diff --git a/.cargo/config.toml b/.cargo/config.toml index a590c044d81..635229119f8 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,3 +3,6 @@ rustflags = "-C link-arg=/STACK:8000000" [target.'cfg(all(target_os = "windows", not(target_env = "msvc")))'] rustflags = "-C link-args=-Wl,--stack,8000000" + +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] diff --git a/.claude/commands/apple-container.md b/.claude/commands/apple-container.md new file mode 100644 index 00000000000..ca6876ad530 --- /dev/null +++ b/.claude/commands/apple-container.md @@ -0,0 +1,46 @@ +--- +allowed-tools: Bash(container *), Bash(cargo *), Read, Grep, Glob +--- + +# Run Tests in Linux Container (Apple `container` CLI) + +Run RustPython tests inside a Linux container using Apple's `container` CLI. +**NEVER use Docker, Podman, or any other container runtime.** Only use the `container` command. + +## Arguments +- `$ARGUMENTS`: Test command to run (e.g., `test_io`, `test_codecs -v`, `test_io -v -m "test_errors"`) + +## Prerequisites + +The `container` CLI is installed via `brew install container`. +The dev image `rustpython-dev` is already built. + +## Steps + +1. **Check if the container is already running** + ```shell + container list 2>/dev/null | grep rustpython-test + ``` + +2. **Start the container if not running** + ```shell + container run -d --name rustpython-test -m 8G -c 4 \ + --mount type=bind,source=/Users/al03219714/Projects/RustPython3,target=/workspace \ + -w /workspace rustpython-dev sleep infinity + ``` + +3. **Run the test inside the container** + ```shell + container exec rustpython-test sh -c "cargo run --release -- -m test $ARGUMENTS" + ``` + +4. **Report results** + - Show test summary (pass/fail counts, expected failures, unexpected successes) + - Highlight any new failures compared to macOS results if available + - Do NOT stop or remove the container after testing (keep it for reuse) + +## Notes +- The workspace is bind-mounted, so local code changes are immediately available +- Use `container exec rustpython-test sh -c "..."` for any command inside the container +- To rebuild after code changes, run: `container exec rustpython-test sh -c "cargo build --release"` +- To stop the container when done: `container rm -f rustpython-test` diff --git a/.claude/commands/investigate-test-failure.md b/.claude/commands/investigate-test-failure.md new file mode 100644 index 00000000000..e9d6b2d2d2c --- /dev/null +++ b/.claude/commands/investigate-test-failure.md @@ -0,0 +1,49 @@ +--- +allowed-tools: Bash(python3:*), Bash(cargo run:*), Read, Grep, Glob, Bash(git add:*), Bash(git commit:*), Bash(cargo fmt:*), Bash(git diff:*), Task +--- + +# Investigate Test Failure + +Investigate why a specific test is failing and determine if it can be fixed or needs an issue. + +## Arguments +- `$ARGUMENTS`: Failed test identifier (e.g., `test_inspect.TestGetSourceBase.test_getsource_reload`) + +## Steps + +1. **Analyze failure cause** + - Read the test code + - Analyze failure message/traceback + - Check related RustPython code + +2. **Verify behavior in CPython** + - Run the test with `python3 -m unittest` to confirm expected behavior + - Document the expected output + +3. **Determine fix feasibility** + - **Simple fix** (import issues, small logic bugs): Fix code → Run `cargo fmt --all` → Pre-commit review → Commit + - **Complex fix** (major unimplemented features): Collect issue info and report to user + + **Pre-commit review process**: + - Run `git diff` to see the changes + - Use Task tool with `general-purpose` subagent to review: + - Compare implementation against cpython/ source code + - Verify the fix aligns with CPython behavior + - Check for any missed edge cases + - Proceed to commit only after review passes + +4. **For complex issues - Collect issue information** + Following `.github/ISSUE_TEMPLATE/report-incompatibility.md` format: + + - **Feature**: Description of missing/broken Python feature + - **Minimal reproduction code**: Smallest code that reproduces the issue + - **CPython behavior**: Result when running with python3 + - **RustPython behavior**: Result when running with cargo run + - **Python Documentation link**: Link to relevant CPython docs + + Report collected information to the user. Issue creation is done only upon user request. + + Example issue creation command: + ``` + gh issue create --template report-incompatibility.md --title "..." --body "..." + ``` diff --git a/.claude/commands/upgrade-pylib-next.md b/.claude/commands/upgrade-pylib-next.md new file mode 100644 index 00000000000..712b79433b3 --- /dev/null +++ b/.claude/commands/upgrade-pylib-next.md @@ -0,0 +1,33 @@ +--- +allowed-tools: Skill(upgrade-pylib), Bash(gh pr list:*) +--- + +# Upgrade Next Python Library + +Find the next Python library module ready for upgrade and run `/upgrade-pylib` for it. + +## Current TODO Status + +!`cargo run --release -- scripts/update_lib todo 2>/dev/null` + +## Open Upgrade PRs + +!`gh pr list --search "Update in:title" --json number,title --template '{{range .}}#{{.number}} {{.title}}{{"\n"}}{{end}}'` + +## Instructions + +From the TODO list above, find modules matching these patterns (in priority order): + +1. `[ ] [no deps]` - Modules with no dependencies (can be upgraded immediately) +2. `[ ] [0/n]` - Modules where all dependencies are already upgraded (e.g., `[0/3]`, `[0/5]`) + +These patterns indicate modules that are ready to upgrade without blocking dependencies. + +**Important**: Skip any modules that already have an open PR in the "Open Upgrade PRs" list above. + +**After identifying a suitable module**, run: +``` +/upgrade-pylib +``` + +If no modules match these criteria, inform the user that all eligible modules have dependencies that need to be upgraded first. diff --git a/.claude/commands/upgrade-pylib.md b/.claude/commands/upgrade-pylib.md new file mode 100644 index 00000000000..d54305d2616 --- /dev/null +++ b/.claude/commands/upgrade-pylib.md @@ -0,0 +1,157 @@ +--- +allowed-tools: Bash(git add:*), Bash(git commit:*), Bash(python3 scripts/update_lib quick:*), Bash(python3 scripts/update_lib auto-mark:*) +--- + +# Upgrade Python Library from CPython + +Upgrade a Python standard library module from CPython to RustPython. + +## Arguments +- `$ARGUMENTS`: Library name to upgrade (e.g., `inspect`, `asyncio`, `json`) + +## Important: Report Tool Issues First + +If during the upgrade process you encounter any of the following issues with `scripts/update_lib`: +- A feature that should be automated but isn't supported +- A bug or unexpected behavior in the tool +- Missing functionality that would make the upgrade easier + +**STOP the upgrade and report the issue first.** Describe: +1. What you were trying to do + - Library name + - The full command executed (e.g. python scripts/update_lib quick cpython/Lib/$ARGUMENTS.py) +2. What went wrong or what's missing +3. Expected vs actual behavior + +This helps improve the tooling for future upgrades. + +## Steps + +1. **Run quick upgrade with update_lib** + - Run: `python3 scripts/update_lib quick $ARGUMENTS` (module name) + - Or: `python3 scripts/update_lib quick cpython/Lib/$ARGUMENTS.py` (library file path) + - Or: `python3 scripts/update_lib quick cpython/Lib/$ARGUMENTS/` (library directory path) + - This will: + - Copy library files (delete existing `Lib/$ARGUMENTS.py` or `Lib/$ARGUMENTS/`, then copy from `cpython/Lib/`) + - Patch test files preserving existing RustPython markers + - Run tests and auto-mark new test failures (not regressions) + - Remove `@unittest.expectedFailure` from tests that now pass + - Create a git commit with the changes + - **Handle warnings**: If you see warnings like `WARNING: TestCFoo does not exist in remote file`, it means the class structure changed and markers couldn't be transferred automatically. These need to be manually restored in step 2 or added in step 3. + +2. **Review git diff and restore RUSTPYTHON-specific changes** + - Run `git diff Lib/test/test_$ARGUMENTS` to review all changes + - **Only restore changes that have explicit `RUSTPYTHON` comments**. Look for: + - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Comments marking RustPython-specific code modifications + - `# TODO: RUSTPYTHON` - Comments marking tests that need work + - Code changes with inline `# ... RUSTPYTHON` comments + - **Do NOT restore other diff changes** - these are likely upstream CPython changes, not RustPython-specific modifications + - When restoring, preserve the original context and formatting + +3. **Investigate test failures with subagent** + - First, get dependent tests using the deps command: + ``` + cargo run --release -- scripts/update_lib deps $ARGUMENTS + ``` + - Look for the line `- [ ] $ARGUMENTS: test_xxx test_yyy ...` to get the direct dependent tests + - Run those tests to collect failures: + ``` + cargo run --release -- -m test test_xxx test_yyy ... 2>&1 | grep -E "^(FAIL|ERROR):" + ``` + - For example, if deps output shows `- [ ] linecache: test_bdb test_inspect test_linecache test_traceback test_zipimport`, run: + ``` + cargo run --release -- -m test test_bdb test_inspect test_linecache test_traceback test_zipimport 2>&1 | grep -E "^(FAIL|ERROR):" + ``` + - For each failure, use the Task tool with `general-purpose` subagent to investigate: + - Subagent should follow the `/investigate-test-failure` skill workflow + - Pass the failed test identifier as the argument (e.g., `test_inspect.TestGetSourceBase.test_getsource_reload`) + - If subagent can fix the issue easily: fix and commit + - If complex issue: subagent collects issue info and reports back (issue creation on user request only) + - Using subagent prevents context pollution in the main conversation + +4. **Mark remaining test failures with auto-mark** + - Run: `python3 scripts/update_lib auto-mark Lib/test/test_$ARGUMENTS.py --mark-failure` + - Or for directory: `python3 scripts/update_lib auto-mark Lib/test/test_$ARGUMENTS/ --mark-failure` + - This will: + - Run tests and mark ALL failing tests with `@unittest.expectedFailure` + - Remove `@unittest.expectedFailure` from tests that now pass + - **Note**: The `--mark-failure` flag marks all failures including regressions. Review the changes before committing. + +5. **Handle panics manually** + - If any tests cause panics/crashes (not just assertion failures), they need `@unittest.skip` instead: + ```python + @unittest.skip("TODO: RUSTPYTHON; panics with 'index out of bounds'") + def test_crashes(self): + ... + ``` + - auto-mark cannot detect panics automatically - check the test output for crash messages + +6. **Handle class-specific failures** + - If a test fails only in the C implementation (TestCFoo) but passes in the Python implementation (TestPyFoo), or vice versa, move the marker to the specific subclass: + ```python + # Base class - no marker here + class TestFoo: + def test_something(self): + ... + + class TestPyFoo(TestFoo, PyTest): pass + + class TestCFoo(TestFoo, CTest): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_something(self): + return super().test_something() + ``` + +7. **Commit the test fixes** + - Run: `git add -u && git commit -m "Mark failing tests"` + - This creates a separate commit for the test markers added in steps 2-6 + +## Example Usage +``` +# Using module names (recommended) +/upgrade-pylib inspect +/upgrade-pylib json +/upgrade-pylib asyncio + +# Using library paths (alternative) +/upgrade-pylib cpython/Lib/inspect.py +/upgrade-pylib cpython/Lib/json/ +``` + +## Example: Restoring RUSTPYTHON changes + +When git diff shows removed RUSTPYTHON-specific code like: +```diff +-# XXX RUSTPYTHON: we don't import _json as fresh since... +-cjson = import_helper.import_fresh_module('json') #, fresh=['_json']) ++cjson = import_helper.import_fresh_module('json', fresh=['_json']) +``` + +You should restore the RustPython version: +```python +# XXX RUSTPYTHON: we don't import _json as fresh since... +cjson = import_helper.import_fresh_module('json') #, fresh=['_json']) +``` + +## Notes +- The cpython/ directory should contain the CPython source that we're syncing from +- `scripts/update_lib` package handles patching and auto-marking: + - `quick` - Combined patch + auto-mark (recommended) + - `migrate` - Only migrate (patch), no test running + - `auto-mark` - Only run tests and mark failures + - `copy-lib` - Copy library files (not tests) +- The patching: + - Transfers `@unittest.expectedFailure` and `@unittest.skip` decorators with `TODO: RUSTPYTHON` markers + - Adds `import unittest # XXX: RUSTPYTHON` if needed for the decorators + - **Limitation**: If a class was restructured (e.g., method overrides removed), update_lib will warn and skip those markers +- The smart auto-mark: + - Marks NEW test failures automatically (tests that didn't exist before) + - Does NOT mark regressions (existing tests that now fail) - these are warnings + - Removes `@unittest.expectedFailure` from tests that now pass +- The script does NOT preserve all RustPython-specific changes - you must review `git diff` and restore them +- Common RustPython markers to look for: + - `# XXX: RUSTPYTHON` or `# XXX RUSTPYTHON` - Inline comments for code modifications + - `# TODO: RUSTPYTHON` - Test skip/failure markers + - Any code with `RUSTPYTHON` in comments that was removed in the diff +- **Important**: Not all changes in the git diff need to be restored. Only restore changes that have explicit `RUSTPYTHON` comments. Other changes are upstream CPython updates. diff --git a/.claude/scripts/setup-env.sh b/.claude/scripts/setup-env.sh new file mode 100755 index 00000000000..f5d28a3f14c --- /dev/null +++ b/.claude/scripts/setup-env.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Claude Code web session startup script +# Sets up the development environment for RustPython + +set -e + +cd /home/user/RustPython + +echo "=== RustPython dev environment setup ===" + +# 1. Ensure python3 points to 3.13+ (needed for scripts/update_lib) +# /usr/local/bin takes precedence over /usr/bin in PATH, +# so we update the symlink there directly. +CURRENT_PY=$(python3 --version 2>&1 | grep -oP '\d+\.\d+') +if [ "$(printf '%s\n' "3.13" "$CURRENT_PY" | sort -V | head -1)" != "3.13" ]; then + echo "Upgrading python3 default to 3.13..." + # Find best available Python >= 3.13 + TARGET="" + for ver in python3.14 python3.13; do + if command -v "$ver" &>/dev/null; then + TARGET=$(command -v "$ver") + break + fi + done + if [ -n "$TARGET" ]; then + # Override /usr/local/bin/python3 if it exists and is outdated + if [ -e /usr/local/bin/python3 ]; then + sudo ln -sf "$TARGET" /usr/local/bin/python3 + fi + # Also set /usr/bin via update-alternatives + sudo update-alternatives --install /usr/bin/python3 python3 "$TARGET" 3 2>/dev/null || true + sudo update-alternatives --set python3 "$TARGET" 2>/dev/null || true + echo "python3 now: $(python3 --version)" + else + echo "WARNING: No Python 3.13+ found. scripts/update_lib may not work." + fi +else + echo "python3 already >= 3.13: $(python3 --version)" +fi + +# 2. Clone CPython source if not present (needed for scripts/update_lib) +if [ ! -d "cpython" ]; then + echo "Cloning CPython v3.14.3 (shallow)..." + git clone --depth 1 --branch v3.14.3 https://github.com/python/cpython.git cpython + echo "CPython source ready." +else + echo "CPython source already present." +fi + +echo "=== Setup complete ===" diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000000..22f0a9a8a01 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "bash .claude/scripts/setup-env.sh" + } + ] + } + ] + } +} diff --git a/.coderabbit.yml b/.coderabbit.yml new file mode 100644 index 00000000000..6a96844e23d --- /dev/null +++ b/.coderabbit.yml @@ -0,0 +1,3 @@ +reviews: + path_filters: + - "!Lib/**" diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt new file mode 100644 index 00000000000..a9fbc8f4318 --- /dev/null +++ b/.cspell.dict/cpython.txt @@ -0,0 +1,233 @@ +ADDOP +aftersign +argdefs +argtypes +asdl +asname +attro +augassign +badcert +badsyntax +baseinfo +basetype +binop +bltin +boolop +BUFMAX +BUILDSTDLIB +bxor +byteswap +cached_tsver +cadata +cafile +calldepth +callinfo +callproc +capath +carg +cellarg +cellvar +cellvars +ceval +cfield +CLASSDEREF +classdict +cmpop +codedepth +CODEUNIT +CONIN +CONOUT +constevaluator +consti +CONVFUNC +convparam +copyslot +cpucount +datastack +defaultdict +denom +deopt +dictbytype +DICTFLAG +dictoffset +distpoint +dynload +elts +eofs +evalloop +excepthandler +exceptiontable +fastlocal +fastlocals +fblock +fblocks +fdescr +ffi_argtypes +fielddesc +fieldlist +fileutils +finalbody +finalizers +firsttraceable +flowgraph +formatfloat +freelist +freevar +freevars +fromlist +getdict +getfunc +getiter +getsets +getslice +globalgetvar +HASARRAY +HASBITFIELD +HASPOINTER +HASSTRUCT +HASUNION +heaptype +hexdigit +HIGHRES +IFUNC +IMMUTABLETYPE +INCREF +inlinedepth +inplace +ismine +ISPOINTER +iteminfo +Itertool +keeped +kwnames +kwonlyarg +kwonlyargs +lasti +libffi +linearise +lineiterator +linetable +loadfast +localsplus +Lshift +lsprof +MAXBLOCKS +maxdepth +metavars +miscompiles +mult +multibytecodec +nameobj +nameop +ncells +nconsts +newargs +newfree +NEWLOCALS +newsemlockobject +nfrees +nkwargs +nkwelts +nlocalsplus +Nondescriptor +noninteger +nops +noraise +nseen +NSIGNALS +numer +opname +opnames +orelse +outparam +outparm +paramfunc +parg +pathconfig +patma +peepholer +phcount +platstdlib +posonlyarg +posonlyargs +prec +preinitialized +pybuilddir +pycore +pyinner +pydecimal +Pyfunc +pylifecycle +pymain +pyrepl +PYTHONTRACEMALLOC +PYTHONUTF8 +pythonw +PYTHREAD_NAME +releasebuffer +repr +resinfo +Rshift +SA_ONSTACK +saveall +scls +setdict +setfunc +setprofileallthreads +SETREF +setresult +setslice +settraceallthreads +SLOTDEFINED +SMALLBUF +SOABI +SSLEOF +stackdepth +stackref +staticbase +stginfo +storefast +stringlib +structseq +subkwargs +subparams +subscr +sval +swappedbytes +sysdict +templatelib +testconsole +threadstate +ticketer +tmptype +tok_oldval +tstate +tvars +typeobject +typeparam +Typeparam +typeparams +typeslots +unaryop +uncollectable +Unhandle +unparse +unparser +untracking +VARKEYWORDS +varkwarg +venvlauncher +venvlaunchert +venvw +venvwlauncher +venvwlaunchert +wbits +weakreflist +weakrefobject +webpki +winconsoleio +withitem +withs +worklist +xstat +XXPRIME diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt new file mode 100644 index 00000000000..2ce5d246d72 --- /dev/null +++ b/.cspell.dict/python-more.txt @@ -0,0 +1,301 @@ +abiflags +abstractmethods +addcompare +aenter +aexit +aiter +altzone +anext +anextawaitable +annotationlib +appendleft +argcount +arrayiterator +arraytype +asend +asyncgen +athrow +backslashreplace +baserepl +basicsize +bdfl +bigcharset +bignum +bivariant +breakpointhook +cformat +chunksize +classcell +classmethods +closefd +closesocket +codepoint +codepoints +codesize +contextvar +cpython +cratio +ctype +ctypes +dealloc +debugbuild +decompressor +defaultaction +descr +dictcomp +dictitems +dictkeys +dictview +digestmod +dllhandle +docstring +docstrings +dunder +endianness +endpos +eventmask +excepthook +exceptiongroup +exitfuncs +extendleft +fastlocals +fdel +fedcba +fget +fileencoding +fillchar +fillvalue +finallyhandler +firstiter +firstlineno +fnctl +frombytes +fromhex +fromunicode +frozensets +fset +fspath +fstring +fstrings +ftruncate +genexpr +genexpressions +getargs +getattro +getcodesize +getdefaultencoding +getfilesystemencodeerrors +getfilesystemencoding +getformat +getframe +getframemodulename +getnewargs +getopt +getpip +getrandom +getrecursionlimit +getrefcount +getsizeof +getswitchinterval +getweakref +getweakrefcount +getweakrefs +getweakrefs +getwindowsversion +gmtoff +groupdict +groupindex +hamt +hostnames +idfunc +idiv +idxs +impls +indexgroup +infj +inittab +Inittab +instancecheck +instanceof +interpchannels +interpqueues +irepeat +isabstractmethod +isbytes +iscased +isfinal +istext +itemiterator +itemsize +iternext +keepends +keyfunc +keyiterator +kwarg +kwargs +kwdefaults +kwonlyargcount +lastgroup +lastindex +linearization +linearize +listcomp +longrange +lvalue +mappingproxy +markupbase +maskpri +maxdigits +MAXGROUPS +MAXREPEAT +maxsplit +maxunicode +memoryview +memoryviewiterator +metaclass +metaclasses +metatype +mformat +mro +mros +multiarch +mymodule +namereplace +nanj +nbytes +ncallbacks +ndigits +ndim +needsfree +nldecoder +nlocals +NOARGS +nonbytes +Nonprintable +onceregistry +origname +ospath +pendingcr +phello +platlibdir +popleft +posixsubprocess +posonly +posonlyargcount +prepending +profilefunc +pycache +pycodecs +pycs +pydatetime +pyexpat +pyio +pymain +PYTHONAPI +PYTHONBREAKPOINT +PYTHONDEBUG +PYTHONDONTWRITEBYTECODE +PYTHONFAULTHANDLER +PYTHONHASHSEED +PYTHONHOME +PYTHONINSPECT +PYTHONINTMAXSTRDIGITS +PYTHONIOENCODING +PYTHONNODEBUGRANGES +PYTHONNOUSERSITE +PYTHONOPTIMIZE +PYTHONPATH +PYTHONPATH +PYTHONSAFEPATH +PYTHONUNBUFFERED +PYTHONVERBOSE +PYTHONWARNDEFAULTENCODING +PYTHONWARNINGS +pytraverse +PYVENV +qualname +quotetabs +radd +rdiv +rdivmod +readall +readbuffer +reconstructor +refcnt +releaselevel +reraised +reverseitemiterator +reverseiterator +reversekeyiterator +reversevalueiterator +rfloordiv +rlshift +rmod +rpow +rrshift +rsub +rtruediv +rvalue +scproxy +seennl +setattro +setcomp +setprofileallthreads +setrecursionlimit +setswitchinterval +settraceallthreads +showwarnmsg +signum +sitebuiltins +slotnames +STACKLESS +stacklevel +stacksize +startpos +subclassable +subclasscheck +subclasshook +subclassing +suboffset +suboffsets +SUBPATTERN +subpatterns +sumprod +surrogateescape +surrogatepass +sysconf +sysconfigdata +sysdict +sysvars +teedata +thisclass +titlecased +tkapp +tobytes +tolist +toreadonly +TPFLAGS +tracefunc +unimportable +unionable +unraisablehook +unsliceable +urandom +valueiterator +vararg +varargs +varnames +warningregistry +warnmsg +warnoptions +warnopts +weaklist +weakproxy +weakrefs +weakrefset +winver +withdata +xmlcharrefreplace +xoptions +xopts +yieldfrom diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt new file mode 100644 index 00000000000..c4457723c6c --- /dev/null +++ b/.cspell.dict/rust-more.txt @@ -0,0 +1,94 @@ +ahash +arrayvec +bidi +biguint +bindgen +bitand +bitflags +bitflagset +bitor +bitvec +bitxor +bstr +byteorder +byteset +caseless +chrono +consts +cranelift +cstring +datelike +deserializer +deserializers +fdiv +flamescope +flate2 +fract +getres +hasher +hexf +hexversion +idents +illumos +ilog +indexmap +insta +keccak +lalrpop +lexopt +libc +libcall +libloading +libz +longlong +Manually +maplit +memmap +memmem +metas +modpow +msvc +muldiv +nanos +nonoverlapping +objclass +peekable +pemfile +powc +powf +powi +prepended +punct +replacen +retag +rmatch +rposition +rsplitn +rustc +rustfmt +rustls +rustyline +seedable +seekfrom +siphash +siphasher +splitn +subsec +thiserror +timelike +timsort +trai +ulonglong +unic +unistd +unraw +unsync +wasip1 +wasip2 +wasmbind +wasmer +wasmtime +widestring +winapi +winresource +winsock diff --git a/.cspell.json b/.cspell.json index ad90153f56a..07fe948c5bf 100644 --- a/.cspell.json +++ b/.cspell.json @@ -1,209 +1,92 @@ // See: https://github.com/streetsidesoftware/cspell/tree/master/packages/cspell { "version": "0.2", + "import": [ + "@cspell/dict-en_us/cspell-ext.json", + // "@cspell/dict-cpp/cspell-ext.json", + "@cspell/dict-python/cspell-ext.json", + "@cspell/dict-rust/cspell-ext.json", + "@cspell/dict-win32/cspell-ext.json", + "@cspell/dict-shell/cspell-ext.json", + ], // language - current active spelling language "language": "en", // dictionaries - list of the names of the dictionaries to use "dictionaries": [ + "cpython", // Sometimes keeping same terms with cpython is easy + "python-more", // Python API terms not listed in python + "rust-more", // Rust API terms not listed in rust "en_US", "softwareTerms", "c", "cpp", "python", - "python-custom", "rust", - "unix", - "posix", - "winapi" + "shell", + "win32" ], // dictionaryDefinitions - this list defines any custom dictionaries to use - "dictionaryDefinitions": [], + "dictionaryDefinitions": [ + { + "name": "cpython", + "path": "./.cspell.dict/cpython.txt" + }, + { + "name": "python-more", + "path": "./.cspell.dict/python-more.txt" + }, + { + "name": "rust-more", + "path": "./.cspell.dict/rust-more.txt" + } + ], "ignorePaths": [ "**/__pycache__/**", + "target/**", "Lib/**" ], // words - list of words to be always considered correct "words": [ - // Rust - "ahash", - "bidi", - "biguint", - "bindgen", - "bitflags", - "bstr", - "byteorder", - "chrono", - "consts", - "cstring", - "flate2", - "fract", - "hasher", - "idents", - "indexmap", - "insta", - "keccak", - "lalrpop", - "libc", - "libz", - "longlong", - "Manually", - "maplit", - "memmap", - "metas", - "modpow", - "nanos", - "peekable", - "powc", - "powf", - "prepended", - "punct", - "replacen", - "rsplitn", - "rustc", - "rustfmt", - "seekfrom", - "splitn", - "subsec", - "timsort", - "trai", - "ulonglong", - "unic", - "unistd", - "winapi", - "winsock", - // Python - "abstractmethods", - "aiter", - "anext", - "arrayiterator", - "arraytype", - "asend", - "athrow", - "basicsize", - "cformat", - "classcell", - "closesocket", - "codepoint", - "codepoints", - "cpython", - "decompressor", - "defaultaction", - "descr", - "dictcomp", - "dictitems", - "dictkeys", - "dictview", - "docstring", - "docstrings", - "dunder", - "eventmask", - "fdel", - "fget", - "fileencoding", - "fillchar", - "finallyhandler", - "frombytes", - "fromhex", - "fromunicode", - "fset", - "fspath", - "fstring", - "fstrings", - "genexpr", - "getattro", - "getformat", - "getnewargs", - "getweakrefcount", - "getweakrefs", - "hostnames", - "idiv", - "impls", - "infj", - "instancecheck", - "instanceof", - "isabstractmethod", - "itemiterator", - "itemsize", - "iternext", - "keyiterator", - "kwarg", - "kwargs", - "linearization", - "linearize", - "listcomp", - "mappingproxy", - "maxsplit", - "memoryview", - "memoryviewiterator", - "metaclass", - "metaclasses", - "metatype", - "mro", - "mros", - "nanj", - "ndigits", - "ndim", - "nonbytes", - "origname", - "posixsubprocess", - "pyexpat", - "PYTHONDEBUG", - "PYTHONHOME", - "PYTHONINSPECT", - "PYTHONOPTIMIZE", - "PYTHONPATH", - "PYTHONPATH", - "PYTHONVERBOSE", - "PYTHONWARNINGS", - "qualname", - "radd", - "rdiv", - "rdivmod", - "reconstructor", - "reversevalueiterator", - "rfloordiv", - "rlshift", - "rmod", - "rpow", - "rrshift", - "rsub", - "rtruediv", - "scproxy", - "setattro", - "setcomp", - "showwarnmsg", - "warnmsg", - "stacklevel", - "subclasscheck", - "subclasshook", - "unionable", - "unraisablehook", - "valueiterator", - "vararg", - "varargs", - "varnames", - "warningregistry", - "warnopts", - "weakproxy", - "xopts", - // RustPython + "RUSTPYTHONPATH", + // RustPython terms + "aiterable", + "alnum", "baseclass", + "boxvec", "Bytecode", "cfgs", "codegen", + "coro", "dedentations", "dedents", "deduped", + "deoptimize", + "downcastable", "downcasted", "dumpable", + "emscripten", + "excs", + "finalizer", + "finalizers", "GetSet", + "groupref", "internable", + "interps", + "jitted", + "jitting", + "kwonly", + "lossily", "makeunicodedata", + "mcache", + "microbenchmark", + "microbenchmarks", "miri", "notrace", - "pyarg", + "oparg", + "openat", "pyarg", "pyargs", + "pyast", "PyAttr", "pyc", "PyClass", @@ -212,6 +95,8 @@ "PyFunction", "pygetset", "pyimpl", + "pylib", + "pymath", "pymember", "PyMethod", "PyModule", @@ -224,64 +109,49 @@ "PyResult", "pyslot", "PyStaticMethod", + "pystone", "pystr", "pystruct", "pystructseq", "pytrace", + "pytype", "reducelib", "richcompare", + "rustix", "RustPython", + "significand", "struc", + "summands", // plural of summand + "sysmodule", "tracebacks", "typealiases", - "Unconstructible", + "typevartuples", + "uncollectable", "unhashable", "uninit", "unraisable", + "unresizable", + "varint", "wasi", + "weaked", "zelf", - // cpython - "argtypes", - "asdl", - "asname", - "augassign", - "badsyntax", - "basetype", - "boolop", - "bxor", - "cellarg", - "cellvar", - "cellvars", - "cmpop", - "dictoffset", - "elts", - "excepthandler", - "finalbody", - "freevar", - "freevars", - "fromlist", - "heaptype", - "IMMUTABLETYPE", - "kwonlyarg", - "kwonlyargs", - "linearise", - "maxdepth", - "mult", - "nkwargs", - "orelse", - "patma", - "posonlyarg", - "posonlyargs", - "prec", - "stackdepth", - "unaryop", - "unparse", - "unparser", - "VARKEYWORDS", - "varkwarg", - "wbits", - "withitem", - "withs" + // unix + "posixshmem", + "shm", + "CLOEXEC", + "codeset", + "endgrent", + "gethrvtime", + "getrusage", + "nanosleep", + "sigaction", + "sighandler", + "WRLCK", + // win32 + "birthtime", + "IFEXEC", + // "stat" + "FIRMLINK", ], // flagWords - list of words to be always considered incorrect "flagWords": [ diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000000..cdd54a47d5b --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,6 @@ +FROM rust:bullseye + +# Install clang +RUN apt-get update \ + && apt-get install -y clang \ + && rm -rf /var/lib/apt/lists/* diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d8641749b59..8838cf6a960 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,25 @@ { - "image": "mcr.microsoft.com/devcontainers/universal:2", - "features": { - "ghcr.io/devcontainers/features/rust:1": {} - } + "name": "Rust", + "build": { + "dockerfile": "Dockerfile" + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + "customizations": { + "vscode": { + "settings": { + "lldb.executable": "/usr/bin/lldb", + // VS Code don't watch files under ./target + "files.watcherExclude": { + "**/target/**": true + }, + "extensions": [ + "rust-lang.rust-analyzer", + "tamasfe.even-better-toml", + "vadimcn.vscode-lldb", + "mutantdino.resourcemonitor" + ] + } + } + }, + "remoteUser": "vscode" } diff --git a/.gemini/config.yaml b/.gemini/config.yaml new file mode 100644 index 00000000000..76afe53388c --- /dev/null +++ b/.gemini/config.yaml @@ -0,0 +1,2 @@ +ignore_patterns: + - "Lib/**" diff --git a/.gitattributes b/.gitattributes index aa993ad110f..d076a34f977 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,70 @@ Lib/** linguist-vendored -Cargo.lock linguist-generated -merge +Cargo.lock linguist-generated *.snap linguist-generated -merge vm/src/stdlib/ast/gen.rs linguist-generated -merge Lib/*.py text working-tree-encoding=UTF-8 eol=LF **/*.rs text working-tree-encoding=UTF-8 eol=LF +crates/rustpython_doc_db/src/*.inc.rs linguist-generated=true + +# Binary data types +*.aif binary +*.aifc binary +*.aiff binary +*.au binary +*.bmp binary +*.exe binary +*.icns binary +*.gif binary +*.ico binary +*.jpg binary +*.pck binary +*.pdf binary +*.png binary +*.psd binary +*.tar binary +*.wav binary +*.whl binary +*.zip binary + +# Text files that should not be subject to eol conversion +[attr]noeol -text + +Lib/test/cjkencodings/* noeol +Lib/test/tokenizedata/coding20731.py noeol +Lib/test/decimaltestdata/*.decTest noeol +Lib/test/test_email/data/*.txt noeol +Lib/test/xmltestdata/* noeol + +# Shell scripts should have LF even on Windows because of Cygwin +Lib/venv/scripts/common/activate text eol=lf +Lib/venv/scripts/posix/* text eol=lf + +# CRLF files +[attr]dos text eol=crlf + +# Language aware diff headers +# https://tekin.co.uk/2020/10/better-git-diff-output-for-ruby-python-elixir-and-more +# https://gist.github.com/tekin/12500956bd56784728e490d8cef9cb81 +*.css diff=css +*.html diff=html +*.py diff=python +*.md diff=markdown + +# Generated files +# https://github.com/github/linguist/blob/master/docs/overrides.md +# +# To always hide generated files in local diffs, mark them as binary: +# $ git config diff.generated.binary true +# +[attr]generated linguist-generated=true diff=generated + +Lib/_opcode_metadata.py generated +Lib/keyword.py generated +Lib/idlelib/help.html generated +Lib/test/certdata/*.pem generated +Lib/test/certdata/*.0 generated +Lib/test/levenshtein_examples.json generated +Lib/test/test_stable_abi_ctypes.py generated +Lib/token.py generated + +.github/workflows/*.lock.yml linguist-generated=true merge=ours \ No newline at end of file diff --git a/.github/actions/install-linux-deps/action.yml b/.github/actions/install-linux-deps/action.yml new file mode 100644 index 00000000000..7900060fb29 --- /dev/null +++ b/.github/actions/install-linux-deps/action.yml @@ -0,0 +1,49 @@ +# This action installs a few dependencies necessary to build RustPython on Linux. +# It can be configured depending on which libraries are needed: +# +# ``` +# - uses: ./.github/actions/install-linux-deps +# with: +# gcc-multilib: true +# musl-tools: false +# ``` +# +# See the `inputs` section for all options and their defaults. Note that you must checkout the +# repository before you can use this action. +# +# This action will only install dependencies when the current operating system is Linux. It will do +# nothing on any other OS (macOS, Windows). + +name: Install Linux dependencies +description: Installs the dependencies necessary to build RustPython on Linux. +inputs: + gcc-multilib: + description: Install gcc-multilib (gcc-multilib) + required: false + default: "false" + musl-tools: + description: Install musl-tools (musl-tools) + required: false + default: "false" + gcc-aarch64-linux-gnu: + description: Install gcc-aarch64-linux-gnu (gcc-aarch64-linux-gnu) + required: false + default: "false" + clang: + description: Install clang (clang) + required: false + default: "false" +runs: + using: composite + steps: + - name: Install Linux dependencies + shell: bash + if: ${{ runner.os == 'Linux' }} + run: > + sudo apt-get update + + sudo apt-get install --no-install-recommends + ${{ fromJSON(inputs.gcc-multilib) && 'gcc-multilib' || '' }} + ${{ fromJSON(inputs.musl-tools) && 'musl-tools' || '' }} + ${{ fromJSON(inputs.clang) && 'clang' || '' }} + ${{ fromJSON(inputs.gcc-aarch64-linux-gnu) && 'gcc-aarch64-linux-gnu linux-libc-dev-arm64-cross libc6-dev-arm64-cross' || '' }} diff --git a/.github/actions/install-macos-deps/action.yml b/.github/actions/install-macos-deps/action.yml new file mode 100644 index 00000000000..46abef197a4 --- /dev/null +++ b/.github/actions/install-macos-deps/action.yml @@ -0,0 +1,47 @@ +# This action installs a few dependencies necessary to build RustPython on macOS. By default it installs +# autoconf, automake and libtool, but can be configured depending on which libraries are needed: +# +# ``` +# - uses: ./.github/actions/install-macos-deps +# with: +# openssl: true +# libtool: false +# ``` +# +# See the `inputs` section for all options and their defaults. Note that you must checkout the +# repository before you can use this action. +# +# This action will only install dependencies when the current operating system is macOS. It will do +# nothing on any other OS (Linux, Windows). + +name: Install macOS dependencies +description: Installs the dependencies necessary to build RustPython on macOS. +inputs: + autoconf: + description: Install autoconf (autoconf) + required: false + default: "true" + automake: + description: Install automake (automake) + required: false + default: "true" + libtool: + description: Install libtool (libtool) + required: false + default: "true" + openssl: + description: Install openssl (openssl@3) + required: false + default: "false" +runs: + using: composite + steps: + - name: Install macOS dependencies + shell: bash + if: ${{ runner.os == 'macOS' }} + run: > + brew install + ${{ fromJSON(inputs.autoconf) && 'autoconf' || '' }} + ${{ fromJSON(inputs.automake) && 'automake' || '' }} + ${{ fromJSON(inputs.libtool) && 'libtool' || '' }} + ${{ fromJSON(inputs.openssl) && 'openssl@3' || '' }} diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json new file mode 100644 index 00000000000..ad986cbd051 --- /dev/null +++ b/.github/aw/actions-lock.json @@ -0,0 +1,14 @@ +{ + "entries": { + "actions/github-script@v8": { + "repo": "actions/github-script", + "version": "v8", + "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" + }, + "github/gh-aw/actions/setup@v0.43.22": { + "repo": "github/gh-aw/actions/setup", + "version": "v0.43.22", + "sha": "fe858c3e14589bf396594a0b106e634d9065823e" + } + } +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..20e088059cc --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,149 @@ +# cspell:ignore manyhow tinyvec zeroize +version: 2 +updates: + - package-ecosystem: cargo + directory: / + schedule: + interval: weekly + groups: + criterion: + patterns: + - "criterion*" + crypto: + patterns: + - "digest" + - "md-5" + - "sha-1" + - "sha2" + - "sha3" + - "blake2" + - "hmac" + - "pbkdf2" + futures: + patterns: + - "futures*" + get-size2: + patterns: + - "get-size*2" + iana-time-zone: + patterns: + - "iana-time-zone*" + jiff: + patterns: + - "jiff*" + lexical: + patterns: + - "lexical*" + libffi: + patterns: + - "libffi*" + malachite: + patterns: + - "malachite*" + manyhow: + patterns: + - "manyhow*" + num: + patterns: + - "num-bigint" + - "num-complex" + - "num-integer" + - "num-iter" + - "num-rational" + - "num-traits" + num_enum: + patterns: + - "num_enum*" + openssl: + patterns: + - "openssl*" + parking_lot: + patterns: + - "parking_lot*" + phf: + patterns: + - "phf*" + plotters: + patterns: + - "plotters*" + portable-atomic: + patterns: + - "portable-atomic*" + pyo3: + patterns: + - "pyo3*" + quote-use: + patterns: + - "quote-use*" + random: + patterns: + - "getrandom" + - "mt19937" + - "rand*" + rayon: + patterns: + - "rayon*" + regex: + patterns: + - "regex*" + result-like: + patterns: + - "result-like*" + security-framework: + patterns: + - "security-framework*" + serde: + patterns: + - "serde" + - "serde_core" + - "serde_derive" + system-configuration: + patterns: + - "system-configuration*" + thiserror: + patterns: + - "thiserror*" + time: + patterns: + - "time*" + tinyvec: + patterns: + - "tinyvec*" + tls_codec: + patterns: + - "tls_codec*" + toml: + patterns: + - "toml*" + wasm-bindgen: + patterns: + - "wasm-bindgen*" + wasmtime: + patterns: + - "cranelift*" + - "wasmtime*" + webpki-root: + patterns: + - "webpki-root*" + windows: + patterns: + - "windows*" + zerocopy: + patterns: + - "zerocopy*" + zeroize: + patterns: + - "zeroize*" + ignore: + # TODO: Remove when we use ruff from crates.io + # for some reason dependabot only updates the Cargo.lock file when dealing + # with git dependencies. i.e. not updating the version in Cargo.toml + - dependency-name: "ruff_*" + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + - package-ecosystem: npm + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4f31ba74c17..ddfa35ab009 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,6 +4,7 @@ on: pull_request: types: [unlabeled, opened, synchronize, reopened] merge_group: + workflow_dispatch: name: CI @@ -15,26 +16,27 @@ concurrency: cancel-in-progress: true env: - CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls,host_env + CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,host_env + # Crates excluded from workspace builds: + # - rustpython_wasm: requires wasm target + # - rustpython-compiler-source: deprecated + # - rustpython-venvlauncher: Windows-only + WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher # Skip additional tests on Windows. They are checked on Linux and MacOS. + # test_glob: many failing tests + # test_pathlib: panic by surrogate chars + # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') + # test_venv: couple of failing tests WINDOWS_SKIPS: >- - test_glob - test_importlib - test_io - test_os + test_rlcompleter test_pathlib test_posixpath - test_shutil test_venv - # configparser: https://github.com/RustPython/RustPython/issues/4995#issuecomment-1582397417 - # socketserver: seems related to configparser crash. - MACOS_SKIPS: >- - test_configparser - test_socketserver # PLATFORM_INDEPENDENT_TESTS are tests that do not depend on the underlying OS. They are currently # only run on Linux to speed up the CI. PLATFORM_INDEPENDENT_TESTS: >- - test_argparse + test__colorize test_array test_asyncgen test_binop @@ -59,7 +61,6 @@ env: test_dis test_enumerate test_exception_variations - test_exceptions test_float test_format test_fractions @@ -83,6 +84,7 @@ env: test_math test_operator test_ordered_dict + test_pep646_syntax test_pow test_raise test_richcmp @@ -97,15 +99,22 @@ env: test_subclassinit test_super test_syntax + test_tstring test_tuple test_types test_unary - test_unicode test_unpack + test_unpack_ex test_weakref test_yield_from + ENV_POLLUTING_TESTS_COMMON: >- + ENV_POLLUTING_TESTS_LINUX: >- + ENV_POLLUTING_TESTS_MACOS: >- + ENV_POLLUTING_TESTS_WINDOWS: >- # Python version targeted by the CI. - PYTHON_VERSION: "3.12.0" + PYTHON_VERSION: "3.14.3" + X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD + X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include jobs: rust_tests: @@ -113,189 +122,299 @@ jobs: env: RUST_BACKTRACE: full name: Run rust tests - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + os: [macos-latest, ubuntu-latest, windows-2025] fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable with: components: clippy - - name: Set up the Windows environment - shell: bash - run: | - choco install llvm openssl --no-progress - if [[ -d "C:\Program Files\OpenSSL-Win64" ]]; then - echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >> $GITHUB_ENV - else - echo "OPENSSL_DIR=C:\Program Files\OpenSSL" >> $GITHUB_ENV - fi - if: runner.os == 'Windows' - - name: Set up the Mac environment - run: brew install autoconf automake libtool - if: runner.os == 'macOS' - - uses: Swatinem/rust-cache@v2 + - name: Install macOS dependencies + uses: ./.github/actions/install-macos-deps + - name: run clippy - run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --exclude rustpython_wasm -- -Dwarnings + run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings - name: run rust tests - run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} - if: runner.os != 'macOS' - # temp skip ssl linking for Mac to avoid CI failure - - name: run rust tests (MacOS no ssl) - run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit - if: runner.os == 'macOS' + run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --verbose --features threading ${{ env.CARGO_ARGS }} - name: check compilation without threading run: cargo check ${{ env.CARGO_ARGS }} - - name: prepare AppleSilicon build - uses: dtolnay/rust-toolchain@stable - with: - target: aarch64-apple-darwin - if: runner.os == 'macOS' - - name: Check compilation for Apple Silicon - run: cargo check --target aarch64-apple-darwin - if: runner.os == 'macOS' - - name: prepare iOS build - uses: dtolnay/rust-toolchain@stable - with: - target: aarch64-apple-ios - if: runner.os == 'macOS' - - name: Check compilation for iOS - run: cargo check --target aarch64-apple-ios - if: runner.os == 'macOS' - - exotic_targets: - if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} - name: Ensure compilation on various targets - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@stable - with: - target: i686-unknown-linux-gnu - - - name: Install gcc-multilib and musl-tools - run: sudo apt-get update && sudo apt-get install gcc-multilib musl-tools - - name: Check compilation for x86 32bit - run: cargo check --target i686-unknown-linux-gnu + - name: check compilation without host_env (sandbox mode) + run: | + cargo check -p rustpython-vm --no-default-features --features compiler + cargo check -p rustpython-stdlib --no-default-features --features compiler + cargo build --no-default-features --features stdlib,importlib,stdio,encodings,freeze-stdlib + if: runner.os == 'Linux' - - uses: dtolnay/rust-toolchain@stable - with: - target: aarch64-linux-android + - name: sandbox smoke test + run: | + target/debug/rustpython extra_tests/snippets/sandbox_smoke.py + target/debug/rustpython extra_tests/snippets/stdlib_re.py + if: runner.os == 'Linux' - - name: Check compilation for android - run: cargo check --target aarch64-linux-android + - name: Test openssl build + run: cargo build --no-default-features --features ssl-openssl + if: runner.os == 'Linux' - - uses: dtolnay/rust-toolchain@stable - with: - target: aarch64-unknown-linux-gnu + # - name: Install tk-dev for tkinter build + # run: sudo apt-get update && sudo apt-get install -y tk-dev + # if: runner.os == 'Linux' - - name: Install gcc-aarch64-linux-gnu - run: sudo apt install gcc-aarch64-linux-gnu - - name: Check compilation for aarch64 linux gnu - run: cargo check --target aarch64-unknown-linux-gnu + # - name: Test tkinter build + # run: cargo build --features tkinter + # if: runner.os == 'Linux' - - uses: dtolnay/rust-toolchain@stable - with: - target: i686-unknown-linux-musl + - name: Test example projects + run: | + cargo run --manifest-path example_projects/barebone/Cargo.toml + cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml + if: runner.os == 'Linux' - - name: Check compilation for musl - run: cargo check --target i686-unknown-linux-musl + - name: run update_lib tests + run: cargo run -- -m unittest discover -s scripts/update_lib/tests -v + env: + PYTHONPATH: scripts + if: runner.os == 'Linux' - - uses: dtolnay/rust-toolchain@stable + cargo_check: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} + name: Ensure compilation on various targets + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + targets: + - aarch64-linux-android + - i686-unknown-linux-gnu + - i686-unknown-linux-musl + - wasm32-wasip2 + - x86_64-unknown-freebsd + dependencies: + gcc-multilib: true + musl-tools: true + - os: ubuntu-latest + targets: + - aarch64-unknown-linux-gnu + dependencies: + gcc-aarch64-linux-gnu: true # conflict with `gcc-multilib` + - os: macos-latest + targets: + - aarch64-apple-ios + - x86_64-apple-darwin + fail-fast: false + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - target: x86_64-unknown-freebsd - - - name: Check compilation for freebsd - run: cargo check --target x86_64-unknown-freebsd + persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 with: - target: wasm32-unknown-unknown + prefix-key: v0-rust-${{ join(matrix.targets, '-') }} - - name: Check compilation for wasm32 - run: cargo check --target wasm32-unknown-unknown --no-default-features + - name: Install dependencies + uses: ./.github/actions/install-linux-deps + with: ${{ matrix.dependencies || fromJSON('{}') }} - uses: dtolnay/rust-toolchain@stable with: - target: x86_64-unknown-freebsd + targets: ${{ join(matrix.targets, ',') }} - - name: Check compilation for freeBSD - run: cargo check --target x86_64-unknown-freebsd - - - name: Prepare repository for redox compilation - run: bash scripts/redox/uncomment-cargo.sh - - name: Check compilation for Redox - uses: coolreader18/redoxer-action@v1 + - name: Setup Android NDK + if: ${{ contains(matrix.targets, 'aarch64-linux-android') }} + id: setup-ndk + uses: nttld/setup-ndk@v1 with: - command: check + ndk-version: r27 + add-to-path: true + + # - name: Prepare repository for redox compilation + # run: bash scripts/redox/uncomment-cargo.sh + # - name: Check compilation for Redox + # uses: coolreader18/redoxer-action@v1 + # with: + # command: check + # args: --ignore-rust-version + + - name: Check compilation + run: | + for target in ${{ join(matrix.targets, ' ') }} + do + echo "::group::${target}" + cargo check --target $target ${{ env.CARGO_ARGS_NO_SSL }} + echo "::endgroup::" + done + env: + CC_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang + AR_aarch64_linux_android: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar + CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang snippets_cpython: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} env: RUST_BACKTRACE: full name: Run snippets and cpython tests - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + os: + - macos-latest + - ubuntu-latest + - windows-2025 fail-fast: false steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Set up the Windows environment - shell: bash - run: | - choco install llvm openssl --no-progress - if [[ -d "C:\Program Files\OpenSSL-Win64" ]]; then - echo "OPENSSL_DIR=C:\Program Files\OpenSSL-Win64" >> $GITHUB_ENV - else - echo "OPENSSL_DIR=C:\Program Files\OpenSSL" >> $GITHUB_ENV - fi - if: runner.os == 'Windows' - - name: Set up the Mac environment - run: brew install autoconf automake libtool openssl@3 - if: runner.os == 'macOS' + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - name: build rustpython - run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }} - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} + + - name: Install macOS dependencies + uses: ./.github/actions/install-macos-deps + with: + openssl: true + + - name: build rustpython + run: cargo build --release --verbose --features=threading,jit ${{ env.CARGO_ARGS }} + - name: run snippets run: python -m pip install -r requirements.txt && pytest -v working-directory: ./extra_tests + - if: runner.os == 'Linux' name: run cpython platform-independent tests - run: - target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true + run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed --timeout 600 -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 45 + - if: runner.os == 'Linux' name: run cpython platform-dependent tests (Linux) - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true + run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 60 + - if: runner.os == 'macOS' name: run cpython platform-dependent tests (MacOS) - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.MACOS_SKIPS }} + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true + run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 50 + - if: runner.os == 'Windows' name: run cpython platform-dependent tests (windows partial - fixme) - run: - target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }} + env: + RUSTPYTHON_SKIP_ENV_POLLUTERS: true + run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }} + timeout-minutes: 50 + + - if: runner.os == 'Linux' + name: run cpython tests to check if env polluters have stopped polluting (Common/Linux) + shell: bash + run: | + for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_LINUX }}; do + for i in $(seq 1 10); do + set +e + target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} + exit_code=$? + set -e + if [ ${exit_code} -eq 3 ]; then + echo "Test ${thing} polluted the environment on attempt ${i}." + break + fi + done + if [ ${exit_code} -ne 3 ]; then + echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" + echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_LINUX in '.github/workflows/ci.yaml'." + echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." + if [ ${exit_code} -ne 0 ]; then + echo "Test ${thing} failed with exit code ${exit_code}." + echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." + fi + exit 1 + fi + done + timeout-minutes: 15 + + - if: runner.os == 'macOS' + name: run cpython tests to check if env polluters have stopped polluting (Common/macOS) + shell: bash + run: | + for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_MACOS }}; do + for i in $(seq 1 10); do + set +e + target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} + exit_code=$? + set -e + if [ ${exit_code} -eq 3 ]; then + echo "Test ${thing} polluted the environment on attempt ${i}." + break + fi + done + if [ ${exit_code} -ne 3 ]; then + echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" + echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_MACOS in '.github/workflows/ci.yaml'." + echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." + if [ ${exit_code} -ne 0 ]; then + echo "Test ${thing} failed with exit code ${exit_code}." + echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." + fi + exit 1 + fi + done + timeout-minutes: 15 + + - if: runner.os == 'Windows' + name: run cpython tests to check if env polluters have stopped polluting (Common/windows) + shell: bash + run: | + for thing in ${{ env.ENV_POLLUTING_TESTS_COMMON }} ${{ env.ENV_POLLUTING_TESTS_WINDOWS }}; do + for i in $(seq 1 10); do + set +e + target/release/rustpython -m test -j 1 --slowest --fail-env-changed --timeout 600 -v ${thing} + exit_code=$? + set -e + if [ ${exit_code} -eq 3 ]; then + echo "Test ${thing} polluted the environment on attempt ${i}." + break + fi + done + if [ ${exit_code} -ne 3 ]; then + echo "Test ${thing} is no longer polluting the environment after ${i} attempts!" + echo "Please remove ${thing} from either ENV_POLLUTING_TESTS_COMMON or ENV_POLLUTING_TESTS_WINDOWS in '.github/workflows/ci.yaml'." + echo "Please also remove the skip decorators that include the word 'POLLUTERS' in ${thing}." + if [ ${exit_code} -ne 0 ]; then + echo "Test ${thing} failed with exit code ${exit_code}." + echo "Please investigate which test item in ${thing} is failing and either mark it as an expected failure or a skip." + fi + exit 1 + fi + done + timeout-minutes: 15 + - if: runner.os != 'Windows' name: check that --install-pip succeeds run: | mkdir site-packages target/release/rustpython --install-pip ensurepip --user - - if: runner.os != 'Windows' - name: Check that ensurepip succeeds. + target/release/rustpython -m pip install six + - name: Check that ensurepip succeeds. run: | target/release/rustpython -m ensurepip target/release/rustpython -c "import pip" @@ -304,57 +423,104 @@ jobs: run: | target/release/rustpython -m venv testvenv testvenv/bin/rustpython -m pip install wheel + - name: Check whats_left is not broken - run: python -I whats_left.py + shell: bash + run: python -I scripts/whats_left.py ${{ env.CARGO_ARGS }} --features jit lint: - name: Check Rust code with rustfmt and clippy + name: Lint Rust & Python code runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: actions/setup-python@v6.2.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Check for redundant test patches + run: python scripts/check_redundant_patches.py + - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy - - name: run rustfmt - run: cargo fmt --check + components: clippy + - name: run clippy on wasm - run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings - - uses: actions/setup-python@v4 + run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings + + - name: Ensure docs generate no warnings + run: cargo doc --locked + + - name: Ensure Lib/_opcode_metadata is updated + run: | + python scripts/generate_opcode_metadata.py + if [ -n "$(git status --porcelain)" ]; then + exit 1 + fi + + - name: Install ruff + uses: astral-sh/ruff-action@4919ec5cf1f49eff0871dbcea0da843445b837e6 # v3.6.1 with: - python-version: ${{ env.PYTHON_VERSION }} - - name: install ruff - run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778 - - name: run python lint - run: ruff extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source + version: "0.15.5" + args: "--version" + + - run: ruff check --diff + + - run: ruff format --check + - name: install prettier run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH + - name: check wasm code with prettier # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506 run: cd wasm && git ls-files -z | xargs -0 prettier --check -u + # Keep cspell check as the last step. This is optional test. + - name: install extra dictionaries + run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell + - name: spell checker + uses: streetsidesoftware/cspell-action@v8 + with: + files: "**/*.rs" + incremental_files_only: true miri: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run tests under miri runs-on: ubuntu-latest + timeout-minutes: 30 + env: + NIGHTLY_CHANNEL: nightly steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@master with: - toolchain: nightly - components: miri + toolchain: ${{ env.NIGHTLY_CHANNEL }} + components: miri - uses: Swatinem/rust-cache@v2 + - name: Run tests under miri - # miri-ignore-leaks because the type-object circular reference means that there will always be - # a memory leak, at least until we have proper cyclic gc - run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test + run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test + env: + # miri-ignore-leaks because the type-object circular reference means that there will always be + # a memory leak, at least until we have proper cyclic gc + MIRIFLAGS: "-Zmiri-ignore-leaks" wasm: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Check the WASM package and demo runs-on: ubuntu-latest + timeout-minutes: 30 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -362,15 +528,18 @@ jobs: run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: install geckodriver run: | - wget https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz + wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz mkdir geckodriver - tar -xzf geckodriver-v0.30.0-linux64.tar.gz -C geckodriver - - uses: actions/setup-python@v4 + tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} - run: python -m pip install -r requirements.txt working-directory: ./wasm/tests - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v6 + with: + cache: "npm" + cache-dependency-path: "wasm/demo/package-lock.json" - name: run test run: | export PATH=$PATH:`pwd`/../../geckodriver @@ -379,6 +548,17 @@ jobs: env: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/demo + - uses: mwilliamson/setup-wabt-action@v3 + with: { wabt-version: "1.0.36" } + - name: check wasm32-unknown without js + run: | + cd example_projects/wasm32_without_js/rustpython-without-js + cargo build + cd .. + if wasm-objdump -xj Import rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm; then + echo "ERROR: wasm32-unknown module expects imports from the host environment" >&2 + fi + cargo run --release --manifest-path wasm-runtime/Cargo.toml rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm - name: build notebook demo if: github.ref == 'refs/heads/release' run: | @@ -390,7 +570,7 @@ jobs: working-directory: ./wasm/notebook - name: Deploy demo to Github Pages if: success() && github.ref == 'refs/heads/release' - uses: peaceiris/actions-gh-pages@v2 + uses: peaceiris/actions-gh-pages@v4 env: ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} PUBLISH_DIR: ./wasm/demo/dist @@ -401,18 +581,28 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run snippets and cpython tests on wasm-wasi runs-on: ubuntu-latest + timeout-minutes: 30 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable with: - target: wasm32-wasi + target: wasm32-wasip1 - uses: Swatinem/rust-cache@v2 - name: Setup Wasmer - uses: wasmerio/setup-wasmer@v2 + uses: wasmerio/setup-wasmer@v3 + - name: Install clang - run: sudo apt-get update && sudo apt-get install clang -y + uses: ./.github/actions/install-linux-deps + with: + clang: true + - name: build rustpython - run: cargo build --release --target wasm32-wasi --features freeze-stdlib,stdlib --verbose + run: cargo build --release --target wasm32-wasip1 --features freeze-stdlib,stdlib --verbose - name: run snippets - run: wasmer run --dir `pwd` target/wasm32-wasi/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py + run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/extra_tests/snippets/stdlib_random.py + - name: run cpython unittest + run: wasmer run --dir `pwd` target/wasm32-wasip1/release/rustpython.wasm -- `pwd`/Lib/test/test_int.py diff --git a/.github/workflows/comment-commands.yml b/.github/workflows/comment-commands.yml new file mode 100644 index 00000000000..3f3402270ea --- /dev/null +++ b/.github/workflows/comment-commands.yml @@ -0,0 +1,21 @@ +name: Comment Commands + +on: + issue_comment: + types: created + +jobs: + issue_assign: + if: (!github.event.issue.pull_request) && github.event.comment.body == 'take' + runs-on: ubuntu-slim + + concurrency: + group: ${{ github.actor }}-issue-assign + + permissions: + issues: write + + steps: + # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651 + - run: | + curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 9176f232c7f..68ef4dea47a 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -1,13 +1,19 @@ on: schedule: - - cron: '0 0 * * 6' + - cron: "0 0 * * 6" workflow_dispatch: + push: + paths: + - .github/workflows/cron-ci.yaml + pull_request: + paths: + - .github/workflows/cron-ci.yaml name: Periodic checks/tasks env: - CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit - PYTHON_VERSION: "3.12.0" + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit + PYTHON_VERSION: "3.14.3" jobs: # codecov collects code coverage data from the rust tests, python snippets and python test suite. @@ -15,34 +21,45 @@ jobs: codecov: name: Collect code coverage data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@cargo-llvm-cov - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} - run: sudo apt-get update && sudo apt-get -y install lcov - name: Run cargo-llvm-cov with Rust tests. - run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit + run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher --verbose --no-default-features --features stdlib,importlib,encodings,ssl-rustls,jit - name: Run cargo-llvm-cov with Python snippets. run: python scripts/cargo-llvm-cov.py continue-on-error: true - name: Run cargo-llvm-cov with Python test suite. - run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed + run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed continue-on-error: true - name: Prepare code coverage data run: cargo llvm-cov report --lcov --output-path='codecov.lcov' - name: Upload to Codecov - uses: codecov/codecov-action@v3 + if: ${{ github.event_name != 'pull_request' }} + uses: codecov/codecov-action@v5 with: file: ./codecov.lcov testdata: name: Collect regression test data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: true + - uses: dtolnay/rust-toolchain@stable - name: build rustpython run: cargo build --release --verbose @@ -51,6 +68,7 @@ jobs: env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib - name: upload tests data to the website + if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} GITHUB_ACTOR: ${{ github.actor }} @@ -70,21 +88,27 @@ jobs: whatsleft: name: Collect what is left data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: true + - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v6.2.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: build rustpython run: cargo build --release --verbose - name: Collect what is left data run: | - chmod +x ./whats_left.py - ./whats_left.py > whats_left.temp + chmod +x ./scripts/whats_left.py + ./scripts/whats_left.py --features "ssl,sqlite" > whats_left.temp env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib - name: Upload data to the website + if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} GITHUB_ACTOR: ${{ github.actor }} @@ -97,6 +121,26 @@ jobs: cd website [ -f ./_data/whats_left.temp ] && cp ./_data/whats_left.temp ./_data/whats_left_lastrun.temp cp ../whats_left.temp ./_data/whats_left.temp + rm ./_data/whats_left/modules.csv + echo -e "module" > ./_data/whats_left/modules.csv + cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv + awk -f - ./_data/whats_left.temp > ./_data/whats_left/builtin_items.csv <<'EOF' + BEGIN { + OFS="," + print "builtin,name,is_inherited" + } + /^# builtin items/ { in_section=1; next } + /^$/ { if (in_section) exit } + in_section { + split($1, a, ".") + rest = "" + idx = index($0, " ") + if (idx > 0) { + rest = substr($0, idx+1) + } + print a[1], $1, rest + } + EOF git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then git push @@ -105,12 +149,17 @@ jobs: benchmark: name: Collect benchmark data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: true + - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v6.2.0 with: - python-version: 3.9 + python-version: ${{ env.PYTHON_VERSION }} - run: cargo install cargo-criterion - name: build benchmarks run: cargo build --release --benches @@ -131,6 +180,7 @@ jobs: mv reports/* . rmdir reports - name: upload benchmark data to the website + if: ${{ github.event_name != 'pull_request' }} env: SSHKEY: ${{ secrets.ACTIONS_TESTS_DATA_DEPLOY_KEY }} run: | @@ -142,7 +192,11 @@ jobs: cd website rm -rf ./assets/criterion cp -r ../target/criterion ./assets/criterion - git add ./assets/criterion + printf '{\n "generated_at": "%s",\n "rustpython_commit": "%s",\n "rustpython_ref": "%s"\n}\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + "${{ github.sha }}" \ + "${{ github.ref_name }}" > ./_data/criterion-metadata.json + git add ./assets/criterion ./_data/criterion-metadata.json if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then git push fi diff --git a/.github/workflows/lib-deps-check.yaml b/.github/workflows/lib-deps-check.yaml new file mode 100644 index 00000000000..0b4c8590811 --- /dev/null +++ b/.github/workflows/lib-deps-check.yaml @@ -0,0 +1,123 @@ +name: Lib Dependencies Check + +on: + pull_request_target: + types: [opened, synchronize, reopened] + paths: + - "Lib/**" + +concurrency: + group: lib-deps-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + PYTHON_VERSION: "3.14.3" + +jobs: + check_deps: + permissions: + pull-requests: write + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout base branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Use base branch for scripts (security: don't run PR code with elevated permissions) + ref: ${{ github.event.pull_request.base.ref }} + fetch-depth: 0 + persist-credentials: false + + - name: Fetch PR head + run: | + git fetch origin ${{ github.event.pull_request.head.sha }} + + - name: Checkout PR Lib files + run: | + # Checkout only Lib/ directory from PR head for accurate comparison + git checkout ${{ github.event.pull_request.head.sha }} -- Lib/ + + - name: Checkout CPython + run: | + git clone --depth 1 --branch "v${{ env.PYTHON_VERSION }}" https://github.com/python/cpython.git cpython + + - name: Get changed Lib files + id: changed-files + run: | + # Get the list of changed files under Lib/ + changed=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'Lib/*.py' 'Lib/**/*.py' | head -50) + echo "Changed files:" + echo "$changed" + + # Extract unique module names + modules="" + for file in $changed; do + if [[ "$file" == Lib/test/* ]]; then + # Test files: Lib/test/test_pydoc.py -> test_pydoc, Lib/test/test_pydoc/foo.py -> test_pydoc + module=$(echo "$file" | sed -E 's|^Lib/test/||; s|\.py$||; s|/.*||') + # Skip non-test files in test/ (e.g., support.py, __init__.py) + if [[ ! "$module" == test_* ]]; then + continue + fi + else + # Lib files: Lib/foo.py -> foo, Lib/foo/__init__.py -> foo + module=$(echo "$file" | sed -E 's|^Lib/||; s|/__init__\.py$||; s|\.py$||; s|/.*||') + fi + if [[ -n "$module" && ! " $modules " =~ " $module " ]]; then + modules="$modules $module" + fi + done + + modules=$(echo "$modules" | xargs) # trim whitespace + echo "Detected modules: $modules" + echo "modules=$modules" >> $GITHUB_OUTPUT + + - name: Setup Python + if: steps.changed-files.outputs.modules != '' + uses: actions/setup-python@v6.2.0 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Run deps check + if: steps.changed-files.outputs.modules != '' + id: deps-check + run: | + # Run deps for all modules at once + python scripts/update_lib deps ${{ steps.changed-files.outputs.modules }} --depth 2 > /tmp/deps_output.txt 2>&1 || true + + # Read output for GitHub Actions + echo "deps_output<> $GITHUB_OUTPUT + cat /tmp/deps_output.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # Check if there's any meaningful output + if [ -s /tmp/deps_output.txt ]; then + echo "has_output=true" >> $GITHUB_OUTPUT + else + echo "has_output=false" >> $GITHUB_OUTPUT + fi + + - name: Post comment + if: steps.deps-check.outputs.has_output == 'true' + uses: marocchino/sticky-pull-request-comment@v3 + with: + header: lib-deps-check + number: ${{ github.event.pull_request.number }} + message: | + ## 📦 Library Dependencies + + The following Lib/ modules were modified. Here are their dependencies: + + ${{ steps.deps-check.outputs.deps_output }} + + **Legend:** + - `[+]` path exists in CPython + - `[x]` up-to-date, `[ ]` outdated + + - name: Remove comment if no Lib changes + if: steps.changed-files.outputs.modules == '' + uses: marocchino/sticky-pull-request-comment@v3 + with: + header: lib-deps-check + number: ${{ github.event.pull_request.number }} + delete: true diff --git a/.github/workflows/pr-format.yaml b/.github/workflows/pr-format.yaml new file mode 100644 index 00000000000..6b11a758668 --- /dev/null +++ b/.github/workflows/pr-format.yaml @@ -0,0 +1,65 @@ +name: Format Check + +# This workflow triggers when a PR is opened/updated +# Posts inline suggestion comments instead of auto-committing +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + - release + +concurrency: + group: format-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + format_check: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Run cargo fmt + run: cargo fmt --all + + - name: Install ruff + uses: astral-sh/ruff-action@4919ec5cf1f49eff0871dbcea0da843445b837e6 # v3.6.1 + with: + version: "0.15.4" + args: "--version" + + - name: Run ruff format + run: ruff format + + - name: Run ruff check import sorting + run: ruff check --select I --fix + + - name: Run generate_opcode_metadata.py + run: python scripts/generate_opcode_metadata.py + + - name: Check for formatting changes + run: | + if ! git diff --exit-code; then + echo "::error::Formatting changes detected. Please run 'cargo fmt --all', 'ruff format', and 'ruff check --select I --fix' locally." + exit 1 + fi + + - name: Post formatting suggestions + if: failure() + uses: reviewdog/action-suggester@v1 + with: + tool_name: auto-format + github_token: ${{ secrets.GITHUB_TOKEN }} + level: warning + filter_mode: diff_context diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..d8eacca71ac --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,190 @@ +name: Release + +on: + schedule: + # 9 AM UTC on every Monday + - cron: "0 9 * * Mon" + workflow_dispatch: + inputs: + pre-release: + type: boolean + description: Mark "Pre-Release" + required: false + default: true + +permissions: + contents: write + +env: + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl + X86_64_PC_WINDOWS_MSVC_OPENSSL_LIB_DIR: C:\Program Files\OpenSSL\lib\VC\x64\MD + X86_64_PC_WINDOWS_MSVC_OPENSSL_INCLUDE_DIR: C:\Program Files\OpenSSL\include + +jobs: + build: + runs-on: ${{ matrix.platform.runner }} + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} + strategy: + matrix: + platform: + - runner: ubuntu-latest + target: x86_64-unknown-linux-gnu +# - runner: ubuntu-latest +# target: i686-unknown-linux-gnu +# - runner: ubuntu-latest +# target: aarch64-unknown-linux-gnu +# - runner: ubuntu-latest +# target: armv7-unknown-linux-gnueabi +# - runner: ubuntu-latest +# target: s390x-unknown-linux-gnu +# - runner: ubuntu-latest +# target: powerpc64le-unknown-linux-gnu + - runner: macos-latest + target: aarch64-apple-darwin +# - runner: macos-latest +# target: x86_64-apple-darwin + - runner: windows-2025 + target: x86_64-pc-windows-msvc +# - runner: windows-2025 +# target: i686-pc-windows-msvc +# - runner: windows-2025 +# target: aarch64-pc-windows-msvc + fail-fast: false + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + - uses: cargo-bins/cargo-binstall@main + + - name: Set up Environment + shell: bash + run: rustup target add ${{ matrix.platform.target }} + - name: Set up MacOS Environment + run: brew install autoconf automake libtool + if: runner.os == 'macOS' + + - name: Build RustPython + run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }} + if: runner.os == 'macOS' + - name: Build RustPython + run: cargo build --release --target=${{ matrix.platform.target }} --verbose --features=threading ${{ env.CARGO_ARGS }},jit + if: runner.os != 'macOS' + + - name: Rename Binary + run: cp target/${{ matrix.platform.target }}/release/rustpython target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }} + if: runner.os != 'Windows' + - name: Rename Binary + run: cp target/${{ matrix.platform.target }}/release/rustpython.exe target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}.exe + if: runner.os == 'Windows' + + - name: Upload Binary Artifacts + uses: actions/upload-artifact@v7.0.0 + with: + name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }} + path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}* + + build-wasm: + runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip1 + + - name: Build RustPython + run: cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release + + - name: Rename Binary + run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm + + - name: Upload Binary Artifacts + uses: actions/upload-artifact@v7.0.0 + with: + name: rustpython-release-wasm32-wasip1 + path: target/rustpython-release-wasm32-wasip1.wasm + + - name: install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - uses: actions/setup-node@v6 + - uses: mwilliamson/setup-wabt-action@v3 + with: { wabt-version: "1.0.30" } + - name: build demo + run: | + npm install + npm run dist + env: + NODE_OPTIONS: "--openssl-legacy-provider" + working-directory: ./wasm/demo + - name: build notebook demo + run: | + npm install + npm run dist + mv dist ../demo/dist/notebook + env: + NODE_OPTIONS: "--openssl-legacy-provider" + working-directory: ./wasm/notebook + - name: Deploy demo to Github Pages + uses: peaceiris/actions-gh-pages@v4 + with: + deploy_key: ${{ secrets.ACTIONS_DEMO_DEPLOY_KEY }} + publish_dir: ./wasm/demo/dist + external_repository: RustPython/demo + publish_branch: master + + release: + runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} + needs: [build, build-wasm] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download Binary Artifacts + uses: actions/download-artifact@v8.0.0 + with: + path: bin + pattern: rustpython-* + merge-multiple: true + + - name: Create Lib Archive + run: | + zip -r bin/rustpython-lib.zip Lib/ + + - name: List Binaries + run: | + ls -lah bin/ + file bin/* + - name: Create Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.ref_name }} + run: ${{ github.run_number }} + PRE_RELEASE_INPUT: ${{ github.event.inputs.pre-release }} + run: | + if [[ "${PRE_RELEASE_INPUT}" == "false" ]]; then + RELEASE_TYPE_NAME=Release + PRERELEASE_ARG= + else + RELEASE_TYPE_NAME=Pre-Release + PRERELEASE_ARG=--prerelease + fi + + today=$(date '+%Y-%m-%d') + gh release create "$today-$tag-$run" \ + --repo="$GITHUB_REPOSITORY" \ + --title="RustPython $RELEASE_TYPE_NAME $today-$tag #$run" \ + --target="$tag" \ + --notes "⚠️ **Important**: To run RustPython, you must download both the binary for your platform AND the \`rustpython-lib.zip\` archive. Extract the Lib directory from the archive to the same location as the binary, or set the \`RUSTPYTHONPATH\` environment variable to point to the Lib directory." \ + --generate-notes \ + $PRERELEASE_ARG \ + bin/rustpython-release-* diff --git a/.github/workflows/update-doc-db.yml b/.github/workflows/update-doc-db.yml new file mode 100644 index 00000000000..b3ad2369e3b --- /dev/null +++ b/.github/workflows/update-doc-db.yml @@ -0,0 +1,124 @@ +name: Update doc DB + +permissions: + contents: write + pull-requests: write + +on: + workflow_dispatch: + inputs: + python-version: + description: Target python version to generate doc db for + type: string + default: "3.14.3" + base-ref: + description: Base branch to create the update branch from + type: string + default: "main" + +defaults: + run: + shell: bash + +jobs: + generate: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + sparse-checkout: | + crates/doc + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ inputs.python-version }} + + - name: Generate docs + run: python crates/doc/generate.py + + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: doc-db-${{ inputs.python-version }}-${{ matrix.os }} + path: "crates/doc/generated/*.json" + if-no-files-found: error + retention-days: 7 + overwrite: true + + merge: + runs-on: ubuntu-latest + needs: generate + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: true + ref: ${{ inputs.base-ref }} + token: ${{ secrets.AUTO_COMMIT_PAT }} + + - name: Create update branch + env: + PYTHON_VERSION: ${{ inputs.python-version }} + run: git switch -c "update-doc-${PYTHON_VERSION}" + + - name: Download generated doc DBs + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + pattern: "doc-db-${{ inputs.python-version }}-**" + path: crates/doc/generated/ + merge-multiple: true + + - name: Transform JSON + env: + PYTHON_VERSION: ${{ inputs.python-version }} + run: | + # Merge all artifacts + jq -s "add" --sort-keys crates/doc/generated/*.json > crates/doc/generated/merged.json + + # Format merged json for the phf macro + jq -r 'to_entries[] | " \(.key | @json) => \(.value | @json),"' crates/doc/generated/merged.json > crates/doc/generated/raw_entries.txt + + OUTPUT_FILE='crates/doc/src/data.inc.rs' + + echo -n '' > $OUTPUT_FILE + + echo '// This file was auto-generated by `.github/workflows/update-doc-db.yml`.' >> $OUTPUT_FILE + echo "// CPython version: ${PYTHON_VERSION}" >> $OUTPUT_FILE + echo '// spell-checker: disable' >> $OUTPUT_FILE + + echo '' >> $OUTPUT_FILE + + echo "pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! {" >> $OUTPUT_FILE + cat crates/doc/generated/raw_entries.txt >> $OUTPUT_FILE + echo '};' >> $OUTPUT_FILE + + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: doc-db-${{ inputs.python-version }} + path: "crates/doc/src/data.inc.rs" + if-no-files-found: error + retention-days: 7 + overwrite: true + + - name: Commit, push and create PR + env: + GH_TOKEN: ${{ secrets.AUTO_COMMIT_PAT }} + PYTHON_VERSION: ${{ inputs.python-version }} + BASE_REF: ${{ inputs.base-ref }} + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + if [ -n "$(git status --porcelain)" ]; then + git add crates/doc/src/data.inc.rs + git commit -m "Update doc DB for CPython ${PYTHON_VERSION}" + git push -u origin HEAD + gh pr create \ + --base "${BASE_REF}" \ + --title "Update doc DB for CPython ${PYTHON_VERSION}" \ + --body "Auto-generated by update-doc-db workflow." + fi diff --git a/.github/workflows/update-libs-status.yaml b/.github/workflows/update-libs-status.yaml new file mode 100644 index 00000000000..53685f88474 --- /dev/null +++ b/.github/workflows/update-libs-status.yaml @@ -0,0 +1,90 @@ +name: Updated libs status + +on: + push: + branches: + - main + paths: + - "Lib/**" + workflow_dispatch: + +permissions: + contents: read + issues: write + +env: + PYTHON_VERSION: "v3.14.3" + ISSUE_ID: "6839" + +jobs: + update-issue: + runs-on: ubuntu-latest + if: ${{ github.repository == 'RustPython/RustPython' }} + steps: + - name: Clone RustPython + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: rustpython + persist-credentials: false + sparse-checkout: |- + Lib + scripts/update_lib + + - name: Clone CPython ${{ env.PYTHON_VERSION }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: python/cpython + path: cpython + ref: ${{ env.PYTHON_VERSION }} + persist-credentials: false + sparse-checkout: | + Lib + + - name: Get current date + id: current_date + run: | + now=$(date -u +"%Y-%m-%d %H:%M:%S") + echo "date=$now" >> "$GITHUB_OUTPUT" + + - name: Write body prefix + run: | + cat > body.txt < + + ## Summary + + Check \`scripts/update_lib\` for tools. As a note, the current latest Python version is \`${{ env.PYTHON_VERSION }}\`. + + Previous versions' issues as reference + - 3.13: #5529 + + + + ## Details + + ${{ steps.current_date.outputs.date }} (UTC) + \`\`\`shell + $ python3 scripts/update_lib todo --done + \`\`\` + EOF + + - name: Run todo + run: python3 rustpython/scripts/update_lib todo --cpython cpython --lib rustpython/Lib --done >> body.txt + + - name: Update GH issue + run: gh issue edit ${{ env.ISSUE_ID }} --body-file ../body.txt + env: + GH_TOKEN: ${{ github.token }} + working-directory: rustpython + + diff --git a/.github/workflows/upgrade-pylib.lock.yml b/.github/workflows/upgrade-pylib.lock.yml new file mode 100644 index 00000000000..e4ae875bd4b --- /dev/null +++ b/.github/workflows/upgrade-pylib.lock.yml @@ -0,0 +1,1093 @@ +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.43.22). DO NOT EDIT. +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# Pick an out-of-sync Python library from the todo list and upgrade it +# by running `scripts/update_lib quick`, then open a pull request. +# +# frontmatter-hash: 3129480d6628afe028911bb8b31b6bb3b5eb251e395f00ed0677922cd21727cb + +name: "Upgrade Python Library" +"on": + workflow_dispatch: + inputs: + name: + description: Module name to upgrade (leave empty to auto-pick) + required: false + type: string + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}" + +run-name: "Upgrade Python Library" + +env: + ISSUE_ID: "6839" + PYTHON_VERSION: v3.14.3 + +# Cache configuration from frontmatter was processed and added to the main job steps + +jobs: + activation: + runs-on: ubuntu-slim + permissions: + contents: read + outputs: + comment_id: "" + comment_repo: "" + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@08a903b1fb2e493a84a57577778fe5dd711f9468 # v0.58.3 + with: + destination: /opt/gh-aw/actions + - name: Check workflow file timestamps + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_WORKFLOW_FILE: "upgrade-pylib.lock.yml" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + outputs: + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + model: ${{ steps.generate_aw_info.outputs.model }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@08a903b1fb2e493a84a57577778fe5dd711f9468 # v0.58.3 + with: + destination: /opt/gh-aw/actions + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.14' + - name: Create gh-aw temp directory + run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + # Cache configuration from frontmatter processed below + - name: Cache (cpython-lib-${{ env.PYTHON_VERSION }}) + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + key: cpython-lib-${{ env.PYTHON_VERSION }} + path: cpython + restore-keys: | + cpython-lib- + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + github.event.pull_request + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Generate agentic run info + id: generate_aw_info + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const fs = require('fs'); + + const awInfo = { + engine_id: "copilot", + engine_name: "GitHub Copilot CLI", + model: process.env.GH_AW_MODEL_AGENT_COPILOT || "", + version: "", + agent_version: "0.0.409", + cli_version: "v0.43.22", + workflow_name: "Upgrade Python Library", + experimental: false, + supports_tools_allowlist: true, + supports_http_transport: true, + run_id: context.runId, + run_number: context.runNumber, + run_attempt: process.env.GITHUB_RUN_ATTEMPT, + repository: context.repo.owner + '/' + context.repo.repo, + ref: context.ref, + sha: context.sha, + actor: context.actor, + event_name: context.eventName, + staged: false, + allowed_domains: ["defaults","rust","python"], + firewall_enabled: true, + awf_version: "v0.16.4", + awmg_version: "", + steps: { + firewall: "squid" + }, + created_at: new Date().toISOString() + }; + + // Write to /tmp/gh-aw directory to avoid inclusion in PR + const tmpPath = '/tmp/gh-aw/aw_info.json'; + fs.writeFileSync(tmpPath, JSON.stringify(awInfo, null, 2)); + console.log('Generated aw_info.json at:', tmpPath); + console.log(JSON.stringify(awInfo, null, 2)); + + // Set model as output for reuse in other steps/jobs + core.setOutput('model', awInfo.model); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.409 + - name: Install awf binary + run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.16.4 + - name: Determine automatic lockdown mode for GitHub MCP server + id: determine-automatic-lockdown + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + with: + script: | + const determineAutomaticLockdown = require('/opt/gh-aw/actions/determine_automatic_lockdown.cjs'); + await determineAutomaticLockdown(github, context, core); + - name: Download container images + run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.16.4 ghcr.io/github/gh-aw-firewall/squid:0.16.4 ghcr.io/github/gh-aw-mcpg:v0.1.4 ghcr.io/github/github-mcp-server:v0.30.3 node:lts-alpine + - name: Write Safe Outputs Config + run: | + mkdir -p /opt/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' + {"create_pull_request":{"expires":30},"missing_data":{},"missing_tool":{},"noop":{"max":1}} + GH_AW_SAFE_OUTPUTS_CONFIG_EOF + cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' + [ + { + "description": "Create a new GitHub pull request to propose code changes. Use this after making file edits to submit them for review and merging. The PR will be created from the current branch with your committed changes. For code review comments on an existing PR, use create_pull_request_review_comment instead. CONSTRAINTS: Maximum 1 pull request(s) can be created. Title will be prefixed with \"Update \". Labels [pylib-sync] will be automatically added.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "Detailed PR description in Markdown. Include what changes were made, why, testing notes, and any breaking changes. Do NOT repeat the title as a heading.", + "type": "string" + }, + "branch": { + "description": "Source branch name containing the changes. If omitted, uses the current working branch.", + "type": "string" + }, + "labels": { + "description": "Labels to categorize the PR (e.g., 'enhancement', 'bugfix'). Labels must exist in the repository.", + "items": { + "type": "string" + }, + "type": "array" + }, + "title": { + "description": "Concise PR title describing the changes. Follow repository conventions (e.g., conventional commits). The title appears as the main heading.", + "type": "string" + } + }, + "required": [ + "title", + "body" + ], + "type": "object" + }, + "name": "create_pull_request" + }, + { + "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "reason": { + "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", + "type": "string" + }, + "tool": { + "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", + "type": "string" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "name": "missing_tool" + }, + { + "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "message": { + "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "name": "noop" + }, + { + "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "context": { + "description": "Additional context about the missing data or where it should come from (max 256 characters).", + "type": "string" + }, + "data_type": { + "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", + "type": "string" + }, + "reason": { + "description": "Explanation of why this data is needed to complete the task (max 256 characters).", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "name": "missing_data" + } + ] + GH_AW_SAFE_OUTPUTS_TOOLS_EOF + cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' + { + "create_pull_request": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "branch": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "labels": { + "type": "array", + "itemType": "string", + "itemSanitize": true, + "itemMaxLength": 128 + }, + "title": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + } + } + GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3001 + + # Set outputs for next steps + { + echo "safe_outputs_api_key=${API_KEY}" + echo "safe_outputs_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "Safe Outputs MCP server will run on port ${PORT}" + + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start + env: + DEBUG: '*' + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR + + bash /opt/gh-aw/actions/start_safe_outputs_server.sh + + - name: Start MCP gateway + id: start-mcp-gateway + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} + GITHUB_MCP_LOCKDOWN: ${{ steps.determine-automatic-lockdown.outputs.lockdown == 'true' && '1' || '0' }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p /tmp/gh-aw/mcp-config + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.4' + + mkdir -p /home/runner/.copilot + cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + { + "mcpServers": { + "github": { + "type": "stdio", + "container": "ghcr.io/github/github-mcp-server:v0.30.3", + "env": { + "GITHUB_LOCKDOWN_MODE": "$GITHUB_MCP_LOCKDOWN", + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}", + "GITHUB_READ_ONLY": "1", + "GITHUB_TOOLSETS": "repos,issues,pull_requests" + } + }, + "safeoutputs": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", + "headers": { + "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_EOF + - name: Generate workflow overview + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); + await generateWorkflowOverview(core); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + bash /opt/gh-aw/actions/create_prompt_first.sh + cat << 'GH_AW_PROMPT_EOF' > "$GH_AW_PROMPT" + + GH_AW_PROMPT_EOF + cat "/opt/gh-aw/prompts/xpia.md" >> "$GH_AW_PROMPT" + cat "/opt/gh-aw/prompts/temp_folder_prompt.md" >> "$GH_AW_PROMPT" + cat "/opt/gh-aw/prompts/markdown.md" >> "$GH_AW_PROMPT" + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + + GitHub API Access Instructions + + The gh CLI is NOT authenticated. Do NOT use gh commands for GitHub operations. + + + To create or modify GitHub resources (issues, discussions, pull requests, etc.), you MUST call the appropriate safe output tool. Simply writing content will NOT work - the workflow requires actual tool calls. + + Temporary IDs: Some safe output tools support a temporary ID field (usually named temporary_id) so you can reference newly-created items elsewhere in the SAME agent output (for example, using #aw_abc1 in a later body). + + **IMPORTANT - temporary_id format rules:** + - If you DON'T need to reference the item later, OMIT the temporary_id field entirely (it will be auto-generated if needed) + - If you DO need cross-references/chaining, you MUST match this EXACT validation regex: /^aw_[A-Za-z0-9]{3,8}$/i + - Format: aw_ prefix followed by 3 to 8 alphanumeric characters (A-Z, a-z, 0-9, case-insensitive) + - Valid alphanumeric characters: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 + - INVALID examples: aw_ab (too short), aw_123456789 (too long), aw_test-id (contains hyphen), aw_id_123 (contains underscore) + - VALID examples: aw_abc, aw_abc1, aw_Test123, aw_A1B2C3D4, aw_12345678 + - To generate valid IDs: use 3-8 random alphanumeric characters or omit the field to let the system auto-generate + + Do NOT invent other aw_* formats — downstream steps will reject them with validation errors matching against /^aw_[A-Za-z0-9]{3,8}$/i. + + Discover available tools from the safeoutputs MCP server. + + **Critical**: Tool calls write structured data that downstream jobs process. Without tool calls, follow-up actions will be skipped. + + **Note**: If you made no other safe output tool calls during this workflow execution, call the "noop" tool to provide a status message indicating completion or that no actions were needed. + + + + The following GitHub context information is available for this workflow: + {{#if __GH_AW_GITHUB_ACTOR__ }} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if __GH_AW_GITHUB_REPOSITORY__ }} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if __GH_AW_GITHUB_WORKSPACE__ }} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} + - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} + - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} + - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} + - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ + {{/if}} + {{#if __GH_AW_GITHUB_RUN_ID__ }} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' >> "$GH_AW_PROMPT" + {{#runtime-import .github/workflows/upgrade-pylib.md}} + GH_AW_PROMPT_EOF + - name: Substitute placeholders + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + with: + script: | + const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_ENV_ISSUE_ID: process.env.GH_AW_ENV_ISSUE_ID, + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, + GH_AW_GITHUB_EVENT_INPUTS_NAME: process.env.GH_AW_GITHUB_EVENT_INPUTS_NAME, + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + } + }); + - name: Interpolate variables and render templates + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_ENV_ISSUE_ID: ${{ env.ISSUE_ID }} + GH_AW_GITHUB_EVENT_INPUTS_NAME: ${{ github.event.inputs.name }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/print_prompt_summary.sh + - name: Clean git credentials + run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 45 + run: | + set -o pipefail + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains '*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com' --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.16.4 --skip-pull \ + -- '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"${GH_AW_MODEL_AGENT_COPILOT:+ --model "$GH_AW_MODEL_AGENT_COPILOT"}' \ + 2>&1 | tee /tmp/gh-aw/agent-stdio.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_MODEL_AGENT_COPILOT: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: | + # Copy Copilot session state files to logs folder for artifact collection + # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them + SESSION_STATE_DIR="$HOME/.copilot/session-state" + LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" + if [ -d "$SESSION_STATE_DIR" ]; then + echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" + mkdir -p "$LOGS_DIR" + cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true + echo "Session state files copied successfully" + else + echo "No session-state directory found at $SESSION_STATE_DIR" + fi + - name: Stop MCP gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Safe Outputs + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: safe-output + path: ${{ env.GH_AW_SAFE_OUTPUTS }} + if-no-files-found: warn + - name: Ingest agent output + id: collect_output + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "*.pythonhosted.org,anaconda.org,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,binstar.org,bootstrap.pypa.io,conda.anaconda.org,conda.binstar.org,crates.io,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,files.pythonhosted.org,github.com,host.docker.internal,index.crates.io,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pip.pypa.io,ppa.launchpad.net,pypi.org,pypi.python.org,raw.githubusercontent.com,registry.npmjs.org,repo.anaconda.com,repo.continuum.io,s.symcb.com,s.symcd.com,security.ubuntu.com,sh.rustup.rs,static.crates.io,static.rust-lang.org,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + await main(); + - name: Upload sanitized agent output + if: always() && env.GH_AW_AGENT_OUTPUT + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: agent-output + path: ${{ env.GH_AW_AGENT_OUTPUT }} + if-no-files-found: warn + - name: Upload engine output files + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: agent_outputs + path: | + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + if-no-files-found: ignore + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP gateway logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: agent-artifacts + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/agent/ + /tmp/gh-aw/aw*.patch + if-no-files-found: ignore + + conclusion: + needs: + - activation + - agent + - detection + - safe_outputs + if: (always()) && (needs.agent.result != 'skipped') + runs-on: ubuntu-slim + permissions: + contents: read + discussions: write + issues: write + pull-requests: write + outputs: + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@08a903b1fb2e493a84a57577778fe5dd711f9468 # v0.58.3 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process No-Op Messages + id: noop + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: 1 + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/noop.cjs'); + await main(); + - name: Record Missing Tool + id: missing_tool + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + await main(); + - name: Handle Agent Failure + id: handle_agent_failure + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "upgrade-pylib" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.agent.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + await main(); + - name: Handle No-Op Message + id: handle_noop_message + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + await main(); + - name: Handle Create Pull Request Error + id: handle_create_pr_error + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_create_pr_error.cjs'); + await main(); + - name: Update reaction comment with completion status + id: conclusion + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} + GH_AW_COMMENT_REPO: ${{ needs.activation.outputs.comment_repo }} + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.result }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/notify_comment_error.cjs'); + await main(); + + detection: + needs: agent + if: needs.agent.outputs.output_types != '' || needs.agent.outputs.has_patch == 'true' + runs-on: ubuntu-latest + permissions: {} + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + timeout-minutes: 10 + outputs: + success: ${{ steps.parse_results.outputs.success }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@08a903b1fb2e493a84a57577778fe5dd711f9468 # v0.58.3 + with: + destination: /opt/gh-aw/actions + - name: Download agent artifacts + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: agent-artifacts + path: /tmp/gh-aw/threat-detection/ + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: agent-output + path: /tmp/gh-aw/threat-detection/ + - name: Echo agent output types + env: + AGENT_OUTPUT_TYPES: ${{ needs.agent.outputs.output_types }} + run: | + echo "Agent output-types: $AGENT_OUTPUT_TYPES" + - name: Setup threat detection + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + WORKFLOW_NAME: "Upgrade Python Library" + WORKFLOW_DESCRIPTION: "Pick an out-of-sync Python library from the todo list and upgrade it\nby running `scripts/update_lib quick`, then open a pull request." + HAS_PATCH: ${{ needs.agent.outputs.has_patch }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + await main(); + - name: Ensure threat-detection directory and log + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh 0.0.409 + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool shell(cat) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(jq) + # --allow-tool shell(ls) + # --allow-tool shell(tail) + # --allow-tool shell(wc) + timeout-minutes: 20 + run: | + set -o pipefail + COPILOT_CLI_INSTRUCTION="$(cat /tmp/gh-aw/aw-prompts/prompt.txt)" + mkdir -p /tmp/ + mkdir -p /tmp/gh-aw/ + mkdir -p /tmp/gh-aw/agent/ + mkdir -p /tmp/gh-aw/sandbox/agent/logs/ + copilot --add-dir /tmp/ --add-dir /tmp/gh-aw/ --add-dir /tmp/gh-aw/agent/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --allow-tool 'shell(cat)' --allow-tool 'shell(grep)' --allow-tool 'shell(head)' --allow-tool 'shell(jq)' --allow-tool 'shell(ls)' --allow-tool 'shell(tail)' --allow-tool 'shell(wc)' --share /tmp/gh-aw/sandbox/agent/logs/conversation.md --prompt "$COPILOT_CLI_INSTRUCTION"${GH_AW_MODEL_DETECTION_COPILOT:+ --model "$GH_AW_MODEL_DETECTION_COPILOT"} 2>&1 | tee /tmp/gh-aw/threat-detection/detection.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + GH_AW_MODEL_DETECTION_COPILOT: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Parse threat detection results + id: parse_results + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + - name: Upload threat detection log + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: threat-detection.log + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + + safe_outputs: + needs: + - activation + - agent + - detection + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.detection.outputs.success == 'true') + runs-on: ubuntu-slim + permissions: + contents: write + issues: write + pull-requests: write + timeout-minutes: 15 + env: + GH_AW_ENGINE_ID: "copilot" + GH_AW_WORKFLOW_ID: "upgrade-pylib" + GH_AW_WORKFLOW_NAME: "Upgrade Python Library" + outputs: + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@08a903b1fb2e493a84a57577778fe5dd711f9468 # v0.58.3 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Download patch artifact + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + with: + name: agent-artifacts + path: /tmp/gh-aw/ + - name: Checkout repository + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + token: ${{ github.token }} + persist-credentials: false + fetch-depth: 1 + - name: Configure Git credentials + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (contains(needs.agent.outputs.output_types, 'create_pull_request')) + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + GIT_TOKEN: ${{ github.token }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${GIT_TOKEN}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_pull_request\":{\"base_branch\":\"${{ github.ref_name }}\",\"draft\":false,\"expires\":30,\"labels\":[\"pylib-sync\"],\"max\":1,\"max_patch_size\":1024,\"title_prefix\":\"Update \"},\"missing_data\":{},\"missing_tool\":{}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + await main(); + diff --git a/.github/workflows/upgrade-pylib.md b/.github/workflows/upgrade-pylib.md new file mode 100644 index 00000000000..f8f0902b6b8 --- /dev/null +++ b/.github/workflows/upgrade-pylib.md @@ -0,0 +1,134 @@ +--- +description: | + Pick an out-of-sync Python library from the todo list and upgrade it + by running `scripts/update_lib quick`, then open a pull request. + +on: + workflow_dispatch: + inputs: + name: + description: "Module name to upgrade (leave empty to auto-pick)" + required: false + type: string + +timeout-minutes: 45 + +permissions: + contents: read + issues: read + pull-requests: read + +network: + allowed: + - defaults + - rust + - python + +engine: copilot + +runtimes: + python: + version: "3.14" + +tools: + bash: + - ":*" + edit: + github: + toolsets: [repos, issues, pull_requests] + read-only: true + +safe-outputs: + create-pull-request: + title-prefix: "Update " + labels: [pylib-sync] + draft: false + expires: 30 + +cache: + key: cpython-lib-${{ env.PYTHON_VERSION }} + path: cpython + restore-keys: + - cpython-lib- + +env: + PYTHON_VERSION: "v3.14.3" + ISSUE_ID: "6839" +--- + +# Upgrade Python Library + +You are an automated maintenance agent for RustPython, a Python 3 interpreter written in Rust. Your task is to upgrade one out-of-sync Python standard library module from CPython. + +## Step 1: Set up the environment + +The CPython source may already be cached. Check if the `cpython` directory exists and has the correct version: + +```bash +if [ -d "cpython/Lib" ]; then + echo "CPython cache hit, skipping clone" +else + git clone --depth 1 --branch "$PYTHON_VERSION" https://github.com/python/cpython.git cpython +fi +``` + +## Step 2: Determine module name + +Run this script to determine the module name: + +```bash +MODULE_NAME="${{ github.event.inputs.name }}" +if [ -z "$MODULE_NAME" ]; then + echo "No module specified, running todo to find one..." + python3 scripts/update_lib todo + echo "Pick one module from the list above that is marked [ ], has no unmet deps, and has a small Δ number." + echo "Do NOT pick: opcode, datetime, random, hashlib, tokenize, pdb, _pyrepl, concurrent, asyncio, multiprocessing, ctypes, idlelib, tkinter, shutil, tarfile, email, unittest" +else + echo "Module specified by user: $MODULE_NAME" +fi +``` + +If the script printed "Module specified by user: ...", use that exact name. If it printed the todo list, pick one suitable module from it. + +## Step 3: Run the upgrade + +Run the quick upgrade command. This will copy the library from CPython, migrate test files preserving RustPython markers, auto-mark test failures, and create a git commit: + +```bash +python3 scripts/update_lib quick +``` + +This takes a while because it builds RustPython (`cargo build --release`) and runs tests to determine which ones pass or fail. + +If the command fails, report the error and stop. Do not try to fix Rust code or modify test files manually. + +## Step 4: Verify the result + +After the script succeeds, check what changed: + +```bash +git log -1 --stat +git diff HEAD~1 --stat +``` + +Make sure the commit was created with the correct message format: `Update from `. + +## Step 5: Create the pull request + +Create a pull request. Reference issue #${{ env.ISSUE_ID }} in the body but do **NOT** use keywords that auto-close issues (Fix, Close, Resolve). + +Use this format for the PR body: + +``` +## Summary + +Upgrade `` from CPython $PYTHON_VERSION. + +Part of #$ISSUE_ID + +## Changes + +- Updated `Lib/` from CPython +- Migrated test files preserving RustPython markers +- Auto-marked test failures with `@expectedFailure` +``` diff --git a/.gitignore b/.gitignore index 23facc8e6e5..d173739854d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,13 +2,15 @@ /*/target **/*.rs.bk **/*.bytecode -__pycache__ +__pycache__/ **/*.pytest_cache .*sw* .repl_history.txt -.vscode +.vscode/ wasm-pack.log .idea/ +.envrc +.python-version flame-graph.html flame.txt @@ -19,3 +21,11 @@ flamescope.json extra_tests/snippets/resources extra_tests/not_impl.py + +Lib/_sysconfig_vars*.json +Lib/site-packages/* +!Lib/site-packages/README.txt +Lib/test/data/* +!Lib/test/data/README +cpython/ + diff --git a/.vscode/launch.json b/.vscode/launch.json index fa6f96c5fd8..6e00e14aff2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,18 +16,6 @@ }, "cwd": "${workspaceFolder}" }, - { - "type": "lldb", - "request": "launch", - "name": "Debug executable 'rustpython' without SSL", - "preLaunchTask": "Build RustPython Debug without SSL", - "program": "target/debug/rustpython", - "args": [], - "env": { - "RUST_BACKTRACE": "1" - }, - "cwd": "${workspaceFolder}" - }, { "type": "lldb", "request": "launch", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 18a3d6010d1..50a52aaf8be 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,28 +1,12 @@ { "version": "2.0.0", "tasks": [ - { - "label": "Build RustPython Debug without SSL", - "type": "shell", - "command": "cargo", - "args": [ - "build", - ], - "problemMatcher": [ - "$rustc", - ], - "group": { - "kind": "build", - "isDefault": true, - }, - }, { "label": "Build RustPython Debug", "type": "shell", "command": "cargo", "args": [ "build", - "--features=ssl" ], "problemMatcher": [ "$rustc", diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..b407328cffb --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,266 @@ +# GitHub Copilot Instructions for RustPython + +This document provides guidelines for working with GitHub Copilot when contributing to the RustPython project. + +## Project Overview + +RustPython is a Python 3 interpreter written in Rust, implementing Python 3.14.0+ compatibility. The project aims to provide: + +- A complete Python-3 environment entirely in Rust (not CPython bindings) +- A clean implementation without compatibility hacks +- Cross-platform support, including WebAssembly compilation +- The ability to embed Python scripting in Rust applications + +## Repository Structure + +- `src/` - Top-level code for the RustPython binary +- `vm/` - The Python virtual machine implementation + - `builtins/` - Python built-in types and functions + - `stdlib/` - Essential standard library modules implemented in Rust, required to run the Python core +- `compiler/` - Python compiler components + - `parser/` - Parser for converting Python source to AST + - `core/` - Bytecode representation in Rust structures + - `codegen/` - AST to bytecode compiler +- `Lib/` - CPython's standard library in Python (copied from CPython). **IMPORTANT**: Do not edit this directory directly; The only allowed operation is copying files from CPython. +- `derive/` - Rust macros for RustPython +- `common/` - Common utilities +- `extra_tests/` - Integration tests and snippets +- `stdlib/` - Non-essential Python standard library modules implemented in Rust (useful but not required for core functionality) +- `wasm/` - WebAssembly support +- `jit/` - Experimental JIT compiler implementation +- `pylib/` - Python standard library packaging (do not modify this directory directly - its contents are generated automatically) + +## AI Agent Rules + +**CRITICAL: Git Operations** +- NEVER create pull requests directly without explicit user permission +- NEVER push commits to remote without explicit user permission +- Always ask the user before performing any git operations that affect the remote repository +- Commits can be created locally when requested, but pushing and PR creation require explicit approval + +## Important Development Notes + +### Running Python Code + +When testing Python code, always use RustPython instead of the standard `python` command: + +```bash +# Use this instead of python script.py +cargo run -- script.py + +# For interactive REPL +cargo run + +# With specific features +cargo run --features jit + +# Release mode (recommended for better performance) +cargo run --release -- script.py +``` + +### Comparing with CPython + +When you need to compare behavior with CPython or run test suites: + +```bash +# Use python command to explicitly run CPython +python my_test_script.py + +# Run RustPython +cargo run -- my_test_script.py +``` + +### Working with the Lib Directory + +The `Lib/` directory contains Python standard library files copied from the CPython repository. Important notes: + +- These files should be edited very conservatively +- Modifications should be minimal and only to work around RustPython limitations +- Tests in `Lib/test` often use one of the following markers: + - Add a `# TODO: RUSTPYTHON` comment when modifications are made + - `unittest.skip("TODO: RustPython ")` + - `unittest.expectedFailure` with `# TODO: RUSTPYTHON ` comment + +### Clean Build + +When you modify bytecode instructions, a full clean is required: + +```bash +rm -r target/debug/build/rustpython-* && find . | grep -E "\.pyc$" | xargs rm -r +``` + +### Testing + +```bash +# Run Rust unit tests +cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher + +# Run Python snippets tests (debug mode recommended for faster compilation) +cargo run -- extra_tests/snippets/builtin_bytes.py + +# Run all Python snippets tests with pytest +cd extra_tests +pytest -v + +# Run the Python test module (release mode recommended for better performance) +cargo run --release -- -m test ${TEST_MODULE} +cargo run --release -- -m test test_unicode # to test test_unicode.py + +# Run the Python test module with specific function +cargo run --release -- -m test test_unicode -k test_unicode_escape +``` + +**Note**: For `extra_tests/snippets` tests, use debug mode (`cargo run`) as compilation is faster. For `unittest` (`-m test`), use release mode (`cargo run --release`) for better runtime performance. + +### Determining What to Implement + +Run `./scripts/whats_left.py` to get a list of unimplemented methods, which is helpful when looking for contribution opportunities. + +## Coding Guidelines + +### Rust Code + +- Follow the default rustfmt code style (`cargo fmt` to format) +- **IMPORTANT**: Always run clippy to lint code (`cargo clippy`) before completing tasks. Fix any warnings or lints that are introduced by your changes +- Follow Rust best practices for error handling and memory management +- Use the macro system (`pyclass`, `pymodule`, `pyfunction`, etc.) when implementing Python functionality in Rust + +#### Comments + +- Do not delete or rewrite existing comments unless they are factually wrong or directly contradict the new code. +- Do not add decorative section separators (e.g. `// -----------`, `// ===`, `/* *** */`). Use `///` doc-comments or short `//` comments only when they add value. + +#### Avoid Duplicate Code in Branches + +When branches differ only in a value but share common logic, extract the differing value first, then call the common logic once. + +**Bad:** +```rust +let result = if condition { + let msg = format!("message A: {x}"); + some_function(msg, shared_arg) +} else { + let msg = format!("message B"); + some_function(msg, shared_arg) +}; +``` + +**Good:** +```rust +let msg = if condition { + format!("message A: {x}") +} else { + format!("message B") +}; +let result = some_function(msg, shared_arg); +``` + +### Python Code + +- **IMPORTANT**: In most cases, Python code should not be edited. Bug fixes should be made through Rust code modifications only +- Follow PEP 8 style for custom Python code +- Use ruff for linting Python code +- Minimize modifications to CPython standard library files + +## Integration Between Rust and Python + +The project provides several mechanisms for integration: + +- `pymodule` macro for creating Python modules in Rust +- `pyclass` macro for implementing Python classes in Rust +- `pyfunction` macro for exposing Rust functions to Python +- `PyObjectRef` and other types for working with Python objects in Rust + +## Common Patterns + +### Implementing a Python Module in Rust + +```rust +#[pymodule] +mod mymodule { + use rustpython_vm::prelude::*; + + #[pyfunction] + fn my_function(value: i32) -> i32 { + value * 2 + } + + #[pyattr] + #[pyclass(name = "MyClass")] + #[derive(Debug, PyPayload)] + struct MyClass { + value: usize, + } + + #[pyclass] + impl MyClass { + #[pymethod] + fn get_value(&self) -> usize { + self.value + } + } +} +``` + +### Adding a Python Module to the Interpreter + +```rust +vm.add_native_module( + "my_module_name".to_owned(), + Box::new(my_module::make_module), +); +``` + +## Building for Different Targets + +### WebAssembly + +```bash +# Build for WASM +cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release +``` + +### JIT Support + +```bash +# Enable JIT support +cargo run --features jit +``` + +### Linux Build and Debug on macOS + +See the "Testing on Linux from macOS" section in [DEVELOPMENT.md](DEVELOPMENT.md#testing-on-linux-from-macos). + +### Building venvlauncher (Windows) + +See DEVELOPMENT.md "CPython Version Upgrade Checklist" section. + +**IMPORTANT**: All 4 venvlauncher binaries use the same source code. Do NOT add multiple `[[bin]]` entries to Cargo.toml. Build once and copy with different names. + +## Test Code Modification Rules + +**CRITICAL: Test code modification restrictions** +- NEVER comment out or delete any test code lines except for removing `@unittest.expectedFailure` decorators and upper TODO comments +- NEVER modify test assertions, test logic, or test data +- When a test cannot pass due to missing language features, keep it as expectedFailure and document the reason +- The only acceptable modifications to test files are: + 1. Removing `@unittest.expectedFailure` decorators and the upper TODO comments when tests actually pass + 2. Adding `@unittest.expectedFailure` decorators when tests cannot be fixed + +**Examples of FORBIDDEN modifications:** +- Commenting out test lines +- Changing test assertions +- Modifying test data or expected results +- Removing test logic + +**Correct approach when tests fail due to unsupported syntax:** +- Keep the test as `@unittest.expectedFailure` +- Document that it requires PEP 695 support +- Focus on tests that can be fixed through Rust code changes only + +## Documentation + +- Check the [architecture document](/architecture/architecture.md) for a high-level overview +- Read the [development guide](/DEVELOPMENT.md) for detailed setup instructions +- Generate documentation with `cargo doc --no-deps --all` +- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/) diff --git a/Cargo.lock b/Cargo.lock index 140f078ff53..9b6f7939c85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,18 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" - -[[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "adler32" @@ -20,32 +14,53 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.3.4", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] [[package]] -name = "android-tzdata" -version = "0.1.1" +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" @@ -57,19 +72,66 @@ dependencies = [ ] [[package]] -name = "ansi_term" -version = "0.12.1" +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ - "winapi", + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", ] [[package]] name = "anyhow" -version = "1.0.69" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "approx" @@ -81,10 +143,19 @@ dependencies = [ ] [[package]] -name = "arrayvec" -version = "0.7.2" +name = "ar_archive_writer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "ascii" @@ -92,37 +163,179 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "asn1-rs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" dependencies = [ - "autocfg", + "bytemuck", ] [[package]] -name = "atty" -version = "0.2.14" +name = "attribute-derive" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +checksum = "05832cdddc8f2650cc2cc187cc2e952b8c133a48eb055f35211f61ee81502d77" dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", + "attribute-derive-macro", + "derive-where", + "manyhow", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7cdbbd4bd005c5d3e2e9c885e6fa575db4f4a3572335b974d8db853b6beb61" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn", ] [[package]] name = "autocfg" -version = "1.1.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-lc-fips-sys" +version = "0.13.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ed8cd42adddefbdb8507fb7443fa9b666631078616b78f70ed22117b5c27d90" +dependencies = [ + "bindgen 0.72.1", + "cc", + "cmake", + "dunce", + "fs_extra", + "regex", +] + +[[package]] +name = "aws-lc-rs" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +dependencies = [ + "aws-lc-fips-sys", + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] [[package]] name = "base64" -version = "0.13.1" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.11.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bindgen" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.11.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] [[package]] name = "bitflags" @@ -132,9 +345,21 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bitflagset" +version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "64b6ee310aa7af14142c8c9121775774ff601ae055ed98ba7fac96098bcde1b9" +dependencies = [ + "num-integer", + "num-traits", + "radium", + "ref-cast", +] [[package]] name = "blake2" @@ -147,64 +372,69 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ "generic-array", ] [[package]] name = "bstr" -version = "0.2.17" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ - "lazy_static 1.4.0", "memchr", "regex-automata", + "serde", ] [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +dependencies = [ + "allocator-api2", +] [[package]] -name = "byteorder" -version = "1.4.3" +name = "bytemuck" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] -name = "bzip2" -version = "0.4.4" +name = "bytes" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" +name = "bzip2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ - "cc", - "libc", - "pkg-config", + "libbz2-rs-sys", ] [[package]] name = "caseless" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f" +checksum = "8b6fd507454086c8edfd769ca6ada439193cdb209c7681712ef6275cccbfe5d8" dependencies = [ - "regex", "unicode-normalization", ] @@ -214,78 +444,213 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" -version = "1.0.79" +version = "1.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cexpr" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.9", ] [[package]] name = "clap" -version = "2.34.0" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim", - "textwrap 0.11.0", - "unicode-width", - "vec_map", + "clap_builder", ] +[[package]] +name = "clap_builder" +version = "4.5.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" + [[package]] name = "clipboard-win" -version = "4.5.0" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" dependencies = [ "error-code", - "str-buf", - "winapi", ] [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "cmake" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" dependencies = [ - "termcolor", - "unicode-width", + "cc", +] + +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", ] [[package]] name = "console" -version = "0.15.5" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ "encode_unicode", - "lazy_static 1.4.0", "libc", - "windows-sys 0.42.0", + "once_cell", + "windows-sys 0.59.0", ] [[package]] @@ -299,16 +664,32 @@ dependencies = [ ] [[package]] -name = "convert_case" -version = "0.4.0" +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -316,95 +697,135 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] [[package]] -name = "cpython" -version = "0.7.1" +name = "cranelift" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3052106c29da7390237bc2310c1928335733b286287754ea85e6093d2495280e" +checksum = "1cc3f22b5e916179fc510801c078f0610a467939cc2809bf5eb8a06fde4aea69" dependencies = [ - "libc", - "num-traits", - "paste", - "python3-sys", + "cranelift-codegen", + "cranelift-frontend", + "cranelift-module", ] [[package]] -name = "cranelift" -version = "0.88.2" +name = "cranelift-assembler-x64" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1b0c164043c16a8ece6813eef609ac2262a32a0bb0f5ed6eecf5d7bfb79ba8" +checksum = "40630d663279bc855bff805d6f5e8a0b6a1867f9df95b010511ac6dc894e9395" dependencies = [ - "cranelift-codegen", - "cranelift-frontend", + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.129.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee6aec5ceb55e5fdbcf7ef677d7c7195531360ff181ce39b2b31df11d57305f" +dependencies = [ + "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52056f6d0584484b57fa6c1a65c1fcb15f3780d8b6a758426d9e3084169b2ddd" +checksum = "9a92d78cc3f087d7e7073828f08d98c7074a3a062b6b29a1b7783ce74305685e" dependencies = [ "cranelift-entity", ] +[[package]] +name = "cranelift-bitset" +version = "0.129.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edcc73d756f2e0d7eda6144fe64a2bc69c624de893cb1be51f1442aed77881d2" +dependencies = [ + "wasmtime-internal-core", +] + [[package]] name = "cranelift-codegen" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fed94c8770dc25d01154c3ffa64ed0b3ba9d583736f305fed7beebe5d9cf74" +checksum = "683d94c2cd0d73b41369b88da1129589bc3a2d99cf49979af1d14751f35b7a1b" dependencies = [ - "arrayvec", "bumpalo", + "cranelift-assembler-x64", "cranelift-bforest", + "cranelift-bitset", "cranelift-codegen-meta", "cranelift-codegen-shared", + "cranelift-control", "cranelift-entity", "cranelift-isle", + "gimli", + "hashbrown 0.15.5", + "libm", "log", "regalloc2", + "rustc-hash", + "serde", "smallvec", "target-lexicon", + "wasmtime-internal-core", ] [[package]] name = "cranelift-codegen-meta" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c451b81faf237d11c7e4f3165eeb6bac61112762c5cfe7b4c0fb7241474358f" +checksum = "235da0e52ee3a0052d0e944c3470ff025b1f4234f6ec4089d3109f2d2ffa6cbd" dependencies = [ + "cranelift-assembler-x64-meta", "cranelift-codegen-shared", + "cranelift-srcgen", + "heck", ] [[package]] name = "cranelift-codegen-shared" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c940133198426d26128f08be2b40b0bd117b84771fd36798969c4d712d81fc" +checksum = "20c07c6c440bd1bf920ff7597a1e743ede1f68dcd400730bd6d389effa7662af" + +[[package]] +name = "cranelift-control" +version = "0.129.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8797c022e02521901e1aee483dea3ed3c67f2bf0a26405c9dd48e8ee7a70944b" +dependencies = [ + "arbitrary", +] [[package]] name = "cranelift-entity" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87a0f1b2fdc18776956370cf8d9b009ded3f855350c480c1c52142510961f352" +checksum = "59d8e72637246edd2cba337939850caa8b201f6315925ec4c156fdd089999699" +dependencies = [ + "cranelift-bitset", + "wasmtime-internal-core", +] [[package]] name = "cranelift-frontend" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34897538b36b216cc8dd324e73263596d51b8cf610da6498322838b2546baf8a" +checksum = "4c31db0085c3dfa131e739c3b26f9f9c84d69a9459627aac1ac4ef8355e3411b" dependencies = [ "cranelift-codegen", "log", @@ -414,18 +835,19 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b2629a569fae540f16a76b70afcc87ad7decb38dc28fa6c648ac73b51e78470" +checksum = "524d804c1ebd8c542e6f64e71aa36934cec17c5da4a9ae3799796220317f5d23" [[package]] name = "cranelift-jit" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625be33ce54cf906c408f5ad9d08caa6e2a09e52d05fd0bd1bd95b132bfbba73" +checksum = "02ca12808d5c1ccf40cb02493a8f1790358f230867fe37735e9af8b76a2262cb" dependencies = [ "anyhow", "cranelift-codegen", + "cranelift-control", "cranelift-entity", "cranelift-module", "cranelift-native", @@ -433,60 +855,67 @@ dependencies = [ "log", "region", "target-lexicon", - "windows-sys 0.36.1", + "wasmtime-internal-jit-icache-coherence", + "windows-sys 0.61.2", ] [[package]] name = "cranelift-module" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883f8d42e07fd6b283941688f6c41a9e3b97fbf2b4ddcfb2756e675b86dc5edb" +checksum = "4d92fca47132ffc3de8783e82a577a2c8aedf85d1e12b92d08863d9af8a76bd4" dependencies = [ "anyhow", "cranelift-codegen", + "cranelift-control", ] [[package]] name = "cranelift-native" -version = "0.88.2" +version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20937dab4e14d3e225c5adfc9c7106bafd4ac669bdb43027b911ff794c6fb318" +checksum = "dc9598f02540e382e1772416eba18e93c5275b746adbbf06ac1f3cf149415270" dependencies = [ "cranelift-codegen", "libc", "target-lexicon", ] +[[package]] +name = "cranelift-srcgen" +version = "0.129.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d953932541249c91e3fa70a75ff1e52adc62979a2a8132145d4b9b3e6d1a9b6a" + [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "criterion" -version = "0.3.6" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ - "atty", + "alloca", + "anes", "cast", + "ciborium", "clap", "criterion-plot", - "csv", - "itertools 0.10.5", - "lazy_static 1.4.0", + "itertools 0.13.0", "num-traits", "oorandom", + "page_size", "plotters", "rayon", "regex", "serde", - "serde_cbor", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -494,156 +923,133 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.4.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", - "itertools 0.10.5", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" -dependencies = [ - "cfg-if", - "crossbeam-utils", + "itertools 0.13.0", ] [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset 0.7.1", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", ] [[package]] -name = "csv" -version = "1.2.0" +name = "csv-core" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af91f40b7355f82b0a891f50e70399475945bb0b0da4f1700ce60761c9d3e359" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", + "memchr", ] [[package]] -name = "csv-core" -version = "0.1.10" +name = "data-encoding" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "memchr", + "const-oid", + "der_derive", + "flagset", + "pem-rfc7468 0.7.0", + "zeroize", ] [[package]] -name = "cxx" -version = "1.0.91" +name = "der-parser" +version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", ] [[package]] -name = "cxx-build" -version = "1.0.91" +name = "der_derive" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ - "cc", - "codespan-reporting", - "once_cell", "proc-macro2", "quote", - "scratch", - "syn 1.0.107", + "syn", ] [[package]] -name = "cxxbridge-flags" -version = "1.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.91" +name = "deranged" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", + "powerfmt", ] [[package]] -name = "derive_more" -version = "0.99.17" +name = "derive-where" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" dependencies = [ - "convert_case", "proc-macro2", "quote", - "rustc_version", - "syn 1.0.107", + "syn", ] [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -671,47 +1077,52 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dns-lookup" -version = "1.0.8" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53ecafc952c4528d9b51a458d1a8904b81783feff9fde08ab6ed2545ff396872" +checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" dependencies = [ "cfg-if", "libc", "socket2", - "winapi", + "windows-sys 0.60.2", ] [[package]] -name = "dyn-clone" -version = "1.0.10" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] -name = "either" -version = "1.8.1" +name = "dyn-clone" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] -name = "embed-doc-image" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af36f591236d9d822425cb6896595658fa558fcebf5ee8accac1d4b92c47166e" -dependencies = [ - "base64", - "proc-macro2", - "quote", - "syn 1.0.107", -] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "encode_unicode" -version = "0.3.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "endian-type" @@ -720,46 +1131,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] -name = "env_logger" -version = "0.9.3" +name = "env_filter" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" dependencies = [ - "atty", "log", - "termcolor", + "regex", ] [[package]] -name = "errno" -version = "0.3.1" +name = "env_home" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" + +[[package]] +name = "env_logger" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ - "cc", "libc", + "windows-sys 0.61.2", ] [[package]] name = "error-code" -version = "2.3.1" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f18991e7bf11e7ffee451b5318b5c1a73c52d0d0ada6e5a3017c8c1ced6a21" -dependencies = [ - "libc", - "str-buf", -] +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" [[package]] name = "exitcode" @@ -767,17 +1187,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "fd-lock" -version = "3.0.12" +version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ae6b3d9530211fb3b12a95374b8b0823be812f53d09e18c5675c0146b09642" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" + +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flame" version = "0.2.2" @@ -793,20 +1231,20 @@ dependencies = [ [[package]] name = "flamer" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b732da54fd4ea34452f2431cf464ac7be94ca4b339c9cd3d3d12eb06fe7aab" +checksum = "7693d9dd1ec1c54f52195dfe255b627f7cec7da33b679cd56de949e662b3db10" dependencies = [ "flame", "quote", - "syn 1.0.107", + "syn", ] [[package]] name = "flamescope" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cc29a6c0dfa26d3a0e80021edda5671eeed79381130897737cdd273ea18909" +checksum = "8168cbad48fdda10be94de9c6319f9e8ac5d3cf0a1abda1864269dfcca3d302a" dependencies = [ "flame", "indexmap", @@ -816,13 +1254,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ - "crc32fast", - "libz-sys", "miniz_oxide", + "zlib-rs 0.6.0", ] [[package]] @@ -831,6 +1268,18 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -847,103 +1296,172 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] -name = "fxhash" -version = "0.2.1" +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "byteorder", + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", ] [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b6d1e2f75c16bfbcd0f95d84f99858a6e2f885c2287d1f5c3a96e8444a34b4" +dependencies = [ + "attribute-derive", + "quote", + "syn", +] + +[[package]] +name = "get-size2" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49cf31a6d70300cf81461098f7797571362387ef4bf85d32ac47eaa59b3a5a1a" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.16.1", + "ordermap", + "smallvec", +] + [[package]] name = "gethostname" -version = "0.2.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "libc", - "winapi", + "rustix", + "windows-link", ] [[package]] name = "getopts" -version = "0.2.21" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ "unicode-width", ] [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", ] [[package]] -name = "glob" -version = "0.3.1" +name = "getrandom" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] [[package]] -name = "half" -version = "1.8.2" +name = "gimli" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" +dependencies = [ + "fnv", + "hashbrown 0.16.1", + "indexmap", + "stable_deref_trait", +] [[package]] -name = "hashbrown" -version = "0.12.3" +name = "glob" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] -name = "heck" -version = "0.4.1" +name = "half" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "hashbrown" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "libc", + "foldhash 0.1.5", ] [[package]] -name = "hermit-abi" -version = "0.2.6" +name = "hashbrown" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "libc", + "foldhash 0.2.0", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -957,124 +1475,212 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "winapi", + "windows-core", ] [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] name = "indexmap" -version = "1.9.3" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ - "autocfg", - "hashbrown", + "equivalent", + "hashbrown 0.16.1", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", ] [[package]] name = "insta" -version = "1.33.0" +version = "1.46.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa511b2e298cd49b1856746f6bb73e17036bcd66b25f5e92cdcdbec9bd75686" +checksum = "e82db8c87c7f1ccecb34ce0c24399b8a73081427f3c7c50a5d597925356115e4" dependencies = [ "console", - "lazy_static 1.4.0", - "linked-hash-map", + "once_cell", "similar", - "yaml-rust", + "tempfile", ] [[package]] -name = "io-lifetimes" -version = "1.0.10" +name = "interpolator" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" [[package]] name = "is-macro" -version = "0.3.0" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4467ed1321b310c2625c5aa6c1b1ffc5de4d9e42668cf697a08fb033ee8265e" +checksum = "1d57a3e447e24c22647738e4607f1df1e0ec6f72e16182c4cd199f647cdfb0e4" dependencies = [ - "Inflector", - "pmutil 0.6.1", + "heck", "proc-macro2", "quote", - "syn 2.0.32", + "syn", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" -version = "0.10.5" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] [[package]] name = "itertools" -version = "0.11.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jiff" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-static" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] -name = "keccak" -version = "0.1.3" +name = "junction" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3afef3b6eff9ce9d8ff9b3601125eec7f0c8cbac7abd14f355d053fa56c98768" +checksum = "8cfc352a66ba903c23239ef51e809508b6fc2b0f90e3476ac7a9ff47e863ae95" dependencies = [ - "cpufeatures", + "scopeguard", + "windows-sys 0.61.2", ] [[package]] -name = "lalrpop-util" -version = "0.20.0" +name = "keccak" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures", +] [[package]] name = "lazy_static" @@ -1084,51 +1690,58 @@ checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-parse-float" -version = "0.8.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ "lexical-parse-integer", "lexical-util", - "static_assertions", ] [[package]] name = "lexical-parse-integer" -version = "0.8.6" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "lexical-util" -version = "0.8.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" -dependencies = [ - "static_assertions", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexopt" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libffi" -version = "3.1.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb06d5b4c428f3cd682943741c39ed4157ae989fffe1094a08eaf7c4014cf60" +checksum = "0498fe5655f857803e156523e644dcdcdc3b3c7edda42ea2afdae2e09b2db87b" dependencies = [ "libc", "libffi-sys", @@ -1136,133 +1749,158 @@ dependencies = [ [[package]] name = "libffi-sys" -version = "2.1.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c6f11e063a27ffe040a9d15f0b661bf41edc2383b7ae0e0ad5a7e7d53d9da3" +checksum = "71d4f1d4ce15091955144350b75db16a96d4a63728500122706fb4d29a26afbb" dependencies = [ "cc", ] [[package]] -name = "libsqlite3-sys" -version = "0.25.2" +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "liblzma" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" dependencies = [ "cc", + "libc", "pkg-config", - "vcpkg", ] [[package]] -name = "libz-sys" -version = "1.1.8" +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "cc", + "bitflags 2.11.0", "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" +dependencies = [ + "cc", "pkg-config", "vcpkg", ] [[package]] -name = "link-cplusplus" -version = "1.0.8" +name = "libz-rs-sys" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" dependencies = [ - "cc", + "zlib-rs 0.5.5", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.17" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lz4_flex" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" dependencies = [ "twox-hash", ] [[package]] name = "mac_address" -version = "1.1.4" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b238e3235c8382b7653c6408ed1b08dd379bdb9fdf990fb0bbae3db2cc0ae963" +checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" dependencies = [ - "nix 0.23.2", + "nix 0.29.0", "winapi", ] [[package]] -name = "mach" -version = "0.3.2" +name = "mach2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" dependencies = [ "libc", ] -[[package]] -name = "malachite" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "220cb36c52aa6eff45559df497abe0e2a4c1209f92279a746a399f622d7b95c7" -dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", -] - [[package]] name = "malachite-base" -version = "0.4.4" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6538136c5daf04126d6be4899f7fe4879b7f8de896dd1b4210fe6de5b94f2555" +checksum = "a8b6f86fdbb1eb9955946be91775239dfcb0acdb1a51bb07d5fc9b8c854f5ccd" dependencies = [ - "itertools 0.11.0", + "hashbrown 0.16.1", + "itertools 0.14.0", + "libm", "ryu", ] [[package]] name = "malachite-bigint" -version = "0.2.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17703a19c80bbdd0b7919f0f104f3b0597f7de4fc4e90a477c15366a5ba03faa" +checksum = "67fcd6e504ffc67db2b3c6d5e90e08054646e2b04f42115a5460bf1c1e37d3bc" dependencies = [ - "derive_more", - "malachite", + "malachite-base", + "malachite-nz", "num-integer", "num-traits", "paste", @@ -1270,26 +1908,50 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.4.4" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0b05577b7a3f09433106460b10304f97fc572f0baabf6640e6cb1e23f5fc52" +checksum = "0197a2f5cfee19d59178e282985c6ca79a9233e26a2adcf40acb693896aa09f6" dependencies = [ - "embed-doc-image", - "itertools 0.11.0", + "itertools 0.14.0", + "libm", "malachite-base", + "wide", ] [[package]] name = "malachite-q" -version = "0.4.4" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1cfdb4016292e6acd832eaee261175f3af8bbee62afeefe4420ebce4c440cb5" +checksum = "be2add95162aede090c48f0ee51bea7d328847ce3180aa44588111f846cc116b" dependencies = [ - "itertools 0.11.0", + "itertools 0.14.0", "malachite-base", "malachite-nz", ] +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "maplit" version = "1.0.2" @@ -1304,62 +1966,61 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" -version = "0.5.8" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" dependencies = [ "libc", ] [[package]] name = "memoffset" -version = "0.6.5" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] [[package]] -name = "memoffset" -version = "0.7.1" +name = "minimal-lexical" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ - "adler", + "adler2", + "simd-adler32", ] [[package]] name = "mt19937" -version = "2.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ca7f22ed370d5991a9caec16a83187e865bc8a532f889670337d5a5689e3a1" +checksum = "56bc7ea7924ea1a79a9e817d0483e39295424cf2b1276cf2b968f9a6c9b63b54" dependencies = [ - "rand_core", + "rand_core 0.9.5", ] [[package]] @@ -1373,118 +2034,157 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 1.3.2", - "cc", + "bitflags 2.11.0", "cfg-if", + "cfg_aliases", "libc", - "memoffset 0.6.5", + "memoffset", ] [[package]] name = "nix" -version = "0.26.2" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.11.0", "cfg-if", + "cfg_aliases", "libc", - "memoffset 0.7.1", - "pin-utils", - "static_assertions", + "memoffset", ] [[package]] -name = "nom8" -version = "0.2.0" +name = "nom" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", ] [[package]] name = "num-complex" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] [[package]] name = "num_enum" -version = "0.5.9" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d829733185c1ca374f17e52b762f24f535ec625d2cc1f070e34c8a9068f341b" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" dependencies = [ "num_enum_derive", + "rustversion", ] [[package]] name = "num_enum_derive" -version = "0.5.9" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2be1598bf1c313dcdd12092e3f1920f463462525a21b7b4e11b4168353d0123e" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ - "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.107", + "syn", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "oorandom" -version = "11.1.3" +version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.62" +version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.11.0", "cfg-if", "foreign-types", "libc", @@ -1495,35 +2195,35 @@ dependencies = [ [[package]] name = "openssl-macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn", ] [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-src" -version = "300.2.1+3.2.0" +version = "300.5.4+3.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fe476c29791a5ca0d1273c697e96085bbabbbea2ef7afd5617e78a4b40332d3" +checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", @@ -1538,11 +2238,20 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978aa494585d3ca4ad74929863093e87cac9790d81fe7aba2b3dc2890643a0fc" +[[package]] +name = "ordermap" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfa78c92071bbd3628c22b1a964f7e0eb201dc1456555db072beb1662ecd6715" +dependencies = [ + "indexmap", +] + [[package]] name = "page_size" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ "libc", "winapi", @@ -1550,9 +2259,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", @@ -1560,178 +2269,402 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +version = "0.9.12" +source = "git+https://github.com/youknowone/parking_lot?branch=rustpython#4392edbe879acc9c0dd94eda53d2205d3ab912c9" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.5.18", "smallvec", - "windows-sys 0.45.0", + "windows-link", ] [[package]] name = "paste" -version = "1.0.12" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + +[[package]] +name = "phf" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared 0.11.3", +] [[package]] name = "phf" -version = "0.11.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_shared", + "phf_macros", + "phf_shared 0.13.1", + "serde", ] [[package]] name = "phf_codegen" -version = "0.11.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", ] [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.5", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "phf_shared", - "rand", + "siphasher", ] [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" dependencies = [ "siphasher", ] [[package]] -name = "pin-utils" -version = "0.1.0" +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "pmutil" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "portable-atomic" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "psm" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +dependencies = [ + "ar_archive_writer", + "cc", +] [[package]] -name = "pkg-config" -version = "0.3.26" +name = "pymath" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "bc10e50b7a1f2cc3887e983721cb51fc7574be0066c84bff3ef9e5c096e8d6d5" +dependencies = [ + "libc", + "libm", + "malachite-bigint", + "num-complex", + "num-integer", + "num-traits", +] [[package]] -name = "plotters" -version = "0.3.4" +name = "pyo3" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" +checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", ] [[package]] -name = "plotters-backend" -version = "0.3.4" +name = "pyo3-build-config" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" +checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" +dependencies = [ + "target-lexicon", +] [[package]] -name = "plotters-svg" -version = "0.3.3" +name = "pyo3-ffi" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" +checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" dependencies = [ - "plotters-backend", + "libc", + "pyo3-build-config", ] [[package]] -name = "pmutil" -version = "0.5.3" +name = "pyo3-macros" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004" +checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" dependencies = [ "proc-macro2", + "pyo3-macros-backend", "quote", - "syn 1.0.107", + "syn", ] [[package]] -name = "pmutil" -version = "0.6.1" +name = "pyo3-macros-backend" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" +checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" dependencies = [ + "heck", "proc-macro2", + "pyo3-build-config", "quote", - "syn 2.0.32", + "syn", ] [[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "proc-macro-crate" -version = "1.3.0" +name = "quote" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66618389e4ec1c7afe67d51a9bf34ff9236480f8d51e7489b7d5ab0303c13f34" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ - "once_cell", - "toml_edit", + "proc-macro2", ] [[package]] -name = "proc-macro2" -version = "1.0.66" +name = "quote-use" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" dependencies = [ - "unicode-ident", + "quote", + "quote-use-macros", ] [[package]] -name = "puruspe" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7765e19fb2ba6fd4373b8d90399f5321683ea7c11b598c6bbaa3a72e9c83b8" - -[[package]] -name = "python3-sys" -version = "0.7.1" +name = "quote-use-macros" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f8b50d72fb3015735aa403eebf19bbd72c093bfeeae24ee798be5f2f1aab52" +checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" dependencies = [ - "libc", - "regex", + "proc-macro-utils", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "quote" -version = "1.0.33" +name = "r-efi" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" -dependencies = [ - "proc-macro2", -] +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "radium" -version = "0.7.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +checksum = "1775bc532a9bfde46e26eba441ca1171b91608d14a3bae71fea371f18a00cffe" +dependencies = [ + "cfg-if", +] [[package]] name = "radix_trie" @@ -1750,8 +2683,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -1761,7 +2704,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1770,14 +2723,23 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] name = "rayon" -version = "1.6.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -1785,14 +2747,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -1803,273 +2763,454 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.11.0", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "regalloc2" -version = "0.3.2" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d43a209257d978ef079f3d446331d0f1794f5e0fc19b306a199983857833a779" +checksum = "08effbc1fa53aaebff69521a5c05640523fab037b34a4a2c109506bc938246fa" dependencies = [ - "fxhash", + "allocator-api2", + "bumpalo", + "hashbrown 0.15.5", "log", - "slice-group-by", + "rustc-hash", "smallvec", ] [[package]] name = "regex" -version = "1.7.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", + "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.1.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "region" -version = "2.2.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" dependencies = [ "bitflags 1.3.2", "libc", - "mach", - "winapi", + "mach2", + "windows-sys 0.52.0", ] [[package]] name = "result-like" -version = "0.4.5" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b80fe0296795a96913be20558326b797a187bb3986ce84ed82dee0fb7414428" +checksum = "bffa194499266bd8a1ac7da6ac7355aa0f81ffa1a5db2baaf20dd13854fd6f4e" dependencies = [ "result-like-derive", ] [[package]] name = "result-like-derive" -version = "0.4.5" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a29c8a4ac7839f1dcb8b899263b501e0d6932f210300c8a0d271323727b35c1" +checksum = "01d3b03471c9700a3a6bd166550daaa6124cb4a146ea139fb028e4edaa8f4277" dependencies = [ - "pmutil 0.5.3", + "pmutil", "proc-macro2", "quote", - "syn 1.0.107", - "syn-ext", + "syn", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "ruff_python_ast" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" +dependencies = [ + "aho-corasick", + "bitflags 2.11.0", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thiserror 2.0.18", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" +dependencies = [ + "bitflags 2.11.0", + "bstr", + "compact_str", + "get-size2", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "rustc-hash", + "static_assertions", + "unicode-ident", + "unicode-normalization", + "unicode_names2 1.3.0", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" +dependencies = [ + "itertools 0.14.0", + "ruff_source_file", + "ruff_text_size", + "unicode-ident", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.0" +source = "git+https://github.com/astral-sh/ruff.git?rev=e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675#e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" +dependencies = [ + "get-size2", ] [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] -name = "rustc_version" -version = "0.4.0" +name = "rusticata-macros" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "semver", + "nom", ] [[package]] name = "rustix" -version = "0.37.11" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.11.0", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted 0.9.0", ] [[package]] name = "rustpython" -version = "0.3.0" +version = "0.5.0" dependencies = [ - "atty", "cfg-if", - "clap", - "cpython", "criterion", "dirs-next", "env_logger", "flame", "flamescope", + "lexopt", "libc", "log", - "python3-sys", + "pyo3", + "ruff_python_parser", "rustpython-compiler", - "rustpython-parser", "rustpython-pylib", "rustpython-stdlib", "rustpython-vm", "rustyline", -] - -[[package]] -name = "rustpython-ast" -version = "0.3.1" -source = "git+https://github.com/RustPython/Parser.git?rev=29c4728dbedc7e69cc2560b9b34058bbba9b1303#29c4728dbedc7e69cc2560b9b34058bbba9b1303" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-literal", - "rustpython-parser-core", - "static_assertions", + "winresource", ] [[package]] name = "rustpython-codegen" -version = "0.3.0" +version = "0.5.0" dependencies = [ "ahash", - "bitflags 2.4.0", + "bitflags 2.11.0", "indexmap", "insta", - "itertools 0.11.0", + "itertools 0.14.0", "log", + "malachite-bigint", + "memchr", "num-complex", "num-traits", - "rustpython-ast", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "rustpython-compiler-core", - "rustpython-parser", - "rustpython-parser-core", + "rustpython-literal", + "rustpython-wtf8", + "thiserror 2.0.18", + "unicode_names2 2.0.0", ] [[package]] name = "rustpython-common" -version = "0.3.0" +version = "0.5.0" dependencies = [ "ascii", - "bitflags 2.4.0", - "bstr", + "bitflags 2.11.0", "cfg-if", - "itertools 0.11.0", + "getrandom 0.3.4", + "itertools 0.14.0", "libc", "lock_api", "malachite-base", "malachite-bigint", "malachite-q", + "nix 0.30.1", "num-complex", "num-traits", - "once_cell", "parking_lot", "radium", - "rand", - "rustpython-format", + "rustpython-literal", + "rustpython-wtf8", "siphasher", - "volatile", + "unicode_names2 2.0.0", "widestring", + "windows-sys 0.61.2", ] [[package]] name = "rustpython-compiler" -version = "0.3.0" +version = "0.5.0" dependencies = [ + "ruff_python_ast", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", - "rustpython-parser", + "thiserror 2.0.18", ] [[package]] name = "rustpython-compiler-core" -version = "0.3.0" +version = "0.5.0" dependencies = [ - "bitflags 2.4.0", - "itertools 0.11.0", + "bitflags 2.11.0", + "bitflagset", + "itertools 0.14.0", "lz4_flex", "malachite-bigint", "num-complex", - "rustpython-parser-core", - "serde", + "ruff_source_file", + "rustpython-wtf8", +] + +[[package]] +name = "rustpython-compiler-source" +version = "0.5.0+deprecated" +dependencies = [ + "ruff_source_file", + "ruff_text_size", ] [[package]] name = "rustpython-derive" -version = "0.3.0" +version = "0.5.0" dependencies = [ "rustpython-compiler", "rustpython-derive-impl", - "syn 1.0.107", + "syn", ] [[package]] name = "rustpython-derive-impl" -version = "0.3.0" +version = "0.5.0" dependencies = [ - "itertools 0.11.0", + "itertools 0.14.0", "maplit", - "once_cell", "proc-macro2", "quote", "rustpython-compiler-core", "rustpython-doc", - "rustpython-parser-core", - "syn 1.0.107", + "syn", "syn-ext", - "textwrap 0.15.2", + "textwrap", ] [[package]] name = "rustpython-doc" -version = "0.3.0" -source = "git+https://github.com/RustPython/__doc__?tag=0.3.0#8b62ce5d796d68a091969c9fa5406276cb483f79" -dependencies = [ - "once_cell", -] - -[[package]] -name = "rustpython-format" -version = "0.3.1" -source = "git+https://github.com/RustPython/Parser.git?rev=29c4728dbedc7e69cc2560b9b34058bbba9b1303#29c4728dbedc7e69cc2560b9b34058bbba9b1303" +version = "0.5.0" dependencies = [ - "bitflags 2.4.0", - "itertools 0.11.0", - "malachite-bigint", - "num-traits", - "rustpython-literal", + "phf 0.13.1", ] [[package]] name = "rustpython-jit" -version = "0.3.0" +version = "0.5.0" dependencies = [ "approx", "cranelift", @@ -2079,118 +3220,110 @@ dependencies = [ "num-traits", "rustpython-compiler-core", "rustpython-derive", - "thiserror", + "rustpython-wtf8", + "thiserror 2.0.18", ] [[package]] name = "rustpython-literal" -version = "0.3.1" -source = "git+https://github.com/RustPython/Parser.git?rev=29c4728dbedc7e69cc2560b9b34058bbba9b1303#29c4728dbedc7e69cc2560b9b34058bbba9b1303" +version = "0.5.0" dependencies = [ "hexf-parse", "is-macro", "lexical-parse-float", "num-traits", + "rand 0.9.2", + "rustpython-wtf8", "unic-ucd-category", ] -[[package]] -name = "rustpython-parser" -version = "0.3.1" -source = "git+https://github.com/RustPython/Parser.git?rev=29c4728dbedc7e69cc2560b9b34058bbba9b1303#29c4728dbedc7e69cc2560b9b34058bbba9b1303" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf", - "phf_codegen", - "rustc-hash", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.3.1" -source = "git+https://github.com/RustPython/Parser.git?rev=29c4728dbedc7e69cc2560b9b34058bbba9b1303#29c4728dbedc7e69cc2560b9b34058bbba9b1303" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.3.1" -source = "git+https://github.com/RustPython/Parser.git?rev=29c4728dbedc7e69cc2560b9b34058bbba9b1303#29c4728dbedc7e69cc2560b9b34058bbba9b1303" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustpython-pylib" -version = "0.3.0" +version = "0.5.0" dependencies = [ "glob", "rustpython-compiler-core", "rustpython-derive", ] +[[package]] +name = "rustpython-sre_engine" +version = "0.5.0" +dependencies = [ + "bitflags 2.11.0", + "criterion", + "num_enum", + "optional", + "rustpython-wtf8", +] + [[package]] name = "rustpython-stdlib" -version = "0.3.0" +version = "0.5.0" dependencies = [ "adler32", "ahash", "ascii", + "aws-lc-rs", "base64", "blake2", "bzip2", "cfg-if", + "chrono", + "constant_time_eq", "crc32fast", "crossbeam-utils", "csv-core", + "der", "digest", "dns-lookup", "dyn-clone", + "flame", "flate2", "foreign-types-shared", "gethostname", "hex", - "itertools 0.11.0", + "hmac", + "indexmap", + "itertools 0.14.0", "libc", + "liblzma", + "liblzma-sys", "libsqlite3-sys", - "libz-sys", + "libz-rs-sys", "mac_address", "malachite-bigint", "md-5", "memchr", "memmap2", "mt19937", - "nix 0.26.2", + "nix 0.30.1", "num-complex", "num-integer", "num-traits", "num_enum", - "once_cell", + "oid-registry", "openssl", "openssl-probe", "openssl-sys", "page_size", "parking_lot", "paste", - "puruspe", - "rand", - "rand_core", + "pbkdf2", + "pem-rfc7468 1.0.0", + "phf 0.13.1", + "pkcs8", + "pymath", + "rand_core 0.9.5", + "ruff_python_ast", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", + "rustix", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-platform-verifier", "rustpython-common", "rustpython-derive", "rustpython-vm", @@ -2200,7 +3333,9 @@ dependencies = [ "sha3", "socket2", "system-configuration", + "tcl-sys", "termios", + "tk-sys", "ucd", "unic-char-property", "unic-normal", @@ -2208,75 +3343,87 @@ dependencies = [ "unic-ucd-bidi", "unic-ucd-category", "unic-ucd-ident", + "unicode-bidi-mirroring", "unicode-casing", - "unicode_names2", + "unicode_names2 2.0.0", "uuid", + "webpki-roots", "widestring", - "winapi", - "windows-sys 0.52.0", - "xml-rs", + "windows-sys 0.61.2", + "x509-cert", + "x509-parser", + "xml", +] + +[[package]] +name = "rustpython-venvlauncher" +version = "0.5.0" +dependencies = [ + "windows-sys 0.61.2", ] [[package]] name = "rustpython-vm" -version = "0.3.0" +version = "0.5.0" dependencies = [ "ahash", "ascii", - "atty", - "bitflags 2.4.0", + "bitflags 2.11.0", "bstr", "caseless", "cfg-if", "chrono", + "constant_time_eq", "crossbeam-utils", + "errno", "exitcode", "flame", "flamer", - "getrandom", + "getrandom 0.3.4", "glob", "half", "hex", "indexmap", "is-macro", - "itertools 0.11.0", + "itertools 0.14.0", + "junction", "libc", + "libffi", + "libloading 0.9.0", "log", "malachite-bigint", "memchr", - "memoffset 0.6.5", - "nix 0.26.2", + "nix 0.30.1", "num-complex", "num-integer", "num-traits", "num_cpus", "num_enum", - "once_cell", "optional", "parking_lot", "paste", - "rand", + "psm", "result-like", - "rustc_version", - "rustpython-ast", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", + "rustix", "rustpython-codegen", "rustpython-common", "rustpython-compiler", "rustpython-compiler-core", "rustpython-derive", - "rustpython-format", "rustpython-jit", "rustpython-literal", - "rustpython-parser", - "rustpython-parser-core", + "rustpython-sre_engine", "rustyline", - "schannel", - "serde", - "sre-engine", + "scoped-tls", + "scopeguard", + "serde_core", "static_assertions", "strum", "strum_macros", - "thiserror", + "thiserror 2.0.18", "thread_local", "timsort", "uname", @@ -2284,28 +3431,35 @@ dependencies = [ "unic-ucd-category", "unic-ucd-ident", "unicode-casing", - "unicode_names2", "wasm-bindgen", "which", "widestring", - "windows", - "windows-sys 0.52.0", - "winreg", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustpython-wtf8" +version = "0.5.0" +dependencies = [ + "ascii", + "bstr", + "itertools 0.14.0", + "memchr", ] [[package]] name = "rustpython_wasm" -version = "0.3.0" +version = "0.5.0" dependencies = [ "console_error_panic_hook", "js-sys", + "ruff_python_parser", "rustpython-common", - "rustpython-parser", "rustpython-pylib", "rustpython-stdlib", "rustpython-vm", - "serde", "serde-wasm-bindgen", + "serde_core", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -2313,38 +3467,55 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rustyline" -version = "11.0.0" +version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfc8644681285d1fb67a467fb3021bfea306b99b4146b166a1fe3ada965eece" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.11.0", "cfg-if", "clipboard-win", - "dirs-next", "fd-lock", + "home", "libc", "log", "memchr", - "nix 0.26.2", + "nix 0.30.1", "radix_trie", - "scopeguard", "unicode-segmentation", "unicode-width", "utf8parse", - "winapi", + "windows-sys 0.60.2", ] [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" + +[[package]] +name = "safe_arch" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "1f7caad094bd561859bcd467734a720c3c1f5d1f338995351fefe2190c45efed" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] [[package]] name = "same-file" @@ -2357,82 +3528,120 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "scratch" -version = "1.0.3" +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + +[[package]] +name = "security-framework" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] [[package]] -name = "semver" -version = "1.0.16" +name = "security-framework-sys" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "serde" -version = "1.0.152" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] [[package]] name = "serde-wasm-bindgen" -version = "0.3.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "618365e8e586c22123d692b72a7d791d5ee697817b65a218cdf12a98870af0f7" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" dependencies = [ - "fnv", "js-sys", "serde", "wasm-bindgen", ] [[package]] -name = "serde_cbor" -version = "0.11.2" +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ - "half", - "serde", + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn", ] [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", - "ryu", + "memchr", "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" +dependencies = [ + "serde_core", ] [[package]] @@ -2447,10 +3656,21 @@ dependencies = [ ] [[package]] -name = "sha2" +name = "sha1" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -2459,107 +3679,128 @@ dependencies = [ [[package]] name = "sha3" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf0c33fae925bdc080598b84bc15c55e7b9a4a43b3c704da051f977469691c9" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ "digest", "keccak", ] +[[package]] +name = "shared-build" +version = "0.2.0" +source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a" +dependencies = [ + "bindgen 0.71.1", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "similar" -version = "2.2.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "siphasher" -version = "0.3.10" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" [[package]] -name = "slice-group-by" -version = "0.3.0" +name = "slab" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallvec" -version = "1.10.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.4.7" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "winapi", + "windows-sys 0.61.2", ] [[package]] -name = "sre-engine" -version = "0.4.1" +name = "spki" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a490c5c46c35dba9a6f5e7ee8e4d67e775eb2d2da0f115750b8d10e1c1ac2d28" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ - "bitflags 1.3.2", - "num_enum", - "optional", + "base64ct", + "der", ] [[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "str-buf" -version = "1.0.6" +name = "stable_deref_trait" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] -name = "strsim" -version = "0.8.0" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strum" -version = "0.24.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ "heck", "proc-macro2", "quote", - "rustversion", - "syn 1.0.107", + "syn", ] [[package]] name = "subtle" -version = "2.4.1" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "1.0.107" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -2567,41 +3808,43 @@ dependencies = [ ] [[package]] -name = "syn" -version = "2.0.32" +name = "syn-ext" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "b126de4ef6c2a628a68609dd00733766c3b015894698a438ebdf374933fc31d1" dependencies = [ "proc-macro2", "quote", - "unicode-ident", + "syn", ] [[package]] -name = "syn-ext" -version = "0.4.0" +name = "synstructure" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b86cb2b68c5b3c078cac02588bc23f3c04bb828c5d3aedd17980876ec6a7be6" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ - "syn 1.0.107", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "system-configuration" -version = "0.5.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 1.3.2", - "core-foundation", + "bitflags 2.11.0", + "core-foundation 0.9.4", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", @@ -2609,17 +3852,30 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.6" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" +checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" [[package]] -name = "termcolor" -version = "1.2.0" +name = "tcl-sys" +version = "0.2.0" +source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a" +dependencies = [ + "pkg-config", + "shared-build", +] + +[[package]] +name = "tempfile" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ - "winapi-util", + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", ] [[package]] @@ -2633,37 +3889,48 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.11.0" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + +[[package]] +name = "thiserror" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "unicode-width", + "thiserror-impl 1.0.69", ] [[package]] -name = "textwrap" -version = "0.15.2" +name = "thiserror" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] [[package]] -name = "thiserror" -version = "1.0.38" +name = "thiserror-impl" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ - "thiserror-impl", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn", ] [[package]] @@ -2679,44 +3946,49 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] name = "time" -version = "0.3.20" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ - "serde", + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", "time-core", + "time-macros", ] [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] -name = "timsort" -version = "0.1.2" +name = "time-macros" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb4fa83bb73adf1c7219f4fe4bf3c0ac5635e4e51e070fad5df745a41bedfb8" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] [[package]] -name = "tiny-keccak" -version = "2.0.2" +name = "timsort" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] +checksum = "639ce8ef6d2ba56be0383a94dd13b92138d58de44c62618303bb798fa92bdc00" [[package]] name = "tinytemplate" @@ -2730,51 +4002,99 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tk-sys" +version = "0.2.0" +source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a" +dependencies = [ + "pkg-config", + "shared-build", +] + +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ - "tinyvec_macros", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "tinyvec_macros" -version = "0.1.1" +name = "toml" +version = "0.9.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "f3afc9a848309fe1aaffaed6e1546a7a14de1f935dc9d89d32afd9a44bab7c46" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] [[package]] name = "toml_datetime" -version = "0.5.1" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] [[package]] -name = "toml_edit" -version = "0.18.1" +name = "toml_parser" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" dependencies = [ - "indexmap", - "nom8", - "toml_datetime", + "winnow", ] +[[package]] +name = "toml_writer" +version = "1.0.6+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" + [[package]] name = "twox-hash" -version = "1.6.3" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "static_assertions", -] +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typenum" -version = "1.16.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "ucd" @@ -2812,17 +4132,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-normal" version = "0.9.0" @@ -2907,174 +4216,185 @@ dependencies = [ "unic-common", ] +[[package]] +name = "unicode-bidi-mirroring" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfa6e8c60bb66d49db113e0125ee8711b7647b5579dc7f5f19c42357ed039fe" + [[package]] name = "unicode-casing" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" +checksum = "061dbb8cc7f108532b6087a0065eff575e892a4bcb503dc57323a197457cc202" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "unicode_names2" -version = "1.1.0" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +dependencies = [ + "phf 0.11.3", + "unicode_names2_generator 1.3.0", +] + +[[package]] +name = "unicode_names2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b2c0942619ae1797f999a0ce7efc6c09592ad30e68e16cdbfdcd48a98c3579" +checksum = "d189085656ca1203291e965444e7f6a2723fbdd1dd9f34f8482e79bafd8338a0" dependencies = [ - "phf", - "unicode_names2_generator", + "phf 0.11.3", + "unicode_names2_generator 2.0.0", ] [[package]] name = "unicode_names2_generator" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d0d66ab60be9799a70f8eb227ea43da7dcc47561dd9102cbadacfe0930113f7" +checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" dependencies = [ "getopts", "log", "phf_codegen", - "rand", - "time", + "rand 0.8.5", ] [[package]] -name = "utf8parse" -version = "0.2.0" +name = "unicode_names2_generator" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" +checksum = "1262662dc96937c71115228ce2e1d30f41db71a7a45d3459e98783ef94052214" +dependencies = [ + "phf_codegen", + "rand 0.8.5", +] [[package]] -name = "uuid" -version = "1.3.0" +name = "untrusted" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" -dependencies = [ - "atomic", - "getrandom", - "rand", - "uuid-macro-internal", -] +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] -name = "uuid-macro-internal" -version = "1.3.0" +name = "untrusted" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b300a878652a387d2a0de915bdae8f1a548f0c6d45e072fe2688794b656cc9" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", -] +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] -name = "vcpkg" -version = "0.2.15" +name = "utf8parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] -name = "vec_map" -version = "0.8.2" +name = "uuid" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +dependencies = [ + "atomic", + "js-sys", + "wasm-bindgen", +] [[package]] -name = "version_check" -version = "0.9.4" +name = "vcpkg" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] -name = "volatile" -version = "0.3.0" +name = "version_check" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e76fae08f03f96e166d2dfda232190638c10e0383841252416f9cfe2ae60e6" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", - "winapi", "winapi-util", ] [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasm-bindgen" -version = "0.2.84" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ - "cfg-if", - "wasm-bindgen-macro", + "wit-bindgen", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.84" +name = "wasm-bindgen" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ - "bumpalo", - "log", + "cfg-if", "once_cell", - "proc-macro2", - "quote", - "syn 1.0.107", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3082,49 +4402,101 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 1.0.107", - "wasm-bindgen-backend", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasmtime-internal-core" +version = "42.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a4a3f055a804a2f3d86e816a9df78a8fa57762212a8506164959224a40cd48" +dependencies = [ + "libm", +] + +[[package]] +name = "wasmtime-internal-jit-icache-coherence" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "c57d24e8d1334a0e5a8b600286ffefa1fc4c3e8176b110dff6fbc1f43c4a599b" +dependencies = [ + "cfg-if", + "libc", + "wasmtime-internal-core", + "windows-sys 0.61.2", +] [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" -version = "4.4.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "d3fabb953106c3c8eea8306e4393700d7657561cb43122571b172bbfb7c7ba1d" dependencies = [ - "either", - "libc", - "once_cell", + "env_home", + "rustix", + "winsafe", +] + +[[package]] +name = "wide" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac11b009ebeae802ed758530b6496784ebfee7a87b9abfbcaf3bbe25b814eb25" +dependencies = [ + "bytemuck", + "safe_arch", ] [[package]] name = "widestring" -version = "0.5.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -3144,11 +4516,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -3158,50 +4530,62 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.52.0" +name = "windows-core" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-core", - "windows-targets 0.52.0", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] -name = "windows-core" -version = "0.52.0" +name = "windows-implement" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ - "windows-targets 0.52.0", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "windows-sys" -version = "0.36.1" +name = "windows-interface" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "windows-sys" -version = "0.42.0" +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows_aarch64_gnullvm 0.42.1", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", - "windows_x86_64_gnullvm 0.42.1", - "windows_x86_64_msvc 0.42.1", + "windows-link", ] [[package]] @@ -3210,248 +4594,350 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets 0.42.1", + "windows-targets 0.42.2", ] [[package]] name = "windows-sys" -version = "0.48.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] name = "windows-sys" -version = "0.52.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.6", ] [[package]] -name = "windows-targets" -version = "0.42.1" +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows_aarch64_gnullvm 0.42.1", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", - "windows_x86_64_gnullvm 0.42.1", - "windows_x86_64_msvc 0.42.1", + "windows-link", ] [[package]] name = "windows-targets" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.1" +name = "windows-targets" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" +name = "windows_aarch64_gnullvm" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" -version = "0.36.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.48.5" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] -name = "windows_i686_gnu" -version = "0.52.0" +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] -name = "windows_i686_msvc" -version = "0.36.1" +name = "windows_i686_gnullvm" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" -version = "0.36.1" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] -name = "windows_x86_64_gnu" -version = "0.52.0" +name = "windows_x86_64_gnullvm" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.0" +name = "windows_x86_64_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.36.1" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" +name = "winnow" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" [[package]] -name = "windows_x86_64_msvc" -version = "0.52.0" +name = "winresource" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e287ced0f21cd11f4035fe946fd3af145f068d1acb708afd248100f89ec7432d" +dependencies = [ + "toml", + "version_check", +] + +[[package]] +name = "winsafe" +version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] -name = "winreg" -version = "0.10.1" +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "x509-cert" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "winapi", + "const-oid", + "der", + "sha1", + "signature", + "spki", + "tls_codec", ] [[package]] -name = "xml-rs" -version = "0.8.14" +name = "x509-parser" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52839dc911083a8ef63efa4d039d1f58b5e409f923e44c80828f206f66e5541c" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static 1.5.0", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "xml" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" + +[[package]] +name = "zerocopy" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71ddd76bcebeed25db614f82bf31a9f4222d3fbba300e6fb6c00afa26cbd4d9d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8187381b52e32220d50b255276aa16a084ec0a9017a0ca2152a1f55c539758d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ - "linked-hash-map", + "proc-macro2", + "quote", + "syn", ] + +[[package]] +name = "zlib-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + +[[package]] +name = "zlib-rs" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" + +[[package]] +name = "zmij" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02aae0f83f69aafc94776e879363e9771d7ecbffe2c7fbb6c14c5e00dfe88439" diff --git a/Cargo.toml b/Cargo.toml index 71e9d221ddb..f0de31e7c62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,114 +1,49 @@ [package] name = "rustpython" -version = "0.3.0" -authors = ["RustPython Team"] -edition = "2021" -rust-version = "1.67.1" description = "A python interpreter written in rust." -repository = "https://github.com/RustPython/RustPython" -license = "MIT" include = ["LICENSE", "Cargo.toml", "src/**/*.rs"] - -[workspace] -resolver = "2" -members = [ - "compiler", "compiler/core", "compiler/codegen", - ".", "common", "derive", "jit", "vm", "pylib", "stdlib", "wasm/lib", "derive-impl", -] - -[workspace.dependencies] -rustpython-compiler-core = { path = "compiler/core", version = "0.3.0" } -rustpython-compiler = { path = "compiler", version = "0.3.0" } -rustpython-codegen = { path = "compiler/codegen", version = "0.3.0" } -rustpython-common = { path = "common", version = "0.3.0" } -rustpython-derive = { path = "derive", version = "0.3.0" } -rustpython-derive-impl = { path = "derive-impl", version = "0.3.0" } -rustpython-jit = { path = "jit", version = "0.3.0" } -rustpython-vm = { path = "vm", default-features = false, version = "0.3.0" } -rustpython-pylib = { path = "pylib", version = "0.3.0" } -rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.3.0" } -rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } - -rustpython-literal = { git = "https://github.com/RustPython/Parser.git", rev = "29c4728dbedc7e69cc2560b9b34058bbba9b1303" } -rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", rev = "29c4728dbedc7e69cc2560b9b34058bbba9b1303" } -rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "29c4728dbedc7e69cc2560b9b34058bbba9b1303" } -rustpython-ast = { git = "https://github.com/RustPython/Parser.git", rev = "29c4728dbedc7e69cc2560b9b34058bbba9b1303" } -rustpython-format = { git = "https://github.com/RustPython/Parser.git", rev = "29c4728dbedc7e69cc2560b9b34058bbba9b1303" } -# rustpython-literal = { path = "../RustPython-parser/literal" } -# rustpython-parser-core = { path = "../RustPython-parser/core" } -# rustpython-parser = { path = "../RustPython-parser/parser" } -# rustpython-ast = { path = "../RustPython-parser/ast" } -# rustpython-format = { path = "../RustPython-parser/format" } - -ahash = "0.8.3" -anyhow = "1.0.45" -ascii = "1.0" -atty = "0.2.14" -bitflags = "2.4.0" -bstr = "0.2.17" -cfg-if = "1.0" -chrono = "0.4.31" -crossbeam-utils = "0.8.16" -flame = "0.2.2" -glob = "0.3" -hex = "0.4.3" -indexmap = { version = "1.9.3", features = ["std"] } -insta = "1.33.0" -itertools = "0.11.0" -is-macro = "0.3.0" -libc = "0.2.151" -log = "0.4.16" -nix = "0.26" -malachite-bigint = "0.2.0" -malachite-q = "0.4.4" -malachite-base = "0.4.4" -num-complex = "0.4.0" -num-integer = "0.1.44" -num-traits = "0.2" -num_enum = "0.5.7" -once_cell = "1.18" -parking_lot = "0.12.1" -paste = "1.0.7" -rand = "0.8.5" -rustyline = "11" -serde = { version = "1.0.133", default-features = false } -schannel = "0.1.22" -static_assertions = "1.1" -syn = "1.0.91" -thiserror = "1.0" -thread_local = "1.1.4" -unicode_names2 = "1.1.0" -widestring = "0.5.1" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true [features] -default = ["threading", "stdlib", "zlib", "importlib"] +default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls", "host_env"] +host_env = ["rustpython-vm/host_env", "rustpython-stdlib?/host_env"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] +stdio = ["rustpython-vm/stdio"] stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"] -flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] -freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] +flame-it = ["rustpython-vm/flame-it", "rustpython-stdlib/flame-it", "flame", "flamescope"] +freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] -zlib = ["stdlib", "rustpython-stdlib/zlib"] -bz2 = ["stdlib", "rustpython-stdlib/bz2"] -ssl = ["rustpython-stdlib/ssl"] -ssl-vendor = ["rustpython-stdlib/ssl-vendor"] +sqlite = ["rustpython-stdlib/sqlite"] +ssl = [] +ssl-rustls = ["ssl", "rustpython-stdlib/ssl-rustls"] +ssl-openssl = ["ssl", "rustpython-stdlib/ssl-openssl"] +ssl-vendor = ["ssl-openssl", "rustpython-stdlib/ssl-vendor"] +tkinter = ["rustpython-stdlib/tkinter"] + +[build-dependencies] +winresource = "0.1" [dependencies] rustpython-compiler = { workspace = true } rustpython-pylib = { workspace = true, optional = true } -rustpython-stdlib = { workspace = true, optional = true } -rustpython-vm = { workspace = true, features = ["compiler"] } -rustpython-parser = { workspace = true } +rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] } +rustpython-vm = { workspace = true, features = ["compiler", "gc"] } +ruff_python_parser = { workspace = true } -atty = { workspace = true } cfg-if = { workspace = true } log = { workspace = true } flame = { workspace = true, optional = true } -clap = "2.34" -dirs = { package = "dirs-next", version = "2.0.0" } -env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] } +lexopt = "0.3" +dirs = { package = "dirs-next", version = "2.0" } +env_logger = "0.11" flamescope = { version = "0.1.2", optional = true } [target.'cfg(windows)'.dependencies] @@ -118,9 +53,9 @@ libc = { workspace = true } rustyline = { workspace = true } [dev-dependencies] -cpython = "0.7.0" -criterion = "0.3.5" -python3-sys = "0.7.1" +criterion = { workspace = true } +pyo3 = { version = "0.28.2", features = ["auto-initialize"] } +rustpython-stdlib = { workspace = true } [[bench]] name = "execution" @@ -142,6 +77,21 @@ opt-level = 3 # https://github.com/rust-lang/rust/issues/92869 # lto = "thin" +# Some crates don't change as much but benefit more from +# more expensive optimization passes, so we selectively +# decrease codegen-units in some cases. +[profile.release.package.rustpython-doc] +codegen-units = 1 + +[profile.release.package.rustpython-literal] +codegen-units = 1 + +[profile.release.package.rustpython-common] +codegen-units = 1 + +[profile.release.package.rustpython-wtf8] +codegen-units = 1 + [profile.bench] lto = "thin" codegen-units = 1 @@ -151,6 +101,148 @@ opt-level = 3 lto = "thin" [patch.crates-io] +parking_lot_core = { git = "https://github.com/youknowone/parking_lot", branch = "rustpython" } # REDOX START, Uncomment when you want to compile/check with redoxer -# nix = { git = "https://github.com/coolreader18/nix", branch = "0.26.2-redox" } # REDOX END + +[package.metadata.packager] +product-name = "RustPython" +identifier = "com.rustpython.rustpython" +description = "An open source Python 3 interpreter written in Rust" +homepage = "https://rustpython.github.io/" +license_file = "LICENSE" +authors = ["RustPython Team"] +publisher = "RustPython Team" +resources = ["LICENSE", "README.md", "Lib"] +icons = ["32x32.png"] + +[package.metadata.packager.nsis] +installer_mode = "both" +template = "installer-config/installer.nsi" + +[package.metadata.packager.wix] +template = "installer-config/installer.wxs" + + +[workspace] +resolver = "2" +members = [ + ".", + "crates/*", +] +exclude = ["pymath"] + +[workspace.package] +version = "0.5.0" +authors = ["RustPython Team"] +edition = "2024" +rust-version = "1.93.0" +repository = "https://github.com/RustPython/RustPython" +license = "MIT" + +[workspace.dependencies] +rustpython-compiler-core = { path = "crates/compiler-core", version = "0.5.0" } +rustpython-compiler = { path = "crates/compiler", version = "0.5.0" } +rustpython-codegen = { path = "crates/codegen", version = "0.5.0" } +rustpython-common = { path = "crates/common", version = "0.5.0" } +rustpython-derive = { path = "crates/derive", version = "0.5.0" } +rustpython-derive-impl = { path = "crates/derive-impl", version = "0.5.0" } +rustpython-jit = { path = "crates/jit", version = "0.5.0" } +rustpython-literal = { path = "crates/literal", version = "0.5.0" } +rustpython-vm = { path = "crates/vm", default-features = false, version = "0.5.0" } +rustpython-pylib = { path = "crates/pylib", version = "0.5.0" } +rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.5.0" } +rustpython-sre_engine = { path = "crates/sre_engine", version = "0.5.0" } +rustpython-wtf8 = { path = "crates/wtf8", version = "0.5.0" } +rustpython-doc = { path = "crates/doc", version = "0.5.0" } + +# Ruff tag 0.15.6 is based on commit e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675 +# at the time of this capture. We use the commit hash to ensure reproducible builds. +ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" } +ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" } +ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" } +ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "e4c7f357777a2fdd34dbe6a98b1b7d3e7488f675" } + +phf = { version = "0.13.1", default-features = false, features = ["macros"]} +ahash = "0.8.12" +ascii = "1.1" +bitflags = "2.11.0" +bitflagset = "0.0.3" +bstr = "1" +bytes = "1.11.1" +cfg-if = "1.0" +chrono = { version = "0.4.44", default-features = false, features = ["clock", "oldtime", "std"] } +constant_time_eq = "0.4" +criterion = { version = "0.8", features = ["html_reports"] } +crossbeam-utils = "0.8.21" +flame = "0.2.2" +getrandom = { version = "0.3", features = ["std"] } +glob = "0.3" +hex = "0.4.3" +indexmap = { version = "2.13.0", features = ["std"] } +insta = "1.46" +itertools = "0.14.0" +is-macro = "0.3.7" +junction = "1.4.2" +libc = "0.2.183" +libffi = "5" +log = "0.4.29" +nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } +malachite-bigint = "0.9.1" +malachite-q = "0.9.1" +malachite-base = "0.9.1" +memchr = "2.8.0" +num-complex = "0.4.6" +num-integer = "0.1.46" +num-traits = "0.2" +num_enum = { version = "0.7", default-features = false } +optional = "0.5" +parking_lot = "0.12.3" +paste = "1.0.15" +proc-macro2 = "1.0.105" +pymath = { version = "0.2.0", features = ["mul_add", "malachite-bigint", "complex"] } +quote = "1.0.45" +radium = "1.1.1" +rand = "0.9" +rand_core = { version = "0.9", features = ["os_rng"] } +rustix = { version = "1.1", features = ["event"] } +rustyline = "17.0.1" +serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] } +schannel = "0.1.28" +scoped-tls = "1" +scopeguard = "1" +static_assertions = "1.1" +strum = "0.27" +strum_macros = "0.27" +syn = "2" +thiserror = "2.0" +thread_local = "1.1.9" +unicode-casing = "0.1.1" +unic-char-property = "0.9.0" +unic-normal = "0.9.0" +unic-ucd-age = "0.9.0" +unic-ucd-bidi = "0.9.0" +unic-ucd-category = "0.9.0" +unic-ucd-ident = "0.9.0" +unicode_names2 = "2.0.0" +unicode-bidi-mirroring = "0.4" +widestring = "1.2.0" +windows-sys = "0.61.2" +wasm-bindgen = "0.2.106" + +# Lints + +[workspace.lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" +elided_lifetimes_in_paths = "warn" + +[workspace.lints.clippy] +alloc_instead_of_core = "warn" +std_instead_of_alloc = "warn" +std_instead_of_core = "warn" +perf = "warn" +style = "warn" +complexity = "warn" +suspicious = "warn" +correctness = "warn" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7c79a011bab..7573f0f2640 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -19,13 +19,13 @@ The contents of the Development Guide include: RustPython requires the following: -- Rust latest stable version (e.g 1.69.0 as of Apr 20 2023) +- Rust latest stable version (e.g 1.92.0 as of Jan 7 2026) - To check Rust version: `rustc --version` - If you have `rustup` on your system, enter to update to the latest stable version: `rustup update stable` - If you do not have Rust installed, use [rustup](https://rustup.rs/) to do so. -- CPython version 3.12 or higher +- CPython version 3.14 or higher - CPython can be installed by your operating system's package manager, from the [Python website](https://www.python.org/downloads/), or using a third-party distribution, such as @@ -65,7 +65,7 @@ $ pytest -v Rust unit tests can be run with `cargo`: ```shell -$ cargo test --workspace --exclude rustpython_wasm +$ cargo test --workspace --exclude rustpython_wasm --exclude rustpython-venvlauncher ``` Python unit tests can be run by compiling RustPython and running the test module: @@ -95,6 +95,41 @@ To run only `test_cmath` (located at `Lib/test/test_cmath`) verbosely: $ cargo run --release -- -m test test_cmath -v ``` +### Testing on Linux from macOS + +You can test RustPython on Linux from macOS using Apple's `container` CLI. + +**Setup (one-time):** + +```shell +# Install container CLI +$ brew install container + +# Disable Rosetta requirement for arm64-only builds +$ defaults write com.apple.container.defaults build.rosetta -bool false + +# Build the development image +$ container build --arch arm64 -t rustpython-dev -f .devcontainer/Dockerfile . +``` + +**Running tests:** + +```shell +# Start a persistent container in background (8GB memory, 4 CPUs for compilation) +$ container run -d --name rustpython-test -m 8G -c 4 \ + --mount type=bind,source=$(pwd),target=/workspace \ + -w /workspace rustpython-dev sleep infinity + +# Run tests inside the container +$ container exec rustpython-test sh -c "cargo run --release -- -m test test_ensurepip" + +# Run any command +$ container exec rustpython-test sh -c "cargo test --workspace" + +# Stop and remove the container when done +$ container rm -f rustpython-test +``` + ## Profiling To profile RustPython, build it in `release` mode with the `flame-it` feature. @@ -118,19 +153,19 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i Understanding a new codebase takes time. Here's a brief view of the repository's structure: -- `compiler/src`: python compilation to bytecode - - `core/src`: python bytecode representation in rust structures - - `parser/src`: python lexing, parsing and ast -- `derive/src`: Rust language extensions and macros specific to rustpython +- `crates/compiler/src`: python compilation to bytecode + - `crates/compiler-core/src`: python bytecode representation in rust structures +- `crates/derive/src` and `crates/derive-impl/src`: Rust language extensions and macros specific to rustpython - `Lib`: Carefully selected / copied files from CPython sourcecode. This is the python side of the standard library. - `test`: CPython test suite -- `vm/src`: python virtual machine +- `crates/vm/src`: python virtual machine - `builtins`: Builtin functions and types - `stdlib`: Standard library parts implemented in rust. - `src`: using the other subcrates to bring rustpython to life. -- `wasm`: Binary crate and resources for WebAssembly build -- `extra_tests`: extra integration test snippets as a supplement to `Lib/test` +- `crates/wasm`: Binary crate and resources for WebAssembly build +- `extra_tests`: extra integration test snippets as a supplement to `Lib/test`. + Add new RustPython-only regression tests here; do not place new tests under `Lib/test`. ## Understanding Internals @@ -140,9 +175,9 @@ implementation is found in the `src` directory (specifically, `src/lib.rs`). The top-level `rustpython` binary depends on several lower-level crates including: -- `rustpython-parser` (implementation in `compiler/parser/src`) -- `rustpython-compiler` (implementation in `compiler/src`) -- `rustpython-vm` (implementation in `vm/src`) +- `ruff_python_parser` and `ruff_python_ast` (external dependencies from the Ruff project) +- `rustpython-compiler` (implementation in `crates/compiler/src`) +- `rustpython-vm` (implementation in `crates/vm/src`) Together, these crates provide the functions of a programming language and enable a line of code to go through a series of steps: @@ -153,31 +188,26 @@ enable a line of code to go through a series of steps: - compile the AST into bytecode - execute the bytecode in the virtual machine (VM). -### rustpython-parser +### Parser and AST -This crate contains the lexer and parser to convert a line of code to -an Abstract Syntax Tree (AST): +RustPython uses the Ruff project's parser and AST implementation: -- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens -- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses - the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax - tree are Rust structs and enums. - - The Parser relies on `LALRPOP`, a Rust parser generator framework. The - LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`. - - More information on parsers and a tutorial can be found in the - [LALRPOP book](https://lalrpop.github.io/lalrpop/). -- AST: `compiler/ast/` implements in Rust the Python types and expressions - represented by the AST nodes. +- Parser: `ruff_python_parser` is used to convert Python source code into tokens + and parse them into an Abstract Syntax Tree (AST) +- AST: `ruff_python_ast` provides the Rust types and expressions represented by + the AST nodes +- These are external dependencies maintained by the Ruff project +- For more information, visit the [Ruff GitHub repository](https://github.com/astral-sh/ruff) ### rustpython-compiler The `rustpython-compiler` crate's purpose is to transform the AST (Abstract Syntax Tree) to bytecode. The implementation of the compiler is found in the -`compiler/src` directory. The compiler implements Python's symbol table, +`crates/compiler/src` directory. The compiler implements Python's symbol table, ast->bytecode compiler, and bytecode optimizer in Rust. -Implementation of bytecode structure in Rust is found in the `compiler/core/src` -directory. `compiler/core/src/bytecode.rs` contains the representation of +Implementation of bytecode structure in Rust is found in the `crates/compiler-core/src` +directory. `crates/compiler-core/src/bytecode.rs` contains the representation of instructions and operations in Rust. Further information about Python's bytecode instructions can be found in the [Python documentation](https://docs.python.org/3/library/dis.html#bytecodes). @@ -185,14 +215,14 @@ bytecode instructions can be found in the ### rustpython-vm The `rustpython-vm` crate has the important job of running the virtual machine that -executes Python's instructions. The `vm/src` directory contains code to +executes Python's instructions. The `crates/vm/src` directory contains code to implement the read and evaluation loop that fetches and dispatches instructions. This directory also contains the implementation of the -Python Standard Library modules in Rust (`vm/src/stdlib`). In Python -everything can be represented as an object. The `vm/src/builtins` directory holds +Python Standard Library modules in Rust (`crates/vm/src/stdlib`). In Python +everything can be represented as an object. The `crates/vm/src/builtins` directory holds the Rust code used to represent different Python objects and their methods. The core implementation of what a Python object is can be found in -`vm/src/object/core.rs`. +`crates/vm/src/object/core.rs`. ### Code generation diff --git a/LICENSE b/LICENSE index 7213274e0f0..e2aa2ed952e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 RustPython Team +Copyright (c) 2025 RustPython Team Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Lib/__future__.py b/Lib/__future__.py index 97dc90c6e46..39720a5e412 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py @@ -33,7 +33,7 @@ to use the feature in question, but may continue to use such imports. MandatoryRelease may also be None, meaning that a planned feature got -dropped. +dropped or that the release version is undetermined. Instances of class _Feature have two corresponding methods, .getOptionalRelease() and .getMandatoryRelease(). @@ -96,7 +96,7 @@ def getMandatoryRelease(self): """Return release in which this feature will become mandatory. This is a 5-tuple, of the same form as sys.version_info, or, if - the feature was dropped, is None. + the feature was dropped, or the release date is undetermined, is None. """ return self.mandatory @@ -143,5 +143,5 @@ def __repr__(self): CO_FUTURE_GENERATOR_STOP) annotations = _Feature((3, 7, 0, "beta", 1), - (3, 11, 0, "alpha", 0), + None, CO_FUTURE_ANNOTATIONS) diff --git a/Lib/_aix_support.py b/Lib/_aix_support.py new file mode 100644 index 00000000000..dadc75c2bf4 --- /dev/null +++ b/Lib/_aix_support.py @@ -0,0 +1,108 @@ +"""Shared AIX support functions.""" + +import sys +import sysconfig + + +# Taken from _osx_support _read_output function +def _read_cmd_output(commandstring, capture_stderr=False): + """Output from successful command execution or None""" + # Similar to os.popen(commandstring, "r").read(), + # but without actually using os.popen because that + # function is not usable during python bootstrap. + import os + import contextlib + fp = open("/tmp/_aix_support.%s"%( + os.getpid(),), "w+b") + + with contextlib.closing(fp) as fp: + if capture_stderr: + cmd = "%s >'%s' 2>&1" % (commandstring, fp.name) + else: + cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) + return fp.read() if not os.system(cmd) else None + + +def _aix_tag(vrtl, bd): + # type: (List[int], int) -> str + # Infer the ABI bitwidth from maxsize (assuming 64 bit as the default) + _sz = 32 if sys.maxsize == (2**31-1) else 64 + _bd = bd if bd != 0 else 9988 + # vrtl[version, release, technology_level] + return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], _bd, _sz) + + +# extract version, release and technology level from a VRMF string +def _aix_vrtl(vrmf): + # type: (str) -> List[int] + v, r, tl = vrmf.split(".")[:3] + return [int(v[-1]), int(r), int(tl)] + + +def _aix_bos_rte(): + # type: () -> Tuple[str, int] + """ + Return a Tuple[str, int] e.g., ['7.1.4.34', 1806] + The fileset bos.rte represents the current AIX run-time level. It's VRMF and + builddate reflect the current ABI levels of the runtime environment. + If no builddate is found give a value that will satisfy pep425 related queries + """ + # All AIX systems to have lslpp installed in this location + # subprocess may not be available during python bootstrap + try: + import subprocess + out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.rte"]) + except ImportError: + out = _read_cmd_output("/usr/bin/lslpp -Lqc bos.rte") + out = out.decode("utf-8") + out = out.strip().split(":") # type: ignore + _bd = int(out[-1]) if out[-1] != '' else 9988 + return (str(out[2]), _bd) + + +def aix_platform(): + # type: () -> str + """ + AIX filesets are identified by four decimal values: V.R.M.F. + V (version) and R (release) can be retrieved using ``uname`` + Since 2007, starting with AIX 5.3 TL7, the M value has been + included with the fileset bos.rte and represents the Technology + Level (TL) of AIX. The F (Fix) value also increases, but is not + relevant for comparing releases and binary compatibility. + For binary compatibility the so-called builddate is needed. + Again, the builddate of an AIX release is associated with bos.rte. + AIX ABI compatibility is described as guaranteed at: https://www.ibm.com/\ + support/knowledgecenter/en/ssw_aix_72/install/binary_compatability.html + + For pep425 purposes the AIX platform tag becomes: + "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(v, r, tl, builddate, bitsize) + e.g., "aix-6107-1415-32" for AIX 6.1 TL7 bd 1415, 32-bit + and, "aix-6107-1415-64" for AIX 6.1 TL7 bd 1415, 64-bit + """ + vrmf, bd = _aix_bos_rte() + return _aix_tag(_aix_vrtl(vrmf), bd) + + +# extract vrtl from the BUILD_GNU_TYPE as an int +def _aix_bgt(): + # type: () -> List[int] + gnu_type = sysconfig.get_config_var("BUILD_GNU_TYPE") + if not gnu_type: + raise ValueError("BUILD_GNU_TYPE is not defined") + return _aix_vrtl(vrmf=gnu_type) + + +def aix_buildtag(): + # type: () -> str + """ + Return the platform_tag of the system Python was built on. + """ + # AIX_BUILDDATE is defined by configure with: + # lslpp -Lcq bos.rte | awk -F: '{ print $NF }' + build_date = sysconfig.get_config_var("AIX_BUILDDATE") + try: + build_date = int(build_date) + except (ValueError, TypeError): + raise ValueError(f"AIX_BUILDDATE is not defined or invalid: " + f"{build_date!r}") + return _aix_tag(_aix_bgt(), build_date) diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 00000000000..a439d03a144 --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,185 @@ +import io +import sys +from threading import RLock +from time import sleep, time + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. +# Messages longer than this will be truncated by logcat. This limit has already +# been reduced at least once in the history of Android (from 4076 to 4068 between +# API level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then we may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + global logcat + logcat = Logcat(android_log_write) + sys.stdout = TextLogStream(stdout_prio, "python.stdout", sys.stdout) + sys.stderr = TextLogStream(stderr_prio, "python.stderr", sys.stderr) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, prio, tag, original=None, **kwargs): + # Respect the -u option. + if original: + kwargs.setdefault("write_through", original.write_through) + fileno = original.fileno() + else: + fileno = None + + # The default is surrogateescape for stdout and backslashreplace for + # stderr, but in the context of an Android log, readability is more + # important than reversibility. + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("errors", "backslashreplace") + + super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs) + self._lock = RLock() + self._pending_bytes = [] + self._pending_bytes_count = 0 + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string into lines first. Note that "".splitlines() == [], so + # nothing will be logged for an empty string. + with self._lock: + for line in s.splitlines(keepends=True): + while line: + chunk = line[:MAX_CHARS_PER_WRITE] + line = line[MAX_CHARS_PER_WRITE:] + self._write_chunk(chunk) + + return len(s) + + # The size and behavior of TextIOWrapper's buffer is not part of its public + # API, so we handle buffering ourselves to avoid truncation. + def _write_chunk(self, s): + b = s.encode(self.encoding, self.errors) + if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE: + self.flush() + + self._pending_bytes.append(b) + self._pending_bytes_count += len(b) + if ( + self.write_through + or b.endswith(b"\n") + or self._pending_bytes_count > MAX_BYTES_PER_WRITE + ): + self.flush() + + def flush(self): + with self._lock: + self.buffer.write(b"".join(self._pending_bytes)) + self._pending_bytes.clear() + self._pending_bytes_count = 0 + + # Since this is a line-based logging system, line buffering cannot be turned + # off, i.e. a newline always causes a flush. + @property + def line_buffering(self): + return True + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, prio, tag, fileno=None): + self.prio = prio + self.tag = tag + self._fileno = fileno + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + logcat.write(self.prio, self.tag, b) + return len(b) + + # This is needed by the test suite --timeout option, which uses faulthandler. + def fileno(self): + if self._fileno is None: + raise io.UnsupportedOperation("fileno") + return self._fileno + + +# When a large volume of data is written to logcat at once, e.g. when a test +# module fails in --verbose3 mode, there's a risk of overflowing logcat's own +# buffer and losing messages. We avoid this by imposing a rate limit using the +# token bucket algorithm, based on a conservative estimate of how fast `adb +# logcat` can consume data. +MAX_BYTES_PER_SECOND = 1024 * 1024 + +# The logcat buffer size of a device can be determined by running `logcat -g`. +# We set the token bucket size to half of the buffer size of our current minimum +# API level, because other things on the system will be producing messages as +# well. +BUCKET_SIZE = 128 * 1024 + +# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 +PER_MESSAGE_OVERHEAD = 28 + + +class Logcat: + def __init__(self, android_log_write): + self.android_log_write = android_log_write + self._lock = RLock() + self._bucket_level = 0 + self._prev_write_time = time() + + def write(self, prio, tag, message): + # Encode null bytes using "modified UTF-8" to avoid them truncating the + # message. + message = message.replace(b"\x00", b"\xc0\x80") + + with self._lock: + now = time() + self._bucket_level += ( + (now - self._prev_write_time) * MAX_BYTES_PER_SECOND) + + # If the bucket level is still below zero, the clock must have gone + # backwards, so reset it to zero and continue. + self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE)) + self._prev_write_time = now + + self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message) + if self._bucket_level < 0: + sleep(-self._bucket_level / MAX_BYTES_PER_SECOND) + + self.android_log_write(prio, tag, message) diff --git a/Lib/_apple_support.py b/Lib/_apple_support.py new file mode 100644 index 00000000000..92febdcf587 --- /dev/null +++ b/Lib/_apple_support.py @@ -0,0 +1,66 @@ +import io +import sys + + +def init_streams(log_write, stdout_level, stderr_level): + # Redirect stdout and stderr to the Apple system log. This method is + # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger + # is enabled. + sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors) + sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors) + + +class SystemLog(io.TextIOWrapper): + def __init__(self, log_write, level, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(LogStream(log_write, level), **kwargs) + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line, so split + # the string before sending it to the superclass. + for line in s.splitlines(keepends=True): + super().write(line) + + return len(s) + + +class LogStream(io.RawIOBase): + def __init__(self, log_write, level): + self.log_write = log_write + self.level = level + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py new file mode 100644 index 00000000000..1c8741b5a55 --- /dev/null +++ b/Lib/_ast_unparse.py @@ -0,0 +1,1161 @@ +# This module contains ``ast.unparse()``, defined here +# to improve the import time for the ``ast`` module. +import sys +from _ast import * +from ast import NodeVisitor +from contextlib import contextmanager, nullcontext +from enum import IntEnum, auto, _simple_enum + +# Large float and imaginary literals get turned into infinities in the AST. +# We unparse those infinities to INFSTR. +_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) + +@_simple_enum(IntEnum) +class _Precedence: + """Precedence table that originated from python grammar.""" + + NAMED_EXPR = auto() # := + TUPLE = auto() # , + YIELD = auto() # 'yield', 'yield from' + TEST = auto() # 'if'-'else', 'lambda' + OR = auto() # 'or' + AND = auto() # 'and' + NOT = auto() # 'not' + CMP = auto() # '<', '>', '==', '>=', '<=', '!=', + # 'in', 'not in', 'is', 'is not' + EXPR = auto() + BOR = EXPR # '|' + BXOR = auto() # '^' + BAND = auto() # '&' + SHIFT = auto() # '<<', '>>' + ARITH = auto() # '+', '-' + TERM = auto() # '*', '@', '/', '%', '//' + FACTOR = auto() # unary '+', '-', '~' + POWER = auto() # '**' + AWAIT = auto() # 'await' + ATOM = auto() + + def next(self): + try: + return self.__class__(self + 1) + except ValueError: + return self + + +_SINGLE_QUOTES = ("'", '"') +_MULTI_QUOTES = ('"""', "'''") +_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) + +class Unparser(NodeVisitor): + """Methods in this class recursively traverse an AST and + output source code for the abstract syntax; original formatting + is disregarded.""" + + def __init__(self): + self._source = [] + self._precedences = {} + self._type_ignores = {} + self._indent = 0 + self._in_try_star = False + self._in_interactive = False + + def interleave(self, inter, f, seq): + """Call f on each item in seq, calling inter() in between.""" + seq = iter(seq) + try: + f(next(seq)) + except StopIteration: + pass + else: + for x in seq: + inter() + f(x) + + def items_view(self, traverser, items): + """Traverse and separate the given *items* with a comma and append it to + the buffer. If *items* is a single item sequence, a trailing comma + will be added.""" + if len(items) == 1: + traverser(items[0]) + self.write(",") + else: + self.interleave(lambda: self.write(", "), traverser, items) + + def maybe_newline(self): + """Adds a newline if it isn't the start of generated source""" + if self._source: + self.write("\n") + + def maybe_semicolon(self): + """Adds a "; " delimiter if it isn't the start of generated source""" + if self._source: + self.write("; ") + + def fill(self, text="", *, allow_semicolon=True): + """Indent a piece of text and append it, according to the current + indentation level, or only delineate with semicolon if applicable""" + if self._in_interactive and not self._indent and allow_semicolon: + self.maybe_semicolon() + self.write(text) + else: + self.maybe_newline() + self.write(" " * self._indent + text) + + def write(self, *text): + """Add new source parts""" + self._source.extend(text) + + @contextmanager + def buffered(self, buffer = None): + if buffer is None: + buffer = [] + + original_source = self._source + self._source = buffer + yield buffer + self._source = original_source + + @contextmanager + def block(self, *, extra = None): + """A context manager for preparing the source for blocks. It adds + the character':', increases the indentation on enter and decreases + the indentation on exit. If *extra* is given, it will be directly + appended after the colon character. + """ + self.write(":") + if extra: + self.write(extra) + self._indent += 1 + yield + self._indent -= 1 + + @contextmanager + def delimit(self, start, end): + """A context manager for preparing the source for expressions. It adds + *start* to the buffer and enters, after exit it adds *end*.""" + + self.write(start) + yield + self.write(end) + + def delimit_if(self, start, end, condition): + if condition: + return self.delimit(start, end) + else: + return nullcontext() + + def require_parens(self, precedence, node): + """Shortcut to adding precedence related parens""" + return self.delimit_if("(", ")", self.get_precedence(node) > precedence) + + def get_precedence(self, node): + return self._precedences.get(node, _Precedence.TEST) + + def set_precedence(self, precedence, *nodes): + for node in nodes: + self._precedences[node] = precedence + + def get_raw_docstring(self, node): + """If a docstring node is found in the body of the *node* parameter, + return that docstring node, None otherwise. + + Logic mirrored from ``_PyAST_GetDocString``.""" + if not isinstance( + node, (AsyncFunctionDef, FunctionDef, ClassDef, Module) + ) or len(node.body) < 1: + return None + node = node.body[0] + if not isinstance(node, Expr): + return None + node = node.value + if isinstance(node, Constant) and isinstance(node.value, str): + return node + + def get_type_comment(self, node): + comment = self._type_ignores.get(node.lineno) or node.type_comment + if comment is not None: + return f" # type: {comment}" + + def traverse(self, node): + if isinstance(node, list): + for item in node: + self.traverse(item) + else: + super().visit(node) + + # Note: as visit() resets the output text, do NOT rely on + # NodeVisitor.generic_visit to handle any nodes (as it calls back in to + # the subclass visit() method, which resets self._source to an empty list) + def visit(self, node): + """Outputs a source code string that, if converted back to an ast + (using ast.parse) will generate an AST equivalent to *node*""" + self._source = [] + self.traverse(node) + return "".join(self._source) + + def _write_docstring_and_traverse_body(self, node): + if (docstring := self.get_raw_docstring(node)): + self._write_docstring(docstring) + self.traverse(node.body[1:]) + else: + self.traverse(node.body) + + def visit_Module(self, node): + self._type_ignores = { + ignore.lineno: f"ignore{ignore.tag}" + for ignore in node.type_ignores + } + try: + self._write_docstring_and_traverse_body(node) + finally: + self._type_ignores.clear() + + def visit_Interactive(self, node): + self._in_interactive = True + try: + self._write_docstring_and_traverse_body(node) + finally: + self._in_interactive = False + + def visit_FunctionType(self, node): + with self.delimit("(", ")"): + self.interleave( + lambda: self.write(", "), self.traverse, node.argtypes + ) + + self.write(" -> ") + self.traverse(node.returns) + + def visit_Expr(self, node): + self.fill() + self.set_precedence(_Precedence.YIELD, node.value) + self.traverse(node.value) + + def visit_NamedExpr(self, node): + with self.require_parens(_Precedence.NAMED_EXPR, node): + self.set_precedence(_Precedence.ATOM, node.target, node.value) + self.traverse(node.target) + self.write(" := ") + self.traverse(node.value) + + def visit_Import(self, node): + self.fill("import ") + self.interleave(lambda: self.write(", "), self.traverse, node.names) + + def visit_ImportFrom(self, node): + self.fill("from ") + self.write("." * (node.level or 0)) + if node.module: + self.write(node.module) + self.write(" import ") + self.interleave(lambda: self.write(", "), self.traverse, node.names) + + def visit_Assign(self, node): + self.fill() + for target in node.targets: + self.set_precedence(_Precedence.TUPLE, target) + self.traverse(target) + self.write(" = ") + self.traverse(node.value) + if type_comment := self.get_type_comment(node): + self.write(type_comment) + + def visit_AugAssign(self, node): + self.fill() + self.traverse(node.target) + self.write(" " + self.binop[node.op.__class__.__name__] + "= ") + self.traverse(node.value) + + def visit_AnnAssign(self, node): + self.fill() + with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)): + self.traverse(node.target) + self.write(": ") + self.traverse(node.annotation) + if node.value: + self.write(" = ") + self.traverse(node.value) + + def visit_Return(self, node): + self.fill("return") + if node.value: + self.write(" ") + self.traverse(node.value) + + def visit_Pass(self, node): + self.fill("pass") + + def visit_Break(self, node): + self.fill("break") + + def visit_Continue(self, node): + self.fill("continue") + + def visit_Delete(self, node): + self.fill("del ") + self.interleave(lambda: self.write(", "), self.traverse, node.targets) + + def visit_Assert(self, node): + self.fill("assert ") + self.traverse(node.test) + if node.msg: + self.write(", ") + self.traverse(node.msg) + + def visit_Global(self, node): + self.fill("global ") + self.interleave(lambda: self.write(", "), self.write, node.names) + + def visit_Nonlocal(self, node): + self.fill("nonlocal ") + self.interleave(lambda: self.write(", "), self.write, node.names) + + def visit_Await(self, node): + with self.require_parens(_Precedence.AWAIT, node): + self.write("await") + if node.value: + self.write(" ") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_Yield(self, node): + with self.require_parens(_Precedence.YIELD, node): + self.write("yield") + if node.value: + self.write(" ") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_YieldFrom(self, node): + with self.require_parens(_Precedence.YIELD, node): + self.write("yield from ") + if not node.value: + raise ValueError("Node can't be used without a value attribute.") + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + + def visit_Raise(self, node): + self.fill("raise") + if not node.exc: + if node.cause: + raise ValueError(f"Node can't use cause without an exception.") + return + self.write(" ") + self.traverse(node.exc) + if node.cause: + self.write(" from ") + self.traverse(node.cause) + + def do_visit_try(self, node): + self.fill("try", allow_semicolon=False) + with self.block(): + self.traverse(node.body) + for ex in node.handlers: + self.traverse(ex) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + if node.finalbody: + self.fill("finally", allow_semicolon=False) + with self.block(): + self.traverse(node.finalbody) + + def visit_Try(self, node): + prev_in_try_star = self._in_try_star + try: + self._in_try_star = False + self.do_visit_try(node) + finally: + self._in_try_star = prev_in_try_star + + def visit_TryStar(self, node): + prev_in_try_star = self._in_try_star + try: + self._in_try_star = True + self.do_visit_try(node) + finally: + self._in_try_star = prev_in_try_star + + def visit_ExceptHandler(self, node): + self.fill("except*" if self._in_try_star else "except", allow_semicolon=False) + if node.type: + self.write(" ") + self.traverse(node.type) + if node.name: + self.write(" as ") + self.write(node.name) + with self.block(): + self.traverse(node.body) + + def visit_ClassDef(self, node): + self.maybe_newline() + for deco in node.decorator_list: + self.fill("@", allow_semicolon=False) + self.traverse(deco) + self.fill("class " + node.name, allow_semicolon=False) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) + with self.delimit_if("(", ")", condition = node.bases or node.keywords): + comma = False + for e in node.bases: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + for e in node.keywords: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + + with self.block(): + self._write_docstring_and_traverse_body(node) + + def visit_FunctionDef(self, node): + self._function_helper(node, "def") + + def visit_AsyncFunctionDef(self, node): + self._function_helper(node, "async def") + + def _function_helper(self, node, fill_suffix): + self.maybe_newline() + for deco in node.decorator_list: + self.fill("@", allow_semicolon=False) + self.traverse(deco) + def_str = fill_suffix + " " + node.name + self.fill(def_str, allow_semicolon=False) + if hasattr(node, "type_params"): + self._type_params_helper(node.type_params) + with self.delimit("(", ")"): + self.traverse(node.args) + if node.returns: + self.write(" -> ") + self.traverse(node.returns) + with self.block(extra=self.get_type_comment(node)): + self._write_docstring_and_traverse_body(node) + + def _type_params_helper(self, type_params): + if type_params is not None and len(type_params) > 0: + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, type_params) + + def visit_TypeVar(self, node): + self.write(node.name) + if node.bound: + self.write(": ") + self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_TypeVarTuple(self, node): + self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_ParamSpec(self, node): + self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) + + def visit_TypeAlias(self, node): + self.fill("type ") + self.traverse(node.name) + self._type_params_helper(node.type_params) + self.write(" = ") + self.traverse(node.value) + + def visit_For(self, node): + self._for_helper("for ", node) + + def visit_AsyncFor(self, node): + self._for_helper("async for ", node) + + def _for_helper(self, fill, node): + self.fill(fill, allow_semicolon=False) + self.set_precedence(_Precedence.TUPLE, node.target) + self.traverse(node.target) + self.write(" in ") + self.traverse(node.iter) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_If(self, node): + self.fill("if ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + # collapse nested ifs into equivalent elifs. + while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If): + node = node.orelse[0] + self.fill("elif ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + # final else + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_While(self, node): + self.fill("while ", allow_semicolon=False) + self.traverse(node.test) + with self.block(): + self.traverse(node.body) + if node.orelse: + self.fill("else", allow_semicolon=False) + with self.block(): + self.traverse(node.orelse) + + def visit_With(self, node): + self.fill("with ", allow_semicolon=False) + self.interleave(lambda: self.write(", "), self.traverse, node.items) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + + def visit_AsyncWith(self, node): + self.fill("async with ", allow_semicolon=False) + self.interleave(lambda: self.write(", "), self.traverse, node.items) + with self.block(extra=self.get_type_comment(node)): + self.traverse(node.body) + + def _str_literal_helper( + self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False + ): + """Helper for writing string literals, minimizing escapes. + Returns the tuple (string literal to write, possible quote types). + """ + def escape_char(c): + # \n and \t are non-printable, but we only escape them if + # escape_special_whitespace is True + if not escape_special_whitespace and c in "\n\t": + return c + # Always escape backslashes and other non-printable characters + if c == "\\" or not c.isprintable(): + return c.encode("unicode_escape").decode("ascii") + return c + + escaped_string = "".join(map(escape_char, string)) + possible_quotes = quote_types + if "\n" in escaped_string: + possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] + possible_quotes = [q for q in possible_quotes if q not in escaped_string] + if not possible_quotes: + # If there aren't any possible_quotes, fallback to using repr + # on the original string. Try to use a quote from quote_types, + # e.g., so that we use triple quotes for docstrings. + string = repr(string) + quote = next((q for q in quote_types if string[0] in q), string[0]) + return string[1:-1], [quote] + if escaped_string: + # Sort so that we prefer '''"''' over """\"""" + possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) + # If we're using triple quotes and we'd need to escape a final + # quote, escape it + if possible_quotes[0][0] == escaped_string[-1]: + assert len(possible_quotes[0]) == 3 + escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] + return escaped_string, possible_quotes + + def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): + """Write string literal value with a best effort attempt to avoid backslashes.""" + string, quote_types = self._str_literal_helper(string, quote_types=quote_types) + quote_type = quote_types[0] + self.write(f"{quote_type}{string}{quote_type}") + + def _ftstring_helper(self, parts): + new_parts = [] + quote_types = list(_ALL_QUOTES) + fallback_to_repr = False + for value, is_constant in parts: + if is_constant: + value, new_quote_types = self._str_literal_helper( + value, + quote_types=quote_types, + escape_special_whitespace=True, + ) + if set(new_quote_types).isdisjoint(quote_types): + fallback_to_repr = True + break + quote_types = new_quote_types + else: + if "\n" in value: + quote_types = [q for q in quote_types if q in _MULTI_QUOTES] + assert quote_types + + new_quote_types = [q for q in quote_types if q not in value] + if new_quote_types: + quote_types = new_quote_types + new_parts.append(value) + + if fallback_to_repr: + # If we weren't able to find a quote type that works for all parts + # of the JoinedStr, fallback to using repr and triple single quotes. + quote_types = ["'''"] + new_parts.clear() + for value, is_constant in parts: + if is_constant: + value = repr('"' + value) # force repr to use single quotes + expected_prefix = "'\"" + assert value.startswith(expected_prefix), repr(value) + value = value[len(expected_prefix):-1] + new_parts.append(value) + + value = "".join(new_parts) + quote_type = quote_types[0] + self.write(f"{quote_type}{value}{quote_type}") + + def _write_ftstring(self, values, prefix): + self.write(prefix) + fstring_parts = [] + for value in values: + with self.buffered() as buffer: + self._write_ftstring_inner(value) + fstring_parts.append( + ("".join(buffer), isinstance(value, Constant)) + ) + self._ftstring_helper(fstring_parts) + + def visit_JoinedStr(self, node): + self._write_ftstring(node.values, "f") + + def visit_TemplateStr(self, node): + self._write_ftstring(node.values, "t") + + def _write_ftstring_inner(self, node, is_format_spec=False): + if isinstance(node, JoinedStr): + # for both the f-string itself, and format_spec + for value in node.values: + self._write_ftstring_inner(value, is_format_spec=is_format_spec) + elif isinstance(node, Constant) and isinstance(node.value, str): + value = node.value.replace("{", "{{").replace("}", "}}") + + if is_format_spec: + value = value.replace("\\", "\\\\") + value = value.replace("'", "\\'") + value = value.replace('"', '\\"') + value = value.replace("\n", "\\n") + self.write(value) + elif isinstance(node, FormattedValue): + self.visit_FormattedValue(node) + elif isinstance(node, Interpolation): + self.visit_Interpolation(node) + else: + raise ValueError(f"Unexpected node inside JoinedStr, {node!r}") + + def _unparse_interpolation_value(self, inner): + unparser = type(self)() + unparser.set_precedence(_Precedence.TEST.next(), inner) + return unparser.visit(inner) + + def _write_interpolation(self, node, use_str_attr=False): + with self.delimit("{", "}"): + if use_str_attr: + expr = node.str + else: + expr = self._unparse_interpolation_value(node.value) + if expr.startswith("{"): + # Separate pair of opening brackets as "{ {" + self.write(" ") + self.write(expr) + if node.conversion != -1: + self.write(f"!{chr(node.conversion)}") + if node.format_spec: + self.write(":") + self._write_ftstring_inner(node.format_spec, is_format_spec=True) + + def visit_FormattedValue(self, node): + self._write_interpolation(node) + + def visit_Interpolation(self, node): + # If `str` is set to `None`, use the `value` to generate the source code. + self._write_interpolation(node, use_str_attr=node.str is not None) + + def visit_Name(self, node): + self.write(node.id) + + def _write_docstring(self, node): + self.fill(allow_semicolon=False) + if node.kind == "u": + self.write("u") + self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) + + def _write_constant(self, value): + if isinstance(value, (float, complex)): + # Substitute overflowing decimal literal for AST infinities, + # and inf - inf for NaNs. + self.write( + repr(value) + .replace("inf", _INFSTR) + .replace("nan", f"({_INFSTR}-{_INFSTR})") + ) + else: + self.write(repr(value)) + + def visit_Constant(self, node): + value = node.value + if isinstance(value, tuple): + with self.delimit("(", ")"): + self.items_view(self._write_constant, value) + elif value is ...: + self.write("...") + else: + if node.kind == "u": + self.write("u") + self._write_constant(node.value) + + def visit_List(self, node): + with self.delimit("[", "]"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + + def visit_ListComp(self, node): + with self.delimit("[", "]"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_GeneratorExp(self, node): + with self.delimit("(", ")"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_SetComp(self, node): + with self.delimit("{", "}"): + self.traverse(node.elt) + for gen in node.generators: + self.traverse(gen) + + def visit_DictComp(self, node): + with self.delimit("{", "}"): + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + for gen in node.generators: + self.traverse(gen) + + def visit_comprehension(self, node): + if node.is_async: + self.write(" async for ") + else: + self.write(" for ") + self.set_precedence(_Precedence.TUPLE, node.target) + self.traverse(node.target) + self.write(" in ") + self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs) + self.traverse(node.iter) + for if_clause in node.ifs: + self.write(" if ") + self.traverse(if_clause) + + def visit_IfExp(self, node): + with self.require_parens(_Precedence.TEST, node): + self.set_precedence(_Precedence.TEST.next(), node.body, node.test) + self.traverse(node.body) + self.write(" if ") + self.traverse(node.test) + self.write(" else ") + self.set_precedence(_Precedence.TEST, node.orelse) + self.traverse(node.orelse) + + def visit_Set(self, node): + if node.elts: + with self.delimit("{", "}"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + else: + # `{}` would be interpreted as a dictionary literal, and + # `set` might be shadowed. Thus: + self.write('{*()}') + + def visit_Dict(self, node): + def write_key_value_pair(k, v): + self.traverse(k) + self.write(": ") + self.traverse(v) + + def write_item(item): + k, v = item + if k is None: + # for dictionary unpacking operator in dicts {**{'y': 2}} + # see PEP 448 for details + self.write("**") + self.set_precedence(_Precedence.EXPR, v) + self.traverse(v) + else: + write_key_value_pair(k, v) + + with self.delimit("{", "}"): + self.interleave( + lambda: self.write(", "), write_item, zip(node.keys, node.values) + ) + + def visit_Tuple(self, node): + with self.delimit_if( + "(", + ")", + len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE + ): + self.items_view(self.traverse, node.elts) + + unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} + unop_precedence = { + "not": _Precedence.NOT, + "~": _Precedence.FACTOR, + "+": _Precedence.FACTOR, + "-": _Precedence.FACTOR, + } + + def visit_UnaryOp(self, node): + operator = self.unop[node.op.__class__.__name__] + operator_precedence = self.unop_precedence[operator] + with self.require_parens(operator_precedence, node): + self.write(operator) + # factor prefixes (+, -, ~) shouldn't be separated + # from the value they belong, (e.g: +1 instead of + 1) + if operator_precedence is not _Precedence.FACTOR: + self.write(" ") + self.set_precedence(operator_precedence, node.operand) + self.traverse(node.operand) + + binop = { + "Add": "+", + "Sub": "-", + "Mult": "*", + "MatMult": "@", + "Div": "/", + "Mod": "%", + "LShift": "<<", + "RShift": ">>", + "BitOr": "|", + "BitXor": "^", + "BitAnd": "&", + "FloorDiv": "//", + "Pow": "**", + } + + binop_precedence = { + "+": _Precedence.ARITH, + "-": _Precedence.ARITH, + "*": _Precedence.TERM, + "@": _Precedence.TERM, + "/": _Precedence.TERM, + "%": _Precedence.TERM, + "<<": _Precedence.SHIFT, + ">>": _Precedence.SHIFT, + "|": _Precedence.BOR, + "^": _Precedence.BXOR, + "&": _Precedence.BAND, + "//": _Precedence.TERM, + "**": _Precedence.POWER, + } + + binop_rassoc = frozenset(("**",)) + def visit_BinOp(self, node): + operator = self.binop[node.op.__class__.__name__] + operator_precedence = self.binop_precedence[operator] + with self.require_parens(operator_precedence, node): + if operator in self.binop_rassoc: + left_precedence = operator_precedence.next() + right_precedence = operator_precedence + else: + left_precedence = operator_precedence + right_precedence = operator_precedence.next() + + self.set_precedence(left_precedence, node.left) + self.traverse(node.left) + self.write(f" {operator} ") + self.set_precedence(right_precedence, node.right) + self.traverse(node.right) + + cmpops = { + "Eq": "==", + "NotEq": "!=", + "Lt": "<", + "LtE": "<=", + "Gt": ">", + "GtE": ">=", + "Is": "is", + "IsNot": "is not", + "In": "in", + "NotIn": "not in", + } + + def visit_Compare(self, node): + with self.require_parens(_Precedence.CMP, node): + self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators) + self.traverse(node.left) + for o, e in zip(node.ops, node.comparators): + self.write(" " + self.cmpops[o.__class__.__name__] + " ") + self.traverse(e) + + boolops = {"And": "and", "Or": "or"} + boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR} + + def visit_BoolOp(self, node): + operator = self.boolops[node.op.__class__.__name__] + operator_precedence = self.boolop_precedence[operator] + + def increasing_level_traverse(node): + nonlocal operator_precedence + operator_precedence = operator_precedence.next() + self.set_precedence(operator_precedence, node) + self.traverse(node) + + with self.require_parens(operator_precedence, node): + s = f" {operator} " + self.interleave(lambda: self.write(s), increasing_level_traverse, node.values) + + def visit_Attribute(self, node): + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + # Special case: 3.__abs__() is a syntax error, so if node.value + # is an integer literal then we need to either parenthesize + # it or add an extra space to get 3 .__abs__(). + if isinstance(node.value, Constant) and isinstance(node.value.value, int): + self.write(" ") + self.write(".") + self.write(node.attr) + + def visit_Call(self, node): + self.set_precedence(_Precedence.ATOM, node.func) + self.traverse(node.func) + with self.delimit("(", ")"): + comma = False + for e in node.args: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + for e in node.keywords: + if comma: + self.write(", ") + else: + comma = True + self.traverse(e) + + def visit_Subscript(self, node): + def is_non_empty_tuple(slice_value): + return ( + isinstance(slice_value, Tuple) + and slice_value.elts + ) + + self.set_precedence(_Precedence.ATOM, node.value) + self.traverse(node.value) + with self.delimit("[", "]"): + if is_non_empty_tuple(node.slice): + # parentheses can be omitted if the tuple isn't empty + self.items_view(self.traverse, node.slice.elts) + else: + self.traverse(node.slice) + + def visit_Starred(self, node): + self.write("*") + self.set_precedence(_Precedence.EXPR, node.value) + self.traverse(node.value) + + def visit_Ellipsis(self, node): + self.write("...") + + def visit_Slice(self, node): + if node.lower: + self.traverse(node.lower) + self.write(":") + if node.upper: + self.traverse(node.upper) + if node.step: + self.write(":") + self.traverse(node.step) + + def visit_Match(self, node): + self.fill("match ", allow_semicolon=False) + self.traverse(node.subject) + with self.block(): + for case in node.cases: + self.traverse(case) + + def visit_arg(self, node): + self.write(node.arg) + if node.annotation: + self.write(": ") + self.traverse(node.annotation) + + def visit_arguments(self, node): + first = True + # normal arguments + all_args = node.posonlyargs + node.args + defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults + for index, elements in enumerate(zip(all_args, defaults), 1): + a, d = elements + if first: + first = False + else: + self.write(", ") + self.traverse(a) + if d: + self.write("=") + self.traverse(d) + if index == len(node.posonlyargs): + self.write(", /") + + # varargs, or bare '*' if no varargs but keyword-only arguments present + if node.vararg or node.kwonlyargs: + if first: + first = False + else: + self.write(", ") + self.write("*") + if node.vararg: + self.write(node.vararg.arg) + if node.vararg.annotation: + self.write(": ") + self.traverse(node.vararg.annotation) + + # keyword-only arguments + if node.kwonlyargs: + for a, d in zip(node.kwonlyargs, node.kw_defaults): + self.write(", ") + self.traverse(a) + if d: + self.write("=") + self.traverse(d) + + # kwargs + if node.kwarg: + if first: + first = False + else: + self.write(", ") + self.write("**" + node.kwarg.arg) + if node.kwarg.annotation: + self.write(": ") + self.traverse(node.kwarg.annotation) + + def visit_keyword(self, node): + if node.arg is None: + self.write("**") + else: + self.write(node.arg) + self.write("=") + self.traverse(node.value) + + def visit_Lambda(self, node): + with self.require_parens(_Precedence.TEST, node): + self.write("lambda") + with self.buffered() as buffer: + self.traverse(node.args) + if buffer: + self.write(" ", *buffer) + self.write(": ") + self.set_precedence(_Precedence.TEST, node.body) + self.traverse(node.body) + + def visit_alias(self, node): + self.write(node.name) + if node.asname: + self.write(" as " + node.asname) + + def visit_withitem(self, node): + self.traverse(node.context_expr) + if node.optional_vars: + self.write(" as ") + self.traverse(node.optional_vars) + + def visit_match_case(self, node): + self.fill("case ", allow_semicolon=False) + self.traverse(node.pattern) + if node.guard: + self.write(" if ") + self.traverse(node.guard) + with self.block(): + self.traverse(node.body) + + def visit_MatchValue(self, node): + self.traverse(node.value) + + def visit_MatchSingleton(self, node): + self._write_constant(node.value) + + def visit_MatchSequence(self, node): + with self.delimit("[", "]"): + self.interleave( + lambda: self.write(", "), self.traverse, node.patterns + ) + + def visit_MatchStar(self, node): + name = node.name + if name is None: + name = "_" + self.write(f"*{name}") + + def visit_MatchMapping(self, node): + def write_key_pattern_pair(pair): + k, p = pair + self.traverse(k) + self.write(": ") + self.traverse(p) + + with self.delimit("{", "}"): + keys = node.keys + self.interleave( + lambda: self.write(", "), + write_key_pattern_pair, + zip(keys, node.patterns, strict=True), + ) + rest = node.rest + if rest is not None: + if keys: + self.write(", ") + self.write(f"**{rest}") + + def visit_MatchClass(self, node): + self.set_precedence(_Precedence.ATOM, node.cls) + self.traverse(node.cls) + with self.delimit("(", ")"): + patterns = node.patterns + self.interleave( + lambda: self.write(", "), self.traverse, patterns + ) + attrs = node.kwd_attrs + if attrs: + def write_attr_pattern(pair): + attr, pattern = pair + self.write(f"{attr}=") + self.traverse(pattern) + + if patterns: + self.write(", ") + self.interleave( + lambda: self.write(", "), + write_attr_pattern, + zip(attrs, node.kwd_patterns, strict=True), + ) + + def visit_MatchAs(self, node): + name = node.name + pattern = node.pattern + if name is None: + self.write("_") + elif pattern is None: + self.write(node.name) + else: + with self.require_parens(_Precedence.TEST, node): + self.set_precedence(_Precedence.BOR, node.pattern) + self.traverse(node.pattern) + self.write(f" as {node.name}") + + def visit_MatchOr(self, node): + with self.require_parens(_Precedence.BOR, node): + self.set_precedence(_Precedence.BOR.next(), *node.patterns) + self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index e96e4c35355..241d40d5740 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -6,6 +6,32 @@ Unit tests are in test_collections. """ +############ Maintenance notes ######################################### +# +# ABCs are different from other standard library modules in that they +# specify compliance tests. In general, once an ABC has been published, +# new methods (either abstract or concrete) cannot be added. +# +# Though classes that inherit from an ABC would automatically receive a +# new mixin method, registered classes would become non-compliant and +# violate the contract promised by ``isinstance(someobj, SomeABC)``. +# +# Though irritating, the correct procedure for adding new abstract or +# mixin methods is to create a new ABC as a subclass of the previous +# ABC. For example, union(), intersection(), and difference() cannot +# be added to Set but could go into a new ABC that extends Set. +# +# Because they are so hard to change, new ABCs should have their APIs +# carefully thought through prior to publication. +# +# Since ABCMeta only checks for the presence of methods, it is possible +# to alter the signature of a method by adding optional arguments +# or changing parameters names. This is still a bit dubious but at +# least it won't cause isinstance() to return an incorrect result. +# +# +####################################################################### + from abc import ABCMeta, abstractmethod import sys @@ -23,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", + "ByteString", "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -59,6 +85,10 @@ def _f(): pass dict_items = type({}.items()) ## misc ## mappingproxy = type(type.__dict__) +def _get_framelocalsproxy(): + return type(sys._getframe().f_locals) +framelocalsproxy = _get_framelocalsproxy() +del _get_framelocalsproxy generator = type((lambda: (yield))()) ## coroutine ## async def _coro(): pass @@ -413,6 +443,21 @@ def __subclasshook__(cls, C): return NotImplemented +class Buffer(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __buffer__(self, flags: int, /) -> memoryview: + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, C): + if cls is Buffer: + return _check_methods(C, "__buffer__") + return NotImplemented + + class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. @@ -440,9 +485,10 @@ def __new__(cls, origin, args): def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() + from annotationlib import type_repr return (f'collections.abc.Callable' - f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' - f'{_type_repr(self.__args__[-1])}]') + f'[[{", ".join([type_repr(a) for a in self.__args__[:-1]])}], ' + f'{type_repr(self.__args__[-1])}]') def __reduce__(self): args = self.__args__ @@ -455,15 +501,8 @@ def __getitem__(self, item): # rather than the default types.GenericAlias object. Most of the # code is copied from typing's _GenericAlias and the builtin # types.GenericAlias. - if not isinstance(item, tuple): item = (item,) - # A special case in PEP 612 where if X = Callable[P, int], - # then X[int, str] == X[[int, str]]. - if (len(self.__parameters__) == 1 - and _is_param_expr(self.__parameters__[0]) - and item and not _is_param_expr(item[0])): - item = (item,) new_args = super().__getitem__(item).__args__ @@ -486,24 +525,6 @@ def _is_param_expr(obj): names = ('ParamSpec', '_ConcatenateGenericAlias') return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) -def _type_repr(obj): - """Return the repr() of an object, special-casing types (internal helper). - - Copied from :mod:`typing` since collections.abc - shouldn't depend on that module. - """ - if isinstance(obj, GenericAlias): - return repr(obj) - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is Ellipsis: - return '...' - if isinstance(obj, FunctionType): - return obj.__name__ - return repr(obj) - class Callable(metaclass=ABCMeta): @@ -803,6 +824,7 @@ def __eq__(self, other): __reversed__ = None Mapping.register(mappingproxy) +Mapping.register(framelocalsproxy) class MappingView(Sized): @@ -940,7 +962,7 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E: D[k] = E[k] + If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v In either case, this is followed by: for k, v in F.items(): D[k] = v ''' @@ -1035,14 +1057,38 @@ def count(self, value): Sequence.register(tuple) Sequence.register(str) +Sequence.register(bytes) Sequence.register(range) Sequence.register(memoryview) - -class ByteString(Sequence): - """This unifies bytes and bytearray. - - XXX Should add all their methods. +class _DeprecateByteStringMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + if name != "ByteString": + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__new__(cls, name, bases, namespace, **kwargs) + + def __instancecheck__(cls, instance): + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__instancecheck__(instance) + +class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): + """Deprecated ABC serving as a common supertype of ``bytes`` and ``bytearray``. + + This ABC is scheduled for removal in Python 3.17. + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the buffer protocol at runtime. For use in type annotations, + either use ``Buffer`` or a union that explicitly specifies the types your + code supports (e.g., ``bytes | bytearray | memoryview``). """ __slots__ = () @@ -1118,4 +1164,4 @@ def __iadd__(self, values): MutableSequence.register(list) -MutableSequence.register(bytearray) # Multiply inheriting, see ByteString +MutableSequence.register(bytearray) diff --git a/Lib/_colorize.py b/Lib/_colorize.py new file mode 100644 index 00000000000..d6673f6692f --- /dev/null +++ b/Lib/_colorize.py @@ -0,0 +1,355 @@ +import os +import sys + +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass, field, Field + +COLORIZE = True + + +# types +if False: + from typing import IO, Self, ClassVar + _theme: Theme + + +class ANSIColors: + RESET = "\x1b[0m" + + BLACK = "\x1b[30m" + BLUE = "\x1b[34m" + CYAN = "\x1b[36m" + GREEN = "\x1b[32m" + GREY = "\x1b[90m" + MAGENTA = "\x1b[35m" + RED = "\x1b[31m" + WHITE = "\x1b[37m" # more like LIGHT GRAY + YELLOW = "\x1b[33m" + + BOLD = "\x1b[1m" + BOLD_BLACK = "\x1b[1;30m" # DARK GRAY + BOLD_BLUE = "\x1b[1;34m" + BOLD_CYAN = "\x1b[1;36m" + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + BOLD_WHITE = "\x1b[1;37m" # actual WHITE + BOLD_YELLOW = "\x1b[1;33m" + + # intense = like bold but without being bold + INTENSE_BLACK = "\x1b[90m" + INTENSE_BLUE = "\x1b[94m" + INTENSE_CYAN = "\x1b[96m" + INTENSE_GREEN = "\x1b[92m" + INTENSE_MAGENTA = "\x1b[95m" + INTENSE_RED = "\x1b[91m" + INTENSE_WHITE = "\x1b[97m" + INTENSE_YELLOW = "\x1b[93m" + + BACKGROUND_BLACK = "\x1b[40m" + BACKGROUND_BLUE = "\x1b[44m" + BACKGROUND_CYAN = "\x1b[46m" + BACKGROUND_GREEN = "\x1b[42m" + BACKGROUND_MAGENTA = "\x1b[45m" + BACKGROUND_RED = "\x1b[41m" + BACKGROUND_WHITE = "\x1b[47m" + BACKGROUND_YELLOW = "\x1b[43m" + + INTENSE_BACKGROUND_BLACK = "\x1b[100m" + INTENSE_BACKGROUND_BLUE = "\x1b[104m" + INTENSE_BACKGROUND_CYAN = "\x1b[106m" + INTENSE_BACKGROUND_GREEN = "\x1b[102m" + INTENSE_BACKGROUND_MAGENTA = "\x1b[105m" + INTENSE_BACKGROUND_RED = "\x1b[101m" + INTENSE_BACKGROUND_WHITE = "\x1b[107m" + INTENSE_BACKGROUND_YELLOW = "\x1b[103m" + + +ColorCodes = set() +NoColors = ANSIColors() + +for attr, code in ANSIColors.__dict__.items(): + if not attr.startswith("__"): + ColorCodes.add(code) + setattr(NoColors, attr, "") + + +# +# Experimental theming support (see gh-133346) +# + +# - Create a theme by copying an existing `Theme` with one or more sections +# replaced, using `default_theme.copy_with()`; +# - create a theme section by copying an existing `ThemeSection` with one or +# more colors replaced, using for example `default_theme.syntax.copy_with()`; +# - create a theme from scratch by instantiating a `Theme` data class with +# the required sections (which are also dataclass instances). +# +# Then call `_colorize.set_theme(your_theme)` to set it. +# +# Put your theme configuration in $PYTHONSTARTUP for the interactive shell, +# or sitecustomize.py in your virtual environment or Python installation for +# other uses. Your applications can call `_colorize.set_theme()` too. +# +# Note that thanks to the dataclasses providing default values for all fields, +# creating a new theme or theme section from scratch is possible without +# specifying all keys. +# +# For example, here's a theme that makes punctuation and operators less prominent: +# +# try: +# from _colorize import set_theme, default_theme, Syntax, ANSIColors +# except ImportError: +# pass +# else: +# theme_with_dim_operators = default_theme.copy_with( +# syntax=Syntax(op=ANSIColors.INTENSE_BLACK), +# ) +# set_theme(theme_with_dim_operators) +# del set_theme, default_theme, Syntax, ANSIColors, theme_with_dim_operators +# +# Guarding the import ensures that your .pythonstartup file will still work in +# Python 3.13 and older. Deleting the variables ensures they don't remain in your +# interactive shell's global scope. + +class ThemeSection(Mapping[str, str]): + """A mixin/base class for theme sections. + + It enables dictionary access to a section, as well as implements convenience + methods. + """ + + # The two types below are just that: types to inform the type checker that the + # mixin will work in context of those fields existing + __dataclass_fields__: ClassVar[dict[str, Field[str]]] + _name_to_value: Callable[[str], str] + + def __post_init__(self) -> None: + name_to_value = {} + for color_name in self.__dataclass_fields__: + name_to_value[color_name] = getattr(self, color_name) + super().__setattr__('_name_to_value', name_to_value.__getitem__) + + def copy_with(self, **kwargs: str) -> Self: + color_state: dict[str, str] = {} + for color_name in self.__dataclass_fields__: + color_state[color_name] = getattr(self, color_name) + color_state.update(kwargs) + return type(self)(**color_state) + + @classmethod + def no_colors(cls) -> Self: + color_state: dict[str, str] = {} + for color_name in cls.__dataclass_fields__: + color_state[color_name] = "" + return cls(**color_state) + + def __getitem__(self, key: str) -> str: + return self._name_to_value(key) + + def __len__(self) -> int: + return len(self.__dataclass_fields__) + + def __iter__(self) -> Iterator[str]: + return iter(self.__dataclass_fields__) + + +@dataclass(frozen=True, kw_only=True) +class Argparse(ThemeSection): + usage: str = ANSIColors.BOLD_BLUE + prog: str = ANSIColors.BOLD_MAGENTA + prog_extra: str = ANSIColors.MAGENTA + heading: str = ANSIColors.BOLD_BLUE + summary_long_option: str = ANSIColors.CYAN + summary_short_option: str = ANSIColors.GREEN + summary_label: str = ANSIColors.YELLOW + summary_action: str = ANSIColors.GREEN + long_option: str = ANSIColors.BOLD_CYAN + short_option: str = ANSIColors.BOLD_GREEN + label: str = ANSIColors.BOLD_YELLOW + action: str = ANSIColors.BOLD_GREEN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Syntax(ThemeSection): + prompt: str = ANSIColors.BOLD_MAGENTA + keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE + builtin: str = ANSIColors.CYAN + comment: str = ANSIColors.RED + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + op: str = ANSIColors.RESET + definition: str = ANSIColors.BOLD + soft_keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Traceback(ThemeSection): + type: str = ANSIColors.BOLD_MAGENTA + message: str = ANSIColors.MAGENTA + filename: str = ANSIColors.MAGENTA + line_no: str = ANSIColors.MAGENTA + frame: str = ANSIColors.MAGENTA + error_highlight: str = ANSIColors.BOLD_RED + error_range: str = ANSIColors.RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Unittest(ThemeSection): + passed: str = ANSIColors.GREEN + warn: str = ANSIColors.YELLOW + fail: str = ANSIColors.RED + fail_info: str = ANSIColors.BOLD_RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True) +class Theme: + """A suite of themes for all sections of Python. + + When adding a new one, remember to also modify `copy_with` and `no_colors` + below. + """ + argparse: Argparse = field(default_factory=Argparse) + syntax: Syntax = field(default_factory=Syntax) + traceback: Traceback = field(default_factory=Traceback) + unittest: Unittest = field(default_factory=Unittest) + + def copy_with( + self, + *, + argparse: Argparse | None = None, + syntax: Syntax | None = None, + traceback: Traceback | None = None, + unittest: Unittest | None = None, + ) -> Self: + """Return a new Theme based on this instance with some sections replaced. + + Themes are immutable to protect against accidental modifications that + could lead to invalid terminal states. + """ + return type(self)( + argparse=argparse or self.argparse, + syntax=syntax or self.syntax, + traceback=traceback or self.traceback, + unittest=unittest or self.unittest, + ) + + @classmethod + def no_colors(cls) -> Self: + """Return a new Theme where colors in all sections are empty strings. + + This allows writing user code as if colors are always used. The color + fields will be ANSI color code strings when colorization is desired + and possible, and empty strings otherwise. + """ + return cls( + argparse=Argparse.no_colors(), + syntax=Syntax.no_colors(), + traceback=Traceback.no_colors(), + unittest=Unittest.no_colors(), + ) + + +def get_colors( + colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None +) -> ANSIColors: + if colorize or can_colorize(file=file): + return ANSIColors() + else: + return NoColors + + +def decolor(text: str) -> str: + """Remove ANSI color codes from a string.""" + for code in ColorCodes: + text = text.replace(code, "") + return text + + +def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + + if file is None: + file = sys.stdout + + if not sys.flags.ignore_environment: + if _safe_getenv("PYTHON_COLORS") == "0": + return False + if _safe_getenv("PYTHON_COLORS") == "1": + return True + if _safe_getenv("NO_COLOR"): + return False + if not COLORIZE: + return False + if _safe_getenv("FORCE_COLOR"): + return True + if _safe_getenv("TERM") == "dumb": + return False + + if not hasattr(file, "fileno"): + return False + + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + + try: + return os.isatty(file.fileno()) + except OSError: + return hasattr(file, "isatty") and file.isatty() + + +default_theme = Theme() +theme_no_color = default_theme.no_colors() + + +def get_theme( + *, + tty_file: IO[str] | IO[bytes] | None = None, + force_color: bool = False, + force_no_color: bool = False, +) -> Theme: + """Returns the currently set theme, potentially in a zero-color variant. + + In cases where colorizing is not possible (see `can_colorize`), the returned + theme contains all empty strings in all color definitions. + See `Theme.no_colors()` for more information. + + It is recommended not to cache the result of this function for extended + periods of time because the user might influence theme selection by + the interactive shell, a debugger, or application-specific code. The + environment (including environment variable state and console configuration + on Windows) can also change in the course of the application life cycle. + """ + if force_color or (not force_no_color and + can_colorize(file=tty_file)): + return _theme + return theme_no_color + + +def set_theme(t: Theme) -> None: + global _theme + + if not isinstance(t, Theme): + raise ValueError(f"Expected Theme object, found {t}") + + _theme = t + + +set_theme(default_theme) diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 17b9010278f..60793c391ae 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -22,7 +22,6 @@ 'tkMessageBox': 'tkinter.messagebox', 'ScrolledText': 'tkinter.scrolledtext', 'Tkconstants': 'tkinter.constants', - 'Tix': 'tkinter.tix', 'ttk': 'tkinter.ttk', 'Tkinter': 'tkinter', 'markupbase': '_markupbase', @@ -257,3 +256,4 @@ for excname in PYTHON3_IMPORTERROR_EXCEPTIONS: REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'ImportError') +del excname diff --git a/Lib/_compression.py b/Lib/_compression.py deleted file mode 100644 index e8b70aa0a3e..00000000000 --- a/Lib/_compression.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Internal classes used by the gzip, lzma and bz2 modules""" - -import io -import sys - -BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size - - -class BaseStream(io.BufferedIOBase): - """Mode-checking helper functions.""" - - def _check_not_closed(self): - if self.closed: - raise ValueError("I/O operation on closed file") - - def _check_can_read(self): - if not self.readable(): - raise io.UnsupportedOperation("File not open for reading") - - def _check_can_write(self): - if not self.writable(): - raise io.UnsupportedOperation("File not open for writing") - - def _check_can_seek(self): - if not self.readable(): - raise io.UnsupportedOperation("Seeking is only supported " - "on files open for reading") - if not self.seekable(): - raise io.UnsupportedOperation("The underlying file object " - "does not support seeking") - - -class DecompressReader(io.RawIOBase): - """Adapts the decompressor API to a RawIOBase reader API""" - - def readable(self): - return True - - def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args): - self._fp = fp - self._eof = False - self._pos = 0 # Current offset in decompressed stream - - # Set to size of decompressed stream once it is known, for SEEK_END - self._size = -1 - - # Save the decompressor factory and arguments. - # If the file contains multiple compressed streams, each - # stream will need a separate decompressor object. A new decompressor - # object is also needed when implementing a backwards seek(). - self._decomp_factory = decomp_factory - self._decomp_args = decomp_args - self._decompressor = self._decomp_factory(**self._decomp_args) - - # Exception class to catch from decompressor signifying invalid - # trailing data to ignore - self._trailing_error = trailing_error - - def close(self): - self._decompressor = None - return super().close() - - def seekable(self): - return self._fp.seekable() - - def readinto(self, b): - with memoryview(b) as view, view.cast("B") as byte_view: - data = self.read(len(byte_view)) - byte_view[:len(data)] = data - return len(data) - - def read(self, size=-1): - if size < 0: - return self.readall() - - if not size or self._eof: - return b"" - data = None # Default if EOF is encountered - # Depending on the input data, our call to the decompressor may not - # return any data. In this case, try again after reading another block. - while True: - if self._decompressor.eof: - rawblock = (self._decompressor.unused_data or - self._fp.read(BUFFER_SIZE)) - if not rawblock: - break - # Continue to next stream. - self._decompressor = self._decomp_factory( - **self._decomp_args) - try: - data = self._decompressor.decompress(rawblock, size) - except self._trailing_error: - # Trailing data isn't a valid compressed stream; ignore it. - break - else: - if self._decompressor.needs_input: - rawblock = self._fp.read(BUFFER_SIZE) - if not rawblock: - raise EOFError("Compressed file ended before the " - "end-of-stream marker was reached") - else: - rawblock = b"" - data = self._decompressor.decompress(rawblock, size) - if data: - break - if not data: - self._eof = True - self._size = self._pos - return b"" - self._pos += len(data) - return data - - def readall(self): - chunks = [] - # sys.maxsize means the max length of output buffer is unlimited, - # so that the whole input buffer can be decompressed within one - # .decompress() call. - while data := self.read(sys.maxsize): - chunks.append(data) - - return b"".join(chunks) - - # Rewind the file to the beginning of the data stream. - def _rewind(self): - self._fp.seek(0) - self._eof = False - self._pos = 0 - self._decompressor = self._decomp_factory(**self._decomp_args) - - def seek(self, offset, whence=io.SEEK_SET): - # Recalculate offset as an absolute file position. - if whence == io.SEEK_SET: - pass - elif whence == io.SEEK_CUR: - offset = self._pos + offset - elif whence == io.SEEK_END: - # Seeking relative to EOF - we need to know the file's size. - if self._size < 0: - while self.read(io.DEFAULT_BUFFER_SIZE): - pass - offset = self._size + offset - else: - raise ValueError("Invalid value for whence: {}".format(whence)) - - # Make it so that offset is the number of bytes to skip forward. - if offset < self._pos: - self._rewind() - else: - offset -= self._pos - - # Read and discard data until we reach the desired position. - while offset > 0: - data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset)) - if not data: - break - offset -= len(data) - - return self._pos - - def tell(self): - """Return the current file position.""" - return self._pos diff --git a/Lib/_dummy_os.py b/Lib/_dummy_os.py index 5bd5ec0a13a..38e287af691 100644 --- a/Lib/_dummy_os.py +++ b/Lib/_dummy_os.py @@ -5,22 +5,30 @@ try: from os import * except ImportError: - import abc + import abc, sys def __getattr__(name): - raise OSError("no os specific module found") + if name in {"_path_normpath", "__path__"}: + raise AttributeError(name) + if name.isupper(): + return 0 + def dummy(*args, **kwargs): + import io + return io.UnsupportedOperation(f"{name}: no os specific module found") + dummy.__name__ = f"dummy_{name}" + return dummy - def _shim(): - import _dummy_os, sys - sys.modules['os'] = _dummy_os - sys.modules['os.path'] = _dummy_os.path + sys.modules['os'] = sys.modules['posix'] = sys.modules[__name__] import posixpath as path - import sys sys.modules['os.path'] = path del sys sep = path.sep + supports_dir_fd = set() + supports_effective_ids = set() + supports_fd = set() + supports_follow_symlinks = set() def fspath(path): diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py index 424b0b3be5e..0630d4e59fa 100644 --- a/Lib/_dummy_thread.py +++ b/Lib/_dummy_thread.py @@ -11,15 +11,35 @@ import _dummy_thread as _thread """ + # Exports only things specified by thread documentation; # skipping obsolete synonyms allocate(), start_new(), exit_thread(). -__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock', - 'interrupt_main', 'LockType', 'RLock', - '_count'] +__all__ = [ + "error", + "start_new_thread", + "exit", + "get_ident", + "allocate_lock", + "interrupt_main", + "LockType", + "RLock", + "_count", + "start_joinable_thread", + "daemon_threads_allowed", + "_shutdown", + "_make_thread_handle", + "_ThreadHandle", + "_get_main_thread_ident", + "_is_main_interpreter", + "_local", +] # A dummy value TIMEOUT_MAX = 2**31 +# Main thread ident for dummy implementation +_MAIN_THREAD_IDENT = -1 + # NOTE: this module can be imported early in the extension building process, # and so top level imports of other modules should be avoided. Instead, all # imports are done when needed on a function-by-function basis. Since threads @@ -27,6 +47,7 @@ error = RuntimeError + def start_new_thread(function, args, kwargs={}): """Dummy implementation of _thread.start_new_thread(). @@ -52,6 +73,7 @@ def start_new_thread(function, args, kwargs={}): pass except: import traceback + traceback.print_exc() _main = True global _interrupt @@ -59,10 +81,58 @@ def start_new_thread(function, args, kwargs={}): _interrupt = False raise KeyboardInterrupt + +def start_joinable_thread(function, handle=None, daemon=True): + """Dummy implementation of _thread.start_joinable_thread(). + + In dummy thread, we just run the function synchronously. + """ + if handle is None: + handle = _ThreadHandle() + try: + function() + except SystemExit: + pass + except: + import traceback + + traceback.print_exc() + handle._set_done() + return handle + + +def daemon_threads_allowed(): + """Dummy implementation of _thread.daemon_threads_allowed().""" + return True + + +def _shutdown(): + """Dummy implementation of _thread._shutdown().""" + pass + + +def _make_thread_handle(ident): + """Dummy implementation of _thread._make_thread_handle().""" + handle = _ThreadHandle() + handle._ident = ident + return handle + + +def _get_main_thread_ident(): + """Dummy implementation of _thread._get_main_thread_ident().""" + return _MAIN_THREAD_IDENT + + +def _is_main_interpreter(): + """Dummy implementation of _thread._is_main_interpreter().""" + return True + + def exit(): """Dummy implementation of _thread.exit().""" raise SystemExit + def get_ident(): """Dummy implementation of _thread.get_ident(). @@ -70,26 +140,31 @@ def get_ident(): available, it is safe to assume that the current process is the only thread. Thus a constant can be safely returned. """ - return -1 + return _MAIN_THREAD_IDENT + def allocate_lock(): """Dummy implementation of _thread.allocate_lock().""" return LockType() + def stack_size(size=None): """Dummy implementation of _thread.stack_size().""" if size is not None: raise error("setting thread stack size not supported") return 0 + def _set_sentinel(): """Dummy implementation of _thread._set_sentinel().""" return LockType() + def _count(): """Dummy implementation of _thread._count().""" return 0 + class LockType(object): """Class implementing dummy implementation of _thread.LockType. @@ -125,6 +200,7 @@ def acquire(self, waitflag=None, timeout=-1): else: if timeout > 0: import time + time.sleep(timeout) return False @@ -153,14 +229,41 @@ def __repr__(self): "locked" if self.locked_status else "unlocked", self.__class__.__module__, self.__class__.__qualname__, - hex(id(self)) + hex(id(self)), ) + +class _ThreadHandle: + """Dummy implementation of _thread._ThreadHandle.""" + + def __init__(self): + self._ident = _MAIN_THREAD_IDENT + self._done = False + + @property + def ident(self): + return self._ident + + def _set_done(self): + self._done = True + + def is_done(self): + return self._done + + def join(self, timeout=None): + # In dummy thread, thread is always done + return + + def __repr__(self): + return f"<_ThreadHandle ident={self._ident}>" + + # Used to signal that interrupt_main was called in a "thread" _interrupt = False # True when not executing in a "thread" _main = True + def interrupt_main(): """Set _interrupt flag to True to have start_new_thread raise KeyboardInterrupt upon exiting.""" @@ -170,6 +273,7 @@ def interrupt_main(): global _interrupt _interrupt = True + class RLock: def __init__(self): self.locked_count = 0 @@ -190,7 +294,7 @@ def release(self): return True def locked(self): - return self.locked_status != 0 + return self.locked_count != 0 def __repr__(self): return "<%s %s.%s object owner=%s count=%s at %s>" % ( @@ -199,5 +303,36 @@ def __repr__(self): self.__class__.__qualname__, get_ident() if self.locked_count else 0, self.locked_count, - hex(id(self)) + hex(id(self)), ) + + +class _local: + """Dummy implementation of _thread._local (thread-local storage).""" + + def __init__(self): + object.__setattr__(self, "_local__impl", {}) + + def __getattribute__(self, name): + if name.startswith("_local__"): + return object.__getattribute__(self, name) + impl = object.__getattribute__(self, "_local__impl") + try: + return impl[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name, value): + if name.startswith("_local__"): + return object.__setattr__(self, name, value) + impl = object.__getattribute__(self, "_local__impl") + impl[name] = value + + def __delattr__(self, name): + if name.startswith("_local__"): + return object.__delattr__(self, name) + impl = object.__getattribute__(self, "_local__impl") + try: + del impl[name] + except KeyError: + raise AttributeError(name) diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py new file mode 100644 index 00000000000..20467a7c2bc --- /dev/null +++ b/Lib/_ios_support.py @@ -0,0 +1,71 @@ +import sys +try: + from ctypes import cdll, c_void_p, c_char_p, util +except ImportError: + # ctypes is an optional module. If it's not present, we're limited in what + # we can tell about the system, but we don't want to prevent the module + # from working. + print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr) + objc = None +else: + # ctypes is available. Load the ObjC library, and wrap the objc_getClass, + # sel_registerName methods + lib = util.find_library("objc") + if lib is None: + # Failed to load the objc library + raise ImportError("ObjC runtime library couldn't be loaded") + + objc = cdll.LoadLibrary(lib) + objc.objc_getClass.restype = c_void_p + objc.objc_getClass.argtypes = [c_char_p] + objc.sel_registerName.restype = c_void_p + objc.sel_registerName.argtypes = [c_char_p] + + +def get_platform_ios(): + # Determine if this is a simulator using the multiarch value + is_simulator = sys.implementation._multiarch.endswith("simulator") + + # We can't use ctypes; abort + if not objc: + return None + + # Most of the methods return ObjC objects + objc.objc_msgSend.restype = c_void_p + # All the methods used have no arguments. + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + + # Equivalent of: + # device = [UIDevice currentDevice] + UIDevice = objc.objc_getClass(b"UIDevice") + SEL_currentDevice = objc.sel_registerName(b"currentDevice") + device = objc.objc_msgSend(UIDevice, SEL_currentDevice) + + # Equivalent of: + # device_systemVersion = [device systemVersion] + SEL_systemVersion = objc.sel_registerName(b"systemVersion") + device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion) + + # Equivalent of: + # device_systemName = [device systemName] + SEL_systemName = objc.sel_registerName(b"systemName") + device_systemName = objc.objc_msgSend(device, SEL_systemName) + + # Equivalent of: + # device_model = [device model] + SEL_model = objc.sel_registerName(b"model") + device_model = objc.objc_msgSend(device, SEL_model) + + # UTF8String returns a const char*; + SEL_UTF8String = objc.sel_registerName(b"UTF8String") + objc.objc_msgSend.restype = c_char_p + + # Equivalent of: + # system = [device_systemName UTF8String] + # release = [device_systemVersion UTF8String] + # model = [device_model UTF8String] + system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode() + release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode() + model = objc.objc_msgSend(device_model, SEL_UTF8String).decode() + + return system, release, model, is_simulator diff --git a/Lib/_markupbase.py b/Lib/_markupbase.py index 3ad7e279960..614f0cd16dd 100644 --- a/Lib/_markupbase.py +++ b/Lib/_markupbase.py @@ -13,7 +13,7 @@ _markedsectionclose = re.compile(r']\s*]\s*>') # An analysis of the MS-Word extensions is available at -# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf +# http://web.archive.org/web/20060321153828/http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf _msmarkedsectionclose = re.compile(r']\s*>') diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py new file mode 100644 index 00000000000..bb55ee423cf --- /dev/null +++ b/Lib/_opcode_metadata.py @@ -0,0 +1,252 @@ +# This file is generated by scripts/generate_opcode_metadata.py +# for RustPython bytecode format (CPython 3.13 compatible opcode numbers). +# Do not edit! + +_specializations = {} + +_specialized_opmap = {} + +opmap = { + 'CACHE': 0, + 'BINARY_SLICE': 1, + 'BUILD_TEMPLATE': 2, + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, + 'CALL_FUNCTION_EX': 4, + 'CHECK_EG_MATCH': 5, + 'CHECK_EXC_MATCH': 6, + 'CLEANUP_THROW': 7, + 'DELETE_SUBSCR': 8, + 'END_FOR': 9, + 'END_SEND': 10, + 'EXIT_INIT_CHECK': 11, + 'FORMAT_SIMPLE': 12, + 'FORMAT_WITH_SPEC': 13, + 'GET_AITER': 14, + 'GET_ANEXT': 15, + 'GET_ITER': 16, + 'RESERVED': 17, + 'GET_LEN': 18, + 'GET_YIELD_FROM_ITER': 19, + 'INTERPRETER_EXIT': 20, + 'LOAD_BUILD_CLASS': 21, + 'LOAD_LOCALS': 22, + 'MAKE_FUNCTION': 23, + 'MATCH_KEYS': 24, + 'MATCH_MAPPING': 25, + 'MATCH_SEQUENCE': 26, + 'NOP': 27, + 'NOT_TAKEN': 28, + 'POP_EXCEPT': 29, + 'POP_ITER': 30, + 'POP_TOP': 31, + 'PUSH_EXC_INFO': 32, + 'PUSH_NULL': 33, + 'RETURN_GENERATOR': 34, + 'RETURN_VALUE': 35, + 'SETUP_ANNOTATIONS': 36, + 'STORE_SLICE': 37, + 'STORE_SUBSCR': 38, + 'TO_BOOL': 39, + 'UNARY_INVERT': 40, + 'UNARY_NEGATIVE': 41, + 'UNARY_NOT': 42, + 'WITH_EXCEPT_START': 43, + 'BINARY_OP': 44, + 'BUILD_INTERPOLATION': 45, + 'BUILD_LIST': 46, + 'BUILD_MAP': 47, + 'BUILD_SET': 48, + 'BUILD_SLICE': 49, + 'BUILD_STRING': 50, + 'BUILD_TUPLE': 51, + 'CALL': 52, + 'CALL_INTRINSIC_1': 53, + 'CALL_INTRINSIC_2': 54, + 'CALL_KW': 55, + 'COMPARE_OP': 56, + 'CONTAINS_OP': 57, + 'CONVERT_VALUE': 58, + 'COPY': 59, + 'COPY_FREE_VARS': 60, + 'DELETE_ATTR': 61, + 'DELETE_DEREF': 62, + 'DELETE_FAST': 63, + 'DELETE_GLOBAL': 64, + 'DELETE_NAME': 65, + 'DICT_MERGE': 66, + 'DICT_UPDATE': 67, + 'END_ASYNC_FOR': 68, + 'EXTENDED_ARG': 69, + 'FOR_ITER': 70, + 'GET_AWAITABLE': 71, + 'IMPORT_FROM': 72, + 'IMPORT_NAME': 73, + 'IS_OP': 74, + 'JUMP_BACKWARD': 75, + 'JUMP_BACKWARD_NO_INTERRUPT': 76, + 'JUMP_FORWARD': 77, + 'LIST_APPEND': 78, + 'LIST_EXTEND': 79, + 'LOAD_ATTR': 80, + 'LOAD_COMMON_CONSTANT': 81, + 'LOAD_CONST': 82, + 'LOAD_DEREF': 83, + 'LOAD_FAST': 84, + 'LOAD_FAST_AND_CLEAR': 85, + 'LOAD_FAST_BORROW': 86, + 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, + 'LOAD_FAST_CHECK': 88, + 'LOAD_FAST_LOAD_FAST': 89, + 'LOAD_FROM_DICT_OR_DEREF': 90, + 'LOAD_FROM_DICT_OR_GLOBALS': 91, + 'LOAD_GLOBAL': 92, + 'LOAD_NAME': 93, + 'LOAD_SMALL_INT': 94, + 'LOAD_SPECIAL': 95, + 'LOAD_SUPER_ATTR': 96, + 'MAKE_CELL': 97, + 'MAP_ADD': 98, + 'MATCH_CLASS': 99, + 'POP_JUMP_IF_FALSE': 100, + 'POP_JUMP_IF_NONE': 101, + 'POP_JUMP_IF_NOT_NONE': 102, + 'POP_JUMP_IF_TRUE': 103, + 'RAISE_VARARGS': 104, + 'RERAISE': 105, + 'SEND': 106, + 'SET_ADD': 107, + 'SET_FUNCTION_ATTRIBUTE': 108, + 'SET_UPDATE': 109, + 'STORE_ATTR': 110, + 'STORE_DEREF': 111, + 'STORE_FAST': 112, + 'STORE_FAST_LOAD_FAST': 113, + 'STORE_FAST_STORE_FAST': 114, + 'STORE_GLOBAL': 115, + 'STORE_NAME': 116, + 'SWAP': 117, + 'UNPACK_EX': 118, + 'UNPACK_SEQUENCE': 119, + 'YIELD_VALUE': 120, + 'RESUME': 128, + 'BINARY_OP_ADD_FLOAT': 129, + 'BINARY_OP_ADD_INT': 130, + 'BINARY_OP_ADD_UNICODE': 131, + 'BINARY_OP_EXTEND': 132, + 'BINARY_OP_MULTIPLY_FLOAT': 133, + 'BINARY_OP_MULTIPLY_INT': 134, + 'BINARY_OP_SUBSCR_DICT': 135, + 'BINARY_OP_SUBSCR_GETITEM': 136, + 'BINARY_OP_SUBSCR_LIST_INT': 137, + 'BINARY_OP_SUBSCR_LIST_SLICE': 138, + 'BINARY_OP_SUBSCR_STR_INT': 139, + 'BINARY_OP_SUBSCR_TUPLE_INT': 140, + 'BINARY_OP_SUBTRACT_FLOAT': 141, + 'BINARY_OP_SUBTRACT_INT': 142, + 'CALL_ALLOC_AND_ENTER_INIT': 143, + 'CALL_BOUND_METHOD_EXACT_ARGS': 144, + 'CALL_BOUND_METHOD_GENERAL': 145, + 'CALL_BUILTIN_CLASS': 146, + 'CALL_BUILTIN_FAST': 147, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, + 'CALL_BUILTIN_O': 149, + 'CALL_ISINSTANCE': 150, + 'CALL_KW_BOUND_METHOD': 151, + 'CALL_KW_NON_PY': 152, + 'CALL_KW_PY': 153, + 'CALL_LEN': 154, + 'CALL_LIST_APPEND': 155, + 'CALL_METHOD_DESCRIPTOR_FAST': 156, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, + 'CALL_METHOD_DESCRIPTOR_O': 159, + 'CALL_NON_PY_GENERAL': 160, + 'CALL_PY_EXACT_ARGS': 161, + 'CALL_PY_GENERAL': 162, + 'CALL_STR_1': 163, + 'CALL_TUPLE_1': 164, + 'CALL_TYPE_1': 165, + 'COMPARE_OP_FLOAT': 166, + 'COMPARE_OP_INT': 167, + 'COMPARE_OP_STR': 168, + 'CONTAINS_OP_DICT': 169, + 'CONTAINS_OP_SET': 170, + 'FOR_ITER_GEN': 171, + 'FOR_ITER_LIST': 172, + 'FOR_ITER_RANGE': 173, + 'FOR_ITER_TUPLE': 174, + 'JUMP_BACKWARD_JIT': 175, + 'JUMP_BACKWARD_NO_JIT': 176, + 'LOAD_ATTR_CLASS': 177, + 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, + 'LOAD_ATTR_INSTANCE_VALUE': 180, + 'LOAD_ATTR_METHOD_LAZY_DICT': 181, + 'LOAD_ATTR_METHOD_NO_DICT': 182, + 'LOAD_ATTR_METHOD_WITH_VALUES': 183, + 'LOAD_ATTR_MODULE': 184, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, + 'LOAD_ATTR_PROPERTY': 187, + 'LOAD_ATTR_SLOT': 188, + 'LOAD_ATTR_WITH_HINT': 189, + 'LOAD_CONST_IMMORTAL': 190, + 'LOAD_CONST_MORTAL': 191, + 'LOAD_GLOBAL_BUILTIN': 192, + 'LOAD_GLOBAL_MODULE': 193, + 'LOAD_SUPER_ATTR_ATTR': 194, + 'LOAD_SUPER_ATTR_METHOD': 195, + 'RESUME_CHECK': 196, + 'SEND_GEN': 197, + 'STORE_ATTR_INSTANCE_VALUE': 198, + 'STORE_ATTR_SLOT': 199, + 'STORE_ATTR_WITH_HINT': 200, + 'STORE_SUBSCR_DICT': 201, + 'STORE_SUBSCR_LIST_INT': 202, + 'TO_BOOL_ALWAYS_TRUE': 203, + 'TO_BOOL_BOOL': 204, + 'TO_BOOL_INT': 205, + 'TO_BOOL_LIST': 206, + 'TO_BOOL_NONE': 207, + 'TO_BOOL_STR': 208, + 'UNPACK_SEQUENCE_LIST': 209, + 'UNPACK_SEQUENCE_TUPLE': 210, + 'UNPACK_SEQUENCE_TWO_TUPLE': 211, + 'INSTRUMENTED_END_FOR': 234, + 'INSTRUMENTED_POP_ITER': 235, + 'INSTRUMENTED_END_SEND': 236, + 'INSTRUMENTED_FOR_ITER': 237, + 'INSTRUMENTED_INSTRUCTION': 238, + 'INSTRUMENTED_JUMP_FORWARD': 239, + 'INSTRUMENTED_NOT_TAKEN': 240, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, + 'INSTRUMENTED_RESUME': 245, + 'INSTRUMENTED_RETURN_VALUE': 246, + 'INSTRUMENTED_YIELD_VALUE': 247, + 'INSTRUMENTED_END_ASYNC_FOR': 248, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, + 'INSTRUMENTED_CALL': 250, + 'INSTRUMENTED_CALL_KW': 251, + 'INSTRUMENTED_CALL_FUNCTION_EX': 252, + 'INSTRUMENTED_JUMP_BACKWARD': 253, + 'INSTRUMENTED_LINE': 254, + 'ENTER_EXECUTOR': 255, + 'ANNOTATIONS_PLACEHOLDER': 256, + 'JUMP': 257, + 'JUMP_IF_FALSE': 258, + 'JUMP_IF_TRUE': 259, + 'JUMP_NO_INTERRUPT': 260, + 'LOAD_CLOSURE': 261, + 'POP_BLOCK': 262, + 'SETUP_CLEANUP': 263, + 'SETUP_FINALLY': 264, + 'SETUP_WITH': 265, + 'STORE_FAST_MAYBE_NULL': 266, +} + +# CPython 3.13 compatible: opcodes < 44 have no argument +HAVE_ARGUMENT = 44 +MIN_INSTRUMENTED_OPCODE = 236 diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index aa66c8b9f41..0cb064fcd79 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -507,6 +507,11 @@ def get_platform_osx(_config_vars, osname, release, machine): # MACOSX_DEPLOYMENT_TARGET. macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') + if macver and '.' not in macver: + # Ensure that the version includes at least a major + # and minor version, even if MACOSX_DEPLOYMENT_TARGET + # is set to a single-label version like "14". + macver += '.0' macrelease = _get_system_version() or macver macver = macver or macrelease diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py new file mode 100644 index 00000000000..55f8c069591 --- /dev/null +++ b/Lib/_py_warnings.py @@ -0,0 +1,869 @@ +"""Python part of the warnings subsystem.""" + +import sys +import _contextvars +import _thread + + +__all__ = ["warn", "warn_explicit", "showwarning", + "formatwarning", "filterwarnings", "simplefilter", + "resetwarnings", "catch_warnings", "deprecated"] + + +# Normally '_wm' is sys.modules['warnings'] but for unit tests it can be +# a different module. User code is allowed to reassign global attributes +# of the 'warnings' module, commonly 'filters' or 'showwarning'. So we +# need to lookup these global attributes dynamically on the '_wm' object, +# rather than binding them earlier. The code in this module consistently uses +# '_wm.' rather than using the globals of this module. If the +# '_warnings' C extension is in use, some globals are replaced by functions +# and variables defined in that extension. +_wm = None + + +def _set_module(module): + global _wm + _wm = module + + +# filters contains a sequence of filter 5-tuples +# The components of the 5-tuple are: +# - an action: error, ignore, always, all, default, module, or once +# - a compiled regex that must match the warning message +# - a class representing the warning category +# - a compiled regex that must match the module that is being warned +# - a line number for the line being warning, or 0 to mean any line +# If either if the compiled regexs are None, match anything. +filters = [] + + +defaultaction = "default" +onceregistry = {} +_lock = _thread.RLock() +_filters_version = 1 + + +# If true, catch_warnings() will use a context var to hold the modified +# filters list. Otherwise, catch_warnings() will operate on the 'filters' +# global of the warnings module. +_use_context = sys.flags.context_aware_warnings + + +class _Context: + def __init__(self, filters): + self._filters = filters + self.log = None # if set to a list, logging is enabled + + def copy(self): + context = _Context(self._filters[:]) + if self.log is not None: + context.log = self.log + return context + + def _record_warning(self, msg): + self.log.append(msg) + + +class _GlobalContext(_Context): + def __init__(self): + self.log = None + + @property + def _filters(self): + # Since there is quite a lot of code that assigns to + # warnings.filters, this needs to return the current value of + # the module global. + try: + return _wm.filters + except AttributeError: + # 'filters' global was deleted. Do we need to actually handle this case? + return [] + + +_global_context = _GlobalContext() + + +_warnings_context = _contextvars.ContextVar('warnings_context') + + +def _get_context(): + if not _use_context: + return _global_context + try: + return _wm._warnings_context.get() + except LookupError: + return _global_context + + +def _set_context(context): + assert _use_context + _wm._warnings_context.set(context) + + +def _new_context(): + assert _use_context + old_context = _wm._get_context() + new_context = old_context.copy() + _wm._set_context(new_context) + return old_context, new_context + + +def _get_filters(): + """Return the current list of filters. This is a non-public API used by + module functions and by the unit tests.""" + return _wm._get_context()._filters + + +def _filters_mutated_lock_held(): + _wm._filters_version += 1 + + +def showwarning(message, category, filename, lineno, file=None, line=None): + """Hook to write a warning to a file; replace if you like.""" + msg = _wm.WarningMessage(message, category, filename, lineno, file, line) + _wm._showwarnmsg_impl(msg) + + +def formatwarning(message, category, filename, lineno, line=None): + """Function to format a warning the standard way.""" + msg = _wm.WarningMessage(message, category, filename, lineno, None, line) + return _wm._formatwarnmsg_impl(msg) + + +def _showwarnmsg_impl(msg): + context = _wm._get_context() + if context.log is not None: + context._record_warning(msg) + return + file = msg.file + if file is None: + file = sys.stderr + if file is None: + # sys.stderr is None when run with pythonw.exe: + # warnings get lost + return + text = _wm._formatwarnmsg(msg) + try: + file.write(text) + except OSError: + # the file (probably stderr) is invalid - this warning gets lost. + pass + + +def _formatwarnmsg_impl(msg): + category = msg.category.__name__ + s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n" + + if msg.line is None: + try: + import linecache + line = linecache.getline(msg.filename, msg.lineno) + except Exception: + # When a warning is logged during Python shutdown, linecache + # and the import machinery don't work anymore + line = None + linecache = None + else: + line = msg.line + if line: + line = line.strip() + s += " %s\n" % line + + if msg.source is not None: + try: + import tracemalloc + # Logging a warning should not raise a new exception: + # catch Exception, not only ImportError and RecursionError. + except Exception: + # don't suggest to enable tracemalloc if it's not available + suggest_tracemalloc = False + tb = None + else: + try: + suggest_tracemalloc = not tracemalloc.is_tracing() + tb = tracemalloc.get_object_traceback(msg.source) + except Exception: + # When a warning is logged during Python shutdown, tracemalloc + # and the import machinery don't work anymore + suggest_tracemalloc = False + tb = None + + if tb is not None: + s += 'Object allocated at (most recent call last):\n' + for frame in tb: + s += (' File "%s", lineno %s\n' + % (frame.filename, frame.lineno)) + + try: + if linecache is not None: + line = linecache.getline(frame.filename, frame.lineno) + else: + line = None + except Exception: + line = None + if line: + line = line.strip() + s += ' %s\n' % line + elif suggest_tracemalloc: + s += (f'{category}: Enable tracemalloc to get the object ' + f'allocation traceback\n') + return s + + +# Keep a reference to check if the function was replaced +_showwarning_orig = showwarning + + +def _showwarnmsg(msg): + """Hook to write a warning to a file; replace if you like.""" + try: + sw = _wm.showwarning + except AttributeError: + pass + else: + if sw is not _showwarning_orig: + # warnings.showwarning() was replaced + if not callable(sw): + raise TypeError("warnings.showwarning() must be set to a " + "function or method") + + sw(msg.message, msg.category, msg.filename, msg.lineno, + msg.file, msg.line) + return + _wm._showwarnmsg_impl(msg) + + +# Keep a reference to check if the function was replaced +_formatwarning_orig = formatwarning + + +def _formatwarnmsg(msg): + """Function to format a warning the standard way.""" + try: + fw = _wm.formatwarning + except AttributeError: + pass + else: + if fw is not _formatwarning_orig: + # warnings.formatwarning() was replaced + return fw(msg.message, msg.category, + msg.filename, msg.lineno, msg.line) + return _wm._formatwarnmsg_impl(msg) + + +def filterwarnings(action, message="", category=Warning, module="", lineno=0, + append=False): + """Insert an entry into the list of warnings filters (at the front). + + 'action' -- one of "error", "ignore", "always", "all", "default", "module", + or "once" + 'message' -- a regex that the warning message must match + 'category' -- a class that the warning must be a subclass of + 'module' -- a regex that the module name must match + 'lineno' -- an integer line number, 0 matches all warnings + 'append' -- if true, append to the list of filters + """ + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(message, str): + raise TypeError("message must be a string") + if not isinstance(category, type) or not issubclass(category, Warning): + raise TypeError("category must be a Warning subclass") + if not isinstance(module, str): + raise TypeError("module must be a string") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") + + if message or module: + import re + + if message: + message = re.compile(message, re.I) + else: + message = None + if module: + module = re.compile(module) + else: + module = None + + _wm._add_filter(action, message, category, module, lineno, append=append) + + +def simplefilter(action, category=Warning, lineno=0, append=False): + """Insert a simple entry into the list of warnings filters (at the front). + + A simple filter matches all modules and messages. + 'action' -- one of "error", "ignore", "always", "all", "default", "module", + or "once" + 'category' -- a class that the warning must be a subclass of + 'lineno' -- an integer line number, 0 matches all warnings + 'append' -- if true, append to the list of filters + """ + if action not in {"error", "ignore", "always", "all", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") + _wm._add_filter(action, None, category, None, lineno, append=append) + + +def _filters_mutated(): + # Even though this function is not part of the public API, it's used by + # a fair amount of user code. + with _wm._lock: + _wm._filters_mutated_lock_held() + + +def _add_filter(*item, append): + with _wm._lock: + filters = _wm._get_filters() + if not append: + # Remove possible duplicate filters, so new one will be placed + # in correct place. If append=True and duplicate exists, do nothing. + try: + filters.remove(item) + except ValueError: + pass + filters.insert(0, item) + else: + if item not in filters: + filters.append(item) + _wm._filters_mutated_lock_held() + + +def resetwarnings(): + """Clear the list of warning filters, so that no filters are active.""" + with _wm._lock: + del _wm._get_filters()[:] + _wm._filters_mutated_lock_held() + + +class _OptionError(Exception): + """Exception used by option processing helpers.""" + pass + + +# Helper to process -W options passed via sys.warnoptions +def _processoptions(args): + for arg in args: + try: + _wm._setoption(arg) + except _wm._OptionError as msg: + print("Invalid -W option ignored:", msg, file=sys.stderr) + + +# Helper for _processoptions() +def _setoption(arg): + parts = arg.split(':') + if len(parts) > 5: + raise _wm._OptionError("too many fields (max 5): %r" % (arg,)) + while len(parts) < 5: + parts.append('') + action, message, category, module, lineno = [s.strip() + for s in parts] + action = _wm._getaction(action) + category = _wm._getcategory(category) + if message or module: + import re + if message: + message = re.escape(message) + if module: + module = re.escape(module) + r'\z' + if lineno: + try: + lineno = int(lineno) + if lineno < 0: + raise ValueError + except (ValueError, OverflowError): + raise _wm._OptionError("invalid lineno %r" % (lineno,)) from None + else: + lineno = 0 + _wm.filterwarnings(action, message, category, module, lineno) + + +# Helper for _setoption() +def _getaction(action): + if not action: + return "default" + for a in ('default', 'always', 'all', 'ignore', 'module', 'once', 'error'): + if a.startswith(action): + return a + raise _wm._OptionError("invalid action: %r" % (action,)) + + +# Helper for _setoption() +def _getcategory(category): + if not category: + return Warning + if '.' not in category: + import builtins as m + klass = category + else: + module, _, klass = category.rpartition('.') + try: + m = __import__(module, None, None, [klass]) + except ImportError: + raise _wm._OptionError("invalid module name: %r" % (module,)) from None + try: + cat = getattr(m, klass) + except AttributeError: + raise _wm._OptionError("unknown warning category: %r" % (category,)) from None + if not issubclass(cat, Warning): + raise _wm._OptionError("invalid warning category: %r" % (category,)) + return cat + + +def _is_internal_filename(filename): + return 'importlib' in filename and '_bootstrap' in filename + + +def _is_filename_to_skip(filename, skip_file_prefixes): + return any(filename.startswith(prefix) for prefix in skip_file_prefixes) + + +def _is_internal_frame(frame): + """Signal whether the frame is an internal CPython implementation detail.""" + return _is_internal_filename(frame.f_code.co_filename) + + +def _next_external_frame(frame, skip_file_prefixes): + """Find the next frame that doesn't involve Python or user internals.""" + frame = frame.f_back + while frame is not None and ( + _is_internal_filename(filename := frame.f_code.co_filename) or + _is_filename_to_skip(filename, skip_file_prefixes)): + frame = frame.f_back + return frame + + +# Code typically replaced by _warnings +def warn(message, category=None, stacklevel=1, source=None, + *, skip_file_prefixes=()): + """Issue a warning, or maybe ignore it or raise an exception.""" + # Check if message is already a Warning object + if isinstance(message, Warning): + category = message.__class__ + # Check category argument + if category is None: + category = UserWarning + if not (isinstance(category, type) and issubclass(category, Warning)): + raise TypeError("category must be a Warning subclass, " + "not '{:s}'".format(type(category).__name__)) + if not isinstance(skip_file_prefixes, tuple): + # The C version demands a tuple for implementation performance. + raise TypeError('skip_file_prefixes must be a tuple of strs.') + if skip_file_prefixes: + stacklevel = max(2, stacklevel) + # Get context information + try: + if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)): + # If frame is too small to care or if the warning originated in + # internal code, then do not try to hide any frames. + frame = sys._getframe(stacklevel) + else: + frame = sys._getframe(1) + # Look for one frame less since the above line starts us off. + for x in range(stacklevel-1): + frame = _next_external_frame(frame, skip_file_prefixes) + if frame is None: + raise ValueError + except ValueError: + globals = sys.__dict__ + filename = "" + lineno = 0 + else: + globals = frame.f_globals + filename = frame.f_code.co_filename + lineno = frame.f_lineno + if '__name__' in globals: + module = globals['__name__'] + else: + module = "" + registry = globals.setdefault("__warningregistry__", {}) + _wm.warn_explicit( + message, + category, + filename, + lineno, + module, + registry, + globals, + source=source, + ) + + +def warn_explicit(message, category, filename, lineno, + module=None, registry=None, module_globals=None, + source=None): + lineno = int(lineno) + if module is None: + module = filename or "" + if module[-3:].lower() == ".py": + module = module[:-3] # XXX What about leading pathname? + if isinstance(message, Warning): + text = str(message) + category = message.__class__ + else: + text = message + message = category(message) + key = (text, category, lineno) + with _wm._lock: + if registry is None: + registry = {} + if registry.get('version', 0) != _wm._filters_version: + registry.clear() + registry['version'] = _wm._filters_version + # Quick test for common case + if registry.get(key): + return + # Search the filters + for item in _wm._get_filters(): + action, msg, cat, mod, ln = item + if ((msg is None or msg.match(text)) and + issubclass(category, cat) and + (mod is None or mod.match(module)) and + (ln == 0 or lineno == ln)): + break + else: + action = _wm.defaultaction + # Early exit actions + if action == "ignore": + return + + if action == "error": + raise message + # Other actions + if action == "once": + registry[key] = 1 + oncekey = (text, category) + if _wm.onceregistry.get(oncekey): + return + _wm.onceregistry[oncekey] = 1 + elif action in {"always", "all"}: + pass + elif action == "module": + registry[key] = 1 + altkey = (text, category, 0) + if registry.get(altkey): + return + registry[altkey] = 1 + elif action == "default": + registry[key] = 1 + else: + # Unrecognized actions are errors + raise RuntimeError( + "Unrecognized action (%r) in warnings.filters:\n %s" % + (action, item)) + + # Prime the linecache for formatting, in case the + # "file" is actually in a zipfile or something. + import linecache + linecache.getlines(filename, module_globals) + + # Print message and context + msg = _wm.WarningMessage(message, category, filename, lineno, source=source) + _wm._showwarnmsg(msg) + + +class WarningMessage(object): + + _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", + "line", "source") + + def __init__(self, message, category, filename, lineno, file=None, + line=None, source=None): + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.file = file + self.line = line + self.source = source + self._category_name = category.__name__ if category else None + + def __str__(self): + return ("{message : %r, category : %r, filename : %r, lineno : %s, " + "line : %r}" % (self.message, self._category_name, + self.filename, self.lineno, self.line)) + + def __repr__(self): + return f'<{type(self).__qualname__} {self}>' + + +class catch_warnings(object): + + """A context manager that copies and restores the warnings filter upon + exiting the context. + + The 'record' argument specifies whether warnings should be captured by a + custom implementation of warnings.showwarning() and be appended to a list + returned by the context manager. Otherwise None is returned by the context + manager. The objects appended to the list are arguments whose attributes + mirror the arguments to showwarning(). + + The 'module' argument is to specify an alternative module to the module + named 'warnings' and imported under that name. This argument is only useful + when testing the warnings module itself. + + If the 'action' argument is not None, the remaining arguments are passed + to warnings.simplefilter() as if it were called immediately on entering the + context. + """ + + def __init__(self, *, record=False, module=None, + action=None, category=Warning, lineno=0, append=False): + """Specify whether to record warnings and if an alternative module + should be used other than sys.modules['warnings']. + + """ + self._record = record + self._module = sys.modules['warnings'] if module is None else module + self._entered = False + if action is None: + self._filter = None + else: + self._filter = (action, category, lineno, append) + + def __repr__(self): + args = [] + if self._record: + args.append("record=True") + if self._module is not sys.modules['warnings']: + args.append("module=%r" % self._module) + name = type(self).__name__ + return "%s(%s)" % (name, ", ".join(args)) + + def __enter__(self): + if self._entered: + raise RuntimeError("Cannot enter %r twice" % self) + self._entered = True + with _wm._lock: + if _use_context: + self._saved_context, context = self._module._new_context() + else: + context = None + self._filters = self._module.filters + self._module.filters = self._filters[:] + self._showwarning = self._module.showwarning + self._showwarnmsg_impl = self._module._showwarnmsg_impl + self._module._filters_mutated_lock_held() + if self._record: + if _use_context: + context.log = log = [] + else: + log = [] + self._module._showwarnmsg_impl = log.append + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig + else: + log = None + if self._filter is not None: + self._module.simplefilter(*self._filter) + return log + + def __exit__(self, *exc_info): + if not self._entered: + raise RuntimeError("Cannot exit %r without entering first" % self) + with _wm._lock: + if _use_context: + self._module._warnings_context.set(self._saved_context) + else: + self._module.filters = self._filters + self._module.showwarning = self._showwarning + self._module._showwarnmsg_impl = self._showwarnmsg_impl + self._module._filters_mutated_lock_held() + + +class deprecated: + """Indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, the type checker + will generate a diagnostic on usage of the deprecated object. + + Usage: + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the ``@overload`` decorator for the attribute to + exist on the overload as returned by ``get_overloads()``. + + See PEP 702 for details. + + """ + def __init__( + self, + message: str, + /, + *, + category: type[Warning] | None = DeprecationWarning, + stacklevel: int = 1, + ) -> None: + if not isinstance(message, str): + raise TypeError( + f"Expected an object of type str for 'message', not {type(message).__name__!r}" + ) + self.message = message + self.category = category + self.stacklevel = stacklevel + + def __call__(self, arg, /): + # Make sure the inner functions created below don't + # retain a reference to self. + msg = self.message + category = self.category + stacklevel = self.stacklevel + if category is None: + arg.__deprecated__ = msg + return arg + elif isinstance(arg, type): + import functools + from types import MethodType + + original_new = arg.__new__ + + @functools.wraps(original_new) + def __new__(cls, /, *args, **kwargs): + if cls is arg: + _wm.warn(msg, category=category, stacklevel=stacklevel + 1) + if original_new is not object.__new__: + return original_new(cls, *args, **kwargs) + # Mirrors a similar check in object.__new__. + elif cls.__init__ is object.__init__ and (args or kwargs): + raise TypeError(f"{cls.__name__}() takes no arguments") + else: + return original_new(cls) + + arg.__new__ = staticmethod(__new__) + + if "__init_subclass__" in arg.__dict__: + # __init_subclass__ is directly present on the decorated class. + # Synthesize a wrapper that calls this method directly. + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python). + # Otherwise, it likely means it's a builtin such as + # object's implementation of __init_subclass__. + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ + + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + _wm.warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + else: + def __init_subclass__(cls, *args, **kwargs): + _wm.warn(msg, category=category, stacklevel=stacklevel + 1) + return super(arg, cls).__init_subclass__(*args, **kwargs) + + arg.__init_subclass__ = classmethod(__init_subclass__) + + arg.__deprecated__ = __new__.__deprecated__ = msg + __init_subclass__.__deprecated__ = msg + return arg + elif callable(arg): + import functools + import inspect + + @functools.wraps(arg) + def wrapper(*args, **kwargs): + _wm.warn(msg, category=category, stacklevel=stacklevel + 1) + return arg(*args, **kwargs) + + if inspect.iscoroutinefunction(arg): + wrapper = inspect.markcoroutinefunction(wrapper) + + arg.__deprecated__ = wrapper.__deprecated__ = msg + return wrapper + else: + raise TypeError( + "@deprecated decorator with non-None category must be applied to " + f"a class or callable, not {arg!r}" + ) + + +_DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" + + +def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): + """Warn that *name* is deprecated or should be removed. + + RuntimeError is raised if *remove* specifies a major/minor tuple older than + the current Python version or the same version but past the alpha. + + The *message* argument is formatted with *name* and *remove* as a Python + version tuple (e.g. (3, 11)). + + """ + remove_formatted = f"{remove[0]}.{remove[1]}" + if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"): + msg = f"{name!r} was slated for removal after Python {remove_formatted} alpha" + raise RuntimeError(msg) + else: + msg = message.format(name=name, remove=remove_formatted) + _wm.warn(msg, DeprecationWarning, stacklevel=3) + + +# Private utility function called by _PyErr_WarnUnawaitedCoroutine +def _warn_unawaited_coroutine(coro): + msg_lines = [ + f"coroutine '{coro.__qualname__}' was never awaited\n" + ] + if coro.cr_origin is not None: + import linecache, traceback + def extract(): + for filename, lineno, funcname in reversed(coro.cr_origin): + line = linecache.getline(filename, lineno) + yield (filename, lineno, funcname, line) + msg_lines.append("Coroutine created at (most recent call last)\n") + msg_lines += traceback.format_list(list(extract())) + msg = "".join(msg_lines).rstrip("\n") + # Passing source= here means that if the user happens to have tracemalloc + # enabled and tracking where the coroutine was created, the warning will + # contain that traceback. This does mean that if they have *both* + # coroutine origin tracking *and* tracemalloc enabled, they'll get two + # partially-redundant tracebacks. If we wanted to be clever we could + # probably detect this case and avoid it, but for now we don't bother. + _wm.warn( + msg, category=RuntimeWarning, stacklevel=2, source=coro + ) + + +def _setup_defaults(): + # Several warning categories are ignored by default in regular builds + if hasattr(sys, 'gettotalrefcount'): + return + _wm.filterwarnings("default", category=DeprecationWarning, module="__main__", append=1) + _wm.simplefilter("ignore", category=DeprecationWarning, append=1) + _wm.simplefilter("ignore", category=PendingDeprecationWarning, append=1) + _wm.simplefilter("ignore", category=ImportWarning, append=1) + _wm.simplefilter("ignore", category=ResourceWarning, append=1) diff --git a/Lib/_pycodecs.py b/Lib/_pycodecs.py index 0741504cc9e..98dec3c614d 100644 --- a/Lib/_pycodecs.py +++ b/Lib/_pycodecs.py @@ -22,10 +22,10 @@ The builtin Unicode codecs use the following interface: - _encode(Unicode_object[,errors='strict']) -> + _encode(Unicode_object[,errors='strict']) -> (string object, bytes consumed) - _decode(char_buffer_obj[,errors='strict']) -> + _decode(char_buffer_obj[,errors='strict']) -> (Unicode object, bytes consumed) _encode() interfaces also accept non-Unicode object as @@ -44,47 +44,76 @@ From PyPy v1.0.0 """ -#from unicodecodec import * - -__all__ = ['register', 'lookup', 'lookup_error', 'register_error', 'encode', 'decode', - 'latin_1_encode', 'mbcs_decode', 'readbuffer_encode', 'escape_encode', - 'utf_8_decode', 'raw_unicode_escape_decode', 'utf_7_decode', - 'unicode_escape_encode', 'latin_1_decode', 'utf_16_decode', - 'unicode_escape_decode', 'ascii_decode', 'charmap_encode', 'charmap_build', - 'unicode_internal_encode', 'unicode_internal_decode', 'utf_16_ex_decode', - 'escape_decode', 'charmap_decode', 'utf_7_encode', 'mbcs_encode', - 'ascii_encode', 'utf_16_encode', 'raw_unicode_escape_encode', 'utf_8_encode', - 'utf_16_le_encode', 'utf_16_be_encode', 'utf_16_le_decode', 'utf_16_be_decode',] +# from unicodecodec import * + +__all__ = [ + "register", + "lookup", + "lookup_error", + "register_error", + "encode", + "decode", + "latin_1_encode", + "mbcs_decode", + "readbuffer_encode", + "escape_encode", + "utf_8_decode", + "raw_unicode_escape_decode", + "utf_7_decode", + "unicode_escape_encode", + "latin_1_decode", + "utf_16_decode", + "unicode_escape_decode", + "ascii_decode", + "charmap_encode", + "charmap_build", + "unicode_internal_encode", + "unicode_internal_decode", + "utf_16_ex_decode", + "escape_decode", + "charmap_decode", + "utf_7_encode", + "mbcs_encode", + "ascii_encode", + "utf_16_encode", + "raw_unicode_escape_encode", + "utf_8_encode", + "utf_16_le_encode", + "utf_16_be_encode", + "utf_16_le_decode", + "utf_16_be_decode", + "utf_32_ex_decode", +] import sys import warnings from _codecs import * -def latin_1_encode( obj, errors='strict'): - """None - """ +def latin_1_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeLatin1(obj, len(obj), errors) res = bytes(res) return res, len(obj) + + # XXX MBCS codec might involve ctypes ? def mbcs_decode(): - """None - """ + """None""" pass -def readbuffer_encode( obj, errors='strict'): - """None - """ + +def readbuffer_encode(obj, errors="strict"): + """None""" if isinstance(obj, str): res = obj.encode() else: res = bytes(obj) return res, len(obj) -def escape_encode( obj, errors='strict'): - """None - """ + +def escape_encode(obj, errors="strict"): + """None""" if not isinstance(obj, bytes): raise TypeError("must be bytes") s = repr(obj).encode() @@ -93,85 +122,88 @@ def escape_encode( obj, errors='strict'): v = v.replace(b"'", b"\\'").replace(b'\\"', b'"') return v, len(obj) -def raw_unicode_escape_decode( data, errors='strict', final=False): - """None - """ - res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final) - res = ''.join(res) - return res, len(data) -def utf_7_decode( data, errors='strict'): - """None - """ - res = PyUnicode_DecodeUTF7(data, len(data), errors) - res = ''.join(res) - return res, len(data) +def raw_unicode_escape_decode(data, errors="strict", final=True): + """None""" + res, consumed = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final) + res = "".join(res) + return res, consumed -def unicode_escape_encode( obj, errors='strict'): - """None - """ + +def utf_7_decode(data, errors="strict", final=False): + """None""" + res, consumed = PyUnicode_DecodeUTF7(data, len(data), errors, final) + res = "".join(res) + return res, consumed + + +def unicode_escape_encode(obj, errors="strict"): + """None""" res = unicodeescape_string(obj, len(obj), 0) - res = b''.join(res) + res = b"".join(res) return res, len(obj) -def latin_1_decode( data, errors='strict'): - """None - """ + +def latin_1_decode(data, errors="strict"): + """None""" res = PyUnicode_DecodeLatin1(data, len(data), errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def utf_16_decode( data, errors='strict', final=False): - """None - """ + +def utf_16_decode(data, errors="strict", final=False): + """None""" consumed = len(data) if final: consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final) - res = ''.join(res) + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "native", final + ) + res = "".join(res) return res, consumed -def unicode_escape_decode( data, errors='strict', final=False): - """None - """ - res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final) - res = ''.join(res) - return res, len(data) +def unicode_escape_decode(data, errors="strict", final=True): + """None""" + res, consumed = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final) + res = "".join(res) + return res, consumed -def ascii_decode( data, errors='strict'): - """None - """ + +def ascii_decode(data, errors="strict"): + """None""" res = PyUnicode_DecodeASCII(data, len(data), errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def charmap_encode(obj, errors='strict', mapping='latin-1'): - """None - """ + +def charmap_encode(obj, errors="strict", mapping="latin-1"): + """None""" res = PyUnicode_EncodeCharmap(obj, len(obj), mapping, errors) res = bytes(res) return res, len(obj) + def charmap_build(s): return {ord(c): i for i, c in enumerate(s)} + if sys.maxunicode == 65535: unicode_bytes = 2 else: unicode_bytes = 4 -def unicode_internal_encode( obj, errors='strict'): - """None - """ + +def unicode_internal_encode(obj, errors="strict"): + """None""" if type(obj) == str: p = bytearray() t = [ord(x) for x in obj] for i in t: b = bytearray() for j in range(unicode_bytes): - b.append(i%256) + b.append(i % 256) i >>= 8 if sys.byteorder == "big": b.reverse() @@ -179,12 +211,12 @@ def unicode_internal_encode( obj, errors='strict'): res = bytes(p) return res, len(res) else: - res = "You can do better than this" # XXX make this right + res = "You can do better than this" # XXX make this right return res, len(res) -def unicode_internal_decode( unistr, errors='strict'): - """None - """ + +def unicode_internal_decode(unistr, errors="strict"): + """None""" if type(unistr) == str: return unistr, len(unistr) else: @@ -198,165 +230,418 @@ def unicode_internal_decode( unistr, errors='strict'): start = 0 stop = unicode_bytes step = 1 - while i < len(unistr)-unicode_bytes+1: + while i < len(unistr) - unicode_bytes + 1: t = 0 h = 0 for j in range(start, stop, step): - t += ord(unistr[i+j])<<(h*8) + t += ord(unistr[i + j]) << (h * 8) h += 1 i += unicode_bytes p += chr(t) - res = ''.join(p) + res = "".join(p) return res, len(res) -def utf_16_ex_decode( data, errors='strict', byteorder=0, final=0): - """None - """ + +def utf_16_ex_decode(data, errors="strict", byteorder=0, final=0): + """None""" if byteorder == 0: - bm = 'native' + bm = "native" elif byteorder == -1: - bm = 'little' + bm = "little" else: - bm = 'big' + bm = "big" consumed = len(data) if final: consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, bm, final) - res = ''.join(res) + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, bm, final + ) + res = "".join(res) return res, consumed, byteorder -# XXX needs error messages when the input is invalid -def escape_decode(data, errors='strict'): - """None - """ + +def utf_32_ex_decode(data, errors="strict", byteorder=0, final=0): + """None""" + if byteorder == 0: + if len(data) < 4: + if final and len(data): + if sys.byteorder == "little": + bm = "little" + else: + bm = "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, bm, final + ) + return "".join(res), consumed, 0 + return "", 0, 0 + if data[0:4] == b"\xff\xfe\x00\x00": + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "little", final + ) + return "".join(res), consumed + 4, -1 + if data[0:4] == b"\x00\x00\xfe\xff": + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "big", final + ) + return "".join(res), consumed + 4, 1 + if sys.byteorder == "little": + bm = "little" + else: + bm = "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, bm, final + ) + return "".join(res), consumed, 0 + + if byteorder == -1: + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "little", final + ) + return "".join(res), consumed, -1 + + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "big", final + ) + return "".join(res), consumed, 1 + + +def _is_hex_digit(b): + return ( + 0x30 <= b <= 0x39 # 0-9 + or 0x41 <= b <= 0x46 # A-F + or 0x61 <= b <= 0x66 + ) # a-f + + +def escape_decode(data, errors="strict"): + if isinstance(data, str): + data = data.encode("latin-1") l = len(data) i = 0 res = bytearray() while i < l: - - if data[i] == '\\': + if data[i] == 0x5C: # '\\' i += 1 if i >= l: raise ValueError("Trailing \\ in string") - else: - if data[i] == '\\': - res += b'\\' - elif data[i] == 'n': - res += b'\n' - elif data[i] == 't': - res += b'\t' - elif data[i] == 'r': - res += b'\r' - elif data[i] == 'b': - res += b'\b' - elif data[i] == '\'': - res += b'\'' - elif data[i] == '\"': - res += b'\"' - elif data[i] == 'f': - res += b'\f' - elif data[i] == 'a': - res += b'\a' - elif data[i] == 'v': - res += b'\v' - elif '0' <= data[i] <= '9': - # emulate a strange wrap-around behavior of CPython: - # \400 is the same as \000 because 0400 == 256 - octal = data[i:i+3] - res.append(int(octal, 8) & 0xFF) - i += 2 - elif data[i] == 'x': - hexa = data[i+1:i+3] - res.append(int(hexa, 16)) + ch = data[i] + if ch == 0x5C: + res.append(0x5C) # \\ + elif ch == 0x27: + res.append(0x27) # \' + elif ch == 0x22: + res.append(0x22) # \" + elif ch == 0x61: + res.append(0x07) # \a + elif ch == 0x62: + res.append(0x08) # \b + elif ch == 0x66: + res.append(0x0C) # \f + elif ch == 0x6E: + res.append(0x0A) # \n + elif ch == 0x72: + res.append(0x0D) # \r + elif ch == 0x74: + res.append(0x09) # \t + elif ch == 0x76: + res.append(0x0B) # \v + elif ch == 0x0A: + pass # \ continuation + elif 0x30 <= ch <= 0x37: # \0-\7 octal + val = ch - 0x30 + if i + 1 < l and 0x30 <= data[i + 1] <= 0x37: + i += 1 + val = (val << 3) | (data[i] - 0x30) + if i + 1 < l and 0x30 <= data[i + 1] <= 0x37: + i += 1 + val = (val << 3) | (data[i] - 0x30) + res.append(val & 0xFF) + elif ch == 0x78: # \x hex + hex_count = 0 + for j in range(1, 3): + if i + j < l and _is_hex_digit(data[i + j]): + hex_count += 1 + else: + break + if hex_count < 2: + if errors == "strict": + raise ValueError("invalid \\x escape at position %d" % (i - 1)) + elif errors == "replace": + res.append(0x3F) # '?' + i += hex_count + else: + res.append(int(bytes(data[i + 1 : i + 3]), 16)) i += 2 + else: + import warnings + + warnings.warn( + '"\\%c" is an invalid escape sequence' % ch + if 0x20 <= ch < 0x7F + else '"\\x%02x" is an invalid escape sequence' % ch, + DeprecationWarning, + stacklevel=2, + ) + res.append(0x5C) + res.append(ch) else: res.append(data[i]) i += 1 - res = bytes(res) - return res, len(res) + return bytes(res), l + -def charmap_decode( data, errors='strict', mapping=None): - """None - """ +def charmap_decode(data, errors="strict", mapping=None): + """None""" res = PyUnicode_DecodeCharmap(data, len(data), mapping, errors) - res = ''.join(res) + res = "".join(res) return res, len(data) -def utf_7_encode( obj, errors='strict'): - """None - """ +def utf_7_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeUTF7(obj, len(obj), 0, 0, errors) - res = b''.join(res) + res = b"".join(res) return res, len(obj) -def mbcs_encode( obj, errors='strict'): - """None - """ + +def mbcs_encode(obj, errors="strict"): + """None""" pass + + ## return (PyUnicode_EncodeMBCS( -## (obj), +## (obj), ## len(obj), ## errors), ## len(obj)) - -def ascii_encode( obj, errors='strict'): - """None - """ + +def ascii_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeASCII(obj, len(obj), errors) res = bytes(res) return res, len(obj) -def utf_16_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'native') + +def utf_16_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "native") res = bytes(res) return res, len(obj) -def raw_unicode_escape_encode( obj, errors='strict'): - """None - """ + +def raw_unicode_escape_encode(obj, errors="strict"): + """None""" res = PyUnicode_EncodeRawUnicodeEscape(obj, len(obj)) res = bytes(res) return res, len(obj) -def utf_16_le_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'little') + +def utf_16_le_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "little") res = bytes(res) return res, len(obj) -def utf_16_be_encode( obj, errors='strict'): - """None - """ - res = PyUnicode_EncodeUTF16(obj, len(obj), errors, 'big') + +def utf_16_be_encode(obj, errors="strict"): + """None""" + res = PyUnicode_EncodeUTF16(obj, len(obj), errors, "big") res = bytes(res) return res, len(obj) -def utf_16_le_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'little', final) - res = ''.join(res) + +def utf_16_le_decode(data, errors="strict", final=0): + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "little", final + ) + res = "".join(res) return res, consumed -def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): - """None - """ - consumed = len(data) - if final: - consumed = 0 - res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'big', final) - res = ''.join(res) + +def utf_16_be_decode(data, errors="strict", final=0): + res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful( + data, len(data), errors, "big", final + ) + res = "".join(res) return res, consumed +def STORECHAR32(ch, byteorder): + """Store a 32-bit character as 4 bytes in the specified byte order.""" + b0 = ch & 0xFF + b1 = (ch >> 8) & 0xFF + b2 = (ch >> 16) & 0xFF + b3 = (ch >> 24) & 0xFF + if byteorder == "little": + return [b0, b1, b2, b3] + else: # big-endian + return [b3, b2, b1, b0] + + +def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"): + """Encode a Unicode string to UTF-32.""" + p = [] + bom = sys.byteorder + + if byteorder == "native": + bom = sys.byteorder + # Add BOM for native encoding + p += STORECHAR32(0xFEFF, bom) + + if byteorder == "little": + bom = "little" + elif byteorder == "big": + bom = "big" + + pos = 0 + while pos < len(s): + ch = ord(s[pos]) + if 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + p += STORECHAR32(ch, bom) + pos += 1 + else: + res, pos = unicode_call_errorhandler( + errors, "utf-32", "surrogates not allowed", s, pos, pos + 1, False + ) + for c in res: + p += STORECHAR32(ord(c), bom) + else: + p += STORECHAR32(ch, bom) + pos += 1 + + return p + + +def utf_32_encode(obj, errors="strict"): + """UTF-32 encoding with BOM.""" + encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, "native") + return bytes(encoded), len(obj) + + +def utf_32_le_encode(obj, errors="strict"): + """UTF-32 little-endian encoding without BOM.""" + encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, "little") + return bytes(encoded), len(obj) + + +def utf_32_be_encode(obj, errors="strict"): + """UTF-32 big-endian encoding without BOM.""" + res = PyUnicode_EncodeUTF32(obj, len(obj), errors, "big") + res = bytes(res) + return res, len(obj) + + +def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=0): + """Decode UTF-32 encoded bytes to Unicode string.""" + if size == 0: + return [], 0, 0 + + result = [] + pos = 0 + aligned_size = (size // 4) * 4 + + while pos + 3 < aligned_size: + if byteorder == "little": + ch = ( + data[pos] + | (data[pos + 1] << 8) + | (data[pos + 2] << 16) + | (data[pos + 3] << 24) + ) + else: # big-endian + ch = ( + (data[pos] << 24) + | (data[pos + 1] << 16) + | (data[pos + 2] << 8) + | data[pos + 3] + ) + + # Validate code point + if ch > 0x10FFFF: + if errors == "strict": + raise UnicodeDecodeError( + "utf-32", + bytes(data), + pos, + pos + 4, + "codepoint not in range(0x110000)", + ) + elif errors == "replace": + result.append("\ufffd") + # 'ignore' - skip this character + pos += 4 + elif 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + result.append(chr(ch)) + pos += 4 + else: + msg = "code point in surrogate code point range(0xd800, 0xe000)" + res, pos = unicode_call_errorhandler( + errors, "utf-32", msg, data, pos, pos + 4, True + ) + result.append(res) + else: + result.append(chr(ch)) + pos += 4 + + # Handle trailing incomplete bytes + if pos < size: + if final: + res, pos = unicode_call_errorhandler( + errors, "utf-32", "truncated data", data, pos, size, True + ) + if res: + result.append(res) + + return result, pos, 0 + + +def utf_32_decode(data, errors="strict", final=0): + """UTF-32 decoding with BOM detection.""" + if len(data) >= 4: + # Check for BOM + if data[0:4] == b"\xff\xfe\x00\x00": + # UTF-32 LE BOM + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "little", final + ) + res = "".join(res) + return res, consumed + 4 + elif data[0:4] == b"\x00\x00\xfe\xff": + # UTF-32 BE BOM + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data[4:], len(data) - 4, errors, "big", final + ) + res = "".join(res) + return res, consumed + 4 + + # Default to little-endian if no BOM + byteorder = "little" if sys.byteorder == "little" else "big" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, byteorder, final + ) + res = "".join(res) + return res, consumed + + +def utf_32_le_decode(data, errors="strict", final=0): + """UTF-32 little-endian decoding without BOM.""" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "little", final + ) + res = "".join(res) + return res, consumed + + +def utf_32_be_decode(data, errors="strict", final=0): + """UTF-32 big-endian decoding without BOM.""" + res, consumed, _ = PyUnicode_DecodeUTF32Stateful( + data, len(data), errors, "big", final + ) + res = "".join(res) + return res, consumed # ---------------------------------------------------------------------- @@ -364,9 +649,9 @@ def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): ##import sys ##""" Python implementation of CPythons builtin unicode codecs. ## -## Generally the functions in this module take a list of characters an returns +## Generally the functions in this module take a list of characters an returns ## a list of characters. -## +## ## For use in the PyPy project""" @@ -376,283 +661,496 @@ def utf_16_be_decode( data, errors='strict', byteorder=0, final = 0): ## 1 - special ## 2 - whitespace (optional) ## 3 - RFC2152 Set O (optional) - + utf7_special = [ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 0, + 0, + 0, + 3, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 3, + 3, + 3, + 0, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 1, + 3, + 3, + 3, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 3, + 3, + 1, + 1, ] -unicode_latin1 = [None]*256 +unicode_latin1 = [None] * 256 def SPECIAL(c, encodeO, encodeWS): c = ord(c) - return (c>127 or utf7_special[c] == 1) or \ - (encodeWS and (utf7_special[(c)] == 2)) or \ - (encodeO and (utf7_special[(c)] == 3)) + return ( + (c > 127 or utf7_special[c] == 1) + or (encodeWS and (utf7_special[(c)] == 2)) + or (encodeO and (utf7_special[(c)] == 3)) + ) + + def B64(n): - return bytes([b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f]]) + return bytes( + [ + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[ + (n) & 0x3F + ] + ] + ) + + def B64CHAR(c): - return (c.isalnum() or (c) == b'+' or (c) == b'/') + return c.isalnum() or (c) == b"+" or (c) == b"/" + + def UB64(c): - if (c) == b'+' : - return 62 - elif (c) == b'/': - return 63 - elif (c) >= b'a': - return ord(c) - 71 - elif (c) >= b'A': - return ord(c) - 65 - else: + if (c) == b"+": + return 62 + elif (c) == b"/": + return 63 + elif (c) >= b"a": + return ord(c) - 71 + elif (c) >= b"A": + return ord(c) - 65 + else: return ord(c) + 4 -def ENCODE( ch, bits) : + +def ENCODE(ch, bits): out = [] - while (bits >= 6): - out += B64(ch >> (bits-6)) - bits -= 6 + while bits >= 6: + out += B64(ch >> (bits - 6)) + bits -= 6 return out, bits -def PyUnicode_DecodeUTF7(s, size, errors): - starts = s - errmsg = "" - inShift = 0 - bitsleft = 0 - charsleft = 0 - surrogate = 0 - p = [] - errorHandler = None - exc = None +def _IS_BASE64(ch): + return ( + (ord("A") <= ch <= ord("Z")) + or (ord("a") <= ch <= ord("z")) + or (ord("0") <= ch <= ord("9")) + or ch == ord("+") + or ch == ord("/") + ) - if (size == 0): - return '' + +def _FROM_BASE64(ch): + if ch == ord("+"): + return 62 + if ch == ord("/"): + return 63 + if ch >= ord("a"): + return ch - 71 + if ch >= ord("A"): + return ch - 65 + if ch >= ord("0"): + return ch - ord("0") + 52 + return -1 + + +def _DECODE_DIRECT(ch): + return ch <= 127 and ch != ord("+") + + +def PyUnicode_DecodeUTF7(s, size, errors, final=False): + if size == 0: + return [], 0 + + p = [] + inShift = False + base64bits = 0 + base64buffer = 0 + surrogate = 0 + startinpos = 0 + shiftOutStart = 0 i = 0 + while i < size: - - ch = bytes([s[i]]) - if (inShift): - if ((ch == b'-') or not B64CHAR(ch)): - inShift = 0 + ch = s[i] + if inShift: + if _IS_BASE64(ch): + base64buffer = (base64buffer << 6) | _FROM_BASE64(ch) + base64bits += 6 i += 1 - - while (bitsleft >= 16): - outCh = ((charsleft) >> (bitsleft-16)) & 0xffff - bitsleft -= 16 - - if (surrogate): - ## We have already generated an error for the high surrogate - ## so let's not bother seeing if the low surrogate is correct or not - surrogate = 0 - elif (0xDC00 <= (outCh) and (outCh) <= 0xDFFF): - ## This is a surrogate pair. Unfortunately we can't represent - ## it in a 16-bit character - surrogate = 1 - msg = "code pairs are not supported" - out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i) - p.append(out) - bitsleft = 0 - break + if base64bits >= 16: + outCh = (base64buffer >> (base64bits - 16)) & 0xFFFF + base64bits -= 16 + base64buffer &= (1 << base64bits) - 1 + if surrogate: + if 0xDC00 <= outCh <= 0xDFFF: + ch2 = ( + 0x10000 + + ((surrogate - 0xD800) << 10) + + (outCh - 0xDC00) + ) + p.append(chr(ch2)) + surrogate = 0 + continue + else: + p.append(chr(surrogate)) + surrogate = 0 + if 0xD800 <= outCh <= 0xDBFF: + surrogate = outCh else: - p.append(chr(outCh )) - #p += out - if (bitsleft >= 6): -## /* The shift sequence has a partial character in it. If -## bitsleft < 6 then we could just classify it as padding -## but that is not the case here */ - msg = "partial character in shift sequence" - out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i) - -## /* According to RFC2152 the remaining bits should be zero. We -## choose to signal an error/insert a replacement character -## here so indicate the potential of a misencoded character. */ - -## /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */ -## if (bitsleft and (charsleft << (sizeof(charsleft) * 8 - bitsleft))): -## raise UnicodeDecodeError, "non-zero padding bits in shift sequence" - if (ch == b'-') : - if ((i < size) and (s[i] == '-')) : - p += '-' - inShift = 1 - - elif SPECIAL(ch, 0, 0) : - raise UnicodeDecodeError("unexpected special character") - - else: - p.append(chr(ord(ch))) + p.append(chr(outCh)) else: - charsleft = (charsleft << 6) | UB64(ch) - bitsleft += 6 - i += 1 -## /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate); - elif ( ch == b'+' ): + inShift = False + if base64bits > 0: + if base64bits >= 6: + i += 1 + errmsg = "partial character in shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, i + ) + p.append(out) + continue + else: + if base64buffer != 0: + i += 1 + errmsg = "non-zero padding bits in shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, i + ) + p.append(out) + continue + if surrogate and _DECODE_DIRECT(ch): + p.append(chr(surrogate)) + surrogate = 0 + if ch == ord("-"): + i += 1 + elif ch == ord("+"): startinpos = i i += 1 - if (i= 6 or (base64bits > 0 and base64buffer != 0): + errmsg = "unterminated shift sequence" + out, i = unicode_call_errorhandler( + errors, "utf-7", errmsg, s, startinpos, size + ) + p.append(out) + + return p, size + + +def _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + c = ord(ch) if isinstance(ch, str) else ch + if c > 127: + return False + if utf7_special[c] == 0: + return True + if utf7_special[c] == 2: + return not encodeWhiteSpace + if utf7_special[c] == 3: + return not encodeSetO + return False - if (inShift) : - #XXX This aint right - endinpos = size - raise UnicodeDecodeError("unterminated shift sequence") - - return p def PyUnicode_EncodeUTF7(s, size, encodeSetO, encodeWhiteSpace, errors): - -# /* It might be possible to tighten this worst case */ inShift = False - i = 0 - bitsleft = 0 - charsleft = 0 + base64bits = 0 + base64buffer = 0 out = [] - for ch in s: - if (not inShift) : - if (ch == '+'): - out.append(b'+-') - elif (SPECIAL(ch, encodeSetO, encodeWhiteSpace)): - charsleft = ord(ch) - bitsleft = 16 - out.append(b'+') - p, bitsleft = ENCODE( charsleft, bitsleft) - out.append(p) - inShift = bitsleft > 0 + + for i, ch in enumerate(s): + ch_ord = ord(ch) + if inShift: + if _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + # shifting out + if base64bits: + out.append(B64(base64buffer << (6 - base64bits))) + base64buffer = 0 + base64bits = 0 + inShift = False + if B64CHAR(ch) or ch == "-": + out.append(b"-") + out.append(bytes([ch_ord])) else: - out.append(bytes([ord(ch)])) + # encode character in base64 + if ch_ord >= 0x10000: + # split into surrogate pair + hi = 0xD800 | ((ch_ord - 0x10000) >> 10) + lo = 0xDC00 | ((ch_ord - 0x10000) & 0x3FF) + base64bits += 16 + base64buffer = (base64buffer << 16) | hi + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + ch_ord = lo + + base64bits += 16 + base64buffer = (base64buffer << 16) | ch_ord + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 else: - if (not SPECIAL(ch, encodeSetO, encodeWhiteSpace)): - out.append(B64((charsleft) << (6-bitsleft))) - charsleft = 0 - bitsleft = 0 -## /* Characters not in the BASE64 set implicitly unshift the sequence -## so no '-' is required, except if the character is itself a '-' */ - if (B64CHAR(ch) or ch == '-'): - out.append(b'-') - inShift = False - out.append(bytes([ord(ch)])) + if ch == "+": + out.append(b"+-") + elif _ENCODE_DIRECT(ch, encodeSetO, encodeWhiteSpace): + out.append(bytes([ch_ord])) else: - bitsleft += 16 - charsleft = (((charsleft) << 16) | ord(ch)) - p, bitsleft = ENCODE(charsleft, bitsleft) - out.append(p) -## /* If the next character is special then we dont' need to terminate -## the shift sequence. If the next character is not a BASE64 character -## or '-' then the shift sequence will be terminated implicitly and we -## don't have to insert a '-'. */ - - if (bitsleft == 0): - if (i + 1 < size): - ch2 = s[i+1] - - if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)): - pass - elif (B64CHAR(ch2) or ch2 == '-'): - out.append(b'-') - inShift = False - else: + out.append(b"+") + inShift = True + # encode character in base64 + if ch_ord >= 0x10000: + hi = 0xD800 | ((ch_ord - 0x10000) >> 10) + lo = 0xDC00 | ((ch_ord - 0x10000) & 0x3FF) + base64bits += 16 + base64buffer = (base64buffer << 16) | hi + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + ch_ord = lo + + base64bits += 16 + base64buffer = (base64buffer << 16) | ch_ord + while base64bits >= 6: + out.append(B64(base64buffer >> (base64bits - 6))) + base64bits -= 6 + base64buffer &= (1 << base64bits) - 1 if base64bits else 0 + + if base64bits == 0: + if i + 1 < size: + ch2 = s[i + 1] + if _ENCODE_DIRECT(ch2, encodeSetO, encodeWhiteSpace): + if B64CHAR(ch2) or ch2 == "-": + out.append(b"-") inShift = False else: - out.append(b'-') + out.append(b"-") inShift = False - i += 1 - - if (bitsleft): - out.append(B64(charsleft << (6-bitsleft) ) ) - out.append(b'-') + + if base64bits: + out.append(B64(base64buffer << (6 - base64bits))) + if inShift: + out.append(b"-") return out -unicode_empty = '' -def unicodeescape_string(s, size, quotes): +unicode_empty = "" + +def unicodeescape_string(s, size, quotes): p = [] - if (quotes) : - if (s.find('\'') != -1 and s.find('"') == -1): + if quotes: + if s.find("'") != -1 and s.find('"') == -1: p.append(b'"') else: - p.append(b'\'') + p.append(b"'") pos = 0 - while (pos < size): + while pos < size: ch = s[pos] - #/* Escape quotes */ - if (quotes and (ch == p[1] or ch == '\\')): - p.append(b'\\%c' % ord(ch)) + # /* Escape quotes */ + if quotes and (ch == p[1] or ch == "\\"): + p.append(b"\\%c" % ord(ch)) pos += 1 continue -#ifdef Py_UNICODE_WIDE - #/* Map 21-bit characters to '\U00xxxxxx' */ - elif (ord(ch) >= 0x10000): - p.append(b'\\U%08x' % ord(ch)) + # ifdef Py_UNICODE_WIDE + # /* Map 21-bit characters to '\U00xxxxxx' */ + elif ord(ch) >= 0x10000: + p.append(b"\\U%08x" % ord(ch)) pos += 1 - continue -#endif - #/* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */ - elif (ord(ch) >= 0xD800 and ord(ch) < 0xDC00): + continue + # endif + # /* Map UTF-16 surrogate pairs to Unicode \UXXXXXXXX escapes */ + elif ord(ch) >= 0xD800 and ord(ch) < 0xDC00: pos += 1 ch2 = s[pos] - - if (ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF): + + if ord(ch2) >= 0xDC00 and ord(ch2) <= 0xDFFF: ucs = (((ord(ch) & 0x03FF) << 10) | (ord(ch2) & 0x03FF)) + 0x00010000 - p.append(b'\\U%08x' % ucs) + p.append(b"\\U%08x" % ucs) pos += 1 continue - - #/* Fall through: isolated surrogates are copied as-is */ + + # /* Fall through: isolated surrogates are copied as-is */ pos -= 1 - - #/* Map 16-bit characters to '\uxxxx' */ - if (ord(ch) >= 256): - p.append(b'\\u%04x' % ord(ch)) - - #/* Map special whitespace to '\t', \n', '\r' */ - elif (ch == '\t'): - p.append(b'\\t') - - elif (ch == '\n'): - p.append(b'\\n') - - elif (ch == '\r'): - p.append(b'\\r') - - elif (ch == '\\'): - p.append(b'\\\\') - - #/* Map non-printable US ASCII to '\xhh' */ - elif (ch < ' ' or ch >= chr(0x7F)) : - p.append(b'\\x%02x' % ord(ch)) - #/* Copy everything else as-is */ + + # /* Map 16-bit characters to '\uxxxx' */ + if ord(ch) >= 256: + p.append(b"\\u%04x" % ord(ch)) + + # /* Map special whitespace to '\t', \n', '\r' */ + elif ch == "\t": + p.append(b"\\t") + + elif ch == "\n": + p.append(b"\\n") + + elif ch == "\r": + p.append(b"\\r") + + elif ch == "\\": + p.append(b"\\\\") + + # /* Map non-printable US ASCII to '\xhh' */ + elif ch < " " or ch >= chr(0x7F): + p.append(b"\\x%02x" % ord(ch)) + # /* Copy everything else as-is */ else: p.append(bytes([ord(ch)])) pos += 1 - if (quotes): + if quotes: p.append(p[0]) return p -def PyUnicode_DecodeASCII(s, size, errors): -# /* ASCII is equivalent to the first 128 ordinals in Unicode. */ - if (size == 1 and ord(s) < 128) : +def PyUnicode_DecodeASCII(s, size, errors): + # /* ASCII is equivalent to the first 128 ordinals in Unicode. */ + if size == 1 and ord(s) < 128: return [chr(ord(s))] - if (size == 0): - return [''] #unicode('') + if size == 0: + return [""] # unicode('') p = [] pos = 0 while pos < len(s): @@ -661,54 +1159,50 @@ def PyUnicode_DecodeASCII(s, size, errors): p += chr(c) pos += 1 else: - res = unicode_call_errorhandler( - errors, "ascii", "ordinal not in range(128)", - s, pos, pos+1) + errors, "ascii", "ordinal not in range(128)", s, pos, pos + 1 + ) p += res[0] pos = res[1] return p -def PyUnicode_EncodeASCII(p, size, errors): +def PyUnicode_EncodeASCII(p, size, errors): return unicode_encode_ucs1(p, size, errors, 128) -def PyUnicode_AsASCIIString(unistr): +def PyUnicode_AsASCIIString(unistr): if not type(unistr) == str: raise TypeError - return PyUnicode_EncodeASCII(str(unistr), - len(str), - None) + return PyUnicode_EncodeASCII(unistr, len(unistr), None) -def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True): - bo = 0 #/* assume native ordering by default */ +def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=True): + bo = 0 # /* assume native ordering by default */ consumed = 0 errmsg = "" - if sys.byteorder == 'little': + if sys.byteorder == "little": ihi = 1 ilo = 0 else: ihi = 0 ilo = 1 - - #/* Unpack UTF-16 encoded data */ + # /* Unpack UTF-16 encoded data */ -## /* Check for BOM marks (U+FEFF) in the input and adjust current -## byte order setting accordingly. In native mode, the leading BOM -## mark is skipped, in all other modes, it is copied to the output -## stream as-is (giving a ZWNBSP character). */ + ## /* Check for BOM marks (U+FEFF) in the input and adjust current + ## byte order setting accordingly. In native mode, the leading BOM + ## mark is skipped, in all other modes, it is copied to the output + ## stream as-is (giving a ZWNBSP character). */ q = 0 p = [] - if byteorder == 'native': - if (size >= 2): + if byteorder == "native": + if size >= 2: bom = (s[ihi] << 8) | s[ilo] -#ifdef BYTEORDER_IS_LITTLE_ENDIAN - if sys.byteorder == 'little': - if (bom == 0xFEFF): + # ifdef BYTEORDER_IS_LITTLE_ENDIAN + if sys.byteorder == "little": + if bom == 0xFEFF: q += 2 bo = -1 elif bom == 0xFFFE: @@ -721,118 +1215,143 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru elif bom == 0xFFFE: q += 2 bo = -1 - elif byteorder == 'little': + elif byteorder == "little": bo = -1 else: bo = 1 - - if (size == 0): - return [''], 0, bo - - if (bo == -1): - #/* force LE */ + + if size == 0: + return [""], 0, bo + + if bo == -1: + # /* force LE */ ihi = 1 ilo = 0 - elif (bo == 1): - #/* force BE */ + elif bo == 1: + # /* force BE */ ihi = 0 ilo = 1 - while (q < len(s)): - - #/* remaining bytes at the end? (size should be even) */ - if (len(s)-q<2): + while q < len(s): + # /* remaining bytes at the end? (size should be even) */ + if len(s) - q < 2: if not final: break - errmsg = "truncated data" - startinpos = q - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) -# /* The remaining input chars are ignored if the callback -## chooses to skip the input */ - - ch = (s[q+ihi] << 8) | s[q+ilo] - q += 2 - - if (ch < 0xD800 or ch > 0xDFFF): + res, q = unicode_call_errorhandler( + errors, "utf-16", "truncated data", s, q, len(s), True + ) + p.append(res) + break + + ch = (s[q + ihi] << 8) | s[q + ilo] + + if ch < 0xD800 or ch > 0xDFFF: p.append(chr(ch)) - continue - - #/* UTF-16 code pair: */ - if (q >= len(s)): - errmsg = "unexpected end of data" - startinpos = q-2 - endinpos = len(s) - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - if (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (s[q+ihi] << 8) | s[q+ilo] q += 2 - if (0xDC00 <= ch2 and ch2 <= 0xDFFF): - #ifndef Py_UNICODE_WIDE - if sys.maxunicode < 65536: - p += [chr(ch), chr(ch2)] + continue + + # /* UTF-16 code pair: high surrogate */ + if 0xD800 <= ch <= 0xDBFF: + if q + 4 <= len(s): + ch2 = (s[q + 2 + ihi] << 8) | s[q + 2 + ilo] + if 0xDC00 <= ch2 <= 0xDFFF: + # Valid surrogate pair - always assemble + p.append(chr((((ch & 0x3FF) << 10) | (ch2 & 0x3FF)) + 0x10000)) + q += 4 + continue else: - p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000)) - #endif + # High surrogate followed by non-low-surrogate + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 + continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "illegal UTF-16 surrogate", s, q, q + 2, True + ) + p.append(res) + else: + # High surrogate at end of data + if not final: + break + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 + continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "unexpected end of data", s, q, len(s), True + ) + p.append(res) + else: + # Low surrogate without preceding high surrogate + if errors == "surrogatepass": + p.append(chr(ch)) + q += 2 continue + res, q = unicode_call_errorhandler( + errors, "utf-16", "illegal encoding", s, q, q + 2, True + ) + p.append(res) - else: - errmsg = "illegal UTF-16 surrogate" - startinpos = q-4 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - - errmsg = "illegal encoding" - startinpos = q-2 - endinpos = startinpos+2 - unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) - return p, q, bo + # moved out of local scope, especially because it didn't # have any nested variables. + def STORECHAR(CH, byteorder): - hi = (CH >> 8) & 0xff - lo = CH & 0xff - if byteorder == 'little': + hi = (CH >> 8) & 0xFF + lo = CH & 0xFF + if byteorder == "little": return [lo, hi] else: return [hi, lo] -def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): -# /* Offsets from p for storing byte pairs in the right order. */ +def PyUnicode_EncodeUTF16(s, size, errors, byteorder="little"): + # /* Offsets from p for storing byte pairs in the right order. */ - p = [] bom = sys.byteorder - if (byteorder == 'native'): - + if byteorder == "native": bom = sys.byteorder p += STORECHAR(0xFEFF, bom) - - if (size == 0): - return "" - if (byteorder == 'little' ): - bom = 'little' - elif (byteorder == 'big'): - bom = 'big' + if byteorder == "little": + bom = "little" + elif byteorder == "big": + bom = "big" - - for c in s: - ch = ord(c) - ch2 = 0 - if (ch >= 0x10000) : - ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF) - ch = 0xD800 | ((ch-0x10000) >> 10) - - p += STORECHAR(ch, bom) - if (ch2): - p += STORECHAR(ch2, bom) + pos = 0 + while pos < len(s): + ch = ord(s[pos]) + if 0xD800 <= ch <= 0xDFFF: + if errors == "surrogatepass": + p += STORECHAR(ch, bom) + pos += 1 + else: + res, pos = unicode_call_errorhandler( + errors, "utf-16", "surrogates not allowed", s, pos, pos + 1, False + ) + for c in res: + cp = ord(c) + cp2 = 0 + if cp >= 0x10000: + cp2 = 0xDC00 | ((cp - 0x10000) & 0x3FF) + cp = 0xD800 | ((cp - 0x10000) >> 10) + p += STORECHAR(cp, bom) + if cp2: + p += STORECHAR(cp2, bom) + else: + ch2 = 0 + if ch >= 0x10000: + ch2 = 0xDC00 | ((ch - 0x10000) & 0x3FF) + ch = 0xD800 | ((ch - 0x10000) >> 10) + p += STORECHAR(ch, bom) + if ch2: + p += STORECHAR(ch2, bom) + pos += 1 return p @@ -840,119 +1359,149 @@ def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'): def PyUnicode_DecodeMBCS(s, size, errors): pass + def PyUnicode_EncodeMBCS(p, size, errors): pass -def unicode_call_errorhandler(errors, encoding, - reason, input, startinpos, endinpos, decode=True): - + +def unicode_call_errorhandler( + errors, encoding, reason, input, startinpos, endinpos, decode=True +): errorHandler = lookup_error(errors) if decode: - exceptionObject = UnicodeDecodeError(encoding, input, startinpos, endinpos, reason) + exceptionObject = UnicodeDecodeError( + encoding, input, startinpos, endinpos, reason + ) else: - exceptionObject = UnicodeEncodeError(encoding, input, startinpos, endinpos, reason) + exceptionObject = UnicodeEncodeError( + encoding, input, startinpos, endinpos, reason + ) res = errorHandler(exceptionObject) - if isinstance(res, tuple) and isinstance(res[0], str) and isinstance(res[1], int): + if ( + isinstance(res, tuple) + and isinstance(res[0], (str, bytes)) + and isinstance(res[1], int) + ): newpos = res[1] - if (newpos < 0): + if newpos < 0: newpos = len(input) + newpos if newpos < 0 or newpos > len(input): - raise IndexError( "position %d from error handler out of bounds" % newpos) + raise IndexError("position %d from error handler out of bounds" % newpos) return res[0], newpos else: - raise TypeError("encoding error handler must return (unicode, int) tuple, not %s" % repr(res)) + raise TypeError( + "encoding error handler must return (unicode, int) tuple, not %s" + % repr(res) + ) + + +# /* --- Latin-1 Codec ------------------------------------------------------ */ -#/* --- Latin-1 Codec ------------------------------------------------------ */ def PyUnicode_DecodeLatin1(s, size, errors): - #/* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ -## if (size == 1): -## return [PyUnicode_FromUnicode(s, 1)] + # /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */ + ## if (size == 1): + ## return [PyUnicode_FromUnicode(s, 1)] pos = 0 p = [] - while (pos < size): + while pos < size: p += chr(s[pos]) pos += 1 return p + def unicode_encode_ucs1(p, size, errors, limit): - if limit == 256: reason = "ordinal not in range(256)" encoding = "latin-1" else: reason = "ordinal not in range(128)" encoding = "ascii" - - if (size == 0): + + if size == 0: return [] res = bytearray() pos = 0 while pos < len(p): - #for ch in p: + # for ch in p: ch = p[pos] - + if ord(ch) < limit: res.append(ord(ch)) pos += 1 else: - #/* startpos for collecting unencodable chars */ - collstart = pos - collend = pos+1 + # /* startpos for collecting unencodable chars */ + collstart = pos + collend = pos + 1 while collend < len(p) and ord(p[collend]) >= limit: collend += 1 - x = unicode_call_errorhandler(errors, encoding, reason, p, collstart, collend, False) - res += x[0].encode() + x = unicode_call_errorhandler( + errors, encoding, reason, p, collstart, collend, False + ) + replacement = x[0] + if isinstance(replacement, bytes): + res += replacement + else: + res += replacement.encode() pos = x[1] - + return res + def PyUnicode_EncodeLatin1(p, size, errors): res = unicode_encode_ucs1(p, size, errors, 256) return res -hexdigits = [ord(hex(i)[-1]) for i in range(16)]+[ord(hex(i)[-1].upper()) for i in range(10, 16)] + +hexdigits = [ord(hex(i)[-1]) for i in range(16)] + [ + ord(hex(i)[-1].upper()) for i in range(10, 16) +] + def hex_number_end(s, pos, digits): target_end = pos + digits - while pos < target_end and pos < len(s) and s[pos] in hexdigits: + while pos < target_end and pos < len(s) and s[pos] in hexdigits: pos += 1 return pos + def hexescape(s, pos, digits, message, errors): ch = 0 p = [] number_end = hex_number_end(s, pos, digits) if number_end - pos != digits: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, number_end) + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 2, number_end + ) p.append(x[0]) pos = x[1] else: - ch = int(s[pos:pos+digits], 16) - #/* when we get here, ch is a 32-bit unicode character */ + ch = int(s[pos : pos + digits], 16) + # /* when we get here, ch is a 32-bit unicode character */ if ch <= sys.maxunicode: p.append(chr(ch)) pos += digits - elif (ch <= 0x10ffff): + elif ch <= 0x10FFFF: ch -= 0x10000 p.append(chr(0xD800 + (ch >> 10))) - p.append(chr(0xDC00 + (ch & 0x03FF))) + p.append(chr(0xDC00 + (ch & 0x03FF))) pos += digits else: message = "illegal Unicode character" - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-2, - pos+digits) + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 2, pos + digits + ) p.append(x[0]) pos = x[1] res = p return res, pos + def PyUnicode_DecodeUnicodeEscape(s, size, errors, final): + if size == 0: + return "", 0 - if (size == 0): - return '' - if isinstance(s, str): s = s.encode() @@ -960,279 +1509,331 @@ def PyUnicode_DecodeUnicodeEscape(s, size, errors, final): p = [] pos = 0 - while (pos < size): -## /* Non-escape characters are interpreted as Unicode ordinals */ - if (chr(s[pos]) != '\\') : + while pos < size: + ## /* Non-escape characters are interpreted as Unicode ordinals */ + if s[pos] != ord("\\"): p.append(chr(s[pos])) pos += 1 continue -## /* \ - Escapes */ - else: - pos += 1 - if pos >= len(s): - errmessage = "\\ at end of string" - unicode_call_errorhandler(errors, "unicodeescape", errmessage, s, pos-1, size) - ch = chr(s[pos]) - pos += 1 - ## /* \x escapes */ - if ch == '\n': pass - elif ch == '\\': p += '\\' - elif ch == '\'': p += '\'' - elif ch == '\"': p += '\"' - elif ch == 'b' : p += '\b' - elif ch == 'f' : p += '\014' #/* FF */ - elif ch == 't' : p += '\t' - elif ch == 'n' : p += '\n' - elif ch == 'r' : p += '\r' - elif ch == 'v' : p += '\013' #break; /* VT */ - elif ch == 'a' : p += '\007' # break; /* BEL, not classic C */ - elif '0' <= ch <= '7': - x = ord(ch) - ord('0') - if pos < size: - ch = chr(s[pos]) - if '0' <= ch <= '7': - pos += 1 - x = (x<<3) + ord(ch) - ord('0') - if pos < size: - ch = chr(s[pos]) - if '0' <= ch <= '7': - pos += 1 - x = (x<<3) + ord(ch) - ord('0') - p.append(chr(x)) - ## /* hex escapes */ - ## /* \xXX */ - elif ch == 'x': + ## /* \ - Escapes */ + escape_start = pos + pos += 1 + if pos >= size: + if not final: + pos = escape_start + break + errmessage = "\\ at end of string" + unicode_call_errorhandler( + errors, "unicodeescape", errmessage, s, pos - 1, size + ) + break + ch = chr(s[pos]) + pos += 1 + ## /* \x escapes */ + if ch == "\n": + pass + elif ch == "\\": + p += "\\" + elif ch == "'": + p += "'" + elif ch == '"': + p += '"' + elif ch == "b": + p += "\b" + elif ch == "f": + p += "\014" # /* FF */ + elif ch == "t": + p += "\t" + elif ch == "n": + p += "\n" + elif ch == "r": + p += "\r" + elif ch == "v": + p += "\013" # break; /* VT */ + elif ch == "a": + p += "\007" # break; /* BEL, not classic C */ + elif "0" <= ch <= "7": + x = ord(ch) - ord("0") + if pos < size: + ch = chr(s[pos]) + if "0" <= ch <= "7": + pos += 1 + x = (x << 3) + ord(ch) - ord("0") + if pos < size: + ch = chr(s[pos]) + if "0" <= ch <= "7": + pos += 1 + x = (x << 3) + ord(ch) - ord("0") + p.append(chr(x)) + ## /* hex escapes */ + ## /* \xXX */ + elif ch in ("x", "u", "U"): + if ch == "x": digits = 2 message = "truncated \\xXX escape" - x = hexescape(s, pos, digits, message, errors) - p += x[0] - pos = x[1] - - # /* \uXXXX */ - elif ch == 'u': + elif ch == "u": digits = 4 message = "truncated \\uXXXX escape" + else: + digits = 8 + message = "truncated \\UXXXXXXXX escape" + number_end = hex_number_end(s, pos, digits) + if number_end - pos != digits: + if not final: + pos = escape_start + break x = hexescape(s, pos, digits, message, errors) p += x[0] pos = x[1] - - # /* \UXXXXXXXX */ - elif ch == 'U': - digits = 8 - message = "truncated \\UXXXXXXXX escape" + else: x = hexescape(s, pos, digits, message, errors) p += x[0] pos = x[1] -## /* \N{name} */ - elif ch == 'N': - message = "malformed \\N character escape" - # pos += 1 - look = pos - try: - import unicodedata - except ImportError: - message = "\\N escapes not supported (can't load unicodedata module)" - unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, size) - if look < size and chr(s[look]) == '{': - #/* look for the closing brace */ - while (look < size and chr(s[look]) != '}'): - look += 1 - if (look > pos+1 and look < size and chr(s[look]) == '}'): - #/* found a name. look it up in the unicode database */ - message = "unknown Unicode character name" - st = s[pos+1:look] - try: - chr_codec = unicodedata.lookup("%s" % st) - except LookupError as e: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) - else: - x = chr_codec, look + 1 - p.append(x[0]) - pos = x[1] - else: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) - else: - x = unicode_call_errorhandler(errors, "unicodeescape", message, s, pos-1, look+1) + ## /* \N{name} */ + elif ch == "N": + message = "malformed \\N character escape" + look = pos + try: + import unicodedata + except ImportError: + message = "\\N escapes not supported (can't load unicodedata module)" + unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, size + ) + continue + if look < size and chr(s[look]) == "{": + # /* look for the closing brace */ + while look < size and chr(s[look]) != "}": + look += 1 + if look > pos + 1 and look < size and chr(s[look]) == "}": + # /* found a name. look it up in the unicode database */ + message = "unknown Unicode character name" + st = s[pos + 1 : look] + try: + chr_codec = unicodedata.lookup("%s" % st) + except LookupError as e: + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + else: + x = chr_codec, look + 1 + p.append(x[0]) + pos = x[1] + else: + if not final: + pos = escape_start + break + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + p.append(x[0]) + pos = x[1] else: - if not found_invalid_escape: - found_invalid_escape = True - warnings.warn("invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2) - p.append('\\') - p.append(ch) - return p + if not final: + pos = escape_start + break + x = unicode_call_errorhandler( + errors, "unicodeescape", message, s, pos - 1, look + 1 + ) + p.append(x[0]) + pos = x[1] + else: + if not found_invalid_escape: + found_invalid_escape = True + warnings.warn( + "invalid escape sequence '\\%c'" % ch, DeprecationWarning, 2 + ) + p.append("\\") + p.append(ch) + return p, pos + def PyUnicode_EncodeRawUnicodeEscape(s, size): - - if (size == 0): - return b'' + if size == 0: + return b"" p = bytearray() for ch in s: -# /* Map 32-bit characters to '\Uxxxxxxxx' */ - if (ord(ch) >= 0x10000): - p += b'\\U%08x' % ord(ch) - elif (ord(ch) >= 256) : -# /* Map 16-bit characters to '\uxxxx' */ - p += b'\\u%04x' % (ord(ch)) -# /* Copy everything else as-is */ + # /* Map 32-bit characters to '\Uxxxxxxxx' */ + if ord(ch) >= 0x10000: + p += b"\\U%08x" % ord(ch) + elif ord(ch) >= 256: + # /* Map 16-bit characters to '\uxxxx' */ + p += b"\\u%04x" % (ord(ch)) + # /* Copy everything else as-is */ else: p.append(ord(ch)) - - #p += '\0' + + # p += '\0' return p -def charmapencode_output(c, mapping): +def charmapencode_output(c, mapping): rep = mapping[c] - if isinstance(rep, int) or isinstance(rep, int): + if isinstance(rep, int): if rep < 256: - return rep + return [rep] else: raise TypeError("character mapping must be in range(256)") elif isinstance(rep, str): - return ord(rep) + return [ord(rep)] + elif isinstance(rep, bytes): + return rep elif rep == None: raise KeyError("character maps to ") else: raise TypeError("character mapping must return integer, None or str") -def PyUnicode_EncodeCharmap(p, size, mapping='latin-1', errors='strict'): -## /* the following variable is used for caching string comparisons -## * -1=not initialized, 0=unknown, 1=strict, 2=replace, -## * 3=ignore, 4=xmlcharrefreplace */ +def PyUnicode_EncodeCharmap(p, size, mapping="latin-1", errors="strict"): + ## /* the following variable is used for caching string comparisons + ## * -1=not initialized, 0=unknown, 1=strict, 2=replace, + ## * 3=ignore, 4=xmlcharrefreplace */ -# /* Default to Latin-1 */ - if mapping == 'latin-1': + # /* Default to Latin-1 */ + if mapping == "latin-1": return PyUnicode_EncodeLatin1(p, size, errors) - if (size == 0): - return b'' + if size == 0: + return b"" inpos = 0 res = [] - while (inpos", p, inpos, inpos+1, False) - try: - res += [charmapencode_output(ord(y), mapping) for y in x[0]] - except KeyError: - raise UnicodeEncodeError("charmap", p, inpos, inpos+1, - "character maps to ") + x = unicode_call_errorhandler( + errors, + "charmap", + "character maps to ", + p, + inpos, + inpos + 1, + False, + ) + replacement = x[0] + if isinstance(replacement, bytes): + res += list(replacement) + else: + try: + for y in replacement: + res += charmapencode_output(ord(y), mapping) + except KeyError: + raise UnicodeEncodeError( + "charmap", p, inpos, inpos + 1, "character maps to " + ) inpos += 1 return res -def PyUnicode_DecodeCharmap(s, size, mapping, errors): -## /* Default to Latin-1 */ - if (mapping == None): +def PyUnicode_DecodeCharmap(s, size, mapping, errors): + ## /* Default to Latin-1 */ + if mapping == None: return PyUnicode_DecodeLatin1(s, size, errors) - if (size == 0): - return '' + if size == 0: + return "" p = [] inpos = 0 - while (inpos< len(s)): - - #/* Get mapping (char ordinal -> integer, Unicode char or None) */ + while inpos < len(s): + # /* Get mapping (char ordinal -> integer, Unicode char or None) */ ch = s[inpos] try: x = mapping[ch] if isinstance(x, int): - if x < 65536: + if x == 0xFFFE: + raise KeyError + if 0 <= x <= 0x10FFFF: p += chr(x) else: - raise TypeError("character mapping must be in range(65536)") + raise TypeError( + "character mapping must be in range(0x%x)" % (0x110000,) + ) elif isinstance(x, str): + if len(x) == 1 and x == "\ufffe": + raise KeyError p += x - elif not x: + elif x is None: raise KeyError else: raise TypeError - except KeyError: - x = unicode_call_errorhandler(errors, "charmap", - "character maps to ", s, inpos, inpos+1) + except (KeyError, IndexError): + x = unicode_call_errorhandler( + errors, "charmap", "character maps to ", s, inpos, inpos + 1 + ) p += x[0] inpos += 1 return p -def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final): - if (size == 0): - return '' +def PyUnicode_DecodeRawUnicodeEscape(s, size, errors, final): + if size == 0: + return "", 0 if isinstance(s, str): s = s.encode() pos = 0 p = [] - while (pos < len(s)): - ch = chr(s[pos]) - #/* Non-escape characters are interpreted as Unicode ordinals */ - if (ch != '\\'): - p.append(ch) + while pos < len(s): + # /* Non-escape characters are interpreted as Unicode ordinals */ + if s[pos] != ord("\\"): + p.append(chr(s[pos])) pos += 1 - continue + continue startinpos = pos -## /* \u-escapes are only interpreted iff the number of leading -## backslashes is odd */ + p_len_before = len(p) + ## /* \u-escapes are only interpreted iff the number of leading + ## backslashes is odd */ bs = pos while pos < size: - if (s[pos] != ord('\\')): + if s[pos] != ord("\\"): break p.append(chr(s[pos])) pos += 1 - - if (pos >= size): + + if pos >= size: + if not final: + del p[p_len_before:] + pos = startinpos break - if (((pos - bs) & 1) == 0 or - (s[pos] != ord('u') and s[pos] != ord('U'))) : + if ((pos - bs) & 1) == 0 or (s[pos] != ord("u") and s[pos] != ord("U")): p.append(chr(s[pos])) pos += 1 continue - + p.pop(-1) - if s[pos] == ord('u'): - count = 4 - else: - count = 8 + count = 4 if s[pos] == ord("u") else 8 pos += 1 - #/* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ + # /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */ number_end = hex_number_end(s, pos, count) if number_end - pos != count: + if not final: + del p[p_len_before:] + pos = startinpos + break res = unicode_call_errorhandler( - errors, "rawunicodeescape", "truncated \\uXXXX", - s, pos-2, number_end) + errors, "rawunicodeescape", "truncated \\uXXXX", s, pos - 2, number_end + ) p.append(res[0]) pos = res[1] else: - x = int(s[pos:pos+count], 16) - #ifndef Py_UNICODE_WIDE - if sys.maxunicode > 0xffff: - if (x > sys.maxunicode): - res = unicode_call_errorhandler( - errors, "rawunicodeescape", "\\Uxxxxxxxx out of range", - s, pos-2, pos+count) - pos = res[1] - p.append(res[0]) - else: - p.append(chr(x)) - pos += count + x = int(s[pos : pos + count], 16) + if x > sys.maxunicode: + res = unicode_call_errorhandler( + errors, + "rawunicodeescape", + "\\Uxxxxxxxx out of range", + s, + pos - 2, + pos + count, + ) + pos = res[1] + p.append(res[0]) else: - if (x > 0x10000): - res = unicode_call_errorhandler( - errors, "rawunicodeescape", "\\Uxxxxxxxx out of range", - s, pos-2, pos+count) - pos = res[1] - p.append(res[0]) - - #endif - else: - p.append(chr(x)) - pos += count + p.append(chr(x)) + pos += count - return p + return p, pos diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py new file mode 100644 index 00000000000..70251dbb653 --- /dev/null +++ b/Lib/_pydatetime.py @@ -0,0 +1,2723 @@ +"""Pure Python implementation of the datetime module.""" + +__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", + "MINYEAR", "MAXYEAR", "UTC") + +__name__ = "datetime" + + +import time as _time +import math as _math +import sys +from operator import index as _index + +def _cmp(x, y): + return 0 if x == y else 1 if x > y else -1 + +def _get_class_module(self): + module_name = self.__class__.__module__ + if module_name == 'datetime': + return 'datetime.' + else: + return '' + +MINYEAR = 1 +MAXYEAR = 9999 +_MAXORDINAL = 3652059 # date.max.toordinal() + +# Utility functions, adapted from Python's Demo/classes/Dates.py, which +# also assumes the current Gregorian calendar indefinitely extended in +# both directions. Difference: Dates.py calls January 1 of year 0 day +# number 1. The code here calls January 1 of year 1 day number 1. This is +# to match the definition of the "proleptic Gregorian" calendar in Dershowitz +# and Reingold's "Calendrical Calculations", where it's the base calendar +# for all computations. See the book for algorithms for converting between +# proleptic Gregorian ordinals and many other calendar systems. + +# -1 is a placeholder for indexing purposes. +_DAYS_IN_MONTH = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + +_DAYS_BEFORE_MONTH = [-1] # -1 is a placeholder for indexing purposes. +dbm = 0 +for dim in _DAYS_IN_MONTH[1:]: + _DAYS_BEFORE_MONTH.append(dbm) + dbm += dim +del dbm, dim + +def _is_leap(year): + "year -> 1 if leap year, else 0." + return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0) + +def _days_before_year(year): + "year -> number of days before January 1st of year." + y = year - 1 + return y*365 + y//4 - y//100 + y//400 + +def _days_in_month(year, month): + "year, month -> number of days in that month in that year." + assert 1 <= month <= 12, month + if month == 2 and _is_leap(year): + return 29 + return _DAYS_IN_MONTH[month] + +def _days_before_month(year, month): + "year, month -> number of days in year preceding first day of month." + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" + return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year)) + +def _ymd2ord(year, month, day): + "year, month, day -> ordinal, considering 01-Jan-0001 as day 1." + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" + dim = _days_in_month(year, month) + assert 1 <= day <= dim, f"day must be in 1..{dim}, not {day}" + return (_days_before_year(year) + + _days_before_month(year, month) + + day) + +_DI400Y = _days_before_year(401) # number of days in 400 years +_DI100Y = _days_before_year(101) # " " " " 100 " +_DI4Y = _days_before_year(5) # " " " " 4 " + +# A 4-year cycle has an extra leap day over what we'd get from pasting +# together 4 single years. +assert _DI4Y == 4 * 365 + 1 + +# Similarly, a 400-year cycle has an extra leap day over what we'd get from +# pasting together 4 100-year cycles. +assert _DI400Y == 4 * _DI100Y + 1 + +# OTOH, a 100-year cycle has one fewer leap day than we'd get from +# pasting together 25 4-year cycles. +assert _DI100Y == 25 * _DI4Y - 1 + +def _ord2ymd(n): + "ordinal -> (year, month, day), considering 01-Jan-0001 as day 1." + + # n is a 1-based index, starting at 1-Jan-1. The pattern of leap years + # repeats exactly every 400 years. The basic strategy is to find the + # closest 400-year boundary at or before n, then work with the offset + # from that boundary to n. Life is much clearer if we subtract 1 from + # n first -- then the values of n at 400-year boundaries are exactly + # those divisible by _DI400Y: + # + # D M Y n n-1 + # -- --- ---- ---------- ---------------- + # 31 Dec -400 -_DI400Y -_DI400Y -1 + # 1 Jan -399 -_DI400Y +1 -_DI400Y 400-year boundary + # ... + # 30 Dec 000 -1 -2 + # 31 Dec 000 0 -1 + # 1 Jan 001 1 0 400-year boundary + # 2 Jan 001 2 1 + # 3 Jan 001 3 2 + # ... + # 31 Dec 400 _DI400Y _DI400Y -1 + # 1 Jan 401 _DI400Y +1 _DI400Y 400-year boundary + n -= 1 + n400, n = divmod(n, _DI400Y) + year = n400 * 400 + 1 # ..., -399, 1, 401, ... + + # Now n is the (non-negative) offset, in days, from January 1 of year, to + # the desired date. Now compute how many 100-year cycles precede n. + # Note that it's possible for n100 to equal 4! In that case 4 full + # 100-year cycles precede the desired day, which implies the desired + # day is December 31 at the end of a 400-year cycle. + n100, n = divmod(n, _DI100Y) + + # Now compute how many 4-year cycles precede it. + n4, n = divmod(n, _DI4Y) + + # And now how many single years. Again n1 can be 4, and again meaning + # that the desired day is December 31 at the end of the 4-year cycle. + n1, n = divmod(n, 365) + + year += n100 * 100 + n4 * 4 + n1 + if n1 == 4 or n100 == 4: + assert n == 0 + return year-1, 12, 31 + + # Now the year is correct, and n is the offset from January 1. We find + # the month via an estimate that's either exact or one too large. + leapyear = n1 == 3 and (n4 != 24 or n100 == 3) + assert leapyear == _is_leap(year) + month = (n + 50) >> 5 + preceding = _DAYS_BEFORE_MONTH[month] + (month > 2 and leapyear) + if preceding > n: # estimate is too large + month -= 1 + preceding -= _DAYS_IN_MONTH[month] + (month == 2 and leapyear) + n -= preceding + assert 0 <= n < _days_in_month(year, month) + + # Now the year and month are correct, and n is the offset from the + # start of that month: we're done! + return year, month, n+1 + +# Month and day names. For localized versions, see the calendar module. +_MONTHNAMES = [None, "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] +_DAYNAMES = [None, "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] + + +def _build_struct_time(y, m, d, hh, mm, ss, dstflag): + wday = (_ymd2ord(y, m, d) + 6) % 7 + dnum = _days_before_month(y, m) + d + return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) + +def _format_time(hh, mm, ss, us, timespec='auto'): + specs = { + 'hours': '{:02d}', + 'minutes': '{:02d}:{:02d}', + 'seconds': '{:02d}:{:02d}:{:02d}', + 'milliseconds': '{:02d}:{:02d}:{:02d}.{:03d}', + 'microseconds': '{:02d}:{:02d}:{:02d}.{:06d}' + } + + if timespec == 'auto': + # Skip trailing microseconds when us==0. + timespec = 'microseconds' if us else 'seconds' + elif timespec == 'milliseconds': + us //= 1000 + try: + fmt = specs[timespec] + except KeyError: + raise ValueError('Unknown timespec value') + else: + return fmt.format(hh, mm, ss, us) + +def _format_offset(off, sep=':'): + s = '' + if off is not None: + if off.days < 0: + sign = "-" + off = -off + else: + sign = "+" + hh, mm = divmod(off, timedelta(hours=1)) + mm, ss = divmod(mm, timedelta(minutes=1)) + s += "%s%02d%s%02d" % (sign, hh, sep, mm) + if ss or ss.microseconds: + s += "%s%02d" % (sep, ss.seconds) + + if ss.microseconds: + s += '.%06d' % ss.microseconds + return s + +_normalize_century = None +def _need_normalize_century(): + global _normalize_century + if _normalize_century is None: + try: + _normalize_century = ( + _time.strftime("%Y", (99, 1, 1, 0, 0, 0, 0, 1, 0)) != "0099") + except ValueError: + _normalize_century = True + return _normalize_century + +# Correctly substitute for %z and %Z escapes in strftime formats. +def _wrap_strftime(object, format, timetuple): + # Don't call utcoffset() or tzname() unless actually needed. + freplace = None # the string to use for %f + zreplace = None # the string to use for %z + colonzreplace = None # the string to use for %:z + Zreplace = None # the string to use for %Z + + # Scan format for %z, %:z and %Z escapes, replacing as needed. + newformat = [] + push = newformat.append + i, n = 0, len(format) + while i < n: + ch = format[i] + i += 1 + if ch == '%': + if i < n: + ch = format[i] + i += 1 + if ch == 'f': + if freplace is None: + freplace = '%06d' % getattr(object, + 'microsecond', 0) + newformat.append(freplace) + elif ch == 'z': + if zreplace is None: + if hasattr(object, "utcoffset"): + zreplace = _format_offset(object.utcoffset(), sep="") + else: + zreplace = "" + assert '%' not in zreplace + newformat.append(zreplace) + elif ch == ':': + if i < n: + ch2 = format[i] + i += 1 + if ch2 == 'z': + if colonzreplace is None: + if hasattr(object, "utcoffset"): + colonzreplace = _format_offset(object.utcoffset(), sep=":") + else: + colonzreplace = "" + assert '%' not in colonzreplace + newformat.append(colonzreplace) + else: + push('%') + push(ch) + push(ch2) + elif ch == 'Z': + if Zreplace is None: + Zreplace = "" + if hasattr(object, "tzname"): + s = object.tzname() + if s is not None: + # strftime is going to have at this: escape % + Zreplace = s.replace('%', '%%') + newformat.append(Zreplace) + # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so + # year 1000 for %G can go on the fast path. + elif ((ch in 'YG' or ch in 'FC') and + object.year < 1000 and _need_normalize_century()): + if ch == 'G': + year = int(_time.strftime("%G", timetuple)) + else: + year = object.year + if ch == 'C': + push('{:02}'.format(year // 100)) + else: + push('{:04}'.format(year)) + if ch == 'F': + push('-{:02}-{:02}'.format(*timetuple[1:3])) + else: + push('%') + push(ch) + else: + push('%') + else: + push(ch) + newformat = "".join(newformat) + return _time.strftime(newformat, timetuple) + +# Helpers for parsing the result of isoformat() +def _is_ascii_digit(c): + return c in "0123456789" + +def _find_isoformat_datetime_separator(dtstr): + # See the comment in _datetimemodule.c:_find_isoformat_datetime_separator + len_dtstr = len(dtstr) + if len_dtstr == 7: + return 7 + + assert len_dtstr > 7 + date_separator = "-" + week_indicator = "W" + + if dtstr[4] == date_separator: + if dtstr[5] == week_indicator: + if len_dtstr < 8: + raise ValueError("Invalid ISO string") + if len_dtstr > 8 and dtstr[8] == date_separator: + if len_dtstr == 9: + raise ValueError("Invalid ISO string") + if len_dtstr > 10 and _is_ascii_digit(dtstr[10]): + # This is as far as we need to resolve the ambiguity for + # the moment - if we have YYYY-Www-##, the separator is + # either a hyphen at 8 or a number at 10. + # + # We'll assume it's a hyphen at 8 because it's way more + # likely that someone will use a hyphen as a separator than + # a number, but at this point it's really best effort + # because this is an extension of the spec anyway. + # TODO(pganssle): Document this + return 8 + return 10 + else: + # YYYY-Www (8) + return 8 + else: + # YYYY-MM-DD (10) + return 10 + else: + if dtstr[4] == week_indicator: + # YYYYWww (7) or YYYYWwwd (8) + idx = 7 + while idx < len_dtstr: + if not _is_ascii_digit(dtstr[idx]): + break + idx += 1 + + if idx < 9: + return idx + + if idx % 2 == 0: + # If the index of the last number is even, it's YYYYWwwd + return 7 + else: + return 8 + else: + # YYYYMMDD (8) + return 8 + + +def _parse_isoformat_date(dtstr): + # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10, + # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator + assert len(dtstr) in (7, 8, 10) + year = int(dtstr[0:4]) + has_sep = dtstr[4] == '-' + + pos = 4 + has_sep + if dtstr[pos:pos + 1] == "W": + # YYYY-?Www-?D? + pos += 1 + weekno = int(dtstr[pos:pos + 2]) + pos += 2 + + dayno = 1 + if len(dtstr) > pos: + if (dtstr[pos:pos + 1] == '-') != has_sep: + raise ValueError("Inconsistent use of dash separator") + + pos += has_sep + + dayno = int(dtstr[pos:pos + 1]) + + return list(_isoweek_to_gregorian(year, weekno, dayno)) + else: + month = int(dtstr[pos:pos + 2]) + pos += 2 + if (dtstr[pos:pos + 1] == "-") != has_sep: + raise ValueError("Inconsistent use of dash separator") + + pos += has_sep + day = int(dtstr[pos:pos + 2]) + + return [year, month, day] + + +_FRACTION_CORRECTION = [100000, 10000, 1000, 100, 10] + + +def _parse_hh_mm_ss_ff(tstr): + # Parses things of the form HH[:?MM[:?SS[{.,}fff[fff]]]] + len_str = len(tstr) + + time_comps = [0, 0, 0, 0] + pos = 0 + for comp in range(0, 3): + if (len_str - pos) < 2: + raise ValueError("Incomplete time component") + + time_comps[comp] = int(tstr[pos:pos+2]) + + pos += 2 + next_char = tstr[pos:pos+1] + + if comp == 0: + has_sep = next_char == ':' + + if not next_char or comp >= 2: + break + + if has_sep and next_char != ':': + raise ValueError("Invalid time separator: %c" % next_char) + + pos += has_sep + + if pos < len_str: + if tstr[pos] not in '.,': + raise ValueError("Invalid microsecond separator") + else: + pos += 1 + if not all(map(_is_ascii_digit, tstr[pos:])): + raise ValueError("Non-digit values in fraction") + + len_remainder = len_str - pos + + if len_remainder >= 6: + to_parse = 6 + else: + to_parse = len_remainder + + time_comps[3] = int(tstr[pos:(pos+to_parse)]) + if to_parse < 6: + time_comps[3] *= _FRACTION_CORRECTION[to_parse-1] + + return time_comps + +def _parse_isoformat_time(tstr): + # Format supported is HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]] + len_str = len(tstr) + if len_str < 2: + raise ValueError("Isoformat time too short") + + # This is equivalent to re.search('[+-Z]', tstr), but faster + tz_pos = (tstr.find('-') + 1 or tstr.find('+') + 1 or tstr.find('Z') + 1) + timestr = tstr[:tz_pos-1] if tz_pos > 0 else tstr + + time_comps = _parse_hh_mm_ss_ff(timestr) + + hour, minute, second, microsecond = time_comps + became_next_day = False + error_from_components = False + if (hour == 24): + if all(time_comp == 0 for time_comp in time_comps[1:]): + hour = 0 + time_comps[0] = hour + became_next_day = True + else: + error_from_components = True + + tzi = None + if tz_pos == len_str and tstr[-1] == 'Z': + tzi = timezone.utc + elif tz_pos > 0: + tzstr = tstr[tz_pos:] + + # Valid time zone strings are: + # HH len: 2 + # HHMM len: 4 + # HH:MM len: 5 + # HHMMSS len: 6 + # HHMMSS.f+ len: 7+ + # HH:MM:SS len: 8 + # HH:MM:SS.f+ len: 10+ + + if len(tzstr) in (0, 1, 3) or tstr[tz_pos-1] == 'Z': + raise ValueError("Malformed time zone string") + + tz_comps = _parse_hh_mm_ss_ff(tzstr) + + if all(x == 0 for x in tz_comps): + tzi = timezone.utc + else: + tzsign = -1 if tstr[tz_pos - 1] == '-' else 1 + + td = timedelta(hours=tz_comps[0], minutes=tz_comps[1], + seconds=tz_comps[2], microseconds=tz_comps[3]) + + tzi = timezone(tzsign * td) + + time_comps.append(tzi) + + return time_comps, became_next_day, error_from_components + +# tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar +def _isoweek_to_gregorian(year, week, day): + # Year is bounded this way because 9999-12-31 is (9999, 52, 5) + if not MINYEAR <= year <= MAXYEAR: + raise ValueError(f"year must be in {MINYEAR}..{MAXYEAR}, not {year}") + + if not 0 < week < 53: + out_of_range = True + + if week == 53: + # ISO years have 53 weeks in them on years starting with a + # Thursday and leap years starting on a Wednesday + first_weekday = _ymd2ord(year, 1, 1) % 7 + if (first_weekday == 4 or (first_weekday == 3 and + _is_leap(year))): + out_of_range = False + + if out_of_range: + raise ValueError(f"Invalid week: {week}") + + if not 0 < day < 8: + raise ValueError(f"Invalid weekday: {day} (range is [1, 7])") + + # Now compute the offset from (Y, 1, 1) in days: + day_offset = (week - 1) * 7 + (day - 1) + + # Calculate the ordinal day for monday, week 1 + day_1 = _isoweek1monday(year) + ord_day = day_1 + day_offset + + return _ord2ymd(ord_day) + + +# Just raise TypeError if the arg isn't None or a string. +def _check_tzname(name): + if name is not None and not isinstance(name, str): + raise TypeError("tzinfo.tzname() must return None or string, " + f"not {type(name).__name__!r}") + +# name is the offset-producing method, "utcoffset" or "dst". +# offset is what it returned. +# If offset isn't None or timedelta, raises TypeError. +# If offset is None, returns None. +# Else offset is checked for being in range. +# If it is, its integer value is returned. Else ValueError is raised. +def _check_utc_offset(name, offset): + assert name in ("utcoffset", "dst") + if offset is None: + return + if not isinstance(offset, timedelta): + raise TypeError(f"tzinfo.{name}() must return None " + f"or timedelta, not {type(offset).__name__!r}") + if not -timedelta(1) < offset < timedelta(1): + raise ValueError("offset must be a timedelta " + "strictly between -timedelta(hours=24) and " + f"timedelta(hours=24), not {offset!r}") + +def _check_date_fields(year, month, day): + year = _index(year) + month = _index(month) + day = _index(day) + if not MINYEAR <= year <= MAXYEAR: + raise ValueError(f"year must be in {MINYEAR}..{MAXYEAR}, not {year}") + if not 1 <= month <= 12: + raise ValueError(f"month must be in 1..12, not {month}") + dim = _days_in_month(year, month) + if not 1 <= day <= dim: + raise ValueError(f"day {day} must be in range 1..{dim} for month {month} in year {year}") + return year, month, day + +def _check_time_fields(hour, minute, second, microsecond, fold): + hour = _index(hour) + minute = _index(minute) + second = _index(second) + microsecond = _index(microsecond) + if not 0 <= hour <= 23: + raise ValueError(f"hour must be in 0..23, not {hour}") + if not 0 <= minute <= 59: + raise ValueError(f"minute must be in 0..59, not {minute}") + if not 0 <= second <= 59: + raise ValueError(f"second must be in 0..59, not {second}") + if not 0 <= microsecond <= 999999: + raise ValueError(f"microsecond must be in 0..999999, not {microsecond}") + if fold not in (0, 1): + raise ValueError(f"fold must be either 0 or 1, not {fold}") + return hour, minute, second, microsecond, fold + +def _check_tzinfo_arg(tz): + if tz is not None and not isinstance(tz, tzinfo): + raise TypeError( + "tzinfo argument must be None or of a tzinfo subclass, " + f"not {type(tz).__name__!r}" + ) + +def _divide_and_round(a, b): + """divide a by b and round result to the nearest integer + + When the ratio is exactly half-way between two integers, + the even integer is returned. + """ + # Based on the reference implementation for divmod_near + # in Objects/longobject.c. + q, r = divmod(a, b) + # round up if either r / b > 0.5, or r / b == 0.5 and q is odd. + # The expression r / b > 0.5 is equivalent to 2 * r > b if b is + # positive, 2 * r < b if b negative. + r *= 2 + greater_than_half = r > b if b > 0 else r < b + if greater_than_half or r == b and q % 2 == 1: + q += 1 + + return q + + +class timedelta: + """Represent the difference between two datetime objects. + + Supported operators: + + - add, subtract timedelta + - unary plus, minus, abs + - compare to timedelta + - multiply, divide by int + + In addition, datetime supports subtraction of two datetime objects + returning a timedelta, and addition or subtraction of a datetime + and a timedelta giving a datetime. + + Representation: (days, seconds, microseconds). + """ + # The representation of (days, seconds, microseconds) was chosen + # arbitrarily; the exact rationale originally specified in the docstring + # was "Because I felt like it." + + __slots__ = '_days', '_seconds', '_microseconds', '_hashcode' + + def __new__(cls, days=0, seconds=0, microseconds=0, + milliseconds=0, minutes=0, hours=0, weeks=0): + # Doing this efficiently and accurately in C is going to be difficult + # and error-prone, due to ubiquitous overflow possibilities, and that + # C double doesn't have enough bits of precision to represent + # microseconds over 10K years faithfully. The code here tries to make + # explicit where go-fast assumptions can be relied on, in order to + # guide the C implementation; it's way more convoluted than speed- + # ignoring auto-overflow-to-long idiomatic Python could be. + + for name, value in ( + ("days", days), + ("seconds", seconds), + ("microseconds", microseconds), + ("milliseconds", milliseconds), + ("minutes", minutes), + ("hours", hours), + ("weeks", weeks) + ): + if not isinstance(value, (int, float)): + raise TypeError( + f"unsupported type for timedelta {name} component: {type(value).__name__}" + ) + + # Final values, all integer. + # s and us fit in 32-bit signed ints; d isn't bounded. + d = s = us = 0 + + # Normalize everything to days, seconds, microseconds. + days += weeks*7 + seconds += minutes*60 + hours*3600 + microseconds += milliseconds*1000 + + # Get rid of all fractions, and normalize s and us. + # Take a deep breath . + if isinstance(days, float): + dayfrac, days = _math.modf(days) + daysecondsfrac, daysecondswhole = _math.modf(dayfrac * (24.*3600.)) + assert daysecondswhole == int(daysecondswhole) # can't overflow + s = int(daysecondswhole) + assert days == int(days) + d = int(days) + else: + daysecondsfrac = 0.0 + d = days + assert isinstance(daysecondsfrac, float) + assert abs(daysecondsfrac) <= 1.0 + assert isinstance(d, int) + assert abs(s) <= 24 * 3600 + # days isn't referenced again before redefinition + + if isinstance(seconds, float): + secondsfrac, seconds = _math.modf(seconds) + assert seconds == int(seconds) + seconds = int(seconds) + secondsfrac += daysecondsfrac + assert abs(secondsfrac) <= 2.0 + else: + secondsfrac = daysecondsfrac + # daysecondsfrac isn't referenced again + assert isinstance(secondsfrac, float) + assert abs(secondsfrac) <= 2.0 + + assert isinstance(seconds, int) + days, seconds = divmod(seconds, 24*3600) + d += days + s += int(seconds) # can't overflow + assert isinstance(s, int) + assert abs(s) <= 2 * 24 * 3600 + # seconds isn't referenced again before redefinition + + usdouble = secondsfrac * 1e6 + assert abs(usdouble) < 2.1e6 # exact value not critical + # secondsfrac isn't referenced again + + if isinstance(microseconds, float): + microseconds = round(microseconds + usdouble) + seconds, microseconds = divmod(microseconds, 1000000) + days, seconds = divmod(seconds, 24*3600) + d += days + s += seconds + else: + microseconds = int(microseconds) + seconds, microseconds = divmod(microseconds, 1000000) + days, seconds = divmod(seconds, 24*3600) + d += days + s += seconds + microseconds = round(microseconds + usdouble) + assert isinstance(s, int) + assert isinstance(microseconds, int) + assert abs(s) <= 3 * 24 * 3600 + assert abs(microseconds) < 3.1e6 + + # Just a little bit of carrying possible for microseconds and seconds. + seconds, us = divmod(microseconds, 1000000) + s += seconds + days, s = divmod(s, 24*3600) + d += days + + assert isinstance(d, int) + assert isinstance(s, int) and 0 <= s < 24*3600 + assert isinstance(us, int) and 0 <= us < 1000000 + + if abs(d) > 999999999: + raise OverflowError("timedelta # of days is too large: %d" % d) + + self = object.__new__(cls) + self._days = d + self._seconds = s + self._microseconds = us + self._hashcode = -1 + return self + + def __repr__(self): + args = [] + if self._days: + args.append("days=%d" % self._days) + if self._seconds: + args.append("seconds=%d" % self._seconds) + if self._microseconds: + args.append("microseconds=%d" % self._microseconds) + if not args: + args.append('0') + return "%s%s(%s)" % (_get_class_module(self), + self.__class__.__qualname__, + ', '.join(args)) + + def __str__(self): + mm, ss = divmod(self._seconds, 60) + hh, mm = divmod(mm, 60) + s = "%d:%02d:%02d" % (hh, mm, ss) + if self._days: + def plural(n): + return n, abs(n) != 1 and "s" or "" + s = ("%d day%s, " % plural(self._days)) + s + if self._microseconds: + s = s + ".%06d" % self._microseconds + return s + + def total_seconds(self): + """Total seconds in the duration.""" + return ((self.days * 86400 + self.seconds) * 10**6 + + self.microseconds) / 10**6 + + # Read-only field accessors + @property + def days(self): + """days""" + return self._days + + @property + def seconds(self): + """seconds""" + return self._seconds + + @property + def microseconds(self): + """microseconds""" + return self._microseconds + + def __add__(self, other): + if isinstance(other, timedelta): + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days + other._days, + self._seconds + other._seconds, + self._microseconds + other._microseconds) + return NotImplemented + + __radd__ = __add__ + + def __sub__(self, other): + if isinstance(other, timedelta): + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days - other._days, + self._seconds - other._seconds, + self._microseconds - other._microseconds) + return NotImplemented + + def __rsub__(self, other): + if isinstance(other, timedelta): + return -self + other + return NotImplemented + + def __neg__(self): + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(-self._days, + -self._seconds, + -self._microseconds) + + def __pos__(self): + return self + + def __abs__(self): + if self._days < 0: + return -self + else: + return self + + def __mul__(self, other): + if isinstance(other, int): + # for CPython compatibility, we cannot use + # our __class__ here, but need a real timedelta + return timedelta(self._days * other, + self._seconds * other, + self._microseconds * other) + if isinstance(other, float): + usec = self._to_microseconds() + a, b = other.as_integer_ratio() + return timedelta(0, 0, _divide_and_round(usec * a, b)) + return NotImplemented + + __rmul__ = __mul__ + + def _to_microseconds(self): + return ((self._days * (24*3600) + self._seconds) * 1000000 + + self._microseconds) + + def __floordiv__(self, other): + if not isinstance(other, (int, timedelta)): + return NotImplemented + usec = self._to_microseconds() + if isinstance(other, timedelta): + return usec // other._to_microseconds() + if isinstance(other, int): + return timedelta(0, 0, usec // other) + + def __truediv__(self, other): + if not isinstance(other, (int, float, timedelta)): + return NotImplemented + usec = self._to_microseconds() + if isinstance(other, timedelta): + return usec / other._to_microseconds() + if isinstance(other, int): + return timedelta(0, 0, _divide_and_round(usec, other)) + if isinstance(other, float): + a, b = other.as_integer_ratio() + return timedelta(0, 0, _divide_and_round(b * usec, a)) + + def __mod__(self, other): + if isinstance(other, timedelta): + r = self._to_microseconds() % other._to_microseconds() + return timedelta(0, 0, r) + return NotImplemented + + def __divmod__(self, other): + if isinstance(other, timedelta): + q, r = divmod(self._to_microseconds(), + other._to_microseconds()) + return q, timedelta(0, 0, r) + return NotImplemented + + # Comparisons of timedelta objects with other. + + def __eq__(self, other): + if isinstance(other, timedelta): + return self._cmp(other) == 0 + else: + return NotImplemented + + def __le__(self, other): + if isinstance(other, timedelta): + return self._cmp(other) <= 0 + else: + return NotImplemented + + def __lt__(self, other): + if isinstance(other, timedelta): + return self._cmp(other) < 0 + else: + return NotImplemented + + def __ge__(self, other): + if isinstance(other, timedelta): + return self._cmp(other) >= 0 + else: + return NotImplemented + + def __gt__(self, other): + if isinstance(other, timedelta): + return self._cmp(other) > 0 + else: + return NotImplemented + + def _cmp(self, other): + assert isinstance(other, timedelta) + return _cmp(self._getstate(), other._getstate()) + + def __hash__(self): + if self._hashcode == -1: + self._hashcode = hash(self._getstate()) + return self._hashcode + + def __bool__(self): + return (self._days != 0 or + self._seconds != 0 or + self._microseconds != 0) + + # Pickle support. + + def _getstate(self): + return (self._days, self._seconds, self._microseconds) + + def __reduce__(self): + return (self.__class__, self._getstate()) + +timedelta.min = timedelta(-999999999) +timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, + microseconds=999999) +timedelta.resolution = timedelta(microseconds=1) + +class date: + """Concrete date type. + + Constructors: + + __new__() + fromtimestamp() + today() + fromordinal() + strptime() + + Operators: + + __repr__, __str__ + __eq__, __le__, __lt__, __ge__, __gt__, __hash__ + __add__, __radd__, __sub__ (add/radd only with timedelta arg) + + Methods: + + timetuple() + toordinal() + weekday() + isoweekday(), isocalendar(), isoformat() + ctime() + strftime() + + Properties (readonly): + year, month, day + """ + __slots__ = '_year', '_month', '_day', '_hashcode' + + def __new__(cls, year, month=None, day=None): + """Constructor. + + Arguments: + + year, month, day (required, base 1) + """ + if (month is None and + isinstance(year, (bytes, str)) and len(year) == 4 and + 1 <= ord(year[2:3]) <= 12): + # Pickle support + if isinstance(year, str): + try: + year = year.encode('latin1') + except UnicodeEncodeError: + # More informative error message. + raise ValueError( + "Failed to encode latin1 string when unpickling " + "a date object. " + "pickle.load(data, encoding='latin1') is assumed.") + self = object.__new__(cls) + self.__setstate(year) + self._hashcode = -1 + return self + year, month, day = _check_date_fields(year, month, day) + self = object.__new__(cls) + self._year = year + self._month = month + self._day = day + self._hashcode = -1 + return self + + # Additional constructors + + @classmethod + def fromtimestamp(cls, t): + "Construct a date from a POSIX timestamp (like time.time())." + if t is None: + raise TypeError("'NoneType' object cannot be interpreted as an integer") + y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) + return cls(y, m, d) + + @classmethod + def today(cls): + "Construct a date from time.time()." + t = _time.time() + return cls.fromtimestamp(t) + + @classmethod + def fromordinal(cls, n): + """Construct a date from a proleptic Gregorian ordinal. + + January 1 of year 1 is day 1. Only the year, month and day are + non-zero in the result. + """ + y, m, d = _ord2ymd(n) + return cls(y, m, d) + + @classmethod + def fromisoformat(cls, date_string): + """Construct a date from a string in ISO 8601 format.""" + + if not isinstance(date_string, str): + raise TypeError('Argument must be a str') + + if not date_string.isascii(): + raise ValueError('Argument must be an ASCII str') + + if len(date_string) not in (7, 8, 10): + raise ValueError(f'Invalid isoformat string: {date_string!r}') + + try: + return cls(*_parse_isoformat_date(date_string)) + except Exception: + raise ValueError(f'Invalid isoformat string: {date_string!r}') + + @classmethod + def fromisocalendar(cls, year, week, day): + """Construct a date from the ISO year, week number and weekday. + + This is the inverse of the date.isocalendar() function""" + return cls(*_isoweek_to_gregorian(year, week, day)) + + @classmethod + def strptime(cls, date_string, format): + """Parse a date string according to the given format (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_date(cls, date_string, format) + + # Conversions to string + + def __repr__(self): + """Convert to formal string, for repr(). + + >>> d = date(2010, 1, 1) + >>> repr(d) + 'datetime.date(2010, 1, 1)' + """ + return "%s%s(%d, %d, %d)" % (_get_class_module(self), + self.__class__.__qualname__, + self._year, + self._month, + self._day) + # XXX These shouldn't depend on time.localtime(), because that + # clips the usable dates to [1970 .. 2038). At least ctime() is + # easily done without using strftime() -- that's better too because + # strftime("%c", ...) is locale specific. + + + def ctime(self): + "Return ctime() style string." + weekday = self.toordinal() % 7 or 7 + return "%s %s %2d 00:00:00 %04d" % ( + _DAYNAMES[weekday], + _MONTHNAMES[self._month], + self._day, self._year) + + def strftime(self, format): + """ + Format using strftime(). + + Example: "%d/%m/%Y, %H:%M:%S" + """ + return _wrap_strftime(self, format, self.timetuple()) + + def __format__(self, fmt): + if not isinstance(fmt, str): + raise TypeError("must be str, not %s" % type(fmt).__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + + def isoformat(self): + """Return the date formatted according to ISO. + + This is 'YYYY-MM-DD'. + + References: + - https://www.w3.org/TR/NOTE-datetime + - https://www.cl.cam.ac.uk/~mgk25/iso-time.html + """ + return "%04d-%02d-%02d" % (self._year, self._month, self._day) + + __str__ = isoformat + + # Read-only field accessors + @property + def year(self): + """year (1-9999)""" + return self._year + + @property + def month(self): + """month (1-12)""" + return self._month + + @property + def day(self): + """day (1-31)""" + return self._day + + # Standard conversions, __eq__, __le__, __lt__, __ge__, __gt__, + # __hash__ (and helpers) + + def timetuple(self): + "Return local time tuple compatible with time.localtime()." + return _build_struct_time(self._year, self._month, self._day, + 0, 0, 0, -1) + + def toordinal(self): + """Return proleptic Gregorian ordinal for the year, month and day. + + January 1 of year 1 is day 1. Only the year, month and day values + contribute to the result. + """ + return _ymd2ord(self._year, self._month, self._day) + + def replace(self, year=None, month=None, day=None): + """Return a new date with new values for the specified fields.""" + if year is None: + year = self._year + if month is None: + month = self._month + if day is None: + day = self._day + return type(self)(year, month, day) + + __replace__ = replace + + # Comparisons of date objects with other. + + def __eq__(self, other): + if isinstance(other, date) and not isinstance(other, datetime): + return self._cmp(other) == 0 + return NotImplemented + + def __le__(self, other): + if isinstance(other, date) and not isinstance(other, datetime): + return self._cmp(other) <= 0 + return NotImplemented + + def __lt__(self, other): + if isinstance(other, date) and not isinstance(other, datetime): + return self._cmp(other) < 0 + return NotImplemented + + def __ge__(self, other): + if isinstance(other, date) and not isinstance(other, datetime): + return self._cmp(other) >= 0 + return NotImplemented + + def __gt__(self, other): + if isinstance(other, date) and not isinstance(other, datetime): + return self._cmp(other) > 0 + return NotImplemented + + def _cmp(self, other): + assert isinstance(other, date) + assert not isinstance(other, datetime) + y, m, d = self._year, self._month, self._day + y2, m2, d2 = other._year, other._month, other._day + return _cmp((y, m, d), (y2, m2, d2)) + + def __hash__(self): + "Hash." + if self._hashcode == -1: + self._hashcode = hash(self._getstate()) + return self._hashcode + + # Computations + + def __add__(self, other): + "Add a date to a timedelta." + if isinstance(other, timedelta): + o = self.toordinal() + other.days + if 0 < o <= _MAXORDINAL: + return type(self).fromordinal(o) + raise OverflowError("result out of range") + return NotImplemented + + __radd__ = __add__ + + def __sub__(self, other): + """Subtract two dates, or a date and a timedelta.""" + if isinstance(other, timedelta): + return self + timedelta(-other.days) + if isinstance(other, date): + days1 = self.toordinal() + days2 = other.toordinal() + return timedelta(days1 - days2) + return NotImplemented + + def weekday(self): + "Return day of the week, where Monday == 0 ... Sunday == 6." + return (self.toordinal() + 6) % 7 + + # Day-of-the-week and week-of-the-year, according to ISO + + def isoweekday(self): + "Return day of the week, where Monday == 1 ... Sunday == 7." + # 1-Jan-0001 is a Monday + return self.toordinal() % 7 or 7 + + def isocalendar(self): + """Return a named tuple containing ISO year, week number, and weekday. + + The first ISO week of the year is the (Mon-Sun) week + containing the year's first Thursday; everything else derives + from that. + + The first week is 1; Monday is 1 ... Sunday is 7. + + ISO calendar algorithm taken from + https://www.phys.uu.nl/~vgent/calendar/isocalendar.htm + (used with permission) + """ + year = self._year + week1monday = _isoweek1monday(year) + today = _ymd2ord(self._year, self._month, self._day) + # Internally, week and day have origin 0 + week, day = divmod(today - week1monday, 7) + if week < 0: + year -= 1 + week1monday = _isoweek1monday(year) + week, day = divmod(today - week1monday, 7) + elif week >= 52: + if today >= _isoweek1monday(year+1): + year += 1 + week = 0 + return _IsoCalendarDate(year, week+1, day+1) + + # Pickle support. + + def _getstate(self): + yhi, ylo = divmod(self._year, 256) + return bytes([yhi, ylo, self._month, self._day]), + + def __setstate(self, string): + yhi, ylo, self._month, self._day = string + self._year = yhi * 256 + ylo + + def __reduce__(self): + return (self.__class__, self._getstate()) + +_date_class = date # so functions w/ args named "date" can get at the class + +date.min = date(1, 1, 1) +date.max = date(9999, 12, 31) +date.resolution = timedelta(days=1) + + +class tzinfo: + """Abstract base class for time zone info classes. + + Subclasses must override the tzname(), utcoffset() and dst() methods. + """ + __slots__ = () + + def tzname(self, dt): + "datetime -> string name of time zone." + raise NotImplementedError("tzinfo subclass must override tzname()") + + def utcoffset(self, dt): + "datetime -> timedelta, positive for east of UTC, negative for west of UTC" + raise NotImplementedError("tzinfo subclass must override utcoffset()") + + def dst(self, dt): + """datetime -> DST offset as timedelta, positive for east of UTC. + + Return 0 if DST not in effect. utcoffset() must include the DST + offset. + """ + raise NotImplementedError("tzinfo subclass must override dst()") + + def fromutc(self, dt): + "datetime in UTC -> datetime in local time." + + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + + dtoff = dt.utcoffset() + if dtoff is None: + raise ValueError("fromutc() requires a non-None utcoffset() " + "result") + + # See the long comment block at the end of this file for an + # explanation of this algorithm. + dtdst = dt.dst() + if dtdst is None: + raise ValueError("fromutc() requires a non-None dst() result") + delta = dtoff - dtdst + if delta: + dt += delta + dtdst = dt.dst() + if dtdst is None: + raise ValueError("fromutc(): dt.dst gave inconsistent " + "results; cannot convert") + return dt + dtdst + + # Pickle support. + + def __reduce__(self): + getinitargs = getattr(self, "__getinitargs__", None) + if getinitargs: + args = getinitargs() + else: + args = () + return (self.__class__, args, self.__getstate__()) + + +class IsoCalendarDate(tuple): + + def __new__(cls, year, week, weekday, /): + return super().__new__(cls, (year, week, weekday)) + + @property + def year(self): + return self[0] + + @property + def week(self): + return self[1] + + @property + def weekday(self): + return self[2] + + def __reduce__(self): + # This code is intended to pickle the object without making the + # class public. See https://bugs.python.org/msg352381 + return (tuple, (tuple(self),)) + + def __repr__(self): + return (f'{self.__class__.__name__}' + f'(year={self[0]}, week={self[1]}, weekday={self[2]})') + + +_IsoCalendarDate = IsoCalendarDate +del IsoCalendarDate +_tzinfo_class = tzinfo + +class time: + """Time with time zone. + + Constructors: + + __new__() + strptime() + + Operators: + + __repr__, __str__ + __eq__, __le__, __lt__, __ge__, __gt__, __hash__ + + Methods: + + strftime() + isoformat() + utcoffset() + tzname() + dst() + + Properties (readonly): + hour, minute, second, microsecond, tzinfo, fold + """ + __slots__ = '_hour', '_minute', '_second', '_microsecond', '_tzinfo', '_hashcode', '_fold' + + def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0): + """Constructor. + + Arguments: + + hour, minute (required) + second, microsecond (default to zero) + tzinfo (default to None) + fold (keyword only, default to zero) + """ + if (isinstance(hour, (bytes, str)) and len(hour) == 6 and + ord(hour[0:1])&0x7F < 24): + # Pickle support + if isinstance(hour, str): + try: + hour = hour.encode('latin1') + except UnicodeEncodeError: + # More informative error message. + raise ValueError( + "Failed to encode latin1 string when unpickling " + "a time object. " + "pickle.load(data, encoding='latin1') is assumed.") + self = object.__new__(cls) + self.__setstate(hour, minute or None) + self._hashcode = -1 + return self + hour, minute, second, microsecond, fold = _check_time_fields( + hour, minute, second, microsecond, fold) + _check_tzinfo_arg(tzinfo) + self = object.__new__(cls) + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond + self._tzinfo = tzinfo + self._hashcode = -1 + self._fold = fold + return self + + @classmethod + def strptime(cls, date_string, format): + """string, format -> new time parsed from a string (like time.strptime()).""" + import _strptime + return _strptime._strptime_datetime_time(cls, date_string, format) + + # Read-only field accessors + @property + def hour(self): + """hour (0-23)""" + return self._hour + + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo + + @property + def fold(self): + return self._fold + + # Standard conversions, __hash__ (and helpers) + + # Comparisons of time objects with other. + + def __eq__(self, other): + if isinstance(other, time): + return self._cmp(other, allow_mixed=True) == 0 + else: + return NotImplemented + + def __le__(self, other): + if isinstance(other, time): + return self._cmp(other) <= 0 + else: + return NotImplemented + + def __lt__(self, other): + if isinstance(other, time): + return self._cmp(other) < 0 + else: + return NotImplemented + + def __ge__(self, other): + if isinstance(other, time): + return self._cmp(other) >= 0 + else: + return NotImplemented + + def __gt__(self, other): + if isinstance(other, time): + return self._cmp(other) > 0 + else: + return NotImplemented + + def _cmp(self, other, allow_mixed=False): + assert isinstance(other, time) + mytz = self._tzinfo + ottz = other._tzinfo + myoff = otoff = None + + if mytz is ottz: + base_compare = True + else: + myoff = self.utcoffset() + otoff = other.utcoffset() + base_compare = myoff == otoff + + if base_compare: + return _cmp((self._hour, self._minute, self._second, + self._microsecond), + (other._hour, other._minute, other._second, + other._microsecond)) + if myoff is None or otoff is None: + if allow_mixed: + return 2 # arbitrary non-zero value + else: + raise TypeError("cannot compare naive and aware times") + myhhmm = self._hour * 60 + self._minute - myoff//timedelta(minutes=1) + othhmm = other._hour * 60 + other._minute - otoff//timedelta(minutes=1) + return _cmp((myhhmm, self._second, self._microsecond), + (othhmm, other._second, other._microsecond)) + + def __hash__(self): + """Hash.""" + if self._hashcode == -1: + if self.fold: + t = self.replace(fold=0) + else: + t = self + tzoff = t.utcoffset() + if not tzoff: # zero or None + self._hashcode = hash(t._getstate()[0]) + else: + h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff, + timedelta(hours=1)) + assert not m % timedelta(minutes=1), "whole minute" + m //= timedelta(minutes=1) + if 0 <= h < 24: + self._hashcode = hash(time(h, m, self.second, self.microsecond)) + else: + self._hashcode = hash((h, m, self.second, self.microsecond)) + return self._hashcode + + # Conversion to string + + def _tzstr(self): + """Return formatted timezone offset (+xx:xx) or an empty string.""" + off = self.utcoffset() + return _format_offset(off) + + def __repr__(self): + """Convert to formal string, for repr().""" + if self._microsecond != 0: + s = ", %d, %d" % (self._second, self._microsecond) + elif self._second != 0: + s = ", %d" % self._second + else: + s = "" + s = "%s%s(%d, %d%s)" % (_get_class_module(self), + self.__class__.__qualname__, + self._hour, self._minute, s) + if self._tzinfo is not None: + assert s[-1:] == ")" + s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" + if self._fold: + assert s[-1:] == ")" + s = s[:-1] + ", fold=1)" + return s + + def isoformat(self, timespec='auto'): + """Return the time formatted according to ISO. + + The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional + part is omitted if self.microsecond == 0. + + The optional argument timespec specifies the number of additional + terms of the time to include. Valid options are 'auto', 'hours', + 'minutes', 'seconds', 'milliseconds' and 'microseconds'. + """ + s = _format_time(self._hour, self._minute, self._second, + self._microsecond, timespec) + tz = self._tzstr() + if tz: + s += tz + return s + + __str__ = isoformat + + @classmethod + def fromisoformat(cls, time_string): + """Construct a time from a string in one of the ISO 8601 formats.""" + if not isinstance(time_string, str): + raise TypeError('fromisoformat: argument must be str') + + # The spec actually requires that time-only ISO 8601 strings start with + # T, but the extended format allows this to be omitted as long as there + # is no ambiguity with date strings. + time_string = time_string.removeprefix('T') + + try: + return cls(*_parse_isoformat_time(time_string)[0]) + except Exception: + raise ValueError(f'Invalid isoformat string: {time_string!r}') + + def strftime(self, format): + """Format using strftime(). The date part of the timestamp passed + to underlying strftime should not be used. + """ + # The year must be >= 1000 else Python's strftime implementation + # can raise a bogus exception. + timetuple = (1900, 1, 1, + self._hour, self._minute, self._second, + 0, 1, -1) + return _wrap_strftime(self, format, timetuple) + + def __format__(self, fmt): + if not isinstance(fmt, str): + raise TypeError("must be str, not %s" % type(fmt).__name__) + if len(fmt) != 0: + return self.strftime(fmt) + return str(self) + + # Timezone functions + + def utcoffset(self): + """Return the timezone offset as timedelta, positive east of UTC + (negative west of UTC).""" + if self._tzinfo is None: + return None + offset = self._tzinfo.utcoffset(None) + _check_utc_offset("utcoffset", offset) + return offset + + def tzname(self): + """Return the timezone name. + + Note that the name is 100% informational -- there's no requirement that + it mean anything in particular. For example, "GMT", "UTC", "-500", + "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. + """ + if self._tzinfo is None: + return None + name = self._tzinfo.tzname(None) + _check_tzname(name) + return name + + def dst(self): + """Return 0 if DST is not in effect, or the DST offset (as timedelta + positive eastward) if DST is in effect. + + This is purely informational; the DST offset has already been added to + the UTC offset returned by utcoffset() if applicable, so there's no + need to consult dst() unless you're interested in displaying the DST + info. + """ + if self._tzinfo is None: + return None + offset = self._tzinfo.dst(None) + _check_utc_offset("dst", offset) + return offset + + def replace(self, hour=None, minute=None, second=None, microsecond=None, + tzinfo=True, *, fold=None): + """Return a new time with new values for the specified fields.""" + if hour is None: + hour = self.hour + if minute is None: + minute = self.minute + if second is None: + second = self.second + if microsecond is None: + microsecond = self.microsecond + if tzinfo is True: + tzinfo = self.tzinfo + if fold is None: + fold = self._fold + return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold) + + __replace__ = replace + + # Pickle support. + + def _getstate(self, protocol=3): + us2, us3 = divmod(self._microsecond, 256) + us1, us2 = divmod(us2, 256) + h = self._hour + if self._fold and protocol > 3: + h += 128 + basestate = bytes([h, self._minute, self._second, + us1, us2, us3]) + if self._tzinfo is None: + return (basestate,) + else: + return (basestate, self._tzinfo) + + def __setstate(self, string, tzinfo): + if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): + raise TypeError("bad tzinfo state arg") + h, self._minute, self._second, us1, us2, us3 = string + if h > 127: + self._fold = 1 + self._hour = h - 128 + else: + self._fold = 0 + self._hour = h + self._microsecond = (((us1 << 8) | us2) << 8) | us3 + self._tzinfo = tzinfo + + def __reduce_ex__(self, protocol): + return (self.__class__, self._getstate(protocol)) + + def __reduce__(self): + return self.__reduce_ex__(2) + +_time_class = time # so functions w/ args named "time" can get at the class + +time.min = time(0, 0, 0) +time.max = time(23, 59, 59, 999999) +time.resolution = timedelta(microseconds=1) + + +class datetime(date): + """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) + + The year, month and day arguments are required. tzinfo may be None, or an + instance of a tzinfo subclass. The remaining arguments may be ints. + """ + __slots__ = time.__slots__ + + def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, + microsecond=0, tzinfo=None, *, fold=0): + if (isinstance(year, (bytes, str)) and len(year) == 10 and + 1 <= ord(year[2:3])&0x7F <= 12): + # Pickle support + if isinstance(year, str): + try: + year = bytes(year, 'latin1') + except UnicodeEncodeError: + # More informative error message. + raise ValueError( + "Failed to encode latin1 string when unpickling " + "a datetime object. " + "pickle.load(data, encoding='latin1') is assumed.") + self = object.__new__(cls) + self.__setstate(year, month) + self._hashcode = -1 + return self + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond, fold = _check_time_fields( + hour, minute, second, microsecond, fold) + _check_tzinfo_arg(tzinfo) + self = object.__new__(cls) + self._year = year + self._month = month + self._day = day + self._hour = hour + self._minute = minute + self._second = second + self._microsecond = microsecond + self._tzinfo = tzinfo + self._hashcode = -1 + self._fold = fold + return self + + # Read-only field accessors + @property + def hour(self): + """hour (0-23)""" + return self._hour + + @property + def minute(self): + """minute (0-59)""" + return self._minute + + @property + def second(self): + """second (0-59)""" + return self._second + + @property + def microsecond(self): + """microsecond (0-999999)""" + return self._microsecond + + @property + def tzinfo(self): + """timezone info object""" + return self._tzinfo + + @property + def fold(self): + return self._fold + + @classmethod + def _fromtimestamp(cls, t, utc, tz): + """Construct a datetime from a POSIX timestamp (like time.time()). + + A timezone info object may be passed in as well. + """ + frac, t = _math.modf(t) + us = round(frac * 1e6) + if us >= 1000000: + t += 1 + us -= 1000000 + elif us < 0: + t -= 1 + us += 1000000 + + converter = _time.gmtime if utc else _time.localtime + y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) + ss = min(ss, 59) # clamp out leap seconds if the platform has them + result = cls(y, m, d, hh, mm, ss, us, tz) + if tz is None and not utc: + # As of version 2015f max fold in IANA database is + # 23 hours at 1969-09-30 13:00:00 in Kwajalein. + # Let's probe 24 hours in the past to detect a transition: + max_fold_seconds = 24 * 3600 + + # On Windows localtime_s throws an OSError for negative values, + # thus we can't perform fold detection for values of time less + # than the max time fold. See comments in _datetimemodule's + # version of this method for more details. + if t < max_fold_seconds and sys.platform.startswith("win"): + return result + + y, m, d, hh, mm, ss = converter(t - max_fold_seconds)[:6] + probe1 = cls(y, m, d, hh, mm, ss, us, tz) + trans = result - probe1 - timedelta(0, max_fold_seconds) + if trans.days < 0: + y, m, d, hh, mm, ss = converter(t + trans // timedelta(0, 1))[:6] + probe2 = cls(y, m, d, hh, mm, ss, us, tz) + if probe2 == result: + result._fold = 1 + elif tz is not None: + result = tz.fromutc(result) + return result + + @classmethod + def fromtimestamp(cls, timestamp, tz=None): + """Construct a datetime from a POSIX timestamp (like time.time()). + + A timezone info object may be passed in as well. + """ + _check_tzinfo_arg(tz) + + return cls._fromtimestamp(timestamp, tz is not None, tz) + + @classmethod + def utcfromtimestamp(cls, t): + """Construct a naive UTC datetime from a POSIX timestamp.""" + import warnings + warnings.warn("datetime.datetime.utcfromtimestamp() is deprecated and scheduled " + "for removal in a future version. Use timezone-aware " + "objects to represent datetimes in UTC: " + "datetime.datetime.fromtimestamp(t, datetime.UTC).", + DeprecationWarning, + stacklevel=2) + return cls._fromtimestamp(t, True, None) + + @classmethod + def now(cls, tz=None): + "Construct a datetime from time.time() and optional time zone info." + t = _time.time() + return cls.fromtimestamp(t, tz) + + @classmethod + def utcnow(cls): + "Construct a UTC datetime from time.time()." + import warnings + warnings.warn("datetime.datetime.utcnow() is deprecated and scheduled for " + "removal in a future version. Use timezone-aware " + "objects to represent datetimes in UTC: " + "datetime.datetime.now(datetime.UTC).", + DeprecationWarning, + stacklevel=2) + t = _time.time() + return cls._fromtimestamp(t, True, None) + + @classmethod + def combine(cls, date, time, tzinfo=True): + "Construct a datetime from a given date and a given time." + if not isinstance(date, _date_class): + raise TypeError("date argument must be a date instance") + if not isinstance(time, _time_class): + raise TypeError("time argument must be a time instance") + if tzinfo is True: + tzinfo = time.tzinfo + return cls(date.year, date.month, date.day, + time.hour, time.minute, time.second, time.microsecond, + tzinfo, fold=time.fold) + + @classmethod + def fromisoformat(cls, date_string): + """Construct a datetime from a string in one of the ISO 8601 formats.""" + if not isinstance(date_string, str): + raise TypeError('fromisoformat: argument must be str') + + if len(date_string) < 7: + raise ValueError(f'Invalid isoformat string: {date_string!r}') + + # Split this at the separator + try: + separator_location = _find_isoformat_datetime_separator(date_string) + dstr = date_string[0:separator_location] + tstr = date_string[(separator_location+1):] + + date_components = _parse_isoformat_date(dstr) + except ValueError: + raise ValueError( + f'Invalid isoformat string: {date_string!r}') from None + + if tstr: + try: + time_components, became_next_day, error_from_components = _parse_isoformat_time(tstr) + except ValueError: + raise ValueError( + f'Invalid isoformat string: {date_string!r}') from None + else: + if error_from_components: + raise ValueError("minute, second, and microsecond must be 0 when hour is 24") + + if became_next_day: + year, month, day = date_components + # Only wrap day/month when it was previously valid + if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + # Calculate midnight of the next day + day += 1 + if day > days_in_month: + day = 1 + month += 1 + if month > 12: + month = 1 + year += 1 + date_components = [year, month, day] + else: + time_components = [0, 0, 0, 0, None] + + return cls(*(date_components + time_components)) + + def timetuple(self): + "Return local time tuple compatible with time.localtime()." + dst = self.dst() + if dst is None: + dst = -1 + elif dst: + dst = 1 + else: + dst = 0 + return _build_struct_time(self.year, self.month, self.day, + self.hour, self.minute, self.second, + dst) + + def _mktime(self): + """Return integer POSIX timestamp.""" + epoch = datetime(1970, 1, 1) + max_fold_seconds = 24 * 3600 + t = (self - epoch) // timedelta(0, 1) + def local(u): + y, m, d, hh, mm, ss = _time.localtime(u)[:6] + return (datetime(y, m, d, hh, mm, ss) - epoch) // timedelta(0, 1) + + # Our goal is to solve t = local(u) for u. + a = local(t) - t + u1 = t - a + t1 = local(u1) + if t1 == t: + # We found one solution, but it may not be the one we need. + # Look for an earlier solution (if `fold` is 0), or a + # later one (if `fold` is 1). + u2 = u1 + (-max_fold_seconds, max_fold_seconds)[self.fold] + b = local(u2) - u2 + if a == b: + return u1 + else: + b = t1 - u1 + assert a != b + u2 = t - b + t2 = local(u2) + if t2 == t: + return u2 + if t1 == t: + return u1 + # We have found both offsets a and b, but neither t - a nor t - b is + # a solution. This means t is in the gap. + return (max, min)[self.fold](u1, u2) + + + def timestamp(self): + "Return POSIX timestamp as float" + if self._tzinfo is None: + s = self._mktime() + return s + self.microsecond / 1e6 + else: + return (self - _EPOCH).total_seconds() + + def utctimetuple(self): + "Return UTC time tuple compatible with time.gmtime()." + offset = self.utcoffset() + if offset: + self -= offset + y, m, d = self.year, self.month, self.day + hh, mm, ss = self.hour, self.minute, self.second + return _build_struct_time(y, m, d, hh, mm, ss, 0) + + def date(self): + "Return the date part." + return date(self._year, self._month, self._day) + + def time(self): + "Return the time part, with tzinfo None." + return time(self.hour, self.minute, self.second, self.microsecond, fold=self.fold) + + def timetz(self): + "Return the time part, with same tzinfo." + return time(self.hour, self.minute, self.second, self.microsecond, + self._tzinfo, fold=self.fold) + + def replace(self, year=None, month=None, day=None, hour=None, + minute=None, second=None, microsecond=None, tzinfo=True, + *, fold=None): + """Return a new datetime with new values for the specified fields.""" + if year is None: + year = self.year + if month is None: + month = self.month + if day is None: + day = self.day + if hour is None: + hour = self.hour + if minute is None: + minute = self.minute + if second is None: + second = self.second + if microsecond is None: + microsecond = self.microsecond + if tzinfo is True: + tzinfo = self.tzinfo + if fold is None: + fold = self.fold + return type(self)(year, month, day, hour, minute, second, + microsecond, tzinfo, fold=fold) + + __replace__ = replace + + def _local_timezone(self): + if self.tzinfo is None: + ts = self._mktime() + # Detect gap + ts2 = self.replace(fold=1-self.fold)._mktime() + if ts2 != ts: # This happens in a gap or a fold + if (ts2 > ts) == self.fold: + ts = ts2 + else: + ts = (self - _EPOCH) // timedelta(seconds=1) + localtm = _time.localtime(ts) + local = datetime(*localtm[:6]) + # Extract TZ data + gmtoff = localtm.tm_gmtoff + zone = localtm.tm_zone + return timezone(timedelta(seconds=gmtoff), zone) + + def astimezone(self, tz=None): + if tz is None: + tz = self._local_timezone() + elif not isinstance(tz, tzinfo): + raise TypeError("tz argument must be an instance of tzinfo") + + mytz = self.tzinfo + if mytz is None: + mytz = self._local_timezone() + myoffset = mytz.utcoffset(self) + else: + myoffset = mytz.utcoffset(self) + if myoffset is None: + mytz = self.replace(tzinfo=None)._local_timezone() + myoffset = mytz.utcoffset(self) + + if tz is mytz: + return self + + # Convert self to UTC, and attach the new time zone object. + utc = (self - myoffset).replace(tzinfo=tz) + + # Convert from UTC to tz's local time. + return tz.fromutc(utc) + + # Ways to produce a string. + + def ctime(self): + "Return ctime() style string." + weekday = self.toordinal() % 7 or 7 + return "%s %s %2d %02d:%02d:%02d %04d" % ( + _DAYNAMES[weekday], + _MONTHNAMES[self._month], + self._day, + self._hour, self._minute, self._second, + self._year) + + def isoformat(self, sep='T', timespec='auto'): + """Return the time formatted according to ISO. + + The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. + By default, the fractional part is omitted if self.microsecond == 0. + + If self.tzinfo is not None, the UTC offset is also attached, giving + a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. + + Optional argument sep specifies the separator between date and + time, default 'T'. + + The optional argument timespec specifies the number of additional + terms of the time to include. Valid options are 'auto', 'hours', + 'minutes', 'seconds', 'milliseconds' and 'microseconds'. + """ + s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + + _format_time(self._hour, self._minute, self._second, + self._microsecond, timespec)) + + off = self.utcoffset() + tz = _format_offset(off) + if tz: + s += tz + + return s + + def __repr__(self): + """Convert to formal string, for repr().""" + L = [self._year, self._month, self._day, # These are never zero + self._hour, self._minute, self._second, self._microsecond] + if L[-1] == 0: + del L[-1] + if L[-1] == 0: + del L[-1] + s = "%s%s(%s)" % (_get_class_module(self), + self.__class__.__qualname__, + ", ".join(map(str, L))) + if self._tzinfo is not None: + assert s[-1:] == ")" + s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" + if self._fold: + assert s[-1:] == ")" + s = s[:-1] + ", fold=1)" + return s + + def __str__(self): + "Convert to string, for str()." + return self.isoformat(sep=' ') + + @classmethod + def strptime(cls, date_string, format): + 'string, format -> new datetime parsed from a string (like time.strptime()).' + import _strptime + return _strptime._strptime_datetime_datetime(cls, date_string, format) + + def utcoffset(self): + """Return the timezone offset as timedelta positive east of UTC (negative west of + UTC).""" + if self._tzinfo is None: + return None + offset = self._tzinfo.utcoffset(self) + _check_utc_offset("utcoffset", offset) + return offset + + def tzname(self): + """Return the timezone name. + + Note that the name is 100% informational -- there's no requirement that + it mean anything in particular. For example, "GMT", "UTC", "-500", + "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. + """ + if self._tzinfo is None: + return None + name = self._tzinfo.tzname(self) + _check_tzname(name) + return name + + def dst(self): + """Return 0 if DST is not in effect, or the DST offset (as timedelta + positive eastward) if DST is in effect. + + This is purely informational; the DST offset has already been added to + the UTC offset returned by utcoffset() if applicable, so there's no + need to consult dst() unless you're interested in displaying the DST + info. + """ + if self._tzinfo is None: + return None + offset = self._tzinfo.dst(self) + _check_utc_offset("dst", offset) + return offset + + # Comparisons of datetime objects with other. + + def __eq__(self, other): + if isinstance(other, datetime): + return self._cmp(other, allow_mixed=True) == 0 + else: + return NotImplemented + + def __le__(self, other): + if isinstance(other, datetime): + return self._cmp(other) <= 0 + else: + return NotImplemented + + def __lt__(self, other): + if isinstance(other, datetime): + return self._cmp(other) < 0 + else: + return NotImplemented + + def __ge__(self, other): + if isinstance(other, datetime): + return self._cmp(other) >= 0 + else: + return NotImplemented + + def __gt__(self, other): + if isinstance(other, datetime): + return self._cmp(other) > 0 + else: + return NotImplemented + + def _cmp(self, other, allow_mixed=False): + assert isinstance(other, datetime) + mytz = self._tzinfo + ottz = other._tzinfo + myoff = otoff = None + + if mytz is ottz: + base_compare = True + else: + myoff = self.utcoffset() + otoff = other.utcoffset() + # Assume that allow_mixed means that we are called from __eq__ + if allow_mixed: + if myoff != self.replace(fold=not self.fold).utcoffset(): + return 2 + if otoff != other.replace(fold=not other.fold).utcoffset(): + return 2 + base_compare = myoff == otoff + + if base_compare: + return _cmp((self._year, self._month, self._day, + self._hour, self._minute, self._second, + self._microsecond), + (other._year, other._month, other._day, + other._hour, other._minute, other._second, + other._microsecond)) + if myoff is None or otoff is None: + if allow_mixed: + return 2 # arbitrary non-zero value + else: + raise TypeError("cannot compare naive and aware datetimes") + # XXX What follows could be done more efficiently... + diff = self - other # this will take offsets into account + if diff.days < 0: + return -1 + return diff and 1 or 0 + + def __add__(self, other): + "Add a datetime and a timedelta." + if not isinstance(other, timedelta): + return NotImplemented + delta = timedelta(self.toordinal(), + hours=self._hour, + minutes=self._minute, + seconds=self._second, + microseconds=self._microsecond) + delta += other + hour, rem = divmod(delta.seconds, 3600) + minute, second = divmod(rem, 60) + if 0 < delta.days <= _MAXORDINAL: + return type(self).combine(date.fromordinal(delta.days), + time(hour, minute, second, + delta.microseconds, + tzinfo=self._tzinfo)) + raise OverflowError("result out of range") + + __radd__ = __add__ + + def __sub__(self, other): + "Subtract two datetimes, or a datetime and a timedelta." + if not isinstance(other, datetime): + if isinstance(other, timedelta): + return self + -other + return NotImplemented + + days1 = self.toordinal() + days2 = other.toordinal() + secs1 = self._second + self._minute * 60 + self._hour * 3600 + secs2 = other._second + other._minute * 60 + other._hour * 3600 + base = timedelta(days1 - days2, + secs1 - secs2, + self._microsecond - other._microsecond) + if self._tzinfo is other._tzinfo: + return base + myoff = self.utcoffset() + otoff = other.utcoffset() + if myoff == otoff: + return base + if myoff is None or otoff is None: + raise TypeError("cannot mix naive and timezone-aware time") + return base + otoff - myoff + + def __hash__(self): + if self._hashcode == -1: + if self.fold: + t = self.replace(fold=0) + else: + t = self + tzoff = t.utcoffset() + if tzoff is None: + self._hashcode = hash(t._getstate()[0]) + else: + days = _ymd2ord(self.year, self.month, self.day) + seconds = self.hour * 3600 + self.minute * 60 + self.second + self._hashcode = hash(timedelta(days, seconds, self.microsecond) - tzoff) + return self._hashcode + + # Pickle support. + + def _getstate(self, protocol=3): + yhi, ylo = divmod(self._year, 256) + us2, us3 = divmod(self._microsecond, 256) + us1, us2 = divmod(us2, 256) + m = self._month + if self._fold and protocol > 3: + m += 128 + basestate = bytes([yhi, ylo, m, self._day, + self._hour, self._minute, self._second, + us1, us2, us3]) + if self._tzinfo is None: + return (basestate,) + else: + return (basestate, self._tzinfo) + + def __setstate(self, string, tzinfo): + if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): + raise TypeError("bad tzinfo state arg") + (yhi, ylo, m, self._day, self._hour, + self._minute, self._second, us1, us2, us3) = string + if m > 127: + self._fold = 1 + self._month = m - 128 + else: + self._fold = 0 + self._month = m + self._year = yhi * 256 + ylo + self._microsecond = (((us1 << 8) | us2) << 8) | us3 + self._tzinfo = tzinfo + + def __reduce_ex__(self, protocol): + return (self.__class__, self._getstate(protocol)) + + def __reduce__(self): + return self.__reduce_ex__(2) + + +datetime.min = datetime(1, 1, 1) +datetime.max = datetime(9999, 12, 31, 23, 59, 59, 999999) +datetime.resolution = timedelta(microseconds=1) + + +def _isoweek1monday(year): + # Helper to calculate the day number of the Monday starting week 1 + THURSDAY = 3 + firstday = _ymd2ord(year, 1, 1) + firstweekday = (firstday + 6) % 7 # See weekday() above + week1monday = firstday - firstweekday + if firstweekday > THURSDAY: + week1monday += 7 + return week1monday + + +class timezone(tzinfo): + __slots__ = '_offset', '_name' + + # Sentinel value to disallow None + _Omitted = object() + def __new__(cls, offset, name=_Omitted): + if not isinstance(offset, timedelta): + raise TypeError("offset must be a timedelta") + if name is cls._Omitted: + if not offset: + return cls.utc + name = None + elif not isinstance(name, str): + raise TypeError("name must be a string") + if not cls._minoffset <= offset <= cls._maxoffset: + raise ValueError("offset must be a timedelta " + "strictly between -timedelta(hours=24) and " + f"timedelta(hours=24), not {offset!r}") + return cls._create(offset, name) + + def __init_subclass__(cls): + raise TypeError("type 'datetime.timezone' is not an acceptable base type") + + @classmethod + def _create(cls, offset, name=None): + self = tzinfo.__new__(cls) + self._offset = offset + self._name = name + return self + + def __getinitargs__(self): + """pickle support""" + if self._name is None: + return (self._offset,) + return (self._offset, self._name) + + def __eq__(self, other): + if isinstance(other, timezone): + return self._offset == other._offset + return NotImplemented + + def __hash__(self): + return hash(self._offset) + + def __repr__(self): + """Convert to formal string, for repr(). + + >>> tz = timezone.utc + >>> repr(tz) + 'datetime.timezone.utc' + >>> tz = timezone(timedelta(hours=-5), 'EST') + >>> repr(tz) + "datetime.timezone(datetime.timedelta(-1, 68400), 'EST')" + """ + if self is self.utc: + return 'datetime.timezone.utc' + if self._name is None: + return "%s%s(%r)" % (_get_class_module(self), + self.__class__.__qualname__, + self._offset) + return "%s%s(%r, %r)" % (_get_class_module(self), + self.__class__.__qualname__, + self._offset, self._name) + + def __str__(self): + return self.tzname(None) + + def utcoffset(self, dt): + if isinstance(dt, datetime) or dt is None: + return self._offset + raise TypeError("utcoffset() argument must be a datetime instance" + " or None") + + def tzname(self, dt): + if isinstance(dt, datetime) or dt is None: + if self._name is None: + return self._name_from_offset(self._offset) + return self._name + raise TypeError("tzname() argument must be a datetime instance" + " or None") + + def dst(self, dt): + if isinstance(dt, datetime) or dt is None: + return None + raise TypeError("dst() argument must be a datetime instance" + " or None") + + def fromutc(self, dt): + if isinstance(dt, datetime): + if dt.tzinfo is not self: + raise ValueError("fromutc: dt.tzinfo " + "is not self") + return dt + self._offset + raise TypeError("fromutc() argument must be a datetime instance" + " or None") + + _maxoffset = timedelta(hours=24, microseconds=-1) + _minoffset = -_maxoffset + + @staticmethod + def _name_from_offset(delta): + if not delta: + return 'UTC' + if delta < timedelta(0): + sign = '-' + delta = -delta + else: + sign = '+' + hours, rest = divmod(delta, timedelta(hours=1)) + minutes, rest = divmod(rest, timedelta(minutes=1)) + seconds = rest.seconds + microseconds = rest.microseconds + if microseconds: + return (f'UTC{sign}{hours:02d}:{minutes:02d}:{seconds:02d}' + f'.{microseconds:06d}') + if seconds: + return f'UTC{sign}{hours:02d}:{minutes:02d}:{seconds:02d}' + return f'UTC{sign}{hours:02d}:{minutes:02d}' + +UTC = timezone.utc = timezone._create(timedelta(0)) + +# bpo-37642: These attributes are rounded to the nearest minute for backwards +# compatibility, even though the constructor will accept a wider range of +# values. This may change in the future. +timezone.min = timezone._create(-timedelta(hours=23, minutes=59)) +timezone.max = timezone._create(timedelta(hours=23, minutes=59)) +_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) + +# Some time zone algebra. For a datetime x, let +# x.n = x stripped of its timezone -- its naive time. +# x.o = x.utcoffset(), and assuming that doesn't raise an exception or +# return None +# x.d = x.dst(), and assuming that doesn't raise an exception or +# return None +# x.s = x's standard offset, x.o - x.d +# +# Now some derived rules, where k is a duration (timedelta). +# +# 1. x.o = x.s + x.d +# This follows from the definition of x.s. +# +# 2. If x and y have the same tzinfo member, x.s = y.s. +# This is actually a requirement, an assumption we need to make about +# sane tzinfo classes. +# +# 3. The naive UTC time corresponding to x is x.n - x.o. +# This is again a requirement for a sane tzinfo class. +# +# 4. (x+k).s = x.s +# This follows from #2, and that datetime.timetz+timedelta preserves tzinfo. +# +# 5. (x+k).n = x.n + k +# Again follows from how arithmetic is defined. +# +# Now we can explain tz.fromutc(x). Let's assume it's an interesting case +# (meaning that the various tzinfo methods exist, and don't blow up or return +# None when called). +# +# The function wants to return a datetime y with timezone tz, equivalent to x. +# x is already in UTC. +# +# By #3, we want +# +# y.n - y.o = x.n [1] +# +# The algorithm starts by attaching tz to x.n, and calling that y. So +# x.n = y.n at the start. Then it wants to add a duration k to y, so that [1] +# becomes true; in effect, we want to solve [2] for k: +# +# (y+k).n - (y+k).o = x.n [2] +# +# By #1, this is the same as +# +# (y+k).n - ((y+k).s + (y+k).d) = x.n [3] +# +# By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start. +# Substituting that into [3], +# +# x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving +# k - (y+k).s - (y+k).d = 0; rearranging, +# k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so +# k = y.s - (y+k).d +# +# On the RHS, (y+k).d can't be computed directly, but y.s can be, and we +# approximate k by ignoring the (y+k).d term at first. Note that k can't be +# very large, since all offset-returning methods return a duration of magnitude +# less than 24 hours. For that reason, if y is firmly in std time, (y+k).d must +# be 0, so ignoring it has no consequence then. +# +# In any case, the new value is +# +# z = y + y.s [4] +# +# It's helpful to step back at look at [4] from a higher level: it's simply +# mapping from UTC to tz's standard time. +# +# At this point, if +# +# z.n - z.o = x.n [5] +# +# we have an equivalent time, and are almost done. The insecurity here is +# at the start of daylight time. Picture US Eastern for concreteness. The wall +# time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good +# sense then. The docs ask that an Eastern tzinfo class consider such a time to +# be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST +# on the day DST starts. We want to return the 1:MM EST spelling because that's +# the only spelling that makes sense on the local wall clock. +# +# In fact, if [5] holds at this point, we do have the standard-time spelling, +# but that takes a bit of proof. We first prove a stronger result. What's the +# difference between the LHS and RHS of [5]? Let +# +# diff = x.n - (z.n - z.o) [6] +# +# Now +# z.n = by [4] +# (y + y.s).n = by #5 +# y.n + y.s = since y.n = x.n +# x.n + y.s = since z and y are have the same tzinfo member, +# y.s = z.s by #2 +# x.n + z.s +# +# Plugging that back into [6] gives +# +# diff = +# x.n - ((x.n + z.s) - z.o) = expanding +# x.n - x.n - z.s + z.o = cancelling +# - z.s + z.o = by #2 +# z.d +# +# So diff = z.d. +# +# If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time +# spelling we wanted in the endcase described above. We're done. Contrarily, +# if z.d = 0, then we have a UTC equivalent, and are also done. +# +# If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to +# add to z (in effect, z is in tz's standard time, and we need to shift the +# local clock into tz's daylight time). +# +# Let +# +# z' = z + z.d = z + diff [7] +# +# and we can again ask whether +# +# z'.n - z'.o = x.n [8] +# +# If so, we're done. If not, the tzinfo class is insane, according to the +# assumptions we've made. This also requires a bit of proof. As before, let's +# compute the difference between the LHS and RHS of [8] (and skipping some of +# the justifications for the kinds of substitutions we've done several times +# already): +# +# diff' = x.n - (z'.n - z'.o) = replacing z'.n via [7] +# x.n - (z.n + diff - z'.o) = replacing diff via [6] +# x.n - (z.n + x.n - (z.n - z.o) - z'.o) = +# x.n - z.n - x.n + z.n - z.o + z'.o = cancel x.n +# - z.n + z.n - z.o + z'.o = cancel z.n +# - z.o + z'.o = #1 twice +# -z.s - z.d + z'.s + z'.d = z and z' have same tzinfo +# z'.d - z.d +# +# So z' is UTC-equivalent to x iff z'.d = z.d at this point. If they are equal, +# we've found the UTC-equivalent so are done. In fact, we stop with [7] and +# return z', not bothering to compute z'.d. +# +# How could z.d and z'd differ? z' = z + z.d [7], so merely moving z' by +# a dst() offset, and starting *from* a time already in DST (we know z.d != 0), +# would have to change the result dst() returns: we start in DST, and moving +# a little further into it takes us out of DST. +# +# There isn't a sane case where this can happen. The closest it gets is at +# the end of DST, where there's an hour in UTC with no spelling in a hybrid +# tzinfo class. In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT. During +# that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM +# UTC) because the docs insist on that, but 0:MM is taken as being in daylight +# time (4:MM UTC). There is no local time mapping to 5:MM UTC. The local +# clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in +# standard time. Since that's what the local clock *does*, we want to map both +# UTC hours 5:MM and 6:MM to 1:MM Eastern. The result is ambiguous +# in local time, but so it goes -- it's the way the local clock works. +# +# When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0, +# so z=0:MM. z.d=60 (minutes) then, so [5] doesn't hold and we keep going. +# z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8] +# (correctly) concludes that z' is not UTC-equivalent to x. +# +# Because we know z.d said z was in daylight time (else [5] would have held and +# we would have stopped then), and we know z.d != z'.d (else [8] would have held +# and we have stopped then), and there are only 2 possible values dst() can +# return in Eastern, it follows that z'.d must be 0 (which it is in the example, +# but the reasoning doesn't depend on the example -- it depends on there being +# two possible dst() outcomes, one zero and the other non-zero). Therefore +# z' must be in standard time, and is the spelling we want in this case. +# +# Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is +# concerned (because it takes z' as being in standard time rather than the +# daylight time we intend here), but returning it gives the real-life "local +# clock repeats an hour" behavior when mapping the "unspellable" UTC hour into +# tz. +# +# When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with +# the 1:MM standard time spelling we want. +# +# So how can this break? One of the assumptions must be violated. Two +# possibilities: +# +# 1) [2] effectively says that y.s is invariant across all y belong to a given +# time zone. This isn't true if, for political reasons or continental drift, +# a region decides to change its base offset from UTC. +# +# 2) There may be versions of "double daylight" time where the tail end of +# the analysis gives up a step too early. I haven't thought about that +# enough to say. +# +# In any case, it's clear that the default fromutc() is strong enough to handle +# "almost all" time zones: so long as the standard offset is invariant, it +# doesn't matter if daylight time transition points change from year to year, or +# if daylight time is skipped in some years; it doesn't matter how large or +# small dst() may get within its bounds; and it doesn't even matter if some +# perverse time zone returns a negative dst()). So a breaking case must be +# pretty bizarre, and a tzinfo subclass can override fromutc() if it is. diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index e7df67dc9b9..97a629fe92c 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -13,104 +13,7 @@ # bug) and will be backported. At this point the spec is stabilizing # and the updates are becoming fewer, smaller, and less significant. -""" -This is an implementation of decimal floating point arithmetic based on -the General Decimal Arithmetic Specification: - - http://speleotrove.com/decimal/decarith.html - -and IEEE standard 854-1987: - - http://en.wikipedia.org/wiki/IEEE_854-1987 - -Decimal floating point has finite precision with arbitrarily large bounds. - -The purpose of this module is to support arithmetic using familiar -"schoolhouse" rules and to avoid some of the tricky representation -issues associated with binary floating point. The package is especially -useful for financial applications or for contexts where users have -expectations that are at odds with binary floating point (for instance, -in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead -of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected -Decimal('0.00')). - -Here are some examples of using the decimal module: - ->>> from decimal import * ->>> setcontext(ExtendedContext) ->>> Decimal(0) -Decimal('0') ->>> Decimal('1') -Decimal('1') ->>> Decimal('-.0123') -Decimal('-0.0123') ->>> Decimal(123456) -Decimal('123456') ->>> Decimal('123.45e12345678') -Decimal('1.2345E+12345680') ->>> Decimal('1.33') + Decimal('1.27') -Decimal('2.60') ->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') -Decimal('-2.20') ->>> dig = Decimal(1) ->>> print(dig / Decimal(3)) -0.333333333 ->>> getcontext().prec = 18 ->>> print(dig / Decimal(3)) -0.333333333333333333 ->>> print(dig.sqrt()) -1 ->>> print(Decimal(3).sqrt()) -1.73205080756887729 ->>> print(Decimal(3) ** 123) -4.85192780976896427E+58 ->>> inf = Decimal(1) / Decimal(0) ->>> print(inf) -Infinity ->>> neginf = Decimal(-1) / Decimal(0) ->>> print(neginf) --Infinity ->>> print(neginf + inf) -NaN ->>> print(neginf * inf) --Infinity ->>> print(dig / 0) -Infinity ->>> getcontext().traps[DivisionByZero] = 1 ->>> print(dig / 0) -Traceback (most recent call last): - ... - ... - ... -decimal.DivisionByZero: x / 0 ->>> c = Context() ->>> c.traps[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> c.divide(Decimal(0), Decimal(0)) -Decimal('NaN') ->>> c.traps[InvalidOperation] = 1 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> print(c.divide(Decimal(0), Decimal(0))) -Traceback (most recent call last): - ... - ... - ... -decimal.InvalidOperation: 0 / 0 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> c.traps[InvalidOperation] = 0 ->>> print(c.divide(Decimal(0), Decimal(0))) -NaN ->>> print(c.flags[InvalidOperation]) -1 ->>> -""" +"""Python decimal arithmetic module""" __all__ = [ # Two major classes @@ -135,13 +38,16 @@ 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', # Functions for manipulating contexts - 'setcontext', 'getcontext', 'localcontext', + 'setcontext', 'getcontext', 'localcontext', 'IEEEContext', # Limits for the C version for compatibility - 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', + 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', 'IEEE_CONTEXT_MAX_BITS', - # C version: compile time choice that enables the thread local context - 'HAVE_THREADS' + # C version: compile time choice that enables the thread local context (deprecated, now always true) + 'HAVE_THREADS', + + # C version: compile time choice that enables the coroutine local context + 'HAVE_CONTEXTVAR' ] __xname__ = __name__ # sys.modules lookup (--without-threads) @@ -156,7 +62,7 @@ try: from collections import namedtuple as _namedtuple - DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent') + DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent', module='decimal') except ImportError: DecimalTuple = lambda *args: args @@ -172,14 +78,17 @@ # Compatibility with the C version HAVE_THREADS = True +HAVE_CONTEXTVAR = True if sys.maxsize == 2**63-1: MAX_PREC = 999999999999999999 MAX_EMAX = 999999999999999999 MIN_EMIN = -999999999999999999 + IEEE_CONTEXT_MAX_BITS = 512 else: MAX_PREC = 425000000 MAX_EMAX = 425000000 MIN_EMIN = -425000000 + IEEE_CONTEXT_MAX_BITS = 256 MIN_ETINY = MIN_EMIN - (MAX_PREC-1) @@ -190,7 +99,7 @@ class DecimalException(ArithmeticError): Used exceptions derive from this. If an exception derives from another exception besides this (such as - Underflow (Inexact, Rounded, Subnormal) that indicates that it is only + Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only called if the others are present. This isn't actually used for anything, though. @@ -238,7 +147,7 @@ class InvalidOperation(DecimalException): x ** (+-)INF An operand is invalid - The result of the operation after these is a quiet positive NaN, + The result of the operation after this is a quiet positive NaN, except when the cause is a signaling NaN, in which case the result is also a quiet NaN, but with the original sign, and an optional diagnostic information. @@ -431,82 +340,40 @@ class FloatOperation(DecimalException, TypeError): ##### Context Functions ################################################## # The getcontext() and setcontext() function manage access to a thread-local -# current context. Py2.4 offers direct support for thread locals. If that -# is not available, use threading.current_thread() which is slower but will -# work for older Pythons. If threads are not part of the build, create a -# mock threading object with threading.local() returning the module namespace. - -try: - import threading -except ImportError: - # Python was compiled without threads; create a mock object instead - class MockThreading(object): - def local(self, sys=sys): - return sys.modules[__xname__] - threading = MockThreading() - del MockThreading - -try: - threading.local - -except AttributeError: +# current context. - # To fix reloading, force it to create a new context - # Old contexts have different exceptions in their dicts, making problems. - if hasattr(threading.current_thread(), '__decimal_context__'): - del threading.current_thread().__decimal_context__ +import contextvars - def setcontext(context): - """Set this thread's context to context.""" - if context in (DefaultContext, BasicContext, ExtendedContext): - context = context.copy() - context.clear_flags() - threading.current_thread().__decimal_context__ = context +_current_context_var = contextvars.ContextVar('decimal_context') - def getcontext(): - """Returns this thread's context. - - If this thread does not yet have a context, returns - a new context and sets this thread's context. - New contexts are copies of DefaultContext. - """ - try: - return threading.current_thread().__decimal_context__ - except AttributeError: - context = Context() - threading.current_thread().__decimal_context__ = context - return context +_context_attributes = frozenset( + ['prec', 'Emin', 'Emax', 'capitals', 'clamp', 'rounding', 'flags', 'traps'] +) -else: +def getcontext(): + """Returns this thread's context. - local = threading.local() - if hasattr(local, '__decimal_context__'): - del local.__decimal_context__ + If this thread does not yet have a context, returns + a new context and sets this thread's context. + New contexts are copies of DefaultContext. + """ + try: + return _current_context_var.get() + except LookupError: + context = Context() + _current_context_var.set(context) + return context + +def setcontext(context): + """Set this thread's context to context.""" + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + _current_context_var.set(context) - def getcontext(_local=local): - """Returns this thread's context. +del contextvars # Don't contaminate the namespace - If this thread does not yet have a context, returns - a new context and sets this thread's context. - New contexts are copies of DefaultContext. - """ - try: - return _local.__decimal_context__ - except AttributeError: - context = Context() - _local.__decimal_context__ = context - return context - - def setcontext(context, _local=local): - """Set this thread's context to context.""" - if context in (DefaultContext, BasicContext, ExtendedContext): - context = context.copy() - context.clear_flags() - _local.__decimal_context__ = context - - del threading, local # Don't contaminate the namespace - -def localcontext(ctx=None): +def localcontext(ctx=None, **kwargs): """Return a context manager for a copy of the supplied context Uses a copy of the current context if no context is specified @@ -542,8 +409,35 @@ def sin(x): >>> print(getcontext().prec) 28 """ - if ctx is None: ctx = getcontext() - return _ContextManager(ctx) + if ctx is None: + ctx = getcontext() + ctx_manager = _ContextManager(ctx) + for key, value in kwargs.items(): + if key not in _context_attributes: + raise TypeError(f"'{key}' is an invalid keyword argument for this function") + setattr(ctx_manager.new_context, key, value) + return ctx_manager + + +def IEEEContext(bits, /): + """ + Return a context object initialized to the proper values for one of the + IEEE interchange formats. The argument must be a multiple of 32 and less + than IEEE_CONTEXT_MAX_BITS. + """ + if bits <= 0 or bits > IEEE_CONTEXT_MAX_BITS or bits % 32: + raise ValueError("argument must be a multiple of 32, " + f"with a maximum of {IEEE_CONTEXT_MAX_BITS}") + + ctx = Context() + ctx.prec = 9 * (bits//32) - 2 + ctx.Emax = 3 * (1 << (bits//16 + 3)) + ctx.Emin = 1 - ctx.Emax + ctx.rounding = ROUND_HALF_EVEN + ctx.clamp = 1 + ctx.traps = dict.fromkeys(_signals, False) + + return ctx ##### Decimal class ####################################################### @@ -553,7 +447,7 @@ def sin(x): # numbers.py for more detail. class Decimal(object): - """Floating point class for decimal arithmetic.""" + """Floating-point class for decimal arithmetic.""" __slots__ = ('_exp','_int','_sign', '_is_special') # Generally, the value of the Decimal instance is given by @@ -711,6 +605,21 @@ def __new__(cls, value="0", context=None): raise TypeError("Cannot convert %r to Decimal" % value) + @classmethod + def from_number(cls, number): + """Converts a real number to a decimal number, exactly. + + >>> Decimal.from_number(314) # int + Decimal('314') + >>> Decimal.from_number(0.1) # float + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_number(Decimal('3.14')) # another decimal instance + Decimal('3.14') + """ + if isinstance(number, (int, Decimal, float)): + return cls(number) + raise TypeError("Cannot convert %r to Decimal" % number) + @classmethod def from_float(cls, f): """Converts a float to a decimal number, exactly. @@ -734,18 +643,23 @@ def from_float(cls, f): """ if isinstance(f, int): # handle integer inputs - return cls(f) - if not isinstance(f, float): - raise TypeError("argument must be int or float.") - if _math.isinf(f) or _math.isnan(f): - return cls(repr(f)) - if _math.copysign(1.0, f) == 1.0: - sign = 0 + sign = 0 if f >= 0 else 1 + k = 0 + coeff = str(abs(f)) + elif isinstance(f, float): + if _math.isinf(f) or _math.isnan(f): + return cls(repr(f)) + if _math.copysign(1.0, f) == 1.0: + sign = 0 + else: + sign = 1 + n, d = abs(f).as_integer_ratio() + k = d.bit_length() - 1 + coeff = str(n*5**k) else: - sign = 1 - n, d = abs(f).as_integer_ratio() - k = d.bit_length() - 1 - result = _dec_from_triple(sign, str(n*5**k), -k) + raise TypeError("argument must be int or float.") + + result = _dec_from_triple(sign, coeff, -k) if cls is Decimal: return result else: @@ -988,7 +902,7 @@ def __hash__(self): if self.is_snan(): raise TypeError('Cannot hash a signaling NaN value.') elif self.is_nan(): - return _PyHASH_NAN + return object.__hash__(self) else: if self._sign: return -_PyHASH_INF @@ -1669,13 +1583,13 @@ def __int__(self): __trunc__ = __int__ + @property def real(self): return self - real = property(real) + @property def imag(self): return Decimal(0) - imag = property(imag) def conjugate(self): return self @@ -2255,10 +2169,16 @@ def _power_exact(self, other, p): else: return None - if xc >= 10**p: + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self + strxc = str(xc) + if len(strxc) > p: return None xe = -e-xe - return _dec_from_triple(0, str(xc), xe) + return _dec_from_triple(0, strxc, xe) # now y is positive; find m and n such that y = m/n if ye >= 0: @@ -2267,7 +2187,7 @@ def _power_exact(self, other, p): if xe != 0 and len(str(abs(yc*xe))) <= -ye: return None xc_bits = _nbits(xc) - if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye: + if len(str(abs(yc)*xc_bits)) <= -ye: return None m, n = yc, 10**(-ye) while m % 2 == n % 2 == 0: @@ -2280,7 +2200,7 @@ def _power_exact(self, other, p): # compute nth root of xc*10**xe if n > 1: # if 1 < xc < 2**n then xc isn't an nth power - if xc != 1 and xc_bits <= n: + if xc_bits <= n: return None xe, rem = divmod(xe, n) @@ -2308,13 +2228,18 @@ def _power_exact(self, other, p): return None xc = xc**m xe *= m - if xc > 10**p: + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: return None # by this point the result *is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary - str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._exp*int(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) @@ -2538,12 +2463,12 @@ def __pow__(self, other, modulo=None, context=None): return ans - def __rpow__(self, other, context=None): + def __rpow__(self, other, modulo=None, context=None): """Swaps self/other and returns __pow__.""" other = _convert_other(other) if other is NotImplemented: return other - return other.__pow__(self, context=context) + return other.__pow__(self, modulo, context=context) def normalize(self, context=None): """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" @@ -3415,7 +3340,10 @@ def _fill_logical(self, context, opa, opb): return opa, opb def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" + """Applies an 'and' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3432,14 +3360,20 @@ def logical_and(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_invert(self, context=None): - """Invert all its digits.""" + """Invert all its digits. + + The self must be logical number. + """ if context is None: context = getcontext() return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), context) def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" + """Applies an 'or' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3456,7 +3390,10 @@ def logical_or(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" + """Applies an 'xor' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3832,6 +3769,10 @@ def __format__(self, specifier, context=None, _localeconv=None): # represented in fixed point; rescale them to 0e0. if not self and self._exp > 0 and spec['type'] in 'fF%': self = self._rescale(0, rounding) + if not self and spec['no_neg_0'] and self._sign: + adjusted_sign = 0 + else: + adjusted_sign = self._sign # figure out placement of the decimal point leftdigits = self._exp + len(self._int) @@ -3862,7 +3803,7 @@ def __format__(self, specifier, context=None, _localeconv=None): # done with the decimal-specific stuff; hand over the rest # of the formatting to the _format_number function - return _format_number(self._sign, intpart, fracpart, exp, spec) + return _format_number(adjusted_sign, intpart, fracpart, exp, spec) def _dec_from_triple(sign, coefficient, exponent, special=False): """Create a decimal instance directly, without any validation, @@ -5672,8 +5613,6 @@ def __init__(self, value=None): def __repr__(self): return "(%r, %r, %r)" % (self.sign, self.int, self.exp) - __str__ = __repr__ - def _normalize(op1, op2, prec = 0): @@ -6169,7 +6108,7 @@ def _convert_for_comparison(self, other, equality_op=False): (?P\d*) # with (possibly empty) diagnostic info. ) # \s* - \Z + \z """, re.VERBOSE | re.IGNORECASE).match _all_zeros = re.compile('0*$').match @@ -6182,7 +6121,7 @@ def _convert_for_comparison(self, other, equality_op=False): # # A format specifier for Decimal looks like: # -# [[fill]align][sign][#][0][minimumwidth][,][.precision][type] +# [[fill]align][sign][z][#][0][minimumwidth][,][.precision][type] _parse_format_specifier_regex = re.compile(r"""\A (?: @@ -6190,13 +6129,18 @@ def _convert_for_comparison(self, other, equality_op=False): (?P[<>=^]) )? (?P[-+ ])? +(?Pz)? (?P\#)? (?P0)? -(?P(?!0)\d+)? -(?P,)? -(?:\.(?P0|(?!0)\d+))? +(?P\d+)? +(?P[,_])? +(?:\. + (?=[\d,_]) # lookahead for digit or separator + (?P\d+)? + (?P[,_])? +)? (?P[eEfFgGn%])? -\Z +\z """, re.VERBOSE|re.DOTALL) del re @@ -6287,6 +6231,9 @@ def _parse_format_specifier(format_spec, _localeconv=None): format_dict['grouping'] = [3, 0] format_dict['decimal_point'] = '.' + if format_dict['frac_separators'] is None: + format_dict['frac_separators'] = '' + return format_dict def _format_align(sign, body, spec): @@ -6406,6 +6353,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): sign = _format_sign(is_negative, spec) + frac_sep = spec['frac_separators'] + if fracpart and frac_sep: + fracpart = frac_sep.join(fracpart[pos:pos + 3] + for pos in range(0, len(fracpart), 3)) + if fracpart or spec['alt']: fracpart = spec['decimal_point'] + fracpart diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 56e9a0cb33c..116ce4f37ec 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -16,15 +16,16 @@ _setmode = None import io -from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) +from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END, Reader, Writer) # noqa: F401 valid_seek_flags = {0, 1, 2} # Hardwired values if hasattr(os, 'SEEK_HOLE') : valid_seek_flags.add(os.SEEK_HOLE) valid_seek_flags.add(os.SEEK_DATA) -# open() uses st_blksize whenever we can -DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes +# open() uses max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) +# when the device block size is available. +DEFAULT_BUFFER_SIZE = 128 * 1024 # bytes # NOTE: Base classes defined here are registered with the "official" ABCs # defined in io.py. We don't use real inheritance though, because we don't want @@ -33,19 +34,17 @@ # Rebind for compatibility BlockingIOError = BlockingIOError -# Does io.IOBase finalizer log the exception if the close() method fails? -# The exception is ignored silently by default in release build. -_IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode) # Does open() check its 'errors' argument? -_CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE +_CHECK_ERRORS = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode) def text_encoding(encoding, stacklevel=2): """ A helper function to choose the text encoding. - When encoding is not None, just return it. - Otherwise, return the default text encoding (i.e. "locale"). + When encoding is not None, this function returns it. + Otherwise, this function returns the default text encoding + (i.e. "locale" or "utf-8" depends on UTF-8 mode). This function emits an EncodingWarning if *encoding* is None and sys.flags.warn_default_encoding is true. @@ -55,7 +54,10 @@ def text_encoding(encoding, stacklevel=2): However, please consider using encoding="utf-8" for new APIs. """ if encoding is None: - encoding = "locale" + if sys.flags.utf8_mode: + encoding = "utf-8" + else: + encoding = "locale" if sys.flags.warn_default_encoding: import warnings warnings.warn("'encoding' argument not specified.", @@ -101,7 +103,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - 'U' universal newline mode (deprecated) ========= =============================================================== The default mode is 'rt' (open for reading text). For binary random @@ -117,20 +118,16 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. - 'U' mode is deprecated and will raise an exception in future versions - of Python. It has no effect in Python 3. Use newline to control - universal newlines mode. - buffering is an optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate the size of a fixed-size chunk buffer. When no buffering argument is given, the default buffering policy works as follows: - * Binary files are buffered in fixed-size chunks; the size of the buffer - is chosen using a heuristic trying to determine the underlying device's - "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`. - On many systems, the buffer will typically be 4096 or 8192 bytes long. + * Binary files are buffered in fixed-size chunks; the size of the buffer + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) + when the device block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -206,7 +203,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, if errors is not None and not isinstance(errors, str): raise TypeError("invalid errors: %r" % errors) modes = set(mode) - if modes - set("axrwb+tU") or len(mode) > len(modes): + if modes - set("axrwb+t") or len(mode) > len(modes): raise ValueError("invalid mode: %r" % mode) creating = "x" in modes reading = "r" in modes @@ -215,13 +212,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, updating = "+" in modes text = "t" in modes binary = "b" in modes - if "U" in modes: - if creating or writing or appending or updating: - raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'") - import warnings - warnings.warn("'U' mode is deprecated", - DeprecationWarning, 2) - reading = True if text and binary: raise ValueError("can't have text and binary mode at once") if creating + reading + writing + appending > 1: @@ -249,18 +239,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, result = raw try: line_buffering = False - if buffering == 1 or buffering < 0 and raw.isatty(): + if buffering == 1 or buffering < 0 and raw._isatty_open_only(): buffering = -1 line_buffering = True if buffering < 0: - buffering = DEFAULT_BUFFER_SIZE - try: - bs = os.fstat(raw.fileno()).st_blksize - except (OSError, AttributeError): - pass - else: - if bs > 1: - buffering = bs + buffering = max(min(raw._blksize, 8192 * 1024), DEFAULT_BUFFER_SIZE) if buffering < 0: raise ValueError("invalid buffering size") if buffering == 0: @@ -311,22 +294,6 @@ def _open_code_with_warning(path): open_code = _open_code_with_warning -def __getattr__(name): - if name == "OpenWrapper": - # bpo-43680: Until Python 3.9, _pyio.open was not a static method and - # builtins.open was set to OpenWrapper to not become a bound method - # when set to a class variable. _io.open is a built-in function whereas - # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now - # a static method, and builtins.open() is now io.open(). - import warnings - warnings.warn('OpenWrapper is deprecated, use open instead', - DeprecationWarning, stacklevel=2) - global OpenWrapper - OpenWrapper = open - return OpenWrapper - raise AttributeError(name) - - # In normal operation, both `UnsupportedOperation`s should be bound to the # same object. try: @@ -338,8 +305,7 @@ class UnsupportedOperation(OSError, ValueError): class IOBase(metaclass=abc.ABCMeta): - """The abstract base class for all I/O classes, acting on streams of - bytes. There is no public constructor. + """The abstract base class for all I/O classes. This class provides dummy implementations for many methods that derived classes can override selectively; the default implementations @@ -441,18 +407,12 @@ def __del__(self): if closed: return - if _IOBASE_EMITS_UNRAISABLE: - self.close() - else: - # The try/except block is in case this is called at program - # exit time, when it's possible that globals have already been - # deleted, and then the close() call might fail. Since - # there's nothing we can do about such failures and they annoy - # the end users, we suppress the traceback. - try: - self.close() - except: - pass + if dealloc_warn := getattr(self, "_dealloc_warn", None): + dealloc_warn(self) + + # If close() fails, the caller logs the exception with + # sys.unraisablehook. close() must be called at the end at __del__(). + self.close() ### Inquiries ### @@ -657,16 +617,15 @@ def read(self, size=-1): n = self.readinto(b) if n is None: return None + if n < 0 or n > len(b): + raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] return bytes(b) def readall(self): """Read until EOF, using multiple read() call.""" res = bytearray() - while True: - data = self.read(DEFAULT_BUFFER_SIZE) - if not data: - break + while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: return bytes(res) @@ -691,8 +650,6 @@ def write(self, b): self._unsupported("write") io.RawIOBase.register(RawIOBase) -from _io import FileIO -RawIOBase.register(FileIO) class BufferedIOBase(IOBase): @@ -899,6 +856,10 @@ def __repr__(self): else: return "<{}.{} name={!r}>".format(modname, clsname, name) + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.raw, "_dealloc_warn", None): + dealloc_warn(source) + ### Lower-level APIs ### def fileno(self): @@ -974,22 +935,22 @@ def read1(self, size=-1): return self.read(size) def write(self, b): - if self.closed: - raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: + if self.closed: + raise ValueError("write to closed file") + n = view.nbytes # Size of any bytes-like object - if n == 0: - return 0 - pos = self._pos - if pos > len(self._buffer): - # Inserts null bytes between the current end of the file - # and the new write position. - padding = b'\x00' * (pos - len(self._buffer)) - self._buffer += padding - self._buffer[pos:pos + n] = b - self._pos += n + if n == 0: + return 0 + + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n return n def seek(self, pos, whence=0): @@ -1154,6 +1115,7 @@ def peek(self, size=0): do at most one raw read to satisfy it. We never return more than self.buffer_size. """ + self._checkClosed("peek of closed file") with self._read_lock: return self._peek_unlocked(size) @@ -1172,6 +1134,7 @@ def read1(self, size=-1): """Reads up to size bytes, with at most one read() system call.""" # Returns up to size bytes. If at least one byte is buffered, we # only return buffered bytes. Otherwise, we do one raw read. + self._checkClosed("read of closed file") if size < 0: size = self.buffer_size if size == 0: @@ -1189,6 +1152,8 @@ def read1(self, size=-1): def _readinto(self, buf, read1): """Read data into *buf* with at most one system call.""" + self._checkClosed("readinto of closed file") + # Need to create a memoryview object of type 'b', otherwise # we may not be able to assign bytes to it, and slicing it # would create a new object. @@ -1233,11 +1198,13 @@ def _readinto(self, buf, read1): return written def tell(self): - return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos + # GH-95782: Keep return value non-negative + return max(_BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos, 0) def seek(self, pos, whence=0): if whence not in valid_seek_flags: raise ValueError("invalid whence value") + self._checkClosed("seek of closed file") with self._read_lock: if whence == 1: pos -= len(self._read_buf) - self._read_pos @@ -1497,6 +1464,17 @@ def write(self, b): return BufferedWriter.write(self, b) +def _new_buffersize(bytes_read): + # Parallels _io/fileio.c new_buffersize + if bytes_read > 65536: + addend = bytes_read >> 3 + else: + addend = 256 + bytes_read + if addend < DEFAULT_BUFFER_SIZE: + addend = DEFAULT_BUFFER_SIZE + return bytes_read + addend + + class FileIO(RawIOBase): _fd = -1 _created = False @@ -1521,6 +1499,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): """ if self._fd >= 0: # Have to close the existing file first. + self._stat_atopen = None try: if self._closefd: os.close(self._fd) @@ -1530,6 +1509,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if isinstance(file, float): raise TypeError('integer argument expected, got float') if isinstance(file, int): + if isinstance(file, bool): + import warnings + warnings.warn("bool is used as a file descriptor", + RuntimeWarning, stacklevel=2) + file = int(file) fd = file if fd < 0: raise ValueError('negative file descriptor') @@ -1588,24 +1572,22 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if not isinstance(fd, int): raise TypeError('expected integer from opener') if fd < 0: - raise OSError('Negative file descriptor') + # bpo-27066: Raise a ValueError for bad value. + raise ValueError(f'opener returned {fd}') owned_fd = fd if not noinherit_flag: os.set_inheritable(fd, False) self._closefd = closefd - fdfstat = os.fstat(fd) + self._stat_atopen = os.fstat(fd) try: - if stat.S_ISDIR(fdfstat.st_mode): + if stat.S_ISDIR(self._stat_atopen.st_mode): raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), file) except AttributeError: # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR # don't exist. pass - self._blksize = getattr(fdfstat, 'st_blksize', 0) - if self._blksize <= 1: - self._blksize = DEFAULT_BUFFER_SIZE if _setmode: # don't translate newlines (\r\n <=> \n) @@ -1622,17 +1604,17 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if e.errno != errno.ESPIPE: raise except: + self._stat_atopen = None if owned_fd is not None: os.close(owned_fd) raise self._fd = fd - def __del__(self): + def _dealloc_warn(self, source): if self._fd >= 0 and self._closefd and not self.closed: import warnings - warnings.warn('unclosed file %r' % (self,), ResourceWarning, + warnings.warn(f'unclosed file {source!r}', ResourceWarning, stacklevel=2, source=self) - self.close() def __getstate__(self): raise TypeError(f"cannot pickle {self.__class__.__name__!r} object") @@ -1651,6 +1633,17 @@ def __repr__(self): return ('<%s name=%r mode=%r closefd=%r>' % (class_name, name, self.mode, self._closefd)) + @property + def _blksize(self): + if self._stat_atopen is None: + return DEFAULT_BUFFER_SIZE + + blksize = getattr(self._stat_atopen, "st_blksize", 0) + # WASI sets blsize to 0 + if not blksize: + return DEFAULT_BUFFER_SIZE + return blksize + def _checkReadable(self): if not self._readable: raise UnsupportedOperation('File not open for reading') @@ -1662,7 +1655,13 @@ def _checkWritable(self, msg=None): def read(self, size=None): """Read at most size bytes, returned as bytes. - Only makes one system call, so less data may be returned than requested + If size is less than 0, read all bytes in the file making + multiple read calls. See ``FileIO.readall``. + + Attempts to make only one system call, retrying only per + PEP 475 (EINTR). This means less data may be returned than + requested. + In non-blocking mode, returns None if no data is available. Return an empty bytes object at EOF. """ @@ -1678,45 +1677,57 @@ def read(self, size=None): def readall(self): """Read all data from the file, returned as bytes. - In non-blocking mode, returns as much as is immediately available, - or None if no data is available. Return an empty bytes object at EOF. + Reads until either there is an error or read() returns size 0 + (indicates EOF). If the file is already at EOF, returns an + empty bytes object. + + In non-blocking mode, returns as much data as could be read + before EAGAIN. If no data is available (EAGAIN is returned + before bytes are read) returns None. """ self._checkClosed() self._checkReadable() - bufsize = DEFAULT_BUFFER_SIZE - try: - pos = os.lseek(self._fd, 0, SEEK_CUR) - end = os.fstat(self._fd).st_size - if end >= pos: - bufsize = end - pos + 1 - except OSError: - pass + if self._stat_atopen is None or self._stat_atopen.st_size <= 0: + bufsize = DEFAULT_BUFFER_SIZE + else: + # In order to detect end of file, need a read() of at least 1 + # byte which returns size 0. Oversize the buffer by 1 byte so the + # I/O can be completed with two read() calls (one for all data, one + # for EOF) without needing to resize the buffer. + bufsize = self._stat_atopen.st_size + 1 - result = bytearray() - while True: - if len(result) >= bufsize: - bufsize = len(result) - bufsize += max(bufsize, DEFAULT_BUFFER_SIZE) - n = bufsize - len(result) - try: - chunk = os.read(self._fd, n) - except BlockingIOError: - if result: - break + if self._stat_atopen.st_size > 65536: + try: + pos = os.lseek(self._fd, 0, SEEK_CUR) + if self._stat_atopen.st_size >= pos: + bufsize = self._stat_atopen.st_size - pos + 1 + except OSError: + pass + + result = bytearray(bufsize) + bytes_read = 0 + try: + while n := os.readinto(self._fd, memoryview(result)[bytes_read:]): + bytes_read += n + if bytes_read >= len(result): + result.resize(_new_buffersize(bytes_read)) + except BlockingIOError: + if not bytes_read: return None - if not chunk: # reached the end of the file - break - result += chunk + assert len(result) - bytes_read >= 1, \ + "os.readinto buffer size 0 will result in erroneous EOF / returns 0" + result.resize(bytes_read) return bytes(result) - def readinto(self, b): + def readinto(self, buffer): """Same as RawIOBase.readinto().""" - m = memoryview(b).cast('B') - data = self.read(len(m)) - n = len(data) - m[:n] = data - return n + self._checkClosed() + self._checkReadable() + try: + return os.readinto(self._fd, buffer) + except BlockingIOError: + return None def write(self, b): """Write bytes b to file, return number written. @@ -1766,6 +1777,7 @@ def truncate(self, size=None): if size is None: size = self.tell() os.ftruncate(self._fd, size) + self._stat_atopen = None return size def close(self): @@ -1775,8 +1787,9 @@ def close(self): called more than once without error. """ if not self.closed: + self._stat_atopen = None try: - if self._closefd: + if self._closefd and self._fd >= 0: os.close(self._fd) finally: super().close() @@ -1813,6 +1826,21 @@ def isatty(self): self._checkClosed() return os.isatty(self._fd) + def _isatty_open_only(self): + """Checks whether the file is a TTY using an open-only optimization. + + TTYs are always character devices. If the interpreter knows a file is + not a character device when it would call ``isatty``, can skip that + call. Inside ``open()`` there is a fresh stat result that contains that + information. Use the stat result to skip a system call. Outside of that + context TOCTOU issues (the fd could be arbitrarily modified by + surrounding code). + """ + if (self._stat_atopen is not None + and not stat.S_ISCHR(self._stat_atopen.st_mode)): + return False + return os.isatty(self._fd) + @property def closefd(self): """True if the file descriptor will be closed by close().""" @@ -1845,7 +1873,7 @@ class TextIOBase(IOBase): """Base class for text I/O. This class provides a character and line based interface to stream - I/O. There is no public constructor. + I/O. """ def read(self, size=-1): @@ -1997,7 +2025,7 @@ class TextIOWrapper(TextIOBase): r"""Character and line based layer over a BufferedIOBase object, buffer. encoding gives the name of the encoding that the stream will be - decoded or encoded with. It defaults to locale.getpreferredencoding(False). + decoded or encoded with. It defaults to locale.getencoding(). errors determines the strictness of encoding and decoding (see the codecs.register) and defaults to "strict". @@ -2031,26 +2059,13 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None, encoding = text_encoding(encoding) if encoding == "locale": - try: - encoding = os.device_encoding(buffer.fileno()) or "locale" - except (AttributeError, UnsupportedOperation): - pass - - if encoding == "locale": - try: - import locale - except ImportError: - # Importing locale may fail if Python is being built - encoding = "utf-8" - else: - encoding = locale.getpreferredencoding(False) + encoding = self._get_locale_encoding() if not isinstance(encoding, str): raise ValueError("invalid encoding: %r" % encoding) if not codecs.lookup(encoding)._is_text_encoding: - msg = ("%r is not a text encoding; " - "use codecs.open() to handle arbitrary codecs") + msg = "%r is not a text encoding" raise LookupError(msg % encoding) if errors is None: @@ -2176,6 +2191,8 @@ def reconfigure(self, *, else: if not isinstance(encoding, str): raise TypeError("invalid encoding: %r" % encoding) + if encoding == "locale": + encoding = self._get_locale_encoding() if newline is Ellipsis: newline = self._readnl @@ -2243,8 +2260,9 @@ def write(self, s): self.buffer.write(b) if self._line_buffering and (haslf or "\r" in s): self.flush() - self._set_decoded_chars('') - self._snapshot = None + if self._snapshot is not None: + self._set_decoded_chars('') + self._snapshot = None if self._decoder: self._decoder.reset() return length @@ -2280,6 +2298,15 @@ def _get_decoded_chars(self, n=None): self._decoded_chars_used += len(chars) return chars + def _get_locale_encoding(self): + try: + import locale + except ImportError: + # Importing locale may fail if Python is being built + return "utf-8" + else: + return locale.getencoding() + def _rewind_decoded_chars(self, n): """Rewind the _decoded_chars buffer.""" if self._decoded_chars_used < n: @@ -2546,11 +2573,15 @@ def read(self, size=None): size = size_index() decoder = self._decoder or self._get_decoder() if size < 0: + chunk = self.buffer.read() + if chunk is None: + raise BlockingIOError("Read returned None.") # Read everything. result = (self._get_decoded_chars() + - decoder.decode(self.buffer.read(), final=True)) - self._set_decoded_chars('') - self._snapshot = None + decoder.decode(chunk, final=True)) + if self._snapshot is not None: + self._set_decoded_chars('') + self._snapshot = None return result else: # Keep reading chunks until we have size characters to return. @@ -2667,6 +2698,10 @@ def readline(self, size=None): def newlines(self): return self._decoder.newlines if self._decoder else None + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.buffer, "_dealloc_warn", None): + dealloc_warn(source) + class StringIO(TextIOWrapper): """Text I/O implementation using an in-memory buffer. diff --git a/Lib/_pylong.py b/Lib/_pylong.py new file mode 100644 index 00000000000..be1acd17ce3 --- /dev/null +++ b/Lib/_pylong.py @@ -0,0 +1,729 @@ +"""Python implementations of some algorithms for use by longobject.c. +The goal is to provide asymptotically faster algorithms that can be +used for operations on integers with many digits. In those cases, the +performance overhead of the Python implementation is not significant +since the asymptotic behavior is what dominates runtime. Functions +provided by this module should be considered private and not part of any +public API. + +Note: for ease of maintainability, please prefer clear code and avoid +"micro-optimizations". This module will only be imported and used for +integers with a huge number of digits. Saving a few microseconds with +tricky or non-obvious code is not worth it. For people looking for +maximum performance, they should use something like gmpy2.""" + +import re +import decimal +try: + import _decimal +except ImportError: + _decimal = None + +# A number of functions have this form, where `w` is a desired number of +# digits in base `base`: +# +# def inner(...w...): +# if w <= LIMIT: +# return something +# lo = w >> 1 +# hi = w - lo +# something involving base**lo, inner(...lo...), j, and inner(...hi...) +# figure out largest w needed +# result = inner(w) +# +# They all had some on-the-fly scheme to cache `base**lo` results for reuse. +# Power is costly. +# +# This routine aims to compute all amd only the needed powers in advance, as +# efficiently as reasonably possible. This isn't trivial, and all the +# on-the-fly methods did needless work in many cases. The driving code above +# changes to: +# +# figure out largest w needed +# mycache = compute_powers(w, base, LIMIT) +# result = inner(w) +# +# and `mycache[lo]` replaces `base**lo` in the inner function. +# +# If an algorithm wants the powers of ceiling(w/2) instead of the floor, +# pass keyword argument `need_hi=True`. +# +# While this does give minor speedups (a few percent at best), the +# primary intent is to simplify the functions using this, by eliminating +# the need for them to craft their own ad-hoc caching schemes. +# +# See code near end of file for a block of code that can be enabled to +# run millions of tests. +def compute_powers(w, base, more_than, *, need_hi=False, show=False): + seen = set() + need = set() + ws = {w} + while ws: + w = ws.pop() # any element is fine to use next + if w in seen or w <= more_than: + continue + seen.add(w) + lo = w >> 1 + hi = w - lo + # only _need_ one here; the other may, or may not, be needed + which = hi if need_hi else lo + need.add(which) + ws.add(which) + if lo != hi: + ws.add(w - which) + + # `need` is the set of exponents needed. To compute them all + # efficiently, possibly add other exponents to `extra`. The goal is + # to ensure that each exponent can be gotten from a smaller one via + # multiplying by the base, squaring it, or squaring and then + # multiplying by the base. + # + # If need_hi is False, this is already the case (w can always be + # gotten from w >> 1 via one of the squaring strategies). But we do + # the work anyway, just in case ;-) + # + # Note that speed is irrelevant. These loops are working on little + # ints (exponents) and go around O(log w) times. The total cost is + # insignificant compared to just one of the bigint multiplies. + cands = need.copy() + extra = set() + while cands: + w = max(cands) + cands.remove(w) + lo = w >> 1 + if lo > more_than and w-1 not in cands and lo not in cands: + extra.add(lo) + cands.add(lo) + assert need_hi or not extra + + d = {} + for n in sorted(need | extra): + lo = n >> 1 + hi = n - lo + if n-1 in d: + if show: + print("* base", end="") + result = d[n-1] * base # cheap! + elif lo in d: + # Multiplying a bigint by itself is about twice as fast + # in CPython provided it's the same object. + if show: + print("square", end="") + result = d[lo] * d[lo] # same object + if hi != lo: + if show: + print(" * base", end="") + assert 2 * lo + 1 == n + result *= base + else: # rare + if show: + print("pow", end='') + result = base ** n + if show: + print(" at", n, "needed" if n in need else "extra") + d[n] = result + + assert need <= d.keys() + if excess := d.keys() - need: + assert need_hi + for n in excess: + del d[n] + return d + +_unbounded_dec_context = decimal.getcontext().copy() +_unbounded_dec_context.prec = decimal.MAX_PREC +_unbounded_dec_context.Emax = decimal.MAX_EMAX +_unbounded_dec_context.Emin = decimal.MIN_EMIN +_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check + +def int_to_decimal(n): + """Asymptotically fast conversion of an 'int' to Decimal.""" + + # Function due to Tim Peters. See GH issue #90716 for details. + # https://github.com/python/cpython/issues/90716 + # + # The implementation in longobject.c of base conversion algorithms + # between power-of-2 and non-power-of-2 bases are quadratic time. + # This function implements a divide-and-conquer algorithm that is + # faster for large numbers. Builds an equal decimal.Decimal in a + # "clever" recursive way. If we want a string representation, we + # apply str to _that_. + + from decimal import Decimal as D + BITLIM = 200 + + # Don't bother caching the "lo" mask in this; the time to compute it is + # tiny compared to the multiply. + def inner(n, w): + if w <= BITLIM: + return D(n) + w2 = w >> 1 + hi = n >> w2 + lo = n & ((1 << w2) - 1) + return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2] + + with decimal.localcontext(_unbounded_dec_context): + nbits = n.bit_length() + w2pow = compute_powers(nbits, D(2), BITLIM) + if n < 0: + negate = True + n = -n + else: + negate = False + result = inner(n, nbits) + if negate: + result = -result + return result + +def int_to_decimal_string(n): + """Asymptotically fast conversion of an 'int' to a decimal string.""" + w = n.bit_length() + if w > 450_000 and _decimal is not None: + # It is only usable with the C decimal implementation. + # _pydecimal.py calls str() on very large integers, which in its + # turn calls int_to_decimal_string(), causing very deep recursion. + return str(int_to_decimal(n)) + + # Fallback algorithm for the case when the C decimal module isn't + # available. This algorithm is asymptotically worse than the algorithm + # using the decimal module, but better than the quadratic time + # implementation in longobject.c. + + DIGLIM = 1000 + def inner(n, w): + if w <= DIGLIM: + return str(n) + w2 = w >> 1 + hi, lo = divmod(n, pow10[w2]) + return inner(hi, w - w2) + inner(lo, w2).zfill(w2) + + # The estimation of the number of decimal digits. + # There is no harm in small error. If we guess too large, there may + # be leading 0's that need to be stripped. If we guess too small, we + # may need to call str() recursively for the remaining highest digits, + # which can still potentially be a large integer. This is manifested + # only if the number has way more than 10**15 digits, that exceeds + # the 52-bit physical address limit in both Intel64 and AMD64. + w = int(w * 0.3010299956639812 + 1) # log10(2) + pow10 = compute_powers(w, 5, DIGLIM) + for k, v in pow10.items(): + pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k + if n < 0: + n = -n + sign = '-' + else: + sign = '' + s = inner(n, w) + if s[0] == '0' and n: + # If our guess of w is too large, there may be leading 0's that + # need to be stripped. + s = s.lstrip('0') + return sign + s + +def _str_to_int_inner(s): + """Asymptotically fast conversion of a 'str' to an 'int'.""" + + # Function due to Bjorn Martinsson. See GH issue #90716 for details. + # https://github.com/python/cpython/issues/90716 + # + # The implementation in longobject.c of base conversion algorithms + # between power-of-2 and non-power-of-2 bases are quadratic time. + # This function implements a divide-and-conquer algorithm making use + # of Python's built in big int multiplication. Since Python uses the + # Karatsuba algorithm for multiplication, the time complexity + # of this function is O(len(s)**1.58). + + DIGLIM = 2048 + + def inner(a, b): + if b - a <= DIGLIM: + return int(s[a:b]) + mid = (a + b + 1) >> 1 + return (inner(mid, b) + + ((inner(a, mid) * w5pow[b - mid]) + << (b - mid))) + + w5pow = compute_powers(len(s), 5, DIGLIM) + return inner(0, len(s)) + + +# Asymptotically faster version, using the C decimal module. See +# comments at the end of the file. This uses decimal arithmetic to +# convert from base 10 to base 256. The latter is just a string of +# bytes, which CPython can convert very efficiently to a Python int. + +# log of 10 to base 256 with best-possible 53-bit precision. Obtained +# via: +# from mpmath import mp +# mp.prec = 1000 +# print(float(mp.log(10, 256)).hex()) +_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415 + +# _spread is for internal testing. It maps a key to the number of times +# that condition obtained in _dec_str_to_int_inner: +# key 0 - quotient guess was right +# key 1 - quotient had to be boosted by 1, one time +# key 999 - one adjustment wasn't enough, so fell back to divmod +from collections import defaultdict +_spread = defaultdict(int) +del defaultdict + +def _dec_str_to_int_inner(s, *, GUARD=8): + # Yes, BYTELIM is "large". Large enough that CPython will usually + # use the Karatsuba _str_to_int_inner to convert the string. This + # allowed reducing the cutoff for calling _this_ function from 3.5M + # to 2M digits. We could almost certainly do even better by + # fine-tuning this and/or using a larger output base than 256. + BYTELIM = 100_000 + D = decimal.Decimal + result = bytearray() + # See notes at end of file for discussion of GUARD. + assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed + + def inner(n, w): + #assert n < D256 ** w # required, but too expensive to check + if w <= BYTELIM: + # XXX Stefan Pochmann discovered that, for 1024-bit ints, + # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`. + # Worse, `int(Decimal) is still quadratic-time for much + # larger ints. So unless/until all that is repaired, the + # seemingly redundant `str(Decimal)` is crucial to speed. + result.extend(int(str(n)).to_bytes(w)) # big-endian default + return + w1 = w >> 1 + w2 = w - w1 + if 0: + # This is maximally clear, but "too slow". `decimal` + # division is asymptotically fast, but we have no way to + # tell it to reuse the high-precision reciprocal it computes + # for pow256[w2], so it has to recompute it over & over & + # over again :-( + hi, lo = divmod(n, pow256[w2][0]) + else: + p256, recip = pow256[w2] + # The integer part will have a number of digits about equal + # to the difference between the log10s of `n` and `pow256` + # (which, since these are integers, is roughly approximated + # by `.adjusted()`). That's the working precision we need, + ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD + hi = +n * +recip # unary `+` chops back to ctx.prec digits + ctx.prec = decimal.MAX_PREC + hi = hi.to_integral_value() # lose the fractional digits + lo = n - hi * p256 + # Because we've been uniformly rounding down, `hi` is a + # lower bound on the correct quotient. + assert lo >= 0 + # Adjust quotient up if needed. It usually isn't. In random + # testing on inputs through 5 billion digit strings, the + # test triggered once in about 200 thousand tries. + count = 0 + if lo >= p256: + count = 1 + lo -= p256 + hi += 1 + if lo >= p256: + # Complete correction via an exact computation. I + # believe it's not possible to get here provided + # GUARD >= 3. It's tested by reducing GUARD below + # that. + count = 999 + hi2, lo = divmod(lo, p256) + hi += hi2 + _spread[count] += 1 + # The assert should always succeed, but way too slow to keep + # enabled. + #assert hi, lo == divmod(n, pow256[w2][0]) + inner(hi, w1) + del hi # at top levels, can free a lot of RAM "early" + inner(lo, w2) + + # How many base 256 digits are needed?. Mathematically, exactly + # floor(log256(int(s))) + 1. There is no cheap way to compute this. + # But we can get an upper bound, and that's necessary for our error + # analysis to make sense. int(s) < 10**len(s), so the log needed is + # < log256(10**len(s)) = len(s) * log256(10). However, using + # finite-precision floating point for this, it's possible that the + # computed value is a little less than the true value. If the true + # value is at - or a little higher than - an integer, we can get an + # off-by-1 error too low. So we add 2 instead of 1 if chopping lost + # a fraction > 0.9. + + # The "WASI" test platform can complain about `len(s)` if it's too + # large to fit in its idea of "an index-sized integer". + lenS = s.__len__() + log_ub = lenS * _LOG_10_BASE_256 + log_ub_as_int = int(log_ub) + w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9) + # And what if we've plain exhausted the limits of HW floats? We + # could compute the log to any desired precision using `decimal`, + # but it's not plausible that anyone will pass a string requiring + # trillions of bytes (unless they're just trying to "break things"). + if w.bit_length() >= 46: + # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double. + raise ValueError(f"cannot convert string of len {lenS} to int") + with decimal.localcontext(_unbounded_dec_context) as ctx: + D256 = D(256) + pow256 = compute_powers(w, D256, BYTELIM, need_hi=True) + rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True) + # We're going to do inexact, chopped arithmetic, multiplying by + # an approximation to the reciprocal of 256**i. We chop to get a + # lower bound on the true integer quotient. Our approximation is + # a lower bound, the multiplication is chopped too, and + # to_integral_value() is also chopped. + ctx.traps[decimal.Inexact] = 0 + ctx.rounding = decimal.ROUND_DOWN + for k, v in pow256.items(): + # No need to save much more precision in the reciprocal than + # the power of 256 has, plus some guard digits to absorb + # most relevant rounding errors. This is highly significant: + # 1/2**i has the same number of significant decimal digits + # as 5**i, generally over twice the number in 2**i, + ctx.prec = v.adjusted() + GUARD + 1 + # The unary "+" chops the reciprocal back to that precision. + pow256[k] = v, +rpow256[k] + del rpow256 # exact reciprocals no longer needed + ctx.prec = decimal.MAX_PREC + inner(D(s), w) + return int.from_bytes(result) + +def int_from_string(s): + """Asymptotically fast version of PyLong_FromString(), conversion + of a string of decimal digits into an 'int'.""" + # PyLong_FromString() has already removed leading +/-, checked for invalid + # use of underscore characters, checked that string consists of only digits + # and underscores, and stripped leading whitespace. The input can still + # contain underscores and have trailing whitespace. + s = s.rstrip().replace('_', '') + func = _str_to_int_inner + if len(s) >= 2_000_000 and _decimal is not None: + func = _dec_str_to_int_inner + return func(s) + +def str_to_int(s): + """Asymptotically fast version of decimal string to 'int' conversion.""" + # FIXME: this doesn't support the full syntax that int() supports. + m = re.match(r'\s*([+-]?)([0-9_]+)\s*', s) + if not m: + raise ValueError('invalid literal for int() with base 10') + v = int_from_string(m.group(2)) + if m.group(1) == '-': + v = -v + return v + + +# Fast integer division, based on code from Mark Dickinson, fast_div.py +# GH-47701. Additional refinements and optimizations by Bjorn Martinsson. The +# algorithm is due to Burnikel and Ziegler, in their paper "Fast Recursive +# Division". + +_DIV_LIMIT = 4000 + + +def _div2n1n(a, b, n): + """Divide a 2n-bit nonnegative integer a by an n-bit positive integer + b, using a recursive divide-and-conquer algorithm. + + Inputs: + n is a positive integer + b is a positive integer with exactly n bits + a is a nonnegative integer such that a < 2**n * b + + Output: + (q, r) such that a = b*q+r and 0 <= r < b. + + """ + if a.bit_length() - n <= _DIV_LIMIT: + return divmod(a, b) + pad = n & 1 + if pad: + a <<= 1 + b <<= 1 + n += 1 + half_n = n >> 1 + mask = (1 << half_n) - 1 + b1, b2 = b >> half_n, b & mask + q1, r = _div3n2n(a >> n, (a >> half_n) & mask, b, b1, b2, half_n) + q2, r = _div3n2n(r, a & mask, b, b1, b2, half_n) + if pad: + r >>= 1 + return q1 << half_n | q2, r + + +def _div3n2n(a12, a3, b, b1, b2, n): + """Helper function for _div2n1n; not intended to be called directly.""" + if a12 >> n == b1: + q, r = (1 << n) - 1, a12 - (b1 << n) + b1 + else: + q, r = _div2n1n(a12, b1, n) + r = (r << n | a3) - q * b2 + while r < 0: + q -= 1 + r += b + return q, r + + +def _int2digits(a, n): + """Decompose non-negative int a into base 2**n + + Input: + a is a non-negative integer + + Output: + List of the digits of a in base 2**n in little-endian order, + meaning the most significant digit is last. The most + significant digit is guaranteed to be non-zero. + If a is 0 then the output is an empty list. + + """ + a_digits = [0] * ((a.bit_length() + n - 1) // n) + + def inner(x, L, R): + if L + 1 == R: + a_digits[L] = x + return + mid = (L + R) >> 1 + shift = (mid - L) * n + upper = x >> shift + lower = x ^ (upper << shift) + inner(lower, L, mid) + inner(upper, mid, R) + + if a: + inner(a, 0, len(a_digits)) + return a_digits + + +def _digits2int(digits, n): + """Combine base-2**n digits into an int. This function is the + inverse of `_int2digits`. For more details, see _int2digits. + """ + + def inner(L, R): + if L + 1 == R: + return digits[L] + mid = (L + R) >> 1 + shift = (mid - L) * n + return (inner(mid, R) << shift) + inner(L, mid) + + return inner(0, len(digits)) if digits else 0 + + +def _divmod_pos(a, b): + """Divide a non-negative integer a by a positive integer b, giving + quotient and remainder.""" + # Use grade-school algorithm in base 2**n, n = nbits(b) + n = b.bit_length() + a_digits = _int2digits(a, n) + + r = 0 + q_digits = [] + for a_digit in reversed(a_digits): + q_digit, r = _div2n1n((r << n) + a_digit, b, n) + q_digits.append(q_digit) + q_digits.reverse() + q = _digits2int(q_digits, n) + return q, r + + +def int_divmod(a, b): + """Asymptotically fast replacement for divmod, for 'int'. + Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b). + """ + if b == 0: + raise ZeroDivisionError('division by zero') + elif b < 0: + q, r = int_divmod(-a, -b) + return q, -r + elif a < 0: + q, r = int_divmod(~a, b) + return ~q, b + ~r + else: + return _divmod_pos(a, b) + + +# Notes on _dec_str_to_int_inner: +# +# Stefan Pochmann worked up a str->int function that used the decimal +# module to, in effect, convert from base 10 to base 256. This is +# "unnatural", in that it requires multiplying and dividing by large +# powers of 2, which `decimal` isn't naturally suited to. But +# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so +# at _some_ point it could be expected to win. +# +# Alas, the crossover point was too high to be of much real interest. I +# (Tim) then worked on ways to replace its division with multiplication +# by a cached reciprocal approximation instead, fixing up errors +# afterwards. This reduced the crossover point significantly, +# +# I revisited the code, and found ways to improve and simplify it. The +# crossover point is at about 3.4 million digits now. +# +# About .adjusted() +# ----------------- +# Restrict to Decimal values x > 0. We don't use negative numbers in the +# code, and I don't want to have to keep typing, e.g., "absolute value". +# +# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't +# look at the digits of x, but instead returns an integer giving x's +# order of magnitude. These are equivalent: +# +# - x.a is the power-of-10 exponent of x's most significant digit. +# - x.a = the infinitely precise floor(log10(x)) +# - x can be written in this form, where f is a real with 1 <= f < 10: +# x = f * 10**x.a +# +# Observation; if x is an integer, len(str(x)) = x.a + 1. +# +# Lemma 1: (x * y).a = x.a + y.a, or one larger +# +# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in +# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If +# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back +# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is +# x.a+y.a+1. +# +# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1 +# +# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a - +# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else +# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from +# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest +# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the +# ceiling is at most 1 larger (with equality iff f/g = 1 exactly). +# +# GUARD digits +# ------------ +# We only want the integer part of divisions, so don't need to build +# the full multiplication tree. But using _just_ the number of +# digits expected in the integer part ignores too much. What's left +# out can have a very significant effect on the quotient. So we use +# GUARD additional digits. +# +# The default 8 is more than enough so no more than 1 correction step +# was ever needed for all inputs tried through 2.5 billion digits. In +# fact, I believe 3 guard digits are always enough - but the proof is +# very involved, so better safe than sorry. +# +# Short course: +# +# If prec is the decimal precision in effect, and we're rounding down, +# the result of an operation is exactly equal to the infinitely precise +# result times 1-e for some real e with 0 <= e < 10**(1-prec). In +# +# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD +# hi = +n * +recip # unary `+` chops to ctx.prec digits +# +# we have 3 visible chopped operations, but there's also a 4th: +# precomputing a truncated `recip` as part of setup. +# +# So the computed product is exactly equal to the true product times +# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an +# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the +# 2nd and higher order terms in the expanded product are too tiny to +# matter). If they're all as large as possible, that's +# +# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis. +# +# That implies the computed product is within 1 of the true product +# provided prec >= log10(true_product) + 1.602. +# +# Here are telegraphic details, rephrasing the initial condition in +# equivalent ways, step by step: +# +# prod - prod * (1 - 4*10**(1-prec)) <= 1 +# prod - prod + prod * 4*10**(1-prec)) <= 1 +# prod * 4*10**(1-prec)) <= 1 +# 10**(log10(prod)) * 4*10**(1-prec)) <= 1 +# 4*10**(1-prec+log10(prod))) <= 1 +# 10**(1-prec+log10(prod))) <= 1/4 +# 1-prec+log10(prod) <= log10(1/4) = -0.602 +# -prec <= -1.602 - log10(prod) +# prec >= log10(prod) + 1.602 +# +# The true product is the same as the true ratio n/p256. By Lemma 2 +# above, n.a - p256.a + 1 is an upper bound on the ceiling of +# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an +# upper bound on the right hand side of the inequality. Any prec >= that +# will work. +# +# But since this is just a sketch of a proof ;-), the code uses the +# empirically tested 8 instead of 3. 5 digits more or less makes no +# practical difference to speed - these ints are huge. And while +# increasing GUARD above 3 may not be necessary, every increase cuts the +# percentage of cases that need a correction at all. +# +# On Computing Reciprocals +# ------------------------ +# In general, the exact reciprocals we compute have over twice as many +# significant digits as needed. 1/256**i has the same number of +# significant decimal digits as 5**i. It's a significant waste of RAM +# to store all those unneeded digits. +# +# So we cut exact reciprocals back to the least precision that can +# be needed so that the error analysis above is valid, +# +# [Note: turns out it's very significantly faster to do it this way than +# to compute 1 / 256**i directly to the desired precision, because the +# power method doesn't require division. It's also faster than computing +# (1/256)**i directly to the desired precision - no material division +# there, but `compute_powers()` is much smarter about _how_ to compute +# all the powers needed than repeated applications of `**` - that +# function invokes `**` for at most the few smallest powers needed.] +# +# The hard part is that chopping back to a shorter width occurs +# _outside_ of `inner`. We can't know then what `prec` `inner()` will +# need. We have to pick, for each value of `w2`, the largest possible +# value `prec` can become when `inner()` is working on `w2`. +# +# This is the `prec` inner() uses: +# max(n.a - p256.a, 0) + GUARD +# and what setup uses (renaming its `v` to `p256` - same thing): +# p256.a + GUARD + 1 +# +# We need that the second is always at least as large as the first, +# which is the same as requiring +# +# n.a - 2 * p256.a <= 1 +# +# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The +# worst case in this context is when w ix even. and then w = 2*w2, so +# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a +# is at most p256.a + p256.a + 1. +# +# So the most n.a - 2 * p256.a can be is +# p256.a + p256.a + 1 - 2 * p256.a = 1. QED +# +# Note: an earlier version of the code split on floor(e/2) instead of on +# the ceiling. The worst case then is odd `w`, and a more involved proof +# was needed to show that adding 4 (instead of 1) may be necessary. +# Basically because, in that case, n may be up to 256 times larger than +# p256**2. Curiously enough, by splitting on the ceiling instead, +# nothing in any proof here actually depends on the output base (256). + +# Enable for brute-force testing of compute_powers(). This takes about a +# minute, because it tries millions of cases. +if 0: + def consumer(w, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit: + return + if w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + exp = compute_powers(w, 1, limit, need_hi=need_hi) + assert exp.keys() == need + + from itertools import chain + for need_hi in (False, True): + for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000): + for w in chain(range(1, 100_000), + (10**i for i in range(5, 30))): + consumer(w, limit, need_hi) diff --git a/Lib/_pyrepl/__init__.py b/Lib/_pyrepl/__init__.py new file mode 100644 index 00000000000..1693cbd0b98 --- /dev/null +++ b/Lib/_pyrepl/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py new file mode 100644 index 00000000000..3fa992eee8e --- /dev/null +++ b/Lib/_pyrepl/__main__.py @@ -0,0 +1,6 @@ +# Important: don't add things to this module, as they will end up in the REPL's +# default globals. Use _pyrepl.main instead. + +if __name__ == "__main__": + from .main import interactive_console as __pyrepl_interactive_console + __pyrepl_interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py new file mode 100644 index 00000000000..d884f880f50 --- /dev/null +++ b/Lib/_pyrepl/_minimal_curses.py @@ -0,0 +1,68 @@ +"""Minimal '_curses' module, the low-level interface for curses module +which is not meant to be used directly. + +Based on ctypes. It's too incomplete to be really called '_curses', so +to use it, you have to import it and stick it in sys.modules['_curses'] +manually. + +Note that there is also a built-in module _minimal_curses which will +hide this one if compiled in. +""" + +import ctypes +import ctypes.util + + +class error(Exception): + pass + + +def _find_clib() -> str: + trylibs = ["ncursesw", "ncurses", "curses"] + + for lib in trylibs: + path = ctypes.util.find_library(lib) + if path: + return path + raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") + + +_clibpath = _find_clib() +clib = ctypes.cdll.LoadLibrary(_clibpath) + +clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] +clib.setupterm.restype = ctypes.c_int + +clib.tigetstr.argtypes = [ctypes.c_char_p] +clib.tigetstr.restype = ctypes.c_ssize_t + +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] +clib.tparm.restype = ctypes.c_char_p + +OK = 0 +ERR = -1 + +# ____________________________________________________________ + + +def setupterm(termstr, fd): + err = ctypes.c_int(0) + result = clib.setupterm(termstr, fd, ctypes.byref(err)) + if result == ERR: + raise error("setupterm() failed (err=%d)" % err.value) + + +def tigetstr(cap): + if not isinstance(cap, bytes): + cap = cap.encode("ascii") + result = clib.tigetstr(cap) + if result == ERR: + return None + return ctypes.cast(result, ctypes.c_char_p).value + + +def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) + if result is None: + raise error("tparm() returned NULL") + return result diff --git a/Lib/_pyrepl/_threading_handler.py b/Lib/_pyrepl/_threading_handler.py new file mode 100644 index 00000000000..82f5e8650a2 --- /dev/null +++ b/Lib/_pyrepl/_threading_handler.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +import traceback + + +TYPE_CHECKING = False +if TYPE_CHECKING: + from threading import Thread + from types import TracebackType + from typing import Protocol + + class ExceptHookArgs(Protocol): + @property + def exc_type(self) -> type[BaseException]: ... + @property + def exc_value(self) -> BaseException | None: ... + @property + def exc_traceback(self) -> TracebackType | None: ... + @property + def thread(self) -> Thread | None: ... + + class ShowExceptions(Protocol): + def __call__(self) -> int: ... + def add(self, s: str) -> None: ... + + from .reader import Reader + + +def install_threading_hook(reader: Reader) -> None: + import threading + + @dataclass + class ExceptHookHandler: + lock: threading.Lock = field(default_factory=threading.Lock) + messages: list[str] = field(default_factory=list) + + def show(self) -> int: + count = 0 + with self.lock: + if not self.messages: + return 0 + reader.restore() + for tb in self.messages: + count += 1 + if tb: + print(tb) + self.messages.clear() + reader.scheduled_commands.append("ctrl-c") + reader.prepare() + return count + + def add(self, s: str) -> None: + with self.lock: + self.messages.append(s) + + def exception(self, args: ExceptHookArgs) -> None: + lines = traceback.format_exception( + args.exc_type, + args.exc_value, + args.exc_traceback, + colorize=reader.can_colorize, + ) # type: ignore[call-overload] + pre = f"\nException in {args.thread.name}:\n" if args.thread else "\n" + tb = pre + "".join(lines) + self.add(tb) + + def __call__(self) -> int: + return self.show() + + + handler = ExceptHookHandler() + reader.threading_hook = handler + threading.excepthook = handler.exception diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py new file mode 100644 index 00000000000..503ca1da329 --- /dev/null +++ b/Lib/_pyrepl/commands.py @@ -0,0 +1,489 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations +import os + +# Categories of actions: +# killing +# yanking +# motion +# editing +# history +# finishing +# [completion] + + +# types +if False: + from .historical_reader import HistoricalReader + + +class Command: + finish: bool = False + kills_digit_arg: bool = True + + def __init__( + self, reader: HistoricalReader, event_name: str, event: list[str] + ) -> None: + # Reader should really be "any reader" but there's too much usage of + # HistoricalReader methods and fields in the code below for us to + # refactor at the moment. + + self.reader = reader + self.event = event + self.event_name = event_name + + def do(self) -> None: + pass + + +class KillCommand(Command): + def kill_range(self, start: int, end: int) -> None: + if start == end: + return + r = self.reader + b = r.buffer + text = b[start:end] + del b[start:end] + if is_kill(r.last_command): + if start < r.pos: + r.kill_ring[-1] = text + r.kill_ring[-1] + else: + r.kill_ring[-1] = r.kill_ring[-1] + text + else: + r.kill_ring.append(text) + r.pos = start + r.dirty = True + + +class YankCommand(Command): + pass + + +class MotionCommand(Command): + pass + + +class EditCommand(Command): + pass + + +class FinishCommand(Command): + finish = True + pass + + +def is_kill(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, KillCommand) + + +def is_yank(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, YankCommand) + + +# etc + + +class digit_arg(Command): + kills_digit_arg = False + + def do(self) -> None: + r = self.reader + c = self.event[-1] + if c == "-": + if r.arg is not None: + r.arg = -r.arg + else: + r.arg = -1 + else: + d = int(c) + if r.arg is None: + r.arg = d + else: + if r.arg < 0: + r.arg = 10 * r.arg - d + else: + r.arg = 10 * r.arg + d + r.dirty = True + + +class clear_screen(Command): + def do(self) -> None: + r = self.reader + r.console.clear() + r.dirty = True + + +class refresh(Command): + def do(self) -> None: + self.reader.dirty = True + + +class repaint(Command): + def do(self) -> None: + self.reader.dirty = True + self.reader.console.repaint() + + +class kill_line(KillCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + eol = r.eol() + for c in b[r.pos : eol]: + if not c.isspace(): + self.kill_range(r.pos, eol) + return + else: + self.kill_range(r.pos, eol + 1) + + +class unix_line_discard(KillCommand): + def do(self) -> None: + r = self.reader + self.kill_range(r.bol(), r.pos) + + +class unix_word_rubout(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.pos, r.eow()) + + +class backward_kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class yank(YankCommand): + def do(self) -> None: + r = self.reader + if not r.kill_ring: + r.error("nothing to yank") + return + r.insert(r.kill_ring[-1]) + + +class yank_pop(YankCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if not r.kill_ring: + r.error("nothing to yank") + return + if not is_yank(r.last_command): + r.error("previous command was not a yank") + return + repl = len(r.kill_ring[-1]) + r.kill_ring.insert(0, r.kill_ring.pop()) + t = r.kill_ring[-1] + b[r.pos - repl : r.pos] = t + r.pos = r.pos - repl + len(t) + r.dirty = True + + +class interrupt(FinishCommand): + def do(self) -> None: + import signal + + self.reader.console.finish() + self.reader.finish() + os.kill(os.getpid(), signal.SIGINT) + + +class ctrl_c(Command): + def do(self) -> None: + self.reader.console.finish() + self.reader.finish() + raise KeyboardInterrupt + + +class suspend(Command): + def do(self) -> None: + import signal + + r = self.reader + p = r.pos + r.console.finish() + os.kill(os.getpid(), signal.SIGSTOP) + ## this should probably be done + ## in a handler for SIGCONT? + r.console.prepare() + r.pos = p + # r.posxy = 0, 0 # XXX this is invalid + r.dirty = True + r.console.screen = [] + + +class up(MotionCommand): + def do(self) -> None: + r = self.reader + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y - 1 + + if r.bol() == 0: + if r.historyi > 0: + r.select_item(r.historyi - 1) + return + r.pos = 0 + r.error("start of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class down(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y + 1 + + if r.eol() == len(b): + if r.historyi < len(r.history): + r.select_item(r.historyi + 1) + r.pos = r.eol(0) + return + r.pos = len(b) + r.error("end of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class left(MotionCommand): + def do(self) -> None: + r = self.reader + for _ in range(r.get_arg()): + p = r.pos - 1 + if p >= 0: + r.pos = p + else: + self.reader.error("start of buffer") + + +class right(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for _ in range(r.get_arg()): + p = r.pos + 1 + if p <= len(b): + r.pos = p + else: + self.reader.error("end of buffer") + + +class beginning_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.bol() + + +class end_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.eol() + + +class home(MotionCommand): + def do(self) -> None: + self.reader.pos = 0 + + +class end(MotionCommand): + def do(self) -> None: + self.reader.pos = len(self.reader.buffer) + + +class forward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.eow() + + +class backward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.bow() + + +class self_insert(EditCommand): + def do(self) -> None: + r = self.reader + text = self.event * r.get_arg() + r.insert(text) + + +class insert_nl(EditCommand): + def do(self) -> None: + r = self.reader + r.insert("\n" * r.get_arg()) + + +class transpose_characters(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + s = r.pos - 1 + if s < 0: + r.error("cannot transpose at start of buffer") + else: + if s == len(b): + s -= 1 + t = min(s + r.get_arg(), len(b) - 1) + c = b[s] + del b[s] + b.insert(t, c) + r.pos = t + r.dirty = True + + +class backspace(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + if r.pos > 0: + r.pos -= 1 + del b[r.pos] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +class delete(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + and self.event[-1] == "\004" + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): + if r.pos != len(b): + del b[r.pos] + r.dirty = True + else: + self.reader.error("end of buffer") + + +class accept(FinishCommand): + def do(self) -> None: + pass + + +class help(Command): + def do(self) -> None: + import _sitebuiltins + + with self.reader.suspend(): + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] + + +class invalid_key(Command): + def do(self) -> None: + pending = self.reader.console.getpending() + s = "".join(self.event) + pending.data + self.reader.error("`%r' not bound" % s) + + +class invalid_command(Command): + def do(self) -> None: + s = self.event_name + self.reader.error("command `%s' not known" % s) + + +class show_history(Command): + def do(self) -> None: + from .pager import get_pager + from site import gethistoryfile # type: ignore[attr-defined] + + history = os.linesep.join(self.reader.history[:]) + self.reader.console.restore() + pager = get_pager() + pager(history, gethistoryfile()) + self.reader.console.prepare() + + # We need to copy over the state so that it's consistent between + # console and reader, and console does not overwrite/append stuff + self.reader.console.screen = self.reader.screen.copy() + self.reader.console.posxy = self.reader.cxy + + +class paste_mode(Command): + + def do(self) -> None: + self.reader.paste_mode = not self.reader.paste_mode + self.reader.dirty = True + + +class enable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = True + self.reader.in_bracketed_paste = True + +class disable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = False + self.reader.in_bracketed_paste = False + self.reader.dirty = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py new file mode 100644 index 00000000000..9a005281dab --- /dev/null +++ b/Lib/_pyrepl/completing_reader.py @@ -0,0 +1,295 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from dataclasses import dataclass, field + +import re +from . import commands, console, reader +from .reader import Reader + + +# types +Command = commands.Command +if False: + from .types import KeySpec, CommandName + + +def prefix(wordlist: list[str], j: int = 0) -> str: + d = {} + i = j + try: + while 1: + for word in wordlist: + d[word[i]] = 1 + if len(d) > 1: + return wordlist[0][j:i] + i += 1 + d = {} + except IndexError: + return wordlist[0][j:i] + return "" + + +STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + +def stripcolor(s: str) -> str: + return STRIPCOLOR_REGEX.sub('', s) + + +def real_len(s: str) -> int: + return len(stripcolor(s)) + + +def left_align(s: str, maxlen: int) -> str: + stripped = stripcolor(s) + if len(stripped) > maxlen: + # too bad, we remove the color + return stripped[:maxlen] + padding = maxlen - len(stripped) + return s + ' '*padding + + +def build_menu( + cons: console.Console, + wordlist: list[str], + start: int, + use_brackets: bool, + sort_in_column: bool, +) -> tuple[list[str], int]: + if use_brackets: + item = "[ %s ]" + padding = 4 + else: + item = "%s " + padding = 2 + maxlen = min(max(map(real_len, wordlist)), cons.width - padding) + cols = int(cons.width / (maxlen + padding)) + rows = int((len(wordlist) - 1)/cols + 1) + + if sort_in_column: + # sort_in_column=False (default) sort_in_column=True + # A B C A D G + # D E F B E + # G C F + # + # "fill" the table with empty words, so we always have the same amout + # of rows for each column + missing = cols*rows - len(wordlist) + wordlist = wordlist + ['']*missing + indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))] + wordlist = [wordlist[i] for i in indexes] + menu = [] + i = start + for r in range(rows): + row = [] + for col in range(cols): + row.append(item % left_align(wordlist[i], maxlen)) + i += 1 + if i >= len(wordlist): + break + menu.append(''.join(row)) + if i >= len(wordlist): + i = 0 + break + if r + 5 > cons.height: + menu.append(" %d more... " % (len(wordlist) - i)) + break + return menu, i + +# this gets somewhat user interface-y, and as a result the logic gets +# very convoluted. +# +# To summarise the summary of the summary:- people are a problem. +# -- The Hitch-Hikers Guide to the Galaxy, Episode 12 + +#### Desired behaviour of the completions commands. +# the considerations are: +# (1) how many completions are possible +# (2) whether the last command was a completion +# (3) if we can assume that the completer is going to return the same set of +# completions: this is controlled by the ``assume_immutable_completions`` +# variable on the reader, which is True by default to match the historical +# behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match +# more closely readline's semantics (this is needed e.g. by +# fancycompleter) +# +# if there's no possible completion, beep at the user and point this out. +# this is easy. +# +# if there's only one possible completion, stick it in. if the last thing +# user did was a completion, point out that he isn't getting anywhere, but +# only if the ``assume_immutable_completions`` is True. +# +# now it gets complicated. +# +# for the first press of a completion key: +# if there's a common prefix, stick it in. + +# irrespective of whether anything got stuck in, if the word is now +# complete, show the "complete but not unique" message + +# if there's no common prefix and if the word is not now complete, +# beep. + +# common prefix -> yes no +# word complete \/ +# yes "cbnu" "cbnu" +# no - beep + +# for the second bang on the completion key +# there will necessarily be no common prefix +# show a menu of the choices. + +# for subsequent bangs, rotate the menu around (if there are sufficient +# choices). + + +class complete(commands.Command): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + last_is_completer = r.last_command_is(self.__class__) + immutable_completions = r.assume_immutable_completions + completions_unchangable = last_is_completer and immutable_completions + stem = r.get_stem() + if not completions_unchangable: + r.cmpltn_menu_choices = r.get_completions(stem) + + completions = r.cmpltn_menu_choices + if not completions: + r.error("no matches") + elif len(completions) == 1: + if completions_unchangable and len(completions[0]) == len(stem): + r.msg = "[ sole completion ]" + r.dirty = True + r.insert(completions[0][len(stem):]) + else: + p = prefix(completions, len(stem)) + if p: + r.insert(p) + if last_is_completer: + r.cmpltn_menu_visible = True + r.cmpltn_message_visible = False + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, r.cmpltn_menu_end, + r.use_brackets, r.sort_in_column) + r.dirty = True + elif not r.cmpltn_menu_visible: + r.cmpltn_message_visible = True + if stem + p in completions: + r.msg = "[ complete but not unique ]" + r.dirty = True + else: + r.msg = "[ not unique ]" + r.dirty = True + + +class self_insert(commands.self_insert): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + + commands.self_insert.do(self) + if r.cmpltn_menu_visible: + stem = r.get_stem() + if len(stem) < 1: + r.cmpltn_reset() + else: + completions = [w for w in r.cmpltn_menu_choices + if w.startswith(stem)] + if completions: + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, 0, + r.use_brackets, r.sort_in_column) + else: + r.cmpltn_reset() + + +@dataclass +class CompletingReader(Reader): + """Adds completion support""" + + ### Class variables + # see the comment for the complete command + assume_immutable_completions = True + use_brackets = True # display completions inside [] + sort_in_column = False + + ### Instance variables + cmpltn_menu: list[str] = field(init=False) + cmpltn_menu_visible: bool = field(init=False) + cmpltn_message_visible: bool = field(init=False) + cmpltn_menu_end: int = field(init=False) + cmpltn_menu_choices: list[str] = field(init=False) + + def __post_init__(self) -> None: + super().__post_init__() + self.cmpltn_reset() + for c in (complete, self_insert): + self.commands[c.__name__] = c + self.commands[c.__name__.replace('_', '-')] = c + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r'\t', 'complete'),) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if not isinstance(cmd, (complete, self_insert)): + self.cmpltn_reset() + + def calc_screen(self) -> list[str]: + screen = super().calc_screen() + if self.cmpltn_menu_visible: + # We display the completions menu below the current prompt + ly = self.lxy[1] + 1 + screen[ly:ly] = self.cmpltn_menu + # If we're not in the middle of multiline edit, don't append to screeninfo + # since that screws up the position calculation in pos2xy function. + # This is a hack to prevent the cursor jumping + # into the completions menu when pressing left or down arrow. + if self.pos != len(self.buffer): + self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) + return screen + + def finish(self) -> None: + super().finish() + self.cmpltn_reset() + + def cmpltn_reset(self) -> None: + self.cmpltn_menu = [] + self.cmpltn_menu_visible = False + self.cmpltn_message_visible = False + self.cmpltn_menu_end = 0 + self.cmpltn_menu_choices = [] + + def get_stem(self) -> str: + st = self.syntax_table + SW = reader.SYNTAX_WORD + b = self.buffer + p = self.pos - 1 + while p >= 0 and st.get(b[p], SW) == SW: + p -= 1 + return ''.join(b[p+1:self.pos]) + + def get_completions(self, stem: str) -> list[str]: + return [] diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py new file mode 100644 index 00000000000..0d78890b4f4 --- /dev/null +++ b/Lib/_pyrepl/console.py @@ -0,0 +1,213 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import _colorize # type: ignore[import-not-found] + +from abc import ABC, abstractmethod +import ast +import code +from dataclasses import dataclass, field +import os.path +import sys + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + from typing import Callable + + +@dataclass +class Event: + evt: str + data: str + raw: bytes = b"" + + +@dataclass +class Console(ABC): + posxy: tuple[int, int] + screen: list[str] = field(default_factory=list) + height: int = 25 + width: int = 80 + + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + self.encoding = encoding or sys.getdefaultencoding() + + if isinstance(f_in, int): + self.input_fd = f_in + else: + self.input_fd = f_in.fileno() + + if isinstance(f_out, int): + self.output_fd = f_out + else: + self.output_fd = f_out.fileno() + + @abstractmethod + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + + @abstractmethod + def prepare(self) -> None: ... + + @abstractmethod + def restore(self) -> None: ... + + @abstractmethod + def move_cursor(self, x: int, y: int) -> None: ... + + @abstractmethod + def set_cursor_vis(self, visible: bool) -> None: ... + + @abstractmethod + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + ... + + @abstractmethod + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + ... + + @abstractmethod + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + ... + + @abstractmethod + def beep(self) -> None: ... + + @abstractmethod + def clear(self) -> None: + """Wipe the screen""" + ... + + @abstractmethod + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + ... + + @abstractmethod + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere).""" + ... + + @abstractmethod + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + ... + + @abstractmethod + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + ... + + @abstractmethod + def wait(self, timeout: float | None) -> bool: + """Wait for an event. The return value is True if an event is + available, False if the timeout has been reached. If timeout is + None, wait forever. The timeout is in milliseconds.""" + ... + + @property + def input_hook(self) -> Callable[[], int] | None: + """Returns the current input hook.""" + ... + + @abstractmethod + def repaint(self) -> None: ... + + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showsyntaxerror(self, filename=None, **kwargs): + super().showsyntaxerror(filename=filename, **kwargs) + + def _excepthook(self, typ, value, tb): + import traceback + lines = traceback.format_exception( + typ, value, tb, + colorize=self.can_colorize, + limit=traceback.BUILTIN_EXCEPTION_LIMIT) + self.write(''.join(lines)) + + def runsource(self, source, filename="", symbol="single"): + try: + tree = self.compile.compiler( + source, + filename, + "exec", + ast.PyCF_ONLY_AST, + incomplete_input=False, + ) + except (SyntaxError, OverflowError, ValueError): + self.showsyntaxerror(filename, source=source) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = self.compile.compiler(item, filename, the_symbol) + except SyntaxError as e: + if e.args[0] == "'await' outside function": + python = os.path.basename(sys.executable) + e.add_note( + f"Try the asyncio REPL ({python} -m asyncio) to use" + f" top-level 'await' and run background asyncio tasks." + ) + self.showsyntaxerror(filename, source=source) + return False + except (OverflowError, ValueError): + self.showsyntaxerror(filename, source=source) + return False + + if code is None: + return True + + self.runcode(code) + return False diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py new file mode 100644 index 00000000000..3a624d9f683 --- /dev/null +++ b/Lib/_pyrepl/curses.py @@ -0,0 +1,33 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +try: + import _curses +except ImportError: + try: + import curses as _curses # type: ignore[no-redef] + except ImportError: + from . import _minimal_curses as _curses # type: ignore[no-redef] + +setupterm = _curses.setupterm +tigetstr = _curses.tigetstr +tparm = _curses.tparm +error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py new file mode 100644 index 00000000000..0468b9a2670 --- /dev/null +++ b/Lib/_pyrepl/fancy_termios.py @@ -0,0 +1,76 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import termios + + +class TermState: + def __init__(self, tuples): + ( + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ) = tuples + + def as_list(self): + return [ + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + # Always return a copy of the control characters list to ensure + # there are not any additional references to self.cc + self.cc[:], + ] + + def copy(self): + return self.__class__(self.as_list()) + + +def tcgetattr(fd): + return TermState(termios.tcgetattr(fd)) + + +def tcsetattr(fd, when, attrs): + termios.tcsetattr(fd, when, attrs.as_list()) + + +class Term(TermState): + TS__init__ = TermState.__init__ + + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + + def save(self): + self.stack.append(self.as_list()) + + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) + + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py new file mode 100644 index 00000000000..c4b95fa2e81 --- /dev/null +++ b/Lib/_pyrepl/historical_reader.py @@ -0,0 +1,419 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field + +from . import commands, input +from .reader import Reader + + +if False: + from .types import SimpleContextManager, KeySpec, CommandName + + +isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + + [ + ("\\%03o" % c, "isearch-add-character") + for c in range(256) + if chr(c).isalpha() and chr(c) != "\\" + ] + + [ + ("\\\\", "self-insert"), + (r"\C-r", "isearch-backwards"), + (r"\C-s", "isearch-forwards"), + (r"\C-c", "isearch-cancel"), + (r"\C-g", "isearch-cancel"), + (r"\", "isearch-backspace"), + ] +) + +ISEARCH_DIRECTION_NONE = "" +ISEARCH_DIRECTION_BACKWARDS = "r" +ISEARCH_DIRECTION_FORWARDS = "f" + + +class next_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == len(r.history): + r.error("end of history list") + return + r.select_item(r.historyi + 1) + + +class previous_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == 0: + r.error("start of history list") + return + r.select_item(r.historyi - 1) + + +class history_search_backward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=False) + + +class history_search_forward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=True) + + +class restore_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi != len(r.history): + if r.get_unicode() != r.history[r.historyi]: + r.buffer = list(r.history[r.historyi]) + r.pos = len(r.buffer) + r.dirty = True + + +class first_history(commands.Command): + def do(self) -> None: + self.reader.select_item(0) + + +class last_history(commands.Command): + def do(self) -> None: + self.reader.select_item(len(self.reader.history)) + + +class operate_and_get_next(commands.FinishCommand): + def do(self) -> None: + self.reader.next_history = self.reader.historyi + 1 + + +class yank_arg(commands.Command): + def do(self) -> None: + r = self.reader + if r.last_command is self.__class__: + r.yank_arg_i += 1 + else: + r.yank_arg_i = 0 + if r.historyi < r.yank_arg_i: + r.error("beginning of history list") + return + a = r.get_arg(-1) + # XXX how to split? + words = r.get_item(r.historyi - r.yank_arg_i - 1).split() + if a < -len(words) or a >= len(words): + r.error("no such arg") + return + w = words[a] + b = r.buffer + if r.yank_arg_i > 0: + o = len(r.yank_arg_yanked) + else: + o = 0 + b[r.pos - o : r.pos] = list(w) + r.yank_arg_yanked = w + r.pos += len(w) - o + r.dirty = True + + +class forward_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_start = r.historyi, r.pos + r.isearch_term = "" + r.dirty = True + r.push_input_trans(r.isearch_trans) + + +class reverse_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.dirty = True + r.isearch_term = "" + r.push_input_trans(r.isearch_trans) + r.isearch_start = r.historyi, r.pos + + +class isearch_cancel(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.pop_input_trans() + r.select_item(r.isearch_start[0]) + r.pos = r.isearch_start[1] + r.dirty = True + + +class isearch_add_character(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + r.isearch_term += self.event[-1] + r.dirty = True + p = r.pos + len(r.isearch_term) - 1 + if b[p : p + 1] != [r.isearch_term[-1]]: + r.isearch_next() + + +class isearch_backspace(commands.Command): + def do(self) -> None: + r = self.reader + if len(r.isearch_term) > 0: + r.isearch_term = r.isearch_term[:-1] + r.dirty = True + else: + r.error("nothing to rubout") + + +class isearch_forwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_next() + + +class isearch_backwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.isearch_next() + + +class isearch_end(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.console.forgetinput() + r.pop_input_trans() + r.dirty = True + + +@dataclass +class HistoricalReader(Reader): + """Adds history support (with incremental history searching) to the + Reader class. + """ + + history: list[str] = field(default_factory=list) + historyi: int = 0 + next_history: int | None = None + transient_history: dict[int, str] = field(default_factory=dict) + isearch_term: str = "" + isearch_direction: str = ISEARCH_DIRECTION_NONE + isearch_start: tuple[int, int] = field(init=False) + isearch_trans: input.KeymapTranslator = field(init=False) + yank_arg_i: int = 0 + yank_arg_yanked: str = "" + + def __post_init__(self) -> None: + super().__post_init__() + for c in [ + next_history, + previous_history, + restore_history, + first_history, + last_history, + yank_arg, + forward_history_isearch, + reverse_history_isearch, + isearch_end, + isearch_add_character, + isearch_cancel, + isearch_add_character, + isearch_backspace, + isearch_forwards, + isearch_backwards, + operate_and_get_next, + history_search_backward, + history_search_forward, + ]: + self.commands[c.__name__] = c + self.commands[c.__name__.replace("_", "-")] = c + self.isearch_start = self.historyi, self.pos + self.isearch_trans = input.KeymapTranslator( + isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character + ) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\", "history-search-forward"), + (r"\x1b[6~", "history-search-forward"), + (r"\", "history-search-backward"), + (r"\x1b[5~", "history-search-backward"), + ) + + def select_item(self, i: int) -> None: + self.transient_history[self.historyi] = self.get_unicode() + buf = self.transient_history.get(i) + if buf is None: + buf = self.history[i].rstrip() + self.buffer = list(buf) + self.historyi = i + self.pos = len(self.buffer) + self.dirty = True + self.last_refresh_cache.invalidated = True + + def get_item(self, i: int) -> str: + if i != len(self.history): + return self.transient_history.get(i, self.history[i]) + else: + return self.transient_history.get(i, self.get_unicode()) + + @contextmanager + def suspend(self) -> SimpleContextManager: + with super().suspend(), self.suspend_history(): + yield + + @contextmanager + def suspend_history(self) -> SimpleContextManager: + try: + old_history = self.history[:] + del self.history[:] + yield + finally: + self.history[:] = old_history + + def prepare(self) -> None: + super().prepare() + try: + self.transient_history = {} + if self.next_history is not None and self.next_history < len(self.history): + self.historyi = self.next_history + self.buffer[:] = list(self.history[self.next_history]) + self.pos = len(self.buffer) + self.transient_history[len(self.history)] = "" + else: + self.historyi = len(self.history) + self.next_history = None + except: + self.restore() + raise + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: + d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] + return "(%s-search `%s') " % (d, self.isearch_term) + else: + return super().get_prompt(lineno, cursor_on_line) + + def search_next(self, *, forwards: bool) -> None: + """Search history for the current line contents up to the cursor. + + Selects the first item found. If nothing is under the cursor, any next + item in history is selected. + """ + pos = self.pos + s = self.get_unicode() + history_index = self.historyi + + # In multiline contexts, we're only interested in the current line. + nl_index = s.rfind('\n', 0, pos) + prefix = s[nl_index + 1:pos] + pos = len(prefix) + + match_prefix = len(prefix) + len_item = 0 + if history_index < len(self.history): + len_item = len(self.get_item(history_index)) + if len_item and pos == len_item: + match_prefix = False + elif not pos: + match_prefix = False + + while 1: + if forwards: + out_of_bounds = history_index >= len(self.history) - 1 + else: + out_of_bounds = history_index == 0 + if out_of_bounds: + if forwards and not match_prefix: + self.pos = 0 + self.buffer = [] + self.dirty = True + else: + self.error("not found") + return + + history_index += 1 if forwards else -1 + s = self.get_item(history_index) + + if not match_prefix: + self.select_item(history_index) + return + + len_acc = 0 + for i, line in enumerate(s.splitlines(keepends=True)): + if line.startswith(prefix): + self.select_item(history_index) + self.pos = pos + len_acc + return + len_acc += len(line) + + def isearch_next(self) -> None: + st = self.isearch_term + p = self.pos + i = self.historyi + s = self.get_unicode() + forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS + while 1: + if forwards: + p = s.find(st, p + 1) + else: + p = s.rfind(st, 0, p + len(st) - 1) + if p != -1: + self.select_item(i) + self.pos = p + return + elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0): + self.error("not found") + return + else: + if forwards: + i += 1 + s = self.get_item(i) + p = -1 + else: + i -= 1 + s = self.get_item(i) + p = len(s) + + def finish(self) -> None: + super().finish() + ret = self.get_unicode() + for i, t in self.transient_history.items(): + if i < len(self.history) and i != self.historyi: + self.history[i] = t + if ret and should_auto_add_history: + self.history.append(ret) + + +should_auto_add_history = True diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py new file mode 100644 index 00000000000..21c24eb5cde --- /dev/null +++ b/Lib/_pyrepl/input.py @@ -0,0 +1,114 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# (naming modules after builtin functions is not such a hot idea...) + +# an KeyTrans instance translates Event objects into Command objects + +# hmm, at what level do we want [C-i] and [tab] to be equivalent? +# [meta-a] and [esc a]? obviously, these are going to be equivalent +# for the UnixConsole, but should they be for PygameConsole? + +# it would in any situation seem to be a bad idea to bind, say, [tab] +# and [C-i] to *different* things... but should binding one bind the +# other? + +# executive, temporary decision: [tab] and [C-i] are distinct, but +# [meta-key] is identified with [esc key]. We demand that any console +# class does quite a lot towards emulating a unix terminal. + +from __future__ import annotations + +from abc import ABC, abstractmethod +import unicodedata +from collections import deque + + +# types +if False: + from .types import EventTuple + + +class InputTranslator(ABC): + @abstractmethod + def push(self, evt: EventTuple) -> None: + pass + + @abstractmethod + def get(self) -> EventTuple | None: + return None + + @abstractmethod + def empty(self) -> bool: + return True + + +class KeymapTranslator(InputTranslator): + def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None): + self.verbose = verbose + from .keymap import compile_keymap, parse_keys + + self.keymap = keymap + self.invalid_cls = invalid_cls + self.character_cls = character_cls + d = {} + for keyspec, command in keymap: + keyseq = tuple(parse_keys(keyspec)) + d[keyseq] = command + if self.verbose: + print(d) + self.k = self.ck = compile_keymap(d, ()) + self.results = deque() + self.stack = [] + + def push(self, evt): + if self.verbose: + print("pushed", evt.data, end="") + key = evt.data + d = self.k.get(key) + if isinstance(d, dict): + if self.verbose: + print("transition") + self.stack.append(key) + self.k = d + else: + if d is None: + if self.verbose: + print("invalid") + if self.stack or len(key) > 1 or unicodedata.category(key) == "C": + self.results.append((self.invalid_cls, self.stack + [key])) + else: + # small optimization: + self.k[key] = self.character_cls + self.results.append((self.character_cls, [key])) + else: + if self.verbose: + print("matched", d) + self.results.append((d, self.stack + [key])) + self.stack = [] + self.k = self.ck + + def get(self): + if self.results: + return self.results.popleft() + else: + return None + + def empty(self) -> bool: + return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py new file mode 100644 index 00000000000..2fb03d19523 --- /dev/null +++ b/Lib/_pyrepl/keymap.py @@ -0,0 +1,213 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +Keymap contains functions for parsing keyspecs and turning keyspecs into +appropriate sequences. + +A keyspec is a string representing a sequence of key presses that can +be bound to a command. All characters other than the backslash represent +themselves. In the traditional manner, a backslash introduces an escape +sequence. + +pyrepl uses its own keyspec format that is meant to be a strict superset of +readline's KEYSEQ format. This means that if a spec is found that readline +accepts that this doesn't, it should be logged as a bug. Note that this means +we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort. + +The extension to readline is that the sequence \\ denotes the +sequence of characters produced by hitting KEY. + +Examples: +`a' - what you get when you hit the `a' key +`\\EOA' - Escape - O - A (up, on my terminal) +`\\' - the up arrow key +`\\' - ditto (keynames are case-insensitive) +`\\C-o', `\\c-o' - control-o +`\\M-.' - meta-period +`\\E.' - ditto (that's how meta works for pyrepl) +`\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' + - all of these are the tab character. +""" + +_escapes = { + "\\": "\\", + "'": "'", + '"': '"', + "a": "\a", + "b": "\b", + "e": "\033", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", +} + +_keynames = { + "backspace": "backspace", + "delete": "delete", + "down": "down", + "end": "end", + "enter": "\r", + "escape": "\033", + "f1": "f1", + "f2": "f2", + "f3": "f3", + "f4": "f4", + "f5": "f5", + "f6": "f6", + "f7": "f7", + "f8": "f8", + "f9": "f9", + "f10": "f10", + "f11": "f11", + "f12": "f12", + "f13": "f13", + "f14": "f14", + "f15": "f15", + "f16": "f16", + "f17": "f17", + "f18": "f18", + "f19": "f19", + "f20": "f20", + "home": "home", + "insert": "insert", + "left": "left", + "page down": "page down", + "page up": "page up", + "return": "\r", + "right": "right", + "space": " ", + "tab": "\t", + "up": "up", +} + + +class KeySpecError(Exception): + pass + + +def parse_keys(keys: str) -> list[str]: + """Parse keys in keyspec format to a sequence of keys.""" + s = 0 + r: list[str] = [] + while s < len(keys): + k, s = _parse_single_key_sequence(keys, s) + r.extend(k) + return r + + +def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]: + ctrl = 0 + meta = 0 + ret = "" + while not ret and s < len(key): + if key[s] == "\\": + c = key[s + 1].lower() + if c in _escapes: + ret = _escapes[c] + s += 2 + elif c == "c": + if key[s + 2] != "-": + raise KeySpecError( + "\\C must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if ctrl: + raise KeySpecError( + "doubled \\C- (char %d of %s)" % (s + 1, repr(key)) + ) + ctrl = 1 + s += 3 + elif c == "m": + if key[s + 2] != "-": + raise KeySpecError( + "\\M must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if meta: + raise KeySpecError( + "doubled \\M- (char %d of %s)" % (s + 1, repr(key)) + ) + meta = 1 + s += 3 + elif c.isdigit(): + n = key[s + 1 : s + 4] + ret = chr(int(n, 8)) + s += 4 + elif c == "x": + n = key[s + 2 : s + 4] + ret = chr(int(n, 16)) + s += 4 + elif c == "<": + t = key.find(">", s) + if t == -1: + raise KeySpecError( + "unterminated \\< starting at char %d of %s" + % (s + 1, repr(key)) + ) + ret = key[s + 2 : t].lower() + if ret not in _keynames: + raise KeySpecError( + "unrecognised keyname `%s' at char %d of %s" + % (ret, s + 2, repr(key)) + ) + ret = _keynames[ret] + s = t + 1 + else: + raise KeySpecError( + "unknown backslash escape %s at char %d of %s" + % (repr(c), s + 2, repr(key)) + ) + else: + ret = key[s] + s += 1 + if ctrl: + if len(ret) == 1: + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + elif ret in {"left", "right"}: + ret = f"ctrl {ret}" + else: + raise KeySpecError("\\C- followed by invalid key") + + result = [ret], s + if meta: + result[0].insert(0, "\033") + return result + + +def compile_keymap(keymap, empty=b""): + r = {} + for key, value in keymap.items(): + if isinstance(key, bytes): + first = key[:1] + else: + first = key[0] + r.setdefault(first, {})[key[1:]] = value + for key, value in r.items(): + if empty in value: + if len(value) != 1: + raise KeySpecError("key definitions for %s clash" % (value.values(),)) + else: + r[key] = value[empty] + else: + r[key] = compile_keymap(value, empty) + return r diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py new file mode 100644 index 00000000000..a6f824dcc4a --- /dev/null +++ b/Lib/_pyrepl/main.py @@ -0,0 +1,59 @@ +import errno +import os +import sys + + +CAN_USE_PYREPL: bool +FAIL_REASON: str +try: + if sys.platform == "win32" and sys.getwindowsversion().build < 10586: + raise RuntimeError("Windows 10 TH2 or later required") + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + from .simple_interact import check + if err := check(): + raise RuntimeError(err) +except Exception as e: + CAN_USE_PYREPL = False + FAIL_REASON = f"warning: can't use pyrepl: {e}" +else: + CAN_USE_PYREPL = True + FAIL_REASON = "" + + +def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + if not CAN_USE_PYREPL: + if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON: + from .trace import trace + trace(FAIL_REASON) + print(FAIL_REASON, file=sys.stderr) + return sys._baserepl() + + if mainmodule: + namespace = mainmodule.__dict__ + else: + import __main__ + namespace = __main__.__dict__ + namespace.pop("__pyrepl_interactive_console", None) + + # sys._baserepl() above does this internally, we do it here + startup_path = os.getenv("PYTHONSTARTUP") + if pythonstartup and startup_path: + sys.audit("cpython.run_startup", startup_path) + + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, namespace) + + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + if not hasattr(sys, "ps2"): + sys.ps2 = "... " + + from .console import InteractiveColoredConsole + from .simple_interact import run_multiline_interactive_console + console = InteractiveColoredConsole(namespace, filename="") + run_multiline_interactive_console(console) diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini new file mode 100644 index 00000000000..395f5945ab7 --- /dev/null +++ b/Lib/_pyrepl/mypy.ini @@ -0,0 +1,24 @@ +# Config file for running mypy on _pyrepl. +# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/_pyrepl +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code,redundant-expr +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py new file mode 100644 index 00000000000..1fddc63e3ee --- /dev/null +++ b/Lib/_pyrepl/pager.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import io +import os +import re +import sys + + +# types +if False: + from typing import Protocol + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plain_pager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfile_pager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'pager', title) + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text: str) -> str: + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s: str) -> str: + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text: str, title: str = '') -> None: + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + has_tty = False + try: + import tty + import termios + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + tty.setcbreak(fd) + has_tty = True + + def getchar() -> str: + return sys.stdin.read(1) + + except (ImportError, AttributeError, io.UnsupportedOperation): + def getchar() -> str: + return sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) + + +def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py new file mode 100644 index 00000000000..dc26bfd3a34 --- /dev/null +++ b/Lib/_pyrepl/reader.py @@ -0,0 +1,816 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import sys + +from contextlib import contextmanager +from dataclasses import dataclass, field, fields +import unicodedata +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] + + +from . import commands, console, input +from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width +from .trace import trace + + +# types +Command = commands.Command +from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def disp_str(buffer: str) -> tuple[str, list[int]]: + """disp_str(buffer:string) -> (string, [int]) + + Return the string that should be the printed representation of + |buffer| and a list detailing where the characters of |buffer| + get used up. E.g.: + + >>> disp_str(chr(3)) + ('^C', [1, 0]) + + """ + b: list[int] = [] + s: list[str] = [] + for c in buffer: + if c == '\x1a': + s.append(c) + b.append(2) + elif ord(c) < 128: + s.append(c) + b.append(1) + elif unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + s.append(c) + b.extend([0] * (len(c) - 1)) + else: + s.append(c) + b.append(str_width(c)) + return "".join(s), b + + +# syntax classes: + +SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) + + +def make_default_syntax_table() -> dict[str, int]: + # XXX perhaps should use some unicodedata here? + st: dict[str, int] = {} + for c in map(chr, range(256)): + st[c] = SYNTAX_SYMBOL + for c in [a for a in map(chr, range(256)) if a.isalnum()]: + st[c] = SYNTAX_WORD + st["\n"] = st[" "] = SYNTAX_WHITESPACE + return st + + +def make_default_commands() -> dict[CommandName, type[Command]]: + result: dict[CommandName, type[Command]] = {} + for v in vars(commands).values(): + if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): + result[v.__name__] = v + result[v.__name__.replace("_", "-")] = v + return result + + +default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [ + (r"\C-a", "beginning-of-line"), + (r"\C-b", "left"), + (r"\C-c", "interrupt"), + (r"\C-d", "delete"), + (r"\C-e", "end-of-line"), + (r"\C-f", "right"), + (r"\C-g", "cancel"), + (r"\C-h", "backspace"), + (r"\C-j", "accept"), + (r"\", "accept"), + (r"\C-k", "kill-line"), + (r"\C-l", "clear-screen"), + (r"\C-m", "accept"), + (r"\C-t", "transpose-characters"), + (r"\C-u", "unix-line-discard"), + (r"\C-w", "unix-word-rubout"), + (r"\C-x\C-u", "upcase-region"), + (r"\C-y", "yank"), + *(() if sys.platform == "win32" else ((r"\C-z", "suspend"), )), + (r"\M-b", "backward-word"), + (r"\M-c", "capitalize-word"), + (r"\M-d", "kill-word"), + (r"\M-f", "forward-word"), + (r"\M-l", "downcase-word"), + (r"\M-t", "transpose-words"), + (r"\M-u", "upcase-word"), + (r"\M-y", "yank-pop"), + (r"\M--", "digit-arg"), + (r"\M-0", "digit-arg"), + (r"\M-1", "digit-arg"), + (r"\M-2", "digit-arg"), + (r"\M-3", "digit-arg"), + (r"\M-4", "digit-arg"), + (r"\M-5", "digit-arg"), + (r"\M-6", "digit-arg"), + (r"\M-7", "digit-arg"), + (r"\M-8", "digit-arg"), + (r"\M-9", "digit-arg"), + (r"\M-\n", "accept"), + ("\\\\", "self-insert"), + (r"\x1b[200~", "enable_bracketed_paste"), + (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x03", "ctrl-c"), + ] + + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] + + [ + (r"\", "up"), + (r"\", "down"), + (r"\", "left"), + (r"\C-\", "backward-word"), + (r"\", "right"), + (r"\C-\", "forward-word"), + (r"\", "delete"), + (r"\x1b[3~", "delete"), + (r"\", "backspace"), + (r"\M-\", "backward-kill-word"), + (r"\", "end-of-line"), # was 'end' + (r"\", "beginning-of-line"), # was 'home' + (r"\", "help"), + (r"\", "show-history"), + (r"\", "paste-mode"), + (r"\EOF", "end"), # the entries in the terminfo database for xterms + (r"\EOH", "home"), # seem to be wrong. this is a less than ideal + # workaround + ] +) + + +@dataclass(slots=True) +class Reader: + """The Reader class implements the bare bones of a command reader, + handling such details as editing and cursor motion. What it does + not support are such things as completion or history support - + these are implemented elsewhere. + + Instance variables of note include: + + * buffer: + A *list* (*not* a string at the moment :-) containing all the + characters that have been entered. + * console: + Hopefully encapsulates the OS dependent stuff. + * pos: + A 0-based index into `buffer' for where the insertion point + is. + * screeninfo: + Ahem. This list contains some info needed to move the + insertion point around reasonably efficiently. + * cxy, lxy: + the position of the insertion point in screen ... + * syntax_table: + Dictionary mapping characters to `syntax class'; read the + emacs docs to see what this means :-) + * commands: + Dictionary mapping command names to command classes. + * arg: + The emacs-style prefix argument. It will be None if no such + argument has been provided. + * dirty: + True if we need to refresh the display. + * kill_ring: + The emacs-style kill-ring; manipulated with yank & yank-pop + * ps1, ps2, ps3, ps4: + prompts. ps1 is the prompt for a one-line input; for a + multiline input it looks like: + ps2> first line of input goes here + ps3> second and further + ps3> lines get ps3 + ... + ps4> and the last one gets ps4 + As with the usual top-level, you can set these to instances if + you like; str() will be called on them (once) at the beginning + of each command. Don't put really long or newline containing + strings here, please! + This is just the default policy; you can change it freely by + overriding get_prompt() (and indeed some standard subclasses + do). + * finished: + handle1 will set this to a true value if a command signals + that we're done. + """ + + console: console.Console + + ## state + buffer: list[str] = field(default_factory=list) + pos: int = 0 + ps1: str = "->> " + ps2: str = "/>> " + ps3: str = "|.. " + ps4: str = R"\__ " + kill_ring: list[list[str]] = field(default_factory=list) + msg: str = "" + arg: int | None = None + dirty: bool = False + finished: bool = False + paste_mode: bool = False + in_bracketed_paste: bool = False + commands: dict[str, type[Command]] = field(default_factory=make_default_commands) + last_command: type[Command] | None = None + syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) + keymap: tuple[tuple[str, str], ...] = () + input_trans: input.KeymapTranslator = field(init=False) + input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + cxy: tuple[int, int] = field(init=False) + lxy: tuple[int, int] = field(init=False) + scheduled_commands: list[str] = field(default_factory=list) + can_colorize: bool = False + threading_hook: Callback | None = None + + ## cached metadata to speed up screen refreshes + @dataclass + class RefreshCache: + in_bracketed_paste: bool = False + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + line_end_offsets: list[int] = field(default_factory=list) + pos: int = field(init=False) + cxy: tuple[int, int] = field(init=False) + dimensions: tuple[int, int] = field(init=False) + invalidated: bool = False + + def update_cache(self, + reader: Reader, + screen: list[str], + screeninfo: list[tuple[int, list[int]]], + ) -> None: + self.in_bracketed_paste = reader.in_bracketed_paste + self.screen = screen.copy() + self.screeninfo = screeninfo.copy() + self.pos = reader.pos + self.cxy = reader.cxy + self.dimensions = reader.console.width, reader.console.height + self.invalidated = False + + def valid(self, reader: Reader) -> bool: + if self.invalidated: + return False + dimensions = reader.console.width, reader.console.height + dimensions_changed = dimensions != self.dimensions + paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste + return not (dimensions_changed or paste_changed) + + def get_cached_location(self, reader: Reader) -> tuple[int, int]: + if self.invalidated: + raise ValueError("Cache is invalidated") + offset = 0 + earliest_common_pos = min(reader.pos, self.pos) + num_common_lines = len(self.line_end_offsets) + while num_common_lines > 0: + offset = self.line_end_offsets[num_common_lines - 1] + if earliest_common_pos > offset: + break + num_common_lines -= 1 + else: + offset = 0 + return offset, num_common_lines + + last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) + + def __post_init__(self) -> None: + # Enable the use of `insert` without a `prepare` call - necessary to + # facilitate the tab completion hack implemented for + # . + self.keymap = self.collect_keymap() + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + self.screeninfo = [(0, [])] + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + self.can_colorize = can_colorize() + + self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.pos = self.pos + self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return default_keymap + + def calc_screen(self) -> list[str]: + """Translate changes in self.buffer into changes in self.console.screen.""" + # Since the last call to calc_screen: + # screen and screeninfo may differ due to a completion menu being shown + # pos and cxy may differ due to edits, cursor movements, or completion menus + + # Lines that are above both the old and new cursor position can't have changed, + # unless the terminal has been resized (which might cause reflowing) or we've + # entered or left paste mode (which changes prompts, causing reflowing). + num_common_lines = 0 + offset = 0 + if self.last_refresh_cache.valid(self): + offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) + + screen = self.last_refresh_cache.screen + del screen[num_common_lines:] + + screeninfo = self.last_refresh_cache.screeninfo + del screeninfo[num_common_lines:] + + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets + del last_refresh_line_end_offsets[num_common_lines:] + + pos = self.pos + pos -= offset + + prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + + lines = "".join(self.buffer[offset:]).split("\n") + + cursor_found = False + lines_beyond_cursor = 0 + for ln, line in enumerate(lines, num_common_lines): + ll = len(line) + if 0 <= pos <= ll: + self.lxy = pos, ln + cursor_found = True + elif cursor_found: + lines_beyond_cursor += 1 + if lines_beyond_cursor > self.console.height: + # No need to keep formatting lines. + # The console can't show them. + break + if prompt_from_cache: + # Only the first line's prompt can come from the cache + prompt_from_cache = False + prompt = "" + else: + prompt = self.get_prompt(ln, ll >= pos >= 0) + while "\n" in prompt: + pre_prompt, _, prompt = prompt.partition("\n") + last_refresh_line_end_offsets.append(offset) + screen.append(pre_prompt) + screeninfo.append((0, [])) + pos -= ll + 1 + prompt, lp = self.process_prompt(prompt) + l, l2 = disp_str(line) + wrapcount = (wlen(l) + lp) // self.console.width + if wrapcount == 0: + offset += ll + 1 # Takes all of the line plus the newline + last_refresh_line_end_offsets.append(offset) + screen.append(prompt + l) + screeninfo.append((lp, l2)) + else: + i = 0 + while l: + prelen = lp if i == 0 else 0 + index_to_wrap_before = 0 + column = 0 + for character_width in l2: + if column + character_width >= self.console.width - prelen: + break + index_to_wrap_before += 1 + column += character_width + pre = prompt if i == 0 else "" + if len(l) > index_to_wrap_before: + offset += index_to_wrap_before + post = "\\" + after = [1] + else: + offset += index_to_wrap_before + 1 # Takes the newline + post = "" + after = [] + last_refresh_line_end_offsets.append(offset) + screen.append(pre + l[:index_to_wrap_before] + post) + screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) + l = l[index_to_wrap_before:] + l2 = l2[index_to_wrap_before:] + i += 1 + self.screeninfo = screeninfo + self.cxy = self.pos2xy() + if self.msg: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + + self.last_refresh_cache.update_cache(self, screen, screeninfo) + return screen + + @staticmethod + def process_prompt(prompt: str) -> tuple[str, int]: + """Process the prompt. + + This means calculate the length of the prompt. The character \x01 + and \x02 are used to bracket ANSI control sequences and need to be + excluded from the length calculation. So also a copy of the prompt + is returned with these control characters removed.""" + + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + + # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars, + # which breaks the logic below so we redefine it here. + def wlen(s: str) -> int: + return sum(str_width(i) for i in s) + + out_prompt = "" + l = wlen(prompt) + pos = 0 + while True: + s = prompt.find("\x01", pos) + if s == -1: + break + e = prompt.find("\x02", s) + if e == -1: + break + # Found start and end brackets, subtract from string length + l = l - (e - s + 1) + keep = prompt[pos:s] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + prompt[s + 1 : e] + pos = e + 1 + keep = prompt[pos:] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + return out_prompt, l + + def bow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break preceding p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p -= 1 + return p + 1 + + def eow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break following p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p += 1 + while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p += 1 + return p + + def bol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break preceding p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + p -= 1 + while p >= 0 and b[p] != "\n": + p -= 1 + return p + 1 + + def eol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break following p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + while p < len(b) and b[p] != "\n": + p += 1 + return p + + def max_column(self, y: int) -> int: + """Return the last x-offset for line y""" + return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + + def max_row(self) -> int: + return len(self.screeninfo) - 1 + + def get_arg(self, default: int = 1) -> int: + """Return any prefix argument that the user has supplied, + returning `default' if there is None. Defaults to 1. + """ + if self.arg is None: + return default + return self.arg + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + """Return what should be in the left-hand margin for line + `lineno'.""" + if self.arg is not None and cursor_on_line: + prompt = f"(arg: {self.arg}) " + elif self.paste_mode and not self.in_bracketed_paste: + prompt = "(paste) " + elif "\n" in self.buffer: + if lineno == 0: + prompt = self.ps2 + elif self.ps4 and lineno == self.buffer.count("\n"): + prompt = self.ps4 + else: + prompt = self.ps3 + else: + prompt = self.ps1 + + if self.can_colorize: + prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + return prompt + + def push_input_trans(self, itrans: input.KeymapTranslator) -> None: + self.input_trans_stack.append(self.input_trans) + self.input_trans = itrans + + def pop_input_trans(self) -> None: + self.input_trans = self.input_trans_stack.pop() + + def setpos_from_xy(self, x: int, y: int) -> None: + """Set pos according to coordinates x, y""" + pos = 0 + i = 0 + while i < y: + prompt_len, character_widths = self.screeninfo[i] + offset = len(character_widths) - character_widths.count(0) + in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + if in_wrapped_line: + pos += offset - 1 # -1 cause backslash is not in buffer + else: + pos += offset + 1 # +1 cause newline is in buffer + i += 1 + + j = 0 + cur_x = self.screeninfo[i][0] + while cur_x < x: + if self.screeninfo[i][1][j] == 0: + continue + cur_x += self.screeninfo[i][1][j] + j += 1 + pos += 1 + + self.pos = pos + + def pos2xy(self) -> tuple[int, int]: + """Return the x, y coordinates of position 'pos'.""" + # this *is* incomprehensible, yes. + p, y = 0, 0 + l2: list[int] = [] + pos = self.pos + assert 0 <= pos <= len(self.buffer) + if pos == len(self.buffer) and len(self.screeninfo) > 0: + y = len(self.screeninfo) - 1 + p, l2 = self.screeninfo[y] + return p + sum(l2) + l2.count(0), y + + for p, l2 in self.screeninfo: + l = len(l2) - l2.count(0) + in_wrapped_line = p + sum(l2) >= self.console.width + offset = l - 1 if in_wrapped_line else l # need to remove backslash + if offset >= pos: + break + + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer + else: + pos -= l + 1 # +1 cause newline is in buffer + y += 1 + return p + sum(l2[:pos]), y + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.dirty = True + + def update_cursor(self) -> None: + """Move the cursor to reflect changes in self.pos""" + self.cxy = self.pos2xy() + self.console.move_cursor(*self.cxy) + + def after_command(self, cmd: Command) -> None: + """This function is called to allow post command cleanup.""" + if getattr(cmd, "kills_digit_arg", True): + if self.arg is not None: + self.dirty = True + self.arg = None + + def prepare(self) -> None: + """Get ready to run. Call restore when finished. You must not + write to the console in between the calls to prepare and + restore.""" + try: + self.console.prepare() + self.arg = None + self.finished = False + del self.buffer[:] + self.pos = 0 + self.dirty = True + self.last_command = None + self.calc_screen() + except BaseException: + self.restore() + raise + + while self.scheduled_commands: + cmd = self.scheduled_commands.pop() + self.do_cmd((cmd, [])) + + def last_command_is(self, cls: type) -> bool: + if not self.last_command: + return False + return issubclass(cls, self.last_command) + + def restore(self) -> None: + """Clean up after a run.""" + self.console.restore() + + @contextmanager + def suspend(self) -> SimpleContextManager: + """A context manager to delegate to another reader.""" + prev_state = {f.name: getattr(self, f.name) for f in fields(self)} + try: + self.restore() + yield + finally: + for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): + setattr(self, arg, prev_state[arg]) + self.prepare() + + def finish(self) -> None: + """Called when a command signals that we're finished.""" + pass + + def error(self, msg: str = "none") -> None: + self.msg = "! " + msg + " " + self.dirty = True + self.console.beep() + + def update_screen(self) -> None: + if self.dirty: + self.refresh() + + def refresh(self) -> None: + """Recalculate and refresh the screen.""" + if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n": + return + + # this call sets up self.cxy, so call it first. + self.screen = self.calc_screen() + self.console.refresh(self.screen, self.cxy) + self.dirty = False + + def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + """`cmd` is a tuple of "event_name" and "event", which in the current + implementation is always just the "buffer" which happens to be a list + of single-character strings.""" + + trace("received command {cmd}", cmd=cmd) + if isinstance(cmd[0], str): + command_type = self.commands.get(cmd[0], commands.invalid_command) + elif isinstance(cmd[0], type): + command_type = cmd[0] + else: + return # nothing to do + + command = command_type(self, *cmd) # type: ignore[arg-type] + command.do() + + self.after_command(command) + + if self.dirty: + self.refresh() + else: + self.update_cursor() + + if not isinstance(cmd, commands.digit_arg): + self.last_command = command_type + + self.finished = bool(command.finish) + if self.finished: + self.console.finish() + self.finish() + + def run_hooks(self) -> None: + threading_hook = self.threading_hook + if threading_hook is None and 'threading' in sys.modules: + from ._threading_handler import install_threading_hook + install_threading_hook(self) + if threading_hook is not None: + try: + threading_hook() + except Exception: + pass + + input_hook = self.console.input_hook + if input_hook: + try: + input_hook() + except Exception: + pass + + def handle1(self, block: bool = True) -> bool: + """Handle a single event. Wait as long as it takes if block + is true (the default), otherwise return False if no event is + pending.""" + + if self.msg: + self.msg = "" + self.dirty = True + + while True: + # We use the same timeout as in readline.c: 100ms + self.run_hooks() + self.console.wait(100) + event = self.console.get_event(block=False) + if not event: + if block: + continue + return False + + translate = True + + if event.evt == "key": + self.input_trans.push(event) + elif event.evt == "scroll": + self.refresh() + elif event.evt == "resize": + self.refresh() + else: + translate = False + + if translate: + cmd = self.input_trans.get() + else: + cmd = [event.evt, event.data] + + if cmd is None: + if block: + continue + return False + + self.do_cmd(cmd) + return True + + def push_char(self, char: int | bytes) -> None: + self.console.push_char(char) + self.handle1(block=False) + + def readline(self, startup_hook: Callback | None = None) -> str: + """Read a line. The implementation of this method also shows + how to drive Reader if you want more control over the event + loop.""" + self.prepare() + try: + if startup_hook is not None: + startup_hook() + self.refresh() + while not self.finished: + self.handle1() + return self.get_unicode() + + finally: + self.restore() + + def bind(self, spec: KeySpec, command: CommandName) -> None: + self.keymap = self.keymap + ((spec, command),) + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def get_unicode(self) -> str: + """Return the current buffer as a unicode string.""" + return "".join(self.buffer) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py new file mode 100644 index 00000000000..888185eb03b --- /dev/null +++ b/Lib/_pyrepl/readline.py @@ -0,0 +1,598 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Alex Gaynor +# Antonio Cuni +# Armin Rigo +# Holger Krekel +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""A compatibility wrapper reimplementing the 'readline' standard module +on top of pyrepl. Not all functionalities are supported. Contains +extensions for multiline input. +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field + +import os +from site import gethistoryfile # type: ignore[attr-defined] +import sys +from rlcompleter import Completer as RLCompleter + +from . import commands, historical_reader +from .completing_reader import CompletingReader +from .console import Console as ConsoleType + +Console: type[ConsoleType] +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import UnixConsole as Console, _error +except ImportError: + from .windows_console import WindowsConsole as Console, _error + +ENCODING = sys.getdefaultencoding() or "latin1" + + +# types +Command = commands.Command +from collections.abc import Callable, Collection +from .types import Callback, Completer, KeySpec, CommandName + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any, Mapping + + +MoreLinesCallable = Callable[[str], bool] + + +__all__ = [ + "add_history", + "clear_history", + "get_begidx", + "get_completer", + "get_completer_delims", + "get_current_history_length", + "get_endidx", + "get_history_item", + "get_history_length", + "get_line_buffer", + "insert_text", + "parse_and_bind", + "read_history_file", + # "read_init_file", + # "redisplay", + "remove_history_item", + "replace_history_item", + "set_auto_history", + "set_completer", + "set_completer_delims", + "set_history_length", + # "set_pre_input_hook", + "set_startup_hook", + "write_history_file", + # ---- multiline extensions ---- + "multiline_input", +] + +# ____________________________________________________________ + +@dataclass +class ReadlineConfig: + readline_completer: Completer | None = None + completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + + +@dataclass(kw_only=True) +class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + # Class fields + assume_immutable_completions = False + use_brackets = False + sort_in_column = True + + # Instance fields + config: ReadlineConfig + more_lines: MoreLinesCallable | None = None + last_used_indentation: str | None = None + + def __post_init__(self) -> None: + super().__post_init__() + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def error(self, msg: str = "none") -> None: + pass # don't show error messages by default + + def get_stem(self) -> str: + b = self.buffer + p = self.pos - 1 + completer_delims = self.config.completer_delims + while p >= 0 and b[p] not in completer_delims: + p -= 1 + return "".join(b[p + 1 : self.pos]) + + def get_completions(self, stem: str) -> list[str]: + if len(stem) == 0 and self.more_lines is not None: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != "\n": + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [" " * num_spaces] + result = [] + function = self.config.readline_completer + if function is not None: + try: + stem = str(stem) # rlcompleter.py seems to not like unicode + except UnicodeEncodeError: + pass # but feed unicode anyway if we have no choice + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + # emulate the behavior of the standard readline that sorts + # the completions before displaying them. + result.sort() + return result + + def get_trimmed_history(self, maxlength: int) -> list[str]: + if maxlength >= 0: + cut = len(self.history) - maxlength + if cut < 0: + cut = 0 + else: + cut = 0 + return self.history[cut:] + + def update_last_used_indentation(self) -> None: + indentation = _get_first_indentation(self.buffer) + if indentation is not None: + self.last_used_indentation = indentation + + # --- simplified support for reading multiline Python statements --- + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\n", "maybe-accept"), + (r"\", "backspace-dedent"), + ) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if self.more_lines is None: + # Force single-line input if we are in raw_input() mode. + # Although there is no direct way to add a \n in this mode, + # multiline buffers can still show up using various + # commands, e.g. navigating the history. + try: + index = self.buffer.index("\n") + except ValueError: + pass + else: + self.buffer = self.buffer[:index] + if self.pos > len(self.buffer): + self.pos = len(self.buffer) + + +def set_auto_history(_should_auto_add_history: bool) -> None: + """Enable or disable automatic history""" + historical_reader.should_auto_add_history = bool(_should_auto_add_history) + + +def _get_this_line_indent(buffer: list[str], pos: int) -> int: + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + + +def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]: + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + + +def _get_first_indentation(buffer: list[str]) -> str | None: + indented_line_start = None + for i in range(len(buffer)): + if (i < len(buffer) - 1 + and buffer[i] == "\n" + and buffer[i + 1] in " \t" + ): + indented_line_start = i + 1 + elif indented_line_start is not None and buffer[i] not in " \t\n": + return ''.join(buffer[indented_line_start : i]) + return None + + +def _should_auto_indent(buffer: list[str], pos: int) -> bool: + # check if last character before "pos" is a colon, ignoring + # whitespaces and comments. + last_char = None + while pos > 0: + pos -= 1 + if last_char is None: + if buffer[pos] not in " \t\n#": # ignore whitespaces and comments + last_char = buffer[pos] + else: + # even if we found a non-whitespace character before + # original pos, we keep going back until newline is reached + # to make sure we ignore comments + if buffer[pos] == "\n": + break + if buffer[pos] == "#": + last_char = None + return last_char == ":" + + +class maybe_accept(commands.Command): + def do(self) -> None: + r: ReadlineAlikeReader + r = self.reader # type: ignore[assignment] + r.dirty = True # this is needed to hide the completion menu, if visible + + if self.reader.in_bracketed_paste: + r.insert("\n") + return + + # if there are already several lines and the cursor + # is not on the last one, always insert a new \n. + text = r.get_unicode() + + if "\n" in r.buffer[r.pos :] or ( + r.more_lines is not None and r.more_lines(text) + ): + def _newline_before_pos(): + before_idx = r.pos - 1 + while before_idx > 0 and text[before_idx].isspace(): + before_idx -= 1 + return text[before_idx : r.pos].count("\n") > 0 + + # if there's already a new line before the cursor then + # even if the cursor is followed by whitespace, we assume + # the user is trying to terminate the block + if _newline_before_pos() and text[r.pos:].isspace(): + self.finish = True + return + + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) + r.insert("\n") + if not self.reader.paste_mode: + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + r.update_last_used_indentation() + if _should_auto_indent(r.buffer, r.pos): + if r.last_used_indentation is not None: + indentation = r.last_used_indentation + else: + # default + indentation = " " * 4 + r.insert(indentation) + elif not self.reader.paste_mode: + self.finish = True + else: + r.insert("\n") + + +class backspace_dedent(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos : r.pos + repeat] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +# ____________________________________________________________ + + +@dataclass(slots=True) +class _ReadlineWrapper: + f_in: int = -1 + f_out: int = -1 + reader: ReadlineAlikeReader | None = field(default=None, repr=False) + saved_history_length: int = -1 + startup_hook: Callback | None = None + config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False) + + def __post_init__(self) -> None: + if self.f_in == -1: + self.f_in = os.dup(0) + if self.f_out == -1: + self.f_out = os.dup(1) + + def get_reader(self) -> ReadlineAlikeReader: + if self.reader is None: + console = Console(self.f_in, self.f_out, encoding=ENCODING) + self.reader = ReadlineAlikeReader(console=console, config=self.config) + return self.reader + + def input(self, prompt: object = "") -> str: + try: + reader = self.get_reader() + except _error: + assert raw_input is not None + return raw_input(prompt) + prompt_str = str(prompt) + reader.ps1 = prompt_str + sys.audit("builtins.input", prompt_str) + result = reader.readline(startup_hook=self.startup_hook) + sys.audit("builtins.input/result", result) + return result + + def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str: + """Read an input on possibly multiple lines, asking for more + lines as long as 'more_lines(unicodetext)' returns an object whose + boolean value is true. + """ + reader = self.get_reader() + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = ps1 + reader.ps2 = ps1 + reader.ps3 = ps2 + reader.ps4 = "" + with warnings.catch_warnings(action="ignore"): + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + def parse_and_bind(self, string: str) -> None: + pass # XXX we don't support parsing GNU-readline-style init files + + def set_completer(self, function: Completer | None = None) -> None: + self.config.readline_completer = function + + def get_completer(self) -> Completer | None: + return self.config.readline_completer + + def set_completer_delims(self, delimiters: Collection[str]) -> None: + self.config.completer_delims = frozenset(delimiters) + + def get_completer_delims(self) -> str: + return "".join(sorted(self.config.completer_delims)) + + def _histline(self, line: str) -> str: + line = line.rstrip("\n") + return line + + def get_history_length(self) -> int: + return self.saved_history_length + + def set_history_length(self, length: int) -> None: + self.saved_history_length = length + + def get_current_history_length(self) -> int: + return len(self.get_reader().history) + + def read_history_file(self, filename: str = gethistoryfile()) -> None: + # multiline extension (really a hack) for the end of lines that + # are actually continuations inside a single multiline_input() + # history item: we use \r\n instead of just \n. If the history + # file is passed to GNU readline, the extra \r are just ignored. + history = self.get_reader().history + + with open(os.path.expanduser(filename), 'rb') as f: + is_editline = f.readline().startswith(b"_HiStOrY_V2_") + if is_editline: + encoding = "unicode-escape" + else: + f.seek(0) + encoding = "utf-8" + + lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')] + buffer = [] + for line in lines: + if line.endswith("\r"): + buffer.append(line+'\n') + else: + line = self._histline(line) + if buffer: + line = self._histline("".join(buffer).replace("\r", "") + line) + del buffer[:] + if line: + history.append(line) + + def write_history_file(self, filename: str = gethistoryfile()) -> None: + maxlength = self.saved_history_length + history = self.get_reader().get_trimmed_history(maxlength) + f = open(os.path.expanduser(filename), "w", + encoding="utf-8", newline="\n") + with f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + + def clear_history(self) -> None: + del self.get_reader().history[:] + + def get_history_item(self, index: int) -> str | None: + history = self.get_reader().history + if 1 <= index <= len(history): + return history[index - 1] + else: + return None # like readline.c + + def remove_history_item(self, index: int) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + del history[index] + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def replace_history_item(self, index: int, line: str) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + history[index] = self._histline(line) + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def add_history(self, line: str) -> None: + self.get_reader().history.append(self._histline(line)) + + def set_startup_hook(self, function: Callback | None = None) -> None: + self.startup_hook = function + + def get_line_buffer(self) -> str: + return self.get_reader().get_unicode() + + def _get_idxs(self) -> tuple[int, int]: + start = cursor = self.get_reader().pos + buf = self.get_line_buffer() + for i in range(cursor - 1, -1, -1): + if buf[i] in self.get_completer_delims(): + break + start = i + return start, cursor + + def get_begidx(self) -> int: + return self._get_idxs()[0] + + def get_endidx(self) -> int: + return self._get_idxs()[1] + + def insert_text(self, text: str) -> None: + self.get_reader().insert(text) + + +_wrapper = _ReadlineWrapper() + +# ____________________________________________________________ +# Public API + +parse_and_bind = _wrapper.parse_and_bind +set_completer = _wrapper.set_completer +get_completer = _wrapper.get_completer +set_completer_delims = _wrapper.set_completer_delims +get_completer_delims = _wrapper.get_completer_delims +get_history_length = _wrapper.get_history_length +set_history_length = _wrapper.set_history_length +get_current_history_length = _wrapper.get_current_history_length +read_history_file = _wrapper.read_history_file +write_history_file = _wrapper.write_history_file +clear_history = _wrapper.clear_history +get_history_item = _wrapper.get_history_item +remove_history_item = _wrapper.remove_history_item +replace_history_item = _wrapper.replace_history_item +add_history = _wrapper.add_history +set_startup_hook = _wrapper.set_startup_hook +get_line_buffer = _wrapper.get_line_buffer +get_begidx = _wrapper.get_begidx +get_endidx = _wrapper.get_endidx +insert_text = _wrapper.insert_text + +# Extension +multiline_input = _wrapper.multiline_input + +# Internal hook +_get_reader = _wrapper.get_reader + +# ____________________________________________________________ +# Stubs + + +def _make_stub(_name: str, _ret: object) -> None: + def stub(*args: object, **kwds: object) -> None: + import warnings + + warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) + + stub.__name__ = _name + globals()[_name] = stub + + +for _name, _ret in [ + ("read_init_file", None), + ("redisplay", None), + ("set_pre_input_hook", None), +]: + assert _name not in globals(), _name + _make_stub(_name, _ret) + +# ____________________________________________________________ + + +def _setup(namespace: Mapping[str, Any]) -> None: + global raw_input + if raw_input is not None: + return # don't run _setup twice + + try: + f_in = sys.stdin.fileno() + f_out = sys.stdout.fileno() + except (AttributeError, ValueError): + return + if not os.isatty(f_in) or not os.isatty(f_out): + return + + _wrapper.f_in = f_in + _wrapper.f_out = f_out + + # set up namespace in rlcompleter, which requires it to be a bona fide dict + if not isinstance(namespace, dict): + namespace = dict(namespace) + _wrapper.config.readline_completer = RLCompleter(namespace).complete + + # this is not really what readline.c does. Better than nothing I guess + import builtins + raw_input = builtins.input + builtins.input = _wrapper.input + + +raw_input: Callable[[object], str] | None = None diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py new file mode 100644 index 00000000000..66e66eae7ea --- /dev/null +++ b/Lib/_pyrepl/simple_interact.py @@ -0,0 +1,167 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""This is an alternative to python_reader which tries to emulate +the CPython prompt as closely as possible, with the exception of +allowing multiline input and multiline history entries. +""" + +from __future__ import annotations + +import _sitebuiltins +import linecache +import functools +import os +import sys +import code + +from .readline import _get_reader, multiline_input + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + + +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import _error +except ModuleNotFoundError: + from .windows_console import _error + +def check() -> str: + """Returns the error message if there is a problem initializing the state.""" + try: + _get_reader() + except _error as e: + if term := os.environ.get("TERM", ""): + term = f"; TERM={term}" + return str(str(e) or repr(e) or "unknown error") + term + return "" + + +def _strip_final_indent(text: str) -> str: + # kill spaces and tabs at the end, but only if they follow '\n'. + # meant to remove the auto-indentation only (although it would of + # course also remove explicitly-added indentation). + short = text.rstrip(" \t") + n = len(short) + if n > 0 and text[n - 1] == "\n": + return short + return text + + +def _clear_screen(): + reader = _get_reader() + reader.scheduled_commands.append("clear_screen") + + +REPL_COMMANDS = { + "exit": _sitebuiltins.Quitter('exit', ''), + "quit": _sitebuiltins.Quitter('quit' ,''), + "copyright": _sitebuiltins._Printer('copyright', sys.copyright), + "help": _sitebuiltins._Helper(), + "clear": _clear_screen, + "\x1a": _sitebuiltins.Quitter('\x1a', ''), +} + + +def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool: + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + lines = src.splitlines(keepends=True) + if len(lines) == 1: + return False + + last_line = lines[-1] + was_indented = last_line.startswith((" ", "\t")) + not_empty = last_line.strip() != "" + incomplete = not last_line.endswith("\n") + return (was_indented or not_empty) and incomplete + else: + return code is None + + +def run_multiline_interactive_console( + console: code.InteractiveConsole, + *, + future_flags: int = 0, +) -> None: + from .readline import _setup + _setup(console.locals) + if future_flags: + console.compile.compiler.flags |= future_flags + + more_lines = functools.partial(_more_lines, console) + input_n = 0 + + def maybe_run_command(statement: str) -> bool: + statement = statement.strip() + if statement in console.locals or statement not in REPL_COMMANDS: + return False + + reader = _get_reader() + reader.history.pop() # skip internal commands in history + command = REPL_COMMANDS[statement] + if callable(command): + # Make sure that history does not change because of commands + with reader.suspend_history(): + command() + return True + return False + + while 1: + try: + try: + sys.stdout.flush() + except Exception: + pass + + ps1 = getattr(sys, "ps1", ">>> ") + ps2 = getattr(sys, "ps2", "... ") + try: + statement = multiline_input(more_lines, ps1, ps2) + except EOFError: + break + + if maybe_run_command(statement): + continue + + input_name = f"" + linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] + assert not more + input_n += 1 + except KeyboardInterrupt: + r = _get_reader() + if r.input_trans is r.isearch_trans: + r.do_cmd(("isearch-end", [""])) + r.pos = len(r.get_unicode()) + r.dirty = True + r.refresh() + r.in_bracketed_paste = False + console.write("\nKeyboardInterrupt\n") + console.resetbuffer() + except MemoryError: + console.write("\nMemoryError\n") + console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py new file mode 100644 index 00000000000..a8eb2433cd3 --- /dev/null +++ b/Lib/_pyrepl/trace.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import os + +# types +if False: + from typing import IO + + +trace_file: IO[str] | None = None +if trace_filename := os.environ.get("PYREPL_TRACE"): + trace_file = open(trace_filename, "a") + + +def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py new file mode 100644 index 00000000000..f9d48b828c7 --- /dev/null +++ b/Lib/_pyrepl/types.py @@ -0,0 +1,8 @@ +from collections.abc import Callable, Iterator + +Callback = Callable[[], object] +SimpleContextManager = Iterator[None] +KeySpec = str # like r"\C-c" +CommandName = str # like "interrupt" +EventTuple = tuple[CommandName, str] +Completer = Callable[[str, int], str | None] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py new file mode 100644 index 00000000000..e69c96b1159 --- /dev/null +++ b/Lib/_pyrepl/unix_console.py @@ -0,0 +1,810 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import errno +import os +import re +import select +import signal +import struct +import termios +import time +import platform +from fcntl import ioctl + +from . import curses +from .console import Console, Event +from .fancy_termios import tcgetattr, tcsetattr +from .trace import trace +from .unix_eventqueue import EventQueue +from .utils import wlen + + +TYPE_CHECKING = False + +# types +if TYPE_CHECKING: + from typing import IO, Literal, overload +else: + overload = lambda func: None + + +class InvalidTerminal(RuntimeError): + pass + + +_error = (termios.error, curses.error, InvalidTerminal) + +SIGWINCH_EVENT = "repaint" + +FIONREAD = getattr(termios, "FIONREAD", None) +TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) + +# ------------ start of baudrate definitions ------------ + +# Add (possibly) missing baudrates (check termios man page) to termios + + +def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: + baudrate_name = "B%d" % rate + if hasattr(termios, baudrate_name): + dictionary[getattr(termios, baudrate_name)] = rate + + +# Check the termios man page (Line speed) to know where these +# values come from. +potential_baudrates = [ + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, +] + +ratedict: dict[int, int] = {} +for rate in potential_baudrates: + add_baudrate_if_supported(ratedict, rate) + +# Clean up variables to avoid unintended usage +del rate, add_baudrate_if_supported + +# ------------ end of baudrate definitions ------------ + +delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") + +try: + poll: type[select.poll] = select.poll +except AttributeError: + # this is exactly the minumum necessary to support what we + # do with poll objects + class MinimalPoll: + def __init__(self): + pass + + def register(self, fd, flag): + self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* + def poll(self, timeout: float | None = None) -> list[int]: + if timeout is None: + r, w, e = select.select([self.fd], [], []) + else: + r, w, e = select.select([self.fd], [], [], timeout/1000) + return r + + poll = MinimalPoll # type: ignore[assignment] + + +class UnixConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + """ + Initialize the UnixConsole. + + Parameters: + - f_in (int or file-like object): Input file descriptor or object. + - f_out (int or file-like object): Output file descriptor or object. + - term (str): Terminal name. + - encoding (str): Encoding to use for I/O operations. + """ + super().__init__(f_in, f_out, term, encoding) + + self.pollob = poll() + self.pollob.register(self.input_fd, select.POLLIN) + self.input_buffer = b"" + self.input_buffer_pos = 0 + curses.setupterm(term or None, self.output_fd) + self.term = term + + @overload + def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + + @overload + def _my_getstr(cap: str, optional: bool) -> bytes | None: ... + + def _my_getstr(cap: str, optional: bool = False) -> bytes | None: + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=True) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) + + self.__setup_movement() + + self.event_queue = EventQueue(self.input_fd, self.encoding) + self.cursor_visible = 1 + + def more_in_buffer(self) -> bool: + return bool( + self.input_buffer + and self.input_buffer_pos < len(self.input_buffer) + ) + + def __read(self, n: int) -> bytes: + if not self.more_in_buffer(): + self.input_buffer = os.read(self.input_fd, 10000) + + ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n] + self.input_buffer_pos += len(ret) + if self.input_buffer_pos >= len(self.input_buffer): + self.input_buffer = b"" + self.input_buffer_pos = 0 + return ret + + + def change_encoding(self, encoding: str) -> None: + """ + Change the encoding used for I/O operations. + + Parameters: + - encoding (str): New encoding to use. + """ + self.encoding = encoding + + def refresh(self, screen, c_xy): + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + if not self.__gone_tall: + while len(self.screen) < min(len(screen), self.height): + self.__hide_cursor() + self.__move(0, len(self.screen) - 1) + self.__write("\n") + self.posxy = 0, len(self.screen) + self.screen.append("") + else: + while len(self.screen) < len(screen): + self.screen.append("") + + if len(screen) > self.height: + self.__gone_tall = 1 + self.__move = self.__move_tall + + px, py = self.posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + # use hardware scrolling if we have it. + if old_offset > offset and self._ri: + self.__hide_cursor() + self.__write_code(self._cup, 0, 0) + self.posxy = 0, old_offset + for i in range(old_offset - offset): + self.__write_code(self._ri) + oldscr.pop(-1) + oldscr.insert(0, "") + elif old_offset < offset and self._ind: + self.__hide_cursor() + self.__write_code(self._cup, self.height - 1, 0) + self.posxy = 0, old_offset + self.height - 1 + for i in range(offset - old_offset): + self.__write_code(self._ind) + oldscr.pop(0) + oldscr.append("") + + self.__offset = offset + + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self.__hide_cursor() + self.__move(0, y) + self.posxy = 0, y + self.__write_code(self._el) + y += 1 + + self.__show_cursor() + + self.screen = screen.copy() + self.move_cursor(cx, cy) + self.flushoutput() + + def move_cursor(self, x, y): + """ + Move the cursor to the specified position on the screen. + + Parameters: + - x (int): X coordinate. + - y (int): Y coordinate. + """ + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(Event("scroll", None)) + else: + self.__move(x, y) + self.posxy = x, y + self.flushoutput() + + def prepare(self): + """ + Prepare the console for input/output operations. + """ + self.__svtermstate = tcgetattr(self.input_fd) + raw = self.__svtermstate.copy() + raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) + raw.oflag &= ~(termios.OPOST) + raw.cflag &= ~(termios.CSIZE | termios.PARENB) + raw.cflag |= termios.CS8 + raw.iflag |= termios.BRKINT + raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) + raw.lflag |= termios.ISIG + raw.cc[termios.VMIN] = 1 + raw.cc[termios.VTIME] = 0 + tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + + # In macOS terminal we need to deactivate line wrap via ANSI escape code + if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + os.write(self.output_fd, b"\033[?7l") + + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__buffer = [] + + self.posxy = 0, 0 + self.__gone_tall = 0 + self.__move = self.__move_short + self.__offset = 0 + + self.__maybe_write_code(self._smkx) + + try: + self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) + except ValueError: + pass + + self.__enable_bracketed_paste() + + def restore(self): + """ + Restore the console to the default state + """ + self.__disable_bracketed_paste() + self.__maybe_write_code(self._rmkx) + self.flushoutput() + tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + + if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + os.write(self.output_fd, b"\033[?7h") + + if hasattr(self, "old_sigwinch"): + signal.signal(signal.SIGWINCH, self.old_sigwinch) + del self.old_sigwinch + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + trace("push char {char!r}", char=char) + self.event_queue.push(char) + + def get_event(self, block: bool = True) -> Event | None: + """ + Get an event from the console event queue. + + Parameters: + - block (bool): Whether to block until an event is available. + + Returns: + - Event: Event object from the event queue. + """ + if not block and not self.wait(timeout=0): + return None + + while self.event_queue.empty(): + while True: + try: + self.push_char(self.__read(1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + return self.event_queue.get() + + def wait(self, timeout: float | None = None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.more_in_buffer() + or bool(self.pollob.poll(timeout)) + ) + + def set_cursor_vis(self, visible): + """ + Set the visibility of the cursor. + + Parameters: + - visible (bool): Visibility flag. + """ + if visible: + self.__show_cursor() + else: + self.__hide_cursor() + + if TIOCGWINSZ: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except (KeyError, TypeError, ValueError): + try: + size = ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + except OSError: + return 25, 80 + height, width = struct.unpack("hhhh", size)[0:2] + if not height: + return 25, 80 + return height, width + + else: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except (KeyError, TypeError, ValueError): + return 25, 80 + + def forgetinput(self): + """ + Discard any pending input on the console. + """ + termios.tcflush(self.input_fd, termios.TCIFLUSH) + + def flushoutput(self): + """ + Flush the output buffer. + """ + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] + + def finish(self): + """ + Finish console operations and flush the output buffer. + """ + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write("\n\r") + self.flushoutput() + + def beep(self): + """ + Emit a beep sound. + """ + self.__maybe_write_code(self._bel) + self.flushoutput() + + if FIONREAD: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + else: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = 10000 + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + def clear(self): + """ + Clear the console screen. + """ + self.__write_code(self._clear) + self.__gone_tall = 1 + self.__move = self.__move_tall + self.posxy = 0, 0 + self.screen = [] + + @property + def input_hook(self): + try: + import posix + except ImportError: + return None + if posix._is_inputhook_installed(): + return posix._inputhook + + def __enable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004h") + + def __disable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004l") + + def __setup_movement(self): + """ + Set up the movement functions based on the terminal capabilities. + """ + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + def __write_changed_line(self, y, oldline, newline, px_coord): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + # if we need to insert a single character right after the first detected change + if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + if ( + y == self.posxy[1] + and x_coord > self.posxy[0] + and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + ): + x_pos = px_pos + x_coord = px_coord + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.posxy = x_coord + character_width, y + + # if it's a single character change in the middle of the line + elif ( + x_coord < minlen + and oldline[x_pos + 1 :] == newline[x_pos + 1 :] + and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + ): + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write(newline[x_pos]) + self.posxy = x_coord + character_width, y + + # if this is the last character to fit in the line and we edit in the middle of the line + elif ( + self.dch1 + and self.ich1 + and wlen(newline) == self.width + and x_coord < wlen(newline) - 2 + and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.posxy = self.width - 2, y + self.__write_code(self.dch1) + + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.posxy = character_width + 1, y + + else: + self.__hide_cursor() + self.__move(x_coord, y) + if wlen(oldline) > wlen(newline): + self.__write_code(self._el) + self.__write(newline[x_pos:]) + self.posxy = wlen(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + assert self._cud1 is not None + assert self._cuu1 is not None + dy = y - self.posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x: int) -> None: + if x != self.posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x: int) -> None: + assert self._cuf1 is not None + assert self._cub1 is not None + dx = x - self.posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x: int) -> None: + dx = x - self.posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint(self): + if not self.__gone_tall: + self.posxy = 0, self.posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py new file mode 100644 index 00000000000..70cfade26e2 --- /dev/null +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -0,0 +1,152 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from collections import deque + +from . import keymap +from .console import Event +from . import curses +from .trace import trace +from termios import tcgetattr, VERASE +import os + + +# Mapping of human-readable key names to their terminal-specific codes +TERMINAL_KEYNAMES = { + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +# Function keys F1-F20 mapping +TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21)) + +# Known CTRL-arrow keycodes +CTRL_ARROW_KEYCODES= { + # for xterm, gnome-terminal, xfce terminal, etc. + b'\033[1;5D': 'ctrl left', + b'\033[1;5C': 'ctrl right', + # for rxvt + b'\033Od': 'ctrl left', + b'\033Oc': 'ctrl right', +} + +def get_terminal_keycodes() -> dict[bytes, str]: + """ + Generates a dictionary mapping terminal keycodes to human-readable names. + """ + keycodes = {} + for key, terminal_code in TERMINAL_KEYNAMES.items(): + keycode = curses.tigetstr(terminal_code) + trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) + if keycode: + keycodes[keycode] = key + keycodes.update(CTRL_ARROW_KEYCODES) + return keycodes + +class EventQueue: + def __init__(self, fd: int, encoding: str) -> None: + self.keycodes = get_terminal_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + self.keycodes[backspace] = "backspace" + self.compiled_keymap = keymap.compile_keymap(self.keycodes) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + ord_char = char if isinstance(char, int) else ord(char) + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) + if char in self.keymap: + if self.keymap is self.compiled_keymap: + #sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, self.flush_buf())) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', bytearray(b'\033'))) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py new file mode 100644 index 00000000000..4651717bd7e --- /dev/null +++ b/Lib/_pyrepl/utils.py @@ -0,0 +1,25 @@ +import re +import unicodedata +import functools + +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +@functools.cache +def str_width(c: str) -> int: + if ord(c) < 128: + return 1 + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + +def wlen(s: str) -> int: + if len(s) == 1 and s != '\x1a': + return str_width(s) + length = sum(str_width(i) for i in s) + # remove lengths of any escape sequences + sequence = ANSI_ESCAPE_SEQUENCE.findall(s) + ctrl_z_cnt = s.count('\x1a') + return length - sum(len(i) for i in sequence) + ctrl_z_cnt diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py new file mode 100644 index 00000000000..fffadd5e2ec --- /dev/null +++ b/Lib/_pyrepl/windows_console.py @@ -0,0 +1,618 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import io +import os +import sys +import time +import msvcrt + +from collections import deque +import ctypes +from ctypes.wintypes import ( + _COORD, + WORD, + SMALL_RECT, + BOOL, + HANDLE, + CHAR, + DWORD, + WCHAR, + SHORT, +) +from ctypes import Structure, POINTER, Union +from .console import Event, Console +from .trace import trace +from .utils import wlen + +try: + from ctypes import GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] +except: + # Keep MyPy happy off Windows + from ctypes import CDLL as WinDLL, cdll as windll + + def GetLastError() -> int: + return 42 + + class WinError(OSError): # type: ignore[no-redef] + def __init__(self, err: int | None, descr: str | None = None) -> None: + self.err = err + self.descr = descr + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + +# Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes +VK_MAP: dict[int, str] = { + 0x23: "end", # VK_END + 0x24: "home", # VK_HOME + 0x25: "left", # VK_LEFT + 0x26: "up", # VK_UP + 0x27: "right", # VK_RIGHT + 0x28: "down", # VK_DOWN + 0x2E: "delete", # VK_DELETE + 0x70: "f1", # VK_F1 + 0x71: "f2", # VK_F2 + 0x72: "f3", # VK_F3 + 0x73: "f4", # VK_F4 + 0x74: "f5", # VK_F5 + 0x75: "f6", # VK_F6 + 0x76: "f7", # VK_F7 + 0x77: "f8", # VK_F8 + 0x78: "f9", # VK_F9 + 0x79: "f10", # VK_F10 + 0x7A: "f11", # VK_F11 + 0x7B: "f12", # VK_F12 + 0x7C: "f13", # VK_F13 + 0x7D: "f14", # VK_F14 + 0x7E: "f15", # VK_F15 + 0x7F: "f16", # VK_F16 + 0x80: "f17", # VK_F17 + 0x81: "f18", # VK_F18 + 0x82: "f19", # VK_F19 + 0x83: "f20", # VK_F20 +} + +# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +ERASE_IN_LINE = "\x1b[K" +MOVE_LEFT = "\x1b[{}D" +MOVE_RIGHT = "\x1b[{}C" +MOVE_UP = "\x1b[{}A" +MOVE_DOWN = "\x1b[{}B" +CLEAR = "\x1b[H\x1b[J" + + +class _error(Exception): + pass + + +class WindowsConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + super().__init__(f_in, f_out, term, encoding) + + SetConsoleMode( + OutHandle, + ENABLE_WRAP_AT_EOL_OUTPUT + | ENABLE_PROCESSED_OUTPUT + | ENABLE_VIRTUAL_TERMINAL_PROCESSING, + ) + self.screen: list[str] = [] + self.width = 80 + self.height = 25 + self.__offset = 0 + self.event_queue: deque[Event] = deque() + try: + self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined] + except ValueError: + # Console I/O is redirected, fallback... + self.out = None + + def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + + while len(self.screen) < min(len(screen), self.height): + self._hide_cursor() + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") + self.posxy = 0, len(self.screen) + self.screen.append("") + + px, py = self.posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + scroll_lines = offset - old_offset + + # Scrolling the buffer as the current input is greater than the visible + # portion of the window. We need to scroll the visible portion and the + # entire history + self._scroll(scroll_lines, self._getscrollbacksize()) + self.posxy = self.posxy[0], self.posxy[1] + scroll_lines + self.__offset += scroll_lines + + for i in range(scroll_lines): + self.screen.append("") + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + self.__offset = offset + + self._hide_cursor() + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self._move_relative(0, y) + self.posxy = 0, y + self._erase_to_end() + y += 1 + + self._show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + + @property + def input_hook(self): + try: + import nt + except ImportError: + return None + if nt._is_inputhook_installed(): + return nt._inputhook + + def __write_changed_line( + self, y: int, oldline: str, newline: str, px_coord: int + ) -> None: + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + self._hide_cursor() + self._move_relative(x_coord, y) + if wlen(oldline) > wlen(newline): + self._erase_to_end() + + self.__write(newline[x_pos:]) + if wlen(newline) == self.width: + # If we wrapped we want to start at the next line + self._move_relative(0, y + 1) + self.posxy = 0, y + 1 + else: + self.posxy = wlen(newline), y + + if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def _scroll( + self, top: int, bottom: int, left: int | None = None, right: int | None = None + ) -> None: + scroll_rect = SMALL_RECT() + scroll_rect.Top = SHORT(top) + scroll_rect.Bottom = SHORT(bottom) + scroll_rect.Left = SHORT(0 if left is None else left) + scroll_rect.Right = SHORT( + self.getheightwidth()[1] - 1 if right is None else right + ) + destination_origin = _COORD() + fill_info = CHAR_INFO() + fill_info.UnicodeChar = " " + + if not ScrollConsoleScreenBuffer( + OutHandle, scroll_rect, None, destination_origin, fill_info + ): + raise WinError(GetLastError()) + + def _hide_cursor(self): + self.__write("\x1b[?25l") + + def _show_cursor(self): + self.__write("\x1b[?25h") + + def _enable_blinking(self): + self.__write("\x1b[?12h") + + def _disable_blinking(self): + self.__write("\x1b[?12l") + + def __write(self, text: str) -> None: + if "\x1a" in text: + text = ''.join(["^Z" if x == '\x1a' else x for x in text]) + + if self.out is not None: + self.out.write(text.encode(self.encoding, "replace")) + self.out.flush() + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + + @property + def screen_xy(self) -> tuple[int, int]: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return info.dwCursorPosition.X, info.dwCursorPosition.Y + + def _erase_to_end(self) -> None: + self.__write(ERASE_IN_LINE) + + def prepare(self) -> None: + trace("prepare") + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.posxy = 0, 0 + self.__gone_tall = 0 + self.__offset = 0 + + def restore(self) -> None: + pass + + def _move_relative(self, x: int, y: int) -> None: + """Moves relative to the current posxy""" + dx = x - self.posxy[0] + dy = y - self.posxy[1] + if dx < 0: + self.__write(MOVE_LEFT.format(-dx)) + elif dx > 0: + self.__write(MOVE_RIGHT.format(dx)) + + if dy < 0: + self.__write(MOVE_UP.format(-dy)) + elif dy > 0: + self.__write(MOVE_DOWN.format(dy)) + + def move_cursor(self, x: int, y: int) -> None: + if x < 0 or y < 0: + raise ValueError(f"Bad cursor position {x}, {y}") + + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(0, Event("scroll", "")) + else: + self._move_relative(x, y) + self.posxy = x, y + + def set_cursor_vis(self, visible: bool) -> None: + if visible: + self._show_cursor() + else: + self._hide_cursor() + + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return ( + info.srWindow.Bottom - info.srWindow.Top + 1, + info.srWindow.Right - info.srWindow.Left + 1, + ) + + def _getscrollbacksize(self) -> int: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + + return info.srWindow.Bottom # type: ignore[no-any-return] + + def _read_input(self, block: bool = True) -> INPUT_RECORD | None: + if not block: + events = DWORD() + if not GetNumberOfConsoleInputEvents(InHandle, events): + raise WinError(GetLastError()) + if not events.value: + return None + + rec = INPUT_RECORD() + read = DWORD() + if not ReadConsoleInput(InHandle, rec, 1, read): + raise WinError(GetLastError()) + + return rec + + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + if self.event_queue: + return self.event_queue.pop() + + while True: + rec = self._read_input(block) + if rec is None: + return None + + if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: + return Event("resize", "") + + if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown: + # Only process keys and keydown events + if block: + continue + return None + + key = rec.Event.KeyEvent.uChar.UnicodeChar + + if rec.Event.KeyEvent.uChar.UnicodeChar == "\r": + # Make enter make unix-like + return Event(evt="key", data="\n", raw=b"\n") + elif rec.Event.KeyEvent.wVirtualKeyCode == 8: + # Turn backspace directly into the command + return Event( + evt="key", + data="backspace", + raw=rec.Event.KeyEvent.uChar.UnicodeChar, + ) + elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00": + # Handle special keys like arrow keys and translate them into the appropriate command + code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode) + if code: + return Event( + evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar + ) + if block: + continue + + return None + + return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar) + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + raise NotImplementedError("push_char not supported on Windows") + + def beep(self) -> None: + self.__write("\x07") + + def clear(self) -> None: + """Wipe the screen""" + self.__write(CLEAR) + self.posxy = 0, 0 + self.screen = [""] + + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self._move_relative(0, min(y, self.height + self.__offset - 1)) + self.__write("\r\n") + + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere). + + All output on Windows is unbuffered so this is a nop""" + pass + + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + if not FlushConsoleInputBuffer(InHandle): + raise WinError(GetLastError()) + + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + return Event("key", "", b"") + + def wait(self, timeout: float | None) -> bool: + """Wait for an event.""" + # Poor man's Windows select loop + start_time = time.time() + while True: + if msvcrt.kbhit(): # type: ignore[attr-defined] + return True + if timeout and time.time() - start_time > timeout / 1000: + return False + time.sleep(0.01) + + def repaint(self) -> None: + raise NotImplementedError("No repaint support") + + +# Windows interop +class CONSOLE_SCREEN_BUFFER_INFO(Structure): + _fields_ = [ + ("dwSize", _COORD), + ("dwCursorPosition", _COORD), + ("wAttributes", WORD), + ("srWindow", SMALL_RECT), + ("dwMaximumWindowSize", _COORD), + ] + + +class CONSOLE_CURSOR_INFO(Structure): + _fields_ = [ + ("dwSize", DWORD), + ("bVisible", BOOL), + ] + + +class CHAR_INFO(Structure): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Attributes", WORD), + ] + + +class Char(Union): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Char", CHAR), + ] + + +class KeyEvent(ctypes.Structure): + _fields_ = [ + ("bKeyDown", BOOL), + ("wRepeatCount", WORD), + ("wVirtualKeyCode", WORD), + ("wVirtualScanCode", WORD), + ("uChar", Char), + ("dwControlKeyState", DWORD), + ] + + +class WindowsBufferSizeEvent(ctypes.Structure): + _fields_ = [("dwSize", _COORD)] + + +class ConsoleEvent(ctypes.Union): + _fields_ = [ + ("KeyEvent", KeyEvent), + ("WindowsBufferSizeEvent", WindowsBufferSizeEvent), + ] + + +class INPUT_RECORD(Structure): + _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)] + + +KEY_EVENT = 0x01 +FOCUS_EVENT = 0x10 +MENU_EVENT = 0x08 +MOUSE_EVENT = 0x02 +WINDOW_BUFFER_SIZE_EVENT = 0x04 + +ENABLE_PROCESSED_OUTPUT = 0x01 +ENABLE_WRAP_AT_EOL_OUTPUT = 0x02 +ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 + +STD_INPUT_HANDLE = -10 +STD_OUTPUT_HANDLE = -11 + +if sys.platform == "win32": + _KERNEL32 = WinDLL("kernel32", use_last_error=True) + + GetStdHandle = windll.kernel32.GetStdHandle + GetStdHandle.argtypes = [DWORD] + GetStdHandle.restype = HANDLE + + GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo + GetConsoleScreenBufferInfo.argtypes = [ + HANDLE, + ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + GetConsoleScreenBufferInfo.restype = BOOL + + ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW + ScrollConsoleScreenBuffer.argtypes = [ + HANDLE, + POINTER(SMALL_RECT), + POINTER(SMALL_RECT), + _COORD, + POINTER(CHAR_INFO), + ] + ScrollConsoleScreenBuffer.restype = BOOL + + SetConsoleMode = _KERNEL32.SetConsoleMode + SetConsoleMode.argtypes = [HANDLE, DWORD] + SetConsoleMode.restype = BOOL + + ReadConsoleInput = _KERNEL32.ReadConsoleInputW + ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] + ReadConsoleInput.restype = BOOL + + GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents + GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] + GetNumberOfConsoleInputEvents.restype = BOOL + + FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer + FlushConsoleInputBuffer.argtypes = [HANDLE] + FlushConsoleInputBuffer.restype = BOOL + + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) + InHandle = GetStdHandle(STD_INPUT_HANDLE) +else: + + def _win_only(*args, **kwargs): + raise NotImplementedError("Windows only") + + GetStdHandle = _win_only + GetConsoleScreenBufferInfo = _win_only + ScrollConsoleScreenBuffer = _win_only + SetConsoleMode = _win_only + ReadConsoleInput = _win_only + GetNumberOfConsoleInputEvents = _win_only + FlushConsoleInputBuffer = _win_only + OutHandle = 0 + InHandle = 0 diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index 3e07ead16eb..c66269a5719 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -10,7 +10,6 @@ import sys - class Quitter(object): def __init__(self, name, eof): self.name = name @@ -48,7 +47,7 @@ def __setup(self): data = None for filename in self.__filenames: try: - with open(filename, "r") as fp: + with open(filename, encoding='utf-8') as fp: data = fp.read() break except OSError: diff --git a/Lib/_strptime.py b/Lib/_strptime.py new file mode 100644 index 00000000000..fc7e369c3d1 --- /dev/null +++ b/Lib/_strptime.py @@ -0,0 +1,822 @@ +"""Strptime-related classes and functions. + +CLASSES: + LocaleTime -- Discovers and stores locale-specific time information + TimeRE -- Creates regexes for pattern matching a string of text containing + time information + +FUNCTIONS: + _getlang -- Figure out what language is being used for the locale + strptime -- Calculates the time struct represented by the passed-in string + +""" +import os +import time +import locale +import calendar +import re +from re import compile as re_compile +from re import sub as re_sub +from re import IGNORECASE +from re import escape as re_escape +from datetime import (date as datetime_date, + timedelta as datetime_timedelta, + timezone as datetime_timezone) +from _thread import allocate_lock as _thread_allocate_lock + +__all__ = [] + +def _getlang(): + # Figure out what the current language is set to. + return locale.getlocale(locale.LC_TIME) + +def _findall(haystack, needle): + # Find all positions of needle in haystack. + if not needle: + return + i = 0 + while True: + i = haystack.find(needle, i) + if i < 0: + break + yield i + i += len(needle) + +def _fixmonths(months): + yield from months + # The lower case of 'İ' ('\u0130') is 'i\u0307'. + # The re module only supports 1-to-1 character matching in + # case-insensitive mode. + for s in months: + if 'i\u0307' in s: + yield s.replace('i\u0307', '\u0130') + +lzh_TW_alt_digits = ( + # 〇:一:二:三:四:五:六:七:八:九 + '\u3007', '\u4e00', '\u4e8c', '\u4e09', '\u56db', + '\u4e94', '\u516d', '\u4e03', '\u516b', '\u4e5d', + # 十:十一:十二:十三:十四:十五:十六:十七:十八:十九 + '\u5341', '\u5341\u4e00', '\u5341\u4e8c', '\u5341\u4e09', '\u5341\u56db', + '\u5341\u4e94', '\u5341\u516d', '\u5341\u4e03', '\u5341\u516b', '\u5341\u4e5d', + # 廿:廿一:廿二:廿三:廿四:廿五:廿六:廿七:廿八:廿九 + '\u5eff', '\u5eff\u4e00', '\u5eff\u4e8c', '\u5eff\u4e09', '\u5eff\u56db', + '\u5eff\u4e94', '\u5eff\u516d', '\u5eff\u4e03', '\u5eff\u516b', '\u5eff\u4e5d', + # 卅:卅一 + '\u5345', '\u5345\u4e00') + + +class LocaleTime(object): + """Stores and handles locale-specific information related to time. + + ATTRIBUTES: + f_weekday -- full weekday names (7-item list) + a_weekday -- abbreviated weekday names (7-item list) + f_month -- full month names (13-item list; dummy value in [0], which + is added by code) + a_month -- abbreviated month names (13-item list, dummy value in + [0], which is added by code) + am_pm -- AM/PM representation (2-item list) + LC_date_time -- format string for date/time representation (string) + LC_date -- format string for date representation (string) + LC_time -- format string for time representation (string) + timezone -- daylight- and non-daylight-savings timezone representation + (2-item list of sets) + lang -- Language used by instance (2-item tuple) + """ + + def __init__(self): + """Set all attributes. + + Order of methods called matters for dependency reasons. + + The locale language is set at the offset and then checked again before + exiting. This is to make sure that the attributes were not set with a + mix of information from more than one locale. This would most likely + happen when using threads where one thread calls a locale-dependent + function while another thread changes the locale while the function in + the other thread is still running. Proper coding would call for + locks to prevent changing the locale while locale-dependent code is + running. The check here is done in case someone does not think about + doing this. + + Only other possible issue is if someone changed the timezone and did + not call tz.tzset . That is an issue for the programmer, though, + since changing the timezone is worthless without that call. + + """ + self.lang = _getlang() + self.__calc_weekday() + self.__calc_month() + self.__calc_am_pm() + self.__calc_alt_digits() + self.__calc_timezone() + self.__calc_date_time() + if _getlang() != self.lang: + raise ValueError("locale changed during initialization") + if time.tzname != self.tzname or time.daylight != self.daylight: + raise ValueError("timezone changed during initialization") + + def __calc_weekday(self): + # Set self.a_weekday and self.f_weekday using the calendar + # module. + a_weekday = [calendar.day_abbr[i].lower() for i in range(7)] + f_weekday = [calendar.day_name[i].lower() for i in range(7)] + self.a_weekday = a_weekday + self.f_weekday = f_weekday + + def __calc_month(self): + # Set self.f_month and self.a_month using the calendar module. + a_month = [calendar.month_abbr[i].lower() for i in range(13)] + f_month = [calendar.month_name[i].lower() for i in range(13)] + self.a_month = a_month + self.f_month = f_month + + def __calc_am_pm(self): + # Set self.am_pm by using time.strftime(). + + # The magic date (1999,3,17,hour,44,55,2,76,0) is not really that + # magical; just happened to have used it everywhere else where a + # static date was needed. + am_pm = [] + for hour in (1, 22): + time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0)) + # br_FR has AM/PM info (' ',' '). + am_pm.append(time.strftime("%p", time_tuple).lower().strip()) + self.am_pm = am_pm + + def __calc_alt_digits(self): + # Set self.LC_alt_digits by using time.strftime(). + + # The magic data should contain all decimal digits. + time_tuple = time.struct_time((1998, 1, 27, 10, 43, 56, 1, 27, 0)) + s = time.strftime("%x%X", time_tuple) + if s.isascii(): + # Fast path -- all digits are ASCII. + self.LC_alt_digits = () + return + + digits = ''.join(sorted(set(re.findall(r'\d', s)))) + if len(digits) == 10 and ord(digits[-1]) == ord(digits[0]) + 9: + # All 10 decimal digits from the same set. + if digits.isascii(): + # All digits are ASCII. + self.LC_alt_digits = () + return + + self.LC_alt_digits = [a + b for a in digits for b in digits] + # Test whether the numbers contain leading zero. + time_tuple2 = time.struct_time((2000, 1, 1, 1, 1, 1, 5, 1, 0)) + if self.LC_alt_digits[1] not in time.strftime("%x %X", time_tuple2): + self.LC_alt_digits[:10] = digits + return + + # Either non-Gregorian calendar or non-decimal numbers. + if {'\u4e00', '\u4e03', '\u4e5d', '\u5341', '\u5eff'}.issubset(s): + # lzh_TW + self.LC_alt_digits = lzh_TW_alt_digits + return + + self.LC_alt_digits = None + + def __calc_date_time(self): + # Set self.LC_date_time, self.LC_date, self.LC_time and + # self.LC_time_ampm by using time.strftime(). + + # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of + # overloaded numbers is minimized. The order in which searches for + # values within the format string is very important; it eliminates + # possible ambiguity for what something represents. + time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) + time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0)) + replacement_pairs = [] + + # Non-ASCII digits + if self.LC_alt_digits or self.LC_alt_digits is None: + for n, d in [(19, '%OC'), (99, '%Oy'), (22, '%OH'), + (44, '%OM'), (55, '%OS'), (17, '%Od'), + (3, '%Om'), (2, '%Ow'), (10, '%OI')]: + if self.LC_alt_digits is None: + s = chr(0x660 + n // 10) + chr(0x660 + n % 10) + replacement_pairs.append((s, d)) + if n < 10: + replacement_pairs.append((s[1], d)) + elif len(self.LC_alt_digits) > n: + replacement_pairs.append((self.LC_alt_digits[n], d)) + else: + replacement_pairs.append((time.strftime(d, time_tuple), d)) + replacement_pairs += [ + ('1999', '%Y'), ('99', '%y'), ('22', '%H'), + ('44', '%M'), ('55', '%S'), ('76', '%j'), + ('17', '%d'), ('03', '%m'), ('3', '%m'), + # '3' needed for when no leading zero. + ('2', '%w'), ('10', '%I'), + ] + + date_time = [] + for directive in ('%c', '%x', '%X', '%r'): + current_format = time.strftime(directive, time_tuple).lower() + current_format = current_format.replace('%', '%%') + # The month and the day of the week formats are treated specially + # because of a possible ambiguity in some locales where the full + # and abbreviated names are equal or names of different types + # are equal. See doc of __find_month_format for more details. + lst, fmt = self.__find_weekday_format(directive) + if lst: + current_format = current_format.replace(lst[2], fmt, 1) + lst, fmt = self.__find_month_format(directive) + if lst: + current_format = current_format.replace(lst[3], fmt, 1) + if self.am_pm[1]: + # Must deal with possible lack of locale info + # manifesting itself as the empty string (e.g., Swedish's + # lack of AM/PM info) or a platform returning a tuple of empty + # strings (e.g., MacOS 9 having timezone as ('','')). + current_format = current_format.replace(self.am_pm[1], '%p') + for tz_values in self.timezone: + for tz in tz_values: + if tz: + current_format = current_format.replace(tz, "%Z") + # Transform all non-ASCII digits to digits in range U+0660 to U+0669. + if not current_format.isascii() and self.LC_alt_digits is None: + current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", + 'f': r"(?P[0-9]{1,6})", + 'H': r"(?P2[0-3]|[0-1]\d|\d| \d)", + 'k': r"(?P2[0-3]|[0-1]\d|\d| \d)", + 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", + 'l': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", + 'G': r"(?P\d\d\d\d)", + 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", + 'm': r"(?P1[0-2]|0[1-9]|[1-9])", + 'M': r"(?P[0-5]\d|\d)", + 'S': r"(?P6[0-1]|[0-5]\d|\d)", + 'U': r"(?P5[0-3]|[0-4]\d|\d)", + 'w': r"(?P[0-6])", + 'u': r"(?P[1-7])", + 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", + # W is set below by using 'U' + 'y': r"(?P\d\d)", + 'Y': r"(?P\d\d\d\d)", + 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", + 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), + 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), + 'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'), + 'b': self.__seqToRE(_fixmonths(self.locale_time.a_month[1:]), 'b'), + 'p': self.__seqToRE(self.locale_time.am_pm, 'p'), + 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone + for tz in tz_names), + 'Z'), + '%': '%'} + if self.locale_time.LC_alt_digits is None: + for d in 'dmyCHIMS': + mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d + mapping['Ow'] = r'(?P\d)' + else: + mapping.update({ + 'Od': self.__seqToRE(self.locale_time.LC_alt_digits[1:32], 'd', + '3[0-1]|[1-2][0-9]|0[1-9]|[1-9]'), + 'Om': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'm', + '1[0-2]|0[1-9]|[1-9]'), + 'Ow': self.__seqToRE(self.locale_time.LC_alt_digits[:7], 'w', + '[0-6]'), + 'Oy': self.__seqToRE(self.locale_time.LC_alt_digits, 'y', + '[0-9][0-9]'), + 'OC': self.__seqToRE(self.locale_time.LC_alt_digits, 'C', + '[0-9][0-9]'), + 'OH': self.__seqToRE(self.locale_time.LC_alt_digits[:24], 'H', + '2[0-3]|[0-1][0-9]|[0-9]'), + 'OI': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'I', + '1[0-2]|0[1-9]|[1-9]'), + 'OM': self.__seqToRE(self.locale_time.LC_alt_digits[:60], 'M', + '[0-5][0-9]|[0-9]'), + 'OS': self.__seqToRE(self.locale_time.LC_alt_digits[:62], 'S', + '6[0-1]|[0-5][0-9]|[0-9]'), + }) + mapping.update({ + 'e': mapping['d'], + 'Oe': mapping['Od'], + 'P': mapping['p'], + 'Op': mapping['p'], + 'W': mapping['U'].replace('U', 'W'), + }) + mapping['W'] = mapping['U'].replace('U', 'W') + + base.__init__(mapping) + base.__setitem__('T', self.pattern('%H:%M:%S')) + base.__setitem__('R', self.pattern('%H:%M')) + base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm)) + base.__setitem__('X', self.pattern(self.locale_time.LC_time)) + base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) + + def __seqToRE(self, to_convert, directive, altregex=None): + """Convert a list to a regex string for matching a directive. + + Want possible matching values to be from longest to shortest. This + prevents the possibility of a match occurring for a value that also + a substring of a larger value that should have matched (e.g., 'abc' + matching when 'abcdef' should have been the match). + + """ + to_convert = sorted(to_convert, key=len, reverse=True) + for value in to_convert: + if value != '': + break + else: + return '' + regex = '|'.join(re_escape(stuff) for stuff in to_convert) + if altregex is not None: + regex += '|' + altregex + return '(?P<%s>%s)' % (directive, regex) + + def pattern(self, format): + """Return regex pattern for the format string. + + Need to make sure that any characters that might be interpreted as + regex syntax are escaped. + + """ + # The sub() call escapes all characters that might be misconstrued + # as regex syntax. Cannot use re.escape since we have to deal with + # format directives (%m, etc.). + format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format) + format = re_sub(r'\s+', r'\\s+', format) + format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR + year_in_format = False + day_of_month_in_format = False + def repl(m): + format_char = m[1] + match format_char: + case 'Y' | 'y' | 'G': + nonlocal year_in_format + year_in_format = True + case 'd': + nonlocal day_of_month_in_format + day_of_month_in_format = True + return self[format_char] + format = re_sub(r'%[-_0^#]*[0-9]*([OE]?\\?.?)', repl, format) + if day_of_month_in_format and not year_in_format: + import warnings + warnings.warn("""\ +Parsing dates involving a day of month without a year specified is ambiguous +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647.""", + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return format + + def compile(self, format): + """Return a compiled re object for the format string.""" + return re_compile(self.pattern(format), IGNORECASE) + +_cache_lock = _thread_allocate_lock() +# DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock +# first! +_TimeRE_cache = TimeRE() +_CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache +_regex_cache = {} + +def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): + """Calculate the Julian day based on the year, week of the year, and day of + the week, with week_start_day representing whether the week of the year + assumes the week starts on Sunday or Monday (6 or 0).""" + first_weekday = datetime_date(year, 1, 1).weekday() + # If we are dealing with the %U directive (week starts on Sunday), it's + # easier to just shift the view to Sunday being the first day of the + # week. + if not week_starts_Mon: + first_weekday = (first_weekday + 1) % 7 + day_of_week = (day_of_week + 1) % 7 + # Need to watch out for a week 0 (when the first day of the year is not + # the same as that specified by %U or %W). + week_0_length = (7 - first_weekday) % 7 + if week_of_year == 0: + return 1 + day_of_week - first_weekday + else: + days_to_week = week_0_length + (7 * (week_of_year - 1)) + return 1 + days_to_week + day_of_week + + +def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a 2-tuple consisting of a time struct and an int containing + the number of microseconds based on the input string and the + format string.""" + + for index, arg in enumerate([data_string, format]): + if not isinstance(arg, str): + msg = "strptime() argument {} must be str, not {}" + raise TypeError(msg.format(index, type(arg))) + + global _TimeRE_cache, _regex_cache + with _cache_lock: + locale_time = _TimeRE_cache.locale_time + if (_getlang() != locale_time.lang or + time.tzname != locale_time.tzname or + time.daylight != locale_time.daylight): + _TimeRE_cache = TimeRE() + _regex_cache.clear() + locale_time = _TimeRE_cache.locale_time + if len(_regex_cache) > _CACHE_MAX_SIZE: + _regex_cache.clear() + format_regex = _regex_cache.get(format) + if not format_regex: + try: + format_regex = _TimeRE_cache.compile(format) + # KeyError raised when a bad format is found; can be specified as + # \\, in which case it was a stray % but with a space after it + except KeyError as err: + bad_directive = err.args[0] + del err + bad_directive = bad_directive.replace('\\s', '') + if not bad_directive: + raise ValueError("stray %% in format '%s'" % format) from None + bad_directive = bad_directive.replace('\\', '', 1) + raise ValueError("'%s' is a bad directive in format '%s'" % + (bad_directive, format)) from None + _regex_cache[format] = format_regex + found = format_regex.match(data_string) + if not found: + raise ValueError("time data %r does not match format %r" % + (data_string, format)) + if len(data_string) != found.end(): + raise ValueError("unconverted data remains: %s" % + data_string[found.end():]) + + iso_year = year = None + month = day = 1 + hour = minute = second = fraction = 0 + tz = -1 + gmtoff = None + gmtoff_fraction = 0 + iso_week = week_of_year = None + week_of_year_start = None + # weekday and julian defaulted to None so as to signal need to calculate + # values + weekday = julian = None + found_dict = found.groupdict() + if locale_time.LC_alt_digits: + def parse_int(s): + try: + return locale_time.LC_alt_digits.index(s) + except ValueError: + return int(s) + else: + parse_int = int + + for group_key in found_dict.keys(): + # Directives not explicitly handled below: + # c, x, X + # handled by making out of other directives + # U, W + # worthless without day of the week + if group_key == 'y': + year = parse_int(found_dict['y']) + if 'C' in found_dict: + century = parse_int(found_dict['C']) + year += century * 100 + else: + # Open Group specification for strptime() states that a %y + #value in the range of [00, 68] is in the century 2000, while + #[69,99] is in the century 1900 + if year <= 68: + year += 2000 + else: + year += 1900 + elif group_key == 'Y': + year = int(found_dict['Y']) + elif group_key == 'G': + iso_year = int(found_dict['G']) + elif group_key == 'm': + month = parse_int(found_dict['m']) + elif group_key == 'B': + month = locale_time.f_month.index(found_dict['B'].lower()) + elif group_key == 'b': + month = locale_time.a_month.index(found_dict['b'].lower()) + elif group_key == 'd': + day = parse_int(found_dict['d']) + elif group_key == 'H': + hour = parse_int(found_dict['H']) + elif group_key == 'I': + hour = parse_int(found_dict['I']) + ampm = found_dict.get('p', '').lower() + # If there was no AM/PM indicator, we'll treat this like AM + if ampm in ('', locale_time.am_pm[0]): + # We're in AM so the hour is correct unless we're + # looking at 12 midnight. + # 12 midnight == 12 AM == hour 0 + if hour == 12: + hour = 0 + elif ampm == locale_time.am_pm[1]: + # We're in PM so we need to add 12 to the hour unless + # we're looking at 12 noon. + # 12 noon == 12 PM == hour 12 + if hour != 12: + hour += 12 + elif group_key == 'M': + minute = parse_int(found_dict['M']) + elif group_key == 'S': + second = parse_int(found_dict['S']) + elif group_key == 'f': + s = found_dict['f'] + # Pad to always return microseconds. + s += "0" * (6 - len(s)) + fraction = int(s) + elif group_key == 'A': + weekday = locale_time.f_weekday.index(found_dict['A'].lower()) + elif group_key == 'a': + weekday = locale_time.a_weekday.index(found_dict['a'].lower()) + elif group_key == 'w': + weekday = int(found_dict['w']) + if weekday == 0: + weekday = 6 + else: + weekday -= 1 + elif group_key == 'u': + weekday = int(found_dict['u']) + weekday -= 1 + elif group_key == 'j': + julian = int(found_dict['j']) + elif group_key in ('U', 'W'): + week_of_year = int(found_dict[group_key]) + if group_key == 'U': + # U starts week on Sunday. + week_of_year_start = 6 + else: + # W starts week on Monday. + week_of_year_start = 0 + elif group_key == 'V': + iso_week = int(found_dict['V']) + elif group_key == 'z': + z = found_dict['z'] + if z == 'Z': + gmtoff = 0 + else: + if z[3] == ':': + z = z[:3] + z[4:] + if len(z) > 5: + if z[5] != ':': + msg = f"Inconsistent use of : in {found_dict['z']}" + raise ValueError(msg) + z = z[:5] + z[6:] + hours = int(z[1:3]) + minutes = int(z[3:5]) + seconds = int(z[5:7] or 0) + gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds + gmtoff_remainder = z[8:] + # Pad to always return microseconds. + gmtoff_remainder_padding = "0" * (6 - len(gmtoff_remainder)) + gmtoff_fraction = int(gmtoff_remainder + gmtoff_remainder_padding) + if z.startswith("-"): + gmtoff = -gmtoff + gmtoff_fraction = -gmtoff_fraction + elif group_key == 'Z': + # Since -1 is default value only need to worry about setting tz if + # it can be something other than -1. + found_zone = found_dict['Z'].lower() + for value, tz_values in enumerate(locale_time.timezone): + if found_zone in tz_values: + # Deal with bad locale setup where timezone names are the + # same and yet time.daylight is true; too ambiguous to + # be able to tell what timezone has daylight savings + if (time.tzname[0] == time.tzname[1] and + time.daylight and found_zone not in ("utc", "gmt")): + break + else: + tz = value + break + + # Deal with the cases where ambiguities arise + # don't assume default values for ISO week/year + if iso_year is not None: + if julian is not None: + raise ValueError("Day of the year directive '%j' is not " + "compatible with ISO year directive '%G'. " + "Use '%Y' instead.") + elif iso_week is None or weekday is None: + raise ValueError("ISO year directive '%G' must be used with " + "the ISO week directive '%V' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + elif iso_week is not None: + if year is None or weekday is None: + raise ValueError("ISO week directive '%V' must be used with " + "the ISO year directive '%G' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + else: + raise ValueError("ISO week directive '%V' is incompatible with " + "the year directive '%Y'. Use the ISO year '%G' " + "instead.") + + leap_year_fix = False + if year is None: + if month == 2 and day == 29: + year = 1904 # 1904 is first leap year of 20th century + leap_year_fix = True + else: + year = 1900 + + # If we know the week of the year and what day of that week, we can figure + # out the Julian day of the year. + if julian is None and weekday is not None: + if week_of_year is not None: + week_starts_Mon = True if week_of_year_start == 0 else False + julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, + week_starts_Mon) + elif iso_year is not None and iso_week is not None: + datetime_result = datetime_date.fromisocalendar(iso_year, iso_week, weekday + 1) + year = datetime_result.year + month = datetime_result.month + day = datetime_result.day + if julian is not None and julian <= 0: + year -= 1 + yday = 366 if calendar.isleap(year) else 365 + julian += yday + + if julian is None: + # Cannot pre-calculate datetime_date() since can change in Julian + # calculation and thus could have different value for the day of + # the week calculation. + # Need to add 1 to result since first day of the year is 1, not 0. + julian = datetime_date(year, month, day).toordinal() - \ + datetime_date(year, 1, 1).toordinal() + 1 + else: # Assume that if they bothered to include Julian day (or if it was + # calculated above with year/week/weekday) it will be accurate. + datetime_result = datetime_date.fromordinal( + (julian - 1) + + datetime_date(year, 1, 1).toordinal()) + year = datetime_result.year + month = datetime_result.month + day = datetime_result.day + if weekday is None: + weekday = datetime_date(year, month, day).weekday() + # Add timezone info + tzname = found_dict.get("Z") + + if leap_year_fix: + # the caller didn't supply a year but asked for Feb 29th. We couldn't + # use the default of 1900 for computations. We set it back to ensure + # that February 29th is smaller than March 1st. + year = 1900 + + return (year, month, day, + hour, minute, second, + weekday, julian, tz, tzname, gmtoff), fraction, gmtoff_fraction + +def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a time struct based on the input string and the + format string.""" + tt = _strptime(data_string, format)[0] + return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) + +def _strptime_datetime_date(cls, data_string, format="%a %b %d %Y"): + """Return a date instance based on the input string and the + format string.""" + tt, _, _ = _strptime(data_string, format) + args = tt[:3] + return cls(*args) + +def _parse_tz(tzname, gmtoff, gmtoff_fraction): + tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction) + if tzname: + return datetime_timezone(tzdelta, tzname) + else: + return datetime_timezone(tzdelta) + +def _strptime_datetime_time(cls, data_string, format="%H:%M:%S"): + """Return a time instance based on the input string and the + format string.""" + tt, fraction, gmtoff_fraction = _strptime(data_string, format) + tzname, gmtoff = tt[-2:] + args = tt[3:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) + +def _strptime_datetime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): + """Return a datetime instance based on the input string and the + format string.""" + tt, fraction, gmtoff_fraction = _strptime(data_string, format) + tzname, gmtoff = tt[-2:] + args = tt[:6] + (fraction,) + if gmtoff is None: + return cls(*args) + else: + tz = _parse_tz(tzname, gmtoff, gmtoff_fraction) + return cls(*args, tz) diff --git a/Lib/_threading_local.py b/Lib/_threading_local.py index e5204339988..0b9e5d3bbf6 100644 --- a/Lib/_threading_local.py +++ b/Lib/_threading_local.py @@ -4,133 +4,6 @@ class. Depending on the version of Python you're using, there may be a faster one available. You should always import the `local` class from `threading`.) - -Thread-local objects support the management of thread-local data. -If you have data that you want to be local to a thread, simply create -a thread-local object and use its attributes: - - >>> mydata = local() - >>> mydata.number = 42 - >>> mydata.number - 42 - -You can also access the local-object's dictionary: - - >>> mydata.__dict__ - {'number': 42} - >>> mydata.__dict__.setdefault('widgets', []) - [] - >>> mydata.widgets - [] - -What's important about thread-local objects is that their data are -local to a thread. If we access the data in a different thread: - - >>> log = [] - >>> def f(): - ... items = sorted(mydata.__dict__.items()) - ... log.append(items) - ... mydata.number = 11 - ... log.append(mydata.number) - - >>> import threading - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[], 11] - -we get different data. Furthermore, changes made in the other thread -don't affect data seen in this thread: - - >>> mydata.number - 42 - -Of course, values you get from a local object, including a __dict__ -attribute, are for whatever thread was current at the time the -attribute was read. For that reason, you generally don't want to save -these values across threads, as they apply only to the thread they -came from. - -You can create custom local objects by subclassing the local class: - - >>> class MyLocal(local): - ... number = 2 - ... initialized = False - ... def __init__(self, **kw): - ... if self.initialized: - ... raise SystemError('__init__ called too many times') - ... self.initialized = True - ... self.__dict__.update(kw) - ... def squared(self): - ... return self.number ** 2 - -This can be useful to support default values, methods and -initialization. Note that if you define an __init__ method, it will be -called each time the local object is used in a separate thread. This -is necessary to initialize each thread's dictionary. - -Now if we create a local object: - - >>> mydata = MyLocal(color='red') - -Now we have a default number: - - >>> mydata.number - 2 - -an initial color: - - >>> mydata.color - 'red' - >>> del mydata.color - -And a method that operates on the data: - - >>> mydata.squared() - 4 - -As before, we can access the data in a separate thread: - - >>> log = [] - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - >>> log - [[('color', 'red'), ('initialized', True)], 11] - -without affecting this thread's data: - - >>> mydata.number - 2 - >>> mydata.color - Traceback (most recent call last): - ... - AttributeError: 'MyLocal' object has no attribute 'color' - -Note that subclasses can define slots, but they are not thread -local. They are shared across threads: - - >>> class MyLocal(local): - ... __slots__ = 'number' - - >>> mydata = MyLocal() - >>> mydata.number = 42 - >>> mydata.color = 'red' - -So, the separate thread: - - >>> thread = threading.Thread(target=f) - >>> thread.start() - >>> thread.join() - -affects what we see: - - >>> # TODO: RUSTPYTHON, __slots__ - >>> mydata.number #doctest: +SKIP - 11 - ->>> del mydata """ from weakref import ref @@ -194,7 +67,6 @@ def thread_deleted(_, idt=idt): @contextmanager def _patch(self): - old = object.__getattribute__(self, '__dict__') impl = object.__getattribute__(self, '_local__impl') try: dct = impl.get_dict() @@ -205,13 +77,12 @@ def _patch(self): with impl.locallock: object.__setattr__(self, '__dict__', dct) yield - object.__setattr__(self, '__dict__', old) class local: __slots__ = '_local__impl', '__dict__' - def __new__(cls, *args, **kw): + def __new__(cls, /, *args, **kw): if (args or kw) and (cls.__init__ is object.__init__): raise TypeError("Initialization arguments are not supported") self = object.__new__(cls) diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 2a27684324d..d1c7fcaeec9 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -8,69 +8,29 @@ __all__ = ['WeakSet'] -class _IterationGuard: - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - class WeakSet: def __init__(self, data=None): self.data = set() + def _remove(item, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(item) - else: - self.data.discard(item) + self.data.discard(item) + self._remove = _remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() if data is not None: self.update(data) - def _commit_removals(self): - pop = self._pending_removals.pop - discard = self.data.discard - while True: - try: - item = pop() - except IndexError: - return - discard(item) - def __iter__(self): - with _IterationGuard(self): - for itemref in self.data: - item = itemref() - if item is not None: - # Caveat: the iterator will keep a strong reference to - # `item` until it is resumed or closed. - yield item + for itemref in self.data.copy(): + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item def __len__(self): - return len(self.data) - len(self._pending_removals) + return len(self.data) def __contains__(self, item): try: @@ -80,25 +40,18 @@ def __contains__(self, item): return wr in self.data def __reduce__(self): - return (self.__class__, (list(self),), - getattr(self, '__dict__', None)) + return self.__class__, (list(self),), self.__getstate__() def add(self, item): - if self._pending_removals: - self._commit_removals() self.data.add(ref(item, self._remove)) def clear(self): - if self._pending_removals: - self._commit_removals() self.data.clear() def copy(self): return self.__class__(self) def pop(self): - if self._pending_removals: - self._commit_removals() while True: try: itemref = self.data.pop() @@ -109,18 +62,12 @@ def pop(self): return item def remove(self, item): - if self._pending_removals: - self._commit_removals() self.data.remove(ref(item)) def discard(self, item): - if self._pending_removals: - self._commit_removals() self.data.discard(ref(item)) def update(self, other): - if self._pending_removals: - self._commit_removals() for element in other: self.add(element) @@ -137,8 +84,6 @@ def difference(self, other): def difference_update(self, other): self.__isub__(other) def __isub__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: @@ -152,8 +97,6 @@ def intersection(self, other): def intersection_update(self, other): self.__iand__(other) def __iand__(self, other): - if self._pending_removals: - self._commit_removals() self.data.intersection_update(ref(item) for item in other) return self @@ -185,8 +128,6 @@ def symmetric_difference(self, other): def symmetric_difference_update(self, other): self.__ixor__(other) def __ixor__(self, other): - if self._pending_removals: - self._commit_removals() if self is other: self.data.clear() else: diff --git a/Lib/abc.py b/Lib/abc.py index 1ecff5e2146..f8a4e11ce9c 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -85,10 +85,6 @@ def my_abstract_property(self): from _abc import (get_cache_token, _abc_init, _abc_register, _abc_instancecheck, _abc_subclasscheck, _get_dump, _reset_registry, _reset_caches) -# TODO: RUSTPYTHON missing _abc module implementation. -except ModuleNotFoundError: - from _py_abc import ABCMeta, get_cache_token - ABCMeta.__module__ = 'abc' except ImportError: from _py_abc import ABCMeta, get_cache_token ABCMeta.__module__ = 'abc' diff --git a/Lib/aifc.py b/Lib/aifc.py deleted file mode 100644 index 5254987e22b..00000000000 --- a/Lib/aifc.py +++ /dev/null @@ -1,984 +0,0 @@ -"""Stuff to parse AIFF-C and AIFF files. - -Unless explicitly stated otherwise, the description below is true -both for AIFF-C files and AIFF files. - -An AIFF-C file has the following structure. - - +-----------------+ - | FORM | - +-----------------+ - | | - +----+------------+ - | | AIFC | - | +------------+ - | | | - | | . | - | | . | - | | . | - +----+------------+ - -An AIFF file has the string "AIFF" instead of "AIFC". - -A chunk consists of an identifier (4 bytes) followed by a size (4 bytes, -big endian order), followed by the data. The size field does not include -the size of the 8 byte header. - -The following chunk types are recognized. - - FVER - (AIFF-C only). - MARK - <# of markers> (2 bytes) - list of markers: - (2 bytes, must be > 0) - (4 bytes) - ("pstring") - COMM - <# of channels> (2 bytes) - <# of sound frames> (4 bytes) - (2 bytes) - (10 bytes, IEEE 80-bit extended - floating point) - in AIFF-C files only: - (4 bytes) - ("pstring") - SSND - (4 bytes, not used by this program) - (4 bytes, not used by this program) - - -A pstring consists of 1 byte length, a string of characters, and 0 or 1 -byte pad to make the total length even. - -Usage. - -Reading AIFF files: - f = aifc.open(file, 'r') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods read(), seek(), and close(). -In some types of audio files, if the setpos() method is not used, -the seek() method is not necessary. - -This returns an instance of a class with the following public methods: - getnchannels() -- returns number of audio channels (1 for - mono, 2 for stereo) - getsampwidth() -- returns sample width in bytes - getframerate() -- returns sampling frequency - getnframes() -- returns number of audio frames - getcomptype() -- returns compression type ('NONE' for AIFF files) - getcompname() -- returns human-readable version of - compression type ('not compressed' for AIFF files) - getparams() -- returns a namedtuple consisting of all of the - above in the above order - getmarkers() -- get the list of marks in the audio file or None - if there are no marks - getmark(id) -- get mark with the specified id (raises an error - if the mark does not exist) - readframes(n) -- returns at most n frames of audio - rewind() -- rewind to the beginning of the audio stream - setpos(pos) -- seek to the specified position - tell() -- return the current position - close() -- close the instance (make it unusable) -The position returned by tell(), the position given to setpos() and -the position of marks are all compatible and have nothing to do with -the actual position in the file. -The close() method is called automatically when the class instance -is destroyed. - -Writing AIFF files: - f = aifc.open(file, 'w') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods write(), tell(), seek(), and -close(). - -This returns an instance of a class with the following public methods: - aiff() -- create an AIFF file (AIFF-C default) - aifc() -- create an AIFF-C file - setnchannels(n) -- set the number of channels - setsampwidth(n) -- set the sample width - setframerate(n) -- set the frame rate - setnframes(n) -- set the number of frames - setcomptype(type, name) - -- set the compression type and the - human-readable compression type - setparams(tuple) - -- set all parameters at once - setmark(id, pos, name) - -- add specified mark to the list of marks - tell() -- return current position in output file (useful - in combination with setmark()) - writeframesraw(data) - -- write audio frames without pathing up the - file header - writeframes(data) - -- write audio frames and patch up the file header - close() -- patch up the file header and close the - output file -You should set the parameters before the first writeframesraw or -writeframes. The total number of frames does not need to be set, -but when it is set to the correct value, the header does not have to -be patched up. -It is best to first set all parameters, perhaps possibly the -compression type, and then write audio frames using writeframesraw. -When all frames have been written, either call writeframes(b'') or -close() to patch up the sizes in the header. -Marks can be added anytime. If there are any marks, you must call -close() after all frames have been written. -The close() method is called automatically when the class instance -is destroyed. - -When a file is opened with the extension '.aiff', an AIFF file is -written, otherwise an AIFF-C file is written. This default can be -changed by calling aiff() or aifc() before the first writeframes or -writeframesraw. -""" - -import struct -import builtins -import warnings - -__all__ = ["Error", "open"] - - -warnings._deprecated(__name__, remove=(3, 13)) - - -class Error(Exception): - pass - -_AIFC_version = 0xA2805140 # Version 1 of AIFF-C - -def _read_long(file): - try: - return struct.unpack('>l', file.read(4))[0] - except struct.error: - raise EOFError from None - -def _read_ulong(file): - try: - return struct.unpack('>L', file.read(4))[0] - except struct.error: - raise EOFError from None - -def _read_short(file): - try: - return struct.unpack('>h', file.read(2))[0] - except struct.error: - raise EOFError from None - -def _read_ushort(file): - try: - return struct.unpack('>H', file.read(2))[0] - except struct.error: - raise EOFError from None - -def _read_string(file): - length = ord(file.read(1)) - if length == 0: - data = b'' - else: - data = file.read(length) - if length & 1 == 0: - dummy = file.read(1) - return data - -_HUGE_VAL = 1.79769313486231e+308 # See - -def _read_float(f): # 10 bytes - expon = _read_short(f) # 2 bytes - sign = 1 - if expon < 0: - sign = -1 - expon = expon + 0x8000 - himant = _read_ulong(f) # 4 bytes - lomant = _read_ulong(f) # 4 bytes - if expon == himant == lomant == 0: - f = 0.0 - elif expon == 0x7FFF: - f = _HUGE_VAL - else: - expon = expon - 16383 - f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63) - return sign * f - -def _write_short(f, x): - f.write(struct.pack('>h', x)) - -def _write_ushort(f, x): - f.write(struct.pack('>H', x)) - -def _write_long(f, x): - f.write(struct.pack('>l', x)) - -def _write_ulong(f, x): - f.write(struct.pack('>L', x)) - -def _write_string(f, s): - if len(s) > 255: - raise ValueError("string exceeds maximum pstring length") - f.write(struct.pack('B', len(s))) - f.write(s) - if len(s) & 1 == 0: - f.write(b'\x00') - -def _write_float(f, x): - import math - if x < 0: - sign = 0x8000 - x = x * -1 - else: - sign = 0 - if x == 0: - expon = 0 - himant = 0 - lomant = 0 - else: - fmant, expon = math.frexp(x) - if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN - expon = sign|0x7FFF - himant = 0 - lomant = 0 - else: # Finite - expon = expon + 16382 - if expon < 0: # denormalized - fmant = math.ldexp(fmant, expon) - expon = 0 - expon = expon | sign - fmant = math.ldexp(fmant, 32) - fsmant = math.floor(fmant) - himant = int(fsmant) - fmant = math.ldexp(fmant - fsmant, 32) - fsmant = math.floor(fmant) - lomant = int(fsmant) - _write_ushort(f, expon) - _write_ulong(f, himant) - _write_ulong(f, lomant) - -with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - from chunk import Chunk -from collections import namedtuple - -_aifc_params = namedtuple('_aifc_params', - 'nchannels sampwidth framerate nframes comptype compname') - -_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' -_aifc_params.sampwidth.__doc__ = 'Sample width in bytes' -_aifc_params.framerate.__doc__ = 'Sampling frequency' -_aifc_params.nframes.__doc__ = 'Number of audio frames' -_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' -_aifc_params.compname.__doc__ = ("""\ -A human-readable version of the compression type -('not compressed' for AIFF files)""") - - -class Aifc_read: - # Variables used in this class: - # - # These variables are available to the user though appropriate - # methods of this class: - # _file -- the open file with methods read(), close(), and seek() - # set through the __init__() method - # _nchannels -- the number of audio channels - # available through the getnchannels() method - # _nframes -- the number of audio frames - # available through the getnframes() method - # _sampwidth -- the number of bytes per audio sample - # available through the getsampwidth() method - # _framerate -- the sampling frequency - # available through the getframerate() method - # _comptype -- the AIFF-C compression type ('NONE' if AIFF) - # available through the getcomptype() method - # _compname -- the human-readable AIFF-C compression type - # available through the getcomptype() method - # _markers -- the marks in the audio file - # available through the getmarkers() and getmark() - # methods - # _soundpos -- the position in the audio stream - # available through the tell() method, set through the - # setpos() method - # - # These variables are used internally only: - # _version -- the AIFF-C version number - # _decomp -- the decompressor from builtin module cl - # _comm_chunk_read -- 1 iff the COMM chunk has been read - # _aifc -- 1 iff reading an AIFF-C file - # _ssnd_seek_needed -- 1 iff positioned correctly in audio - # file for readframes() - # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk - # _framesize -- size of one frame in the file - - _file = None # Set here since __del__ checks it - - def initfp(self, file): - self._version = 0 - self._convert = None - self._markers = [] - self._soundpos = 0 - self._file = file - chunk = Chunk(file) - if chunk.getname() != b'FORM': - raise Error('file does not start with FORM id') - formdata = chunk.read(4) - if formdata == b'AIFF': - self._aifc = 0 - elif formdata == b'AIFC': - self._aifc = 1 - else: - raise Error('not an AIFF or AIFF-C file') - self._comm_chunk_read = 0 - self._ssnd_chunk = None - while 1: - self._ssnd_seek_needed = 1 - try: - chunk = Chunk(self._file) - except EOFError: - break - chunkname = chunk.getname() - if chunkname == b'COMM': - self._read_comm_chunk(chunk) - self._comm_chunk_read = 1 - elif chunkname == b'SSND': - self._ssnd_chunk = chunk - dummy = chunk.read(8) - self._ssnd_seek_needed = 0 - elif chunkname == b'FVER': - self._version = _read_ulong(chunk) - elif chunkname == b'MARK': - self._readmark(chunk) - chunk.skip() - if not self._comm_chunk_read or not self._ssnd_chunk: - raise Error('COMM chunk and/or SSND chunk missing') - - def __init__(self, f): - if isinstance(f, str): - file_object = builtins.open(f, 'rb') - try: - self.initfp(file_object) - except: - file_object.close() - raise - else: - # assume it is an open file object already - self.initfp(f) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - # - # User visible methods. - # - def getfp(self): - return self._file - - def rewind(self): - self._ssnd_seek_needed = 1 - self._soundpos = 0 - - def close(self): - file = self._file - if file is not None: - self._file = None - file.close() - - def tell(self): - return self._soundpos - - def getnchannels(self): - return self._nchannels - - def getnframes(self): - return self._nframes - - def getsampwidth(self): - return self._sampwidth - - def getframerate(self): - return self._framerate - - def getcomptype(self): - return self._comptype - - def getcompname(self): - return self._compname - -## def getversion(self): -## return self._version - - def getparams(self): - return _aifc_params(self.getnchannels(), self.getsampwidth(), - self.getframerate(), self.getnframes(), - self.getcomptype(), self.getcompname()) - - def getmarkers(self): - if len(self._markers) == 0: - return None - return self._markers - - def getmark(self, id): - for marker in self._markers: - if id == marker[0]: - return marker - raise Error('marker {0!r} does not exist'.format(id)) - - def setpos(self, pos): - if pos < 0 or pos > self._nframes: - raise Error('position not in range') - self._soundpos = pos - self._ssnd_seek_needed = 1 - - def readframes(self, nframes): - if self._ssnd_seek_needed: - self._ssnd_chunk.seek(0) - dummy = self._ssnd_chunk.read(8) - pos = self._soundpos * self._framesize - if pos: - self._ssnd_chunk.seek(pos + 8) - self._ssnd_seek_needed = 0 - if nframes == 0: - return b'' - data = self._ssnd_chunk.read(nframes * self._framesize) - if self._convert and data: - data = self._convert(data) - self._soundpos = self._soundpos + len(data) // (self._nchannels - * self._sampwidth) - return data - - # - # Internal methods. - # - - def _alaw2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.alaw2lin(data, 2) - - def _ulaw2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.ulaw2lin(data, 2) - - def _adpcm2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - if not hasattr(self, '_adpcmstate'): - # first time - self._adpcmstate = None - data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate) - return data - - def _sowt2lin(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.byteswap(data, 2) - - def _read_comm_chunk(self, chunk): - self._nchannels = _read_short(chunk) - self._nframes = _read_long(chunk) - self._sampwidth = (_read_short(chunk) + 7) // 8 - self._framerate = int(_read_float(chunk)) - if self._sampwidth <= 0: - raise Error('bad sample width') - if self._nchannels <= 0: - raise Error('bad # of channels') - self._framesize = self._nchannels * self._sampwidth - if self._aifc: - #DEBUG: SGI's soundeditor produces a bad size :-( - kludge = 0 - if chunk.chunksize == 18: - kludge = 1 - warnings.warn('Warning: bad COMM chunk size') - chunk.chunksize = 23 - #DEBUG end - self._comptype = chunk.read(4) - #DEBUG start - if kludge: - length = ord(chunk.file.read(1)) - if length & 1 == 0: - length = length + 1 - chunk.chunksize = chunk.chunksize + length - chunk.file.seek(-1, 1) - #DEBUG end - self._compname = _read_string(chunk) - if self._comptype != b'NONE': - if self._comptype == b'G722': - self._convert = self._adpcm2lin - elif self._comptype in (b'ulaw', b'ULAW'): - self._convert = self._ulaw2lin - elif self._comptype in (b'alaw', b'ALAW'): - self._convert = self._alaw2lin - elif self._comptype in (b'sowt', b'SOWT'): - self._convert = self._sowt2lin - else: - raise Error('unsupported compression type') - self._sampwidth = 2 - else: - self._comptype = b'NONE' - self._compname = b'not compressed' - - def _readmark(self, chunk): - nmarkers = _read_short(chunk) - # Some files appear to contain invalid counts. - # Cope with this by testing for EOF. - try: - for i in range(nmarkers): - id = _read_short(chunk) - pos = _read_long(chunk) - name = _read_string(chunk) - if pos or name: - # some files appear to have - # dummy markers consisting of - # a position 0 and name '' - self._markers.append((id, pos, name)) - except EOFError: - w = ('Warning: MARK chunk contains only %s marker%s instead of %s' % - (len(self._markers), '' if len(self._markers) == 1 else 's', - nmarkers)) - warnings.warn(w) - -class Aifc_write: - # Variables used in this class: - # - # These variables are user settable through appropriate methods - # of this class: - # _file -- the open file with methods write(), close(), tell(), seek() - # set through the __init__() method - # _comptype -- the AIFF-C compression type ('NONE' in AIFF) - # set through the setcomptype() or setparams() method - # _compname -- the human-readable AIFF-C compression type - # set through the setcomptype() or setparams() method - # _nchannels -- the number of audio channels - # set through the setnchannels() or setparams() method - # _sampwidth -- the number of bytes per audio sample - # set through the setsampwidth() or setparams() method - # _framerate -- the sampling frequency - # set through the setframerate() or setparams() method - # _nframes -- the number of audio frames written to the header - # set through the setnframes() or setparams() method - # _aifc -- whether we're writing an AIFF-C file or an AIFF file - # set through the aifc() method, reset through the - # aiff() method - # - # These variables are used internally only: - # _version -- the AIFF-C version number - # _comp -- the compressor from builtin module cl - # _nframeswritten -- the number of audio frames actually written - # _datalength -- the size of the audio samples written to the header - # _datawritten -- the size of the audio samples actually written - - _file = None # Set here since __del__ checks it - - def __init__(self, f): - if isinstance(f, str): - file_object = builtins.open(f, 'wb') - try: - self.initfp(file_object) - except: - file_object.close() - raise - - # treat .aiff file extensions as non-compressed audio - if f.endswith('.aiff'): - self._aifc = 0 - else: - # assume it is an open file object already - self.initfp(f) - - def initfp(self, file): - self._file = file - self._version = _AIFC_version - self._comptype = b'NONE' - self._compname = b'not compressed' - self._convert = None - self._nchannels = 0 - self._sampwidth = 0 - self._framerate = 0 - self._nframes = 0 - self._nframeswritten = 0 - self._datawritten = 0 - self._datalength = 0 - self._markers = [] - self._marklength = 0 - self._aifc = 1 # AIFF-C is default - - def __del__(self): - self.close() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - # - # User visible methods. - # - def aiff(self): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._aifc = 0 - - def aifc(self): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._aifc = 1 - - def setnchannels(self, nchannels): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if nchannels < 1: - raise Error('bad # of channels') - self._nchannels = nchannels - - def getnchannels(self): - if not self._nchannels: - raise Error('number of channels not set') - return self._nchannels - - def setsampwidth(self, sampwidth): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if sampwidth < 1 or sampwidth > 4: - raise Error('bad sample width') - self._sampwidth = sampwidth - - def getsampwidth(self): - if not self._sampwidth: - raise Error('sample width not set') - return self._sampwidth - - def setframerate(self, framerate): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if framerate <= 0: - raise Error('bad frame rate') - self._framerate = framerate - - def getframerate(self): - if not self._framerate: - raise Error('frame rate not set') - return self._framerate - - def setnframes(self, nframes): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._nframes = nframes - - def getnframes(self): - return self._nframeswritten - - def setcomptype(self, comptype, compname): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if comptype not in (b'NONE', b'ulaw', b'ULAW', - b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'): - raise Error('unsupported compression type') - self._comptype = comptype - self._compname = compname - - def getcomptype(self): - return self._comptype - - def getcompname(self): - return self._compname - -## def setversion(self, version): -## if self._nframeswritten: -## raise Error, 'cannot change parameters after starting to write' -## self._version = version - - def setparams(self, params): - nchannels, sampwidth, framerate, nframes, comptype, compname = params - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if comptype not in (b'NONE', b'ulaw', b'ULAW', - b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'): - raise Error('unsupported compression type') - self.setnchannels(nchannels) - self.setsampwidth(sampwidth) - self.setframerate(framerate) - self.setnframes(nframes) - self.setcomptype(comptype, compname) - - def getparams(self): - if not self._nchannels or not self._sampwidth or not self._framerate: - raise Error('not all parameters set') - return _aifc_params(self._nchannels, self._sampwidth, self._framerate, - self._nframes, self._comptype, self._compname) - - def setmark(self, id, pos, name): - if id <= 0: - raise Error('marker ID must be > 0') - if pos < 0: - raise Error('marker position must be >= 0') - if not isinstance(name, bytes): - raise Error('marker name must be bytes') - for i in range(len(self._markers)): - if id == self._markers[i][0]: - self._markers[i] = id, pos, name - return - self._markers.append((id, pos, name)) - - def getmark(self, id): - for marker in self._markers: - if id == marker[0]: - return marker - raise Error('marker {0!r} does not exist'.format(id)) - - def getmarkers(self): - if len(self._markers) == 0: - return None - return self._markers - - def tell(self): - return self._nframeswritten - - def writeframesraw(self, data): - if not isinstance(data, (bytes, bytearray)): - data = memoryview(data).cast('B') - self._ensure_header_written(len(data)) - nframes = len(data) // (self._sampwidth * self._nchannels) - if self._convert: - data = self._convert(data) - self._file.write(data) - self._nframeswritten = self._nframeswritten + nframes - self._datawritten = self._datawritten + len(data) - - def writeframes(self, data): - self.writeframesraw(data) - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - - def close(self): - if self._file is None: - return - try: - self._ensure_header_written(0) - if self._datawritten & 1: - # quick pad to even size - self._file.write(b'\x00') - self._datawritten = self._datawritten + 1 - self._writemarkers() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten or \ - self._marklength: - self._patchheader() - finally: - # Prevent ref cycles - self._convert = None - f = self._file - self._file = None - f.close() - - # - # Internal methods. - # - - def _lin2alaw(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.lin2alaw(data, 2) - - def _lin2ulaw(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.lin2ulaw(data, 2) - - def _lin2adpcm(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - if not hasattr(self, '_adpcmstate'): - self._adpcmstate = None - data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate) - return data - - def _lin2sowt(self, data): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - import audioop - return audioop.byteswap(data, 2) - - def _ensure_header_written(self, datasize): - if not self._nframeswritten: - if self._comptype in (b'ULAW', b'ulaw', - b'ALAW', b'alaw', b'G722', - b'sowt', b'SOWT'): - if not self._sampwidth: - self._sampwidth = 2 - if self._sampwidth != 2: - raise Error('sample width must be 2 when compressing ' - 'with ulaw/ULAW, alaw/ALAW, sowt/SOWT ' - 'or G7.22 (ADPCM)') - if not self._nchannels: - raise Error('# channels not specified') - if not self._sampwidth: - raise Error('sample width not specified') - if not self._framerate: - raise Error('sampling rate not specified') - self._write_header(datasize) - - def _init_compression(self): - if self._comptype == b'G722': - self._convert = self._lin2adpcm - elif self._comptype in (b'ulaw', b'ULAW'): - self._convert = self._lin2ulaw - elif self._comptype in (b'alaw', b'ALAW'): - self._convert = self._lin2alaw - elif self._comptype in (b'sowt', b'SOWT'): - self._convert = self._lin2sowt - - def _write_header(self, initlength): - if self._aifc and self._comptype != b'NONE': - self._init_compression() - self._file.write(b'FORM') - if not self._nframes: - self._nframes = initlength // (self._nchannels * self._sampwidth) - self._datalength = self._nframes * self._nchannels * self._sampwidth - if self._datalength & 1: - self._datalength = self._datalength + 1 - if self._aifc: - if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'): - self._datalength = self._datalength // 2 - if self._datalength & 1: - self._datalength = self._datalength + 1 - elif self._comptype == b'G722': - self._datalength = (self._datalength + 3) // 4 - if self._datalength & 1: - self._datalength = self._datalength + 1 - try: - self._form_length_pos = self._file.tell() - except (AttributeError, OSError): - self._form_length_pos = None - commlength = self._write_form_length(self._datalength) - if self._aifc: - self._file.write(b'AIFC') - self._file.write(b'FVER') - _write_ulong(self._file, 4) - _write_ulong(self._file, self._version) - else: - self._file.write(b'AIFF') - self._file.write(b'COMM') - _write_ulong(self._file, commlength) - _write_short(self._file, self._nchannels) - if self._form_length_pos is not None: - self._nframes_pos = self._file.tell() - _write_ulong(self._file, self._nframes) - if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'): - _write_short(self._file, 8) - else: - _write_short(self._file, self._sampwidth * 8) - _write_float(self._file, self._framerate) - if self._aifc: - self._file.write(self._comptype) - _write_string(self._file, self._compname) - self._file.write(b'SSND') - if self._form_length_pos is not None: - self._ssnd_length_pos = self._file.tell() - _write_ulong(self._file, self._datalength + 8) - _write_ulong(self._file, 0) - _write_ulong(self._file, 0) - - def _write_form_length(self, datalength): - if self._aifc: - commlength = 18 + 5 + len(self._compname) - if commlength & 1: - commlength = commlength + 1 - verslength = 12 - else: - commlength = 18 - verslength = 0 - _write_ulong(self._file, 4 + verslength + self._marklength + \ - 8 + commlength + 16 + datalength) - return commlength - - def _patchheader(self): - curpos = self._file.tell() - if self._datawritten & 1: - datalength = self._datawritten + 1 - self._file.write(b'\x00') - else: - datalength = self._datawritten - if datalength == self._datalength and \ - self._nframes == self._nframeswritten and \ - self._marklength == 0: - self._file.seek(curpos, 0) - return - self._file.seek(self._form_length_pos, 0) - dummy = self._write_form_length(datalength) - self._file.seek(self._nframes_pos, 0) - _write_ulong(self._file, self._nframeswritten) - self._file.seek(self._ssnd_length_pos, 0) - _write_ulong(self._file, datalength + 8) - self._file.seek(curpos, 0) - self._nframes = self._nframeswritten - self._datalength = datalength - - def _writemarkers(self): - if len(self._markers) == 0: - return - self._file.write(b'MARK') - length = 2 - for marker in self._markers: - id, pos, name = marker - length = length + len(name) + 1 + 6 - if len(name) & 1 == 0: - length = length + 1 - _write_ulong(self._file, length) - self._marklength = length + 8 - _write_short(self._file, len(self._markers)) - for marker in self._markers: - id, pos, name = marker - _write_short(self._file, id) - _write_ulong(self._file, pos) - _write_string(self._file, name) - -def open(f, mode=None): - if mode is None: - if hasattr(f, 'mode'): - mode = f.mode - else: - mode = 'rb' - if mode in ('r', 'rb'): - return Aifc_read(f) - elif mode in ('w', 'wb'): - return Aifc_write(f) - else: - raise Error("mode must be 'r', 'rb', 'w', or 'wb'") - - -if __name__ == '__main__': - import sys - if not sys.argv[1:]: - sys.argv.append('/usr/demos/data/audio/bach.aiff') - fn = sys.argv[1] - with open(fn, 'r') as f: - print("Reading", fn) - print("nchannels =", f.getnchannels()) - print("nframes =", f.getnframes()) - print("sampwidth =", f.getsampwidth()) - print("framerate =", f.getframerate()) - print("comptype =", f.getcomptype()) - print("compname =", f.getcompname()) - if sys.argv[2:]: - gn = sys.argv[2] - print("Writing", gn) - with open(gn, 'w') as g: - g.setparams(f.getparams()) - while 1: - data = f.readframes(1024) - if not data: - break - g.writeframes(data) - print("Done.") diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py new file mode 100644 index 00000000000..832d160de7f --- /dev/null +++ b/Lib/annotationlib.py @@ -0,0 +1,1152 @@ +"""Helpers for introspecting and wrapping annotations.""" + +import ast +import builtins +import enum +import keyword +import sys +import types + +__all__ = [ + "Format", + "ForwardRef", + "call_annotate_function", + "call_evaluate_function", + "get_annotate_from_class_namespace", + "get_annotations", + "annotations_to_string", + "type_repr", +] + + +class Format(enum.IntEnum): + VALUE = 1 + VALUE_WITH_FAKE_GLOBALS = 2 + FORWARDREF = 3 + STRING = 4 + + +_sentinel = object() +# Following `NAME_ERROR_MSG` in `ceval_macros.h`: +_NAME_ERROR_MSG = "name '{name:.200}' is not defined" + + +# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be +# preserved for compatibility with the old typing.ForwardRef class. The remaining +# names are private. +_SLOTS = ( + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__globals__", + "__extra_names__", + "__code__", + "__ast_node__", + "__cell__", + "__owner__", + "__stringifier_dict__", +) + + +class ForwardRef: + """Wrapper that holds a forward reference. + + Constructor arguments: + * arg: a string representing the code to be evaluated. + * module: the module where the forward reference was created. + Must be a string, not a module object. + * owner: The owning object (module, class, or function). + * is_argument: Does nothing, retained for compatibility. + * is_class: True if the forward reference was created in class scope. + + """ + + __slots__ = _SLOTS + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_argument=True, + is_class=False, + ): + if not isinstance(arg, str): + raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + self.__arg__ = arg + self.__forward_is_argument__ = is_argument + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__owner__ = owner + # These are always set to None here but may be non-None if a ForwardRef + # is created through __class__ assignment on a _Stringifier object. + self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). + self.__cell__ = None + self.__extra_names__ = None + # These are initially None but serve as a cache and may be set to a non-None + # value later. + self.__code__ = None + self.__ast_node__ = None + + def __init_subclass__(cls, /, *args, **kwds): + raise TypeError("Cannot subclass ForwardRef") + + def evaluate( + self, + *, + globals=None, + locals=None, + type_params=None, + owner=None, + format=Format.VALUE, + ): + """Evaluate the forward reference and return the value. + + If the forward reference cannot be evaluated, raise an exception. + """ + match format: + case Format.STRING: + return self.__forward_arg__ + case Format.VALUE: + is_forwardref_format = False + case Format.FORWARDREF: + is_forwardref_format = True + case _: + raise NotImplementedError(format) + if isinstance(self.__cell__, types.CellType): + try: + return self.__cell__.cell_contents + except ValueError: + pass + if owner is None: + owner = self.__owner__ + + if globals is None and self.__forward_module__ is not None: + globals = getattr( + sys.modules.get(self.__forward_module__, None), "__dict__", None + ) + if globals is None: + globals = self.__globals__ + if globals is None: + if isinstance(owner, type): + module_name = getattr(owner, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + globals = getattr(module, "__dict__", None) + elif isinstance(owner, types.ModuleType): + globals = getattr(owner, "__dict__", None) + elif callable(owner): + globals = getattr(owner, "__globals__", None) + + # If we pass None to eval() below, the globals of this module are used. + if globals is None: + globals = {} + + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + + if locals is None: + locals = {} + if isinstance(owner, type): + locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params is not None: + for param in type_params: + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + + if self.__extra_names__: + locals.update(self.__extra_names__) + + arg = self.__forward_arg__ + if arg.isidentifier() and not keyword.iskeyword(arg): + if arg in locals: + return locals[arg] + elif arg in globals: + return globals[arg] + elif hasattr(builtins, arg): + return getattr(builtins, arg) + elif is_forwardref_format: + return self + else: + raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg) + else: + code = self.__forward_code__ + try: + return eval(code, globals=globals, locals=locals) + except Exception: + if not is_forwardref_format: + raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. + new_locals = _StringifierDict( + {**builtins.__dict__, **globals, **locals}, + globals=globals, + owner=owner, + is_class=self.__forward_is_class__, + format=format, + ) + try: + result = eval(code, globals=globals, locals=new_locals) + except Exception: + return self + else: + new_locals.transmogrify(self.__cell__) + return result + + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + import typing + import warnings + + if type_params is _sentinel: + typing._deprecation_warning_for_no_type_params_passed( + "typing.ForwardRef._evaluate" + ) + type_params = () + warnings._deprecated( + "ForwardRef._evaluate", + "{name} is a private API and is retained for compatibility, but will be removed" + " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", + remove=(3, 16), + ) + return typing.evaluate_forward_ref( + self, + globals=globalns, + locals=localns, + type_params=type_params, + _recursive_guard=recursive_guard, + ) + + @property + def __forward_arg__(self): + if self.__arg__ is not None: + return self.__arg__ + if self.__ast_node__ is not None: + self.__arg__ = ast.unparse(self.__ast_node__) + return self.__arg__ + raise AssertionError( + "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" + ) + + @property + def __forward_code__(self): + if self.__code__ is not None: + return self.__code__ + arg = self.__forward_arg__ + try: + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") + except SyntaxError: + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + return self.__code__ + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + # Use "is" here because we use id() for this in __hash__ + # because dictionaries are not hashable. + and self.__globals__ is other.__globals__ + and self.__forward_is_class__ == other.__forward_is_class__ + # Two separate cells are always considered unequal in forward refs. + and ( + {name: id(cell) for name, cell in self.__cell__.items()} + == {name: id(cell) for name, cell in other.__cell__.items()} + if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) + else self.__cell__ is other.__cell__ + ) + and self.__owner__ == other.__owner__ + and ( + (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == + (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None) + ) + ) + + def __hash__(self): + return hash(( + self.__forward_arg__, + self.__forward_module__, + id(self.__globals__), # dictionaries are not hashable, so hash by identity + self.__forward_is_class__, + ( # cells are not hashable as well + tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) + if isinstance(self.__cell__, dict) else id(self.__cell__), + ), + self.__owner__, + tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, + )) + + def __or__(self, other): + return types.UnionType[self, other] + + def __ror__(self, other): + return types.UnionType[other, self] + + def __repr__(self): + extra = [] + if self.__forward_module__ is not None: + extra.append(f", module={self.__forward_module__!r}") + if self.__forward_is_class__: + extra.append(", is_class=True") + if self.__owner__ is not None: + extra.append(f", owner={self.__owner__!r}") + return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + + +_Template = type(t"") + + +class _Stringifier: + # Must match the slots on ForwardRef, so we can turn an instance of one into an + # instance of the other in place. + __slots__ = _SLOTS + + def __init__( + self, + node, + globals=None, + owner=None, + is_class=False, + cell=None, + *, + stringifier_dict, + extra_names=None, + ): + # Either an AST node or a simple str (for the common case where a ForwardRef + # represent a single name). + assert isinstance(node, (ast.AST, str)) + self.__arg__ = None + self.__forward_is_argument__ = False + self.__forward_is_class__ = is_class + self.__forward_module__ = None + self.__code__ = None + self.__ast_node__ = node + self.__globals__ = globals + self.__extra_names__ = extra_names + self.__cell__ = cell + self.__owner__ = owner + self.__stringifier_dict__ = stringifier_dict + + def __convert_to_ast(self, other): + if isinstance(other, _Stringifier): + if isinstance(other.__ast_node__, str): + return ast.Name(id=other.__ast_node__), other.__extra_names__ + return other.__ast_node__, other.__extra_names__ + elif type(other) is _Template: + return _template_to_ast(other), None + elif ( + # In STRING format we don't bother with the create_unique_name() dance; + # it's better to emit the repr() of the object instead of an opaque name. + self.__stringifier_dict__.format == Format.STRING + or other is None + or type(other) in (str, int, float, bool, complex) + ): + return ast.Constant(value=other), None + elif type(other) is dict: + extra_names = {} + keys = [] + values = [] + for key, value in other.items(): + new_key, new_extra_names = self.__convert_to_ast(key) + if new_extra_names is not None: + extra_names.update(new_extra_names) + keys.append(new_key) + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + values.append(new_value) + return ast.Dict(keys, values), extra_names + elif type(other) in (list, tuple, set): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] + return ast_class(elts), extra_names + else: + name = self.__stringifier_dict__.create_unique_name() + return ast.Name(id=name), {name: other} + + def __convert_to_ast_getitem(self, other): + if isinstance(other, slice): + extra_names = {} + + def conv(obj): + if obj is None: + return None + new_obj, new_extra_names = self.__convert_to_ast(obj) + if new_extra_names is not None: + extra_names.update(new_extra_names) + return new_obj + + return ast.Slice( + lower=conv(other.start), + upper=conv(other.stop), + step=conv(other.step), + ), extra_names + else: + return self.__convert_to_ast(other) + + def __get_ast(self): + node = self.__ast_node__ + if isinstance(node, str): + return ast.Name(id=node) + return node + + def __make_new(self, node, extra_names=None): + new_extra_names = {} + if self.__extra_names__ is not None: + new_extra_names.update(self.__extra_names__) + if extra_names is not None: + new_extra_names.update(extra_names) + stringifier = _Stringifier( + node, + self.__globals__, + self.__owner__, + self.__forward_is_class__, + stringifier_dict=self.__stringifier_dict__, + extra_names=new_extra_names or None, + ) + self.__stringifier_dict__.stringifiers.append(stringifier) + return stringifier + + # Must implement this since we set __eq__. We hash by identity so that + # stringifiers in dict keys are kept separate. + def __hash__(self): + return id(self) + + def __getitem__(self, other): + # Special case, to avoid stringifying references to class-scoped variables + # as '__classdict__["x"]'. + if self.__ast_node__ == "__classdict__": + raise KeyError + if isinstance(other, tuple): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast_getitem(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + other = ast.Tuple(elts) + else: + other, extra_names = self.__convert_to_ast_getitem(other) + assert isinstance(other, ast.AST), repr(other) + return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names) + + def __getattr__(self, attr): + return self.__make_new(ast.Attribute(self.__get_ast(), attr)) + + def __call__(self, *args, **kwargs): + extra_names = {} + ast_args = [] + for arg in args: + new_arg, new_extra_names = self.__convert_to_ast(arg) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_args.append(new_arg) + ast_kwargs = [] + for key, value in kwargs.items(): + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_kwargs.append(ast.keyword(key, new_value)) + return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names) + + def __iter__(self): + yield self.__make_new(ast.Starred(self.__get_ast())) + + def __repr__(self): + if isinstance(self.__ast_node__, str): + return self.__ast_node__ + return ast.unparse(self.__ast_node__) + + def __format__(self, format_spec): + raise TypeError("Cannot stringify annotation containing string formatting") + + def _make_binop(op: ast.AST): + def binop(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(self.__get_ast(), op, rhs), extra_names + ) + + return binop + + __add__ = _make_binop(ast.Add()) + __sub__ = _make_binop(ast.Sub()) + __mul__ = _make_binop(ast.Mult()) + __matmul__ = _make_binop(ast.MatMult()) + __truediv__ = _make_binop(ast.Div()) + __mod__ = _make_binop(ast.Mod()) + __lshift__ = _make_binop(ast.LShift()) + __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) + __xor__ = _make_binop(ast.BitXor()) + __and__ = _make_binop(ast.BitAnd()) + __floordiv__ = _make_binop(ast.FloorDiv()) + __pow__ = _make_binop(ast.Pow()) + + del _make_binop + + def _make_rbinop(op: ast.AST): + def rbinop(self, other): + new_other, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(new_other, op, self.__get_ast()), extra_names + ) + + return rbinop + + __radd__ = _make_rbinop(ast.Add()) + __rsub__ = _make_rbinop(ast.Sub()) + __rmul__ = _make_rbinop(ast.Mult()) + __rmatmul__ = _make_rbinop(ast.MatMult()) + __rtruediv__ = _make_rbinop(ast.Div()) + __rmod__ = _make_rbinop(ast.Mod()) + __rlshift__ = _make_rbinop(ast.LShift()) + __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) + __rxor__ = _make_rbinop(ast.BitXor()) + __rand__ = _make_rbinop(ast.BitAnd()) + __rfloordiv__ = _make_rbinop(ast.FloorDiv()) + __rpow__ = _make_rbinop(ast.Pow()) + + del _make_rbinop + + def _make_compare(op): + def compare(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.Compare( + left=self.__get_ast(), + ops=[op], + comparators=[rhs], + ), + extra_names, + ) + + return compare + + __lt__ = _make_compare(ast.Lt()) + __le__ = _make_compare(ast.LtE()) + __eq__ = _make_compare(ast.Eq()) + __ne__ = _make_compare(ast.NotEq()) + __gt__ = _make_compare(ast.Gt()) + __ge__ = _make_compare(ast.GtE()) + + del _make_compare + + def _make_unary_op(op): + def unary_op(self): + return self.__make_new(ast.UnaryOp(op, self.__get_ast())) + + return unary_op + + __invert__ = _make_unary_op(ast.Invert()) + __pos__ = _make_unary_op(ast.UAdd()) + __neg__ = _make_unary_op(ast.USub()) + + del _make_unary_op + + +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" + values = [] + interp_count = 0 + for part in template: + match part: + case str(): + values.append(ast.Constant(value=part)) + case _: + interp = ast.Interpolation( + str=part.expression, + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, + ) + values.append(interp) + interp_count += 1 + return ast.TemplateStr(values=values) + + +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + +class _StringifierDict(dict): + def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): + super().__init__(namespace) + self.namespace = namespace + self.globals = globals + self.owner = owner + self.is_class = is_class + self.stringifiers = [] + self.next_id = 1 + self.format = format + + def __missing__(self, key): + fwdref = _Stringifier( + key, + globals=self.globals, + owner=self.owner, + is_class=self.is_class, + stringifier_dict=self, + ) + self.stringifiers.append(fwdref) + return fwdref + + def transmogrify(self, cell_dict): + for obj in self.stringifiers: + obj.__class__ = ForwardRef + obj.__stringifier_dict__ = None # not needed for ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict + + def create_unique_name(self): + name = f"__annotationlib_name_{self.next_id}__" + self.next_id += 1 + return name + + +def call_evaluate_function(evaluate, format, *, owner=None): + """Call an evaluate function. Evaluate functions are normally generated for + the value of type aliases and the bounds, constraints, and defaults of + type parameter objects. + """ + return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) + + +def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): + """Call an __annotate__ function. __annotate__ functions are normally + generated by the compiler to defer the evaluation of annotations. They + can be called with any of the format arguments in the Format enum, but + compiler-generated __annotate__ functions only support the VALUE format. + This function provides additional functionality to call __annotate__ + functions with the FORWARDREF and STRING formats. + + *annotate* must be an __annotate__ function, which takes a single argument + and returns a dict of annotations. + + *format* must be a member of the Format enum or one of the corresponding + integer values. + + *owner* can be the object that owns the annotations (i.e., the module, + class, or function that the __annotate__ function derives from). With the + FORWARDREF format, it is used to provide better evaluation capabilities + on the generated ForwardRef objects. + + """ + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + try: + return annotate(format) + except NotImplementedError: + pass + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special + # environment where every name lookup results in an instance of _Stringifier. + # _Stringifier supports every dunder operation and returns a new _Stringifier. + # At the end, we get a dictionary that mostly contains _Stringifier objects (or + # possibly constants if the annotate function uses them directly). We then + # convert each of those into a string to get an approximation of the + # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + + globals = _StringifierDict({}, format=format) + is_class = isinstance(owner, type) + closure, _ = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) + if _is_evaluate: + return _stringify_single(annos) + return { + key: _stringify_single(val) + for key, val in annos.items() + } + elif format == Format.FORWARDREF: + # FORWARDREF is implemented similarly to STRING, but there are two changes, + # at the beginning and the end of the process. + # First, while STRING uses an empty dictionary as the namespace, so that all + # name lookups result in _Stringifier objects, FORWARDREF uses the globals + # and builtins, so that defined names map to their real values. + # Second, instead of returning strings, we want to return either real values + # or ForwardRef objects. To do this, we keep track of all _Stringifier objects + # created while the annotation is being evaluated, and at the end we convert + # them all to ForwardRef objects by assigning to __class__. To make this + # technique work, we have to ensure that the _Stringifier and ForwardRef + # classes share the same attributes. + # We use this technique because while the annotations are being evaluated, + # we want to support all operations that the language allows, including even + # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately + # reconstruct the source. But in the dictionary that we eventually return, we + # want to return objects with more user-friendly behavior, such as an __eq__ + # that returns a bool and an defined set of attributes. + namespace = {**annotate.__builtins__, **annotate.__globals__} + is_class = isinstance(owner, type) + globals = _StringifierDict( + namespace, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=True + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + try: + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) + except Exception: + pass + else: + globals.transmogrify(cell_dict) + return result + + # Try again, but do not provide any globals. This allows us to return + # a value in certain cases where an exception gets raised during evaluation. + globals = _StringifierDict( + {}, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + globals.transmogrify(cell_dict) + if _is_evaluate: + if isinstance(result, ForwardRef): + return result.evaluate(format=Format.FORWARDREF) + else: + return result + else: + return { + key: ( + val.evaluate(format=Format.FORWARDREF) + if isinstance(val, ForwardRef) + else val + ) + for key, val in result.items() + } + elif format == Format.VALUE: + # Should be impossible because __annotate__ functions must not raise + # NotImplementedError for this format. + raise RuntimeError("annotate function does not support VALUE format") + else: + raise ValueError(f"Invalid format: {format!r}") + + +def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): + if not annotate.__closure__: + return None, None + new_closure = [] + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell + new_cell = None + if allow_evaluation: + try: + cell.cell_contents + except ValueError: + pass + else: + new_cell = cell + if new_cell is None: + fwdref = _Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=stringifier_dict, + ) + stringifier_dict.stringifiers.append(fwdref) + new_cell = types.CellType(fwdref) + new_closure.append(new_cell) + return tuple(new_closure), cell_dict + + +def _stringify_single(anno): + if anno is ...: + return "..." + # We have to handle str specially to support PEP 563 stringified annotations. + elif isinstance(anno, str): + return anno + elif isinstance(anno, _Template): + return ast.unparse(_template_to_ast(anno)) + else: + return repr(anno) + + +def get_annotate_from_class_namespace(obj): + """Retrieve the annotate function from a class namespace dictionary. + + Return None if the namespace does not contain an annotate function. + This is useful in metaclass ``__new__`` methods to retrieve the annotate function. + """ + try: + return obj["__annotate__"] + except KeyError: + return obj.get("__annotate_func__", None) + + +def get_annotations( + obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE +): + """Compute the annotations dict for an object. + + obj may be a callable, class, module, or other object with + __annotate__ or __annotations__ attributes. + Passing any other object raises TypeError. + + The *format* parameter controls the format in which annotations are returned, + and must be a member of the Format enum or its integer equivalent. + For the VALUE format, the __annotations__ is tried first; if it + does not exist, the __annotate__ function is called. The + FORWARDREF format uses __annotations__ if it exists and can be + evaluated, and otherwise falls back to calling the __annotate__ function. + The SOURCE format tries __annotate__ first, and falls back to + using __annotations__, stringified using annotations_to_string(). + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if eval_str and format != Format.VALUE: + raise ValueError("eval_str=True is only supported with format=Format.VALUE") + + match format: + case Format.VALUE: + # For VALUE, we first look at __annotations__ + ann = _get_dunder_annotations(obj) + + # If it's not there, try __annotate__ instead + if ann is None: + ann = _get_and_call_annotate(obj, format) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + ann = _get_dunder_annotations(obj) + except Exception: + pass + else: + if ann is not None: + return dict(ann) + + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is None: + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + ann = _get_dunder_annotations(obj) + case Format.STRING: + # For STRING, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return dict(ann) + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + if ann is not None: + return annotations_to_string(ann) + case Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + case _: + raise ValueError(f"Unsupported format {format!r}") + + if ann is None: + if isinstance(obj, type) or callable(obj): + return {} + raise TypeError(f"{obj!r} does not have annotations") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if globals is None or locals is None: + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + else: + obj_globals = obj_locals = unwrap = None + + if unwrap is not None: + while True: + if hasattr(unwrap, "__wrapped__"): + unwrap = unwrap.__wrapped__ + continue + if functools := sys.modules.get("functools"): + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + if locals is None: + locals = {} + locals = {param.__name__: param for param in type_params} | locals + + return_value = { + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) + for key, value in ann.items() + } + return return_value + + +def type_repr(value): + """Convert a Python value to a format suitable for use with the STRING format. + + This is intended as a helper for tools that support the STRING format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, (type, types.FunctionType, types.BuiltinFunctionType)): + if value.__module__ == "builtins": + return value.__qualname__ + return f"{value.__module__}.{value.__qualname__}" + elif isinstance(value, _Template): + tree = _template_to_ast(value) + return ast.unparse(tree) + if value is ...: + return "..." + return repr(value) + + +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format. + + Always returns a fresh a dictionary. + """ + return { + n: t if isinstance(t, str) else type_repr(t) + for n, t in annotations.items() + } + + +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.lstrip().startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + +def _get_and_call_annotate(obj, format): + """Get the __annotate__ function and call it. + + May not return a fresh dictionary. + """ + annotate = getattr(obj, "__annotate__", None) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return ann + return None + + +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + +def _get_dunder_annotations(obj): + """Return the annotations for an object, checking that it is a dictionary. + + Does not return a fresh dictionary. + """ + # This special case is needed to support types defined under + # from __future__ import annotations, where accessing the __annotations__ + # attribute directly might return annotations for the wrong class. + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return None + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return None + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return ann diff --git a/Lib/argparse.py b/Lib/argparse.py index 543d9944f9e..1d7d34f9924 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -18,11 +18,12 @@ 'integers', metavar='int', nargs='+', type=int, help='an integer to be summed') parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), + '--log', help='the file where the sum should be written') args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() + with (open(args.log, 'w') if args.log is not None + else contextlib.nullcontext(sys.stdout)) as log: + log.write('%s' % sum(args.integers)) The module contains the following public classes: @@ -39,7 +40,8 @@ - FileType -- A factory for defining types of files to be created. As the example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. + the type= argument of add_argument() calls. Deprecated since + Python 3.14. - Action -- The base class for parser actions. Typically actions are selected by passing strings like 'store_true' or 'append_const' to @@ -89,8 +91,6 @@ import re as _re import sys as _sys -import warnings - from gettext import gettext as _, ngettext SUPPRESS = '==SUPPRESS==' @@ -161,18 +161,21 @@ class HelpFormatter(object): provided by the class are considered an implementation detail. """ - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - + def __init__( + self, + prog, + indent_increment=2, + max_help_position=24, + width=None, + color=True, + ): # default setting for width if width is None: import shutil width = shutil.get_terminal_size().columns width -= 2 + self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -189,9 +192,20 @@ def __init__(self, self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') + def _set_color(self, color): + from _colorize import can_colorize, decolor, get_theme + + if color and can_colorize(): + self._theme = get_theme(force_color=True).argparse + self._decolor = decolor + else: + self._theme = get_theme(force_no_color=True).argparse + self._decolor = lambda text: text + # =============================== # Section and indentation methods # =============================== + def _indent(self): self._current_indent += self._indent_increment self._level += 1 @@ -225,7 +239,12 @@ def format_help(self): # add the heading if the section was non-empty if self.heading is not SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) + heading_text = _('%(heading)s:') % dict(heading=self.heading) + t = self.formatter._theme + heading = ( + f'{" " * current_indent}' + f'{t.heading}{heading_text}{t.reset}\n' + ) else: heading = '' @@ -238,6 +257,7 @@ def _add_item(self, func, args): # ======================== # Message building methods # ======================== + def start_section(self, heading): self._indent() section = self._Section(self, self._current_section, heading) @@ -261,14 +281,13 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation - invocations = [get_invocation(action)] + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) + invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): - invocations.append(get_invocation(subaction)) + invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) # update the maximum item length - invocation_length = max(map(len, invocations)) - action_length = invocation_length + self._current_indent + action_length = max(invocation_lengths) self._action_max_length = max(self._action_max_length, action_length) @@ -282,6 +301,7 @@ def add_arguments(self, actions): # ======================= # Help-formatting methods # ======================= + def format_help(self): help = self._root_section.format_help() if help: @@ -295,51 +315,39 @@ def _join_parts(self, part_strings): if part and part is not SUPPRESS]) def _format_usage(self, usage, actions, groups, prefix): + t = self._theme + if prefix is None: prefix = _('usage: ') # if usage is specified, use that if usage is not None: - usage = usage % dict(prog=self._prog) + usage = ( + t.prog_extra + + usage + % {"prog": f"{t.prog}{self._prog}{t.reset}{t.prog_extra}"} + + t.reset + ) # if no optionals or positionals are available, usage is just prog elif usage is None and not actions: - usage = '%(prog)s' % dict(prog=self._prog) + usage = f"{t.prog}{self._prog}{t.reset}" # if optionals and positionals are available, calculate usage elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent - if len(prefix) + len(usage) > text_width: + if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - part_regexp = ( - r'\(.*?\)+(?=\s|$)|' - r'\[.*?\]+(?=\s|$)|' - r'\S+' - ) - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -351,12 +359,13 @@ def get_lines(parts, indent, prefix=None): else: line_len = indent_length - 1 for part in parts: - if line_len + 1 + len(part) > text_width and line: + part_len = len(self._decolor(part)) + if line_len + 1 + part_len > text_width and line: lines.append(indent + ' '.join(line)) line = [] line_len = indent_length - 1 line.append(part) - line_len += len(part) + 1 + line_len += part_len + 1 if line: lines.append(indent + ' '.join(line)) if prefix is not None: @@ -364,8 +373,9 @@ def get_lines(parts, indent, prefix=None): return lines # if prog is short, follow it with optionals or positionals - if len(prefix) + len(prog) <= 0.75 * text_width: - indent = ' ' * (len(prefix) + len(prog) + 1) + prog_len = len(self._decolor(prog)) + if len(prefix) + prog_len <= 0.75 * text_width: + indent = ' ' * (len(prefix) + prog_len + 1) if opt_parts: lines = get_lines([prog] + opt_parts, indent, prefix) lines.extend(get_lines(pos_parts, indent)) @@ -388,121 +398,120 @@ def get_lines(parts, indent, prefix=None): # join lines into usage usage = '\n'.join(lines) - # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) - - def _format_actions_usage(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') + usage = usage.removeprefix(prog) + usage = f"{t.prog}{prog}{t.reset}{usage}" - try: - start = actions.index(group._group_actions[0]) - except ValueError: - continue - else: - group_action_count = len(group._group_actions) - end = start + group_action_count - if actions[start:end] == group._group_actions: + # prefix with 'usage:' + return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - suppressed_actions_count = 0 - for action in group._group_actions: - group_actions.add(action) - if action.help is SUPPRESS: - suppressed_actions_count += 1 + def _is_long_option(self, string): + return len(string) > 2 - exposed_actions_count = group_action_count - suppressed_actions_count + def _get_actions_usage_parts(self, actions, groups): + """Get usage parts with split index for optionals/positionals. - if not group.required: - if start in inserts: - inserts[start] += ' [' - else: - inserts[start] = '[' - if end in inserts: - inserts[end] += ']' - else: - inserts[end] = ']' - elif exposed_actions_count > 1: - if start in inserts: - inserts[start] += ' (' - else: - inserts[start] = '(' - if end in inserts: - inserts[end] += ')' - else: - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) + for group in groups: + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] - for i, action in enumerate(actions): - - # suppressed arguments are marked with None - # remove | separators for suppressed arguments - if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = self._format_args(action, default) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # add the action string to the list - parts.append(part) - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] + t = self._theme + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part # add the action string to the list parts.append(part) - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = text.strip() - - # return the text - return text + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: @@ -518,6 +527,7 @@ def _format_action(self, action): help_width = max(self._width - help_position, 11) action_width = help_position - self._current_indent - 2 action_header = self._format_action_invocation(action) + action_header_no_color = self._decolor(action_header) # no help; start on same line and add a final newline if not action.help: @@ -525,9 +535,15 @@ def _format_action(self, action): action_header = '%*s%s\n' % tup # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header + elif len(action_header_no_color) <= action_width: + # calculate widths without color codes + action_header_color = action_header + tup = self._current_indent, '', action_width, action_header_no_color action_header = '%*s%-*s ' % tup + # swap in the colored header + action_header = action_header.replace( + action_header_no_color, action_header_color + ) indent_first = 0 # long action name; start on the next line @@ -560,35 +576,48 @@ def _format_action(self, action): return self._join_parts(parts) def _format_action_invocation(self, action): + t = self._theme + if not action.option_strings: default = self._get_default_metavar_for_positional(action) - metavar, = self._metavar_formatter(action, default)(1) - return metavar + return ( + t.action + + ' '.join(self._metavar_formatter(action, default)(1)) + + t.reset + ) else: - parts = [] + + def color_option_strings(strings): + parts = [] + for s in strings: + if self._is_long_option(s): + parts.append(f"{t.long_option}{s}{t.reset}") + else: + parts.append(f"{t.short_option}{s}{t.reset}") + return parts # if the Optional doesn't take a value, format is: # -s, --long if action.nargs == 0: - parts.extend(action.option_strings) + option_strings = color_option_strings(action.option_strings) + return ', '.join(option_strings) # if the Optional takes a value, format is: - # -s ARGS, --long ARGS + # -s, --long ARGS else: default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) + option_strings = color_option_strings(action.option_strings) + args_string = ( + f"{t.label}{self._format_args(action, default)}{t.reset}" + ) + return ', '.join(option_strings) + ' ' + args_string def _metavar_formatter(self, action, default_metavar): if action.metavar is not None: result = action.metavar elif action.choices is not None: - choice_strs = [str(choice) for choice in action.choices] - result = '{%s}' % ','.join(choice_strs) + result = '{%s}' % ','.join(map(str, action.choices)) else: result = default_metavar @@ -628,17 +657,19 @@ def _format_args(self, action, default_metavar): return result def _expand_help(self, action): + help_string = self._get_help_string(action) + if '%' not in help_string: + return help_string params = dict(vars(action), prog=self._prog) for name in list(params): - if params[name] is SUPPRESS: + value = params[name] + if value is SUPPRESS: del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ + elif hasattr(value, '__name__'): + params[name] = value.__name__ if params.get('choices') is not None: - choices_str = ', '.join([str(c) for c in params['choices']]) - params['choices'] = choices_str - return self._get_help_string(action) % params + params['choices'] = ', '.join(map(str, params['choices'])) + return help_string % params def _iter_indented_subactions(self, action): try: @@ -704,23 +735,18 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter): """ def _get_help_string(self, action): - """ - Add the default value to the option help message. - - ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't - already present. This code will do that, detecting cornercases to - prevent duplicates or cases where it wouldn't make sense to the end - user. - """ help = action.help if help is None: help = '' - if '%(default)' not in help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' + if ( + '%(default)' not in help + and action.default is not SUPPRESS + and not action.required + ): + defaulting_nargs = (OPTIONAL, ZERO_OR_MORE) + if action.option_strings or action.nargs in defaulting_nargs: + help += _(' (default: %(default)s)') return help @@ -750,11 +776,19 @@ def _get_action_name(argument): elif argument.option_strings: return '/'.join(argument.option_strings) elif argument.metavar not in (None, SUPPRESS): - return argument.metavar + metavar = argument.metavar + if not isinstance(metavar, tuple): + return metavar + if argument.nargs == ZERO_OR_MORE and len(metavar) == 2: + return '%s[, %s]' % metavar + elif argument.nargs == ONE_OR_MORE: + return '%s[, %s]' % metavar + else: + return ', '.join(metavar) elif argument.dest not in (None, SUPPRESS): return argument.dest elif argument.choices: - return '{' + ','.join(argument.choices) + '}' + return '{%s}' % ','.join(map(str, argument.choices)) else: return None @@ -849,7 +883,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): self.option_strings = option_strings self.dest = dest self.nargs = nargs @@ -860,6 +895,7 @@ def __init__(self, self.required = required self.help = help self.metavar = metavar + self.deprecated = deprecated def _get_kwargs(self): names = [ @@ -873,6 +909,7 @@ def _get_kwargs(self): 'required', 'help', 'metavar', + 'deprecated', ] return [(name, getattr(self, name)) for name in names] @@ -880,59 +917,37 @@ def format_usage(self): return self.option_strings[0] def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - + raise NotImplementedError('.__call__() not defined') -# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. -_deprecated_default = object() class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, default=None, - type=_deprecated_default, - choices=_deprecated_default, required=False, help=None, - metavar=_deprecated_default): + deprecated=False): _option_strings = [] for option_string in option_strings: _option_strings.append(option_string) if option_string.startswith('--'): + if option_string.startswith('--no-'): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') option_string = '--no-' + option_string[2:] _option_strings.append(option_string) - # We need `_deprecated` special value to ban explicit arguments that - # match default value. Like: - # parser.add_argument('-f', action=BooleanOptionalAction, type=int) - for field_name in ('type', 'choices', 'metavar'): - if locals()[field_name] is not _deprecated_default: - warnings._deprecated( - field_name, - "{name!r} is deprecated as of Python 3.12 and will be " - "removed in Python {remove}.", - remove=(3, 14)) - - if type is _deprecated_default: - type = None - if choices is _deprecated_default: - choices = None - if metavar is _deprecated_default: - metavar = None - super().__init__( option_strings=_option_strings, dest=dest, nargs=0, default=default, - type=type, - choices=choices, required=required, help=help, - metavar=metavar) + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): @@ -955,7 +970,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for store actions must be != 0; if you ' 'have nothing to store, actions such as store ' @@ -972,7 +988,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values) @@ -987,7 +1004,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_StoreConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -995,7 +1013,8 @@ def __init__(self, const=const, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, self.const) @@ -1008,14 +1027,16 @@ def __init__(self, dest, default=False, required=False, - help=None): + help=None, + deprecated=False): super(_StoreTrueAction, self).__init__( option_strings=option_strings, dest=dest, const=True, - default=default, + deprecated=deprecated, required=required, - help=help) + help=help, + default=default) class _StoreFalseAction(_StoreConstAction): @@ -1025,14 +1046,16 @@ def __init__(self, dest, default=True, required=False, - help=None): + help=None, + deprecated=False): super(_StoreFalseAction, self).__init__( option_strings=option_strings, dest=dest, const=False, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) class _AppendAction(Action): @@ -1047,7 +1070,8 @@ def __init__(self, choices=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): if nargs == 0: raise ValueError('nargs for append actions must be != 0; if arg ' 'strings are not supplying the value to append, ' @@ -1064,7 +1088,8 @@ def __init__(self, choices=choices, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1082,7 +1107,8 @@ def __init__(self, default=None, required=False, help=None, - metavar=None): + metavar=None, + deprecated=False): super(_AppendConstAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1091,7 +1117,8 @@ def __init__(self, default=default, required=required, help=help, - metavar=metavar) + metavar=metavar, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) @@ -1107,14 +1134,16 @@ def __init__(self, dest, default=None, required=False, - help=None): + help=None, + deprecated=False): super(_CountAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, default=default, required=required, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): count = getattr(namespace, self.dest, None) @@ -1129,13 +1158,15 @@ def __init__(self, option_strings, dest=SUPPRESS, default=SUPPRESS, - help=None): + help=None, + deprecated=False): super(_HelpAction, self).__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, - help=help) + help=help, + deprecated=deprecated) def __call__(self, parser, namespace, values, option_string=None): parser.print_help() @@ -1149,7 +1180,10 @@ def __init__(self, version=None, dest=SUPPRESS, default=SUPPRESS, - help="show program's version number and exit"): + help=None, + deprecated=False): + if help is None: + help = _("show program's version number and exit") super(_VersionAction, self).__init__( option_strings=option_strings, dest=dest, @@ -1193,6 +1227,8 @@ def __init__(self, self._parser_class = parser_class self._name_parser_map = {} self._choices_actions = [] + self._deprecated = set() + self._color = True super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1203,34 +1239,45 @@ def __init__(self, help=help, metavar=metavar) - def add_parser(self, name, **kwargs): + def add_parser(self, name, *, deprecated=False, **kwargs): # set prog from the existing prefix if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + # set color + if kwargs.get('color') is None: + kwargs['color'] = self._color + aliases = kwargs.pop('aliases', ()) if name in self._name_parser_map: - raise ArgumentError(self, _('conflicting subparser: %s') % name) + raise ValueError(f'conflicting subparser: {name}') for alias in aliases: if alias in self._name_parser_map: - raise ArgumentError( - self, _('conflicting subparser alias: %s') % alias) + raise ValueError(f'conflicting subparser alias: {alias}') # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) + else: + choice_action = None # create the parser and add it to the map parser = self._parser_class(**kwargs) + if choice_action is not None: + parser._check_help(choice_action) self._name_parser_map[name] = parser # make parser available under aliases also for alias in aliases: self._name_parser_map[alias] = parser + if deprecated: + self._deprecated.add(name) + self._deprecated.update(aliases) + return parser def _get_subactions(self): @@ -1246,13 +1293,17 @@ def __call__(self, parser, namespace, values, option_string=None): # select the parser try: - parser = self._name_parser_map[parser_name] + subparser = self._name_parser_map[parser_name] except KeyError: args = {'parser_name': parser_name, 'choices': ', '.join(self._name_parser_map)} msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args raise ArgumentError(self, msg) + if parser_name in self._deprecated: + parser._warning(_("command '%(parser_name)s' is deprecated") % + {'parser_name': parser_name}) + # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them @@ -1260,12 +1311,13 @@ def __call__(self, parser, namespace, values, option_string=None): # In case this subparser defines new defaults, we parse them # in a new namespace object and then update the original # namespace for the relevant parts. - subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + subnamespace, arg_strings = subparser.parse_known_args(arg_strings, None) for key, value in vars(subnamespace).items(): setattr(namespace, key, value) if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + if not hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + setattr(namespace, _UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) class _ExtendAction(_AppendAction): @@ -1280,7 +1332,7 @@ def __call__(self, parser, namespace, values, option_string=None): # ============== class FileType(object): - """Factory for creating file object types + """Deprecated factory for creating file object types Instances of FileType are typically passed as type= arguments to the ArgumentParser add_argument() method. @@ -1297,6 +1349,12 @@ class FileType(object): """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + import warnings + warnings.warn( + "FileType is deprecated. Simply open files after parsing arguments.", + category=PendingDeprecationWarning, + stacklevel=2 + ) self._mode = mode self._bufsize = bufsize self._encoding = encoding @@ -1400,7 +1458,7 @@ def __init__(self, self._defaults = {} # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + self._negative_number_matcher = _re.compile(r'-\.?\d') # whether or not there are any optionals that look like negative # numbers -- uses a list so it can be shared and edited @@ -1409,6 +1467,7 @@ def __init__(self, # ==================== # Registration methods # ==================== + def register(self, registry_name, value, object): registry = self._registries.setdefault(registry_name, {}) registry[value] = object @@ -1419,6 +1478,7 @@ def _registry_get(self, registry_name, value, default=None): # ================================== # Namespace default accessor methods # ================================== + def set_defaults(self, **kwargs): self._defaults.update(kwargs) @@ -1438,6 +1498,7 @@ def get_default(self, dest): # ======================= # Adding argument actions # ======================= + def add_argument(self, *args, **kwargs): """ add_argument(dest, ..., name=value, ...) @@ -1450,7 +1511,8 @@ def add_argument(self, *args, **kwargs): chars = self.prefix_chars if not args or len(args) == 1 and args[0][0] not in chars: if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') + raise TypeError('dest supplied twice for positional argument,' + ' did you mean metavar?') kwargs = self._get_positional_kwargs(*args, **kwargs) # otherwise, we're adding an optional argument @@ -1466,27 +1528,34 @@ def add_argument(self, *args, **kwargs): kwargs['default'] = self.argument_default # create the action object, and add it to the parser + action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action "%s"' % (action_class,)) + raise ValueError(f'unknown action {action_class!r}') action = action_class(**kwargs) + # raise an error if action for positional argument does not + # consume arguments + if not action.option_strings and action.nargs == 0: + raise ValueError(f'action {action_name!r} is not valid for positional arguments') + # raise an error if the action type is not callable type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - raise ValueError('%r is not callable' % (type_func,)) + raise TypeError(f'{type_func!r} is not callable') if type_func is FileType: - raise ValueError('%r is a FileType class object, instance of it' - ' must be passed' % (type_func,)) + raise TypeError(f'{type_func!r} is a FileType class object, ' + f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: - self._get_formatter()._format_args(action, None) + formatter._format_args(action, None) except TypeError: raise ValueError("length of metavar tuple does not match nargs") - + self._check_help(action) return self._add_action(action) def add_argument_group(self, *args, **kwargs): @@ -1528,8 +1597,10 @@ def _add_container_actions(self, container): title_group_map = {} for group in self._action_groups: if group.title in title_group_map: - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) + # This branch could happen if a derived class added + # groups with duplicated titles in __init__ + msg = f'cannot merge actions - two groups are named {group.title!r}' + raise ValueError(msg) title_group_map[group.title] = group # map each action to its group @@ -1552,7 +1623,11 @@ def _add_container_actions(self, container): # NOTE: if add_mutually_exclusive_group ever gains title= and # description= then this code will need to be expanded as above for group in container._mutually_exclusive_groups: - mutex_group = self.add_mutually_exclusive_group( + if group._container is container: + cont = self + else: + cont = title_group_map[group._container.title] + mutex_group = cont.add_mutually_exclusive_group( required=group.required) # map the actions to their new mutex group @@ -1566,14 +1641,15 @@ def _add_container_actions(self, container): def _get_positional_kwargs(self, dest, **kwargs): # make sure required is not specified if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") + msg = "'required' is an invalid argument for positionals" raise TypeError(msg) # mark positional arguments as required if at least one is # always required - if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: - kwargs['required'] = True - if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + nargs = kwargs.get('nargs') + if nargs == 0: + raise ValueError('nargs for positionals must be != 0') + if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS]: kwargs['required'] = True # return the keyword arguments with no option strings @@ -1586,11 +1662,9 @@ def _get_optional_kwargs(self, *args, **kwargs): for option_string in args: # error on strings that don't start with an appropriate prefix if not option_string[0] in self.prefix_chars: - args = {'option': option_string, - 'prefix_chars': self.prefix_chars} - msg = _('invalid option string %(option)r: ' - 'must start with a character %(prefix_chars)r') - raise ValueError(msg % args) + raise ValueError( + f'invalid option string {option_string!r}: ' + f'must start with a character {self.prefix_chars!r}') # strings starting with two prefix characters are long options option_strings.append(option_string) @@ -1606,8 +1680,8 @@ def _get_optional_kwargs(self, *args, **kwargs): dest_option_string = option_strings[0] dest = dest_option_string.lstrip(self.prefix_chars) if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) + msg = f'dest= is required for options like {option_string!r}' + raise TypeError(msg) dest = dest.replace('-', '_') # return the updated keyword arguments @@ -1623,8 +1697,8 @@ def _get_handler(self): try: return getattr(self, handler_func_name) except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) + msg = f'invalid conflict_resolution value: {self.conflict_handler!r}' + raise ValueError(msg) def _check_conflict(self, action): @@ -1663,10 +1737,26 @@ def _handle_conflict_resolve(self, action, conflicting_actions): if not action.option_strings: action.container._remove_action(action) + def _check_help(self, action): + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() + try: + formatter._expand_help(action) + except (ValueError, TypeError, KeyError) as exc: + raise ValueError('badly formed help string') from exc + class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): + if 'prefix_chars' in kwargs: + import warnings + depr_msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + warnings.warn(depr_msg, DeprecationWarning, stacklevel=3) + # add any missing keyword arguments by checking the container update = kwargs.setdefault update('conflict_handler', container.conflict_handler) @@ -1698,13 +1788,7 @@ def _remove_action(self, action): self._group_actions.remove(action) def add_argument_group(self, *args, **kwargs): - warnings.warn( - "Nesting argument groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_argument_group(*args, **kwargs) - + raise ValueError('argument groups cannot be nested') class _MutuallyExclusiveGroup(_ArgumentGroup): @@ -1715,7 +1799,7 @@ def __init__(self, container, required=False): def _add_action(self, action): if action.required: - msg = _('mutually exclusive arguments must be optional') + msg = 'mutually exclusive arguments must be optional' raise ValueError(msg) action = self._container._add_action(action) self._group_actions.append(action) @@ -1725,13 +1809,29 @@ def _remove_action(self, action): self._container._remove_action(action) self._group_actions.remove(action) - def add_mutually_exclusive_group(self, *args, **kwargs): - warnings.warn( - "Nesting mutually exclusive groups is deprecated.", - category=DeprecationWarning, - stacklevel=2 - ) - return super().add_mutually_exclusive_group(*args, **kwargs) + def add_mutually_exclusive_group(self, **kwargs): + raise ValueError('mutually exclusive groups cannot be nested') + +def _prog_name(prog=None): + if prog is not None: + return prog + arg0 = _sys.argv[0] + try: + modspec = _sys.modules['__main__'].__spec__ + except (KeyError, AttributeError): + # possibly PYTHONSTARTUP or -X presite or other weird edge case + # no good answer here, so fall back to the default + modspec = None + if modspec is None: + # simple script + return _os.path.basename(arg0) + py = _os.path.basename(_sys.executable) + if modspec.name != '__main__': + # imported module or package + modname = modspec.name.removesuffix('.__main__') + return f'{py} -m {modname}' + # directory or ZIP file + return f'{py} {arg0}' class ArgumentParser(_AttributeHolder, _ActionsContainer): @@ -1754,6 +1854,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): - allow_abbrev -- Allow long options to be abbreviated unambiguously - exit_on_error -- Determines whether or not ArgumentParser exits with error info when an error occurs + - suggest_on_error - Enables suggestions for mistyped argument choices + and subparser names (default: ``False``) + - color - Allow color output in help messages (default: ``False``) """ def __init__(self, @@ -1769,19 +1872,18 @@ def __init__(self, conflict_handler='error', add_help=True, allow_abbrev=True, - exit_on_error=True): - + exit_on_error=True, + *, + suggest_on_error=False, + color=True, + ): superinit = super(ArgumentParser, self).__init__ superinit(description=description, prefix_chars=prefix_chars, argument_default=argument_default, conflict_handler=conflict_handler) - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog + self.prog = _prog_name(prog) self.usage = usage self.epilog = epilog self.formatter_class = formatter_class @@ -1789,6 +1891,11 @@ def __init__(self, self.add_help = add_help self.allow_abbrev = allow_abbrev self.exit_on_error = exit_on_error + self.suggest_on_error = suggest_on_error + self.color = color + + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) @@ -1811,17 +1918,16 @@ def identity(string): # add parent arguments and defaults for parent in parents: + if not isinstance(parent, ArgumentParser): + raise TypeError('parents must be a list of ArgumentParser') self._add_container_actions(parent) - try: - defaults = parent._defaults - except AttributeError: - pass - else: - self._defaults.update(defaults) + defaults = parent._defaults + self._defaults.update(defaults) # ======================= # Pretty __repr__ methods # ======================= + def _get_kwargs(self): names = [ 'prog', @@ -1836,16 +1942,17 @@ def _get_kwargs(self): # ================================== # Optional/Positional adding methods # ================================== + def add_subparsers(self, **kwargs): if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) + raise ValueError('cannot have multiple subparser arguments') # add the parser class to the arguments if it's not present kwargs.setdefault('parser_class', type(self)) if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) + title = kwargs.pop('title', _('subcommands')) + description = kwargs.pop('description', None) self._subparsers = self.add_argument_group(title, description) else: self._subparsers = self._positionals @@ -1853,15 +1960,19 @@ def add_subparsers(self, **kwargs): # prog defaults to the usage message of this parser, skipping # optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') + formatter.add_usage(None, positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list parsers_class = self._pop_action_class(kwargs, 'parsers') action = parsers_class(option_strings=[], **kwargs) + action._color = self.color + self._check_help(action) self._subparsers._add_action(action) # return the created parsers action @@ -1887,14 +1998,21 @@ def _get_positional_actions(self): # ===================================== # Command line argument parsing methods # ===================================== + def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_args(self, args=None, namespace=None): + return self._parse_known_args2(args, namespace, intermixed=False) + + def _parse_known_args2(self, args, namespace, intermixed): if args is None: # args default to the system args args = _sys.argv[1:] @@ -1921,18 +2039,18 @@ def parse_known_args(self, args=None, namespace=None): # parse the arguments and exit if there are any errors if self.exit_on_error: try: - namespace, args = self._parse_known_args(args, namespace) + namespace, args = self._parse_known_args(args, namespace, intermixed) except ArgumentError as err: self.error(str(err)) else: - namespace, args = self._parse_known_args(args, namespace) + namespace, args = self._parse_known_args(args, namespace, intermixed) if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) return namespace, args - def _parse_known_args(self, arg_strings, namespace): + def _parse_known_args(self, arg_strings, namespace, intermixed): # replace arg strings that are file references if self.fromfile_prefix_chars is not None: arg_strings = self._read_args_from_files(arg_strings) @@ -1964,11 +2082,11 @@ def _parse_known_args(self, arg_strings, namespace): # otherwise, add the arg to the arg strings # and note the index if it was an option else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: + option_tuples = self._parse_optional(arg_string) + if option_tuples is None: pattern = 'A' else: - option_string_indices[i] = option_tuple + option_string_indices[i] = option_tuples pattern = 'O' arg_string_pattern_parts.append(pattern) @@ -1978,15 +2096,15 @@ def _parse_known_args(self, arg_strings, namespace): # converts arg strings to the appropriate and then takes the action seen_actions = set() seen_non_default_actions = set() + warned = set() def take_action(action, argument_strings, option_string=None): seen_actions.add(action) argument_values = self._get_values(action, argument_strings) # error if this argument is not allowed with other previously - # seen arguments, assuming that actions that use the default - # value don't really count as "present" - if argument_values is not action.default: + # seen arguments + if action.option_strings or argument_strings: seen_non_default_actions.add(action) for conflict_action in action_conflicts.get(action, []): if conflict_action in seen_non_default_actions: @@ -2003,8 +2121,16 @@ def take_action(action, argument_strings, option_string=None): def consume_optional(start_index): # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple + option_tuples = option_string_indices[start_index] + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, sep, explicit_arg in option_tuples]) + args = {'option': arg_strings[start_index], 'matches': options} + msg = _('ambiguous option: %(option)s could match %(matches)s') + raise ArgumentError(None, msg % args) + + action, option_string, sep, explicit_arg = option_tuples[0] # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -2015,6 +2141,7 @@ def consume_optional(start_index): # if we found no optional action, skip it if action is None: extras.append(arg_strings[start_index]) + extras_pattern.append('O') return start_index + 1 # if there is an explicit argument, try to match the @@ -2031,18 +2158,28 @@ def consume_optional(start_index): and option_string[1] not in chars and explicit_arg != '' ): + if sep or explicit_arg[0] in chars: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] - new_explicit_arg = explicit_arg[1:] or None optionals_map = self._option_string_actions if option_string in optionals_map: action = optionals_map[option_string] - explicit_arg = new_explicit_arg + explicit_arg = explicit_arg[1:] + if not explicit_arg: + sep = explicit_arg = None + elif explicit_arg[0] == '=': + sep = '=' + explicit_arg = explicit_arg[1:] + else: + sep = '' else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - + extras.append(char + explicit_arg) + extras_pattern.append('O') + stop = start_index + 1 + break # if the action expect exactly one argument, we've # successfully matched the option; exit the loop elif arg_count == 1: @@ -2073,6 +2210,10 @@ def consume_optional(start_index): # the Optional's string args stopped assert action_tuples for action, args, option_string in action_tuples: + if action.deprecated and option_string not in warned: + self._warning(_("option '%(option)s' is deprecated") % + {'option': option_string}) + warned.add(option_string) take_action(action, args, option_string) return stop @@ -2091,7 +2232,20 @@ def consume_positionals(start_index): # and add the Positional and its args to the list for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] + # Strip out the first '--' if it is not in REMAINDER arg. + if action.nargs == PARSER: + if arg_strings_pattern[start_index] == '-': + assert args[0] == '--' + args.remove('--') + elif action.nargs != REMAINDER: + if (arg_strings_pattern.find('-', start_index, + start_index + arg_count) >= 0): + args.remove('--') start_index += arg_count + if args and action.deprecated and action.dest not in warned: + self._warning(_("argument '%(argument_name)s' is deprecated") % + {'argument_name': action.dest}) + warned.add(action.dest) take_action(action, args) # slice off the Positionals that we just parsed and return the @@ -2102,6 +2256,7 @@ def consume_positionals(start_index): # consume Positionals and Optionals alternately, until we have # passed the last option string extras = [] + extras_pattern = [] start_index = 0 if option_string_indices: max_option_string_index = max(option_string_indices) @@ -2110,11 +2265,12 @@ def consume_positionals(start_index): while start_index <= max_option_string_index: # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) - if start_index != next_option_string_index: + next_option_string_index = start_index + while next_option_string_index <= max_option_string_index: + if next_option_string_index in option_string_indices: + break + next_option_string_index += 1 + if not intermixed and start_index != next_option_string_index: positionals_end_index = consume_positionals(start_index) # only try to parse the next optional if we didn't consume @@ -2130,16 +2286,35 @@ def consume_positionals(start_index): if start_index not in option_string_indices: strings = arg_strings[start_index:next_option_string_index] extras.extend(strings) + extras_pattern.extend(arg_strings_pattern[start_index:next_option_string_index]) start_index = next_option_string_index # consume the next optional and any arguments for it start_index = consume_optional(start_index) - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) + if not intermixed: + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) - # if we didn't consume all the argument strings, there were extras - extras.extend(arg_strings[stop_index:]) + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + else: + extras.extend(arg_strings[start_index:]) + extras_pattern.extend(arg_strings_pattern[start_index:]) + extras_pattern = ''.join(extras_pattern) + assert len(extras_pattern) == len(extras) + # consume all positionals + arg_strings = [s for s, c in zip(extras, extras_pattern) if c != 'O'] + arg_strings_pattern = extras_pattern.replace('O', '') + stop_index = consume_positionals(0) + # leave unknown optionals and non-consumed positionals in extras + for i, c in enumerate(extras_pattern): + if not stop_index: + break + if c != 'O': + stop_index -= 1 + extras[i] = None + extras = [s for s in extras if s is not None] # make sure all required actions were present and also convert # action defaults which were not given as arguments @@ -2161,7 +2336,7 @@ def consume_positionals(start_index): self._get_value(action, action.default)) if required_actions: - self.error(_('the following arguments are required: %s') % + raise ArgumentError(None, _('the following arguments are required: %s') % ', '.join(required_actions)) # make sure all required groups had one option present @@ -2177,7 +2352,7 @@ def consume_positionals(start_index): for action in group._group_actions if action.help is not SUPPRESS] msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) + raise ArgumentError(None, msg % ' '.join(names)) # return the updated namespace and the extra arguments return namespace, extras @@ -2204,7 +2379,7 @@ def _read_args_from_files(self, arg_strings): arg_strings = self._read_args_from_files(arg_strings) new_arg_strings.extend(arg_strings) except OSError as err: - self.error(str(err)) + raise ArgumentError(None, str(err)) # return the modified argument list return new_arg_strings @@ -2237,18 +2412,19 @@ def _match_argument(self, action, arg_strings_pattern): def _match_arguments_partial(self, actions, arg_strings_pattern): # progressively shorten the actions list by slicing off the # final actions until we find a match - result = [] for i in range(len(actions), 0, -1): actions_slice = actions[:i] pattern = ''.join([self._get_nargs_pattern(action) for action in actions_slice]) match = _re.match(pattern, arg_strings_pattern) if match is not None: - result.extend([len(string) for string in match.groups()]) - break - - # return the list of arg string counts - return result + result = [len(string) for string in match.groups()] + if (match.end() < len(arg_strings_pattern) + and arg_strings_pattern[match.end()] == 'O'): + while result and not result[-1]: + del result[-1] + return result + return [] def _parse_optional(self, arg_string): # if it's an empty string, it was meant to be a positional @@ -2262,36 +2438,24 @@ def _parse_optional(self, arg_string): # if the option string is present in the parser, return the action if arg_string in self._option_string_actions: action = self._option_string_actions[arg_string] - return action, arg_string, None + return [(action, arg_string, None, None)] # if it's just a single character, it was meant to be positional if len(arg_string) == 1: return None # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg + option_string, sep, explicit_arg = arg_string.partition('=') + if sep and option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return [(action, option_string, sep, explicit_arg)] # search through all possible prefixes of the option string # and all actions in the parser for possible interpretations option_tuples = self._get_option_tuples(arg_string) - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) - args = {'option': arg_string, 'matches': options} - msg = _('ambiguous option: %(option)s could match %(matches)s') - self.error(msg % args) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple + if option_tuples: + return option_tuples # if it was not found as an option, but it looks like a negative # number, it was meant to be positional @@ -2306,7 +2470,7 @@ def _parse_optional(self, arg_string): # it was meant to be an optional but there is no such option # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None + return [(None, arg_string, None, None)] def _get_option_tuples(self, option_string): result = [] @@ -2316,39 +2480,38 @@ def _get_option_tuples(self, option_string): chars = self.prefix_chars if option_string[0] in chars and option_string[1] in chars: if self.allow_abbrev: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None for option_string in self._option_string_actions: if option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, sep, explicit_arg result.append(tup) # single character options can be concatenated with their arguments # but multiple character options always have to have their argument # separate elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string - explicit_arg = None + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None short_option_prefix = option_string[:2] short_explicit_arg = option_string[2:] for option_string in self._option_string_actions: if option_string == short_option_prefix: action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg + tup = action, option_string, '', short_explicit_arg result.append(tup) - elif option_string.startswith(option_prefix): + elif self.allow_abbrev and option_string.startswith(option_prefix): action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg + tup = action, option_string, sep, explicit_arg result.append(tup) # shouldn't ever get here else: - self.error(_('unexpected option string: %s') % option_string) + raise ArgumentError(None, _('unexpected option string: %s') % option_string) # return the collected option tuples return result @@ -2357,43 +2520,40 @@ def _get_nargs_pattern(self, action): # in all examples below, we have to allow for '--' args # which are represented as '-' in the pattern nargs = action.nargs + # if this is an optional action, -- is not allowed + option = action.option_strings # the default (None) is assumed to be a single argument if nargs is None: - nargs_pattern = '(-*A-*)' + nargs_pattern = '([A])' if option else '(-*A-*)' # allow zero or one arguments elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' + nargs_pattern = '(A?)' if option else '(-*A?-*)' # allow zero or more arguments elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' + nargs_pattern = '(A*)' if option else '(-*[A-]*)' # allow one or more arguments elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' + nargs_pattern = '(A+)' if option else '(-*A[A-]*)' # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' + nargs_pattern = '([AO]*)' if option else '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' + nargs_pattern = '(A[AO]*)' if option else '(-*A[-AO]*)' # suppress action, like nargs=0 elif nargs == SUPPRESS: - nargs_pattern = '(-*-*)' + nargs_pattern = '()' if option else '(-*)' # all others should be integers else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') + nargs_pattern = '([AO]{%d})' % nargs if option else '((?:-*A){%d}-*)' % nargs # return the pattern return nargs_pattern @@ -2405,8 +2565,11 @@ def _get_nargs_pattern(self, action): def parse_intermixed_args(self, args=None, namespace=None): args, argv = self.parse_known_intermixed_args(args, namespace) if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) + msg = _('unrecognized arguments: %s') % ' '.join(argv) + if self.exit_on_error: + self.error(msg) + else: + raise ArgumentError(None, msg) return args def parse_known_intermixed_args(self, args=None, namespace=None): @@ -2417,10 +2580,6 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # are then parsed. If the parser definition is incompatible with the # intermixed assumptions (e.g. use of REMAINDER, subparsers) a # TypeError is raised. - # - # positionals are 'deactivated' by setting nargs and default to - # SUPPRESS. This blocks the addition of that positional to the - # namespace positionals = self._get_positional_actions() a = [action for action in positionals @@ -2429,80 +2588,21 @@ def parse_known_intermixed_args(self, args=None, namespace=None): raise TypeError('parse_intermixed_args: positional arg' ' with nargs=%s'%a[0].nargs) - if [action.dest for group in self._mutually_exclusive_groups - for action in group._group_actions if action in positionals]: - raise TypeError('parse_intermixed_args: positional in' - ' mutuallyExclusiveGroup') - - try: - save_usage = self.usage - try: - if self.usage is None: - # capture the full usage for use in error messages - self.usage = self.format_usage()[7:] - for action in positionals: - # deactivate positionals - action.save_nargs = action.nargs - # action.nargs = 0 - action.nargs = SUPPRESS - action.save_default = action.default - action.default = SUPPRESS - namespace, remaining_args = self.parse_known_args(args, - namespace) - for action in positionals: - # remove the empty positional values from namespace - if (hasattr(namespace, action.dest) - and getattr(namespace, action.dest)==[]): - from warnings import warn - warn('Do not expect %s in %s' % (action.dest, namespace)) - delattr(namespace, action.dest) - finally: - # restore nargs and usage before exiting - for action in positionals: - action.nargs = action.save_nargs - action.default = action.save_default - optionals = self._get_optional_actions() - try: - # parse positionals. optionals aren't normally required, but - # they could be, so make sure they aren't. - for action in optionals: - action.save_required = action.required - action.required = False - for group in self._mutually_exclusive_groups: - group.save_required = group.required - group.required = False - namespace, extras = self.parse_known_args(remaining_args, - namespace) - finally: - # restore parser values before exiting - for action in optionals: - action.required = action.save_required - for group in self._mutually_exclusive_groups: - group.required = group.save_required - finally: - self.usage = save_usage - return namespace, extras + return self._parse_known_args2(args, namespace, intermixed=True) # ======================== # Value conversion methods # ======================== - def _get_values(self, action, arg_strings): - # for everything but PARSER, REMAINDER args, strip out first '--' - if action.nargs not in [PARSER, REMAINDER]: - try: - arg_strings.remove('--') - except ValueError: - pass + def _get_values(self, action, arg_strings): # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: if action.option_strings: value = action.const else: value = action.default - if isinstance(value, str): + if isinstance(value, str) and value is not SUPPRESS: value = self._get_value(action, value) - self._check_value(action, value) # when nargs='*' on a positional, if there were no command-line # args, use the default if it is anything other than None @@ -2510,11 +2610,8 @@ def _get_values(self, action, arg_strings): not action.option_strings): if action.default is not None: value = action.default - self._check_value(action, value) else: - # since arg_strings is always [] at this point - # there is no need to use self._check_value(action, value) - value = arg_strings + value = [] # single argument or optional argument produces a single value elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: @@ -2547,8 +2644,7 @@ def _get_values(self, action, arg_strings): def _get_value(self, action, arg_string): type_func = self._registry_get('type', action.type, action.type) if not callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) + raise TypeError(f'{type_func!r} is not callable') # convert the value to the appropriate type try: @@ -2571,15 +2667,33 @@ def _get_value(self, action, arg_string): def _check_value(self, action, value): # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - args = {'value': value, - 'choices': ', '.join(map(repr, action.choices))} + choices = action.choices + if choices is None: + return + + if isinstance(choices, str): + choices = iter(choices) + + if value not in choices: + args = {'value': str(value), + 'choices': ', '.join(map(str, action.choices))} msg = _('invalid choice: %(value)r (choose from %(choices)s)') + + if self.suggest_on_error and isinstance(value, str): + if all(isinstance(choice, str) for choice in action.choices): + import difflib + suggestions = difflib.get_close_matches(value, action.choices, 1) + if suggestions: + args['closest'] = suggestions[0] + msg = _('invalid choice: %(value)r, maybe you meant %(closest)r? ' + '(choose from %(choices)s)') + raise ArgumentError(action, msg % args) # ======================= # Help-formatting methods # ======================= + def format_usage(self): formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, @@ -2610,11 +2724,21 @@ def format_help(self): return formatter.format_help() def _get_formatter(self): - return self.formatter_class(prog=self.prog) + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(self.color) + return formatter + + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter # ===================== # Help-printing methods # ===================== + def print_usage(self, file=None): if file is None: file = _sys.stdout @@ -2636,6 +2760,7 @@ def _print_message(self, message, file=None): # =============== # Exiting methods # =============== + def exit(self, status=0, message=None): if message: self._print_message(message, _sys.stderr) @@ -2653,3 +2778,7 @@ def error(self, message): self.print_usage(_sys.stderr) args = {'prog': self.prog, 'message': message} self.exit(2, _('%(prog)s: error: %(message)s\n') % args) + + def _warning(self, message): + args = {'prog': self.prog, 'message': message} + self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr) diff --git a/Lib/ast.py b/Lib/ast.py index 07044706dc3..2f11683ecf7 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1,44 +1,38 @@ """ - ast - ~~~ - - The `ast` module helps Python applications to process trees of the Python - abstract syntax grammar. The abstract syntax itself might change with - each Python release; this module helps to find out programmatically what - the current grammar looks like and allows modifications of it. - - An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as - a flag to the `compile()` builtin function or by using the `parse()` - function from this module. The result will be a tree of objects whose - classes all inherit from `ast.AST`. - - A modified abstract syntax tree can be compiled into a Python code object - using the built-in `compile()` function. - - Additionally various helper functions are provided that make working with - the trees simpler. The main intention of the helper functions and this - module in general is to provide an easy to use interface for libraries - that work tightly with the python syntax (template engines for example). - - - :copyright: Copyright 2008 by Armin Ronacher. - :license: Python License. +The `ast` module helps Python applications to process trees of the Python +abstract syntax grammar. The abstract syntax itself might change with +each Python release; this module helps to find out programmatically what +the current grammar looks like and allows modifications of it. + +An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as +a flag to the `compile()` builtin function or by using the `parse()` +function from this module. The result will be a tree of objects whose +classes all inherit from `ast.AST`. + +A modified abstract syntax tree can be compiled into a Python code object +using the built-in `compile()` function. + +Additionally various helper functions are provided that make working with +the trees simpler. The main intention of the helper functions and this +module in general is to provide an easy to use interface for libraries +that work tightly with the python syntax (template engines for example). + +:copyright: Copyright 2008 by Armin Ronacher. +:license: Python License. """ -import sys -import re from _ast import * -from contextlib import contextmanager, nullcontext -from enum import IntEnum, auto, _simple_enum def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None): + type_comments=False, feature_version=None, optimize=-1): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). Pass type_comments=True to get back type comments where the syntax allows. """ flags = PyCF_ONLY_AST + if optimize > 0: + flags |= PyCF_OPTIMIZED_AST if type_comments: flags |= PyCF_TYPE_COMMENTS if feature_version is None: @@ -50,7 +44,7 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version) + _feature_version=feature_version, optimize=optimize) def literal_eval(node_or_string): @@ -112,7 +106,11 @@ def _convert(node): return _convert(node_or_string) -def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): +def dump( + node, annotate_fields=True, include_attributes=False, + *, + indent=None, show_empty=False, +): """ Return a formatted dump of the tree in node. This is mainly useful for debugging purposes. If annotate_fields is true (by default), @@ -123,6 +121,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): include_attributes can be set to true. If indent is a non-negative integer or string, then the tree will be pretty-printed with that indent level. None (the default) selects the single line representation. + If show_empty is False, then empty lists and fields that are None + will be omitted from the output for better readability. """ def _format(node, level=0): if indent is not None: @@ -135,6 +135,7 @@ def _format(node, level=0): if isinstance(node, AST): cls = type(node) args = [] + args_buffer = [] allsimple = True keywords = annotate_fields for name in node._fields: @@ -146,6 +147,16 @@ def _format(node, level=0): if value is None and getattr(cls, name, ...) is None: keywords = True continue + if not show_empty: + if value == []: + field_type = cls._field_types.get(name, object) + if getattr(field_type, '__origin__', ...) is list: + if not keywords: + args_buffer.append(repr(value)) + continue + if not keywords: + args.extend(args_buffer) + args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: @@ -304,12 +315,18 @@ def get_docstring(node, clean=True): return text -_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") +_line_pattern = None def _splitlines_no_ff(source, maxlines=None): """Split a string into lines ignoring form feed and other chars. This mimics how the Python parser splits source code. """ + global _line_pattern + if _line_pattern is None: + # lazily computed to speedup import time of `ast` + import re + _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + lines = [] for lineno, match in enumerate(_line_pattern.finditer(source), 1): if maxlines is not None and lineno > maxlines: @@ -380,6 +397,88 @@ def walk(node): yield node +def compare( + a, + b, + /, + *, + compare_attributes=False, +): + """Recursively compares two ASTs. + + compare_attributes affects whether AST attributes are considered + in the comparison. If compare_attributes is False (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + might differ in whitespace or similar details. + """ + + sentinel = object() # handle the possibility of a missing attribute/field + + def _compare(a, b): + # Compare two fields on an AST object, which may themselves be + # AST objects, lists of AST objects, or primitive ASDL types + # like identifiers and constants. + if isinstance(a, AST): + return compare( + a, + b, + compare_attributes=compare_attributes, + ) + elif isinstance(a, list): + # If a field is repeated, then both objects will represent + # the value as a list. + if len(a) != len(b): + return False + for a_item, b_item in zip(a, b): + if not _compare(a_item, b_item): + return False + else: + return True + else: + return type(a) is type(b) and a == b + + def _compare_fields(a, b): + if a._fields != b._fields: + return False + for field in a._fields: + a_field = getattr(a, field, sentinel) + b_field = getattr(b, field, sentinel) + if a_field is sentinel and b_field is sentinel: + # both nodes are missing a field at runtime + continue + if a_field is sentinel or b_field is sentinel: + # one of the node is missing a field + return False + if not _compare(a_field, b_field): + return False + else: + return True + + def _compare_attributes(a, b): + if a._attributes != b._attributes: + return False + # Attributes are always ints. + for attr in a._attributes: + a_attr = getattr(a, attr, sentinel) + b_attr = getattr(b, attr, sentinel) + if a_attr is sentinel and b_attr is sentinel: + # both nodes are missing an attribute at runtime + continue + if a_attr != b_attr: + return False + else: + return True + + if type(a) is not type(b): + return False + if not _compare_fields(a, b): + return False + if compare_attributes and not _compare_attributes(a, b): + return False + return True + + class NodeVisitor(object): """ A node visitor base class that walks the abstract syntax tree and calls a @@ -416,27 +515,6 @@ def generic_visit(self, node): elif isinstance(value, AST): self.visit(value) - def visit_Constant(self, node): - value = node.value - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - DeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - class NodeTransformer(NodeVisitor): """ @@ -496,151 +574,6 @@ def generic_visit(self, node): setattr(node, field, new_node) return node - -_DEPRECATED_VALUE_ALIAS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; use value instead" -) -_DEPRECATED_CLASS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; " - "use ast.Constant instead" -) - - -# If the ast module is loaded more than once, only add deprecated methods once -if not hasattr(Constant, 'n'): - # The following code is for backward compatibility. - # It will be removed in future. - - def _n_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _n_setter(self, value): - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - def _s_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _s_setter(self, value): - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - Constant.n = property(_n_getter, _n_setter) - Constant.s = property(_s_getter, _s_setter) - -class _ABC(type): - - def __init__(cls, *args): - cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" - - def __instancecheck__(cls, inst): - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", - message=_DEPRECATED_CLASS_MESSAGE, - remove=(3, 14) - ) - if not isinstance(inst, Constant): - return False - if cls in _const_types: - try: - value = inst.value - except AttributeError: - return False - else: - return ( - isinstance(value, _const_types[cls]) and - not isinstance(value, _const_types_not.get(cls, ())) - ) - return type.__instancecheck__(cls, inst) - -def _new(cls, *args, **kwargs): - for key in kwargs: - if key not in cls._fields: - # arbitrary keyword arguments are accepted - continue - pos = cls._fields.index(key) - if pos < len(args): - raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(*args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -class Num(Constant, metaclass=_ABC): - _fields = ('n',) - __new__ = _new - -class Str(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class Bytes(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class NameConstant(Constant, metaclass=_ABC): - __new__ = _new - -class Ellipsis(Constant, metaclass=_ABC): - _fields = () - - def __new__(cls, *args, **kwargs): - if cls is _ast_Ellipsis: - import warnings - warnings._deprecated( - "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(..., *args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -# Keep another reference to Ellipsis in the global namespace -# so it can be referenced in Ellipsis.__new__ -# (The original "Ellipsis" name is removed from the global namespace later on) -_ast_Ellipsis = Ellipsis - -_const_types = { - Num: (int, float, complex), - Str: (str,), - Bytes: (bytes,), - NameConstant: (type(None), bool), - Ellipsis: (type(...),), -} -_const_types_not = { - Num: (bool,), -} - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} - class slice(AST): """Deprecated AST node class.""" @@ -681,1132 +614,22 @@ class Param(expr_context): """Deprecated AST node class. Unused in Python 3.""" -# Large float and imaginary literals get turned into infinities in the AST. -# We unparse those infinities to INFSTR. -_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) - -@_simple_enum(IntEnum) -class _Precedence: - """Precedence table that originated from python grammar.""" - - NAMED_EXPR = auto() # := - TUPLE = auto() # , - YIELD = auto() # 'yield', 'yield from' - TEST = auto() # 'if'-'else', 'lambda' - OR = auto() # 'or' - AND = auto() # 'and' - NOT = auto() # 'not' - CMP = auto() # '<', '>', '==', '>=', '<=', '!=', - # 'in', 'not in', 'is', 'is not' - EXPR = auto() - BOR = EXPR # '|' - BXOR = auto() # '^' - BAND = auto() # '&' - SHIFT = auto() # '<<', '>>' - ARITH = auto() # '+', '-' - TERM = auto() # '*', '@', '/', '%', '//' - FACTOR = auto() # unary '+', '-', '~' - POWER = auto() # '**' - AWAIT = auto() # 'await' - ATOM = auto() - - def next(self): - try: - return self.__class__(self + 1) - except ValueError: - return self - - -_SINGLE_QUOTES = ("'", '"') -_MULTI_QUOTES = ('"""', "'''") -_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) - -class _Unparser(NodeVisitor): - """Methods in this class recursively traverse an AST and - output source code for the abstract syntax; original formatting - is disregarded.""" - - def __init__(self, *, _avoid_backslashes=False): - self._source = [] - self._precedences = {} - self._type_ignores = {} - self._indent = 0 - self._avoid_backslashes = _avoid_backslashes - self._in_try_star = False - - def interleave(self, inter, f, seq): - """Call f on each item in seq, calling inter() in between.""" - seq = iter(seq) - try: - f(next(seq)) - except StopIteration: - pass - else: - for x in seq: - inter() - f(x) - - def items_view(self, traverser, items): - """Traverse and separate the given *items* with a comma and append it to - the buffer. If *items* is a single item sequence, a trailing comma - will be added.""" - if len(items) == 1: - traverser(items[0]) - self.write(",") - else: - self.interleave(lambda: self.write(", "), traverser, items) - - def maybe_newline(self): - """Adds a newline if it isn't the start of generated source""" - if self._source: - self.write("\n") - - def fill(self, text=""): - """Indent a piece of text and append it, according to the current - indentation level""" - self.maybe_newline() - self.write(" " * self._indent + text) - - def write(self, *text): - """Add new source parts""" - self._source.extend(text) - - @contextmanager - def buffered(self, buffer = None): - if buffer is None: - buffer = [] - - original_source = self._source - self._source = buffer - yield buffer - self._source = original_source - - @contextmanager - def block(self, *, extra = None): - """A context manager for preparing the source for blocks. It adds - the character':', increases the indentation on enter and decreases - the indentation on exit. If *extra* is given, it will be directly - appended after the colon character. - """ - self.write(":") - if extra: - self.write(extra) - self._indent += 1 - yield - self._indent -= 1 - - @contextmanager - def delimit(self, start, end): - """A context manager for preparing the source for expressions. It adds - *start* to the buffer and enters, after exit it adds *end*.""" - - self.write(start) - yield - self.write(end) - - def delimit_if(self, start, end, condition): - if condition: - return self.delimit(start, end) - else: - return nullcontext() - - def require_parens(self, precedence, node): - """Shortcut to adding precedence related parens""" - return self.delimit_if("(", ")", self.get_precedence(node) > precedence) - - def get_precedence(self, node): - return self._precedences.get(node, _Precedence.TEST) - - def set_precedence(self, precedence, *nodes): - for node in nodes: - self._precedences[node] = precedence - - def get_raw_docstring(self, node): - """If a docstring node is found in the body of the *node* parameter, - return that docstring node, None otherwise. - - Logic mirrored from ``_PyAST_GetDocString``.""" - if not isinstance( - node, (AsyncFunctionDef, FunctionDef, ClassDef, Module) - ) or len(node.body) < 1: - return None - node = node.body[0] - if not isinstance(node, Expr): - return None - node = node.value - if isinstance(node, Constant) and isinstance(node.value, str): - return node - - def get_type_comment(self, node): - comment = self._type_ignores.get(node.lineno) or node.type_comment - if comment is not None: - return f" # type: {comment}" - - def traverse(self, node): - if isinstance(node, list): - for item in node: - self.traverse(item) - else: - super().visit(node) - - # Note: as visit() resets the output text, do NOT rely on - # NodeVisitor.generic_visit to handle any nodes (as it calls back in to - # the subclass visit() method, which resets self._source to an empty list) - def visit(self, node): - """Outputs a source code string that, if converted back to an ast - (using ast.parse) will generate an AST equivalent to *node*""" - self._source = [] - self.traverse(node) - return "".join(self._source) - - def _write_docstring_and_traverse_body(self, node): - if (docstring := self.get_raw_docstring(node)): - self._write_docstring(docstring) - self.traverse(node.body[1:]) - else: - self.traverse(node.body) - - def visit_Module(self, node): - self._type_ignores = { - ignore.lineno: f"ignore{ignore.tag}" - for ignore in node.type_ignores - } - self._write_docstring_and_traverse_body(node) - self._type_ignores.clear() - - def visit_FunctionType(self, node): - with self.delimit("(", ")"): - self.interleave( - lambda: self.write(", "), self.traverse, node.argtypes - ) - - self.write(" -> ") - self.traverse(node.returns) - - def visit_Expr(self, node): - self.fill() - self.set_precedence(_Precedence.YIELD, node.value) - self.traverse(node.value) - - def visit_NamedExpr(self, node): - with self.require_parens(_Precedence.NAMED_EXPR, node): - self.set_precedence(_Precedence.ATOM, node.target, node.value) - self.traverse(node.target) - self.write(" := ") - self.traverse(node.value) - - def visit_Import(self, node): - self.fill("import ") - self.interleave(lambda: self.write(", "), self.traverse, node.names) - - def visit_ImportFrom(self, node): - self.fill("from ") - self.write("." * (node.level or 0)) - if node.module: - self.write(node.module) - self.write(" import ") - self.interleave(lambda: self.write(", "), self.traverse, node.names) - - def visit_Assign(self, node): - self.fill() - for target in node.targets: - self.set_precedence(_Precedence.TUPLE, target) - self.traverse(target) - self.write(" = ") - self.traverse(node.value) - if type_comment := self.get_type_comment(node): - self.write(type_comment) - - def visit_AugAssign(self, node): - self.fill() - self.traverse(node.target) - self.write(" " + self.binop[node.op.__class__.__name__] + "= ") - self.traverse(node.value) - - def visit_AnnAssign(self, node): - self.fill() - with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)): - self.traverse(node.target) - self.write(": ") - self.traverse(node.annotation) - if node.value: - self.write(" = ") - self.traverse(node.value) - - def visit_Return(self, node): - self.fill("return") - if node.value: - self.write(" ") - self.traverse(node.value) - - def visit_Pass(self, node): - self.fill("pass") - - def visit_Break(self, node): - self.fill("break") - - def visit_Continue(self, node): - self.fill("continue") - - def visit_Delete(self, node): - self.fill("del ") - self.interleave(lambda: self.write(", "), self.traverse, node.targets) - - def visit_Assert(self, node): - self.fill("assert ") - self.traverse(node.test) - if node.msg: - self.write(", ") - self.traverse(node.msg) - - def visit_Global(self, node): - self.fill("global ") - self.interleave(lambda: self.write(", "), self.write, node.names) - - def visit_Nonlocal(self, node): - self.fill("nonlocal ") - self.interleave(lambda: self.write(", "), self.write, node.names) - - def visit_Await(self, node): - with self.require_parens(_Precedence.AWAIT, node): - self.write("await") - if node.value: - self.write(" ") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_Yield(self, node): - with self.require_parens(_Precedence.YIELD, node): - self.write("yield") - if node.value: - self.write(" ") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_YieldFrom(self, node): - with self.require_parens(_Precedence.YIELD, node): - self.write("yield from ") - if not node.value: - raise ValueError("Node can't be used without a value attribute.") - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - - def visit_Raise(self, node): - self.fill("raise") - if not node.exc: - if node.cause: - raise ValueError(f"Node can't use cause without an exception.") - return - self.write(" ") - self.traverse(node.exc) - if node.cause: - self.write(" from ") - self.traverse(node.cause) - - def do_visit_try(self, node): - self.fill("try") - with self.block(): - self.traverse(node.body) - for ex in node.handlers: - self.traverse(ex) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - if node.finalbody: - self.fill("finally") - with self.block(): - self.traverse(node.finalbody) - - def visit_Try(self, node): - prev_in_try_star = self._in_try_star - try: - self._in_try_star = False - self.do_visit_try(node) - finally: - self._in_try_star = prev_in_try_star - - def visit_TryStar(self, node): - prev_in_try_star = self._in_try_star - try: - self._in_try_star = True - self.do_visit_try(node) - finally: - self._in_try_star = prev_in_try_star - - def visit_ExceptHandler(self, node): - self.fill("except*" if self._in_try_star else "except") - if node.type: - self.write(" ") - self.traverse(node.type) - if node.name: - self.write(" as ") - self.write(node.name) - with self.block(): - self.traverse(node.body) - - def visit_ClassDef(self, node): - self.maybe_newline() - for deco in node.decorator_list: - self.fill("@") - self.traverse(deco) - self.fill("class " + node.name) - if hasattr(node, "type_params"): - self._type_params_helper(node.type_params) - with self.delimit_if("(", ")", condition = node.bases or node.keywords): - comma = False - for e in node.bases: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - for e in node.keywords: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - - with self.block(): - self._write_docstring_and_traverse_body(node) - - def visit_FunctionDef(self, node): - self._function_helper(node, "def") - - def visit_AsyncFunctionDef(self, node): - self._function_helper(node, "async def") - - def _function_helper(self, node, fill_suffix): - self.maybe_newline() - for deco in node.decorator_list: - self.fill("@") - self.traverse(deco) - def_str = fill_suffix + " " + node.name - self.fill(def_str) - if hasattr(node, "type_params"): - self._type_params_helper(node.type_params) - with self.delimit("(", ")"): - self.traverse(node.args) - if node.returns: - self.write(" -> ") - self.traverse(node.returns) - with self.block(extra=self.get_type_comment(node)): - self._write_docstring_and_traverse_body(node) - - def _type_params_helper(self, type_params): - if type_params is not None and len(type_params) > 0: - with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, type_params) - - def visit_TypeVar(self, node): - self.write(node.name) - if node.bound: - self.write(": ") - self.traverse(node.bound) - - def visit_TypeVarTuple(self, node): - self.write("*" + node.name) - - def visit_ParamSpec(self, node): - self.write("**" + node.name) - - def visit_TypeAlias(self, node): - self.fill("type ") - self.traverse(node.name) - self._type_params_helper(node.type_params) - self.write(" = ") - self.traverse(node.value) - - def visit_For(self, node): - self._for_helper("for ", node) - - def visit_AsyncFor(self, node): - self._for_helper("async for ", node) - - def _for_helper(self, fill, node): - self.fill(fill) - self.set_precedence(_Precedence.TUPLE, node.target) - self.traverse(node.target) - self.write(" in ") - self.traverse(node.iter) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_If(self, node): - self.fill("if ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - # collapse nested ifs into equivalent elifs. - while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If): - node = node.orelse[0] - self.fill("elif ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - # final else - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_While(self, node): - self.fill("while ") - self.traverse(node.test) - with self.block(): - self.traverse(node.body) - if node.orelse: - self.fill("else") - with self.block(): - self.traverse(node.orelse) - - def visit_With(self, node): - self.fill("with ") - self.interleave(lambda: self.write(", "), self.traverse, node.items) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - - def visit_AsyncWith(self, node): - self.fill("async with ") - self.interleave(lambda: self.write(", "), self.traverse, node.items) - with self.block(extra=self.get_type_comment(node)): - self.traverse(node.body) - - def _str_literal_helper( - self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False - ): - """Helper for writing string literals, minimizing escapes. - Returns the tuple (string literal to write, possible quote types). - """ - def escape_char(c): - # \n and \t are non-printable, but we only escape them if - # escape_special_whitespace is True - if not escape_special_whitespace and c in "\n\t": - return c - # Always escape backslashes and other non-printable characters - if c == "\\" or not c.isprintable(): - return c.encode("unicode_escape").decode("ascii") - return c - - escaped_string = "".join(map(escape_char, string)) - possible_quotes = quote_types - if "\n" in escaped_string: - possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] - possible_quotes = [q for q in possible_quotes if q not in escaped_string] - if not possible_quotes: - # If there aren't any possible_quotes, fallback to using repr - # on the original string. Try to use a quote from quote_types, - # e.g., so that we use triple quotes for docstrings. - string = repr(string) - quote = next((q for q in quote_types if string[0] in q), string[0]) - return string[1:-1], [quote] - if escaped_string: - # Sort so that we prefer '''"''' over """\"""" - possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) - # If we're using triple quotes and we'd need to escape a final - # quote, escape it - if possible_quotes[0][0] == escaped_string[-1]: - assert len(possible_quotes[0]) == 3 - escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] - return escaped_string, possible_quotes - - def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): - """Write string literal value with a best effort attempt to avoid backslashes.""" - string, quote_types = self._str_literal_helper(string, quote_types=quote_types) - quote_type = quote_types[0] - self.write(f"{quote_type}{string}{quote_type}") - - def visit_JoinedStr(self, node): - self.write("f") - - fstring_parts = [] - for value in node.values: - with self.buffered() as buffer: - self._write_fstring_inner(value) - fstring_parts.append( - ("".join(buffer), isinstance(value, Constant)) - ) - - new_fstring_parts = [] - quote_types = list(_ALL_QUOTES) - fallback_to_repr = False - for value, is_constant in fstring_parts: - if is_constant: - value, new_quote_types = self._str_literal_helper( - value, - quote_types=quote_types, - escape_special_whitespace=True, - ) - if set(new_quote_types).isdisjoint(quote_types): - fallback_to_repr = True - break - quote_types = new_quote_types - elif "\n" in value: - quote_types = [q for q in quote_types if q in _MULTI_QUOTES] - assert quote_types - new_fstring_parts.append(value) - - if fallback_to_repr: - # If we weren't able to find a quote type that works for all parts - # of the JoinedStr, fallback to using repr and triple single quotes. - quote_types = ["'''"] - new_fstring_parts.clear() - for value, is_constant in fstring_parts: - if is_constant: - value = repr('"' + value) # force repr to use single quotes - expected_prefix = "'\"" - assert value.startswith(expected_prefix), repr(value) - value = value[len(expected_prefix):-1] - new_fstring_parts.append(value) - - value = "".join(new_fstring_parts) - quote_type = quote_types[0] - self.write(f"{quote_type}{value}{quote_type}") - - def _write_fstring_inner(self, node): - if isinstance(node, JoinedStr): - # for both the f-string itself, and format_spec - for value in node.values: - self._write_fstring_inner(value) - elif isinstance(node, Constant) and isinstance(node.value, str): - value = node.value.replace("{", "{{").replace("}", "}}") - self.write(value) - elif isinstance(node, FormattedValue): - self.visit_FormattedValue(node) - else: - raise ValueError(f"Unexpected node inside JoinedStr, {node!r}") - - def visit_FormattedValue(self, node): - def unparse_inner(inner): - unparser = type(self)() - unparser.set_precedence(_Precedence.TEST.next(), inner) - return unparser.visit(inner) - - with self.delimit("{", "}"): - expr = unparse_inner(node.value) - if expr.startswith("{"): - # Separate pair of opening brackets as "{ {" - self.write(" ") - self.write(expr) - if node.conversion != -1: - self.write(f"!{chr(node.conversion)}") - if node.format_spec: - self.write(":") - self._write_fstring_inner(node.format_spec) - - def visit_Name(self, node): - self.write(node.id) - - def _write_docstring(self, node): - self.fill() - if node.kind == "u": - self.write("u") - self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) - - def _write_constant(self, value): - if isinstance(value, (float, complex)): - # Substitute overflowing decimal literal for AST infinities, - # and inf - inf for NaNs. - self.write( - repr(value) - .replace("inf", _INFSTR) - .replace("nan", f"({_INFSTR}-{_INFSTR})") - ) - elif self._avoid_backslashes and isinstance(value, str): - self._write_str_avoiding_backslashes(value) - else: - self.write(repr(value)) - - def visit_Constant(self, node): - value = node.value - if isinstance(value, tuple): - with self.delimit("(", ")"): - self.items_view(self._write_constant, value) - elif value is ...: - self.write("...") - else: - if node.kind == "u": - self.write("u") - self._write_constant(node.value) - - def visit_List(self, node): - with self.delimit("[", "]"): - self.interleave(lambda: self.write(", "), self.traverse, node.elts) - - def visit_ListComp(self, node): - with self.delimit("[", "]"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_GeneratorExp(self, node): - with self.delimit("(", ")"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_SetComp(self, node): - with self.delimit("{", "}"): - self.traverse(node.elt) - for gen in node.generators: - self.traverse(gen) - - def visit_DictComp(self, node): - with self.delimit("{", "}"): - self.traverse(node.key) - self.write(": ") - self.traverse(node.value) - for gen in node.generators: - self.traverse(gen) - - def visit_comprehension(self, node): - if node.is_async: - self.write(" async for ") - else: - self.write(" for ") - self.set_precedence(_Precedence.TUPLE, node.target) - self.traverse(node.target) - self.write(" in ") - self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs) - self.traverse(node.iter) - for if_clause in node.ifs: - self.write(" if ") - self.traverse(if_clause) - - def visit_IfExp(self, node): - with self.require_parens(_Precedence.TEST, node): - self.set_precedence(_Precedence.TEST.next(), node.body, node.test) - self.traverse(node.body) - self.write(" if ") - self.traverse(node.test) - self.write(" else ") - self.set_precedence(_Precedence.TEST, node.orelse) - self.traverse(node.orelse) - - def visit_Set(self, node): - if node.elts: - with self.delimit("{", "}"): - self.interleave(lambda: self.write(", "), self.traverse, node.elts) - else: - # `{}` would be interpreted as a dictionary literal, and - # `set` might be shadowed. Thus: - self.write('{*()}') - - def visit_Dict(self, node): - def write_key_value_pair(k, v): - self.traverse(k) - self.write(": ") - self.traverse(v) - - def write_item(item): - k, v = item - if k is None: - # for dictionary unpacking operator in dicts {**{'y': 2}} - # see PEP 448 for details - self.write("**") - self.set_precedence(_Precedence.EXPR, v) - self.traverse(v) - else: - write_key_value_pair(k, v) - - with self.delimit("{", "}"): - self.interleave( - lambda: self.write(", "), write_item, zip(node.keys, node.values) - ) - - def visit_Tuple(self, node): - with self.delimit_if( - "(", - ")", - len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE - ): - self.items_view(self.traverse, node.elts) - - unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"} - unop_precedence = { - "not": _Precedence.NOT, - "~": _Precedence.FACTOR, - "+": _Precedence.FACTOR, - "-": _Precedence.FACTOR, - } - - def visit_UnaryOp(self, node): - operator = self.unop[node.op.__class__.__name__] - operator_precedence = self.unop_precedence[operator] - with self.require_parens(operator_precedence, node): - self.write(operator) - # factor prefixes (+, -, ~) shouldn't be separated - # from the value they belong, (e.g: +1 instead of + 1) - if operator_precedence is not _Precedence.FACTOR: - self.write(" ") - self.set_precedence(operator_precedence, node.operand) - self.traverse(node.operand) - - binop = { - "Add": "+", - "Sub": "-", - "Mult": "*", - "MatMult": "@", - "Div": "/", - "Mod": "%", - "LShift": "<<", - "RShift": ">>", - "BitOr": "|", - "BitXor": "^", - "BitAnd": "&", - "FloorDiv": "//", - "Pow": "**", - } - - binop_precedence = { - "+": _Precedence.ARITH, - "-": _Precedence.ARITH, - "*": _Precedence.TERM, - "@": _Precedence.TERM, - "/": _Precedence.TERM, - "%": _Precedence.TERM, - "<<": _Precedence.SHIFT, - ">>": _Precedence.SHIFT, - "|": _Precedence.BOR, - "^": _Precedence.BXOR, - "&": _Precedence.BAND, - "//": _Precedence.TERM, - "**": _Precedence.POWER, - } - - binop_rassoc = frozenset(("**",)) - def visit_BinOp(self, node): - operator = self.binop[node.op.__class__.__name__] - operator_precedence = self.binop_precedence[operator] - with self.require_parens(operator_precedence, node): - if operator in self.binop_rassoc: - left_precedence = operator_precedence.next() - right_precedence = operator_precedence - else: - left_precedence = operator_precedence - right_precedence = operator_precedence.next() - - self.set_precedence(left_precedence, node.left) - self.traverse(node.left) - self.write(f" {operator} ") - self.set_precedence(right_precedence, node.right) - self.traverse(node.right) - - cmpops = { - "Eq": "==", - "NotEq": "!=", - "Lt": "<", - "LtE": "<=", - "Gt": ">", - "GtE": ">=", - "Is": "is", - "IsNot": "is not", - "In": "in", - "NotIn": "not in", - } - - def visit_Compare(self, node): - with self.require_parens(_Precedence.CMP, node): - self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators) - self.traverse(node.left) - for o, e in zip(node.ops, node.comparators): - self.write(" " + self.cmpops[o.__class__.__name__] + " ") - self.traverse(e) - - boolops = {"And": "and", "Or": "or"} - boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR} - - def visit_BoolOp(self, node): - operator = self.boolops[node.op.__class__.__name__] - operator_precedence = self.boolop_precedence[operator] - - def increasing_level_traverse(node): - nonlocal operator_precedence - operator_precedence = operator_precedence.next() - self.set_precedence(operator_precedence, node) - self.traverse(node) - - with self.require_parens(operator_precedence, node): - s = f" {operator} " - self.interleave(lambda: self.write(s), increasing_level_traverse, node.values) - - def visit_Attribute(self, node): - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - # Special case: 3.__abs__() is a syntax error, so if node.value - # is an integer literal then we need to either parenthesize - # it or add an extra space to get 3 .__abs__(). - if isinstance(node.value, Constant) and isinstance(node.value.value, int): - self.write(" ") - self.write(".") - self.write(node.attr) - - def visit_Call(self, node): - self.set_precedence(_Precedence.ATOM, node.func) - self.traverse(node.func) - with self.delimit("(", ")"): - comma = False - for e in node.args: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - for e in node.keywords: - if comma: - self.write(", ") - else: - comma = True - self.traverse(e) - - def visit_Subscript(self, node): - def is_non_empty_tuple(slice_value): - return ( - isinstance(slice_value, Tuple) - and slice_value.elts - ) - - self.set_precedence(_Precedence.ATOM, node.value) - self.traverse(node.value) - with self.delimit("[", "]"): - if is_non_empty_tuple(node.slice): - # parentheses can be omitted if the tuple isn't empty - self.items_view(self.traverse, node.slice.elts) - else: - self.traverse(node.slice) - - def visit_Starred(self, node): - self.write("*") - self.set_precedence(_Precedence.EXPR, node.value) - self.traverse(node.value) - - def visit_Ellipsis(self, node): - self.write("...") - - def visit_Slice(self, node): - if node.lower: - self.traverse(node.lower) - self.write(":") - if node.upper: - self.traverse(node.upper) - if node.step: - self.write(":") - self.traverse(node.step) - - def visit_Match(self, node): - self.fill("match ") - self.traverse(node.subject) - with self.block(): - for case in node.cases: - self.traverse(case) - - def visit_arg(self, node): - self.write(node.arg) - if node.annotation: - self.write(": ") - self.traverse(node.annotation) - - def visit_arguments(self, node): - first = True - # normal arguments - all_args = node.posonlyargs + node.args - defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults - for index, elements in enumerate(zip(all_args, defaults), 1): - a, d = elements - if first: - first = False - else: - self.write(", ") - self.traverse(a) - if d: - self.write("=") - self.traverse(d) - if index == len(node.posonlyargs): - self.write(", /") - - # varargs, or bare '*' if no varargs but keyword-only arguments present - if node.vararg or node.kwonlyargs: - if first: - first = False - else: - self.write(", ") - self.write("*") - if node.vararg: - self.write(node.vararg.arg) - if node.vararg.annotation: - self.write(": ") - self.traverse(node.vararg.annotation) - - # keyword-only arguments - if node.kwonlyargs: - for a, d in zip(node.kwonlyargs, node.kw_defaults): - self.write(", ") - self.traverse(a) - if d: - self.write("=") - self.traverse(d) - - # kwargs - if node.kwarg: - if first: - first = False - else: - self.write(", ") - self.write("**" + node.kwarg.arg) - if node.kwarg.annotation: - self.write(": ") - self.traverse(node.kwarg.annotation) - - def visit_keyword(self, node): - if node.arg is None: - self.write("**") - else: - self.write(node.arg) - self.write("=") - self.traverse(node.value) - - def visit_Lambda(self, node): - with self.require_parens(_Precedence.TEST, node): - self.write("lambda") - with self.buffered() as buffer: - self.traverse(node.args) - if buffer: - self.write(" ", *buffer) - self.write(": ") - self.set_precedence(_Precedence.TEST, node.body) - self.traverse(node.body) - - def visit_alias(self, node): - self.write(node.name) - if node.asname: - self.write(" as " + node.asname) - - def visit_withitem(self, node): - self.traverse(node.context_expr) - if node.optional_vars: - self.write(" as ") - self.traverse(node.optional_vars) - - def visit_match_case(self, node): - self.fill("case ") - self.traverse(node.pattern) - if node.guard: - self.write(" if ") - self.traverse(node.guard) - with self.block(): - self.traverse(node.body) - - def visit_MatchValue(self, node): - self.traverse(node.value) - - def visit_MatchSingleton(self, node): - self._write_constant(node.value) - - def visit_MatchSequence(self, node): - with self.delimit("[", "]"): - self.interleave( - lambda: self.write(", "), self.traverse, node.patterns - ) - - def visit_MatchStar(self, node): - name = node.name - if name is None: - name = "_" - self.write(f"*{name}") - - def visit_MatchMapping(self, node): - def write_key_pattern_pair(pair): - k, p = pair - self.traverse(k) - self.write(": ") - self.traverse(p) - - with self.delimit("{", "}"): - keys = node.keys - self.interleave( - lambda: self.write(", "), - write_key_pattern_pair, - zip(keys, node.patterns, strict=True), - ) - rest = node.rest - if rest is not None: - if keys: - self.write(", ") - self.write(f"**{rest}") - - def visit_MatchClass(self, node): - self.set_precedence(_Precedence.ATOM, node.cls) - self.traverse(node.cls) - with self.delimit("(", ")"): - patterns = node.patterns - self.interleave( - lambda: self.write(", "), self.traverse, patterns - ) - attrs = node.kwd_attrs - if attrs: - def write_attr_pattern(pair): - attr, pattern = pair - self.write(f"{attr}=") - self.traverse(pattern) - - if patterns: - self.write(", ") - self.interleave( - lambda: self.write(", "), - write_attr_pattern, - zip(attrs, node.kwd_patterns, strict=True), - ) - - def visit_MatchAs(self, node): - name = node.name - pattern = node.pattern - if name is None: - self.write("_") - elif pattern is None: - self.write(node.name) - else: - with self.require_parens(_Precedence.TEST, node): - self.set_precedence(_Precedence.BOR, node.pattern) - self.traverse(node.pattern) - self.write(f" as {node.name}") - - def visit_MatchOr(self, node): - with self.require_parens(_Precedence.BOR, node): - self.set_precedence(_Precedence.BOR.next(), *node.patterns) - self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) - def unparse(ast_obj): - unparser = _Unparser() + global _Unparser + try: + unparser = _Unparser() + except NameError: + from _ast_unparse import Unparser as _Unparser + unparser = _Unparser() return unparser.visit(ast_obj) -_deprecated_globals = { - name: globals().pop(name) - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') -} - -def __getattr__(name): - if name in _deprecated_globals: - globals()[name] = value = _deprecated_globals[name] - import warnings - warnings._deprecated( - f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return value - raise AttributeError(f"module 'ast' has no attribute '{name}'") - - -def main(): +def main(args=None): import argparse + import sys - parser = argparse.ArgumentParser(prog='python -m ast') - parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?', - default='-', + parser = argparse.ArgumentParser(color=True) + parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), @@ -1818,12 +641,40 @@ def main(): 'column offsets') parser.add_argument('-i', '--indent', type=int, default=3, help='indentation of nodes (number of spaces)') - args = parser.parse_args() + parser.add_argument('--feature-version', + type=str, default=None, metavar='VERSION', + help='Python version in the format 3.x ' + '(for example, 3.10)') + parser.add_argument('-O', '--optimize', + type=int, default=-1, metavar='LEVEL', + help='optimization level for parser (default -1)') + parser.add_argument('--show-empty', default=False, action='store_true', + help='show empty lists and fields in dump output') + args = parser.parse_args(args) + + if args.infile == '-': + name = '' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + + # Process feature_version + feature_version = None + if args.feature_version: + try: + major, minor = map(int, args.feature_version.split('.', 1)) + except ValueError: + parser.error('Invalid format for --feature-version; ' + 'expected format 3.x (for example, 3.10)') + + feature_version = (major, minor) - with args.infile as infile: - source = infile.read() - tree = parse(source, args.infile.name, args.mode, type_comments=args.no_type_comments) - print(dump(tree, include_attributes=args.include_attributes, indent=args.indent)) + tree = parse(source, name, args.mode, type_comments=args.no_type_comments, + feature_version=feature_version, optimize=args.optimize) + print(dump(tree, include_attributes=args.include_attributes, + indent=args.indent, show_empty=args.show_empty)) if __name__ == '__main__': main() diff --git a/Lib/asynchat.py b/Lib/asynchat.py deleted file mode 100644 index fc1146adbb1..00000000000 --- a/Lib/asynchat.py +++ /dev/null @@ -1,307 +0,0 @@ -# -*- Mode: Python; tab-width: 4 -*- -# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp -# Author: Sam Rushing - -# ====================================================================== -# Copyright 1996 by Sam Rushing -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Sam -# Rushing not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# ====================================================================== - -r"""A class supporting chat-style (command/response) protocols. - -This class adds support for 'chat' style protocols - where one side -sends a 'command', and the other sends a response (examples would be -the common internet protocols - smtp, nntp, ftp, etc..). - -The handle_read() method looks at the input stream for the current -'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n' -for multi-line output), calling self.found_terminator() on its -receipt. - -for example: -Say you build an async nntp client using this class. At the start -of the connection, you'll have self.terminator set to '\r\n', in -order to process the single-line greeting. Just before issuing a -'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST -command will be accumulated (using your own 'collect_incoming_data' -method) up to the terminator, and then control will be returned to -you - by calling your self.found_terminator() method. -""" -import asyncore -from collections import deque - - -class async_chat(asyncore.dispatcher): - """This is an abstract class. You must derive from this class, and add - the two methods collect_incoming_data() and found_terminator()""" - - # these are overridable defaults - - ac_in_buffer_size = 65536 - ac_out_buffer_size = 65536 - - # we don't want to enable the use of encoding by default, because that is a - # sign of an application bug that we don't want to pass silently - - use_encoding = 0 - encoding = 'latin-1' - - def __init__(self, sock=None, map=None): - # for string terminator matching - self.ac_in_buffer = b'' - - # we use a list here rather than io.BytesIO for a few reasons... - # del lst[:] is faster than bio.truncate(0) - # lst = [] is faster than bio.truncate(0) - self.incoming = [] - - # we toss the use of the "simple producer" and replace it with - # a pure deque, which the original fifo was a wrapping of - self.producer_fifo = deque() - asyncore.dispatcher.__init__(self, sock, map) - - def collect_incoming_data(self, data): - raise NotImplementedError("must be implemented in subclass") - - def _collect_incoming_data(self, data): - self.incoming.append(data) - - def _get_data(self): - d = b''.join(self.incoming) - del self.incoming[:] - return d - - def found_terminator(self): - raise NotImplementedError("must be implemented in subclass") - - def set_terminator(self, term): - """Set the input delimiter. - - Can be a fixed string of any length, an integer, or None. - """ - if isinstance(term, str) and self.use_encoding: - term = bytes(term, self.encoding) - elif isinstance(term, int) and term < 0: - raise ValueError('the number of received bytes must be positive') - self.terminator = term - - def get_terminator(self): - return self.terminator - - # grab some more data from the socket, - # throw it to the collector method, - # check for the terminator, - # if found, transition to the next state. - - def handle_read(self): - - try: - data = self.recv(self.ac_in_buffer_size) - except BlockingIOError: - return - except OSError as why: - self.handle_error() - return - - if isinstance(data, str) and self.use_encoding: - data = bytes(str, self.encoding) - self.ac_in_buffer = self.ac_in_buffer + data - - # Continue to search for self.terminator in self.ac_in_buffer, - # while calling self.collect_incoming_data. The while loop - # is necessary because we might read several data+terminator - # combos with a single recv(4096). - - while self.ac_in_buffer: - lb = len(self.ac_in_buffer) - terminator = self.get_terminator() - if not terminator: - # no terminator, collect it all - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - elif isinstance(terminator, int): - # numeric terminator - n = terminator - if lb < n: - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - self.terminator = self.terminator - lb - else: - self.collect_incoming_data(self.ac_in_buffer[:n]) - self.ac_in_buffer = self.ac_in_buffer[n:] - self.terminator = 0 - self.found_terminator() - else: - # 3 cases: - # 1) end of buffer matches terminator exactly: - # collect data, transition - # 2) end of buffer matches some prefix: - # collect data to the prefix - # 3) end of buffer does not match any prefix: - # collect data - terminator_len = len(terminator) - index = self.ac_in_buffer.find(terminator) - if index != -1: - # we found the terminator - if index > 0: - # don't bother reporting the empty string - # (source of subtle bugs) - self.collect_incoming_data(self.ac_in_buffer[:index]) - self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:] - # This does the Right Thing if the terminator - # is changed here. - self.found_terminator() - else: - # check for a prefix of the terminator - index = find_prefix_at_end(self.ac_in_buffer, terminator) - if index: - if index != lb: - # we found a prefix, collect up to the prefix - self.collect_incoming_data(self.ac_in_buffer[:-index]) - self.ac_in_buffer = self.ac_in_buffer[-index:] - break - else: - # no prefix, collect it all - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - - def handle_write(self): - self.initiate_send() - - def handle_close(self): - self.close() - - def push(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) - sabs = self.ac_out_buffer_size - if len(data) > sabs: - for i in range(0, len(data), sabs): - self.producer_fifo.append(data[i:i+sabs]) - else: - self.producer_fifo.append(data) - self.initiate_send() - - def push_with_producer(self, producer): - self.producer_fifo.append(producer) - self.initiate_send() - - def readable(self): - "predicate for inclusion in the readable for select()" - # cannot use the old predicate, it violates the claim of the - # set_terminator method. - - # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size) - return 1 - - def writable(self): - "predicate for inclusion in the writable for select()" - return self.producer_fifo or (not self.connected) - - def close_when_done(self): - "automatically close this channel once the outgoing queue is empty" - self.producer_fifo.append(None) - - def initiate_send(self): - while self.producer_fifo and self.connected: - first = self.producer_fifo[0] - # handle empty string/buffer or None entry - if not first: - del self.producer_fifo[0] - if first is None: - self.handle_close() - return - - # handle classic producer behavior - obs = self.ac_out_buffer_size - try: - data = first[:obs] - except TypeError: - data = first.more() - if data: - self.producer_fifo.appendleft(data) - else: - del self.producer_fifo[0] - continue - - if isinstance(data, str) and self.use_encoding: - data = bytes(data, self.encoding) - - # send the data - try: - num_sent = self.send(data) - except OSError: - self.handle_error() - return - - if num_sent: - if num_sent < len(data) or obs < len(first): - self.producer_fifo[0] = first[num_sent:] - else: - del self.producer_fifo[0] - # we tried to send some actual data - return - - def discard_buffers(self): - # Emergencies only! - self.ac_in_buffer = b'' - del self.incoming[:] - self.producer_fifo.clear() - - -class simple_producer: - - def __init__(self, data, buffer_size=512): - self.data = data - self.buffer_size = buffer_size - - def more(self): - if len(self.data) > self.buffer_size: - result = self.data[:self.buffer_size] - self.data = self.data[self.buffer_size:] - return result - else: - result = self.data - self.data = b'' - return result - - -# Given 'haystack', see if any prefix of 'needle' is at its end. This -# assumes an exact match has already been checked. Return the number of -# characters matched. -# for example: -# f_p_a_e("qwerty\r", "\r\n") => 1 -# f_p_a_e("qwertydkjf", "\r\n") => 0 -# f_p_a_e("qwerty\r\n", "\r\n") => - -# this could maybe be made faster with a computed regex? -# [answer: no; circa Python-2.0, Jan 2001] -# new python: 28961/s -# old python: 18307/s -# re: 12820/s -# regex: 14035/s - -def find_prefix_at_end(haystack, needle): - l = len(needle) - 1 - while l and not haystack.endswith(needle[:l]): - l -= 1 - return l diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index ff69378ba97..32a5dbae03a 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -1,23 +1,16 @@ """The asyncio package, tracking PEP 3156.""" # flake8: noqa -import sys - -import selectors -# XXX RustPython TODO: _overlapped -if sys.platform == 'win32' and False: - # Similar thing for _overlapped. - try: - from . import _overlapped - except ImportError: - import _overlapped # Will also be exported. +import sys # This relies on each of the submodules having an __all__ variable. from .base_events import * from .coroutines import * from .events import * +from .exceptions import * from .futures import * +from .graph import * from .locks import * from .protocols import * from .runners import * @@ -25,12 +18,17 @@ from .streams import * from .subprocess import * from .tasks import * +from .taskgroups import * +from .timeouts import * +from .threads import * from .transports import * __all__ = (base_events.__all__ + coroutines.__all__ + events.__all__ + + exceptions.__all__ + futures.__all__ + + graph.__all__ + locks.__all__ + protocols.__all__ + runners.__all__ + @@ -38,6 +36,9 @@ streams.__all__ + subprocess.__all__ + tasks.__all__ + + taskgroups.__all__ + + threads.__all__ + + timeouts.__all__ + transports.__all__) if sys.platform == 'win32': # pragma: no cover @@ -46,3 +47,28 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ + +def __getattr__(name: str): + import warnings + + match name: + case "AbstractEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return events._AbstractEventLoopPolicy + case "DefaultEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + if sys.platform == 'win32': + return windows_events._DefaultEventLoopPolicy + return unix_events._DefaultEventLoopPolicy + case "WindowsSelectorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsSelectorEventLoopPolicy + # Else fall through to the AttributeError below. + case "WindowsProactorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsProactorEventLoopPolicy + # Else fall through to the AttributeError below. + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py new file mode 100644 index 00000000000..e07dd52a2a5 --- /dev/null +++ b/Lib/asyncio/__main__.py @@ -0,0 +1,239 @@ +import argparse +import ast +import asyncio +import asyncio.tools +import concurrent.futures +import contextvars +import inspect +import os +import site +import sys +import threading +import types +import warnings + +from _colorize import get_theme +from _pyrepl.console import InteractiveColoredConsole + +from . import futures + + +class AsyncIOInteractiveConsole(InteractiveColoredConsole): + + def __init__(self, locals, loop): + super().__init__(locals, filename="") + self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT + + self.loop = loop + self.context = contextvars.copy_context() + + def runcode(self, code): + global return_code + future = concurrent.futures.Future() + + def callback(): + global return_code + global repl_future + global keyboard_interrupted + + repl_future = None + keyboard_interrupted = False + + func = types.FunctionType(code, self.locals) + try: + coro = func() + except SystemExit as se: + return_code = se.code + self.loop.stop() + return + except KeyboardInterrupt as ex: + keyboard_interrupted = True + future.set_exception(ex) + return + except BaseException as ex: + future.set_exception(ex) + return + + if not inspect.iscoroutine(coro): + future.set_result(coro) + return + + try: + repl_future = self.loop.create_task(coro, context=self.context) + futures._chain_future(repl_future, future) + except BaseException as exc: + future.set_exception(exc) + + self.loop.call_soon_threadsafe(callback, context=self.context) + + try: + return future.result() + except SystemExit as se: + return_code = se.code + self.loop.stop() + return + except BaseException: + if keyboard_interrupted: + if not CAN_USE_PYREPL: + self.write("\nKeyboardInterrupt\n") + else: + self.showtraceback() + return self.STATEMENT_FAILED + +class REPLThread(threading.Thread): + + def run(self): + global return_code + + try: + banner = ( + f'asyncio REPL {sys.version} on {sys.platform}\n' + f'Use "await" directly instead of "asyncio.run()".\n' + f'Type "help", "copyright", "credits" or "license" ' + f'for more information.\n' + ) + + console.write(banner) + + if startup_path := os.getenv("PYTHONSTARTUP"): + sys.audit("cpython.run_startup", startup_path) + + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, console.locals) + + ps1 = getattr(sys, "ps1", ">>> ") + if CAN_USE_PYREPL: + theme = get_theme().syntax + ps1 = f"{theme.prompt}{ps1}{theme.reset}" + console.write(f"{ps1}import asyncio\n") + + if CAN_USE_PYREPL: + from _pyrepl.simple_interact import ( + run_multiline_interactive_console, + ) + try: + run_multiline_interactive_console(console) + except SystemExit: + # expected via the `exit` and `quit` commands + pass + except BaseException: + # unexpected issue + console.showtraceback() + console.write("Internal error, ") + return_code = 1 + else: + console.interact(banner="", exitmsg="") + finally: + warnings.filterwarnings( + 'ignore', + message=r'^coroutine .* was never awaited$', + category=RuntimeWarning) + + loop.call_soon_threadsafe(loop.stop) + + def interrupt(self) -> None: + if not CAN_USE_PYREPL: + return + + from _pyrepl.simple_interact import _get_reader + r = _get_reader() + if r.threading_hook is not None: + r.threading_hook.add("") # type: ignore + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog="python3 -m asyncio", + description="Interactive asyncio shell and CLI tools", + color=True, + ) + subparsers = parser.add_subparsers(help="sub-commands", dest="command") + ps = subparsers.add_parser( + "ps", help="Display a table of all pending tasks in a process" + ) + ps.add_argument("pid", type=int, help="Process ID to inspect") + pstree = subparsers.add_parser( + "pstree", help="Display a tree of all pending tasks in a process" + ) + pstree.add_argument("pid", type=int, help="Process ID to inspect") + args = parser.parse_args() + match args.command: + case "ps": + asyncio.tools.display_awaited_by_tasks_table(args.pid) + sys.exit(0) + case "pstree": + asyncio.tools.display_awaited_by_tasks_tree(args.pid) + sys.exit(0) + case None: + pass # continue to the interactive shell + case _: + # shouldn't happen as an invalid command-line wouldn't parse + # but let's keep it for the next person adding a command + print(f"error: unhandled command {args.command}", file=sys.stderr) + parser.print_usage(file=sys.stderr) + sys.exit(1) + + sys.audit("cpython.run_stdin") + + if os.getenv('PYTHON_BASIC_REPL'): + CAN_USE_PYREPL = False + else: + from _pyrepl.main import CAN_USE_PYREPL + + return_code = 0 + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + repl_locals = {'asyncio': asyncio} + for key in {'__name__', '__package__', + '__loader__', '__spec__', + '__builtins__', '__file__'}: + repl_locals[key] = locals()[key] + + console = AsyncIOInteractiveConsole(repl_locals, loop) + + repl_future = None + keyboard_interrupted = False + + try: + import readline # NoQA + except ImportError: + readline = None + + interactive_hook = getattr(sys, "__interactivehook__", None) + + if interactive_hook is not None: + sys.audit("cpython.run_interactivehook", interactive_hook) + interactive_hook() + + if interactive_hook is site.register_readline: + # Fix the completer function to use the interactive console locals + try: + import rlcompleter + except: + pass + else: + if readline is not None: + completer = rlcompleter.Completer(console.locals) + readline.set_completer(completer.complete) + + repl_thread = REPLThread(name="Interactive thread") + repl_thread.daemon = True + repl_thread.start() + + while True: + try: + loop.run_forever() + except KeyboardInterrupt: + keyboard_interrupted = True + if repl_future and not repl_future.done(): + repl_future.cancel() + repl_thread.interrupt() + continue + else: + break + + console.write('exiting asyncio REPL...\n') + sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 2df379933c3..8cbb71f7085 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -14,13 +14,14 @@ """ import collections +import collections.abc import concurrent.futures +import errno import heapq -import inspect import itertools -import logging import os import socket +import stat import subprocess import threading import time @@ -29,16 +30,27 @@ import warnings import weakref -from . import compat +try: + import ssl +except ImportError: # pragma: no cover + ssl = None + +from . import constants from . import coroutines from . import events +from . import exceptions from . import futures +from . import protocols +from . import sslproto +from . import staggered from . import tasks -from .coroutines import coroutine +from . import timeouts +from . import transports +from . import trsock from .log import logger -__all__ = ['BaseEventLoop'] +__all__ = 'BaseEventLoop','Server', # Minimum number of _scheduled timer handles before cleanup of @@ -49,10 +61,11 @@ # before cleanup of cancelled handles is performed. _MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5 -# Exceptions which must not call the exception handler in fatal error -# methods (_fatal_error()) -_FATAL_ERROR_IGNORE = (BrokenPipeError, - ConnectionResetError, ConnectionAbortedError) + +_HAS_IPv6 = hasattr(socket, 'AF_INET6') + +# Maximum timeout passed to select to avoid OS limitations +MAXIMUM_SELECT_TIMEOUT = 24 * 3600 def _format_handle(handle): @@ -84,21 +97,7 @@ def _set_reuseport(sock): 'SO_REUSEPORT defined but not implemented.') -def _is_stream_socket(sock): - # Linux's socket.type is a bitmask that can include extra info - # about socket, therefore we can't do simple - # `sock_type == socket.SOCK_STREAM`. - return (sock.type & socket.SOCK_STREAM) == socket.SOCK_STREAM - - -def _is_dgram_socket(sock): - # Linux's socket.type is a bitmask that can include extra info - # about socket, therefore we can't do simple - # `sock_type == socket.SOCK_DGRAM`. - return (sock.type & socket.SOCK_DGRAM) == socket.SOCK_DGRAM - - -def _ipaddr_info(host, port, family, type, proto): +def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0): # Try to skip getaddrinfo if "host" is already an IP. Users might have # handled name resolution in their own code and pass in resolved IPs. if not hasattr(socket, 'inet_pton'): @@ -109,11 +108,6 @@ def _ipaddr_info(host, port, family, type, proto): return None if type == socket.SOCK_STREAM: - # Linux only: - # getaddrinfo() can raise when socket.type is a bit mask. - # So if socket.type is a bit mask of SOCK_STREAM, and say - # SOCK_NONBLOCK, we simply return None, which will trigger - # a call to getaddrinfo() letting it process this request. proto = socket.IPPROTO_TCP elif type == socket.SOCK_DGRAM: proto = socket.IPPROTO_UDP @@ -135,7 +129,7 @@ def _ipaddr_info(host, port, family, type, proto): if family == socket.AF_UNSPEC: afs = [socket.AF_INET] - if hasattr(socket, 'AF_INET6'): + if _HAS_IPv6: afs.append(socket.AF_INET6) else: afs = [family] @@ -151,7 +145,10 @@ def _ipaddr_info(host, port, family, type, proto): try: socket.inet_pton(af, host) # The host has already been resolved. - return af, type, proto, '', (host, port) + if _HAS_IPv6 and af == socket.AF_INET6: + return af, type, proto, '', (host, port, flowinfo, scopeid) + else: + return af, type, proto, '', (host, port) except OSError: pass @@ -159,75 +156,262 @@ def _ipaddr_info(host, port, family, type, proto): return None -def _ensure_resolved(address, *, family=0, type=socket.SOCK_STREAM, proto=0, - flags=0, loop): - host, port = address[:2] - info = _ipaddr_info(host, port, family, type, proto) - if info is not None: - # "host" is already a resolved IP. - fut = loop.create_future() - fut.set_result([info]) - return fut - else: - return loop.getaddrinfo(host, port, family=family, type=type, - proto=proto, flags=flags) +def _interleave_addrinfos(addrinfos, first_address_family_count=1): + """Interleave list of addrinfo tuples by family.""" + # Group addresses by family + addrinfos_by_family = collections.OrderedDict() + for addr in addrinfos: + family = addr[0] + if family not in addrinfos_by_family: + addrinfos_by_family[family] = [] + addrinfos_by_family[family].append(addr) + addrinfos_lists = list(addrinfos_by_family.values()) + + reordered = [] + if first_address_family_count > 1: + reordered.extend(addrinfos_lists[0][:first_address_family_count - 1]) + del addrinfos_lists[0][:first_address_family_count - 1] + reordered.extend( + a for a in itertools.chain.from_iterable( + itertools.zip_longest(*addrinfos_lists) + ) if a is not None) + return reordered def _run_until_complete_cb(fut): - exc = fut._exception - if (isinstance(exc, BaseException) - and not isinstance(exc, Exception)): - # Issue #22429: run_forever() already finished, no need to - # stop it. - return - fut._loop.stop() + if not fut.cancelled(): + exc = fut.exception() + if isinstance(exc, (SystemExit, KeyboardInterrupt)): + # Issue #22429: run_forever() already finished, no need to + # stop it. + return + futures._get_loop(fut).stop() + + +if hasattr(socket, 'TCP_NODELAY'): + def _set_nodelay(sock): + if (sock.family in {socket.AF_INET, socket.AF_INET6} and + sock.type == socket.SOCK_STREAM and + sock.proto == socket.IPPROTO_TCP): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) +else: + def _set_nodelay(sock): + pass + + +def _check_ssl_socket(sock): + if ssl is not None and isinstance(sock, ssl.SSLSocket): + raise TypeError("Socket cannot be of type SSLSocket") + + +class _SendfileFallbackProtocol(protocols.Protocol): + def __init__(self, transp): + if not isinstance(transp, transports._FlowControlMixin): + raise TypeError("transport should be _FlowControlMixin instance") + self._transport = transp + self._proto = transp.get_protocol() + self._should_resume_reading = transp.is_reading() + self._should_resume_writing = transp._protocol_paused + transp.pause_reading() + transp.set_protocol(self) + if self._should_resume_writing: + self._write_ready_fut = self._transport._loop.create_future() + else: + self._write_ready_fut = None + + async def drain(self): + if self._transport.is_closing(): + raise ConnectionError("Connection closed by peer") + fut = self._write_ready_fut + if fut is None: + return + await fut + + def connection_made(self, transport): + raise RuntimeError("Invalid state: " + "connection should have been established already.") + + def connection_lost(self, exc): + if self._write_ready_fut is not None: + # Never happens if peer disconnects after sending the whole content + # Thus disconnection is always an exception from user perspective + if exc is None: + self._write_ready_fut.set_exception( + ConnectionError("Connection is closed by peer")) + else: + self._write_ready_fut.set_exception(exc) + self._proto.connection_lost(exc) + + def pause_writing(self): + if self._write_ready_fut is not None: + return + self._write_ready_fut = self._transport._loop.create_future() + + def resume_writing(self): + if self._write_ready_fut is None: + return + self._write_ready_fut.set_result(False) + self._write_ready_fut = None + + def data_received(self, data): + raise RuntimeError("Invalid state: reading should be paused") + + def eof_received(self): + raise RuntimeError("Invalid state: reading should be paused") + + async def restore(self): + self._transport.set_protocol(self._proto) + if self._should_resume_reading: + self._transport.resume_reading() + if self._write_ready_fut is not None: + # Cancel the future. + # Basically it has no effect because protocol is switched back, + # no code should wait for it anymore. + self._write_ready_fut.cancel() + if self._should_resume_writing: + self._proto.resume_writing() class Server(events.AbstractServer): - def __init__(self, loop, sockets): + def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, + ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop - self.sockets = sockets - self._active_count = 0 + self._sockets = sockets + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] + self._protocol_factory = protocol_factory + self._backlog = backlog + self._ssl_context = ssl_context + self._ssl_handshake_timeout = ssl_handshake_timeout + self._ssl_shutdown_timeout = ssl_shutdown_timeout + self._serving = False + self._serving_forever_fut = None def __repr__(self): - return '<%s sockets=%r>' % (self.__class__.__name__, self.sockets) + return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): - assert self.sockets is not None - self._active_count += 1 + def _attach(self, transport): + assert self._sockets is not None + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self.sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() + def _wakeup(self): + waiters = self._waiters + self._waiters = None + for waiter in waiters: + if not waiter.done(): + waiter.set_result(None) + + def _start_serving(self): + if self._serving: + return + self._serving = True + for sock in self._sockets: + sock.listen(self._backlog) + self._loop._start_serving( + self._protocol_factory, sock, self._ssl_context, + self, self._backlog, self._ssl_handshake_timeout, + self._ssl_shutdown_timeout) + + def get_loop(self): + return self._loop + + def is_serving(self): + return self._serving + + @property + def sockets(self): + if self._sockets is None: + return () + return tuple(trsock.TransportSocket(s) for s in self._sockets) + def close(self): - sockets = self.sockets + sockets = self._sockets if sockets is None: return - self.sockets = None + self._sockets = None + for sock in sockets: self._loop._stop_serving(sock) - if self._active_count == 0: + + self._serving = False + + if (self._serving_forever_fut is not None and + not self._serving_forever_fut.done()): + self._serving_forever_fut.cancel() + self._serving_forever_fut = None + + if len(self._clients) == 0: self._wakeup() - def _wakeup(self): - waiters = self._waiters - self._waiters = None - for waiter in waiters: - if not waiter.done(): - waiter.set_result(waiter) + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + + async def start_serving(self): + self._start_serving() + # Skip one loop iteration so that all 'loop.add_reader' + # go through. + await tasks.sleep(0) + + async def serve_forever(self): + if self._serving_forever_fut is not None: + raise RuntimeError( + f'server {self!r} is already being awaited on serve_forever()') + if self._sockets is None: + raise RuntimeError(f'server {self!r} is closed') + + self._start_serving() + self._serving_forever_fut = self._loop.create_future() + + try: + await self._serving_forever_fut + except exceptions.CancelledError: + try: + self.close() + await self.wait_closed() + finally: + raise + finally: + self._serving_forever_fut = None - @coroutine - def wait_closed(self): - if self.sockets is None or self._waiters is None: + async def wait_closed(self): + """Wait until server is closed and all connections are dropped. + + - If the server is not closed, wait. + - If it is closed, but there are still active connections, wait. + + Anyone waiting here will be unblocked once both conditions + (server is closed and all connections have been dropped) + have become true, in either order. + + Historical note: In 3.11 and before, this was broken, returning + immediately if the server was already closed, even if there + were still active connections. An attempted fix in 3.12.0 was + still broken, returning immediately if the server was still + open and there were no active connections. Hopefully in 3.12.1 + we have it right. + """ + # Waiters are unblocked by self._wakeup(), which is called + # from two places: self.close() and self._detach(), but only + # when both conditions have become true. To signal that this + # has happened, self._wakeup() sets self._waiters to None. + if self._waiters is None: return waiter = self._loop.create_future() self._waiters.append(waiter) - yield from waiter + await waiter class BaseEventLoop(events.AbstractEventLoop): @@ -243,50 +427,55 @@ def __init__(self): # Identifier of the thread running the event loop, or None if the # event loop is not running self._thread_id = None - self._clock_resolution = 1e-06 #time.get_clock_info('monotonic').resolution + self._clock_resolution = time.get_clock_info('monotonic').resolution self._exception_handler = None - self.set_debug((not sys.flags.ignore_environment - and bool(os.environ.get('PYTHONASYNCIODEBUG')))) + self.set_debug(coroutines._is_debug_mode()) + # The preserved state of async generator hooks. + self._old_agen_hooks = None # In debug mode, if the execution of a callback or a step of a task # exceed this duration in seconds, the slow callback/task is logged. self.slow_callback_duration = 0.1 self._current_handle = None self._task_factory = None - self._coroutine_wrapper_set = False - - if hasattr(sys, 'get_asyncgen_hooks'): - # Python >= 3.6 - # A weak set of all asynchronous generators that are - # being iterated by the loop. - self._asyncgens = weakref.WeakSet() - else: - self._asyncgens = None + self._coroutine_origin_tracking_enabled = False + self._coroutine_origin_tracking_saved_depth = None + # A weak set of all asynchronous generators that are + # being iterated by the loop. + self._asyncgens = weakref.WeakSet() # Set to True when `loop.shutdown_asyncgens` is called. self._asyncgens_shutdown_called = False + # Set to True when `loop.shutdown_default_executor` is called. + self._executor_shutdown_called = False def __repr__(self): - return ('<%s running=%s closed=%s debug=%s>' - % (self.__class__.__name__, self.is_running(), - self.is_closed(), self.get_debug())) + return ( + f'<{self.__class__.__name__} running={self.is_running()} ' + f'closed={self.is_closed()} debug={self.get_debug()}>' + ) def create_future(self): """Create a Future object attached to the loop.""" return futures.Future(loop=self) - def create_task(self, coro): - """Schedule a coroutine object. + def create_task(self, coro, **kwargs): + """Schedule or begin executing a coroutine object. Return a task object. """ self._check_closed() - if self._task_factory is None: - task = tasks.Task(coro, loop=self) - if task._source_traceback: - del task._source_traceback[-1] - else: - task = self._task_factory(self, coro) - return task + if self._task_factory is not None: + return self._task_factory(self, coro, **kwargs) + + task = tasks.Task(coro, loop=self, **kwargs) + if task._source_traceback: + del task._source_traceback[-1] + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->BaseEventLoop.create_task->task + del task def set_task_factory(self, factory): """Set a task factory that will be used by loop.create_task(). @@ -294,9 +483,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object. The callable - must return a Future. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the active + event loop, 'coro' will be a coroutine object, and **kwargs will be + arbitrary keyword arguments that should be passed on to Task. + The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -311,9 +501,13 @@ def _make_socket_transport(self, sock, protocol, waiter=None, *, """Create socket transport.""" raise NotImplementedError - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): + def _make_ssl_transport( + self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + call_connection_made=True): """Create SSL transport.""" raise NotImplementedError @@ -332,10 +526,9 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, """Create write pipe transport.""" raise NotImplementedError - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): + async def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): """Create subprocess transport.""" raise NotImplementedError @@ -356,29 +549,29 @@ def _check_closed(self): if self._closed: raise RuntimeError('Event loop is closed') + def _check_default_executor(self): + if self._executor_shutdown_called: + raise RuntimeError('Executor shutdown has been called') + def _asyncgen_finalizer_hook(self, agen): self._asyncgens.discard(agen) if not self.is_closed(): - self.create_task(agen.aclose()) - # Wake up the loop if the finalizer was called from - # a different thread. - self._write_to_self() + self.call_soon_threadsafe(self.create_task, agen.aclose()) def _asyncgen_firstiter_hook(self, agen): if self._asyncgens_shutdown_called: warnings.warn( - "asynchronous generator {!r} was scheduled after " - "loop.shutdown_asyncgens() call".format(agen), + f"asynchronous generator {agen!r} was scheduled after " + f"loop.shutdown_asyncgens() call", ResourceWarning, source=self) self._asyncgens.add(agen) - @coroutine - def shutdown_asyncgens(self): + async def shutdown_asyncgens(self): """Shutdown all active asynchronous generators.""" self._asyncgens_shutdown_called = True - if self._asyncgens is None or not len(self._asyncgens): + if not len(self._asyncgens): # If Python version is <3.6 or we don't have any asynchronous # generators alive. return @@ -386,48 +579,106 @@ def shutdown_asyncgens(self): closing_agens = list(self._asyncgens) self._asyncgens.clear() - shutdown_coro = tasks.gather( + results = await tasks.gather( *[ag.aclose() for ag in closing_agens], - return_exceptions=True, - loop=self) + return_exceptions=True) - results = yield from shutdown_coro for result, agen in zip(results, closing_agens): if isinstance(result, Exception): self.call_exception_handler({ - 'message': 'an error occurred during closing of ' - 'asynchronous generator {!r}'.format(agen), + 'message': f'an error occurred during closing of ' + f'asynchronous generator {agen!r}', 'exception': result, 'asyncgen': agen }) - def run_forever(self): - """Run until stop() is called.""" - self._check_closed() + async def shutdown_default_executor(self, timeout=None): + """Schedule the shutdown of the default executor. + + The timeout parameter specifies the amount of time the executor will + be given to finish joining. The default value is None, which means + that the executor will be given an unlimited amount of time. + """ + self._executor_shutdown_called = True + if self._default_executor is None: + return + future = self.create_future() + thread = threading.Thread(target=self._do_shutdown, args=(future,)) + thread.start() + try: + async with timeouts.timeout(timeout): + await future + except TimeoutError: + warnings.warn("The executor did not finishing joining " + f"its threads within {timeout} seconds.", + RuntimeWarning, stacklevel=2) + self._default_executor.shutdown(wait=False) + else: + thread.join() + + def _do_shutdown(self, future): + try: + self._default_executor.shutdown(wait=True) + if not self.is_closed(): + self.call_soon_threadsafe(futures._set_result_unless_cancelled, + future, None) + except Exception as ex: + if not self.is_closed() and not future.cancelled(): + self.call_soon_threadsafe(future.set_exception, ex) + + def _check_running(self): if self.is_running(): raise RuntimeError('This event loop is already running') if events._get_running_loop() is not None: raise RuntimeError( 'Cannot run the event loop while another loop is running') - self._set_coroutine_wrapper(self._debug) + + def _run_forever_setup(self): + """Prepare the run loop to process events. + + This method exists so that custom event loop subclasses (e.g., event loops + that integrate a GUI event loop with Python's event loop) have access to all the + loop setup logic. + """ + self._check_closed() + self._check_running() + self._set_coroutine_origin_tracking(self._debug) + + self._old_agen_hooks = sys.get_asyncgen_hooks() self._thread_id = threading.get_ident() - if self._asyncgens is not None: - old_agen_hooks = sys.get_asyncgen_hooks() - sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, - finalizer=self._asyncgen_finalizer_hook) + sys.set_asyncgen_hooks( + firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook + ) + + events._set_running_loop(self) + + def _run_forever_cleanup(self): + """Clean up after an event loop finishes the looping over events. + + This method exists so that custom event loop subclasses (e.g., event loops + that integrate a GUI event loop with Python's event loop) have access to all the + loop cleanup logic. + """ + self._stopping = False + self._thread_id = None + events._set_running_loop(None) + self._set_coroutine_origin_tracking(False) + # Restore any pre-existing async generator hooks. + if self._old_agen_hooks is not None: + sys.set_asyncgen_hooks(*self._old_agen_hooks) + self._old_agen_hooks = None + + def run_forever(self): + """Run until stop() is called.""" + self._run_forever_setup() try: - events._set_running_loop(self) while True: self._run_once() if self._stopping: break finally: - self._stopping = False - self._thread_id = None - events._set_running_loop(None) - self._set_coroutine_wrapper(False) - if self._asyncgens is not None: - sys.set_asyncgen_hooks(*old_agen_hooks) + self._run_forever_cleanup() def run_until_complete(self, future): """Run until the Future is done. @@ -441,6 +692,7 @@ def run_until_complete(self, future): Return the Future's result, or raise its exception. """ self._check_closed() + self._check_running() new_task = not futures.isfuture(future) future = tasks.ensure_future(future, loop=self) @@ -459,7 +711,8 @@ def run_until_complete(self, future): # local task. future.exception() raise - future.remove_done_callback(_run_until_complete_cb) + finally: + future.remove_done_callback(_run_until_complete_cb) if not future.done(): raise RuntimeError('Event loop stopped before Future completed.') @@ -490,6 +743,7 @@ def close(self): self._closed = True self._ready.clear() self._scheduled.clear() + self._executor_shutdown_called = True executor = self._default_executor if executor is not None: self._default_executor = None @@ -499,16 +753,11 @@ def is_closed(self): """Returns True if the event loop was closed.""" return self._closed - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if not self.is_closed(): - warnings.warn("unclosed event loop %r" % self, ResourceWarning, - source=self) - if not self.is_running(): - self.close() + def __del__(self, _warn=warnings.warn): + if not self.is_closed(): + _warn(f"unclosed event loop {self!r}", ResourceWarning, source=self) + if not self.is_running(): + self.close() def is_running(self): """Returns True if the event loop is running.""" @@ -523,7 +772,7 @@ def time(self): """ return time.monotonic() - def call_later(self, delay, callback, *args): + def call_later(self, delay, callback, *args, context=None): """Arrange for a callback to be called at a given time. Return a Handle: an opaque object with a cancel() method that @@ -533,34 +782,39 @@ def call_later(self, delay, callback, *args): always relative to the current time. Each callback will be called exactly once. If two callbacks - are scheduled for exactly the same time, it undefined which + are scheduled for exactly the same time, it is undefined which will be called first. Any positional arguments after the callback will be passed to the callback when it is called. """ - timer = self.call_at(self.time() + delay, callback, *args) + if delay is None: + raise TypeError('delay must not be None') + timer = self.call_at(self.time() + delay, callback, *args, + context=context) if timer._source_traceback: del timer._source_traceback[-1] return timer - def call_at(self, when, callback, *args): + def call_at(self, when, callback, *args, context=None): """Like call_later(), but uses an absolute time. Absolute time corresponds to the event loop's time() method. """ + if when is None: + raise TypeError("when cannot be None") self._check_closed() if self._debug: self._check_thread() self._check_callback(callback, 'call_at') - timer = events.TimerHandle(when, callback, args, self) + timer = events.TimerHandle(when, callback, args, self, context) if timer._source_traceback: del timer._source_traceback[-1] heapq.heappush(self._scheduled, timer) timer._scheduled = True return timer - def call_soon(self, callback, *args): + def call_soon(self, callback, *args, context=None): """Arrange for a callback to be called as soon as possible. This operates as a FIFO queue: callbacks are called in the @@ -574,24 +828,23 @@ def call_soon(self, callback, *args): if self._debug: self._check_thread() self._check_callback(callback, 'call_soon') - handle = self._call_soon(callback, args) + handle = self._call_soon(callback, args, context) if handle._source_traceback: del handle._source_traceback[-1] return handle def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines.iscoroutinefunction(callback)): + coroutines._iscoroutinefunction(callback)): raise TypeError( - "coroutines cannot be used with {}()".format(method)) + f"coroutines cannot be used with {method}()") if not callable(callback): raise TypeError( - 'a callable object was expected by {}(), got {!r}'.format( - method, callback)) - + f'a callable object was expected by {method}(), ' + f'got {callback!r}') - def _call_soon(self, callback, args): - handle = events.Handle(callback, args, self) + def _call_soon(self, callback, args, context): + handle = events.Handle(callback, args, self, context) if handle._source_traceback: del handle._source_traceback[-1] self._ready.append(handle) @@ -614,12 +867,15 @@ def _check_thread(self): "Non-thread-safe operation invoked on an event loop other " "than the current one") - def call_soon_threadsafe(self, callback, *args): + def call_soon_threadsafe(self, callback, *args, context=None): """Like call_soon(), but thread-safe.""" self._check_closed() if self._debug: self._check_callback(callback, 'call_soon_threadsafe') - handle = self._call_soon(callback, args) + handle = events._ThreadSafeHandle(callback, args, self, context) + self._ready.append(handle) + if handle._source_traceback: + del handle._source_traceback[-1] if handle._source_traceback: del handle._source_traceback[-1] self._write_to_self() @@ -631,24 +887,31 @@ def run_in_executor(self, executor, func, *args): self._check_callback(func, 'run_in_executor') if executor is None: executor = self._default_executor + # Only check when the default executor is being used + self._check_default_executor() if executor is None: - executor = concurrent.futures.ThreadPoolExecutor() + executor = concurrent.futures.ThreadPoolExecutor( + thread_name_prefix='asyncio' + ) self._default_executor = executor - return futures.wrap_future(executor.submit(func, *args), loop=self) + return futures.wrap_future( + executor.submit(func, *args), loop=self) def set_default_executor(self, executor): + if not isinstance(executor, concurrent.futures.ThreadPoolExecutor): + raise TypeError('executor must be ThreadPoolExecutor instance') self._default_executor = executor def _getaddrinfo_debug(self, host, port, family, type, proto, flags): - msg = ["%s:%r" % (host, port)] + msg = [f"{host}:{port!r}"] if family: - msg.append('family=%r' % family) + msg.append(f'family={family!r}') if type: - msg.append('type=%r' % type) + msg.append(f'type={type!r}') if proto: - msg.append('proto=%r' % proto) + msg.append(f'proto={proto!r}') if flags: - msg.append('flags=%r' % flags) + msg.append(f'flags={flags!r}') msg = ', '.join(msg) logger.debug('Get address info %s', msg) @@ -656,33 +919,156 @@ def _getaddrinfo_debug(self, host, port, family, type, proto, flags): addrinfo = socket.getaddrinfo(host, port, family, type, proto, flags) dt = self.time() - t0 - msg = ('Getting address info %s took %.3f ms: %r' - % (msg, dt * 1e3, addrinfo)) + msg = f'Getting address info {msg} took {dt * 1e3:.3f}ms: {addrinfo!r}' if dt >= self.slow_callback_duration: logger.info(msg) else: logger.debug(msg) return addrinfo - def getaddrinfo(self, host, port, *, - family=0, type=0, proto=0, flags=0): + async def getaddrinfo(self, host, port, *, + family=0, type=0, proto=0, flags=0): if self._debug: - return self.run_in_executor(None, self._getaddrinfo_debug, - host, port, family, type, proto, flags) + getaddr_func = self._getaddrinfo_debug else: - return self.run_in_executor(None, socket.getaddrinfo, - host, port, family, type, proto, flags) + getaddr_func = socket.getaddrinfo - def getnameinfo(self, sockaddr, flags=0): - return self.run_in_executor(None, socket.getnameinfo, sockaddr, flags) + return await self.run_in_executor( + None, getaddr_func, host, port, family, type, proto, flags) - @coroutine - def create_connection(self, protocol_factory, host=None, port=None, *, - ssl=None, family=0, proto=0, flags=0, sock=None, - local_addr=None, server_hostname=None): + async def getnameinfo(self, sockaddr, flags=0): + return await self.run_in_executor( + None, socket.getnameinfo, sockaddr, flags) + + async def sock_sendfile(self, sock, file, offset=0, count=None, + *, fallback=True): + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + _check_ssl_socket(sock) + self._check_sendfile_params(sock, file, offset, count) + try: + return await self._sock_sendfile_native(sock, file, + offset, count) + except exceptions.SendfileNotAvailableError as exc: + if not fallback: + raise + return await self._sock_sendfile_fallback(sock, file, + offset, count) + + async def _sock_sendfile_native(self, sock, file, offset, count): + # NB: sendfile syscall is not supported for SSL sockets and + # non-mmap files even if sendfile is supported by OS + raise exceptions.SendfileNotAvailableError( + f"syscall sendfile is not available for socket {sock!r} " + f"and file {file!r} combination") + + async def _sock_sendfile_fallback(self, sock, file, offset, count): + if offset: + file.seek(offset) + blocksize = ( + min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) + if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE + ) + buf = bytearray(blocksize) + total_sent = 0 + try: + while True: + if count: + blocksize = min(count - total_sent, blocksize) + if blocksize <= 0: + break + view = memoryview(buf)[:blocksize] + read = await self.run_in_executor(None, file.readinto, view) + if not read: + break # EOF + await self.sock_sendall(sock, view[:read]) + total_sent += read + return total_sent + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset + total_sent) + + def _check_sendfile_params(self, sock, file, offset, count): + if 'b' not in getattr(file, 'mode', 'b'): + raise ValueError("file should be opened in binary mode") + if not sock.type == socket.SOCK_STREAM: + raise ValueError("only SOCK_STREAM type sockets are supported") + if count is not None: + if not isinstance(count, int): + raise TypeError( + "count must be a positive integer (got {!r})".format(count)) + if count <= 0: + raise ValueError( + "count must be a positive integer (got {!r})".format(count)) + if not isinstance(offset, int): + raise TypeError( + "offset must be a non-negative integer (got {!r})".format( + offset)) + if offset < 0: + raise ValueError( + "offset must be a non-negative integer (got {!r})".format( + offset)) + + async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): + """Create, bind and connect one socket.""" + my_exceptions = [] + exceptions.append(my_exceptions) + family, type_, proto, _, address = addr_info + sock = None + try: + try: + sock = socket.socket(family=family, type=type_, proto=proto) + sock.setblocking(False) + if local_addr_infos is not None: + for lfamily, _, _, _, laddr in local_addr_infos: + # skip local addresses of different family + if lfamily != family: + continue + try: + sock.bind(laddr) + break + except OSError as exc: + msg = ( + f'error while attempting to bind on ' + f'address {laddr!r}: {str(exc).lower()}' + ) + exc = OSError(exc.errno, msg) + my_exceptions.append(exc) + else: # all bind attempts failed + if my_exceptions: + raise my_exceptions.pop() + else: + raise OSError(f"no matching local address with {family=} found") + await self.sock_connect(sock, address) + return sock + except OSError as exc: + my_exceptions.append(exc) + raise + except: + if sock is not None: + try: + sock.close() + except OSError: + # An error when closing a newly created socket is + # not important, but it can overwrite more important + # non-OSError error. So ignore it. + pass + raise + finally: + exceptions = my_exceptions = None + + async def create_connection( + self, protocol_factory, host=None, port=None, + *, ssl=None, family=0, + proto=0, flags=0, sock=None, + local_addr=None, server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + happy_eyeballs_delay=None, interleave=None, + all_errors=False): """Connect to a TCP server. - Create a streaming transport connection to a given Internet host and + Create a streaming transport connection to a given internet host and port: socket family AF_INET or socket.AF_INET6 depending on host (or family if specified), socket type SOCK_STREAM. protocol_factory must be a callable returning a protocol instance. @@ -710,85 +1096,96 @@ def create_connection(self, protocol_factory, host=None, port=None, *, 'when using ssl without a host') server_hostname = host + if ssl_handshake_timeout is not None and not ssl: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') + + if ssl_shutdown_timeout is not None and not ssl: + raise ValueError( + 'ssl_shutdown_timeout is only meaningful with ssl') + + if sock is not None: + _check_ssl_socket(sock) + + if happy_eyeballs_delay is not None and interleave is None: + # If using happy eyeballs, default to interleave addresses by family + interleave = 1 + if host is not None or port is not None: if sock is not None: raise ValueError( 'host/port and sock can not be specified at the same time') - f1 = _ensure_resolved((host, port), family=family, - type=socket.SOCK_STREAM, proto=proto, - flags=flags, loop=self) - fs = [f1] - if local_addr is not None: - f2 = _ensure_resolved(local_addr, family=family, - type=socket.SOCK_STREAM, proto=proto, - flags=flags, loop=self) - fs.append(f2) - else: - f2 = None - - yield from tasks.wait(fs, loop=self) - - infos = f1.result() + infos = await self._ensure_resolved( + (host, port), family=family, + type=socket.SOCK_STREAM, proto=proto, flags=flags, loop=self) if not infos: raise OSError('getaddrinfo() returned empty list') - if f2 is not None: - laddr_infos = f2.result() + + if local_addr is not None: + laddr_infos = await self._ensure_resolved( + local_addr, family=family, + type=socket.SOCK_STREAM, proto=proto, + flags=flags, loop=self) if not laddr_infos: raise OSError('getaddrinfo() returned empty list') + else: + laddr_infos = None + + if interleave: + infos = _interleave_addrinfos(infos, interleave) exceptions = [] - for family, type, proto, cname, address in infos: + if happy_eyeballs_delay is None: + # not using happy eyeballs + for addrinfo in infos: + try: + sock = await self._connect_sock( + exceptions, addrinfo, laddr_infos) + break + except OSError: + continue + else: # using happy eyeballs + sock = (await staggered.staggered_race( + ( + # can't use functools.partial as it keeps a reference + # to exceptions + lambda addrinfo=addrinfo: self._connect_sock( + exceptions, addrinfo, laddr_infos + ) + for addrinfo in infos + ), + happy_eyeballs_delay, + loop=self, + ))[0] # can't use sock, _, _ as it keeks a reference to exceptions + + if sock is None: + exceptions = [exc for sub in exceptions for exc in sub] try: - sock = socket.socket(family=family, type=type, proto=proto) - sock.setblocking(False) - if f2 is not None: - for _, _, _, _, laddr in laddr_infos: - try: - sock.bind(laddr) - break - except OSError as exc: - exc = OSError( - exc.errno, 'error while ' - 'attempting to bind on address ' - '{!r}: {}'.format( - laddr, exc.strerror.lower())) - exceptions.append(exc) - else: - sock.close() - sock = None - continue - if self._debug: - logger.debug("connect %r to %r", sock, address) - yield from self.sock_connect(sock, address) - except OSError as exc: - if sock is not None: - sock.close() - exceptions.append(exc) - except: - if sock is not None: - sock.close() - raise - else: - break - else: - if len(exceptions) == 1: - raise exceptions[0] - else: - # If they all have the same str(), raise one. - model = str(exceptions[0]) - if all(str(exc) == model for exc in exceptions): + if all_errors: + raise ExceptionGroup("create_connection failed", exceptions) + if len(exceptions) == 1: raise exceptions[0] - # Raise a combined exception so the user can see all - # the various error messages. - raise OSError('Multiple exceptions: {}'.format( - ', '.join(str(exc) for exc in exceptions))) + elif exceptions: + # If they all have the same str(), raise one. + model = str(exceptions[0]) + if all(str(exc) == model for exc in exceptions): + raise exceptions[0] + # Raise a combined exception so the user can see all + # the various error messages. + raise OSError('Multiple exceptions: {}'.format( + ', '.join(str(exc) for exc in exceptions))) + else: + # No exceptions were collected, raise a timeout error + raise TimeoutError('create_connection failed') + finally: + exceptions = None else: if sock is None: raise ValueError( 'host and port was not specified and no sock specified') - if not _is_stream_socket(sock): + if sock.type != socket.SOCK_STREAM: # We allow AF_INET, AF_INET6, AF_UNIX as long as they # are SOCK_STREAM. # We support passing AF_UNIX sockets even though we have @@ -796,10 +1193,12 @@ def create_connection(self, protocol_factory, host=None, port=None, *, # Disallowing AF_UNIX in this method, breaks backwards # compatibility. raise ValueError( - 'A Stream Socket was expected, got {!r}'.format(sock)) + f'A Stream Socket was expected, got {sock!r}') - transport, protocol = yield from self._create_connection_transport( - sock, protocol_factory, ssl, server_hostname) + transport, protocol = await self._create_connection_transport( + sock, protocol_factory, ssl, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) if self._debug: # Get the socket from the transport because SSL transport closes # the old socket and creates a new SSL socket @@ -808,9 +1207,11 @@ def create_connection(self, protocol_factory, host=None, port=None, *, sock, host, port, transport, protocol) return transport, protocol - @coroutine - def _create_connection_transport(self, sock, protocol_factory, ssl, - server_hostname, server_side=False): + async def _create_connection_transport( + self, sock, protocol_factory, ssl, + server_hostname, server_side=False, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): sock.setblocking(False) @@ -820,42 +1221,166 @@ def _create_connection_transport(self, sock, protocol_factory, ssl, sslcontext = None if isinstance(ssl, bool) else ssl transport = self._make_ssl_transport( sock, protocol, sslcontext, waiter, - server_side=server_side, server_hostname=server_hostname) + server_side=server_side, server_hostname=server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) else: transport = self._make_socket_transport(sock, protocol, waiter) try: - yield from waiter + await waiter except: transport.close() raise return transport, protocol - @coroutine - def create_datagram_endpoint(self, protocol_factory, - local_addr=None, remote_addr=None, *, - family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, - allow_broadcast=None, sock=None): + async def sendfile(self, transport, file, offset=0, count=None, + *, fallback=True): + """Send a file to transport. + + Return the total number of bytes which were sent. + + The method uses high-performance os.sendfile if available. + + file must be a regular file object opened in binary mode. + + offset tells from where to start reading the file. If specified, + count is the total number of bytes to transmit as opposed to + sending the file until EOF is reached. File position is updated on + return or also in case of error in which case file.tell() + can be used to figure out the number of bytes + which were sent. + + fallback set to True makes asyncio to manually read and send + the file when the platform does not support the sendfile syscall + (e.g. Windows or SSL socket on Unix). + + Raise SendfileNotAvailableError if the system does not support + sendfile syscall and fallback is False. + """ + if transport.is_closing(): + raise RuntimeError("Transport is closing") + mode = getattr(transport, '_sendfile_compatible', + constants._SendfileMode.UNSUPPORTED) + if mode is constants._SendfileMode.UNSUPPORTED: + raise RuntimeError( + f"sendfile is not supported for transport {transport!r}") + if mode is constants._SendfileMode.TRY_NATIVE: + try: + return await self._sendfile_native(transport, file, + offset, count) + except exceptions.SendfileNotAvailableError as exc: + if not fallback: + raise + + if not fallback: + raise RuntimeError( + f"fallback is disabled and native sendfile is not " + f"supported for transport {transport!r}") + + return await self._sendfile_fallback(transport, file, + offset, count) + + async def _sendfile_native(self, transp, file, offset, count): + raise exceptions.SendfileNotAvailableError( + "sendfile syscall is not supported") + + async def _sendfile_fallback(self, transp, file, offset, count): + if offset: + file.seek(offset) + blocksize = min(count, 16384) if count else 16384 + buf = bytearray(blocksize) + total_sent = 0 + proto = _SendfileFallbackProtocol(transp) + try: + while True: + if count: + blocksize = min(count - total_sent, blocksize) + if blocksize <= 0: + return total_sent + view = memoryview(buf)[:blocksize] + read = await self.run_in_executor(None, file.readinto, view) + if not read: + return total_sent # EOF + transp.write(view[:read]) + await proto.drain() + total_sent += read + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset + total_sent) + await proto.restore() + + async def start_tls(self, transport, protocol, sslcontext, *, + server_side=False, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): + """Upgrade transport to TLS. + + Return a new transport that *protocol* should start using + immediately. + """ + if ssl is None: + raise RuntimeError('Python ssl module is not available') + + if not isinstance(sslcontext, ssl.SSLContext): + raise TypeError( + f'sslcontext is expected to be an instance of ssl.SSLContext, ' + f'got {sslcontext!r}') + + if not getattr(transport, '_start_tls_compatible', False): + raise TypeError( + f'transport {transport!r} is not supported by start_tls()') + + waiter = self.create_future() + ssl_protocol = sslproto.SSLProtocol( + self, protocol, sslcontext, waiter, + server_side, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout, + call_connection_made=False) + + # Pause early so that "ssl_protocol.data_received()" doesn't + # have a chance to get called before "ssl_protocol.connection_made()". + transport.pause_reading() + + transport.set_protocol(ssl_protocol) + conmade_cb = self.call_soon(ssl_protocol.connection_made, transport) + resume_cb = self.call_soon(transport.resume_reading) + + try: + await waiter + except BaseException: + transport.close() + conmade_cb.cancel() + resume_cb.cancel() + raise + + return ssl_protocol._app_transport + + async def create_datagram_endpoint(self, protocol_factory, + local_addr=None, remote_addr=None, *, + family=0, proto=0, flags=0, + reuse_port=None, + allow_broadcast=None, sock=None): """Create datagram connection.""" if sock is not None: - if not _is_dgram_socket(sock): + if sock.type == socket.SOCK_STREAM: raise ValueError( - 'A UDP Socket was expected, got {!r}'.format(sock)) + f'A datagram socket was expected, got {sock!r}') if (local_addr or remote_addr or family or proto or flags or - reuse_address or reuse_port or allow_broadcast): + reuse_port or allow_broadcast): # show the problematic kwargs in exception msg opts = dict(local_addr=local_addr, remote_addr=remote_addr, family=family, proto=proto, flags=flags, - reuse_address=reuse_address, reuse_port=reuse_port, + reuse_port=reuse_port, allow_broadcast=allow_broadcast) - problems = ', '.join( - '{}={}'.format(k, v) for k, v in opts.items() if v) + problems = ', '.join(f'{k}={v}' for k, v in opts.items() if v) raise ValueError( - 'socket modifier keyword arguments can not be used ' - 'when sock is specified. ({})'.format(problems)) + f'socket modifier keyword arguments can not be used ' + f'when sock is specified. ({problems})') sock.setblocking(False) r_addr = None else: @@ -863,15 +1388,34 @@ def create_datagram_endpoint(self, protocol_factory, if family == 0: raise ValueError('unexpected address family') addr_pairs_info = (((family, proto), (None, None)),) + elif hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX: + for addr in (local_addr, remote_addr): + if addr is not None and not isinstance(addr, str): + raise TypeError('string is expected') + + if local_addr and local_addr[0] not in (0, '\x00'): + try: + if stat.S_ISSOCK(os.stat(local_addr).st_mode): + os.remove(local_addr) + except FileNotFoundError: + pass + except OSError as err: + # Directory may have permissions only to create socket. + logger.error('Unable to check or remove stale UNIX ' + 'socket %r: %r', + local_addr, err) + + addr_pairs_info = (((family, proto), + (local_addr, remote_addr)), ) else: # join address by (family, protocol) - addr_infos = collections.OrderedDict() + addr_infos = {} # Using order preserving dict for idx, addr in ((0, local_addr), (1, remote_addr)): if addr is not None: - assert isinstance(addr, tuple) and len(addr) == 2, ( - '2-tuple is expected') + if not (isinstance(addr, tuple) and len(addr) == 2): + raise TypeError('2-tuple is expected') - infos = yield from _ensure_resolved( + infos = await self._ensure_resolved( addr, family=family, type=socket.SOCK_DGRAM, proto=proto, flags=flags, loop=self) if not infos: @@ -894,9 +1438,6 @@ def create_datagram_endpoint(self, protocol_factory, exceptions = [] - if reuse_address is None: - reuse_address = os.name == 'posix' and sys.platform != 'cygwin' - for ((family, proto), (local_address, remote_address)) in addr_pairs_info: sock = None @@ -904,9 +1445,6 @@ def create_datagram_endpoint(self, protocol_factory, try: sock = socket.socket( family=family, type=socket.SOCK_DGRAM, proto=proto) - if reuse_address: - sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if reuse_port: _set_reuseport(sock) if allow_broadcast: @@ -917,7 +1455,8 @@ def create_datagram_endpoint(self, protocol_factory, if local_addr: sock.bind(local_address) if remote_addr: - yield from self.sock_connect(sock, remote_address) + if not allow_broadcast: + await self.sock_connect(sock, remote_address) r_addr = remote_address except OSError as exc: if sock is not None: @@ -947,36 +1486,51 @@ def create_datagram_endpoint(self, protocol_factory, remote_addr, transport, protocol) try: - yield from waiter + await waiter except: transport.close() raise return transport, protocol - @coroutine - def _create_server_getaddrinfo(self, host, port, family, flags): - infos = yield from _ensure_resolved((host, port), family=family, + async def _ensure_resolved(self, address, *, + family=0, type=socket.SOCK_STREAM, + proto=0, flags=0, loop): + host, port = address[:2] + info = _ipaddr_info(host, port, family, type, proto, *address[2:]) + if info is not None: + # "host" is already a resolved IP. + return [info] + else: + return await loop.getaddrinfo(host, port, family=family, type=type, + proto=proto, flags=flags) + + async def _create_server_getaddrinfo(self, host, port, family, flags): + infos = await self._ensure_resolved((host, port), family=family, type=socket.SOCK_STREAM, flags=flags, loop=self) if not infos: - raise OSError('getaddrinfo({!r}) returned empty list'.format(host)) + raise OSError(f'getaddrinfo({host!r}) returned empty list') return infos - @coroutine - def create_server(self, protocol_factory, host=None, port=None, - *, - family=socket.AF_UNSPEC, - flags=socket.AI_PASSIVE, - sock=None, - backlog=100, - ssl=None, - reuse_address=None, - reuse_port=None): + async def create_server( + self, protocol_factory, host=None, port=None, + *, + family=socket.AF_UNSPEC, + flags=socket.AI_PASSIVE, + sock=None, + backlog=100, + ssl=None, + reuse_address=None, + reuse_port=None, + keep_alive=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + start_serving=True): """Create a TCP server. - The host parameter can be a string, in that case the TCP server is bound - to host and port. + The host parameter can be a string, in that case the TCP server is + bound to host and port. The host parameter can also be a sequence of strings and in that case the TCP server is bound to all hosts of the sequence. If a host @@ -990,19 +1544,30 @@ def create_server(self, protocol_factory, host=None, port=None, """ if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') + + if ssl_handshake_timeout is not None and ssl is None: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') + + if ssl_shutdown_timeout is not None and ssl is None: + raise ValueError( + 'ssl_shutdown_timeout is only meaningful with ssl') + + if sock is not None: + _check_ssl_socket(sock) + if host is not None or port is not None: if sock is not None: raise ValueError( 'host/port and sock can not be specified at the same time') - AF_INET6 = getattr(socket, 'AF_INET6', 0) if reuse_address is None: - reuse_address = os.name == 'posix' and sys.platform != 'cygwin' + reuse_address = os.name == "posix" and sys.platform != "cygwin" sockets = [] if host == '': hosts = [None] elif (isinstance(host, str) or - not isinstance(host, collections.Iterable)): + not isinstance(host, collections.abc.Iterable)): hosts = [host] else: hosts = host @@ -1010,7 +1575,7 @@ def create_server(self, protocol_factory, host=None, port=None, fs = [self._create_server_getaddrinfo(host, port, family=family, flags=flags) for host in hosts] - infos = yield from tasks.gather(*fs, loop=self) + infos = await tasks.gather(*fs) infos = set(itertools.chain.from_iterable(infos)) completed = False @@ -1030,21 +1595,41 @@ def create_server(self, protocol_factory, host=None, port=None, if reuse_address: sock.setsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - if reuse_port: + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if reuse_port and af in (socket.AF_INET, socket.AF_INET6): _set_reuseport(sock) + if keep_alive: + sock.setsockopt( + socket.SOL_SOCKET, socket.SO_KEEPALIVE, True) # Disable IPv4/IPv6 dual stack support (enabled by # default on Linux) which makes a single socket # listen on both address families. - if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'): + if (_HAS_IPv6 and + af == socket.AF_INET6 and + hasattr(socket, 'IPPROTO_IPV6')): sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, True) try: sock.bind(sa) except OSError as err: - raise OSError(err.errno, 'error while attempting ' - 'to bind on address %r: %s' - % (sa, err.strerror.lower())) + msg = ('error while attempting ' + 'to bind on address %r: %s' + % (sa, str(err).lower())) + if err.errno == errno.EADDRNOTAVAIL: + # Assume the family is not enabled (bpo-30945) + sockets.pop() + sock.close() + if self._debug: + logger.warning(msg) + continue + raise OSError(err.errno, msg) from None + + if not sockets: + raise OSError('could not bind on any address out of %r' + % ([info[4] for info in infos],)) + completed = True finally: if not completed: @@ -1053,36 +1638,48 @@ def create_server(self, protocol_factory, host=None, port=None, else: if sock is None: raise ValueError('Neither host/port nor sock were specified') - if not _is_stream_socket(sock): - raise ValueError( - 'A Stream Socket was expected, got {!r}'.format(sock)) + if sock.type != socket.SOCK_STREAM: + raise ValueError(f'A Stream Socket was expected, got {sock!r}') sockets = [sock] - server = Server(self, sockets) for sock in sockets: - sock.listen(backlog) sock.setblocking(False) - self._start_serving(protocol_factory, sock, ssl, server, backlog) + + server = Server(self, sockets, protocol_factory, + ssl, backlog, ssl_handshake_timeout, + ssl_shutdown_timeout) + if start_serving: + server._start_serving() + # Skip one loop iteration so that all 'loop.add_reader' + # go through. + await tasks.sleep(0) + if self._debug: logger.info("%r is serving", server) return server - @coroutine - def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None): - """Handle an accepted connection. + async def connect_accepted_socket( + self, protocol_factory, sock, + *, ssl=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): + if sock.type != socket.SOCK_STREAM: + raise ValueError(f'A Stream Socket was expected, got {sock!r}') - This is used by servers that accept connections outside of - asyncio but that use asyncio to handle connections. + if ssl_handshake_timeout is not None and not ssl: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') - This method is a coroutine. When completed, the coroutine - returns a (transport, protocol) pair. - """ - if not _is_stream_socket(sock): + if ssl_shutdown_timeout is not None and not ssl: raise ValueError( - 'A Stream Socket was expected, got {!r}'.format(sock)) + 'ssl_shutdown_timeout is only meaningful with ssl') - transport, protocol = yield from self._create_connection_transport( - sock, protocol_factory, ssl, '', server_side=True) + _check_ssl_socket(sock) + + transport, protocol = await self._create_connection_transport( + sock, protocol_factory, ssl, '', server_side=True, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) if self._debug: # Get the socket from the transport because SSL transport closes # the old socket and creates a new SSL socket @@ -1090,14 +1687,13 @@ def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None): logger.debug("%r handled: (%r, %r)", sock, transport, protocol) return transport, protocol - @coroutine - def connect_read_pipe(self, protocol_factory, pipe): + async def connect_read_pipe(self, protocol_factory, pipe): protocol = protocol_factory() waiter = self.create_future() transport = self._make_read_pipe_transport(pipe, protocol, waiter) try: - yield from waiter + await waiter except: transport.close() raise @@ -1107,14 +1703,13 @@ def connect_read_pipe(self, protocol_factory, pipe): pipe.fileno(), transport, protocol) return transport, protocol - @coroutine - def connect_write_pipe(self, protocol_factory, pipe): + async def connect_write_pipe(self, protocol_factory, pipe): protocol = protocol_factory() waiter = self.create_future() transport = self._make_write_pipe_transport(pipe, protocol, waiter) try: - yield from waiter + await waiter except: transport.close() raise @@ -1127,21 +1722,24 @@ def connect_write_pipe(self, protocol_factory, pipe): def _log_subprocess(self, msg, stdin, stdout, stderr): info = [msg] if stdin is not None: - info.append('stdin=%s' % _format_pipe(stdin)) + info.append(f'stdin={_format_pipe(stdin)}') if stdout is not None and stderr == subprocess.STDOUT: - info.append('stdout=stderr=%s' % _format_pipe(stdout)) + info.append(f'stdout=stderr={_format_pipe(stdout)}') else: if stdout is not None: - info.append('stdout=%s' % _format_pipe(stdout)) + info.append(f'stdout={_format_pipe(stdout)}') if stderr is not None: - info.append('stderr=%s' % _format_pipe(stderr)) + info.append(f'stderr={_format_pipe(stderr)}') logger.debug(' '.join(info)) - @coroutine - def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=False, shell=True, bufsize=0, - **kwargs): + async def subprocess_shell(self, protocol_factory, cmd, *, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=False, + shell=True, bufsize=0, + encoding=None, errors=None, text=None, + **kwargs): if not isinstance(cmd, (bytes, str)): raise ValueError("cmd must be a string") if universal_newlines: @@ -1150,45 +1748,57 @@ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, raise ValueError("shell must be True") if bufsize != 0: raise ValueError("bufsize must be 0") + if text: + raise ValueError("text must be False") + if encoding is not None: + raise ValueError("encoding must be None") + if errors is not None: + raise ValueError("errors must be None") + protocol = protocol_factory() + debug_log = None if self._debug: # don't log parameters: they may contain sensitive information # (password) and may be too long debug_log = 'run shell command %r' % cmd self._log_subprocess(debug_log, stdin, stdout, stderr) - transport = yield from self._make_subprocess_transport( + transport = await self._make_subprocess_transport( protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs) - if self._debug: + if self._debug and debug_log is not None: logger.info('%s: %r', debug_log, transport) return transport, protocol - @coroutine - def subprocess_exec(self, protocol_factory, program, *args, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=False, - shell=False, bufsize=0, **kwargs): + async def subprocess_exec(self, protocol_factory, program, *args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=False, + shell=False, bufsize=0, + encoding=None, errors=None, text=None, + **kwargs): if universal_newlines: raise ValueError("universal_newlines must be False") if shell: raise ValueError("shell must be False") if bufsize != 0: raise ValueError("bufsize must be 0") + if text: + raise ValueError("text must be False") + if encoding is not None: + raise ValueError("encoding must be None") + if errors is not None: + raise ValueError("errors must be None") + popen_args = (program,) + args - for arg in popen_args: - if not isinstance(arg, (str, bytes)): - raise TypeError("program arguments must be " - "a bytes or text string, not %s" - % type(arg).__name__) protocol = protocol_factory() + debug_log = None if self._debug: # don't log parameters: they may contain sensitive information # (password) and may be too long - debug_log = 'execute program %r' % program + debug_log = f'execute program {program!r}' self._log_subprocess(debug_log, stdin, stdout, stderr) - transport = yield from self._make_subprocess_transport( + transport = await self._make_subprocess_transport( protocol, popen_args, False, stdin, stdout, stderr, bufsize, **kwargs) - if self._debug: + if self._debug and debug_log is not None: logger.info('%s: %r', debug_log, transport) return transport, protocol @@ -1210,8 +1820,8 @@ def set_exception_handler(self, handler): documentation for details about context). """ if handler is not None and not callable(handler): - raise TypeError('A callable object or None is expected, ' - 'got {!r}'.format(handler)) + raise TypeError(f'A callable object or None is expected, ' + f'got {handler!r}') self._exception_handler = handler def default_exception_handler(self, context): @@ -1221,6 +1831,11 @@ def default_exception_handler(self, context): handler is set, and can be called by a custom exception handler that wants to defer to the default behavior. + This default handler logs the error message and other + context-dependent information. In debug mode, a truncated + stack trace is also appended showing where the given object + (e.g. a handle or future or task) was created, if any. + The context parameter has the same meaning as in `call_exception_handler()`. """ @@ -1234,10 +1849,11 @@ def default_exception_handler(self, context): else: exc_info = False - if ('source_traceback' not in context - and self._current_handle is not None - and self._current_handle._source_traceback): - context['handle_traceback'] = self._current_handle._source_traceback + if ('source_traceback' not in context and + self._current_handle is not None and + self._current_handle._source_traceback): + context['handle_traceback'] = \ + self._current_handle._source_traceback log_lines = [message] for key in sorted(context): @@ -1254,7 +1870,7 @@ def default_exception_handler(self, context): value += tb.rstrip() else: value = repr(value) - log_lines.append('{}: {}'.format(key, value)) + log_lines.append(f'{key}: {value}') logger.error('\n'.join(log_lines), exc_info=exc_info) @@ -1266,10 +1882,13 @@ def call_exception_handler(self, context): - 'message': Error message; - 'exception' (optional): Exception object; - 'future' (optional): Future instance; + - 'task' (optional): Task instance; - 'handle' (optional): Handle instance; - 'protocol' (optional): Protocol instance; - 'transport' (optional): Transport instance; - 'socket' (optional): Socket instance; + - 'source_traceback' (optional): Traceback of the source; + - 'handle_traceback' (optional): Traceback of the handle; - 'asyncgen' (optional): Asynchronous generator that caused the exception. @@ -1282,7 +1901,9 @@ def call_exception_handler(self, context): if self._exception_handler is None: try: self.default_exception_handler(context) - except Exception: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: # Second protection layer for unexpected errors # in the default implementation, as well as for subclassed # event loops with overloaded "default_exception_handler". @@ -1290,8 +1911,25 @@ def call_exception_handler(self, context): exc_info=True) else: try: - self._exception_handler(self, context) - except Exception as exc: + ctx = None + thing = context.get("task") + if thing is None: + # Even though Futures don't have a context, + # Task is a subclass of Future, + # and sometimes the 'future' key holds a Task. + thing = context.get("future") + if thing is None: + # Handles also have a context. + thing = context.get("handle") + if thing is not None and hasattr(thing, "get_context"): + ctx = thing.get_context() + if ctx is not None and hasattr(ctx, "run"): + ctx.run(self._exception_handler, self, context) + else: + self._exception_handler(self, context) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: # Exception in the user set custom exception handler. try: # Let's try default handler. @@ -1300,7 +1938,9 @@ def call_exception_handler(self, context): 'exception': exc, 'context': context, }) - except Exception: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: # Guard 'default_exception_handler' in case it is # overloaded. logger.error('Exception in default exception handler ' @@ -1309,12 +1949,9 @@ def call_exception_handler(self, context): exc_info=True) def _add_callback(self, handle): - """Add a Handle to _scheduled (TimerHandle) or _ready.""" - assert isinstance(handle, events.Handle), 'A Handle is required here' - if handle._cancelled: - return - assert not isinstance(handle, events.TimerHandle) - self._ready.append(handle) + """Add a Handle to _ready.""" + if not handle._cancelled: + self._ready.append(handle) def _add_callback_signalsafe(self, handle): """Like _add_callback() but called from a signal handler.""" @@ -1362,32 +1999,16 @@ def _run_once(self): timeout = 0 elif self._scheduled: # Compute the desired timeout. - when = self._scheduled[0]._when - timeout = max(0, when - self.time()) - - if self._debug and timeout != 0: - t0 = self.time() - event_list = self._selector.select(timeout) - dt = self.time() - t0 - if dt >= 1.0: - level = logging.INFO - else: - level = logging.DEBUG - nevent = len(event_list) - if timeout is None: - logger.log(level, 'poll took %.3f ms: %s events', - dt * 1e3, nevent) - elif nevent: - logger.log(level, - 'poll %.3f ms took %.3f ms: %s events', - timeout * 1e3, dt * 1e3, nevent) - elif dt >= 1.0: - logger.log(level, - 'poll %.3f ms took %.3f ms: timeout', - timeout * 1e3, dt * 1e3) - else: - event_list = self._selector.select(timeout) + timeout = self._scheduled[0]._when - self.time() + if timeout > MAXIMUM_SELECT_TIMEOUT: + timeout = MAXIMUM_SELECT_TIMEOUT + elif timeout < 0: + timeout = 0 + + event_list = self._selector.select(timeout) self._process_events(event_list) + # Needed to break cycles when an exception occurs. + event_list = None # Handle 'later' callbacks that are ready. end_time = self.time() + self._clock_resolution @@ -1425,38 +2046,20 @@ def _run_once(self): handle._run() handle = None # Needed to break cycles when an exception occurs. - def _set_coroutine_wrapper(self, enabled): - try: - set_wrapper = sys.set_coroutine_wrapper - get_wrapper = sys.get_coroutine_wrapper - except AttributeError: - return - - enabled = bool(enabled) - if self._coroutine_wrapper_set == enabled: + def _set_coroutine_origin_tracking(self, enabled): + if bool(enabled) == bool(self._coroutine_origin_tracking_enabled): return - wrapper = coroutines.debug_wrapper - current_wrapper = get_wrapper() - if enabled: - if current_wrapper not in (None, wrapper): - warnings.warn( - "loop.set_debug(True): cannot set debug coroutine " - "wrapper; another wrapper is already set %r" % - current_wrapper, RuntimeWarning) - else: - set_wrapper(wrapper) - self._coroutine_wrapper_set = True + self._coroutine_origin_tracking_saved_depth = ( + sys.get_coroutine_origin_tracking_depth()) + sys.set_coroutine_origin_tracking_depth( + constants.DEBUG_STACK_DEPTH) else: - if current_wrapper not in (None, wrapper): - warnings.warn( - "loop.set_debug(False): cannot unset debug coroutine " - "wrapper; another wrapper was set %r" % - current_wrapper, RuntimeWarning) - else: - set_wrapper(None) - self._coroutine_wrapper_set = False + sys.set_coroutine_origin_tracking_depth( + self._coroutine_origin_tracking_saved_depth) + + self._coroutine_origin_tracking_enabled = enabled def get_debug(self): return self._debug @@ -1465,4 +2068,4 @@ def set_debug(self, enabled): self._debug = enabled if self.is_running(): - self._set_coroutine_wrapper(enabled) + self.call_soon_threadsafe(self._set_coroutine_origin_tracking, enabled) diff --git a/Lib/asyncio/base_futures.py b/Lib/asyncio/base_futures.py index 01259a062e6..7987963bd99 100644 --- a/Lib/asyncio/base_futures.py +++ b/Lib/asyncio/base_futures.py @@ -1,18 +1,8 @@ -__all__ = [] +__all__ = () -import concurrent.futures._base import reprlib -from . import events - -Error = concurrent.futures._base.Error -CancelledError = concurrent.futures.CancelledError -TimeoutError = concurrent.futures.TimeoutError - - -class InvalidStateError(Error): - """The operation is not allowed in this state.""" - +from . import format_helpers # States for Future. _PENDING = 'PENDING' @@ -38,17 +28,17 @@ def _format_callbacks(cb): cb = '' def format_cb(callback): - return events._format_callback_source(callback, ()) + return format_helpers._format_callback_source(callback, ()) if size == 1: - cb = format_cb(cb[0]) + cb = format_cb(cb[0][0]) elif size == 2: - cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1])) + cb = '{}, {}'.format(format_cb(cb[0][0]), format_cb(cb[1][0])) elif size > 2: - cb = '{}, <{} more>, {}'.format(format_cb(cb[0]), + cb = '{}, <{} more>, {}'.format(format_cb(cb[0][0]), size - 2, - format_cb(cb[-1])) - return 'cb=[%s]' % cb + format_cb(cb[-1][0])) + return f'cb=[{cb}]' def _future_repr_info(future): @@ -57,15 +47,21 @@ def _future_repr_info(future): info = [future._state.lower()] if future._state == _FINISHED: if future._exception is not None: - info.append('exception={!r}'.format(future._exception)) + info.append(f'exception={future._exception!r}') else: # use reprlib to limit the length of the output, especially # for very long strings result = reprlib.repr(future._result) - info.append('result={}'.format(result)) + info.append(f'result={result}') if future._callbacks: info.append(_format_callbacks(future._callbacks)) if future._source_traceback: frame = future._source_traceback[-1] - info.append('created at %s:%s' % (frame[0], frame[1])) + info.append(f'created at {frame[0]}:{frame[1]}') return info + + +@reprlib.recursive_repr() +def _future_repr(future): + info = ' '.join(_future_repr_info(future)) + return f'<{future.__class__.__name__} {info}>' diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index a00d9d5732f..321a4e5d5d1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -1,11 +1,12 @@ import collections import subprocess import warnings +import os +import signal +import sys -from . import compat from . import protocols from . import transports -from .coroutines import coroutine from .log import logger @@ -25,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -59,9 +61,9 @@ def __repr__(self): if self._closed: info.append('closed') if self._pid is not None: - info.append('pid=%s' % self._pid) + info.append(f'pid={self._pid}') if self._returncode is not None: - info.append('returncode=%s' % self._returncode) + info.append(f'returncode={self._returncode}') elif self._pid is not None: info.append('running') else: @@ -69,19 +71,19 @@ def __repr__(self): stdin = self._pipes.get(0) if stdin is not None: - info.append('stdin=%s' % stdin.pipe) + info.append(f'stdin={stdin.pipe}') stdout = self._pipes.get(1) stderr = self._pipes.get(2) if stdout is not None and stderr is stdout: - info.append('stdout=stderr=%s' % stdout.pipe) + info.append(f'stdout=stderr={stdout.pipe}') else: if stdout is not None: - info.append('stdout=%s' % stdout.pipe) + info.append(f'stdout={stdout.pipe}') if stderr is not None: - info.append('stderr=%s' % stderr.pipe) + info.append(f'stderr={stderr.pipe}') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): raise NotImplementedError @@ -103,33 +105,35 @@ def close(self): for proto in self._pipes.values(): if proto is None: continue - proto.pipe.close() - - if (self._proc is not None - # the child process finished? - and self._returncode is None - # the child process finished but the transport was not notified yet? - and self._proc.poll() is None - ): + # See gh-114177 + # skip closing the pipe if loop is already closed + # this can happen e.g. when loop is closed immediately after + # process is killed + if self._loop and not self._loop.is_closed(): + proto.pipe.close() + + if (self._proc is not None and + # has the child process finished? + self._returncode is None and + # the child process has finished, but the + # transport hasn't been notified yet? + self._proc.poll() is None): + if self._loop.get_debug(): logger.warning('Close running child process: kill %r', self) try: self._proc.kill() - except ProcessLookupError: + except (ProcessLookupError, PermissionError): + # the process may have already exited or may be running setuid pass # Don't clear the _proc reference yet: _post_init() may still run - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning, - source=self) - self.close() + def __del__(self, _warn=warnings.warn): + if not self._closed: + _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) + self.close() def get_pid(self): return self._pid @@ -147,38 +151,51 @@ def _check_proc(self): if self._proc is None: raise ProcessLookupError() - def send_signal(self, signal): - self._check_proc() - self._proc.send_signal(signal) + if sys.platform == 'win32': + def send_signal(self, signal): + self._check_proc() + self._proc.send_signal(signal) + + def terminate(self): + self._check_proc() + self._proc.terminate() + + def kill(self): + self._check_proc() + self._proc.kill() + else: + def send_signal(self, signal): + self._check_proc() + try: + os.kill(self._proc.pid, signal) + except ProcessLookupError: + pass - def terminate(self): - self._check_proc() - self._proc.terminate() + def terminate(self): + self.send_signal(signal.SIGTERM) - def kill(self): - self._check_proc() - self._proc.kill() + def kill(self): + self.send_signal(signal.SIGKILL) - @coroutine - def _connect_pipes(self, waiter): + async def _connect_pipes(self, waiter): try: proc = self._proc loop = self._loop if proc.stdin is not None: - _, pipe = yield from loop.connect_write_pipe( + _, pipe = await loop.connect_write_pipe( lambda: WriteSubprocessPipeProto(self, 0), proc.stdin) self._pipes[0] = pipe if proc.stdout is not None: - _, pipe = yield from loop.connect_read_pipe( + _, pipe = await loop.connect_read_pipe( lambda: ReadSubprocessPipeProto(self, 1), proc.stdout) self._pipes[1] = pipe if proc.stderr is not None: - _, pipe = yield from loop.connect_read_pipe( + _, pipe = await loop.connect_read_pipe( lambda: ReadSubprocessPipeProto(self, 2), proc.stderr) self._pipes[2] = pipe @@ -189,12 +206,15 @@ def _connect_pipes(self, waiter): for callback, data in self._pending_calls: loop.call_soon(callback, *data) self._pending_calls = None - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: if waiter is not None and not waiter.cancelled(): waiter.set_exception(exc) else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -213,24 +233,17 @@ def _process_exited(self, returncode): assert returncode is not None, returncode assert self._returncode is None, self._returncode if self._loop.get_debug(): - logger.info('%r exited with return code %r', - self, returncode) + logger.info('%r exited with return code %r', self, returncode) self._returncode = returncode if self._proc.returncode is None: # asyncio uses a child watcher: copy the status into the Popen # object. On Python 3.6, it is required to avoid a ResourceWarning. self._proc.returncode = returncode self._call(self._protocol.process_exited) - self._try_finish() - # wake up futures waiting for wait() - for waiter in self._exit_waiters: - if not waiter.cancelled(): - waiter.set_result(returncode) - self._exit_waiters = None + self._try_finish() - @coroutine - def _wait(self): + async def _wait(self): """Wait until the process exit and return the process return code. This method is a coroutine.""" @@ -239,12 +252,21 @@ def _wait(self): waiter = self._loop.create_future() self._exit_waiters.append(waiter) - return (yield from waiter) + return await waiter def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True @@ -254,6 +276,11 @@ def _call_connection_lost(self, exc): try: self._protocol.connection_lost(exc) finally: + # wake up futures waiting for wait() + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) + self._exit_waiters = None self._loop = None self._proc = None self._protocol = None @@ -271,8 +298,7 @@ def connection_made(self, transport): self.pipe = transport def __repr__(self): - return ('<%s fd=%s pipe=%r>' - % (self.__class__.__name__, self.fd, self.pipe)) + return f'<{self.__class__.__name__} fd={self.fd} pipe={self.pipe!r}>' def connection_lost(self, exc): self.disconnected = True diff --git a/Lib/asyncio/base_tasks.py b/Lib/asyncio/base_tasks.py index 5f34434c576..c907b683413 100644 --- a/Lib/asyncio/base_tasks.py +++ b/Lib/asyncio/base_tasks.py @@ -1,4 +1,5 @@ import linecache +import reprlib import traceback from . import base_futures @@ -8,25 +9,42 @@ def _task_repr_info(task): info = base_futures._future_repr_info(task) - if task._must_cancel: + if task.cancelling() and not task.done(): # replace status info[0] = 'cancelling' - coro = coroutines._format_coroutine(task._coro) - info.insert(1, 'coro=<%s>' % coro) + info.insert(1, 'name=%r' % task.get_name()) if task._fut_waiter is not None: - info.insert(2, 'wait_for=%r' % task._fut_waiter) + info.insert(2, f'wait_for={task._fut_waiter!r}') + + if task._coro: + coro = coroutines._format_coroutine(task._coro) + info.insert(2, f'coro=<{coro}>') + return info +@reprlib.recursive_repr() +def _task_repr(task): + info = ' '.join(_task_repr_info(task)) + return f'<{task.__class__.__name__} {info}>' + + def _task_get_stack(task, limit): frames = [] - try: - # 'async def' coroutines + if hasattr(task._coro, 'cr_frame'): + # case 1: 'async def' coroutines f = task._coro.cr_frame - except AttributeError: + elif hasattr(task._coro, 'gi_frame'): + # case 2: legacy coroutines f = task._coro.gi_frame + elif hasattr(task._coro, 'ag_frame'): + # case 3: async generators + f = task._coro.ag_frame + else: + # case 4: unknown objects + f = None if f is not None: while f is not None: if limit is not None: @@ -61,15 +79,15 @@ def _task_print_stack(task, limit, file): linecache.checkcache(filename) line = linecache.getline(filename, lineno, f.f_globals) extracted_list.append((filename, lineno, name, line)) + exc = task._exception if not extracted_list: - print('No stack for %r' % task, file=file) + print(f'No stack for {task!r}', file=file) elif exc is not None: - print('Traceback for %r (most recent call last):' % task, - file=file) + print(f'Traceback for {task!r} (most recent call last):', file=file) else: - print('Stack for %r (most recent call last):' % task, - file=file) + print(f'Stack for {task!r} (most recent call last):', file=file) + traceback.print_list(extracted_list, file=file) if exc is not None: for line in traceback.format_exception_only(exc.__class__, exc): diff --git a/Lib/asyncio/compat.py b/Lib/asyncio/compat.py deleted file mode 100644 index 4790bb4a35f..00000000000 --- a/Lib/asyncio/compat.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Compatibility helpers for the different Python versions.""" - -import sys - -PY34 = sys.version_info >= (3, 4) -PY35 = sys.version_info >= (3, 5) -PY352 = sys.version_info >= (3, 5, 2) - - -def flatten_list_bytes(list_of_data): - """Concatenate a sequence of bytes-like objects.""" - if not PY34: - # On Python 3.3 and older, bytes.join() doesn't handle - # memoryview. - list_of_data = ( - bytes(data) if isinstance(data, memoryview) else data - for data in list_of_data) - return b''.join(list_of_data) diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index f9e123281e2..b60c1e4236a 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -1,7 +1,41 @@ -"""Constants.""" +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + +import enum # After the connection is lost, log warnings after this many write()s. LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5 # Seconds to wait before retrying accept(). ACCEPT_RETRY_DELAY = 1 + +# Number of stack entries to capture in debug mode. +# The larger the number, the slower the operation in debug mode +# (see extract_stack() in format_helpers.py). +DEBUG_STACK_DEPTH = 10 + +# Number of seconds to wait for SSL handshake to complete +# The default timeout matches that of Nginx. +SSL_HANDSHAKE_TIMEOUT = 60.0 + +# Number of seconds to wait for SSL shutdown to complete +# The default timeout mimics lingering_time +SSL_SHUTDOWN_TIMEOUT = 30.0 + +# Used in sendfile fallback code. We use fallback for platforms +# that don't support sendfile, or for TLS connections. +SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 256 + +FLOW_CONTROL_HIGH_WATER_SSL_READ = 256 # KiB +FLOW_CONTROL_HIGH_WATER_SSL_WRITE = 512 # KiB + +# Default timeout for joining the threads in the threadpool +THREAD_JOIN_TIMEOUT = 300 + +# The enum should be here to break circular dependencies between +# base_events and sslproto +class _SendfileMode(enum.Enum): + UNSUPPORTED = enum.auto() + TRY_NATIVE = enum.auto() + FALLBACK = enum.auto() diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index 08e94412b33..a51319cb72a 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -1,249 +1,16 @@ -__all__ = ['coroutine', - 'iscoroutinefunction', 'iscoroutine'] +__all__ = 'iscoroutinefunction', 'iscoroutine' -import functools +import collections.abc import inspect -import opcode import os import sys -import traceback import types -from . import compat -from . import events -from . import base_futures -from .log import logger - -# Opcode of "yield from" instruction -_YIELD_FROM = opcode.opmap['YIELD_FROM'] - -# If you set _DEBUG to true, @coroutine will wrap the resulting -# generator objects in a CoroWrapper instance (defined below). That -# instance will log a message when the generator is never iterated -# over, which may happen when you forget to use "yield from" with a -# coroutine call. Note that the value of the _DEBUG flag is taken -# when the decorator is used, so to be of any use it must be set -# before you define your coroutines. A downside of using this feature -# is that tracebacks show entries for the CoroWrapper.__next__ method -# when _DEBUG is true. -_DEBUG = (not sys.flags.ignore_environment and - bool(os.environ.get('PYTHONASYNCIODEBUG'))) - - -try: - _types_coroutine = types.coroutine - _types_CoroutineType = types.CoroutineType -except AttributeError: - # Python 3.4 - _types_coroutine = None - _types_CoroutineType = None - -try: - _inspect_iscoroutinefunction = inspect.iscoroutinefunction -except AttributeError: - # Python 3.4 - _inspect_iscoroutinefunction = lambda func: False - -try: - from collections.abc import Coroutine as _CoroutineABC, \ - Awaitable as _AwaitableABC -except ImportError: - _CoroutineABC = _AwaitableABC = None - - -# Check for CPython issue #21209 -def has_yield_from_bug(): - class MyGen: - def __init__(self): - self.send_args = None - def __iter__(self): - return self - def __next__(self): - return 42 - def send(self, *what): - self.send_args = what - return None - def yield_from_gen(gen): - yield from gen - value = (1, 2, 3) - gen = MyGen() - coro = yield_from_gen(gen) - next(coro) - coro.send(value) - return gen.send_args != (value,) -_YIELD_FROM_BUG = has_yield_from_bug() -del has_yield_from_bug - - -def debug_wrapper(gen): - # This function is called from 'sys.set_coroutine_wrapper'. - # We only wrap here coroutines defined via 'async def' syntax. - # Generator-based coroutines are wrapped in @coroutine - # decorator. - return CoroWrapper(gen, None) - - -class CoroWrapper: - # Wrapper for coroutine object in _DEBUG mode. - - def __init__(self, gen, func=None): - assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen - self.gen = gen - self.func = func # Used to unwrap @coroutine decorator - self._source_traceback = traceback.extract_stack(sys._getframe(1)) - self.__name__ = getattr(gen, '__name__', None) - self.__qualname__ = getattr(gen, '__qualname__', None) - - def __repr__(self): - coro_repr = _format_coroutine(self) - if self._source_traceback: - frame = self._source_traceback[-1] - coro_repr += ', created at %s:%s' % (frame[0], frame[1]) - return '<%s %s>' % (self.__class__.__name__, coro_repr) - - def __iter__(self): - return self - - def __next__(self): - return self.gen.send(None) - - if _YIELD_FROM_BUG: - # For for CPython issue #21209: using "yield from" and a custom - # generator, generator.send(tuple) unpacks the tuple instead of passing - # the tuple unchanged. Check if the caller is a generator using "yield - # from" to decide if the parameter should be unpacked or not. - def send(self, *value): - frame = sys._getframe() - caller = frame.f_back - assert caller.f_lasti >= 0 - if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM: - value = value[0] - return self.gen.send(value) - else: - def send(self, value): - return self.gen.send(value) - - def throw(self, type, value=None, traceback=None): - return self.gen.throw(type, value, traceback) - - def close(self): - return self.gen.close() - - @property - def gi_frame(self): - return self.gen.gi_frame - - @property - def gi_running(self): - return self.gen.gi_running - - @property - def gi_code(self): - return self.gen.gi_code - - if compat.PY35: - - def __await__(self): - cr_await = getattr(self.gen, 'cr_await', None) - if cr_await is not None: - raise RuntimeError( - "Cannot await on coroutine {!r} while it's " - "awaiting for {!r}".format(self.gen, cr_await)) - return self - - @property - def gi_yieldfrom(self): - return self.gen.gi_yieldfrom - - @property - def cr_await(self): - return self.gen.cr_await - - @property - def cr_running(self): - return self.gen.cr_running - - @property - def cr_code(self): - return self.gen.cr_code - - @property - def cr_frame(self): - return self.gen.cr_frame - - def __del__(self): - # Be careful accessing self.gen.frame -- self.gen might not exist. - gen = getattr(self, 'gen', None) - frame = getattr(gen, 'gi_frame', None) - if frame is None: - frame = getattr(gen, 'cr_frame', None) - if frame is not None and frame.f_lasti == -1: - msg = '%r was never yielded from' % self - tb = getattr(self, '_source_traceback', ()) - if tb: - tb = ''.join(traceback.format_list(tb)) - msg += ('\nCoroutine object created at ' - '(most recent call last):\n') - msg += tb.rstrip() - logger.error(msg) - - -def coroutine(func): - """Decorator to mark coroutines. - - If the coroutine is not yielded from before it is destroyed, - an error message is logged. - """ - if _inspect_iscoroutinefunction(func): - # In Python 3.5 that's all we need to do for coroutines - # defiend with "async def". - # Wrapping in CoroWrapper will happen via - # 'sys.set_coroutine_wrapper' function. - return func - - if inspect.isgeneratorfunction(func): - coro = func - else: - @functools.wraps(func) - def coro(*args, **kw): - res = func(*args, **kw) - if (base_futures.isfuture(res) or inspect.isgenerator(res) or - isinstance(res, CoroWrapper)): - res = yield from res - elif _AwaitableABC is not None: - # If 'func' returns an Awaitable (new in 3.5) we - # want to run it. - try: - await_meth = res.__await__ - except AttributeError: - pass - else: - if isinstance(res, _AwaitableABC): - res = yield from await_meth() - return res - - if not _DEBUG: - if _types_coroutine is None: - wrapper = coro - else: - wrapper = _types_coroutine(coro) - else: - @functools.wraps(func) - def wrapper(*args, **kwds): - w = CoroWrapper(coro(*args, **kwds), func=func) - if w._source_traceback: - del w._source_traceback[-1] - # Python < 3.5 does not implement __qualname__ - # on generator objects, so we set it manually. - # We use getattr as some callables (such as - # functools.partial may lack __qualname__). - w.__name__ = getattr(func, '__name__', None) - w.__qualname__ = getattr(func, '__qualname__', None) - return w - - wrapper._is_coroutine = _is_coroutine # For iscoroutinefunction(). - return wrapper +def _is_debug_mode(): + # See: https://docs.python.org/3/library/asyncio-dev.html#asyncio-debug-mode. + return sys.flags.dev_mode or (not sys.flags.ignore_environment and + bool(os.environ.get('PYTHONASYNCIODEBUG'))) # A marker for iscoroutinefunction. @@ -251,94 +18,101 @@ def wrapper(*args, **kwds): def iscoroutinefunction(func): + import warnings """Return True if func is a decorated coroutine function.""" - return (getattr(func, '_is_coroutine', None) is _is_coroutine or - _inspect_iscoroutinefunction(func)) + warnings._deprecated("asyncio.iscoroutinefunction", + f"{warnings._DEPRECATED_MSG}; " + "use inspect.iscoroutinefunction() instead", + remove=(3,16)) + return _iscoroutinefunction(func) + +def _iscoroutinefunction(func): + return (inspect.iscoroutinefunction(func) or + getattr(func, '_is_coroutine', None) is _is_coroutine) -_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper) -if _CoroutineABC is not None: - _COROUTINE_TYPES += (_CoroutineABC,) -if _types_CoroutineType is not None: - # Prioritize native coroutine check to speed-up - # asyncio.iscoroutine. - _COROUTINE_TYPES = (_types_CoroutineType,) + _COROUTINE_TYPES + +# Prioritize native coroutine check to speed-up +# asyncio.iscoroutine. +_COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine) +_iscoroutine_typecache = set() def iscoroutine(obj): """Return True if obj is a coroutine object.""" - return isinstance(obj, _COROUTINE_TYPES) + if type(obj) in _iscoroutine_typecache: + return True + + if isinstance(obj, _COROUTINE_TYPES): + # Just in case we don't want to cache more than 100 + # positive types. That shouldn't ever happen, unless + # someone stressing the system on purpose. + if len(_iscoroutine_typecache) < 100: + _iscoroutine_typecache.add(type(obj)) + return True + else: + return False def _format_coroutine(coro): assert iscoroutine(coro) - if not hasattr(coro, 'cr_code') and not hasattr(coro, 'gi_code'): - # Most likely a built-in type or a Cython coroutine. - - # Built-in types might not have __qualname__ or __name__. - coro_name = getattr( - coro, '__qualname__', - getattr(coro, '__name__', type(coro).__name__)) - coro_name = '{}()'.format(coro_name) + def get_name(coro): + # Coroutines compiled with Cython sometimes don't have + # proper __qualname__ or __name__. While that is a bug + # in Cython, asyncio shouldn't crash with an AttributeError + # in its __repr__ functions. + if hasattr(coro, '__qualname__') and coro.__qualname__: + coro_name = coro.__qualname__ + elif hasattr(coro, '__name__') and coro.__name__: + coro_name = coro.__name__ + else: + # Stop masking Cython bugs, expose them in a friendly way. + coro_name = f'<{type(coro).__name__} without __name__>' + return f'{coro_name}()' - running = False + def is_running(coro): try: - running = coro.cr_running + return coro.cr_running except AttributeError: try: - running = coro.gi_running + return coro.gi_running except AttributeError: - pass + return False - if running: - return '{} running'.format(coro_name) - else: - return coro_name - - coro_name = None - if isinstance(coro, CoroWrapper): - func = coro.func - coro_name = coro.__qualname__ - if coro_name is not None: - coro_name = '{}()'.format(coro_name) - else: - func = coro + coro_code = None + if hasattr(coro, 'cr_code') and coro.cr_code: + coro_code = coro.cr_code + elif hasattr(coro, 'gi_code') and coro.gi_code: + coro_code = coro.gi_code - if coro_name is None: - coro_name = events._format_callback(func, (), {}) + coro_name = get_name(coro) - try: - coro_code = coro.gi_code - except AttributeError: - coro_code = coro.cr_code + if not coro_code: + # Built-in types might not have __qualname__ or __name__. + if is_running(coro): + return f'{coro_name} running' + else: + return coro_name - try: + coro_frame = None + if hasattr(coro, 'gi_frame') and coro.gi_frame: coro_frame = coro.gi_frame - except AttributeError: + elif hasattr(coro, 'cr_frame') and coro.cr_frame: coro_frame = coro.cr_frame - filename = coro_code.co_filename + # If Cython's coroutine has a fake code object without proper + # co_filename -- expose that. + filename = coro_code.co_filename or '' + lineno = 0 - if (isinstance(coro, CoroWrapper) and - not inspect.isgeneratorfunction(coro.func) and - coro.func is not None): - source = events._get_function_source(coro.func) - if source is not None: - filename, lineno = source - if coro_frame is None: - coro_repr = ('%s done, defined at %s:%s' - % (coro_name, filename, lineno)) - else: - coro_repr = ('%s running, defined at %s:%s' - % (coro_name, filename, lineno)) - elif coro_frame is not None: + + if coro_frame is not None: lineno = coro_frame.f_lineno - coro_repr = ('%s running at %s:%s' - % (coro_name, filename, lineno)) + coro_repr = f'{coro_name} running at {filename}:{lineno}' + else: lineno = coro_code.co_firstlineno - coro_repr = ('%s done, defined at %s:%s' - % (coro_name, filename, lineno)) + coro_repr = f'{coro_name} done, defined at {filename}:{lineno}' return coro_repr diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 466db6d9a3d..a7fb55982ab 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -1,96 +1,55 @@ """Event loop and event loop policy.""" -__all__ = ['AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - 'get_event_loop_policy', 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', - '_set_running_loop', 'get_running_loop', - '_get_running_loop', - ] - -import functools -import inspect -import reprlib +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + +__all__ = ( + "AbstractEventLoop", + "AbstractServer", + "Handle", + "TimerHandle", + "get_event_loop_policy", + "set_event_loop_policy", + "get_event_loop", + "set_event_loop", + "new_event_loop", + "_set_running_loop", + "get_running_loop", + "_get_running_loop", +) + +import contextvars +import os +import signal import socket import subprocess import sys import threading -import traceback +import warnings -from asyncio import compat - - -def _get_function_source(func): - if compat.PY34: - func = inspect.unwrap(func) - elif hasattr(func, '__wrapped__'): - func = func.__wrapped__ - if inspect.isfunction(func): - code = func.__code__ - return (code.co_filename, code.co_firstlineno) - if isinstance(func, functools.partial): - return _get_function_source(func.func) - if compat.PY34 and isinstance(func, functools.partialmethod): - return _get_function_source(func.func) - return None - - -def _format_args_and_kwargs(args, kwargs): - """Format function arguments and keyword arguments. - - Special case for a single parameter: ('hello',) is formatted as ('hello'). - """ - # use reprlib to limit the length of the output - items = [] - if args: - items.extend(reprlib.repr(arg) for arg in args) - if kwargs: - items.extend('{}={}'.format(k, reprlib.repr(v)) - for k, v in kwargs.items()) - return '(' + ', '.join(items) + ')' - - -def _format_callback(func, args, kwargs, suffix=''): - if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) - - if hasattr(func, '__qualname__'): - func_repr = getattr(func, '__qualname__') - elif hasattr(func, '__name__'): - func_repr = getattr(func, '__name__') - else: - func_repr = repr(func) - - func_repr += _format_args_and_kwargs(args, kwargs) - if suffix: - func_repr += suffix - return func_repr - -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) - source = _get_function_source(func) - if source: - func_repr += ' at %s:%s' % source - return func_repr +from . import format_helpers class Handle: """Object returned by callback registration methods.""" __slots__ = ('_callback', '_args', '_cancelled', '_loop', - '_source_traceback', '_repr', '__weakref__') + '_source_traceback', '_repr', '__weakref__', + '_context') - def __init__(self, callback, args, loop): + def __init__(self, callback, args, loop, context=None): + if context is None: + context = contextvars.copy_context() + self._context = context self._loop = loop self._callback = callback self._args = args self._cancelled = False self._repr = None if self._loop.get_debug(): - self._source_traceback = traceback.extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) else: self._source_traceback = None @@ -99,17 +58,22 @@ def _repr_info(self): if self._cancelled: info.append('cancelled') if self._callback is not None: - info.append(_format_callback_source(self._callback, self._args)) + info.append(format_helpers._format_callback_source( + self._callback, self._args, + debug=self._loop.get_debug())) if self._source_traceback: frame = self._source_traceback[-1] - info.append('created at %s:%s' % (frame[0], frame[1])) + info.append(f'created at {frame[0]}:{frame[1]}') return info def __repr__(self): if self._repr is not None: return self._repr info = self._repr_info() - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) + + def get_context(self): + return self._context def cancel(self): if not self._cancelled: @@ -122,12 +86,19 @@ def cancel(self): self._callback = None self._args = None + def cancelled(self): + return self._cancelled + def _run(self): try: - self._callback(*self._args) - except Exception as exc: - cb = _format_callback_source(self._callback, self._args) - msg = 'Exception in callback {}'.format(cb) + self._context.run(self._callback, *self._args) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + cb = format_helpers._format_callback_source( + self._callback, self._args, + debug=self._loop.get_debug()) + msg = f'Exception in callback {cb}' context = { 'message': msg, 'exception': exc, @@ -138,15 +109,42 @@ def _run(self): self._loop.call_exception_handler(context) self = None # Needed to break cycles when an exception occurs. +# _ThreadSafeHandle is used for callbacks scheduled with call_soon_threadsafe +# and is thread safe unlike Handle which is not thread safe. +class _ThreadSafeHandle(Handle): + + __slots__ = ('_lock',) + + def __init__(self, callback, args, loop, context=None): + super().__init__(callback, args, loop, context) + self._lock = threading.RLock() + + def cancel(self): + with self._lock: + return super().cancel() + + def cancelled(self): + with self._lock: + return super().cancelled() + + def _run(self): + # The event loop checks for cancellation without holding the lock + # It is possible that the handle is cancelled after the check + # but before the callback is called so check it again after acquiring + # the lock and return without calling the callback if it is cancelled. + with self._lock: + if self._cancelled: + return + return super()._run() + class TimerHandle(Handle): """Object returned by timed callback registration methods.""" __slots__ = ['_scheduled', '_when'] - def __init__(self, when, callback, args, loop): - assert when is not None - super().__init__(callback, args, loop) + def __init__(self, when, callback, args, loop, context=None): + super().__init__(callback, args, loop, context) if self._source_traceback: del self._source_traceback[-1] self._when = when @@ -155,27 +153,31 @@ def __init__(self, when, callback, args, loop): def _repr_info(self): info = super()._repr_info() pos = 2 if self._cancelled else 1 - info.insert(pos, 'when=%s' % self._when) + info.insert(pos, f'when={self._when}') return info def __hash__(self): return hash(self._when) def __lt__(self, other): - return self._when < other._when + if isinstance(other, TimerHandle): + return self._when < other._when + return NotImplemented def __le__(self, other): - if self._when < other._when: - return True - return self.__eq__(other) + if isinstance(other, TimerHandle): + return self._when < other._when or self.__eq__(other) + return NotImplemented def __gt__(self, other): - return self._when > other._when + if isinstance(other, TimerHandle): + return self._when > other._when + return NotImplemented def __ge__(self, other): - if self._when > other._when: - return True - return self.__eq__(other) + if isinstance(other, TimerHandle): + return self._when > other._when or self.__eq__(other) + return NotImplemented def __eq__(self, other): if isinstance(other, TimerHandle): @@ -185,26 +187,68 @@ def __eq__(self, other): self._cancelled == other._cancelled) return NotImplemented - def __ne__(self, other): - equal = self.__eq__(other) - return NotImplemented if equal is NotImplemented else not equal - def cancel(self): if not self._cancelled: self._loop._timer_handle_cancelled(self) super().cancel() + def when(self): + """Return a scheduled callback time. + + The time is an absolute timestamp, using the same time + reference as loop.time(). + """ + return self._when + class AbstractServer: """Abstract server returned by create_server().""" def close(self): """Stop serving. This leaves existing connections open.""" - return NotImplemented + raise NotImplementedError + + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + + def get_loop(self): + """Get the event loop the Server object is attached to.""" + raise NotImplementedError + + def is_serving(self): + """Return True if the server is accepting connections.""" + raise NotImplementedError + + async def start_serving(self): + """Start accepting connections. - def wait_closed(self): + This method is idempotent, so it can be called when + the server is already being serving. + """ + raise NotImplementedError + + async def serve_forever(self): + """Start accepting connections until the coroutine is cancelled. + + The server is closed when the coroutine is cancelled. + """ + raise NotImplementedError + + async def wait_closed(self): """Coroutine to wait until service is closed.""" - return NotImplemented + raise NotImplementedError + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + self.close() + await self.wait_closed() class AbstractEventLoop: @@ -250,23 +294,27 @@ def close(self): """ raise NotImplementedError - def shutdown_asyncgens(self): + async def shutdown_asyncgens(self): """Shutdown all active asynchronous generators.""" raise NotImplementedError + async def shutdown_default_executor(self): + """Schedule the shutdown of the default executor.""" + raise NotImplementedError + # Methods scheduling callbacks. All these return Handles. def _timer_handle_cancelled(self, handle): """Notification that a TimerHandle has been cancelled.""" raise NotImplementedError - def call_soon(self, callback, *args): - return self.call_later(0, callback, *args) + def call_soon(self, callback, *args, context=None): + return self.call_later(0, callback, *args, context=context) - def call_later(self, delay, callback, *args): + def call_later(self, delay, callback, *args, context=None): raise NotImplementedError - def call_at(self, when, callback, *args): + def call_at(self, when, callback, *args, context=None): raise NotImplementedError def time(self): @@ -277,12 +325,12 @@ def create_future(self): # Method scheduling a coroutine object: create a task. - def create_task(self, coro): + def create_task(self, coro, **kwargs): raise NotImplementedError # Methods for interacting with threads. - def call_soon_threadsafe(self, callback, *args): + def call_soon_threadsafe(self, callback, *args, context=None): raise NotImplementedError def run_in_executor(self, executor, func, *args): @@ -293,21 +341,32 @@ def set_default_executor(self, executor): # Network I/O methods returning Futures. - def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0): + async def getaddrinfo(self, host, port, *, + family=0, type=0, proto=0, flags=0): raise NotImplementedError - def getnameinfo(self, sockaddr, flags=0): + async def getnameinfo(self, sockaddr, flags=0): raise NotImplementedError - def create_connection(self, protocol_factory, host=None, port=None, *, - ssl=None, family=0, proto=0, flags=0, sock=None, - local_addr=None, server_hostname=None): + async def create_connection( + self, protocol_factory, host=None, port=None, + *, ssl=None, family=0, proto=0, + flags=0, sock=None, local_addr=None, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + happy_eyeballs_delay=None, interleave=None): raise NotImplementedError - def create_server(self, protocol_factory, host=None, port=None, *, - family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, - sock=None, backlog=100, ssl=None, reuse_address=None, - reuse_port=None): + async def create_server( + self, protocol_factory, host=None, port=None, + *, family=socket.AF_UNSPEC, + flags=socket.AI_PASSIVE, sock=None, backlog=100, + ssl=None, reuse_address=None, reuse_port=None, + keep_alive=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + start_serving=True): """A coroutine which creates a TCP server bound to host and port. The return value is a Server object which can be used to stop @@ -315,8 +374,8 @@ def create_server(self, protocol_factory, host=None, port=None, *, If host is an empty string or None all interfaces are assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be a - sequence (e.g. list) of hosts to bind to. + one for IPv4 and another one for IPv6). The host parameter can also be + a sequence (e.g. list) of hosts to bind to. family can be set to either AF_INET or AF_INET6 to force the socket to use IPv4 or IPv6. If not set it will be determined @@ -342,22 +401,65 @@ def create_server(self, protocol_factory, host=None, port=None, *, the same port as other existing endpoints are bound to, so long as they all set this flag when being created. This option is not supported on Windows. + + keep_alive set to True keeps connections active by enabling the + periodic transmission of messages. + + ssl_handshake_timeout is the time in seconds that an SSL server + will wait for completion of the SSL handshake before aborting the + connection. Default is 60s. + + ssl_shutdown_timeout is the time in seconds that an SSL server + will wait for completion of the SSL shutdown procedure + before aborting the connection. Default is 30s. + + start_serving set to True (default) causes the created server + to start accepting connections immediately. When set to False, + the user should await Server.start_serving() or Server.serve_forever() + to make the server to start accepting connections. """ raise NotImplementedError - def create_unix_connection(self, protocol_factory, path, *, - ssl=None, sock=None, - server_hostname=None): + async def sendfile(self, transport, file, offset=0, count=None, + *, fallback=True): + """Send a file through a transport. + + Return an amount of sent bytes. + """ raise NotImplementedError - def create_unix_server(self, protocol_factory, path, *, - sock=None, backlog=100, ssl=None): + async def start_tls(self, transport, protocol, sslcontext, *, + server_side=False, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): + """Upgrade a transport to TLS. + + Return a new transport that *protocol* should start using + immediately. + """ + raise NotImplementedError + + async def create_unix_connection( + self, protocol_factory, path=None, *, + ssl=None, sock=None, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): + raise NotImplementedError + + async def create_unix_server( + self, protocol_factory, path=None, *, + sock=None, backlog=100, ssl=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + start_serving=True): """A coroutine which creates a UNIX Domain Socket server. The return value is a Server object, which can be used to stop the service. - path is a str, representing a file systsem path to bind the + path is a str, representing a file system path to bind the server socket to. sock can optionally be specified in order to use a preexisting @@ -368,14 +470,40 @@ def create_unix_server(self, protocol_factory, path, *, ssl can be set to an SSLContext to enable SSL over the accepted connections. + + ssl_handshake_timeout is the time in seconds that an SSL server + will wait for the SSL handshake to complete (defaults to 60s). + + ssl_shutdown_timeout is the time in seconds that an SSL server + will wait for the SSL shutdown to finish (defaults to 30s). + + start_serving set to True (default) causes the created server + to start accepting connections immediately. When set to False, + the user should await Server.start_serving() or Server.serve_forever() + to make the server to start accepting connections. + """ + raise NotImplementedError + + async def connect_accepted_socket( + self, protocol_factory, sock, + *, ssl=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): + """Handle an accepted connection. + + This is used by servers that accept connections outside of + asyncio, but use asyncio to handle connections. + + This method is a coroutine. When completed, the coroutine + returns a (transport, protocol) pair. """ raise NotImplementedError - def create_datagram_endpoint(self, protocol_factory, - local_addr=None, remote_addr=None, *, - family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, - allow_broadcast=None, sock=None): + async def create_datagram_endpoint(self, protocol_factory, + local_addr=None, remote_addr=None, *, + family=0, proto=0, flags=0, + reuse_address=None, reuse_port=None, + allow_broadcast=None, sock=None): """A coroutine which creates a datagram endpoint. This method will try to establish the endpoint in the background. @@ -383,8 +511,8 @@ def create_datagram_endpoint(self, protocol_factory, protocol_factory must be a callable returning a protocol instance. - socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_DGRAM. + socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on + host (or family if specified), socket type SOCK_DGRAM. reuse_address tells the kernel to reuse a local socket in TIME_WAIT state, without waiting for its natural timeout to @@ -408,7 +536,7 @@ def create_datagram_endpoint(self, protocol_factory, # Pipes and subprocesses. - def connect_read_pipe(self, protocol_factory, pipe): + async def connect_read_pipe(self, protocol_factory, pipe): """Register read pipe in event loop. Set the pipe to non-blocking mode. protocol_factory should instantiate object with Protocol interface. @@ -418,10 +546,10 @@ def connect_read_pipe(self, protocol_factory, pipe): # The reason to accept file-like object instead of just file descriptor # is: we need to own pipe and close it at transport finishing # Can got complicated errors if pass f.fileno(), - # close fd in pipe transport then close f and vise versa. + # close fd in pipe transport then close f and vice versa. raise NotImplementedError - def connect_write_pipe(self, protocol_factory, pipe): + async def connect_write_pipe(self, protocol_factory, pipe): """Register write pipe in event loop. protocol_factory should instantiate object with BaseProtocol interface. @@ -431,17 +559,21 @@ def connect_write_pipe(self, protocol_factory, pipe): # The reason to accept file-like object instead of just file descriptor # is: we need to own pipe and close it at transport finishing # Can got complicated errors if pass f.fileno(), - # close fd in pipe transport then close f and vise versa. + # close fd in pipe transport then close f and vice versa. raise NotImplementedError - def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs): + async def subprocess_shell(self, protocol_factory, cmd, *, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs): raise NotImplementedError - def subprocess_exec(self, protocol_factory, *args, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs): + async def subprocess_exec(self, protocol_factory, *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs): raise NotImplementedError # Ready-based callback registration methods. @@ -463,16 +595,32 @@ def remove_writer(self, fd): # Completion based I/O methods returning Futures. - def sock_recv(self, sock, nbytes): + async def sock_recv(self, sock, nbytes): + raise NotImplementedError + + async def sock_recv_into(self, sock, buf): + raise NotImplementedError + + async def sock_recvfrom(self, sock, bufsize): + raise NotImplementedError + + async def sock_recvfrom_into(self, sock, buf, nbytes=0): raise NotImplementedError - def sock_sendall(self, sock, data): + async def sock_sendall(self, sock, data): raise NotImplementedError - def sock_connect(self, sock, address): + async def sock_sendto(self, sock, data, address): raise NotImplementedError - def sock_accept(self, sock): + async def sock_connect(self, sock, address): + raise NotImplementedError + + async def sock_accept(self, sock): + raise NotImplementedError + + async def sock_sendfile(self, sock, file, offset=0, count=None, + *, fallback=None): raise NotImplementedError # Signal handling. @@ -514,13 +662,13 @@ def set_debug(self, enabled): raise NotImplementedError -class AbstractEventLoopPolicy: +class _AbstractEventLoopPolicy: """Abstract policy for accessing the event loop.""" def get_event_loop(self): """Get the event loop for the current context. - Returns an event loop object implementing the BaseEventLoop interface, + Returns an event loop object implementing the AbstractEventLoop interface, or raises an exception in case no event loop has been set for the current context and the current policy does not specify to create one. @@ -537,18 +685,7 @@ def new_event_loop(self): the current context, set_event_loop must be called explicitly.""" raise NotImplementedError - # Child processes handling (Unix only). - - def get_child_watcher(self): - "Get the watcher for child processes." - raise NotImplementedError - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - raise NotImplementedError - - -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): +class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): """Default policy implementation for accessing the event loop. In this policy, each thread has its own event loop. However, we @@ -565,29 +702,25 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): class _Local(threading.local): _loop = None - _set_called = False def __init__(self): self._local = self._Local() def get_event_loop(self): - """Get the event loop. + """Get the event loop for the current context. - This may be None or an instance of EventLoop. + Returns an instance of EventLoop or raises an exception. """ - if (self._local._loop is None and - not self._local._set_called and - isinstance(threading.current_thread(), threading._MainThread)): - self.set_event_loop(self.new_event_loop()) if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) + return self._local._loop def set_event_loop(self, loop): """Set the event loop.""" - self._local._set_called = True - assert loop is None or isinstance(loop, AbstractEventLoop) + if loop is not None and not isinstance(loop, AbstractEventLoop): + raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") self._local._loop = loop def new_event_loop(self): @@ -611,7 +744,9 @@ def new_event_loop(self): # A TLS for the running event loop, used by _get_running_loop. class _RunningLoop(threading.local): - _loop = None + loop_pid = (None, None) + + _running_loop = _RunningLoop() @@ -633,7 +768,10 @@ def _get_running_loop(): This is a low-level function intended to be used by event loops. This function is thread-specific. """ - return _running_loop._loop + # NOTE: this function is implemented in C (see _asynciomodule.c) + running_loop, pid = _running_loop.loop_pid + if running_loop is not None and pid == os.getpid(): + return running_loop def _set_running_loop(loop): @@ -642,32 +780,43 @@ def _set_running_loop(loop): This is a low-level function intended to be used by event loops. This function is thread-specific. """ - _running_loop._loop = loop + # NOTE: this function is implemented in C (see _asynciomodule.c) + _running_loop.loop_pid = (loop, os.getpid()) def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import DefaultEventLoopPolicy - _event_loop_policy = DefaultEventLoopPolicy() + if sys.platform == 'win32': + from .windows_events import _DefaultEventLoopPolicy + else: + from .unix_events import _DefaultEventLoopPolicy + _event_loop_policy = _DefaultEventLoopPolicy() -def get_event_loop_policy(): +def _get_event_loop_policy(): """Get the current event loop policy.""" if _event_loop_policy is None: _init_event_loop_policy() return _event_loop_policy +def get_event_loop_policy(): + warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16)) + return _get_event_loop_policy() -def set_event_loop_policy(policy): +def _set_event_loop_policy(policy): """Set the current event loop policy. If policy is None, the default policy is restored.""" global _event_loop_policy - assert policy is None or isinstance(policy, AbstractEventLoopPolicy) + if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): + raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy +def set_event_loop_policy(policy): + warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16)) + _set_event_loop_policy(policy) def get_event_loop(): """Return an asyncio event loop. @@ -678,28 +827,52 @@ def get_event_loop(): If there is no running event loop set, the function will return the result of `get_event_loop_policy().get_event_loop()` call. """ + # NOTE: this function is implemented in C (see _asynciomodule.c) current_loop = _get_running_loop() if current_loop is not None: return current_loop - return get_event_loop_policy().get_event_loop() + return _get_event_loop_policy().get_event_loop() def set_event_loop(loop): """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - get_event_loop_policy().set_event_loop(loop) + _get_event_loop_policy().set_event_loop(loop) def new_event_loop(): """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return get_event_loop_policy().new_event_loop() - - -def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() - - -def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) + return _get_event_loop_policy().new_event_loop() + + +# Alias pure-Python implementations for testing purposes. +_py__get_running_loop = _get_running_loop +_py__set_running_loop = _set_running_loop +_py_get_running_loop = get_running_loop +_py_get_event_loop = get_event_loop + + +try: + # get_event_loop() is one of the most frequently called + # functions in asyncio. Pure Python implementation is + # about 4 times slower than C-accelerated. + from _asyncio import (_get_running_loop, _set_running_loop, + get_running_loop, get_event_loop) +except ImportError: + pass +else: + # Alias C implementations for testing purposes. + _c__get_running_loop = _get_running_loop + _c__set_running_loop = _set_running_loop + _c_get_running_loop = get_running_loop + _c_get_event_loop = get_event_loop + + +if hasattr(os, 'fork'): + def on_fork(): + # Reset the loop and wakeupfd in the forked child process. + if _event_loop_policy is not None: + _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() + _set_running_loop(None) + signal.set_wakeup_fd(-1) + + os.register_at_fork(after_in_child=on_fork) diff --git a/Lib/asyncio/exceptions.py b/Lib/asyncio/exceptions.py new file mode 100644 index 00000000000..5ece595aad6 --- /dev/null +++ b/Lib/asyncio/exceptions.py @@ -0,0 +1,62 @@ +"""asyncio exceptions.""" + + +__all__ = ('BrokenBarrierError', + 'CancelledError', 'InvalidStateError', 'TimeoutError', + 'IncompleteReadError', 'LimitOverrunError', + 'SendfileNotAvailableError') + + +class CancelledError(BaseException): + """The Future or Task was cancelled.""" + + +TimeoutError = TimeoutError # make local alias for the standard exception + + +class InvalidStateError(Exception): + """The operation is not allowed in this state.""" + + +class SendfileNotAvailableError(RuntimeError): + """Sendfile syscall is not available. + + Raised if OS does not support sendfile syscall for given socket or + file type. + """ + + +class IncompleteReadError(EOFError): + """ + Incomplete read error. Attributes: + + - partial: read bytes string before the end of stream was reached + - expected: total number of expected bytes (or None if unknown) + """ + def __init__(self, partial, expected): + r_expected = 'undefined' if expected is None else repr(expected) + super().__init__(f'{len(partial)} bytes read on a total of ' + f'{r_expected} expected bytes') + self.partial = partial + self.expected = expected + + def __reduce__(self): + return type(self), (self.partial, self.expected) + + +class LimitOverrunError(Exception): + """Reached the buffer limit while looking for a separator. + + Attributes: + - consumed: total number of to be consumed bytes. + """ + def __init__(self, message, consumed): + super().__init__(message) + self.consumed = consumed + + def __reduce__(self): + return type(self), (self.args[0], self.consumed) + + +class BrokenBarrierError(RuntimeError): + """Barrier is broken by barrier.abort() call.""" diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py new file mode 100644 index 00000000000..93737b7708a --- /dev/null +++ b/Lib/asyncio/format_helpers.py @@ -0,0 +1,84 @@ +import functools +import inspect +import reprlib +import sys +import traceback + +from . import constants + + +def _get_function_source(func): + func = inspect.unwrap(func) + if inspect.isfunction(func): + code = func.__code__ + return (code.co_filename, code.co_firstlineno) + if isinstance(func, functools.partial): + return _get_function_source(func.func) + if isinstance(func, functools.partialmethod): + return _get_function_source(func.func) + return None + + +def _format_callback_source(func, args, *, debug=False): + func_repr = _format_callback(func, args, None, debug=debug) + source = _get_function_source(func) + if source: + func_repr += f' at {source[0]}:{source[1]}' + return func_repr + + +def _format_args_and_kwargs(args, kwargs, *, debug=False): + """Format function arguments and keyword arguments. + + Special case for a single parameter: ('hello',) is formatted as ('hello'). + + Note that this function only returns argument details when + debug=True is specified, as arguments may contain sensitive + information. + """ + if not debug: + return '()' + + # use reprlib to limit the length of the output + items = [] + if args: + items.extend(reprlib.repr(arg) for arg in args) + if kwargs: + items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items()) + return '({})'.format(', '.join(items)) + + +def _format_callback(func, args, kwargs, *, debug=False, suffix=''): + if isinstance(func, functools.partial): + suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix + return _format_callback(func.func, func.args, func.keywords, + debug=debug, suffix=suffix) + + if hasattr(func, '__qualname__') and func.__qualname__: + func_repr = func.__qualname__ + elif hasattr(func, '__name__') and func.__name__: + func_repr = func.__name__ + else: + func_repr = repr(func) + + func_repr += _format_args_and_kwargs(args, kwargs, debug=debug) + if suffix: + func_repr += suffix + return func_repr + + +def extract_stack(f=None, limit=None): + """Replacement for traceback.extract_stack() that only does the + necessary work for asyncio debug mode. + """ + if f is None: + f = sys._getframe().f_back + if limit is None: + # Limit the amount of work to a reasonable amount, as extract_stack() + # can be called for each coroutine and future in debug mode. + limit = constants.DEBUG_STACK_DEPTH + stack = traceback.StackSummary.extract(traceback.walk_stack(f), + limit=limit, + lookup_lines=False) + stack.reverse() + return stack diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 82c03330ada..d1df6707302 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -1,21 +1,22 @@ """A Future class similar to the one in PEP 3148.""" -__all__ = ['CancelledError', 'TimeoutError', 'InvalidStateError', - 'Future', 'wrap_future', 'isfuture'] +__all__ = ( + 'Future', 'wrap_future', 'isfuture', + 'future_add_to_awaited_by', 'future_discard_from_awaited_by', +) import concurrent.futures +import contextvars import logging import sys -import traceback +from types import GenericAlias from . import base_futures -from . import compat from . import events +from . import exceptions +from . import format_helpers -CancelledError = base_futures.CancelledError -InvalidStateError = base_futures.InvalidStateError -TimeoutError = base_futures.TimeoutError isfuture = base_futures.isfuture @@ -27,101 +28,22 @@ STACK_DEBUG = logging.DEBUG - 1 # heavy-duty debugging -class _TracebackLogger: - """Helper to log a traceback upon destruction if not cleared. - - This solves a nasty problem with Futures and Tasks that have an - exception set: if nobody asks for the exception, the exception is - never logged. This violates the Zen of Python: 'Errors should - never pass silently. Unless explicitly silenced.' - - However, we don't want to log the exception as soon as - set_exception() is called: if the calling code is written - properly, it will get the exception and handle it properly. But - we *do* want to log it if result() or exception() was never called - -- otherwise developers waste a lot of time wondering why their - buggy code fails silently. - - An earlier attempt added a __del__() method to the Future class - itself, but this backfired because the presence of __del__() - prevents garbage collection from breaking cycles. A way out of - this catch-22 is to avoid having a __del__() method on the Future - class itself, but instead to have a reference to a helper object - with a __del__() method that logs the traceback, where we ensure - that the helper object doesn't participate in cycles, and only the - Future has a reference to it. - - The helper object is added when set_exception() is called. When - the Future is collected, and the helper is present, the helper - object is also collected, and its __del__() method will log the - traceback. When the Future's result() or exception() method is - called (and a helper object is present), it removes the helper - object, after calling its clear() method to prevent it from - logging. - - One downside is that we do a fair amount of work to extract the - traceback from the exception, even when it is never logged. It - would seem cheaper to just store the exception object, but that - references the traceback, which references stack frames, which may - reference the Future, which references the _TracebackLogger, and - then the _TracebackLogger would be included in a cycle, which is - what we're trying to avoid! As an optimization, we don't - immediately format the exception; we only do the work when - activate() is called, which call is delayed until after all the - Future's callbacks have run. Since usually a Future has at least - one callback (typically set by 'yield from') and usually that - callback extracts the callback, thereby removing the need to - format the exception. - - PS. I don't claim credit for this solution. I first heard of it - in a discussion about closing files when they are collected. - """ - - __slots__ = ('loop', 'source_traceback', 'exc', 'tb') - - def __init__(self, future, exc): - self.loop = future._loop - self.source_traceback = future._source_traceback - self.exc = exc - self.tb = None - - def activate(self): - exc = self.exc - if exc is not None: - self.exc = None - self.tb = traceback.format_exception(exc.__class__, exc, - exc.__traceback__) - - def clear(self): - self.exc = None - self.tb = None - - def __del__(self): - if self.tb: - msg = 'Future/Task exception was never retrieved\n' - if self.source_traceback: - src = ''.join(traceback.format_list(self.source_traceback)) - msg += 'Future/Task created at (most recent call last):\n' - msg += '%s\n' % src.rstrip() - msg += ''.join(self.tb).rstrip() - self.loop.call_exception_handler({'message': msg}) - - class Future: """This class is *almost* compatible with concurrent.futures.Future. Differences: + - This class is not thread-safe. + - result() and exception() do not take a timeout argument and raise an exception when the future isn't done yet. - Callbacks registered with add_done_callback() are always called - via the event loop's call_soon_threadsafe(). + via the event loop's call_soon(). - This class is not compatible with the wait() and as_completed() methods in the concurrent.futures package. - (In Python 3.4 or later we may be able to unify the implementations.) """ # Class variables serving as defaults for instance variables. @@ -130,19 +52,25 @@ class Future: _exception = None _loop = None _source_traceback = None + _cancel_message = None + # A saved CancelledError for later chaining as an exception context. + _cancelled_exc = None # This field is used for a dual purpose: # - Its presence is a marker to declare that a class implements # the Future protocol (i.e. is intended to be duck-type compatible). # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. - # - It is set by __iter__() below so that Task._step() can tell - # the difference between `yield from Future()` (correct) vs. + # - It is set by __iter__() below so that Task.__step() can tell + # the difference between + # `await Future()` or `yield from Future()` (correct) vs. # `yield Future()` (incorrect). _asyncio_future_blocking = False - _log_traceback = False # Used for Python 3.4 and later - _tb_logger = None # Used for Python 3.3 only + # Used by the capture_call_stack() API. + __asyncio_awaited_by = None + + __log_traceback = False def __init__(self, *, loop=None): """Initialize the future. @@ -157,50 +85,86 @@ def __init__(self, *, loop=None): self._loop = loop self._callbacks = [] if self._loop.get_debug(): - self._source_traceback = traceback.extract_stack(sys._getframe(1)) - - _repr_info = base_futures._future_repr_info + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, ' '.join(self._repr_info())) - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if not self._log_traceback: - # set_exception() was not called, or result() or exception() - # has consumed the exception - return - exc = self._exception - context = { - 'message': ('%s exception was never retrieved' - % self.__class__.__name__), - 'exception': exc, - 'future': self, - } - if self._source_traceback: - context['source_traceback'] = self._source_traceback - self._loop.call_exception_handler(context) - - def __class_getitem__(cls, type): - return cls - - def cancel(self): + return base_futures._future_repr(self) + + def __del__(self): + if not self.__log_traceback: + # set_exception() was not called, or result() or exception() + # has consumed the exception + return + exc = self._exception + context = { + 'message': + f'{self.__class__.__name__} exception was never retrieved', + 'exception': exc, + 'future': self, + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + + __class_getitem__ = classmethod(GenericAlias) + + @property + def _log_traceback(self): + return self.__log_traceback + + @_log_traceback.setter + def _log_traceback(self, val): + if val: + raise ValueError('_log_traceback can only be set to False') + self.__log_traceback = False + + @property + def _asyncio_awaited_by(self): + if self.__asyncio_awaited_by is None: + return None + return frozenset(self.__asyncio_awaited_by) + + def get_loop(self): + """Return the event loop the Future is bound to.""" + loop = self._loop + if loop is None: + raise RuntimeError("Future object is not initialized.") + return loop + + def _make_cancelled_error(self): + """Create the CancelledError to raise if the Future is cancelled. + + This should only be called once when handling a cancellation since + it erases the saved context exception value. + """ + if self._cancelled_exc is not None: + exc = self._cancelled_exc + self._cancelled_exc = None + return exc + + if self._cancel_message is None: + exc = exceptions.CancelledError() + else: + exc = exceptions.CancelledError(self._cancel_message) + return exc + + def cancel(self, msg=None): """Cancel the future and schedule callbacks. If the future is already done or cancelled, return False. Otherwise, change the future's state to cancelled, schedule the callbacks and return True. """ + self.__log_traceback = False if self._state != _PENDING: return False self._state = _CANCELLED - self._schedule_callbacks() + self._cancel_message = msg + self.__schedule_callbacks() return True - def _schedule_callbacks(self): + def __schedule_callbacks(self): """Internal: Ask the event loop to call all callbacks. The callbacks are scheduled to be called as soon as possible. Also @@ -211,8 +175,8 @@ def _schedule_callbacks(self): return self._callbacks[:] = [] - for callback in callbacks: - self._loop.call_soon(callback, self) + for callback, ctx in callbacks: + self._loop.call_soon(callback, self, context=ctx) def cancelled(self): """Return True if the future was cancelled.""" @@ -236,15 +200,12 @@ def result(self): the future is done and has an exception set, this exception is raised. """ if self._state == _CANCELLED: - raise CancelledError + raise self._make_cancelled_error() if self._state != _FINISHED: - raise InvalidStateError('Result is not ready.') - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None + raise exceptions.InvalidStateError('Result is not ready.') + self.__log_traceback = False if self._exception is not None: - raise self._exception + raise self._exception.with_traceback(self._exception_tb) return self._result def exception(self): @@ -256,16 +217,13 @@ def exception(self): InvalidStateError. """ if self._state == _CANCELLED: - raise CancelledError + raise self._make_cancelled_error() if self._state != _FINISHED: - raise InvalidStateError('Exception is not set.') - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None + raise exceptions.InvalidStateError('Exception is not set.') + self.__log_traceback = False return self._exception - def add_done_callback(self, fn): + def add_done_callback(self, fn, *, context=None): """Add a callback to be run when the future becomes done. The callback is called with a single argument - the future object. If @@ -273,9 +231,11 @@ def add_done_callback(self, fn): scheduled with call_soon. """ if self._state != _PENDING: - self._loop.call_soon(fn, self) + self._loop.call_soon(fn, self, context=context) else: - self._callbacks.append(fn) + if context is None: + context = contextvars.copy_context() + self._callbacks.append((fn, context)) # New method not in PEP 3148. @@ -284,7 +244,9 @@ def remove_done_callback(self, fn): Returns the number of callbacks removed. """ - filtered_callbacks = [f for f in self._callbacks if f != fn] + filtered_callbacks = [(f, ctx) + for (f, ctx) in self._callbacks + if f != fn] removed_count = len(self._callbacks) - len(filtered_callbacks) if removed_count: self._callbacks[:] = filtered_callbacks @@ -299,10 +261,10 @@ def set_result(self, result): InvalidStateError. """ if self._state != _PENDING: - raise InvalidStateError('{}: {!r}'.format(self._state, self)) + raise exceptions.InvalidStateError(f'{self._state}: {self!r}') self._result = result self._state = _FINISHED - self._schedule_callbacks() + self.__schedule_callbacks() def set_exception(self, exception): """Mark the future done and set an exception. @@ -311,38 +273,49 @@ def set_exception(self, exception): InvalidStateError. """ if self._state != _PENDING: - raise InvalidStateError('{}: {!r}'.format(self._state, self)) + raise exceptions.InvalidStateError(f'{self._state}: {self!r}') if isinstance(exception, type): exception = exception() - if type(exception) is StopIteration: - raise TypeError("StopIteration interacts badly with generators " - "and cannot be raised into a Future") + if isinstance(exception, StopIteration): + new_exc = RuntimeError("StopIteration interacts badly with " + "generators and cannot be raised into a " + "Future") + new_exc.__cause__ = exception + new_exc.__context__ = exception + exception = new_exc self._exception = exception + self._exception_tb = exception.__traceback__ self._state = _FINISHED - self._schedule_callbacks() - if compat.PY34: - self._log_traceback = True - else: - self._tb_logger = _TracebackLogger(self, exception) - # Arrange for the logger to be activated after all callbacks - # have had a chance to call result() or exception(). - self._loop.call_soon(self._tb_logger.activate) + self.__schedule_callbacks() + self.__log_traceback = True - def __iter__(self): + def __await__(self): if not self.done(): self._asyncio_future_blocking = True yield self # This tells Task to wait for completion. - assert self.done(), "yield from wasn't used with future" + if not self.done(): + raise RuntimeError("await wasn't used with future") return self.result() # May raise too. - if compat.PY35: - __await__ = __iter__ # make compatible with 'await' expression + __iter__ = __await__ # make compatible with 'yield from'. # Needed for testing purposes. _PyFuture = Future +def _get_loop(fut): + # Tries to call Future.get_loop() if it's available. + # Otherwise fallbacks to using the old '_loop' property. + try: + get_loop = fut.get_loop + except AttributeError: + pass + else: + return get_loop() + return fut._loop + + def _set_result_unless_cancelled(fut, result): """Helper setting the result only if the future was not cancelled.""" if fut.cancelled(): @@ -350,6 +323,16 @@ def _set_result_unless_cancelled(fut, result): fut.set_result(result) +def _convert_future_exc(exc): + exc_class = type(exc) + if exc_class is concurrent.futures.CancelledError: + return exceptions.CancelledError(*exc.args).with_traceback(exc.__traceback__) + elif exc_class is concurrent.futures.InvalidStateError: + return exceptions.InvalidStateError(*exc.args).with_traceback(exc.__traceback__) + else: + return exc + + def _set_concurrent_future_state(concurrent, source): """Copy state from a future to a concurrent.futures.Future.""" assert source.done() @@ -359,7 +342,7 @@ def _set_concurrent_future_state(concurrent, source): return exception = source.exception() if exception is not None: - concurrent.set_exception(exception) + concurrent.set_exception(_convert_future_exc(exception)) else: result = source.result() concurrent.set_result(result) @@ -379,7 +362,7 @@ def _copy_future_state(source, dest): else: exception = source.exception() if exception is not None: - dest.set_exception(exception) + dest.set_exception(_convert_future_exc(exception)) else: result = source.result() dest.set_result(result) @@ -398,8 +381,8 @@ def _chain_future(source, destination): if not isfuture(destination) and not isinstance(destination, concurrent.futures.Future): raise TypeError('A future is required for destination argument') - source_loop = source._loop if isfuture(source) else None - dest_loop = destination._loop if isfuture(destination) else None + source_loop = _get_loop(source) if isfuture(source) else None + dest_loop = _get_loop(destination) if isfuture(destination) else None def _set_state(future, other): if isfuture(future): @@ -415,9 +398,14 @@ def _call_check_cancel(destination): source_loop.call_soon_threadsafe(source.cancel) def _call_set_state(source): + if (destination.cancelled() and + dest_loop is not None and dest_loop.is_closed()): + return if dest_loop is None or dest_loop is source_loop: _set_state(destination, source) else: + if dest_loop.is_closed(): + return dest_loop.call_soon_threadsafe(_set_state, destination, source) destination.add_done_callback(_call_check_cancel) @@ -429,7 +417,7 @@ def wrap_future(future, *, loop=None): if isfuture(future): return future assert isinstance(future, concurrent.futures.Future), \ - 'concurrent.futures.Future is expected, got {!r}'.format(future) + f'concurrent.futures.Future is expected, got {future!r}' if loop is None: loop = events.get_event_loop() new_future = loop.create_future() @@ -437,6 +425,49 @@ def wrap_future(future, *, loop=None): return new_future +def future_add_to_awaited_by(fut, waiter, /): + """Record that `fut` is awaited on by `waiter`.""" + # For the sake of keeping the implementation minimal and assuming + # that most of asyncio users use the built-in Futures and Tasks + # (or their subclasses), we only support native Future objects + # and their subclasses. + # + # Longer version: tracking requires storing the caller-callee + # dependency somewhere. One obvious choice is to store that + # information right in the future itself in a dedicated attribute. + # This means that we'd have to require all duck-type compatible + # futures to implement a specific attribute used by asyncio for + # the book keeping. Another solution would be to store that in + # a global dictionary. The downside here is that that would create + # strong references and any scenario where the "add" call isn't + # followed by a "discard" call would lead to a memory leak. + # Using WeakDict would resolve that issue, but would complicate + # the C code (_asynciomodule.c). The bottom line here is that + # it's not clear that all this work would be worth the effort. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is None: + fut._Future__asyncio_awaited_by = set() + fut._Future__asyncio_awaited_by.add(waiter) + + +def future_discard_from_awaited_by(fut, waiter, /): + """Record that `fut` is no longer awaited on by `waiter`.""" + # See the comment in "future_add_to_awaited_by()" body for + # details on implementation. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._Future__asyncio_awaited_by is not None: + fut._Future__asyncio_awaited_by.discard(waiter) + + +_py_future_add_to_awaited_by = future_add_to_awaited_by +_py_future_discard_from_awaited_by = future_discard_from_awaited_by + try: import _asyncio except ImportError: @@ -444,3 +475,7 @@ def wrap_future(future, *, loop=None): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future + future_add_to_awaited_by = _asyncio.future_add_to_awaited_by + future_discard_from_awaited_by = _asyncio.future_discard_from_awaited_by + _c_future_add_to_awaited_by = future_add_to_awaited_by + _c_future_discard_from_awaited_by = future_discard_from_awaited_by diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py new file mode 100644 index 00000000000..b5bfeb1630a --- /dev/null +++ b/Lib/asyncio/graph.py @@ -0,0 +1,276 @@ +"""Introspection utils for tasks call graphs.""" + +import dataclasses +import io +import sys +import types + +from . import events +from . import futures +from . import tasks + +__all__ = ( + 'capture_call_graph', + 'format_call_graph', + 'print_call_graph', + 'FrameCallGraphEntry', + 'FutureCallGraph', +) + +# Sadly, we can't re-use the traceback module's datastructures as those +# are tailored for error reporting, whereas we need to represent an +# async call graph. +# +# Going with pretty verbose names as we'd like to export them to the +# top level asyncio namespace, and want to avoid future name clashes. + + +@dataclasses.dataclass(frozen=True, slots=True) +class FrameCallGraphEntry: + frame: types.FrameType + + +@dataclasses.dataclass(frozen=True, slots=True) +class FutureCallGraph: + future: futures.Future + call_stack: tuple["FrameCallGraphEntry", ...] + awaited_by: tuple["FutureCallGraph", ...] + + +def _build_graph_for_future( + future: futures.Future, + *, + limit: int | None = None, +) -> FutureCallGraph: + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + coro = None + if get_coro := getattr(future, 'get_coro', None): + coro = get_coro() if limit != 0 else None + + st: list[FrameCallGraphEntry] = [] + awaited_by: list[FutureCallGraph] = [] + + while coro is not None: + if hasattr(coro, 'cr_await'): + # A native coroutine or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.cr_await + elif hasattr(coro, 'ag_await'): + # A native async generator or duck-type compatible iterator + st.append(FrameCallGraphEntry(coro.cr_frame)) + coro = coro.ag_await + else: + break + + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + if limit > 0: + st = st[:limit] + elif limit < 0: + st = st[limit:] + st.reverse() + return FutureCallGraph(future, tuple(st), tuple(awaited_by)) + + +def capture_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> FutureCallGraph | None: + """Capture the async call graph for the current task or the provided Future. + + The graph is represented with three data structures: + + * FutureCallGraph(future, call_stack, awaited_by) + + Where 'future' is an instance of asyncio.Future or asyncio.Task. + + 'call_stack' is a tuple of FrameGraphEntry objects. + + 'awaited_by' is a tuple of FutureCallGraph objects. + + * FrameCallGraphEntry(frame) + + Where 'frame' is a frame object of a regular Python function + in the call stack. + + Receives an optional 'future' argument. If not passed, + the current task will be used. If there's no current task, the function + returns None. + + If "capture_call_graph()" is introspecting *the current task*, the + optional keyword-only 'depth' argument can be used to skip the specified + number of frames from top of the stack. + + If the optional keyword-only 'limit' argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If 'limit' is positive, the entries left are the closest to + the invocation point. If 'limit' is negative, the topmost entries are + left. If 'limit' is omitted or None, all entries are present. + If 'limit' is 0, the call stack is not captured at all, only + "awaited by" information is present. + """ + + loop = events._get_running_loop() + + if future is not None: + # Check if we're in a context of a running event loop; + # if yes - check if the passed future is the currently + # running task or not. + if loop is None or future is not tasks.current_task(loop=loop): + return _build_graph_for_future(future, limit=limit) + # else: future is the current task, move on. + else: + if loop is None: + raise RuntimeError( + 'capture_call_graph() is called outside of a running ' + 'event loop and no *future* to introspect was provided') + future = tasks.current_task(loop=loop) + + if future is None: + # This isn't a generic call stack introspection utility. If we + # can't determine the current task and none was provided, we + # just return. + return None + + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + call_stack: list[FrameCallGraphEntry] = [] + + f = sys._getframe(depth) if limit != 0 else None + try: + while f is not None: + is_async = f.f_generator is not None + call_stack.append(FrameCallGraphEntry(f)) + + if is_async: + if f.f_back is not None and f.f_back.f_generator is None: + # We've reached the bottom of the coroutine stack, which + # must be the Task that runs it. + break + + f = f.f_back + finally: + del f + + awaited_by = [] + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + limit *= -1 + if limit > 0: + call_stack = call_stack[:limit] + elif limit < 0: + call_stack = call_stack[limit:] + + return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) + + +def format_call_graph( + future: futures.Future | None = None, + /, + *, + depth: int = 1, + limit: int | None = None, +) -> str: + """Return the async call graph as a string for `future`. + + If `future` is not provided, format the call graph for the current task. + """ + + def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: + def add_line(line: str) -> None: + buf.append(level * ' ' + line) + + if isinstance(st.future, tasks.Task): + add_line( + f'* Task(name={st.future.get_name()!r}, id={id(st.future):#x})' + ) + else: + add_line( + f'* Future(id={id(st.future):#x})' + ) + + if st.call_stack: + add_line( + f' + Call stack:' + ) + for ste in st.call_stack: + f = ste.frame + + if f.f_generator is None: + f = ste.frame + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {f.f_code.co_qualname}()' + ) + else: + c = f.f_generator + + try: + f = c.cr_frame + code = c.cr_code + tag = 'async' + except AttributeError: + try: + f = c.ag_frame + code = c.ag_code + tag = 'async generator' + except AttributeError: + f = c.gi_frame + code = c.gi_code + tag = 'generator' + + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag} {code.co_qualname}()' + ) + + if st.awaited_by: + add_line( + f' + Awaited by:' + ) + for fut in st.awaited_by: + render_level(fut, buf, level + 1) + + graph = capture_call_graph(future, depth=depth + 1, limit=limit) + if graph is None: + return "" + + buf: list[str] = [] + try: + render_level(graph, buf, 0) + finally: + # 'graph' has references to frames so we should + # make sure it's GC'ed as soon as we don't need it. + del graph + return '\n'.join(buf) + +def print_call_graph( + future: futures.Future | None = None, + /, + *, + file: io.Writer[str] | None = None, + depth: int = 1, + limit: int | None = None, +) -> None: + """Print the async call graph for the current task or the provided Future.""" + print(format_call_graph(future, depth=depth, limit=limit), file=file) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index deefc938ecf..fa3a94764b5 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,123 +1,55 @@ """Synchronization primitives.""" -__all__ = ['Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore'] +__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', + 'BoundedSemaphore', 'Barrier') import collections +import enum -from . import compat -from . import events -from . import futures -from .coroutines import coroutine +from . import exceptions +from . import mixins - -class _ContextManager: - """Context manager. - - This enables the following idiom for acquiring and releasing a - lock around a block: - - with (yield from lock): - - - while failing loudly when accidentally using: - - with lock: - - """ - - def __init__(self, lock): - self._lock = lock - - def __enter__(self): +class _ContextManagerMixin: + async def __aenter__(self): + await self.acquire() # We have no use for the "as ..." clause in the with # statement for locks. return None - def __exit__(self, *args): - try: - self._lock.release() - finally: - self._lock = None # Crudely prevent reuse. - - -class _ContextManagerMixin: - def __enter__(self): - raise RuntimeError( - '"yield from" should be used as context manager expression') + async def __aexit__(self, exc_type, exc, tb): + self.release() - def __exit__(self, *args): - # This must exist because __enter__ exists, even though that - # always raises; that's how the with-statement works. - pass - @coroutine - def __iter__(self): - # This is not a coroutine. It is meant to enable the idiom: - # - # with (yield from lock): - # - # - # as an alternative to: - # - # yield from lock.acquire() - # try: - # - # finally: - # lock.release() - yield from self.acquire() - return _ContextManager(self) - - if compat.PY35: - - def __await__(self): - # To make "with await lock" work. - yield from self.acquire() - return _ContextManager(self) - - @coroutine - def __aenter__(self): - yield from self.acquire() - # We have no use for the "as ..." clause in the with - # statement for locks. - return None - - @coroutine - def __aexit__(self, exc_type, exc, tb): - self.release() - - -class Lock(_ContextManagerMixin): +class Lock(_ContextManagerMixin, mixins._LoopBoundMixin): """Primitive lock objects. A primitive lock is a synchronization primitive that is not owned - by a particular coroutine when locked. A primitive lock is in one + by a particular task when locked. A primitive lock is in one of two states, 'locked' or 'unlocked'. It is created in the unlocked state. It has two basic methods, acquire() and release(). When the state is unlocked, acquire() changes the state to locked and returns immediately. When the state is locked, acquire() blocks until a call to release() in - another coroutine changes it to unlocked, then the acquire() call + another task changes it to unlocked, then the acquire() call resets it to locked and returns. The release() method should only be called in the locked state; it changes the state to unlocked and returns immediately. If an attempt is made to release an unlocked lock, a RuntimeError will be raised. - When more than one coroutine is blocked in acquire() waiting for - the state to turn to unlocked, only one coroutine proceeds when a - release() call resets the state to unlocked; first coroutine which - is blocked in acquire() is being processed. - - acquire() is a coroutine and should be called with 'yield from'. + When more than one task is blocked in acquire() waiting for + the state to turn to unlocked, only one task proceeds when a + release() call resets the state to unlocked; successive release() + calls will unblock tasks in FIFO order. - Locks also support the context management protocol. '(yield from lock)' - should be used as the context manager expression. + Locks also support the asynchronous context management protocol. + 'async with lock' statement should be used. Usage: lock = Lock() ... - yield from lock + await lock.acquire() try: ... finally: @@ -127,63 +59,76 @@ class Lock(_ContextManagerMixin): lock = Lock() ... - with (yield from lock): + async with lock: ... Lock objects can be tested for locking state: if not lock.locked(): - yield from lock + await lock.acquire() else: # lock is acquired ... """ - def __init__(self, *, loop=None): - self._waiters = collections.deque() + def __init__(self): + self._waiters = None self._locked = False - if loop is not None: - self._loop = loop - else: - self._loop = events.get_event_loop() def __repr__(self): res = super().__repr__() extra = 'locked' if self._locked else 'unlocked' if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) + extra = f'{extra}, waiters:{len(self._waiters)}' + return f'<{res[1:-1]} [{extra}]>' def locked(self): """Return True if lock is acquired.""" return self._locked - @coroutine - def acquire(self): + async def acquire(self): """Acquire a lock. This method blocks until the lock is unlocked, then sets it to locked and returns True. """ - if not self._locked and all(w.cancelled() for w in self._waiters): + # Implement fair scheduling, where thread always waits + # its turn. Jumping the queue if all are cancelled is an optimization. + if (not self._locked and (self._waiters is None or + all(w.cancelled() for w in self._waiters))): self._locked = True return True - fut = self._loop.create_future() + if self._waiters is None: + self._waiters = collections.deque() + fut = self._get_loop().create_future() self._waiters.append(fut) + try: - yield from fut - self._locked = True - return True - finally: - self._waiters.remove(fut) + try: + await fut + finally: + self._waiters.remove(fut) + except exceptions.CancelledError: + # Currently the only exception designed be able to occur here. + + # Ensure the lock invariant: If lock is not claimed (or about + # to be claimed by us) and there is a Task in waiters, + # ensure that the Task at the head will run. + if not self._locked: + self._wake_up_first() + raise + + # assert self._locked is False + self._locked = True + return True def release(self): """Release a lock. When the lock is locked, reset it to unlocked, and return. - If any other coroutines are blocked waiting for the lock to become + If any other tasks are blocked waiting for the lock to become unlocked, allow exactly one of them to proceed. When invoked on an unlocked lock, a RuntimeError is raised. @@ -192,16 +137,25 @@ def release(self): """ if self._locked: self._locked = False - # Wake up the first waiter who isn't cancelled. - for fut in self._waiters: - if not fut.done(): - fut.set_result(True) - break + self._wake_up_first() else: raise RuntimeError('Lock is not acquired.') + def _wake_up_first(self): + """Ensure that the first waiter will wake up.""" + if not self._waiters: + return + try: + fut = next(iter(self._waiters)) + except StopIteration: + return -class Event: + # .done() means that the waiter is already set to wake up. + if not fut.done(): + fut.set_result(True) + + +class Event(mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Event. Class implementing event objects. An event manages a flag that can be set @@ -210,28 +164,24 @@ class Event: false. """ - def __init__(self, *, loop=None): + def __init__(self): self._waiters = collections.deque() self._value = False - if loop is not None: - self._loop = loop - else: - self._loop = events.get_event_loop() def __repr__(self): res = super().__repr__() extra = 'set' if self._value else 'unset' if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) + extra = f'{extra}, waiters:{len(self._waiters)}' + return f'<{res[1:-1]} [{extra}]>' def is_set(self): """Return True if and only if the internal flag is true.""" return self._value def set(self): - """Set the internal flag to true. All coroutines waiting for it to - become true are awakened. Coroutine that call wait() once the flag is + """Set the internal flag to true. All tasks waiting for it to + become true are awakened. Tasks that call wait() once the flag is true will not block at all. """ if not self._value: @@ -242,51 +192,43 @@ def set(self): fut.set_result(True) def clear(self): - """Reset the internal flag to false. Subsequently, coroutines calling + """Reset the internal flag to false. Subsequently, tasks calling wait() will block until set() is called to set the internal flag to true again.""" self._value = False - @coroutine - def wait(self): + async def wait(self): """Block until the internal flag is true. If the internal flag is true on entry, return True - immediately. Otherwise, block until another coroutine calls + immediately. Otherwise, block until another task calls set() to set the flag to true, then return True. """ if self._value: return True - fut = self._loop.create_future() + fut = self._get_loop().create_future() self._waiters.append(fut) try: - yield from fut + await fut return True finally: self._waiters.remove(fut) -class Condition(_ContextManagerMixin): +class Condition(_ContextManagerMixin, mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Condition. This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. + allows one or more tasks to wait until they are notified by another + task. A new Lock object is created and used as the underlying lock. """ - def __init__(self, lock=None, *, loop=None): - if loop is not None: - self._loop = loop - else: - self._loop = events.get_event_loop() - + def __init__(self, lock=None): if lock is None: - lock = Lock(loop=self._loop) - elif lock._loop is not self._loop: - raise ValueError("loop argument must agree with lock") + lock = Lock() self._lock = lock # Export the lock's locked(), acquire() and release() methods. @@ -300,72 +242,95 @@ def __repr__(self): res = super().__repr__() extra = 'locked' if self.locked() else 'unlocked' if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) + extra = f'{extra}, waiters:{len(self._waiters)}' + return f'<{res[1:-1]} [{extra}]>' - @coroutine - def wait(self): + async def wait(self): """Wait until notified. - If the calling coroutine has not acquired the lock when this + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. This method releases the underlying lock, and then blocks until it is awakened by a notify() or notify_all() call for - the same condition variable in another coroutine. Once + the same condition variable in another task. Once awakened, it re-acquires the lock and returns True. + + This method may return spuriously, + which is why the caller should always + re-check the state and be prepared to wait() again. """ if not self.locked(): raise RuntimeError('cannot wait on un-acquired lock') + fut = self._get_loop().create_future() self.release() try: - fut = self._loop.create_future() - self._waiters.append(fut) try: - yield from fut - return True - finally: - self._waiters.remove(fut) - - finally: - # Must reacquire lock even if wait is cancelled - while True: + self._waiters.append(fut) try: - yield from self.acquire() - break - except futures.CancelledError: - pass + await fut + return True + finally: + self._waiters.remove(fut) - @coroutine - def wait_for(self, predicate): + finally: + # Must re-acquire lock even if wait is cancelled. + # We only catch CancelledError here, since we don't want any + # other (fatal) errors with the future to cause us to spin. + err = None + while True: + try: + await self.acquire() + break + except exceptions.CancelledError as e: + err = e + + if err is not None: + try: + raise err # Re-raise most recent exception instance. + finally: + err = None # Break reference cycles. + except BaseException: + # Any error raised out of here _may_ have occurred after this Task + # believed to have been successfully notified. + # Make sure to notify another Task instead. This may result + # in a "spurious wakeup", which is allowed as part of the + # Condition Variable protocol. + self._notify(1) + raise + + async def wait_for(self, predicate): """Wait until a predicate becomes true. - The predicate should be a callable which result will be - interpreted as a boolean value. The final predicate value is + The predicate should be a callable whose result will be + interpreted as a boolean value. The method will repeatedly + wait() until it evaluates to true. The final predicate value is the return value. """ result = predicate() while not result: - yield from self.wait() + await self.wait() result = predicate() return result def notify(self, n=1): - """By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method + """By default, wake up one task waiting on this condition, if any. + If the calling task has not acquired the lock when this method is called, a RuntimeError is raised. - This method wakes up at most n of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. + This method wakes up n of the tasks waiting for the condition + variable; if fewer than n are waiting, they are all awoken. - Note: an awakened coroutine does not actually return from its + Note: an awakened task does not actually return from its wait() call until it can reacquire the lock. Since notify() does not release the lock, its caller should. """ if not self.locked(): raise RuntimeError('cannot notify on un-acquired lock') + self._notify(n) + def _notify(self, n): idx = 0 for fut in self._waiters: if idx >= n: @@ -376,15 +341,15 @@ def notify(self, n=1): fut.set_result(False) def notify_all(self): - """Wake up all threads waiting on this condition. This method acts - like notify(), but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, + """Wake up all tasks waiting on this condition. This method acts + like notify(), but wakes up all waiting tasks instead of one. If the + calling task has not acquired the lock when this method is called, a RuntimeError is raised. """ self.notify(len(self._waiters)) -class Semaphore(_ContextManagerMixin): +class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): """A Semaphore implementation. A semaphore manages an internal counter which is decremented by each @@ -399,67 +364,89 @@ class Semaphore(_ContextManagerMixin): ValueError is raised. """ - def __init__(self, value=1, *, loop=None): + def __init__(self, value=1): if value < 0: raise ValueError("Semaphore initial value must be >= 0") + self._waiters = None self._value = value - self._waiters = collections.deque() - if loop is not None: - self._loop = loop - else: - self._loop = events.get_event_loop() def __repr__(self): res = super().__repr__() - extra = 'locked' if self.locked() else 'unlocked,value:{}'.format( - self._value) + extra = 'locked' if self.locked() else f'unlocked, value:{self._value}' if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) - - def _wake_up_next(self): - while self._waiters: - waiter = self._waiters.popleft() - if not waiter.done(): - waiter.set_result(None) - return + extra = f'{extra}, waiters:{len(self._waiters)}' + return f'<{res[1:-1]} [{extra}]>' def locked(self): - """Returns True if semaphore can not be acquired immediately.""" - return self._value == 0 + """Returns True if semaphore cannot be acquired immediately.""" + # Due to state, or FIFO rules (must allow others to run first). + return self._value == 0 or ( + any(not w.cancelled() for w in (self._waiters or ()))) - @coroutine - def acquire(self): + async def acquire(self): """Acquire a semaphore. If the internal counter is larger than zero on entry, decrement it by one and return True immediately. If it is - zero on entry, block, waiting until some other coroutine has + zero on entry, block, waiting until some other task has called release() to make it larger than 0, and then return True. """ - while self._value <= 0: - fut = self._loop.create_future() - self._waiters.append(fut) + if not self.locked(): + # Maintain FIFO, wait for others to start even if _value > 0. + self._value -= 1 + return True + + if self._waiters is None: + self._waiters = collections.deque() + fut = self._get_loop().create_future() + self._waiters.append(fut) + + try: try: - yield from fut - except: - # See the similar code in Queue.get. - fut.cancel() - if self._value > 0 and not fut.cancelled(): - self._wake_up_next() - raise - self._value -= 1 + await fut + finally: + self._waiters.remove(fut) + except exceptions.CancelledError: + # Currently the only exception designed be able to occur here. + if fut.done() and not fut.cancelled(): + # Our Future was successfully set to True via _wake_up_next(), + # but we are not about to successfully acquire(). Therefore we + # must undo the bookkeeping already done and attempt to wake + # up someone else. + self._value += 1 + raise + + finally: + # New waiters may have arrived but had to wait due to FIFO. + # Wake up as many as are allowed. + while self._value > 0: + if not self._wake_up_next(): + break # There was no-one to wake up. return True def release(self): """Release a semaphore, incrementing the internal counter by one. - When it was zero on entry and another coroutine is waiting for it to - become larger than zero again, wake up that coroutine. + + When it was zero on entry and another task is waiting for it to + become larger than zero again, wake up that task. """ self._value += 1 self._wake_up_next() + def _wake_up_next(self): + """Wake up the first waiter that isn't done.""" + if not self._waiters: + return False + + for fut in self._waiters: + if not fut.done(): + self._value -= 1 + fut.set_result(True) + # `fut` is now `done()` and not `cancelled()`. + return True + return False + class BoundedSemaphore(Semaphore): """A bounded semaphore implementation. @@ -468,11 +455,163 @@ class BoundedSemaphore(Semaphore): above the initial value. """ - def __init__(self, value=1, *, loop=None): + def __init__(self, value=1): self._bound_value = value - super().__init__(value, loop=loop) + super().__init__(value) def release(self): if self._value >= self._bound_value: raise ValueError('BoundedSemaphore released too many times') super().release() + + + +class _BarrierState(enum.Enum): + FILLING = 'filling' + DRAINING = 'draining' + RESETTING = 'resetting' + BROKEN = 'broken' + + +class Barrier(mixins._LoopBoundMixin): + """Asyncio equivalent to threading.Barrier + + Implements a Barrier primitive. + Useful for synchronizing a fixed number of tasks at known synchronization + points. Tasks block on 'wait()' and are simultaneously awoken once they + have all made their call. + """ + + def __init__(self, parties): + """Create a barrier, initialised to 'parties' tasks.""" + if parties < 1: + raise ValueError('parties must be >= 1') + + self._cond = Condition() # notify all tasks when state changes + + self._parties = parties + self._state = _BarrierState.FILLING + self._count = 0 # count tasks in Barrier + + def __repr__(self): + res = super().__repr__() + extra = f'{self._state.value}' + if not self.broken: + extra += f', waiters:{self.n_waiting}/{self.parties}' + return f'<{res[1:-1]} [{extra}]>' + + async def __aenter__(self): + # wait for the barrier reaches the parties number + # when start draining release and return index of waited task + return await self.wait() + + async def __aexit__(self, *args): + pass + + async def wait(self): + """Wait for the barrier. + + When the specified number of tasks have started waiting, they are all + simultaneously awoken. + Returns an unique and individual index number from 0 to 'parties-1'. + """ + async with self._cond: + await self._block() # Block while the barrier drains or resets. + try: + index = self._count + self._count += 1 + if index + 1 == self._parties: + # We release the barrier + await self._release() + else: + await self._wait() + return index + finally: + self._count -= 1 + # Wake up any tasks waiting for barrier to drain. + self._exit() + + async def _block(self): + # Block until the barrier is ready for us, + # or raise an exception if it is broken. + # + # It is draining or resetting, wait until done + # unless a CancelledError occurs + await self._cond.wait_for( + lambda: self._state not in ( + _BarrierState.DRAINING, _BarrierState.RESETTING + ) + ) + + # see if the barrier is in a broken state + if self._state is _BarrierState.BROKEN: + raise exceptions.BrokenBarrierError("Barrier aborted") + + async def _release(self): + # Release the tasks waiting in the barrier. + + # Enter draining state. + # Next waiting tasks will be blocked until the end of draining. + self._state = _BarrierState.DRAINING + self._cond.notify_all() + + async def _wait(self): + # Wait in the barrier until we are released. Raise an exception + # if the barrier is reset or broken. + + # wait for end of filling + # unless a CancelledError occurs + await self._cond.wait_for(lambda: self._state is not _BarrierState.FILLING) + + if self._state in (_BarrierState.BROKEN, _BarrierState.RESETTING): + raise exceptions.BrokenBarrierError("Abort or reset of barrier") + + def _exit(self): + # If we are the last tasks to exit the barrier, signal any tasks + # waiting for the barrier to drain. + if self._count == 0: + if self._state in (_BarrierState.RESETTING, _BarrierState.DRAINING): + self._state = _BarrierState.FILLING + self._cond.notify_all() + + async def reset(self): + """Reset the barrier to the initial state. + + Any tasks currently waiting will get the BrokenBarrier exception + raised. + """ + async with self._cond: + if self._count > 0: + if self._state is not _BarrierState.RESETTING: + #reset the barrier, waking up tasks + self._state = _BarrierState.RESETTING + else: + self._state = _BarrierState.FILLING + self._cond.notify_all() + + async def abort(self): + """Place the barrier into a 'broken' state. + + Useful in case of error. Any currently waiting tasks and tasks + attempting to 'wait()' will have BrokenBarrierError raised. + """ + async with self._cond: + self._state = _BarrierState.BROKEN + self._cond.notify_all() + + @property + def parties(self): + """Return the number of tasks required to trip the barrier.""" + return self._parties + + @property + def n_waiting(self): + """Return the number of tasks currently waiting at the barrier.""" + if self._state is _BarrierState.FILLING: + return self._count + return 0 + + @property + def broken(self): + """Return True if the barrier is in a broken state.""" + return self._state is _BarrierState.BROKEN diff --git a/Lib/asyncio/mixins.py b/Lib/asyncio/mixins.py new file mode 100644 index 00000000000..c6bf97329e9 --- /dev/null +++ b/Lib/asyncio/mixins.py @@ -0,0 +1,21 @@ +"""Event loop mixins.""" + +import threading +from . import events + +_global_lock = threading.Lock() + + +class _LoopBoundMixin: + _loop = None + + def _get_loop(self): + loop = events._get_running_loop() + + if self._loop is None: + with _global_lock: + if self._loop is None: + self._loop = loop + if loop is not self._loop: + raise RuntimeError(f'{self!r} is bound to a different event loop') + return loop diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index ff12877fae2..f404273c3ae 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -4,20 +4,45 @@ proactor is only implemented on Windows with IOCP. """ -__all__ = ['BaseProactorEventLoop'] +__all__ = 'BaseProactorEventLoop', +import io +import os import socket import warnings +import signal +import threading +import collections from . import base_events -from . import compat from . import constants from . import futures +from . import exceptions +from . import protocols from . import sslproto from . import transports +from . import trsock from .log import logger +def _set_socket_extra(transport, sock): + transport._extra['socket'] = trsock.TransportSocket(sock) + + try: + transport._extra['sockname'] = sock.getsockname() + except socket.error: + if transport._loop.get_debug(): + logger.warning( + "getsockname() failed on %r", sock, exc_info=True) + + if 'peername' not in transport._extra: + try: + transport._extra['peername'] = sock.getpeername() + except socket.error: + # UDP sockets may not have a peer name + transport._extra['peername'] = None + + class _ProactorBasePipeTransport(transports._FlowControlMixin, transports.BaseTransport): """Base class for pipe and socket transports.""" @@ -27,7 +52,7 @@ def __init__(self, loop, sock, protocol, waiter=None, super().__init__(extra, loop) self._set_extra(sock) self._sock = sock - self._protocol = protocol + self.set_protocol(protocol) self._server = server self._buffer = None # None or bytearray. self._read_fut = None @@ -35,9 +60,10 @@ def __init__(self, loop, sock, protocol, waiter=None, self._pending_write = 0 self._conn_lost = 0 self._closing = False # Set when close() called. + self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -51,17 +77,16 @@ def __repr__(self): elif self._closing: info.append('closing') if self._sock is not None: - info.append('fd=%s' % self._sock.fileno()) + info.append(f'fd={self._sock.fileno()}') if self._read_fut is not None: - info.append('read=%s' % self._read_fut) + info.append(f'read={self._read_fut!r}') if self._write_fut is not None: - info.append("write=%r" % self._write_fut) + info.append(f'write={self._write_fut!r}') if self._buffer: - bufsize = len(self._buffer) - info.append('write_bufsize=%s' % bufsize) + info.append(f'write_bufsize={len(self._buffer)}') if self._eof_written: info.append('EOF written') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def _set_extra(self, sock): self._extra['pipe'] = sock @@ -86,31 +111,33 @@ def close(self): self._read_fut.cancel() self._read_fut = None - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning, - source=self) - self.close() + def __del__(self, _warn=warnings.warn): + if self._sock is not None: + _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) + self._sock.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): - if self._loop.get_debug(): - logger.debug("%r: %s", self, message, exc_info=True) - else: - self._loop.call_exception_handler({ - 'message': message, - 'exception': exc, - 'transport': self, - 'protocol': self._protocol, - }) - self._force_close(exc) + try: + if isinstance(exc, OSError): + if self._loop.get_debug(): + logger.debug("%r: %s", self, message, exc_info=True) + else: + self._loop.call_exception_handler({ + 'message': message, + 'exception': exc, + 'transport': self, + 'protocol': self._protocol, + }) + finally: + self._force_close(exc) def _force_close(self, exc): - if self._closing: + if self._empty_waiter is not None and not self._empty_waiter.done(): + if exc is None: + self._empty_waiter.set_result(None) + else: + self._empty_waiter.set_exception(exc) + if self._closing and self._called_connection_lost: return self._closing = True self._conn_lost += 1 @@ -125,6 +152,8 @@ def _force_close(self, exc): self._loop.call_soon(self._call_connection_lost, exc) def _call_connection_lost(self, exc): + if self._called_connection_lost: + return try: self._protocol.connection_lost(exc) finally: @@ -132,14 +161,15 @@ def _call_connection_lost(self, exc): # end then it may fail with ERROR_NETNAME_DELETED if we # just close our end. First calling shutdown() seems to # cure it, but maybe using DisconnectEx() would be better. - if hasattr(self._sock, 'shutdown'): + if hasattr(self._sock, 'shutdown') and self._sock.fileno() != -1: self._sock.shutdown(socket.SHUT_RDWR) self._sock.close() self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None + self._called_connection_lost = True def get_write_buffer_size(self): size = self._pending_write @@ -153,53 +183,127 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport, """Transport for read pipes.""" def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): + extra=None, server=None, buffer_size=65536): + self._pending_data_length = -1 + self._paused = True super().__init__(loop, sock, protocol, waiter, extra, server) - self._paused = False + + self._data = bytearray(buffer_size) self._loop.call_soon(self._loop_reading) + self._paused = False + + def is_reading(self): + return not self._paused and not self._closing def pause_reading(self): - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') + if self._closing or self._paused: + return self._paused = True + + # bpo-33694: Don't cancel self._read_fut because cancelling an + # overlapped WSASend() loss silently data with the current proactor + # implementation. + # + # If CancelIoEx() fails with ERROR_NOT_FOUND, it means that WSASend() + # completed (even if HasOverlappedIoCompleted() returns 0), but + # Overlapped.cancel() currently silently ignores the ERROR_NOT_FOUND + # error. Once the overlapped is ignored, the IOCP loop will ignores the + # completion I/O event and so not read the result of the overlapped + # WSARecv(). + if self._loop.get_debug(): logger.debug("%r pauses reading", self) def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: + if self._closing or not self._paused: return - self._loop.call_soon(self._loop_reading, self._read_fut) + + self._paused = False + if self._read_fut is None: + self._loop.call_soon(self._loop_reading, None) + + length = self._pending_data_length + self._pending_data_length = -1 + if length > -1: + # Call the protocol method after calling _loop_reading(), + # since the protocol can decide to pause reading again. + self._loop.call_soon(self._data_received, self._data[:length], length) + if self._loop.get_debug(): logger.debug("%r resumes reading", self) - def _loop_reading(self, fut=None): + def _eof_received(self): + if self._loop.get_debug(): + logger.debug("%r received EOF", self) + + try: + keep_open = self._protocol.eof_received() + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error( + exc, 'Fatal error: protocol.eof_received() call failed.') + return + + if not keep_open: + self.close() + + def _data_received(self, data, length): if self._paused: + # Don't call any protocol method while reading is paused. + # The protocol will be called on resume_reading(). + assert self._pending_data_length == -1 + self._pending_data_length = length return - data = None + if length == 0: + self._eof_received() + return + + if isinstance(self._protocol, protocols.BufferedProtocol): + try: + protocols._feed_data_to_buffered_proto(self._protocol, data) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error(exc, + 'Fatal error: protocol.buffer_updated() ' + 'call failed.') + return + else: + self._protocol.data_received(data) + + def _loop_reading(self, fut=None): + length = -1 + data = None try: if fut is not None: assert self._read_fut is fut or (self._read_fut is None and self._closing) self._read_fut = None - data = fut.result() # deliver data later in "finally" clause + if fut.done(): + # deliver data later in "finally" clause + length = fut.result() + if length == 0: + # we got end-of-file so no need to reschedule a new read + return + + # It's a new slice so make it immutable so protocols upstream don't have problems + data = bytes(memoryview(self._data)[:length]) + else: + # the future will be replaced by next proactor.recv call + fut.cancel() if self._closing: # since close() has been called we ignore any read data - data = None return - if data == b'': - # we got end-of-file so no need to reschedule a new read - return + # bpo-33694: buffer_updated() has currently no fast path because of + # a data loss issue caused by overlapped WSASend() cancellation. - # reschedule a new read - self._read_fut = self._loop._proactor.recv(self._sock, 4096) + if not self._paused: + # reschedule a new read + self._read_fut = self._loop._proactor.recv_into(self._sock, self._data) except ConnectionAbortedError as exc: if not self._closing: self._fatal_error(exc, 'Fatal read error on pipe transport') @@ -210,32 +314,36 @@ def _loop_reading(self, fut=None): self._force_close(exc) except OSError as exc: self._fatal_error(exc, 'Fatal read error on pipe transport') - except futures.CancelledError: + except exceptions.CancelledError: if not self._closing: raise else: - self._read_fut.add_done_callback(self._loop_reading) + if not self._paused: + self._read_fut.add_done_callback(self._loop_reading) finally: - if data: - self._protocol.data_received(data) - elif data is not None: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if not keep_open: - self.close() + if length > -1: + self._data_received(data, length) class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, transports.WriteTransport): """Transport for write pipes.""" + _start_tls_compatible = True + + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self._empty_waiter = None + def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) + raise TypeError( + f"data argument must be a bytes-like object, " + f"not {type(data).__name__}") if self._eof_written: raise RuntimeError('write_eof() already called') + if self._empty_waiter is not None: + raise RuntimeError('unable to write; sendfile is in progress') if not data: return @@ -267,6 +375,10 @@ def write(self, data): def _loop_writing(self, f=None, data=None): try: + if f is not None and self._write_fut is None and self._closing: + # XXX most likely self._force_close() has been called, and + # it has set self._write_fut to None. + return assert f is self._write_fut self._write_fut = None self._pending_write = 0 @@ -295,6 +407,8 @@ def _loop_writing(self, f=None, data=None): self._maybe_pause_protocol() else: self._write_fut.add_done_callback(self._loop_writing) + if self._empty_waiter is not None and self._write_fut is None: + self._empty_waiter.set_result(None) except ConnectionResetError as exc: self._force_close(exc) except OSError as exc: @@ -309,6 +423,17 @@ def write_eof(self): def abort(self): self._force_close(None) + def _make_empty_waiter(self): + if self._empty_waiter is not None: + raise RuntimeError("Empty waiter is already set") + self._empty_waiter = self._loop.create_future() + if self._write_fut is None: + self._empty_waiter.set_result(None) + return self._empty_waiter + + def _reset_empty_waiter(self): + self._empty_waiter = None + class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport): def __init__(self, *args, **kw): @@ -332,6 +457,137 @@ def _pipe_closed(self, fut): self.close() +class _ProactorDatagramTransport(_ProactorBasePipeTransport, + transports.DatagramTransport): + max_size = 256 * 1024 + _header_size = 8 + + def __init__(self, loop, sock, protocol, address=None, + waiter=None, extra=None): + self._address = address + self._empty_waiter = None + self._buffer_size = 0 + # We don't need to call _protocol.connection_made() since our base + # constructor does it for us. + super().__init__(loop, sock, protocol, waiter=waiter, extra=extra) + + # The base constructor sets _buffer = None, so we set it here + self._buffer = collections.deque() + self._loop.call_soon(self._loop_reading) + + def _set_extra(self, sock): + _set_socket_extra(self, sock) + + def get_write_buffer_size(self): + return self._buffer_size + + def abort(self): + self._force_close(None) + + def sendto(self, data, addr=None): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be bytes-like object (%r)', + type(data)) + + if self._address is not None and addr not in (None, self._address): + raise ValueError( + f'Invalid address: must be None or {self._address}') + + if self._conn_lost and self._address: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.sendto() raised exception.') + self._conn_lost += 1 + return + + # Ensure that what we buffer is immutable. + self._buffer.append((bytes(data), addr)) + self._buffer_size += len(data) + self._header_size + + if self._write_fut is None: + # No current write operations are active, kick one off + self._loop_writing() + # else: A write operation is already kicked off + + self._maybe_pause_protocol() + + def _loop_writing(self, fut=None): + try: + if self._conn_lost: + return + + assert fut is self._write_fut + self._write_fut = None + if fut: + # We are in a _loop_writing() done callback, get the result + fut.result() + + if not self._buffer or (self._conn_lost and self._address): + # The connection has been closed + if self._closing: + self._loop.call_soon(self._call_connection_lost, None) + return + + data, addr = self._buffer.popleft() + self._buffer_size -= len(data) + self._header_size + if self._address is not None: + self._write_fut = self._loop._proactor.send(self._sock, + data) + else: + self._write_fut = self._loop._proactor.sendto(self._sock, + data, + addr=addr) + except OSError as exc: + self._protocol.error_received(exc) + except Exception as exc: + self._fatal_error(exc, 'Fatal write error on datagram transport') + else: + self._write_fut.add_done_callback(self._loop_writing) + self._maybe_resume_protocol() + + def _loop_reading(self, fut=None): + data = None + try: + if self._conn_lost: + return + + assert self._read_fut is fut or (self._read_fut is None and + self._closing) + + self._read_fut = None + if fut is not None: + res = fut.result() + + if self._closing: + # since close() has been called we ignore any read data + data = None + return + + if self._address is not None: + data, addr = res, self._address + else: + data, addr = res + + if self._conn_lost: + return + if self._address is not None: + self._read_fut = self._loop._proactor.recv(self._sock, + self.max_size) + else: + self._read_fut = self._loop._proactor.recvfrom(self._sock, + self.max_size) + except OSError as exc: + self._protocol.error_received(exc) + except exceptions.CancelledError: + if not self._closing: + raise + else: + if self._read_fut is not None: + self._read_fut.add_done_callback(self._loop_reading) + finally: + if data: + self._protocol.datagram_received(data, addr) + + class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport, _ProactorBaseWritePipeTransport, transports.Transport): @@ -349,21 +605,15 @@ class _ProactorSocketTransport(_ProactorReadPipeTransport, transports.Transport): """Transport for connected sockets.""" + _sendfile_compatible = constants._SendfileMode.TRY_NATIVE + + def __init__(self, loop, sock, protocol, waiter=None, + extra=None, server=None): + super().__init__(loop, sock, protocol, waiter, extra, server) + base_events._set_nodelay(sock) + def _set_extra(self, sock): - self._extra['socket'] = sock - try: - self._extra['sockname'] = sock.getsockname() - except (socket.error, AttributeError): - if self._loop.get_debug(): - logger.warning("getsockname() failed on %r", - sock, exc_info=True) - if 'peername' not in self._extra: - try: - self._extra['peername'] = sock.getpeername() - except (socket.error, AttributeError): - if self._loop.get_debug(): - logger.warning("getpeername() failed on %r", - sock, exc_info=True) + _set_socket_extra(self, sock) def can_write_eof(self): return True @@ -387,26 +637,35 @@ def __init__(self, proactor): self._accept_futures = {} # socket file descriptor => Future proactor.set_loop(self) self._make_self_pipe() + if threading.current_thread() is threading.main_thread(): + # wakeup fd can only be installed to a file descriptor from the main thread + signal.set_wakeup_fd(self._csock.fileno()) def _make_socket_transport(self, sock, protocol, waiter=None, extra=None, server=None): return _ProactorSocketTransport(self, sock, protocol, waiter, extra, server) - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): - if not sslproto._is_sslproto_available(): - raise NotImplementedError("Proactor event loop requires Python 3.5" - " or newer (ssl.MemoryBIO) to support " - "SSL") - - ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, - server_side, server_hostname) + def _make_ssl_transport( + self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): + ssl_protocol = sslproto.SSLProtocol( + self, protocol, sslcontext, waiter, + server_side, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) _ProactorSocketTransport(self, rawsock, ssl_protocol, extra=extra, server=server) return ssl_protocol._app_transport + def _make_datagram_transport(self, sock, protocol, + address=None, waiter=None, extra=None): + return _ProactorDatagramTransport(self, sock, protocol, address, + waiter, extra) + def _make_duplex_pipe_transport(self, sock, protocol, waiter=None, extra=None): return _ProactorDuplexPipeTransport(self, @@ -428,6 +687,8 @@ def close(self): if self.is_closed(): return + if threading.current_thread() is threading.main_thread(): + signal.set_wakeup_fd(-1) # Call these methods before closing the event loop (before calling # BaseEventLoop.close), because they can schedule callbacks with # call_soon(), which is forbidden when the event loop is closed. @@ -440,20 +701,75 @@ def close(self): # Close the event loop super().close() - def sock_recv(self, sock, n): - return self._proactor.recv(sock, n) + async def sock_recv(self, sock, n): + return await self._proactor.recv(sock, n) - def sock_sendall(self, sock, data): - return self._proactor.send(sock, data) + async def sock_recv_into(self, sock, buf): + return await self._proactor.recv_into(sock, buf) - def sock_connect(self, sock, address): - return self._proactor.connect(sock, address) + async def sock_recvfrom(self, sock, bufsize): + return await self._proactor.recvfrom(sock, bufsize) - def sock_accept(self, sock): - return self._proactor.accept(sock) + async def sock_recvfrom_into(self, sock, buf, nbytes=0): + if not nbytes: + nbytes = len(buf) - def _socketpair(self): - raise NotImplementedError + return await self._proactor.recvfrom_into(sock, buf, nbytes) + + async def sock_sendall(self, sock, data): + return await self._proactor.send(sock, data) + + async def sock_sendto(self, sock, data, address): + return await self._proactor.sendto(sock, data, 0, address) + + async def sock_connect(self, sock, address): + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + return await self._proactor.connect(sock, address) + + async def sock_accept(self, sock): + return await self._proactor.accept(sock) + + async def _sock_sendfile_native(self, sock, file, offset, count): + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise exceptions.SendfileNotAvailableError("not a regular file") + try: + fsize = os.fstat(fileno).st_size + except OSError: + raise exceptions.SendfileNotAvailableError("not a regular file") + blocksize = count if count else fsize + if not blocksize: + return 0 # empty file + + blocksize = min(blocksize, 0xffff_ffff) + end_pos = min(offset + count, fsize) if count else fsize + offset = min(offset, fsize) + total_sent = 0 + try: + while True: + blocksize = min(end_pos - offset, blocksize) + if blocksize <= 0: + return total_sent + await self._proactor.sendfile(sock, file, offset, blocksize) + offset += blocksize + total_sent += blocksize + finally: + if total_sent > 0: + file.seek(offset) + + async def _sendfile_native(self, transp, file, offset, count): + resume_reading = transp.is_reading() + transp.pause_reading() + await transp._make_empty_waiter() + try: + return await self.sock_sendfile(transp._sock, file, offset, count, + fallback=False) + finally: + transp._reset_empty_waiter() + if resume_reading: + transp.resume_reading() def _close_self_pipe(self): if self._self_reading_future is not None: @@ -467,21 +783,30 @@ def _close_self_pipe(self): def _make_self_pipe(self): # A self-socket, really. :-) - self._ssock, self._csock = self._socketpair() + self._ssock, self._csock = socket.socketpair() self._ssock.setblocking(False) self._csock.setblocking(False) self._internal_fds += 1 - self.call_soon(self._loop_self_reading) def _loop_self_reading(self, f=None): try: if f is not None: f.result() # may raise + if self._self_reading_future is not f: + # When we scheduled this Future, we assigned it to + # _self_reading_future. If it's not there now, something has + # tried to cancel the loop while this callback was still in the + # queue (see windows_events.ProactorEventLoop.run_forever). In + # that case stop here instead of continuing to schedule a new + # iteration. + return f = self._proactor.recv(self._ssock, 4096) - except futures.CancelledError: + except exceptions.CancelledError: # _close_self_pipe() has been called, stop waiting for data return - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self.call_exception_handler({ 'message': 'Error on reading from the event loop self pipe', 'exception': exc, @@ -492,10 +817,27 @@ def _loop_self_reading(self, f=None): f.add_done_callback(self._loop_self_reading) def _write_to_self(self): - self._csock.send(b'\0') + # This may be called from a different thread, possibly after + # _close_self_pipe() has been called or even while it is + # running. Guard for self._csock being None or closed. When + # a socket is closed, send() raises OSError (with errno set to + # EBADF, but let's not rely on the exact error code). + csock = self._csock + if csock is None: + return + + try: + csock.send(b'\0') + except OSError: + if self._debug: + logger.debug("Fail to write a null byte into the " + "self-pipe socket", + exc_info=True) def _start_serving(self, protocol_factory, sock, - sslcontext=None, server=None, backlog=100): + sslcontext=None, server=None, backlog=100, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): def loop(f=None): try: @@ -508,7 +850,9 @@ def loop(f=None): if sslcontext is not None: self._make_ssl_transport( conn, protocol, sslcontext, server_side=True, - extra={'peername': addr}, server=server) + extra={'peername': addr}, server=server, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) else: self._make_socket_transport( conn, protocol, @@ -521,13 +865,13 @@ def loop(f=None): self.call_exception_handler({ 'message': 'Accept failed on a socket', 'exception': exc, - 'socket': sock, + 'socket': trsock.TransportSocket(sock), }) sock.close() elif self._debug: logger.debug("Accept failed on socket %r", sock, exc_info=True) - except futures.CancelledError: + except exceptions.CancelledError: sock.close() else: self._accept_futures[sock.fileno()] = f @@ -545,6 +889,8 @@ def _stop_accept_futures(self): self._accept_futures.clear() def _stop_serving(self, sock): - self._stop_accept_futures() + future = self._accept_futures.pop(sock.fileno(), None) + if future: + future.cancel() self._proactor._stop_serving(sock) sock.close() diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py index 80fcac9a82d..09987b164c6 100644 --- a/Lib/asyncio/protocols.py +++ b/Lib/asyncio/protocols.py @@ -1,7 +1,9 @@ -"""Abstract Protocol class.""" +"""Abstract Protocol base classes.""" -__all__ = ['BaseProtocol', 'Protocol', 'DatagramProtocol', - 'SubprocessProtocol'] +__all__ = ( + 'BaseProtocol', 'Protocol', 'DatagramProtocol', + 'SubprocessProtocol', 'BufferedProtocol', +) class BaseProtocol: @@ -14,6 +16,8 @@ class BaseProtocol: write-only transport like write pipe """ + __slots__ = () + def connection_made(self, transport): """Called when a connection is made. @@ -85,6 +89,8 @@ class Protocol(BaseProtocol): * CL: connection_lost() """ + __slots__ = () + def data_received(self, data): """Called when some data is received. @@ -100,9 +106,64 @@ def eof_received(self): """ +class BufferedProtocol(BaseProtocol): + """Interface for stream protocol with manual buffer control. + + Event methods, such as `create_server` and `create_connection`, + accept factories that return protocols that implement this interface. + + The idea of BufferedProtocol is that it allows to manually allocate + and control the receive buffer. Event loops can then use the buffer + provided by the protocol to avoid unnecessary data copies. This + can result in noticeable performance improvement for protocols that + receive big amounts of data. Sophisticated protocols can allocate + the buffer only once at creation time. + + State machine of calls: + + start -> CM [-> GB [-> BU?]]* [-> ER?] -> CL -> end + + * CM: connection_made() + * GB: get_buffer() + * BU: buffer_updated() + * ER: eof_received() + * CL: connection_lost() + """ + + __slots__ = () + + def get_buffer(self, sizehint): + """Called to allocate a new receive buffer. + + *sizehint* is a recommended minimal size for the returned + buffer. When set to -1, the buffer size can be arbitrary. + + Must return an object that implements the + :ref:`buffer protocol `. + It is an error to return a zero-sized buffer. + """ + + def buffer_updated(self, nbytes): + """Called when the buffer was updated with the received data. + + *nbytes* is the total number of bytes that were written to + the buffer. + """ + + def eof_received(self): + """Called when the other end calls write_eof() or equivalent. + + If this returns a false value (including None), the transport + will close itself. If it returns a true value, closing the + transport is up to the protocol. + """ + + class DatagramProtocol(BaseProtocol): """Interface for datagram protocol.""" + __slots__ = () + def datagram_received(self, data, addr): """Called when some datagram is received.""" @@ -116,6 +177,8 @@ def error_received(self, exc): class SubprocessProtocol(BaseProtocol): """Interface for protocol for subprocess calls.""" + __slots__ = () + def pipe_data_received(self, fd, data): """Called when the subprocess writes data into stdout/stderr pipe. @@ -132,3 +195,22 @@ def pipe_connection_lost(self, fd, exc): def process_exited(self): """Called when subprocess has exited.""" + + +def _feed_data_to_buffered_proto(proto, data): + data_len = len(data) + while data_len: + buf = proto.get_buffer(data_len) + buf_len = len(buf) + if not buf_len: + raise RuntimeError('get_buffer() returned an empty buffer') + + if buf_len >= data_len: + buf[:data_len] = data + proto.buffer_updated(data_len) + return + else: + buf[:buf_len] = data[:buf_len] + proto.buffer_updated(buf_len) + data = data[buf_len:] + data_len = len(data) diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index e16c46ae738..084fccaaff2 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,35 +1,40 @@ -"""Queues""" - -__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty'] +__all__ = ( + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'QueueFull', + 'QueueEmpty', + 'QueueShutDown', +) import collections import heapq +from types import GenericAlias -from . import compat -from . import events from . import locks -from .coroutines import coroutine +from . import mixins class QueueEmpty(Exception): - """Exception raised when Queue.get_nowait() is called on a Queue object - which is empty. - """ + """Raised when Queue.get_nowait() is called on an empty Queue.""" pass class QueueFull(Exception): - """Exception raised when the Queue.put_nowait() method is called on a Queue - object which is full. - """ + """Raised when the Queue.put_nowait() method is called on a full Queue.""" + pass + + +class QueueShutDown(Exception): + """Raised when putting on to or getting from a shut-down Queue.""" pass -class Queue: +class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "yield from put()" will block when the + is an integer greater than 0, then "await put()" will block when the queue reaches maxsize, until an item is removed by get(). Unlike the standard library Queue, you can reliably know this Queue's size @@ -37,11 +42,7 @@ class Queue: interrupted between calling qsize() and doing an operation on the Queue. """ - def __init__(self, maxsize=0, *, loop=None): - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop + def __init__(self, maxsize=0): self._maxsize = maxsize # Futures. @@ -49,9 +50,10 @@ def __init__(self, maxsize=0, *, loop=None): # Futures. self._putters = collections.deque() self._unfinished_tasks = 0 - self._finished = locks.Event(loop=self._loop) + self._finished = locks.Event() self._finished.set() self._init(maxsize) + self._is_shutdown = False # These three are overridable in subclasses. @@ -75,25 +77,25 @@ def _wakeup_next(self, waiters): break def __repr__(self): - return '<{} at {:#x} {}>'.format( - type(self).__name__, id(self), self._format()) + return f'<{type(self).__name__} at {id(self):#x} {self._format()}>' def __str__(self): - return '<{} {}>'.format(type(self).__name__, self._format()) + return f'<{type(self).__name__} {self._format()}>' - def __class_getitem__(cls, type): - return cls + __class_getitem__ = classmethod(GenericAlias) def _format(self): - result = 'maxsize={!r}'.format(self._maxsize) + result = f'maxsize={self._maxsize!r}' if getattr(self, '_queue', None): - result += ' _queue={!r}'.format(list(self._queue)) + result += f' _queue={list(self._queue)!r}' if self._getters: - result += ' _getters[{}]'.format(len(self._getters)) + result += f' _getters[{len(self._getters)}]' if self._putters: - result += ' _putters[{}]'.format(len(self._putters)) + result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: - result += ' tasks={}'.format(self._unfinished_tasks) + result += f' tasks={self._unfinished_tasks}' + if self._is_shutdown: + result += ' shutdown' return result def qsize(self): @@ -120,22 +122,30 @@ def full(self): else: return self.qsize() >= self._maxsize - @coroutine - def put(self, item): + async def put(self, item): """Put an item into the queue. Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. - This method is a coroutine. + Raises QueueShutDown if the queue has been shut down. """ while self.full(): - putter = self._loop.create_future() + if self._is_shutdown: + raise QueueShutDown + putter = self._get_loop().create_future() self._putters.append(putter) try: - yield from putter + await putter except: putter.cancel() # Just in case putter is not done yet. + try: + # Clean self._putters from canceled putters. + self._putters.remove(putter) + except ValueError: + # The putter could be removed from self._putters by a + # previous get_nowait call or a shutdown call. + pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take # the call. Wake up the next in line. @@ -147,7 +157,11 @@ def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise QueueFull. + + Raises QueueShutDown if the queue has been shut down. """ + if self._is_shutdown: + raise QueueShutDown if self.full(): raise QueueFull self._put(item) @@ -155,21 +169,30 @@ def put_nowait(self, item): self._finished.clear() self._wakeup_next(self._getters) - @coroutine - def get(self): + async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. - This method is a coroutine. + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ while self.empty(): - getter = self._loop.create_future() + if self._is_shutdown and self.empty(): + raise QueueShutDown + getter = self._get_loop().create_future() self._getters.append(getter) try: - yield from getter + await getter except: getter.cancel() # Just in case getter is not done yet. + try: + # Clean self._getters from canceled getters. + self._getters.remove(getter) + except ValueError: + # The getter could be removed from self._getters by a + # previous put_nowait call, or a shutdown call. + pass if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take # the call. Wake up the next in line. @@ -181,8 +204,13 @@ def get_nowait(self): """Remove and return an item from the queue. Return an item if one is immediately available, else raise QueueEmpty. + + Raises QueueShutDown if the queue has been shut down and is empty, or + if the queue has been shut down immediately. """ if self.empty(): + if self._is_shutdown: + raise QueueShutDown raise QueueEmpty item = self._get() self._wakeup_next(self._putters) @@ -208,8 +236,7 @@ def task_done(self): if self._unfinished_tasks == 0: self._finished.set() - @coroutine - def join(self): + async def join(self): """Block until all items in the queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the @@ -218,7 +245,37 @@ def join(self): When the count of unfinished tasks drops to zero, join() unblocks. """ if self._unfinished_tasks > 0: - yield from self._finished.wait() + await self._finished.wait() + + def shutdown(self, immediate=False): + """Shut-down the queue, making queue gets and puts raise QueueShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. + """ + self._is_shutdown = True + if immediate: + while not self.empty(): + self._get() + if self._unfinished_tasks > 0: + self._unfinished_tasks -= 1 + if self._unfinished_tasks == 0: + self._finished.set() + # All getters need to re-check queue-empty to raise ShutDown + while self._getters: + getter = self._getters.popleft() + if not getter.done(): + getter.set_result(None) + while self._putters: + putter = self._putters.popleft() + if not putter.done(): + putter.set_result(None) class PriorityQueue(Queue): @@ -248,9 +305,3 @@ def _put(self, item): def _get(self): return self._queue.pop() - - -if not compat.PY35: - JoinableQueue = Queue - """Deprecated alias for Queue.""" - __all__.append('JoinableQueue') diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index c3a696ef579..ba37e003a65 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -1,26 +1,192 @@ -__all__ = ['run'] - +__all__ = ('Runner', 'run') + +import contextvars +import enum +import functools +import inspect +import threading +import signal from . import coroutines from . import events +from . import exceptions from . import tasks +from . import constants + +class _State(enum.Enum): + CREATED = "created" + INITIALIZED = "initialized" + CLOSED = "closed" + + +class Runner: + """A context manager that controls event loop life cycle. + + The context manager always creates a new event loop, + allows to run async functions inside it, + and properly finalizes the loop at the context manager exit. + + If debug is True, the event loop will be run in debug mode. + If loop_factory is passed, it is used for new event loop creation. + + asyncio.run(main(), debug=True) + + is a shortcut for + + with asyncio.Runner(debug=True) as runner: + runner.run(main()) + + The run() method can be called multiple times within the runner's context. + + This can be useful for interactive console (e.g. IPython), + unittest runners, console tools, -- everywhere when async code + is called from existing sync framework and where the preferred single + asyncio.run() call doesn't work. + + """ + + # Note: the class is final, it is not intended for inheritance. + + def __init__(self, *, debug=None, loop_factory=None): + self._state = _State.CREATED + self._debug = debug + self._loop_factory = loop_factory + self._loop = None + self._context = None + self._interrupt_count = 0 + self._set_event_loop = False + def __enter__(self): + self._lazy_init() + return self -def run(main, *, debug=False): - """Run a coroutine. + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + """Shutdown and close event loop.""" + if self._state is not _State.INITIALIZED: + return + try: + loop = self._loop + _cancel_all_tasks(loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete( + loop.shutdown_default_executor(constants.THREAD_JOIN_TIMEOUT)) + finally: + if self._set_event_loop: + events.set_event_loop(None) + loop.close() + self._loop = None + self._state = _State.CLOSED + + def get_loop(self): + """Return embedded event loop.""" + self._lazy_init() + return self._loop + + def run(self, coro, *, context=None): + """Run code in the embedded event loop.""" + if events._get_running_loop() is not None: + # fail fast with short traceback + raise RuntimeError( + "Runner.run() cannot be called from a running event loop") + + self._lazy_init() + + if not coroutines.iscoroutine(coro): + if inspect.isawaitable(coro): + async def _wrap_awaitable(awaitable): + return await awaitable + + coro = _wrap_awaitable(coro) + else: + raise TypeError('An asyncio.Future, a coroutine or an ' + 'awaitable is required') + + if context is None: + context = self._context + + task = self._loop.create_task(coro, context=context) + + if (threading.current_thread() is threading.main_thread() + and signal.getsignal(signal.SIGINT) is signal.default_int_handler + ): + sigint_handler = functools.partial(self._on_sigint, main_task=task) + try: + signal.signal(signal.SIGINT, sigint_handler) + except ValueError: + # `signal.signal` may throw if `threading.main_thread` does + # not support signals (e.g. embedded interpreter with signals + # not registered - see gh-91880) + sigint_handler = None + else: + sigint_handler = None + + self._interrupt_count = 0 + try: + return self._loop.run_until_complete(task) + except exceptions.CancelledError: + if self._interrupt_count > 0: + uncancel = getattr(task, "uncancel", None) + if uncancel is not None and uncancel() == 0: + raise KeyboardInterrupt() + raise # CancelledError + finally: + if (sigint_handler is not None + and signal.getsignal(signal.SIGINT) is sigint_handler + ): + signal.signal(signal.SIGINT, signal.default_int_handler) + + def _lazy_init(self): + if self._state is _State.CLOSED: + raise RuntimeError("Runner is closed") + if self._state is _State.INITIALIZED: + return + if self._loop_factory is None: + self._loop = events.new_event_loop() + if not self._set_event_loop: + # Call set_event_loop only once to avoid calling + # attach_loop multiple times on child watchers + events.set_event_loop(self._loop) + self._set_event_loop = True + else: + self._loop = self._loop_factory() + if self._debug is not None: + self._loop.set_debug(self._debug) + self._context = contextvars.copy_context() + self._state = _State.INITIALIZED + + def _on_sigint(self, signum, frame, main_task): + self._interrupt_count += 1 + if self._interrupt_count == 1 and not main_task.done(): + main_task.cancel() + # wakeup loop if it is blocked by select() with long timeout + self._loop.call_soon_threadsafe(lambda: None) + return + raise KeyboardInterrupt() + + +def run(main, *, debug=None, loop_factory=None): + """Execute the coroutine and return the result. This function runs the passed coroutine, taking care of - managing the asyncio event loop and finalizing asynchronous - generators. + managing the asyncio event loop, finalizing asynchronous + generators and closing the default executor. This function cannot be called when another asyncio event loop is running in the same thread. If debug is True, the event loop will be run in debug mode. + If loop_factory is passed, it is used for new event loop creation. This function always creates a new event loop and closes it at the end. It should be used as a main entry point for asyncio programs, and should ideally only be called once. + The executor is given a timeout duration of 5 minutes to shutdown. + If the executor hasn't finished within that duration, a warning is + emitted and the executor is closed. + Example: async def main(): @@ -30,24 +196,12 @@ async def main(): asyncio.run(main()) """ if events._get_running_loop() is not None: + # fail fast with short traceback raise RuntimeError( "asyncio.run() cannot be called from a running event loop") - if not coroutines.iscoroutine(main): - raise ValueError("a coroutine was expected, got {!r}".format(main)) - - loop = events.new_event_loop() - try: - events.set_event_loop(loop) - loop.set_debug(debug) - return loop.run_until_complete(main) - finally: - try: - _cancel_all_tasks(loop) - loop.run_until_complete(loop.shutdown_asyncgens()) - finally: - events.set_event_loop(None) - loop.close() + with Runner(debug=debug, loop_factory=loop_factory) as runner: + return runner.run(main) def _cancel_all_tasks(loop): @@ -58,8 +212,7 @@ def _cancel_all_tasks(loop): for task in to_cancel: task.cancel() - loop.run_until_complete( - tasks.gather(*to_cancel, loop=loop, return_exceptions=True)) + loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True)) for task in to_cancel: if task.cancelled(): diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 9dbe550b017..ff7e16df3c6 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -4,11 +4,14 @@ also includes support for signal handling, see the unix_events sub-module. """ -__all__ = ['BaseSelectorEventLoop'] +__all__ = 'BaseSelectorEventLoop', import collections import errno import functools +import itertools +import os +import selectors import socket import warnings import weakref @@ -18,16 +21,23 @@ ssl = None from . import base_events -from . import compat from . import constants from . import events from . import futures -from . import selectors -from . import transports +from . import protocols from . import sslproto -from .coroutines import coroutine +from . import transports +from . import trsock from .log import logger +_HAS_SENDMSG = hasattr(socket.socket, 'sendmsg') + +if _HAS_SENDMSG: + try: + SC_IOV_MAX = os.sysconf('SC_IOV_MAX') + except OSError: + # Fallback to send + _HAS_SENDMSG = False def _test_selector_event(selector, fd, event): # Test if the selector is monitoring 'event' events @@ -40,17 +50,6 @@ def _test_selector_event(selector, fd, event): return bool(key.events & event) -if hasattr(socket, 'TCP_NODELAY'): - def _set_nodelay(sock): - if (sock.family in {socket.AF_INET, socket.AF_INET6} and - sock.type == socket.SOCK_STREAM and - sock.proto == socket.IPPROTO_TCP): - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) -else: - def _set_nodelay(sock): - pass - - class BaseSelectorEventLoop(base_events.BaseEventLoop): """Selector event loop. @@ -69,36 +68,31 @@ def __init__(self, selector=None): def _make_socket_transport(self, sock, protocol, waiter=None, *, extra=None, server=None): + self._ensure_fd_no_transport(sock) return _SelectorSocketTransport(self, sock, protocol, waiter, extra, server) - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): - if not sslproto._is_sslproto_available(): - return self._make_legacy_ssl_transport( - rawsock, protocol, sslcontext, waiter, - server_side=server_side, server_hostname=server_hostname, - extra=extra, server=server) - - ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, - server_side, server_hostname) + def _make_ssl_transport( + self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None, + ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, + ): + self._ensure_fd_no_transport(rawsock) + ssl_protocol = sslproto.SSLProtocol( + self, protocol, sslcontext, waiter, + server_side, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout + ) _SelectorSocketTransport(self, rawsock, ssl_protocol, extra=extra, server=server) return ssl_protocol._app_transport - def _make_legacy_ssl_transport(self, rawsock, protocol, sslcontext, - waiter, *, - server_side=False, server_hostname=None, - extra=None, server=None): - # Use the legacy API: SSL_write, SSL_read, etc. The legacy API is used - # on Python 3.4 and older, when ssl.MemoryBIO is not available. - return _SelectorSslTransport( - self, rawsock, protocol, sslcontext, waiter, - server_side, server_hostname, extra, server) - def _make_datagram_transport(self, sock, protocol, address=None, waiter=None, extra=None): + self._ensure_fd_no_transport(sock) return _SelectorDatagramTransport(self, sock, protocol, address, waiter, extra) @@ -113,9 +107,6 @@ def close(self): self._selector.close() self._selector = None - def _socketpair(self): - raise NotImplementedError - def _close_self_pipe(self): self._remove_reader(self._ssock.fileno()) self._ssock.close() @@ -126,7 +117,7 @@ def _close_self_pipe(self): def _make_self_pipe(self): # A self-socket, really. :-) - self._ssock, self._csock = self._socketpair() + self._ssock, self._csock = socket.socketpair() self._ssock.setblocking(False) self._csock.setblocking(False) self._internal_fds += 1 @@ -154,36 +145,48 @@ def _write_to_self(self): # a socket is closed, send() raises OSError (with errno set to # EBADF, but let's not rely on the exact error code). csock = self._csock - if csock is not None: - try: - csock.send(b'\0') - except OSError: - if self._debug: - logger.debug("Fail to write a null byte into the " - "self-pipe socket", - exc_info=True) + if csock is None: + return + + try: + csock.send(b'\0') + except OSError: + if self._debug: + logger.debug("Fail to write a null byte into the " + "self-pipe socket", + exc_info=True) def _start_serving(self, protocol_factory, sock, - sslcontext=None, server=None, backlog=100): + sslcontext=None, server=None, backlog=100, + ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): self._add_reader(sock.fileno(), self._accept_connection, - protocol_factory, sock, sslcontext, server, backlog) - - def _accept_connection(self, protocol_factory, sock, - sslcontext=None, server=None, backlog=100): + protocol_factory, sock, sslcontext, server, backlog, + ssl_handshake_timeout, ssl_shutdown_timeout) + + def _accept_connection( + self, protocol_factory, sock, + sslcontext=None, server=None, backlog=100, + ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): # This method is only called once for each event loop tick where the # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. # See https://bugs.python.org/issue27906 for more details. - for _ in range(backlog): + for _ in range(backlog + 1): try: conn, addr = sock.accept() if self._debug: logger.debug("%r got a new connection from %r: %r", server, addr, conn) conn.setblocking(False) - except (BlockingIOError, InterruptedError, ConnectionAbortedError): - # Early exit because the socket accept buffer is empty. - return None + except ConnectionAbortedError: + # Discard connections that were aborted before accept(). + continue + except (BlockingIOError, InterruptedError): + # Early exit because of a signal or + # the socket accept buffer is empty. + return except OSError as exc: # There's nowhere to send the error, so just log it. if exc.errno in (errno.EMFILE, errno.ENFILE, @@ -194,24 +197,28 @@ def _accept_connection(self, protocol_factory, sock, self.call_exception_handler({ 'message': 'socket.accept() out of system resource', 'exception': exc, - 'socket': sock, + 'socket': trsock.TransportSocket(sock), }) self._remove_reader(sock.fileno()) self.call_later(constants.ACCEPT_RETRY_DELAY, self._start_serving, protocol_factory, sock, sslcontext, server, - backlog) + backlog, ssl_handshake_timeout, + ssl_shutdown_timeout) else: raise # The event loop will catch, log and ignore it. else: extra = {'peername': addr} - accept = self._accept_connection2(protocol_factory, conn, extra, - sslcontext, server) + accept = self._accept_connection2( + protocol_factory, conn, extra, sslcontext, server, + ssl_handshake_timeout, ssl_shutdown_timeout) self.create_task(accept) - @coroutine - def _accept_connection2(self, protocol_factory, conn, extra, - sslcontext=None, server=None): + async def _accept_connection2( + self, protocol_factory, conn, extra, + sslcontext=None, server=None, + ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): protocol = None transport = None try: @@ -220,24 +227,32 @@ def _accept_connection2(self, protocol_factory, conn, extra, if sslcontext: transport = self._make_ssl_transport( conn, protocol, sslcontext, waiter=waiter, - server_side=True, extra=extra, server=server) + server_side=True, extra=extra, server=server, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) else: transport = self._make_socket_transport( conn, protocol, waiter=waiter, extra=extra, server=server) try: - yield from waiter - except: + await waiter + except BaseException: transport.close() + # gh-109534: When an exception is raised by the SSLProtocol object the + # exception set in this future can keep the protocol object alive and + # cause a reference cycle. + waiter = None raise + # It's now up to the protocol to handle the connection. - # It's now up to the protocol to handle the connection. - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: if self._debug: context = { - 'message': ('Error on transport creation ' - 'for incoming connection'), + 'message': + 'Error on transport creation for incoming connection', 'exception': exc, } if protocol is not None: @@ -247,22 +262,24 @@ def _accept_connection2(self, protocol_factory, conn, extra, self.call_exception_handler(context) def _ensure_fd_no_transport(self, fd): - try: - transport = self._transports[fd] - except KeyError: - pass - else: - if not transport.is_closing(): - raise RuntimeError( - 'File descriptor {!r} is used by transport {!r}'.format( - fd, transport)) + fileno = fd + if not isinstance(fileno, int): + try: + fileno = int(fileno.fileno()) + except (AttributeError, TypeError, ValueError): + # This code matches selectors._fileobj_to_fd function. + raise ValueError(f"Invalid file object: {fd!r}") from None + transport = self._transports.get(fileno) + if transport and not transport.is_closing(): + raise RuntimeError( + f'File descriptor {fd!r} is used by transport ' + f'{transport!r}') def _add_reader(self, fd, callback, *args): self._check_closed() - handle = events.Handle(callback, args, self) - try: - key = self._selector.get_key(fd) - except KeyError: + handle = events.Handle(callback, args, self, None) + key = self._selector.get_map().get(fd) + if key is None: self._selector.register(fd, selectors.EVENT_READ, (handle, None)) else: @@ -271,34 +288,32 @@ def _add_reader(self, fd, callback, *args): (handle, writer)) if reader is not None: reader.cancel() + return handle def _remove_reader(self, fd): if self.is_closed(): return False - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: return False + mask, (reader, writer) = key.events, key.data + mask &= ~selectors.EVENT_READ + if not mask: + self._selector.unregister(fd) else: - mask, (reader, writer) = key.events, key.data - mask &= ~selectors.EVENT_READ - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (None, writer)) + self._selector.modify(fd, mask, (None, writer)) - if reader is not None: - reader.cancel() - return True - else: - return False + if reader is not None: + reader.cancel() + return True + else: + return False def _add_writer(self, fd, callback, *args): self._check_closed() - handle = events.Handle(callback, args, self) - try: - key = self._selector.get_key(fd) - except KeyError: + handle = events.Handle(callback, args, self, None) + key = self._selector.get_map().get(fd) + if key is None: self._selector.register(fd, selectors.EVENT_WRITE, (None, handle)) else: @@ -307,34 +322,33 @@ def _add_writer(self, fd, callback, *args): (reader, handle)) if writer is not None: writer.cancel() + return handle def _remove_writer(self, fd): """Remove a writer callback.""" if self.is_closed(): return False - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: return False + mask, (reader, writer) = key.events, key.data + # Remove both writer and connector. + mask &= ~selectors.EVENT_WRITE + if not mask: + self._selector.unregister(fd) else: - mask, (reader, writer) = key.events, key.data - # Remove both writer and connector. - mask &= ~selectors.EVENT_WRITE - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (reader, None)) + self._selector.modify(fd, mask, (reader, None)) - if writer is not None: - writer.cancel() - return True - else: - return False + if writer is not None: + writer.cancel() + return True + else: + return False def add_reader(self, fd, callback, *args): """Add a reader callback.""" self._ensure_fd_no_transport(fd) - return self._add_reader(fd, callback, *args) + self._add_reader(fd, callback, *args) def remove_reader(self, fd): """Remove a reader callback.""" @@ -344,111 +358,294 @@ def remove_reader(self, fd): def add_writer(self, fd, callback, *args): """Add a writer callback..""" self._ensure_fd_no_transport(fd) - return self._add_writer(fd, callback, *args) + self._add_writer(fd, callback, *args) def remove_writer(self, fd): """Remove a writer callback.""" self._ensure_fd_no_transport(fd) return self._remove_writer(fd) - def sock_recv(self, sock, n): + async def sock_recv(self, sock, n): """Receive data from the socket. The return value is a bytes object representing the data received. The maximum amount of data to be received at once is specified by nbytes. - - This method is a coroutine. """ + base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") + try: + return sock.recv(n) + except (BlockingIOError, InterruptedError): + pass fut = self.create_future() - self._sock_recv(fut, False, sock, n) - return fut + fd = sock.fileno() + self._ensure_fd_no_transport(fd) + handle = self._add_reader(fd, self._sock_recv, fut, sock, n) + fut.add_done_callback( + functools.partial(self._sock_read_done, fd, handle=handle)) + return await fut - def _sock_recv(self, fut, registered, sock, n): + def _sock_read_done(self, fd, fut, handle=None): + if handle is None or not handle.cancelled(): + self.remove_reader(fd) + + def _sock_recv(self, fut, sock, n): # _sock_recv() can add itself as an I/O callback if the operation can't # be done immediately. Don't use it directly, call sock_recv(). - fd = sock.fileno() - if registered: - # Remove the callback early. It should be rare that the - # selector says the fd is ready but the call still returns - # EAGAIN, and I am willing to take a hit in that case in - # order to simplify the common case. - self.remove_reader(fd) - if fut.cancelled(): + if fut.done(): return try: data = sock.recv(n) except (BlockingIOError, InterruptedError): - self.add_reader(fd, self._sock_recv, fut, True, sock, n) - except Exception as exc: + return # try again next time + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: fut.set_exception(exc) else: fut.set_result(data) - def sock_sendall(self, sock, data): - """Send data to the socket. - - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + async def sock_recv_into(self, sock, buf): + """Receive data from the socket. - This method is a coroutine. + The received data is written into *buf* (a writable buffer). + The return value is the number of bytes written. """ + base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") + try: + return sock.recv_into(buf) + except (BlockingIOError, InterruptedError): + pass fut = self.create_future() - if data: - self._sock_sendall(fut, False, sock, data) + fd = sock.fileno() + self._ensure_fd_no_transport(fd) + handle = self._add_reader(fd, self._sock_recv_into, fut, sock, buf) + fut.add_done_callback( + functools.partial(self._sock_read_done, fd, handle=handle)) + return await fut + + def _sock_recv_into(self, fut, sock, buf): + # _sock_recv_into() can add itself as an I/O callback if the operation + # can't be done immediately. Don't use it directly, call + # sock_recv_into(). + if fut.done(): + return + try: + nbytes = sock.recv_into(buf) + except (BlockingIOError, InterruptedError): + return # try again next time + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + fut.set_exception(exc) else: - fut.set_result(None) - return fut + fut.set_result(nbytes) + + async def sock_recvfrom(self, sock, bufsize): + """Receive a datagram from a datagram socket. - def _sock_sendall(self, fut, registered, sock, data): + The return value is a tuple of (bytes, address) representing the + datagram received and the address it came from. + The maximum amount of data to be received at once is specified by + nbytes. + """ + base_events._check_ssl_socket(sock) + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + try: + return sock.recvfrom(bufsize) + except (BlockingIOError, InterruptedError): + pass + fut = self.create_future() fd = sock.fileno() + self._ensure_fd_no_transport(fd) + handle = self._add_reader(fd, self._sock_recvfrom, fut, sock, bufsize) + fut.add_done_callback( + functools.partial(self._sock_read_done, fd, handle=handle)) + return await fut + + def _sock_recvfrom(self, fut, sock, bufsize): + # _sock_recvfrom() can add itself as an I/O callback if the operation + # can't be done immediately. Don't use it directly, call + # sock_recvfrom(). + if fut.done(): + return + try: + result = sock.recvfrom(bufsize) + except (BlockingIOError, InterruptedError): + return # try again next time + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + fut.set_exception(exc) + else: + fut.set_result(result) - if registered: - self.remove_writer(fd) - if fut.cancelled(): + async def sock_recvfrom_into(self, sock, buf, nbytes=0): + """Receive data from the socket. + + The received data is written into *buf* (a writable buffer). + The return value is a tuple of (number of bytes written, address). + """ + base_events._check_ssl_socket(sock) + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + if not nbytes: + nbytes = len(buf) + + try: + return sock.recvfrom_into(buf, nbytes) + except (BlockingIOError, InterruptedError): + pass + fut = self.create_future() + fd = sock.fileno() + self._ensure_fd_no_transport(fd) + handle = self._add_reader(fd, self._sock_recvfrom_into, fut, sock, buf, + nbytes) + fut.add_done_callback( + functools.partial(self._sock_read_done, fd, handle=handle)) + return await fut + + def _sock_recvfrom_into(self, fut, sock, buf, bufsize): + # _sock_recv_into() can add itself as an I/O callback if the operation + # can't be done immediately. Don't use it directly, call + # sock_recv_into(). + if fut.done(): return + try: + result = sock.recvfrom_into(buf, bufsize) + except (BlockingIOError, InterruptedError): + return # try again next time + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + fut.set_exception(exc) + else: + fut.set_result(result) + async def sock_sendall(self, sock, data): + """Send data to the socket. + + The socket must be connected to a remote socket. This method continues + to send data from data until either all data has been sent or an + error occurs. None is returned on success. On error, an exception is + raised, and there is no way to determine how much data, if any, was + successfully processed by the receiving end of the connection. + """ + base_events._check_ssl_socket(sock) + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") try: n = sock.send(data) except (BlockingIOError, InterruptedError): n = 0 - except Exception as exc: + + if n == len(data): + # all data sent + return + + fut = self.create_future() + fd = sock.fileno() + self._ensure_fd_no_transport(fd) + # use a trick with a list in closure to store a mutable state + handle = self._add_writer(fd, self._sock_sendall, fut, sock, + memoryview(data), [n]) + fut.add_done_callback( + functools.partial(self._sock_write_done, fd, handle=handle)) + return await fut + + def _sock_sendall(self, fut, sock, view, pos): + if fut.done(): + # Future cancellation can be scheduled on previous loop iteration + return + start = pos[0] + try: + n = sock.send(view[start:]) + except (BlockingIOError, InterruptedError): + return + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: fut.set_exception(exc) return - if n == len(data): + start += n + + if start == len(view): fut.set_result(None) else: - if n: - data = data[n:] - self.add_writer(fd, self._sock_sendall, fut, True, sock, data) + pos[0] = start + + async def sock_sendto(self, sock, data, address): + """Send data to the socket. + + The socket must be connected to a remote socket. This method continues + to send data from data until either all data has been sent or an + error occurs. None is returned on success. On error, an exception is + raised, and there is no way to determine how much data, if any, was + successfully processed by the receiving end of the connection. + """ + base_events._check_ssl_socket(sock) + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + try: + return sock.sendto(data, address) + except (BlockingIOError, InterruptedError): + pass + + fut = self.create_future() + fd = sock.fileno() + self._ensure_fd_no_transport(fd) + # use a trick with a list in closure to store a mutable state + handle = self._add_writer(fd, self._sock_sendto, fut, sock, data, + address) + fut.add_done_callback( + functools.partial(self._sock_write_done, fd, handle=handle)) + return await fut + + def _sock_sendto(self, fut, sock, data, address): + if fut.done(): + # Future cancellation can be scheduled on previous loop iteration + return + try: + n = sock.sendto(data, 0, address) + except (BlockingIOError, InterruptedError): + return + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + fut.set_exception(exc) + else: + fut.set_result(n) - @coroutine - def sock_connect(self, sock, address): + async def sock_connect(self, sock, address): """Connect to a remote socket at address. This method is a coroutine. """ + base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") - if not hasattr(socket, 'AF_UNIX') or sock.family != socket.AF_UNIX: - resolved = base_events._ensure_resolved( - address, family=sock.family, proto=sock.proto, loop=self) - if not resolved.done(): - yield from resolved - _, _, _, _, address = resolved.result()[0] + if sock.family == socket.AF_INET or ( + base_events._HAS_IPv6 and sock.family == socket.AF_INET6): + resolved = await self._ensure_resolved( + address, family=sock.family, type=sock.type, proto=sock.proto, + loop=self, + ) + _, _, _, _, address = resolved[0] fut = self.create_future() self._sock_connect(fut, sock, address) - return (yield from fut) + try: + return await fut + finally: + # Needed to break cycles when an exception occurs. + fut = None def _sock_connect(self, fut, sock, address): fd = sock.fileno() @@ -459,66 +656,91 @@ def _sock_connect(self, fut, sock, address): # connection runs in background. We have to wait until the socket # becomes writable to be notified when the connection succeed or # fails. + self._ensure_fd_no_transport(fd) + handle = self._add_writer( + fd, self._sock_connect_cb, fut, sock, address) fut.add_done_callback( - functools.partial(self._sock_connect_done, fd)) - self.add_writer(fd, self._sock_connect_cb, fut, sock, address) - except Exception as exc: + functools.partial(self._sock_write_done, fd, handle=handle)) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: fut.set_exception(exc) else: fut.set_result(None) + finally: + fut = None - def _sock_connect_done(self, fd, fut): - self.remove_writer(fd) + def _sock_write_done(self, fd, fut, handle=None): + if handle is None or not handle.cancelled(): + self.remove_writer(fd) def _sock_connect_cb(self, fut, sock, address): - if fut.cancelled(): + if fut.done(): return try: err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: # Jump to any except clause below. - raise OSError(err, 'Connect call failed %s' % (address,)) + raise OSError(err, f'Connect call failed {address}') except (BlockingIOError, InterruptedError): # socket is still registered, the callback will be retried later pass - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: fut.set_exception(exc) else: fut.set_result(None) + finally: + fut = None - def sock_accept(self, sock): + async def sock_accept(self, sock): """Accept a connection. The socket must be bound to an address and listening for connections. The return value is a pair (conn, address) where conn is a new socket object usable to send and receive data on the connection, and address is the address bound to the socket on the other end of the connection. - - This method is a coroutine. """ + base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() - self._sock_accept(fut, False, sock) - return fut + self._sock_accept(fut, sock) + return await fut - def _sock_accept(self, fut, registered, sock): + def _sock_accept(self, fut, sock): fd = sock.fileno() - if registered: - self.remove_reader(fd) - if fut.cancelled(): - return try: conn, address = sock.accept() conn.setblocking(False) except (BlockingIOError, InterruptedError): - self.add_reader(fd, self._sock_accept, fut, True, sock) - except Exception as exc: + self._ensure_fd_no_transport(fd) + handle = self._add_reader(fd, self._sock_accept, fut, sock) + fut.add_done_callback( + functools.partial(self._sock_read_done, fd, handle=handle)) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: fut.set_exception(exc) else: fut.set_result((conn, address)) + async def _sendfile_native(self, transp, file, offset, count): + del self._transports[transp._sock_fd] + resume_reading = transp.is_reading() + transp.pause_reading() + await transp._make_empty_waiter() + try: + return await self.sock_sendfile(transp._sock, file, offset, count, + fallback=False) + finally: + transp._reset_empty_waiter() + if resume_reading: + transp.resume_reading() + self._transports[transp._sock_fd] = transp + def _process_events(self, event_list): for key, mask in event_list: fileobj, (reader, writer) = key.fileobj, key.data @@ -543,8 +765,6 @@ class _SelectorTransport(transports._FlowControlMixin, max_size = 256 * 1024 # Buffer size passed to recv(). - _buffer_factory = bytearray # Constructs initial value for self._buffer. - # Attribute used in the destructor: it must be set even if the constructor # is not called (see _SelectorSslTransport which may start by raising an # exception) @@ -552,8 +772,11 @@ class _SelectorTransport(transports._FlowControlMixin, def __init__(self, loop, sock, protocol, extra=None, server=None): super().__init__(extra, loop) - self._extra['socket'] = sock - self._extra['sockname'] = sock.getsockname() + self._extra['socket'] = trsock.TransportSocket(sock) + try: + self._extra['sockname'] = sock.getsockname() + except OSError: + self._extra['sockname'] = None if 'peername' not in self._extra: try: self._extra['peername'] = sock.getpeername() @@ -561,14 +784,18 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._extra['peername'] = None self._sock = sock self._sock_fd = sock.fileno() - self._protocol = protocol - self._protocol_connected = True + + self._protocol_connected = False + self.set_protocol(protocol) + self._server = server - self._buffer = self._buffer_factory() + self._buffer = collections.deque() self._conn_lost = 0 # Set when call to connection_lost scheduled. self._closing = False # Set when close() called. + self._paused = False # Set when pause_reading() called + if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -577,7 +804,7 @@ def __repr__(self): info.append('closed') elif self._closing: info.append('closing') - info.append('fd=%s' % self._sock_fd) + info.append(f'fd={self._sock_fd}') # test if the transport was closed if self._loop is not None and not self._loop.is_closed(): polling = _test_selector_event(self._loop._selector, @@ -596,14 +823,15 @@ def __repr__(self): state = 'idle' bufsize = self.get_write_buffer_size() - info.append('write=<%s, bufsize=%s>' % (state, bufsize)) - return '<%s>' % ' '.join(info) + info.append(f'write=<{state}, bufsize={bufsize}>') + return '<{}>'.format(' '.join(info)) def abort(self): self._force_close(None) def set_protocol(self, protocol): self._protocol = protocol + self._protocol_connected = True def get_protocol(self): return self._protocol @@ -611,6 +839,25 @@ def get_protocol(self): def is_closing(self): return self._closing + def is_reading(self): + return not self.is_closing() and not self._paused + + def pause_reading(self): + if not self.is_reading(): + return + self._paused = True + self._loop._remove_reader(self._sock_fd) + if self._loop.get_debug(): + logger.debug("%r pauses reading", self) + + def resume_reading(self): + if self._closing or not self._paused: + return + self._paused = False + self._add_reader(self._sock_fd, self._read_ready) + if self._loop.get_debug(): + logger.debug("%r resumes reading", self) + def close(self): if self._closing: return @@ -621,19 +868,16 @@ def close(self): self._loop._remove_writer(self._sock_fd) self._loop.call_soon(self._call_connection_lost, None) - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning, - source=self) - self._sock.close() + def __del__(self, _warn=warnings.warn): + if self._sock is not None: + _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) + self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if isinstance(exc, OSError): if self._loop.get_debug(): logger.debug("%r: %s", self, message, exc_info=True) else: @@ -668,85 +912,150 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): - return len(self._buffer) + return sum(map(len, self._buffer)) + + def _add_reader(self, fd, callback, *args): + if not self.is_reading(): + return + self._loop._add_reader(fd, callback, *args) class _SelectorSocketTransport(_SelectorTransport): + _start_tls_compatible = True + _sendfile_compatible = constants._SendfileMode.TRY_NATIVE + def __init__(self, loop, sock, protocol, waiter=None, extra=None, server=None): + + self._read_ready_cb = None super().__init__(loop, sock, protocol, extra, server) self._eof = False - self._paused = False - + self._empty_waiter = None + if _HAS_SENDMSG: + self._write_ready = self._write_sendmsg + else: + self._write_ready = self._write_send # Disable the Nagle algorithm -- small writes will be # sent without waiting for the TCP ACK. This generally # decreases the latency (in some cases significantly.) - _set_nodelay(self._sock) + base_events._set_nodelay(self._sock) self._loop.call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._loop._add_reader, + self._loop.call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called self._loop.call_soon(futures._set_result_unless_cancelled, waiter, None) - def pause_reading(self): - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') - self._paused = True - self._loop._remove_reader(self._sock_fd) - if self._loop.get_debug(): - logger.debug("%r pauses reading", self) + def set_protocol(self, protocol): + if isinstance(protocol, protocols.BufferedProtocol): + self._read_ready_cb = self._read_ready__get_buffer + else: + self._read_ready_cb = self._read_ready__data_received - def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: - return - self._loop._add_reader(self._sock_fd, self._read_ready) - if self._loop.get_debug(): - logger.debug("%r resumes reading", self) + super().set_protocol(protocol) def _read_ready(self): + self._read_ready_cb() + + def _read_ready__get_buffer(self): + if self._conn_lost: + return + + try: + buf = self._protocol.get_buffer(-1) + if not len(buf): + raise RuntimeError('get_buffer() returned an empty buffer') + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error( + exc, 'Fatal error: protocol.get_buffer() call failed.') + return + + try: + nbytes = self._sock.recv_into(buf) + except (BlockingIOError, InterruptedError): + return + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error(exc, 'Fatal read error on socket transport') + return + + if not nbytes: + self._read_ready__on_eof() + return + + try: + self._protocol.buffer_updated(nbytes) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error( + exc, 'Fatal error: protocol.buffer_updated() call failed.') + + def _read_ready__data_received(self): if self._conn_lost: return try: data = self._sock.recv(self.max_size) except (BlockingIOError, InterruptedError): - pass - except Exception as exc: + return + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self._fatal_error(exc, 'Fatal read error on socket transport') + return + + if not data: + self._read_ready__on_eof() + return + + try: + self._protocol.data_received(data) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error( + exc, 'Fatal error: protocol.data_received() call failed.') + + def _read_ready__on_eof(self): + if self._loop.get_debug(): + logger.debug("%r received EOF", self) + + try: + keep_open = self._protocol.eof_received() + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error( + exc, 'Fatal error: protocol.eof_received() call failed.') + return + + if keep_open: + # We're keeping the connection open so the + # protocol can write more, but we still can't + # receive more, so remove the reader callback. + self._loop._remove_reader(self._sock_fd) else: - if data: - self._protocol.data_received(data) - else: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if keep_open: - # We're keeping the connection open so the - # protocol can write more, but we still can't - # receive more, so remove the reader callback. - self._loop._remove_reader(self._sock_fd) - else: - self.close() + self.close() def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be a bytes-like object, ' - 'not %r' % type(data).__name__) + raise TypeError(f'data argument must be a bytes, bytearray, or memoryview ' + f'object, not {type(data).__name__!r}') if self._eof: raise RuntimeError('Cannot call write() after write_eof()') + if self._empty_waiter is not None: + raise RuntimeError('unable to write; sendfile is in progress') if not data: return @@ -762,268 +1071,108 @@ def write(self, data): n = self._sock.send(data) except (BlockingIOError, InterruptedError): pass - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self._fatal_error(exc, 'Fatal write error on socket transport') return else: - data = data[n:] + data = memoryview(data)[n:] if not data: return # Not all was written; register write handler. self._loop._add_writer(self._sock_fd, self._write_ready) # Add it to the buffer. - self._buffer.extend(data) + self._buffer.append(data) self._maybe_pause_protocol() - def _write_ready(self): - assert self._buffer, 'Data should not be empty' + def _get_sendmsg_buffer(self): + return itertools.islice(self._buffer, SC_IOV_MAX) + def _write_sendmsg(self): + assert self._buffer, 'Data should not be empty' if self._conn_lost: return try: - n = self._sock.send(self._buffer) + nbytes = self._sock.sendmsg(self._get_sendmsg_buffer()) + self._adjust_leftover_buffer(nbytes) except (BlockingIOError, InterruptedError): pass - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() self._fatal_error(exc, 'Fatal write error on socket transport') + if self._empty_waiter is not None: + self._empty_waiter.set_exception(exc) else: - if n: - del self._buffer[:n] self._maybe_resume_protocol() # May append to buffer. if not self._buffer: self._loop._remove_writer(self._sock_fd) + if self._empty_waiter is not None: + self._empty_waiter.set_result(None) if self._closing: self._call_connection_lost(None) elif self._eof: self._sock.shutdown(socket.SHUT_WR) - def write_eof(self): - if self._eof: - return - self._eof = True - if not self._buffer: - self._sock.shutdown(socket.SHUT_WR) - - def can_write_eof(self): - return True - - -class _SelectorSslTransport(_SelectorTransport): - - _buffer_factory = bytearray - - def __init__(self, loop, rawsock, protocol, sslcontext, waiter=None, - server_side=False, server_hostname=None, - extra=None, server=None): - if ssl is None: - raise RuntimeError('stdlib ssl module not available') - - if not sslcontext: - sslcontext = sslproto._create_transport_context(server_side, server_hostname) - - wrap_kwargs = { - 'server_side': server_side, - 'do_handshake_on_connect': False, - } - if server_hostname and not server_side: - wrap_kwargs['server_hostname'] = server_hostname - sslsock = sslcontext.wrap_socket(rawsock, **wrap_kwargs) - - super().__init__(loop, sslsock, protocol, extra, server) - # the protocol connection is only made after the SSL handshake - self._protocol_connected = False - - self._server_hostname = server_hostname - self._waiter = waiter - self._sslcontext = sslcontext - self._paused = False - - # SSL-specific extra info. (peercert is set later) - self._extra.update(sslcontext=sslcontext) - - if self._loop.get_debug(): - logger.debug("%r starts SSL handshake", self) - start_time = self._loop.time() - else: - start_time = None - self._on_handshake(start_time) - - def _wakeup_waiter(self, exc=None): - if self._waiter is None: - return - if not self._waiter.cancelled(): - if exc is not None: - self._waiter.set_exception(exc) + def _adjust_leftover_buffer(self, nbytes: int) -> None: + buffer = self._buffer + while nbytes: + b = buffer.popleft() + b_len = len(b) + if b_len <= nbytes: + nbytes -= b_len else: - self._waiter.set_result(None) - self._waiter = None - - def _on_handshake(self, start_time): - try: - self._sock.do_handshake() - except ssl.SSLWantReadError: - self._loop._add_reader(self._sock_fd, - self._on_handshake, start_time) - return - except ssl.SSLWantWriteError: - self._loop._add_writer(self._sock_fd, - self._on_handshake, start_time) - return - except BaseException as exc: - if self._loop.get_debug(): - logger.warning("%r: SSL handshake failed", - self, exc_info=True) - self._loop._remove_reader(self._sock_fd) - self._loop._remove_writer(self._sock_fd) - self._sock.close() - self._wakeup_waiter(exc) - if isinstance(exc, Exception): - return - else: - raise - - self._loop._remove_reader(self._sock_fd) - self._loop._remove_writer(self._sock_fd) - - peercert = self._sock.getpeercert() - if not hasattr(self._sslcontext, 'check_hostname'): - # Verify hostname if requested, Python 3.4+ uses check_hostname - # and checks the hostname in do_handshake() - if (self._server_hostname and - self._sslcontext.verify_mode != ssl.CERT_NONE): - try: - ssl.match_hostname(peercert, self._server_hostname) - except Exception as exc: - if self._loop.get_debug(): - logger.warning("%r: SSL handshake failed " - "on matching the hostname", - self, exc_info=True) - self._sock.close() - self._wakeup_waiter(exc) - return - - # Add extra info that becomes available after handshake. - self._extra.update(peercert=peercert, - cipher=self._sock.cipher(), - compression=self._sock.compression(), - ssl_object=self._sock, - ) - - self._read_wants_write = False - self._write_wants_read = False - self._loop._add_reader(self._sock_fd, self._read_ready) - self._protocol_connected = True - self._loop.call_soon(self._protocol.connection_made, self) - # only wake up the waiter when connection_made() has been called - self._loop.call_soon(self._wakeup_waiter) - - if self._loop.get_debug(): - dt = self._loop.time() - start_time - logger.debug("%r: SSL handshake took %.1f ms", self, dt * 1e3) - - def pause_reading(self): - # XXX This is a bit icky, given the comment at the top of - # _read_ready(). Is it possible to evoke a deadlock? I don't - # know, although it doesn't look like it; write() will still - # accept more data for the buffer and eventually the app will - # call resume_reading() again, and things will flow again. - - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') - self._paused = True - self._loop._remove_reader(self._sock_fd) - if self._loop.get_debug(): - logger.debug("%r pauses reading", self) - - def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: - return - self._loop._add_reader(self._sock_fd, self._read_ready) - if self._loop.get_debug(): - logger.debug("%r resumes reading", self) + buffer.appendleft(b[nbytes:]) + break - def _read_ready(self): + def _write_send(self): + assert self._buffer, 'Data should not be empty' if self._conn_lost: return - if self._write_wants_read: - self._write_wants_read = False - self._write_ready() - - if self._buffer: - self._loop._add_writer(self._sock_fd, self._write_ready) - try: - data = self._sock.recv(self.max_size) - except (BlockingIOError, InterruptedError, ssl.SSLWantReadError): + buffer = self._buffer.popleft() + n = self._sock.send(buffer) + if n != len(buffer): + # Not all data was written + self._buffer.appendleft(buffer[n:]) + except (BlockingIOError, InterruptedError): pass - except ssl.SSLWantWriteError: - self._read_wants_write = True - self._loop._remove_reader(self._sock_fd) - self._loop._add_writer(self._sock_fd, self._write_ready) - except Exception as exc: - self._fatal_error(exc, 'Fatal read error on SSL transport') + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._loop._remove_writer(self._sock_fd) + self._buffer.clear() + self._fatal_error(exc, 'Fatal write error on socket transport') + if self._empty_waiter is not None: + self._empty_waiter.set_exception(exc) else: - if data: - self._protocol.data_received(data) - else: - try: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if keep_open: - logger.warning('returning true from eof_received() ' - 'has no effect when using ssl') - finally: - self.close() - - def _write_ready(self): - if self._conn_lost: - return - if self._read_wants_write: - self._read_wants_write = False - self._read_ready() - - if not (self._paused or self._closing): - self._loop._add_reader(self._sock_fd, self._read_ready) - - if self._buffer: - try: - n = self._sock.send(self._buffer) - except (BlockingIOError, InterruptedError, ssl.SSLWantWriteError): - n = 0 - except ssl.SSLWantReadError: - n = 0 - self._loop._remove_writer(self._sock_fd) - self._write_wants_read = True - except Exception as exc: + self._maybe_resume_protocol() # May append to buffer. + if not self._buffer: self._loop._remove_writer(self._sock_fd) - self._buffer.clear() - self._fatal_error(exc, 'Fatal write error on SSL transport') - return - - if n: - del self._buffer[:n] - - self._maybe_resume_protocol() # May append to buffer. + if self._empty_waiter is not None: + self._empty_waiter.set_result(None) + if self._closing: + self._call_connection_lost(None) + elif self._eof: + self._sock.shutdown(socket.SHUT_WR) + def write_eof(self): + if self._closing or self._eof: + return + self._eof = True if not self._buffer: - self._loop._remove_writer(self._sock_fd) - if self._closing: - self._call_connection_lost(None) + self._sock.shutdown(socket.SHUT_WR) - def write(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be a bytes-like object, ' - 'not %r' % type(data).__name__) - if not data: + def writelines(self, list_of_data): + if self._eof: + raise RuntimeError('Cannot call writelines() after write_eof()') + if self._empty_waiter is not None: + raise RuntimeError('unable to writelines; sendfile is in progress') + if not list_of_data: return if self._conn_lost: @@ -1032,28 +1181,54 @@ def write(self, data): self._conn_lost += 1 return - if not self._buffer: + self._buffer.extend([memoryview(data) for data in list_of_data]) + self._write_ready() + # If the entire buffer couldn't be written, register a write handler + if self._buffer: self._loop._add_writer(self._sock_fd, self._write_ready) - - # Add it to the buffer. - self._buffer.extend(data) - self._maybe_pause_protocol() + self._maybe_pause_protocol() def can_write_eof(self): - return False + return True + + def _call_connection_lost(self, exc): + try: + super()._call_connection_lost(exc) + finally: + self._write_ready = None + if self._empty_waiter is not None: + self._empty_waiter.set_exception( + ConnectionError("Connection is closed by peer")) + + def _make_empty_waiter(self): + if self._empty_waiter is not None: + raise RuntimeError("Empty waiter is already set") + self._empty_waiter = self._loop.create_future() + if not self._buffer: + self._empty_waiter.set_result(None) + return self._empty_waiter + + def _reset_empty_waiter(self): + self._empty_waiter = None + + def close(self): + self._read_ready_cb = None + super().close() -class _SelectorDatagramTransport(_SelectorTransport): +class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): _buffer_factory = collections.deque + _header_size = 8 def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): super().__init__(loop, sock, protocol, extra) self._address = address + self._buffer_size = 0 self._loop.call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._loop._add_reader, + self._loop.call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -1061,7 +1236,7 @@ def __init__(self, loop, sock, protocol, address=None, waiter, None) def get_write_buffer_size(self): - return sum(len(data) for data, _ in self._buffer) + return self._buffer_size def _read_ready(self): if self._conn_lost: @@ -1072,21 +1247,23 @@ def _read_ready(self): pass except OSError as exc: self._protocol.error_received(exc) - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self._fatal_error(exc, 'Fatal read error on datagram transport') else: self._protocol.datagram_received(data, addr) def sendto(self, data, addr=None): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be a bytes-like object, ' - 'not %r' % type(data).__name__) - if not data: - return + raise TypeError(f'data argument must be a bytes-like object, ' + f'not {type(data).__name__!r}') - if self._address and addr not in (None, self._address): - raise ValueError('Invalid address: must be None or %s' % - (self._address,)) + if self._address: + if addr not in (None, self._address): + raise ValueError( + f'Invalid address: must be None or {self._address}') + addr = self._address if self._conn_lost and self._address: if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: @@ -1097,7 +1274,7 @@ def sendto(self, data, addr=None): if not self._buffer: # Attempt to send it right away first. try: - if self._address: + if self._extra['peername']: self._sock.send(data) else: self._sock.sendto(data, addr) @@ -1107,32 +1284,39 @@ def sendto(self, data, addr=None): except OSError as exc: self._protocol.error_received(exc) return - except Exception as exc: - self._fatal_error(exc, - 'Fatal write error on datagram transport') + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error( + exc, 'Fatal write error on datagram transport') return # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) + self._buffer_size += len(data) + self._header_size self._maybe_pause_protocol() def _sendto_ready(self): while self._buffer: data, addr = self._buffer.popleft() + self._buffer_size -= len(data) + self._header_size try: - if self._address: + if self._extra['peername']: self._sock.send(data) else: self._sock.sendto(data, addr) except (BlockingIOError, InterruptedError): self._buffer.appendleft((data, addr)) # Try again later. + self._buffer_size += len(data) + self._header_size break except OSError as exc: self._protocol.error_received(exc) return - except Exception as exc: - self._fatal_error(exc, - 'Fatal write error on datagram transport') + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._fatal_error( + exc, 'Fatal write error on datagram transport') return self._maybe_resume_protocol() # May append to buffer. diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 7ad28d6aa00..74c5f0d5ca0 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -1,16 +1,48 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + import collections +import enum import warnings try: import ssl except ImportError: # pragma: no cover ssl = None -from . import base_events -from . import compat +from . import constants +from . import exceptions from . import protocols from . import transports from .log import logger +if ssl is not None: + SSLAgainErrors = (ssl.SSLWantReadError, ssl.SSLSyscallError) + + +class SSLProtocolState(enum.Enum): + UNWRAPPED = "UNWRAPPED" + DO_HANDSHAKE = "DO_HANDSHAKE" + WRAPPED = "WRAPPED" + FLUSHING = "FLUSHING" + SHUTDOWN = "SHUTDOWN" + + +class AppProtocolState(enum.Enum): + # This tracks the state of app protocol (https://git.io/fj59P): + # + # INIT -cm-> CON_MADE [-dr*->] [-er-> EOF?] -cl-> CON_LOST + # + # * cm: connection_made() + # * dr: data_received() + # * er: eof_received() + # * cl: connection_lost() + + STATE_INIT = "STATE_INIT" + STATE_CON_MADE = "STATE_CON_MADE" + STATE_EOF = "STATE_EOF" + STATE_CON_LOST = "STATE_CON_LOST" + def _create_transport_context(server_side, server_hostname): if server_side: @@ -19,286 +51,43 @@ def _create_transport_context(server_side, server_hostname): # Client side may pass ssl=True to use a default # context; in that case the sslcontext passed is None. # The default is secure for client connections. - if hasattr(ssl, 'create_default_context'): - # Python 3.4+: use up-to-date strong settings. - sslcontext = ssl.create_default_context() - if not server_hostname: - sslcontext.check_hostname = False - else: - # Fallback for Python 3.3. - sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - sslcontext.options |= ssl.OP_NO_SSLv2 - sslcontext.options |= ssl.OP_NO_SSLv3 - sslcontext.set_default_verify_paths() - sslcontext.verify_mode = ssl.CERT_REQUIRED + # Python 3.4+: use up-to-date strong settings. + sslcontext = ssl.create_default_context() + if not server_hostname: + sslcontext.check_hostname = False return sslcontext -def _is_sslproto_available(): - return hasattr(ssl, "MemoryBIO") - - -# States of an _SSLPipe. -_UNWRAPPED = "UNWRAPPED" -_DO_HANDSHAKE = "DO_HANDSHAKE" -_WRAPPED = "WRAPPED" -_SHUTDOWN = "SHUTDOWN" - - -class _SSLPipe(object): - """An SSL "Pipe". - - An SSL pipe allows you to communicate with an SSL/TLS protocol instance - through memory buffers. It can be used to implement a security layer for an - existing connection where you don't have access to the connection's file - descriptor, or for some reason you don't want to use it. - - An SSL pipe can be in "wrapped" and "unwrapped" mode. In unwrapped mode, - data is passed through untransformed. In wrapped mode, application level - data is encrypted to SSL record level data and vice versa. The SSL record - level is the lowest level in the SSL protocol suite and is what travels - as-is over the wire. - - An SslPipe initially is in "unwrapped" mode. To start SSL, call - do_handshake(). To shutdown SSL again, call unwrap(). - """ - - max_size = 256 * 1024 # Buffer size passed to read() - - def __init__(self, context, server_side, server_hostname=None): - """ - The *context* argument specifies the ssl.SSLContext to use. - - The *server_side* argument indicates whether this is a server side or - client side transport. - - The optional *server_hostname* argument can be used to specify the - hostname you are connecting to. You may only specify this parameter if - the _ssl module supports Server Name Indication (SNI). - """ - self._context = context - self._server_side = server_side - self._server_hostname = server_hostname - self._state = _UNWRAPPED - self._incoming = ssl.MemoryBIO() - self._outgoing = ssl.MemoryBIO() - self._sslobj = None - self._need_ssldata = False - self._handshake_cb = None - self._shutdown_cb = None - - @property - def context(self): - """The SSL context passed to the constructor.""" - return self._context - - @property - def ssl_object(self): - """The internal ssl.SSLObject instance. - - Return None if the pipe is not wrapped. - """ - return self._sslobj - - @property - def need_ssldata(self): - """Whether more record level data is needed to complete a handshake - that is currently in progress.""" - return self._need_ssldata - - @property - def wrapped(self): - """ - Whether a security layer is currently in effect. - - Return False during handshake. - """ - return self._state == _WRAPPED - - def do_handshake(self, callback=None): - """Start the SSL handshake. - - Return a list of ssldata. A ssldata element is a list of buffers - - The optional *callback* argument can be used to install a callback that - will be called when the handshake is complete. The callback will be - called with None if successful, else an exception instance. - """ - if self._state != _UNWRAPPED: - raise RuntimeError('handshake in progress or completed') - self._sslobj = self._context.wrap_bio( - self._incoming, self._outgoing, - server_side=self._server_side, - server_hostname=self._server_hostname) - self._state = _DO_HANDSHAKE - self._handshake_cb = callback - ssldata, appdata = self.feed_ssldata(b'', only_handshake=True) - assert len(appdata) == 0 - return ssldata - - def shutdown(self, callback=None): - """Start the SSL shutdown sequence. - - Return a list of ssldata. A ssldata element is a list of buffers - - The optional *callback* argument can be used to install a callback that - will be called when the shutdown is complete. The callback will be - called without arguments. - """ - if self._state == _UNWRAPPED: - raise RuntimeError('no security layer present') - if self._state == _SHUTDOWN: - raise RuntimeError('shutdown in progress') - assert self._state in (_WRAPPED, _DO_HANDSHAKE) - self._state = _SHUTDOWN - self._shutdown_cb = callback - ssldata, appdata = self.feed_ssldata(b'') - assert appdata == [] or appdata == [b''] - return ssldata - - def feed_eof(self): - """Send a potentially "ragged" EOF. - - This method will raise an SSL_ERROR_EOF exception if the EOF is - unexpected. - """ - self._incoming.write_eof() - ssldata, appdata = self.feed_ssldata(b'') - assert appdata == [] or appdata == [b''] - - def feed_ssldata(self, data, only_handshake=False): - """Feed SSL record level data into the pipe. - - The data must be a bytes instance. It is OK to send an empty bytes - instance. This can be used to get ssldata for a handshake initiated by - this endpoint. - - Return a (ssldata, appdata) tuple. The ssldata element is a list of - buffers containing SSL data that needs to be sent to the remote SSL. - - The appdata element is a list of buffers containing plaintext data that - needs to be forwarded to the application. The appdata list may contain - an empty buffer indicating an SSL "close_notify" alert. This alert must - be acknowledged by calling shutdown(). - """ - if self._state == _UNWRAPPED: - # If unwrapped, pass plaintext data straight through. - if data: - appdata = [data] - else: - appdata = [] - return ([], appdata) - - self._need_ssldata = False - if data: - self._incoming.write(data) - - ssldata = [] - appdata = [] - try: - if self._state == _DO_HANDSHAKE: - # Call do_handshake() until it doesn't raise anymore. - self._sslobj.do_handshake() - self._state = _WRAPPED - if self._handshake_cb: - self._handshake_cb(None) - if only_handshake: - return (ssldata, appdata) - # Handshake done: execute the wrapped block - - if self._state == _WRAPPED: - # Main state: read data from SSL until close_notify - while True: - chunk = self._sslobj.read(self.max_size) - appdata.append(chunk) - if not chunk: # close_notify - break +def add_flowcontrol_defaults(high, low, kb): + if high is None: + if low is None: + hi = kb * 1024 + else: + lo = low + hi = 4 * lo + else: + hi = high + if low is None: + lo = hi // 4 + else: + lo = low - elif self._state == _SHUTDOWN: - # Call shutdown() until it doesn't raise anymore. - self._sslobj.unwrap() - self._sslobj = None - self._state = _UNWRAPPED - if self._shutdown_cb: - self._shutdown_cb() - - elif self._state == _UNWRAPPED: - # Drain possible plaintext data after close_notify. - appdata.append(self._incoming.read()) - except (ssl.SSLError, ssl.CertificateError) as exc: - if getattr(exc, 'errno', None) not in ( - ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE, - ssl.SSL_ERROR_SYSCALL): - if self._state == _DO_HANDSHAKE and self._handshake_cb: - self._handshake_cb(exc) - raise - self._need_ssldata = (exc.errno == ssl.SSL_ERROR_WANT_READ) - - # Check for record level data that needs to be sent back. - # Happens for the initial handshake and renegotiations. - if self._outgoing.pending: - ssldata.append(self._outgoing.read()) - return (ssldata, appdata) - - def feed_appdata(self, data, offset=0): - """Feed plaintext data into the pipe. - - Return an (ssldata, offset) tuple. The ssldata element is a list of - buffers containing record level data that needs to be sent to the - remote SSL instance. The offset is the number of plaintext bytes that - were processed, which may be less than the length of data. - - NOTE: In case of short writes, this call MUST be retried with the SAME - buffer passed into the *data* argument (i.e. the id() must be the - same). This is an OpenSSL requirement. A further particularity is that - a short write will always have offset == 0, because the _ssl module - does not enable partial writes. And even though the offset is zero, - there will still be encrypted data in ssldata. - """ - assert 0 <= offset <= len(data) - if self._state == _UNWRAPPED: - # pass through data in unwrapped mode - if offset < len(data): - ssldata = [data[offset:]] - else: - ssldata = [] - return (ssldata, len(data)) + if not hi >= lo >= 0: + raise ValueError('high (%r) must be >= low (%r) must be >= 0' % + (hi, lo)) - ssldata = [] - view = memoryview(data) - while True: - self._need_ssldata = False - try: - if offset < len(view): - offset += self._sslobj.write(view[offset:]) - except ssl.SSLError as exc: - # It is not allowed to call write() after unwrap() until the - # close_notify is acknowledged. We return the condition to the - # caller as a short write. - if exc.reason == 'PROTOCOL_IS_SHUTDOWN': - exc.errno = ssl.SSL_ERROR_WANT_READ - if exc.errno not in (ssl.SSL_ERROR_WANT_READ, - ssl.SSL_ERROR_WANT_WRITE, - ssl.SSL_ERROR_SYSCALL): - raise - self._need_ssldata = (exc.errno == ssl.SSL_ERROR_WANT_READ) - - # See if there's any record level data back for us. - if self._outgoing.pending: - ssldata.append(self._outgoing.read()) - if offset == len(view) or self._need_ssldata: - break - return (ssldata, offset) + return hi, lo class _SSLProtocolTransport(transports._FlowControlMixin, transports.Transport): - def __init__(self, loop, ssl_protocol, app_protocol): + _start_tls_compatible = True + _sendfile_compatible = constants._SendfileMode.FALLBACK + + def __init__(self, loop, ssl_protocol): self._loop = loop - # SSLProtocol instance self._ssl_protocol = ssl_protocol - self._app_protocol = app_protocol self._closed = False def get_extra_info(self, name, default=None): @@ -306,13 +95,13 @@ def get_extra_info(self, name, default=None): return self._ssl_protocol._get_extra_info(name, default) def set_protocol(self, protocol): - self._app_protocol = protocol + self._ssl_protocol._set_app_protocol(protocol) def get_protocol(self): - return self._app_protocol + return self._ssl_protocol._app_protocol def is_closing(self): - return self._closed + return self._closed or self._ssl_protocol._is_transport_closing() def close(self): """Close the transport. @@ -322,18 +111,21 @@ def close(self): protocol's connection_lost() method will (eventually) called with None as its argument. """ - self._closed = True - self._ssl_protocol._start_shutdown() - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning, - source=self) - self.close() + if not self._closed: + self._closed = True + self._ssl_protocol._start_shutdown() + else: + self._ssl_protocol = None + + def __del__(self, _warnings=warnings): + if not self._closed: + self._closed = True + _warnings.warn( + "unclosed transport ", ResourceWarning) + + def is_reading(self): + return not self._ssl_protocol._app_reading_paused def pause_reading(self): """Pause the receiving end. @@ -341,7 +133,7 @@ def pause_reading(self): No data will be passed to the protocol's data_received() method until resume_reading() is called. """ - self._ssl_protocol._transport.pause_reading() + self._ssl_protocol._pause_reading() def resume_reading(self): """Resume the receiving end. @@ -349,7 +141,7 @@ def resume_reading(self): Data received will once again be passed to the protocol's data_received() method. """ - self._ssl_protocol._transport.resume_reading() + self._ssl_protocol._resume_reading() def set_write_buffer_limits(self, high=None, low=None): """Set the high- and low-water limits for write flow control. @@ -370,11 +162,51 @@ def set_write_buffer_limits(self, high=None, low=None): reduces opportunities for doing I/O and computation concurrently. """ - self._ssl_protocol._transport.set_write_buffer_limits(high, low) + self._ssl_protocol._set_write_buffer_limits(high, low) + self._ssl_protocol._control_app_writing() + + def get_write_buffer_limits(self): + return (self._ssl_protocol._outgoing_low_water, + self._ssl_protocol._outgoing_high_water) def get_write_buffer_size(self): - """Return the current size of the write buffer.""" - return self._ssl_protocol._transport.get_write_buffer_size() + """Return the current size of the write buffers.""" + return self._ssl_protocol._get_write_buffer_size() + + def set_read_buffer_limits(self, high=None, low=None): + """Set the high- and low-water limits for read flow control. + + These two values control when to call the upstream transport's + pause_reading() and resume_reading() methods. If specified, + the low-water limit must be less than or equal to the + high-water limit. Neither value can be negative. + + The defaults are implementation-specific. If only the + high-water limit is given, the low-water limit defaults to an + implementation-specific value less than or equal to the + high-water limit. Setting high to zero forces low to zero as + well, and causes pause_reading() to be called whenever the + buffer becomes non-empty. Setting low to zero causes + resume_reading() to be called only once the buffer is empty. + Use of zero for either limit is generally sub-optimal as it + reduces opportunities for doing I/O and computation + concurrently. + """ + self._ssl_protocol._set_read_buffer_limits(high, low) + self._ssl_protocol._control_ssl_reading() + + def get_read_buffer_limits(self): + return (self._ssl_protocol._incoming_low_water, + self._ssl_protocol._incoming_high_water) + + def get_read_buffer_size(self): + """Return the current size of the read buffer.""" + return self._ssl_protocol._get_read_buffer_size() + + @property + def _protocol_paused(self): + # Required for sendfile fallback pause_writing/resume_writing logic + return self._ssl_protocol._app_writing_paused def write(self, data): """Write some data bytes to the transport. @@ -383,11 +215,26 @@ def write(self, data): to be sent out asynchronously. """ if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError("data: expecting a bytes-like instance, got {!r}" - .format(type(data).__name__)) + raise TypeError(f"data: expecting a bytes-like instance, " + f"got {type(data).__name__}") if not data: return - self._ssl_protocol._write_appdata(data) + self._ssl_protocol._write_appdata((data,)) + + def writelines(self, list_of_data): + """Write a list (or any iterable) of data bytes to the transport. + + The default implementation concatenates the arguments and + calls write() on the result. + """ + self._ssl_protocol._write_appdata(list_of_data) + + def write_eof(self): + """Close the write end after flushing buffered data. + + This raises :exc:`NotImplementedError` right now. + """ + raise NotImplementedError def can_write_eof(self): """Return True if this transport supports write_eof(), False if not.""" @@ -400,24 +247,53 @@ def abort(self): The protocol's connection_lost() method will (eventually) be called with None as its argument. """ - self._ssl_protocol._abort() + self._force_close(None) + def _force_close(self, exc): + self._closed = True + if self._ssl_protocol is not None: + self._ssl_protocol._abort(exc) + + def _test__append_write_backlog(self, data): + # for test only + self._ssl_protocol._write_backlog.append(data) + self._ssl_protocol._write_buffer_size += len(data) -class SSLProtocol(protocols.Protocol): - """SSL protocol. - Implementation of SSL on top of a socket using incoming and outgoing - buffers which are ssl.MemoryBIO objects. - """ +class SSLProtocol(protocols.BufferedProtocol): + max_size = 256 * 1024 # Buffer size passed to read() + + _handshake_start_time = None + _handshake_timeout_handle = None + _shutdown_timeout_handle = None def __init__(self, loop, app_protocol, sslcontext, waiter, server_side=False, server_hostname=None, - call_connection_made=True): + call_connection_made=True, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): if ssl is None: - raise RuntimeError('stdlib ssl module not available') + raise RuntimeError("stdlib ssl module not available") + + self._ssl_buffer = bytearray(self.max_size) + self._ssl_buffer_view = memoryview(self._ssl_buffer) + + if ssl_handshake_timeout is None: + ssl_handshake_timeout = constants.SSL_HANDSHAKE_TIMEOUT + elif ssl_handshake_timeout <= 0: + raise ValueError( + f"ssl_handshake_timeout should be a positive number, " + f"got {ssl_handshake_timeout}") + if ssl_shutdown_timeout is None: + ssl_shutdown_timeout = constants.SSL_SHUTDOWN_TIMEOUT + elif ssl_shutdown_timeout <= 0: + raise ValueError( + f"ssl_shutdown_timeout should be a positive number, " + f"got {ssl_shutdown_timeout}") if not sslcontext: - sslcontext = _create_transport_context(server_side, server_hostname) + sslcontext = _create_transport_context( + server_side, server_hostname) self._server_side = server_side if server_hostname and not server_side: @@ -435,17 +311,55 @@ def __init__(self, loop, app_protocol, sslcontext, waiter, self._waiter = waiter self._loop = loop - self._app_protocol = app_protocol - self._app_transport = _SSLProtocolTransport(self._loop, - self, self._app_protocol) - # _SSLPipe instance (None until the connection is made) - self._sslpipe = None - self._session_established = False - self._in_handshake = False - self._in_shutdown = False + self._set_app_protocol(app_protocol) + self._app_transport = None + self._app_transport_created = False # transport, ex: SelectorSocketTransport self._transport = None - self._call_connection_made = call_connection_made + self._ssl_handshake_timeout = ssl_handshake_timeout + self._ssl_shutdown_timeout = ssl_shutdown_timeout + # SSL and state machine + self._incoming = ssl.MemoryBIO() + self._outgoing = ssl.MemoryBIO() + self._state = SSLProtocolState.UNWRAPPED + self._conn_lost = 0 # Set when connection_lost called + if call_connection_made: + self._app_state = AppProtocolState.STATE_INIT + else: + self._app_state = AppProtocolState.STATE_CON_MADE + self._sslobj = self._sslcontext.wrap_bio( + self._incoming, self._outgoing, + server_side=self._server_side, + server_hostname=self._server_hostname) + + # Flow Control + + self._ssl_writing_paused = False + + self._app_reading_paused = False + + self._ssl_reading_paused = False + self._incoming_high_water = 0 + self._incoming_low_water = 0 + self._set_read_buffer_limits() + self._eof_received = False + + self._app_writing_paused = False + self._outgoing_high_water = 0 + self._outgoing_low_water = 0 + self._set_write_buffer_limits() + self._get_app_transport() + + def _set_app_protocol(self, app_protocol): + self._app_protocol = app_protocol + # Make fast hasattr check first + if (hasattr(app_protocol, 'get_buffer') and + isinstance(app_protocol, protocols.BufferedProtocol)): + self._app_protocol_get_buffer = app_protocol.get_buffer + self._app_protocol_buffer_updated = app_protocol.buffer_updated + self._app_protocol_is_buffer = True + else: + self._app_protocol_is_buffer = False def _wakeup_waiter(self, exc=None): if self._waiter is None: @@ -457,15 +371,23 @@ def _wakeup_waiter(self, exc=None): self._waiter.set_result(None) self._waiter = None + def _get_app_transport(self): + if self._app_transport is None: + if self._app_transport_created: + raise RuntimeError('Creating _SSLProtocolTransport twice') + self._app_transport = _SSLProtocolTransport(self._loop, self) + self._app_transport_created = True + return self._app_transport + + def _is_transport_closing(self): + return self._transport is not None and self._transport.is_closing() + def connection_made(self, transport): """Called when the low-level connection is made. Start the SSL handshake. """ self._transport = transport - self._sslpipe = _SSLPipe(self._sslcontext, - self._server_side, - self._server_hostname) self._start_handshake() def connection_lost(self, exc): @@ -475,48 +397,58 @@ def connection_lost(self, exc): meaning a regular EOF is received or the connection was aborted or closed). """ - if self._session_established: - self._session_established = False - self._loop.call_soon(self._app_protocol.connection_lost, exc) + self._write_backlog.clear() + self._outgoing.read() + self._conn_lost += 1 + + # Just mark the app transport as closed so that its __dealloc__ + # doesn't complain. + if self._app_transport is not None: + self._app_transport._closed = True + + if self._state != SSLProtocolState.DO_HANDSHAKE: + if ( + self._app_state == AppProtocolState.STATE_CON_MADE or + self._app_state == AppProtocolState.STATE_EOF + ): + self._app_state = AppProtocolState.STATE_CON_LOST + self._loop.call_soon(self._app_protocol.connection_lost, exc) + self._set_state(SSLProtocolState.UNWRAPPED) self._transport = None self._app_transport = None + self._app_protocol = None self._wakeup_waiter(exc) - def pause_writing(self): - """Called when the low-level transport's buffer goes over - the high-water mark. - """ - self._app_protocol.pause_writing() + if self._shutdown_timeout_handle: + self._shutdown_timeout_handle.cancel() + self._shutdown_timeout_handle = None + if self._handshake_timeout_handle: + self._handshake_timeout_handle.cancel() + self._handshake_timeout_handle = None - def resume_writing(self): - """Called when the low-level transport's buffer drains below - the low-water mark. - """ - self._app_protocol.resume_writing() + def get_buffer(self, n): + want = n + if want <= 0 or want > self.max_size: + want = self.max_size + if len(self._ssl_buffer) < want: + self._ssl_buffer = bytearray(want) + self._ssl_buffer_view = memoryview(self._ssl_buffer) + return self._ssl_buffer_view - def data_received(self, data): - """Called when some SSL data is received. + def buffer_updated(self, nbytes): + self._incoming.write(self._ssl_buffer_view[:nbytes]) - The argument is a bytes object. - """ - try: - ssldata, appdata = self._sslpipe.feed_ssldata(data) - except ssl.SSLError as e: - if self._loop.get_debug(): - logger.warning('%r: SSL error %s (reason %s)', - self, e.errno, e.reason) - self._abort() - return + if self._state == SSLProtocolState.DO_HANDSHAKE: + self._do_handshake() - for chunk in ssldata: - self._transport.write(chunk) + elif self._state == SSLProtocolState.WRAPPED: + self._do_read() - for chunk in appdata: - if chunk: - self._app_protocol.data_received(chunk) - else: - self._start_shutdown() - break + elif self._state == SSLProtocolState.FLUSHING: + self._do_flush() + + elif self._state == SSLProtocolState.SHUTDOWN: + self._do_shutdown() def eof_received(self): """Called when the other end of the low-level stream @@ -526,36 +458,80 @@ def eof_received(self): will close itself. If it returns a true value, closing the transport is up to the protocol. """ + self._eof_received = True try: if self._loop.get_debug(): logger.debug("%r received EOF", self) - self._wakeup_waiter(ConnectionResetError) + if self._state == SSLProtocolState.DO_HANDSHAKE: + self._on_handshake_complete(ConnectionResetError) - if not self._in_handshake: - keep_open = self._app_protocol.eof_received() - if keep_open: - logger.warning('returning true from eof_received() ' - 'has no effect when using ssl') - finally: + elif self._state == SSLProtocolState.WRAPPED: + self._set_state(SSLProtocolState.FLUSHING) + if self._app_reading_paused: + return True + else: + self._do_flush() + + elif self._state == SSLProtocolState.FLUSHING: + self._do_write() + self._set_state(SSLProtocolState.SHUTDOWN) + self._do_shutdown() + + elif self._state == SSLProtocolState.SHUTDOWN: + self._do_shutdown() + + except Exception: self._transport.close() + raise def _get_extra_info(self, name, default=None): if name in self._extra: return self._extra[name] - else: + elif self._transport is not None: return self._transport.get_extra_info(name, default) + else: + return default - def _start_shutdown(self): - if self._in_shutdown: - return - self._in_shutdown = True - self._write_appdata(b'') + def _set_state(self, new_state): + allowed = False + + if new_state == SSLProtocolState.UNWRAPPED: + allowed = True + + elif ( + self._state == SSLProtocolState.UNWRAPPED and + new_state == SSLProtocolState.DO_HANDSHAKE + ): + allowed = True - def _write_appdata(self, data): - self._write_backlog.append((data, 0)) - self._write_buffer_size += len(data) - self._process_write_backlog() + elif ( + self._state == SSLProtocolState.DO_HANDSHAKE and + new_state == SSLProtocolState.WRAPPED + ): + allowed = True + + elif ( + self._state == SSLProtocolState.WRAPPED and + new_state == SSLProtocolState.FLUSHING + ): + allowed = True + + elif ( + self._state == SSLProtocolState.FLUSHING and + new_state == SSLProtocolState.SHUTDOWN + ): + allowed = True + + if allowed: + self._state = new_state + + else: + raise RuntimeError( + 'cannot switch state from {} to {}'.format( + self._state, new_state)) + + # Handshake flow def _start_handshake(self): if self._loop.get_debug(): @@ -563,42 +539,58 @@ def _start_handshake(self): self._handshake_start_time = self._loop.time() else: self._handshake_start_time = None - self._in_handshake = True - # (b'', 1) is a special value in _process_write_backlog() to do - # the SSL handshake - self._write_backlog.append((b'', 1)) - self._loop.call_soon(self._process_write_backlog) + + self._set_state(SSLProtocolState.DO_HANDSHAKE) + + # start handshake timeout count down + self._handshake_timeout_handle = \ + self._loop.call_later(self._ssl_handshake_timeout, + self._check_handshake_timeout) + + self._do_handshake() + + def _check_handshake_timeout(self): + if self._state == SSLProtocolState.DO_HANDSHAKE: + msg = ( + f"SSL handshake is taking longer than " + f"{self._ssl_handshake_timeout} seconds: " + f"aborting the connection" + ) + self._fatal_error(ConnectionAbortedError(msg)) + + def _do_handshake(self): + try: + self._sslobj.do_handshake() + except SSLAgainErrors: + self._process_outgoing() + except ssl.SSLError as exc: + self._on_handshake_complete(exc) + else: + self._on_handshake_complete(None) def _on_handshake_complete(self, handshake_exc): - self._in_handshake = False + if self._handshake_timeout_handle is not None: + self._handshake_timeout_handle.cancel() + self._handshake_timeout_handle = None - sslobj = self._sslpipe.ssl_object + sslobj = self._sslobj try: - if handshake_exc is not None: + if handshake_exc is None: + self._set_state(SSLProtocolState.WRAPPED) + else: raise handshake_exc peercert = sslobj.getpeercert() - if not hasattr(self._sslcontext, 'check_hostname'): - # Verify hostname if requested, Python 3.4+ uses check_hostname - # and checks the hostname in do_handshake() - if (self._server_hostname - and self._sslcontext.verify_mode != ssl.CERT_NONE): - ssl.match_hostname(peercert, self._server_hostname) - except BaseException as exc: - if self._loop.get_debug(): - if isinstance(exc, ssl.CertificateError): - logger.warning("%r: SSL handshake failed " - "on verifying the certificate", - self, exc_info=True) - else: - logger.warning("%r: SSL handshake failed", - self, exc_info=True) - self._transport.close() - if isinstance(exc, Exception): - self._wakeup_waiter(exc) - return + except Exception as exc: + handshake_exc = None + self._set_state(SSLProtocolState.UNWRAPPED) + if isinstance(exc, ssl.CertificateError): + msg = 'SSL handshake failed on verifying the certificate' else: - raise + msg = 'SSL handshake failed' + self._fatal_error(exc, msg) + self._wakeup_waiter(exc) + return if self._loop.get_debug(): dt = self._loop.time() - self._handshake_start_time @@ -608,85 +600,330 @@ def _on_handshake_complete(self, handshake_exc): self._extra.update(peercert=peercert, cipher=sslobj.cipher(), compression=sslobj.compression(), - ssl_object=sslobj, - ) - if self._call_connection_made: - self._app_protocol.connection_made(self._app_transport) + ssl_object=sslobj) + if self._app_state == AppProtocolState.STATE_INIT: + self._app_state = AppProtocolState.STATE_CON_MADE + self._app_protocol.connection_made(self._get_app_transport()) self._wakeup_waiter() - self._session_established = True - # In case transport.write() was already called. Don't call - # immediately _process_write_backlog(), but schedule it: - # _on_handshake_complete() can be called indirectly from - # _process_write_backlog(), and _process_write_backlog() is not - # reentrant. - self._loop.call_soon(self._process_write_backlog) - - def _process_write_backlog(self): - # Try to make progress on the write backlog. - if self._transport is None: + self._do_read() + + # Shutdown flow + + def _start_shutdown(self): + if ( + self._state in ( + SSLProtocolState.FLUSHING, + SSLProtocolState.SHUTDOWN, + SSLProtocolState.UNWRAPPED + ) + ): + return + if self._app_transport is not None: + self._app_transport._closed = True + if self._state == SSLProtocolState.DO_HANDSHAKE: + self._abort(None) + else: + self._set_state(SSLProtocolState.FLUSHING) + self._shutdown_timeout_handle = self._loop.call_later( + self._ssl_shutdown_timeout, + self._check_shutdown_timeout + ) + self._do_flush() + + def _check_shutdown_timeout(self): + if ( + self._state in ( + SSLProtocolState.FLUSHING, + SSLProtocolState.SHUTDOWN + ) + ): + self._transport._force_close( + exceptions.TimeoutError('SSL shutdown timed out')) + + def _do_flush(self): + self._do_read() + self._set_state(SSLProtocolState.SHUTDOWN) + self._do_shutdown() + + def _do_shutdown(self): + try: + if not self._eof_received: + self._sslobj.unwrap() + except SSLAgainErrors: + self._process_outgoing() + except ssl.SSLError as exc: + self._on_shutdown_complete(exc) + else: + self._process_outgoing() + self._call_eof_received() + self._on_shutdown_complete(None) + + def _on_shutdown_complete(self, shutdown_exc): + if self._shutdown_timeout_handle is not None: + self._shutdown_timeout_handle.cancel() + self._shutdown_timeout_handle = None + + if shutdown_exc: + self._fatal_error(shutdown_exc) + else: + self._loop.call_soon(self._transport.close) + + def _abort(self, exc): + self._set_state(SSLProtocolState.UNWRAPPED) + if self._transport is not None: + self._transport._force_close(exc) + + # Outgoing flow + + def _write_appdata(self, list_of_data): + if ( + self._state in ( + SSLProtocolState.FLUSHING, + SSLProtocolState.SHUTDOWN, + SSLProtocolState.UNWRAPPED + ) + ): + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('SSL connection is closed') + self._conn_lost += 1 return + for data in list_of_data: + self._write_backlog.append(data) + self._write_buffer_size += len(data) + try: - for i in range(len(self._write_backlog)): - data, offset = self._write_backlog[0] - if data: - ssldata, offset = self._sslpipe.feed_appdata(data, offset) - elif offset: - ssldata = self._sslpipe.do_handshake( - self._on_handshake_complete) - offset = 1 + if self._state == SSLProtocolState.WRAPPED: + self._do_write() + + except Exception as ex: + self._fatal_error(ex, 'Fatal error on SSL protocol') + + def _do_write(self): + try: + while self._write_backlog: + data = self._write_backlog[0] + count = self._sslobj.write(data) + data_len = len(data) + if count < data_len: + self._write_backlog[0] = data[count:] + self._write_buffer_size -= count else: - ssldata = self._sslpipe.shutdown(self._finalize) - offset = 1 - - for chunk in ssldata: - self._transport.write(chunk) - - if offset < len(data): - self._write_backlog[0] = (data, offset) - # A short write means that a write is blocked on a read - # We need to enable reading if it is paused! - assert self._sslpipe.need_ssldata - if self._transport._paused: - self._transport.resume_reading() + del self._write_backlog[0] + self._write_buffer_size -= data_len + except SSLAgainErrors: + pass + self._process_outgoing() + + def _process_outgoing(self): + if not self._ssl_writing_paused: + data = self._outgoing.read() + if len(data): + self._transport.write(data) + self._control_app_writing() + + # Incoming flow + + def _do_read(self): + if ( + self._state not in ( + SSLProtocolState.WRAPPED, + SSLProtocolState.FLUSHING, + ) + ): + return + try: + if not self._app_reading_paused: + if self._app_protocol_is_buffer: + self._do_read__buffered() + else: + self._do_read__copied() + if self._write_backlog: + self._do_write() + else: + self._process_outgoing() + self._control_ssl_reading() + except Exception as ex: + self._fatal_error(ex, 'Fatal error on SSL protocol') + + def _do_read__buffered(self): + offset = 0 + count = 1 + + buf = self._app_protocol_get_buffer(self._get_read_buffer_size()) + wants = len(buf) + + try: + count = self._sslobj.read(wants, buf) + + if count > 0: + offset = count + while offset < wants: + count = self._sslobj.read(wants - offset, buf[offset:]) + if count > 0: + offset += count + else: + break + else: + self._loop.call_soon(self._do_read) + except SSLAgainErrors: + pass + if offset > 0: + self._app_protocol_buffer_updated(offset) + if not count: + # close_notify + self._call_eof_received() + self._start_shutdown() + + def _do_read__copied(self): + chunk = b'1' + zero = True + one = False + + try: + while True: + chunk = self._sslobj.read(self.max_size) + if not chunk: break + if zero: + zero = False + one = True + first = chunk + elif one: + one = False + data = [first, chunk] + else: + data.append(chunk) + except SSLAgainErrors: + pass + if one: + self._app_protocol.data_received(first) + elif not zero: + self._app_protocol.data_received(b''.join(data)) + if not chunk: + # close_notify + self._call_eof_received() + self._start_shutdown() + + def _call_eof_received(self): + try: + if self._app_state == AppProtocolState.STATE_CON_MADE: + self._app_state = AppProtocolState.STATE_EOF + keep_open = self._app_protocol.eof_received() + if keep_open: + logger.warning('returning true from eof_received() ' + 'has no effect when using ssl') + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as ex: + self._fatal_error(ex, 'Error calling eof_received()') - # An entire chunk from the backlog was processed. We can - # delete it and reduce the outstanding buffer size. - del self._write_backlog[0] - self._write_buffer_size -= len(data) - except BaseException as exc: - if self._in_handshake: - # BaseExceptions will be re-raised in _on_handshake_complete. - self._on_handshake_complete(exc) - else: - self._fatal_error(exc, 'Fatal error on SSL transport') - if not isinstance(exc, Exception): - # BaseException + # Flow control for writes from APP socket + + def _control_app_writing(self): + size = self._get_write_buffer_size() + if size >= self._outgoing_high_water and not self._app_writing_paused: + self._app_writing_paused = True + try: + self._app_protocol.pause_writing() + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: + self._loop.call_exception_handler({ + 'message': 'protocol.pause_writing() failed', + 'exception': exc, + 'transport': self._app_transport, + 'protocol': self, + }) + elif size <= self._outgoing_low_water and self._app_writing_paused: + self._app_writing_paused = False + try: + self._app_protocol.resume_writing() + except (KeyboardInterrupt, SystemExit): raise + except BaseException as exc: + self._loop.call_exception_handler({ + 'message': 'protocol.resume_writing() failed', + 'exception': exc, + 'transport': self._app_transport, + 'protocol': self, + }) + + def _get_write_buffer_size(self): + return self._outgoing.pending + self._write_buffer_size + + def _set_write_buffer_limits(self, high=None, low=None): + high, low = add_flowcontrol_defaults( + high, low, constants.FLOW_CONTROL_HIGH_WATER_SSL_WRITE) + self._outgoing_high_water = high + self._outgoing_low_water = low + + # Flow control for reads to APP socket + + def _pause_reading(self): + self._app_reading_paused = True + + def _resume_reading(self): + if self._app_reading_paused: + self._app_reading_paused = False + + def resume(): + if self._state == SSLProtocolState.WRAPPED: + self._do_read() + elif self._state == SSLProtocolState.FLUSHING: + self._do_flush() + elif self._state == SSLProtocolState.SHUTDOWN: + self._do_shutdown() + self._loop.call_soon(resume) + + # Flow control for reads from SSL socket + + def _control_ssl_reading(self): + size = self._get_read_buffer_size() + if size >= self._incoming_high_water and not self._ssl_reading_paused: + self._ssl_reading_paused = True + self._transport.pause_reading() + elif size <= self._incoming_low_water and self._ssl_reading_paused: + self._ssl_reading_paused = False + self._transport.resume_reading() + + def _set_read_buffer_limits(self, high=None, low=None): + high, low = add_flowcontrol_defaults( + high, low, constants.FLOW_CONTROL_HIGH_WATER_SSL_READ) + self._incoming_high_water = high + self._incoming_low_water = low + + def _get_read_buffer_size(self): + return self._incoming.pending + + # Flow control for writes to SSL socket + + def pause_writing(self): + """Called when the low-level transport's buffer goes over + the high-water mark. + """ + assert not self._ssl_writing_paused + self._ssl_writing_paused = True + + def resume_writing(self): + """Called when the low-level transport's buffer drains below + the low-water mark. + """ + assert self._ssl_writing_paused + self._ssl_writing_paused = False + self._process_outgoing() def _fatal_error(self, exc, message='Fatal error on transport'): - # Should be called from exception handler only. - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if self._transport: + self._transport._force_close(exc) + + if isinstance(exc, OSError): if self._loop.get_debug(): logger.debug("%r: %s", self, message, exc_info=True) - else: + elif not isinstance(exc, exceptions.CancelledError): self._loop.call_exception_handler({ 'message': message, 'exception': exc, 'transport': self._transport, 'protocol': self, }) - if self._transport: - self._transport._force_close(exc) - - def _finalize(self): - if self._transport is not None: - self._transport.close() - - def _abort(self): - if self._transport is not None: - try: - self._transport.abort() - finally: - self._finalize() diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py new file mode 100644 index 00000000000..2ad65d8648e --- /dev/null +++ b/Lib/asyncio/staggered.py @@ -0,0 +1,179 @@ +"""Support for running coroutines in parallel with staggered start times.""" + +__all__ = 'staggered_race', + +import contextlib + +from . import events +from . import exceptions as exceptions_mod +from . import locks +from . import tasks +from . import futures + + +async def staggered_race(coro_fns, delay, *, loop=None): + """Run coroutines with staggered start times and take the first to finish. + + This method takes an iterable of coroutine functions. The first one is + started immediately. From then on, whenever the immediately preceding one + fails (raises an exception), or when *delay* seconds has passed, the next + coroutine is started. This continues until one of the coroutines complete + successfully, in which case all others are cancelled, or until all + coroutines fail. + + The coroutines provided should be well-behaved in the following way: + + * They should only ``return`` if completed successfully. + + * They should always raise an exception if they did not complete + successfully. In particular, if they handle cancellation, they should + probably reraise, like this:: + + try: + # do work + except asyncio.CancelledError: + # undo partially completed work + raise + + Args: + coro_fns: an iterable of coroutine functions, i.e. callables that + return a coroutine object when called. Use ``functools.partial`` or + lambdas to pass arguments. + + delay: amount of time, in seconds, between starting coroutines. If + ``None``, the coroutines will run sequentially. + + loop: the event loop to use. + + Returns: + tuple *(winner_result, winner_index, exceptions)* where + + - *winner_result*: the result of the winning coroutine, or ``None`` + if no coroutines won. + + - *winner_index*: the index of the winning coroutine in + ``coro_fns``, or ``None`` if no coroutines won. If the winning + coroutine may return None on success, *winner_index* can be used + to definitively determine whether any coroutine won. + + - *exceptions*: list of exceptions returned by the coroutines. + ``len(exceptions)`` is equal to the number of coroutines actually + started, and the order is the same as in ``coro_fns``. The winning + coroutine's entry is ``None``. + + """ + # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. + loop = loop or events.get_running_loop() + parent_task = tasks.current_task(loop) + enum_coro_fns = enumerate(coro_fns) + winner_result = None + winner_index = None + unhandled_exceptions = [] + exceptions = [] + running_tasks = set() + on_completed_fut = None + + def task_done(task): + running_tasks.discard(task) + futures.future_discard_from_awaited_by(task, parent_task) + if ( + on_completed_fut is not None + and not on_completed_fut.done() + and not running_tasks + ): + on_completed_fut.set_result(None) + + if task.cancelled(): + return + + exc = task.exception() + if exc is None: + return + unhandled_exceptions.append(exc) + + async def run_one_coro(ok_to_start, previous_failed) -> None: + # in eager tasks this waits for the calling task to append this task + # to running_tasks, in regular tasks this wait is a no-op that does + # not yield a future. See gh-124309. + await ok_to_start.wait() + # Wait for the previous task to finish, or for delay seconds + if previous_failed is not None: + with contextlib.suppress(exceptions_mod.TimeoutError): + # Use asyncio.wait_for() instead of asyncio.wait() here, so + # that if we get cancelled at this point, Event.wait() is also + # cancelled, otherwise there will be a "Task destroyed but it is + # pending" later. + await tasks.wait_for(previous_failed.wait(), delay) + # Get the next coroutine to run + try: + this_index, coro_fn = next(enum_coro_fns) + except StopIteration: + return + # Start task that will run the next coroutine + this_failed = locks.Event() + next_ok_to_start = locks.Event() + next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) + futures.future_add_to_awaited_by(next_task, parent_task) + running_tasks.add(next_task) + next_task.add_done_callback(task_done) + # next_task has been appended to running_tasks so next_task is ok to + # start. + next_ok_to_start.set() + # Prepare place to put this coroutine's exceptions if not won + exceptions.append(None) + assert len(exceptions) == this_index + 1 + + try: + result = await coro_fn() + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as e: + exceptions[this_index] = e + this_failed.set() # Kickstart the next coroutine + else: + # Store winner's results + nonlocal winner_index, winner_result + assert winner_index is None + winner_index = this_index + winner_result = result + # Cancel all other tasks. We take care to not cancel the current + # task as well. If we do so, then since there is no `await` after + # here and CancelledError are usually thrown at one, we will + # encounter a curious corner case where the current task will end + # up as done() == True, cancelled() == False, exception() == + # asyncio.CancelledError. This behavior is specified in + # https://bugs.python.org/issue30048 + current_task = tasks.current_task(loop) + for t in running_tasks: + if t is not current_task: + t.cancel() + + propagate_cancellation_error = None + try: + ok_to_start = locks.Event() + first_task = loop.create_task(run_one_coro(ok_to_start, None)) + futures.future_add_to_awaited_by(first_task, parent_task) + running_tasks.add(first_task) + first_task.add_done_callback(task_done) + # first_task has been appended to running_tasks so first_task is ok to start. + ok_to_start.set() + propagate_cancellation_error = None + # Make sure no tasks are left running if we leave this function + while running_tasks: + on_completed_fut = loop.create_future() + try: + await on_completed_fut + except exceptions_mod.CancelledError as ex: + propagate_cancellation_error = ex + for task in running_tasks: + task.cancel(*ex.args) + on_completed_fut = None + if __debug__ and unhandled_exceptions: + # If run_one_coro raises an unhandled exception, it's probably a + # programming error, and I want to see it. + raise ExceptionGroup("staggered race failed", unhandled_exceptions) + if propagate_cancellation_error is not None: + raise propagate_cancellation_error + return winner_result, winner_index, exceptions + finally: + del exceptions, propagate_cancellation_error, unhandled_exceptions, parent_task diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index a82cc79acaa..64aac4cc50d 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -1,55 +1,30 @@ -"""Stream-related things.""" - -__all__ = ['StreamReader', 'StreamWriter', 'StreamReaderProtocol', - 'open_connection', 'start_server', - 'IncompleteReadError', - 'LimitOverrunError', - ] +__all__ = ( + 'StreamReader', 'StreamWriter', 'StreamReaderProtocol', + 'open_connection', 'start_server') +import collections import socket +import sys +import warnings +import weakref if hasattr(socket, 'AF_UNIX'): - __all__.extend(['open_unix_connection', 'start_unix_server']) + __all__ += ('open_unix_connection', 'start_unix_server') from . import coroutines -from . import compat from . import events +from . import exceptions +from . import format_helpers from . import protocols -from .coroutines import coroutine from .log import logger +from .tasks import sleep -_DEFAULT_LIMIT = 2 ** 16 - - -class IncompleteReadError(EOFError): - """ - Incomplete read error. Attributes: - - - partial: read bytes string before the end of stream was reached - - expected: total number of expected bytes (or None if unknown) - """ - def __init__(self, partial, expected): - super().__init__("%d bytes read on a total of %r expected bytes" - % (len(partial), expected)) - self.partial = partial - self.expected = expected - - -class LimitOverrunError(Exception): - """Reached the buffer limit while looking for a separator. - - Attributes: - - consumed: total number of to be consumed bytes. - """ - def __init__(self, message, consumed): - super().__init__(message) - self.consumed = consumed +_DEFAULT_LIMIT = 2 ** 16 # 64 KiB -@coroutine -def open_connection(host=None, port=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): +async def open_connection(host=None, port=None, *, + limit=_DEFAULT_LIMIT, **kwds): """A wrapper for create_connection() returning a (reader, writer) pair. The reader returned is a StreamReader instance; the writer is a @@ -67,19 +42,17 @@ def open_connection(host=None, port=None, *, StreamReaderProtocol classes, just copy the code -- there's really nothing special here except some convenience.) """ - if loop is None: - loop = events.get_event_loop() + loop = events.get_running_loop() reader = StreamReader(limit=limit, loop=loop) protocol = StreamReaderProtocol(reader, loop=loop) - transport, _ = yield from loop.create_connection( + transport, _ = await loop.create_connection( lambda: protocol, host, port, **kwds) writer = StreamWriter(transport, protocol, reader, loop) return reader, writer -@coroutine -def start_server(client_connected_cb, host=None, port=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): +async def start_server(client_connected_cb, host=None, port=None, *, + limit=_DEFAULT_LIMIT, **kwds): """Start a socket server, call back for each client connected. The first parameter, `client_connected_cb`, takes two parameters: @@ -94,15 +67,13 @@ def start_server(client_connected_cb, host=None, port=None, *, positional host and port, with various optional keyword arguments following. The return value is the same as loop.create_server(). - Additional optional keyword arguments are loop (to set the event loop - instance to use) and limit (to set the buffer limit passed to the - StreamReader). + Additional optional keyword argument is limit (to set the buffer + limit passed to the StreamReader). The return value is the same as loop.create_server(), i.e. a Server object which can be used to stop the service. """ - if loop is None: - loop = events.get_event_loop() + loop = events.get_running_loop() def factory(): reader = StreamReader(limit=limit, loop=loop) @@ -110,31 +81,28 @@ def factory(): loop=loop) return protocol - return (yield from loop.create_server(factory, host, port, **kwds)) + return await loop.create_server(factory, host, port, **kwds) if hasattr(socket, 'AF_UNIX'): # UNIX Domain Sockets are supported on this platform - @coroutine - def open_unix_connection(path=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): + async def open_unix_connection(path=None, *, + limit=_DEFAULT_LIMIT, **kwds): """Similar to `open_connection` but works with UNIX Domain Sockets.""" - if loop is None: - loop = events.get_event_loop() + loop = events.get_running_loop() + reader = StreamReader(limit=limit, loop=loop) protocol = StreamReaderProtocol(reader, loop=loop) - transport, _ = yield from loop.create_unix_connection( + transport, _ = await loop.create_unix_connection( lambda: protocol, path, **kwds) writer = StreamWriter(transport, protocol, reader, loop) return reader, writer - @coroutine - def start_unix_server(client_connected_cb, path=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): + async def start_unix_server(client_connected_cb, path=None, *, + limit=_DEFAULT_LIMIT, **kwds): """Similar to `start_server` but works with UNIX Domain Sockets.""" - if loop is None: - loop = events.get_event_loop() + loop = events.get_running_loop() def factory(): reader = StreamReader(limit=limit, loop=loop) @@ -142,14 +110,14 @@ def factory(): loop=loop) return protocol - return (yield from loop.create_unix_server(factory, path, **kwds)) + return await loop.create_unix_server(factory, path, **kwds) class FlowControlMixin(protocols.Protocol): """Reusable flow control logic for StreamWriter.drain(). This implements the protocol methods pause_writing(), - resume_reading() and connection_lost(). If the subclass overrides + resume_writing() and connection_lost(). If the subclass overrides these it must call the super methods. StreamWriter.drain() must wait for _drain_helper() coroutine. @@ -161,7 +129,7 @@ def __init__(self, loop=None): else: self._loop = loop self._paused = False - self._drain_waiter = None + self._drain_waiters = collections.deque() self._connection_lost = False def pause_writing(self): @@ -176,39 +144,37 @@ def resume_writing(self): if self._loop.get_debug(): logger.debug("%r resumes writing", self) - waiter = self._drain_waiter - if waiter is not None: - self._drain_waiter = None + for waiter in self._drain_waiters: if not waiter.done(): waiter.set_result(None) def connection_lost(self, exc): self._connection_lost = True - # Wake up the writer if currently paused. + # Wake up the writer(s) if currently paused. if not self._paused: return - waiter = self._drain_waiter - if waiter is None: - return - self._drain_waiter = None - if waiter.done(): - return - if exc is None: - waiter.set_result(None) - else: - waiter.set_exception(exc) - @coroutine - def _drain_helper(self): + for waiter in self._drain_waiters: + if not waiter.done(): + if exc is None: + waiter.set_result(None) + else: + waiter.set_exception(exc) + + async def _drain_helper(self): if self._connection_lost: raise ConnectionResetError('Connection lost') if not self._paused: return - waiter = self._drain_waiter - assert waiter is None or waiter.cancelled() waiter = self._loop.create_future() - self._drain_waiter = waiter - yield from waiter + self._drain_waiters.append(waiter) + try: + await waiter + finally: + self._drain_waiters.remove(waiter) + + def _get_close_waiter(self, stream): + raise NotImplementedError class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): @@ -220,40 +186,104 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): call inappropriate methods of the protocol.) """ + _source_traceback = None + def __init__(self, stream_reader, client_connected_cb=None, loop=None): super().__init__(loop=loop) - self._stream_reader = stream_reader - self._stream_writer = None + if stream_reader is not None: + self._stream_reader_wr = weakref.ref(stream_reader) + self._source_traceback = stream_reader._source_traceback + else: + self._stream_reader_wr = None + if client_connected_cb is not None: + # This is a stream created by the `create_server()` function. + # Keep a strong reference to the reader until a connection + # is established. + self._strong_reader = stream_reader + self._reject_connection = False + self._task = None + self._transport = None self._client_connected_cb = client_connected_cb self._over_ssl = False + self._closed = self._loop.create_future() + + @property + def _stream_reader(self): + if self._stream_reader_wr is None: + return None + return self._stream_reader_wr() + + def _replace_transport(self, transport): + loop = self._loop + self._transport = transport + self._over_ssl = transport.get_extra_info('sslcontext') is not None def connection_made(self, transport): - self._stream_reader.set_transport(transport) + if self._reject_connection: + context = { + 'message': ('An open stream was garbage collected prior to ' + 'establishing network connection; ' + 'call "stream.close()" explicitly.') + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + transport.abort() + return + self._transport = transport + reader = self._stream_reader + if reader is not None: + reader.set_transport(transport) self._over_ssl = transport.get_extra_info('sslcontext') is not None if self._client_connected_cb is not None: - self._stream_writer = StreamWriter(transport, self, - self._stream_reader, - self._loop) - res = self._client_connected_cb(self._stream_reader, - self._stream_writer) + writer = StreamWriter(transport, self, reader, self._loop) + res = self._client_connected_cb(reader, writer) if coroutines.iscoroutine(res): - self._loop.create_task(res) + def callback(task): + if task.cancelled(): + transport.close() + return + exc = task.exception() + if exc is not None: + self._loop.call_exception_handler({ + 'message': 'Unhandled exception in client_connected_cb', + 'exception': exc, + 'transport': transport, + }) + transport.close() + + self._task = self._loop.create_task(res) + self._task.add_done_callback(callback) + + self._strong_reader = None def connection_lost(self, exc): - if self._stream_reader is not None: + reader = self._stream_reader + if reader is not None: + if exc is None: + reader.feed_eof() + else: + reader.set_exception(exc) + if not self._closed.done(): if exc is None: - self._stream_reader.feed_eof() + self._closed.set_result(None) else: - self._stream_reader.set_exception(exc) + self._closed.set_exception(exc) super().connection_lost(exc) - self._stream_reader = None + self._stream_reader_wr = None self._stream_writer = None + self._task = None + self._transport = None def data_received(self, data): - self._stream_reader.feed_data(data) + reader = self._stream_reader + if reader is not None: + reader.feed_data(data) def eof_received(self): - self._stream_reader.feed_eof() + reader = self._stream_reader + if reader is not None: + reader.feed_eof() if self._over_ssl: # Prevent a warning in SSLProtocol.eof_received: # "returning true from eof_received() @@ -261,6 +291,20 @@ def eof_received(self): return False return True + def _get_close_waiter(self, stream): + return self._closed + + def __del__(self): + # Prevent reports about unhandled exceptions. + # Better than self._closed._log_traceback = False hack + try: + closed = self._closed + except AttributeError: + pass # failed constructor + else: + if closed.done() and not closed.cancelled(): + closed.exception() + class StreamWriter: """Wraps a Transport. @@ -279,12 +323,14 @@ def __init__(self, transport, protocol, reader, loop): assert reader is None or isinstance(reader, StreamReader) self._reader = reader self._loop = loop + self._complete_fut = self._loop.create_future() + self._complete_fut.set_result(None) def __repr__(self): - info = [self.__class__.__name__, 'transport=%r' % self._transport] + info = [self.__class__.__name__, f'transport={self._transport!r}'] if self._reader is not None: - info.append('reader=%r' % self._reader) - return '<%s>' % ' '.join(info) + info.append(f'reader={self._reader!r}') + return '<{}>'.format(' '.join(info)) @property def transport(self): @@ -305,36 +351,68 @@ def can_write_eof(self): def close(self): return self._transport.close() + def is_closing(self): + return self._transport.is_closing() + + async def wait_closed(self): + await self._protocol._get_close_waiter(self) + def get_extra_info(self, name, default=None): return self._transport.get_extra_info(name, default) - @coroutine - def drain(self): + async def drain(self): """Flush the write buffer. The intended use is to write w.write(data) - yield from w.drain() + await w.drain() """ if self._reader is not None: exc = self._reader.exception() if exc is not None: raise exc - if self._transport is not None: - if self._transport.is_closing(): - # Yield to the event loop so connection_lost() may be - # called. Without this, _drain_helper() would return - # immediately, and code that calls - # write(...); yield from drain() - # in a loop would never call connection_lost(), so it - # would not see an error when the socket is closed. - yield - yield from self._protocol._drain_helper() - + if self._transport.is_closing(): + # Wait for protocol.connection_lost() call + # Raise connection closing error if any, + # ConnectionResetError otherwise + # Yield to the event loop so connection_lost() may be + # called. Without this, _drain_helper() would return + # immediately, and code that calls + # write(...); await drain() + # in a loop would never call connection_lost(), so it + # would not see an error when the socket is closed. + await sleep(0) + await self._protocol._drain_helper() + + async def start_tls(self, sslcontext, *, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): + """Upgrade an existing stream-based connection to TLS.""" + server_side = self._protocol._client_connected_cb is not None + protocol = self._protocol + await self.drain() + new_transport = await self._loop.start_tls( # type: ignore + self._transport, protocol, sslcontext, + server_side=server_side, server_hostname=server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) + self._transport = new_transport + protocol._replace_transport(new_transport) + + def __del__(self, warnings=warnings): + if not self._transport.is_closing(): + if self._loop.is_closed(): + warnings.warn("loop is closed", ResourceWarning) + else: + self.close() + warnings.warn(f"unclosed {self!r}", ResourceWarning) class StreamReader: + _source_traceback = None + def __init__(self, limit=_DEFAULT_LIMIT, loop=None): # The line length limit is a security feature; # it also doubles as half the buffer limit. @@ -353,24 +431,27 @@ def __init__(self, limit=_DEFAULT_LIMIT, loop=None): self._exception = None self._transport = None self._paused = False + if self._loop.get_debug(): + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) def __repr__(self): info = ['StreamReader'] if self._buffer: - info.append('%d bytes' % len(self._buffer)) + info.append(f'{len(self._buffer)} bytes') if self._eof: info.append('eof') if self._limit != _DEFAULT_LIMIT: - info.append('l=%d' % self._limit) + info.append(f'limit={self._limit}') if self._waiter: - info.append('w=%r' % self._waiter) + info.append(f'waiter={self._waiter!r}') if self._exception: - info.append('e=%r' % self._exception) + info.append(f'exception={self._exception!r}') if self._transport: - info.append('t=%r' % self._transport) + info.append(f'transport={self._transport!r}') if self._paused: info.append('paused') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def exception(self): return self._exception @@ -431,8 +512,7 @@ def feed_data(self, data): else: self._paused = True - @coroutine - def _wait_for_data(self, func_name): + async def _wait_for_data(self, func_name): """Wait until feed_data() or feed_eof() is called. If stream was paused, automatically resume it. @@ -442,8 +522,9 @@ def _wait_for_data(self, func_name): # would have an unexpected behaviour. It would not possible to know # which coroutine would get the next data. if self._waiter is not None: - raise RuntimeError('%s() called while another coroutine is ' - 'already waiting for incoming data' % func_name) + raise RuntimeError( + f'{func_name}() called while another coroutine is ' + f'already waiting for incoming data') assert not self._eof, '_wait_for_data after EOF' @@ -455,12 +536,11 @@ def _wait_for_data(self, func_name): self._waiter = self._loop.create_future() try: - yield from self._waiter + await self._waiter finally: self._waiter = None - @coroutine - def readline(self): + async def readline(self): """Read chunk of data from the stream until newline (b'\n') is found. On success, return chunk that ends with newline. If only partial @@ -479,10 +559,10 @@ def readline(self): sep = b'\n' seplen = len(sep) try: - line = yield from self.readuntil(sep) - except IncompleteReadError as e: + line = await self.readuntil(sep) + except exceptions.IncompleteReadError as e: return e.partial - except LimitOverrunError as e: + except exceptions.LimitOverrunError as e: if self._buffer.startswith(sep, e.consumed): del self._buffer[:e.consumed + seplen] else: @@ -491,8 +571,7 @@ def readline(self): raise ValueError(e.args[0]) return line - @coroutine - def readuntil(self, separator=b'\n'): + async def readuntil(self, separator=b'\n'): """Read data from the stream until ``separator`` is found. On success, the data and separator will be removed from the @@ -511,20 +590,34 @@ def readuntil(self, separator=b'\n'): If the data cannot be read because of over limit, a LimitOverrunError exception will be raised, and the data will be left in the internal buffer, so it can be read again. + + The ``separator`` may also be a tuple of separators. In this + case the return value will be the shortest possible that has any + separator as the suffix. For the purposes of LimitOverrunError, + the shortest possible separator is considered to be the one that + matched. """ - seplen = len(separator) - if seplen == 0: + if isinstance(separator, tuple): + # Makes sure shortest matches wins + separator = sorted(separator, key=len) + else: + separator = [separator] + if not separator: + raise ValueError('Separator should contain at least one element') + min_seplen = len(separator[0]) + max_seplen = len(separator[-1]) + if min_seplen == 0: raise ValueError('Separator should be at least one-byte string') if self._exception is not None: raise self._exception # Consume whole buffer except last bytes, which length is - # one less than seplen. Let's check corner cases with - # separator='SEPARATOR': + # one less than max_seplen. Let's check corner cases with + # separator[-1]='SEPARATOR': # * we have received almost complete separator (without last # byte). i.e buffer='some textSEPARATO'. In this case we - # can safely consume len(separator) - 1 bytes. + # can safely consume max_seplen - 1 bytes. # * last byte of buffer is first byte of separator, i.e. # buffer='abcdefghijklmnopqrS'. We may safely consume # everything except that last byte, but this require to @@ -537,66 +630,75 @@ def readuntil(self, separator=b'\n'): # messages :) # `offset` is the number of bytes from the beginning of the buffer - # where there is no occurrence of `separator`. + # where there is no occurrence of any `separator`. offset = 0 - # Loop until we find `separator` in the buffer, exceed the buffer size, + # Loop until we find a `separator` in the buffer, exceed the buffer size, # or an EOF has happened. while True: buflen = len(self._buffer) - # Check if we now have enough data in the buffer for `separator` to - # fit. - if buflen - offset >= seplen: - isep = self._buffer.find(separator, offset) - - if isep != -1: - # `separator` is in the buffer. `isep` will be used later - # to retrieve the data. + # Check if we now have enough data in the buffer for shortest + # separator to fit. + if buflen - offset >= min_seplen: + match_start = None + match_end = None + for sep in separator: + isep = self._buffer.find(sep, offset) + + if isep != -1: + # `separator` is in the buffer. `match_start` and + # `match_end` will be used later to retrieve the + # data. + end = isep + len(sep) + if match_end is None or end < match_end: + match_end = end + match_start = isep + if match_end is not None: break # see upper comment for explanation. - offset = buflen + 1 - seplen + offset = max(0, buflen + 1 - max_seplen) if offset > self._limit: - raise LimitOverrunError( + raise exceptions.LimitOverrunError( 'Separator is not found, and chunk exceed the limit', offset) # Complete message (with full separator) may be present in buffer # even when EOF flag is set. This may happen when the last chunk # adds data which makes separator be found. That's why we check for - # EOF *ater* inspecting the buffer. + # EOF *after* inspecting the buffer. if self._eof: chunk = bytes(self._buffer) self._buffer.clear() - raise IncompleteReadError(chunk, None) + raise exceptions.IncompleteReadError(chunk, None) # _wait_for_data() will resume reading if stream was paused. - yield from self._wait_for_data('readuntil') + await self._wait_for_data('readuntil') - if isep > self._limit: - raise LimitOverrunError( - 'Separator is found, but chunk is longer than limit', isep) + if match_start > self._limit: + raise exceptions.LimitOverrunError( + 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:isep + seplen] - del self._buffer[:isep + seplen] + chunk = self._buffer[:match_end] + del self._buffer[:match_end] self._maybe_resume_transport() return bytes(chunk) - @coroutine - def read(self, n=-1): + async def read(self, n=-1): """Read up to `n` bytes from the stream. - If n is not provided, or set to -1, read until EOF and return all read - bytes. If the EOF was received and the internal buffer is empty, return - an empty bytes object. + If `n` is not provided or set to -1, + read until EOF, then return all read bytes. + If EOF was received and the internal buffer is empty, + return an empty bytes object. - If n is zero, return empty bytes object immediately. + If `n` is 0, return an empty bytes object immediately. - If n is positive, this function try to read `n` bytes, and may return - less or equal bytes than requested, but at least one byte. If EOF was - received before any byte is read, this function returns empty byte - object. + If `n` is positive, return at most `n` available bytes + as soon as at least 1 byte is available in the internal buffer. + If EOF is received before any byte is read, return an empty + bytes object. Returned value is not limited with limit, configured at stream creation. @@ -618,24 +720,23 @@ def read(self, n=-1): # bytes. So just call self.read(self._limit) until EOF. blocks = [] while True: - block = yield from self.read(self._limit) + block = await self.read(self._limit) if not block: break blocks.append(block) return b''.join(blocks) if not self._buffer and not self._eof: - yield from self._wait_for_data('read') + await self._wait_for_data('read') # This will work right even if buffer is less than n bytes - data = bytes(self._buffer[:n]) + data = bytes(memoryview(self._buffer)[:n]) del self._buffer[:n] self._maybe_resume_transport() return data - @coroutine - def readexactly(self, n): + async def readexactly(self, n): """Read exactly `n` bytes. Raise an IncompleteReadError if EOF is reached before `n` bytes can be @@ -663,33 +764,24 @@ def readexactly(self, n): if self._eof: incomplete = bytes(self._buffer) self._buffer.clear() - raise IncompleteReadError(incomplete, n) + raise exceptions.IncompleteReadError(incomplete, n) - yield from self._wait_for_data('readexactly') + await self._wait_for_data('readexactly') if len(self._buffer) == n: data = bytes(self._buffer) self._buffer.clear() else: - data = bytes(self._buffer[:n]) + data = bytes(memoryview(self._buffer)[:n]) del self._buffer[:n] self._maybe_resume_transport() return data - if compat.PY35: - @coroutine - def __aiter__(self): - return self - - @coroutine - def __anext__(self): - val = yield from self.readline() - if val == b'': - raise StopAsyncIteration - return val - - if compat.PY352: - # In Python 3.5.2 and greater, __aiter__ should return - # the asynchronous iterator directly. - def __aiter__(self): - return self + def __aiter__(self): + return self + + async def __anext__(self): + val = await self.readline() + if val == b'': + raise StopAsyncIteration + return val diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index b2f5304f772..043359bbd03 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -1,4 +1,4 @@ -__all__ = ['create_subprocess_exec', 'create_subprocess_shell'] +__all__ = 'create_subprocess_exec', 'create_subprocess_shell' import subprocess @@ -6,7 +6,6 @@ from . import protocols from . import streams from . import tasks -from .coroutines import coroutine from .log import logger @@ -24,16 +23,19 @@ def __init__(self, limit, loop): self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None + self._process_exited = False + self._pipe_fds = [] + self._stdin_closed = self._loop.create_future() def __repr__(self): info = [self.__class__.__name__] if self.stdin is not None: - info.append('stdin=%r' % self.stdin) + info.append(f'stdin={self.stdin!r}') if self.stdout is not None: - info.append('stdout=%r' % self.stdout) + info.append(f'stdout={self.stdout!r}') if self.stderr is not None: - info.append('stderr=%r' % self.stderr) - return '<%s>' % ' '.join(info) + info.append(f'stderr={self.stderr!r}') + return '<{}>'.format(' '.join(info)) def connection_made(self, transport): self._transport = transport @@ -43,12 +45,14 @@ def connection_made(self, transport): self.stdout = streams.StreamReader(limit=self._limit, loop=self._loop) self.stdout.set_transport(stdout_transport) + self._pipe_fds.append(1) stderr_transport = transport.get_pipe_transport(2) if stderr_transport is not None: self.stderr = streams.StreamReader(limit=self._limit, loop=self._loop) self.stderr.set_transport(stderr_transport) + self._pipe_fds.append(2) stdin_transport = transport.get_pipe_transport(0) if stdin_transport is not None: @@ -73,6 +77,13 @@ def pipe_connection_lost(self, fd, exc): if pipe is not None: pipe.close() self.connection_lost(exc) + if exc is None: + self._stdin_closed.set_result(None) + else: + self._stdin_closed.set_exception(exc) + # Since calling `wait_closed()` is not mandatory, + # we shouldn't log the traceback if this is not awaited. + self._stdin_closed._log_traceback = False return if fd == 1: reader = self.stdout @@ -80,15 +91,28 @@ def pipe_connection_lost(self, fd, exc): reader = self.stderr else: reader = None - if reader != None: + if reader is not None: if exc is None: reader.feed_eof() else: reader.set_exception(exc) + if fd in self._pipe_fds: + self._pipe_fds.remove(fd) + self._maybe_close_transport() + def process_exited(self): - self._transport.close() - self._transport = None + self._process_exited = True + self._maybe_close_transport() + + def _maybe_close_transport(self): + if len(self._pipe_fds) == 0 and self._process_exited: + self._transport.close() + self._transport = None + + def _get_close_waiter(self, stream): + if stream is self.stdin: + return self._stdin_closed class Process: @@ -102,18 +126,15 @@ def __init__(self, transport, protocol, loop): self.pid = transport.get_pid() def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, self.pid) + return f'<{self.__class__.__name__} {self.pid}>' @property def returncode(self): return self._transport.get_returncode() - @coroutine - def wait(self): - """Wait until the process exit and return the process return code. - - This method is a coroutine.""" - return (yield from self._transport._wait()) + async def wait(self): + """Wait until the process exit and return the process return code.""" + return await self._transport._wait() def send_signal(self, signal): self._transport.send_signal(signal) @@ -124,17 +145,19 @@ def terminate(self): def kill(self): self._transport.kill() - @coroutine - def _feed_stdin(self, input): + async def _feed_stdin(self, input): debug = self._loop.get_debug() - self.stdin.write(input) - if debug: - logger.debug('%r communicate: feed stdin (%s bytes)', - self, len(input)) try: - yield from self.stdin.drain() + if input is not None: + self.stdin.write(input) + if debug: + logger.debug( + '%r communicate: feed stdin (%s bytes)', self, len(input)) + + await self.stdin.drain() except (BrokenPipeError, ConnectionResetError) as exc: - # communicate() ignores BrokenPipeError and ConnectionResetError + # communicate() ignores BrokenPipeError and ConnectionResetError. + # write() and drain() can raise these exceptions. if debug: logger.debug('%r communicate: stdin got %r', self, exc) @@ -142,12 +165,10 @@ def _feed_stdin(self, input): logger.debug('%r communicate: close stdin', self) self.stdin.close() - @coroutine - def _noop(self): + async def _noop(self): return None - @coroutine - def _read_stream(self, fd): + async def _read_stream(self, fd): transport = self._transport.get_pipe_transport(fd) if fd == 2: stream = self.stderr @@ -157,16 +178,15 @@ def _read_stream(self, fd): if self._loop.get_debug(): name = 'stdout' if fd == 1 else 'stderr' logger.debug('%r communicate: read %s', self, name) - output = yield from stream.read() + output = await stream.read() if self._loop.get_debug(): name = 'stdout' if fd == 1 else 'stderr' logger.debug('%r communicate: close %s', self, name) transport.close() return output - @coroutine - def communicate(self, input=None): - if input is not None: + async def communicate(self, input=None): + if self.stdin is not None: stdin = self._feed_stdin(input) else: stdin = self._noop() @@ -178,36 +198,32 @@ def communicate(self, input=None): stderr = self._read_stream(2) else: stderr = self._noop() - stdin, stdout, stderr = yield from tasks.gather(stdin, stdout, stderr, - loop=self._loop) - yield from self.wait() + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + await self.wait() return (stdout, stderr) -@coroutine -def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, - loop=None, limit=streams._DEFAULT_LIMIT, **kwds): - if loop is None: - loop = events.get_event_loop() +async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, + limit=streams._DEFAULT_LIMIT, **kwds): + loop = events.get_running_loop() protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, loop=loop) - transport, protocol = yield from loop.subprocess_shell( - protocol_factory, - cmd, stdin=stdin, stdout=stdout, - stderr=stderr, **kwds) + transport, protocol = await loop.subprocess_shell( + protocol_factory, + cmd, stdin=stdin, stdout=stdout, + stderr=stderr, **kwds) return Process(transport, protocol, loop) -@coroutine -def create_subprocess_exec(program, *args, stdin=None, stdout=None, - stderr=None, loop=None, - limit=streams._DEFAULT_LIMIT, **kwds): - if loop is None: - loop = events.get_event_loop() + +async def create_subprocess_exec(program, *args, stdin=None, stdout=None, + stderr=None, limit=streams._DEFAULT_LIMIT, + **kwds): + loop = events.get_running_loop() protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, loop=loop) - transport, protocol = yield from loop.subprocess_exec( - protocol_factory, - program, *args, - stdin=stdin, stdout=stdout, - stderr=stderr, **kwds) + transport, protocol = await loop.subprocess_exec( + protocol_factory, + program, *args, + stdin=stdin, stdout=stdout, + stderr=stderr, **kwds) return Process(transport, protocol, loop) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py new file mode 100644 index 00000000000..00e8f6d5d1a --- /dev/null +++ b/Lib/asyncio/taskgroups.py @@ -0,0 +1,280 @@ +# Adapted with permission from the EdgeDB project; +# license: PSFL. + + +__all__ = ("TaskGroup",) + +from . import events +from . import exceptions +from . import futures +from . import tasks + + +class TaskGroup: + """Asynchronous context manager for managing groups of tasks. + + Example use: + + async with asyncio.TaskGroup() as group: + task1 = group.create_task(some_coroutine(...)) + task2 = group.create_task(other_coroutine(...)) + print("Both tasks have completed now.") + + All tasks are awaited when the context manager exits. + + Any exceptions other than `asyncio.CancelledError` raised within + a task will cancel all remaining tasks and wait for them to exit. + The exceptions are then combined and raised as an `ExceptionGroup`. + """ + def __init__(self): + self._entered = False + self._exiting = False + self._aborting = False + self._loop = None + self._parent_task = None + self._parent_cancel_requested = False + self._tasks = set() + self._errors = [] + self._base_error = None + self._on_completed_fut = None + + def __repr__(self): + info = [''] + if self._tasks: + info.append(f'tasks={len(self._tasks)}') + if self._errors: + info.append(f'errors={len(self._errors)}') + if self._aborting: + info.append('cancelling') + elif self._entered: + info.append('entered') + + info_str = ' '.join(info) + return f'' + + async def __aenter__(self): + if self._entered: + raise RuntimeError( + f"TaskGroup {self!r} has already been entered") + if self._loop is None: + self._loop = events.get_running_loop() + self._parent_task = tasks.current_task(self._loop) + if self._parent_task is None: + raise RuntimeError( + f'TaskGroup {self!r} cannot determine the parent task') + self._entered = True + + return self + + async def __aexit__(self, et, exc, tb): + tb = None + try: + return await self._aexit(et, exc) + finally: + # Exceptions are heavy objects that can have object + # cycles (bad for GC); let's not keep a reference to + # a bunch of them. It would be nicer to use a try/finally + # in __aexit__ directly but that introduced some diff noise + self._parent_task = None + self._errors = None + self._base_error = None + exc = None + + async def _aexit(self, et, exc): + self._exiting = True + + if (exc is not None and + self._is_base_error(exc) and + self._base_error is None): + self._base_error = exc + + if et is not None and issubclass(et, exceptions.CancelledError): + propagate_cancellation_error = exc + else: + propagate_cancellation_error = None + + if et is not None: + if not self._aborting: + # Our parent task is being cancelled: + # + # async with TaskGroup() as g: + # g.create_task(...) + # await ... # <- CancelledError + # + # or there's an exception in "async with": + # + # async with TaskGroup() as g: + # g.create_task(...) + # 1 / 0 + # + self._abort() + + # We use while-loop here because "self._on_completed_fut" + # can be cancelled multiple times if our parent task + # is being cancelled repeatedly (or even once, when + # our own cancellation is already in progress) + while self._tasks: + if self._on_completed_fut is None: + self._on_completed_fut = self._loop.create_future() + + try: + await self._on_completed_fut + except exceptions.CancelledError as ex: + if not self._aborting: + # Our parent task is being cancelled: + # + # async def wrapper(): + # async with TaskGroup() as g: + # g.create_task(foo) + # + # "wrapper" is being cancelled while "foo" is + # still running. + propagate_cancellation_error = ex + self._abort() + + self._on_completed_fut = None + + assert not self._tasks + + if self._base_error is not None: + try: + raise self._base_error + finally: + exc = None + + if self._parent_cancel_requested: + # If this flag is set we *must* call uncancel(). + if self._parent_task.uncancel() == 0: + # If there are no pending cancellations left, + # don't propagate CancelledError. + propagate_cancellation_error = None + + # Propagate CancelledError if there is one, except if there + # are other errors -- those have priority. + try: + if propagate_cancellation_error is not None and not self._errors: + try: + raise propagate_cancellation_error + finally: + exc = None + finally: + propagate_cancellation_error = None + + if et is not None and not issubclass(et, exceptions.CancelledError): + self._errors.append(exc) + + if self._errors: + # If the parent task is being cancelled from the outside + # of the taskgroup, un-cancel and re-cancel the parent task, + # which will keep the cancel count stable. + if self._parent_task.cancelling(): + self._parent_task.uncancel() + self._parent_task.cancel() + try: + raise BaseExceptionGroup( + 'unhandled errors in a TaskGroup', + self._errors, + ) from None + finally: + exc = None + + + def create_task(self, coro, **kwargs): + """Create a new task in this group and return it. + + Similar to `asyncio.create_task`. + """ + if not self._entered: + coro.close() + raise RuntimeError(f"TaskGroup {self!r} has not been entered") + if self._exiting and not self._tasks: + coro.close() + raise RuntimeError(f"TaskGroup {self!r} is finished") + if self._aborting: + coro.close() + raise RuntimeError(f"TaskGroup {self!r} is shutting down") + task = self._loop.create_task(coro, **kwargs) + + futures.future_add_to_awaited_by(task, self._parent_task) + + # Always schedule the done callback even if the task is + # already done (e.g. if the coro was able to complete eagerly), + # otherwise if the task completes with an exception then it will cancel + # the current task too early. gh-128550, gh-128588 + self._tasks.add(task) + task.add_done_callback(self._on_task_done) + try: + return task + finally: + # gh-128552: prevent a refcycle of + # task.exception().__traceback__->TaskGroup.create_task->task + del task + + # Since Python 3.8 Tasks propagate all exceptions correctly, + # except for KeyboardInterrupt and SystemExit which are + # still considered special. + + def _is_base_error(self, exc: BaseException) -> bool: + assert isinstance(exc, BaseException) + return isinstance(exc, (SystemExit, KeyboardInterrupt)) + + def _abort(self): + self._aborting = True + + for t in self._tasks: + if not t.done(): + t.cancel() + + def _on_task_done(self, task): + self._tasks.discard(task) + + futures.future_discard_from_awaited_by(task, self._parent_task) + + if self._on_completed_fut is not None and not self._tasks: + if not self._on_completed_fut.done(): + self._on_completed_fut.set_result(True) + + if task.cancelled(): + return + + exc = task.exception() + if exc is None: + return + + self._errors.append(exc) + if self._is_base_error(exc) and self._base_error is None: + self._base_error = exc + + if self._parent_task.done(): + # Not sure if this case is possible, but we want to handle + # it anyways. + self._loop.call_exception_handler({ + 'message': f'Task {task!r} has errored out but its parent ' + f'task {self._parent_task} is already completed', + 'exception': exc, + 'task': task, + }) + return + + if not self._aborting and not self._parent_cancel_requested: + # If parent task *is not* being cancelled, it means that we want + # to manually cancel it to abort whatever is being run right now + # in the TaskGroup. But we want to mark parent task as + # "not cancelled" later in __aexit__. Example situation that + # we need to handle: + # + # async def foo(): + # try: + # async with TaskGroup() as g: + # g.create_task(crash_soon()) + # await something # <- this needs to be canceled + # # by the TaskGroup, e.g. + # # foo() needs to be cancelled + # except Exception: + # # Ignore any exceptions raised in the TaskGroup + # pass + # await something_else # this line has to be called + # # after TaskGroup is finished. + self._abort() + self._parent_cancel_requested = True + self._parent_task.cancel() diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 8a8427fe680..fbd5c39a7c5 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -1,24 +1,37 @@ """Support for tasks, coroutines and the scheduler.""" -__all__ = ['Task', 'create_task', - 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', - 'wait', 'wait_for', 'as_completed', 'sleep', 'async', - 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', - 'all_tasks' - ] +__all__ = ( + 'Task', 'create_task', + 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', + 'wait', 'wait_for', 'as_completed', 'sleep', + 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', + 'current_task', 'all_tasks', + 'create_eager_task_factory', 'eager_task_factory', + '_register_task', '_unregister_task', '_enter_task', '_leave_task', +) import concurrent.futures +import contextvars import functools import inspect -import warnings +import itertools +import math +import types import weakref +from types import GenericAlias from . import base_tasks -from . import compat from . import coroutines from . import events +from . import exceptions from . import futures -from .coroutines import coroutine +from . import queues +from . import timeouts + +# Helper to generate new task names +# This uses itertools.count() instead of a "+= 1" operation because the latter +# is not thread safe. See bpo-11866 for a longer explanation. +_task_name_counter = itertools.count(1).__next__ def current_task(loop=None): @@ -32,102 +45,106 @@ def all_tasks(loop=None): """Return a set of all tasks for the loop.""" if loop is None: loop = events.get_running_loop() - return {t for t in _all_tasks - if futures._get_loop(t) is loop and not t.done()} - - -def _all_tasks_compat(loop=None): - # Different from "all_task()" by returning *all* Tasks, including - # the completed ones. Used to implement deprecated "Tasks.all_task()" - # method. - if loop is None: - loop = events.get_event_loop() - return {t for t in _all_tasks if futures._get_loop(t) is loop} + # capturing the set of eager tasks first, so if an eager task "graduates" + # to a regular task in another thread, we don't risk missing it. + eager_tasks = list(_eager_tasks) + return {t for t in itertools.chain(_scheduled_tasks, eager_tasks) + if futures._get_loop(t) is loop and not t.done()} -def _set_task_name(task, name): - if name is not None: - try: - set_name = task.set_name - except AttributeError: - pass - else: - set_name(name) +class Task(futures._PyFuture): # Inherit Python Task implementation + # from a Python Future implementation. -class Task(futures.Future): """A coroutine wrapped in a Future.""" # An important invariant maintained while a Task not done: + # _fut_waiter is either None or a Future. The Future + # can be either done() or not done(). + # The task can be in any of 3 states: # - # - Either _fut_waiter is None, and _step() is scheduled; - # - or _fut_waiter is some Future, and _step() is *not* scheduled. + # - 1: _fut_waiter is not None and not _fut_waiter.done(): + # __step() is *not* scheduled and the Task is waiting for _fut_waiter. + # - 2: (_fut_waiter is None or _fut_waiter.done()) and __step() is scheduled: + # the Task is waiting for __step() to be executed. + # - 3: _fut_waiter is None and __step() is *not* scheduled: + # the Task is currently executing (in __step()). # - # The only transition from the latter to the former is through - # _wakeup(). When _fut_waiter is not None, one of its callbacks - # must be _wakeup(). + # * In state 1, one of the callbacks of __fut_waiter must be __wakeup(). + # * The transition from 1 to 2 happens when _fut_waiter becomes done(), + # as it schedules __wakeup() to be called (which calls __step() so + # we way that __step() is scheduled). + # * It transitions from 2 to 3 when __step() is executed, and it clears + # _fut_waiter to None. + + # If False, don't log a message if the task is destroyed while its + # status is still pending + _log_destroy_pending = True - # Weak set containing all tasks alive. - _all_tasks = weakref.WeakSet() + def __init__(self, coro, *, loop=None, name=None, context=None, + eager_start=False): + super().__init__(loop=loop) + if self._source_traceback: + del self._source_traceback[-1] + if not coroutines.iscoroutine(coro): + # raise after Future.__init__(), attrs are required for __del__ + # prevent logging for pending task in __del__ + self._log_destroy_pending = False + raise TypeError(f"a coroutine was expected, got {coro!r}") + + if name is None: + self._name = f'Task-{_task_name_counter()}' + else: + self._name = str(name) - # Dictionary containing tasks that are currently active in - # all running event loops. {EventLoop: Task} - _current_tasks = {} + self._num_cancels_requested = 0 + self._must_cancel = False + self._fut_waiter = None + self._coro = coro + if context is None: + self._context = contextvars.copy_context() + else: + self._context = context - # If False, don't log a message if the task is destroyed whereas its - # status is still pending - _log_destroy_pending = True + if eager_start and self._loop.is_running(): + self.__eager_start() + else: + self._loop.call_soon(self.__step, context=self._context) + _py_register_task(self) - @classmethod - def current_task(cls, loop=None): - """Return the currently running task in an event loop or None. + def __del__(self): + if self._state == futures._PENDING and self._log_destroy_pending: + context = { + 'task': self, + 'message': 'Task was destroyed but it is pending!', + } + if self._source_traceback: + context['source_traceback'] = self._source_traceback + self._loop.call_exception_handler(context) + super().__del__() - By default the current task for the current event loop is returned. + __class_getitem__ = classmethod(GenericAlias) - None is returned when called not in the context of a Task. - """ - if loop is None: - loop = events.get_event_loop() - return cls._current_tasks.get(loop) + def __repr__(self): + return base_tasks._task_repr(self) - @classmethod - def all_tasks(cls, loop=None): - """Return a set of all tasks for an event loop. + def get_coro(self): + return self._coro - By default all tasks for the current event loop are returned. - """ - if loop is None: - loop = events.get_event_loop() - return {t for t in cls._all_tasks if t._loop is loop} + def get_context(self): + return self._context - def __init__(self, coro, *, loop=None): - assert coroutines.iscoroutine(coro), repr(coro) - super().__init__(loop=loop) - if self._source_traceback: - del self._source_traceback[-1] - self._coro = coro - self._fut_waiter = None - self._must_cancel = False - self._loop.call_soon(self._step) - self.__class__._all_tasks.add(self) - - # On Python 3.3 or older, objects with a destructor that are part of a - # reference cycle are never destroyed. That's not the case any more on - # Python 3.4 thanks to the PEP 442. - if compat.PY34: - def __del__(self): - if self._state == futures._PENDING and self._log_destroy_pending: - context = { - 'task': self, - 'message': 'Task was destroyed but it is pending!', - } - if self._source_traceback: - context['source_traceback'] = self._source_traceback - self._loop.call_exception_handler(context) - futures.Future.__del__(self) - - def _repr_info(self): - return base_tasks._task_repr_info(self) + def get_name(self): + return self._name + + def set_name(self, value): + self._name = str(value) + + def set_result(self, result): + raise RuntimeError('Task does not support set_result operation') + + def set_exception(self, exception): + raise RuntimeError('Task does not support set_exception operation') def get_stack(self, *, limit=None): """Return the list of stack frames for this task's coroutine. @@ -163,7 +180,7 @@ def print_stack(self, *, limit=None, file=None): """ return base_tasks._task_print_stack(self, limit, file) - def cancel(self): + def cancel(self, msg=None): """Request that this task cancel itself. This arranges for a CancelledError to be thrown into the @@ -182,31 +199,89 @@ def cancel(self): task will be marked as cancelled when the wrapped coroutine terminates with a CancelledError exception (even if cancel() was not called). + + This also increases the task's count of cancellation requests. """ + self._log_traceback = False if self.done(): return False + self._num_cancels_requested += 1 + # These two lines are controversial. See discussion starting at + # https://github.com/python/cpython/pull/31394#issuecomment-1053545331 + # Also remember that this is duplicated in _asynciomodule.c. + # if self._num_cancels_requested > 1: + # return False if self._fut_waiter is not None: - if self._fut_waiter.cancel(): + if self._fut_waiter.cancel(msg=msg): # Leave self._fut_waiter; it may be a Task that # catches and ignores the cancellation so we may have # to cancel it again later. return True - # It must be the case that self._step is already scheduled. + # It must be the case that self.__step is already scheduled. self._must_cancel = True + self._cancel_message = msg return True - def _step(self, exc=None): - assert not self.done(), \ - '_step(): already done: {!r}, {!r}'.format(self, exc) + def cancelling(self): + """Return the count of the task's cancellation requests. + + This count is incremented when .cancel() is called + and may be decremented using .uncancel(). + """ + return self._num_cancels_requested + + def uncancel(self): + """Decrement the task's count of cancellation requests. + + This should be called by the party that called `cancel()` on the task + beforehand. + + Returns the remaining number of cancellation requests. + """ + if self._num_cancels_requested > 0: + self._num_cancels_requested -= 1 + if self._num_cancels_requested == 0: + self._must_cancel = False + return self._num_cancels_requested + + def __eager_start(self): + prev_task = _py_swap_current_task(self._loop, self) + try: + _py_register_eager_task(self) + try: + self._context.run(self.__step_run_and_handle_result, None) + finally: + _py_unregister_eager_task(self) + finally: + try: + curtask = _py_swap_current_task(self._loop, prev_task) + assert curtask is self + finally: + if self.done(): + self._coro = None + self = None # Needed to break cycles when an exception occurs. + else: + _py_register_task(self) + + def __step(self, exc=None): + if self.done(): + raise exceptions.InvalidStateError( + f'__step(): already done: {self!r}, {exc!r}') if self._must_cancel: - if not isinstance(exc, futures.CancelledError): - exc = futures.CancelledError() + if not isinstance(exc, exceptions.CancelledError): + exc = self._make_cancelled_error() self._must_cancel = False - coro = self._coro self._fut_waiter = None - self.__class__._current_tasks[self._loop] = self - # Call either coro.throw(exc) or coro.send(None). + _py_enter_task(self._loop, self) + try: + self.__step_run_and_handle_result(exc) + finally: + _py_leave_task(self._loop, self) + self = None # Needed to break cycles when an exception occurs. + + def __step_run_and_handle_result(self, exc): + coro = self._coro try: if exc is None: # We use the `send` method directly, because coroutines @@ -215,79 +290,87 @@ def _step(self, exc=None): else: result = coro.throw(exc) except StopIteration as exc: - self.set_result(exc.value) - except futures.CancelledError: + if self._must_cancel: + # Task is cancelled right before coro stops. + self._must_cancel = False + super().cancel(msg=self._cancel_message) + else: + super().set_result(exc.value) + except exceptions.CancelledError as exc: + # Save the original exception so we can chain it later. + self._cancelled_exc = exc super().cancel() # I.e., Future.cancel(self). - except Exception as exc: - self.set_exception(exc) - except BaseException as exc: - self.set_exception(exc) + except (KeyboardInterrupt, SystemExit) as exc: + super().set_exception(exc) raise + except BaseException as exc: + super().set_exception(exc) else: blocking = getattr(result, '_asyncio_future_blocking', None) if blocking is not None: # Yielded Future must come from Future.__iter__(). - if result._loop is not self._loop: + if futures._get_loop(result) is not self._loop: + new_exc = RuntimeError( + f'Task {self!r} got Future ' + f'{result!r} attached to a different loop') self._loop.call_soon( - self._step, - RuntimeError( - 'Task {!r} got Future {!r} attached to a ' - 'different loop'.format(self, result))) + self.__step, new_exc, context=self._context) elif blocking: if result is self: + new_exc = RuntimeError( + f'Task cannot await on itself: {self!r}') self._loop.call_soon( - self._step, - RuntimeError( - 'Task cannot await on itself: {!r}'.format( - self))) + self.__step, new_exc, context=self._context) else: + futures.future_add_to_awaited_by(result, self) result._asyncio_future_blocking = False - result.add_done_callback(self._wakeup) + result.add_done_callback( + self.__wakeup, context=self._context) self._fut_waiter = result if self._must_cancel: - if self._fut_waiter.cancel(): + if self._fut_waiter.cancel( + msg=self._cancel_message): self._must_cancel = False else: + new_exc = RuntimeError( + f'yield was used instead of yield from ' + f'in task {self!r} with {result!r}') self._loop.call_soon( - self._step, - RuntimeError( - 'yield was used instead of yield from ' - 'in task {!r} with {!r}'.format(self, result))) + self.__step, new_exc, context=self._context) + elif result is None: # Bare yield relinquishes control for one event loop iteration. - self._loop.call_soon(self._step) + self._loop.call_soon(self.__step, context=self._context) elif inspect.isgenerator(result): # Yielding a generator is just wrong. + new_exc = RuntimeError( + f'yield was used instead of yield from for ' + f'generator in task {self!r} with {result!r}') self._loop.call_soon( - self._step, - RuntimeError( - 'yield was used instead of yield from for ' - 'generator in task {!r} with {}'.format( - self, result))) + self.__step, new_exc, context=self._context) else: # Yielding something else is an error. + new_exc = RuntimeError(f'Task got bad yield: {result!r}') self._loop.call_soon( - self._step, - RuntimeError( - 'Task got bad yield: {!r}'.format(result))) + self.__step, new_exc, context=self._context) finally: - self.__class__._current_tasks.pop(self._loop) self = None # Needed to break cycles when an exception occurs. - def _wakeup(self, future): + def __wakeup(self, future): + futures.future_discard_from_awaited_by(future, self) try: future.result() - except Exception as exc: + except BaseException as exc: # This may also be a cancellation. - self._step(exc) + self.__step(exc) else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, + # If we call `__step(value, None)` instead of `__step()`, # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. - self._step() + self.__step() self = None # Needed to break cycles when an exception occurs. @@ -303,15 +386,13 @@ def _wakeup(self, future): Task = _CTask = _asyncio.Task -def create_task(coro, *, name=None): +def create_task(coro, **kwargs): """Schedule the execution of a coroutine object in a spawn task. Return a Task object. """ loop = events.get_running_loop() - task = loop.create_task(coro) - _set_task_name(task, name) - return task + return loop.create_task(coro, **kwargs) # wait() and as_completed() similar to those in PEP 3148. @@ -321,36 +402,34 @@ def create_task(coro, *, name=None): ALL_COMPLETED = concurrent.futures.ALL_COMPLETED -@coroutine -def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): - """Wait for the Futures and coroutines given by fs to complete. - - The sequence futures must not be empty. +async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED): + """Wait for the Futures or Tasks given by fs to complete. - Coroutines will be wrapped in Tasks. + The fs iterable must not be empty. Returns two sets of Future: (done, pending). Usage: - done, pending = yield from asyncio.wait(fs) + done, pending = await asyncio.wait(fs) Note: This does not raise TimeoutError! Futures that aren't done when the timeout occurs are returned in the second set. """ if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError("expect a list of futures, not %s" % type(fs).__name__) + raise TypeError(f"expect a list of futures, not {type(fs).__name__}") if not fs: - raise ValueError('Set of coroutines/Futures is empty.') + raise ValueError('Set of Tasks/Futures is empty.') if return_when not in (FIRST_COMPLETED, FIRST_EXCEPTION, ALL_COMPLETED): - raise ValueError('Invalid return_when value: {}'.format(return_when)) + raise ValueError(f'Invalid return_when value: {return_when}') - if loop is None: - loop = events.get_event_loop() + fs = set(fs) - fs = {ensure_future(f, loop=loop) for f in set(fs)} + if any(coroutines.iscoroutine(f) for f in fs): + raise TypeError("Passing coroutines is forbidden, use tasks explicitly.") - return (yield from _wait(fs, timeout, return_when, loop)) + loop = events.get_running_loop() + return await _wait(fs, timeout, return_when, loop) def _release_waiter(waiter, *args): @@ -358,8 +437,7 @@ def _release_waiter(waiter, *args): waiter.set_result(None) -@coroutine -def wait_for(fut, timeout, *, loop=None): +async def wait_for(fut, timeout): """Wait for the single Future or coroutine to complete, with timeout. Coroutine will be wrapped in Task. @@ -370,43 +448,47 @@ def wait_for(fut, timeout, *, loop=None): If the wait is cancelled, the task is also cancelled. + If the task suppresses the cancellation and returns a value instead, + that value is returned. + This function is a coroutine. """ - if loop is None: - loop = events.get_event_loop() + # The special case for timeout <= 0 is for the following case: + # + # async def test_waitfor(): + # func_started = False + # + # async def func(): + # nonlocal func_started + # func_started = True + # + # try: + # await asyncio.wait_for(func(), 0) + # except asyncio.TimeoutError: + # assert not func_started + # else: + # assert False + # + # asyncio.run(test_waitfor()) - if timeout is None: - return (yield from fut) - waiter = loop.create_future() - timeout_handle = loop.call_later(timeout, _release_waiter, waiter) - cb = functools.partial(_release_waiter, waiter) + if timeout is not None and timeout <= 0: + fut = ensure_future(fut) - fut = ensure_future(fut, loop=loop) - fut.add_done_callback(cb) + if fut.done(): + return fut.result() - try: - # wait until the future completes or the timeout + await _cancel_and_wait(fut) try: - yield from waiter - except futures.CancelledError: - fut.remove_done_callback(cb) - fut.cancel() - raise - - if fut.done(): return fut.result() - else: - fut.remove_done_callback(cb) - fut.cancel() - raise futures.TimeoutError() - finally: - timeout_handle.cancel() + except exceptions.CancelledError as exc: + raise TimeoutError from exc + async with timeouts.timeout(timeout): + return await fut -@coroutine -def _wait(fs, timeout, return_when, loop): - """Internal helper for wait() and wait_for(). +async def _wait(fs, timeout, return_when, loop): + """Internal helper for wait(). The fs argument must be a collection of Futures. """ @@ -416,6 +498,7 @@ def _wait(fs, timeout, return_when, loop): if timeout is not None: timeout_handle = loop.call_later(timeout, _release_waiter, waiter) counter = len(fs) + cur_task = current_task() def _on_completion(f): nonlocal counter @@ -428,19 +511,22 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) + futures.future_discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) + futures.future_add_to_awaited_by(f, cur_task) try: - yield from waiter + await waiter finally: if timeout_handle is not None: timeout_handle.cancel() + for f in fs: + f.remove_done_callback(_on_completion) done, pending = set(), set() for f in fs: - f.remove_done_callback(_on_completion) if f.done(): done.add(f) else: @@ -448,99 +534,174 @@ def _on_completion(f): return done, pending -# This is *not* a @coroutine! It is just an iterator (yielding Futures). -def as_completed(fs, *, loop=None, timeout=None): - """Return an iterator whose values are coroutines. +async def _cancel_and_wait(fut): + """Cancel the *fut* future or task and wait until it completes.""" - When waiting for the yielded coroutines you'll get the results (or - exceptions!) of the original Futures (or coroutines), in the order - in which and as soon as they complete. + loop = events.get_running_loop() + waiter = loop.create_future() + cb = functools.partial(_release_waiter, waiter) + fut.add_done_callback(cb) - This differs from PEP 3148; the proper way to use this is: + try: + fut.cancel() + # We cannot wait on *fut* directly to make + # sure _cancel_and_wait itself is reliably cancellable. + await waiter + finally: + fut.remove_done_callback(cb) - for f in as_completed(fs): - result = yield from f # The 'yield from' may raise. - # Use result. - If a timeout is specified, the 'yield from' will raise - TimeoutError when the timeout occurs before all Futures are done. +class _AsCompletedIterator: + """Iterator of awaitables representing tasks of asyncio.as_completed. - Note: The futures 'f' are not necessarily members of fs. + As an asynchronous iterator, iteration yields futures as they finish. As a + plain iterator, new coroutines are yielded that will return or raise the + result of the next underlying future to complete. """ - if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError("expect a list of futures, not %s" % type(fs).__name__) - loop = loop if loop is not None else events.get_event_loop() - todo = {ensure_future(f, loop=loop) for f in set(fs)} - from .queues import Queue # Import here to avoid circular import problem. - done = Queue(loop=loop) - timeout_handle = None + def __init__(self, aws, timeout): + self._done = queues.Queue() + self._timeout_handle = None - def _on_timeout(): + loop = events.get_event_loop() + todo = {ensure_future(aw, loop=loop) for aw in set(aws)} for f in todo: - f.remove_done_callback(_on_completion) - done.put_nowait(None) # Queue a dummy value for _wait_for_one(). - todo.clear() # Can't do todo.remove(f) in the loop. + f.add_done_callback(self._handle_completion) + if todo and timeout is not None: + self._timeout_handle = ( + loop.call_later(timeout, self._handle_timeout) + ) + self._todo = todo + self._todo_left = len(todo) + + def __aiter__(self): + return self + + def __iter__(self): + return self + + async def __anext__(self): + if not self._todo_left: + raise StopAsyncIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return await self._wait_for_one() + + def __next__(self): + if not self._todo_left: + raise StopIteration + assert self._todo_left > 0 + self._todo_left -= 1 + return self._wait_for_one(resolve=True) + + def _handle_timeout(self): + for f in self._todo: + f.remove_done_callback(self._handle_completion) + self._done.put_nowait(None) # Sentinel for _wait_for_one(). + self._todo.clear() # Can't do todo.remove(f) in the loop. + + def _handle_completion(self, f): + if not self._todo: + return # _handle_timeout() was here first. + self._todo.remove(f) + self._done.put_nowait(f) + if not self._todo and self._timeout_handle is not None: + self._timeout_handle.cancel() + + async def _wait_for_one(self, resolve=False): + # Wait for the next future to be done and return it unless resolve is + # set, in which case return either the result of the future or raise + # an exception. + f = await self._done.get() + if f is None: + # Dummy value from _handle_timeout(). + raise exceptions.TimeoutError + return f.result() if resolve else f - def _on_completion(f): - if not todo: - return # _on_timeout() was here first. - todo.remove(f) - done.put_nowait(f) - if not todo and timeout_handle is not None: - timeout_handle.cancel() - @coroutine - def _wait_for_one(): - f = yield from done.get() - if f is None: - # Dummy value from _on_timeout(). - raise futures.TimeoutError - return f.result() # May raise f.exception(). +def as_completed(fs, *, timeout=None): + """Create an iterator of awaitables or their results in completion order. - for f in todo: - f.add_done_callback(_on_completion) - if todo and timeout is not None: - timeout_handle = loop.call_later(timeout, _on_timeout) - for _ in range(len(todo)): - yield _wait_for_one() + Run the supplied awaitables concurrently. The returned object can be + iterated to obtain the results of the awaitables as they finish. + The object returned can be iterated as an asynchronous iterator or a plain + iterator. When asynchronous iteration is used, the originally-supplied + awaitables are yielded if they are tasks or futures. This makes it easy to + correlate previously-scheduled tasks with their results: -@coroutine -def sleep(delay, result=None, *, loop=None): - """Coroutine that completes after a given time (in seconds).""" - if delay == 0: - yield - return result + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] - if loop is None: - loop = events.get_event_loop() - future = loop.create_future() - h = future._loop.call_later(delay, - futures._set_result_unless_cancelled, - future, result) - try: - return (yield from future) - finally: - h.cancel() + async for earliest_connect in as_completed(tasks): + # earliest_connect is done. The result can be obtained by + # awaiting it or calling earliest_connect.result() + reader, writer = await earliest_connect + if earliest_connect is ipv6_connect: + print("IPv6 connection established.") + else: + print("IPv4 connection established.") -def async_(coro_or_future, *, loop=None): - """Wrap a coroutine in a future. + During asynchronous iteration, implicitly-created tasks will be yielded for + supplied awaitables that aren't tasks or futures. + + When used as a plain iterator, each iteration yields a new coroutine that + returns the result or raises the exception of the next completed awaitable. + This pattern is compatible with Python versions older than 3.13: + + ipv4_connect = create_task(open_connection("127.0.0.1", 80)) + ipv6_connect = create_task(open_connection("::1", 80)) + tasks = [ipv4_connect, ipv6_connect] + + for next_connect in as_completed(tasks): + # next_connect is not one of the original task objects. It must be + # awaited to obtain the result value or raise the exception of the + # awaitable that finishes next. + reader, writer = await next_connect + + A TimeoutError is raised if the timeout occurs before all awaitables are + done. This is raised by the async for loop during asynchronous iteration or + by the coroutines yielded during plain iteration. + """ + if inspect.isawaitable(fs): + raise TypeError( + f"expects an iterable of awaitables, not {type(fs).__name__}" + ) + + return _AsCompletedIterator(fs, timeout) - If the argument is a Future, it is returned directly. - This function is deprecated in 3.5. Use asyncio.ensure_future() instead. +@types.coroutine +def __sleep0(): + """Skip one event loop run cycle. + + This is a private helper for 'asyncio.sleep()', used + when the 'delay' is set to 0. It uses a bare 'yield' + expression (which Task.__step knows how to handle) + instead of creating a Future object. """ + yield + - warnings.warn("asyncio.async() function is deprecated, use ensure_future()", - DeprecationWarning) +async def sleep(delay, result=None): + """Coroutine that completes after a given time (in seconds).""" + if delay <= 0: + await __sleep0() + return result - return ensure_future(coro_or_future, loop=loop) + if math.isnan(delay): + raise ValueError("Invalid delay: NaN (not a number)") -# Silence DeprecationWarning: -globals()['async'] = async_ -async_.__name__ = 'async' -del async_ + loop = events.get_running_loop() + future = loop.create_future() + h = loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) + try: + return await future + finally: + h.cancel() def ensure_future(coro_or_future, *, loop=None): @@ -549,30 +710,30 @@ def ensure_future(coro_or_future, *, loop=None): If the argument is a Future, it is returned directly. """ if futures.isfuture(coro_or_future): - if loop is not None and loop is not coro_or_future._loop: - raise ValueError('loop argument must agree with Future') + if loop is not None and loop is not futures._get_loop(coro_or_future): + raise ValueError('The future belongs to a different loop than ' + 'the one specified as the loop argument') return coro_or_future - elif coroutines.iscoroutine(coro_or_future): - if loop is None: - loop = events.get_event_loop() - task = loop.create_task(coro_or_future) - if task._source_traceback: - del task._source_traceback[-1] - return task - elif compat.PY35 and inspect.isawaitable(coro_or_future): - return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) - else: - raise TypeError('A Future, a coroutine or an awaitable is required') - - -@coroutine -def _wrap_awaitable(awaitable): - """Helper for asyncio.ensure_future(). + should_close = True + if not coroutines.iscoroutine(coro_or_future): + if inspect.isawaitable(coro_or_future): + async def _wrap_awaitable(awaitable): + return await awaitable + + coro_or_future = _wrap_awaitable(coro_or_future) + should_close = False + else: + raise TypeError('An asyncio.Future, a coroutine or an awaitable ' + 'is required') - Wraps awaitable (an object with __await__) into a coroutine - that will later be wrapped in a Task by ensure_future(). - """ - return (yield from awaitable.__await__()) + if loop is None: + loop = events.get_event_loop() + try: + return loop.create_task(coro_or_future) + except RuntimeError: + if should_close: + coro_or_future.close() + raise class _GatheringFuture(futures.Future): @@ -583,23 +744,29 @@ class _GatheringFuture(futures.Future): cancelled. """ - def __init__(self, children, *, loop=None): + def __init__(self, children, *, loop): + assert loop is not None super().__init__(loop=loop) self._children = children + self._cancel_requested = False - def cancel(self): + def cancel(self, msg=None): if self.done(): return False ret = False for child in self._children: - if child.cancel(): + if child.cancel(msg=msg): ret = True + if ret: + # If any child tasks were actually cancelled, we should + # propagate the cancellation request regardless of + # *return_exceptions* argument. See issue 32684. + self._cancel_requested = True return ret -def gather(*coros_or_futures, loop=None, return_exceptions=False): - """Return a future aggregating results from the given coroutines - or futures. +def gather(*coros_or_futures, return_exceptions=False): + """Return a future aggregating results from the given coroutines/futures. Coroutines will be wrapped in a future and scheduled in the event loop. They will not necessarily be scheduled in the same order as @@ -620,77 +787,157 @@ def gather(*coros_or_futures, loop=None, return_exceptions=False): the outer Future is *not* cancelled in this case. (This is to prevent the cancellation of one child to cause other children to be cancelled.) + + If *return_exceptions* is False, cancelling gather() after it + has been marked done won't cancel any submitted awaitables. + For instance, gather can be marked done after propagating an + exception to the caller, therefore, calling ``gather.cancel()`` + after catching an exception (raised by one of the awaitables) from + gather won't cancel any other awaitables. """ if not coros_or_futures: - if loop is None: - loop = events.get_event_loop() + loop = events.get_event_loop() outer = loop.create_future() outer.set_result([]) return outer - arg_to_fut = {} - for arg in set(coros_or_futures): - if not futures.isfuture(arg): - fut = ensure_future(arg, loop=loop) - if loop is None: - loop = fut._loop - # The caller cannot control this future, the "destroy pending task" - # warning should not be emitted. - fut._log_destroy_pending = False - else: - fut = arg - if loop is None: - loop = fut._loop - elif fut._loop is not loop: - raise ValueError("futures are tied to different event loops") - arg_to_fut[arg] = fut - - children = [arg_to_fut[arg] for arg in coros_or_futures] - nchildren = len(children) - outer = _GatheringFuture(children, loop=loop) - nfinished = 0 - results = [None] * nchildren + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None - def _done_callback(i, fut): + def _done_callback(fut, cur_task=cur_task): nonlocal nfinished - if outer.done(): + nfinished += 1 + + if cur_task is not None: + futures.future_discard_from_awaited_by(fut, cur_task) + + if outer is None or outer.done(): if not fut.cancelled(): # Mark exception retrieved. fut.exception() return - if fut.cancelled(): - res = futures.CancelledError() - if not return_exceptions: - outer.set_exception(res) - return - elif fut._exception is not None: - res = fut.exception() # Mark exception retrieved. - if not return_exceptions: - outer.set_exception(res) + if not return_exceptions: + if fut.cancelled(): + # Check if 'fut' is cancelled first, as + # 'fut.exception()' will *raise* a CancelledError + # instead of returning it. + exc = fut._make_cancelled_error() + outer.set_exception(exc) return + else: + exc = fut.exception() + if exc is not None: + outer.set_exception(exc) + return + + if nfinished == nfuts: + # All futures are done; create a list of results + # and set it to the 'outer' future. + results = [] + + for fut in children: + if fut.cancelled(): + # Check if 'fut' is cancelled first, as 'fut.exception()' + # will *raise* a CancelledError instead of returning it. + # Also, since we're adding the exception return value + # to 'results' instead of raising it, don't bother + # setting __context__. This also lets us preserve + # calling '_make_cancelled_error()' at most once. + res = exceptions.CancelledError( + '' if fut._cancel_message is None else + fut._cancel_message) + else: + res = fut.exception() + if res is None: + res = fut.result() + results.append(res) + + if outer._cancel_requested: + # If gather is being cancelled we must propagate the + # cancellation regardless of *return_exceptions* argument. + # See issue 32684. + exc = fut._make_cancelled_error() + outer.set_exception(exc) + else: + outer.set_result(results) + + arg_to_fut = {} + children = [] + nfuts = 0 + nfinished = 0 + done_futs = [] + outer = None # bpo-46672 + for arg in coros_or_futures: + if arg not in arg_to_fut: + fut = ensure_future(arg, loop=loop) + if loop is None: + loop = futures._get_loop(fut) + if fut is not arg: + # 'arg' was not a Future, therefore, 'fut' is a new + # Future created specifically for 'arg'. Since the caller + # can't control it, disable the "destroy pending task" + # warning. + fut._log_destroy_pending = False + nfuts += 1 + arg_to_fut[arg] = fut + if fut.done(): + done_futs.append(fut) + else: + if cur_task is not None: + futures.future_add_to_awaited_by(fut, cur_task) + fut.add_done_callback(_done_callback) + else: - res = fut._result - results[i] = res - nfinished += 1 - if nfinished == nchildren: - outer.set_result(results) + # There's a duplicate Future object in coros_or_futures. + fut = arg_to_fut[arg] + + children.append(fut) - for i, fut in enumerate(children): - fut.add_done_callback(functools.partial(_done_callback, i)) + outer = _GatheringFuture(children, loop=loop) + # Run done callbacks after GatheringFuture created so any post-processing + # can be performed at this point + # optimization: in the special case that *all* futures finished eagerly, + # this will effectively complete the gather eagerly, with the last + # callback setting the result (or exception) on outer before returning it + for fut in done_futs: + _done_callback(fut) return outer -def shield(arg, *, loop=None): +def _log_on_exception(fut): + if fut.cancelled(): + return + + exc = fut.exception() + if exc is None: + return + + context = { + 'message': + f'{exc.__class__.__name__} exception in shielded future', + 'exception': exc, + 'future': fut, + } + if fut._source_traceback: + context['source_traceback'] = fut._source_traceback + fut._loop.call_exception_handler(context) + + +def shield(arg): """Wait for a future, shielding it from cancellation. The statement - res = yield from shield(something()) + task = asyncio.create_task(something()) + res = await shield(task) is exactly equivalent to the statement - res = yield from something() + res = await something() *except* that if the coroutine containing it is cancelled, the task running in something() is not cancelled. From the POV of @@ -702,23 +949,34 @@ def shield(arg, *, loop=None): If you want to completely ignore cancellation (not recommended) you can combine shield() with a try/except clause, as follows: + task = asyncio.create_task(something()) try: - res = yield from shield(something()) + res = await shield(task) except CancelledError: res = None + + Save a reference to tasks passed to this function, to avoid + a task disappearing mid-execution. The event loop only keeps + weak references to tasks. A task that isn't referenced elsewhere + may get garbage collected at any time, even before it's done. """ - inner = ensure_future(arg, loop=loop) + inner = ensure_future(arg) if inner.done(): # Shortcut. return inner - loop = inner._loop + loop = futures._get_loop(inner) outer = loop.create_future() - def _done_callback(inner): + if loop is not None and (cur_task := current_task(loop)) is not None: + futures.future_add_to_awaited_by(inner, cur_task) + else: + cur_task = None + + def _clear_awaited_by_callback(inner): + futures.future_discard_from_awaited_by(inner, cur_task) + + def _inner_done_callback(inner): if outer.cancelled(): - if not inner.cancelled(): - # Mark inner's result as retrieved. - inner.exception() return if inner.cancelled(): @@ -730,7 +988,19 @@ def _done_callback(inner): else: outer.set_result(inner.result()) - inner.add_done_callback(_done_callback) + def _outer_done_callback(outer): + if not inner.done(): + inner.remove_done_callback(_inner_done_callback) + # Keep only one callback to log on cancel + inner.remove_done_callback(_log_on_exception) + inner.add_done_callback(_log_on_exception) + + if cur_task is not None: + inner.add_done_callback(_clear_awaited_by_callback) + + + inner.add_done_callback(_inner_done_callback) + outer.add_done_callback(_outer_done_callback) return outer @@ -746,7 +1016,9 @@ def run_coroutine_threadsafe(coro, loop): def callback(): try: futures._chain_future(ensure_future(coro, loop=loop), future) - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: if future.set_running_or_notify_cancel(): future.set_exception(exc) raise @@ -755,8 +1027,40 @@ def callback(): return future -# WeakSet containing all alive tasks. -_all_tasks = weakref.WeakSet() +def create_eager_task_factory(custom_task_constructor): + """Create a function suitable for use as a task factory on an event-loop. + + Example usage: + + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) + + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any callable + that returns a Task-compatible object and has a signature compatible + with `Task.__init__`; it must have the `eager_start` keyword argument. + + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ + + def factory(loop, coro, *, eager_start=True, **kwargs): + return custom_task_constructor( + coro, loop=loop, eager_start=eager_start, **kwargs) + + return factory + + +eager_task_factory = create_eager_task_factory(Task) + + +# Collectively these two sets hold references to the complete set of active +# tasks. Eagerly executed tasks use a faster regular set as an optimization +# but may graduate to a WeakSet if the task blocks on IO. +_scheduled_tasks = weakref.WeakSet() +_eager_tasks = set() # Dictionary containing tasks that are currently active in # all running event loops. {EventLoop: Task} @@ -764,8 +1068,13 @@ def callback(): def _register_task(task): - """Register a new task in asyncio as executed by loop.""" - _all_tasks.add(task) + """Register an asyncio Task scheduled to run on an event loop.""" + _scheduled_tasks.add(task) + + +def _register_eager_task(task): + """Register an asyncio Task about to be eagerly executed.""" + _eager_tasks.add(task) def _enter_task(loop, task): @@ -784,25 +1093,49 @@ def _leave_task(loop, task): del _current_tasks[loop] +def _swap_current_task(loop, task): + prev_task = _current_tasks.get(loop) + if task is None: + del _current_tasks[loop] + else: + _current_tasks[loop] = task + return prev_task + + def _unregister_task(task): - """Unregister a task.""" - _all_tasks.discard(task) + """Unregister a completed, scheduled Task.""" + _scheduled_tasks.discard(task) + +def _unregister_eager_task(task): + """Unregister a task which finished its first eager step.""" + _eager_tasks.discard(task) + +_py_current_task = current_task _py_register_task = _register_task +_py_register_eager_task = _register_eager_task _py_unregister_task = _unregister_task +_py_unregister_eager_task = _unregister_eager_task _py_enter_task = _enter_task _py_leave_task = _leave_task - +_py_swap_current_task = _swap_current_task +_py_all_tasks = all_tasks try: - from _asyncio import (_register_task, _unregister_task, - _enter_task, _leave_task, - _all_tasks, _current_tasks) + from _asyncio import (_register_task, _register_eager_task, + _unregister_task, _unregister_eager_task, + _enter_task, _leave_task, _swap_current_task, + current_task, all_tasks) except ImportError: pass else: + _c_current_task = current_task _c_register_task = _register_task + _c_register_eager_task = _register_eager_task _c_unregister_task = _unregister_task + _c_unregister_eager_task = _unregister_eager_task _c_enter_task = _enter_task _c_leave_task = _leave_task + _c_swap_current_task = _swap_current_task + _c_all_tasks = all_tasks diff --git a/Lib/asyncio/test_utils.py b/Lib/asyncio/test_utils.py deleted file mode 100644 index 99e3839f456..00000000000 --- a/Lib/asyncio/test_utils.py +++ /dev/null @@ -1,503 +0,0 @@ -"""Utilities shared by tests.""" - -import collections -import contextlib -import io -import logging -import os -import re -import socket -import socketserver -import sys -import tempfile -import threading -import time -import unittest -import weakref - -from unittest import mock - -from http.server import HTTPServer -from wsgiref.simple_server import WSGIRequestHandler, WSGIServer - -try: - import ssl -except ImportError: # pragma: no cover - ssl = None - -from . import base_events -from . import compat -from . import events -from . import futures -from . import selectors -from . import tasks -from .coroutines import coroutine -from .log import logger - - -if sys.platform == 'win32': # pragma: no cover - from .windows_utils import socketpair -else: - from socket import socketpair # pragma: no cover - - -def dummy_ssl_context(): - if ssl is None: - return None - else: - return ssl.SSLContext(ssl.PROTOCOL_SSLv23) - - -def run_briefly(loop): - @coroutine - def once(): - pass - gen = once() - t = loop.create_task(gen) - # Don't log a warning if the task is not done after run_until_complete(). - # It occurs if the loop is stopped or if a task raises a BaseException. - t._log_destroy_pending = False - try: - loop.run_until_complete(t) - finally: - gen.close() - - -def run_until(loop, pred, timeout=30): - deadline = time.time() + timeout - while not pred(): - if timeout is not None: - timeout = deadline - time.time() - if timeout <= 0: - raise futures.TimeoutError() - loop.run_until_complete(tasks.sleep(0.001, loop=loop)) - - -def run_once(loop): - """Legacy API to run once through the event loop. - - This is the recommended pattern for test code. It will poll the - selector once and run all callbacks scheduled in response to I/O - events. - """ - loop.call_soon(loop.stop) - loop.run_forever() - - -class SilentWSGIRequestHandler(WSGIRequestHandler): - - def get_stderr(self): - return io.StringIO() - - def log_message(self, format, *args): - pass - - -class SilentWSGIServer(WSGIServer): - - request_timeout = 2 - - def get_request(self): - request, client_addr = super().get_request() - request.settimeout(self.request_timeout) - return request, client_addr - - def handle_error(self, request, client_address): - pass - - -class SSLWSGIServerMixin: - - def finish_request(self, request, client_address): - # The relative location of our test directory (which - # contains the ssl key and certificate files) differs - # between the stdlib and stand-alone asyncio. - # Prefer our own if we can find it. - here = os.path.join(os.path.dirname(__file__), '..', 'tests') - if not os.path.isdir(here): - here = os.path.join(os.path.dirname(os.__file__), - 'test', 'test_asyncio') - keyfile = os.path.join(here, 'ssl_key.pem') - certfile = os.path.join(here, 'ssl_cert.pem') - context = ssl.SSLContext() - context.load_cert_chain(certfile, keyfile) - - ssock = context.wrap_socket(request, server_side=True) - try: - self.RequestHandlerClass(ssock, client_address, self) - ssock.close() - except OSError: - # maybe socket has been closed by peer - pass - - -class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer): - pass - - -def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls): - - def app(environ, start_response): - status = '200 OK' - headers = [('Content-type', 'text/plain')] - start_response(status, headers) - return [b'Test message'] - - # Run the test WSGI server in a separate thread in order not to - # interfere with event handling in the main thread - server_class = server_ssl_cls if use_ssl else server_cls - httpd = server_class(address, SilentWSGIRequestHandler) - httpd.set_app(app) - httpd.address = httpd.server_address - server_thread = threading.Thread( - target=lambda: httpd.serve_forever(poll_interval=0.05)) - server_thread.start() - try: - yield httpd - finally: - httpd.shutdown() - httpd.server_close() - server_thread.join() - - -if hasattr(socket, 'AF_UNIX'): - - class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer): - - def server_bind(self): - socketserver.UnixStreamServer.server_bind(self) - self.server_name = '127.0.0.1' - self.server_port = 80 - - - class UnixWSGIServer(UnixHTTPServer, WSGIServer): - - request_timeout = 2 - - def server_bind(self): - UnixHTTPServer.server_bind(self) - self.setup_environ() - - def get_request(self): - request, client_addr = super().get_request() - request.settimeout(self.request_timeout) - # Code in the stdlib expects that get_request - # will return a socket and a tuple (host, port). - # However, this isn't true for UNIX sockets, - # as the second return value will be a path; - # hence we return some fake data sufficient - # to get the tests going - return request, ('127.0.0.1', '') - - - class SilentUnixWSGIServer(UnixWSGIServer): - - def handle_error(self, request, client_address): - pass - - - class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer): - pass - - - def gen_unix_socket_path(): - with tempfile.NamedTemporaryFile() as file: - return file.name - - - @contextlib.contextmanager - def unix_socket_path(): - path = gen_unix_socket_path() - try: - yield path - finally: - try: - os.unlink(path) - except OSError: - pass - - - @contextlib.contextmanager - def run_test_unix_server(*, use_ssl=False): - with unix_socket_path() as path: - yield from _run_test_server(address=path, use_ssl=use_ssl, - server_cls=SilentUnixWSGIServer, - server_ssl_cls=UnixSSLWSGIServer) - - -@contextlib.contextmanager -def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False): - yield from _run_test_server(address=(host, port), use_ssl=use_ssl, - server_cls=SilentWSGIServer, - server_ssl_cls=SSLWSGIServer) - - -def make_test_protocol(base): - dct = {} - for name in dir(base): - if name.startswith('__') and name.endswith('__'): - # skip magic names - continue - dct[name] = MockCallback(return_value=None) - return type('TestProtocol', (base,) + base.__bases__, dct)() - - -class TestSelector(selectors.BaseSelector): - - def __init__(self): - self.keys = {} - - def register(self, fileobj, events, data=None): - key = selectors.SelectorKey(fileobj, 0, events, data) - self.keys[fileobj] = key - return key - - def unregister(self, fileobj): - return self.keys.pop(fileobj) - - def select(self, timeout): - return [] - - def get_map(self): - return self.keys - - -class TestLoop(base_events.BaseEventLoop): - """Loop for unittests. - - It manages self time directly. - If something scheduled to be executed later then - on next loop iteration after all ready handlers done - generator passed to __init__ is calling. - - Generator should be like this: - - def gen(): - ... - when = yield ... - ... = yield time_advance - - Value returned by yield is absolute time of next scheduled handler. - Value passed to yield is time advance to move loop's time forward. - """ - - def __init__(self, gen=None): - super().__init__() - - if gen is None: - def gen(): - yield - self._check_on_close = False - else: - self._check_on_close = True - - self._gen = gen() - next(self._gen) - self._time = 0 - self._clock_resolution = 1e-9 - self._timers = [] - self._selector = TestSelector() - - self.readers = {} - self.writers = {} - self.reset_counters() - - self._transports = weakref.WeakValueDictionary() - - def time(self): - return self._time - - def advance_time(self, advance): - """Move test time forward.""" - if advance: - self._time += advance - - def close(self): - super().close() - if self._check_on_close: - try: - self._gen.send(0) - except StopIteration: - pass - else: # pragma: no cover - raise AssertionError("Time generator is not finished") - - def _add_reader(self, fd, callback, *args): - self.readers[fd] = events.Handle(callback, args, self) - - def _remove_reader(self, fd): - self.remove_reader_count[fd] += 1 - if fd in self.readers: - del self.readers[fd] - return True - else: - return False - - def assert_reader(self, fd, callback, *args): - assert fd in self.readers, 'fd {} is not registered'.format(fd) - handle = self.readers[fd] - assert handle._callback == callback, '{!r} != {!r}'.format( - handle._callback, callback) - assert handle._args == args, '{!r} != {!r}'.format( - handle._args, args) - - def _add_writer(self, fd, callback, *args): - self.writers[fd] = events.Handle(callback, args, self) - - def _remove_writer(self, fd): - self.remove_writer_count[fd] += 1 - if fd in self.writers: - del self.writers[fd] - return True - else: - return False - - def assert_writer(self, fd, callback, *args): - assert fd in self.writers, 'fd {} is not registered'.format(fd) - handle = self.writers[fd] - assert handle._callback == callback, '{!r} != {!r}'.format( - handle._callback, callback) - assert handle._args == args, '{!r} != {!r}'.format( - handle._args, args) - - def _ensure_fd_no_transport(self, fd): - try: - transport = self._transports[fd] - except KeyError: - pass - else: - raise RuntimeError( - 'File descriptor {!r} is used by transport {!r}'.format( - fd, transport)) - - def add_reader(self, fd, callback, *args): - """Add a reader callback.""" - self._ensure_fd_no_transport(fd) - return self._add_reader(fd, callback, *args) - - def remove_reader(self, fd): - """Remove a reader callback.""" - self._ensure_fd_no_transport(fd) - return self._remove_reader(fd) - - def add_writer(self, fd, callback, *args): - """Add a writer callback..""" - self._ensure_fd_no_transport(fd) - return self._add_writer(fd, callback, *args) - - def remove_writer(self, fd): - """Remove a writer callback.""" - self._ensure_fd_no_transport(fd) - return self._remove_writer(fd) - - def reset_counters(self): - self.remove_reader_count = collections.defaultdict(int) - self.remove_writer_count = collections.defaultdict(int) - - def _run_once(self): - super()._run_once() - for when in self._timers: - advance = self._gen.send(when) - self.advance_time(advance) - self._timers = [] - - def call_at(self, when, callback, *args): - self._timers.append(when) - return super().call_at(when, callback, *args) - - def _process_events(self, event_list): - return - - def _write_to_self(self): - pass - - -def MockCallback(**kwargs): - return mock.Mock(spec=['__call__'], **kwargs) - - -class MockPattern(str): - """A regex based str with a fuzzy __eq__. - - Use this helper with 'mock.assert_called_with', or anywhere - where a regex comparison between strings is needed. - - For instance: - mock_call.assert_called_with(MockPattern('spam.*ham')) - """ - def __eq__(self, other): - return bool(re.search(str(self), other, re.S)) - - -def get_function_source(func): - source = events._get_function_source(func) - if source is None: - raise ValueError("unable to get the source of %r" % (func,)) - return source - - -class TestCase(unittest.TestCase): - def set_event_loop(self, loop, *, cleanup=True): - assert loop is not None - # ensure that the event loop is passed explicitly in asyncio - events.set_event_loop(None) - if cleanup: - self.addCleanup(loop.close) - - def new_test_loop(self, gen=None): - loop = TestLoop(gen) - self.set_event_loop(loop) - return loop - - def setUp(self): - self._get_running_loop = events._get_running_loop - events._get_running_loop = lambda: None - - def tearDown(self): - events._get_running_loop = self._get_running_loop - - events.set_event_loop(None) - - # Detect CPython bug #23353: ensure that yield/yield-from is not used - # in an except block of a generator - self.assertEqual(sys.exc_info(), (None, None, None)) - - if not compat.PY34: - # Python 3.3 compatibility - def subTest(self, *args, **kwargs): - class EmptyCM: - def __enter__(self): - pass - def __exit__(self, *exc): - pass - return EmptyCM() - - -@contextlib.contextmanager -def disable_logger(): - """Context manager to disable asyncio logger. - - For example, it can be used to ignore warnings in debug mode. - """ - old_level = logger.level - try: - logger.setLevel(logging.CRITICAL+1) - yield - finally: - logger.setLevel(old_level) - - -def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, - family=socket.AF_INET): - """Create a mock of a non-blocking socket.""" - sock = mock.MagicMock(socket.socket) - sock.proto = proto - sock.type = type - sock.family = family - sock.gettimeout.return_value = 0.0 - return sock - - -def force_legacy_ssl_support(): - return mock.patch('asyncio.sslproto._is_sslproto_available', - return_value=False) diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py new file mode 100644 index 00000000000..db048a8231d --- /dev/null +++ b/Lib/asyncio/threads.py @@ -0,0 +1,25 @@ +"""High-level support for working with threads in asyncio""" + +import functools +import contextvars + +from . import events + + +__all__ = "to_thread", + + +async def to_thread(func, /, *args, **kwargs): + """Asynchronously run function *func* in a separate thread. + + Any *args and **kwargs supplied for this function are directly passed + to *func*. Also, the current :class:`contextvars.Context` is propagated, + allowing context variables from the main thread to be accessed in the + separate thread. + + Return a coroutine that can be awaited to get the eventual result of *func*. + """ + loop = events.get_running_loop() + ctx = contextvars.copy_context() + func_call = functools.partial(ctx.run, func, *args, **kwargs) + return await loop.run_in_executor(None, func_call) diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py new file mode 100644 index 00000000000..09342dc7c13 --- /dev/null +++ b/Lib/asyncio/timeouts.py @@ -0,0 +1,182 @@ +import enum + +from types import TracebackType + +from . import events +from . import exceptions +from . import tasks + + +__all__ = ( + "Timeout", + "timeout", + "timeout_at", +) + + +class _State(enum.Enum): + CREATED = "created" + ENTERED = "active" + EXPIRING = "expiring" + EXPIRED = "expired" + EXITED = "finished" + + +class Timeout: + """Asynchronous context manager for cancelling overdue coroutines. + + Use `timeout()` or `timeout_at()` rather than instantiating this class directly. + """ + + def __init__(self, when: float | None) -> None: + """Schedule a timeout that will trigger at a given loop time. + + - If `when` is `None`, the timeout will never trigger. + - If `when < loop.time()`, the timeout will trigger on the next + iteration of the event loop. + """ + self._state = _State.CREATED + + self._timeout_handler: events.TimerHandle | None = None + self._task: tasks.Task | None = None + self._when = when + + def when(self) -> float | None: + """Return the current deadline.""" + return self._when + + def reschedule(self, when: float | None) -> None: + """Reschedule the timeout.""" + if self._state is not _State.ENTERED: + if self._state is _State.CREATED: + raise RuntimeError("Timeout has not been entered") + raise RuntimeError( + f"Cannot change state of {self._state.value} Timeout", + ) + + self._when = when + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + + if when is None: + self._timeout_handler = None + else: + loop = events.get_running_loop() + if when <= loop.time(): + self._timeout_handler = loop.call_soon(self._on_timeout) + else: + self._timeout_handler = loop.call_at(when, self._on_timeout) + + def expired(self) -> bool: + """Is timeout expired during execution?""" + return self._state in (_State.EXPIRING, _State.EXPIRED) + + def __repr__(self) -> str: + info = [''] + if self._state is _State.ENTERED: + when = round(self._when, 3) if self._when is not None else None + info.append(f"when={when}") + info_str = ' '.join(info) + return f"" + + async def __aenter__(self) -> "Timeout": + if self._state is not _State.CREATED: + raise RuntimeError("Timeout has already been entered") + task = tasks.current_task() + if task is None: + raise RuntimeError("Timeout should be used inside a task") + self._state = _State.ENTERED + self._task = task + self._cancelling = self._task.cancelling() + self.reschedule(self._when) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool | None: + assert self._state in (_State.ENTERED, _State.EXPIRING) + + if self._timeout_handler is not None: + self._timeout_handler.cancel() + self._timeout_handler = None + + if self._state is _State.EXPIRING: + self._state = _State.EXPIRED + + if self._task.uncancel() <= self._cancelling and exc_type is not None: + # Since there are no new cancel requests, we're + # handling this. + if issubclass(exc_type, exceptions.CancelledError): + raise TimeoutError from exc_val + elif exc_val is not None: + self._insert_timeout_error(exc_val) + if isinstance(exc_val, ExceptionGroup): + for exc in exc_val.exceptions: + self._insert_timeout_error(exc) + elif self._state is _State.ENTERED: + self._state = _State.EXITED + + return None + + def _on_timeout(self) -> None: + assert self._state is _State.ENTERED + self._task.cancel() + self._state = _State.EXPIRING + # drop the reference early + self._timeout_handler = None + + @staticmethod + def _insert_timeout_error(exc_val: BaseException) -> None: + while exc_val.__context__ is not None: + if isinstance(exc_val.__context__, exceptions.CancelledError): + te = TimeoutError() + te.__context__ = te.__cause__ = exc_val.__context__ + exc_val.__context__ = te + break + exc_val = exc_val.__context__ + + +def timeout(delay: float | None) -> Timeout: + """Timeout async context manager. + + Useful in cases when you want to apply timeout logic around block + of code or in cases when asyncio.wait_for is not suitable. For example: + + >>> async with asyncio.timeout(10): # 10 seconds timeout + ... await long_running_task() + + + delay - value in seconds or None to disable timeout logic + + long_running_task() is interrupted by raising asyncio.CancelledError, + the top-most affected timeout() context manager converts CancelledError + into TimeoutError. + """ + loop = events.get_running_loop() + return Timeout(loop.time() + delay if delay is not None else None) + + +def timeout_at(when: float | None) -> Timeout: + """Schedule the timeout at absolute time. + + Like timeout() but argument gives absolute time in the same clock system + as loop.time(). + + Please note: it is not POSIX time but a time with + undefined starting base, e.g. the time of the system power on. + + >>> async with asyncio.timeout_at(loop.time() + 10): + ... await long_running_task() + + + when - a deadline when timeout occurs or None to disable timeout logic + + long_running_task() is interrupted by raising asyncio.CancelledError, + the top-most affected timeout() context manager converts CancelledError + into TimeoutError. + """ + return Timeout(when) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py new file mode 100644 index 00000000000..f39e11fdd51 --- /dev/null +++ b/Lib/asyncio/tools.py @@ -0,0 +1,276 @@ +"""Tools to analyze tasks running in asyncio programs.""" + +from collections import defaultdict, namedtuple +from itertools import count +from enum import Enum +import sys +from _remote_debugging import RemoteUnwinder, FrameInfo + +class NodeType(Enum): + COROUTINE = 1 + TASK = 2 + + +class CycleFoundException(Exception): + """Raised when there is a cycle when drawing the call tree.""" + def __init__( + self, + cycles: list[list[int]], + id2name: dict[int, str], + ) -> None: + super().__init__(cycles, id2name) + self.cycles = cycles + self.id2name = id2name + + + +# ─── indexing helpers ─────────────────────────────────────────── +def _format_stack_entry(elem: str|FrameInfo) -> str: + if not isinstance(elem, str): + if elem.lineno == 0 and elem.filename == "": + return f"{elem.funcname}" + else: + return f"{elem.funcname} {elem.filename}:{elem.lineno}" + return elem + + +def _index(result): + id2name, awaits, task_stacks = {}, [], {} + for awaited_info in result: + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_name = task_info.task_name + id2name[task_id] = task_name + + # Store the internal coroutine stack for this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + call_stack = coro_info.call_stack + internal_stack = [_format_stack_entry(frame) for frame in call_stack] + task_stacks[task_id] = internal_stack + + # Add the awaited_by relationships (external dependencies) + if task_info.awaited_by: + for coro_info in task_info.awaited_by: + call_stack = coro_info.call_stack + parent_task_id = coro_info.task_name + stack = [_format_stack_entry(frame) for frame in call_stack] + awaits.append((parent_task_id, stack, task_id)) + return id2name, awaits, task_stacks + + +def _build_tree(id2name, awaits, task_stacks): + id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()} + children = defaultdict(list) + cor_nodes = defaultdict(dict) # Maps parent -> {frame_name: node_key} + next_cor_id = count(1) + + def get_or_create_cor_node(parent, frame): + """Get existing coroutine node or create new one under parent""" + if frame in cor_nodes[parent]: + return cor_nodes[parent][frame] + + node_key = (NodeType.COROUTINE, f"c{next(next_cor_id)}") + id2label[node_key] = frame + children[parent].append(node_key) + cor_nodes[parent][frame] = node_key + return node_key + + # Build task dependency tree with coroutine frames + for parent_id, stack, child_id in awaits: + cur = (NodeType.TASK, parent_id) + for frame in reversed(stack): + cur = get_or_create_cor_node(cur, frame) + + child_key = (NodeType.TASK, child_id) + if child_key not in children[cur]: + children[cur].append(child_key) + + # Add coroutine stacks for leaf tasks + awaiting_tasks = {parent_id for parent_id, _, _ in awaits} + for task_id in id2name: + if task_id not in awaiting_tasks and task_id in task_stacks: + cur = (NodeType.TASK, task_id) + for frame in reversed(task_stacks[task_id]): + cur = get_or_create_cor_node(cur, frame) + + return id2label, children + + +def _roots(id2label, children): + all_children = {c for kids in children.values() for c in kids} + return [n for n in id2label if n not in all_children] + +# ─── detect cycles in the task-to-task graph ─────────────────────── +def _task_graph(awaits): + """Return {parent_task_id: {child_task_id, …}, …}.""" + g = defaultdict(set) + for parent_id, _stack, child_id in awaits: + g[parent_id].add(child_id) + return g + + +def _find_cycles(graph): + """ + Depth-first search for back-edges. + + Returns a list of cycles (each cycle is a list of task-ids) or an + empty list if the graph is acyclic. + """ + WHITE, GREY, BLACK = 0, 1, 2 + color = defaultdict(lambda: WHITE) + path, cycles = [], [] + + def dfs(v): + color[v] = GREY + path.append(v) + for w in graph.get(v, ()): + if color[w] == WHITE: + dfs(w) + elif color[w] == GREY: # back-edge → cycle! + i = path.index(w) + cycles.append(path[i:] + [w]) # make a copy + color[v] = BLACK + path.pop() + + for v in list(graph): + if color[v] == WHITE: + dfs(v) + return cycles + + +# ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_all_awaited_by() + + +def build_async_tree(result, task_emoji="(T)", cor_emoji=""): + """ + Build a list of strings for pretty-print an async call tree. + + The call tree is produced by `get_all_async_stacks()`, prefixing tasks + with `task_emoji` and coroutine frames with `cor_emoji`. + """ + id2name, awaits, task_stacks = _index(result) + g = _task_graph(awaits) + cycles = _find_cycles(g) + if cycles: + raise CycleFoundException(cycles, id2name) + labels, children = _build_tree(id2name, awaits, task_stacks) + + def pretty(node): + flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + return f"{flag} {labels[node]}" + + def render(node, prefix="", last=True, buf=None): + if buf is None: + buf = [] + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") + new_pref = prefix + (" " if last else "│ ") + kids = children.get(node, []) + for i, kid in enumerate(kids): + render(kid, new_pref, i == len(kids) - 1, buf) + return buf + + return [render(root) for root in _roots(labels, children)] + + +def build_task_table(result): + id2name, _, _ = _index(result) + table = [] + + for awaited_info in result: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + # Get task info + task_id = task_info.task_id + task_name = task_info.task_name + + # Build coroutine stack string + frames = [frame for coro in task_info.coroutine_stack + for frame in coro.call_stack] + coro_stack = " -> ".join(_format_stack_entry(x).split(" ")[0] + for x in frames) + + # Handle tasks with no awaiters + if not task_info.awaited_by: + table.append([thread_id, hex(task_id), task_name, coro_stack, + "", "", "0x0"]) + continue + + # Handle tasks with awaiters + for coro_info in task_info.awaited_by: + parent_id = coro_info.task_name + awaiter_frames = [_format_stack_entry(x).split(" ")[0] + for x in coro_info.call_stack] + awaiter_chain = " -> ".join(awaiter_frames) + awaiter_name = id2name.get(parent_id, "Unknown") + parent_id_str = (hex(parent_id) if isinstance(parent_id, int) + else str(parent_id)) + + table.append([thread_id, hex(task_id), task_name, coro_stack, + awaiter_chain, awaiter_name, parent_id_str]) + + return table + +def _print_cycle_exception(exception: CycleFoundException): + print("ERROR: await-graph contains cycles - cannot print a tree!", file=sys.stderr) + print("", file=sys.stderr) + for c in exception.cycles: + inames = " → ".join(exception.id2name.get(tid, hex(tid)) for tid in c) + print(f"cycle: {inames}", file=sys.stderr) + + +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + ) + sys.exit(1) + + +def _get_awaited_by_tasks(pid: int) -> list: + try: + return get_all_awaited_by(pid) + except RuntimeError as e: + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}") + sys.exit(1) + except PermissionError as e: + exit_with_permission_help_text() + + +def display_awaited_by_tasks_table(pid: int) -> None: + """Build and print a table of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + table = build_task_table(tasks) + # Print the table in a simple tabular format + print( + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}" + ) + print("-" * 180) + for row in table: + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") + + +def display_awaited_by_tasks_tree(pid: int) -> None: + """Build and print a tree of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + try: + result = build_async_tree(tasks) + except CycleFoundException as e: + _print_cycle_exception(e) + sys.exit(1) + + for tree in result: + print("\n".join(tree)) diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py index 0db08757156..34c7ad44ffd 100644 --- a/Lib/asyncio/transports.py +++ b/Lib/asyncio/transports.py @@ -1,15 +1,16 @@ """Abstract Transport class.""" -from asyncio import compat - -__all__ = ['BaseTransport', 'ReadTransport', 'WriteTransport', - 'Transport', 'DatagramTransport', 'SubprocessTransport', - ] +__all__ = ( + 'BaseTransport', 'ReadTransport', 'WriteTransport', + 'Transport', 'DatagramTransport', 'SubprocessTransport', +) class BaseTransport: """Base class for transports.""" + __slots__ = ('_extra',) + def __init__(self, extra=None): if extra is None: extra = {} @@ -28,8 +29,8 @@ def close(self): Buffered data will be flushed asynchronously. No more data will be received. After all buffered data is flushed, the - protocol's connection_lost() method will (eventually) called - with None as its argument. + protocol's connection_lost() method will (eventually) be + called with None as its argument. """ raise NotImplementedError @@ -45,6 +46,12 @@ def get_protocol(self): class ReadTransport(BaseTransport): """Interface for read-only transports.""" + __slots__ = () + + def is_reading(self): + """Return True if the transport is receiving.""" + raise NotImplementedError + def pause_reading(self): """Pause the receiving end. @@ -65,6 +72,8 @@ def resume_reading(self): class WriteTransport(BaseTransport): """Interface for write-only transports.""" + __slots__ = () + def set_write_buffer_limits(self, high=None, low=None): """Set the high- and low-water limits for write flow control. @@ -90,6 +99,12 @@ def get_write_buffer_size(self): """Return the current size of the write buffer.""" raise NotImplementedError + def get_write_buffer_limits(self): + """Get the high and low watermarks for write flow control. + Return a tuple (low, high) where low and high are + positive number of bytes.""" + raise NotImplementedError + def write(self, data): """Write some data bytes to the transport. @@ -104,7 +119,7 @@ def writelines(self, list_of_data): The default implementation concatenates the arguments and calls write() on the result. """ - data = compat.flatten_list_bytes(list_of_data) + data = b''.join(list_of_data) self.write(data) def write_eof(self): @@ -151,10 +166,14 @@ class Transport(ReadTransport, WriteTransport): except writelines(), which calls write() in a loop. """ + __slots__ = () + class DatagramTransport(BaseTransport): """Interface for datagram (UDP) transports.""" + __slots__ = () + def sendto(self, data, addr=None): """Send data to the transport. @@ -162,6 +181,8 @@ def sendto(self, data, addr=None): to be sent out asynchronously. addr is target socket address. If addr is None use target address pointed on transport creation. + If data is an empty bytes object a zero-length datagram will be + sent. """ raise NotImplementedError @@ -177,6 +198,8 @@ def abort(self): class SubprocessTransport(BaseTransport): + __slots__ = () + def get_pid(self): """Get subprocess id.""" raise NotImplementedError @@ -244,6 +267,8 @@ class _FlowControlMixin(Transport): resume_writing() may be called. """ + __slots__ = ('_loop', '_protocol_paused', '_high_water', '_low_water') + def __init__(self, extra=None, loop=None): super().__init__(extra) assert loop is not None @@ -259,7 +284,9 @@ def _maybe_pause_protocol(self): self._protocol_paused = True try: self._protocol.pause_writing() - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self._loop.call_exception_handler({ 'message': 'protocol.pause_writing() failed', 'exception': exc, @@ -269,11 +296,13 @@ def _maybe_pause_protocol(self): def _maybe_resume_protocol(self): if (self._protocol_paused and - self.get_write_buffer_size() <= self._low_water): + self.get_write_buffer_size() <= self._low_water): self._protocol_paused = False try: self._protocol.resume_writing() - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self._loop.call_exception_handler({ 'message': 'protocol.resume_writing() failed', 'exception': exc, @@ -287,14 +316,16 @@ def get_write_buffer_limits(self): def _set_write_buffer_limits(self, high=None, low=None): if high is None: if low is None: - high = 64*1024 + high = 64 * 1024 else: - high = 4*low + high = 4 * low if low is None: low = high // 4 + if not high >= low >= 0: - raise ValueError('high (%r) must be >= low (%r) must be >= 0' % - (high, low)) + raise ValueError( + f'high ({high!r}) must be >= low ({low!r}) must be >= 0') + self._high_water = high self._low_water = low diff --git a/Lib/asyncio/trsock.py b/Lib/asyncio/trsock.py new file mode 100644 index 00000000000..c1f20473b32 --- /dev/null +++ b/Lib/asyncio/trsock.py @@ -0,0 +1,98 @@ +import socket + + +class TransportSocket: + + """A socket-like wrapper for exposing real transport sockets. + + These objects can be safely returned by APIs like + `transport.get_extra_info('socket')`. All potentially disruptive + operations (like "socket.close()") are banned. + """ + + __slots__ = ('_sock',) + + def __init__(self, sock: socket.socket): + self._sock = sock + + @property + def family(self): + return self._sock.family + + @property + def type(self): + return self._sock.type + + @property + def proto(self): + return self._sock.proto + + def __repr__(self): + s = ( + f"" + + def __getstate__(self): + raise TypeError("Cannot serialize asyncio.TransportSocket object") + + def fileno(self): + return self._sock.fileno() + + def dup(self): + return self._sock.dup() + + def get_inheritable(self): + return self._sock.get_inheritable() + + def shutdown(self, how): + # asyncio doesn't currently provide a high-level transport API + # to shutdown the connection. + self._sock.shutdown(how) + + def getsockopt(self, *args, **kwargs): + return self._sock.getsockopt(*args, **kwargs) + + def setsockopt(self, *args, **kwargs): + self._sock.setsockopt(*args, **kwargs) + + def getpeername(self): + return self._sock.getpeername() + + def getsockname(self): + return self._sock.getsockname() + + def getsockbyname(self): + return self._sock.getsockbyname() + + def settimeout(self, value): + if value == 0: + return + raise ValueError( + 'settimeout(): only 0 timeout is allowed on transport sockets') + + def gettimeout(self): + return 0 + + def setblocking(self, flag): + if not flag: + return + raise ValueError( + 'setblocking(): transport sockets cannot be blocking') diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 9db09b9d9b8..1c1458127db 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1,7 +1,10 @@ """Selector event loop for Unix with signal handling.""" import errno +import io +import itertools import os +import selectors import signal import socket import stat @@ -10,25 +13,24 @@ import threading import warnings - from . import base_events from . import base_subprocess -from . import compat from . import constants from . import coroutines from . import events +from . import exceptions from . import futures from . import selector_events -from . import selectors +from . import tasks from . import transports -from .coroutines import coroutine from .log import logger -__all__ = ['SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'DefaultEventLoopPolicy', - ] +__all__ = ( + 'SelectorEventLoop', + 'EventLoop', +) + if sys.platform == 'win32': # pragma: no cover raise ImportError('Signals are not really supported on Windows') @@ -39,11 +41,14 @@ def _sighandler_noop(signum, frame): pass -try: - _fspath = os.fspath -except AttributeError: - # Python 3.5 or earlier - _fspath = lambda path: path +def waitstatus_to_exitcode(status): + try: + return os.waitstatus_to_exitcode(status) + except ValueError: + # The child exited, but we don't understand its status. + # This shouldn't happen, but if it does, let's just + # return that status; perhaps that helps debug it. + return status class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): @@ -55,14 +60,25 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} - - def _socketpair(self): - return socket.socketpair() + self._unix_server_sockets = {} + if can_use_pidfd(): + self._watcher = _PidfdChildWatcher() + else: + self._watcher = _ThreadedChildWatcher() def close(self): super().close() - for sig in list(self._signal_handlers): - self.remove_signal_handler(sig) + if not sys.is_finalizing(): + for sig in list(self._signal_handlers): + self.remove_signal_handler(sig) + else: + if self._signal_handlers: + warnings.warn(f"Closing the loop {self!r} " + f"on interpreter shutdown " + f"stage, skipping signal handlers removal", + ResourceWarning, + source=self) + self._signal_handlers.clear() def _process_self_data(self, data): for signum in data: @@ -77,8 +93,8 @@ def add_signal_handler(self, sig, callback, *args): Raise ValueError if the signal number is invalid or uncatchable. Raise RuntimeError if there is a problem setting up the handler. """ - if (coroutines.iscoroutine(callback) - or coroutines.iscoroutinefunction(callback)): + if (coroutines.iscoroutine(callback) or + coroutines._iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -92,12 +108,12 @@ def add_signal_handler(self, sig, callback, *args): except (ValueError, OSError) as exc: raise RuntimeError(str(exc)) - handle = events.Handle(callback, args, self) + handle = events.Handle(callback, args, self, None) self._signal_handlers[sig] = handle try: # Register a dummy signal handler to ask Python to write the signal - # number in the wakup file descriptor. _process_self_data() will + # number in the wakeup file descriptor. _process_self_data() will # read signal numbers from this file descriptor to handle signals. signal.signal(sig, _sighandler_noop) @@ -112,7 +128,7 @@ def add_signal_handler(self, sig, callback, *args): logger.info('set_wakeup_fd(-1) failed: %s', nexc) if exc.errno == errno.EINVAL: - raise RuntimeError('sig {} cannot be caught'.format(sig)) + raise RuntimeError(f'sig {sig} cannot be caught') else: raise @@ -146,7 +162,7 @@ def remove_signal_handler(self, sig): signal.signal(sig, handler) except OSError as exc: if exc.errno == errno.EINVAL: - raise RuntimeError('sig {} cannot be caught'.format(sig)) + raise RuntimeError(f'sig {sig} cannot be caught') else: raise @@ -165,11 +181,10 @@ def _check_signal(self, sig): Raise RuntimeError if there is a problem setting up the handler. """ if not isinstance(sig, int): - raise TypeError('sig must be an int, not {!r}'.format(sig)) + raise TypeError(f'sig must be an int, not {sig!r}') - if not (1 <= sig < signal.NSIG): - raise ValueError( - 'sig {} out of range(1, {})'.format(sig, signal.NSIG)) + if sig not in signal.valid_signals(): + raise ValueError(f'invalid signal number {sig}') def _make_read_pipe_transport(self, pipe, protocol, waiter=None, extra=None): @@ -179,43 +194,37 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None, extra=None): return _UnixWritePipeTransport(self, pipe, protocol, waiter, extra) - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): - with events.get_child_watcher() as watcher: - waiter = self.create_future() - transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - - watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) - try: - yield from waiter - except Exception as exc: - # Workaround CPython bug #23353: using yield/yield-from in an - # except block of a generator doesn't clear properly - # sys.exc_info() - err = exc - else: - err = None - - if err is not None: - transp.close() - yield from transp._wait() - raise err + async def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): + watcher = self._watcher + waiter = self.create_future() + transp = _UnixSubprocessTransport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + waiter=waiter, extra=extra, + **kwargs) + watcher.add_child_handler(transp.get_pid(), + self._child_watcher_callback, transp) + try: + await waiter + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: + transp.close() + await transp._wait() + raise return transp def _child_watcher_callback(self, pid, returncode, transp): self.call_soon_threadsafe(transp._process_exited, returncode) - @coroutine - def create_unix_connection(self, protocol_factory, path, *, - ssl=None, sock=None, - server_hostname=None): + async def create_unix_connection( + self, protocol_factory, path=None, *, + ssl=None, sock=None, + server_hostname=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): assert server_hostname is None or isinstance(server_hostname, str) if ssl: if server_hostname is None: @@ -224,16 +233,23 @@ def create_unix_connection(self, protocol_factory, path, *, else: if server_hostname is not None: raise ValueError('server_hostname is only meaningful with ssl') + if ssl_handshake_timeout is not None: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') + if ssl_shutdown_timeout is not None: + raise ValueError( + 'ssl_shutdown_timeout is only meaningful with ssl') if path is not None: if sock is not None: raise ValueError( 'path and sock can not be specified at the same time') + path = os.fspath(path) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) try: sock.setblocking(False) - yield from self.sock_connect(sock, path) + await self.sock_connect(sock, path) except: sock.close() raise @@ -242,28 +258,40 @@ def create_unix_connection(self, protocol_factory, path, *, if sock is None: raise ValueError('no path and sock were specified') if (sock.family != socket.AF_UNIX or - not base_events._is_stream_socket(sock)): + sock.type != socket.SOCK_STREAM): raise ValueError( - 'A UNIX Domain Stream Socket was expected, got {!r}' - .format(sock)) + f'A UNIX Domain Stream Socket was expected, got {sock!r}') sock.setblocking(False) - transport, protocol = yield from self._create_connection_transport( - sock, protocol_factory, ssl, server_hostname) + transport, protocol = await self._create_connection_transport( + sock, protocol_factory, ssl, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) return transport, protocol - @coroutine - def create_unix_server(self, protocol_factory, path=None, *, - sock=None, backlog=100, ssl=None): + async def create_unix_server( + self, protocol_factory, path=None, *, + sock=None, backlog=100, ssl=None, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None, + start_serving=True, cleanup_socket=True): if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') + if ssl_handshake_timeout is not None and not ssl: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') + + if ssl_shutdown_timeout is not None and not ssl: + raise ValueError( + 'ssl_shutdown_timeout is only meaningful with ssl') + if path is not None: if sock is not None: raise ValueError( 'path and sock can not be specified at the same time') - path = _fspath(path) + path = os.fspath(path) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # Check for abstract socket. `str` and `bytes` paths are supported. @@ -275,7 +303,8 @@ def create_unix_server(self, protocol_factory, path=None, *, pass except OSError as err: # Directory may have permissions only to create socket. - logger.error('Unable to check or remove stale UNIX socket %r: %r', path, err) + logger.error('Unable to check or remove stale UNIX socket ' + '%r: %r', path, err) try: sock.bind(path) @@ -284,7 +313,7 @@ def create_unix_server(self, protocol_factory, path=None, *, if exc.errno == errno.EADDRINUSE: # Let's improve the error message by adding # with what exact address it occurs. - msg = 'Address {!r} is already in use'.format(path) + msg = f'Address {path!r} is already in use' raise OSError(errno.EADDRINUSE, msg) from None else: raise @@ -297,28 +326,159 @@ def create_unix_server(self, protocol_factory, path=None, *, 'path was not specified, and no sock specified') if (sock.family != socket.AF_UNIX or - not base_events._is_stream_socket(sock)): + sock.type != socket.SOCK_STREAM): raise ValueError( - 'A UNIX Domain Stream Socket was expected, got {!r}' - .format(sock)) + f'A UNIX Domain Stream Socket was expected, got {sock!r}') + + if cleanup_socket: + path = sock.getsockname() + # Check for abstract socket. `str` and `bytes` paths are supported. + if path[0] not in (0, '\x00'): + try: + self._unix_server_sockets[sock] = os.stat(path).st_ino + except FileNotFoundError: + pass - server = base_events.Server(self, [sock]) - sock.listen(backlog) sock.setblocking(False) - self._start_serving(protocol_factory, sock, ssl, server) + server = base_events.Server(self, [sock], protocol_factory, + ssl, backlog, ssl_handshake_timeout, + ssl_shutdown_timeout) + if start_serving: + server._start_serving() + # Skip one loop iteration so that all 'loop.add_reader' + # go through. + await tasks.sleep(0) + return server + async def _sock_sendfile_native(self, sock, file, offset, count): + try: + os.sendfile + except AttributeError: + raise exceptions.SendfileNotAvailableError( + "os.sendfile() is not available") + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise exceptions.SendfileNotAvailableError("not a regular file") + try: + fsize = os.fstat(fileno).st_size + except OSError: + raise exceptions.SendfileNotAvailableError("not a regular file") + blocksize = count if count else fsize + if not blocksize: + return 0 # empty file + + fut = self.create_future() + self._sock_sendfile_native_impl(fut, None, sock, fileno, + offset, count, blocksize, 0) + return await fut + + def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, + offset, count, blocksize, total_sent): + fd = sock.fileno() + if registered_fd is not None: + # Remove the callback early. It should be rare that the + # selector says the fd is ready but the call still returns + # EAGAIN, and I am willing to take a hit in that case in + # order to simplify the common case. + self.remove_writer(registered_fd) + if fut.cancelled(): + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + return + if count: + blocksize = count - total_sent + if blocksize <= 0: + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_result(total_sent) + return -#if hasattr(os, 'set_blocking'): -# def _set_nonblocking(fd): -# os.set_blocking(fd, False) -#else: -# import fcntl + # On 32-bit architectures truncate to 1GiB to avoid OverflowError + blocksize = min(blocksize, sys.maxsize//2 + 1) -# def _set_nonblocking(fd): -# flags = fcntl.fcntl(fd, fcntl.F_GETFL) -# flags = flags | os.O_NONBLOCK -# fcntl.fcntl(fd, fcntl.F_SETFL, flags) + try: + sent = os.sendfile(fd, fileno, offset, blocksize) + except (BlockingIOError, InterruptedError): + if registered_fd is None: + self._sock_add_cancellation_callback(fut, sock) + self.add_writer(fd, self._sock_sendfile_native_impl, fut, + fd, sock, fileno, + offset, count, blocksize, total_sent) + except OSError as exc: + if (registered_fd is not None and + exc.errno == errno.ENOTCONN and + type(exc) is not ConnectionError): + # If we have an ENOTCONN and this isn't a first call to + # sendfile(), i.e. the connection was closed in the middle + # of the operation, normalize the error to ConnectionError + # to make it consistent across all Posix systems. + new_exc = ConnectionError( + "socket is not connected", errno.ENOTCONN) + new_exc.__cause__ = exc + exc = new_exc + if total_sent == 0: + # We can get here for different reasons, the main + # one being 'file' is not a regular mmap(2)-like + # file, in which case we'll fall back on using + # plain send(). + err = exceptions.SendfileNotAvailableError( + "os.sendfile call failed") + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_exception(err) + else: + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_exception(exc) + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_exception(exc) + else: + if sent == 0: + # EOF + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_result(total_sent) + else: + offset += sent + total_sent += sent + if registered_fd is None: + self._sock_add_cancellation_callback(fut, sock) + self.add_writer(fd, self._sock_sendfile_native_impl, fut, + fd, sock, fileno, + offset, count, blocksize, total_sent) + + def _sock_sendfile_update_filepos(self, fileno, offset, total_sent): + if total_sent > 0: + os.lseek(fileno, offset, os.SEEK_SET) + + def _sock_add_cancellation_callback(self, fut, sock): + def cb(fut): + if fut.cancelled(): + fd = sock.fileno() + if fd != -1: + self.remove_writer(fd) + fut.add_done_callback(cb) + + def _stop_serving(self, sock): + # Is this a unix socket that needs cleanup? + if sock in self._unix_server_sockets: + path = sock.getsockname() + else: + path = None + + super()._stop_serving(sock) + + if path is not None: + prev_ino = self._unix_server_sockets[sock] + del self._unix_server_sockets[sock] + try: + if os.stat(path).st_ino == prev_ino: + os.unlink(path) + except FileNotFoundError: + pass + except OSError as err: + logger.error('Unable to clean up listening UNIX socket ' + '%r: %r', path, err) class _UnixReadPipeTransport(transports.ReadTransport): @@ -333,6 +493,7 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): self._fileno = pipe.fileno() self._protocol = protocol self._closing = False + self._paused = False mode = os.fstat(self._fileno).st_mode if not (stat.S_ISFIFO(mode) or @@ -343,29 +504,36 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): self._protocol = None raise ValueError("Pipe transport is for pipes/sockets only.") - _set_nonblocking(self._fileno) + os.set_blocking(self._fileno, False) self._loop.call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._loop._add_reader, + self._loop.call_soon(self._add_reader, self._fileno, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called self._loop.call_soon(futures._set_result_unless_cancelled, waiter, None) + def _add_reader(self, fd, callback): + if not self.is_reading(): + return + self._loop._add_reader(fd, callback) + + def is_reading(self): + return not self._paused and not self._closing + def __repr__(self): info = [self.__class__.__name__] if self._pipe is None: info.append('closed') elif self._closing: info.append('closing') - info.append('fd=%s' % self._fileno) + info.append(f'fd={self._fileno}') selector = getattr(self._loop, '_selector', None) if self._pipe is not None and selector is not None: polling = selector_events._test_selector_event( - selector, - self._fileno, selectors.EVENT_READ) + selector, self._fileno, selectors.EVENT_READ) if polling: info.append('polling') else: @@ -374,7 +542,7 @@ def __repr__(self): info.append('open') else: info.append('closed') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def _read_ready(self): try: @@ -395,10 +563,20 @@ def _read_ready(self): self._loop.call_soon(self._call_connection_lost, None) def pause_reading(self): + if not self.is_reading(): + return + self._paused = True self._loop._remove_reader(self._fileno) + if self._loop.get_debug(): + logger.debug("%r pauses reading", self) def resume_reading(self): + if self._closing or not self._paused: + return + self._paused = False self._loop._add_reader(self._fileno, self._read_ready) + if self._loop.get_debug(): + logger.debug("%r resumes reading", self) def set_protocol(self, protocol): self._protocol = protocol @@ -413,15 +591,10 @@ def close(self): if not self._closing: self._close(None) - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning, - source=self) - self._pipe.close() + def __del__(self, _warn=warnings.warn): + if self._pipe is not None: + _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) + self._pipe.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): # should be called by exception handler only @@ -476,7 +649,7 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): raise ValueError("Pipe transport is only for " "pipes, sockets and character devices") - _set_nonblocking(self._fileno) + os.set_blocking(self._fileno, False) self._loop.call_soon(self._protocol.connection_made, self) # On AIX, the reader trick (to be notified when the read end of the @@ -498,24 +671,23 @@ def __repr__(self): info.append('closed') elif self._closing: info.append('closing') - info.append('fd=%s' % self._fileno) + info.append(f'fd={self._fileno}') selector = getattr(self._loop, '_selector', None) if self._pipe is not None and selector is not None: polling = selector_events._test_selector_event( - selector, - self._fileno, selectors.EVENT_WRITE) + selector, self._fileno, selectors.EVENT_WRITE) if polling: info.append('polling') else: info.append('idle') bufsize = self.get_write_buffer_size() - info.append('bufsize=%s' % bufsize) + info.append(f'bufsize={bufsize}') elif self._pipe is not None: info.append('open') else: info.append('closed') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def get_write_buffer_size(self): return len(self._buffer) @@ -549,7 +721,9 @@ def write(self, data): n = os.write(self._fileno, data) except (BlockingIOError, InterruptedError): n = 0 - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self._conn_lost += 1 self._fatal_error(exc, 'Fatal write error on pipe transport') return @@ -569,7 +743,9 @@ def _write_ready(self): n = os.write(self._fileno, self._buffer) except (BlockingIOError, InterruptedError): pass - except Exception as exc: + except (SystemExit, KeyboardInterrupt): + raise + except BaseException as exc: self._buffer.clear() self._conn_lost += 1 # Remove writer here, _fatal_error() doesn't it @@ -614,22 +790,17 @@ def close(self): # write_eof is all what we needed to close the write pipe self.write_eof() - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning, - source=self) - self._pipe.close() + def __del__(self, _warn=warnings.warn): + if self._pipe is not None: + _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) + self._pipe.close() def abort(self): self._close(None) def _fatal_error(self, exc, message='Fatal error on pipe transport'): # should be called by exception handler only - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if isinstance(exc, OSError): if self._loop.get_debug(): logger.debug("%r: %s", self, message, exc_info=True) else: @@ -659,227 +830,105 @@ def _call_connection_lost(self, exc): self._loop = None -#if hasattr(os, 'set_inheritable'): -# # Python 3.4 and newer -# _set_inheritable = os.set_inheritable -#else: -# import fcntl -# -# def _set_inheritable(fd, inheritable): -# cloexec_flag = getattr(fcntl, 'FD_CLOEXEC', 1) -# -# old = fcntl.fcntl(fd, fcntl.F_GETFD) -# if not inheritable: -# fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) -# else: -# fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w = None - if stdin == subprocess.PIPE: - # Use a socket pair for stdin, since not all platforms + if stdin == subprocess.PIPE and sys.platform.startswith('aix'): + # Use a socket pair for stdin on AIX, since it does not # support selecting read events on the write end of a # socket (which we use in order to detect closing of the - # other end). Notably this is needed on AIX, and works - # just fine on other platforms. - stdin, stdin_w = self._loop._socketpair() - - # Mark the write end of the stdin pipe as non-inheritable, - # needed by close_fds=False on Python 3.3 and older - # (Python 3.4 implements the PEP 446, socketpair returns - # non-inheritable sockets) - _set_inheritable(stdin_w.fileno(), False) - self._proc = subprocess.Popen( - args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, - universal_newlines=False, bufsize=bufsize, **kwargs) - if stdin_w is not None: - stdin.close() - self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize) - - -class AbstractChildWatcher: - """Abstract base class for monitoring child processes. - - Objects derived from this class monitor a collection of subprocesses and - report their termination or interruption by a signal. - - New callbacks are registered with .add_child_handler(). Starting a new - process must be done within a 'with' block to allow the watcher to suspend - its activity until the new process if fully registered (this is needed to - prevent a race condition in some implementations). - - Example: - with watcher: - proc = subprocess.Popen("sleep 1") - watcher.add_child_handler(proc.pid, callback) - - Notes: - Implementations of this class must be thread-safe. - - Since child watcher objects may catch the SIGCHLD signal and call - waitpid(-1), there should be only one active object per process. - """ - - def add_child_handler(self, pid, callback, *args): - """Register a new child handler. - - Arrange for callback(pid, returncode, *args) to be called when - process 'pid' terminates. Specifying another callback for the same - process replaces the previous handler. - - Note: callback() must be thread-safe. - """ - raise NotImplementedError() - - def remove_child_handler(self, pid): - """Removes the handler for process 'pid'. - - The function returns True if the handler was successfully removed, - False if there was nothing to remove.""" - - raise NotImplementedError() - - def attach_loop(self, loop): - """Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then it is - first detached before attaching to the new loop. - - Note: loop may be None. - """ - raise NotImplementedError() - - def close(self): - """Close the watcher. - - This must be called to make sure that any underlying resource is freed. - """ - raise NotImplementedError() - - def __enter__(self): - """Enter the watcher's context and allow starting new processes - - This function must return self""" - raise NotImplementedError() - - def __exit__(self, a, b, c): - """Exit the watcher's context""" - raise NotImplementedError() - - -class BaseChildWatcher(AbstractChildWatcher): - - def __init__(self): - self._loop = None - self._callbacks = {} - - def close(self): - self.attach_loop(None) - - def _do_waitpid(self, expected_pid): - raise NotImplementedError() - - def _do_waitpid_all(self): - raise NotImplementedError() - - def attach_loop(self, loop): - assert loop is None or isinstance(loop, events.AbstractEventLoop) + # other end). + stdin, stdin_w = socket.socketpair() + try: + self._proc = subprocess.Popen( + args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, + universal_newlines=False, bufsize=bufsize, **kwargs) + if stdin_w is not None: + stdin.close() + self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize) + stdin_w = None + finally: + if stdin_w is not None: + stdin.close() + stdin_w.close() - if self._loop is not None and loop is None and self._callbacks: - warnings.warn( - 'A loop is being detached ' - 'from a child watcher with pending handlers', - RuntimeWarning) - if self._loop is not None: - self._loop.remove_signal_handler(signal.SIGCHLD) +class _PidfdChildWatcher: + """Child watcher implementation using Linux's pid file descriptors. - self._loop = loop - if loop is not None: - loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) + This child watcher polls process file descriptors (pidfds) to await child + process termination. In some respects, PidfdChildWatcher is a "Goldilocks" + child watcher implementation. It doesn't require signals or threads, doesn't + interfere with any processes launched outside the event loop, and scales + linearly with the number of subprocesses launched by the event loop. The + main disadvantage is that pidfds are specific to Linux, and only work on + recent (5.3+) kernels. + """ - # Prevent a race condition in case a child terminated - # during the switch. - self._do_waitpid_all() + def add_child_handler(self, pid, callback, *args): + loop = events.get_running_loop() + pidfd = os.pidfd_open(pid) + loop._add_reader(pidfd, self._do_wait, pid, pidfd, callback, args) - def _sig_chld(self): + def _do_wait(self, pid, pidfd, callback, args): + loop = events.get_running_loop() + loop._remove_reader(pidfd) try: - self._do_waitpid_all() - except Exception as exc: - # self._loop should always be available here - # as '_sig_chld' is added as a signal handler - # in 'attach_loop' - self._loop.call_exception_handler({ - 'message': 'Unknown exception in SIGCHLD handler', - 'exception': exc, - }) - - def _compute_returncode(self, status): - if os.WIFSIGNALED(status): - # The child process died because of a signal. - return -os.WTERMSIG(status) - elif os.WIFEXITED(status): - # The child process exited (e.g sys.exit()). - return os.WEXITSTATUS(status) + _, status = os.waitpid(pid, 0) + except ChildProcessError: + # The child process is already reaped + # (may happen if waitpid() is called elsewhere). + returncode = 255 + logger.warning( + "child process pid %d exit status already read: " + " will report returncode 255", + pid) else: - # The child exited, but we don't understand its status. - # This shouldn't happen, but if it does, let's just - # return that status; perhaps that helps debug it. - return status + returncode = waitstatus_to_exitcode(status) + + os.close(pidfd) + callback(pid, returncode, *args) +class _ThreadedChildWatcher: + """Threaded child watcher implementation. -class SafeChildWatcher(BaseChildWatcher): - """'Safe' child watcher implementation. + The watcher uses a thread per process + for waiting for the process finish. - This implementation avoids disrupting other code spawning processes by - polling explicitly each process in the SIGCHLD handler instead of calling - os.waitpid(-1). + It doesn't require subscription on POSIX signal + but a thread creation is not free. - This is a safe solution but it has a significant overhead when handling a - big number of children (O(n) each time SIGCHLD is raised) + The watcher has O(1) complexity, its performance doesn't depend + on amount of spawn processes. """ - def close(self): - self._callbacks.clear() - super().close() - - def __enter__(self): - return self + def __init__(self): + self._pid_counter = itertools.count(0) + self._threads = {} - def __exit__(self, a, b, c): - pass + def __del__(self, _warn=warnings.warn): + threads = [thread for thread in list(self._threads.values()) + if thread.is_alive()] + if threads: + _warn(f"{self.__class__} has registered but not finished child processes", + ResourceWarning, + source=self) def add_child_handler(self, pid, callback, *args): - if self._loop is None: - raise RuntimeError( - "Cannot add child handler, " - "the child watcher does not have a loop attached") - - self._callbacks[pid] = (callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): + loop = events.get_running_loop() + thread = threading.Thread(target=self._do_waitpid, + name=f"asyncio-waitpid-{next(self._pid_counter)}", + args=(loop, pid, callback, args), + daemon=True) + self._threads[pid] = thread + thread.start() + + def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) + pid, status = os.waitpid(expected_pid, 0) except ChildProcessError: # The child process is already reaped # (may happen if waitpid() is called elsewhere). @@ -889,186 +938,35 @@ def _do_waitpid(self, expected_pid): "Unknown child process pid %d, will report returncode 255", pid) else: - if pid == 0: - # The child process is still alive. - return - - returncode = self._compute_returncode(status) - if self._loop.get_debug(): + returncode = waitstatus_to_exitcode(status) + if loop.get_debug(): logger.debug('process %s exited with returncode %s', expected_pid, returncode) - try: - callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - if self._loop.get_debug(): - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) + if loop.is_closed(): + logger.warning("Loop %r that handles pid %r is closed", loop, pid) else: - callback(pid, returncode, *args) - - -class FastChildWatcher(BaseChildWatcher): - """'Fast' child watcher implementation. - - This implementation reaps every terminated processes by calling - os.waitpid(-1) directly, possibly breaking other code spawning processes - and waiting for their termination. - - There is no noticeable overhead when handling a big number of children - (O(1) each time a child terminates). - """ - def __init__(self): - super().__init__() - self._lock = threading.Lock() - self._zombies = {} - self._forks = 0 + loop.call_soon_threadsafe(callback, pid, returncode, *args) - def close(self): - self._callbacks.clear() - self._zombies.clear() - super().close() - - def __enter__(self): - with self._lock: - self._forks += 1 + self._threads.pop(expected_pid) - return self - - def __exit__(self, a, b, c): - with self._lock: - self._forks -= 1 - - if self._forks or not self._zombies: - return +def can_use_pidfd(): + if not hasattr(os, 'pidfd_open'): + return False + try: + pid = os.getpid() + os.close(os.pidfd_open(pid, 0)) + except OSError: + # blocked by security policy like SECCOMP + return False + return True - collateral_victims = str(self._zombies) - self._zombies.clear() - logger.warning( - "Caught subprocesses termination from unknown pids: %s", - collateral_victims) - - def add_child_handler(self, pid, callback, *args): - assert self._forks, "Must use the context manager" - - if self._loop is None: - raise RuntimeError( - "Cannot add child handler, " - "the child watcher does not have a loop attached") - - with self._lock: - try: - returncode = self._zombies.pop(pid) - except KeyError: - # The child is running. - self._callbacks[pid] = callback, args - return - - # The child is dead already. We can fire the callback. - callback(pid, returncode, *args) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - # Because of signal coalescing, we must keep calling waitpid() as - # long as we're able to reap a child. - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - # No more child processes exist. - return - else: - if pid == 0: - # A child process is still alive. - return - - returncode = self._compute_returncode(status) - - with self._lock: - try: - callback, args = self._callbacks.pop(pid) - except KeyError: - # unknown child - if self._forks: - # It may not be registered yet. - self._zombies[pid] = returncode - if self._loop.get_debug(): - logger.debug('unknown process %s exited ' - 'with returncode %s', - pid, returncode) - continue - callback = None - else: - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - pid, returncode) - - if callback is None: - logger.warning( - "Caught subprocess termination from unknown pid: " - "%d -> %d", pid, returncode) - else: - callback(pid, returncode, *args) - - -class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - """UNIX event loop policy with a watcher for child processes.""" +class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): + """UNIX event loop policy""" _loop_factory = _UnixSelectorEventLoop - def __init__(self): - super().__init__() - self._watcher = None - - def _init_watcher(self): - with events._lock: - if self._watcher is None: # pragma: no branch - self._watcher = SafeChildWatcher() - if isinstance(threading.current_thread(), - threading._MainThread): - self._watcher.attach_loop(self._local._loop) - - def set_event_loop(self, loop): - """Set the event loop. - - As a side effect, if a child watcher was set before, then calling - .set_event_loop() from the main thread will call .attach_loop(loop) on - the child watcher. - """ - - super().set_event_loop(loop) - - if self._watcher is not None and \ - isinstance(threading.current_thread(), threading._MainThread): - self._watcher.attach_loop(loop) - - def get_child_watcher(self): - """Get the watcher for child processes. - - If not yet set, a SafeChildWatcher object is automatically created. - """ - if self._watcher is None: - self._init_watcher() - - return self._watcher - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - - assert watcher is None or isinstance(watcher, AbstractChildWatcher) - - if self._watcher is not None: - self._watcher.close() - - self._watcher = watcher SelectorEventLoop = _UnixSelectorEventLoop -DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy +EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 2c68bc526a2..5f75b17d8ca 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -1,32 +1,41 @@ """Selector and proactor event loops for Windows.""" +import sys + +if sys.platform != 'win32': # pragma: no cover + raise ImportError('win32 only') + +import _overlapped import _winapi import errno +from functools import partial import math +import msvcrt import socket import struct +import time import weakref from . import events from . import base_subprocess from . import futures +from . import exceptions from . import proactor_events from . import selector_events from . import tasks from . import windows_utils -# XXX RustPython TODO: _overlapped -# from . import _overlapped -from .coroutines import coroutine from .log import logger -__all__ = ['SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', - ] +__all__ = ( + 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', + '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', + '_WindowsProactorEventLoopPolicy', 'EventLoop', +) -NULL = 0 -INFINITE = 0xffffffff +NULL = _winapi.NULL +INFINITE = _winapi.INFINITE ERROR_CONNECTION_REFUSED = 1225 ERROR_CONNECTION_ABORTED = 1236 @@ -53,7 +62,7 @@ def _repr_info(self): info = super()._repr_info() if self._ov is not None: state = 'pending' if self._ov.pending else 'completed' - info.insert(1, 'overlapped=<%s, %#x>' % (state, self._ov.address)) + info.insert(1, f'overlapped=<{state}, {self._ov.address:#x}>') return info def _cancel_overlapped(self): @@ -72,9 +81,9 @@ def _cancel_overlapped(self): self._loop.call_exception_handler(context) self._ov = None - def cancel(self): + def cancel(self, msg=None): self._cancel_overlapped() - return super().cancel() + return super().cancel(msg=msg) def set_exception(self, exception): super().set_exception(exception) @@ -109,12 +118,12 @@ def _poll(self): def _repr_info(self): info = super()._repr_info() - info.append('handle=%#x' % self._handle) + info.append(f'handle={self._handle:#x}') if self._handle is not None: state = 'signaled' if self._poll() else 'waiting' info.append(state) if self._wait_handle is not None: - info.append('wait_handle=%#x' % self._wait_handle) + info.append(f'wait_handle={self._wait_handle:#x}') return info def _unregister_wait_cb(self, fut): @@ -146,9 +155,9 @@ def _unregister_wait(self): self._unregister_wait_cb(None) - def cancel(self): + def cancel(self, msg=None): self._unregister_wait() - return super().cancel() + return super().cancel(msg=msg) def set_exception(self, exception): self._unregister_wait() @@ -297,9 +306,6 @@ def close(self): class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Windows version of selector event loop.""" - def _socketpair(self): - return windows_utils.socketpair() - class ProactorEventLoop(proactor_events.BaseProactorEventLoop): """Windows version of proactor event loop using IOCP.""" @@ -309,20 +315,35 @@ def __init__(self, proactor=None): proactor = IocpProactor() super().__init__(proactor) - def _socketpair(self): - return windows_utils.socketpair() - - @coroutine - def create_pipe_connection(self, protocol_factory, address): + def _run_forever_setup(self): + assert self._self_reading_future is None + self.call_soon(self._loop_self_reading) + super()._run_forever_setup() + + def _run_forever_cleanup(self): + super()._run_forever_cleanup() + if self._self_reading_future is not None: + ov = self._self_reading_future._ov + self._self_reading_future.cancel() + # self_reading_future always uses IOCP, so even though it's + # been cancelled, we need to make sure that the IOCP message + # is received so that the kernel is not holding on to the + # memory, possibly causing memory corruption later. Only + # unregister it if IO is complete in all respects. Otherwise + # we need another _poll() later to complete the IO. + if ov is not None and not ov.pending: + self._proactor._unregister(ov) + self._self_reading_future = None + + async def create_pipe_connection(self, protocol_factory, address): f = self._proactor.connect_pipe(address) - pipe = yield from f + pipe = await f protocol = protocol_factory() trans = self._make_duplex_pipe_transport(pipe, protocol, extra={'addr': address}) return trans, protocol - @coroutine - def start_serving_pipe(self, protocol_factory, address): + async def start_serving_pipe(self, protocol_factory, address): server = PipeServer(address) def loop_accept_pipe(f=None): @@ -347,6 +368,10 @@ def loop_accept_pipe(f=None): return f = self._proactor.accept_pipe(pipe) + except BrokenPipeError: + if pipe and pipe.fileno() != -1: + pipe.close() + self.call_soon(loop_accept_pipe) except OSError as exc: if pipe and pipe.fileno() != -1: self.call_exception_handler({ @@ -358,7 +383,8 @@ def loop_accept_pipe(f=None): elif self._debug: logger.warning("Accept pipe failed on pipe %r", pipe, exc_info=True) - except futures.CancelledError: + self.call_soon(loop_accept_pipe) + except exceptions.CancelledError: if pipe: pipe.close() else: @@ -368,28 +394,22 @@ def loop_accept_pipe(f=None): self.call_soon(loop_accept_pipe) return [server] - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): + async def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): waiter = self.create_future() transp = _WindowsSubprocessTransport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, waiter=waiter, extra=extra, **kwargs) try: - yield from waiter - except Exception as exc: - # Workaround CPython bug #23353: using yield/yield-from in an - # except block of a generator doesn't clear properly sys.exc_info() - err = exc - else: - err = None - - if err is not None: + await waiter + except (SystemExit, KeyboardInterrupt): + raise + except BaseException: transp.close() - yield from transp._wait() - raise err + await transp._wait() + raise return transp @@ -397,7 +417,7 @@ def _make_subprocess_transport(self, protocol, args, shell, class IocpProactor: """Proactor implementation using IOCP.""" - def __init__(self, concurrency=0xffffffff): + def __init__(self, concurrency=INFINITE): self._loop = None self._results = [] self._iocp = _overlapped.CreateIoCompletionPort( @@ -407,10 +427,16 @@ def __init__(self, concurrency=0xffffffff): self._unregistered = [] self._stopped_serving = weakref.WeakSet() + def _check_closed(self): + if self._iocp is None: + raise RuntimeError('IocpProactor is closed') + def __repr__(self): - return ('<%s overlapped#=%s result#=%s>' - % (self.__class__.__name__, len(self._cache), - len(self._results))) + info = ['overlapped#=%s' % len(self._cache), + 'result#=%s' % len(self._results)] + if self._iocp is None: + info.append('closed') + return '<%s %s>' % (self.__class__.__name__, " ".join(info)) def set_loop(self, loop): self._loop = loop @@ -420,13 +446,40 @@ def select(self, timeout=None): self._poll(timeout) tmp = self._results self._results = [] - return tmp + try: + return tmp + finally: + # Needed to break cycles when an exception occurs. + tmp = None def _result(self, value): fut = self._loop.create_future() fut.set_result(value) return fut + @staticmethod + def finish_socket_func(trans, key, ov): + try: + return ov.getresult() + except OSError as exc: + if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED, + _overlapped.ERROR_OPERATION_ABORTED): + raise ConnectionResetError(*exc.args) + else: + raise + + @classmethod + def _finish_recvfrom(cls, trans, key, ov, *, empty_result): + try: + return cls.finish_socket_func(trans, key, ov) + except OSError as exc: + # WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same + # socket is used to send to an address that is not listening. + if exc.winerror == _overlapped.ERROR_PORT_UNREACHABLE: + return empty_result, None + else: + raise + def recv(self, conn, nbytes, flags=0): self._register_with_iocp(conn) ov = _overlapped.Overlapped(NULL) @@ -438,16 +491,50 @@ def recv(self, conn, nbytes, flags=0): except BrokenPipeError: return self._result(b'') - def finish_recv(trans, key, ov): - try: - return ov.getresult() - except OSError as exc: - if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: - raise ConnectionResetError(*exc.args) - else: - raise + return self._register(ov, conn, self.finish_socket_func) + + def recv_into(self, conn, buf, flags=0): + self._register_with_iocp(conn) + ov = _overlapped.Overlapped(NULL) + try: + if isinstance(conn, socket.socket): + ov.WSARecvInto(conn.fileno(), buf, flags) + else: + ov.ReadFileInto(conn.fileno(), buf) + except BrokenPipeError: + return self._result(0) - return self._register(ov, conn, finish_recv) + return self._register(ov, conn, self.finish_socket_func) + + def recvfrom(self, conn, nbytes, flags=0): + self._register_with_iocp(conn) + ov = _overlapped.Overlapped(NULL) + try: + ov.WSARecvFrom(conn.fileno(), nbytes, flags) + except BrokenPipeError: + return self._result((b'', None)) + + return self._register(ov, conn, partial(self._finish_recvfrom, + empty_result=b'')) + + def recvfrom_into(self, conn, buf, flags=0): + self._register_with_iocp(conn) + ov = _overlapped.Overlapped(NULL) + try: + ov.WSARecvFromInto(conn.fileno(), buf, flags) + except BrokenPipeError: + return self._result((0, None)) + + return self._register(ov, conn, partial(self._finish_recvfrom, + empty_result=0)) + + def sendto(self, conn, buf, flags=0, addr=None): + self._register_with_iocp(conn) + ov = _overlapped.Overlapped(NULL) + + ov.WSASendTo(conn.fileno(), buf, flags, addr) + + return self._register(ov, conn, self.finish_socket_func) def send(self, conn, buf, flags=0): self._register_with_iocp(conn) @@ -457,16 +544,7 @@ def send(self, conn, buf, flags=0): else: ov.WriteFile(conn.fileno(), buf) - def finish_send(trans, key, ov): - try: - return ov.getresult() - except OSError as exc: - if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: - raise ConnectionResetError(*exc.args) - else: - raise - - return self._register(ov, conn, finish_send) + return self._register(ov, conn, self.finish_socket_func) def accept(self, listener): self._register_with_iocp(listener) @@ -483,12 +561,11 @@ def finish_accept(trans, key, ov): conn.settimeout(listener.gettimeout()) return conn, conn.getpeername() - @coroutine - def accept_coro(future, conn): + async def accept_coro(future, conn): # Coroutine closing the accept socket if the future is cancelled try: - yield from future - except futures.CancelledError: + await future + except exceptions.CancelledError: conn.close() raise @@ -498,6 +575,14 @@ def accept_coro(future, conn): return future def connect(self, conn, address): + if conn.type == socket.SOCK_DGRAM: + # WSAConnect will complete immediately for UDP sockets so we don't + # need to register any IOCP operation + _overlapped.WSAConnect(conn.fileno(), address) + fut = self._loop.create_future() + fut.set_result(None) + return fut + self._register_with_iocp(conn) # The socket needs to be locally bound before we call ConnectEx(). try: @@ -520,6 +605,18 @@ def finish_connect(trans, key, ov): return self._register(ov, conn, finish_connect) + def sendfile(self, sock, file, offset, count): + self._register_with_iocp(sock) + ov = _overlapped.Overlapped(NULL) + offset_low = offset & 0xffff_ffff + offset_high = (offset >> 32) & 0xffff_ffff + ov.TransmitFile(sock.fileno(), + msvcrt.get_osfhandle(file.fileno()), + offset_low, offset_high, + count, 0, 0) + + return self._register(ov, sock, self.finish_socket_func) + def accept_pipe(self, pipe): self._register_with_iocp(pipe) ov = _overlapped.Overlapped(NULL) @@ -537,13 +634,12 @@ def finish_accept_pipe(trans, key, ov): return self._register(ov, pipe, finish_accept_pipe) - @coroutine - def connect_pipe(self, address): + async def connect_pipe(self, address): delay = CONNECT_PIPE_INIT_DELAY while True: - # Unfortunately there is no way to do an overlapped connect to a pipe. - # Call CreateFile() in a loop until it doesn't fail with - # ERROR_PIPE_BUSY + # Unfortunately there is no way to do an overlapped connect to + # a pipe. Call CreateFile() in a loop until it doesn't fail with + # ERROR_PIPE_BUSY. try: handle = _overlapped.ConnectPipe(address) break @@ -553,7 +649,7 @@ def connect_pipe(self, address): # ConnectPipe() failed with ERROR_PIPE_BUSY: retry later delay = min(delay * 2, CONNECT_PIPE_MAX_DELAY) - yield from tasks.sleep(delay, loop=self._loop) + await tasks.sleep(delay) return windows_utils.PipeHandle(handle) @@ -573,6 +669,8 @@ def _wait_cancel(self, event, done_callback): return fut def _wait_for_handle(self, handle, timeout, _is_cancel): + self._check_closed() + if timeout is None: ms = _winapi.INFINITE else: @@ -615,6 +713,8 @@ def _register_with_iocp(self, obj): # that succeed immediately. def _register(self, ov, obj, callback): + self._check_closed() + # Return a future which will be set with the result of the # operation when it completes. The future's value is actually # the value returned by callback(). @@ -651,6 +751,7 @@ def _unregister(self, ov): already be signalled (pending in the proactor event queue). It is also safe if the event is never signalled (because it was cancelled). """ + self._check_closed() self._unregistered.append(ov) def _get_accept_socket(self, family): @@ -707,8 +808,10 @@ def _poll(self, timeout=None): else: f.set_result(value) self._results.append(f) + finally: + f = None - # Remove unregisted futures + # Remove unregistered futures for ov in self._unregistered: self._cache.pop(ov.address, None) self._unregistered.clear() @@ -720,8 +823,12 @@ def _stop_serving(self, obj): self._stopped_serving.add(obj) def close(self): + if self._iocp is None: + # already closed + return + # Cancel remaining registered operations. - for address, (fut, ov, obj, callback) in list(self._cache.items()): + for fut, ov, obj, callback in list(self._cache.values()): if fut.cancelled(): # Nothing to do with cancelled futures pass @@ -742,14 +849,25 @@ def close(self): context['source_traceback'] = fut._source_traceback self._loop.call_exception_handler(context) + # Wait until all cancelled overlapped complete: don't exit with running + # overlapped to prevent a crash. Display progress every second if the + # loop is still running. + msg_update = 1.0 + start_time = time.monotonic() + next_msg = start_time + msg_update while self._cache: - if not self._poll(1): - logger.debug('taking long time to close proactor') + if next_msg <= time.monotonic(): + logger.debug('%r is running after closing for %.1f seconds', + self, time.monotonic() - start_time) + next_msg = time.monotonic() + msg_update + + # handle a few events, or timeout + self._poll(msg_update) self._results = [] - if self._iocp is not None: - _winapi.CloseHandle(self._iocp) - self._iocp = None + + _winapi.CloseHandle(self._iocp) + self._iocp = None def __del__(self): self.close() @@ -773,8 +891,13 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop -class _WindowsDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): +class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): _loop_factory = SelectorEventLoop -DefaultEventLoopPolicy = _WindowsDefaultEventLoopPolicy +class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): + _loop_factory = ProactorEventLoop + + +_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy +EventLoop = ProactorEventLoop diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index 7c63fb904b3..ef277fac3e2 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -1,6 +1,4 @@ -""" -Various Windows specific bits and pieces -""" +"""Various Windows specific bits and pieces.""" import sys @@ -11,13 +9,12 @@ import itertools import msvcrt import os -import socket import subprocess import tempfile import warnings -__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle'] +__all__ = 'pipe', 'Popen', 'PIPE', 'PipeHandle' # Constants/globals @@ -29,61 +26,14 @@ _mmap_counter = itertools.count() -if hasattr(socket, 'socketpair'): - # Since Python 3.5, socket.socketpair() is now also available on Windows - socketpair = socket.socketpair -else: - # Replacement for socket.socketpair() - def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): - """A socket pair usable as a self-pipe, for Windows. - - Origin: https://gist.github.com/4325783, by Geert Jansen. - Public domain. - """ - if family == socket.AF_INET: - host = '127.0.0.1' - elif family == socket.AF_INET6: - host = '::1' - else: - raise ValueError("Only AF_INET and AF_INET6 socket address " - "families are supported") - if type != socket.SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with setblocking(0) - # that prevents us from having to create a thread. - lsock = socket.socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen(1) - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket.socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) - - # Replacement for os.pipe() using handles instead of fds def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp(prefix=r'\\.\pipe\python-pipe-%d-%d-' % - (os.getpid(), next(_mmap_counter))) + address = tempfile.mktemp( + prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format( + os.getpid(), next(_mmap_counter))) if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX @@ -138,10 +88,10 @@ def __init__(self, handle): def __repr__(self): if self._handle is not None: - handle = 'handle=%r' % self._handle + handle = f'handle={self._handle!r}' else: handle = 'closed' - return '<%s %s>' % (self.__class__.__name__, handle) + return f'<{self.__class__.__name__} {handle}>' @property def handle(self): @@ -149,7 +99,7 @@ def handle(self): def fileno(self): if self._handle is None: - raise ValueError("I/O operatioon on closed pipe") + raise ValueError("I/O operation on closed pipe") return self._handle def close(self, *, CloseHandle=_winapi.CloseHandle): @@ -157,10 +107,9 @@ def close(self, *, CloseHandle=_winapi.CloseHandle): CloseHandle(self._handle) self._handle = None - def __del__(self): + def __del__(self, _warn=warnings.warn): if self._handle is not None: - warnings.warn("unclosed %r" % self, ResourceWarning, - source=self) + _warn(f"unclosed {self!r}", ResourceWarning, source=self) self.close() def __enter__(self): diff --git a/Lib/asyncore.py b/Lib/asyncore.py deleted file mode 100644 index 0e92be3ad19..00000000000 --- a/Lib/asyncore.py +++ /dev/null @@ -1,642 +0,0 @@ -# -*- Mode: Python -*- -# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp -# Author: Sam Rushing - -# ====================================================================== -# Copyright 1996 by Sam Rushing -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Sam -# Rushing not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# ====================================================================== - -"""Basic infrastructure for asynchronous socket service clients and servers. - -There are only two ways to have a program on a single processor do "more -than one thing at a time". Multi-threaded programming is the simplest and -most popular way to do it, but there is another very different technique, -that lets you have nearly all the advantages of multi-threading, without -actually using multiple threads. it's really only practical if your program -is largely I/O bound. If your program is CPU bound, then pre-emptive -scheduled threads are probably what you really need. Network servers are -rarely CPU-bound, however. - -If your operating system supports the select() system call in its I/O -library (and nearly all do), then you can use it to juggle multiple -communication channels at once; doing other work while your I/O is taking -place in the "background." Although this strategy can seem strange and -complex, especially at first, it is in many ways easier to understand and -control than multi-threaded programming. The module documented here solves -many of the difficult problems for you, making the task of building -sophisticated high-performance network servers and clients a snap. -""" - -import select -import socket -import sys -import time -import warnings - -import os -from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ - ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ - errorcode - -_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, - EBADF}) - -try: - socket_map -except NameError: - socket_map = {} - -def _strerror(err): - try: - return os.strerror(err) - except (ValueError, OverflowError, NameError): - if err in errorcode: - return errorcode[err] - return "Unknown error %s" %err - -class ExitNow(Exception): - pass - -_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit) - -def read(obj): - try: - obj.handle_read_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def write(obj): - try: - obj.handle_write_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def _exception(obj): - try: - obj.handle_expt_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def readwrite(obj, flags): - try: - if flags & select.POLLIN: - obj.handle_read_event() - if flags & select.POLLOUT: - obj.handle_write_event() - if flags & select.POLLPRI: - obj.handle_expt_event() - if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): - obj.handle_close() - except OSError as e: - if e.args[0] not in _DISCONNECTED: - obj.handle_error() - else: - obj.handle_close() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def poll(timeout=0.0, map=None): - if map is None: - map = socket_map - if map: - r = []; w = []; e = [] - for fd, obj in list(map.items()): - is_r = obj.readable() - is_w = obj.writable() - if is_r: - r.append(fd) - # accepting sockets should not be writable - if is_w and not obj.accepting: - w.append(fd) - if is_r or is_w: - e.append(fd) - if [] == r == w == e: - time.sleep(timeout) - return - - r, w, e = select.select(r, w, e, timeout) - - for fd in r: - obj = map.get(fd) - if obj is None: - continue - read(obj) - - for fd in w: - obj = map.get(fd) - if obj is None: - continue - write(obj) - - for fd in e: - obj = map.get(fd) - if obj is None: - continue - _exception(obj) - -def poll2(timeout=0.0, map=None): - # Use the poll() support added to the select module in Python 2.0 - if map is None: - map = socket_map - if timeout is not None: - # timeout is in milliseconds - timeout = int(timeout*1000) - pollster = select.poll() - if map: - for fd, obj in list(map.items()): - flags = 0 - if obj.readable(): - flags |= select.POLLIN | select.POLLPRI - # accepting sockets should not be writable - if obj.writable() and not obj.accepting: - flags |= select.POLLOUT - if flags: - pollster.register(fd, flags) - - r = pollster.poll(timeout) - for fd, flags in r: - obj = map.get(fd) - if obj is None: - continue - readwrite(obj, flags) - -poll3 = poll2 # Alias for backward compatibility - -def loop(timeout=30.0, use_poll=False, map=None, count=None): - if map is None: - map = socket_map - - if use_poll and hasattr(select, 'poll'): - poll_fun = poll2 - else: - poll_fun = poll - - if count is None: - while map: - poll_fun(timeout, map) - - else: - while map and count > 0: - poll_fun(timeout, map) - count = count - 1 - -class dispatcher: - - debug = False - connected = False - accepting = False - connecting = False - closing = False - addr = None - ignore_log_types = frozenset({'warning'}) - - def __init__(self, sock=None, map=None): - if map is None: - self._map = socket_map - else: - self._map = map - - self._fileno = None - - if sock: - # Set to nonblocking just to make sure for cases where we - # get a socket from a blocking source. - sock.setblocking(0) - self.set_socket(sock, map) - self.connected = True - # The constructor no longer requires that the socket - # passed be connected. - try: - self.addr = sock.getpeername() - except OSError as err: - if err.args[0] in (ENOTCONN, EINVAL): - # To handle the case where we got an unconnected - # socket. - self.connected = False - else: - # The socket is broken in some unknown way, alert - # the user and remove it from the map (to prevent - # polling of broken sockets). - self.del_channel(map) - raise - else: - self.socket = None - - def __repr__(self): - status = [self.__class__.__module__+"."+self.__class__.__qualname__] - if self.accepting and self.addr: - status.append('listening') - elif self.connected: - status.append('connected') - if self.addr is not None: - try: - status.append('%s:%d' % self.addr) - except TypeError: - status.append(repr(self.addr)) - return '<%s at %#x>' % (' '.join(status), id(self)) - - def add_channel(self, map=None): - #self.log_info('adding channel %s' % self) - if map is None: - map = self._map - map[self._fileno] = self - - def del_channel(self, map=None): - fd = self._fileno - if map is None: - map = self._map - if fd in map: - #self.log_info('closing channel %d:%s' % (fd, self)) - del map[fd] - self._fileno = None - - def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM): - self.family_and_type = family, type - sock = socket.socket(family, type) - sock.setblocking(0) - self.set_socket(sock) - - def set_socket(self, sock, map=None): - self.socket = sock - self._fileno = sock.fileno() - self.add_channel(map) - - def set_reuse_addr(self): - # try to re-use a server port if possible - try: - self.socket.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, - self.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR) | 1 - ) - except OSError: - pass - - # ================================================== - # predicates for select() - # these are used as filters for the lists of sockets - # to pass to select(). - # ================================================== - - def readable(self): - return True - - def writable(self): - return True - - # ================================================== - # socket object methods. - # ================================================== - - def listen(self, num): - self.accepting = True - if os.name == 'nt' and num > 5: - num = 5 - return self.socket.listen(num) - - def bind(self, addr): - self.addr = addr - return self.socket.bind(addr) - - def connect(self, address): - self.connected = False - self.connecting = True - err = self.socket.connect_ex(address) - if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ - or err == EINVAL and os.name == 'nt': - self.addr = address - return - if err in (0, EISCONN): - self.addr = address - self.handle_connect_event() - else: - raise OSError(err, errorcode[err]) - - def accept(self): - # XXX can return either an address pair or None - try: - conn, addr = self.socket.accept() - except TypeError: - return None - except OSError as why: - if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): - return None - else: - raise - else: - return conn, addr - - def send(self, data): - try: - result = self.socket.send(data) - return result - except OSError as why: - if why.args[0] == EWOULDBLOCK: - return 0 - elif why.args[0] in _DISCONNECTED: - self.handle_close() - return 0 - else: - raise - - def recv(self, buffer_size): - try: - data = self.socket.recv(buffer_size) - if not data: - # a closed connection is indicated by signaling - # a read condition, and having recv() return 0. - self.handle_close() - return b'' - else: - return data - except OSError as why: - # winsock sometimes raises ENOTCONN - if why.args[0] in _DISCONNECTED: - self.handle_close() - return b'' - else: - raise - - def close(self): - self.connected = False - self.accepting = False - self.connecting = False - self.del_channel() - if self.socket is not None: - try: - self.socket.close() - except OSError as why: - if why.args[0] not in (ENOTCONN, EBADF): - raise - - # log and log_info may be overridden to provide more sophisticated - # logging and warning methods. In general, log is for 'hit' logging - # and 'log_info' is for informational, warning and error logging. - - def log(self, message): - sys.stderr.write('log: %s\n' % str(message)) - - def log_info(self, message, type='info'): - if type not in self.ignore_log_types: - print('%s: %s' % (type, message)) - - def handle_read_event(self): - if self.accepting: - # accepting sockets are never connected, they "spawn" new - # sockets that are connected - self.handle_accept() - elif not self.connected: - if self.connecting: - self.handle_connect_event() - self.handle_read() - else: - self.handle_read() - - def handle_connect_event(self): - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - raise OSError(err, _strerror(err)) - self.handle_connect() - self.connected = True - self.connecting = False - - def handle_write_event(self): - if self.accepting: - # Accepting sockets shouldn't get a write event. - # We will pretend it didn't happen. - return - - if not self.connected: - if self.connecting: - self.handle_connect_event() - self.handle_write() - - def handle_expt_event(self): - # handle_expt_event() is called if there might be an error on the - # socket, or if there is OOB data - # check for the error condition first - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - # we can get here when select.select() says that there is an - # exceptional condition on the socket - # since there is an error, we'll go ahead and close the socket - # like we would in a subclassed handle_read() that received no - # data - self.handle_close() - else: - self.handle_expt() - - def handle_error(self): - nil, t, v, tbinfo = compact_traceback() - - # sometimes a user repr method will crash. - try: - self_repr = repr(self) - except: - self_repr = '<__repr__(self) failed for object at %0x>' % id(self) - - self.log_info( - 'uncaptured python exception, closing channel %s (%s:%s %s)' % ( - self_repr, - t, - v, - tbinfo - ), - 'error' - ) - self.handle_close() - - def handle_expt(self): - self.log_info('unhandled incoming priority event', 'warning') - - def handle_read(self): - self.log_info('unhandled read event', 'warning') - - def handle_write(self): - self.log_info('unhandled write event', 'warning') - - def handle_connect(self): - self.log_info('unhandled connect event', 'warning') - - def handle_accept(self): - pair = self.accept() - if pair is not None: - self.handle_accepted(*pair) - - def handle_accepted(self, sock, addr): - sock.close() - self.log_info('unhandled accepted event', 'warning') - - def handle_close(self): - self.log_info('unhandled close event', 'warning') - self.close() - -# --------------------------------------------------------------------------- -# adds simple buffered output capability, useful for simple clients. -# [for more sophisticated usage use asynchat.async_chat] -# --------------------------------------------------------------------------- - -class dispatcher_with_send(dispatcher): - - def __init__(self, sock=None, map=None): - dispatcher.__init__(self, sock, map) - self.out_buffer = b'' - - def initiate_send(self): - num_sent = 0 - num_sent = dispatcher.send(self, self.out_buffer[:65536]) - self.out_buffer = self.out_buffer[num_sent:] - - def handle_write(self): - self.initiate_send() - - def writable(self): - return (not self.connected) or len(self.out_buffer) - - def send(self, data): - if self.debug: - self.log_info('sending %s' % repr(data)) - self.out_buffer = self.out_buffer + data - self.initiate_send() - -# --------------------------------------------------------------------------- -# used for debugging. -# --------------------------------------------------------------------------- - -def compact_traceback(): - t, v, tb = sys.exc_info() - tbinfo = [] - if not tb: # Must have a traceback - raise AssertionError("traceback does not exist") - while tb: - tbinfo.append(( - tb.tb_frame.f_code.co_filename, - tb.tb_frame.f_code.co_name, - str(tb.tb_lineno) - )) - tb = tb.tb_next - - # just to be safe - del tb - - file, function, line = tbinfo[-1] - info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) - return (file, function, line), t, v, info - -def close_all(map=None, ignore_all=False): - if map is None: - map = socket_map - for x in list(map.values()): - try: - x.close() - except OSError as x: - if x.args[0] == EBADF: - pass - elif not ignore_all: - raise - except _reraised_exceptions: - raise - except: - if not ignore_all: - raise - map.clear() - -# Asynchronous File I/O: -# -# After a little research (reading man pages on various unixen, and -# digging through the linux kernel), I've determined that select() -# isn't meant for doing asynchronous file i/o. -# Heartening, though - reading linux/mm/filemap.c shows that linux -# supports asynchronous read-ahead. So _MOST_ of the time, the data -# will be sitting in memory for us already when we go to read it. -# -# What other OS's (besides NT) support async file i/o? [VMS?] -# -# Regardless, this is useful for pipes, and stdin/stdout... - -if os.name == 'posix': - class file_wrapper: - # Here we override just enough to make a file - # look like a socket for the purposes of asyncore. - # The passed fd is automatically os.dup()'d - - def __init__(self, fd): - self.fd = os.dup(fd) - - def __del__(self): - if self.fd >= 0: - warnings.warn("unclosed file %r" % self, ResourceWarning, - source=self) - self.close() - - def recv(self, *args): - return os.read(self.fd, *args) - - def send(self, *args): - return os.write(self.fd, *args) - - def getsockopt(self, level, optname, buflen=None): - if (level == socket.SOL_SOCKET and - optname == socket.SO_ERROR and - not buflen): - return 0 - raise NotImplementedError("Only asyncore specific behaviour " - "implemented.") - - read = recv - write = send - - def close(self): - if self.fd < 0: - return - fd = self.fd - self.fd = -1 - os.close(fd) - - def fileno(self): - return self.fd - - class file_dispatcher(dispatcher): - - def __init__(self, fd, map=None): - dispatcher.__init__(self, None, map) - self.connected = True - try: - fd = fd.fileno() - except AttributeError: - pass - self.set_file(fd) - # set it to non-blocking mode - os.set_blocking(fd, False) - - def set_file(self, fd): - self.socket = file_wrapper(fd) - self._fileno = self.socket.fileno() - self.add_channel() diff --git a/Lib/base64.py b/Lib/base64.py old mode 100755 new mode 100644 index e233647ee76..f95132a4274 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,12 +1,9 @@ -#! /usr/bin/env python3 - """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere -import re import struct import binascii @@ -18,7 +15,7 @@ 'b64encode', 'b64decode', 'b32encode', 'b32decode', 'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode', # Base85 and Ascii85 encodings - 'b85encode', 'b85decode', 'a85encode', 'a85decode', + 'b85encode', 'b85decode', 'a85encode', 'a85decode', 'z85encode', 'z85decode', # Standard Base64 encoding 'standard_b64encode', 'standard_b64decode', # Some common Base64 alternatives. As referenced by RFC 3458, see thread @@ -164,7 +161,6 @@ def urlsafe_b64decode(s): _b32rev = {} def _b32encode(alphabet, s): - global _b32tab2 # Delay the initialization of the table to not waste memory # if the function is never called if alphabet not in _b32tab2: @@ -200,7 +196,6 @@ def _b32encode(alphabet, s): return bytes(encoded) def _b32decode(alphabet, s, casefold=False, map01=None): - global _b32rev # Delay the initialization of the table to not waste memory # if the function is never called if alphabet not in _b32rev: @@ -288,7 +283,7 @@ def b16decode(s, casefold=False): s = _bytes_from_decode_data(s) if casefold: s = s.upper() - if re.search(b'[^0-9A-F]', s): + if s.translate(None, delete=b'0123456789ABCDEF'): raise binascii.Error('Non-base16 digit found') return binascii.unhexlify(s) @@ -334,7 +329,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): wrapcol controls whether the output should have newline (b'\\n') characters added to it. If this is non-zero, each output line will be at most this - many characters long. + many characters long, excluding the trailing newline. pad controls whether the input is padded to a multiple of 4 before encoding. Note that the btoa implementation always pads. @@ -467,9 +462,12 @@ def b85decode(b): # Delay the initialization of tables to not waste memory # if the function is never called if _b85dec is None: - _b85dec = [None] * 256 + # we don't assign to _b85dec directly to avoid issues when + # multiple threads call this function simultaneously + b85dec_tmp = [None] * 256 for i, c in enumerate(_b85alphabet): - _b85dec[c] = i + b85dec_tmp[c] = i + _b85dec = b85dec_tmp b = _bytes_from_decode_data(b) padding = (-len(b)) % 5 @@ -499,6 +497,33 @@ def b85decode(b): result = result[:-padding] return result +_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') +# Translating b85 valid but z85 invalid chars to b'\x00' is required +# to prevent them from being decoded as b85 valid chars. +_z85_b85_decode_diff = b';_`|~' +_z85_decode_translation = bytes.maketrans( + _z85alphabet + _z85_b85_decode_diff, + _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) +) +_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) + +def z85encode(s): + """Encode bytes-like object b in z85 format and return a bytes object.""" + return b85encode(s).translate(_z85_encode_translation) + +def z85decode(s): + """Decode the z85-encoded bytes-like object or ASCII string b + + The result is returned as a bytes object. + """ + s = _bytes_from_decode_data(s) + s = s.translate(_z85_decode_translation) + try: + return b85decode(s) + except ValueError as e: + raise ValueError(e.args[0].replace('base85', 'z85')) from None + # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it # though. The files should be opened in binary mode. @@ -579,7 +604,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Lib/bdb.py b/Lib/bdb.py index 0f3eec653ba..79da4bab9c9 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -2,7 +2,10 @@ import fnmatch import sys +import threading import os +import weakref +from contextlib import contextmanager from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -14,6 +17,166 @@ class BdbQuit(Exception): """Exception to give up completely.""" +E = sys.monitoring.events + +class _MonitoringTracer: + EVENT_CALLBACK_MAP = { + E.PY_START: 'call', + E.PY_RESUME: 'call', + E.PY_THROW: 'call', + E.LINE: 'line', + E.JUMP: 'jump', + E.PY_RETURN: 'return', + E.PY_YIELD: 'return', + E.PY_UNWIND: 'unwind', + E.RAISE: 'exception', + E.STOP_ITERATION: 'exception', + E.INSTRUCTION: 'opcode', + } + + GLOBAL_EVENTS = E.PY_START | E.PY_RESUME | E.PY_THROW | E.PY_UNWIND | E.RAISE + LOCAL_EVENTS = E.LINE | E.JUMP | E.PY_RETURN | E.PY_YIELD | E.STOP_ITERATION + + def __init__(self): + self._tool_id = sys.monitoring.DEBUGGER_ID + self._name = 'bdbtracer' + self._tracefunc = None + self._disable_current_event = False + self._tracing_thread = None + self._enabled = False + + def start_trace(self, tracefunc): + self._tracefunc = tracefunc + self._tracing_thread = threading.current_thread() + curr_tool = sys.monitoring.get_tool(self._tool_id) + if curr_tool is None: + sys.monitoring.use_tool_id(self._tool_id, self._name) + elif curr_tool == self._name: + sys.monitoring.clear_tool_id(self._tool_id) + else: + raise ValueError('Another debugger is using the monitoring tool') + E = sys.monitoring.events + all_events = 0 + for event, cb_name in self.EVENT_CALLBACK_MAP.items(): + callback = self.callback_wrapper(getattr(self, f'{cb_name}_callback'), event) + sys.monitoring.register_callback(self._tool_id, event, callback) + if event != E.INSTRUCTION: + all_events |= event + self.update_local_events() + sys.monitoring.set_events(self._tool_id, self.GLOBAL_EVENTS) + self._enabled = True + + def stop_trace(self): + self._enabled = False + self._tracing_thread = None + curr_tool = sys.monitoring.get_tool(self._tool_id) + if curr_tool != self._name: + return + sys.monitoring.clear_tool_id(self._tool_id) + sys.monitoring.free_tool_id(self._tool_id) + + def disable_current_event(self): + self._disable_current_event = True + + def restart_events(self): + if sys.monitoring.get_tool(self._tool_id) == self._name: + sys.monitoring.restart_events() + + def callback_wrapper(self, func, event): + import functools + + @functools.wraps(func) + def wrapper(*args): + if self._tracing_thread != threading.current_thread(): + return + try: + frame = sys._getframe().f_back + ret = func(frame, *args) + if self._enabled and frame.f_trace: + self.update_local_events() + if ( + self._disable_current_event + and event not in (E.PY_THROW, E.PY_UNWIND, E.RAISE) + ): + return sys.monitoring.DISABLE + else: + return ret + except BaseException: + self.stop_trace() + sys._getframe().f_back.f_trace = None + raise + finally: + self._disable_current_event = False + + return wrapper + + def call_callback(self, frame, code, *args): + local_tracefunc = self._tracefunc(frame, 'call', None) + if local_tracefunc is not None: + frame.f_trace = local_tracefunc + if self._enabled: + sys.monitoring.set_local_events(self._tool_id, code, self.LOCAL_EVENTS) + + def return_callback(self, frame, code, offset, retval): + if frame.f_trace: + frame.f_trace(frame, 'return', retval) + + def unwind_callback(self, frame, code, *args): + if frame.f_trace: + frame.f_trace(frame, 'return', None) + + def line_callback(self, frame, code, *args): + if frame.f_trace and frame.f_trace_lines: + frame.f_trace(frame, 'line', None) + + def jump_callback(self, frame, code, inst_offset, dest_offset): + if dest_offset > inst_offset: + return sys.monitoring.DISABLE + inst_lineno = self._get_lineno(code, inst_offset) + dest_lineno = self._get_lineno(code, dest_offset) + if inst_lineno != dest_lineno: + return sys.monitoring.DISABLE + if frame.f_trace and frame.f_trace_lines: + frame.f_trace(frame, 'line', None) + + def exception_callback(self, frame, code, offset, exc): + if frame.f_trace: + if exc.__traceback__ and hasattr(exc.__traceback__, 'tb_frame'): + tb = exc.__traceback__ + while tb: + if tb.tb_frame.f_locals.get('self') is self: + return + tb = tb.tb_next + frame.f_trace(frame, 'exception', (type(exc), exc, exc.__traceback__)) + + def opcode_callback(self, frame, code, offset): + if frame.f_trace and frame.f_trace_opcodes: + frame.f_trace(frame, 'opcode', None) + + def update_local_events(self, frame=None): + if sys.monitoring.get_tool(self._tool_id) != self._name: + return + if frame is None: + frame = sys._getframe().f_back + while frame is not None: + if frame.f_trace is not None: + if frame.f_trace_opcodes: + events = self.LOCAL_EVENTS | E.INSTRUCTION + else: + events = self.LOCAL_EVENTS + sys.monitoring.set_local_events(self._tool_id, frame.f_code, events) + frame = frame.f_back + + def _get_lineno(self, code, offset): + import dis + last_lineno = None + for start, lineno in dis.findlinestarts(code): + if offset < start: + return last_lineno + last_lineno = lineno + return last_lineno + + class Bdb: """Generic Python debugger base class. @@ -28,11 +191,24 @@ class Bdb: is determined by the __name__ in the frame globals. """ - def __init__(self, skip=None): + def __init__(self, skip=None, backend='settrace'): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None + self.trace_opcodes = False + self.enterframe = None + self.cmdframe = None + self.cmdlineno = None + self.code_linenos = weakref.WeakKeyDictionary() + self.backend = backend + if backend == 'monitoring': + self.monitoring_tracer = _MonitoringTracer() + elif backend == 'settrace': + self.monitoring_tracer = None + else: + raise ValueError(f"Invalid backend '{backend}'") self._load_breaks() @@ -53,6 +229,18 @@ def canonic(self, filename): self.fncache[filename] = canonic return canonic + def start_trace(self): + if self.monitoring_tracer: + self.monitoring_tracer.start_trace(self.trace_dispatch) + else: + sys.settrace(self.trace_dispatch) + + def stop_trace(self): + if self.monitoring_tracer: + self.monitoring_tracer.stop_trace() + else: + sys.settrace(None) + def reset(self): """Set values of attributes as ready to start debugging.""" import linecache @@ -60,6 +248,12 @@ def reset(self): self.botframe = None self._set_stopinfo(None, None) + @contextmanager + def set_enterframe(self, frame): + self.enterframe = frame + yield + self.enterframe = None + def trace_dispatch(self, frame, event, arg): """Dispatch a trace function for debugged frames based on the event. @@ -84,24 +278,28 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ - if self.quitting: - return # None - if event == 'line': - return self.dispatch_line(frame) - if event == 'call': - return self.dispatch_call(frame, arg) - if event == 'return': - return self.dispatch_return(frame, arg) - if event == 'exception': - return self.dispatch_exception(frame, arg) - if event == 'c_call': - return self.trace_dispatch - if event == 'c_exception': - return self.trace_dispatch - if event == 'c_return': + + with self.set_enterframe(frame): + if self.quitting: + return # None + if event == 'line': + return self.dispatch_line(frame) + if event == 'call': + return self.dispatch_call(frame, arg) + if event == 'return': + return self.dispatch_return(frame, arg) + if event == 'exception': + return self.dispatch_exception(frame, arg) + if event == 'c_call': + return self.trace_dispatch + if event == 'c_exception': + return self.trace_dispatch + if event == 'c_return': + return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) + print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch - print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) - return self.trace_dispatch def dispatch_line(self, frame): """Invoke user function and return trace function for line event. @@ -110,9 +308,17 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) + self.restart_events() if self.quitting: raise BdbQuit + elif not self.get_break(frame.f_code.co_filename, frame.f_lineno): + self.disable_current_event() return self.trace_dispatch def dispatch_call(self, frame, arg): @@ -128,12 +334,18 @@ def dispatch_call(self, frame, arg): self.botframe = frame.f_back # (CT) Note that this may also be None! return self.trace_dispatch if not (self.stop_here(frame) or self.break_anywhere(frame)): - # No need to trace this function + # We already know there's no breakpoint in this function + # If it's a next/until/return command, we don't need any CALL event + # and we don't need to set the f_trace on any new frame. + # If it's a step command, it must either hit stop_here, or skip the + # whole module. Either way, we don't need the CALL event here. + self.disable_current_event() return # None # Ignore call events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: return self.trace_dispatch self.user_call(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit return self.trace_dispatch @@ -147,16 +359,25 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: # Ignore return events in generator except when stepping. if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: + # It's possible to trigger a StopIteration exception in + # the caller so we must set the trace function in the caller + self._set_caller_tracefunc(frame) return self.trace_dispatch try: self.frame_returning = frame self.user_return(frame, arg) + self.restart_events() finally: self.frame_returning = None if self.quitting: raise BdbQuit # The user issued a 'next' or 'until' command. if self.stopframe is frame and self.stoplineno != -1: self._set_stopinfo(None, None) + # The previous frame might not have f_trace set, unless we are + # issuing a command that does not expect to stop, we should set + # f_trace + if self.stoplineno != -1: + self._set_caller_tracefunc(frame) return self.trace_dispatch def dispatch_exception(self, frame, arg): @@ -173,6 +394,7 @@ def dispatch_exception(self, frame, arg): if not (frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] is StopIteration and arg[2] is None): self.user_exception(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit # Stop at the StopIteration or GeneratorExit exception when the user # has set stopframe in a generator by issuing a return command, or a @@ -182,10 +404,26 @@ def dispatch_exception(self, frame, arg): and self.stopframe.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] in (StopIteration, GeneratorExit)): self.user_exception(frame, arg) + self.restart_events() if self.quitting: raise BdbQuit return self.trace_dispatch + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + + Opcode event will always trigger the user callback. For now the only + opcode event is from an inline set_trace() and we want to stop there + unconditionally. + """ + self.user_opcode(frame) + self.restart_events() + if self.quitting: raise BdbQuit + return self.trace_dispatch + # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. @@ -249,9 +487,25 @@ def do_clear(self, arg): raise NotImplementedError("subclass of bdb must implement do_clear()") def break_anywhere(self, frame): - """Return True if there is any breakpoint for frame's filename. + """Return True if there is any breakpoint in that frame + """ + filename = self.canonic(frame.f_code.co_filename) + if filename not in self.breaks: + return False + for lineno in self.breaks[filename]: + if self._lineno_in_frame(lineno, frame): + return True + return False + + def _lineno_in_frame(self, lineno, frame): + """Return True if the line number is in the frame's code object. """ - return self.canonic(frame.f_code.co_filename) in self.breaks + code = frame.f_code + if lineno < code.co_firstlineno: + return False + if code not in self.code_linenos: + self.code_linenos[code] = set(lineno for _, _, lineno in code.co_lines()) + return lineno in self.code_linenos[code] # Derived classes should override the user_* methods # to gain control. @@ -272,7 +526,24 @@ def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass + + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + if self.monitoring_tracer: + self.monitoring_tracer.update_local_events() + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -285,6 +556,21 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self, current_frame): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame, unless + # the caller is the botframe. + caller_frame = current_frame.f_back + if caller_frame and not caller_frame.f_trace and caller_frame is not self.botframe: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -299,24 +585,22 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) + + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_stopinfo(None, None, opcode=True) def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: - self._set_stopinfo(frame, None, -1) + self._set_stopinfo(frame, frame, -1) else: self._set_stopinfo(frame.f_back, frame) @@ -325,15 +609,21 @@ def set_trace(self, frame=None): If frame is not specified, debugging starts from caller's frame. """ + self.stop_trace() if frame is None: frame = sys._getframe().f_back self.reset() - while frame: - frame.f_trace = self.trace_dispatch - self.botframe = frame - frame = frame.f_back - self.set_step() - sys.settrace(self.trace_dispatch) + with self.set_enterframe(frame): + while frame: + frame.f_trace = self.trace_dispatch + self.botframe = frame + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work + frame.f_trace_lines = True + frame = frame.f_back + self.set_stepinstr() + self.enterframe = None + self.start_trace() def set_continue(self): """Stop only at breakpoints or when finished. @@ -344,11 +634,16 @@ def set_continue(self): self._set_stopinfo(self.botframe, None, -1) if not self.breaks: # no breakpoints; run without debugger overhead - sys.settrace(None) + self.stop_trace() frame = sys._getframe().f_back while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + if self.backend == 'monitoring': + self.monitoring_tracer.update_local_events() + self.frame_trace_lines_opcodes = {} def set_quit(self): """Set quitting attribute to True. @@ -358,7 +653,7 @@ def set_quit(self): self.stopframe = self.botframe self.returnframe = None self.quitting = True - sys.settrace(None) + self.stop_trace() # Derived classes and clients can call the following methods # to manipulate breakpoints. These methods return an @@ -387,6 +682,14 @@ def set_break(self, filename, lineno, temporary=False, cond=None, return 'Line %s:%d does not exist' % (filename, lineno) self._add_to_breaks(filename, lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) + # After we set a new breakpoint, we need to search through all frames + # and set f_trace to trace_dispatch if there could be a breakpoint in + # that frame. + frame = self.enterframe + while frame: + if self.break_anywhere(frame): + frame.f_trace = self.trace_dispatch + frame = frame.f_back return None def _load_breaks(self): @@ -578,6 +881,16 @@ def format_stack_entry(self, frame_lineno, lprefix=': '): s += f'{lprefix}Warning: lineno is None' return s + def disable_current_event(self): + """Disable the current event.""" + if self.backend == 'monitoring': + self.monitoring_tracer.disable_current_event() + + def restart_events(self): + """Restart all events.""" + if self.backend == 'monitoring': + self.monitoring_tracer.restart_events() + # The following methods can be called by clients to use # a debugger to debug a statement or an expression. # Both can be given as a string, or a code object. @@ -595,14 +908,14 @@ def run(self, cmd, globals=None, locals=None): self.reset() if isinstance(cmd, str): cmd = compile(cmd, "", "exec") - sys.settrace(self.trace_dispatch) + self.start_trace() try: exec(cmd, globals, locals) except BdbQuit: pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() def runeval(self, expr, globals=None, locals=None): """Debug an expression executed via the eval() function. @@ -615,14 +928,14 @@ def runeval(self, expr, globals=None, locals=None): if locals is None: locals = globals self.reset() - sys.settrace(self.trace_dispatch) + self.start_trace() try: return eval(expr, globals, locals) except BdbQuit: pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() def runctx(self, cmd, globals, locals): """For backwards-compatibility. Defers to run().""" @@ -637,7 +950,7 @@ def runcall(self, func, /, *args, **kwds): Return the result of the function call. """ self.reset() - sys.settrace(self.trace_dispatch) + self.start_trace() res = None try: res = func(*args, **kwds) @@ -645,7 +958,7 @@ def runcall(self, func, /, *args, **kwds): pass finally: self.quitting = True - sys.settrace(None) + self.stop_trace() return res diff --git a/Lib/bz2.py b/Lib/bz2.py index fabe4f73c8d..eb58f4da596 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -10,20 +10,20 @@ __author__ = "Nadeem Vawda " from builtins import open as _builtin_open +from compression._common import _streams import io import os -import _compression from _bz2 import BZ2Compressor, BZ2Decompressor -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 -class BZ2File(_compression.BaseStream): +class BZ2File(_streams.BaseStream): """A file object providing transparent bzip2 (de)compression. @@ -54,7 +54,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9): """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") @@ -88,7 +88,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9): raise TypeError("filename must be a str, bytes, file or PathLike object") if self._mode == _MODE_READ: - raw = _compression.DecompressReader(self._fp, + raw = _streams.DecompressReader(self._fp, BZ2Decompressor, trailing_error=OSError) self._buffer = io.BufferedReader(raw) else: @@ -100,7 +100,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -115,13 +115,21 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED self._buffer = None @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" @@ -240,7 +248,7 @@ def writelines(self, seq): Line separators are not added between the written byte strings. """ - return _compression.BaseStream.writelines(self, seq) + return _streams.BaseStream.writelines(self, seq) def seek(self, offset, whence=io.SEEK_SET): """Change the file position. diff --git a/Lib/calendar.py b/Lib/calendar.py index 657396439c9..18f76d52ff8 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -7,6 +7,7 @@ import sys import datetime +from enum import IntEnum, global_enum import locale as _locale from itertools import repeat @@ -16,6 +17,9 @@ "timegm", "month_name", "month_abbr", "day_name", "day_abbr", "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", "LocaleHTMLCalendar", "weekheader", + "Day", "Month", "JANUARY", "FEBRUARY", "MARCH", + "APRIL", "MAY", "JUNE", "JULY", + "AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER", "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY", "SUNDAY"] @@ -23,7 +27,9 @@ error = ValueError # Exceptions raised for bad input -class IllegalMonthError(ValueError): +# This is trick for backward compatibility. Since 3.13, we will raise IllegalMonthError instead of +# IndexError for bad month number(out of 1-12). But we can't remove IndexError for backward compatibility. +class IllegalMonthError(ValueError, IndexError): def __init__(self, month): self.month = month def __str__(self): @@ -37,9 +43,47 @@ def __str__(self): return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday -# Constants for months referenced later -January = 1 -February = 2 +def __getattr__(name): + if name in ('January', 'February'): + import warnings + warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead", + DeprecationWarning, stacklevel=2) + if name == 'January': + return 1 + else: + return 2 + + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") + + +# Constants for months +@global_enum +class Month(IntEnum): + JANUARY = 1 + FEBRUARY = 2 + MARCH = 3 + APRIL = 4 + MAY = 5 + JUNE = 6 + JULY = 7 + AUGUST = 8 + SEPTEMBER = 9 + OCTOBER = 10 + NOVEMBER = 11 + DECEMBER = 12 + + +# Constants for days +@global_enum +class Day(IntEnum): + MONDAY = 0 + TUESDAY = 1 + WEDNESDAY = 2 + THURSDAY = 3 + FRIDAY = 4 + SATURDAY = 5 + SUNDAY = 6 + # Number of days per month (except for February in leap years) mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -95,9 +139,6 @@ def __len__(self): month_name = _localized_month('%B') month_abbr = _localized_month('%b') -# Constants for weekdays -(MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY) = range(7) - def isleap(year): """Return True for leap years, False for non-leap years.""" @@ -116,21 +157,24 @@ def weekday(year, month, day): """Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31).""" if not datetime.MINYEAR <= year <= datetime.MAXYEAR: year = 2000 + year % 400 - return datetime.date(year, month, day).weekday() + return Day(datetime.date(year, month, day).weekday()) -def monthrange(year, month): - """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for - year, month.""" +def _validate_month(month): if not 1 <= month <= 12: raise IllegalMonthError(month) + +def monthrange(year, month): + """Return weekday of first day of month (0-6 ~ Mon-Sun) + and number of days (28-31) for year, month.""" + _validate_month(month) day1 = weekday(year, month, 1) - ndays = mdays[month] + (month == February and isleap(year)) + ndays = mdays[month] + (month == FEBRUARY and isleap(year)) return day1, ndays def _monthlen(year, month): - return mdays[month] + (month == February and isleap(year)) + return mdays[month] + (month == FEBRUARY and isleap(year)) def _prevmonth(year, month): @@ -260,10 +304,7 @@ def yeardatescalendar(self, year, width=3): Each month contains between 4 and 6 weeks and each week contains 1-7 days. Days are datetime.date objects. """ - months = [ - self.monthdatescalendar(year, i) - for i in range(January, January+12) - ] + months = [self.monthdatescalendar(year, m) for m in Month] return [months[i:i+width] for i in range(0, len(months), width) ] def yeardays2calendar(self, year, width=3): @@ -273,10 +314,7 @@ def yeardays2calendar(self, year, width=3): (day number, weekday number) tuples. Day numbers outside this month are zero. """ - months = [ - self.monthdays2calendar(year, i) - for i in range(January, January+12) - ] + months = [self.monthdays2calendar(year, m) for m in Month] return [months[i:i+width] for i in range(0, len(months), width) ] def yeardayscalendar(self, year, width=3): @@ -285,10 +323,7 @@ def yeardayscalendar(self, year, width=3): yeardatescalendar()). Entries in the week lists are day numbers. Day numbers outside this month are zero. """ - months = [ - self.monthdayscalendar(year, i) - for i in range(January, January+12) - ] + months = [self.monthdayscalendar(year, m) for m in Month] return [months[i:i+width] for i in range(0, len(months), width) ] @@ -340,6 +375,8 @@ def formatmonthname(self, theyear, themonth, width, withyear=True): """ Return a formatted month name. """ + _validate_month(themonth) + s = month_name[themonth] if withyear: s = "%s %r" % (s, theyear) @@ -391,6 +428,7 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): headers = (header for k in months) a(formatstring(headers, colwidth, c).rstrip()) a('\n'*l) + # max number of weeks for this row height = max(len(cal) for cal in row) for j in range(height): @@ -470,6 +508,7 @@ def formatmonthname(self, theyear, themonth, withyear=True): """ Return a month name as a table row. """ + _validate_month(themonth) if withyear: s = '%s %s' % (month_name[themonth], theyear) else: @@ -509,7 +548,7 @@ def formatyear(self, theyear, width=3): a('\n') a('%s' % ( width, self.cssclass_year_head, theyear)) - for i in range(January, January+12, width): + for i in range(JANUARY, JANUARY+12, width): # months in this row months = range(i, min(i+width, 13)) a('') @@ -555,8 +594,6 @@ def __enter__(self): _locale.setlocale(_locale.LC_TIME, self.locale) def __exit__(self, *args): - if self.oldlocale is None: - return _locale.setlocale(_locale.LC_TIME, self.oldlocale) @@ -610,6 +647,117 @@ def formatmonthname(self, theyear, themonth, withyear=True): with different_locale(self.locale): return super().formatmonthname(theyear, themonth, withyear) + +class _CLIDemoCalendar(TextCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + def formatweek(self, theweek, width, *, highlight_day=None): + """ + Returns a single week in a string (no newline). + """ + if highlight_day: + from _colorize import get_colors + + ansi = get_colors() + highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" + reset = ansi.RESET + else: + highlight = reset = "" + + return ' '.join( + ( + f"{highlight}{self.formatday(d, wd, width)}{reset}" + if d == highlight_day + else self.formatday(d, wd, width) + ) + for (d, wd) in theweek + ) + + def formatmonth(self, theyear, themonth, w=0, l=0): + """ + Return a month's calendar string (multi-line). + """ + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month == themonth + ): + highlight_day = self.highlight_day.day + else: + highlight_day = None + w = max(2, w) + l = max(1, l) + s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1) + s = s.rstrip() + s += '\n' * l + s += self.formatweekheader(w).rstrip() + s += '\n' * l + for week in self.monthdays2calendar(theyear, themonth): + s += self.formatweek(week, w, highlight_day=highlight_day).rstrip() + s += '\n' * l + return s + + def formatyear(self, theyear, w=2, l=1, c=6, m=3): + """ + Returns a year's calendar as a multi-line string. + """ + w = max(2, w) + l = max(1, l) + c = max(2, c) + colwidth = (w + 1) * 7 - 1 + v = [] + a = v.append + a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + a('\n'*l) + header = self.formatweekheader(w) + for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): + # months in this row + months = range(m*i+1, min(m*(i+1)+1, 13)) + a('\n'*l) + names = (self.formatmonthname(theyear, k, colwidth, False) + for k in months) + a(formatstring(names, colwidth, c).rstrip()) + a('\n'*l) + headers = (header for k in months) + a(formatstring(headers, colwidth, c).rstrip()) + a('\n'*l) + + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month in months + ): + month_pos = months.index(self.highlight_day.month) + else: + month_pos = None + + # max number of weeks for this row + height = max(len(cal) for cal in row) + for j in range(height): + weeks = [] + for k, cal in enumerate(row): + if j >= len(cal): + weeks.append('') + else: + day = ( + self.highlight_day.day if k == month_pos else None + ) + weeks.append( + self.formatweek(cal[j], w, highlight_day=day) + ) + a(formatstring(weeks, colwidth, c).rstrip()) + a('\n' * l) + return ''.join(v) + + +class _CLIDemoLocaleCalendar(LocaleTextCalendar, _CLIDemoCalendar): + def __init__(self, highlight_day=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.highlight_day = highlight_day + + # Support for old module level interface c = TextCalendar() @@ -660,9 +808,9 @@ def timegm(tuple): return seconds -def main(args): +def main(args=None): import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) textgroup = parser.add_argument_group('text only arguments') htmlgroup = parser.add_argument_group('html only arguments') textgroup.add_argument( @@ -693,7 +841,7 @@ def main(args): parser.add_argument( "-L", "--locale", default=None, - help="locale to be used from month and weekday names" + help="locale to use for month and weekday names" ) parser.add_argument( "-e", "--encoding", @@ -706,10 +854,15 @@ def main(args): choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "-f", "--first-weekday", + type=int, default=0, + help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + ) parser.add_argument( "year", nargs='?', type=int, - help="year number (1-9999)" + help="year number" ) parser.add_argument( "month", @@ -717,42 +870,47 @@ def main(args): help="month number (1-12, text only)" ) - options = parser.parse_args(args[1:]) + options = parser.parse_args(args) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") sys.exit(1) locale = options.locale, options.encoding + today = datetime.date.today() if options.type == "html": + if options.month: + parser.error("incorrect number of arguments") + sys.exit(1) if options.locale: cal = LocaleHTMLCalendar(locale=locale) else: cal = HTMLCalendar() + cal.setfirstweekday(options.first_weekday) encoding = options.encoding if encoding is None: encoding = sys.getdefaultencoding() optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write if options.year is None: - write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif options.month is None: - write(cal.formatyearpage(options.year, **optdict)) + write(cal.formatyearpage(today.year, **optdict)) else: - parser.error("incorrect number of arguments") - sys.exit(1) + write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: - cal = LocaleTextCalendar(locale=locale) + cal = _CLIDemoLocaleCalendar(highlight_day=today, locale=locale) else: - cal = TextCalendar() + cal = _CLIDemoCalendar(highlight_day=today) + cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months + else: + _validate_month(options.month) if options.year is None: - result = cal.formatyear(datetime.date.today().year, **optdict) + result = cal.formatyear(today.year, **optdict) elif options.month is None: result = cal.formatyear(options.year, **optdict) else: @@ -765,4 +923,4 @@ def main(args): if __name__ == "__main__": - main(sys.argv) + main() diff --git a/Lib/cgi.py b/Lib/cgi.py deleted file mode 100755 index 8787567be7c..00000000000 --- a/Lib/cgi.py +++ /dev/null @@ -1,1012 +0,0 @@ -#! /usr/local/bin/python - -# NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is -# intentionally NOT "/usr/bin/env python". On many systems -# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI -# scripts, and /usr/local/bin is the default directory where Python is -# installed, so /usr/bin/env would be unable to find python. Granted, -# binary installations by Linux vendors often install Python in -# /usr/bin. So let those vendors patch cgi.py to match their choice -# of installation. - -"""Support module for CGI (Common Gateway Interface) scripts. - -This module defines a number of utilities for use by CGI scripts -written in Python. - -The global variable maxlen can be set to an integer indicating the maximum size -of a POST request. POST requests larger than this size will result in a -ValueError being raised during parsing. The default value of this variable is 0, -meaning the request size is unlimited. -""" - -# History -# ------- -# -# Michael McLay started this module. Steve Majewski changed the -# interface to SvFormContentDict and FormContentDict. The multipart -# parsing was inspired by code submitted by Andreas Paepcke. Guido van -# Rossum rewrote, reformatted and documented the module and is currently -# responsible for its maintenance. -# - -__version__ = "2.6" - - -# Imports -# ======= - -from io import StringIO, BytesIO, TextIOWrapper -from collections.abc import Mapping -import sys -import os -import urllib.parse -from email.parser import FeedParser -from email.message import Message -import html -import locale -import tempfile -import warnings - -__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart", - "parse_header", "test", "print_exception", "print_environ", - "print_form", "print_directory", "print_arguments", - "print_environ_usage"] - - -warnings._deprecated(__name__, remove=(3,13)) - -# Logging support -# =============== - -logfile = "" # Filename to log to, if not empty -logfp = None # File object to log to, if not None - -def initlog(*allargs): - """Write a log message, if there is a log file. - - Even though this function is called initlog(), you should always - use log(); log is a variable that is set either to initlog - (initially), to dolog (once the log file has been opened), or to - nolog (when logging is disabled). - - The first argument is a format string; the remaining arguments (if - any) are arguments to the % operator, so e.g. - log("%s: %s", "a", "b") - will write "a: b" to the log file, followed by a newline. - - If the global logfp is not None, it should be a file object to - which log data is written. - - If the global logfp is None, the global logfile may be a string - giving a filename to open, in append mode. This file should be - world writable!!! If the file can't be opened, logging is - silently disabled (since there is no safe place where we could - send an error message). - - """ - global log, logfile, logfp - warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead", - DeprecationWarning, stacklevel=2) - if logfile and not logfp: - try: - logfp = open(logfile, "a", encoding="locale") - except OSError: - pass - if not logfp: - log = nolog - else: - log = dolog - log(*allargs) - -def dolog(fmt, *args): - """Write a log message to the log file. See initlog() for docs.""" - logfp.write(fmt%args + "\n") - -def nolog(*allargs): - """Dummy function, assigned to log when logging is disabled.""" - pass - -def closelog(): - """Close the log file.""" - global log, logfile, logfp - logfile = '' - if logfp: - logfp.close() - logfp = None - log = initlog - -log = initlog # The current logging function - - -# Parsing functions -# ================= - -# Maximum input we will accept when REQUEST_METHOD is POST -# 0 ==> unlimited input -maxlen = 0 - -def parse(fp=None, environ=os.environ, keep_blank_values=0, - strict_parsing=0, separator='&'): - """Parse a query in the environment or from a file (default stdin) - - Arguments, all optional: - - fp : file pointer; default: sys.stdin.buffer - - environ : environment dictionary; default: os.environ - - keep_blank_values: flag indicating whether blank values in - percent-encoded forms should be treated as blank strings. - A true value indicates that blanks should be retained as - blank strings. The default false value indicates that - blank values are to be ignored and treated as if they were - not included. - - strict_parsing: flag indicating what to do with parsing errors. - If false (the default), errors are silently ignored. - If true, errors raise a ValueError exception. - - separator: str. The symbol to use for separating the query arguments. - Defaults to &. - """ - if fp is None: - fp = sys.stdin - - # field keys and values (except for files) are returned as strings - # an encoding is required to decode the bytes read from self.fp - if hasattr(fp,'encoding'): - encoding = fp.encoding - else: - encoding = 'latin-1' - - # fp.read() must return bytes - if isinstance(fp, TextIOWrapper): - fp = fp.buffer - - if not 'REQUEST_METHOD' in environ: - environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone - if environ['REQUEST_METHOD'] == 'POST': - ctype, pdict = parse_header(environ['CONTENT_TYPE']) - if ctype == 'multipart/form-data': - return parse_multipart(fp, pdict, separator=separator) - elif ctype == 'application/x-www-form-urlencoded': - clength = int(environ['CONTENT_LENGTH']) - if maxlen and clength > maxlen: - raise ValueError('Maximum content length exceeded') - qs = fp.read(clength).decode(encoding) - else: - qs = '' # Unknown content-type - if 'QUERY_STRING' in environ: - if qs: qs = qs + '&' - qs = qs + environ['QUERY_STRING'] - elif sys.argv[1:]: - if qs: qs = qs + '&' - qs = qs + sys.argv[1] - environ['QUERY_STRING'] = qs # XXX Shouldn't, really - elif 'QUERY_STRING' in environ: - qs = environ['QUERY_STRING'] - else: - if sys.argv[1:]: - qs = sys.argv[1] - else: - qs = "" - environ['QUERY_STRING'] = qs # XXX Shouldn't, really - return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, - encoding=encoding, separator=separator) - - -def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): - """Parse multipart input. - - Arguments: - fp : input file - pdict: dictionary containing other parameters of content-type header - encoding, errors: request encoding and error handler, passed to - FieldStorage - - Returns a dictionary just like parse_qs(): keys are the field names, each - value is a list of values for that field. For non-file fields, the value - is a list of strings. - """ - # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always - # represented as 7bit US-ASCII. - boundary = pdict['boundary'].decode('ascii') - ctype = "multipart/form-data; boundary={}".format(boundary) - headers = Message() - headers.set_type(ctype) - try: - headers['Content-Length'] = pdict['CONTENT-LENGTH'] - except KeyError: - pass - fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, - environ={'REQUEST_METHOD': 'POST'}, separator=separator) - return {k: fs.getlist(k) for k in fs} - -def _parseparam(s): - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) - if end < 0: - end = len(s) - f = s[:end] - yield f.strip() - s = s[end:] - -def parse_header(line): - """Parse a Content-type like header. - - Return the main content-type and a dictionary of options. - - """ - parts = _parseparam(';' + line) - key = parts.__next__() - pdict = {} - for p in parts: - i = p.find('=') - if i >= 0: - name = p[:i].strip().lower() - value = p[i+1:].strip() - if len(value) >= 2 and value[0] == value[-1] == '"': - value = value[1:-1] - value = value.replace('\\\\', '\\').replace('\\"', '"') - pdict[name] = value - return key, pdict - - -# Classes for field storage -# ========================= - -class MiniFieldStorage: - - """Like FieldStorage, for use when no file uploads are possible.""" - - # Dummy attributes - filename = None - list = None - type = None - file = None - type_options = {} - disposition = None - disposition_options = {} - headers = {} - - def __init__(self, name, value): - """Constructor from field name and value.""" - self.name = name - self.value = value - # self.file = StringIO(value) - - def __repr__(self): - """Return printable representation.""" - return "MiniFieldStorage(%r, %r)" % (self.name, self.value) - - -class FieldStorage: - - """Store a sequence of fields, reading multipart/form-data. - - This class provides naming, typing, files stored on disk, and - more. At the top level, it is accessible like a dictionary, whose - keys are the field names. (Note: None can occur as a field name.) - The items are either a Python list (if there's multiple values) or - another FieldStorage or MiniFieldStorage object. If it's a single - object, it has the following attributes: - - name: the field name, if specified; otherwise None - - filename: the filename, if specified; otherwise None; this is the - client side filename, *not* the file name on which it is - stored (that's a temporary file you don't deal with) - - value: the value as a *string*; for file uploads, this - transparently reads the file every time you request the value - and returns *bytes* - - file: the file(-like) object from which you can read the data *as - bytes* ; None if the data is stored a simple string - - type: the content-type, or None if not specified - - type_options: dictionary of options specified on the content-type - line - - disposition: content-disposition, or None if not specified - - disposition_options: dictionary of corresponding options - - headers: a dictionary(-like) object (sometimes email.message.Message or a - subclass thereof) containing *all* headers - - The class is subclassable, mostly for the purpose of overriding - the make_file() method, which is called internally to come up with - a file open for reading and writing. This makes it possible to - override the default choice of storing all files in a temporary - directory and unlinking them as soon as they have been opened. - - """ - def __init__(self, fp=None, headers=None, outerboundary=b'', - environ=os.environ, keep_blank_values=0, strict_parsing=0, - limit=None, encoding='utf-8', errors='replace', - max_num_fields=None, separator='&'): - """Constructor. Read multipart/* until last part. - - Arguments, all optional: - - fp : file pointer; default: sys.stdin.buffer - (not used when the request method is GET) - Can be : - 1. a TextIOWrapper object - 2. an object whose read() and readline() methods return bytes - - headers : header dictionary-like object; default: - taken from environ as per CGI spec - - outerboundary : terminating multipart boundary - (for internal use only) - - environ : environment dictionary; default: os.environ - - keep_blank_values: flag indicating whether blank values in - percent-encoded forms should be treated as blank strings. - A true value indicates that blanks should be retained as - blank strings. The default false value indicates that - blank values are to be ignored and treated as if they were - not included. - - strict_parsing: flag indicating what to do with parsing errors. - If false (the default), errors are silently ignored. - If true, errors raise a ValueError exception. - - limit : used internally to read parts of multipart/form-data forms, - to exit from the reading loop when reached. It is the difference - between the form content-length and the number of bytes already - read - - encoding, errors : the encoding and error handler used to decode the - binary stream to strings. Must be the same as the charset defined - for the page sending the form (content-type : meta http-equiv or - header) - - max_num_fields: int. If set, then __init__ throws a ValueError - if there are more than n fields read by parse_qsl(). - - """ - method = 'GET' - self.keep_blank_values = keep_blank_values - self.strict_parsing = strict_parsing - self.max_num_fields = max_num_fields - self.separator = separator - if 'REQUEST_METHOD' in environ: - method = environ['REQUEST_METHOD'].upper() - self.qs_on_post = None - if method == 'GET' or method == 'HEAD': - if 'QUERY_STRING' in environ: - qs = environ['QUERY_STRING'] - elif sys.argv[1:]: - qs = sys.argv[1] - else: - qs = "" - qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape') - fp = BytesIO(qs) - if headers is None: - headers = {'content-type': - "application/x-www-form-urlencoded"} - if headers is None: - headers = {} - if method == 'POST': - # Set default content-type for POST to what's traditional - headers['content-type'] = "application/x-www-form-urlencoded" - if 'CONTENT_TYPE' in environ: - headers['content-type'] = environ['CONTENT_TYPE'] - if 'QUERY_STRING' in environ: - self.qs_on_post = environ['QUERY_STRING'] - if 'CONTENT_LENGTH' in environ: - headers['content-length'] = environ['CONTENT_LENGTH'] - else: - if not (isinstance(headers, (Mapping, Message))): - raise TypeError("headers must be mapping or an instance of " - "email.message.Message") - self.headers = headers - if fp is None: - self.fp = sys.stdin.buffer - # self.fp.read() must return bytes - elif isinstance(fp, TextIOWrapper): - self.fp = fp.buffer - else: - if not (hasattr(fp, 'read') and hasattr(fp, 'readline')): - raise TypeError("fp must be file pointer") - self.fp = fp - - self.encoding = encoding - self.errors = errors - - if not isinstance(outerboundary, bytes): - raise TypeError('outerboundary must be bytes, not %s' - % type(outerboundary).__name__) - self.outerboundary = outerboundary - - self.bytes_read = 0 - self.limit = limit - - # Process content-disposition header - cdisp, pdict = "", {} - if 'content-disposition' in self.headers: - cdisp, pdict = parse_header(self.headers['content-disposition']) - self.disposition = cdisp - self.disposition_options = pdict - self.name = None - if 'name' in pdict: - self.name = pdict['name'] - self.filename = None - if 'filename' in pdict: - self.filename = pdict['filename'] - self._binary_file = self.filename is not None - - # Process content-type header - # - # Honor any existing content-type header. But if there is no - # content-type header, use some sensible defaults. Assume - # outerboundary is "" at the outer level, but something non-false - # inside a multi-part. The default for an inner part is text/plain, - # but for an outer part it should be urlencoded. This should catch - # bogus clients which erroneously forget to include a content-type - # header. - # - # See below for what we do if there does exist a content-type header, - # but it happens to be something we don't understand. - if 'content-type' in self.headers: - ctype, pdict = parse_header(self.headers['content-type']) - elif self.outerboundary or method != 'POST': - ctype, pdict = "text/plain", {} - else: - ctype, pdict = 'application/x-www-form-urlencoded', {} - self.type = ctype - self.type_options = pdict - if 'boundary' in pdict: - self.innerboundary = pdict['boundary'].encode(self.encoding, - self.errors) - else: - self.innerboundary = b"" - - clen = -1 - if 'content-length' in self.headers: - try: - clen = int(self.headers['content-length']) - except ValueError: - pass - if maxlen and clen > maxlen: - raise ValueError('Maximum content length exceeded') - self.length = clen - if self.limit is None and clen >= 0: - self.limit = clen - - self.list = self.file = None - self.done = 0 - if ctype == 'application/x-www-form-urlencoded': - self.read_urlencoded() - elif ctype[:10] == 'multipart/': - self.read_multi(environ, keep_blank_values, strict_parsing) - else: - self.read_single() - - def __del__(self): - try: - self.file.close() - except AttributeError: - pass - - def __enter__(self): - return self - - def __exit__(self, *args): - self.file.close() - - def __repr__(self): - """Return a printable representation.""" - return "FieldStorage(%r, %r, %r)" % ( - self.name, self.filename, self.value) - - def __iter__(self): - return iter(self.keys()) - - def __getattr__(self, name): - if name != 'value': - raise AttributeError(name) - if self.file: - self.file.seek(0) - value = self.file.read() - self.file.seek(0) - elif self.list is not None: - value = self.list - else: - value = None - return value - - def __getitem__(self, key): - """Dictionary style indexing.""" - if self.list is None: - raise TypeError("not indexable") - found = [] - for item in self.list: - if item.name == key: found.append(item) - if not found: - raise KeyError(key) - if len(found) == 1: - return found[0] - else: - return found - - def getvalue(self, key, default=None): - """Dictionary style get() method, including 'value' lookup.""" - if key in self: - value = self[key] - if isinstance(value, list): - return [x.value for x in value] - else: - return value.value - else: - return default - - def getfirst(self, key, default=None): - """ Return the first value received.""" - if key in self: - value = self[key] - if isinstance(value, list): - return value[0].value - else: - return value.value - else: - return default - - def getlist(self, key): - """ Return list of received values.""" - if key in self: - value = self[key] - if isinstance(value, list): - return [x.value for x in value] - else: - return [value.value] - else: - return [] - - def keys(self): - """Dictionary style keys() method.""" - if self.list is None: - raise TypeError("not indexable") - return list(set(item.name for item in self.list)) - - def __contains__(self, key): - """Dictionary style __contains__ method.""" - if self.list is None: - raise TypeError("not indexable") - return any(item.name == key for item in self.list) - - def __len__(self): - """Dictionary style len(x) support.""" - return len(self.keys()) - - def __bool__(self): - if self.list is None: - raise TypeError("Cannot be converted to bool.") - return bool(self.list) - - def read_urlencoded(self): - """Internal: read data in query string format.""" - qs = self.fp.read(self.length) - if not isinstance(qs, bytes): - raise ValueError("%s should return bytes, got %s" \ - % (self.fp, type(qs).__name__)) - qs = qs.decode(self.encoding, self.errors) - if self.qs_on_post: - qs += '&' + self.qs_on_post - query = urllib.parse.parse_qsl( - qs, self.keep_blank_values, self.strict_parsing, - encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields, separator=self.separator) - self.list = [MiniFieldStorage(key, value) for key, value in query] - self.skip_lines() - - FieldStorageClass = None - - def read_multi(self, environ, keep_blank_values, strict_parsing): - """Internal: read a part that is itself multipart.""" - ib = self.innerboundary - if not valid_boundary(ib): - raise ValueError('Invalid boundary in multipart form: %r' % (ib,)) - self.list = [] - if self.qs_on_post: - query = urllib.parse.parse_qsl( - self.qs_on_post, self.keep_blank_values, self.strict_parsing, - encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields, separator=self.separator) - self.list.extend(MiniFieldStorage(key, value) for key, value in query) - - klass = self.FieldStorageClass or self.__class__ - first_line = self.fp.readline() # bytes - if not isinstance(first_line, bytes): - raise ValueError("%s should return bytes, got %s" \ - % (self.fp, type(first_line).__name__)) - self.bytes_read += len(first_line) - - # Ensure that we consume the file until we've hit our inner boundary - while (first_line.strip() != (b"--" + self.innerboundary) and - first_line): - first_line = self.fp.readline() - self.bytes_read += len(first_line) - - # Propagate max_num_fields into the sub class appropriately - max_num_fields = self.max_num_fields - if max_num_fields is not None: - max_num_fields -= len(self.list) - - while True: - parser = FeedParser() - hdr_text = b"" - while True: - data = self.fp.readline() - hdr_text += data - if not data.strip(): - break - if not hdr_text: - break - # parser takes strings, not bytes - self.bytes_read += len(hdr_text) - parser.feed(hdr_text.decode(self.encoding, self.errors)) - headers = parser.close() - - # Some clients add Content-Length for part headers, ignore them - if 'content-length' in headers: - del headers['content-length'] - - limit = None if self.limit is None \ - else self.limit - self.bytes_read - part = klass(self.fp, headers, ib, environ, keep_blank_values, - strict_parsing, limit, - self.encoding, self.errors, max_num_fields, self.separator) - - if max_num_fields is not None: - max_num_fields -= 1 - if part.list: - max_num_fields -= len(part.list) - if max_num_fields < 0: - raise ValueError('Max number of fields exceeded') - - self.bytes_read += part.bytes_read - self.list.append(part) - if part.done or self.bytes_read >= self.length > 0: - break - self.skip_lines() - - def read_single(self): - """Internal: read an atomic part.""" - if self.length >= 0: - self.read_binary() - self.skip_lines() - else: - self.read_lines() - self.file.seek(0) - - bufsize = 8*1024 # I/O buffering size for copy to file - - def read_binary(self): - """Internal: read binary data.""" - self.file = self.make_file() - todo = self.length - if todo >= 0: - while todo > 0: - data = self.fp.read(min(todo, self.bufsize)) # bytes - if not isinstance(data, bytes): - raise ValueError("%s should return bytes, got %s" - % (self.fp, type(data).__name__)) - self.bytes_read += len(data) - if not data: - self.done = -1 - break - self.file.write(data) - todo = todo - len(data) - - def read_lines(self): - """Internal: read lines until EOF or outerboundary.""" - if self._binary_file: - self.file = self.__file = BytesIO() # store data as bytes for files - else: - self.file = self.__file = StringIO() # as strings for other fields - if self.outerboundary: - self.read_lines_to_outerboundary() - else: - self.read_lines_to_eof() - - def __write(self, line): - """line is always bytes, not string""" - if self.__file is not None: - if self.__file.tell() + len(line) > 1000: - self.file = self.make_file() - data = self.__file.getvalue() - self.file.write(data) - self.__file = None - if self._binary_file: - # keep bytes - self.file.write(line) - else: - # decode to string - self.file.write(line.decode(self.encoding, self.errors)) - - def read_lines_to_eof(self): - """Internal: read lines until EOF.""" - while 1: - line = self.fp.readline(1<<16) # bytes - self.bytes_read += len(line) - if not line: - self.done = -1 - break - self.__write(line) - - def read_lines_to_outerboundary(self): - """Internal: read lines until outerboundary. - Data is read as bytes: boundaries and line ends must be converted - to bytes for comparisons. - """ - next_boundary = b"--" + self.outerboundary - last_boundary = next_boundary + b"--" - delim = b"" - last_line_lfend = True - _read = 0 - while 1: - - if self.limit is not None and 0 <= self.limit <= _read: - break - line = self.fp.readline(1<<16) # bytes - self.bytes_read += len(line) - _read += len(line) - if not line: - self.done = -1 - break - if delim == b"\r": - line = delim + line - delim = b"" - if line.startswith(b"--") and last_line_lfend: - strippedline = line.rstrip() - if strippedline == next_boundary: - break - if strippedline == last_boundary: - self.done = 1 - break - odelim = delim - if line.endswith(b"\r\n"): - delim = b"\r\n" - line = line[:-2] - last_line_lfend = True - elif line.endswith(b"\n"): - delim = b"\n" - line = line[:-1] - last_line_lfend = True - elif line.endswith(b"\r"): - # We may interrupt \r\n sequences if they span the 2**16 - # byte boundary - delim = b"\r" - line = line[:-1] - last_line_lfend = False - else: - delim = b"" - last_line_lfend = False - self.__write(odelim + line) - - def skip_lines(self): - """Internal: skip lines until outer boundary if defined.""" - if not self.outerboundary or self.done: - return - next_boundary = b"--" + self.outerboundary - last_boundary = next_boundary + b"--" - last_line_lfend = True - while True: - line = self.fp.readline(1<<16) - self.bytes_read += len(line) - if not line: - self.done = -1 - break - if line.endswith(b"--") and last_line_lfend: - strippedline = line.strip() - if strippedline == next_boundary: - break - if strippedline == last_boundary: - self.done = 1 - break - last_line_lfend = line.endswith(b'\n') - - def make_file(self): - """Overridable: return a readable & writable file. - - The file will be used as follows: - - data is written to it - - seek(0) - - data is read from it - - The file is opened in binary mode for files, in text mode - for other fields - - This version opens a temporary file for reading and writing, - and immediately deletes (unlinks) it. The trick (on Unix!) is - that the file can still be used, but it can't be opened by - another process, and it will automatically be deleted when it - is closed or when the current process terminates. - - If you want a more permanent file, you derive a class which - overrides this method. If you want a visible temporary file - that is nevertheless automatically deleted when the script - terminates, try defining a __del__ method in a derived class - which unlinks the temporary files you have created. - - """ - if self._binary_file: - return tempfile.TemporaryFile("wb+") - else: - return tempfile.TemporaryFile("w+", - encoding=self.encoding, newline = '\n') - - -# Test/debug code -# =============== - -def test(environ=os.environ): - """Robust test CGI script, usable as main program. - - Write minimal HTTP headers and dump all information provided to - the script in HTML form. - - """ - print("Content-type: text/html") - print() - sys.stderr = sys.stdout - try: - form = FieldStorage() # Replace with other classes to test those - print_directory() - print_arguments() - print_form(form) - print_environ(environ) - print_environ_usage() - def f(): - exec("testing print_exception() -- italics?") - def g(f=f): - f() - print("

What follows is a test, not an actual exception:

") - g() - except: - print_exception() - - print("

Second try with a small maxlen...

") - - global maxlen - maxlen = 50 - try: - form = FieldStorage() # Replace with other classes to test those - print_directory() - print_arguments() - print_form(form) - print_environ(environ) - except: - print_exception() - -def print_exception(type=None, value=None, tb=None, limit=None): - if type is None: - type, value, tb = sys.exc_info() - import traceback - print() - print("

Traceback (most recent call last):

") - list = traceback.format_tb(tb, limit) + \ - traceback.format_exception_only(type, value) - print("
%s%s
" % ( - html.escape("".join(list[:-1])), - html.escape(list[-1]), - )) - del tb - -def print_environ(environ=os.environ): - """Dump the shell environment as HTML.""" - keys = sorted(environ.keys()) - print() - print("

Shell Environment:

") - print("
") - for key in keys: - print("
", html.escape(key), "
", html.escape(environ[key])) - print("
") - print() - -def print_form(form): - """Dump the contents of a form as HTML.""" - keys = sorted(form.keys()) - print() - print("

Form Contents:

") - if not keys: - print("

No form fields.") - print("

") - for key in keys: - print("
" + html.escape(key) + ":", end=' ') - value = form[key] - print("" + html.escape(repr(type(value))) + "") - print("
" + html.escape(repr(value))) - print("
") - print() - -def print_directory(): - """Dump the current directory as HTML.""" - print() - print("

Current Working Directory:

") - try: - pwd = os.getcwd() - except OSError as msg: - print("OSError:", html.escape(str(msg))) - else: - print(html.escape(pwd)) - print() - -def print_arguments(): - print() - print("

Command Line Arguments:

") - print() - print(sys.argv) - print() - -def print_environ_usage(): - """Dump a list of environment variables used by CGI as HTML.""" - print(""" -

These environment variables could have been set:

-
    -
  • AUTH_TYPE -
  • CONTENT_LENGTH -
  • CONTENT_TYPE -
  • DATE_GMT -
  • DATE_LOCAL -
  • DOCUMENT_NAME -
  • DOCUMENT_ROOT -
  • DOCUMENT_URI -
  • GATEWAY_INTERFACE -
  • LAST_MODIFIED -
  • PATH -
  • PATH_INFO -
  • PATH_TRANSLATED -
  • QUERY_STRING -
  • REMOTE_ADDR -
  • REMOTE_HOST -
  • REMOTE_IDENT -
  • REMOTE_USER -
  • REQUEST_METHOD -
  • SCRIPT_NAME -
  • SERVER_NAME -
  • SERVER_PORT -
  • SERVER_PROTOCOL -
  • SERVER_ROOT -
  • SERVER_SOFTWARE -
-In addition, HTTP headers sent by the server may be passed in the -environment as well. Here are some common variable names: -
    -
  • HTTP_ACCEPT -
  • HTTP_CONNECTION -
  • HTTP_HOST -
  • HTTP_PRAGMA -
  • HTTP_REFERER -
  • HTTP_USER_AGENT -
-""") - - -# Utilities -# ========= - -def valid_boundary(s): - import re - if isinstance(s, bytes): - _vb_pattern = b"^[ -~]{0,200}[!-~]$" - else: - _vb_pattern = "^[ -~]{0,200}[!-~]$" - return re.match(_vb_pattern, s) - -# Invoke mainline -# =============== - -# Call test() when this file is run as a script (not imported as a module) -if __name__ == '__main__': - test() diff --git a/Lib/cgitb.py b/Lib/cgitb.py deleted file mode 100644 index 8ce0e833a98..00000000000 --- a/Lib/cgitb.py +++ /dev/null @@ -1,332 +0,0 @@ -"""More comprehensive traceback formatting for Python scripts. - -To enable this module, do: - - import cgitb; cgitb.enable() - -at the top of your script. The optional arguments to enable() are: - - display - if true, tracebacks are displayed in the web browser - logdir - if set, tracebacks are written to files in this directory - context - number of lines of source code to show for each stack frame - format - 'text' or 'html' controls the output format - -By default, tracebacks are displayed but not saved, the context is 5 lines -and the output format is 'html' (for backwards compatibility with the -original use of this module) - -Alternatively, if you have caught an exception and want cgitb to display it -for you, call cgitb.handler(). The optional argument to handler() is a -3-item tuple (etype, evalue, etb) just like the value of sys.exc_info(). -The default handler displays output as HTML. - -""" -import inspect -import keyword -import linecache -import os -import pydoc -import sys -import tempfile -import time -import tokenize -import traceback -import warnings -from html import escape as html_escape - -warnings._deprecated(__name__, remove=(3, 13)) - - -def reset(): - """Return a string that resets the CGI and browser to a known state.""" - return ''' - --> --> - - ''' - -__UNDEF__ = [] # a special sentinel object -def small(text): - if text: - return '' + text + '' - else: - return '' - -def strong(text): - if text: - return '' + text + '' - else: - return '' - -def grey(text): - if text: - return '' + text + '' - else: - return '' - -def lookup(name, frame, locals): - """Find the value for a given name in the given environment.""" - if name in locals: - return 'local', locals[name] - if name in frame.f_globals: - return 'global', frame.f_globals[name] - if '__builtins__' in frame.f_globals: - builtins = frame.f_globals['__builtins__'] - if type(builtins) is type({}): - if name in builtins: - return 'builtin', builtins[name] - else: - if hasattr(builtins, name): - return 'builtin', getattr(builtins, name) - return None, __UNDEF__ - -def scanvars(reader, frame, locals): - """Scan one logical line of Python and look up values of variables used.""" - vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__ - for ttype, token, start, end, line in tokenize.generate_tokens(reader): - if ttype == tokenize.NEWLINE: break - if ttype == tokenize.NAME and token not in keyword.kwlist: - if lasttoken == '.': - if parent is not __UNDEF__: - value = getattr(parent, token, __UNDEF__) - vars.append((prefix + token, prefix, value)) - else: - where, value = lookup(token, frame, locals) - vars.append((token, where, value)) - elif token == '.': - prefix += lasttoken + '.' - parent = value - else: - parent, prefix = None, '' - lasttoken = token - return vars - -def html(einfo, context=5): - """Return a nice HTML document describing a given traceback.""" - etype, evalue, etb = einfo - if isinstance(etype, type): - etype = etype.__name__ - pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable - date = time.ctime(time.time()) - head = f''' - - - - - -
 
- 
-{html_escape(str(etype))}		 -{pyver}
{date}
-

A problem occurred in a Python script. Here is the sequence of -function calls leading up to the error, in the order they occurred.

''' - - indent = '' + small(' ' * 5) + ' ' - frames = [] - records = inspect.getinnerframes(etb, context) - for frame, file, lnum, func, lines, index in records: - if file: - file = os.path.abspath(file) - link = '%s' % (file, pydoc.html.escape(file)) - else: - file = link = '?' - args, varargs, varkw, locals = inspect.getargvalues(frame) - call = '' - if func != '?': - call = 'in ' + strong(pydoc.html.escape(func)) - if func != "": - call += inspect.formatargvalues(args, varargs, varkw, locals, - formatvalue=lambda value: '=' + pydoc.html.repr(value)) - - highlight = {} - def reader(lnum=[lnum]): - highlight[lnum[0]] = 1 - try: return linecache.getline(file, lnum[0]) - finally: lnum[0] += 1 - vars = scanvars(reader, frame, locals) - - rows = ['%s%s %s' % - (' ', link, call)] - if index is not None: - i = lnum - index - for line in lines: - num = small(' ' * (5-len(str(i))) + str(i)) + ' ' - if i in highlight: - line = '=>%s%s' % (num, pydoc.html.preformat(line)) - rows.append('%s' % line) - else: - line = '  %s%s' % (num, pydoc.html.preformat(line)) - rows.append('%s' % grey(line)) - i += 1 - - done, dump = {}, [] - for name, where, value in vars: - if name in done: continue - done[name] = 1 - if value is not __UNDEF__: - if where in ('global', 'builtin'): - name = ('%s ' % where) + strong(name) - elif where == 'local': - name = strong(name) - else: - name = where + strong(name.split('.')[-1]) - dump.append('%s = %s' % (name, pydoc.html.repr(value))) - else: - dump.append(name + ' undefined') - - rows.append('%s' % small(grey(', '.join(dump)))) - frames.append(''' - -%s
''' % '\n'.join(rows)) - - exception = ['

%s: %s' % (strong(pydoc.html.escape(str(etype))), - pydoc.html.escape(str(evalue)))] - for name in dir(evalue): - if name[:1] == '_': continue - value = pydoc.html.repr(getattr(evalue, name)) - exception.append('\n
%s%s =\n%s' % (indent, name, value)) - - return head + ''.join(frames) + ''.join(exception) + ''' - - - -''' % pydoc.html.escape( - ''.join(traceback.format_exception(etype, evalue, etb))) - -def text(einfo, context=5): - """Return a plain text document describing a given traceback.""" - etype, evalue, etb = einfo - if isinstance(etype, type): - etype = etype.__name__ - pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable - date = time.ctime(time.time()) - head = "%s\n%s\n%s\n" % (str(etype), pyver, date) + ''' -A problem occurred in a Python script. Here is the sequence of -function calls leading up to the error, in the order they occurred. -''' - - frames = [] - records = inspect.getinnerframes(etb, context) - for frame, file, lnum, func, lines, index in records: - file = file and os.path.abspath(file) or '?' - args, varargs, varkw, locals = inspect.getargvalues(frame) - call = '' - if func != '?': - call = 'in ' + func - if func != "": - call += inspect.formatargvalues(args, varargs, varkw, locals, - formatvalue=lambda value: '=' + pydoc.text.repr(value)) - - highlight = {} - def reader(lnum=[lnum]): - highlight[lnum[0]] = 1 - try: return linecache.getline(file, lnum[0]) - finally: lnum[0] += 1 - vars = scanvars(reader, frame, locals) - - rows = [' %s %s' % (file, call)] - if index is not None: - i = lnum - index - for line in lines: - num = '%5d ' % i - rows.append(num+line.rstrip()) - i += 1 - - done, dump = {}, [] - for name, where, value in vars: - if name in done: continue - done[name] = 1 - if value is not __UNDEF__: - if where == 'global': name = 'global ' + name - elif where != 'local': name = where + name.split('.')[-1] - dump.append('%s = %s' % (name, pydoc.text.repr(value))) - else: - dump.append(name + ' undefined') - - rows.append('\n'.join(dump)) - frames.append('\n%s\n' % '\n'.join(rows)) - - exception = ['%s: %s' % (str(etype), str(evalue))] - for name in dir(evalue): - value = pydoc.text.repr(getattr(evalue, name)) - exception.append('\n%s%s = %s' % (" "*4, name, value)) - - return head + ''.join(frames) + ''.join(exception) + ''' - -The above is a description of an error in a Python program. Here is -the original traceback: - -%s -''' % ''.join(traceback.format_exception(etype, evalue, etb)) - -class Hook: - """A hook to replace sys.excepthook that shows tracebacks in HTML.""" - - def __init__(self, display=1, logdir=None, context=5, file=None, - format="html"): - self.display = display # send tracebacks to browser if true - self.logdir = logdir # log tracebacks to files if not None - self.context = context # number of source code lines per frame - self.file = file or sys.stdout # place to send the output - self.format = format - - def __call__(self, etype, evalue, etb): - self.handle((etype, evalue, etb)) - - def handle(self, info=None): - info = info or sys.exc_info() - if self.format == "html": - self.file.write(reset()) - - formatter = (self.format=="html") and html or text - plain = False - try: - doc = formatter(info, self.context) - except: # just in case something goes wrong - doc = ''.join(traceback.format_exception(*info)) - plain = True - - if self.display: - if plain: - doc = pydoc.html.escape(doc) - self.file.write('

' + doc + '
\n') - else: - self.file.write(doc + '\n') - else: - self.file.write('

A problem occurred in a Python script.\n') - - if self.logdir is not None: - suffix = ['.txt', '.html'][self.format=="html"] - (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir) - - try: - with os.fdopen(fd, 'w') as file: - file.write(doc) - msg = '%s contains the description of this error.' % path - except: - msg = 'Tried to save traceback to %s, but failed.' % path - - if self.format == 'html': - self.file.write('

%s

\n' % msg) - else: - self.file.write(msg + '\n') - try: - self.file.flush() - except: pass - -handler = Hook().handle -def enable(display=1, logdir=None, context=5, format="html"): - """Install an exception handler that formats tracebacks as HTML. - - The optional argument 'display' can be set to 0 to suppress sending the - traceback to the browser, and 'logdir' can be set to a directory to cause - tracebacks to be written to files there.""" - sys.excepthook = Hook(display=display, logdir=logdir, - context=context, format=format) diff --git a/Lib/chunk.py b/Lib/chunk.py deleted file mode 100644 index 618781efd11..00000000000 --- a/Lib/chunk.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Simple class to read IFF chunks. - -An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File -Format)) has the following structure: - -+----------------+ -| ID (4 bytes) | -+----------------+ -| size (4 bytes) | -+----------------+ -| data | -| ... | -+----------------+ - -The ID is a 4-byte string which identifies the type of chunk. - -The size field (a 32-bit value, encoded using big-endian byte order) -gives the size of the whole chunk, including the 8-byte header. - -Usually an IFF-type file consists of one or more chunks. The proposed -usage of the Chunk class defined here is to instantiate an instance at -the start of each chunk and read from the instance until it reaches -the end, after which a new instance can be instantiated. At the end -of the file, creating a new instance will fail with an EOFError -exception. - -Usage: -while True: - try: - chunk = Chunk(file) - except EOFError: - break - chunktype = chunk.getname() - while True: - data = chunk.read(nbytes) - if not data: - pass - # do something with data - -The interface is file-like. The implemented methods are: -read, close, seek, tell, isatty. -Extra methods are: skip() (called by close, skips to the end of the chunk), -getname() (returns the name (ID) of the chunk) - -The __init__ method has one required argument, a file-like object -(including a chunk instance), and one optional argument, a flag which -specifies whether or not chunks are aligned on 2-byte boundaries. The -default is 1, i.e. aligned. -""" - -import warnings - -warnings._deprecated(__name__, remove=(3, 13)) - -class Chunk: - def __init__(self, file, align=True, bigendian=True, inclheader=False): - import struct - self.closed = False - self.align = align # whether to align to word (2-byte) boundaries - if bigendian: - strflag = '>' - else: - strflag = '<' - self.file = file - self.chunkname = file.read(4) - if len(self.chunkname) < 4: - raise EOFError - try: - self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0] - except struct.error: - raise EOFError from None - if inclheader: - self.chunksize = self.chunksize - 8 # subtract header - self.size_read = 0 - try: - self.offset = self.file.tell() - except (AttributeError, OSError): - self.seekable = False - else: - self.seekable = True - - def getname(self): - """Return the name (ID) of the current chunk.""" - return self.chunkname - - def getsize(self): - """Return the size of the current chunk.""" - return self.chunksize - - def close(self): - if not self.closed: - try: - self.skip() - finally: - self.closed = True - - def isatty(self): - if self.closed: - raise ValueError("I/O operation on closed file") - return False - - def seek(self, pos, whence=0): - """Seek to specified position into the chunk. - Default position is 0 (start of chunk). - If the file is not seekable, this will result in an error. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if not self.seekable: - raise OSError("cannot seek") - if whence == 1: - pos = pos + self.size_read - elif whence == 2: - pos = pos + self.chunksize - if pos < 0 or pos > self.chunksize: - raise RuntimeError - self.file.seek(self.offset + pos, 0) - self.size_read = pos - - def tell(self): - if self.closed: - raise ValueError("I/O operation on closed file") - return self.size_read - - def read(self, size=-1): - """Read at most size bytes from the chunk. - If size is omitted or negative, read until the end - of the chunk. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if self.size_read >= self.chunksize: - return b'' - if size < 0: - size = self.chunksize - self.size_read - if size > self.chunksize - self.size_read: - size = self.chunksize - self.size_read - data = self.file.read(size) - self.size_read = self.size_read + len(data) - if self.size_read == self.chunksize and \ - self.align and \ - (self.chunksize & 1): - dummy = self.file.read(1) - self.size_read = self.size_read + len(dummy) - return data - - def skip(self): - """Skip the rest of the chunk. - If you are not interested in the contents of the chunk, - this method should be called so that the file points to - the start of the next chunk. - """ - - if self.closed: - raise ValueError("I/O operation on closed file") - if self.seekable: - try: - n = self.chunksize - self.size_read - # maybe fix alignment - if self.align and (self.chunksize & 1): - n = n + 1 - self.file.seek(n, 1) - self.size_read = self.size_read + n - return - except OSError: - pass - while self.size_read < self.chunksize: - n = min(8192, self.chunksize - self.size_read) - dummy = self.read(n) - if not dummy: - raise EOFError diff --git a/Lib/cmd.py b/Lib/cmd.py index 88ee7d3ddc4..51495fb3216 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -5,16 +5,16 @@ 1. End of file on input is processed as the command 'EOF'. 2. A command is parsed out of each line by collecting the prefix composed of characters in the identchars member. -3. A command `foo' is dispatched to a method 'do_foo()'; the do_ method +3. A command 'foo' is dispatched to a method 'do_foo()'; the do_ method is passed a single argument consisting of the remainder of the line. 4. Typing an empty line repeats the last command. (Actually, it calls the - method `emptyline', which may be overridden in a subclass.) -5. There is a predefined `help' method. Given an argument `topic', it - calls the command `help_topic'. With no arguments, it lists all topics + method 'emptyline', which may be overridden in a subclass.) +5. There is a predefined 'help' method. Given an argument 'topic', it + calls the command 'help_topic'. With no arguments, it lists all topics with defined help_ functions, broken into up to three topics; documented commands, miscellaneous help topics, and undocumented commands. -6. The command '?' is a synonym for `help'. The command '!' is a synonym - for `shell', if a do_shell method exists. +6. The command '?' is a synonym for 'help'. The command '!' is a synonym + for 'shell', if a do_shell method exists. 7. If completion is enabled, completing commands will be done automatically, and completing of commands args is done by calling complete_foo() with arguments text, line, begidx, endidx. text is string we are matching @@ -23,31 +23,34 @@ indexes of the text being matched, which could be used to provide different completion depending upon which position the argument is in. -The `default' method may be overridden to intercept commands for which there +The 'default' method may be overridden to intercept commands for which there is no do_ method. -The `completedefault' method may be overridden to intercept completions for +The 'completedefault' method may be overridden to intercept completions for commands that have no complete_ method. -The data member `self.ruler' sets the character used to draw separator lines +The data member 'self.ruler' sets the character used to draw separator lines in the help messages. If empty, no ruler line is drawn. It defaults to "=". -If the value of `self.intro' is nonempty when the cmdloop method is called, +If the value of 'self.intro' is nonempty when the cmdloop method is called, it is printed out on interpreter startup. This value may be overridden via an optional argument to the cmdloop() method. -The data members `self.doc_header', `self.misc_header', and -`self.undoc_header' set the headers used for the help function's +The data members 'self.doc_header', 'self.misc_header', and +'self.undoc_header' set the headers used for the help function's listings of documented functions, miscellaneous topics, and undocumented functions respectively. """ -import string, sys +import sys __all__ = ["Cmd"] PROMPT = '(Cmd) ' -IDENTCHARS = string.ascii_letters + string.digits + '_' +IDENTCHARS = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' + 'abcdefghijklmnopqrstuvwxyz' + '0123456789' + '_') class Cmd: """A simple framework for writing line-oriented command interpreters. @@ -108,7 +111,15 @@ def cmdloop(self, intro=None): import readline self.old_completer = readline.get_completer() readline.set_completer(self.complete) - readline.parse_and_bind(self.completekey+": complete") + if readline.backend == "editline": + if self.completekey == 'tab': + # libedit uses "^I" instead of "tab" + command_string = "bind ^I rl_complete" + else: + command_string = f"bind {self.completekey} rl_complete" + else: + command_string = f"{self.completekey}: complete" + readline.parse_and_bind(command_string) except ImportError: pass try: @@ -210,9 +221,8 @@ def onecmd(self, line): if cmd == '': return self.default(line) else: - try: - func = getattr(self, 'do_' + cmd) - except AttributeError: + func = getattr(self, 'do_' + cmd, None) + if func is None: return self.default(line) return func(arg) @@ -263,7 +273,7 @@ def complete(self, text, state): endidx = readline.get_endidx() - stripped if begidx>0: cmd, args, foo = self.parseline(line) - if cmd == '': + if not cmd: compfunc = self.completedefault else: try: @@ -296,8 +306,11 @@ def do_help(self, arg): try: func = getattr(self, 'help_' + arg) except AttributeError: + from inspect import cleandoc + try: doc=getattr(self, 'do_' + arg).__doc__ + doc = cleandoc(doc) if doc: self.stdout.write("%s\n"%str(doc)) return diff --git a/Lib/code.py b/Lib/code.py index 76000f8c8b2..b134886dc26 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -5,6 +5,7 @@ # Inspired by similar code by Jeff Epler and Fredrik Lundh. +import builtins import sys import traceback from codeop import CommandCompiler, compile_command @@ -24,10 +25,10 @@ class InteractiveInterpreter: def __init__(self, locals=None): """Constructor. - The optional 'locals' argument specifies the dictionary in - which code will be executed; it defaults to a newly created - dictionary with key "__name__" set to "__console__" and key - "__doc__" set to None. + The optional 'locals' argument specifies a mapping to use as the + namespace in which code will be executed; it defaults to a newly + created dictionary with key "__name__" set to "__console__" and + key "__doc__" set to None. """ if locals is None: @@ -63,7 +64,7 @@ def runsource(self, source, filename="", symbol="single"): code = self.compile(source, filename, symbol) except (OverflowError, SyntaxError, ValueError): # Case 1 - self.showsyntaxerror(filename) + self.showsyntaxerror(filename, source=source) return False if code is None: @@ -93,7 +94,7 @@ def runcode(self, code): except: self.showtraceback() - def showsyntaxerror(self, filename=None): + def showsyntaxerror(self, filename=None, **kwargs): """Display the syntax error that just occurred. This doesn't display a stack trace because there isn't one. @@ -105,28 +106,14 @@ def showsyntaxerror(self, filename=None): The output is written by self.write(), below. """ - type, value, tb = sys.exc_info() - sys.last_type = type - sys.last_value = value - sys.last_traceback = tb - if filename and type is SyntaxError: - # Work hard to stuff the correct filename in the exception - try: - msg, (dummy_filename, lineno, offset, line) = value.args - except ValueError: - # Not the format we expect; leave it alone - pass - else: - # Stuff in the right filename - value = SyntaxError(msg, (filename, lineno, offset, line)) - sys.last_value = value - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(type, value, tb) + try: + typ, value, tb = sys.exc_info() + if filename and issubclass(typ, SyntaxError): + value.filename = filename + source = kwargs.pop('source', "") + self._showtraceback(typ, value, None, source) + finally: + typ = value = tb = None def showtraceback(self): """Display the exception that just occurred. @@ -136,18 +123,46 @@ def showtraceback(self): The output is written by self.write(), below. """ - sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() - sys.last_traceback = last_tb try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) - if sys.excepthook is sys.__excepthook__: - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(ei[0], ei[1], last_tb) + typ, value, tb = sys.exc_info() + self._showtraceback(typ, value, tb.tb_next, "") finally: - last_tb = ei = None + typ = value = tb = None + + def _showtraceback(self, typ, value, tb, source): + sys.last_type = typ + sys.last_traceback = tb + value = value.with_traceback(tb) + # Set the line of text that the exception refers to + lines = source.splitlines() + if (source and typ is SyntaxError + and not value.text and value.lineno is not None + and len(lines) >= value.lineno): + value.text = lines[value.lineno - 1] + sys.last_exc = sys.last_value = value + if sys.excepthook is sys.__excepthook__: + self._excepthook(typ, value, tb) + else: + # If someone has set sys.excepthook, we let that take precedence + # over self.write + try: + sys.excepthook(typ, value, tb) + except SystemExit: + raise + except BaseException as e: + e.__context__ = None + e = e.with_traceback(e.__traceback__.tb_next) + print('Error in sys.excepthook:', file=sys.stderr) + sys.__excepthook__(type(e), e, e.__traceback__) + print(file=sys.stderr) + print('Original exception was:', file=sys.stderr) + sys.__excepthook__(typ, value, tb) + + def _excepthook(self, typ, value, tb): + # This method is being overwritten in + # _pyrepl.console.InteractiveColoredConsole + lines = traceback.format_exception(typ, value, tb) + self.write(''.join(lines)) def write(self, data): """Write a string. @@ -167,7 +182,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename=""): + def __init__(self, locals=None, filename="", *, local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -179,6 +194,7 @@ def __init__(self, locals=None, filename=""): """ InteractiveInterpreter.__init__(self, locals) self.filename = filename + self.local_exit = local_exit self.resetbuffer() def resetbuffer(self): @@ -203,12 +219,17 @@ def interact(self, banner=None, exitmsg=None): """ try: sys.ps1 + delete_ps1_after = False except AttributeError: sys.ps1 = ">>> " + delete_ps1_after = True try: - sys.ps2 + _ps2 = sys.ps2 + delete_ps2_after = False except AttributeError: sys.ps2 = "... " + delete_ps2_after = True + cprt = 'Type "help", "copyright", "credits" or "license" for more information.' if banner is None: self.write("Python %s on %s\n%s\n(%s)\n" % @@ -217,29 +238,72 @@ def interact(self, banner=None, exitmsg=None): elif banner: self.write("%s\n" % str(banner)) more = 0 - while 1: - try: - if more: - prompt = sys.ps2 - else: - prompt = sys.ps1 + + # When the user uses exit() or quit() in their interactive shell + # they probably just want to exit the created shell, not the whole + # process. exit and quit in builtins closes sys.stdin which makes + # it super difficult to restore + # + # When self.local_exit is True, we overwrite the builtins so + # exit() and quit() only raises SystemExit and we can catch that + # to only exit the interactive shell + + _exit = None + _quit = None + + if self.local_exit: + if hasattr(builtins, "exit"): + _exit = builtins.exit + builtins.exit = Quitter("exit") + + if hasattr(builtins, "quit"): + _quit = builtins.quit + builtins.quit = Quitter("quit") + + try: + while True: try: - line = self.raw_input(prompt) - except EOFError: - self.write("\n") - break - else: - more = self.push(line) - except KeyboardInterrupt: - self.write("\nKeyboardInterrupt\n") - self.resetbuffer() - more = 0 - if exitmsg is None: - self.write('now exiting %s...\n' % self.__class__.__name__) - elif exitmsg != '': - self.write('%s\n' % exitmsg) - - def push(self, line): + if more: + prompt = sys.ps2 + else: + prompt = sys.ps1 + try: + line = self.raw_input(prompt) + except EOFError: + self.write("\n") + break + else: + more = self.push(line) + except KeyboardInterrupt: + self.write("\nKeyboardInterrupt\n") + self.resetbuffer() + more = 0 + except SystemExit as e: + if self.local_exit: + self.write("\n") + break + else: + raise e + finally: + # restore exit and quit in builtins if they were modified + if _exit is not None: + builtins.exit = _exit + + if _quit is not None: + builtins.quit = _quit + + if delete_ps1_after: + del sys.ps1 + + if delete_ps2_after: + del sys.ps2 + + if exitmsg is None: + self.write('now exiting %s...\n' % self.__class__.__name__) + elif exitmsg != '': + self.write('%s\n' % exitmsg) + + def push(self, line, filename=None, _symbol="single"): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -255,7 +319,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename, symbol=_symbol) if not more: self.resetbuffer() return more @@ -274,8 +340,22 @@ def raw_input(self, prompt=""): return input(prompt) +class Quitter: + def __init__(self, name): + self.name = name + if sys.platform == "win32": + self.eof = 'Ctrl-Z plus Return' + else: + self.eof = 'Ctrl-D (i.e. EOF)' + + def __repr__(self): + return f'Use {self.name} or {self.eof} to exit' + + def __call__(self, code=None): + raise SystemExit(code) + -def interact(banner=None, readfunc=None, local=None, exitmsg=None): +def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False): """Closely emulate the interactive Python interpreter. This is a backwards compatible interface to the InteractiveConsole @@ -288,14 +368,15 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): readfunc -- if not None, replaces InteractiveConsole.raw_input() local -- passed to InteractiveInterpreter.__init__() exitmsg -- passed to InteractiveConsole.interact() + local_exit -- passed to InteractiveConsole.__init__() """ - console = InteractiveConsole(local) + console = InteractiveConsole(local, local_exit=local_exit) if readfunc is not None: console.raw_input = readfunc else: try: - import readline + import readline # noqa: F401 except ImportError: pass console.interact(banner, exitmsg) @@ -304,7 +385,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument('-q', action='store_true', help="don't print version and copyright messages") args = parser.parse_args() diff --git a/Lib/codecs.py b/Lib/codecs.py index e6ad6e3a052..e4a8010aba9 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -111,6 +111,9 @@ def __repr__(self): (self.__class__.__module__, self.__class__.__qualname__, self.name, id(self)) + def __getnewargs__(self): + return tuple(self) + class Codec: """ Defines the interface for stateless encoders/decoders. @@ -414,6 +417,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamReader(Codec): @@ -612,7 +618,7 @@ def readlines(self, sizehint=None, keepends=True): method and are included in the list entries. sizehint, if given, is ignored since there is no efficient - way to finding the true end-of-line. + way of finding the true end-of-line. """ data = self.read() @@ -663,6 +669,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamReaderWriter: @@ -700,13 +709,13 @@ def read(self, size=-1): return self.reader.read(size) - def readline(self, size=None): + def readline(self, size=None, keepends=True): - return self.reader.readline(size) + return self.reader.readline(size, keepends) - def readlines(self, sizehint=None): + def readlines(self, sizehint=None, keepends=True): - return self.reader.readlines(sizehint) + return self.reader.readlines(sizehint, keepends) def __next__(self): @@ -750,6 +759,9 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### class StreamRecoder: @@ -866,10 +878,12 @@ def __enter__(self): def __exit__(self, type, value, tb): self.stream.close() + def __reduce_ex__(self, proto): + raise TypeError("can't serialize %s" % self.__class__.__name__) + ### Shortcuts def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): - """ Open an encoded file using the given mode and return a wrapped version providing transparent encoding/decoding. @@ -878,7 +892,8 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): codecs. Output is also codec dependent and will usually be Unicode as well. - Underlying encoded files are always opened in binary mode. + If encoding is not None, then the + underlying encoded files are always opened in binary mode. The default file mode is 'r', meaning to open the file in read mode. encoding specifies the encoding which is to be used for the @@ -896,8 +911,11 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1): .encoding which allows querying the used encoding. This attribute is only available if an encoding was specified as parameter. - """ + import warnings + warnings.warn("codecs.open() is deprecated. Use open() instead.", + DeprecationWarning, stacklevel=2) + if encoding is not None and \ 'b' not in mode: # Force opening of the file in binary mode @@ -1093,34 +1111,15 @@ def make_encoding_map(decoding_map): ### error handlers -try: - strict_errors = lookup_error("strict") - ignore_errors = lookup_error("ignore") - replace_errors = lookup_error("replace") - xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") - backslashreplace_errors = lookup_error("backslashreplace") - namereplace_errors = lookup_error("namereplace") -except LookupError: - # In --disable-unicode builds, these error handler are missing - strict_errors = None - ignore_errors = None - replace_errors = None - xmlcharrefreplace_errors = None - backslashreplace_errors = None - namereplace_errors = None +strict_errors = lookup_error("strict") +ignore_errors = lookup_error("ignore") +replace_errors = lookup_error("replace") +xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") +backslashreplace_errors = lookup_error("backslashreplace") +namereplace_errors = lookup_error("namereplace") # Tell modulefinder that using codecs probably needs the encodings # package _false = 0 if _false: - import encodings - -### Tests - -if __name__ == '__main__': - - # Make stdout translate Latin-1 output into UTF-8 output - sys.stdout = EncodedFile(sys.stdout, 'latin-1', 'utf-8') - - # Have stdin translate Latin-1 input into UTF-8 input - sys.stdin = EncodedFile(sys.stdin, 'utf-8', 'latin-1') + import encodings # noqa: F401 diff --git a/Lib/codeop.py b/Lib/codeop.py index 4dd096574bb..8cac00442d9 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -44,9 +44,10 @@ # Caveat emptor: These flags are undocumented on purpose and depending # on their effect outside the standard library is **unsupported**. PyCF_DONT_IMPLY_DEDENT = 0x200 +PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 -def _maybe_compile(compiler, source, filename, symbol): +def _maybe_compile(compiler, source, filename, symbol, flags): # Check for source consisting of only blank lines and comments. for line in source.split("\n"): line = line.strip() @@ -60,36 +61,26 @@ def _maybe_compile(compiler, source, filename, symbol): with warnings.catch_warnings(): warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning)) try: - compiler(source, filename, symbol) + compiler(source, filename, symbol, flags=flags) except SyntaxError: # Let other compile() errors propagate. try: - compiler(source + "\n", filename, symbol) + compiler(source + "\n", filename, symbol, flags=flags) + return None + except _IncompleteInputError as e: return None except SyntaxError as e: - if "incomplete input" in str(e): - return None + pass # fallthrough return compiler(source, filename, symbol, incomplete_input=False) -def _is_syntax_error(err1, err2): - rep1 = repr(err1) - rep2 = repr(err2) - if "was never closed" in rep1 and "was never closed" in rep2: - return False - if rep1 == rep2: - return True - return False - -def _compile(source, filename, symbol, incomplete_input=True): - flags = 0 +def _compile(source, filename, symbol, incomplete_input=True, *, flags=0): if incomplete_input: flags |= PyCF_ALLOW_INCOMPLETE_INPUT flags |= PyCF_DONT_IMPLY_DEDENT return compile(source, filename, symbol, flags) - -def compile_command(source, filename="", symbol="single"): +def compile_command(source, filename="", symbol="single", flags=0): r"""Compile a command and determine whether it is incomplete. Arguments: @@ -108,7 +99,7 @@ def compile_command(source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(_compile, source, filename, symbol) + return _maybe_compile(_compile, source, filename, symbol, flags) class Compile: """Instances of this class behave much like the built-in compile @@ -118,12 +109,14 @@ class Compile: def __init__(self): self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT - def __call__(self, source, filename, symbol, **kwargs): - flags = self.flags + def __call__(self, source, filename, symbol, flags=0, **kwargs): + flags |= self.flags if kwargs.get('incomplete_input', True) is False: flags &= ~PyCF_DONT_IMPLY_DEDENT flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT codeob = compile(source, filename, symbol, flags, True) + if flags & PyCF_ONLY_AST: + return codeob # this is an ast.Module in this case for feature in _features: if codeob.co_flags & feature.compiler_flag: self.flags |= feature.compiler_flag @@ -158,4 +151,4 @@ def __call__(self, source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(self.compiler, source, filename, symbol) + return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 59a2d520fea..3d3bbd7a39a 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -29,6 +29,9 @@ import _collections_abc import sys as _sys +_sys.modules['collections.abc'] = _collections_abc +abc = _collections_abc + from itertools import chain as _chain from itertools import repeat as _repeat from itertools import starmap as _starmap @@ -45,15 +48,20 @@ else: _collections_abc.MutableSequence.register(deque) +try: + # Expose _deque_iterator to support pickling deque iterators + from _collections import _deque_iterator # noqa: F401 +except ImportError: + pass + try: from _collections import defaultdict except ImportError: - # FIXME: try to implement defaultdict in collections.rs rather than in Python - # I (coolreader18) couldn't figure out some class stuff with __new__ and - # __init__ and __missing__ and subclassing built-in types from Rust, so I went - # with this instead. + # TODO: RUSTPYTHON - implement defaultdict in Rust from ._defaultdict import defaultdict +heapq = None # Lazily imported + ################################################################################ ### OrderedDict @@ -94,17 +102,19 @@ class OrderedDict(dict): # Individual links are kept alive by the hard reference in self.__map. # Those hard references disappear when a key is deleted from an OrderedDict. + def __new__(cls, /, *args, **kwds): + "Create the ordered dict object and set up the underlying structures." + self = dict.__new__(cls) + self.__hardroot = _Link() + self.__root = root = _proxy(self.__hardroot) + root.prev = root.next = root + self.__map = {} + return self + def __init__(self, other=(), /, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries. Keyword argument order is preserved. ''' - try: - self.__root - except AttributeError: - self.__hardroot = _Link() - self.__root = root = _proxy(self.__hardroot) - root.prev = root.next = root - self.__map = {} self.__update(other, **kwds) def __setitem__(self, key, value, @@ -271,7 +281,7 @@ def __repr__(self): 'od.__repr__() <==> repr(od)' if not self: return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, list(self.items())) + return '%s(%r)' % (self.__class__.__name__, dict(self.items())) def __reduce__(self): 'Return state information for pickling' @@ -454,7 +464,7 @@ def _make(cls, iterable): def _replace(self, /, **kwds): result = self._make(_map(kwds.pop, field_names, self)) if kwds: - raise ValueError(f'Got unexpected field names: {list(kwds)!r}') + raise TypeError(f'Got unexpected field names: {list(kwds)!r}') return result _replace.__doc__ = (f'Return a new {typename} object replacing specified ' @@ -492,6 +502,7 @@ def __getnewargs__(self): '_field_defaults': field_defaults, '__new__': __new__, '_make': _make, + '__replace__': _replace, '_replace': _replace, '__repr__': __repr__, '_asdict': _asdict, @@ -511,9 +522,12 @@ def __getnewargs__(self): # specified a particular module. if module is None: try: - module = _sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass + module = _sys._getframemodulename(1) or '__main__' + except AttributeError: + try: + module = _sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass if module is not None: result.__module__ = module @@ -582,7 +596,7 @@ class Counter(dict): # References: # http://en.wikipedia.org/wiki/Multiset # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html - # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm + # http://www.java2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm # http://code.activestate.com/recipes/259174/ # Knuth, TAOCP Vol. II section 4.6.3 @@ -622,7 +636,10 @@ def most_common(self, n=None): return sorted(self.items(), key=_itemgetter(1), reverse=True) # Lazy import to speedup Python startup time - import heapq + global heapq + if heapq is None: + import heapq + return heapq.nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): @@ -632,7 +649,8 @@ def elements(self): >>> sorted(c.elements()) ['A', 'A', 'B', 'B', 'C', 'C'] - # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + >>> import math >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) >>> math.prod(prime_factors.elements()) @@ -673,7 +691,7 @@ def update(self, iterable=None, /, **kwds): ''' # The regular dict.update() operation makes no sense here because the - # replace behavior results in the some of original untouched counts + # replace behavior results in some of the original untouched counts # being mixed-in with all of the other counts for a mismash that # doesn't have a straight-forward interpretation in most counting # contexts. Instead, we implement straight-addition. Both the inputs @@ -1008,19 +1026,22 @@ def __getitem__(self, key): return self.__missing__(key) # support subclasses that define __missing__ def get(self, key, default=None): - return self[key] if key in self else default + return self[key] if key in self else default # needs to make use of __contains__ def __len__(self): return len(set().union(*self.maps)) # reuses stored hash values if possible def __iter__(self): d = {} - for mapping in reversed(self.maps): - d.update(dict.fromkeys(mapping)) # reuses stored hash values if possible + for mapping in map(dict.fromkeys, reversed(self.maps)): + d |= mapping # reuses stored hash values if possible return iter(d) def __contains__(self, key): - return any(key in m for m in self.maps) + for mapping in self.maps: + if key in mapping: + return True + return False def __bool__(self): return any(self.maps) @@ -1030,9 +1051,9 @@ def __repr__(self): return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})' @classmethod - def fromkeys(cls, iterable, *args): - 'Create a ChainMap with a single dict created from the iterable.' - return cls(dict.fromkeys(iterable, *args)) + def fromkeys(cls, iterable, value=None, /): + 'Create a new ChainMap with keys from iterable and values set to value.' + return cls(dict.fromkeys(iterable, value)) def copy(self): 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' @@ -1136,10 +1157,17 @@ def __delitem__(self, key): def __iter__(self): return iter(self.data) - # Modify __contains__ to work correctly when __missing__ is present + # Modify __contains__ and get() to work like dict + # does when __missing__ is present. def __contains__(self, key): return key in self.data + def get(self, key, default=None): + if key in self: + return self[key] + return default + + # Now, add the methods in dicts but not in MutableMapping def __repr__(self): return repr(self.data) @@ -1468,6 +1496,8 @@ def format_map(self, mapping): return self.data.format_map(mapping) def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.index(sub, start, end) def isalpha(self): @@ -1536,6 +1566,8 @@ def rfind(self, sub, start=0, end=_sys.maxsize): return self.data.rfind(sub, start, end) def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.rindex(sub, start, end) def rjust(self, width, *args): diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py deleted file mode 100644 index 86ca8b8a841..00000000000 --- a/Lib/collections/abc.py +++ /dev/null @@ -1,3 +0,0 @@ -from _collections_abc import * -from _collections_abc import __all__ -from _collections_abc import _CallableGenericAlias diff --git a/Lib/colorsys.py b/Lib/colorsys.py index 9bdc83e3772..e97f91718a3 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -24,7 +24,7 @@ __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", "rgb_to_hsv","hsv_to_rgb"] -# Some floating point constants +# Some floating-point constants ONE_THIRD = 1.0/3.0 ONE_SIXTH = 1.0/6.0 @@ -83,7 +83,7 @@ def rgb_to_hls(r, g, b): if l <= 0.5: s = rangec / sumc else: - s = rangec / (2.0-sumc) + s = rangec / (2.0-maxc-minc) # Not always 2.0-sumc: gh-106498. rc = (maxc-r) / rangec gc = (maxc-g) / rangec bc = (maxc-b) / rangec diff --git a/Lib/compileall.py b/Lib/compileall.py index a388931fb5a..67fe370451e 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -97,9 +97,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True if workers != 1 and ProcessPoolExecutor is not None: + import multiprocessing + if multiprocessing.get_start_method() == 'fork': + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None # If workers == 0, let ProcessPoolExecutor choose workers = workers or None - with ProcessPoolExecutor(max_workers=workers) as executor: + with ProcessPoolExecutor(max_workers=workers, + mp_context=mp_context) as executor: results = executor.map(partial(compile_file, ddir=ddir, force=force, rx=rx, quiet=quiet, @@ -110,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, prependdir=prependdir, limit_sl_dest=limit_sl_dest, hardlink_dupes=hardlink_dupes), - files) + files, + chunksize=4) success = min(results, default=True) else: for file in files: @@ -166,13 +173,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) - ddir_parts = list(fullname_parts) - - for spart, opart in zip(stripdir_parts, fullname_parts): - if spart == opart: - ddir_parts.remove(spart) - dfile = os.path.join(*ddir_parts) + if stripdir_parts != fullname_parts[:len(stripdir_parts)]: + if quiet < 2: + print("The stripdir path {!r} is not a valid prefix for " + "source path {!r}; ignoring".format(stripdir, fullname)) + else: + dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) if prependdir is not None: if dfile is None: @@ -310,7 +317,9 @@ def main(): import argparse parser = argparse.ArgumentParser( - description='Utilities to support installing Python libraries.') + description='Utilities to support installing Python libraries.', + color=True, + ) parser.add_argument('-l', action='store_const', const=0, default=None, dest='maxlevels', help="don't recurse into subdirectories") diff --git a/wasm/demo/src/browser_module.rs b/Lib/compression/__init__.py similarity index 100% rename from wasm/demo/src/browser_module.rs rename to Lib/compression/__init__.py diff --git a/Lib/compression/_common/__init__.py b/Lib/compression/_common/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/compression/_common/_streams.py b/Lib/compression/_common/_streams.py new file mode 100644 index 00000000000..9f367d4e304 --- /dev/null +++ b/Lib/compression/_common/_streams.py @@ -0,0 +1,162 @@ +"""Internal classes used by compression modules""" + +import io +import sys + +BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size + + +class BaseStream(io.BufferedIOBase): + """Mode-checking helper functions.""" + + def _check_not_closed(self): + if self.closed: + raise ValueError("I/O operation on closed file") + + def _check_can_read(self): + if not self.readable(): + raise io.UnsupportedOperation("File not open for reading") + + def _check_can_write(self): + if not self.writable(): + raise io.UnsupportedOperation("File not open for writing") + + def _check_can_seek(self): + if not self.readable(): + raise io.UnsupportedOperation("Seeking is only supported " + "on files open for reading") + if not self.seekable(): + raise io.UnsupportedOperation("The underlying file object " + "does not support seeking") + + +class DecompressReader(io.RawIOBase): + """Adapts the decompressor API to a RawIOBase reader API""" + + def readable(self): + return True + + def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args): + self._fp = fp + self._eof = False + self._pos = 0 # Current offset in decompressed stream + + # Set to size of decompressed stream once it is known, for SEEK_END + self._size = -1 + + # Save the decompressor factory and arguments. + # If the file contains multiple compressed streams, each + # stream will need a separate decompressor object. A new decompressor + # object is also needed when implementing a backwards seek(). + self._decomp_factory = decomp_factory + self._decomp_args = decomp_args + self._decompressor = self._decomp_factory(**self._decomp_args) + + # Exception class to catch from decompressor signifying invalid + # trailing data to ignore + self._trailing_error = trailing_error + + def close(self): + self._decompressor = None + return super().close() + + def seekable(self): + return self._fp.seekable() + + def readinto(self, b): + with memoryview(b) as view, view.cast("B") as byte_view: + data = self.read(len(byte_view)) + byte_view[:len(data)] = data + return len(data) + + def read(self, size=-1): + if size < 0: + return self.readall() + + if not size or self._eof: + return b"" + data = None # Default if EOF is encountered + # Depending on the input data, our call to the decompressor may not + # return any data. In this case, try again after reading another block. + while True: + if self._decompressor.eof: + rawblock = (self._decompressor.unused_data or + self._fp.read(BUFFER_SIZE)) + if not rawblock: + break + # Continue to next stream. + self._decompressor = self._decomp_factory( + **self._decomp_args) + try: + data = self._decompressor.decompress(rawblock, size) + except self._trailing_error: + # Trailing data isn't a valid compressed stream; ignore it. + break + else: + if self._decompressor.needs_input: + rawblock = self._fp.read(BUFFER_SIZE) + if not rawblock: + raise EOFError("Compressed file ended before the " + "end-of-stream marker was reached") + else: + rawblock = b"" + data = self._decompressor.decompress(rawblock, size) + if data: + break + if not data: + self._eof = True + self._size = self._pos + return b"" + self._pos += len(data) + return data + + def readall(self): + chunks = [] + # sys.maxsize means the max length of output buffer is unlimited, + # so that the whole input buffer can be decompressed within one + # .decompress() call. + while data := self.read(sys.maxsize): + chunks.append(data) + + return b"".join(chunks) + + # Rewind the file to the beginning of the data stream. + def _rewind(self): + self._fp.seek(0) + self._eof = False + self._pos = 0 + self._decompressor = self._decomp_factory(**self._decomp_args) + + def seek(self, offset, whence=io.SEEK_SET): + # Recalculate offset as an absolute file position. + if whence == io.SEEK_SET: + pass + elif whence == io.SEEK_CUR: + offset = self._pos + offset + elif whence == io.SEEK_END: + # Seeking relative to EOF - we need to know the file's size. + if self._size < 0: + while self.read(io.DEFAULT_BUFFER_SIZE): + pass + offset = self._size + offset + else: + raise ValueError("Invalid value for whence: {}".format(whence)) + + # Make it so that offset is the number of bytes to skip forward. + if offset < self._pos: + self._rewind() + else: + offset -= self._pos + + # Read and discard data until we reach the desired position. + while offset > 0: + data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset)) + if not data: + break + offset -= len(data) + + return self._pos + + def tell(self): + """Return the current file position.""" + return self._pos diff --git a/Lib/compression/bz2.py b/Lib/compression/bz2.py new file mode 100644 index 00000000000..16815d6cd20 --- /dev/null +++ b/Lib/compression/bz2.py @@ -0,0 +1,5 @@ +import bz2 +__doc__ = bz2.__doc__ +del bz2 + +from bz2 import * diff --git a/Lib/compression/gzip.py b/Lib/compression/gzip.py new file mode 100644 index 00000000000..552f48f948a --- /dev/null +++ b/Lib/compression/gzip.py @@ -0,0 +1,5 @@ +import gzip +__doc__ = gzip.__doc__ +del gzip + +from gzip import * diff --git a/Lib/compression/lzma.py b/Lib/compression/lzma.py new file mode 100644 index 00000000000..b4bc7ccb1db --- /dev/null +++ b/Lib/compression/lzma.py @@ -0,0 +1,5 @@ +import lzma +__doc__ = lzma.__doc__ +del lzma + +from lzma import * diff --git a/Lib/compression/zlib.py b/Lib/compression/zlib.py new file mode 100644 index 00000000000..3aa7e2db90e --- /dev/null +++ b/Lib/compression/zlib.py @@ -0,0 +1,5 @@ +import zlib +__doc__ = zlib.__doc__ +del zlib + +from zlib import * diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py new file mode 100644 index 00000000000..84b25914b0a --- /dev/null +++ b/Lib/compression/zstd/__init__.py @@ -0,0 +1,242 @@ +"""Python bindings to the Zstandard (zstd) compression library (RFC-8878).""" + +__all__ = ( + # compression.zstd + 'COMPRESSION_LEVEL_DEFAULT', + 'compress', + 'CompressionParameter', + 'decompress', + 'DecompressionParameter', + 'finalize_dict', + 'get_frame_info', + 'Strategy', + 'train_dict', + + # compression.zstd._zstdfile + 'open', + 'ZstdFile', + + # _zstd + 'get_frame_size', + 'zstd_version', + 'zstd_version_info', + 'ZstdCompressor', + 'ZstdDecompressor', + 'ZstdDict', + 'ZstdError', +) + +import _zstd +import enum +from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError, + get_frame_size, zstd_version) +from compression.zstd._zstdfile import ZstdFile, open, _nbytes + +# zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE) +zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100), + _zstd.zstd_version_number % 100) +"""Version number of the runtime zstd library as a tuple of integers.""" + +COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT +"""The default compression level for Zstandard, currently '3'.""" + + +class FrameInfo: + """Information about a Zstandard frame.""" + + __slots__ = 'decompressed_size', 'dictionary_id' + + def __init__(self, decompressed_size, dictionary_id): + super().__setattr__('decompressed_size', decompressed_size) + super().__setattr__('dictionary_id', dictionary_id) + + def __repr__(self): + return (f'FrameInfo(decompressed_size={self.decompressed_size}, ' + f'dictionary_id={self.dictionary_id})') + + def __setattr__(self, name, _): + raise AttributeError(f"can't set attribute {name!r}") + + +def get_frame_info(frame_buffer): + """Get Zstandard frame information from a frame header. + + *frame_buffer* is a bytes-like object. It should start from the beginning + of a frame, and needs to include at least the frame header (6 to 18 bytes). + + The returned FrameInfo object has two attributes. + 'decompressed_size' is the size in bytes of the data in the frame when + decompressed, or None when the decompressed size is unknown. + 'dictionary_id' is an int in the range (0, 2**32). The special value 0 + means that the dictionary ID was not recorded in the frame header, + the frame may or may not need a dictionary to be decoded, + and the ID of such a dictionary is not specified. + """ + return FrameInfo(*_zstd.get_frame_info(frame_buffer)) + + +def train_dict(samples, dict_size): + """Return a ZstdDict representing a trained Zstandard dictionary. + + *samples* is an iterable of samples, where a sample is a bytes-like + object representing a file. + + *dict_size* is the dictionary's maximum size, in bytes. + """ + if not isinstance(dict_size, int): + ds_cls = type(dict_size).__qualname__ + raise TypeError(f'dict_size must be an int object, not {ds_cls!r}.') + + samples = tuple(samples) + chunks = b''.join(samples) + chunk_sizes = tuple(_nbytes(sample) for sample in samples) + if not chunks: + raise ValueError("samples contained no data; can't train dictionary.") + dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size) + return ZstdDict(dict_content) + + +def finalize_dict(zstd_dict, /, samples, dict_size, level): + """Return a ZstdDict representing a finalized Zstandard dictionary. + + Given a custom content as a basis for dictionary, and a set of samples, + finalize *zstd_dict* by adding headers and statistics according to the + Zstandard dictionary format. + + You may compose an effective dictionary content by hand, which is used as + basis dictionary, and use some samples to finalize a dictionary. The basis + dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. + + *samples* is an iterable of samples, where a sample is a bytes-like object + representing a file. + *dict_size* is the dictionary's maximum size, in bytes. + *level* is the expected compression level. The statistics for each + compression level differ, so tuning the dictionary to the compression level + can provide improvements. + """ + + if not isinstance(zstd_dict, ZstdDict): + raise TypeError('zstd_dict argument should be a ZstdDict object.') + if not isinstance(dict_size, int): + raise TypeError('dict_size argument should be an int object.') + if not isinstance(level, int): + raise TypeError('level argument should be an int object.') + + samples = tuple(samples) + chunks = b''.join(samples) + chunk_sizes = tuple(_nbytes(sample) for sample in samples) + if not chunks: + raise ValueError("The samples are empty content, can't finalize the " + "dictionary.") + dict_content = _zstd.finalize_dict(zstd_dict.dict_content, chunks, + chunk_sizes, dict_size, level) + return ZstdDict(dict_content) + + +def compress(data, level=None, options=None, zstd_dict=None): + """Return Zstandard compressed *data* as bytes. + + *level* is an int specifying the compression level to use, defaulting to + COMPRESSION_LEVEL_DEFAULT ('3'). + *options* is a dict object that contains advanced compression + parameters. See CompressionParameter for more on options. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See + the function train_dict for how to train a ZstdDict on sample data. + + For incremental compression, use a ZstdCompressor instead. + """ + comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) + return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME) + + +def decompress(data, zstd_dict=None, options=None): + """Decompress one or more frames of Zstandard compressed *data*. + + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See + the function train_dict for how to train a ZstdDict on sample data. + *options* is a dict object that contains advanced compression + parameters. See DecompressionParameter for more on options. + + For incremental decompression, use a ZstdDecompressor instead. + """ + results = [] + while True: + decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) + results.append(decomp.decompress(data)) + if not decomp.eof: + raise ZstdError('Compressed data ended before the ' + 'end-of-stream marker was reached') + data = decomp.unused_data + if not data: + break + return b''.join(results) + + +class CompressionParameter(enum.IntEnum): + """Compression parameters.""" + + compression_level = _zstd.ZSTD_c_compressionLevel + window_log = _zstd.ZSTD_c_windowLog + hash_log = _zstd.ZSTD_c_hashLog + chain_log = _zstd.ZSTD_c_chainLog + search_log = _zstd.ZSTD_c_searchLog + min_match = _zstd.ZSTD_c_minMatch + target_length = _zstd.ZSTD_c_targetLength + strategy = _zstd.ZSTD_c_strategy + + enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching + ldm_hash_log = _zstd.ZSTD_c_ldmHashLog + ldm_min_match = _zstd.ZSTD_c_ldmMinMatch + ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog + ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog + + content_size_flag = _zstd.ZSTD_c_contentSizeFlag + checksum_flag = _zstd.ZSTD_c_checksumFlag + dict_id_flag = _zstd.ZSTD_c_dictIDFlag + + nb_workers = _zstd.ZSTD_c_nbWorkers + job_size = _zstd.ZSTD_c_jobSize + overlap_log = _zstd.ZSTD_c_overlapLog + + def bounds(self): + """Return the (lower, upper) int bounds of a compression parameter. + + Both the lower and upper bounds are inclusive. + """ + return _zstd.get_param_bounds(self.value, is_compress=True) + + +class DecompressionParameter(enum.IntEnum): + """Decompression parameters.""" + + window_log_max = _zstd.ZSTD_d_windowLogMax + + def bounds(self): + """Return the (lower, upper) int bounds of a decompression parameter. + + Both the lower and upper bounds are inclusive. + """ + return _zstd.get_param_bounds(self.value, is_compress=False) + + +class Strategy(enum.IntEnum): + """Compression strategies, listed from fastest to strongest. + + Note that new strategies might be added in the future. + Only the order (from fast to strong) is guaranteed, + the numeric value might change. + """ + + fast = _zstd.ZSTD_fast + dfast = _zstd.ZSTD_dfast + greedy = _zstd.ZSTD_greedy + lazy = _zstd.ZSTD_lazy + lazy2 = _zstd.ZSTD_lazy2 + btlazy2 = _zstd.ZSTD_btlazy2 + btopt = _zstd.ZSTD_btopt + btultra = _zstd.ZSTD_btultra + btultra2 = _zstd.ZSTD_btultra2 + + +# Check validity of the CompressionParameter & DecompressionParameter types +_zstd.set_parameter_types(CompressionParameter, DecompressionParameter) diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py new file mode 100644 index 00000000000..d709f5efc65 --- /dev/null +++ b/Lib/compression/zstd/_zstdfile.py @@ -0,0 +1,345 @@ +import io +from os import PathLike +from _zstd import ZstdCompressor, ZstdDecompressor, ZSTD_DStreamOutSize +from compression._common import _streams + +__all__ = ('ZstdFile', 'open') + +_MODE_CLOSED = 0 +_MODE_READ = 1 +_MODE_WRITE = 2 + + +def _nbytes(dat, /): + if isinstance(dat, (bytes, bytearray)): + return len(dat) + with memoryview(dat) as mv: + return mv.nbytes + + +class ZstdFile(_streams.BaseStream): + """A file-like object providing transparent Zstandard (de)compression. + + A ZstdFile can act as a wrapper for an existing file object, or refer + directly to a named file on disk. + + ZstdFile provides a *binary* file interface. Data is read and returned as + bytes, and may only be written to objects that support the Buffer Protocol. + """ + + FLUSH_BLOCK = ZstdCompressor.FLUSH_BLOCK + FLUSH_FRAME = ZstdCompressor.FLUSH_FRAME + + def __init__(self, file, /, mode='r', *, + level=None, options=None, zstd_dict=None): + """Open a Zstandard compressed file in binary mode. + + *file* can be either an file-like object, or a file name to open. + + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for + creating exclusively, or 'a' for appending. These can equivalently be + given as 'rb', 'wb', 'xb' and 'ab' respectively. + + *level* is an optional int specifying the compression level to use, + or COMPRESSION_LEVEL_DEFAULT if not given. + + *options* is an optional dict for advanced compression parameters. + See CompressionParameter and DecompressionParameter for the possible + options. + + *zstd_dict* is an optional ZstdDict object, a pre-trained Zstandard + dictionary. See train_dict() to train ZstdDict on sample data. + """ + self._fp = None + self._close_fp = False + self._mode = _MODE_CLOSED + self._buffer = None + + if not isinstance(mode, str): + raise ValueError('mode must be a str') + if options is not None and not isinstance(options, dict): + raise TypeError('options must be a dict or None') + mode = mode.removesuffix('b') # handle rb, wb, xb, ab + if mode == 'r': + if level is not None: + raise TypeError('level is illegal in read mode') + self._mode = _MODE_READ + elif mode in {'w', 'a', 'x'}: + if level is not None and not isinstance(level, int): + raise TypeError('level must be int or None') + self._mode = _MODE_WRITE + self._compressor = ZstdCompressor(level=level, options=options, + zstd_dict=zstd_dict) + self._pos = 0 + else: + raise ValueError(f'Invalid mode: {mode!r}') + + if isinstance(file, (str, bytes, PathLike)): + self._fp = io.open(file, f'{mode}b') + self._close_fp = True + elif ((mode == 'r' and hasattr(file, 'read')) + or (mode != 'r' and hasattr(file, 'write'))): + self._fp = file + else: + raise TypeError('file must be a file-like object ' + 'or a str, bytes, or PathLike object') + + if self._mode == _MODE_READ: + raw = _streams.DecompressReader( + self._fp, + ZstdDecompressor, + zstd_dict=zstd_dict, + options=options, + ) + self._buffer = io.BufferedReader(raw) + + def close(self): + """Flush and close the file. + + May be called multiple times. Once the file has been closed, + any other operation on it will raise ValueError. + """ + if self._fp is None: + return + try: + if self._mode == _MODE_READ: + if getattr(self, '_buffer', None): + self._buffer.close() + self._buffer = None + elif self._mode == _MODE_WRITE: + self.flush(self.FLUSH_FRAME) + self._compressor = None + finally: + self._mode = _MODE_CLOSED + try: + if self._close_fp: + self._fp.close() + finally: + self._fp = None + self._close_fp = False + + def write(self, data, /): + """Write a bytes-like object *data* to the file. + + Returns the number of uncompressed bytes written, which is + always the length of data in bytes. Note that due to buffering, + the file on disk may not reflect the data written until .flush() + or .close() is called. + """ + self._check_can_write() + + length = _nbytes(data) + + compressed = self._compressor.compress(data) + self._fp.write(compressed) + self._pos += length + return length + + def flush(self, mode=FLUSH_BLOCK): + """Flush remaining data to the underlying stream. + + The mode argument can be FLUSH_BLOCK or FLUSH_FRAME. Abuse of this + method will reduce compression ratio, use it only when necessary. + + If the program is interrupted afterwards, all data can be recovered. + To ensure saving to disk, also need to use os.fsync(fd). + + This method does nothing in reading mode. + """ + if self._mode == _MODE_READ: + return + self._check_not_closed() + if mode not in {self.FLUSH_BLOCK, self.FLUSH_FRAME}: + raise ValueError('Invalid mode argument, expected either ' + 'ZstdFile.FLUSH_FRAME or ' + 'ZstdFile.FLUSH_BLOCK') + if self._compressor.last_mode == mode: + return + # Flush zstd block/frame, and write. + data = self._compressor.flush(mode) + self._fp.write(data) + if hasattr(self._fp, 'flush'): + self._fp.flush() + + def read(self, size=-1): + """Read up to size uncompressed bytes from the file. + + If size is negative or omitted, read until EOF is reached. + Returns b'' if the file is already at EOF. + """ + if size is None: + size = -1 + self._check_can_read() + return self._buffer.read(size) + + def read1(self, size=-1): + """Read up to size uncompressed bytes, while trying to avoid + making multiple reads from the underlying stream. Reads up to a + buffer's worth of data if size is negative. + + Returns b'' if the file is at EOF. + """ + self._check_can_read() + if size < 0: + # Note this should *not* be io.DEFAULT_BUFFER_SIZE. + # ZSTD_DStreamOutSize is the minimum amount to read guaranteeing + # a full block is read. + size = ZSTD_DStreamOutSize + return self._buffer.read1(size) + + def readinto(self, b): + """Read bytes into b. + + Returns the number of bytes read (0 for EOF). + """ + self._check_can_read() + return self._buffer.readinto(b) + + def readinto1(self, b): + """Read bytes into b, while trying to avoid making multiple reads + from the underlying stream. + + Returns the number of bytes read (0 for EOF). + """ + self._check_can_read() + return self._buffer.readinto1(b) + + def readline(self, size=-1): + """Read a line of uncompressed bytes from the file. + + The terminating newline (if present) is retained. If size is + non-negative, no more than size bytes will be read (in which + case the line may be incomplete). Returns b'' if already at EOF. + """ + self._check_can_read() + return self._buffer.readline(size) + + def seek(self, offset, whence=io.SEEK_SET): + """Change the file position. + + The new position is specified by offset, relative to the + position indicated by whence. Possible values for whence are: + + 0: start of stream (default): offset must not be negative + 1: current stream position + 2: end of stream; offset must not be positive + + Returns the new file position. + + Note that seeking is emulated, so depending on the arguments, + this operation may be extremely slow. + """ + self._check_can_read() + + # BufferedReader.seek() checks seekable + return self._buffer.seek(offset, whence) + + def peek(self, size=-1): + """Return buffered data without advancing the file position. + + Always returns at least one byte of data, unless at EOF. + The exact number of bytes returned is unspecified. + """ + # Relies on the undocumented fact that BufferedReader.peek() always + # returns at least one byte (except at EOF) + self._check_can_read() + return self._buffer.peek(size) + + def __next__(self): + if ret := self._buffer.readline(): + return ret + raise StopIteration + + def tell(self): + """Return the current file position.""" + self._check_not_closed() + if self._mode == _MODE_READ: + return self._buffer.tell() + elif self._mode == _MODE_WRITE: + return self._pos + + def fileno(self): + """Return the file descriptor for the underlying file.""" + self._check_not_closed() + return self._fp.fileno() + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' + + @property + def closed(self): + """True if this file is closed.""" + return self._mode == _MODE_CLOSED + + def seekable(self): + """Return whether the file supports seeking.""" + return self.readable() and self._buffer.seekable() + + def readable(self): + """Return whether the file was opened for reading.""" + self._check_not_closed() + return self._mode == _MODE_READ + + def writable(self): + """Return whether the file was opened for writing.""" + self._check_not_closed() + return self._mode == _MODE_WRITE + + +def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, + encoding=None, errors=None, newline=None): + """Open a Zstandard compressed file in binary or text mode. + + file can be either a file name (given as a str, bytes, or PathLike object), + in which case the named file is opened, or it can be an existing file object + to read from or write to. + + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', + 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. + + The level, options, and zstd_dict parameters specify the settings the same + as ZstdFile. + + When using read mode (decompression), the options parameter is a dict + representing advanced decompression options. The level parameter is not + supported in this case. When using write mode (compression), only one of + level, an int representing the compression level, or options, a dict + representing advanced compression options, may be passed. In both modes, + zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary. + + For binary mode, this function is equivalent to the ZstdFile constructor: + ZstdFile(filename, mode, ...). In this case, the encoding, errors and + newline parameters must not be provided. + + For text mode, an ZstdFile object is created, and wrapped in an + io.TextIOWrapper instance with the specified encoding, error handling + behavior, and line ending(s). + """ + + text_mode = 't' in mode + mode = mode.replace('t', '') + + if text_mode: + if 'b' in mode: + raise ValueError(f'Invalid mode: {mode!r}') + else: + if encoding is not None: + raise ValueError('Argument "encoding" not supported in binary mode') + if errors is not None: + raise ValueError('Argument "errors" not supported in binary mode') + if newline is not None: + raise ValueError('Argument "newline" not supported in binary mode') + + binary_file = ZstdFile(file, mode, level=level, options=options, + zstd_dict=zstd_dict) + + if text_mode: + return io.TextIOWrapper(binary_file, encoding, errors, newline) + else: + return binary_file diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index d746aeac50a..72de617a5b6 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -23,6 +23,7 @@ 'ALL_COMPLETED', 'CancelledError', 'TimeoutError', + 'InvalidStateError', 'BrokenExecutor', 'Future', 'Executor', @@ -50,4 +51,4 @@ def __getattr__(name): ThreadPoolExecutor = te return te - raise AttributeError(f"module {__name__} has no attribute {name}") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index cf119ac6437..7d69a5baead 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -50,9 +50,7 @@ class CancelledError(Error): """The Future was cancelled.""" pass -class TimeoutError(Error): - """The operation exceeded the given deadline.""" - pass +TimeoutError = TimeoutError # make local alias for the standard exception class InvalidStateError(Error): """The operation is not allowed in this state.""" @@ -284,7 +282,7 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): A named 2-tuple of sets. The first set, named 'done', contains the futures that completed (is finished or cancelled) before the wait completed. The second set, named 'not_done', contains uncompleted - futures. Duplicate futures given to *fs* are removed and will be + futures. Duplicate futures given to *fs* are removed and will be returned only once. """ fs = set(fs) @@ -312,6 +310,18 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): done.update(waiter.finished_futures) return DoneAndNotDoneFutures(done, fs - done) + +def _result_or_cancel(fut, timeout=None): + try: + try: + return fut.result(timeout) + finally: + fut.cancel() + finally: + # Break a reference cycle with the exception in self._exception + del fut + + class Future(object): """Represents the result of an asynchronous computation.""" @@ -386,7 +396,7 @@ def done(self): return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED] def __get_result(self): - if self._exception: + if self._exception is not None: try: raise self._exception finally: @@ -606,9 +616,9 @@ def result_iterator(): while fs: # Careful not to keep a reference to the popped future if timeout is None: - yield fs.pop().result() + yield _result_or_cancel(fs.pop()) else: - yield fs.pop().result(end_time - time.monotonic()) + yield _result_or_cancel(fs.pop(), end_time - time.monotonic()) finally: for future in fs: future.cancel() diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 57941e485d8..0dee8303ba2 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -49,6 +49,8 @@ from concurrent.futures import _base import queue import multiprocessing as mp +# This import is required to load the multiprocessing.connection submodule +# so that it can be accessed later as `mp.connection` import multiprocessing.connection from multiprocessing.queues import Queue import threading @@ -56,7 +58,7 @@ from functools import partial import itertools import sys -import traceback +from traceback import format_exception _threads_wakeups = weakref.WeakKeyDictionary() @@ -66,22 +68,31 @@ class _ThreadWakeup: def __init__(self): self._closed = False + self._lock = threading.Lock() self._reader, self._writer = mp.Pipe(duplex=False) def close(self): - if not self._closed: - self._closed = True - self._writer.close() - self._reader.close() + # Please note that we do not take the self._lock when + # calling clear() (to avoid deadlocking) so this method can + # only be called safely from the same thread as all calls to + # clear() even if you hold the lock. Otherwise we + # might try to read from the closed pipe. + with self._lock: + if not self._closed: + self._closed = True + self._writer.close() + self._reader.close() def wakeup(self): - if not self._closed: - self._writer.send_bytes(b"") + with self._lock: + if not self._closed: + self._writer.send_bytes(b"") def clear(self): - if not self._closed: - while self._reader.poll(): - self._reader.recv_bytes() + if self._closed: + raise RuntimeError('operation on closed _ThreadWakeup') + while self._reader.poll(): + self._reader.recv_bytes() def _python_exit(): @@ -123,8 +134,7 @@ def __str__(self): class _ExceptionWithTraceback: def __init__(self, exc, tb): - tb = traceback.format_exception(type(exc), exc, tb) - tb = ''.join(tb) + tb = ''.join(format_exception(type(exc), exc, tb)) self.exc = exc # Traceback object needs to be garbage-collected as its frames # contain references to all the objects in the exception scope @@ -145,10 +155,11 @@ def __init__(self, future, fn, args, kwargs): self.kwargs = kwargs class _ResultItem(object): - def __init__(self, work_id, exception=None, result=None): + def __init__(self, work_id, exception=None, result=None, exit_pid=None): self.work_id = work_id self.exception = exception self.result = result + self.exit_pid = exit_pid class _CallItem(object): def __init__(self, work_id, fn, args, kwargs): @@ -160,20 +171,17 @@ def __init__(self, work_id, fn, args, kwargs): class _SafeQueue(Queue): """Safe Queue set exception to the future object linked to a job""" - def __init__(self, max_size=0, *, ctx, pending_work_items, shutdown_lock, - thread_wakeup): + def __init__(self, max_size=0, *, ctx, pending_work_items, thread_wakeup): self.pending_work_items = pending_work_items - self.shutdown_lock = shutdown_lock self.thread_wakeup = thread_wakeup super().__init__(max_size, ctx=ctx) def _on_queue_feeder_error(self, e, obj): if isinstance(obj, _CallItem): - tb = traceback.format_exception(type(e), e, e.__traceback__) + tb = format_exception(type(e), e, e.__traceback__) e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb))) work_item = self.pending_work_items.pop(obj.work_id, None) - with self.shutdown_lock: - self.thread_wakeup.wakeup() + self.thread_wakeup.wakeup() # work_item can be None if another process terminated. In this # case, the executor_manager_thread fails all work_items # with BrokenProcessPool @@ -183,16 +191,6 @@ def _on_queue_feeder_error(self, e, obj): super()._on_queue_feeder_error(e, obj) -def _get_chunks(*iterables, chunksize): - """ Iterates over zip()ed iterables in chunks. """ - it = zip(*iterables) - while True: - chunk = tuple(itertools.islice(it, chunksize)) - if not chunk: - return - yield chunk - - def _process_chunk(fn, chunk): """ Processes a chunk of an iterable passed to map. @@ -205,17 +203,19 @@ def _process_chunk(fn, chunk): return [fn(*args) for args in chunk] -def _sendback_result(result_queue, work_id, result=None, exception=None): +def _sendback_result(result_queue, work_id, result=None, exception=None, + exit_pid=None): """Safely send back the given result or exception""" try: result_queue.put(_ResultItem(work_id, result=result, - exception=exception)) + exception=exception, exit_pid=exit_pid)) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) - result_queue.put(_ResultItem(work_id, exception=exc)) + result_queue.put(_ResultItem(work_id, exception=exc, + exit_pid=exit_pid)) -def _process_worker(call_queue, result_queue, initializer, initargs): +def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=None): """Evaluates calls from call_queue and places the results in result_queue. This worker is run in a separate process. @@ -236,25 +236,38 @@ def _process_worker(call_queue, result_queue, initializer, initargs): # The parent will notice that the process stopped and # mark the pool broken return + num_tasks = 0 + exit_pid = None while True: call_item = call_queue.get(block=True) if call_item is None: # Wake up queue management thread result_queue.put(os.getpid()) return + + if max_tasks is not None: + num_tasks += 1 + if num_tasks >= max_tasks: + exit_pid = os.getpid() + try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) - _sendback_result(result_queue, call_item.work_id, exception=exc) + _sendback_result(result_queue, call_item.work_id, exception=exc, + exit_pid=exit_pid) else: - _sendback_result(result_queue, call_item.work_id, result=r) + _sendback_result(result_queue, call_item.work_id, result=r, + exit_pid=exit_pid) del r # Liberate the resource as soon as possible, to avoid holding onto # open files or shared memory that is not needed anymore del call_item + if exit_pid is not None: + return + class _ExecutorManagerThread(threading.Thread): """Manages the communication between this process and the worker processes. @@ -284,11 +297,10 @@ def __init__(self, executor): # if there is no pending work item. def weakref_cb(_, thread_wakeup=self.thread_wakeup, - shutdown_lock=self.shutdown_lock): - mp.util.debug('Executor collected: triggering callback for' + mp_util_debug=mp.util.debug): + mp_util_debug('Executor collected: triggering callback for' ' QueueManager wakeup') - with shutdown_lock: - thread_wakeup.wakeup() + thread_wakeup.wakeup() self.executor_reference = weakref.ref(executor, weakref_cb) @@ -305,6 +317,10 @@ def weakref_cb(_, # A queue.Queue of work ids e.g. Queue([5, 6, ...]). self.work_ids_queue = executor._work_ids + # Maximum number of tasks a worker process can execute before + # exiting safely + self.max_tasks_per_child = executor._max_tasks_per_child + # A dict mapping work ids to _WorkItems e.g. # {5: <_WorkItem...>, 6: <_WorkItem...>, ...} self.pending_work_items = executor._pending_work_items @@ -315,7 +331,14 @@ def run(self): # Main loop for the executor manager thread. while True: - self.add_call_item_to_queue() + # gh-109047: During Python finalization, self.call_queue.put() + # creation of a thread can fail with RuntimeError. + try: + self.add_call_item_to_queue() + except BaseException as exc: + cause = format_exception(exc) + self.terminate_broken(cause) + return result_item, is_broken, cause = self.wait_result_broken_or_wakeup() @@ -324,19 +347,32 @@ def run(self): return if result_item is not None: self.process_result_item(result_item) + + process_exited = result_item.exit_pid is not None + if process_exited: + p = self.processes.pop(result_item.exit_pid) + p.join() + # Delete reference to result_item to avoid keeping references # while waiting on new results. del result_item - # attempt to increment idle process count - executor = self.executor_reference() - if executor is not None: - executor._idle_worker_semaphore.release() - del executor + if executor := self.executor_reference(): + if process_exited: + with self.shutdown_lock: + executor._adjust_process_count() + else: + executor._idle_worker_semaphore.release() + del executor if self.is_shutting_down(): self.flag_executor_shutting_down() + # When only canceled futures remain in pending_work_items, our + # next call to wait_result_broken_or_wakeup would hang forever. + # This makes sure we have some running futures or none at all. + self.add_call_item_to_queue() + # Since no new work items can be added, it is safe to shutdown # this thread if there are no pending work items. if not self.pending_work_items: @@ -386,14 +422,13 @@ def wait_result_broken_or_wakeup(self): try: result_item = result_reader.recv() is_broken = False - except BaseException as e: - cause = traceback.format_exception(type(e), e, e.__traceback__) + except BaseException as exc: + cause = format_exception(exc) elif wakeup_reader in ready: is_broken = False - with self.shutdown_lock: - self.thread_wakeup.clear() + self.thread_wakeup.clear() return result_item, is_broken, cause @@ -401,24 +436,14 @@ def process_result_item(self, result_item): # Process the received a result_item. This can be either the PID of a # worker that exited gracefully or a _ResultItem - if isinstance(result_item, int): - # Clean shutdown of a worker using its PID - # (avoids marking the executor broken) - assert self.is_shutting_down() - p = self.processes.pop(result_item) - p.join() - if not self.processes: - self.join_executor_internals() - return - else: - # Received a _ResultItem so mark the future as completed. - work_item = self.pending_work_items.pop(result_item.work_id, None) - # work_item can be None if another process terminated (see above) - if work_item is not None: - if result_item.exception: - work_item.future.set_exception(result_item.exception) - else: - work_item.future.set_result(result_item.result) + # Received a _ResultItem so mark the future as completed. + work_item = self.pending_work_items.pop(result_item.work_id, None) + # work_item can be None if another process terminated (see above) + if work_item is not None: + if result_item.exception is not None: + work_item.future.set_exception(result_item.exception) + else: + work_item.future.set_result(result_item.result) def is_shutting_down(self): # Check whether we should start shutting down the executor. @@ -430,7 +455,7 @@ def is_shutting_down(self): return (_global_shutdown or executor is None or executor._shutdown_thread) - def terminate_broken(self, cause): + def _terminate_broken(self, cause): # Terminate the executor because it is in a broken state. The cause # argument can be used to display more information on the error that # lead the executor into becoming broken. @@ -455,7 +480,14 @@ def terminate_broken(self, cause): # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): - work_item.future.set_exception(bpe) + try: + work_item.future.set_exception(bpe) + except _base.InvalidStateError: + # set_exception() fails if the future is cancelled: ignore it. + # Trying to check if the future is cancelled before calling + # set_exception() would leave a race condition if the future is + # cancelled between the check and set_exception(). + pass # Delete references to object. See issue16284 del work_item self.pending_work_items.clear() @@ -465,8 +497,14 @@ def terminate_broken(self, cause): for p in self.processes.values(): p.terminate() + self.call_queue._terminate_broken() + # clean up resources - self.join_executor_internals() + self._join_executor_internals(broken=True) + + def terminate_broken(self, cause): + with self.shutdown_lock: + self._terminate_broken(cause) def flag_executor_shutting_down(self): # Flag the executor as shutting down and cancel remaining tasks if @@ -509,15 +547,24 @@ def shutdown_workers(self): break def join_executor_internals(self): - self.shutdown_workers() + with self.shutdown_lock: + self._join_executor_internals() + + def _join_executor_internals(self, broken=False): + # If broken, call_queue was closed and so can no longer be used. + if not broken: + self.shutdown_workers() + # Release the queue's resources as soon as possible. self.call_queue.close() self.call_queue.join_thread() - with self.shutdown_lock: - self.thread_wakeup.close() + self.thread_wakeup.close() + # If .join() is not called on the created processes then # some ctx.Queue methods may deadlock on Mac OS X. for p in self.processes.values(): + if broken: + p.terminate() p.join() def get_n_children_alive(self): @@ -582,22 +629,29 @@ class BrokenProcessPool(_base.BrokenExecutor): class ProcessPoolExecutor(_base.Executor): def __init__(self, max_workers=None, mp_context=None, - initializer=None, initargs=()): + initializer=None, initargs=(), *, max_tasks_per_child=None): """Initializes a new ProcessPoolExecutor instance. Args: max_workers: The maximum number of processes that can be used to execute the given calls. If None or not given then as many worker processes will be created as the machine has processors. - mp_context: A multiprocessing context to launch the workers. This + mp_context: A multiprocessing context to launch the workers created + using the multiprocessing.get_context('start method') API. This object should provide SimpleQueue, Queue and Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. + max_tasks_per_child: The maximum number of tasks a worker process + can complete before it will exit and be replaced with a fresh + worker process. The default of None means worker process will + live as long as the executor. Requires a non-'fork' mp_context + start method. When given, we default to using 'spawn' if no + mp_context is supplied. """ _check_system_limits() if max_workers is None: - self._max_workers = os.cpu_count() or 1 + self._max_workers = os.process_cpu_count() or 1 if sys.platform == 'win32': self._max_workers = min(_MAX_WINDOWS_WORKERS, self._max_workers) @@ -612,7 +666,10 @@ def __init__(self, max_workers=None, mp_context=None, self._max_workers = max_workers if mp_context is None: - mp_context = mp.get_context() + if max_tasks_per_child is not None: + mp_context = mp.get_context("spawn") + else: + mp_context = mp.get_context() self._mp_context = mp_context # https://github.com/python/cpython/issues/90622 @@ -624,6 +681,18 @@ def __init__(self, max_workers=None, mp_context=None, self._initializer = initializer self._initargs = initargs + if max_tasks_per_child is not None: + if not isinstance(max_tasks_per_child, int): + raise TypeError("max_tasks_per_child must be an integer") + elif max_tasks_per_child <= 0: + raise ValueError("max_tasks_per_child must be >= 1") + if self._mp_context.get_start_method(allow_none=False) == "fork": + # https://github.com/python/cpython/issues/90622 + raise ValueError("max_tasks_per_child is incompatible with" + " the 'fork' multiprocessing start method;" + " supply a different mp_context.") + self._max_tasks_per_child = max_tasks_per_child + # Management thread self._executor_manager_thread = None @@ -646,7 +715,9 @@ def __init__(self, max_workers=None, mp_context=None, # as it could result in a deadlock if a worker process dies with the # _result_queue write lock still acquired. # - # _shutdown_lock must be locked to access _ThreadWakeup. + # Care must be taken to only call clear and close from the + # executor_manager_thread, since _ThreadWakeup.clear() is not protected + # by a lock. self._executor_manager_thread_wakeup = _ThreadWakeup() # Create communication channels for the executor @@ -657,7 +728,6 @@ def __init__(self, max_workers=None, mp_context=None, self._call_queue = _SafeQueue( max_size=queue_size, ctx=self._mp_context, pending_work_items=self._pending_work_items, - shutdown_lock=self._shutdown_lock, thread_wakeup=self._executor_manager_thread_wakeup) # Killed worker processes can produce spurious "broken pipe" # tracebacks in the queue's own worker thread. But we detect killed @@ -677,6 +747,11 @@ def _start_executor_manager_thread(self): self._executor_manager_thread_wakeup def _adjust_process_count(self): + # gh-132969: avoid error when state is reset and executor is still running, + # which will happen when shutdown(wait=False) is called. + if self._processes is None: + return + # if there's an idle process, we don't need to spawn a new one. if self._idle_worker_semaphore.acquire(blocking=False): return @@ -705,7 +780,8 @@ def _spawn_process(self): args=(self._call_queue, self._result_queue, self._initializer, - self._initargs)) + self._initargs, + self._max_tasks_per_child)) p.start() self._processes[p.pid] = p @@ -759,7 +835,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1): raise ValueError("chunksize must be >= 1.") results = super().map(partial(_process_chunk, fn), - _get_chunks(*iterables, chunksize=chunksize), + itertools.batched(zip(*iterables), chunksize), timeout=timeout) return _chain_from_iterable_of_lists(results) diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 493861d314d..9021dde48ef 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -37,14 +37,14 @@ def _python_exit(): threading._register_atexit(_python_exit) # At fork, reinitialize the `_global_shutdown_lock` lock in the child process -# TODO RUSTPYTHON - _at_fork_reinit is not implemented yet -if hasattr(os, 'register_at_fork') and hasattr(_global_shutdown_lock, '_at_fork_reinit'): +if hasattr(os, 'register_at_fork'): os.register_at_fork(before=_global_shutdown_lock.acquire, after_in_child=_global_shutdown_lock._at_fork_reinit, after_in_parent=_global_shutdown_lock.release) + os.register_at_fork(after_in_child=_threads_queues.clear) -class _WorkItem(object): +class _WorkItem: def __init__(self, future, fn, args, kwargs): self.future = future self.fn = fn @@ -79,17 +79,20 @@ def _worker(executor_reference, work_queue, initializer, initargs): return try: while True: - work_item = work_queue.get(block=True) - if work_item is not None: - work_item.run() - # Delete references to object. See issue16284 - del work_item - - # attempt to increment idle count + try: + work_item = work_queue.get_nowait() + except queue.Empty: + # attempt to increment idle count if queue is empty executor = executor_reference() if executor is not None: executor._idle_semaphore.release() del executor + work_item = work_queue.get(block=True) + + if work_item is not None: + work_item.run() + # Delete references to object. See GH-60488 + del work_item continue executor = executor_reference() @@ -137,10 +140,10 @@ def __init__(self, max_workers=None, thread_name_prefix='', # * CPU bound task which releases GIL # * I/O bound task (which releases GIL, of course) # - # We use cpu_count + 4 for both types of tasks. + # We use process_cpu_count + 4 for both types of tasks. # But we limit it to 32 to avoid consuming surprisingly large resource # on many core machine. - max_workers = min(32, (os.cpu_count() or 1) + 4) + max_workers = min(32, (os.process_cpu_count() or 1) + 4) if max_workers <= 0: raise ValueError("max_workers must be greater than 0") diff --git a/Lib/configparser.py b/Lib/configparser.py index df2d7e335d9..d435a5c2fe0 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -18,8 +18,8 @@ delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', - interpolation=, converters=): - + interpolation=, converters=, + allow_unnamed_section=False): Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -59,7 +59,7 @@ instance. It will be used as the handler for option value pre-processing when using getters. RawConfigParser objects don't do any sort of interpolation, whereas ConfigParser uses an instance of - BasicInterpolation. The library also provides a ``zc.buildbot`` + BasicInterpolation. The library also provides a ``zc.buildout`` inspired ExtendedInterpolation implementation. When `converters` is given, it should be a dictionary where each key @@ -68,6 +68,10 @@ converter gets its corresponding get*() method on the parser object and section proxies. + When `allow_unnamed_section` is True (default: False), options + without section are accepted: the section for these is + ``configparser.UNNAMED_SECTION``. + sections() Return all the configuration section names, sans DEFAULT. @@ -139,24 +143,27 @@ between keys and values are surrounded by spaces. """ -from collections.abc import MutableMapping +# Do not import dataclasses; overhead is unacceptable (gh-117703) + +from collections.abc import Iterable, MutableMapping from collections import ChainMap as _ChainMap +import contextlib import functools import io import itertools import os import re import sys -import warnings -__all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError", +__all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", - "ConfigParser", "SafeConfigParser", "RawConfigParser", + "MultilineContinuationError", "UnnamedSectionDisabledError", + "InvalidWriteError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", - "LegacyInterpolation", "SectionProxy", "ConverterMapping", - "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"] + "SectionProxy", "ConverterMapping", + "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") _default_dict = dict DEFAULTSECT = "DEFAULT" @@ -298,44 +305,33 @@ def __init__(self, option, section, rawval): class ParsingError(Error): """Raised when a configuration file does not follow legal syntax.""" - def __init__(self, source=None, filename=None): - # Exactly one of `source'/`filename' arguments has to be given. - # `filename' kept for compatibility. - if filename and source: - raise ValueError("Cannot specify both `filename' and `source'. " - "Use `source'.") - elif not filename and not source: - raise ValueError("Required argument `source' not given.") - elif filename: - source = filename - Error.__init__(self, 'Source contains parsing errors: %r' % source) + def __init__(self, source, *args): + super().__init__(f'Source contains parsing errors: {source!r}') self.source = source self.errors = [] self.args = (source, ) - - @property - def filename(self): - """Deprecated, use `source'.""" - warnings.warn( - "The 'filename' attribute will be removed in Python 3.12. " - "Use 'source' instead.", - DeprecationWarning, stacklevel=2 - ) - return self.source - - @filename.setter - def filename(self, value): - """Deprecated, user `source'.""" - warnings.warn( - "The 'filename' attribute will be removed in Python 3.12. " - "Use 'source' instead.", - DeprecationWarning, stacklevel=2 - ) - self.source = value + if args: + self.append(*args) def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, line) + self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + + def combine(self, others): + for other in others: + for error in other.errors: + self.append(*error) + return self + + @staticmethod + def _raise_all(exceptions: Iterable['ParsingError']): + """ + Combine any number of ParsingErrors into one and raise it. + """ + exceptions = iter(exceptions) + with contextlib.suppress(StopIteration): + raise next(exceptions).combine(exceptions) + class MissingSectionHeaderError(ParsingError): @@ -352,6 +348,44 @@ def __init__(self, filename, lineno, line): self.args = (filename, lineno, line) +class MultilineContinuationError(ParsingError): + """Raised when a key without value is followed by continuation line""" + def __init__(self, filename, lineno, line): + Error.__init__( + self, + "Key without value continued with an indented line.\n" + "file: %r, line: %d\n%r" + %(filename, lineno, line)) + self.source = filename + self.lineno = lineno + self.line = line + self.args = (filename, lineno, line) + + +class UnnamedSectionDisabledError(Error): + """Raised when an attempt to use UNNAMED_SECTION is made with the + feature disabled.""" + def __init__(self): + Error.__init__(self, "Support for UNNAMED_SECTION is disabled.") + + +class _UnnamedSection: + + def __repr__(self): + return "" + +class InvalidWriteError(Error): + """Raised when attempting to write data that the parser would read back differently. + ex: writing a key which begins with the section header pattern would read back as a + new section """ + + def __init__(self, msg=''): + Error.__init__(self, msg) + + +UNNAMED_SECTION = _UnnamedSection() + + # Used in parser getters to indicate the default behaviour when a specific # option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. @@ -507,6 +541,8 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, except (KeyError, NoSectionError, NoOptionError): raise InterpolationMissingOptionError( option, section, rawval, ":".join(path)) from None + if v is None: + continue if "$" in v: self._interpolate_some(parser, opt, accum, v, sect, dict(parser.items(sect, raw=True)), @@ -520,51 +556,51 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, "found: %r" % (rest,)) -class LegacyInterpolation(Interpolation): - """Deprecated interpolation used in old versions of ConfigParser. - Use BasicInterpolation or ExtendedInterpolation instead.""" +class _ReadState: + elements_added : set[str] + cursect : dict[str, str] | None = None + sectname : str | None = None + optname : str | None = None + lineno : int = 0 + indent_level : int = 0 + errors : list[ParsingError] - _KEYCRE = re.compile(r"%\(([^)]*)\)s|.") + def __init__(self): + self.elements_added = set() + self.errors = list() - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - warnings.warn( - "LegacyInterpolation has been deprecated since Python 3.2 " - "and will be removed from the configparser module in Python 3.13. " - "Use BasicInterpolation or ExtendedInterpolation instead.", - DeprecationWarning, stacklevel=2 - ) - def before_get(self, parser, section, option, value, vars): - rawval = value - depth = MAX_INTERPOLATION_DEPTH - while depth: # Loop through this until it's done - depth -= 1 - if value and "%(" in value: - replace = functools.partial(self._interpolation_replace, - parser=parser) - value = self._KEYCRE.sub(replace, value) - try: - value = value % vars - except KeyError as e: - raise InterpolationMissingOptionError( - option, section, rawval, e.args[0]) from None - else: - break - if value and "%(" in value: - raise InterpolationDepthError(option, section, rawval) - return value +class _Line(str): + __slots__ = 'clean', 'has_comments' - def before_set(self, parser, section, option, value): - return value + def __new__(cls, val, *args, **kwargs): + return super().__new__(cls, val) - @staticmethod - def _interpolation_replace(match, parser): - s = match.group(1) - if s is None: - return match.group() - else: - return "%%(%s)s" % parser.optionxform(s) + def __init__(self, val, comments): + trimmed = val.strip() + self.clean = comments.strip(trimmed) + self.has_comments = trimmed != self.clean + + +class _CommentSpec: + def __init__(self, full_prefixes, inline_prefixes): + full_patterns = ( + # prefix at the beginning of a line + fr'^({re.escape(prefix)}).*' + for prefix in full_prefixes + ) + inline_patterns = ( + # prefix at the beginning of the line or following a space + fr'(^|\s)({re.escape(prefix)}.*)' + for prefix in inline_prefixes + ) + self.pattern = re.compile('|'.join(itertools.chain(full_patterns, inline_patterns))) + + def strip(self, text): + return self.pattern.sub('', text).rstrip() + + def wrap(self, text): + return _Line(text, self) class RawConfigParser(MutableMapping): @@ -613,7 +649,8 @@ def __init__(self, defaults=None, dict_type=_default_dict, comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET, converters=_UNSET): + interpolation=_UNSET, converters=_UNSET, + allow_unnamed_section=False,): self._dict = dict_type self._sections = self._dict() @@ -632,8 +669,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._comment_prefixes = tuple(comment_prefixes or ()) - self._inline_comment_prefixes = tuple(inline_comment_prefixes or ()) + self._comments = _CommentSpec(comment_prefixes or (), inline_comment_prefixes or ()) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -652,6 +688,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._converters.update(converters) if defaults: self._read_defaults(defaults) + self._allow_unnamed_section = allow_unnamed_section def defaults(self): return self._defaults @@ -670,6 +707,10 @@ def add_section(self, section): if section == self.default_section: raise ValueError('Invalid section name: %r' % section) + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + if section in self._sections: raise DuplicateSectionError(section) self._sections[section] = self._dict() @@ -753,7 +794,8 @@ def read_dict(self, dictionary, source=''): """ elements_added = set() for section, keys in dictionary.items(): - section = str(section) + if section is not UNNAMED_SECTION: + section = str(section) try: self.add_section(section) except (DuplicateSectionError, ValueError): @@ -769,15 +811,6 @@ def read_dict(self, dictionary, source=''): elements_added.add((section, key)) self.set(section, key, value) - def readfp(self, fp, filename=None): - """Deprecated, use read_file instead.""" - warnings.warn( - "This method will be removed in Python 3.12. " - "Use 'parser.read_file()' instead.", - DeprecationWarning, stacklevel=2 - ) - self.read_file(fp, source=filename) - def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. @@ -934,14 +967,21 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) + if UNNAMED_SECTION in self._sections and self._sections[UNNAMED_SECTION]: + self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) + for section in self._sections: + if section is UNNAMED_SECTION: + continue self._write_section(fp, section, self._sections[section].items(), d) - def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp`.""" - fp.write("[{}]\n".format(section_name)) + def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): + """Write a single section to the specified 'fp'.""" + if not unnamed: + fp.write("[{}]\n".format(section_name)) for key, value in section_items: + self._validate_key_contents(key) value = self._interpolation.before_write(self, section_name, key, value) if value is not None or not self._allow_no_value: @@ -1026,110 +1066,111 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - elements_added = set() - cursect = None # None, or a dictionary - sectname = None - optname = None - lineno = 0 - indent_level = 0 - e = None # None, or an exception - for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize - # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes - # strip full line comments - for prefix in self._comment_prefixes: - if line.strip().startswith(prefix): - comment_start = 0 - break - if comment_start == sys.maxsize: - comment_start = None - value = line[:comment_start].strip() - if not value: + try: + ParsingError._raise_all(self._read_inner(fp, fpname)) + finally: + self._join_multiline_values() + + def _read_inner(self, fp, fpname): + st = _ReadState() + + for st.lineno, line in enumerate(map(self._comments.wrap, fp), start=1): + if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no # comment on the line - if (comment_start is None and - cursect is not None and - optname and - cursect[optname] is not None): - cursect[optname].append('') # newlines added at join + if (not line.has_comments and + st.cursect is not None and + st.optname and + st.cursect[st.optname] is not None): + st.cursect[st.optname].append('') # newlines added at join else: # empty line marks end of value - indent_level = sys.maxsize + st.indent_level = sys.maxsize continue - # continuation line? + first_nonspace = self.NONSPACECRE.search(line) - cur_indent_level = first_nonspace.start() if first_nonspace else 0 - if (cursect is not None and optname and - cur_indent_level > indent_level): - cursect[optname].append(value) - # a section header or option header? - else: - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults - else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header in the file? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? - else: - mo = self._optcre.match(value) - if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] - else: - # valueless option handling - cursect[optname] = None - else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) - self._join_multiline_values() - # if any parsing errors occurred, raise an exception - if e: - raise e + st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 + + if self._handle_continuation_line(st, line, fpname): + continue + + self._handle_rest(st, line, fpname) + + return st.errors + + def _handle_continuation_line(self, st, line, fpname): + # continuation line? + is_continue = (st.cursect is not None and st.optname and + st.cur_indent_level > st.indent_level) + if is_continue: + if st.cursect[st.optname] is None: + raise MultilineContinuationError(fpname, st.lineno, line) + st.cursect[st.optname].append(line.clean) + return is_continue + + def _handle_rest(self, st, line, fpname): + # a section header or option header? + if self._allow_unnamed_section and st.cursect is None: + self._handle_header(st, UNNAMED_SECTION, fpname) + + st.indent_level = st.cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(line.clean) + + if not mo and st.cursect is None: + raise MissingSectionHeaderError(fpname, st.lineno, line) + + self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname) + + def _handle_header(self, st, sectname, fpname): + st.sectname = sectname + if st.sectname in self._sections: + if self._strict and st.sectname in st.elements_added: + raise DuplicateSectionError(st.sectname, fpname, + st.lineno) + st.cursect = self._sections[st.sectname] + st.elements_added.add(st.sectname) + elif st.sectname == self.default_section: + st.cursect = self._defaults + else: + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + # So sections can't start with a continuation line + st.optname = None + + def _handle_option(self, st, line, fpname): + # an option line? + st.indent_level = st.cur_indent_level + + mo = self._optcre.match(line.clean) + if not mo: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + st.errors.append(ParsingError(fpname, st.lineno, line)) + return + + st.optname, vi, optval = mo.group('option', 'vi', 'value') + if not st.optname: + st.errors.append(ParsingError(fpname, st.lineno, line)) + st.optname = self.optionxform(st.optname.rstrip()) + if (self._strict and + (st.sectname, st.optname) in st.elements_added): + raise DuplicateOptionError(st.sectname, st.optname, + fpname, st.lineno) + st.elements_added.add((st.sectname, st.optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + st.cursect[st.optname] = [optval] + else: + # valueless option handling + st.cursect[st.optname] = None def _join_multiline_values(self): defaults = self.default_section, self._defaults @@ -1149,12 +1190,6 @@ def _read_defaults(self, defaults): for key, value in defaults.items(): self._defaults[self.optionxform(key)] = value - def _handle_error(self, exc, fpname, lineno, line): - if not exc: - exc = ParsingError(fpname) - exc.append(lineno, repr(line)) - return exc - def _unify_values(self, section, vars): """Create a sequence of lookups with 'vars' taking priority over the 'section' which takes priority over the DEFAULTSECT. @@ -1182,21 +1217,32 @@ def _convert_to_boolean(self, value): raise ValueError('Not a boolean: %s' % value) return self.BOOLEAN_STATES[value.lower()] + def _validate_key_contents(self, key): + """Raises an InvalidWriteError for any keys containing + delimiters or that begins with the section header pattern""" + if re.match(self.SECTCRE, key): + raise InvalidWriteError( + f"Cannot write key {key}; begins with section pattern") + for delim in self._delimiters: + if delim in key: + raise InvalidWriteError( + f"Cannot write key {key}; contains delimiter {delim}") + def _validate_value_types(self, *, section="", option="", value=""): - """Raises a TypeError for non-string values. + """Raises a TypeError for illegal non-string values. - The only legal non-string value if we allow valueless - options is None, so we need to check if the value is a - string if: - - we do not allow valueless options, or - - we allow valueless options but the value is not None + Legal non-string values are UNNAMED_SECTION and falsey values if + they are allowed. For compatibility reasons this method is not used in classic set() for RawConfigParsers. It is invoked in every case for mapping protocol access and in ConfigParser.set(). """ - if not isinstance(section, str): - raise TypeError("section names must be strings") + if section is UNNAMED_SECTION: + if not self._allow_unnamed_section: + raise UnnamedSectionDisabledError + elif not isinstance(section, str): + raise TypeError("section names must be strings or UNNAMED_SECTION") if not isinstance(option, str): raise TypeError("option keys must be strings") if not self._allow_no_value or value: @@ -1240,19 +1286,6 @@ def _read_defaults(self, defaults): self._interpolation = hold_interpolation -class SafeConfigParser(ConfigParser): - """ConfigParser alias for backwards compatibility purposes.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - warnings.warn( - "The SafeConfigParser class has been renamed to ConfigParser " - "in Python 3.2. This alias will be removed in Python 3.12." - " Use ConfigParser directly instead.", - DeprecationWarning, stacklevel=2 - ) - - class SectionProxy(MutableMapping): """A proxy for a single section from a parser.""" diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 58e9a498878..5b646fabca0 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -20,6 +20,8 @@ class AbstractContextManager(abc.ABC): __class_getitem__ = classmethod(GenericAlias) + __slots__ = () + def __enter__(self): """Return `self` upon entering the runtime context.""" return self @@ -42,6 +44,8 @@ class AbstractAsyncContextManager(abc.ABC): __class_getitem__ = classmethod(GenericAlias) + __slots__ = () + async def __aenter__(self): """Return `self` upon entering the runtime context.""" return self @@ -145,14 +149,17 @@ def __exit__(self, typ, value, traceback): except StopIteration: return False else: - raise RuntimeError("generator didn't stop") + try: + raise RuntimeError("generator didn't stop") + finally: + self.gen.close() else: if value is None: # Need to force instantiation so we can reliably # tell if we get the same exception back value = typ() try: - self.gen.throw(typ, value, traceback) + self.gen.throw(value) except StopIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration @@ -187,7 +194,10 @@ def __exit__(self, typ, value, traceback): raise exc.__traceback__ = traceback return False - raise RuntimeError("generator didn't stop after throw()") + try: + raise RuntimeError("generator didn't stop after throw()") + finally: + self.gen.close() class _AsyncGeneratorContextManager( _GeneratorContextManagerBase, @@ -212,14 +222,17 @@ async def __aexit__(self, typ, value, traceback): except StopAsyncIteration: return False else: - raise RuntimeError("generator didn't stop") + try: + raise RuntimeError("generator didn't stop") + finally: + await self.gen.aclose() else: if value is None: # Need to force instantiation so we can reliably # tell if we get the same exception back value = typ() try: - await self.gen.athrow(typ, value, traceback) + await self.gen.athrow(value) except StopAsyncIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration @@ -254,7 +267,10 @@ async def __aexit__(self, typ, value, traceback): raise exc.__traceback__ = traceback return False - raise RuntimeError("generator didn't stop after athrow()") + try: + raise RuntimeError("generator didn't stop after athrow()") + finally: + await self.gen.aclose() def contextmanager(func): @@ -441,7 +457,16 @@ def __exit__(self, exctype, excinst, exctb): # exactly reproduce the limitations of the CPython interpreter. # # See http://bugs.python.org/issue12029 for more details - return exctype is not None and issubclass(exctype, self._exceptions) + if exctype is None: + return + if issubclass(exctype, self._exceptions): + return True + if issubclass(exctype, BaseExceptionGroup): + match, rest = excinst.split(self._exceptions) + if rest is None: + return True + raise rest + return False class _BaseExitStack: @@ -544,11 +569,12 @@ def __enter__(self): return self def __exit__(self, *exc_details): - received_exc = exc_details[0] is not None + exc = exc_details[1] + received_exc = exc is not None # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements - frame_exc = sys.exc_info()[1] + frame_exc = sys.exception() def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: @@ -571,24 +597,28 @@ def _fix_exception_context(new_exc, old_exc): is_sync, cb = self._exit_callbacks.pop() assert is_sync try: + if exc is None: + exc_details = None, None, None + else: + exc_details = type(exc), exc, exc.__traceback__ if cb(*exc_details): suppressed_exc = True pending_raise = False - exc_details = (None, None, None) - except: - new_exc_details = sys.exc_info() + exc = None + except BaseException as new_exc: # simulate the stack of exceptions by setting the context - _fix_exception_context(new_exc_details[1], exc_details[1]) + _fix_exception_context(new_exc, exc) pending_raise = True - exc_details = new_exc_details + exc = new_exc + if pending_raise: try: - # bare "raise exc_details[1]" replaces our carefully + # bare "raise exc" replaces our carefully # set-up context - fixed_ctx = exc_details[1].__context__ - raise exc_details[1] + fixed_ctx = exc.__context__ + raise exc except BaseException: - exc_details[1].__context__ = fixed_ctx + exc.__context__ = fixed_ctx raise return received_exc and suppressed_exc @@ -684,11 +714,12 @@ async def __aenter__(self): return self async def __aexit__(self, *exc_details): - received_exc = exc_details[0] is not None + exc = exc_details[1] + received_exc = exc is not None # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements - frame_exc = sys.exc_info()[1] + frame_exc = sys.exception() def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: @@ -710,6 +741,10 @@ def _fix_exception_context(new_exc, old_exc): while self._exit_callbacks: is_sync, cb = self._exit_callbacks.pop() try: + if exc is None: + exc_details = None, None, None + else: + exc_details = type(exc), exc, exc.__traceback__ if is_sync: cb_suppress = cb(*exc_details) else: @@ -718,21 +753,21 @@ def _fix_exception_context(new_exc, old_exc): if cb_suppress: suppressed_exc = True pending_raise = False - exc_details = (None, None, None) - except: - new_exc_details = sys.exc_info() + exc = None + except BaseException as new_exc: # simulate the stack of exceptions by setting the context - _fix_exception_context(new_exc_details[1], exc_details[1]) + _fix_exception_context(new_exc, exc) pending_raise = True - exc_details = new_exc_details + exc = new_exc + if pending_raise: try: - # bare "raise exc_details[1]" replaces our carefully + # bare "raise exc" replaces our carefully # set-up context - fixed_ctx = exc_details[1].__context__ - raise exc_details[1] + fixed_ctx = exc.__context__ + raise exc except BaseException: - exc_details[1].__context__ = fixed_ctx + exc.__context__ = fixed_ctx raise return received_exc and suppressed_exc diff --git a/Lib/contextvars.py b/Lib/contextvars.py index d78c80dfe6f..14514f185e0 100644 --- a/Lib/contextvars.py +++ b/Lib/contextvars.py @@ -1,4 +1,8 @@ +import _collections_abc from _contextvars import Context, ContextVar, Token, copy_context __all__ = ('Context', 'ContextVar', 'Token', 'copy_context') + + +_collections_abc.Mapping.register(Context) diff --git a/Lib/copy.py b/Lib/copy.py index 1b276afe081..c64fc076179 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -4,8 +4,9 @@ import copy - x = copy.copy(y) # make a shallow copy of y - x = copy.deepcopy(y) # make a deep copy of y + x = copy.copy(y) # make a shallow copy of y + x = copy.deepcopy(y) # make a deep copy of y + x = copy.replace(y, a=1, b=2) # new object with fields replaced, as defined by `__replace__` For module specific errors, copy.Error is raised. @@ -56,12 +57,7 @@ class Error(Exception): pass error = Error # backward compatibility -try: - from org.python.core import PyStringMap -except ImportError: - PyStringMap = None - -__all__ = ["Error", "copy", "deepcopy"] +__all__ = ["Error", "copy", "deepcopy", "replace"] def copy(x): """Shallow copy operation on arbitrary Python objects. @@ -71,13 +67,15 @@ def copy(x): cls = type(x) - copier = _copy_dispatch.get(cls) - if copier: - return copier(x) + if cls in _copy_atomic_types: + return x + if cls in _copy_builtin_containers: + return cls.copy(x) + if issubclass(cls, type): # treat it as a regular class: - return _copy_immutable(x) + return x copier = getattr(cls, "__copy__", None) if copier is not None: @@ -102,28 +100,12 @@ def copy(x): return _reconstruct(x, None, *rv) -_copy_dispatch = d = {} - -def _copy_immutable(x): - return x -for t in (type(None), int, float, bool, complex, str, tuple, +_copy_atomic_types = {types.NoneType, int, float, bool, complex, str, tuple, bytes, frozenset, type, range, slice, property, - types.BuiltinFunctionType, type(Ellipsis), type(NotImplemented), - types.FunctionType, weakref.ref): - d[t] = _copy_immutable -t = getattr(types, "CodeType", None) -if t is not None: - d[t] = _copy_immutable - -d[list] = list.copy -d[dict] = dict.copy -d[set] = set.copy -d[bytearray] = bytearray.copy - -if PyStringMap is not None: - d[PyStringMap] = PyStringMap.copy - -del d, t + types.BuiltinFunctionType, types.EllipsisType, + types.NotImplementedType, types.FunctionType, types.CodeType, + weakref.ref, super} +_copy_builtin_containers = {list, dict, set, bytearray} def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. @@ -131,22 +113,25 @@ def deepcopy(x, memo=None, _nil=[]): See the module's __doc__ string for more info. """ - if memo is None: - memo = {} + cls = type(x) - d = id(x) - y = memo.get(d, _nil) - if y is not _nil: - return y + if cls in _atomic_types: + return x - cls = type(x) + d = id(x) + if memo is None: + memo = {} + else: + y = memo.get(d, _nil) + if y is not _nil: + return y copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): - y = _deepcopy_atomic(x, memo) + y = x # atomic copy else: copier = getattr(x, "__deepcopy__", None) if copier is not None: @@ -177,26 +162,12 @@ def deepcopy(x, memo=None, _nil=[]): _keep_alive(x, memo) # Make sure x lives at least as long as d return y +_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType, + int, float, bool, complex, bytes, str, types.CodeType, type, range, + types.BuiltinFunctionType, types.FunctionType, weakref.ref, property} + _deepcopy_dispatch = d = {} -def _deepcopy_atomic(x, memo): - return x -d[type(None)] = _deepcopy_atomic -d[type(Ellipsis)] = _deepcopy_atomic -d[type(NotImplemented)] = _deepcopy_atomic -d[int] = _deepcopy_atomic -d[float] = _deepcopy_atomic -d[bool] = _deepcopy_atomic -d[complex] = _deepcopy_atomic -d[bytes] = _deepcopy_atomic -d[str] = _deepcopy_atomic -d[types.CodeType] = _deepcopy_atomic -d[type] = _deepcopy_atomic -d[range] = _deepcopy_atomic -d[types.BuiltinFunctionType] = _deepcopy_atomic -d[types.FunctionType] = _deepcopy_atomic -d[weakref.ref] = _deepcopy_atomic -d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] @@ -231,8 +202,6 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy): y[deepcopy(key, memo)] = deepcopy(value, memo) return y d[dict] = _deepcopy_dict -if PyStringMap is not None: - d[PyStringMap] = _deepcopy_dict def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) @@ -301,4 +270,17 @@ def _reconstruct(x, memo, func, args, y[key] = value return y -del types, weakref, PyStringMap +del types, weakref + + +def replace(obj, /, **changes): + """Return a new object replacing specified fields with new values. + + This is especially useful for immutable objects, like named tuples or + frozen dataclasses. + """ + cls = obj.__class__ + func = getattr(cls, '__replace__', None) + if func is None: + raise TypeError(f"replace() does not support {cls.__name__} objects") + return func(obj, **changes) diff --git a/Lib/copyreg.py b/Lib/copyreg.py index dfc463c49a3..a5e8add4a55 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -25,16 +25,21 @@ def constructor(object): # Example: provide pickling support for complex numbers. -try: - complex -except NameError: - pass -else: +def pickle_complex(c): + return complex, (c.real, c.imag) - def pickle_complex(c): - return complex, (c.real, c.imag) +pickle(complex, pickle_complex, complex) - pickle(complex, pickle_complex, complex) +def pickle_union(obj): + import typing, operator + return operator.getitem, (typing.Union, obj.__args__) + +pickle(type(int | str), pickle_union) + +def pickle_super(obj): + return super, (obj.__thisclass__, obj.__self__) + +pickle(super, pickle_super) # Support for pickling new-style objects @@ -48,6 +53,7 @@ def _reconstructor(cls, base, state): return obj _HEAPTYPE = 1<<9 +_new_type = type(int.__new__) # Python code for object.__reduce_ex__ for protocols 0 and 1 @@ -57,6 +63,9 @@ def _reduce_ex(self, proto): for base in cls.__mro__: if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: break + new = base.__new__ + if isinstance(new, _new_type) and new.__self__ is base: + break else: base = object # not really reachable if base is object: @@ -79,6 +88,10 @@ def _reduce_ex(self, proto): except AttributeError: dict = None else: + if (type(self).__getstate__ is object.__getstate__ and + getattr(self, "__slots__", None)): + raise TypeError("a class that defines __slots__ without " + "defining __getstate__ cannot be pickled") dict = getstate() if dict: return _reconstructor, args, dict diff --git a/Lib/csv.py b/Lib/csv.py index 2f38bb1a197..0a627ba7a51 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -1,23 +1,89 @@ -""" -csv.py - read/write/investigate CSV files +r""" +CSV parsing and writing. + +This module provides classes that assist in the reading and writing +of Comma Separated Value (CSV) files, and implements the interface +described by PEP 305. Although many CSV files are simple to parse, +the format is not formally defined by a stable specification and +is subtle enough that parsing lines of a CSV file with something +like line.split(",") is bound to fail. The module supports three +basic APIs: reading, writing, and registration of dialects. + + +DIALECT REGISTRATION: + +Readers and writers support a dialect argument, which is a convenient +handle on a group of settings. When the dialect argument is a string, +it identifies one of the dialects previously registered with the module. +If it is a class or instance, the attributes of the argument are used as +the settings for the reader or writer: + + class excel: + delimiter = ',' + quotechar = '"' + escapechar = None + doublequote = True + skipinitialspace = False + lineterminator = '\r\n' + quoting = QUOTE_MINIMAL + +SETTINGS: + + * quotechar - specifies a one-character string to use as the + quoting character. It defaults to '"'. + * delimiter - specifies a one-character string to use as the + field separator. It defaults to ','. + * skipinitialspace - specifies how to interpret spaces which + immediately follow a delimiter. It defaults to False, which + means that spaces immediately following a delimiter is part + of the following field. + * lineterminator - specifies the character sequence which should + terminate rows. + * quoting - controls when quotes should be generated by the writer. + It can take on any of the following module constants: + + csv.QUOTE_MINIMAL means only when required, for example, when a + field contains either the quotechar or the delimiter + csv.QUOTE_ALL means that quotes are always placed around fields. + csv.QUOTE_NONNUMERIC means that quotes are always placed around + fields which do not parse as integers or floating-point + numbers. + csv.QUOTE_STRINGS means that quotes are always placed around + fields which are strings. Note that the Python value None + is not a string. + csv.QUOTE_NOTNULL means that quotes are only placed around fields + that are not the Python value None. + csv.QUOTE_NONE means that quotes are never placed around fields. + * escapechar - specifies a one-character string used to escape + the delimiter when quoting is set to QUOTE_NONE. + * doublequote - controls the handling of quotes inside fields. When + True, two consecutive quotes are interpreted as one during read, + and when writing, each quote character embedded in the data is + written as two quotes """ -import re -from _csv import Error, writer, reader, \ +import types +from _csv import Error, writer, reader, register_dialect, \ + unregister_dialect, get_dialect, list_dialects, \ + field_size_limit, \ QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \ - __doc__ + QUOTE_STRINGS, QUOTE_NOTNULL +from _csv import Dialect as _Dialect -from collections import OrderedDict from io import StringIO __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", - "Error", "Dialect", "__doc__", "excel", "excel_tab", + "QUOTE_STRINGS", "QUOTE_NOTNULL", + "Error", "Dialect", "excel", "excel_tab", "field_size_limit", "reader", "writer", - "Sniffer", - "unregister_dialect", "__version__", "DictReader", "DictWriter", + "register_dialect", "get_dialect", "list_dialects", "Sniffer", + "unregister_dialect", "DictReader", "DictWriter", "unix_dialect"] +__version__ = "1.0" + + class Dialect: """Describe a CSV dialect. @@ -46,8 +112,8 @@ def _validate(self): try: _Dialect(self) except TypeError as e: - # We do this for compatibility with py2.3 - raise Error(str(e)) + # Re-raise to get a traceback showing more user code. + raise Error(str(e)) from None class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" @@ -57,10 +123,12 @@ class excel(Dialect): skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL +register_dialect("excel", excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' +register_dialect("excel-tab", excel_tab) class unix_dialect(Dialect): """Describe the usual properties of Unix-generated CSV files.""" @@ -70,11 +138,14 @@ class unix_dialect(Dialect): skipinitialspace = False lineterminator = '\n' quoting = QUOTE_ALL +register_dialect("unix", unix_dialect) class DictReader: def __init__(self, f, fieldnames=None, restkey=None, restval=None, dialect="excel", *args, **kwds): + if fieldnames is not None and iter(fieldnames) is fieldnames: + fieldnames = list(fieldnames) self._fieldnames = fieldnames # list of keys for the dict self.restkey = restkey # key to catch long rows self.restval = restval # default value for short rows @@ -111,7 +182,7 @@ def __next__(self): # values while row == []: row = next(self.reader) - d = OrderedDict(zip(self.fieldnames, row)) + d = dict(zip(self.fieldnames, row)) lf = len(self.fieldnames) lr = len(row) if lf < lr: @@ -121,13 +192,18 @@ def __next__(self): d[key] = self.restval return d + __class_getitem__ = classmethod(types.GenericAlias) + class DictWriter: def __init__(self, f, fieldnames, restval="", extrasaction="raise", dialect="excel", *args, **kwds): + if fieldnames is not None and iter(fieldnames) is fieldnames: + fieldnames = list(fieldnames) self.fieldnames = fieldnames # list of keys for the dict self.restval = restval # for writing short dicts - if extrasaction.lower() not in ("raise", "ignore"): + extrasaction = extrasaction.lower() + if extrasaction not in ("raise", "ignore"): raise ValueError("extrasaction (%s) must be 'raise' or 'ignore'" % extrasaction) self.extrasaction = extrasaction @@ -135,7 +211,7 @@ def __init__(self, f, fieldnames, restval="", extrasaction="raise", def writeheader(self): header = dict(zip(self.fieldnames, self.fieldnames)) - self.writerow(header) + return self.writerow(header) def _dict_to_list(self, rowdict): if self.extrasaction == "raise": @@ -151,11 +227,8 @@ def writerow(self, rowdict): def writerows(self, rowdicts): return self.writer.writerows(map(self._dict_to_list, rowdicts)) -# Guard Sniffer's type checking against builds that exclude complex() -try: - complex -except NameError: - complex = float + __class_getitem__ = classmethod(types.GenericAlias) + class Sniffer: ''' @@ -207,6 +280,7 @@ def _guess_quote_and_delimiter(self, data, delimiters): If there is no quotechar the delimiter can't be determined this way. """ + import re matches = [] for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", @@ -404,14 +478,10 @@ def has_header(self, sample): continue # skip rows that have irregular number of columns for col in list(columnTypes.keys()): - - for thisType in [int, float, complex]: - try: - thisType(row[col]) - break - except (ValueError, OverflowError): - pass - else: + thisType = complex + try: + thisType(row[col]) + except (ValueError, OverflowError): # fallback to length of string thisType = len(row[col]) @@ -427,7 +497,7 @@ def has_header(self, sample): # on whether it's a header hasHeader = 0 for col, colType in columnTypes.items(): - if type(colType) == type(0): # it's a length + if isinstance(colType, int): # it's a length if len(header[col]) != colType: hasHeader += 1 else: diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 2e9d4c5e723..04ec0270148 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -1,6 +1,8 @@ """create and manipulate C data types in Python""" -import os as _os, sys as _sys +import os as _os +import sys as _sys +import sysconfig as _sysconfig import types as _types __version__ = "1.1.0" @@ -12,6 +14,7 @@ from _ctypes import RTLD_LOCAL, RTLD_GLOBAL from _ctypes import ArgumentError from _ctypes import SIZEOF_TIME_T +from _ctypes import CField from struct import calcsize as _calcsize @@ -19,7 +22,7 @@ raise Exception("Version number mismatch", __version__, _ctypes_version) if _os.name == "nt": - from _ctypes import FormatError + from _ctypes import COMError, CopyComPointer, FormatError DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": @@ -107,7 +110,7 @@ class CFunctionType(_CFuncPtr): return CFunctionType if _os.name == "nt": - from _ctypes import LoadLibrary as _dlopen + from _ctypes import LoadLibrary as _LoadLibrary from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL _win_functype_cache = {} @@ -161,6 +164,7 @@ def __repr__(self): return super().__repr__() except ValueError: return "%s()" % type(self).__name__ + __class_getitem__ = classmethod(_types.GenericAlias) _check_size(py_object, "P") class c_short(_SimpleCData): @@ -205,6 +209,18 @@ class c_longdouble(_SimpleCData): if sizeof(c_longdouble) == sizeof(c_double): c_longdouble = c_double +try: + class c_double_complex(_SimpleCData): + _type_ = "D" + _check_size(c_double_complex) + class c_float_complex(_SimpleCData): + _type_ = "F" + _check_size(c_float_complex) + class c_longdouble_complex(_SimpleCData): + _type_ = "G" +except AttributeError: + pass + if _calcsize("l") == _calcsize("q"): # if long and long long have the same size, make c_longlong an alias for c_long c_longlong = c_long @@ -252,7 +268,72 @@ class c_void_p(_SimpleCData): class c_bool(_SimpleCData): _type_ = "?" -from _ctypes import POINTER, pointer, _pointer_type_cache +def POINTER(cls): + """Create and return a new ctypes pointer type. + + Pointer types are cached and reused internally, + so calling this function repeatedly is cheap. + """ + if cls is None: + return c_void_p + try: + return cls.__pointer_type__ + except AttributeError: + pass + if isinstance(cls, str): + # handle old-style incomplete types (see test_ctypes.test_incomplete) + import warnings + warnings._deprecated("ctypes.POINTER with string", remove=(3, 19)) + try: + return _pointer_type_cache_fallback[cls] + except KeyError: + result = type(f'LP_{cls}', (_Pointer,), {}) + _pointer_type_cache_fallback[cls] = result + return result + + # create pointer type and set __pointer_type__ for cls + return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls}) + +def pointer(obj): + """Create a new pointer instance, pointing to 'obj'. + + The returned object is of the type POINTER(type(obj)). Note that if you + just want to pass a pointer to an object to a foreign function call, you + should use byref(obj) which is much faster. + """ + typ = POINTER(type(obj)) + return typ(obj) + +class _PointerTypeCache: + def __setitem__(self, cls, pointer_type): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + cls.__pointer_type__ = pointer_type + except AttributeError: + _pointer_type_cache_fallback[cls] = pointer_type + + def __getitem__(self, cls): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + return _pointer_type_cache_fallback[cls] + + def get(self, cls, default=None): + import warnings + warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19)) + try: + return cls.__pointer_type__ + except AttributeError: + return _pointer_type_cache_fallback.get(cls, default) + + def __contains__(self, cls): + return hasattr(cls, '__pointer_type__') + +_pointer_type_cache_fallback = {} +_pointer_type_cache = _PointerTypeCache() class c_wchar_p(_SimpleCData): _type_ = "Z" @@ -263,7 +344,7 @@ class c_wchar(_SimpleCData): _type_ = "u" def _reset_cache(): - _pointer_type_cache.clear() + _pointer_type_cache_fallback.clear() _c_functype_cache.clear() if _os.name == "nt": _win_functype_cache.clear() @@ -271,7 +352,6 @@ def _reset_cache(): POINTER(c_wchar).from_param = c_wchar_p.from_param # _SimpleCData.c_char_p_from_param POINTER(c_char).from_param = c_char_p.from_param - _pointer_type_cache[None] = c_void_p def create_unicode_buffer(init, size=None): """create_unicode_buffer(aString) -> character array @@ -302,17 +382,11 @@ def create_unicode_buffer(init, size=None): raise TypeError(init) -# XXX Deprecated def SetPointerType(pointer, cls): - if _pointer_type_cache.get(cls, None) is not None: - raise RuntimeError("This type already exists in the cache") - if id(pointer) not in _pointer_type_cache: - raise RuntimeError("What's this???") + import warnings + warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) pointer.set_type(cls) - _pointer_type_cache[cls] = pointer - del _pointer_type_cache[id(pointer)] -# XXX Deprecated def ARRAY(typ, len): return typ * len @@ -344,39 +418,59 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None): - self._name = name - flags = self._func_flags_ - if use_errno: - flags |= _FUNCFLAG_USE_ERRNO - if use_last_error: - flags |= _FUNCFLAG_USE_LASTERROR - if _sys.platform.startswith("aix"): - """When the name contains ".a(" and ends with ")", - e.g., "libFOO.a(libFOO.so)" - this is taken to be an - archive(member) syntax for dlopen(), and the mode is adjusted. - Otherwise, name is presented to dlopen() as a file argument. - """ - if name and name.endswith(")") and ".a(" in name: - mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW ) - if _os.name == "nt": - if winmode is not None: - mode = winmode - else: - import nt - mode = nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS - if '/' in name or '\\' in name: - self._name = nt._getfullpathname(self._name) - mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR - class _FuncPtr(_CFuncPtr): - _flags_ = flags + _flags_ = self._func_flags_ _restype_ = self._func_restype_ + if use_errno: + _flags_ |= _FUNCFLAG_USE_ERRNO + if use_last_error: + _flags_ |= _FUNCFLAG_USE_LASTERROR + self._FuncPtr = _FuncPtr + if name: + name = _os.fspath(name) - if handle is None: - self._handle = _dlopen(self._name, mode) - else: - self._handle = handle + self._handle = self._load_library(name, mode, handle, winmode) + + if _os.name == "nt": + def _load_library(self, name, mode, handle, winmode): + if winmode is None: + import nt as _nt + winmode = _nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS + # WINAPI LoadLibrary searches for a DLL if the given name + # is not fully qualified with an explicit drive. For POSIX + # compatibility, and because the DLL search path no longer + # contains the working directory, begin by fully resolving + # any name that contains a path separator. + if name is not None and ('/' in name or '\\' in name): + name = _nt._getfullpathname(name) + winmode |= _nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR + self._name = name + if handle is not None: + return handle + return _LoadLibrary(self._name, winmode) + + else: + def _load_library(self, name, mode, handle, winmode): + # If the filename that has been provided is an iOS/tvOS/watchOS + # .fwork file, dereference the location to the true origin of the + # binary. + if name and name.endswith(".fwork"): + with open(name) as f: + name = _os.path.join( + _os.path.dirname(_sys.executable), + f.read().strip() + ) + if _sys.platform.startswith("aix"): + """When the name contains ".a(" and ends with ")", + e.g., "libFOO.a(libFOO.so)" - this is taken to be an + archive(member) syntax for dlopen(), and the mode is adjusted. + Otherwise, name is presented to dlopen() as a file argument. + """ + if name and name.endswith(")") and ".a(" in name: + mode |= _os.RTLD_MEMBER | _os.RTLD_NOW + self._name = name + return _dlopen(name, mode) def __repr__(self): return "<%s '%s', handle %x at %#x>" % \ @@ -464,8 +558,9 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif _sys.platform == "cygwin": - pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) +elif _sys.platform in ["android", "cygwin"]: + # These are Unix-like platforms which use a dynamically-linked libpython. + pythonapi = PyDLL(_sysconfig.get_config_var("LDLIBRARY")) else: pythonapi = PyDLL(None) @@ -497,6 +592,7 @@ def WinError(code=None, descr=None): # functions from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr +from _ctypes import _memoryview_at_addr ## void *memmove(void *, const void *, size_t); memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) @@ -517,11 +613,19 @@ def cast(obj, typ): _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) def string_at(ptr, size=-1): - """string_at(addr[, size]) -> string + """string_at(ptr[, size]) -> string - Return the string at addr.""" + Return the byte string at void *ptr.""" return _string_at(ptr, size) +_memoryview_at = PYFUNCTYPE( + py_object, c_void_p, c_ssize_t, c_int)(_memoryview_at_addr) +def memoryview_at(ptr, size, readonly=False): + """memoryview_at(ptr, size[, readonly]) -> memoryview + + Return a memoryview representing the memory at void *ptr.""" + return _memoryview_at(ptr, size, bool(readonly)) + try: from _ctypes import _wstring_at_addr except ImportError: @@ -529,9 +633,9 @@ def string_at(ptr, size=-1): else: _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string + """wstring_at(ptr[, size]) -> string - Return the string at addr.""" + Return the wide-character string at void *ptr.""" return _wstring_at(ptr, size) diff --git a/Lib/ctypes/_endian.py b/Lib/ctypes/_endian.py index 34dee64b1a6..6382dd22b8a 100644 --- a/Lib/ctypes/_endian.py +++ b/Lib/ctypes/_endian.py @@ -1,5 +1,5 @@ import sys -from ctypes import * +from ctypes import Array, Structure, Union _array_type = type(Array) @@ -15,8 +15,8 @@ def _other_endian(typ): # if typ is array if isinstance(typ, _array_type): return _other_endian(typ._type_) * typ._length_ - # if typ is structure - if issubclass(typ, Structure): + # if typ is structure or union + if issubclass(typ, (Structure, Union)): return typ raise TypeError("This type does not support other endian: %s" % typ) @@ -37,7 +37,7 @@ class _swapped_union_meta(_swapped_meta, type(Union)): pass ################################################################ # Note: The Structure metaclass checks for the *presence* (not the -# value!) of a _swapped_bytes_ attribute to determine the bit order in +# value!) of a _swappedbytes_ attribute to determine the bit order in # structures containing bit fields. if sys.byteorder == "little": diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py new file mode 100644 index 00000000000..2048ccb6a1c --- /dev/null +++ b/Lib/ctypes/_layout.py @@ -0,0 +1,330 @@ +"""Python implementation of computing the layout of a struct/union + +This code is internal and tightly coupled to the C part. The interface +may change at any time. +""" + +import sys +import warnings + +from _ctypes import CField, buffer_info +import ctypes + +def round_down(n, multiple): + assert n >= 0 + assert multiple > 0 + return (n // multiple) * multiple + +def round_up(n, multiple): + assert n >= 0 + assert multiple > 0 + return ((n + multiple - 1) // multiple) * multiple + +_INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 + + +class StructUnionLayout: + def __init__(self, fields, size, align, format_spec): + # sequence of CField objects + self.fields = fields + + # total size of the aggregate (rounded up to alignment) + self.size = size + + # total alignment requirement of the aggregate + self.align = align + + # buffer format specification (as a string, UTF-8 but bes + # kept ASCII-only) + self.format_spec = format_spec + + +def get_layout(cls, input_fields, is_struct, base): + """Return a StructUnionLayout for the given class. + + Called by PyCStructUnionType_update_stginfo when _fields_ is assigned + to a class. + """ + # Currently there are two modes, selectable using the '_layout_' attribute: + # + # 'gcc-sysv' mode places fields one after another, bit by bit. + # But "each bit field must fit within a single object of its specified + # type" (GCC manual, section 15.8 "Bit Field Packing"). When it doesn't, + # we insert a few bits of padding to avoid that. + # + # 'ms' mode works similar except for bitfield packing. Adjacent + # bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit + # if the integral types are the same size and if the next bit-field fits + # into the current allocation unit without crossing the boundary imposed + # by the common alignment requirements of the bit-fields. + # + # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields + # for details. + + # We do not support zero length bitfields (we use bitsize != 0 + # elsewhere to indicate a bitfield). Here, non-bitfields have bit_size + # set to size*8. + + # For clarity, variables that count bits have `bit` in their names. + + pack = getattr(cls, '_pack_', None) + + layout = getattr(cls, '_layout_', None) + if layout is None: + if sys.platform == 'win32': + gcc_layout = False + elif pack: + if is_struct: + base_type_name = 'Structure' + else: + base_type_name = 'Union' + warnings._deprecated( + '_pack_ without _layout_', + f"Due to '_pack_', the '{cls.__name__}' {base_type_name} will " + + "use memory layout compatible with MSVC (Windows). " + + "If this is intended, set _layout_ to 'ms'. " + + "The implicit default is deprecated and slated to become " + + "an error in Python {remove}.", + remove=(3, 19), + ) + gcc_layout = False + else: + gcc_layout = True + elif layout == 'ms': + gcc_layout = False + elif layout == 'gcc-sysv': + gcc_layout = True + else: + raise ValueError(f'unknown _layout_: {layout!r}') + + align = getattr(cls, '_align_', 1) + if align < 0: + raise ValueError('_align_ must be a non-negative integer') + elif align == 0: + # Setting `_align_ = 0` amounts to using the default alignment + align = 1 + + if base: + align = max(ctypes.alignment(base), align) + + swapped_bytes = hasattr(cls, '_swappedbytes_') + if swapped_bytes: + big_endian = sys.byteorder == 'little' + else: + big_endian = sys.byteorder == 'big' + + if pack is not None: + try: + pack = int(pack) + except (TypeError, ValueError): + raise ValueError("_pack_ must be an integer") + if pack < 0: + raise ValueError("_pack_ must be a non-negative integer") + if pack > _INT_MAX: + raise ValueError("_pack_ too big") + if gcc_layout: + raise ValueError('_pack_ is not compatible with gcc-sysv layout') + + result_fields = [] + + if is_struct: + format_spec_parts = ["T{"] + else: + format_spec_parts = ["B"] + + last_field_bit_size = 0 # used in MS layout only + + # `8 * next_byte_offset + next_bit_offset` points to where the + # next field would start. + next_bit_offset = 0 + next_byte_offset = 0 + + # size if this was a struct (sum of field sizes, plus padding) + struct_size = 0 + # max of field sizes; only meaningful for unions + union_size = 0 + + if base: + struct_size = ctypes.sizeof(base) + if gcc_layout: + next_bit_offset = struct_size * 8 + else: + next_byte_offset = struct_size + + last_size = struct_size + for i, field in enumerate(input_fields): + if not is_struct: + # Unions start fresh each time + last_field_bit_size = 0 + next_bit_offset = 0 + next_byte_offset = 0 + + # Unpack the field + field = tuple(field) + try: + name, ctype = field + except (ValueError, TypeError): + try: + name, ctype, bit_size = field + except (ValueError, TypeError) as exc: + raise ValueError( + '_fields_ must be a sequence of (name, C type) pairs ' + + 'or (name, C type, bit size) triples') from exc + is_bitfield = True + if bit_size <= 0: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + type_size = ctypes.sizeof(ctype) + if bit_size > type_size * 8: + raise ValueError( + f'number of bits invalid for bit field {name!r}') + else: + is_bitfield = False + type_size = ctypes.sizeof(ctype) + bit_size = type_size * 8 + + type_bit_size = type_size * 8 + type_align = ctypes.alignment(ctype) or 1 + type_bit_align = type_align * 8 + + if gcc_layout: + # We don't use next_byte_offset here + assert pack is None + assert next_byte_offset == 0 + + # Determine whether the bit field, if placed at the next + # free bit, fits within a single object of its specified type. + # That is: determine a "slot", sized & aligned for the + # specified type, which contains the bitfield's beginning: + slot_start_bit = round_down(next_bit_offset, type_bit_align) + slot_end_bit = slot_start_bit + type_bit_size + # And see if it also contains the bitfield's last bit: + field_end_bit = next_bit_offset + bit_size + if field_end_bit > slot_end_bit: + # It doesn't: add padding (bump up to the next + # alignment boundary) + next_bit_offset = round_up(next_bit_offset, type_bit_align) + + offset = round_down(next_bit_offset, type_bit_align) // 8 + if is_bitfield: + bit_offset = next_bit_offset - 8 * offset + assert bit_offset <= type_bit_size + else: + assert offset == next_bit_offset / 8 + + next_bit_offset += bit_size + struct_size = round_up(next_bit_offset, 8) // 8 + else: + if pack: + type_align = min(pack, type_align) + + # next_byte_offset points to end of current bitfield. + # next_bit_offset is generally non-positive, + # and 8 * next_byte_offset + next_bit_offset points just behind + # the end of the last field we placed. + if ( + (0 < next_bit_offset + bit_size) + or (type_bit_size != last_field_bit_size) + ): + # Close the previous bitfield (if any) + # and start a new bitfield + next_byte_offset = round_up(next_byte_offset, type_align) + + next_byte_offset += type_size + + last_field_bit_size = type_bit_size + # Reminder: 8 * (next_byte_offset) + next_bit_offset + # points to where we would start a new field, namely + # just behind where we placed the last field plus an + # allowance for alignment. + next_bit_offset = -last_field_bit_size + + assert type_bit_size == last_field_bit_size + + offset = next_byte_offset - last_field_bit_size // 8 + if is_bitfield: + assert 0 <= (last_field_bit_size + next_bit_offset) + bit_offset = last_field_bit_size + next_bit_offset + if type_bit_size: + assert (last_field_bit_size + next_bit_offset) < type_bit_size + + next_bit_offset += bit_size + struct_size = next_byte_offset + + if is_bitfield and big_endian: + # On big-endian architectures, bit fields are also laid out + # starting with the big end. + bit_offset = type_bit_size - bit_size - bit_offset + + # Add the format spec parts + if is_struct: + padding = offset - last_size + format_spec_parts.append(padding_spec(padding)) + + fieldfmt, bf_ndim, bf_shape = buffer_info(ctype) + + if bf_shape: + format_spec_parts.extend(( + "(", + ','.join(str(n) for n in bf_shape), + ")", + )) + + if fieldfmt is None: + fieldfmt = "B" + if isinstance(name, bytes): + # a bytes name would be rejected later, but we check early + # to avoid a BytesWarning with `python -bb` + raise TypeError( + f"field {name!r}: name must be a string, not bytes") + format_spec_parts.append(f"{fieldfmt}:{name}:") + + result_fields.append(CField( + name=name, + type=ctype, + byte_size=type_size, + byte_offset=offset, + bit_size=bit_size if is_bitfield else None, + bit_offset=bit_offset if is_bitfield else None, + index=i, + + # Do not use CField outside ctypes, yet. + # The constructor is internal API and may change without warning. + _internal_use=True, + )) + if is_bitfield and not gcc_layout: + assert type_bit_size > 0 + + align = max(align, type_align) + last_size = struct_size + if not is_struct: + union_size = max(struct_size, union_size) + + if is_struct: + total_size = struct_size + else: + total_size = union_size + + # Adjust the size according to the alignment requirements + aligned_size = round_up(total_size, align) + + # Finish up the format spec + if is_struct: + padding = aligned_size - total_size + format_spec_parts.append(padding_spec(padding)) + format_spec_parts.append("}") + + return StructUnionLayout( + fields=result_fields, + size=aligned_size, + align=align, + format_spec="".join(format_spec_parts), + ) + + +def padding_spec(padding): + if padding <= 0: + return "" + if padding == 1: + return "x" + return f"{padding}x" diff --git a/Lib/ctypes/test/__init__.py b/Lib/ctypes/test/__init__.py deleted file mode 100644 index 6e496fa5a52..00000000000 --- a/Lib/ctypes/test/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import unittest -from test import support -from test.support import import_helper - - -# skip tests if _ctypes was not built -ctypes = import_helper.import_module('ctypes') -ctypes_symbols = dir(ctypes) - -def need_symbol(name): - return unittest.skipUnless(name in ctypes_symbols, - '{!r} is required'.format(name)) - -def load_tests(*args): - return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/ctypes/test/__main__.py b/Lib/ctypes/test/__main__.py deleted file mode 100644 index 362a9ec8cff..00000000000 --- a/Lib/ctypes/test/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from ctypes.test import load_tests -import unittest - -unittest.main() diff --git a/Lib/ctypes/test/test_arrays.py b/Lib/ctypes/test/test_arrays.py deleted file mode 100644 index 14603b7049c..00000000000 --- a/Lib/ctypes/test/test_arrays.py +++ /dev/null @@ -1,238 +0,0 @@ -import unittest -from test.support import bigmemtest, _2G -import sys -from ctypes import * - -from ctypes.test import need_symbol - -formats = "bBhHiIlLqQfd" - -formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ - c_long, c_ulonglong, c_float, c_double, c_longdouble - -class ArrayTestCase(unittest.TestCase): - def test_simple(self): - # create classes holding simple numeric types, and check - # various properties. - - init = list(range(15, 25)) - - for fmt in formats: - alen = len(init) - int_array = ARRAY(fmt, alen) - - ia = int_array(*init) - # length of instance ok? - self.assertEqual(len(ia), alen) - - # slot values ok? - values = [ia[i] for i in range(alen)] - self.assertEqual(values, init) - - # out-of-bounds accesses should be caught - with self.assertRaises(IndexError): ia[alen] - with self.assertRaises(IndexError): ia[-alen-1] - - # change the items - from operator import setitem - new_values = list(range(42, 42+alen)) - [setitem(ia, n, new_values[n]) for n in range(alen)] - values = [ia[i] for i in range(alen)] - self.assertEqual(values, new_values) - - # are the items initialized to 0? - ia = int_array() - values = [ia[i] for i in range(alen)] - self.assertEqual(values, [0] * alen) - - # Too many initializers should be caught - self.assertRaises(IndexError, int_array, *range(alen*2)) - - CharArray = ARRAY(c_char, 3) - - ca = CharArray(b"a", b"b", b"c") - - # Should this work? It doesn't: - # CharArray("abc") - self.assertRaises(TypeError, CharArray, "abc") - - self.assertEqual(ca[0], b"a") - self.assertEqual(ca[1], b"b") - self.assertEqual(ca[2], b"c") - self.assertEqual(ca[-3], b"a") - self.assertEqual(ca[-2], b"b") - self.assertEqual(ca[-1], b"c") - - self.assertEqual(len(ca), 3) - - # cannot delete items - from operator import delitem - self.assertRaises(TypeError, delitem, ca, 0) - - def test_step_overflow(self): - a = (c_int * 5)() - a[3::sys.maxsize] = (1,) - self.assertListEqual(a[3::sys.maxsize], [1]) - a = (c_char * 5)() - a[3::sys.maxsize] = b"A" - self.assertEqual(a[3::sys.maxsize], b"A") - a = (c_wchar * 5)() - a[3::sys.maxsize] = u"X" - self.assertEqual(a[3::sys.maxsize], u"X") - - def test_numeric_arrays(self): - - alen = 5 - - numarray = ARRAY(c_int, alen) - - na = numarray() - values = [na[i] for i in range(alen)] - self.assertEqual(values, [0] * alen) - - na = numarray(*[c_int()] * alen) - values = [na[i] for i in range(alen)] - self.assertEqual(values, [0]*alen) - - na = numarray(1, 2, 3, 4, 5) - values = [i for i in na] - self.assertEqual(values, [1, 2, 3, 4, 5]) - - na = numarray(*map(c_int, (1, 2, 3, 4, 5))) - values = [i for i in na] - self.assertEqual(values, [1, 2, 3, 4, 5]) - - def test_classcache(self): - self.assertIsNot(ARRAY(c_int, 3), ARRAY(c_int, 4)) - self.assertIs(ARRAY(c_int, 3), ARRAY(c_int, 3)) - - def test_from_address(self): - # Failed with 0.9.8, reported by JUrner - p = create_string_buffer(b"foo") - sz = (c_char * 3).from_address(addressof(p)) - self.assertEqual(sz[:], b"foo") - self.assertEqual(sz[::], b"foo") - self.assertEqual(sz[::-1], b"oof") - self.assertEqual(sz[::3], b"f") - self.assertEqual(sz[1:4:2], b"o") - self.assertEqual(sz.value, b"foo") - - @need_symbol('create_unicode_buffer') - def test_from_addressW(self): - p = create_unicode_buffer("foo") - sz = (c_wchar * 3).from_address(addressof(p)) - self.assertEqual(sz[:], "foo") - self.assertEqual(sz[::], "foo") - self.assertEqual(sz[::-1], "oof") - self.assertEqual(sz[::3], "f") - self.assertEqual(sz[1:4:2], "o") - self.assertEqual(sz.value, "foo") - - def test_cache(self): - # Array types are cached internally in the _ctypes extension, - # in a WeakValueDictionary. Make sure the array type is - # removed from the cache when the itemtype goes away. This - # test will not fail, but will show a leak in the testsuite. - - # Create a new type: - class my_int(c_int): - pass - # Create a new array type based on it: - t1 = my_int * 1 - t2 = my_int * 1 - self.assertIs(t1, t2) - - def test_subclass(self): - class T(Array): - _type_ = c_int - _length_ = 13 - class U(T): - pass - class V(U): - pass - class W(V): - pass - class X(T): - _type_ = c_short - class Y(T): - _length_ = 187 - - for c in [T, U, V, W]: - self.assertEqual(c._type_, c_int) - self.assertEqual(c._length_, 13) - self.assertEqual(c()._type_, c_int) - self.assertEqual(c()._length_, 13) - - self.assertEqual(X._type_, c_short) - self.assertEqual(X._length_, 13) - self.assertEqual(X()._type_, c_short) - self.assertEqual(X()._length_, 13) - - self.assertEqual(Y._type_, c_int) - self.assertEqual(Y._length_, 187) - self.assertEqual(Y()._type_, c_int) - self.assertEqual(Y()._length_, 187) - - def test_bad_subclass(self): - with self.assertRaises(AttributeError): - class T(Array): - pass - with self.assertRaises(AttributeError): - class T(Array): - _type_ = c_int - with self.assertRaises(AttributeError): - class T(Array): - _length_ = 13 - - def test_bad_length(self): - with self.assertRaises(ValueError): - class T(Array): - _type_ = c_int - _length_ = - sys.maxsize * 2 - with self.assertRaises(ValueError): - class T(Array): - _type_ = c_int - _length_ = -1 - with self.assertRaises(TypeError): - class T(Array): - _type_ = c_int - _length_ = 1.87 - with self.assertRaises(OverflowError): - class T(Array): - _type_ = c_int - _length_ = sys.maxsize * 2 - - def test_zero_length(self): - # _length_ can be zero. - class T(Array): - _type_ = c_int - _length_ = 0 - - def test_empty_element_struct(self): - class EmptyStruct(Structure): - _fields_ = [] - - obj = (EmptyStruct * 2)() # bpo37188: Floating point exception - self.assertEqual(sizeof(obj), 0) - - def test_empty_element_array(self): - class EmptyArray(Array): - _type_ = c_int - _length_ = 0 - - obj = (EmptyArray * 2)() # bpo37188: Floating point exception - self.assertEqual(sizeof(obj), 0) - - def test_bpo36504_signed_int_overflow(self): - # The overflow check in PyCArrayType_new() could cause signed integer - # overflow. - with self.assertRaises(OverflowError): - c_char * sys.maxsize * 2 - - @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') - @bigmemtest(size=_2G, memuse=1, dry_run=False) - def test_large_array(self, size): - c_char * size - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_bitfields.py b/Lib/ctypes/test/test_bitfields.py deleted file mode 100644 index 66acd62e685..00000000000 --- a/Lib/ctypes/test/test_bitfields.py +++ /dev/null @@ -1,297 +0,0 @@ -from ctypes import * -from ctypes.test import need_symbol -from test import support -import unittest -import os - -import _ctypes_test - -class BITS(Structure): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] - -func = CDLL(_ctypes_test.__file__).unpack_bitfields -func.argtypes = POINTER(BITS), c_char - -##for n in "ABCDEFGHIMNOPQRS": -## print n, hex(getattr(BITS, n).size), getattr(BITS, n).offset - -class C_Test(unittest.TestCase): - - def test_ints(self): - for i in range(512): - for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) - - # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior - @support.skip_if_sanitizer(ub=True) - def test_shorts(self): - b = BITS() - name = "M" - if func(byref(b), name.encode('ascii')) == 999: - self.skipTest("Compiler does not support signed short bitfields") - for i in range(256): - for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) - -signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) -unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) -int_types = unsigned_int_types + signed_int_types - -class BitFieldTest(unittest.TestCase): - - def test_longlong(self): - class X(Structure): - _fields_ = [("a", c_longlong, 1), - ("b", c_longlong, 62), - ("c", c_longlong, 1)] - - self.assertEqual(sizeof(X), sizeof(c_longlong)) - x = X() - x.a, x.b, x.c = -1, 7, -1 - self.assertEqual((x.a, x.b, x.c), (-1, 7, -1)) - - def test_ulonglong(self): - class X(Structure): - _fields_ = [("a", c_ulonglong, 1), - ("b", c_ulonglong, 62), - ("c", c_ulonglong, 1)] - - self.assertEqual(sizeof(X), sizeof(c_longlong)) - x = X() - self.assertEqual((x.a, x.b, x.c), (0, 0, 0)) - x.a, x.b, x.c = 7, 7, 7 - self.assertEqual((x.a, x.b, x.c), (1, 7, 1)) - - def test_signed(self): - for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)*2) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) - - - def test_unsigned(self): - for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) - - - def fail_fields(self, *fields): - return self.get_except(type(Structure), "X", (), - {"_fields_": fields}) - - def test_nonint_types(self): - # bit fields are not allowed on non-integer types. - result = self.fail_fields(("a", c_char_p, 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char_p')) - - result = self.fail_fields(("a", c_void_p, 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_void_p')) - - if c_int != c_long: - result = self.fail_fields(("a", POINTER(c_int), 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type LP_c_int')) - - result = self.fail_fields(("a", c_char, 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) - - class Dummy(Structure): - _fields_ = [] - - result = self.fail_fields(("a", Dummy, 1)) - self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) - - @need_symbol('c_wchar') - def test_c_wchar(self): - result = self.fail_fields(("a", c_wchar, 1)) - self.assertEqual(result, - (TypeError, 'bit fields not allowed for type c_wchar')) - - def test_single_bitfield_size(self): - for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) - - result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) - - class X(Structure): - _fields_ = [("a", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) - - def test_multi_bitfields_size(self): - class X(Structure): - _fields_ = [("a", c_short, 1), - ("b", c_short, 14), - ("c", c_short, 1)] - self.assertEqual(sizeof(X), sizeof(c_short)) - - class X(Structure): - _fields_ = [("a", c_short, 1), - ("a1", c_short), - ("b", c_short, 14), - ("c", c_short, 1)] - self.assertEqual(sizeof(X), sizeof(c_short)*3) - self.assertEqual(X.a.offset, 0) - self.assertEqual(X.a1.offset, sizeof(c_short)) - self.assertEqual(X.b.offset, sizeof(c_short)*2) - self.assertEqual(X.c.offset, sizeof(c_short)*2) - - class X(Structure): - _fields_ = [("a", c_short, 3), - ("b", c_short, 14), - ("c", c_short, 14)] - self.assertEqual(sizeof(X), sizeof(c_short)*3) - self.assertEqual(X.a.offset, sizeof(c_short)*0) - self.assertEqual(X.b.offset, sizeof(c_short)*1) - self.assertEqual(X.c.offset, sizeof(c_short)*2) - - - def get_except(self, func, *args, **kw): - try: - func(*args, **kw) - except Exception as detail: - return detail.__class__, str(detail) - - def test_mixed_1(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 4)] - if os.name == "nt": - self.assertEqual(sizeof(X), sizeof(c_int)*2) - else: - self.assertEqual(sizeof(X), sizeof(c_int)) - - def test_mixed_2(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_int, 32)] - self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int)) - - def test_mixed_3(self): - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - self.assertEqual(sizeof(X), sizeof(c_byte)) - - def test_mixed_4(self): - class X(Structure): - _fields_ = [("a", c_short, 4), - ("b", c_short, 4), - ("c", c_int, 24), - ("d", c_short, 4), - ("e", c_short, 4), - ("f", c_int, 24)] - # MSVC does NOT combine c_short and c_int into one field, GCC - # does (unless GCC is run with '-mms-bitfields' which - # produces code compatible with MSVC). - if os.name == "nt": - self.assertEqual(sizeof(X), sizeof(c_int) * 4) - else: - self.assertEqual(sizeof(X), sizeof(c_int) * 2) - - def test_anon_bitfields(self): - # anonymous bit-fields gave a strange error message - class X(Structure): - _fields_ = [("a", c_byte, 4), - ("b", c_ubyte, 4)] - class Y(Structure): - _anonymous_ = ["_"] - _fields_ = [("_", X)] - - @need_symbol('c_uint32') - def test_uint32(self): - class X(Structure): - _fields_ = [("a", c_uint32, 32)] - x = X() - x.a = 10 - self.assertEqual(x.a, 10) - x.a = 0xFDCBA987 - self.assertEqual(x.a, 0xFDCBA987) - - @need_symbol('c_uint64') - def test_uint64(self): - class X(Structure): - _fields_ = [("a", c_uint64, 64)] - x = X() - x.a = 10 - self.assertEqual(x.a, 10) - x.a = 0xFEDCBA9876543211 - self.assertEqual(x.a, 0xFEDCBA9876543211) - - @need_symbol('c_uint32') - def test_uint32_swap_little_endian(self): - # Issue #23319 - class Little(LittleEndianStructure): - _fields_ = [("a", c_uint32, 24), - ("b", c_uint32, 4), - ("c", c_uint32, 4)] - b = bytearray(4) - x = Little.from_buffer(b) - x.a = 0xabcdef - x.b = 1 - x.c = 2 - self.assertEqual(b, b'\xef\xcd\xab\x21') - - @need_symbol('c_uint32') - def test_uint32_swap_big_endian(self): - # Issue #23319 - class Big(BigEndianStructure): - _fields_ = [("a", c_uint32, 24), - ("b", c_uint32, 4), - ("c", c_uint32, 4)] - b = bytearray(4) - x = Big.from_buffer(b) - x.a = 0xabcdef - x.b = 1 - x.c = 2 - self.assertEqual(b, b'\xab\xcd\xef\x12') - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_checkretval.py b/Lib/ctypes/test/test_checkretval.py deleted file mode 100644 index e9567dc3912..00000000000 --- a/Lib/ctypes/test/test_checkretval.py +++ /dev/null @@ -1,36 +0,0 @@ -import unittest - -from ctypes import * -from ctypes.test import need_symbol - -class CHECKED(c_int): - def _check_retval_(value): - # Receives a CHECKED instance. - return str(value.value) - _check_retval_ = staticmethod(_check_retval_) - -class Test(unittest.TestCase): - - def test_checkretval(self): - - import _ctypes_test - dll = CDLL(_ctypes_test.__file__) - self.assertEqual(42, dll._testfunc_p_p(42)) - - dll._testfunc_p_p.restype = CHECKED - self.assertEqual("42", dll._testfunc_p_p(42)) - - dll._testfunc_p_p.restype = None - self.assertEqual(None, dll._testfunc_p_p(42)) - - del dll._testfunc_p_p.restype - self.assertEqual(42, dll._testfunc_p_p(42)) - - @need_symbol('oledll') - def test_oledll(self): - self.assertRaises(OSError, - oledll.oleaut32.CreateTypeLib2, - 0, None, None) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_delattr.py b/Lib/ctypes/test/test_delattr.py deleted file mode 100644 index 0f4d58691b5..00000000000 --- a/Lib/ctypes/test/test_delattr.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest -from ctypes import * - -class X(Structure): - _fields_ = [("foo", c_int)] - -class TestCase(unittest.TestCase): - def test_simple(self): - self.assertRaises(TypeError, - delattr, c_int(42), "value") - - def test_chararray(self): - self.assertRaises(TypeError, - delattr, (c_char * 5)(), "value") - - def test_struct(self): - self.assertRaises(TypeError, - delattr, X(), "foo") - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_errno.py b/Lib/ctypes/test/test_errno.py deleted file mode 100644 index 3685164dde6..00000000000 --- a/Lib/ctypes/test/test_errno.py +++ /dev/null @@ -1,76 +0,0 @@ -import unittest, os, errno -import threading - -from ctypes import * -from ctypes.util import find_library - -class Test(unittest.TestCase): - def test_open(self): - libc_name = find_library("c") - if libc_name is None: - raise unittest.SkipTest("Unable to find C library") - libc = CDLL(libc_name, use_errno=True) - if os.name == "nt": - libc_open = libc._open - else: - libc_open = libc.open - - libc_open.argtypes = c_char_p, c_int - - self.assertEqual(libc_open(b"", 0), -1) - self.assertEqual(get_errno(), errno.ENOENT) - - self.assertEqual(set_errno(32), errno.ENOENT) - self.assertEqual(get_errno(), 32) - - def _worker(): - set_errno(0) - - libc = CDLL(libc_name, use_errno=False) - if os.name == "nt": - libc_open = libc._open - else: - libc_open = libc.open - libc_open.argtypes = c_char_p, c_int - self.assertEqual(libc_open(b"", 0), -1) - self.assertEqual(get_errno(), 0) - - t = threading.Thread(target=_worker) - t.start() - t.join() - - self.assertEqual(get_errno(), 32) - set_errno(0) - - @unittest.skipUnless(os.name == "nt", 'Test specific to Windows') - def test_GetLastError(self): - dll = WinDLL("kernel32", use_last_error=True) - GetModuleHandle = dll.GetModuleHandleA - GetModuleHandle.argtypes = [c_wchar_p] - - self.assertEqual(0, GetModuleHandle("foo")) - self.assertEqual(get_last_error(), 126) - - self.assertEqual(set_last_error(32), 126) - self.assertEqual(get_last_error(), 32) - - def _worker(): - set_last_error(0) - - dll = WinDLL("kernel32", use_last_error=False) - GetModuleHandle = dll.GetModuleHandleW - GetModuleHandle.argtypes = [c_wchar_p] - GetModuleHandle("bar") - - self.assertEqual(get_last_error(), 0) - - t = threading.Thread(target=_worker) - t.start() - t.join() - - self.assertEqual(get_last_error(), 32) - - set_last_error(0) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py deleted file mode 100644 index 1ff9d019b13..00000000000 --- a/Lib/ctypes/test/test_find.py +++ /dev/null @@ -1,127 +0,0 @@ -import unittest -import unittest.mock -import os.path -import sys -import test.support -from test.support import os_helper -from ctypes import * -from ctypes.util import find_library - -# On some systems, loading the OpenGL libraries needs the RTLD_GLOBAL mode. -class Test_OpenGL_libs(unittest.TestCase): - @classmethod - def setUpClass(cls): - lib_gl = lib_glu = lib_gle = None - if sys.platform == "win32": - lib_gl = find_library("OpenGL32") - lib_glu = find_library("Glu32") - elif sys.platform == "darwin": - lib_gl = lib_glu = find_library("OpenGL") - else: - lib_gl = find_library("GL") - lib_glu = find_library("GLU") - lib_gle = find_library("gle") - - ## print, for debugging - if test.support.verbose: - print("OpenGL libraries:") - for item in (("GL", lib_gl), - ("GLU", lib_glu), - ("gle", lib_gle)): - print("\t", item) - - cls.gl = cls.glu = cls.gle = None - if lib_gl: - try: - cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) - except OSError: - pass - if lib_glu: - try: - cls.glu = CDLL(lib_glu, RTLD_GLOBAL) - except OSError: - pass - if lib_gle: - try: - cls.gle = CDLL(lib_gle) - except OSError: - pass - - @classmethod - def tearDownClass(cls): - cls.gl = cls.glu = cls.gle = None - - def test_gl(self): - if self.gl is None: - self.skipTest('lib_gl not available') - self.gl.glClearIndex - - def test_glu(self): - if self.glu is None: - self.skipTest('lib_glu not available') - self.glu.gluBeginCurve - - def test_gle(self): - if self.gle is None: - self.skipTest('lib_gle not available') - self.gle.gleGetJoinStyle - - def test_shell_injection(self): - result = find_library('; echo Hello shell > ' + os_helper.TESTFN) - self.assertFalse(os.path.lexists(os_helper.TESTFN)) - self.assertIsNone(result) - - -@unittest.skipUnless(sys.platform.startswith('linux'), - 'Test only valid for Linux') -class FindLibraryLinux(unittest.TestCase): - def test_find_on_libpath(self): - import subprocess - import tempfile - - try: - p = subprocess.Popen(['gcc', '--version'], stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) - out, _ = p.communicate() - except OSError: - raise unittest.SkipTest('gcc, needed for test, not available') - with tempfile.TemporaryDirectory() as d: - # create an empty temporary file - srcname = os.path.join(d, 'dummy.c') - libname = 'py_ctypes_test_dummy' - dstname = os.path.join(d, 'lib%s.so' % libname) - with open(srcname, 'wb') as f: - pass - self.assertTrue(os.path.exists(srcname)) - # compile the file to a shared library - cmd = ['gcc', '-o', dstname, '--shared', - '-Wl,-soname,lib%s.so' % libname, srcname] - out = subprocess.check_output(cmd) - self.assertTrue(os.path.exists(dstname)) - # now check that the .so can't be found (since not in - # LD_LIBRARY_PATH) - self.assertIsNone(find_library(libname)) - # now add the location to LD_LIBRARY_PATH - with os_helper.EnvironmentVarGuard() as env: - KEY = 'LD_LIBRARY_PATH' - if KEY not in env: - v = d - else: - v = '%s:%s' % (env[KEY], d) - env.set(KEY, v) - # now check that the .so can be found (since in - # LD_LIBRARY_PATH) - self.assertEqual(find_library(libname), 'lib%s.so' % libname) - - def test_find_library_with_gcc(self): - with unittest.mock.patch("ctypes.util._findSoname_ldconfig", lambda *args: None): - self.assertNotEqual(find_library('c'), None) - - def test_find_library_with_ld(self): - with unittest.mock.patch("ctypes.util._findSoname_ldconfig", lambda *args: None), \ - unittest.mock.patch("ctypes.util._findLib_gcc", lambda *args: None): - self.assertNotEqual(find_library('c'), None) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_funcptr.py b/Lib/ctypes/test/test_funcptr.py deleted file mode 100644 index e0b9b54e97f..00000000000 --- a/Lib/ctypes/test/test_funcptr.py +++ /dev/null @@ -1,132 +0,0 @@ -import unittest -from ctypes import * - -try: - WINFUNCTYPE -except NameError: - # fake to enable this test on Linux - WINFUNCTYPE = CFUNCTYPE - -import _ctypes_test -lib = CDLL(_ctypes_test.__file__) - -class CFuncPtrTestCase(unittest.TestCase): - def test_basic(self): - X = WINFUNCTYPE(c_int, c_int, c_int) - - def func(*args): - return len(args) - - x = X(func) - self.assertEqual(x.restype, c_int) - self.assertEqual(x.argtypes, (c_int, c_int)) - self.assertEqual(sizeof(x), sizeof(c_voidp)) - self.assertEqual(sizeof(X), sizeof(c_voidp)) - - def test_first(self): - StdCallback = WINFUNCTYPE(c_int, c_int, c_int) - CdeclCallback = CFUNCTYPE(c_int, c_int, c_int) - - def func(a, b): - return a + b - - s = StdCallback(func) - c = CdeclCallback(func) - - self.assertEqual(s(1, 2), 3) - self.assertEqual(c(1, 2), 3) - # The following no longer raises a TypeError - it is now - # possible, as in C, to call cdecl functions with more parameters. - #self.assertRaises(TypeError, c, 1, 2, 3) - self.assertEqual(c(1, 2, 3, 4, 5, 6), 3) - if not WINFUNCTYPE is CFUNCTYPE: - self.assertRaises(TypeError, s, 1, 2, 3) - - def test_structures(self): - WNDPROC = WINFUNCTYPE(c_long, c_int, c_int, c_int, c_int) - - def wndproc(hwnd, msg, wParam, lParam): - return hwnd + msg + wParam + lParam - - HINSTANCE = c_int - HICON = c_int - HCURSOR = c_int - LPCTSTR = c_char_p - - class WNDCLASS(Structure): - _fields_ = [("style", c_uint), - ("lpfnWndProc", WNDPROC), - ("cbClsExtra", c_int), - ("cbWndExtra", c_int), - ("hInstance", HINSTANCE), - ("hIcon", HICON), - ("hCursor", HCURSOR), - ("lpszMenuName", LPCTSTR), - ("lpszClassName", LPCTSTR)] - - wndclass = WNDCLASS() - wndclass.lpfnWndProc = WNDPROC(wndproc) - - WNDPROC_2 = WINFUNCTYPE(c_long, c_int, c_int, c_int, c_int) - - # This is no longer true, now that WINFUNCTYPE caches created types internally. - ## # CFuncPtr subclasses are compared by identity, so this raises a TypeError: - ## self.assertRaises(TypeError, setattr, wndclass, - ## "lpfnWndProc", WNDPROC_2(wndproc)) - # instead: - - self.assertIs(WNDPROC, WNDPROC_2) - # 'wndclass.lpfnWndProc' leaks 94 references. Why? - self.assertEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10) - - - f = wndclass.lpfnWndProc - - del wndclass - del wndproc - - self.assertEqual(f(10, 11, 12, 13), 46) - - def test_dllfunctions(self): - - def NoNullHandle(value): - if not value: - raise WinError() - return value - - strchr = lib.my_strchr - strchr.restype = c_char_p - strchr.argtypes = (c_char_p, c_char) - self.assertEqual(strchr(b"abcdefghi", b"b"), b"bcdefghi") - self.assertEqual(strchr(b"abcdefghi", b"x"), None) - - - strtok = lib.my_strtok - strtok.restype = c_char_p - # Neither of this does work: strtok changes the buffer it is passed -## strtok.argtypes = (c_char_p, c_char_p) -## strtok.argtypes = (c_string, c_char_p) - - def c_string(init): - size = len(init) + 1 - return (c_char*size)(*init) - - s = b"a\nb\nc" - b = c_string(s) - -## b = (c_char * (len(s)+1))() -## b.value = s - -## b = c_string(s) - self.assertEqual(strtok(b, b"\n"), b"a") - self.assertEqual(strtok(None, b"\n"), b"b") - self.assertEqual(strtok(None, b"\n"), b"c") - self.assertEqual(strtok(None, b"\n"), None) - - def test_abstract(self): - from ctypes import _CFuncPtr - - self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid") - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_functions.py b/Lib/ctypes/test/test_functions.py deleted file mode 100644 index fc571700ce3..00000000000 --- a/Lib/ctypes/test/test_functions.py +++ /dev/null @@ -1,384 +0,0 @@ -""" -Here is probably the place to write the docs, since the test-cases -show how the type behave. - -Later... -""" - -from ctypes import * -from ctypes.test import need_symbol -import sys, unittest - -try: - WINFUNCTYPE -except NameError: - # fake to enable this test on Linux - WINFUNCTYPE = CFUNCTYPE - -import _ctypes_test -dll = CDLL(_ctypes_test.__file__) -if sys.platform == "win32": - windll = WinDLL(_ctypes_test.__file__) - -class POINT(Structure): - _fields_ = [("x", c_int), ("y", c_int)] -class RECT(Structure): - _fields_ = [("left", c_int), ("top", c_int), - ("right", c_int), ("bottom", c_int)] -class FunctionTestCase(unittest.TestCase): - - def test_mro(self): - # in Python 2.3, this raises TypeError: MRO conflict among bases classes, - # in Python 2.2 it works. - # - # But in early versions of _ctypes.c, the result of tp_new - # wasn't checked, and it even crashed Python. - # Found by Greg Chapman. - - with self.assertRaises(TypeError): - class X(object, Array): - _length_ = 5 - _type_ = "i" - - from _ctypes import _Pointer - with self.assertRaises(TypeError): - class X(object, _Pointer): - pass - - from _ctypes import _SimpleCData - with self.assertRaises(TypeError): - class X(object, _SimpleCData): - _type_ = "i" - - with self.assertRaises(TypeError): - class X(object, Structure): - _fields_ = [] - - @need_symbol('c_wchar') - def test_wchar_parm(self): - f = dll._testfunc_i_bhilfd - f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] - result = f(1, "x", 3, 4, 5.0, 6.0) - self.assertEqual(result, 139) - self.assertEqual(type(result), int) - - @need_symbol('c_wchar') - def test_wchar_result(self): - f = dll._testfunc_i_bhilfd - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - f.restype = c_wchar - result = f(0, 0, 0, 0, 0, 0) - self.assertEqual(result, '\x00') - - def test_voidresult(self): - f = dll._testfunc_v - f.restype = None - f.argtypes = [c_int, c_int, POINTER(c_int)] - result = c_int() - self.assertEqual(None, f(1, 2, byref(result))) - self.assertEqual(result.value, 3) - - def test_intresult(self): - f = dll._testfunc_i_bhilfd - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - f.restype = c_int - result = f(1, 2, 3, 4, 5.0, 6.0) - self.assertEqual(result, 21) - self.assertEqual(type(result), int) - - result = f(-1, -2, -3, -4, -5.0, -6.0) - self.assertEqual(result, -21) - self.assertEqual(type(result), int) - - # If we declare the function to return a short, - # is the high part split off? - f.restype = c_short - result = f(1, 2, 3, 4, 5.0, 6.0) - self.assertEqual(result, 21) - self.assertEqual(type(result), int) - - result = f(1, 2, 3, 0x10004, 5.0, 6.0) - self.assertEqual(result, 21) - self.assertEqual(type(result), int) - - # You cannot assign character format codes as restype any longer - self.assertRaises(TypeError, setattr, f, "restype", "i") - - def test_floatresult(self): - f = dll._testfunc_f_bhilfd - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - f.restype = c_float - result = f(1, 2, 3, 4, 5.0, 6.0) - self.assertEqual(result, 21) - self.assertEqual(type(result), float) - - result = f(-1, -2, -3, -4, -5.0, -6.0) - self.assertEqual(result, -21) - self.assertEqual(type(result), float) - - def test_doubleresult(self): - f = dll._testfunc_d_bhilfd - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - f.restype = c_double - result = f(1, 2, 3, 4, 5.0, 6.0) - self.assertEqual(result, 21) - self.assertEqual(type(result), float) - - result = f(-1, -2, -3, -4, -5.0, -6.0) - self.assertEqual(result, -21) - self.assertEqual(type(result), float) - - @need_symbol('c_longdouble') - def test_longdoubleresult(self): - f = dll._testfunc_D_bhilfD - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble] - f.restype = c_longdouble - result = f(1, 2, 3, 4, 5.0, 6.0) - self.assertEqual(result, 21) - self.assertEqual(type(result), float) - - result = f(-1, -2, -3, -4, -5.0, -6.0) - self.assertEqual(result, -21) - self.assertEqual(type(result), float) - - @need_symbol('c_longlong') - def test_longlongresult(self): - f = dll._testfunc_q_bhilfd - f.restype = c_longlong - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] - result = f(1, 2, 3, 4, 5.0, 6.0) - self.assertEqual(result, 21) - - f = dll._testfunc_q_bhilfdq - f.restype = c_longlong - f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double, c_longlong] - result = f(1, 2, 3, 4, 5.0, 6.0, 21) - self.assertEqual(result, 42) - - def test_stringresult(self): - f = dll._testfunc_p_p - f.argtypes = None - f.restype = c_char_p - result = f(b"123") - self.assertEqual(result, b"123") - - result = f(None) - self.assertEqual(result, None) - - def test_pointers(self): - f = dll._testfunc_p_p - f.restype = POINTER(c_int) - f.argtypes = [POINTER(c_int)] - - # This only works if the value c_int(42) passed to the - # function is still alive while the pointer (the result) is - # used. - - v = c_int(42) - - self.assertEqual(pointer(v).contents.value, 42) - result = f(pointer(v)) - self.assertEqual(type(result), POINTER(c_int)) - self.assertEqual(result.contents.value, 42) - - # This on works... - result = f(pointer(v)) - self.assertEqual(result.contents.value, v.value) - - p = pointer(c_int(99)) - result = f(p) - self.assertEqual(result.contents.value, 99) - - arg = byref(v) - result = f(arg) - self.assertNotEqual(result.contents, v.value) - - self.assertRaises(ArgumentError, f, byref(c_short(22))) - - # It is dangerous, however, because you don't control the lifetime - # of the pointer: - result = f(byref(c_int(99))) - self.assertNotEqual(result.contents, 99) - - ################################################################ - def test_shorts(self): - f = dll._testfunc_callback_i_if - - args = [] - expected = [262144, 131072, 65536, 32768, 16384, 8192, 4096, 2048, - 1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1] - - def callback(v): - args.append(v) - return v - - CallBack = CFUNCTYPE(c_int, c_int) - - cb = CallBack(callback) - f(2**18, cb) - self.assertEqual(args, expected) - - ################################################################ - - - def test_callbacks(self): - f = dll._testfunc_callback_i_if - f.restype = c_int - f.argtypes = None - - MyCallback = CFUNCTYPE(c_int, c_int) - - def callback(value): - #print "called back with", value - return value - - cb = MyCallback(callback) - result = f(-10, cb) - self.assertEqual(result, -18) - - # test with prototype - f.argtypes = [c_int, MyCallback] - cb = MyCallback(callback) - result = f(-10, cb) - self.assertEqual(result, -18) - - AnotherCallback = WINFUNCTYPE(c_int, c_int, c_int, c_int, c_int) - - # check that the prototype works: we call f with wrong - # argument types - cb = AnotherCallback(callback) - self.assertRaises(ArgumentError, f, -10, cb) - - - def test_callbacks_2(self): - # Can also use simple datatypes as argument type specifiers - # for the callback function. - # In this case the call receives an instance of that type - f = dll._testfunc_callback_i_if - f.restype = c_int - - MyCallback = CFUNCTYPE(c_int, c_int) - - f.argtypes = [c_int, MyCallback] - - def callback(value): - #print "called back with", value - self.assertEqual(type(value), int) - return value - - cb = MyCallback(callback) - result = f(-10, cb) - self.assertEqual(result, -18) - - @need_symbol('c_longlong') - def test_longlong_callbacks(self): - - f = dll._testfunc_callback_q_qf - f.restype = c_longlong - - MyCallback = CFUNCTYPE(c_longlong, c_longlong) - - f.argtypes = [c_longlong, MyCallback] - - def callback(value): - self.assertIsInstance(value, int) - return value & 0x7FFFFFFF - - cb = MyCallback(callback) - - self.assertEqual(13577625587, f(1000000000000, cb)) - - def test_errors(self): - self.assertRaises(AttributeError, getattr, dll, "_xxx_yyy") - self.assertRaises(ValueError, c_int.in_dll, dll, "_xxx_yyy") - - def test_byval(self): - - # without prototype - ptin = POINT(1, 2) - ptout = POINT() - # EXPORT int _testfunc_byval(point in, point *pout) - result = dll._testfunc_byval(ptin, byref(ptout)) - got = result, ptout.x, ptout.y - expected = 3, 1, 2 - self.assertEqual(got, expected) - - # with prototype - ptin = POINT(101, 102) - ptout = POINT() - dll._testfunc_byval.argtypes = (POINT, POINTER(POINT)) - dll._testfunc_byval.restype = c_int - result = dll._testfunc_byval(ptin, byref(ptout)) - got = result, ptout.x, ptout.y - expected = 203, 101, 102 - self.assertEqual(got, expected) - - def test_struct_return_2H(self): - class S2H(Structure): - _fields_ = [("x", c_short), - ("y", c_short)] - dll.ret_2h_func.restype = S2H - dll.ret_2h_func.argtypes = [S2H] - inp = S2H(99, 88) - s2h = dll.ret_2h_func(inp) - self.assertEqual((s2h.x, s2h.y), (99*2, 88*3)) - - @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') - def test_struct_return_2H_stdcall(self): - class S2H(Structure): - _fields_ = [("x", c_short), - ("y", c_short)] - - windll.s_ret_2h_func.restype = S2H - windll.s_ret_2h_func.argtypes = [S2H] - s2h = windll.s_ret_2h_func(S2H(99, 88)) - self.assertEqual((s2h.x, s2h.y), (99*2, 88*3)) - - def test_struct_return_8H(self): - class S8I(Structure): - _fields_ = [("a", c_int), - ("b", c_int), - ("c", c_int), - ("d", c_int), - ("e", c_int), - ("f", c_int), - ("g", c_int), - ("h", c_int)] - dll.ret_8i_func.restype = S8I - dll.ret_8i_func.argtypes = [S8I] - inp = S8I(9, 8, 7, 6, 5, 4, 3, 2) - s8i = dll.ret_8i_func(inp) - self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), - (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) - - @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') - def test_struct_return_8H_stdcall(self): - class S8I(Structure): - _fields_ = [("a", c_int), - ("b", c_int), - ("c", c_int), - ("d", c_int), - ("e", c_int), - ("f", c_int), - ("g", c_int), - ("h", c_int)] - windll.s_ret_8i_func.restype = S8I - windll.s_ret_8i_func.argtypes = [S8I] - inp = S8I(9, 8, 7, 6, 5, 4, 3, 2) - s8i = windll.s_ret_8i_func(inp) - self.assertEqual( - (s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), - (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) - - def test_sf1651235(self): - # see https://www.python.org/sf/1651235 - - proto = CFUNCTYPE(c_int, RECT, POINT) - def callback(*args): - return 0 - - callback = proto(callback) - self.assertRaises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT())) - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_incomplete.py b/Lib/ctypes/test/test_incomplete.py deleted file mode 100644 index 00c430ef53c..00000000000 --- a/Lib/ctypes/test/test_incomplete.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest -from ctypes import * - -################################################################ -# -# The incomplete pointer example from the tutorial -# - -class MyTestCase(unittest.TestCase): - - def test_incomplete_example(self): - lpcell = POINTER("cell") - class cell(Structure): - _fields_ = [("name", c_char_p), - ("next", lpcell)] - - SetPointerType(lpcell, cell) - - c1 = cell() - c1.name = b"foo" - c2 = cell() - c2.name = b"bar" - - c1.next = pointer(c2) - c2.next = pointer(c1) - - p = c1 - - result = [] - for i in range(8): - result.append(p.name) - p = p.next[0] - self.assertEqual(result, [b"foo", b"bar"] * 4) - - # to not leak references, we must clean _pointer_type_cache - from ctypes import _pointer_type_cache - del _pointer_type_cache[cell] - -################################################################ - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_init.py b/Lib/ctypes/test/test_init.py deleted file mode 100644 index 75fad112a01..00000000000 --- a/Lib/ctypes/test/test_init.py +++ /dev/null @@ -1,40 +0,0 @@ -from ctypes import * -import unittest - -class X(Structure): - _fields_ = [("a", c_int), - ("b", c_int)] - new_was_called = False - - def __new__(cls): - result = super().__new__(cls) - result.new_was_called = True - return result - - def __init__(self): - self.a = 9 - self.b = 12 - -class Y(Structure): - _fields_ = [("x", X)] - - -class InitTest(unittest.TestCase): - def test_get(self): - # make sure the only accessing a nested structure - # doesn't call the structure's __new__ and __init__ - y = Y() - self.assertEqual((y.x.a, y.x.b), (0, 0)) - self.assertEqual(y.x.new_was_called, False) - - # But explicitly creating an X structure calls __new__ and __init__, of course. - x = X() - self.assertEqual((x.a, x.b), (9, 12)) - self.assertEqual(x.new_was_called, True) - - y.x = x - self.assertEqual((y.x.a, y.x.b), (9, 12)) - self.assertEqual(y.x.new_was_called, False) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_keeprefs.py b/Lib/ctypes/test/test_keeprefs.py deleted file mode 100644 index 94c02573fa1..00000000000 --- a/Lib/ctypes/test/test_keeprefs.py +++ /dev/null @@ -1,153 +0,0 @@ -from ctypes import * -import unittest - -class SimpleTestCase(unittest.TestCase): - def test_cint(self): - x = c_int() - self.assertEqual(x._objects, None) - x.value = 42 - self.assertEqual(x._objects, None) - x = c_int(99) - self.assertEqual(x._objects, None) - - def test_ccharp(self): - x = c_char_p() - self.assertEqual(x._objects, None) - x.value = b"abc" - self.assertEqual(x._objects, b"abc") - x = c_char_p(b"spam") - self.assertEqual(x._objects, b"spam") - -class StructureTestCase(unittest.TestCase): - def test_cint_struct(self): - class X(Structure): - _fields_ = [("a", c_int), - ("b", c_int)] - - x = X() - self.assertEqual(x._objects, None) - x.a = 42 - x.b = 99 - self.assertEqual(x._objects, None) - - def test_ccharp_struct(self): - class X(Structure): - _fields_ = [("a", c_char_p), - ("b", c_char_p)] - x = X() - self.assertEqual(x._objects, None) - - x.a = b"spam" - x.b = b"foo" - self.assertEqual(x._objects, {"0": b"spam", "1": b"foo"}) - - def test_struct_struct(self): - class POINT(Structure): - _fields_ = [("x", c_int), ("y", c_int)] - class RECT(Structure): - _fields_ = [("ul", POINT), ("lr", POINT)] - - r = RECT() - r.ul.x = 0 - r.ul.y = 1 - r.lr.x = 2 - r.lr.y = 3 - self.assertEqual(r._objects, None) - - r = RECT() - pt = POINT(1, 2) - r.ul = pt - self.assertEqual(r._objects, {'0': {}}) - r.ul.x = 22 - r.ul.y = 44 - self.assertEqual(r._objects, {'0': {}}) - r.lr = POINT() - self.assertEqual(r._objects, {'0': {}, '1': {}}) - -class ArrayTestCase(unittest.TestCase): - def test_cint_array(self): - INTARR = c_int * 3 - - ia = INTARR() - self.assertEqual(ia._objects, None) - ia[0] = 1 - ia[1] = 2 - ia[2] = 3 - self.assertEqual(ia._objects, None) - - class X(Structure): - _fields_ = [("x", c_int), - ("a", INTARR)] - - x = X() - x.x = 1000 - x.a[0] = 42 - x.a[1] = 96 - self.assertEqual(x._objects, None) - x.a = ia - self.assertEqual(x._objects, {'1': {}}) - -class PointerTestCase(unittest.TestCase): - def test_p_cint(self): - i = c_int(42) - x = pointer(i) - self.assertEqual(x._objects, {'1': i}) - -class DeletePointerTestCase(unittest.TestCase): - @unittest.skip('test disabled') - def test_X(self): - class X(Structure): - _fields_ = [("p", POINTER(c_char_p))] - x = X() - i = c_char_p("abc def") - from sys import getrefcount as grc - print("2?", grc(i)) - x.p = pointer(i) - print("3?", grc(i)) - for i in range(320): - c_int(99) - x.p[0] - print(x.p[0]) -## del x -## print "2?", grc(i) -## del i - import gc - gc.collect() - for i in range(320): - c_int(99) - x.p[0] - print(x.p[0]) - print(x.p.contents) -## print x._objects - - x.p[0] = "spam spam" -## print x.p[0] - print("+" * 42) - print(x._objects) - -class PointerToStructure(unittest.TestCase): - def test(self): - class POINT(Structure): - _fields_ = [("x", c_int), ("y", c_int)] - class RECT(Structure): - _fields_ = [("a", POINTER(POINT)), - ("b", POINTER(POINT))] - r = RECT() - p1 = POINT(1, 2) - - r.a = pointer(p1) - r.b = pointer(p1) -## from pprint import pprint as pp -## pp(p1._objects) -## pp(r._objects) - - r.a[0].x = 42 - r.a[0].y = 99 - - # to avoid leaking when tests are run several times - # clean up the types left in the cache. - from ctypes import _pointer_type_cache - del _pointer_type_cache[POINT] - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_libc.py b/Lib/ctypes/test/test_libc.py deleted file mode 100644 index 56285b5ff81..00000000000 --- a/Lib/ctypes/test/test_libc.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest - -from ctypes import * -import _ctypes_test - -lib = CDLL(_ctypes_test.__file__) - -def three_way_cmp(x, y): - """Return -1 if x < y, 0 if x == y and 1 if x > y""" - return (x > y) - (x < y) - -class LibTest(unittest.TestCase): - def test_sqrt(self): - lib.my_sqrt.argtypes = c_double, - lib.my_sqrt.restype = c_double - self.assertEqual(lib.my_sqrt(4.0), 2.0) - import math - self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) - - def test_qsort(self): - comparefunc = CFUNCTYPE(c_int, POINTER(c_char), POINTER(c_char)) - lib.my_qsort.argtypes = c_void_p, c_size_t, c_size_t, comparefunc - lib.my_qsort.restype = None - - def sort(a, b): - return three_way_cmp(a[0], b[0]) - - chars = create_string_buffer(b"spam, spam, and spam") - lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort)) - self.assertEqual(chars.raw, b" ,,aaaadmmmnpppsss\x00") - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_loading.py b/Lib/ctypes/test/test_loading.py deleted file mode 100644 index ea892277c4e..00000000000 --- a/Lib/ctypes/test/test_loading.py +++ /dev/null @@ -1,182 +0,0 @@ -from ctypes import * -import os -import shutil -import subprocess -import sys -import unittest -import test.support -from test.support import import_helper -from test.support import os_helper -from ctypes.util import find_library - -libc_name = None - -def setUpModule(): - global libc_name - if os.name == "nt": - libc_name = find_library("c") - elif sys.platform == "cygwin": - libc_name = "cygwin1.dll" - else: - libc_name = find_library("c") - - if test.support.verbose: - print("libc_name is", libc_name) - -class LoaderTest(unittest.TestCase): - - unknowndll = "xxrandomnamexx" - - def test_load(self): - if libc_name is None: - self.skipTest('could not find libc') - CDLL(libc_name) - CDLL(os.path.basename(libc_name)) - self.assertRaises(OSError, CDLL, self.unknowndll) - - def test_load_version(self): - if libc_name is None: - self.skipTest('could not find libc') - if os.path.basename(libc_name) != 'libc.so.6': - self.skipTest('wrong libc path for test') - cdll.LoadLibrary("libc.so.6") - # linux uses version, libc 9 should not exist - self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9") - self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) - - def test_find(self): - for name in ("c", "m"): - lib = find_library(name) - if lib: - cdll.LoadLibrary(lib) - CDLL(lib) - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_library(self): - # CRT is no longer directly loadable. See issue23606 for the - # discussion about alternative approaches. - #self.assertIsNotNone(libc_name) - if test.support.verbose: - print(find_library("kernel32")) - print(find_library("user32")) - - if os.name == "nt": - windll.kernel32.GetModuleHandleW - windll["kernel32"].GetModuleHandleW - windll.LoadLibrary("kernel32").GetModuleHandleW - WinDLL("kernel32").GetModuleHandleW - # embedded null character - self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0") - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_ordinal_functions(self): - import _ctypes_test - dll = WinDLL(_ctypes_test.__file__) - # We load the same function both via ordinal and name - func_ord = dll[2] - func_name = dll.GetString - # addressof gets the address where the function pointer is stored - a_ord = addressof(func_ord) - a_name = addressof(func_name) - f_ord_addr = c_void_p.from_address(a_ord).value - f_name_addr = c_void_p.from_address(a_name).value - self.assertEqual(hex(f_ord_addr), hex(f_name_addr)) - - self.assertRaises(AttributeError, dll.__getitem__, 1234) - - @unittest.skipUnless(os.name == "nt", 'Windows-specific test') - def test_1703286_A(self): - from _ctypes import LoadLibrary, FreeLibrary - # On winXP 64-bit, advapi32 loads at an address that does - # NOT fit into a 32-bit integer. FreeLibrary must be able - # to accept this address. - - # These are tests for https://www.python.org/sf/1703286 - handle = LoadLibrary("advapi32") - FreeLibrary(handle) - - @unittest.skipUnless(os.name == "nt", 'Windows-specific test') - def test_1703286_B(self): - # Since on winXP 64-bit advapi32 loads like described - # above, the (arbitrarily selected) CloseEventLog function - # also has a high address. 'call_function' should accept - # addresses so large. - from _ctypes import call_function - advapi32 = windll.advapi32 - # Calling CloseEventLog with a NULL argument should fail, - # but the call should not segfault or so. - self.assertEqual(0, advapi32.CloseEventLog(None)) - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p - proc = windll.kernel32.GetProcAddress(advapi32._handle, - b"CloseEventLog") - self.assertTrue(proc) - # This is the real test: call the function via 'call_function' - self.assertEqual(0, call_function(proc, (None,))) - - @unittest.skipUnless(os.name == "nt", - 'test specific to Windows') - def test_load_dll_with_flags(self): - _sqlite3 = import_helper.import_module("_sqlite3") - src = _sqlite3.__file__ - if src.lower().endswith("_d.pyd"): - ext = "_d.dll" - else: - ext = ".dll" - - with os_helper.temp_dir() as tmp: - # We copy two files and load _sqlite3.dll (formerly .pyd), - # which has a dependency on sqlite3.dll. Then we test - # loading it in subprocesses to avoid it starting in memory - # for each test. - target = os.path.join(tmp, "_sqlite3.dll") - shutil.copy(src, target) - shutil.copy(os.path.join(os.path.dirname(src), "sqlite3" + ext), - os.path.join(tmp, "sqlite3" + ext)) - - def should_pass(command): - with self.subTest(command): - subprocess.check_output( - [sys.executable, "-c", - "from ctypes import *; import nt;" + command], - cwd=tmp - ) - - def should_fail(command): - with self.subTest(command): - with self.assertRaises(subprocess.CalledProcessError): - subprocess.check_output( - [sys.executable, "-c", - "from ctypes import *; import nt;" + command], - cwd=tmp, stderr=subprocess.STDOUT, - ) - - # Default load should not find this in CWD - should_fail("WinDLL('_sqlite3.dll')") - - # Relative path (but not just filename) should succeed - should_pass("WinDLL('./_sqlite3.dll')") - - # Insecure load flags should succeed - # Clear the DLL directory to avoid safe search settings propagating - should_pass("windll.kernel32.SetDllDirectoryW(None); WinDLL('_sqlite3.dll', winmode=0)") - - # Full path load without DLL_LOAD_DIR shouldn't find dependency - should_fail("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + - "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32)") - - # Full path load with DLL_LOAD_DIR should succeed - should_pass("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + - "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32|" + - "nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)") - - # User-specified directory should succeed - should_pass("import os; p = os.add_dll_directory(os.getcwd());" + - "WinDLL('_sqlite3.dll'); p.close()") - - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_memfunctions.py b/Lib/ctypes/test/test_memfunctions.py deleted file mode 100644 index e784b9a7068..00000000000 --- a/Lib/ctypes/test/test_memfunctions.py +++ /dev/null @@ -1,79 +0,0 @@ -import sys -from test import support -import unittest -from ctypes import * -from ctypes.test import need_symbol - -class MemFunctionsTest(unittest.TestCase): - @unittest.skip('test disabled') - def test_overflow(self): - # string_at and wstring_at must use the Python calling - # convention (which acquires the GIL and checks the Python - # error flag). Provoke an error and catch it; see also issue - # #3554: - self.assertRaises((OverflowError, MemoryError, SystemError), - lambda: wstring_at(u"foo", sys.maxint - 1)) - self.assertRaises((OverflowError, MemoryError, SystemError), - lambda: string_at("foo", sys.maxint - 1)) - - def test_memmove(self): - # large buffers apparently increase the chance that the memory - # is allocated in high address space. - a = create_string_buffer(1000000) - p = b"Hello, World" - result = memmove(a, p, len(p)) - self.assertEqual(a.value, b"Hello, World") - - self.assertEqual(string_at(result), b"Hello, World") - self.assertEqual(string_at(result, 5), b"Hello") - self.assertEqual(string_at(result, 16), b"Hello, World\0\0\0\0") - self.assertEqual(string_at(result, 0), b"") - - def test_memset(self): - a = create_string_buffer(1000000) - result = memset(a, ord('x'), 16) - self.assertEqual(a.value, b"xxxxxxxxxxxxxxxx") - - self.assertEqual(string_at(result), b"xxxxxxxxxxxxxxxx") - self.assertEqual(string_at(a), b"xxxxxxxxxxxxxxxx") - self.assertEqual(string_at(a, 20), b"xxxxxxxxxxxxxxxx\0\0\0\0") - - def test_cast(self): - a = (c_ubyte * 32)(*map(ord, "abcdef")) - self.assertEqual(cast(a, c_char_p).value, b"abcdef") - self.assertEqual(cast(a, POINTER(c_byte))[:7], - [97, 98, 99, 100, 101, 102, 0]) - self.assertEqual(cast(a, POINTER(c_byte))[:7:], - [97, 98, 99, 100, 101, 102, 0]) - self.assertEqual(cast(a, POINTER(c_byte))[6:-1:-1], - [0, 102, 101, 100, 99, 98, 97]) - self.assertEqual(cast(a, POINTER(c_byte))[:7:2], - [97, 99, 101, 0]) - self.assertEqual(cast(a, POINTER(c_byte))[:7:7], - [97]) - - @support.refcount_test - def test_string_at(self): - s = string_at(b"foo bar") - # XXX The following may be wrong, depending on how Python - # manages string instances - self.assertEqual(2, sys.getrefcount(s)) - self.assertTrue(s, "foo bar") - - self.assertEqual(string_at(b"foo bar", 7), b"foo bar") - self.assertEqual(string_at(b"foo bar", 3), b"foo") - - @need_symbol('create_unicode_buffer') - def test_wstring_at(self): - p = create_unicode_buffer("Hello, World") - a = create_unicode_buffer(1000000) - result = memmove(a, p, len(p) * sizeof(c_wchar)) - self.assertEqual(a.value, "Hello, World") - - self.assertEqual(wstring_at(a), "Hello, World") - self.assertEqual(wstring_at(a, 5), "Hello") - self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") - self.assertEqual(wstring_at(a, 0), "") - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_numbers.py b/Lib/ctypes/test/test_numbers.py deleted file mode 100644 index db500e812be..00000000000 --- a/Lib/ctypes/test/test_numbers.py +++ /dev/null @@ -1,295 +0,0 @@ -from ctypes import * -import unittest -import struct - -def valid_ranges(*types): - # given a sequence of numeric types, collect their _type_ - # attribute, which is a single format character compatible with - # the struct module, use the struct module to calculate the - # minimum and maximum value allowed for this format. - # Returns a list of (min, max) values. - result = [] - for t in types: - fmt = t._type_ - size = struct.calcsize(fmt) - a = struct.unpack(fmt, (b"\x00"*32)[:size])[0] - b = struct.unpack(fmt, (b"\xFF"*32)[:size])[0] - c = struct.unpack(fmt, (b"\x7F"+b"\x00"*32)[:size])[0] - d = struct.unpack(fmt, (b"\x80"+b"\xFF"*32)[:size])[0] - result.append((min(a, b, c, d), max(a, b, c, d))) - return result - -ArgType = type(byref(c_int(0))) - -unsigned_types = [c_ubyte, c_ushort, c_uint, c_ulong] -signed_types = [c_byte, c_short, c_int, c_long, c_longlong] - -bool_types = [] - -float_types = [c_double, c_float] - -try: - c_ulonglong - c_longlong -except NameError: - pass -else: - unsigned_types.append(c_ulonglong) - signed_types.append(c_longlong) - -try: - c_bool -except NameError: - pass -else: - bool_types.append(c_bool) - -unsigned_ranges = valid_ranges(*unsigned_types) -signed_ranges = valid_ranges(*signed_types) -bool_values = [True, False, 0, 1, -1, 5000, 'test', [], [1]] - -################################################################ - -class NumberTestCase(unittest.TestCase): - - def test_default_init(self): - # default values are set to zero - for t in signed_types + unsigned_types + float_types: - self.assertEqual(t().value, 0) - - def test_unsigned_values(self): - # the value given to the constructor is available - # as the 'value' attribute - for t, (l, h) in zip(unsigned_types, unsigned_ranges): - self.assertEqual(t(l).value, l) - self.assertEqual(t(h).value, h) - - def test_signed_values(self): - # see above - for t, (l, h) in zip(signed_types, signed_ranges): - self.assertEqual(t(l).value, l) - self.assertEqual(t(h).value, h) - - def test_bool_values(self): - from operator import truth - for t, v in zip(bool_types, bool_values): - self.assertEqual(t(v).value, truth(v)) - - def test_typeerror(self): - # Only numbers are allowed in the constructor, - # otherwise TypeError is raised - for t in signed_types + unsigned_types + float_types: - self.assertRaises(TypeError, t, "") - self.assertRaises(TypeError, t, None) - - @unittest.skip('test disabled') - def test_valid_ranges(self): - # invalid values of the correct type - # raise ValueError (not OverflowError) - for t, (l, h) in zip(unsigned_types, unsigned_ranges): - self.assertRaises(ValueError, t, l-1) - self.assertRaises(ValueError, t, h+1) - - def test_from_param(self): - # the from_param class method attribute always - # returns PyCArgObject instances - for t in signed_types + unsigned_types + float_types: - self.assertEqual(ArgType, type(t.from_param(0))) - - def test_byref(self): - # calling byref returns also a PyCArgObject instance - for t in signed_types + unsigned_types + float_types + bool_types: - parm = byref(t()) - self.assertEqual(ArgType, type(parm)) - - - def test_floats(self): - # c_float and c_double can be created from - # Python int and float - class FloatLike(object): - def __float__(self): - return 2.0 - f = FloatLike() - for t in float_types: - self.assertEqual(t(2.0).value, 2.0) - self.assertEqual(t(2).value, 2.0) - self.assertEqual(t(2).value, 2.0) - self.assertEqual(t(f).value, 2.0) - - def test_integers(self): - class FloatLike(object): - def __float__(self): - return 2.0 - f = FloatLike() - class IntLike(object): - def __int__(self): - return 2 - d = IntLike() - class IndexLike(object): - def __index__(self): - return 2 - i = IndexLike() - # integers cannot be constructed from floats, - # but from integer-like objects - for t in signed_types + unsigned_types: - self.assertRaises(TypeError, t, 3.14) - self.assertRaises(TypeError, t, f) - self.assertRaises(TypeError, t, d) - self.assertEqual(t(i).value, 2) - - def test_sizes(self): - for t in signed_types + unsigned_types + float_types + bool_types: - try: - size = struct.calcsize(t._type_) - except struct.error: - continue - # sizeof of the type... - self.assertEqual(sizeof(t), size) - # and sizeof of an instance - self.assertEqual(sizeof(t()), size) - - def test_alignments(self): - for t in signed_types + unsigned_types + float_types: - code = t._type_ # the typecode - align = struct.calcsize("c%c" % code) - struct.calcsize(code) - - # alignment of the type... - self.assertEqual((code, alignment(t)), - (code, align)) - # and alignment of an instance - self.assertEqual((code, alignment(t())), - (code, align)) - - def test_int_from_address(self): - from array import array - for t in signed_types + unsigned_types: - # the array module doesn't support all format codes - # (no 'q' or 'Q') - try: - array(t._type_) - except ValueError: - continue - a = array(t._type_, [100]) - - # v now is an integer at an 'external' memory location - v = t.from_address(a.buffer_info()[0]) - self.assertEqual(v.value, a[0]) - self.assertEqual(type(v), t) - - # changing the value at the memory location changes v's value also - a[0] = 42 - self.assertEqual(v.value, a[0]) - - - def test_float_from_address(self): - from array import array - for t in float_types: - a = array(t._type_, [3.14]) - v = t.from_address(a.buffer_info()[0]) - self.assertEqual(v.value, a[0]) - self.assertIs(type(v), t) - a[0] = 2.3456e17 - self.assertEqual(v.value, a[0]) - self.assertIs(type(v), t) - - def test_char_from_address(self): - from ctypes import c_char - from array import array - - a = array('b', [0]) - a[0] = ord('x') - v = c_char.from_address(a.buffer_info()[0]) - self.assertEqual(v.value, b'x') - self.assertIs(type(v), c_char) - - a[0] = ord('?') - self.assertEqual(v.value, b'?') - - # array does not support c_bool / 't' - @unittest.skip('test disabled') - def test_bool_from_address(self): - from ctypes import c_bool - from array import array - a = array(c_bool._type_, [True]) - v = t.from_address(a.buffer_info()[0]) - self.assertEqual(v.value, a[0]) - self.assertEqual(type(v) is t) - a[0] = False - self.assertEqual(v.value, a[0]) - self.assertEqual(type(v) is t) - - def test_init(self): - # c_int() can be initialized from Python's int, and c_int. - # Not from c_long or so, which seems strange, abc should - # probably be changed: - self.assertRaises(TypeError, c_int, c_long(42)) - - def test_float_overflow(self): - import sys - big_int = int(sys.float_info.max) * 2 - for t in float_types + [c_longdouble]: - self.assertRaises(OverflowError, t, big_int) - if (hasattr(t, "__ctype_be__")): - self.assertRaises(OverflowError, t.__ctype_be__, big_int) - if (hasattr(t, "__ctype_le__")): - self.assertRaises(OverflowError, t.__ctype_le__, big_int) - - @unittest.skip('test disabled') - def test_perf(self): - check_perf() - -from ctypes import _SimpleCData -class c_int_S(_SimpleCData): - _type_ = "i" - __slots__ = [] - -def run_test(rep, msg, func, arg=None): -## items = [None] * rep - items = range(rep) - from time import perf_counter as clock - if arg is not None: - start = clock() - for i in items: - func(arg); func(arg); func(arg); func(arg); func(arg) - stop = clock() - else: - start = clock() - for i in items: - func(); func(); func(); func(); func() - stop = clock() - print("%15s: %.2f us" % (msg, ((stop-start)*1e6/5/rep))) - -def check_perf(): - # Construct 5 objects - from ctypes import c_int - - REP = 200000 - - run_test(REP, "int()", int) - run_test(REP, "int(999)", int) - run_test(REP, "c_int()", c_int) - run_test(REP, "c_int(999)", c_int) - run_test(REP, "c_int_S()", c_int_S) - run_test(REP, "c_int_S(999)", c_int_S) - -# Python 2.3 -OO, win2k, P4 700 MHz: -# -# int(): 0.87 us -# int(999): 0.87 us -# c_int(): 3.35 us -# c_int(999): 3.34 us -# c_int_S(): 3.23 us -# c_int_S(999): 3.24 us - -# Python 2.2 -OO, win2k, P4 700 MHz: -# -# int(): 0.89 us -# int(999): 0.89 us -# c_int(): 9.99 us -# c_int(999): 10.02 us -# c_int_S(): 9.87 us -# c_int_S(999): 9.85 us - -if __name__ == '__main__': -## check_perf() - unittest.main() diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py deleted file mode 100644 index 38af7ac13d7..00000000000 --- a/Lib/ctypes/test/test_parameters.py +++ /dev/null @@ -1,250 +0,0 @@ -import unittest -from ctypes.test import need_symbol -import test.support - -class SimpleTypesTestCase(unittest.TestCase): - - def setUp(self): - import ctypes - try: - from _ctypes import set_conversion_mode - except ImportError: - pass - else: - self.prev_conv_mode = set_conversion_mode("ascii", "strict") - - def tearDown(self): - try: - from _ctypes import set_conversion_mode - except ImportError: - pass - else: - set_conversion_mode(*self.prev_conv_mode) - - def test_subclasses(self): - from ctypes import c_void_p, c_char_p - # ctypes 0.9.5 and before did overwrite from_param in SimpleType_new - class CVOIDP(c_void_p): - def from_param(cls, value): - return value * 2 - from_param = classmethod(from_param) - - class CCHARP(c_char_p): - def from_param(cls, value): - return value * 4 - from_param = classmethod(from_param) - - self.assertEqual(CVOIDP.from_param("abc"), "abcabc") - self.assertEqual(CCHARP.from_param("abc"), "abcabcabcabc") - - @need_symbol('c_wchar_p') - def test_subclasses_c_wchar_p(self): - from ctypes import c_wchar_p - - class CWCHARP(c_wchar_p): - def from_param(cls, value): - return value * 3 - from_param = classmethod(from_param) - - self.assertEqual(CWCHARP.from_param("abc"), "abcabcabc") - - # XXX Replace by c_char_p tests - def test_cstrings(self): - from ctypes import c_char_p - - # c_char_p.from_param on a Python String packs the string - # into a cparam object - s = b"123" - self.assertIs(c_char_p.from_param(s)._obj, s) - - # new in 0.9.1: convert (encode) unicode to ascii - self.assertEqual(c_char_p.from_param(b"123")._obj, b"123") - self.assertRaises(TypeError, c_char_p.from_param, "123\377") - self.assertRaises(TypeError, c_char_p.from_param, 42) - - # calling c_char_p.from_param with a c_char_p instance - # returns the argument itself: - a = c_char_p(b"123") - self.assertIs(c_char_p.from_param(a), a) - - @need_symbol('c_wchar_p') - def test_cw_strings(self): - from ctypes import c_wchar_p - - c_wchar_p.from_param("123") - - self.assertRaises(TypeError, c_wchar_p.from_param, 42) - self.assertRaises(TypeError, c_wchar_p.from_param, b"123\377") - - pa = c_wchar_p.from_param(c_wchar_p("123")) - self.assertEqual(type(pa), c_wchar_p) - - def test_int_pointers(self): - from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer - LPINT = POINTER(c_int) - -## p = pointer(c_int(42)) -## x = LPINT.from_param(p) - x = LPINT.from_param(pointer(c_int(42))) - self.assertEqual(x.contents.value, 42) - self.assertEqual(LPINT(c_int(42)).contents.value, 42) - - self.assertEqual(LPINT.from_param(None), None) - - if c_int != c_long: - self.assertRaises(TypeError, LPINT.from_param, pointer(c_long(42))) - self.assertRaises(TypeError, LPINT.from_param, pointer(c_uint(42))) - self.assertRaises(TypeError, LPINT.from_param, pointer(c_short(42))) - - def test_byref_pointer(self): - # The from_param class method of POINTER(typ) classes accepts what is - # returned by byref(obj), it type(obj) == typ - from ctypes import c_short, c_uint, c_int, c_long, POINTER, byref - LPINT = POINTER(c_int) - - LPINT.from_param(byref(c_int(42))) - - self.assertRaises(TypeError, LPINT.from_param, byref(c_short(22))) - if c_int != c_long: - self.assertRaises(TypeError, LPINT.from_param, byref(c_long(22))) - self.assertRaises(TypeError, LPINT.from_param, byref(c_uint(22))) - - def test_byref_pointerpointer(self): - # See above - from ctypes import c_short, c_uint, c_int, c_long, pointer, POINTER, byref - - LPLPINT = POINTER(POINTER(c_int)) - LPLPINT.from_param(byref(pointer(c_int(42)))) - - self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_short(22)))) - if c_int != c_long: - self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_long(22)))) - self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_uint(22)))) - - def test_array_pointers(self): - from ctypes import c_short, c_uint, c_int, c_long, POINTER - INTARRAY = c_int * 3 - ia = INTARRAY() - self.assertEqual(len(ia), 3) - self.assertEqual([ia[i] for i in range(3)], [0, 0, 0]) - - # Pointers are only compatible with arrays containing items of - # the same type! - LPINT = POINTER(c_int) - LPINT.from_param((c_int*3)()) - self.assertRaises(TypeError, LPINT.from_param, c_short*3) - self.assertRaises(TypeError, LPINT.from_param, c_long*3) - self.assertRaises(TypeError, LPINT.from_param, c_uint*3) - - def test_noctypes_argtype(self): - import _ctypes_test - from ctypes import CDLL, c_void_p, ArgumentError - - func = CDLL(_ctypes_test.__file__)._testfunc_p_p - func.restype = c_void_p - # TypeError: has no from_param method - self.assertRaises(TypeError, setattr, func, "argtypes", (object,)) - - class Adapter(object): - def from_param(cls, obj): - return None - - func.argtypes = (Adapter(),) - self.assertEqual(func(None), None) - self.assertEqual(func(object()), None) - - class Adapter(object): - def from_param(cls, obj): - return obj - - func.argtypes = (Adapter(),) - # don't know how to convert parameter 1 - self.assertRaises(ArgumentError, func, object()) - self.assertEqual(func(c_void_p(42)), 42) - - class Adapter(object): - def from_param(cls, obj): - raise ValueError(obj) - - func.argtypes = (Adapter(),) - # ArgumentError: argument 1: ValueError: 99 - self.assertRaises(ArgumentError, func, 99) - - def test_abstract(self): - from ctypes import (Array, Structure, Union, _Pointer, - _SimpleCData, _CFuncPtr) - - self.assertRaises(TypeError, Array.from_param, 42) - self.assertRaises(TypeError, Structure.from_param, 42) - self.assertRaises(TypeError, Union.from_param, 42) - self.assertRaises(TypeError, _CFuncPtr.from_param, 42) - self.assertRaises(TypeError, _Pointer.from_param, 42) - self.assertRaises(TypeError, _SimpleCData.from_param, 42) - - @test.support.cpython_only - def test_issue31311(self): - # __setstate__ should neither raise a SystemError nor crash in case - # of a bad __dict__. - from ctypes import Structure - - class BadStruct(Structure): - @property - def __dict__(self): - pass - with self.assertRaises(TypeError): - BadStruct().__setstate__({}, b'foo') - - class WorseStruct(Structure): - @property - def __dict__(self): - 1/0 - with self.assertRaises(ZeroDivisionError): - WorseStruct().__setstate__({}, b'foo') - - def test_parameter_repr(self): - from ctypes import ( - c_bool, - c_char, - c_wchar, - c_byte, - c_ubyte, - c_short, - c_ushort, - c_int, - c_uint, - c_long, - c_ulong, - c_longlong, - c_ulonglong, - c_float, - c_double, - c_longdouble, - c_char_p, - c_wchar_p, - c_void_p, - ) - self.assertRegex(repr(c_bool.from_param(True)), r"^$") - self.assertEqual(repr(c_char.from_param(97)), "") - self.assertRegex(repr(c_wchar.from_param('a')), r"^$") - self.assertEqual(repr(c_byte.from_param(98)), "") - self.assertEqual(repr(c_ubyte.from_param(98)), "") - self.assertEqual(repr(c_short.from_param(511)), "") - self.assertEqual(repr(c_ushort.from_param(511)), "") - self.assertRegex(repr(c_int.from_param(20000)), r"^$") - self.assertRegex(repr(c_uint.from_param(20000)), r"^$") - self.assertRegex(repr(c_long.from_param(20000)), r"^$") - self.assertRegex(repr(c_ulong.from_param(20000)), r"^$") - self.assertRegex(repr(c_longlong.from_param(20000)), r"^$") - self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") - self.assertEqual(repr(c_float.from_param(1.5)), "") - self.assertEqual(repr(c_double.from_param(1.5)), "") - self.assertEqual(repr(c_double.from_param(1e300)), "") - self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") - self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") - self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") - self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") - -################################################################ - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py deleted file mode 100644 index efffc80a66f..00000000000 --- a/Lib/ctypes/test/test_pep3118.py +++ /dev/null @@ -1,235 +0,0 @@ -import unittest -from ctypes import * -import re, sys - -if sys.byteorder == "little": - THIS_ENDIAN = "<" - OTHER_ENDIAN = ">" -else: - THIS_ENDIAN = ">" - OTHER_ENDIAN = "<" - -def normalize(format): - # Remove current endian specifier and white space from a format - # string - if format is None: - return "" - format = format.replace(OTHER_ENDIAN, THIS_ENDIAN) - return re.sub(r"\s", "", format) - -class Test(unittest.TestCase): - - def test_native_types(self): - for tp, fmt, shape, itemtp in native_types: - ob = tp() - v = memoryview(ob) - try: - self.assertEqual(normalize(v.format), normalize(fmt)) - if shape: - self.assertEqual(len(v), shape[0]) - else: - self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob)) - self.assertEqual(v.itemsize, sizeof(itemtp)) - self.assertEqual(v.shape, shape) - # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides - # if requested. memoryview currently reconstructs missing - # stride information, so this assert will fail. - # self.assertEqual(v.strides, ()) - - # they are always read/write - self.assertFalse(v.readonly) - - if v.shape: - n = 1 - for dim in v.shape: - n = n * dim - self.assertEqual(n * v.itemsize, len(v.tobytes())) - except: - # so that we can see the failing type - print(tp) - raise - - def test_endian_types(self): - for tp, fmt, shape, itemtp in endian_types: - ob = tp() - v = memoryview(ob) - try: - self.assertEqual(v.format, fmt) - if shape: - self.assertEqual(len(v), shape[0]) - else: - self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob)) - self.assertEqual(v.itemsize, sizeof(itemtp)) - self.assertEqual(v.shape, shape) - # XXX Issue #12851 - # self.assertEqual(v.strides, ()) - - # they are always read/write - self.assertFalse(v.readonly) - - if v.shape: - n = 1 - for dim in v.shape: - n = n * dim - self.assertEqual(n, len(v)) - except: - # so that we can see the failing type - print(tp) - raise - -# define some structure classes - -class Point(Structure): - _fields_ = [("x", c_long), ("y", c_long)] - -class PackedPoint(Structure): - _pack_ = 2 - _fields_ = [("x", c_long), ("y", c_long)] - -class Point2(Structure): - pass -Point2._fields_ = [("x", c_long), ("y", c_long)] - -class EmptyStruct(Structure): - _fields_ = [] - -class aUnion(Union): - _fields_ = [("a", c_int)] - -class StructWithArrays(Structure): - _fields_ = [("x", c_long * 3 * 2), ("y", Point * 4)] - -class Incomplete(Structure): - pass - -class Complete(Structure): - pass -PComplete = POINTER(Complete) -Complete._fields_ = [("a", c_long)] - -################################################################ -# -# This table contains format strings as they look on little endian -# machines. The test replaces '<' with '>' on big endian machines. -# - -# Platform-specific type codes -s_bool = {1: '?', 2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_bool)] -s_short = {2: 'h', 4: 'l', 8: 'q'}[sizeof(c_short)] -s_ushort = {2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_ushort)] -s_int = {2: 'h', 4: 'i', 8: 'q'}[sizeof(c_int)] -s_uint = {2: 'H', 4: 'I', 8: 'Q'}[sizeof(c_uint)] -s_long = {4: 'l', 8: 'q'}[sizeof(c_long)] -s_ulong = {4: 'L', 8: 'Q'}[sizeof(c_ulong)] -s_longlong = "q" -s_ulonglong = "Q" -s_float = "f" -s_double = "d" -s_longdouble = "g" - -# Alias definitions in ctypes/__init__.py -if c_int is c_long: - s_int = s_long -if c_uint is c_ulong: - s_uint = s_ulong -if c_longlong is c_long: - s_longlong = s_long -if c_ulonglong is c_ulong: - s_ulonglong = s_ulong -if c_longdouble is c_double: - s_longdouble = s_double - - -native_types = [ - # type format shape calc itemsize - - ## simple types - - (c_char, "l:x:>l:y:}".replace('l', s_long), (), BEPoint), - (LEPoint, "T{l:x:>l:y:}".replace('l', s_long), (), POINTER(BEPoint)), - (POINTER(LEPoint), "&T{)") - self.assertEqual(repr(py_object(42)), "py_object(42)") - self.assertEqual(repr(py_object(object)), "py_object(%r)" % object) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_random_things.py b/Lib/ctypes/test/test_random_things.py deleted file mode 100644 index 2988e275cf4..00000000000 --- a/Lib/ctypes/test/test_random_things.py +++ /dev/null @@ -1,77 +0,0 @@ -from ctypes import * -import contextlib -from test import support -import unittest -import sys - - -def callback_func(arg): - 42 / arg - raise ValueError(arg) - -@unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') -class call_function_TestCase(unittest.TestCase): - # _ctypes.call_function is deprecated and private, but used by - # Gary Bishp's readline module. If we have it, we must test it as well. - - def test(self): - from _ctypes import call_function - windll.kernel32.LoadLibraryA.restype = c_void_p - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p - - hdll = windll.kernel32.LoadLibraryA(b"kernel32") - funcaddr = windll.kernel32.GetProcAddress(hdll, b"GetModuleHandleA") - - self.assertEqual(call_function(funcaddr, (None,)), - windll.kernel32.GetModuleHandleA(None)) - -class CallbackTracbackTestCase(unittest.TestCase): - # When an exception is raised in a ctypes callback function, the C - # code prints a traceback. - # - # This test makes sure the exception types *and* the exception - # value is printed correctly. - # - # Changed in 0.9.3: No longer is '(in callback)' prepended to the - # error message - instead an additional frame for the C code is - # created, then a full traceback printed. When SystemExit is - # raised in a callback function, the interpreter exits. - - @contextlib.contextmanager - def expect_unraisable(self, exc_type, exc_msg=None): - with support.catch_unraisable_exception() as cm: - yield - - self.assertIsInstance(cm.unraisable.exc_value, exc_type) - if exc_msg is not None: - self.assertEqual(str(cm.unraisable.exc_value), exc_msg) - self.assertEqual(cm.unraisable.err_msg, - "Exception ignored on calling ctypes " - "callback function") - self.assertIs(cm.unraisable.object, callback_func) - - def test_ValueError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - with self.expect_unraisable(ValueError, '42'): - cb(42) - - def test_IntegerDivisionError(self): - cb = CFUNCTYPE(c_int, c_int)(callback_func) - with self.expect_unraisable(ZeroDivisionError): - cb(0) - - def test_FloatDivisionError(self): - cb = CFUNCTYPE(c_int, c_double)(callback_func) - with self.expect_unraisable(ZeroDivisionError): - cb(0.0) - - def test_TypeErrorDivisionError(self): - cb = CFUNCTYPE(c_int, c_char_p)(callback_func) - err_msg = "unsupported operand type(s) for /: 'int' and 'bytes'" - with self.expect_unraisable(TypeError, err_msg): - cb(b"spam") - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/ctypes/test/test_refcounts.py deleted file mode 100644 index 48958cd2a60..00000000000 --- a/Lib/ctypes/test/test_refcounts.py +++ /dev/null @@ -1,116 +0,0 @@ -import unittest -from test import support -import ctypes -import gc - -MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) -OtherCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_ulonglong) - -import _ctypes_test -dll = ctypes.CDLL(_ctypes_test.__file__) - -class RefcountTestCase(unittest.TestCase): - - @support.refcount_test - def test_1(self): - from sys import getrefcount as grc - - f = dll._testfunc_callback_i_if - f.restype = ctypes.c_int - f.argtypes = [ctypes.c_int, MyCallback] - - def callback(value): - #print "called back with", value - return value - - self.assertEqual(grc(callback), 2) - cb = MyCallback(callback) - - self.assertGreater(grc(callback), 2) - result = f(-10, cb) - self.assertEqual(result, -18) - cb = None - - gc.collect() - - self.assertEqual(grc(callback), 2) - - - @support.refcount_test - def test_refcount(self): - from sys import getrefcount as grc - def func(*args): - pass - # this is the standard refcount for func - self.assertEqual(grc(func), 2) - - # the CFuncPtr instance holds at least one refcount on func: - f = OtherCallback(func) - self.assertGreater(grc(func), 2) - - # and may release it again - del f - self.assertGreaterEqual(grc(func), 2) - - # but now it must be gone - gc.collect() - self.assertEqual(grc(func), 2) - - class X(ctypes.Structure): - _fields_ = [("a", OtherCallback)] - x = X() - x.a = OtherCallback(func) - - # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(grc(func), 2) - - # and may release it again - del x - self.assertGreaterEqual(grc(func), 2) - - # and now it must be gone again - gc.collect() - self.assertEqual(grc(func), 2) - - f = OtherCallback(func) - - # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(grc(func), 2) - - # create a cycle - f.cycle = f - - del f - gc.collect() - self.assertEqual(grc(func), 2) - -class AnotherLeak(unittest.TestCase): - def test_callback(self): - import sys - - proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) - def func(a, b): - return a * b * 2 - f = proto(func) - - a = sys.getrefcount(ctypes.c_int) - f(1, 2) - self.assertEqual(sys.getrefcount(ctypes.c_int), a) - - @support.refcount_test - def test_callback_py_object_none_return(self): - # bpo-36880: test that returning None from a py_object callback - # does not decrement the refcount of None. - - for FUNCTYPE in (ctypes.CFUNCTYPE, ctypes.PYFUNCTYPE): - with self.subTest(FUNCTYPE=FUNCTYPE): - @FUNCTYPE(ctypes.py_object) - def func(): - return None - - # Check that calling func does not affect None's refcount. - for _ in range(10000): - func() - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/ctypes/test/test_repr.py b/Lib/ctypes/test/test_repr.py deleted file mode 100644 index 60a2c803453..00000000000 --- a/Lib/ctypes/test/test_repr.py +++ /dev/null @@ -1,29 +0,0 @@ -from ctypes import * -import unittest - -subclasses = [] -for base in [c_byte, c_short, c_int, c_long, c_longlong, - c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong, - c_float, c_double, c_longdouble, c_bool]: - class X(base): - pass - subclasses.append(X) - -class X(c_char): - pass - -# This test checks if the __repr__ is correct for subclasses of simple types - -class ReprTest(unittest.TestCase): - def test_numbers(self): - for typ in subclasses: - base = typ.__bases__[0] - self.assertTrue(repr(base(42)).startswith(base.__name__)) - self.assertEqual("'), ('__ge__', '>='), ]: - if _set_new_attribute(cls, name, - _cmp_fn(name, op, self_tuple, other_tuple, - globals=globals)): - raise TypeError(f'Cannot overwrite attribute {name} ' - f'in class {cls.__name__}. Consider using ' - 'functools.total_ordering') + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + func_builder.add_fn(name, + ('self', 'other'), + [ ' if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + ' return NotImplemented'], + overwrite_error='Consider using functools.total_ordering') if frozen: - for fn in _frozen_get_del_attr(cls, field_list, globals): - if _set_new_attribute(cls, fn.__name__, fn): - raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' - f'in class {cls.__name__}') + _frozen_get_del_attr(cls, field_list, func_builder) # Decide if/how we're going to create a hash function. hash_action = _hash_action[bool(unsafe_hash), @@ -1083,22 +1207,36 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, bool(frozen), has_explicit_hash] if hash_action: - # No need to call _set_new_attribute here, since by the time - # we're here the overwriting is unconditional. - cls.__hash__ = hash_action(cls, field_list, globals) + cls.__hash__ = hash_action(cls, field_list, func_builder) + + # Generate the methods and add them to the class. This needs to be done + # before the __doc__ logic below, since inspect will look at the __init__ + # signature. + func_builder.add_fns_to_class(cls) if not getattr(cls, '__doc__'): # Create a class doc-string. - cls.__doc__ = (cls.__name__ + - str(inspect.signature(cls)).replace(' -> None', '')) + try: + # In some cases fetching a signature is not possible. + # But, we surely should not fail in this case. + text_sig = str(inspect.signature( + cls, + annotation_format=annotationlib.Format.FORWARDREF, + )).replace(' -> None', '') + except (TypeError, ValueError): + text_sig = '' + cls.__doc__ = (cls.__name__ + text_sig) if match_args: - # I could probably compute this once + # I could probably compute this once. _set_new_attribute(cls, '__match_args__', tuple(f.name for f in std_init_fields)) + # It's an error to specify weakref_slot if slots is False. + if weakref_slot and not slots: + raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen) + cls = _add_slots(cls, frozen, weakref_slot, fields) abc.update_abstractmethods(cls) @@ -1106,7 +1244,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # _dataclass_getstate and _dataclass_setstate are needed for pickling frozen -# classes with slots. These could be slighly more performant if we generated +# classes with slots. These could be slightly more performant if we generated # the code instead of iterating over fields. But that can be a project for # another day, if performance becomes an issue. def _dataclass_getstate(self): @@ -1119,61 +1257,181 @@ def _dataclass_setstate(self, state): object.__setattr__(self, field.name, value) -def _add_slots(cls, is_frozen): - # Need to create a new class, since we can't set __slots__ - # after a class has been created. +def _get_slots(cls): + match cls.__dict__.get('__slots__'): + # `__dictoffset__` and `__weakrefoffset__` can tell us whether + # the base type has dict/weakref slots, in a way that works correctly + # for both Python classes and C extension types. Extension types + # don't use `__slots__` for slot creation + case None: + slots = [] + if getattr(cls, '__weakrefoffset__', -1) != 0: + slots.append('__weakref__') + if getattr(cls, '__dictoffset__', -1) != 0: + slots.append('__dict__') + yield from slots + case str(slot): + yield slot + # Slots may be any iterable, but we cannot handle an iterator + # because it will already be (partially) consumed. + case iterable if not hasattr(iterable, '__next__'): + yield from iterable + case _: + raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") + + +def _update_func_cell_for__class__(f, oldcls, newcls): + # Returns True if we update a cell, else False. + if f is None: + # f will be None in the case of a property where not all of + # fget, fset, and fdel are used. Nothing to do in that case. + return False + try: + idx = f.__code__.co_freevars.index("__class__") + except ValueError: + # This function doesn't reference __class__, so nothing to do. + return False + # Fix the cell to point to the new class, if it's already pointing + # at the old class. I'm not convinced that the "is oldcls" test + # is needed, but other than performance can't hurt. + closure = f.__closure__[idx] + if closure.cell_contents is oldcls: + closure.cell_contents = newcls + return True + return False + + +def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + seen_docs = False + slots = {} + for slot in itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ): + doc = getattr(defined_fields.get(slot), 'doc', None) + if doc is not None: + seen_docs = True + slots[slot] = doc + + # We only return dict if there's at least one doc member, + # otherwise we return tuple, which is the old default format. + if seen_docs: + return slots + return tuple(slots) + + +def _add_slots(cls, is_frozen, weakref_slot, defined_fields): + # Need to create a new class, since we can't set __slots__ after a + # class has been created, and the @dataclass decorator is called + # after the class is created. # Make sure __slots__ isn't already set. if '__slots__' in cls.__dict__: raise TypeError(f'{cls.__name__} already specifies __slots__') + # gh-102069: Remove existing __weakref__ descriptor. + # gh-135228: Make sure the original class can be garbage collected. + sys._clear_type_descriptors(cls) + # Create a new dict for our new class. cls_dict = dict(cls.__dict__) field_names = tuple(f.name for f in fields(cls)) - cls_dict['__slots__'] = field_names + # Make sure slots don't overlap with those in base classes. + inherited_slots = set( + itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) + ) + + cls_dict["__slots__"] = _create_slots( + defined_fields, inherited_slots, field_names, weakref_slot, + ) + for field_name in field_names: # Remove our attributes, if present. They'll still be # available in _MARKER. cls_dict.pop(field_name, None) - # Remove __dict__ itself. - cls_dict.pop('__dict__', None) - # And finally create the class. qualname = getattr(cls, '__qualname__', None) - cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) if qualname is not None: - cls.__qualname__ = qualname + newcls.__qualname__ = qualname if is_frozen: # Need this for pickling frozen classes with slots. - cls.__getstate__ = _dataclass_getstate - cls.__setstate__ = _dataclass_setstate + if '__getstate__' not in cls_dict: + newcls.__getstate__ = _dataclass_getstate + if '__setstate__' not in cls_dict: + newcls.__setstate__ = _dataclass_setstate + + # Fix up any closures which reference __class__. This is used to + # fix zero argument super so that it points to the correct class + # (the newly created one, which we're returning) and not the + # original class. We can break out of this loop as soon as we + # make an update, since all closures for a class will share a + # given cell. + for member in newcls.__dict__.values(): + # If this is a wrapped function, unwrap it. + member = inspect.unwrap(member) + + if isinstance(member, types.FunctionType): + if _update_func_cell_for__class__(member, cls, newcls): + break + elif isinstance(member, property): + if (_update_func_cell_for__class__(member.fget, cls, newcls) + or _update_func_cell_for__class__(member.fset, cls, newcls) + or _update_func_cell_for__class__(member.fdel, cls, newcls)): + break + + # Get new annotations to remove references to the original class + # in forward references + newcls_ann = annotationlib.get_annotations( + newcls, format=annotationlib.Format.FORWARDREF) + + # Fix references in dataclass Fields + for f in getattr(newcls, _FIELDS).values(): + try: + ann = newcls_ann[f.name] + except KeyError: + pass + else: + f.type = ann - return cls + # Fix the class reference in the __annotate__ method + init = newcls.__init__ + if init_annotate := getattr(init, "__annotate__", None): + if getattr(init_annotate, "__generated_by_dataclasses__", False): + _update_func_cell_for__class__(init_annotate, cls, newcls) + + return newcls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, - kw_only=False, slots=False): - """Returns the same class as was passed in, with dunder methods - added based on the fields defined in the class. + kw_only=False, slots=False, weakref_slot=False): + """Add dunder methods based on the fields defined in the class. Examines PEP 526 __annotations__ to determine fields. - If init is true, an __init__() method is added to the class. If - repr is true, a __repr__() method is added. If order is true, rich + If init is true, an __init__() method is added to the class. If repr + is true, a __repr__() method is added. If order is true, rich comparison dunder methods are added. If unsafe_hash is true, a - __hash__() method function is added. If frozen is true, fields may - not be assigned to after instance creation. If match_args is true, - the __match_args__ tuple is added. If kw_only is true, then by - default all fields are keyword-only. If slots is true, an - __slots__ attribute is added. + __hash__() method is added. If frozen is true, fields may not be + assigned to after instance creation. If match_args is true, the + __match_args__ tuple is added. If kw_only is true, then by default + all fields are keyword-only. If slots is true, a new class with a + __slots__ attribute is returned. """ def wrap(cls): return _process_class(cls, init, repr, eq, order, unsafe_hash, - frozen, match_args, kw_only, slots) + frozen, match_args, kw_only, slots, + weakref_slot) # See if we're being called as @dataclass or @dataclass(). if cls is None: @@ -1195,7 +1453,7 @@ def fields(class_or_instance): try: fields = getattr(class_or_instance, _FIELDS) except AttributeError: - raise TypeError('must be called with a dataclass type or instance') + raise TypeError('must be called with a dataclass type or instance') from None # Exclude pseudo-fields. Note that fields is sorted by insertion # order, so the order of the tuple is as the fields were defined. @@ -1210,7 +1468,7 @@ def _is_dataclass_instance(obj): def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj) + cls = obj if isinstance(obj, type) else type(obj) return hasattr(cls, _FIELDS) @@ -1218,7 +1476,7 @@ def asdict(obj, *, dict_factory=dict): """Return the fields of a dataclass instance as a new dictionary mapping field names to field values. - Example usage: + Example usage:: @dataclass class C: @@ -1231,7 +1489,7 @@ class C: If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") @@ -1239,42 +1497,69 @@ class C: def _asdict_inner(obj, dict_factory): - if _is_dataclass_instance(obj): - result = [] - for f in fields(obj): - value = _asdict_inner(getattr(obj, f.name), dict_factory) - result.append((f.name, value)) - return dict_factory(result) - elif isinstance(obj, tuple) and hasattr(obj, '_fields'): - # obj is a namedtuple. Recurse into it, but the returned - # object is another namedtuple of the same type. This is - # similar to how other list- or tuple-derived classes are - # treated (see below), but we just need to create them - # differently because a namedtuple's __init__ needs to be - # called differently (see bpo-34363). - - # I'm not using namedtuple's _asdict() - # method, because: - # - it does not recurse in to the namedtuple fields and - # convert them to dicts (using dict_factory). - # - I don't actually want to return a dict here. The main - # use case here is json.dumps, and it handles converting - # namedtuples to lists. Admittedly we're losing some - # information here when we produce a json list instead of a - # dict. Note that if we returned dicts here instead of - # namedtuples, we could no longer call asdict() on a data - # structure where a namedtuple was used as a dict key. - - return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) - elif isinstance(obj, (list, tuple)): + obj_type = type(obj) + if obj_type in _ATOMIC_TYPES: + return obj + elif hasattr(obj_type, _FIELDS): + # dataclass instance: fast path for the common case + if dict_factory is dict: + return { + f.name: _asdict_inner(getattr(obj, f.name), dict) + for f in fields(obj) + } + else: + return dict_factory([ + (f.name, _asdict_inner(getattr(obj, f.name), dict_factory)) + for f in fields(obj) + ]) + # handle the builtin types first for speed; subclasses handled below + elif obj_type is list: + return [_asdict_inner(v, dict_factory) for v in obj] + elif obj_type is dict: + return { + _asdict_inner(k, dict_factory): _asdict_inner(v, dict_factory) + for k, v in obj.items() + } + elif obj_type is tuple: + return tuple([_asdict_inner(v, dict_factory) for v in obj]) + elif issubclass(obj_type, tuple): + if hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + + # I'm not using namedtuple's _asdict() + # method, because: + # - it does not recurse in to the namedtuple fields and + # convert them to dicts (using dict_factory). + # - I don't actually want to return a dict here. The main + # use case here is json.dumps, and it handles converting + # namedtuples to lists. Admittedly we're losing some + # information here when we produce a json list instead of a + # dict. Note that if we returned dicts here instead of + # namedtuples, we could no longer call asdict() on a data + # structure where a namedtuple was used as a dict key. + return obj_type(*[_asdict_inner(v, dict_factory) for v in obj]) + else: + return obj_type(_asdict_inner(v, dict_factory) for v in obj) + elif issubclass(obj_type, dict): + if hasattr(obj_type, 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = obj_type(obj.default_factory) + for k, v in obj.items(): + result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) + return result + return obj_type((_asdict_inner(k, dict_factory), + _asdict_inner(v, dict_factory)) + for k, v in obj.items()) + elif issubclass(obj_type, list): # Assume we can create an object of this type by passing in a - # generator (which is not true for namedtuples, handled - # above). - return type(obj)(_asdict_inner(v, dict_factory) for v in obj) - elif isinstance(obj, dict): - return type(obj)((_asdict_inner(k, dict_factory), - _asdict_inner(v, dict_factory)) - for k, v in obj.items()) + # generator + return obj_type(_asdict_inner(v, dict_factory) for v in obj) else: return copy.deepcopy(obj) @@ -1289,13 +1574,13 @@ class C: x: int y: int - c = C(1, 2) - assert astuple(c) == (1, 2) + c = C(1, 2) + assert astuple(c) == (1, 2) If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): @@ -1304,12 +1589,13 @@ class C: def _astuple_inner(obj, tuple_factory): - if _is_dataclass_instance(obj): - result = [] - for f in fields(obj): - value = _astuple_inner(getattr(obj, f.name), tuple_factory) - result.append(value) - return tuple_factory(result) + if type(obj) in _ATOMIC_TYPES: + return obj + elif _is_dataclass_instance(obj): + return tuple_factory([ + _astuple_inner(getattr(obj, f.name), tuple_factory) + for f in fields(obj) + ]) elif isinstance(obj, tuple) and hasattr(obj, '_fields'): # obj is a namedtuple. Recurse into it, but the returned # object is another namedtuple of the same type. This is @@ -1324,7 +1610,15 @@ def _astuple_inner(obj, tuple_factory): # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) elif isinstance(obj, dict): - return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + obj_type = type(obj) + if hasattr(obj_type, 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = obj_type(getattr(obj, 'default_factory')) + for k, v in obj.items(): + result[_astuple_inner(k, tuple_factory)] = _astuple_inner(v, tuple_factory) + return result + return obj_type((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) for k, v in obj.items()) else: return copy.deepcopy(obj) @@ -1332,17 +1626,18 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, - frozen=False, match_args=True, kw_only=False, slots=False): + frozen=False, match_args=True, kw_only=False, slots=False, + weakref_slot=False, module=None, decorator=dataclass): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable of either (name), (name, type) or (name, type, Field) objects. If type is omitted, use the string 'typing.Any'. Field objects are created by - the equivalent of calling 'field(name, type [, Field-info])'. + the equivalent of calling 'field(name, type [, Field-info])'.:: C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,)) - is equivalent to: + is equivalent to:: @dataclass class C(Base): @@ -1352,8 +1647,11 @@ class C(Base): For the bases and namespace parameters, see the builtin type() function. - The parameters init, repr, eq, order, unsafe_hash, and frozen are passed to - dataclass(). + The parameters init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, + slots, and weakref_slot are passed to dataclass(). + + If module parameter is defined, the '__module__' attribute of the dataclass is + set to that value. """ if namespace is None: @@ -1367,7 +1665,7 @@ class C(Base): for item in fields: if isinstance(item, str): name = item - tp = 'typing.Any' + tp = _ANY_MARKER elif len(item) == 2: name, tp, = item elif len(item) == 3: @@ -1386,26 +1684,77 @@ class C(Base): seen.add(name) annotations[name] = tp + # We initially block the VALUE format, because inside dataclass() we'll + # call get_annotations(), which will try the VALUE format first. If we don't + # block, that means we'd always end up eagerly importing typing here, which + # is what we're trying to avoid. + value_blocked = True + + def annotate_method(format): + def get_any(): + match format: + case annotationlib.Format.STRING: + return 'typing.Any' + case annotationlib.Format.FORWARDREF: + typing = sys.modules.get("typing") + if typing is None: + return annotationlib.ForwardRef("Any", module="typing") + else: + return typing.Any + case annotationlib.Format.VALUE: + if value_blocked: + raise NotImplementedError + from typing import Any + return Any + case _: + raise NotImplementedError + annos = { + ann: get_any() if t is _ANY_MARKER else t + for ann, t in annotations.items() + } + if format == annotationlib.Format.STRING: + return annotationlib.annotations_to_string(annos) + else: + return annos + # Update 'ns' with the user-supplied namespace plus our calculated values. def exec_body_callback(ns): ns.update(namespace) ns.update(defaults) - ns['__annotations__'] = annotations # We use `types.new_class()` instead of simply `type()` to allow dynamic creation # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) + # For now, set annotations including the _ANY_MARKER. + cls.__annotate__ = annotate_method - # Apply the normal decorator. - return dataclass(cls, init=init, repr=repr, eq=eq, order=order, - unsafe_hash=unsafe_hash, frozen=frozen, - match_args=match_args, kw_only=kw_only, slots=slots) + # For pickling to work, the __module__ variable needs to be set to the frame + # where the dataclass is created. + if module is None: + try: + module = sys._getframemodulename(1) or '__main__' + except AttributeError: + try: + module = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + cls.__module__ = module + + # Apply the normal provided decorator. + cls = decorator(cls, init=init, repr=repr, eq=eq, order=order, + unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, kw_only=kw_only, slots=slots, + weakref_slot=weakref_slot) + # Now that the class is ready, allow the VALUE format. + value_blocked = False + return cls def replace(obj, /, **changes): """Return a new object replacing specified fields with new values. - This is especially useful for frozen classes. Example usage: + This is especially useful for frozen classes. Example usage:: @dataclass(frozen=True) class C: @@ -1415,18 +1764,20 @@ class C: c = C(1, 2) c1 = replace(c, x=3) assert c1.x == 3 and c1.y == 2 - """ - - # We're going to mutate 'changes', but that's okay because it's a - # new dict, even if called with 'replace(obj, **my_changes)'. - + """ if not _is_dataclass_instance(obj): raise TypeError("replace() should be called on dataclass instances") + return _replace(obj, **changes) + + +def _replace(self, /, **changes): + # We're going to mutate 'changes', but that's okay because it's a + # new dict, even if called with 'replace(self, **my_changes)'. # It's an error to have init=False fields in 'changes'. - # If a field is not in 'changes', read its value from the provided obj. + # If a field is not in 'changes', read its value from the provided 'self'. - for f in getattr(obj, _FIELDS).values(): + for f in getattr(self, _FIELDS).values(): # Only consider normal fields or InitVars. if f._field_type is _FIELD_CLASSVAR: continue @@ -1434,20 +1785,20 @@ class C: if not f.init: # Error if this field is specified in changes. if f.name in changes: - raise ValueError(f'field {f.name} is declared with ' - 'init=False, it cannot be specified with ' - 'replace()') + raise TypeError(f'field {f.name} is declared with ' + f'init=False, it cannot be specified with ' + f'replace()') continue if f.name not in changes: if f._field_type is _FIELD_INITVAR and f.default is MISSING: - raise ValueError(f"InitVar {f.name!r} " - 'must be specified with replace()') - changes[f.name] = getattr(obj, f.name) + raise TypeError(f"InitVar {f.name!r} " + f'must be specified with replace()') + changes[f.name] = getattr(self, f.name) # Create the new object, which calls __init__() and # __post_init__() (if defined), using all of the init fields we've # added and/or left in 'changes'. If there are values supplied in # changes that aren't fields, this will correctly raise a # TypeError. - return obj.__class__(**changes) + return self.__class__(**changes) diff --git a/Lib/datetime.py b/Lib/datetime.py index 353e48b68ca..14f30556584 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1,2524 +1,13 @@ -"""Concrete date/time and related types. +"""Specific date/time and related types. -See http://www.iana.org/time-zones/repository/tz-link.html for +See https://data.iana.org/time-zones/tz-link.html for time zone and DST data sources. """ -__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", - "MINYEAR", "MAXYEAR", "UTC") - - -import time as _time -import math as _math -import sys -from operator import index as _index - -def _cmp(x, y): - return 0 if x == y else 1 if x > y else -1 - -MINYEAR = 1 -MAXYEAR = 9999 -_MAXORDINAL = 3652059 # date.max.toordinal() - -# Utility functions, adapted from Python's Demo/classes/Dates.py, which -# also assumes the current Gregorian calendar indefinitely extended in -# both directions. Difference: Dates.py calls January 1 of year 0 day -# number 1. The code here calls January 1 of year 1 day number 1. This is -# to match the definition of the "proleptic Gregorian" calendar in Dershowitz -# and Reingold's "Calendrical Calculations", where it's the base calendar -# for all computations. See the book for algorithms for converting between -# proleptic Gregorian ordinals and many other calendar systems. - -# -1 is a placeholder for indexing purposes. -_DAYS_IN_MONTH = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] - -_DAYS_BEFORE_MONTH = [-1] # -1 is a placeholder for indexing purposes. -dbm = 0 -for dim in _DAYS_IN_MONTH[1:]: - _DAYS_BEFORE_MONTH.append(dbm) - dbm += dim -del dbm, dim - -def _is_leap(year): - "year -> 1 if leap year, else 0." - return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0) - -def _days_before_year(year): - "year -> number of days before January 1st of year." - y = year - 1 - return y*365 + y//4 - y//100 + y//400 - -def _days_in_month(year, month): - "year, month -> number of days in that month in that year." - assert 1 <= month <= 12, month - if month == 2 and _is_leap(year): - return 29 - return _DAYS_IN_MONTH[month] - -def _days_before_month(year, month): - "year, month -> number of days in year preceding first day of month." - assert 1 <= month <= 12, 'month must be in 1..12' - return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year)) - -def _ymd2ord(year, month, day): - "year, month, day -> ordinal, considering 01-Jan-0001 as day 1." - assert 1 <= month <= 12, 'month must be in 1..12' - dim = _days_in_month(year, month) - assert 1 <= day <= dim, ('day must be in 1..%d' % dim) - return (_days_before_year(year) + - _days_before_month(year, month) + - day) - -_DI400Y = _days_before_year(401) # number of days in 400 years -_DI100Y = _days_before_year(101) # " " " " 100 " -_DI4Y = _days_before_year(5) # " " " " 4 " - -# A 4-year cycle has an extra leap day over what we'd get from pasting -# together 4 single years. -assert _DI4Y == 4 * 365 + 1 - -# Similarly, a 400-year cycle has an extra leap day over what we'd get from -# pasting together 4 100-year cycles. -assert _DI400Y == 4 * _DI100Y + 1 - -# OTOH, a 100-year cycle has one fewer leap day than we'd get from -# pasting together 25 4-year cycles. -assert _DI100Y == 25 * _DI4Y - 1 - -def _ord2ymd(n): - "ordinal -> (year, month, day), considering 01-Jan-0001 as day 1." - - # n is a 1-based index, starting at 1-Jan-1. The pattern of leap years - # repeats exactly every 400 years. The basic strategy is to find the - # closest 400-year boundary at or before n, then work with the offset - # from that boundary to n. Life is much clearer if we subtract 1 from - # n first -- then the values of n at 400-year boundaries are exactly - # those divisible by _DI400Y: - # - # D M Y n n-1 - # -- --- ---- ---------- ---------------- - # 31 Dec -400 -_DI400Y -_DI400Y -1 - # 1 Jan -399 -_DI400Y +1 -_DI400Y 400-year boundary - # ... - # 30 Dec 000 -1 -2 - # 31 Dec 000 0 -1 - # 1 Jan 001 1 0 400-year boundary - # 2 Jan 001 2 1 - # 3 Jan 001 3 2 - # ... - # 31 Dec 400 _DI400Y _DI400Y -1 - # 1 Jan 401 _DI400Y +1 _DI400Y 400-year boundary - n -= 1 - n400, n = divmod(n, _DI400Y) - year = n400 * 400 + 1 # ..., -399, 1, 401, ... - - # Now n is the (non-negative) offset, in days, from January 1 of year, to - # the desired date. Now compute how many 100-year cycles precede n. - # Note that it's possible for n100 to equal 4! In that case 4 full - # 100-year cycles precede the desired day, which implies the desired - # day is December 31 at the end of a 400-year cycle. - n100, n = divmod(n, _DI100Y) - - # Now compute how many 4-year cycles precede it. - n4, n = divmod(n, _DI4Y) - - # And now how many single years. Again n1 can be 4, and again meaning - # that the desired day is December 31 at the end of the 4-year cycle. - n1, n = divmod(n, 365) - - year += n100 * 100 + n4 * 4 + n1 - if n1 == 4 or n100 == 4: - assert n == 0 - return year-1, 12, 31 - - # Now the year is correct, and n is the offset from January 1. We find - # the month via an estimate that's either exact or one too large. - leapyear = n1 == 3 and (n4 != 24 or n100 == 3) - assert leapyear == _is_leap(year) - month = (n + 50) >> 5 - preceding = _DAYS_BEFORE_MONTH[month] + (month > 2 and leapyear) - if preceding > n: # estimate is too large - month -= 1 - preceding -= _DAYS_IN_MONTH[month] + (month == 2 and leapyear) - n -= preceding - assert 0 <= n < _days_in_month(year, month) - - # Now the year and month are correct, and n is the offset from the - # start of that month: we're done! - return year, month, n+1 - -# Month and day names. For localized versions, see the calendar module. -_MONTHNAMES = [None, "Jan", "Feb", "Mar", "Apr", "May", "Jun", - "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] -_DAYNAMES = [None, "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] - - -def _build_struct_time(y, m, d, hh, mm, ss, dstflag): - wday = (_ymd2ord(y, m, d) + 6) % 7 - dnum = _days_before_month(y, m) + d - return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) - -def _format_time(hh, mm, ss, us, timespec='auto'): - specs = { - 'hours': '{:02d}', - 'minutes': '{:02d}:{:02d}', - 'seconds': '{:02d}:{:02d}:{:02d}', - 'milliseconds': '{:02d}:{:02d}:{:02d}.{:03d}', - 'microseconds': '{:02d}:{:02d}:{:02d}.{:06d}' - } - - if timespec == 'auto': - # Skip trailing microseconds when us==0. - timespec = 'microseconds' if us else 'seconds' - elif timespec == 'milliseconds': - us //= 1000 - try: - fmt = specs[timespec] - except KeyError: - raise ValueError('Unknown timespec value') - else: - return fmt.format(hh, mm, ss, us) - -def _format_offset(off): - s = '' - if off is not None: - if off.days < 0: - sign = "-" - off = -off - else: - sign = "+" - hh, mm = divmod(off, timedelta(hours=1)) - mm, ss = divmod(mm, timedelta(minutes=1)) - s += "%s%02d:%02d" % (sign, hh, mm) - if ss or ss.microseconds: - s += ":%02d" % ss.seconds - - if ss.microseconds: - s += '.%06d' % ss.microseconds - return s - -# Correctly substitute for %z and %Z escapes in strftime formats. -def _wrap_strftime(object, format, timetuple): - # Don't call utcoffset() or tzname() unless actually needed. - freplace = None # the string to use for %f - zreplace = None # the string to use for %z - Zreplace = None # the string to use for %Z - - # Scan format for %z and %Z escapes, replacing as needed. - newformat = [] - push = newformat.append - i, n = 0, len(format) - while i < n: - ch = format[i] - i += 1 - if ch == '%': - if i < n: - ch = format[i] - i += 1 - if ch == 'f': - if freplace is None: - freplace = '%06d' % getattr(object, - 'microsecond', 0) - newformat.append(freplace) - elif ch == 'z': - if zreplace is None: - zreplace = "" - if hasattr(object, "utcoffset"): - offset = object.utcoffset() - if offset is not None: - sign = '+' - if offset.days < 0: - offset = -offset - sign = '-' - h, rest = divmod(offset, timedelta(hours=1)) - m, rest = divmod(rest, timedelta(minutes=1)) - s = rest.seconds - u = offset.microseconds - if u: - zreplace = '%c%02d%02d%02d.%06d' % (sign, h, m, s, u) - elif s: - zreplace = '%c%02d%02d%02d' % (sign, h, m, s) - else: - zreplace = '%c%02d%02d' % (sign, h, m) - assert '%' not in zreplace - newformat.append(zreplace) - elif ch == 'Z': - if Zreplace is None: - Zreplace = "" - if hasattr(object, "tzname"): - s = object.tzname() - if s is not None: - # strftime is going to have at this: escape % - Zreplace = s.replace('%', '%%') - newformat.append(Zreplace) - else: - push('%') - push(ch) - else: - push('%') - else: - push(ch) - newformat = "".join(newformat) - return _time.strftime(newformat, timetuple) - -# Helpers for parsing the result of isoformat() -def _parse_isoformat_date(dtstr): - # It is assumed that this function will only be called with a - # string of length exactly 10, and (though this is not used) ASCII-only - year = int(dtstr[0:4]) - if dtstr[4] != '-': - raise ValueError('Invalid date separator: %s' % dtstr[4]) - - month = int(dtstr[5:7]) - - if dtstr[7] != '-': - raise ValueError('Invalid date separator') - - day = int(dtstr[8:10]) - - return [year, month, day] - -def _parse_hh_mm_ss_ff(tstr): - # Parses things of the form HH[:MM[:SS[.fff[fff]]]] - len_str = len(tstr) - - time_comps = [0, 0, 0, 0] - pos = 0 - for comp in range(0, 3): - if (len_str - pos) < 2: - raise ValueError('Incomplete time component') - - time_comps[comp] = int(tstr[pos:pos+2]) - - pos += 2 - next_char = tstr[pos:pos+1] - - if not next_char or comp >= 2: - break - - if next_char != ':': - raise ValueError('Invalid time separator: %c' % next_char) - - pos += 1 - - if pos < len_str: - if tstr[pos] != '.': - raise ValueError('Invalid microsecond component') - else: - pos += 1 - - len_remainder = len_str - pos - if len_remainder not in (3, 6): - raise ValueError('Invalid microsecond component') - - time_comps[3] = int(tstr[pos:]) - if len_remainder == 3: - time_comps[3] *= 1000 - - return time_comps - -def _parse_isoformat_time(tstr): - # Format supported is HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]] - len_str = len(tstr) - if len_str < 2: - raise ValueError('Isoformat time too short') - - # This is equivalent to re.search('[+-]', tstr), but faster - tz_pos = (tstr.find('-') + 1 or tstr.find('+') + 1) - timestr = tstr[:tz_pos-1] if tz_pos > 0 else tstr - - time_comps = _parse_hh_mm_ss_ff(timestr) - - tzi = None - if tz_pos > 0: - tzstr = tstr[tz_pos:] - - # Valid time zone strings are: - # HH:MM len: 5 - # HH:MM:SS len: 8 - # HH:MM:SS.ffffff len: 15 - - if len(tzstr) not in (5, 8, 15): - raise ValueError('Malformed time zone string') - - tz_comps = _parse_hh_mm_ss_ff(tzstr) - if all(x == 0 for x in tz_comps): - tzi = timezone.utc - else: - tzsign = -1 if tstr[tz_pos - 1] == '-' else 1 - - td = timedelta(hours=tz_comps[0], minutes=tz_comps[1], - seconds=tz_comps[2], microseconds=tz_comps[3]) - - tzi = timezone(tzsign * td) - - time_comps.append(tzi) - - return time_comps - - -# Just raise TypeError if the arg isn't None or a string. -def _check_tzname(name): - if name is not None and not isinstance(name, str): - raise TypeError("tzinfo.tzname() must return None or string, " - "not '%s'" % type(name)) - -# name is the offset-producing method, "utcoffset" or "dst". -# offset is what it returned. -# If offset isn't None or timedelta, raises TypeError. -# If offset is None, returns None. -# Else offset is checked for being in range. -# If it is, its integer value is returned. Else ValueError is raised. -def _check_utc_offset(name, offset): - assert name in ("utcoffset", "dst") - if offset is None: - return - if not isinstance(offset, timedelta): - raise TypeError("tzinfo.%s() must return None " - "or timedelta, not '%s'" % (name, type(offset))) - if not -timedelta(1) < offset < timedelta(1): - raise ValueError("%s()=%s, must be strictly between " - "-timedelta(hours=24) and timedelta(hours=24)" % - (name, offset)) - -def _check_date_fields(year, month, day): - year = _index(year) - month = _index(month) - day = _index(day) - if not MINYEAR <= year <= MAXYEAR: - raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) - if not 1 <= month <= 12: - raise ValueError('month must be in 1..12', month) - dim = _days_in_month(year, month) - if not 1 <= day <= dim: - raise ValueError('day must be in 1..%d' % dim, day) - return year, month, day - -def _check_time_fields(hour, minute, second, microsecond, fold): - hour = _index(hour) - minute = _index(minute) - second = _index(second) - microsecond = _index(microsecond) - if not 0 <= hour <= 23: - raise ValueError('hour must be in 0..23', hour) - if not 0 <= minute <= 59: - raise ValueError('minute must be in 0..59', minute) - if not 0 <= second <= 59: - raise ValueError('second must be in 0..59', second) - if not 0 <= microsecond <= 999999: - raise ValueError('microsecond must be in 0..999999', microsecond) - if fold not in (0, 1): - raise ValueError('fold must be either 0 or 1', fold) - return hour, minute, second, microsecond, fold - -def _check_tzinfo_arg(tz): - if tz is not None and not isinstance(tz, tzinfo): - raise TypeError("tzinfo argument must be None or of a tzinfo subclass") - -def _cmperror(x, y): - raise TypeError("can't compare '%s' to '%s'" % ( - type(x).__name__, type(y).__name__)) - -def _divide_and_round(a, b): - """divide a by b and round result to the nearest integer - - When the ratio is exactly half-way between two integers, - the even integer is returned. - """ - # Based on the reference implementation for divmod_near - # in Objects/longobject.c. - q, r = divmod(a, b) - # round up if either r / b > 0.5, or r / b == 0.5 and q is odd. - # The expression r / b > 0.5 is equivalent to 2 * r > b if b is - # positive, 2 * r < b if b negative. - r *= 2 - greater_than_half = r > b if b > 0 else r < b - if greater_than_half or r == b and q % 2 == 1: - q += 1 - - return q - - -class timedelta: - """Represent the difference between two datetime objects. - - Supported operators: - - - add, subtract timedelta - - unary plus, minus, abs - - compare to timedelta - - multiply, divide by int - - In addition, datetime supports subtraction of two datetime objects - returning a timedelta, and addition or subtraction of a datetime - and a timedelta giving a datetime. - - Representation: (days, seconds, microseconds). Why? Because I - felt like it. - """ - __slots__ = '_days', '_seconds', '_microseconds', '_hashcode' - - def __new__(cls, days=0, seconds=0, microseconds=0, - milliseconds=0, minutes=0, hours=0, weeks=0): - # Doing this efficiently and accurately in C is going to be difficult - # and error-prone, due to ubiquitous overflow possibilities, and that - # C double doesn't have enough bits of precision to represent - # microseconds over 10K years faithfully. The code here tries to make - # explicit where go-fast assumptions can be relied on, in order to - # guide the C implementation; it's way more convoluted than speed- - # ignoring auto-overflow-to-long idiomatic Python could be. - - # XXX Check that all inputs are ints or floats. - - # Final values, all integer. - # s and us fit in 32-bit signed ints; d isn't bounded. - d = s = us = 0 - - # Normalize everything to days, seconds, microseconds. - days += weeks*7 - seconds += minutes*60 + hours*3600 - microseconds += milliseconds*1000 - - # Get rid of all fractions, and normalize s and us. - # Take a deep breath . - if isinstance(days, float): - dayfrac, days = _math.modf(days) - daysecondsfrac, daysecondswhole = _math.modf(dayfrac * (24.*3600.)) - assert daysecondswhole == int(daysecondswhole) # can't overflow - s = int(daysecondswhole) - assert days == int(days) - d = int(days) - else: - daysecondsfrac = 0.0 - d = days - assert isinstance(daysecondsfrac, float) - assert abs(daysecondsfrac) <= 1.0 - assert isinstance(d, int) - assert abs(s) <= 24 * 3600 - # days isn't referenced again before redefinition - - if isinstance(seconds, float): - secondsfrac, seconds = _math.modf(seconds) - assert seconds == int(seconds) - seconds = int(seconds) - secondsfrac += daysecondsfrac - assert abs(secondsfrac) <= 2.0 - else: - secondsfrac = daysecondsfrac - # daysecondsfrac isn't referenced again - assert isinstance(secondsfrac, float) - assert abs(secondsfrac) <= 2.0 - - assert isinstance(seconds, int) - days, seconds = divmod(seconds, 24*3600) - d += days - s += int(seconds) # can't overflow - assert isinstance(s, int) - assert abs(s) <= 2 * 24 * 3600 - # seconds isn't referenced again before redefinition - - usdouble = secondsfrac * 1e6 - assert abs(usdouble) < 2.1e6 # exact value not critical - # secondsfrac isn't referenced again - - if isinstance(microseconds, float): - microseconds = round(microseconds + usdouble) - seconds, microseconds = divmod(microseconds, 1000000) - days, seconds = divmod(seconds, 24*3600) - d += days - s += seconds - else: - microseconds = int(microseconds) - seconds, microseconds = divmod(microseconds, 1000000) - days, seconds = divmod(seconds, 24*3600) - d += days - s += seconds - microseconds = round(microseconds + usdouble) - assert isinstance(s, int) - assert isinstance(microseconds, int) - assert abs(s) <= 3 * 24 * 3600 - assert abs(microseconds) < 3.1e6 - - # Just a little bit of carrying possible for microseconds and seconds. - seconds, us = divmod(microseconds, 1000000) - s += seconds - days, s = divmod(s, 24*3600) - d += days - - assert isinstance(d, int) - assert isinstance(s, int) and 0 <= s < 24*3600 - assert isinstance(us, int) and 0 <= us < 1000000 - - if abs(d) > 999999999: - raise OverflowError("timedelta # of days is too large: %d" % d) - - self = object.__new__(cls) - self._days = d - self._seconds = s - self._microseconds = us - self._hashcode = -1 - return self - - def __repr__(self): - args = [] - if self._days: - args.append("days=%d" % self._days) - if self._seconds: - args.append("seconds=%d" % self._seconds) - if self._microseconds: - args.append("microseconds=%d" % self._microseconds) - if not args: - args.append('0') - return "%s.%s(%s)" % (self.__class__.__module__, - self.__class__.__qualname__, - ', '.join(args)) - - def __str__(self): - mm, ss = divmod(self._seconds, 60) - hh, mm = divmod(mm, 60) - s = "%d:%02d:%02d" % (hh, mm, ss) - if self._days: - def plural(n): - return n, abs(n) != 1 and "s" or "" - s = ("%d day%s, " % plural(self._days)) + s - if self._microseconds: - s = s + ".%06d" % self._microseconds - return s - - def total_seconds(self): - """Total seconds in the duration.""" - return ((self.days * 86400 + self.seconds) * 10**6 + - self.microseconds) / 10**6 - - # Read-only field accessors - @property - def days(self): - """days""" - return self._days - - @property - def seconds(self): - """seconds""" - return self._seconds - - @property - def microseconds(self): - """microseconds""" - return self._microseconds - - def __add__(self, other): - if isinstance(other, timedelta): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(self._days + other._days, - self._seconds + other._seconds, - self._microseconds + other._microseconds) - return NotImplemented - - __radd__ = __add__ - - def __sub__(self, other): - if isinstance(other, timedelta): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(self._days - other._days, - self._seconds - other._seconds, - self._microseconds - other._microseconds) - return NotImplemented - - def __rsub__(self, other): - if isinstance(other, timedelta): - return -self + other - return NotImplemented - - def __neg__(self): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(-self._days, - -self._seconds, - -self._microseconds) - - def __pos__(self): - return self - - def __abs__(self): - if self._days < 0: - return -self - else: - return self - - def __mul__(self, other): - if isinstance(other, int): - # for CPython compatibility, we cannot use - # our __class__ here, but need a real timedelta - return timedelta(self._days * other, - self._seconds * other, - self._microseconds * other) - if isinstance(other, float): - usec = self._to_microseconds() - a, b = other.as_integer_ratio() - return timedelta(0, 0, _divide_and_round(usec * a, b)) - return NotImplemented - - __rmul__ = __mul__ - - def _to_microseconds(self): - return ((self._days * (24*3600) + self._seconds) * 1000000 + - self._microseconds) - - def __floordiv__(self, other): - if not isinstance(other, (int, timedelta)): - return NotImplemented - usec = self._to_microseconds() - if isinstance(other, timedelta): - return usec // other._to_microseconds() - if isinstance(other, int): - return timedelta(0, 0, usec // other) - - def __truediv__(self, other): - if not isinstance(other, (int, float, timedelta)): - return NotImplemented - usec = self._to_microseconds() - if isinstance(other, timedelta): - return usec / other._to_microseconds() - if isinstance(other, int): - return timedelta(0, 0, _divide_and_round(usec, other)) - if isinstance(other, float): - a, b = other.as_integer_ratio() - return timedelta(0, 0, _divide_and_round(b * usec, a)) - - def __mod__(self, other): - if isinstance(other, timedelta): - r = self._to_microseconds() % other._to_microseconds() - return timedelta(0, 0, r) - return NotImplemented - - def __divmod__(self, other): - if isinstance(other, timedelta): - q, r = divmod(self._to_microseconds(), - other._to_microseconds()) - return q, timedelta(0, 0, r) - return NotImplemented - - # Comparisons of timedelta objects with other. - - def __eq__(self, other): - if isinstance(other, timedelta): - return self._cmp(other) == 0 - else: - return NotImplemented - - def __le__(self, other): - if isinstance(other, timedelta): - return self._cmp(other) <= 0 - else: - return NotImplemented - - def __lt__(self, other): - if isinstance(other, timedelta): - return self._cmp(other) < 0 - else: - return NotImplemented - - def __ge__(self, other): - if isinstance(other, timedelta): - return self._cmp(other) >= 0 - else: - return NotImplemented - - def __gt__(self, other): - if isinstance(other, timedelta): - return self._cmp(other) > 0 - else: - return NotImplemented - - def _cmp(self, other): - assert isinstance(other, timedelta) - return _cmp(self._getstate(), other._getstate()) - - def __hash__(self): - if self._hashcode == -1: - self._hashcode = hash(self._getstate()) - return self._hashcode - - def __bool__(self): - return (self._days != 0 or - self._seconds != 0 or - self._microseconds != 0) - - # Pickle support. - - def _getstate(self): - return (self._days, self._seconds, self._microseconds) - - def __reduce__(self): - return (self.__class__, self._getstate()) - -timedelta.min = timedelta(-999999999) -timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59, - microseconds=999999) -timedelta.resolution = timedelta(microseconds=1) - -class date: - """Concrete date type. - - Constructors: - - __new__() - fromtimestamp() - today() - fromordinal() - - Operators: - - __repr__, __str__ - __eq__, __le__, __lt__, __ge__, __gt__, __hash__ - __add__, __radd__, __sub__ (add/radd only with timedelta arg) - - Methods: - - timetuple() - toordinal() - weekday() - isoweekday(), isocalendar(), isoformat() - ctime() - strftime() - - Properties (readonly): - year, month, day - """ - __slots__ = '_year', '_month', '_day', '_hashcode' - - def __new__(cls, year, month=None, day=None): - """Constructor. - - Arguments: - - year, month, day (required, base 1) - """ - if (month is None and - isinstance(year, (bytes, str)) and len(year) == 4 and - 1 <= ord(year[2:3]) <= 12): - # Pickle support - if isinstance(year, str): - try: - year = year.encode('latin1') - except UnicodeEncodeError: - # More informative error message. - raise ValueError( - "Failed to encode latin1 string when unpickling " - "a date object. " - "pickle.load(data, encoding='latin1') is assumed.") - self = object.__new__(cls) - self.__setstate(year) - self._hashcode = -1 - return self - year, month, day = _check_date_fields(year, month, day) - self = object.__new__(cls) - self._year = year - self._month = month - self._day = day - self._hashcode = -1 - return self - - # Additional constructors - - @classmethod - def fromtimestamp(cls, t): - "Construct a date from a POSIX timestamp (like time.time())." - y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) - return cls(y, m, d) - - @classmethod - def today(cls): - "Construct a date from time.time()." - t = _time.time() - return cls.fromtimestamp(t) - - @classmethod - def fromordinal(cls, n): - """Construct a date from a proleptic Gregorian ordinal. - - January 1 of year 1 is day 1. Only the year, month and day are - non-zero in the result. - """ - y, m, d = _ord2ymd(n) - return cls(y, m, d) - - @classmethod - def fromisoformat(cls, date_string): - """Construct a date from the output of date.isoformat().""" - if not isinstance(date_string, str): - raise TypeError('fromisoformat: argument must be str') - - try: - assert len(date_string) == 10 - return cls(*_parse_isoformat_date(date_string)) - except Exception: - raise ValueError(f'Invalid isoformat string: {date_string!r}') - - @classmethod - def fromisocalendar(cls, year, week, day): - """Construct a date from the ISO year, week number and weekday. - - This is the inverse of the date.isocalendar() function""" - # Year is bounded this way because 9999-12-31 is (9999, 52, 5) - if not MINYEAR <= year <= MAXYEAR: - raise ValueError(f"Year is out of range: {year}") - - if not 0 < week < 53: - out_of_range = True - - if week == 53: - # ISO years have 53 weeks in them on years starting with a - # Thursday and leap years starting on a Wednesday - first_weekday = _ymd2ord(year, 1, 1) % 7 - if (first_weekday == 4 or (first_weekday == 3 and - _is_leap(year))): - out_of_range = False - - if out_of_range: - raise ValueError(f"Invalid week: {week}") - - if not 0 < day < 8: - raise ValueError(f"Invalid weekday: {day} (range is [1, 7])") - - # Now compute the offset from (Y, 1, 1) in days: - day_offset = (week - 1) * 7 + (day - 1) - - # Calculate the ordinal day for monday, week 1 - day_1 = _isoweek1monday(year) - ord_day = day_1 + day_offset - - return cls(*_ord2ymd(ord_day)) - - # Conversions to string - - def __repr__(self): - """Convert to formal string, for repr(). - - >>> dt = datetime(2010, 1, 1) - >>> repr(dt) - 'datetime.datetime(2010, 1, 1, 0, 0)' - - >>> dt = datetime(2010, 1, 1, tzinfo=timezone.utc) - >>> repr(dt) - 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' - """ - return "%s.%s(%d, %d, %d)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._year, - self._month, - self._day) - # XXX These shouldn't depend on time.localtime(), because that - # clips the usable dates to [1970 .. 2038). At least ctime() is - # easily done without using strftime() -- that's better too because - # strftime("%c", ...) is locale specific. - - - def ctime(self): - "Return ctime() style string." - weekday = self.toordinal() % 7 or 7 - return "%s %s %2d 00:00:00 %04d" % ( - _DAYNAMES[weekday], - _MONTHNAMES[self._month], - self._day, self._year) - - def strftime(self, fmt): - "Format using strftime()." - return _wrap_strftime(self, fmt, self.timetuple()) - - def __format__(self, fmt): - if not isinstance(fmt, str): - raise TypeError("must be str, not %s" % type(fmt).__name__) - if len(fmt) != 0: - return self.strftime(fmt) - return str(self) - - def isoformat(self): - """Return the date formatted according to ISO. - - This is 'YYYY-MM-DD'. - - References: - - http://www.w3.org/TR/NOTE-datetime - - http://www.cl.cam.ac.uk/~mgk25/iso-time.html - """ - return "%04d-%02d-%02d" % (self._year, self._month, self._day) - - __str__ = isoformat - - # Read-only field accessors - @property - def year(self): - """year (1-9999)""" - return self._year - - @property - def month(self): - """month (1-12)""" - return self._month - - @property - def day(self): - """day (1-31)""" - return self._day - - # Standard conversions, __eq__, __le__, __lt__, __ge__, __gt__, - # __hash__ (and helpers) - - def timetuple(self): - "Return local time tuple compatible with time.localtime()." - return _build_struct_time(self._year, self._month, self._day, - 0, 0, 0, -1) - - def toordinal(self): - """Return proleptic Gregorian ordinal for the year, month and day. - - January 1 of year 1 is day 1. Only the year, month and day values - contribute to the result. - """ - return _ymd2ord(self._year, self._month, self._day) - - def replace(self, year=None, month=None, day=None): - """Return a new date with new values for the specified fields.""" - if year is None: - year = self._year - if month is None: - month = self._month - if day is None: - day = self._day - return type(self)(year, month, day) - - # Comparisons of date objects with other. - - def __eq__(self, other): - if isinstance(other, date): - return self._cmp(other) == 0 - return NotImplemented - - def __le__(self, other): - if isinstance(other, date): - return self._cmp(other) <= 0 - return NotImplemented - - def __lt__(self, other): - if isinstance(other, date): - return self._cmp(other) < 0 - return NotImplemented - - def __ge__(self, other): - if isinstance(other, date): - return self._cmp(other) >= 0 - return NotImplemented - - def __gt__(self, other): - if isinstance(other, date): - return self._cmp(other) > 0 - return NotImplemented - - def _cmp(self, other): - assert isinstance(other, date) - y, m, d = self._year, self._month, self._day - y2, m2, d2 = other._year, other._month, other._day - return _cmp((y, m, d), (y2, m2, d2)) - - def __hash__(self): - "Hash." - if self._hashcode == -1: - self._hashcode = hash(self._getstate()) - return self._hashcode - - # Computations - - def __add__(self, other): - "Add a date to a timedelta." - if isinstance(other, timedelta): - o = self.toordinal() + other.days - if 0 < o <= _MAXORDINAL: - return type(self).fromordinal(o) - raise OverflowError("result out of range") - return NotImplemented - - __radd__ = __add__ - - def __sub__(self, other): - """Subtract two dates, or a date and a timedelta.""" - if isinstance(other, timedelta): - return self + timedelta(-other.days) - if isinstance(other, date): - days1 = self.toordinal() - days2 = other.toordinal() - return timedelta(days1 - days2) - return NotImplemented - - def weekday(self): - "Return day of the week, where Monday == 0 ... Sunday == 6." - return (self.toordinal() + 6) % 7 - - # Day-of-the-week and week-of-the-year, according to ISO - - def isoweekday(self): - "Return day of the week, where Monday == 1 ... Sunday == 7." - # 1-Jan-0001 is a Monday - return self.toordinal() % 7 or 7 - - def isocalendar(self): - """Return a named tuple containing ISO year, week number, and weekday. - - The first ISO week of the year is the (Mon-Sun) week - containing the year's first Thursday; everything else derives - from that. - - The first week is 1; Monday is 1 ... Sunday is 7. - - ISO calendar algorithm taken from - http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm - (used with permission) - """ - year = self._year - week1monday = _isoweek1monday(year) - today = _ymd2ord(self._year, self._month, self._day) - # Internally, week and day have origin 0 - week, day = divmod(today - week1monday, 7) - if week < 0: - year -= 1 - week1monday = _isoweek1monday(year) - week, day = divmod(today - week1monday, 7) - elif week >= 52: - if today >= _isoweek1monday(year+1): - year += 1 - week = 0 - return _IsoCalendarDate(year, week+1, day+1) - - # Pickle support. - - def _getstate(self): - yhi, ylo = divmod(self._year, 256) - return bytes([yhi, ylo, self._month, self._day]), - - def __setstate(self, string): - yhi, ylo, self._month, self._day = string - self._year = yhi * 256 + ylo - - def __reduce__(self): - return (self.__class__, self._getstate()) - -_date_class = date # so functions w/ args named "date" can get at the class - -date.min = date(1, 1, 1) -date.max = date(9999, 12, 31) -date.resolution = timedelta(days=1) - - -class tzinfo: - """Abstract base class for time zone info classes. - - Subclasses must override the name(), utcoffset() and dst() methods. - """ - __slots__ = () - - def tzname(self, dt): - "datetime -> string name of time zone." - raise NotImplementedError("tzinfo subclass must override tzname()") - - def utcoffset(self, dt): - "datetime -> timedelta, positive for east of UTC, negative for west of UTC" - raise NotImplementedError("tzinfo subclass must override utcoffset()") - - def dst(self, dt): - """datetime -> DST offset as timedelta, positive for east of UTC. - - Return 0 if DST not in effect. utcoffset() must include the DST - offset. - """ - raise NotImplementedError("tzinfo subclass must override dst()") - - def fromutc(self, dt): - "datetime in UTC -> datetime in local time." - - if not isinstance(dt, datetime): - raise TypeError("fromutc() requires a datetime argument") - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") - - dtoff = dt.utcoffset() - if dtoff is None: - raise ValueError("fromutc() requires a non-None utcoffset() " - "result") - - # See the long comment block at the end of this file for an - # explanation of this algorithm. - dtdst = dt.dst() - if dtdst is None: - raise ValueError("fromutc() requires a non-None dst() result") - delta = dtoff - dtdst - if delta: - dt += delta - dtdst = dt.dst() - if dtdst is None: - raise ValueError("fromutc(): dt.dst gave inconsistent " - "results; cannot convert") - return dt + dtdst - - # Pickle support. - - def __reduce__(self): - getinitargs = getattr(self, "__getinitargs__", None) - if getinitargs: - args = getinitargs() - else: - args = () - getstate = getattr(self, "__getstate__", None) - if getstate: - state = getstate() - else: - state = getattr(self, "__dict__", None) or None - if state is None: - return (self.__class__, args) - else: - return (self.__class__, args, state) - - -class IsoCalendarDate(tuple): - - def __new__(cls, year, week, weekday, /): - return super().__new__(cls, (year, week, weekday)) - - @property - def year(self): - return self[0] - - @property - def week(self): - return self[1] - - @property - def weekday(self): - return self[2] - - def __reduce__(self): - # This code is intended to pickle the object without making the - # class public. See https://bugs.python.org/msg352381 - return (tuple, (tuple(self),)) - - def __repr__(self): - return (f'{self.__class__.__name__}' - f'(year={self[0]}, week={self[1]}, weekday={self[2]})') - - -_IsoCalendarDate = IsoCalendarDate -del IsoCalendarDate -_tzinfo_class = tzinfo - -class time: - """Time with time zone. - - Constructors: - - __new__() - - Operators: - - __repr__, __str__ - __eq__, __le__, __lt__, __ge__, __gt__, __hash__ - - Methods: - - strftime() - isoformat() - utcoffset() - tzname() - dst() - - Properties (readonly): - hour, minute, second, microsecond, tzinfo, fold - """ - __slots__ = '_hour', '_minute', '_second', '_microsecond', '_tzinfo', '_hashcode', '_fold' - - def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0): - """Constructor. - - Arguments: - - hour, minute (required) - second, microsecond (default to zero) - tzinfo (default to None) - fold (keyword only, default to zero) - """ - if (isinstance(hour, (bytes, str)) and len(hour) == 6 and - ord(hour[0:1])&0x7F < 24): - # Pickle support - if isinstance(hour, str): - try: - hour = hour.encode('latin1') - except UnicodeEncodeError: - # More informative error message. - raise ValueError( - "Failed to encode latin1 string when unpickling " - "a time object. " - "pickle.load(data, encoding='latin1') is assumed.") - self = object.__new__(cls) - self.__setstate(hour, minute or None) - self._hashcode = -1 - return self - hour, minute, second, microsecond, fold = _check_time_fields( - hour, minute, second, microsecond, fold) - _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) - self._hour = hour - self._minute = minute - self._second = second - self._microsecond = microsecond - self._tzinfo = tzinfo - self._hashcode = -1 - self._fold = fold - return self - - # Read-only field accessors - @property - def hour(self): - """hour (0-23)""" - return self._hour - - @property - def minute(self): - """minute (0-59)""" - return self._minute - - @property - def second(self): - """second (0-59)""" - return self._second - - @property - def microsecond(self): - """microsecond (0-999999)""" - return self._microsecond - - @property - def tzinfo(self): - """timezone info object""" - return self._tzinfo - - @property - def fold(self): - return self._fold - - # Standard conversions, __hash__ (and helpers) - - # Comparisons of time objects with other. - - def __eq__(self, other): - if isinstance(other, time): - return self._cmp(other, allow_mixed=True) == 0 - else: - return NotImplemented - - def __le__(self, other): - if isinstance(other, time): - return self._cmp(other) <= 0 - else: - return NotImplemented - - def __lt__(self, other): - if isinstance(other, time): - return self._cmp(other) < 0 - else: - return NotImplemented - - def __ge__(self, other): - if isinstance(other, time): - return self._cmp(other) >= 0 - else: - return NotImplemented - - def __gt__(self, other): - if isinstance(other, time): - return self._cmp(other) > 0 - else: - return NotImplemented - - def _cmp(self, other, allow_mixed=False): - assert isinstance(other, time) - mytz = self._tzinfo - ottz = other._tzinfo - myoff = otoff = None - - if mytz is ottz: - base_compare = True - else: - myoff = self.utcoffset() - otoff = other.utcoffset() - base_compare = myoff == otoff - - if base_compare: - return _cmp((self._hour, self._minute, self._second, - self._microsecond), - (other._hour, other._minute, other._second, - other._microsecond)) - if myoff is None or otoff is None: - if allow_mixed: - return 2 # arbitrary non-zero value - else: - raise TypeError("cannot compare naive and aware times") - myhhmm = self._hour * 60 + self._minute - myoff//timedelta(minutes=1) - othhmm = other._hour * 60 + other._minute - otoff//timedelta(minutes=1) - return _cmp((myhhmm, self._second, self._microsecond), - (othhmm, other._second, other._microsecond)) - - def __hash__(self): - """Hash.""" - if self._hashcode == -1: - if self.fold: - t = self.replace(fold=0) - else: - t = self - tzoff = t.utcoffset() - if not tzoff: # zero or None - self._hashcode = hash(t._getstate()[0]) - else: - h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff, - timedelta(hours=1)) - assert not m % timedelta(minutes=1), "whole minute" - m //= timedelta(minutes=1) - if 0 <= h < 24: - self._hashcode = hash(time(h, m, self.second, self.microsecond)) - else: - self._hashcode = hash((h, m, self.second, self.microsecond)) - return self._hashcode - - # Conversion to string - - def _tzstr(self): - """Return formatted timezone offset (+xx:xx) or an empty string.""" - off = self.utcoffset() - return _format_offset(off) - - def __repr__(self): - """Convert to formal string, for repr().""" - if self._microsecond != 0: - s = ", %d, %d" % (self._second, self._microsecond) - elif self._second != 0: - s = ", %d" % self._second - else: - s = "" - s= "%s.%s(%d, %d%s)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._hour, self._minute, s) - if self._tzinfo is not None: - assert s[-1:] == ")" - s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" - if self._fold: - assert s[-1:] == ")" - s = s[:-1] + ", fold=1)" - return s - - def isoformat(self, timespec='auto'): - """Return the time formatted according to ISO. - - The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional - part is omitted if self.microsecond == 0. - - The optional argument timespec specifies the number of additional - terms of the time to include. Valid options are 'auto', 'hours', - 'minutes', 'seconds', 'milliseconds' and 'microseconds'. - """ - s = _format_time(self._hour, self._minute, self._second, - self._microsecond, timespec) - tz = self._tzstr() - if tz: - s += tz - return s - - __str__ = isoformat - - @classmethod - def fromisoformat(cls, time_string): - """Construct a time from the output of isoformat().""" - if not isinstance(time_string, str): - raise TypeError('fromisoformat: argument must be str') - - try: - return cls(*_parse_isoformat_time(time_string)) - except Exception: - raise ValueError(f'Invalid isoformat string: {time_string!r}') - - - def strftime(self, fmt): - """Format using strftime(). The date part of the timestamp passed - to underlying strftime should not be used. - """ - # The year must be >= 1000 else Python's strftime implementation - # can raise a bogus exception. - timetuple = (1900, 1, 1, - self._hour, self._minute, self._second, - 0, 1, -1) - return _wrap_strftime(self, fmt, timetuple) - - def __format__(self, fmt): - if not isinstance(fmt, str): - raise TypeError("must be str, not %s" % type(fmt).__name__) - if len(fmt) != 0: - return self.strftime(fmt) - return str(self) - - # Timezone functions - - def utcoffset(self): - """Return the timezone offset as timedelta, positive east of UTC - (negative west of UTC).""" - if self._tzinfo is None: - return None - offset = self._tzinfo.utcoffset(None) - _check_utc_offset("utcoffset", offset) - return offset - - def tzname(self): - """Return the timezone name. - - Note that the name is 100% informational -- there's no requirement that - it mean anything in particular. For example, "GMT", "UTC", "-500", - "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. - """ - if self._tzinfo is None: - return None - name = self._tzinfo.tzname(None) - _check_tzname(name) - return name - - def dst(self): - """Return 0 if DST is not in effect, or the DST offset (as timedelta - positive eastward) if DST is in effect. - - This is purely informational; the DST offset has already been added to - the UTC offset returned by utcoffset() if applicable, so there's no - need to consult dst() unless you're interested in displaying the DST - info. - """ - if self._tzinfo is None: - return None - offset = self._tzinfo.dst(None) - _check_utc_offset("dst", offset) - return offset - - def replace(self, hour=None, minute=None, second=None, microsecond=None, - tzinfo=True, *, fold=None): - """Return a new time with new values for the specified fields.""" - if hour is None: - hour = self.hour - if minute is None: - minute = self.minute - if second is None: - second = self.second - if microsecond is None: - microsecond = self.microsecond - if tzinfo is True: - tzinfo = self.tzinfo - if fold is None: - fold = self._fold - return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold) - - # Pickle support. - - def _getstate(self, protocol=3): - us2, us3 = divmod(self._microsecond, 256) - us1, us2 = divmod(us2, 256) - h = self._hour - if self._fold and protocol > 3: - h += 128 - basestate = bytes([h, self._minute, self._second, - us1, us2, us3]) - if self._tzinfo is None: - return (basestate,) - else: - return (basestate, self._tzinfo) - - def __setstate(self, string, tzinfo): - if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): - raise TypeError("bad tzinfo state arg") - h, self._minute, self._second, us1, us2, us3 = string - if h > 127: - self._fold = 1 - self._hour = h - 128 - else: - self._fold = 0 - self._hour = h - self._microsecond = (((us1 << 8) | us2) << 8) | us3 - self._tzinfo = tzinfo - - def __reduce_ex__(self, protocol): - return (self.__class__, self._getstate(protocol)) - - def __reduce__(self): - return self.__reduce_ex__(2) - -_time_class = time # so functions w/ args named "time" can get at the class - -time.min = time(0, 0, 0) -time.max = time(23, 59, 59, 999999) -time.resolution = timedelta(microseconds=1) - - -class datetime(date): - """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) - - The year, month and day arguments are required. tzinfo may be None, or an - instance of a tzinfo subclass. The remaining arguments may be ints. - """ - __slots__ = date.__slots__ + time.__slots__ - - def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, - microsecond=0, tzinfo=None, *, fold=0): - if (isinstance(year, (bytes, str)) and len(year) == 10 and - 1 <= ord(year[2:3])&0x7F <= 12): - # Pickle support - if isinstance(year, str): - try: - year = bytes(year, 'latin1') - except UnicodeEncodeError: - # More informative error message. - raise ValueError( - "Failed to encode latin1 string when unpickling " - "a datetime object. " - "pickle.load(data, encoding='latin1') is assumed.") - self = object.__new__(cls) - self.__setstate(year, month) - self._hashcode = -1 - return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond, fold = _check_time_fields( - hour, minute, second, microsecond, fold) - _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) - self._year = year - self._month = month - self._day = day - self._hour = hour - self._minute = minute - self._second = second - self._microsecond = microsecond - self._tzinfo = tzinfo - self._hashcode = -1 - self._fold = fold - return self - - # Read-only field accessors - @property - def hour(self): - """hour (0-23)""" - return self._hour - - @property - def minute(self): - """minute (0-59)""" - return self._minute - - @property - def second(self): - """second (0-59)""" - return self._second - - @property - def microsecond(self): - """microsecond (0-999999)""" - return self._microsecond - - @property - def tzinfo(self): - """timezone info object""" - return self._tzinfo - - @property - def fold(self): - return self._fold - - @classmethod - def _fromtimestamp(cls, t, utc, tz): - """Construct a datetime from a POSIX timestamp (like time.time()). - - A timezone info object may be passed in as well. - """ - frac, t = _math.modf(t) - us = round(frac * 1e6) - if us >= 1000000: - t += 1 - us -= 1000000 - elif us < 0: - t -= 1 - us += 1000000 - - converter = _time.gmtime if utc else _time.localtime - y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) - ss = min(ss, 59) # clamp out leap seconds if the platform has them - result = cls(y, m, d, hh, mm, ss, us, tz) - if tz is None and not utc: - # As of version 2015f max fold in IANA database is - # 23 hours at 1969-09-30 13:00:00 in Kwajalein. - # Let's probe 24 hours in the past to detect a transition: - max_fold_seconds = 24 * 3600 - - # On Windows localtime_s throws an OSError for negative values, - # thus we can't perform fold detection for values of time less - # than the max time fold. See comments in _datetimemodule's - # version of this method for more details. - if t < max_fold_seconds and sys.platform.startswith("win"): - return result - - y, m, d, hh, mm, ss = converter(t - max_fold_seconds)[:6] - probe1 = cls(y, m, d, hh, mm, ss, us, tz) - trans = result - probe1 - timedelta(0, max_fold_seconds) - if trans.days < 0: - y, m, d, hh, mm, ss = converter(t + trans // timedelta(0, 1))[:6] - probe2 = cls(y, m, d, hh, mm, ss, us, tz) - if probe2 == result: - result._fold = 1 - elif tz is not None: - result = tz.fromutc(result) - return result - - @classmethod - def fromtimestamp(cls, t, tz=None): - """Construct a datetime from a POSIX timestamp (like time.time()). - - A timezone info object may be passed in as well. - """ - _check_tzinfo_arg(tz) - - return cls._fromtimestamp(t, tz is not None, tz) - - @classmethod - def utcfromtimestamp(cls, t): - """Construct a naive UTC datetime from a POSIX timestamp.""" - return cls._fromtimestamp(t, True, None) - - @classmethod - def now(cls, tz=None): - "Construct a datetime from time.time() and optional time zone info." - t = _time.time() - return cls.fromtimestamp(t, tz) - - @classmethod - def utcnow(cls): - "Construct a UTC datetime from time.time()." - t = _time.time() - return cls.utcfromtimestamp(t) - - @classmethod - def combine(cls, date, time, tzinfo=True): - "Construct a datetime from a given date and a given time." - if not isinstance(date, _date_class): - raise TypeError("date argument must be a date instance") - if not isinstance(time, _time_class): - raise TypeError("time argument must be a time instance") - if tzinfo is True: - tzinfo = time.tzinfo - return cls(date.year, date.month, date.day, - time.hour, time.minute, time.second, time.microsecond, - tzinfo, fold=time.fold) - - @classmethod - def fromisoformat(cls, date_string): - """Construct a datetime from the output of datetime.isoformat().""" - if not isinstance(date_string, str): - raise TypeError('fromisoformat: argument must be str') - - # Split this at the separator - dstr = date_string[0:10] - tstr = date_string[11:] - - try: - date_components = _parse_isoformat_date(dstr) - except ValueError: - raise ValueError(f'Invalid isoformat string: {date_string!r}') - - if tstr: - try: - time_components = _parse_isoformat_time(tstr) - except ValueError: - raise ValueError(f'Invalid isoformat string: {date_string!r}') - else: - time_components = [0, 0, 0, 0, None] - - return cls(*(date_components + time_components)) - - def timetuple(self): - "Return local time tuple compatible with time.localtime()." - dst = self.dst() - if dst is None: - dst = -1 - elif dst: - dst = 1 - else: - dst = 0 - return _build_struct_time(self.year, self.month, self.day, - self.hour, self.minute, self.second, - dst) - - def _mktime(self): - """Return integer POSIX timestamp.""" - epoch = datetime(1970, 1, 1) - max_fold_seconds = 24 * 3600 - t = (self - epoch) // timedelta(0, 1) - def local(u): - y, m, d, hh, mm, ss = _time.localtime(u)[:6] - return (datetime(y, m, d, hh, mm, ss) - epoch) // timedelta(0, 1) - - # Our goal is to solve t = local(u) for u. - a = local(t) - t - u1 = t - a - t1 = local(u1) - if t1 == t: - # We found one solution, but it may not be the one we need. - # Look for an earlier solution (if `fold` is 0), or a - # later one (if `fold` is 1). - u2 = u1 + (-max_fold_seconds, max_fold_seconds)[self.fold] - b = local(u2) - u2 - if a == b: - return u1 - else: - b = t1 - u1 - assert a != b - u2 = t - b - t2 = local(u2) - if t2 == t: - return u2 - if t1 == t: - return u1 - # We have found both offsets a and b, but neither t - a nor t - b is - # a solution. This means t is in the gap. - return (max, min)[self.fold](u1, u2) - - - def timestamp(self): - "Return POSIX timestamp as float" - if self._tzinfo is None: - s = self._mktime() - return s + self.microsecond / 1e6 - else: - return (self - _EPOCH).total_seconds() - - def utctimetuple(self): - "Return UTC time tuple compatible with time.gmtime()." - offset = self.utcoffset() - if offset: - self -= offset - y, m, d = self.year, self.month, self.day - hh, mm, ss = self.hour, self.minute, self.second - return _build_struct_time(y, m, d, hh, mm, ss, 0) - - def date(self): - "Return the date part." - return date(self._year, self._month, self._day) - - def time(self): - "Return the time part, with tzinfo None." - return time(self.hour, self.minute, self.second, self.microsecond, fold=self.fold) - - def timetz(self): - "Return the time part, with same tzinfo." - return time(self.hour, self.minute, self.second, self.microsecond, - self._tzinfo, fold=self.fold) - - def replace(self, year=None, month=None, day=None, hour=None, - minute=None, second=None, microsecond=None, tzinfo=True, - *, fold=None): - """Return a new datetime with new values for the specified fields.""" - if year is None: - year = self.year - if month is None: - month = self.month - if day is None: - day = self.day - if hour is None: - hour = self.hour - if minute is None: - minute = self.minute - if second is None: - second = self.second - if microsecond is None: - microsecond = self.microsecond - if tzinfo is True: - tzinfo = self.tzinfo - if fold is None: - fold = self.fold - return type(self)(year, month, day, hour, minute, second, - microsecond, tzinfo, fold=fold) - - def _local_timezone(self): - if self.tzinfo is None: - ts = self._mktime() - else: - ts = (self - _EPOCH) // timedelta(seconds=1) - localtm = _time.localtime(ts) - local = datetime(*localtm[:6]) - # Extract TZ data - gmtoff = localtm.tm_gmtoff - zone = localtm.tm_zone - return timezone(timedelta(seconds=gmtoff), zone) - - def astimezone(self, tz=None): - if tz is None: - tz = self._local_timezone() - elif not isinstance(tz, tzinfo): - raise TypeError("tz argument must be an instance of tzinfo") - - mytz = self.tzinfo - if mytz is None: - mytz = self._local_timezone() - myoffset = mytz.utcoffset(self) - else: - myoffset = mytz.utcoffset(self) - if myoffset is None: - mytz = self.replace(tzinfo=None)._local_timezone() - myoffset = mytz.utcoffset(self) - - if tz is mytz: - return self - - # Convert self to UTC, and attach the new time zone object. - utc = (self - myoffset).replace(tzinfo=tz) - - # Convert from UTC to tz's local time. - return tz.fromutc(utc) - - # Ways to produce a string. - - def ctime(self): - "Return ctime() style string." - weekday = self.toordinal() % 7 or 7 - return "%s %s %2d %02d:%02d:%02d %04d" % ( - _DAYNAMES[weekday], - _MONTHNAMES[self._month], - self._day, - self._hour, self._minute, self._second, - self._year) - - def isoformat(self, sep='T', timespec='auto'): - """Return the time formatted according to ISO. - - The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. - By default, the fractional part is omitted if self.microsecond == 0. - - If self.tzinfo is not None, the UTC offset is also attached, giving - giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. - - Optional argument sep specifies the separator between date and - time, default 'T'. - - The optional argument timespec specifies the number of additional - terms of the time to include. Valid options are 'auto', 'hours', - 'minutes', 'seconds', 'milliseconds' and 'microseconds'. - """ - s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + - _format_time(self._hour, self._minute, self._second, - self._microsecond, timespec)) - - off = self.utcoffset() - tz = _format_offset(off) - if tz: - s += tz - - return s - - def __repr__(self): - """Convert to formal string, for repr().""" - L = [self._year, self._month, self._day, # These are never zero - self._hour, self._minute, self._second, self._microsecond] - if L[-1] == 0: - del L[-1] - if L[-1] == 0: - del L[-1] - s = "%s.%s(%s)" % (self.__class__.__module__, - self.__class__.__qualname__, - ", ".join(map(str, L))) - if self._tzinfo is not None: - assert s[-1:] == ")" - s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" - if self._fold: - assert s[-1:] == ")" - s = s[:-1] + ", fold=1)" - return s - - def __str__(self): - "Convert to string, for str()." - return self.isoformat(sep=' ') - - @classmethod - def strptime(cls, date_string, format): - 'string, format -> new datetime parsed from a string (like time.strptime()).' - import _strptime - return _strptime._strptime_datetime(cls, date_string, format) - - def utcoffset(self): - """Return the timezone offset as timedelta positive east of UTC (negative west of - UTC).""" - if self._tzinfo is None: - return None - offset = self._tzinfo.utcoffset(self) - _check_utc_offset("utcoffset", offset) - return offset - - def tzname(self): - """Return the timezone name. - - Note that the name is 100% informational -- there's no requirement that - it mean anything in particular. For example, "GMT", "UTC", "-500", - "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. - """ - if self._tzinfo is None: - return None - name = self._tzinfo.tzname(self) - _check_tzname(name) - return name - - def dst(self): - """Return 0 if DST is not in effect, or the DST offset (as timedelta - positive eastward) if DST is in effect. - - This is purely informational; the DST offset has already been added to - the UTC offset returned by utcoffset() if applicable, so there's no - need to consult dst() unless you're interested in displaying the DST - info. - """ - if self._tzinfo is None: - return None - offset = self._tzinfo.dst(self) - _check_utc_offset("dst", offset) - return offset - - # Comparisons of datetime objects with other. - - def __eq__(self, other): - if isinstance(other, datetime): - return self._cmp(other, allow_mixed=True) == 0 - elif not isinstance(other, date): - return NotImplemented - else: - return False - - def __le__(self, other): - if isinstance(other, datetime): - return self._cmp(other) <= 0 - elif not isinstance(other, date): - return NotImplemented - else: - _cmperror(self, other) - - def __lt__(self, other): - if isinstance(other, datetime): - return self._cmp(other) < 0 - elif not isinstance(other, date): - return NotImplemented - else: - _cmperror(self, other) - - def __ge__(self, other): - if isinstance(other, datetime): - return self._cmp(other) >= 0 - elif not isinstance(other, date): - return NotImplemented - else: - _cmperror(self, other) - - def __gt__(self, other): - if isinstance(other, datetime): - return self._cmp(other) > 0 - elif not isinstance(other, date): - return NotImplemented - else: - _cmperror(self, other) - - def _cmp(self, other, allow_mixed=False): - assert isinstance(other, datetime) - mytz = self._tzinfo - ottz = other._tzinfo - myoff = otoff = None - - if mytz is ottz: - base_compare = True - else: - myoff = self.utcoffset() - otoff = other.utcoffset() - # Assume that allow_mixed means that we are called from __eq__ - if allow_mixed: - if myoff != self.replace(fold=not self.fold).utcoffset(): - return 2 - if otoff != other.replace(fold=not other.fold).utcoffset(): - return 2 - base_compare = myoff == otoff - - if base_compare: - return _cmp((self._year, self._month, self._day, - self._hour, self._minute, self._second, - self._microsecond), - (other._year, other._month, other._day, - other._hour, other._minute, other._second, - other._microsecond)) - if myoff is None or otoff is None: - if allow_mixed: - return 2 # arbitrary non-zero value - else: - raise TypeError("cannot compare naive and aware datetimes") - # XXX What follows could be done more efficiently... - diff = self - other # this will take offsets into account - if diff.days < 0: - return -1 - return diff and 1 or 0 - - def __add__(self, other): - "Add a datetime and a timedelta." - if not isinstance(other, timedelta): - return NotImplemented - delta = timedelta(self.toordinal(), - hours=self._hour, - minutes=self._minute, - seconds=self._second, - microseconds=self._microsecond) - delta += other - hour, rem = divmod(delta.seconds, 3600) - minute, second = divmod(rem, 60) - if 0 < delta.days <= _MAXORDINAL: - return type(self).combine(date.fromordinal(delta.days), - time(hour, minute, second, - delta.microseconds, - tzinfo=self._tzinfo)) - raise OverflowError("result out of range") - - __radd__ = __add__ - - def __sub__(self, other): - "Subtract two datetimes, or a datetime and a timedelta." - if not isinstance(other, datetime): - if isinstance(other, timedelta): - return self + -other - return NotImplemented - - days1 = self.toordinal() - days2 = other.toordinal() - secs1 = self._second + self._minute * 60 + self._hour * 3600 - secs2 = other._second + other._minute * 60 + other._hour * 3600 - base = timedelta(days1 - days2, - secs1 - secs2, - self._microsecond - other._microsecond) - if self._tzinfo is other._tzinfo: - return base - myoff = self.utcoffset() - otoff = other.utcoffset() - if myoff == otoff: - return base - if myoff is None or otoff is None: - raise TypeError("cannot mix naive and timezone-aware time") - return base + otoff - myoff - - def __hash__(self): - if self._hashcode == -1: - if self.fold: - t = self.replace(fold=0) - else: - t = self - tzoff = t.utcoffset() - if tzoff is None: - self._hashcode = hash(t._getstate()[0]) - else: - days = _ymd2ord(self.year, self.month, self.day) - seconds = self.hour * 3600 + self.minute * 60 + self.second - self._hashcode = hash(timedelta(days, seconds, self.microsecond) - tzoff) - return self._hashcode - - # Pickle support. - - def _getstate(self, protocol=3): - yhi, ylo = divmod(self._year, 256) - us2, us3 = divmod(self._microsecond, 256) - us1, us2 = divmod(us2, 256) - m = self._month - if self._fold and protocol > 3: - m += 128 - basestate = bytes([yhi, ylo, m, self._day, - self._hour, self._minute, self._second, - us1, us2, us3]) - if self._tzinfo is None: - return (basestate,) - else: - return (basestate, self._tzinfo) - - def __setstate(self, string, tzinfo): - if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): - raise TypeError("bad tzinfo state arg") - (yhi, ylo, m, self._day, self._hour, - self._minute, self._second, us1, us2, us3) = string - if m > 127: - self._fold = 1 - self._month = m - 128 - else: - self._fold = 0 - self._month = m - self._year = yhi * 256 + ylo - self._microsecond = (((us1 << 8) | us2) << 8) | us3 - self._tzinfo = tzinfo - - def __reduce_ex__(self, protocol): - return (self.__class__, self._getstate(protocol)) - - def __reduce__(self): - return self.__reduce_ex__(2) - - -datetime.min = datetime(1, 1, 1) -datetime.max = datetime(9999, 12, 31, 23, 59, 59, 999999) -datetime.resolution = timedelta(microseconds=1) - - -def _isoweek1monday(year): - # Helper to calculate the day number of the Monday starting week 1 - # XXX This could be done more efficiently - THURSDAY = 3 - firstday = _ymd2ord(year, 1, 1) - firstweekday = (firstday + 6) % 7 # See weekday() above - week1monday = firstday - firstweekday - if firstweekday > THURSDAY: - week1monday += 7 - return week1monday - - -class timezone(tzinfo): - __slots__ = '_offset', '_name' - - # Sentinel value to disallow None - _Omitted = object() - def __new__(cls, offset, name=_Omitted): - if not isinstance(offset, timedelta): - raise TypeError("offset must be a timedelta") - if name is cls._Omitted: - if not offset: - return cls.utc - name = None - elif not isinstance(name, str): - raise TypeError("name must be a string") - if not cls._minoffset <= offset <= cls._maxoffset: - raise ValueError("offset must be a timedelta " - "strictly between -timedelta(hours=24) and " - "timedelta(hours=24).") - return cls._create(offset, name) - - @classmethod - def _create(cls, offset, name=None): - self = tzinfo.__new__(cls) - self._offset = offset - self._name = name - return self - - def __getinitargs__(self): - """pickle support""" - if self._name is None: - return (self._offset,) - return (self._offset, self._name) - - def __eq__(self, other): - if isinstance(other, timezone): - return self._offset == other._offset - return NotImplemented - - def __hash__(self): - return hash(self._offset) - - def __repr__(self): - """Convert to formal string, for repr(). - - >>> tz = timezone.utc - >>> repr(tz) - 'datetime.timezone.utc' - >>> tz = timezone(timedelta(hours=-5), 'EST') - >>> repr(tz) - "datetime.timezone(datetime.timedelta(-1, 68400), 'EST')" - """ - if self is self.utc: - return 'datetime.timezone.utc' - if self._name is None: - return "%s.%s(%r)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._offset) - return "%s.%s(%r, %r)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._offset, self._name) - - def __str__(self): - return self.tzname(None) - - def utcoffset(self, dt): - if isinstance(dt, datetime) or dt is None: - return self._offset - raise TypeError("utcoffset() argument must be a datetime instance" - " or None") - - def tzname(self, dt): - if isinstance(dt, datetime) or dt is None: - if self._name is None: - return self._name_from_offset(self._offset) - return self._name - raise TypeError("tzname() argument must be a datetime instance" - " or None") - - def dst(self, dt): - if isinstance(dt, datetime) or dt is None: - return None - raise TypeError("dst() argument must be a datetime instance" - " or None") - - def fromutc(self, dt): - if isinstance(dt, datetime): - if dt.tzinfo is not self: - raise ValueError("fromutc: dt.tzinfo " - "is not self") - return dt + self._offset - raise TypeError("fromutc() argument must be a datetime instance" - " or None") - - _maxoffset = timedelta(hours=24, microseconds=-1) - _minoffset = -_maxoffset - - @staticmethod - def _name_from_offset(delta): - if not delta: - return 'UTC' - if delta < timedelta(0): - sign = '-' - delta = -delta - else: - sign = '+' - hours, rest = divmod(delta, timedelta(hours=1)) - minutes, rest = divmod(rest, timedelta(minutes=1)) - seconds = rest.seconds - microseconds = rest.microseconds - if microseconds: - return (f'UTC{sign}{hours:02d}:{minutes:02d}:{seconds:02d}' - f'.{microseconds:06d}') - if seconds: - return f'UTC{sign}{hours:02d}:{minutes:02d}:{seconds:02d}' - return f'UTC{sign}{hours:02d}:{minutes:02d}' - -UTC = timezone.utc = timezone._create(timedelta(0)) -# bpo-37642: These attributes are rounded to the nearest minute for backwards -# compatibility, even though the constructor will accept a wider range of -# values. This may change in the future. -timezone.min = timezone._create(-timedelta(hours=23, minutes=59)) -timezone.max = timezone._create(timedelta(hours=23, minutes=59)) -_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) - -# Some time zone algebra. For a datetime x, let -# x.n = x stripped of its timezone -- its naive time. -# x.o = x.utcoffset(), and assuming that doesn't raise an exception or -# return None -# x.d = x.dst(), and assuming that doesn't raise an exception or -# return None -# x.s = x's standard offset, x.o - x.d -# -# Now some derived rules, where k is a duration (timedelta). -# -# 1. x.o = x.s + x.d -# This follows from the definition of x.s. -# -# 2. If x and y have the same tzinfo member, x.s = y.s. -# This is actually a requirement, an assumption we need to make about -# sane tzinfo classes. -# -# 3. The naive UTC time corresponding to x is x.n - x.o. -# This is again a requirement for a sane tzinfo class. -# -# 4. (x+k).s = x.s -# This follows from #2, and that datetime.timetz+timedelta preserves tzinfo. -# -# 5. (x+k).n = x.n + k -# Again follows from how arithmetic is defined. -# -# Now we can explain tz.fromutc(x). Let's assume it's an interesting case -# (meaning that the various tzinfo methods exist, and don't blow up or return -# None when called). -# -# The function wants to return a datetime y with timezone tz, equivalent to x. -# x is already in UTC. -# -# By #3, we want -# -# y.n - y.o = x.n [1] -# -# The algorithm starts by attaching tz to x.n, and calling that y. So -# x.n = y.n at the start. Then it wants to add a duration k to y, so that [1] -# becomes true; in effect, we want to solve [2] for k: -# -# (y+k).n - (y+k).o = x.n [2] -# -# By #1, this is the same as -# -# (y+k).n - ((y+k).s + (y+k).d) = x.n [3] -# -# By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start. -# Substituting that into [3], -# -# x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving -# k - (y+k).s - (y+k).d = 0; rearranging, -# k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so -# k = y.s - (y+k).d -# -# On the RHS, (y+k).d can't be computed directly, but y.s can be, and we -# approximate k by ignoring the (y+k).d term at first. Note that k can't be -# very large, since all offset-returning methods return a duration of magnitude -# less than 24 hours. For that reason, if y is firmly in std time, (y+k).d must -# be 0, so ignoring it has no consequence then. -# -# In any case, the new value is -# -# z = y + y.s [4] -# -# It's helpful to step back at look at [4] from a higher level: it's simply -# mapping from UTC to tz's standard time. -# -# At this point, if -# -# z.n - z.o = x.n [5] -# -# we have an equivalent time, and are almost done. The insecurity here is -# at the start of daylight time. Picture US Eastern for concreteness. The wall -# time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good -# sense then. The docs ask that an Eastern tzinfo class consider such a time to -# be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST -# on the day DST starts. We want to return the 1:MM EST spelling because that's -# the only spelling that makes sense on the local wall clock. -# -# In fact, if [5] holds at this point, we do have the standard-time spelling, -# but that takes a bit of proof. We first prove a stronger result. What's the -# difference between the LHS and RHS of [5]? Let -# -# diff = x.n - (z.n - z.o) [6] -# -# Now -# z.n = by [4] -# (y + y.s).n = by #5 -# y.n + y.s = since y.n = x.n -# x.n + y.s = since z and y are have the same tzinfo member, -# y.s = z.s by #2 -# x.n + z.s -# -# Plugging that back into [6] gives -# -# diff = -# x.n - ((x.n + z.s) - z.o) = expanding -# x.n - x.n - z.s + z.o = cancelling -# - z.s + z.o = by #2 -# z.d -# -# So diff = z.d. -# -# If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time -# spelling we wanted in the endcase described above. We're done. Contrarily, -# if z.d = 0, then we have a UTC equivalent, and are also done. -# -# If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to -# add to z (in effect, z is in tz's standard time, and we need to shift the -# local clock into tz's daylight time). -# -# Let -# -# z' = z + z.d = z + diff [7] -# -# and we can again ask whether -# -# z'.n - z'.o = x.n [8] -# -# If so, we're done. If not, the tzinfo class is insane, according to the -# assumptions we've made. This also requires a bit of proof. As before, let's -# compute the difference between the LHS and RHS of [8] (and skipping some of -# the justifications for the kinds of substitutions we've done several times -# already): -# -# diff' = x.n - (z'.n - z'.o) = replacing z'.n via [7] -# x.n - (z.n + diff - z'.o) = replacing diff via [6] -# x.n - (z.n + x.n - (z.n - z.o) - z'.o) = -# x.n - z.n - x.n + z.n - z.o + z'.o = cancel x.n -# - z.n + z.n - z.o + z'.o = cancel z.n -# - z.o + z'.o = #1 twice -# -z.s - z.d + z'.s + z'.d = z and z' have same tzinfo -# z'.d - z.d -# -# So z' is UTC-equivalent to x iff z'.d = z.d at this point. If they are equal, -# we've found the UTC-equivalent so are done. In fact, we stop with [7] and -# return z', not bothering to compute z'.d. -# -# How could z.d and z'd differ? z' = z + z.d [7], so merely moving z' by -# a dst() offset, and starting *from* a time already in DST (we know z.d != 0), -# would have to change the result dst() returns: we start in DST, and moving -# a little further into it takes us out of DST. -# -# There isn't a sane case where this can happen. The closest it gets is at -# the end of DST, where there's an hour in UTC with no spelling in a hybrid -# tzinfo class. In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT. During -# that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM -# UTC) because the docs insist on that, but 0:MM is taken as being in daylight -# time (4:MM UTC). There is no local time mapping to 5:MM UTC. The local -# clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in -# standard time. Since that's what the local clock *does*, we want to map both -# UTC hours 5:MM and 6:MM to 1:MM Eastern. The result is ambiguous -# in local time, but so it goes -- it's the way the local clock works. -# -# When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0, -# so z=0:MM. z.d=60 (minutes) then, so [5] doesn't hold and we keep going. -# z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8] -# (correctly) concludes that z' is not UTC-equivalent to x. -# -# Because we know z.d said z was in daylight time (else [5] would have held and -# we would have stopped then), and we know z.d != z'.d (else [8] would have held -# and we have stopped then), and there are only 2 possible values dst() can -# return in Eastern, it follows that z'.d must be 0 (which it is in the example, -# but the reasoning doesn't depend on the example -- it depends on there being -# two possible dst() outcomes, one zero and the other non-zero). Therefore -# z' must be in standard time, and is the spelling we want in this case. -# -# Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is -# concerned (because it takes z' as being in standard time rather than the -# daylight time we intend here), but returning it gives the real-life "local -# clock repeats an hour" behavior when mapping the "unspellable" UTC hour into -# tz. -# -# When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with -# the 1:MM standard time spelling we want. -# -# So how can this break? One of the assumptions must be violated. Two -# possibilities: -# -# 1) [2] effectively says that y.s is invariant across all y belong to a given -# time zone. This isn't true if, for political reasons or continental drift, -# a region decides to change its base offset from UTC. -# -# 2) There may be versions of "double daylight" time where the tail end of -# the analysis gives up a step too early. I haven't thought about that -# enough to say. -# -# In any case, it's clear that the default fromutc() is strong enough to handle -# "almost all" time zones: so long as the standard offset is invariant, it -# doesn't matter if daylight time transition points change from year to year, or -# if daylight time is skipped in some years; it doesn't matter how large or -# small dst() may get within its bounds; and it doesn't even matter if some -# perverse time zone returns a negative dst()). So a breaking case must be -# pretty bizarre, and a tzinfo subclass can override fromutc() if it is. - try: from _datetime import * except ImportError: - pass -else: - # Clean up unused names - del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y, - _DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time, - _check_date_fields, _check_time_fields, - _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror, - _date_class, _days_before_month, _days_before_year, _days_in_month, - _format_time, _format_offset, _index, _is_leap, _isoweek1monday, _math, - _ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord, - _divide_and_round, _parse_isoformat_date, _parse_isoformat_time, - _parse_hh_mm_ss_ff, _IsoCalendarDate) - # XXX Since import * above excludes names that start with _, - # docstring does not get overwritten. In the future, it may be - # appropriate to maintain a single module level docstring and - # remove the following line. - from _datetime import __doc__ + from _pydatetime import * + +__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", + "MINYEAR", "MAXYEAR", "UTC") diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index f65da521af4..4fdbc54e74c 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -5,7 +5,7 @@ import dbm d = dbm.open(file, 'w', 0o666) -The returned object is a dbm.gnu, dbm.ndbm or dbm.dumb object, dependent on the +The returned object is a dbm.sqlite3, dbm.gnu, dbm.ndbm or dbm.dumb database object, dependent on the type of database being opened (determined by the whichdb function) in the case of an existing dbm. If the dbm does not exist and the create or new flag ('c' or 'n') was specified, the dbm type will be determined by the availability of @@ -38,7 +38,7 @@ class error(Exception): pass -_names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] +_names = ['dbm.sqlite3', 'dbm.gnu', 'dbm.ndbm', 'dbm.dumb'] _defaultmod = None _modules = {} @@ -109,17 +109,18 @@ def whichdb(filename): """ # Check for ndbm first -- this has a .pag and a .dir file + filename = os.fsencode(filename) try: - f = io.open(filename + ".pag", "rb") + f = io.open(filename + b".pag", "rb") f.close() - f = io.open(filename + ".dir", "rb") + f = io.open(filename + b".dir", "rb") f.close() return "dbm.ndbm" except OSError: # some dbm emulations based on Berkeley DB generate a .db file # some do not, but they should be caught by the bsd checks try: - f = io.open(filename + ".db", "rb") + f = io.open(filename + b".db", "rb") f.close() # guarantee we can actually open the file using dbm # kind of overkill, but since we are dealing with emulations @@ -134,12 +135,12 @@ def whichdb(filename): # Check for dumbdbm next -- this has a .dir and a .dat file try: # First check for presence of files - os.stat(filename + ".dat") - size = os.stat(filename + ".dir").st_size + os.stat(filename + b".dat") + size = os.stat(filename + b".dir").st_size # dumbdbm files with no keys are empty if size == 0: return "dbm.dumb" - f = io.open(filename + ".dir", "rb") + f = io.open(filename + b".dir", "rb") try: if f.read(1) in (b"'", b'"'): return "dbm.dumb" @@ -163,6 +164,10 @@ def whichdb(filename): if len(s) != 4: return "" + # Check for SQLite3 header string. + if s16 == b"SQLite format 3\0": + return "dbm.sqlite3" + # Convert to 4-byte int in native byte order -- return "" if impossible try: (magic,) = struct.unpack("=l", s) diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 864ad371ec9..def120ffc37 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -46,6 +46,7 @@ class _Database(collections.abc.MutableMapping): _io = _io # for _commit() def __init__(self, filebasename, mode, flag='c'): + filebasename = self._os.fsencode(filebasename) self._mode = mode self._readonly = (flag == 'r') @@ -54,14 +55,14 @@ def __init__(self, filebasename, mode, flag='c'): # where key is the string key, pos is the offset into the dat # file of the associated value's first byte, and siz is the number # of bytes in the associated value. - self._dirfile = filebasename + '.dir' + self._dirfile = filebasename + b'.dir' # The data file is a binary file pointed into by the directory # file, and holds the values associated with keys. Each value # begins at a _BLOCKSIZE-aligned byte offset, and is a raw # binary 8-bit string value. - self._datfile = filebasename + '.dat' - self._bakfile = filebasename + '.bak' + self._datfile = filebasename + b'.dat' + self._bakfile = filebasename + b'.bak' # The index is an in-memory dict, mirroring the directory file. self._index = None # maps keys to (pos, siz) pairs @@ -97,7 +98,8 @@ def _update(self, flag): except OSError: if flag not in ('c', 'n'): raise - self._modified = True + with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: + self._chmod(self._dirfile) else: with f: for line in f: @@ -133,6 +135,7 @@ def _commit(self): # position; UTF-8, though, does care sometimes. entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair) f.write(entry) + self._modified = False sync = _commit diff --git a/Lib/dbm/gnu.py b/Lib/dbm/gnu.py new file mode 100644 index 00000000000..b07a1defffd --- /dev/null +++ b/Lib/dbm/gnu.py @@ -0,0 +1,3 @@ +"""Provide the _gdbm module as a dbm submodule.""" + +from _gdbm import * diff --git a/Lib/dbm/ndbm.py b/Lib/dbm/ndbm.py new file mode 100644 index 00000000000..23056a29ef2 --- /dev/null +++ b/Lib/dbm/ndbm.py @@ -0,0 +1,3 @@ +"""Provide the _dbm module as a dbm submodule.""" + +from _dbm import * diff --git a/Lib/dbm/sqlite3.py b/Lib/dbm/sqlite3.py new file mode 100644 index 00000000000..d0eed54e0f8 --- /dev/null +++ b/Lib/dbm/sqlite3.py @@ -0,0 +1,144 @@ +import os +import sqlite3 +from pathlib import Path +from contextlib import suppress, closing +from collections.abc import MutableMapping + +BUILD_TABLE = """ + CREATE TABLE IF NOT EXISTS Dict ( + key BLOB UNIQUE NOT NULL, + value BLOB NOT NULL + ) +""" +GET_SIZE = "SELECT COUNT (key) FROM Dict" +LOOKUP_KEY = "SELECT value FROM Dict WHERE key = CAST(? AS BLOB)" +STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))" +DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)" +ITER_KEYS = "SELECT key FROM Dict" + + +class error(OSError): + pass + + +_ERR_CLOSED = "DBM object has already been closed" +_ERR_REINIT = "DBM object does not support reinitialization" + + +def _normalize_uri(path): + path = Path(path) + uri = path.absolute().as_uri() + while "//" in uri: + uri = uri.replace("//", "/") + return uri + + +class _Database(MutableMapping): + + def __init__(self, path, /, *, flag, mode): + if hasattr(self, "_cx"): + raise error(_ERR_REINIT) + + path = os.fsdecode(path) + match flag: + case "r": + flag = "ro" + case "w": + flag = "rw" + case "c": + flag = "rwc" + Path(path).touch(mode=mode, exist_ok=True) + case "n": + flag = "rwc" + Path(path).unlink(missing_ok=True) + Path(path).touch(mode=mode) + case _: + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n', " + f"not {flag!r}") + + # We use the URI format when opening the database. + uri = _normalize_uri(path) + uri = f"{uri}?mode={flag}" + if flag == "ro": + # Add immutable=1 to allow read-only SQLite access even if wal/shm missing + uri += "&immutable=1" + + try: + self._cx = sqlite3.connect(uri, autocommit=True, uri=True) + except sqlite3.Error as exc: + raise error(str(exc)) + + if flag != "ro": + # This is an optimization only; it's ok if it fails. + with suppress(sqlite3.OperationalError): + self._cx.execute("PRAGMA journal_mode = wal") + + if flag == "rwc": + self._execute(BUILD_TABLE) + + def _execute(self, *args, **kwargs): + if not self._cx: + raise error(_ERR_CLOSED) + try: + return closing(self._cx.execute(*args, **kwargs)) + except sqlite3.Error as exc: + raise error(str(exc)) + + def __len__(self): + with self._execute(GET_SIZE) as cu: + row = cu.fetchone() + return row[0] + + def __getitem__(self, key): + with self._execute(LOOKUP_KEY, (key,)) as cu: + row = cu.fetchone() + if not row: + raise KeyError(key) + return row[0] + + def __setitem__(self, key, value): + self._execute(STORE_KV, (key, value)) + + def __delitem__(self, key): + with self._execute(DELETE_KEY, (key,)) as cu: + if not cu.rowcount: + raise KeyError(key) + + def __iter__(self): + try: + with self._execute(ITER_KEYS) as cu: + for row in cu: + yield row[0] + except sqlite3.Error as exc: + raise error(str(exc)) + + def close(self): + if self._cx: + self._cx.close() + self._cx = None + + def keys(self): + return list(super().keys()) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def open(filename, /, flag="r", mode=0o666): + """Open a dbm.sqlite3 database and return the dbm object. + + The 'filename' parameter is the name of the database file. + + The optional 'flag' parameter can be one of ...: + 'r' (default): open an existing database for read only access + 'w': open an existing database for read/write access + 'c': create a database if it does not exist; open for read/write access + 'n': always create a new, empty database; open for read/write access + + The optional 'mode' parameter is the Unix file access mode of the database; + only used when creating a new database. Default: 0o666. + """ + return _Database(filename, flag=flag, mode=mode) diff --git a/Lib/decimal.py b/Lib/decimal.py index 7746ea26010..530bdfb3895 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,11 +1,109 @@ +"""Decimal fixed-point and floating-point arithmetic. + +This is an implementation of decimal floating-point arithmetic based on +the General Decimal Arithmetic Specification: + + http://speleotrove.com/decimal/decarith.html + +and IEEE standard 854-1987: + + http://en.wikipedia.org/wiki/IEEE_854-1987 + +Decimal floating point has finite precision with arbitrarily large bounds. + +The purpose of this module is to support arithmetic using familiar +"schoolhouse" rules and to avoid some of the tricky representation +issues associated with binary floating point. The package is especially +useful for financial applications or for contexts where users have +expectations that are at odds with binary floating point (for instance, +in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead +of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected +Decimal('0.00')). + +Here are some examples of using the decimal module: + +>>> from decimal import * +>>> setcontext(ExtendedContext) +>>> Decimal(0) +Decimal('0') +>>> Decimal('1') +Decimal('1') +>>> Decimal('-.0123') +Decimal('-0.0123') +>>> Decimal(123456) +Decimal('123456') +>>> Decimal('123.45e12345678') +Decimal('1.2345E+12345680') +>>> Decimal('1.33') + Decimal('1.27') +Decimal('2.60') +>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') +Decimal('-2.20') +>>> dig = Decimal(1) +>>> print(dig / Decimal(3)) +0.333333333 +>>> getcontext().prec = 18 +>>> print(dig / Decimal(3)) +0.333333333333333333 +>>> print(dig.sqrt()) +1 +>>> print(Decimal(3).sqrt()) +1.73205080756887729 +>>> print(Decimal(3) ** 123) +4.85192780976896427E+58 +>>> inf = Decimal(1) / Decimal(0) +>>> print(inf) +Infinity +>>> neginf = Decimal(-1) / Decimal(0) +>>> print(neginf) +-Infinity +>>> print(neginf + inf) +NaN +>>> print(neginf * inf) +-Infinity +>>> print(dig / 0) +Infinity +>>> getcontext().traps[DivisionByZero] = 1 +>>> print(dig / 0) +Traceback (most recent call last): + ... + ... + ... +decimal.DivisionByZero: x / 0 +>>> c = Context() +>>> c.traps[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> c.divide(Decimal(0), Decimal(0)) +Decimal('NaN') +>>> c.traps[InvalidOperation] = 1 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> print(c.divide(Decimal(0), Decimal(0))) +Traceback (most recent call last): + ... + ... + ... +decimal.InvalidOperation: 0 / 0 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> c.traps[InvalidOperation] = 0 +>>> print(c.divide(Decimal(0), Decimal(0))) +NaN +>>> print(c.flags[InvalidOperation]) +1 +>>> +""" try: from _decimal import * - from _decimal import __doc__ - from _decimal import __version__ - from _decimal import __libmpdec_version__ + from _decimal import __version__ # noqa: F401 + from _decimal import __libmpdec_version__ # noqa: F401 except ImportError: - from _pydecimal import * - from _pydecimal import __doc__ - from _pydecimal import __version__ - from _pydecimal import __libmpdec_version__ + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/difflib.py b/Lib/difflib.py index ba0b256969e..ac1ba4a6e4e 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -78,8 +78,8 @@ class SequenceMatcher: sequences. As a rule of thumb, a .ratio() value over 0.6 means the sequences are close matches: - >>> print(round(s.ratio(), 3)) - 0.866 + >>> print(round(s.ratio(), 2)) + 0.87 >>> If you're only interested in where the sequences match, @@ -908,87 +908,85 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - - # don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far - best_ratio, cutoff = 0.74, 0.75 + # Don't synch up unless the lines have a similarity score above + # cutoff. Previously only the smallest pair was handled here, + # and if there are many pairs with the best ratio, recursion + # could grow very deep, and runtime cubic. See: + # https://github.com/python/cpython/issues/119105 + # + # Later, more pathological cases prompted removing recursion + # entirely. + cutoff = 0.74999 cruncher = SequenceMatcher(self.charjunk) - eqi, eqj = None, None # 1st indices of equal lines (if any) + crqr = cruncher.real_quick_ratio + cqr = cruncher.quick_ratio + cr = cruncher.ratio - # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch up - # on junk -- unless we have to) + WINDOW = 10 + best_i = best_j = None + dump_i, dump_j = alo, blo # smallest indices not yet resolved for j in range(blo, bhi): - bj = b[j] - cruncher.set_seq2(bj) - for i in range(alo, ahi): - ai = a[i] - if ai == bj: - if eqi is None: - eqi, eqj = i, j - continue - cruncher.set_seq1(ai) - # computing similarity is expensive, so use the quick - # upper bounds first -- have seen this speed up messy - # compares by a factor of 3. - # note that ratio() is only expensive to compute the first - # time it's called on a sequence pair; the expensive part - # of the computation is cached by cruncher - if cruncher.real_quick_ratio() > best_ratio and \ - cruncher.quick_ratio() > best_ratio and \ - cruncher.ratio() > best_ratio: - best_ratio, best_i, best_j = cruncher.ratio(), i, j - if best_ratio < cutoff: - # no non-identical "pretty close" pair - if eqi is None: - # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) - return - # no close pair, but an identical pair -- synch up on that - best_i, best_j, best_ratio = eqi, eqj, 1.0 - else: - # there's a close pair, so forget the identical pair (if any) - eqi = None - - # a[best_i] very similar to b[best_j]; eqi is None iff they're not - # identical - - # pump out diffs from before the synch point - yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) - - # do intraline marking on the synch pair - aelt, belt = a[best_i], b[best_j] - if eqi is None: - # pump out a '-', '?', '+', '?' quad for the synched lines - atags = btags = "" - cruncher.set_seqs(aelt, belt) - for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): - la, lb = ai2 - ai1, bj2 - bj1 - if tag == 'replace': - atags += '^' * la - btags += '^' * lb - elif tag == 'delete': - atags += '-' * la - elif tag == 'insert': - btags += '+' * lb - elif tag == 'equal': - atags += ' ' * la - btags += ' ' * lb - else: - raise ValueError('unknown tag %r' % (tag,)) - yield from self._qformat(aelt, belt, atags, btags) - else: - # the synch pair is identical - yield ' ' + aelt + cruncher.set_seq2(b[j]) + # Search the corresponding i's within WINDOW for rhe highest + # ratio greater than `cutoff`. + aequiv = alo + (j - blo) + arange = range(max(aequiv - WINDOW, dump_i), + min(aequiv + WINDOW + 1, ahi)) + if not arange: # likely exit if `a` is shorter than `b` + break + best_ratio = cutoff + for i in arange: + cruncher.set_seq1(a[i]) + # Ordering by cheapest to most expensive ratio is very + # valuable, most often getting out early. + if (crqr() > best_ratio + and cqr() > best_ratio + and cr() > best_ratio): + best_i, best_j, best_ratio = i, j, cr() + + if best_i is None: + # found nothing to synch on yet - move to next j + continue - # pump out diffs from after the synch point - yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + # pump out straight replace from before this synch pair + yield from self._fancy_helper(a, dump_i, best_i, + b, dump_j, best_j) + # do intraline marking on the synch pair + aelt, belt = a[best_i], b[best_j] + if aelt != belt: + # pump out a '-', '?', '+', '?' quad for the synched lines + atags = btags = "" + cruncher.set_seqs(aelt, belt) + for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): + la, lb = ai2 - ai1, bj2 - bj1 + if tag == 'replace': + atags += '^' * la + btags += '^' * lb + elif tag == 'delete': + atags += '-' * la + elif tag == 'insert': + btags += '+' * lb + elif tag == 'equal': + atags += ' ' * la + btags += ' ' * lb + else: + raise ValueError('unknown tag %r' % (tag,)) + yield from self._qformat(aelt, belt, atags, btags) + else: + # the synch pair is identical + yield ' ' + aelt + dump_i, dump_j = best_i + 1, best_j + 1 + best_i = best_j = None + + # pump out straight replace from after the last synch pair + yield from self._fancy_helper(a, dump_i, ahi, + b, dump_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: - g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + g = self._plain_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: @@ -1040,11 +1038,9 @@ def _qformat(self, aline, bline, atags, btags): # remaining is that perhaps it was really the case that " volatile" # was inserted after "private". I can live with that . -import re - -def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): +def IS_LINE_JUNK(line, pat=None): r""" - Return True for ignorable line: iff `line` is blank or contains a single '#'. + Return True for ignorable line: if `line` is blank or contains a single '#'. Examples: @@ -1056,6 +1052,11 @@ def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): False """ + if pat is None: + # Default: match '#' or the empty string + return line.strip() in '#' + # Previous versions used the undocumented parameter 'pat' as a + # match function. Retain this behaviour for compatibility. return pat(line) is not None def IS_CHARACTER_JUNK(ch, ws=" \t"): @@ -1266,6 +1267,12 @@ def _check_types(a, b, *args): if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) + if isinstance(a, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(a).__name__) + if isinstance(b, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(b).__name__) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) @@ -1628,13 +1635,22 @@ def _line_pair_iterator(): """ _styles = """ - table.diff {font-family:Courier; border:medium;} + :root {color-scheme: light dark} + table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium} .diff_header {background-color:#e0e0e0} td.diff_header {text-align:right} .diff_next {background-color:#c0c0c0} - .diff_add {background-color:#aaffaa} + .diff_add {background-color:palegreen} .diff_chg {background-color:#ffff77} - .diff_sub {background-color:#ffaaaa}""" + .diff_sub {background-color:#ffaaaa} + + @media (prefers-color-scheme: dark) { + .diff_header {background-color:#666} + .diff_next {background-color:#393939} + .diff_add {background-color:darkgreen} + .diff_chg {background-color:#847415} + .diff_sub {background-color:darkred} + }""" _table_template = """ '). \ replace('\t',' ') -del re def restore(delta, which): r""" @@ -2047,10 +2062,3 @@ def restore(delta, which): for line in delta: if line[:2] in prefixes: yield line[2:] - -def _test(): - import doctest, difflib - return doctest.testmod(difflib) - -if __name__ == "__main__": - _test() diff --git a/Lib/dis.py b/Lib/dis.py index 53c85555bc9..d6d2c1386dd 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -1,18 +1,1157 @@ -from _dis import * +"""Disassembler of Python byte code into mnemonics.""" +import sys +import types +import collections +import io -# Disassembling a file by following cpython Lib/dis.py -def _test(): - """Simple test program to disassemble a file.""" +from opcode import * +from opcode import ( + __all__ as _opcodes_all, + _cache_format, + _inline_cache_entries, + _nb_ops, + _common_constants, + _intrinsic_1_descs, + _intrinsic_2_descs, + _special_method_names, + _specializations, + _specialized_opmap, +) + +from _opcode import get_executor + +__all__ = ["code_info", "dis", "disassemble", "distb", "disco", + "findlinestarts", "findlabels", "show_code", + "get_instructions", "Instruction", "Bytecode"] + _opcodes_all +del _opcodes_all + +_have_code = (types.MethodType, types.FunctionType, types.CodeType, + classmethod, staticmethod, type) + +CONVERT_VALUE = opmap['CONVERT_VALUE'] + +SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] +FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate') + +ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] +LOAD_GLOBAL = opmap['LOAD_GLOBAL'] +LOAD_SMALL_INT = opmap['LOAD_SMALL_INT'] +BINARY_OP = opmap['BINARY_OP'] +JUMP_BACKWARD = opmap['JUMP_BACKWARD'] +FOR_ITER = opmap['FOR_ITER'] +SEND = opmap['SEND'] +LOAD_ATTR = opmap['LOAD_ATTR'] +LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] +CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] +CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] +LOAD_COMMON_CONSTANT = opmap['LOAD_COMMON_CONSTANT'] +LOAD_SPECIAL = opmap['LOAD_SPECIAL'] +LOAD_FAST_LOAD_FAST = opmap['LOAD_FAST_LOAD_FAST'] +LOAD_FAST_BORROW_LOAD_FAST_BORROW = opmap['LOAD_FAST_BORROW_LOAD_FAST_BORROW'] +STORE_FAST_LOAD_FAST = opmap['STORE_FAST_LOAD_FAST'] +STORE_FAST_STORE_FAST = opmap['STORE_FAST_STORE_FAST'] +IS_OP = opmap['IS_OP'] +CONTAINS_OP = opmap['CONTAINS_OP'] +END_ASYNC_FOR = opmap['END_ASYNC_FOR'] + +CACHE = opmap["CACHE"] + +_all_opname = list(opname) +_all_opmap = dict(opmap) +for name, op in _specialized_opmap.items(): + # fill opname and opmap + assert op < len(_all_opname) + _all_opname[op] = name + _all_opmap[name] = op + +deoptmap = { + specialized: base for base, family in _specializations.items() for specialized in family +} + +def _try_compile(source, name): + """Attempts to compile the given source, first as an expression and + then as a statement if the first approach fails. + + Utility function to accept strings in functions that otherwise + expect code objects + """ + try: + return compile(source, name, 'eval') + except SyntaxError: + pass + return compile(source, name, 'exec') + +def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, + show_offsets=False, show_positions=False): + """Disassemble classes, methods, functions, and other compiled objects. + + With no argument, disassemble the last traceback. + + Compiled objects currently include generator objects, async generator + objects, and coroutine objects, all of which store their code object + in a special attribute. + """ + if x is None: + distb(file=file, show_caches=show_caches, adaptive=adaptive, + show_offsets=show_offsets, show_positions=show_positions) + return + # Extract functions from methods. + if hasattr(x, '__func__'): + x = x.__func__ + # Extract compiled code objects from... + if hasattr(x, '__code__'): # ...a function, or + x = x.__code__ + elif hasattr(x, 'gi_code'): #...a generator object, or + x = x.gi_code + elif hasattr(x, 'ag_code'): #...an asynchronous generator object, or + x = x.ag_code + elif hasattr(x, 'cr_code'): #...a coroutine. + x = x.cr_code + # Perform the disassembly. + if hasattr(x, '__dict__'): # Class or module + items = sorted(x.__dict__.items()) + for name, x1 in items: + if isinstance(x1, _have_code): + print("Disassembly of %s:" % name, file=file) + try: + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + except TypeError as msg: + print("Sorry:", msg, file=file) + print(file=file) + elif hasattr(x, 'co_code'): # Code object + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + elif isinstance(x, (bytes, bytearray)): # Raw bytecode + labels_map = _make_labels_map(x) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + offset_width=len(str(max(len(x) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches) + arg_resolver = ArgResolver(labels_map=labels_map) + _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) + elif isinstance(x, str): # Source code + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + else: + raise TypeError("don't know how to disassemble %s objects" % + type(x).__name__) + +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): + """Disassemble a traceback (default: last traceback).""" + if tb is None: + try: + if hasattr(sys, 'last_exc'): + tb = sys.last_exc.__traceback__ + else: + tb = sys.last_traceback + except AttributeError: + raise RuntimeError("no last traceback to disassemble") from None + while tb.tb_next: tb = tb.tb_next + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + +# The inspect module interrogates this dictionary to build its +# list of CO_* constants. It is also used by pretty_flags to +# turn the co_flags field into a human readable list. +COMPILER_FLAG_NAMES = { + 1: "OPTIMIZED", + 2: "NEWLOCALS", + 4: "VARARGS", + 8: "VARKEYWORDS", + 16: "NESTED", + 32: "GENERATOR", + 64: "NOFREE", + 128: "COROUTINE", + 256: "ITERABLE_COROUTINE", + 512: "ASYNC_GENERATOR", + 0x4000000: "HAS_DOCSTRING", + 0x8000000: "METHOD", +} + +def pretty_flags(flags): + """Return pretty representation of code flags.""" + names = [] + for i in range(32): + flag = 1<" + +# Sentinel to represent values that cannot be calculated +UNKNOWN = _Unknown() + +def _get_code_object(x): + """Helper to handle methods, compiled or raw code objects, and strings.""" + # Extract functions from methods. + if hasattr(x, '__func__'): + x = x.__func__ + # Extract compiled code objects from... + if hasattr(x, '__code__'): # ...a function, or + x = x.__code__ + elif hasattr(x, 'gi_code'): #...a generator object, or + x = x.gi_code + elif hasattr(x, 'ag_code'): #...an asynchronous generator object, or + x = x.ag_code + elif hasattr(x, 'cr_code'): #...a coroutine. + x = x.cr_code + # Handle source code. + if isinstance(x, str): + x = _try_compile(x, "") + # By now, if we don't have a code object, we can't disassemble x. + if hasattr(x, 'co_code'): + return x + raise TypeError("don't know how to disassemble %s objects" % + type(x).__name__) + +def _deoptop(op): + name = _all_opname[op] + return _all_opmap[deoptmap[name]] if name in deoptmap else op + +def _get_code_array(co, adaptive): + if adaptive: + code = co._co_code_adaptive + res = [] + found = False + for i in range(0, len(code), 2): + op, arg = code[i], code[i+1] + if op == ENTER_EXECUTOR: + try: + ex = get_executor(co, i) + except (ValueError, RuntimeError): + ex = None + + if ex: + op, arg = ex.get_opcode(), ex.get_oparg() + found = True + + res.append(op.to_bytes()) + res.append(arg.to_bytes()) + return code if not found else b''.join(res) + else: + return co.co_code + +def code_info(x): + """Formatted details of methods, functions, or code.""" + return _format_code_info(_get_code_object(x)) + +def _format_code_info(co): + lines = [] + lines.append("Name: %s" % co.co_name) + lines.append("Filename: %s" % co.co_filename) + lines.append("Argument count: %s" % co.co_argcount) + lines.append("Positional-only arguments: %s" % co.co_posonlyargcount) + lines.append("Kw-only arguments: %s" % co.co_kwonlyargcount) + lines.append("Number of locals: %s" % co.co_nlocals) + lines.append("Stack size: %s" % co.co_stacksize) + lines.append("Flags: %s" % pretty_flags(co.co_flags)) + if co.co_consts: + lines.append("Constants:") + for i_c in enumerate(co.co_consts): + lines.append("%4d: %r" % i_c) + if co.co_names: + lines.append("Names:") + for i_n in enumerate(co.co_names): + lines.append("%4d: %s" % i_n) + if co.co_varnames: + lines.append("Variable names:") + for i_n in enumerate(co.co_varnames): + lines.append("%4d: %s" % i_n) + if co.co_freevars: + lines.append("Free variables:") + for i_n in enumerate(co.co_freevars): + lines.append("%4d: %s" % i_n) + if co.co_cellvars: + lines.append("Cell variables:") + for i_n in enumerate(co.co_cellvars): + lines.append("%4d: %s" % i_n) + return "\n".join(lines) + +def show_code(co, *, file=None): + """Print details of methods, functions, or code to *file*. + + If *file* is not provided, the output is printed on stdout. + """ + print(code_info(co), file=file) + +Positions = collections.namedtuple( + 'Positions', + [ + 'lineno', + 'end_lineno', + 'col_offset', + 'end_col_offset', + ], + defaults=[None] * 4 +) + +_Instruction = collections.namedtuple( + "_Instruction", + [ + 'opname', + 'opcode', + 'arg', + 'argval', + 'argrepr', + 'offset', + 'start_offset', + 'starts_line', + 'line_number', + 'label', + 'positions', + 'cache_info', + ], + defaults=[None, None, None] +) + +_Instruction.opname.__doc__ = "Human readable name for operation" +_Instruction.opcode.__doc__ = "Numeric code for operation" +_Instruction.arg.__doc__ = "Numeric argument to operation (if any), otherwise None" +_Instruction.argval.__doc__ = "Resolved arg value (if known), otherwise same as arg" +_Instruction.argrepr.__doc__ = "Human readable description of operation argument" +_Instruction.offset.__doc__ = "Start index of operation within bytecode sequence" +_Instruction.start_offset.__doc__ = ( + "Start index of operation within bytecode sequence, including extended args if present; " + "otherwise equal to Instruction.offset" +) +_Instruction.starts_line.__doc__ = "True if this opcode starts a source line, otherwise False" +_Instruction.line_number.__doc__ = "source line number associated with this opcode (if any), otherwise None" +_Instruction.label.__doc__ = "A label (int > 0) if this instruction is a jump target, otherwise None" +_Instruction.positions.__doc__ = "dis.Positions object holding the span of source code covered by this instruction" +_Instruction.cache_info.__doc__ = "list of (name, size, data), one for each cache entry of the instruction" + +_ExceptionTableEntryBase = collections.namedtuple("_ExceptionTableEntryBase", + "start end target depth lasti") + +class _ExceptionTableEntry(_ExceptionTableEntryBase): + pass + +_OPNAME_WIDTH = 20 +_OPARG_WIDTH = 5 + +def _get_cache_size(opname): + return _inline_cache_entries.get(opname, 0) + +def _get_jump_target(op, arg, offset): + """Gets the bytecode offset of the jump target if this is a jump instruction. + + Otherwise return None. + """ + deop = _deoptop(op) + caches = _get_cache_size(_all_opname[deop]) + if deop in hasjrel: + if _is_backward_jump(deop): + arg = -arg + target = offset + 2 + arg*2 + target += 2 * caches + elif deop in hasjabs: + target = arg*2 + else: + target = None + return target + +class Instruction(_Instruction): + """Details for a bytecode operation. + + Defined fields: + opname - human readable name for operation + opcode - numeric code for operation + arg - numeric argument to operation (if any), otherwise None + argval - resolved arg value (if known), otherwise same as arg + argrepr - human readable description of operation argument + offset - start index of operation within bytecode sequence + start_offset - start index of operation within bytecode sequence including extended args if present; + otherwise equal to Instruction.offset + starts_line - True if this opcode starts a source line, otherwise False + line_number - source line number associated with this opcode (if any), otherwise None + label - A label if this instruction is a jump target, otherwise None + positions - Optional dis.Positions object holding the span of source code + covered by this instruction + cache_info - information about the format and content of the instruction's cache + entries (if any) + """ + + @staticmethod + def make( + opname, arg, argval, argrepr, offset, start_offset, starts_line, + line_number, label=None, positions=None, cache_info=None + ): + return Instruction(opname, _all_opmap[opname], arg, argval, argrepr, offset, + start_offset, starts_line, line_number, label, positions, cache_info) + + @property + def oparg(self): + """Alias for Instruction.arg.""" + return self.arg + + @property + def baseopcode(self): + """Numeric code for the base operation if operation is specialized. + + Otherwise equal to Instruction.opcode. + """ + return _deoptop(self.opcode) + + @property + def baseopname(self): + """Human readable name for the base operation if operation is specialized. + + Otherwise equal to Instruction.opname. + """ + return opname[self.baseopcode] + + @property + def cache_offset(self): + """Start index of the cache entries following the operation.""" + return self.offset + 2 + + @property + def end_offset(self): + """End index of the cache entries following the operation.""" + return self.cache_offset + _get_cache_size(_all_opname[self.opcode])*2 + + @property + def jump_target(self): + """Bytecode index of the jump target if this is a jump operation. + + Otherwise return None. + """ + return _get_jump_target(self.opcode, self.arg, self.offset) + + @property + def is_jump_target(self): + """True if other code jumps to here, otherwise False""" + return self.label is not None + + def __str__(self): + output = io.StringIO() + formatter = Formatter(file=output) + formatter.print_instruction(self, False) + return output.getvalue() + + +class Formatter: + + def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, + line_offset=0, show_caches=False, *, show_positions=False): + """Create a Formatter + + *file* where to write the output + *lineno_width* sets the width of the source location field (0 omits it). + Should be large enough for a line number or full positions (depending + on the value of *show_positions*). + *offset_width* sets the width of the instruction offset field + *label_width* sets the width of the label field + *show_caches* is a boolean indicating whether to display cache lines + *show_positions* is a boolean indicating whether full positions should + be reported instead of only the line numbers. + """ + self.file = file + self.lineno_width = lineno_width + self.offset_width = offset_width + self.label_width = label_width + self.show_caches = show_caches + self.show_positions = show_positions + + def print_instruction(self, instr, mark_as_current=False): + self.print_instruction_line(instr, mark_as_current) + if self.show_caches and instr.cache_info: + offset = instr.offset + for name, size, data in instr.cache_info: + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + self.print_instruction_line( + Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions), + False) + + def print_instruction_line(self, instr, mark_as_current): + """Format instruction details for inclusion in disassembly output.""" + lineno_width = self.lineno_width + offset_width = self.offset_width + label_width = self.label_width + + new_source_line = (lineno_width > 0 and + instr.starts_line and + instr.offset > 0) + if new_source_line: + print(file=self.file) + + fields = [] + # Column: Source code locations information + if lineno_width: + if self.show_positions: + # reporting positions instead of just line numbers + if instr_positions := instr.positions: + if all(p is None for p in instr_positions): + positions_str = _NO_LINENO + else: + ps = tuple('?' if p is None else p for p in instr_positions) + positions_str = f"{ps[0]}:{ps[2]}-{ps[1]}:{ps[3]}" + fields.append(f'{positions_str:{lineno_width}}') + else: + fields.append(' ' * lineno_width) + else: + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" + lineno_fmt = lineno_fmt % lineno_width + lineno = _NO_LINENO if instr.line_number is None else instr.line_number + fields.append(lineno_fmt % lineno) + else: + fields.append(' ' * lineno_width) + # Column: Label + if instr.label is not None: + lbl = f"L{instr.label}:" + fields.append(f"{lbl:>{label_width}}") + else: + fields.append(' ' * label_width) + # Column: Instruction offset from start of code sequence + if offset_width > 0: + fields.append(f"{repr(instr.offset):>{offset_width}} ") + # Column: Current instruction indicator + if mark_as_current: + fields.append('-->') + else: + fields.append(' ') + # Column: Opcode name + fields.append(instr.opname.ljust(_OPNAME_WIDTH)) + # Column: Opcode argument + if instr.arg is not None: + arg = repr(instr.arg) + # If opname is longer than _OPNAME_WIDTH, we allow it to overflow into + # the space reserved for oparg. This results in fewer misaligned opargs + # in the disassembly output. + opname_excess = max(0, len(instr.opname) - _OPNAME_WIDTH) + fields.append(repr(instr.arg).rjust(_OPARG_WIDTH - opname_excess)) + # Column: Opcode argument details + if instr.argrepr: + fields.append('(' + instr.argrepr + ')') + print(' '.join(fields).rstrip(), file=self.file) + + def print_exception_table(self, exception_entries): + file = self.file + if exception_entries: + print("ExceptionTable:", file=file) + for entry in exception_entries: + lasti = " lasti" if entry.lasti else "" + start = entry.start_label + end = entry.end_label + target = entry.target_label + print(f" L{start} to L{end} -> L{target} [{entry.depth}]{lasti}", file=file) + + +class ArgResolver: + def __init__(self, co_consts=None, names=None, varname_from_oparg=None, labels_map=None): + self.co_consts = co_consts + self.names = names + self.varname_from_oparg = varname_from_oparg + self.labels_map = labels_map or {} + + def offset_from_jump_arg(self, op, arg, offset): + deop = _deoptop(op) + if deop in hasjabs: + return arg * 2 + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + return argval + return None + + def get_label_for_offset(self, offset): + return self.labels_map.get(offset, None) + + def get_argval_argrepr(self, op, arg, offset): + get_name = None if self.names is None else self.names.__getitem__ + argval = None + argrepr = '' + deop = _deoptop(op) + if arg is not None: + # Set argval to the dereferenced value of the argument when + # available, and argrepr to the string representation of argval. + # _disassemble_bytes needs the string repr of the + # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. + argval = arg + if deop in hasconst: + argval, argrepr = _get_const_info(deop, arg, self.co_consts) + elif deop in hasname: + if deop == LOAD_GLOBAL: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL" + elif deop == LOAD_ATTR: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + elif deop == LOAD_SUPER_ATTR: + argval, argrepr = _get_name_info(arg//4, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + else: + argval, argrepr = _get_name_info(arg, get_name) + elif deop in hasjump or deop in hasexc: + argval = self.offset_from_jump_arg(op, arg, offset) + lbl = self.get_label_for_offset(argval) + assert lbl is not None + preposition = "from" if deop == END_ASYNC_FOR else "to" + argrepr = f"{preposition} L{lbl}" + elif deop in (LOAD_FAST_LOAD_FAST, LOAD_FAST_BORROW_LOAD_FAST_BORROW, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): + arg1 = arg >> 4 + arg2 = arg & 15 + val1, argrepr1 = _get_name_info(arg1, self.varname_from_oparg) + val2, argrepr2 = _get_name_info(arg2, self.varname_from_oparg) + argrepr = argrepr1 + ", " + argrepr2 + argval = val1, val2 + elif deop in haslocal or deop in hasfree: + argval, argrepr = _get_name_info(arg, self.varname_from_oparg) + elif deop in hascompare: + argval = cmp_op[arg >> 5] + argrepr = argval + if arg & 16: + argrepr = f"bool({argrepr})" + elif deop == CONVERT_VALUE: + argval = (None, str, repr, ascii)[arg] + argrepr = ('', 'str', 'repr', 'ascii')[arg] + elif deop == SET_FUNCTION_ATTRIBUTE: + argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) + if arg & (1<> 1 + lasti = bool(dl&1) + entries.append(_ExceptionTableEntry(start, end, target, depth, lasti)) + except StopIteration: + return entries + +def _is_backward_jump(op): + return opname[op] in ('JUMP_BACKWARD', + 'JUMP_BACKWARD_NO_INTERRUPT', + 'END_ASYNC_FOR') # Not really a jump, but it has a "target" + +def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, + original_code=None, arg_resolver=None): + """Iterate over the instructions in a bytecode string. + + Generates a sequence of Instruction namedtuples giving the details of each + opcode. + + """ + # Use the basic, unadaptive code for finding labels and actually walking the + # bytecode, since replacements like ENTER_EXECUTOR and INSTRUMENTED_* can + # mess that logic up pretty badly: + original_code = original_code or code + co_positions = co_positions or iter(()) + + starts_line = False + local_line_number = None + line_number = None + for offset, start_offset, op, arg in _unpack_opargs(original_code): + if linestarts is not None: + starts_line = offset in linestarts + if starts_line: + local_line_number = linestarts[offset] + if local_line_number is not None: + line_number = local_line_number + line_offset + else: + line_number = None + positions = Positions(*next(co_positions, ())) + deop = _deoptop(op) + op = code[offset] + + if arg_resolver: + argval, argrepr = arg_resolver.get_argval_argrepr(op, arg, offset) + else: + argval, argrepr = arg, repr(arg) + + caches = _get_cache_size(_all_opname[deop]) + # Advance the co_positions iterator: + for _ in range(caches): + next(co_positions, ()) + + if caches: + cache_info = [] + cache_offset = offset + for name, size in _cache_format[opname[deop]].items(): + data = code[cache_offset + 2: cache_offset + 2 + 2 * size] + cache_offset += size * 2 + cache_info.append((name, size, data)) + else: + cache_info = None + + label = arg_resolver.get_label_for_offset(offset) if arg_resolver else None + yield Instruction(_all_opname[op], op, arg, argval, argrepr, + offset, start_offset, starts_line, line_number, + label, positions, cache_info) + + +def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, + show_offsets=False, show_positions=False): + """Disassemble a code object.""" + linestarts = dict(findlinestarts(co)) + exception_entries = _parse_exception_table(co) + if show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(linestarts) + labels_map = _make_labels_map(co.co_code, exception_entries=exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + lineno_width=lineno_width, + offset_width=len(str(max(len(co.co_code) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches, + show_positions=show_positions) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(_get_code_array(co, adaptive), lasti, linestarts, + exception_entries=exception_entries, co_positions=co.co_positions(), + original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) + +def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): + disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + if depth is None or depth > 0: + if depth is not None: + depth = depth - 1 + for x in co.co_consts: + if hasattr(x, 'co_code'): + print(file=file) + print("Disassembly of %r:" % (x,), file=file) + _disassemble_recursive( + x, file=file, depth=depth, show_caches=show_caches, + adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions + ) + + +def _make_labels_map(original_code, exception_entries=()): + jump_targets = set(findlabels(original_code)) + labels = set(jump_targets) + for start, end, target, _, _ in exception_entries: + labels.add(start) + labels.add(end) + labels.add(target) + labels = sorted(labels) + labels_map = {offset: i+1 for (i, offset) in enumerate(sorted(labels))} + for e in exception_entries: + e.start_label = labels_map[e.start] + e.end_label = labels_map[e.end] + e.target_label = labels_map[e.target] + return labels_map + +_NO_LINENO = ' --' + +def _get_lineno_width(linestarts): + if linestarts is None: + return 0 + maxlineno = max(filter(None, linestarts.values()), default=-1) + if maxlineno == -1: + # Omit the line number column entirely if we have no line number info + return 0 + lineno_width = max(3, len(str(maxlineno))) + if lineno_width < len(_NO_LINENO) and None in linestarts.values(): + lineno_width = len(_NO_LINENO) + return lineno_width + +def _get_positions_width(code): + # Positions are formatted as 'LINE:COL-ENDLINE:ENDCOL ' (note trailing space). + # A missing component appears as '?', and when all components are None, we + # render '_NO_LINENO'. thus the minimum width is 1 + len(_NO_LINENO). + # + # If all values are missing, positions are not printed (i.e. positions_width = 0). + has_value = False + values_width = 0 + for positions in code.co_positions(): + has_value |= any(isinstance(p, int) for p in positions) + width = sum(1 if p is None else len(str(p)) for p in positions) + values_width = max(width, values_width) + if has_value: + # 3 = number of separators in a normal format + return 1 + max(len(_NO_LINENO), 3 + values_width) + return 0 + +def _disassemble_bytes(code, lasti=-1, linestarts=None, + *, line_offset=0, exception_entries=(), + co_positions=None, original_code=None, + arg_resolver=None, formatter=None): + + assert formatter is not None + assert arg_resolver is not None + + instrs = _get_instructions_bytes(code, linestarts=linestarts, + line_offset=line_offset, + co_positions=co_positions, + original_code=original_code, + arg_resolver=arg_resolver) + + print_instructions(instrs, exception_entries, formatter, lasti=lasti) + + +def print_instructions(instrs, exception_entries, formatter, lasti=-1): + for instr in instrs: + # Each CACHE takes 2 bytes + is_current_instr = instr.offset <= lasti \ + <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) + formatter.print_instruction(instr, is_current_instr) + + formatter.print_exception_table(exception_entries) + +def _disassemble_str(source, **kwargs): + """Compile the source string, then disassemble the code object.""" + _disassemble_recursive(_try_compile(source, ''), **kwargs) + +disco = disassemble # XXX For backwards compatibility + + +# Rely on C `int` being 32 bits for oparg +_INT_BITS = 32 +# Value for c int when it overflows +_INT_OVERFLOW = 2 ** (_INT_BITS - 1) + +def _unpack_opargs(code): + extended_arg = 0 + extended_args_offset = 0 # Number of EXTENDED_ARG instructions preceding the current instruction + caches = 0 + for i in range(0, len(code), 2): + # Skip inline CACHE entries: + if caches: + caches -= 1 + continue + op = code[i] + deop = _deoptop(op) + caches = _get_cache_size(_all_opname[deop]) + if deop in hasarg: + arg = code[i+1] | extended_arg + extended_arg = (arg << 8) if deop == EXTENDED_ARG else 0 + # The oparg is stored as a signed integer + # If the value exceeds its upper limit, it will overflow and wrap + # to a negative integer + if extended_arg >= _INT_OVERFLOW: + extended_arg -= 2 * _INT_OVERFLOW + else: + arg = None + extended_arg = 0 + if deop == EXTENDED_ARG: + extended_args_offset += 1 + yield (i, i, op, arg) + else: + start_offset = i - extended_args_offset*2 + yield (i, start_offset, op, arg) + extended_args_offset = 0 + +def findlabels(code): + """Detect all offsets in a byte code which are jump targets. + + Return the list of offsets. + + """ + labels = [] + for offset, _, op, arg in _unpack_opargs(code): + if arg is not None: + label = _get_jump_target(op, arg, offset) + if label is None: + continue + if label not in labels: + labels.append(label) + return labels + +def findlinestarts(code): + """Find the offsets in a byte code which are start of lines in the source. + + Generate pairs (offset, lineno) + lineno will be an integer or None the offset does not have a source line. + """ + + lastline = False # None is a valid line number + for start, end, line in code.co_lines(): + if line is not lastline: + lastline = line + yield start, line + return + +def _find_imports(co): + """Find import statements in the code + + Generate triplets (name, level, fromlist) where + name is the imported module and level, fromlist are + the corresponding args to __import__. + """ + IMPORT_NAME = opmap['IMPORT_NAME'] + + consts = co.co_consts + names = co.co_names + opargs = [(op, arg) for _, _, op, arg in _unpack_opargs(co.co_code) + if op != EXTENDED_ARG] + for i, (op, oparg) in enumerate(opargs): + if op == IMPORT_NAME and i >= 2: + from_op = opargs[i-1] + level_op = opargs[i-2] + if (from_op[0] in hasconst and + (level_op[0] in hasconst or level_op[0] == LOAD_SMALL_INT)): + level = _get_const_value(level_op[0], level_op[1], consts) + fromlist = _get_const_value(from_op[0], from_op[1], consts) + yield (names[oparg], level, fromlist) + +def _find_store_names(co): + """Find names of variables which are written in the code + + Generate sequence of strings + """ + STORE_OPS = { + opmap['STORE_NAME'], + opmap['STORE_GLOBAL'] + } + + names = co.co_names + for _, _, op, arg in _unpack_opargs(co.co_code): + if op in STORE_OPS: + yield names[arg] + + +class Bytecode: + """The bytecode operations of a piece of code + + Instantiate this with a function, method, other compiled object, string of + code, or a code object (as returned by compile()). + + Iterating over this yields the bytecode operations as Instruction instances. + """ + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): + self.codeobj = co = _get_code_object(x) + if first_line is None: + self.first_line = co.co_firstlineno + self._line_offset = 0 + else: + self.first_line = first_line + self._line_offset = first_line - co.co_firstlineno + self._linestarts = dict(findlinestarts(co)) + self._original_object = x + self.current_offset = current_offset + self.exception_entries = _parse_exception_table(co) + self.show_caches = show_caches + self.adaptive = adaptive + self.show_offsets = show_offsets + self.show_positions = show_positions + + def __iter__(self): + co = self.codeobj + original_code = co.co_code + labels_map = _make_labels_map(original_code, self.exception_entries) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + return _get_instructions_bytes(_get_code_array(co, self.adaptive), + linestarts=self._linestarts, + line_offset=self._line_offset, + co_positions=co.co_positions(), + original_code=original_code, + arg_resolver=arg_resolver) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, + self._original_object) + + @classmethod + def from_traceback(cls, tb, *, show_caches=False, adaptive=False): + """ Construct a Bytecode from the given traceback """ + while tb.tb_next: + tb = tb.tb_next + return cls( + tb.tb_frame.f_code, current_offset=tb.tb_lasti, show_caches=show_caches, adaptive=adaptive + ) + + def info(self): + """Return formatted information about the code object.""" + return _format_code_info(self.codeobj) + + def dis(self): + """Return a formatted view of the bytecode operations.""" + co = self.codeobj + if self.current_offset is not None: + offset = self.current_offset + else: + offset = -1 + with io.StringIO() as output: + code = _get_code_array(co, self.adaptive) + offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 + if self.show_positions: + lineno_width = _get_positions_width(co) + else: + lineno_width = _get_lineno_width(self._linestarts) + labels_map = _make_labels_map(co.co_code, self.exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=output, + lineno_width=lineno_width, + offset_width=offset_width, + label_width=label_width, + line_offset=self._line_offset, + show_caches=self.show_caches, + show_positions=self.show_positions) + + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(code, + linestarts=self._linestarts, + line_offset=self._line_offset, + lasti=offset, + exception_entries=self.exception_entries, + co_positions=co.co_positions(), + original_code=co.co_code, + arg_resolver=arg_resolver, + formatter=formatter) + return output.getvalue() + + +def main(args=None): import argparse - parser = argparse.ArgumentParser() - parser.add_argument('infile', type=argparse.FileType('rb'), nargs='?', default='-') - args = parser.parse_args() - with args.infile as infile: - source = infile.read() - code = compile(source, args.infile.name, "exec") - dis(code) + parser = argparse.ArgumentParser(color=True) + parser.add_argument('-C', '--show-caches', action='store_true', + help='show inline caches') + parser.add_argument('-O', '--show-offsets', action='store_true', + help='show instruction offsets') + parser.add_argument('-P', '--show-positions', action='store_true', + help='show instruction positions') + parser.add_argument('-S', '--specialized', action='store_true', + help='show specialized bytecode') + parser.add_argument('infile', nargs='?', default='-') + args = parser.parse_args(args=args) + if args.infile == '-': + name = '' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + code = compile(source, name, "exec") + dis(code, show_caches=args.show_caches, adaptive=args.specialized, + show_offsets=args.show_offsets, show_positions=args.show_positions) if __name__ == "__main__": - _test() + main() diff --git a/Lib/distutils/README b/Lib/distutils/README deleted file mode 100644 index 408a203b85d..00000000000 --- a/Lib/distutils/README +++ /dev/null @@ -1,13 +0,0 @@ -This directory contains the Distutils package. - -There's a full documentation available at: - - http://docs.python.org/distutils/ - -The Distutils-SIG web page is also a good starting point: - - http://www.python.org/sigs/distutils-sig/ - -WARNING : Distutils must remain compatible with 2.3 - -$Id$ diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py deleted file mode 100644 index d823d040a1c..00000000000 --- a/Lib/distutils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""distutils - -The main package for the Python Module Distribution Utilities. Normally -used from a setup script as - - from distutils.core import setup - - setup (...) -""" - -import sys - -__version__ = sys.version[:sys.version.index(' ')] diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py deleted file mode 100644 index 30b3b473985..00000000000 --- a/Lib/distutils/_msvccompiler.py +++ /dev/null @@ -1,574 +0,0 @@ -"""distutils._msvccompiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for Microsoft Visual Studio 2015. - -The module is compatible with VS 2015 and later. You can find legacy support -for older versions in distutils.msvc9compiler and distutils.msvccompiler. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) -# ported to VS 2005 and VS 2008 by Christian Heimes -# ported to VS 2015 by Steve Dower - -import os -import shutil -import stat -import subprocess -import winreg - -from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import CCompiler, gen_lib_options -from distutils import log -from distutils.util import get_platform - -from itertools import count - -def _find_vc2015(): - try: - key = winreg.OpenKeyEx( - winreg.HKEY_LOCAL_MACHINE, - r"Software\Microsoft\VisualStudio\SxS\VC7", - access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY - ) - except OSError: - log.debug("Visual C++ is not registered") - return None, None - - best_version = 0 - best_dir = None - with key: - for i in count(): - try: - v, vc_dir, vt = winreg.EnumValue(key, i) - except OSError: - break - if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): - try: - version = int(float(v)) - except (ValueError, TypeError): - continue - if version >= 14 and version > best_version: - best_version, best_dir = version, vc_dir - return best_version, best_dir - -def _find_vc2017(): - import _distutils_findvs - import threading - - best_version = 0, # tuple for full version comparisons - best_dir = None - - # We need to call findall() on its own thread because it will - # initialize COM. - all_packages = [] - def _getall(): - all_packages.extend(_distutils_findvs.findall()) - t = threading.Thread(target=_getall) - t.start() - t.join() - - for name, version_str, path, packages in all_packages: - if 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64' in packages: - vc_dir = os.path.join(path, 'VC', 'Auxiliary', 'Build') - if not os.path.isdir(vc_dir): - continue - try: - version = tuple(int(i) for i in version_str.split('.')) - except (ValueError, TypeError): - continue - if version > best_version: - best_version, best_dir = version, vc_dir - try: - best_version = best_version[0] - except IndexError: - best_version = None - return best_version, best_dir - -def _find_vcvarsall(plat_spec): - best_version, best_dir = _find_vc2017() - vcruntime = None - vcruntime_plat = 'x64' if 'amd64' in plat_spec else 'x86' - if best_version: - vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**", - "Microsoft.VC141.CRT", "vcruntime140.dll") - try: - import glob - vcruntime = glob.glob(vcredist, recursive=True)[-1] - except (ImportError, OSError, LookupError): - vcruntime = None - - if not best_version: - best_version, best_dir = _find_vc2015() - if best_version: - vcruntime = os.path.join(best_dir, 'redist', vcruntime_plat, - "Microsoft.VC140.CRT", "vcruntime140.dll") - - if not best_version: - log.debug("No suitable Visual C++ version found") - return None, None - - vcvarsall = os.path.join(best_dir, "vcvarsall.bat") - if not os.path.isfile(vcvarsall): - log.debug("%s cannot be found", vcvarsall) - return None, None - - if not vcruntime or not os.path.isfile(vcruntime): - log.debug("%s cannot be found", vcruntime) - vcruntime = None - - return vcvarsall, vcruntime - -def _get_vc_env(plat_spec): - if os.getenv("DISTUTILS_USE_SDK"): - return { - key.lower(): value - for key, value in os.environ.items() - } - - vcvarsall, vcruntime = _find_vcvarsall(plat_spec) - if not vcvarsall: - raise DistutilsPlatformError("Unable to find vcvarsall.bat") - - try: - out = subprocess.check_output( - 'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec), - stderr=subprocess.STDOUT, - ).decode('utf-16le', errors='replace') - except subprocess.CalledProcessError as exc: - log.error(exc.output) - raise DistutilsPlatformError("Error executing {}" - .format(exc.cmd)) - - env = { - key.lower(): value - for key, _, value in - (line.partition('=') for line in out.splitlines()) - if key and value - } - - if vcruntime: - env['py_vcruntime_redist'] = vcruntime - return env - -def _find_exe(exe, paths=None): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - if not paths: - paths = os.getenv('path').split(os.pathsep) - for p in paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - return exe - -# A map keyed by get_platform() return values to values accepted by -# 'vcvarsall.bat'. Always cross-compile from x86 to work with the -# lighter-weight MSVC installs that do not include native 64-bit tools. -PLAT_TO_VCVARS = { - 'win32' : 'x86', - 'win-amd64' : 'x86_amd64', -} - -# A set containing the DLLs that are guaranteed to be available for -# all micro versions of this Python version. Known extension -# dependencies that are not in this set will be copied to the output -# path. -_BUNDLED_DLLS = frozenset(['vcruntime140.dll']) - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - # target platform (.plat_name is consistent with 'bdist') - self.plat_name = None - self.initialized = False - - def initialize(self, plat_name=None): - # multi-init means we would need to check platform same each time... - assert not self.initialized, "don't init multiple times" - if plat_name is None: - plat_name = get_platform() - # sanity check for platforms to prevent obscure errors later. - if plat_name not in PLAT_TO_VCVARS: - raise DistutilsPlatformError("--plat-name must be one of {}" - .format(tuple(PLAT_TO_VCVARS))) - - # Get the vcvarsall.bat spec for the requested platform. - plat_spec = PLAT_TO_VCVARS[plat_name] - - vc_env = _get_vc_env(plat_spec) - if not vc_env: - raise DistutilsPlatformError("Unable to find a compatible " - "Visual Studio installation.") - - self._paths = vc_env.get('path', '') - paths = self._paths.split(os.pathsep) - self.cc = _find_exe("cl.exe", paths) - self.linker = _find_exe("link.exe", paths) - self.lib = _find_exe("lib.exe", paths) - self.rc = _find_exe("rc.exe", paths) # resource compiler - self.mc = _find_exe("mc.exe", paths) # message compiler - self.mt = _find_exe("mt.exe", paths) # message compiler - self._vcruntime_redist = vc_env.get('py_vcruntime_redist', '') - - for dir in vc_env.get('include', '').split(os.pathsep): - if dir: - self.add_include_dir(dir.rstrip(os.sep)) - - for dir in vc_env.get('lib', '').split(os.pathsep): - if dir: - self.add_library_dir(dir.rstrip(os.sep)) - - self.preprocess_options = None - # If vcruntime_redist is available, link against it dynamically. Otherwise, - # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib - # later to dynamically link to ucrtbase but not vcruntime. - self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG' - ] - self.compile_options.append('/MD' if self._vcruntime_redist else '/MT') - - self.compile_options_debug = [ - '/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG' - ] - - ldflags = [ - '/nologo', '/INCREMENTAL:NO', '/LTCG' - ] - if not self._vcruntime_redist: - ldflags.extend(('/nodefaultlib:libucrt.lib', 'ucrt.lib')) - - ldflags_debug = [ - '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL' - ] - - self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1'] - self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1'] - self.ldflags_shared = [*ldflags, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] - self.ldflags_shared_debug = [*ldflags_debug, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] - self.ldflags_static = [*ldflags] - self.ldflags_static_debug = [*ldflags_debug] - - self._ldflags = { - (CCompiler.EXECUTABLE, None): self.ldflags_exe, - (CCompiler.EXECUTABLE, False): self.ldflags_exe, - (CCompiler.EXECUTABLE, True): self.ldflags_exe_debug, - (CCompiler.SHARED_OBJECT, None): self.ldflags_shared, - (CCompiler.SHARED_OBJECT, False): self.ldflags_shared, - (CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug, - (CCompiler.SHARED_LIBRARY, None): self.ldflags_static, - (CCompiler.SHARED_LIBRARY, False): self.ldflags_static, - (CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug, - } - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - ext_map = { - **{ext: self.obj_extension for ext in self.src_extensions}, - **{ext: self.res_extension for ext in self._rc_extensions + self._mc_extensions}, - } - - output_dir = output_dir or '' - - def make_out_path(p): - base, ext = os.path.splitext(p) - if strip_dir: - base = os.path.basename(base) - else: - _, base = os.path.splitdrive(base) - if base.startswith((os.path.sep, os.path.altsep)): - base = base[1:] - try: - # XXX: This may produce absurdly long paths. We should check - # the length of the result and trim base until we fit within - # 260 characters. - return os.path.join(output_dir, base + ext_map[ext]) - except LookupError: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError("Don't know how to compile {}".format(p)) - - return list(map(make_out_path, source_filenames)) - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - - add_cpp_opts = False - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - add_cpp_opts = True - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + [output_opt, input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src]) - base, _ = os.path.splitext(os.path.basename (src)) - rc_file = os.path.join(rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc, "/fo" + obj, rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile {} to {}" - .format(src, obj)) - - args = [self.cc] + compile_opts + pp_opts - if add_cpp_opts: - args.append('/EHsc') - args.append(input_opt) - args.append("/Fo" + obj) - args.extend(extra_postargs) - - try: - self.spawn(args) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - objects, output_dir = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - log.debug('Executing "%s" %s', self.lib, ' '.join(lib_args)) - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - if runtime_library_dirs: - self.warn("I don't know what to do with 'runtime_library_dirs': " - + str(runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ldflags = self._ldflags[target_desc, debug] - - export_opts = ["/EXPORT:" + sym for sym in (export_symbols or [])] - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - build_temp = os.path.dirname(objects[0]) - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - build_temp, - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - output_dir = os.path.dirname(os.path.abspath(output_filename)) - self.mkpath(output_dir) - try: - log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args)) - self.spawn([self.linker] + ld_args) - self._copy_vcruntime(output_dir) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def _copy_vcruntime(self, output_dir): - vcruntime = self._vcruntime_redist - if not vcruntime or not os.path.isfile(vcruntime): - return - - if os.path.basename(vcruntime).lower() in _BUNDLED_DLLS: - return - - log.debug('Copying "%s"', vcruntime) - vcruntime = shutil.copy(vcruntime, output_dir) - os.chmod(vcruntime, stat.S_IWRITE) - - def spawn(self, cmd): - old_path = os.getenv('path') - try: - os.environ['path'] = self._paths - return super().spawn(cmd) - finally: - os.environ['path'] = old_path - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC") - - def library_option(self, lib): - return self.library_filename(lib) - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename(name)) - if os.path.isfile(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/Lib/distutils/archive_util.py b/Lib/distutils/archive_util.py deleted file mode 100644 index b002dc3b845..00000000000 --- a/Lib/distutils/archive_util.py +++ /dev/null @@ -1,256 +0,0 @@ -"""distutils.archive_util - -Utility functions for creating archive files (tarballs, zip files, -that sort of thing).""" - -import os -from warnings import warn -import sys - -try: - import zipfile -except ImportError: - zipfile = None - - -from distutils.errors import DistutilsExecError -from distutils.spawn import spawn -from distutils.dir_util import mkpath -from distutils import log - -try: - from pwd import getpwnam -except ImportError: - getpwnam = None - -try: - from grp import getgrnam -except ImportError: - getgrnam = None - -def _get_gid(name): - """Returns a gid, given a group name.""" - if getgrnam is None or name is None: - return None - try: - result = getgrnam(name) - except KeyError: - result = None - if result is not None: - return result[2] - return None - -def _get_uid(name): - """Returns an uid, given a user name.""" - if getpwnam is None or name is None: - return None - try: - result = getpwnam(name) - except KeyError: - result = None - if result is not None: - return result[2] - return None - -def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, - owner=None, group=None): - """Create a (possibly compressed) tar file from all the files under - 'base_dir'. - - 'compress' must be "gzip" (the default), "bzip2", "xz", "compress", or - None. ("compress" will be deprecated in Python 3.2) - - 'owner' and 'group' can be used to define an owner and a group for the - archive that is being built. If not provided, the current owner and group - will be used. - - The output tar file will be named 'base_dir' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", ".xz" or ".Z"). - - Returns the output filename. - """ - tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', 'xz': 'xz', None: '', - 'compress': ''} - compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'xz': '.xz', - 'compress': '.Z'} - - # flags for compression program, each element of list will be an argument - if compress is not None and compress not in compress_ext.keys(): - raise ValueError( - "bad value for 'compress': must be None, 'gzip', 'bzip2', " - "'xz' or 'compress'") - - archive_name = base_name + '.tar' - if compress != 'compress': - archive_name += compress_ext.get(compress, '') - - mkpath(os.path.dirname(archive_name), dry_run=dry_run) - - # creating the tarball - import tarfile # late import so Python build itself doesn't break - - log.info('Creating tar archive') - - uid = _get_uid(owner) - gid = _get_gid(group) - - def _set_uid_gid(tarinfo): - if gid is not None: - tarinfo.gid = gid - tarinfo.gname = group - if uid is not None: - tarinfo.uid = uid - tarinfo.uname = owner - return tarinfo - - if not dry_run: - tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress]) - try: - tar.add(base_dir, filter=_set_uid_gid) - finally: - tar.close() - - # compression using `compress` - if compress == 'compress': - warn("'compress' will be deprecated.", PendingDeprecationWarning) - # the option varies depending on the platform - compressed_name = archive_name + compress_ext[compress] - if sys.platform == 'win32': - cmd = [compress, archive_name, compressed_name] - else: - cmd = [compress, '-f', archive_name] - spawn(cmd, dry_run=dry_run) - return compressed_name - - return archive_name - -def make_zipfile(base_name, base_dir, verbose=0, dry_run=0): - """Create a zip file from all the files under 'base_dir'. - - The output zip file will be named 'base_name' + ".zip". Uses either the - "zipfile" Python module (if available) or the InfoZIP "zip" utility - (if installed and found on the default search path). If neither tool is - available, raises DistutilsExecError. Returns the name of the output zip - file. - """ - zip_filename = base_name + ".zip" - mkpath(os.path.dirname(zip_filename), dry_run=dry_run) - - # If zipfile module is not available, try spawning an external - # 'zip' command. - if zipfile is None: - if verbose: - zipoptions = "-r" - else: - zipoptions = "-rq" - - try: - spawn(["zip", zipoptions, zip_filename, base_dir], - dry_run=dry_run) - except DistutilsExecError: - # XXX really should distinguish between "couldn't find - # external 'zip' command" and "zip failed". - raise DistutilsExecError(("unable to create zip file '%s': " - "could neither import the 'zipfile' module nor " - "find a standalone zip utility") % zip_filename) - - else: - log.info("creating '%s' and adding '%s' to it", - zip_filename, base_dir) - - if not dry_run: - try: - zip = zipfile.ZipFile(zip_filename, "w", - compression=zipfile.ZIP_DEFLATED) - except RuntimeError: - zip = zipfile.ZipFile(zip_filename, "w", - compression=zipfile.ZIP_STORED) - - if base_dir != os.curdir: - path = os.path.normpath(os.path.join(base_dir, '')) - zip.write(path, path) - log.info("adding '%s'", path) - for dirpath, dirnames, filenames in os.walk(base_dir): - for name in dirnames: - path = os.path.normpath(os.path.join(dirpath, name, '')) - zip.write(path, path) - log.info("adding '%s'", path) - for name in filenames: - path = os.path.normpath(os.path.join(dirpath, name)) - if os.path.isfile(path): - zip.write(path, path) - log.info("adding '%s'", path) - zip.close() - - return zip_filename - -ARCHIVE_FORMATS = { - 'gztar': (make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"), - 'bztar': (make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"), - 'xztar': (make_tarball, [('compress', 'xz')], "xz'ed tar-file"), - 'ztar': (make_tarball, [('compress', 'compress')], "compressed tar file"), - 'tar': (make_tarball, [('compress', None)], "uncompressed tar file"), - 'zip': (make_zipfile, [],"ZIP file") - } - -def check_archive_formats(formats): - """Returns the first format from the 'format' list that is unknown. - - If all formats are known, returns None - """ - for format in formats: - if format not in ARCHIVE_FORMATS: - return format - return None - -def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, - dry_run=0, owner=None, group=None): - """Create an archive file (eg. zip or tar). - - 'base_name' is the name of the file to create, minus any format-specific - extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", "xztar", or "ztar". - - 'root_dir' is a directory that will be the root directory of the - archive; ie. we typically chdir into 'root_dir' before creating the - archive. 'base_dir' is the directory where we start archiving from; - ie. 'base_dir' will be the common prefix of all files and - directories in the archive. 'root_dir' and 'base_dir' both default - to the current directory. Returns the name of the archive file. - - 'owner' and 'group' are used when creating a tar archive. By default, - uses the current owner and group. - """ - save_cwd = os.getcwd() - if root_dir is not None: - log.debug("changing into '%s'", root_dir) - base_name = os.path.abspath(base_name) - if not dry_run: - os.chdir(root_dir) - - if base_dir is None: - base_dir = os.curdir - - kwargs = {'dry_run': dry_run} - - try: - format_info = ARCHIVE_FORMATS[format] - except KeyError: - raise ValueError("unknown archive format '%s'" % format) - - func = format_info[0] - for arg, val in format_info[1]: - kwargs[arg] = val - - if format != 'zip': - kwargs['owner'] = owner - kwargs['group'] = group - - try: - filename = func(base_name, base_dir, **kwargs) - finally: - if root_dir is not None: - log.debug("changing back to '%s'", save_cwd) - os.chdir(save_cwd) - - return filename diff --git a/Lib/distutils/bcppcompiler.py b/Lib/distutils/bcppcompiler.py deleted file mode 100644 index 9f4c432d90e..00000000000 --- a/Lib/distutils/bcppcompiler.py +++ /dev/null @@ -1,393 +0,0 @@ -"""distutils.bcppcompiler - -Contains BorlandCCompiler, an implementation of the abstract CCompiler class -for the Borland C++ compiler. -""" - -# This implementation by Lyle Johnson, based on the original msvccompiler.py -# module and using the directions originally published by Gordon Williams. - -# XXX looks like there's a LOT of overlap between these two classes: -# someone should sit down and factor out the common code as -# WindowsCCompiler! --GPW - - -import os -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError, UnknownFileError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.file_util import write_file -from distutils.dep_util import newer -from distutils import log - -class BCPPCompiler(CCompiler) : - """Concrete class that implements an interface to the Borland C/C++ - compiler, as defined by the CCompiler abstract class. - """ - - compiler_type = 'bcpp' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = _c_extensions + _cpp_extensions - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - - def __init__ (self, - verbose=0, - dry_run=0, - force=0): - - CCompiler.__init__ (self, verbose, dry_run, force) - - # These executables are assumed to all be in the path. - # Borland doesn't seem to use any special registry settings to - # indicate their installation locations. - - self.cc = "bcc32.exe" - self.linker = "ilink32.exe" - self.lib = "tlib.exe" - - self.preprocess_options = None - self.compile_options = ['/tWM', '/O2', '/q', '/g0'] - self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] - - self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] - self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] - self.ldflags_static = [] - self.ldflags_exe = ['/Gn', '/q', '/x'] - self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] - - - # -- Worker methods ------------------------------------------------ - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - compile_opts = extra_preargs or [] - compile_opts.append ('-c') - if debug: - compile_opts.extend (self.compile_options_debug) - else: - compile_opts.extend (self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - # XXX why do the normpath here? - src = os.path.normpath(src) - obj = os.path.normpath(obj) - # XXX _setup_compile() did a mkpath() too but before the normpath. - # Is it possible to skip the normpath? - self.mkpath(os.path.dirname(obj)) - - if ext == '.res': - # This is already a binary file -- skip it. - continue # the 'for' loop - if ext == '.rc': - # This needs to be compiled to a .res file -- do it now. - try: - self.spawn (["brcc32", "-fo", obj, src]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue # the 'for' loop - - # The next two are both for the real compiler. - if ext in self._c_extensions: - input_opt = "" - elif ext in self._cpp_extensions: - input_opt = "-P" - else: - # Unknown file type -- no extra options. The compiler - # will probably fail, but let it just in case this is a - # file the compiler recognizes even if we don't. - input_opt = "" - - output_opt = "-o" + obj - - # Compiler command line syntax is: "bcc32 [options] file(s)". - # Note that the source file names must appear at the end of - # the command line. - try: - self.spawn ([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs + [src]) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - # compile () - - - def create_static_lib (self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - (objects, output_dir) = self._fix_object_args (objects, output_dir) - output_filename = \ - self.library_filename (output_libname, output_dir=output_dir) - - if self._need_link (objects, output_filename): - lib_args = [output_filename, '/u'] + objects - if debug: - pass # XXX what goes here? - try: - self.spawn ([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # create_static_lib () - - - def link (self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - # XXX this ignores 'build_temp'! should follow the lead of - # msvccompiler.py - - (objects, output_dir) = self._fix_object_args (objects, output_dir) - (libraries, library_dirs, runtime_library_dirs) = \ - self._fix_lib_args (libraries, library_dirs, runtime_library_dirs) - - if runtime_library_dirs: - log.warn("I don't know what to do with 'runtime_library_dirs': %s", - str(runtime_library_dirs)) - - if output_dir is not None: - output_filename = os.path.join (output_dir, output_filename) - - if self._need_link (objects, output_filename): - - # Figure out linker args based on type of target. - if target_desc == CCompiler.EXECUTABLE: - startup_obj = 'c0w32' - if debug: - ld_args = self.ldflags_exe_debug[:] - else: - ld_args = self.ldflags_exe[:] - else: - startup_obj = 'c0d32' - if debug: - ld_args = self.ldflags_shared_debug[:] - else: - ld_args = self.ldflags_shared[:] - - - # Create a temporary exports file for use by the linker - if export_symbols is None: - def_file = '' - else: - head, tail = os.path.split (output_filename) - modname, ext = os.path.splitext (tail) - temp_dir = os.path.dirname(objects[0]) # preserve tree structure - def_file = os.path.join (temp_dir, '%s.def' % modname) - contents = ['EXPORTS'] - for sym in (export_symbols or []): - contents.append(' %s=_%s' % (sym, sym)) - self.execute(write_file, (def_file, contents), - "writing %s" % def_file) - - # Borland C++ has problems with '/' in paths - objects2 = map(os.path.normpath, objects) - # split objects in .obj and .res files - # Borland C++ needs them at different positions in the command line - objects = [startup_obj] - resources = [] - for file in objects2: - (base, ext) = os.path.splitext(os.path.normcase(file)) - if ext == '.res': - resources.append(file) - else: - objects.append(file) - - - for l in library_dirs: - ld_args.append("/L%s" % os.path.normpath(l)) - ld_args.append("/L.") # we sometimes use relative paths - - # list of object files - ld_args.extend(objects) - - # XXX the command-line syntax for Borland C++ is a bit wonky; - # certain filenames are jammed together in one big string, but - # comma-delimited. This doesn't mesh too well with the - # Unix-centric attitude (with a DOS/Windows quoting hack) of - # 'spawn()', so constructing the argument list is a bit - # awkward. Note that doing the obvious thing and jamming all - # the filenames and commas into one argument would be wrong, - # because 'spawn()' would quote any filenames with spaces in - # them. Arghghh!. Apparently it works fine as coded... - - # name of dll/exe file - ld_args.extend([',',output_filename]) - # no map file and start libraries - ld_args.append(',,') - - for lib in libraries: - # see if we find it and if there is a bcpp specific lib - # (xxx_bcpp.lib) - libfile = self.find_library_file(library_dirs, lib, debug) - if libfile is None: - ld_args.append(lib) - # probably a BCPP internal library -- don't warn - else: - # full name which prefers bcpp_xxx.lib over xxx.lib - ld_args.append(libfile) - - # some default libraries - ld_args.append ('import32') - ld_args.append ('cw32mt') - - # def file for export symbols - ld_args.extend([',',def_file]) - # add resource files - ld_args.append(',') - ld_args.extend(resources) - - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath (os.path.dirname (output_filename)) - try: - self.spawn ([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # link () - - # -- Miscellaneous methods ----------------------------------------- - - - def find_library_file (self, dirs, lib, debug=0): - # List of effective library names to try, in order of preference: - # xxx_bcpp.lib is better than xxx.lib - # and xxx_d.lib is better than xxx.lib if debug is set - # - # The "_bcpp" suffix is to handle a Python installation for people - # with multiple compilers (primarily Distutils hackers, I suspect - # ;-). The idea is they'd have one static library for each - # compiler they care about, since (almost?) every Windows compiler - # seems to have a different format for static libraries. - if debug: - dlib = (lib + "_d") - try_names = (dlib + "_bcpp", lib + "_bcpp", dlib, lib) - else: - try_names = (lib + "_bcpp", lib) - - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename(name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # overwrite the one from CCompiler to support rc and res-files - def object_filenames (self, - source_filenames, - strip_dir=0, - output_dir=''): - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - # use normcase to make sure '.rc' is really '.rc' and not '.RC' - (base, ext) = os.path.splitext (os.path.normcase(src_name)) - if ext not in (self.src_extensions + ['.rc','.res']): - raise UnknownFileError("unknown file type '%s' (from '%s')" % \ - (ext, src_name)) - if strip_dir: - base = os.path.basename (base) - if ext == '.res': - # these can go unchanged - obj_names.append (os.path.join (output_dir, base + ext)) - elif ext == '.rc': - # these need to be compiled to .res-files - obj_names.append (os.path.join (output_dir, base + '.res')) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - # object_filenames () - - def preprocess (self, - source, - output_file=None, - macros=None, - include_dirs=None, - extra_preargs=None, - extra_postargs=None): - - (_, macros, include_dirs) = \ - self._fix_compile_args(None, macros, include_dirs) - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = ['cpp32.exe'] + pp_opts - if output_file is not None: - pp_args.append('-o' + output_file) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or the - # source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - print(msg) - raise CompileError(msg) - - # preprocess() diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py deleted file mode 100644 index b71d1d39bcd..00000000000 --- a/Lib/distutils/ccompiler.py +++ /dev/null @@ -1,1115 +0,0 @@ -"""distutils.ccompiler - -Contains CCompiler, an abstract base class that defines the interface -for the Distutils compiler abstraction model.""" - -import sys, os, re -from distutils.errors import * -from distutils.spawn import spawn -from distutils.file_util import move_file -from distutils.dir_util import mkpath -from distutils.dep_util import newer_pairwise, newer_group -from distutils.util import split_quoted, execute -from distutils import log - -class CCompiler: - """Abstract base class to define the interface that must be implemented - by real compiler classes. Also has some utility methods used by - several compiler classes. - - The basic idea behind a compiler abstraction class is that each - instance can be used for all the compile/link steps in building a - single project. Thus, attributes common to all of those compile and - link steps -- include directories, macros to define, libraries to link - against, etc. -- are attributes of the compiler instance. To allow for - variability in how individual files are treated, most of those - attributes may be varied on a per-compilation or per-link basis. - """ - - # 'compiler_type' is a class attribute that identifies this class. It - # keeps code that wants to know what kind of compiler it's dealing with - # from having to import all possible compiler classes just to do an - # 'isinstance'. In concrete CCompiler subclasses, 'compiler_type' - # should really, really be one of the keys of the 'compiler_class' - # dictionary (see below -- used by the 'new_compiler()' factory - # function) -- authors of new compiler interface classes are - # responsible for updating 'compiler_class'! - compiler_type = None - - # XXX things not handled by this compiler abstraction model: - # * client can't provide additional options for a compiler, - # e.g. warning, optimization, debugging flags. Perhaps this - # should be the domain of concrete compiler abstraction classes - # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base - # class should have methods for the common ones. - # * can't completely override the include or library searchg - # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". - # I'm not sure how widely supported this is even by Unix - # compilers, much less on other platforms. And I'm even less - # sure how useful it is; maybe for cross-compiling, but - # support for that is a ways off. (And anyways, cross - # compilers probably have a dedicated binary with the - # right paths compiled in. I hope.) - # * can't do really freaky things with the library list/library - # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against - # different versions of libfoo.a in different locations. I - # think this is useless without the ability to null out the - # library search path anyways. - - - # Subclasses that rely on the standard filename generation methods - # implemented below should override these; see the comment near - # those methods ('object_filenames()' et. al.) for details: - src_extensions = None # list of strings - obj_extension = None # string - static_lib_extension = None - shared_lib_extension = None # string - static_lib_format = None # format string - shared_lib_format = None # prob. same as static_lib_format - exe_extension = None # string - - # Default language settings. language_map is used to detect a source - # file or Extension target language, checking source filenames. - # language_order is used to detect the language precedence, when deciding - # what language to use when mixing source types. For example, if some - # extension has two files with ".c" extension, and one with ".cpp", it - # is still linked as c++. - language_map = {".c" : "c", - ".cc" : "c++", - ".cpp" : "c++", - ".cxx" : "c++", - ".m" : "objc", - } - language_order = ["c++", "objc", "c"] - - def __init__(self, verbose=0, dry_run=0, force=0): - self.dry_run = dry_run - self.force = force - self.verbose = verbose - - # 'output_dir': a common output directory for object, library, - # shared object, and shared library files - self.output_dir = None - - # 'macros': a list of macro definitions (or undefinitions). A - # macro definition is a 2-tuple (name, value), where the value is - # either a string or None (no explicit value). A macro - # undefinition is a 1-tuple (name,). - self.macros = [] - - # 'include_dirs': a list of directories to search for include files - self.include_dirs = [] - - # 'libraries': a list of libraries to include in any link - # (library names, not filenames: eg. "foo" not "libfoo.a") - self.libraries = [] - - # 'library_dirs': a list of directories to search for libraries - self.library_dirs = [] - - # 'runtime_library_dirs': a list of directories to search for - # shared libraries/objects at runtime - self.runtime_library_dirs = [] - - # 'objects': a list of object files (or similar, such as explicitly - # named library files) to include on any link - self.objects = [] - - for key in self.executables.keys(): - self.set_executable(key, self.executables[key]) - - def set_executables(self, **kwargs): - """Define the executables (and options for them) that will be run - to perform the various stages of compilation. The exact set of - executables that may be specified here depends on the compiler - class (via the 'executables' class attribute), but most will have: - compiler the C/C++ compiler - linker_so linker used to create shared objects and libraries - linker_exe linker used to create binary executables - archiver static library creator - - On platforms with a command-line (Unix, DOS/Windows), each of these - is a string that will be split into executable name and (optional) - list of arguments. (Splitting the string is done similarly to how - Unix shells operate: words are delimited by spaces, but quotes and - backslashes can override this. See - 'distutils.util.split_quoted()'.) - """ - - # Note that some CCompiler implementation classes will define class - # attributes 'cpp', 'cc', etc. with hard-coded executable names; - # this is appropriate when a compiler class is for exactly one - # compiler/OS combination (eg. MSVCCompiler). Other compiler - # classes (UnixCCompiler, in particular) are driven by information - # discovered at run-time, since there are many different ways to do - # basically the same things with Unix C compilers. - - for key in kwargs: - if key not in self.executables: - raise ValueError("unknown executable '%s' for class %s" % - (key, self.__class__.__name__)) - self.set_executable(key, kwargs[key]) - - def set_executable(self, key, value): - if isinstance(value, str): - setattr(self, key, split_quoted(value)) - else: - setattr(self, key, value) - - def _find_macro(self, name): - i = 0 - for defn in self.macros: - if defn[0] == name: - return i - i += 1 - return None - - def _check_macro_definitions(self, definitions): - """Ensures that every element of 'definitions' is a valid macro - definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do - nothing if all definitions are OK, raise TypeError otherwise. - """ - for defn in definitions: - if not (isinstance(defn, tuple) and - (len(defn) in (1, 2) and - (isinstance (defn[1], str) or defn[1] is None)) and - isinstance (defn[0], str)): - raise TypeError(("invalid macro definition '%s': " % defn) + \ - "must be tuple (string,), (string, string), or " + \ - "(string, None)") - - - # -- Bookkeeping methods ------------------------------------------- - - def define_macro(self, name, value=None): - """Define a preprocessor macro for all compilations driven by this - compiler object. The optional parameter 'value' should be a - string; if it is not supplied, then the macro will be defined - without an explicit value and the exact outcome depends on the - compiler used (XXX true? does ANSI say anything about this?) - """ - # Delete from the list of macro definitions/undefinitions if - # already there (so that this one will take precedence). - i = self._find_macro (name) - if i is not None: - del self.macros[i] - - self.macros.append((name, value)) - - def undefine_macro(self, name): - """Undefine a preprocessor macro for all compilations driven by - this compiler object. If the same macro is defined by - 'define_macro()' and undefined by 'undefine_macro()' the last call - takes precedence (including multiple redefinitions or - undefinitions). If the macro is redefined/undefined on a - per-compilation basis (ie. in the call to 'compile()'), then that - takes precedence. - """ - # Delete from the list of macro definitions/undefinitions if - # already there (so that this one will take precedence). - i = self._find_macro (name) - if i is not None: - del self.macros[i] - - undefn = (name,) - self.macros.append(undefn) - - def add_include_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - header files. The compiler is instructed to search directories in - the order in which they are supplied by successive calls to - 'add_include_dir()'. - """ - self.include_dirs.append(dir) - - def set_include_dirs(self, dirs): - """Set the list of directories that will be searched to 'dirs' (a - list of strings). Overrides any preceding calls to - 'add_include_dir()'; subsequence calls to 'add_include_dir()' add - to the list passed to 'set_include_dirs()'. This does not affect - any list of standard include directories that the compiler may - search by default. - """ - self.include_dirs = dirs[:] - - def add_library(self, libname): - """Add 'libname' to the list of libraries that will be included in - all links driven by this compiler object. Note that 'libname' - should *not* be the name of a file containing a library, but the - name of the library itself: the actual filename will be inferred by - the linker, the compiler, or the compiler class (depending on the - platform). - - The linker will be instructed to link against libraries in the - order they were supplied to 'add_library()' and/or - 'set_libraries()'. It is perfectly valid to duplicate library - names; the linker will be instructed to link against libraries as - many times as they are mentioned. - """ - self.libraries.append(libname) - - def set_libraries(self, libnames): - """Set the list of libraries to be included in all links driven by - this compiler object to 'libnames' (a list of strings). This does - not affect any standard system libraries that the linker may - include by default. - """ - self.libraries = libnames[:] - - def add_library_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - libraries specified to 'add_library()' and 'set_libraries()'. The - linker will be instructed to search for libraries in the order they - are supplied to 'add_library_dir()' and/or 'set_library_dirs()'. - """ - self.library_dirs.append(dir) - - def set_library_dirs(self, dirs): - """Set the list of library search directories to 'dirs' (a list of - strings). This does not affect any standard library search path - that the linker may search by default. - """ - self.library_dirs = dirs[:] - - def add_runtime_library_dir(self, dir): - """Add 'dir' to the list of directories that will be searched for - shared libraries at runtime. - """ - self.runtime_library_dirs.append(dir) - - def set_runtime_library_dirs(self, dirs): - """Set the list of directories to search for shared libraries at - runtime to 'dirs' (a list of strings). This does not affect any - standard search path that the runtime linker may search by - default. - """ - self.runtime_library_dirs = dirs[:] - - def add_link_object(self, object): - """Add 'object' to the list of object files (or analogues, such as - explicitly named library files or the output of "resource - compilers") to be included in every link driven by this compiler - object. - """ - self.objects.append(object) - - def set_link_objects(self, objects): - """Set the list of object files (or analogues) to be included in - every link to 'objects'. This does not affect any standard object - files that the linker may include by default (such as system - libraries). - """ - self.objects = objects[:] - - - # -- Private utility methods -------------------------------------- - # (here for the convenience of subclasses) - - # Helper method to prep compiler in subclass compile() methods - - def _setup_compile(self, outdir, macros, incdirs, sources, depends, - extra): - """Process arguments and decide which source files to compile.""" - if outdir is None: - outdir = self.output_dir - elif not isinstance(outdir, str): - raise TypeError("'output_dir' must be a string or None") - - if macros is None: - macros = self.macros - elif isinstance(macros, list): - macros = macros + (self.macros or []) - else: - raise TypeError("'macros' (if supplied) must be a list of tuples") - - if incdirs is None: - incdirs = self.include_dirs - elif isinstance(incdirs, (list, tuple)): - incdirs = list(incdirs) + (self.include_dirs or []) - else: - raise TypeError( - "'include_dirs' (if supplied) must be a list of strings") - - if extra is None: - extra = [] - - # Get the list of expected output (object) files - objects = self.object_filenames(sources, strip_dir=0, - output_dir=outdir) - assert len(objects) == len(sources) - - pp_opts = gen_preprocess_options(macros, incdirs) - - build = {} - for i in range(len(sources)): - src = sources[i] - obj = objects[i] - ext = os.path.splitext(src)[1] - self.mkpath(os.path.dirname(obj)) - build[obj] = (src, ext) - - return macros, objects, extra, pp_opts, build - - def _get_cc_args(self, pp_opts, debug, before): - # works for unixccompiler, cygwinccompiler - cc_args = pp_opts + ['-c'] - if debug: - cc_args[:0] = ['-g'] - if before: - cc_args[:0] = before - return cc_args - - def _fix_compile_args(self, output_dir, macros, include_dirs): - """Typecheck and fix-up some of the arguments to the 'compile()' - method, and return fixed-up values. Specifically: if 'output_dir' - is None, replaces it with 'self.output_dir'; ensures that 'macros' - is a list, and augments it with 'self.macros'; ensures that - 'include_dirs' is a list, and augments it with 'self.include_dirs'. - Guarantees that the returned values are of the correct type, - i.e. for 'output_dir' either string or None, and for 'macros' and - 'include_dirs' either list or None. - """ - if output_dir is None: - output_dir = self.output_dir - elif not isinstance(output_dir, str): - raise TypeError("'output_dir' must be a string or None") - - if macros is None: - macros = self.macros - elif isinstance(macros, list): - macros = macros + (self.macros or []) - else: - raise TypeError("'macros' (if supplied) must be a list of tuples") - - if include_dirs is None: - include_dirs = self.include_dirs - elif isinstance(include_dirs, (list, tuple)): - include_dirs = list(include_dirs) + (self.include_dirs or []) - else: - raise TypeError( - "'include_dirs' (if supplied) must be a list of strings") - - return output_dir, macros, include_dirs - - def _prep_compile(self, sources, output_dir, depends=None): - """Decide which souce files must be recompiled. - - Determine the list of object files corresponding to 'sources', - and figure out which ones really need to be recompiled. - Return a list of all object files and a dictionary telling - which source files can be skipped. - """ - # Get the list of expected output (object) files - objects = self.object_filenames(sources, output_dir=output_dir) - assert len(objects) == len(sources) - - # Return an empty dict for the "which source files can be skipped" - # return value to preserve API compatibility. - return objects, {} - - def _fix_object_args(self, objects, output_dir): - """Typecheck and fix up some arguments supplied to various methods. - Specifically: ensure that 'objects' is a list; if output_dir is - None, replace with self.output_dir. Return fixed versions of - 'objects' and 'output_dir'. - """ - if not isinstance(objects, (list, tuple)): - raise TypeError("'objects' must be a list or tuple of strings") - objects = list(objects) - - if output_dir is None: - output_dir = self.output_dir - elif not isinstance(output_dir, str): - raise TypeError("'output_dir' must be a string or None") - - return (objects, output_dir) - - def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): - """Typecheck and fix up some of the arguments supplied to the - 'link_*' methods. Specifically: ensure that all arguments are - lists, and augment them with their permanent versions - (eg. 'self.libraries' augments 'libraries'). Return a tuple with - fixed versions of all arguments. - """ - if libraries is None: - libraries = self.libraries - elif isinstance(libraries, (list, tuple)): - libraries = list (libraries) + (self.libraries or []) - else: - raise TypeError( - "'libraries' (if supplied) must be a list of strings") - - if library_dirs is None: - library_dirs = self.library_dirs - elif isinstance(library_dirs, (list, tuple)): - library_dirs = list (library_dirs) + (self.library_dirs or []) - else: - raise TypeError( - "'library_dirs' (if supplied) must be a list of strings") - - if runtime_library_dirs is None: - runtime_library_dirs = self.runtime_library_dirs - elif isinstance(runtime_library_dirs, (list, tuple)): - runtime_library_dirs = (list(runtime_library_dirs) + - (self.runtime_library_dirs or [])) - else: - raise TypeError("'runtime_library_dirs' (if supplied) " - "must be a list of strings") - - return (libraries, library_dirs, runtime_library_dirs) - - def _need_link(self, objects, output_file): - """Return true if we need to relink the files listed in 'objects' - to recreate 'output_file'. - """ - if self.force: - return True - else: - if self.dry_run: - newer = newer_group (objects, output_file, missing='newer') - else: - newer = newer_group (objects, output_file) - return newer - - def detect_language(self, sources): - """Detect the language of a given file, or list of files. Uses - language_map, and language_order to do the job. - """ - if not isinstance(sources, list): - sources = [sources] - lang = None - index = len(self.language_order) - for source in sources: - base, ext = os.path.splitext(source) - extlang = self.language_map.get(ext) - try: - extindex = self.language_order.index(extlang) - if extindex < index: - lang = extlang - index = extindex - except ValueError: - pass - return lang - - - # -- Worker methods ------------------------------------------------ - # (must be implemented by subclasses) - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - """Preprocess a single C/C++ source file, named in 'source'. - Output will be written to file named 'output_file', or stdout if - 'output_file' not supplied. 'macros' is a list of macro - definitions as for 'compile()', which will augment the macros set - with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a - list of directory names that will be added to the default list. - - Raises PreprocessError on failure. - """ - pass - - def compile(self, sources, output_dir=None, macros=None, - include_dirs=None, debug=0, extra_preargs=None, - extra_postargs=None, depends=None): - """Compile one or more source files. - - 'sources' must be a list of filenames, most likely C/C++ - files, but in reality anything that can be handled by a - particular compiler and compiler class (eg. MSVCCompiler can - handle resource files in 'sources'). Return a list of object - filenames, one per source filename in 'sources'. Depending on - the implementation, not all source files will necessarily be - compiled, but all corresponding object filenames will be - returned. - - If 'output_dir' is given, object files will be put under it, while - retaining their original path component. That is, "foo/bar.c" - normally compiles to "foo/bar.o" (for a Unix implementation); if - 'output_dir' is "build", then it would compile to - "build/foo/bar.o". - - 'macros', if given, must be a list of macro definitions. A macro - definition is either a (name, value) 2-tuple or a (name,) 1-tuple. - The former defines a macro; if the value is None, the macro is - defined without an explicit value. The 1-tuple case undefines a - macro. Later definitions/redefinitions/ undefinitions take - precedence. - - 'include_dirs', if given, must be a list of strings, the - directories to add to the default include file search path for this - compilation only. - - 'debug' is a boolean; if true, the compiler will be instructed to - output debug symbols in (or alongside) the object file(s). - - 'extra_preargs' and 'extra_postargs' are implementation- dependent. - On platforms that have the notion of a command-line (e.g. Unix, - DOS/Windows), they are most likely lists of strings: extra - command-line arguments to prepand/append to the compiler command - line. On other platforms, consult the implementation class - documentation. In any event, they are intended as an escape hatch - for those occasions when the abstract compiler framework doesn't - cut the mustard. - - 'depends', if given, is a list of filenames that all targets - depend on. If a source file is older than any file in - depends, then the source file will be recompiled. This - supports dependency tracking, but only at a coarse - granularity. - - Raises CompileError on failure. - """ - # A concrete compiler class can either override this method - # entirely or implement _compile(). - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) - - # Return *all* object filenames, not just the ones we just built. - return objects - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - """Compile 'src' to product 'obj'.""" - # A concrete compiler class that does not override compile() - # should implement _compile(). - pass - - def create_static_lib(self, objects, output_libname, output_dir=None, - debug=0, target_lang=None): - """Link a bunch of stuff together to create a static library file. - The "bunch of stuff" consists of the list of object files supplied - as 'objects', the extra object files supplied to - 'add_link_object()' and/or 'set_link_objects()', the libraries - supplied to 'add_library()' and/or 'set_libraries()', and the - libraries supplied as 'libraries' (if any). - - 'output_libname' should be a library name, not a filename; the - filename will be inferred from the library name. 'output_dir' is - the directory where the library file will be put. - - 'debug' is a boolean; if true, debugging information will be - included in the library (note that on most platforms, it is the - compile step where this matters: the 'debug' flag is included here - just for consistency). - - 'target_lang' is the target language for which the given objects - are being compiled. This allows specific linkage time treatment of - certain languages. - - Raises LibError on failure. - """ - pass - - - # values for target_desc parameter in link() - SHARED_OBJECT = "shared_object" - SHARED_LIBRARY = "shared_library" - EXECUTABLE = "executable" - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - """Link a bunch of stuff together to create an executable or - shared library file. - - The "bunch of stuff" consists of the list of object files supplied - as 'objects'. 'output_filename' should be a filename. If - 'output_dir' is supplied, 'output_filename' is relative to it - (i.e. 'output_filename' can provide directory components if - needed). - - 'libraries' is a list of libraries to link against. These are - library names, not filenames, since they're translated into - filenames in a platform-specific way (eg. "foo" becomes "libfoo.a" - on Unix and "foo.lib" on DOS/Windows). However, they can include a - directory component, which means the linker will look in that - specific directory rather than searching all the normal locations. - - 'library_dirs', if supplied, should be a list of directories to - search for libraries that were specified as bare library names - (ie. no directory component). These are on top of the system - default and those supplied to 'add_library_dir()' and/or - 'set_library_dirs()'. 'runtime_library_dirs' is a list of - directories that will be embedded into the shared library and used - to search for other shared libraries that *it* depends on at - run-time. (This may only be relevant on Unix.) - - 'export_symbols' is a list of symbols that the shared library will - export. (This appears to be relevant only on Windows.) - - 'debug' is as for 'compile()' and 'create_static_lib()', with the - slight distinction that it actually matters on most platforms (as - opposed to 'create_static_lib()', which includes a 'debug' flag - mostly for form's sake). - - 'extra_preargs' and 'extra_postargs' are as for 'compile()' (except - of course that they supply command-line arguments for the - particular linker being used). - - 'target_lang' is the target language for which the given objects - are being compiled. This allows specific linkage time treatment of - certain languages. - - Raises LinkError on failure. - """ - raise NotImplementedError - - - # Old 'link_*()' methods, rewritten to use the new 'link()' method. - - def link_shared_lib(self, - objects, - output_libname, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - self.link(CCompiler.SHARED_LIBRARY, objects, - self.library_filename(output_libname, lib_type='shared'), - output_dir, - libraries, library_dirs, runtime_library_dirs, - export_symbols, debug, - extra_preargs, extra_postargs, build_temp, target_lang) - - - def link_shared_object(self, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - self.link(CCompiler.SHARED_OBJECT, objects, - output_filename, output_dir, - libraries, library_dirs, runtime_library_dirs, - export_symbols, debug, - extra_preargs, extra_postargs, build_temp, target_lang) - - - def link_executable(self, - objects, - output_progname, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - target_lang=None): - self.link(CCompiler.EXECUTABLE, objects, - self.executable_filename(output_progname), output_dir, - libraries, library_dirs, runtime_library_dirs, None, - debug, extra_preargs, extra_postargs, None, target_lang) - - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function; there is - # no appropriate default implementation so subclasses should - # implement all of these. - - def library_dir_option(self, dir): - """Return the compiler option to add 'dir' to the list of - directories searched for libraries. - """ - raise NotImplementedError - - def runtime_library_dir_option(self, dir): - """Return the compiler option to add 'dir' to the list of - directories searched for runtime libraries. - """ - raise NotImplementedError - - def library_option(self, lib): - """Return the compiler option to add 'lib' to the list of libraries - linked into the shared library or executable. - """ - raise NotImplementedError - - def has_function(self, funcname, includes=None, include_dirs=None, - libraries=None, library_dirs=None): - """Return a boolean indicating whether funcname is supported on - the current platform. The optional arguments can be used to - augment the compilation environment. - """ - # this can't be included at module scope because it tries to - # import math which might not be available at that point - maybe - # the necessary logic should just be inlined? - import tempfile - if includes is None: - includes = [] - if include_dirs is None: - include_dirs = [] - if libraries is None: - libraries = [] - if library_dirs is None: - library_dirs = [] - fd, fname = tempfile.mkstemp(".c", funcname, text=True) - f = os.fdopen(fd, "w") - try: - for incl in includes: - f.write("""#include "%s"\n""" % incl) - f.write("""\ -main (int argc, char **argv) { - %s(); -} -""" % funcname) - finally: - f.close() - try: - objects = self.compile([fname], include_dirs=include_dirs) - except CompileError: - return False - - try: - self.link_executable(objects, "a.out", - libraries=libraries, - library_dirs=library_dirs) - except (LinkError, TypeError): - return False - return True - - def find_library_file (self, dirs, lib, debug=0): - """Search the specified list of directories for a static or shared - library file 'lib' and return the full path to that file. If - 'debug' true, look for a debugging version (if that makes sense on - the current platform). Return None if 'lib' wasn't found in any of - the specified directories. - """ - raise NotImplementedError - - # -- Filename generation methods ----------------------------------- - - # The default implementation of the filename generating methods are - # prejudiced towards the Unix/DOS/Windows view of the world: - # * object files are named by replacing the source file extension - # (eg. .c/.cpp -> .o/.obj) - # * library files (shared or static) are named by plugging the - # library name and extension into a format string, eg. - # "lib%s.%s" % (lib_name, ".a") for Unix static libraries - # * executables are named by appending an extension (possibly - # empty) to the program name: eg. progname + ".exe" for - # Windows - # - # To reduce redundant code, these methods expect to find - # several attributes in the current object (presumably defined - # as class attributes): - # * src_extensions - - # list of C/C++ source file extensions, eg. ['.c', '.cpp'] - # * obj_extension - - # object file extension, eg. '.o' or '.obj' - # * static_lib_extension - - # extension for static library files, eg. '.a' or '.lib' - # * shared_lib_extension - - # extension for shared library/object files, eg. '.so', '.dll' - # * static_lib_format - - # format string for generating static library filenames, - # eg. 'lib%s.%s' or '%s.%s' - # * shared_lib_format - # format string for generating shared library filenames - # (probably same as static_lib_format, since the extension - # is one of the intended parameters to the format string) - # * exe_extension - - # extension for executable files, eg. '' or '.exe' - - def object_filenames(self, source_filenames, strip_dir=0, output_dir=''): - if output_dir is None: - output_dir = '' - obj_names = [] - for src_name in source_filenames: - base, ext = os.path.splitext(src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - raise UnknownFileError( - "unknown file type '%s' (from '%s')" % (ext, src_name)) - if strip_dir: - base = os.path.basename(base) - obj_names.append(os.path.join(output_dir, - base + self.obj_extension)) - return obj_names - - def shared_object_filename(self, basename, strip_dir=0, output_dir=''): - assert output_dir is not None - if strip_dir: - basename = os.path.basename(basename) - return os.path.join(output_dir, basename + self.shared_lib_extension) - - def executable_filename(self, basename, strip_dir=0, output_dir=''): - assert output_dir is not None - if strip_dir: - basename = os.path.basename(basename) - return os.path.join(output_dir, basename + (self.exe_extension or '')) - - def library_filename(self, libname, lib_type='static', # or 'shared' - strip_dir=0, output_dir=''): - assert output_dir is not None - if lib_type not in ("static", "shared", "dylib", "xcode_stub"): - raise ValueError( - "'lib_type' must be \"static\", \"shared\", \"dylib\", or \"xcode_stub\"") - fmt = getattr(self, lib_type + "_lib_format") - ext = getattr(self, lib_type + "_lib_extension") - - dir, base = os.path.split(libname) - filename = fmt % (base, ext) - if strip_dir: - dir = '' - - return os.path.join(output_dir, dir, filename) - - - # -- Utility methods ----------------------------------------------- - - def announce(self, msg, level=1): - log.debug(msg) - - def debug_print(self, msg): - from distutils.debug import DEBUG - if DEBUG: - print(msg) - - def warn(self, msg): - sys.stderr.write("warning: %s\n" % msg) - - def execute(self, func, args, msg=None, level=1): - execute(func, args, msg, self.dry_run) - - def spawn(self, cmd): - spawn(cmd, dry_run=self.dry_run) - - def move_file(self, src, dst): - return move_file(src, dst, dry_run=self.dry_run) - - def mkpath (self, name, mode=0o777): - mkpath(name, mode, dry_run=self.dry_run) - - -# Map a sys.platform/os.name ('posix', 'nt') to the default compiler -# type for that platform. Keys are interpreted as re match -# patterns. Order is important; platform mappings are preferred over -# OS names. -_default_compilers = ( - - # Platform string mappings - - # on a cygwin built python we can use gcc like an ordinary UNIXish - # compiler - ('cygwin.*', 'unix'), - - # OS name mappings - ('posix', 'unix'), - ('nt', 'msvc'), - - ) - -def get_default_compiler(osname=None, platform=None): - """Determine the default compiler to use for the given platform. - - osname should be one of the standard Python OS names (i.e. the - ones returned by os.name) and platform the common value - returned by sys.platform for the platform in question. - - The default values are os.name and sys.platform in case the - parameters are not given. - """ - if osname is None: - osname = os.name - if platform is None: - platform = sys.platform - for pattern, compiler in _default_compilers: - if re.match(pattern, platform) is not None or \ - re.match(pattern, osname) is not None: - return compiler - # Default to Unix compiler - return 'unix' - -# Map compiler types to (module_name, class_name) pairs -- ie. where to -# find the code that implements an interface to this compiler. (The module -# is assumed to be in the 'distutils' package.) -compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', - "standard UNIX-style compiler"), - 'msvc': ('_msvccompiler', 'MSVCCompiler', - "Microsoft Visual C++"), - 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', - "Cygwin port of GNU C Compiler for Win32"), - 'mingw32': ('cygwinccompiler', 'Mingw32CCompiler', - "Mingw32 port of GNU C Compiler for Win32"), - 'bcpp': ('bcppcompiler', 'BCPPCompiler', - "Borland C++ Compiler"), - } - -def show_compilers(): - """Print list of available compilers (used by the "--help-compiler" - options to "build", "build_ext", "build_clib"). - """ - # XXX this "knows" that the compiler option it's describing is - # "--compiler", which just happens to be the case for the three - # commands that use it. - from distutils.fancy_getopt import FancyGetopt - compilers = [] - for compiler in compiler_class.keys(): - compilers.append(("compiler="+compiler, None, - compiler_class[compiler][2])) - compilers.sort() - pretty_printer = FancyGetopt(compilers) - pretty_printer.print_help("List of available compilers:") - - -def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): - """Generate an instance of some CCompiler subclass for the supplied - platform/compiler combination. 'plat' defaults to 'os.name' - (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler - for that platform. Currently only 'posix' and 'nt' are supported, and - the default compilers are "traditional Unix interface" (UnixCCompiler - class) and Visual C++ (MSVCCompiler class). Note that it's perfectly - possible to ask for a Unix compiler object under Windows, and a - Microsoft compiler object under Unix -- if you supply a value for - 'compiler', 'plat' is ignored. - """ - if plat is None: - plat = os.name - - try: - if compiler is None: - compiler = get_default_compiler(plat) - - (module_name, class_name, long_description) = compiler_class[compiler] - except KeyError: - msg = "don't know how to compile C/C++ code on platform '%s'" % plat - if compiler is not None: - msg = msg + " with '%s' compiler" % compiler - raise DistutilsPlatformError(msg) - - try: - module_name = "distutils." + module_name - __import__ (module_name) - module = sys.modules[module_name] - klass = vars(module)[class_name] - except ImportError: - raise DistutilsModuleError( - "can't compile C/C++ code: unable to load module '%s'" % \ - module_name) - except KeyError: - raise DistutilsModuleError( - "can't compile C/C++ code: unable to find class '%s' " - "in module '%s'" % (class_name, module_name)) - - # XXX The None is necessary to preserve backwards compatibility - # with classes that expect verbose to be the first positional - # argument. - return klass(None, dry_run, force) - - -def gen_preprocess_options(macros, include_dirs): - """Generate C pre-processor options (-D, -U, -I) as used by at least - two types of compilers: the typical Unix compiler and Visual C++. - 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) - means undefine (-U) macro 'name', and (name,value) means define (-D) - macro 'name' to 'value'. 'include_dirs' is just a list of directory - names to be added to the header file search path (-I). Returns a list - of command-line options suitable for either Unix compilers or Visual - C++. - """ - # XXX it would be nice (mainly aesthetic, and so we don't generate - # stupid-looking command lines) to go over 'macros' and eliminate - # redundant definitions/undefinitions (ie. ensure that only the - # latest mention of a particular macro winds up on the command - # line). I don't think it's essential, though, since most (all?) - # Unix C compilers only pay attention to the latest -D or -U - # mention of a macro on their command line. Similar situation for - # 'include_dirs'. I'm punting on both for now. Anyways, weeding out - # redundancies like this should probably be the province of - # CCompiler, since the data structures used are inherited from it - # and therefore common to all CCompiler classes. - pp_opts = [] - for macro in macros: - if not (isinstance(macro, tuple) and 1 <= len(macro) <= 2): - raise TypeError( - "bad macro definition '%s': " - "each element of 'macros' list must be a 1- or 2-tuple" - % macro) - - if len(macro) == 1: # undefine this macro - pp_opts.append("-U%s" % macro[0]) - elif len(macro) == 2: - if macro[1] is None: # define with no explicit value - pp_opts.append("-D%s" % macro[0]) - else: - # XXX *don't* need to be clever about quoting the - # macro value here, because we're going to avoid the - # shell at all costs when we spawn the command! - pp_opts.append("-D%s=%s" % macro) - - for dir in include_dirs: - pp_opts.append("-I%s" % dir) - return pp_opts - - -def gen_lib_options (compiler, library_dirs, runtime_library_dirs, libraries): - """Generate linker options for searching library directories and - linking with specific libraries. 'libraries' and 'library_dirs' are, - respectively, lists of library names (not filenames!) and search - directories. Returns a list of command-line options suitable for use - with some compiler (depending on the two format strings passed in). - """ - lib_opts = [] - - for dir in library_dirs: - lib_opts.append(compiler.library_dir_option(dir)) - - for dir in runtime_library_dirs: - opt = compiler.runtime_library_dir_option(dir) - if isinstance(opt, list): - lib_opts = lib_opts + opt - else: - lib_opts.append(opt) - - # XXX it's important that we *not* remove redundant library mentions! - # sometimes you really do have to say "-lfoo -lbar -lfoo" in order to - # resolve all symbols. I just hope we never have to say "-lfoo obj.o - # -lbar" to get things to work -- that's certainly a possibility, but a - # pretty nasty way to arrange your C code. - - for lib in libraries: - (lib_dir, lib_name) = os.path.split(lib) - if lib_dir: - lib_file = compiler.find_library_file([lib_dir], lib_name) - if lib_file: - lib_opts.append(lib_file) - else: - compiler.warn("no library file corresponding to " - "'%s' found (skipping)" % lib) - else: - lib_opts.append(compiler.library_option (lib)) - return lib_opts diff --git a/Lib/distutils/cmd.py b/Lib/distutils/cmd.py deleted file mode 100644 index 939f7959457..00000000000 --- a/Lib/distutils/cmd.py +++ /dev/null @@ -1,434 +0,0 @@ -"""distutils.cmd - -Provides the Command class, the base class for the command classes -in the distutils.command package. -""" - -import sys, os, re -from distutils.errors import DistutilsOptionError -from distutils import util, dir_util, file_util, archive_util, dep_util -from distutils import log - -class Command: - """Abstract base class for defining command classes, the "worker bees" - of the Distutils. A useful analogy for command classes is to think of - them as subroutines with local variables called "options". The options - are "declared" in 'initialize_options()' and "defined" (given their - final values, aka "finalized") in 'finalize_options()', both of which - must be defined by every command class. The distinction between the - two is necessary because option values might come from the outside - world (command line, config file, ...), and any options dependent on - other options must be computed *after* these outside influences have - been processed -- hence 'finalize_options()'. The "body" of the - subroutine, where it does all its work based on the values of its - options, is the 'run()' method, which must also be implemented by every - command class. - """ - - # 'sub_commands' formalizes the notion of a "family" of commands, - # eg. "install" as the parent with sub-commands "install_lib", - # "install_headers", etc. The parent of a family of commands - # defines 'sub_commands' as a class attribute; it's a list of - # (command_name : string, predicate : unbound_method | string | None) - # tuples, where 'predicate' is a method of the parent command that - # determines whether the corresponding command is applicable in the - # current situation. (Eg. we "install_headers" is only applicable if - # we have any C header files to install.) If 'predicate' is None, - # that command is always applicable. - # - # 'sub_commands' is usually defined at the *end* of a class, because - # predicates can be unbound methods, so they must already have been - # defined. The canonical example is the "install" command. - sub_commands = [] - - - # -- Creation/initialization methods ------------------------------- - - def __init__(self, dist): - """Create and initialize a new Command object. Most importantly, - invokes the 'initialize_options()' method, which is the real - initializer and depends on the actual command being - instantiated. - """ - # late import because of mutual dependence between these classes - from distutils.dist import Distribution - - if not isinstance(dist, Distribution): - raise TypeError("dist must be a Distribution instance") - if self.__class__ is Command: - raise RuntimeError("Command is an abstract class") - - self.distribution = dist - self.initialize_options() - - # Per-command versions of the global flags, so that the user can - # customize Distutils' behaviour command-by-command and let some - # commands fall back on the Distribution's behaviour. None means - # "not defined, check self.distribution's copy", while 0 or 1 mean - # false and true (duh). Note that this means figuring out the real - # value of each flag is a touch complicated -- hence "self._dry_run" - # will be handled by __getattr__, below. - # XXX This needs to be fixed. - self._dry_run = None - - # verbose is largely ignored, but needs to be set for - # backwards compatibility (I think)? - self.verbose = dist.verbose - - # Some commands define a 'self.force' option to ignore file - # timestamps, but methods defined *here* assume that - # 'self.force' exists for all commands. So define it here - # just to be safe. - self.force = None - - # The 'help' flag is just used for command-line parsing, so - # none of that complicated bureaucracy is needed. - self.help = 0 - - # 'finalized' records whether or not 'finalize_options()' has been - # called. 'finalize_options()' itself should not pay attention to - # this flag: it is the business of 'ensure_finalized()', which - # always calls 'finalize_options()', to respect/update it. - self.finalized = 0 - - # XXX A more explicit way to customize dry_run would be better. - def __getattr__(self, attr): - if attr == 'dry_run': - myval = getattr(self, "_" + attr) - if myval is None: - return getattr(self.distribution, attr) - else: - return myval - else: - raise AttributeError(attr) - - def ensure_finalized(self): - if not self.finalized: - self.finalize_options() - self.finalized = 1 - - # Subclasses must define: - # initialize_options() - # provide default values for all options; may be customized by - # setup script, by options from config file(s), or by command-line - # options - # finalize_options() - # decide on the final values for all options; this is called - # after all possible intervention from the outside world - # (command-line, option file, etc.) has been processed - # run() - # run the command: do whatever it is we're here to do, - # controlled by the command's various option values - - def initialize_options(self): - """Set default values for all the options that this command - supports. Note that these defaults may be overridden by other - commands, by the setup script, by config files, or by the - command-line. Thus, this is not the place to code dependencies - between options; generally, 'initialize_options()' implementations - are just a bunch of "self.foo = None" assignments. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - def finalize_options(self): - """Set final values for all the options that this command supports. - This is always called as late as possible, ie. after any option - assignments from the command-line or from other commands have been - done. Thus, this is the place to code option dependencies: if - 'foo' depends on 'bar', then it is safe to set 'foo' from 'bar' as - long as 'foo' still has the same value it was assigned in - 'initialize_options()'. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - - def dump_options(self, header=None, indent=""): - from distutils.fancy_getopt import longopt_xlate - if header is None: - header = "command options for '%s':" % self.get_command_name() - self.announce(indent + header, level=log.INFO) - indent = indent + " " - for (option, _, _) in self.user_options: - option = option.translate(longopt_xlate) - if option[-1] == "=": - option = option[:-1] - value = getattr(self, option) - self.announce(indent + "%s = %s" % (option, value), - level=log.INFO) - - def run(self): - """A command's raison d'etre: carry out the action it exists to - perform, controlled by the options initialized in - 'initialize_options()', customized by other commands, the setup - script, the command-line, and config files, and finalized in - 'finalize_options()'. All terminal output and filesystem - interaction should be done by 'run()'. - - This method must be implemented by all command classes. - """ - raise RuntimeError("abstract method -- subclass %s must override" - % self.__class__) - - def announce(self, msg, level=1): - """If the current verbosity level is of greater than or equal to - 'level' print 'msg' to stdout. - """ - log.log(level, msg) - - def debug_print(self, msg): - """Print 'msg' to stdout if the global DEBUG (taken from the - DISTUTILS_DEBUG environment variable) flag is true. - """ - from distutils.debug import DEBUG - if DEBUG: - print(msg) - sys.stdout.flush() - - - # -- Option validation methods ------------------------------------- - # (these are very handy in writing the 'finalize_options()' method) - # - # NB. the general philosophy here is to ensure that a particular option - # value meets certain type and value constraints. If not, we try to - # force it into conformance (eg. if we expect a list but have a string, - # split the string on comma and/or whitespace). If we can't force the - # option into conformance, raise DistutilsOptionError. Thus, command - # classes need do nothing more than (eg.) - # self.ensure_string_list('foo') - # and they can be guaranteed that thereafter, self.foo will be - # a list of strings. - - def _ensure_stringlike(self, option, what, default=None): - val = getattr(self, option) - if val is None: - setattr(self, option, default) - return default - elif not isinstance(val, str): - raise DistutilsOptionError("'%s' must be a %s (got `%s`)" - % (option, what, val)) - return val - - def ensure_string(self, option, default=None): - """Ensure that 'option' is a string; if not defined, set it to - 'default'. - """ - self._ensure_stringlike(option, "string", default) - - def ensure_string_list(self, option): - r"""Ensure that 'option' is a list of strings. If 'option' is - currently a string, we split it either on /,\s*/ or /\s+/, so - "foo bar baz", "foo,bar,baz", and "foo, bar baz" all become - ["foo", "bar", "baz"]. - """ - val = getattr(self, option) - if val is None: - return - elif isinstance(val, str): - setattr(self, option, re.split(r',\s*|\s+', val)) - else: - if isinstance(val, list): - ok = all(isinstance(v, str) for v in val) - else: - ok = False - if not ok: - raise DistutilsOptionError( - "'%s' must be a list of strings (got %r)" - % (option, val)) - - def _ensure_tested_string(self, option, tester, what, error_fmt, - default=None): - val = self._ensure_stringlike(option, what, default) - if val is not None and not tester(val): - raise DistutilsOptionError(("error in '%s' option: " + error_fmt) - % (option, val)) - - def ensure_filename(self, option): - """Ensure that 'option' is the name of an existing file.""" - self._ensure_tested_string(option, os.path.isfile, - "filename", - "'%s' does not exist or is not a file") - - def ensure_dirname(self, option): - self._ensure_tested_string(option, os.path.isdir, - "directory name", - "'%s' does not exist or is not a directory") - - - # -- Convenience methods for commands ------------------------------ - - def get_command_name(self): - if hasattr(self, 'command_name'): - return self.command_name - else: - return self.__class__.__name__ - - def set_undefined_options(self, src_cmd, *option_pairs): - """Set the values of any "undefined" options from corresponding - option values in some other command object. "Undefined" here means - "is None", which is the convention used to indicate that an option - has not been changed between 'initialize_options()' and - 'finalize_options()'. Usually called from 'finalize_options()' for - options that depend on some other command rather than another - option of the same command. 'src_cmd' is the other command from - which option values will be taken (a command object will be created - for it if necessary); the remaining arguments are - '(src_option,dst_option)' tuples which mean "take the value of - 'src_option' in the 'src_cmd' command object, and copy it to - 'dst_option' in the current command object". - """ - # Option_pairs: list of (src_option, dst_option) tuples - src_cmd_obj = self.distribution.get_command_obj(src_cmd) - src_cmd_obj.ensure_finalized() - for (src_option, dst_option) in option_pairs: - if getattr(self, dst_option) is None: - setattr(self, dst_option, getattr(src_cmd_obj, src_option)) - - def get_finalized_command(self, command, create=1): - """Wrapper around Distribution's 'get_command_obj()' method: find - (create if necessary and 'create' is true) the command object for - 'command', call its 'ensure_finalized()' method, and return the - finalized command object. - """ - cmd_obj = self.distribution.get_command_obj(command, create) - cmd_obj.ensure_finalized() - return cmd_obj - - # XXX rename to 'get_reinitialized_command()'? (should do the - # same in dist.py, if so) - def reinitialize_command(self, command, reinit_subcommands=0): - return self.distribution.reinitialize_command(command, - reinit_subcommands) - - def run_command(self, command): - """Run some other command: uses the 'run_command()' method of - Distribution, which creates and finalizes the command object if - necessary and then invokes its 'run()' method. - """ - self.distribution.run_command(command) - - def get_sub_commands(self): - """Determine the sub-commands that are relevant in the current - distribution (ie., that need to be run). This is based on the - 'sub_commands' class attribute: each tuple in that list may include - a method that we call to determine if the subcommand needs to be - run for the current distribution. Return a list of command names. - """ - commands = [] - for (cmd_name, method) in self.sub_commands: - if method is None or method(self): - commands.append(cmd_name) - return commands - - - # -- External world manipulation ----------------------------------- - - def warn(self, msg): - log.warn("warning: %s: %s\n", self.get_command_name(), msg) - - def execute(self, func, args, msg=None, level=1): - util.execute(func, args, msg, dry_run=self.dry_run) - - def mkpath(self, name, mode=0o777): - dir_util.mkpath(name, mode, dry_run=self.dry_run) - - def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, - link=None, level=1): - """Copy a file respecting verbose, dry-run and force flags. (The - former two default to whatever is in the Distribution object, and - the latter defaults to false for commands that don't define it.)""" - return file_util.copy_file(infile, outfile, preserve_mode, - preserve_times, not self.force, link, - dry_run=self.dry_run) - - def copy_tree(self, infile, outfile, preserve_mode=1, preserve_times=1, - preserve_symlinks=0, level=1): - """Copy an entire directory tree respecting verbose, dry-run, - and force flags. - """ - return dir_util.copy_tree(infile, outfile, preserve_mode, - preserve_times, preserve_symlinks, - not self.force, dry_run=self.dry_run) - - def move_file (self, src, dst, level=1): - """Move a file respecting dry-run flag.""" - return file_util.move_file(src, dst, dry_run=self.dry_run) - - def spawn(self, cmd, search_path=1, level=1): - """Spawn an external command respecting dry-run flag.""" - from distutils.spawn import spawn - spawn(cmd, search_path, dry_run=self.dry_run) - - def make_archive(self, base_name, format, root_dir=None, base_dir=None, - owner=None, group=None): - return archive_util.make_archive(base_name, format, root_dir, base_dir, - dry_run=self.dry_run, - owner=owner, group=group) - - def make_file(self, infiles, outfile, func, args, - exec_msg=None, skip_msg=None, level=1): - """Special case of 'execute()' for operations that process one or - more input files and generate one output file. Works just like - 'execute()', except the operation is skipped and a different - message printed if 'outfile' already exists and is newer than all - files listed in 'infiles'. If the command defined 'self.force', - and it is true, then the command is unconditionally run -- does no - timestamp checks. - """ - if skip_msg is None: - skip_msg = "skipping %s (inputs unchanged)" % outfile - - # Allow 'infiles' to be a single string - if isinstance(infiles, str): - infiles = (infiles,) - elif not isinstance(infiles, (list, tuple)): - raise TypeError( - "'infiles' must be a string, or a list or tuple of strings") - - if exec_msg is None: - exec_msg = "generating %s from %s" % (outfile, ', '.join(infiles)) - - # If 'outfile' must be regenerated (either because it doesn't - # exist, is out-of-date, or the 'force' flag is true) then - # perform the action that presumably regenerates it - if self.force or dep_util.newer_group(infiles, outfile): - self.execute(func, args, exec_msg, level) - # Otherwise, print the "skip" message - else: - log.debug(skip_msg) - -# XXX 'install_misc' class not currently used -- it was the base class for -# both 'install_scripts' and 'install_data', but they outgrew it. It might -# still be useful for 'install_headers', though, so I'm keeping it around -# for the time being. - -class install_misc(Command): - """Common base class for installing some files in a subdirectory. - Currently used by install_data and install_scripts. - """ - - user_options = [('install-dir=', 'd', "directory to install the files to")] - - def initialize_options (self): - self.install_dir = None - self.outfiles = [] - - def _install_dir_from(self, dirname): - self.set_undefined_options('install', (dirname, 'install_dir')) - - def _copy_files(self, filelist): - self.outfiles = [] - if not filelist: - return - self.mkpath(self.install_dir) - for f in filelist: - self.copy_file(f, self.install_dir) - self.outfiles.append(os.path.join(self.install_dir, f)) - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/__init__.py b/Lib/distutils/command/__init__.py deleted file mode 100644 index 481eea9fd4b..00000000000 --- a/Lib/distutils/command/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -"""distutils.command - -Package containing implementation of all the standard Distutils -commands.""" - -__all__ = ['build', - 'build_py', - 'build_ext', - 'build_clib', - 'build_scripts', - 'clean', - 'install', - 'install_lib', - 'install_headers', - 'install_scripts', - 'install_data', - 'sdist', - 'register', - 'bdist', - 'bdist_dumb', - 'bdist_rpm', - 'bdist_wininst', - 'check', - 'upload', - # These two are reserved for future use: - #'bdist_sdux', - #'bdist_pkgtool', - # Note: - # bdist_packager is not included because it only provides - # an abstract base class - ] diff --git a/Lib/distutils/command/bdist.py b/Lib/distutils/command/bdist.py deleted file mode 100644 index 014871d280e..00000000000 --- a/Lib/distutils/command/bdist.py +++ /dev/null @@ -1,143 +0,0 @@ -"""distutils.command.bdist - -Implements the Distutils 'bdist' command (create a built [binary] -distribution).""" - -import os -from distutils.core import Command -from distutils.errors import * -from distutils.util import get_platform - - -def show_formats(): - """Print list of available formats (arguments to "--format" option). - """ - from distutils.fancy_getopt import FancyGetopt - formats = [] - for format in bdist.format_commands: - formats.append(("formats=" + format, None, - bdist.format_command[format][1])) - pretty_printer = FancyGetopt(formats) - pretty_printer.print_help("List of available distribution formats:") - - -class bdist(Command): - - description = "create a built (binary) distribution" - - user_options = [('bdist-base=', 'b', - "temporary directory for creating built distributions"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('formats=', None, - "formats for distribution (comma-separated list)"), - ('dist-dir=', 'd', - "directory to put final built distributions in " - "[default: dist]"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('owner=', 'u', - "Owner name used when creating a tar file" - " [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file" - " [default: current group]"), - ] - - boolean_options = ['skip-build'] - - help_options = [ - ('help-formats', None, - "lists available distribution formats", show_formats), - ] - - # The following commands do not take a format option from bdist - no_format_option = ('bdist_rpm',) - - # This won't do in reality: will need to distinguish RPM-ish Linux, - # Debian-ish Linux, Solaris, FreeBSD, ..., Windows, Mac OS. - default_format = {'posix': 'gztar', - 'nt': 'zip'} - - # Establish the preferred order (for the --help-formats option). - format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', - 'wininst', 'zip', 'msi'] - - # And the real information. - format_command = {'rpm': ('bdist_rpm', "RPM distribution"), - 'gztar': ('bdist_dumb', "gzip'ed tar file"), - 'bztar': ('bdist_dumb', "bzip2'ed tar file"), - 'xztar': ('bdist_dumb', "xz'ed tar file"), - 'ztar': ('bdist_dumb', "compressed tar file"), - 'tar': ('bdist_dumb', "tar file"), - 'wininst': ('bdist_wininst', - "Windows executable installer"), - 'zip': ('bdist_dumb', "ZIP file"), - 'msi': ('bdist_msi', "Microsoft Installer") - } - - - def initialize_options(self): - self.bdist_base = None - self.plat_name = None - self.formats = None - self.dist_dir = None - self.skip_build = 0 - self.group = None - self.owner = None - - def finalize_options(self): - # have to finalize 'plat_name' before 'bdist_base' - if self.plat_name is None: - if self.skip_build: - self.plat_name = get_platform() - else: - self.plat_name = self.get_finalized_command('build').plat_name - - # 'bdist_base' -- parent of per-built-distribution-format - # temporary directories (eg. we'll probably have - # "build/bdist./dumb", "build/bdist./rpm", etc.) - if self.bdist_base is None: - build_base = self.get_finalized_command('build').build_base - self.bdist_base = os.path.join(build_base, - 'bdist.' + self.plat_name) - - self.ensure_string_list('formats') - if self.formats is None: - try: - self.formats = [self.default_format[os.name]] - except KeyError: - raise DistutilsPlatformError( - "don't know how to create built distributions " - "on platform %s" % os.name) - - if self.dist_dir is None: - self.dist_dir = "dist" - - def run(self): - # Figure out which sub-commands we need to run. - commands = [] - for format in self.formats: - try: - commands.append(self.format_command[format][0]) - except KeyError: - raise DistutilsOptionError("invalid format '%s'" % format) - - # Reinitialize and run each command. - for i in range(len(self.formats)): - cmd_name = commands[i] - sub_cmd = self.reinitialize_command(cmd_name) - if cmd_name not in self.no_format_option: - sub_cmd.format = self.formats[i] - - # passing the owner and group names for tar archiving - if cmd_name == 'bdist_dumb': - sub_cmd.owner = self.owner - sub_cmd.group = self.group - - # If we're going to need to run this command again, tell it to - # keep its temporary files around so subsequent runs go faster. - if cmd_name in commands[i+1:]: - sub_cmd.keep_temp = 1 - self.run_command(cmd_name) diff --git a/Lib/distutils/command/bdist_dumb.py b/Lib/distutils/command/bdist_dumb.py deleted file mode 100644 index f0d6b5b8cd8..00000000000 --- a/Lib/distutils/command/bdist_dumb.py +++ /dev/null @@ -1,123 +0,0 @@ -"""distutils.command.bdist_dumb - -Implements the Distutils 'bdist_dumb' command (create a "dumb" built -distribution -- i.e., just an archive to be unpacked under $prefix or -$exec_prefix).""" - -import os -from distutils.core import Command -from distutils.util import get_platform -from distutils.dir_util import remove_tree, ensure_relative -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_dumb(Command): - - description = "create a \"dumb\" built distribution" - - user_options = [('bdist-dir=', 'd', - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('format=', 'f', - "archive format to create (tar, gztar, bztar, xztar, " - "ztar, zip)"), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('relative', None, - "build the archive using relative paths " - "(default: false)"), - ('owner=', 'u', - "Owner name used when creating a tar file" - " [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file" - " [default: current group]"), - ] - - boolean_options = ['keep-temp', 'skip-build', 'relative'] - - default_format = { 'posix': 'gztar', - 'nt': 'zip' } - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.format = None - self.keep_temp = 0 - self.dist_dir = None - self.skip_build = None - self.relative = 0 - self.owner = None - self.group = None - - def finalize_options(self): - if self.bdist_dir is None: - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'dumb') - - if self.format is None: - try: - self.format = self.default_format[os.name] - except KeyError: - raise DistutilsPlatformError( - "don't know how to create dumb built distributions " - "on platform %s" % os.name) - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ('skip_build', 'skip_build')) - - def run(self): - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.root = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - - log.info("installing to %s", self.bdist_dir) - self.run_command('install') - - # And make an archive relative to the root of the - # pseudo-installation tree. - archive_basename = "%s.%s" % (self.distribution.get_fullname(), - self.plat_name) - - pseudoinstall_root = os.path.join(self.dist_dir, archive_basename) - if not self.relative: - archive_root = self.bdist_dir - else: - if (self.distribution.has_ext_modules() and - (install.install_base != install.install_platbase)): - raise DistutilsPlatformError( - "can't make a dumb built distribution where " - "base and platbase are different (%s, %s)" - % (repr(install.install_base), - repr(install.install_platbase))) - else: - archive_root = os.path.join(self.bdist_dir, - ensure_relative(install.install_base)) - - # Make the archive - filename = self.make_archive(pseudoinstall_root, - self.format, root_dir=archive_root, - owner=self.owner, group=self.group) - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - self.distribution.dist_files.append(('bdist_dumb', pyversion, - filename)) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) diff --git a/Lib/distutils/command/bdist_msi.py b/Lib/distutils/command/bdist_msi.py deleted file mode 100644 index 80104c372d9..00000000000 --- a/Lib/distutils/command/bdist_msi.py +++ /dev/null @@ -1,741 +0,0 @@ -# Copyright (C) 2005, 2006 Martin von Löwis -# Licensed to PSF under a Contributor Agreement. -# The bdist_wininst command proper -# based on bdist_wininst -""" -Implements the bdist_msi command. -""" - -import sys, os -from distutils.core import Command -from distutils.dir_util import remove_tree -from distutils.sysconfig import get_python_version -from distutils.version import StrictVersion -from distutils.errors import DistutilsOptionError -from distutils.util import get_platform -from distutils import log -import msilib -from msilib import schema, sequence, text -from msilib import Directory, Feature, Dialog, add_data - -class PyDialog(Dialog): - """Dialog class with a fixed layout: controls at the top, then a ruler, - then a list of buttons: back, next, cancel. Optionally a bitmap at the - left.""" - def __init__(self, *args, **kw): - """Dialog(database, name, x, y, w, h, attributes, title, first, - default, cancel, bitmap=true)""" - Dialog.__init__(self, *args) - ruler = self.h - 36 - bmwidth = 152*ruler/328 - #if kw.get("bitmap", True): - # self.bitmap("Bitmap", 0, 0, bmwidth, ruler, "PythonWin") - self.line("BottomLine", 0, ruler, self.w, 0) - - def title(self, title): - "Set the title text of the dialog at the top." - # name, x, y, w, h, flags=Visible|Enabled|Transparent|NoPrefix, - # text, in VerdanaBold10 - self.text("Title", 15, 10, 320, 60, 0x30003, - r"{\VerdanaBold10}%s" % title) - - def back(self, title, next, name = "Back", active = 1): - """Add a back button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 180, self.h-27 , 56, 17, flags, title, next) - - def cancel(self, title, next, name = "Cancel", active = 1): - """Add a cancel button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 304, self.h-27, 56, 17, flags, title, next) - - def next(self, title, next, name = "Next", active = 1): - """Add a Next button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 236, self.h-27, 56, 17, flags, title, next) - - def xbutton(self, name, title, next, xpos): - """Add a button with a given title, the tab-next button, - its name in the Control table, giving its x position; the - y-position is aligned with the other buttons. - - Return the button, so that events can be associated""" - return self.pushbutton(name, int(self.w*xpos - 28), self.h-27, 56, 17, 3, title, next) - -class bdist_msi(Command): - - description = "create a Microsoft Installer (.msi) binary distribution" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - all_versions = ['2.0', '2.1', '2.2', '2.3', '2.4', - '2.5', '2.6', '2.7', '2.8', '2.9', - '3.0', '3.1', '3.2', '3.3', '3.4', - '3.5', '3.6', '3.7', '3.8', '3.9'] - other_version = 'X' - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.versions = None - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'msi') - - short_version = get_python_version() - if (not self.target_version) and self.distribution.has_ext_modules(): - self.target_version = short_version - - if self.target_version: - self.versions = [self.target_version] - if not self.skip_build and self.distribution.has_ext_modules()\ - and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" - " option must be specified" % (short_version,)) - else: - self.versions = list(self.all_versions) - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.pre_install_script: - raise DistutilsOptionError( - "the pre-install-script feature is not yet implemented") - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - self.install_script_key = None - - def run(self): - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.prefix = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - self.mkpath(self.dist_dir) - fullname = self.distribution.get_fullname() - installer_name = self.get_installer_filename(fullname) - installer_name = os.path.abspath(installer_name) - if os.path.exists(installer_name): os.unlink(installer_name) - - metadata = self.distribution.metadata - author = metadata.author - if not author: - author = metadata.maintainer - if not author: - author = "UNKNOWN" - version = metadata.get_version() - # ProductVersion must be strictly numeric - # XXX need to deal with prerelease versions - sversion = "%d.%d.%d" % StrictVersion(version).version - # Prefix ProductName with Python x.y, so that - # it sorts together with the other Python packages - # in Add-Remove-Programs (APR) - fullname = self.distribution.get_fullname() - if self.target_version: - product_name = "Python %s %s" % (self.target_version, fullname) - else: - product_name = "Python %s" % (fullname) - self.db = msilib.init_database(installer_name, schema, - product_name, msilib.gen_uuid(), - sversion, author) - msilib.add_tables(self.db, sequence) - props = [('DistVersion', version)] - email = metadata.author_email or metadata.maintainer_email - if email: - props.append(("ARPCONTACT", email)) - if metadata.url: - props.append(("ARPURLINFOABOUT", metadata.url)) - if props: - add_data(self.db, 'Property', props) - - self.add_find_python() - self.add_files() - self.add_scripts() - self.add_ui() - self.db.Commit() - - if hasattr(self.distribution, 'dist_files'): - tup = 'bdist_msi', self.target_version or 'any', fullname - self.distribution.dist_files.append(tup) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def add_files(self): - db = self.db - cab = msilib.CAB("distfiles") - rootdir = os.path.abspath(self.bdist_dir) - - root = Directory(db, cab, None, rootdir, "TARGETDIR", "SourceDir") - f = Feature(db, "Python", "Python", "Everything", - 0, 1, directory="TARGETDIR") - - items = [(f, root, '')] - for version in self.versions + [self.other_version]: - target = "TARGETDIR" + version - name = default = "Python" + version - desc = "Everything" - if version is self.other_version: - title = "Python from another location" - level = 2 - else: - title = "Python %s from registry" % version - level = 1 - f = Feature(db, name, title, desc, 1, level, directory=target) - dir = Directory(db, cab, root, rootdir, target, default) - items.append((f, dir, version)) - db.Commit() - - seen = {} - for feature, dir, version in items: - todo = [dir] - while todo: - dir = todo.pop() - for file in os.listdir(dir.absolute): - afile = os.path.join(dir.absolute, file) - if os.path.isdir(afile): - short = "%s|%s" % (dir.make_short(file), file) - default = file + version - newdir = Directory(db, cab, dir, file, default, short) - todo.append(newdir) - else: - if not dir.component: - dir.start_component(dir.logical, feature, 0) - if afile not in seen: - key = seen[afile] = dir.add_file(file) - if file==self.install_script: - if self.install_script_key: - raise DistutilsOptionError( - "Multiple files with name %s" % file) - self.install_script_key = '[#%s]' % key - else: - key = seen[afile] - add_data(self.db, "DuplicateFile", - [(key + version, dir.component, key, None, dir.logical)]) - db.Commit() - cab.commit(db) - - def add_find_python(self): - """Adds code to the installer to compute the location of Python. - - Properties PYTHON.MACHINE.X.Y and PYTHON.USER.X.Y will be set from the - registry for each version of Python. - - Properties TARGETDIRX.Y will be set from PYTHON.USER.X.Y if defined, - else from PYTHON.MACHINE.X.Y. - - Properties PYTHONX.Y will be set to TARGETDIRX.Y\\python.exe""" - - start = 402 - for ver in self.versions: - install_path = r"SOFTWARE\Python\PythonCore\%s\InstallPath" % ver - machine_reg = "python.machine." + ver - user_reg = "python.user." + ver - machine_prop = "PYTHON.MACHINE." + ver - user_prop = "PYTHON.USER." + ver - machine_action = "PythonFromMachine" + ver - user_action = "PythonFromUser" + ver - exe_action = "PythonExe" + ver - target_dir_prop = "TARGETDIR" + ver - exe_prop = "PYTHON" + ver - if msilib.Win64: - # type: msidbLocatorTypeRawValue + msidbLocatorType64bit - Type = 2+16 - else: - Type = 2 - add_data(self.db, "RegLocator", - [(machine_reg, 2, install_path, None, Type), - (user_reg, 1, install_path, None, Type)]) - add_data(self.db, "AppSearch", - [(machine_prop, machine_reg), - (user_prop, user_reg)]) - add_data(self.db, "CustomAction", - [(machine_action, 51+256, target_dir_prop, "[" + machine_prop + "]"), - (user_action, 51+256, target_dir_prop, "[" + user_prop + "]"), - (exe_action, 51+256, exe_prop, "[" + target_dir_prop + "]\\python.exe"), - ]) - add_data(self.db, "InstallExecuteSequence", - [(machine_action, machine_prop, start), - (user_action, user_prop, start + 1), - (exe_action, None, start + 2), - ]) - add_data(self.db, "InstallUISequence", - [(machine_action, machine_prop, start), - (user_action, user_prop, start + 1), - (exe_action, None, start + 2), - ]) - add_data(self.db, "Condition", - [("Python" + ver, 0, "NOT TARGETDIR" + ver)]) - start += 4 - assert start < 500 - - def add_scripts(self): - if self.install_script: - start = 6800 - for ver in self.versions + [self.other_version]: - install_action = "install_script." + ver - exe_prop = "PYTHON" + ver - add_data(self.db, "CustomAction", - [(install_action, 50, exe_prop, self.install_script_key)]) - add_data(self.db, "InstallExecuteSequence", - [(install_action, "&Python%s=3" % ver, start)]) - start += 1 - # XXX pre-install scripts are currently refused in finalize_options() - # but if this feature is completed, it will also need to add - # entries for each version as the above code does - if self.pre_install_script: - scriptfn = os.path.join(self.bdist_dir, "preinstall.bat") - f = open(scriptfn, "w") - # The batch file will be executed with [PYTHON], so that %1 - # is the path to the Python interpreter; %0 will be the path - # of the batch file. - # rem =""" - # %1 %0 - # exit - # """ - # - f.write('rem ="""\n%1 %0\nexit\n"""\n') - f.write(open(self.pre_install_script).read()) - f.close() - add_data(self.db, "Binary", - [("PreInstall", msilib.Binary(scriptfn)) - ]) - add_data(self.db, "CustomAction", - [("PreInstall", 2, "PreInstall", None) - ]) - add_data(self.db, "InstallExecuteSequence", - [("PreInstall", "NOT Installed", 450)]) - - - def add_ui(self): - db = self.db - x = y = 50 - w = 370 - h = 300 - title = "[ProductName] Setup" - - # see "Dialog Style Bits" - modal = 3 # visible | modal - modeless = 1 # visible - track_disk_space = 32 - - # UI customization properties - add_data(db, "Property", - # See "DefaultUIFont Property" - [("DefaultUIFont", "DlgFont8"), - # See "ErrorDialog Style Bit" - ("ErrorDialog", "ErrorDlg"), - ("Progress1", "Install"), # modified in maintenance type dlg - ("Progress2", "installs"), - ("MaintenanceForm_Action", "Repair"), - # possible values: ALL, JUSTME - ("WhichUsers", "ALL") - ]) - - # Fonts, see "TextStyle Table" - add_data(db, "TextStyle", - [("DlgFont8", "Tahoma", 9, None, 0), - ("DlgFontBold8", "Tahoma", 8, None, 1), #bold - ("VerdanaBold10", "Verdana", 10, None, 1), - ("VerdanaRed9", "Verdana", 9, 255, 0), - ]) - - # UI Sequences, see "InstallUISequence Table", "Using a Sequence Table" - # Numbers indicate sequence; see sequence.py for how these action integrate - add_data(db, "InstallUISequence", - [("PrepareDlg", "Not Privileged or Windows9x or Installed", 140), - ("WhichUsersDlg", "Privileged and not Windows9x and not Installed", 141), - # In the user interface, assume all-users installation if privileged. - ("SelectFeaturesDlg", "Not Installed", 1230), - # XXX no support for resume installations yet - #("ResumeDlg", "Installed AND (RESUME OR Preselected)", 1240), - ("MaintenanceTypeDlg", "Installed AND NOT RESUME AND NOT Preselected", 1250), - ("ProgressDlg", None, 1280)]) - - add_data(db, 'ActionText', text.ActionText) - add_data(db, 'UIText', text.UIText) - ##################################################################### - # Standard dialogs: FatalError, UserExit, ExitDialog - fatal=PyDialog(db, "FatalError", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - fatal.title("[ProductName] Installer ended prematurely") - fatal.back("< Back", "Finish", active = 0) - fatal.cancel("Cancel", "Back", active = 0) - fatal.text("Description1", 15, 70, 320, 80, 0x30003, - "[ProductName] setup ended prematurely because of an error. Your system has not been modified. To install this program at a later time, please run the installation again.") - fatal.text("Description2", 15, 155, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c=fatal.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Exit") - - user_exit=PyDialog(db, "UserExit", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - user_exit.title("[ProductName] Installer was interrupted") - user_exit.back("< Back", "Finish", active = 0) - user_exit.cancel("Cancel", "Back", active = 0) - user_exit.text("Description1", 15, 70, 320, 80, 0x30003, - "[ProductName] setup was interrupted. Your system has not been modified. " - "To install this program at a later time, please run the installation again.") - user_exit.text("Description2", 15, 155, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c = user_exit.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Exit") - - exit_dialog = PyDialog(db, "ExitDialog", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - exit_dialog.title("Completing the [ProductName] Installer") - exit_dialog.back("< Back", "Finish", active = 0) - exit_dialog.cancel("Cancel", "Back", active = 0) - exit_dialog.text("Description", 15, 235, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c = exit_dialog.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Return") - - ##################################################################### - # Required dialog: FilesInUse, ErrorDlg - inuse = PyDialog(db, "FilesInUse", - x, y, w, h, - 19, # KeepModeless|Modal|Visible - title, - "Retry", "Retry", "Retry", bitmap=False) - inuse.text("Title", 15, 6, 200, 15, 0x30003, - r"{\DlgFontBold8}Files in Use") - inuse.text("Description", 20, 23, 280, 20, 0x30003, - "Some files that need to be updated are currently in use.") - inuse.text("Text", 20, 55, 330, 50, 3, - "The following applications are using files that need to be updated by this setup. Close these applications and then click Retry to continue the installation or Cancel to exit it.") - inuse.control("List", "ListBox", 20, 107, 330, 130, 7, "FileInUseProcess", - None, None, None) - c=inuse.back("Exit", "Ignore", name="Exit") - c.event("EndDialog", "Exit") - c=inuse.next("Ignore", "Retry", name="Ignore") - c.event("EndDialog", "Ignore") - c=inuse.cancel("Retry", "Exit", name="Retry") - c.event("EndDialog","Retry") - - # See "Error Dialog". See "ICE20" for the required names of the controls. - error = Dialog(db, "ErrorDlg", - 50, 10, 330, 101, - 65543, # Error|Minimize|Modal|Visible - title, - "ErrorText", None, None) - error.text("ErrorText", 50,9,280,48,3, "") - #error.control("ErrorIcon", "Icon", 15, 9, 24, 24, 5242881, None, "py.ico", None, None) - error.pushbutton("N",120,72,81,21,3,"No",None).event("EndDialog","ErrorNo") - error.pushbutton("Y",240,72,81,21,3,"Yes",None).event("EndDialog","ErrorYes") - error.pushbutton("A",0,72,81,21,3,"Abort",None).event("EndDialog","ErrorAbort") - error.pushbutton("C",42,72,81,21,3,"Cancel",None).event("EndDialog","ErrorCancel") - error.pushbutton("I",81,72,81,21,3,"Ignore",None).event("EndDialog","ErrorIgnore") - error.pushbutton("O",159,72,81,21,3,"Ok",None).event("EndDialog","ErrorOk") - error.pushbutton("R",198,72,81,21,3,"Retry",None).event("EndDialog","ErrorRetry") - - ##################################################################### - # Global "Query Cancel" dialog - cancel = Dialog(db, "CancelDlg", 50, 10, 260, 85, 3, title, - "No", "No", "No") - cancel.text("Text", 48, 15, 194, 30, 3, - "Are you sure you want to cancel [ProductName] installation?") - #cancel.control("Icon", "Icon", 15, 15, 24, 24, 5242881, None, - # "py.ico", None, None) - c=cancel.pushbutton("Yes", 72, 57, 56, 17, 3, "Yes", "No") - c.event("EndDialog", "Exit") - - c=cancel.pushbutton("No", 132, 57, 56, 17, 3, "No", "Yes") - c.event("EndDialog", "Return") - - ##################################################################### - # Global "Wait for costing" dialog - costing = Dialog(db, "WaitForCostingDlg", 50, 10, 260, 85, modal, title, - "Return", "Return", "Return") - costing.text("Text", 48, 15, 194, 30, 3, - "Please wait while the installer finishes determining your disk space requirements.") - c = costing.pushbutton("Return", 102, 57, 56, 17, 3, "Return", None) - c.event("EndDialog", "Exit") - - ##################################################################### - # Preparation dialog: no user input except cancellation - prep = PyDialog(db, "PrepareDlg", x, y, w, h, modeless, title, - "Cancel", "Cancel", "Cancel") - prep.text("Description", 15, 70, 320, 40, 0x30003, - "Please wait while the Installer prepares to guide you through the installation.") - prep.title("Welcome to the [ProductName] Installer") - c=prep.text("ActionText", 15, 110, 320, 20, 0x30003, "Pondering...") - c.mapping("ActionText", "Text") - c=prep.text("ActionData", 15, 135, 320, 30, 0x30003, None) - c.mapping("ActionData", "Text") - prep.back("Back", None, active=0) - prep.next("Next", None, active=0) - c=prep.cancel("Cancel", None) - c.event("SpawnDialog", "CancelDlg") - - ##################################################################### - # Feature (Python directory) selection - seldlg = PyDialog(db, "SelectFeaturesDlg", x, y, w, h, modal, title, - "Next", "Next", "Cancel") - seldlg.title("Select Python Installations") - - seldlg.text("Hint", 15, 30, 300, 20, 3, - "Select the Python locations where %s should be installed." - % self.distribution.get_fullname()) - - seldlg.back("< Back", None, active=0) - c = seldlg.next("Next >", "Cancel") - order = 1 - c.event("[TARGETDIR]", "[SourceDir]", ordering=order) - for version in self.versions + [self.other_version]: - order += 1 - c.event("[TARGETDIR]", "[TARGETDIR%s]" % version, - "FEATURE_SELECTED AND &Python%s=3" % version, - ordering=order) - c.event("SpawnWaitDialog", "WaitForCostingDlg", ordering=order + 1) - c.event("EndDialog", "Return", ordering=order + 2) - c = seldlg.cancel("Cancel", "Features") - c.event("SpawnDialog", "CancelDlg") - - c = seldlg.control("Features", "SelectionTree", 15, 60, 300, 120, 3, - "FEATURE", None, "PathEdit", None) - c.event("[FEATURE_SELECTED]", "1") - ver = self.other_version - install_other_cond = "FEATURE_SELECTED AND &Python%s=3" % ver - dont_install_other_cond = "FEATURE_SELECTED AND &Python%s<>3" % ver - - c = seldlg.text("Other", 15, 200, 300, 15, 3, - "Provide an alternate Python location") - c.condition("Enable", install_other_cond) - c.condition("Show", install_other_cond) - c.condition("Disable", dont_install_other_cond) - c.condition("Hide", dont_install_other_cond) - - c = seldlg.control("PathEdit", "PathEdit", 15, 215, 300, 16, 1, - "TARGETDIR" + ver, None, "Next", None) - c.condition("Enable", install_other_cond) - c.condition("Show", install_other_cond) - c.condition("Disable", dont_install_other_cond) - c.condition("Hide", dont_install_other_cond) - - ##################################################################### - # Disk cost - cost = PyDialog(db, "DiskCostDlg", x, y, w, h, modal, title, - "OK", "OK", "OK", bitmap=False) - cost.text("Title", 15, 6, 200, 15, 0x30003, - r"{\DlgFontBold8}Disk Space Requirements") - cost.text("Description", 20, 20, 280, 20, 0x30003, - "The disk space required for the installation of the selected features.") - cost.text("Text", 20, 53, 330, 60, 3, - "The highlighted volumes (if any) do not have enough disk space " - "available for the currently selected features. You can either " - "remove some files from the highlighted volumes, or choose to " - "install less features onto local drive(s), or select different " - "destination drive(s).") - cost.control("VolumeList", "VolumeCostList", 20, 100, 330, 150, 393223, - None, "{120}{70}{70}{70}{70}", None, None) - cost.xbutton("OK", "Ok", None, 0.5).event("EndDialog", "Return") - - ##################################################################### - # WhichUsers Dialog. Only available on NT, and for privileged users. - # This must be run before FindRelatedProducts, because that will - # take into account whether the previous installation was per-user - # or per-machine. We currently don't support going back to this - # dialog after "Next" was selected; to support this, we would need to - # find how to reset the ALLUSERS property, and how to re-run - # FindRelatedProducts. - # On Windows9x, the ALLUSERS property is ignored on the command line - # and in the Property table, but installer fails according to the documentation - # if a dialog attempts to set ALLUSERS. - whichusers = PyDialog(db, "WhichUsersDlg", x, y, w, h, modal, title, - "AdminInstall", "Next", "Cancel") - whichusers.title("Select whether to install [ProductName] for all users of this computer.") - # A radio group with two options: allusers, justme - g = whichusers.radiogroup("AdminInstall", 15, 60, 260, 50, 3, - "WhichUsers", "", "Next") - g.add("ALL", 0, 5, 150, 20, "Install for all users") - g.add("JUSTME", 0, 25, 150, 20, "Install just for me") - - whichusers.back("Back", None, active=0) - - c = whichusers.next("Next >", "Cancel") - c.event("[ALLUSERS]", "1", 'WhichUsers="ALL"', 1) - c.event("EndDialog", "Return", ordering = 2) - - c = whichusers.cancel("Cancel", "AdminInstall") - c.event("SpawnDialog", "CancelDlg") - - ##################################################################### - # Installation Progress dialog (modeless) - progress = PyDialog(db, "ProgressDlg", x, y, w, h, modeless, title, - "Cancel", "Cancel", "Cancel", bitmap=False) - progress.text("Title", 20, 15, 200, 15, 0x30003, - r"{\DlgFontBold8}[Progress1] [ProductName]") - progress.text("Text", 35, 65, 300, 30, 3, - "Please wait while the Installer [Progress2] [ProductName]. " - "This may take several minutes.") - progress.text("StatusLabel", 35, 100, 35, 20, 3, "Status:") - - c=progress.text("ActionText", 70, 100, w-70, 20, 3, "Pondering...") - c.mapping("ActionText", "Text") - - #c=progress.text("ActionData", 35, 140, 300, 20, 3, None) - #c.mapping("ActionData", "Text") - - c=progress.control("ProgressBar", "ProgressBar", 35, 120, 300, 10, 65537, - None, "Progress done", None, None) - c.mapping("SetProgress", "Progress") - - progress.back("< Back", "Next", active=False) - progress.next("Next >", "Cancel", active=False) - progress.cancel("Cancel", "Back").event("SpawnDialog", "CancelDlg") - - ################################################################### - # Maintenance type: repair/uninstall - maint = PyDialog(db, "MaintenanceTypeDlg", x, y, w, h, modal, title, - "Next", "Next", "Cancel") - maint.title("Welcome to the [ProductName] Setup Wizard") - maint.text("BodyText", 15, 63, 330, 42, 3, - "Select whether you want to repair or remove [ProductName].") - g=maint.radiogroup("RepairRadioGroup", 15, 108, 330, 60, 3, - "MaintenanceForm_Action", "", "Next") - #g.add("Change", 0, 0, 200, 17, "&Change [ProductName]") - g.add("Repair", 0, 18, 200, 17, "&Repair [ProductName]") - g.add("Remove", 0, 36, 200, 17, "Re&move [ProductName]") - - maint.back("< Back", None, active=False) - c=maint.next("Finish", "Cancel") - # Change installation: Change progress dialog to "Change", then ask - # for feature selection - #c.event("[Progress1]", "Change", 'MaintenanceForm_Action="Change"', 1) - #c.event("[Progress2]", "changes", 'MaintenanceForm_Action="Change"', 2) - - # Reinstall: Change progress dialog to "Repair", then invoke reinstall - # Also set list of reinstalled features to "ALL" - c.event("[REINSTALL]", "ALL", 'MaintenanceForm_Action="Repair"', 5) - c.event("[Progress1]", "Repairing", 'MaintenanceForm_Action="Repair"', 6) - c.event("[Progress2]", "repairs", 'MaintenanceForm_Action="Repair"', 7) - c.event("Reinstall", "ALL", 'MaintenanceForm_Action="Repair"', 8) - - # Uninstall: Change progress to "Remove", then invoke uninstall - # Also set list of removed features to "ALL" - c.event("[REMOVE]", "ALL", 'MaintenanceForm_Action="Remove"', 11) - c.event("[Progress1]", "Removing", 'MaintenanceForm_Action="Remove"', 12) - c.event("[Progress2]", "removes", 'MaintenanceForm_Action="Remove"', 13) - c.event("Remove", "ALL", 'MaintenanceForm_Action="Remove"', 14) - - # Close dialog when maintenance action scheduled - c.event("EndDialog", "Return", 'MaintenanceForm_Action<>"Change"', 20) - #c.event("NewDialog", "SelectFeaturesDlg", 'MaintenanceForm_Action="Change"', 21) - - maint.cancel("Cancel", "RepairRadioGroup").event("SpawnDialog", "CancelDlg") - - def get_installer_filename(self, fullname): - # Factored out to allow overriding in subclasses - if self.target_version: - base_name = "%s.%s-py%s.msi" % (fullname, self.plat_name, - self.target_version) - else: - base_name = "%s.%s.msi" % (fullname, self.plat_name) - installer_name = os.path.join(self.dist_dir, base_name) - return installer_name diff --git a/Lib/distutils/command/bdist_rpm.py b/Lib/distutils/command/bdist_rpm.py deleted file mode 100644 index 02f10dd89d9..00000000000 --- a/Lib/distutils/command/bdist_rpm.py +++ /dev/null @@ -1,582 +0,0 @@ -"""distutils.command.bdist_rpm - -Implements the Distutils 'bdist_rpm' command (create RPM source and binary -distributions).""" - -import subprocess, sys, os -from distutils.core import Command -from distutils.debug import DEBUG -from distutils.util import get_platform -from distutils.file_util import write_file -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_rpm(Command): - - description = "create an RPM distribution" - - user_options = [ - ('bdist-base=', None, - "base directory for creating built distributions"), - ('rpm-base=', None, - "base directory for creating RPMs (defaults to \"rpm\" under " - "--bdist-base; must be specified for RPM 2)"), - ('dist-dir=', 'd', - "directory to put final RPM files in " - "(and .spec files if --spec-only)"), - ('python=', None, - "path to Python interpreter to hard-code in the .spec file " - "(default: \"python\")"), - ('fix-python', None, - "hard-code the exact path to the current Python interpreter in " - "the .spec file"), - ('spec-only', None, - "only regenerate spec file"), - ('source-only', None, - "only generate source RPM"), - ('binary-only', None, - "only generate binary RPM"), - ('use-bzip2', None, - "use bzip2 instead of gzip to create source distribution"), - - # More meta-data: too RPM-specific to put in the setup script, - # but needs to go in the .spec file -- so we make these options - # to "bdist_rpm". The idea is that packagers would put this - # info in setup.cfg, although they are of course free to - # supply it on the command line. - ('distribution-name=', None, - "name of the (Linux) distribution to which this " - "RPM applies (*not* the name of the module distribution!)"), - ('group=', None, - "package classification [default: \"Development/Libraries\"]"), - ('release=', None, - "RPM release number"), - ('serial=', None, - "RPM serial number"), - ('vendor=', None, - "RPM \"vendor\" (eg. \"Joe Blow \") " - "[default: maintainer or author from setup script]"), - ('packager=', None, - "RPM packager (eg. \"Jane Doe \") " - "[default: vendor]"), - ('doc-files=', None, - "list of documentation files (space or comma-separated)"), - ('changelog=', None, - "RPM changelog"), - ('icon=', None, - "name of icon file"), - ('provides=', None, - "capabilities provided by this package"), - ('requires=', None, - "capabilities required by this package"), - ('conflicts=', None, - "capabilities which conflict with this package"), - ('build-requires=', None, - "capabilities required to build this package"), - ('obsoletes=', None, - "capabilities made obsolete by this package"), - ('no-autoreq', None, - "do not automatically calculate dependencies"), - - # Actions to take when building RPM - ('keep-temp', 'k', - "don't clean up RPM build directory"), - ('no-keep-temp', None, - "clean up RPM build directory [default]"), - ('use-rpm-opt-flags', None, - "compile with RPM_OPT_FLAGS when building from source RPM"), - ('no-rpm-opt-flags', None, - "do not pass any RPM CFLAGS to compiler"), - ('rpm3-mode', None, - "RPM 3 compatibility mode (default)"), - ('rpm2-mode', None, - "RPM 2 compatibility mode"), - - # Add the hooks necessary for specifying custom scripts - ('prep-script=', None, - "Specify a script for the PREP phase of RPM building"), - ('build-script=', None, - "Specify a script for the BUILD phase of RPM building"), - - ('pre-install=', None, - "Specify a script for the pre-INSTALL phase of RPM building"), - ('install-script=', None, - "Specify a script for the INSTALL phase of RPM building"), - ('post-install=', None, - "Specify a script for the post-INSTALL phase of RPM building"), - - ('pre-uninstall=', None, - "Specify a script for the pre-UNINSTALL phase of RPM building"), - ('post-uninstall=', None, - "Specify a script for the post-UNINSTALL phase of RPM building"), - - ('clean-script=', None, - "Specify a script for the CLEAN phase of RPM building"), - - ('verify-script=', None, - "Specify a script for the VERIFY phase of the RPM build"), - - # Allow a packager to explicitly force an architecture - ('force-arch=', None, - "Force an architecture onto the RPM build process"), - - ('quiet', 'q', - "Run the INSTALL phase of RPM building in quiet mode"), - ] - - boolean_options = ['keep-temp', 'use-rpm-opt-flags', 'rpm3-mode', - 'no-autoreq', 'quiet'] - - negative_opt = {'no-keep-temp': 'keep-temp', - 'no-rpm-opt-flags': 'use-rpm-opt-flags', - 'rpm2-mode': 'rpm3-mode'} - - - def initialize_options(self): - self.bdist_base = None - self.rpm_base = None - self.dist_dir = None - self.python = None - self.fix_python = None - self.spec_only = None - self.binary_only = None - self.source_only = None - self.use_bzip2 = None - - self.distribution_name = None - self.group = None - self.release = None - self.serial = None - self.vendor = None - self.packager = None - self.doc_files = None - self.changelog = None - self.icon = None - - self.prep_script = None - self.build_script = None - self.install_script = None - self.clean_script = None - self.verify_script = None - self.pre_install = None - self.post_install = None - self.pre_uninstall = None - self.post_uninstall = None - self.prep = None - self.provides = None - self.requires = None - self.conflicts = None - self.build_requires = None - self.obsoletes = None - - self.keep_temp = 0 - self.use_rpm_opt_flags = 1 - self.rpm3_mode = 1 - self.no_autoreq = 0 - - self.force_arch = None - self.quiet = 0 - - def finalize_options(self): - self.set_undefined_options('bdist', ('bdist_base', 'bdist_base')) - if self.rpm_base is None: - if not self.rpm3_mode: - raise DistutilsOptionError( - "you must specify --rpm-base in RPM 2 mode") - self.rpm_base = os.path.join(self.bdist_base, "rpm") - - if self.python is None: - if self.fix_python: - self.python = sys.executable - else: - self.python = "python3" - elif self.fix_python: - raise DistutilsOptionError( - "--python and --fix-python are mutually exclusive options") - - if os.name != 'posix': - raise DistutilsPlatformError("don't know how to create RPM " - "distributions on platform %s" % os.name) - if self.binary_only and self.source_only: - raise DistutilsOptionError( - "cannot supply both '--source-only' and '--binary-only'") - - # don't pass CFLAGS to pure python distributions - if not self.distribution.has_ext_modules(): - self.use_rpm_opt_flags = 0 - - self.set_undefined_options('bdist', ('dist_dir', 'dist_dir')) - self.finalize_package_data() - - def finalize_package_data(self): - self.ensure_string('group', "Development/Libraries") - self.ensure_string('vendor', - "%s <%s>" % (self.distribution.get_contact(), - self.distribution.get_contact_email())) - self.ensure_string('packager') - self.ensure_string_list('doc_files') - if isinstance(self.doc_files, list): - for readme in ('README', 'README.txt'): - if os.path.exists(readme) and readme not in self.doc_files: - self.doc_files.append(readme) - - self.ensure_string('release', "1") - self.ensure_string('serial') # should it be an int? - - self.ensure_string('distribution_name') - - self.ensure_string('changelog') - # Format changelog correctly - self.changelog = self._format_changelog(self.changelog) - - self.ensure_filename('icon') - - self.ensure_filename('prep_script') - self.ensure_filename('build_script') - self.ensure_filename('install_script') - self.ensure_filename('clean_script') - self.ensure_filename('verify_script') - self.ensure_filename('pre_install') - self.ensure_filename('post_install') - self.ensure_filename('pre_uninstall') - self.ensure_filename('post_uninstall') - - # XXX don't forget we punted on summaries and descriptions -- they - # should be handled here eventually! - - # Now *this* is some meta-data that belongs in the setup script... - self.ensure_string_list('provides') - self.ensure_string_list('requires') - self.ensure_string_list('conflicts') - self.ensure_string_list('build_requires') - self.ensure_string_list('obsoletes') - - self.ensure_string('force_arch') - - def run(self): - if DEBUG: - print("before _get_package_data():") - print("vendor =", self.vendor) - print("packager =", self.packager) - print("doc_files =", self.doc_files) - print("changelog =", self.changelog) - - # make directories - if self.spec_only: - spec_dir = self.dist_dir - self.mkpath(spec_dir) - else: - rpm_dir = {} - for d in ('SOURCES', 'SPECS', 'BUILD', 'RPMS', 'SRPMS'): - rpm_dir[d] = os.path.join(self.rpm_base, d) - self.mkpath(rpm_dir[d]) - spec_dir = rpm_dir['SPECS'] - - # Spec file goes into 'dist_dir' if '--spec-only specified', - # build/rpm. otherwise. - spec_path = os.path.join(spec_dir, - "%s.spec" % self.distribution.get_name()) - self.execute(write_file, - (spec_path, - self._make_spec_file()), - "writing '%s'" % spec_path) - - if self.spec_only: # stop if requested - return - - # Make a source distribution and copy to SOURCES directory with - # optional icon. - saved_dist_files = self.distribution.dist_files[:] - sdist = self.reinitialize_command('sdist') - if self.use_bzip2: - sdist.formats = ['bztar'] - else: - sdist.formats = ['gztar'] - self.run_command('sdist') - self.distribution.dist_files = saved_dist_files - - source = sdist.get_archive_files()[0] - source_dir = rpm_dir['SOURCES'] - self.copy_file(source, source_dir) - - if self.icon: - if os.path.exists(self.icon): - self.copy_file(self.icon, source_dir) - else: - raise DistutilsFileError( - "icon file '%s' does not exist" % self.icon) - - # build package - log.info("building RPMs") - rpm_cmd = ['rpm'] - if os.path.exists('/usr/bin/rpmbuild') or \ - os.path.exists('/bin/rpmbuild'): - rpm_cmd = ['rpmbuild'] - - if self.source_only: # what kind of RPMs? - rpm_cmd.append('-bs') - elif self.binary_only: - rpm_cmd.append('-bb') - else: - rpm_cmd.append('-ba') - rpm_cmd.extend(['--define', '__python %s' % self.python]) - if self.rpm3_mode: - rpm_cmd.extend(['--define', - '_topdir %s' % os.path.abspath(self.rpm_base)]) - if not self.keep_temp: - rpm_cmd.append('--clean') - - if self.quiet: - rpm_cmd.append('--quiet') - - rpm_cmd.append(spec_path) - # Determine the binary rpm names that should be built out of this spec - # file - # Note that some of these may not be really built (if the file - # list is empty) - nvr_string = "%{name}-%{version}-%{release}" - src_rpm = nvr_string + ".src.rpm" - non_src_rpm = "%{arch}/" + nvr_string + ".%{arch}.rpm" - q_cmd = r"rpm -q --qf '%s %s\n' --specfile '%s'" % ( - src_rpm, non_src_rpm, spec_path) - - out = os.popen(q_cmd) - try: - binary_rpms = [] - source_rpm = None - while True: - line = out.readline() - if not line: - break - l = line.strip().split() - assert(len(l) == 2) - binary_rpms.append(l[1]) - # The source rpm is named after the first entry in the spec file - if source_rpm is None: - source_rpm = l[0] - - status = out.close() - if status: - raise DistutilsExecError("Failed to execute: %s" % repr(q_cmd)) - - finally: - out.close() - - self.spawn(rpm_cmd) - - if not self.dry_run: - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - - if not self.binary_only: - srpm = os.path.join(rpm_dir['SRPMS'], source_rpm) - assert(os.path.exists(srpm)) - self.move_file(srpm, self.dist_dir) - filename = os.path.join(self.dist_dir, source_rpm) - self.distribution.dist_files.append( - ('bdist_rpm', pyversion, filename)) - - if not self.source_only: - for rpm in binary_rpms: - rpm = os.path.join(rpm_dir['RPMS'], rpm) - if os.path.exists(rpm): - self.move_file(rpm, self.dist_dir) - filename = os.path.join(self.dist_dir, - os.path.basename(rpm)) - self.distribution.dist_files.append( - ('bdist_rpm', pyversion, filename)) - - def _dist_path(self, path): - return os.path.join(self.dist_dir, os.path.basename(path)) - - def _make_spec_file(self): - """Generate the text of an RPM spec file and return it as a - list of strings (one per line). - """ - # definitions and headers - spec_file = [ - '%define name ' + self.distribution.get_name(), - '%define version ' + self.distribution.get_version().replace('-','_'), - '%define unmangled_version ' + self.distribution.get_version(), - '%define release ' + self.release.replace('-','_'), - '', - 'Summary: ' + self.distribution.get_description(), - ] - - # Workaround for #14443 which affects some RPM based systems such as - # RHEL6 (and probably derivatives) - vendor_hook = subprocess.getoutput('rpm --eval %{__os_install_post}') - # Generate a potential replacement value for __os_install_post (whilst - # normalizing the whitespace to simplify the test for whether the - # invocation of brp-python-bytecompile passes in __python): - vendor_hook = '\n'.join([' %s \\' % line.strip() - for line in vendor_hook.splitlines()]) - problem = "brp-python-bytecompile \\\n" - fixed = "brp-python-bytecompile %{__python} \\\n" - fixed_hook = vendor_hook.replace(problem, fixed) - if fixed_hook != vendor_hook: - spec_file.append('# Workaround for http://bugs.python.org/issue14443') - spec_file.append('%define __os_install_post ' + fixed_hook + '\n') - - # put locale summaries into spec file - # XXX not supported for now (hard to put a dictionary - # in a config file -- arg!) - #for locale in self.summaries.keys(): - # spec_file.append('Summary(%s): %s' % (locale, - # self.summaries[locale])) - - spec_file.extend([ - 'Name: %{name}', - 'Version: %{version}', - 'Release: %{release}',]) - - # XXX yuck! this filename is available from the "sdist" command, - # but only after it has run: and we create the spec file before - # running "sdist", in case of --spec-only. - if self.use_bzip2: - spec_file.append('Source0: %{name}-%{unmangled_version}.tar.bz2') - else: - spec_file.append('Source0: %{name}-%{unmangled_version}.tar.gz') - - spec_file.extend([ - 'License: ' + self.distribution.get_license(), - 'Group: ' + self.group, - 'BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot', - 'Prefix: %{_prefix}', ]) - - if not self.force_arch: - # noarch if no extension modules - if not self.distribution.has_ext_modules(): - spec_file.append('BuildArch: noarch') - else: - spec_file.append( 'BuildArch: %s' % self.force_arch ) - - for field in ('Vendor', - 'Packager', - 'Provides', - 'Requires', - 'Conflicts', - 'Obsoletes', - ): - val = getattr(self, field.lower()) - if isinstance(val, list): - spec_file.append('%s: %s' % (field, ' '.join(val))) - elif val is not None: - spec_file.append('%s: %s' % (field, val)) - - - if self.distribution.get_url() != 'UNKNOWN': - spec_file.append('Url: ' + self.distribution.get_url()) - - if self.distribution_name: - spec_file.append('Distribution: ' + self.distribution_name) - - if self.build_requires: - spec_file.append('BuildRequires: ' + - ' '.join(self.build_requires)) - - if self.icon: - spec_file.append('Icon: ' + os.path.basename(self.icon)) - - if self.no_autoreq: - spec_file.append('AutoReq: 0') - - spec_file.extend([ - '', - '%description', - self.distribution.get_long_description() - ]) - - # put locale descriptions into spec file - # XXX again, suppressed because config file syntax doesn't - # easily support this ;-( - #for locale in self.descriptions.keys(): - # spec_file.extend([ - # '', - # '%description -l ' + locale, - # self.descriptions[locale], - # ]) - - # rpm scripts - # figure out default build script - def_setup_call = "%s %s" % (self.python,os.path.basename(sys.argv[0])) - def_build = "%s build" % def_setup_call - if self.use_rpm_opt_flags: - def_build = 'env CFLAGS="$RPM_OPT_FLAGS" ' + def_build - - # insert contents of files - - # XXX this is kind of misleading: user-supplied options are files - # that we open and interpolate into the spec file, but the defaults - # are just text that we drop in as-is. Hmmm. - - install_cmd = ('%s install -O1 --root=$RPM_BUILD_ROOT ' - '--record=INSTALLED_FILES') % def_setup_call - - script_options = [ - ('prep', 'prep_script', "%setup -n %{name}-%{unmangled_version}"), - ('build', 'build_script', def_build), - ('install', 'install_script', install_cmd), - ('clean', 'clean_script', "rm -rf $RPM_BUILD_ROOT"), - ('verifyscript', 'verify_script', None), - ('pre', 'pre_install', None), - ('post', 'post_install', None), - ('preun', 'pre_uninstall', None), - ('postun', 'post_uninstall', None), - ] - - for (rpm_opt, attr, default) in script_options: - # Insert contents of file referred to, if no file is referred to - # use 'default' as contents of script - val = getattr(self, attr) - if val or default: - spec_file.extend([ - '', - '%' + rpm_opt,]) - if val: - spec_file.extend(open(val, 'r').read().split('\n')) - else: - spec_file.append(default) - - - # files section - spec_file.extend([ - '', - '%files -f INSTALLED_FILES', - '%defattr(-,root,root)', - ]) - - if self.doc_files: - spec_file.append('%doc ' + ' '.join(self.doc_files)) - - if self.changelog: - spec_file.extend([ - '', - '%changelog',]) - spec_file.extend(self.changelog) - - return spec_file - - def _format_changelog(self, changelog): - """Format the changelog correctly and convert it to a list of strings - """ - if not changelog: - return changelog - new_changelog = [] - for line in changelog.strip().split('\n'): - line = line.strip() - if line[0] == '*': - new_changelog.extend(['', line]) - elif line[0] == '-': - new_changelog.append(line) - else: - new_changelog.append(' ' + line) - - # strip trailing newline inserted by first changelog entry - if not new_changelog[0]: - del new_changelog[0] - - return new_changelog diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py deleted file mode 100644 index 1db47f9b983..00000000000 --- a/Lib/distutils/command/bdist_wininst.py +++ /dev/null @@ -1,367 +0,0 @@ -"""distutils.command.bdist_wininst - -Implements the Distutils 'bdist_wininst' command: create a windows installer -exe-program.""" - -import sys, os -from distutils.core import Command -from distutils.util import get_platform -from distutils.dir_util import create_tree, remove_tree -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_wininst(Command): - - description = "create an executable installer for MS Windows" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('bitmap=', 'b', - "bitmap to use for the installer instead of python-powered logo"), - ('title=', 't', - "title to display on the installer background instead of default"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ('user-access-control=', None, - "specify Vista's UAC handling - 'none'/default=no " - "handling, 'auto'=use UAC if target Python installed for " - "all users, 'force'=always use UAC"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.bitmap = None - self.title = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.user_access_control = None - - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - if self.skip_build and self.plat_name: - # If build is skipped and plat_name is overridden, bdist will - # not see the correct 'plat_name' - so set that up manually. - bdist = self.distribution.get_command_obj('bdist') - bdist.plat_name = self.plat_name - # next the command will be initialized using that name - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'wininst') - - if not self.target_version: - self.target_version = "" - - if not self.skip_build and self.distribution.has_ext_modules(): - short_version = get_python_version() - if self.target_version and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" \ - " option must be specified" % (short_version,)) - self.target_version = short_version - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - - def run(self): - if (sys.platform != "win32" and - (self.distribution.has_ext_modules() or - self.distribution.has_c_libraries())): - raise DistutilsPlatformError \ - ("distribution contains extensions and/or C libraries; " - "must be compiled on a Windows 32 platform") - - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.root = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - install.plat_name = self.plat_name - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - # Use a custom scheme for the zip-file, because we have to decide - # at installation time which scheme to use. - for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'): - value = key.upper() - if key == 'headers': - value = value + '/Include/$dist_name' - setattr(install, - 'install_' + key, - value) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - # And make an archive relative to the root of the - # pseudo-installation tree. - from tempfile import mktemp - archive_basename = mktemp() - fullname = self.distribution.get_fullname() - arcname = self.make_archive(archive_basename, "zip", - root_dir=self.bdist_dir) - # create an exe containing the zip-file - self.create_exe(arcname, fullname, self.bitmap) - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - self.distribution.dist_files.append(('bdist_wininst', pyversion, - self.get_installer_filename(fullname))) - # remove the zip-file again - log.debug("removing temporary file '%s'", arcname) - os.remove(arcname) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def get_inidata(self): - # Return data describing the installation. - lines = [] - metadata = self.distribution.metadata - - # Write the [metadata] section. - lines.append("[metadata]") - - # 'info' will be displayed in the installer's dialog box, - # describing the items to be installed. - info = (metadata.long_description or '') + '\n' - - # Escape newline characters - def escape(s): - return s.replace("\n", "\\n") - - for name in ["author", "author_email", "description", "maintainer", - "maintainer_email", "name", "url", "version"]: - data = getattr(metadata, name, "") - if data: - info = info + ("\n %s: %s" % \ - (name.capitalize(), escape(data))) - lines.append("%s=%s" % (name, escape(data))) - - # The [setup] section contains entries controlling - # the installer runtime. - lines.append("\n[Setup]") - if self.install_script: - lines.append("install_script=%s" % self.install_script) - lines.append("info=%s" % escape(info)) - lines.append("target_compile=%d" % (not self.no_target_compile)) - lines.append("target_optimize=%d" % (not self.no_target_optimize)) - if self.target_version: - lines.append("target_version=%s" % self.target_version) - if self.user_access_control: - lines.append("user_access_control=%s" % self.user_access_control) - - title = self.title or self.distribution.get_fullname() - lines.append("title=%s" % escape(title)) - import time - import distutils - build_info = "Built %s with distutils-%s" % \ - (time.ctime(time.time()), distutils.__version__) - lines.append("build_info=%s" % build_info) - return "\n".join(lines) - - def create_exe(self, arcname, fullname, bitmap=None): - import struct - - self.mkpath(self.dist_dir) - - cfgdata = self.get_inidata() - - installer_name = self.get_installer_filename(fullname) - self.announce("creating %s" % installer_name) - - if bitmap: - bitmapdata = open(bitmap, "rb").read() - bitmaplen = len(bitmapdata) - else: - bitmaplen = 0 - - file = open(installer_name, "wb") - file.write(self.get_exe_bytes()) - if bitmap: - file.write(bitmapdata) - - # Convert cfgdata from unicode to ascii, mbcs encoded - if isinstance(cfgdata, str): - cfgdata = cfgdata.encode("mbcs") - - # Append the pre-install script - cfgdata = cfgdata + b"\0" - if self.pre_install_script: - # We need to normalize newlines, so we open in text mode and - # convert back to bytes. "latin-1" simply avoids any possible - # failures. - with open(self.pre_install_script, "r", - encoding="latin-1") as script: - script_data = script.read().encode("latin-1") - cfgdata = cfgdata + script_data + b"\n\0" - else: - # empty pre-install script - cfgdata = cfgdata + b"\0" - file.write(cfgdata) - - # The 'magic number' 0x1234567B is used to make sure that the - # binary layout of 'cfgdata' is what the wininst.exe binary - # expects. If the layout changes, increment that number, make - # the corresponding changes to the wininst.exe sources, and - # recompile them. - header = struct.pack("' under the base build directory. We only use one of - # them for a given distribution, though -- - if self.build_purelib is None: - self.build_purelib = os.path.join(self.build_base, 'lib') - if self.build_platlib is None: - self.build_platlib = os.path.join(self.build_base, - 'lib' + plat_specifier) - - # 'build_lib' is the actual directory that we will use for this - # particular module distribution -- if user didn't supply it, pick - # one of 'build_purelib' or 'build_platlib'. - if self.build_lib is None: - if self.distribution.ext_modules: - self.build_lib = self.build_platlib - else: - self.build_lib = self.build_purelib - - # 'build_temp' -- temporary directory for compiler turds, - # "build/temp." - if self.build_temp is None: - self.build_temp = os.path.join(self.build_base, - 'temp' + plat_specifier) - if self.build_scripts is None: - self.build_scripts = os.path.join(self.build_base, - 'scripts-%d.%d' % sys.version_info[:2]) - - if self.executable is None: - self.executable = os.path.normpath(sys.executable) - - if isinstance(self.parallel, str): - try: - self.parallel = int(self.parallel) - except ValueError: - raise DistutilsOptionError("parallel should be an integer") - - def run(self): - # Run all relevant sub-commands. This will be some subset of: - # - build_py - pure Python modules - # - build_clib - standalone C libraries - # - build_ext - Python extensions - # - build_scripts - (Python) scripts - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - - # -- Predicates for the sub-command list --------------------------- - - def has_pure_modules(self): - return self.distribution.has_pure_modules() - - def has_c_libraries(self): - return self.distribution.has_c_libraries() - - def has_ext_modules(self): - return self.distribution.has_ext_modules() - - def has_scripts(self): - return self.distribution.has_scripts() - - - sub_commands = [('build_py', has_pure_modules), - ('build_clib', has_c_libraries), - ('build_ext', has_ext_modules), - ('build_scripts', has_scripts), - ] diff --git a/Lib/distutils/command/build_clib.py b/Lib/distutils/command/build_clib.py deleted file mode 100644 index 3e20ef23cd8..00000000000 --- a/Lib/distutils/command/build_clib.py +++ /dev/null @@ -1,209 +0,0 @@ -"""distutils.command.build_clib - -Implements the Distutils 'build_clib' command, to build a C/C++ library -that is included in the module distribution and needed by an extension -module.""" - - -# XXX this module has *lots* of code ripped-off quite transparently from -# build_ext.py -- not surprisingly really, as the work required to build -# a static library from a collection of C source files is not really all -# that different from what's required to build a shared object file from -# a collection of C source files. Nevertheless, I haven't done the -# necessary refactoring to account for the overlap in code between the -# two modules, mainly because a number of subtle details changed in the -# cut 'n paste. Sigh. - -import os -from distutils.core import Command -from distutils.errors import * -from distutils.sysconfig import customize_compiler -from distutils import log - -def show_compilers(): - from distutils.ccompiler import show_compilers - show_compilers() - - -class build_clib(Command): - - description = "build C/C++ libraries used by Python extensions" - - user_options = [ - ('build-clib=', 'b', - "directory to build C/C++ libraries to"), - ('build-temp=', 't', - "directory to put temporary build by-products"), - ('debug', 'g', - "compile with debugging information"), - ('force', 'f', - "forcibly build everything (ignore file timestamps)"), - ('compiler=', 'c', - "specify the compiler type"), - ] - - boolean_options = ['debug', 'force'] - - help_options = [ - ('help-compiler', None, - "list available compilers", show_compilers), - ] - - def initialize_options(self): - self.build_clib = None - self.build_temp = None - - # List of libraries to build - self.libraries = None - - # Compilation options for all libraries - self.include_dirs = None - self.define = None - self.undef = None - self.debug = None - self.force = 0 - self.compiler = None - - - def finalize_options(self): - # This might be confusing: both build-clib and build-temp default - # to build-temp as defined by the "build" command. This is because - # I think that C libraries are really just temporary build - # by-products, at least from the point of view of building Python - # extensions -- but I want to keep my options open. - self.set_undefined_options('build', - ('build_temp', 'build_clib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force')) - - self.libraries = self.distribution.libraries - if self.libraries: - self.check_library_list(self.libraries) - - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - if isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - # XXX same as for build_ext -- what about 'self.define' and - # 'self.undef' ? - - - def run(self): - if not self.libraries: - return - - # Yech -- this is cut 'n pasted from build_ext.py! - from distutils.ccompiler import new_compiler - self.compiler = new_compiler(compiler=self.compiler, - dry_run=self.dry_run, - force=self.force) - customize_compiler(self.compiler) - - if self.include_dirs is not None: - self.compiler.set_include_dirs(self.include_dirs) - if self.define is not None: - # 'define' option is a list of (name,value) tuples - for (name,value) in self.define: - self.compiler.define_macro(name, value) - if self.undef is not None: - for macro in self.undef: - self.compiler.undefine_macro(macro) - - self.build_libraries(self.libraries) - - - def check_library_list(self, libraries): - """Ensure that the list of libraries is valid. - - `library` is presumably provided as a command option 'libraries'. - This method checks that it is a list of 2-tuples, where the tuples - are (library_name, build_info_dict). - - Raise DistutilsSetupError if the structure is invalid anywhere; - just returns otherwise. - """ - if not isinstance(libraries, list): - raise DistutilsSetupError( - "'libraries' option must be a list of tuples") - - for lib in libraries: - if not isinstance(lib, tuple) and len(lib) != 2: - raise DistutilsSetupError( - "each element of 'libraries' must a 2-tuple") - - name, build_info = lib - - if not isinstance(name, str): - raise DistutilsSetupError( - "first element of each tuple in 'libraries' " - "must be a string (the library name)") - - if '/' in name or (os.sep != '/' and os.sep in name): - raise DistutilsSetupError("bad library name '%s': " - "may not contain directory separators" % lib[0]) - - if not isinstance(build_info, dict): - raise DistutilsSetupError( - "second element of each tuple in 'libraries' " - "must be a dictionary (build info)") - - - def get_library_names(self): - # Assume the library list is valid -- 'check_library_list()' is - # called from 'finalize_options()', so it should be! - if not self.libraries: - return None - - lib_names = [] - for (lib_name, build_info) in self.libraries: - lib_names.append(lib_name) - return lib_names - - - def get_source_files(self): - self.check_library_list(self.libraries) - filenames = [] - for (lib_name, build_info) in self.libraries: - sources = build_info.get('sources') - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'libraries' option (library '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % lib_name) - - filenames.extend(sources) - return filenames - - - def build_libraries(self, libraries): - for (lib_name, build_info) in libraries: - sources = build_info.get('sources') - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'libraries' option (library '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % lib_name) - sources = list(sources) - - log.info("building '%s' library", lib_name) - - # First, compile the source code to object files in the library - # directory. (This should probably change to putting object - # files in a temporary build directory.) - macros = build_info.get('macros') - include_dirs = build_info.get('include_dirs') - objects = self.compiler.compile(sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=include_dirs, - debug=self.debug) - - # Now "link" the object files together into a static library. - # (On Unix at least, this isn't really linking -- it just - # builds an archive. Whatever.) - self.compiler.create_static_lib(objects, lib_name, - output_dir=self.build_clib, - debug=self.debug) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py deleted file mode 100644 index acf2fc5484a..00000000000 --- a/Lib/distutils/command/build_ext.py +++ /dev/null @@ -1,755 +0,0 @@ -"""distutils.command.build_ext - -Implements the Distutils 'build_ext' command, for building extension -modules (currently limited to C extensions, should accommodate C++ -extensions ASAP).""" - -import contextlib -import os -import re -import sys -from distutils.core import Command -from distutils.errors import * -from distutils.sysconfig import customize_compiler, get_python_version -from distutils.sysconfig import get_config_h_filename -from distutils.dep_util import newer_group -from distutils.extension import Extension -from distutils.util import get_platform -from distutils import log - -from site import USER_BASE - -# An extension name is just a dot-separated list of Python NAMEs (ie. -# the same as a fully-qualified module name). -extension_name_re = re.compile \ - (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$') - - -def show_compilers (): - from distutils.ccompiler import show_compilers - show_compilers() - - -class build_ext(Command): - - description = "build C/C++ extensions (compile/link to build directory)" - - # XXX thoughts on how to deal with complex command-line options like - # these, i.e. how to make it so fancy_getopt can suck them off the - # command line and make it look like setup.py defined the appropriate - # lists of tuples of what-have-you. - # - each command needs a callback to process its command-line options - # - Command.__init__() needs access to its share of the whole - # command line (must ultimately come from - # Distribution.parse_command_line()) - # - it then calls the current command class' option-parsing - # callback to deal with weird options like -D, which have to - # parse the option text and churn out some custom data - # structure - # - that data structure (in this case, a list of 2-tuples) - # will then be present in the command object by the time - # we get to finalize_options() (i.e. the constructor - # takes care of both command-line and client options - # in between initialize_options() and finalize_options()) - - sep_by = " (separated by '%s')" % os.pathsep - user_options = [ - ('build-lib=', 'b', - "directory for compiled extension modules"), - ('build-temp=', 't', - "directory for temporary files (build by-products)"), - ('plat-name=', 'p', - "platform name to cross-compile for, if supported " - "(default: %s)" % get_platform()), - ('inplace', 'i', - "ignore build-lib and put compiled extensions into the source " + - "directory alongside your pure Python modules"), - ('include-dirs=', 'I', - "list of directories to search for header files" + sep_by), - ('define=', 'D', - "C preprocessor macros to define"), - ('undef=', 'U', - "C preprocessor macros to undefine"), - ('libraries=', 'l', - "external C libraries to link with"), - ('library-dirs=', 'L', - "directories to search for external C libraries" + sep_by), - ('rpath=', 'R', - "directories to search for shared C libraries at runtime"), - ('link-objects=', 'O', - "extra explicit link objects to include in the link"), - ('debug', 'g', - "compile/link with debugging information"), - ('force', 'f', - "forcibly build everything (ignore file timestamps)"), - ('compiler=', 'c', - "specify the compiler type"), - ('parallel=', 'j', - "number of parallel build jobs"), - ('swig-cpp', None, - "make SWIG create C++ files (default is C)"), - ('swig-opts=', None, - "list of SWIG command line options"), - ('swig=', None, - "path to the SWIG executable"), - ('user', None, - "add user include, library and rpath") - ] - - boolean_options = ['inplace', 'debug', 'force', 'swig-cpp', 'user'] - - help_options = [ - ('help-compiler', None, - "list available compilers", show_compilers), - ] - - def initialize_options(self): - self.extensions = None - self.build_lib = None - self.plat_name = None - self.build_temp = None - self.inplace = 0 - self.package = None - - self.include_dirs = None - self.define = None - self.undef = None - self.libraries = None - self.library_dirs = None - self.rpath = None - self.link_objects = None - self.debug = None - self.force = None - self.compiler = None - self.swig = None - self.swig_cpp = None - self.swig_opts = None - self.user = None - self.parallel = None - - def finalize_options(self): - from distutils import sysconfig - - self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force'), - ('parallel', 'parallel'), - ('plat_name', 'plat_name'), - ) - - if self.package is None: - self.package = self.distribution.ext_package - - self.extensions = self.distribution.ext_modules - - # Make sure Python's include directories (for Python.h, pyconfig.h, - # etc.) are in the include search path. - py_include = sysconfig.get_python_inc() - plat_py_include = sysconfig.get_python_inc(plat_specific=1) - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - if isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - # If in a virtualenv, add its include directory - # Issue 16116 - if sys.exec_prefix != sys.base_exec_prefix: - self.include_dirs.append(os.path.join(sys.exec_prefix, 'include')) - - # Put the Python "system" include dir at the end, so that - # any local include dirs take precedence. - self.include_dirs.append(py_include) - if plat_py_include != py_include: - self.include_dirs.append(plat_py_include) - - self.ensure_string_list('libraries') - self.ensure_string_list('link_objects') - - # Life is easier if we're not forever checking for None, so - # simplify these options to empty lists if unset - if self.libraries is None: - self.libraries = [] - if self.library_dirs is None: - self.library_dirs = [] - elif isinstance(self.library_dirs, str): - self.library_dirs = self.library_dirs.split(os.pathsep) - - if self.rpath is None: - self.rpath = [] - elif isinstance(self.rpath, str): - self.rpath = self.rpath.split(os.pathsep) - - # for extensions under windows use different directories - # for Release and Debug builds. - # also Python's library directory must be appended to library_dirs - if os.name == 'nt': - # the 'libs' directory is for binary installs - we assume that - # must be the *native* platform. But we don't really support - # cross-compiling via a binary install anyway, so we let it go. - self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) - if sys.base_exec_prefix != sys.prefix: # Issue 16116 - self.library_dirs.append(os.path.join(sys.base_exec_prefix, 'libs')) - if self.debug: - self.build_temp = os.path.join(self.build_temp, "Debug") - else: - self.build_temp = os.path.join(self.build_temp, "Release") - - # Append the source distribution include and library directories, - # this allows distutils on windows to work in the source tree - self.include_dirs.append(os.path.dirname(get_config_h_filename())) - _sys_home = getattr(sys, '_home', None) - if _sys_home: - self.library_dirs.append(_sys_home) - - # Use the .lib files for the correct architecture - if self.plat_name == 'win32': - suffix = 'win32' - else: - # win-amd64 or win-ia64 - suffix = self.plat_name[4:] - new_lib = os.path.join(sys.exec_prefix, 'PCbuild') - if suffix: - new_lib = os.path.join(new_lib, suffix) - self.library_dirs.append(new_lib) - - # for extensions under Cygwin and AtheOS Python's library directory must be - # appended to library_dirs - if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos': - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): - # building third party extensions - self.library_dirs.append(os.path.join(sys.prefix, "lib", - "python" + get_python_version(), - "config")) - else: - # building python standard extensions - self.library_dirs.append('.') - - # For building extensions with a shared Python library, - # Python's library directory must be appended to library_dirs - # See Issues: #1600860, #4366 - if False and (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if not sysconfig.python_build: - # building third party extensions - self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) - else: - # building python standard extensions - self.library_dirs.append('.') - - # The argument parsing will result in self.define being a string, but - # it has to be a list of 2-tuples. All the preprocessor symbols - # specified by the 'define' option will be set to '1'. Multiple - # symbols can be separated with commas. - - if self.define: - defines = self.define.split(',') - self.define = [(symbol, '1') for symbol in defines] - - # The option for macros to undefine is also a string from the - # option parsing, but has to be a list. Multiple symbols can also - # be separated with commas here. - if self.undef: - self.undef = self.undef.split(',') - - if self.swig_opts is None: - self.swig_opts = [] - else: - self.swig_opts = self.swig_opts.split(' ') - - # Finally add the user include and library directories if requested - if self.user: - user_include = os.path.join(USER_BASE, "include") - user_lib = os.path.join(USER_BASE, "lib") - if os.path.isdir(user_include): - self.include_dirs.append(user_include) - if os.path.isdir(user_lib): - self.library_dirs.append(user_lib) - self.rpath.append(user_lib) - - if isinstance(self.parallel, str): - try: - self.parallel = int(self.parallel) - except ValueError: - raise DistutilsOptionError("parallel should be an integer") - - def run(self): - from distutils.ccompiler import new_compiler - - # 'self.extensions', as supplied by setup.py, is a list of - # Extension instances. See the documentation for Extension (in - # distutils.extension) for details. - # - # For backwards compatibility with Distutils 0.8.2 and earlier, we - # also allow the 'extensions' list to be a list of tuples: - # (ext_name, build_info) - # where build_info is a dictionary containing everything that - # Extension instances do except the name, with a few things being - # differently named. We convert these 2-tuples to Extension - # instances as needed. - - if not self.extensions: - return - - # If we were asked to build any C/C++ libraries, make sure that the - # directory where we put them is in the library search path for - # linking extensions. - if self.distribution.has_c_libraries(): - build_clib = self.get_finalized_command('build_clib') - self.libraries.extend(build_clib.get_library_names() or []) - self.library_dirs.append(build_clib.build_clib) - - # Setup the CCompiler object that we'll use to do all the - # compiling and linking - self.compiler = new_compiler(compiler=self.compiler, - verbose=self.verbose, - dry_run=self.dry_run, - force=self.force) - customize_compiler(self.compiler) - # If we are cross-compiling, init the compiler now (if we are not - # cross-compiling, init would not hurt, but people may rely on - # late initialization of compiler even if they shouldn't...) - if os.name == 'nt' and self.plat_name != get_platform(): - self.compiler.initialize(self.plat_name) - - # And make sure that any compile/link-related options (which might - # come from the command-line or from the setup script) are set in - # that CCompiler object -- that way, they automatically apply to - # all compiling and linking done here. - if self.include_dirs is not None: - self.compiler.set_include_dirs(self.include_dirs) - if self.define is not None: - # 'define' option is a list of (name,value) tuples - for (name, value) in self.define: - self.compiler.define_macro(name, value) - if self.undef is not None: - for macro in self.undef: - self.compiler.undefine_macro(macro) - if self.libraries is not None: - self.compiler.set_libraries(self.libraries) - if self.library_dirs is not None: - self.compiler.set_library_dirs(self.library_dirs) - if self.rpath is not None: - self.compiler.set_runtime_library_dirs(self.rpath) - if self.link_objects is not None: - self.compiler.set_link_objects(self.link_objects) - - # Now actually compile and link everything. - self.build_extensions() - - def check_extensions_list(self, extensions): - """Ensure that the list of extensions (presumably provided as a - command option 'extensions') is valid, i.e. it is a list of - Extension objects. We also support the old-style list of 2-tuples, - where the tuples are (ext_name, build_info), which are converted to - Extension instances here. - - Raise DistutilsSetupError if the structure is invalid anywhere; - just returns otherwise. - """ - if not isinstance(extensions, list): - raise DistutilsSetupError( - "'ext_modules' option must be a list of Extension instances") - - for i, ext in enumerate(extensions): - if isinstance(ext, Extension): - continue # OK! (assume type-checking done - # by Extension constructor) - - if not isinstance(ext, tuple) or len(ext) != 2: - raise DistutilsSetupError( - "each element of 'ext_modules' option must be an " - "Extension instance or 2-tuple") - - ext_name, build_info = ext - - log.warn("old-style (ext_name, build_info) tuple found in " - "ext_modules for extension '%s' " - "-- please convert to Extension instance", ext_name) - - if not (isinstance(ext_name, str) and - extension_name_re.match(ext_name)): - raise DistutilsSetupError( - "first element of each tuple in 'ext_modules' " - "must be the extension name (a string)") - - if not isinstance(build_info, dict): - raise DistutilsSetupError( - "second element of each tuple in 'ext_modules' " - "must be a dictionary (build info)") - - # OK, the (ext_name, build_info) dict is type-safe: convert it - # to an Extension instance. - ext = Extension(ext_name, build_info['sources']) - - # Easy stuff: one-to-one mapping from dict elements to - # instance attributes. - for key in ('include_dirs', 'library_dirs', 'libraries', - 'extra_objects', 'extra_compile_args', - 'extra_link_args'): - val = build_info.get(key) - if val is not None: - setattr(ext, key, val) - - # Medium-easy stuff: same syntax/semantics, different names. - ext.runtime_library_dirs = build_info.get('rpath') - if 'def_file' in build_info: - log.warn("'def_file' element of build info dict " - "no longer supported") - - # Non-trivial stuff: 'macros' split into 'define_macros' - # and 'undef_macros'. - macros = build_info.get('macros') - if macros: - ext.define_macros = [] - ext.undef_macros = [] - for macro in macros: - if not (isinstance(macro, tuple) and len(macro) in (1, 2)): - raise DistutilsSetupError( - "'macros' element of build info dict " - "must be 1- or 2-tuple") - if len(macro) == 1: - ext.undef_macros.append(macro[0]) - elif len(macro) == 2: - ext.define_macros.append(macro) - - extensions[i] = ext - - def get_source_files(self): - self.check_extensions_list(self.extensions) - filenames = [] - - # Wouldn't it be neat if we knew the names of header files too... - for ext in self.extensions: - filenames.extend(ext.sources) - return filenames - - def get_outputs(self): - # Sanity check the 'extensions' list -- can't assume this is being - # done in the same run as a 'build_extensions()' call (in fact, we - # can probably assume that it *isn't*!). - self.check_extensions_list(self.extensions) - - # And build the list of output (built) filenames. Note that this - # ignores the 'inplace' flag, and assumes everything goes in the - # "build" tree. - outputs = [] - for ext in self.extensions: - outputs.append(self.get_ext_fullpath(ext.name)) - return outputs - - def build_extensions(self): - # First, sanity-check the 'extensions' list - self.check_extensions_list(self.extensions) - if self.parallel: - self._build_extensions_parallel() - else: - self._build_extensions_serial() - - def _build_extensions_parallel(self): - workers = self.parallel - if self.parallel is True: - workers = os.cpu_count() # may return None - try: - from concurrent.futures import ThreadPoolExecutor - except ImportError: - workers = None - - if workers is None: - self._build_extensions_serial() - return - - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = [executor.submit(self.build_extension, ext) - for ext in self.extensions] - for ext, fut in zip(self.extensions, futures): - with self._filter_build_errors(ext): - fut.result() - - def _build_extensions_serial(self): - for ext in self.extensions: - with self._filter_build_errors(ext): - self.build_extension(ext) - - @contextlib.contextmanager - def _filter_build_errors(self, ext): - try: - yield - except (CCompilerError, DistutilsError, CompileError) as e: - if not ext.optional: - raise - self.warn('building extension "%s" failed: %s' % - (ext.name, e)) - - def build_extension(self, ext): - sources = ext.sources - if sources is None or not isinstance(sources, (list, tuple)): - raise DistutilsSetupError( - "in 'ext_modules' option (extension '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % ext.name) - sources = list(sources) - - ext_path = self.get_ext_fullpath(ext.name) - depends = sources + ext.depends - if not (self.force or newer_group(depends, ext_path, 'newer')): - log.debug("skipping '%s' extension (up-to-date)", ext.name) - return - else: - log.info("building '%s' extension", ext.name) - - # First, scan the sources for SWIG definition files (.i), run - # SWIG on 'em to create .c files, and modify the sources list - # accordingly. - sources = self.swig_sources(sources, ext) - - # Next, compile the source code to object files. - - # XXX not honouring 'define_macros' or 'undef_macros' -- the - # CCompiler API needs to change to accommodate this, and I - # want to do one thing at a time! - - # Two possible sources for extra compiler arguments: - # - 'extra_compile_args' in Extension object - # - CFLAGS environment variable (not particularly - # elegant, but people seem to expect it and I - # guess it's useful) - # The environment variable should take precedence, and - # any sensible compiler will give precedence to later - # command line args. Hence we combine them in order: - extra_args = ext.extra_compile_args or [] - - macros = ext.define_macros[:] - for undef in ext.undef_macros: - macros.append((undef,)) - - objects = self.compiler.compile(sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=ext.include_dirs, - debug=self.debug, - extra_postargs=extra_args, - depends=ext.depends) - - # XXX outdated variable, kept here in case third-part code - # needs it. - self._built_objects = objects[:] - - # Now link the object files together into a "shared object" -- - # of course, first we have to figure out all the other things - # that go into the mix. - if ext.extra_objects: - objects.extend(ext.extra_objects) - extra_args = ext.extra_link_args or [] - - # Detect target language, if not provided - language = ext.language or self.compiler.detect_language(sources) - - self.compiler.link_shared_object( - objects, ext_path, - libraries=self.get_libraries(ext), - library_dirs=ext.library_dirs, - runtime_library_dirs=ext.runtime_library_dirs, - extra_postargs=extra_args, - export_symbols=self.get_export_symbols(ext), - debug=self.debug, - build_temp=self.build_temp, - target_lang=language) - - def swig_sources(self, sources, extension): - """Walk the list of source files in 'sources', looking for SWIG - interface (.i) files. Run SWIG on all that are found, and - return a modified 'sources' list with SWIG source files replaced - by the generated C (or C++) files. - """ - new_sources = [] - swig_sources = [] - swig_targets = {} - - # XXX this drops generated C/C++ files into the source tree, which - # is fine for developers who want to distribute the generated - # source -- but there should be an option to put SWIG output in - # the temp dir. - - if self.swig_cpp: - log.warn("--swig-cpp is deprecated - use --swig-opts=-c++") - - if self.swig_cpp or ('-c++' in self.swig_opts) or \ - ('-c++' in extension.swig_opts): - target_ext = '.cpp' - else: - target_ext = '.c' - - for source in sources: - (base, ext) = os.path.splitext(source) - if ext == ".i": # SWIG interface file - new_sources.append(base + '_wrap' + target_ext) - swig_sources.append(source) - swig_targets[source] = new_sources[-1] - else: - new_sources.append(source) - - if not swig_sources: - return new_sources - - swig = self.swig or self.find_swig() - swig_cmd = [swig, "-python"] - swig_cmd.extend(self.swig_opts) - if self.swig_cpp: - swig_cmd.append("-c++") - - # Do not override commandline arguments - if not self.swig_opts: - for o in extension.swig_opts: - swig_cmd.append(o) - - for source in swig_sources: - target = swig_targets[source] - log.info("swigging %s to %s", source, target) - self.spawn(swig_cmd + ["-o", target, source]) - - return new_sources - - def find_swig(self): - """Return the name of the SWIG executable. On Unix, this is - just "swig" -- it should be in the PATH. Tries a bit harder on - Windows. - """ - if os.name == "posix": - return "swig" - elif os.name == "nt": - # Look for SWIG in its standard installation directory on - # Windows (or so I presume!). If we find it there, great; - # if not, act like Unix and assume it's in the PATH. - for vers in ("1.3", "1.2", "1.1"): - fn = os.path.join("c:\\swig%s" % vers, "swig.exe") - if os.path.isfile(fn): - return fn - else: - return "swig.exe" - else: - raise DistutilsPlatformError( - "I don't know how to find (much less run) SWIG " - "on platform '%s'" % os.name) - - # -- Name generators ----------------------------------------------- - # (extension names, filenames, whatever) - def get_ext_fullpath(self, ext_name): - """Returns the path of the filename for a given extension. - - The file is located in `build_lib` or directly in the package - (inplace option). - """ - fullname = self.get_ext_fullname(ext_name) - modpath = fullname.split('.') - filename = self.get_ext_filename(modpath[-1]) - - if not self.inplace: - # no further work needed - # returning : - # build_dir/package/path/filename - filename = os.path.join(*modpath[:-1]+[filename]) - return os.path.join(self.build_lib, filename) - - # the inplace option requires to find the package directory - # using the build_py command for that - package = '.'.join(modpath[0:-1]) - build_py = self.get_finalized_command('build_py') - package_dir = os.path.abspath(build_py.get_package_dir(package)) - - # returning - # package_dir/filename - return os.path.join(package_dir, filename) - - def get_ext_fullname(self, ext_name): - """Returns the fullname of a given extension name. - - Adds the `package.` prefix""" - if self.package is None: - return ext_name - else: - return self.package + '.' + ext_name - - def get_ext_filename(self, ext_name): - r"""Convert the name of an extension (eg. "foo.bar") into the name - of the file from which it will be loaded (eg. "foo/bar.so", or - "foo\bar.pyd"). - """ - from distutils.sysconfig import get_config_var - ext_path = ext_name.split('.') - ext_suffix = get_config_var('EXT_SUFFIX') - return os.path.join(*ext_path) + ext_suffix - - def get_export_symbols(self, ext): - """Return the list of symbols that a shared extension has to - export. This either uses 'ext.export_symbols' or, if it's not - provided, "PyInit_" + module_name. Only relevant on Windows, where - the .pyd file (DLL) must export the module "PyInit_" function. - """ - initfunc_name = "PyInit_" + ext.name.split('.')[-1] - if initfunc_name not in ext.export_symbols: - ext.export_symbols.append(initfunc_name) - return ext.export_symbols - - def get_libraries(self, ext): - """Return the list of libraries to link against when building a - shared extension. On most platforms, this is just 'ext.libraries'; - on Windows, we add the Python library (eg. python20.dll). - """ - # The python library is always needed on Windows. For MSVC, this - # is redundant, since the library is mentioned in a pragma in - # pyconfig.h that MSVC groks. The other Windows compilers all seem - # to need it mentioned explicitly, though, so that's what we do. - # Append '_d' to the python import library on debug builds. - if sys.platform == "win32": - from distutils._msvccompiler import MSVCCompiler - if not isinstance(self.compiler, MSVCCompiler): - template = "python%d%d" - if self.debug: - template = template + '_d' - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - else: - return ext.libraries - elif sys.platform[:6] == "cygwin": - template = "python%d.%d" - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - elif sys.platform[:6] == "atheos": - from distutils import sysconfig - - template = "python%d.%d" - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # Get SHLIBS from Makefile - extra = [] - for lib in sysconfig.get_config_var('SHLIBS').split(): - if lib.startswith('-l'): - extra.append(lib[2:]) - else: - extra.append(lib) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib, "m"] + extra - elif sys.platform == 'darwin': - # Don't use the default code below - return ext.libraries - elif sys.platform[:3] == 'aix': - # Don't use the default code below - return ext.libraries - else: - from distutils import sysconfig - if False and sysconfig.get_config_var('Py_ENABLE_SHARED'): - pythonlib = 'python{}.{}{}'.format( - sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff, - sysconfig.get_config_var('ABIFLAGS')) - return ext.libraries + [pythonlib] - else: - return ext.libraries diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py deleted file mode 100644 index cf0ca57c320..00000000000 --- a/Lib/distutils/command/build_py.py +++ /dev/null @@ -1,416 +0,0 @@ -"""distutils.command.build_py - -Implements the Distutils 'build_py' command.""" - -import os -import importlib.util -import sys -from glob import glob - -from distutils.core import Command -from distutils.errors import * -from distutils.util import convert_path, Mixin2to3 -from distutils import log - -class build_py (Command): - - description = "\"build\" pure Python modules (copy to build directory)" - - user_options = [ - ('build-lib=', 'd', "directory to \"build\" (copy) to"), - ('compile', 'c', "compile .py to .pyc"), - ('no-compile', None, "don't compile .py files [default]"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - ('force', 'f', "forcibly build everything (ignore file timestamps)"), - ] - - boolean_options = ['compile', 'force'] - negative_opt = {'no-compile' : 'compile'} - - def initialize_options(self): - self.build_lib = None - self.py_modules = None - self.package = None - self.package_data = None - self.package_dir = None - self.compile = 0 - self.optimize = 0 - self.force = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_lib', 'build_lib'), - ('force', 'force')) - - # Get the distribution options that are aliases for build_py - # options -- list of packages and list of modules. - self.packages = self.distribution.packages - self.py_modules = self.distribution.py_modules - self.package_data = self.distribution.package_data - self.package_dir = {} - if self.distribution.package_dir: - for name, path in self.distribution.package_dir.items(): - self.package_dir[name] = convert_path(path) - self.data_files = self.get_data_files() - - # Ick, copied straight from install_lib.py (fancy_getopt needs a - # type system! Hell, *everything* needs a type system!!!) - if not isinstance(self.optimize, int): - try: - self.optimize = int(self.optimize) - assert 0 <= self.optimize <= 2 - except (ValueError, AssertionError): - raise DistutilsOptionError("optimize must be 0, 1, or 2") - - def run(self): - # XXX copy_file by default preserves atime and mtime. IMHO this is - # the right thing to do, but perhaps it should be an option -- in - # particular, a site administrator might want installed files to - # reflect the time of installation rather than the last - # modification time before the installed release. - - # XXX copy_file by default preserves mode, which appears to be the - # wrong thing to do: if a file is read-only in the working - # directory, we want it to be installed read/write so that the next - # installation of the same module distribution can overwrite it - # without problems. (This might be a Unix-specific issue.) Thus - # we turn off 'preserve_mode' when copying to the build directory, - # since the build directory is supposed to be exactly what the - # installation will look like (ie. we preserve mode when - # installing). - - # Two options control which modules will be installed: 'packages' - # and 'py_modules'. The former lets us work with whole packages, not - # specifying individual modules at all; the latter is for - # specifying modules one-at-a-time. - - if self.py_modules: - self.build_modules() - if self.packages: - self.build_packages() - self.build_package_data() - - self.byte_compile(self.get_outputs(include_bytecode=0)) - - def get_data_files(self): - """Generate list of '(package,src_dir,build_dir,filenames)' tuples""" - data = [] - if not self.packages: - return data - for package in self.packages: - # Locate package source directory - src_dir = self.get_package_dir(package) - - # Compute package build directory - build_dir = os.path.join(*([self.build_lib] + package.split('.'))) - - # Length of path to strip from found files - plen = 0 - if src_dir: - plen = len(src_dir)+1 - - # Strip directory from globbed filenames - filenames = [ - file[plen:] for file in self.find_data_files(package, src_dir) - ] - data.append((package, src_dir, build_dir, filenames)) - return data - - def find_data_files(self, package, src_dir): - """Return filenames for package's data files in 'src_dir'""" - globs = (self.package_data.get('', []) - + self.package_data.get(package, [])) - files = [] - for pattern in globs: - # Each pattern has to be converted to a platform-specific path - filelist = glob(os.path.join(src_dir, convert_path(pattern))) - # Files that match more than one pattern are only added once - files.extend([fn for fn in filelist if fn not in files - and os.path.isfile(fn)]) - return files - - def build_package_data(self): - """Copy data files into build directory""" - lastdir = None - for package, src_dir, build_dir, filenames in self.data_files: - for filename in filenames: - target = os.path.join(build_dir, filename) - self.mkpath(os.path.dirname(target)) - self.copy_file(os.path.join(src_dir, filename), target, - preserve_mode=False) - - def get_package_dir(self, package): - """Return the directory, relative to the top of the source - distribution, where package 'package' should be found - (at least according to the 'package_dir' option, if any).""" - path = package.split('.') - - if not self.package_dir: - if path: - return os.path.join(*path) - else: - return '' - else: - tail = [] - while path: - try: - pdir = self.package_dir['.'.join(path)] - except KeyError: - tail.insert(0, path[-1]) - del path[-1] - else: - tail.insert(0, pdir) - return os.path.join(*tail) - else: - # Oops, got all the way through 'path' without finding a - # match in package_dir. If package_dir defines a directory - # for the root (nameless) package, then fallback on it; - # otherwise, we might as well have not consulted - # package_dir at all, as we just use the directory implied - # by 'tail' (which should be the same as the original value - # of 'path' at this point). - pdir = self.package_dir.get('') - if pdir is not None: - tail.insert(0, pdir) - - if tail: - return os.path.join(*tail) - else: - return '' - - def check_package(self, package, package_dir): - # Empty dir name means current directory, which we can probably - # assume exists. Also, os.path.exists and isdir don't know about - # my "empty string means current dir" convention, so we have to - # circumvent them. - if package_dir != "": - if not os.path.exists(package_dir): - raise DistutilsFileError( - "package directory '%s' does not exist" % package_dir) - if not os.path.isdir(package_dir): - raise DistutilsFileError( - "supposed package directory '%s' exists, " - "but is not a directory" % package_dir) - - # Require __init__.py for all but the "root package" - if package: - init_py = os.path.join(package_dir, "__init__.py") - if os.path.isfile(init_py): - return init_py - else: - log.warn(("package init file '%s' not found " + - "(or not a regular file)"), init_py) - - # Either not in a package at all (__init__.py not expected), or - # __init__.py doesn't exist -- so don't return the filename. - return None - - def check_module(self, module, module_file): - if not os.path.isfile(module_file): - log.warn("file %s (for module %s) not found", module_file, module) - return False - else: - return True - - def find_package_modules(self, package, package_dir): - self.check_package(package, package_dir) - module_files = glob(os.path.join(package_dir, "*.py")) - modules = [] - setup_script = os.path.abspath(self.distribution.script_name) - - for f in module_files: - abs_f = os.path.abspath(f) - if abs_f != setup_script: - module = os.path.splitext(os.path.basename(f))[0] - modules.append((package, module, f)) - else: - self.debug_print("excluding %s" % setup_script) - return modules - - def find_modules(self): - """Finds individually-specified Python modules, ie. those listed by - module name in 'self.py_modules'. Returns a list of tuples (package, - module_base, filename): 'package' is a tuple of the path through - package-space to the module; 'module_base' is the bare (no - packages, no dots) module name, and 'filename' is the path to the - ".py" file (relative to the distribution root) that implements the - module. - """ - # Map package names to tuples of useful info about the package: - # (package_dir, checked) - # package_dir - the directory where we'll find source files for - # this package - # checked - true if we have checked that the package directory - # is valid (exists, contains __init__.py, ... ?) - packages = {} - - # List of (package, module, filename) tuples to return - modules = [] - - # We treat modules-in-packages almost the same as toplevel modules, - # just the "package" for a toplevel is empty (either an empty - # string or empty list, depending on context). Differences: - # - don't check for __init__.py in directory for empty package - for module in self.py_modules: - path = module.split('.') - package = '.'.join(path[0:-1]) - module_base = path[-1] - - try: - (package_dir, checked) = packages[package] - except KeyError: - package_dir = self.get_package_dir(package) - checked = 0 - - if not checked: - init_py = self.check_package(package, package_dir) - packages[package] = (package_dir, 1) - if init_py: - modules.append((package, "__init__", init_py)) - - # XXX perhaps we should also check for just .pyc files - # (so greedy closed-source bastards can distribute Python - # modules too) - module_file = os.path.join(package_dir, module_base + ".py") - if not self.check_module(module, module_file): - continue - - modules.append((package, module_base, module_file)) - - return modules - - def find_all_modules(self): - """Compute the list of all modules that will be built, whether - they are specified one-module-at-a-time ('self.py_modules') or - by whole packages ('self.packages'). Return a list of tuples - (package, module, module_file), just like 'find_modules()' and - 'find_package_modules()' do.""" - modules = [] - if self.py_modules: - modules.extend(self.find_modules()) - if self.packages: - for package in self.packages: - package_dir = self.get_package_dir(package) - m = self.find_package_modules(package, package_dir) - modules.extend(m) - return modules - - def get_source_files(self): - return [module[-1] for module in self.find_all_modules()] - - def get_module_outfile(self, build_dir, package, module): - outfile_path = [build_dir] + list(package) + [module + ".py"] - return os.path.join(*outfile_path) - - def get_outputs(self, include_bytecode=1): - modules = self.find_all_modules() - outputs = [] - for (package, module, module_file) in modules: - package = package.split('.') - filename = self.get_module_outfile(self.build_lib, package, module) - outputs.append(filename) - if include_bytecode: - if self.compile: - outputs.append(importlib.util.cache_from_source( - filename, optimization='')) - if self.optimize > 0: - outputs.append(importlib.util.cache_from_source( - filename, optimization=self.optimize)) - - outputs += [ - os.path.join(build_dir, filename) - for package, src_dir, build_dir, filenames in self.data_files - for filename in filenames - ] - - return outputs - - def build_module(self, module, module_file, package): - if isinstance(package, str): - package = package.split('.') - elif not isinstance(package, (list, tuple)): - raise TypeError( - "'package' must be a string (dot-separated), list, or tuple") - - # Now put the module source file into the "build" area -- this is - # easy, we just copy it somewhere under self.build_lib (the build - # directory for Python source). - outfile = self.get_module_outfile(self.build_lib, package, module) - dir = os.path.dirname(outfile) - self.mkpath(dir) - return self.copy_file(module_file, outfile, preserve_mode=0) - - def build_modules(self): - modules = self.find_modules() - for (package, module, module_file) in modules: - # Now "build" the module -- ie. copy the source file to - # self.build_lib (the build directory for Python source). - # (Actually, it gets copied to the directory for this package - # under self.build_lib.) - self.build_module(module, module_file, package) - - def build_packages(self): - for package in self.packages: - # Get list of (package, module, module_file) tuples based on - # scanning the package directory. 'package' is only included - # in the tuple so that 'find_modules()' and - # 'find_package_tuples()' have a consistent interface; it's - # ignored here (apart from a sanity check). Also, 'module' is - # the *unqualified* module name (ie. no dots, no package -- we - # already know its package!), and 'module_file' is the path to - # the .py file, relative to the current directory - # (ie. including 'package_dir'). - package_dir = self.get_package_dir(package) - modules = self.find_package_modules(package, package_dir) - - # Now loop over the modules we found, "building" each one (just - # copy it to self.build_lib). - for (package_, module, module_file) in modules: - assert package == package_ - self.build_module(module, module_file, package) - - def byte_compile(self, files): - if sys.dont_write_bytecode: - self.warn('byte-compiling is disabled, skipping.') - return - - from distutils.util import byte_compile - prefix = self.build_lib - if prefix[-1] != os.sep: - prefix = prefix + os.sep - - # XXX this code is essentially the same as the 'byte_compile() - # method of the "install_lib" command, except for the determination - # of the 'prefix' string. Hmmm. - if self.compile: - byte_compile(files, optimize=0, - force=self.force, prefix=prefix, dry_run=self.dry_run) - if self.optimize > 0: - byte_compile(files, optimize=self.optimize, - force=self.force, prefix=prefix, dry_run=self.dry_run) - -class build_py_2to3(build_py, Mixin2to3): - def run(self): - self.updated_files = [] - - # Base class code - if self.py_modules: - self.build_modules() - if self.packages: - self.build_packages() - self.build_package_data() - - # 2to3 - self.run_2to3(self.updated_files) - - # Remaining base class code - self.byte_compile(self.get_outputs(include_bytecode=0)) - - def build_module(self, module, module_file, package): - res = build_py.build_module(self, module, module_file, package) - if res[1]: - # file was copied - self.updated_files.append(res[0]) - return res diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py deleted file mode 100644 index ccc70e64650..00000000000 --- a/Lib/distutils/command/build_scripts.py +++ /dev/null @@ -1,160 +0,0 @@ -"""distutils.command.build_scripts - -Implements the Distutils 'build_scripts' command.""" - -import os, re -from stat import ST_MODE -from distutils import sysconfig -from distutils.core import Command -from distutils.dep_util import newer -from distutils.util import convert_path, Mixin2to3 -from distutils import log -import tokenize - -# check if Python is called on the first line with this expression -first_line_re = re.compile(b'^#!.*python[0-9.]*([ \t].*)?$') - -class build_scripts(Command): - - description = "\"build\" scripts (copy and fixup #! line)" - - user_options = [ - ('build-dir=', 'd', "directory to \"build\" (copy) to"), - ('force', 'f', "forcibly build everything (ignore file timestamps"), - ('executable=', 'e', "specify final destination interpreter path"), - ] - - boolean_options = ['force'] - - - def initialize_options(self): - self.build_dir = None - self.scripts = None - self.force = None - self.executable = None - self.outfiles = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_scripts', 'build_dir'), - ('force', 'force'), - ('executable', 'executable')) - self.scripts = self.distribution.scripts - - def get_source_files(self): - return self.scripts - - def run(self): - if not self.scripts: - return - self.copy_scripts() - - - def copy_scripts(self): - r"""Copy each script listed in 'self.scripts'; if it's marked as a - Python script in the Unix way (first line matches 'first_line_re', - ie. starts with "\#!" and contains "python"), then adjust the first - line to refer to the current Python interpreter as we copy. - """ - self.mkpath(self.build_dir) - outfiles = [] - updated_files = [] - for script in self.scripts: - adjust = False - script = convert_path(script) - outfile = os.path.join(self.build_dir, os.path.basename(script)) - outfiles.append(outfile) - - if not self.force and not newer(script, outfile): - log.debug("not copying %s (up-to-date)", script) - continue - - # Always open the file, but ignore failures in dry-run mode -- - # that way, we'll get accurate feedback if we can read the - # script. - try: - f = open(script, "rb") - except OSError: - if not self.dry_run: - raise - f = None - else: - encoding, lines = tokenize.detect_encoding(f.readline) - f.seek(0) - first_line = f.readline() - if not first_line: - self.warn("%s is an empty file (skipping)" % script) - continue - - match = first_line_re.match(first_line) - if match: - adjust = True - post_interp = match.group(1) or b'' - - if adjust: - log.info("copying and adjusting %s -> %s", script, - self.build_dir) - updated_files.append(outfile) - if not self.dry_run: - if not sysconfig.python_build: - executable = self.executable - else: - executable = os.path.join( - sysconfig.get_config_var("BINDIR"), - "python%s%s" % (sysconfig.get_config_var("VERSION"), - sysconfig.get_config_var("EXE"))) - executable = os.fsencode(executable) - shebang = b"#!" + executable + post_interp + b"\n" - # Python parser starts to read a script using UTF-8 until - # it gets a #coding:xxx cookie. The shebang has to be the - # first line of a file, the #coding:xxx cookie cannot be - # written before. So the shebang has to be decodable from - # UTF-8. - try: - shebang.decode('utf-8') - except UnicodeDecodeError: - raise ValueError( - "The shebang ({!r}) is not decodable " - "from utf-8".format(shebang)) - # If the script is encoded to a custom encoding (use a - # #coding:xxx cookie), the shebang has to be decodable from - # the script encoding too. - try: - shebang.decode(encoding) - except UnicodeDecodeError: - raise ValueError( - "The shebang ({!r}) is not decodable " - "from the script encoding ({})" - .format(shebang, encoding)) - with open(outfile, "wb") as outf: - outf.write(shebang) - outf.writelines(f.readlines()) - if f: - f.close() - else: - if f: - f.close() - updated_files.append(outfile) - self.copy_file(script, outfile) - - if os.name == 'posix': - for file in outfiles: - if self.dry_run: - log.info("changing mode of %s", file) - else: - oldmode = os.stat(file)[ST_MODE] & 0o7777 - newmode = (oldmode | 0o555) & 0o7777 - if newmode != oldmode: - log.info("changing mode of %s from %o to %o", - file, oldmode, newmode) - os.chmod(file, newmode) - # XXX should we modify self.outfiles? - return outfiles, updated_files - -class build_scripts_2to3(build_scripts, Mixin2to3): - - def copy_scripts(self): - outfiles, updated_files = build_scripts.copy_scripts(self) - if not self.dry_run: - self.run_2to3(updated_files) - return outfiles, updated_files diff --git a/Lib/distutils/command/check.py b/Lib/distutils/command/check.py deleted file mode 100644 index 7ebe707cff4..00000000000 --- a/Lib/distutils/command/check.py +++ /dev/null @@ -1,145 +0,0 @@ -"""distutils.command.check - -Implements the Distutils 'check' command. -""" -from distutils.core import Command -from distutils.errors import DistutilsSetupError - -try: - # docutils is installed - from docutils.utils import Reporter - from docutils.parsers.rst import Parser - from docutils import frontend - from docutils import nodes - from io import StringIO - - class SilentReporter(Reporter): - - def __init__(self, source, report_level, halt_level, stream=None, - debug=0, encoding='ascii', error_handler='replace'): - self.messages = [] - Reporter.__init__(self, source, report_level, halt_level, stream, - debug, encoding, error_handler) - - def system_message(self, level, message, *children, **kwargs): - self.messages.append((level, message, children, kwargs)) - return nodes.system_message(message, level=level, - type=self.levels[level], - *children, **kwargs) - - HAS_DOCUTILS = True -except Exception: - # Catch all exceptions because exceptions besides ImportError probably - # indicate that docutils is not ported to Py3k. - HAS_DOCUTILS = False - -class check(Command): - """This command checks the meta-data of the package. - """ - description = ("perform some checks on the package") - user_options = [('metadata', 'm', 'Verify meta-data'), - ('restructuredtext', 'r', - ('Checks if long string meta-data syntax ' - 'are reStructuredText-compliant')), - ('strict', 's', - 'Will exit with an error if a check fails')] - - boolean_options = ['metadata', 'restructuredtext', 'strict'] - - def initialize_options(self): - """Sets default values for options.""" - self.restructuredtext = 0 - self.metadata = 1 - self.strict = 0 - self._warnings = 0 - - def finalize_options(self): - pass - - def warn(self, msg): - """Counts the number of warnings that occurs.""" - self._warnings += 1 - return Command.warn(self, msg) - - def run(self): - """Runs the command.""" - # perform the various tests - if self.metadata: - self.check_metadata() - if self.restructuredtext: - if HAS_DOCUTILS: - self.check_restructuredtext() - elif self.strict: - raise DistutilsSetupError('The docutils package is needed.') - - # let's raise an error in strict mode, if we have at least - # one warning - if self.strict and self._warnings > 0: - raise DistutilsSetupError('Please correct your package.') - - def check_metadata(self): - """Ensures that all required elements of meta-data are supplied. - - name, version, URL, (author and author_email) or - (maintainer and maintainer_email)). - - Warns if any are missing. - """ - metadata = self.distribution.metadata - - missing = [] - for attr in ('name', 'version', 'url'): - if not (hasattr(metadata, attr) and getattr(metadata, attr)): - missing.append(attr) - - if missing: - self.warn("missing required meta-data: %s" % ', '.join(missing)) - if metadata.author: - if not metadata.author_email: - self.warn("missing meta-data: if 'author' supplied, " + - "'author_email' must be supplied too") - elif metadata.maintainer: - if not metadata.maintainer_email: - self.warn("missing meta-data: if 'maintainer' supplied, " + - "'maintainer_email' must be supplied too") - else: - self.warn("missing meta-data: either (author and author_email) " + - "or (maintainer and maintainer_email) " + - "must be supplied") - - def check_restructuredtext(self): - """Checks if the long string fields are reST-compliant.""" - data = self.distribution.get_long_description() - for warning in self._check_rst_data(data): - line = warning[-1].get('line') - if line is None: - warning = warning[1] - else: - warning = '%s (line %s)' % (warning[1], line) - self.warn(warning) - - def _check_rst_data(self, data): - """Returns warnings when the provided data doesn't compile.""" - source_path = StringIO() - parser = Parser() - settings = frontend.OptionParser(components=(Parser,)).get_default_values() - settings.tab_width = 4 - settings.pep_references = None - settings.rfc_references = None - reporter = SilentReporter(source_path, - settings.report_level, - settings.halt_level, - stream=settings.warning_stream, - debug=settings.debug, - encoding=settings.error_encoding, - error_handler=settings.error_encoding_error_handler) - - document = nodes.document(settings, reporter, source=source_path) - document.note_source(source_path, -1) - try: - parser.parse(data, document) - except AttributeError as e: - reporter.messages.append( - (-1, 'Could not finish the parsing: %s.' % e, '', {})) - - return reporter.messages diff --git a/Lib/distutils/command/clean.py b/Lib/distutils/command/clean.py deleted file mode 100644 index 0cb27016621..00000000000 --- a/Lib/distutils/command/clean.py +++ /dev/null @@ -1,76 +0,0 @@ -"""distutils.command.clean - -Implements the Distutils 'clean' command.""" - -# contributed by Bastian Kleineidam , added 2000-03-18 - -import os -from distutils.core import Command -from distutils.dir_util import remove_tree -from distutils import log - -class clean(Command): - - description = "clean up temporary files from 'build' command" - user_options = [ - ('build-base=', 'b', - "base build directory (default: 'build.build-base')"), - ('build-lib=', None, - "build directory for all modules (default: 'build.build-lib')"), - ('build-temp=', 't', - "temporary build directory (default: 'build.build-temp')"), - ('build-scripts=', None, - "build directory for scripts (default: 'build.build-scripts')"), - ('bdist-base=', None, - "temporary directory for built distributions"), - ('all', 'a', - "remove all build output, not just temporary by-products") - ] - - boolean_options = ['all'] - - def initialize_options(self): - self.build_base = None - self.build_lib = None - self.build_temp = None - self.build_scripts = None - self.bdist_base = None - self.all = None - - def finalize_options(self): - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib'), - ('build_scripts', 'build_scripts'), - ('build_temp', 'build_temp')) - self.set_undefined_options('bdist', - ('bdist_base', 'bdist_base')) - - def run(self): - # remove the build/temp. directory (unless it's already - # gone) - if os.path.exists(self.build_temp): - remove_tree(self.build_temp, dry_run=self.dry_run) - else: - log.debug("'%s' does not exist -- can't clean it", - self.build_temp) - - if self.all: - # remove build directories - for directory in (self.build_lib, - self.bdist_base, - self.build_scripts): - if os.path.exists(directory): - remove_tree(directory, dry_run=self.dry_run) - else: - log.warn("'%s' does not exist -- can't clean it", - directory) - - # just for the heck of it, try to remove the base build directory: - # we might have emptied it right now, but if not we don't care - if not self.dry_run: - try: - os.rmdir(self.build_base) - log.info("removing '%s'", self.build_base) - except OSError: - pass diff --git a/Lib/distutils/command/command_template b/Lib/distutils/command/command_template deleted file mode 100644 index 6106819db84..00000000000 --- a/Lib/distutils/command/command_template +++ /dev/null @@ -1,33 +0,0 @@ -"""distutils.command.x - -Implements the Distutils 'x' command. -""" - -# created 2000/mm/dd, John Doe - -__revision__ = "$Id$" - -from distutils.core import Command - - -class x(Command): - - # Brief (40-50 characters) description of the command - description = "" - - # List of option tuples: long name, short name (None if no short - # name), and help string. - user_options = [('', '', - ""), - ] - - def initialize_options(self): - self. = None - self. = None - self. = None - - def finalize_options(self): - if self.x is None: - self.x = - - def run(self): diff --git a/Lib/distutils/command/config.py b/Lib/distutils/command/config.py deleted file mode 100644 index 4ae153d1943..00000000000 --- a/Lib/distutils/command/config.py +++ /dev/null @@ -1,347 +0,0 @@ -"""distutils.command.config - -Implements the Distutils 'config' command, a (mostly) empty command class -that exists mainly to be sub-classed by specific module distributions and -applications. The idea is that while every "config" command is different, -at least they're all named the same, and users always see "config" in the -list of standard commands. Also, this is a good place to put common -configure-like tasks: "try to compile this C code", or "figure out where -this header file lives". -""" - -import os, re - -from distutils.core import Command -from distutils.errors import DistutilsExecError -from distutils.sysconfig import customize_compiler -from distutils import log - -LANG_EXT = {"c": ".c", "c++": ".cxx"} - -class config(Command): - - description = "prepare to build" - - user_options = [ - ('compiler=', None, - "specify the compiler type"), - ('cc=', None, - "specify the compiler executable"), - ('include-dirs=', 'I', - "list of directories to search for header files"), - ('define=', 'D', - "C preprocessor macros to define"), - ('undef=', 'U', - "C preprocessor macros to undefine"), - ('libraries=', 'l', - "external C libraries to link with"), - ('library-dirs=', 'L', - "directories to search for external C libraries"), - - ('noisy', None, - "show every action (compile, link, run, ...) taken"), - ('dump-source', None, - "dump generated source files before attempting to compile them"), - ] - - - # The three standard command methods: since the "config" command - # does nothing by default, these are empty. - - def initialize_options(self): - self.compiler = None - self.cc = None - self.include_dirs = None - self.libraries = None - self.library_dirs = None - - # maximal output for now - self.noisy = 1 - self.dump_source = 1 - - # list of temporary files generated along-the-way that we have - # to clean at some point - self.temp_files = [] - - def finalize_options(self): - if self.include_dirs is None: - self.include_dirs = self.distribution.include_dirs or [] - elif isinstance(self.include_dirs, str): - self.include_dirs = self.include_dirs.split(os.pathsep) - - if self.libraries is None: - self.libraries = [] - elif isinstance(self.libraries, str): - self.libraries = [self.libraries] - - if self.library_dirs is None: - self.library_dirs = [] - elif isinstance(self.library_dirs, str): - self.library_dirs = self.library_dirs.split(os.pathsep) - - def run(self): - pass - - # Utility methods for actual "config" commands. The interfaces are - # loosely based on Autoconf macros of similar names. Sub-classes - # may use these freely. - - def _check_compiler(self): - """Check that 'self.compiler' really is a CCompiler object; - if not, make it one. - """ - # We do this late, and only on-demand, because this is an expensive - # import. - from distutils.ccompiler import CCompiler, new_compiler - if not isinstance(self.compiler, CCompiler): - self.compiler = new_compiler(compiler=self.compiler, - dry_run=self.dry_run, force=1) - customize_compiler(self.compiler) - if self.include_dirs: - self.compiler.set_include_dirs(self.include_dirs) - if self.libraries: - self.compiler.set_libraries(self.libraries) - if self.library_dirs: - self.compiler.set_library_dirs(self.library_dirs) - - def _gen_temp_sourcefile(self, body, headers, lang): - filename = "_configtest" + LANG_EXT[lang] - file = open(filename, "w") - if headers: - for header in headers: - file.write("#include <%s>\n" % header) - file.write("\n") - file.write(body) - if body[-1] != "\n": - file.write("\n") - file.close() - return filename - - def _preprocess(self, body, headers, include_dirs, lang): - src = self._gen_temp_sourcefile(body, headers, lang) - out = "_configtest.i" - self.temp_files.extend([src, out]) - self.compiler.preprocess(src, out, include_dirs=include_dirs) - return (src, out) - - def _compile(self, body, headers, include_dirs, lang): - src = self._gen_temp_sourcefile(body, headers, lang) - if self.dump_source: - dump_file(src, "compiling '%s':" % src) - (obj,) = self.compiler.object_filenames([src]) - self.temp_files.extend([src, obj]) - self.compiler.compile([src], include_dirs=include_dirs) - return (src, obj) - - def _link(self, body, headers, include_dirs, libraries, library_dirs, - lang): - (src, obj) = self._compile(body, headers, include_dirs, lang) - prog = os.path.splitext(os.path.basename(src))[0] - self.compiler.link_executable([obj], prog, - libraries=libraries, - library_dirs=library_dirs, - target_lang=lang) - - if self.compiler.exe_extension is not None: - prog = prog + self.compiler.exe_extension - self.temp_files.append(prog) - - return (src, obj, prog) - - def _clean(self, *filenames): - if not filenames: - filenames = self.temp_files - self.temp_files = [] - log.info("removing: %s", ' '.join(filenames)) - for filename in filenames: - try: - os.remove(filename) - except OSError: - pass - - - # XXX these ignore the dry-run flag: what to do, what to do? even if - # you want a dry-run build, you still need some sort of configuration - # info. My inclination is to make it up to the real config command to - # consult 'dry_run', and assume a default (minimal) configuration if - # true. The problem with trying to do it here is that you'd have to - # return either true or false from all the 'try' methods, neither of - # which is correct. - - # XXX need access to the header search path and maybe default macros. - - def try_cpp(self, body=None, headers=None, include_dirs=None, lang="c"): - """Construct a source file from 'body' (a string containing lines - of C/C++ code) and 'headers' (a list of header files to include) - and run it through the preprocessor. Return true if the - preprocessor succeeded, false if there were any errors. - ('body' probably isn't of much use, but what the heck.) - """ - from distutils.ccompiler import CompileError - self._check_compiler() - ok = True - try: - self._preprocess(body, headers, include_dirs, lang) - except CompileError: - ok = False - - self._clean() - return ok - - def search_cpp(self, pattern, body=None, headers=None, include_dirs=None, - lang="c"): - """Construct a source file (just like 'try_cpp()'), run it through - the preprocessor, and return true if any line of the output matches - 'pattern'. 'pattern' should either be a compiled regex object or a - string containing a regex. If both 'body' and 'headers' are None, - preprocesses an empty file -- which can be useful to determine the - symbols the preprocessor and compiler set by default. - """ - self._check_compiler() - src, out = self._preprocess(body, headers, include_dirs, lang) - - if isinstance(pattern, str): - pattern = re.compile(pattern) - - file = open(out) - match = False - while True: - line = file.readline() - if line == '': - break - if pattern.search(line): - match = True - break - - file.close() - self._clean() - return match - - def try_compile(self, body, headers=None, include_dirs=None, lang="c"): - """Try to compile a source file built from 'body' and 'headers'. - Return true on success, false otherwise. - """ - from distutils.ccompiler import CompileError - self._check_compiler() - try: - self._compile(body, headers, include_dirs, lang) - ok = True - except CompileError: - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - def try_link(self, body, headers=None, include_dirs=None, libraries=None, - library_dirs=None, lang="c"): - """Try to compile and link a source file, built from 'body' and - 'headers', to executable form. Return true on success, false - otherwise. - """ - from distutils.ccompiler import CompileError, LinkError - self._check_compiler() - try: - self._link(body, headers, include_dirs, - libraries, library_dirs, lang) - ok = True - except (CompileError, LinkError): - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - def try_run(self, body, headers=None, include_dirs=None, libraries=None, - library_dirs=None, lang="c"): - """Try to compile, link to an executable, and run a program - built from 'body' and 'headers'. Return true on success, false - otherwise. - """ - from distutils.ccompiler import CompileError, LinkError - self._check_compiler() - try: - src, obj, exe = self._link(body, headers, include_dirs, - libraries, library_dirs, lang) - self.spawn([exe]) - ok = True - except (CompileError, LinkError, DistutilsExecError): - ok = False - - log.info(ok and "success!" or "failure.") - self._clean() - return ok - - - # -- High-level methods -------------------------------------------- - # (these are the ones that are actually likely to be useful - # when implementing a real-world config command!) - - def check_func(self, func, headers=None, include_dirs=None, - libraries=None, library_dirs=None, decl=0, call=0): - """Determine if function 'func' is available by constructing a - source file that refers to 'func', and compiles and links it. - If everything succeeds, returns true; otherwise returns false. - - The constructed source file starts out by including the header - files listed in 'headers'. If 'decl' is true, it then declares - 'func' (as "int func()"); you probably shouldn't supply 'headers' - and set 'decl' true in the same call, or you might get errors about - a conflicting declarations for 'func'. Finally, the constructed - 'main()' function either references 'func' or (if 'call' is true) - calls it. 'libraries' and 'library_dirs' are used when - linking. - """ - self._check_compiler() - body = [] - if decl: - body.append("int %s ();" % func) - body.append("int main () {") - if call: - body.append(" %s();" % func) - else: - body.append(" %s;" % func) - body.append("}") - body = "\n".join(body) + "\n" - - return self.try_link(body, headers, include_dirs, - libraries, library_dirs) - - def check_lib(self, library, library_dirs=None, headers=None, - include_dirs=None, other_libraries=[]): - """Determine if 'library' is available to be linked against, - without actually checking that any particular symbols are provided - by it. 'headers' will be used in constructing the source file to - be compiled, but the only effect of this is to check if all the - header files listed are available. Any libraries listed in - 'other_libraries' will be included in the link, in case 'library' - has symbols that depend on other libraries. - """ - self._check_compiler() - return self.try_link("int main (void) { }", headers, include_dirs, - [library] + other_libraries, library_dirs) - - def check_header(self, header, include_dirs=None, library_dirs=None, - lang="c"): - """Determine if the system header file named by 'header_file' - exists and can be found by the preprocessor; return true if so, - false otherwise. - """ - return self.try_cpp(body="/* No body */", headers=[header], - include_dirs=include_dirs) - - -def dump_file(filename, head=None): - """Dumps a file content into log.info. - - If head is not None, will be dumped before the file content. - """ - if head is None: - log.info('%s', filename) - else: - log.info(head) - file = open(filename) - try: - log.info(file.read()) - finally: - file.close() diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py deleted file mode 100644 index fd3357ea78e..00000000000 --- a/Lib/distutils/command/install.py +++ /dev/null @@ -1,705 +0,0 @@ -"""distutils.command.install - -Implements the Distutils 'install' command.""" - -import sys -import os - -from distutils import log -from distutils.core import Command -from distutils.debug import DEBUG -from distutils.sysconfig import get_config_vars -from distutils.errors import DistutilsPlatformError -from distutils.file_util import write_file -from distutils.util import convert_path, subst_vars, change_root -from distutils.util import get_platform -from distutils.errors import DistutilsOptionError - -from site import USER_BASE -from site import USER_SITE -HAS_USER_SITE = True - -WINDOWS_SCHEME = { - 'purelib': '$base/Lib/site-packages', - 'platlib': '$base/Lib/site-packages', - 'headers': '$base/Include/$dist_name', - 'scripts': '$base/Scripts', - 'data' : '$base', -} - -INSTALL_SCHEMES = { - 'unix_prefix': { - 'purelib': '$base/lib/python$py_version_short/site-packages', - 'platlib': '$platbase/lib/python$py_version_short/site-packages', - 'headers': '$base/include/python$py_version_short$abiflags/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'unix_local': { - 'purelib': '$base/local/lib/python$py_version_short/dist-packages', - 'platlib': '$platbase/local/lib/python$py_version_short/dist-packages', - 'headers': '$base/local/include/python$py_version_short/$dist_name', - 'scripts': '$base/local/bin', - 'data' : '$base/local', - }, - 'deb_system': { - 'purelib': '$base/lib/python3/dist-packages', - 'platlib': '$platbase/lib/python3/dist-packages', - 'headers': '$base/include/python$py_version_short/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'unix_home': { - 'purelib': '$base/lib/python', - 'platlib': '$base/lib/python', - 'headers': '$base/include/python/$dist_name', - 'scripts': '$base/bin', - 'data' : '$base', - }, - 'nt': WINDOWS_SCHEME, - } - -# user site schemes -if HAS_USER_SITE: - INSTALL_SCHEMES['nt_user'] = { - 'purelib': '$usersite', - 'platlib': '$usersite', - 'headers': '$userbase/Python$py_version_nodot/Include/$dist_name', - 'scripts': '$userbase/Python$py_version_nodot/Scripts', - 'data' : '$userbase', - } - - INSTALL_SCHEMES['unix_user'] = { - 'purelib': '$usersite', - 'platlib': '$usersite', - 'headers': - '$userbase/include/python$py_version_short$abiflags/$dist_name', - 'scripts': '$userbase/bin', - 'data' : '$userbase', - } - -# XXX RUSTPYTHON: replace python with rustpython in all these paths -for group in INSTALL_SCHEMES.values(): - for key in group.keys(): - group[key] = group[key].replace("Python", "RustPython").replace("python", "rustpython") - -# The keys to an installation scheme; if any new types of files are to be -# installed, be sure to add an entry to every installation scheme above, -# and to SCHEME_KEYS here. -SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data') - - -class install(Command): - - description = "install everything from build directory" - - user_options = [ - # Select installation scheme and set base director(y|ies) - ('prefix=', None, - "installation prefix"), - ('exec-prefix=', None, - "(Unix only) prefix for platform-specific files"), - ('home=', None, - "(Unix only) home directory to install under"), - - # Or, just set the base director(y|ies) - ('install-base=', None, - "base installation directory (instead of --prefix or --home)"), - ('install-platbase=', None, - "base installation directory for platform-specific files " + - "(instead of --exec-prefix or --home)"), - ('root=', None, - "install everything relative to this alternate root directory"), - - # Or, explicitly set the installation scheme - ('install-purelib=', None, - "installation directory for pure Python module distributions"), - ('install-platlib=', None, - "installation directory for non-pure module distributions"), - ('install-lib=', None, - "installation directory for all module distributions " + - "(overrides --install-purelib and --install-platlib)"), - - ('install-headers=', None, - "installation directory for C/C++ headers"), - ('install-scripts=', None, - "installation directory for Python scripts"), - ('install-data=', None, - "installation directory for data files"), - - # Byte-compilation options -- see install_lib.py for details, as - # these are duplicated from there (but only install_lib does - # anything with them). - ('compile', 'c', "compile .py to .pyc [default]"), - ('no-compile', None, "don't compile .py files"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - - # Miscellaneous control options - ('force', 'f', - "force installation (overwrite any existing files)"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - - # Where to install documentation (eventually!) - #('doc-format=', None, "format of documentation to generate"), - #('install-man=', None, "directory for Unix man pages"), - #('install-html=', None, "directory for HTML documentation"), - #('install-info=', None, "directory for GNU info files"), - - ('record=', None, - "filename in which to record list of installed files"), - - ('install-layout=', None, - "installation layout to choose (known values: deb, unix)"), - ] - - boolean_options = ['compile', 'force', 'skip-build'] - - if HAS_USER_SITE: - user_options.append(('user', None, - "install in user site-package '%s'" % USER_SITE)) - boolean_options.append('user') - - negative_opt = {'no-compile' : 'compile'} - - - def initialize_options(self): - """Initializes options.""" - # High-level options: these select both an installation base - # and scheme. - self.prefix = None - self.exec_prefix = None - self.home = None - self.user = 0 - self.prefix_option = None - - # These select only the installation base; it's up to the user to - # specify the installation scheme (currently, that means supplying - # the --install-{platlib,purelib,scripts,data} options). - self.install_base = None - self.install_platbase = None - self.root = None - - # These options are the actual installation directories; if not - # supplied by the user, they are filled in using the installation - # scheme implied by prefix/exec-prefix/home and the contents of - # that installation scheme. - self.install_purelib = None # for pure module distributions - self.install_platlib = None # non-pure (dists w/ extensions) - self.install_headers = None # for C/C++ headers - self.install_lib = None # set to either purelib or platlib - self.install_scripts = None - self.install_data = None - self.install_userbase = USER_BASE - self.install_usersite = USER_SITE - - # enable custom installation, known values: deb - self.install_layout = None - self.multiarch = None - - self.compile = None - self.optimize = None - - # Deprecated - # These two are for putting non-packagized distributions into their - # own directory and creating a .pth file if it makes sense. - # 'extra_path' comes from the setup file; 'install_path_file' can - # be turned off if it makes no sense to install a .pth file. (But - # better to install it uselessly than to guess wrong and not - # install it when it's necessary and would be used!) Currently, - # 'install_path_file' is always true unless some outsider meddles - # with it. - self.extra_path = None - self.install_path_file = 1 - - # 'force' forces installation, even if target files are not - # out-of-date. 'skip_build' skips running the "build" command, - # handy if you know it's not necessary. 'warn_dir' (which is *not* - # a user option, it's just there so the bdist_* commands can turn - # it off) determines whether we warn about installing to a - # directory not in sys.path. - self.force = 0 - self.skip_build = 0 - self.warn_dir = 1 - - # These are only here as a conduit from the 'build' command to the - # 'install_*' commands that do the real work. ('build_base' isn't - # actually used anywhere, but it might be useful in future.) They - # are not user options, because if the user told the install - # command where the build directory is, that wouldn't affect the - # build command. - self.build_base = None - self.build_lib = None - - # Not defined yet because we don't know anything about - # documentation yet. - #self.install_man = None - #self.install_html = None - #self.install_info = None - - self.record = None - - - # -- Option finalizing methods ------------------------------------- - # (This is rather more involved than for most commands, - # because this is where the policy for installing third- - # party Python modules on various platforms given a wide - # array of user input is decided. Yes, it's quite complex!) - - def finalize_options(self): - """Finalizes options.""" - # This method (and its pliant slaves, like 'finalize_unix()', - # 'finalize_other()', and 'select_scheme()') is where the default - # installation directories for modules, extension modules, and - # anything else we care to install from a Python module - # distribution. Thus, this code makes a pretty important policy - # statement about how third-party stuff is added to a Python - # installation! Note that the actual work of installation is done - # by the relatively simple 'install_*' commands; they just take - # their orders from the installation directory options determined - # here. - - # Check for errors/inconsistencies in the options; first, stuff - # that's wrong on any platform. - - if ((self.prefix or self.exec_prefix or self.home) and - (self.install_base or self.install_platbase)): - raise DistutilsOptionError( - "must supply either prefix/exec-prefix/home or " + - "install-base/install-platbase -- not both") - - if self.home and (self.prefix or self.exec_prefix): - raise DistutilsOptionError( - "must supply either home or prefix/exec-prefix -- not both") - - if self.user and (self.prefix or self.exec_prefix or self.home or - self.install_base or self.install_platbase): - raise DistutilsOptionError("can't combine user with prefix, " - "exec_prefix/home, or install_(plat)base") - - # Next, stuff that's wrong (or dubious) only on certain platforms. - if os.name != "posix": - if self.exec_prefix: - self.warn("exec-prefix option ignored on this platform") - self.exec_prefix = None - - # Now the interesting logic -- so interesting that we farm it out - # to other methods. The goal of these methods is to set the final - # values for the install_{lib,scripts,data,...} options, using as - # input a heady brew of prefix, exec_prefix, home, install_base, - # install_platbase, user-supplied versions of - # install_{purelib,platlib,lib,scripts,data,...}, and the - # INSTALL_SCHEME dictionary above. Phew! - - self.dump_dirs("pre-finalize_{unix,other}") - - if os.name == 'posix': - self.finalize_unix() - else: - self.finalize_other() - - self.dump_dirs("post-finalize_{unix,other}()") - - # Expand configuration variables, tilde, etc. in self.install_base - # and self.install_platbase -- that way, we can use $base or - # $platbase in the other installation directories and not worry - # about needing recursive variable expansion (shudder). - - py_version = sys.version.split()[0] - (prefix, exec_prefix) = get_config_vars('prefix', 'exec_prefix') - try: - abiflags = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - abiflags = '' - self.config_vars = {'dist_name': self.distribution.get_name(), - 'dist_version': self.distribution.get_version(), - 'dist_fullname': self.distribution.get_fullname(), - 'py_version': py_version, - 'py_version_short': '%d.%d' % sys.version_info[:2], - 'py_version_nodot': '%d%d' % sys.version_info[:2], - 'sys_prefix': prefix, - 'prefix': prefix, - 'sys_exec_prefix': exec_prefix, - 'exec_prefix': exec_prefix, - 'abiflags': abiflags, - } - - if HAS_USER_SITE: - self.config_vars['userbase'] = self.install_userbase - self.config_vars['usersite'] = self.install_usersite - - self.expand_basedirs() - - self.dump_dirs("post-expand_basedirs()") - - # Now define config vars for the base directories so we can expand - # everything else. - self.config_vars['base'] = self.install_base - self.config_vars['platbase'] = self.install_platbase - - if DEBUG: - from pprint import pprint - print("config vars:") - pprint(self.config_vars) - - # Expand "~" and configuration variables in the installation - # directories. - self.expand_dirs() - - self.dump_dirs("post-expand_dirs()") - - # Create directories in the home dir: - if self.user: - self.create_home_path() - - # Pick the actual directory to install all modules to: either - # install_purelib or install_platlib, depending on whether this - # module distribution is pure or not. Of course, if the user - # already specified install_lib, use their selection. - if self.install_lib is None: - if self.distribution.ext_modules: # has extensions: non-pure - self.install_lib = self.install_platlib - else: - self.install_lib = self.install_purelib - - - # Convert directories from Unix /-separated syntax to the local - # convention. - self.convert_paths('lib', 'purelib', 'platlib', - 'scripts', 'data', 'headers', - 'userbase', 'usersite') - - # Deprecated - # Well, we're not actually fully completely finalized yet: we still - # have to deal with 'extra_path', which is the hack for allowing - # non-packagized module distributions (hello, Numerical Python!) to - # get their own directories. - self.handle_extra_path() - self.install_libbase = self.install_lib # needed for .pth file - self.install_lib = os.path.join(self.install_lib, self.extra_dirs) - - # If a new root directory was supplied, make all the installation - # dirs relative to it. - if self.root is not None: - self.change_roots('libbase', 'lib', 'purelib', 'platlib', - 'scripts', 'data', 'headers') - - self.dump_dirs("after prepending root") - - # Find out the build directories, ie. where to install from. - self.set_undefined_options('build', - ('build_base', 'build_base'), - ('build_lib', 'build_lib')) - - # Punt on doc directories for now -- after all, we're punting on - # documentation completely! - - def dump_dirs(self, msg): - """Dumps the list of user options.""" - if not DEBUG: - return - from distutils.fancy_getopt import longopt_xlate - log.debug(msg + ":") - for opt in self.user_options: - opt_name = opt[0] - if opt_name[-1] == "=": - opt_name = opt_name[0:-1] - if opt_name in self.negative_opt: - opt_name = self.negative_opt[opt_name] - opt_name = opt_name.translate(longopt_xlate) - val = not getattr(self, opt_name) - else: - opt_name = opt_name.translate(longopt_xlate) - val = getattr(self, opt_name) - log.debug(" %s: %s", opt_name, val) - - def finalize_unix(self): - """Finalizes options for posix platforms.""" - if self.install_base is not None or self.install_platbase is not None: - if ((self.install_lib is None and - self.install_purelib is None and - self.install_platlib is None) or - self.install_headers is None or - self.install_scripts is None or - self.install_data is None): - raise DistutilsOptionError( - "install-base or install-platbase supplied, but " - "installation scheme is incomplete") - return - - if self.user: - if self.install_userbase is None: - raise DistutilsPlatformError( - "User base directory is not specified") - self.install_base = self.install_platbase = self.install_userbase - self.select_scheme("unix_user") - elif self.home is not None: - self.install_base = self.install_platbase = self.home - self.select_scheme("unix_home") - else: - self.prefix_option = self.prefix - if self.prefix is None: - if self.exec_prefix is not None: - raise DistutilsOptionError( - "must not supply exec-prefix without prefix") - - self.prefix = os.path.normpath(sys.prefix) - self.exec_prefix = os.path.normpath(sys.exec_prefix) - - else: - if self.exec_prefix is None: - self.exec_prefix = self.prefix - - self.install_base = self.prefix - self.install_platbase = self.exec_prefix - if self.install_layout: - if self.install_layout.lower() in ['deb']: - import sysconfig - self.multiarch = sysconfig.get_config_var('MULTIARCH') - self.select_scheme("deb_system") - elif self.install_layout.lower() in ['unix']: - self.select_scheme("unix_prefix") - else: - raise DistutilsOptionError( - "unknown value for --install-layout") - elif ((self.prefix_option and - os.path.normpath(self.prefix) != '/usr/local') - or sys.base_prefix != sys.prefix - or 'PYTHONUSERBASE' in os.environ - or 'VIRTUAL_ENV' in os.environ - or 'real_prefix' in sys.__dict__): - self.select_scheme("unix_prefix") - else: - if os.path.normpath(self.prefix) == '/usr/local': - self.prefix = self.exec_prefix = '/usr' - self.install_base = self.install_platbase = '/usr' - self.select_scheme("unix_local") - - def finalize_other(self): - """Finalizes options for non-posix platforms""" - if self.user: - if self.install_userbase is None: - raise DistutilsPlatformError( - "User base directory is not specified") - self.install_base = self.install_platbase = self.install_userbase - self.select_scheme(os.name + "_user") - elif self.home is not None: - self.install_base = self.install_platbase = self.home - self.select_scheme("unix_home") - else: - if self.prefix is None: - self.prefix = os.path.normpath(sys.prefix) - - self.install_base = self.install_platbase = self.prefix - try: - self.select_scheme(os.name) - except KeyError: - raise DistutilsPlatformError( - "I don't know how to install stuff on '%s'" % os.name) - - def select_scheme(self, name): - """Sets the install directories by applying the install schemes.""" - # it's the caller's problem if they supply a bad name! - scheme = INSTALL_SCHEMES[name] - for key in SCHEME_KEYS: - attrname = 'install_' + key - if getattr(self, attrname) is None: - setattr(self, attrname, scheme[key]) - - def _expand_attrs(self, attrs): - for attr in attrs: - val = getattr(self, attr) - if val is not None: - if os.name == 'posix' or os.name == 'nt': - val = os.path.expanduser(val) - val = subst_vars(val, self.config_vars) - setattr(self, attr, val) - - def expand_basedirs(self): - """Calls `os.path.expanduser` on install_base, install_platbase and - root.""" - self._expand_attrs(['install_base', 'install_platbase', 'root']) - - def expand_dirs(self): - """Calls `os.path.expanduser` on install dirs.""" - self._expand_attrs(['install_purelib', 'install_platlib', - 'install_lib', 'install_headers', - 'install_scripts', 'install_data',]) - - def convert_paths(self, *names): - """Call `convert_path` over `names`.""" - for name in names: - attr = "install_" + name - setattr(self, attr, convert_path(getattr(self, attr))) - - def handle_extra_path(self): - """Set `path_file` and `extra_dirs` using `extra_path`.""" - if self.extra_path is None: - self.extra_path = self.distribution.extra_path - - if self.extra_path is not None: - log.warn( - "Distribution option extra_path is deprecated. " - "See issue27919 for details." - ) - if isinstance(self.extra_path, str): - self.extra_path = self.extra_path.split(',') - - if len(self.extra_path) == 1: - path_file = extra_dirs = self.extra_path[0] - elif len(self.extra_path) == 2: - path_file, extra_dirs = self.extra_path - else: - raise DistutilsOptionError( - "'extra_path' option must be a list, tuple, or " - "comma-separated string with 1 or 2 elements") - - # convert to local form in case Unix notation used (as it - # should be in setup scripts) - extra_dirs = convert_path(extra_dirs) - else: - path_file = None - extra_dirs = '' - - # XXX should we warn if path_file and not extra_dirs? (in which - # case the path file would be harmless but pointless) - self.path_file = path_file - self.extra_dirs = extra_dirs - - def change_roots(self, *names): - """Change the install directories pointed by name using root.""" - for name in names: - attr = "install_" + name - setattr(self, attr, change_root(self.root, getattr(self, attr))) - - def create_home_path(self): - """Create directories under ~.""" - if not self.user: - return - home = convert_path(os.path.expanduser("~")) - for name, path in self.config_vars.items(): - if path.startswith(home) and not os.path.isdir(path): - self.debug_print("os.makedirs('%s', 0o700)" % path) - os.makedirs(path, 0o700) - - # -- Command execution methods ------------------------------------- - - def run(self): - """Runs the command.""" - # Obviously have to build before we can install - if not self.skip_build: - self.run_command('build') - # If we built for any other platform, we can't install. - build_plat = self.distribution.get_command_obj('build').plat_name - # check warn_dir - it is a clue that the 'install' is happening - # internally, and not to sys.path, so we don't check the platform - # matches what we are running. - if self.warn_dir and build_plat != get_platform(): - raise DistutilsPlatformError("Can't install when " - "cross-compiling") - - # Run all sub-commands (at least those that need to be run) - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - if self.path_file: - self.create_path_file() - - # write list of installed files, if requested. - if self.record: - outputs = self.get_outputs() - if self.root: # strip any package prefix - root_len = len(self.root) - for counter in range(len(outputs)): - outputs[counter] = outputs[counter][root_len:] - self.execute(write_file, - (self.record, outputs), - "writing list of installed files to '%s'" % - self.record) - - sys_path = map(os.path.normpath, sys.path) - sys_path = map(os.path.normcase, sys_path) - install_lib = os.path.normcase(os.path.normpath(self.install_lib)) - if (self.warn_dir and - not (self.path_file and self.install_path_file) and - install_lib not in sys_path): - log.debug(("modules installed to '%s', which is not in " - "Python's module search path (sys.path) -- " - "you'll have to change the search path yourself"), - self.install_lib) - - def create_path_file(self): - """Creates the .pth file""" - filename = os.path.join(self.install_libbase, - self.path_file + ".pth") - if self.install_path_file: - self.execute(write_file, - (filename, [self.extra_dirs]), - "creating %s" % filename) - else: - self.warn("path file '%s' not created" % filename) - - - # -- Reporting methods --------------------------------------------- - - def get_outputs(self): - """Assembles the outputs of all the sub-commands.""" - outputs = [] - for cmd_name in self.get_sub_commands(): - cmd = self.get_finalized_command(cmd_name) - # Add the contents of cmd.get_outputs(), ensuring - # that outputs doesn't contain duplicate entries - for filename in cmd.get_outputs(): - if filename not in outputs: - outputs.append(filename) - - if self.path_file and self.install_path_file: - outputs.append(os.path.join(self.install_libbase, - self.path_file + ".pth")) - - return outputs - - def get_inputs(self): - """Returns the inputs of all the sub-commands""" - # XXX gee, this looks familiar ;-( - inputs = [] - for cmd_name in self.get_sub_commands(): - cmd = self.get_finalized_command(cmd_name) - inputs.extend(cmd.get_inputs()) - - return inputs - - # -- Predicates for sub-command list ------------------------------- - - def has_lib(self): - """Returns true if the current distribution has any Python - modules to install.""" - return (self.distribution.has_pure_modules() or - self.distribution.has_ext_modules()) - - def has_headers(self): - """Returns true if the current distribution has any headers to - install.""" - return self.distribution.has_headers() - - def has_scripts(self): - """Returns true if the current distribution has any scripts to. - install.""" - return self.distribution.has_scripts() - - def has_data(self): - """Returns true if the current distribution has any data to. - install.""" - return self.distribution.has_data_files() - - # 'sub_commands': a list of commands this command might have to run to - # get its work done. See cmd.py for more info. - sub_commands = [('install_lib', has_lib), - ('install_headers', has_headers), - ('install_scripts', has_scripts), - ('install_data', has_data), - ('install_egg_info', lambda self:True), - ] diff --git a/Lib/distutils/command/install_data.py b/Lib/distutils/command/install_data.py deleted file mode 100644 index 947cd76a99e..00000000000 --- a/Lib/distutils/command/install_data.py +++ /dev/null @@ -1,79 +0,0 @@ -"""distutils.command.install_data - -Implements the Distutils 'install_data' command, for installing -platform-independent data files.""" - -# contributed by Bastian Kleineidam - -import os -from distutils.core import Command -from distutils.util import change_root, convert_path - -class install_data(Command): - - description = "install data files" - - user_options = [ - ('install-dir=', 'd', - "base directory for installing data files " - "(default: installation base dir)"), - ('root=', None, - "install everything relative to this alternate root directory"), - ('force', 'f', "force installation (overwrite existing files)"), - ] - - boolean_options = ['force'] - - def initialize_options(self): - self.install_dir = None - self.outfiles = [] - self.root = None - self.force = 0 - self.data_files = self.distribution.data_files - self.warn_dir = 1 - - def finalize_options(self): - self.set_undefined_options('install', - ('install_data', 'install_dir'), - ('root', 'root'), - ('force', 'force'), - ) - - def run(self): - self.mkpath(self.install_dir) - for f in self.data_files: - if isinstance(f, str): - # it's a simple file, so copy it - f = convert_path(f) - if self.warn_dir: - self.warn("setup script did not provide a directory for " - "'%s' -- installing right in '%s'" % - (f, self.install_dir)) - (out, _) = self.copy_file(f, self.install_dir) - self.outfiles.append(out) - else: - # it's a tuple with path to install to and a list of files - dir = convert_path(f[0]) - if not os.path.isabs(dir): - dir = os.path.join(self.install_dir, dir) - elif self.root: - dir = change_root(self.root, dir) - self.mkpath(dir) - - if f[1] == []: - # If there are no files listed, the user must be - # trying to create an empty directory, so add the - # directory to the list of output files. - self.outfiles.append(dir) - else: - # Copy files, adding them to the list of output files. - for data in f[1]: - data = convert_path(data) - (out, _) = self.copy_file(data, dir) - self.outfiles.append(out) - - def get_inputs(self): - return self.data_files or [] - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/install_egg_info.py b/Lib/distutils/command/install_egg_info.py deleted file mode 100644 index 0a71b610005..00000000000 --- a/Lib/distutils/command/install_egg_info.py +++ /dev/null @@ -1,97 +0,0 @@ -"""distutils.command.install_egg_info - -Implements the Distutils 'install_egg_info' command, for installing -a package's PKG-INFO metadata.""" - - -from distutils.cmd import Command -from distutils import log, dir_util -import os, sys, re - -class install_egg_info(Command): - """Install an .egg-info file for the package""" - - description = "Install package's PKG-INFO metadata as an .egg-info file" - user_options = [ - ('install-dir=', 'd', "directory to install to"), - ('install-layout', None, "custom installation layout"), - ] - - def initialize_options(self): - self.install_dir = None - self.install_layout = None - self.prefix_option = None - - def finalize_options(self): - self.set_undefined_options('install_lib',('install_dir','install_dir')) - self.set_undefined_options('install',('install_layout','install_layout')) - self.set_undefined_options('install',('prefix_option','prefix_option')) - if self.install_layout: - if not self.install_layout.lower() in ['deb', 'unix']: - raise DistutilsOptionError( - "unknown value for --install-layout") - no_pyver = (self.install_layout.lower() == 'deb') - elif self.prefix_option: - no_pyver = False - else: - no_pyver = True - if no_pyver: - basename = "%s-%s.egg-info" % ( - to_filename(safe_name(self.distribution.get_name())), - to_filename(safe_version(self.distribution.get_version())) - ) - else: - basename = "%s-%s-py%d.%d.egg-info" % ( - to_filename(safe_name(self.distribution.get_name())), - to_filename(safe_version(self.distribution.get_version())), - *sys.version_info[:2] - ) - self.target = os.path.join(self.install_dir, basename) - self.outputs = [self.target] - - def run(self): - target = self.target - if os.path.isdir(target) and not os.path.islink(target): - dir_util.remove_tree(target, dry_run=self.dry_run) - elif os.path.exists(target): - self.execute(os.unlink,(self.target,),"Removing "+target) - elif not os.path.isdir(self.install_dir): - self.execute(os.makedirs, (self.install_dir,), - "Creating "+self.install_dir) - log.info("Writing %s", target) - if not self.dry_run: - with open(target, 'w', encoding='UTF-8') as f: - self.distribution.metadata.write_pkg_file(f) - - def get_outputs(self): - return self.outputs - - -# The following routines are taken from setuptools' pkg_resources module and -# can be replaced by importing them from pkg_resources once it is included -# in the stdlib. - -def safe_name(name): - """Convert an arbitrary string to a standard distribution name - - Any runs of non-alphanumeric/. characters are replaced with a single '-'. - """ - return re.sub('[^A-Za-z0-9.]+', '-', name) - - -def safe_version(version): - """Convert an arbitrary string to a standard version string - - Spaces become dots, and all other non-alphanumeric characters become - dashes, with runs of multiple dashes condensed to a single dash. - """ - version = version.replace(' ','.') - return re.sub('[^A-Za-z0-9.]+', '-', version) - - -def to_filename(name): - """Convert a project or version name to its filename-escaped form - - Any '-' characters are currently replaced with '_'. - """ - return name.replace('-','_') diff --git a/Lib/distutils/command/install_headers.py b/Lib/distutils/command/install_headers.py deleted file mode 100644 index 9bb0b18dc0d..00000000000 --- a/Lib/distutils/command/install_headers.py +++ /dev/null @@ -1,47 +0,0 @@ -"""distutils.command.install_headers - -Implements the Distutils 'install_headers' command, to install C/C++ header -files to the Python include directory.""" - -from distutils.core import Command - - -# XXX force is never used -class install_headers(Command): - - description = "install C/C++ header files" - - user_options = [('install-dir=', 'd', - "directory to install header files to"), - ('force', 'f', - "force installation (overwrite existing files)"), - ] - - boolean_options = ['force'] - - def initialize_options(self): - self.install_dir = None - self.force = 0 - self.outfiles = [] - - def finalize_options(self): - self.set_undefined_options('install', - ('install_headers', 'install_dir'), - ('force', 'force')) - - - def run(self): - headers = self.distribution.headers - if not headers: - return - - self.mkpath(self.install_dir) - for header in headers: - (out, _) = self.copy_file(header, self.install_dir) - self.outfiles.append(out) - - def get_inputs(self): - return self.distribution.headers or [] - - def get_outputs(self): - return self.outfiles diff --git a/Lib/distutils/command/install_lib.py b/Lib/distutils/command/install_lib.py deleted file mode 100644 index eef63626ff7..00000000000 --- a/Lib/distutils/command/install_lib.py +++ /dev/null @@ -1,221 +0,0 @@ -"""distutils.command.install_lib - -Implements the Distutils 'install_lib' command -(install all Python modules).""" - -import os -import importlib.util -import sys - -from distutils.core import Command -from distutils.errors import DistutilsOptionError - - -# Extension for Python source files. -PYTHON_SOURCE_EXTENSION = ".py" - -class install_lib(Command): - - description = "install all Python modules (extensions and pure Python)" - - # The byte-compilation options are a tad confusing. Here are the - # possible scenarios: - # 1) no compilation at all (--no-compile --no-optimize) - # 2) compile .pyc only (--compile --no-optimize; default) - # 3) compile .pyc and "opt-1" .pyc (--compile --optimize) - # 4) compile "opt-1" .pyc only (--no-compile --optimize) - # 5) compile .pyc and "opt-2" .pyc (--compile --optimize-more) - # 6) compile "opt-2" .pyc only (--no-compile --optimize-more) - # - # The UI for this is two options, 'compile' and 'optimize'. - # 'compile' is strictly boolean, and only decides whether to - # generate .pyc files. 'optimize' is three-way (0, 1, or 2), and - # decides both whether to generate .pyc files and what level of - # optimization to use. - - user_options = [ - ('install-dir=', 'd', "directory to install to"), - ('build-dir=','b', "build directory (where to install from)"), - ('force', 'f', "force installation (overwrite existing files)"), - ('compile', 'c', "compile .py to .pyc [default]"), - ('no-compile', None, "don't compile .py files"), - ('optimize=', 'O', - "also compile with optimization: -O1 for \"python -O\", " - "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), - ('skip-build', None, "skip the build steps"), - ] - - boolean_options = ['force', 'compile', 'skip-build'] - negative_opt = {'no-compile' : 'compile'} - - def initialize_options(self): - # let the 'install' command dictate our installation directory - self.install_dir = None - self.build_dir = None - self.force = 0 - self.compile = None - self.optimize = None - self.skip_build = None - self.multiarch = None # if we should rename the extensions - - def finalize_options(self): - # Get all the information we need to install pure Python modules - # from the umbrella 'install' command -- build (source) directory, - # install (target) directory, and whether to compile .py files. - self.set_undefined_options('install', - ('build_lib', 'build_dir'), - ('install_lib', 'install_dir'), - ('force', 'force'), - ('compile', 'compile'), - ('optimize', 'optimize'), - ('skip_build', 'skip_build'), - ('multiarch', 'multiarch'), - ) - - if self.compile is None: - self.compile = True - if self.optimize is None: - self.optimize = False - - if not isinstance(self.optimize, int): - try: - self.optimize = int(self.optimize) - if self.optimize not in (0, 1, 2): - raise AssertionError - except (ValueError, AssertionError): - raise DistutilsOptionError("optimize must be 0, 1, or 2") - - def run(self): - # Make sure we have built everything we need first - self.build() - - # Install everything: simply dump the entire contents of the build - # directory to the installation directory (that's the beauty of - # having a build directory!) - outfiles = self.install() - - # (Optionally) compile .py to .pyc - if outfiles is not None and self.distribution.has_pure_modules(): - self.byte_compile(outfiles) - - # -- Top-level worker functions ------------------------------------ - # (called from 'run()') - - def build(self): - if not self.skip_build: - if self.distribution.has_pure_modules(): - self.run_command('build_py') - if self.distribution.has_ext_modules(): - self.run_command('build_ext') - - def install(self): - if os.path.isdir(self.build_dir): - import distutils.dir_util - distutils.dir_util._multiarch = self.multiarch - outfiles = self.copy_tree(self.build_dir, self.install_dir) - else: - self.warn("'%s' does not exist -- no Python modules to install" % - self.build_dir) - return - return outfiles - - def byte_compile(self, files): - if sys.dont_write_bytecode: - self.warn('byte-compiling is disabled, skipping.') - return - - from distutils.util import byte_compile - - # Get the "--root" directory supplied to the "install" command, - # and use it as a prefix to strip off the purported filename - # encoded in bytecode files. This is far from complete, but it - # should at least generate usable bytecode in RPM distributions. - install_root = self.get_finalized_command('install').root - - if self.compile: - byte_compile(files, optimize=0, - force=self.force, prefix=install_root, - dry_run=self.dry_run) - if self.optimize > 0: - byte_compile(files, optimize=self.optimize, - force=self.force, prefix=install_root, - verbose=self.verbose, dry_run=self.dry_run) - - - # -- Utility methods ----------------------------------------------- - - def _mutate_outputs(self, has_any, build_cmd, cmd_option, output_dir): - if not has_any: - return [] - - build_cmd = self.get_finalized_command(build_cmd) - build_files = build_cmd.get_outputs() - build_dir = getattr(build_cmd, cmd_option) - - prefix_len = len(build_dir) + len(os.sep) - outputs = [] - for file in build_files: - outputs.append(os.path.join(output_dir, file[prefix_len:])) - - return outputs - - def _bytecode_filenames(self, py_filenames): - bytecode_files = [] - for py_file in py_filenames: - # Since build_py handles package data installation, the - # list of outputs can contain more than just .py files. - # Make sure we only report bytecode for the .py files. - ext = os.path.splitext(os.path.normcase(py_file))[1] - if ext != PYTHON_SOURCE_EXTENSION: - continue - if self.compile: - bytecode_files.append(importlib.util.cache_from_source( - py_file, optimization='')) - if self.optimize > 0: - bytecode_files.append(importlib.util.cache_from_source( - py_file, optimization=self.optimize)) - - return bytecode_files - - - # -- External interface -------------------------------------------- - # (called by outsiders) - - def get_outputs(self): - """Return the list of files that would be installed if this command - were actually run. Not affected by the "dry-run" flag or whether - modules have actually been built yet. - """ - pure_outputs = \ - self._mutate_outputs(self.distribution.has_pure_modules(), - 'build_py', 'build_lib', - self.install_dir) - if self.compile: - bytecode_outputs = self._bytecode_filenames(pure_outputs) - else: - bytecode_outputs = [] - - ext_outputs = \ - self._mutate_outputs(self.distribution.has_ext_modules(), - 'build_ext', 'build_lib', - self.install_dir) - - return pure_outputs + bytecode_outputs + ext_outputs - - def get_inputs(self): - """Get the list of files that are input to this command, ie. the - files that get installed as they are named in the build tree. - The files in this list correspond one-to-one to the output - filenames returned by 'get_outputs()'. - """ - inputs = [] - - if self.distribution.has_pure_modules(): - build_py = self.get_finalized_command('build_py') - inputs.extend(build_py.get_outputs()) - - if self.distribution.has_ext_modules(): - build_ext = self.get_finalized_command('build_ext') - inputs.extend(build_ext.get_outputs()) - - return inputs diff --git a/Lib/distutils/command/install_scripts.py b/Lib/distutils/command/install_scripts.py deleted file mode 100644 index 31a1130ee54..00000000000 --- a/Lib/distutils/command/install_scripts.py +++ /dev/null @@ -1,60 +0,0 @@ -"""distutils.command.install_scripts - -Implements the Distutils 'install_scripts' command, for installing -Python scripts.""" - -# contributed by Bastian Kleineidam - -import os -from distutils.core import Command -from distutils import log -from stat import ST_MODE - - -class install_scripts(Command): - - description = "install scripts (Python or otherwise)" - - user_options = [ - ('install-dir=', 'd', "directory to install scripts to"), - ('build-dir=','b', "build directory (where to install from)"), - ('force', 'f', "force installation (overwrite existing files)"), - ('skip-build', None, "skip the build steps"), - ] - - boolean_options = ['force', 'skip-build'] - - def initialize_options(self): - self.install_dir = None - self.force = 0 - self.build_dir = None - self.skip_build = None - - def finalize_options(self): - self.set_undefined_options('build', ('build_scripts', 'build_dir')) - self.set_undefined_options('install', - ('install_scripts', 'install_dir'), - ('force', 'force'), - ('skip_build', 'skip_build'), - ) - - def run(self): - if not self.skip_build: - self.run_command('build_scripts') - self.outfiles = self.copy_tree(self.build_dir, self.install_dir) - if os.name == 'posix': - # Set the executable bits (owner, group, and world) on - # all the scripts we just installed. - for file in self.get_outputs(): - if self.dry_run: - log.info("changing mode of %s", file) - else: - mode = ((os.stat(file)[ST_MODE]) | 0o555) & 0o7777 - log.info("changing mode of %s to %o", file, mode) - os.chmod(file, mode) - - def get_inputs(self): - return self.distribution.scripts or [] - - def get_outputs(self): - return self.outfiles or [] diff --git a/Lib/distutils/command/register.py b/Lib/distutils/command/register.py deleted file mode 100644 index 0fac94e9e54..00000000000 --- a/Lib/distutils/command/register.py +++ /dev/null @@ -1,304 +0,0 @@ -"""distutils.command.register - -Implements the Distutils 'register' command (register with the repository). -""" - -# created 2002/10/21, Richard Jones - -import getpass -import io -import urllib.parse, urllib.request -from warnings import warn - -from distutils.core import PyPIRCCommand -from distutils.errors import * -from distutils import log - -class register(PyPIRCCommand): - - description = ("register the distribution with the Python package index") - user_options = PyPIRCCommand.user_options + [ - ('list-classifiers', None, - 'list the valid Trove classifiers'), - ('strict', None , - 'Will stop the registering if the meta-data are not fully compliant') - ] - boolean_options = PyPIRCCommand.boolean_options + [ - 'verify', 'list-classifiers', 'strict'] - - sub_commands = [('check', lambda self: True)] - - def initialize_options(self): - PyPIRCCommand.initialize_options(self) - self.list_classifiers = 0 - self.strict = 0 - - def finalize_options(self): - PyPIRCCommand.finalize_options(self) - # setting options for the `check` subcommand - check_options = {'strict': ('register', self.strict), - 'restructuredtext': ('register', 1)} - self.distribution.command_options['check'] = check_options - - def run(self): - self.finalize_options() - self._set_config() - - # Run sub commands - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - if self.dry_run: - self.verify_metadata() - elif self.list_classifiers: - self.classifiers() - else: - self.send_metadata() - - def check_metadata(self): - """Deprecated API.""" - warn("distutils.command.register.check_metadata is deprecated, \ - use the check command instead", PendingDeprecationWarning) - check = self.distribution.get_command_obj('check') - check.ensure_finalized() - check.strict = self.strict - check.restructuredtext = 1 - check.run() - - def _set_config(self): - ''' Reads the configuration file and set attributes. - ''' - config = self._read_pypirc() - if config != {}: - self.username = config['username'] - self.password = config['password'] - self.repository = config['repository'] - self.realm = config['realm'] - self.has_config = True - else: - if self.repository not in ('pypi', self.DEFAULT_REPOSITORY): - raise ValueError('%s not found in .pypirc' % self.repository) - if self.repository == 'pypi': - self.repository = self.DEFAULT_REPOSITORY - self.has_config = False - - def classifiers(self): - ''' Fetch the list of classifiers from the server. - ''' - url = self.repository+'?:action=list_classifiers' - response = urllib.request.urlopen(url) - log.info(self._read_pypi_response(response)) - - def verify_metadata(self): - ''' Send the metadata to the package index server to be checked. - ''' - # send the info to the server and report the result - (code, result) = self.post_to_server(self.build_post_data('verify')) - log.info('Server response (%s): %s', code, result) - - def send_metadata(self): - ''' Send the metadata to the package index server. - - Well, do the following: - 1. figure who the user is, and then - 2. send the data as a Basic auth'ed POST. - - First we try to read the username/password from $HOME/.pypirc, - which is a ConfigParser-formatted file with a section - [distutils] containing username and password entries (both - in clear text). Eg: - - [distutils] - index-servers = - pypi - - [pypi] - username: fred - password: sekrit - - Otherwise, to figure who the user is, we offer the user three - choices: - - 1. use existing login, - 2. register as a new user, or - 3. set the password to a random string and email the user. - - ''' - # see if we can short-cut and get the username/password from the - # config - if self.has_config: - choice = '1' - username = self.username - password = self.password - else: - choice = 'x' - username = password = '' - - # get the user's login info - choices = '1 2 3 4'.split() - while choice not in choices: - self.announce('''\ -We need to know who you are, so please choose either: - 1. use your existing login, - 2. register as a new user, - 3. have the server generate a new password for you (and email it to you), or - 4. quit -Your selection [default 1]: ''', log.INFO) - choice = input() - if not choice: - choice = '1' - elif choice not in choices: - print('Please choose one of the four options!') - - if choice == '1': - # get the username and password - while not username: - username = input('Username: ') - while not password: - password = getpass.getpass('Password: ') - - # set up the authentication - auth = urllib.request.HTTPPasswordMgr() - host = urllib.parse.urlparse(self.repository)[1] - auth.add_password(self.realm, host, username, password) - # send the info to the server and report the result - code, result = self.post_to_server(self.build_post_data('submit'), - auth) - self.announce('Server response (%s): %s' % (code, result), - log.INFO) - - # possibly save the login - if code == 200: - if self.has_config: - # sharing the password in the distribution instance - # so the upload command can reuse it - self.distribution.password = password - else: - self.announce(('I can store your PyPI login so future ' - 'submissions will be faster.'), log.INFO) - self.announce('(the login will be stored in %s)' % \ - self._get_rc_file(), log.INFO) - choice = 'X' - while choice.lower() not in 'yn': - choice = input('Save your login (y/N)?') - if not choice: - choice = 'n' - if choice.lower() == 'y': - self._store_pypirc(username, password) - - elif choice == '2': - data = {':action': 'user'} - data['name'] = data['password'] = data['email'] = '' - data['confirm'] = None - while not data['name']: - data['name'] = input('Username: ') - while data['password'] != data['confirm']: - while not data['password']: - data['password'] = getpass.getpass('Password: ') - while not data['confirm']: - data['confirm'] = getpass.getpass(' Confirm: ') - if data['password'] != data['confirm']: - data['password'] = '' - data['confirm'] = None - print("Password and confirm don't match!") - while not data['email']: - data['email'] = input(' EMail: ') - code, result = self.post_to_server(data) - if code != 200: - log.info('Server response (%s): %s', code, result) - else: - log.info('You will receive an email shortly.') - log.info(('Follow the instructions in it to ' - 'complete registration.')) - elif choice == '3': - data = {':action': 'password_reset'} - data['email'] = '' - while not data['email']: - data['email'] = input('Your email address: ') - code, result = self.post_to_server(data) - log.info('Server response (%s): %s', code, result) - - def build_post_data(self, action): - # figure the data to send - the metadata plus some additional - # information used by the package server - meta = self.distribution.metadata - data = { - ':action': action, - 'metadata_version' : '1.0', - 'name': meta.get_name(), - 'version': meta.get_version(), - 'summary': meta.get_description(), - 'home_page': meta.get_url(), - 'author': meta.get_contact(), - 'author_email': meta.get_contact_email(), - 'license': meta.get_licence(), - 'description': meta.get_long_description(), - 'keywords': meta.get_keywords(), - 'platform': meta.get_platforms(), - 'classifiers': meta.get_classifiers(), - 'download_url': meta.get_download_url(), - # PEP 314 - 'provides': meta.get_provides(), - 'requires': meta.get_requires(), - 'obsoletes': meta.get_obsoletes(), - } - if data['provides'] or data['requires'] or data['obsoletes']: - data['metadata_version'] = '1.1' - return data - - def post_to_server(self, data, auth=None): - ''' Post a query to the server, and return a string response. - ''' - if 'name' in data: - self.announce('Registering %s to %s' % (data['name'], - self.repository), - log.INFO) - # Build up the MIME payload for the urllib2 POST data - boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = '\n--' + boundary - end_boundary = sep_boundary + '--' - body = io.StringIO() - for key, value in data.items(): - # handle multiple entries for the same name - if type(value) not in (type([]), type( () )): - value = [value] - for value in value: - value = str(value) - body.write(sep_boundary) - body.write('\nContent-Disposition: form-data; name="%s"'%key) - body.write("\n\n") - body.write(value) - if value and value[-1] == '\r': - body.write('\n') # write an extra newline (lurve Macs) - body.write(end_boundary) - body.write("\n") - body = body.getvalue().encode("utf-8") - - # build the Request - headers = { - 'Content-type': 'multipart/form-data; boundary=%s; charset=utf-8'%boundary, - 'Content-length': str(len(body)) - } - req = urllib.request.Request(self.repository, body, headers) - - # handle HTTP and include the Basic Auth handler - opener = urllib.request.build_opener( - urllib.request.HTTPBasicAuthHandler(password_mgr=auth) - ) - data = '' - try: - result = opener.open(req) - except urllib.error.HTTPError as e: - if self.show_response: - data = e.fp.read() - result = e.code, e.msg - except urllib.error.URLError as e: - result = 500, str(e) - else: - if self.show_response: - data = self._read_pypi_response(result) - result = 200, 'OK' - if self.show_response: - msg = '\n'.join(('-' * 75, data, '-' * 75)) - self.announce(msg, log.INFO) - return result diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py deleted file mode 100644 index 4fd1d4715de..00000000000 --- a/Lib/distutils/command/sdist.py +++ /dev/null @@ -1,456 +0,0 @@ -"""distutils.command.sdist - -Implements the Distutils 'sdist' command (create a source distribution).""" - -import os -import sys -from types import * -from glob import glob -from warnings import warn - -from distutils.core import Command -from distutils import dir_util, dep_util, file_util, archive_util -from distutils.text_file import TextFile -from distutils.errors import * -from distutils.filelist import FileList -from distutils import log -from distutils.util import convert_path - -def show_formats(): - """Print all possible values for the 'formats' option (used by - the "--help-formats" command-line option). - """ - from distutils.fancy_getopt import FancyGetopt - from distutils.archive_util import ARCHIVE_FORMATS - formats = [] - for format in ARCHIVE_FORMATS.keys(): - formats.append(("formats=" + format, None, - ARCHIVE_FORMATS[format][2])) - formats.sort() - FancyGetopt(formats).print_help( - "List of available source distribution formats:") - -class sdist(Command): - - description = "create a source distribution (tarball, zip file, etc.)" - - def checking_metadata(self): - """Callable used for the check sub-command. - - Placed here so user_options can view it""" - return self.metadata_check - - user_options = [ - ('template=', 't', - "name of manifest template file [default: MANIFEST.in]"), - ('manifest=', 'm', - "name of manifest file [default: MANIFEST]"), - ('use-defaults', None, - "include the default file set in the manifest " - "[default; disable with --no-defaults]"), - ('no-defaults', None, - "don't include the default file set"), - ('prune', None, - "specifically exclude files/directories that should not be " - "distributed (build tree, RCS/CVS dirs, etc.) " - "[default; disable with --no-prune]"), - ('no-prune', None, - "don't automatically exclude anything"), - ('manifest-only', 'o', - "just regenerate the manifest and then stop " - "(implies --force-manifest)"), - ('force-manifest', 'f', - "forcibly regenerate the manifest and carry on as usual. " - "Deprecated: now the manifest is always regenerated."), - ('formats=', None, - "formats for source distribution (comma-separated list)"), - ('keep-temp', 'k', - "keep the distribution tree around after creating " + - "archive file(s)"), - ('dist-dir=', 'd', - "directory to put the source distribution archive(s) in " - "[default: dist]"), - ('metadata-check', None, - "Ensure that all required elements of meta-data " - "are supplied. Warn if any missing. [default]"), - ('owner=', 'u', - "Owner name used when creating a tar file [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file [default: current group]"), - ] - - boolean_options = ['use-defaults', 'prune', - 'manifest-only', 'force-manifest', - 'keep-temp', 'metadata-check'] - - help_options = [ - ('help-formats', None, - "list available distribution formats", show_formats), - ] - - negative_opt = {'no-defaults': 'use-defaults', - 'no-prune': 'prune' } - - sub_commands = [('check', checking_metadata)] - - def initialize_options(self): - # 'template' and 'manifest' are, respectively, the names of - # the manifest template and manifest file. - self.template = None - self.manifest = None - - # 'use_defaults': if true, we will include the default file set - # in the manifest - self.use_defaults = 1 - self.prune = 1 - - self.manifest_only = 0 - self.force_manifest = 0 - - self.formats = ['gztar'] - self.keep_temp = 0 - self.dist_dir = None - - self.archive_files = None - self.metadata_check = 1 - self.owner = None - self.group = None - - def finalize_options(self): - if self.manifest is None: - self.manifest = "MANIFEST" - if self.template is None: - self.template = "MANIFEST.in" - - self.ensure_string_list('formats') - - bad_format = archive_util.check_archive_formats(self.formats) - if bad_format: - raise DistutilsOptionError( - "unknown archive format '%s'" % bad_format) - - if self.dist_dir is None: - self.dist_dir = "dist" - - def run(self): - # 'filelist' contains the list of files that will make up the - # manifest - self.filelist = FileList() - - # Run sub commands - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - # Do whatever it takes to get the list of files to process - # (process the manifest template, read an existing manifest, - # whatever). File list is accumulated in 'self.filelist'. - self.get_file_list() - - # If user just wanted us to regenerate the manifest, stop now. - if self.manifest_only: - return - - # Otherwise, go ahead and create the source distribution tarball, - # or zipfile, or whatever. - self.make_distribution() - - def check_metadata(self): - """Deprecated API.""" - warn("distutils.command.sdist.check_metadata is deprecated, \ - use the check command instead", PendingDeprecationWarning) - check = self.distribution.get_command_obj('check') - check.ensure_finalized() - check.run() - - def get_file_list(self): - """Figure out the list of files to include in the source - distribution, and put it in 'self.filelist'. This might involve - reading the manifest template (and writing the manifest), or just - reading the manifest, or just using the default file set -- it all - depends on the user's options. - """ - # new behavior when using a template: - # the file list is recalculated every time because - # even if MANIFEST.in or setup.py are not changed - # the user might have added some files in the tree that - # need to be included. - # - # This makes --force the default and only behavior with templates. - template_exists = os.path.isfile(self.template) - if not template_exists and self._manifest_is_not_generated(): - self.read_manifest() - self.filelist.sort() - self.filelist.remove_duplicates() - return - - if not template_exists: - self.warn(("manifest template '%s' does not exist " + - "(using default file list)") % - self.template) - self.filelist.findall() - - if self.use_defaults: - self.add_defaults() - - if template_exists: - self.read_template() - - if self.prune: - self.prune_file_list() - - self.filelist.sort() - self.filelist.remove_duplicates() - self.write_manifest() - - def add_defaults(self): - """Add all the default files to self.filelist: - - README or README.txt - - setup.py - - test/test*.py - - all pure Python modules mentioned in setup script - - all files pointed by package_data (build_py) - - all files defined in data_files. - - all files defined as scripts. - - all C sources listed as part of extensions or C libraries - in the setup script (doesn't catch C headers!) - Warns if (README or README.txt) or setup.py are missing; everything - else is optional. - """ - standards = [('README', 'README.txt'), self.distribution.script_name] - for fn in standards: - if isinstance(fn, tuple): - alts = fn - got_it = False - for fn in alts: - if os.path.exists(fn): - got_it = True - self.filelist.append(fn) - break - - if not got_it: - self.warn("standard file not found: should have one of " + - ', '.join(alts)) - else: - if os.path.exists(fn): - self.filelist.append(fn) - else: - self.warn("standard file '%s' not found" % fn) - - optional = ['test/test*.py', 'setup.cfg'] - for pattern in optional: - files = filter(os.path.isfile, glob(pattern)) - self.filelist.extend(files) - - # build_py is used to get: - # - python modules - # - files defined in package_data - build_py = self.get_finalized_command('build_py') - - # getting python files - if self.distribution.has_pure_modules(): - self.filelist.extend(build_py.get_source_files()) - - # getting package_data files - # (computed in build_py.data_files by build_py.finalize_options) - for pkg, src_dir, build_dir, filenames in build_py.data_files: - for filename in filenames: - self.filelist.append(os.path.join(src_dir, filename)) - - # getting distribution.data_files - if self.distribution.has_data_files(): - for item in self.distribution.data_files: - if isinstance(item, str): # plain file - item = convert_path(item) - if os.path.isfile(item): - self.filelist.append(item) - else: # a (dirname, filenames) tuple - dirname, filenames = item - for f in filenames: - f = convert_path(f) - if os.path.isfile(f): - self.filelist.append(f) - - if self.distribution.has_ext_modules(): - build_ext = self.get_finalized_command('build_ext') - self.filelist.extend(build_ext.get_source_files()) - - if self.distribution.has_c_libraries(): - build_clib = self.get_finalized_command('build_clib') - self.filelist.extend(build_clib.get_source_files()) - - if self.distribution.has_scripts(): - build_scripts = self.get_finalized_command('build_scripts') - self.filelist.extend(build_scripts.get_source_files()) - - def read_template(self): - """Read and parse manifest template file named by self.template. - - (usually "MANIFEST.in") The parsing and processing is done by - 'self.filelist', which updates itself accordingly. - """ - log.info("reading manifest template '%s'", self.template) - template = TextFile(self.template, strip_comments=1, skip_blanks=1, - join_lines=1, lstrip_ws=1, rstrip_ws=1, - collapse_join=1) - - try: - while True: - line = template.readline() - if line is None: # end of file - break - - try: - self.filelist.process_template_line(line) - # the call above can raise a DistutilsTemplateError for - # malformed lines, or a ValueError from the lower-level - # convert_path function - except (DistutilsTemplateError, ValueError) as msg: - self.warn("%s, line %d: %s" % (template.filename, - template.current_line, - msg)) - finally: - template.close() - - def prune_file_list(self): - """Prune off branches that might slip into the file list as created - by 'read_template()', but really don't belong there: - * the build tree (typically "build") - * the release tree itself (only an issue if we ran "sdist" - previously with --keep-temp, or it aborted) - * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories - """ - build = self.get_finalized_command('build') - base_dir = self.distribution.get_fullname() - - self.filelist.exclude_pattern(None, prefix=build.build_base) - self.filelist.exclude_pattern(None, prefix=base_dir) - - if sys.platform == 'win32': - seps = r'/|\\' - else: - seps = '/' - - vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr', - '_darcs'] - vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps) - self.filelist.exclude_pattern(vcs_ptrn, is_regex=1) - - def write_manifest(self): - """Write the file list in 'self.filelist' (presumably as filled in - by 'add_defaults()' and 'read_template()') to the manifest file - named by 'self.manifest'. - """ - if self._manifest_is_not_generated(): - log.info("not writing to manually maintained " - "manifest file '%s'" % self.manifest) - return - - content = self.filelist.files[:] - content.insert(0, '# file GENERATED by distutils, do NOT edit') - self.execute(file_util.write_file, (self.manifest, content), - "writing manifest file '%s'" % self.manifest) - - def _manifest_is_not_generated(self): - # check for special comment used in 3.1.3 and higher - if not os.path.isfile(self.manifest): - return False - - fp = open(self.manifest) - try: - first_line = fp.readline() - finally: - fp.close() - return first_line != '# file GENERATED by distutils, do NOT edit\n' - - def read_manifest(self): - """Read the manifest file (named by 'self.manifest') and use it to - fill in 'self.filelist', the list of files to include in the source - distribution. - """ - log.info("reading manifest file '%s'", self.manifest) - manifest = open(self.manifest) - for line in manifest: - # ignore comments and blank lines - line = line.strip() - if line.startswith('#') or not line: - continue - self.filelist.append(line) - manifest.close() - - def make_release_tree(self, base_dir, files): - """Create the directory tree that will become the source - distribution archive. All directories implied by the filenames in - 'files' are created under 'base_dir', and then we hard link or copy - (if hard linking is unavailable) those files into place. - Essentially, this duplicates the developer's source tree, but in a - directory named after the distribution, containing only the files - to be distributed. - """ - # Create all the directories under 'base_dir' necessary to - # put 'files' there; the 'mkpath()' is just so we don't die - # if the manifest happens to be empty. - self.mkpath(base_dir) - dir_util.create_tree(base_dir, files, dry_run=self.dry_run) - - # And walk over the list of files, either making a hard link (if - # os.link exists) to each one that doesn't already exist in its - # corresponding location under 'base_dir', or copying each file - # that's out-of-date in 'base_dir'. (Usually, all files will be - # out-of-date, because by default we blow away 'base_dir' when - # we're done making the distribution archives.) - - if hasattr(os, 'link'): # can make hard links on this system - link = 'hard' - msg = "making hard links in %s..." % base_dir - else: # nope, have to copy - link = None - msg = "copying files to %s..." % base_dir - - if not files: - log.warn("no files to distribute -- empty manifest?") - else: - log.info(msg) - for file in files: - if not os.path.isfile(file): - log.warn("'%s' not a regular file -- skipping", file) - else: - dest = os.path.join(base_dir, file) - self.copy_file(file, dest, link=link) - - self.distribution.metadata.write_pkg_info(base_dir) - - def make_distribution(self): - """Create the source distribution(s). First, we create the release - tree with 'make_release_tree()'; then, we create all required - archive files (according to 'self.formats') from the release tree. - Finally, we clean up by blowing away the release tree (unless - 'self.keep_temp' is true). The list of archive files created is - stored so it can be retrieved later by 'get_archive_files()'. - """ - # Don't warn about missing meta-data here -- should be (and is!) - # done elsewhere. - base_dir = self.distribution.get_fullname() - base_name = os.path.join(self.dist_dir, base_dir) - - self.make_release_tree(base_dir, self.filelist.files) - archive_files = [] # remember names of files we create - # tar archive must be created last to avoid overwrite and remove - if 'tar' in self.formats: - self.formats.append(self.formats.pop(self.formats.index('tar'))) - - for fmt in self.formats: - file = self.make_archive(base_name, fmt, base_dir=base_dir, - owner=self.owner, group=self.group) - archive_files.append(file) - self.distribution.dist_files.append(('sdist', '', file)) - - self.archive_files = archive_files - - if not self.keep_temp: - dir_util.remove_tree(base_dir, dry_run=self.dry_run) - - def get_archive_files(self): - """Return the list of archive files created when the command - was run, or None if the command hasn't run yet. - """ - return self.archive_files diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py deleted file mode 100644 index 32dda359bad..00000000000 --- a/Lib/distutils/command/upload.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -distutils.command.upload - -Implements the Distutils 'upload' subcommand (upload package to a package -index). -""" - -import os -import io -import platform -import hashlib -from base64 import standard_b64encode -from urllib.request import urlopen, Request, HTTPError -from urllib.parse import urlparse -from distutils.errors import DistutilsError, DistutilsOptionError -from distutils.core import PyPIRCCommand -from distutils.spawn import spawn -from distutils import log - -class upload(PyPIRCCommand): - - description = "upload binary package to PyPI" - - user_options = PyPIRCCommand.user_options + [ - ('sign', 's', - 'sign files to upload using gpg'), - ('identity=', 'i', 'GPG identity used to sign files'), - ] - - boolean_options = PyPIRCCommand.boolean_options + ['sign'] - - def initialize_options(self): - PyPIRCCommand.initialize_options(self) - self.username = '' - self.password = '' - self.show_response = 0 - self.sign = False - self.identity = None - - def finalize_options(self): - PyPIRCCommand.finalize_options(self) - if self.identity and not self.sign: - raise DistutilsOptionError( - "Must use --sign for --identity to have meaning" - ) - config = self._read_pypirc() - if config != {}: - self.username = config['username'] - self.password = config['password'] - self.repository = config['repository'] - self.realm = config['realm'] - - # getting the password from the distribution - # if previously set by the register command - if not self.password and self.distribution.password: - self.password = self.distribution.password - - def run(self): - if not self.distribution.dist_files: - msg = ("Must create and upload files in one command " - "(e.g. setup.py sdist upload)") - raise DistutilsOptionError(msg) - for command, pyversion, filename in self.distribution.dist_files: - self.upload_file(command, pyversion, filename) - - def upload_file(self, command, pyversion, filename): - # Makes sure the repository URL is compliant - schema, netloc, url, params, query, fragments = \ - urlparse(self.repository) - if params or query or fragments: - raise AssertionError("Incompatible url %s" % self.repository) - - if schema not in ('http', 'https'): - raise AssertionError("unsupported schema " + schema) - - # Sign if requested - if self.sign: - gpg_args = ["gpg", "--detach-sign", "-a", filename] - if self.identity: - gpg_args[2:2] = ["--local-user", self.identity] - spawn(gpg_args, - dry_run=self.dry_run) - - # Fill in the data - send all the meta-data in case we need to - # register a new release - f = open(filename,'rb') - try: - content = f.read() - finally: - f.close() - meta = self.distribution.metadata - data = { - # action - ':action': 'file_upload', - 'protocol_version': '1', - - # identify release - 'name': meta.get_name(), - 'version': meta.get_version(), - - # file content - 'content': (os.path.basename(filename),content), - 'filetype': command, - 'pyversion': pyversion, - 'md5_digest': hashlib.md5(content).hexdigest(), - - # additional meta-data - 'metadata_version': '1.0', - 'summary': meta.get_description(), - 'home_page': meta.get_url(), - 'author': meta.get_contact(), - 'author_email': meta.get_contact_email(), - 'license': meta.get_licence(), - 'description': meta.get_long_description(), - 'keywords': meta.get_keywords(), - 'platform': meta.get_platforms(), - 'classifiers': meta.get_classifiers(), - 'download_url': meta.get_download_url(), - # PEP 314 - 'provides': meta.get_provides(), - 'requires': meta.get_requires(), - 'obsoletes': meta.get_obsoletes(), - } - comment = '' - if command == 'bdist_rpm': - dist, version, id = platform.dist() - if dist: - comment = 'built for %s %s' % (dist, version) - elif command == 'bdist_dumb': - comment = 'built for %s' % platform.platform(terse=1) - data['comment'] = comment - - if self.sign: - data['gpg_signature'] = (os.path.basename(filename) + ".asc", - open(filename+".asc", "rb").read()) - - # set up the authentication - user_pass = (self.username + ":" + self.password).encode('ascii') - # The exact encoding of the authentication string is debated. - # Anyway PyPI only accepts ascii for both username or password. - auth = "Basic " + standard_b64encode(user_pass).decode('ascii') - - # Build up the MIME payload for the POST data - boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = b'\r\n--' + boundary.encode('ascii') - end_boundary = sep_boundary + b'--\r\n' - body = io.BytesIO() - for key, value in data.items(): - title = '\r\nContent-Disposition: form-data; name="%s"' % key - # handle multiple entries for the same name - if not isinstance(value, list): - value = [value] - for value in value: - if type(value) is tuple: - title += '; filename="%s"' % value[0] - value = value[1] - else: - value = str(value).encode('utf-8') - body.write(sep_boundary) - body.write(title.encode('utf-8')) - body.write(b"\r\n\r\n") - body.write(value) - body.write(end_boundary) - body = body.getvalue() - - msg = "Submitting %s to %s" % (filename, self.repository) - self.announce(msg, log.INFO) - - # build the Request - headers = { - 'Content-type': 'multipart/form-data; boundary=%s' % boundary, - 'Content-length': str(len(body)), - 'Authorization': auth, - } - - request = Request(self.repository, data=body, - headers=headers) - # send the data - try: - result = urlopen(request) - status = result.getcode() - reason = result.msg - except HTTPError as e: - status = e.code - reason = e.msg - except OSError as e: - self.announce(str(e), log.ERROR) - raise - - if status == 200: - self.announce('Server response (%s): %s' % (status, reason), - log.INFO) - if self.show_response: - text = self._read_pypi_response(result) - msg = '\n'.join(('-' * 75, text, '-' * 75)) - self.announce(msg, log.INFO) - else: - msg = 'Upload failed (%s): %s' % (status, reason) - self.announce(msg, log.ERROR) - raise DistutilsError(msg) diff --git a/Lib/distutils/config.py b/Lib/distutils/config.py deleted file mode 100644 index bf8d8dd2f5a..00000000000 --- a/Lib/distutils/config.py +++ /dev/null @@ -1,131 +0,0 @@ -"""distutils.pypirc - -Provides the PyPIRCCommand class, the base class for the command classes -that uses .pypirc in the distutils.command package. -""" -import os -from configparser import RawConfigParser - -from distutils.cmd import Command - -DEFAULT_PYPIRC = """\ -[distutils] -index-servers = - pypi - -[pypi] -username:%s -password:%s -""" - -class PyPIRCCommand(Command): - """Base command that knows how to handle the .pypirc file - """ - DEFAULT_REPOSITORY = 'https://upload.pypi.org/legacy/' - DEFAULT_REALM = 'pypi' - repository = None - realm = None - - user_options = [ - ('repository=', 'r', - "url of repository [default: %s]" % \ - DEFAULT_REPOSITORY), - ('show-response', None, - 'display full response text from server')] - - boolean_options = ['show-response'] - - def _get_rc_file(self): - """Returns rc file path.""" - return os.path.join(os.path.expanduser('~'), '.pypirc') - - def _store_pypirc(self, username, password): - """Creates a default .pypirc file.""" - rc = self._get_rc_file() - with os.fdopen(os.open(rc, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: - f.write(DEFAULT_PYPIRC % (username, password)) - - def _read_pypirc(self): - """Reads the .pypirc file.""" - rc = self._get_rc_file() - if os.path.exists(rc): - self.announce('Using PyPI login from %s' % rc) - repository = self.repository or self.DEFAULT_REPOSITORY - realm = self.realm or self.DEFAULT_REALM - - config = RawConfigParser() - config.read(rc) - sections = config.sections() - if 'distutils' in sections: - # let's get the list of servers - index_servers = config.get('distutils', 'index-servers') - _servers = [server.strip() for server in - index_servers.split('\n') - if server.strip() != ''] - if _servers == []: - # nothing set, let's try to get the default pypi - if 'pypi' in sections: - _servers = ['pypi'] - else: - # the file is not properly defined, returning - # an empty dict - return {} - for server in _servers: - current = {'server': server} - current['username'] = config.get(server, 'username') - - # optional params - for key, default in (('repository', - self.DEFAULT_REPOSITORY), - ('realm', self.DEFAULT_REALM), - ('password', None)): - if config.has_option(server, key): - current[key] = config.get(server, key) - else: - current[key] = default - - # work around people having "repository" for the "pypi" - # section of their config set to the HTTP (rather than - # HTTPS) URL - if (server == 'pypi' and - repository in (self.DEFAULT_REPOSITORY, 'pypi')): - current['repository'] = self.DEFAULT_REPOSITORY - return current - - if (current['server'] == repository or - current['repository'] == repository): - return current - elif 'server-login' in sections: - # old format - server = 'server-login' - if config.has_option(server, 'repository'): - repository = config.get(server, 'repository') - else: - repository = self.DEFAULT_REPOSITORY - return {'username': config.get(server, 'username'), - 'password': config.get(server, 'password'), - 'repository': repository, - 'server': server, - 'realm': self.DEFAULT_REALM} - - return {} - - def _read_pypi_response(self, response): - """Read and decode a PyPI HTTP response.""" - import cgi - content_type = response.getheader('content-type', 'text/plain') - encoding = cgi.parse_header(content_type)[1].get('charset', 'ascii') - return response.read().decode(encoding) - - def initialize_options(self): - """Initialize options.""" - self.repository = None - self.realm = None - self.show_response = 0 - - def finalize_options(self): - """Finalizes options.""" - if self.repository is None: - self.repository = self.DEFAULT_REPOSITORY - if self.realm is None: - self.realm = self.DEFAULT_REALM diff --git a/Lib/distutils/core.py b/Lib/distutils/core.py deleted file mode 100644 index d603d4a45a7..00000000000 --- a/Lib/distutils/core.py +++ /dev/null @@ -1,234 +0,0 @@ -"""distutils.core - -The only module that needs to be imported to use the Distutils; provides -the 'setup' function (which is to be called from the setup script). Also -indirectly provides the Distribution and Command classes, although they are -really defined in distutils.dist and distutils.cmd. -""" - -import os -import sys - -from distutils.debug import DEBUG -from distutils.errors import * - -# Mainly import these so setup scripts can "from distutils.core import" them. -from distutils.dist import Distribution -from distutils.cmd import Command -from distutils.config import PyPIRCCommand -from distutils.extension import Extension - -# This is a barebones help message generated displayed when the user -# runs the setup script with no arguments at all. More useful help -# is generated with various --help options: global help, list commands, -# and per-command help. -USAGE = """\ -usage: %(script)s [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] - or: %(script)s --help [cmd1 cmd2 ...] - or: %(script)s --help-commands - or: %(script)s cmd --help -""" - -def gen_usage (script_name): - script = os.path.basename(script_name) - return USAGE % vars() - - -# Some mild magic to control the behaviour of 'setup()' from 'run_setup()'. -_setup_stop_after = None -_setup_distribution = None - -# Legal keyword arguments for the setup() function -setup_keywords = ('distclass', 'script_name', 'script_args', 'options', - 'name', 'version', 'author', 'author_email', - 'maintainer', 'maintainer_email', 'url', 'license', - 'description', 'long_description', 'keywords', - 'platforms', 'classifiers', 'download_url', - 'requires', 'provides', 'obsoletes', - ) - -# Legal keyword arguments for the Extension constructor -extension_keywords = ('name', 'sources', 'include_dirs', - 'define_macros', 'undef_macros', - 'library_dirs', 'libraries', 'runtime_library_dirs', - 'extra_objects', 'extra_compile_args', 'extra_link_args', - 'swig_opts', 'export_symbols', 'depends', 'language') - -def setup (**attrs): - """The gateway to the Distutils: do everything your setup script needs - to do, in a highly flexible and user-driven way. Briefly: create a - Distribution instance; find and parse config files; parse the command - line; run each Distutils command found there, customized by the options - supplied to 'setup()' (as keyword arguments), in config files, and on - the command line. - - The Distribution instance might be an instance of a class supplied via - the 'distclass' keyword argument to 'setup'; if no such class is - supplied, then the Distribution class (in dist.py) is instantiated. - All other arguments to 'setup' (except for 'cmdclass') are used to set - attributes of the Distribution instance. - - The 'cmdclass' argument, if supplied, is a dictionary mapping command - names to command classes. Each command encountered on the command line - will be turned into a command class, which is in turn instantiated; any - class found in 'cmdclass' is used in place of the default, which is - (for command 'foo_bar') class 'foo_bar' in module - 'distutils.command.foo_bar'. The command class must provide a - 'user_options' attribute which is a list of option specifiers for - 'distutils.fancy_getopt'. Any command-line options between the current - and the next command are used to set attributes of the current command - object. - - When the entire command-line has been successfully parsed, calls the - 'run()' method on each command object in turn. This method will be - driven entirely by the Distribution object (which each command object - has a reference to, thanks to its constructor), and the - command-specific options that became attributes of each command - object. - """ - - global _setup_stop_after, _setup_distribution - - # Determine the distribution class -- either caller-supplied or - # our Distribution (see below). - klass = attrs.get('distclass') - if klass: - del attrs['distclass'] - else: - klass = Distribution - - if 'script_name' not in attrs: - attrs['script_name'] = os.path.basename(sys.argv[0]) - if 'script_args' not in attrs: - attrs['script_args'] = sys.argv[1:] - - # Create the Distribution instance, using the remaining arguments - # (ie. everything except distclass) to initialize it - try: - _setup_distribution = dist = klass(attrs) - except DistutilsSetupError as msg: - if 'name' not in attrs: - raise SystemExit("error in setup command: %s" % msg) - else: - raise SystemExit("error in %s setup command: %s" % \ - (attrs['name'], msg)) - - if _setup_stop_after == "init": - return dist - - # Find and parse the config file(s): they will override options from - # the setup script, but be overridden by the command line. - dist.parse_config_files() - - if DEBUG: - print("options (after parsing config files):") - dist.dump_option_dicts() - - if _setup_stop_after == "config": - return dist - - # Parse the command line and override config files; any - # command-line errors are the end user's fault, so turn them into - # SystemExit to suppress tracebacks. - try: - ok = dist.parse_command_line() - except DistutilsArgError as msg: - raise SystemExit(gen_usage(dist.script_name) + "\nerror: %s" % msg) - - if DEBUG: - print("options (after parsing command line):") - dist.dump_option_dicts() - - if _setup_stop_after == "commandline": - return dist - - # And finally, run all the commands found on the command line. - if ok: - try: - dist.run_commands() - except KeyboardInterrupt: - raise SystemExit("interrupted") - except OSError as exc: - if DEBUG: - sys.stderr.write("error: %s\n" % (exc,)) - raise - else: - raise SystemExit("error: %s" % (exc,)) - - except (DistutilsError, - CCompilerError) as msg: - if DEBUG: - raise - else: - raise SystemExit("error: " + str(msg)) - - return dist - -# setup () - - -def run_setup (script_name, script_args=None, stop_after="run"): - """Run a setup script in a somewhat controlled environment, and - return the Distribution instance that drives things. This is useful - if you need to find out the distribution meta-data (passed as - keyword args from 'script' to 'setup()', or the contents of the - config files or command-line. - - 'script_name' is a file that will be read and run with 'exec()'; - 'sys.argv[0]' will be replaced with 'script' for the duration of the - call. 'script_args' is a list of strings; if supplied, - 'sys.argv[1:]' will be replaced by 'script_args' for the duration of - the call. - - 'stop_after' tells 'setup()' when to stop processing; possible - values: - init - stop after the Distribution instance has been created and - populated with the keyword arguments to 'setup()' - config - stop after config files have been parsed (and their data - stored in the Distribution instance) - commandline - stop after the command-line ('sys.argv[1:]' or 'script_args') - have been parsed (and the data stored in the Distribution) - run [default] - stop after all commands have been run (the same as if 'setup()' - had been called in the usual way - - Returns the Distribution instance, which provides all information - used to drive the Distutils. - """ - if stop_after not in ('init', 'config', 'commandline', 'run'): - raise ValueError("invalid value for 'stop_after': %r" % (stop_after,)) - - global _setup_stop_after, _setup_distribution - _setup_stop_after = stop_after - - save_argv = sys.argv.copy() - g = {'__file__': script_name} - try: - try: - sys.argv[0] = script_name - if script_args is not None: - sys.argv[1:] = script_args - with open(script_name, 'rb') as f: - exec(f.read(), g) - finally: - sys.argv = save_argv - _setup_stop_after = None - except SystemExit: - # Hmm, should we do something if exiting with a non-zero code - # (ie. error)? - pass - - if _setup_distribution is None: - raise RuntimeError(("'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?") % \ - script_name) - - # I wonder if the setup script's namespace -- g and l -- would be of - # any interest to callers? - #print "_setup_distribution:", _setup_distribution - return _setup_distribution - -# run_setup () diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py deleted file mode 100644 index 1c369903477..00000000000 --- a/Lib/distutils/cygwinccompiler.py +++ /dev/null @@ -1,405 +0,0 @@ -"""distutils.cygwinccompiler - -Provides the CygwinCCompiler class, a subclass of UnixCCompiler that -handles the Cygwin port of the GNU C compiler to Windows. It also contains -the Mingw32CCompiler class which handles the mingw32 port of GCC (same as -cygwin in no-cygwin mode). -""" - -# problems: -# -# * if you use a msvc compiled python version (1.5.2) -# 1. you have to insert a __GNUC__ section in its config.h -# 2. you have to generate an import library for its dll -# - create a def-file for python??.dll -# - create an import library using -# dlltool --dllname python15.dll --def python15.def \ -# --output-lib libpython15.a -# -# see also http://starship.python.net/crew/kernr/mingw32/Notes.html -# -# * We put export_symbols in a def-file, and don't use -# --export-all-symbols because it doesn't worked reliable in some -# tested configurations. And because other windows compilers also -# need their symbols specified this no serious problem. -# -# tested configurations: -# -# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works -# (after patching python's config.h and for C++ some other include files) -# see also http://starship.python.net/crew/kernr/mingw32/Notes.html -# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works -# (ld doesn't support -shared, so we use dllwrap) -# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now -# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 -# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html -# - using gcc -mdll instead dllwrap doesn't work without -static because -# it tries to link against dlls instead their import libraries. (If -# it finds the dll first.) -# By specifying -static we force ld to link against the import libraries, -# this is windows standard and there are normally not the necessary symbols -# in the dlls. -# *** only the version of June 2000 shows these problems -# * cygwin gcc 3.2/ld 2.13.90 works -# (ld supports -shared) -# * mingw gcc 3.2/ld 2.13 works -# (ld supports -shared) - -import os -import sys -import copy -from subprocess import Popen, PIPE, check_output -import re - -from distutils.ccompiler import gen_preprocess_options, gen_lib_options -from distutils.unixccompiler import UnixCCompiler -from distutils.file_util import write_file -from distutils.errors import (DistutilsExecError, CCompilerError, - CompileError, UnknownFileError) -from distutils import log -from distutils.version import LooseVersion -from distutils.spawn import find_executable - -def get_msvcr(): - """Include the appropriate MSVC runtime library if Python was built - with MSVC 7.0 or later. - """ - msc_pos = sys.version.find('MSC v.') - if msc_pos != -1: - msc_ver = sys.version[msc_pos+6:msc_pos+10] - if msc_ver == '1300': - # MSVC 7.0 - return ['msvcr70'] - elif msc_ver == '1310': - # MSVC 7.1 - return ['msvcr71'] - elif msc_ver == '1400': - # VS2005 / MSVC 8.0 - return ['msvcr80'] - elif msc_ver == '1500': - # VS2008 / MSVC 9.0 - return ['msvcr90'] - elif msc_ver == '1600': - # VS2010 / MSVC 10.0 - return ['msvcr100'] - else: - raise ValueError("Unknown MS Compiler version %s " % msc_ver) - - -class CygwinCCompiler(UnixCCompiler): - """ Handles the Cygwin port of the GNU C compiler to Windows. - """ - compiler_type = 'cygwin' - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".dll" - static_lib_format = "lib%s%s" - shared_lib_format = "%s%s" - exe_extension = ".exe" - - def __init__(self, verbose=0, dry_run=0, force=0): - - UnixCCompiler.__init__(self, verbose, dry_run, force) - - status, details = check_config_h() - self.debug_print("Python's GCC status: %s (details: %s)" % - (status, details)) - if status is not CONFIG_H_OK: - self.warn( - "Python's pyconfig.h doesn't seem to support your compiler. " - "Reason: %s. " - "Compiling may fail because of undefined preprocessor macros." - % details) - - self.gcc_version, self.ld_version, self.dllwrap_version = \ - get_versions() - self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % - (self.gcc_version, - self.ld_version, - self.dllwrap_version) ) - - # ld_version >= "2.10.90" and < "2.13" should also be able to use - # gcc -mdll instead of dllwrap - # Older dllwraps had own version numbers, newer ones use the - # same as the rest of binutils ( also ld ) - # dllwrap 2.10.90 is buggy - if self.ld_version >= "2.10.90": - self.linker_dll = "gcc" - else: - self.linker_dll = "dllwrap" - - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: - shared_option = "-mdll -static" - - # Hard-code GCC because that's what this is all about. - # XXX optimization, warnings etc. should be customizable. - self.set_executables(compiler='gcc -mcygwin -O -Wall', - compiler_so='gcc -mcygwin -mdll -O -Wall', - compiler_cxx='g++ -mcygwin -O -Wall', - linker_exe='gcc -mcygwin', - linker_so=('%s -mcygwin %s' % - (self.linker_dll, shared_option))) - - # cygwin and mingw32 need different sets of libraries - if self.gcc_version == "2.91.57": - # cygwin shouldn't need msvcrt, but without the dlls will crash - # (gcc version 2.91.57) -- perhaps something about initialization - self.dll_libraries=["msvcrt"] - self.warn( - "Consider upgrading to a newer version of gcc") - else: - # Include the appropriate MSVC runtime library if Python was built - # with MSVC 7.0 or later. - self.dll_libraries = get_msvcr() - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - """Compiles the source by spawning GCC and windres if needed.""" - if ext == '.rc' or ext == '.res': - # gcc needs '.res' and '.rc' compiled to object files !!! - try: - self.spawn(["windres", "-i", src, "-o", obj]) - except DistutilsExecError as msg: - raise CompileError(msg) - else: # for other files use the C-compiler - try: - self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def link(self, target_desc, objects, output_filename, output_dir=None, - libraries=None, library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - """Link the objects.""" - # use separate copies, so we can modify the lists - extra_preargs = copy.copy(extra_preargs or []) - libraries = copy.copy(libraries or []) - objects = copy.copy(objects or []) - - # Additional libraries - libraries.extend(self.dll_libraries) - - # handle export symbols by creating a def-file - # with executables this only works with gcc/ld as linker - if ((export_symbols is not None) and - (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): - # (The linker doesn't do anything if output is up-to-date. - # So it would probably better to check if we really need this, - # but for this we had to insert some unchanged parts of - # UnixCCompiler, and this is not what we want.) - - # we want to put some files in the same directory as the - # object files are, build_temp doesn't help much - # where are the object files - temp_dir = os.path.dirname(objects[0]) - # name of dll to give the helper files the same base name - (dll_name, dll_extension) = os.path.splitext( - os.path.basename(output_filename)) - - # generate the filenames for these files - def_file = os.path.join(temp_dir, dll_name + ".def") - lib_file = os.path.join(temp_dir, 'lib' + dll_name + ".a") - - # Generate .def file - contents = [ - "LIBRARY %s" % os.path.basename(output_filename), - "EXPORTS"] - for sym in export_symbols: - contents.append(sym) - self.execute(write_file, (def_file, contents), - "writing %s" % def_file) - - # next add options for def-file and to creating import libraries - - # dllwrap uses different options than gcc/ld - if self.linker_dll == "dllwrap": - extra_preargs.extend(["--output-lib", lib_file]) - # for dllwrap we have to use a special option - extra_preargs.extend(["--def", def_file]) - # we use gcc/ld here and can be sure ld is >= 2.9.10 - else: - # doesn't work: bfd_close build\...\libfoo.a: Invalid operation - #extra_preargs.extend(["-Wl,--out-implib,%s" % lib_file]) - # for gcc/ld the def-file is specified as any object files - objects.append(def_file) - - #end: if ((export_symbols is not None) and - # (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): - - # who wants symbols and a many times larger output file - # should explicitly switch the debug mode on - # otherwise we let dllwrap/ld strip the output file - # (On my machine: 10KB < stripped_file < ??100KB - # unstripped_file = stripped_file + XXX KB - # ( XXX=254 for a typical python extension)) - if not debug: - extra_preargs.append("-s") - - UnixCCompiler.link(self, target_desc, objects, output_filename, - output_dir, libraries, library_dirs, - runtime_library_dirs, - None, # export_symbols, we do this in our def-file - debug, extra_preargs, extra_postargs, build_temp, - target_lang) - - # -- Miscellaneous methods ----------------------------------------- - - def object_filenames(self, source_filenames, strip_dir=0, output_dir=''): - """Adds supports for rc and res files.""" - if output_dir is None: - output_dir = '' - obj_names = [] - for src_name in source_filenames: - # use normcase to make sure '.rc' is really '.rc' and not '.RC' - base, ext = os.path.splitext(os.path.normcase(src_name)) - if ext not in (self.src_extensions + ['.rc','.res']): - raise UnknownFileError("unknown file type '%s' (from '%s')" % \ - (ext, src_name)) - if strip_dir: - base = os.path.basename (base) - if ext in ('.res', '.rc'): - # these need to be compiled to object files - obj_names.append (os.path.join(output_dir, - base + ext + self.obj_extension)) - else: - obj_names.append (os.path.join(output_dir, - base + self.obj_extension)) - return obj_names - -# the same as cygwin plus some additional parameters -class Mingw32CCompiler(CygwinCCompiler): - """ Handles the Mingw32 port of the GNU C compiler to Windows. - """ - compiler_type = 'mingw32' - - def __init__(self, verbose=0, dry_run=0, force=0): - - CygwinCCompiler.__init__ (self, verbose, dry_run, force) - - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: - shared_option = "-mdll -static" - - # A real mingw32 doesn't need to specify a different entry point, - # but cygwin 2.91.57 in no-cygwin-mode needs it. - if self.gcc_version <= "2.91.57": - entry_point = '--entry _DllMain@12' - else: - entry_point = '' - - if is_cygwingcc(): - raise CCompilerError( - 'Cygwin gcc cannot be used with --compiler=mingw32') - - self.set_executables(compiler='gcc -O -Wall', - compiler_so='gcc -mdll -O -Wall', - compiler_cxx='g++ -O -Wall', - linker_exe='gcc', - linker_so='%s %s %s' - % (self.linker_dll, shared_option, - entry_point)) - # Maybe we should also append -mthreads, but then the finished - # dlls need another dll (mingwm10.dll see Mingw32 docs) - # (-mthreads: Support thread-safe exception handling on `Mingw32') - - # no additional libraries needed - self.dll_libraries=[] - - # Include the appropriate MSVC runtime library if Python was built - # with MSVC 7.0 or later. - self.dll_libraries = get_msvcr() - -# Because these compilers aren't configured in Python's pyconfig.h file by -# default, we should at least warn the user if he is using an unmodified -# version. - -CONFIG_H_OK = "ok" -CONFIG_H_NOTOK = "not ok" -CONFIG_H_UNCERTAIN = "uncertain" - -def check_config_h(): - """Check if the current Python installation appears amenable to building - extensions with GCC. - - Returns a tuple (status, details), where 'status' is one of the following - constants: - - - CONFIG_H_OK: all is well, go ahead and compile - - CONFIG_H_NOTOK: doesn't look good - - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h - - 'details' is a human-readable string explaining the situation. - - Note there are two ways to conclude "OK": either 'sys.version' contains - the string "GCC" (implying that this Python was built with GCC), or the - installed "pyconfig.h" contains the string "__GNUC__". - """ - - # XXX since this function also checks sys.version, it's not strictly a - # "pyconfig.h" check -- should probably be renamed... - - from distutils import sysconfig - - # if sys.version contains GCC then python was compiled with GCC, and the - # pyconfig.h file should be OK - if "GCC" in sys.version: - return CONFIG_H_OK, "sys.version mentions 'GCC'" - - # let's see if __GNUC__ is mentioned in python.h - fn = sysconfig.get_config_h_filename() - try: - config_h = open(fn) - try: - if "__GNUC__" in config_h.read(): - return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn - else: - return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn - finally: - config_h.close() - except OSError as exc: - return (CONFIG_H_UNCERTAIN, - "couldn't read '%s': %s" % (fn, exc.strerror)) - -RE_VERSION = re.compile(br'(\d+\.\d+(\.\d+)*)') - -def _find_exe_version(cmd): - """Find the version of an executable by running `cmd` in the shell. - - If the command is not found, or the output does not match - `RE_VERSION`, returns None. - """ - executable = cmd.split()[0] - if find_executable(executable) is None: - return None - out = Popen(cmd, shell=True, stdout=PIPE).stdout - try: - out_string = out.read() - finally: - out.close() - result = RE_VERSION.search(out_string) - if result is None: - return None - # LooseVersion works with strings - # so we need to decode our bytes - return LooseVersion(result.group(1).decode()) - -def get_versions(): - """ Try to find out the versions of gcc, ld and dllwrap. - - If not possible it returns None for it. - """ - commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] - return tuple([_find_exe_version(cmd) for cmd in commands]) - -def is_cygwingcc(): - '''Try to determine if the gcc that would be used is from cygwin.''' - out_string = check_output(['gcc', '-dumpmachine']) - return out_string.strip().endswith(b'cygwin') diff --git a/Lib/distutils/debug.py b/Lib/distutils/debug.py deleted file mode 100644 index daf1660f0d8..00000000000 --- a/Lib/distutils/debug.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - -# If DISTUTILS_DEBUG is anything other than the empty string, we run in -# debug mode. -DEBUG = os.environ.get('DISTUTILS_DEBUG') diff --git a/Lib/distutils/dep_util.py b/Lib/distutils/dep_util.py deleted file mode 100644 index d74f5e4e92f..00000000000 --- a/Lib/distutils/dep_util.py +++ /dev/null @@ -1,92 +0,0 @@ -"""distutils.dep_util - -Utility functions for simple, timestamp-based dependency of files -and groups of files; also, function based entirely on such -timestamp dependency analysis.""" - -import os -from distutils.errors import DistutilsFileError - - -def newer (source, target): - """Return true if 'source' exists and is more recently modified than - 'target', or if 'source' exists and 'target' doesn't. Return false if - both exist and 'target' is the same age or younger than 'source'. - Raise DistutilsFileError if 'source' does not exist. - """ - if not os.path.exists(source): - raise DistutilsFileError("file '%s' does not exist" % - os.path.abspath(source)) - if not os.path.exists(target): - return 1 - - from stat import ST_MTIME - mtime1 = os.stat(source)[ST_MTIME] - mtime2 = os.stat(target)[ST_MTIME] - - return mtime1 > mtime2 - -# newer () - - -def newer_pairwise (sources, targets): - """Walk two filename lists in parallel, testing if each source is newer - than its corresponding target. Return a pair of lists (sources, - targets) where source is newer than target, according to the semantics - of 'newer()'. - """ - if len(sources) != len(targets): - raise ValueError("'sources' and 'targets' must be same length") - - # build a pair of lists (sources, targets) where source is newer - n_sources = [] - n_targets = [] - for i in range(len(sources)): - if newer(sources[i], targets[i]): - n_sources.append(sources[i]) - n_targets.append(targets[i]) - - return (n_sources, n_targets) - -# newer_pairwise () - - -def newer_group (sources, target, missing='error'): - """Return true if 'target' is out-of-date with respect to any file - listed in 'sources'. In other words, if 'target' exists and is newer - than every file in 'sources', return false; otherwise return true. - 'missing' controls what we do when a source file is missing; the - default ("error") is to blow up with an OSError from inside 'stat()'; - if it is "ignore", we silently drop any missing source files; if it is - "newer", any missing source files make us assume that 'target' is - out-of-date (this is handy in "dry-run" mode: it'll make you pretend to - carry out commands that wouldn't work because inputs are missing, but - that doesn't matter because you're not actually going to run the - commands). - """ - # If the target doesn't even exist, then it's definitely out-of-date. - if not os.path.exists(target): - return 1 - - # Otherwise we have to find out the hard way: if *any* source file - # is more recent than 'target', then 'target' is out-of-date and - # we can immediately return true. If we fall through to the end - # of the loop, then 'target' is up-to-date and we return false. - from stat import ST_MTIME - target_mtime = os.stat(target)[ST_MTIME] - for source in sources: - if not os.path.exists(source): - if missing == 'error': # blow up when we stat() the file - pass - elif missing == 'ignore': # missing source dropped from - continue # target's dependency list - elif missing == 'newer': # missing source means target is - return 1 # out-of-date - - source_mtime = os.stat(source)[ST_MTIME] - if source_mtime > target_mtime: - return 1 - else: - return 0 - -# newer_group () diff --git a/Lib/distutils/dir_util.py b/Lib/distutils/dir_util.py deleted file mode 100644 index df4d751c942..00000000000 --- a/Lib/distutils/dir_util.py +++ /dev/null @@ -1,223 +0,0 @@ -"""distutils.dir_util - -Utility functions for manipulating directories and directory trees.""" - -import os -import errno -from distutils.errors import DistutilsFileError, DistutilsInternalError -from distutils import log - -# cache for by mkpath() -- in addition to cheapening redundant calls, -# eliminates redundant "creating /foo/bar/baz" messages in dry-run mode -_path_created = {} - -# I don't use os.makedirs because a) it's new to Python 1.5.2, and -# b) it blows up if the directory already exists (I want to silently -# succeed in that case). -def mkpath(name, mode=0o777, verbose=1, dry_run=0): - """Create a directory and any missing ancestor directories. - - If the directory already exists (or if 'name' is the empty string, which - means the current directory, which of course exists), then do nothing. - Raise DistutilsFileError if unable to create some directory along the way - (eg. some sub-path exists, but is a file rather than a directory). - If 'verbose' is true, print a one-line summary of each mkdir to stdout. - Return the list of directories actually created. - """ - - global _path_created - - # Detect a common bug -- name is None - if not isinstance(name, str): - raise DistutilsInternalError( - "mkpath: 'name' must be a string (got %r)" % (name,)) - - # XXX what's the better way to handle verbosity? print as we create - # each directory in the path (the current behaviour), or only announce - # the creation of the whole path? (quite easy to do the latter since - # we're not using a recursive algorithm) - - name = os.path.normpath(name) - created_dirs = [] - if os.path.isdir(name) or name == '': - return created_dirs - if _path_created.get(os.path.abspath(name)): - return created_dirs - - (head, tail) = os.path.split(name) - tails = [tail] # stack of lone dirs to create - - while head and tail and not os.path.isdir(head): - (head, tail) = os.path.split(head) - tails.insert(0, tail) # push next higher dir onto stack - - # now 'head' contains the deepest directory that already exists - # (that is, the child of 'head' in 'name' is the highest directory - # that does *not* exist) - for d in tails: - #print "head = %s, d = %s: " % (head, d), - head = os.path.join(head, d) - abs_head = os.path.abspath(head) - - if _path_created.get(abs_head): - continue - - if verbose >= 1: - log.info("creating %s", head) - - if not dry_run: - try: - os.mkdir(head, mode) - except OSError as exc: - if not (exc.errno == errno.EEXIST and os.path.isdir(head)): - raise DistutilsFileError( - "could not create '%s': %s" % (head, exc.args[-1])) - created_dirs.append(head) - - _path_created[abs_head] = 1 - return created_dirs - -def create_tree(base_dir, files, mode=0o777, verbose=1, dry_run=0): - """Create all the empty directories under 'base_dir' needed to put 'files' - there. - - 'base_dir' is just the name of a directory which doesn't necessarily - exist yet; 'files' is a list of filenames to be interpreted relative to - 'base_dir'. 'base_dir' + the directory portion of every file in 'files' - will be created if it doesn't already exist. 'mode', 'verbose' and - 'dry_run' flags are as for 'mkpath()'. - """ - # First get the list of directories to create - need_dir = set() - for file in files: - need_dir.add(os.path.join(base_dir, os.path.dirname(file))) - - # Now create them - for dir in sorted(need_dir): - mkpath(dir, mode, verbose=verbose, dry_run=dry_run) - -import sysconfig -_multiarch = None - -def copy_tree(src, dst, preserve_mode=1, preserve_times=1, - preserve_symlinks=0, update=0, verbose=1, dry_run=0): - """Copy an entire directory tree 'src' to a new location 'dst'. - - Both 'src' and 'dst' must be directory names. If 'src' is not a - directory, raise DistutilsFileError. If 'dst' does not exist, it is - created with 'mkpath()'. The end result of the copy is that every - file in 'src' is copied to 'dst', and directories under 'src' are - recursively copied to 'dst'. Return the list of files that were - copied or might have been copied, using their output name. The - return value is unaffected by 'update' or 'dry_run': it is simply - the list of all files under 'src', with the names changed to be - under 'dst'. - - 'preserve_mode' and 'preserve_times' are the same as for - 'copy_file'; note that they only apply to regular files, not to - directories. If 'preserve_symlinks' is true, symlinks will be - copied as symlinks (on platforms that support them!); otherwise - (the default), the destination of the symlink will be copied. - 'update' and 'verbose' are the same as for 'copy_file'. - """ - from distutils.file_util import copy_file - - if not dry_run and not os.path.isdir(src): - raise DistutilsFileError( - "cannot copy tree '%s': not a directory" % src) - try: - names = os.listdir(src) - except OSError as e: - if dry_run: - names = [] - else: - raise DistutilsFileError( - "error listing files in '%s': %s" % (src, e.strerror)) - - ext_suffix = sysconfig.get_config_var ('EXT_SUFFIX') - _multiarch = sysconfig.get_config_var ('MULTIARCH') - if ext_suffix.endswith(_multiarch + ext_suffix[-3:]): - new_suffix = None - else: - new_suffix = "%s-%s%s" % (ext_suffix[:-3], _multiarch, ext_suffix[-3:]) - - if not dry_run: - mkpath(dst, verbose=verbose) - - outputs = [] - - for n in names: - src_name = os.path.join(src, n) - dst_name = os.path.join(dst, n) - if new_suffix and _multiarch and n.endswith(ext_suffix) and not n.endswith(new_suffix): - dst_name = os.path.join(dst, n.replace(ext_suffix, new_suffix)) - log.info("renaming extension %s -> %s", n, n.replace(ext_suffix, new_suffix)) - - if n.startswith('.nfs'): - # skip NFS rename files - continue - - if preserve_symlinks and os.path.islink(src_name): - link_dest = os.readlink(src_name) - if verbose >= 1: - log.info("linking %s -> %s", dst_name, link_dest) - if not dry_run: - os.symlink(link_dest, dst_name) - outputs.append(dst_name) - - elif os.path.isdir(src_name): - outputs.extend( - copy_tree(src_name, dst_name, preserve_mode, - preserve_times, preserve_symlinks, update, - verbose=verbose, dry_run=dry_run)) - else: - copy_file(src_name, dst_name, preserve_mode, - preserve_times, update, verbose=verbose, - dry_run=dry_run) - outputs.append(dst_name) - - return outputs - -def _build_cmdtuple(path, cmdtuples): - """Helper for remove_tree().""" - for f in os.listdir(path): - real_f = os.path.join(path,f) - if os.path.isdir(real_f) and not os.path.islink(real_f): - _build_cmdtuple(real_f, cmdtuples) - else: - cmdtuples.append((os.remove, real_f)) - cmdtuples.append((os.rmdir, path)) - -def remove_tree(directory, verbose=1, dry_run=0): - """Recursively remove an entire directory tree. - - Any errors are ignored (apart from being reported to stdout if 'verbose' - is true). - """ - global _path_created - - if verbose >= 1: - log.info("removing '%s' (and everything under it)", directory) - if dry_run: - return - cmdtuples = [] - _build_cmdtuple(directory, cmdtuples) - for cmd in cmdtuples: - try: - cmd[0](cmd[1]) - # remove dir from cache if it's already there - abspath = os.path.abspath(cmd[1]) - if abspath in _path_created: - del _path_created[abspath] - except OSError as exc: - log.warn("error removing %s: %s", directory, exc) - -def ensure_relative(path): - """Take the full path 'path', and make it a relative path. - - This is useful to make 'path' the second argument to os.path.join(). - """ - drive, path = os.path.splitdrive(path) - if path[0:1] == os.sep: - path = drive + path[1:] - return path diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py deleted file mode 100644 index 62a24516cfa..00000000000 --- a/Lib/distutils/dist.py +++ /dev/null @@ -1,1236 +0,0 @@ -"""distutils.dist - -Provides the Distribution class, which represents the module distribution -being built/installed/distributed. -""" - -import sys -import os -import re -from email import message_from_file - -try: - import warnings -except ImportError: - warnings = None - -from distutils.errors import * -from distutils.fancy_getopt import FancyGetopt, translate_longopt -from distutils.util import check_environ, strtobool, rfc822_escape -from distutils import log -from distutils.debug import DEBUG - -# Regex to define acceptable Distutils command names. This is not *quite* -# the same as a Python NAME -- I don't allow leading underscores. The fact -# that they're very similar is no coincidence; the default naming scheme is -# to look for a Python module named after the command. -command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') - - -class Distribution: - """The core of the Distutils. Most of the work hiding behind 'setup' - is really done within a Distribution instance, which farms the work out - to the Distutils commands specified on the command line. - - Setup scripts will almost never instantiate Distribution directly, - unless the 'setup()' function is totally inadequate to their needs. - However, it is conceivable that a setup script might wish to subclass - Distribution for some specialized purpose, and then pass the subclass - to 'setup()' as the 'distclass' keyword argument. If so, it is - necessary to respect the expectations that 'setup' has of Distribution. - See the code for 'setup()', in core.py, for details. - """ - - # 'global_options' describes the command-line options that may be - # supplied to the setup script prior to any actual commands. - # Eg. "./setup.py -n" or "./setup.py --quiet" both take advantage of - # these global options. This list should be kept to a bare minimum, - # since every global option is also valid as a command option -- and we - # don't want to pollute the commands with too many options that they - # have minimal control over. - # The fourth entry for verbose means that it can be repeated. - global_options = [ - ('verbose', 'v', "run verbosely (default)", 1), - ('quiet', 'q', "run quietly (turns verbosity off)"), - ('dry-run', 'n', "don't actually do anything"), - ('help', 'h', "show detailed help message"), - ('no-user-cfg', None, - 'ignore pydistutils.cfg in your home directory'), - ] - - # 'common_usage' is a short (2-3 line) string describing the common - # usage of the setup script. - common_usage = """\ -Common commands: (see '--help-commands' for more) - - setup.py build will build the package underneath 'build/' - setup.py install will install the package -""" - - # options that are not propagated to the commands - display_options = [ - ('help-commands', None, - "list all available commands"), - ('name', None, - "print package name"), - ('version', 'V', - "print package version"), - ('fullname', None, - "print -"), - ('author', None, - "print the author's name"), - ('author-email', None, - "print the author's email address"), - ('maintainer', None, - "print the maintainer's name"), - ('maintainer-email', None, - "print the maintainer's email address"), - ('contact', None, - "print the maintainer's name if known, else the author's"), - ('contact-email', None, - "print the maintainer's email address if known, else the author's"), - ('url', None, - "print the URL for this package"), - ('license', None, - "print the license of the package"), - ('licence', None, - "alias for --license"), - ('description', None, - "print the package description"), - ('long-description', None, - "print the long package description"), - ('platforms', None, - "print the list of platforms"), - ('classifiers', None, - "print the list of classifiers"), - ('keywords', None, - "print the list of keywords"), - ('provides', None, - "print the list of packages/modules provided"), - ('requires', None, - "print the list of packages/modules required"), - ('obsoletes', None, - "print the list of packages/modules made obsolete") - ] - display_option_names = [translate_longopt(x[0]) for x in display_options] - - # negative options are options that exclude other options - negative_opt = {'quiet': 'verbose'} - - # -- Creation/initialization methods ------------------------------- - - def __init__(self, attrs=None): - """Construct a new Distribution instance: initialize all the - attributes of a Distribution, and then use 'attrs' (a dictionary - mapping attribute names to values) to assign some of those - attributes their "real" values. (Any attributes not mentioned in - 'attrs' will be assigned to some null value: 0, None, an empty list - or dictionary, etc.) Most importantly, initialize the - 'command_obj' attribute to the empty dictionary; this will be - filled in with real command objects by 'parse_command_line()'. - """ - - # Default values for our command-line options - self.verbose = 1 - self.dry_run = 0 - self.help = 0 - for attr in self.display_option_names: - setattr(self, attr, 0) - - # Store the distribution meta-data (name, version, author, and so - # forth) in a separate object -- we're getting to have enough - # information here (and enough command-line options) that it's - # worth it. Also delegate 'get_XXX()' methods to the 'metadata' - # object in a sneaky and underhanded (but efficient!) way. - self.metadata = DistributionMetadata() - for basename in self.metadata._METHOD_BASENAMES: - method_name = "get_" + basename - setattr(self, method_name, getattr(self.metadata, method_name)) - - # 'cmdclass' maps command names to class objects, so we - # can 1) quickly figure out which class to instantiate when - # we need to create a new command object, and 2) have a way - # for the setup script to override command classes - self.cmdclass = {} - - # 'command_packages' is a list of packages in which commands - # are searched for. The factory for command 'foo' is expected - # to be named 'foo' in the module 'foo' in one of the packages - # named here. This list is searched from the left; an error - # is raised if no named package provides the command being - # searched for. (Always access using get_command_packages().) - self.command_packages = None - - # 'script_name' and 'script_args' are usually set to sys.argv[0] - # and sys.argv[1:], but they can be overridden when the caller is - # not necessarily a setup script run from the command-line. - self.script_name = None - self.script_args = None - - # 'command_options' is where we store command options between - # parsing them (from config files, the command-line, etc.) and when - # they are actually needed -- ie. when the command in question is - # instantiated. It is a dictionary of dictionaries of 2-tuples: - # command_options = { command_name : { option : (source, value) } } - self.command_options = {} - - # 'dist_files' is the list of (command, pyversion, file) that - # have been created by any dist commands run so far. This is - # filled regardless of whether the run is dry or not. pyversion - # gives sysconfig.get_python_version() if the dist file is - # specific to a Python version, 'any' if it is good for all - # Python versions on the target platform, and '' for a source - # file. pyversion should not be used to specify minimum or - # maximum required Python versions; use the metainfo for that - # instead. - self.dist_files = [] - - # These options are really the business of various commands, rather - # than of the Distribution itself. We provide aliases for them in - # Distribution as a convenience to the developer. - self.packages = None - self.package_data = {} - self.package_dir = None - self.py_modules = None - self.libraries = None - self.headers = None - self.ext_modules = None - self.ext_package = None - self.include_dirs = None - self.extra_path = None - self.scripts = None - self.data_files = None - self.password = '' - - # And now initialize bookkeeping stuff that can't be supplied by - # the caller at all. 'command_obj' maps command names to - # Command instances -- that's how we enforce that every command - # class is a singleton. - self.command_obj = {} - - # 'have_run' maps command names to boolean values; it keeps track - # of whether we have actually run a particular command, to make it - # cheap to "run" a command whenever we think we might need to -- if - # it's already been done, no need for expensive filesystem - # operations, we just check the 'have_run' dictionary and carry on. - # It's only safe to query 'have_run' for a command class that has - # been instantiated -- a false value will be inserted when the - # command object is created, and replaced with a true value when - # the command is successfully run. Thus it's probably best to use - # '.get()' rather than a straight lookup. - self.have_run = {} - - # Now we'll use the attrs dictionary (ultimately, keyword args from - # the setup script) to possibly override any or all of these - # distribution options. - - if attrs: - # Pull out the set of command options and work on them - # specifically. Note that this order guarantees that aliased - # command options will override any supplied redundantly - # through the general options dictionary. - options = attrs.get('options') - if options is not None: - del attrs['options'] - for (command, cmd_options) in options.items(): - opt_dict = self.get_option_dict(command) - for (opt, val) in cmd_options.items(): - opt_dict[opt] = ("setup script", val) - - if 'licence' in attrs: - attrs['license'] = attrs['licence'] - del attrs['licence'] - msg = "'licence' distribution option is deprecated; use 'license'" - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") - - # Now work on the rest of the attributes. Any attribute that's - # not already defined is invalid! - for (key, val) in attrs.items(): - if hasattr(self.metadata, "set_" + key): - getattr(self.metadata, "set_" + key)(val) - elif hasattr(self.metadata, key): - setattr(self.metadata, key, val) - elif hasattr(self, key): - setattr(self, key, val) - else: - msg = "Unknown distribution option: %s" % repr(key) - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") - - # no-user-cfg is handled before other command line args - # because other args override the config files, and this - # one is needed before we can load the config files. - # If attrs['script_args'] wasn't passed, assume false. - # - # This also make sure we just look at the global options - self.want_user_cfg = True - - if self.script_args is not None: - for arg in self.script_args: - if not arg.startswith('-'): - break - if arg == '--no-user-cfg': - self.want_user_cfg = False - break - - self.finalize_options() - - def get_option_dict(self, command): - """Get the option dictionary for a given command. If that - command's option dictionary hasn't been created yet, then create it - and return the new dictionary; otherwise, return the existing - option dictionary. - """ - dict = self.command_options.get(command) - if dict is None: - dict = self.command_options[command] = {} - return dict - - def dump_option_dicts(self, header=None, commands=None, indent=""): - from pprint import pformat - - if commands is None: # dump all command option dicts - commands = sorted(self.command_options.keys()) - - if header is not None: - self.announce(indent + header) - indent = indent + " " - - if not commands: - self.announce(indent + "no commands known yet") - return - - for cmd_name in commands: - opt_dict = self.command_options.get(cmd_name) - if opt_dict is None: - self.announce(indent + - "no option dict for '%s' command" % cmd_name) - else: - self.announce(indent + - "option dict for '%s' command:" % cmd_name) - out = pformat(opt_dict) - for line in out.split('\n'): - self.announce(indent + " " + line) - - # -- Config file finding/parsing methods --------------------------- - - def find_config_files(self): - """Find as many configuration files as should be processed for this - platform, and return a list of filenames in the order in which they - should be parsed. The filenames returned are guaranteed to exist - (modulo nasty race conditions). - - There are three possible config files: distutils.cfg in the - Distutils installation directory (ie. where the top-level - Distutils __inst__.py file lives), a file in the user's home - directory named .pydistutils.cfg on Unix and pydistutils.cfg - on Windows/Mac; and setup.cfg in the current directory. - - The file in the user's home directory can be disabled with the - --no-user-cfg option. - """ - files = [] - check_environ() - - # Where to look for the system-wide Distutils config file - sys_dir = os.path.dirname(sys.modules['distutils'].__file__) - - # Look for the system config file - sys_file = os.path.join(sys_dir, "distutils.cfg") - if os.path.isfile(sys_file): - files.append(sys_file) - - # What to call the per-user config file - if os.name == 'posix': - user_filename = ".pydistutils.cfg" - else: - user_filename = "pydistutils.cfg" - - # And look for the user config file - if self.want_user_cfg: - user_file = os.path.join(os.path.expanduser('~'), user_filename) - if os.path.isfile(user_file): - files.append(user_file) - - # All platforms support local setup.cfg - local_file = "setup.cfg" - if os.path.isfile(local_file): - files.append(local_file) - - if DEBUG: - self.announce("using config files: %s" % ', '.join(files)) - - return files - - def parse_config_files(self, filenames=None): - from configparser import ConfigParser - - # Ignore install directory options if we have a venv - if sys.prefix != sys.base_prefix: - ignore_options = [ - 'install-base', 'install-platbase', 'install-lib', - 'install-platlib', 'install-purelib', 'install-headers', - 'install-scripts', 'install-data', 'prefix', 'exec-prefix', - 'home', 'user', 'root'] - else: - ignore_options = [] - - ignore_options = frozenset(ignore_options) - - if filenames is None: - filenames = self.find_config_files() - - if DEBUG: - self.announce("Distribution.parse_config_files():") - - parser = ConfigParser() - for filename in filenames: - if DEBUG: - self.announce(" reading %s" % filename) - parser.read(filename) - for section in parser.sections(): - options = parser.options(section) - opt_dict = self.get_option_dict(section) - - for opt in options: - if opt != '__name__' and opt not in ignore_options: - val = parser.get(section,opt) - opt = opt.replace('-', '_') - opt_dict[opt] = (filename, val) - - # Make the ConfigParser forget everything (so we retain - # the original filenames that options come from) - parser.__init__() - - # If there was a "global" section in the config file, use it - # to set Distribution options. - - if 'global' in self.command_options: - for (opt, (src, val)) in self.command_options['global'].items(): - alias = self.negative_opt.get(opt) - try: - if alias: - setattr(self, alias, not strtobool(val)) - elif opt in ('verbose', 'dry_run'): # ugh! - setattr(self, opt, strtobool(val)) - else: - setattr(self, opt, val) - except ValueError as msg: - raise DistutilsOptionError(msg) - - # -- Command-line parsing methods ---------------------------------- - - def parse_command_line(self): - """Parse the setup script's command line, taken from the - 'script_args' instance attribute (which defaults to 'sys.argv[1:]' - -- see 'setup()' in core.py). This list is first processed for - "global options" -- options that set attributes of the Distribution - instance. Then, it is alternately scanned for Distutils commands - and options for that command. Each new command terminates the - options for the previous command. The allowed options for a - command are determined by the 'user_options' attribute of the - command class -- thus, we have to be able to load command classes - in order to parse the command line. Any error in that 'options' - attribute raises DistutilsGetoptError; any error on the - command-line raises DistutilsArgError. If no Distutils commands - were found on the command line, raises DistutilsArgError. Return - true if command-line was successfully parsed and we should carry - on with executing commands; false if no errors but we shouldn't - execute commands (currently, this only happens if user asks for - help). - """ - # - # We now have enough information to show the Macintosh dialog - # that allows the user to interactively specify the "command line". - # - toplevel_options = self._get_toplevel_options() - - # We have to parse the command line a bit at a time -- global - # options, then the first command, then its options, and so on -- - # because each command will be handled by a different class, and - # the options that are valid for a particular class aren't known - # until we have loaded the command class, which doesn't happen - # until we know what the command is. - - self.commands = [] - parser = FancyGetopt(toplevel_options + self.display_options) - parser.set_negative_aliases(self.negative_opt) - parser.set_aliases({'licence': 'license'}) - args = parser.getopt(args=self.script_args, object=self) - option_order = parser.get_option_order() - log.set_verbosity(self.verbose) - - # for display options we return immediately - if self.handle_display_options(option_order): - return - while args: - args = self._parse_command_opts(parser, args) - if args is None: # user asked for help (and got it) - return - - # Handle the cases of --help as a "global" option, ie. - # "setup.py --help" and "setup.py --help command ...". For the - # former, we show global options (--verbose, --dry-run, etc.) - # and display-only options (--name, --version, etc.); for the - # latter, we omit the display-only options and show help for - # each command listed on the command line. - if self.help: - self._show_help(parser, - display_options=len(self.commands) == 0, - commands=self.commands) - return - - # Oops, no commands found -- an end-user error - if not self.commands: - raise DistutilsArgError("no commands supplied") - - # All is well: return true - return True - - def _get_toplevel_options(self): - """Return the non-display options recognized at the top level. - - This includes options that are recognized *only* at the top - level as well as options recognized for commands. - """ - return self.global_options + [ - ("command-packages=", None, - "list of packages that provide distutils commands"), - ] - - def _parse_command_opts(self, parser, args): - """Parse the command-line options for a single command. - 'parser' must be a FancyGetopt instance; 'args' must be the list - of arguments, starting with the current command (whose options - we are about to parse). Returns a new version of 'args' with - the next command at the front of the list; will be the empty - list if there are no more commands on the command line. Returns - None if the user asked for help on this command. - """ - # late import because of mutual dependence between these modules - from distutils.cmd import Command - - # Pull the current command from the head of the command line - command = args[0] - if not command_re.match(command): - raise SystemExit("invalid command name '%s'" % command) - self.commands.append(command) - - # Dig up the command class that implements this command, so we - # 1) know that it's a valid command, and 2) know which options - # it takes. - try: - cmd_class = self.get_command_class(command) - except DistutilsModuleError as msg: - raise DistutilsArgError(msg) - - # Require that the command class be derived from Command -- want - # to be sure that the basic "command" interface is implemented. - if not issubclass(cmd_class, Command): - raise DistutilsClassError( - "command class %s must subclass Command" % cmd_class) - - # Also make sure that the command object provides a list of its - # known options. - if not (hasattr(cmd_class, 'user_options') and - isinstance(cmd_class.user_options, list)): - msg = ("command class %s must provide " - "'user_options' attribute (a list of tuples)") - raise DistutilsClassError(msg % cmd_class) - - # If the command class has a list of negative alias options, - # merge it in with the global negative aliases. - negative_opt = self.negative_opt - if hasattr(cmd_class, 'negative_opt'): - negative_opt = negative_opt.copy() - negative_opt.update(cmd_class.negative_opt) - - # Check for help_options in command class. They have a different - # format (tuple of four) so we need to preprocess them here. - if (hasattr(cmd_class, 'help_options') and - isinstance(cmd_class.help_options, list)): - help_options = fix_help_options(cmd_class.help_options) - else: - help_options = [] - - # All commands support the global options too, just by adding - # in 'global_options'. - parser.set_option_table(self.global_options + - cmd_class.user_options + - help_options) - parser.set_negative_aliases(negative_opt) - (args, opts) = parser.getopt(args[1:]) - if hasattr(opts, 'help') and opts.help: - self._show_help(parser, display_options=0, commands=[cmd_class]) - return - - if (hasattr(cmd_class, 'help_options') and - isinstance(cmd_class.help_options, list)): - help_option_found=0 - for (help_option, short, desc, func) in cmd_class.help_options: - if hasattr(opts, parser.get_attr_name(help_option)): - help_option_found=1 - if callable(func): - func() - else: - raise DistutilsClassError( - "invalid help function %r for help option '%s': " - "must be a callable object (function, etc.)" - % (func, help_option)) - - if help_option_found: - return - - # Put the options from the command-line into their official - # holding pen, the 'command_options' dictionary. - opt_dict = self.get_option_dict(command) - for (name, value) in vars(opts).items(): - opt_dict[name] = ("command line", value) - - return args - - def finalize_options(self): - """Set final values for all the options on the Distribution - instance, analogous to the .finalize_options() method of Command - objects. - """ - for attr in ('keywords', 'platforms'): - value = getattr(self.metadata, attr) - if value is None: - continue - if isinstance(value, str): - value = [elm.strip() for elm in value.split(',')] - setattr(self.metadata, attr, value) - - def _show_help(self, parser, global_options=1, display_options=1, - commands=[]): - """Show help for the setup script command-line in the form of - several lists of command-line options. 'parser' should be a - FancyGetopt instance; do not expect it to be returned in the - same state, as its option table will be reset to make it - generate the correct help text. - - If 'global_options' is true, lists the global options: - --verbose, --dry-run, etc. If 'display_options' is true, lists - the "display-only" options: --name, --version, etc. Finally, - lists per-command help for every command name or command class - in 'commands'. - """ - # late import because of mutual dependence between these modules - from distutils.core import gen_usage - from distutils.cmd import Command - - if global_options: - if display_options: - options = self._get_toplevel_options() - else: - options = self.global_options - parser.set_option_table(options) - parser.print_help(self.common_usage + "\nGlobal options:") - print('') - - if display_options: - parser.set_option_table(self.display_options) - parser.print_help( - "Information display options (just display " + - "information, ignore any commands)") - print('') - - for command in self.commands: - if isinstance(command, type) and issubclass(command, Command): - klass = command - else: - klass = self.get_command_class(command) - if (hasattr(klass, 'help_options') and - isinstance(klass.help_options, list)): - parser.set_option_table(klass.user_options + - fix_help_options(klass.help_options)) - else: - parser.set_option_table(klass.user_options) - parser.print_help("Options for '%s' command:" % klass.__name__) - print('') - - print(gen_usage(self.script_name)) - - def handle_display_options(self, option_order): - """If there were any non-global "display-only" options - (--help-commands or the metadata display options) on the command - line, display the requested info and return true; else return - false. - """ - from distutils.core import gen_usage - - # User just wants a list of commands -- we'll print it out and stop - # processing now (ie. if they ran "setup --help-commands foo bar", - # we ignore "foo bar"). - if self.help_commands: - self.print_commands() - print('') - print(gen_usage(self.script_name)) - return 1 - - # If user supplied any of the "display metadata" options, then - # display that metadata in the order in which the user supplied the - # metadata options. - any_display_options = 0 - is_display_option = {} - for option in self.display_options: - is_display_option[option[0]] = 1 - - for (opt, val) in option_order: - if val and is_display_option.get(opt): - opt = translate_longopt(opt) - value = getattr(self.metadata, "get_"+opt)() - if opt in ['keywords', 'platforms']: - print(','.join(value)) - elif opt in ('classifiers', 'provides', 'requires', - 'obsoletes'): - print('\n'.join(value)) - else: - print(value) - any_display_options = 1 - - return any_display_options - - def print_command_list(self, commands, header, max_length): - """Print a subset of the list of all commands -- used by - 'print_commands()'. - """ - print(header + ":") - - for cmd in commands: - klass = self.cmdclass.get(cmd) - if not klass: - klass = self.get_command_class(cmd) - try: - description = klass.description - except AttributeError: - description = "(no description available)" - - print(" %-*s %s" % (max_length, cmd, description)) - - def print_commands(self): - """Print out a help message listing all available commands with a - description of each. The list is divided into "standard commands" - (listed in distutils.command.__all__) and "extra commands" - (mentioned in self.cmdclass, but not a standard command). The - descriptions come from the command class attribute - 'description'. - """ - import distutils.command - std_commands = distutils.command.__all__ - is_std = {} - for cmd in std_commands: - is_std[cmd] = 1 - - extra_commands = [] - for cmd in self.cmdclass.keys(): - if not is_std.get(cmd): - extra_commands.append(cmd) - - max_length = 0 - for cmd in (std_commands + extra_commands): - if len(cmd) > max_length: - max_length = len(cmd) - - self.print_command_list(std_commands, - "Standard commands", - max_length) - if extra_commands: - print() - self.print_command_list(extra_commands, - "Extra commands", - max_length) - - def get_command_list(self): - """Get a list of (command, description) tuples. - The list is divided into "standard commands" (listed in - distutils.command.__all__) and "extra commands" (mentioned in - self.cmdclass, but not a standard command). The descriptions come - from the command class attribute 'description'. - """ - # Currently this is only used on Mac OS, for the Mac-only GUI - # Distutils interface (by Jack Jansen) - import distutils.command - std_commands = distutils.command.__all__ - is_std = {} - for cmd in std_commands: - is_std[cmd] = 1 - - extra_commands = [] - for cmd in self.cmdclass.keys(): - if not is_std.get(cmd): - extra_commands.append(cmd) - - rv = [] - for cmd in (std_commands + extra_commands): - klass = self.cmdclass.get(cmd) - if not klass: - klass = self.get_command_class(cmd) - try: - description = klass.description - except AttributeError: - description = "(no description available)" - rv.append((cmd, description)) - return rv - - # -- Command class/object methods ---------------------------------- - - def get_command_packages(self): - """Return a list of packages from which commands are loaded.""" - pkgs = self.command_packages - if not isinstance(pkgs, list): - if pkgs is None: - pkgs = '' - pkgs = [pkg.strip() for pkg in pkgs.split(',') if pkg != ''] - if "distutils.command" not in pkgs: - pkgs.insert(0, "distutils.command") - self.command_packages = pkgs - return pkgs - - def get_command_class(self, command): - """Return the class that implements the Distutils command named by - 'command'. First we check the 'cmdclass' dictionary; if the - command is mentioned there, we fetch the class object from the - dictionary and return it. Otherwise we load the command module - ("distutils.command." + command) and fetch the command class from - the module. The loaded class is also stored in 'cmdclass' - to speed future calls to 'get_command_class()'. - - Raises DistutilsModuleError if the expected module could not be - found, or if that module does not define the expected class. - """ - klass = self.cmdclass.get(command) - if klass: - return klass - - for pkgname in self.get_command_packages(): - module_name = "%s.%s" % (pkgname, command) - klass_name = command - - try: - __import__(module_name) - module = sys.modules[module_name] - except ImportError: - continue - - try: - klass = getattr(module, klass_name) - except AttributeError: - raise DistutilsModuleError( - "invalid command '%s' (no class '%s' in module '%s')" - % (command, klass_name, module_name)) - - self.cmdclass[command] = klass - return klass - - raise DistutilsModuleError("invalid command '%s'" % command) - - def get_command_obj(self, command, create=1): - """Return the command object for 'command'. Normally this object - is cached on a previous call to 'get_command_obj()'; if no command - object for 'command' is in the cache, then we either create and - return it (if 'create' is true) or return None. - """ - cmd_obj = self.command_obj.get(command) - if not cmd_obj and create: - if DEBUG: - self.announce("Distribution.get_command_obj(): " - "creating '%s' command object" % command) - - klass = self.get_command_class(command) - cmd_obj = self.command_obj[command] = klass(self) - self.have_run[command] = 0 - - # Set any options that were supplied in config files - # or on the command line. (NB. support for error - # reporting is lame here: any errors aren't reported - # until 'finalize_options()' is called, which means - # we won't report the source of the error.) - options = self.command_options.get(command) - if options: - self._set_command_options(cmd_obj, options) - - return cmd_obj - - def _set_command_options(self, command_obj, option_dict=None): - """Set the options for 'command_obj' from 'option_dict'. Basically - this means copying elements of a dictionary ('option_dict') to - attributes of an instance ('command'). - - 'command_obj' must be a Command instance. If 'option_dict' is not - supplied, uses the standard option dictionary for this command - (from 'self.command_options'). - """ - command_name = command_obj.get_command_name() - if option_dict is None: - option_dict = self.get_option_dict(command_name) - - if DEBUG: - self.announce(" setting options for '%s' command:" % command_name) - for (option, (source, value)) in option_dict.items(): - if DEBUG: - self.announce(" %s = %s (from %s)" % (option, value, - source)) - try: - bool_opts = [translate_longopt(o) - for o in command_obj.boolean_options] - except AttributeError: - bool_opts = [] - try: - neg_opt = command_obj.negative_opt - except AttributeError: - neg_opt = {} - - try: - is_string = isinstance(value, str) - if option in neg_opt and is_string: - setattr(command_obj, neg_opt[option], not strtobool(value)) - elif option in bool_opts and is_string: - setattr(command_obj, option, strtobool(value)) - elif hasattr(command_obj, option): - setattr(command_obj, option, value) - else: - raise DistutilsOptionError( - "error in %s: command '%s' has no such option '%s'" - % (source, command_name, option)) - except ValueError as msg: - raise DistutilsOptionError(msg) - - def reinitialize_command(self, command, reinit_subcommands=0): - """Reinitializes a command to the state it was in when first - returned by 'get_command_obj()': ie., initialized but not yet - finalized. This provides the opportunity to sneak option - values in programmatically, overriding or supplementing - user-supplied values from the config files and command line. - You'll have to re-finalize the command object (by calling - 'finalize_options()' or 'ensure_finalized()') before using it for - real. - - 'command' should be a command name (string) or command object. If - 'reinit_subcommands' is true, also reinitializes the command's - sub-commands, as declared by the 'sub_commands' class attribute (if - it has one). See the "install" command for an example. Only - reinitializes the sub-commands that actually matter, ie. those - whose test predicates return true. - - Returns the reinitialized command object. - """ - from distutils.cmd import Command - if not isinstance(command, Command): - command_name = command - command = self.get_command_obj(command_name) - else: - command_name = command.get_command_name() - - if not command.finalized: - return command - command.initialize_options() - command.finalized = 0 - self.have_run[command_name] = 0 - self._set_command_options(command) - - if reinit_subcommands: - for sub in command.get_sub_commands(): - self.reinitialize_command(sub, reinit_subcommands) - - return command - - # -- Methods that operate on the Distribution ---------------------- - - def announce(self, msg, level=log.INFO): - log.log(level, msg) - - def run_commands(self): - """Run each command that was seen on the setup script command line. - Uses the list of commands found and cache of command objects - created by 'get_command_obj()'. - """ - for cmd in self.commands: - self.run_command(cmd) - - # -- Methods that operate on its Commands -------------------------- - - def run_command(self, command): - """Do whatever it takes to run a command (including nothing at all, - if the command has already been run). Specifically: if we have - already created and run the command named by 'command', return - silently without doing anything. If the command named by 'command' - doesn't even have a command object yet, create one. Then invoke - 'run()' on that command object (or an existing one). - """ - # Already been here, done that? then return silently. - if self.have_run.get(command): - return - - log.info("running %s", command) - cmd_obj = self.get_command_obj(command) - cmd_obj.ensure_finalized() - cmd_obj.run() - self.have_run[command] = 1 - - # -- Distribution query methods ------------------------------------ - - def has_pure_modules(self): - return len(self.packages or self.py_modules or []) > 0 - - def has_ext_modules(self): - return self.ext_modules and len(self.ext_modules) > 0 - - def has_c_libraries(self): - return self.libraries and len(self.libraries) > 0 - - def has_modules(self): - return self.has_pure_modules() or self.has_ext_modules() - - def has_headers(self): - return self.headers and len(self.headers) > 0 - - def has_scripts(self): - return self.scripts and len(self.scripts) > 0 - - def has_data_files(self): - return self.data_files and len(self.data_files) > 0 - - def is_pure(self): - return (self.has_pure_modules() and - not self.has_ext_modules() and - not self.has_c_libraries()) - - # -- Metadata query methods ---------------------------------------- - - # If you're looking for 'get_name()', 'get_version()', and so forth, - # they are defined in a sneaky way: the constructor binds self.get_XXX - # to self.metadata.get_XXX. The actual code is in the - # DistributionMetadata class, below. - -class DistributionMetadata: - """Dummy class to hold the distribution meta-data: name, version, - author, and so forth. - """ - - _METHOD_BASENAMES = ("name", "version", "author", "author_email", - "maintainer", "maintainer_email", "url", - "license", "description", "long_description", - "keywords", "platforms", "fullname", "contact", - "contact_email", "classifiers", "download_url", - # PEP 314 - "provides", "requires", "obsoletes", - ) - - def __init__(self, path=None): - if path is not None: - self.read_pkg_file(open(path)) - else: - self.name = None - self.version = None - self.author = None - self.author_email = None - self.maintainer = None - self.maintainer_email = None - self.url = None - self.license = None - self.description = None - self.long_description = None - self.keywords = None - self.platforms = None - self.classifiers = None - self.download_url = None - # PEP 314 - self.provides = None - self.requires = None - self.obsoletes = None - - def read_pkg_file(self, file): - """Reads the metadata values from a file object.""" - msg = message_from_file(file) - - def _read_field(name): - value = msg[name] - if value == 'UNKNOWN': - return None - return value - - def _read_list(name): - values = msg.get_all(name, None) - if values == []: - return None - return values - - metadata_version = msg['metadata-version'] - self.name = _read_field('name') - self.version = _read_field('version') - self.description = _read_field('summary') - # we are filling author only. - self.author = _read_field('author') - self.maintainer = None - self.author_email = _read_field('author-email') - self.maintainer_email = None - self.url = _read_field('home-page') - self.license = _read_field('license') - - if 'download-url' in msg: - self.download_url = _read_field('download-url') - else: - self.download_url = None - - self.long_description = _read_field('description') - self.description = _read_field('summary') - - if 'keywords' in msg: - self.keywords = _read_field('keywords').split(',') - - self.platforms = _read_list('platform') - self.classifiers = _read_list('classifier') - - # PEP 314 - these fields only exist in 1.1 - if metadata_version == '1.1': - self.requires = _read_list('requires') - self.provides = _read_list('provides') - self.obsoletes = _read_list('obsoletes') - else: - self.requires = None - self.provides = None - self.obsoletes = None - - def write_pkg_info(self, base_dir): - """Write the PKG-INFO file into the release tree. - """ - with open(os.path.join(base_dir, 'PKG-INFO'), 'w', - encoding='UTF-8') as pkg_info: - self.write_pkg_file(pkg_info) - - def write_pkg_file(self, file): - """Write the PKG-INFO format data to a file object. - """ - version = '1.0' - if (self.provides or self.requires or self.obsoletes or - self.classifiers or self.download_url): - version = '1.1' - - file.write('Metadata-Version: %s\n' % version) - file.write('Name: %s\n' % self.get_name()) - file.write('Version: %s\n' % self.get_version()) - file.write('Summary: %s\n' % self.get_description()) - file.write('Home-page: %s\n' % self.get_url()) - file.write('Author: %s\n' % self.get_contact()) - file.write('Author-email: %s\n' % self.get_contact_email()) - file.write('License: %s\n' % self.get_license()) - if self.download_url: - file.write('Download-URL: %s\n' % self.download_url) - - long_desc = rfc822_escape(self.get_long_description()) - file.write('Description: %s\n' % long_desc) - - keywords = ','.join(self.get_keywords()) - if keywords: - file.write('Keywords: %s\n' % keywords) - - self._write_list(file, 'Platform', self.get_platforms()) - self._write_list(file, 'Classifier', self.get_classifiers()) - - # PEP 314 - self._write_list(file, 'Requires', self.get_requires()) - self._write_list(file, 'Provides', self.get_provides()) - self._write_list(file, 'Obsoletes', self.get_obsoletes()) - - def _write_list(self, file, name, values): - for value in values: - file.write('%s: %s\n' % (name, value)) - - # -- Metadata query methods ---------------------------------------- - - def get_name(self): - return self.name or "UNKNOWN" - - def get_version(self): - return self.version or "0.0.0" - - def get_fullname(self): - return "%s-%s" % (self.get_name(), self.get_version()) - - def get_author(self): - return self.author or "UNKNOWN" - - def get_author_email(self): - return self.author_email or "UNKNOWN" - - def get_maintainer(self): - return self.maintainer or "UNKNOWN" - - def get_maintainer_email(self): - return self.maintainer_email or "UNKNOWN" - - def get_contact(self): - return self.maintainer or self.author or "UNKNOWN" - - def get_contact_email(self): - return self.maintainer_email or self.author_email or "UNKNOWN" - - def get_url(self): - return self.url or "UNKNOWN" - - def get_license(self): - return self.license or "UNKNOWN" - get_licence = get_license - - def get_description(self): - return self.description or "UNKNOWN" - - def get_long_description(self): - return self.long_description or "UNKNOWN" - - def get_keywords(self): - return self.keywords or [] - - def get_platforms(self): - return self.platforms or ["UNKNOWN"] - - def get_classifiers(self): - return self.classifiers or [] - - def get_download_url(self): - return self.download_url or "UNKNOWN" - - # PEP 314 - def get_requires(self): - return self.requires or [] - - def set_requires(self, value): - import distutils.versionpredicate - for v in value: - distutils.versionpredicate.VersionPredicate(v) - self.requires = value - - def get_provides(self): - return self.provides or [] - - def set_provides(self, value): - value = [v.strip() for v in value] - for v in value: - import distutils.versionpredicate - distutils.versionpredicate.split_provision(v) - self.provides = value - - def get_obsoletes(self): - return self.obsoletes or [] - - def set_obsoletes(self, value): - import distutils.versionpredicate - for v in value: - distutils.versionpredicate.VersionPredicate(v) - self.obsoletes = value - -def fix_help_options(options): - """Convert a 4-tuple 'help_options' list as found in various command - classes to the 3-tuple form required by FancyGetopt. - """ - new_options = [] - for help_tuple in options: - new_options.append(help_tuple[0:3]) - return new_options diff --git a/Lib/distutils/errors.py b/Lib/distutils/errors.py deleted file mode 100644 index 8b93059e19f..00000000000 --- a/Lib/distutils/errors.py +++ /dev/null @@ -1,97 +0,0 @@ -"""distutils.errors - -Provides exceptions used by the Distutils modules. Note that Distutils -modules may raise standard exceptions; in particular, SystemExit is -usually raised for errors that are obviously the end-user's fault -(eg. bad command-line arguments). - -This module is safe to use in "from ... import *" mode; it only exports -symbols whose names start with "Distutils" and end with "Error".""" - -class DistutilsError (Exception): - """The root of all Distutils evil.""" - pass - -class DistutilsModuleError (DistutilsError): - """Unable to load an expected module, or to find an expected class - within some module (in particular, command modules and classes).""" - pass - -class DistutilsClassError (DistutilsError): - """Some command class (or possibly distribution class, if anyone - feels a need to subclass Distribution) is found not to be holding - up its end of the bargain, ie. implementing some part of the - "command "interface.""" - pass - -class DistutilsGetoptError (DistutilsError): - """The option table provided to 'fancy_getopt()' is bogus.""" - pass - -class DistutilsArgError (DistutilsError): - """Raised by fancy_getopt in response to getopt.error -- ie. an - error in the command line usage.""" - pass - -class DistutilsFileError (DistutilsError): - """Any problems in the filesystem: expected file not found, etc. - Typically this is for problems that we detect before OSError - could be raised.""" - pass - -class DistutilsOptionError (DistutilsError): - """Syntactic/semantic errors in command options, such as use of - mutually conflicting options, or inconsistent options, - badly-spelled values, etc. No distinction is made between option - values originating in the setup script, the command line, config - files, or what-have-you -- but if we *know* something originated in - the setup script, we'll raise DistutilsSetupError instead.""" - pass - -class DistutilsSetupError (DistutilsError): - """For errors that can be definitely blamed on the setup script, - such as invalid keyword arguments to 'setup()'.""" - pass - -class DistutilsPlatformError (DistutilsError): - """We don't know how to do something on the current platform (but - we do know how to do it on some platform) -- eg. trying to compile - C files on a platform not supported by a CCompiler subclass.""" - pass - -class DistutilsExecError (DistutilsError): - """Any problems executing an external program (such as the C - compiler, when compiling C files).""" - pass - -class DistutilsInternalError (DistutilsError): - """Internal inconsistencies or impossibilities (obviously, this - should never be seen if the code is working!).""" - pass - -class DistutilsTemplateError (DistutilsError): - """Syntax error in a file list template.""" - -class DistutilsByteCompileError(DistutilsError): - """Byte compile error.""" - -# Exception classes used by the CCompiler implementation classes -class CCompilerError (Exception): - """Some compile/link operation failed.""" - -class PreprocessError (CCompilerError): - """Failure to preprocess one or more C/C++ files.""" - -class CompileError (CCompilerError): - """Failure to compile one or more C/C++ source files.""" - -class LibError (CCompilerError): - """Failure to create a static library from one or more C/C++ object - files.""" - -class LinkError (CCompilerError): - """Failure to link one or more C/C++ object files into an executable - or shared library file.""" - -class UnknownFileError (CCompilerError): - """Attempt to process an unknown file type.""" diff --git a/Lib/distutils/extension.py b/Lib/distutils/extension.py deleted file mode 100644 index c507da360aa..00000000000 --- a/Lib/distutils/extension.py +++ /dev/null @@ -1,240 +0,0 @@ -"""distutils.extension - -Provides the Extension class, used to describe C/C++ extension -modules in setup scripts.""" - -import os -import warnings - -# This class is really only used by the "build_ext" command, so it might -# make sense to put it in distutils.command.build_ext. However, that -# module is already big enough, and I want to make this class a bit more -# complex to simplify some common cases ("foo" module in "foo.c") and do -# better error-checking ("foo.c" actually exists). -# -# Also, putting this in build_ext.py means every setup script would have to -# import that large-ish module (indirectly, through distutils.core) in -# order to do anything. - -class Extension: - """Just a collection of attributes that describes an extension - module and everything needed to build it (hopefully in a portable - way, but there are hooks that let you be as unportable as you need). - - Instance attributes: - name : string - the full name of the extension, including any packages -- ie. - *not* a filename or pathname, but Python dotted name - sources : [string] - list of source filenames, relative to the distribution root - (where the setup script lives), in Unix form (slash-separated) - for portability. Source files may be C, C++, SWIG (.i), - platform-specific resource files, or whatever else is recognized - by the "build_ext" command as source for a Python extension. - include_dirs : [string] - list of directories to search for C/C++ header files (in Unix - form for portability) - define_macros : [(name : string, value : string|None)] - list of macros to define; each macro is defined using a 2-tuple, - where 'value' is either the string to define it to or None to - define it without a particular value (equivalent of "#define - FOO" in source or -DFOO on Unix C compiler command line) - undef_macros : [string] - list of macros to undefine explicitly - library_dirs : [string] - list of directories to search for C/C++ libraries at link time - libraries : [string] - list of library names (not filenames or paths) to link against - runtime_library_dirs : [string] - list of directories to search for C/C++ libraries at run time - (for shared extensions, this is when the extension is loaded) - extra_objects : [string] - list of extra files to link with (eg. object files not implied - by 'sources', static library that must be explicitly specified, - binary resource files, etc.) - extra_compile_args : [string] - any extra platform- and compiler-specific information to use - when compiling the source files in 'sources'. For platforms and - compilers where "command line" makes sense, this is typically a - list of command-line arguments, but for other platforms it could - be anything. - extra_link_args : [string] - any extra platform- and compiler-specific information to use - when linking object files together to create the extension (or - to create a new static Python interpreter). Similar - interpretation as for 'extra_compile_args'. - export_symbols : [string] - list of symbols to be exported from a shared extension. Not - used on all platforms, and not generally necessary for Python - extensions, which typically export exactly one symbol: "init" + - extension_name. - swig_opts : [string] - any extra options to pass to SWIG if a source file has the .i - extension. - depends : [string] - list of files that the extension depends on - language : string - extension language (i.e. "c", "c++", "objc"). Will be detected - from the source extensions if not provided. - optional : boolean - specifies that a build failure in the extension should not abort the - build process, but simply not install the failing extension. - """ - - # When adding arguments to this constructor, be sure to update - # setup_keywords in core.py. - def __init__(self, name, sources, - include_dirs=None, - define_macros=None, - undef_macros=None, - library_dirs=None, - libraries=None, - runtime_library_dirs=None, - extra_objects=None, - extra_compile_args=None, - extra_link_args=None, - export_symbols=None, - swig_opts = None, - depends=None, - language=None, - optional=None, - **kw # To catch unknown keywords - ): - if not isinstance(name, str): - raise AssertionError("'name' must be a string") - if not (isinstance(sources, list) and - all(isinstance(v, str) for v in sources)): - raise AssertionError("'sources' must be a list of strings") - - self.name = name - self.sources = sources - self.include_dirs = include_dirs or [] - self.define_macros = define_macros or [] - self.undef_macros = undef_macros or [] - self.library_dirs = library_dirs or [] - self.libraries = libraries or [] - self.runtime_library_dirs = runtime_library_dirs or [] - self.extra_objects = extra_objects or [] - self.extra_compile_args = extra_compile_args or [] - self.extra_link_args = extra_link_args or [] - self.export_symbols = export_symbols or [] - self.swig_opts = swig_opts or [] - self.depends = depends or [] - self.language = language - self.optional = optional - - # If there are unknown keyword options, warn about them - if len(kw) > 0: - options = [repr(option) for option in kw] - options = ', '.join(sorted(options)) - msg = "Unknown Extension options: %s" % options - warnings.warn(msg) - - def __repr__(self): - return '<%s.%s(%r) at %#x>' % ( - self.__class__.__module__, - self.__class__.__qualname__, - self.name, - id(self)) - - -def read_setup_file(filename): - """Reads a Setup file and returns Extension instances.""" - from distutils.sysconfig import (parse_makefile, expand_makefile_vars, - _variable_rx) - - from distutils.text_file import TextFile - from distutils.util import split_quoted - - # First pass over the file to gather "VAR = VALUE" assignments. - vars = parse_makefile(filename) - - # Second pass to gobble up the real content: lines of the form - # ... [ ...] [ ...] [ ...] - file = TextFile(filename, - strip_comments=1, skip_blanks=1, join_lines=1, - lstrip_ws=1, rstrip_ws=1) - try: - extensions = [] - - while True: - line = file.readline() - if line is None: # eof - break - if _variable_rx.match(line): # VAR=VALUE, handled in first pass - continue - - if line[0] == line[-1] == "*": - file.warn("'%s' lines not handled yet" % line) - continue - - line = expand_makefile_vars(line, vars) - words = split_quoted(line) - - # NB. this parses a slightly different syntax than the old - # makesetup script: here, there must be exactly one extension per - # line, and it must be the first word of the line. I have no idea - # why the old syntax supported multiple extensions per line, as - # they all wind up being the same. - - module = words[0] - ext = Extension(module, []) - append_next_word = None - - for word in words[1:]: - if append_next_word is not None: - append_next_word.append(word) - append_next_word = None - continue - - suffix = os.path.splitext(word)[1] - switch = word[0:2] ; value = word[2:] - - if suffix in (".c", ".cc", ".cpp", ".cxx", ".c++", ".m", ".mm"): - # hmm, should we do something about C vs. C++ sources? - # or leave it up to the CCompiler implementation to - # worry about? - ext.sources.append(word) - elif switch == "-I": - ext.include_dirs.append(value) - elif switch == "-D": - equals = value.find("=") - if equals == -1: # bare "-DFOO" -- no value - ext.define_macros.append((value, None)) - else: # "-DFOO=blah" - ext.define_macros.append((value[0:equals], - value[equals+2:])) - elif switch == "-U": - ext.undef_macros.append(value) - elif switch == "-C": # only here 'cause makesetup has it! - ext.extra_compile_args.append(word) - elif switch == "-l": - ext.libraries.append(value) - elif switch == "-L": - ext.library_dirs.append(value) - elif switch == "-R": - ext.runtime_library_dirs.append(value) - elif word == "-rpath": - append_next_word = ext.runtime_library_dirs - elif word == "-Xlinker": - append_next_word = ext.extra_link_args - elif word == "-Xcompiler": - append_next_word = ext.extra_compile_args - elif switch == "-u": - ext.extra_link_args.append(word) - if not value: - append_next_word = ext.extra_link_args - elif suffix in (".a", ".so", ".sl", ".o", ".dylib"): - # NB. a really faithful emulation of makesetup would - # append a .o file to extra_objects only if it - # had a slash in it; otherwise, it would s/.o/.c/ - # and append it to sources. Hmmmm. - ext.extra_objects.append(word) - else: - file.warn("unrecognized argument '%s'" % word) - - extensions.append(ext) - finally: - file.close() - - return extensions diff --git a/Lib/distutils/fancy_getopt.py b/Lib/distutils/fancy_getopt.py deleted file mode 100644 index 7d170dd2773..00000000000 --- a/Lib/distutils/fancy_getopt.py +++ /dev/null @@ -1,457 +0,0 @@ -"""distutils.fancy_getopt - -Wrapper around the standard getopt module that provides the following -additional features: - * short and long options are tied together - * options have help strings, so fancy_getopt could potentially - create a complete usage summary - * options set attributes of a passed-in object -""" - -import sys, string, re -import getopt -from distutils.errors import * - -# Much like command_re in distutils.core, this is close to but not quite -# the same as a Python NAME -- except, in the spirit of most GNU -# utilities, we use '-' in place of '_'. (The spirit of LISP lives on!) -# The similarities to NAME are again not a coincidence... -longopt_pat = r'[a-zA-Z](?:[a-zA-Z0-9-]*)' -longopt_re = re.compile(r'^%s$' % longopt_pat) - -# For recognizing "negative alias" options, eg. "quiet=!verbose" -neg_alias_re = re.compile("^(%s)=!(%s)$" % (longopt_pat, longopt_pat)) - -# This is used to translate long options to legitimate Python identifiers -# (for use as attributes of some object). -longopt_xlate = str.maketrans('-', '_') - -class FancyGetopt: - """Wrapper around the standard 'getopt()' module that provides some - handy extra functionality: - * short and long options are tied together - * options have help strings, and help text can be assembled - from them - * options set attributes of a passed-in object - * boolean options can have "negative aliases" -- eg. if - --quiet is the "negative alias" of --verbose, then "--quiet" - on the command line sets 'verbose' to false - """ - - def __init__(self, option_table=None): - # The option table is (currently) a list of tuples. The - # tuples may have 3 or four values: - # (long_option, short_option, help_string [, repeatable]) - # if an option takes an argument, its long_option should have '=' - # appended; short_option should just be a single character, no ':' - # in any case. If a long_option doesn't have a corresponding - # short_option, short_option should be None. All option tuples - # must have long options. - self.option_table = option_table - - # 'option_index' maps long option names to entries in the option - # table (ie. those 3-tuples). - self.option_index = {} - if self.option_table: - self._build_index() - - # 'alias' records (duh) alias options; {'foo': 'bar'} means - # --foo is an alias for --bar - self.alias = {} - - # 'negative_alias' keeps track of options that are the boolean - # opposite of some other option - self.negative_alias = {} - - # These keep track of the information in the option table. We - # don't actually populate these structures until we're ready to - # parse the command-line, since the 'option_table' passed in here - # isn't necessarily the final word. - self.short_opts = [] - self.long_opts = [] - self.short2long = {} - self.attr_name = {} - self.takes_arg = {} - - # And 'option_order' is filled up in 'getopt()'; it records the - # original order of options (and their values) on the command-line, - # but expands short options, converts aliases, etc. - self.option_order = [] - - def _build_index(self): - self.option_index.clear() - for option in self.option_table: - self.option_index[option[0]] = option - - def set_option_table(self, option_table): - self.option_table = option_table - self._build_index() - - def add_option(self, long_option, short_option=None, help_string=None): - if long_option in self.option_index: - raise DistutilsGetoptError( - "option conflict: already an option '%s'" % long_option) - else: - option = (long_option, short_option, help_string) - self.option_table.append(option) - self.option_index[long_option] = option - - def has_option(self, long_option): - """Return true if the option table for this parser has an - option with long name 'long_option'.""" - return long_option in self.option_index - - def get_attr_name(self, long_option): - """Translate long option name 'long_option' to the form it - has as an attribute of some object: ie., translate hyphens - to underscores.""" - return long_option.translate(longopt_xlate) - - def _check_alias_dict(self, aliases, what): - assert isinstance(aliases, dict) - for (alias, opt) in aliases.items(): - if alias not in self.option_index: - raise DistutilsGetoptError(("invalid %s '%s': " - "option '%s' not defined") % (what, alias, alias)) - if opt not in self.option_index: - raise DistutilsGetoptError(("invalid %s '%s': " - "aliased option '%s' not defined") % (what, alias, opt)) - - def set_aliases(self, alias): - """Set the aliases for this option parser.""" - self._check_alias_dict(alias, "alias") - self.alias = alias - - def set_negative_aliases(self, negative_alias): - """Set the negative aliases for this option parser. - 'negative_alias' should be a dictionary mapping option names to - option names, both the key and value must already be defined - in the option table.""" - self._check_alias_dict(negative_alias, "negative alias") - self.negative_alias = negative_alias - - def _grok_option_table(self): - """Populate the various data structures that keep tabs on the - option table. Called by 'getopt()' before it can do anything - worthwhile. - """ - self.long_opts = [] - self.short_opts = [] - self.short2long.clear() - self.repeat = {} - - for option in self.option_table: - if len(option) == 3: - long, short, help = option - repeat = 0 - elif len(option) == 4: - long, short, help, repeat = option - else: - # the option table is part of the code, so simply - # assert that it is correct - raise ValueError("invalid option tuple: %r" % (option,)) - - # Type- and value-check the option names - if not isinstance(long, str) or len(long) < 2: - raise DistutilsGetoptError(("invalid long option '%s': " - "must be a string of length >= 2") % long) - - if (not ((short is None) or - (isinstance(short, str) and len(short) == 1))): - raise DistutilsGetoptError("invalid short option '%s': " - "must a single character or None" % short) - - self.repeat[long] = repeat - self.long_opts.append(long) - - if long[-1] == '=': # option takes an argument? - if short: short = short + ':' - long = long[0:-1] - self.takes_arg[long] = 1 - else: - # Is option is a "negative alias" for some other option (eg. - # "quiet" == "!verbose")? - alias_to = self.negative_alias.get(long) - if alias_to is not None: - if self.takes_arg[alias_to]: - raise DistutilsGetoptError( - "invalid negative alias '%s': " - "aliased option '%s' takes a value" - % (long, alias_to)) - - self.long_opts[-1] = long # XXX redundant?! - self.takes_arg[long] = 0 - - # If this is an alias option, make sure its "takes arg" flag is - # the same as the option it's aliased to. - alias_to = self.alias.get(long) - if alias_to is not None: - if self.takes_arg[long] != self.takes_arg[alias_to]: - raise DistutilsGetoptError( - "invalid alias '%s': inconsistent with " - "aliased option '%s' (one of them takes a value, " - "the other doesn't" - % (long, alias_to)) - - # Now enforce some bondage on the long option name, so we can - # later translate it to an attribute name on some object. Have - # to do this a bit late to make sure we've removed any trailing - # '='. - if not longopt_re.match(long): - raise DistutilsGetoptError( - "invalid long option name '%s' " - "(must be letters, numbers, hyphens only" % long) - - self.attr_name[long] = self.get_attr_name(long) - if short: - self.short_opts.append(short) - self.short2long[short[0]] = long - - def getopt(self, args=None, object=None): - """Parse command-line options in args. Store as attributes on object. - - If 'args' is None or not supplied, uses 'sys.argv[1:]'. If - 'object' is None or not supplied, creates a new OptionDummy - object, stores option values there, and returns a tuple (args, - object). If 'object' is supplied, it is modified in place and - 'getopt()' just returns 'args'; in both cases, the returned - 'args' is a modified copy of the passed-in 'args' list, which - is left untouched. - """ - if args is None: - args = sys.argv[1:] - if object is None: - object = OptionDummy() - created_object = True - else: - created_object = False - - self._grok_option_table() - - short_opts = ' '.join(self.short_opts) - try: - opts, args = getopt.getopt(args, short_opts, self.long_opts) - except getopt.error as msg: - raise DistutilsArgError(msg) - - for opt, val in opts: - if len(opt) == 2 and opt[0] == '-': # it's a short option - opt = self.short2long[opt[1]] - else: - assert len(opt) > 2 and opt[:2] == '--' - opt = opt[2:] - - alias = self.alias.get(opt) - if alias: - opt = alias - - if not self.takes_arg[opt]: # boolean option? - assert val == '', "boolean option can't have value" - alias = self.negative_alias.get(opt) - if alias: - opt = alias - val = 0 - else: - val = 1 - - attr = self.attr_name[opt] - # The only repeating option at the moment is 'verbose'. - # It has a negative option -q quiet, which should set verbose = 0. - if val and self.repeat.get(attr) is not None: - val = getattr(object, attr, 0) + 1 - setattr(object, attr, val) - self.option_order.append((opt, val)) - - # for opts - if created_object: - return args, object - else: - return args - - def get_option_order(self): - """Returns the list of (option, value) tuples processed by the - previous run of 'getopt()'. Raises RuntimeError if - 'getopt()' hasn't been called yet. - """ - if self.option_order is None: - raise RuntimeError("'getopt()' hasn't been called yet") - else: - return self.option_order - - def generate_help(self, header=None): - """Generate help text (a list of strings, one per suggested line of - output) from the option table for this FancyGetopt object. - """ - # Blithely assume the option table is good: probably wouldn't call - # 'generate_help()' unless you've already called 'getopt()'. - - # First pass: determine maximum length of long option names - max_opt = 0 - for option in self.option_table: - long = option[0] - short = option[1] - l = len(long) - if long[-1] == '=': - l = l - 1 - if short is not None: - l = l + 5 # " (-x)" where short == 'x' - if l > max_opt: - max_opt = l - - opt_width = max_opt + 2 + 2 + 2 # room for indent + dashes + gutter - - # Typical help block looks like this: - # --foo controls foonabulation - # Help block for longest option looks like this: - # --flimflam set the flim-flam level - # and with wrapped text: - # --flimflam set the flim-flam level (must be between - # 0 and 100, except on Tuesdays) - # Options with short names will have the short name shown (but - # it doesn't contribute to max_opt): - # --foo (-f) controls foonabulation - # If adding the short option would make the left column too wide, - # we push the explanation off to the next line - # --flimflam (-l) - # set the flim-flam level - # Important parameters: - # - 2 spaces before option block start lines - # - 2 dashes for each long option name - # - min. 2 spaces between option and explanation (gutter) - # - 5 characters (incl. space) for short option name - - # Now generate lines of help text. (If 80 columns were good enough - # for Jesus, then 78 columns are good enough for me!) - line_width = 78 - text_width = line_width - opt_width - big_indent = ' ' * opt_width - if header: - lines = [header] - else: - lines = ['Option summary:'] - - for option in self.option_table: - long, short, help = option[:3] - text = wrap_text(help, text_width) - if long[-1] == '=': - long = long[0:-1] - - # Case 1: no short option at all (makes life easy) - if short is None: - if text: - lines.append(" --%-*s %s" % (max_opt, long, text[0])) - else: - lines.append(" --%-*s " % (max_opt, long)) - - # Case 2: we have a short option, so we have to include it - # just after the long option - else: - opt_names = "%s (-%s)" % (long, short) - if text: - lines.append(" --%-*s %s" % - (max_opt, opt_names, text[0])) - else: - lines.append(" --%-*s" % opt_names) - - for l in text[1:]: - lines.append(big_indent + l) - return lines - - def print_help(self, header=None, file=None): - if file is None: - file = sys.stdout - for line in self.generate_help(header): - file.write(line + "\n") - - -def fancy_getopt(options, negative_opt, object, args): - parser = FancyGetopt(options) - parser.set_negative_aliases(negative_opt) - return parser.getopt(args, object) - - -WS_TRANS = {ord(_wschar) : ' ' for _wschar in string.whitespace} - -def wrap_text(text, width): - """wrap_text(text : string, width : int) -> [string] - - Split 'text' into multiple lines of no more than 'width' characters - each, and return the list of strings that results. - """ - if text is None: - return [] - if len(text) <= width: - return [text] - - text = text.expandtabs() - text = text.translate(WS_TRANS) - chunks = re.split(r'( +|-+)', text) - chunks = [ch for ch in chunks if ch] # ' - ' results in empty strings - lines = [] - - while chunks: - cur_line = [] # list of chunks (to-be-joined) - cur_len = 0 # length of current line - - while chunks: - l = len(chunks[0]) - if cur_len + l <= width: # can squeeze (at least) this chunk in - cur_line.append(chunks[0]) - del chunks[0] - cur_len = cur_len + l - else: # this line is full - # drop last chunk if all space - if cur_line and cur_line[-1][0] == ' ': - del cur_line[-1] - break - - if chunks: # any chunks left to process? - # if the current line is still empty, then we had a single - # chunk that's too big too fit on a line -- so we break - # down and break it up at the line width - if cur_len == 0: - cur_line.append(chunks[0][0:width]) - chunks[0] = chunks[0][width:] - - # all-whitespace chunks at the end of a line can be discarded - # (and we know from the re.split above that if a chunk has - # *any* whitespace, it is *all* whitespace) - if chunks[0][0] == ' ': - del chunks[0] - - # and store this line in the list-of-all-lines -- as a single - # string, of course! - lines.append(''.join(cur_line)) - - return lines - - -def translate_longopt(opt): - """Convert a long option name to a valid Python identifier by - changing "-" to "_". - """ - return opt.translate(longopt_xlate) - - -class OptionDummy: - """Dummy class just used as a place to hold command-line option - values as instance attributes.""" - - def __init__(self, options=[]): - """Create a new OptionDummy instance. The attributes listed in - 'options' will be initialized to None.""" - for opt in options: - setattr(self, opt, None) - - -if __name__ == "__main__": - text = """\ -Tra-la-la, supercalifragilisticexpialidocious. -How *do* you spell that odd word, anyways? -(Someone ask Mary -- she'll know [or she'll -say, "How should I know?"].)""" - - for w in (10, 20, 30, 40): - print("width: %d" % w) - print("\n".join(wrap_text(text, w))) - print() diff --git a/Lib/distutils/file_util.py b/Lib/distutils/file_util.py deleted file mode 100644 index b3fee35a6cc..00000000000 --- a/Lib/distutils/file_util.py +++ /dev/null @@ -1,238 +0,0 @@ -"""distutils.file_util - -Utility functions for operating on single files. -""" - -import os -from distutils.errors import DistutilsFileError -from distutils import log - -# for generating verbose output in 'copy_file()' -_copy_action = { None: 'copying', - 'hard': 'hard linking', - 'sym': 'symbolically linking' } - - -def _copy_file_contents(src, dst, buffer_size=16*1024): - """Copy the file 'src' to 'dst'; both must be filenames. Any error - opening either file, reading from 'src', or writing to 'dst', raises - DistutilsFileError. Data is read/written in chunks of 'buffer_size' - bytes (default 16k). No attempt is made to handle anything apart from - regular files. - """ - # Stolen from shutil module in the standard library, but with - # custom error-handling added. - fsrc = None - fdst = None - try: - try: - fsrc = open(src, 'rb') - except OSError as e: - raise DistutilsFileError("could not open '%s': %s" % (src, e.strerror)) - - if os.path.exists(dst): - try: - os.unlink(dst) - except OSError as e: - raise DistutilsFileError( - "could not delete '%s': %s" % (dst, e.strerror)) - - try: - fdst = open(dst, 'wb') - except OSError as e: - raise DistutilsFileError( - "could not create '%s': %s" % (dst, e.strerror)) - - while True: - try: - buf = fsrc.read(buffer_size) - except OSError as e: - raise DistutilsFileError( - "could not read from '%s': %s" % (src, e.strerror)) - - if not buf: - break - - try: - fdst.write(buf) - except OSError as e: - raise DistutilsFileError( - "could not write to '%s': %s" % (dst, e.strerror)) - finally: - if fdst: - fdst.close() - if fsrc: - fsrc.close() - -def copy_file(src, dst, preserve_mode=1, preserve_times=1, update=0, - link=None, verbose=1, dry_run=0): - """Copy a file 'src' to 'dst'. If 'dst' is a directory, then 'src' is - copied there with the same name; otherwise, it must be a filename. (If - the file exists, it will be ruthlessly clobbered.) If 'preserve_mode' - is true (the default), the file's mode (type and permission bits, or - whatever is analogous on the current platform) is copied. If - 'preserve_times' is true (the default), the last-modified and - last-access times are copied as well. If 'update' is true, 'src' will - only be copied if 'dst' does not exist, or if 'dst' does exist but is - older than 'src'. - - 'link' allows you to make hard links (os.link) or symbolic links - (os.symlink) instead of copying: set it to "hard" or "sym"; if it is - None (the default), files are copied. Don't set 'link' on systems that - don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. If hardlink fails, falls back to - _copy_file_contents(). - - Under Mac OS, uses the native file copy function in macostools; on - other systems, uses '_copy_file_contents()' to copy file contents. - - Return a tuple (dest_name, copied): 'dest_name' is the actual name of - the output file, and 'copied' is true if the file was copied (or would - have been copied, if 'dry_run' true). - """ - # XXX if the destination file already exists, we clobber it if - # copying, but blow up if linking. Hmmm. And I don't know what - # macostools.copyfile() does. Should definitely be consistent, and - # should probably blow up if destination exists and we would be - # changing it (ie. it's not already a hard/soft link to src OR - # (not update) and (src newer than dst). - - from distutils.dep_util import newer - from stat import ST_ATIME, ST_MTIME, ST_MODE, S_IMODE - - if not os.path.isfile(src): - raise DistutilsFileError( - "can't copy '%s': doesn't exist or not a regular file" % src) - - if os.path.isdir(dst): - dir = dst - dst = os.path.join(dst, os.path.basename(src)) - else: - dir = os.path.dirname(dst) - - if update and not newer(src, dst): - if verbose >= 1: - log.debug("not copying %s (output up-to-date)", src) - return (dst, 0) - - try: - action = _copy_action[link] - except KeyError: - raise ValueError("invalid value '%s' for 'link' argument" % link) - - if verbose >= 1: - if os.path.basename(dst) == os.path.basename(src): - log.info("%s %s -> %s", action, src, dir) - else: - log.info("%s %s -> %s", action, src, dst) - - if dry_run: - return (dst, 1) - - # If linking (hard or symbolic), use the appropriate system call - # (Unix only, of course, but that's the caller's responsibility) - elif link == 'hard': - if not (os.path.exists(dst) and os.path.samefile(src, dst)): - try: - os.link(src, dst) - return (dst, 1) - except OSError: - # If hard linking fails, fall back on copying file - # (some special filesystems don't support hard linking - # even under Unix, see issue #8876). - pass - elif link == 'sym': - if not (os.path.exists(dst) and os.path.samefile(src, dst)): - os.symlink(src, dst) - return (dst, 1) - - # Otherwise (non-Mac, not linking), copy the file contents and - # (optionally) copy the times and mode. - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) - - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) - - return (dst, 1) - - -# XXX I suspect this is Unix-specific -- need porting help! -def move_file (src, dst, - verbose=1, - dry_run=0): - - """Move a file 'src' to 'dst'. If 'dst' is a directory, the file will - be moved into it with the same name; otherwise, 'src' is just renamed - to 'dst'. Return the new full name of the file. - - Handles cross-device moves on Unix using 'copy_file()'. What about - other systems??? - """ - from os.path import exists, isfile, isdir, basename, dirname - import errno - - if verbose >= 1: - log.info("moving %s -> %s", src, dst) - - if dry_run: - return dst - - if not isfile(src): - raise DistutilsFileError("can't move '%s': not a regular file" % src) - - if isdir(dst): - dst = os.path.join(dst, basename(src)) - elif exists(dst): - raise DistutilsFileError( - "can't move '%s': destination '%s' already exists" % - (src, dst)) - - if not isdir(dirname(dst)): - raise DistutilsFileError( - "can't move '%s': destination '%s' not a valid path" % - (src, dst)) - - copy_it = False - try: - os.rename(src, dst) - except OSError as e: - (num, msg) = e.args - if num == errno.EXDEV: - copy_it = True - else: - raise DistutilsFileError( - "couldn't move '%s' to '%s': %s" % (src, dst, msg)) - - if copy_it: - copy_file(src, dst, verbose=verbose) - try: - os.unlink(src) - except OSError as e: - (num, msg) = e.args - try: - os.unlink(dst) - except OSError: - pass - raise DistutilsFileError( - "couldn't move '%s' to '%s' by copy/delete: " - "delete '%s' failed: %s" - % (src, dst, src, msg)) - return dst - - -def write_file (filename, contents): - """Create a file with the specified name and write 'contents' (a - sequence of strings without line terminators) to it. - """ - f = open(filename, "w") - try: - for line in contents: - f.write(line + "\n") - finally: - f.close() diff --git a/Lib/distutils/filelist.py b/Lib/distutils/filelist.py deleted file mode 100644 index c92d5fdba39..00000000000 --- a/Lib/distutils/filelist.py +++ /dev/null @@ -1,327 +0,0 @@ -"""distutils.filelist - -Provides the FileList class, used for poking about the filesystem -and building lists of files. -""" - -import os, re -import fnmatch -import functools -from distutils.util import convert_path -from distutils.errors import DistutilsTemplateError, DistutilsInternalError -from distutils import log - -class FileList: - """A list of files built by on exploring the filesystem and filtered by - applying various patterns to what we find there. - - Instance attributes: - dir - directory from which files will be taken -- only used if - 'allfiles' not supplied to constructor - files - list of filenames currently being built/filtered/manipulated - allfiles - complete list of files under consideration (ie. without any - filtering applied) - """ - - def __init__(self, warn=None, debug_print=None): - # ignore argument to FileList, but keep them for backwards - # compatibility - self.allfiles = None - self.files = [] - - def set_allfiles(self, allfiles): - self.allfiles = allfiles - - def findall(self, dir=os.curdir): - self.allfiles = findall(dir) - - def debug_print(self, msg): - """Print 'msg' to stdout if the global DEBUG (taken from the - DISTUTILS_DEBUG environment variable) flag is true. - """ - from distutils.debug import DEBUG - if DEBUG: - print(msg) - - # -- List-like methods --------------------------------------------- - - def append(self, item): - self.files.append(item) - - def extend(self, items): - self.files.extend(items) - - def sort(self): - # Not a strict lexical sort! - sortable_files = sorted(map(os.path.split, self.files)) - self.files = [] - for sort_tuple in sortable_files: - self.files.append(os.path.join(*sort_tuple)) - - - # -- Other miscellaneous utility methods --------------------------- - - def remove_duplicates(self): - # Assumes list has been sorted! - for i in range(len(self.files) - 1, 0, -1): - if self.files[i] == self.files[i - 1]: - del self.files[i] - - - # -- "File template" methods --------------------------------------- - - def _parse_template_line(self, line): - words = line.split() - action = words[0] - - patterns = dir = dir_pattern = None - - if action in ('include', 'exclude', - 'global-include', 'global-exclude'): - if len(words) < 2: - raise DistutilsTemplateError( - "'%s' expects ..." % action) - patterns = [convert_path(w) for w in words[1:]] - elif action in ('recursive-include', 'recursive-exclude'): - if len(words) < 3: - raise DistutilsTemplateError( - "'%s' expects ..." % action) - dir = convert_path(words[1]) - patterns = [convert_path(w) for w in words[2:]] - elif action in ('graft', 'prune'): - if len(words) != 2: - raise DistutilsTemplateError( - "'%s' expects a single " % action) - dir_pattern = convert_path(words[1]) - else: - raise DistutilsTemplateError("unknown action '%s'" % action) - - return (action, patterns, dir, dir_pattern) - - def process_template_line(self, line): - # Parse the line: split it up, make sure the right number of words - # is there, and return the relevant words. 'action' is always - # defined: it's the first word of the line. Which of the other - # three are defined depends on the action; it'll be either - # patterns, (dir and patterns), or (dir_pattern). - (action, patterns, dir, dir_pattern) = self._parse_template_line(line) - - # OK, now we know that the action is valid and we have the - # right number of words on the line for that action -- so we - # can proceed with minimal error-checking. - if action == 'include': - self.debug_print("include " + ' '.join(patterns)) - for pattern in patterns: - if not self.include_pattern(pattern, anchor=1): - log.warn("warning: no files found matching '%s'", - pattern) - - elif action == 'exclude': - self.debug_print("exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.exclude_pattern(pattern, anchor=1): - log.warn(("warning: no previously-included files " - "found matching '%s'"), pattern) - - elif action == 'global-include': - self.debug_print("global-include " + ' '.join(patterns)) - for pattern in patterns: - if not self.include_pattern(pattern, anchor=0): - log.warn(("warning: no files found matching '%s' " - "anywhere in distribution"), pattern) - - elif action == 'global-exclude': - self.debug_print("global-exclude " + ' '.join(patterns)) - for pattern in patterns: - if not self.exclude_pattern(pattern, anchor=0): - log.warn(("warning: no previously-included files matching " - "'%s' found anywhere in distribution"), - pattern) - - elif action == 'recursive-include': - self.debug_print("recursive-include %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.include_pattern(pattern, prefix=dir): - log.warn(("warning: no files found matching '%s' " - "under directory '%s'"), - pattern, dir) - - elif action == 'recursive-exclude': - self.debug_print("recursive-exclude %s %s" % - (dir, ' '.join(patterns))) - for pattern in patterns: - if not self.exclude_pattern(pattern, prefix=dir): - log.warn(("warning: no previously-included files matching " - "'%s' found under directory '%s'"), - pattern, dir) - - elif action == 'graft': - self.debug_print("graft " + dir_pattern) - if not self.include_pattern(None, prefix=dir_pattern): - log.warn("warning: no directories found matching '%s'", - dir_pattern) - - elif action == 'prune': - self.debug_print("prune " + dir_pattern) - if not self.exclude_pattern(None, prefix=dir_pattern): - log.warn(("no previously-included directories found " - "matching '%s'"), dir_pattern) - else: - raise DistutilsInternalError( - "this cannot happen: invalid action '%s'" % action) - - - # -- Filtering/selection methods ----------------------------------- - - def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): - """Select strings (presumably filenames) from 'self.files' that - match 'pattern', a Unix-style wildcard (glob) pattern. Patterns - are not quite the same as implemented by the 'fnmatch' module: '*' - and '?' match non-special characters, where "special" is platform- - dependent: slash on Unix; colon, slash, and backslash on - DOS/Windows; and colon on Mac OS. - - If 'anchor' is true (the default), then the pattern match is more - stringent: "*.py" will match "foo.py" but not "foo/bar.py". If - 'anchor' is false, both of these will match. - - If 'prefix' is supplied, then only filenames starting with 'prefix' - (itself a pattern) and ending with 'pattern', with anything in between - them, will match. 'anchor' is ignored in this case. - - If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and - 'pattern' is assumed to be either a string containing a regex or a - regex object -- no translation is done, the regex is just compiled - and used as-is. - - Selected strings will be added to self.files. - - Return True if files are found, False otherwise. - """ - # XXX docstring lying about what the special chars are? - files_found = False - pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) - self.debug_print("include_pattern: applying regex r'%s'" % - pattern_re.pattern) - - # delayed loading of allfiles list - if self.allfiles is None: - self.findall() - - for name in self.allfiles: - if pattern_re.search(name): - self.debug_print(" adding " + name) - self.files.append(name) - files_found = True - return files_found - - - def exclude_pattern (self, pattern, - anchor=1, prefix=None, is_regex=0): - """Remove strings (presumably filenames) from 'files' that match - 'pattern'. Other parameters are the same as for - 'include_pattern()', above. - The list 'self.files' is modified in place. - Return True if files are found, False otherwise. - """ - files_found = False - pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) - self.debug_print("exclude_pattern: applying regex r'%s'" % - pattern_re.pattern) - for i in range(len(self.files)-1, -1, -1): - if pattern_re.search(self.files[i]): - self.debug_print(" removing " + self.files[i]) - del self.files[i] - files_found = True - return files_found - - -# ---------------------------------------------------------------------- -# Utility functions - -def _find_all_simple(path): - """ - Find all files under 'path' - """ - results = ( - os.path.join(base, file) - for base, dirs, files in os.walk(path, followlinks=True) - for file in files - ) - return filter(os.path.isfile, results) - - -def findall(dir=os.curdir): - """ - Find all files under 'dir' and return the list of full filenames. - Unless dir is '.', return full filenames with dir prepended. - """ - files = _find_all_simple(dir) - if dir == os.curdir: - make_rel = functools.partial(os.path.relpath, start=dir) - files = map(make_rel, files) - return list(files) - - -def glob_to_re(pattern): - """Translate a shell-like glob pattern to a regular expression; return - a string containing the regex. Differs from 'fnmatch.translate()' in - that '*' does not match "special characters" (which are - platform-specific). - """ - pattern_re = fnmatch.translate(pattern) - - # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which - # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, - # and by extension they shouldn't match such "special characters" under - # any OS. So change all non-escaped dots in the RE to match any - # character except the special characters (currently: just os.sep). - sep = os.sep - if os.sep == '\\': - # we're using a regex to manipulate a regex, so we need - # to escape the backslash twice - sep = r'\\\\' - escaped = r'\1[^%s]' % sep - pattern_re = re.sub(r'((?= self.threshold: - if args: - msg = msg % args - if level in (WARN, ERROR, FATAL): - stream = sys.stderr - else: - stream = sys.stdout - try: - stream.write('%s\n' % msg) - except UnicodeEncodeError: - # emulate backslashreplace error handler - encoding = stream.encoding - msg = msg.encode(encoding, "backslashreplace").decode(encoding) - stream.write('%s\n' % msg) - stream.flush() - - def log(self, level, msg, *args): - self._log(level, msg, args) - - def debug(self, msg, *args): - self._log(DEBUG, msg, args) - - def info(self, msg, *args): - self._log(INFO, msg, args) - - def warn(self, msg, *args): - self._log(WARN, msg, args) - - def error(self, msg, *args): - self._log(ERROR, msg, args) - - def fatal(self, msg, *args): - self._log(FATAL, msg, args) - -_global_log = Log() -log = _global_log.log -debug = _global_log.debug -info = _global_log.info -warn = _global_log.warn -error = _global_log.error -fatal = _global_log.fatal - -def set_threshold(level): - # return the old threshold for use from tests - old = _global_log.threshold - _global_log.threshold = level - return old - -def set_verbosity(v): - if v <= 0: - set_threshold(WARN) - elif v == 1: - set_threshold(INFO) - elif v >= 2: - set_threshold(DEBUG) diff --git a/Lib/distutils/msvc9compiler.py b/Lib/distutils/msvc9compiler.py deleted file mode 100644 index 21191276227..00000000000 --- a/Lib/distutils/msvc9compiler.py +++ /dev/null @@ -1,791 +0,0 @@ -"""distutils.msvc9compiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio 2008. - -The module is compatible with VS 2005 and VS 2008. You can find legacy support -for older versions of VS in distutils.msvccompiler. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) -# ported to VS2005 and VS 2008 by Christian Heimes - -import os -import subprocess -import sys -import re - -from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import CCompiler, gen_preprocess_options, \ - gen_lib_options -from distutils import log -from distutils.util import get_platform - -import winreg - -RegOpenKeyEx = winreg.OpenKeyEx -RegEnumKey = winreg.EnumKey -RegEnumValue = winreg.EnumValue -RegError = winreg.error - -HKEYS = (winreg.HKEY_USERS, - winreg.HKEY_CURRENT_USER, - winreg.HKEY_LOCAL_MACHINE, - winreg.HKEY_CLASSES_ROOT) - -NATIVE_WIN64 = (sys.platform == 'win32' and sys.maxsize > 2**32) -if NATIVE_WIN64: - # Visual C++ is a 32-bit application, so we need to look in - # the corresponding registry branch, if we're running a - # 64-bit Python on Win64 - VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" - WINSDK_BASE = r"Software\Wow6432Node\Microsoft\Microsoft SDKs\Windows" - NET_BASE = r"Software\Wow6432Node\Microsoft\.NETFramework" -else: - VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" - WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" - NET_BASE = r"Software\Microsoft\.NETFramework" - -# A map keyed by get_platform() return values to values accepted by -# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is -# the param to cross-compile on x86 targeting amd64.) -PLAT_TO_VCVARS = { - 'win32' : 'x86', - 'win-amd64' : 'amd64', - 'win-ia64' : 'ia64', -} - -class Reg: - """Helper class to read values from the registry - """ - - def get_value(cls, path, key): - for base in HKEYS: - d = cls.read_values(base, path) - if d and key in d: - return d[key] - raise KeyError(key) - get_value = classmethod(get_value) - - def read_keys(cls, base, key): - """Return list of registry keys.""" - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while True: - try: - k = RegEnumKey(handle, i) - except RegError: - break - L.append(k) - i += 1 - return L - read_keys = classmethod(read_keys) - - def read_values(cls, base, key): - """Return dict of registry keys and values. - - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while True: - try: - name, value, type = RegEnumValue(handle, i) - except RegError: - break - name = name.lower() - d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) - i += 1 - return d - read_values = classmethod(read_values) - - def convert_mbcs(s): - dec = getattr(s, "decode", None) - if dec is not None: - try: - s = dec("mbcs") - except UnicodeError: - pass - return s - convert_mbcs = staticmethod(convert_mbcs) - -class MacroExpander: - - def __init__(self, version): - self.macros = {} - self.vsbase = VS_BASE % version - self.load_macros(version) - - def set_macro(self, macro, path, key): - self.macros["$(%s)" % macro] = Reg.get_value(path, key) - - def load_macros(self, version): - self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") - self.set_macro("FrameworkDir", NET_BASE, "installroot") - try: - if version >= 8.0: - self.set_macro("FrameworkSDKDir", NET_BASE, - "sdkinstallrootv2.0") - else: - raise KeyError("sdkinstallrootv2.0") - except KeyError: - raise DistutilsPlatformError( - """Python was built with Visual Studio 2008; -extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2008 was not found on this system. If you have Cygwin installed, -you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - - if version >= 9.0: - self.set_macro("FrameworkVersion", self.vsbase, "clr version") - self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") - else: - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = Reg.get_value(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] - - def sub(self, s): - for k, v in self.macros.items(): - s = s.replace(k, v) - return s - -def get_build_version(): - """Return the version of MSVC that was used to build Python. - - For Python 2.3 and up, the version number is included in - sys.version. For earlier versions, assume the compiler is MSVC 6. - """ - prefix = "MSC v." - i = sys.version.find(prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - # v13 was skipped and should be v14 - majorVersion += 1 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def normalize_and_reduce_paths(paths): - """Return a list of normalized paths with duplicates removed. - - The current order of paths is maintained. - """ - # Paths are normalized so things like: /a and /a/ aren't both preserved. - reduced_paths = [] - for p in paths: - np = os.path.normpath(p) - # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. - if np not in reduced_paths: - reduced_paths.append(np) - return reduced_paths - -def removeDuplicates(variable): - """Remove duplicate values of an environment variable. - """ - oldList = variable.split(os.pathsep) - newList = [] - for i in oldList: - if i not in newList: - newList.append(i) - newVariable = os.pathsep.join(newList) - return newVariable - -def find_vcvarsall(version): - """Find the vcvarsall.bat file - - At first it tries to find the productdir of VS 2008 in the registry. If - that fails it falls back to the VS90COMNTOOLS env var. - """ - vsbase = VS_BASE % version - try: - productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, - "productdir") - except KeyError: - log.debug("Unable to find productdir in registry") - productdir = None - - if not productdir or not os.path.isdir(productdir): - toolskey = "VS%0.f0COMNTOOLS" % version - toolsdir = os.environ.get(toolskey, None) - - if toolsdir and os.path.isdir(toolsdir): - productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") - productdir = os.path.abspath(productdir) - if not os.path.isdir(productdir): - log.debug("%s is not a valid directory" % productdir) - return None - else: - log.debug("Env var %s is not set or invalid" % toolskey) - if not productdir: - log.debug("No productdir found") - return None - vcvarsall = os.path.join(productdir, "vcvarsall.bat") - if os.path.isfile(vcvarsall): - return vcvarsall - log.debug("Unable to find vcvarsall.bat") - return None - -def query_vcvarsall(version, arch="x86"): - """Launch vcvarsall.bat and read the settings from its environment - """ - vcvarsall = find_vcvarsall(version) - interesting = set(("include", "lib", "libpath", "path")) - result = {} - - if vcvarsall is None: - raise DistutilsPlatformError("Unable to find vcvarsall.bat") - log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) - popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - try: - stdout, stderr = popen.communicate() - if popen.wait() != 0: - raise DistutilsPlatformError(stderr.decode("mbcs")) - - stdout = stdout.decode("mbcs") - for line in stdout.split("\n"): - line = Reg.convert_mbcs(line) - if '=' not in line: - continue - line = line.strip() - key, value = line.split('=', 1) - key = key.lower() - if key in interesting: - if value.endswith(os.pathsep): - value = value[:-1] - result[key] = removeDuplicates(value) - - finally: - popen.stdout.close() - popen.stderr.close() - - if len(result) != len(interesting): - raise ValueError(str(list(result.keys()))) - - return result - -# More globals -VERSION = get_build_version() -if VERSION < 8.0: - raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) -# MACROS = MacroExpander(VERSION) - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = VERSION - self.__root = r"Software\Microsoft\VisualStudio" - # self.__macros = MACROS - self.__paths = [] - # target platform (.plat_name is consistent with 'bdist') - self.plat_name = None - self.__arch = None # deprecated name - self.initialized = False - - def initialize(self, plat_name=None): - # multi-init means we would need to check platform same each time... - assert not self.initialized, "don't init multiple times" - if plat_name is None: - plat_name = get_platform() - # sanity check for platforms to prevent obscure errors later. - ok_plats = 'win32', 'win-amd64', 'win-ia64' - if plat_name not in ok_plats: - raise DistutilsPlatformError("--plat-name must be one of %s" % - (ok_plats,)) - - if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): - # Assume that the SDK set up everything alright; don't try to be - # smarter - self.cc = "cl.exe" - self.linker = "link.exe" - self.lib = "lib.exe" - self.rc = "rc.exe" - self.mc = "mc.exe" - else: - # On x86, 'vcvars32.bat amd64' creates an env that doesn't work; - # to cross compile, you use 'x86_amd64'. - # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross - # compile use 'x86' (ie, it runs the x86 compiler directly) - # No idea how itanium handles this, if at all. - if plat_name == get_platform() or plat_name == 'win32': - # native build or cross-compile to win32 - plat_spec = PLAT_TO_VCVARS[plat_name] - else: - # cross compile from win32 -> some 64bit - plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \ - PLAT_TO_VCVARS[plat_name] - - vc_env = query_vcvarsall(VERSION, plat_spec) - - self.__paths = vc_env['path'].split(os.pathsep) - os.environ['lib'] = vc_env['lib'] - os.environ['include'] = vc_env['include'] - - if len(self.__paths) == 0: - raise DistutilsPlatformError("Python was built with %s, " - "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." - % self.__product) - - self.cc = self.find_exe("cl.exe") - self.linker = self.find_exe("link.exe") - self.lib = self.find_exe("lib.exe") - self.rc = self.find_exe("rc.exe") # resource compiler - self.mc = self.find_exe("mc.exe") # message compiler - #self.set_path_env_var('lib') - #self.set_path_env_var('include') - - # extend the MSVC path with the current path - try: - for p in os.environ['path'].split(';'): - self.__paths.append(p) - except KeyError: - pass - self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = ";".join(self.__paths) - - self.preprocess_options = None - if self.__arch == "x86": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', - '/Z7', '/D_DEBUG'] - else: - # Win64 - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', - '/Z7', '/D_DEBUG'] - - self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] - if self.__version >= 7: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' - ] - self.ldflags_static = [ '/nologo'] - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - # Copied from ccompiler.py, extended to return .res as 'object'-file - # for .rc input file - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - (base, ext) = os.path.splitext (src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError ("Don't know how to compile %s" % src_name) - if strip_dir: - base = os.path.basename (base) - if ext in self._rc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - elif ext in self._mc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append ('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + - [output_opt] + [input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) - base, _ = os.path.splitext (os.path.basename (src)) - rc_file = os.path.join (rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc] + - ["/fo" + obj] + [rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile %s to %s" - % (src, obj)) - - output_opt = "/Fo" + obj - try: - self.spawn([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - (libraries, library_dirs, runtime_library_dirs) = fixed_args - - if runtime_library_dirs: - self.warn ("I don't know what to do with 'runtime_library_dirs': " - + str (runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - if target_desc == CCompiler.EXECUTABLE: - if debug: - ldflags = self.ldflags_shared_debug[1:] - else: - ldflags = self.ldflags_shared[1:] - else: - if debug: - ldflags = self.ldflags_shared_debug - else: - ldflags = self.ldflags_shared - - export_opts = [] - for sym in (export_symbols or []): - export_opts.append("/EXPORT:" + sym) - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - build_temp = os.path.dirname(objects[0]) - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - build_temp, - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - self.manifest_setup_ldargs(output_filename, build_temp, ld_args) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath(os.path.dirname(output_filename)) - try: - self.spawn([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - # embed the manifest - # XXX - this is somewhat fragile - if mt.exe fails, distutils - # will still consider the DLL up-to-date, but it will not have a - # manifest. Maybe we should link to a temp file? OTOH, that - # implies a build environment error that shouldn't go undetected. - mfinfo = self.manifest_get_embed_info(target_desc, ld_args) - if mfinfo is not None: - mffilename, mfid = mfinfo - out_arg = '-outputresource:%s;%s' % (output_filename, mfid) - try: - self.spawn(['mt.exe', '-nologo', '-manifest', - mffilename, out_arg]) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def manifest_setup_ldargs(self, output_filename, build_temp, ld_args): - # If we need a manifest at all, an embedded manifest is recommended. - # See MSDN article titled - # "How to: Embed a Manifest Inside a C/C++ Application" - # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) - # Ask the linker to generate the manifest in the temp dir, so - # we can check it, and possibly embed it, later. - temp_manifest = os.path.join( - build_temp, - os.path.basename(output_filename) + ".manifest") - ld_args.append('/MANIFESTFILE:' + temp_manifest) - - def manifest_get_embed_info(self, target_desc, ld_args): - # If a manifest should be embedded, return a tuple of - # (manifest_filename, resource_id). Returns None if no manifest - # should be embedded. See http://bugs.python.org/issue7833 for why - # we want to avoid any manifest for extension modules if we can) - for arg in ld_args: - if arg.startswith("/MANIFESTFILE:"): - temp_manifest = arg.split(":", 1)[1] - break - else: - # no /MANIFESTFILE so nothing to do. - return None - if target_desc == CCompiler.EXECUTABLE: - # by default, executables always get the manifest with the - # CRT referenced. - mfid = 1 - else: - # Extension modules try and avoid any manifest if possible. - mfid = 2 - temp_manifest = self._remove_visual_c_ref(temp_manifest) - if temp_manifest is None: - return None - return temp_manifest, mfid - - def _remove_visual_c_ref(self, manifest_file): - try: - # Remove references to the Visual C runtime, so they will - # fall through to the Visual C dependency of Python.exe. - # This way, when installed for a restricted user (e.g. - # runtimes are not in WinSxS folder, but in Python's own - # folder), the runtimes do not need to be in every folder - # with .pyd's. - # Returns either the filename of the modified manifest or - # None if no manifest should be embedded. - manifest_f = open(manifest_file) - try: - manifest_buf = manifest_f.read() - finally: - manifest_f.close() - pattern = re.compile( - r"""|)""", - re.DOTALL) - manifest_buf = re.sub(pattern, "", manifest_buf) - pattern = r"\s*" - manifest_buf = re.sub(pattern, "", manifest_buf) - # Now see if any other assemblies are referenced - if not, we - # don't want a manifest embedded. - pattern = re.compile( - r"""|)""", re.DOTALL) - if re.search(pattern, manifest_buf) is None: - return None - - manifest_f = open(manifest_file, 'w') - try: - manifest_f.write(manifest_buf) - return manifest_file - finally: - manifest_f.close() - except OSError: - pass - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC++") - - def library_option(self, lib): - return self.library_filename(lib) - - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename (name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # Helper methods for using the MSVC registry settings - - def find_exe(self, exe): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - for p in self.__paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - - # didn't find it; try existing path - for p in os.environ['Path'].split(';'): - fn = os.path.join(os.path.abspath(p),exe) - if os.path.isfile(fn): - return fn - - return exe diff --git a/Lib/distutils/msvccompiler.py b/Lib/distutils/msvccompiler.py deleted file mode 100644 index 1048cd41593..00000000000 --- a/Lib/distutils/msvccompiler.py +++ /dev/null @@ -1,643 +0,0 @@ -"""distutils.msvccompiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for the Microsoft Visual Studio. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) - -import sys, os -from distutils.errors import \ - DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils import log - -_can_read_reg = False -try: - import winreg - - _can_read_reg = True - hkey_mod = winreg - - RegOpenKeyEx = winreg.OpenKeyEx - RegEnumKey = winreg.EnumKey - RegEnumValue = winreg.EnumValue - RegError = winreg.error - -except ImportError: - try: - import win32api - import win32con - _can_read_reg = True - hkey_mod = win32con - - RegOpenKeyEx = win32api.RegOpenKeyEx - RegEnumKey = win32api.RegEnumKey - RegEnumValue = win32api.RegEnumValue - RegError = win32api.error - except ImportError: - log.info("Warning: Can't read registry to find the " - "necessary compiler setting\n" - "Make sure that Python modules winreg, " - "win32api or win32con are installed.") - pass - -if _can_read_reg: - HKEYS = (hkey_mod.HKEY_USERS, - hkey_mod.HKEY_CURRENT_USER, - hkey_mod.HKEY_LOCAL_MACHINE, - hkey_mod.HKEY_CLASSES_ROOT) - -def read_keys(base, key): - """Return list of registry keys.""" - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - L = [] - i = 0 - while True: - try: - k = RegEnumKey(handle, i) - except RegError: - break - L.append(k) - i += 1 - return L - -def read_values(base, key): - """Return dict of registry keys and values. - - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while True: - try: - name, value, type = RegEnumValue(handle, i) - except RegError: - break - name = name.lower() - d[convert_mbcs(name)] = convert_mbcs(value) - i += 1 - return d - -def convert_mbcs(s): - dec = getattr(s, "decode", None) - if dec is not None: - try: - s = dec("mbcs") - except UnicodeError: - pass - return s - -class MacroExpander: - def __init__(self, version): - self.macros = {} - self.load_macros(version) - - def set_macro(self, macro, path, key): - for base in HKEYS: - d = read_values(base, path) - if d: - self.macros["$(%s)" % macro] = d[key] - break - - def load_macros(self, version): - vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version - self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") - self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") - net = r"Software\Microsoft\.NETFramework" - self.set_macro("FrameworkDir", net, "installroot") - try: - if version > 7.0: - self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") - else: - self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") - except KeyError as exc: # - raise DistutilsPlatformError( - """Python was built with Visual Studio 2003; -extensions must be built with a compiler than can generate compatible binaries. -Visual Studio 2003 was not found on this system. If you have Cygwin installed, -you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") - - p = r"Software\Microsoft\NET Framework Setup\Product" - for base in HKEYS: - try: - h = RegOpenKeyEx(base, p) - except RegError: - continue - key = RegEnumKey(h, 0) - d = read_values(base, r"%s\%s" % (p, key)) - self.macros["$(FrameworkVersion)"] = d["version"] - - def sub(self, s): - for k, v in self.macros.items(): - s = s.replace(k, v) - return s - -def get_build_version(): - """Return the version of MSVC that was used to build Python. - - For Python 2.3 and up, the version number is included in - sys.version. For earlier versions, assume the compiler is MSVC 6. - """ - prefix = "MSC v." - i = sys.version.find(prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - # v13 was skipped and should be v14 - majorVersion += 1 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - -def get_build_architecture(): - """Return the processor architecture. - - Possible results are "Intel", "Itanium", or "AMD64". - """ - - prefix = " bit (" - i = sys.version.find(prefix) - if i == -1: - return "Intel" - j = sys.version.find(")", i) - return sys.version[i+len(prefix):j] - -def normalize_and_reduce_paths(paths): - """Return a list of normalized paths with duplicates removed. - - The current order of paths is maintained. - """ - # Paths are normalized so things like: /a and /a/ aren't both preserved. - reduced_paths = [] - for p in paths: - np = os.path.normpath(p) - # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. - if np not in reduced_paths: - reduced_paths.append(np) - return reduced_paths - - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - self.__version = get_build_version() - self.__arch = get_build_architecture() - if self.__arch == "Intel": - # x86 - if self.__version >= 7: - self.__root = r"Software\Microsoft\VisualStudio" - self.__macros = MacroExpander(self.__version) - else: - self.__root = r"Software\Microsoft\Devstudio" - self.__product = "Visual Studio version %s" % self.__version - else: - # Win64. Assume this was built with the platform SDK - self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) - - self.initialized = False - - def initialize(self): - self.__paths = [] - if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): - # Assume that the SDK set up everything alright; don't try to be - # smarter - self.cc = "cl.exe" - self.linker = "link.exe" - self.lib = "lib.exe" - self.rc = "rc.exe" - self.mc = "mc.exe" - else: - self.__paths = self.get_msvc_paths("path") - - if len(self.__paths) == 0: - raise DistutilsPlatformError("Python was built with %s, " - "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." - % self.__product) - - self.cc = self.find_exe("cl.exe") - self.linker = self.find_exe("link.exe") - self.lib = self.find_exe("lib.exe") - self.rc = self.find_exe("rc.exe") # resource compiler - self.mc = self.find_exe("mc.exe") # message compiler - self.set_path_env_var('lib') - self.set_path_env_var('include') - - # extend the MSVC path with the current path - try: - for p in os.environ['path'].split(';'): - self.__paths.append(p) - except KeyError: - pass - self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = ";".join(self.__paths) - - self.preprocess_options = None - if self.__arch == "Intel": - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GX' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', - '/Z7', '/D_DEBUG'] - else: - # Win64 - self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , - '/DNDEBUG'] - self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', - '/Z7', '/D_DEBUG'] - - self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] - if self.__version >= 7: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' - ] - else: - self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG' - ] - self.ldflags_static = [ '/nologo'] - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - # Copied from ccompiler.py, extended to return .res as 'object'-file - # for .rc input file - if output_dir is None: output_dir = '' - obj_names = [] - for src_name in source_filenames: - (base, ext) = os.path.splitext (src_name) - base = os.path.splitdrive(base)[1] # Chop off the drive - base = base[os.path.isabs(base):] # If abs, chop off leading / - if ext not in self.src_extensions: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError ("Don't know how to compile %s" % src_name) - if strip_dir: - base = os.path.basename (base) - if ext in self._rc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - elif ext in self._mc_extensions: - obj_names.append (os.path.join (output_dir, - base + self.res_extension)) - else: - obj_names.append (os.path.join (output_dir, - base + self.obj_extension)) - return obj_names - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append ('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + - [output_opt] + [input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) - base, _ = os.path.splitext (os.path.basename (src)) - rc_file = os.path.join (rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc] + - ["/fo" + obj] + [rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile %s to %s" - % (src, obj)) - - output_opt = "/Fo" + obj - try: - self.spawn([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - (objects, output_dir) = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - (libraries, library_dirs, runtime_library_dirs) = fixed_args - - if runtime_library_dirs: - self.warn ("I don't know what to do with 'runtime_library_dirs': " - + str (runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - if target_desc == CCompiler.EXECUTABLE: - if debug: - ldflags = self.ldflags_shared_debug[1:] - else: - ldflags = self.ldflags_shared[1:] - else: - if debug: - ldflags = self.ldflags_shared_debug - else: - ldflags = self.ldflags_shared - - export_opts = [] - for sym in (export_symbols or []): - export_opts.append("/EXPORT:" + sym) - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - os.path.dirname(objects[0]), - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - self.mkpath(os.path.dirname(output_filename)) - try: - self.spawn([self.linker] + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC++") - - def library_option(self, lib): - return self.library_filename(lib) - - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename (name)) - if os.path.exists(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None - - # Helper methods for using the MSVC registry settings - - def find_exe(self, exe): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - for p in self.__paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - - # didn't find it; try existing path - for p in os.environ['Path'].split(';'): - fn = os.path.join(os.path.abspath(p),exe) - if os.path.isfile(fn): - return fn - - return exe - - def get_msvc_paths(self, path, platform='x86'): - """Get a list of devstudio directories (include, lib or path). - - Return a list of strings. The list will be empty if unable to - access the registry or appropriate registry keys not found. - """ - if not _can_read_reg: - return [] - - path = path + " dirs" - if self.__version >= 7: - key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" - % (self.__root, self.__version)) - else: - key = (r"%s\6.0\Build System\Components\Platforms" - r"\Win32 (%s)\Directories" % (self.__root, platform)) - - for base in HKEYS: - d = read_values(base, key) - if d: - if self.__version >= 7: - return self.__macros.sub(d[path]).split(";") - else: - return d[path].split(";") - # MSVC 6 seems to create the registry entries we need only when - # the GUI is run. - if self.__version == 6: - for base in HKEYS: - if read_values(base, r"%s\6.0" % self.__root) is not None: - self.warn("It seems you have Visual Studio 6 installed, " - "but the expected registry settings are not present.\n" - "You must at least run the Visual Studio GUI once " - "so that these entries are created.") - break - return [] - - def set_path_env_var(self, name): - """Set environment variable 'name' to an MSVC path type value. - - This is equivalent to a SET command prior to execution of spawned - commands. - """ - - if name == "lib": - p = self.get_msvc_paths("library") - else: - p = self.get_msvc_paths(name) - if p: - os.environ[name] = ';'.join(p) - - -if get_build_version() >= 8.0: - log.debug("Importing new compiler from distutils.msvc9compiler") - OldMSVCCompiler = MSVCCompiler - from distutils.msvc9compiler import MSVCCompiler - # get_build_architecture not really relevant now we support cross-compile - from distutils.msvc9compiler import MacroExpander diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py deleted file mode 100644 index 53876880932..00000000000 --- a/Lib/distutils/spawn.py +++ /dev/null @@ -1,192 +0,0 @@ -"""distutils.spawn - -Provides the 'spawn()' function, a front-end to various platform- -specific functions for launching another program in a sub-process. -Also provides the 'find_executable()' to search the path for a given -executable name. -""" - -import sys -import os - -from distutils.errors import DistutilsPlatformError, DistutilsExecError -from distutils.debug import DEBUG -from distutils import log - -def spawn(cmd, search_path=1, verbose=0, dry_run=0): - """Run another program, specified as a command list 'cmd', in a new process. - - 'cmd' is just the argument list for the new process, ie. - cmd[0] is the program to run and cmd[1:] are the rest of its arguments. - There is no way to run a program with a name different from that of its - executable. - - If 'search_path' is true (the default), the system's executable - search path will be used to find the program; otherwise, cmd[0] - must be the exact path to the executable. If 'dry_run' is true, - the command will not actually be run. - - Raise DistutilsExecError if running the program fails in any way; just - return on success. - """ - # cmd is documented as a list, but just in case some code passes a tuple - # in, protect our %-formatting code against horrible death - cmd = list(cmd) - if os.name == 'posix': - _spawn_posix(cmd, search_path, dry_run=dry_run) - elif os.name == 'nt': - _spawn_nt(cmd, search_path, dry_run=dry_run) - else: - raise DistutilsPlatformError( - "don't know how to spawn programs on platform '%s'" % os.name) - -def _nt_quote_args(args): - """Quote command-line arguments for DOS/Windows conventions. - - Just wraps every argument which contains blanks in double quotes, and - returns a new argument list. - """ - # XXX this doesn't seem very robust to me -- but if the Windows guys - # say it'll work, I guess I'll have to accept it. (What if an arg - # contains quotes? What other magic characters, other than spaces, - # have to be escaped? Is there an escaping mechanism other than - # quoting?) - for i, arg in enumerate(args): - if ' ' in arg: - args[i] = '"%s"' % arg - return args - -def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0): - executable = cmd[0] - cmd = _nt_quote_args(cmd) - if search_path: - # either we find one or it stays the same - executable = find_executable(executable) or executable - log.info(' '.join([executable] + cmd[1:])) - if not dry_run: - # spawn for NT requires a full path to the .exe - try: - rc = os.spawnv(os.P_WAIT, executable, cmd) - except OSError as exc: - # this seems to happen when the command isn't found - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed: %s" % (cmd, exc.args[-1])) - if rc != 0: - # and this reflects the command running but failing - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed with exit status %d" % (cmd, rc)) - -if sys.platform == 'darwin': - from distutils import sysconfig - _cfg_target = None - _cfg_target_split = None - -def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0): - log.info(' '.join(cmd)) - if dry_run: - return - executable = cmd[0] - exec_fn = search_path and os.execvp or os.execv - env = None - if sys.platform == 'darwin': - global _cfg_target, _cfg_target_split - if _cfg_target is None: - _cfg_target = sysconfig.get_config_var( - 'MACOSX_DEPLOYMENT_TARGET') or '' - if _cfg_target: - _cfg_target_split = [int(x) for x in _cfg_target.split('.')] - if _cfg_target: - # ensure that the deployment target of build process is not less - # than that used when the interpreter was built. This ensures - # extension modules are built with correct compatibility values - cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) - if _cfg_target_split > [int(x) for x in cur_target.split('.')]: - my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' - 'now "%s" but "%s" during configure' - % (cur_target, _cfg_target)) - raise DistutilsPlatformError(my_msg) - env = dict(os.environ, - MACOSX_DEPLOYMENT_TARGET=cur_target) - exec_fn = search_path and os.execvpe or os.execve - pid = os.fork() - if pid == 0: # in the child - try: - if env is None: - exec_fn(executable, cmd) - else: - exec_fn(executable, cmd, env) - except OSError as e: - if not DEBUG: - cmd = executable - sys.stderr.write("unable to execute %r: %s\n" - % (cmd, e.strerror)) - os._exit(1) - - if not DEBUG: - cmd = executable - sys.stderr.write("unable to execute %r for unknown reasons" % cmd) - os._exit(1) - else: # in the parent - # Loop until the child either exits or is terminated by a signal - # (ie. keep waiting if it's merely stopped) - while True: - try: - pid, status = os.waitpid(pid, 0) - except OSError as exc: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed: %s" % (cmd, exc.args[-1])) - if os.WIFSIGNALED(status): - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r terminated by signal %d" - % (cmd, os.WTERMSIG(status))) - elif os.WIFEXITED(status): - exit_status = os.WEXITSTATUS(status) - if exit_status == 0: - return # hey, it succeeded! - else: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "command %r failed with exit status %d" - % (cmd, exit_status)) - elif os.WIFSTOPPED(status): - continue - else: - if not DEBUG: - cmd = executable - raise DistutilsExecError( - "unknown error executing %r: termination status %d" - % (cmd, status)) - -def find_executable(executable, path=None): - """Tries to find 'executable' in the directories listed in 'path'. - - A string listing directories separated by 'os.pathsep'; defaults to - os.environ['PATH']. Returns the complete filename or None if not found. - """ - if path is None: - path = os.environ.get('PATH', os.defpath) - - paths = path.split(os.pathsep) - base, ext = os.path.splitext(executable) - - if (sys.platform == 'win32') and (ext != '.exe'): - executable = executable + '.exe' - - if not os.path.isfile(executable): - for p in paths: - f = os.path.join(p, executable) - if os.path.isfile(f): - # the file exists, we have a shot at spawn working - return f - return None - else: - return executable diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py deleted file mode 100644 index 3a5984f5c01..00000000000 --- a/Lib/distutils/sysconfig.py +++ /dev/null @@ -1,556 +0,0 @@ -"""Provide access to Python's configuration information. The specific -configuration variables available depend heavily on the platform and -configuration. The values may be retrieved using -get_config_var(name), and the list of variables is available via -get_config_vars().keys(). Additional convenience functions are also -available. - -Written by: Fred L. Drake, Jr. -Email: -""" - -import _imp -import os -import re -import sys - -from .errors import DistutilsPlatformError - -# These are needed in a couple of spots, so just compute them once. -PREFIX = os.path.normpath(sys.prefix) -EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -BASE_PREFIX = os.path.normpath(sys.base_prefix) -BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) - -# Path to the base directory of the project. On Windows the binary may -# live in project/PCbuild/win32 or project/PCbuild/amd64. -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) -else: - if sys.executable: - project_base = os.path.dirname(os.path.abspath(sys.executable)) - else: - # sys.executable can be empty if argv[0] has been changed and Python is - # unable to retrieve the real program name - project_base = os.getcwd() - - -# python_build: (Boolean) if true, we're either building Python or -# building an extension with an un-installed Python, so we use -# different (hard-wired) directories. -def _is_python_source_dir(d): - for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): - return True - return False - -_sys_home = getattr(sys, '_home', None) - -if os.name == 'nt': - def _fix_pcbuild(d): - if d and os.path.normcase(d).startswith( - os.path.normcase(os.path.join(PREFIX, "PCbuild"))): - return PREFIX - return d - project_base = _fix_pcbuild(project_base) - _sys_home = _fix_pcbuild(_sys_home) - -def _python_build(): - if _sys_home: - return _is_python_source_dir(_sys_home) - return _is_python_source_dir(project_base) - -python_build = _python_build() - - -# Calculate the build qualifier flags if they are defined. Adding the flags -# to the include and lib directories only makes sense for an installation, not -# an in-source build. -build_flags = '' -try: - if not python_build: - build_flags = sys.abiflags -except AttributeError: - # It's not a configure-based build, so the sys module doesn't have - # this attribute, which is fine. - pass - -def get_python_version(): - """Return a string containing the major and minor Python version, - leaving off the patchlevel. Sample return values could be '1.5' - or '2.2'. - """ - return '%d.%d' % sys.version_info[:2] - - -def get_python_inc(plat_specific=0, prefix=None): - """Return the directory containing installed Python header files. - - If 'plat_specific' is false (the default), this is the path to the - non-platform-specific header files, i.e. Python.h and so on; - otherwise, this is the path to platform-specific header files - (namely pyconfig.h). - - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - if os.name == "posix": - if python_build: - # Assume the executable is in the build directory. The - # pyconfig.h file should be in the same directory. Since - # the build directory may not be the source directory, we - # must use "srcdir" from the makefile to find the "Include" - # directory. - if plat_specific: - return _sys_home or project_base - else: - incdir = os.path.join(get_config_var('srcdir'), 'Include') - return os.path.normpath(incdir) - python_dir = 'python' + get_python_version() + build_flags - return os.path.join(prefix, "include", python_dir) - elif os.name == "nt": - if python_build: - # Include both the include and PC dir to ensure we can find - # pyconfig.h - return (os.path.join(prefix, "include") + os.path.pathsep + - os.path.join(prefix, "PC")) - return os.path.join(prefix, "include") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its C header files " - "on platform '%s'" % os.name) - - -def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): - """Return the directory containing the Python library (standard or - site additions). - - If 'plat_specific' is true, return the directory containing - platform-specific modules, i.e. any module from a non-pure-Python - module distribution; otherwise, return the platform-shared library - directory. If 'standard_lib' is true, return the directory - containing standard Python library modules; otherwise, return the - directory for site-specific modules. - - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. - """ - if prefix is None: - if standard_lib: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - else: - prefix = plat_specific and EXEC_PREFIX or PREFIX - - if os.name == "posix": - if plat_specific or standard_lib: - # Platform-specific modules (any module from a non-pure-Python - # module distribution) or standard Python library modules. - libdir = sys.platlibdir - else: - # Pure Python - libdir = "lib" - libpython = os.path.join(prefix, libdir, - # XXX RUSTPYTHON: changed from python->rustpython - "rustpython" + get_python_version()) - if standard_lib: - return libpython - else: - return os.path.join(libpython, "site-packages") - elif os.name == "nt": - if standard_lib: - return os.path.join(prefix, "Lib") - else: - return os.path.join(prefix, "Lib", "site-packages") - else: - raise DistutilsPlatformError( - "I don't know where Python installs its library " - "on platform '%s'" % os.name) - - - -def customize_compiler(compiler): - """Do any platform-specific customization of a CCompiler instance. - - Mainly needed on Unix, so we can plug in the information that - varies across Unices and is stored in Python's Makefile. - """ - if compiler.compiler_type == "unix": - if sys.platform == "darwin": - # Perform first-time customization of compiler-related - # config vars on OS X now that we know we need a compiler. - # This is primarily to support Pythons from binary - # installers. The kind and paths to build tools on - # the user system may vary significantly from the system - # that Python itself was built on. Also the user OS - # version and build tools may not support the same set - # of CPU architectures for universal builds. - global _config_vars - # Use get_config_var() to ensure _config_vars is initialized. - if not get_config_var('CUSTOMIZED_OSX_COMPILER'): - import _osx_support - _osx_support.customize_compiler(_config_vars) - _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' - - (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ - get_config_vars('CC', 'CXX', 'CFLAGS', - 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') - - if 'CC' in os.environ: - newcc = os.environ['CC'] - if (sys.platform == 'darwin' - and 'LDSHARED' not in os.environ - and ldshared.startswith(cc)): - # On OS X, if CC is overridden, use that as the default - # command for LDSHARED as well - ldshared = newcc + ldshared[len(cc):] - cc = newcc - if 'CXX' in os.environ: - cxx = os.environ['CXX'] - if 'LDSHARED' in os.environ: - ldshared = os.environ['LDSHARED'] - if 'CPP' in os.environ: - cpp = os.environ['CPP'] - else: - cpp = cc + " -E" # not always - if 'LDFLAGS' in os.environ: - ldshared = ldshared + ' ' + os.environ['LDFLAGS'] - if 'CFLAGS' in os.environ: - cflags = cflags + ' ' + os.environ['CFLAGS'] - ldshared = ldshared + ' ' + os.environ['CFLAGS'] - if 'CPPFLAGS' in os.environ: - cpp = cpp + ' ' + os.environ['CPPFLAGS'] - cflags = cflags + ' ' + os.environ['CPPFLAGS'] - ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] - if 'AR' in os.environ: - ar = os.environ['AR'] - if 'ARFLAGS' in os.environ: - archiver = ar + ' ' + os.environ['ARFLAGS'] - else: - archiver = ar + ' ' + ar_flags - - cc_cmd = cc + ' ' + cflags - compiler.set_executables( - preprocessor=cpp, - compiler=cc_cmd, - compiler_so=cc_cmd + ' ' + ccshared, - compiler_cxx=cxx, - linker_so=ldshared, - linker_exe=cc, - archiver=archiver) - - compiler.shared_lib_extension = shlib_suffix - - -def get_config_h_filename(): - """Return full pathname of installed pyconfig.h file.""" - if python_build: - if os.name == "nt": - inc_dir = os.path.join(_sys_home or project_base, "PC") - else: - inc_dir = _sys_home or project_base - else: - inc_dir = get_python_inc(plat_specific=1) - - return os.path.join(inc_dir, 'pyconfig.h') - - -def get_makefile_filename(): - """Return full pathname of installed Makefile from the Python build.""" - if python_build: - return os.path.join(_sys_home or project_base, "Makefile") - lib_dir = get_python_lib(plat_specific=0, standard_lib=1) - config_file = 'config-{}{}'.format(get_python_version(), build_flags) - if hasattr(sys.implementation, '_multiarch'): - config_file += '-%s' % sys.implementation._multiarch - return os.path.join(lib_dir, config_file, 'Makefile') - - -def parse_config_h(fp, g=None): - """Parse a config.h-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - if g is None: - g = {} - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - # - while True: - line = fp.readline() - if not line: - break - m = define_rx.match(line) - if m: - n, v = m.group(1, 2) - try: v = int(v) - except ValueError: pass - g[n] = v - else: - m = undef_rx.match(line) - if m: - g[m.group(1)] = 0 - return g - - -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") -_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)") -_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}") - -def parse_makefile(fn, g=None): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - from distutils.text_file import TextFile - fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1, errors="surrogateescape") - - if g is None: - g = {} - done = {} - notdone = {} - - while True: - line = fp.readline() - if line is None: # eof - break - m = _variable_rx.match(line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # Variables with a 'PY_' prefix in the makefile. These need to - # be made available without that prefix through sysconfig. - # Special care is needed to ensure that variable expansion works, even - # if the expansion uses the name without a prefix. - renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - - # do variable interpolation here - while notdone: - for name in list(notdone): - value = notdone[name] - m = _findvar1_rx.search(value) or _findvar2_rx.search(value) - if m: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if name.startswith('PY_') and name[3:] in renamed_variables: - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - else: - done[n] = item = "" - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - del notdone[name] - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - else: - # bogus variable reference; just drop it since we can't deal - del notdone[name] - - fp.close() - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - g.update(done) - return g - - -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. - """ - - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. - - while True: - m = _findvar1_rx.search(s) or _findvar2_rx.search(s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] - else: - break - return s - - -_config_vars = None - -def _init_posix(): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see the sysconfig module - name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars - global _config_vars - _config_vars = {} - _config_vars.update(build_time_vars) - - -def _init_nt(): - """Initialize the module as appropriate for NT""" - g = {} - # set basic install directories - g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) - g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) - - # XXX hmmm.. a normal install puts include files here - g['INCLUDEPY'] = get_python_inc(plat_specific=0) - - g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - g['EXE'] = ".exe" - g['VERSION'] = get_python_version().replace(".", "") - g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) - - global _config_vars - _config_vars = g - - -def get_config_vars(*args): - """With no arguments, return a dictionary of all configuration - variables relevant for the current platform. Generally this includes - everything needed to build extensions and install both pure modules and - extensions. On Unix, this means every variable defined in Python's - installed Makefile; on Windows it's a much smaller set. - - With arguments, return a list of values that result from looking up - each argument in the configuration variable dictionary. - """ - global _config_vars - if _config_vars is None: - func = globals().get("_init_" + os.name) - if func: - func() - else: - _config_vars = {} - - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _config_vars['prefix'] = PREFIX - _config_vars['exec_prefix'] = EXEC_PREFIX - - # For backward compatibility, see issue19555 - SO = _config_vars.get('EXT_SUFFIX') - if SO is not None: - _config_vars['SO'] = SO - - # Always convert srcdir to an absolute path - srcdir = _config_vars.get('srcdir', project_base) - if os.name == 'posix': - if python_build: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir)) - - # Convert srcdir into an absolute path if it appears necessary. - # Normally it is relative to the build directory. However, during - # testing, for example, we might be running a non-installed python - # from a different directory. - if python_build and os.name == "posix": - base = project_base - if (not os.path.isabs(_config_vars['srcdir']) and - base != os.getcwd()): - # srcdir is relative and we are not in the same directory - # as the executable. Assume executable is in the build - # directory and make srcdir absolute. - srcdir = os.path.join(base, _config_vars['srcdir']) - _config_vars['srcdir'] = os.path.normpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_config_vars) - - if args: - vals = [] - for name in args: - vals.append(_config_vars.get(name)) - return vals - else: - return _config_vars - -def get_config_var(name): - """Return the value of a single variable using the dictionary - returned by 'get_config_vars()'. Equivalent to - get_config_vars().get(name) - """ - if name == 'SO': - import warnings - warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2) - return get_config_vars().get(name) diff --git a/Lib/distutils/text_file.py b/Lib/distutils/text_file.py deleted file mode 100644 index 93abad38f43..00000000000 --- a/Lib/distutils/text_file.py +++ /dev/null @@ -1,286 +0,0 @@ -"""text_file - -provides the TextFile class, which gives an interface to text files -that (optionally) takes care of stripping comments, ignoring blank -lines, and joining lines with backslashes.""" - -import sys, io - - -class TextFile: - """Provides a file-like object that takes care of all the things you - commonly want to do when processing a text file that has some - line-by-line syntax: strip comments (as long as "#" is your - comment character), skip blank lines, join adjacent lines by - escaping the newline (ie. backslash at end of line), strip - leading and/or trailing whitespace. All of these are optional - and independently controllable. - - Provides a 'warn()' method so you can generate warning messages that - report physical line number, even if the logical line in question - spans multiple physical lines. Also provides 'unreadline()' for - implementing line-at-a-time lookahead. - - Constructor is called as: - - TextFile (filename=None, file=None, **options) - - It bombs (RuntimeError) if both 'filename' and 'file' are None; - 'filename' should be a string, and 'file' a file object (or - something that provides 'readline()' and 'close()' methods). It is - recommended that you supply at least 'filename', so that TextFile - can include it in warning messages. If 'file' is not supplied, - TextFile creates its own using 'io.open()'. - - The options are all boolean, and affect the value returned by - 'readline()': - strip_comments [default: true] - strip from "#" to end-of-line, as well as any whitespace - leading up to the "#" -- unless it is escaped by a backslash - lstrip_ws [default: false] - strip leading whitespace from each line before returning it - rstrip_ws [default: true] - strip trailing whitespace (including line terminator!) from - each line before returning it - skip_blanks [default: true} - skip lines that are empty *after* stripping comments and - whitespace. (If both lstrip_ws and rstrip_ws are false, - then some lines may consist of solely whitespace: these will - *not* be skipped, even if 'skip_blanks' is true.) - join_lines [default: false] - if a backslash is the last non-newline character on a line - after stripping comments and whitespace, join the following line - to it to form one "logical line"; if N consecutive lines end - with a backslash, then N+1 physical lines will be joined to - form one logical line. - collapse_join [default: false] - strip leading whitespace from lines that are joined to their - predecessor; only matters if (join_lines and not lstrip_ws) - errors [default: 'strict'] - error handler used to decode the file content - - Note that since 'rstrip_ws' can strip the trailing newline, the - semantics of 'readline()' must differ from those of the builtin file - object's 'readline()' method! In particular, 'readline()' returns - None for end-of-file: an empty string might just be a blank line (or - an all-whitespace line), if 'rstrip_ws' is true but 'skip_blanks' is - not.""" - - default_options = { 'strip_comments': 1, - 'skip_blanks': 1, - 'lstrip_ws': 0, - 'rstrip_ws': 1, - 'join_lines': 0, - 'collapse_join': 0, - 'errors': 'strict', - } - - def __init__(self, filename=None, file=None, **options): - """Construct a new TextFile object. At least one of 'filename' - (a string) and 'file' (a file-like object) must be supplied. - They keyword argument options are described above and affect - the values returned by 'readline()'.""" - if filename is None and file is None: - raise RuntimeError("you must supply either or both of 'filename' and 'file'") - - # set values for all options -- either from client option hash - # or fallback to default_options - for opt in self.default_options.keys(): - if opt in options: - setattr(self, opt, options[opt]) - else: - setattr(self, opt, self.default_options[opt]) - - # sanity check client option hash - for opt in options.keys(): - if opt not in self.default_options: - raise KeyError("invalid TextFile option '%s'" % opt) - - if file is None: - self.open(filename) - else: - self.filename = filename - self.file = file - self.current_line = 0 # assuming that file is at BOF! - - # 'linebuf' is a stack of lines that will be emptied before we - # actually read from the file; it's only populated by an - # 'unreadline()' operation - self.linebuf = [] - - def open(self, filename): - """Open a new file named 'filename'. This overrides both the - 'filename' and 'file' arguments to the constructor.""" - self.filename = filename - self.file = io.open(self.filename, 'r', errors=self.errors) - self.current_line = 0 - - def close(self): - """Close the current file and forget everything we know about it - (filename, current line number).""" - file = self.file - self.file = None - self.filename = None - self.current_line = None - file.close() - - def gen_error(self, msg, line=None): - outmsg = [] - if line is None: - line = self.current_line - outmsg.append(self.filename + ", ") - if isinstance(line, (list, tuple)): - outmsg.append("lines %d-%d: " % tuple(line)) - else: - outmsg.append("line %d: " % line) - outmsg.append(str(msg)) - return "".join(outmsg) - - def error(self, msg, line=None): - raise ValueError("error: " + self.gen_error(msg, line)) - - def warn(self, msg, line=None): - """Print (to stderr) a warning message tied to the current logical - line in the current file. If the current logical line in the - file spans multiple physical lines, the warning refers to the - whole range, eg. "lines 3-5". If 'line' supplied, it overrides - the current line number; it may be a list or tuple to indicate a - range of physical lines, or an integer for a single physical - line.""" - sys.stderr.write("warning: " + self.gen_error(msg, line) + "\n") - - def readline(self): - """Read and return a single logical line from the current file (or - from an internal buffer if lines have previously been "unread" - with 'unreadline()'). If the 'join_lines' option is true, this - may involve reading multiple physical lines concatenated into a - single string. Updates the current line number, so calling - 'warn()' after 'readline()' emits a warning about the physical - line(s) just read. Returns None on end-of-file, since the empty - string can occur if 'rstrip_ws' is true but 'strip_blanks' is - not.""" - # If any "unread" lines waiting in 'linebuf', return the top - # one. (We don't actually buffer read-ahead data -- lines only - # get put in 'linebuf' if the client explicitly does an - # 'unreadline()'. - if self.linebuf: - line = self.linebuf[-1] - del self.linebuf[-1] - return line - - buildup_line = '' - - while True: - # read the line, make it None if EOF - line = self.file.readline() - if line == '': - line = None - - if self.strip_comments and line: - - # Look for the first "#" in the line. If none, never - # mind. If we find one and it's the first character, or - # is not preceded by "\", then it starts a comment -- - # strip the comment, strip whitespace before it, and - # carry on. Otherwise, it's just an escaped "#", so - # unescape it (and any other escaped "#"'s that might be - # lurking in there) and otherwise leave the line alone. - - pos = line.find("#") - if pos == -1: # no "#" -- no comments - pass - - # It's definitely a comment -- either "#" is the first - # character, or it's elsewhere and unescaped. - elif pos == 0 or line[pos-1] != "\\": - # Have to preserve the trailing newline, because it's - # the job of a later step (rstrip_ws) to remove it -- - # and if rstrip_ws is false, we'd better preserve it! - # (NB. this means that if the final line is all comment - # and has no trailing newline, we will think that it's - # EOF; I think that's OK.) - eol = (line[-1] == '\n') and '\n' or '' - line = line[0:pos] + eol - - # If all that's left is whitespace, then skip line - # *now*, before we try to join it to 'buildup_line' -- - # that way constructs like - # hello \\ - # # comment that should be ignored - # there - # result in "hello there". - if line.strip() == "": - continue - else: # it's an escaped "#" - line = line.replace("\\#", "#") - - # did previous line end with a backslash? then accumulate - if self.join_lines and buildup_line: - # oops: end of file - if line is None: - self.warn("continuation line immediately precedes " - "end-of-file") - return buildup_line - - if self.collapse_join: - line = line.lstrip() - line = buildup_line + line - - # careful: pay attention to line number when incrementing it - if isinstance(self.current_line, list): - self.current_line[1] = self.current_line[1] + 1 - else: - self.current_line = [self.current_line, - self.current_line + 1] - # just an ordinary line, read it as usual - else: - if line is None: # eof - return None - - # still have to be careful about incrementing the line number! - if isinstance(self.current_line, list): - self.current_line = self.current_line[1] + 1 - else: - self.current_line = self.current_line + 1 - - # strip whitespace however the client wants (leading and - # trailing, or one or the other, or neither) - if self.lstrip_ws and self.rstrip_ws: - line = line.strip() - elif self.lstrip_ws: - line = line.lstrip() - elif self.rstrip_ws: - line = line.rstrip() - - # blank line (whether we rstrip'ed or not)? skip to next line - # if appropriate - if (line == '' or line == '\n') and self.skip_blanks: - continue - - if self.join_lines: - if line[-1] == '\\': - buildup_line = line[:-1] - continue - - if line[-2:] == '\\\n': - buildup_line = line[0:-2] + '\n' - continue - - # well, I guess there's some actual content there: return it - return line - - def readlines(self): - """Read and return the list of all logical lines remaining in the - current file.""" - lines = [] - while True: - line = self.readline() - if line is None: - return lines - lines.append(line) - - def unreadline(self, line): - """Push 'line' (a string) onto an internal buffer that will be - checked by future 'readline()' calls. Handy for implementing - a parser with line-at-a-time lookahead.""" - self.linebuf.append(line) diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py deleted file mode 100644 index 4524417e668..00000000000 --- a/Lib/distutils/unixccompiler.py +++ /dev/null @@ -1,333 +0,0 @@ -"""distutils.unixccompiler - -Contains the UnixCCompiler class, a subclass of CCompiler that handles -the "typical" Unix-style command-line C compiler: - * macros defined with -Dname[=value] - * macros undefined with -Uname - * include search directories specified with -Idir - * libraries specified with -lllib - * library search directories specified with -Ldir - * compile handled by 'cc' (or similar) executable with -c option: - compiles .c to .o - * link static library handled by 'ar' command (possibly with 'ranlib') - * link shared library handled by 'cc -shared' -""" - -import os, sys, re - -from distutils import sysconfig -from distutils.dep_util import newer -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.errors import \ - DistutilsExecError, CompileError, LibError, LinkError -from distutils import log - -if sys.platform == 'darwin': - import _osx_support - -# XXX Things not currently handled: -# * optimization/debug/warning flags; we just use whatever's in Python's -# Makefile and live with it. Is this adequate? If not, we might -# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, -# SunCCompiler, and I suspect down that road lies madness. -# * even if we don't know a warning flag from an optimization flag, -# we need some way for outsiders to feed preprocessor/compiler/linker -# flags in to us -- eg. a sysadmin might want to mandate certain flags -# via a site config file, or a user might want to set something for -# compiling this module distribution only via the setup.py command -# line, whatever. As long as these options come from something on the -# current system, they can be as system-dependent as they like, and we -# should just happily stuff them into the preprocessor/compiler/linker -# options and carry on. - - -class UnixCCompiler(CCompiler): - - compiler_type = 'unix' - - # These are used by CCompiler in two places: the constructor sets - # instance attributes 'preprocessor', 'compiler', etc. from them, and - # 'set_executable()' allows any of these to be set. The defaults here - # are pretty generic; they will probably have to be set by an outsider - # (eg. using information discovered by the sysconfig about building - # Python extensions). - executables = {'preprocessor' : None, - 'compiler' : ["cc"], - 'compiler_so' : ["cc"], - 'compiler_cxx' : ["cc"], - 'linker_so' : ["cc", "-shared"], - 'linker_exe' : ["cc"], - 'archiver' : ["ar", "-cr"], - 'ranlib' : None, - } - - if sys.platform[:6] == "darwin": - executables['ranlib'] = ["ranlib"] - - # Needed for the filename generation methods provided by the base - # class, CCompiler. NB. whoever instantiates/uses a particular - # UnixCCompiler instance should set 'shared_lib_ext' -- we set a - # reasonable common default here, but it's not necessarily used on all - # Unices! - - src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".so" - dylib_lib_extension = ".dylib" - xcode_stub_lib_extension = ".tbd" - static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" - xcode_stub_lib_format = dylib_lib_format - if sys.platform == "cygwin": - exe_extension = ".exe" - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - fixed_args = self._fix_compile_args(None, macros, include_dirs) - ignore, macros, include_dirs = fixed_args - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = self.preprocessor + pp_opts - if output_file: - pp_args.extend(['-o', output_file]) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or we're - # generating output to stdout, or there's a target output file and - # the source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - raise CompileError(msg) - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - compiler_so = self.compiler_so - if sys.platform == 'darwin': - compiler_so = _osx_support.compiler_fixup(compiler_so, - cc_args + extra_postargs) - try: - self.spawn(compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def create_static_lib(self, objects, output_libname, - output_dir=None, debug=0, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - - output_filename = \ - self.library_filename(output_libname, output_dir=output_dir) - - if self._need_link(objects, output_filename): - self.mkpath(os.path.dirname(output_filename)) - self.spawn(self.archiver + - [output_filename] + - objects + self.objects) - - # Not many Unices required ranlib anymore -- SunOS 4.x is, I - # think the only major Unix that does. Maybe we need some - # platform intelligence here to skip ranlib if it's not - # needed -- or maybe Python's configure script took care of - # it for us, hence the check for leading colon. - if self.ranlib: - try: - self.spawn(self.ranlib + [output_filename]) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def link(self, target_desc, objects, - output_filename, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - # filter out standard library paths, which are not explicitely needed - # for linking - system_libdirs = ['/lib', '/lib64', '/usr/lib', '/usr/lib64'] - multiarch = sysconfig.get_config_var("MULTIARCH") - if multiarch: - system_libdirs.extend(['/lib/%s' % multiarch, '/usr/lib/%s' % multiarch]) - library_dirs = [dir for dir in library_dirs - if not dir in system_libdirs] - runtime_library_dirs = [dir for dir in runtime_library_dirs - if not dir in system_libdirs] - - lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if not isinstance(output_dir, (str, type(None))): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - if target_lang == "c++" and self.compiler_cxx: - # skip over environment variable settings if /usr/bin/env - # is used to set up the linker's environment. - # This is needed on OSX. Note: this assumes that the - # normal and C++ compiler have the same environment - # settings. - i = 0 - if os.path.basename(linker[0]) == "env": - i = 1 - while '=' in linker[i]: - i += 1 - linker[i] = self.compiler_cxx[i] - - if sys.platform == 'darwin': - linker = _osx_support.compiler_fixup(linker, ld_args) - - self.spawn(linker + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "-L" + dir - - def _is_gcc(self, compiler_name): - return "gcc" in compiler_name or "g++" in compiler_name - - def runtime_library_dir_option(self, dir): - # XXX Hackish, at the very least. See Python bug #445902: - # http://sourceforge.net/tracker/index.php - # ?func=detail&aid=445902&group_id=5470&atid=105470 - # Linkers on different platforms need different options to - # specify that directories need to be added to the list of - # directories searched for dependencies when a dynamic library - # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to - # be told to pass the -R option through to the linker, whereas - # other compilers and gcc on other systems just know this. - # Other compilers may need something slightly different. At - # this time, there's no way to determine this information from - # the configuration data stored in the Python installation, so - # we use this hack. - compiler = os.path.basename(sysconfig.get_config_var("CC")) - if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir - elif sys.platform[:7] == "freebsd": - return "-Wl,-rpath=" + dir - elif sys.platform[:5] == "hp-ux": - if self._is_gcc(compiler): - return ["-Wl,+s", "-L" + dir] - return ["+s", "-L" + dir] - elif sys.platform[:7] == "irix646" or sys.platform[:6] == "osf1V5": - return ["-rpath", dir] - else: - if self._is_gcc(compiler): - # gcc on non-GNU systems does not need -Wl, but can - # use it anyway. Since distutils has always passed in - # -Wl whenever gcc was used in the past it is probably - # safest to keep doing so. - if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH - # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir - else: - return "-Wl,-R" + dir - else: - # No idea how --enable-new-dtags would be passed on to - # ld if this system was using GNU ld. Don't know if a - # system like this even exists. - return "-R" + dir - - def library_option(self, lib): - return "-l" + lib - - def find_library_file(self, dirs, lib, debug=0): - shared_f = self.library_filename(lib, lib_type='shared') - dylib_f = self.library_filename(lib, lib_type='dylib') - xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') - static_f = self.library_filename(lib, lib_type='static') - - if sys.platform == 'darwin': - # On OSX users can specify an alternate SDK using - # '-isysroot', calculate the SDK root if it is specified - # (and use it further on) - # - # Note that, as of Xcode 7, Apple SDKs may contain textual stub - # libraries with .tbd extensions rather than the normal .dylib - # shared libraries installed in /. The Apple compiler tool - # chain handles this transparently but it can cause problems - # for programs that are being built with an SDK and searching - # for specific libraries. Callers of find_library_file need to - # keep in mind that the base filename of the returned SDK library - # file might have a different extension from that of the library - # file installed on the running system, for example: - # /Applications/Xcode.app/Contents/Developer/Platforms/ - # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ - # usr/lib/libedit.tbd - # vs - # /usr/lib/libedit.dylib - cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s+(\S+)', cflags) - if m is None: - sysroot = '/' - else: - sysroot = m.group(1) - - - - for dir in dirs: - shared = os.path.join(dir, shared_f) - dylib = os.path.join(dir, dylib_f) - static = os.path.join(dir, static_f) - xcode_stub = os.path.join(dir, xcode_stub_f) - - if sys.platform == 'darwin' and ( - dir.startswith('/System/') or ( - dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): - - shared = os.path.join(sysroot, dir[1:], shared_f) - dylib = os.path.join(sysroot, dir[1:], dylib_f) - static = os.path.join(sysroot, dir[1:], static_f) - xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) - - # We're second-guessing the linker here, with not much hard - # data to go on: GCC seems to prefer the shared library, so I'm - # assuming that *all* Unix C compilers do. And of course I'm - # ignoring even GCC's "-static" option. So sue me. - if os.path.exists(dylib): - return dylib - elif os.path.exists(xcode_stub): - return xcode_stub - elif os.path.exists(shared): - return shared - elif os.path.exists(static): - return static - - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py deleted file mode 100644 index fdcf6fabae2..00000000000 --- a/Lib/distutils/util.py +++ /dev/null @@ -1,557 +0,0 @@ -"""distutils.util - -Miscellaneous utility functions -- anything that doesn't fit into -one of the other *util.py modules. -""" - -import os -import re -import importlib.util -import string -import sys -from distutils.errors import DistutilsPlatformError -from distutils.dep_util import newer -from distutils.spawn import spawn -from distutils import log -from distutils.errors import DistutilsByteCompileError - -def get_platform (): - """Return a string that identifies the current platform. This is used - mainly to distinguish platform-specific build directories and - platform-specific built distributions. Typically includes the OS name - and version and the architecture (as supplied by 'os.uname()'), - although the exact information included depends on the OS; eg. for IRIX - the architecture isn't particularly important (IRIX only runs on SGI - hardware), but for Linux the kernel version isn't particularly - important. - - Examples of returned values: - linux-i586 - linux-alpha (?) - solaris-2.6-sun4u - irix-5.3 - irix64-6.2 - - Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) - win-ia64 (64bit Windows on Itanium) - win32 (all others - specifically, sys.platform is returned) - - For other non-POSIX platforms, currently just returns 'sys.platform'. - """ - if os.name == 'nt': - # sniff sys.version for architecture. - prefix = " bit (" - i = sys.version.find(prefix) - if i == -1: - return sys.platform - j = sys.version.find(")", i) - look = sys.version[i+len(prefix):j].lower() - if look == 'amd64': - return 'win-amd64' - if look == 'itanium': - return 'win-ia64' - return sys.platform - - # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - - if os.name != "posix" or not hasattr(os, 'uname'): - # XXX what about the architecture? NT is Intel or Alpha, - # Mac OS is M68k or PPC, etc. - return sys.platform - - # Try to distinguish various flavours of Unix - - (osname, host, release, version, machine) = os.uname() - - # Convert the OS name to lowercase, remove '/' characters - # (to accommodate BSD/OS), and translate spaces (for "Power Macintosh") - osname = osname.lower().replace('/', '') - machine = machine.replace(' ', '_') - machine = machine.replace('/', '-') - - if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return "%s-%s" % (osname, machine) - elif osname[:5] == "sunos": - if release[0] >= "5": # SunOS 5 == Solaris 2 - osname = "solaris" - release = "%d.%s" % (int(release[0]) - 3, release[2:]) - # We can't use "platform.architecture()[0]" because a - # bootstrap problem. We use a dict to get an error - # if some suspicious happens. - bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} - machine += ".%s" % bitness[sys.maxsize] - # fall through to standard osname-release-machine representation - elif osname[:4] == "irix": # could be "irix64"! - return "%s-%s" % (osname, release) - elif osname[:3] == "aix": - return "%s-%s.%s" % (osname, version, release) - elif osname[:6] == "cygwin": - osname = "cygwin" - rel_re = re.compile (r'[\d.]+', re.ASCII) - m = rel_re.match(release) - if m: - release = m.group() - elif osname[:6] == "darwin": - import _osx_support, distutils.sysconfig - osname, release, machine = _osx_support.get_platform_osx( - distutils.sysconfig.get_config_vars(), - osname, release, machine) - - return "%s-%s-%s" % (osname, release, machine) - -# get_platform () - - -def convert_path (pathname): - """Return 'pathname' as a name that will work on the native filesystem, - i.e. split it on '/' and put it back together again using the current - directory separator. Needed because filenames in the setup script are - always supplied in Unix style, and have to be converted to the local - convention before we can actually use them in the filesystem. Raises - ValueError on non-Unix-ish systems if 'pathname' either starts or - ends with a slash. - """ - if os.sep == '/': - return pathname - if not pathname: - return pathname - if pathname[0] == '/': - raise ValueError("path '%s' cannot be absolute" % pathname) - if pathname[-1] == '/': - raise ValueError("path '%s' cannot end with '/'" % pathname) - - paths = pathname.split('/') - while '.' in paths: - paths.remove('.') - if not paths: - return os.curdir - return os.path.join(*paths) - -# convert_path () - - -def change_root (new_root, pathname): - """Return 'pathname' with 'new_root' prepended. If 'pathname' is - relative, this is equivalent to "os.path.join(new_root,pathname)". - Otherwise, it requires making 'pathname' relative and then joining the - two, which is tricky on DOS/Windows and Mac OS. - """ - if os.name == 'posix': - if not os.path.isabs(pathname): - return os.path.join(new_root, pathname) - else: - return os.path.join(new_root, pathname[1:]) - - elif os.name == 'nt': - (drive, path) = os.path.splitdrive(pathname) - if path[0] == '\\': - path = path[1:] - return os.path.join(new_root, path) - - else: - raise DistutilsPlatformError("nothing known about platform '%s'" % os.name) - - -_environ_checked = 0 -def check_environ (): - """Ensure that 'os.environ' has all the environment variables we - guarantee that users can use in config files, command-line options, - etc. Currently this includes: - HOME - user's home directory (Unix only) - PLAT - description of the current platform, including hardware - and OS (see 'get_platform()') - """ - global _environ_checked - if _environ_checked: - return - - if os.name == 'posix' and 'HOME' not in os.environ: - import pwd - os.environ['HOME'] = pwd.getpwuid(os.getuid())[5] - - if 'PLAT' not in os.environ: - os.environ['PLAT'] = get_platform() - - _environ_checked = 1 - - -def subst_vars (s, local_vars): - """Perform shell/Perl-style variable substitution on 'string'. Every - occurrence of '$' followed by a name is considered a variable, and - variable is substituted by the value found in the 'local_vars' - dictionary, or in 'os.environ' if it's not in 'local_vars'. - 'os.environ' is first checked/augmented to guarantee that it contains - certain values: see 'check_environ()'. Raise ValueError for any - variables not found in either 'local_vars' or 'os.environ'. - """ - check_environ() - def _subst (match, local_vars=local_vars): - var_name = match.group(1) - if var_name in local_vars: - return str(local_vars[var_name]) - else: - return os.environ[var_name] - - try: - return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s) - except KeyError as var: - raise ValueError("invalid variable '$%s'" % var) - -# subst_vars () - - -def grok_environment_error (exc, prefix="error: "): - # Function kept for backward compatibility. - # Used to try clever things with EnvironmentErrors, - # but nowadays str(exception) produces good messages. - return prefix + str(exc) - - -# Needed by 'split_quoted()' -_wordchars_re = _squote_re = _dquote_re = None -def _init_regex(): - global _wordchars_re, _squote_re, _dquote_re - _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) - _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") - _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') - -def split_quoted (s): - """Split a string up according to Unix shell-like rules for quotes and - backslashes. In short: words are delimited by spaces, as long as those - spaces are not escaped by a backslash, or inside a quoted string. - Single and double quotes are equivalent, and the quote characters can - be backslash-escaped. The backslash is stripped from any two-character - escape sequence, leaving only the escaped character. The quote - characters are stripped from any quoted string. Returns a list of - words. - """ - - # This is a nice algorithm for splitting up a single string, since it - # doesn't require character-by-character examination. It was a little - # bit of a brain-bender to get it working right, though... - if _wordchars_re is None: _init_regex() - - s = s.strip() - words = [] - pos = 0 - - while s: - m = _wordchars_re.match(s, pos) - end = m.end() - if end == len(s): - words.append(s[:end]) - break - - if s[end] in string.whitespace: # unescaped, unquoted whitespace: now - words.append(s[:end]) # we definitely have a word delimiter - s = s[end:].lstrip() - pos = 0 - - elif s[end] == '\\': # preserve whatever is being escaped; - # will become part of the current word - s = s[:end] + s[end+1:] - pos = end+1 - - else: - if s[end] == "'": # slurp singly-quoted string - m = _squote_re.match(s, end) - elif s[end] == '"': # slurp doubly-quoted string - m = _dquote_re.match(s, end) - else: - raise RuntimeError("this can't happen (bad char '%c')" % s[end]) - - if m is None: - raise ValueError("bad string (mismatched %s quotes?)" % s[end]) - - (beg, end) = m.span() - s = s[:beg] + s[beg+1:end-1] + s[end:] - pos = m.end() - 2 - - if pos >= len(s): - words.append(s) - break - - return words - -# split_quoted () - - -def execute (func, args, msg=None, verbose=0, dry_run=0): - """Perform some action that affects the outside world (eg. by - writing to the filesystem). Such actions are special because they - are disabled by the 'dry_run' flag. This method takes care of all - that bureaucracy for you; all you have to do is supply the - function to call and an argument tuple for it (to embody the - "external action" being performed), and an optional message to - print. - """ - if msg is None: - msg = "%s%r" % (func.__name__, args) - if msg[-2:] == ',)': # correct for singleton tuple - msg = msg[0:-2] + ')' - - log.info(msg) - if not dry_run: - func(*args) - - -def strtobool (val): - """Convert a string representation of truth to true (1) or false (0). - - True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values - are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if - 'val' is anything else. - """ - val = val.lower() - if val in ('y', 'yes', 't', 'true', 'on', '1'): - return 1 - elif val in ('n', 'no', 'f', 'false', 'off', '0'): - return 0 - else: - raise ValueError("invalid truth value %r" % (val,)) - - -def byte_compile (py_files, - optimize=0, force=0, - prefix=None, base_dir=None, - verbose=1, dry_run=0, - direct=None): - """Byte-compile a collection of Python source files to .pyc - files in a __pycache__ subdirectory. 'py_files' is a list - of files to compile; any files that don't end in ".py" are silently - skipped. 'optimize' must be one of the following: - 0 - don't optimize - 1 - normal optimization (like "python -O") - 2 - extra optimization (like "python -OO") - If 'force' is true, all files are recompiled regardless of - timestamps. - - The source filename encoded in each bytecode file defaults to the - filenames listed in 'py_files'; you can modify these with 'prefix' and - 'basedir'. 'prefix' is a string that will be stripped off of each - source filename, and 'base_dir' is a directory name that will be - prepended (after 'prefix' is stripped). You can supply either or both - (or neither) of 'prefix' and 'base_dir', as you wish. - - If 'dry_run' is true, doesn't actually do anything that would - affect the filesystem. - - Byte-compilation is either done directly in this interpreter process - with the standard py_compile module, or indirectly by writing a - temporary script and executing it. Normally, you should let - 'byte_compile()' figure out to use direct compilation or not (see - the source for details). The 'direct' flag is used by the script - generated in indirect mode; unless you know what you're doing, leave - it set to None. - """ - - # Late import to fix a bootstrap issue: _posixsubprocess is built by - # setup.py, but setup.py uses distutils. - import subprocess - - # nothing is done if sys.dont_write_bytecode is True - if sys.dont_write_bytecode: - raise DistutilsByteCompileError('byte-compiling is disabled.') - - # First, if the caller didn't force us into direct or indirect mode, - # figure out which mode we should be in. We take a conservative - # approach: choose direct mode *only* if the current interpreter is - # in debug mode and optimize is 0. If we're not in debug mode (-O - # or -OO), we don't know which level of optimization this - # interpreter is running with, so we can't do direct - # byte-compilation and be certain that it's the right thing. Thus, - # always compile indirectly if the current interpreter is in either - # optimize mode, or if either optimization level was requested by - # the caller. - if direct is None: - direct = (__debug__ and optimize == 0) - - # "Indirect" byte-compilation: write a temporary script and then - # run it with the appropriate flags. - if not direct: - try: - from tempfile import mkstemp - (script_fd, script_name) = mkstemp(".py") - except ImportError: - from tempfile import mktemp - (script_fd, script_name) = None, mktemp(".py") - log.info("writing byte-compilation script '%s'", script_name) - if not dry_run: - if script_fd is not None: - script = os.fdopen(script_fd, "w") - else: - script = open(script_name, "w") - - script.write("""\ -from distutils.util import byte_compile -files = [ -""") - - # XXX would be nice to write absolute filenames, just for - # safety's sake (script should be more robust in the face of - # chdir'ing before running it). But this requires abspath'ing - # 'prefix' as well, and that breaks the hack in build_lib's - # 'byte_compile()' method that carefully tacks on a trailing - # slash (os.sep really) to make sure the prefix here is "just - # right". This whole prefix business is rather delicate -- the - # problem is that it's really a directory, but I'm treating it - # as a dumb string, so trailing slashes and so forth matter. - - #py_files = map(os.path.abspath, py_files) - #if prefix: - # prefix = os.path.abspath(prefix) - - script.write(",\n".join(map(repr, py_files)) + "]\n") - script.write(""" -byte_compile(files, optimize=%r, force=%r, - prefix=%r, base_dir=%r, - verbose=%r, dry_run=0, - direct=1) -""" % (optimize, force, prefix, base_dir, verbose)) - - script.close() - - cmd = [sys.executable] - cmd.extend(subprocess._optim_args_from_interpreter_flags()) - cmd.append(script_name) - spawn(cmd, dry_run=dry_run) - execute(os.remove, (script_name,), "removing %s" % script_name, - dry_run=dry_run) - - # "Direct" byte-compilation: use the py_compile module to compile - # right here, right now. Note that the script generated in indirect - # mode simply calls 'byte_compile()' in direct mode, a weird sort of - # cross-process recursion. Hey, it works! - else: - from py_compile import compile - - for file in py_files: - if file[-3:] != ".py": - # This lets us be lazy and not filter filenames in - # the "install_lib" command. - continue - - # Terminology from the py_compile module: - # cfile - byte-compiled file - # dfile - purported source filename (same as 'file' by default) - if optimize >= 0: - opt = '' if optimize == 0 else optimize - cfile = importlib.util.cache_from_source( - file, optimization=opt) - else: - cfile = importlib.util.cache_from_source(file) - dfile = file - if prefix: - if file[:len(prefix)] != prefix: - raise ValueError("invalid prefix: filename %r doesn't start with %r" - % (file, prefix)) - dfile = dfile[len(prefix):] - if base_dir: - dfile = os.path.join(base_dir, dfile) - - cfile_base = os.path.basename(cfile) - if direct: - if force or newer(file, cfile): - log.info("byte-compiling %s to %s", file, cfile_base) - if not dry_run: - compile(file, cfile, dfile) - else: - log.debug("skipping byte-compilation of %s to %s", - file, cfile_base) - -# byte_compile () - -def rfc822_escape (header): - """Return a version of the string escaped for inclusion in an - RFC-822 header, by ensuring there are 8 spaces space after each newline. - """ - lines = header.split('\n') - sep = '\n' + 8 * ' ' - return sep.join(lines) - -# 2to3 support - -def run_2to3(files, fixer_names=None, options=None, explicit=None): - """Invoke 2to3 on a list of Python files. - The files should all come from the build area, as the - modification is done in-place. To reduce the build time, - only files modified since the last invocation of this - function should be passed in the files argument.""" - - if not files: - return - - # Make this class local, to delay import of 2to3 - from lib2to3.refactor import RefactoringTool, get_fixers_from_package - class DistutilsRefactoringTool(RefactoringTool): - def log_error(self, msg, *args, **kw): - log.error(msg, *args) - - def log_message(self, msg, *args): - log.info(msg, *args) - - def log_debug(self, msg, *args): - log.debug(msg, *args) - - if fixer_names is None: - fixer_names = get_fixers_from_package('lib2to3.fixes') - r = DistutilsRefactoringTool(fixer_names, options=options) - r.refactor(files, write=True) - -def copydir_run_2to3(src, dest, template=None, fixer_names=None, - options=None, explicit=None): - """Recursively copy a directory, only copying new and changed files, - running run_2to3 over all newly copied Python modules afterward. - - If you give a template string, it's parsed like a MANIFEST.in. - """ - from distutils.dir_util import mkpath - from distutils.file_util import copy_file - from distutils.filelist import FileList - filelist = FileList() - curdir = os.getcwd() - os.chdir(src) - try: - filelist.findall() - finally: - os.chdir(curdir) - filelist.files[:] = filelist.allfiles - if template: - for line in template.splitlines(): - line = line.strip() - if not line: continue - filelist.process_template_line(line) - copied = [] - for filename in filelist.files: - outname = os.path.join(dest, filename) - mkpath(os.path.dirname(outname)) - res = copy_file(os.path.join(src, filename), outname, update=1) - if res[1]: copied.append(outname) - run_2to3([fn for fn in copied if fn.lower().endswith('.py')], - fixer_names=fixer_names, options=options, explicit=explicit) - return copied - -class Mixin2to3: - '''Mixin class for commands that run 2to3. - To configure 2to3, setup scripts may either change - the class variables, or inherit from individual commands - to override how 2to3 is invoked.''' - - # provide list of fixers to run; - # defaults to all from lib2to3.fixers - fixer_names = None - - # options dictionary - options = None - - # list of fixers to invoke even though they are marked as explicit - explicit = None - - def run_2to3(self, files): - return run_2to3(files, self.fixer_names, self.options, self.explicit) diff --git a/Lib/distutils/version.py b/Lib/distutils/version.py deleted file mode 100644 index af14cc13481..00000000000 --- a/Lib/distutils/version.py +++ /dev/null @@ -1,343 +0,0 @@ -# -# distutils/version.py -# -# Implements multiple version numbering conventions for the -# Python Module Distribution Utilities. -# -# $Id$ -# - -"""Provides classes to represent module version numbers (one class for -each style of version numbering). There are currently two such classes -implemented: StrictVersion and LooseVersion. - -Every version number class implements the following interface: - * the 'parse' method takes a string and parses it to some internal - representation; if the string is an invalid version number, - 'parse' raises a ValueError exception - * the class constructor takes an optional string argument which, - if supplied, is passed to 'parse' - * __str__ reconstructs the string that was passed to 'parse' (or - an equivalent string -- ie. one that will generate an equivalent - version number instance) - * __repr__ generates Python code to recreate the version number instance - * _cmp compares the current instance with either another instance - of the same class or a string (which will be parsed to an instance - of the same class, thus must follow the same rules) -""" - -import re - -class Version: - """Abstract base class for version numbering classes. Just provides - constructor (__init__) and reproducer (__repr__), because those - seem to be the same for all version numbering classes; and route - rich comparisons to _cmp. - """ - - def __init__ (self, vstring=None): - if vstring: - self.parse(vstring) - - def __repr__ (self): - return "%s ('%s')" % (self.__class__.__name__, str(self)) - - def __eq__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c == 0 - - def __lt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c < 0 - - def __le__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c <= 0 - - def __gt__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c > 0 - - def __ge__(self, other): - c = self._cmp(other) - if c is NotImplemented: - return c - return c >= 0 - - -# Interface for version-number classes -- must be implemented -# by the following classes (the concrete ones -- Version should -# be treated as an abstract class). -# __init__ (string) - create and take same action as 'parse' -# (string parameter is optional) -# parse (string) - convert a string representation to whatever -# internal representation is appropriate for -# this style of version numbering -# __str__ (self) - convert back to a string; should be very similar -# (if not identical to) the string supplied to parse -# __repr__ (self) - generate Python code to recreate -# the instance -# _cmp (self, other) - compare two version numbers ('other' may -# be an unparsed version string, or another -# instance of your version class) - - -class StrictVersion (Version): - - """Version numbering for anal retentives and software idealists. - Implements the standard interface for version number classes as - described above. A version number consists of two or three - dot-separated numeric components, with an optional "pre-release" tag - on the end. The pre-release tag consists of the letter 'a' or 'b' - followed by a number. If the numeric components of two version - numbers are equal, then one with a pre-release tag will always - be deemed earlier (lesser) than one without. - - The following are valid version numbers (shown in the order that - would be obtained by sorting according to the supplied cmp function): - - 0.4 0.4.0 (these two are equivalent) - 0.4.1 - 0.5a1 - 0.5b3 - 0.5 - 0.9.6 - 1.0 - 1.0.4a3 - 1.0.4b1 - 1.0.4 - - The following are examples of invalid version numbers: - - 1 - 2.7.2.2 - 1.3.a4 - 1.3pl1 - 1.3c4 - - The rationale for this version numbering system will be explained - in the distutils documentation. - """ - - version_re = re.compile(r'^(\d+) \. (\d+) (\. (\d+))? ([ab](\d+))?$', - re.VERBOSE | re.ASCII) - - - def parse (self, vstring): - match = self.version_re.match(vstring) - if not match: - raise ValueError("invalid version number '%s'" % vstring) - - (major, minor, patch, prerelease, prerelease_num) = \ - match.group(1, 2, 4, 5, 6) - - if patch: - self.version = tuple(map(int, [major, minor, patch])) - else: - self.version = tuple(map(int, [major, minor])) + (0,) - - if prerelease: - self.prerelease = (prerelease[0], int(prerelease_num)) - else: - self.prerelease = None - - - def __str__ (self): - - if self.version[2] == 0: - vstring = '.'.join(map(str, self.version[0:2])) - else: - vstring = '.'.join(map(str, self.version)) - - if self.prerelease: - vstring = vstring + self.prerelease[0] + str(self.prerelease[1]) - - return vstring - - - def _cmp (self, other): - if isinstance(other, str): - other = StrictVersion(other) - - if self.version != other.version: - # numeric versions don't match - # prerelease stuff doesn't matter - if self.version < other.version: - return -1 - else: - return 1 - - # have to compare prerelease - # case 1: neither has prerelease; they're equal - # case 2: self has prerelease, other doesn't; other is greater - # case 3: self doesn't have prerelease, other does: self is greater - # case 4: both have prerelease: must compare them! - - if (not self.prerelease and not other.prerelease): - return 0 - elif (self.prerelease and not other.prerelease): - return -1 - elif (not self.prerelease and other.prerelease): - return 1 - elif (self.prerelease and other.prerelease): - if self.prerelease == other.prerelease: - return 0 - elif self.prerelease < other.prerelease: - return -1 - else: - return 1 - else: - assert False, "never get here" - -# end class StrictVersion - - -# The rules according to Greg Stein: -# 1) a version number has 1 or more numbers separated by a period or by -# sequences of letters. If only periods, then these are compared -# left-to-right to determine an ordering. -# 2) sequences of letters are part of the tuple for comparison and are -# compared lexicographically -# 3) recognize the numeric components may have leading zeroes -# -# The LooseVersion class below implements these rules: a version number -# string is split up into a tuple of integer and string components, and -# comparison is a simple tuple comparison. This means that version -# numbers behave in a predictable and obvious way, but a way that might -# not necessarily be how people *want* version numbers to behave. There -# wouldn't be a problem if people could stick to purely numeric version -# numbers: just split on period and compare the numbers as tuples. -# However, people insist on putting letters into their version numbers; -# the most common purpose seems to be: -# - indicating a "pre-release" version -# ('alpha', 'beta', 'a', 'b', 'pre', 'p') -# - indicating a post-release patch ('p', 'pl', 'patch') -# but of course this can't cover all version number schemes, and there's -# no way to know what a programmer means without asking him. -# -# The problem is what to do with letters (and other non-numeric -# characters) in a version number. The current implementation does the -# obvious and predictable thing: keep them as strings and compare -# lexically within a tuple comparison. This has the desired effect if -# an appended letter sequence implies something "post-release": -# eg. "0.99" < "0.99pl14" < "1.0", and "5.001" < "5.001m" < "5.002". -# -# However, if letters in a version number imply a pre-release version, -# the "obvious" thing isn't correct. Eg. you would expect that -# "1.5.1" < "1.5.2a2" < "1.5.2", but under the tuple/lexical comparison -# implemented here, this just isn't so. -# -# Two possible solutions come to mind. The first is to tie the -# comparison algorithm to a particular set of semantic rules, as has -# been done in the StrictVersion class above. This works great as long -# as everyone can go along with bondage and discipline. Hopefully a -# (large) subset of Python module programmers will agree that the -# particular flavour of bondage and discipline provided by StrictVersion -# provides enough benefit to be worth using, and will submit their -# version numbering scheme to its domination. The free-thinking -# anarchists in the lot will never give in, though, and something needs -# to be done to accommodate them. -# -# Perhaps a "moderately strict" version class could be implemented that -# lets almost anything slide (syntactically), and makes some heuristic -# assumptions about non-digits in version number strings. This could -# sink into special-case-hell, though; if I was as talented and -# idiosyncratic as Larry Wall, I'd go ahead and implement a class that -# somehow knows that "1.2.1" < "1.2.2a2" < "1.2.2" < "1.2.2pl3", and is -# just as happy dealing with things like "2g6" and "1.13++". I don't -# think I'm smart enough to do it right though. -# -# In any case, I've coded the test suite for this module (see -# ../test/test_version.py) specifically to fail on things like comparing -# "1.2a2" and "1.2". That's not because the *code* is doing anything -# wrong, it's because the simple, obvious design doesn't match my -# complicated, hairy expectations for real-world version numbers. It -# would be a snap to fix the test suite to say, "Yep, LooseVersion does -# the Right Thing" (ie. the code matches the conception). But I'd rather -# have a conception that matches common notions about version numbers. - -class LooseVersion (Version): - - """Version numbering for anarchists and software realists. - Implements the standard interface for version number classes as - described above. A version number consists of a series of numbers, - separated by either periods or strings of letters. When comparing - version numbers, the numeric components will be compared - numerically, and the alphabetic components lexically. The following - are all valid version numbers, in no particular order: - - 1.5.1 - 1.5.2b2 - 161 - 3.10a - 8.02 - 3.4j - 1996.07.12 - 3.2.pl0 - 3.1.1.6 - 2g6 - 11g - 0.960923 - 2.2beta29 - 1.13++ - 5.5.kw - 2.0b1pl0 - - In fact, there is no such thing as an invalid version number under - this scheme; the rules for comparison are simple and predictable, - but may not always give the results you want (for some definition - of "want"). - """ - - component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) - - def __init__ (self, vstring=None): - if vstring: - self.parse(vstring) - - - def parse (self, vstring): - # I've given up on thinking I can reconstruct the version string - # from the parsed tuple -- so I just store the string here for - # use by __str__ - self.vstring = vstring - components = [x for x in self.component_re.split(vstring) - if x and x != '.'] - for i, obj in enumerate(components): - try: - components[i] = int(obj) - except ValueError: - pass - - self.version = components - - - def __str__ (self): - return self.vstring - - - def __repr__ (self): - return "LooseVersion ('%s')" % str(self) - - - def _cmp (self, other): - if isinstance(other, str): - other = LooseVersion(other) - - if self.version == other.version: - return 0 - if self.version < other.version: - return -1 - if self.version > other.version: - return 1 - - -# end class LooseVersion diff --git a/Lib/distutils/versionpredicate.py b/Lib/distutils/versionpredicate.py deleted file mode 100644 index 062c98f2489..00000000000 --- a/Lib/distutils/versionpredicate.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Module for parsing and testing package version predicate strings. -""" -import re -import distutils.version -import operator - - -re_validPackage = re.compile(r"(?i)^\s*([a-z_]\w*(?:\.[a-z_]\w*)*)(.*)", - re.ASCII) -# (package) (rest) - -re_paren = re.compile(r"^\s*\((.*)\)\s*$") # (list) inside of parentheses -re_splitComparison = re.compile(r"^\s*(<=|>=|<|>|!=|==)\s*([^\s,]+)\s*$") -# (comp) (version) - - -def splitUp(pred): - """Parse a single version comparison. - - Return (comparison string, StrictVersion) - """ - res = re_splitComparison.match(pred) - if not res: - raise ValueError("bad package restriction syntax: %r" % pred) - comp, verStr = res.groups() - return (comp, distutils.version.StrictVersion(verStr)) - -compmap = {"<": operator.lt, "<=": operator.le, "==": operator.eq, - ">": operator.gt, ">=": operator.ge, "!=": operator.ne} - -class VersionPredicate: - """Parse and test package version predicates. - - >>> v = VersionPredicate('pyepat.abc (>1.0, <3333.3a1, !=1555.1b3)') - - The `name` attribute provides the full dotted name that is given:: - - >>> v.name - 'pyepat.abc' - - The str() of a `VersionPredicate` provides a normalized - human-readable version of the expression:: - - >>> print(v) - pyepat.abc (> 1.0, < 3333.3a1, != 1555.1b3) - - The `satisfied_by()` method can be used to determine with a given - version number is included in the set described by the version - restrictions:: - - >>> v.satisfied_by('1.1') - True - >>> v.satisfied_by('1.4') - True - >>> v.satisfied_by('1.0') - False - >>> v.satisfied_by('4444.4') - False - >>> v.satisfied_by('1555.1b3') - False - - `VersionPredicate` is flexible in accepting extra whitespace:: - - >>> v = VersionPredicate(' pat( == 0.1 ) ') - >>> v.name - 'pat' - >>> v.satisfied_by('0.1') - True - >>> v.satisfied_by('0.2') - False - - If any version numbers passed in do not conform to the - restrictions of `StrictVersion`, a `ValueError` is raised:: - - >>> v = VersionPredicate('p1.p2.p3.p4(>=1.0, <=1.3a1, !=1.2zb3)') - Traceback (most recent call last): - ... - ValueError: invalid version number '1.2zb3' - - It the module or package name given does not conform to what's - allowed as a legal module or package name, `ValueError` is - raised:: - - >>> v = VersionPredicate('foo-bar') - Traceback (most recent call last): - ... - ValueError: expected parenthesized list: '-bar' - - >>> v = VersionPredicate('foo bar (12.21)') - Traceback (most recent call last): - ... - ValueError: expected parenthesized list: 'bar (12.21)' - - """ - - def __init__(self, versionPredicateStr): - """Parse a version predicate string. - """ - # Fields: - # name: package name - # pred: list of (comparison string, StrictVersion) - - versionPredicateStr = versionPredicateStr.strip() - if not versionPredicateStr: - raise ValueError("empty package restriction") - match = re_validPackage.match(versionPredicateStr) - if not match: - raise ValueError("bad package name in %r" % versionPredicateStr) - self.name, paren = match.groups() - paren = paren.strip() - if paren: - match = re_paren.match(paren) - if not match: - raise ValueError("expected parenthesized list: %r" % paren) - str = match.groups()[0] - self.pred = [splitUp(aPred) for aPred in str.split(",")] - if not self.pred: - raise ValueError("empty parenthesized list in %r" - % versionPredicateStr) - else: - self.pred = [] - - def __str__(self): - if self.pred: - seq = [cond + " " + str(ver) for cond, ver in self.pred] - return self.name + " (" + ", ".join(seq) + ")" - else: - return self.name - - def satisfied_by(self, version): - """True if version is compatible with all the predicates in self. - The parameter version must be acceptable to the StrictVersion - constructor. It may be either a string or StrictVersion. - """ - for cond, ver in self.pred: - if not compmap[cond](version, ver): - return False - return True - - -_provision_rx = None - -def split_provision(value): - """Return the name and optional version number of a provision. - - The version number, if given, will be returned as a `StrictVersion` - instance, otherwise it will be `None`. - - >>> split_provision('mypkg') - ('mypkg', None) - >>> split_provision(' mypkg( 1.2 ) ') - ('mypkg', StrictVersion ('1.2')) - """ - global _provision_rx - if _provision_rx is None: - _provision_rx = re.compile( - r"([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)(?:\s*\(\s*([^)\s]+)\s*\))?$", - re.ASCII) - value = value.strip() - m = _provision_rx.match(value) - if not m: - raise ValueError("illegal provides specification: %r" % value) - ver = m.group(2) or None - if ver: - ver = distutils.version.StrictVersion(ver) - return m.group(1), ver diff --git a/Lib/doctest.py b/Lib/doctest.py index 65466b49834..a66888d8fc9 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -94,6 +94,7 @@ def _test(): import __future__ import difflib +import functools import inspect import linecache import os @@ -102,10 +103,30 @@ def _test(): import sys import traceback import unittest -from io import StringIO # XXX: RUSTPYTHON; , IncrementalNewlineDecoder +from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple +import _colorize # Used in doctests +from _colorize import ANSIColors, can_colorize + + +__unittest = True + +class TestResults(namedtuple('TestResults', 'failed attempted')): + def __new__(cls, failed, attempted, *, skipped=0): + results = super().__new__(cls, failed, attempted) + results.skipped = skipped + return results + + def __repr__(self): + if self.skipped: + return (f'TestResults(failed={self.failed}, ' + f'attempted={self.attempted}, ' + f'skipped={self.skipped})') + else: + # Leave the repr() unchanged for backward compatibility + # if skipped is zero + return super().__repr__() -TestResults = namedtuple('TestResults', 'failed attempted') # There are 4 basic classes: # - Example: a pair, plus an intra-docstring line number. @@ -207,7 +228,13 @@ def _normalize_module(module, depth=2): elif isinstance(module, str): return __import__(module, globals(), locals(), ["*"]) elif module is None: - return sys.modules[sys._getframe(depth).f_globals['__name__']] + try: + try: + return sys.modules[sys._getframemodulename(depth)] + except AttributeError: + return sys.modules[sys._getframe(depth).f_globals['__name__']] + except KeyError: + pass else: raise TypeError("Expected a module, string, or None") @@ -229,10 +256,7 @@ def _load_testfile(filename, package, module_relative, encoding): file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - - # TODO: RUSTPYTHON; use _newline_convert once io.IncrementalNewlineDecoder is implemented - return file_contents.replace(os.linesep, '\n'), filename - # return _newline_convert(file_contents), filename + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename @@ -368,11 +392,11 @@ def __init__(self, out): # still use input() to get user input self.use_rawinput = 1 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): self.__debugger_used = True if frame is None: frame = sys._getframe().f_back - pdb.Pdb.set_trace(self, frame) + pdb.Pdb.set_trace(self, frame, commands=commands) def set_continue(self): # Calling set_continue unconditionally would break unit test @@ -572,9 +596,11 @@ def __hash__(self): def __lt__(self, other): if not isinstance(other, DocTest): return NotImplemented - return ((self.name, self.filename, self.lineno, id(self)) + self_lno = self.lineno if self.lineno is not None else -1 + other_lno = other.lineno if other.lineno is not None else -1 + return ((self.name, self.filename, self_lno, id(self)) < - (other.name, other.filename, other.lineno, id(other))) + (other.name, other.filename, other_lno, id(other))) ###################################################################### ## 3. DocTestParser @@ -959,7 +985,8 @@ def _from_module(self, module, object): return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ - elif inspect.ismethoddescriptor(object): + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): if hasattr(object, '__objclass__'): obj_mod = object.__objclass__.__module__ elif hasattr(object, '__module__'): @@ -1106,7 +1133,7 @@ def _find_lineno(self, obj, source_lines): if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) + re.escape(getattr(obj, '__name__', '-'))) for i, line in enumerate(source_lines): if pat.match(line): lineno = i @@ -1114,13 +1141,24 @@ def _find_lineno(self, obj, source_lines): # Find the line number for functions & methods. if inspect.ismethod(obj): obj = obj.__func__ - if inspect.isfunction(obj) and getattr(obj, '__doc__', None): + if isinstance(obj, property): + obj = obj.fget + if isinstance(obj, functools.cached_property): + obj = obj.func + if inspect.isroutine(obj) and getattr(obj, '__doc__', None): # We don't use `docstring` var here, because `obj` can be changed. - obj = obj.__code__ + obj = inspect.unwrap(obj) + try: + obj = obj.__code__ + except AttributeError: + # Functions implemented in C don't necessarily + # have a __code__ attribute. + # If there's no code, there's no lineno + return None if inspect.istraceback(obj): obj = obj.tb_frame if inspect.isframe(obj): obj = obj.f_code if inspect.iscode(obj): - lineno = getattr(obj, 'co_firstlineno', None)-1 + lineno = obj.co_firstlineno - 1 # Find the line number where the docstring starts. Assume # that it's the first line that begins with a quote mark. @@ -1146,8 +1184,10 @@ class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. The `run` method is used to process a single DocTest case. It - returns a tuple `(f, t)`, where `t` is the number of test cases - tried, and `f` is the number of test cases that failed. + returns a TestResults instance. + + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) @@ -1160,27 +1200,29 @@ class DocTestRunner: _TestClass.square -> TestResults(failed=0, attempted=1) The `summarize` method prints a summary of all the test cases that - have been run by the runner, and returns an aggregated `(f, t)` - tuple: + have been run by the runner, and returns an aggregated TestResults + instance: >>> runner.summarize(verbose=1) 4 items passed all tests: 2 tests in _TestClass 2 tests in _TestClass.__init__ 2 tests in _TestClass.get - 1 tests in _TestClass.square + 1 test in _TestClass.square 7 tests in 4 items. - 7 passed and 0 failed. + 7 passed. Test passed. TestResults(failed=0, attempted=7) - The aggregated number of tried examples and failed examples is - also available via the `tries` and `failures` attributes: + The aggregated number of tried examples and failed examples is also + available via the `tries`, `failures` and `skips` attributes: >>> runner.tries 7 >>> runner.failures 0 + >>> runner.skips + 0 The comparison between expected outputs and actual outputs is done by an `OutputChecker`. This comparison may be customized with a @@ -1190,13 +1232,15 @@ class DocTestRunner: `OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. - First, an output function (`out) can be passed to + First, an output function (`out`) can be passed to `TestRunner.run`; this function will be called with strings that should be displayed. It defaults to `sys.stdout.write`. If capturing the output is not sufficient, then the display output can be also customized by subclassing DocTestRunner, and overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. + + >>> _colorize.COLORIZE = save_colorize """ # This divider string is used to separate failure messages, and to # separate sections of the summary. @@ -1229,7 +1273,8 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Keep track of the examples we've run. self.tries = 0 self.failures = 0 - self._name2ft = {} + self.skips = 0 + self._stats = {} # Create a fake output target for capturing doctest output. self._fakeout = _SpoofOut() @@ -1274,7 +1319,10 @@ def report_unexpected_exception(self, out, test, example, exc_info): 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) def _failure_header(self, test, example): - out = [self.DIVIDER] + red, reset = ( + (ANSIColors.RED, ANSIColors.RESET) if can_colorize() else ("", "") + ) + out = [f"{red}{self.DIVIDER}{reset}"] if test.filename: if test.lineno is not None and example.lineno is not None: lineno = test.lineno + example.lineno + 1 @@ -1298,13 +1346,11 @@ def __run(self, test, compileflags, out): Run the examples in `test`. Write the outcome of each example with one of the `DocTestRunner.report_*` methods, using the writer function `out`. `compileflags` is the set of compiler - flags that should be used to execute examples. Return a tuple - `(f, t)`, where `t` is the number of examples tried, and `f` - is the number of examples that failed. The examples are run - in the namespace `test.globs`. + flags that should be used to execute examples. Return a TestResults + instance. The examples are run in the namespace `test.globs`. """ - # Keep track of the number of failures and tries. - failures = tries = 0 + # Keep track of the number of failed, attempted, skipped examples. + failures = attempted = skips = 0 # Save the option flags (since option directives can be used # to modify them). @@ -1316,6 +1362,7 @@ def __run(self, test, compileflags, out): # Process each example. for examplenum, example in enumerate(test.examples): + attempted += 1 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress # reporting after the first failure. @@ -1333,10 +1380,10 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + skips += 1 continue # Record that we started this example. - tries += 1 if not quiet: self.report_start(out, test, example) @@ -1353,11 +1400,11 @@ def __run(self, test, compileflags, out): exec(compile(example.source, filename, "single", compileflags, True), test.globs) self.debugger.set_continue() # ==== Example Finished ==== - exception = None + exc_info = None except KeyboardInterrupt: raise - except: - exception = sys.exc_info() + except BaseException as exc: + exc_info = type(exc), exc, exc.__traceback__.tb_next self.debugger.set_continue() # ==== Example Finished ==== got = self._fakeout.getvalue() # the actual output @@ -1366,15 +1413,32 @@ def __run(self, test, compileflags, out): # If the example executed without raising any exceptions, # verify its output. - if exception is None: + if exc_info is None: if check(example.want, got, self.optionflags): outcome = SUCCESS # The example raised an exception: check if it was expected. else: - exc_msg = traceback.format_exception_only(*exception[:2])[-1] + formatted_ex = traceback.format_exception_only(*exc_info[:2]) + if issubclass(exc_info[0], SyntaxError): + # SyntaxError / IndentationError is special: + # we don't care about the carets / suggestions / etc + # We only care about the error message and notes. + # They start with `SyntaxError:` (or any other class name) + exception_line_prefixes = ( + f"{exc_info[0].__qualname__}:", + f"{exc_info[0].__module__}.{exc_info[0].__qualname__}:", + ) + exc_msg_index = next( + index + for index, line in enumerate(formatted_ex) + if line.startswith(exception_line_prefixes) + ) + formatted_ex = formatted_ex[exc_msg_index:] + + exc_msg = "".join(formatted_ex) if not quiet: - got += _exception_traceback(exception) + got += _exception_traceback(exc_info) # If `example.exc_msg` is None, then we weren't expecting # an exception. @@ -1403,7 +1467,7 @@ def __run(self, test, compileflags, out): elif outcome is BOOM: if not quiet: self.report_unexpected_exception(out, test, example, - exception) + exc_info) failures += 1 else: assert False, ("unknown outcome", outcome) @@ -1414,19 +1478,22 @@ def __run(self, test, compileflags, out): # Restore the option flags (in case they were modified) self.optionflags = original_optionflags - # Record and return the number of failures and tries. - self.__record_outcome(test, failures, tries) - return TestResults(failures, tries) + # Record and return the number of failures and attempted. + self.__record_outcome(test, failures, attempted, skips) + return TestResults(failures, attempted, skipped=skips) - def __record_outcome(self, test, f, t): + def __record_outcome(self, test, failures, tries, skips): """ - Record the fact that the given DocTest (`test`) generated `f` - failures out of `t` tried examples. + Record the fact that the given DocTest (`test`) generated `failures` + failures out of `tries` tried examples. """ - f2, t2 = self._name2ft.get(test.name, (0,0)) - self._name2ft[test.name] = (f+f2, t+t2) - self.failures += f - self.tries += t + failures2, tries2, skips2 = self._stats.get(test.name, (0, 0, 0)) + self._stats[test.name] = (failures + failures2, + tries + tries2, + skips + skips2) + self.failures += failures + self.tries += tries + self.skips += skips __LINECACHE_FILENAME_RE = re.compile(r'.+)' @@ -1495,7 +1562,11 @@ def out(s): # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ - + saved_can_colorize = _colorize.can_colorize + _colorize.can_colorize = lambda *args, **kwargs: False + color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} + for key in color_variables: + color_variables[key] = os.environ.pop(key, None) try: return self.__run(test, compileflags, out) finally: @@ -1504,6 +1575,10 @@ def out(s): sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook + _colorize.can_colorize = saved_can_colorize + for key, value in color_variables.items(): + if value is not None: + os.environ[key] = value if clear_globs: test.globs.clear() import builtins @@ -1515,9 +1590,7 @@ def out(s): def summarize(self, verbose=None): """ Print a summary of all the test cases that have been run by - this DocTestRunner, and return a tuple `(f, t)`, where `f` is - the total number of failed examples, and `t` is the total - number of tried examples. + this DocTestRunner, and return a TestResults instance. The optional `verbose` argument controls how detailed the summary is. If the verbosity is not specified, then the @@ -1525,66 +1598,98 @@ def summarize(self, verbose=None): """ if verbose is None: verbose = self._verbose - notests = [] - passed = [] - failed = [] - totalt = totalf = 0 - for x in self._name2ft.items(): - name, (f, t) = x - assert f <= t - totalt += t - totalf += f - if t == 0: + + notests, passed, failed = [], [], [] + total_tries = total_failures = total_skips = 0 + + for name, (failures, tries, skips) in self._stats.items(): + assert failures <= tries + total_tries += tries + total_failures += failures + total_skips += skips + + if tries == 0: notests.append(name) - elif f == 0: - passed.append( (name, t) ) + elif failures == 0: + passed.append((name, tries)) else: - failed.append(x) + failed.append((name, (failures, tries, skips))) + + ansi = _colorize.get_colors() + bold_green = ansi.BOLD_GREEN + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if verbose: if notests: - print(len(notests), "items had no tests:") + print(f"{_n_items(notests)} had no tests:") notests.sort() - for thing in notests: - print(" ", thing) + for name in notests: + print(f" {name}") + if passed: - print(len(passed), "items passed all tests:") - passed.sort() - for thing, count in passed: - print(" %3d tests in %s" % (count, thing)) + print(f"{green}{_n_items(passed)} passed all tests:{reset}") + for name, count in sorted(passed): + s = "" if count == 1 else "s" + print(f" {green}{count:3d} test{s} in {name}{reset}") + if failed: - print(self.DIVIDER) - print(len(failed), "items had failures:") - failed.sort() - for thing, (f, t) in failed: - print(" %3d of %3d in %s" % (f, t, thing)) + print(f"{red}{self.DIVIDER}{reset}") + print(f"{_n_items(failed)} had failures:") + for name, (failures, tries, skips) in sorted(failed): + print(f" {failures:3d} of {tries:3d} in {name}") + if verbose: - print(totalt, "tests in", len(self._name2ft), "items.") - print(totalt - totalf, "passed and", totalf, "failed.") - if totalf: - print("***Test Failed***", totalf, "failures.") + s = "" if total_tries == 1 else "s" + print(f"{total_tries} test{s} in {_n_items(self._stats)}.") + + and_f = ( + f" and {red}{total_failures} failed{reset}" + if total_failures else "" + ) + print(f"{green}{total_tries - total_failures} passed{reset}{and_f}.") + + if total_failures: + s = "" if total_failures == 1 else "s" + msg = f"{bold_red}***Test Failed*** {total_failures} failure{s}{reset}" + if total_skips: + s = "" if total_skips == 1 else "s" + msg = f"{msg} and {yellow}{total_skips} skipped test{s}{reset}" + print(f"{msg}.") elif verbose: - print("Test passed.") - return TestResults(totalf, totalt) + print(f"{bold_green}Test passed.{reset}") + + return TestResults(total_failures, total_tries, skipped=total_skips) #///////////////////////////////////////////////////////////////// # Backward compatibility cruft to maintain doctest.master. #///////////////////////////////////////////////////////////////// def merge(self, other): - d = self._name2ft - for name, (f, t) in other._name2ft.items(): + d = self._stats + for name, (failures, tries, skips) in other._stats.items(): if name in d: - # Don't print here by default, since doing - # so breaks some of the buildbots - #print("*** DocTestRunner.merge: '" + name + "' in both" \ - # " testers; summing outcomes.") - f2, t2 = d[name] - f = f + f2 - t = t + t2 - d[name] = f, t + failures2, tries2, skips2 = d[name] + failures = failures + failures2 + tries = tries + tries2 + skips = skips + skips2 + d[name] = (failures, tries, skips) + + +def _n_items(items: list | dict) -> str: + """ + Helper to pluralise the number of items in a list. + """ + n = len(items) + s = "" if n == 1 else "s" + return f"{n} item{s}" + class OutputChecker: """ - A class used to check the whether the actual output from a doctest + A class used to check whether the actual output from a doctest example matches the expected output. `OutputChecker` defines two methods: `check_output`, which compares a given pair of outputs, and returns true if they match; and `output_difference`, which @@ -1889,8 +1994,8 @@ def testmod(m=None, name=None, globs=None, verbose=None, from module m (or the current module if m is not supplied), starting with m.__doc__. - Also test examples reachable from dict m.__test__ if it exists and is - not None. m.__test__ maps names to functions, classes and strings; + Also test examples reachable from dict m.__test__ if it exists. + m.__test__ maps names to functions, classes and strings; function and class docstrings are tested even if the name is private; strings are tested directly, as if they were docstrings. @@ -1980,7 +2085,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, @@ -2103,7 +2209,8 @@ class doctest.Tester, then merges the results into (or creates) else: master.merge(runner) - return TestResults(runner.failures, runner.tries) + return TestResults(runner.failures, runner.tries, skipped=runner.skips) + def run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0): @@ -2178,13 +2285,13 @@ def __init__(self, test, optionflags=0, setUp=None, tearDown=None, unittest.TestCase.__init__(self) self._dt_optionflags = optionflags self._dt_checker = checker - self._dt_globs = test.globs.copy() self._dt_test = test self._dt_setUp = setUp self._dt_tearDown = tearDown def setUp(self): test = self._dt_test + self._dt_globs = test.globs.copy() if self._dt_setUp is not None: self._dt_setUp(test) @@ -2215,13 +2322,14 @@ def runTest(self): try: runner.DIVIDER = "-"*70 - failures, tries = runner.run( - test, out=new.write, clear_globs=False) + results = runner.run(test, out=new.write, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") finally: sys.stdout = old - if failures: - raise self.failureException(self.format_failure(new.getvalue())) + if results.failed: + raise self.failureException(self.format_failure(new.getvalue().rstrip('\n'))) def format_failure(self, err): test = self._dt_test @@ -2631,7 +2739,7 @@ def testsource(module, name): return testsrc def debug_src(src, pm=False, globs=None): - """Debug a single doctest docstring, in argument `src`'""" + """Debug a single doctest docstring, in argument `src`""" testsrc = script_from_examples(src) debug_script(testsrc, pm, globs) @@ -2767,7 +2875,7 @@ def get(self): def _test(): import argparse - parser = argparse.ArgumentParser(description="doctest runner") + parser = argparse.ArgumentParser(description="doctest runner", color=True) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print very verbose output for all tests') parser.add_argument('-o', '--option', action='append', diff --git a/Lib/dummy_threading.py b/Lib/dummy_threading.py index 1bb7eee338a..662f3b89a9a 100644 --- a/Lib/dummy_threading.py +++ b/Lib/dummy_threading.py @@ -6,6 +6,7 @@ regardless of whether ``_thread`` was available which is not desired. """ + from sys import modules as sys_modules import _dummy_thread @@ -19,35 +20,38 @@ # Could have checked if ``_thread`` was not in sys.modules and gone # a different route, but decided to mirror technique used with # ``threading`` below. - if '_thread' in sys_modules: - held_thread = sys_modules['_thread'] + if "_thread" in sys_modules: + held_thread = sys_modules["_thread"] holding_thread = True # Must have some module named ``_thread`` that implements its API # in order to initially import ``threading``. - sys_modules['_thread'] = sys_modules['_dummy_thread'] + sys_modules["_thread"] = sys_modules["_dummy_thread"] - if 'threading' in sys_modules: + if "threading" in sys_modules: # If ``threading`` is already imported, might as well prevent # trying to import it more than needed by saving it if it is # already imported before deleting it. - held_threading = sys_modules['threading'] + held_threading = sys_modules["threading"] holding_threading = True - del sys_modules['threading'] + del sys_modules["threading"] - if '_threading_local' in sys_modules: + if "_threading_local" in sys_modules: # If ``_threading_local`` is already imported, might as well prevent # trying to import it more than needed by saving it if it is # already imported before deleting it. - held__threading_local = sys_modules['_threading_local'] + held__threading_local = sys_modules["_threading_local"] holding__threading_local = True - del sys_modules['_threading_local'] + del sys_modules["_threading_local"] import threading + # Need a copy of the code kept somewhere... - sys_modules['_dummy_threading'] = sys_modules['threading'] - del sys_modules['threading'] - sys_modules['_dummy__threading_local'] = sys_modules['_threading_local'] - del sys_modules['_threading_local'] + sys_modules["_dummy_threading"] = sys_modules["threading"] + del sys_modules["threading"] + # _threading_local may not be imported if _thread._local is available + if "_threading_local" in sys_modules: + sys_modules["_dummy__threading_local"] = sys_modules["_threading_local"] + del sys_modules["_threading_local"] from _dummy_threading import * from _dummy_threading import __all__ @@ -55,23 +59,23 @@ # Put back ``threading`` if we overwrote earlier if holding_threading: - sys_modules['threading'] = held_threading + sys_modules["threading"] = held_threading del held_threading del holding_threading # Put back ``_threading_local`` if we overwrote earlier if holding__threading_local: - sys_modules['_threading_local'] = held__threading_local + sys_modules["_threading_local"] = held__threading_local del held__threading_local del holding__threading_local # Put back ``thread`` if we overwrote, else del the entry we made if holding_thread: - sys_modules['_thread'] = held_thread + sys_modules["_thread"] = held_thread del held_thread else: - del sys_modules['_thread'] + del sys_modules["_thread"] del holding_thread del _dummy_thread diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py index fae872439ed..9fa47783004 100644 --- a/Lib/email/__init__.py +++ b/Lib/email/__init__.py @@ -25,7 +25,6 @@ ] - # Some convenience routines. Don't import Parser and Message as side-effects # of importing email since those cascadingly import most of the rest of the # email package. diff --git a/Lib/email/_encoded_words.py b/Lib/email/_encoded_words.py index 5eaab36ed0a..6795a606de0 100644 --- a/Lib/email/_encoded_words.py +++ b/Lib/email/_encoded_words.py @@ -62,7 +62,7 @@ # regex based decoder. _q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub, - lambda m: bytes([int(m.group(1), 16)])) + lambda m: bytes.fromhex(m.group(1).decode())) def decode_q(encoded): encoded = encoded.replace(b'_', b' ') @@ -98,30 +98,42 @@ def len_q(bstring): # def decode_b(encoded): - defects = [] + # First try encoding with validate=True, fixing the padding if needed. + # This will succeed only if encoded includes no invalid characters. pad_err = len(encoded) % 4 - if pad_err: - defects.append(errors.InvalidBase64PaddingDefect()) - padded_encoded = encoded + b'==='[:4-pad_err] - else: - padded_encoded = encoded + missing_padding = b'==='[:4-pad_err] if pad_err else b'' try: - return base64.b64decode(padded_encoded, validate=True), defects + return ( + base64.b64decode(encoded + missing_padding, validate=True), + [errors.InvalidBase64PaddingDefect()] if pad_err else [], + ) except binascii.Error: - # Since we had correct padding, this must an invalid char error. - defects = [errors.InvalidBase64CharactersDefect()] + # Since we had correct padding, this is likely an invalid char error. + # # The non-alphabet characters are ignored as far as padding - # goes, but we don't know how many there are. So we'll just - # try various padding lengths until something works. - for i in 0, 1, 2, 3: + # goes, but we don't know how many there are. So try without adding + # padding to see if it works. + try: + return ( + base64.b64decode(encoded, validate=False), + [errors.InvalidBase64CharactersDefect()], + ) + except binascii.Error: + # Add as much padding as could possibly be necessary (extra padding + # is ignored). try: - return base64.b64decode(encoded+b'='*i, validate=False), defects + return ( + base64.b64decode(encoded + b'==', validate=False), + [errors.InvalidBase64CharactersDefect(), + errors.InvalidBase64PaddingDefect()], + ) except binascii.Error: - if i==0: - defects.append(errors.InvalidBase64PaddingDefect()) - else: - # This should never happen. - raise AssertionError("unexpected binascii.Error") + # This only happens when the encoded string's length is 1 more + # than a multiple of 4, which is invalid. + # + # bpo-27397: Just return the encoded string since there's no + # way to decode. + return encoded, [errors.InvalidBase64LengthDefect()] def encode_b(bstring): return base64.b64encode(bstring).decode('ascii') @@ -167,15 +179,15 @@ def decode(ew): # Turn the CTE decoded bytes into unicode. try: string = bstring.decode(charset) - except UnicodeError: + except UnicodeDecodeError: defects.append(errors.UndecodableBytesDefect("Encoded word " - "contains bytes not decodable using {} charset".format(charset))) + f"contains bytes not decodable using {charset!r} charset")) string = bstring.decode(charset, 'surrogateescape') - except LookupError: + except (LookupError, UnicodeEncodeError): string = bstring.decode('ascii', 'surrogateescape') if charset.lower() != 'unknown-8bit': - defects.append(errors.CharsetError("Unknown charset {} " - "in encoded word; decoded as unknown bytes".format(charset))) + defects.append(errors.CharsetError(f"Unknown charset {charset!r} " + f"in encoded word; decoded as unknown bytes")) return string, charset, lang, defects diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 57d01fbcb0f..91243378dc0 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -68,9 +68,9 @@ """ import re +import sys import urllib # For urllib.parse.unquote from string import hexdigits -from collections import OrderedDict from operator import itemgetter from email import _encoded_words as _ew from email import errors @@ -92,93 +92,31 @@ ASPECIALS = TSPECIALS | set("*'%") ATTRIBUTE_ENDS = ASPECIALS | WSP EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') +NLSET = {'\n', '\r'} +SPECIALSNL = SPECIALS | NLSET -def quote_string(value): - return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' -# -# Accumulator for header folding -# +def make_quoted_pairs(value): + """Escape dquote and backslash for use within a quoted-string.""" + return str(value).replace('\\', '\\\\').replace('"', '\\"') -class _Folded: - def __init__(self, maxlen, policy): - self.maxlen = maxlen - self.policy = policy - self.lastlen = 0 - self.stickyspace = None - self.firstline = True - self.done = [] - self.current = [] +def quote_string(value): + escaped = make_quoted_pairs(value) + return f'"{escaped}"' - def newline(self): - self.done.extend(self.current) - self.done.append(self.policy.linesep) - self.current.clear() - self.lastlen = 0 - def finalize(self): - if self.current: - self.newline() +# Match a RFC 2047 word, looks like =?utf-8?q?someword?= +rfc2047_matcher = re.compile(r''' + =\? # literal =? + [^?]* # charset + \? # literal ? + [qQbB] # literal 'q' or 'b', case insensitive + \? # literal ? + .*? # encoded word + \?= # literal ?= +''', re.VERBOSE | re.MULTILINE) - def __str__(self): - return ''.join(self.done) - - def append(self, stoken): - self.current.append(stoken) - - def append_if_fits(self, token, stoken=None): - if stoken is None: - stoken = str(token) - l = len(stoken) - if self.stickyspace is not None: - stickyspace_len = len(self.stickyspace) - if self.lastlen + stickyspace_len + l <= self.maxlen: - self.current.append(self.stickyspace) - self.lastlen += stickyspace_len - self.current.append(stoken) - self.lastlen += l - self.stickyspace = None - self.firstline = False - return True - if token.has_fws: - ws = token.pop_leading_fws() - if ws is not None: - self.stickyspace += str(ws) - stickyspace_len += len(ws) - token._fold(self) - return True - if stickyspace_len and l + 1 <= self.maxlen: - margin = self.maxlen - l - if 0 < margin < stickyspace_len: - trim = stickyspace_len - margin - self.current.append(self.stickyspace[:trim]) - self.stickyspace = self.stickyspace[trim:] - stickyspace_len = trim - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.lastlen = l + stickyspace_len - self.stickyspace = None - self.firstline = False - return True - if not self.firstline: - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.stickyspace = None - self.firstline = False - return True - if self.lastlen + l <= self.maxlen: - self.current.append(stoken) - self.lastlen += l - return True - if l < self.maxlen: - self.newline() - self.current.append(stoken) - self.lastlen = l - return True - return False # # TokenList and its subclasses @@ -187,6 +125,8 @@ def append_if_fits(self, token, stoken=None): class TokenList(list): token_type = None + syntactic_break = True + ew_combine_allowed = True def __init__(self, *args, **kw): super().__init__(*args, **kw) @@ -207,84 +147,13 @@ def value(self): def all_defects(self): return sum((x.all_defects for x in self), self.defects) - # - # Folding API - # - # parts(): - # - # return a list of objects that constitute the "higher level syntactic - # objects" specified by the RFC as the best places to fold a header line. - # The returned objects must include leading folding white space, even if - # this means mutating the underlying parse tree of the object. Each object - # is only responsible for returning *its* parts, and should not drill down - # to any lower level except as required to meet the leading folding white - # space constraint. - # - # _fold(folded): - # - # folded: the result accumulator. This is an instance of _Folded. - # (XXX: I haven't finished factoring this out yet, the folding code - # pretty much uses this as a state object.) When the folded.current - # contains as much text as will fit, the _fold method should call - # folded.newline. - # folded.lastlen: the current length of the test stored in folded.current. - # folded.maxlen: The maximum number of characters that may appear on a - # folded line. Differs from the policy setting in that "no limit" is - # represented by +inf, which means it can be used in the trivially - # logical fashion in comparisons. - # - # Currently no subclasses implement parts, and I think this will remain - # true. A subclass only needs to implement _fold when the generic version - # isn't sufficient. _fold will need to be implemented primarily when it is - # possible for encoded words to appear in the specialized token-list, since - # there is no generic algorithm that can know where exactly the encoded - # words are allowed. A _fold implementation is responsible for filling - # lines in the same general way that the top level _fold does. It may, and - # should, call the _fold method of sub-objects in a similar fashion to that - # of the top level _fold. - # - # XXX: I'm hoping it will be possible to factor the existing code further - # to reduce redundancy and make the logic clearer. - - @property - def parts(self): - klass = self.__class__ - this = [] - for token in self: - if token.startswith_fws(): - if this: - yield this[0] if len(this)==1 else klass(this) - this.clear() - end_ws = token.pop_trailing_ws() - this.append(token) - if end_ws: - yield klass(this) - this = [end_ws] - if this: - yield this[0] if len(this)==1 else klass(this) - def startswith_fws(self): return self[0].startswith_fws() - def pop_leading_fws(self): - if self[0].token_type == 'fws': - return self.pop(0) - return self[0].pop_leading_fws() - - def pop_trailing_ws(self): - if self[-1].token_type == 'cfws': - return self.pop(-1) - return self[-1].pop_trailing_ws() - @property - def has_fws(self): - for part in self: - if part.has_fws: - return True - return False - - def has_leading_comment(self): - return self[0].has_leading_comment() + def as_ew_allowed(self): + """True if all top level tokens of this part may be RFC2047 encoded.""" + return all(part.as_ew_allowed for part in self) @property def comments(self): @@ -294,71 +163,13 @@ def comments(self): return comments def fold(self, *, policy): - # max_line_length 0/None means no limit, ie: infinitely long. - maxlen = policy.max_line_length or float("+inf") - folded = _Folded(maxlen, policy) - self._fold(folded) - folded.finalize() - return str(folded) - - def as_encoded_word(self, charset): - # This works only for things returned by 'parts', which include - # the leading fws, if any, that should be used. - res = [] - ws = self.pop_leading_fws() - if ws: - res.append(ws) - trailer = self.pop(-1) if self[-1].token_type=='fws' else '' - res.append(_ew.encode(str(self), charset)) - res.append(trailer) - return ''.join(res) - - def cte_encode(self, charset, policy): - res = [] - for part in self: - res.append(part.cte_encode(charset, policy)) - return ''.join(res) - - def _fold(self, folded): - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - # XXX: this should be a policy setting when utf8 is False. - charset = 'utf-8' - tstr = part.cte_encode(charset, folded.policy) - tlen = len(tstr) - if folded.append_if_fits(part, tstr): - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - # Peel off the leading whitespace and make it sticky, to - # avoid infinite recursion. - folded.stickyspace = str(part.pop(0)) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # There are no fold points in this one; it is too long for a single - # line and can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() + return _refold_parse_tree(self, policy=policy) def pprint(self, indent=''): - print('\n'.join(self._pp(indent=''))) + print(self.ppstr(indent=indent)) def ppstr(self, indent=''): - return '\n'.join(self._pp(indent='')) + return '\n'.join(self._pp(indent=indent)) def _pp(self, indent=''): yield '{}{}/{}('.format( @@ -390,213 +201,35 @@ def comments(self): class UnstructuredTokenList(TokenList): - token_type = 'unstructured' - def _fold(self, folded): - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - is_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None: - # We've already done an EW, combine this one with it - # if there's room. - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - oldlastlen = sum(len(x) for x in folded.current[:last_ew]) - schunk = str(chunk) - lchunk = len(schunk) - if oldlastlen + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = oldlastlen + lchunk - continue - tstr = part.as_encoded_word(charset) - is_ew = True - if folded.append_if_fits(part, tstr): - if is_ew: - last_ew = len(folded.current) - 1 - continue - if is_ew or last_ew: - # It's too big to fit on the line, but since we've - # got encoded words we can use encoded word folding. - part._fold_as_ew(folded) - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # It can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() - last_ew = None - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - if last_ew is None: - res.append(part.cte_encode(charset, policy)) - last_ew = len(res) - else: - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res.append(tl.as_encoded_word(charset)) - return ''.join(res) - class Phrase(TokenList): - token_type = 'phrase' - def _fold(self, folded): - # As with Unstructured, we can have pure ASCII with or without - # surrogateescape encoded bytes, or we could have unicode. But this - # case is more complicated, since we have to deal with the various - # sub-token types and how they can be composed in the face of - # unicode-that-needs-CTE-encoding, and the fact that if a token a - # comment that becomes a barrier across which we can't compose encoded - # words. - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - has_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None and not part.has_leading_comment(): - # We've already done an EW, let's see if we can combine - # this one with it. The last_ew logic ensures that all we - # have at this point is atoms, no comments or quoted - # strings. So we can treat the text between the last - # encoded word and the content of this token as - # unstructured text, and things will work correctly. But - # we have to strip off any trailing comment on this token - # first, and if it is a quoted string we have to pull out - # the content (we're encoding it, so it no longer needs to - # be quoted). - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - schunk = str(chunk) - lchunk = len(schunk) - if last_ew + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = sum(len(x) for x in folded.current) - continue - tstr = part.as_encoded_word(charset) - tlen = len(tstr) - has_ew = True - if folded.append_if_fits(part, tstr): - if has_ew and not part.comments: - last_ew = len(folded.current) - 1 - elif part.comments or part.token_type == 'quoted-string': - # If a comment is involved we can't combine EWs. And if a - # quoted string is involved, it's not worth the effort to - # try to combine them. - last_ew = None - continue - part._fold(folded) - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - is_ew = False - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - is_ew = True - if last_ew is None: - if not part.comments: - last_ew = len(res) - res.append(part.cte_encode(charset, policy)) - elif not part.has_leading_comment(): - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res[last_ew:] = [tl.as_encoded_word(charset)] - if part.comments or (not is_ew and part.token_type == 'quoted-string'): - last_ew = None - return ''.join(res) - class Word(TokenList): - token_type = 'word' class CFWSList(WhiteSpaceTokenList): - token_type = 'cfws' - def has_leading_comment(self): - return bool(self.comments) - class Atom(TokenList): - token_type = 'atom' class Token(TokenList): - token_type = 'token' + encode_as_ew = False class EncodedWord(TokenList): - token_type = 'encoded-word' cte = None charset = None lang = None - @property - def encoded(self): - if self.cte is not None: - return self.cte - _ew.encode(str(self), self.charset) - - class QuotedString(TokenList): @@ -812,7 +445,10 @@ def route(self): def addr_spec(self): for x in self: if x.token_type == 'addr-spec': - return x.addr_spec + if x.local_part: + return x.addr_spec + else: + return quote_string(x.local_part) + x.addr_spec else: return '<>' @@ -867,6 +503,7 @@ def display_name(self): class Domain(TokenList): token_type = 'domain' + as_ew_allowed = False @property def domain(self): @@ -874,18 +511,23 @@ def domain(self): class DotAtom(TokenList): - token_type = 'dot-atom' class DotAtomText(TokenList): - token_type = 'dot-atom-text' + as_ew_allowed = True + + +class NoFoldLiteral(TokenList): + token_type = 'no-fold-literal' + as_ew_allowed = False class AddrSpec(TokenList): token_type = 'addr-spec' + as_ew_allowed = False @property def local_part(self): @@ -918,24 +560,30 @@ def addr_spec(self): class ObsLocalPart(TokenList): token_type = 'obs-local-part' + as_ew_allowed = False class DisplayName(Phrase): token_type = 'display-name' + ew_combine_allowed = False @property def display_name(self): res = TokenList(self) + if len(res) == 0: + return res.value if res[0].token_type == 'cfws': res.pop(0) else: - if res[0][0].token_type == 'cfws': + if (isinstance(res[0], TokenList) and + res[0][0].token_type == 'cfws'): res[0] = TokenList(res[0][1:]) if res[-1].token_type == 'cfws': res.pop() else: - if res[-1][-1].token_type == 'cfws': + if (isinstance(res[-1], TokenList) and + res[-1][-1].token_type == 'cfws'): res[-1] = TokenList(res[-1][:-1]) return res.value @@ -948,11 +596,15 @@ def value(self): for x in self: if x.token_type == 'quoted-string': quote = True - if quote: + if len(self) != 0 and quote: pre = post = '' - if self[0].token_type=='cfws' or self[0][0].token_type=='cfws': + if (self[0].token_type == 'cfws' or + isinstance(self[0], TokenList) and + self[0][0].token_type == 'cfws'): pre = ' ' - if self[-1].token_type=='cfws' or self[-1][-1].token_type=='cfws': + if (self[-1].token_type == 'cfws' or + isinstance(self[-1], TokenList) and + self[-1][-1].token_type == 'cfws'): post = ' ' return pre+quote_string(self.display_name)+post else: @@ -962,6 +614,7 @@ def value(self): class LocalPart(TokenList): token_type = 'local-part' + as_ew_allowed = False @property def value(self): @@ -997,6 +650,7 @@ def local_part(self): class DomainLiteral(TokenList): token_type = 'domain-literal' + as_ew_allowed = False @property def domain(self): @@ -1083,6 +737,7 @@ def stripped_value(self): class MimeParameters(TokenList): token_type = 'mime-parameters' + syntactic_break = False @property def params(self): @@ -1091,7 +746,7 @@ def params(self): # to assume the RFC 2231 pieces can come in any order. However, we # output them in the order that we first see a given name, which gives # us a stable __str__. - params = OrderedDict() + params = {} # Using order preserving dict from Python 3.7+ for token in self: if not token.token_type.endswith('parameter'): continue @@ -1142,7 +797,7 @@ def params(self): else: try: value = value.decode(charset, 'surrogateescape') - except LookupError: + except (LookupError, UnicodeEncodeError): # XXX: there should really be a custom defect for # unknown character set to make it easy to find, # because otherwise unknown charset is a silent @@ -1167,6 +822,10 @@ def __str__(self): class ParameterizedHeaderValue(TokenList): + # Set this false so that the value doesn't wind up on a new line even + # if it and the parameters would fit there but not on the first line. + syntactic_break = False + @property def params(self): for token in reversed(self): @@ -1174,58 +833,50 @@ def params(self): return token.params return {} - @property - def parts(self): - if self and self[-1].token_type == 'mime-parameters': - # We don't want to start a new line if all of the params don't fit - # after the value, so unwrap the parameter list. - return TokenList(self[:-1] + self[-1]) - return TokenList(self).parts - class ContentType(ParameterizedHeaderValue): - token_type = 'content-type' + as_ew_allowed = False maintype = 'text' subtype = 'plain' class ContentDisposition(ParameterizedHeaderValue): - token_type = 'content-disposition' + as_ew_allowed = False content_disposition = None class ContentTransferEncoding(TokenList): - token_type = 'content-transfer-encoding' + as_ew_allowed = False cte = '7bit' class HeaderLabel(TokenList): - token_type = 'header-label' + as_ew_allowed = False -class Header(TokenList): +class MsgID(TokenList): + token_type = 'msg-id' + as_ew_allowed = False - token_type = 'header' + def fold(self, policy): + # message-id tokens may not be folded. + return str(self) + policy.linesep + + +class MessageID(MsgID): + token_type = 'message-id' - def _fold(self, folded): - folded.append(str(self.pop(0))) - folded.lastlen = len(folded.current[0]) - # The first line of the header is different from all others: we don't - # want to start a new object on a new line if it has any fold points in - # it that would allow part of it to be on the first header line. - # Further, if the first fold point would fit on the new line, we want - # to do that, but if it doesn't we want to put it on the first line. - # Folded supports this via the stickyspace attribute. If this - # attribute is not None, it does the special handling. - folded.stickyspace = str(self.pop(0)) if self[0].token_type == 'cfws' else '' - rest = self.pop(0) - if self: - raise ValueError("Malformed Header token list") - rest._fold(folded) + +class InvalidMessageID(MessageID): + token_type = 'invalid-message-id' + + +class Header(TokenList): + token_type = 'header' # @@ -1234,6 +885,10 @@ def _fold(self, folded): class Terminal(str): + as_ew_allowed = True + ew_combine_allowed = True + syntactic_break = True + def __new__(cls, value, token_type): self = super().__new__(cls, value) self.token_type = token_type @@ -1243,6 +898,9 @@ def __new__(cls, value, token_type): def __repr__(self): return "{}({})".format(self.__class__.__name__, super().__repr__()) + def pprint(self): + print(self.__class__.__name__ + '/' + self.token_type) + @property def all_defects(self): return list(self.defects) @@ -1256,29 +914,14 @@ def _pp(self, indent=''): '' if not self.defects else ' {}'.format(self.defects), )] - def cte_encode(self, charset, policy): - value = str(self) - try: - value.encode('us-ascii') - return value - except UnicodeEncodeError: - return _ew.encode(value, charset) - def pop_trailing_ws(self): # This terminates the recursion. return None - def pop_leading_fws(self): - # This terminates the recursion. - return None - @property def comments(self): return [] - def has_leading_comment(self): - return False - def __getnewargs__(self): return(str(self), self.token_type) @@ -1292,8 +935,6 @@ def value(self): def startswith_fws(self): return True - has_fws = True - class ValueTerminal(Terminal): @@ -1304,11 +945,6 @@ def value(self): def startswith_fws(self): return False - has_fws = False - - def as_encoded_word(self, charset): - return _ew.encode(str(self), charset) - class EWWhiteSpaceTerminal(WhiteSpaceTerminal): @@ -1316,14 +952,12 @@ class EWWhiteSpaceTerminal(WhiteSpaceTerminal): def value(self): return '' - @property - def encoded(self): - return self[:] - def __str__(self): return '' - has_fws = True + +class _InvalidEwError(errors.HeaderParseError): + """Invalid encoded word found while parsing headers.""" # XXX these need to become classes and used as instances so @@ -1331,6 +965,8 @@ def __str__(self): # up other parse trees. Maybe should have tests for that, too. DOT = ValueTerminal('.', 'dot') ListSeparator = ValueTerminal(',', 'list-separator') +ListSeparator.as_ew_allowed = False +ListSeparator.syntactic_break = False RouteComponentMarker = ValueTerminal('@', 'route-component-marker') # @@ -1356,15 +992,14 @@ def __str__(self): _wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split _non_atom_end_matcher = re.compile(r"[^{}]+".format( - ''.join(ATOM_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(ATOM_ENDS)))).match _non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall _non_token_end_matcher = re.compile(r"[^{}]+".format( - ''.join(TOKEN_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(TOKEN_ENDS)))).match _non_attribute_end_matcher = re.compile(r"[^{}]+".format( - ''.join(ATTRIBUTE_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(ATTRIBUTE_ENDS)))).match _non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format( - ''.join(EXTENDED_ATTRIBUTE_ENDS).replace( - '\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(EXTENDED_ATTRIBUTE_ENDS)))).match def _validate_xtext(xtext): """If input token contains ASCII non-printables, register a defect.""" @@ -1385,6 +1020,8 @@ def _get_ptext_to_endchars(value, endchars): a flag that is True iff there were any quoted printables decoded. """ + if not value: + return '', '', False fragment, *remainder = _wsp_splitter(value, 1) vchars = [] escape = False @@ -1418,7 +1055,7 @@ def get_fws(value): fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws') return fws, newvalue -def get_encoded_word(value): +def get_encoded_word(value, terminal_type='vtext'): """ encoded-word = "=?" charset "?" encoding "?" encoded-text "?=" """ @@ -1431,7 +1068,10 @@ def get_encoded_word(value): raise errors.HeaderParseError( "expected encoded word but found {}".format(value)) remstr = ''.join(remainder) - if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits: + if (len(remstr) > 1 and + remstr[0] in hexdigits and + remstr[1] in hexdigits and + tok.count('?') < 2): # The ? after the CTE was followed by an encoded word escape (=XX). rest, *remainder = remstr.split('?=', 1) tok = tok + '?=' + rest @@ -1442,8 +1082,8 @@ def get_encoded_word(value): value = ''.join(remainder) try: text, charset, lang, defects = _ew.decode('=?' + tok + '?=') - except ValueError: - raise errors.HeaderParseError( + except (ValueError, KeyError): + raise _InvalidEwError( "encoded word format invalid: '{}'".format(ew.cte)) ew.charset = charset ew.lang = lang @@ -1454,10 +1094,14 @@ def get_encoded_word(value): ew.append(token) continue chars, *remainder = _wsp_splitter(text, 1) - vtext = ValueTerminal(chars, 'vtext') + vtext = ValueTerminal(chars, terminal_type) _validate_xtext(vtext) ew.append(vtext) text = ''.join(remainder) + # Encoded words should be followed by a WS + if value and value[0] not in WSP: + ew.defects.append(errors.InvalidHeaderDefect( + "missing trailing whitespace after encoded-word")) return ew, value def get_unstructured(value): @@ -1489,9 +1133,12 @@ def get_unstructured(value): token, value = get_fws(value) unstructured.append(token) continue + valid_ew = True if value.startswith('=?'): try: - token, value = get_encoded_word(value) + token, value = get_encoded_word(value, 'utext') + except _InvalidEwError: + valid_ew = False except errors.HeaderParseError: # XXX: Need to figure out how to register defects when # appropriate here. @@ -1510,7 +1157,15 @@ def get_unstructured(value): unstructured.append(token) continue tok, *remainder = _wsp_splitter(value, 1) - vtext = ValueTerminal(tok, 'vtext') + # Split in the middle of an atom if there is a rfc2047 encoded word + # which does not have WSP on both sides. The defect will be registered + # the next time through the loop. + # This needs to only be performed when the encoded word is valid; + # otherwise, performing it on an invalid encoded word can cause + # the parser to go in an infinite loop. + if valid_ew and rfc2047_matcher.search(tok): + tok, *remainder = value.partition('=?') + vtext = ValueTerminal(tok, 'utext') _validate_xtext(vtext) unstructured.append(vtext) value = ''.join(remainder) @@ -1571,21 +1226,33 @@ def get_bare_quoted_string(value): value is the text between the quote marks, with whitespace preserved and quoted pairs decoded. """ - if value[0] != '"': + if not value or value[0] != '"': raise errors.HeaderParseError( "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() value = value[1:] + if value and value[0] == '"': + token, value = get_qcontent(value) + bare_quoted_string.append(token) while value and value[0] != '"': if value[0] in WSP: token, value = get_fws(value) elif value[:2] == '=?': + valid_ew = False try: token, value = get_encoded_word(value) bare_quoted_string.defects.append(errors.InvalidHeaderDefect( "encoded word inside quoted string")) + valid_ew = True except errors.HeaderParseError: token, value = get_qcontent(value) + # Collapse the whitespace between two encoded words that occur in a + # bare-quoted-string. + if valid_ew and len(bare_quoted_string) > 1: + if (bare_quoted_string[-1].token_type == 'fws' and + bare_quoted_string[-2].token_type == 'encoded-word'): + bare_quoted_string[-1] = EWWhiteSpaceTerminal( + bare_quoted_string[-1], 'fws') else: token, value = get_qcontent(value) bare_quoted_string.append(token) @@ -1742,6 +1409,9 @@ def get_word(value): leader, value = get_cfws(value) else: leader = None + if not value: + raise errors.HeaderParseError( + "Expected 'atom' or 'quoted-string' but found nothing.") if value[0]=='"': token, value = get_quoted_string(value) elif value[0] in SPECIALS: @@ -1797,7 +1467,7 @@ def get_local_part(value): """ local_part = LocalPart() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1863,13 +1533,18 @@ def get_obs_local_part(value): raise token, value = get_cfws(value) obs_local_part.append(token) + if not obs_local_part: + raise errors.HeaderParseError( + "expected obs-local-part but found '{}'".format(value)) if (obs_local_part[0].token_type == 'dot' or obs_local_part[0].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[1].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid leading '.' in local part")) if (obs_local_part[-1].token_type == 'dot' or obs_local_part[-1].token_type=='cfws' and + len(obs_local_part) > 1 and obs_local_part[-2].token_type=='dot'): obs_local_part.defects.append(errors.InvalidHeaderDefect( "Invalid trailing '.' in local part")) @@ -1900,7 +1575,7 @@ def get_dtext(value): def _check_for_early_dl_end(value, domain_literal): if value: return False - domain_literal.append(errors.InvalidHeaderDefect( + domain_literal.defects.append(errors.InvalidHeaderDefect( "end of input inside domain-literal")) domain_literal.append(ValueTerminal(']', 'domain-literal-end')) return True @@ -1919,9 +1594,9 @@ def get_domain_literal(value): raise errors.HeaderParseError("expected '[' at start of domain-literal " "but found '{}'".format(value)) value = value[1:] + domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if _check_for_early_dl_end(value, domain_literal): return domain_literal, value - domain_literal.append(ValueTerminal('[', 'domain-literal-start')) if value[0] in WSP: token, value = get_fws(value) domain_literal.append(token) @@ -1951,7 +1626,7 @@ def get_domain(value): """ domain = Domain() leader = None - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: raise errors.HeaderParseError( @@ -1966,6 +1641,8 @@ def get_domain(value): token, value = get_dot_atom(value) except errors.HeaderParseError: token, value = get_atom(value) + if value and value[0] == '@': + raise errors.HeaderParseError('Invalid Domain') if leader is not None: token[:0] = [leader] domain.append(token) @@ -1989,7 +1666,7 @@ def get_addr_spec(value): addr_spec.append(token) if not value or value[0] != '@': addr_spec.defects.append(errors.InvalidHeaderDefect( - "add-spec local part with no domain")) + "addr-spec local part with no domain")) return addr_spec, value addr_spec.append(ValueTerminal('@', 'address-at-symbol')) token, value = get_domain(value[1:]) @@ -2025,6 +1702,8 @@ def get_obs_route(value): if value[0] in CFWS_LEADER: token, value = get_cfws(value) obs_route.append(token) + if not value: + break if value[0] == '@': obs_route.append(RouteComponentMarker) token, value = get_domain(value[1:]) @@ -2043,7 +1722,7 @@ def get_angle_addr(value): """ angle_addr = AngleAddr() - if value[0] in CFWS_LEADER: + if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) angle_addr.append(token) if not value or value[0] != '<': @@ -2053,7 +1732,7 @@ def get_angle_addr(value): value = value[1:] # Although it is not legal per RFC5322, SMTP uses '<>' in certain # circumstances. - if value[0] == '>': + if value and value[0] == '>': angle_addr.append(ValueTerminal('>', 'angle-addr-end')) angle_addr.defects.append(errors.InvalidHeaderDefect( "null addr-spec in angle-addr")) @@ -2105,6 +1784,9 @@ def get_name_addr(value): name_addr = NameAddr() # Both the optional display name and the angle-addr can start with cfws. leader = None + if not value: + raise errors.HeaderParseError( + "expected name-addr but found '{}'".format(value)) if value[0] in CFWS_LEADER: leader, value = get_cfws(value) if not value: @@ -2119,7 +1801,10 @@ def get_name_addr(value): raise errors.HeaderParseError( "expected name-addr but found '{}'".format(token)) if leader is not None: - token[0][:0] = [leader] + if isinstance(token[0], TokenList): + token[0][:0] = [leader] + else: + token[:0] = [leader] leader = None name_addr.append(token) token, value = get_angle_addr(value) @@ -2281,7 +1966,7 @@ def get_group(value): if not value: group.defects.append(errors.InvalidHeaderDefect( "end of header in group")) - if value[0] != ';': + elif value[0] != ';': raise errors.HeaderParseError( "expected ';' at end of group but found {}".format(value)) group.append(ValueTerminal(';', 'group-terminator')) @@ -2335,7 +2020,7 @@ def get_address_list(value): try: token, value = get_address(value) address_list.append(token) - except errors.HeaderParseError as err: + except errors.HeaderParseError: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) @@ -2370,10 +2055,122 @@ def get_address_list(value): address_list.defects.append(errors.InvalidHeaderDefect( "invalid address in address-list")) if value: # Must be a , at this point. - address_list.append(ValueTerminal(',', 'list-separator')) + address_list.append(ListSeparator) value = value[1:] return address_list, value + +def get_no_fold_literal(value): + """ no-fold-literal = "[" *dtext "]" + """ + no_fold_literal = NoFoldLiteral() + if not value: + raise errors.HeaderParseError( + "expected no-fold-literal but found '{}'".format(value)) + if value[0] != '[': + raise errors.HeaderParseError( + "expected '[' at the start of no-fold-literal " + "but found '{}'".format(value)) + no_fold_literal.append(ValueTerminal('[', 'no-fold-literal-start')) + value = value[1:] + token, value = get_dtext(value) + no_fold_literal.append(token) + if not value or value[0] != ']': + raise errors.HeaderParseError( + "expected ']' at the end of no-fold-literal " + "but found '{}'".format(value)) + no_fold_literal.append(ValueTerminal(']', 'no-fold-literal-end')) + return no_fold_literal, value[1:] + +def get_msg_id(value): + """msg-id = [CFWS] "<" id-left '@' id-right ">" [CFWS] + id-left = dot-atom-text / obs-id-left + id-right = dot-atom-text / no-fold-literal / obs-id-right + no-fold-literal = "[" *dtext "]" + """ + msg_id = MsgID() + if value and value[0] in CFWS_LEADER: + token, value = get_cfws(value) + msg_id.append(token) + if not value or value[0] != '<': + raise errors.HeaderParseError( + "expected msg-id but found '{}'".format(value)) + msg_id.append(ValueTerminal('<', 'msg-id-start')) + value = value[1:] + # Parse id-left. + try: + token, value = get_dot_atom_text(value) + except errors.HeaderParseError: + try: + # obs-id-left is same as local-part of add-spec. + token, value = get_obs_local_part(value) + msg_id.defects.append(errors.ObsoleteHeaderDefect( + "obsolete id-left in msg-id")) + except errors.HeaderParseError: + raise errors.HeaderParseError( + "expected dot-atom-text or obs-id-left" + " but found '{}'".format(value)) + msg_id.append(token) + if not value or value[0] != '@': + msg_id.defects.append(errors.InvalidHeaderDefect( + "msg-id with no id-right")) + # Even though there is no id-right, if the local part + # ends with `>` let's just parse it too and return + # along with the defect. + if value and value[0] == '>': + msg_id.append(ValueTerminal('>', 'msg-id-end')) + value = value[1:] + return msg_id, value + msg_id.append(ValueTerminal('@', 'address-at-symbol')) + value = value[1:] + # Parse id-right. + try: + token, value = get_dot_atom_text(value) + except errors.HeaderParseError: + try: + token, value = get_no_fold_literal(value) + except errors.HeaderParseError: + try: + token, value = get_domain(value) + msg_id.defects.append(errors.ObsoleteHeaderDefect( + "obsolete id-right in msg-id")) + except errors.HeaderParseError: + raise errors.HeaderParseError( + "expected dot-atom-text, no-fold-literal or obs-id-right" + " but found '{}'".format(value)) + msg_id.append(token) + if value and value[0] == '>': + value = value[1:] + else: + msg_id.defects.append(errors.InvalidHeaderDefect( + "missing trailing '>' on msg-id")) + msg_id.append(ValueTerminal('>', 'msg-id-end')) + if value and value[0] in CFWS_LEADER: + token, value = get_cfws(value) + msg_id.append(token) + return msg_id, value + + +def parse_message_id(value): + """message-id = "Message-ID:" msg-id CRLF + """ + message_id = MessageID() + try: + token, value = get_msg_id(value) + message_id.append(token) + except errors.HeaderParseError as ex: + token = get_unstructured(value) + message_id = InvalidMessageID(token) + message_id.defects.append( + errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex))) + else: + # Value after parsing a valid msg_id should be None. + if value: + message_id.defects.append(errors.InvalidHeaderDefect( + "Unexpected {!r}".format(value))) + + return message_id + # # XXX: As I begin to add additional header parsers, I'm realizing we probably # have two level of parser routines: the get_XXX methods that get a token in @@ -2615,8 +2412,8 @@ def get_section(value): digits += value[0] value = value[1:] if digits[0] == '0' and digits != '0': - section.defects.append(errors.InvalidHeaderError("section number" - "has an invalid leading 0")) + section.defects.append(errors.InvalidHeaderDefect( + "section number has an invalid leading 0")) section.number = int(digits) section.append(ValueTerminal(digits, 'digits')) return section, value @@ -2679,7 +2476,6 @@ def get_parameter(value): raise errors.HeaderParseError("Parameter not followed by '='") param.append(ValueTerminal('=', 'parameter-separator')) value = value[1:] - leader = None if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) param.append(token) @@ -2754,7 +2550,7 @@ def get_parameter(value): if value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {!r}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if value and value[0] != "'": token, value = get_attrtext(value) @@ -2763,7 +2559,7 @@ def get_parameter(value): if not value or value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if remainder is not None: # Treat the rest of value as bare quoted string content. @@ -2771,6 +2567,9 @@ def get_parameter(value): while value: if value[0] in WSP: token, value = get_fws(value) + elif value[0] == '"': + token = ValueTerminal('"', 'DQUOTE') + value = value[1:] else: token, value = get_qcontent(value) v.append(token) @@ -2791,7 +2590,7 @@ def parse_mime_parameters(value): the formal RFC grammar, but it is more convenient for us for the set of parameters to be treated as its own TokenList. - This is 'parse' routine because it consumes the reminaing value, but it + This is 'parse' routine because it consumes the remaining value, but it would never be called to parse a full header. Instead it is called to parse everything after the non-parameter value of a specific MIME header. @@ -2801,7 +2600,7 @@ def parse_mime_parameters(value): try: token, value = get_parameter(value) mime_parameters.append(token) - except errors.HeaderParseError as err: + except errors.HeaderParseError: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) @@ -2859,7 +2658,6 @@ def parse_content_type_header(value): don't do that. """ ctype = ContentType() - recover = False if not value: ctype.defects.append(errors.HeaderMissingRequiredValue( "Missing content type specification")) @@ -2968,3 +2766,332 @@ def parse_content_transfer_encoding_header(value): token, value = get_phrase(value) cte_header.append(token) return cte_header + + +# +# Header folding +# +# Header folding is complex, with lots of rules and corner cases. The +# following code does its best to obey the rules and handle the corner +# cases, but you can be sure there are few bugs:) +# +# This folder generally canonicalizes as it goes, preferring the stringified +# version of each token. The tokens contain information that supports the +# folder, including which tokens can be encoded in which ways. +# +# Folded text is accumulated in a simple list of strings ('lines'), each +# one of which should be less than policy.max_line_length ('maxlen'). +# + +def _steal_trailing_WSP_if_exists(lines): + wsp = '' + if lines and lines[-1] and lines[-1][-1] in WSP: + wsp = lines[-1][-1] + lines[-1] = lines[-1][:-1] + return wsp + +def _refold_parse_tree(parse_tree, *, policy): + """Return string of contents of parse_tree folded according to RFC rules. + + """ + # max_line_length 0/None means no limit, ie: infinitely long. + maxlen = policy.max_line_length or sys.maxsize + encoding = 'utf-8' if policy.utf8 else 'us-ascii' + lines = [''] # Folded lines to be output + leading_whitespace = '' # When we have whitespace between two encoded + # words, we may need to encode the whitespace + # at the beginning of the second word. + last_ew = None # Points to the last encoded character if there's an ew on + # the line + last_charset = None + wrap_as_ew_blocked = 0 + want_encoding = False # This is set to True if we need to encode this part + end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') + parts = list(parse_tree) + while parts: + part = parts.pop(0) + if part is end_ew_not_allowed: + wrap_as_ew_blocked -= 1 + continue + tstr = str(part) + if not want_encoding: + if part.token_type in ('ptext', 'vtext'): + # Encode if tstr contains special characters. + want_encoding = not SPECIALSNL.isdisjoint(tstr) + else: + # Encode if tstr contains newlines. + want_encoding = not NLSET.isdisjoint(tstr) + try: + tstr.encode(encoding) + charset = encoding + except UnicodeEncodeError: + if any(isinstance(x, errors.UndecodableBytesDefect) + for x in part.all_defects): + charset = 'unknown-8bit' + else: + # If policy.utf8 is false this should really be taken from a + # 'charset' property on the policy. + charset = 'utf-8' + want_encoding = True + + if part.token_type == 'mime-parameters': + # Mime parameter folding (using RFC2231) is extra special. + _fold_mime_parameters(part, lines, maxlen, encoding) + continue + + if want_encoding and not wrap_as_ew_blocked: + if not part.as_ew_allowed: + want_encoding = False + last_ew = None + if part.syntactic_break: + encoded_part = part.fold(policy=policy)[:-len(policy.linesep)] + if policy.linesep not in encoded_part: + # It fits on a single line + if len(encoded_part) > maxlen - len(lines[-1]): + # But not on this one, so start a new one. + newline = _steal_trailing_WSP_if_exists(lines) + # XXX what if encoded_part has no leading FWS? + lines.append(newline) + lines[-1] += encoded_part + continue + # Either this is not a major syntactic break, so we don't + # want it on a line by itself even if it fits, or it + # doesn't fit on a line by itself. Either way, fall through + # to unpacking the subparts and wrapping them. + if not hasattr(part, 'encode'): + # It's not a Terminal, do each piece individually. + parts = list(part) + parts + want_encoding = False + continue + elif part.as_ew_allowed: + # It's a terminal, wrap it as an encoded word, possibly + # combining it with previously encoded words if allowed. + if (last_ew is not None and + charset != last_charset and + (last_charset == 'unknown-8bit' or + last_charset == 'utf-8' and charset != 'us-ascii')): + last_ew = None + last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, + part.ew_combine_allowed, charset, leading_whitespace) + # This whitespace has been added to the lines in _fold_as_ew() + # so clear it now. + leading_whitespace = '' + last_charset = charset + want_encoding = False + continue + else: + # It's a terminal which should be kept non-encoded + # (e.g. a ListSeparator). + last_ew = None + want_encoding = False + # fall through + + if len(tstr) <= maxlen - len(lines[-1]): + lines[-1] += tstr + continue + + # This part is too long to fit. The RFC wants us to break at + # "major syntactic breaks", so unless we don't consider this + # to be one, check if it will fit on the next line by itself. + leading_whitespace = '' + if (part.syntactic_break and + len(tstr) + 1 <= maxlen): + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + # We're going to fold the data onto a new line here. Due to + # the way encoded strings handle continuation lines, we need to + # be prepared to encode any whitespace if the next line turns + # out to start with an encoded word. + lines.append(newline + tstr) + + whitespace_accumulator = [] + for char in lines[-1]: + if char not in WSP: + break + whitespace_accumulator.append(char) + leading_whitespace = ''.join(whitespace_accumulator) + last_ew = None + continue + if not hasattr(part, 'encode'): + # It's not a terminal, try folding the subparts. + newparts = list(part) + if part.token_type == 'bare-quoted-string': + # To fold a quoted string we need to create a list of terminal + # tokens that will render the leading and trailing quotes + # and use quoted pairs in the value as appropriate. + newparts = ( + [ValueTerminal('"', 'ptext')] + + [ValueTerminal(make_quoted_pairs(p), 'ptext') + for p in newparts] + + [ValueTerminal('"', 'ptext')]) + if not part.as_ew_allowed: + wrap_as_ew_blocked += 1 + newparts.append(end_ew_not_allowed) + parts = newparts + parts + continue + if part.as_ew_allowed and not wrap_as_ew_blocked: + # It doesn't need CTE encoding, but encode it anyway so we can + # wrap it. + parts.insert(0, part) + want_encoding = True + continue + # We can't figure out how to wrap, it, so give up. + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + else: + # We can't fold it onto the next line either... + lines[-1] += tstr + + return policy.linesep.join(lines) + policy.linesep + +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, leading_whitespace): + """Fold string to_encode into lines as encoded word, combining if allowed. + Return the new value for last_ew, or None if ew_combine_allowed is False. + + If there is already an encoded word in the last line of lines (indicated by + a non-None value for last_ew) and ew_combine_allowed is true, decode the + existing ew, combine it with to_encode, and re-encode. Otherwise, encode + to_encode. In either case, split to_encode as necessary so that the + encoded segments fit within maxlen. + + """ + if last_ew is not None and ew_combine_allowed: + to_encode = str( + get_unstructured(lines[-1][last_ew:] + to_encode)) + lines[-1] = lines[-1][:last_ew] + elif to_encode[0] in WSP: + # We're joining this to non-encoded text, so don't encode + # the leading blank. + leading_wsp = to_encode[0] + to_encode = to_encode[1:] + if (len(lines[-1]) == maxlen): + lines.append(_steal_trailing_WSP_if_exists(lines)) + lines[-1] += leading_wsp + + trailing_wsp = '' + if to_encode[-1] in WSP: + # Likewise for the trailing space. + trailing_wsp = to_encode[-1] + to_encode = to_encode[:-1] + new_last_ew = len(lines[-1]) if last_ew is None else last_ew + + encode_as = 'utf-8' if charset == 'us-ascii' else charset + + # The RFC2047 chrome takes up 7 characters plus the length + # of the charset name. + chrome_len = len(encode_as) + 7 + + if (chrome_len + 1) >= maxlen: + raise errors.HeaderParseError( + "max_line_length is too small to fit an encoded word") + + while to_encode: + remaining_space = maxlen - len(lines[-1]) + text_space = remaining_space - chrome_len - len(leading_whitespace) + if text_space <= 0: + lines.append(' ') + continue + + # If we are at the start of a continuation line, prepend whitespace + # (we only want to do this when the line starts with an encoded word + # but if we're folding in this helper function, then we know that we + # are going to be writing out an encoded word.) + if len(lines) > 1 and len(lines[-1]) == 1 and leading_whitespace: + encoded_word = _ew.encode(leading_whitespace, charset=encode_as) + lines[-1] += encoded_word + leading_whitespace = '' + + to_encode_word = to_encode[:text_space] + encoded_word = _ew.encode(to_encode_word, charset=encode_as) + excess = len(encoded_word) - remaining_space + while excess > 0: + # Since the chunk to encode is guaranteed to fit into less than 100 characters, + # shrinking it by one at a time shouldn't take long. + to_encode_word = to_encode_word[:-1] + encoded_word = _ew.encode(to_encode_word, charset=encode_as) + excess = len(encoded_word) - remaining_space + lines[-1] += encoded_word + to_encode = to_encode[len(to_encode_word):] + leading_whitespace = '' + + if to_encode: + lines.append(' ') + new_last_ew = len(lines[-1]) + lines[-1] += trailing_wsp + return new_last_ew if ew_combine_allowed else None + +def _fold_mime_parameters(part, lines, maxlen, encoding): + """Fold TokenList 'part' into the 'lines' list as mime parameters. + + Using the decoded list of parameters and values, format them according to + the RFC rules, including using RFC2231 encoding if the value cannot be + expressed in 'encoding' and/or the parameter+value is too long to fit + within 'maxlen'. + + """ + # Special case for RFC2231 encoding: start from decoded values and use + # RFC2231 encoding iff needed. + # + # Note that the 1 and 2s being added to the length calculations are + # accounting for the possibly-needed spaces and semicolons we'll be adding. + # + for name, value in part.params: + # XXX What if this ';' puts us over maxlen the first time through the + # loop? We should split the header value onto a newline in that case, + # but to do that we need to recognize the need earlier or reparse the + # header, so I'm going to ignore that bug for now. It'll only put us + # one character over. + if not lines[-1].rstrip().endswith(';'): + lines[-1] += ';' + charset = encoding + error_handler = 'strict' + try: + value.encode(encoding) + encoding_required = False + except UnicodeEncodeError: + encoding_required = True + if utils._has_surrogates(value): + charset = 'unknown-8bit' + error_handler = 'surrogateescape' + else: + charset = 'utf-8' + if encoding_required: + encoded_value = urllib.parse.quote( + value, safe='', errors=error_handler) + tstr = "{}*={}''{}".format(name, charset, encoded_value) + else: + tstr = '{}={}'.format(name, quote_string(value)) + if len(lines[-1]) + len(tstr) + 1 < maxlen: + lines[-1] = lines[-1] + ' ' + tstr + continue + elif len(tstr) + 2 <= maxlen: + lines.append(' ' + tstr) + continue + # We need multiple sections. We are allowed to mix encoded and + # non-encoded sections, but we aren't going to. We'll encode them all. + section = 0 + extra_chrome = charset + "''" + while value: + chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome) + if maxlen <= chrome_len + 3: + # We need room for the leading blank, the trailing semicolon, + # and at least one character of the value. If we don't + # have that, we'd be stuck, so in that case fall back to + # the RFC standard width. + maxlen = 78 + splitpoint = maxchars = maxlen - chrome_len - 2 + while True: + partial = value[:splitpoint] + encoded_value = urllib.parse.quote( + partial, safe='', errors=error_handler) + if len(encoded_value) <= maxchars: + break + splitpoint -= 1 + lines.append(" {}*{}*={}{}".format( + name, section, extra_chrome, encoded_value)) + extra_chrome = '' + section += 1 + value = value[splitpoint:] + if value: + lines[-1] += ';' diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index cdfa3729adc..565af0cf361 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -13,7 +13,7 @@ 'quote', ] -import time, calendar +import time SPACE = ' ' EMPTYSTRING = '' @@ -65,8 +65,10 @@ def _parsedate_tz(data): """ if not data: - return + return None data = data.split() + if not data: # This happens for whitespace-only input. + return None # The FWS after the comma after the day-of-week is optional, so search and # adjust for this. if data[0].endswith(',') or data[0].lower() in _daynames: @@ -93,6 +95,8 @@ def _parsedate_tz(data): return None data = data[:5] [dd, mm, yy, tm, tz] = data + if not (dd and mm and yy): + return None mm = mm.lower() if mm not in _monthnames: dd, mm = mm, dd.lower() @@ -108,6 +112,8 @@ def _parsedate_tz(data): yy, tm = tm, yy if yy[-1] == ',': yy = yy[:-1] + if not yy: + return None if not yy[0].isdigit(): yy, tz = tz, yy if tm[-1] == ',': @@ -126,6 +132,8 @@ def _parsedate_tz(data): tss = 0 elif len(tm) == 3: [thh, tmm, tss] = tm + else: + return None else: return None try: @@ -138,8 +146,9 @@ def _parsedate_tz(data): return None # Check for a yy specified in two-digit format, then convert it to the # appropriate four-digit format, according to the POSIX standard. RFC 822 - # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) - # mandates a 4-digit yy. For more information, see the documentation for + # calls for a two-digit yy, but RFC 2822 (which obsoletes RFC 822) already + # mandated a 4-digit yy, and RFC 5322 (which obsoletes RFC 2822) continues + # this requirement. For more information, see the documentation for # the time module. if yy < 100: # The year is between 1969 and 1999 (inclusive). @@ -186,6 +195,9 @@ def mktime_tz(data): # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: + # Delay the import, since mktime_tz is rarely used + import calendar + t = calendar.timegm(data) return t - data[9] @@ -222,9 +234,11 @@ def __init__(self, field): self.CR = '\r\n' self.FWS = self.LWS + self.CR self.atomends = self.specials + self.LWS + self.CR - # Note that RFC 2822 now specifies `.' as obs-phrase, meaning that it - # is obsolete syntax. RFC 2822 requires that we recognize obsolete - # syntax, so allow dots in phrases. + # Note that RFC 2822 section 4.1 introduced '.' as obs-phrase to handle + # existing practice (periods in display names), even though it was not + # allowed in RFC 822. RFC 5322 section 4.1 (which obsoletes RFC 2822) + # continues this requirement. We must recognize obsolete syntax, so + # allow dots in phrases. self.phraseends = self.atomends.replace('.', '') self.field = field self.commentlist = [] @@ -379,7 +393,12 @@ def getaddrspec(self): aslist.append('@') self.pos += 1 self.gotonext() - return EMPTYSTRING.join(aslist) + self.getdomain() + domain = self.getdomain() + if not domain: + # Invalid domain, return an empty address instead of returning a + # local part to denote failed parsing. + return EMPTYSTRING + return EMPTYSTRING.join(aslist) + domain def getdomain(self): """Get the complete domain name from an address.""" @@ -394,6 +413,10 @@ def getdomain(self): elif self.field[self.pos] == '.': self.pos += 1 sdlist.append('.') + elif self.field[self.pos] == '@': + # bpo-34155: Don't parse domains with two `@` like + # `a@malicious.org@important.com`. + return EMPTYSTRING elif self.field[self.pos] in self.atomends: break else: diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index df4649676ae..0d486c90a9c 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -152,11 +152,18 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): mangle_from_ -- a flag that, when True escapes From_ lines in the body of the message by putting a `>' in front of them. This is used when the message is being - serialized by a generator. Default: True. + serialized by a generator. Default: False. message_factory -- the class to use to create new message objects. If the value is None, the default is Message. + verify_generated_headers + -- if true, the generator verifies that each header + they are properly folded, so that a parser won't + treat it as multiple headers, start-of-body, or + part of another header. + This is a check against custom Header & fold() + implementations. """ raise_on_defect = False @@ -165,6 +172,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): max_line_length = 78 mangle_from_ = False message_factory = None + verify_generated_headers = True def handle_defect(self, obj, defect): """Based on policy, either raise defect or call register_defect. @@ -294,12 +302,12 @@ def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): @@ -361,8 +369,12 @@ def _fold(self, name, value, sanitize): # Assume it is a Header-like object. h = value if h is not None: - parts.append(h.encode(linesep=self.linesep, - maxlinelen=self.max_line_length)) + # The Header class interprets a value of None for maxlinelen as the + # default value of 78, as recommended by RFC 5322 section 2.1.1. + maxlinelen = 0 + if self.max_line_length is not None: + maxlinelen = self.max_line_length + parts.append(h.encode(linesep=self.linesep, maxlinelen=maxlinelen)) parts.append(self.linesep) return ''.join(parts) diff --git a/Lib/email/architecture.rst b/Lib/email/architecture.rst index 78572ae63b4..fcd10bde132 100644 --- a/Lib/email/architecture.rst +++ b/Lib/email/architecture.rst @@ -66,7 +66,7 @@ data payloads. Message Lifecycle ----------------- -The general lifecyle of a message is: +The general lifecycle of a message is: Creation A `Message` object can be created by a Parser, or it can be diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index 17f0818f6ca..4cdf22666e3 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -45,7 +45,6 @@ MISC_LEN = 7 - # Helpers def header_length(bytearray): """Return the length of s when it is encoded with base64.""" @@ -57,7 +56,6 @@ def header_length(bytearray): return n - def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with Base64 encoding in a given charset. @@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'): return '=?%s?b?%s?=' % (charset, encoded) - def body_encode(s, maxlinelen=76, eol=NL): r"""Encode a string with base64. @@ -84,7 +81,7 @@ def body_encode(s, maxlinelen=76, eol=NL): in an email. """ if not s: - return s + return "" encvec = [] max_unencoded = maxlinelen * 3 // 4 @@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL): return EMPTYSTRING.join(encvec) - def decode(string): """Decode a raw base64 string, returning a bytes object. diff --git a/Lib/email/charset.py b/Lib/email/charset.py index ee564040c68..043801107b6 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -18,7 +18,6 @@ from email.encoders import encode_7or8bit - # Flags for types of header encodings QP = 1 # Quoted-Printable BASE64 = 2 # Base64 @@ -32,7 +31,6 @@ EMPTYSTRING = '' - # Defaults CHARSETS = { # input header enc body enc output conv @@ -104,7 +102,6 @@ } - # Convenience functions for extending the above mappings def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): """Add character set properties to the global registry. @@ -112,8 +109,8 @@ def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): charset is the input character set, and must be the canonical name of a character set. - Optional header_enc and body_enc is either Charset.QP for - quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for + Optional header_enc and body_enc is either charset.QP for + quoted-printable, charset.BASE64 for base64 encoding, charset.SHORTEST for the shortest of qp or base64 encoding, or None for no encoding. SHORTEST is only valid for header_enc. It describes how message headers and message bodies in the input charset are to be encoded. Default is no @@ -153,7 +150,6 @@ def add_codec(charset, codecname): CODEC_MAP[charset] = codecname - # Convenience function for encoding strings, taking into account # that they might be unknown-8bit (ie: have surrogate-escaped bytes) def _encode(string, codec): @@ -163,7 +159,6 @@ def _encode(string, codec): return string.encode(codec) - class Charset: """Map character sets to their email properties. @@ -185,13 +180,13 @@ class Charset: header_encoding: If the character set must be encoded before it can be used in an email header, this attribute will be set to - Charset.QP (for quoted-printable), Charset.BASE64 (for - base64 encoding), or Charset.SHORTEST for the shortest of + charset.QP (for quoted-printable), charset.BASE64 (for + base64 encoding), or charset.SHORTEST for the shortest of QP or BASE64 encoding. Otherwise, it will be None. body_encoding: Same as header_encoding, but describes the encoding for the mail message's body, which indeed may be different than the - header encoding. Charset.SHORTEST is not allowed for + header encoding. charset.SHORTEST is not allowed for body_encoding. output_charset: Some character sets must be converted before they can be @@ -241,11 +236,9 @@ def __init__(self, input_charset=DEFAULT_CHARSET): self.output_codec = CODEC_MAP.get(self.output_charset, self.output_charset) - def __str__(self): + def __repr__(self): return self.input_charset.lower() - __repr__ = __str__ - def __eq__(self, other): return str(self) == str(other).lower() @@ -348,7 +341,6 @@ def header_encode_lines(self, string, maxlengths): if not lines and not current_line: lines.append(None) else: - separator = (' ' if lines else '') joined_line = EMPTYSTRING.join(current_line) header_bytes = _encode(joined_line, codec) lines.append(encoder(header_bytes)) diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index b904ded94c9..11d1536db27 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -2,6 +2,7 @@ import email.charset import email.message import email.errors +import sys from email import quoprimime class ContentManager: @@ -72,12 +73,14 @@ def get_non_text_content(msg): return msg.get_payload(decode=True) for maintype in 'audio image video application'.split(): raw_data_manager.add_get_handler(maintype, get_non_text_content) +del maintype def get_message_content(msg): return msg.get_payload(0) for subtype in 'rfc822 external-body'.split(): raw_data_manager.add_get_handler('message/'+subtype, get_message_content) +del subtype def get_and_fixup_unknown_message_content(msg): @@ -140,22 +143,23 @@ def _encode_base64(data, max_line_length): def _encode_text(string, charset, cte, policy): + # If max_line_length is 0 or None, there is no limit. + maxlen = policy.max_line_length or sys.maxsize lines = string.encode(charset).splitlines() linesep = policy.linesep.encode('ascii') def embedded_body(lines): return linesep.join(lines) + linesep def normal_body(lines): return b'\n'.join(lines) + b'\n' - if cte==None: + if cte is None: # Use heuristics to decide on the "best" encoding. - try: - return '7bit', normal_body(lines).decode('ascii') - except UnicodeDecodeError: - pass - if (policy.cte_type == '8bit' and - max(len(x) for x in lines) <= policy.max_line_length): - return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') + if max(map(len, lines), default=0) <= maxlen: + try: + return '7bit', normal_body(lines).decode('ascii') + except UnicodeDecodeError: + pass + if policy.cte_type == '8bit': + return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') sniff = embedded_body(lines[:10]) - sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), - policy.max_line_length) + sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), maxlen) sniff_base64 = binascii.b2a_base64(sniff) # This is a little unfair to qp; it includes lineseps, base64 doesn't. if len(sniff_qp) > len(sniff_base64): @@ -170,9 +174,9 @@ def normal_body(lines): return b'\n'.join(lines) + b'\n' data = normal_body(lines).decode('ascii', 'surrogateescape') elif cte == 'quoted-printable': data = quoprimime.body_encode(normal_body(lines).decode('latin-1'), - policy.max_line_length) + maxlen) elif cte == 'base64': - data = _encode_base64(embedded_body(lines), policy.max_line_length) + data = _encode_base64(embedded_body(lines), maxlen) else: raise ValueError("Unknown content transfer encoding {}".format(cte)) return cte, data @@ -238,9 +242,7 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64', data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True) data = data.decode('ascii') elif cte == '7bit': - # Make sure it really is only ASCII. The early warning here seems - # worth the overhead...if you care write your own content manager :). - data.encode('ascii') + data = data.decode('ascii') elif cte in ('8bit', 'binary'): data = data.decode('ascii', 'surrogateescape') msg.set_payload(data) @@ -248,3 +250,4 @@ def set_bytes_content(msg, data, maintype, subtype, cte='base64', _finalize_set(msg, disposition, filename, cid, params) for typ in (bytes, bytearray, memoryview): raw_data_manager.add_set_handler(typ, set_bytes_content) +del typ diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 0a66acb6240..17bd1ab7b19 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -16,7 +16,6 @@ from quopri import encodestring as _encodestring - def _qencode(s): enc = _encodestring(s, quotetabs=True) # Must encode spaces, which quopri.encodestring() doesn't do @@ -34,7 +33,6 @@ def encode_base64(msg): msg['Content-Transfer-Encoding'] = 'base64' - def encode_quopri(msg): """Encode the message's payload in quoted-printable. @@ -46,7 +44,6 @@ def encode_quopri(msg): msg['Content-Transfer-Encoding'] = 'quoted-printable' - def encode_7or8bit(msg): """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" orig = msg.get_payload(decode=True) @@ -64,6 +61,5 @@ def encode_7or8bit(msg): msg['Content-Transfer-Encoding'] = '7bit' - def encode_noop(msg): """Do nothing.""" diff --git a/Lib/email/errors.py b/Lib/email/errors.py index 791239fa6a5..02aa5eced6a 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -29,6 +29,10 @@ class CharsetError(MessageError): """An illegal charset was given.""" +class HeaderWriteError(MessageError): + """Error while writing headers.""" + + # These are parsing defects which the parser was able to work around. class MessageDefect(ValueError): """Base class for a message defect.""" @@ -73,6 +77,9 @@ class InvalidBase64PaddingDefect(MessageDefect): class InvalidBase64CharactersDefect(MessageDefect): """base64 encoded sequence had characters not in base64 alphabet""" +class InvalidBase64LengthDefect(MessageDefect): + """base64 encoded sequence had invalid length (1 mod 4)""" + # These errors are specific to header parsing. class HeaderDefect(MessageDefect): @@ -105,3 +112,6 @@ class NonASCIILocalPartDefect(HeaderDefect): """local_part contains non-ASCII characters""" # This defect only occurs during unicode parsing, not when # parsing messages decoded from binary. + +class InvalidDateDefect(HeaderDefect): + """Header has unparsable or invalid date""" diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 7c07ca86457..bc773f38030 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -32,16 +32,17 @@ NLCRE_bol = re.compile(r'(\r\n|\r|\n)') NLCRE_eol = re.compile(r'(\r\n|\r|\n)\Z') NLCRE_crack = re.compile(r'(\r\n|\r|\n)') -# RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character +# RFC 5322 section 3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character # except controls, SP, and ":". headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') EMPTYSTRING = '' NL = '\n' +boundaryendRE = re.compile( + r'(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') NeedMoreData = object() - class BufferedSubFile(object): """A file-ish object that can have new data loaded into it. @@ -132,7 +133,6 @@ def __next__(self): return line - class FeedParser: """A feed-style parser of email.""" @@ -189,7 +189,7 @@ def close(self): assert not self._msgstack # Look for final set of defects if root.get_content_maintype() == 'multipart' \ - and not root.is_multipart(): + and not root.is_multipart() and not self._headersonly: defect = errors.MultipartInvariantViolationDefect() self.policy.handle_defect(root, defect) return root @@ -266,7 +266,7 @@ def _parsegen(self): yield NeedMoreData continue break - msg = self._pop_message() + self._pop_message() # We need to pop the EOF matcher in order to tell if we're at # the end of the current file, not the end of the last block # of message headers. @@ -294,7 +294,7 @@ def _parsegen(self): return if self._cur.get_content_maintype() == 'message': # The message claims to be a message/* type, then what follows is - # another RFC 2822 message. + # another RFC 5322 message. for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData @@ -320,7 +320,7 @@ def _parsegen(self): self._cur.set_payload(EMPTYSTRING.join(lines)) return # Make sure a valid content type was specified per RFC 2045:6.4. - if (self._cur.get('content-transfer-encoding', '8bit').lower() + if (str(self._cur.get('content-transfer-encoding', '8bit')).lower() not in ('7bit', '8bit', 'binary')): defect = errors.InvalidMultipartContentTransferEncodingDefect() self.policy.handle_defect(self._cur, defect) @@ -329,9 +329,10 @@ def _parsegen(self): # this onto the input stream until we've scanned past the # preamble. separator = '--' + boundary - boundaryre = re.compile( - '(?P' + re.escape(separator) + - r')(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') + def boundarymatch(line): + if not line.startswith(separator): + return None + return boundaryendRE.match(line, len(separator)) capturing_preamble = True preamble = [] linesep = False @@ -343,7 +344,7 @@ def _parsegen(self): continue if line == '': break - mo = boundaryre.match(line) + mo = boundarymatch(line) if mo: # If we're looking at the end boundary, we're done with # this multipart. If there was a newline at the end of @@ -375,13 +376,13 @@ def _parsegen(self): if line is NeedMoreData: yield NeedMoreData continue - mo = boundaryre.match(line) + mo = boundarymatch(line) if not mo: self._input.unreadline(line) break # Recurse to parse this subpart; the input stream points # at the subpart's first line. - self._input.push_eof_matcher(boundaryre.match) + self._input.push_eof_matcher(boundarymatch) for retval in self._parsegen(): if retval is NeedMoreData: yield NeedMoreData diff --git a/Lib/email/generator.py b/Lib/email/generator.py index ae670c2353c..ce94f5c56fe 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -14,15 +14,16 @@ from copy import deepcopy from io import StringIO, BytesIO from email.utils import _has_surrogates +from email.errors import HeaderWriteError UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. NLCRE = re.compile(r'\r\n|\r|\n') fcre = re.compile(r'^From ', re.MULTILINE) +NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]') - class Generator: """Generates output from a Message object tree. @@ -49,7 +50,7 @@ def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, expanded to 8 spaces) than maxheaderlen, the header will split as defined in the Header class. Set maxheaderlen to zero to disable header wrapping. The default is 78, as recommended (but not required) - by RFC 2822. + by RFC 5322 section 2.1.1. The policy keyword specifies a policy object that controls a number of aspects of the generator's operation. If no policy is specified, @@ -170,7 +171,7 @@ def _write(self, msg): # parameter. # # The way we do this, so as to make the _handle_*() methods simpler, - # is to cache any subpart writes into a buffer. The we write the + # is to cache any subpart writes into a buffer. Then we write the # headers and the buffer contents. That way, subpart handlers can # Do The Right Thing, and can still modify the Content-Type: header if # necessary. @@ -186,7 +187,11 @@ def _write(self, msg): # If we munged the cte, copy the message again and re-fix the CTE. if munge_cte: msg = deepcopy(msg) - msg.replace_header('content-transfer-encoding', munge_cte[0]) + # Preserve the header order if the CTE header already exists. + if msg.get('content-transfer-encoding') is None: + msg['Content-Transfer-Encoding'] = munge_cte[0] + else: + msg.replace_header('content-transfer-encoding', munge_cte[0]) msg.replace_header('content-type', munge_cte[1]) # Write the headers. First we see if the message object wants to # handle that itself. If not, we'll do it generically. @@ -219,7 +224,16 @@ def _dispatch(self, msg): def _write_headers(self, msg): for h, v in msg.raw_items(): - self.write(self.policy.fold(h, v)) + folded = self.policy.fold(h, v) + if self.policy.verify_generated_headers: + linesep = self.policy.linesep + if not folded.endswith(self.policy.linesep): + raise HeaderWriteError( + f'folded header does not end with {linesep!r}: {folded!r}') + if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)): + raise HeaderWriteError( + f'folded header contains newline: {folded!r}') + self.write(folded) # A blank line always separates headers from body self.write(self._NL) @@ -240,7 +254,7 @@ def _handle_text(self, msg): # existing message. msg = deepcopy(msg) del msg['content-transfer-encoding'] - msg.set_payload(payload, charset) + msg.set_payload(msg._payload, charset) payload = msg.get_payload() self._munge_cte = (msg['content-transfer-encoding'], msg['content-type']) @@ -388,7 +402,7 @@ def _make_boundary(cls, text=None): def _compile_re(cls, s, flags): return re.compile(s, flags) - + class BytesGenerator(Generator): """Generates a bytes version of a Message object tree. @@ -439,7 +453,6 @@ def _compile_re(cls, s, flags): return re.compile(s.encode('ascii'), flags) - _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): @@ -499,7 +512,6 @@ def _dispatch(self, msg): }, file=self) - # Helper used by Generator._make_boundary _width = len(repr(sys.maxsize-1)) _fmt = '%%0%dd' % _width diff --git a/Lib/email/header.py b/Lib/email/header.py index c7b2dd9f310..a0aadb97ca6 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -36,11 +36,11 @@ =\? # literal =? (?P[^?]*?) # non-greedy up to the next ? is the charset \? # literal ? - (?P[qb]) # either a "q" or a "b", case insensitive + (?P[qQbB]) # either a "q" or a "b", case insensitive \? # literal ? (?P.*?) # non-greedy up to the next ?= is the encoded string \?= # literal ?= - ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE) + ''', re.VERBOSE | re.MULTILINE) # Field name regexp, including trailing colon, but not separating whitespace, # according to RFC 2822. Character range is from tilde to exclamation mark. @@ -52,25 +52,29 @@ _embedded_header = re.compile(r'\n[^ \t]+:') - # Helpers _max_append = email.quoprimime._max_append - def decode_header(header): """Decode a message header value without converting charset. - Returns a list of (string, charset) pairs containing each of the decoded - parts of the header. Charset is None for non-encoded parts of the header, - otherwise a lower-case string containing the name of the character set - specified in the encoded string. + For historical reasons, this function may return either: + + 1. A list of length 1 containing a pair (str, None). + 2. A list of (bytes, charset) pairs containing each of the decoded + parts of the header. Charset is None for non-encoded parts of the header, + otherwise a lower-case string containing the name of the character set + specified in the encoded string. header may be a string that may or may not contain RFC2047 encoded words, or it may be a Header object. An email.errors.HeaderParseError may be raised when certain decoding error occurs (e.g. a base64 decoding exception). + + This function exists for backwards compatibility only. For new code, we + recommend using email.headerregistry.HeaderRegistry instead. """ # If it is a Header object, we can just return the encoded chunks. if hasattr(header, '_chunks'): @@ -152,7 +156,6 @@ def decode_header(header): return collapsed - def make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' '): """Create a Header from a sequence of pairs as returned by decode_header() @@ -164,6 +167,9 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, This function takes one of those sequence of pairs and returns a Header instance. Optional maxlinelen, header_name, and continuation_ws are as in the Header constructor. + + This function exists for backwards compatibility only, and is not + recommended for use in new code. """ h = Header(maxlinelen=maxlinelen, header_name=header_name, continuation_ws=continuation_ws) @@ -175,7 +181,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, return h - class Header: def __init__(self, s=None, charset=None, maxlinelen=None, header_name=None, @@ -409,7 +414,6 @@ def _normalize(self): self._chunks = chunks - class _ValueFormatter: def __init__(self, headerlen, maxlen, continuation_ws, splitchars): self._maxlen = maxlen @@ -431,7 +435,7 @@ def newline(self): if end_of_line != (' ', ''): self._current_line.push(*end_of_line) if len(self._current_line) > 0: - if self._current_line.is_onlyws(): + if self._current_line.is_onlyws() and self._lines: self._lines[-1] += str(self._current_line) else: self._lines.append(str(self._current_line)) diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 0fc2231e5cb..543141dc427 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -2,10 +2,6 @@ This module provides an implementation of the HeaderRegistry API. The implementation is designed to flexibly follow RFC5322 rules. - -Eventually HeaderRegistry will be a public API, but it isn't yet, -and will probably change some before that happens. - """ from types import MappingProxyType @@ -31,6 +27,11 @@ def __init__(self, display_name='', username='', domain='', addr_spec=None): without any Content Transfer Encoding. """ + + inputs = ''.join(filter(None, (display_name, username, domain, addr_spec))) + if '\r' in inputs or '\n' in inputs: + raise ValueError("invalid arguments; address parts cannot contain CR or LF") + # This clause with its potential 'raise' may only happen when an # application program creates an Address object using an addr_spec # keyword. The email library code itself must always supply username @@ -69,11 +70,9 @@ def addr_spec(self): """The addr_spec (username@domain) portion of the address, quoted according to RFC 5322 rules, but with no Content Transfer Encoding. """ - nameset = set(self.username) - if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS): - lp = parser.quote_string(self.username) - else: - lp = self.username + lp = self.username + if not parser.DOT_ATOM_ENDS.isdisjoint(lp): + lp = parser.quote_string(lp) if self.domain: return lp + '@' + self.domain if not lp: @@ -86,19 +85,17 @@ def __repr__(self): self.display_name, self.username, self.domain) def __str__(self): - nameset = set(self.display_name) - if len(nameset) > len(nameset-parser.SPECIALS): - disp = parser.quote_string(self.display_name) - else: - disp = self.display_name + disp = self.display_name + if not parser.SPECIALS.isdisjoint(disp): + disp = parser.quote_string(disp) if disp: addr_spec = '' if self.addr_spec=='<>' else self.addr_spec return "{} <{}>".format(disp, addr_spec) return self.addr_spec def __eq__(self, other): - if type(other) != type(self): - return False + if not isinstance(other, Address): + return NotImplemented return (self.display_name == other.display_name and self.username == other.username and self.domain == other.domain) @@ -141,17 +138,15 @@ def __str__(self): if self.display_name is None and len(self.addresses)==1: return str(self.addresses[0]) disp = self.display_name - if disp is not None: - nameset = set(disp) - if len(nameset) > len(nameset-parser.SPECIALS): - disp = parser.quote_string(disp) + if disp is not None and not parser.SPECIALS.isdisjoint(disp): + disp = parser.quote_string(disp) adrstr = ", ".join(str(x) for x in self.addresses) adrstr = ' ' + adrstr if adrstr else adrstr return "{}:{};".format(disp, adrstr) def __eq__(self, other): - if type(other) != type(self): - return False + if not isinstance(other, Group): + return NotImplemented return (self.display_name == other.display_name and self.addresses == other.addresses) @@ -223,7 +218,7 @@ def __reduce__(self): self.__class__.__bases__, str(self), ), - self.__dict__) + self.__getstate__()) @classmethod def _reconstruct(cls, value): @@ -245,13 +240,16 @@ def fold(self, *, policy): the header name and the ': ' separator. """ - # At some point we need to only put fws here if it was in the source. + # At some point we need to put fws here if it was in the source. header = parser.Header([ parser.HeaderLabel([ parser.ValueTerminal(self.name, 'header-name'), parser.ValueTerminal(':', 'header-sep')]), - parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), - self._parse_tree]) + ]) + if self._parse_tree: + header.append( + parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')])) + header.append(self._parse_tree) return header.fold(policy=policy) @@ -300,7 +298,14 @@ def parse(cls, value, kwds): kwds['parse_tree'] = parser.TokenList() return if isinstance(value, str): - value = utils.parsedate_to_datetime(value) + kwds['decoded'] = value + try: + value = utils.parsedate_to_datetime(value) + except ValueError: + kwds['defects'].append(errors.InvalidDateDefect('Invalid date value or format')) + kwds['datetime'] = None + kwds['parse_tree'] = parser.TokenList() + return kwds['datetime'] = value kwds['decoded'] = utils.format_datetime(kwds['datetime']) kwds['parse_tree'] = cls.value_parser(kwds['decoded']) @@ -369,8 +374,8 @@ def groups(self): @property def addresses(self): if self._addresses is None: - self._addresses = tuple([address for group in self._groups - for address in group.addresses]) + self._addresses = tuple(address for group in self._groups + for address in group.addresses) return self._addresses @@ -517,6 +522,18 @@ def cte(self): return self._cte +class MessageIDHeader: + + max_count = 1 + value_parser = staticmethod(parser.parse_message_id) + + @classmethod + def parse(cls, value, kwds): + kwds['parse_tree'] = parse_tree = cls.value_parser(value) + kwds['decoded'] = str(parse_tree) + kwds['defects'].extend(parse_tree.all_defects) + + # The header factory # _default_header_map = { @@ -539,6 +556,7 @@ def cte(self): 'content-type': ContentTypeHeader, 'content-disposition': ContentDispositionHeader, 'content-transfer-encoding': ContentTransferEncodingHeader, + 'message-id': MessageIDHeader, } class HeaderRegistry: diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index b5502ee9752..3410935e38f 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -15,7 +15,6 @@ from io import StringIO - # This function will become a method of the Message class def walk(self): """Walk over the message tree, yielding each subpart. @@ -29,7 +28,6 @@ def walk(self): yield from subpart.walk() - # These two functions are imported into the Iterators.py interface module. def body_line_iterator(msg, decode=False): """Iterate over the parts, returning string payloads line-by-line. @@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None): yield subpart - def _structure(msg, fp=None, level=0, include_default=False): """A handy debugging aid""" if fp is None: diff --git a/Lib/email/message.py b/Lib/email/message.py index b6512f2198a..80f01d66a33 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -6,15 +6,15 @@ __all__ = ['Message', 'EmailMessage'] +import binascii import re -import uu import quopri from io import BytesIO, StringIO # Intrapackage imports from email import utils from email import errors -from email._policybase import Policy, compat32 +from email._policybase import compat32 from email import charset as _charset from email._encoded_words import decode_b Charset = _charset.Charset @@ -35,7 +35,7 @@ def _splitparam(param): if not sep: return a.strip(), None return a.strip(), b.strip() - + def _formatparam(param, value=None, quote=True): """Convenience function to format and return a key=value pair. @@ -74,19 +74,25 @@ def _parseparam(s): # RDM This might be a Header, so for now stringify it. s = ';' + str(s) plist = [] - while s[:1] == ';': - s = s[1:] - end = s.find(';') - while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: - end = s.find(';', end + 1) + start = 0 + while s.find(';', start) == start: + start += 1 + end = s.find(';', start) + ind, diff = start, 0 + while end > 0: + diff += s.count('"', ind, end) - s.count('\\"', ind, end) + if diff % 2 == 0: + break + end, ind = ind, s.find(';', end + 1) if end < 0: end = len(s) - f = s[:end] - if '=' in f: - i = f.index('=') - f = f[:i].strip().lower() + '=' + f[i+1:].strip() + i = s.find('=', start, end) + if i == -1: + f = s[start:end] + else: + f = s[start:i].rstrip().lower() + '=' + s[i+1:end].lstrip() plist.append(f.strip()) - s = s[end:] + start = end return plist @@ -101,11 +107,41 @@ def _unquotevalue(value): return utils.unquote(value) - +def _decode_uu(encoded): + """Decode uuencoded data.""" + decoded_lines = [] + encoded_lines_iter = iter(encoded.splitlines()) + for line in encoded_lines_iter: + if line.startswith(b"begin "): + mode, _, path = line.removeprefix(b"begin ").partition(b" ") + try: + int(mode, base=8) + except ValueError: + continue + else: + break + else: + raise ValueError("`begin` line not found") + for line in encoded_lines_iter: + if not line: + raise ValueError("Truncated input") + elif line.strip(b' \t\r\n\f') == b'end': + break + try: + decoded_line = binascii.a2b_uu(line) + except binascii.Error: + # Workaround for broken uuencoders by /Fredrik Lundh + nbytes = (((line[0]-32) & 63) * 4 + 5) // 3 + decoded_line = binascii.a2b_uu(line[:nbytes]) + decoded_lines.append(decoded_line) + + return b''.join(decoded_lines) + + class Message: """Basic message object. - A message object is defined as something that has a bunch of RFC 2822 + A message object is defined as something that has a bunch of RFC 5322 headers and a payload. It may optionally have an envelope header (a.k.a. Unix-From or From_ header). If the message is a container (i.e. a multipart or a message/rfc822), then the payload is a list of Message @@ -141,7 +177,7 @@ def as_string(self, unixfrom=False, maxheaderlen=0, policy=None): header. For backward compatibility reasons, if maxheaderlen is not specified it defaults to 0, so you must override it explicitly if you want a different maxheaderlen. 'policy' is passed to the - Generator instance used to serialize the mesasge; if it is not + Generator instance used to serialize the message; if it is not specified the policy associated with the message instance is used. If the message object contains binary data that is not encoded @@ -256,28 +292,35 @@ def get_payload(self, i=None, decode=False): if i is not None and not isinstance(self._payload, list): raise TypeError('Expected list, got %s' % type(self._payload)) payload = self._payload - # cte might be a Header, so for now stringify it. - cte = str(self.get('content-transfer-encoding', '')).lower() + cte = self.get('content-transfer-encoding', '') + if hasattr(cte, 'cte'): + cte = cte.cte + else: + # cte might be a Header, so for now stringify it. + cte = str(cte).strip().lower() # payload may be bytes here. - if isinstance(payload, str): - if utils._has_surrogates(payload): - bpayload = payload.encode('ascii', 'surrogateescape') - if not decode: + if not decode: + if isinstance(payload, str) and utils._has_surrogates(payload): + try: + bpayload = payload.encode('ascii', 'surrogateescape') try: - payload = bpayload.decode(self.get_param('charset', 'ascii'), 'replace') + payload = bpayload.decode(self.get_content_charset('ascii'), 'replace') except LookupError: payload = bpayload.decode('ascii', 'replace') - elif decode: - try: - bpayload = payload.encode('ascii') - except UnicodeError: - # This won't happen for RFC compliant messages (messages - # containing only ASCII code points in the unicode input). - # If it does happen, turn the string into bytes in a way - # guaranteed not to fail. - bpayload = payload.encode('raw-unicode-escape') - if not decode: + except UnicodeEncodeError: + pass return payload + if isinstance(payload, str): + try: + bpayload = payload.encode('ascii', 'surrogateescape') + except UnicodeEncodeError: + # This won't happen for RFC compliant messages (messages + # containing only ASCII code points in the unicode input). + # If it does happen, turn the string into bytes in a way + # guaranteed not to fail. + bpayload = payload.encode('raw-unicode-escape') + else: + bpayload = payload if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': @@ -288,13 +331,10 @@ def get_payload(self, i=None, decode=False): self.policy.handle_defect(self, defect) return value elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): - in_file = BytesIO(bpayload) - out_file = BytesIO() try: - uu.decode(in_file, out_file, quiet=True) - return out_file.getvalue() - except uu.Error: - # Some decoding problem + return _decode_uu(bpayload) + except ValueError: + # Some decoding problem. return bpayload if isinstance(payload, str): return bpayload @@ -312,7 +352,7 @@ def set_payload(self, payload, charset=None): return if not isinstance(charset, Charset): charset = Charset(charset) - payload = payload.encode(charset.output_charset) + payload = payload.encode(charset.output_charset, 'surrogateescape') if hasattr(payload, 'decode'): self._payload = payload.decode('ascii', 'surrogateescape') else: @@ -421,7 +461,11 @@ def __delitem__(self, name): self._headers = newheaders def __contains__(self, name): - return name.lower() in [k.lower() for k, v in self._headers] + name_lower = name.lower() + for k, v in self._headers: + if name_lower == k.lower(): + return True + return False def __iter__(self): for field, value in self._headers: @@ -528,7 +572,7 @@ def add_header(self, _name, _value, **_params): msg.add_header('content-disposition', 'attachment', filename='bud.gif') msg.add_header('content-disposition', 'attachment', - filename=('utf-8', '', Fußballer.ppt')) + filename=('utf-8', '', 'Fußballer.ppt')) msg.add_header('content-disposition', 'attachment', filename='Fußballer.ppt')) """ @@ -948,7 +992,7 @@ def __init__(self, policy=None): if policy is None: from email.policy import default policy = default - Message.__init__(self, policy) + super().__init__(policy) def as_string(self, unixfrom=False, maxheaderlen=None, policy=None): @@ -958,14 +1002,14 @@ def as_string(self, unixfrom=False, maxheaderlen=None, policy=None): header. maxheaderlen is retained for backward compatibility with the base Message class, but defaults to None, meaning that the policy value for max_line_length controls the header maximum length. 'policy' is - passed to the Generator instance used to serialize the mesasge; if it + passed to the Generator instance used to serialize the message; if it is not specified the policy associated with the message instance is used. """ policy = self.policy if policy is None else policy if maxheaderlen is None: maxheaderlen = policy.max_line_length - return super().as_string(maxheaderlen=maxheaderlen, policy=policy) + return super().as_string(unixfrom, maxheaderlen, policy) def __str__(self): return self.as_string(policy=self.policy.clone(utf8=True)) @@ -982,7 +1026,7 @@ def _find_body(self, part, preferencelist): if subtype in preferencelist: yield (preferencelist.index(subtype), part) return - if maintype != 'multipart': + if maintype != 'multipart' or not self.is_multipart(): return if subtype != 'related': for subpart in part.iter_parts(): @@ -1041,7 +1085,16 @@ def iter_attachments(self): maintype, subtype = self.get_content_type().split('/') if maintype != 'multipart' or subtype == 'alternative': return - parts = self.get_payload().copy() + payload = self.get_payload() + # Certain malformed messages can have content type set to `multipart/*` + # but still have single part body, in which case payload.copy() can + # fail with AttributeError. + try: + parts = payload.copy() + except AttributeError: + # payload is not a list, it is most probably a string. + return + if maintype == 'multipart' and subtype == 'related': # For related, we treat everything but the root as an attachment. # The root may be indicated by 'start'; if there's no start or we @@ -1078,7 +1131,7 @@ def iter_parts(self): Return an empty iterator for a non-multipart. """ - if self.get_content_maintype() == 'multipart': + if self.is_multipart(): yield from self.get_payload() def get_content(self, *args, content_manager=None, **kw): diff --git a/Lib/email/mime/application.py b/Lib/email/mime/application.py index 6877e554e10..f67cbad3f03 100644 --- a/Lib/email/mime/application.py +++ b/Lib/email/mime/application.py @@ -17,7 +17,7 @@ def __init__(self, _data, _subtype='octet-stream', _encoder=encoders.encode_base64, *, policy=None, **_params): """Create an application/* type MIME document. - _data is a string containing the raw application data. + _data contains the bytes for the raw application data. _subtype is the MIME content type subtype, defaulting to 'octet-stream'. diff --git a/Lib/email/mime/audio.py b/Lib/email/mime/audio.py index 4bcd7b224a8..aa0c4905cbb 100644 --- a/Lib/email/mime/audio.py +++ b/Lib/email/mime/audio.py @@ -6,39 +6,10 @@ __all__ = ['MIMEAudio'] -import sndhdr - -from io import BytesIO from email import encoders from email.mime.nonmultipart import MIMENonMultipart - -_sndhdr_MIMEmap = {'au' : 'basic', - 'wav' :'x-wav', - 'aiff':'x-aiff', - 'aifc':'x-aiff', - } - -# There are others in sndhdr that don't have MIME types. :( -# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma?? -def _whatsnd(data): - """Try to identify a sound file type. - - sndhdr.what() has a pretty cruddy interface, unfortunately. This is why - we re-do it here. It would be easier to reverse engineer the Unix 'file' - command and use the standard 'magic' file, as shipped with a modern Unix. - """ - hdr = data[:512] - fakefile = BytesIO(hdr) - for testfn in sndhdr.tests: - res = testfn(hdr, fakefile) - if res is not None: - return _sndhdr_MIMEmap.get(res[0]) - return None - - - class MIMEAudio(MIMENonMultipart): """Class for generating audio/* MIME documents.""" @@ -46,8 +17,8 @@ def __init__(self, _audiodata, _subtype=None, _encoder=encoders.encode_base64, *, policy=None, **_params): """Create an audio/* type MIME document. - _audiodata is a string containing the raw audio data. If this data - can be decoded by the standard Python `sndhdr' module, then the + _audiodata contains the bytes for the raw audio data. If this data + can be decoded as au, wav, aiff, or aifc, then the subtype will be automatically included in the Content-Type header. Otherwise, you can specify the specific audio subtype via the _subtype parameter. If _subtype is not given, and no subtype can be @@ -65,10 +36,62 @@ def __init__(self, _audiodata, _subtype=None, header. """ if _subtype is None: - _subtype = _whatsnd(_audiodata) + _subtype = _what(_audiodata) if _subtype is None: raise TypeError('Could not find audio MIME subtype') MIMENonMultipart.__init__(self, 'audio', _subtype, policy=policy, **_params) self.set_payload(_audiodata) _encoder(self) + + +_rules = [] + + +# Originally from the sndhdr module. +# +# There are others in sndhdr that don't have MIME types. :( +# Additional ones to be added to sndhdr? midi, mp3, realaudio, wma?? +def _what(data): + # Try to identify a sound file type. + # + # sndhdr.what() had a pretty cruddy interface, unfortunately. This is why + # we re-do it here. It would be easier to reverse engineer the Unix 'file' + # command and use the standard 'magic' file, as shipped with a modern Unix. + for testfn in _rules: + if res := testfn(data): + return res + else: + return None + + +def rule(rulefunc): + _rules.append(rulefunc) + return rulefunc + + +@rule +def _aiff(h): + if not h.startswith(b'FORM'): + return None + if h[8:12] in {b'AIFC', b'AIFF'}: + return 'x-aiff' + else: + return None + + +@rule +def _au(h): + if h.startswith(b'.snd'): + return 'basic' + else: + return None + + +@rule +def _wav(h): + # 'RIFF' 'WAVE' 'fmt ' + if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ': + return None + else: + return "x-wav" diff --git a/Lib/email/mime/base.py b/Lib/email/mime/base.py index 1a3f9b51f6c..f601f621cec 100644 --- a/Lib/email/mime/base.py +++ b/Lib/email/mime/base.py @@ -11,7 +11,6 @@ from email import message - class MIMEBase(message.Message): """Base class for MIME specializations.""" diff --git a/Lib/email/mime/image.py b/Lib/email/mime/image.py index 92724643cde..4b7f2f9cbad 100644 --- a/Lib/email/mime/image.py +++ b/Lib/email/mime/image.py @@ -6,13 +6,10 @@ __all__ = ['MIMEImage'] -import imghdr - from email import encoders from email.mime.nonmultipart import MIMENonMultipart - class MIMEImage(MIMENonMultipart): """Class for generating image/* type MIME documents.""" @@ -20,11 +17,11 @@ def __init__(self, _imagedata, _subtype=None, _encoder=encoders.encode_base64, *, policy=None, **_params): """Create an image/* type MIME document. - _imagedata is a string containing the raw image data. If this data - can be decoded by the standard Python `imghdr' module, then the - subtype will be automatically included in the Content-Type header. - Otherwise, you can specify the specific image subtype via the _subtype - parameter. + _imagedata contains the bytes for the raw image data. If the data + type can be detected (jpeg, png, gif, tiff, rgb, pbm, pgm, ppm, + rast, xbm, bmp, webp, and exr attempted), then the subtype will be + automatically included in the Content-Type header. Otherwise, you can + specify the specific image subtype via the _subtype parameter. _encoder is a function which will perform the actual encoding for transport of the image data. It takes one argument, which is this @@ -37,11 +34,119 @@ def __init__(self, _imagedata, _subtype=None, constructor, which turns them into parameters on the Content-Type header. """ - if _subtype is None: - _subtype = imghdr.what(None, _imagedata) + _subtype = _what(_imagedata) if _subtype is None else _subtype if _subtype is None: raise TypeError('Could not guess image MIME subtype') MIMENonMultipart.__init__(self, 'image', _subtype, policy=policy, **_params) self.set_payload(_imagedata) _encoder(self) + + +_rules = [] + + +# Originally from the imghdr module. +def _what(data): + for rule in _rules: + if res := rule(data): + return res + else: + return None + + +def rule(rulefunc): + _rules.append(rulefunc) + return rulefunc + + +@rule +def _jpeg(h): + """JPEG data with JFIF or Exif markers; and raw JPEG""" + if h[6:10] in (b'JFIF', b'Exif'): + return 'jpeg' + elif h[:4] == b'\xff\xd8\xff\xdb': + return 'jpeg' + + +@rule +def _png(h): + if h.startswith(b'\211PNG\r\n\032\n'): + return 'png' + + +@rule +def _gif(h): + """GIF ('87 and '89 variants)""" + if h[:6] in (b'GIF87a', b'GIF89a'): + return 'gif' + + +@rule +def _tiff(h): + """TIFF (can be in Motorola or Intel byte order)""" + if h[:2] in (b'MM', b'II'): + return 'tiff' + + +@rule +def _rgb(h): + """SGI image library""" + if h.startswith(b'\001\332'): + return 'rgb' + + +@rule +def _pbm(h): + """PBM (portable bitmap)""" + if len(h) >= 3 and \ + h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r': + return 'pbm' + + +@rule +def _pgm(h): + """PGM (portable graymap)""" + if len(h) >= 3 and \ + h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r': + return 'pgm' + + +@rule +def _ppm(h): + """PPM (portable pixmap)""" + if len(h) >= 3 and \ + h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r': + return 'ppm' + + +@rule +def _rast(h): + """Sun raster file""" + if h.startswith(b'\x59\xA6\x6A\x95'): + return 'rast' + + +@rule +def _xbm(h): + """X bitmap (X10 or X11)""" + if h.startswith(b'#define '): + return 'xbm' + + +@rule +def _bmp(h): + if h.startswith(b'BM'): + return 'bmp' + + +@rule +def _webp(h): + if h.startswith(b'RIFF') and h[8:12] == b'WEBP': + return 'webp' + + +@rule +def _exr(h): + if h.startswith(b'\x76\x2f\x31\x01'): + return 'exr' diff --git a/Lib/email/mime/message.py b/Lib/email/mime/message.py index 07e4f2d1196..61836b5a786 100644 --- a/Lib/email/mime/message.py +++ b/Lib/email/mime/message.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEMessage(MIMENonMultipart): """Class representing message/* MIME documents.""" diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 2d3f288810d..94d81c771a4 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -9,7 +9,6 @@ from email.mime.base import MIMEBase - class MIMEMultipart(MIMEBase): """Base class for MIME multipart/* type messages.""" diff --git a/Lib/email/mime/nonmultipart.py b/Lib/email/mime/nonmultipart.py index e1f51968b59..a41386eb148 100644 --- a/Lib/email/mime/nonmultipart.py +++ b/Lib/email/mime/nonmultipart.py @@ -10,7 +10,6 @@ from email.mime.base import MIMEBase - class MIMENonMultipart(MIMEBase): """Base class for MIME non-multipart type messages.""" diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index 35b44238300..7672b789138 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -6,11 +6,9 @@ __all__ = ['MIMEText'] -from email.charset import Charset from email.mime.nonmultipart import MIMENonMultipart - class MIMEText(MIMENonMultipart): """Class for generating text/* type MIME documents.""" @@ -37,6 +35,6 @@ def __init__(self, _text, _subtype='plain', _charset=None, *, policy=None): _charset = 'utf-8' MIMENonMultipart.__init__(self, 'text', _subtype, policy=policy, - **{'charset': str(_charset)}) + charset=str(_charset)) self.set_payload(_text, _charset) diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 555b1725606..e3003118ce1 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -2,7 +2,7 @@ # Author: Barry Warsaw, Thomas Wouters, Anthony Baxter # Contact: email-sig@python.org -"""A parser of RFC 2822 and MIME email messages.""" +"""A parser of RFC 5322 and MIME email messages.""" __all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser', 'FeedParser', 'BytesFeedParser'] @@ -13,17 +13,16 @@ from email._policybase import compat32 - class Parser: def __init__(self, _class=None, *, policy=compat32): - """Parser of RFC 2822 and MIME email messages. + """Parser of RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The string must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + The string must be formatted as a block of RFC 5322 headers and header + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the string or by a blank line. @@ -50,10 +49,7 @@ def parse(self, fp, headersonly=False): feedparser = FeedParser(self._class, policy=self.policy) if headersonly: feedparser._set_headersonly() - while True: - data = fp.read(8192) - if not data: - break + while data := fp.read(8192): feedparser.feed(data) return feedparser.close() @@ -68,7 +64,6 @@ def parsestr(self, text, headersonly=False): return self.parse(StringIO(text), headersonly=headersonly) - class HeaderParser(Parser): def parse(self, fp, headersonly=True): return Parser.parse(self, fp, True) @@ -76,18 +71,18 @@ def parse(self, fp, headersonly=True): def parsestr(self, text, headersonly=True): return Parser.parsestr(self, text, True) - + class BytesParser: def __init__(self, *args, **kw): - """Parser of binary RFC 2822 and MIME email messages. + """Parser of binary RFC 5322 and MIME email messages. Creates an in-memory object tree representing the email message, which can then be manipulated and turned over to a Generator to return the textual representation of the message. - The input must be formatted as a block of RFC 2822 headers and header - continuation lines, optionally preceded by a `Unix-from' header. The + The input must be formatted as a block of RFC 5322 headers and header + continuation lines, optionally preceded by a 'Unix-from' header. The header block is terminated either by the end of the input or by a blank line. diff --git a/Lib/email/policy.py b/Lib/email/policy.py index 5131311ac5e..6e109b65011 100644 --- a/Lib/email/policy.py +++ b/Lib/email/policy.py @@ -3,6 +3,7 @@ """ import re +import sys from email._policybase import Policy, Compat32, compat32, _extend_docstrings from email.utils import _has_surrogates from email.headerregistry import HeaderRegistry as HeaderRegistry @@ -20,7 +21,7 @@ 'HTTP', ] -linesep_splitter = re.compile(r'\n|\r') +linesep_splitter = re.compile(r'\n|\r\n?') @_extend_docstrings class EmailPolicy(Policy): @@ -118,13 +119,13 @@ def header_source_parse(self, sourcelines): """+ The name is parsed as everything up to the ':' and returned unmodified. The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and + remainder of the first line joined with all subsequent lines, and stripping any trailing carriage return or linefeed characters. (This is the same as Compat32). """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = ''.join((value, *sourcelines[1:])).lstrip(' \t\r\n') return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): @@ -203,14 +204,22 @@ def fold_binary(self, name, value): def _fold(self, name, value, refold_binary=False): if hasattr(value, 'name'): return value.fold(policy=self) - maxlen = self.max_line_length if self.max_line_length else float('inf') - lines = value.splitlines() + maxlen = self.max_line_length if self.max_line_length else sys.maxsize + # We can't use splitlines here because it splits on more than \r and \n. + lines = linesep_splitter.split(value) refold = (self.refold_source == 'all' or self.refold_source == 'long' and (lines and len(lines[0])+len(name)+2 > maxlen or any(len(x) > maxlen for x in lines[1:]))) - if refold or refold_binary and _has_surrogates(value): + + if not refold: + if not self.utf8: + refold = not value.isascii() + elif refold_binary: + refold = _has_surrogates(value) + if refold: return self.header_factory(name, ''.join(lines)).fold(policy=self) + return name + ': ' + self.linesep.join(lines) + self.linesep diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py index c543eb59ae7..27fcbb5a26e 100644 --- a/Lib/email/quoprimime.py +++ b/Lib/email/quoprimime.py @@ -148,6 +148,7 @@ def header_encode(header_bytes, charset='iso-8859-1'): _QUOPRI_BODY_ENCODE_MAP = _QUOPRI_BODY_MAP[:] for c in b'\r\n': _QUOPRI_BODY_ENCODE_MAP[c] = chr(c) +del c def body_encode(body, maxlinelen=76, eol=NL): """Encode with quoted-printable, wrapping at maxlinelen characters. @@ -173,7 +174,7 @@ def body_encode(body, maxlinelen=76, eol=NL): if not body: return body - # quote speacial characters + # quote special characters body = body.translate(_QUOPRI_BODY_ENCODE_MAP) soft_break = '=' + eol diff --git a/Lib/email/utils.py b/Lib/email/utils.py index a759d23308d..e4d35f06abc 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -25,8 +25,6 @@ import os import re import time -import random -import socket import datetime import urllib.parse @@ -36,9 +34,6 @@ from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz -# Intrapackage imports -from email.charset import Charset - COMMASPACE = ', ' EMPTYSTRING = '' UEMPTYSTRING = '' @@ -48,11 +43,12 @@ specialsre = re.compile(r'[][\\()<>@,:;".]') escapesre = re.compile(r'[\\"]') + def _has_surrogates(s): - """Return True if s contains surrogate-escaped binary data.""" + """Return True if s may contain surrogate-escaped binary data.""" # This check is based on the fact that unless there are surrogates, utf8 # (Python's default encoding) can encode any string. This is the fastest - # way to check for surrogates, see issue 11454 for timings. + # way to check for surrogates, see bpo-11454 (moved to gh-55663) for timings. try: s.encode() return False @@ -81,7 +77,7 @@ def formataddr(pair, charset='utf-8'): If the first element of pair is false, then the second element is returned unmodified. - Optional charset if given is the character set that is used to encode + The optional charset is the character set that is used to encode realname in case realname is not ASCII safe. Can be an instance of str or a Charset-like object which has a header_encode method. Default is 'utf-8'. @@ -94,6 +90,8 @@ def formataddr(pair, charset='utf-8'): name.encode('ascii') except UnicodeEncodeError: if isinstance(charset, str): + # lazy import to improve module import time + from email.charset import Charset charset = Charset(charset) encoded_name = charset.header_encode(name) return "%s <%s>" % (encoded_name, address) @@ -106,24 +104,127 @@ def formataddr(pair, charset='utf-8'): return address +def _iter_escaped_chars(addr): + pos = 0 + escape = False + for pos, ch in enumerate(addr): + if escape: + yield (pos, '\\' + ch) + escape = False + elif ch == '\\': + escape = True + else: + yield (pos, ch) + if escape: + yield (pos, '\\') + + +def _strip_quoted_realnames(addr): + """Strip real names between quotes.""" + if '"' not in addr: + # Fast path + return addr + + start = 0 + open_pos = None + result = [] + for pos, ch in _iter_escaped_chars(addr): + if ch == '"': + if open_pos is None: + open_pos = pos + else: + if start != open_pos: + result.append(addr[start:open_pos]) + start = pos + 1 + open_pos = None -def getaddresses(fieldvalues): - """Return a list of (REALNAME, EMAIL) for each fieldvalue.""" - all = COMMASPACE.join(fieldvalues) - a = _AddressList(all) - return a.addresslist + if start < len(addr): + result.append(addr[start:]) + return ''.join(result) -ecre = re.compile(r''' - =\? # literal =? - (?P[^?]*?) # non-greedy up to the next ? is the charset - \? # literal ? - (?P[qb]) # either a "q" or a "b", case insensitive - \? # literal ? - (?P.*?) # non-greedy up to the next ?= is the atom - \?= # literal ?= - ''', re.VERBOSE | re.IGNORECASE) +supports_strict_parsing = True + +def getaddresses(fieldvalues, *, strict=True): + """Return a list of (REALNAME, EMAIL) or ('','') for each fieldvalue. + + When parsing fails for a fieldvalue, a 2-tuple of ('', '') is returned in + its place. + + If strict is true, use a strict parser which rejects malformed inputs. + """ + + # If strict is true, if the resulting list of parsed addresses is greater + # than the number of fieldvalues in the input list, a parsing error has + # occurred and consequently a list containing a single empty 2-tuple [('', + # '')] is returned in its place. This is done to avoid invalid output. + # + # Malformed input: getaddresses(['alice@example.com ']) + # Invalid output: [('', 'alice@example.com'), ('', 'bob@example.com')] + # Safe output: [('', '')] + + if not strict: + all = COMMASPACE.join(str(v) for v in fieldvalues) + a = _AddressList(all) + return a.addresslist + + fieldvalues = [str(v) for v in fieldvalues] + fieldvalues = _pre_parse_validation(fieldvalues) + addr = COMMASPACE.join(fieldvalues) + a = _AddressList(addr) + result = _post_parse_validation(a.addresslist) + + # Treat output as invalid if the number of addresses is not equal to the + # expected number of addresses. + n = 0 + for v in fieldvalues: + # When a comma is used in the Real Name part it is not a deliminator. + # So strip those out before counting the commas. + v = _strip_quoted_realnames(v) + # Expected number of addresses: 1 + number of commas + n += 1 + v.count(',') + if len(result) != n: + return [('', '')] + + return result + + +def _check_parenthesis(addr): + # Ignore parenthesis in quoted real names. + addr = _strip_quoted_realnames(addr) + + opens = 0 + for pos, ch in _iter_escaped_chars(addr): + if ch == '(': + opens += 1 + elif ch == ')': + opens -= 1 + if opens < 0: + return False + return (opens == 0) + + +def _pre_parse_validation(email_header_fields): + accepted_values = [] + for v in email_header_fields: + if not _check_parenthesis(v): + v = "('', '')" + accepted_values.append(v) + + return accepted_values + + +def _post_parse_validation(parsed_email_header_tuples): + accepted_values = [] + # The parser would have parsed a correctly formatted domain-literal + # The existence of an [ after parsing indicates a parsing failure + for v in parsed_email_header_tuples: + if '[' in v[1]: + v = ('', '') + accepted_values.append(v) + + return accepted_values def _format_timetuple_and_zone(timetuple, zone): @@ -140,7 +241,7 @@ def formatdate(timeval=None, localtime=False, usegmt=False): Fri, 09 Nov 2001 01:08:47 -0000 - Optional timeval if given is a floating point time value as accepted by + Optional timeval if given is a floating-point time value as accepted by gmtime() and localtime(), otherwise the current time is used. Optional localtime is a flag that when True, interprets timeval, and @@ -155,13 +256,13 @@ def formatdate(timeval=None, localtime=False, usegmt=False): # 2822 requires that day and month names be the English abbreviations. if timeval is None: timeval = time.time() - if localtime or usegmt: - dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc) - else: - dt = datetime.datetime.utcfromtimestamp(timeval) + dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc) + if localtime: dt = dt.astimezone() usegmt = False + elif not usegmt: + dt = dt.replace(tzinfo=None) return format_datetime(dt, usegmt) def format_datetime(dt, usegmt=False): @@ -193,6 +294,11 @@ def make_msgid(idstring=None, domain=None): portion of the message id after the '@'. It defaults to the locally defined hostname. """ + # Lazy imports to speedup module import time + # (no other functions in email.utils need these modules) + import random + import socket + timeval = int(time.time()*100) pid = os.getpid() randint = random.getrandbits(64) @@ -207,17 +313,43 @@ def make_msgid(idstring=None, domain=None): def parsedate_to_datetime(data): - *dtuple, tz = _parsedate_tz(data) + parsed_date_tz = _parsedate_tz(data) + if parsed_date_tz is None: + raise ValueError('Invalid date value or format "%s"' % str(data)) + *dtuple, tz = parsed_date_tz if tz is None: return datetime.datetime(*dtuple[:6]) return datetime.datetime(*dtuple[:6], tzinfo=datetime.timezone(datetime.timedelta(seconds=tz))) -def parseaddr(addr): - addrs = _AddressList(addr).addresslist - if not addrs: - return '', '' +def parseaddr(addr, *, strict=True): + """ + Parse addr into its constituent realname and email address parts. + + Return a tuple of realname and email address, unless the parse fails, in + which case return a 2-tuple of ('', ''). + + If strict is True, use a strict parser which rejects malformed inputs. + """ + if not strict: + addrs = _AddressList(addr).addresslist + if not addrs: + return ('', '') + return addrs[0] + + if isinstance(addr, list): + addr = addr[0] + + if not isinstance(addr, str): + return ('', '') + + addr = _pre_parse_validation([addr])[0] + addrs = _post_parse_validation(_AddressList(addr).addresslist) + + if not addrs or len(addrs) > 1: + return ('', '') + return addrs[0] @@ -265,21 +397,13 @@ def decode_params(params): params is a sequence of 2-tuples containing (param name, string value). """ - # Copy params so we don't mess with the original - params = params[:] - new_params = [] + new_params = [params[0]] # Map parameter's name to a list of continuations. The values are a # 3-tuple of the continuation number, the string value, and a flag # specifying whether a particular segment is %-encoded. rfc2231_params = {} - name, value = params.pop(0) - new_params.append((name, value)) - while params: - name, value = params.pop(0) - if name.endswith('*'): - encoded = True - else: - encoded = False + for name, value in params[1:]: + encoded = name.endswith('*') value = unquote(value) mo = rfc2231_continuation.match(name) if mo: @@ -293,8 +417,14 @@ def decode_params(params): for name, continuations in rfc2231_params.items(): value = [] extended = False - # Sort by number - continuations.sort() + # Sort by number, treating None as 0 if there is no 0, + # and ignore it if there is already a 0. + has_zero = any(x[0] == 0 for x in continuations) + if has_zero: + continuations = [x for x in continuations if x[0] is not None] + else: + continuations = [(x[0] or 0, x[1], x[2]) for x in continuations] + continuations.sort(key=lambda x: x[0]) # And now append all values in numerical order, converting # %-encodings for the encoded segments. If any of the # continuation names ends in a *, then the entire string, after @@ -342,41 +472,23 @@ def collapse_rfc2231_value(value, errors='replace', # better than not having it. # -def localtime(dt=None, isdst=-1): +def localtime(dt=None, isdst=None): """Return local time as an aware datetime object. If called without arguments, return current time. Otherwise *dt* argument should be a datetime instance, and it is converted to the local time zone according to the system time zone database. If *dt* is naive (that is, dt.tzinfo is None), it is assumed to be in local time. - In this case, a positive or zero value for *isdst* causes localtime to - presume initially that summer time (for example, Daylight Saving Time) - is or is not (respectively) in effect for the specified time. A - negative value for *isdst* causes the localtime() function to attempt - to divine whether summer time is in effect for the specified time. + The isdst parameter is ignored. """ + if isdst is not None: + import warnings + warnings._deprecated( + "The 'isdst' parameter to 'localtime'", + message='{name} is deprecated and slated for removal in Python {remove}', + remove=(3, 14), + ) if dt is None: - return datetime.datetime.now(datetime.timezone.utc).astimezone() - if dt.tzinfo is not None: - return dt.astimezone() - # We have a naive datetime. Convert to a (localtime) timetuple and pass to - # system mktime together with the isdst hint. System mktime will return - # seconds since epoch. - tm = dt.timetuple()[:-1] + (isdst,) - seconds = time.mktime(tm) - localtm = time.localtime(seconds) - try: - delta = datetime.timedelta(seconds=localtm.tm_gmtoff) - tz = datetime.timezone(delta, localtm.tm_zone) - except AttributeError: - # Compute UTC offset and compare with the value implied by tm_isdst. - # If the values match, use the zone name implied by tm_isdst. - delta = dt - datetime.datetime(*time.gmtime(seconds)[:6]) - dst = time.daylight and localtm.tm_isdst > 0 - gmtoff = -(time.altzone if dst else time.timezone) - if delta == datetime.timedelta(seconds=gmtoff): - tz = datetime.timezone(delta, time.tzname[dst]) - else: - tz = datetime.timezone(delta) - return dt.replace(tzinfo=tz) + dt = datetime.datetime.now() + return dt.astimezone() diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index 4b37d3321c9..298177eb800 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -156,15 +156,22 @@ def search_function(encoding): codecs.register(search_function) if sys.platform == 'win32': - def _alias_mbcs(encoding): + from ._win_cp_codecs import create_win32_code_page_codec + + def win32_code_page_search_function(encoding): + encoding = encoding.lower() + if not encoding.startswith('cp'): + return None try: - import _winapi - ansi_code_page = "cp%s" % _winapi.GetACP() - if encoding == ansi_code_page: - import encodings.mbcs - return encodings.mbcs.getregentry() - except ImportError: - # Imports may fail while we are shutting down - pass + cp = int(encoding[2:]) + except ValueError: + return None + # Test if the code page is supported + try: + codecs.code_page_encode(cp, 'x') + except (OverflowError, OSError): + return None + + return create_win32_code_page_codec(cp) - codecs.register(_alias_mbcs) + codecs.register(win32_code_page_search_function) diff --git a/Lib/encodings/_win_cp_codecs.py b/Lib/encodings/_win_cp_codecs.py new file mode 100644 index 00000000000..4f8eb886794 --- /dev/null +++ b/Lib/encodings/_win_cp_codecs.py @@ -0,0 +1,36 @@ +import codecs + +def create_win32_code_page_codec(cp): + from codecs import code_page_encode, code_page_decode + + def encode(input, errors='strict'): + return code_page_encode(cp, input, errors) + + def decode(input, errors='strict'): + return code_page_decode(cp, input, errors, True) + + class IncrementalEncoder(codecs.IncrementalEncoder): + def encode(self, input, final=False): + return code_page_encode(cp, input, self.errors)[0] + + class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + class StreamWriter(codecs.StreamWriter): + def encode(self, input, errors='strict'): + return code_page_encode(cp, input, errors) + + class StreamReader(codecs.StreamReader): + def decode(self, input, errors, final): + return code_page_decode(cp, input, errors, final) + + return codecs.CodecInfo( + name=f'cp{cp}', + encode=encode, + decode=decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamreader=StreamReader, + streamwriter=StreamWriter, + ) diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index d85afd6d5cf..4ecb6b6e297 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -204,11 +204,17 @@ 'csibm869' : 'cp869', 'ibm869' : 'cp869', + # cp874 codec + '874' : 'cp874', + 'ms874' : 'cp874', + 'windows_874' : 'cp874', + # cp932 codec '932' : 'cp932', 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', + 'windows_31j' : 'cp932', # cp949 codec '949' : 'cp949', @@ -240,6 +246,7 @@ 'ks_c_5601_1987' : 'euc_kr', 'ksx1001' : 'euc_kr', 'ks_x_1001' : 'euc_kr', + 'cseuckr' : 'euc_kr', # gb18030 codec 'gb18030_2000' : 'gb18030', @@ -398,6 +405,8 @@ 'iso_8859_8' : 'iso8859_8', 'iso_8859_8_1988' : 'iso8859_8', 'iso_ir_138' : 'iso8859_8', + 'iso_8859_8_i' : 'iso8859_8', + 'iso_8859_8_e' : 'iso8859_8', # iso8859_9 codec 'csisolatin5' : 'iso8859_9', diff --git a/Lib/encodings/cp65001.py b/Lib/encodings/cp65001.py deleted file mode 100644 index 95cb2aecf0c..00000000000 --- a/Lib/encodings/cp65001.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Code page 65001: Windows UTF-8 (CP_UTF8). -""" - -import codecs -import functools - -if not hasattr(codecs, 'code_page_encode'): - raise LookupError("cp65001 encoding is only available on Windows") - -### Codec APIs - -encode = functools.partial(codecs.code_page_encode, 65001) -_decode = functools.partial(codecs.code_page_decode, 65001) - -def decode(input, errors='strict'): - return codecs.code_page_decode(65001, input, errors, True) - -class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return encode(input, self.errors)[0] - -class IncrementalDecoder(codecs.BufferedIncrementalDecoder): - _buffer_decode = _decode - -class StreamWriter(codecs.StreamWriter): - encode = encode - -class StreamReader(codecs.StreamReader): - decode = _decode - -### encodings module API - -def getregentry(): - return codecs.CodecInfo( - name='cp65001', - encode=encode, - decode=decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, - ) diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index ea4058512fe..0c90b4c9fe1 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -11,7 +11,7 @@ sace_prefix = "xn--" # This assumes query strings, so AllowUnassigned is true -def nameprep(label): +def nameprep(label): # type: (str) -> str # Map newlabel = [] for c in label: @@ -25,7 +25,7 @@ def nameprep(label): label = unicodedata.normalize("NFKC", label) # Prohibit - for c in label: + for i, c in enumerate(label): if stringprep.in_table_c12(c) or \ stringprep.in_table_c22(c) or \ stringprep.in_table_c3(c) or \ @@ -35,42 +35,49 @@ def nameprep(label): stringprep.in_table_c7(c) or \ stringprep.in_table_c8(c) or \ stringprep.in_table_c9(c): - raise UnicodeError("Invalid character %r" % c) + raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}") # Check bidi RandAL = [stringprep.in_table_d1(x) for x in label] - for c in RandAL: - if c: - # There is a RandAL char in the string. Must perform further - # tests: - # 1) The characters in section 5.8 MUST be prohibited. - # This is table C.8, which was already checked - # 2) If a string contains any RandALCat character, the string - # MUST NOT contain any LCat character. - if any(stringprep.in_table_d2(x) for x in label): - raise UnicodeError("Violation of BIDI requirement 2") - - # 3) If a string contains any RandALCat character, a - # RandALCat character MUST be the first character of the - # string, and a RandALCat character MUST be the last - # character of the string. - if not RandAL[0] or not RandAL[-1]: - raise UnicodeError("Violation of BIDI requirement 3") + if any(RandAL): + # There is a RandAL char in the string. Must perform further + # tests: + # 1) The characters in section 5.8 MUST be prohibited. + # This is table C.8, which was already checked + # 2) If a string contains any RandALCat character, the string + # MUST NOT contain any LCat character. + for i, x in enumerate(label): + if stringprep.in_table_d2(x): + raise UnicodeEncodeError("idna", label, i, i+1, + "Violation of BIDI requirement 2") + # 3) If a string contains any RandALCat character, a + # RandALCat character MUST be the first character of the + # string, and a RandALCat character MUST be the last + # character of the string. + if not RandAL[0]: + raise UnicodeEncodeError("idna", label, 0, 1, + "Violation of BIDI requirement 3") + if not RandAL[-1]: + raise UnicodeEncodeError("idna", label, len(label)-1, len(label), + "Violation of BIDI requirement 3") return label -def ToASCII(label): +def ToASCII(label): # type: (str) -> bytes try: # Step 1: try ASCII - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 3: UseSTD3ASCIIRules is false, so # Skip to step 8. - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + if 0 < len(label_ascii) < 64: + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 2: nameprep label = nameprep(label) @@ -78,31 +85,48 @@ def ToASCII(label): # Step 3: UseSTD3ASCIIRules is false # Step 4: try ASCII try: - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 8. if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 5: Check ACE prefix - if label.startswith(sace_prefix): - raise UnicodeError("Label starts with ACE prefix") + if label.lower().startswith(sace_prefix): + raise UnicodeEncodeError( + "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix") # Step 6: Encode with PUNYCODE - label = label.encode("punycode") + label_ascii = label.encode("punycode") # Step 7: Prepend ACE prefix - label = ace_prefix + label + label_ascii = ace_prefix + label_ascii # Step 8: Check size - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + # do not check for empty as we prepend ace_prefix. + if len(label_ascii) < 64: + return label_ascii + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") def ToUnicode(label): + if len(label) > 1024: + # Protection from https://github.com/python/cpython/issues/98433. + # https://datatracker.ietf.org/doc/html/rfc5894#section-6 + # doesn't specify a label size limit prior to NAMEPREP. But having + # one makes practical sense. + # This leaves ample room for nameprep() to remove Nothing characters + # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still + # preventing us from wasting time decoding a big thing that'll just + # hit the actual <= 63 length limit in Step 6. + if isinstance(label, str): + label = label.encode("utf-8", errors="backslashreplace") + raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long") # Step 1: Check for ASCII if isinstance(label, bytes): pure_ascii = True @@ -110,25 +134,32 @@ def ToUnicode(label): try: label = label.encode("ascii") pure_ascii = True - except UnicodeError: + except UnicodeEncodeError: pure_ascii = False if not pure_ascii: + assert isinstance(label, str) # Step 2: Perform nameprep label = nameprep(label) # It doesn't say this, but apparently, it should be ASCII now try: label = label.encode("ascii") - except UnicodeError: - raise UnicodeError("Invalid character in IDN label") + except UnicodeEncodeError as exc: + raise UnicodeEncodeError("idna", label, exc.start, exc.end, + "Invalid character in IDN label") # Step 3: Check for ACE prefix - if not label.startswith(ace_prefix): + assert isinstance(label, bytes) + if not label.lower().startswith(ace_prefix): return str(label, "ascii") # Step 4: Remove ACE prefix label1 = label[len(ace_prefix):] # Step 5: Decode using PUNYCODE - result = label1.decode("punycode") + try: + result = label1.decode("punycode") + except UnicodeDecodeError as exc: + offset = len(ace_prefix) + raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason) # Step 6: Apply ToASCII label2 = ToASCII(result) @@ -136,7 +167,8 @@ def ToUnicode(label): # Step 7: Compare the result of step 6 with the one of step 3 # label2 will already be in lower case. if str(label, "ascii").lower() != str(label2, "ascii"): - raise UnicodeError("IDNA does not round-trip", label, label2) + raise UnicodeDecodeError("idna", label, 0, len(label), + f"IDNA does not round-trip, '{label!r}' != '{label2!r}'") # Step 8: return the result of step 5 return result @@ -148,7 +180,7 @@ def encode(self, input, errors='strict'): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return b'', 0 @@ -160,11 +192,16 @@ def encode(self, input, errors='strict'): else: # ASCII name: fast path labels = result.split(b'.') - for label in labels[:-1]: - if not (0 < len(label) < 64): - raise UnicodeError("label empty or too long") - if len(labels[-1]) >= 64: - raise UnicodeError("label too long") + for i, label in enumerate(labels[:-1]): + if len(label) == 0: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+1, + "label empty") + for i, label in enumerate(labels): + if len(label) >= 64: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+len(label), + "label too long") return result, len(input) result = bytearray() @@ -174,17 +211,27 @@ def encode(self, input, errors='strict'): del labels[-1] else: trailing_dot = b'' - for label in labels: + for i, label in enumerate(labels): if result: # Join with U+002E result.extend(b'.') - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError( + "idna", + input, + offset + exc.start, + offset + exc.end, + exc.reason, + ) return bytes(result+trailing_dot), len(input) def decode(self, input, errors='strict'): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return "", 0 @@ -194,7 +241,7 @@ def decode(self, input, errors='strict'): # XXX obviously wrong, see #3232 input = bytes(input) - if ace_prefix not in input: + if ace_prefix not in input.lower(): # Fast path try: return input.decode('ascii'), len(input) @@ -210,8 +257,15 @@ def decode(self, input, errors='strict'): trailing_dot = '' result = [] - for label in labels: - result.append(ToUnicode(label)) + for i, label in enumerate(labels): + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(x) for x in labels[:i]) + len(labels[:i]) + raise UnicodeDecodeError( + "idna", input, offset+exc.start, offset+exc.end, exc.reason) + else: + result.append(u_label) return ".".join(result)+trailing_dot, len(input) @@ -219,7 +273,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): def _buffer_encode(self, input, errors, final): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return (b'', 0) @@ -243,7 +297,16 @@ def _buffer_encode(self, input, errors, final): # Join with U+002E result.extend(b'.') size += 1 - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeEncodeError( + "idna", + input, + size + exc.start, + size + exc.end, + exc.reason, + ) size += len(label) result += trailing_dot @@ -253,7 +316,7 @@ def _buffer_encode(self, input, errors, final): class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return ("", 0) @@ -263,7 +326,11 @@ def _buffer_decode(self, input, errors, final): labels = dots.split(input) else: # Must be ASCII string - input = str(input, "ascii") + try: + input = str(input, "ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError("idna", input, + exc.start, exc.end, exc.reason) labels = input.split(".") trailing_dot = '' @@ -280,7 +347,18 @@ def _buffer_decode(self, input, errors, final): result = [] size = 0 for label in labels: - result.append(ToUnicode(label)) + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError( + "idna", + input.encode("ascii", errors="backslashreplace"), + size + exc.start, + size + exc.end, + exc.reason, + ) + else: + result.append(u_label) if size: size += 1 size += len(label) diff --git a/Lib/encodings/mac_centeuro.py b/Lib/encodings/mac_centeuro.py deleted file mode 100644 index 5785a0ec12d..00000000000 --- a/Lib/encodings/mac_centeuro.py +++ /dev/null @@ -1,307 +0,0 @@ -""" Python Character Mapping Codec mac_centeuro generated from 'MAPPINGS/VENDORS/APPLE/CENTEURO.TXT' with gencodec.py. - -"""#" - -import codecs - -### Codec APIs - -class Codec(codecs.Codec): - - def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_table) - - def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) - -class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_table)[0] - -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] - -class StreamWriter(Codec,codecs.StreamWriter): - pass - -class StreamReader(Codec,codecs.StreamReader): - pass - -### encodings module API - -def getregentry(): - return codecs.CodecInfo( - name='mac-centeuro', - encode=Codec().encode, - decode=Codec().decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, - ) - - -### Decoding Table - -decoding_table = ( - '\x00' # 0x00 -> CONTROL CHARACTER - '\x01' # 0x01 -> CONTROL CHARACTER - '\x02' # 0x02 -> CONTROL CHARACTER - '\x03' # 0x03 -> CONTROL CHARACTER - '\x04' # 0x04 -> CONTROL CHARACTER - '\x05' # 0x05 -> CONTROL CHARACTER - '\x06' # 0x06 -> CONTROL CHARACTER - '\x07' # 0x07 -> CONTROL CHARACTER - '\x08' # 0x08 -> CONTROL CHARACTER - '\t' # 0x09 -> CONTROL CHARACTER - '\n' # 0x0A -> CONTROL CHARACTER - '\x0b' # 0x0B -> CONTROL CHARACTER - '\x0c' # 0x0C -> CONTROL CHARACTER - '\r' # 0x0D -> CONTROL CHARACTER - '\x0e' # 0x0E -> CONTROL CHARACTER - '\x0f' # 0x0F -> CONTROL CHARACTER - '\x10' # 0x10 -> CONTROL CHARACTER - '\x11' # 0x11 -> CONTROL CHARACTER - '\x12' # 0x12 -> CONTROL CHARACTER - '\x13' # 0x13 -> CONTROL CHARACTER - '\x14' # 0x14 -> CONTROL CHARACTER - '\x15' # 0x15 -> CONTROL CHARACTER - '\x16' # 0x16 -> CONTROL CHARACTER - '\x17' # 0x17 -> CONTROL CHARACTER - '\x18' # 0x18 -> CONTROL CHARACTER - '\x19' # 0x19 -> CONTROL CHARACTER - '\x1a' # 0x1A -> CONTROL CHARACTER - '\x1b' # 0x1B -> CONTROL CHARACTER - '\x1c' # 0x1C -> CONTROL CHARACTER - '\x1d' # 0x1D -> CONTROL CHARACTER - '\x1e' # 0x1E -> CONTROL CHARACTER - '\x1f' # 0x1F -> CONTROL CHARACTER - ' ' # 0x20 -> SPACE - '!' # 0x21 -> EXCLAMATION MARK - '"' # 0x22 -> QUOTATION MARK - '#' # 0x23 -> NUMBER SIGN - '$' # 0x24 -> DOLLAR SIGN - '%' # 0x25 -> PERCENT SIGN - '&' # 0x26 -> AMPERSAND - "'" # 0x27 -> APOSTROPHE - '(' # 0x28 -> LEFT PARENTHESIS - ')' # 0x29 -> RIGHT PARENTHESIS - '*' # 0x2A -> ASTERISK - '+' # 0x2B -> PLUS SIGN - ',' # 0x2C -> COMMA - '-' # 0x2D -> HYPHEN-MINUS - '.' # 0x2E -> FULL STOP - '/' # 0x2F -> SOLIDUS - '0' # 0x30 -> DIGIT ZERO - '1' # 0x31 -> DIGIT ONE - '2' # 0x32 -> DIGIT TWO - '3' # 0x33 -> DIGIT THREE - '4' # 0x34 -> DIGIT FOUR - '5' # 0x35 -> DIGIT FIVE - '6' # 0x36 -> DIGIT SIX - '7' # 0x37 -> DIGIT SEVEN - '8' # 0x38 -> DIGIT EIGHT - '9' # 0x39 -> DIGIT NINE - ':' # 0x3A -> COLON - ';' # 0x3B -> SEMICOLON - '<' # 0x3C -> LESS-THAN SIGN - '=' # 0x3D -> EQUALS SIGN - '>' # 0x3E -> GREATER-THAN SIGN - '?' # 0x3F -> QUESTION MARK - '@' # 0x40 -> COMMERCIAL AT - 'A' # 0x41 -> LATIN CAPITAL LETTER A - 'B' # 0x42 -> LATIN CAPITAL LETTER B - 'C' # 0x43 -> LATIN CAPITAL LETTER C - 'D' # 0x44 -> LATIN CAPITAL LETTER D - 'E' # 0x45 -> LATIN CAPITAL LETTER E - 'F' # 0x46 -> LATIN CAPITAL LETTER F - 'G' # 0x47 -> LATIN CAPITAL LETTER G - 'H' # 0x48 -> LATIN CAPITAL LETTER H - 'I' # 0x49 -> LATIN CAPITAL LETTER I - 'J' # 0x4A -> LATIN CAPITAL LETTER J - 'K' # 0x4B -> LATIN CAPITAL LETTER K - 'L' # 0x4C -> LATIN CAPITAL LETTER L - 'M' # 0x4D -> LATIN CAPITAL LETTER M - 'N' # 0x4E -> LATIN CAPITAL LETTER N - 'O' # 0x4F -> LATIN CAPITAL LETTER O - 'P' # 0x50 -> LATIN CAPITAL LETTER P - 'Q' # 0x51 -> LATIN CAPITAL LETTER Q - 'R' # 0x52 -> LATIN CAPITAL LETTER R - 'S' # 0x53 -> LATIN CAPITAL LETTER S - 'T' # 0x54 -> LATIN CAPITAL LETTER T - 'U' # 0x55 -> LATIN CAPITAL LETTER U - 'V' # 0x56 -> LATIN CAPITAL LETTER V - 'W' # 0x57 -> LATIN CAPITAL LETTER W - 'X' # 0x58 -> LATIN CAPITAL LETTER X - 'Y' # 0x59 -> LATIN CAPITAL LETTER Y - 'Z' # 0x5A -> LATIN CAPITAL LETTER Z - '[' # 0x5B -> LEFT SQUARE BRACKET - '\\' # 0x5C -> REVERSE SOLIDUS - ']' # 0x5D -> RIGHT SQUARE BRACKET - '^' # 0x5E -> CIRCUMFLEX ACCENT - '_' # 0x5F -> LOW LINE - '`' # 0x60 -> GRAVE ACCENT - 'a' # 0x61 -> LATIN SMALL LETTER A - 'b' # 0x62 -> LATIN SMALL LETTER B - 'c' # 0x63 -> LATIN SMALL LETTER C - 'd' # 0x64 -> LATIN SMALL LETTER D - 'e' # 0x65 -> LATIN SMALL LETTER E - 'f' # 0x66 -> LATIN SMALL LETTER F - 'g' # 0x67 -> LATIN SMALL LETTER G - 'h' # 0x68 -> LATIN SMALL LETTER H - 'i' # 0x69 -> LATIN SMALL LETTER I - 'j' # 0x6A -> LATIN SMALL LETTER J - 'k' # 0x6B -> LATIN SMALL LETTER K - 'l' # 0x6C -> LATIN SMALL LETTER L - 'm' # 0x6D -> LATIN SMALL LETTER M - 'n' # 0x6E -> LATIN SMALL LETTER N - 'o' # 0x6F -> LATIN SMALL LETTER O - 'p' # 0x70 -> LATIN SMALL LETTER P - 'q' # 0x71 -> LATIN SMALL LETTER Q - 'r' # 0x72 -> LATIN SMALL LETTER R - 's' # 0x73 -> LATIN SMALL LETTER S - 't' # 0x74 -> LATIN SMALL LETTER T - 'u' # 0x75 -> LATIN SMALL LETTER U - 'v' # 0x76 -> LATIN SMALL LETTER V - 'w' # 0x77 -> LATIN SMALL LETTER W - 'x' # 0x78 -> LATIN SMALL LETTER X - 'y' # 0x79 -> LATIN SMALL LETTER Y - 'z' # 0x7A -> LATIN SMALL LETTER Z - '{' # 0x7B -> LEFT CURLY BRACKET - '|' # 0x7C -> VERTICAL LINE - '}' # 0x7D -> RIGHT CURLY BRACKET - '~' # 0x7E -> TILDE - '\x7f' # 0x7F -> CONTROL CHARACTER - '\xc4' # 0x80 -> LATIN CAPITAL LETTER A WITH DIAERESIS - '\u0100' # 0x81 -> LATIN CAPITAL LETTER A WITH MACRON - '\u0101' # 0x82 -> LATIN SMALL LETTER A WITH MACRON - '\xc9' # 0x83 -> LATIN CAPITAL LETTER E WITH ACUTE - '\u0104' # 0x84 -> LATIN CAPITAL LETTER A WITH OGONEK - '\xd6' # 0x85 -> LATIN CAPITAL LETTER O WITH DIAERESIS - '\xdc' # 0x86 -> LATIN CAPITAL LETTER U WITH DIAERESIS - '\xe1' # 0x87 -> LATIN SMALL LETTER A WITH ACUTE - '\u0105' # 0x88 -> LATIN SMALL LETTER A WITH OGONEK - '\u010c' # 0x89 -> LATIN CAPITAL LETTER C WITH CARON - '\xe4' # 0x8A -> LATIN SMALL LETTER A WITH DIAERESIS - '\u010d' # 0x8B -> LATIN SMALL LETTER C WITH CARON - '\u0106' # 0x8C -> LATIN CAPITAL LETTER C WITH ACUTE - '\u0107' # 0x8D -> LATIN SMALL LETTER C WITH ACUTE - '\xe9' # 0x8E -> LATIN SMALL LETTER E WITH ACUTE - '\u0179' # 0x8F -> LATIN CAPITAL LETTER Z WITH ACUTE - '\u017a' # 0x90 -> LATIN SMALL LETTER Z WITH ACUTE - '\u010e' # 0x91 -> LATIN CAPITAL LETTER D WITH CARON - '\xed' # 0x92 -> LATIN SMALL LETTER I WITH ACUTE - '\u010f' # 0x93 -> LATIN SMALL LETTER D WITH CARON - '\u0112' # 0x94 -> LATIN CAPITAL LETTER E WITH MACRON - '\u0113' # 0x95 -> LATIN SMALL LETTER E WITH MACRON - '\u0116' # 0x96 -> LATIN CAPITAL LETTER E WITH DOT ABOVE - '\xf3' # 0x97 -> LATIN SMALL LETTER O WITH ACUTE - '\u0117' # 0x98 -> LATIN SMALL LETTER E WITH DOT ABOVE - '\xf4' # 0x99 -> LATIN SMALL LETTER O WITH CIRCUMFLEX - '\xf6' # 0x9A -> LATIN SMALL LETTER O WITH DIAERESIS - '\xf5' # 0x9B -> LATIN SMALL LETTER O WITH TILDE - '\xfa' # 0x9C -> LATIN SMALL LETTER U WITH ACUTE - '\u011a' # 0x9D -> LATIN CAPITAL LETTER E WITH CARON - '\u011b' # 0x9E -> LATIN SMALL LETTER E WITH CARON - '\xfc' # 0x9F -> LATIN SMALL LETTER U WITH DIAERESIS - '\u2020' # 0xA0 -> DAGGER - '\xb0' # 0xA1 -> DEGREE SIGN - '\u0118' # 0xA2 -> LATIN CAPITAL LETTER E WITH OGONEK - '\xa3' # 0xA3 -> POUND SIGN - '\xa7' # 0xA4 -> SECTION SIGN - '\u2022' # 0xA5 -> BULLET - '\xb6' # 0xA6 -> PILCROW SIGN - '\xdf' # 0xA7 -> LATIN SMALL LETTER SHARP S - '\xae' # 0xA8 -> REGISTERED SIGN - '\xa9' # 0xA9 -> COPYRIGHT SIGN - '\u2122' # 0xAA -> TRADE MARK SIGN - '\u0119' # 0xAB -> LATIN SMALL LETTER E WITH OGONEK - '\xa8' # 0xAC -> DIAERESIS - '\u2260' # 0xAD -> NOT EQUAL TO - '\u0123' # 0xAE -> LATIN SMALL LETTER G WITH CEDILLA - '\u012e' # 0xAF -> LATIN CAPITAL LETTER I WITH OGONEK - '\u012f' # 0xB0 -> LATIN SMALL LETTER I WITH OGONEK - '\u012a' # 0xB1 -> LATIN CAPITAL LETTER I WITH MACRON - '\u2264' # 0xB2 -> LESS-THAN OR EQUAL TO - '\u2265' # 0xB3 -> GREATER-THAN OR EQUAL TO - '\u012b' # 0xB4 -> LATIN SMALL LETTER I WITH MACRON - '\u0136' # 0xB5 -> LATIN CAPITAL LETTER K WITH CEDILLA - '\u2202' # 0xB6 -> PARTIAL DIFFERENTIAL - '\u2211' # 0xB7 -> N-ARY SUMMATION - '\u0142' # 0xB8 -> LATIN SMALL LETTER L WITH STROKE - '\u013b' # 0xB9 -> LATIN CAPITAL LETTER L WITH CEDILLA - '\u013c' # 0xBA -> LATIN SMALL LETTER L WITH CEDILLA - '\u013d' # 0xBB -> LATIN CAPITAL LETTER L WITH CARON - '\u013e' # 0xBC -> LATIN SMALL LETTER L WITH CARON - '\u0139' # 0xBD -> LATIN CAPITAL LETTER L WITH ACUTE - '\u013a' # 0xBE -> LATIN SMALL LETTER L WITH ACUTE - '\u0145' # 0xBF -> LATIN CAPITAL LETTER N WITH CEDILLA - '\u0146' # 0xC0 -> LATIN SMALL LETTER N WITH CEDILLA - '\u0143' # 0xC1 -> LATIN CAPITAL LETTER N WITH ACUTE - '\xac' # 0xC2 -> NOT SIGN - '\u221a' # 0xC3 -> SQUARE ROOT - '\u0144' # 0xC4 -> LATIN SMALL LETTER N WITH ACUTE - '\u0147' # 0xC5 -> LATIN CAPITAL LETTER N WITH CARON - '\u2206' # 0xC6 -> INCREMENT - '\xab' # 0xC7 -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xbb' # 0xC8 -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - '\u2026' # 0xC9 -> HORIZONTAL ELLIPSIS - '\xa0' # 0xCA -> NO-BREAK SPACE - '\u0148' # 0xCB -> LATIN SMALL LETTER N WITH CARON - '\u0150' # 0xCC -> LATIN CAPITAL LETTER O WITH DOUBLE ACUTE - '\xd5' # 0xCD -> LATIN CAPITAL LETTER O WITH TILDE - '\u0151' # 0xCE -> LATIN SMALL LETTER O WITH DOUBLE ACUTE - '\u014c' # 0xCF -> LATIN CAPITAL LETTER O WITH MACRON - '\u2013' # 0xD0 -> EN DASH - '\u2014' # 0xD1 -> EM DASH - '\u201c' # 0xD2 -> LEFT DOUBLE QUOTATION MARK - '\u201d' # 0xD3 -> RIGHT DOUBLE QUOTATION MARK - '\u2018' # 0xD4 -> LEFT SINGLE QUOTATION MARK - '\u2019' # 0xD5 -> RIGHT SINGLE QUOTATION MARK - '\xf7' # 0xD6 -> DIVISION SIGN - '\u25ca' # 0xD7 -> LOZENGE - '\u014d' # 0xD8 -> LATIN SMALL LETTER O WITH MACRON - '\u0154' # 0xD9 -> LATIN CAPITAL LETTER R WITH ACUTE - '\u0155' # 0xDA -> LATIN SMALL LETTER R WITH ACUTE - '\u0158' # 0xDB -> LATIN CAPITAL LETTER R WITH CARON - '\u2039' # 0xDC -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK - '\u203a' # 0xDD -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - '\u0159' # 0xDE -> LATIN SMALL LETTER R WITH CARON - '\u0156' # 0xDF -> LATIN CAPITAL LETTER R WITH CEDILLA - '\u0157' # 0xE0 -> LATIN SMALL LETTER R WITH CEDILLA - '\u0160' # 0xE1 -> LATIN CAPITAL LETTER S WITH CARON - '\u201a' # 0xE2 -> SINGLE LOW-9 QUOTATION MARK - '\u201e' # 0xE3 -> DOUBLE LOW-9 QUOTATION MARK - '\u0161' # 0xE4 -> LATIN SMALL LETTER S WITH CARON - '\u015a' # 0xE5 -> LATIN CAPITAL LETTER S WITH ACUTE - '\u015b' # 0xE6 -> LATIN SMALL LETTER S WITH ACUTE - '\xc1' # 0xE7 -> LATIN CAPITAL LETTER A WITH ACUTE - '\u0164' # 0xE8 -> LATIN CAPITAL LETTER T WITH CARON - '\u0165' # 0xE9 -> LATIN SMALL LETTER T WITH CARON - '\xcd' # 0xEA -> LATIN CAPITAL LETTER I WITH ACUTE - '\u017d' # 0xEB -> LATIN CAPITAL LETTER Z WITH CARON - '\u017e' # 0xEC -> LATIN SMALL LETTER Z WITH CARON - '\u016a' # 0xED -> LATIN CAPITAL LETTER U WITH MACRON - '\xd3' # 0xEE -> LATIN CAPITAL LETTER O WITH ACUTE - '\xd4' # 0xEF -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX - '\u016b' # 0xF0 -> LATIN SMALL LETTER U WITH MACRON - '\u016e' # 0xF1 -> LATIN CAPITAL LETTER U WITH RING ABOVE - '\xda' # 0xF2 -> LATIN CAPITAL LETTER U WITH ACUTE - '\u016f' # 0xF3 -> LATIN SMALL LETTER U WITH RING ABOVE - '\u0170' # 0xF4 -> LATIN CAPITAL LETTER U WITH DOUBLE ACUTE - '\u0171' # 0xF5 -> LATIN SMALL LETTER U WITH DOUBLE ACUTE - '\u0172' # 0xF6 -> LATIN CAPITAL LETTER U WITH OGONEK - '\u0173' # 0xF7 -> LATIN SMALL LETTER U WITH OGONEK - '\xdd' # 0xF8 -> LATIN CAPITAL LETTER Y WITH ACUTE - '\xfd' # 0xF9 -> LATIN SMALL LETTER Y WITH ACUTE - '\u0137' # 0xFA -> LATIN SMALL LETTER K WITH CEDILLA - '\u017b' # 0xFB -> LATIN CAPITAL LETTER Z WITH DOT ABOVE - '\u0141' # 0xFC -> LATIN CAPITAL LETTER L WITH STROKE - '\u017c' # 0xFD -> LATIN SMALL LETTER Z WITH DOT ABOVE - '\u0122' # 0xFE -> LATIN CAPITAL LETTER G WITH CEDILLA - '\u02c7' # 0xFF -> CARON -) - -### Encoding table -encoding_table=codecs.charmap_build(decoding_table) diff --git a/Lib/encodings/palmos.py b/Lib/encodings/palmos.py index c506d654523..df164ca5b95 100644 --- a/Lib/encodings/palmos.py +++ b/Lib/encodings/palmos.py @@ -201,7 +201,7 @@ def getregentry(): '\u02dc' # 0x98 -> SMALL TILDE '\u2122' # 0x99 -> TRADE MARK SIGN '\u0161' # 0x9A -> LATIN SMALL LETTER S WITH CARON - '\x9b' # 0x9B -> + '\u203a' # 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK '\u0153' # 0x9C -> LATIN SMALL LIGATURE OE '\x9d' # 0x9D -> '\x9e' # 0x9E -> diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 1c572644707..4622fc8c920 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -1,4 +1,4 @@ -""" Codec for the Punicode encoding, as specified in RFC 3492 +""" Codec for the Punycode encoding, as specified in RFC 3492 Written by Martin v. Löwis. """ @@ -131,10 +131,11 @@ def decode_generalized_number(extended, extpos, bias, errors): j = 0 while 1: try: - char = ord(extended[extpos]) + char = extended[extpos] except IndexError: if errors == "strict": - raise UnicodeError("incomplete punicode string") + raise UnicodeDecodeError("punycode", extended, extpos, extpos+1, + "incomplete punycode string") return extpos + 1, None extpos += 1 if 0x41 <= char <= 0x5A: # A-Z @@ -142,8 +143,8 @@ def decode_generalized_number(extended, extpos, bias, errors): elif 0x30 <= char <= 0x39: digit = char - 22 # 0x30-26 elif errors == "strict": - raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos-1]) + raise UnicodeDecodeError("punycode", extended, extpos-1, extpos, + f"Invalid extended code point '{extended[extpos-1]}'") else: return extpos, None t = T(j, bias) @@ -155,11 +156,14 @@ def decode_generalized_number(extended, extpos, bias, errors): def insertion_sort(base, extended, errors): - """3.2 Insertion unsort coding""" + """3.2 Insertion sort coding""" + # This function raises UnicodeDecodeError with position in the extended. + # Caller should add the offset. char = 0x80 pos = -1 bias = 72 extpos = 0 + while extpos < len(extended): newpos, delta = decode_generalized_number(extended, extpos, bias, errors) @@ -171,7 +175,9 @@ def insertion_sort(base, extended, errors): char += pos // (len(base) + 1) if char > 0x10FFFF: if errors == "strict": - raise UnicodeError("Invalid character U+%x" % char) + raise UnicodeDecodeError( + "punycode", extended, pos-1, pos, + f"Invalid character U+{char:x}") char = ord('?') pos = pos % (len(base) + 1) base = base[:pos] + chr(char) + base[pos:] @@ -187,11 +193,21 @@ def punycode_decode(text, errors): pos = text.rfind(b"-") if pos == -1: base = "" - extended = str(text, "ascii").upper() + extended = text.upper() else: - base = str(text[:pos], "ascii", errors) - extended = str(text[pos+1:], "ascii").upper() - return insertion_sort(base, extended, errors) + try: + base = str(text[:pos], "ascii", errors) + except UnicodeDecodeError as exc: + raise UnicodeDecodeError("ascii", text, exc.start, exc.end, + exc.reason) from None + extended = text[pos+1:].upper() + try: + return insertion_sort(base, extended, errors) + except UnicodeDecodeError as exc: + offset = pos + 1 + raise UnicodeDecodeError("punycode", text, + offset+exc.start, offset+exc.end, + exc.reason) from None ### Codec APIs @@ -203,7 +219,7 @@ def encode(self, input, errors='strict'): def decode(self, input, errors='strict'): if errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") res = punycode_decode(input, errors) return res, len(input) @@ -214,7 +230,7 @@ def encode(self, input, final=False): class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): if self.errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+self.errors) + raise UnicodeError(f"Unsupported error handling: {self.errors}") return punycode_decode(input, self.errors) class StreamWriter(Codec,codecs.StreamWriter): diff --git a/Lib/encodings/undefined.py b/Lib/encodings/undefined.py index 4690288355c..082771e1c86 100644 --- a/Lib/encodings/undefined.py +++ b/Lib/encodings/undefined.py @@ -1,6 +1,6 @@ """ Python 'undefined' Codec - This codec will always raise a ValueError exception when being + This codec will always raise a UnicodeError exception when being used. It is intended for use by the site.py file to switch off automatic string to Unicode coercion. diff --git a/Lib/encodings/unicode_internal.py b/Lib/encodings/unicode_internal.py deleted file mode 100644 index df3e7752d20..00000000000 --- a/Lib/encodings/unicode_internal.py +++ /dev/null @@ -1,45 +0,0 @@ -""" Python 'unicode-internal' Codec - - -Written by Marc-Andre Lemburg (mal@lemburg.com). - -(c) Copyright CNRI, All Rights Reserved. NO WARRANTY. - -""" -import codecs - -### Codec APIs - -class Codec(codecs.Codec): - - # Note: Binding these as C functions will result in the class not - # converting them to methods. This is intended. - encode = codecs.unicode_internal_encode - decode = codecs.unicode_internal_decode - -class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return codecs.unicode_internal_encode(input, self.errors)[0] - -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.unicode_internal_decode(input, self.errors)[0] - -class StreamWriter(Codec,codecs.StreamWriter): - pass - -class StreamReader(Codec,codecs.StreamReader): - pass - -### encodings module API - -def getregentry(): - return codecs.CodecInfo( - name='unicode-internal', - encode=Codec.encode, - decode=Codec.decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamwriter=StreamWriter, - streamreader=StreamReader, - ) diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py index c61248242be..d3b99800266 100644 --- a/Lib/encodings/utf_16.py +++ b/Lib/encodings/utf_16.py @@ -64,7 +64,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_16_be_decode elif consumed >= 2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -138,7 +138,7 @@ def decode(self, input, errors='strict'): elif byteorder == 1: self.decode = codecs.utf_16_be_decode elif consumed>=2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py index cdf84d14129..1924bedbb74 100644 --- a/Lib/encodings/utf_32.py +++ b/Lib/encodings/utf_32.py @@ -59,7 +59,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_32_be_decode elif consumed >= 4: - raise UnicodeError("UTF-32 stream does not start with BOM") + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -132,8 +132,8 @@ def decode(self, input, errors='strict'): self.decode = codecs.utf_32_le_decode elif byteorder == 1: self.decode = codecs.utf_32_be_decode - elif consumed>=4: - raise UnicodeError("UTF-32 stream does not start with BOM") + elif consumed >= 4: + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 1a2f57c07ba..21bbfad0fe6 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,80 +1,64 @@ -import collections import os -import os.path import subprocess import sys import sysconfig import tempfile +from contextlib import nullcontext from importlib import resources +from pathlib import Path +from shutil import copy2 __all__ = ["version", "bootstrap"] -_PACKAGE_NAMES = ('setuptools', 'pip') -_SETUPTOOLS_VERSION = "65.5.0" -_PIP_VERSION = "22.3.1" -_PROJECTS = [ - ("setuptools", _SETUPTOOLS_VERSION, "py3"), - ("pip", _PIP_VERSION, "py3"), -] - -# Packages bundled in ensurepip._bundled have wheel_name set. -# Packages from WHEEL_PKG_DIR have wheel_path set. -_Package = collections.namedtuple('Package', - ('version', 'wheel_name', 'wheel_path')) +_PIP_VERSION = "25.3" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora # installs wheel packages in the /usr/share/python-wheels/ directory and don't # install the ensurepip._bundled package. -_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') +if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None: + _WHEEL_PKG_DIR = Path(_pkg_dir).resolve() +else: + _WHEEL_PKG_DIR = None -def _find_packages(path): - packages = {} +def _find_wheel_pkg_dir_pip(): + if _WHEEL_PKG_DIR is None: + # NOTE: The compile-time `WHEEL_PKG_DIR` is unset so there is no place + # NOTE: for looking up the wheels. + return None + + dist_matching_wheels = _WHEEL_PKG_DIR.glob('pip-*.whl') try: - filenames = os.listdir(path) - except OSError: - # Ignore: path doesn't exist or permission error - filenames = () - # Make the code deterministic if a directory contains multiple wheel files - # of the same package, but don't attempt to implement correct version - # comparison since this case should not happen. - filenames = sorted(filenames) - for filename in filenames: - # filename is like 'pip-21.2.4-py3-none-any.whl' - if not filename.endswith(".whl"): - continue - for name in _PACKAGE_NAMES: - prefix = name + '-' - if filename.startswith(prefix): - break - else: - continue - - # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' - version = filename.removeprefix(prefix).partition('-')[0] - wheel_path = os.path.join(path, filename) - packages[name] = _Package(version, None, wheel_path) - return packages - - -def _get_packages(): - global _PACKAGES, _WHEEL_PKG_DIR - if _PACKAGES is not None: - return _PACKAGES - - packages = {} - for name, version, py_tag in _PROJECTS: - wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" - packages[name] = _Package(version, wheel_name, None) - if _WHEEL_PKG_DIR: - dir_packages = _find_packages(_WHEEL_PKG_DIR) - # only used the wheel package directory if all packages are found there - if all(name in dir_packages for name in _PACKAGE_NAMES): - packages = dir_packages - _PACKAGES = packages - return packages -_PACKAGES = None + last_matching_dist_wheel = sorted(dist_matching_wheels)[-1] + except IndexError: + # NOTE: `WHEEL_PKG_DIR` does not contain any wheel files for `pip`. + return None + + return nullcontext(last_matching_dist_wheel) + + +def _get_pip_whl_path_ctx(): + # Prefer pip from the wheel package directory, if present. + if (alternative_pip_wheel_path := _find_wheel_pkg_dir_pip()) is not None: + return alternative_pip_wheel_path + + return resources.as_file( + resources.files('ensurepip') + / '_bundled' + / f'pip-{_PIP_VERSION}-py3-none-any.whl' + ) + + +def _get_pip_version(): + with _get_pip_whl_path_ctx() as bundled_wheel_path: + wheel_name = bundled_wheel_path.name + return ( + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' + wheel_name. + removeprefix('pip-'). + partition('-')[0] + ) def _run_pip(args, additional_paths=None): @@ -107,7 +91,7 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _get_packages()['pip'].version + return _get_pip_version() def _disable_pip_configuration_settings(): @@ -153,40 +137,26 @@ def _bootstrap(*, root=None, upgrade=False, user=False, _disable_pip_configuration_settings() - # By default, installing pip and setuptools installs all of the + # By default, installing pip installs all of the # following scripts (X.Y == running Python version): # - # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # pip, pipX, pipX.Y # # pip 1.5+ allows ensurepip to request that some of those be left out if altinstall: - # omit pip, pipX and easy_install + # omit pip, pipX os.environ["ENSUREPIP_OPTIONS"] = "altinstall" elif not default_pip: - # omit pip and easy_install + # omit pip os.environ["ENSUREPIP_OPTIONS"] = "install" with tempfile.TemporaryDirectory() as tmpdir: # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path - additional_paths = [] - for name, package in _get_packages().items(): - if package.wheel_name: - # Use bundled wheel package - wheel_name = package.wheel_name - wheel_path = resources.files("ensurepip") / "_bundled" / wheel_name - whl = wheel_path.read_bytes() - else: - # Use the wheel package directory - with open(package.wheel_path, "rb") as fp: - whl = fp.read() - wheel_name = os.path.basename(package.wheel_path) - - filename = os.path.join(tmpdir, wheel_name) - with open(filename, "wb") as fp: - fp.write(whl) - - additional_paths.append(filename) + tmpdir_path = Path(tmpdir) + with _get_pip_whl_path_ctx() as bundled_wheel_path: + tmp_wheel_path = tmpdir_path / bundled_wheel_path.name + copy2(bundled_wheel_path, tmp_wheel_path) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -199,7 +169,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) + return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) + def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -229,12 +200,12 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) + return _run_pip([*args, "pip"]) def _main(argv=None): import argparse - parser = argparse.ArgumentParser(prog="python -m ensurepip") + parser = argparse.ArgumentParser(color=True) parser.add_argument( "--version", action="version", @@ -271,14 +242,14 @@ def _main(argv=None): action="store_true", default=False, help=("Make an alternate install, installing only the X.Y versioned " - "scripts (Default: pipX, pipX.Y, easy_install-X.Y)."), + "scripts (Default: pipX, pipX.Y)."), ) parser.add_argument( "--default-pip", action="store_true", default=False, help=("Make a default pip install, installing the unqualified pip " - "and easy_install in addition to the versioned scripts."), + "in addition to the versioned scripts."), ) args = parser.parse_args(argv) diff --git a/Lib/ensurepip/_bundled/pip-22.3.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-22.3.1-py3-none-any.whl deleted file mode 100644 index c5b7753e757..00000000000 Binary files a/Lib/ensurepip/_bundled/pip-22.3.1-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl new file mode 100644 index 00000000000..755e1aa0c3d Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-25.3-py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl deleted file mode 100644 index 123a13e2c6b..00000000000 Binary files a/Lib/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py index b257904328d..4183c28a809 100644 --- a/Lib/ensurepip/_uninstall.py +++ b/Lib/ensurepip/_uninstall.py @@ -6,7 +6,7 @@ def _main(argv=None): - parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/enum.py b/Lib/enum.py index 7cffb71863c..b4551da1c17 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,12 +1,10 @@ import sys import builtins as bltns from types import MappingProxyType, DynamicClassAttribute -from operator import or_ as _or_ -from functools import reduce __all__ = [ - 'EnumType', 'EnumMeta', + 'EnumType', 'EnumMeta', 'EnumDict', 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'ReprEnum', 'auto', 'unique', 'property', 'verify', 'member', 'nonmember', 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', @@ -63,8 +61,8 @@ def _is_sunder(name): return ( len(name) > 2 and name[0] == name[-1] == '_' and - name[1:2] != '_' and - name[-2:-1] != '_' + name[1] != '_' and + name[-2] != '_' ) def _is_internal_class(cls_name, obj): @@ -83,7 +81,6 @@ def _is_private(cls_name, name): if ( len(name) > pat_len and name.startswith(pattern) - and name[pat_len:pat_len+1] != ['_'] and (name[-1] != '_' or name[-2] != '_') ): return True @@ -132,7 +129,7 @@ def show_flag_values(value): def bin(num, max_bits=None): """ Like built-in bin(), except negative values are represented in - twos-compliment, and the leading bit always indicates sign + twos-complement, and the leading bit always indicates sign (0=positive, 1=negative). >>> bin(10) @@ -141,6 +138,7 @@ def bin(num, max_bits=None): '0b1 0101' """ + num = num.__index__() ceiling = 2 ** (num).bit_length() if num >= 0: s = bltns.bin(num + ceiling).replace('1', '0', 1) @@ -153,18 +151,10 @@ def bin(num, max_bits=None): digits = (sign[-1] * max_bits + digits)[-max_bits:] return "%s %s" % (sign, digits) -def _dedent(text): - """ - Like textwrap.dedent. Rewritten because we cannot import textwrap. - """ - lines = text.split('\n') - blanks = 0 - for i, ch in enumerate(lines[0]): - if ch != ' ': - break - for j, l in enumerate(lines): - lines[j] = l[i:] - return '\n'.join(lines) +class _not_given: + def __repr__(self): + return('') +_not_given = _not_given() class _auto_null: def __repr__(self): @@ -206,7 +196,7 @@ def __get__(self, instance, ownerclass=None): # use previous enum.property return self.fget(instance) elif self._attr_type == 'attr': - # look up previous attibute + # look up previous attribute return getattr(self._cls_type, self.name) elif self._attr_type == 'desc': # use previous descriptor @@ -283,9 +273,10 @@ def __set_name__(self, enum_class, member_name): enum_member._sort_order_ = len(enum_class._member_names_) if Flag is not None and issubclass(enum_class, Flag): - enum_class._flag_mask_ |= value - if _is_single_bit(value): - enum_class._singles_mask_ |= value + if isinstance(value, int): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 # If another member with the same value was already defined, the @@ -313,72 +304,40 @@ def __set_name__(self, enum_class, member_name): elif ( Flag is not None and issubclass(enum_class, Flag) + and isinstance(value, int) and _is_single_bit(value) ): # no other instances found, record this member in _member_names_ enum_class._member_names_.append(member_name) - # if necessary, get redirect in place and then add it to _member_map_ - found_descriptor = None - descriptor_type = None - class_type = None - for base in enum_class.__mro__[1:]: - attr = base.__dict__.get(member_name) - if attr is not None: - if isinstance(attr, (property, DynamicClassAttribute)): - found_descriptor = attr - class_type = base - descriptor_type = 'enum' - break - elif _is_descriptor(attr): - found_descriptor = attr - descriptor_type = descriptor_type or 'desc' - class_type = class_type or base - continue - else: - descriptor_type = 'attr' - class_type = base - if found_descriptor: - redirect = property() - redirect.member = enum_member - redirect.__set_name__(enum_class, member_name) - if descriptor_type in ('enum','desc'): - # earlier descriptor found; copy fget, fset, fdel to this one. - redirect.fget = getattr(found_descriptor, 'fget', None) - redirect._get = getattr(found_descriptor, '__get__', None) - redirect.fset = getattr(found_descriptor, 'fset', None) - redirect._set = getattr(found_descriptor, '__set__', None) - redirect.fdel = getattr(found_descriptor, 'fdel', None) - redirect._del = getattr(found_descriptor, '__delete__', None) - redirect._attr_type = descriptor_type - redirect._cls_type = class_type - setattr(enum_class, member_name, redirect) - else: - setattr(enum_class, member_name, enum_member) - # now add to _member_map_ (even aliases) - enum_class._member_map_[member_name] = enum_member + + enum_class._add_member_(member_name, enum_member) try: # This may fail if value is not hashable. We can't add the value # to the map, and by-value lookups for this value will be # linear. enum_class._value2member_map_.setdefault(value, enum_member) + if value not in enum_class._hashable_values_: + enum_class._hashable_values_.append(value) except TypeError: # keep track of the value in a list so containment checks are quick enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(member_name, []).append(value) -class _EnumDict(dict): +class EnumDict(dict): """ Track enum member order and ensure member names are not reused. EnumType will use the names found in self._member_names as the enumeration member names. """ - def __init__(self): + def __init__(self, cls_name=None): super().__init__() - self._member_names = {} # use a dict to keep insertion order + self._member_names = {} # use a dict -- faster look-up than a list, and keeps insertion order since 3.7 self._last_values = [] self._ignore = [] self._auto_called = False + self._cls_name = cls_name def __setitem__(self, key, value): """ @@ -389,23 +348,19 @@ def __setitem__(self, key, value): Single underscore (sunder) names are reserved. """ - if _is_internal_class(self._cls_name, value): - import warnings - warnings.warn( - "In 3.13 classes created inside an enum will not become a member. " - "Use the `member` decorator to keep the current behavior.", - DeprecationWarning, - stacklevel=2, - ) - if _is_private(self._cls_name, key): - # also do nothing, name will be a normal attribute + if self._cls_name is not None and _is_private(self._cls_name, key): + # do nothing, name will be a normal attribute pass elif _is_sunder(key): if key not in ( '_order_', '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', - ): + '_add_alias_', '_add_value_alias_', + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' % (key, ) @@ -441,10 +396,9 @@ def __setitem__(self, key, value): value = value.value elif _is_descriptor(value): pass - # TODO: uncomment next three lines in 3.13 - # elif _is_internal_class(self._cls_name, value): - # # do nothing, name will be a normal attribute - # pass + elif self._cls_name is not None and _is_internal_class(self._cls_name, value): + # do nothing, name will be a normal attribute + pass else: if key in self: # enum overwriting a descriptor? @@ -457,10 +411,11 @@ def __setitem__(self, key, value): if isinstance(value, auto): single = True value = (value, ) - if type(value) is tuple and any(isinstance(v, auto) for v in value): + if isinstance(value, tuple) and any(isinstance(v, auto) for v in value): # insist on an actual tuple, no subclasses, in keeping with only supporting # top-level auto() usage (not contained in any other data structure) auto_valued = [] + t = type(value) for v in value: if isinstance(v, auto): non_auto_store = False @@ -475,12 +430,21 @@ def __setitem__(self, key, value): if single: value = auto_valued[0] else: - value = tuple(auto_valued) + try: + # accepts iterable as multiple arguments? + value = t(auto_valued) + except TypeError: + # then pass them in singly + value = t(*auto_valued) self._member_names[key] = None if non_auto_store: self._last_values.append(value) super().__setitem__(key, value) + @property + def member_names(self): + return list(self._member_names) + def update(self, members, **more_members): try: for name in members.keys(): @@ -491,6 +455,8 @@ def update(self, members, **more_members): for name, value in more_members.items(): self[name] = value +_EnumDict = EnumDict # keep private name for backwards compatibility + class EnumType(type): """ @@ -502,8 +468,7 @@ def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members_(cls, bases) # create the namespace dict - enum_dict = _EnumDict() - enum_dict._cls_name = cls + enum_dict = EnumDict(cls) # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) if first_enum is not None: @@ -564,7 +529,9 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_member_names_'] = [] classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} - classdict['_unhashable_values_'] = [] + classdict['_hashable_values_'] = [] # for comparing with non-hashable types + classdict['_unhashable_values_'] = [] # e.g. frozenset() with set() + classdict['_unhashable_values_map_'] = {} classdict['_member_type_'] = member_type # now set the __repr__ for the value classdict['_value_repr_'] = metacls._find_data_repr_(cls, bases) @@ -579,15 +546,16 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_all_bits_'] = 0 classdict['_inverted_'] = None try: - exc = None + classdict['_%s__in_progress' % cls] = True enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) - except RuntimeError as e: - # any exceptions raised by member.__new__ will get converted to a - # RuntimeError, so get that original exception back and raise it instead - exc = e.__cause__ or e - if exc is not None: - raise exc - # + classdict['_%s__in_progress' % cls] = False + delattr(enum_class, '_%s__in_progress' % cls) + except Exception as e: + # since 3.12 the note "Error calling __set_name__ on '_proto_member' instance ..." + # is tacked on to the error instead of raising a RuntimeError, so discard it + if hasattr(e, '__notes__'): + del e.__notes__ + raise # update classdict with any changes made by __init_subclass__ classdict.update(enum_class.__dict__) # @@ -706,7 +674,7 @@ def __bool__(cls): """ return True - def __call__(cls, value, names=None, *values, module=None, qualname=None, type=None, start=1, boundary=None): + def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, type=None, start=1, boundary=None): """ Either returns an existing member, or creates a new enum class. @@ -735,18 +703,18 @@ def __call__(cls, value, names=None, *values, module=None, qualname=None, type=N """ if cls._member_map_: # simple value lookup if members exist - if names: + if names is not _not_given: value = (value, names) + values return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type - if names is None and type is None: + if names is _not_given and type is None: # no body? no data-type? possibly wrong usage raise TypeError( f"{cls} has no members; specify `names=()` if you meant to create a new, empty, enum" ) return cls._create_( class_name=value, - names=names, + names=None if names is _not_given else names, module=module, qualname=qualname, type=type, @@ -760,10 +728,20 @@ def __contains__(cls, value): `value` is in `cls` if: 1) `value` is a member of `cls`, or 2) `value` is the value of one of the `cls`'s members. + 3) `value` is a pseudo-member (flags) """ if isinstance(value, cls): return True - return value in cls._value2member_map_ or value in cls._unhashable_values_ + if issubclass(cls, Flag): + try: + result = cls._missing_(value) + return isinstance(result, cls) + except ValueError: + pass + return ( + value in cls._unhashable_values_ # both structures are lists + or value in cls._hashable_values_ + ) def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute @@ -1059,7 +1037,70 @@ def _find_new_(mcls, classdict, member_type, first_enum): else: use_args = True return __new__, save_new, use_args -EnumMeta = EnumType + + def _add_member_(cls, name, member): + # _value_ structures are not updated + if name in cls._member_map_: + if cls._member_map_[name] is not member: + raise NameError('%r is already bound: %r' % (name, cls._member_map_[name])) + return + # + # if necessary, get redirect in place and then add it to _member_map_ + found_descriptor = None + descriptor_type = None + class_type = None + for base in cls.__mro__[1:]: + attr = base.__dict__.get(name) + if attr is not None: + if isinstance(attr, (property, DynamicClassAttribute)): + found_descriptor = attr + class_type = base + descriptor_type = 'enum' + break + elif _is_descriptor(attr): + found_descriptor = attr + descriptor_type = descriptor_type or 'desc' + class_type = class_type or base + continue + else: + descriptor_type = 'attr' + class_type = base + if found_descriptor: + redirect = property() + redirect.member = member + redirect.__set_name__(cls, name) + if descriptor_type in ('enum', 'desc'): + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = getattr(found_descriptor, 'fget', None) + redirect._get = getattr(found_descriptor, '__get__', None) + redirect.fset = getattr(found_descriptor, 'fset', None) + redirect._set = getattr(found_descriptor, '__set__', None) + redirect.fdel = getattr(found_descriptor, 'fdel', None) + redirect._del = getattr(found_descriptor, '__delete__', None) + redirect._attr_type = descriptor_type + redirect._cls_type = class_type + setattr(cls, name, redirect) + else: + setattr(cls, name, member) + # now add to _member_map_ (even aliases) + cls._member_map_[name] = member + + @property + def __signature__(cls): + from inspect import Parameter, Signature + if cls._member_names_: + return Signature([Parameter('values', Parameter.VAR_POSITIONAL)]) + else: + return Signature([Parameter('new_class_name', Parameter.POSITIONAL_ONLY), + Parameter('names', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('module', Parameter.KEYWORD_ONLY, default=None), + Parameter('qualname', Parameter.KEYWORD_ONLY, default=None), + Parameter('type', Parameter.KEYWORD_ONLY, default=None), + Parameter('start', Parameter.KEYWORD_ONLY, default=1), + Parameter('boundary', Parameter.KEYWORD_ONLY, default=None)]) + + +EnumMeta = EnumType # keep EnumMeta name for backwards compatibility class Enum(metaclass=EnumType): @@ -1102,13 +1143,6 @@ class Enum(metaclass=EnumType): attributes -- see the documentation for details. """ - @classmethod - def __signature__(cls): - if cls._member_names_: - return '(*values)' - else: - return '(new_class_name, /, names, *, module=None, qualname=None, type=None, start=1, boundary=None)' - def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' @@ -1125,12 +1159,17 @@ def __new__(cls, value): pass except TypeError: # not there, now do long search -- O(n) behavior - for member in cls._member_map_.values(): - if member._value_ == value: - return member + for name, unhashable_values in cls._unhashable_values_map_.items(): + if value in unhashable_values: + return cls[name] + for name, member in cls._member_map_.items(): + if value == member._value_: + return cls[name] # still not found -- verify that members exist, in-case somebody got here mistakenly # (such as via super when trying to override __new__) if not cls._member_map_: + if getattr(cls, '_%s__in_progress' % cls.__name__, False): + raise TypeError('do not use `super().__new__; call the appropriate __new__ directly') from None raise TypeError("%r has no members defined" % cls) # # still not found -- try _missing_ hook @@ -1165,8 +1204,33 @@ def __new__(cls, value): exc = None ve_exc = None - def __init__(self, *args, **kwds): - pass + def _add_alias_(self, name): + self.__class__._add_member_(name, self) + + def _add_value_alias_(self, value): + cls = self.__class__ + try: + if value in cls._value2member_map_: + if cls._value2member_map_[value] is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + except TypeError: + # unhashable value, do long search + for m in cls._member_map_.values(): + if m._value_ == value: + if m is not self: + raise ValueError('%r is already bound: %r' % (value, cls._value2member_map_[value])) + return + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + cls._value2member_map_.setdefault(value, self) + cls._hashable_values_.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + cls._unhashable_values_.append(value) + cls._unhashable_values_map_.setdefault(self.name, []).append(value) @staticmethod def _generate_next_value_(name, start, count, last_values): @@ -1181,28 +1245,13 @@ def _generate_next_value_(name, start, count, last_values): if not last_values: return start try: - last = last_values[-1] - last_values.sort() - if last == last_values[-1]: - # no difference between old and new methods - return last + 1 - else: - # trigger old method (with warning) - raise TypeError + last_value = sorted(last_values).pop() except TypeError: - import warnings - warnings.warn( - "In 3.13 the default `auto()`/`_generate_next_value_` will require all values to be sortable and support adding +1\n" - "and the value returned will be the largest value in the enum incremented by 1", - DeprecationWarning, - stacklevel=3, - ) - for v in reversed(last_values): - try: - return v + 1 - except TypeError: - pass - return start + raise TypeError('unable to sort non-numeric values') from None + try: + return last_value + 1 + except TypeError: + raise TypeError('unable to increment %r' % (last_value, )) from None @classmethod def _missing_(cls, value): @@ -1217,14 +1266,13 @@ def __str__(self): def __dir__(self): """ - Returns all members and all public methods + Returns public methods and other interesting attributes. """ - if self.__class__._member_type_ is object: - interesting = set(['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'value']) - else: + interesting = set() + if self.__class__._member_type_ is not object: interesting = set(object.__dir__(self)) for name in getattr(self, '__dict__', []): - if name[0] != '_': + if name[0] != '_' and name not in self._member_map_: interesting.add(name) for cls in self.__class__.mro(): for name, obj in cls.__dict__.items(): @@ -1237,7 +1285,7 @@ def __dir__(self): else: # in case it was added by `dir(self)` interesting.discard(name) - else: + elif name not in self._member_map_: interesting.add(name) names = sorted( set(['__class__', '__doc__', '__eq__', '__hash__', '__module__']) @@ -1525,37 +1573,50 @@ def __str__(self): def __bool__(self): return bool(self._value_) + def _get_value(self, flag): + if isinstance(flag, self.__class__): + return flag._value_ + elif self._member_type_ is not object and isinstance(flag, self._member_type_): + return flag + return NotImplemented + def __or__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") value = self._value_ - return self.__class__(value | other) + return self.__class__(value | other_value) def __and__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") value = self._value_ - return self.__class__(value & other) + return self.__class__(value & other_value) def __xor__(self, other): - if isinstance(other, self.__class__): - other = other._value_ - elif self._member_type_ is not object and isinstance(other, self._member_type_): - other = other - else: + other_value = self._get_value(other) + if other_value is NotImplemented: return NotImplemented + + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") value = self._value_ - return self.__class__(value ^ other) + return self.__class__(value ^ other_value) def __invert__(self): + if self._get_value(self) is None: + raise TypeError(f"'{self}' cannot be inverted") + if self._inverted_ is None: if self._boundary_ in (EJECT, KEEP): self._inverted_ = self.__class__(~self._value_) @@ -1622,7 +1683,7 @@ def global_flag_repr(self): cls_name = self.__class__.__name__ if self._name_ is None: return "%s.%s(%r)" % (module, cls_name, self._value_) - if _is_single_bit(self): + if _is_single_bit(self._value_): return '%s.%s' % (module, self._name_) if self._boundary_ is not FlagBoundary.KEEP: return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) @@ -1665,7 +1726,7 @@ def _simple_enum(etype=Enum, *, boundary=None, use_args=None): Class decorator that converts a normal class into an :class:`Enum`. No safety checks are done, and some advanced behavior (such as :func:`__init_subclass__`) is not available. Enum creation can be faster - using :func:`simple_enum`. + using :func:`_simple_enum`. >>> from enum import Enum, _simple_enum >>> @_simple_enum(Enum) @@ -1696,7 +1757,9 @@ def convert_class(cls): body['_member_names_'] = member_names = [] body['_member_map_'] = member_map = {} body['_value2member_map_'] = value2member_map = {} - body['_unhashable_values_'] = [] + body['_hashable_values_'] = hashable_values = [] + body['_unhashable_values_'] = unhashable_values = [] + body['_unhashable_values_map_'] = {} body['_member_type_'] = member_type = etype._member_type_ body['_value_repr_'] = etype._value_repr_ if issubclass(etype, Flag): @@ -1743,35 +1806,42 @@ def convert_class(cls): for name, value in attrs.items(): if isinstance(value, auto) and auto.value is _auto_null: value = gnv(name, 1, len(member_names), gnv_last_values) - if value in value2member_map: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member.value in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member member._sort_order_ = len(member_names) + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) value2member_map[value] = member + hashable_values.append(value) if _is_single_bit(value): # not a multi-bit alias, record in _member_names_ and _flag_mask_ member_names.append(name) @@ -1793,37 +1863,53 @@ def convert_class(cls): if value.value is _auto_null: value.value = gnv(name, 1, len(member_names), gnv_last_values) value = value.value - if value in value2member_map: + # create basic member (possibly isolate value for alias check) + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + # now check if alias + try: + contained = value2member_map.get(member._value_) + except TypeError: + contained = None + if member._value_ in unhashable_values or member._value_ in hashable_values: + for m in enum_class: + if m._value_ == member._value_: + contained = m + break + if contained is not None: # an alias to an existing member - member = value2member_map[value] - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member + contained._add_alias_(name) else: - # create the member - if use_args: - if not isinstance(value, tuple): - value = (value, ) - member = new_member(enum_class, *value) - value = value[0] - else: - member = new_member(enum_class) - if __new__ is None: - member._value_ = value + # finish creating member member._name_ = name member.__objclass__ = enum_class member.__init__(value) member._sort_order_ = len(member_names) - redirect = property() - redirect.member = member - redirect.__set_name__(enum_class, name) - setattr(enum_class, name, redirect) - member_map[name] = member - value2member_map[value] = member + if name not in ('name', 'value'): + setattr(enum_class, name, member) + member_map[name] = member + else: + enum_class._add_member_(name, member) member_names.append(name) gnv_last_values.append(value) + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, member) + if value not in hashable_values: + hashable_values.append(value) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + enum_class._unhashable_values_map_.setdefault(name, []).append(value) if '__new__' in body: enum_class.__new_member__ = enum_class.__new__ enum_class.__new__ = Enum.__new__ @@ -1880,7 +1966,7 @@ def __call__(self, enumeration): if 2**i not in values: missing.append(2**i) elif enum_type == 'enum': - # check for powers of one + # check for missing consecutive integers for i in range(low+1, high): if i not in values: missing.append(i) @@ -1908,7 +1994,8 @@ def __call__(self, enumeration): missed = [v for v in values if v not in member_values] if missed: missing_names.append(name) - missing_value |= reduce(_or_, missed) + for val in missed: + missing_value |= val if missing_names: if len(missing_names) == 1: alias = 'alias %s is missing' % missing_names[0] @@ -1941,8 +2028,7 @@ def _test_simple_enum(checked_enum, simple_enum): ... RED = auto() ... GREEN = auto() ... BLUE = auto() - ... # TODO: RUSTPYTHON - >>> _test_simple_enum(CheckedColor, Color) # doctest: +SKIP + >>> _test_simple_enum(CheckedColor, Color) If differences are found, a :exc:`TypeError` is raised. """ @@ -1957,7 +2043,8 @@ def _test_simple_enum(checked_enum, simple_enum): + list(simple_enum._member_map_.keys()) ) for key in set(checked_keys + simple_keys): - if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__'): + if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', + '__static_attributes__', '__firstlineno__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 950b2afd4c1..c5b8d854d77 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -10,10 +10,7 @@ """ -try: - import os -except ImportError: - import _dummy_os as os +import os import stat from itertools import filterfalse from types import GenericAlias @@ -91,12 +88,15 @@ def _do_cmp(f1, f2): class dircmp: """A class that manages the comparison of 2 directories. - dircmp(a, b, ignore=None, hide=None) + dircmp(a, b, ignore=None, hide=None, *, shallow=True) A and B are directories. IGNORE is a list of names to ignore, defaults to DEFAULT_IGNORES. HIDE is a list of names to hide, defaults to [os.curdir, os.pardir]. + SHALLOW specifies whether to just check the stat signature (do not read + the files). + defaults to True. High level usage: x = dircmp(dir1, dir2) @@ -124,7 +124,7 @@ class dircmp: in common_dirs. """ - def __init__(self, a, b, ignore=None, hide=None): # Initialize + def __init__(self, a, b, ignore=None, hide=None, *, shallow=True): # Initialize self.left = a self.right = b if hide is None: @@ -135,6 +135,7 @@ def __init__(self, a, b, ignore=None, hide=None): # Initialize self.ignore = DEFAULT_IGNORES else: self.ignore = ignore + self.shallow = shallow def phase0(self): # Compare everything except common subdirectories self.left_list = _filter(os.listdir(self.left), @@ -160,17 +161,19 @@ def phase2(self): # Distinguish files, directories, funnies a_path = os.path.join(self.left, x) b_path = os.path.join(self.right, x) - ok = 1 + ok = True try: a_stat = os.stat(a_path) - except OSError: + except (OSError, ValueError): + # See https://github.com/python/cpython/issues/122400 + # for the rationale for protecting against ValueError. # print('Can\'t stat', a_path, ':', why.args[1]) - ok = 0 + ok = False try: b_stat = os.stat(b_path) - except OSError: + except (OSError, ValueError): # print('Can\'t stat', b_path, ':', why.args[1]) - ok = 0 + ok = False if ok: a_type = stat.S_IFMT(a_stat.st_mode) @@ -187,7 +190,7 @@ def phase2(self): # Distinguish files, directories, funnies self.common_funny.append(x) def phase3(self): # Find out differences between common files - xx = cmpfiles(self.left, self.right, self.common_files) + xx = cmpfiles(self.left, self.right, self.common_files, self.shallow) self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories @@ -199,7 +202,8 @@ def phase4(self): # Find out differences between common subdirectories for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) - self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide) + self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide, + shallow=self.shallow) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() @@ -245,7 +249,7 @@ def report_full_closure(self): # Report on self and subdirs recursively methodmap = dict(subdirs=phase4, same_files=phase3, diff_files=phase3, funny_files=phase3, - common_dirs = phase2, common_files=phase2, common_funny=phase2, + common_dirs=phase2, common_files=phase2, common_funny=phase2, common=phase1, left_only=phase1, right_only=phase1, left_list=phase0, right_list=phase0) @@ -283,12 +287,12 @@ def cmpfiles(a, b, common, shallow=True): # Return: # 0 for equal # 1 for different -# 2 for funny cases (can't stat, etc.) +# 2 for funny cases (can't stat, NUL bytes, etc.) # def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) - except OSError: + except (OSError, ValueError): return 2 diff --git a/Lib/fileinput.py b/Lib/fileinput.py index e234dc9ea65..3dba3d2fbfa 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -53,7 +53,7 @@ sequence must be accessed in strictly sequential order; sequence access and readline() cannot be mixed. -Optional in-place filtering: if the keyword argument inplace=1 is +Optional in-place filtering: if the keyword argument inplace=True is passed to input() or to the FileInput constructor, the file is moved to a backup file and standard output is directed to the input file. This makes it possible to write a filter that rewrites its input file @@ -399,7 +399,7 @@ def isstdin(self): def hook_compressed(filename, mode, *, encoding=None, errors=None): - if encoding is None: # EncodingWarning is emitted in FileInput() already. + if encoding is None and "b" not in mode: # EncodingWarning is emitted in FileInput() already. encoding = "locale" ext = os.path.splitext(filename)[1] if ext == '.gz': diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index fee59bf73ff..10e1c936688 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -9,18 +9,15 @@ The function translate(PATTERN) returns a regular expression corresponding to PATTERN. (It does not compile it.) """ + +import functools +import itertools import os import posixpath import re -import functools -__all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] +__all__ = ["filter", "filterfalse", "fnmatch", "fnmatchcase", "translate"] -# Build a thread-safe incrementing counter to help create unique regexp group -# names across calls. -from itertools import count -_nextgroupnum = count().__next__ -del count def fnmatch(name, pat): """Test whether FILENAME matches PATTERN. @@ -41,7 +38,8 @@ def fnmatch(name, pat): pat = os.path.normcase(pat) return fnmatchcase(name, pat) -@functools.lru_cache(maxsize=256, typed=True) + +@functools.lru_cache(maxsize=32768, typed=True) def _compile_pattern(pat): if isinstance(pat, bytes): pat_str = str(pat, 'ISO-8859-1') @@ -51,6 +49,7 @@ def _compile_pattern(pat): res = translate(pat) return re.compile(res).match + def filter(names, pat): """Construct a list from those elements of the iterable NAMES that match PAT.""" result = [] @@ -67,6 +66,22 @@ def filter(names, pat): result.append(name) return result + +def filterfalse(names, pat): + """Construct a list from those elements of the iterable NAMES that do not match PAT.""" + pat = os.path.normcase(pat) + match = _compile_pattern(pat) + if os.path is posixpath: + # normcase on posix is NOP. Optimize it away from the loop. + return list(itertools.filterfalse(match, names)) + + result = [] + for name in names: + if match(os.path.normcase(name)) is None: + result.append(name) + return result + + def fnmatchcase(name, pat): """Test whether FILENAME matches PATTERN, including case. @@ -83,19 +98,32 @@ def translate(pat): There is no way to quote meta-characters. """ - STAR = object() + parts, star_indices = _translate(pat, '*', '.') + return _join_translated_parts(parts, star_indices) + + +_re_setops_sub = re.compile(r'([&~|])').sub +_re_escape = functools.lru_cache(maxsize=512)(re.escape) + + +def _translate(pat, star, question_mark): res = [] add = res.append + star_indices = [] + i, n = 0, len(pat) while i < n: c = pat[i] i = i+1 if c == '*': + # store the position of the wildcard + star_indices.append(len(res)) + add(star) # compress consecutive `*` into one - if (not res) or res[-1] is not STAR: - add(STAR) + while i < n and pat[i] == '*': + i += 1 elif c == '?': - add('.') + add(question_mark) elif c == '[': j = i if j < n and pat[j] == '!': @@ -134,8 +162,6 @@ def translate(pat): # Hyphens that create ranges shouldn't be escaped. stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') for s in chunks) - # Escape set operations (&&, ~~ and ||). - stuff = re.sub(r'([&~|])', r'\\\1', stuff) i = j+1 if not stuff: # Empty range: never match. @@ -144,56 +170,40 @@ def translate(pat): # Negated empty range: match any character. add('.') else: + # Escape set operations (&&, ~~ and ||). + stuff = _re_setops_sub(r'\\\1', stuff) if stuff[0] == '!': stuff = '^' + stuff[1:] elif stuff[0] in ('^', '['): stuff = '\\' + stuff add(f'[{stuff}]') else: - add(re.escape(c)) - assert i == n - - # Deal with STARs. - inp = res - res = [] - add = res.append - i, n = 0, len(inp) - # Fixed pieces at the start? - while i < n and inp[i] is not STAR: - add(inp[i]) - i += 1 - # Now deal with STAR fixed STAR fixed ... - # For an interior `STAR fixed` pairing, we want to do a minimal - # .*? match followed by `fixed`, with no possibility of backtracking. - # We can't spell that directly, but can trick it into working by matching - # .*?fixed - # in a lookahead assertion, save the matched part in a group, then - # consume that group via a backreference. If the overall match fails, - # the lookahead assertion won't try alternatives. So the translation is: - # (?=(?P.*?fixed))(?P=name) - # Group names are created as needed: g0, g1, g2, ... - # The numbers are obtained from _nextgroupnum() to ensure they're unique - # across calls and across threads. This is because people rely on the - # undocumented ability to join multiple translate() results together via - # "|" to build large regexps matching "one of many" shell patterns. - while i < n: - assert inp[i] is STAR - i += 1 - if i == n: - add(".*") - break - assert inp[i] is not STAR - fixed = [] - while i < n and inp[i] is not STAR: - fixed.append(inp[i]) - i += 1 - fixed = "".join(fixed) - if i == n: - add(".*") - add(fixed) - else: - groupnum = _nextgroupnum() - add(f"(?=(?P.*?{fixed}))(?P=g{groupnum})") + add(_re_escape(c)) assert i == n - res = "".join(res) - return fr'(?s:{res})\Z' + return res, star_indices + + +def _join_translated_parts(parts, star_indices): + if not star_indices: + return fr'(?s:{"".join(parts)})\z' + iter_star_indices = iter(star_indices) + j = next(iter_star_indices) + buffer = parts[:j] # fixed pieces at the start + append, extend = buffer.append, buffer.extend + i = j + 1 + for j in iter_star_indices: + # Now deal with STAR fixed STAR fixed ... + # For an interior `STAR fixed` pairing, we want to do a minimal + # .*? match followed by `fixed`, with no possibility of backtracking. + # Atomic groups ("(?>...)") allow us to spell that directly. + # Note: people rely on the undocumented ability to join multiple + # translate() results together via "|" to build large regexps matching + # "one of many" shell patterns. + append('(?>.*?') + extend(parts[i:j]) + append(')') + i = j + 1 + append('.*') + extend(parts[i:]) + res = ''.join(buffer) + return fr'(?s:{res})\z' diff --git a/Lib/fractions.py b/Lib/fractions.py index f9ac882ec00..a497ee19935 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -3,7 +3,7 @@ """Fraction, infinite-precision, rational numbers.""" -from decimal import Decimal +import functools import math import numbers import operator @@ -20,21 +20,165 @@ # _PyHASH_MODULUS. _PyHASH_INF = sys.hash_info.inf +@functools.lru_cache(maxsize = 1 << 14) +def _hash_algorithm(numerator, denominator): + + # To make sure that the hash of a Fraction agrees with the hash + # of a numerically equal integer, float or Decimal instance, we + # follow the rules for numeric hashes outlined in the + # documentation. (See library docs, 'Built-in Types'). + + try: + dinv = pow(denominator, -1, _PyHASH_MODULUS) + except ValueError: + # ValueError means there is no modular inverse. + hash_ = _PyHASH_INF + else: + # The general algorithm now specifies that the absolute value of + # the hash is + # (|N| * dinv) % P + # where N is self._numerator and P is _PyHASH_MODULUS. That's + # optimized here in two ways: first, for a non-negative int i, + # hash(i) == i % P, but the int hash implementation doesn't need + # to divide, and is faster than doing % P explicitly. So we do + # hash(|N| * dinv) + # instead. Second, N is unbounded, so its product with dinv may + # be arbitrarily expensive to compute. The final answer is the + # same if we use the bounded |N| % P instead, which can again + # be done with an int hash() call. If 0 <= i < P, hash(i) == i, + # so this nested hash() call wastes a bit of time making a + # redundant copy when |N| < P, but can save an arbitrarily large + # amount of computation for large |N|. + hash_ = hash(hash(abs(numerator)) * dinv) + result = hash_ if numerator >= 0 else -hash_ + return -2 if result == -1 else result + _RATIONAL_FORMAT = re.compile(r""" - \A\s* # optional whitespace at the start, - (?P[-+]?) # an optional sign, then - (?=\d|\.\d) # lookahead for digit or .digit - (?P\d*|\d+(_\d+)*) # numerator (possibly empty) - (?: # followed by - (?:/(?P\d+(_\d+)*))? # an optional denominator - | # or - (?:\.(?Pd*|\d+(_\d+)*))? # an optional fractional part - (?:E(?P[-+]?\d+(_\d+)*))? # and optional exponent + \A\s* # optional whitespace at the start, + (?P[-+]?) # an optional sign, then + (?=\d|\.\d) # lookahead for digit or .digit + (?P\d*|\d+(_\d+)*) # numerator (possibly empty) + (?: # followed by + (?:\s*/\s*(?P\d+(_\d+)*))? # an optional denominator + | # or + (?:\.(?P\d*|\d+(_\d+)*))? # an optional fractional part + (?:E(?P[-+]?\d+(_\d+)*))? # and optional exponent ) - \s*\Z # and optional whitespace to finish + \s*\z # and optional whitespace to finish """, re.VERBOSE | re.IGNORECASE) +# Helpers for formatting + +def _round_to_exponent(n, d, exponent, no_neg_zero=False): + """Round a rational number to the nearest multiple of a given power of 10. + + Rounds the rational number n/d to the nearest integer multiple of + 10**exponent, rounding to the nearest even integer multiple in the case of + a tie. Returns a pair (sign: bool, significand: int) representing the + rounded value (-1)**sign * significand * 10**exponent. + + If no_neg_zero is true, then the returned sign will always be False when + the significand is zero. Otherwise, the sign reflects the sign of the + input. + + d must be positive, but n and d need not be relatively prime. + """ + if exponent >= 0: + d *= 10**exponent + else: + n *= 10**-exponent + + # The divmod quotient is correct for round-ties-towards-positive-infinity; + # In the case of a tie, we zero out the least significant bit of q. + q, r = divmod(n + (d >> 1), d) + if r == 0 and d & 1 == 0: + q &= -2 + + sign = q < 0 if no_neg_zero else n < 0 + return sign, abs(q) + + +def _round_to_figures(n, d, figures): + """Round a rational number to a given number of significant figures. + + Rounds the rational number n/d to the given number of significant figures + using the round-ties-to-even rule, and returns a triple + (sign: bool, significand: int, exponent: int) representing the rounded + value (-1)**sign * significand * 10**exponent. + + In the special case where n = 0, returns a significand of zero and + an exponent of 1 - figures, for compatibility with formatting. + Otherwise, the returned significand satisfies + 10**(figures - 1) <= significand < 10**figures. + + d must be positive, but n and d need not be relatively prime. + figures must be positive. + """ + # Special case for n == 0. + if n == 0: + return False, 0, 1 - figures + + # Find integer m satisfying 10**(m - 1) <= abs(n)/d <= 10**m. (If abs(n)/d + # is a power of 10, either of the two possible values for m is fine.) + str_n, str_d = str(abs(n)), str(d) + m = len(str_n) - len(str_d) + (str_d <= str_n) + + # Round to a multiple of 10**(m - figures). The significand we get + # satisfies 10**(figures - 1) <= significand <= 10**figures. + exponent = m - figures + sign, significand = _round_to_exponent(n, d, exponent) + + # Adjust in the case where significand == 10**figures, to ensure that + # 10**(figures - 1) <= significand < 10**figures. + if len(str(significand)) == figures + 1: + significand //= 10 + exponent += 1 + + return sign, significand, exponent + + +# Pattern for matching non-float-style format specifications. +_GENERAL_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P.)? + (?P[<>=^]) + )? + (?P[-+ ]?) + # Alt flag forces a slash and denominator in the output, even for + # integer-valued Fraction objects. + (?P\#)? + # We don't implement the zeropad flag since there's no single obvious way + # to interpret it. + (?P0|[1-9][0-9]*)? + (?P[,_])? +""", re.DOTALL | re.VERBOSE).fullmatch + + +# Pattern for matching float-style format specifications; +# supports 'e', 'E', 'f', 'F', 'g', 'G' and '%' presentation types. +_FLOAT_FORMAT_SPECIFICATION_MATCHER = re.compile(r""" + (?: + (?P.)? + (?P[<>=^]) + )? + (?P[-+ ]?) + (?Pz)? + (?P\#)? + # A '0' that's *not* followed by another digit is parsed as a minimum width + # rather than a zeropad flag. + (?P0(?=[0-9]))? + (?P[0-9]+)? + (?P[,_])? + (?:\. + (?=[,_0-9]) # lookahead for digit or separator + (?P[0-9]+)? + (?P[,_])? + )? + (?P[eEfFgG%]) +""", re.DOTALL | re.VERBOSE).fullmatch + + class Fraction(numbers.Rational): """This class implements rational numbers. @@ -59,7 +203,7 @@ class Fraction(numbers.Rational): __slots__ = ('_numerator', '_denominator') # We're immutable, so use __new__ not __init__ - def __new__(cls, numerator=0, denominator=None, *, _normalize=True): + def __new__(cls, numerator=0, denominator=None): """Constructs a Rational. Takes a string like '3/2' or '1.5', another Rational instance, a @@ -103,7 +247,9 @@ def __new__(cls, numerator=0, denominator=None, *, _normalize=True): self._denominator = numerator.denominator return self - elif isinstance(numerator, (float, Decimal)): + elif (isinstance(numerator, float) or + (not isinstance(numerator, type) and + hasattr(numerator, 'as_integer_ratio'))): # Exact conversion self._numerator, self._denominator = numerator.as_integer_ratio() return self @@ -137,8 +283,8 @@ def __new__(cls, numerator=0, denominator=None, *, _normalize=True): numerator = -numerator else: - raise TypeError("argument should be a string " - "or a Rational instance") + raise TypeError("argument should be a string or a Rational " + "instance or have the as_integer_ratio() method") elif type(numerator) is int is type(denominator): pass # *very* normal case @@ -155,16 +301,37 @@ def __new__(cls, numerator=0, denominator=None, *, _normalize=True): if denominator == 0: raise ZeroDivisionError('Fraction(%s, 0)' % numerator) - if _normalize: - g = math.gcd(numerator, denominator) - if denominator < 0: - g = -g - numerator //= g - denominator //= g + g = math.gcd(numerator, denominator) + if denominator < 0: + g = -g + numerator //= g + denominator //= g self._numerator = numerator self._denominator = denominator return self + @classmethod + def from_number(cls, number): + """Converts a finite real number to a rational number, exactly. + + Beware that Fraction.from_number(0.3) != Fraction(3, 10). + + """ + if type(number) is int: + return cls._from_coprime_ints(number, 1) + + elif isinstance(number, numbers.Rational): + return cls._from_coprime_ints(number.numerator, number.denominator) + + elif (isinstance(number, float) or + (not isinstance(number, type) and + hasattr(number, 'as_integer_ratio'))): + return cls._from_coprime_ints(*number.as_integer_ratio()) + + else: + raise TypeError("argument should be a Rational instance or " + "have the as_integer_ratio() method") + @classmethod def from_float(cls, f): """Converts a finite float to a rational number, exactly. @@ -177,7 +344,7 @@ def from_float(cls, f): elif not isinstance(f, float): raise TypeError("%s.from_float() only takes floats, not %r (%s)" % (cls.__name__, f, type(f).__name__)) - return cls(*f.as_integer_ratio()) + return cls._from_coprime_ints(*f.as_integer_ratio()) @classmethod def from_decimal(cls, dec): @@ -189,13 +356,28 @@ def from_decimal(cls, dec): raise TypeError( "%s.from_decimal() only takes Decimals, not %r (%s)" % (cls.__name__, dec, type(dec).__name__)) - return cls(*dec.as_integer_ratio()) + return cls._from_coprime_ints(*dec.as_integer_ratio()) + + @classmethod + def _from_coprime_ints(cls, numerator, denominator, /): + """Convert a pair of ints to a rational number, for internal use. + + The ratio of integers should be in lowest terms and the denominator + should be positive. + """ + obj = super(Fraction, cls).__new__(cls) + obj._numerator = numerator + obj._denominator = denominator + return obj + + def is_integer(self): + """Return True if the Fraction is an integer.""" + return self._denominator == 1 def as_integer_ratio(self): - """Return the integer ratio as a tuple. + """Return a pair of integers, whose ratio is equal to the original Fraction. - Return a tuple of two integers, whose ratio is equal to the - Fraction and with a positive denominator. + The ratio is in lowest terms and has a positive denominator. """ return (self._numerator, self._denominator) @@ -245,14 +427,16 @@ def limit_denominator(self, max_denominator=1000000): break p0, q0, p1, q1 = p1, q1, p0+a*p1, q2 n, d = d, n-a*d - k = (max_denominator-q0)//q1 - bound1 = Fraction(p0+k*p1, q0+k*q1) - bound2 = Fraction(p1, q1) - if abs(bound2 - self) <= abs(bound1-self): - return bound2 + + # Determine which of the candidates (p0+k*p1)/(q0+k*q1) and p1/q1 is + # closer to self. The distance between them is 1/(q1*(q0+k*q1)), while + # the distance from p1/q1 to self is d/(q1*self._denominator). So we + # need to compare 2*(q0+k*q1) with self._denominator/d. + if 2*d*(q0+k*q1) <= self._denominator: + return Fraction._from_coprime_ints(p1, q1) else: - return bound1 + return Fraction._from_coprime_ints(p0+k*p1, q0+k*q1) @property def numerator(a): @@ -274,7 +458,163 @@ def __str__(self): else: return '%s/%s' % (self._numerator, self._denominator) - def _operator_fallbacks(monomorphic_operator, fallback_operator): + def _format_general(self, match): + """Helper method for __format__. + + Handles fill, alignment, signs, and thousands separators in the + case of no presentation type. + """ + # Validate and parse the format specifier. + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + alternate_form = bool(match["alt"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] or '' + + # Determine the body and sign representation. + n, d = self._numerator, self._denominator + if d > 1 or alternate_form: + body = f"{abs(n):{thousands_sep}}/{d:{thousands_sep}}" + else: + body = f"{abs(n):{thousands_sep}}" + sign = '-' if n < 0 else pos_sign + + # Pad with fill character if necessary and return. + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body + + def _format_float_style(self, match): + """Helper method for __format__; handles float presentation types.""" + fill = match["fill"] or " " + align = match["align"] or ">" + pos_sign = "" if match["sign"] == "-" else match["sign"] + no_neg_zero = bool(match["no_neg_zero"]) + alternate_form = bool(match["alt"]) + zeropad = bool(match["zeropad"]) + minimumwidth = int(match["minimumwidth"] or "0") + thousands_sep = match["thousands_sep"] + precision = int(match["precision"] or "6") + frac_sep = match["frac_separators"] or "" + presentation_type = match["presentation_type"] + trim_zeros = presentation_type in "gG" and not alternate_form + trim_point = not alternate_form + exponent_indicator = "E" if presentation_type in "EFG" else "e" + + if align == '=' and fill == '0': + zeropad = True + + # Round to get the digits we need, figure out where to place the point, + # and decide whether to use scientific notation. 'point_pos' is the + # relative to the _end_ of the digit string: that is, it's the number + # of digits that should follow the point. + if presentation_type in "fF%": + exponent = -precision + if presentation_type == "%": + exponent -= 2 + negative, significand = _round_to_exponent( + self._numerator, self._denominator, exponent, no_neg_zero) + scientific = False + point_pos = precision + else: # presentation_type in "eEgG" + figures = ( + max(precision, 1) + if presentation_type in "gG" + else precision + 1 + ) + negative, significand, exponent = _round_to_figures( + self._numerator, self._denominator, figures) + scientific = ( + presentation_type in "eE" + or exponent > 0 + or exponent + figures <= -4 + ) + point_pos = figures - 1 if scientific else -exponent + + # Get the suffix - the part following the digits, if any. + if presentation_type == "%": + suffix = "%" + elif scientific: + suffix = f"{exponent_indicator}{exponent + point_pos:+03d}" + else: + suffix = "" + + # String of output digits, padded sufficiently with zeros on the left + # so that we'll have at least one digit before the decimal point. + digits = f"{significand:0{point_pos + 1}d}" + + # Before padding, the output has the form f"{sign}{leading}{trailing}", + # where `leading` includes thousands separators if necessary and + # `trailing` includes the decimal separator where appropriate. + sign = "-" if negative else pos_sign + leading = digits[: len(digits) - point_pos] + frac_part = digits[len(digits) - point_pos :] + if trim_zeros: + frac_part = frac_part.rstrip("0") + separator = "" if trim_point and not frac_part else "." + if frac_sep: + frac_part = frac_sep.join(frac_part[pos:pos + 3] + for pos in range(0, len(frac_part), 3)) + trailing = separator + frac_part + suffix + + # Do zero padding if required. + if zeropad: + min_leading = minimumwidth - len(sign) - len(trailing) + # When adding thousands separators, they'll be added to the + # zero-padded portion too, so we need to compensate. + leading = leading.zfill( + 3 * min_leading // 4 + 1 if thousands_sep else min_leading + ) + + # Insert thousands separators if required. + if thousands_sep: + first_pos = 1 + (len(leading) - 1) % 3 + leading = leading[:first_pos] + "".join( + thousands_sep + leading[pos : pos + 3] + for pos in range(first_pos, len(leading), 3) + ) + + # We now have a sign and a body. Pad with fill character if necessary + # and return. + body = leading + trailing + padding = fill * (minimumwidth - len(sign) - len(body)) + if align == ">": + return padding + sign + body + elif align == "<": + return sign + body + padding + elif align == "^": + half = len(padding) // 2 + return padding[:half] + sign + body + padding[half:] + else: # align == "=" + return sign + padding + body + + def __format__(self, format_spec, /): + """Format this fraction according to the given format specification.""" + + if match := _GENERAL_FORMAT_SPECIFICATION_MATCHER(format_spec): + return self._format_general(match) + + if match := _FLOAT_FORMAT_SPECIFICATION_MATCHER(format_spec): + # Refuse the temptation to guess if both alignment _and_ + # zero padding are specified. + if match["align"] is None or match["zeropad"] is None: + return self._format_float_style(match) + + raise ValueError( + f"Invalid format specifier {format_spec!r} " + f"for object of type {type(self).__name__!r}" + ) + + def _operator_fallbacks(monomorphic_operator, fallback_operator, + handle_complex=True): """Generates forward and reverse operators given a purely-rational operator and a function from the operator module. @@ -355,12 +695,14 @@ class doesn't subclass a concrete type, there's no """ def forward(a, b): - if isinstance(b, (int, Fraction)): + if isinstance(b, Fraction): return monomorphic_operator(a, b) + elif isinstance(b, int): + return monomorphic_operator(a, Fraction(b)) elif isinstance(b, float): return fallback_operator(float(a), b) - elif isinstance(b, complex): - return fallback_operator(complex(a), b) + elif handle_complex and isinstance(b, complex): + return fallback_operator(float(a), b) else: return NotImplemented forward.__name__ = '__' + fallback_operator.__name__ + '__' @@ -369,11 +711,11 @@ def forward(a, b): def reverse(b, a): if isinstance(a, numbers.Rational): # Includes ints. - return monomorphic_operator(a, b) + return monomorphic_operator(Fraction(a), b) elif isinstance(a, numbers.Real): return fallback_operator(float(a), float(b)) - elif isinstance(a, numbers.Complex): - return fallback_operator(complex(a), complex(b)) + elif handle_complex and isinstance(a, numbers.Complex): + return fallback_operator(complex(a), float(b)) else: return NotImplemented reverse.__name__ = '__r' + fallback_operator.__name__ + '__' @@ -451,40 +793,40 @@ def reverse(b, a): def _add(a, b): """a + b""" - na, da = a.numerator, a.denominator - nb, db = b.numerator, b.denominator + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator g = math.gcd(da, db) if g == 1: - return Fraction(na * db + da * nb, da * db, _normalize=False) + return Fraction._from_coprime_ints(na * db + da * nb, da * db) s = da // g t = na * (db // g) + nb * s g2 = math.gcd(t, g) if g2 == 1: - return Fraction(t, s * db, _normalize=False) - return Fraction(t // g2, s * (db // g2), _normalize=False) + return Fraction._from_coprime_ints(t, s * db) + return Fraction._from_coprime_ints(t // g2, s * (db // g2)) __add__, __radd__ = _operator_fallbacks(_add, operator.add) def _sub(a, b): """a - b""" - na, da = a.numerator, a.denominator - nb, db = b.numerator, b.denominator + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator g = math.gcd(da, db) if g == 1: - return Fraction(na * db - da * nb, da * db, _normalize=False) + return Fraction._from_coprime_ints(na * db - da * nb, da * db) s = da // g t = na * (db // g) - nb * s g2 = math.gcd(t, g) if g2 == 1: - return Fraction(t, s * db, _normalize=False) - return Fraction(t // g2, s * (db // g2), _normalize=False) + return Fraction._from_coprime_ints(t, s * db) + return Fraction._from_coprime_ints(t // g2, s * (db // g2)) __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub) def _mul(a, b): """a * b""" - na, da = a.numerator, a.denominator - nb, db = b.numerator, b.denominator + na, da = a._numerator, a._denominator + nb, db = b._numerator, b._denominator g1 = math.gcd(na, db) if g1 > 1: na //= g1 @@ -493,15 +835,17 @@ def _mul(a, b): if g2 > 1: nb //= g2 da //= g2 - return Fraction(na * nb, db * da, _normalize=False) + return Fraction._from_coprime_ints(na * nb, db * da) __mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul) def _div(a, b): """a / b""" # Same as _mul(), with inversed b. - na, da = a.numerator, a.denominator - nb, db = b.numerator, b.denominator + nb, db = b._numerator, b._denominator + if nb == 0: + raise ZeroDivisionError('Fraction(%s, 0)' % db) + na, da = a._numerator, a._denominator g1 = math.gcd(na, nb) if g1 > 1: na //= g1 @@ -513,7 +857,7 @@ def _div(a, b): n, d = na * db, nb * da if d < 0: n, d = -n, -d - return Fraction(n, d, _normalize=False) + return Fraction._from_coprime_ints(n, d) __truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv) @@ -521,7 +865,7 @@ def _floordiv(a, b): """a // b""" return (a.numerator * b.denominator) // (a.denominator * b.numerator) - __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv) + __floordiv__, __rfloordiv__ = _operator_fallbacks(_floordiv, operator.floordiv, False) def _divmod(a, b): """(a // b, a % b)""" @@ -529,16 +873,16 @@ def _divmod(a, b): div, n_mod = divmod(a.numerator * db, da * b.numerator) return div, Fraction(n_mod, da * db) - __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod) + __divmod__, __rdivmod__ = _operator_fallbacks(_divmod, divmod, False) def _mod(a, b): """a % b""" da, db = a.denominator, b.denominator return Fraction((a.numerator * db) % (b.numerator * da), da * db) - __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod) + __mod__, __rmod__ = _operator_fallbacks(_mod, operator.mod, False) - def __pow__(a, b): + def __pow__(a, b, modulo=None): """a ** b If b is not an integer, the result will be a float or complex @@ -546,30 +890,36 @@ def __pow__(a, b): result will be rational. """ + if modulo is not None: + return NotImplemented if isinstance(b, numbers.Rational): if b.denominator == 1: power = b.numerator if power >= 0: - return Fraction(a._numerator ** power, - a._denominator ** power, - _normalize=False) - elif a._numerator >= 0: - return Fraction(a._denominator ** -power, - a._numerator ** -power, - _normalize=False) + return Fraction._from_coprime_ints(a._numerator ** power, + a._denominator ** power) + elif a._numerator > 0: + return Fraction._from_coprime_ints(a._denominator ** -power, + a._numerator ** -power) + elif a._numerator == 0: + raise ZeroDivisionError('Fraction(%s, 0)' % + a._denominator ** -power) else: - return Fraction((-a._denominator) ** -power, - (-a._numerator) ** -power, - _normalize=False) + return Fraction._from_coprime_ints((-a._denominator) ** -power, + (-a._numerator) ** -power) else: # A fractional power will generally produce an # irrational number. return float(a) ** float(b) - else: + elif isinstance(b, (float, complex)): return float(a) ** b + else: + return NotImplemented - def __rpow__(b, a): + def __rpow__(b, a, modulo=None): """a ** b""" + if modulo is not None: + return NotImplemented if b._denominator == 1 and b._numerator >= 0: # If a is an int, keep it that way if possible. return a ** b._numerator @@ -584,15 +934,15 @@ def __rpow__(b, a): def __pos__(a): """+a: Coerces a subclass instance to Fraction""" - return Fraction(a._numerator, a._denominator, _normalize=False) + return Fraction._from_coprime_ints(a._numerator, a._denominator) def __neg__(a): """-a""" - return Fraction(-a._numerator, a._denominator, _normalize=False) + return Fraction._from_coprime_ints(-a._numerator, a._denominator) def __abs__(a): """abs(a)""" - return Fraction(abs(a._numerator), a._denominator, _normalize=False) + return Fraction._from_coprime_ints(abs(a._numerator), a._denominator) def __int__(a, _index=operator.index): """int(a)""" @@ -610,12 +960,12 @@ def __trunc__(a): def __floor__(a): """math.floor(a)""" - return a.numerator // a.denominator + return a._numerator // a._denominator def __ceil__(a): """math.ceil(a)""" # The negations cleverly convince floordiv to return the ceiling. - return -(-a.numerator // a.denominator) + return -(-a._numerator // a._denominator) def __round__(self, ndigits=None): """round(self, ndigits) @@ -623,10 +973,11 @@ def __round__(self, ndigits=None): Rounds half toward even. """ if ndigits is None: - floor, remainder = divmod(self.numerator, self.denominator) - if remainder * 2 < self.denominator: + d = self._denominator + floor, remainder = divmod(self._numerator, d) + if remainder * 2 < d: return floor - elif remainder * 2 > self.denominator: + elif remainder * 2 > d: return floor + 1 # Deal with the half case: elif floor % 2 == 0: @@ -644,36 +995,7 @@ def __round__(self, ndigits=None): def __hash__(self): """hash(self)""" - - # To make sure that the hash of a Fraction agrees with the hash - # of a numerically equal integer, float or Decimal instance, we - # follow the rules for numeric hashes outlined in the - # documentation. (See library docs, 'Built-in Types'). - - try: - dinv = pow(self._denominator, -1, _PyHASH_MODULUS) - except ValueError: - # ValueError means there is no modular inverse. - hash_ = _PyHASH_INF - else: - # The general algorithm now specifies that the absolute value of - # the hash is - # (|N| * dinv) % P - # where N is self._numerator and P is _PyHASH_MODULUS. That's - # optimized here in two ways: first, for a non-negative int i, - # hash(i) == i % P, but the int hash implementation doesn't need - # to divide, and is faster than doing % P explicitly. So we do - # hash(|N| * dinv) - # instead. Second, N is unbounded, so its product with dinv may - # be arbitrarily expensive to compute. The final answer is the - # same if we use the bounded |N| % P instead, which can again - # be done with an int hash() call. If 0 <= i < P, hash(i) == i, - # so this nested hash() call wastes a bit of time making a - # redundant copy when |N| < P, but can save an arbitrarily large - # amount of computation for large |N|. - hash_ = hash(hash(abs(self._numerator)) * dinv) - result = hash_ if self._numerator >= 0 else -hash_ - return -2 if result == -1 else result + return _hash_algorithm(self._numerator, self._denominator) def __eq__(a, b): """a == b""" diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 7c5a50715f6..50771e8c17c 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -343,7 +343,7 @@ def ntransfercmd(self, cmd, rest=None): connection and the expected size of the transfer. The expected size may be None if it could not be determined. - Optional `rest' argument can be a string that is sent as the + Optional 'rest' argument can be a string that is sent as the argument to a REST command. This is essentially a server marker used to tell the server to skip over any data up to the given marker. @@ -434,10 +434,7 @@ def retrbinary(self, cmd, callback, blocksize=8192, rest=None): """ self.voidcmd('TYPE I') with self.transfercmd(cmd, rest) as conn: - while 1: - data = conn.recv(blocksize) - if not data: - break + while data := conn.recv(blocksize): callback(data) # shutdown ssl layer if _SSLSocket is not None and isinstance(conn, _SSLSocket): @@ -496,10 +493,7 @@ def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None): """ self.voidcmd('TYPE I') with self.transfercmd(cmd, rest) as conn: - while 1: - buf = fp.read(blocksize) - if not buf: - break + while buf := fp.read(blocksize): conn.sendall(buf) if callback: callback(buf) @@ -561,7 +555,7 @@ def dir(self, *args): LIST command. (This *should* only be used for a pathname.)''' cmd = 'LIST' func = None - if args[-1:] and type(args[-1]) != type(''): + if args[-1:] and not isinstance(args[-1], str): args, func = args[:-1], args[-1] for arg in args: if arg: @@ -713,28 +707,12 @@ class FTP_TLS(FTP): '221 Goodbye.' >>> ''' - ssl_version = ssl.PROTOCOL_TLS_CLIENT def __init__(self, host='', user='', passwd='', acct='', - keyfile=None, certfile=None, context=None, - timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, - encoding='utf-8'): - if context is not None and keyfile is not None: - raise ValueError("context and keyfile arguments are mutually " - "exclusive") - if context is not None and certfile is not None: - raise ValueError("context and certfile arguments are mutually " - "exclusive") - if keyfile is not None or certfile is not None: - import warnings - warnings.warn("keyfile and certfile are deprecated, use a " - "custom context instead", DeprecationWarning, 2) - self.keyfile = keyfile - self.certfile = certfile + *, context=None, timeout=_GLOBAL_DEFAULT_TIMEOUT, + source_address=None, encoding='utf-8'): if context is None: - context = ssl._create_stdlib_context(self.ssl_version, - certfile=certfile, - keyfile=keyfile) + context = ssl._create_stdlib_context() self.context = context self._prot_p = False super().__init__(host, user, passwd, acct, @@ -749,7 +727,7 @@ def auth(self): '''Set up secure control connection by using TLS/SSL.''' if isinstance(self.sock, ssl.SSLSocket): raise ValueError("Already using TLS") - if self.ssl_version >= ssl.PROTOCOL_TLS: + if self.context.protocol >= ssl.PROTOCOL_TLS: resp = self.voidcmd('AUTH TLS') else: resp = self.voidcmd('AUTH SSL') @@ -922,11 +900,17 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): def test(): '''Test program. - Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... + Usage: ftplib [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... + + Options: + -d increase debugging level + -r[file] set alternate ~/.netrc file - -d dir - -l list - -p password + Commands: + -l[dir] list directory + -d[dir] change the current directory + -p toggle passive and active mode + file retrieve the file and write it to stdout ''' if len(sys.argv) < 2: @@ -952,15 +936,14 @@ def test(): netrcobj = netrc.netrc(rcfile) except OSError: if rcfile is not None: - sys.stderr.write("Could not open account file" - " -- using anonymous login.") + print("Could not open account file -- using anonymous login.", + file=sys.stderr) else: try: userid, acct, passwd = netrcobj.authenticators(host) - except KeyError: + except (KeyError, TypeError): # no account for host - sys.stderr.write( - "No account -- using anonymous login.") + print("No account -- using anonymous login.", file=sys.stderr) ftp.login(userid, passwd, acct) for file in sys.argv[2:]: if file[:2] == '-l': @@ -973,7 +956,9 @@ def test(): ftp.set_pasv(not ftp.passiveserver) else: ftp.retrbinary('RETR ' + file, \ - sys.stdout.write, 1024) + sys.stdout.buffer.write, 1024) + sys.stdout.buffer.flush() + sys.stdout.flush() ftp.quit() diff --git a/Lib/functools.py b/Lib/functools.py index 2ae4290f983..df4660eef3f 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -6,21 +6,21 @@ # Written by Nick Coghlan , # Raymond Hettinger , # and Łukasz Langa . -# Copyright (C) 2006-2013 Python Software Foundation. +# Copyright (C) 2006 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', - 'cached_property'] + 'cached_property', 'Placeholder'] from abc import get_cache_token from collections import namedtuple -# import types, weakref # Deferred to single_dispatch() +# import weakref # Deferred to single_dispatch() +from operator import itemgetter from reprlib import recursive_repr +from types import GenericAlias, MethodType, MappingProxyType, UnionType from _thread import RLock -from types import GenericAlias - ################################################################################ ### update_wrapper() and wraps() decorator @@ -30,7 +30,7 @@ # wrapper functions that can handle naive introspection WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', - '__annotations__', '__type_params__') + '__annotate__', '__type_params__') WRAPPER_UPDATES = ('__dict__',) def update_wrapper(wrapper, wrapped, @@ -236,14 +236,16 @@ def __ge__(self, other): def reduce(function, sequence, initial=_initial_missing): """ - reduce(function, iterable[, initial]) -> value - - Apply a function of two arguments cumulatively to the items of a sequence - or iterable, from left to right, so as to reduce the iterable to a single - value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates - ((((1+2)+3)+4)+5). If initial is present, it is placed before the items - of the iterable in the calculation, and serves as a default when the - iterable is empty. + reduce(function, iterable, /[, initial]) -> value + + Apply a function of two arguments cumulatively to the items of an iterable, from left to right. + + This effectively reduces the iterable to a single value. If initial is present, + it is placed before the items of the iterable in the calculation, and serves as + a default when the iterable is empty. + + For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) + calculates ((((1 + 2) + 3) + 4) + 5). """ it = iter(sequence) @@ -262,53 +264,138 @@ def reduce(function, sequence, initial=_initial_missing): return value -try: - from _functools import reduce -except ImportError: - pass - ################################################################################ ### partial() argument application ################################################################################ -# Purely functional, no descriptor behaviour -class partial: - """New function with partial application of the given arguments - and keywords. + +class _PlaceholderType: + """The type of the Placeholder singleton. + + Used as a placeholder for partial arguments. """ + __instance = None + __slots__ = () + + def __init_subclass__(cls, *args, **kwargs): + raise TypeError(f"type '{cls.__name__}' is not an acceptable base type") - __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" + def __new__(cls): + if cls.__instance is None: + cls.__instance = object.__new__(cls) + return cls.__instance - def __new__(cls, func, /, *args, **keywords): + def __repr__(self): + return 'Placeholder' + + def __reduce__(self): + return 'Placeholder' + +Placeholder = _PlaceholderType() + +def _partial_prepare_merger(args): + if not args: + return 0, None + nargs = len(args) + order = [] + j = nargs + for i, a in enumerate(args): + if a is Placeholder: + order.append(j) + j += 1 + else: + order.append(i) + phcount = j - nargs + merger = itemgetter(*order) if phcount else None + return phcount, merger + +def _partial_new(cls, func, /, *args, **keywords): + if issubclass(cls, partial): + base_cls = partial if not callable(func): raise TypeError("the first argument must be callable") + else: + base_cls = partialmethod + # func could be a descriptor like classmethod which isn't callable + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError(f"the first argument {func!r} must be a callable " + "or a descriptor") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + for value in keywords.values(): + if value is Placeholder: + raise TypeError("Placeholder cannot be passed as a keyword argument") + if isinstance(func, base_cls): + pto_phcount = func._phcount + tot_args = func.args + if args: + tot_args += args + if pto_phcount: + # merge args with args of `func` which is `partial` + nargs = len(args) + if nargs < pto_phcount: + tot_args += (Placeholder,) * (pto_phcount - nargs) + tot_args = func._merger(tot_args) + if nargs > pto_phcount: + tot_args += args[pto_phcount:] + phcount, merger = _partial_prepare_merger(tot_args) + else: # works for both pto_phcount == 0 and != 0 + phcount, merger = pto_phcount, func._merger + keywords = {**func.keywords, **keywords} + func = func.func + else: + tot_args = args + phcount, merger = _partial_prepare_merger(tot_args) + + self = object.__new__(cls) + self.func = func + self.args = tot_args + self.keywords = keywords + self._phcount = phcount + self._merger = merger + return self + +def _partial_repr(self): + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" - if hasattr(func, "func"): - args = func.args + args - keywords = {**func.keywords, **keywords} - func = func.func +# Purely functional, no descriptor behaviour +class partial: + """New function with partial application of the given arguments + and keywords. + """ - self = super(partial, cls).__new__(cls) + __slots__ = ("func", "args", "keywords", "_phcount", "_merger", + "__dict__", "__weakref__") - self.func = func - self.args = args - self.keywords = keywords - return self + __new__ = _partial_new + __repr__ = recursive_repr()(_partial_repr) def __call__(self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partial' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(*self.args, *args, **keywords) + return self.func(*pto_args, *args, **keywords) - @recursive_repr() - def __repr__(self): - qualname = type(self).__qualname__ - args = [repr(self.func)] - args.extend(repr(x) for x in self.args) - args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - if type(self).__module__ == "functools": - return f"functools.{qualname}({', '.join(args)})" - return f"{qualname}({', '.join(args)})" + def __get__(self, obj, objtype=None): + if obj is None: + return self + return MethodType(self, obj) def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -325,6 +412,10 @@ def __setstate__(self, state): (namespace is not None and not isinstance(namespace, dict))): raise TypeError("invalid partial state") + if args and args[-1] is Placeholder: + raise TypeError("trailing Placeholders are not allowed") + phcount, merger = _partial_prepare_merger(args) + args = tuple(args) # just in case it's a subclass if kwds is None: kwds = {} @@ -337,57 +428,45 @@ def __setstate__(self, state): self.func = func self.args = args self.keywords = kwds + self._phcount = phcount + self._merger = merger + + __class_getitem__ = classmethod(GenericAlias) + try: - from _functools import partial + from _functools import partial, Placeholder, _PlaceholderType except ImportError: pass # Descriptor version -class partialmethod(object): +class partialmethod: """Method descriptor with partial application of the given arguments and keywords. Supports wrapping existing descriptors and handles non-descriptor callables as instance methods. """ - - def __init__(self, func, /, *args, **keywords): - if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) - - # func could be a descriptor like classmethod which isn't callable, - # so we can't inherit from partial (it verifies func is callable) - if isinstance(func, partialmethod): - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called - self.func = func.func - self.args = func.args + args - self.keywords = {**func.keywords, **keywords} - else: - self.func = func - self.args = args - self.keywords = keywords - - def __repr__(self): - args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) - format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + __new__ = _partial_new + __repr__ = _partial_repr def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): + phcount = self._phcount + if phcount: + try: + pto_args = self._merger(self.args + args) + args = args[phcount:] + except IndexError: + raise TypeError("missing positional arguments " + "in 'partialmethod' call; expected " + f"at least {phcount}, got {len(args)}") + else: + pto_args = self.args keywords = {**self.keywords, **keywords} - return self.func(cls_or_self, *self.args, *args, **keywords) + return self.func(cls_or_self, *pto_args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ - _method._partialmethod = self + _method.__partialmethod__ = self return _method def __get__(self, obj, cls=None): @@ -423,28 +502,23 @@ def _unwrap_partial(func): func = func.func return func +def _unwrap_partialmethod(func): + prev = None + while func is not prev: + prev = func + while isinstance(getattr(func, "__partialmethod__", None), partialmethod): + func = func.__partialmethod__ + while isinstance(func, partialmethod): + func = getattr(func, 'func') + func = _unwrap_partial(func) + return func + ################################################################################ ### LRU Cache function decorator ################################################################################ _CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) -class _HashedSeq(list): - """ This class guarantees that hash() will be called no more than once - per element. This is important because the lru_cache() will hash - the key multiple times on a cache miss. - - """ - - __slots__ = 'hashvalue' - - def __init__(self, tup, hash=hash): - self[:] = tup - self.hashvalue = hash(tup) - - def __hash__(self): - return self.hashvalue - def _make_key(args, kwds, typed, kwd_mark = (object(),), fasttypes = {int, str}, @@ -474,7 +548,7 @@ def _make_key(args, kwds, typed, key += tuple(type(v) for v in kwds.values()) elif len(key) == 1 and type(key[0]) in fasttypes: return key[0] - return _HashedSeq(key) + return key def lru_cache(maxsize=128, typed=False): """Least-recently-used cache decorator. @@ -483,8 +557,9 @@ def lru_cache(maxsize=128, typed=False): can grow without bound. If *typed* is True, arguments of different types will be cached separately. - For example, f(3.0) and f(3) will be treated as distinct calls with - distinct results. + For example, f(decimal.Decimal("3.0")) and f(3.0) will be treated as + distinct calls with distinct results. Some types such as str and int may + be cached separately even when typed is false. Arguments to the cached function must be hashable. @@ -660,7 +735,7 @@ def cache(user_function, /): def _c3_merge(sequences): """Merges MROs in *sequences* to a single MRO using the C3 algorithm. - Adapted from https://www.python.org/download/releases/2.3/mro/. + Adapted from https://docs.python.org/3/howto/mro.html. """ result = [] @@ -809,7 +884,7 @@ def singledispatch(func): # There are many programs that use functools without singledispatch, so we # trade-off making singledispatch marginally slower for the benefit of # making start-up of such applications slightly faster. - import types, weakref + import weakref registry = {} dispatch_cache = weakref.WeakKeyDictionary() @@ -838,16 +913,11 @@ def dispatch(cls): dispatch_cache[cls] = impl return impl - def _is_union_type(cls): - from typing import get_origin, Union - return get_origin(cls) in {Union, types.UnionType} - def _is_valid_dispatch_type(cls): if isinstance(cls, type): return True - from typing import get_args - return (_is_union_type(cls) and - all(isinstance(arg, type) for arg in get_args(cls))) + return (isinstance(cls, UnionType) and + all(isinstance(arg, type) for arg in cls.__args__)) def register(cls, func=None): """generic_func.register(cls, func) -> func @@ -865,8 +935,8 @@ def register(cls, func=None): f"Invalid first argument to `register()`. " f"{cls!r} is not a class or union type." ) - ann = getattr(cls, '__annotations__', {}) - if not ann: + ann = getattr(cls, '__annotate__', None) + if ann is None: raise TypeError( f"Invalid first argument to `register()`: {cls!r}. " f"Use either `@register(some_class)` or plain `@register` " @@ -876,23 +946,27 @@ def register(cls, func=None): # only import typing if annotation parsing is necessary from typing import get_type_hints - argname, cls = next(iter(get_type_hints(func).items())) + from annotationlib import Format, ForwardRef + argname, cls = next(iter(get_type_hints(func, format=Format.FORWARDREF).items())) if not _is_valid_dispatch_type(cls): - if _is_union_type(cls): + if isinstance(cls, UnionType): raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} not all arguments are classes." ) + elif isinstance(cls, ForwardRef): + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} is an unresolved forward reference." + ) else: raise TypeError( f"Invalid annotation for {argname!r}. " f"{cls!r} is not a class." ) - if _is_union_type(cls): - from typing import get_args - - for arg in get_args(cls): + if isinstance(cls, UnionType): + for arg in cls.__args__: registry[arg] = func else: registry[cls] = func @@ -905,14 +979,13 @@ def wrapper(*args, **kw): if not args: raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return dispatch(args[0].__class__)(*args, **kw) funcname = getattr(func, '__name__', 'singledispatch function') registry[object] = func wrapper.register = register wrapper.dispatch = dispatch - wrapper.registry = types.MappingProxyType(registry) + wrapper.registry = MappingProxyType(registry) wrapper._clear_cache = dispatch_cache.clear update_wrapper(wrapper, func) return wrapper @@ -922,8 +995,7 @@ def wrapper(*args, **kw): class singledispatchmethod: """Single-dispatch generic method descriptor. - Supports wrapping existing descriptors and handles non-descriptor - callables as instance methods. + Supports wrapping existing descriptors. """ def __init__(self, func): @@ -941,19 +1013,77 @@ def register(self, cls, method=None): return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): - def _method(*args, **kwargs): - method = self.dispatcher.dispatch(args[0].__class__) - return method.__get__(obj, cls)(*args, **kwargs) - - _method.__isabstractmethod__ = self.__isabstractmethod__ - _method.register = self.register - update_wrapper(_method, self.func) - return _method + return _singledispatchmethod_get(self, obj, cls) @property def __isabstractmethod__(self): return getattr(self.func, '__isabstractmethod__', False) + def __repr__(self): + try: + name = self.func.__qualname__ + except AttributeError: + try: + name = self.func.__name__ + except AttributeError: + name = '?' + return f'' + +class _singledispatchmethod_get: + def __init__(self, unbound, obj, cls): + self._unbound = unbound + self._dispatch = unbound.dispatcher.dispatch + self._obj = obj + self._cls = cls + # Set instance attributes which cannot be handled in __getattr__() + # because they conflict with type descriptors. + func = unbound.func + try: + self.__module__ = func.__module__ + except AttributeError: + pass + try: + self.__doc__ = func.__doc__ + except AttributeError: + pass + + def __repr__(self): + try: + name = self.__qualname__ + except AttributeError: + try: + name = self.__name__ + except AttributeError: + name = '?' + if self._obj is not None: + return f'' + else: + return f'' + + def __call__(self, /, *args, **kwargs): + if not args: + funcname = getattr(self._unbound.func, '__name__', + 'singledispatchmethod method') + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') + return self._dispatch(args[0].__class__).__get__(self._obj, self._cls)(*args, **kwargs) + + def __getattr__(self, name): + # Resolve these attributes lazily to speed up creation of + # the _singledispatchmethod_get instance. + if name not in {'__name__', '__qualname__', '__isabstractmethod__', + '__annotations__', '__type_params__'}: + raise AttributeError + return getattr(self._unbound.func, name) + + @property + def __wrapped__(self): + return self._unbound.func + + @property + def register(self): + return self._unbound.register + ################################################################################ ### cached_property() - property result cached as instance attribute @@ -966,6 +1096,7 @@ def __init__(self, func): self.func = func self.attrname = None self.__doc__ = func.__doc__ + self.__module__ = func.__module__ def __set_name__(self, owner, name): if self.attrname is None: @@ -1004,3 +1135,31 @@ def __get__(self, instance, owner=None): return val __class_getitem__ = classmethod(GenericAlias) + +def _warn_python_reduce_kwargs(py_reduce): + @wraps(py_reduce) + def wrapper(*args, **kwargs): + if 'function' in kwargs or 'sequence' in kwargs: + import os + import warnings + warnings.warn( + 'Calling functools.reduce with keyword arguments ' + '"function" or "sequence" ' + 'is deprecated in Python 3.14 and will be ' + 'forbidden in Python 3.16.', + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return py_reduce(*args, **kwargs) + return wrapper + +reduce = _warn_python_reduce_kwargs(reduce) +del _warn_python_reduce_kwargs + +# The import of the C accelerated version of reduce() has been moved +# here due to gh-121676. In Python 3.16, _warn_python_reduce_kwargs() +# should be removed and the import block should be moved back right +# after the definition of reduce(). +try: + from _functools import reduce +except ImportError: + pass diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 309759af257..9363f564aab 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -3,15 +3,12 @@ Do not use directly. The OS specific modules import the appropriate functions from this module themselves. """ -try: - import os -except ImportError: - import _dummy_os as os +import os import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile', - 'samestat'] + 'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink', + 'lexists', 'samefile', 'sameopenfile', 'samestat', 'ALLOW_MISSING'] # Does a path exist? @@ -25,6 +22,15 @@ def exists(path): return True +# Being true for dangling symbolic links is also useful. +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + os.lstat(path) + except (OSError, ValueError): + return False + return True + # This follows symbolic links, so both islink() and isdir() can be true # for the same path on systems that support symlinks def isfile(path): @@ -48,6 +54,33 @@ def isdir(s): return stat.S_ISDIR(st.st_mode) +# Is a path a symbolic link? +# This will always return false on systems where os.lstat doesn't exist. + +def islink(path): + """Test whether a path is a symbolic link""" + try: + st = os.lstat(path) + except (OSError, ValueError, AttributeError): + return False + return stat.S_ISLNK(st.st_mode) + + +# Is a path a junction? +def isjunction(path): + """Test whether a path is a junction + Junctions are not supported on the current platform""" + os.fspath(path) + return False + + +def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive. + Dev Drives are not supported on the current platform""" + os.fspath(path) + return False + + def getsize(filename): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size @@ -156,3 +189,12 @@ def _check_arg_types(funcname, *args): f'os.PathLike object, not {s.__class__.__name__!r}') from None if hasstr and hasbytes: raise TypeError("Can't mix strings and bytes in path components") from None + +# A singleton with a true boolean value. +@object.__new__ +class ALLOW_MISSING: + """Special value for use in realpath().""" + def __repr__(self): + return 'os.path.ALLOW_MISSING' + def __reduce__(self): + return self.__class__.__name__ diff --git a/Lib/getopt.py b/Lib/getopt.py index 9d4cab1bac3..25f3e2439b3 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -2,8 +2,8 @@ This module helps scripts to parse the command line arguments in sys.argv. It supports the same conventions as the Unix getopt() -function (including the special meanings of arguments of the form `-' -and `--'). Long options similar to those supported by GNU software +function (including the special meanings of arguments of the form '-' +and '--'). Long options similar to those supported by GNU software may be used as well via an optional third argument. This module provides two functions and an exception: @@ -24,21 +24,14 @@ # TODO for gnu_getopt(): # # - GNU getopt_long_only mechanism -# - allow the caller to specify ordering -# - RETURN_IN_ORDER option -# - GNU extension with '-' as first character of option string -# - optional arguments, specified by double colons # - an option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] import os -try: - from gettext import gettext as _ -except ImportError: - # Bootstrapping Python: gettext's dependencies not built yet - def _(s): return s +from gettext import gettext as _ + class GetoptError(Exception): opt = '' @@ -61,12 +54,14 @@ def getopt(args, shortopts, longopts = []): running program. Typically, this means "sys.argv[1:]". shortopts is the string of option letters that the script wants to recognize, with options that require an argument followed by a - colon (i.e., the same format that Unix getopt() uses). If + colon and options that accept an optional argument followed by + two colons (i.e., the same format that Unix getopt() uses). If specified, longopts is a list of strings with the names of the long options which should be supported. The leading '--' characters should not be included in the option name. Options which require an argument should be followed by an equal sign - ('='). + ('='). Options which accept an optional argument should be + followed by an equal sign and question mark ('=?'). The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments @@ -81,7 +76,7 @@ def getopt(args, shortopts, longopts = []): """ opts = [] - if type(longopts) == type(""): + if isinstance(longopts, str): longopts = [longopts] else: longopts = list(longopts) @@ -105,7 +100,7 @@ def gnu_getopt(args, shortopts, longopts = []): processing options as soon as a non-option argument is encountered. - If the first character of the option string is `+', or if the + If the first character of the option string is '+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. @@ -118,8 +113,13 @@ def gnu_getopt(args, shortopts, longopts = []): else: longopts = list(longopts) + return_in_order = False + if shortopts.startswith('-'): + shortopts = shortopts[1:] + all_options_first = False + return_in_order = True # Allow options after non-option arguments? - if shortopts.startswith('+'): + elif shortopts.startswith('+'): shortopts = shortopts[1:] all_options_first = True elif os.environ.get("POSIXLY_CORRECT"): @@ -133,8 +133,14 @@ def gnu_getopt(args, shortopts, longopts = []): break if args[0][:2] == '--': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_longs(opts, args[0][2:], longopts, args[1:]) elif args[0][:1] == '-' and args[0] != '-': + if return_in_order and prog_args: + opts.append((None, prog_args)) + prog_args = [] opts, args = do_shorts(opts, args[0][1:], shortopts, args[1:]) else: if all_options_first: @@ -156,7 +162,7 @@ def do_longs(opts, opt, longopts, args): has_arg, opt = long_has_args(opt, longopts) if has_arg: - if optarg is None: + if optarg is None and has_arg != '?': if not args: raise GetoptError(_('option --%s requires argument') % opt, opt) optarg, args = args[0], args[1:] @@ -177,13 +183,19 @@ def long_has_args(opt, longopts): return False, opt elif opt + '=' in possibilities: return True, opt - # No exact match, so better be unique. + elif opt + '=?' in possibilities: + return '?', opt + # Possibilities must be unique to be accepted if len(possibilities) > 1: - # XXX since possibilities contains all valid continuations, might be - # nice to work them into the error msg - raise GetoptError(_('option --%s not a unique prefix') % opt, opt) + raise GetoptError( + _("option --%s not a unique prefix; possible options: %s") + % (opt, ", ".join(possibilities)), + opt, + ) assert len(possibilities) == 1 unique_match = possibilities[0] + if unique_match.endswith('=?'): + return '?', unique_match[:-2] has_arg = unique_match.endswith('=') if has_arg: unique_match = unique_match[:-1] @@ -192,8 +204,9 @@ def long_has_args(opt, longopts): def do_shorts(opts, optstring, shortopts, args): while optstring != '': opt, optstring = optstring[0], optstring[1:] - if short_has_arg(opt, shortopts): - if optstring == '': + has_arg = short_has_arg(opt, shortopts) + if has_arg: + if optstring == '' and has_arg != '?': if not args: raise GetoptError(_('option -%s requires argument') % opt, opt) @@ -207,7 +220,11 @@ def do_shorts(opts, optstring, shortopts, args): def short_has_arg(opt, shortopts): for i in range(len(shortopts)): if opt == shortopts[i] != ':': - return shortopts.startswith(':', i+1) + if not shortopts.startswith(':', i+1): + return False + if shortopts.startswith('::', i+1): + return '?' + return True raise GetoptError(_('option -%s not recognized') % opt, opt) if __name__ == '__main__': diff --git a/Lib/getpass.py b/Lib/getpass.py index 6970d8adfba..3d9bb1f0d14 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -1,6 +1,7 @@ """Utilities to get a password and/or the current user name. -getpass(prompt[, stream]) - Prompt for a password, with echo turned off. +getpass(prompt[, stream[, echo_char]]) - Prompt for a password, with echo +turned off and optional keyboard feedback. getuser() - Get the user name from the environment or password database. GetPassWarning - This UserWarning is issued when getpass() cannot prevent @@ -18,7 +19,6 @@ import io import os import sys -import warnings __all__ = ["getpass","getuser","GetPassWarning"] @@ -26,13 +26,15 @@ class GetPassWarning(UserWarning): pass -def unix_getpass(prompt='Password: ', stream=None): +def unix_getpass(prompt='Password: ', stream=None, *, echo_char=None): """Prompt for a password, with echo turned off. Args: prompt: Written on stream to ask for the input. Default: 'Password: ' stream: A writable file object to display the prompt. Defaults to the tty. If no tty is available defaults to sys.stderr. + echo_char: A single ASCII character to mask input (e.g., '*'). + If None, input is hidden. Returns: The seKr3t input. Raises: @@ -41,6 +43,8 @@ def unix_getpass(prompt='Password: ', stream=None): Always restores terminal settings before returning. """ + _check_echo_char(echo_char) + passwd = None with contextlib.ExitStack() as stack: try: @@ -69,12 +73,16 @@ def unix_getpass(prompt='Password: ', stream=None): old = termios.tcgetattr(fd) # a copy to save new = old[:] new[3] &= ~termios.ECHO # 3 == 'lflags' + if echo_char: + new[3] &= ~termios.ICANON tcsetattr_flags = termios.TCSAFLUSH if hasattr(termios, 'TCSASOFT'): tcsetattr_flags |= termios.TCSASOFT try: termios.tcsetattr(fd, tcsetattr_flags, new) - passwd = _raw_input(prompt, stream, input=input) + passwd = _raw_input(prompt, stream, input=input, + echo_char=echo_char) + finally: termios.tcsetattr(fd, tcsetattr_flags, old) stream.flush() # issue7208 @@ -94,10 +102,11 @@ def unix_getpass(prompt='Password: ', stream=None): return passwd -def win_getpass(prompt='Password: ', stream=None): +def win_getpass(prompt='Password: ', stream=None, *, echo_char=None): """Prompt for password with echo off, using Windows getwch().""" if sys.stdin is not sys.__stdin__: return fallback_getpass(prompt, stream) + _check_echo_char(echo_char) for c in prompt: msvcrt.putwch(c) @@ -109,24 +118,48 @@ def win_getpass(prompt='Password: ', stream=None): if c == '\003': raise KeyboardInterrupt if c == '\b': + if echo_char and pw: + msvcrt.putwch('\b') + msvcrt.putwch(' ') + msvcrt.putwch('\b') pw = pw[:-1] else: pw = pw + c + if echo_char: + msvcrt.putwch(echo_char) msvcrt.putwch('\r') msvcrt.putwch('\n') return pw -def fallback_getpass(prompt='Password: ', stream=None): +def fallback_getpass(prompt='Password: ', stream=None, *, echo_char=None): + _check_echo_char(echo_char) + import warnings warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: stream = sys.stderr print("Warning: Password input may be echoed.", file=stream) - return _raw_input(prompt, stream) + return _raw_input(prompt, stream, echo_char=echo_char) + +def _check_echo_char(echo_char): + # Single-character ASCII excluding control characters + if echo_char is None: + return + if not isinstance(echo_char, str): + raise TypeError("'echo_char' must be a str or None, not " + f"{type(echo_char).__name__}") + if not ( + len(echo_char) == 1 + and echo_char.isprintable() + and echo_char.isascii() + ): + raise ValueError("'echo_char' must be a single printable ASCII " + f"character, got: {echo_char!r}") -def _raw_input(prompt="", stream=None, input=None): + +def _raw_input(prompt="", stream=None, input=None, echo_char=None): # This doesn't save the string in the GNU readline history. if not stream: stream = sys.stderr @@ -143,6 +176,8 @@ def _raw_input(prompt="", stream=None, input=None): stream.write(prompt) stream.flush() # NOTE: The Python C API calls flockfile() (and unlock) during readline. + if echo_char: + return _readline_with_echo_char(stream, input, echo_char) line = input.readline() if not line: raise EOFError @@ -151,12 +186,45 @@ def _raw_input(prompt="", stream=None, input=None): return line +def _readline_with_echo_char(stream, input, echo_char): + passwd = "" + eof_pressed = False + while True: + char = input.read(1) + if char == '\n' or char == '\r': + break + elif char == '\x03': + raise KeyboardInterrupt + elif char == '\x7f' or char == '\b': + if passwd: + stream.write("\b \b") + stream.flush() + passwd = passwd[:-1] + elif char == '\x04': + if eof_pressed: + break + else: + eof_pressed = True + elif char == '\x00': + continue + else: + passwd += char + stream.write(echo_char) + stream.flush() + eof_pressed = False + return passwd + + def getuser(): """Get the username from the environment or password database. First try various environment variables, then the password database. This works on Windows as long as USERNAME is set. + Any failure to find a username raises OSError. + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` + were raised. """ for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): @@ -164,9 +232,12 @@ def getuser(): if user: return user - # If this fails, the exception will "explain" why - import pwd - return pwd.getpwuid(os.getuid())[0] + try: + import pwd + return pwd.getpwuid(os.getuid())[0] + except (ImportError, KeyError) as e: + raise OSError('No username set in the environment') from e + # Bind the name getpass to the appropriate function try: diff --git a/Lib/gettext.py b/Lib/gettext.py index 4c3b80b0239..6c11ab2b1eb 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -41,22 +41,18 @@ # to do binary searches and lazy initializations. Or you might want to use # the undocumented double-hash algorithm for .mo files with hash tables, but # you'll need to study the GNU gettext code to do this. -# -# - Support Solaris .mo file formats. Unfortunately, we've been unable to -# find this format documented anywhere. -import locale +import operator import os -import re import sys __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog', - 'find', 'translation', 'install', 'textdomain', 'bindtextdomain', - 'bind_textdomain_codeset', - 'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext', - 'ldngettext', 'lngettext', 'ngettext', + 'bindtextdomain', 'find', 'translation', 'install', + 'textdomain', 'dgettext', 'dngettext', 'gettext', + 'ngettext', 'pgettext', 'dpgettext', 'npgettext', + 'dnpgettext' ] _default_localedir = os.path.join(sys.base_prefix, 'share', 'locale') @@ -70,21 +66,26 @@ # https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms # http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y -_token_pattern = re.compile(r""" - (?P[ \t]+) | # spaces and horizontal tabs - (?P[0-9]+\b) | # decimal integer - (?Pn\b) | # only n is allowed - (?P[()]) | - (?P[-*/%+?:]|[>, - # <=, >=, ==, !=, &&, ||, - # ? : - # unary and bitwise ops - # not allowed - (?P\w+|.) # invalid token - """, re.VERBOSE|re.DOTALL) +_token_pattern = None def _tokenize(plural): - for mo in re.finditer(_token_pattern, plural): + global _token_pattern + if _token_pattern is None: + import re + _token_pattern = re.compile(r""" + (?P[ \t]+) | # spaces and horizontal tabs + (?P[0-9]+\b) | # decimal integer + (?Pn\b) | # only n is allowed + (?P[()]) | + (?P[-*/%+?:]|[>, + # <=, >=, ==, !=, &&, ||, + # ? : + # unary and bitwise ops + # not allowed + (?P\w+|.) # invalid token + """, re.VERBOSE|re.DOTALL) + + for mo in _token_pattern.finditer(plural): kind = mo.lastgroup if kind == 'WHITESPACES': continue @@ -94,12 +95,14 @@ def _tokenize(plural): yield value yield '' + def _error(value): if value: return ValueError('unexpected token in plural form: %s' % value) else: return ValueError('unexpected end of plural form') + _binary_ops = ( ('||',), ('&&',), @@ -111,6 +114,7 @@ def _error(value): _binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops} _c2py_ops = {'||': 'or', '&&': 'and', '/': '//'} + def _parse(tokens, priority=-1): result = '' nexttok = next(tokens) @@ -160,18 +164,34 @@ def _parse(tokens, priority=-1): return result, nexttok + def _as_int(n): try: - i = round(n) + round(n) except TypeError: raise TypeError('Plural value must be an integer, got %s' % (n.__class__.__name__,)) from None + return _as_int2(n) + +def _as_int2(n): + try: + return operator.index(n) + except TypeError: + pass + import warnings + frame = sys._getframe(1) + stacklevel = 2 + while frame.f_back is not None and frame.f_globals.get('__name__') == __name__: + stacklevel += 1 + frame = frame.f_back warnings.warn('Plural value must be an integer, got %s' % (n.__class__.__name__,), - DeprecationWarning, 4) + DeprecationWarning, + stacklevel) return n + def c2py(plural): """Gets a C expression as used in PO files for plural forms and returns a Python function that implements an equivalent expression. @@ -195,7 +215,7 @@ def c2py(plural): elif c == ')': depth -= 1 - ns = {'_as_int': _as_int} + ns = {'_as_int': _as_int, '__name__': __name__} exec('''if True: def func(n): if not isinstance(n, int): @@ -209,6 +229,7 @@ def func(n): def _expand_lang(loc): + import locale loc = locale.normalize(loc) COMPONENT_CODESET = 1 << 0 COMPONENT_TERRITORY = 1 << 1 @@ -249,12 +270,10 @@ def _expand_lang(loc): return ret - class NullTranslations: def __init__(self, fp=None): self._info = {} self._charset = None - self._output_charset = None self._fallback = None if fp is not None: self._parse(fp) @@ -273,31 +292,28 @@ def gettext(self, message): return self._fallback.gettext(message) return message - def lgettext(self, message): - if self._fallback: - return self._fallback.lgettext(message) - if self._output_charset: - return message.encode(self._output_charset) - return message.encode(locale.getpreferredencoding()) - def ngettext(self, msgid1, msgid2, n): if self._fallback: return self._fallback.ngettext(msgid1, msgid2, n) + n = _as_int2(n) if n == 1: return msgid1 else: return msgid2 - def lngettext(self, msgid1, msgid2, n): + def pgettext(self, context, message): + if self._fallback: + return self._fallback.pgettext(context, message) + return message + + def npgettext(self, context, msgid1, msgid2, n): if self._fallback: - return self._fallback.lngettext(msgid1, msgid2, n) + return self._fallback.npgettext(context, msgid1, msgid2, n) + n = _as_int2(n) if n == 1: - tmsg = msgid1 + return msgid1 else: - tmsg = msgid2 - if self._output_charset: - return tmsg.encode(self._output_charset) - return tmsg.encode(locale.getpreferredencoding()) + return msgid2 def info(self): return self._info @@ -305,24 +321,13 @@ def info(self): def charset(self): return self._charset - def output_charset(self): - return self._output_charset - - def set_output_charset(self, charset): - self._output_charset = charset - def install(self, names=None): import builtins builtins.__dict__['_'] = self.gettext - if hasattr(names, "__contains__"): - if "gettext" in names: - builtins.__dict__['gettext'] = builtins.__dict__['_'] - if "ngettext" in names: - builtins.__dict__['ngettext'] = self.ngettext - if "lgettext" in names: - builtins.__dict__['lgettext'] = self.lgettext - if "lngettext" in names: - builtins.__dict__['lngettext'] = self.lngettext + if names is not None: + allowed = {'gettext', 'ngettext', 'npgettext', 'pgettext'} + for name in allowed & set(names): + builtins.__dict__[name] = getattr(self, name) class GNUTranslations(NullTranslations): @@ -330,6 +335,10 @@ class GNUTranslations(NullTranslations): LE_MAGIC = 0x950412de BE_MAGIC = 0xde120495 + # The encoding of a msgctxt and a msgid in a .mo file is + # msgctxt + "\x04" + msgid (gettext version >= 0.15) + CONTEXT = "%s\x04%s" + # Acceptable .mo versions VERSIONS = (0, 1) @@ -385,6 +394,9 @@ def _parse(self, fp): item = b_item.decode().strip() if not item: continue + # Skip over comment lines: + if item.startswith('#-#-#-#-#') and item.endswith('#-#-#-#-#'): + continue k = v = None if ':' in item: k, v = item.split(':', 1) @@ -423,46 +435,48 @@ def _parse(self, fp): masteridx += 8 transidx += 8 - def lgettext(self, message): + def gettext(self, message): missing = object() tmsg = self._catalog.get(message, missing) if tmsg is missing: - if self._fallback: - return self._fallback.lgettext(message) - tmsg = message - if self._output_charset: - return tmsg.encode(self._output_charset) - return tmsg.encode(locale.getpreferredencoding()) + tmsg = self._catalog.get((message, self.plural(1)), missing) + if tmsg is not missing: + return tmsg + if self._fallback: + return self._fallback.gettext(message) + return message - def lngettext(self, msgid1, msgid2, n): + def ngettext(self, msgid1, msgid2, n): try: tmsg = self._catalog[(msgid1, self.plural(n))] except KeyError: if self._fallback: - return self._fallback.lngettext(msgid1, msgid2, n) + return self._fallback.ngettext(msgid1, msgid2, n) if n == 1: tmsg = msgid1 else: tmsg = msgid2 - if self._output_charset: - return tmsg.encode(self._output_charset) - return tmsg.encode(locale.getpreferredencoding()) + return tmsg - def gettext(self, message): + def pgettext(self, context, message): + ctxt_msg_id = self.CONTEXT % (context, message) missing = object() - tmsg = self._catalog.get(message, missing) + tmsg = self._catalog.get(ctxt_msg_id, missing) if tmsg is missing: - if self._fallback: - return self._fallback.gettext(message) - return message - return tmsg + tmsg = self._catalog.get((ctxt_msg_id, self.plural(1)), missing) + if tmsg is not missing: + return tmsg + if self._fallback: + return self._fallback.pgettext(context, message) + return message - def ngettext(self, msgid1, msgid2, n): + def npgettext(self, context, msgid1, msgid2, n): + ctxt_msg_id = self.CONTEXT % (context, msgid1) try: - tmsg = self._catalog[(msgid1, self.plural(n))] + tmsg = self._catalog[ctxt_msg_id, self.plural(n)] except KeyError: if self._fallback: - return self._fallback.ngettext(msgid1, msgid2, n) + return self._fallback.npgettext(context, msgid1, msgid2, n) if n == 1: tmsg = msgid1 else: @@ -507,12 +521,12 @@ def find(domain, localedir=None, languages=None, all=False): return result - # a mapping between absolute .mo file path and Translation object _translations = {} + def translation(domain, localedir=None, languages=None, - class_=None, fallback=False, codeset=None): + class_=None, fallback=False): if class_ is None: class_ = GNUTranslations mofiles = find(domain, localedir, languages, all=True) @@ -538,8 +552,6 @@ def translation(domain, localedir=None, languages=None, # are not used. import copy t = copy.copy(t) - if codeset: - t.set_output_charset(codeset) if result is None: result = t else: @@ -547,16 +559,13 @@ def translation(domain, localedir=None, languages=None, return result -def install(domain, localedir=None, codeset=None, names=None): - t = translation(domain, localedir, fallback=True, codeset=codeset) +def install(domain, localedir=None, *, names=None): + t = translation(domain, localedir, fallback=True) t.install(names) - # a mapping b/w domains and locale directories _localedirs = {} -# a mapping b/w domains and codesets -_localecodesets = {} # current global domain, `messages' used for compatibility w/ GNU gettext _current_domain = 'messages' @@ -575,63 +584,61 @@ def bindtextdomain(domain, localedir=None): return _localedirs.get(domain, _default_localedir) -def bind_textdomain_codeset(domain, codeset=None): - global _localecodesets - if codeset is not None: - _localecodesets[domain] = codeset - return _localecodesets.get(domain) - - def dgettext(domain, message): try: - t = translation(domain, _localedirs.get(domain, None), - codeset=_localecodesets.get(domain)) + t = translation(domain, _localedirs.get(domain, None)) except OSError: return message return t.gettext(message) -def ldgettext(domain, message): - codeset = _localecodesets.get(domain) - try: - t = translation(domain, _localedirs.get(domain, None), codeset=codeset) - except OSError: - return message.encode(codeset or locale.getpreferredencoding()) - return t.lgettext(message) def dngettext(domain, msgid1, msgid2, n): try: - t = translation(domain, _localedirs.get(domain, None), - codeset=_localecodesets.get(domain)) + t = translation(domain, _localedirs.get(domain, None)) except OSError: + n = _as_int2(n) if n == 1: return msgid1 else: return msgid2 return t.ngettext(msgid1, msgid2, n) -def ldngettext(domain, msgid1, msgid2, n): - codeset = _localecodesets.get(domain) + +def dpgettext(domain, context, message): + try: + t = translation(domain, _localedirs.get(domain, None)) + except OSError: + return message + return t.pgettext(context, message) + + +def dnpgettext(domain, context, msgid1, msgid2, n): try: - t = translation(domain, _localedirs.get(domain, None), codeset=codeset) + t = translation(domain, _localedirs.get(domain, None)) except OSError: + n = _as_int2(n) if n == 1: - tmsg = msgid1 + return msgid1 else: - tmsg = msgid2 - return tmsg.encode(codeset or locale.getpreferredencoding()) - return t.lngettext(msgid1, msgid2, n) + return msgid2 + return t.npgettext(context, msgid1, msgid2, n) + def gettext(message): return dgettext(_current_domain, message) -def lgettext(message): - return ldgettext(_current_domain, message) def ngettext(msgid1, msgid2, n): return dngettext(_current_domain, msgid1, msgid2, n) -def lngettext(msgid1, msgid2, n): - return ldngettext(_current_domain, msgid1, msgid2, n) + +def pgettext(context, message): + return dpgettext(_current_domain, context, message) + + +def npgettext(context, msgid1, msgid2, n): + return dnpgettext(_current_domain, context, msgid1, msgid2, n) + # dcgettext() has been deemed unnecessary and is not implemented. @@ -641,7 +648,7 @@ def lngettext(msgid1, msgid2, n): # import gettext # cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR) # _ = cat.gettext -# print _('Hello World') +# print(_('Hello World')) # The resulting catalog object currently don't support access through a # dictionary API, which was supported (but apparently unused) in GNOME diff --git a/Lib/glob.py b/Lib/glob.py index a7256422d52..f1a87c82fc5 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -4,11 +4,14 @@ import os import re import fnmatch +import functools import itertools +import operator import stat import sys -__all__ = ["glob", "iglob", "escape"] + +__all__ = ["glob", "iglob", "escape", "translate"] def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): @@ -19,6 +22,9 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, dot are special cases that are not matched by '*' and '?' patterns by default. + The order of the returned list is undefined. Sort it if you need a + particular order. + If `include_hidden` is true, the patterns '*', '?', '**' will match hidden directories. @@ -37,6 +43,9 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, dot are special cases that are not matched by '*' and '?' patterns. + The order of the returned paths is undefined. Sort them if you need a + particular order. + If recursive is true, the pattern '**' will match any files and zero or more directories and subdirectories. """ @@ -104,8 +113,8 @@ def _iglob(pathname, root_dir, dir_fd, recursive, dironly, def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False): names = _listdir(dirname, dir_fd, dironly) - if include_hidden or not _ishidden(pattern): - names = (x for x in names if include_hidden or not _ishidden(x)) + if not (include_hidden or _ishidden(pattern)): + names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): @@ -119,12 +128,19 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -# Following functions are not public but can be used by third-party code. +_deprecated_function_message = ( + "{name} is deprecated and will be removed in Python {remove}. Use " + "glob.glob and pass a directory to its root_dir argument instead." +) def glob0(dirname, pattern): + import warnings + warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): + import warnings + warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) return _glob1(dirname, pattern, None, False) # This helper function recursively yields relative pathnames inside a literal @@ -132,7 +148,8 @@ def glob1(dirname, pattern): def _glob2(dirname, pattern, dir_fd, dironly, include_hidden=False): assert _isrecursive(pattern) - yield pattern[:0] + if not dirname or _isdir(dirname, dir_fd): + yield pattern[:0] yield from _rlistdir(dirname, dir_fd, dironly, include_hidden=include_hidden) @@ -248,4 +265,289 @@ def escape(pathname): return drive + pathname +_special_parts = ('', '.', '..') _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) +_no_recurse_symlinks = object() + + +def translate(pat, *, recursive=False, include_hidden=False, seps=None): + """Translate a pathname with shell wildcards to a regular expression. + + If `recursive` is true, the pattern segment '**' will match any number of + path segments. + + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). + + If a sequence of separator characters is given to `seps`, they will be + used to split the pattern into segments and match path separators. If not + given, os.path.sep and os.path.altsep (where available) are used. + """ + if not seps: + if os.path.altsep: + seps = (os.path.sep, os.path.altsep) + else: + seps = os.path.sep + escaped_seps = ''.join(map(re.escape, seps)) + any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps + not_sep = f'[^{escaped_seps}]' + if include_hidden: + one_last_segment = f'{not_sep}+' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:.+{any_sep})?' + any_last_segments = '.*' + else: + one_last_segment = f'[^{escaped_seps}.]{not_sep}*' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:{one_segment})*' + any_last_segments = f'{any_segments}(?:{one_last_segment})?' + + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == '*': + results.append(one_segment if idx < last_part_idx else one_last_segment) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)[0]) + if idx < last_part_idx: + results.append(any_sep) + res = ''.join(results) + return fr'(?s:{res})\z' + + +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, seps, case_sensitive, recursive=True): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = translate(pat, recursive=recursive, include_hidden=True, seps=seps) + return re.compile(regex, flags=flags).match + + +class _GlobberBase: + """Abstract class providing shell-style pattern matching and globbing. + """ + + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): + self.sep = sep + self.case_sensitive = case_sensitive + self.case_pedantic = case_pedantic + self.recursive = recursive + + # Abstract methods + + @staticmethod + def lexists(path): + """Implements os.path.lexists(). + """ + raise NotImplementedError + + @staticmethod + def scandir(path): + """Like os.scandir(), but generates (entry, name, path) tuples. + """ + raise NotImplementedError + + @staticmethod + def concat_path(path, text): + """Implements path concatenation. + """ + raise NotImplementedError + + # High-level methods + + def compile(self, pat, altsep=None): + seps = (self.sep, altsep) if altsep else self.sep + return _compile_pattern(pat, seps, self.case_sensitive, self.recursive) + + def selector(self, parts): + """Returns a function that selects from a given path, walking and + filtering according to the glob-style pattern parts in *parts*. + """ + if not parts: + return self.select_exists + part = parts.pop() + if self.recursive and part == '**': + selector = self.recursive_selector + elif part in _special_parts: + selector = self.special_selector + elif not self.case_pedantic and magic_check.search(part) is None: + selector = self.literal_selector + else: + selector = self.wildcard_selector + return selector(part, parts) + + def special_selector(self, part, parts): + """Returns a function that selects special children of the given path. + """ + if parts: + part += self.sep + select_next = self.selector(parts) + + def select_special(path, exists=False): + path = self.concat_path(path, part) + return select_next(path, exists) + return select_special + + def literal_selector(self, part, parts): + """Returns a function that selects a literal descendant of a path. + """ + + # Optimization: consume and join any subsequent literal parts here, + # rather than leaving them for the next selector. This reduces the + # number of string concatenation operations. + while parts and magic_check.search(parts[-1]) is None: + part += self.sep + parts.pop() + if parts: + part += self.sep + + select_next = self.selector(parts) + + def select_literal(path, exists=False): + path = self.concat_path(path, part) + return select_next(path, exists=False) + return select_literal + + def wildcard_selector(self, part, parts): + """Returns a function that selects direct children of a given path, + filtering by pattern. + """ + + match = None if part == '*' else self.compile(part) + dir_only = bool(parts) + if dir_only: + select_next = self.selector(parts) + + def select_wildcard(path, exists=False): + try: + entries = self.scandir(path) + except OSError: + pass + else: + for entry, entry_name, entry_path in entries: + if match is None or match(entry_name): + if dir_only: + try: + if not entry.is_dir(): + continue + except OSError: + continue + entry_path = self.concat_path(entry_path, self.sep) + yield from select_next(entry_path, exists=True) + else: + yield entry_path + return select_wildcard + + def recursive_selector(self, part, parts): + """Returns a function that selects a given path and all its children, + recursively, filtering by pattern. + """ + # Optimization: consume following '**' parts, which have no effect. + while parts and parts[-1] == '**': + parts.pop() + + # Optimization: consume and join any following non-special parts here, + # rather than leaving them for the next selector. They're used to + # build a regular expression, which we use to filter the results of + # the recursive walk. As a result, non-special pattern segments + # following a '**' wildcard don't require additional filesystem access + # to expand. + follow_symlinks = self.recursive is not _no_recurse_symlinks + if follow_symlinks: + while parts and parts[-1] not in _special_parts: + part += self.sep + parts.pop() + + match = None if part == '**' else self.compile(part) + dir_only = bool(parts) + select_next = self.selector(parts) + + def select_recursive(path, exists=False): + match_pos = len(str(path)) + if match is None or match(str(path), match_pos): + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) + + def select_recursive_step(stack, match_pos): + path = stack.pop() + try: + entries = self.scandir(path) + except OSError: + pass + else: + for entry, _entry_name, entry_path in entries: + is_dir = False + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + is_dir = True + except OSError: + pass + + if is_dir or not dir_only: + entry_path_str = str(entry_path) + if dir_only: + entry_path = self.concat_path(entry_path, self.sep) + if match is None or match(entry_path_str, match_pos): + if dir_only: + yield from select_next(entry_path, exists=True) + else: + # Optimization: directly yield the path if this is + # last pattern part. + yield entry_path + if is_dir: + stack.append(entry_path) + + return select_recursive + + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. + """ + if exists: + # Optimization: this path is already known to exist, e.g. because + # it was returned from os.scandir(), so we skip calling lstat(). + yield path + elif self.lexists(path): + yield path + + +class _StringGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for string paths. + """ + lexists = staticmethod(os.path.lexists) + concat_path = operator.add + + @staticmethod + def scandir(path): + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with os.scandir(path) as scandir_it: + entries = list(scandir_it) + return ((entry, entry.name, entry.path) for entry in entries) + + +class _PathGlobber(_GlobberBase): + """Provides shell-style pattern matching and globbing for pathlib paths. + """ + + @staticmethod + def lexists(path): + return path.info.exists(follow_symlinks=False) + + @staticmethod + def scandir(path): + return ((child.info, child.name, child) for child in path.iterdir()) + + @staticmethod + def concat_path(path, text): + return path.with_segments(str(path) + text) diff --git a/Lib/graphlib.py b/Lib/graphlib.py index 636545648e1..7961c9c5cac 100644 --- a/Lib/graphlib.py +++ b/Lib/graphlib.py @@ -90,20 +90,24 @@ def prepare(self): still be used to obtain as many nodes as possible until cycles block more progress. After a call to this function, the graph cannot be modified and therefore no more nodes can be added using "add". + + Raise ValueError if nodes have already been passed out of the sorter. + """ - if self._ready_nodes is not None: - raise ValueError("cannot prepare() more than once") + if self._npassedout > 0: + raise ValueError("cannot prepare() after starting sort") - self._ready_nodes = [ - i.node for i in self._node2info.values() if i.npredecessors == 0 - ] + if self._ready_nodes is None: + self._ready_nodes = [ + i.node for i in self._node2info.values() if i.npredecessors == 0 + ] # ready_nodes is set before we look for cycles on purpose: # if the user wants to catch the CycleError, that's fine, # they can continue using the instance to grab as many # nodes as possible before cycles block more progress cycle = self._find_cycle() if cycle: - raise CycleError(f"nodes are in a cycle", cycle) + raise CycleError("nodes are in a cycle", cycle) def get_ready(self): """Return a tuple of all the nodes that are ready. @@ -154,7 +158,7 @@ def done(self, *nodes): This method unblocks any successor of each node in *nodes* for being returned in the future by a call to "get_ready". - Raises :exec:`ValueError` if any node in *nodes* has already been marked as + Raises ValueError if any node in *nodes* has already been marked as processed by a previous call to this method, if a node was not added to the graph by using "add" or if called without calling "prepare" previously or if node has not yet been returned by "get_ready". diff --git a/Lib/gzip.py b/Lib/gzip.py index 5b20e5ba698..c00f51858de 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -5,22 +5,30 @@ # based on Andrew Kuchling's minigzip.py distributed with the zlib module -import struct, sys, time, os -import zlib import builtins import io -import _compression +import os +import struct +import sys +import time +import weakref +import zlib +from compression._common import _streams __all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"] FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 -READ, WRITE = 1, 2 +READ = 'rb' +WRITE = 'wb' _COMPRESS_LEVEL_FAST = 1 _COMPRESS_LEVEL_TRADEOFF = 6 _COMPRESS_LEVEL_BEST = 9 +READ_BUFFER_SIZE = 128 * 1024 +_WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE + def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, encoding=None, errors=None, newline=None): @@ -118,7 +126,25 @@ class BadGzipFile(OSError): """Exception raised in some cases for invalid gzip files.""" -class GzipFile(_compression.BaseStream): +class _WriteBufferStream(io.RawIOBase): + """Minimal object to pass WriteBuffer flushes into GzipFile""" + def __init__(self, gzip_file): + self.gzip_file = weakref.ref(gzip_file) + + def write(self, data): + gzip_file = self.gzip_file() + if gzip_file is None: + raise RuntimeError("lost gzip_file") + return gzip_file._write_raw(data) + + def seekable(self): + return False + + def writable(self): + return True + + +class GzipFile(_streams.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. @@ -160,65 +186,74 @@ def __init__(self, filename=None, mode=None, and 9 is slowest and produces the most compression. 0 is no compression at all. The default is 9. - The mtime argument is an optional numeric timestamp to be written - to the last modification time field in the stream when compressing. - If omitted or None, the current time is used. + The optional mtime argument is the timestamp requested by gzip. The time + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. + If mtime is omitted or None, the current time is used. Use mtime = 0 + to generate a compressed stream that does not depend on creation time. """ + # Ensure attributes exist at __del__ + self.mode = None + self.fileobj = None + self._buffer = None + if mode and ('t' in mode or 'U' in mode): raise ValueError("Invalid mode: {!r}".format(mode)) if mode and 'b' not in mode: mode += 'b' - if fileobj is None: - fileobj = self.myfileobj = builtins.open(filename, mode or 'rb') - if filename is None: - filename = getattr(fileobj, 'name', '') - if not isinstance(filename, (str, bytes)): - filename = '' - else: - filename = os.fspath(filename) - origmode = mode - if mode is None: - mode = getattr(fileobj, 'mode', 'rb') - - if mode.startswith('r'): - self.mode = READ - raw = _GzipReader(fileobj) - self._buffer = io.BufferedReader(raw) - self.name = filename - - elif mode.startswith(('w', 'a', 'x')): - if origmode is None: - import warnings - warnings.warn( - "GzipFile was opened for writing, but this will " - "change in future Python releases. " - "Specify the mode argument for opening it for writing.", - FutureWarning, 2) - self.mode = WRITE - self._init_write(filename) - self.compress = zlib.compressobj(compresslevel, - zlib.DEFLATED, - -zlib.MAX_WBITS, - zlib.DEF_MEM_LEVEL, - 0) - self._write_mtime = mtime - else: - raise ValueError("Invalid mode: {!r}".format(mode)) - self.fileobj = fileobj + try: + if fileobj is None: + fileobj = self.myfileobj = builtins.open(filename, mode or 'rb') + if filename is None: + filename = getattr(fileobj, 'name', '') + if not isinstance(filename, (str, bytes)): + filename = '' + else: + filename = os.fspath(filename) + origmode = mode + if mode is None: + mode = getattr(fileobj, 'mode', 'rb') + + + if mode.startswith('r'): + self.mode = READ + raw = _GzipReader(fileobj) + self._buffer = io.BufferedReader(raw) + self.name = filename + + elif mode.startswith(('w', 'a', 'x')): + if origmode is None: + import warnings + warnings.warn( + "GzipFile was opened for writing, but this will " + "change in future Python releases. " + "Specify the mode argument for opening it for writing.", + FutureWarning, 2) + self.mode = WRITE + self._init_write(filename) + self.compress = zlib.compressobj(compresslevel, + zlib.DEFLATED, + -zlib.MAX_WBITS, + zlib.DEF_MEM_LEVEL, + 0) + self._write_mtime = mtime + self._buffer_size = _WRITE_BUFFER_SIZE + self._buffer = io.BufferedWriter(_WriteBufferStream(self), + buffer_size=self._buffer_size) + else: + raise ValueError("Invalid mode: {!r}".format(mode)) - if self.mode == WRITE: - self._write_gzip_header(compresslevel) + self.fileobj = fileobj - @property - def filename(self): - import warnings - warnings.warn("use the name attribute", DeprecationWarning, 2) - if self.mode == WRITE and self.name[-3:] != ".gz": - return self.name + ".gz" - return self.name + if self.mode == WRITE: + self._write_gzip_header(compresslevel) + except: + # Avoid a ResourceWarning if the write fails, + # eg read-only file or KeyboardInterrupt + self._close() + raise @property def mtime(self): @@ -237,6 +272,11 @@ def _init_write(self, filename): self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc + def tell(self): + self._check_not_closed() + self._buffer.flush() + return super().tell() + def _write_gzip_header(self, compresslevel): self.fileobj.write(b'\037\213') # magic header self.fileobj.write(b'\010') # compression method @@ -278,6 +318,10 @@ def write(self,data): if self.fileobj is None: raise ValueError("write() on closed GzipFile object") + return self._buffer.write(data) + + def _write_raw(self, data): + # Called by our self._buffer underlying WriteBufferStream. if isinstance(data, (bytes, bytearray)): length = len(data) else: @@ -293,11 +337,15 @@ def write(self,data): return length - def read(self, size=-1): - self._check_not_closed() + def _check_read(self, caller): if self.mode != READ: import errno - raise OSError(errno.EBADF, "read() on write-only GzipFile object") + msg = f"{caller}() on write-only GzipFile object" + raise OSError(errno.EBADF, msg) + + def read(self, size=-1): + self._check_not_closed() + self._check_read("read") return self._buffer.read(size) def read1(self, size=-1): @@ -305,19 +353,25 @@ def read1(self, size=-1): Reads up to a buffer's worth of data if size is negative.""" self._check_not_closed() - if self.mode != READ: - import errno - raise OSError(errno.EBADF, "read1() on write-only GzipFile object") + self._check_read("read1") if size < 0: size = io.DEFAULT_BUFFER_SIZE return self._buffer.read1(size) + def readinto(self, b): + self._check_not_closed() + self._check_read("readinto") + return self._buffer.readinto(b) + + def readinto1(self, b): + self._check_not_closed() + self._check_read("readinto1") + return self._buffer.readinto1(b) + def peek(self, n): self._check_not_closed() - if self.mode != READ: - import errno - raise OSError(errno.EBADF, "peek() on write-only GzipFile object") + self._check_read("peek") return self._buffer.peek(n) @property @@ -328,9 +382,11 @@ def close(self): fileobj = self.fileobj if fileobj is None: return - self.fileobj = None + if self._buffer is None or self._buffer.closed: + return try: if self.mode == WRITE: + self._buffer.flush() fileobj.write(self.compress.flush()) write32u(fileobj, self.crc) # self.size may exceed 2 GiB, or even 4 GiB @@ -338,14 +394,19 @@ def close(self): elif self.mode == READ: self._buffer.close() finally: - myfileobj = self.myfileobj - if myfileobj: - self.myfileobj = None - myfileobj.close() + self._close() + + def _close(self): + self.fileobj = None + myfileobj = self.myfileobj + if myfileobj is not None: + self.myfileobj = None + myfileobj.close() def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_not_closed() if self.mode == WRITE: + self._buffer.flush() # Ensure the compressor's buffer is flushed self.fileobj.write(self.compress.flush(zlib_mode)) self.fileobj.flush() @@ -376,6 +437,9 @@ def seekable(self): def seek(self, offset, whence=io.SEEK_SET): if self.mode == WRITE: + self._check_not_closed() + # Flush buffer to ensure validity of self.offset + self._buffer.flush() if whence != io.SEEK_SET: if whence == io.SEEK_CUR: offset = self.offset + offset @@ -384,10 +448,10 @@ def seek(self, offset, whence=io.SEEK_SET): if offset < self.offset: raise OSError('Negative seek in write mode') count = offset - self.offset - chunk = b'\0' * 1024 - for i in range(count // 1024): + chunk = b'\0' * self._buffer_size + for i in range(count // self._buffer_size): self.write(chunk) - self.write(b'\0' * (count % 1024)) + self.write(b'\0' * (count % self._buffer_size)) elif self.mode == READ: self._check_not_closed() return self._buffer.seek(offset, whence) @@ -398,6 +462,13 @@ def readline(self, size=-1): self._check_not_closed() return self._buffer.readline(size) + def __del__(self): + if self.mode == WRITE and not self.closed: + import warnings + warnings.warn("unclosed GzipFile", + ResourceWarning, source=self, stacklevel=2) + + super().__del__() def _read_exact(fp, n): '''Read exactly *n* bytes from `fp` @@ -452,9 +523,9 @@ def _read_gzip_header(fp): return last_mtime -class _GzipReader(_compression.DecompressReader): +class _GzipReader(_streams.DecompressReader): def __init__(self, fp): - super().__init__(_PaddedFile(fp), zlib.decompressobj, + super().__init__(_PaddedFile(fp), zlib._ZlibDecompressor, wbits=-zlib.MAX_WBITS) # Set flag indicating start of a new member self._new_member = True @@ -502,12 +573,13 @@ def read(self, size=-1): self._new_member = False # Read a chunk of data from the file - buf = self._fp.read(io.DEFAULT_BUFFER_SIZE) + if self._decompressor.needs_input: + buf = self._fp.read(READ_BUFFER_SIZE) + uncompress = self._decompressor.decompress(buf, size) + else: + uncompress = self._decompressor.decompress(b"", size) - uncompress = self._decompressor.decompress(buf, size) - if self._decompressor.unconsumed_tail != b"": - self._fp.prepend(self._decompressor.unconsumed_tail) - elif self._decompressor.unused_data != b"": + if self._decompressor.unused_data != b"": # Prepend the already read bytes to the fileobj so they can # be seen by _read_eof() and _read_gzip_header() self._fp.prepend(self._decompressor.unused_data) @@ -518,14 +590,11 @@ def read(self, size=-1): raise EOFError("Compressed file ended before the " "end-of-stream marker was reached") - self._add_read_data( uncompress ) + self._crc = zlib.crc32(uncompress, self._crc) + self._stream_size += len(uncompress) self._pos += len(uncompress) return uncompress - def _add_read_data(self, data): - self._crc = zlib.crc32(data, self._crc) - self._stream_size = self._stream_size + len(data) - def _read_eof(self): # We've read to the end of the file # We check that the computed CRC and size of the @@ -552,43 +621,21 @@ def _rewind(self): self._new_member = True -def _create_simple_gzip_header(compresslevel: int, - mtime = None) -> bytes: - """ - Write a simple gzip header with no extra fields. - :param compresslevel: Compresslevel used to determine the xfl bytes. - :param mtime: The mtime (must support conversion to a 32-bit integer). - :return: A bytes object representing the gzip header. - """ - if mtime is None: - mtime = time.time() - if compresslevel == _COMPRESS_LEVEL_BEST: - xfl = 2 - elif compresslevel == _COMPRESS_LEVEL_FAST: - xfl = 4 - else: - xfl = 0 - # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra - # fields added to header), mtime, xfl and os (255 for unknown OS). - return struct.pack(" blocksize: - password = new(hash_name, password).digest() - password = password + b'\x00' * (blocksize - len(password)) - inner.update(password.translate(_trans_36)) - outer.update(password.translate(_trans_5C)) - - def prf(msg, inner=inner, outer=outer): - # PBKDF2_HMAC uses the password as key. We can re-use the same - # digest objects and just update copies to skip initialization. - icpy = inner.copy() - ocpy = outer.copy() - icpy.update(msg) - ocpy.update(icpy.digest()) - return ocpy.digest() - - if iterations < 1: - raise ValueError(iterations) - if dklen is None: - dklen = outer.digest_size - if dklen < 1: - raise ValueError(dklen) - - dkey = b'' - loop = 1 - from_bytes = int.from_bytes - while len(dkey) < dklen: - prev = prf(salt + loop.to_bytes(4)) - # endianness doesn't matter here as long to / from use the same - rkey = from_bytes(prev) - for i in range(iterations - 1): - prev = prf(prev) - # rkey = rkey ^ prev - rkey ^= from_bytes(prev) - loop += 1 - dkey += rkey.to_bytes(inner.digest_size) - - return dkey[:dklen] + pass + try: # OpenSSL's scrypt requires OpenSSL 1.1+ - from _hashlib import scrypt + from _hashlib import scrypt # noqa: F401 except ImportError: pass @@ -294,6 +231,8 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): view = memoryview(buf) while True: size = fileobj.readinto(buf) + if size is None: + raise BlockingIOError("I/O operation would block.") if size == 0: break # EOF digestobj.update(view[:size]) diff --git a/Lib/heapq.py b/Lib/heapq.py index 2fd9d1ff4bf..17f62dd2d58 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -42,7 +42,7 @@ property of a heap is that a[0] is always its smallest element. The strange invariant above is meant to be an efficient memory -representation for a tournament. The numbers below are `k', not a[k]: +representation for a tournament. The numbers below are 'k', not a[k]: 0 @@ -55,7 +55,7 @@ 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 -In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In +In the tree above, each cell 'k' is topping '2*k+1' and '2*k+2'. In a usual binary tournament we see in sports, each cell is the winner over the two cells it tops, and we can trace the winner down the tree to see all opponents s/he had. However, in many computer applications @@ -78,7 +78,7 @@ not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition means the smallest scheduled -time. When an event schedule other events for execution, they are +time. When an event schedules other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). @@ -91,14 +91,14 @@ Heaps are also very useful in big disk sorts. You most probably all know that a big sort implies producing "runs" (which are pre-sorted -sequences, which size is usually related to the amount of CPU memory), +sequences, whose size is usually related to the amount of CPU memory), followed by a merging passes for these runs, which merging is often very cleverly organised[1]. It is very important that the initial sort produces the longest runs possible. Tournaments are a good way -to that. If, using all the memory available to hold a tournament, you -replace and percolate items that happen to fit the current run, you'll -produce runs which are twice the size of the memory for random input, -and much better for input fuzzily ordered. +to achieve that. If, using all the memory available to hold a +tournament, you replace and percolate items that happen to fit the +current run, you'll produce runs which are twice the size of the +memory for random input, and much better for input fuzzily ordered. Moreover, if you output the 0'th item on disk and get an input which may not fit in the current tournament (because the value "wins" over @@ -110,7 +110,7 @@ effective! In a word, heaps are useful memory structures to know. I use them in -a few applications, and I think it is good to keep a `heap' module +a few applications, and I think it is good to keep a 'heap' module around. :-) -------------------- @@ -126,8 +126,9 @@ From all times, sorting has always been a Great Art! :-) """ -__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', - 'nlargest', 'nsmallest', 'heappushpop'] +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'heappushpop', + 'heappush_max', 'heappop_max', 'heapify_max', 'heapreplace_max', + 'heappushpop_max', 'nlargest', 'nsmallest', 'merge'] def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" @@ -178,7 +179,7 @@ def heapify(x): for i in reversed(range(n//2)): _siftup(x, i) -def _heappop_max(heap): +def heappop_max(heap): """Maxheap version of a heappop.""" lastelt = heap.pop() # raises appropriate IndexError if heap is empty if heap: @@ -188,19 +189,32 @@ def _heappop_max(heap): return returnitem return lastelt -def _heapreplace_max(heap, item): +def heapreplace_max(heap, item): """Maxheap version of a heappop followed by a heappush.""" returnitem = heap[0] # raises appropriate IndexError if heap is empty heap[0] = item _siftup_max(heap, 0) return returnitem -def _heapify_max(x): +def heappush_max(heap, item): + """Maxheap version of a heappush.""" + heap.append(item) + _siftdown_max(heap, 0, len(heap)-1) + +def heappushpop_max(heap, item): + """Maxheap fast version of a heappush followed by a heappop.""" + if heap and item < heap[0]: + item, heap[0] = heap[0], item + _siftup_max(heap, 0) + return item + +def heapify_max(x): """Transform list into a maxheap, in-place, in O(len(x)) time.""" n = len(x) for i in reversed(range(n//2)): _siftup_max(x, i) + # 'heap' is a heap at all indices >= startpos, except possibly for pos. pos # is the index of a leaf with a possibly out-of-order value. Restore the # heap invariant. @@ -335,9 +349,9 @@ def merge(*iterables, key=None, reverse=False): h_append = h.append if reverse: - _heapify = _heapify_max - _heappop = _heappop_max - _heapreplace = _heapreplace_max + _heapify = heapify_max + _heappop = heappop_max + _heapreplace = heapreplace_max direction = -1 else: _heapify = heapify @@ -490,10 +504,10 @@ def nsmallest(n, iterable, key=None): result = [(elem, i) for i, elem in zip(range(n), it)] if not result: return result - _heapify_max(result) + heapify_max(result) top = result[0][0] order = n - _heapreplace = _heapreplace_max + _heapreplace = heapreplace_max for elem in it: if elem < top: _heapreplace(result, (elem, order)) @@ -507,10 +521,10 @@ def nsmallest(n, iterable, key=None): result = [(key(elem), i, elem) for i, elem in zip(range(n), it)] if not result: return result - _heapify_max(result) + heapify_max(result) top = result[0][0] order = n - _heapreplace = _heapreplace_max + _heapreplace = heapreplace_max for elem in it: k = key(elem) if k < top: @@ -583,19 +597,13 @@ def nlargest(n, iterable, key=None): from _heapq import * except ImportError: pass -try: - from _heapq import _heapreplace_max -except ImportError: - pass -try: - from _heapq import _heapify_max -except ImportError: - pass -try: - from _heapq import _heappop_max -except ImportError: - pass +# For backwards compatibility +_heappop_max = heappop_max +_heapreplace_max = heapreplace_max +_heappush_max = heappush_max +_heappushpop_max = heappushpop_max +_heapify_max = heapify_max if __name__ == "__main__": diff --git a/Lib/hmac.py b/Lib/hmac.py index 8b4f920db95..2d6016cda11 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -3,7 +3,6 @@ Implements the HMAC algorithm as described by RFC 2104. """ -import warnings as _warnings try: import _hashlib as _hashopenssl except ImportError: @@ -14,7 +13,10 @@ compare_digest = _hashopenssl.compare_digest _functype = type(_hashopenssl.openssl_sha256) # builtin type -import hashlib as _hashlib +try: + import _hmac +except ImportError: + _hmac = None trans_5C = bytes((x ^ 0x5C) for x in range(256)) trans_36 = bytes((x ^ 0x36) for x in range(256)) @@ -24,11 +26,27 @@ digest_size = None +def _get_digest_constructor(digest_like): + if callable(digest_like): + return digest_like + if isinstance(digest_like, str): + def digest_wrapper(d=b''): + import hashlib + return hashlib.new(digest_like, d) + else: + def digest_wrapper(d=b''): + return digest_like.new(d) + return digest_wrapper + + class HMAC: """RFC 2104 HMAC class. Also complies with RFC 4231. This supports the API for Cryptographic Hash Functions (PEP 247). """ + + # Note: self.blocksize is the default blocksize; self.block_size + # is effective block size as well as the public API attribute. blocksize = 64 # 512-bit HMAC; can be changed in subclasses. __slots__ = ( @@ -50,31 +68,47 @@ def __init__(self, key, msg=None, digestmod=''): """ if not isinstance(key, (bytes, bytearray)): - raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) + raise TypeError(f"key: expected bytes or bytearray, " + f"but got {type(key).__name__!r}") if not digestmod: - raise TypeError("Missing required parameter 'digestmod'.") + raise TypeError("Missing required argument 'digestmod'.") + self.__init(key, msg, digestmod) + + def __init(self, key, msg, digestmod): if _hashopenssl and isinstance(digestmod, (str, _functype)): try: - self._init_hmac(key, msg, digestmod) - except _hashopenssl.UnsupportedDigestmodError: - self._init_old(key, msg, digestmod) - else: - self._init_old(key, msg, digestmod) + self._init_openssl_hmac(key, msg, digestmod) + return + except _hashopenssl.UnsupportedDigestmodError: # pragma: no cover + pass + if _hmac and isinstance(digestmod, str): + try: + self._init_builtin_hmac(key, msg, digestmod) + return + except _hmac.UnknownHashError: # pragma: no cover + pass + self._init_old(key, msg, digestmod) - def _init_hmac(self, key, msg, digestmod): + def _init_openssl_hmac(self, key, msg, digestmod): self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod) + self._inner = self._outer = None # because the slots are defined + self.digest_size = self._hmac.digest_size + self.block_size = self._hmac.block_size + + _init_hmac = _init_openssl_hmac # for backward compatibility (if any) + + def _init_builtin_hmac(self, key, msg, digestmod): + self._hmac = _hmac.new(key, msg, digestmod=digestmod) + self._inner = self._outer = None # because the slots are defined self.digest_size = self._hmac.digest_size self.block_size = self._hmac.block_size def _init_old(self, key, msg, digestmod): - if callable(digestmod): - digest_cons = digestmod - elif isinstance(digestmod, str): - digest_cons = lambda d=b'': _hashlib.new(digestmod, d) - else: - digest_cons = lambda d=b'': digestmod.new(d) + import warnings + + digest_cons = _get_digest_constructor(digestmod) self._hmac = None self._outer = digest_cons() @@ -84,21 +118,19 @@ def _init_old(self, key, msg, digestmod): if hasattr(self._inner, 'block_size'): blocksize = self._inner.block_size if blocksize < 16: - _warnings.warn('block_size of %d seems too small; using our ' - 'default of %d.' % (blocksize, self.blocksize), - RuntimeWarning, 2) - blocksize = self.blocksize + warnings.warn(f"block_size of {blocksize} seems too small; " + f"using our default of {self.blocksize}.", + RuntimeWarning, 2) + blocksize = self.blocksize # pragma: no cover else: - _warnings.warn('No block_size attribute on given digest object; ' - 'Assuming %d.' % (self.blocksize), - RuntimeWarning, 2) - blocksize = self.blocksize + warnings.warn("No block_size attribute on given digest object; " + f"Assuming {self.blocksize}.", + RuntimeWarning, 2) + blocksize = self.blocksize # pragma: no cover if len(key) > blocksize: key = digest_cons(key).digest() - # self.blocksize is the default blocksize. self.block_size is - # effective block size as well as the public API attribute. self.block_size = blocksize key = key.ljust(blocksize, b'\0') @@ -127,6 +159,7 @@ def copy(self): # Call __new__ directly to avoid the expensive __init__. other = self.__class__.__new__(self.__class__) other.digest_size = self.digest_size + other.block_size = self.block_size if self._hmac: other._hmac = self._hmac.copy() other._inner = other._outer = None @@ -164,6 +197,7 @@ def hexdigest(self): h = self._current() return h.hexdigest() + def new(key, msg=None, digestmod=''): """Create a new hashing object and return it. @@ -193,25 +227,41 @@ def digest(key, msg, digest): A hashlib constructor returning a new hash object. *OR* A module supporting PEP 247. """ - if _hashopenssl is not None and isinstance(digest, (str, _functype)): + if _hashopenssl and isinstance(digest, (str, _functype)): try: return _hashopenssl.hmac_digest(key, msg, digest) + except OverflowError: + # OpenSSL's HMAC limits the size of the key to INT_MAX. + # Instead of falling back to HACL* implementation which + # may still not be supported due to a too large key, we + # directly switch to the pure Python fallback instead + # even if we could have used streaming HMAC for small keys + # but large messages. + return _compute_digest_fallback(key, msg, digest) except _hashopenssl.UnsupportedDigestmodError: pass - if callable(digest): - digest_cons = digest - elif isinstance(digest, str): - digest_cons = lambda d=b'': _hashlib.new(digest, d) - else: - digest_cons = lambda d=b'': digest.new(d) + if _hmac and isinstance(digest, str): + try: + return _hmac.compute_digest(key, msg, digest) + except (OverflowError, _hmac.UnknownHashError): + # HACL* HMAC limits the size of the key to UINT32_MAX + # so we fallback to the pure Python implementation even + # if streaming HMAC may have been used for small keys + # and large messages. + pass + + return _compute_digest_fallback(key, msg, digest) + +def _compute_digest_fallback(key, msg, digest): + digest_cons = _get_digest_constructor(digest) inner = digest_cons() outer = digest_cons() blocksize = getattr(inner, 'block_size', 64) if len(key) > blocksize: key = digest_cons(key).digest() - key = key + b'\x00' * (blocksize - len(key)) + key = key.ljust(blocksize, b'\0') inner.update(key.translate(trans_36)) outer.update(key.translate(trans_5C)) inner.update(msg) diff --git a/Lib/html/__init__.py b/Lib/html/__init__.py index da0a0a3ce70..1543460ca33 100644 --- a/Lib/html/__init__.py +++ b/Lib/html/__init__.py @@ -25,7 +25,7 @@ def escape(s, quote=True): return s -# see http://www.w3.org/TR/html5/syntax.html#tokenizing-character-references +# see https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state _invalid_charrefs = { 0x00: '\ufffd', # REPLACEMENT CHARACTER diff --git a/Lib/html/entities.py b/Lib/html/entities.py index dc508631ac4..eb6dc121905 100644 --- a/Lib/html/entities.py +++ b/Lib/html/entities.py @@ -3,8 +3,7 @@ __all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs'] -# maps the HTML entity name to the Unicode code point -# from https://html.spec.whatwg.org/multipage/named-characters.html +# maps HTML4 entity name to the Unicode code point name2codepoint = { 'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1 'Aacute': 0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1 @@ -261,7 +260,11 @@ } -# maps the HTML5 named character references to the equivalent Unicode character(s) +# HTML5 named character references +# Generated by Tools/build/parse_html5_entities.py +# from https://html.spec.whatwg.org/entities.json and +# https://html.spec.whatwg.org/multipage/named-characters.html. +# Map HTML5 named character references to the equivalent Unicode character(s). html5 = { 'Aacute': '\xc1', 'aacute': '\xe1', diff --git a/Lib/html/parser.py b/Lib/html/parser.py index bef0f4fe4bf..80fb8c3f929 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -12,6 +12,7 @@ import _markupbase from html import unescape +from html.entities import html5 as html5_entities __all__ = ['HTMLParser'] @@ -23,20 +24,52 @@ entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +incomplete_charref = re.compile('&#(?:[0-9]|[xX][0-9a-fA-F])') +attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') +endtagopen = re.compile('') -commentclose = re.compile(r'--\s*>') +commentclose = re.compile(r'--!?>') +commentabruptclose = re.compile(r'-?>') # Note: -# 1) if you change tagfind/attrfind remember to update locatestarttagend too; -# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# 1) if you change tagfind/attrfind remember to update locatetagend too; +# 2) if you change tagfind/attrfind and/or locatetagend the parser will # explode, so don't do it. -# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state -# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state -tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') -attrfind_tolerant = re.compile( - r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' - r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +# see the HTML5 specs section "13.2.5.6 Tag open state", +# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state". +# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state +# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state +# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*') +attrfind_tolerant = re.compile(r""" + ( + (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + ) + ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + ('[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space +""", re.VERBOSE) +locatetagend = re.compile(r""" + [a-zA-Z][^\t\n\r\f />]* # tag name + [\t\n\r\f /]* # optional whitespace before attribute name + (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + [\t\n\r\f /]* # possibly followed by a space + )* + >? +""", re.VERBOSE) +# The following variables are not used, but are temporarily left for +# backward compatibility. locatestarttagend_tolerant = re.compile(r""" <[a-zA-Z][^\t\n\r\f />\x00]* # tag name (?:[\s/]* # optional whitespace before attribute name @@ -53,10 +86,24 @@ \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') -# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between -# ') +# Character reference processing logic specific to attribute values +# See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state +def _replace_attr_charref(match): + ref = match.group(0) + # Numeric / hex char refs must always be unescaped + if ref.startswith('&#'): + return unescape(ref) + # Named character / entity references must only be unescaped + # if they are an exact match, and they are not followed by an equals sign + if not ref.endswith('=') and ref[1:] in html5_entities: + return unescape(ref) + # Otherwise do not unescape + return ref + +def _unescape_attrvalue(s): + return attr_charref.sub(_replace_attr_charref, s) class HTMLParser(_markupbase.ParserBase): @@ -81,15 +128,25 @@ class HTMLParser(_markupbase.ParserBase): argument. """ - CDATA_CONTENT_ELEMENTS = ("script", "style") + # See the HTML5 specs section "13.4 Parsing HTML fragments". + # https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments + # CDATA_CONTENT_ELEMENTS are parsed in RAWTEXT mode + CDATA_CONTENT_ELEMENTS = ("script", "style", "xmp", "iframe", "noembed", "noframes") + RCDATA_CONTENT_ELEMENTS = ("textarea", "title") - def __init__(self, *, convert_charrefs=True): + def __init__(self, *, convert_charrefs=True, scripting=False): """Initialize and reset this instance. - If convert_charrefs is True (the default), all character references + If convert_charrefs is true (the default), all character references are automatically converted to the corresponding Unicode characters. + + If *scripting* is false (the default), the content of the + ``noscript`` element is parsed normally; if it's true, + it's returned as is without being parsed. """ + super().__init__() self.convert_charrefs = convert_charrefs + self.scripting = scripting self.reset() def reset(self): @@ -98,7 +155,9 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None - _markupbase.ParserBase.reset(self) + self._support_cdata = True + self._escapable = True + super().reset() def feed(self, data): r"""Feed data to the parser. @@ -119,13 +178,35 @@ def get_starttag_text(self): """Return full source of start tag: '<...>'.""" return self.__starttag_text - def set_cdata_mode(self, elem): + def set_cdata_mode(self, elem, *, escapable=False): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'' % self.cdata_elem, re.I) + self._escapable = escapable + if self.cdata_elem == 'plaintext': + self.interesting = re.compile(r'\z') + elif escapable and not self.convert_charrefs: + self.interesting = re.compile(r'&|])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) + else: + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True + + def _set_support_cdata(self, flag=True): + """Enable or disable support of the CDATA sections. + If enabled, "<[CDATA[" starts a CDATA section which ends with "]]>". + If disabled, "<[CDATA[" starts a bogus comments which ends with ">". + + This method is not called by default. Its purpose is to be called + in custom handle_starttag() and handle_endtag() methods, with + value that depends on the adjusted current node. + See https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state + for details. + """ + self._support_cdata = flag # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is @@ -146,7 +227,7 @@ def goahead(self, end): # & near the end and see if it's followed by a space or ;. amppos = rawdata.rfind('&', max(i, n-34)) if (amppos >= 0 and - not re.compile(r'[\s;]').search(rawdata, amppos)): + not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)): break # wait till we get all the text j = n else: @@ -158,7 +239,7 @@ def goahead(self, end): break j = n if i < j: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:j])) else: self.handle_data(rawdata[i:j]) @@ -176,7 +257,7 @@ def goahead(self, end): k = self.parse_pi(i) elif startswith("', i + 1) - if k < 0: - k = rawdata.find('<', i + 1) - if k < 0: - k = i + 1 - else: - k += 1 - if self.convert_charrefs and not self.cdata_elem: - self.handle_data(unescape(rawdata[i:k])) + if starttagopen.match(rawdata, i): # < + letter + pass + elif startswith("', i+9) + if j < 0: + return -1 + self.unknown_decl(rawdata[i+3: j]) + return j + 3 elif rawdata[i:i+9].lower() == ' gtpos = rawdata.find('>', i+9) @@ -271,12 +382,27 @@ def parse_html_declaration(self, i): else: return self.parse_bogus_comment(i) + # Internal -- parse comment, return length or -1 if not terminated + # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state + def parse_comment(self, i, report=True): + rawdata = self.rawdata + assert rawdata.startswith(' \n # \" --> " # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k+1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j+1:j+4], 8))) - i = j + 4 - return _nulljoin(res) + return _unquote_sub(_unquote_replace, str) # The _getdate() routine is used to set the expiration time in the cookie's HTTP # header. By default, _getdate() returns the current time in the appropriate # "expires" format for a Set-Cookie header. The one optional argument is an # offset from now, in seconds. For example, an offset of -3600 means "one hour -# ago". The offset may be a floating point number. +# ago". The offset may be a floating-point number. # _weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] @@ -282,17 +273,19 @@ class Morsel(dict): "httponly" : "HttpOnly", "version" : "Version", "samesite" : "SameSite", + "partitioned": "Partitioned", } - _flags = {'secure', 'httponly'} + _reserved_defaults = dict.fromkeys(_reserved, "") + + _flags = {'secure', 'httponly', 'partitioned'} def __init__(self): # Set defaults self._key = self._value = self._coded_value = None # Set default attributes - for key in self._reserved: - dict.__setitem__(self, key, "") + dict.update(self, self._reserved_defaults) @property def key(self): @@ -310,12 +303,16 @@ def __setitem__(self, K, V): K = K.lower() if not K in self._reserved: raise CookieError("Invalid attribute %r" % (K,)) + if _has_control_character(K, V): + raise CookieError(f"Control characters are not allowed in cookies {K!r} {V!r}") dict.__setitem__(self, K, V) def setdefault(self, key, val=None): key = key.lower() if key not in self._reserved: raise CookieError("Invalid attribute %r" % (key,)) + if _has_control_character(key, val): + raise CookieError("Control characters are not allowed in cookies %r %r" % (key, val,)) return dict.setdefault(self, key, val) def __eq__(self, morsel): @@ -351,6 +348,9 @@ def set(self, key, val, coded_val): raise CookieError('Attempt to set a reserved key %r' % (key,)) if not _is_legal_key(key): raise CookieError('Illegal key %r' % (key,)) + if _has_control_character(key, val, coded_val): + raise CookieError( + "Control characters are not allowed in cookies %r %r %r" % (key, val, coded_val,)) # It's a good key, so save it. self._key = key @@ -442,9 +442,11 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any doublequoted string + "(?:[^\\"]|\\.)*" # Any double-quoted string | # or - \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr + # Special case for "expires" attr + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day + [\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Date and time in specific format | # or [""" + _LegalValueChars + r"""]* # Any word or empty string ) # End of group 'val' @@ -502,7 +504,10 @@ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): result = [] items = sorted(self.items()) for key, value in items: - result.append(value.output(attrs, header)) + value_output = value.output(attrs, header) + if _has_control_character(value_output): + raise CookieError("Control characters are not allowed in cookies") + result.append(value_output) return sep.join(result) __str__ = output diff --git a/Lib/http/server.py b/Lib/http/server.py index 58abadf7377..ac1f57c29f0 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -2,18 +2,18 @@ Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST, -and CGIHTTPRequestHandler for CGI scripts. +and (deprecated) CGIHTTPRequestHandler for CGI scripts. -It does, however, optionally implement HTTP/1.1 persistent connections, -as of version 0.3. +It does, however, optionally implement HTTP/1.1 persistent connections. Notes on CGIHTTPRequestHandler ------------------------------ -This class implements GET and POST requests to cgi-bin scripts. +This class is deprecated. It implements GET and POST requests to cgi-bin scripts. -If the os.fork() function is not present (e.g. on Windows), -subprocess.Popen() is used as a fallback, with slightly altered semantics. +If the os.fork() function is not present (Windows), subprocess.Popen() is used, +with slightly altered but never documented semantics. Use from a threaded +process is likely to trigger a warning at os.fork() time. In all cases, the implementation is intentionally naive -- all requests are executed synchronously. @@ -83,8 +83,10 @@ __version__ = "0.6" __all__ = [ - "HTTPServer", "ThreadingHTTPServer", "BaseHTTPRequestHandler", - "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler", + "HTTPServer", "ThreadingHTTPServer", + "HTTPSServer", "ThreadingHTTPSServer", + "BaseHTTPRequestHandler", "SimpleHTTPRequestHandler", + "CGIHTTPRequestHandler", ] import copy @@ -93,12 +95,13 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath import select import shutil -import socket # For gethostbyaddr() +import socket import socketserver import sys import time @@ -109,11 +112,15 @@ # Default error message template DEFAULT_ERROR_MESSAGE = """\ - - + + - + + Error response @@ -127,9 +134,14 @@ DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8" +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + class HTTPServer(socketserver.TCPServer): - allow_reuse_address = 1 # Seems to make sense in testing environment + allow_reuse_address = True # Seems to make sense in testing environment + allow_reuse_port = False def server_bind(self): """Override server_bind to store the server name.""" @@ -143,6 +155,47 @@ class ThreadingHTTPServer(socketserver.ThreadingMixIn, HTTPServer): daemon_threads = True +class HTTPSServer(HTTPServer): + def __init__(self, server_address, RequestHandlerClass, + bind_and_activate=True, *, certfile, keyfile=None, + password=None, alpn_protocols=None): + try: + import ssl + except ImportError: + raise RuntimeError("SSL module is missing; " + "HTTPS support is unavailable") + + self.ssl = ssl + self.certfile = certfile + self.keyfile = keyfile + self.password = password + # Support by default HTTP/1.1 + self.alpn_protocols = ( + ["http/1.1"] if alpn_protocols is None else alpn_protocols + ) + + super().__init__(server_address, + RequestHandlerClass, + bind_and_activate) + + def server_activate(self): + """Wrap the socket in SSLSocket.""" + super().server_activate() + context = self._create_context() + self.socket = context.wrap_socket(self.socket, server_side=True) + + def _create_context(self): + """Create a secure SSL context.""" + context = self.ssl.create_default_context(self.ssl.Purpose.CLIENT_AUTH) + context.load_cert_chain(self.certfile, self.keyfile, self.password) + context.set_alpn_protocols(self.alpn_protocols) + return context + + +class ThreadingHTTPSServer(socketserver.ThreadingMixIn, HTTPSServer): + daemon_threads = True + + class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """HTTP request handler base class. @@ -275,6 +328,7 @@ def parse_request(self): error response has already been sent back. """ + is_http_0_9 = False self.command = None # set in case of error on the first line self.request_version = version = self.default_request_version self.close_connection = True @@ -300,6 +354,10 @@ def parse_request(self): # - Leading zeros MUST be ignored by recipients. if len(version_number) != 2: raise ValueError + if any(not component.isdigit() for component in version_number): + raise ValueError("non digit in http version") + if any(len(component) > 10 for component in version_number): + raise ValueError("unreasonable length http version") version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): self.send_error( @@ -328,8 +386,21 @@ def parse_request(self): HTTPStatus.BAD_REQUEST, "Bad HTTP/0.9 request type (%r)" % command) return False + is_http_0_9 = True self.command, self.path = command, path + # gh-87389: The purpose of replacing '//' with '/' is to protect + # against open redirect attacks possibly triggered if the path starts + # with '//' because http clients treat //path as an absolute URI + # without scheme (similar to http://path) rather than a path. + if self.path.startswith('//'): + self.path = '/' + self.path.lstrip('/') # Reduce to a single / + + # For HTTP/0.9, headers are not expected at all. + if is_http_0_9: + self.headers = {} + return True + # Examine the headers and look for a Connection directive. try: self.headers = http.client.parse_headers(self.rfile, @@ -556,6 +627,11 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + _control_char_table[ord('\\')] = r'\\' + def log_message(self, format, *args): """Log an arbitrary message. @@ -571,12 +647,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" @@ -637,6 +717,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """ server_version = "SimpleHTTP/" + __version__ + index_pages = ("index.html", "index.htm") extensions_map = _encodings_map_default = { '.gz': 'application/gzip', '.Z': 'application/octet-stream', @@ -680,7 +761,7 @@ def send_head(self): f = None if os.path.isdir(path): parts = urllib.parse.urlsplit(self.path) - if not parts.path.endswith('/'): + if not parts.path.endswith(('/', '%2f', '%2F')): # redirect browser - doing basically what apache does self.send_response(HTTPStatus.MOVED_PERMANENTLY) new_parts = (parts[0], parts[1], parts[2] + '/', @@ -690,9 +771,9 @@ def send_head(self): self.send_header("Content-Length", "0") self.end_headers() return None - for index in "index.html", "index.htm": + for index in self.index_pages: index = os.path.join(path, index) - if os.path.exists(index): + if os.path.isfile(index): path = index break else: @@ -702,7 +783,7 @@ def send_head(self): # The test for this was added in test_httpserver.py # However, some OS platforms accept a trailingSlash as a filename # See discussion on python-dev and Issue34711 regarding - # parseing and rejection of filenames with a trailing slash + # parsing and rejection of filenames with a trailing slash if path.endswith("/"): self.send_error(HTTPStatus.NOT_FOUND, "File not found") return None @@ -770,21 +851,24 @@ def list_directory(self, path): return None list.sort(key=lambda a: a.lower()) r = [] + displaypath = self.path + displaypath = displaypath.split('#', 1)[0] + displaypath = displaypath.split('?', 1)[0] try: - displaypath = urllib.parse.unquote(self.path, + displaypath = urllib.parse.unquote(displaypath, errors='surrogatepass') except UnicodeDecodeError: - displaypath = urllib.parse.unquote(path) + displaypath = urllib.parse.unquote(displaypath) displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() - title = 'Directory listing for %s' % displaypath - r.append('') - r.append('\n') - r.append('' % enc) - r.append('%s\n' % title) - r.append('\n

%s

' % title) + title = f'Directory listing for {displaypath}' + r.append('') + r.append('') + r.append('') + r.append(f'') + r.append('') + r.append(f'{title}\n') + r.append(f'\n

{title}

') r.append('
\n
    ') for name in list: fullname = os.path.join(path, name) @@ -820,14 +904,14 @@ def translate_path(self, path): """ # abandon query parameters - path = path.split('?',1)[0] - path = path.split('#',1)[0] + path = path.split('#', 1)[0] + path = path.split('?', 1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 - trailing_slash = path.rstrip().endswith('/') try: path = urllib.parse.unquote(path, errors='surrogatepass') except UnicodeDecodeError: path = urllib.parse.unquote(path) + trailing_slash = path.endswith('/') path = posixpath.normpath(path) words = path.split('/') words = filter(None, words) @@ -877,7 +961,7 @@ def guess_type(self, path): ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] - guess, _ = mimetypes.guess_type(path) + guess, _ = mimetypes.guess_file_type(path) if guess: return guess return 'application/octet-stream' @@ -966,6 +1050,12 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): """ + def __init__(self, *args, **kwargs): + import warnings + warnings._deprecated("http.server.CGIHTTPRequestHandler", + remove=(3, 15)) + super().__init__(*args, **kwargs) + # Determine platform specifics have_fork = hasattr(os, 'fork') @@ -1078,7 +1168,7 @@ def run_cgi(self): "CGI script is not executable (%r)" % scriptname) return - # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html + # Reference: https://www6.uniovi.es/~antonio/ncsa_httpd/cgi/env.html # XXX Much of the following could be prepared ahead of time! env = copy.deepcopy(os.environ) env['SERVER_SOFTWARE'] = self.version_string() @@ -1198,7 +1288,18 @@ def run_cgi(self): env = env ) if self.command.lower() == "post" and nbytes > 0: - data = self.rfile.read(nbytes) + cursize = 0 + data = self.rfile.read(min(nbytes, _MIN_READ_BUF_SIZE)) + while len(data) < nbytes and len(data) != cursize: + cursize = len(data) + # This is a geometric increase in read size (never more + # than doubling out the current length of data per loop + # iteration). + delta = min(cursize, nbytes - cursize) + try: + data += self.rfile.read(delta) + except TimeoutError: + break else: data = None # throw away additional data [see bug #427345] @@ -1230,7 +1331,8 @@ def _get_best_family(*address): def test(HandlerClass=BaseHTTPRequestHandler, ServerClass=ThreadingHTTPServer, - protocol="HTTP/1.0", port=8000, bind=None): + protocol="HTTP/1.0", port=8000, bind=None, + tls_cert=None, tls_key=None, tls_password=None): """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the port argument). @@ -1238,12 +1340,20 @@ def test(HandlerClass=BaseHTTPRequestHandler, """ ServerClass.address_family, addr = _get_best_family(bind, port) HandlerClass.protocol_version = protocol - with ServerClass(addr, HandlerClass) as httpd: + + if tls_cert: + server = ServerClass(addr, HandlerClass, certfile=tls_cert, + keyfile=tls_key, password=tls_password) + else: + server = ServerClass(addr, HandlerClass) + + with server as httpd: host, port = httpd.socket.getsockname()[:2] url_host = f'[{host}]' if ':' in host else host + protocol = 'HTTPS' if tls_cert else 'HTTP' print( - f"Serving HTTP on {host} port {port} " - f"(http://{url_host}:{port}/) ..." + f"Serving {protocol} on {host} port {port} " + f"({protocol.lower()}://{url_host}:{port}/) ..." ) try: httpd.serve_forever() @@ -1255,26 +1365,51 @@ def test(HandlerClass=BaseHTTPRequestHandler, import argparse import contextlib - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument('--cgi', action='store_true', help='run as CGI server') - parser.add_argument('--bind', '-b', metavar='ADDRESS', - help='specify alternate bind address ' + parser.add_argument('-b', '--bind', metavar='ADDRESS', + help='bind to this address ' '(default: all interfaces)') - parser.add_argument('--directory', '-d', default=os.getcwd(), - help='specify alternate directory ' + parser.add_argument('-d', '--directory', default=os.getcwd(), + help='serve this directory ' '(default: current directory)') - parser.add_argument('port', action='store', default=8000, type=int, - nargs='?', - help='specify alternate port (default: 8000)') + parser.add_argument('-p', '--protocol', metavar='VERSION', + default='HTTP/1.0', + help='conform to this HTTP version ' + '(default: %(default)s)') + parser.add_argument('--tls-cert', metavar='PATH', + help='path to the TLS certificate chain file') + parser.add_argument('--tls-key', metavar='PATH', + help='path to the TLS key file') + parser.add_argument('--tls-password-file', metavar='PATH', + help='path to the password file for the TLS key') + parser.add_argument('port', default=8000, type=int, nargs='?', + help='bind to this port ' + '(default: %(default)s)') args = parser.parse_args() + + if not args.tls_cert and args.tls_key: + parser.error("--tls-key requires --tls-cert to be set") + + tls_key_password = None + if args.tls_password_file: + if not args.tls_cert: + parser.error("--tls-password-file requires --tls-cert to be set") + + try: + with open(args.tls_password_file, "r", encoding="utf-8") as f: + tls_key_password = f.read().strip() + except OSError as e: + parser.error(f"Failed to read TLS password file: {e}") + if args.cgi: handler_class = CGIHTTPRequestHandler else: handler_class = SimpleHTTPRequestHandler # ensure dual-stack is not disabled; ref #38907 - class DualStackServer(ThreadingHTTPServer): + class DualStackServerMixin: def server_bind(self): # suppress exception when protocol is IPv4 @@ -1287,9 +1422,20 @@ def finish_request(self, request, client_address): self.RequestHandlerClass(request, client_address, self, directory=args.directory) + class HTTPDualStackServer(DualStackServerMixin, ThreadingHTTPServer): + pass + class HTTPSDualStackServer(DualStackServerMixin, ThreadingHTTPSServer): + pass + + ServerClass = HTTPSDualStackServer if args.tls_cert else HTTPDualStackServer + test( HandlerClass=handler_class, - ServerClass=DualStackServer, + ServerClass=ServerClass, port=args.port, bind=args.bind, + protocol=args.protocol, + tls_cert=args.tls_cert, + tls_key=args.tls_key, + tls_password=tls_key_password, ) diff --git a/Lib/imaplib.py b/Lib/imaplib.py new file mode 100644 index 00000000000..cbe129b3e7c --- /dev/null +++ b/Lib/imaplib.py @@ -0,0 +1,1967 @@ +"""IMAP4 client. + +Based on RFC 2060. + +Public class: IMAP4 +Public variable: Debug +Public functions: Internaldate2tuple + Int2AP + ParseFlags + Time2Internaldate +""" + +# Author: Piers Lauder December 1997. +# +# Authentication code contributed by Donn Cave June 1998. +# String method conversion by ESR, February 2001. +# GET/SETACL contributed by Anthony Baxter April 2001. +# IMAP4_SSL contributed by Tino Lange March 2002. +# GET/SETQUOTA contributed by Andreas Zeidler June 2002. +# PROXYAUTH contributed by Rick Holbert November 2002. +# GET/SETANNOTATION contributed by Tomas Lindroos June 2005. +# IDLE contributed by Forest August 2024. + +__version__ = "2.60" + +import binascii, errno, random, re, socket, subprocess, sys, time, calendar +from datetime import datetime, timezone, timedelta +from io import DEFAULT_BUFFER_SIZE + +try: + import ssl + HAVE_SSL = True +except ImportError: + HAVE_SSL = False + +__all__ = ["IMAP4", "IMAP4_stream", "Internaldate2tuple", + "Int2AP", "ParseFlags", "Time2Internaldate"] + +# Globals + +CRLF = b'\r\n' +Debug = 0 +IMAP4_PORT = 143 +IMAP4_SSL_PORT = 993 +AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first + +# Maximal line length when calling readline(). This is to prevent +# reading arbitrary length lines. RFC 3501 and 2060 (IMAP 4rev1) +# don't specify a line length. RFC 2683 suggests limiting client +# command lines to 1000 octets and that servers should be prepared +# to accept command lines up to 8000 octets, so we used to use 10K here. +# In the modern world (eg: gmail) the response to, for example, a +# search command can be quite large, so we now use 1M. +_MAXLINE = 1000000 + + +# Commands + +Commands = { + # name valid states + 'APPEND': ('AUTH', 'SELECTED'), + 'AUTHENTICATE': ('NONAUTH',), + 'CAPABILITY': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'CHECK': ('SELECTED',), + 'CLOSE': ('SELECTED',), + 'COPY': ('SELECTED',), + 'CREATE': ('AUTH', 'SELECTED'), + 'DELETE': ('AUTH', 'SELECTED'), + 'DELETEACL': ('AUTH', 'SELECTED'), + 'ENABLE': ('AUTH', ), + 'EXAMINE': ('AUTH', 'SELECTED'), + 'EXPUNGE': ('SELECTED',), + 'FETCH': ('SELECTED',), + 'GETACL': ('AUTH', 'SELECTED'), + 'GETANNOTATION':('AUTH', 'SELECTED'), + 'GETQUOTA': ('AUTH', 'SELECTED'), + 'GETQUOTAROOT': ('AUTH', 'SELECTED'), + 'IDLE': ('AUTH', 'SELECTED'), + 'MYRIGHTS': ('AUTH', 'SELECTED'), + 'LIST': ('AUTH', 'SELECTED'), + 'LOGIN': ('NONAUTH',), + 'LOGOUT': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'LSUB': ('AUTH', 'SELECTED'), + 'MOVE': ('SELECTED',), + 'NAMESPACE': ('AUTH', 'SELECTED'), + 'NOOP': ('NONAUTH', 'AUTH', 'SELECTED', 'LOGOUT'), + 'PARTIAL': ('SELECTED',), # NB: obsolete + 'PROXYAUTH': ('AUTH',), + 'RENAME': ('AUTH', 'SELECTED'), + 'SEARCH': ('SELECTED',), + 'SELECT': ('AUTH', 'SELECTED'), + 'SETACL': ('AUTH', 'SELECTED'), + 'SETANNOTATION':('AUTH', 'SELECTED'), + 'SETQUOTA': ('AUTH', 'SELECTED'), + 'SORT': ('SELECTED',), + 'STARTTLS': ('NONAUTH',), + 'STATUS': ('AUTH', 'SELECTED'), + 'STORE': ('SELECTED',), + 'SUBSCRIBE': ('AUTH', 'SELECTED'), + 'THREAD': ('SELECTED',), + 'UID': ('SELECTED',), + 'UNSUBSCRIBE': ('AUTH', 'SELECTED'), + 'UNSELECT': ('SELECTED',), + } + +# Patterns to match server responses + +Continuation = re.compile(br'\+( (?P.*))?') +Flags = re.compile(br'.*FLAGS \((?P[^\)]*)\)') +InternalDate = re.compile(br'.*INTERNALDATE "' + br'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' + br' (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])' + br' (?P[-+])(?P[0-9][0-9])(?P[0-9][0-9])' + br'"') +# Literal is no longer used; kept for backward compatibility. +Literal = re.compile(br'.*{(?P\d+)}$', re.ASCII) +MapCRLF = re.compile(br'\r\n|\r|\n') +# We no longer exclude the ']' character from the data portion of the response +# code, even though it violates the RFC. Popular IMAP servers such as Gmail +# allow flags with ']', and there are programs (including imaplib!) that can +# produce them. The problem with this is if the 'text' portion of the response +# includes a ']' we'll parse the response wrong (which is the point of the RFC +# restriction). However, that seems less likely to be a problem in practice +# than being unable to correctly parse flags that include ']' chars, which +# was reported as a real-world problem in issue #21815. +Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P.*))?\]') +Untagged_response = re.compile(br'\* (?P[A-Z-]+)( (?P.*))?') +# Untagged_status is no longer used; kept for backward compatibility +Untagged_status = re.compile( + br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?', re.ASCII) +# We compile these in _mode_xxx. +_Literal = br'.*{(?P\d+)}$' +_Untagged_status = br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?' + + + +class IMAP4: + + r"""IMAP4 client class. + + Instantiate with: IMAP4([host[, port[, timeout=None]]]) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 port). + timeout - socket timeout (default: None) + If timeout is not given or is None, + the global default socket timeout is used + + All IMAP4rev1 commands are supported by methods of the same + name (in lowercase). + + All arguments to commands are converted to strings, except for + AUTHENTICATE, and the last argument to APPEND which is passed as + an IMAP4 literal. If necessary (the string contains any + non-printing characters or white-space and isn't enclosed with + either parentheses or double quotes) each string is quoted. + However, the 'password' argument to the LOGIN command is always + quoted. If you want to avoid having an argument string quoted + (eg: the 'flags' argument to STORE) then enclose the string in + parentheses (eg: "(\Deleted)"). + + Each command returns a tuple: (type, [data, ...]) where 'type' + is usually 'OK' or 'NO', and 'data' is either the text from the + tagged response, or untagged results from command. Each 'data' + is either a string, or a tuple. If a tuple, then the first part + is the header of the response, and the second part contains + the data (ie: 'literal' value). + + Errors raise the exception class .error(""). + IMAP4 server errors raise .abort(""), + which is a sub-class of 'error'. Mailbox status changes + from READ-WRITE to READ-ONLY raise the exception class + .readonly(""), which is a sub-class of 'abort'. + + "error" exceptions imply a program error. + "abort" exceptions imply the connection should be reset, and + the command re-tried. + "readonly" exceptions imply the command should be re-tried. + + Note: to use this module, you must read the RFCs pertaining to the + IMAP4 protocol, as the semantics of the arguments to each IMAP4 + command are left to the invoker, not to mention the results. Also, + most IMAP servers implement a sub-set of the commands available here. + """ + + class error(Exception): pass # Logical errors - debug required + class abort(error): pass # Service errors - close and retry + class readonly(abort): pass # Mailbox status changed to READ-ONLY + class _responsetimeout(TimeoutError): pass # No response during IDLE + + def __init__(self, host='', port=IMAP4_PORT, timeout=None): + self.debug = Debug + self.state = 'LOGOUT' + self.literal = None # A literal argument to a command + self.tagged_commands = {} # Tagged commands awaiting response + self.untagged_responses = {} # {typ: [data, ...], ...} + self.continuation_response = '' # Last continuation response + self._idle_responses = [] # Response queue for idle iteration + self._idle_capture = False # Whether to queue responses for idle + self.is_readonly = False # READ-ONLY desired state + self.tagnum = 0 + self._tls_established = False + self._mode_ascii() + self._readbuf = [] + + # Open socket to server. + + self.open(host, port, timeout) + + try: + self._connect() + except Exception: + try: + self.shutdown() + except OSError: + pass + raise + + def _mode_ascii(self): + self.utf8_enabled = False + self._encoding = 'ascii' + self.Literal = re.compile(_Literal, re.ASCII) + self.Untagged_status = re.compile(_Untagged_status, re.ASCII) + + + def _mode_utf8(self): + self.utf8_enabled = True + self._encoding = 'utf-8' + self.Literal = re.compile(_Literal) + self.Untagged_status = re.compile(_Untagged_status) + + + def _connect(self): + # Create unique tag for this session, + # and compile tagged response matcher. + + self.tagpre = Int2AP(random.randint(4096, 65535)) + self.tagre = re.compile(br'(?P' + + self.tagpre + + br'\d+) (?P[A-Z]+) (?P.*)', re.ASCII) + + # Get server welcome message, + # request and store CAPABILITY response. + + if __debug__: + self._cmd_log_len = 10 + self._cmd_log_idx = 0 + self._cmd_log = {} # Last '_cmd_log_len' interactions + if self.debug >= 1: + self._mesg('imaplib version %s' % __version__) + self._mesg('new IMAP4 connection, tag=%s' % self.tagpre) + + self.welcome = self._get_response() + if 'PREAUTH' in self.untagged_responses: + self.state = 'AUTH' + elif 'OK' in self.untagged_responses: + self.state = 'NONAUTH' + else: + raise self.error(self.welcome) + + self._get_capabilities() + if __debug__: + if self.debug >= 3: + self._mesg('CAPABILITIES: %r' % (self.capabilities,)) + + for version in AllowedVersions: + if not version in self.capabilities: + continue + self.PROTOCOL_VERSION = version + return + + raise self.error('server not IMAP4 compliant') + + + def __getattr__(self, attr): + # Allow UPPERCASE variants of IMAP4 command methods. + if attr in Commands: + return getattr(self, attr.lower()) + raise AttributeError("Unknown IMAP4 command: '%s'" % attr) + + def __enter__(self): + return self + + def __exit__(self, *args): + if self.state == "LOGOUT": + return + + try: + self.logout() + except OSError: + pass + + + # Overridable methods + + + def _create_socket(self, timeout): + # Default value of IMAP4.host is '', but socket.getaddrinfo() + # (which is used by socket.create_connection()) expects None + # as a default value for host. + if timeout is not None and not timeout: + raise ValueError('Non-blocking socket (timeout=0) is not supported') + host = None if not self.host else self.host + sys.audit("imaplib.open", self, self.host, self.port) + address = (host, self.port) + if timeout is not None: + return socket.create_connection(address, timeout) + return socket.create_connection(address) + + def open(self, host='', port=IMAP4_PORT, timeout=None): + """Setup connection to remote server on "host:port" + (default: localhost:standard IMAP4 port). + This connection will be used by the routines: + read, readline, send, shutdown. + """ + self.host = host + self.port = port + self.sock = self._create_socket(timeout) + self._file = self.sock.makefile('rb') + + + @property + def file(self): + # The old 'file' attribute is no longer used now that we do our own + # read() and readline() buffering, with which it conflicts. + # As an undocumented interface, it should never have been accessed by + # external code, and therefore does not warrant deprecation. + # Nevertheless, we provide this property for now, to avoid suddenly + # breaking any code in the wild that might have been using it in a + # harmless way. + import warnings + warnings.warn( + 'IMAP4.file is unsupported, can cause errors, and may be removed.', + RuntimeWarning, + stacklevel=2) + return self._file + + + def read(self, size): + """Read 'size' bytes from remote.""" + # We need buffered read() to continue working after socket timeouts, + # since we use them during IDLE. Unfortunately, the standard library's + # SocketIO implementation makes this impossible, by setting a permanent + # error condition instead of letting the caller decide how to handle a + # timeout. We therefore implement our own buffered read(). + # https://github.com/python/cpython/issues/51571 + # + # Reading in chunks instead of delegating to a single + # BufferedReader.read() call also means we avoid its preallocation + # of an unreasonably large memory block if a malicious server claims + # it will send a huge literal without actually sending one. + # https://github.com/python/cpython/issues/119511 + + parts = [] + + while size > 0: + + if len(parts) < len(self._readbuf): + buf = self._readbuf[len(parts)] + else: + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break + if not buf: + break + self._readbuf.append(buf) + + if len(buf) >= size: + parts.append(buf[:size]) + self._readbuf = [buf[size:]] + self._readbuf[len(parts):] + break + parts.append(buf) + size -= len(buf) + + return b''.join(parts) + + + def readline(self): + """Read line from remote.""" + # The comment in read() explains why we implement our own readline(). + + LF = b'\n' + parts = [] + length = 0 + + while length < _MAXLINE: + + if len(parts) < len(self._readbuf): + buf = self._readbuf[len(parts)] + else: + try: + buf = self.sock.recv(DEFAULT_BUFFER_SIZE) + except ConnectionError: + break + if not buf: + break + self._readbuf.append(buf) + + pos = buf.find(LF) + if pos != -1: + pos += 1 + parts.append(buf[:pos]) + self._readbuf = [buf[pos:]] + self._readbuf[len(parts):] + break + parts.append(buf) + length += len(buf) + + line = b''.join(parts) + if len(line) > _MAXLINE: + raise self.error("got more than %d bytes" % _MAXLINE) + return line + + + def send(self, data): + """Send data to remote.""" + sys.audit("imaplib.send", self, data) + self.sock.sendall(data) + + + def shutdown(self): + """Close I/O established in "open".""" + self._file.close() + try: + self.sock.shutdown(socket.SHUT_RDWR) + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): + raise + finally: + self.sock.close() + + + def socket(self): + """Return socket instance used to connect to IMAP4 server. + + socket = .socket() + """ + return self.sock + + + + # Utility methods + + + def recent(self): + """Return most recent 'RECENT' responses if any exist, + else prompt server for an update using the 'NOOP' command. + + (typ, [data]) = .recent() + + 'data' is None if no new messages, + else list of RECENT responses, most recent last. + """ + name = 'RECENT' + typ, dat = self._untagged_response('OK', [None], name) + if dat[-1]: + return typ, dat + typ, dat = self.noop() # Prod server for response + return self._untagged_response(typ, dat, name) + + + def response(self, code): + """Return data for response 'code' if received, or None. + + Old value for response 'code' is cleared. + + (code, [data]) = .response(code) + """ + return self._untagged_response(code, [None], code.upper()) + + + + # IMAP4 commands + + + def append(self, mailbox, flags, date_time, message): + """Append message to named mailbox. + + (typ, [data]) = .append(mailbox, flags, date_time, message) + + All args except 'message' can be None. + """ + name = 'APPEND' + if not mailbox: + mailbox = 'INBOX' + if flags: + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags + else: + flags = None + if date_time: + date_time = Time2Internaldate(date_time) + else: + date_time = None + literal = MapCRLF.sub(CRLF, message) + self.literal = literal + return self._simple_command(name, mailbox, flags, date_time) + + + def authenticate(self, mechanism, authobject): + """Authenticate command - requires response processing. + + 'mechanism' specifies which authentication mechanism is to + be used - it must appear in .capabilities in the + form AUTH=. + + 'authobject' must be a callable object: + + data = authobject(response) + + It will be called to process server continuation responses; the + response argument it is passed will be a bytes. It should return bytes + data that will be base64 encoded and sent to the server. It should + return None if the client abort response '*' should be sent instead. + """ + mech = mechanism.upper() + # XXX: shouldn't this code be removed, not commented out? + #cap = 'AUTH=%s' % mech + #if not cap in self.capabilities: # Let the server decide! + # raise self.error("Server doesn't allow %s authentication." % mech) + self.literal = _Authenticator(authobject).process + typ, dat = self._simple_command('AUTHENTICATE', mech) + if typ != 'OK': + raise self.error(dat[-1].decode('utf-8', 'replace')) + self.state = 'AUTH' + return typ, dat + + + def capability(self): + """(typ, [data]) = .capability() + Fetch capabilities list from server.""" + + name = 'CAPABILITY' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def check(self): + """Checkpoint mailbox on server. + + (typ, [data]) = .check() + """ + return self._simple_command('CHECK') + + + def close(self): + """Close currently selected mailbox. + + Deleted messages are removed from writable mailbox. + This is the recommended command before 'LOGOUT'. + + (typ, [data]) = .close() + """ + try: + typ, dat = self._simple_command('CLOSE') + finally: + self.state = 'AUTH' + return typ, dat + + + def copy(self, message_set, new_mailbox): + """Copy 'message_set' messages onto end of 'new_mailbox'. + + (typ, [data]) = .copy(message_set, new_mailbox) + """ + return self._simple_command('COPY', message_set, new_mailbox) + + + def create(self, mailbox): + """Create new mailbox. + + (typ, [data]) = .create(mailbox) + """ + return self._simple_command('CREATE', mailbox) + + + def delete(self, mailbox): + """Delete old mailbox. + + (typ, [data]) = .delete(mailbox) + """ + return self._simple_command('DELETE', mailbox) + + def deleteacl(self, mailbox, who): + """Delete the ACLs (remove any rights) set for who on mailbox. + + (typ, [data]) = .deleteacl(mailbox, who) + """ + return self._simple_command('DELETEACL', mailbox, who) + + def enable(self, capability): + """Send an RFC5161 enable string to the server. + + (typ, [data]) = .enable(capability) + """ + if 'ENABLE' not in self.capabilities: + raise IMAP4.error("Server does not support ENABLE") + typ, data = self._simple_command('ENABLE', capability) + if typ == 'OK' and 'UTF8=ACCEPT' in capability.upper(): + self._mode_utf8() + return typ, data + + def expunge(self): + """Permanently remove deleted items from selected mailbox. + + Generates 'EXPUNGE' response for each deleted message. + + (typ, [data]) = .expunge() + + 'data' is list of 'EXPUNGE'd message numbers in order received. + """ + name = 'EXPUNGE' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def fetch(self, message_set, message_parts): + """Fetch (parts of) messages. + + (typ, [data, ...]) = .fetch(message_set, message_parts) + + 'message_parts' should be a string of selected parts + enclosed in parentheses, eg: "(UID BODY[TEXT])". + + 'data' are tuples of message part envelope and data. + """ + name = 'FETCH' + typ, dat = self._simple_command(name, message_set, message_parts) + return self._untagged_response(typ, dat, name) + + + def getacl(self, mailbox): + """Get the ACLs for a mailbox. + + (typ, [data]) = .getacl(mailbox) + """ + typ, dat = self._simple_command('GETACL', mailbox) + return self._untagged_response(typ, dat, 'ACL') + + + def getannotation(self, mailbox, entry, attribute): + """(typ, [data]) = .getannotation(mailbox, entry, attribute) + Retrieve ANNOTATIONs.""" + + typ, dat = self._simple_command('GETANNOTATION', mailbox, entry, attribute) + return self._untagged_response(typ, dat, 'ANNOTATION') + + + def getquota(self, root): + """Get the quota root's resource usage and limits. + + Part of the IMAP4 QUOTA extension defined in rfc2087. + + (typ, [data]) = .getquota(root) + """ + typ, dat = self._simple_command('GETQUOTA', root) + return self._untagged_response(typ, dat, 'QUOTA') + + + def getquotaroot(self, mailbox): + """Get the list of quota roots for the named mailbox. + + (typ, [[QUOTAROOT responses...], [QUOTA responses]]) = .getquotaroot(mailbox) + """ + typ, dat = self._simple_command('GETQUOTAROOT', mailbox) + typ, quota = self._untagged_response(typ, dat, 'QUOTA') + typ, quotaroot = self._untagged_response(typ, dat, 'QUOTAROOT') + return typ, [quotaroot, quota] + + + def idle(self, duration=None): + """Return an iterable IDLE context manager producing untagged responses. + If the argument is not None, limit iteration to 'duration' seconds. + + with M.idle(duration=29 * 60) as idler: + for typ, data in idler: + print(typ, data) + + Note: 'duration' requires a socket connection (not IMAP4_stream). + """ + return Idler(self, duration) + + + def list(self, directory='""', pattern='*'): + """List mailbox names in directory matching pattern. + + (typ, [data]) = .list(directory='""', pattern='*') + + 'data' is list of LIST responses. + """ + name = 'LIST' + typ, dat = self._simple_command(name, directory, pattern) + return self._untagged_response(typ, dat, name) + + + def login(self, user, password): + """Identify client using plaintext password. + + (typ, [data]) = .login(user, password) + + NB: 'password' will be quoted. + """ + typ, dat = self._simple_command('LOGIN', user, self._quote(password)) + if typ != 'OK': + raise self.error(dat[-1]) + self.state = 'AUTH' + return typ, dat + + + def login_cram_md5(self, user, password): + """ Force use of CRAM-MD5 authentication. + + (typ, [data]) = .login_cram_md5(user, password) + """ + self.user, self.password = user, password + return self.authenticate('CRAM-MD5', self._CRAM_MD5_AUTH) + + + def _CRAM_MD5_AUTH(self, challenge): + """ Authobject to use with CRAM-MD5 authentication. """ + import hmac + + if isinstance(self.password, str): + password = self.password.encode('utf-8') + else: + password = self.password + + try: + authcode = hmac.HMAC(password, challenge, 'md5') + except ValueError: # HMAC-MD5 is not available + raise self.error("CRAM-MD5 authentication is not supported") + return f"{self.user} {authcode.hexdigest()}" + + + def logout(self): + """Shutdown connection to server. + + (typ, [data]) = .logout() + + Returns server 'BYE' response. + """ + self.state = 'LOGOUT' + typ, dat = self._simple_command('LOGOUT') + self.shutdown() + return typ, dat + + + def lsub(self, directory='""', pattern='*'): + """List 'subscribed' mailbox names in directory matching pattern. + + (typ, [data, ...]) = .lsub(directory='""', pattern='*') + + 'data' are tuples of message part envelope and data. + """ + name = 'LSUB' + typ, dat = self._simple_command(name, directory, pattern) + return self._untagged_response(typ, dat, name) + + def myrights(self, mailbox): + """Show my ACLs for a mailbox (i.e. the rights that I have on mailbox). + + (typ, [data]) = .myrights(mailbox) + """ + typ,dat = self._simple_command('MYRIGHTS', mailbox) + return self._untagged_response(typ, dat, 'MYRIGHTS') + + def namespace(self): + """ Returns IMAP namespaces ala rfc2342 + + (typ, [data, ...]) = .namespace() + """ + name = 'NAMESPACE' + typ, dat = self._simple_command(name) + return self._untagged_response(typ, dat, name) + + + def noop(self): + """Send NOOP command. + + (typ, [data]) = .noop() + """ + if __debug__: + if self.debug >= 3: + self._dump_ur(self.untagged_responses) + return self._simple_command('NOOP') + + + def partial(self, message_num, message_part, start, length): + """Fetch truncated part of a message. + + (typ, [data, ...]) = .partial(message_num, message_part, start, length) + + 'data' is tuple of message part envelope and data. + """ + name = 'PARTIAL' + typ, dat = self._simple_command(name, message_num, message_part, start, length) + return self._untagged_response(typ, dat, 'FETCH') + + + def proxyauth(self, user): + """Assume authentication as "user". + + Allows an authorised administrator to proxy into any user's + mailbox. + + (typ, [data]) = .proxyauth(user) + """ + + name = 'PROXYAUTH' + return self._simple_command('PROXYAUTH', user) + + + def rename(self, oldmailbox, newmailbox): + """Rename old mailbox name to new. + + (typ, [data]) = .rename(oldmailbox, newmailbox) + """ + return self._simple_command('RENAME', oldmailbox, newmailbox) + + + def search(self, charset, *criteria): + """Search mailbox for matching messages. + + (typ, [data]) = .search(charset, criterion, ...) + + 'data' is space separated list of matching message numbers. + If UTF8 is enabled, charset MUST be None. + """ + name = 'SEARCH' + if charset: + if self.utf8_enabled: + raise IMAP4.error("Non-None charset not valid in UTF8 mode") + typ, dat = self._simple_command(name, 'CHARSET', charset, *criteria) + else: + typ, dat = self._simple_command(name, *criteria) + return self._untagged_response(typ, dat, name) + + + def select(self, mailbox='INBOX', readonly=False): + """Select a mailbox. + + Flush all untagged responses. + + (typ, [data]) = .select(mailbox='INBOX', readonly=False) + + 'data' is count of messages in mailbox ('EXISTS' response). + + Mandated responses are ('FLAGS', 'EXISTS', 'RECENT', 'UIDVALIDITY'), so + other responses should be obtained via .response('FLAGS') etc. + """ + self.untagged_responses = {} # Flush old responses. + self.is_readonly = readonly + if readonly: + name = 'EXAMINE' + else: + name = 'SELECT' + typ, dat = self._simple_command(name, mailbox) + if typ != 'OK': + self.state = 'AUTH' # Might have been 'SELECTED' + return typ, dat + self.state = 'SELECTED' + if 'READ-ONLY' in self.untagged_responses \ + and not readonly: + if __debug__: + if self.debug >= 1: + self._dump_ur(self.untagged_responses) + raise self.readonly('%s is not writable' % mailbox) + return typ, self.untagged_responses.get('EXISTS', [None]) + + + def setacl(self, mailbox, who, what): + """Set a mailbox acl. + + (typ, [data]) = .setacl(mailbox, who, what) + """ + return self._simple_command('SETACL', mailbox, who, what) + + + def setannotation(self, *args): + """(typ, [data]) = .setannotation(mailbox[, entry, attribute]+) + Set ANNOTATIONs.""" + + typ, dat = self._simple_command('SETANNOTATION', *args) + return self._untagged_response(typ, dat, 'ANNOTATION') + + + def setquota(self, root, limits): + """Set the quota root's resource limits. + + (typ, [data]) = .setquota(root, limits) + """ + typ, dat = self._simple_command('SETQUOTA', root, limits) + return self._untagged_response(typ, dat, 'QUOTA') + + + def sort(self, sort_criteria, charset, *search_criteria): + """IMAP4rev1 extension SORT command. + + (typ, [data]) = .sort(sort_criteria, charset, search_criteria, ...) + """ + name = 'SORT' + #if not name in self.capabilities: # Let the server decide! + # raise self.error('unimplemented extension command: %s' % name) + if (sort_criteria[0],sort_criteria[-1]) != ('(',')'): + sort_criteria = '(%s)' % sort_criteria + typ, dat = self._simple_command(name, sort_criteria, charset, *search_criteria) + return self._untagged_response(typ, dat, name) + + + def starttls(self, ssl_context=None): + name = 'STARTTLS' + if not HAVE_SSL: + raise self.error('SSL support missing') + if self._tls_established: + raise self.abort('TLS session already established') + if name not in self.capabilities: + raise self.abort('TLS not supported by server') + # Generate a default SSL context if none was passed. + if ssl_context is None: + ssl_context = ssl._create_stdlib_context() + typ, dat = self._simple_command(name) + if typ == 'OK': + self.sock = ssl_context.wrap_socket(self.sock, + server_hostname=self.host) + self._file = self.sock.makefile('rb') + self._tls_established = True + self._get_capabilities() + else: + raise self.error("Couldn't establish TLS session") + return self._untagged_response(typ, dat, name) + + + def status(self, mailbox, names): + """Request named status conditions for mailbox. + + (typ, [data]) = .status(mailbox, names) + """ + name = 'STATUS' + #if self.PROTOCOL_VERSION == 'IMAP4': # Let the server decide! + # raise self.error('%s unimplemented in IMAP4 (obtain IMAP4rev1 server, or re-code)' % name) + typ, dat = self._simple_command(name, mailbox, names) + return self._untagged_response(typ, dat, name) + + + def store(self, message_set, command, flags): + """Alters flag dispositions for messages in mailbox. + + (typ, [data]) = .store(message_set, command, flags) + """ + if (flags[0],flags[-1]) != ('(',')'): + flags = '(%s)' % flags # Avoid quoting the flags + typ, dat = self._simple_command('STORE', message_set, command, flags) + return self._untagged_response(typ, dat, 'FETCH') + + + def subscribe(self, mailbox): + """Subscribe to new mailbox. + + (typ, [data]) = .subscribe(mailbox) + """ + return self._simple_command('SUBSCRIBE', mailbox) + + + def thread(self, threading_algorithm, charset, *search_criteria): + """IMAPrev1 extension THREAD command. + + (type, [data]) = .thread(threading_algorithm, charset, search_criteria, ...) + """ + name = 'THREAD' + typ, dat = self._simple_command(name, threading_algorithm, charset, *search_criteria) + return self._untagged_response(typ, dat, name) + + + def uid(self, command, *args): + """Execute "command arg ..." with messages identified by UID, + rather than message number. + + (typ, [data]) = .uid(command, arg1, arg2, ...) + + Returns response appropriate to 'command'. + """ + command = command.upper() + if not command in Commands: + raise self.error("Unknown IMAP4 UID command: %s" % command) + if self.state not in Commands[command]: + raise self.error("command %s illegal in state %s, " + "only allowed in states %s" % + (command, self.state, + ', '.join(Commands[command]))) + name = 'UID' + typ, dat = self._simple_command(name, command, *args) + if command in ('SEARCH', 'SORT', 'THREAD'): + name = command + else: + name = 'FETCH' + return self._untagged_response(typ, dat, name) + + + def unsubscribe(self, mailbox): + """Unsubscribe from old mailbox. + + (typ, [data]) = .unsubscribe(mailbox) + """ + return self._simple_command('UNSUBSCRIBE', mailbox) + + + def unselect(self): + """Free server's resources associated with the selected mailbox + and returns the server to the authenticated state. + This command performs the same actions as CLOSE, except + that no messages are permanently removed from the currently + selected mailbox. + + (typ, [data]) = .unselect() + """ + try: + typ, data = self._simple_command('UNSELECT') + finally: + self.state = 'AUTH' + return typ, data + + + def xatom(self, name, *args): + """Allow simple extension commands + notified by server in CAPABILITY response. + + Assumes command is legal in current state. + + (typ, [data]) = .xatom(name, arg, ...) + + Returns response appropriate to extension command 'name'. + """ + name = name.upper() + #if not name in self.capabilities: # Let the server decide! + # raise self.error('unknown extension command: %s' % name) + if not name in Commands: + Commands[name] = (self.state,) + return self._simple_command(name, *args) + + + + # Private methods + + + def _append_untagged(self, typ, dat): + if dat is None: + dat = b'' + + # During idle, queue untagged responses for delivery via iteration + if self._idle_capture: + # Responses containing literal strings are passed to us one data + # fragment at a time, while others arrive in a single call. + if (not self._idle_responses or + isinstance(self._idle_responses[-1][1][-1], bytes)): + # We are not continuing a fragmented response; start a new one + self._idle_responses.append((typ, [dat])) + else: + # We are continuing a fragmented response; append the fragment + response = self._idle_responses[-1] + assert response[0] == typ + response[1].append(dat) + if __debug__ and self.debug >= 5: + self._mesg(f'idle: queue untagged {typ} {dat!r}') + return + + ur = self.untagged_responses + if __debug__: + if self.debug >= 5: + self._mesg('untagged_responses[%s] %s += ["%r"]' % + (typ, len(ur.get(typ,'')), dat)) + if typ in ur: + ur[typ].append(dat) + else: + ur[typ] = [dat] + + + def _check_bye(self): + bye = self.untagged_responses.get('BYE') + if bye: + raise self.abort(bye[-1].decode(self._encoding, 'replace')) + + + def _command(self, name, *args): + + if self.state not in Commands[name]: + self.literal = None + raise self.error("command %s illegal in state %s, " + "only allowed in states %s" % + (name, self.state, + ', '.join(Commands[name]))) + + for typ in ('OK', 'NO', 'BAD'): + if typ in self.untagged_responses: + del self.untagged_responses[typ] + + if 'READ-ONLY' in self.untagged_responses \ + and not self.is_readonly: + raise self.readonly('mailbox status changed to READ-ONLY') + + tag = self._new_tag() + name = bytes(name, self._encoding) + data = tag + b' ' + name + for arg in args: + if arg is None: continue + if isinstance(arg, str): + arg = bytes(arg, self._encoding) + data = data + b' ' + arg + + literal = self.literal + if literal is not None: + self.literal = None + if type(literal) is type(self._command): + literator = literal + else: + literator = None + if self.utf8_enabled: + data = data + bytes(' UTF8 (~{%s}' % len(literal), self._encoding) + literal = literal + b')' + else: + data = data + bytes(' {%s}' % len(literal), self._encoding) + + if __debug__: + if self.debug >= 4: + self._mesg('> %r' % data) + else: + self._log('> %r' % data) + + try: + self.send(data + CRLF) + except OSError as val: + raise self.abort('socket error: %s' % val) + + if literal is None: + return tag + + while 1: + # Wait for continuation response + + while self._get_response(): + if self.tagged_commands[tag]: # BAD/NO? + return tag + + # Send literal + + if literator: + literal = literator(self.continuation_response) + + if __debug__: + if self.debug >= 4: + self._mesg('write literal size %s' % len(literal)) + + try: + self.send(literal) + self.send(CRLF) + except OSError as val: + raise self.abort('socket error: %s' % val) + + if not literator: + break + + return tag + + + def _command_complete(self, name, tag): + logout = (name == 'LOGOUT') + # BYE is expected after LOGOUT + if not logout: + self._check_bye() + try: + typ, data = self._get_tagged_response(tag, expect_bye=logout) + except self.abort as val: + raise self.abort('command: %s => %s' % (name, val)) + except self.error as val: + raise self.error('command: %s => %s' % (name, val)) + if not logout: + self._check_bye() + if typ == 'BAD': + raise self.error('%s command error: %s %s' % (name, typ, data)) + return typ, data + + + def _get_capabilities(self): + typ, dat = self.capability() + if dat == [None]: + raise self.error('no CAPABILITY response from server') + dat = str(dat[-1], self._encoding) + dat = dat.upper() + self.capabilities = tuple(dat.split()) + + + def _get_response(self, start_timeout=False): + + # Read response and store. + # + # Returns None for continuation responses, + # otherwise first response line received. + # + # If start_timeout is given, temporarily uses it as a socket + # timeout while waiting for the start of a response, raising + # _responsetimeout if one doesn't arrive. (Used by Idler.) + + if start_timeout is not False and self.sock: + assert start_timeout is None or start_timeout > 0 + saved_timeout = self.sock.gettimeout() + self.sock.settimeout(start_timeout) + try: + resp = self._get_line() + except TimeoutError as err: + raise self._responsetimeout from err + finally: + self.sock.settimeout(saved_timeout) + else: + resp = self._get_line() + + # Command completion response? + + if self._match(self.tagre, resp): + tag = self.mo.group('tag') + if not tag in self.tagged_commands: + raise self.abort('unexpected tagged response: %r' % resp) + + typ = self.mo.group('type') + typ = str(typ, self._encoding) + dat = self.mo.group('data') + self.tagged_commands[tag] = (typ, [dat]) + else: + dat2 = None + + # '*' (untagged) responses? + + if not self._match(Untagged_response, resp): + if self._match(self.Untagged_status, resp): + dat2 = self.mo.group('data2') + + if self.mo is None: + # Only other possibility is '+' (continuation) response... + + if self._match(Continuation, resp): + self.continuation_response = self.mo.group('data') + return None # NB: indicates continuation + + raise self.abort("unexpected response: %r" % resp) + + typ = self.mo.group('type') + typ = str(typ, self._encoding) + dat = self.mo.group('data') + if dat is None: dat = b'' # Null untagged response + if dat2: dat = dat + b' ' + dat2 + + # Is there a literal to come? + + while self._match(self.Literal, dat): + + # Read literal direct from connection. + + size = int(self.mo.group('size')) + if __debug__: + if self.debug >= 4: + self._mesg('read literal size %s' % size) + data = self.read(size) + + # Store response with literal as tuple + + self._append_untagged(typ, (dat, data)) + + # Read trailer - possibly containing another literal + + dat = self._get_line() + + self._append_untagged(typ, dat) + + # Bracketed response information? + + if typ in ('OK', 'NO', 'BAD') and self._match(Response_code, dat): + typ = self.mo.group('type') + typ = str(typ, self._encoding) + self._append_untagged(typ, self.mo.group('data')) + + if __debug__: + if self.debug >= 1 and typ in ('NO', 'BAD', 'BYE'): + self._mesg('%s response: %r' % (typ, dat)) + + return resp + + + def _get_tagged_response(self, tag, expect_bye=False): + + while 1: + result = self.tagged_commands[tag] + if result is not None: + del self.tagged_commands[tag] + return result + + if expect_bye: + typ = 'BYE' + bye = self.untagged_responses.pop(typ, None) + if bye is not None: + # Server replies to the "LOGOUT" command with "BYE" + return (typ, bye) + + # If we've seen a BYE at this point, the socket will be + # closed, so report the BYE now. + self._check_bye() + + # Some have reported "unexpected response" exceptions. + # Note that ignoring them here causes loops. + # Instead, send me details of the unexpected response and + # I'll update the code in '_get_response()'. + + try: + self._get_response() + except self.abort as val: + if __debug__: + if self.debug >= 1: + self.print_log() + raise + + + def _get_line(self): + + line = self.readline() + if not line: + raise self.abort('socket error: EOF') + + # Protocol mandates all lines terminated by CRLF + if not line.endswith(b'\r\n'): + raise self.abort('socket error: unterminated line: %r' % line) + + line = line[:-2] + if __debug__: + if self.debug >= 4: + self._mesg('< %r' % line) + else: + self._log('< %r' % line) + return line + + + def _match(self, cre, s): + + # Run compiled regular expression match method on 's'. + # Save result, return success. + + self.mo = cre.match(s) + if __debug__: + if self.mo is not None and self.debug >= 5: + self._mesg("\tmatched %r => %r" % (cre.pattern, self.mo.groups())) + return self.mo is not None + + + def _new_tag(self): + + tag = self.tagpre + bytes(str(self.tagnum), self._encoding) + self.tagnum = self.tagnum + 1 + self.tagged_commands[tag] = None + return tag + + + def _quote(self, arg): + + arg = arg.replace('\\', '\\\\') + arg = arg.replace('"', '\\"') + + return '"' + arg + '"' + + + def _simple_command(self, name, *args): + + return self._command_complete(name, self._command(name, *args)) + + + def _untagged_response(self, typ, dat, name): + if typ == 'NO': + return typ, dat + if not name in self.untagged_responses: + return typ, [None] + data = self.untagged_responses.pop(name) + if __debug__: + if self.debug >= 5: + self._mesg('untagged_responses[%s] => %s' % (name, data)) + return typ, data + + + if __debug__: + + def _mesg(self, s, secs=None): + if secs is None: + secs = time.time() + tm = time.strftime('%M:%S', time.localtime(secs)) + sys.stderr.write(' %s.%02d %s\n' % (tm, (secs*100)%100, s)) + sys.stderr.flush() + + def _dump_ur(self, untagged_resp_dict): + if not untagged_resp_dict: + return + items = (f'{key}: {value!r}' + for key, value in untagged_resp_dict.items()) + self._mesg('untagged responses dump:' + '\n\t\t'.join(items)) + + def _log(self, line): + # Keep log of last '_cmd_log_len' interactions for debugging. + self._cmd_log[self._cmd_log_idx] = (line, time.time()) + self._cmd_log_idx += 1 + if self._cmd_log_idx >= self._cmd_log_len: + self._cmd_log_idx = 0 + + def print_log(self): + self._mesg('last %d IMAP4 interactions:' % len(self._cmd_log)) + i, n = self._cmd_log_idx, self._cmd_log_len + while n: + try: + self._mesg(*self._cmd_log[i]) + except: + pass + i += 1 + if i >= self._cmd_log_len: + i = 0 + n -= 1 + + +class Idler: + """Iterable IDLE context manager: start IDLE & produce untagged responses. + + An object of this type is returned by the IMAP4.idle() method. + + Note: The name and structure of this class are subject to change. + """ + + def __init__(self, imap, duration=None): + if 'IDLE' not in imap.capabilities: + raise imap.error("Server does not support IMAP4 IDLE") + if duration is not None and not imap.sock: + # IMAP4_stream pipes don't support timeouts + raise imap.error('duration requires a socket connection') + self._duration = duration + self._deadline = None + self._imap = imap + self._tag = None + self._saved_state = None + + def __enter__(self): + imap = self._imap + assert not imap._idle_responses + assert not imap._idle_capture + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle start duration={self._duration}') + + # Start capturing untagged responses before sending IDLE, + # so we can deliver via iteration any that arrive while + # the IDLE command continuation request is still pending. + imap._idle_capture = True + + try: + self._tag = imap._command('IDLE') + # As with any command, the server is allowed to send us unrelated, + # untagged responses before acting on IDLE. These lines will be + # returned by _get_response(). When the server is ready, it will + # send an IDLE continuation request, indicated by _get_response() + # returning None. We therefore process responses in a loop until + # this occurs. + while resp := imap._get_response(): + if imap.tagged_commands[self._tag]: + typ, data = imap.tagged_commands.pop(self._tag) + if typ == 'NO': + raise imap.error(f'idle denied: {data}') + raise imap.abort(f'unexpected status response: {resp}') + + if __debug__ and imap.debug >= 4: + prompt = imap.continuation_response + imap._mesg(f'idle continuation prompt: {prompt}') + except BaseException: + imap._idle_capture = False + raise + + if self._duration is not None: + self._deadline = time.monotonic() + self._duration + + self._saved_state = imap.state + imap.state = 'IDLING' + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + imap = self._imap + + if __debug__ and imap.debug >= 4: + imap._mesg('idle done') + imap.state = self._saved_state + + # Stop intercepting untagged responses before sending DONE, + # since we can no longer deliver them via iteration. + imap._idle_capture = False + + # If we captured untagged responses while the IDLE command + # continuation request was still pending, but the user did not + # iterate over them before exiting IDLE, we must put them + # someplace where the user can retrieve them. The only + # sensible place for this is the untagged_responses dict, + # despite its unfortunate inability to preserve the relative + # order of different response types. + if leftovers := len(imap._idle_responses): + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle quit with {leftovers} leftover responses') + while imap._idle_responses: + typ, data = imap._idle_responses.pop(0) + # Append one fragment at a time, just as _get_response() does + for datum in data: + imap._append_untagged(typ, datum) + + try: + imap.send(b'DONE' + CRLF) + status, [msg] = imap._command_complete('IDLE', self._tag) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle status: {status} {msg!r}') + except OSError: + if not exc_type: + raise + + return False # Do not suppress context body exceptions + + def __iter__(self): + return self + + def _pop(self, timeout, default=('', None)): + # Get the next response, or a default value on timeout. + # The timeout arg can be an int or float, or None for no timeout. + # Timeouts require a socket connection (not IMAP4_stream). + # This method ignores self._duration. + + # Historical Note: + # The timeout was originally implemented using select() after + # checking for the presence of already-buffered data. + # That allowed timeouts on pipe connetions like IMAP4_stream. + # However, it seemed possible that SSL data arriving without any + # IMAP data afterward could cause select() to indicate available + # application data when there was none, leading to a read() call + # that would block with no timeout. It was unclear under what + # conditions this would happen in practice. Our implementation was + # changed to use socket timeouts instead of select(), just to be + # safe. + + imap = self._imap + if imap.state != 'IDLING': + raise imap.error('_pop() only works during IDLE') + + if imap._idle_responses: + # Response is ready to return to the user + resp = imap._idle_responses.pop(0) + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) de-queued {resp[0]}') + return resp + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) reading') + + if timeout is not None: + if timeout <= 0: + return default + timeout = float(timeout) # Required by socket.settimeout() + + try: + imap._get_response(timeout) # Reads line, calls _append_untagged() + except IMAP4._responsetimeout: + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) done') + return default + + resp = imap._idle_responses.pop(0) + + if __debug__ and imap.debug >= 4: + imap._mesg(f'idle _pop({timeout}) read {resp[0]}') + return resp + + def __next__(self): + imap = self._imap + + if self._duration is None: + timeout = None + else: + timeout = self._deadline - time.monotonic() + typ, data = self._pop(timeout) + + if not typ: + if __debug__ and imap.debug >= 4: + imap._mesg('idle iterator exhausted') + raise StopIteration + + return typ, data + + def burst(self, interval=0.1): + """Yield a burst of responses no more than 'interval' seconds apart. + + with M.idle() as idler: + # get a response and any others following by < 0.1 seconds + batch = list(idler.burst()) + print(f'processing {len(batch)} responses...') + print(batch) + + Note: This generator requires a socket connection (not IMAP4_stream). + """ + if not self._imap.sock: + raise self._imap.error('burst() requires a socket connection') + + try: + yield next(self) + except StopIteration: + return + + while response := self._pop(interval, None): + yield response + + +if HAVE_SSL: + + class IMAP4_SSL(IMAP4): + + """IMAP4 client class over SSL connection + + Instantiate with: IMAP4_SSL([host[, port[, ssl_context[, timeout=None]]]]) + + host - host's name (default: localhost); + port - port number (default: standard IMAP4 SSL port); + ssl_context - a SSLContext object that contains your certificate chain + and private key (default: None) + timeout - socket timeout (default: None) If timeout is not given or is None, + the global default socket timeout is used + + for more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, host='', port=IMAP4_SSL_PORT, + *, ssl_context=None, timeout=None): + if ssl_context is None: + ssl_context = ssl._create_stdlib_context() + self.ssl_context = ssl_context + IMAP4.__init__(self, host, port, timeout) + + def _create_socket(self, timeout): + sock = IMAP4._create_socket(self, timeout) + return self.ssl_context.wrap_socket(sock, + server_hostname=self.host) + + def open(self, host='', port=IMAP4_SSL_PORT, timeout=None): + """Setup connection to remote server on "host:port". + (default: localhost:standard IMAP4 SSL port). + This connection will be used by the routines: + read, readline, send, shutdown. + """ + IMAP4.open(self, host, port, timeout) + + __all__.append("IMAP4_SSL") + + +class IMAP4_stream(IMAP4): + + """IMAP4 client class over a stream + + Instantiate with: IMAP4_stream(command) + + "command" - a string that can be passed to subprocess.Popen() + + for more documentation see the docstring of the parent class IMAP4. + """ + + + def __init__(self, command): + self.command = command + IMAP4.__init__(self) + + + def open(self, host=None, port=None, timeout=None): + """Setup a stream connection. + This connection will be used by the routines: + read, readline, send, shutdown. + """ + self.host = None # For compatibility with parent class + self.port = None + self.sock = None + self._file = None + self.process = subprocess.Popen(self.command, + bufsize=DEFAULT_BUFFER_SIZE, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + shell=True, close_fds=True) + self.writefile = self.process.stdin + self.readfile = self.process.stdout + + def read(self, size): + """Read 'size' bytes from remote.""" + return self.readfile.read(size) + + + def readline(self): + """Read line from remote.""" + return self.readfile.readline() + + + def send(self, data): + """Send data to remote.""" + self.writefile.write(data) + self.writefile.flush() + + + def shutdown(self): + """Close I/O established in "open".""" + self.readfile.close() + self.writefile.close() + self.process.wait() + + + +class _Authenticator: + + """Private class to provide en/decoding + for base64-based authentication conversation. + """ + + def __init__(self, mechinst): + self.mech = mechinst # Callable object to provide/process data + + def process(self, data): + ret = self.mech(self.decode(data)) + if ret is None: + return b'*' # Abort conversation + return self.encode(ret) + + def encode(self, inp): + # + # Invoke binascii.b2a_base64 iteratively with + # short even length buffers, strip the trailing + # line feed from the result and append. "Even" + # means a number that factors to both 6 and 8, + # so when it gets to the end of the 8-bit input + # there's no partial 6-bit output. + # + oup = b'' + if isinstance(inp, str): + inp = inp.encode('utf-8') + while inp: + if len(inp) > 48: + t = inp[:48] + inp = inp[48:] + else: + t = inp + inp = b'' + e = binascii.b2a_base64(t) + if e: + oup = oup + e[:-1] + return oup + + def decode(self, inp): + if not inp: + return b'' + return binascii.a2b_base64(inp) + +Months = ' Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split(' ') +Mon2num = {s.encode():n+1 for n, s in enumerate(Months[1:])} + +def Internaldate2tuple(resp): + """Parse an IMAP4 INTERNALDATE string. + + Return corresponding local time. The return value is a + time.struct_time tuple or None if the string has wrong format. + """ + + mo = InternalDate.match(resp) + if not mo: + return None + + mon = Mon2num[mo.group('mon')] + zonen = mo.group('zonen') + + day = int(mo.group('day')) + year = int(mo.group('year')) + hour = int(mo.group('hour')) + min = int(mo.group('min')) + sec = int(mo.group('sec')) + zoneh = int(mo.group('zoneh')) + zonem = int(mo.group('zonem')) + + # INTERNALDATE timezone must be subtracted to get UT + + zone = (zoneh*60 + zonem)*60 + if zonen == b'-': + zone = -zone + + tt = (year, mon, day, hour, min, sec, -1, -1, -1) + utc = calendar.timegm(tt) - zone + + return time.localtime(utc) + + + +def Int2AP(num): + + """Convert integer to A-P string representation.""" + + val = b''; AP = b'ABCDEFGHIJKLMNOP' + num = int(abs(num)) + while num: + num, mod = divmod(num, 16) + val = AP[mod:mod+1] + val + return val + + + +def ParseFlags(resp): + + """Convert IMAP4 flags response to python tuple.""" + + mo = Flags.match(resp) + if not mo: + return () + + return tuple(mo.group('flags').split()) + + +def Time2Internaldate(date_time): + + """Convert date_time to IMAP4 INTERNALDATE representation. + + Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'. The + date_time argument can be a number (int or float) representing + seconds since epoch (as returned by time.time()), a 9-tuple + representing local time, an instance of time.struct_time (as + returned by time.localtime()), an aware datetime instance or a + double-quoted string. In the last case, it is assumed to already + be in the correct format. + """ + if isinstance(date_time, (int, float)): + dt = datetime.fromtimestamp(date_time, + timezone.utc).astimezone() + elif isinstance(date_time, tuple): + try: + gmtoff = date_time.tm_gmtoff + except AttributeError: + if time.daylight: + dst = date_time[8] + if dst == -1: + dst = time.localtime(time.mktime(date_time))[8] + gmtoff = -(time.timezone, time.altzone)[dst] + else: + gmtoff = -time.timezone + delta = timedelta(seconds=gmtoff) + dt = datetime(*date_time[:6], tzinfo=timezone(delta)) + elif isinstance(date_time, datetime): + if date_time.tzinfo is None: + raise ValueError("date_time must be aware") + dt = date_time + elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'): + return date_time # Assume in correct format + else: + raise ValueError("date_time not of a known type") + fmt = '"%d-{}-%Y %H:%M:%S %z"'.format(Months[dt.month]) + return dt.strftime(fmt) + + + +if __name__ == '__main__': + + # To test: invoke either as 'python imaplib.py [IMAP4_server_hostname]' + # or 'python imaplib.py -s "rsh IMAP4_server_hostname exec /etc/rimapd"' + # to test the IMAP4_stream class + + import getopt, getpass + + try: + optlist, args = getopt.getopt(sys.argv[1:], 'd:s:') + except getopt.error as val: + optlist, args = (), () + + stream_command = None + for opt,val in optlist: + if opt == '-d': + Debug = int(val) + elif opt == '-s': + stream_command = val + if not args: args = (stream_command,) + + if not args: args = ('',) + + host = args[0] + + USER = getpass.getuser() + PASSWD = getpass.getpass("IMAP password for %s on %s: " % (USER, host or "localhost")) + + test_mesg = 'From: %(user)s@localhost%(lf)sSubject: IMAP4 test%(lf)s%(lf)sdata...%(lf)s' % {'user':USER, 'lf':'\n'} + test_seq1 = ( + ('login', (USER, PASSWD)), + ('create', ('/tmp/xxx 1',)), + ('rename', ('/tmp/xxx 1', '/tmp/yyy')), + ('CREATE', ('/tmp/yyz 2',)), + ('append', ('/tmp/yyz 2', None, None, test_mesg)), + ('list', ('/tmp', 'yy*')), + ('select', ('/tmp/yyz 2',)), + ('search', (None, 'SUBJECT', 'test')), + ('fetch', ('1', '(FLAGS INTERNALDATE RFC822)')), + ('store', ('1', 'FLAGS', r'(\Deleted)')), + ('namespace', ()), + ('expunge', ()), + ('recent', ()), + ('close', ()), + ) + + test_seq2 = ( + ('select', ()), + ('response',('UIDVALIDITY',)), + ('uid', ('SEARCH', 'ALL')), + ('response', ('EXISTS',)), + ('append', (None, None, None, test_mesg)), + ('recent', ()), + ('logout', ()), + ) + + def run(cmd, args): + M._mesg('%s %s' % (cmd, args)) + typ, dat = getattr(M, cmd)(*args) + M._mesg('%s => %s %s' % (cmd, typ, dat)) + if typ == 'NO': raise dat[0] + return dat + + try: + if stream_command: + M = IMAP4_stream(stream_command) + else: + M = IMAP4(host) + if M.state == 'AUTH': + test_seq1 = test_seq1[1:] # Login not needed + M._mesg('PROTOCOL_VERSION = %s' % M.PROTOCOL_VERSION) + M._mesg('CAPABILITIES = %r' % (M.capabilities,)) + + for cmd,args in test_seq1: + run(cmd, args) + + for ml in run('list', ('/tmp/', 'yy%')): + mo = re.match(r'.*"([^"]+)"$', ml) + if mo: path = mo.group(1) + else: path = ml.split()[-1] + run('delete', (path,)) + + for cmd,args in test_seq2: + dat = run(cmd, args) + + if (cmd,args) != ('uid', ('SEARCH', 'ALL')): + continue + + uid = dat[-1].split() + if not uid: continue + run('uid', ('FETCH', '%s' % uid[-1], + '(FLAGS INTERNALDATE RFC822.SIZE RFC822.HEADER RFC822.TEXT)')) + + print('\nAll tests OK.') + + except: + print('\nTests failed.') + + if not Debug: + print(''' +If you would like to see debugging output, +try: %s -d5 +''' % sys.argv[0]) + + raise diff --git a/Lib/imghdr.py b/Lib/imghdr.py deleted file mode 100644 index 6a372e66c7f..00000000000 --- a/Lib/imghdr.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Recognize image file formats based on their first few bytes.""" - -from os import PathLike -import warnings - -__all__ = ["what"] - - -warnings._deprecated(__name__, remove=(3, 13)) - - -#-------------------------# -# Recognize image headers # -#-------------------------# - -def what(file, h=None): - f = None - try: - if h is None: - if isinstance(file, (str, PathLike)): - f = open(file, 'rb') - h = f.read(32) - else: - location = file.tell() - h = file.read(32) - file.seek(location) - for tf in tests: - res = tf(h, f) - if res: - return res - finally: - if f: f.close() - return None - - -#---------------------------------# -# Subroutines per image file type # -#---------------------------------# - -tests = [] - -def test_jpeg(h, f): - """JPEG data with JFIF or Exif markers; and raw JPEG""" - if h[6:10] in (b'JFIF', b'Exif'): - return 'jpeg' - elif h[:4] == b'\xff\xd8\xff\xdb': - return 'jpeg' - -tests.append(test_jpeg) - -def test_png(h, f): - if h.startswith(b'\211PNG\r\n\032\n'): - return 'png' - -tests.append(test_png) - -def test_gif(h, f): - """GIF ('87 and '89 variants)""" - if h[:6] in (b'GIF87a', b'GIF89a'): - return 'gif' - -tests.append(test_gif) - -def test_tiff(h, f): - """TIFF (can be in Motorola or Intel byte order)""" - if h[:2] in (b'MM', b'II'): - return 'tiff' - -tests.append(test_tiff) - -def test_rgb(h, f): - """SGI image library""" - if h.startswith(b'\001\332'): - return 'rgb' - -tests.append(test_rgb) - -def test_pbm(h, f): - """PBM (portable bitmap)""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'14' and h[2] in b' \t\n\r': - return 'pbm' - -tests.append(test_pbm) - -def test_pgm(h, f): - """PGM (portable graymap)""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'25' and h[2] in b' \t\n\r': - return 'pgm' - -tests.append(test_pgm) - -def test_ppm(h, f): - """PPM (portable pixmap)""" - if len(h) >= 3 and \ - h[0] == ord(b'P') and h[1] in b'36' and h[2] in b' \t\n\r': - return 'ppm' - -tests.append(test_ppm) - -def test_rast(h, f): - """Sun raster file""" - if h.startswith(b'\x59\xA6\x6A\x95'): - return 'rast' - -tests.append(test_rast) - -def test_xbm(h, f): - """X bitmap (X10 or X11)""" - if h.startswith(b'#define '): - return 'xbm' - -tests.append(test_xbm) - -def test_bmp(h, f): - if h.startswith(b'BM'): - return 'bmp' - -tests.append(test_bmp) - -def test_webp(h, f): - if h.startswith(b'RIFF') and h[8:12] == b'WEBP': - return 'webp' - -tests.append(test_webp) - -def test_exr(h, f): - if h.startswith(b'\x76\x2f\x31\x01'): - return 'exr' - -tests.append(test_exr) - -#--------------------# -# Small test program # -#--------------------# - -def test(): - import sys - recursive = 0 - if sys.argv[1:] and sys.argv[1] == '-r': - del sys.argv[1:2] - recursive = 1 - try: - if sys.argv[1:]: - testall(sys.argv[1:], recursive, 1) - else: - testall(['.'], recursive, 1) - except KeyboardInterrupt: - sys.stderr.write('\n[Interrupted]\n') - sys.exit(1) - -def testall(list, recursive, toplevel): - import sys - import os - for filename in list: - if os.path.isdir(filename): - print(filename + '/:', end=' ') - if recursive or toplevel: - print('recursing down:') - import glob - names = glob.glob(os.path.join(glob.escape(filename), '*')) - testall(names, recursive, 0) - else: - print('*** directory (use -r) ***') - else: - print(filename + ':', end=' ') - sys.stdout.flush() - try: - print(what(filename)) - except OSError: - print('*** not found ***') - -if __name__ == '__main__': - test() diff --git a/Lib/imp.py b/Lib/imp.py deleted file mode 100644 index fc42c157658..00000000000 --- a/Lib/imp.py +++ /dev/null @@ -1,346 +0,0 @@ -"""This module provides the components needed to build your own __import__ -function. Undocumented functions are obsolete. - -In most cases it is preferred you consider using the importlib module's -functionality over this module. - -""" -# (Probably) need to stay in _imp -from _imp import (lock_held, acquire_lock, release_lock, - get_frozen_object, is_frozen_package, - init_frozen, is_builtin, is_frozen, - _fix_co_filename, _frozen_module_names) -try: - from _imp import create_dynamic -except ImportError: - # Platform doesn't support dynamic loading. - create_dynamic = None - -from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name -from importlib._bootstrap_external import SourcelessFileLoader - -from importlib import machinery -from importlib import util -import importlib -import os -import sys -import tokenize -import types -import warnings - -warnings.warn("the imp module is deprecated in favour of importlib and slated " - "for removal in Python 3.12; " - "see the module's documentation for alternative uses", - DeprecationWarning, stacklevel=2) - -# DEPRECATED -SEARCH_ERROR = 0 -PY_SOURCE = 1 -PY_COMPILED = 2 -C_EXTENSION = 3 -PY_RESOURCE = 4 -PKG_DIRECTORY = 5 -C_BUILTIN = 6 -PY_FROZEN = 7 -PY_CODERESOURCE = 8 -IMP_HOOK = 9 - - -def new_module(name): - """**DEPRECATED** - - Create a new module. - - The module is not entered into sys.modules. - - """ - return types.ModuleType(name) - - -def get_magic(): - """**DEPRECATED** - - Return the magic number for .pyc files. - """ - return util.MAGIC_NUMBER - - -def get_tag(): - """Return the magic tag for .pyc files.""" - return sys.implementation.cache_tag - - -def cache_from_source(path, debug_override=None): - """**DEPRECATED** - - Given the path to a .py file, return the path to its .pyc file. - - The .py file does not need to exist; this simply returns the path to the - .pyc file calculated as if the .py file were imported. - - If debug_override is not None, then it must be a boolean and is used in - place of sys.flags.optimize. - - If sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - return util.cache_from_source(path, debug_override) - - -def source_from_cache(path): - """**DEPRECATED** - - Given the path to a .pyc. file, return the path to its .py file. - - The .pyc file does not need to exist; this simply returns the path to - the .py file calculated to correspond to the .pyc file. If path does - not conform to PEP 3147 format, ValueError will be raised. If - sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - return util.source_from_cache(path) - - -def get_suffixes(): - """**DEPRECATED**""" - extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES] - source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] - bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] - - return extensions + source + bytecode - - -class NullImporter: - - """**DEPRECATED** - - Null import object. - - """ - - def __init__(self, path): - if path == '': - raise ImportError('empty pathname', path='') - elif os.path.isdir(path): - raise ImportError('existing directory', path=path) - - def find_module(self, fullname): - """Always returns None.""" - return None - - -class _HackedGetData: - - """Compatibility support for 'file' arguments of various load_*() - functions.""" - - def __init__(self, fullname, path, file=None): - super().__init__(fullname, path) - self.file = file - - def get_data(self, path): - """Gross hack to contort loader to deal w/ load_*()'s bad API.""" - if self.file and path == self.path: - # The contract of get_data() requires us to return bytes. Reopen the - # file in binary mode if needed. - if not self.file.closed: - file = self.file - if 'b' not in file.mode: - file.close() - if self.file.closed: - self.file = file = open(self.path, 'rb') - - with file: - return file.read() - else: - return super().get_data(path) - - -class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader): - - """Compatibility support for implementing load_source().""" - - -def load_source(name, pathname, file=None): - loader = _LoadSourceCompatibility(name, pathname, file) - spec = util.spec_from_file_location(name, pathname, loader=loader) - if name in sys.modules: - module = _exec(spec, sys.modules[name]) - else: - module = _load(spec) - # To allow reloading to potentially work, use a non-hacked loader which - # won't rely on a now-closed file object. - module.__loader__ = machinery.SourceFileLoader(name, pathname) - module.__spec__.loader = module.__loader__ - return module - - -class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader): - - """Compatibility support for implementing load_compiled().""" - - -def load_compiled(name, pathname, file=None): - """**DEPRECATED**""" - loader = _LoadCompiledCompatibility(name, pathname, file) - spec = util.spec_from_file_location(name, pathname, loader=loader) - if name in sys.modules: - module = _exec(spec, sys.modules[name]) - else: - module = _load(spec) - # To allow reloading to potentially work, use a non-hacked loader which - # won't rely on a now-closed file object. - module.__loader__ = SourcelessFileLoader(name, pathname) - module.__spec__.loader = module.__loader__ - return module - - -def load_package(name, path): - """**DEPRECATED**""" - if os.path.isdir(path): - extensions = (machinery.SOURCE_SUFFIXES[:] + - machinery.BYTECODE_SUFFIXES[:]) - for extension in extensions: - init_path = os.path.join(path, '__init__' + extension) - if os.path.exists(init_path): - path = init_path - break - else: - raise ValueError('{!r} is not a package'.format(path)) - spec = util.spec_from_file_location(name, path, - submodule_search_locations=[]) - if name in sys.modules: - return _exec(spec, sys.modules[name]) - else: - return _load(spec) - - -def load_module(name, file, filename, details): - """**DEPRECATED** - - Load a module, given information returned by find_module(). - - The module name must include the full package name, if any. - - """ - suffix, mode, type_ = details - if mode and (not mode.startswith('r') or '+' in mode): - raise ValueError('invalid file open mode {!r}'.format(mode)) - elif file is None and type_ in {PY_SOURCE, PY_COMPILED}: - msg = 'file object required for import (type code {})'.format(type_) - raise ValueError(msg) - elif type_ == PY_SOURCE: - return load_source(name, filename, file) - elif type_ == PY_COMPILED: - return load_compiled(name, filename, file) - elif type_ == C_EXTENSION and load_dynamic is not None: - if file is None: - with open(filename, 'rb') as opened_file: - return load_dynamic(name, filename, opened_file) - else: - return load_dynamic(name, filename, file) - elif type_ == PKG_DIRECTORY: - return load_package(name, filename) - elif type_ == C_BUILTIN: - return init_builtin(name) - elif type_ == PY_FROZEN: - return init_frozen(name) - else: - msg = "Don't know how to import {} (type code {})".format(name, type_) - raise ImportError(msg, name=name) - - -def find_module(name, path=None): - """**DEPRECATED** - - Search for a module. - - If path is omitted or None, search for a built-in, frozen or special - module and continue search in sys.path. The module name cannot - contain '.'; to search for a submodule of a package, pass the - submodule name and the package's __path__. - - """ - if not isinstance(name, str): - raise TypeError("'name' must be a str, not {}".format(type(name))) - elif not isinstance(path, (type(None), list)): - # Backwards-compatibility - raise RuntimeError("'path' must be None or a list, " - "not {}".format(type(path))) - - if path is None: - if is_builtin(name): - return None, None, ('', '', C_BUILTIN) - elif is_frozen(name): - return None, None, ('', '', PY_FROZEN) - else: - path = sys.path - - for entry in path: - package_directory = os.path.join(entry, name) - for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: - package_file_name = '__init__' + suffix - file_path = os.path.join(package_directory, package_file_name) - if os.path.isfile(file_path): - return None, package_directory, ('', '', PKG_DIRECTORY) - for suffix, mode, type_ in get_suffixes(): - file_name = name + suffix - file_path = os.path.join(entry, file_name) - if os.path.isfile(file_path): - break - else: - continue - break # Break out of outer loop when breaking out of inner loop. - else: - raise ImportError(_ERR_MSG.format(name), name=name) - - encoding = None - if 'b' not in mode: - with open(file_path, 'rb') as file: - encoding = tokenize.detect_encoding(file.readline)[0] - file = open(file_path, mode, encoding=encoding) - return file, file_path, (suffix, mode, type_) - - -def reload(module): - """**DEPRECATED** - - Reload the module and return it. - - The module must have been successfully imported before. - - """ - return importlib.reload(module) - - -def init_builtin(name): - """**DEPRECATED** - - Load and return a built-in module by name, or None is such module doesn't - exist - """ - try: - return _builtin_from_name(name) - except ImportError: - return None - - -if create_dynamic: - def load_dynamic(name, path, file=None): - """**DEPRECATED** - - Load an extension module. - """ - import importlib.machinery - loader = importlib.machinery.ExtensionFileLoader(name, path) - - # Issue #24748: Skip the sys.modules check in _load_module_shim; - # always load new extension - spec = importlib.machinery.ModuleSpec( - name=name, loader=loader, origin=path) - return _load(spec) - -else: - load_dynamic = None diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 707c081cb2c..a7d57561ead 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -54,8 +54,6 @@ # Fully bootstrapped at this point, import whatever you like, circular # dependencies and startup overhead minimisation permitting :) -import warnings - # Public API ######################################################### @@ -105,7 +103,7 @@ def reload(module): try: name = module.__name__ except AttributeError: - raise TypeError("reload() argument must be a module") + raise TypeError("reload() argument must be a module") from None if sys.modules.get(name) is not module: raise ImportError(f"module {name} not in sys.modules", name=name) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 093a0b82456..499da1e04ef 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -53,7 +53,7 @@ def _new_module(name): # For a list that can have a weakref to it. class _List(list): - pass + __slots__ = ("__weakref__",) # Copied from weakref.py with some simplifications and modifications unique to @@ -382,6 +382,9 @@ def release(self): self.waiters.pop() self.wakeup.release() + def locked(self): + return bool(self.count) + def __repr__(self): return f'_ModuleLock({self.name!r}) at {id(self)}' @@ -490,8 +493,7 @@ def _call_with_frames_removed(f, *args, **kwds): def _verbose_message(message, *args, verbosity=1): """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - # XXX RUSTPYTHON: hasattr check because we might be bootstrapping and we wouldn't have stderr yet - if sys.flags.verbose >= verbosity and hasattr(sys, "stderr"): + if sys.flags.verbose >= verbosity: if not message.startswith(('#', 'import ')): message = '# ' + message print(message.format(*args), file=sys.stderr) @@ -527,7 +529,7 @@ def _load_module_shim(self, fullname): """ msg = ("the load_module() method is deprecated and slated for removal in " - "Python 3.12; use exec_module() instead") + "Python 3.15; use exec_module() instead") _warnings.warn(msg, DeprecationWarning) spec = spec_from_loader(fullname, self) if fullname in sys.modules: @@ -825,10 +827,16 @@ def _module_repr_from_spec(spec): """Return the repr to use for the module.""" name = '?' if spec.name is None else spec.name if spec.origin is None: - if spec.loader is None: + loader = spec.loader + if loader is None: return f'' + elif ( + _bootstrap_external is not None + and isinstance(loader, _bootstrap_external.NamespaceLoader) + ): + return f'' else: - return f'' + return f'' else: if spec.has_location: return f'' @@ -1129,7 +1137,7 @@ def find_spec(cls, fullname, path=None, target=None): # part of the importer), instead of here (the finder part). # The loader is the usual place to get the data that will # be loaded into the module. (For example, see _LoaderBasics - # in _bootstra_external.py.) Most importantly, this importer + # in _bootstrap_external.py.) Most importantly, this importer # is simpler if we wait to get the data. # However, getting as much data in the finder as possible # to later load the module is okay, and sometimes important. @@ -1236,10 +1244,12 @@ def _find_spec(name, path, target=None): """Find a module's spec.""" meta_path = sys.meta_path if meta_path is None: - # PyImport_Cleanup() is running or has been called. raise ImportError("sys.meta_path is None, Python is likely " "shutting down") + # gh-130094: Copy sys.meta_path so that we have a consistent view of the + # list while iterating over it. + meta_path = list(meta_path) if not meta_path: _warnings.warn('sys.meta_path is empty', ImportWarning) @@ -1294,7 +1304,6 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' def _find_and_load_unlocked(name, import_): path = None @@ -1304,8 +1313,9 @@ def _find_and_load_unlocked(name, import_): if parent not in sys.modules: _call_with_frames_removed(import_, parent) # Crazy side-effects! - if name in sys.modules: - return sys.modules[name] + module = sys.modules.get(name) + if module is not None: + return module parent_module = sys.modules[parent] try: path = parent_module.__path__ @@ -1313,6 +1323,12 @@ def _find_and_load_unlocked(name, import_): msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ + if getattr(parent_spec, '_initializing', False): + _call_with_frames_removed(import_, parent) + # Crazy side-effects (again)! + module = sys.modules.get(name) + if module is not None: + return module child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 73ac4405cb5..95ce14b2c39 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -52,7 +52,7 @@ # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', -_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) @@ -81,6 +81,11 @@ def _pack_uint32(x): return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') +def _unpack_uint64(data): + """Convert 8 bytes in little-endian to an integer.""" + assert len(data) == 8 + return int.from_bytes(data, 'little') + def _unpack_uint32(data): """Convert 4 bytes in little-endian to an integer.""" assert len(data) == 4 @@ -203,7 +208,7 @@ def _write_atomic(path, data, mode=0o666): try: # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. - with _io.FileIO(fd, 'wb') as file: + with _io.open(fd, 'wb') as file: file.write(data) _os.replace(path_tmp, path) except OSError: @@ -216,254 +221,7 @@ def _write_atomic(path, data, mode=0o666): _code_type = type(_write_atomic.__code__) - -# Finder/loader utility code ############################################### - -# Magic word to reject .pyc files generated by other Python versions. -# It should change for each incompatible change to the bytecode. -# -# The value of CR and LF is incorporated so if you ever read or write -# a .pyc file in text mode the magic number will be wrong; also, the -# Apple MPW compiler swaps their values, botching string constants. -# -# There were a variety of old schemes for setting the magic number. -# The current working scheme is to increment the previous value by -# 10. -# -# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -# number also includes a new "magic tag", i.e. a human readable string used -# to represent the magic number in __pycache__ directories. When you change -# the magic number, you must also set a new unique magic tag. Generally this -# can be named after the Python major version of the magic number bump, but -# it can really be anything, as long as it's different than anything else -# that's come before. The tags are included in the following table, starting -# with Python 3.2a0. -# -# Known values: -# Python 1.5: 20121 -# Python 1.5.1: 20121 -# Python 1.5.2: 20121 -# Python 1.6: 50428 -# Python 2.0: 50823 -# Python 2.0.1: 50823 -# Python 2.1: 60202 -# Python 2.1.1: 60202 -# Python 2.1.2: 60202 -# Python 2.2: 60717 -# Python 2.3a0: 62011 -# Python 2.3a0: 62021 -# Python 2.3a0: 62011 (!) -# Python 2.4a0: 62041 -# Python 2.4a3: 62051 -# Python 2.4b1: 62061 -# Python 2.5a0: 62071 -# Python 2.5a0: 62081 (ast-branch) -# Python 2.5a0: 62091 (with) -# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) -# Python 2.5b3: 62101 (fix wrong code: for x, in ...) -# Python 2.5b3: 62111 (fix wrong code: x += yield) -# Python 2.5c1: 62121 (fix wrong lnotab with for loops and -# storing constants that should have been removed) -# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) -# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) -# Python 2.6a1: 62161 (WITH_CLEANUP optimization) -# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) -# Python 2.7a0: 62181 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 2.7a0 62191 (introduce SETUP_WITH) -# Python 2.7a0 62201 (introduce BUILD_SET) -# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -# Python 3000: 3000 -# 3010 (removed UNARY_CONVERT) -# 3020 (added BUILD_SET) -# 3030 (added keyword-only parameters) -# 3040 (added signature annotations) -# 3050 (print becomes a function) -# 3060 (PEP 3115 metaclass syntax) -# 3061 (string literals become unicode) -# 3071 (PEP 3109 raise changes) -# 3081 (PEP 3137 make __file__ and __name__ unicode) -# 3091 (kill str8 interning) -# 3101 (merge from 2.6a0, see 62151) -# 3103 (__file__ points to source file) -# Python 3.0a4: 3111 (WITH_CLEANUP optimization). -# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT - #3021) -# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: -# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) -# Python 3.1a1: 3151 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE - #4715) -# Python 3.2a1: 3160 (add SETUP_WITH #6101) -# tag: cpython-32 -# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) -# tag: cpython-32 -# Python 3.2a3 3180 (add DELETE_DEREF #4617) -# Python 3.3a1 3190 (__class__ super closure changed) -# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) -# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) -# Python 3.3a2 3220 (changed PEP 380 implementation #14230) -# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) -# Python 3.4a1 3250 (evaluate positional default arguments before -# keyword-only defaults #16967) -# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override -# free vars #17853) -# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) -# Python 3.4a1 3280 (remove implicit class argument) -# Python 3.4a4 3290 (changes to __qualname__ computation #19301) -# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) -# Python 3.4rc2 3310 (alter __qualname__ computation #20625) -# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) -# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) -# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) -# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) -# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) -# Python 3.6a2 3370 (16 bit wordcode #26647) -# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) -# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE -# #27095) -# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) -# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes -# #27985) -# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL - #27213) -# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) -# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) -# Python 3.6rc1 3379 (more thorough __class__ validation #23722) -# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) -# Python 3.7a2 3391 (update GET_AITER #31709) -# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) -# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the first stmt in the body; -# this might affected the first line number #32911) -# Python 3.8a1 3400 (move frame block handling to compiler #17611) -# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) -# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) -# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict -# comprehensions #35224) -# Python 3.8b2 3412 (Swap the position of positional args and positional -# only args in ast.arguments #37593) -# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) -# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) -# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) -# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) -# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) -# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) -# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) -# Python 3.10a1 3430 (Make 'annotations' future by default) -# Python 3.10a1 3431 (New line number table format -- PEP 626) -# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) -# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) -# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) -# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). -# Python 3.10b1 3436 (Add GEN_START bytecode #43683) -# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) -# Python 3.10b1 3438 Safer line number table handling. -# Python 3.10b1 3439 (Add ROT_N) -# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) -# Python 3.11a1 3451 (Add CALL_METHOD_KW) -# Python 3.11a1 3452 (drop nlocals from marshaled code objects) -# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) -# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) -# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) -# Python 3.11a1 3456 (interleave cell args bpo-43693) -# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) -# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) -# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) -# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) -# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) -# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change -# MATCH_CLASS and MATCH_KEYS, and add COPY) -# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the -# active exception) -# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into -# BINARY_OP) -# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) -# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) -# Python 3.11a4 3467 (Change CALL_xxx opcodes) -# Python 3.11a4 3468 (Add SEND opcode) -# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) -# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) -# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) -# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) -# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) -# Python 3.11a4 3474 (Add RESUME opcode) -# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) -# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) -# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and -# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) -# Python 3.11a5 3478 (New CALL opcodes) -# Python 3.11a5 3479 (Add PUSH_NULL opcode) -# Python 3.11a5 3480 (New CALL opcodes, second iteration) -# Python 3.11a5 3481 (Use inline cache for BINARY_OP) -# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) -# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) -# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and -# STORE_ATTR) -# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) -# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) -# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) -# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) -# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) -# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) -# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, -# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) -# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) -# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) -# Python 3.11a7 3494 (New location info table) -# Python 3.12a1 3500 (Remove PRECALL opcode) -# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) -# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) -# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) -# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) -# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) -# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) -# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) -# Python 3.12a1 3508 (Add CLEANUP_THROW) -# Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) -# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) -# Python 3.12a2 3512 (Remove all unused consts from code objects) -# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) -# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) -# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) -# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) -# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a6 3518 (Add RETURN_CONST instruction) -# Python 3.12a6 3519 (Modify SEND instruction) -# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) -# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) -# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) -# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) -# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) -# Python 3.12b1 3525 (Shrink the CALL caches) -# Python 3.12b1 3526 (Add instrumentation support) -# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) -# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) -# Python 3.12b1 3529 (Inline list/dict/set comprehensions) -# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) -# Python 3.12b1 3531 (Add PEP 695 changes) - -# Python 3.13 will start with 3550 - -# Please don't copy-paste the same pre-release tag for new entries above!!! -# You should always use the *upcoming* tag. For example, if 3.12a6 came out -# a week ago, I should put "Python 3.12a7" next to my new magic number. - -# MAGIC must change whenever the bytecode emitted by the compiler may no -# longer be understood by older implementations of the eval loop (usually -# due to the addition of new opcodes). -# -# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. -# -# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array -# in PC/launcher.c must also be updated. - -MAGIC_NUMBER = (3531).to_bytes(2, 'little') + b'\r\n' - -_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c +MAGIC_NUMBER = _imp.pyc_magic_number_token.to_bytes(4, 'little') _PYCACHE = '__pycache__' _OPT = 'opt-' @@ -535,7 +293,8 @@ def cache_from_source(path, debug_override=None, *, optimization=None): # Strip initial drive from a Windows path. We know we have an absolute # path here, so the second part of the check rules out a POSIX path that # happens to contain a colon at the second character. - if head[1] == ':' and head[0] not in path_separators: + # Slicing avoids issues with an empty (or short) `head`. + if head[1:2] == ':' and head[0:1] not in path_separators: head = head[2:] # Strip initial path separator from `head` to complete the conversion @@ -954,6 +713,12 @@ def _search_registry(cls, fullname): @classmethod def find_spec(cls, fullname, path=None, target=None): + _warnings.warn('importlib.machinery.WindowsRegistryFinder is ' + 'deprecated; use site configuration instead. ' + 'Future versions of Python may not enable this ' + 'finder by default.', + DeprecationWarning, stacklevel=2) + filepath = cls._search_registry(fullname) if filepath is None: return None @@ -1102,7 +867,7 @@ def get_code(self, fullname): _imp.check_hash_based_pycs == 'always')): source_bytes = self.get_data(source_path) source_hash = _imp.source_hash( - _RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) _validate_hash_pyc(data, source_hash, fullname, @@ -1131,7 +896,7 @@ def get_code(self, fullname): source_mtime is not None): if hash_based: if source_hash is None: - source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, + source_hash = _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) data = _code_to_hash_pyc(code_object, source_hash, check_source) else: @@ -1437,7 +1202,7 @@ class PathFinder: @staticmethod def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders - stored in sys.path_importer_caches (where implemented).""" + stored in sys.path_importer_cache (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): # Drop entry if finder name is a relative path. The current # working directory may have changed. @@ -1449,6 +1214,9 @@ def invalidate_caches(): # https://bugs.python.org/issue45703 _NamespacePath._epoch += 1 + from importlib.metadata import MetadataPathFinder + MetadataPathFinder.invalidate_caches() + @staticmethod def _path_hooks(path): """Search sys.path_hooks for a finder for 'path'.""" @@ -1473,7 +1241,7 @@ def _path_importer_cache(cls, path): if path == '': try: path = _os.getcwd() - except FileNotFoundError: + except (FileNotFoundError, PermissionError): # Don't cache the failure as the cwd can easily change to # a valid directory later on. return None @@ -1690,6 +1458,52 @@ def __repr__(self): return f'FileFinder({self.path!r})' +class AppleFrameworkLoader(ExtensionFileLoader): + """A loader for modules that have been packaged as frameworks for + compatibility with Apple's iOS App Store policies. + """ + def create_module(self, spec): + # If the ModuleSpec has been created by the FileFinder, it will have + # been created with an origin pointing to the .fwork file. We need to + # redirect this to the location in the Frameworks folder, using the + # content of the .fwork file. + if spec.origin.endswith(".fwork"): + with _io.FileIO(spec.origin, 'r') as file: + framework_binary = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + spec.origin = _path_join(bundle_path, framework_binary) + + # If the loader is created based on the spec for a loaded module, the + # path will be pointing at the Framework location. If this occurs, + # get the original .fwork location to use as the module's __file__. + if self.path.endswith(".fwork"): + path = self.path + else: + with _io.FileIO(self.path + ".origin", 'r') as file: + origin = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + path = _path_join(bundle_path, origin) + + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) + + _bootstrap._verbose_message( + "Apple framework extension module {!r} loaded from {!r} (path {!r})", + spec.name, + spec.origin, + path, + ) + + # Ensure that the __file__ points at the .fwork location + try: + module.__file__ = path + except AttributeError: + # Not important enough to report. + # (The error is also ignored in _bootstrap._init_module_attrs or + # import_run_extension in import.c) + pass + + return module + # Import setup ############################################################### def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1722,10 +1536,17 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + extension_loaders = [] + if hasattr(_imp, 'create_dynamic'): + if sys.platform in {"ios", "tvos", "watchos"}: + extension_loaders = [(AppleFrameworkLoader, [ + suffix.replace(".so", ".fwork") + for suffix in _imp.extension_suffixes() + ])] + extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] + return extension_loaders + [source, bytecode] def _set_bootstrap_module(_bootstrap_module): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b56fa94eb9c..1e47495f65f 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,9 +13,6 @@ _frozen_importlib_external = _bootstrap_external from ._abc import Loader import abc -import warnings - -from .resources import abc as _resources_abc __all__ = [ @@ -25,19 +22,6 @@ ] -def __getattr__(name): - """ - For backwards compatibility, continue to make names - from _resources_abc available through this module. #93963 - """ - if name in _resources_abc.__all__: - obj = getattr(_resources_abc, name) - warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - def _register(abstract_cls, *classes): for cls in classes: abstract_cls.register(cls) @@ -80,10 +64,13 @@ def invalidate_caches(self): class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their - back-end storage. + back-end storage to facilitate reading data to perform an import. This ABC represents one of the optional protocols specified by PEP 302. + For directly loading resources, use TraversableResources instead. This class + primarily exists for backwards compatibility with other ABCs in this module. + """ @abc.abstractmethod @@ -180,7 +167,11 @@ def get_code(self, fullname): else: return self.source_to_code(source, path) -_register(ExecutionLoader, machinery.ExtensionFileLoader) +_register( + ExecutionLoader, + machinery.ExtensionFileLoader, + machinery.AppleFrameworkLoader, +) class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): @@ -211,6 +202,10 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo def path_mtime(self, path): """Return the (int) modification time for the path (str).""" + import warnings + warnings.warn('SourceLoader.path_mtime is deprecated in favour of ' + 'SourceLoader.path_stats().', + DeprecationWarning, stacklevel=2) if self.path_stats.__func__ is SourceLoader.path_stats: raise OSError return int(self.path_stats(path)['mtime']) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index d9a19a13f7b..63d726445c3 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -3,18 +3,48 @@ from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter -from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, - OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, - EXTENSION_SUFFIXES) +from ._bootstrap_external import ( + SOURCE_SUFFIXES, BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, + DEBUG_BYTECODE_SUFFIXES as _DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES as _OPTIMIZED_BYTECODE_SUFFIXES +) from ._bootstrap_external import WindowsRegistryFinder from ._bootstrap_external import PathFinder from ._bootstrap_external import FileFinder from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] + + +def __getattr__(name): + import warnings + + if name == 'DEBUG_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.DEBUG_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _DEBUG_BYTECODE_SUFFIXES + elif name == 'OPTIMIZED_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _OPTIMIZED_BYTECODE_SUFFIXES + + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 56ee4038328..8ce62dd864f 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,9 +1,13 @@ +from __future__ import annotations + import os import re import abc -import csv import sys +import json import email +import types +import inspect import pathlib import zipfile import operator @@ -12,11 +16,9 @@ import functools import itertools import posixpath -import contextlib import collections -import inspect -from . import _adapters, _meta +from . import _meta from ._collections import FreezableDefaultDict, Pair from ._functools import method_cache, pass_none from ._itertools import always_iterable, unique_everseen @@ -26,8 +28,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import List, Mapping, Optional, cast - +from typing import Any, Iterable, List, Mapping, Match, Optional, Set, cast __all__ = [ 'Distribution', @@ -48,20 +49,14 @@ class PackageNotFoundError(ModuleNotFoundError): """The package was not found.""" - def __str__(self): + def __str__(self) -> str: return f"No package metadata was found for {self.name}" @property - def name(self): + def name(self) -> str: # type: ignore[override] (name,) = self.args return name - # TODO: RUSTPYTHON; the entire setter is added to avoid errors - @name.setter - def name(self, value): - import sys - sys.stderr.write("set value to PackageNotFoundError ignored\n") - class Sectioned: """ @@ -124,38 +119,11 @@ def read(text, filter_=None): yield Pair(name, value) @staticmethod - def valid(line): + def valid(line: str): return line and not line.startswith('#') -class DeprecatedTuple: - """ - Provide subscript item access for backward compatibility. - - >>> recwarn = getfixture('recwarn') - >>> ep = EntryPoint(name='name', value='value', group='group') - >>> ep[:] - ('name', 'value', 'group') - >>> ep[0] - 'name' - >>> len(recwarn) - 1 - """ - - # Do not remove prior to 2023-05-01 or Python 3.13 - _warn = functools.partial( - warnings.warn, - "EntryPoint tuple interface is deprecated. Access members by name.", - DeprecationWarning, - stacklevel=2, - ) - - def __getitem__(self, item): - self._warn() - return self._key()[item] - - -class EntryPoint(DeprecatedTuple): +class EntryPoint: """An entry point as defined by Python packaging conventions. See `the packaging docs on entry points @@ -197,34 +165,37 @@ class EntryPoint(DeprecatedTuple): value: str group: str - dist: Optional['Distribution'] = None + dist: Optional[Distribution] = None - def __init__(self, name, value, group): + def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) - def load(self): + def load(self) -> Any: """Load the entry point from its definition. If only a module is indicated by the value, return that module. Otherwise, return the named object. """ - match = self.pattern.match(self.value) + match = cast(Match, self.pattern.match(self.value)) module = import_module(match.group('module')) attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) @property - def module(self): + def module(self) -> str: match = self.pattern.match(self.value) + assert match is not None return match.group('module') @property - def attr(self): + def attr(self) -> str: match = self.pattern.match(self.value) + assert match is not None return match.group('attr') @property - def extras(self): + def extras(self) -> List[str]: match = self.pattern.match(self.value) + assert match is not None return re.findall(r'\w+', match.group('extras') or '') def _for(self, dist): @@ -272,7 +243,7 @@ def __repr__(self): f'group={self.group!r})' ) - def __hash__(self): + def __hash__(self) -> int: return hash(self._key()) @@ -283,7 +254,7 @@ class EntryPoints(tuple): __slots__ = () - def __getitem__(self, name): # -> EntryPoint: + def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] """ Get the EntryPoint in self matching name. """ @@ -292,7 +263,14 @@ def __getitem__(self, name): # -> EntryPoint: except StopIteration: raise KeyError(name) - def select(self, **params): + def __repr__(self): + """ + Repr with classname and tuple constructor to + signal that we deviate from regular tuple behavior. + """ + return '%s(%r)' % (self.__class__.__name__, tuple(self)) + + def select(self, **params) -> EntryPoints: """ Select entry points from self that match the given parameters (typically group and/or name). @@ -300,14 +278,14 @@ def select(self, **params): return EntryPoints(ep for ep in self if ep.matches(**params)) @property - def names(self): + def names(self) -> Set[str]: """ Return the set of all names of all entry points. """ return {ep.name for ep in self} @property - def groups(self): + def groups(self) -> Set[str]: """ Return the set of all groups of all entry points. """ @@ -328,28 +306,31 @@ def _from_text(text): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - def read_text(self, encoding='utf-8'): - with self.locate().open(encoding=encoding) as stream: - return stream.read() + hash: Optional[FileHash] + size: int + dist: Distribution + + def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] + return self.locate().read_text(encoding=encoding) - def read_binary(self): - with self.locate().open('rb') as stream: - return stream.read() + def read_binary(self) -> bytes: + return self.locate().read_bytes() - def locate(self): + def locate(self) -> SimplePath: """Return a path-like object for this path""" return self.dist.locate_file(self) class FileHash: - def __init__(self, spec): + def __init__(self, spec: str) -> None: self.mode, _, self.value = spec.partition('=') - def __repr__(self): + def __repr__(self) -> str: return f'' class DeprecatedNonAbstract: + # Required until Python 3.14 def __new__(cls, *args, **kwargs): all_names = { name for subclass in inspect.getmro(cls) for name in vars(subclass) @@ -369,25 +350,48 @@ def __new__(cls, *args, **kwargs): class Distribution(DeprecatedNonAbstract): - """A Python distribution package.""" + """ + An abstract Python distribution package. + + Custom providers may derive from this class and define + the abstract methods to provide a concrete implementation + for their environment. Some providers may opt to override + the default implementation of some properties to bypass + the file-reading mechanism. + """ @abc.abstractmethod def read_text(self, filename) -> Optional[str]: """Attempt to load metadata file given by the name. + Python distribution metadata is organized by blobs of text + typically represented as "files" in the metadata directory + (e.g. package-1.0.dist-info). These files include things + like: + + - METADATA: The distribution metadata including fields + like Name and Version and Description. + - entry_points.txt: A series of entry points as defined in + `the entry points spec `_. + - RECORD: A record of files according to + `this recording spec `_. + + A package may provide any set of files, including those + not listed here or none at all. + :param filename: The name of the file in the distribution info. :return: The text if found, otherwise None. """ @abc.abstractmethod - def locate_file(self, path): + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ - Given a path to a file in this distribution, return a path + Given a path to a file in this distribution, return a SimplePath to it. """ @classmethod - def from_name(cls, name: str): + def from_name(cls, name: str) -> Distribution: """Return the Distribution for the given package name. :param name: The name of the distribution package to search for. @@ -400,21 +404,23 @@ def from_name(cls, name: str): if not name: raise ValueError("A distribution name is required.") try: - return next(cls.discover(name=name)) + return next(iter(cls.discover(name=name))) except StopIteration: raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs): + def discover( + cls, *, context: Optional[DistributionFinder.Context] = None, **kwargs + ) -> Iterable[Distribution]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing a context. :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. + :return: Iterable of Distribution objects for packages matching + the context. """ - context = kwargs.pop('context', None) if context and kwargs: raise ValueError("cannot accept context and kwargs") context = context or DistributionFinder.Context(**kwargs) @@ -423,8 +429,8 @@ def discover(cls, **kwargs): ) @staticmethod - def at(path): - """Return a Distribution for the indicated metadata path + def at(path: str | os.PathLike[str]) -> Distribution: + """Return a Distribution for the indicated metadata path. :param path: a string or path-like object :return: a concrete Distribution instance for the path @@ -433,7 +439,7 @@ def at(path): @staticmethod def _discover_resolvers(): - """Search the meta_path for resolvers.""" + """Search the meta_path for resolvers (MetadataPathFinders).""" declared = ( getattr(finder, 'find_distributions', None) for finder in sys.meta_path ) @@ -444,8 +450,15 @@ def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. + metadata per the + `Core metadata specifications `_. + + Custom providers may provide the METADATA file or override this + property. """ + # deferred for performance (python/cpython#109829) + from . import _adapters + opt_text = ( self.read_text('METADATA') or self.read_text('PKG-INFO') @@ -458,7 +471,7 @@ def metadata(self) -> _meta.PackageMetadata: return _adapters.Message(email.message_from_string(text)) @property - def name(self): + def name(self) -> str: """Return the 'Name' metadata for the distribution package.""" return self.metadata['Name'] @@ -468,16 +481,22 @@ def _normalized_name(self): return Prepared.normalize(self.name) @property - def version(self): + def version(self) -> str: """Return the 'Version' metadata for the distribution package.""" return self.metadata['Version'] @property - def entry_points(self): + def entry_points(self) -> EntryPoints: + """ + Return EntryPoints for this distribution. + + Custom providers may provide the ``entry_points.txt`` file + or override this property. + """ return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property - def files(self): + def files(self) -> Optional[List[PackagePath]]: """Files in this distribution. :return: List of PackagePath for this distribution or None @@ -486,6 +505,10 @@ def files(self): (i.e. RECORD for dist-info, or installed-files.txt or SOURCES.txt for egg-info) is missing. Result may be empty if the metadata exists but is empty. + + Custom providers are recommended to provide a "RECORD" file (in + ``read_text``) or override this property to allow for callers to be + able to resolve filenames provided by the package. """ def make_file(name, hash=None, size_str=None): @@ -497,6 +520,10 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): + # Delay csv import, since Distribution.files is not as widely used + # as other parts of importlib.metadata + import csv + return starmap(make_file, csv.reader(lines)) @pass_none @@ -513,7 +540,7 @@ def skip_missing_files(package_paths): def _read_files_distinfo(self): """ - Read the lines of RECORD + Read the lines of RECORD. """ text = self.read_text('RECORD') return text and text.splitlines() @@ -540,7 +567,7 @@ def _read_files_egginfo_installed(self): paths = ( (subdir / name) .resolve() - .relative_to(self.locate_file('').resolve()) + .relative_to(self.locate_file('').resolve(), walk_up=True) .as_posix() for name in text.splitlines() ) @@ -562,7 +589,7 @@ def _read_files_egginfo_sources(self): return text and map('"{}"'.format, text.splitlines()) @property - def requires(self): + def requires(self) -> Optional[List[str]]: """Generated requirements specified for this Distribution""" reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() return reqs and list(reqs) @@ -613,10 +640,23 @@ def url_req_space(req): space = url_req_space(section.value) yield section.value + space + quoted_marker(section.name) + @property + def origin(self): + return self._load_json('direct_url.json') + + def _load_json(self, filename): + return pass_none(json.loads)( + self.read_text(filename), + object_hook=lambda data: types.SimpleNamespace(**data), + ) + class DistributionFinder(MetaPathFinder): """ A MetaPathFinder capable of discovering installed distributions. + + Custom providers should implement this interface in order to + supply metadata. """ class Context: @@ -629,6 +669,17 @@ class Context: Each DistributionFinder may expect any parameters and should attempt to honor the canonical parameters defined below when appropriate. + + This mechanism gives a custom provider a means to + solicit additional details from the caller beyond + "name" and "path" when searching distributions. + For example, imagine a provider that exposes suites + of packages in either a "public" or "private" ``realm``. + A caller may wish to query only for distributions in + a particular realm and could call + ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. """ name = None @@ -641,7 +692,7 @@ def __init__(self, **kwargs): vars(self).update(kwargs) @property - def path(self): + def path(self) -> List[str]: """ The sequence of directory path that a distribution finder should search. @@ -652,7 +703,7 @@ def path(self): return vars(self).get('path', sys.path) @abc.abstractmethod - def find_distributions(self, context=Context()): + def find_distributions(self, context=Context()) -> Iterable[Distribution]: """ Find distributions. @@ -664,11 +715,18 @@ def find_distributions(self, context=Context()): class FastPath: """ - Micro-optimized class for searching a path for - children. + Micro-optimized class for searching a root for children. + + Root is a path on the file system that may contain metadata + directories either as natural directories or within a zip file. >>> FastPath('').children() ['...'] + + FastPath objects are cached and recycled for any given root. + + >>> FastPath('foobar') is FastPath('foobar') + True """ @functools.lru_cache() # type: ignore @@ -710,7 +768,19 @@ def lookup(self, mtime): class Lookup: + """ + A micro-optimized class for searching a (fast) path for metadata. + """ + def __init__(self, path: FastPath): + """ + Calculate all of the children representing metadata. + + From the children in the path, calculate early all of the + children that appear to represent metadata (infos) or legacy + metadata (eggs). + """ + base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") self.infos = FreezableDefaultDict(list) @@ -731,7 +801,10 @@ def __init__(self, path: FastPath): self.infos.freeze() self.eggs.freeze() - def search(self, prepared): + def search(self, prepared: Prepared): + """ + Yield all infos and eggs matching the Prepared query. + """ infos = ( self.infos[prepared.normalized] if prepared @@ -747,13 +820,28 @@ def search(self, prepared): class Prepared: """ - A prepared search for metadata on a possibly-named package. + A prepared search query for metadata on a possibly-named package. + + Pre-calculates the normalization to prevent repeated operations. + + >>> none = Prepared(None) + >>> none.normalized + >>> none.legacy_normalized + >>> bool(none) + False + >>> sample = Prepared('Sample__Pkg-name.foo') + >>> sample.normalized + 'sample_pkg_name_foo' + >>> sample.legacy_normalized + 'sample__pkg_name.foo' + >>> bool(sample) + True """ normalized = None legacy_normalized = None - def __init__(self, name): + def __init__(self, name: Optional[str]): self.name = name if name is None: return @@ -781,7 +869,9 @@ def __bool__(self): class MetadataPathFinder(DistributionFinder): @classmethod - def find_distributions(cls, context=DistributionFinder.Context()): + def find_distributions( + cls, context=DistributionFinder.Context() + ) -> Iterable[PathDistribution]: """ Find distributions. @@ -801,19 +891,20 @@ def _search_paths(cls, name, paths): path.search(prepared) for path in map(FastPath, paths) ) - def invalidate_caches(cls): + @classmethod + def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() class PathDistribution(Distribution): - def __init__(self, path: SimplePath): + def __init__(self, path: SimplePath) -> None: """Construct a distribution. :param path: SimplePath indicating the metadata directory. """ self._path = path - def read_text(self, filename): + def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -823,9 +914,11 @@ def read_text(self, filename): ): return self._path.joinpath(filename).read_text(encoding='utf-8') + return None + read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path): + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: return self._path.parent / path @property @@ -858,7 +951,7 @@ def _name_from_stem(stem): return name -def distribution(distribution_name): +def distribution(distribution_name: str) -> Distribution: """Get the ``Distribution`` instance for the named package. :param distribution_name: The name of the distribution package as a string. @@ -867,7 +960,7 @@ def distribution(distribution_name): return Distribution.from_name(distribution_name) -def distributions(**kwargs): +def distributions(**kwargs) -> Iterable[Distribution]: """Get all ``Distribution`` instances in the current environment. :return: An iterable of ``Distribution`` instances. @@ -875,7 +968,7 @@ def distributions(**kwargs): return Distribution.discover(**kwargs) -def metadata(distribution_name) -> _meta.PackageMetadata: +def metadata(distribution_name: str) -> _meta.PackageMetadata: """Get the metadata for the named package. :param distribution_name: The name of the distribution package to query. @@ -884,7 +977,7 @@ def metadata(distribution_name) -> _meta.PackageMetadata: return Distribution.from_name(distribution_name).metadata -def version(distribution_name): +def version(distribution_name: str) -> str: """Get the version string for the named package. :param distribution_name: The name of the distribution package to query. @@ -918,7 +1011,7 @@ def entry_points(**params) -> EntryPoints: return EntryPoints(eps).select(**params) -def files(distribution_name): +def files(distribution_name: str) -> Optional[List[PackagePath]]: """Return a list of files for the named package. :param distribution_name: The name of the distribution package to query. @@ -927,11 +1020,11 @@ def files(distribution_name): return distribution(distribution_name).files -def requires(distribution_name): +def requires(distribution_name: str) -> Optional[List[str]]: """ Return a list of requirements for the named package. - :return: An iterator of requirements, suitable for + :return: An iterable of requirements, suitable for packaging.requirement.Requirement. """ return distribution(distribution_name).requires @@ -958,13 +1051,42 @@ def _top_level_declared(dist): return (dist.read_text('top_level.txt') or '').split() +def _topmost(name: PackagePath) -> Optional[str]: + """ + Return the top-most parent as long as there is a parent. + """ + top, *rest = name.parts + return top if rest else None + + +def _get_toplevel_name(name: PackagePath) -> str: + """ + Infer a possibly importable module name from a name presumed on + sys.path. + + >>> _get_toplevel_name(PackagePath('foo.py')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo.pyc')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo/__init__.py')) + 'foo' + >>> _get_toplevel_name(PackagePath('foo.pth')) + 'foo.pth' + >>> _get_toplevel_name(PackagePath('foo.dist-info')) + 'foo.dist-info' + """ + return _topmost(name) or ( + # python/typeshed#10328 + inspect.getmodulename(name) # type: ignore + or str(name) + ) + + def _top_level_inferred(dist): - opt_names = { - f.parts[0] if len(f.parts) > 1 else inspect.getmodulename(f) - for f in always_iterable(dist.files) - } + opt_names = set(map(_get_toplevel_name, always_iterable(dist.files))) - @pass_none def importable_name(name): return '.' not in name diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py index 6aed69a3085..59116880895 100644 --- a/Lib/importlib/metadata/_adapters.py +++ b/Lib/importlib/metadata/_adapters.py @@ -53,7 +53,7 @@ def __iter__(self): def __getitem__(self, item): """ Warn users that a ``KeyError`` can be expected when a - mising key is supplied. Ref python/importlib_metadata#371. + missing key is supplied. Ref python/importlib_metadata#371. """ res = super().__getitem__(item) if res is None: diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index c9a7ef906a8..1927d0f624d 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import os from typing import Protocol from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload @@ -6,30 +9,27 @@ class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover + def __len__(self) -> int: ... # pragma: no cover - def __contains__(self, item: str) -> bool: - ... # pragma: no cover + def __contains__(self, item: str) -> bool: ... # pragma: no cover - def __getitem__(self, key: str) -> str: - ... # pragma: no cover + def __getitem__(self, key: str) -> str: ... # pragma: no cover - def __iter__(self) -> Iterator[str]: - ... # pragma: no cover + def __iter__(self) -> Iterator[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: None = None) -> Optional[str]: - ... # pragma: no cover + def get( + self, name: str, failobj: None = None + ) -> Optional[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: _T) -> Union[str, _T]: - ... # pragma: no cover + def get(self, name: str, failobj: _T) -> Union[str, _T]: ... # pragma: no cover # overload per python/importlib_metadata#435 @overload - def get_all(self, name: str, failobj: None = None) -> Optional[List[Any]]: - ... # pragma: no cover + def get_all( + self, name: str, failobj: None = None + ) -> Optional[List[Any]]: ... # pragma: no cover @overload def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: @@ -44,20 +44,24 @@ def json(self) -> Dict[str, Union[str, List[str]]]: """ -class SimplePath(Protocol[_T]): +class SimplePath(Protocol): """ - A minimal subset of pathlib.Path required by PathDistribution. + A minimal subset of pathlib.Path required by Distribution. """ - def joinpath(self) -> _T: - ... # pragma: no cover + def joinpath( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover - def __truediv__(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def __truediv__( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover @property - def parent(self) -> _T: - ... # pragma: no cover + def parent(self) -> SimplePath: ... # pragma: no cover + + def read_text(self, encoding=None) -> str: ... # pragma: no cover + + def read_bytes(self) -> bytes: ... # pragma: no cover - def read_text(self) -> str: - ... # pragma: no cover + def exists(self) -> bool: ... # pragma: no cover diff --git a/Lib/importlib/metadata/diagnose.py b/Lib/importlib/metadata/diagnose.py new file mode 100644 index 00000000000..e405471ac4d --- /dev/null +++ b/Lib/importlib/metadata/diagnose.py @@ -0,0 +1,21 @@ +import sys + +from . import Distribution + + +def inspect(path): + print("Inspecting", path) + dists = list(Distribution.discover(path=[path])) + if not dists: + return + print("Found", len(dists), "packages:", end=' ') + print(', '.join(dist.name for dist in dists)) + + +def run(): + for path in sys.path: + inspect(path) + + +if __name__ == '__main__': + run() diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index 34e3a9950cc..723c9f9eb33 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -1,20 +1,27 @@ -"""Read resources contained within a package.""" +""" +Read resources contained within a package. + +This codebase is shared between importlib.resources in the stdlib +and importlib_resources in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" from ._common import ( as_file, files, Package, + Anchor, ) -from ._legacy import ( +from ._functional import ( contents, + is_resource, open_binary, - read_binary, open_text, - read_text, - is_resource, path, - Resource, + read_binary, + read_text, ) from .abc import ResourceReader @@ -22,11 +29,11 @@ __all__ = [ 'Package', - 'Resource', + 'Anchor', 'ResourceReader', 'as_file', - 'contents', 'files', + 'contents', 'is_resource', 'open_binary', 'open_text', diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index b402e05116e..4e9014c45a0 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -12,8 +12,6 @@ from typing import Union, Optional, cast from .abc import ResourceReader, Traversable -from ._adapters import wrap_spec - Package = Union[types.ModuleType, str] Anchor = Package @@ -27,6 +25,8 @@ def package_to_anchor(func): >>> files('a', 'b') Traceback (most recent call last): TypeError: files() takes from 0 to 1 positional arguments but 2 were given + + Remove this compatibility in Python 3.14. """ undefined = object() @@ -66,10 +66,10 @@ def get_resource_reader(package: types.ModuleType) -> Optional[ResourceReader]: # zipimport.zipimporter does not support weak references, resulting in a # TypeError. That seems terrible. spec = package.__spec__ - reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore + reader = getattr(spec.loader, 'get_resource_reader', None) # type: ignore[union-attr] if reader is None: return None - return reader(spec.name) # type: ignore + return reader(spec.name) # type: ignore[union-attr] @functools.singledispatch @@ -77,12 +77,12 @@ def resolve(cand: Optional[Anchor]) -> types.ModuleType: return cast(types.ModuleType, cand) -@resolve.register(str) # TODO: RUSTPYTHON; manual type annotation +@resolve.register def _(cand: str) -> types.ModuleType: return importlib.import_module(cand) -@resolve.register(type(None)) # TODO: RUSTPYTHON; manual type annotation +@resolve.register def _(cand: None) -> types.ModuleType: return resolve(_infer_caller().f_globals['__name__']) @@ -93,12 +93,13 @@ def _infer_caller(): """ def is_this_file(frame_info): - return frame_info.filename == __file__ + return frame_info.filename == stack[0].filename def is_wrapper(frame_info): return frame_info.function == 'wrapper' - not_this_file = itertools.filterfalse(is_this_file, inspect.stack()) + stack = inspect.stack() + not_this_file = itertools.filterfalse(is_this_file, stack) # also exclude 'wrapper' due to singledispatch in the call stack callers = itertools.filterfalse(is_wrapper, not_this_file) return next(callers).frame @@ -109,6 +110,9 @@ def from_package(package: types.ModuleType): Return a Traversable object for the given package. """ + # deferred for performance (python/cpython#109829) + from ._adapters import wrap_spec + spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() @@ -179,7 +183,7 @@ def _(path): @contextlib.contextmanager def _temp_path(dir: tempfile.TemporaryDirectory): """ - Wrap tempfile.TemporyDirectory to return a pathlib object. + Wrap tempfile.TemporaryDirectory to return a pathlib object. """ with dir as result: yield pathlib.Path(result) diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py new file mode 100644 index 00000000000..f59416f2dd6 --- /dev/null +++ b/Lib/importlib/resources/_functional.py @@ -0,0 +1,81 @@ +"""Simplified function-based API for importlib.resources""" + +import warnings + +from ._common import files, as_file + + +_MISSING = object() + + +def open_binary(anchor, *path_names): + """Open for binary reading the *resource* within *package*.""" + return _get_resource(anchor, path_names).open('rb') + + +def open_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Open for text reading the *resource* within *package*.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.open('r', encoding=encoding, errors=errors) + + +def read_binary(anchor, *path_names): + """Read and return contents of *resource* within *package* as bytes.""" + return _get_resource(anchor, path_names).read_bytes() + + +def read_text(anchor, *path_names, encoding=_MISSING, errors='strict'): + """Read and return contents of *resource* within *package* as str.""" + encoding = _get_encoding_arg(path_names, encoding) + resource = _get_resource(anchor, path_names) + return resource.read_text(encoding=encoding, errors=errors) + + +def path(anchor, *path_names): + """Return the path to the *resource* as an actual file system path.""" + return as_file(_get_resource(anchor, path_names)) + + +def is_resource(anchor, *path_names): + """Return ``True`` if there is a resource named *name* in the package, + + Otherwise returns ``False``. + """ + return _get_resource(anchor, path_names).is_file() + + +def contents(anchor, *path_names): + """Return an iterable over the named resources within the package. + + The iterable returns :class:`str` resources (e.g. files). + The iterable does not recurse into subdirectories. + """ + warnings.warn( + "importlib.resources.contents is deprecated. " + "Use files(anchor).iterdir() instead.", + DeprecationWarning, + stacklevel=1, + ) + return (resource.name for resource in _get_resource(anchor, path_names).iterdir()) + + +def _get_encoding_arg(path_names, encoding): + # For compatibility with versions where *encoding* was a positional + # argument, it needs to be given explicitly when there are multiple + # *path_names*. + # This limitation can be removed in Python 3.15. + if encoding is _MISSING: + if len(path_names) > 1: + raise TypeError( + "'encoding' argument required with multiple path names", + ) + else: + return 'utf-8' + return encoding + + +def _get_resource(anchor, path_names): + if anchor is None: + raise TypeError("anchor must be module or string, got None") + return files(anchor).joinpath(*path_names) diff --git a/Lib/importlib/resources/_legacy.py b/Lib/importlib/resources/_legacy.py deleted file mode 100644 index b1ea8105dad..00000000000 --- a/Lib/importlib/resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index c3cdf769cbe..70fc7e2b9c0 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -1,8 +1,14 @@ +from __future__ import annotations + import collections +import contextlib import itertools import pathlib import operator +import re +import warnings import zipfile +from collections.abc import Iterator from . import abc @@ -31,8 +37,10 @@ def files(self): class ZipReader(abc.TraversableResources): def __init__(self, loader, module): - _, _, name = module.rpartition('.') - self.prefix = loader.prefix.replace('\\', '/') + name + '/' + self.prefix = loader.prefix.replace('\\', '/') + if loader.is_package(module): + _, _, name = module.rpartition('.') + self.prefix += name + '/' self.archive = loader.archive def open_resource(self, resource): @@ -62,7 +70,7 @@ class MultiplexedPath(abc.Traversable): """ def __init__(self, *paths): - self._paths = list(map(pathlib.Path, remove_duplicates(paths))) + self._paths = list(map(_ensure_traversable, remove_duplicates(paths))) if not self._paths: message = 'MultiplexedPath must contain at least one path' raise FileNotFoundError(message) @@ -130,7 +138,40 @@ class NamespaceReader(abc.TraversableResources): def __init__(self, namespace_path): if 'NamespacePath' not in str(namespace_path): raise ValueError('Invalid path') - self.path = MultiplexedPath(*list(namespace_path)) + self.path = MultiplexedPath(*filter(bool, map(self._resolve, namespace_path))) + + @classmethod + def _resolve(cls, path_str) -> abc.Traversable | None: + r""" + Given an item from a namespace path, resolve it to a Traversable. + + path_str might be a directory on the filesystem or a path to a + zipfile plus the path within the zipfile, e.g. ``/foo/bar`` or + ``/foo/baz.zip/inner_dir`` or ``foo\baz.zip\inner_dir\sub``. + + path_str might also be a sentinel used by editable packages to + trigger other behaviors (see python/importlib_resources#311). + In that case, return None. + """ + dirs = (cand for cand in cls._candidate_paths(path_str) if cand.is_dir()) + return next(dirs, None) + + @classmethod + def _candidate_paths(cls, path_str: str) -> Iterator[abc.Traversable]: + yield pathlib.Path(path_str) + yield from cls._resolve_zip_path(path_str) + + @staticmethod + def _resolve_zip_path(path_str: str): + for match in reversed(list(re.finditer(r'[\\/]', path_str))): + with contextlib.suppress( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + ): + inner = path_str[match.end() :].replace('\\', '/') + '/' + yield zipfile.Path(path_str[: match.start()], inner.lstrip('/')) def resource_path(self, resource): """ @@ -142,3 +183,21 @@ def resource_path(self, resource): def files(self): return self.path + + +def _ensure_traversable(path): + """ + Convert deprecated string arguments to traversables (pathlib.Path). + + Remove with Python 3.15. + """ + if not isinstance(path, str): + return path + + warnings.warn( + "String arguments are deprecated. Pass a Traversable instead.", + DeprecationWarning, + stacklevel=3, + ) + + return pathlib.Path(path) diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 7770c922c84..2e75299b13a 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -77,7 +77,7 @@ class ResourceHandle(Traversable): def __init__(self, parent: ResourceContainer, name: str): self.parent = parent - self.name = name # type: ignore + self.name = name # type: ignore[misc] def is_file(self): return True @@ -88,7 +88,7 @@ def is_dir(self): def open(self, mode='r', *args, **kwargs): stream = self.parent.reader.open_binary(self.name) if 'b' not in mode: - stream = io.TextIOWrapper(*args, **kwargs) + stream = io.TextIOWrapper(stream, *args, **kwargs) return stream def joinpath(self, name): diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index f4d6e823315..2b564e9b52e 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,7 +5,6 @@ from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER -from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache @@ -18,7 +17,7 @@ def source_hash(source_bytes): "Return the hash of *source_bytes* as used in hash-based pyc files." - return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + return _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) def resolve_name(name, package): @@ -135,7 +134,7 @@ class _incompatible_extension_module_restrictions: may not be imported in a subinterpreter. That implies modules that do not implement multi-phase init or that explicitly of out. - Likewise for modules import in a subinterpeter with its own GIL + Likewise for modules import in a subinterpreter with its own GIL when the extension does not support a per-interpreter GIL. This implies the module does not have a Py_mod_multiple_interpreters slot set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. @@ -145,7 +144,7 @@ class _incompatible_extension_module_restrictions: You can get the same effect as this function by implementing the basic interface of multi-phase init (PEP 489) and lying about - support for mulitple interpreters (or per-interpreter GIL). + support for multiple interpreters (or per-interpreter GIL). """ def __init__(self, *, disable_check): @@ -171,36 +170,57 @@ class _LazyModule(types.ModuleType): def __getattribute__(self, attr): """Trigger the load of the module and return the attribute.""" - # All module metadata must be garnered from __spec__ in order to avoid - # using mutated values. - # Stop triggering this method. - self.__class__ = types.ModuleType - # Get the original name to make sure no object substitution occurred - # in sys.modules. - original_name = self.__spec__.name - # Figure out exactly what attributes were mutated between the creation - # of the module and now. - attrs_then = self.__spec__.loader_state['__dict__'] - attrs_now = self.__dict__ - attrs_updated = {} - for key, value in attrs_now.items(): - # Code that set the attribute may have kept a reference to the - # assigned object, making identity more important than equality. - if key not in attrs_then: - attrs_updated[key] = value - elif id(attrs_now[key]) != id(attrs_then[key]): - attrs_updated[key] = value - self.__spec__.loader.exec_module(self) - # If exec_module() was used directly there is no guarantee the module - # object was put into sys.modules. - if original_name in sys.modules: - if id(self) != id(sys.modules[original_name]): - raise ValueError(f"module object for {original_name!r} " - "substituted in sys.modules during a lazy " - "load") - # Update after loading since that's what would happen in an eager - # loading situation. - self.__dict__.update(attrs_updated) + __spec__ = object.__getattribute__(self, '__spec__') + loader_state = __spec__.loader_state + with loader_state['lock']: + # Only the first thread to get the lock should trigger the load + # and reset the module's class. The rest can now getattr(). + if object.__getattribute__(self, '__class__') is _LazyModule: + __class__ = loader_state['__class__'] + + # Reentrant calls from the same thread must be allowed to proceed without + # triggering the load again. + # exec_module() and self-referential imports are the primary ways this can + # happen, but in any case we must return something to avoid deadlock. + if loader_state['is_loading']: + return __class__.__getattribute__(self, attr) + loader_state['is_loading'] = True + + __dict__ = __class__.__getattribute__(self, '__dict__') + + # All module metadata must be gathered from __spec__ in order to avoid + # using mutated values. + # Get the original name to make sure no object substitution occurred + # in sys.modules. + original_name = __spec__.name + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = loader_state['__dict__'] + attrs_now = __dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + # Code that set an attribute may have kept a reference to the + # assigned object, making identity more important than equality. + if key not in attrs_then: + attrs_updated[key] = value + elif id(attrs_now[key]) != id(attrs_then[key]): + attrs_updated[key] = value + __spec__.loader.exec_module(self) + # If exec_module() was used directly there is no guarantee the module + # object was put into sys.modules. + if original_name in sys.modules: + if id(self) != id(sys.modules[original_name]): + raise ValueError(f"module object for {original_name!r} " + "substituted in sys.modules during a lazy " + "load") + # Update after loading since that's what would happen in an eager + # loading situation. + __dict__.update(attrs_updated) + # Finally, stop triggering this method, if the module did not + # already update its own __class__. + if isinstance(self, _LazyModule): + object.__setattr__(self, '__class__', __class__) + return getattr(self, attr) def __delattr__(self, attr): @@ -235,6 +255,9 @@ def create_module(self, spec): def exec_module(self, module): """Make the module load lazily.""" + # Threading is only needed for lazy loading, and importlib.util can + # be pulled in at interpreter startup, so defer until needed. + import threading module.__spec__.loader = self.loader module.__loader__ = self.loader # Don't need to worry about deep-copying as trying to set an attribute @@ -244,5 +267,13 @@ def exec_module(self, module): loader_state = {} loader_state['__dict__'] = module.__dict__.copy() loader_state['__class__'] = module.__class__ + loader_state['lock'] = threading.RLock() + loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/inspect.py b/Lib/inspect.py index a84f3346b35..3cee85f39a6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -6,9 +6,9 @@ Here are some of the useful functions provided by this module: - ismodule(), isclass(), ismethod(), isfunction(), isgeneratorfunction(), - isgenerator(), istraceback(), isframe(), iscode(), isbuiltin(), - isroutine() - check object types + ismodule(), isclass(), ismethod(), ispackage(), isfunction(), + isgeneratorfunction(), isgenerator(), istraceback(), isframe(), + iscode(), isbuiltin(), isroutine() - check object types getmembers() - get members of an object that satisfy a given condition getfile(), getsourcefile(), getsource() - find an object's source code @@ -31,7 +31,121 @@ __author__ = ('Ka-Ping Yee ', 'Yury Selivanov ') +__all__ = [ + "AGEN_CLOSED", + "AGEN_CREATED", + "AGEN_RUNNING", + "AGEN_SUSPENDED", + "ArgInfo", + "Arguments", + "Attribute", + "BlockFinder", + "BoundArguments", + "BufferFlags", + "CORO_CLOSED", + "CORO_CREATED", + "CORO_RUNNING", + "CORO_SUSPENDED", + "CO_ASYNC_GENERATOR", + "CO_COROUTINE", + "CO_GENERATOR", + "CO_ITERABLE_COROUTINE", + "CO_NESTED", + "CO_NEWLOCALS", + "CO_NOFREE", + "CO_OPTIMIZED", + "CO_VARARGS", + "CO_VARKEYWORDS", + "CO_HAS_DOCSTRING", + "CO_METHOD", + "ClassFoundException", + "ClosureVars", + "EndOfBlock", + "FrameInfo", + "FullArgSpec", + "GEN_CLOSED", + "GEN_CREATED", + "GEN_RUNNING", + "GEN_SUSPENDED", + "Parameter", + "Signature", + "TPFLAGS_IS_ABSTRACT", + "Traceback", + "classify_class_attrs", + "cleandoc", + "currentframe", + "findsource", + "formatannotation", + "formatannotationrelativeto", + "formatargvalues", + "get_annotations", + "getabsfile", + "getargs", + "getargvalues", + "getasyncgenlocals", + "getasyncgenstate", + "getattr_static", + "getblock", + "getcallargs", + "getclasstree", + "getclosurevars", + "getcomments", + "getcoroutinelocals", + "getcoroutinestate", + "getdoc", + "getfile", + "getframeinfo", + "getfullargspec", + "getgeneratorlocals", + "getgeneratorstate", + "getinnerframes", + "getlineno", + "getmembers", + "getmembers_static", + "getmodule", + "getmodulename", + "getmro", + "getouterframes", + "getsource", + "getsourcefile", + "getsourcelines", + "indentsize", + "isabstract", + "isasyncgen", + "isasyncgenfunction", + "isawaitable", + "isbuiltin", + "isclass", + "iscode", + "iscoroutine", + "iscoroutinefunction", + "isdatadescriptor", + "isframe", + "isfunction", + "isgenerator", + "isgeneratorfunction", + "isgetsetdescriptor", + "ismemberdescriptor", + "ismethod", + "ismethoddescriptor", + "ismethodwrapper", + "ismodule", + "ispackage", + "isroutine", + "istraceback", + "markcoroutinefunction", + "signature", + "stack", + "trace", + "unwrap", + "walktree", +] + + import abc +from annotationlib import Format, ForwardRef +from annotationlib import get_annotations # re-exported +import ast import dis import collections.abc import enum @@ -44,58 +158,51 @@ import tokenize import token import types -import warnings import functools import builtins +from keyword import iskeyword from operator import attrgetter from collections import namedtuple, OrderedDict +from weakref import ref as make_weakref # Create constants for the compiler flags in Include/code.h # We try to get them from dis to avoid duplication mod_dict = globals() for k, v in dis.COMPILER_FLAG_NAMES.items(): mod_dict["CO_" + v] = k +del k, v, mod_dict # See Include/object.h TPFLAGS_IS_ABSTRACT = 1 << 20 + # ----------------------------------------------------------- type-checking def ismodule(object): - """Return true if the object is a module. - - Module objects provide these attributes: - __cached__ pathname to byte compiled file - __doc__ documentation string - __file__ filename (missing for built-in modules)""" + """Return true if the object is a module.""" return isinstance(object, types.ModuleType) def isclass(object): - """Return true if the object is a class. - - Class objects provide these attributes: - __doc__ documentation string - __module__ name of module in which this class was defined""" + """Return true if the object is a class.""" return isinstance(object, type) def ismethod(object): - """Return true if the object is an instance method. - - Instance method objects provide these attributes: - __doc__ documentation string - __name__ name with which this method was defined - __func__ function object containing implementation of method - __self__ instance to which this method is bound""" + """Return true if the object is an instance method.""" return isinstance(object, types.MethodType) +def ispackage(object): + """Return true if the object is a package.""" + return ismodule(object) and hasattr(object, "__path__") + def ismethoddescriptor(object): """Return true if the object is a method descriptor. But not if ismethod() or isclass() or isfunction() are true. This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute but not a __set__ - attribute, but beyond that the set of attributes varies. __name__ is - usually sensible, and __doc__ often is. + An object passing this test has a __get__ attribute, but not a + __set__ attribute or a __delete__ attribute. Beyond that, the set + of attributes varies; __name__ is usually sensible, and __doc__ + often is. Methods implemented via descriptors that also pass one of the other tests return false from the ismethoddescriptor() test, simply because @@ -105,7 +212,9 @@ def ismethoddescriptor(object): # mutual exclusion return False tp = type(object) - return hasattr(tp, "__get__") and not hasattr(tp, "__set__") + return (hasattr(tp, "__get__") + and not hasattr(tp, "__set__") + and not hasattr(tp, "__delete__")) def isdatadescriptor(object): """Return true if the object is a data descriptor. @@ -161,21 +270,28 @@ def isfunction(object): Function objects provide these attributes: __doc__ documentation string __name__ name with which this function was defined + __qualname__ qualified name of this function + __module__ name of the module the function was defined in or None __code__ code object containing compiled function bytecode __defaults__ tuple of any default values for arguments __globals__ global namespace in which this function was defined __annotations__ dict of parameter annotations - __kwdefaults__ dict of keyword only parameters with defaults""" + __kwdefaults__ dict of keyword only parameters with defaults + __dict__ namespace which is supporting arbitrary function attributes + __closure__ a tuple of cells or None + __type_params__ tuple of type parameters""" return isinstance(object, types.FunctionType) def _has_code_flag(f, flag): """Return true if ``f`` is a function (or a method or functools.partial - wrapper wrapping a function) whose code object has the given ``flag`` + wrapper wrapping a function or a functools.partialmethod wrapping a + function) whose code object has the given ``flag`` set in its flags.""" + f = functools._unwrap_partialmethod(f) while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) - if not isfunction(f): + if not (isfunction(f) or _signature_is_functionlike(f)): return False return bool(f.__code__.co_flags & flag) @@ -186,12 +302,31 @@ def isgeneratorfunction(obj): See help(isfunction) for a list of attributes.""" return _has_code_flag(obj, CO_GENERATOR) +# A marker for markcoroutinefunction and iscoroutinefunction. +_is_coroutine_mark = object() + +def _has_coroutine_mark(f): + while ismethod(f): + f = f.__func__ + f = functools._unwrap_partial(f) + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_mark + +def markcoroutinefunction(func): + """ + Decorator to ensure callable is recognised as a coroutine function. + """ + if hasattr(func, '__func__'): + func = func.__func__ + func._is_coroutine_marker = _is_coroutine_mark + return func + def iscoroutinefunction(obj): """Return true if the object is a coroutine function. - Coroutine functions are defined with "async def" syntax. + Coroutine functions are normally defined with "async def" syntax, but may + be marked via markcoroutinefunction. """ - return _has_code_flag(obj, CO_COROUTINE) + return _has_code_flag(obj, CO_COROUTINE) or _has_coroutine_mark(obj) def isasyncgenfunction(obj): """Return true if the object is an asynchronous generator function. @@ -209,17 +344,19 @@ def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: - __iter__ defined to support iteration over container - close raises a new GeneratorExit exception inside the - generator to terminate the iteration gi_code code object gi_frame frame object or possibly None once the generator has been exhausted gi_running set to 1 when generator is executing, 0 otherwise - next return the next item from the container - send resumes the generator and "sends" a value that becomes + gi_suspended set to 1 when the generator is suspended at a yield point, 0 otherwise + gi_yieldfrom object being iterated by yield from or None + + __iter__() defined to support iteration over container + close() raises a new GeneratorExit exception inside the + generator to terminate the iteration + send() resumes the generator and "sends" a value that becomes the result of the current yield-expression - throw used to raise an exception inside the generator""" + throw() used to raise an exception inside the generator""" return isinstance(object, types.GeneratorType) def iscoroutine(object): @@ -254,7 +391,11 @@ def isframe(object): f_lasti index of last attempted instruction in bytecode f_lineno current line number in Python source code f_locals local namespace seen by this frame - f_trace tracing function for this frame, or None""" + f_trace tracing function for this frame, or None + f_trace_lines is a tracing event triggered for each source line? + f_trace_opcodes are per-opcode events being requested? + + clear() used to clear all references to local variables""" return isinstance(object, types.FrameType) def iscode(object): @@ -271,15 +412,21 @@ def iscode(object): co_flags bitmap: 1=optimized | 2=newlocals | 4=*arg | 8=**arg | 16=nested | 32=generator | 64=nofree | 128=coroutine | 256=iterable_coroutine | 512=async_generator + | 0x4000000=has_docstring co_freevars tuple of names of free variables co_posonlyargcount number of positional only arguments co_kwonlyargcount number of keyword only arguments (not including ** arg) co_lnotab encoded mapping of line numbers to bytecode indices co_name name with which this code object was defined - co_names tuple of names of local variables + co_names tuple of names other than arguments and function locals co_nlocals number of local variables co_stacksize virtual machine stack space required - co_varnames tuple of names of arguments and local variables""" + co_varnames tuple of names of arguments and local variables + co_qualname fully qualified function name + + co_lines() returns an iterator that yields successive bytecode ranges + co_positions() returns an iterator of source code positions for each bytecode instruction + replace() returns a copy of the code object with a new values""" return isinstance(object, types.CodeType) def isbuiltin(object): @@ -291,28 +438,31 @@ def isbuiltin(object): __self__ instance to which a method is bound, or None""" return isinstance(object, types.BuiltinFunctionType) +def ismethodwrapper(object): + """Return true if the object is a method wrapper.""" + return isinstance(object, types.MethodWrapperType) + def isroutine(object): """Return true if the object is any kind of function or method.""" return (isbuiltin(object) or isfunction(object) or ismethod(object) - or ismethoddescriptor(object)) + or ismethoddescriptor(object) + or ismethodwrapper(object) + or isinstance(object, functools._singledispatchmethod_get)) def isabstract(object): """Return true if the object is an abstract base class (ABC).""" if not isinstance(object, type): return False - # TODO: RUSTPYTHON - # TPFLAGS_IS_ABSTRACT is not being set for abstract classes, so this implementation differs from CPython. - # if object.__flags__ & TPFLAGS_IS_ABSTRACT: - # return True + if object.__flags__ & TPFLAGS_IS_ABSTRACT: + return True if not issubclass(type(object), abc.ABCMeta): return False if hasattr(object, '__abstractmethods__'): # It looks like ABCMeta.__new__ has finished running; # TPFLAGS_IS_ABSTRACT should have been accurate. - # return False - return bool(getattr(object, '__abstractmethods__')) + return False # It looks like ABCMeta.__new__ has not finished running yet; we're # probably in __init_subclass__. We'll look for abstractmethods manually. for name, value in object.__dict__.items(): @@ -325,32 +475,30 @@ def isabstract(object): return True return False -def getmembers(object, predicate=None): - """Return all members of an object as (name, value) pairs sorted by name. - Optionally, only return members that satisfy a given predicate.""" - if isclass(object): - mro = (object,) + getmro(object) - else: - mro = () +def _getmembers(object, predicate, getter): results = [] processed = set() names = dir(object) - # :dd any DynamicClassAttributes to the list of names if object is a class; - # this may result in duplicate entries if, for example, a virtual - # attribute with the same name as a DynamicClassAttribute exists - try: - for base in object.__bases__: - for k, v in base.__dict__.items(): - if isinstance(v, types.DynamicClassAttribute): - names.append(k) - except AttributeError: - pass + if isclass(object): + mro = getmro(object) + # add any DynamicClassAttributes to the list of names if object is a class; + # this may result in duplicate entries if, for example, a virtual + # attribute with the same name as a DynamicClassAttribute exists + try: + for base in object.__bases__: + for k, v in base.__dict__.items(): + if isinstance(v, types.DynamicClassAttribute): + names.append(k) + except AttributeError: + pass + else: + mro = () for key in names: # First try to get the value via getattr. Some descriptors don't # like calling their __get__ (see bug #1785), so fall back to # looking in the __dict__. try: - value = getattr(object, key) + value = getter(object, key) # handle the duplicate key if key in processed: raise AttributeError @@ -369,6 +517,25 @@ def getmembers(object, predicate=None): results.sort(key=lambda pair: pair[0]) return results +def getmembers(object, predicate=None): + """Return all members of an object as (name, value) pairs sorted by name. + Optionally, only return members that satisfy a given predicate.""" + return _getmembers(object, predicate, getattr) + +def getmembers_static(object, predicate=None): + """Return all members of an object as (name, value) pairs sorted by name + without triggering dynamic lookup via the descriptor protocol, + __getattr__ or __getattribute__. Optionally, only return members that + satisfy a given predicate. + + Note: this function may not be able to retrieve all members + that getmembers can fetch (like dynamically created attributes) + and may find members that getmembers can't (like descriptors + that raise AttributeError). It can also return descriptor objects + instead of instance members in some cases. + """ + return _getmembers(object, predicate, getattr_static) + Attribute = namedtuple('Attribute', 'name kind defining_class object') def classify_class_attrs(cls): @@ -409,7 +576,7 @@ def classify_class_attrs(cls): # attribute with the same name as a DynamicClassAttribute exists. for base in mro: for k, v in base.__dict__.items(): - if isinstance(v, types.DynamicClassAttribute): + if isinstance(v, types.DynamicClassAttribute) and v.fget is not None: names.append(k) result = [] processed = set() @@ -432,7 +599,7 @@ def classify_class_attrs(cls): if name == '__dict__': raise Exception("__dict__ is special, don't want the proxy") get_obj = getattr(cls, name) - except Exception as exc: + except Exception: pass else: homecls = getattr(get_obj, "__objclass__", homecls) @@ -509,18 +676,14 @@ def unwrap(func, *, stop=None): :exc:`ValueError` is raised if a cycle is encountered. """ - if stop is None: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') - else: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting # Memoise by id to tolerate non-hashable objects, but store objects to # ensure they aren't destroyed, which would allow their IDs to be reused. memo = {id(f): f} recursion_limit = sys.getrecursionlimit() - while _is_wrapper(func): + while not isinstance(func, type) and hasattr(func, '__wrapped__'): + if stop is not None and stop(func): + break func = func.__wrapped__ id_func = id(func) if (id_func in memo) or (len(memo) >= recursion_limit): @@ -581,9 +744,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): @@ -630,29 +792,28 @@ def cleandoc(doc): Any whitespace that can be uniformly removed from the second line onwards is removed.""" - try: - lines = doc.expandtabs().split('\n') - except UnicodeError: - return None - else: - # Find minimum indentation of any non-blank lines after first line. - margin = sys.maxsize - for line in lines[1:]: - content = len(line.lstrip()) - if content: - indent = len(line) - content - margin = min(margin, indent) - # Remove indentation. - if lines: - lines[0] = lines[0].lstrip() - if margin < sys.maxsize: - for i in range(1, len(lines)): lines[i] = lines[i][margin:] - # Remove any trailing or leading blank lines. - while lines and not lines[-1]: - lines.pop() - while lines and not lines[0]: - lines.pop(0) - return '\n'.join(lines) + lines = doc.expandtabs().split('\n') + + # Find minimum indentation of any non-blank lines after first line. + margin = sys.maxsize + for line in lines[1:]: + content = len(line.lstrip(' ')) + if content: + indent = len(line) - content + margin = min(margin, indent) + # Remove indentation. + if lines: + lines[0] = lines[0].lstrip(' ') + if margin < sys.maxsize: + for i in range(1, len(lines)): + lines[i] = lines[i][margin:] + # Remove any trailing or leading blank lines. + while lines and not lines[-1]: + lines.pop() + while lines and not lines[0]: + lines.pop(0) + return '\n'.join(lines) + def getfile(object): """Work out which source or compiled file an object was defined in.""" @@ -665,6 +826,8 @@ def getfile(object): module = sys.modules.get(object.__module__) if getattr(module, '__file__', None): return module.__file__ + if object.__module__ == '__main__': + raise OSError('source code not available') raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): object = object.__func__ @@ -697,21 +860,27 @@ def getsourcefile(object): Return None if no way can be identified to get the source. """ filename = getfile(object) - all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:] - all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:] + all_bytecode_suffixes = importlib.machinery.BYTECODE_SUFFIXES[:] if any(filename.endswith(s) for s in all_bytecode_suffixes): filename = (os.path.splitext(filename)[0] + importlib.machinery.SOURCE_SUFFIXES[0]) elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None + elif filename.endswith(".fwork"): + # Apple mobile framework markers are another type of non-source file + return None + + # return a filename found in the linecache even if it doesn't exist on disk + if filename in linecache.cache: + return filename if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader - if getattr(getmodule(object, filename), '__loader__', None) is not None: + module = getmodule(object, filename) + if getattr(module, '__loader__', None) is not None: return filename - # or it is in the linecache - if filename in linecache.cache: + elif getattr(getattr(module, "__spec__", None), "loader", None) is not None: return filename def getabsfile(object, _filename=None): @@ -732,19 +901,20 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) # Try the cache again with the absolute file name try: file = getabsfile(object, _filename) - except TypeError: + except (TypeError, FileNotFoundError): return None if file in modulesbyfile: return sys.modules.get(modulesbyfile[file]) # Update the filename to module name cache and check yet again # Copy sys.modules in order to cope with changes while iterating - for modname, module in list(sys.modules.items()): + for modname, module in sys.modules.copy().items(): if ismodule(module) and hasattr(module, '__file__'): f = module.__file__ if f == _filesbymodname.get(modname, None): @@ -772,6 +942,11 @@ def getmodule(object, _filename=None): if builtinobject is object: return builtin + +class ClassFoundException(Exception): + pass + + def findsource(object): """Return the entire source file and starting line number for an object. @@ -789,12 +964,14 @@ def findsource(object): # Allow filenames in form of "" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): + if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'): raise OSError('source code not available') module = getmodule(object, file) if module: lines = linecache.getlines(file, module.__dict__) + if not lines and file.startswith('<') and hasattr(object, "__code__"): + lines = linecache._getlines_from_code(object.__code__) else: lines = linecache.getlines(file) if not lines: @@ -804,27 +981,13 @@ def findsource(object): return lines, 0 if isclass(object): - name = object.__name__ - pat = re.compile(r'^(\s*)class\s*' + name + r'\b') - # make some effort to find the best matching class definition: - # use the one with the least indentation, which is the one - # that's most probably not inside a function definition. - candidates = [] - for i in range(len(lines)): - match = pat.match(lines[i]) - if match: - # if it's at toplevel, it's already the best one - if lines[i][0] == 'c': - return lines, i - # else add whitespace to candidate list - candidates.append((match.group(1), i)) - if candidates: - # this will sort by whitespace, and by line number, - # less whitespace first - candidates.sort() - return lines, candidates[0][1] - else: - raise OSError('could not find class definition') + try: + lnum = vars(object)['__firstlineno__'] - 1 + except (TypeError, KeyError): + raise OSError('source code not available') + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ @@ -838,10 +1001,8 @@ def findsource(object): if not hasattr(object, 'co_firstlineno'): raise OSError('could not find function definition') lnum = object.co_firstlineno - 1 - pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(? 0: - if pat.match(lines[lnum]): break - lnum = lnum - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') return lines, lnum raise OSError('could not find code object') @@ -896,43 +1057,43 @@ class BlockFinder: """Provide a tokeneater() method to detect the end of a code block.""" def __init__(self): self.indent = 0 - self.islambda = False + self.singleline = False self.started = False self.passline = False self.indecorator = False - self.decoratorhasargs = False self.last = 1 + self.body_col0 = None def tokeneater(self, type, token, srowcol, erowcol, line): if not self.started and not self.indecorator: + if type in (tokenize.INDENT, tokenize.COMMENT, tokenize.NL): + pass + elif token == "async": + pass # skip any decorators - if token == "@": + elif token == "@": self.indecorator = True - # look for the first "def", "class" or "lambda" - elif token in ("def", "class", "lambda"): - if token == "lambda": - self.islambda = True + else: + # For "def" and "class" scan to the end of the block. + # For "lambda" and generator expression scan to + # the end of the logical line. + self.singleline = token not in ("def", "class") self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] - if self.islambda: # lambdas always end at the first NEWLINE + if self.singleline: raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator - if self.indecorator and not self.decoratorhasargs: + if self.indecorator: self.indecorator = False elif self.passline: pass elif type == tokenize.INDENT: + if self.body_col0 is None and self.started: + self.body_col0 = erowcol[1] self.indent = self.indent + 1 self.passline = True elif type == tokenize.DEDENT: @@ -942,6 +1103,10 @@ def tokeneater(self, type, token, srowcol, erowcol, line): # not e.g. for "if: else:" or "try: finally:" blocks) if self.indent <= 0: raise EndOfBlock + elif type == tokenize.COMMENT: + if self.body_col0 is not None and srowcol[1] >= self.body_col0: + # Include comments if indented at least as much as the block + self.last = srowcol[0] elif self.indent == 0 and type not in (tokenize.COMMENT, tokenize.NL): # any other token on the same indentation level end the previous # block as well, except the pseudo-tokens COMMENT and NL. @@ -956,6 +1121,14 @@ def getblock(lines): blockfinder.tokeneater(*_token) except (EndOfBlock, IndentationError): pass + except SyntaxError as e: + if "unmatched" not in e.msg: + raise e from None + _, *_token_info = _token + try: + blockfinder.tokeneater(tokenize.NEWLINE, *_token_info) + except (EndOfBlock, IndentationError): + pass return lines[:blockfinder.last] def getsourcelines(object): @@ -1043,7 +1216,6 @@ def getargs(co): nkwargs = co.co_kwonlyargcount args = list(names[:nargs]) kwonlyargs = list(names[nargs:nargs+nkwargs]) - step = 0 nargs += nkwargs varargs = None @@ -1055,37 +1227,6 @@ def getargs(co): varkw = co.co_varnames[nargs] return Arguments(args + kwonlyargs, varargs, varkw) -ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - -def getargspec(func): - """Get the names and default values of a function's parameters. - - A tuple of four things is returned: (args, varargs, keywords, defaults). - 'args' is a list of the argument names, including keyword-only argument names. - 'varargs' and 'keywords' are the names of the * and ** parameters or None. - 'defaults' is an n-tuple of the default values of the last n parameters. - - This function is deprecated, as it does not support annotations or - keyword-only parameters and will raise ValueError if either is present - on the supplied callable. - - For a more structured introspection API, use inspect.signature() instead. - - Alternatively, use getfullargspec() for an API with a similar namedtuple - based interface, but full support for annotations and keyword-only - parameters. - - Deprecated since Python 3.5, use `inspect.getfullargspec()`. - """ - warnings.warn("inspect.getargspec() is deprecated since Python 3.0, " - "use inspect.signature() or inspect.getfullargspec()", - DeprecationWarning, stacklevel=2) - args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - getfullargspec(func) - if kwonlyargs or ann: - raise ValueError("Function has keyword-only parameters or annotations" - ", use inspect.signature() API which can support them") - return ArgSpec(args, varargs, varkw, defaults) FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') @@ -1126,7 +1267,8 @@ def getfullargspec(func): sig = _signature_from_callable(func, follow_wrapper_chains=False, skip_bound_arg=False, - sigcls=Signature) + sigcls=Signature, + eval_str=False) except Exception as ex: # Most of the times 'signature' will raise ValueError. # But, it can also raise AttributeError, and, maybe something @@ -1139,7 +1281,6 @@ def getfullargspec(func): varkw = None posonlyargs = [] kwonlyargs = [] - defaults = () annotations = {} defaults = () kwdefaults = {} @@ -1195,13 +1336,22 @@ def getargvalues(frame): args, varargs, varkw = getargs(frame.f_code) return ArgInfo(args, varargs, varkw, frame.f_locals) -def formatannotation(annotation, base_module=None): +def formatannotation(annotation, base_module=None, *, quote_annotation_strings=True): + if not quote_annotation_strings and isinstance(annotation, str): + return annotation if getattr(annotation, '__module__', None) == 'typing': - return repr(annotation).replace('typing.', '') + def repl(match): + text = match.group() + return text.removeprefix('typing.') + return re.sub(r'[\w\.]+', repl, repr(annotation)) + if isinstance(annotation, types.GenericAlias): + return str(annotation) if isinstance(annotation, type): if annotation.__module__ in ('builtins', base_module): return annotation.__qualname__ return annotation.__module__+'.'+annotation.__qualname__ + if isinstance(annotation, ForwardRef): + return annotation.__forward_arg__ return repr(annotation) def formatannotationrelativeto(object): @@ -1210,63 +1360,6 @@ def _formatannotation(annotation): return formatannotation(annotation, module) return _formatannotation -def formatargspec(args, varargs=None, varkw=None, defaults=None, - kwonlyargs=(), kwonlydefaults={}, annotations={}, - formatarg=str, - formatvarargs=lambda name: '*' + name, - formatvarkw=lambda name: '**' + name, - formatvalue=lambda value: '=' + repr(value), - formatreturns=lambda text: ' -> ' + text, - formatannotation=formatannotation): - """Format an argument spec from the values returned by getfullargspec. - - The first seven arguments are (args, varargs, varkw, defaults, - kwonlyargs, kwonlydefaults, annotations). The other five arguments - are the corresponding optional formatting functions that are called to - turn names and values into strings. The last argument is an optional - function to format the sequence of arguments. - - Deprecated since Python 3.5: use the `signature` function and `Signature` - objects. - """ - - from warnings import warn - - warn("`formatargspec` is deprecated since Python 3.5. Use `signature` and " - "the `Signature` object directly", - DeprecationWarning, - stacklevel=2) - - def formatargandannotation(arg): - result = formatarg(arg) - if arg in annotations: - result += ': ' + formatannotation(annotations[arg]) - return result - specs = [] - if defaults: - firstdefault = len(args) - len(defaults) - for i, arg in enumerate(args): - spec = formatargandannotation(arg) - if defaults and i >= firstdefault: - spec = spec + formatvalue(defaults[i - firstdefault]) - specs.append(spec) - if varargs is not None: - specs.append(formatvarargs(formatargandannotation(varargs))) - else: - if kwonlyargs: - specs.append('*') - if kwonlyargs: - for kwonlyarg in kwonlyargs: - spec = formatargandannotation(kwonlyarg) - if kwonlydefaults and kwonlyarg in kwonlydefaults: - spec += formatvalue(kwonlydefaults[kwonlyarg]) - specs.append(spec) - if varkw is not None: - specs.append(formatvarkw(formatargandannotation(varkw))) - result = '(' + ', '.join(specs) + ')' - if 'return' in annotations: - result += formatreturns(formatannotation(annotations['return'])) - return result def formatargvalues(args, varargs, varkw, locals, formatarg=str, @@ -1425,11 +1518,15 @@ def getclosurevars(func): global_vars = {} builtin_vars = {} unbound_names = set() - for name in code.co_names: - if name in ("None", "True", "False"): - # Because these used to be builtins instead of keywords, they - # may still show up as name references. We ignore them. - continue + global_names = set() + for instruction in dis.get_instructions(code): + opname = instruction.opname + name = instruction.argval + if opname == "LOAD_ATTR": + unbound_names.add(name) + elif opname == "LOAD_GLOBAL": + global_names.add(name) + for name in global_names: try: global_vars[name] = global_ns[name] except KeyError: @@ -1443,7 +1540,30 @@ def getclosurevars(func): # -------------------------------------------------- stack frame extraction -Traceback = namedtuple('Traceback', 'filename lineno function code_context index') +_Traceback = namedtuple('_Traceback', 'filename lineno function code_context index') + +class Traceback(_Traceback): + def __new__(cls, filename, lineno, function, code_context, index, *, positions=None): + instance = super().__new__(cls, filename, lineno, function, code_context, index) + instance.positions = positions + return instance + + def __repr__(self): + return ('Traceback(filename={!r}, lineno={!r}, function={!r}, ' + 'code_context={!r}, index={!r}, positions={!r})'.format( + self.filename, self.lineno, self.function, self.code_context, + self.index, self.positions)) + +def _get_code_position_from_tb(tb): + code, instruction_index = tb.tb_frame.f_code, tb.tb_lasti + return _get_code_position(code, instruction_index) + +def _get_code_position(code, instruction_index): + if instruction_index < 0: + return (None, None, None, None) + positions_gen = code.co_positions() + # The nth entry in code.co_positions() corresponds to instruction (2*n)th since Python 3.10+ + return next(itertools.islice(positions_gen, instruction_index // 2, None)) def getframeinfo(frame, context=1): """Get information about a frame or traceback object. @@ -1454,10 +1574,20 @@ def getframeinfo(frame, context=1): The optional second argument specifies the number of lines of context to return, which are centered around the current line.""" if istraceback(frame): + positions = _get_code_position_from_tb(frame) lineno = frame.tb_lineno frame = frame.tb_frame else: lineno = frame.f_lineno + positions = _get_code_position(frame.f_code, frame.f_lasti) + + if positions[0] is None: + frame, *positions = (frame, lineno, *positions[1:]) + else: + frame, *positions = (frame, *positions) + + lineno = positions[0] + if not isframe(frame): raise TypeError('{!r} is not a frame or traceback object'.format(frame)) @@ -1475,14 +1605,26 @@ def getframeinfo(frame, context=1): else: lines = index = None - return Traceback(filename, lineno, frame.f_code.co_name, lines, index) + return Traceback(filename, lineno, frame.f_code.co_name, lines, + index, positions=dis.Positions(*positions)) def getlineno(frame): """Get the line number from a frame object, allowing for optimization.""" # FrameType.f_lineno is now a descriptor that grovels co_lnotab return frame.f_lineno -FrameInfo = namedtuple('FrameInfo', ('frame',) + Traceback._fields) +_FrameInfo = namedtuple('_FrameInfo', ('frame',) + Traceback._fields) +class FrameInfo(_FrameInfo): + def __new__(cls, frame, filename, lineno, function, code_context, index, *, positions=None): + instance = super().__new__(cls, frame, filename, lineno, function, code_context, index) + instance.positions = positions + return instance + + def __repr__(self): + return ('FrameInfo(frame={!r}, filename={!r}, lineno={!r}, function={!r}, ' + 'code_context={!r}, index={!r}, positions={!r})'.format( + self.frame, self.filename, self.lineno, self.function, + self.code_context, self.index, self.positions)) def getouterframes(frame, context=1): """Get a list of records for a frame and all higher (calling) frames. @@ -1491,8 +1633,9 @@ def getouterframes(frame, context=1): name, a list of lines of context, and index within the context.""" framelist = [] while frame: - frameinfo = (frame,) + getframeinfo(frame, context) - framelist.append(FrameInfo(*frameinfo)) + traceback_info = getframeinfo(frame, context) + frameinfo = (frame,) + traceback_info + framelist.append(FrameInfo(*frameinfo, positions=traceback_info.positions)) frame = frame.f_back return framelist @@ -1503,8 +1646,9 @@ def getinnerframes(tb, context=1): name, a list of lines of context, and index within the context.""" framelist = [] while tb: - frameinfo = (tb.tb_frame,) + getframeinfo(tb, context) - framelist.append(FrameInfo(*frameinfo)) + traceback_info = getframeinfo(tb, context) + frameinfo = (tb.tb_frame,) + traceback_info + framelist.append(FrameInfo(*frameinfo, positions=traceback_info.positions)) tb = tb.tb_next return framelist @@ -1518,15 +1662,17 @@ def stack(context=1): def trace(context=1): """Return a list of records for the stack below the current exception.""" - return getinnerframes(sys.exc_info()[2], context) + exc = sys.exception() + tb = None if exc is None else exc.__traceback__ + return getinnerframes(tb, context) # ------------------------------------------------ static version of getattr _sentinel = object() +_static_getmro = type.__dict__['__mro__'].__get__ +_get_dunder_dict_of_class = type.__dict__["__dict__"].__get__ -def _static_getmro(klass): - return type.__dict__['__mro__'].__get__(klass) def _check_instance(obj, attr): instance_dict = {} @@ -1539,34 +1685,43 @@ def _check_instance(obj, attr): def _check_class(klass, attr): for entry in _static_getmro(klass): - if _shadowed_dict(type(entry)) is _sentinel: - try: - return entry.__dict__[attr] - except KeyError: - pass + if _shadowed_dict(type(entry)) is _sentinel and attr in entry.__dict__: + return entry.__dict__[attr] return _sentinel -def _is_type(obj): - try: - _static_getmro(obj) - except TypeError: - return False - return True -def _shadowed_dict(klass): - dict_attr = type.__dict__["__dict__"] - for entry in _static_getmro(klass): - try: - class_dict = dict_attr.__get__(entry)["__dict__"] - except KeyError: - pass - else: +@functools.lru_cache() +def _shadowed_dict_from_weakref_mro_tuple(*weakref_mro): + for weakref_entry in weakref_mro: + # Normally we'd have to check whether the result of weakref_entry() + # is None here, in case the object the weakref is pointing to has died. + # In this specific case, however, we know that the only caller of this + # function is `_shadowed_dict()`, and that therefore this weakref is + # guaranteed to point to an object that is still alive. + entry = weakref_entry() + dunder_dict = _get_dunder_dict_of_class(entry) + if '__dict__' in dunder_dict: + class_dict = dunder_dict['__dict__'] if not (type(class_dict) is types.GetSetDescriptorType and class_dict.__name__ == "__dict__" and class_dict.__objclass__ is entry): return class_dict return _sentinel + +def _shadowed_dict(klass): + # gh-118013: the inner function here is decorated with lru_cache for + # performance reasons, *but* make sure not to pass strong references + # to the items in the mro. Doing so can lead to unexpected memory + # consumption in cases where classes are dynamically created and + # destroyed, and the dynamically created classes happen to be the only + # objects that hold strong references to other objects that take up a + # significant amount of memory. + return _shadowed_dict_from_weakref_mro_tuple( + *[make_weakref(entry) for entry in _static_getmro(klass)] + ) + + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the descriptor protocol, __getattr__ or __getattribute__. @@ -1579,8 +1734,10 @@ def getattr_static(obj, attr, default=_sentinel): documentation for details. """ instance_result = _sentinel - if not _is_type(obj): - klass = type(obj) + + objtype = type(obj) + if type not in _static_getmro(objtype): + klass = objtype dict_attr = _shadowed_dict(klass) if (dict_attr is _sentinel or type(dict_attr) is types.MemberDescriptorType): @@ -1591,8 +1748,10 @@ def getattr_static(obj, attr, default=_sentinel): klass_result = _check_class(klass, attr) if instance_result is not _sentinel and klass_result is not _sentinel: - if (_check_class(type(klass_result), '__get__') is not _sentinel and - _check_class(type(klass_result), '__set__') is not _sentinel): + if _check_class(type(klass_result), "__get__") is not _sentinel and ( + _check_class(type(klass_result), "__set__") is not _sentinel + or _check_class(type(klass_result), "__delete__") is not _sentinel + ): return klass_result if instance_result is not _sentinel: @@ -1603,11 +1762,11 @@ def getattr_static(obj, attr, default=_sentinel): if obj is klass: # for types we check the metaclass too for entry in _static_getmro(type(klass)): - if _shadowed_dict(type(entry)) is _sentinel: - try: - return entry.__dict__[attr] - except KeyError: - pass + if ( + _shadowed_dict(type(entry)) is _sentinel + and attr in entry.__dict__ + ): + return entry.__dict__[attr] if default is not _sentinel: return default raise AttributeError(attr) @@ -1631,11 +1790,11 @@ def getgeneratorstate(generator): """ if generator.gi_running: return GEN_RUNNING + if generator.gi_suspended: + return GEN_SUSPENDED if generator.gi_frame is None: return GEN_CLOSED - if generator.gi_frame.f_lasti == -1: - return GEN_CREATED - return GEN_SUSPENDED + return GEN_CREATED def getgeneratorlocals(generator): @@ -1673,11 +1832,11 @@ def getcoroutinestate(coroutine): """ if coroutine.cr_running: return CORO_RUNNING + if coroutine.cr_suspended: + return CORO_SUSPENDED if coroutine.cr_frame is None: return CORO_CLOSED - if coroutine.cr_frame.f_lasti == -1: - return CORO_CREATED - return CORO_SUSPENDED + return CORO_CREATED def getcoroutinelocals(coroutine): @@ -1693,35 +1852,89 @@ def getcoroutinelocals(coroutine): return {} +# ----------------------------------- asynchronous generator introspection + +AGEN_CREATED = 'AGEN_CREATED' +AGEN_RUNNING = 'AGEN_RUNNING' +AGEN_SUSPENDED = 'AGEN_SUSPENDED' +AGEN_CLOSED = 'AGEN_CLOSED' + + +def getasyncgenstate(agen): + """Get current state of an asynchronous generator object. + + Possible states are: + AGEN_CREATED: Waiting to start execution. + AGEN_RUNNING: Currently being executed by the interpreter. + AGEN_SUSPENDED: Currently suspended at a yield expression. + AGEN_CLOSED: Execution has completed. + """ + if agen.ag_running: + return AGEN_RUNNING + if agen.ag_suspended: + return AGEN_SUSPENDED + if agen.ag_frame is None: + return AGEN_CLOSED + return AGEN_CREATED + + +def getasyncgenlocals(agen): + """ + Get the mapping of asynchronous generator local variables to their current + values. + + A dict is returned, with the keys the local variable names and values the + bound values.""" + + if not isasyncgen(agen): + raise TypeError(f"{agen!r} is not a Python async generator") + + frame = getattr(agen, "ag_frame", None) + if frame is not None: + return agen.ag_frame.f_locals + else: + return {} + + ############################################################################### ### Function Signature Object (PEP 362) ############################################################################### -_WrapperDescriptor = type(type.__call__) -_MethodWrapper = type(all.__call__) -_ClassMethodWrapper = type(int.__dict__['from_bytes']) - -_NonUserDefinedCallables = (_WrapperDescriptor, - _MethodWrapper, - _ClassMethodWrapper, +_NonUserDefinedCallables = (types.WrapperDescriptorType, + types.MethodWrapperType, + types.ClassMethodDescriptorType, types.BuiltinFunctionType) -def _signature_get_user_defined_method(cls, method_name): +def _signature_get_user_defined_method(cls, method_name, *, follow_wrapper_chains=True): """Private helper. Checks if ``cls`` has an attribute named ``method_name`` and returns it only if it is a pure python function. """ - try: - meth = getattr(cls, method_name) - except AttributeError: - return + if method_name == '__new__': + meth = getattr(cls, method_name, None) else: - if not isinstance(meth, _NonUserDefinedCallables): - # Once '__signature__' will be added to 'C'-level - # callables, this check won't be necessary - return meth + meth = getattr_static(cls, method_name, None) + if meth is None: + return None + + # NOTE: The meth may wraps a non-user-defined callable. + # In this case, we treat the meth as non-user-defined callable too. + # (e.g. cls.__new__ generated by @warnings.deprecated) + unwrapped_meth = None + if follow_wrapper_chains: + unwrapped_meth = unwrap(meth, stop=(lambda m: hasattr(m, "__signature__") + or _signature_is_builtin(m))) + + if (isinstance(meth, _NonUserDefinedCallables) + or isinstance(unwrapped_meth, _NonUserDefinedCallables)): + # Once '__signature__' will be added to 'C'-level + # callables, this check won't be necessary + return None + if method_name != '__new__': + meth = _descriptor_get(meth, cls) + return meth def _signature_get_partial(wrapped_sig, partial, extra_args=()): @@ -1756,7 +1969,12 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): if param.kind is _POSITIONAL_ONLY: # If positional-only parameter is bound by partial, # it effectively disappears from the signature - new_params.pop(param_name) + # However, if it is a Placeholder it is not removed + # And also looses default value + if arg_value is functools.Placeholder: + new_params[param_name] = param.replace(default=_empty) + else: + new_params.pop(param_name) continue if param.kind is _POSITIONAL_OR_KEYWORD: @@ -1778,7 +1996,17 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()): new_params[param_name] = param.replace(default=arg_value) else: # was passed as a positional argument - new_params.pop(param.name) + # Do not pop if it is a Placeholder + # also change kind to positional only + # and remove default + if arg_value is functools.Placeholder: + new_param = param.replace( + kind=_POSITIONAL_ONLY, + default=_empty + ) + new_params[param_name] = new_param + else: + new_params.pop(param_name) continue if param.kind is _KEYWORD_ONLY: @@ -1834,8 +2062,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): @@ -1854,36 +2084,11 @@ def _signature_is_functionlike(obj): code = getattr(obj, '__code__', None) defaults = getattr(obj, '__defaults__', _void) # Important to use _void ... kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here - annotations = getattr(obj, '__annotations__', None) return (isinstance(code, types.CodeType) and isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and - (kwdefaults is None or isinstance(kwdefaults, dict)) and - isinstance(annotations, dict)) - - -def _signature_get_bound_param(spec): - """ Private helper to get first parameter name from a - __text_signature__ of a builtin method, which should - be in the following format: '($param1, ...)'. - Assumptions are that the first argument won't have - a default value or an annotation. - """ - - assert spec.startswith('($') - - pos = spec.find(',') - if pos == -1: - pos = spec.find(')') - - cpos = spec.find(':') - assert cpos == -1 or cpos > pos - - cpos = spec.find('=') - assert cpos == -1 or cpos > pos - - return spec[2:pos] + (kwdefaults is None or isinstance(kwdefaults, dict))) def _signature_strip_non_python_syntax(signature): @@ -1891,26 +2096,21 @@ def _signature_strip_non_python_syntax(signature): Private helper function. Takes a signature in Argument Clinic's extended signature format. - Returns a tuple of three things: - * that signature re-rendered in standard Python syntax, + Returns a tuple of two things: + * that signature re-rendered in standard Python syntax, and * the index of the "self" parameter (generally 0), or None if - the function does not have a "self" parameter, and - * the index of the last "positional only" parameter, - or None if the signature has no positional-only parameters. + the function does not have a "self" parameter. """ if not signature: - return signature, None, None + return signature, None self_parameter = None - last_positional_only = None - lines = [l.encode('ascii') for l in signature.split('\n')] + lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) - delayed_comma = False - skip_next_comma = False text = [] add = text.append @@ -1927,49 +2127,27 @@ def _signature_strip_non_python_syntax(signature): if type == OP: if string == ',': - if skip_next_comma: - skip_next_comma = False - else: - assert not delayed_comma - delayed_comma = True - current_parameter += 1 - continue - - if string == '/': - assert not skip_next_comma - assert last_positional_only is None - skip_next_comma = True - last_positional_only = current_parameter - 1 - continue + current_parameter += 1 - if (type == ERRORTOKEN) and (string == '$'): + if (type == OP) and (string == '$'): assert self_parameter is None self_parameter = current_parameter continue - if delayed_comma: - delayed_comma = False - if not ((type == OP) and (string == ')')): - add(', ') add(string) if (string == ','): add(' ') - clean_signature = ''.join(text) - return clean_signature, self_parameter, last_positional_only + clean_signature = ''.join(text).strip().replace("\n", "") + return clean_signature, self_parameter def _signature_fromstr(cls, obj, s, skip_bound_arg=True): """Private helper to parse content of '__text_signature__' and return a Signature based on it. """ - # Lazy import ast because it's relatively heavy and - # it's not used for other than this function. - import ast - Parameter = cls._parameter_cls - clean_signature, self_parameter, last_positional_only = \ - _signature_strip_non_python_syntax(s) + clean_signature, self_parameter = _signature_strip_non_python_syntax(s) program = "def foo" + clean_signature + ": pass" @@ -1985,11 +2163,15 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): parameters = [] empty = Parameter.empty - invalid = object() module = None module_dict = {} + module_name = getattr(obj, '__module__', None) + if not module_name: + objclass = getattr(obj, '__objclass__', None) + module_name = getattr(objclass, '__module__', None) + if module_name: module = sys.modules.get(module_name, None) if module: @@ -2009,11 +2191,11 @@ def wrap_value(s): try: value = eval(s, sys_module_dict) except NameError: - raise RuntimeError() + raise ValueError if isinstance(value, (str, int, float, bytes, bool, type(None))): return ast.Constant(value) - raise RuntimeError() + raise ValueError class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): @@ -2023,7 +2205,7 @@ def visit_Attribute(self, node): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - raise RuntimeError() + raise ValueError a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) @@ -2033,33 +2215,43 @@ def visit_Name(self, node): raise ValueError() return wrap_value(node.id) + def visit_BinOp(self, node): + # Support constant folding of a couple simple binary operations + # commonly used to define default values in text signatures + left = self.visit(node.left) + right = self.visit(node.right) + if not isinstance(left, ast.Constant) or not isinstance(right, ast.Constant): + raise ValueError + if isinstance(node.op, ast.Add): + return ast.Constant(left.value + right.value) + elif isinstance(node.op, ast.Sub): + return ast.Constant(left.value - right.value) + elif isinstance(node.op, ast.BitOr): + return ast.Constant(left.value | right.value) + raise ValueError + def p(name_node, default_node, default=empty): name = parse_name(name_node) - if name is invalid: - return None if default_node and default_node is not _empty: try: default_node = RewriteSymbolics().visit(default_node) - o = ast.literal_eval(default_node) + default = ast.literal_eval(default_node) except ValueError: - o = invalid - if o is invalid: - return None - default = o if o is not invalid else default + raise ValueError("{!r} builtin has invalid signature".format(obj)) from None parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters - args = reversed(f.args.args) - defaults = reversed(f.args.defaults) - iter = itertools.zip_longest(args, defaults, fillvalue=None) - if last_positional_only is not None: - kind = Parameter.POSITIONAL_ONLY - else: - kind = Parameter.POSITIONAL_OR_KEYWORD - for i, (name, default) in enumerate(reversed(list(iter))): + total_non_kw_args = len(f.args.posonlyargs) + len(f.args.args) + required_non_kw_args = total_non_kw_args - len(f.args.defaults) + defaults = itertools.chain(itertools.repeat(None, required_non_kw_args), f.args.defaults) + + kind = Parameter.POSITIONAL_ONLY + for (name, default) in zip(f.args.posonlyargs, defaults): + p(name, default) + + kind = Parameter.POSITIONAL_OR_KEYWORD + for (name, default) in zip(f.args.args, defaults): p(name, default) - if i == last_positional_only: - kind = Parameter.POSITIONAL_OR_KEYWORD # *args if f.args.vararg: @@ -2112,7 +2304,9 @@ def _signature_from_builtin(cls, func, skip_bound_arg=True): return _signature_fromstr(cls, func, s, skip_bound_arg) -def _signature_from_function(cls, func, skip_bound_arg=True): +def _signature_from_function(cls, func, skip_bound_arg=True, + globals=None, locals=None, eval_str=False, + *, annotation_format=Format.VALUE): """Private helper: constructs Signature for the given python function.""" is_duck_function = False @@ -2138,7 +2332,8 @@ def _signature_from_function(cls, func, skip_bound_arg=True): positional = arg_names[:pos_count] keyword_only_count = func_code.co_kwonlyargcount keyword_only = arg_names[pos_count:pos_count + keyword_only_count] - annotations = func.__annotations__ + annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str, + format=annotation_format) defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ @@ -2206,26 +2401,44 @@ def _signature_from_function(cls, func, skip_bound_arg=True): __validate_parameters__=is_duck_function) +def _descriptor_get(descriptor, obj): + if isclass(descriptor): + return descriptor + get = getattr(type(descriptor), '__get__', _sentinel) + if get is _sentinel: + return descriptor + return get(descriptor, obj, type(obj)) + + def _signature_from_callable(obj, *, follow_wrapper_chains=True, skip_bound_arg=True, - sigcls): + globals=None, + locals=None, + eval_str=False, + sigcls, + annotation_format=Format.VALUE): """Private helper function to get signature for arbitrary callable objects. """ + _get_signature_of = functools.partial(_signature_from_callable, + follow_wrapper_chains=follow_wrapper_chains, + skip_bound_arg=skip_bound_arg, + globals=globals, + locals=locals, + sigcls=sigcls, + eval_str=eval_str, + annotation_format=annotation_format) + if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) if isinstance(obj, types.MethodType): # In this case we skip the first parameter of the underlying # function (usually `self` or `cls`). - sig = _signature_from_callable( - obj.__func__, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + sig = _get_signature_of(obj.__func__) if skip_bound_arg: return _signature_bound_method(sig) @@ -2234,16 +2447,15 @@ def _signature_from_callable(obj, *, # Was this function wrapped by a decorator? if follow_wrapper_chains: - obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__"))) + # Unwrap until we find an explicit signature or a MethodType (which will be + # handled explicitly below). + obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__") + or isinstance(f, types.MethodType))) if isinstance(obj, types.MethodType): # If the unwrapped object is a *method*, we might want to # skip its first parameter (self). # See test_signature_wrapped_bound_method for details. - return _signature_from_callable( - obj, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + return _get_signature_of(obj) try: sig = obj.__signature__ @@ -2258,7 +2470,7 @@ def _signature_from_callable(obj, *, return sig try: - partialmethod = obj._partialmethod + partialmethod = obj.__partialmethod__ except AttributeError: pass else: @@ -2270,11 +2482,7 @@ def _signature_from_callable(obj, *, # (usually `self`, or `cls`) will not be passed # automatically (as for boundmethods) - wrapped_sig = _signature_from_callable( - partialmethod.func, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + wrapped_sig = _get_signature_of(partialmethod.func) sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] @@ -2286,124 +2494,121 @@ def _signature_from_callable(obj, *, sig_params = tuple(sig.parameters.values()) assert (not sig_params or first_wrapped_param is not sig_params[0]) + # If there were placeholders set, + # first param is transformed to positional only + if partialmethod.args.count(functools.Placeholder): + first_wrapped_param = first_wrapped_param.replace( + kind=Parameter.POSITIONAL_ONLY) new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) + if isinstance(obj, functools.partial): + wrapped_sig = _get_signature_of(obj.func) + return _signature_get_partial(wrapped_sig, obj) + if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type # of a Python function (Cython functions, for instance), then: return _signature_from_function(sigcls, obj, - skip_bound_arg=skip_bound_arg) + skip_bound_arg=skip_bound_arg, + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, skip_bound_arg=skip_bound_arg) - if isinstance(obj, functools.partial): - wrapped_sig = _signature_from_callable( - obj.func, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - return _signature_get_partial(wrapped_sig, obj) - - sig = None if isinstance(obj, type): # obj is a class or a metaclass # First, let's see if it has an overloaded __call__ defined # in its metaclass - call = _signature_get_user_defined_method(type(obj), '__call__') + call = _signature_get_user_defined_method( + type(obj), + '__call__', + follow_wrapper_chains=follow_wrapper_chains, + ) if call is not None: - sig = _signature_from_callable( - call, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - else: - # Now we check if the 'obj' class has a '__new__' method - new = _signature_get_user_defined_method(obj, '__new__') - if new is not None: - sig = _signature_from_callable( - new, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + return _get_signature_of(call) + + # NOTE: The user-defined method can be a function with a thin wrapper + # around object.__new__ (e.g., generated by `@warnings.deprecated`) + new = _signature_get_user_defined_method( + obj, + '__new__', + follow_wrapper_chains=follow_wrapper_chains, + ) + init = _signature_get_user_defined_method( + obj, + '__init__', + follow_wrapper_chains=follow_wrapper_chains, + ) + + # Go through the MRO and see if any class has user-defined + # pure Python __new__ or __init__ method + for base in obj.__mro__: + # Now we check if the 'obj' class has an own '__new__' method + if new is not None and '__new__' in base.__dict__: + sig = _get_signature_of(new) + if skip_bound_arg: + sig = _signature_bound_method(sig) + return sig + # or an own '__init__' method + elif init is not None and '__init__' in base.__dict__: + return _get_signature_of(init) + + # At this point we know, that `obj` is a class, with no user- + # defined '__init__', '__new__', or class-level '__call__' + + for base in obj.__mro__[:-1]: + # Since '__text_signature__' is implemented as a + # descriptor that extracts text signature from the + # class docstring, if 'obj' is derived from a builtin + # class, its own '__text_signature__' may be 'None'. + # Therefore, we go through the MRO (except the last + # class in there, which is 'object') to find the first + # class with non-empty text signature. + try: + text_sig = base.__text_signature__ + except AttributeError: + pass else: - # Finally, we should have at least __init__ implemented - init = _signature_get_user_defined_method(obj, '__init__') - if init is not None: - sig = _signature_from_callable( - init, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - - if sig is None: - # At this point we know, that `obj` is a class, with no user- - # defined '__init__', '__new__', or class-level '__call__' - - for base in obj.__mro__[:-1]: - # Since '__text_signature__' is implemented as a - # descriptor that extracts text signature from the - # class docstring, if 'obj' is derived from a builtin - # class, its own '__text_signature__' may be 'None'. - # Therefore, we go through the MRO (except the last - # class in there, which is 'object') to find the first - # class with non-empty text signature. - try: - text_sig = base.__text_signature__ - except AttributeError: - pass - else: - if text_sig: - # If 'obj' class has a __text_signature__ attribute: - # return a signature based on it - return _signature_fromstr(sigcls, obj, text_sig) - - # No '__text_signature__' was found for the 'obj' class. - # Last option is to check if its '__init__' is - # object.__init__ or type.__init__. - if type not in obj.__mro__: - # We have a class (not metaclass), but no user-defined - # __init__ or __new__ for it - if (obj.__init__ is object.__init__ and - obj.__new__ is object.__new__): - # Return a signature of 'object' builtin. - return sigcls.from_callable(object) - else: - raise ValueError( - 'no signature found for builtin type {!r}'.format(obj)) + if text_sig: + # If 'base' class has a __text_signature__ attribute: + # return a signature based on it + return _signature_fromstr(sigcls, base, text_sig) + + # No '__text_signature__' was found for the 'obj' class. + # Last option is to check if its '__init__' is + # object.__init__ or type.__init__. + if type not in obj.__mro__: + obj_init = obj.__init__ + obj_new = obj.__new__ + if follow_wrapper_chains: + obj_init = unwrap(obj_init) + obj_new = unwrap(obj_new) + # We have a class (not metaclass), but no user-defined + # __init__ or __new__ for it + if obj_init is object.__init__ and obj_new is object.__new__: + # Return a signature of 'object' builtin. + return sigcls.from_callable(object) + else: + raise ValueError( + 'no signature found for builtin type {!r}'.format(obj)) - elif not isinstance(obj, _NonUserDefinedCallables): + else: # An object with __call__ - # We also check that the 'obj' is not an instance of - # _WrapperDescriptor or _MethodWrapper to avoid - # infinite recursion (and even potential segfault) - call = _signature_get_user_defined_method(type(obj), '__call__') + call = getattr_static(type(obj), '__call__', None) if call is not None: try: - sig = _signature_from_callable( - call, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - except ValueError as ex: - msg = 'no signature found for {!r}'.format(obj) - raise ValueError(msg) from ex - - if sig is not None: - # For classes and objects we skip the first parameter of their - # __call__, __new__, or __init__ methods - if skip_bound_arg: - return _signature_bound_method(sig) - else: - return sig - - if isinstance(obj, types.BuiltinFunctionType): - # Raise a nicer error message for builtins - msg = 'no signature found for builtin function {!r}'.format(obj) - raise ValueError(msg) + text_sig = obj.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + return _signature_fromstr(sigcls, obj, text_sig) + call = _descriptor_get(call, obj) + return _get_signature_of(call) raise ValueError('callable {!r} is not supported by signature'.format(obj)) @@ -2417,18 +2622,21 @@ class _empty: class _ParameterKind(enum.IntEnum): - POSITIONAL_ONLY = 0 - POSITIONAL_OR_KEYWORD = 1 - VAR_POSITIONAL = 2 - KEYWORD_ONLY = 3 - VAR_KEYWORD = 4 + POSITIONAL_ONLY = 'positional-only' + POSITIONAL_OR_KEYWORD = 'positional or keyword' + VAR_POSITIONAL = 'variadic positional' + KEYWORD_ONLY = 'keyword-only' + VAR_KEYWORD = 'variadic keyword' + + def __new__(cls, description): + value = len(cls.__members__) + member = int.__new__(cls, value) + member._value_ = value + member.description = description + return member def __str__(self): - return self._name_ - - @property - def description(self): - return _PARAM_NAME_MAPPING[self] + return self.name _POSITIONAL_ONLY = _ParameterKind.POSITIONAL_ONLY _POSITIONAL_OR_KEYWORD = _ParameterKind.POSITIONAL_OR_KEYWORD @@ -2436,14 +2644,6 @@ def description(self): _KEYWORD_ONLY = _ParameterKind.KEYWORD_ONLY _VAR_KEYWORD = _ParameterKind.VAR_KEYWORD -_PARAM_NAME_MAPPING = { - _POSITIONAL_ONLY: 'positional-only', - _POSITIONAL_OR_KEYWORD: 'positional or keyword', - _VAR_POSITIONAL: 'variadic positional', - _KEYWORD_ONLY: 'keyword-only', - _VAR_KEYWORD: 'variadic keyword' -} - class Parameter: """Represents a parameter in a function signature. @@ -2512,7 +2712,10 @@ def __init__(self, name, kind, *, default=_empty, annotation=_empty): self._kind = _POSITIONAL_ONLY name = 'implicit{}'.format(name[1:]) - if not name.isidentifier(): + # It's possible for C functions to have a positional-only parameter + # where the name is a keyword, so for compatibility we'll allow it. + is_keyword = iskeyword(name) and self._kind is not _POSITIONAL_ONLY + if is_keyword or not name.isidentifier(): raise ValueError('{!r} is not a valid parameter name'.format(name)) self._name = name @@ -2562,13 +2765,17 @@ def replace(self, *, name=_void, kind=_void, return type(self)(name, kind, default=default, annotation=annotation) def __str__(self): + return self._format() + + def _format(self, *, quote_annotation_strings=True): kind = self.kind formatted = self._name # Add annotation and default value if self._annotation is not _empty: - formatted = '{}: {}'.format(formatted, - formatannotation(self._annotation)) + annotation = formatannotation(self._annotation, + quote_annotation_strings=quote_annotation_strings) + formatted = '{}: {}'.format(formatted, annotation) if self._default is not _empty: if self._annotation is not _empty: @@ -2583,11 +2790,13 @@ def __str__(self): return formatted + __replace__ = replace + def __repr__(self): return '<{} "{}">'.format(self.__class__.__name__, self) def __hash__(self): - return hash((self.name, self.kind, self.annotation, self.default)) + return hash((self._name, self._kind, self._annotation, self._default)) def __eq__(self, other): if self is other: @@ -2606,7 +2815,7 @@ class BoundArguments: Has the following public attributes: - * arguments : OrderedDict + * arguments : dict An ordered mutable mapping of parameters' names to arguments' values. Does not contain arguments' default values. * signature : Signature @@ -2706,7 +2915,7 @@ def apply_defaults(self): # Signature.bind_partial(). continue new_arguments.append((name, val)) - self.arguments = OrderedDict(new_arguments) + self.arguments = dict(new_arguments) def __eq__(self, other): if self is other: @@ -2772,12 +2981,20 @@ def __init__(self, parameters=None, *, return_annotation=_empty, if __validate_parameters__: params = OrderedDict() top_kind = _POSITIONAL_ONLY - kind_defaults = False + seen_default = False + seen_var_parameters = set() - for idx, param in enumerate(parameters): + for param in parameters: kind = param.kind name = param.name + if kind in (_VAR_POSITIONAL, _VAR_KEYWORD): + if kind in seen_var_parameters: + msg = f'more than one {kind.description} parameter' + raise ValueError(msg) + + seen_var_parameters.add(kind) + if kind < top_kind: msg = ( 'wrong parameter order: {} parameter before {} ' @@ -2787,21 +3004,19 @@ def __init__(self, parameters=None, *, return_annotation=_empty, kind.description) raise ValueError(msg) elif kind > top_kind: - kind_defaults = False top_kind = kind if kind in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD): if param.default is _empty: - if kind_defaults: + if seen_default: # No default for this parameter, but the - # previous parameter of the same kind had - # a default + # previous parameter of had a default msg = 'non-default argument follows default ' \ 'argument' raise ValueError(msg) else: # There is a default for this parameter. - kind_defaults = True + seen_default = True if name in params: msg = 'duplicate parameter name: {!r}'.format(name) @@ -2809,41 +3024,20 @@ def __init__(self, parameters=None, *, return_annotation=_empty, params[name] = param else: - params = OrderedDict(((param.name, param) - for param in parameters)) + params = OrderedDict((param.name, param) for param in parameters) self._parameters = types.MappingProxyType(params) self._return_annotation = return_annotation @classmethod - def from_function(cls, func): - """Constructs Signature for the given python function. - - Deprecated since Python 3.5, use `Signature.from_callable()`. - """ - - warnings.warn("inspect.Signature.from_function() is deprecated since " - "Python 3.5, use Signature.from_callable()", - DeprecationWarning, stacklevel=2) - return _signature_from_function(cls, func) - - @classmethod - def from_builtin(cls, func): - """Constructs Signature for the given builtin function. - - Deprecated since Python 3.5, use `Signature.from_callable()`. - """ - - warnings.warn("inspect.Signature.from_builtin() is deprecated since " - "Python 3.5, use Signature.from_callable()", - DeprecationWarning, stacklevel=2) - return _signature_from_builtin(cls, func) - - @classmethod - def from_callable(cls, obj, *, follow_wrapped=True): + def from_callable(cls, obj, *, + follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Constructs Signature for the given callable object.""" return _signature_from_callable(obj, sigcls=cls, - follow_wrapper_chains=follow_wrapped) + follow_wrapper_chains=follow_wrapped, + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) @property def parameters(self): @@ -2868,6 +3062,8 @@ def replace(self, *, parameters=_void, return_annotation=_void): return type(self)(parameters, return_annotation=return_annotation) + __replace__ = replace + def _hash_basis(self): params = tuple(param for param in self.parameters.values() if param.kind != _KEYWORD_ONLY) @@ -2892,12 +3088,14 @@ def __eq__(self, other): def _bind(self, args, kwargs, *, partial=False): """Private method. Don't use directly.""" - arguments = OrderedDict() + arguments = {} parameters = iter(self.parameters.values()) parameters_ex = () arg_vals = iter(args) + pos_only_param_in_kwargs = [] + while True: # Let's iterate through the positional arguments and corresponding # parameters @@ -2918,10 +3116,13 @@ def _bind(self, args, kwargs, *, partial=False): break elif param.name in kwargs: if param.kind == _POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None + if param.default is _empty: + msg = f'missing a required positional-only argument: {param.name!r}' + raise TypeError(msg) + # Raise a TypeError once we are sure there is no + # **kwargs param later. + pos_only_param_in_kwargs.append(param) + continue parameters_ex = (param,) break elif (param.kind == _VAR_KEYWORD or @@ -2938,8 +3139,12 @@ def _bind(self, args, kwargs, *, partial=False): parameters_ex = (param,) break else: - msg = 'missing a required argument: {arg!r}' - msg = msg.format(arg=param.name) + if param.kind == _KEYWORD_ONLY: + argtype = ' keyword-only' + else: + argtype = '' + msg = 'missing a required{argtype} argument: {arg!r}' + msg = msg.format(arg=param.name, argtype=argtype) raise TypeError(msg) from None else: # We have a positional argument to process @@ -2999,20 +3204,22 @@ def _bind(self, args, kwargs, *, partial=False): format(arg=param_name)) from None else: - if param.kind == _POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. \ - format(arg=param.name)) - arguments[param_name] = arg_val if kwargs: if kwargs_param is not None: # Process our '**kwargs'-like parameter arguments[kwargs_param.name] = kwargs + elif pos_only_param_in_kwargs: + raise TypeError( + 'got some positional-only arguments passed as ' + 'keyword arguments: {arg!r}'.format( + arg=', '.join( + param.name + for param in pos_only_param_in_kwargs + ), + ), + ) else: raise TypeError( 'got an unexpected keyword argument {arg!r}'.format( @@ -3046,11 +3253,26 @@ def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, self) def __str__(self): + return self.format() + + def format(self, *, max_width=None, quote_annotation_strings=True): + """Create a string representation of the Signature object. + + If *max_width* integer is passed, + signature will try to fit into the *max_width*. + If signature is longer than *max_width*, + all parameters will be on separate lines. + + If *quote_annotation_strings* is False, annotations + in the signature are displayed without opening and closing quotation + marks. This is useful when the signature was created with the + STRING format or when ``from __future__ import annotations`` was used. + """ result = [] render_pos_only_separator = False render_kw_only_separator = True for param in self.parameters.values(): - formatted = str(param) + formatted = param._format(quote_annotation_strings=quote_annotation_strings) kind = param.kind @@ -3083,17 +3305,45 @@ def __str__(self): result.append('/') rendered = '({})'.format(', '.join(result)) + if max_width is not None and len(rendered) > max_width: + rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: - anno = formatannotation(self.return_annotation) + anno = formatannotation(self.return_annotation, + quote_annotation_strings=quote_annotation_strings) rendered += ' -> {}'.format(anno) return rendered -def signature(obj, *, follow_wrapped=True): +def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False, + annotation_format=Format.VALUE): """Get a signature object for the passed callable.""" - return Signature.from_callable(obj, follow_wrapped=follow_wrapped) + return Signature.from_callable(obj, follow_wrapped=follow_wrapped, + globals=globals, locals=locals, eval_str=eval_str, + annotation_format=annotation_format) + + +class BufferFlags(enum.IntFlag): + SIMPLE = 0x0 + WRITABLE = 0x1 + FORMAT = 0x4 + ND = 0x8 + STRIDES = 0x10 | ND + C_CONTIGUOUS = 0x20 | STRIDES + F_CONTIGUOUS = 0x40 | STRIDES + ANY_CONTIGUOUS = 0x80 | STRIDES + INDIRECT = 0x100 | STRIDES + CONTIG = ND | WRITABLE + CONTIG_RO = ND + STRIDED = STRIDES | WRITABLE + STRIDED_RO = STRIDES + RECORDS = STRIDES | WRITABLE | FORMAT + RECORDS_RO = STRIDES | FORMAT + FULL = INDIRECT | WRITABLE | FORMAT + FULL_RO = INDIRECT | FORMAT + READ = 0x100 + WRITE = 0x200 def _main(): @@ -3101,7 +3351,7 @@ def _main(): import argparse import importlib - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument( 'object', help="The object to be analysed. " diff --git a/Lib/io.py b/Lib/io.py index a8a31c34712..63ffadb1d38 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -45,42 +45,20 @@ "FileIO", "BytesIO", "StringIO", "BufferedIOBase", "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper", - "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END"] + "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END", + "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder", + "Reader", "Writer"] import _io import abc +from _collections_abc import _check_methods from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, - open, open_code, BytesIO, StringIO, BufferedReader, + open, open_code, FileIO, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, - # XXX RUSTPYTHON TODO: IncrementalNewlineDecoder - # IncrementalNewlineDecoder, text_encoding, TextIOWrapper) - text_encoding, TextIOWrapper) + IncrementalNewlineDecoder, text_encoding, TextIOWrapper) -try: - from _io import FileIO -except ImportError: - pass - -def __getattr__(name): - if name == "OpenWrapper": - # bpo-43680: Until Python 3.9, _pyio.open was not a static method and - # builtins.open was set to OpenWrapper to not become a bound method - # when set to a class variable. _io.open is a built-in function whereas - # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now - # a static method, and builtins.open() is now io.open(). - import warnings - warnings.warn('OpenWrapper is deprecated, use open instead', - DeprecationWarning, stacklevel=2) - global OpenWrapper - OpenWrapper = open - return OpenWrapper - raise AttributeError(name) - - -# Pretend this exception was created here. -UnsupportedOperation.__module__ = "io" # for seek() SEEK_SET = 0 @@ -102,10 +80,7 @@ class BufferedIOBase(_io._BufferedIOBase, IOBase): class TextIOBase(_io._TextIOBase, IOBase): __doc__ = _io._TextIOBase.__doc__ -try: - RawIOBase.register(FileIO) -except NameError: - pass +RawIOBase.register(FileIO) for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom, BufferedRWPair): @@ -121,3 +96,55 @@ class TextIOBase(_io._TextIOBase, IOBase): pass else: RawIOBase.register(_WindowsConsoleIO) + +# +# Static Typing Support +# + +GenericAlias = type(list[int]) + + +class Reader(metaclass=abc.ABCMeta): + """Protocol for simple I/O reader instances. + + This protocol only supports blocking I/O. + """ + + __slots__ = () + + @abc.abstractmethod + def read(self, size=..., /): + """Read data from the input stream and return it. + + If *size* is specified, at most *size* items (bytes/characters) will be + read. + """ + + @classmethod + def __subclasshook__(cls, C): + if cls is Reader: + return _check_methods(C, "read") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) + + +class Writer(metaclass=abc.ABCMeta): + """Protocol for simple I/O writer instances. + + This protocol only supports blocking I/O. + """ + + __slots__ = () + + @abc.abstractmethod + def write(self, data, /): + """Write *data* to the output stream and return the number of items written.""" + + @classmethod + def __subclasshook__(cls, C): + if cls is Writer: + return _check_methods(C, "write") + return NotImplemented + + __class_getitem__ = classmethod(GenericAlias) diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 1cb71d8032e..ca732e4f2e8 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -239,7 +239,7 @@ def summarize_address_range(first, last): else: raise ValueError('unknown IP version') - ip_bits = first._max_prefixlen + ip_bits = first.max_prefixlen first_int = first._ip last_int = last._ip while first_int <= last_int: @@ -310,7 +310,7 @@ def collapse_addresses(addresses): [IPv4Network('192.0.2.0/24')] Args: - addresses: An iterator of IPv4Network or IPv6Network objects. + addresses: An iterable of IPv4Network or IPv6Network objects. Returns: An iterator of the collapsed IPv(4|6)Network objects. @@ -326,12 +326,12 @@ def collapse_addresses(addresses): # split IP addresses and networks for ip in addresses: if isinstance(ip, _BaseAddress): - if ips and ips[-1]._version != ip._version: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) ips.append(ip) - elif ip._prefixlen == ip._max_prefixlen: - if ips and ips[-1]._version != ip._version: + elif ip._prefixlen == ip.max_prefixlen: + if ips and ips[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, ips[-1])) try: @@ -339,7 +339,7 @@ def collapse_addresses(addresses): except AttributeError: ips.append(ip.network_address) else: - if nets and nets[-1]._version != ip._version: + if nets and nets[-1].version != ip.version: raise TypeError("%s and %s are not of the same version" % ( ip, nets[-1])) nets.append(ip) @@ -407,26 +407,21 @@ def reverse_pointer(self): """ return self._reverse_pointer() - @property - def version(self): - msg = '%200s has no version specified' % (type(self),) - raise NotImplementedError(msg) - def _check_int_address(self, address): if address < 0: msg = "%d (< 0) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._version)) + raise AddressValueError(msg % (address, self.version)) if address > self._ALL_ONES: msg = "%d (>= 2**%d) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._max_prefixlen, - self._version)) + raise AddressValueError(msg % (address, self.max_prefixlen, + self.version)) def _check_packed_address(self, address, expected_len): address_len = len(address) if address_len != expected_len: msg = "%r (len %d != %d) is not permitted as an IPv%d address" raise AddressValueError(msg % (address, address_len, - expected_len, self._version)) + expected_len, self.version)) @classmethod def _ip_int_from_prefix(cls, prefixlen): @@ -455,12 +450,12 @@ def _prefix_from_ip_int(cls, ip_int): ValueError: If the input intermingles zeroes & ones """ trailing_zeroes = _count_righthand_zero_bits(ip_int, - cls._max_prefixlen) - prefixlen = cls._max_prefixlen - trailing_zeroes + cls.max_prefixlen) + prefixlen = cls.max_prefixlen - trailing_zeroes leading_ones = ip_int >> trailing_zeroes all_ones = (1 << prefixlen) - 1 if leading_ones != all_ones: - byteslen = cls._max_prefixlen // 8 + byteslen = cls.max_prefixlen // 8 details = ip_int.to_bytes(byteslen, 'big') msg = 'Netmask pattern %r mixes zeroes & ones' raise ValueError(msg % details) @@ -492,7 +487,7 @@ def _prefix_from_prefix_string(cls, prefixlen_str): prefixlen = int(prefixlen_str) except ValueError: cls._report_invalid_netmask(prefixlen_str) - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen_str) return prefixlen @@ -542,7 +537,7 @@ def _split_addr_prefix(cls, address): """ # a packed address or integer if isinstance(address, (bytes, int)): - return address, cls._max_prefixlen + return address, cls.max_prefixlen if not isinstance(address, tuple): # Assume input argument to be string or any object representation @@ -552,7 +547,7 @@ def _split_addr_prefix(cls, address): # Constructing from a tuple (addr, [mask]) if len(address) > 1: return address - return address[0], cls._max_prefixlen + return address[0], cls.max_prefixlen def __reduce__(self): return self.__class__, (str(self),) @@ -577,14 +572,14 @@ def __int__(self): def __eq__(self, other): try: return (self._ip == other._ip - and self._version == other._version) + and self.version == other.version) except AttributeError: return NotImplemented def __lt__(self, other): if not isinstance(other, _BaseAddress): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self._ip != other._ip: @@ -613,7 +608,7 @@ def __hash__(self): return hash(hex(int(self._ip))) def _get_address_key(self): - return (self._version, self) + return (self.version, self) def __reduce__(self): return self.__class__, (self._ip,) @@ -649,15 +644,15 @@ def __format__(self, fmt): # Set some defaults if fmt_base == 'n': - if self._version == 4: + if self.version == 4: fmt_base = 'b' # Binary is default for ipv4 else: fmt_base = 'x' # Hex is default for ipv6 if fmt_base == 'b': - padlen = self._max_prefixlen + padlen = self.max_prefixlen else: - padlen = self._max_prefixlen // 4 + padlen = self.max_prefixlen // 4 if grouping: padlen += padlen // 4 - 1 @@ -716,7 +711,7 @@ def __getitem__(self, n): def __lt__(self, other): if not isinstance(other, _BaseNetwork): return NotImplemented - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same version' % ( self, other)) if self.network_address != other.network_address: @@ -727,18 +722,18 @@ def __lt__(self, other): def __eq__(self, other): try: - return (self._version == other._version and + return (self.version == other.version and self.network_address == other.network_address and int(self.netmask) == int(other.netmask)) except AttributeError: return NotImplemented def __hash__(self): - return hash(int(self.network_address) ^ int(self.netmask)) + return hash((int(self.network_address), int(self.netmask))) def __contains__(self, other): # always false if one is v4 and the other is v6. - if self._version != other._version: + if self.version != other.version: return False # dealing with another network. if isinstance(other, _BaseNetwork): @@ -829,7 +824,7 @@ def address_exclude(self, other): ValueError: If other is not completely contained by self. """ - if not self._version == other._version: + if not self.version == other.version: raise TypeError("%s and %s are not of the same version" % ( self, other)) @@ -901,10 +896,10 @@ def compare_networks(self, other): """ # does this need to raise a ValueError? - if self._version != other._version: + if self.version != other.version: raise TypeError('%s and %s are not of the same type' % ( self, other)) - # self._version == other._version below here: + # self.version == other.version below here: if self.network_address < other.network_address: return -1 if self.network_address > other.network_address: @@ -924,7 +919,7 @@ def _get_networks_key(self): and list.sort(). """ - return (self._version, self.network_address, self.netmask) + return (self.version, self.network_address, self.netmask) def subnets(self, prefixlen_diff=1, new_prefix=None): """The subnets which join to make the current subnet. @@ -952,7 +947,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): number means a larger network) """ - if self._prefixlen == self._max_prefixlen: + if self._prefixlen == self.max_prefixlen: yield self return @@ -967,7 +962,7 @@ def subnets(self, prefixlen_diff=1, new_prefix=None): raise ValueError('prefix length diff must be > 0') new_prefixlen = self._prefixlen + prefixlen_diff - if new_prefixlen > self._max_prefixlen: + if new_prefixlen > self.max_prefixlen: raise ValueError( 'prefix length diff %d is invalid for netblock %s' % ( new_prefixlen, self)) @@ -1036,7 +1031,7 @@ def is_multicast(self): def _is_subnet_of(a, b): try: # Always false if one is v4 and the other is v6. - if a._version != b._version: + if a.version != b.version: raise TypeError(f"{a} and {b} are not of the same version") return (b.network_address <= a.network_address and b.broadcast_address >= a.broadcast_address) @@ -1086,7 +1081,11 @@ def is_private(self): """ return any(self.network_address in priv_network and self.broadcast_address in priv_network - for priv_network in self._constants._private_networks) + for priv_network in self._constants._private_networks) and all( + self.network_address not in network and + self.broadcast_address not in network + for network in self._constants._private_networks_exceptions + ) @property def is_global(self): @@ -1142,11 +1141,11 @@ class _BaseV4: """ __slots__ = () - _version = 4 + version = 4 # Equivalent to 255.255.255.255 or 32 bits of 1's. _ALL_ONES = (2**IPV4LENGTH) - 1 - _max_prefixlen = IPV4LENGTH + max_prefixlen = IPV4LENGTH # There are only a handful of valid v4 netmasks, so we cache them all # when constructed (see _make_netmask()). _netmask_cache = {} @@ -1166,7 +1165,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: try: @@ -1264,15 +1263,6 @@ def _reverse_pointer(self): reverse_octets = str(self).split('.')[::-1] return '.'.join(reverse_octets) + '.in-addr.arpa' - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv4Address(_BaseV4, _BaseAddress): """Represent and manipulate single IPv4 Addresses.""" @@ -1333,18 +1323,41 @@ def is_reserved(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property @functools.lru_cache() def is_global(self): + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. + """ return self not in self._constants._public_network and not self.is_private @property @@ -1389,6 +1402,16 @@ def is_link_local(self): """ return self in self._constants._linklocal_network + @property + def ipv6_mapped(self): + """Return the IPv4-mapped IPv6 address. + + Returns: + The IPv4-mapped IPv6 address per RFC 4291. + + """ + return IPv6Address(f'::ffff:{self}') + class IPv4Interface(IPv4Address): @@ -1519,10 +1542,10 @@ def __init__(self, address, strict=True): self.network_address = IPv4Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == (self._max_prefixlen): - self.hosts = lambda: [IPv4Address(addr)] + elif self._prefixlen == (self.max_prefixlen): + self.hosts = lambda: iter((IPv4Address(addr),)) @property @functools.lru_cache() @@ -1548,13 +1571,15 @@ class _IPv4Constants: _public_network = IPv4Network('100.64.0.0/10') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml _private_networks = [ IPv4Network('0.0.0.0/8'), IPv4Network('10.0.0.0/8'), IPv4Network('127.0.0.0/8'), IPv4Network('169.254.0.0/16'), IPv4Network('172.16.0.0/12'), - IPv4Network('192.0.0.0/29'), + IPv4Network('192.0.0.0/24'), IPv4Network('192.0.0.170/31'), IPv4Network('192.0.2.0/24'), IPv4Network('192.168.0.0/16'), @@ -1565,6 +1590,11 @@ class _IPv4Constants: IPv4Network('255.255.255.255/32'), ] + _private_networks_exceptions = [ + IPv4Network('192.0.0.9/32'), + IPv4Network('192.0.0.10/32'), + ] + _reserved_network = IPv4Network('240.0.0.0/4') _unspecified_address = IPv4Address('0.0.0.0') @@ -1584,11 +1614,11 @@ class _BaseV6: """ __slots__ = () - _version = 6 + version = 6 _ALL_ONES = (2**IPV6LENGTH) - 1 _HEXTET_COUNT = 8 _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') - _max_prefixlen = IPV6LENGTH + max_prefixlen = IPV6LENGTH # There are only a bunch of valid v6 netmasks, so we cache them all # when constructed (see _make_netmask()). @@ -1606,7 +1636,7 @@ def _make_netmask(cls, arg): if arg not in cls._netmask_cache: if isinstance(arg, int): prefixlen = arg - if not (0 <= prefixlen <= cls._max_prefixlen): + if not (0 <= prefixlen <= cls.max_prefixlen): cls._report_invalid_netmask(prefixlen) else: prefixlen = cls._prefix_from_prefix_string(arg) @@ -1630,8 +1660,18 @@ def _ip_int_from_string(cls, ip_str): """ if not ip_str: raise AddressValueError('Address cannot be empty') - - parts = ip_str.split(':') + if len(ip_str) > 45: + shorten = ip_str + if len(shorten) > 100: + shorten = f'{ip_str[:45]}({len(ip_str)-90} chars elided){ip_str[-45:]}' + raise AddressValueError(f"At most 45 characters expected in " + f"{shorten!r}") + + # We want to allow more parts than the max to be 'split' + # to preserve the correct error message when there are + # too many parts combined with '::' + _max_parts = cls._HEXTET_COUNT + 1 + parts = ip_str.split(':', maxsplit=_max_parts) # An IPv6 address needs at least 2 colons (3 parts). _min_parts = 3 @@ -1651,7 +1691,6 @@ def _ip_int_from_string(cls, ip_str): # An IPv6 address can't have more than 8 colons (9 parts). # The extra colon comes from using the "::" notation for a single # leading or trailing zero part. - _max_parts = cls._HEXTET_COUNT + 1 if len(parts) > _max_parts: msg = "At most %d colons permitted in %r" % (_max_parts-1, ip_str) raise AddressValueError(msg) @@ -1821,9 +1860,6 @@ def _string_from_ip_int(cls, ip_int=None): def _explode_shorthand_ip_string(self): """Expand a shortened IPv6 address. - Args: - ip_str: A string, the IPv6 address. - Returns: A string, the expanded IPv6 address. @@ -1871,15 +1907,6 @@ def _split_scope_id(ip_str): raise AddressValueError('Invalid IPv6 address: "%r"' % ip_str) return addr, scope_id - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - class IPv6Address(_BaseV6, _BaseAddress): """Represent and manipulate single IPv6 Addresses.""" @@ -1926,8 +1953,49 @@ def __init__(self, address): self._ip = self._ip_int_from_string(addr_str) + def _explode_shorthand_ip_string(self): + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + return super()._explode_shorthand_ip_string() + prefix_len = 30 + raw_exploded_str = super()._explode_shorthand_ip_string() + return f"{raw_exploded_str[:prefix_len]}{ipv4_mapped!s}" + + def _reverse_pointer(self): + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + return super()._reverse_pointer() + prefix_len = 30 + raw_exploded_str = super()._explode_shorthand_ip_string()[:prefix_len] + # ipv4 encoded using hexadecimal nibbles instead of decimals + ipv4_int = ipv4_mapped._ip + reverse_chars = f"{raw_exploded_str}{ipv4_int:008x}"[::-1].replace(':', '') + return '.'.join(reverse_chars) + '.ip6.arpa' + + def _ipv4_mapped_ipv6_to_str(self): + """Return convenient text representation of IPv4-mapped IPv6 address + + See RFC 4291 2.5.5.2, 2.2 p.3 for details. + + Returns: + A string, 'x:x:x:x:x:x:d.d.d.d', where the 'x's are the hexadecimal values of + the six high-order 16-bit pieces of the address, and the 'd's are + the decimal values of the four low-order 8-bit pieces of the + address (standard IPv4 representation) as defined in RFC 4291 2.2 p.3. + + """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + raise AddressValueError("Can not apply to non-IPv4-mapped IPv6 address %s" % str(self)) + high_order_bits = self._ip >> 32 + return "%s:%s" % (self._string_from_ip_int(high_order_bits), str(ipv4_mapped)) + def __str__(self): - ip_str = super().__str__() + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is None: + ip_str = super().__str__() + else: + ip_str = self._ipv4_mapped_ipv6_to_str() return ip_str + '%' + self._scope_id if self._scope_id else ip_str def __hash__(self): @@ -1941,6 +2009,9 @@ def __eq__(self, other): return False return self._scope_id == getattr(other, '_scope_id', None) + def __reduce__(self): + return (self.__class__, (str(self),)) + @property def scope_id(self): """Identifier of a particular zone of the address's scope. @@ -1967,6 +2038,9 @@ def is_multicast(self): See RFC 2373 2.7 for details. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_multicast return self in self._constants._multicast_network @property @@ -1978,6 +2052,9 @@ def is_reserved(self): reserved IPv6 Network ranges. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_reserved return any(self in x for x in self._constants._reserved_networks) @property @@ -1988,6 +2065,9 @@ def is_link_local(self): A boolean, True if the address is reserved per RFC 4291. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_link_local return self in self._constants._linklocal_network @property @@ -2007,28 +2087,46 @@ def is_site_local(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry, or is ipv4_mapped and is - reserved in the iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + address.is_private == address.ipv4_mapped.is_private + + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ ipv4_mapped = self.ipv4_mapped if ipv4_mapped is not None: return ipv4_mapped.is_private - return any(self in net for net in self._constants._private_networks) + return ( + any(self in net for net in self._constants._private_networks) + and all(self not in net for net in self._constants._private_networks_exceptions) + ) @property def is_global(self): - """Test if this address is allocated for public networks. + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_global return not self.is_private @property @@ -2040,6 +2138,9 @@ def is_unspecified(self): RFC 2373 2.5.2. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_unspecified return self._ip == 0 @property @@ -2051,6 +2152,9 @@ def is_loopback(self): RFC 2373 2.5.3. """ + ipv4_mapped = self.ipv4_mapped + if ipv4_mapped is not None: + return ipv4_mapped.is_loopback return self._ip == 1 @property @@ -2167,7 +2271,7 @@ def is_unspecified(self): @property def is_loopback(self): - return self._ip == 1 and self.network.is_loopback + return super().is_loopback and self.network.is_loopback class IPv6Network(_BaseV6, _BaseNetwork): @@ -2229,10 +2333,10 @@ def __init__(self, address, strict=True): self.network_address = IPv6Address(packed & int(self.netmask)) - if self._prefixlen == (self._max_prefixlen - 1): + if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ - elif self._prefixlen == self._max_prefixlen: - self.hosts = lambda: [IPv6Address(addr)] + elif self._prefixlen == self.max_prefixlen: + self.hosts = lambda: iter((IPv6Address(addr),)) def hosts(self): """Generate Iterator over usable hosts in a network. @@ -2268,19 +2372,33 @@ class _IPv6Constants: _multicast_network = IPv6Network('ff00::/8') + # Not globally reachable address blocks listed on + # https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml _private_networks = [ IPv6Network('::1/128'), IPv6Network('::/128'), IPv6Network('::ffff:0:0/96'), + IPv6Network('64:ff9b:1::/48'), IPv6Network('100::/64'), IPv6Network('2001::/23'), - IPv6Network('2001:2::/48'), IPv6Network('2001:db8::/32'), - IPv6Network('2001:10::/28'), + # IANA says N/A, let's consider it not globally reachable to be safe + IPv6Network('2002::/16'), + # RFC 9637: https://www.rfc-editor.org/rfc/rfc9637.html#section-6-2.2 + IPv6Network('3fff::/20'), IPv6Network('fc00::/7'), IPv6Network('fe80::/10'), ] + _private_networks_exceptions = [ + IPv6Network('2001:1::1/128'), + IPv6Network('2001:1::2/128'), + IPv6Network('2001:3::/32'), + IPv6Network('2001:4:112::/48'), + IPv6Network('2001:20::/28'), + IPv6Network('2001:30::/28'), + ] + _reserved_networks = [ IPv6Network('::/8'), IPv6Network('100::/8'), IPv6Network('200::/7'), IPv6Network('400::/6'), diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index e4c21daaf3e..9eaa4f3fbc1 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -1,4 +1,4 @@ -r"""JSON (JavaScript Object Notation) is a subset of +r"""JSON (JavaScript Object Notation) is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. @@ -86,13 +86,13 @@ '[2.0, 1.0]' -Using json.tool from the shell to validate and pretty-print:: +Using json from the shell to validate and pretty-print:: - $ echo '{"json":"obj"}' | python -m json.tool + $ echo '{"json":"obj"}' | python -m json { "json": "obj" } - $ echo '{ 1.2:3.4}' | python -m json.tool + $ echo '{ 1.2:3.4}' | python -m json Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ __version__ = '2.0.9' @@ -128,8 +128,9 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the strings written to ``fp`` can - contain non-ASCII characters if they appear in strings contained in - ``obj``. Otherwise, all such characters are escaped in JSON strings. + contain non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in JSON + strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -145,10 +146,11 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -189,9 +191,10 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. - If ``ensure_ascii`` is false, then the return value can contain non-ASCII - characters if they appear in strings contained in ``obj``. Otherwise, all - such characters are escaped in JSON strings. + If ``ensure_ascii`` is false, then the return value can contain + non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in + JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -207,10 +210,11 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -281,11 +285,12 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. @@ -306,11 +311,12 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to diff --git a/Lib/json/__main__.py b/Lib/json/__main__.py new file mode 100644 index 00000000000..1808eaddb62 --- /dev/null +++ b/Lib/json/__main__.py @@ -0,0 +1,20 @@ +"""Command-line tool to validate and pretty-print JSON + +Usage:: + + $ echo '{"json":"obj"}' | python -m json + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m json + Expecting property name enclosed in double quotes: line 1 column 3 (char 2) + +""" +import json.tool + + +if __name__ == '__main__': + try: + json.tool.main() + except BrokenPipeError as exc: + raise SystemExit(exc.errno) diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index 239bacdb0fd..db87724a897 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -41,7 +41,6 @@ def _from_serde(cls, msg, doc, line, col): pos += col return cls(msg, doc, pos) - # Note that this exception is used from _json def __init__(self, msg, doc, pos): lineno = doc.count('\n', 0, pos) + 1 @@ -65,17 +64,18 @@ def __reduce__(self): } +HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS) STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) BACKSLASH = { '"': '"', '\\': '\\', '/': '/', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', } -def _decode_uXXXX(s, pos): - esc = s[pos + 1:pos + 5] - if len(esc) == 4 and esc[1] not in 'xX': +def _decode_uXXXX(s, pos, _m=HEXDIGITS.match): + esc = _m(s, pos + 1) + if esc is not None: try: - return int(esc, 16) + return int(esc.group(), 16) except ValueError: pass msg = "Invalid \\uXXXX escape" @@ -215,10 +215,13 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook, break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + comma_idx = end - 1 end = _w(s, end).end() nextchar = s[end:end + 1] end += 1 if nextchar != '"': + if nextchar == '}': + raise JSONDecodeError("Illegal trailing comma before end of object", s, comma_idx) raise JSONDecodeError( "Expecting property name enclosed in double quotes", s, end - 1) if object_pairs_hook is not None: @@ -255,19 +258,23 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + comma_idx = end - 1 try: if s[end] in _ws: end += 1 if s[end] in _ws: end = _w(s, end + 1).end() + nextchar = s[end:end + 1] except IndexError: pass + if nextchar == ']': + raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx) return values, end class JSONDecoder(object): - """Simple JSON decoder + """Simple JSON decoder Performs the following translations in decoding by default: @@ -304,10 +311,10 @@ def __init__(self, *, object_hook=None, parse_float=None, place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). - ``object_pairs_hook``, if specified will be called with the result of - every JSON object decoded with an ordered list of pairs. The return - value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. + ``object_pairs_hook``, if specified will be called with the result + of every JSON object decoded with an ordered list of pairs. The + return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 21bff2c1a1f..5cf6d64f3ea 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -30,6 +30,7 @@ for i in range(0x20): ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) +del i INFINITY = float('inf') @@ -71,7 +72,7 @@ def replace(match): c_encode_basestring_ascii or py_encode_basestring_ascii) class JSONEncoder(object): - """Extensible JSON encoder for Python data structures. + """Extensible JSON encoder for Python data structures. Supports the following objects and types by default: @@ -107,12 +108,13 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, """Constructor for JSONEncoder, with sensible defaults. If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, float or None. If - skipkeys is True, such items are simply skipped. + encoding of keys that are not str, int, float, bool or None. + If skipkeys is True, such items are simply skipped. - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming non-ASCII characters escaped. If - ensure_ascii is false, the output can contain non-ASCII characters. + If ensure_ascii is true, the output is guaranteed to be str objects + with all incoming non-ASCII and non-printable characters escaped. + If ensure_ascii is false, the output can contain non-ASCII and + non-printable characters. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -133,14 +135,15 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, indent level. An indent level of 0 will only insert newlines. None is the most compact representation. - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. + If specified, separators should be an (item_separator, + key_separator) tuple. The default is (', ', ': ') if *indent* is + ``None`` and (',', ': ') otherwise. To get the most compact JSON + representation, you should specify (',', ':') to eliminate + whitespace. If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. + that can't otherwise be serialized. It should return a JSON + encodable version of the object or raise a ``TypeError``. """ @@ -173,7 +176,7 @@ def default(self, o): else: return list(iterable) # Let the base class default method raise the TypeError - return JSONEncoder.default(self, o) + return super().default(o) """ raise TypeError(f'Object of type {o.__class__.__name__} ' @@ -243,15 +246,18 @@ def floatstr(o, allow_nan=self.allow_nan, return text - if (_one_shot and c_make_encoder is not None - and self.indent is None): + if self.indent is None or isinstance(self.indent, str): + indent = self.indent + else: + indent = ' ' * self.indent + if _one_shot and c_make_encoder is not None: _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, + markers, self.default, _encoder, indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, + markers, self.default, _encoder, indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) @@ -271,9 +277,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _intstr=int.__repr__, ): - if _indent is not None and not isinstance(_indent, str): - _indent = ' ' * _indent - def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' @@ -292,37 +295,40 @@ def _iterencode_list(lst, _current_indent_level): else: newline_indent = None separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: + for i, value in enumerate(lst): + if i: buf = separator - if isinstance(value, str): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, int): - # Subclasses of int/float may override __repr__, but we still - # want to encode them as integers/floats in JSON. One example - # within the standard library is IntEnum. - yield buf + _intstr(value) - elif isinstance(value, float): - # see comment above for int - yield buf + _floatstr(value) - else: - yield buf - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, int): + # Subclasses of int/float may override __repr__, but we still + # want to encode them as integers/floats in JSON. One example + # within the standard library is IntEnum. + yield buf + _intstr(value) + elif isinstance(value, float): + # see comment above for int + yield buf + _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(lst).__name__} item {i}') + raise if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -344,7 +350,6 @@ def _iterencode_dict(dct, _current_indent_level): _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level item_separator = _item_separator + newline_indent - yield newline_indent else: newline_indent = None item_separator = _item_separator @@ -377,33 +382,41 @@ def _iterencode_dict(dct, _current_indent_level): f'not {key.__class__.__name__}') if first: first = False + if newline_indent is not None: + yield newline_indent else: yield item_separator yield _encoder(key) yield _key_separator - if isinstance(value, str): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, int): - # see comment for int/float in _make_iterencode - yield _intstr(value) - elif isinstance(value, float): - # see comment for int/float in _make_iterencode - yield _floatstr(value) - else: - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) + try: + if isinstance(value, str): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, int): + # see comment for int/float in _make_iterencode + yield _intstr(value) + elif isinstance(value, float): + # see comment for int/float in _make_iterencode + yield _floatstr(value) else: - chunks = _iterencode(value, _current_indent_level) - yield from chunks - if newline_indent is not None: + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + yield from chunks + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') + raise + if not first and newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level yield '}' @@ -435,8 +448,14 @@ def _iterencode(o, _current_indent_level): if markerid in markers: raise ValueError("Circular reference detected") markers[markerid] = o - o = _default(o) - yield from _iterencode(o, _current_indent_level) + newobj = _default(o) + try: + yield from _iterencode(newobj, _current_indent_level) + except GeneratorExit: + raise + except BaseException as exc: + exc.add_note(f'when serializing {type(o).__name__} object') + raise if markers is not None: del markers[markerid] return _iterencode diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py index 7a61cfc2d24..090897515fe 100644 --- a/Lib/json/scanner.py +++ b/Lib/json/scanner.py @@ -9,7 +9,7 @@ __all__ = ['make_scanner'] NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', + r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?', (re.VERBOSE | re.MULTILINE | re.DOTALL)) def py_make_scanner(context): diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 0490b8c0be1..1967817add8 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -1,32 +1,54 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 3 (char 2) +"""Command-line tool to validate and pretty-print JSON +See `json.__main__` for a usage example (invocation as +`python -m json.tool` is supported for backwards compatibility). """ import argparse import json +import re import sys -from pathlib import Path +from _colorize import get_theme, can_colorize + + +# The string we are colorizing is valid JSON, +# so we can use a looser but simpler regex to match +# the various parts, most notably strings and numbers, +# where the regex given by the spec is much more complex. +_color_pattern = re.compile(r''' + (?P"(\\.|[^"\\])*")(?=:) | + (?P"(\\.|[^"\\])*") | + (?PNaN|-?Infinity|[0-9\-+.Ee]+) | + (?Ptrue|false) | + (?Pnull) +''', re.VERBOSE) + +_group_to_theme_color = { + "key": "definition", + "string": "string", + "number": "number", + "boolean": "keyword", + "null": "keyword", +} + + +def _colorize_json(json_str, theme): + def _replace_match_callback(match): + for group, color in _group_to_theme_color.items(): + if m := match.group(group): + return f"{theme[color]}{m}{theme.reset}" + return match.group() + + return re.sub(_color_pattern, _replace_match_callback, json_str) def main(): - prog = 'python -m json.tool' description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(prog=prog, description=description) + parser = argparse.ArgumentParser(description=description, color=True) parser.add_argument('infile', nargs='?', - type=argparse.FileType(encoding="utf-8"), help='a JSON file to be validated or pretty-printed', - default=sys.stdin) + default='-') parser.add_argument('outfile', nargs='?', - type=Path, help='write the output of infile to outfile', default=None) parser.add_argument('--sort-keys', action='store_true', default=False, @@ -59,27 +81,41 @@ def main(): dump_args['indent'] = None dump_args['separators'] = ',', ':' - with options.infile as infile: + try: + if options.infile == '-': + infile = sys.stdin + else: + infile = open(options.infile, encoding='utf-8') try: if options.json_lines: objs = (json.loads(line) for line in infile) else: objs = (json.load(infile),) + finally: + if infile is not sys.stdin: + infile.close() - if options.outfile is None: - out = sys.stdout + if options.outfile is None: + outfile = sys.stdout + else: + outfile = open(options.outfile, 'w', encoding='utf-8') + with outfile: + if can_colorize(file=outfile): + t = get_theme(tty_file=outfile).syntax + for obj in objs: + json_str = json.dumps(obj, **dump_args) + outfile.write(_colorize_json(json_str, t)) + outfile.write('\n') else: - out = options.outfile.open('w', encoding='utf-8') - with out as outfile: for obj in objs: json.dump(obj, outfile, **dump_args) outfile.write('\n') - except ValueError as e: - raise SystemExit(e) + except ValueError as e: + raise SystemExit(e) if __name__ == '__main__': try: main() except BrokenPipeError as exc: - sys.exit(exc.errno) + raise SystemExit(exc.errno) diff --git a/Lib/keyword.py b/Lib/keyword.py index cc2b46b7229..e22c837835e 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -56,7 +56,8 @@ softkwlist = [ '_', 'case', - 'match' + 'match', + 'type' ] iskeyword = frozenset(kwlist).__contains__ diff --git a/Lib/linecache.py b/Lib/linecache.py index 97644a8e379..ef3b2d9136b 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,17 +5,13 @@ that name. """ -import functools -import sys -import os -import tokenize - __all__ = ["getline", "clearcache", "checkcache", "lazycache"] # The cache. Maps filenames to either a thunk which will provide source code, # or a tuple (size, mtime, lines, fullname) once loaded. cache = {} +_interactive_cache = {} def clearcache(): @@ -37,10 +33,9 @@ def getlines(filename, module_globals=None): """Get the lines for a Python source file from the cache. Update the cache if it doesn't contain an entry for this file already.""" - if filename in cache: - entry = cache[filename] - if len(entry) != 1: - return cache[filename][2] + entry = cache.get(filename, None) + if entry is not None and len(entry) != 1: + return entry[2] try: return updatecache(filename, module_globals) @@ -49,28 +44,59 @@ def getlines(filename, module_globals=None): return [] +def _getline_from_code(filename, lineno): + lines = _getlines_from_code(filename) + if 1 <= lineno <= len(lines): + return lines[lineno - 1] + return '' + +def _make_key(code): + return (code.co_filename, code.co_qualname, code.co_firstlineno) + +def _getlines_from_code(code): + code_id = _make_key(code) + entry = _interactive_cache.get(code_id, None) + if entry is not None and len(entry) != 1: + return entry[2] + return [] + + +def _source_unavailable(filename): + """Return True if the source code is unavailable for such file name.""" + return ( + not filename + or (filename.startswith('<') + and filename.endswith('>') + and not filename.startswith('')): + # These imports are not at top level because linecache is in the critical + # path of the interpreter startup and importing os and sys take a lot of time + # and slows down the startup sequence. + try: + import os + import sys + import tokenize + except ImportError: + # These import can fail if the interpreter is shutting down return [] - fullname = filename + entry = cache.pop(filename, None) + if _source_unavailable(filename): + return [] + + if filename.startswith('')): - return False + return None # Try for a __loader__, if available if module_globals and '__name__' in module_globals: - name = module_globals['__name__'] - if (loader := module_globals.get('__loader__')) is None: - if spec := module_globals.get('__spec__'): - try: - loader = spec.loader - except AttributeError: - pass + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + loader = getattr(spec, 'loader', None) + if loader is None: + loader = module_globals.get('__loader__') get_source = getattr(loader, 'get_source', None) if name and get_source: - get_lines = functools.partial(get_source, name) - cache[filename] = (get_lines,) - return True - return False + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) + return (get_lines,) + return None + + + +def _register_code(code, string, name): + entry = (len(string), + None, + [line + '\n' for line in string.splitlines()], + name) + stack = [code] + while stack: + code = stack.pop() + for const in code.co_consts: + if isinstance(const, type(code)): + stack.append(const) + key = _make_key(code) + _interactive_cache[key] = entry diff --git a/Lib/locale.py b/Lib/locale.py index 7a7694e1bfb..dfedc6386cb 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -13,7 +13,6 @@ import sys import encodings import encodings.aliases -import re import _collections_abc from builtins import str as _builtin_str import functools @@ -25,8 +24,8 @@ # Yuck: LC_MESSAGES is non-standard: can't tell whether it exists before # trying the import. So __all__ is also fiddled at the end of the file. __all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error", - "setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm", - "str", "atof", "atoi", "format", "format_string", "currency", + "setlocale", "localeconv", "strcoll", "strxfrm", + "str", "atof", "atoi", "format_string", "currency", "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY", "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding"] @@ -177,8 +176,7 @@ def _strip_padding(s, amount): amount -= 1 return s[lpos:rpos+1] -_percent_re = re.compile(r'%(?:\((?P.*?)\))?' - r'(?P[-#0-9 +*.hlL]*?)[eEfFgGdiouxXcrs%]') +_percent_re = None def _format(percent, value, grouping=False, monetary=False, *additional): if additional: @@ -217,6 +215,13 @@ def format_string(f, val, grouping=False, monetary=False): Grouping is applied if the third parameter is true. Conversion uses monetary thousands separator and grouping strings if forth parameter monetary is true.""" + global _percent_re + if _percent_re is None: + import re + + _percent_re = re.compile(r'%(?:\((?P.*?)\))?(?P[-#0-9 +*.hlL]*?)[eEfFgGdiouxXcrs%]') + percents = list(_percent_re.finditer(f)) new_f = _percent_re.sub('%s', f) @@ -247,21 +252,6 @@ def format_string(f, val, grouping=False, monetary=False): return new_f % val -def format(percent, value, grouping=False, monetary=False, *additional): - """Deprecated, use format_string instead.""" - import warnings - warnings.warn( - "This method will be removed in a future version of Python. " - "Use 'locale.format_string()' instead.", - DeprecationWarning, stacklevel=2 - ) - - match = _percent_re.match(percent) - if not match or len(match.group())!= len(percent): - raise ValueError(("format() must be given exactly one %%char " - "format specifier, %s not valid") % repr(percent)) - return _format(percent, value, grouping, monetary, *additional) - def currency(val, symbol=True, grouping=False, international=False): """Formats val according to the currency settings in the current locale.""" @@ -556,11 +546,15 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): """ import warnings - warnings.warn( - "Use setlocale(), getencoding() and getlocale() instead", - DeprecationWarning, stacklevel=2 - ) + warnings._deprecated( + "locale.getdefaultlocale", + "{name!r} is deprecated and slated for removal in Python {remove}. " + "Use setlocale(), getencoding() and getlocale() instead.", + remove=(3, 15)) + return _getdefaultlocale(envvars) + +def _getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): try: # check if it's supported by the _locale module import _locale @@ -625,40 +619,15 @@ def setlocale(category, locale=None): locale = normalize(_build_localename(locale)) return _setlocale(category, locale) -def resetlocale(category=LC_ALL): - - """ Sets the locale for category to the default setting. - - The default setting is determined by calling - getdefaultlocale(). category defaults to LC_ALL. - - """ - import warnings - warnings.warn( - 'Use locale.setlocale(locale.LC_ALL, "") instead', - DeprecationWarning, stacklevel=2 - ) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - loc = getdefaultlocale() - - _setlocale(category, _build_localename(loc)) - try: from _locale import getencoding except ImportError: + # When _locale.getencoding() is missing, locale.getencoding() uses the + # Python filesystem encoding. def getencoding(): - if hasattr(sys, 'getandroidapilevel'): - # On Android langinfo.h and CODESET are missing, and UTF-8 is - # always used in mbstowcs() and wcstombs(). - return 'utf-8' - encoding = getdefaultlocale()[1] - if encoding is None: - # LANG not set, default to UTF-8 - encoding = 'utf-8' - return encoding + return sys.getfilesystemencoding() + try: CODESET @@ -896,6 +865,28 @@ def getpreferredencoding(do_setlocale=True): # updated 'ca_es@valencia' -> 'ca_ES.ISO8859-15@valencia' to 'ca_ES.UTF-8@valencia' # updated 'kk_kz' -> 'kk_KZ.RK1048' to 'kk_KZ.ptcp154' # updated 'russian' -> 'ru_RU.ISO8859-5' to 'ru_RU.KOI8-R' +# +# SS 2025-02-04: +# Updated alias mapping with glibc 2.41 supported locales and the latest +# X lib alias mapping. +# +# These are the differences compared to the old mapping (Python 3.13.1 +# and older): +# +# updated 'c.utf8' -> 'C.UTF-8' to 'en_US.UTF-8' +# updated 'de_it' -> 'de_IT.ISO8859-1' to 'de_IT.UTF-8' +# removed 'de_li.utf8' +# updated 'en_il' -> 'en_IL.UTF-8' to 'en_IL.ISO8859-1' +# removed 'english.iso88591' +# updated 'es_cu' -> 'es_CU.UTF-8' to 'es_CU.ISO8859-1' +# updated 'russian' -> 'ru_RU.KOI8-R' to 'ru_RU.ISO8859-5' +# updated 'sr@latn' -> 'sr_CS.UTF-8@latin' to 'sr_RS.UTF-8@latin' +# removed 'univ' +# removed 'universal' +# +# SS 2025-06-10: +# Remove 'c.utf8' -> 'en_US.UTF-8' because 'en_US.UTF-8' does not exist +# on all platforms. locale_alias = { 'a3': 'az_AZ.KOI8-C', @@ -975,7 +966,6 @@ def getpreferredencoding(do_setlocale=True): 'c.ascii': 'C', 'c.en': 'C', 'c.iso88591': 'en_US.ISO8859-1', - 'c.utf8': 'en_US.UTF-8', 'c_c': 'C', 'c_c.c': 'C', 'ca': 'ca_ES.ISO8859-1', @@ -992,6 +982,7 @@ def getpreferredencoding(do_setlocale=True): 'chr_us': 'chr_US.UTF-8', 'ckb_iq': 'ckb_IQ.UTF-8', 'cmn_tw': 'cmn_TW.UTF-8', + 'crh_ru': 'crh_RU.UTF-8', 'crh_ua': 'crh_UA.UTF-8', 'croatian': 'hr_HR.ISO8859-2', 'cs': 'cs_CZ.ISO8859-2', @@ -1013,11 +1004,12 @@ def getpreferredencoding(do_setlocale=True): 'de_be': 'de_BE.ISO8859-1', 'de_ch': 'de_CH.ISO8859-1', 'de_de': 'de_DE.ISO8859-1', - 'de_it': 'de_IT.ISO8859-1', - 'de_li.utf8': 'de_LI.UTF-8', + 'de_it': 'de_IT.UTF-8', + 'de_li': 'de_LI.ISO8859-1', 'de_lu': 'de_LU.ISO8859-1', 'deutsch': 'de_DE.ISO8859-1', 'doi_in': 'doi_IN.UTF-8', + 'dsb_de': 'dsb_DE.UTF-8', 'dutch': 'nl_NL.ISO8859-1', 'dutch.iso88591': 'nl_BE.ISO8859-1', 'dv_mv': 'dv_MV.UTF-8', @@ -1040,7 +1032,7 @@ def getpreferredencoding(do_setlocale=True): 'en_gb': 'en_GB.ISO8859-1', 'en_hk': 'en_HK.ISO8859-1', 'en_ie': 'en_IE.ISO8859-1', - 'en_il': 'en_IL.UTF-8', + 'en_il': 'en_IL.ISO8859-1', 'en_in': 'en_IN.ISO8859-1', 'en_ng': 'en_NG.UTF-8', 'en_nz': 'en_NZ.ISO8859-1', @@ -1056,7 +1048,6 @@ def getpreferredencoding(do_setlocale=True): 'en_zw.utf8': 'en_ZS.UTF-8', 'eng_gb': 'en_GB.ISO8859-1', 'english': 'en_EN.ISO8859-1', - 'english.iso88591': 'en_US.ISO8859-1', 'english_uk': 'en_GB.ISO8859-1', 'english_united-states': 'en_US.ISO8859-1', 'english_united-states.437': 'C', @@ -1072,7 +1063,7 @@ def getpreferredencoding(do_setlocale=True): 'es_cl': 'es_CL.ISO8859-1', 'es_co': 'es_CO.ISO8859-1', 'es_cr': 'es_CR.ISO8859-1', - 'es_cu': 'es_CU.UTF-8', + 'es_cu': 'es_CU.ISO8859-1', 'es_do': 'es_DO.ISO8859-1', 'es_ec': 'es_EC.ISO8859-1', 'es_es': 'es_ES.ISO8859-1', @@ -1122,6 +1113,7 @@ def getpreferredencoding(do_setlocale=True): 'ga_ie': 'ga_IE.ISO8859-1', 'galego': 'gl_ES.ISO8859-1', 'galician': 'gl_ES.ISO8859-1', + 'gbm_in': 'gbm_IN.UTF-8', 'gd': 'gd_GB.ISO8859-1', 'gd_gb': 'gd_GB.ISO8859-1', 'ger_de': 'de_DE.ISO8859-1', @@ -1162,6 +1154,7 @@ def getpreferredencoding(do_setlocale=True): 'icelandic': 'is_IS.ISO8859-1', 'id': 'id_ID.ISO8859-1', 'id_id': 'id_ID.ISO8859-1', + 'ie': 'ie.UTF-8', 'ig_ng': 'ig_NG.UTF-8', 'ik_ca': 'ik_CA.UTF-8', 'in': 'id_ID.ISO8859-1', @@ -1216,6 +1209,7 @@ def getpreferredencoding(do_setlocale=True): 'ks_in': 'ks_IN.UTF-8', 'ks_in@devanagari.utf8': 'ks_IN.UTF-8@devanagari', 'ku_tr': 'ku_TR.ISO8859-9', + 'kv_ru': 'kv_RU.UTF-8', 'kw': 'kw_GB.ISO8859-1', 'kw_gb': 'kw_GB.ISO8859-1', 'ky': 'ky_KG.UTF-8', @@ -1234,6 +1228,7 @@ def getpreferredencoding(do_setlocale=True): 'lo_la.mulelao1': 'lo_LA.MULELAO-1', 'lt': 'lt_LT.ISO8859-13', 'lt_lt': 'lt_LT.ISO8859-13', + 'ltg_lv.utf8': 'ltg_LV.UTF-8', 'lv': 'lv_LV.ISO8859-13', 'lv_lv': 'lv_LV.ISO8859-13', 'lzh_tw': 'lzh_TW.UTF-8', @@ -1241,6 +1236,7 @@ def getpreferredencoding(do_setlocale=True): 'mai': 'mai_IN.UTF-8', 'mai_in': 'mai_IN.UTF-8', 'mai_np': 'mai_NP.UTF-8', + 'mdf_ru': 'mdf_RU.UTF-8', 'mfe_mu': 'mfe_MU.UTF-8', 'mg_mg': 'mg_MG.ISO8859-15', 'mhr_ru': 'mhr_RU.UTF-8', @@ -1254,6 +1250,7 @@ def getpreferredencoding(do_setlocale=True): 'ml_in': 'ml_IN.UTF-8', 'mn_mn': 'mn_MN.UTF-8', 'mni_in': 'mni_IN.UTF-8', + 'mnw_mm': 'mnw_MM.UTF-8', 'mr': 'mr_IN.UTF-8', 'mr_in': 'mr_IN.UTF-8', 'ms': 'ms_MY.ISO8859-1', @@ -1322,6 +1319,7 @@ def getpreferredencoding(do_setlocale=True): 'pt_pt': 'pt_PT.ISO8859-1', 'quz_pe': 'quz_PE.UTF-8', 'raj_in': 'raj_IN.UTF-8', + 'rif_ma': 'rif_MA.UTF-8', 'ro': 'ro_RO.ISO8859-2', 'ro_ro': 'ro_RO.ISO8859-2', 'romanian': 'ro_RO.ISO8859-2', @@ -1329,12 +1327,14 @@ def getpreferredencoding(do_setlocale=True): 'ru_ru': 'ru_RU.UTF-8', 'ru_ua': 'ru_UA.KOI8-U', 'rumanian': 'ro_RO.ISO8859-2', - 'russian': 'ru_RU.KOI8-R', + 'russian': 'ru_RU.ISO8859-5', 'rw': 'rw_RW.ISO8859-1', 'rw_rw': 'rw_RW.ISO8859-1', 'sa_in': 'sa_IN.UTF-8', + 'sah_ru': 'sah_RU.UTF-8', 'sat_in': 'sat_IN.UTF-8', 'sc_it': 'sc_IT.UTF-8', + 'scn_it': 'scn_IT.UTF-8', 'sd': 'sd_IN.UTF-8', 'sd_in': 'sd_IN.UTF-8', 'sd_in@devanagari.utf8': 'sd_IN.UTF-8@devanagari', @@ -1376,7 +1376,7 @@ def getpreferredencoding(do_setlocale=True): 'sq_mk': 'sq_MK.UTF-8', 'sr': 'sr_RS.UTF-8', 'sr@cyrillic': 'sr_RS.UTF-8', - 'sr@latn': 'sr_CS.UTF-8@latin', + 'sr@latn': 'sr_RS.UTF-8@latin', 'sr_cs': 'sr_CS.UTF-8', 'sr_cs.iso88592@latn': 'sr_CS.ISO8859-2', 'sr_cs@latn': 'sr_CS.UTF-8@latin', @@ -1395,14 +1395,17 @@ def getpreferredencoding(do_setlocale=True): 'sr_yu@cyrillic': 'sr_RS.UTF-8', 'ss': 'ss_ZA.ISO8859-1', 'ss_za': 'ss_ZA.ISO8859-1', + 'ssy_er': 'ssy_ER.UTF-8', 'st': 'st_ZA.ISO8859-1', 'st_za': 'st_ZA.ISO8859-1', + 'su_id': 'su_ID.UTF-8', 'sv': 'sv_SE.ISO8859-1', 'sv_fi': 'sv_FI.ISO8859-1', 'sv_se': 'sv_SE.ISO8859-1', 'sw_ke': 'sw_KE.UTF-8', 'sw_tz': 'sw_TZ.UTF-8', 'swedish': 'sv_SE.ISO8859-1', + 'syr': 'syr.UTF-8', 'szl_pl': 'szl_PL.UTF-8', 'ta': 'ta_IN.TSCII-0', 'ta_in': 'ta_IN.TSCII-0', @@ -1429,6 +1432,7 @@ def getpreferredencoding(do_setlocale=True): 'tn': 'tn_ZA.ISO8859-15', 'tn_za': 'tn_ZA.ISO8859-15', 'to_to': 'to_TO.UTF-8', + 'tok': 'tok.UTF-8', 'tpi_pg': 'tpi_PG.UTF-8', 'tr': 'tr_TR.ISO8859-9', 'tr_cy': 'tr_CY.ISO8859-9', @@ -1443,8 +1447,7 @@ def getpreferredencoding(do_setlocale=True): 'ug_cn': 'ug_CN.UTF-8', 'uk': 'uk_UA.KOI8-U', 'uk_ua': 'uk_UA.KOI8-U', - 'univ': 'en_US.utf', - 'universal': 'en_US.utf', + 'univ.utf8': 'en_US.UTF-8', 'universal.utf8@ucs4': 'en_US.UTF-8', 'unm_us': 'unm_US.UTF-8', 'ur': 'ur_PK.CP1256', @@ -1473,6 +1476,7 @@ def getpreferredencoding(do_setlocale=True): 'yo_ng': 'yo_NG.UTF-8', 'yue_hk': 'yue_HK.UTF-8', 'yuw_pg': 'yuw_PG.UTF-8', + 'zgh_ma': 'zgh_MA.UTF-8', 'zh': 'zh_CN.eucCN', 'zh_cn': 'zh_CN.gb2312', 'zh_cn.big5': 'zh_TW.big5', @@ -1496,7 +1500,8 @@ def getpreferredencoding(do_setlocale=True): # to include every locale up to Windows Vista. # # NOTE: this mapping is incomplete. If your language is missing, please -# submit a bug report to the Python bug tracker at http://bugs.python.org/ +# submit a bug report as detailed in the Python devguide at: +# https://devguide.python.org/triage/issue-tracker/ # Make sure you include the missing language identifier and the suggested # locale code. # @@ -1742,17 +1747,6 @@ def _init_categories(categories=categories): print(' Encoding: ', enc or '(undefined)') print() - print() - print('Locale settings after calling resetlocale():') - print('-'*72) - resetlocale() - for name,category in categories.items(): - print(name, '...') - lang, enc = getlocale(category) - print(' Language: ', lang or '(undefined)') - print(' Encoding: ', enc or '(undefined)') - print() - try: setlocale(LC_ALL, "") except: diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 19bd2bc20b2..357d127c090 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,13 +18,14 @@ Logging package for Python. Based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ import sys, os, time, io, re, traceback, warnings, weakref, collections.abc +from types import GenericAlias from string import Template from string import Formatter as StrFormatter @@ -37,7 +38,8 @@ 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', - 'lastResort', 'raiseExceptions'] + 'lastResort', 'raiseExceptions', 'getLevelNamesMapping', + 'getHandlerByName', 'getHandlerNames'] import threading @@ -54,7 +56,7 @@ # #_startTime is used as the base when calculating the relative time of events # -_startTime = time.time() +_startTime = time.time_ns() # #raiseExceptions is used to see if exceptions during handling should be @@ -63,20 +65,25 @@ raiseExceptions = True # -# If you don't want threading information in the log, set this to zero +# If you don't want threading information in the log, set this to False # logThreads = True # -# If you don't want multiprocessing information in the log, set this to zero +# If you don't want multiprocessing information in the log, set this to False # logMultiprocessing = True # -# If you don't want process information in the log, set this to zero +# If you don't want process information in the log, set this to False # logProcesses = True +# +# If you don't want asyncio task information in the log, set this to False +# +logAsyncioTasks = True + #--------------------------------------------------------------------------- # Level related stuff #--------------------------------------------------------------------------- @@ -116,6 +123,9 @@ 'NOTSET': NOTSET, } +def getLevelNamesMapping(): + return _nameToLevel.copy() + def getLevelName(level): """ Return the textual or numeric representation of logging level 'level'. @@ -149,22 +159,19 @@ def addLevelName(level, levelName): This is used when converting levels to text during message formatting. """ - _acquireLock() - try: #unlikely to cause an exception, but you never know... + with _lock: _levelToName[level] = levelName _nameToLevel[levelName] = level - finally: - _releaseLock() -if hasattr(sys, '_getframe'): - currentframe = lambda: sys._getframe(3) +if hasattr(sys, "_getframe"): + currentframe = lambda: sys._getframe(1) else: #pragma: no cover def currentframe(): """Return the frame object for the caller's stack frame.""" try: raise Exception - except Exception: - return sys.exc_info()[2].tb_frame.f_back + except Exception as exc: + return exc.__traceback__.tb_frame.f_back # # _srcfile is used when walking the stack to check when we've got the first @@ -181,13 +188,18 @@ def currentframe(): _srcfile = os.path.normcase(addLevelName.__code__.co_filename) # _srcfile is only used in conjunction with sys._getframe(). -# To provide compatibility with older versions of Python, set _srcfile -# to None if _getframe() is not available; this value will prevent -# findCaller() from being called. You can also do this if you want to avoid -# the overhead of fetching caller information, even when _getframe() is -# available. -#if not hasattr(sys, '_getframe'): -# _srcfile = None +# Setting _srcfile to None will prevent findCaller() from being called. This +# way, you can avoid the overhead of fetching caller information. + +# The following is based on warnings._is_internal_frame. It makes sure that +# frames of the import mechanism are skipped when logging at module level and +# using a stacklevel value greater than one. +def _is_internal_frame(frame): + """Signal whether the frame is a CPython or logging module internal.""" + filename = os.path.normcase(frame.f_code.co_filename) + return filename == _srcfile or ( + "importlib" in filename and "_bootstrap" in filename + ) def _checkLevel(level): @@ -216,21 +228,27 @@ def _checkLevel(level): # _lock = threading.RLock() -def _acquireLock(): +def _prepareFork(): """ - Acquire the module-level lock for serializing access to shared data. + Prepare to fork a new child process by acquiring the module-level lock. - This should be released with _releaseLock(). + This should be used in conjunction with _afterFork(). """ - if _lock: + # Wrap the lock acquisition in a try-except to prevent the lock from being + # abandoned in the event of an asynchronous exception. See gh-106238. + try: _lock.acquire() + except BaseException: + _lock.release() + raise -def _releaseLock(): +def _afterFork(): """ - Release the module-level lock acquired by calling _acquireLock(). + After a new child process has been forked, release the module-level lock. + + This should be used in conjunction with _prepareFork(). """ - if _lock: - _lock.release() + _lock.release() # Prevent a held logging lock from blocking a child from logging. @@ -245,23 +263,20 @@ def _register_at_fork_reinit_lock(instance): _at_fork_reinit_lock_weakset = weakref.WeakSet() def _register_at_fork_reinit_lock(instance): - _acquireLock() - try: + with _lock: _at_fork_reinit_lock_weakset.add(instance) - finally: - _releaseLock() def _after_at_fork_child_reinit_locks(): for handler in _at_fork_reinit_lock_weakset: handler._at_fork_reinit() - # _acquireLock() was called in the parent before forking. + # _prepareFork() was called in the parent before forking. # The lock is reinitialized to unlocked state. _lock._at_fork_reinit() - os.register_at_fork(before=_acquireLock, + os.register_at_fork(before=_prepareFork, after_in_child=_after_at_fork_child_reinit_locks, - after_in_parent=_releaseLock) + after_in_parent=_afterFork) #--------------------------------------------------------------------------- @@ -285,7 +300,7 @@ def __init__(self, name, level, pathname, lineno, """ Initialize a logging record with interesting information. """ - ct = time.time() + ct = time.time_ns() self.name = name self.msg = msg # @@ -324,9 +339,17 @@ def __init__(self, name, level, pathname, lineno, self.stack_info = sinfo self.lineno = lineno self.funcName = func - self.created = ct - self.msecs = (ct - int(ct)) * 1000 - self.relativeCreated = (self.created - _startTime) * 1000 + self.created = ct / 1e9 # ns to float seconds + # Get the number of whole milliseconds (0-999) in the fractional part of seconds. + # Eg: 1_677_903_920_999_998_503 ns --> 999_998_503 ns--> 999 ms + # Convert to float by adding 0.0 for historical reasons. See gh-89047 + self.msecs = (ct % 1_000_000_000) // 1_000_000 + 0.0 + if self.msecs == 999.0 and int(self.created) != ct // 1_000_000_000: + # ns -> sec conversion can round up, e.g: + # 1_677_903_920_999_999_900 ns --> 1_677_903_921.0 sec + self.msecs = 0.0 + + self.relativeCreated = (ct - _startTime) / 1e6 if logThreads: self.thread = threading.get_ident() self.threadName = threading.current_thread().name @@ -352,6 +375,15 @@ def __init__(self, name, level, pathname, lineno, else: self.process = None + self.taskName = None + if logAsyncioTasks: + asyncio = sys.modules.get('asyncio') + if asyncio: + try: + self.taskName = asyncio.current_task().get_name() + except Exception: + pass + def __repr__(self): return ''%(self.name, self.levelno, self.pathname, self.lineno, self.msg) @@ -487,7 +519,7 @@ def __init__(self, *args, **kwargs): def usesTime(self): fmt = self._fmt - return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0 + return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_search) >= 0 def validate(self): pattern = Template.pattern @@ -548,7 +580,7 @@ class Formatter(object): %(lineno)d Source line number where the logging call was issued (if available) %(funcName)s Function name - %(created)f Time when the LogRecord was created (time.time() + %(created)f Time when the LogRecord was created (time.time_ns() / 1e9 return value) %(asctime)s Textual time when the LogRecord was created %(msecs)d Millisecond portion of the creation time @@ -557,7 +589,9 @@ class Formatter(object): (typically at application startup time) %(thread)d Thread ID (if available) %(threadName)s Thread name (if available) + %(taskName)s Task name (if available) %(process)d Process ID (if available) + %(processName)s Process name (if available) %(message)s The result of record.getMessage(), computed just as the record is emitted """ @@ -633,7 +667,7 @@ def formatException(self, ei): # See issues #9427, #1553375. Commented out for now. #if getattr(self, 'fullstack', False): # traceback.print_stack(tb.tb_frame.f_back, file=sio) - traceback.print_exception(ei[0], ei[1], tb, None, sio) + traceback.print_exception(ei[0], ei[1], tb, limit=None, file=sio) s = sio.getvalue() sio.close() if s[-1:] == "\n": @@ -808,23 +842,36 @@ def filter(self, record): Determine if a record is loggable by consulting all the filters. The default is to allow the record to be logged; any filter can veto - this and the record is then dropped. Returns a zero value if a record - is to be dropped, else non-zero. + this by returning a false value. + If a filter attached to a handler returns a log record instance, + then that instance is used in place of the original log record in + any further processing of the event by that handler. + If a filter returns any other true value, the original log record + is used in any further processing of the event by that handler. + + If none of the filters return false values, this method returns + a log record. + If any of the filters return a false value, this method returns + a false value. .. versionchanged:: 3.2 Allow filters to be just callables. + + .. versionchanged:: 3.12 + Allow filters to return a LogRecord instead of + modifying it in place. """ - rv = True for f in self.filters: if hasattr(f, 'filter'): result = f.filter(record) else: result = f(record) # assume callable - will raise if not if not result: - rv = False - break - return rv + return False + if isinstance(result, LogRecord): + record = result + return record #--------------------------------------------------------------------------- # Handler classes and functions @@ -841,24 +888,36 @@ def _removeHandlerRef(wr): # set to None. It can also be called from another thread. So we need to # pre-emptively grab the necessary globals and check if they're None, # to prevent race conditions and failures during interpreter shutdown. - acquire, release, handlers = _acquireLock, _releaseLock, _handlerList - if acquire and release and handlers: - acquire() - try: - if wr in handlers: + handlers, lock = _handlerList, _lock + if lock and handlers: + with lock: + try: handlers.remove(wr) - finally: - release() + except ValueError: + pass def _addHandlerRef(handler): """ Add a handler to the internal cleanup list using a weak reference. """ - _acquireLock() - try: + with _lock: _handlerList.append(weakref.ref(handler, _removeHandlerRef)) - finally: - _releaseLock() + + +def getHandlerByName(name): + """ + Get a handler with the specified *name*, or None if there isn't one with + that name. + """ + return _handlers.get(name) + + +def getHandlerNames(): + """ + Return all known handler names as an immutable set. + """ + return frozenset(_handlers) + class Handler(Filterer): """ @@ -887,15 +946,12 @@ def get_name(self): return self._name def set_name(self, name): - _acquireLock() - try: + with _lock: if self._name in _handlers: del _handlers[self._name] self._name = name if name: _handlers[name] = self - finally: - _releaseLock() name = property(get_name, set_name) @@ -958,16 +1014,17 @@ def handle(self, record): Emission depends on filters which may have been added to the handler. Wrap the actual emission of the record with acquisition/release of - the I/O thread lock. Returns whether the filter passed the record for - emission. + the I/O thread lock. + + Returns an instance of the log record that was emitted + if it passed all filters, otherwise a false value is returned. """ rv = self.filter(record) + if isinstance(rv, LogRecord): + record = rv if rv: - self.acquire() - try: + with self.lock: self.emit(record) - finally: - self.release() return rv def setFormatter(self, fmt): @@ -995,13 +1052,10 @@ def close(self): methods. """ #get the module data lock, as we're updating a shared structure. - _acquireLock() - try: #unlikely to raise an exception, but you never know... + with _lock: self._closed = True if self._name and self._name in _handlers: del _handlers[self._name] - finally: - _releaseLock() def handleError(self, record): """ @@ -1016,14 +1070,14 @@ def handleError(self, record): The record which was being processed is passed in to this method. """ if raiseExceptions and sys.stderr: # see issue 13807 - t, v, tb = sys.exc_info() + exc = sys.exception() try: sys.stderr.write('--- Logging error ---\n') - traceback.print_exception(t, v, tb, None, sys.stderr) + traceback.print_exception(exc, limit=None, file=sys.stderr) sys.stderr.write('Call stack:\n') # Walk the stack frame up until we're out of logging, # so as to print the calling context. - frame = tb.tb_frame + frame = exc.__traceback__.tb_frame while (frame and os.path.dirname(frame.f_code.co_filename) == __path__[0]): frame = frame.f_back @@ -1048,7 +1102,7 @@ def handleError(self, record): except OSError: #pragma: no cover pass # see issue 5971 finally: - del t, v, tb + del exc def __repr__(self): level = getLevelName(self.level) @@ -1078,12 +1132,9 @@ def flush(self): """ Flushes the stream. """ - self.acquire() - try: + with self.lock: if self.stream and hasattr(self.stream, "flush"): self.stream.flush() - finally: - self.release() def emit(self, record): """ @@ -1119,12 +1170,9 @@ def setStream(self, stream): result = None else: result = self.stream - self.acquire() - try: + with self.lock: self.flush() self.stream = stream - finally: - self.release() return result def __repr__(self): @@ -1136,6 +1184,8 @@ def __repr__(self): name += ' ' return '<%s %s(%s)>' % (self.__class__.__name__, name, level) + __class_getitem__ = classmethod(GenericAlias) + class FileHandler(StreamHandler): """ @@ -1172,8 +1222,7 @@ def close(self): """ Closes the stream. """ - self.acquire() - try: + with self.lock: try: if self.stream: try: @@ -1189,8 +1238,6 @@ def close(self): # Also see Issue #42378: we also rely on # self._closed being set to True there StreamHandler.close(self) - finally: - self.release() def _open(self): """ @@ -1326,8 +1373,7 @@ def getLogger(self, name): rv = None if not isinstance(name, str): raise TypeError('A logger name must be a string') - _acquireLock() - try: + with _lock: if name in self.loggerDict: rv = self.loggerDict[name] if isinstance(rv, PlaceHolder): @@ -1342,8 +1388,6 @@ def getLogger(self, name): rv.manager = self self.loggerDict[name] = rv self._fixupParents(rv) - finally: - _releaseLock() return rv def setLoggerClass(self, klass): @@ -1406,12 +1450,11 @@ def _clear_cache(self): Called when level changes are made """ - _acquireLock() - for logger in self.loggerDict.values(): - if isinstance(logger, Logger): - logger._cache.clear() - self.root._cache.clear() - _releaseLock() + with _lock: + for logger in self.loggerDict.values(): + if isinstance(logger, Logger): + logger._cache.clear() + self.root._cache.clear() #--------------------------------------------------------------------------- # Logger classes and functions @@ -1432,6 +1475,8 @@ class Logger(Filterer): level, and "input.csv", "input.xls" and "input.gnu" for the sub-levels. There is no arbitrary limit to the depth of nesting. """ + _tls = threading.local() + def __init__(self, name, level=NOTSET): """ Initialize the logger with a name and an optional level. @@ -1459,7 +1504,7 @@ def debug(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.debug("Houston, we have a %s", "thorny problem", exc_info=1) + logger.debug("Houston, we have a %s", "thorny problem", exc_info=True) """ if self.isEnabledFor(DEBUG): self._log(DEBUG, msg, args, **kwargs) @@ -1471,7 +1516,7 @@ def info(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.info("Houston, we have a %s", "interesting problem", exc_info=1) + logger.info("Houston, we have a %s", "notable problem", exc_info=True) """ if self.isEnabledFor(INFO): self._log(INFO, msg, args, **kwargs) @@ -1483,7 +1528,7 @@ def warning(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1) + logger.warning("Houston, we have a %s", "bit of a problem", exc_info=True) """ if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) @@ -1500,7 +1545,7 @@ def error(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.error("Houston, we have a %s", "major problem", exc_info=1) + logger.error("Houston, we have a %s", "major problem", exc_info=True) """ if self.isEnabledFor(ERROR): self._log(ERROR, msg, args, **kwargs) @@ -1518,7 +1563,7 @@ def critical(self, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.critical("Houston, we have a %s", "major disaster", exc_info=1) + logger.critical("Houston, we have a %s", "major disaster", exc_info=True) """ if self.isEnabledFor(CRITICAL): self._log(CRITICAL, msg, args, **kwargs) @@ -1536,7 +1581,7 @@ def log(self, level, msg, *args, **kwargs): To pass exception information, use the keyword argument exc_info with a true value, e.g. - logger.log(level, "We have a %s", "mysterious problem", exc_info=1) + logger.log(level, "We have a %s", "mysterious problem", exc_info=True) """ if not isinstance(level, int): if raiseExceptions: @@ -1554,33 +1599,31 @@ def findCaller(self, stack_info=False, stacklevel=1): f = currentframe() #On some versions of IronPython, currentframe() returns None if #IronPython isn't run with -X:Frames. - if f is not None: - f = f.f_back - orig_f = f - while f and stacklevel > 1: - f = f.f_back - stacklevel -= 1 - if not f: - f = orig_f - rv = "(unknown file)", 0, "(unknown function)", None - while hasattr(f, "f_code"): - co = f.f_code - filename = os.path.normcase(co.co_filename) - if filename == _srcfile: - f = f.f_back - continue - sinfo = None - if stack_info: - sio = io.StringIO() - sio.write('Stack (most recent call last):\n') + if f is None: + return "(unknown file)", 0, "(unknown function)", None + while stacklevel > 0: + next_f = f.f_back + if next_f is None: + ## We've got options here. + ## If we want to use the last (deepest) frame: + break + ## If we want to mimic the warnings module: + #return ("sys", 1, "(unknown function)", None) + ## If we want to be pedantic: + #raise ValueError("call stack is not deep enough") + f = next_f + if not _is_internal_frame(f): + stacklevel -= 1 + co = f.f_code + sinfo = None + if stack_info: + with io.StringIO() as sio: + sio.write("Stack (most recent call last):\n") traceback.print_stack(f, file=sio) sinfo = sio.getvalue() if sinfo[-1] == '\n': sinfo = sinfo[:-1] - sio.close() - rv = (co.co_filename, f.f_lineno, co.co_name, sinfo) - break - return rv + return co.co_filename, f.f_lineno, co.co_name, sinfo def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None): @@ -1630,30 +1673,35 @@ def handle(self, record): This method is used for unpickled records received from a socket, as well as those created locally. Logger-level filtering is applied. """ - if (not self.disabled) and self.filter(record): + if self._is_disabled(): + return + + self._tls.in_progress = True + try: + maybe_record = self.filter(record) + if not maybe_record: + return + if isinstance(maybe_record, LogRecord): + record = maybe_record self.callHandlers(record) + finally: + self._tls.in_progress = False def addHandler(self, hdlr): """ Add the specified handler to this logger. """ - _acquireLock() - try: + with _lock: if not (hdlr in self.handlers): self.handlers.append(hdlr) - finally: - _releaseLock() def removeHandler(self, hdlr): """ Remove the specified handler from this logger. """ - _acquireLock() - try: + with _lock: if hdlr in self.handlers: self.handlers.remove(hdlr) - finally: - _releaseLock() def hasHandlers(self): """ @@ -1725,22 +1773,19 @@ def isEnabledFor(self, level): """ Is this logger enabled for level 'level'? """ - if self.disabled: + if self._is_disabled(): return False try: return self._cache[level] except KeyError: - _acquireLock() - try: + with _lock: if self.manager.disable >= level: is_enabled = self._cache[level] = False else: is_enabled = self._cache[level] = ( level >= self.getEffectiveLevel() ) - finally: - _releaseLock() return is_enabled def getChild(self, suffix): @@ -1762,13 +1807,32 @@ def getChild(self, suffix): suffix = '.'.join((self.name, suffix)) return self.manager.getLogger(suffix) + def getChildren(self): + + def _hierlevel(logger): + if logger is logger.manager.root: + return 0 + return 1 + logger.name.count('.') + + d = self.manager.loggerDict + with _lock: + # exclude PlaceHolders - the last check is to ensure that lower-level + # descendants aren't returned - if there are placeholders, a logger's + # parent field might point to a grandparent or ancestor thereof. + return set(item for item in d.values() + if isinstance(item, Logger) and item.parent is self and + _hierlevel(item) == 1 + _hierlevel(item.parent)) + + def _is_disabled(self): + # We need to use getattr as it will only be set the first time a log + # message is recorded on any given thread + return self.disabled or getattr(self._tls, 'in_progress', False) + def __repr__(self): level = getLevelName(self.getEffectiveLevel()) return '<%s %s (%s)>' % (self.__class__.__name__, self.name, level) def __reduce__(self): - # In general, only the root logger will not be accessible via its name. - # However, the root logger's class has its own __reduce__ method. if getLogger(self.name) is not self: import pickle raise pickle.PicklingError('logger cannot be pickled') @@ -1798,7 +1862,7 @@ class LoggerAdapter(object): information in logging output. """ - def __init__(self, logger, extra=None): + def __init__(self, logger, extra=None, merge_extra=False): """ Initialize the adapter with a logger and a dict-like object which provides contextual information. This constructor signature allows @@ -1808,9 +1872,20 @@ def __init__(self, logger, extra=None): following example: adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2")) + + By default, LoggerAdapter objects will drop the "extra" argument + passed on the individual log calls to use its own instead. + + Initializing it with merge_extra=True will instead merge both + maps when logging, the individual call extra taking precedence + over the LoggerAdapter instance extra + + .. versionchanged:: 3.13 + The *merge_extra* argument was added. """ self.logger = logger self.extra = extra + self.merge_extra = merge_extra def process(self, msg, kwargs): """ @@ -1822,7 +1897,10 @@ def process(self, msg, kwargs): Normally, you'll only need to override this one method in a LoggerAdapter subclass for your specific needs. """ - kwargs["extra"] = self.extra + if self.merge_extra and "extra" in kwargs: + kwargs["extra"] = {**self.extra, **kwargs["extra"]} + else: + kwargs["extra"] = self.extra return msg, kwargs # @@ -1902,18 +1980,11 @@ def hasHandlers(self): """ return self.logger.hasHandlers() - def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False): + def _log(self, level, msg, args, **kwargs): """ Low-level log implementation, proxied to allow nested logger adapters. """ - return self.logger._log( - level, - msg, - args, - exc_info=exc_info, - extra=extra, - stack_info=stack_info, - ) + return self.logger._log(level, msg, args, **kwargs) @property def manager(self): @@ -1932,6 +2003,8 @@ def __repr__(self): level = getLevelName(logger.getEffectiveLevel()) return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level) + __class_getitem__ = classmethod(GenericAlias) + root = RootLogger(WARNING) Logger.root = root Logger.manager = Manager(Logger.root) @@ -1971,7 +2044,7 @@ def basicConfig(**kwargs): that this argument is incompatible with 'filename' - if both are present, 'stream' is ignored. handlers If specified, this should be an iterable of already created - handlers, which will be added to the root handler. Any handler + handlers, which will be added to the root logger. Any handler in the list which does not have a formatter assigned will be assigned the formatter created in this function. force If this keyword is specified as true, any existing handlers @@ -2010,8 +2083,7 @@ def basicConfig(**kwargs): """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads - _acquireLock() - try: + with _lock: force = kwargs.pop('force', False) encoding = kwargs.pop('encoding', None) errors = kwargs.pop('errors', 'backslashreplace') @@ -2060,8 +2132,6 @@ def basicConfig(**kwargs): if kwargs: keys = ', '.join(kwargs.keys()) raise ValueError('Unrecognised argument(s): %s' % keys) - finally: - _releaseLock() #--------------------------------------------------------------------------- # Utility functions at module level. @@ -2179,7 +2249,11 @@ def shutdown(handlerList=_handlerList): if h: try: h.acquire() - h.flush() + # MemoryHandlers might not want to be flushed on close, + # but circular imports prevent us scoping this to just + # those handlers. hence the default to True. + if getattr(h, 'flushOnClose', True): + h.flush() h.close() except (OSError, ValueError): # Ignore errors which might be caused @@ -2242,7 +2316,9 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): logger = getLogger("py.warnings") if not logger.handlers: logger.addHandler(NullHandler()) - logger.warning("%s", s) + # bpo-46557: Log str(s) as msg instead of logger.warning("%s", s) + # since some log aggregation tools group logs by the msg arg + logger.warning(str(s)) def captureWarnings(capture): """ diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 3bc63b78621..190b4f92259 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -1,4 +1,4 @@ -# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2023 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -19,18 +19,20 @@ is based on PEP 282 and comments thereto in comp.lang.python, and influenced by Apache's log4j system. -Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ import errno +import functools import io import logging import logging.handlers +import os +import queue import re import struct -import sys import threading import traceback @@ -59,28 +61,34 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non """ import configparser + if isinstance(fname, str): + if not os.path.exists(fname): + raise FileNotFoundError(f"{fname} doesn't exist") + elif not os.path.getsize(fname): + raise RuntimeError(f'{fname} is an empty file') + if isinstance(fname, configparser.RawConfigParser): cp = fname else: - cp = configparser.ConfigParser(defaults) - if hasattr(fname, 'readline'): - cp.read_file(fname) - else: - encoding = io.text_encoding(encoding) - cp.read(fname, encoding=encoding) + try: + cp = configparser.ConfigParser(defaults) + if hasattr(fname, 'readline'): + cp.read_file(fname) + else: + encoding = io.text_encoding(encoding) + cp.read(fname, encoding=encoding) + except configparser.ParsingError as e: + raise RuntimeError(f'{fname} is invalid: {e}') formatters = _create_formatters(cp) # critical section - logging._acquireLock() - try: + with logging._lock: _clearExistingHandlers() # Handlers add themselves to logging._handlers handlers = _install_handlers(cp, formatters) _install_loggers(cp, handlers, disable_existing_loggers) - finally: - logging._releaseLock() def _resolve(name): @@ -113,11 +121,18 @@ def _create_formatters(cp): fs = cp.get(sectname, "format", raw=True, fallback=None) dfs = cp.get(sectname, "datefmt", raw=True, fallback=None) stl = cp.get(sectname, "style", raw=True, fallback='%') + defaults = cp.get(sectname, "defaults", raw=True, fallback=None) + c = logging.Formatter class_name = cp[sectname].get("class") if class_name: c = _resolve(class_name) - f = c(fs, dfs, stl) + + if defaults is not None: + defaults = eval(defaults, vars(logging)) + f = c(fs, dfs, stl, defaults=defaults) + else: + f = c(fs, dfs, stl) formatters[form] = f return formatters @@ -360,7 +375,7 @@ class BaseConfigurator(object): WORD_PATTERN = re.compile(r'^\s*(\w+)\s*') DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*') - INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*') + INDEX_PATTERN = re.compile(r'^\[([^\[\]]*)\]\s*') DIGIT_PATTERN = re.compile(r'^\d+$') value_converters = { @@ -392,11 +407,9 @@ def resolve(self, s): self.importer(used) found = getattr(found, frag) return found - except ImportError: - e, tb = sys.exc_info()[1:] + except ImportError as e: v = ValueError('Cannot resolve %r: %s' % (s, e)) - v.__cause__, v.__traceback__ = e, tb - raise v + raise v from e def ext_convert(self, value): """Default converter for the ext:// protocol.""" @@ -469,10 +482,10 @@ def configure_custom(self, config): c = config.pop('()') if not callable(c): c = self.resolve(c) - props = config.pop('.', None) # Check for valid identifiers - kwargs = {k: config[k] for k in config if valid_ident(k)} + kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))} result = c(**kwargs) + props = config.pop('.', None) if props: for name, value in props.items(): setattr(result, name, value) @@ -484,6 +497,33 @@ def as_tuple(self, value): value = tuple(value) return value +def _is_queue_like_object(obj): + """Check that *obj* implements the Queue API.""" + if isinstance(obj, (queue.Queue, queue.SimpleQueue)): + return True + # defer importing multiprocessing as much as possible + from multiprocessing.queues import Queue as MPQueue + if isinstance(obj, MPQueue): + return True + # Depending on the multiprocessing start context, we cannot create + # a multiprocessing.managers.BaseManager instance 'mm' to get the + # runtime type of mm.Queue() or mm.JoinableQueue() (see gh-119819). + # + # Since we only need an object implementing the Queue API, we only + # do a protocol check, but we do not use typing.runtime_checkable() + # and typing.Protocol to reduce import time (see gh-121723). + # + # Ideally, we would have wanted to simply use strict type checking + # instead of a protocol-based type checking since the latter does + # not check the method signatures. + # + # Note that only 'put_nowait' and 'get' are required by the logging + # queue handler and queue listener (see gh-124653) and that other + # methods are either optional or unused. + minimal_queue_interface = ['put_nowait', 'get'] + return all(callable(getattr(obj, method, None)) + for method in minimal_queue_interface) + class DictConfigurator(BaseConfigurator): """ Configure logging using a dictionary-like object to describe the @@ -500,8 +540,7 @@ def configure(self): raise ValueError("Unsupported version: %s" % config['version']) incremental = config.pop('incremental', False) EMPTY_DICT = {} - logging._acquireLock() - try: + with logging._lock: if incremental: handlers = config.get('handlers', EMPTY_DICT) for name in handlers: @@ -566,7 +605,7 @@ def configure(self): handler.name = name handlers[name] = handler except Exception as e: - if 'target not configured yet' in str(e.__cause__): + if ' not configured yet' in str(e.__cause__): deferred.append(name) else: raise ValueError('Unable to configure handler ' @@ -645,8 +684,6 @@ def configure(self): except Exception as e: raise ValueError('Unable to configure root ' 'logger') from e - finally: - logging._releaseLock() def configure_formatter(self, config): """Configure a formatter from a dictionary.""" @@ -657,10 +694,9 @@ def configure_formatter(self, config): except TypeError as te: if "'format'" not in str(te): raise - #Name of parameter changed from fmt to format. - #Retry with old name. - #This is so that code can be used with older Python versions - #(e.g. by Django) + # logging.Formatter and its subclasses expect the `fmt` + # parameter instead of `format`. Retry passing configuration + # with `fmt`. config['fmt'] = config.pop('format') config['()'] = factory result = self.configure_custom(config) @@ -669,18 +705,27 @@ def configure_formatter(self, config): dfmt = config.get('datefmt', None) style = config.get('style', '%') cname = config.get('class', None) + defaults = config.get('defaults', None) if not cname: c = logging.Formatter else: c = _resolve(cname) + kwargs = {} + + # Add defaults only if it exists. + # Prevents TypeError in custom formatter callables that do not + # accept it. + if defaults is not None: + kwargs['defaults'] = defaults + # A TypeError would be raised if "validate" key is passed in with a formatter callable # that does not accept "validate" as a parameter if 'validate' in config: # if user hasn't mentioned it, the default will be fine - result = c(fmt, dfmt, style, config['validate']) + result = c(fmt, dfmt, style, config['validate'], **kwargs) else: - result = c(fmt, dfmt, style) + result = c(fmt, dfmt, style, **kwargs) return result @@ -697,10 +742,29 @@ def add_filters(self, filterer, filters): """Add filters to a filterer from a list of names.""" for f in filters: try: - filterer.addFilter(self.config['filters'][f]) + if callable(f) or callable(getattr(f, 'filter', None)): + filter_ = f + else: + filter_ = self.config['filters'][f] + filterer.addFilter(filter_) except Exception as e: raise ValueError('Unable to add filter %r' % f) from e + def _configure_queue_handler(self, klass, **kwargs): + if 'queue' in kwargs: + q = kwargs.pop('queue') + else: + q = queue.Queue() # unbounded + + rhl = kwargs.pop('respect_handler_level', False) + lklass = kwargs.pop('listener', logging.handlers.QueueListener) + handlers = kwargs.pop('handlers', []) + + listener = lklass(q, *handlers, respect_handler_level=rhl) + handler = klass(q, **kwargs) + handler.listener = listener + return handler + def configure_handler(self, config): """Configure a handler from a dictionary.""" config_copy = dict(config) # for restoring in case of error @@ -720,28 +784,87 @@ def configure_handler(self, config): factory = c else: cname = config.pop('class') - klass = self.resolve(cname) - #Special case for handler which refers to another handler - if issubclass(klass, logging.handlers.MemoryHandler) and\ - 'target' in config: - try: - th = self.config['handlers'][config['target']] - if not isinstance(th, logging.Handler): - config.update(config_copy) # restore for deferred cfg - raise TypeError('target not configured yet') - config['target'] = th - except Exception as e: - raise ValueError('Unable to set target handler ' - '%r' % config['target']) from e + if callable(cname): + klass = cname + else: + klass = self.resolve(cname) + if issubclass(klass, logging.handlers.MemoryHandler): + if 'flushLevel' in config: + config['flushLevel'] = logging._checkLevel(config['flushLevel']) + if 'target' in config: + # Special case for handler which refers to another handler + try: + tn = config['target'] + th = self.config['handlers'][tn] + if not isinstance(th, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('target not configured yet') + config['target'] = th + except Exception as e: + raise ValueError('Unable to set target handler %r' % tn) from e + elif issubclass(klass, logging.handlers.QueueHandler): + # Another special case for handler which refers to other handlers + # if 'handlers' not in config: + # raise ValueError('No handlers specified for a QueueHandler') + if 'queue' in config: + qspec = config['queue'] + + if isinstance(qspec, str): + q = self.resolve(qspec) + if not callable(q): + raise TypeError('Invalid queue specifier %r' % qspec) + config['queue'] = q() + elif isinstance(qspec, dict): + if '()' not in qspec: + raise TypeError('Invalid queue specifier %r' % qspec) + config['queue'] = self.configure_custom(dict(qspec)) + elif not _is_queue_like_object(qspec): + raise TypeError('Invalid queue specifier %r' % qspec) + + if 'listener' in config: + lspec = config['listener'] + if isinstance(lspec, type): + if not issubclass(lspec, logging.handlers.QueueListener): + raise TypeError('Invalid listener specifier %r' % lspec) + else: + if isinstance(lspec, str): + listener = self.resolve(lspec) + if isinstance(listener, type) and\ + not issubclass(listener, logging.handlers.QueueListener): + raise TypeError('Invalid listener specifier %r' % lspec) + elif isinstance(lspec, dict): + if '()' not in lspec: + raise TypeError('Invalid listener specifier %r' % lspec) + listener = self.configure_custom(dict(lspec)) + else: + raise TypeError('Invalid listener specifier %r' % lspec) + if not callable(listener): + raise TypeError('Invalid listener specifier %r' % lspec) + config['listener'] = listener + if 'handlers' in config: + hlist = [] + try: + for hn in config['handlers']: + h = self.config['handlers'][hn] + if not isinstance(h, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('Required handler %r ' + 'is not configured yet' % hn) + hlist.append(h) + except Exception as e: + raise ValueError('Unable to set required handler %r' % hn) from e + config['handlers'] = hlist elif issubclass(klass, logging.handlers.SMTPHandler) and\ 'mailhost' in config: config['mailhost'] = self.as_tuple(config['mailhost']) elif issubclass(klass, logging.handlers.SysLogHandler) and\ 'address' in config: config['address'] = self.as_tuple(config['address']) - factory = klass - props = config.pop('.', None) - kwargs = {k: config[k] for k in config if valid_ident(k)} + if issubclass(klass, logging.handlers.QueueHandler): + factory = functools.partial(self._configure_queue_handler, klass) + else: + factory = klass + kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))} try: result = factory(**kwargs) except TypeError as te: @@ -759,6 +882,7 @@ def configure_handler(self, config): result.setLevel(logging._checkLevel(level)) if filters: self.add_filters(result, filters) + props = config.pop('.', None) if props: for name, value in props.items(): setattr(result, name, value) @@ -794,6 +918,7 @@ def configure_logger(self, name, config, incremental=False): """Configure a non-root logger from a dictionary.""" logger = logging.getLogger(name) self.common_logger_config(logger, config, incremental) + logger.disabled = False propagate = config.get('propagate', None) if propagate is not None: logger.propagate = propagate @@ -886,9 +1011,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): ThreadingTCPServer.__init__(self, (host, port), handler) - logging._acquireLock() - self.abort = 0 - logging._releaseLock() + with logging._lock: + self.abort = 0 self.timeout = 1 self.ready = ready self.verify = verify @@ -902,9 +1026,8 @@ def serve_until_stopped(self): self.timeout) if rd: self.handle_request() - logging._acquireLock() - abort = self.abort - logging._releaseLock() + with logging._lock: + abort = self.abort self.server_close() class Server(threading.Thread): @@ -925,9 +1048,8 @@ def run(self): self.port = server.server_address[1] self.ready.set() global _listener - logging._acquireLock() - _listener = server - logging._releaseLock() + with logging._lock: + _listener = server server.serve_until_stopped() return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify) @@ -937,10 +1059,7 @@ def stopListening(): Stop the listening server which was created with a call to listen(). """ global _listener - logging._acquireLock() - try: + with logging._lock: if _listener: _listener.abort = 1 _listener = None - finally: - logging._releaseLock() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 61a39958c0a..d3ea06c731e 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -23,11 +23,17 @@ To use, simply 'import logging.handlers' and log away! """ -import io, logging, socket, os, pickle, struct, time, re -from stat import ST_DEV, ST_INO, ST_MTIME +import copy +import io +import logging +import os +import pickle import queue +import re +import socket +import struct import threading -import copy +import time # # Some constants... @@ -187,15 +193,18 @@ def shouldRollover(self, record): Basically, see if the supplied record would cause the file to exceed the size limit we have. """ - # See bpo-45401: Never rollover anything other than regular files - if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): - return False if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? + pos = self.stream.tell() + if not pos: + # gh-116263: Never rollover an empty file + return False msg = "%s\n" % self.format(record) - self.stream.seek(0, 2) #due to non-posix-compliant Windows feature - if self.stream.tell() + len(msg) >= self.maxBytes: + if pos + len(msg) >= self.maxBytes: + # See bpo-45401: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + return False return True return False @@ -232,19 +241,19 @@ def __init__(self, filename, when='h', interval=1, backupCount=0, if self.when == 'S': self.interval = 1 # one second self.suffix = "%Y-%m-%d_%H-%M-%S" - self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$" + extMatch = r"(?= self.rolloverAt: + # See #89564: Never rollover anything other than regular files + if os.path.exists(self.baseFilename) and not os.path.isfile(self.baseFilename): + # The file is not a regular file, so do not rollover, but do + # set the next rollover time to avoid repeated checks. + self.rolloverAt = self.computeRollover(t) + return False + return True return False @@ -365,32 +388,28 @@ def getFilesToDelete(self): dirName, baseName = os.path.split(self.baseFilename) fileNames = os.listdir(dirName) result = [] - # See bpo-44753: Don't use the extension when computing the prefix. - n, e = os.path.splitext(baseName) - prefix = n + '.' - plen = len(prefix) - for fileName in fileNames: - if self.namer is None: - # Our files will always start with baseName - if not fileName.startswith(baseName): - continue - else: - # Our files could be just about anything after custom naming, but - # likely candidates are of the form - # foo.log.DATETIME_SUFFIX or foo.DATETIME_SUFFIX.log - if (not fileName.startswith(baseName) and fileName.endswith(e) and - len(fileName) > (plen + 1) and not fileName[plen+1].isdigit()): - continue - - if fileName[:plen] == prefix: - suffix = fileName[plen:] - # See bpo-45628: The date/time suffix could be anywhere in the - # filename - parts = suffix.split('.') - for part in parts: - if self.extMatch.match(part): + if self.namer is None: + prefix = baseName + '.' + plen = len(prefix) + for fileName in fileNames: + if fileName[:plen] == prefix: + suffix = fileName[plen:] + if self.extMatch.fullmatch(suffix): + result.append(os.path.join(dirName, fileName)) + else: + for fileName in fileNames: + # Our files could be just about anything after custom naming, + # but they should contain the datetime suffix. + # Try to find the datetime suffix in the file name and verify + # that the file name can be generated by this handler. + m = self.extMatch.search(fileName) + while m: + dfn = self.namer(self.baseFilename + "." + m[0]) + if os.path.basename(dfn) == fileName: result.append(os.path.join(dirName, fileName)) break + m = self.extMatch.search(fileName, m.start() + 1) + if len(result) < self.backupCount: result = [] else: @@ -406,17 +425,14 @@ def doRollover(self): then we have to get a list of matching filenames, sort them and remove the one with the oldest suffix. """ - if self.stream: - self.stream.close() - self.stream = None # get the time that this sequence started at and make it a TimeTuple currentTime = int(time.time()) - dstNow = time.localtime(currentTime)[-1] t = self.rolloverAt - self.interval if self.utc: timeTuple = time.gmtime(t) else: timeTuple = time.localtime(t) + dstNow = time.localtime(currentTime)[-1] dstThen = timeTuple[-1] if dstNow != dstThen: if dstNow: @@ -427,26 +443,19 @@ def doRollover(self): dfn = self.rotation_filename(self.baseFilename + "." + time.strftime(self.suffix, timeTuple)) if os.path.exists(dfn): - os.remove(dfn) + # Already rolled over. + return + + if self.stream: + self.stream.close() + self.stream = None self.rotate(self.baseFilename, dfn) if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) if not self.delay: self.stream = self._open() - newRolloverAt = self.computeRollover(currentTime) - while newRolloverAt <= currentTime: - newRolloverAt = newRolloverAt + self.interval - #If DST changes and midnight or weekly rollover, adjust for this. - if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc: - dstAtRollover = time.localtime(newRolloverAt)[-1] - if dstNow != dstAtRollover: - if not dstNow: # DST kicks in before next rollover, so we need to deduct an hour - addend = -3600 - else: # DST bows out before next rollover, so we need to add an hour - addend = 3600 - newRolloverAt += addend - self.rolloverAt = newRolloverAt + self.rolloverAt = self.computeRollover(currentTime) class WatchedFileHandler(logging.FileHandler): """ @@ -462,8 +471,7 @@ class WatchedFileHandler(logging.FileHandler): This handler is not appropriate for use under Windows, because under Windows open files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need - for such a handler. Furthermore, ST_INO is not supported under - Windows; stat always returns zero for this value. + for such a handler. This handler is based on a suggestion and patch by Chad J. Schroeder. @@ -479,9 +487,11 @@ def __init__(self, filename, mode='a', encoding=None, delay=False, self._statstream() def _statstream(self): - if self.stream: - sres = os.fstat(self.stream.fileno()) - self.dev, self.ino = sres[ST_DEV], sres[ST_INO] + if self.stream is None: + return + sres = os.fstat(self.stream.fileno()) + self.dev = sres.st_dev + self.ino = sres.st_ino def reopenIfNeeded(self): """ @@ -491,6 +501,9 @@ def reopenIfNeeded(self): has, close the old stream and reopen the file to get the current stream. """ + if self.stream is None: + return + # Reduce the chance of race conditions by stat'ing by path only # once and then fstat'ing our new fd if we opened a new log stream. # See issue #14632: Thanks to John Mulligan for the problem report @@ -498,18 +511,23 @@ def reopenIfNeeded(self): try: # stat the file by path, checking for existence sres = os.stat(self.baseFilename) + + # compare file system stat with that of our stream file handle + reopen = (sres.st_dev != self.dev or sres.st_ino != self.ino) except FileNotFoundError: - sres = None - # compare file system stat with that of our stream file handle - if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino: - if self.stream is not None: - # we have an open file handle, clean it up - self.stream.flush() - self.stream.close() - self.stream = None # See Issue #21742: _open () might fail. - # open a new file handle and get new stat info from that fd - self.stream = self._open() - self._statstream() + reopen = True + + if not reopen: + return + + # we have an open file handle, clean it up + self.stream.flush() + self.stream.close() + self.stream = None # See Issue #21742: _open () might fail. + + # open a new file handle and get new stat info from that fd + self.stream = self._open() + self._statstream() def emit(self, record): """ @@ -679,15 +697,12 @@ def close(self): """ Closes the socket. """ - self.acquire() - try: + with self.lock: sock = self.sock if sock: self.sock = None sock.close() logging.Handler.close(self) - finally: - self.release() class DatagramHandler(SocketHandler): """ @@ -829,10 +844,8 @@ class SysLogHandler(logging.Handler): "local7": LOG_LOCAL7, } - #The map below appears to be trivially lowercasing the key. However, - #there's more to it than meets the eye - in some locales, lowercasing - #gives unexpected results. See SF #1524081: in the Turkish locale, - #"INFO".lower() != "info" + # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept + # for backwards compatibility. priority_map = { "DEBUG" : "debug", "INFO" : "info", @@ -859,12 +872,49 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), self.address = address self.facility = facility self.socktype = socktype + self.socket = None + self.createSocket() + + def _connect_unixsocket(self, address): + use_socktype = self.socktype + if use_socktype is None: + use_socktype = socket.SOCK_DGRAM + self.socket = socket.socket(socket.AF_UNIX, use_socktype) + try: + self.socket.connect(address) + # it worked, so set self.socktype to the used type + self.socktype = use_socktype + except OSError: + self.socket.close() + if self.socktype is not None: + # user didn't specify falling back, so fail + raise + use_socktype = socket.SOCK_STREAM + self.socket = socket.socket(socket.AF_UNIX, use_socktype) + try: + self.socket.connect(address) + # it worked, so set self.socktype to the used type + self.socktype = use_socktype + except OSError: + self.socket.close() + raise + + def createSocket(self): + """ + Try to create a socket and, if it's not a datagram socket, connect it + to the other end. This method is called during handler initialization, + but it's not regarded as an error if the other end isn't listening yet + --- the method will be called again when emitting an event, + if there is no socket at that point. + """ + address = self.address + socktype = self.socktype if isinstance(address, str): self.unixsocket = True # Syslog server may be unavailable during handler initialisation. # C's openlog() function also ignores connection errors. - # Moreover, we ignore these errors while logging, so it not worse + # Moreover, we ignore these errors while logging, so it's not worse # to ignore it also here. try: self._connect_unixsocket(address) @@ -895,30 +945,6 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), self.socket = sock self.socktype = socktype - def _connect_unixsocket(self, address): - use_socktype = self.socktype - if use_socktype is None: - use_socktype = socket.SOCK_DGRAM - self.socket = socket.socket(socket.AF_UNIX, use_socktype) - try: - self.socket.connect(address) - # it worked, so set self.socktype to the used type - self.socktype = use_socktype - except OSError: - self.socket.close() - if self.socktype is not None: - # user didn't specify falling back, so fail - raise - use_socktype = socket.SOCK_STREAM - self.socket = socket.socket(socket.AF_UNIX, use_socktype) - try: - self.socket.connect(address) - # it worked, so set self.socktype to the used type - self.socktype = use_socktype - except OSError: - self.socket.close() - raise - def encodePriority(self, facility, priority): """ Encode the facility and priority. You can pass in strings or @@ -936,12 +962,12 @@ def close(self): """ Closes the socket. """ - self.acquire() - try: - self.socket.close() + with self.lock: + sock = self.socket + if sock: + self.socket = None + sock.close() logging.Handler.close(self) - finally: - self.release() def mapPriority(self, levelName): """ @@ -978,6 +1004,10 @@ def emit(self, record): # Message is a string. Convert to bytes as required by RFC 5424 msg = msg.encode('utf-8') msg = prio + msg + + if not self.socket: + self.createSocket() + if self.unixsocket: try: self.socket.send(msg) @@ -1010,7 +1040,8 @@ def __init__(self, mailhost, fromaddr, toaddrs, subject, only be used when authentication credentials are supplied. The tuple will be either an empty tuple, or a single-value tuple with the name of a keyfile, or a 2-value tuple with the names of the keyfile and - certificate file. (This tuple is passed to the `starttls` method). + certificate file. (This tuple is passed to the + `ssl.SSLContext.load_cert_chain` method). A timeout in seconds can be specified for the SMTP connection (the default is one second). """ @@ -1063,8 +1094,23 @@ def emit(self, record): msg.set_content(self.format(record)) if self.username: if self.secure is not None: + import ssl + + try: + keyfile = self.secure[0] + except IndexError: + keyfile = None + + try: + certfile = self.secure[1] + except IndexError: + certfile = None + + context = ssl._create_stdlib_context( + certfile=certfile, keyfile=keyfile + ) smtp.ehlo() - smtp.starttls(*self.secure) + smtp.starttls(context=context) smtp.ehlo() smtp.login(self.username, self.password) smtp.send_message(msg) @@ -1094,7 +1140,16 @@ def __init__(self, appname, dllname=None, logtype="Application"): dllname = os.path.join(dllname[0], r'win32service.pyd') self.dllname = dllname self.logtype = logtype - self._welu.AddSourceToRegistry(appname, dllname, logtype) + # Administrative privileges are required to add a source to the registry. + # This may not be available for a user that just wants to add to an + # existing source - handle this specific case. + try: + self._welu.AddSourceToRegistry(appname, dllname, logtype) + except Exception as e: + # This will probably be a pywintypes.error. Only raise if it's not + # an "access denied" error, else let it pass + if getattr(e, 'winerror', None) != 5: # not access denied + raise self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE self.typemap = { logging.DEBUG : win32evtlog.EVENTLOG_INFORMATION_TYPE, @@ -1300,11 +1355,8 @@ def flush(self): This version just zaps the buffer to empty. """ - self.acquire() - try: + with self.lock: self.buffer.clear() - finally: - self.release() def close(self): """ @@ -1354,11 +1406,8 @@ def setTarget(self, target): """ Set the target handler for this handler. """ - self.acquire() - try: + with self.lock: self.target = target - finally: - self.release() def flush(self): """ @@ -1366,16 +1415,13 @@ def flush(self): records to the target, if there is one. Override if you want different behaviour. - The record buffer is also cleared by this operation. + The record buffer is only cleared if a target has been set. """ - self.acquire() - try: + with self.lock: if self.target: for record in self.buffer: self.target.handle(record) self.buffer.clear() - finally: - self.release() def close(self): """ @@ -1386,12 +1432,9 @@ def close(self): if self.flushOnClose: self.flush() finally: - self.acquire() - try: + with self.lock: self.target = None BufferingHandler.close(self) - finally: - self.release() class QueueHandler(logging.Handler): @@ -1411,6 +1454,7 @@ def __init__(self, queue): """ logging.Handler.__init__(self) self.queue = queue + self.listener = None # will be set to listener if configured via dictConfig() def enqueue(self, record): """ @@ -1424,12 +1468,15 @@ def enqueue(self, record): def prepare(self, record): """ - Prepares a record for queuing. The object returned by this method is + Prepare a record for queuing. The object returned by this method is enqueued. - The base implementation formats the record to merge the message - and arguments, and removes unpickleable items from the record - in-place. + The base implementation formats the record to merge the message and + arguments, and removes unpickleable items from the record in-place. + Specifically, it overwrites the record's `msg` and + `message` attributes with the merged message (obtained by + calling the handler's `format` method), and sets the `args`, + `exc_info` and `exc_text` attributes to None. You might want to override this method if you want to convert the record to a dict or JSON string, or send a modified copy @@ -1439,7 +1486,7 @@ def prepare(self, record): # (if there's exception data), and also returns the formatted # message. We can then use this to replace the original # msg + args, as these might be unpickleable. We also zap the - # exc_info and exc_text attributes, as they are no longer + # exc_info, exc_text and stack_info attributes, as they are no longer # needed and, if not None, will typically not be pickleable. msg = self.format(record) # bpo-35726: make copy of record to avoid affecting other handlers in the chain. @@ -1449,6 +1496,7 @@ def prepare(self, record): record.args = None record.exc_info = None record.exc_text = None + record.stack_info = None return record def emit(self, record): @@ -1497,6 +1545,9 @@ def start(self): This starts up a background thread to monitor the queue for LogRecords to process. """ + if self._thread is not None: + raise RuntimeError("Listener already started") + self._thread = t = threading.Thread(target=self._monitor) t.daemon = True t.start() @@ -1568,6 +1619,7 @@ def stop(self): Note that if you don't call this before your application exits, there may be some records still left on the queue, which won't be processed. """ - self.enqueue_sentinel() - self._thread.join() - self._thread = None + if self._thread: # see gh-114706 - allow calling this more than once + self.enqueue_sentinel() + self._thread.join() + self._thread = None diff --git a/Lib/lzma.py b/Lib/lzma.py new file mode 100644 index 00000000000..316066d024e --- /dev/null +++ b/Lib/lzma.py @@ -0,0 +1,364 @@ +"""Interface to the liblzma compression library. + +This module provides a class for reading and writing compressed files, +classes for incremental (de)compression, and convenience functions for +one-shot (de)compression. + +These classes and functions support both the XZ and legacy LZMA +container formats, as well as raw compressed data streams. +""" + +__all__ = [ + "CHECK_NONE", "CHECK_CRC32", "CHECK_CRC64", "CHECK_SHA256", + "CHECK_ID_MAX", "CHECK_UNKNOWN", + "FILTER_LZMA1", "FILTER_LZMA2", "FILTER_DELTA", "FILTER_X86", "FILTER_IA64", + "FILTER_ARM", "FILTER_ARMTHUMB", "FILTER_POWERPC", "FILTER_SPARC", + "FORMAT_AUTO", "FORMAT_XZ", "FORMAT_ALONE", "FORMAT_RAW", + "MF_HC3", "MF_HC4", "MF_BT2", "MF_BT3", "MF_BT4", + "MODE_FAST", "MODE_NORMAL", "PRESET_DEFAULT", "PRESET_EXTREME", + + "LZMACompressor", "LZMADecompressor", "LZMAFile", "LZMAError", + "open", "compress", "decompress", "is_check_supported", +] + +import builtins +import io +import os +from compression._common import _streams +from _lzma import * +from _lzma import _encode_filter_properties, _decode_filter_properties # noqa: F401 + + +# Value 0 no longer used +_MODE_READ = 1 +# Value 2 no longer used +_MODE_WRITE = 3 + + +class LZMAFile(_streams.BaseStream): + + """A file object providing transparent LZMA (de)compression. + + An LZMAFile can act as a wrapper for an existing file object, or + refer directly to a named file on disk. + + Note that LZMAFile provides a *binary* file interface - data read + is returned as bytes, and data to be written must be given as bytes. + """ + + def __init__(self, filename=None, mode="r", *, + format=None, check=-1, preset=None, filters=None): + """Open an LZMA-compressed file in binary mode. + + filename can be either an actual file name (given as a str, + bytes, or PathLike object), in which case the named file is + opened, or it can be an existing file object to read from or + write to. + + mode can be "r" for reading (default), "w" for (over)writing, + "x" for creating exclusively, or "a" for appending. These can + equivalently be given as "rb", "wb", "xb" and "ab" respectively. + + format specifies the container format to use for the file. + If mode is "r", this defaults to FORMAT_AUTO. Otherwise, the + default is FORMAT_XZ. + + check specifies the integrity check to use. This argument can + only be used when opening a file for writing. For FORMAT_XZ, + the default is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not + support integrity checks - for these formats, check must be + omitted, or be CHECK_NONE. + + When opening a file for reading, the *preset* argument is not + meaningful, and should be omitted. The *filters* argument should + also be omitted, except when format is FORMAT_RAW (in which case + it is required). + + When opening a file for writing, the settings used by the + compressor can be specified either as a preset compression + level (with the *preset* argument), or in detail as a custom + filter chain (with the *filters* argument). For FORMAT_XZ and + FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset + level. For FORMAT_RAW, the caller must always specify a filter + chain; the raw compressor does not support preset compression + levels. + + preset (if provided) should be an integer in the range 0-9, + optionally OR-ed with the constant PRESET_EXTREME. + + filters (if provided) should be a sequence of dicts. Each dict + should have an entry for "id" indicating ID of the filter, plus + additional entries for options to the filter. + """ + self._fp = None + self._closefp = False + self._mode = None + + if mode in ("r", "rb"): + if check != -1: + raise ValueError("Cannot specify an integrity check " + "when opening a file for reading") + if preset is not None: + raise ValueError("Cannot specify a preset compression " + "level when opening a file for reading") + if format is None: + format = FORMAT_AUTO + mode_code = _MODE_READ + elif mode in ("w", "wb", "a", "ab", "x", "xb"): + if format is None: + format = FORMAT_XZ + mode_code = _MODE_WRITE + self._compressor = LZMACompressor(format=format, check=check, + preset=preset, filters=filters) + self._pos = 0 + else: + raise ValueError("Invalid mode: {!r}".format(mode)) + + if isinstance(filename, (str, bytes, os.PathLike)): + if "b" not in mode: + mode += "b" + self._fp = builtins.open(filename, mode) + self._closefp = True + self._mode = mode_code + elif hasattr(filename, "read") or hasattr(filename, "write"): + self._fp = filename + self._mode = mode_code + else: + raise TypeError("filename must be a str, bytes, file or PathLike object") + + if self._mode == _MODE_READ: + raw = _streams.DecompressReader(self._fp, LZMADecompressor, + trailing_error=LZMAError, format=format, filters=filters) + self._buffer = io.BufferedReader(raw) + + def close(self): + """Flush and close the file. + + May be called more than once without error. Once the file is + closed, any other operation on it will raise a ValueError. + """ + if self.closed: + return + try: + if self._mode == _MODE_READ: + self._buffer.close() + self._buffer = None + elif self._mode == _MODE_WRITE: + self._fp.write(self._compressor.flush()) + self._compressor = None + finally: + try: + if self._closefp: + self._fp.close() + finally: + self._fp = None + self._closefp = False + + @property + def closed(self): + """True if this file is closed.""" + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' + + def fileno(self): + """Return the file descriptor for the underlying file.""" + self._check_not_closed() + return self._fp.fileno() + + def seekable(self): + """Return whether the file supports seeking.""" + return self.readable() and self._buffer.seekable() + + def readable(self): + """Return whether the file was opened for reading.""" + self._check_not_closed() + return self._mode == _MODE_READ + + def writable(self): + """Return whether the file was opened for writing.""" + self._check_not_closed() + return self._mode == _MODE_WRITE + + def peek(self, size=-1): + """Return buffered data without advancing the file position. + + Always returns at least one byte of data, unless at EOF. + The exact number of bytes returned is unspecified. + """ + self._check_can_read() + # Relies on the undocumented fact that BufferedReader.peek() always + # returns at least one byte (except at EOF) + return self._buffer.peek(size) + + def read(self, size=-1): + """Read up to size uncompressed bytes from the file. + + If size is negative or omitted, read until EOF is reached. + Returns b"" if the file is already at EOF. + """ + self._check_can_read() + return self._buffer.read(size) + + def read1(self, size=-1): + """Read up to size uncompressed bytes, while trying to avoid + making multiple reads from the underlying stream. Reads up to a + buffer's worth of data if size is negative. + + Returns b"" if the file is at EOF. + """ + self._check_can_read() + if size < 0: + size = io.DEFAULT_BUFFER_SIZE + return self._buffer.read1(size) + + def readline(self, size=-1): + """Read a line of uncompressed bytes from the file. + + The terminating newline (if present) is retained. If size is + non-negative, no more than size bytes will be read (in which + case the line may be incomplete). Returns b'' if already at EOF. + """ + self._check_can_read() + return self._buffer.readline(size) + + def write(self, data): + """Write a bytes object to the file. + + Returns the number of uncompressed bytes written, which is + always the length of data in bytes. Note that due to buffering, + the file on disk may not reflect the data written until close() + is called. + """ + self._check_can_write() + if isinstance(data, (bytes, bytearray)): + length = len(data) + else: + # accept any data that supports the buffer protocol + data = memoryview(data) + length = data.nbytes + + compressed = self._compressor.compress(data) + self._fp.write(compressed) + self._pos += length + return length + + def seek(self, offset, whence=io.SEEK_SET): + """Change the file position. + + The new position is specified by offset, relative to the + position indicated by whence. Possible values for whence are: + + 0: start of stream (default): offset must not be negative + 1: current stream position + 2: end of stream; offset must not be positive + + Returns the new file position. + + Note that seeking is emulated, so depending on the parameters, + this operation may be extremely slow. + """ + self._check_can_seek() + return self._buffer.seek(offset, whence) + + def tell(self): + """Return the current file position.""" + self._check_not_closed() + if self._mode == _MODE_READ: + return self._buffer.tell() + return self._pos + + +def open(filename, mode="rb", *, + format=None, check=-1, preset=None, filters=None, + encoding=None, errors=None, newline=None): + """Open an LZMA-compressed file in binary or text mode. + + filename can be either an actual file name (given as a str, bytes, + or PathLike object), in which case the named file is opened, or it + can be an existing file object to read from or write to. + + The mode argument can be "r", "rb" (default), "w", "wb", "x", "xb", + "a", or "ab" for binary mode, or "rt", "wt", "xt", or "at" for text + mode. + + The format, check, preset and filters arguments specify the + compression settings, as for LZMACompressor, LZMADecompressor and + LZMAFile. + + For binary mode, this function is equivalent to the LZMAFile + constructor: LZMAFile(filename, mode, ...). In this case, the + encoding, errors and newline arguments must not be provided. + + For text mode, an LZMAFile object is created, and wrapped in an + io.TextIOWrapper instance with the specified encoding, error + handling behavior, and line ending(s). + + """ + if "t" in mode: + if "b" in mode: + raise ValueError("Invalid mode: %r" % (mode,)) + else: + if encoding is not None: + raise ValueError("Argument 'encoding' not supported in binary mode") + if errors is not None: + raise ValueError("Argument 'errors' not supported in binary mode") + if newline is not None: + raise ValueError("Argument 'newline' not supported in binary mode") + + lz_mode = mode.replace("t", "") + binary_file = LZMAFile(filename, lz_mode, format=format, check=check, + preset=preset, filters=filters) + + if "t" in mode: + encoding = io.text_encoding(encoding) + return io.TextIOWrapper(binary_file, encoding, errors, newline) + else: + return binary_file + + +def compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None): + """Compress a block of data. + + Refer to LZMACompressor's docstring for a description of the + optional arguments *format*, *check*, *preset* and *filters*. + + For incremental compression, use an LZMACompressor instead. + """ + comp = LZMACompressor(format, check, preset, filters) + return comp.compress(data) + comp.flush() + + +def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None): + """Decompress a block of data. + + Refer to LZMADecompressor's docstring for a description of the + optional arguments *format*, *check* and *filters*. + + For incremental decompression, use an LZMADecompressor instead. + """ + results = [] + while True: + decomp = LZMADecompressor(format, memlimit, filters) + try: + res = decomp.decompress(data) + except LZMAError: + if results: + break # Leftover data is not a valid LZMA/XZ stream; ignore it. + else: + raise # Error on the first iteration; bail out. + results.append(res) + if not decomp.eof: + raise LZMAError("Compressed data ended before the " + "end-of-stream marker was reached") + data = decomp.unused_data + if not data: + break + return b"".join(results) diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 70da07ed2e9..364af6bb010 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -395,6 +395,56 @@ def get_file(self, key): f = open(os.path.join(self._path, self._lookup(key)), 'rb') return _ProxyFile(f) + def get_info(self, key): + """Get the keyed message's "info" as a string.""" + subpath = self._lookup(key) + if self.colon in subpath: + return subpath.split(self.colon)[-1] + return '' + + def set_info(self, key, info: str): + """Set the keyed message's "info" string.""" + if not isinstance(info, str): + raise TypeError(f'info must be a string: {type(info)}') + old_subpath = self._lookup(key) + new_subpath = old_subpath.split(self.colon)[0] + if info: + new_subpath += self.colon + info + if new_subpath == old_subpath: + return + old_path = os.path.join(self._path, old_subpath) + new_path = os.path.join(self._path, new_subpath) + os.rename(old_path, new_path) + self._toc[key] = new_subpath + + def get_flags(self, key): + """Return as a string the standard flags that are set on the keyed message.""" + info = self.get_info(key) + if info.startswith('2,'): + return info[2:] + return '' + + def set_flags(self, key, flags: str): + """Set the given flags and unset all others on the keyed message.""" + if not isinstance(flags, str): + raise TypeError(f'flags must be a string: {type(flags)}') + # TODO: check if flags are valid standard flag characters? + self.set_info(key, '2,' + ''.join(sorted(set(flags)))) + + def add_flag(self, key, flag: str): + """Set the given flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + # TODO: check that flag is a valid standard flag character? + self.set_flags(key, ''.join(set(self.get_flags(key)) | set(flag))) + + def remove_flag(self, key, flag: str): + """Unset the given string flag(s) without changing others on the keyed message.""" + if not isinstance(flag, str): + raise TypeError(f'flag must be a string: {type(flag)}') + if self.get_flags(key): + self.set_flags(key, ''.join(set(self.get_flags(key)) - set(flag))) + def iterkeys(self): """Return an iterator over keys.""" self._refresh() @@ -540,6 +590,8 @@ def _refresh(self): for subdir in self._toc_mtimes: path = self._paths[subdir] for entry in os.listdir(path): + if entry.startswith('.'): + continue p = os.path.join(path, entry) if os.path.isdir(p): continue @@ -698,9 +750,13 @@ def flush(self): _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) + # Make sure the new file's mode and owner are the same as the old file's + info = os.stat(self._path) + os.chmod(new_file.name, info.st_mode) + try: + os.chown(new_file.name, info.st_uid, info.st_gid) + except (AttributeError, OSError): + pass try: os.rename(new_file.name, self._path) except FileExistsError: @@ -778,10 +834,11 @@ def get_message(self, key): """Return a Message representation or raise a KeyError.""" start, stop = self._lookup(key) self._file.seek(start) - from_line = self._file.readline().replace(linesep, b'') + from_line = self._file.readline().replace(linesep, b'').decode('ascii') string = self._file.read(stop - self._file.tell()) msg = self._message_factory(string.replace(linesep, b'\n')) - msg.set_from(from_line[5:].decode('ascii')) + msg.set_unixfrom(from_line) + msg.set_from(from_line[5:]) return msg def get_string(self, key, from_=False): @@ -1089,10 +1146,24 @@ def __len__(self): """Return a count of messages in the mailbox.""" return len(list(self.iterkeys())) + def _open_mh_sequences_file(self, text): + mode = '' if text else 'b' + kwargs = {'encoding': 'ASCII'} if text else {} + path = os.path.join(self._path, '.mh_sequences') + while True: + try: + return open(path, 'r+' + mode, **kwargs) + except FileNotFoundError: + pass + try: + return open(path, 'x+' + mode, **kwargs) + except FileExistsError: + pass + def lock(self): """Lock the mailbox.""" if not self._locked: - self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') + self._file = self._open_mh_sequences_file(text=False) _lock_file(self._file) self._locked = True @@ -1146,7 +1217,11 @@ def remove_folder(self, folder): def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} - with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: + try: + f = open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') + except FileNotFoundError: + return results + with f: all_keys = set(self.keys()) for line in f: try: @@ -1169,7 +1244,7 @@ def get_sequences(self): def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" - f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') + f = self._open_mh_sequences_file(text=True) try: os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) for name, keys in sequences.items(): @@ -1956,10 +2031,7 @@ def readlines(self, sizehint=None): def __iter__(self): """Iterate over lines.""" - while True: - line = self.readline() - if not line: - return + while line := self.readline(): yield line def tell(self): @@ -2111,11 +2183,7 @@ def _unlock_file(f): def _create_carefully(path): """Create a file if it doesn't exist and open for reading and writing.""" - fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666) - try: - return open(path, 'rb+') - finally: - os.close(fd) + return open(path, 'xb+') def _create_temporary(path): """Create a temp file based on path and open for reading and writing.""" diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 954bb0a7453..7d0f4c1fd40 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -23,10 +23,11 @@ read_mime_types(file) -- parse one file, return a dictionary or None """ -import os -import sys -import posixpath -import urllib.parse +try: + from _winapi import _mimetypes_read_windows_registry +except ImportError: + _mimetypes_read_windows_registry = None + try: import winreg as _winreg except ImportError: @@ -34,7 +35,7 @@ __all__ = [ "knownfiles", "inited", "MimeTypes", - "guess_type", "guess_all_extensions", "guess_extension", + "guess_type", "guess_file_type", "guess_all_extensions", "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", "encodings_map", "types_map", "common_types" ] @@ -88,7 +89,21 @@ def add_type(self, type, ext, strict=True): If strict is true, information will be added to list of standard types, else to the list of non-standard types. + + Valid extensions are empty or start with a '.'. """ + if ext and not ext.startswith('.'): + from warnings import _deprecated + + _deprecated( + "Undotted extensions", + "Using undotted extensions is deprecated and " + "will raise a ValueError in Python {remove}", + remove=(3, 16), + ) + + if not type: + return self.types_map[strict][ext] = type exts = self.types_map_inv[strict].setdefault(type, []) if ext not in exts: @@ -110,11 +125,21 @@ def guess_type(self, url, strict=True): mapped to '.tar.gz'. (This is table-driven too, using the dictionary suffix_map.) - Optional `strict' argument when False adds a bunch of commonly found, + Optional 'strict' argument when False adds a bunch of commonly found, but non-standard types. """ + # Lazy import to improve module import time + import os + import urllib.parse + + # TODO: Deprecate accepting file paths (in particular path-like objects). url = os.fspath(url) - scheme, url = urllib.parse._splittype(url) + p = urllib.parse.urlparse(url) + if p.scheme and len(p.scheme) > 1: + scheme = p.scheme + url = p.path + else: + return self.guess_file_type(url, strict=strict) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data @@ -134,26 +159,43 @@ def guess_type(self, url, strict=True): if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None - base, ext = posixpath.splitext(url) - while ext in self.suffix_map: - base, ext = posixpath.splitext(base + self.suffix_map[ext]) + + # Lazy import to improve module import time + import posixpath + + return self._guess_file_type(url, strict, posixpath.splitext) + + def guess_file_type(self, path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path instead of URL. + """ + # Lazy import to improve module import time + import os + + path = os.fsdecode(path) + path = os.path.splitdrive(path)[1] + return self._guess_file_type(path, strict, os.path.splitext) + + def _guess_file_type(self, path, strict, splitext): + base, ext = splitext(path) + while (ext_lower := ext.lower()) in self.suffix_map: + base, ext = splitext(base + self.suffix_map[ext_lower]) + # encodings_map is case sensitive if ext in self.encodings_map: encoding = self.encodings_map[ext] - base, ext = posixpath.splitext(base) + base, ext = splitext(base) else: encoding = None + ext = ext.lower() types_map = self.types_map[True] if ext in types_map: return types_map[ext], encoding - elif ext.lower() in types_map: - return types_map[ext.lower()], encoding elif strict: return None, encoding types_map = self.types_map[False] if ext in types_map: return types_map[ext], encoding - elif ext.lower() in types_map: - return types_map[ext.lower()], encoding else: return None, encoding @@ -163,13 +205,13 @@ def guess_all_extensions(self, type, strict=True): Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, - but would be mapped to the MIME type `type' by guess_type(). + but would be mapped to the MIME type 'type' by guess_type(). - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ type = type.lower() - extensions = self.types_map_inv[True].get(type, []) + extensions = list(self.types_map_inv[True].get(type, [])) if not strict: for ext in self.types_map_inv[False].get(type, []): if ext not in extensions: @@ -182,11 +224,11 @@ def guess_extension(self, type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ extensions = self.guess_all_extensions(type, strict) @@ -213,10 +255,7 @@ def readfp(self, fp, strict=True): list of standard types, else to the list of non-standard types. """ - while 1: - line = fp.readline() - if not line: - break + while line := fp.readline(): words = line.split() for i in range(len(words)): if words[i][0] == '#': @@ -237,10 +276,21 @@ def read_windows_registry(self, strict=True): types. """ - # Windows only - if not _winreg: + if not _mimetypes_read_windows_registry and not _winreg: return + add_type = self.add_type + if strict: + add_type = lambda type, ext: self.add_type(type, ext, True) + + # Accelerated function if it is available + if _mimetypes_read_windows_registry: + _mimetypes_read_windows_registry(add_type) + elif _winreg: + self._read_windows_registry(add_type) + + @classmethod + def _read_windows_registry(cls, add_type): def enum_types(mimedb): i = 0 while True: @@ -265,7 +315,7 @@ def enum_types(mimedb): subkey, 'Content Type') if datatype != _winreg.REG_SZ: continue - self.add_type(mimetype, subkeyname, strict) + add_type(mimetype, subkeyname) except OSError: continue @@ -284,7 +334,7 @@ def guess_type(url, strict=True): to ".tar.gz". (This is table-driven too, using the dictionary suffix_map). - Optional `strict' argument when false adds a bunch of commonly found, but + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -292,17 +342,27 @@ def guess_type(url, strict=True): return _db.guess_type(url, strict) +def guess_file_type(path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path instead of URL. + """ + if _db is None: + init() + return _db.guess_file_type(path, strict=strict) + + def guess_all_extensions(type, strict=True): """Guess the extensions for a file based on its MIME type. Return value is a list of strings giving the possible filename extensions, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data - stream, but would be mapped to the MIME type `type' by - guess_type(). If no extension can be guessed for `type', None + stream, but would be mapped to the MIME type 'type' by + guess_type(). If no extension can be guessed for 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -315,10 +375,10 @@ def guess_extension(type, strict=True): Return value is a string giving a filename extension, including the leading dot ('.'). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the - MIME type `type' by guess_type(). If no extension can be guessed for - `type', None is returned. + MIME type 'type' by guess_type(). If no extension can be guessed for + 'type', None is returned. - Optional `strict' argument when false adds a bunch of commonly found, + Optional 'strict' argument when false adds a bunch of commonly found, but non-standard types. """ if _db is None: @@ -349,8 +409,8 @@ def init(files=None): if files is None or _db is None: db = MimeTypes() - if _winreg: - db.read_windows_registry() + # Quick return if not supported + db.read_windows_registry() if files is None: files = knownfiles @@ -359,6 +419,9 @@ def init(files=None): else: db = _db + # Lazy import to improve module import time + import os + for file in files: if os.path.isfile(file): db.read(file) @@ -401,23 +464,28 @@ def _default_mime_types(): '.Z': 'compress', '.bz2': 'bzip2', '.xz': 'xz', + '.br': 'br', } # Before adding new types, make sure they are either registered with IANA, - # at http://www.iana.org/assignments/media-types + # at https://www.iana.org/assignments/media-types/media-types.xhtml # or extensions, i.e. using the x- prefix # If you add to these, please keep them sorted by mime type. # Make sure the entry with the preferred file extension for a particular mime type # appears before any others of the same mimetype. types_map = _types_map_default = { - '.js' : 'application/javascript', - '.mjs' : 'application/javascript', + '.js' : 'text/javascript', + '.mjs' : 'text/javascript', + '.epub' : 'application/epub+zip', + '.gz' : 'application/gzip', '.json' : 'application/json', '.webmanifest': 'application/manifest+json', '.doc' : 'application/msword', '.dot' : 'application/msword', '.wiz' : 'application/msword', + '.nq' : 'application/n-quads', + '.nt' : 'application/n-triples', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', '.dll' : 'application/octet-stream', @@ -426,24 +494,37 @@ def _default_mime_types(): '.obj' : 'application/octet-stream', '.so' : 'application/octet-stream', '.oda' : 'application/oda', + '.ogx' : 'application/ogg', '.pdf' : 'application/pdf', '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', '.ai' : 'application/postscript', '.eps' : 'application/postscript', + '.trig' : 'application/trig', '.m3u' : 'application/vnd.apple.mpegurl', '.m3u8' : 'application/vnd.apple.mpegurl', '.xls' : 'application/vnd.ms-excel', '.xlb' : 'application/vnd.ms-excel', + '.eot' : 'application/vnd.ms-fontobject', '.ppt' : 'application/vnd.ms-powerpoint', '.pot' : 'application/vnd.ms-powerpoint', '.ppa' : 'application/vnd.ms-powerpoint', '.pps' : 'application/vnd.ms-powerpoint', '.pwz' : 'application/vnd.ms-powerpoint', + '.odg' : 'application/vnd.oasis.opendocument.graphics', + '.odp' : 'application/vnd.oasis.opendocument.presentation', + '.ods' : 'application/vnd.oasis.opendocument.spreadsheet', + '.odt' : 'application/vnd.oasis.opendocument.text', + '.pptx' : 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.xlsx' : 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.docx' : 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.rar' : 'application/vnd.rar', '.wasm' : 'application/wasm', + '.7z' : 'application/x-7z-compressed', '.bcpio' : 'application/x-bcpio', '.cpio' : 'application/x-cpio', '.csh' : 'application/x-csh', + '.deb' : 'application/x-debian-package', '.dvi' : 'application/x-dvi', '.gtar' : 'application/x-gtar', '.hdf' : 'application/x-hdf', @@ -453,10 +534,12 @@ def _default_mime_types(): '.cdf' : 'application/x-netcdf', '.nc' : 'application/x-netcdf', '.p12' : 'application/x-pkcs12', + '.php' : 'application/x-httpd-php', '.pfx' : 'application/x-pkcs12', '.ram' : 'application/x-pn-realaudio', '.pyc' : 'application/x-python-code', '.pyo' : 'application/x-python-code', + '.rpm' : 'application/x-rpm', '.sh' : 'application/x-sh', '.shar' : 'application/x-shar', '.swf' : 'application/x-shockwave-flash', @@ -479,29 +562,61 @@ def _default_mime_types(): '.rdf' : 'application/xml', '.wsdl' : 'application/xml', '.xpdl' : 'application/xml', + '.yaml' : 'application/yaml', + '.yml' : 'application/yaml', '.zip' : 'application/zip', + '.3gp' : 'audio/3gpp', + '.3gpp' : 'audio/3gpp', + '.3g2' : 'audio/3gpp2', + '.3gpp2' : 'audio/3gpp2', + '.aac' : 'audio/aac', + '.adts' : 'audio/aac', + '.loas' : 'audio/aac', + '.ass' : 'audio/aac', '.au' : 'audio/basic', '.snd' : 'audio/basic', + '.flac' : 'audio/flac', + '.mka' : 'audio/matroska', + '.m4a' : 'audio/mp4', '.mp3' : 'audio/mpeg', '.mp2' : 'audio/mpeg', + '.ogg' : 'audio/ogg', + '.opus' : 'audio/opus', '.aif' : 'audio/x-aiff', '.aifc' : 'audio/x-aiff', '.aiff' : 'audio/x-aiff', '.ra' : 'audio/x-pn-realaudio', - '.wav' : 'audio/x-wav', + '.wav' : 'audio/vnd.wave', + '.otf' : 'font/otf', + '.ttf' : 'font/ttf', + '.weba' : 'audio/webm', + '.woff' : 'font/woff', + '.woff2' : 'font/woff2', + '.avif' : 'image/avif', '.bmp' : 'image/bmp', + '.emf' : 'image/emf', + '.fits' : 'image/fits', + '.g3' : 'image/g3fax', '.gif' : 'image/gif', '.ief' : 'image/ief', + '.jp2' : 'image/jp2', '.jpg' : 'image/jpeg', '.jpe' : 'image/jpeg', '.jpeg' : 'image/jpeg', + '.jpm' : 'image/jpm', + '.jpx' : 'image/jpx', + '.heic' : 'image/heic', + '.heif' : 'image/heif', '.png' : 'image/png', '.svg' : 'image/svg+xml', + '.t38' : 'image/t38', '.tiff' : 'image/tiff', '.tif' : 'image/tiff', + '.tfx' : 'image/tiff-fx', '.ico' : 'image/vnd.microsoft.icon', + '.webp' : 'image/webp', + '.wmf' : 'image/wmf', '.ras' : 'image/x-cmu-raster', - '.bmp' : 'image/x-ms-bmp', '.pnm' : 'image/x-portable-anymap', '.pbm' : 'image/x-portable-bitmap', '.pgm' : 'image/x-portable-graymap', @@ -514,34 +629,49 @@ def _default_mime_types(): '.mht' : 'message/rfc822', '.mhtml' : 'message/rfc822', '.nws' : 'message/rfc822', + '.gltf' : 'model/gltf+json', + '.glb' : 'model/gltf-binary', + '.stl' : 'model/stl', '.css' : 'text/css', '.csv' : 'text/csv', '.html' : 'text/html', '.htm' : 'text/html', + '.md' : 'text/markdown', + '.markdown': 'text/markdown', + '.n3' : 'text/n3', '.txt' : 'text/plain', '.bat' : 'text/plain', '.c' : 'text/plain', '.h' : 'text/plain', '.ksh' : 'text/plain', '.pl' : 'text/plain', + '.srt' : 'text/plain', '.rtx' : 'text/richtext', + '.rtf' : 'text/rtf', '.tsv' : 'text/tab-separated-values', + '.vtt' : 'text/vtt', '.py' : 'text/x-python', + '.rst' : 'text/x-rst', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', '.sgml' : 'text/x-sgml', '.vcf' : 'text/x-vcard', '.xml' : 'text/xml', + '.mkv' : 'video/matroska', + '.mk3d' : 'video/matroska-3d', '.mp4' : 'video/mp4', '.mpeg' : 'video/mpeg', '.m1v' : 'video/mpeg', '.mpa' : 'video/mpeg', '.mpe' : 'video/mpeg', '.mpg' : 'video/mpeg', + '.ogv' : 'video/ogg', '.mov' : 'video/quicktime', '.qt' : 'video/quicktime', '.webm' : 'video/webm', - '.avi' : 'video/x-msvideo', + '.avi' : 'video/vnd.avi', + '.m4v' : 'video/x-m4v', + '.wmv' : 'video/x-ms-wmv', '.movie' : 'video/x-sgi-movie', } @@ -551,6 +681,7 @@ def _default_mime_types(): # Please sort these too common_types = _common_types_default = { '.rtf' : 'application/rtf', + '.apk' : 'application/vnd.android.package-archive', '.midi': 'audio/midi', '.mid' : 'audio/midi', '.jpg' : 'image/jpg', @@ -564,51 +695,53 @@ def _default_mime_types(): _default_mime_types() -def _main(): - import getopt - - USAGE = """\ -Usage: mimetypes.py [options] type - -Options: - --help / -h -- print this message and exit - --lenient / -l -- additionally search of some common, but non-standard - types. - --extension / -e -- guess extension instead of type - -More than one type argument may be given. -""" - - def usage(code, msg=''): - print(USAGE) - if msg: print(msg) - sys.exit(code) - - try: - opts, args = getopt.getopt(sys.argv[1:], 'hle', - ['help', 'lenient', 'extension']) - except getopt.error as msg: - usage(1, msg) - - strict = 1 - extension = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-l', '--lenient'): - strict = 0 - elif opt in ('-e', '--extension'): - extension = 1 - for gtype in args: - if extension: - guess = guess_extension(gtype, strict) - if not guess: print("I don't know anything about type", gtype) - else: print(guess) - else: - guess, encoding = guess_type(gtype, strict) - if not guess: print("I don't know anything about type", gtype) - else: print('type:', guess, 'encoding:', encoding) +def _parse_args(args): + from argparse import ArgumentParser + + parser = ArgumentParser( + description='map filename extensions to MIME types', color=True + ) + parser.add_argument( + '-e', '--extension', + action='store_true', + help='guess extension instead of type' + ) + parser.add_argument( + '-l', '--lenient', + action='store_true', + help='additionally search for common but non-standard types' + ) + parser.add_argument('type', nargs='+', help='a type to search') + args = parser.parse_args(args) + return args, parser.format_help() + + +def _main(args=None): + """Run the mimetypes command-line interface and return a text to print.""" + args, help_text = _parse_args(args) + + results = [] + if args.extension: + for gtype in args.type: + guess = guess_extension(gtype, not args.lenient) + if guess: + results.append(str(guess)) + else: + results.append(f"error: unknown type {gtype}") + return results + else: + for gtype in args.type: + guess, encoding = guess_type(gtype, not args.lenient) + if guess: + results.append(f"type: {guess} encoding: {encoding}") + else: + results.append(f"error: media type unknown for {gtype}") + return results if __name__ == '__main__': - _main() + import sys + + results = _main() + print("\n".join(results)) + sys.exit(any(result.startswith("error: ") for result in results)) diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py new file mode 100644 index 00000000000..ac478ee7f51 --- /dev/null +++ b/Lib/modulefinder.py @@ -0,0 +1,671 @@ +"""Find modules used by a script, using introspection.""" + +import dis +import importlib._bootstrap_external +import importlib.machinery +import marshal +import os +import io +import sys + +# Old imp constants: + +_SEARCH_ERROR = 0 +_PY_SOURCE = 1 +_PY_COMPILED = 2 +_C_EXTENSION = 3 +_PKG_DIRECTORY = 5 +_C_BUILTIN = 6 +_PY_FROZEN = 7 + +# Modulefinder does a good job at simulating Python's, but it can not +# handle __path__ modifications packages make at runtime. Therefore there +# is a mechanism whereby you can register extra paths in this map for a +# package, and it will be honored. + +# Note this is a mapping is lists of paths. +packagePathMap = {} + +# A Public interface +def AddPackagePath(packagename, path): + packagePathMap.setdefault(packagename, []).append(path) + +replacePackageMap = {} + +# This ReplacePackage mechanism allows modulefinder to work around +# situations in which a package injects itself under the name +# of another package into sys.modules at runtime by calling +# ReplacePackage("real_package_name", "faked_package_name") +# before running ModuleFinder. + +def ReplacePackage(oldname, newname): + replacePackageMap[oldname] = newname + + +def _find_module(name, path=None): + """An importlib reimplementation of imp.find_module (for our purposes).""" + + # It's necessary to clear the caches for our Finder first, in case any + # modules are being added/deleted/modified at runtime. In particular, + # test_modulefinder.py changes file tree contents in a cache-breaking way: + + importlib.machinery.PathFinder.invalidate_caches() + + spec = importlib.machinery.PathFinder.find_spec(name, path) + + if spec is None: + raise ImportError("No module named {name!r}".format(name=name), name=name) + + # Some special cases: + + if spec.loader is importlib.machinery.BuiltinImporter: + return None, None, ("", "", _C_BUILTIN) + + if spec.loader is importlib.machinery.FrozenImporter: + return None, None, ("", "", _PY_FROZEN) + + file_path = spec.origin + + if spec.loader.is_package(name): + return None, os.path.dirname(file_path), ("", "", _PKG_DIRECTORY) + + if isinstance(spec.loader, importlib.machinery.SourceFileLoader): + kind = _PY_SOURCE + + elif isinstance( + spec.loader, ( + importlib.machinery.ExtensionFileLoader, + importlib.machinery.AppleFrameworkLoader, + ) + ): + kind = _C_EXTENSION + + elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): + kind = _PY_COMPILED + + else: # Should never happen. + return None, None, ("", "", _SEARCH_ERROR) + + file = io.open_code(file_path) + suffix = os.path.splitext(file_path)[-1] + + return file, file_path, (suffix, "rb", kind) + + +class Module: + + def __init__(self, name, file=None, path=None): + self.__name__ = name + self.__file__ = file + self.__path__ = path + self.__code__ = None + # The set of global names that are assigned to in the module. + # This includes those names imported through starimports of + # Python modules. + self.globalnames = {} + # The set of starimports this module did that could not be + # resolved, ie. a starimport from a non-Python module. + self.starimports = {} + + def __repr__(self): + s = "Module(%r" % (self.__name__,) + if self.__file__ is not None: + s = s + ", %r" % (self.__file__,) + if self.__path__ is not None: + s = s + ", %r" % (self.__path__,) + s = s + ")" + return s + +class ModuleFinder: + + def __init__(self, path=None, debug=0, excludes=None, replace_paths=None): + if path is None: + path = sys.path + self.path = path + self.modules = {} + self.badmodules = {} + self.debug = debug + self.indent = 0 + self.excludes = excludes if excludes is not None else [] + self.replace_paths = replace_paths if replace_paths is not None else [] + self.processed_paths = [] # Used in debugging only + + def msg(self, level, str, *args): + if level <= self.debug: + for i in range(self.indent): + print(" ", end=' ') + print(str, end=' ') + for arg in args: + print(repr(arg), end=' ') + print() + + def msgin(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent + 1 + self.msg(*args) + + def msgout(self, *args): + level = args[0] + if level <= self.debug: + self.indent = self.indent - 1 + self.msg(*args) + + def run_script(self, pathname): + self.msg(2, "run_script", pathname) + with io.open_code(pathname) as fp: + stuff = ("", "rb", _PY_SOURCE) + self.load_module('__main__', fp, pathname, stuff) + + def load_file(self, pathname): + dir, name = os.path.split(pathname) + name, ext = os.path.splitext(name) + with io.open_code(pathname) as fp: + stuff = (ext, "rb", _PY_SOURCE) + self.load_module(name, fp, pathname, stuff) + + def import_hook(self, name, caller=None, fromlist=None, level=-1): + self.msg(3, "import_hook", name, caller, fromlist, level) + parent = self.determine_parent(caller, level=level) + q, tail = self.find_head_package(parent, name) + m = self.load_tail(q, tail) + if not fromlist: + return q + if m.__path__: + self.ensure_fromlist(m, fromlist) + return None + + def determine_parent(self, caller, level=-1): + self.msgin(4, "determine_parent", caller, level) + if not caller or level == 0: + self.msgout(4, "determine_parent -> None") + return None + pname = caller.__name__ + if level >= 1: # relative import + if caller.__path__: + level -= 1 + if level == 0: + parent = self.modules[pname] + assert parent is caller + self.msgout(4, "determine_parent ->", parent) + return parent + if pname.count(".") < level: + raise ImportError("relative importpath too deep") + pname = ".".join(pname.split(".")[:-level]) + parent = self.modules[pname] + self.msgout(4, "determine_parent ->", parent) + return parent + if caller.__path__: + parent = self.modules[pname] + assert caller is parent + self.msgout(4, "determine_parent ->", parent) + return parent + if '.' in pname: + i = pname.rfind('.') + pname = pname[:i] + parent = self.modules[pname] + assert parent.__name__ == pname + self.msgout(4, "determine_parent ->", parent) + return parent + self.msgout(4, "determine_parent -> None") + return None + + def find_head_package(self, parent, name): + self.msgin(4, "find_head_package", parent, name) + if '.' in name: + i = name.find('.') + head = name[:i] + tail = name[i+1:] + else: + head = name + tail = "" + if parent: + qname = "%s.%s" % (parent.__name__, head) + else: + qname = head + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + if parent: + qname = head + parent = None + q = self.import_module(head, qname, parent) + if q: + self.msgout(4, "find_head_package ->", (q, tail)) + return q, tail + self.msgout(4, "raise ImportError: No module named", qname) + raise ImportError("No module named " + qname) + + def load_tail(self, q, tail): + self.msgin(4, "load_tail", q, tail) + m = q + while tail: + i = tail.find('.') + if i < 0: i = len(tail) + head, tail = tail[:i], tail[i+1:] + mname = "%s.%s" % (m.__name__, head) + m = self.import_module(head, mname, m) + if not m: + self.msgout(4, "raise ImportError: No module named", mname) + raise ImportError("No module named " + mname) + self.msgout(4, "load_tail ->", m) + return m + + def ensure_fromlist(self, m, fromlist, recursive=0): + self.msg(4, "ensure_fromlist", m, fromlist, recursive) + for sub in fromlist: + if sub == "*": + if not recursive: + all = self.find_all_submodules(m) + if all: + self.ensure_fromlist(m, all, 1) + elif not hasattr(m, sub): + subname = "%s.%s" % (m.__name__, sub) + submod = self.import_module(sub, subname, m) + if not submod: + raise ImportError("No module named " + subname) + + def find_all_submodules(self, m): + if not m.__path__: + return + modules = {} + # 'suffixes' used to be a list hardcoded to [".py", ".pyc"]. + # But we must also collect Python extension modules - although + # we cannot separate normal dlls from Python extensions. + suffixes = [] + suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] + suffixes += importlib.machinery.SOURCE_SUFFIXES[:] + suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] + for dir in m.__path__: + try: + names = os.listdir(dir) + except OSError: + self.msg(2, "can't list directory", dir) + continue + for name in names: + mod = None + for suff in suffixes: + n = len(suff) + if name[-n:] == suff: + mod = name[:-n] + break + if mod and mod != "__init__": + modules[mod] = mod + return modules.keys() + + def import_module(self, partname, fqname, parent): + self.msgin(3, "import_module", partname, fqname, parent) + try: + m = self.modules[fqname] + except KeyError: + pass + else: + self.msgout(3, "import_module ->", m) + return m + if fqname in self.badmodules: + self.msgout(3, "import_module -> None") + return None + if parent and parent.__path__ is None: + self.msgout(3, "import_module -> None") + return None + try: + fp, pathname, stuff = self.find_module(partname, + parent and parent.__path__, parent) + except ImportError: + self.msgout(3, "import_module ->", None) + return None + + try: + m = self.load_module(fqname, fp, pathname, stuff) + finally: + if fp: + fp.close() + if parent: + setattr(parent, partname, m) + self.msgout(3, "import_module ->", m) + return m + + def load_module(self, fqname, fp, pathname, file_info): + suffix, mode, type = file_info + self.msgin(2, "load_module", fqname, fp and "fp", pathname) + if type == _PKG_DIRECTORY: + m = self.load_package(fqname, pathname) + self.msgout(2, "load_module ->", m) + return m + if type == _PY_SOURCE: + co = compile(fp.read(), pathname, 'exec') + elif type == _PY_COMPILED: + try: + data = fp.read() + importlib._bootstrap_external._classify_pyc(data, fqname, {}) + except ImportError as exc: + self.msgout(2, "raise ImportError: " + str(exc), pathname) + raise + co = marshal.loads(memoryview(data)[16:]) + else: + co = None + m = self.add_module(fqname) + m.__file__ = pathname + if co: + if self.replace_paths: + co = self.replace_paths_in_code(co) + m.__code__ = co + self.scan_code(co, m) + self.msgout(2, "load_module ->", m) + return m + + def _add_badmodule(self, name, caller): + if name not in self.badmodules: + self.badmodules[name] = {} + if caller: + self.badmodules[name][caller.__name__] = 1 + else: + self.badmodules[name]["-"] = 1 + + def _safe_import_hook(self, name, caller, fromlist, level=-1): + # wrapper for self.import_hook() that won't raise ImportError + if name in self.badmodules: + self._add_badmodule(name, caller) + return + try: + self.import_hook(name, caller, level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + self._add_badmodule(name, caller) + except SyntaxError as msg: + self.msg(2, "SyntaxError:", str(msg)) + self._add_badmodule(name, caller) + else: + if fromlist: + for sub in fromlist: + fullname = name + "." + sub + if fullname in self.badmodules: + self._add_badmodule(fullname, caller) + continue + try: + self.import_hook(name, caller, [sub], level=level) + except ImportError as msg: + self.msg(2, "ImportError:", str(msg)) + self._add_badmodule(fullname, caller) + + def scan_opcodes(self, co): + # Scan the code, and yield 'interesting' opcode combinations + for name in dis._find_store_names(co): + yield "store", (name,) + for name, level, fromlist in dis._find_imports(co): + if level == 0: # absolute import + yield "absolute_import", (fromlist, name) + else: # relative import + yield "relative_import", (level, fromlist, name) + + def scan_code(self, co, m): + code = co.co_code + scanner = self.scan_opcodes + for what, args in scanner(co): + if what == "store": + name, = args + m.globalnames[name] = 1 + elif what == "absolute_import": + fromlist, name = args + have_star = 0 + if fromlist is not None: + if "*" in fromlist: + have_star = 1 + fromlist = [f for f in fromlist if f != "*"] + self._safe_import_hook(name, m, fromlist, level=0) + if have_star: + # We've encountered an "import *". If it is a Python module, + # the code has already been parsed and we can suck out the + # global names. + mm = None + if m.__path__: + # At this point we don't know whether 'name' is a + # submodule of 'm' or a global module. Let's just try + # the full name first. + mm = self.modules.get(m.__name__ + "." + name) + if mm is None: + mm = self.modules.get(name) + if mm is not None: + m.globalnames.update(mm.globalnames) + m.starimports.update(mm.starimports) + if mm.__code__ is None: + m.starimports[name] = 1 + else: + m.starimports[name] = 1 + elif what == "relative_import": + level, fromlist, name = args + if name: + self._safe_import_hook(name, m, fromlist, level=level) + else: + parent = self.determine_parent(m, level=level) + self._safe_import_hook(parent.__name__, None, fromlist, level=0) + else: + # We don't expect anything else from the generator. + raise RuntimeError(what) + + for c in co.co_consts: + if isinstance(c, type(co)): + self.scan_code(c, m) + + def load_package(self, fqname, pathname): + self.msgin(2, "load_package", fqname, pathname) + newname = replacePackageMap.get(fqname) + if newname: + fqname = newname + m = self.add_module(fqname) + m.__file__ = pathname + m.__path__ = [pathname] + + # As per comment at top of file, simulate runtime __path__ additions. + m.__path__ = m.__path__ + packagePathMap.get(fqname, []) + + fp, buf, stuff = self.find_module("__init__", m.__path__) + try: + self.load_module(fqname, fp, buf, stuff) + self.msgout(2, "load_package ->", m) + return m + finally: + if fp: + fp.close() + + def add_module(self, fqname): + if fqname in self.modules: + return self.modules[fqname] + self.modules[fqname] = m = Module(fqname) + return m + + def find_module(self, name, path, parent=None): + if parent is not None: + # assert path is not None + fullname = parent.__name__+'.'+name + else: + fullname = name + if fullname in self.excludes: + self.msgout(3, "find_module -> Excluded", fullname) + raise ImportError(name) + + if path is None: + if name in sys.builtin_module_names: + return (None, None, ("", "", _C_BUILTIN)) + + path = self.path + + return _find_module(name, path) + + def report(self): + """Print a report to stdout, listing the found modules with their + paths, as well as modules that are missing, or seem to be missing. + """ + print() + print(" %-25s %s" % ("Name", "File")) + print(" %-25s %s" % ("----", "----")) + # Print modules found + keys = sorted(self.modules.keys()) + for key in keys: + m = self.modules[key] + if m.__path__: + print("P", end=' ') + else: + print("m", end=' ') + print("%-25s" % key, m.__file__ or "") + + # Print missing modules + missing, maybe = self.any_missing_maybe() + if missing: + print() + print("Missing modules:") + for name in missing: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + # Print modules that may be missing, but then again, maybe not... + if maybe: + print() + print("Submodules that appear to be missing, but could also be", end=' ') + print("global names in the parent package:") + for name in maybe: + mods = sorted(self.badmodules[name].keys()) + print("?", name, "imported from", ', '.join(mods)) + + def any_missing(self): + """Return a list of modules that appear to be missing. Use + any_missing_maybe() if you want to know which modules are + certain to be missing, and which *may* be missing. + """ + missing, maybe = self.any_missing_maybe() + return missing + maybe + + def any_missing_maybe(self): + """Return two lists, one with modules that are certainly missing + and one with modules that *may* be missing. The latter names could + either be submodules *or* just global names in the package. + + The reason it can't always be determined is that it's impossible to + tell which names are imported when "from module import *" is done + with an extension module, short of actually importing it. + """ + missing = [] + maybe = [] + for name in self.badmodules: + if name in self.excludes: + continue + i = name.rfind(".") + if i < 0: + missing.append(name) + continue + subname = name[i+1:] + pkgname = name[:i] + pkg = self.modules.get(pkgname) + if pkg is not None: + if pkgname in self.badmodules[name]: + # The package tried to import this module itself and + # failed. It's definitely missing. + missing.append(name) + elif subname in pkg.globalnames: + # It's a global in the package: definitely not missing. + pass + elif pkg.starimports: + # It could be missing, but the package did an "import *" + # from a non-Python module, so we simply can't be sure. + maybe.append(name) + else: + # It's not a global in the package, the package didn't + # do funny star imports, it's very likely to be missing. + # The symbol could be inserted into the package from the + # outside, but since that's not good style we simply list + # it missing. + missing.append(name) + else: + missing.append(name) + missing.sort() + maybe.sort() + return missing, maybe + + def replace_paths_in_code(self, co): + new_filename = original_filename = os.path.normpath(co.co_filename) + for f, r in self.replace_paths: + if original_filename.startswith(f): + new_filename = r + original_filename[len(f):] + break + + if self.debug and original_filename not in self.processed_paths: + if new_filename != original_filename: + self.msgout(2, "co_filename %r changed to %r" \ + % (original_filename,new_filename,)) + else: + self.msgout(2, "co_filename %r remains unchanged" \ + % (original_filename,)) + self.processed_paths.append(original_filename) + + consts = list(co.co_consts) + for i in range(len(consts)): + if isinstance(consts[i], type(co)): + consts[i] = self.replace_paths_in_code(consts[i]) + + return co.replace(co_consts=tuple(consts), co_filename=new_filename) + + +def test(): + # Parse command line + import getopt + try: + opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") + except getopt.error as msg: + print(msg) + return + + # Process options + debug = 1 + domods = 0 + addpath = [] + exclude = [] + for o, a in opts: + if o == '-d': + debug = debug + 1 + if o == '-m': + domods = 1 + if o == '-p': + addpath = addpath + a.split(os.pathsep) + if o == '-q': + debug = 0 + if o == '-x': + exclude.append(a) + + # Provide default arguments + if not args: + script = "hello.py" + else: + script = args[0] + + # Set the path based on sys.path and the script directory + path = sys.path[:] + path[0] = os.path.dirname(script) + path = addpath + path + if debug > 1: + print("path:") + for item in path: + print(" ", repr(item)) + + # Create the module finder and turn its crank + mf = ModuleFinder(path, debug, exclude) + for arg in args[1:]: + if arg == '-m': + domods = 1 + continue + if domods: + if arg[-2:] == '.*': + mf.import_hook(arg[:-2], None, ["*"]) + else: + mf.import_hook(arg) + else: + mf.load_file(arg) + mf.run_script(script) + mf.report() + return mf # for -i debugging + + +if __name__ == '__main__': + try: + mf = test() + except KeyboardInterrupt: + print("\n[interrupted]") diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 510e4b5aba4..abd88adf76e 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -9,6 +9,7 @@ __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ] +import errno import io import os import sys @@ -18,7 +19,6 @@ import tempfile import itertools -import _multiprocessing from . import util @@ -27,6 +27,7 @@ _ForkingPickler = reduction.ForkingPickler try: + import _multiprocessing import _winapi from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE except ImportError: @@ -73,12 +74,7 @@ def arbitrary_address(family): if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': - # Prefer abstract sockets if possible to avoid problems with the address - # size. When coding portable applications, some implementations have - # sun_path as short as 92 bytes in the sockaddr_un struct. - if util.abstract_sockets_supported: - return f"\0listener-{os.getpid()}-{next(_mmap_counter)}" - return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) + return tempfile.mktemp(prefix='sock-', dir=util.get_temp_dir()) elif family == 'AF_PIPE': return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % (os.getpid(), next(_mmap_counter)), dir="") @@ -188,10 +184,9 @@ def send_bytes(self, buf, offset=0, size=None): self._check_closed() self._check_writable() m = memoryview(buf) - # HACK for byte-indexing of non-bytewise buffers (e.g. array.array) if m.itemsize > 1: - m = memoryview(bytes(m)) - n = len(m) + m = m.cast('B') + n = m.nbytes if offset < 0: raise ValueError("offset is negative") if n < offset: @@ -277,12 +272,22 @@ class PipeConnection(_ConnectionBase): with FILE_FLAG_OVERLAPPED. """ _got_empty_message = False + _send_ov = None def _close(self, _CloseHandle=_winapi.CloseHandle): + ov = self._send_ov + if ov is not None: + # Interrupt WaitForMultipleObjects() in _send_bytes() + ov.cancel() _CloseHandle(self._handle) def _send_bytes(self, buf): + if self._send_ov is not None: + # A connection should only be used by a single thread + raise ValueError("concurrent send_bytes() calls " + "are not supported") ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True) + self._send_ov = ov try: if err == _winapi.ERROR_IO_PENDING: waitres = _winapi.WaitForMultipleObjects( @@ -292,7 +297,13 @@ def _send_bytes(self, buf): ov.cancel() raise finally: + self._send_ov = None nwritten, err = ov.GetOverlappedResult(True) + if err == _winapi.ERROR_OPERATION_ABORTED: + # close() was called by another thread while + # WaitForMultipleObjects() was waiting for the overlapped + # operation. + raise OSError(errno.EPIPE, "handle is closed") assert err == 0 assert nwritten == len(buf) @@ -465,8 +476,9 @@ def accept(self): ''' if self._listener is None: raise OSError('listener is closed') + c = self._listener.accept() - if self._authkey: + if self._authkey is not None: deliver_challenge(c, self._authkey) answer_challenge(c, self._authkey) return c @@ -728,39 +740,227 @@ def PipeClient(address): # Authentication stuff # -MESSAGE_LENGTH = 20 +MESSAGE_LENGTH = 40 # MUST be > 20 -CHALLENGE = b'#CHALLENGE#' -WELCOME = b'#WELCOME#' -FAILURE = b'#FAILURE#' +_CHALLENGE = b'#CHALLENGE#' +_WELCOME = b'#WELCOME#' +_FAILURE = b'#FAILURE#' -def deliver_challenge(connection, authkey): +# multiprocessing.connection Authentication Handshake Protocol Description +# (as documented for reference after reading the existing code) +# ============================================================================= +# +# On Windows: native pipes with "overlapped IO" are used to send the bytes, +# instead of the length prefix SIZE scheme described below. (ie: the OS deals +# with message sizes for us) +# +# Protocol error behaviors: +# +# On POSIX, any failure to receive the length prefix into SIZE, for SIZE greater +# than the requested maxsize to receive, or receiving fewer than SIZE bytes +# results in the connection being closed and auth to fail. +# +# On Windows, receiving too few bytes is never a low level _recv_bytes read +# error, receiving too many will trigger an error only if receive maxsize +# value was larger than 128 OR the if the data arrived in smaller pieces. +# +# Serving side Client side +# ------------------------------ --------------------------------------- +# 0. Open a connection on the pipe. +# 1. Accept connection. +# 2. Random 20+ bytes -> MESSAGE +# Modern servers always send +# more than 20 bytes and include +# a {digest} prefix on it with +# their preferred HMAC digest. +# Legacy ones send ==20 bytes. +# 3. send 4 byte length (net order) +# prefix followed by: +# b'#CHALLENGE#' + MESSAGE +# 4. Receive 4 bytes, parse as network byte +# order integer. If it is -1, receive an +# additional 8 bytes, parse that as network +# byte order. The result is the length of +# the data that follows -> SIZE. +# 5. Receive min(SIZE, 256) bytes -> M1 +# 6. Assert that M1 starts with: +# b'#CHALLENGE#' +# 7. Strip that prefix from M1 into -> M2 +# 7.1. Parse M2: if it is exactly 20 bytes in +# length this indicates a legacy server +# supporting only HMAC-MD5. Otherwise the +# 7.2. preferred digest is looked up from an +# expected "{digest}" prefix on M2. No prefix +# or unsupported digest? <- AuthenticationError +# 7.3. Put divined algorithm name in -> D_NAME +# 8. Compute HMAC-D_NAME of AUTHKEY, M2 -> C_DIGEST +# 9. Send 4 byte length prefix (net order) +# followed by C_DIGEST bytes. +# 10. Receive 4 or 4+8 byte length +# prefix (#4 dance) -> SIZE. +# 11. Receive min(SIZE, 256) -> C_D. +# 11.1. Parse C_D: legacy servers +# accept it as is, "md5" -> D_NAME +# 11.2. modern servers check the length +# of C_D, IF it is 16 bytes? +# 11.2.1. "md5" -> D_NAME +# and skip to step 12. +# 11.3. longer? expect and parse a "{digest}" +# prefix into -> D_NAME. +# Strip the prefix and store remaining +# bytes in -> C_D. +# 11.4. Don't like D_NAME? <- AuthenticationError +# 12. Compute HMAC-D_NAME of AUTHKEY, +# MESSAGE into -> M_DIGEST. +# 13. Compare M_DIGEST == C_D: +# 14a: Match? Send length prefix & +# b'#WELCOME#' +# <- RETURN +# 14b: Mismatch? Send len prefix & +# b'#FAILURE#' +# <- CLOSE & AuthenticationError +# 15. Receive 4 or 4+8 byte length prefix (net +# order) again as in #4 into -> SIZE. +# 16. Receive min(SIZE, 256) bytes -> M3. +# 17. Compare M3 == b'#WELCOME#': +# 17a. Match? <- RETURN +# 17b. Mismatch? <- CLOSE & AuthenticationError +# +# If this RETURNed, the connection remains open: it has been authenticated. +# +# Length prefixes are used consistently. Even on the legacy protocol, this +# was good fortune and allowed us to evolve the protocol by using the length +# of the opening challenge or length of the returned digest as a signal as +# to which protocol the other end supports. + +_ALLOWED_DIGESTS = frozenset( + {b'md5', b'sha256', b'sha384', b'sha3_256', b'sha3_384'}) +_MAX_DIGEST_LEN = max(len(_) for _ in _ALLOWED_DIGESTS) + +# Old hmac-md5 only server versions from Python <=3.11 sent a message of this +# length. It happens to not match the length of any supported digest so we can +# use a message of this length to indicate that we should work in backwards +# compatible md5-only mode without a {digest_name} prefix on our response. +_MD5ONLY_MESSAGE_LENGTH = 20 +_MD5_DIGEST_LEN = 16 +_LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN) + + +def _get_digest_name_and_payload(message): # type: (bytes) -> tuple[str, bytes] + """Returns a digest name and the payload for a response hash. + + If a legacy protocol is detected based on the message length + or contents the digest name returned will be empty to indicate + legacy mode where MD5 and no digest prefix should be sent. + """ + # modern message format: b"{digest}payload" longer than 20 bytes + # legacy message format: 16 or 20 byte b"payload" + if len(message) in _LEGACY_LENGTHS: + # Either this was a legacy server challenge, or we're processing + # a reply from a legacy client that sent an unprefixed 16-byte + # HMAC-MD5 response. All messages using the modern protocol will + # be longer than either of these lengths. + return '', message + if (message.startswith(b'{') and + (curly := message.find(b'}', 1, _MAX_DIGEST_LEN+2)) > 0): + digest = message[1:curly] + if digest in _ALLOWED_DIGESTS: + payload = message[curly+1:] + return digest.decode('ascii'), payload + raise AuthenticationError( + 'unsupported message length, missing digest prefix, ' + f'or unsupported digest: {message=}') + + +def _create_response(authkey, message): + """Create a MAC based on authkey and message + + The MAC algorithm defaults to HMAC-MD5, unless MD5 is not available or + the message has a '{digest_name}' prefix. For legacy HMAC-MD5, the response + is the raw MAC, otherwise the response is prefixed with '{digest_name}', + e.g. b'{sha256}abcdefg...' + + Note: The MAC protects the entire message including the digest_name prefix. + """ import hmac + digest_name = _get_digest_name_and_payload(message)[0] + # The MAC protects the entire message: digest header and payload. + if not digest_name: + # Legacy server without a {digest} prefix on message. + # Generate a legacy non-prefixed HMAC-MD5 reply. + try: + return hmac.new(authkey, message, 'md5').digest() + except ValueError: + # HMAC-MD5 is not available (FIPS mode?), fall back to + # HMAC-SHA2-256 modern protocol. The legacy server probably + # doesn't support it and will reject us anyways. :shrug: + digest_name = 'sha256' + # Modern protocol, indicate the digest used in the reply. + response = hmac.new(authkey, message, digest_name).digest() + return b'{%s}%s' % (digest_name.encode('ascii'), response) + + +def _verify_challenge(authkey, message, response): + """Verify MAC challenge + + If our message did not include a digest_name prefix, the client is allowed + to select a stronger digest_name from _ALLOWED_DIGESTS. + + In case our message is prefixed, a client cannot downgrade to a weaker + algorithm, because the MAC is calculated over the entire message + including the '{digest_name}' prefix. + """ + import hmac + response_digest, response_mac = _get_digest_name_and_payload(response) + response_digest = response_digest or 'md5' + try: + expected = hmac.new(authkey, message, response_digest).digest() + except ValueError: + raise AuthenticationError(f'{response_digest=} unsupported') + if len(expected) != len(response_mac): + raise AuthenticationError( + f'expected {response_digest!r} of length {len(expected)} ' + f'got {len(response_mac)}') + if not hmac.compare_digest(expected, response_mac): + raise AuthenticationError('digest received was wrong') + + +def deliver_challenge(connection, authkey: bytes, digest_name='sha256'): if not isinstance(authkey, bytes): raise ValueError( "Authkey must be bytes, not {0!s}".format(type(authkey))) + assert MESSAGE_LENGTH > _MD5ONLY_MESSAGE_LENGTH, "protocol constraint" message = os.urandom(MESSAGE_LENGTH) - connection.send_bytes(CHALLENGE + message) - digest = hmac.new(authkey, message, 'md5').digest() + message = b'{%s}%s' % (digest_name.encode('ascii'), message) + # Even when sending a challenge to a legacy client that does not support + # digest prefixes, they'll take the entire thing as a challenge and + # respond to it with a raw HMAC-MD5. + connection.send_bytes(_CHALLENGE + message) response = connection.recv_bytes(256) # reject large message - if response == digest: - connection.send_bytes(WELCOME) + try: + _verify_challenge(authkey, message, response) + except AuthenticationError: + connection.send_bytes(_FAILURE) + raise else: - connection.send_bytes(FAILURE) - raise AuthenticationError('digest received was wrong') + connection.send_bytes(_WELCOME) -def answer_challenge(connection, authkey): - import hmac + +def answer_challenge(connection, authkey: bytes): if not isinstance(authkey, bytes): raise ValueError( "Authkey must be bytes, not {0!s}".format(type(authkey))) message = connection.recv_bytes(256) # reject large message - assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message - message = message[len(CHALLENGE):] - digest = hmac.new(authkey, message, 'md5').digest() + if not message.startswith(_CHALLENGE): + raise AuthenticationError( + f'Protocol error, expected challenge: {message=}') + message = message[len(_CHALLENGE):] + if len(message) < _MD5ONLY_MESSAGE_LENGTH: + raise AuthenticationError(f'challenge too short: {len(message)} bytes') + digest = _create_response(authkey, message) connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message - if response != WELCOME: + if response != _WELCOME: raise AuthenticationError('digest sent was rejected') # @@ -812,8 +1012,20 @@ def _exhaustive_wait(handles, timeout): # returning the first signalled might create starvation issues.) L = list(handles) ready = [] + # Windows limits WaitForMultipleObjects at 64 handles, and we use a + # few for synchronisation, so we switch to batched waits at 60. + if len(L) > 60: + try: + res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout) + except TimeoutError: + return [] + ready.extend(L[i] for i in res) + if res: + L = [h for i, h in enumerate(L) if i > res[0] & i not in res] + timeout = 0 while L: - res = _winapi.WaitForMultipleObjects(L, False, timeout) + short_L = L[:60] if len(L) > 60 else L + res = _winapi.WaitForMultipleObjects(short_L, False, timeout) if res == WAIT_TIMEOUT: break elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): @@ -943,7 +1155,7 @@ def wait(object_list, timeout=None): return ready # -# Make connection and socket objects sharable if possible +# Make connection and socket objects shareable if possible # if sys.platform == 'win32': diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index 8d0525d5d62..f395e8b04d0 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -145,7 +145,7 @@ def freeze_support(self): '''Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. ''' - if sys.platform == 'win32' and getattr(sys, 'frozen', False): + if self.get_start_method() == 'spawn' and getattr(sys, 'frozen', False): from .spawn import freeze_support freeze_support() @@ -223,6 +223,10 @@ class Process(process.BaseProcess): def _Popen(process_obj): return _default_context.get_context().Process._Popen(process_obj) + @staticmethod + def _after_fork(): + return _default_context.get_context().Process._after_fork() + class DefaultContext(BaseContext): Process = Process @@ -254,6 +258,7 @@ def get_start_method(self, allow_none=False): return self._actual_context._name def get_all_start_methods(self): + """Returns a list of the supported start methods, default first.""" if sys.platform == 'win32': return ['spawn'] else: @@ -283,6 +288,11 @@ def _Popen(process_obj): from .popen_spawn_posix import Popen return Popen(process_obj) + @staticmethod + def _after_fork(): + # process is spawned, nothing to do + pass + class ForkServerProcess(process.BaseProcess): _start_method = 'forkserver' @staticmethod @@ -326,6 +336,11 @@ def _Popen(process_obj): from .popen_spawn_win32 import Popen return Popen(process_obj) + @staticmethod + def _after_fork(): + # process is spawned, nothing to do + pass + class SpawnContext(BaseContext): _name = 'spawn' Process = SpawnProcess diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 22a911a7a29..e243442e7a1 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,3 +1,4 @@ +import atexit import errno import os import selectors @@ -61,7 +62,7 @@ def _stop_unlocked(self): def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' - if not all(type(mod) is str for mod in self._preload_modules): + if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') self._preload_modules = modules_names @@ -126,12 +127,13 @@ def ensure_running(self): cmd = ('from multiprocessing.forkserver import main; ' + 'main(%d, %d, %r, **%r)') + main_kws = {} if self._preload_modules: - desired_keys = {'main_path', 'sys_path'} data = spawn.get_preparation_data('ignore') - data = {x: y for x, y in data.items() if x in desired_keys} - else: - data = {} + if 'sys_path' in data: + main_kws['sys_path'] = data['sys_path'] + if 'init_main_from_path' in data: + main_kws['main_path'] = data['init_main_from_path'] with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -146,7 +148,7 @@ def ensure_running(self): try: fds_to_pass = [listener.fileno(), alive_r] cmd %= (listener.fileno(), alive_r, self._preload_modules, - data) + main_kws) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] @@ -167,6 +169,8 @@ def ensure_running(self): def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): '''Run forkserver.''' if preload: + if sys_path is not None: + sys.path[:] = sys_path if '__main__' in preload and main_path is not None: process.current_process()._inheriting = True try: @@ -179,6 +183,10 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): except ImportError: pass + # gh-135335: flush stdout/stderr in case any of the preloaded modules + # wrote to them, otherwise children might inherit buffered data + util._flush_std_streams() + util._close_stdin() sig_r, sig_w = os.pipe() @@ -271,6 +279,8 @@ def sigchld_handler(*_unused): selector.close() unused_fds = [alive_r, child_w, sig_r, sig_w] unused_fds.extend(pid_to_fd.values()) + atexit._clear() + atexit.register(util._exit_function) code = _serve_one(child_r, fds, unused_fds, old_handlers) @@ -278,6 +288,7 @@ def sigchld_handler(*_unused): sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: + atexit._run_exitfuncs() os._exit(code) else: # Send pid to client process diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 22292c78b7b..ef791c27516 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -49,11 +49,11 @@ def reduce_array(a): reduction.register(array.array, reduce_array) view_types = [type(getattr({}, name)()) for name in ('items','keys','values')] -if view_types[0] is not list: # only needed in Py3.0 - def rebuild_as_list(obj): - return list, (list(obj),) - for view_type in view_types: - reduction.register(view_type, rebuild_as_list) +def rebuild_as_list(obj): + return list, (list(obj),) +for view_type in view_types: + reduction.register(view_type, rebuild_as_list) +del view_type, view_types # # Type for identifying shared objects @@ -90,7 +90,10 @@ def dispatch(c, id, methodname, args=(), kwds={}): kind, result = c.recv() if kind == '#RETURN': return result - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def convert_to_error(kind, result): if kind == '#ERROR': @@ -153,7 +156,7 @@ def __init__(self, registry, address, authkey, serializer): Listener, Client = listener_client[serializer] # do authentication later - self.listener = Listener(address=address, backlog=16) + self.listener = Listener(address=address, backlog=128) self.address = self.listener.address self.id_to_obj = {'0': (None, ())} @@ -433,7 +436,6 @@ def incref(self, c, ident): self.id_to_refcount[ident] = 1 self.id_to_obj[ident] = \ self.id_to_local_proxy_obj[ident] - obj, exposed, gettypeid = self.id_to_obj[ident] util.debug('Server re-enabled tracking & INCREF %r', ident) else: raise ke @@ -497,7 +499,7 @@ class BaseManager(object): _Server = Server def __init__(self, address=None, authkey=None, serializer='pickle', - ctx=None): + ctx=None, *, shutdown_timeout=1.0): if authkey is None: authkey = process.current_process().authkey self._address = address # XXX not final address if eg ('', 0) @@ -507,6 +509,7 @@ def __init__(self, address=None, authkey=None, serializer='pickle', self._serializer = serializer self._Listener, self._Client = listener_client[serializer] self._ctx = ctx or get_context() + self._shutdown_timeout = shutdown_timeout def get_server(self): ''' @@ -570,8 +573,8 @@ def start(self, initializer=None, initargs=()): self._state.value = State.STARTED self.shutdown = util.Finalize( self, type(self)._finalize_manager, - args=(self._process, self._address, self._authkey, - self._state, self._Client), + args=(self._process, self._address, self._authkey, self._state, + self._Client, self._shutdown_timeout), exitpriority=0 ) @@ -656,7 +659,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.shutdown() @staticmethod - def _finalize_manager(process, address, authkey, state, _Client): + def _finalize_manager(process, address, authkey, state, _Client, + shutdown_timeout): ''' Shutdown the manager process; will be registered as a finalizer ''' @@ -671,15 +675,17 @@ def _finalize_manager(process, address, authkey, state, _Client): except Exception: pass - process.join(timeout=1.0) + process.join(timeout=shutdown_timeout) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): util.info('trying to `terminate()` manager process') process.terminate() - process.join(timeout=1.0) + process.join(timeout=shutdown_timeout) if process.is_alive(): util.info('manager still alive after terminate') + process.kill() + process.join() state.value = State.SHUTDOWN try: @@ -752,22 +758,29 @@ class BaseProxy(object): _address_to_local = {} _mutex = util.ForkAwareThreadLock() + # Each instance gets a `_serial` number. Unlike `id(...)`, this number + # is never reused. + _next_serial = 1 + def __init__(self, token, serializer, manager=None, authkey=None, exposed=None, incref=True, manager_owned=False): with BaseProxy._mutex: - tls_idset = BaseProxy._address_to_local.get(token.address, None) - if tls_idset is None: - tls_idset = util.ForkAwareLocal(), ProcessLocalSet() - BaseProxy._address_to_local[token.address] = tls_idset + tls_serials = BaseProxy._address_to_local.get(token.address, None) + if tls_serials is None: + tls_serials = util.ForkAwareLocal(), ProcessLocalSet() + BaseProxy._address_to_local[token.address] = tls_serials + + self._serial = BaseProxy._next_serial + BaseProxy._next_serial += 1 # self._tls is used to record the connection used by this # thread to communicate with the manager at token.address - self._tls = tls_idset[0] + self._tls = tls_serials[0] - # self._idset is used to record the identities of all shared - # objects for which the current process owns references and + # self._all_serials is a set used to record the identities of all + # shared objects for which the current process owns references and # which are in the manager at token.address - self._idset = tls_idset[1] + self._all_serials = tls_serials[1] self._token = token self._id = self._token.id @@ -830,7 +843,10 @@ def _callmethod(self, methodname, args=(), kwds={}): conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def _getvalue(self): ''' @@ -847,20 +863,20 @@ def _incref(self): dispatch(conn, None, 'incref', (self._id,)) util.debug('INCREF %r', self._token.id) - self._idset.add(self._id) + self._all_serials.add(self._serial) state = self._manager and self._manager._state self._close = util.Finalize( self, BaseProxy._decref, - args=(self._token, self._authkey, state, - self._tls, self._idset, self._Client), + args=(self._token, self._serial, self._authkey, state, + self._tls, self._all_serials, self._Client), exitpriority=10 ) @staticmethod - def _decref(token, authkey, state, tls, idset, _Client): - idset.discard(token.id) + def _decref(token, serial, authkey, state, tls, idset, _Client): + idset.discard(serial) # check whether manager is still alive if state is None or state.value == State.STARTED: @@ -1156,15 +1172,19 @@ def __imul__(self, value): self._callmethod('__imul__', (value,)) return self + __class_getitem__ = classmethod(types.GenericAlias) + -DictProxy = MakeProxyType('DictProxy', ( +_BaseDictProxy = MakeProxyType('DictProxy', ( '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', '__setitem__', 'clear', 'copy', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) -DictProxy._method_to_typeid_ = { +_BaseDictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } +class DictProxy(_BaseDictProxy): + __class_getitem__ = classmethod(types.GenericAlias) ArrayProxy = MakeProxyType('ArrayProxy', ( @@ -1338,7 +1358,6 @@ def __init__(self, *args, **kwargs): def __del__(self): util.debug(f"{self.__class__.__name__}.__del__ by pid {getpid()}") - pass def get_server(self): 'Better than monkeypatching for now; merge into Server ultimately' diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index bbe05a550c3..f979890170b 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -200,9 +200,12 @@ def __init__(self, processes=None, initializer=None, initargs=(), self._initargs = initargs if processes is None: - processes = os.cpu_count() or 1 + processes = os.process_cpu_count() or 1 if processes < 1: raise ValueError("Number of processes must be at least 1") + if maxtasksperchild is not None: + if not isinstance(maxtasksperchild, int) or maxtasksperchild <= 0: + raise ValueError("maxtasksperchild must be a positive int or None") if initializer is not None and not callable(initializer): raise TypeError('initializer must be a callable') @@ -693,7 +696,7 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, if (not result_handler.is_alive()) and (len(cache) != 0): raise AssertionError( - "Cannot have cache with result_hander not alive") + "Cannot have cache with result_handler not alive") result_handler._state = TERMINATE change_notifier.put(None) diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf476..a57ef6bdad5 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -1,3 +1,4 @@ +import atexit import os import signal @@ -66,10 +67,13 @@ def _launch(self, process_obj): self.pid = os.fork() if self.pid == 0: try: + atexit._clear() + atexit.register(util._exit_function) os.close(parent_r) os.close(parent_w) code = process_obj._bootstrap(parent_sentinel=child_r) finally: + atexit._run_exitfuncs() os._exit(code) else: os.close(child_w) diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py index 24b8634523e..cccd659ae77 100644 --- a/Lib/multiprocessing/popen_spawn_posix.py +++ b/Lib/multiprocessing/popen_spawn_posix.py @@ -57,6 +57,10 @@ def _launch(self, process_obj): self._fds.extend([child_r, child_w]) self.pid = util.spawnv_passfds(spawn.get_executable(), cmd, self._fds) + os.close(child_r) + child_r = None + os.close(child_w) + child_w = None self.sentinel = parent_r with open(parent_w, 'wb', closefd=False) as f: f.write(fp.getbuffer()) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 9c4098d0fa4..62fb0ddbf91 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -3,6 +3,7 @@ import signal import sys import _winapi +from subprocess import STARTUPINFO, STARTF_FORCEOFFFEEDBACK from .context import reduction, get_spawning_popen, set_spawning_popen from . import spawn @@ -14,6 +15,7 @@ # # +# Exit code used by Popen.terminate() TERMINATE = 0x10000 WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") @@ -54,25 +56,27 @@ def __init__(self, process_obj): wfd = msvcrt.open_osfhandle(whandle, 0) cmd = spawn.get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle) - cmd = ' '.join('"%s"' % x for x in cmd) python_exe = spawn.get_executable() # bpo-35797: When running in a venv, we bypass the redirect # executor and launch our base Python. if WINENV and _path_eq(python_exe, sys.executable): - python_exe = sys._base_executable + cmd[0] = python_exe = sys._base_executable env = os.environ.copy() env["__PYVENV_LAUNCHER__"] = sys.executable else: env = None + cmd = ' '.join('"%s"' % x for x in cmd) + with open(wfd, 'wb', closefd=True) as to_child: # start process try: hp, ht, pid, tid = _winapi.CreateProcess( python_exe, cmd, - None, None, False, 0, env, None, None) + None, None, False, 0, env, None, + STARTUPINFO(dwFlags=STARTF_FORCEOFFFEEDBACK)) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) @@ -99,18 +103,20 @@ def duplicate_for_child(self, handle): return reduction.duplicate(handle, self.sentinel) def wait(self, timeout=None): - if self.returncode is None: - if timeout is None: - msecs = _winapi.INFINITE - else: - msecs = max(0, int(timeout * 1000 + 0.5)) - - res = _winapi.WaitForSingleObject(int(self._handle), msecs) - if res == _winapi.WAIT_OBJECT_0: - code = _winapi.GetExitCodeProcess(self._handle) - if code == TERMINATE: - code = -signal.SIGTERM - self.returncode = code + if self.returncode is not None: + return self.returncode + + if timeout is None: + msecs = _winapi.INFINITE + else: + msecs = max(0, int(timeout * 1000 + 0.5)) + + res = _winapi.WaitForSingleObject(int(self._handle), msecs) + if res == _winapi.WAIT_OBJECT_0: + code = _winapi.GetExitCodeProcess(self._handle) + if code == TERMINATE: + code = -signal.SIGTERM + self.returncode = code return self.returncode @@ -118,12 +124,22 @@ def poll(self): return self.wait(timeout=0) def terminate(self): - if self.returncode is None: - try: - _winapi.TerminateProcess(int(self._handle), TERMINATE) - except OSError: - if self.wait(timeout=1.0) is None: - raise + if self.returncode is not None: + return + + try: + _winapi.TerminateProcess(int(self._handle), TERMINATE) + except PermissionError: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + code = _winapi.GetExitCodeProcess(int(self._handle)) + if code == _winapi.STILL_ACTIVE: + raise + + # gh-113009: Don't set self.returncode. Even if GetExitCodeProcess() + # returns an exit code different than STILL_ACTIVE, the process can + # still be running. Only set self.returncode once WaitForSingleObject() + # returns WAIT_OBJECT_0 in wait(). kill = terminate diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 0b2e0b45b23..b45f7df476f 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -61,7 +61,7 @@ def parent_process(): def _cleanup(): # check for processes which have finished for p in list(_children): - if p._popen.poll() is not None: + if (child_popen := p._popen) and child_popen.poll() is not None: _children.discard(p) # @@ -304,18 +304,14 @@ def _bootstrap(self, parent_sentinel=None): if threading._HAVE_THREAD_NATIVE_ID: threading.main_thread()._set_native_id() try: - util._finalizer_registry.clear() - util._run_after_forkers() + self._after_fork() finally: # delay finalization of the old process object until after # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') - try: - self.run() - exitcode = 0 - finally: - util._exit_function() + self.run() + exitcode = 0 except SystemExit as e: if e.code is None: exitcode = 0 @@ -336,6 +332,13 @@ def _bootstrap(self, parent_sentinel=None): return exitcode + @staticmethod + def _after_fork(): + from . import util + util._finalizer_registry.clear() + util._run_after_forkers() + + # # We subclass bytes to avoid accidental transmission of auth keys over network # @@ -427,6 +430,7 @@ def close(self): for name, signum in list(signal.__dict__.items()): if name[:3]=='SIG' and '_' not in name: _exitcode_to_name[-signum] = f'-{name}' +del name, signum # For debug and leak testing _dangling = WeakSet() diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index f37f114a968..925f0439000 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -20,8 +20,6 @@ from queue import Empty, Full -import _multiprocessing - from . import connection from . import context _ForkingPickler = context.reduction.ForkingPickler @@ -158,6 +156,20 @@ def cancel_join_thread(self): except AttributeError: pass + def _terminate_broken(self): + # Close a Queue on error. + + # gh-94777: Prevent queue writing to a pipe which is no longer read. + self._reader.close() + + # gh-107219: Close the connection writer which can unblock + # Queue._feed() if it was stuck in send_bytes(). + if sys.platform == 'win32': + self._writer.close() + + self.close() + self.join_thread() + def _start_thread(self): debug('Queue._start_thread()') @@ -169,13 +181,19 @@ def _start_thread(self): self._wlock, self._reader.close, self._writer.close, self._ignore_epipe, self._on_queue_feeder_error, self._sem), - name='QueueFeederThread' + name='QueueFeederThread', + daemon=True, ) - self._thread.daemon = True - debug('doing self._thread.start()') - self._thread.start() - debug('... done self._thread.start()') + try: + debug('doing self._thread.start()') + self._thread.start() + debug('... done self._thread.start()') + except: + # gh-109047: During Python finalization, creating a thread + # can fail with RuntimeError. + self._thread = None + raise if not self._joincancelled: self._jointhread = Finalize( @@ -280,6 +298,8 @@ def _on_queue_feeder_error(e, obj): import traceback traceback.print_exc() + __class_getitem__ = classmethod(types.GenericAlias) + _sentinel = object() diff --git a/Lib/multiprocessing/resource_sharer.py b/Lib/multiprocessing/resource_sharer.py index 66076509a12..b8afb0fbed3 100644 --- a/Lib/multiprocessing/resource_sharer.py +++ b/Lib/multiprocessing/resource_sharer.py @@ -123,7 +123,7 @@ def _start(self): from .connection import Listener assert self._listener is None, "Already have Listener" util.debug('starting listener and thread for sending handles') - self._listener = Listener(authkey=process.current_process().authkey) + self._listener = Listener(authkey=process.current_process().authkey, backlog=128) self._address = self._listener.address t = threading.Thread(target=self._serve) t.daemon = True diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index cc42dbdda05..22e3bbcf21b 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -15,11 +15,15 @@ # this resource tracker process, "killall python" would probably leave unlinked # resources. +import base64 import os import signal import sys import threading import warnings +from collections import deque + +import json from . import spawn from . import util @@ -29,8 +33,12 @@ _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask') _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM) +def cleanup_noop(name): + raise RuntimeError('noop should never be registered or cleaned up') + _CLEANUP_FUNCS = { - 'noop': lambda: None, + 'noop': cleanup_noop, + 'dummy': lambda name: None, # Dummy resource used in tests } if os.name == 'posix': @@ -51,25 +59,86 @@ }) +class ReentrantCallError(RuntimeError): + pass + + class ResourceTracker(object): def __init__(self): - self._lock = threading.Lock() + self._lock = threading.RLock() self._fd = None self._pid = None + self._exitcode = None + self._reentrant_messages = deque() - def _stop(self): - with self._lock: - if self._fd is None: - # not running - return + # True to use colon-separated lines, rather than JSON lines, + # for internal communication. (Mainly for testing). + # Filenames not supported by the simple format will always be sent + # using JSON. + # The reader should understand all formats. + self._use_simple_format = True + + def _reentrant_call_error(self): + # gh-109629: this happens if an explicit call to the ResourceTracker + # gets interrupted by a garbage collection, invoking a finalizer (*) + # that itself calls back into ResourceTracker. + # (*) for example the SemLock finalizer + raise ReentrantCallError( + "Reentrant call into the multiprocessing resource tracker") + + def __del__(self): + # making sure child processess are cleaned before ResourceTracker + # gets destructed. + # see https://github.com/python/cpython/issues/88887 + self._stop(use_blocking_lock=False) + + def _stop(self, use_blocking_lock=True): + if use_blocking_lock: + with self._lock: + self._stop_locked() + else: + acquired = self._lock.acquire(blocking=False) + try: + self._stop_locked() + finally: + if acquired: + self._lock.release() + + def _stop_locked( + self, + close=os.close, + waitpid=os.waitpid, + waitstatus_to_exitcode=os.waitstatus_to_exitcode, + ): + # This shouldn't happen (it might when called by a finalizer) + # so we check for it anyway. + if self._lock._recursion_count() > 1: + raise self._reentrant_call_error() + if self._fd is None: + # not running + return + if self._pid is None: + return - # closing the "alive" file descriptor stops main() - os.close(self._fd) - self._fd = None + # closing the "alive" file descriptor stops main() + close(self._fd) + self._fd = None - os.waitpid(self._pid, 0) + try: + _, status = waitpid(self._pid, 0) + except ChildProcessError: self._pid = None + self._exitcode = None + return + + self._pid = None + + try: + self._exitcode = waitstatus_to_exitcode(status) + except ValueError: + # os.waitstatus_to_exitcode may raise an exception for invalid values + self._exitcode = None def getfd(self): self.ensure_running() @@ -80,71 +149,119 @@ def ensure_running(self): This can be run from any process. Usually a child process will use the resource created by its parent.''' + return self._ensure_running_and_write() + + def _teardown_dead_process(self): + os.close(self._fd) + + # Clean-up to avoid dangling processes. + try: + # _pid can be None if this process is a child from another + # python process, which has started the resource_tracker. + if self._pid is not None: + os.waitpid(self._pid, 0) + except ChildProcessError: + # The resource_tracker has already been terminated. + pass + self._fd = None + self._pid = None + self._exitcode = None + + warnings.warn('resource_tracker: process died unexpectedly, ' + 'relaunching. Some resources might leak.') + + def _launch(self): + fds_to_pass = [] + try: + fds_to_pass.append(sys.stderr.fileno()) + except Exception: + pass + r, w = os.pipe() + try: + fds_to_pass.append(r) + # process will out live us, so no need to wait on pid + exe = spawn.get_executable() + args = [ + exe, + *util._args_from_interpreter_flags(), + '-c', + f'from multiprocessing.resource_tracker import main;main({r})', + ] + # bpo-33613: Register a signal mask that will block the signals. + # This signal mask will be inherited by the child that is going + # to be spawned and will protect the child from a race condition + # that can make the child die before it registers signal handlers + # for SIGINT and SIGTERM. The mask is unregistered after spawning + # the child. + prev_sigmask = None + try: + if _HAVE_SIGMASK: + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + pid = util.spawnv_passfds(exe, args, fds_to_pass) + finally: + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) + except: + os.close(w) + raise + else: + self._fd = w + self._pid = pid + finally: + os.close(r) + + def _make_probe_message(self): + """Return a probe message.""" + if self._use_simple_format: + return b'PROBE:0:noop\n' + return ( + json.dumps( + {"cmd": "PROBE", "rtype": "noop"}, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ).encode("ascii") + + def _ensure_running_and_write(self, msg=None): with self._lock: + if self._lock._recursion_count() > 1: + # The code below is certainly not reentrant-safe, so bail out + if msg is None: + raise self._reentrant_call_error() + return self._reentrant_messages.append(msg) + if self._fd is not None: # resource tracker was launched before, is it still running? - if self._check_alive(): - # => still alive - return - # => dead, launch it again - os.close(self._fd) - - # Clean-up to avoid dangling processes. + if msg is None: + to_send = self._make_probe_message() + else: + to_send = msg try: - # _pid can be None if this process is a child from another - # python process, which has started the resource_tracker. - if self._pid is not None: - os.waitpid(self._pid, 0) - except ChildProcessError: - # The resource_tracker has already been terminated. - pass - self._fd = None - self._pid = None + self._write(to_send) + except OSError: + self._teardown_dead_process() + self._launch() - warnings.warn('resource_tracker: process died unexpectedly, ' - 'relaunching. Some resources might leak.') + msg = None # message was sent in probe + else: + self._launch() - fds_to_pass = [] - try: - fds_to_pass.append(sys.stderr.fileno()) - except Exception: - pass - cmd = 'from multiprocessing.resource_tracker import main;main(%d)' - r, w = os.pipe() + while True: try: - fds_to_pass.append(r) - # process will out live us, so no need to wait on pid - exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() - args += ['-c', cmd % r] - # bpo-33613: Register a signal mask that will block the signals. - # This signal mask will be inherited by the child that is going - # to be spawned and will protect the child from a race condition - # that can make the child die before it registers signal handlers - # for SIGINT and SIGTERM. The mask is unregistered after spawning - # the child. - try: - if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) - pid = util.spawnv_passfds(exe, args, fds_to_pass) - finally: - if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) - except: - os.close(w) - raise - else: - self._fd = w - self._pid = pid - finally: - os.close(r) + reentrant_msg = self._reentrant_messages.popleft() + except IndexError: + break + self._write(reentrant_msg) + if msg is not None: + self._write(msg) def _check_alive(self): '''Check that the pipe has not been closed by sending a probe.''' try: # We cannot use send here as it calls ensure_running, creating # a cycle. - os.write(self._fd, b'PROBE:0:noop\n') + os.write(self._fd, self._make_probe_message()) except OSError: return False else: @@ -158,17 +275,42 @@ def unregister(self, name, rtype): '''Unregister name of resource with resource tracker.''' self._send('UNREGISTER', name, rtype) - def _send(self, cmd, name, rtype): - self.ensure_running() - msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') - if len(name) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('name too long') + def _write(self, msg): nbytes = os.write(self._fd, msg) - assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( - nbytes, len(msg)) + assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" + + def _send(self, cmd, name, rtype): + if self._use_simple_format and '\n' not in name: + msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") + if len(msg) > 512: + # posix guarantees that writes to a pipe of less than PIPE_BUF + # bytes are atomic, and that PIPE_BUF >= 512 + raise ValueError('msg too long') + self._ensure_running_and_write(msg) + return + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) + # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. + # POSIX shm_open() and sem_open() require the name, including its leading slash, + # to be at most NAME_MAX bytes (255 on Linux) + # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char + # escape like \uDC80. + # As we want the overall message to be kept atomic and therefore smaller than 512, + # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name + # will not exceed 340 bytes. + b = name.encode('utf-8', 'surrogateescape') + if len(b) > 255: + raise ValueError('shared memory name too long (max 255 bytes)') + b64 = base64.urlsafe_b64encode(b).decode('ascii') + + payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64} + msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii") + + # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. + assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" + assert msg.startswith(b'{') + + self._ensure_running_and_write(msg) _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running @@ -176,6 +318,31 @@ def _send(self, cmd, name, rtype): unregister = _resource_tracker.unregister getfd = _resource_tracker.getfd + +def _decode_message(line): + if line.startswith(b'{'): + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + else: + cmd, rest = line.strip().decode('ascii').split(':', maxsplit=1) + name, rtype = rest.rsplit(':', maxsplit=1) + return cmd, rtype, name + + def main(fd): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc @@ -191,12 +358,14 @@ def main(fd): pass cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()} + exit_code = 0 + try: # keep track of registered/unregistered resources with open(fd, 'rb') as f: for line in f: try: - cmd, name, rtype = line.strip().decode('ascii').split(':') + cmd, rtype, name = _decode_message(line) cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( @@ -212,6 +381,7 @@ def main(fd): else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: + exit_code = 3 try: sys.excepthook(*sys.exc_info()) except: @@ -221,9 +391,17 @@ def main(fd): for rtype, rtype_cache in cache.items(): if rtype_cache: try: - warnings.warn('resource_tracker: There appear to be %d ' - 'leaked %s objects to clean up at shutdown' % - (len(rtype_cache), rtype)) + exit_code = 1 + if rtype == 'dummy': + # The test 'dummy' resource is expected to leak. + # We skip the warning (and *only* the warning) for it. + pass + else: + warnings.warn( + f'resource_tracker: There appear to be ' + f'{len(rtype_cache)} leaked {rtype} objects to ' + f'clean up at shutdown: {rtype_cache}' + ) except Exception: pass for name in rtype_cache: @@ -234,6 +412,9 @@ def main(fd): try: _CLEANUP_FUNCS[rtype](name) except Exception as e: + exit_code = 2 warnings.warn('resource_tracker: %r: %s' % (name, e)) finally: pass + + sys.exit(exit_code) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 122b3fcebf3..67e70fdc27c 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -23,6 +23,7 @@ import _posixshmem _USE_POSIX = True +from . import resource_tracker _O_CREX = os.O_CREAT | os.O_EXCL @@ -70,8 +71,9 @@ class SharedMemory: _flags = os.O_RDWR _mode = 0o600 _prepend_leading_slash = True if _USE_POSIX else False + _track = True - def __init__(self, name=None, create=False, size=0): + def __init__(self, name=None, create=False, size=0, *, track=True): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: @@ -81,6 +83,7 @@ def __init__(self, name=None, create=False, size=0): if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") + self._track = track if _USE_POSIX: # POSIX Shared Memory @@ -115,9 +118,8 @@ def __init__(self, name=None, create=False, size=0): except OSError: self.unlink() raise - - from .resource_tracker import register - register(self._name, "shared_memory") + if self._track: + resource_tracker.register(self._name, "shared_memory") else: @@ -173,7 +175,10 @@ def __init__(self, name=None, create=False, size=0): ) finally: _winapi.CloseHandle(h_map) - size = _winapi.VirtualQuerySize(p_buf) + try: + size = _winapi.VirtualQuerySize(p_buf) + finally: + _winapi.UnmapViewOfFile(p_buf) self._mmap = mmap.mmap(-1, size, tagname=name) self._size = size @@ -233,13 +238,20 @@ def close(self): def unlink(self): """Requests that the underlying shared memory block be destroyed. - In order to ensure proper cleanup of resources, unlink should be - called once (and only once) across all processes which have access - to the shared memory block.""" + Unlink should be called once (and only once) across all handles + which have access to the shared memory block, even if these + handles belong to different processes. Closing and unlinking may + happen in any order, but trying to access data inside a shared + memory block after unlinking may result in memory errors, + depending on platform. + + This method has no effect on Windows, where the only way to + delete a shared memory block is to close all handles.""" + if _USE_POSIX and self._name: - from .resource_tracker import unregister _posixshmem.shm_unlink(self._name) - unregister(self._name, "shared_memory") + if self._track: + resource_tracker.unregister(self._name, "shared_memory") _encoding = "utf8" diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 7cc129e2610..daac1ecc34b 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -31,20 +31,25 @@ WINSERVICE = False else: WINEXE = getattr(sys, 'frozen', False) - WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") - -if WINSERVICE: - _python_exe = os.path.join(sys.exec_prefix, 'python.exe') -else: - _python_exe = sys.executable + WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe") def set_executable(exe): global _python_exe - _python_exe = exe + if exe is None: + _python_exe = exe + elif sys.platform == 'win32': + _python_exe = os.fsdecode(exe) + else: + _python_exe = os.fsencode(exe) def get_executable(): return _python_exe +if WINSERVICE: + set_executable(os.path.join(sys.exec_prefix, 'python.exe')) +else: + set_executable(sys.executable) + # # # @@ -86,7 +91,8 @@ def get_command_line(**kwds): prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' prog %= ', '.join('%s=%r' % item for item in kwds.items()) opts = util._args_from_interpreter_flags() - return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork'] + exe = get_executable() + return [exe] + opts + ['-c', prog, '--multiprocessing-fork'] def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): @@ -144,7 +150,11 @@ def _check_not_importing_main(): ... The "freeze_support()" line can be omitted if the program - is not going to be frozen to produce an executable.''') + is not going to be frozen to produce an executable. + + To fix this issue, refer to the "Safe importing of main module" + section in https://docs.python.org/3/library/multiprocessing.html + ''') def get_preparation_data(name): diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index d0be48f1fd7..870c91349b9 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -50,8 +50,8 @@ class SemLock(object): def __init__(self, kind, value, maxvalue, *, ctx): if ctx is None: ctx = context._default_context.get_context() - name = ctx.get_start_method() - unlink_now = sys.platform == 'win32' or name == 'fork' + self._is_fork_ctx = ctx.get_start_method() == 'fork' + unlink_now = sys.platform == 'win32' or self._is_fork_ctx for i in range(100): try: sl = self._semlock = _multiprocessing.SemLock( @@ -103,6 +103,11 @@ def __getstate__(self): if sys.platform == 'win32': h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: + if self._is_fork_ctx: + raise RuntimeError('A SemLock created in a fork context is being ' + 'shared with a process in a spawn context. This is ' + 'not supported. Please use the same context to create ' + 'multiprocessing objects and Process.') h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) @@ -110,6 +115,8 @@ def __setstate__(self, state): self._semlock = _multiprocessing.SemLock._rebuild(*state) util.debug('recreated blocker with handle %r' % state[0]) self._make_methods() + # Ensure that deserialized SemLock can be serialized again (gh-108520). + self._is_fork_ctx = False @staticmethod def _make_name(): @@ -167,7 +174,7 @@ def __repr__(self): name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name - elif self._semlock._get_value() == 1: + elif not self._semlock._is_zero(): name = 'None' elif self._semlock._count() > 0: name = 'SomeOtherThread' @@ -193,7 +200,7 @@ def __repr__(self): if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name count = self._semlock._count() - elif self._semlock._get_value() == 1: + elif not self._semlock._is_zero(): name, count = 'None', 0 elif self._semlock._count() > 0: name, count = 'SomeOtherThread', 'nonzero' @@ -353,6 +360,9 @@ def wait(self, timeout=None): return True return False + def __repr__(self): + set_status = 'set' if self.is_set() else 'unset' + return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>" # # Barrier # diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 9e07a4e93e5..4c8425064fe 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -34,6 +34,7 @@ DEBUG = 10 INFO = 20 SUBWARNING = 25 +WARNING = 30 LOGGER_NAME = 'multiprocessing' DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' @@ -43,19 +44,23 @@ def sub_debug(msg, *args): if _logger: - _logger.log(SUBDEBUG, msg, *args) + _logger.log(SUBDEBUG, msg, *args, stacklevel=2) def debug(msg, *args): if _logger: - _logger.log(DEBUG, msg, *args) + _logger.log(DEBUG, msg, *args, stacklevel=2) def info(msg, *args): if _logger: - _logger.log(INFO, msg, *args) + _logger.log(INFO, msg, *args, stacklevel=2) + +def _warn(msg, *args): + if _logger: + _logger.log(WARNING, msg, *args, stacklevel=2) def sub_warning(msg, *args): if _logger: - _logger.log(SUBWARNING, msg, *args) + _logger.log(SUBWARNING, msg, *args, stacklevel=2) def get_logger(): ''' @@ -64,8 +69,7 @@ def get_logger(): global _logger import logging - logging._acquireLock() - try: + with logging._lock: if not _logger: _logger = logging.getLogger(LOGGER_NAME) @@ -79,9 +83,6 @@ def get_logger(): atexit._exithandlers.remove((_exit_function, (), {})) atexit._exithandlers.append((_exit_function, (), {})) - finally: - logging._releaseLock() - return _logger def log_to_stderr(level=None): @@ -106,11 +107,7 @@ def log_to_stderr(level=None): # Abstract socket support def _platform_supports_abstract_sockets(): - if sys.platform == "linux": - return True - if hasattr(sys, 'getandroidapilevel'): - return True - return False + return sys.platform in ("linux", "android") def is_abstract_socket_namespace(address): @@ -129,6 +126,23 @@ def is_abstract_socket_namespace(address): # Function returning a temp directory which will be removed on exit # +# Maximum length of a NULL-terminated [1] socket file path is usually +# between 92 and 108 [2], but Linux is known to use a size of 108 [3]. +# BSD-based systems usually use a size of 104 or 108 and Windows does +# not create AF_UNIX sockets. +# +# [1]: https://github.com/python/cpython/issues/140734 +# [2]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html +# [3]: https://man7.org/linux/man-pages/man7/unix.7.html + +if sys.platform == 'linux': + _SUN_PATH_MAX = 108 +elif sys.platform.startswith(('openbsd', 'freebsd')): + _SUN_PATH_MAX = 104 +else: + # On Windows platforms, we do not create AF_UNIX sockets. + _SUN_PATH_MAX = None if os.name == 'nt' else 92 + def _remove_temp_dir(rmtree, tempdir): rmtree(tempdir) @@ -138,12 +152,69 @@ def _remove_temp_dir(rmtree, tempdir): if current_process is not None: current_process._config['tempdir'] = None +def _get_base_temp_dir(tempfile): + """Get a temporary directory where socket files will be created. + + To prevent additional imports, pass a pre-imported 'tempfile' module. + """ + if os.name == 'nt': + return None + # Most of the time, the default temporary directory is /tmp. Thus, + # listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do + # not have a path length exceeding SUN_PATH_MAX. + # + # If users specify their own temporary directory, we may be unable + # to create those files. Therefore, we fall back to the system-wide + # temporary directory /tmp, assumed to exist on POSIX systems. + # + # See https://github.com/python/cpython/issues/132124. + base_tempdir = tempfile.gettempdir() + # Files created in a temporary directory are suffixed by a string + # generated by tempfile._RandomNameSequence, which, by design, + # is 8 characters long. + # + # Thus, the socket file path length (without NULL terminator) will be: + # + # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') + sun_path_len = len(base_tempdir) + 14 + 14 + # Strict inequality to account for the NULL terminator. + # See https://github.com/python/cpython/issues/140734. + if sun_path_len < _SUN_PATH_MAX: + return base_tempdir + # Fallback to the default system-wide temporary directory. + # This ignores user-defined environment variables. + # + # On POSIX systems, /tmp MUST be writable by any application [1]. + # We however emit a warning if this is not the case to prevent + # obscure errors later in the execution. + # + # On some legacy systems, /var/tmp and /usr/tmp can be present + # and will be used instead. + # + # [1]: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s18.html + dirlist = ['/tmp', '/var/tmp', '/usr/tmp'] + try: + base_system_tempdir = tempfile._get_default_tempdir(dirlist) + except FileNotFoundError: + _warn("Process-wide temporary directory %s will not be usable for " + "creating socket files and no usable system-wide temporary " + "directory was found in %s", base_tempdir, dirlist) + # At this point, the system-wide temporary directory is not usable + # but we may assume that the user-defined one is, even if we will + # not be able to write socket files out there. + return base_tempdir + _warn("Ignoring user-defined temporary directory: %s", base_tempdir) + # at most max(map(len, dirlist)) + 14 + 14 = 36 characters + assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX + return base_system_tempdir + def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: import shutil, tempfile - tempdir = tempfile.mkdtemp(prefix='pymp-') + base_tempdir = _get_base_temp_dir(tempfile) + tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) info('created temp directory %s', tempdir) # keep a strong reference to shutil.rmtree(), since the finalizer # can be called late during Python shutdown @@ -450,9 +521,9 @@ def spawnv_passfds(path, args, passfds): errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( - args, [os.fsencode(path)], True, passfds, None, None, + args, [path], True, passfds, None, None, -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, - False, False, None, None, None, -1, None) + False, False, -1, None, None, None, -1, None) finally: os.close(errpipe_read) os.close(errpipe_write) diff --git a/Lib/netrc.py b/Lib/netrc.py index c1358aac6ae..bd003e80a48 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -2,11 +2,24 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 -import os, shlex, stat +import os, stat __all__ = ["netrc", "NetrcParseError"] +def _can_security_check(): + # On WASI, getuid() is indicated as a stub but it may also be missing. + return os.name == 'posix' and hasattr(os, 'getuid') + + +def _getpwuid(uid): + try: + import pwd + return pwd.getpwuid(uid)[0] + except (ImportError, LookupError): + return f'uid {uid}' + + class NetrcParseError(Exception): """Exception raised on syntax errors in the .netrc file.""" def __init__(self, msg, filename=None, lineno=None): @@ -142,18 +155,12 @@ def _parse(self, file, fp, default_netrc): self._security_check(fp, default_netrc, self.hosts[entryname][0]) def _security_check(self, fp, default_netrc, login): - if os.name == 'posix' and default_netrc and login != "anonymous": + if _can_security_check() and default_netrc and login != "anonymous": prop = os.fstat(fp.fileno()) - if prop.st_uid != os.getuid(): - import pwd - try: - fowner = pwd.getpwuid(prop.st_uid)[0] - except KeyError: - fowner = 'uid %s' % prop.st_uid - try: - user = pwd.getpwuid(os.getuid())[0] - except KeyError: - user = 'uid %s' % os.getuid() + current_user_id = os.getuid() + if prop.st_uid != current_user_id: + fowner = _getpwuid(prop.st_uid) + user = _getpwuid(current_user_id) raise NetrcParseError( (f"~/.netrc file owner ({fowner}, {user}) does not match" " current user")) diff --git a/Lib/nntplib.py b/Lib/nntplib.py deleted file mode 100644 index dddea059982..00000000000 --- a/Lib/nntplib.py +++ /dev/null @@ -1,1093 +0,0 @@ -"""An NNTP client class based on: -- RFC 977: Network News Transfer Protocol -- RFC 2980: Common NNTP Extensions -- RFC 3977: Network News Transfer Protocol (version 2) - -Example: - ->>> from nntplib import NNTP ->>> s = NNTP('news') ->>> resp, count, first, last, name = s.group('comp.lang.python') ->>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) -Group comp.lang.python has 51 articles, range 5770 to 5821 ->>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last)) ->>> resp = s.quit() ->>> - -Here 'resp' is the server response line. -Error responses are turned into exceptions. - -To post an article from a file: ->>> f = open(filename, 'rb') # file containing article, including header ->>> resp = s.post(f) ->>> - -For descriptions of all methods, read the comments in the code below. -Note that all arguments and return values representing article numbers -are strings, not numbers, since they are rarely used for calculations. -""" - -# RFC 977 by Brian Kantor and Phil Lapsley. -# xover, xgtitle, xpath, date methods by Kevan Heydon - -# Incompatible changes from the 2.x nntplib: -# - all commands are encoded as UTF-8 data (using the "surrogateescape" -# error handler), except for raw message data (POST, IHAVE) -# - all responses are decoded as UTF-8 data (using the "surrogateescape" -# error handler), except for raw message data (ARTICLE, HEAD, BODY) -# - the `file` argument to various methods is keyword-only -# -# - NNTP.date() returns a datetime object -# - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object, -# rather than a pair of (date, time) strings. -# - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples -# - NNTP.descriptions() returns a dict mapping group names to descriptions -# - NNTP.xover() returns a list of dicts mapping field names (header or metadata) -# to field values; each dict representing a message overview. -# - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo) -# tuple. -# - the "internal" methods have been marked private (they now start with -# an underscore) - -# Other changes from the 2.x/3.1 nntplib: -# - automatic querying of capabilities at connect -# - New method NNTP.getcapabilities() -# - New method NNTP.over() -# - New helper function decode_header() -# - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and -# arbitrary iterables yielding lines. -# - An extensive test suite :-) - -# TODO: -# - return structured data (GroupInfo etc.) everywhere -# - support HDR - -# Imports -import re -import socket -import collections -import datetime -import sys -import warnings - -try: - import ssl -except ImportError: - _have_ssl = False -else: - _have_ssl = True - -from email.header import decode_header as _email_decode_header -from socket import _GLOBAL_DEFAULT_TIMEOUT - -__all__ = ["NNTP", - "NNTPError", "NNTPReplyError", "NNTPTemporaryError", - "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError", - "decode_header", - ] - -warnings._deprecated(__name__, remove=(3, 13)) - -# maximal line length when calling readline(). This is to prevent -# reading arbitrary length lines. RFC 3977 limits NNTP line length to -# 512 characters, including CRLF. We have selected 2048 just to be on -# the safe side. -_MAXLINE = 2048 - - -# Exceptions raised when an error or invalid response is received -class NNTPError(Exception): - """Base class for all nntplib exceptions""" - def __init__(self, *args): - Exception.__init__(self, *args) - try: - self.response = args[0] - except IndexError: - self.response = 'No response given' - -class NNTPReplyError(NNTPError): - """Unexpected [123]xx reply""" - pass - -class NNTPTemporaryError(NNTPError): - """4xx errors""" - pass - -class NNTPPermanentError(NNTPError): - """5xx errors""" - pass - -class NNTPProtocolError(NNTPError): - """Response does not begin with [1-5]""" - pass - -class NNTPDataError(NNTPError): - """Error in response data""" - pass - - -# Standard port used by NNTP servers -NNTP_PORT = 119 -NNTP_SSL_PORT = 563 - -# Response numbers that are followed by additional text (e.g. article) -_LONGRESP = { - '100', # HELP - '101', # CAPABILITIES - '211', # LISTGROUP (also not multi-line with GROUP) - '215', # LIST - '220', # ARTICLE - '221', # HEAD, XHDR - '222', # BODY - '224', # OVER, XOVER - '225', # HDR - '230', # NEWNEWS - '231', # NEWGROUPS - '282', # XGTITLE -} - -# Default decoded value for LIST OVERVIEW.FMT if not supported -_DEFAULT_OVERVIEW_FMT = [ - "subject", "from", "date", "message-id", "references", ":bytes", ":lines"] - -# Alternative names allowed in LIST OVERVIEW.FMT response -_OVERVIEW_FMT_ALTERNATIVES = { - 'bytes': ':bytes', - 'lines': ':lines', -} - -# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) -_CRLF = b'\r\n' - -GroupInfo = collections.namedtuple('GroupInfo', - ['group', 'last', 'first', 'flag']) - -ArticleInfo = collections.namedtuple('ArticleInfo', - ['number', 'message_id', 'lines']) - - -# Helper function(s) -def decode_header(header_str): - """Takes a unicode string representing a munged header value - and decodes it as a (possibly non-ASCII) readable value.""" - parts = [] - for v, enc in _email_decode_header(header_str): - if isinstance(v, bytes): - parts.append(v.decode(enc or 'ascii')) - else: - parts.append(v) - return ''.join(parts) - -def _parse_overview_fmt(lines): - """Parse a list of string representing the response to LIST OVERVIEW.FMT - and return a list of header/metadata names. - Raises NNTPDataError if the response is not compliant - (cf. RFC 3977, section 8.4).""" - fmt = [] - for line in lines: - if line[0] == ':': - # Metadata name (e.g. ":bytes") - name, _, suffix = line[1:].partition(':') - name = ':' + name - else: - # Header name (e.g. "Subject:" or "Xref:full") - name, _, suffix = line.partition(':') - name = name.lower() - name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name) - # Should we do something with the suffix? - fmt.append(name) - defaults = _DEFAULT_OVERVIEW_FMT - if len(fmt) < len(defaults): - raise NNTPDataError("LIST OVERVIEW.FMT response too short") - if fmt[:len(defaults)] != defaults: - raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields") - return fmt - -def _parse_overview(lines, fmt, data_process_func=None): - """Parse the response to an OVER or XOVER command according to the - overview format `fmt`.""" - n_defaults = len(_DEFAULT_OVERVIEW_FMT) - overview = [] - for line in lines: - fields = {} - article_number, *tokens = line.split('\t') - article_number = int(article_number) - for i, token in enumerate(tokens): - if i >= len(fmt): - # XXX should we raise an error? Some servers might not - # support LIST OVERVIEW.FMT and still return additional - # headers. - continue - field_name = fmt[i] - is_metadata = field_name.startswith(':') - if i >= n_defaults and not is_metadata: - # Non-default header names are included in full in the response - # (unless the field is totally empty) - h = field_name + ": " - if token and token[:len(h)].lower() != h: - raise NNTPDataError("OVER/XOVER response doesn't include " - "names of additional headers") - token = token[len(h):] if token else None - fields[fmt[i]] = token - overview.append((article_number, fields)) - return overview - -def _parse_datetime(date_str, time_str=None): - """Parse a pair of (date, time) strings, and return a datetime object. - If only the date is given, it is assumed to be date and time - concatenated together (e.g. response to the DATE command). - """ - if time_str is None: - time_str = date_str[-6:] - date_str = date_str[:-6] - hours = int(time_str[:2]) - minutes = int(time_str[2:4]) - seconds = int(time_str[4:]) - year = int(date_str[:-4]) - month = int(date_str[-4:-2]) - day = int(date_str[-2:]) - # RFC 3977 doesn't say how to interpret 2-char years. Assume that - # there are no dates before 1970 on Usenet. - if year < 70: - year += 2000 - elif year < 100: - year += 1900 - return datetime.datetime(year, month, day, hours, minutes, seconds) - -def _unparse_datetime(dt, legacy=False): - """Format a date or datetime object as a pair of (date, time) strings - in the format required by the NEWNEWS and NEWGROUPS commands. If a - date object is passed, the time is assumed to be midnight (00h00). - - The returned representation depends on the legacy flag: - * if legacy is False (the default): - date has the YYYYMMDD format and time the HHMMSS format - * if legacy is True: - date has the YYMMDD format and time the HHMMSS format. - RFC 3977 compliant servers should understand both formats; therefore, - legacy is only needed when talking to old servers. - """ - if not isinstance(dt, datetime.datetime): - time_str = "000000" - else: - time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt) - y = dt.year - if legacy: - y = y % 100 - date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt) - else: - date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt) - return date_str, time_str - - -if _have_ssl: - - def _encrypt_on(sock, context, hostname): - """Wrap a socket in SSL/TLS. Arguments: - - sock: Socket to wrap - - context: SSL context to use for the encrypted connection - Returns: - - sock: New, encrypted socket. - """ - # Generate a default SSL context if none was passed. - if context is None: - context = ssl._create_stdlib_context() - return context.wrap_socket(sock, server_hostname=hostname) - - -# The classes themselves -class NNTP: - # UTF-8 is the character set for all NNTP commands and responses: they - # are automatically encoded (when sending) and decoded (and receiving) - # by this class. - # However, some multi-line data blocks can contain arbitrary bytes (for - # example, latin-1 or utf-16 data in the body of a message). Commands - # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message - # data will therefore only accept and produce bytes objects. - # Furthermore, since there could be non-compliant servers out there, - # we use 'surrogateescape' as the error handler for fault tolerance - # and easy round-tripping. This could be useful for some applications - # (e.g. NNTP gateways). - - encoding = 'utf-8' - errors = 'surrogateescape' - - def __init__(self, host, port=NNTP_PORT, user=None, password=None, - readermode=None, usenetrc=False, - timeout=_GLOBAL_DEFAULT_TIMEOUT): - """Initialize an instance. Arguments: - - host: hostname to connect to - - port: port to connect to (default the standard NNTP port) - - user: username to authenticate with - - password: password to use with username - - readermode: if true, send 'mode reader' command after - connecting. - - usenetrc: allow loading username and password from ~/.netrc file - if not specified explicitly - - timeout: timeout (in seconds) used for socket connections - - readermode is sometimes necessary if you are connecting to an - NNTP server on the local machine and intend to call - reader-specific commands, such as `group'. If you get - unexpected NNTPPermanentErrors, you might need to set - readermode. - """ - self.host = host - self.port = port - self.sock = self._create_socket(timeout) - self.file = None - try: - self.file = self.sock.makefile("rwb") - self._base_init(readermode) - if user or usenetrc: - self.login(user, password, usenetrc) - except: - if self.file: - self.file.close() - self.sock.close() - raise - - def _base_init(self, readermode): - """Partial initialization for the NNTP protocol. - This instance method is extracted for supporting the test code. - """ - self.debugging = 0 - self.welcome = self._getresp() - - # Inquire about capabilities (RFC 3977). - self._caps = None - self.getcapabilities() - - # 'MODE READER' is sometimes necessary to enable 'reader' mode. - # However, the order in which 'MODE READER' and 'AUTHINFO' need to - # arrive differs between some NNTP servers. If _setreadermode() fails - # with an authorization failed error, it will set this to True; - # the login() routine will interpret that as a request to try again - # after performing its normal function. - # Enable only if we're not already in READER mode anyway. - self.readermode_afterauth = False - if readermode and 'READER' not in self._caps: - self._setreadermode() - if not self.readermode_afterauth: - # Capabilities might have changed after MODE READER - self._caps = None - self.getcapabilities() - - # RFC 4642 2.2.2: Both the client and the server MUST know if there is - # a TLS session active. A client MUST NOT attempt to start a TLS - # session if a TLS session is already active. - self.tls_on = False - - # Log in and encryption setup order is left to subclasses. - self.authenticated = False - - def __enter__(self): - return self - - def __exit__(self, *args): - is_connected = lambda: hasattr(self, "file") - if is_connected(): - try: - self.quit() - except (OSError, EOFError): - pass - finally: - if is_connected(): - self._close() - - def _create_socket(self, timeout): - if timeout is not None and not timeout: - raise ValueError('Non-blocking socket (timeout=0) is not supported') - sys.audit("nntplib.connect", self, self.host, self.port) - return socket.create_connection((self.host, self.port), timeout) - - def getwelcome(self): - """Get the welcome message from the server - (this is read and squirreled away by __init__()). - If the response code is 200, posting is allowed; - if it 201, posting is not allowed.""" - - if self.debugging: print('*welcome*', repr(self.welcome)) - return self.welcome - - def getcapabilities(self): - """Get the server capabilities, as read by __init__(). - If the CAPABILITIES command is not supported, an empty dict is - returned.""" - if self._caps is None: - self.nntp_version = 1 - self.nntp_implementation = None - try: - resp, caps = self.capabilities() - except (NNTPPermanentError, NNTPTemporaryError): - # Server doesn't support capabilities - self._caps = {} - else: - self._caps = caps - if 'VERSION' in caps: - # The server can advertise several supported versions, - # choose the highest. - self.nntp_version = max(map(int, caps['VERSION'])) - if 'IMPLEMENTATION' in caps: - self.nntp_implementation = ' '.join(caps['IMPLEMENTATION']) - return self._caps - - def set_debuglevel(self, level): - """Set the debugging level. Argument 'level' means: - 0: no debugging output (default) - 1: print commands and responses but not body text etc. - 2: also print raw lines read and sent before stripping CR/LF""" - - self.debugging = level - debug = set_debuglevel - - def _putline(self, line): - """Internal: send one line to the server, appending CRLF. - The `line` must be a bytes-like object.""" - sys.audit("nntplib.putline", self, line) - line = line + _CRLF - if self.debugging > 1: print('*put*', repr(line)) - self.file.write(line) - self.file.flush() - - def _putcmd(self, line): - """Internal: send one command to the server (through _putline()). - The `line` must be a unicode string.""" - if self.debugging: print('*cmd*', repr(line)) - line = line.encode(self.encoding, self.errors) - self._putline(line) - - def _getline(self, strip_crlf=True): - """Internal: return one line from the server, stripping _CRLF. - Raise EOFError if the connection is closed. - Returns a bytes object.""" - line = self.file.readline(_MAXLINE +1) - if len(line) > _MAXLINE: - raise NNTPDataError('line too long') - if self.debugging > 1: - print('*get*', repr(line)) - if not line: raise EOFError - if strip_crlf: - if line[-2:] == _CRLF: - line = line[:-2] - elif line[-1:] in _CRLF: - line = line[:-1] - return line - - def _getresp(self): - """Internal: get a response from the server. - Raise various errors if the response indicates an error. - Returns a unicode string.""" - resp = self._getline() - if self.debugging: print('*resp*', repr(resp)) - resp = resp.decode(self.encoding, self.errors) - c = resp[:1] - if c == '4': - raise NNTPTemporaryError(resp) - if c == '5': - raise NNTPPermanentError(resp) - if c not in '123': - raise NNTPProtocolError(resp) - return resp - - def _getlongresp(self, file=None): - """Internal: get a response plus following text from the server. - Raise various errors if the response indicates an error. - - Returns a (response, lines) tuple where `response` is a unicode - string and `lines` is a list of bytes objects. - If `file` is a file-like object, it must be open in binary mode. - """ - - openedFile = None - try: - # If a string was passed then open a file with that name - if isinstance(file, (str, bytes)): - openedFile = file = open(file, "wb") - - resp = self._getresp() - if resp[:3] not in _LONGRESP: - raise NNTPReplyError(resp) - - lines = [] - if file is not None: - # XXX lines = None instead? - terminators = (b'.' + _CRLF, b'.\n') - while 1: - line = self._getline(False) - if line in terminators: - break - if line.startswith(b'..'): - line = line[1:] - file.write(line) - else: - terminator = b'.' - while 1: - line = self._getline() - if line == terminator: - break - if line.startswith(b'..'): - line = line[1:] - lines.append(line) - finally: - # If this method created the file, then it must close it - if openedFile: - openedFile.close() - - return resp, lines - - def _shortcmd(self, line): - """Internal: send a command and get the response. - Same return value as _getresp().""" - self._putcmd(line) - return self._getresp() - - def _longcmd(self, line, file=None): - """Internal: send a command and get the response plus following text. - Same return value as _getlongresp().""" - self._putcmd(line) - return self._getlongresp(file) - - def _longcmdstring(self, line, file=None): - """Internal: send a command and get the response plus following text. - Same as _longcmd() and _getlongresp(), except that the returned `lines` - are unicode strings rather than bytes objects. - """ - self._putcmd(line) - resp, list = self._getlongresp(file) - return resp, [line.decode(self.encoding, self.errors) - for line in list] - - def _getoverviewfmt(self): - """Internal: get the overview format. Queries the server if not - already done, else returns the cached value.""" - try: - return self._cachedoverviewfmt - except AttributeError: - pass - try: - resp, lines = self._longcmdstring("LIST OVERVIEW.FMT") - except NNTPPermanentError: - # Not supported by server? - fmt = _DEFAULT_OVERVIEW_FMT[:] - else: - fmt = _parse_overview_fmt(lines) - self._cachedoverviewfmt = fmt - return fmt - - def _grouplist(self, lines): - # Parse lines into "group last first flag" - return [GroupInfo(*line.split()) for line in lines] - - def capabilities(self): - """Process a CAPABILITIES command. Not supported by all servers. - Return: - - resp: server response if successful - - caps: a dictionary mapping capability names to lists of tokens - (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] }) - """ - caps = {} - resp, lines = self._longcmdstring("CAPABILITIES") - for line in lines: - name, *tokens = line.split() - caps[name] = tokens - return resp, caps - - def newgroups(self, date, *, file=None): - """Process a NEWGROUPS command. Arguments: - - date: a date or datetime object - Return: - - resp: server response if successful - - list: list of newsgroup names - """ - if not isinstance(date, (datetime.date, datetime.date)): - raise TypeError( - "the date parameter must be a date or datetime object, " - "not '{:40}'".format(date.__class__.__name__)) - date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) - cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str) - resp, lines = self._longcmdstring(cmd, file) - return resp, self._grouplist(lines) - - def newnews(self, group, date, *, file=None): - """Process a NEWNEWS command. Arguments: - - group: group name or '*' - - date: a date or datetime object - Return: - - resp: server response if successful - - list: list of message ids - """ - if not isinstance(date, (datetime.date, datetime.date)): - raise TypeError( - "the date parameter must be a date or datetime object, " - "not '{:40}'".format(date.__class__.__name__)) - date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) - cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str) - return self._longcmdstring(cmd, file) - - def list(self, group_pattern=None, *, file=None): - """Process a LIST or LIST ACTIVE command. Arguments: - - group_pattern: a pattern indicating which groups to query - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of (group, last, first, flag) (strings) - """ - if group_pattern is not None: - command = 'LIST ACTIVE ' + group_pattern - else: - command = 'LIST' - resp, lines = self._longcmdstring(command, file) - return resp, self._grouplist(lines) - - def _getdescriptions(self, group_pattern, return_all): - line_pat = re.compile('^(?P[^ \t]+)[ \t]+(.*)$') - # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first - resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern) - if not resp.startswith('215'): - # Now the deprecated XGTITLE. This either raises an error - # or succeeds with the same output structure as LIST - # NEWSGROUPS. - resp, lines = self._longcmdstring('XGTITLE ' + group_pattern) - groups = {} - for raw_line in lines: - match = line_pat.search(raw_line.strip()) - if match: - name, desc = match.group(1, 2) - if not return_all: - return desc - groups[name] = desc - if return_all: - return resp, groups - else: - # Nothing found - return '' - - def description(self, group): - """Get a description for a single group. If more than one - group matches ('group' is a pattern), return the first. If no - group matches, return an empty string. - - This elides the response code from the server, since it can - only be '215' or '285' (for xgtitle) anyway. If the response - code is needed, use the 'descriptions' method. - - NOTE: This neither checks for a wildcard in 'group' nor does - it check whether the group actually exists.""" - return self._getdescriptions(group, False) - - def descriptions(self, group_pattern): - """Get descriptions for a range of groups.""" - return self._getdescriptions(group_pattern, True) - - def group(self, name): - """Process a GROUP command. Argument: - - group: the group name - Returns: - - resp: server response if successful - - count: number of articles - - first: first article number - - last: last article number - - name: the group name - """ - resp = self._shortcmd('GROUP ' + name) - if not resp.startswith('211'): - raise NNTPReplyError(resp) - words = resp.split() - count = first = last = 0 - n = len(words) - if n > 1: - count = words[1] - if n > 2: - first = words[2] - if n > 3: - last = words[3] - if n > 4: - name = words[4].lower() - return resp, int(count), int(first), int(last), name - - def help(self, *, file=None): - """Process a HELP command. Argument: - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of strings returned by the server in response to the - HELP command - """ - return self._longcmdstring('HELP', file) - - def _statparse(self, resp): - """Internal: parse the response line of a STAT, NEXT, LAST, - ARTICLE, HEAD or BODY command.""" - if not resp.startswith('22'): - raise NNTPReplyError(resp) - words = resp.split() - art_num = int(words[1]) - message_id = words[2] - return resp, art_num, message_id - - def _statcmd(self, line): - """Internal: process a STAT, NEXT or LAST command.""" - resp = self._shortcmd(line) - return self._statparse(resp) - - def stat(self, message_spec=None): - """Process a STAT command. Argument: - - message_spec: article number or message id (if not specified, - the current article is selected) - Returns: - - resp: server response if successful - - art_num: the article number - - message_id: the message id - """ - if message_spec: - return self._statcmd('STAT {0}'.format(message_spec)) - else: - return self._statcmd('STAT') - - def next(self): - """Process a NEXT command. No arguments. Return as for STAT.""" - return self._statcmd('NEXT') - - def last(self): - """Process a LAST command. No arguments. Return as for STAT.""" - return self._statcmd('LAST') - - def _artcmd(self, line, file=None): - """Internal: process a HEAD, BODY or ARTICLE command.""" - resp, lines = self._longcmd(line, file) - resp, art_num, message_id = self._statparse(resp) - return resp, ArticleInfo(art_num, message_id, lines) - - def head(self, message_spec=None, *, file=None): - """Process a HEAD command. Argument: - - message_spec: article number or message id - - file: filename string or file object to store the headers in - Returns: - - resp: server response if successful - - ArticleInfo: (article number, message id, list of header lines) - """ - if message_spec is not None: - cmd = 'HEAD {0}'.format(message_spec) - else: - cmd = 'HEAD' - return self._artcmd(cmd, file) - - def body(self, message_spec=None, *, file=None): - """Process a BODY command. Argument: - - message_spec: article number or message id - - file: filename string or file object to store the body in - Returns: - - resp: server response if successful - - ArticleInfo: (article number, message id, list of body lines) - """ - if message_spec is not None: - cmd = 'BODY {0}'.format(message_spec) - else: - cmd = 'BODY' - return self._artcmd(cmd, file) - - def article(self, message_spec=None, *, file=None): - """Process an ARTICLE command. Argument: - - message_spec: article number or message id - - file: filename string or file object to store the article in - Returns: - - resp: server response if successful - - ArticleInfo: (article number, message id, list of article lines) - """ - if message_spec is not None: - cmd = 'ARTICLE {0}'.format(message_spec) - else: - cmd = 'ARTICLE' - return self._artcmd(cmd, file) - - def slave(self): - """Process a SLAVE command. Returns: - - resp: server response if successful - """ - return self._shortcmd('SLAVE') - - def xhdr(self, hdr, str, *, file=None): - """Process an XHDR command (optional server extension). Arguments: - - hdr: the header type (e.g. 'subject') - - str: an article nr, a message id, or a range nr1-nr2 - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of (nr, value) strings - """ - pat = re.compile('^([0-9]+) ?(.*)\n?') - resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file) - def remove_number(line): - m = pat.match(line) - return m.group(1, 2) if m else line - return resp, [remove_number(line) for line in lines] - - def xover(self, start, end, *, file=None): - """Process an XOVER command (optional server extension) Arguments: - - start: start of range - - end: end of range - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of dicts containing the response fields - """ - resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end), - file) - fmt = self._getoverviewfmt() - return resp, _parse_overview(lines, fmt) - - def over(self, message_spec, *, file=None): - """Process an OVER command. If the command isn't supported, fall - back to XOVER. Arguments: - - message_spec: - - either a message id, indicating the article to fetch - information about - - or a (start, end) tuple, indicating a range of article numbers; - if end is None, information up to the newest message will be - retrieved - - or None, indicating the current article number must be used - - file: Filename string or file object to store the result in - Returns: - - resp: server response if successful - - list: list of dicts containing the response fields - - NOTE: the "message id" form isn't supported by XOVER - """ - cmd = 'OVER' if 'OVER' in self._caps else 'XOVER' - if isinstance(message_spec, (tuple, list)): - start, end = message_spec - cmd += ' {0}-{1}'.format(start, end or '') - elif message_spec is not None: - cmd = cmd + ' ' + message_spec - resp, lines = self._longcmdstring(cmd, file) - fmt = self._getoverviewfmt() - return resp, _parse_overview(lines, fmt) - - def date(self): - """Process the DATE command. - Returns: - - resp: server response if successful - - date: datetime object - """ - resp = self._shortcmd("DATE") - if not resp.startswith('111'): - raise NNTPReplyError(resp) - elem = resp.split() - if len(elem) != 2: - raise NNTPDataError(resp) - date = elem[1] - if len(date) != 14: - raise NNTPDataError(resp) - return resp, _parse_datetime(date, None) - - def _post(self, command, f): - resp = self._shortcmd(command) - # Raises a specific exception if posting is not allowed - if not resp.startswith('3'): - raise NNTPReplyError(resp) - if isinstance(f, (bytes, bytearray)): - f = f.splitlines() - # We don't use _putline() because: - # - we don't want additional CRLF if the file or iterable is already - # in the right format - # - we don't want a spurious flush() after each line is written - for line in f: - if not line.endswith(_CRLF): - line = line.rstrip(b"\r\n") + _CRLF - if line.startswith(b'.'): - line = b'.' + line - self.file.write(line) - self.file.write(b".\r\n") - self.file.flush() - return self._getresp() - - def post(self, data): - """Process a POST command. Arguments: - - data: bytes object, iterable or file containing the article - Returns: - - resp: server response if successful""" - return self._post('POST', data) - - def ihave(self, message_id, data): - """Process an IHAVE command. Arguments: - - message_id: message-id of the article - - data: file containing the article - Returns: - - resp: server response if successful - Note that if the server refuses the article an exception is raised.""" - return self._post('IHAVE {0}'.format(message_id), data) - - def _close(self): - try: - if self.file: - self.file.close() - del self.file - finally: - self.sock.close() - - def quit(self): - """Process a QUIT command and close the socket. Returns: - - resp: server response if successful""" - try: - resp = self._shortcmd('QUIT') - finally: - self._close() - return resp - - def login(self, user=None, password=None, usenetrc=True): - if self.authenticated: - raise ValueError("Already logged in.") - if not user and not usenetrc: - raise ValueError( - "At least one of `user` and `usenetrc` must be specified") - # If no login/password was specified but netrc was requested, - # try to get them from ~/.netrc - # Presume that if .netrc has an entry, NNRP authentication is required. - try: - if usenetrc and not user: - import netrc - credentials = netrc.netrc() - auth = credentials.authenticators(self.host) - if auth: - user = auth[0] - password = auth[2] - except OSError: - pass - # Perform NNTP authentication if needed. - if not user: - return - resp = self._shortcmd('authinfo user ' + user) - if resp.startswith('381'): - if not password: - raise NNTPReplyError(resp) - else: - resp = self._shortcmd('authinfo pass ' + password) - if not resp.startswith('281'): - raise NNTPPermanentError(resp) - # Capabilities might have changed after login - self._caps = None - self.getcapabilities() - # Attempt to send mode reader if it was requested after login. - # Only do so if we're not in reader mode already. - if self.readermode_afterauth and 'READER' not in self._caps: - self._setreadermode() - # Capabilities might have changed after MODE READER - self._caps = None - self.getcapabilities() - - def _setreadermode(self): - try: - self.welcome = self._shortcmd('mode reader') - except NNTPPermanentError: - # Error 5xx, probably 'not implemented' - pass - except NNTPTemporaryError as e: - if e.response.startswith('480'): - # Need authorization before 'mode reader' - self.readermode_afterauth = True - else: - raise - - if _have_ssl: - def starttls(self, context=None): - """Process a STARTTLS command. Arguments: - - context: SSL context to use for the encrypted connection - """ - # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if - # a TLS session already exists. - if self.tls_on: - raise ValueError("TLS is already enabled.") - if self.authenticated: - raise ValueError("TLS cannot be started after authentication.") - resp = self._shortcmd('STARTTLS') - if resp.startswith('382'): - self.file.close() - self.sock = _encrypt_on(self.sock, context, self.host) - self.file = self.sock.makefile("rwb") - self.tls_on = True - # Capabilities may change after TLS starts up, so ask for them - # again. - self._caps = None - self.getcapabilities() - else: - raise NNTPError("TLS failed to start.") - - -if _have_ssl: - class NNTP_SSL(NNTP): - - def __init__(self, host, port=NNTP_SSL_PORT, - user=None, password=None, ssl_context=None, - readermode=None, usenetrc=False, - timeout=_GLOBAL_DEFAULT_TIMEOUT): - """This works identically to NNTP.__init__, except for the change - in default port and the `ssl_context` argument for SSL connections. - """ - self.ssl_context = ssl_context - super().__init__(host, port, user, password, readermode, - usenetrc, timeout) - - def _create_socket(self, timeout): - sock = super()._create_socket(timeout) - try: - sock = _encrypt_on(sock, self.ssl_context, self.host) - except: - sock.close() - raise - else: - return sock - - __all__.append("NNTP_SSL") - - -# Test retrieval when run as a script. -if __name__ == '__main__': - import argparse - - parser = argparse.ArgumentParser(description="""\ - nntplib built-in demo - display the latest articles in a newsgroup""") - parser.add_argument('-g', '--group', default='gmane.comp.python.general', - help='group to fetch messages from (default: %(default)s)') - parser.add_argument('-s', '--server', default='news.gmane.io', - help='NNTP server hostname (default: %(default)s)') - parser.add_argument('-p', '--port', default=-1, type=int, - help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT)) - parser.add_argument('-n', '--nb-articles', default=10, type=int, - help='number of articles to fetch (default: %(default)s)') - parser.add_argument('-S', '--ssl', action='store_true', default=False, - help='use NNTP over SSL') - args = parser.parse_args() - - port = args.port - if not args.ssl: - if port == -1: - port = NNTP_PORT - s = NNTP(host=args.server, port=port) - else: - if port == -1: - port = NNTP_SSL_PORT - s = NNTP_SSL(host=args.server, port=port) - - caps = s.getcapabilities() - if 'STARTTLS' in caps: - s.starttls() - resp, count, first, last, name = s.group(args.group) - print('Group', name, 'has', count, 'articles, range', first, 'to', last) - - def cut(s, lim): - if len(s) > lim: - s = s[:lim - 4] + "..." - return s - - first = str(int(last) - args.nb_articles + 1) - resp, overviews = s.xover(first, last) - for artnum, over in overviews: - author = decode_header(over['from']).split('<', 1)[0] - subject = decode_header(over['subject']) - lines = int(over[':lines']) - print("{:7} {:20} {:42} ({})".format( - artnum, cut(author, 20), cut(subject, 42), lines) - ) - - s.quit() diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 0444b0f65d1..01f060e70be 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -19,18 +19,17 @@ import os import sys -import stat import genericpath from genericpath import * - -__all__ = ["normcase","isabs","join","splitdrive","split","splitext", +__all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", - "ismount", "expanduser","expandvars","normpath","abspath", - "curdir","pardir","sep","pathsep","defpath","altsep", + "ismount","isreserved","expanduser","expandvars","normpath", + "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", - "samefile", "sameopenfile", "samestat", "commonpath"] + "samefile", "sameopenfile", "samestat", "commonpath", "isjunction", + "isdevdrive", "ALLOW_MISSING"] def _get_bothseps(path): if isinstance(path, bytes): @@ -78,12 +77,6 @@ def normcase(s): return s.replace('/', '\\').lower() -# Return whether a path is absolute. -# Trivial in Posix, harder on Windows. -# For Windows it is absolute if it starts with a slash or backslash (current -# volume), or if a pathname after the volume-letter-and-colon or UNC-resource -# starts with a slash or backslash. - def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) @@ -91,16 +84,15 @@ def isabs(s): sep = b'\\' altsep = b'/' colon_sep = b':\\' + double_sep = b'\\\\' else: sep = '\\' altsep = '/' colon_sep = ':\\' + double_sep = '\\\\' s = s[:3].replace(altsep, sep) # Absolute: UNC, device, and paths with a drive and root. - # LEGACY BUG: isabs("/x") should be false since the path has no drive. - if s.startswith(sep) or s.startswith(colon_sep, 1): - return True - return False + return s.startswith(colon_sep, 1) or s.startswith(double_sep) # Join two (or more) paths. @@ -109,27 +101,27 @@ def join(path, *paths): if isinstance(path, bytes): sep = b'\\' seps = b'\\/' - colon = b':' + colon_seps = b':\\/' else: sep = '\\' seps = '\\/' - colon = ':' + colon_seps = ':\\/' try: - if not paths: - path[:0] + sep #23780: Ensure compatible data type even if p is null. - result_drive, result_path = splitdrive(path) - for p in map(os.fspath, paths): - p_drive, p_path = splitdrive(p) - if p_path and p_path[0] in seps: + result_drive, result_root, result_path = splitroot(path) + for p in paths: + p_drive, p_root, p_path = splitroot(p) + if p_root: # Second path is absolute if p_drive or not result_drive: result_drive = p_drive + result_root = p_root result_path = p_path continue elif p_drive and p_drive != result_drive: if p_drive.lower() != result_drive.lower(): # Different drives => ignore the first path entirely result_drive = p_drive + result_root = p_root result_path = p_path continue # Same drive in different case @@ -139,10 +131,10 @@ def join(path, *paths): result_path = result_path + sep result_path = result_path + p_path ## add separator between UNC and non-absolute path - if (result_path and result_path[0] not in seps and - result_drive and result_drive[-1:] != colon): + if (result_path and not result_root and + result_drive and result_drive[-1] not in colon_seps): return result_drive + sep + result_path - return result_drive + result_path + return result_drive + result_root + result_path except (TypeError, AttributeError, BytesWarning): genericpath._check_arg_types('join', path, *paths) raise @@ -170,34 +162,56 @@ def splitdrive(p): Paths cannot contain both a drive letter and a UNC path. """ - p = os.fspath(p) - if len(p) >= 2: + drive, root, tail = splitroot(p) + return drive, root + tail + + +try: + from nt import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. + + The tail contains anything after the root.""" + p = os.fspath(p) if isinstance(p, bytes): sep = b'\\' altsep = b'/' colon = b':' unc_prefix = b'\\\\?\\UNC\\' + empty = b'' else: sep = '\\' altsep = '/' colon = ':' unc_prefix = '\\\\?\\UNC\\' + empty = '' normp = p.replace(altsep, sep) - if normp[0:2] == sep * 2: - # UNC drives, e.g. \\server\share or \\?\UNC\server\share - # Device drives, e.g. \\.\device or \\?\device - start = 8 if normp[:8].upper() == unc_prefix else 2 - index = normp.find(sep, start) - if index == -1: - return p, p[:0] - index2 = normp.find(sep, index + 1) - if index2 == -1: - return p, p[:0] - return p[:index2], p[index2:] - if normp[1:2] == colon: - # Drive-letter drives, e.g. X: - return p[:2], p[2:] - return p[:0], p + if normp[:1] == sep: + if normp[1:2] == sep: + # UNC drives, e.g. \\server\share or \\?\UNC\server\share + # Device drives, e.g. \\.\device or \\?\device + start = 8 if normp[:8].upper() == unc_prefix else 2 + index = normp.find(sep, start) + if index == -1: + return p, empty, empty + index2 = normp.find(sep, index + 1) + if index2 == -1: + return p, empty, empty + return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + else: + # Relative path with root, e.g. \Windows + return empty, p[:1], p[1:] + elif normp[1:2] == colon: + if normp[2:3] == sep: + # Absolute drive-letter path, e.g. X:\Windows + return p[:2], p[2:3], p[3:] + else: + # Relative path with drive, e.g. X:Windows + return p[:2], empty, p[2:] + else: + # Relative path, e.g. Windows + return empty, empty, p # Split a path in head (everything up to the last '/') and tail (the @@ -212,15 +226,13 @@ def split(p): Either part may be empty.""" p = os.fspath(p) seps = _get_bothseps(p) - d, p = splitdrive(p) + d, r, p = splitroot(p) # set i to index beyond p's last slash i = len(p) while i and p[i-1] not in seps: i -= 1 head, tail = p[:i], p[i:] # now tail has no slashes - # remove trailing slashes from head, unless it's all slashes - head = head.rstrip(seps) or head - return d + head, tail + return d + r + head.rstrip(seps), tail # Split a path in root and extension. @@ -250,28 +262,6 @@ def dirname(p): """Returns the directory component of a pathname""" return split(p)[0] -# Is a path a symbolic link? -# This will always return false on systems where os.lstat doesn't exist. - -def islink(path): - """Test whether a path is a symbolic link. - This will always return false for Windows prior to 6.0. - """ - try: - st = os.lstat(path) - except (OSError, ValueError, AttributeError): - return False - return stat.S_ISLNK(st.st_mode) - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - st = os.lstat(path) - except (OSError, ValueError): - return False - return True # Is a path a mount point? # Any drive letter root (eg c:\) @@ -293,10 +283,10 @@ def ismount(path): path = os.fspath(path) seps = _get_bothseps(path) path = abspath(path) - root, rest = splitdrive(path) - if root and root[0] in seps: - return (not rest) or (rest in seps) - if rest and rest in seps: + drive, root, rest = splitroot(path) + if drive and drive[0] in seps: + return not rest + if root and not rest: return True if _getvolumepathname: @@ -307,6 +297,40 @@ def ismount(path): return False +_reserved_chars = frozenset( + {chr(i) for i in range(32)} | + {'"', '*', ':', '<', '>', '?', '|', '/', '\\'} +) + +_reserved_names = frozenset( + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | + {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} +) + +def isreserved(path): + """Return true if the pathname is reserved by the system.""" + # Refer to "Naming Files, Paths, and Namespaces": + # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + path = os.fsdecode(splitroot(path)[2]).replace(altsep, sep) + return any(_isreservedname(name) for name in reversed(path.split(sep))) + +def _isreservedname(name): + """Return true if the filename is reserved by the system.""" + # Trailing dots and spaces are reserved. + if name[-1:] in ('.', ' '): + return name not in ('.', '..') + # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved. + # ASCII control characters (0-31) are reserved. + # Colon is reserved for file streams (e.g. "name:stream[:type]"). + if _reserved_chars.intersection(name): + return True + # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules + # are complex and vary across Windows versions. On the side of + # caution, return True for names that may not be reserved. + return name.partition('.')[0].rstrip(' ').upper() in _reserved_names + + # Expand paths beginning with '~' or '~user'. # '~' means $HOME; '~user' means that user's home directory. # If the path doesn't begin with '~', or if the user or $HOME is unknown, @@ -322,24 +346,23 @@ def expanduser(path): If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): + seps = b'\\/' tilde = b'~' else: + seps = '\\/' tilde = '~' if not path.startswith(tilde): return path i, n = 1, len(path) - while i < n and path[i] not in _get_bothseps(path): + while i < n and path[i] not in seps: i += 1 if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] - elif not 'HOMEPATH' in os.environ: + elif 'HOMEPATH' not in os.environ: return path else: - try: - drive = os.environ['HOMEDRIVE'] - except KeyError: - drive = '' + drive = os.environ.get('HOMEDRIVE', '') userhome = join(drive, os.environ['HOMEPATH']) if i != 1: #~user @@ -377,17 +400,23 @@ def expanduser(path): # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^|<>='. +_varpattern = r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)" +_varsub = None +_varsubb = None + def expandvars(path): """Expand shell variables of the forms $var, ${var} and %var%. Unknown variables are left unchanged.""" path = os.fspath(path) + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path and b'%' not in path: return path - import string - varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') - quote = b'\'' + if not _varsubb: + import re + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb percent = b'%' brace = b'{' rbrace = b'}' @@ -396,101 +425,51 @@ def expandvars(path): else: if '$' not in path and '%' not in path: return path - import string - varchars = string.ascii_letters + string.digits + '_-' - quote = '\'' + if not _varsub: + import re + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub percent = '%' brace = '{' rbrace = '}' dollar = '$' environ = os.environ - res = path[:0] - index = 0 - pathlen = len(path) - while index < pathlen: - c = path[index:index+1] - if c == quote: # no expansion within single quotes - path = path[index + 1:] - pathlen = len(path) - try: - index = path.index(c) - res += c + path[:index + 1] - except ValueError: - res += c + path - index = pathlen - 1 - elif c == percent: # variable or '%' - if path[index + 1:index + 2] == percent: - res += c - index += 1 - else: - path = path[index+1:] - pathlen = len(path) - try: - index = path.index(percent) - except ValueError: - res += percent + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = percent + var + percent - res += value - elif c == dollar: # variable or '$$' - if path[index + 1:index + 2] == dollar: - res += c - index += 1 - elif path[index + 1:index + 2] == brace: - path = path[index+2:] - pathlen = len(path) - try: - index = path.index(rbrace) - except ValueError: - res += dollar + brace + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + brace + var + rbrace - res += value - else: - var = path[:0] - index += 1 - c = path[index:index + 1] - while c and c in varchars: - var += c - index += 1 - c = path[index:index + 1] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + var - res += value - if c: - index -= 1 + + def repl(m): + lastindex = m.lastindex + if lastindex is None: + return m[0] + name = m[lastindex] + if lastindex == 1: + if name == percent: + return name + if not name.endswith(percent): + return m[0] + name = name[:-1] else: - res += c - index += 1 - return res + if name == dollar: + return name + if name.startswith(brace): + if not name.endswith(rbrace): + return m[0] + name = name[1:-1] + + try: + if environ is None: + return os.fsencode(os.environ[os.fsdecode(name)]) + else: + return environ[name] + except KeyError: + return m[0] + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! try: - from nt import _path_normpath + from nt import _path_normpath as normpath except ImportError: def normpath(path): @@ -507,13 +486,8 @@ def normpath(path): curdir = '.' pardir = '..' path = path.replace(altsep, sep) - prefix, path = splitdrive(path) - - # collapse initial backslashes - if path.startswith(sep): - prefix += sep - path = path.lstrip(sep) - + drive, root, path = splitroot(path) + prefix = drive + root comps = path.split(sep) i = 0 while i < len(comps): @@ -523,7 +497,7 @@ def normpath(path): if i > 0 and comps[i-1] != pardir: del comps[i-1:i+1] i -= 1 - elif i == 0 and prefix.endswith(sep): + elif i == 0 and root: del comps[i] else: i += 1 @@ -534,37 +508,22 @@ def normpath(path): comps.append(curdir) return prefix + sep.join(comps) -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - - -def _abspath_fallback(path): - """Return the absolute version of a path as a fallback function in case - `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for - more. - - """ - - path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) - return normpath(path) # Return an absolute path. try: from nt import _getfullpathname except ImportError: # not running on Windows - mock up something sensible - abspath = _abspath_fallback + def abspath(path): + """Return the absolute version of a path.""" + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + cwd = os.getcwdb() + else: + cwd = os.getcwd() + path = join(cwd, path) + return normpath(path) else: # use native Windows method on Windows def abspath(path): @@ -572,15 +531,36 @@ def abspath(path): try: return _getfullpathname(normpath(path)) except (OSError, ValueError): - return _abspath_fallback(path) + # See gh-75230, handle outside for cleaner traceback + pass + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + sep = b'\\' + getcwd = os.getcwdb + else: + sep = '\\' + getcwd = os.getcwd + drive, root, path = splitroot(path) + # Either drive or root can be nonempty, but not both. + if drive or root: + try: + path = join(_getfullpathname(drive + root), path) + except (OSError, ValueError): + # Drive "\0:" cannot exist; use the root directory. + path = drive + sep + path + else: + path = join(getcwd(), path) + return normpath(path) try: - from nt import _getfinalpathname, readlink as _nt_readlink + from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink except ImportError: # realpath is a no-op on systems without _getfinalpathname support. - realpath = abspath + def realpath(path, *, strict=False): + return abspath(path) else: - def _readlink_deep(path): + def _readlink_deep(path, ignored_error=OSError): # These error codes indicate that we should stop reading links and # return the path we currently have. # 1: ERROR_INVALID_FUNCTION @@ -613,7 +593,7 @@ def _readlink_deep(path): path = old_path break path = normpath(join(dirname(old_path), path)) - except OSError as ex: + except ignored_error as ex: if ex.winerror in allowed_winerror: break raise @@ -622,7 +602,7 @@ def _readlink_deep(path): break return path - def _getfinalpathname_nonstrict(path): + def _getfinalpathname_nonstrict(path, ignored_error=OSError): # These error codes indicate that we should stop resolving the path # and return the value we currently have. # 1: ERROR_INVALID_FUNCTION @@ -638,9 +618,10 @@ def _getfinalpathname_nonstrict(path): # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME # 161: ERROR_BAD_PATHNAME + # 1005: ERROR_UNRECOGNIZED_VOLUME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1005, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. @@ -649,23 +630,29 @@ def _getfinalpathname_nonstrict(path): try: path = _getfinalpathname(path) return join(path, tail) if tail else path - except OSError as ex: + except ignored_error as ex: if ex.winerror not in allowed_winerror: raise try: # The OS could not resolve this path fully, so we attempt # to follow the link ourselves. If we succeed, join the tail # and return. - new_path = _readlink_deep(path) + new_path = _readlink_deep(path, + ignored_error=ignored_error) if new_path != path: return join(new_path, tail) if tail else new_path - except OSError: + except ignored_error: # If we fail to readlink(), let's keep traversing pass - path, name = split(path) - # TODO (bpo-38186): Request the real file name from the directory - # entry using FindFirstFileW. For now, we will return the path - # as best we have it + # If we get these errors, try to get the real name of the file without accessing it. + if ex.winerror in (1, 5, 32, 50, 87, 1920, 1921): + try: + name = _findfirstfile(path) + path, _ = split(path) + except ignored_error: + path, name = split(path) + else: + path, name = split(path) if path and not name: return path + tail tail = join(name, tail) if tail else name @@ -679,7 +666,8 @@ def realpath(path, *, strict=False): new_unc_prefix = b'\\\\' cwd = os.getcwdb() # bpo-38081: Special case for realpath(b'nul') - if normcase(path) == normcase(os.fsencode(devnull)): + devnull = b'nul' + if normcase(path) == devnull: return b'\\\\.\\NUL' else: prefix = '\\\\?\\' @@ -687,19 +675,36 @@ def realpath(path, *, strict=False): new_unc_prefix = '\\\\' cwd = os.getcwd() # bpo-38081: Special case for realpath('nul') - if normcase(path) == normcase(devnull): + devnull = 'nul' + if normcase(path) == devnull: return '\\\\.\\NUL' had_prefix = path.startswith(prefix) + + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + strict = True + elif strict: + ignored_error = () + else: + ignored_error = OSError + if not had_prefix and not isabs(path): path = join(cwd, path) try: path = _getfinalpathname(path) initial_winerror = 0 - except OSError as ex: + except ValueError as ex: + # gh-106242: Raised for embedded null characters + # In strict modes, we convert into an OSError. + # Non-strict mode returns the path as-is, since we've already + # made it absolute. if strict: - raise + raise OSError(str(ex)) from None + path = normpath(path) + except ignored_error as ex: initial_winerror = ex.winerror - path = _getfinalpathname_nonstrict(path) + path = _getfinalpathname_nonstrict(path, + ignored_error=ignored_error) # The path returned by _getfinalpathname will always start with \\?\ - # strip off that prefix unless it was already provided on the original # path. @@ -714,6 +719,10 @@ def realpath(path, *, strict=False): try: if _getfinalpathname(spath) == path: path = spath + except ValueError as ex: + # Unexpected, as an invalid path should not have gained a prefix + # at any point, but we ignore this error just in case. + pass except OSError as ex: # If the path does not exist and originally did not exist, then # strip the prefix anyway. @@ -722,13 +731,15 @@ def realpath(path, *, strict=False): return path -# Win9x family and earlier have no Unicode filename support. -supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and - sys.getwindowsversion()[3] >= 2) +# All supported version have Unicode filename support. +supports_unicode_filenames = True def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + if not path: + raise ValueError("no path specified") + if isinstance(path, bytes): sep = b'\\' curdir = b'.' @@ -740,22 +751,20 @@ def relpath(path, start=None): if start is None: start = curdir + else: + start = os.fspath(start) - if not path: - raise ValueError("no path specified") - - start = os.fspath(start) try: - start_abs = abspath(normpath(start)) - path_abs = abspath(normpath(path)) - start_drive, start_rest = splitdrive(start_abs) - path_drive, path_rest = splitdrive(path_abs) + start_abs = abspath(start) + path_abs = abspath(path) + start_drive, _, start_rest = splitroot(start_abs) + path_drive, _, path_rest = splitroot(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) - start_list = [x for x in start_rest.split(sep) if x] - path_list = [x for x in path_rest.split(sep) if x] + start_list = start_rest.split(sep) if start_rest else [] + path_list = path_rest.split(sep) if path_rest else [] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): @@ -766,29 +775,28 @@ def relpath(path, start=None): rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise -# Return the longest common sub-path of the sequence of paths given as input. +# Return the longest common sub-path of the iterable of paths given as input. # The function is case-insensitive and 'separator-insensitive', i.e. if the # only difference between two paths is the use of '\' versus '/' as separator, # they are deemed to be equal. # # However, the returned path will have the standard '\' separator (even if the # given paths had the alternative '/' separator) and will have the case of the -# first path given in the sequence. Additionally, any trailing separator is +# first path given in the iterable. Additionally, any trailing separator is # stripped from the returned path. def commonpath(paths): - """Given a sequence of path names, returns the longest common sub-path.""" - + """Given an iterable of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) if not paths: - raise ValueError('commonpath() arg is an empty sequence') + raise ValueError('commonpath() arg is an empty iterable') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'\\' altsep = b'/' @@ -799,21 +807,22 @@ def commonpath(paths): curdir = '.' try: - drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths] - split_paths = [p.split(sep) for d, p in drivesplits] - - try: - isabs, = set(p[:1] == sep for d, p in drivesplits) - except ValueError: - raise ValueError("Can't mix absolute and relative paths") from None + drivesplits = [splitroot(p.replace(altsep, sep).lower()) for p in paths] + split_paths = [p.split(sep) for d, r, p in drivesplits] # Check that all drive letters or UNC paths match. The check is made only # now otherwise type errors for mixing strings and bytes would not be # caught. - if len(set(d for d, p in drivesplits)) != 1: + if len({d for d, r, p in drivesplits}) != 1: raise ValueError("Paths don't have the same drive") - drive, path = splitdrive(paths[0].replace(altsep, sep)) + drive, root, path = splitroot(paths[0].replace(altsep, sep)) + if len({r for d, r, p in drivesplits}) != 1: + if drive: + raise ValueError("Can't mix absolute and relative paths") + else: + raise ValueError("Can't mix rooted and not-rooted paths") + common = path.split(sep) common = [c for c in common if c and c != curdir] @@ -827,19 +836,35 @@ def commonpath(paths): else: common = common[:len(s1)] - prefix = drive + sep if isabs else drive - return prefix + sep.join(common) + return drive + root + sep.join(common) except (TypeError, AttributeError): genericpath._check_arg_types('commonpath', *paths) raise try: - # The genericpath.isdir implementation uses os.stat and checks the mode - # attribute to tell whether or not the path is a directory. - # This is overkill on Windows - just pass the path to GetFileAttributes - # and check the attribute from there. - from nt import _isdir as isdir + # The isdir(), isfile(), islink(), exists() and lexists() implementations + # in genericpath use os.stat(). This is overkill on Windows. Use simpler + # builtin functions if they are available. + from nt import _path_isdir as isdir + from nt import _path_isfile as isfile + from nt import _path_islink as islink + from nt import _path_isjunction as isjunction + from nt import _path_exists as exists + from nt import _path_lexists as lexists +except ImportError: + # Use genericpath.* as imported above + pass + + +try: + from nt import _path_isdevdrive + def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive.""" + try: + return _path_isdevdrive(abspath(path)) + except OSError: + return False except ImportError: - # Use genericpath.isdir as imported above. + # Use genericpath.isdevdrive as imported above pass diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py index 61852aff589..57c7858dff0 100644 --- a/Lib/nturl2path.py +++ b/Lib/nturl2path.py @@ -3,7 +3,15 @@ This module only exists to provide OS-specific code for urllib.requests, thus do not use directly. """ -# Testing is done through test_urllib. +# Testing is done through test_nturl2path. + +import warnings + + +warnings._deprecated( + __name__, + message=f"{warnings._DEPRECATED_MSG}; use 'urllib.request' instead", + remove=(3, 19)) def url2pathname(url): """OS-specific conversion from a relative URL of the 'file' scheme @@ -14,33 +22,25 @@ def url2pathname(url): # ///C:/foo/bar/spam.foo # become # C:\foo\bar\spam.foo - import string, urllib.parse - # Windows itself uses ":" even in URLs. - url = url.replace(':', '|') - if not '|' in url: - # No drive specifier, just convert slashes - if url[:4] == '////': - # path is something like ////host/path/on/remote/host - # convert this to \\host\path\on\remote\host - # (notice halving of slashes at the start of the path) - url = url[2:] - components = url.split('/') - # make sure not to convert quoted slashes :-) - return urllib.parse.unquote('\\'.join(components)) - comp = url.split('|') - if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: - error = 'Bad URL: ' + url - raise OSError(error) - drive = comp[0][-1].upper() - components = comp[1].split('/') - path = drive + ':' - for comp in components: - if comp: - path = path + '\\' + urllib.parse.unquote(comp) - # Issue #11474 - handing url such as |c/| - if path.endswith(':') and url.endswith('/'): - path += '\\' - return path + import urllib.parse + if url[:3] == '///': + # URL has an empty authority section, so the path begins on the third + # character. + url = url[2:] + elif url[:12] == '//localhost/': + # Skip past 'localhost' authority. + url = url[11:] + if url[:3] == '///': + # Skip past extra slash before UNC drive in URL path. + url = url[1:] + else: + if url[:1] == '/' and url[2:3] in (':', '|'): + # Skip past extra slash before DOS drive in URL path. + url = url[1:] + if url[1:2] == '|': + # Older URLs use a pipe after a drive letter + url = url[:1] + ':' + url[2:] + return urllib.parse.unquote(url.replace('/', '\\')) def pathname2url(p): """OS-specific conversion from a file system path to a relative URL @@ -49,33 +49,26 @@ def pathname2url(p): # C:\foo\bar\spam.foo # becomes # ///C:/foo/bar/spam.foo + import ntpath import urllib.parse # First, clean up some special forms. We are going to sacrifice # the additional information anyway - if p[:4] == '\\\\?\\': + p = p.replace('\\', '/') + if p[:4] == '//?/': p = p[4:] - if p[:4].upper() == 'UNC\\': - p = '\\' + p[4:] - elif p[1:2] != ':': - raise OSError('Bad path: ' + p) - if not ':' in p: - # No drive specifier, just convert slashes and quote the name - if p[:2] == '\\\\': - # path is something like \\host\path\on\remote\host - # convert this to ////host/path/on/remote/host - # (notice doubling of slashes at the start of the path) - p = '\\\\' + p - components = p.split('\\') - return urllib.parse.quote('/'.join(components)) - comp = p.split(':', maxsplit=2) - if len(comp) != 2 or len(comp[0]) > 1: - error = 'Bad path: ' + p - raise OSError(error) + if p[:4].upper() == 'UNC/': + p = '//' + p[4:] + drive, root, tail = ntpath.splitroot(p) + if drive: + if drive[1:] == ':': + # DOS drive specified. Add three slashes to the start, producing + # an authority section with a zero-length authority, and a path + # section starting with a single slash. + drive = f'///{drive}' + drive = urllib.parse.quote(drive, safe='/:') + elif root: + # Add explicitly empty authority to path beginning with one slash. + root = f'//{root}' - drive = urllib.parse.quote(comp[0].upper()) - components = comp[1].split('\\') - path = '///' + drive + ':' - for comp in components: - if comp: - path = path + '/' + urllib.parse.quote(comp) - return path + tail = urllib.parse.quote(tail) + return drive + root + tail diff --git a/Lib/numbers.py b/Lib/numbers.py index 0985dd85f60..37fddb89177 100644 --- a/Lib/numbers.py +++ b/Lib/numbers.py @@ -5,6 +5,31 @@ TODO: Fill out more detailed documentation on the operators.""" +############ Maintenance notes ######################################### +# +# ABCs are different from other standard library modules in that they +# specify compliance tests. In general, once an ABC has been published, +# new methods (either abstract or concrete) cannot be added. +# +# Though classes that inherit from an ABC would automatically receive a +# new mixin method, registered classes would become non-compliant and +# violate the contract promised by ``isinstance(someobj, SomeABC)``. +# +# Though irritating, the correct procedure for adding new abstract or +# mixin methods is to create a new ABC as a subclass of the previous +# ABC. +# +# Because they are so hard to change, new ABCs should have their APIs +# carefully thought through prior to publication. +# +# Since ABCMeta only checks for the presence of methods, it is possible +# to alter the signature of a method by adding optional arguments +# or changing parameter names. This is still a bit dubious but at +# least it won't cause isinstance() to return an incorrect result. +# +# +####################################################################### + from abc import ABCMeta, abstractmethod __all__ = ["Number", "Complex", "Real", "Rational", "Integral"] @@ -118,7 +143,7 @@ def __rtruediv__(self, other): @abstractmethod def __pow__(self, exponent): - """self**exponent; should promote to float or complex when necessary.""" + """self ** exponent; should promote to float or complex when necessary.""" raise NotImplementedError @abstractmethod @@ -167,7 +192,7 @@ def __trunc__(self): """trunc(self): Truncates self to an Integral. Returns an Integral i such that: - * i>0 iff self>0; + * i > 0 iff self > 0; * abs(i) <= abs(self); * for any Integral j satisfying the first two conditions, abs(i) >= abs(j) [i.e. i has "maximal" abs among those]. @@ -203,7 +228,7 @@ def __divmod__(self, other): return (self // other, self % other) def __rdivmod__(self, other): - """divmod(other, self): The pair (self // other, self % other). + """divmod(other, self): The pair (other // self, other % self). Sometimes this can be computed faster than the pair of operations. @@ -265,18 +290,27 @@ def conjugate(self): class Rational(Real): - """.numerator and .denominator should be in lowest terms.""" + """To Real, Rational adds numerator and denominator properties. + + The numerator and denominator values should be in lowest terms, + with a positive denominator. + """ __slots__ = () @property @abstractmethod def numerator(self): + """The numerator of a rational number in lowest terms.""" raise NotImplementedError @property @abstractmethod def denominator(self): + """The denominator of a rational number in lowest terms. + + This denominator should be positive. + """ raise NotImplementedError # Concrete implementation of Real's conversion to float. diff --git a/Lib/opcode.py b/Lib/opcode.py index ab6b765b4b7..0e9520b6832 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,424 +4,73 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG"] -# It's a chicken-and-egg I'm afraid: -# We're imported before _opcode's made. -# With exception unheeded -# (stack_effect is not needed) -# Both our chickens and eggs are allayed. -# --Larry Hastings, 2013/11/23 +__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap", + "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname", + "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"] -try: - from _opcode import stack_effect - __all__.append('stack_effect') -except ImportError: - pass +import builtins +import _opcode +from _opcode import stack_effect -cmp_op = ('<', '<=', '==', '!=', '>', '>=') - -hasarg = [] -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasexc = [] - -def is_pseudo(op): - return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE - -oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, - haslocal, hascompare, hasfree, hasexc] - -opmap = {} - -## pseudo opcodes (used in the compiler) mapped to the values -## they can become in the actual code. -_pseudo_ops = {} - -def def_op(name, op): - opmap[name] = op - -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) - -def pseudo_op(name, op, real_ops): - def_op(name, op) - _pseudo_ops[name] = real_ops - # add the pseudo opcode to the lists its targets are in - for oplist in oplists: - res = [opmap[rop] in oplist for rop in real_ops] - if any(res): - assert all(res) - oplist.append(op) - - -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('CACHE', 0) -def_op('POP_TOP', 1) -def_op('PUSH_NULL', 2) - -def_op('NOP', 9) -def_op('UNARY_POSITIVE', 10) -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) - -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_SLICE', 26) -def_op('STORE_SLICE', 27) - -def_op('GET_LEN', 30) -def_op('MATCH_MAPPING', 31) -def_op('MATCH_SEQUENCE', 32) -def_op('MATCH_KEYS', 33) - -def_op('PUSH_EXC_INFO', 35) -def_op('CHECK_EXC_MATCH', 36) -def_op('CHECK_EG_MATCH', 37) - -def_op('WITH_EXCEPT_START', 49) -def_op('GET_AITER', 50) -def_op('GET_ANEXT', 51) -def_op('BEFORE_ASYNC_WITH', 52) -def_op('BEFORE_WITH', 53) -def_op('END_ASYNC_FOR', 54) -def_op('CLEANUP_THROW', 55) - -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) - -# TODO: RUSTPYTHON -# Delete below def_op after updating coroutines.py -def_op('YIELD_FROM', 72) - -def_op('GET_ITER', 68) -def_op('GET_YIELD_FROM_ITER', 69) -def_op('PRINT_EXPR', 70) -def_op('LOAD_BUILD_CLASS', 71) - -def_op('LOAD_ASSERTION_ERROR', 74) -def_op('RETURN_GENERATOR', 75) - -def_op('LIST_TO_TUPLE', 82) -def_op('RETURN_VALUE', 83) -def_op('IMPORT_STAR', 84) -def_op('SETUP_ANNOTATIONS', 85) - -def_op('ASYNC_GEN_WRAP', 87) -def_op('PREP_RERAISE_STAR', 88) -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # real opcodes from here have an argument: - -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" -def_op('SWAP', 99) -def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries -name_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list -jrel_op('JUMP_FORWARD', 110) # Number of words to skip -jrel_op('JUMP_IF_FALSE_OR_POP', 111) # Number of words to skip -jrel_op('JUMP_IF_TRUE_OR_POP', 112) # "" -jrel_op('POP_JUMP_IF_FALSE', 114) -jrel_op('POP_JUMP_IF_TRUE', 115) -name_op('LOAD_GLOBAL', 116) # Index in name list -def_op('IS_OP', 117) -def_op('CONTAINS_OP', 118) -def_op('RERAISE', 119) -def_op('COPY', 120) -def_op('BINARY_OP', 122) -jrel_op('SEND', 123) # Number of bytes to skip -def_op('LOAD_FAST', 124) # Local variable number, no null check -haslocal.append(124) -def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) -def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) -def_op('LOAD_FAST_CHECK', 127) # Local variable number -haslocal.append(127) -jrel_op('POP_JUMP_IF_NOT_NONE', 128) -jrel_op('POP_JUMP_IF_NONE', 129) -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('GET_AWAITABLE', 131) -def_op('MAKE_FUNCTION', 132) # Flags -def_op('BUILD_SLICE', 133) # Number of items -jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) -def_op('MAKE_CELL', 135) -hasfree.append(135) -def_op('LOAD_CLOSURE', 136) -hasfree.append(136) -def_op('LOAD_DEREF', 137) -hasfree.append(137) -def_op('STORE_DEREF', 138) -hasfree.append(138) -def_op('DELETE_DEREF', 139) -hasfree.append(139) -jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) - -def_op('CALL_FUNCTION_EX', 142) # Flags - -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) -def_op('LOAD_CLASSDEREF', 148) -hasfree.append(148) -def_op('COPY_FREE_VARS', 149) -def_op('YIELD_VALUE', 150) -def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py -def_op('MATCH_CLASS', 152) - -def_op('FORMAT_VALUE', 155) -def_op('BUILD_CONST_KEY_MAP', 156) -def_op('BUILD_STRING', 157) - -def_op('LIST_EXTEND', 162) -def_op('SET_UPDATE', 163) -def_op('DICT_MERGE', 164) -def_op('DICT_UPDATE', 165) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, # noqa: F401 + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) # noqa: F401 +EXTENDED_ARG = opmap['EXTENDED_ARG'] -def_op('CALL', 171) -def_op('KW_NAMES', 172) -hasconst.append(172) +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] +for m in (opmap, _specialized_opmap): + for op, i in m.items(): + opname[i] = op +cmp_op = ('<', '<=', '==', '!=', '>', '>=') -hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) - -MIN_PSEUDO_OPCODE = 256 - -pseudo_op('SETUP_FINALLY', 256, ['NOP']) -hasexc.append(256) -pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -hasexc.append(257) -pseudo_op('SETUP_WITH', 258, ['NOP']) -hasexc.append(258) -pseudo_op('POP_BLOCK', 259, ['NOP']) - -pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) -pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) - -pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR']) - -MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 - -del def_op, name_op, jrel_op, jabs_op, pseudo_op - -opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] -for op, i in opmap.items(): - opname[i] = op +# These lists are documented as part of the dis module's API +hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] +hasconst = [op for op in opmap.values() if _opcode.has_const(op)] +hasname = [op for op in opmap.values() if _opcode.has_name(op)] +hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] +hasjrel = hasjump # for backward compatibility +hasjabs = [] +hasfree = [op for op in opmap.values() if _opcode.has_free(op)] +haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] -_nb_ops = [ - ("NB_ADD", "+"), - ("NB_AND", "&"), - ("NB_FLOOR_DIVIDE", "//"), - ("NB_LSHIFT", "<<"), - ("NB_MATRIX_MULTIPLY", "@"), - ("NB_MULTIPLY", "*"), - ("NB_REMAINDER", "%"), - ("NB_OR", "|"), - ("NB_POWER", "**"), - ("NB_RSHIFT", ">>"), - ("NB_SUBTRACT", "-"), - ("NB_TRUE_DIVIDE", "/"), - ("NB_XOR", "^"), - ("NB_INPLACE_ADD", "+="), - ("NB_INPLACE_AND", "&="), - ("NB_INPLACE_FLOOR_DIVIDE", "//="), - ("NB_INPLACE_LSHIFT", "<<="), - ("NB_INPLACE_MATRIX_MULTIPLY", "@="), - ("NB_INPLACE_MULTIPLY", "*="), - ("NB_INPLACE_REMAINDER", "%="), - ("NB_INPLACE_OR", "|="), - ("NB_INPLACE_POWER", "**="), - ("NB_INPLACE_RSHIFT", ">>="), - ("NB_INPLACE_SUBTRACT", "-="), - ("NB_INPLACE_TRUE_DIVIDE", "/="), - ("NB_INPLACE_XOR", "^="), -] +_intrinsic_1_descs = _opcode.get_intrinsic1_descs() +_intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_special_method_names = _opcode.get_special_method_names() +_common_constants = [builtins.AssertionError, builtins.NotImplementedError, + builtins.tuple, builtins.all, builtins.any] +_nb_ops = _opcode.get_nb_ops() -_specializations = { - "BINARY_OP": [ - "BINARY_OP_ADAPTIVE", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_ADD_INT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_SUBTRACT_INT", - ], - "BINARY_SUBSCR": [ - "BINARY_SUBSCR_ADAPTIVE", - "BINARY_SUBSCR_DICT", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_TUPLE_INT", - ], - "CALL": [ - "CALL_ADAPTIVE", - "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_NO_KW_BUILTIN_FAST", - "CALL_NO_KW_BUILTIN_O", - "CALL_NO_KW_ISINSTANCE", - "CALL_NO_KW_LEN", - "CALL_NO_KW_LIST_APPEND", - "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "CALL_NO_KW_METHOD_DESCRIPTOR_O", - "CALL_NO_KW_STR_1", - "CALL_NO_KW_TUPLE_1", - "CALL_NO_KW_TYPE_1", - ], - "COMPARE_OP": [ - "COMPARE_OP_ADAPTIVE", - "COMPARE_OP_FLOAT_JUMP", - "COMPARE_OP_INT_JUMP", - "COMPARE_OP_STR_JUMP", - ], - "EXTENDED_ARG": [ - "EXTENDED_ARG_QUICK", - ], - "FOR_ITER": [ - "FOR_ITER_ADAPTIVE", - "FOR_ITER_LIST", - "FOR_ITER_RANGE", - ], - "JUMP_BACKWARD": [ - "JUMP_BACKWARD_QUICK", - ], - "LOAD_ATTR": [ - "LOAD_ATTR_ADAPTIVE", - # These potentially push [NULL, bound method] onto the stack. - "LOAD_ATTR_CLASS", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_WITH_HINT", - # These will always push [unbound method, self] onto the stack. - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_WITH_DICT", - "LOAD_ATTR_METHOD_WITH_VALUES", - ], - "LOAD_CONST": [ - "LOAD_CONST__LOAD_FAST", - ], - "LOAD_FAST": [ - "LOAD_FAST__LOAD_CONST", - "LOAD_FAST__LOAD_FAST", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_ADAPTIVE", - "LOAD_GLOBAL_BUILTIN", - "LOAD_GLOBAL_MODULE", - ], - "RESUME": [ - "RESUME_QUICK", - ], - "STORE_ATTR": [ - "STORE_ATTR_ADAPTIVE", - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "STORE_FAST": [ - "STORE_FAST__LOAD_FAST", - "STORE_FAST__STORE_FAST", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_ADAPTIVE", - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_ADAPTIVE", - "UNPACK_SEQUENCE_LIST", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_TWO_TUPLE", - ], -} -_specialized_instructions = [ - opcode for family in _specializations.values() for opcode in family -] -_specialization_stats = [ - "success", - "failure", - "hit", - "deferred", - "miss", - "deopt", -] +hascompare = [opmap["COMPARE_OP"]] _cache_format = { "LOAD_GLOBAL": { "counter": 1, "index": 1, - "module_keys_version": 2, + "module_keys_version": 1, "builtin_keys_version": 1, }, "BINARY_OP": { "counter": 1, + "descr": 4, }, "UNPACK_SEQUENCE": { "counter": 1, }, "COMPARE_OP": { "counter": 1, - "mask": 1, }, - "BINARY_SUBSCR": { + "CONTAINS_OP": { "counter": 1, - "type_version": 2, - "func_version": 1, }, "FOR_ITER": { "counter": 1, }, + "LOAD_SUPER_ATTR": { + "counter": 1, + }, "LOAD_ATTR": { "counter": 1, "version": 2, @@ -436,13 +85,38 @@ def pseudo_op(name, op, real_ops): "CALL": { "counter": 1, "func_version": 2, - "min_args": 1, + }, + "CALL_KW": { + "counter": 1, + "func_version": 2, }, "STORE_SUBSCR": { "counter": 1, }, + "SEND": { + "counter": 1, + }, + "JUMP_BACKWARD": { + "counter": 1, + }, + "TO_BOOL": { + "counter": 1, + "version": 2, + }, + "POP_JUMP_IF_TRUE": { + "counter": 1, + }, + "POP_JUMP_IF_FALSE": { + "counter": 1, + }, + "POP_JUMP_IF_NONE": { + "counter": 1, + }, + "POP_JUMP_IF_NOT_NONE": { + "counter": 1, + }, } -_inline_cache_entries = [ - sum(_cache_format.get(opname[opcode], {}).values()) for opcode in range(256) -] +_inline_cache_entries = { + name : sum(value.values()) for (name, value) in _cache_format.items() +} diff --git a/Lib/operator.py b/Lib/operator.py index 30116c1189a..1b765522f85 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -14,8 +14,8 @@ 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', - 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', - 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'is_', 'is_none', 'is_not', 'is_not_none', 'isub', 'itemgetter', 'itruediv', + 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub', 'truediv', 'truth', 'xor'] @@ -66,6 +66,14 @@ def is_not(a, b): "Same as a is not b." return a is not b +def is_none(a): + "Same as a is None." + return a is None + +def is_not_none(a): + "Same as a is not None." + return a is not None + # Mathematical/Bitwise Operations *********************************************# def abs(a): @@ -239,7 +247,7 @@ class attrgetter: """ __slots__ = ('_attrs', '_call') - def __init__(self, attr, *attrs): + def __init__(self, attr, /, *attrs): if not attrs: if not isinstance(attr, str): raise TypeError('attribute name must be a string') @@ -257,7 +265,7 @@ def func(obj): return tuple(getter(obj) for getter in getters) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -276,7 +284,7 @@ class itemgetter: """ __slots__ = ('_items', '_call') - def __init__(self, item, *items): + def __init__(self, item, /, *items): if not items: self._items = (item,) def func(obj): @@ -288,7 +296,7 @@ def func(obj): return tuple(obj[i] for i in items) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -315,7 +323,7 @@ def __init__(self, name, /, *args, **kwargs): self._args = args self._kwargs = kwargs - def __call__(self, obj): + def __call__(self, obj, /): return getattr(obj, self._name)(*self._args, **self._kwargs) def __repr__(self): @@ -415,7 +423,7 @@ def ixor(a, b): except ImportError: pass else: - from _operator import __doc__ + from _operator import __doc__ # noqa: F401 # All of these "__func__ = func" assignments have to happen after importing # from _operator to make sure they're set to the right function diff --git a/Lib/optparse.py b/Lib/optparse.py index 1c450c6fcbe..38cf16d21ef 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -43,7 +43,7 @@ __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. -Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. +Copyright (c) 2002 Python Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -74,7 +74,8 @@ """ import sys, os -import textwrap +from gettext import gettext as _, ngettext + def _repr(self): return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self) @@ -86,19 +87,6 @@ def _repr(self): # Id: help.py 527 2006-07-23 15:21:30Z greg # Id: errors.py 509 2006-04-20 00:58:24Z gward -try: - from gettext import gettext, ngettext -except ImportError: - def gettext(message): - return message - - def ngettext(singular, plural, n): - if n == 1: - return singular - return plural - -_ = gettext - class OptParseError (Exception): def __init__(self, msg): @@ -263,6 +251,7 @@ def _format_text(self, text): Format a paragraph of free-form text for inclusion in the help output at the current indentation level. """ + import textwrap text_width = max(self.width - self.current_indent, 11) indent = " "*self.current_indent return textwrap.fill(text, @@ -319,6 +308,7 @@ def format_option(self, option): indent_first = 0 result.append(opts) if option.help: + import textwrap help_text = self.expand_default(option) help_lines = textwrap.wrap(help_text, self.help_width) result.append("%*s%s\n" % (indent_first, "", help_lines[0])) diff --git a/Lib/os.py b/Lib/os.py index d26cfc99939..ac03b416390 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -10,7 +10,7 @@ - os.extsep is the extension separator (always '.') - os.altsep is the alternate pathname separator (None or '/') - os.pathsep is the component separator used in $PATH etc - - os.linesep is the line separator in text files ('\r' or '\n' or '\r\n') + - os.linesep is the line separator in text files ('\n' or '\r\n') - os.defpath is the default search path for executables - os.devnull is the file path of the null device ('/dev/null', etc.) @@ -64,6 +64,10 @@ def _get_exports_list(module): from posix import _have_functions except ImportError: pass + try: + from posix import _create_environ + except ImportError: + pass import posix __all__.extend(_get_exports_list(posix)) @@ -88,6 +92,10 @@ def _get_exports_list(module): from nt import _have_functions except ImportError: pass + try: + from nt import _create_environ + except ImportError: + pass else: raise ImportError('no os specific module found') @@ -110,6 +118,7 @@ def _add(str, fn): _add("HAVE_FCHMODAT", "chmod") _add("HAVE_FCHOWNAT", "chown") _add("HAVE_FSTATAT", "stat") + _add("HAVE_LSTAT", "lstat") _add("HAVE_FUTIMESAT", "utime") _add("HAVE_LINKAT", "link") _add("HAVE_MKDIRAT", "mkdir") @@ -131,6 +140,7 @@ def _add(str, fn): _set = set() _add("HAVE_FCHDIR", "chdir") _add("HAVE_FCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") _add("HAVE_FCHOWN", "chown") _add("HAVE_FDOPENDIR", "listdir") _add("HAVE_FDOPENDIR", "scandir") @@ -171,6 +181,7 @@ def _add(str, fn): _add("HAVE_FSTATAT", "stat") _add("HAVE_LCHFLAGS", "chflags") _add("HAVE_LCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") if _exists("lchown"): # mac os x10.3 _add("HAVE_LCHOWN", "chown") _add("HAVE_LINKAT", "link") @@ -279,6 +290,10 @@ def renames(old, new): __all__.extend(["makedirs", "removedirs", "renames"]) +# Private sentinel that makes walk() classify all symlinks and junctions as +# regular files. +_walk_symlinks_as_files = object() + def walk(top, topdown=True, onerror=None, followlinks=False): """Directory tree generator. @@ -288,7 +303,8 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirpath, dirnames, filenames dirpath is a string, the path to the directory. dirnames is a list of - the names of the subdirectories in dirpath (excluding '.' and '..'). + the names of the subdirectories in dirpath (including symlinks to directories, + and excluding '.' and '..'). filenames is a list of the names of the non-directory files in dirpath. Note that the names in the lists are just names, with no path components. To get a full path (which begins with top) to a file or directory in @@ -330,98 +346,91 @@ def walk(top, topdown=True, onerror=None, followlinks=False): import os from os.path import join, getsize - for root, dirs, files in os.walk('python/Lib/email'): - print(root, "consumes", end="") - print(sum(getsize(join(root, name)) for name in files), end="") + for root, dirs, files in os.walk('python/Lib/xml'): + print(root, "consumes ") + print(sum(getsize(join(root, name)) for name in files), end=" ") print("bytes in", len(files), "non-directory files") - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories + if '__pycache__' in dirs: + dirs.remove('__pycache__') # don't visit __pycache__ directories """ sys.audit("os.walk", top, topdown, onerror, followlinks) - return _walk(fspath(top), topdown, onerror, followlinks) - -def _walk(top, topdown, onerror, followlinks): - dirs = [] - nondirs = [] - walk_dirs = [] - - # We may not have read permission for top, in which case we can't - # get a list of the files the directory contains. os.walk - # always suppressed the exception then, rather than blow up for a - # minor reason when (say) a thousand readable directories are still - # left to visit. That logic is copied here. - try: - # Note that scandir is global in this module due - # to earlier import-*. - scandir_it = scandir(top) - except OSError as error: - if onerror is not None: - onerror(error) - return - with scandir_it: - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - return - - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider that the entry is not - # a directory, same behaviour than os.path.isdir(). - is_dir = False + stack = [fspath(top)] + islink, join = path.islink, path.join + while stack: + top = stack.pop() + if isinstance(top, tuple): + yield top + continue - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) + dirs = [] + nondirs = [] + walk_dirs = [] - if not topdown and is_dir: - # Bottom-up: recurse into sub-directory, but exclude symlinks to - # directories if followlinks is False - if followlinks: - walk_into = True - else: + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. + # We suppress the exception here, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. + try: + with scandir(top) as entries: + for entry in entries: try: - is_symlink = entry.is_symlink() + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() except OSError: - # If is_symlink() raises an OSError, consider that the - # entry is not a symbolic link, same behaviour than - # os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - walk_dirs.append(entry.path) - - # Yield before recursion if going top down - if topdown: - yield top, dirs, nondirs - - # Recurse into sub-directories - islink, join = path.islink, path.join - for dirname in dirs: - new_path = join(top, dirname) - # Issue #23605: os.path.islink() is used instead of caching - # entry.is_symlink() result during the loop on os.scandir() because - # the caller can replace the directory entry during the "yield" - # above. - if followlinks or not islink(new_path): - yield from _walk(new_path, topdown, onerror, followlinks) - else: - # Recurse into sub-directories - for new_path in walk_dirs: - yield from _walk(new_path, topdown, onerror, followlinks) - # Yield after recursion if going bottom up - yield top, dirs, nondirs + # If is_dir() raises an OSError, consider the entry not to + # be a directory, same behaviour as os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: traverse into sub-directory, but exclude + # symlinks to directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider the + # entry not to be a symbolic link, same behaviour + # as os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + walk_dirs.append(entry.path) + except OSError as error: + if onerror is not None: + onerror(error) + continue + + if topdown: + # Yield before sub-directory traversal if going top down + yield top, dirs, nondirs + # Traverse into sub-directories + for dirname in reversed(dirs): + new_path = join(top, dirname) + # bpo-23605: os.path.islink() is used instead of caching + # entry.is_symlink() result during the loop on os.scandir() because + # the caller can replace the directory entry during the "yield" + # above. + if followlinks or not islink(new_path): + stack.append(new_path) + else: + # Yield after sub-directory traversal if going bottom up + stack.append((top, dirs, nondirs)) + # Traverse into sub-directories + for new_path in reversed(walk_dirs): + stack.append(new_path) __all__.append("walk") @@ -452,35 +461,69 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= Example: import os - for root, dirs, files, rootfd in os.fwalk('python/Lib/email'): + for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'): print(root, "consumes", end="") print(sum(os.stat(name, dir_fd=rootfd).st_size for name in files), end="") print("bytes in", len(files), "non-directory files") - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories + if '__pycache__' in dirs: + dirs.remove('__pycache__') # don't visit __pycache__ directories """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) - if not isinstance(top, int) or not hasattr(top, '__index__'): - top = fspath(top) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - if not follow_symlinks: - orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY, dir_fd=dir_fd) + top = fspath(top) + stack = [(_fwalk_walk, (True, dir_fd, top, top, None))] + isbytes = isinstance(top, bytes) try: - if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and - path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, isinstance(top, bytes), - topdown, onerror, follow_symlinks) + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) finally: - close(topfd) - - def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): + # Close any file descriptors still on the stack. + while stack: + action, value = stack.pop() + if action == _fwalk_close: + close(value) + + # Each item in the _fwalk() stack is a pair (action, args). + _fwalk_walk = 0 # args: (isroot, dirfd, toppath, topname, entry) + _fwalk_yield = 1 # args: (toppath, dirnames, filenames, topfd) + _fwalk_close = 2 # args: dirfd + + def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. + action, value = stack.pop() + if action == _fwalk_close: + close(value) + return + elif action == _fwalk_yield: + yield value + return + assert action == _fwalk_walk + isroot, dirfd, toppath, topname, entry = value + try: + if not follow_symlinks: + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + if entry is None: + orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + else: + orig_st = entry.stat(follow_symlinks=False) + topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + except OSError as err: + if isroot: + raise + if onerror is not None: + onerror(err) + return + stack.append((_fwalk_close, topfd)) + if not follow_symlinks: + if isroot and not st.S_ISDIR(orig_st.st_mode): + return + if not path.samestat(orig_st, stat(topfd)): + return + scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -506,31 +549,18 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd + else: + stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) - for name in dirs if entries is None else zip(dirs, entries): - try: - if not follow_symlinks: - if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) - else: - assert entries is not None - name, entry = name - orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY, dir_fd=topfd) - except OSError as err: - if onerror is not None: - onerror(err) - continue - try: - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, isbytes, - topdown, onerror, follow_symlinks) - finally: - close(dirfd) - - if not topdown: - yield toppath, dirs, nondirs, topfd + toppath = path.join(toppath, toppath[:0]) # Add trailing slash. + if entries is None: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, None)) + for name in dirs[::-1]) + else: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, entry)) + for name, entry in zip(dirs[::-1], entries[::-1])) __all__.append("fwalk") @@ -704,9 +734,11 @@ def __len__(self): return len(self._data) def __repr__(self): - return 'environ({{{}}})'.format(', '.join( - ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value)) - for key, value in self._data.items()))) + formatted_items = ", ".join( + f"{self.decodekey(key)!r}: {self.decodevalue(value)!r}" + for key, value in self._data.items() + ) + return f"environ({{{formatted_items}}})" def copy(self): return dict(self) @@ -734,7 +766,7 @@ def __ror__(self, other): new.update(self) return new -def _createenviron(): +def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): @@ -764,9 +796,24 @@ def decode(value): encode, decode) # unicode environ -environ = _createenviron() -del _createenviron +environ = _create_environ_mapping() +del _create_environ_mapping + + +if _exists("_create_environ"): + def reload_environ(): + data = _create_environ() + if name == 'nt': + encodekey = environ.encodekey + data = {encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + env_data = environ._data + env_data.clear() + env_data.update(data) + __all__.append("reload_environ") def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. @@ -980,7 +1027,7 @@ def popen(cmd, mode="r", buffering=-1): raise ValueError("invalid mode %r" % mode) if buffering == 0 or buffering is None: raise ValueError("popen() does not support unbuffered streams") - import subprocess, io + import subprocess if mode == "r": proc = subprocess.Popen(cmd, shell=True, text=True, @@ -1053,6 +1100,12 @@ def _fspath(path): else: raise TypeError("expected str, bytes or os.PathLike object, " "not " + path_type.__name__) + except TypeError: + if path_type.__fspath__ is None: + raise TypeError("expected str, bytes or os.PathLike object, " + "not " + path_type.__name__) from None + else: + raise if isinstance(path_repr, (str, bytes)): return path_repr else: @@ -1071,6 +1124,8 @@ class PathLike(abc.ABC): """Abstract base class for implementing the file system path protocol.""" + __slots__ = () + @abc.abstractmethod def __fspath__(self): """Return the file system path representation of the object.""" @@ -1120,3 +1175,17 @@ def add_dll_directory(path): cookie, nt._remove_dll_directory ) + + +if _exists('sched_getaffinity') and sys._get_cpu_count_config() < 0: + def process_cpu_count(): + """ + Get the number of CPUs of the current process. + + Return the number of logical CPUs usable by the calling thread of the + current process. Return None if indeterminable. + """ + return len(sched_getaffinity(0)) +else: + # Just an alias to cpu_count() (same docstring) + process_cpu_count = cpu_count diff --git a/Lib/pathlib.py b/Lib/pathlib.py deleted file mode 100644 index f4aab1c0ce5..00000000000 --- a/Lib/pathlib.py +++ /dev/null @@ -1,1464 +0,0 @@ -import fnmatch -import functools -import io -import ntpath -import os -import posixpath -import re -import sys -import warnings -from _collections_abc import Sequence -from errno import EINVAL, ENOENT, ENOTDIR, EBADF, ELOOP -from operator import attrgetter -from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -from urllib.parse import quote_from_bytes as urlquote_from_bytes - - -__all__ = [ - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - -# -# Internals -# - -_WINERROR_NOT_READY = 21 # drive exists but is not accessible -_WINERROR_INVALID_NAME = 123 # fix for bpo-35306 -_WINERROR_CANT_RESOLVE_FILENAME = 1921 # broken symlink pointing to itself - -# EBADF - guard against macOS `stat` throwing EBADF -_IGNORED_ERROS = (ENOENT, ENOTDIR, EBADF, ELOOP) - -_IGNORED_WINERRORS = ( - _WINERROR_NOT_READY, - _WINERROR_INVALID_NAME, - _WINERROR_CANT_RESOLVE_FILENAME) - -def _ignore_error(exception): - # XXX RUSTPYTHON: added check for FileNotFoundError, file.exists() on windows throws it - # but with a errno==ESRCH for some reason - return (isinstance(exception, FileNotFoundError) or - getattr(exception, 'errno', None) in _IGNORED_ERROS or - getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) - - -def _is_wildcard_pattern(pat): - # Whether this pattern needs actual matching using fnmatch, or can - # be looked up directly as a file. - return "*" in pat or "?" in pat or "[" in pat - - -class _Flavour(object): - """A flavour implements a particular (platform-specific) set of path - semantics.""" - - def __init__(self): - self.join = self.sep.join - - def parse_parts(self, parts): - parsed = [] - sep = self.sep - altsep = self.altsep - drv = root = '' - it = reversed(parts) - for part in it: - if not part: - continue - if altsep: - part = part.replace(altsep, sep) - drv, root, rel = self.splitroot(part) - if sep in rel: - for x in reversed(rel.split(sep)): - if x and x != '.': - parsed.append(sys.intern(x)) - else: - if rel and rel != '.': - parsed.append(sys.intern(rel)) - if drv or root: - if not drv: - # If no drive is present, try to find one in the previous - # parts. This makes the result of parsing e.g. - # ("C:", "/", "a") reasonably intuitive. - for part in it: - if not part: - continue - if altsep: - part = part.replace(altsep, sep) - drv = self.splitroot(part)[0] - if drv: - break - break - if drv or root: - parsed.append(drv + root) - parsed.reverse() - return drv, root, parsed - - def join_parsed_parts(self, drv, root, parts, drv2, root2, parts2): - """ - Join the two paths represented by the respective - (drive, root, parts) tuples. Return a new (drive, root, parts) tuple. - """ - if root2: - if not drv2 and drv: - return drv, root2, [drv + root2] + parts2[1:] - elif drv2: - if drv2 == drv or self.casefold(drv2) == self.casefold(drv): - # Same drive => second path is relative to the first - return drv, root, parts + parts2[1:] - else: - # Second path is non-anchored (common case) - return drv, root, parts + parts2 - return drv2, root2, parts2 - - -class _WindowsFlavour(_Flavour): - # Reference for Windows paths can be found at - # http://msdn.microsoft.com/en-us/library/aa365247%28v=vs.85%29.aspx - - sep = '\\' - altsep = '/' - has_drv = True - pathmod = ntpath - - is_supported = (os.name == 'nt') - - drive_letters = set('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ') - ext_namespace_prefix = '\\\\?\\' - - reserved_names = ( - {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | - {'COM%s' % c for c in '123456789\xb9\xb2\xb3'} | - {'LPT%s' % c for c in '123456789\xb9\xb2\xb3'} - ) - - # Interesting findings about extended paths: - # * '\\?\c:\a' is an extended path, which bypasses normal Windows API - # path processing. Thus relative paths are not resolved and slash is not - # translated to backslash. It has the native NT path limit of 32767 - # characters, but a bit less after resolving device symbolic links, - # such as '\??\C:' => '\Device\HarddiskVolume2'. - # * '\\?\c:/a' looks for a device named 'C:/a' because slash is a - # regular name character in the object namespace. - # * '\\?\c:\foo/bar' is invalid because '/' is illegal in NT filesystems. - # The only path separator at the filesystem level is backslash. - # * '//?/c:\a' and '//?/c:/a' are effectively equivalent to '\\.\c:\a' and - # thus limited to MAX_PATH. - # * Prior to Windows 8, ANSI API bytes paths are limited to MAX_PATH, - # even with the '\\?\' prefix. - - def splitroot(self, part, sep=sep): - first = part[0:1] - second = part[1:2] - if (second == sep and first == sep): - # XXX extended paths should also disable the collapsing of "." - # components (according to MSDN docs). - prefix, part = self._split_extended_path(part) - first = part[0:1] - second = part[1:2] - else: - prefix = '' - third = part[2:3] - if (second == sep and first == sep and third != sep): - # is a UNC path: - # vvvvvvvvvvvvvvvvvvvvv root - # \\machine\mountpoint\directory\etc\... - # directory ^^^^^^^^^^^^^^ - index = part.find(sep, 2) - if index != -1: - index2 = part.find(sep, index + 1) - # a UNC path can't have two slashes in a row - # (after the initial two) - if index2 != index + 1: - if index2 == -1: - index2 = len(part) - if prefix: - return prefix + part[1:index2], sep, part[index2+1:] - else: - return part[:index2], sep, part[index2+1:] - drv = root = '' - if second == ':' and first in self.drive_letters: - drv = part[:2] - part = part[2:] - first = third - if first == sep: - root = first - part = part.lstrip(sep) - return prefix + drv, root, part - - def casefold(self, s): - return s.lower() - - def casefold_parts(self, parts): - return [p.lower() for p in parts] - - def compile_pattern(self, pattern): - return re.compile(fnmatch.translate(pattern), re.IGNORECASE).fullmatch - - def _split_extended_path(self, s, ext_prefix=ext_namespace_prefix): - prefix = '' - if s.startswith(ext_prefix): - prefix = s[:4] - s = s[4:] - if s.startswith('UNC\\'): - prefix += s[:3] - s = '\\' + s[3:] - return prefix, s - - def is_reserved(self, parts): - # NOTE: the rules for reserved names seem somewhat complicated - # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not - # exist). We err on the side of caution and return True for paths - # which are not considered reserved by Windows. - if not parts: - return False - if parts[0].startswith('\\\\'): - # UNC paths are never reserved - return False - name = parts[-1].partition('.')[0].partition(':')[0].rstrip(' ') - return name.upper() in self.reserved_names - - def make_uri(self, path): - # Under Windows, file URIs use the UTF-8 encoding. - drive = path.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - rest = path.as_posix()[2:].lstrip('/') - return 'file:///%s/%s' % ( - drive, urlquote_from_bytes(rest.encode('utf-8'))) - else: - # It's a path on a network drive => 'file://host/share/a/b' - return 'file:' + urlquote_from_bytes(path.as_posix().encode('utf-8')) - - -class _PosixFlavour(_Flavour): - sep = '/' - altsep = '' - has_drv = False - pathmod = posixpath - - is_supported = (os.name != 'nt') - - def splitroot(self, part, sep=sep): - if part and part[0] == sep: - stripped_part = part.lstrip(sep) - # According to POSIX path resolution: - # http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap04.html#tag_04_11 - # "A pathname that begins with two successive slashes may be - # interpreted in an implementation-defined manner, although more - # than two leading slashes shall be treated as a single slash". - if len(part) - len(stripped_part) == 2: - return '', sep * 2, stripped_part - else: - return '', sep, stripped_part - else: - return '', '', part - - def casefold(self, s): - return s - - def casefold_parts(self, parts): - return parts - - def compile_pattern(self, pattern): - return re.compile(fnmatch.translate(pattern)).fullmatch - - def is_reserved(self, parts): - return False - - def make_uri(self, path): - # We represent the path using the local filesystem encoding, - # for portability to other applications. - bpath = bytes(path) - return 'file://' + urlquote_from_bytes(bpath) - - -_windows_flavour = _WindowsFlavour() -_posix_flavour = _PosixFlavour() - - -class _Accessor: - """An accessor implements a particular (system-specific or not) way of - accessing paths on the filesystem.""" - - -class _NormalAccessor(_Accessor): - - stat = os.stat - - open = io.open - - listdir = os.listdir - - scandir = os.scandir - - chmod = os.chmod - - mkdir = os.mkdir - - unlink = os.unlink - - if hasattr(os, "link"): - link = os.link - else: - def link(self, src, dst): - raise NotImplementedError("os.link() not available on this system") - - rmdir = os.rmdir - - rename = os.rename - - replace = os.replace - - if hasattr(os, "symlink"): - symlink = os.symlink - else: - def symlink(self, src, dst, target_is_directory=False): - raise NotImplementedError("os.symlink() not available on this system") - - def touch(self, path, mode=0o666, exist_ok=True): - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(path, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(path, flags, mode) - os.close(fd) - - if hasattr(os, "readlink"): - readlink = os.readlink - else: - def readlink(self, path): - raise NotImplementedError("os.readlink() not available on this system") - - def owner(self, path): - try: - import pwd - return pwd.getpwuid(self.stat(path).st_uid).pw_name - except ImportError: - raise NotImplementedError("Path.owner() is unsupported on this system") - - def group(self, path): - try: - import grp - return grp.getgrgid(self.stat(path).st_gid).gr_name - except ImportError: - raise NotImplementedError("Path.group() is unsupported on this system") - - getcwd = os.getcwd - - expanduser = staticmethod(os.path.expanduser) - - realpath = staticmethod(os.path.realpath) - - -_normal_accessor = _NormalAccessor() - - -# -# Globbing helpers -# - -def _make_selector(pattern_parts, flavour): - pat = pattern_parts[0] - child_parts = pattern_parts[1:] - if pat == '**': - cls = _RecursiveWildcardSelector - elif '**' in pat: - raise ValueError("Invalid pattern: '**' can only be an entire path component") - elif _is_wildcard_pattern(pat): - cls = _WildcardSelector - else: - cls = _PreciseSelector - return cls(pat, child_parts, flavour) - -if hasattr(functools, "lru_cache"): - _make_selector = functools.lru_cache()(_make_selector) - - -class _Selector: - """A selector matches a specific glob pattern part against the children - of a given path.""" - - def __init__(self, child_parts, flavour): - self.child_parts = child_parts - if child_parts: - self.successor = _make_selector(child_parts, flavour) - self.dironly = True - else: - self.successor = _TerminatingSelector() - self.dironly = False - - def select_from(self, parent_path): - """Iterate over all child paths of `parent_path` matched by this - selector. This can contain parent_path itself.""" - path_cls = type(parent_path) - is_dir = path_cls.is_dir - exists = path_cls.exists - scandir = parent_path._accessor.scandir - if not is_dir(parent_path): - return iter([]) - return self._select_from(parent_path, is_dir, exists, scandir) - - -class _TerminatingSelector: - - def _select_from(self, parent_path, is_dir, exists, scandir): - yield parent_path - - -class _PreciseSelector(_Selector): - - def __init__(self, name, child_parts, flavour): - self.name = name - _Selector.__init__(self, child_parts, flavour) - - def _select_from(self, parent_path, is_dir, exists, scandir): - try: - path = parent_path._make_child_relpath(self.name) - if (is_dir if self.dironly else exists)(path): - for p in self.successor._select_from(path, is_dir, exists, scandir): - yield p - except PermissionError: - return - - -class _WildcardSelector(_Selector): - - def __init__(self, pat, child_parts, flavour): - self.match = flavour.compile_pattern(pat) - _Selector.__init__(self, child_parts, flavour) - - def _select_from(self, parent_path, is_dir, exists, scandir): - try: - with scandir(parent_path) as scandir_it: - entries = list(scandir_it) - for entry in entries: - if self.dironly: - try: - # "entry.is_dir()" can raise PermissionError - # in some cases (see bpo-38894), which is not - # among the errors ignored by _ignore_error() - if not entry.is_dir(): - continue - except OSError as e: - if not _ignore_error(e): - raise - continue - name = entry.name - if self.match(name): - path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir): - yield p - except PermissionError: - return - - -class _RecursiveWildcardSelector(_Selector): - - def __init__(self, pat, child_parts, flavour): - _Selector.__init__(self, child_parts, flavour) - - def _iterate_directories(self, parent_path, is_dir, scandir): - yield parent_path - try: - with scandir(parent_path) as scandir_it: - entries = list(scandir_it) - for entry in entries: - entry_is_dir = False - try: - entry_is_dir = entry.is_dir() - except OSError as e: - if not _ignore_error(e): - raise - if entry_is_dir and not entry.is_symlink(): - path = parent_path._make_child_relpath(entry.name) - for p in self._iterate_directories(path, is_dir, scandir): - yield p - except PermissionError: - return - - def _select_from(self, parent_path, is_dir, exists, scandir): - try: - yielded = set() - try: - successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, is_dir, scandir): - for p in successor_select(starting_point, is_dir, exists, scandir): - if p not in yielded: - yield p - yielded.add(p) - finally: - yielded.clear() - except PermissionError: - return - - -# -# Public API -# - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_pathcls', '_drv', '_root', '_parts') - - def __init__(self, path): - # We don't store the instance to avoid reference cycles - self._pathcls = type(path) - self._drv = path._drv - self._root = path._root - self._parts = path._parts - - def __len__(self): - if self._drv or self._root: - return len(self._parts) - 1 - else: - return len(self._parts) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._pathcls._from_parsed_parts(self._drv, self._root, - self._parts[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(self._pathcls.__name__) - - -class PurePath(object): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - __slots__ = ( - '_drv', '_root', '_parts', - '_str', '_hash', '_pparts', '_cached_cparts', - ) - - def __new__(cls, *args): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return cls._from_parts(args) - - def __reduce__(self): - # Using the parts tuple helps share interned path parts - # when pickling related paths. - return (self.__class__, tuple(self._parts)) - - @classmethod - def _parse_args(cls, args): - # This is useful when you don't want to create an instance, just - # canonicalize some constructor arguments. - parts = [] - for a in args: - if isinstance(a, PurePath): - parts += a._parts - else: - a = os.fspath(a) - if isinstance(a, str): - # Force-cast str subclasses to str (issue #21127) - parts.append(str(a)) - else: - raise TypeError( - "argument should be a str object or an os.PathLike " - "object returning str, not %r" - % type(a)) - return cls._flavour.parse_parts(parts) - - @classmethod - def _from_parts(cls, args): - # We need to call _parse_args on the instance, so as to get the - # right flavour. - self = object.__new__(cls) - drv, root, parts = self._parse_args(args) - self._drv = drv - self._root = root - self._parts = parts - return self - - @classmethod - def _from_parsed_parts(cls, drv, root, parts): - self = object.__new__(cls) - self._drv = drv - self._root = root - self._parts = parts - return self - - @classmethod - def _format_parsed_parts(cls, drv, root, parts): - if drv or root: - return drv + root + cls._flavour.join(parts[1:]) - else: - return cls._flavour.join(parts) - - def _make_child(self, args): - drv, root, parts = self._parse_args(args) - drv, root, parts = self._flavour.join_parsed_parts( - self._drv, self._root, self._parts, drv, root, parts) - return self._from_parsed_parts(drv, root, parts) - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self._drv, self._root, - self._parts) or '.' - return self._str - - def __fspath__(self): - return str(self) - - def as_posix(self): - """Return the string representation of the path with forward (/) - slashes.""" - f = self._flavour - return str(self).replace(f.sep, '/') - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def as_uri(self): - """Return the path as a 'file' URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - return self._flavour.make_uri(self) - - @property - def _cparts(self): - # Cached casefolded parts, for hashing and comparison - try: - return self._cached_cparts - except AttributeError: - self._cached_cparts = self._flavour.casefold_parts(self._parts) - return self._cached_cparts - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._cparts == other._cparts and self._flavour is other._flavour - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(tuple(self._cparts)) - return self._hash - - def __lt__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._cparts < other._cparts - - def __le__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._cparts <= other._cparts - - def __gt__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._cparts > other._cparts - - def __ge__(self, other): - if not isinstance(other, PurePath) or self._flavour is not other._flavour: - return NotImplemented - return self._cparts >= other._cparts - - def __class_getitem__(cls, type): - return cls - - drive = property(attrgetter('_drv'), - doc="""The drive prefix (letter or UNC path), if any.""") - - root = property(attrgetter('_root'), - doc="""The root of the path, if any.""") - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - anchor = self._drv + self._root - return anchor - - @property - def name(self): - """The final path component, if any.""" - parts = self._parts - if len(parts) == (1 if (self._drv or self._root) else 0): - return '' - return parts[-1] - - @property - def suffix(self): - """ - The final component's last suffix, if any. - - This includes the leading period. For example: '.txt' - """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' - - @property - def suffixes(self): - """ - A list of the final component's suffixes, if any. - - These include the leading periods. For example: ['.tar', '.gz'] - """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] - - @property - def stem(self): - """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name - - def with_name(self, name): - """Return a new path with the file name changed.""" - if not self.name: - raise ValueError("%r has an empty name" % (self,)) - drv, root, parts = self._flavour.parse_parts((name,)) - if (not name or name[-1] in [self._flavour.sep, self._flavour.altsep] - or drv or root or len(parts) != 1): - raise ValueError("Invalid name %r" % (name)) - return self._from_parsed_parts(self._drv, self._root, - self._parts[:-1] + [name]) - - def with_stem(self, stem): - """Return a new path with the stem changed.""" - return self.with_name(stem + self.suffix) - - def with_suffix(self, suffix): - """Return a new path with the file suffix changed. If the path - has no suffix, add given suffix. If the given suffix is an empty - string, remove the suffix from the path. - """ - f = self._flavour - if f.sep in suffix or f.altsep and f.altsep in suffix: - raise ValueError("Invalid suffix %r" % (suffix,)) - if suffix and not suffix.startswith('.') or suffix == '.': - raise ValueError("Invalid suffix %r" % (suffix)) - name = self.name - if not name: - raise ValueError("%r has an empty name" % (self,)) - old_suffix = self.suffix - if not old_suffix: - name = name + suffix - else: - name = name[:-len(old_suffix)] + suffix - return self._from_parsed_parts(self._drv, self._root, - self._parts[:-1] + [name]) - - def relative_to(self, *other): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - a subpath of the other path), raise ValueError. - """ - # For the purpose of this method, drive and root are considered - # separate parts, i.e.: - # Path('c:/').relative_to('c:') gives Path('/') - # Path('c:/').relative_to('/') raise ValueError - if not other: - raise TypeError("need at least one argument") - parts = self._parts - drv = self._drv - root = self._root - if root: - abs_parts = [drv, root] + parts[1:] - else: - abs_parts = parts - to_drv, to_root, to_parts = self._parse_args(other) - if to_root: - to_abs_parts = [to_drv, to_root] + to_parts[1:] - else: - to_abs_parts = to_parts - n = len(to_abs_parts) - cf = self._flavour.casefold_parts - if (root or drv) if n == 0 else cf(abs_parts[:n]) != cf(to_abs_parts): - formatted = self._format_parsed_parts(to_drv, to_root, to_parts) - raise ValueError("{!r} is not in the subpath of {!r}" - " OR one path is relative and the other is absolute." - .format(str(self), str(formatted))) - return self._from_parsed_parts('', root if n == 1 else '', - abs_parts[n:]) - - def is_relative_to(self, *other): - """Return True if the path is relative to another path or False. - """ - try: - self.relative_to(*other) - return True - except ValueError: - return False - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - # We cache the tuple to avoid building a new one each time .parts - # is accessed. XXX is this necessary? - try: - return self._pparts - except AttributeError: - self._pparts = tuple(self._parts) - return self._pparts - - def joinpath(self, *args): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self._make_child(args) - - def __truediv__(self, key): - try: - return self._make_child((key,)) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self._from_parts([key] + self._parts) - except TypeError: - return NotImplemented - - @property - def parent(self): - """The logical parent of the path.""" - drv = self._drv - root = self._root - parts = self._parts - if len(parts) == 1 and (drv or root): - return self - return self._from_parsed_parts(drv, root, parts[:-1]) - - @property - def parents(self): - """A sequence of this path's logical parents.""" - return _PathParents(self) - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - if not self._root: - return False - return not self._flavour.has_drv or bool(self._drv) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - return self._flavour.is_reserved(self._parts) - - def match(self, path_pattern): - """ - Return True if this path matches the given pattern. - """ - cf = self._flavour.casefold - path_pattern = cf(path_pattern) - drv, root, pat_parts = self._flavour.parse_parts((path_pattern,)) - if not pat_parts: - raise ValueError("empty pattern") - if drv and drv != cf(self._drv): - return False - if root and root != cf(self._root): - return False - parts = self._cparts - if drv or root: - if len(pat_parts) != len(parts): - return False - pat_parts = pat_parts[1:] - elif len(pat_parts) > len(parts): - return False - for part, pat in zip(reversed(parts), reversed(pat_parts)): - if not fnmatch.fnmatchcase(part, pat): - return False - return True - -# Can't subclass os.PathLike from PurePath and keep the constructor -# optimizations in PurePath._parse_args(). -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - _flavour = _posix_flavour - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - _flavour = _windows_flavour - __slots__ = () - - -# Filesystem-accessing classes - - -class Path(PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - _accessor = _normal_accessor - __slots__ = () - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - self = cls._from_parts(args) - if not self._flavour.is_supported: - raise NotImplementedError("cannot instantiate %r on your system" - % (cls.__name__,)) - return self - - def _make_child_relpath(self, part): - # This is an optimization used for dir walking. `part` must be - # a single part relative to this path. - parts = self._parts + [part] - return self._from_parsed_parts(self._drv, self._root, parts) - - def __enter__(self): - return self - - def __exit__(self, t, v, tb): - # https://bugs.python.org/issue39682 - # In previous versions of pathlib, this method marked this path as - # closed; subsequent attempts to perform I/O would raise an IOError. - # This functionality was never documented, and had the effect of - # making Path objects mutable, contrary to PEP 428. In Python 3.9 the - # _closed attribute was removed, and this method made a no-op. - # This method and __enter__()/__exit__() should be deprecated and - # removed in the future. - pass - - # Public API - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory - (as returned by os.getcwd()). - """ - return cls(cls._accessor.getcwd()) - - @classmethod - def home(cls): - """Return a new path pointing to the user's home directory (as - returned by os.path.expanduser('~')). - """ - return cls("~").expanduser() - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). - """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self._accessor.stat(other_path) - return os.path.samestat(st, other_st) - - def iterdir(self): - """Iterate over the files in this directory. Does not yield any - result for the special paths '.' and '..'. - """ - for name in self._accessor.listdir(self): - if name in {'.', '..'}: - # Yielding a path object for these makes little sense - continue - yield self._make_child_relpath(name) - - def glob(self, pattern): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - if not pattern: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - selector = _make_selector(tuple(pattern_parts), self._flavour) - for p in selector.select_from(self): - yield p - - def rglob(self, pattern): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) - if drv or root: - raise NotImplementedError("Non-relative patterns are unsupported") - selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour) - for p in selector.select_from(self): - yield p - - def absolute(self): - """Return an absolute version of this path. This function works - even if the path doesn't point to anything. - - No normalization is done, i.e. all '.' and '..' will be kept along. - Use resolve() to get the canonical path to a file. - """ - # XXX untested yet! - if self.is_absolute(): - return self - # FIXME this must defer to the specific flavour (and, under Windows, - # use nt._getfullpathname()) - return self._from_parts([self._accessor.getcwd()] + self._parts) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it (for example turning slashes into backslashes under - Windows). - """ - - def check_eloop(e): - winerror = getattr(e, 'winerror', 0) - if e.errno == ELOOP or winerror == _WINERROR_CANT_RESOLVE_FILENAME: - raise RuntimeError("Symlink loop from %r" % e.filename) - - try: - s = self._accessor.realpath(self, strict=strict) - except OSError as e: - check_eloop(e) - raise - p = self._from_parts((s,)) - - # In non-strict mode, realpath() doesn't raise on symlink loops. - # Ensure we get an exception by calling stat() - if not strict: - try: - p.stat() - except OSError as e: - check_eloop(e) - return p - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return self._accessor.stat(self, follow_symlinks=follow_symlinks) - - def owner(self): - """ - Return the login name of the file owner. - """ - return self._accessor.owner(self) - - def group(self): - """ - Return the group name of the file gid. - """ - return self._accessor.group(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return self._accessor.open(self, mode, buffering, encoding, errors, - newline) - - def read_bytes(self): - """ - Open the file in bytes mode, read it, and close the file. - """ - with self.open(mode='rb') as f: - return f.read() - - def read_text(self, encoding=None, errors=None): - """ - Open the file in text mode, read it, and close the file. - """ - encoding = io.text_encoding(encoding) - with self.open(mode='r', encoding=encoding, errors=errors) as f: - return f.read() - - def write_bytes(self, data): - """ - Open the file in bytes mode, write to it, and close the file. - """ - # type-check for the buffer interface before truncating the file - view = memoryview(data) - with self.open(mode='wb') as f: - return f.write(view) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - if not isinstance(data, str): - raise TypeError('data must be str, not %s' % - data.__class__.__name__) - encoding = io.text_encoding(encoding) - with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: - return f.write(data) - - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - path = self._accessor.readlink(self) - return self._from_parts((path,)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - self._accessor.touch(self, mode, exist_ok) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - self._accessor.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - self._accessor.chmod(self, mode, follow_symlinks=follow_symlinks) - - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - self._accessor.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - self._accessor.rmdir(self) - - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - self._accessor.rename(self, target) - return self.__class__(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - self._accessor.replace(self, target) - return self.__class__(target) - - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - self._accessor.symlink(target, self, target_is_directory) - - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - self._accessor.link(target, self) - - def link_to(self, target): - """ - Make the target path a hard link pointing to this path. - - Note this function does not make this path a hard link to *target*, - despite the implication of the function and argument names. The order - of arguments (target, link) is the reverse of Path.symlink_to, but - matches that of os.link. - - Deprecated since Python 3.10 and scheduled for removal in Python 3.12. - Use `hardlink_to()` instead. - """ - warnings.warn("pathlib.Path.link_to() is deprecated and is scheduled " - "for removal in Python 3.12. " - "Use pathlib.Path.hardlink_to() instead.", - DeprecationWarning, stacklevel=2) - self._accessor.link(self, target) - - # Convenience functions for querying the stat results - - def exists(self): - """ - Whether this path exists. - """ - try: - self.stat() - except OSError as e: - if not _ignore_error(e): - raise - return False - except ValueError: - # Non-encodable path - return False - return True - - def is_dir(self): - """ - Whether this path is a directory. - """ - try: - return S_ISDIR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_file(self): - """ - Whether this path is a regular file (also True for symlinks pointing - to regular files). - """ - try: - return S_ISREG(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_mount(self): - """ - Check if this path is a POSIX mount point - """ - # Need to exist and be a dir - if not self.exists() or not self.is_dir(): - return False - - try: - parent_dev = self.parent.stat().st_dev - except OSError: - return False - - dev = self.stat().st_dev - if dev != parent_dev: - return True - ino = self.stat().st_ino - parent_ino = self.parent.stat().st_ino - return ino == parent_ino - - def is_symlink(self): - """ - Whether this path is a symbolic link. - """ - try: - return S_ISLNK(self.lstat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist - return False - except ValueError: - # Non-encodable path - return False - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except OSError as e: - if not _ignore_error(e): - raise - # Path doesn't exist or is a broken symlink - # (see http://web.archive.org/web/20200623061726/https://bitbucket.org/pitrou/pathlib/issues/12/ ) - return False - except ValueError: - # Non-encodable path - return False - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self._drv or self._root) and - self._parts and self._parts[0][:1] == '~'): - homedir = self._accessor.expanduser(self._parts[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - return self._from_parts([homedir] + self._parts[1:]) - - return self - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - def is_mount(self): - raise NotImplementedError("Path.is_mount() is unsupported on this system") diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py new file mode 100644 index 00000000000..0d763d1f0dc --- /dev/null +++ b/Lib/pathlib/__init__.py @@ -0,0 +1,1307 @@ +"""Object-oriented filesystem paths. + +This module provides classes to represent abstract paths and concrete +paths with operations that have semantics appropriate for different +operating systems. +""" + +import io +import ntpath +import operator +import os +import posixpath +import sys +from errno import * +from glob import _StringGlobber, _no_recurse_symlinks +from itertools import chain +from stat import S_ISDIR, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from _collections_abc import Sequence + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from pathlib._os import ( + PathInfo, DirEntryInfo, + ensure_different_files, ensure_distinct_paths, + copyfile2, copyfileobj, magic_open, copy_info, +) + + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is attempted. + """ + pass + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePath: + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unjoined string paths. This is set in + # the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is not self.parser: + # GH-103631: Convert separators for backwards compatibility. + paths.append(arg.as_posix()) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + self._raw_paths = paths + + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + return type(self)(*pathsegments) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + return drv, root, [x for x in rel.split(sep) if x and x != '.'] + + @classmethod + def _parse_pattern(cls, pattern): + """Parse a glob pattern to a list of parts. This is much like + _parse_path, except: + + - Rather than normalizing and returning the drive and root, we raise + NotImplementedError if either are present. + - If the path has no real parts, we raise ValueError. + - If the path ends in a slash, then a final empty part is added. + """ + drv, root, rel = cls.parser.splitroot(pattern) + if root or drv: + raise NotImplementedError("Non-relative patterns are unsupported") + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + rel = rel.replace(altsep, sep) + parts = [x for x in rel.split(sep) if x and x != '.'] + if not parts: + raise ValueError(f"Unacceptable pattern: {str(pattern)!r}") + elif rel.endswith(sep): + # GH-65238: preserve trailing slash in glob patterns. + parts.append('') + return parts + + def as_posix(self): + """Return the string representation of the path with forward (/) + slashes.""" + return str(self).replace(self.parser.sep, '/') + + @property + def _raw_path(self): + paths = self._raw_paths + if len(paths) == 1: + return paths[0] + elif paths: + # Join path segments from the initializer. + return self.parser.join(*paths) + else: + return '' + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + stem = self.stem + if not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") + elif suffix and not suffix.startswith('.'): + raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) + + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if i != -1: + stem = name[:i] + # Stem must contain at least one non-dot character. + if stem.lstrip('.'): + return stem + return name + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name.lstrip('.') + i = name.rfind('.') + if i != -1: + return name[i:] + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + return ['.' + ext for ext in self.name.lstrip('.').split('.')[1:]] + + def relative_to(self, other, *, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if not hasattr(other, 'with_segments'): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other): + """Return True if the path is relative to another path or False. + """ + if not hasattr(other, 'with_segments'): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + import warnings + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings._deprecated("pathlib.PurePath.is_reserved", msg, remove=(3, 15)) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + import warnings + msg = ("pathlib.PurePath.as_uri() is deprecated and scheduled " + "for removal in Python 3.19. Use pathlib.Path.as_uri().") + warnings._deprecated("pathlib.PurePath.as_uri", msg, remove=(3, 19)) + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + def full_match(self, pattern, *, case_sensitive=None): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + if not hasattr(pattern, 'with_segments'): + pattern = self.with_segments(pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path = str(self) if self.parts else '' + pattern = str(pattern) if pattern.parts else '' + globber = _StringGlobber(self.parser.sep, case_sensitive, recursive=True) + return globber.compile(pattern)(path) is not None + + def match(self, path_pattern, *, case_sensitive=None): + """ + Return True if this path matches the given pattern. If the pattern is + relative, matching is done from the right; otherwise, the entire path + is matched. The recursive wildcard '**' is *not* supported by this + method. + """ + if not hasattr(path_pattern, 'with_segments'): + path_pattern = self.with_segments(path_pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + path_parts = self.parts[::-1] + pattern_parts = path_pattern.parts[::-1] + if not pattern_parts: + raise ValueError("empty pattern") + if len(path_parts) < len(pattern_parts): + return False + if len(path_parts) > len(pattern_parts) and path_pattern.anchor: + return False + globber = _StringGlobber(self.parser.sep, case_sensitive) + for path_part, pattern_part in zip(path_parts, pattern_parts): + match = globber.compile(pattern_part) + if match(path_part) is None: + return False + return True + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = ('_info',) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + @property + def info(self): + """ + A PathInfo object that exposes the file type and other file attributes + of this path. + """ + try: + return self._info + except AttributeError: + self._info = PathInfo(self) + return self._info + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return os.lstat(self) + + def exists(self, *, follow_symlinks=True): + """ + Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. + """ + if follow_symlinks: + return os.path.exists(self) + return os.path.lexists(self) + + def is_dir(self, *, follow_symlinks=True): + """ + Whether this path is a directory. + """ + if follow_symlinks: + return os.path.isdir(self) + try: + return S_ISDIR(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_file(self, *, follow_symlinks=True): + """ + Whether this path is a regular file (also True for symlinks pointing + to regular files). + """ + if follow_symlinks: + return os.path.isfile(self) + try: + return S_ISREG(self.stat(follow_symlinks=follow_symlinks).st_mode) + except (OSError, ValueError): + return False + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_symlink(self): + """ + Whether this path is a symbolic link. + """ + return os.path.islink(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with self.open(mode='rb', buffering=0) as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + with self.open(mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with self.open(mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def _from_dir_entry(self, dir_entry, path_str): + path = self.with_segments(path_str) + path._str = path_str + path._info = DirEntryInfo(dir_entry) + return path + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + entries = list(scandir_it) + if root_dir == '.': + return (self._from_dir_entry(e, e.name) for e in entries) + else: + return (self._from_dir_entry(e, e.path) for e in entries) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if case_sensitive is None: + case_sensitive = self.parser is posixpath + case_pedantic = False + else: + # The user has expressed a case sensitivity choice, but we don't + # know the case sensitivity of the underlying filesystem, so we + # must use scandir() for everything, including non-wildcard parts. + case_pedantic = True + parts = self._parse_pattern(pattern) + recursive = True if recurse_symlinks else _no_recurse_symlinks + globber = _StringGlobber(self.parser.sep, case_sensitive, case_pedantic, recursive) + select = globber.selector(parts[::-1]) + root = str(self) + paths = select(self.parser.join(root, '')) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + pattern = self.parser.join('**', pattern) + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + if not follow_symlinks: + follow_symlinks = os._walk_symlinks_as_files + results = os.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + cwd = os.getcwd() + path = cls(cwd) + path._str = cwd # getcwd() returns a normalized path + return path + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + else: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + f = f"{type(self).__name__}.owner()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + else: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + f = f"{type(self).__name__}.group()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + else: + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + f = f"{type(self).__name__}.readlink()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def _delete(self): + """ + Delete this file or directory (including all sub-directories). + """ + if self.is_symlink() or self.is_junction(): + self.unlink() + elif self.is_dir(): + # Lazy import to improve module import time + import shutil + shutil.rmtree(self) + else: + self.unlink() + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + return target + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + return target + + def copy(self, target, **kwargs): + """ + Recursively copy this file or directory tree to the given destination. + """ + if not hasattr(target, 'with_segments'): + target = self.with_segments(target) + ensure_distinct_paths(self, target) + target._copy_from(self, **kwargs) + return target.joinpath() # Empty join to ensure fresh metadata. + + def copy_into(self, target_dir, **kwargs): + """ + Copy this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif hasattr(target_dir, 'with_segments'): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.copy(target, **kwargs) + + def _copy_from(self, source, follow_symlinks=True, preserve_metadata=False): + """ + Recursively copy the given path to this path. + """ + if not follow_symlinks and source.info.is_symlink(): + self._copy_from_symlink(source, preserve_metadata) + elif source.info.is_dir(): + children = source.iterdir() + os.mkdir(self) + for child in children: + self.joinpath(child.name)._copy_from( + child, follow_symlinks, preserve_metadata) + if preserve_metadata: + copy_info(source.info, self) + else: + self._copy_from_file(source, preserve_metadata) + + def _copy_from_file(self, source, preserve_metadata=False): + ensure_different_files(source, self) + with magic_open(source, 'rb') as source_f: + with open(self, 'wb') as target_f: + copyfileobj(source_f, target_f) + if preserve_metadata: + copy_info(source.info, self) + + if copyfile2: + # Use fast OS routine for local file copying where available. + _copy_from_file_fallback = _copy_from_file + def _copy_from_file(self, source, preserve_metadata=False): + try: + source = os.fspath(source) + except TypeError: + pass + else: + copyfile2(source, str(self)) + return + self._copy_from_file_fallback(source, preserve_metadata) + + if os.name == 'nt': + # If a directory-symlink is copied *before* its target, then + # os.symlink() incorrectly creates a file-symlink on Windows. Avoid + # this by passing *target_is_dir* to os.symlink() on Windows. + def _copy_from_symlink(self, source, preserve_metadata=False): + os.symlink(str(source.readlink()), self, source.info.is_dir()) + if preserve_metadata: + copy_info(source.info, self, follow_symlinks=False) + else: + def _copy_from_symlink(self, source, preserve_metadata=False): + os.symlink(str(source.readlink()), self) + if preserve_metadata: + copy_info(source.info, self, follow_symlinks=False) + + def move(self, target): + """ + Recursively move this file or directory tree to the given destination. + """ + # Use os.replace() if the target is os.PathLike and on the same FS. + try: + target = self.with_segments(target) + except TypeError: + pass + else: + ensure_different_files(self, target) + try: + os.replace(self, target) + except OSError as err: + if err.errno != EXDEV: + raise + else: + return target.joinpath() # Empty join to ensure fresh metadata. + # Fall back to copy+delete. + target = self.copy(target, follow_symlinks=False, preserve_metadata=True) + self._delete() + return target + + def move_into(self, target_dir): + """ + Move this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + elif hasattr(target_dir, 'with_segments'): + target = target_dir / name + else: + target = self.with_segments(target_dir, name) + return self.move(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + else: + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + f = f"{type(self).__name__}.symlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + else: + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + f = f"{type(self).__name__}.hardlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + homedir = os.path.expanduser("~") + if homedir == "~": + raise RuntimeError("Could not determine home directory.") + return cls(homedir) + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative paths can't be expressed as file URIs") + from urllib.request import pathname2url + return pathname2url(str(self), add_scheme=True) + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + from urllib.error import URLError + from urllib.request import url2pathname + try: + path = cls(url2pathname(uri, require_scheme=True)) + except URLError as exc: + raise ValueError(exc.reason) from None + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 00000000000..58e137f2a92 --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,12 @@ +""" +This module exists so that pathlib objects pickled under Python 3.13 can be +unpickled in 3.14+. +""" + +from pathlib import * + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", +] diff --git a/Lib/pathlib/_os.py b/Lib/pathlib/_os.py new file mode 100644 index 00000000000..039836941dd --- /dev/null +++ b/Lib/pathlib/_os.py @@ -0,0 +1,530 @@ +""" +Low-level OS functionality wrappers used by pathlib. +""" + +from errno import * +from io import TextIOWrapper, text_encoding +from stat import S_ISDIR, S_ISREG, S_ISLNK, S_IMODE +import os +import sys +try: + import fcntl +except ImportError: + fcntl = None +try: + import posix +except ImportError: + posix = None +try: + import _winapi +except ImportError: + _winapi = None + + +def _get_copy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + + +if fcntl and hasattr(fcntl, 'FICLONE'): + def _ficlone(source_fd, target_fd): + """ + Perform a lightweight copy of two files, where the data blocks are + copied only when modified. This is known as Copy on Write (CoW), + instantaneous copy or reflink. + """ + fcntl.ioctl(target_fd, fcntl.FICLONE, source_fd) +else: + _ficlone = None + + +if posix and hasattr(posix, '_fcopyfile'): + def _fcopyfile(source_fd, target_fd): + """ + Copy a regular file content using high-performance fcopyfile(3) + syscall (macOS). + """ + posix._fcopyfile(source_fd, target_fd, posix._COPYFILE_DATA) +else: + _fcopyfile = None + + +if hasattr(os, 'copy_file_range'): + def _copy_file_range(source_fd, target_fd): + """ + Copy data from one regular mmap-like fd to another by using a + high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side + copy. + This should work on Linux >= 4.5 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.copy_file_range(source_fd, target_fd, blocksize, + offset_dst=offset) + if sent == 0: + break # EOF + offset += sent +else: + _copy_file_range = None + + +if hasattr(os, 'sendfile'): + def _sendfile(source_fd, target_fd): + """Copy data from one regular mmap-like fd to another by using + high-performance sendfile(2) syscall. + This should work on Linux >= 2.6.33 only. + """ + blocksize = _get_copy_blocksize(source_fd) + offset = 0 + while True: + sent = os.sendfile(target_fd, source_fd, offset, blocksize) + if sent == 0: + break # EOF + offset += sent +else: + _sendfile = None + + +if _winapi and hasattr(_winapi, 'CopyFile2'): + def copyfile2(source, target): + """ + Copy from one file to another using CopyFile2 (Windows only). + """ + _winapi.CopyFile2(source, target, 0) +else: + copyfile2 = None + + +def copyfileobj(source_f, target_f): + """ + Copy data from file-like object source_f to file-like object target_f. + """ + try: + source_fd = source_f.fileno() + target_fd = target_f.fileno() + except Exception: + pass # Fall through to generic code. + else: + try: + # Use OS copy-on-write where available. + if _ficlone: + try: + _ficlone(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EBADF, EOPNOTSUPP, ETXTBSY, EXDEV): + raise err + + # Use OS copy where available. + if _fcopyfile: + try: + _fcopyfile(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (EINVAL, ENOTSUP): + raise err + if _copy_file_range: + try: + _copy_file_range(source_fd, target_fd) + return + except OSError as err: + if err.errno not in (ETXTBSY, EXDEV): + raise err + if _sendfile: + try: + _sendfile(source_fd, target_fd) + return + except OSError as err: + if err.errno != ENOTSOCK: + raise err + except OSError as err: + # Produce more useful error messages. + err.filename = source_f.name + err.filename2 = target_f.name + raise err + + # Last resort: copy with fileobj read() and write(). + read_source = source_f.read + write_target = target_f.write + while buf := read_source(1024 * 1024): + write_target(buf) + + +def magic_open(path, mode='r', buffering=-1, encoding=None, errors=None, + newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + text = 'b' not in mode + if text: + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + try: + return open(path, mode, buffering, encoding, errors, newline) + except TypeError: + pass + cls = type(path) + mode = ''.join(sorted(c for c in mode if c not in 'bt')) + if text: + try: + attr = getattr(cls, f'__open_{mode}__') + except AttributeError: + pass + else: + return attr(path, buffering, encoding, errors, newline) + elif encoding is not None: + raise ValueError("binary mode doesn't take an encoding argument") + elif errors is not None: + raise ValueError("binary mode doesn't take an errors argument") + elif newline is not None: + raise ValueError("binary mode doesn't take a newline argument") + + try: + attr = getattr(cls, f'__open_{mode}b__') + except AttributeError: + pass + else: + stream = attr(path, buffering) + if text: + stream = TextIOWrapper(stream, encoding, errors, newline) + return stream + + raise TypeError(f"{cls.__name__} can't be opened with mode {mode!r}") + + +def ensure_distinct_paths(source, target): + """ + Raise OSError(EINVAL) if the other path is within this path. + """ + # Note: there is no straightforward, foolproof algorithm to determine + # if one directory is within another (a particularly perverse example + # would be a single network share mounted in one location via NFS, and + # in another location via CIFS), so we simply checks whether the + # other path is lexically equal to, or within, this path. + if source == target: + err = OSError(EINVAL, "Source and target are the same path") + elif source in target.parents: + err = OSError(EINVAL, "Source path is a parent of target path") + else: + return + err.filename = str(source) + err.filename2 = str(target) + raise err + + +def ensure_different_files(source, target): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + source_file_id = source.info._file_id + target_file_id = target.info._file_id + except AttributeError: + if source != target: + return + else: + try: + if source_file_id() != target_file_id(): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(source) + err.filename2 = str(target) + raise err + + +def copy_info(info, target, follow_symlinks=True): + """Copy metadata from the given PathInfo to the given local path.""" + copy_times_ns = ( + hasattr(info, '_access_time_ns') and + hasattr(info, '_mod_time_ns') and + (follow_symlinks or os.utime in os.supports_follow_symlinks)) + if copy_times_ns: + t0 = info._access_time_ns(follow_symlinks=follow_symlinks) + t1 = info._mod_time_ns(follow_symlinks=follow_symlinks) + os.utime(target, ns=(t0, t1), follow_symlinks=follow_symlinks) + + # We must copy extended attributes before the file is (potentially) + # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. + copy_xattrs = ( + hasattr(info, '_xattrs') and + hasattr(os, 'setxattr') and + (follow_symlinks or os.setxattr in os.supports_follow_symlinks)) + if copy_xattrs: + xattrs = info._xattrs(follow_symlinks=follow_symlinks) + for attr, value in xattrs: + try: + os.setxattr(target, attr, value, follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + + copy_posix_permissions = ( + hasattr(info, '_posix_permissions') and + (follow_symlinks or os.chmod in os.supports_follow_symlinks)) + if copy_posix_permissions: + posix_permissions = info._posix_permissions(follow_symlinks=follow_symlinks) + try: + os.chmod(target, posix_permissions, follow_symlinks=follow_symlinks) + except NotImplementedError: + # if we got a NotImplementedError, it's because + # * follow_symlinks=False, + # * lchown() is unavailable, and + # * either + # * fchownat() is unavailable or + # * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW. + # (it returned ENOSUP.) + # therefore we're out of options--we simply cannot chown the + # symlink. give up, suppress the error. + # (which is what shutil always did in this circumstance.) + pass + + copy_bsd_flags = ( + hasattr(info, '_bsd_flags') and + hasattr(os, 'chflags') and + (follow_symlinks or os.chflags in os.supports_follow_symlinks)) + if copy_bsd_flags: + bsd_flags = info._bsd_flags(follow_symlinks=follow_symlinks) + try: + os.chflags(target, bsd_flags, follow_symlinks=follow_symlinks) + except OSError as why: + if why.errno not in (EOPNOTSUPP, ENOTSUP): + raise + + +class _PathInfoBase: + __slots__ = ('_path', '_stat_result', '_lstat_result') + + def __init__(self, path): + self._path = str(path) + + def __repr__(self): + path_type = "WindowsPath" if os.name == "nt" else "PosixPath" + return f"<{path_type}.info>" + + def _stat(self, *, follow_symlinks=True, ignore_errors=False): + """Return the status as an os.stat_result, or None if stat() fails and + ignore_errors is true.""" + if follow_symlinks: + try: + result = self._stat_result + except AttributeError: + pass + else: + if ignore_errors or result is not None: + return result + try: + self._stat_result = os.stat(self._path) + except (OSError, ValueError): + self._stat_result = None + if not ignore_errors: + raise + return self._stat_result + else: + try: + result = self._lstat_result + except AttributeError: + pass + else: + if ignore_errors or result is not None: + return result + try: + self._lstat_result = os.lstat(self._path) + except (OSError, ValueError): + self._lstat_result = None + if not ignore_errors: + raise + return self._lstat_result + + def _posix_permissions(self, *, follow_symlinks=True): + """Return the POSIX file permissions.""" + return S_IMODE(self._stat(follow_symlinks=follow_symlinks).st_mode) + + def _file_id(self, *, follow_symlinks=True): + """Returns the identifier of the file.""" + st = self._stat(follow_symlinks=follow_symlinks) + return st.st_dev, st.st_ino + + def _access_time_ns(self, *, follow_symlinks=True): + """Return the access time in nanoseconds.""" + return self._stat(follow_symlinks=follow_symlinks).st_atime_ns + + def _mod_time_ns(self, *, follow_symlinks=True): + """Return the modify time in nanoseconds.""" + return self._stat(follow_symlinks=follow_symlinks).st_mtime_ns + + if hasattr(os.stat_result, 'st_flags'): + def _bsd_flags(self, *, follow_symlinks=True): + """Return the flags.""" + return self._stat(follow_symlinks=follow_symlinks).st_flags + + if hasattr(os, 'listxattr'): + def _xattrs(self, *, follow_symlinks=True): + """Return the xattrs as a list of (attr, value) pairs, or an empty + list if extended attributes aren't supported.""" + try: + return [ + (attr, os.getxattr(self._path, attr, follow_symlinks=follow_symlinks)) + for attr in os.listxattr(self._path, follow_symlinks=follow_symlinks)] + except OSError as err: + if err.errno not in (EPERM, ENOTSUP, ENODATA, EINVAL, EACCES): + raise + return [] + + +class _WindowsPathInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information for Windows paths. Don't try to construct it yourself.""" + __slots__ = ('_exists', '_is_dir', '_is_file', '_is_symlink') + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + if not follow_symlinks and self.is_symlink(): + return True + try: + return self._exists + except AttributeError: + if os.path.exists(self._path): + self._exists = True + return True + else: + self._exists = self._is_dir = self._is_file = False + return False + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + if not follow_symlinks and self.is_symlink(): + return False + try: + return self._is_dir + except AttributeError: + if os.path.isdir(self._path): + self._is_dir = self._exists = True + return True + else: + self._is_dir = False + return False + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + if not follow_symlinks and self.is_symlink(): + return False + try: + return self._is_file + except AttributeError: + if os.path.isfile(self._path): + self._is_file = self._exists = True + return True + else: + self._is_file = False + return False + + def is_symlink(self): + """Whether this path is a symbolic link.""" + try: + return self._is_symlink + except AttributeError: + self._is_symlink = os.path.islink(self._path) + return self._is_symlink + + +class _PosixPathInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information for POSIX paths. Don't try to construct it yourself.""" + __slots__ = () + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return True + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return S_ISDIR(st.st_mode) + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + st = self._stat(follow_symlinks=follow_symlinks, ignore_errors=True) + if st is None: + return False + return S_ISREG(st.st_mode) + + def is_symlink(self): + """Whether this path is a symbolic link.""" + st = self._stat(follow_symlinks=False, ignore_errors=True) + if st is None: + return False + return S_ISLNK(st.st_mode) + + +PathInfo = _WindowsPathInfo if os.name == 'nt' else _PosixPathInfo + + +class DirEntryInfo(_PathInfoBase): + """Implementation of pathlib.types.PathInfo that provides status + information by querying a wrapped os.DirEntry object. Don't try to + construct it yourself.""" + __slots__ = ('_entry',) + + def __init__(self, entry): + super().__init__(entry.path) + self._entry = entry + + def _stat(self, *, follow_symlinks=True, ignore_errors=False): + try: + return self._entry.stat(follow_symlinks=follow_symlinks) + except OSError: + if not ignore_errors: + raise + return None + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + if not follow_symlinks: + return True + return self._stat(ignore_errors=True) is not None + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + try: + return self._entry.is_dir(follow_symlinks=follow_symlinks) + except OSError: + return False + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + try: + return self._entry.is_file(follow_symlinks=follow_symlinks) + except OSError: + return False + + def is_symlink(self): + """Whether this path is a symbolic link.""" + try: + return self._entry.is_symlink() + except OSError: + return False diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py new file mode 100644 index 00000000000..d8f5c34a1a7 --- /dev/null +++ b/Lib/pathlib/types.py @@ -0,0 +1,430 @@ +""" +Protocols for supporting classes in pathlib. +""" + +# This module also provides abstract base classes for rich path objects. +# These ABCs are a *private* part of the Python standard library, but they're +# made available as a PyPI package called "pathlib-abc". It's possible they'll +# become an official part of the standard library in future. +# +# Three ABCs are provided -- _JoinablePath, _ReadablePath and _WritablePath + + +from abc import ABC, abstractmethod +from glob import _PathGlobber +from io import text_encoding +from pathlib._os import magic_open, ensure_distinct_paths, ensure_different_files, copyfileobj +from pathlib import PurePath, Path +from typing import Optional, Protocol, runtime_checkable + + +def _explode_path(path, split): + """ + Split the path into a 2-tuple (anchor, parts), where *anchor* is the + uppermost parent of the path (equivalent to path.parents[-1]), and + *parts* is a reversed list of parts following the anchor. + """ + parent, name = split(path) + names = [] + while path != parent: + names.append(name) + path = parent + parent, name = split(path) + return path, names + + +@runtime_checkable +class _PathParser(Protocol): + """Protocol for path parsers, which do low-level path manipulation. + + Path parsers provide a subset of the os.path API, specifically those + functions needed to provide JoinablePath functionality. Each JoinablePath + subclass references its path parser via a 'parser' class attribute. + """ + + sep: str + altsep: Optional[str] + def split(self, path: str) -> tuple[str, str]: ... + def splitext(self, path: str) -> tuple[str, str]: ... + def normcase(self, path: str) -> str: ... + + +@runtime_checkable +class PathInfo(Protocol): + """Protocol for path info objects, which support querying the file type. + Methods may return cached results. + """ + def exists(self, *, follow_symlinks: bool = True) -> bool: ... + def is_dir(self, *, follow_symlinks: bool = True) -> bool: ... + def is_file(self, *, follow_symlinks: bool = True) -> bool: ... + def is_symlink(self) -> bool: ... + + +class _JoinablePath(ABC): + """Abstract base class for pure path objects. + + This class *does not* provide several magic methods that are defined in + its implementation PurePath. They are: __init__, __fspath__, __bytes__, + __reduce__, __hash__, __eq__, __lt__, __le__, __gt__, __ge__. + """ + __slots__ = () + + @property + @abstractmethod + def parser(self): + """Implementation of pathlib._types.Parser used for low-level path + parsing and manipulation. + """ + raise NotImplementedError + + @abstractmethod + def with_segments(self, *pathsegments): + """Construct a new path object from any number of path-like objects. + Subclasses may override this method to customize how new path objects + are created from methods like `iterdir()`. + """ + raise NotImplementedError + + @abstractmethod + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + raise NotImplementedError + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return _explode_path(str(self), self.parser.split)[0] + + @property + def name(self): + """The final path component, if any.""" + return self.parser.split(str(self))[1] + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + return self.parser.splitext(self.name)[1] + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + split = self.parser.splitext + stem, suffix = split(self.name) + suffixes = [] + while suffix: + suffixes.append(suffix) + stem, suffix = split(stem) + return suffixes[::-1] + + @property + def stem(self): + """The final path component, minus its last suffix.""" + return self.parser.splitext(self.name)[0] + + def with_name(self, name): + """Return a new path with the file name changed.""" + split = self.parser.split + if split(name)[0]: + raise ValueError(f"Invalid name {name!r}") + path = str(self) + path = path.removesuffix(split(path)[1]) + name + return self.with_segments(path) + + def with_stem(self, stem): + """Return a new path with the stem changed.""" + suffix = self.suffix + if not suffix: + return self.with_name(stem) + elif not stem: + # If the suffix is non-empty, we can't make the stem empty. + raise ValueError(f"{self!r} has a non-empty suffix") + else: + return self.with_name(stem + suffix) + + def with_suffix(self, suffix): + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. + """ + stem = self.stem + if not stem: + # If the stem is empty, we can't make the suffix non-empty. + raise ValueError(f"{self!r} has an empty name") + elif suffix and not suffix.startswith('.'): + raise ValueError(f"Invalid suffix {suffix!r}") + else: + return self.with_name(stem + suffix) + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + anchor, parts = _explode_path(str(self), self.parser.split) + if anchor: + parts.append(anchor) + return tuple(reversed(parts)) + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(str(self), *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(str(self), key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, str(self)) + except TypeError: + return NotImplemented + + @property + def parent(self): + """The logical parent of the path.""" + path = str(self) + parent = self.parser.split(path)[0] + if path != parent: + return self.with_segments(parent) + return self + + @property + def parents(self): + """A sequence of this path's logical parents.""" + split = self.parser.split + path = str(self) + parent = split(path)[0] + parents = [] + while path != parent: + parents.append(self.with_segments(parent)) + path = parent + parent = split(path)[0] + return tuple(parents) + + def full_match(self, pattern): + """ + Return True if this path matches the given glob-style pattern. The + pattern is matched against the entire path. + """ + case_sensitive = self.parser.normcase('Aa') == 'Aa' + globber = _PathGlobber(self.parser.sep, case_sensitive, recursive=True) + match = globber.compile(pattern, altsep=self.parser.altsep) + return match(str(self)) is not None + + +class _ReadablePath(_JoinablePath): + """Abstract base class for readable path objects. + + The Path class implements this ABC for local filesystem paths. Users may + create subclasses to implement readable virtual filesystem paths, such as + paths in archive files or on remote storage systems. + """ + __slots__ = () + + @property + @abstractmethod + def info(self): + """ + A PathInfo object that exposes the file type and other file attributes + of this path. + """ + raise NotImplementedError + + @abstractmethod + def __open_rb__(self, buffering=-1): + """ + Open the file pointed to by this path for reading in binary mode and + return a file object, like open(mode='rb'). + """ + raise NotImplementedError + + def read_bytes(self): + """ + Open the file in bytes mode, read it, and close the file. + """ + with magic_open(self, mode='rb', buffering=0) as f: + return f.read() + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + with magic_open(self, mode='r', encoding=encoding, errors=errors, newline=newline) as f: + return f.read() + + @abstractmethod + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + raise NotImplementedError + + def glob(self, pattern, *, recurse_symlinks=True): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + anchor, parts = _explode_path(pattern, self.parser.split) + if anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + elif not parts: + raise ValueError(f"Unacceptable pattern: {pattern!r}") + elif not recurse_symlinks: + raise NotImplementedError("recurse_symlinks=False is unsupported") + case_sensitive = self.parser.normcase('Aa') == 'Aa' + globber = _PathGlobber(self.parser.sep, case_sensitive, recursive=True) + select = globber.selector(parts) + return select(self.joinpath('')) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + paths = [self] + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) + try: + for child in path.iterdir(): + if child.info.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(child) + dirnames.append(child.name) + else: + filenames.append(child.name) + except OSError as error: + if on_error is not None: + on_error(error) + if not top_down: + while not isinstance(paths.pop(), tuple): + pass + continue + if top_down: + yield path, dirnames, filenames + paths += [path.joinpath(d) for d in reversed(dirnames)] + + @abstractmethod + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + raise NotImplementedError + + def copy(self, target, **kwargs): + """ + Recursively copy this file or directory tree to the given destination. + """ + ensure_distinct_paths(self, target) + target._copy_from(self, **kwargs) + return target.joinpath() # Empty join to ensure fresh metadata. + + def copy_into(self, target_dir, **kwargs): + """ + Copy this file or directory tree into the given existing directory. + """ + name = self.name + if not name: + raise ValueError(f"{self!r} has an empty name") + return self.copy(target_dir / name, **kwargs) + + +class _WritablePath(_JoinablePath): + """Abstract base class for writable path objects. + + The Path class implements this ABC for local filesystem paths. Users may + create subclasses to implement writable virtual filesystem paths, such as + paths in archive files or on remote storage systems. + """ + __slots__ = () + + @abstractmethod + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + raise NotImplementedError + + @abstractmethod + def mkdir(self): + """ + Create a new directory at this given path. + """ + raise NotImplementedError + + @abstractmethod + def __open_wb__(self, buffering=-1): + """ + Open the file pointed to by this path for writing in binary mode and + return a file object, like open(mode='wb'). + """ + raise NotImplementedError + + def write_bytes(self, data): + """ + Open the file in bytes mode, write to it, and close the file. + """ + # type-check for the buffer interface before truncating the file + view = memoryview(data) + with magic_open(self, mode='wb') as f: + return f.write(view) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = text_encoding(encoding) + if not isinstance(data, str): + raise TypeError('data must be str, not %s' % + data.__class__.__name__) + with magic_open(self, mode='w', encoding=encoding, errors=errors, newline=newline) as f: + return f.write(data) + + def _copy_from(self, source, follow_symlinks=True): + """ + Recursively copy the given path to this path. + """ + stack = [(source, self)] + while stack: + src, dst = stack.pop() + if not follow_symlinks and src.info.is_symlink(): + dst.symlink_to(str(src.readlink()), src.info.is_dir()) + elif src.info.is_dir(): + children = src.iterdir() + dst.mkdir() + for child in children: + stack.append((child, dst.joinpath(child.name))) + else: + ensure_different_files(src, dst) + with magic_open(src, 'rb') as source_f: + with magic_open(dst, 'wb') as target_f: + copyfileobj(source_f, target_f) + + +_JoinablePath.register(PurePath) +_ReadablePath.register(Path) +_WritablePath.register(Path) diff --git a/Lib/pdb.py b/Lib/pdb.py index bf503f1e73e..ec6cf06e58b 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -185,6 +185,15 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.commands_bnum = None # The breakpoint number for which we are # defining a list + def set_trace(self, frame=None, *, commands=None): + if frame is None: + frame = sys._getframe().f_back + + if commands is not None: + self.rcLines.extend(commands) + + super().set_trace(frame) + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt diff --git a/Lib/pickle.py b/Lib/pickle.py index f027e043204..beaefae0479 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -26,12 +26,11 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache -from itertools import islice +from itertools import batched from functools import partial import sys from sys import maxsize from struct import pack, unpack -import re import io import codecs import _compat_pickle @@ -51,7 +50,7 @@ bytes_types = (bytes, bytearray) # These are purely informational; no code uses these. -format_version = "4.0" # File format version we write +format_version = "5.0" # File format version we write compatible_formats = ["1.0", # Original protocol 0 "1.1", # Protocol 0 with INST added "1.2", # Original protocol 1 @@ -68,7 +67,7 @@ # The protocol we write by default. May be less than HIGHEST_PROTOCOL. # Only bump this if the oldest still supported version of Python already # includes it. -DEFAULT_PROTOCOL = 4 +DEFAULT_PROTOCOL = 5 class PickleError(Exception): """A common base class for the other pickling exceptions.""" @@ -98,12 +97,6 @@ class _Stop(Exception): def __init__(self, value): self.value = value -# Jython has PyStringMap; it's a dict subclass with string keys -try: - from org.python.core import PyStringMap -except ImportError: - PyStringMap = None - # Pickle opcodes. See pickletools.py for extensive docs. The listing # here is in kind-of alphabetical order of 1-character pickle code. # pickletools groups them by purpose. @@ -194,7 +187,7 @@ def __init__(self, value): NEXT_BUFFER = b'\x97' # push next out-of-band buffer READONLY_BUFFER = b'\x98' # make top of stack readonly -__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)]) +__all__.extend(x for x in dir() if x.isupper() and not x.startswith('_')) class _Framer: @@ -319,37 +312,46 @@ def load_frame(self, frame_size): # Tools used for pickling. -def _getattribute(obj, name): - for subpath in name.split('.'): - if subpath == '': - raise AttributeError("Can't get local attribute {!r} on {!r}" - .format(name, obj)) - try: - parent = obj - obj = getattr(obj, subpath) - except AttributeError: - raise AttributeError("Can't get attribute {!r} on {!r}" - .format(name, obj)) from None - return obj, parent +def _getattribute(obj, dotted_path): + for subpath in dotted_path: + obj = getattr(obj, subpath) + return obj def whichmodule(obj, name): """Find the module an object belong to.""" + dotted_path = name.split('.') module_name = getattr(obj, '__module__', None) - if module_name is not None: - return module_name - # Protect the iteration by using a list copy of sys.modules against dynamic - # modules that trigger imports of other modules upon calls to getattr. - for module_name, module in sys.modules.copy().items(): - if (module_name == '__main__' - or module_name == '__mp_main__' # bpo-42406 - or module is None): - continue - try: - if _getattribute(module, name)[0] is obj: - return module_name - except AttributeError: - pass - return '__main__' + if '' in dotted_path: + raise PicklingError(f"Can't pickle local object {obj!r}") + if module_name is None: + # Protect the iteration by using a list copy of sys.modules against dynamic + # modules that trigger imports of other modules upon calls to getattr. + for module_name, module in sys.modules.copy().items(): + if (module_name == '__main__' + or module_name == '__mp_main__' # bpo-42406 + or module is None): + continue + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + pass + module_name = '__main__' + + try: + __import__(module_name, level=0) + module = sys.modules[module_name] + except (ImportError, ValueError, KeyError) as exc: + raise PicklingError(f"Can't pickle {obj!r}: {exc!s}") + try: + if _getattribute(module, dotted_path) is obj: + return module_name + except AttributeError: + raise PicklingError(f"Can't pickle {obj!r}: " + f"it's not found as {module_name}.{name}") + + raise PicklingError( + f"Can't pickle {obj!r}: it's not the same object as {module_name}.{name}") def encode_long(x): r"""Encode a long to a two's complement little-endian binary string. @@ -401,6 +403,15 @@ def decode_long(data): """ return int.from_bytes(data, byteorder='little', signed=True) +def _T(obj): + cls = type(obj) + module = cls.__module__ + if module in (None, 'builtins', '__main__'): + return cls.__qualname__ + return f'{module}.{cls.__qualname__}' + + +_NoValue = object() # Pickling machinery @@ -412,7 +423,7 @@ def __init__(self, file, protocol=None, *, fix_imports=True, The optional *protocol* argument tells the pickler to use the given protocol; supported protocols are 0, 1, 2, 3, 4 and 5. - The default protocol is 4. It was introduced in Python 3.4, and + The default protocol is 5. It was introduced in Python 3.8, and is incompatible with previous versions. Specifying a negative protocol version selects the highest @@ -536,10 +547,11 @@ def save(self, obj, save_persistent_id=True): self.framer.commit_frame() # Check for persistent id (defined by a subclass) - pid = self.persistent_id(obj) - if pid is not None and save_persistent_id: - self.save_pers(pid) - return + if save_persistent_id: + pid = self.persistent_id(obj) + if pid is not None: + self.save_pers(pid) + return # Check the memo x = self.memo.get(id(obj)) @@ -548,8 +560,8 @@ def save(self, obj, save_persistent_id=True): return rv = NotImplemented - reduce = getattr(self, "reducer_override", None) - if reduce is not None: + reduce = getattr(self, "reducer_override", _NoValue) + if reduce is not _NoValue: rv = reduce(obj) if rv is NotImplemented: @@ -562,8 +574,8 @@ def save(self, obj, save_persistent_id=True): # Check private dispatch table if any, or else # copyreg.dispatch_table - reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) - if reduce is not None: + reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue) + if reduce is not _NoValue: rv = reduce(obj) else: # Check for a class with a custom metaclass; treat as regular @@ -573,34 +585,37 @@ def save(self, obj, save_persistent_id=True): return # Check for a __reduce_ex__ method, fall back to __reduce__ - reduce = getattr(obj, "__reduce_ex__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce_ex__", _NoValue) + if reduce is not _NoValue: rv = reduce(self.proto) else: - reduce = getattr(obj, "__reduce__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce__", _NoValue) + if reduce is not _NoValue: rv = reduce() else: - raise PicklingError("Can't pickle %r object: %r" % - (t.__name__, obj)) + raise PicklingError(f"Can't pickle {_T(t)} object") # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): self.save_global(obj, rv) return - # Assert that reduce() returned a tuple - if not isinstance(rv, tuple): - raise PicklingError("%s must return string or tuple" % reduce) - - # Assert that it returned an appropriately sized tuple - l = len(rv) - if not (2 <= l <= 6): - raise PicklingError("Tuple returned by %s must have " - "two to six elements" % reduce) - - # Save the reduce() output and finally memoize the object - self.save_reduce(obj=obj, *rv) + try: + # Assert that reduce() returned a tuple + if not isinstance(rv, tuple): + raise PicklingError(f'__reduce__ must return a string or tuple, not {_T(rv)}') + + # Assert that it returned an appropriately sized tuple + l = len(rv) + if not (2 <= l <= 6): + raise PicklingError("tuple returned by __reduce__ " + "must contain 2 through 6 elements") + + # Save the reduce() output and finally memoize the object + self.save_reduce(obj=obj, *rv) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} object') + raise def persistent_id(self, obj): # This exists so a subclass can override it @@ -622,10 +637,12 @@ def save_reduce(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, *, obj=None): # This API is called by some subclasses - if not isinstance(args, tuple): - raise PicklingError("args from save_reduce() must be a tuple") if not callable(func): - raise PicklingError("func from save_reduce() must be callable") + raise PicklingError(f"first item of the tuple returned by __reduce__ " + f"must be callable, not {_T(func)}") + if not isinstance(args, tuple): + raise PicklingError(f"second item of the tuple returned by __reduce__ " + f"must be a tuple, not {_T(args)}") save = self.save write = self.write @@ -634,19 +651,30 @@ def save_reduce(self, func, args, state=None, listitems=None, if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): - raise PicklingError("args[0] from {} args has no __new__" - .format(func_name)) + raise PicklingError("first argument to __newobj_ex__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError("args[0] from {} args has the wrong class" - .format(func_name)) + raise PicklingError(f"first argument to __newobj_ex__() " + f"must be {obj.__class__!r}, not {cls!r}") if self.proto >= 4: - save(cls) - save(args) - save(kwargs) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + save(kwargs) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ_EX) else: func = partial(cls.__new__, cls, *args, **kwargs) - save(func) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise save(()) write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": @@ -678,18 +706,33 @@ def save_reduce(self, func, args, state=None, listitems=None, # Python 2.2). cls = args[0] if not hasattr(cls, "__new__"): - raise PicklingError( - "args[0] from __newobj__ args has no __new__") + raise PicklingError("first argument to __newobj__() has no __new__") if obj is not None and cls is not obj.__class__: - raise PicklingError( - "args[0] from __newobj__ args has the wrong class") + raise PicklingError(f"first argument to __newobj__() " + f"must be {obj.__class__!r}, not {cls!r}") args = args[1:] - save(cls) - save(args) + try: + save(cls) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} class') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} __new__ arguments') + raise write(NEWOBJ) else: - save(func) - save(args) + try: + save(func) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor') + raise + try: + save(args) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} reconstructor arguments') + raise write(REDUCE) if obj is not None: @@ -707,23 +750,35 @@ def save_reduce(self, func, args, state=None, listitems=None, # items and dict items (as (key, value) tuples), or None. if listitems is not None: - self._batch_appends(listitems) + self._batch_appends(listitems, obj) if dictitems is not None: - self._batch_setitems(dictitems) + self._batch_setitems(dictitems, obj) if state is not None: if state_setter is None: - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(BUILD) else: # If a state_setter is specified, call it instead of load_build # to update obj's with its previous state. # First, push state_setter and its tuple of expected arguments # (obj, state) onto the stack. - save(state_setter) + try: + save(state_setter) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state setter') + raise save(obj) # simple BINGET opcode as obj is already memoized. - save(state) + try: + save(state) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} state') + raise write(TUPLE2) # Trigger a state_setter(obj, state) function call. write(REDUCE) @@ -786,14 +841,10 @@ def save_float(self, obj): self.write(FLOAT + repr(obj).encode("ascii") + b'\n') dispatch[float] = save_float - def save_bytes(self, obj): - if self.proto < 3: - if not obj: # bytes object is empty - self.save_reduce(bytes, (), obj=obj) - else: - self.save_reduce(codecs.encode, - (str(obj, 'latin1'), 'latin1'), obj=obj) - return + def _save_bytes_no_memo(self, obj): + # helper for writing bytes objects for protocol >= 3 + # without memoizing them + assert self.proto >= 3 n = len(obj) if n <= 0xff: self.write(SHORT_BINBYTES + pack("= 5 + # without memoizing them + assert self.proto >= 5 + n = len(obj) + if n >= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BYTEARRAY8 + pack("= self.framer._FRAME_SIZE_TARGET: - self._write_large_bytes(BYTEARRAY8 + pack("= 5") with obj.raw() as m: if not m.contiguous: @@ -836,10 +903,18 @@ def save_picklebuffer(self, obj): if in_band: # Write data in-band # XXX The C implementation avoids a copy here + buf = m.tobytes() + in_memo = id(buf) in self.memo if m.readonly: - self.save_bytes(m.tobytes()) + if in_memo: + self._save_bytes_no_memo(buf) + else: + self.save_bytes(buf) else: - self.save_bytearray(m.tobytes()) + if in_memo: + self._save_bytearray_no_memo(buf) + else: + self.save_bytearray(buf) else: # Write data out-of-band self.write(NEXT_BUFFER) @@ -861,13 +936,13 @@ def save_str(self, obj): else: self.write(BINUNICODE + pack("= 2: - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise # Subtle. Same as in the big comment below. if id(obj) in memo: get = self.get(memo[id(obj)][0]) @@ -898,8 +977,12 @@ def save_tuple(self, obj): # has more than 3 elements. write = self.write write(MARK) - for element in obj: - save(element) + for i, element in enumerate(obj): + try: + save(element) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise if id(obj) in memo: # Subtle. d was not in memo when we entered save_tuple(), so @@ -929,38 +1012,47 @@ def save_list(self, obj): self.write(MARK + LIST) self.memoize(obj) - self._batch_appends(obj) + self._batch_appends(obj, obj) dispatch[list] = save_list _BATCHSIZE = 1000 - def _batch_appends(self, items): + def _batch_appends(self, items, obj): # Helper to batch up APPENDS sequences save = self.save write = self.write if not self.bin: - for x in items: - save(x) + for i, x in enumerate(items): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPEND) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + start = 0 + for batch in batched(items, self._BATCHSIZE): + batch_len = len(batch) + if batch_len != 1: write(MARK) - for x in tmp: - save(x) + for i, x in enumerate(batch, start): + try: + save(x) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {i}') + raise write(APPENDS) - elif n: - save(tmp[0]) + else: + try: + save(batch[0]) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {start}') + raise write(APPEND) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return + start += batch_len def save_dict(self, obj): if self.bin: @@ -969,13 +1061,11 @@ def save_dict(self, obj): self.write(MARK + DICT) self.memoize(obj) - self._batch_setitems(obj.items()) + self._batch_setitems(obj.items(), obj) dispatch[dict] = save_dict - if PyStringMap is not None: - dispatch[PyStringMap] = save_dict - def _batch_setitems(self, items): + def _batch_setitems(self, items, obj): # Helper to batch up SETITEMS sequences; proto >= 1 only save = self.save write = self.write @@ -983,28 +1073,34 @@ def _batch_setitems(self, items): if not self.bin: for k, v in items: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) return - it = iter(items) - while True: - tmp = list(islice(it, self._BATCHSIZE)) - n = len(tmp) - if n > 1: + for batch in batched(items, self._BATCHSIZE): + if len(batch) != 1: write(MARK) - for k, v in tmp: + for k, v in batch: save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEMS) - elif n: - k, v = tmp[0] + else: + k, v = batch[0] save(k) - save(v) + try: + save(v) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} item {k!r}') + raise write(SETITEM) - # else tmp is empty, and we're done - if n < self._BATCHSIZE: - return def save_set(self, obj): save = self.save @@ -1017,17 +1113,15 @@ def save_set(self, obj): write(EMPTY_SET) self.memoize(obj) - it = iter(obj) - while True: - batch = list(islice(it, self._BATCHSIZE)) - n = len(batch) - if n > 0: - write(MARK) + for batch in batched(obj, self._BATCHSIZE): + write(MARK) + try: for item in batch: save(item) - write(ADDITEMS) - if n < self._BATCHSIZE: - return + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise + write(ADDITEMS) dispatch[set] = save_set def save_frozenset(self, obj): @@ -1039,8 +1133,12 @@ def save_frozenset(self, obj): return write(MARK) - for item in obj: - save(item) + try: + for item in obj: + save(item) + except BaseException as exc: + exc.add_note(f'when serializing {_T(obj)} element') + raise if id(obj) in self.memo: # If the object is already in the memo, this means it is @@ -1059,48 +1157,59 @@ def save_global(self, obj, name=None): if name is None: name = getattr(obj, '__qualname__', None) - if name is None: - name = obj.__name__ + if name is None: + name = obj.__name__ module_name = whichmodule(obj, name) - try: - __import__(module_name, level=0) - module = sys.modules[module_name] - obj2, parent = _getattribute(module, name) - except (ImportError, KeyError, AttributeError): - raise PicklingError( - "Can't pickle %r: it's not found as %s.%s" % - (obj, module_name, name)) from None - else: - if obj2 is not obj: - raise PicklingError( - "Can't pickle %r: it's not the same object as %s.%s" % - (obj, module_name, name)) - if self.proto >= 2: - code = _extension_registry.get((module_name, name)) - if code: - assert code > 0 + code = _extension_registry.get((module_name, name), _NoValue) + if code is not _NoValue: if code <= 0xff: - write(EXT1 + pack("= 3. + if self.proto >= 4: self.save(module_name) self.save(name) write(STACK_GLOBAL) - elif parent is not module: - self.save_reduce(getattr, (parent, lastname)) - elif self.proto >= 3: - write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + - bytes(name, "utf-8") + b'\n') + elif '.' in name: + # In protocol < 4, objects with multi-part __qualname__ + # are represented as + # getattr(getattr(..., attrname1), attrname2). + dotted_path = name.split('.') + name = dotted_path.pop(0) + save = self.save + for attrname in dotted_path: + save(getattr) + if self.proto < 2: + write(MARK) + self._save_toplevel_by_name(module_name, name) + for attrname in dotted_path: + save(attrname) + if self.proto < 2: + write(TUPLE) + else: + write(TUPLE2) + write(REDUCE) + else: + self._save_toplevel_by_name(module_name, name) + + self.memoize(obj) + + def _save_toplevel_by_name(self, module_name, name): + if self.proto >= 3: + # Non-ASCII identifiers are supported only with protocols >= 3. + encoding = "utf-8" else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING @@ -1109,15 +1218,19 @@ def save_global(self, obj, name=None): module_name, name = r_name_mapping[(module_name, name)] elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] - try: - write(GLOBAL + bytes(module_name, "ascii") + b'\n' + - bytes(name, "ascii") + b'\n') - except UnicodeEncodeError: - raise PicklingError( - "can't pickle global identifier '%s.%s' using " - "pickle protocol %i" % (module, name, self.proto)) from None - - self.memoize(obj) + encoding = "ascii" + try: + self.write(GLOBAL + bytes(module_name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle module identifier {module_name!r} using " + f"pickle protocol {self.proto}") + try: + self.write(bytes(name, encoding) + b'\n') + except UnicodeEncodeError: + raise PicklingError( + f"can't pickle global identifier {name!r} using " + f"pickle protocol {self.proto}") def save_type(self, obj): if obj is type(None): @@ -1273,7 +1386,7 @@ def load_int(self): elif data == TRUE[1:]: val = True else: - val = int(data, 0) + val = int(data) self.append(val) dispatch[INT[0]] = load_int @@ -1293,7 +1406,7 @@ def load_long(self): val = self.readline()[:-1] if val and val[-1] == b'L'[0]: val = val[:-1] - self.append(int(val, 0)) + self.append(int(val)) dispatch[LONG[0]] = load_long def load_long1(self): @@ -1489,7 +1602,7 @@ def _instantiate(self, klass, args): value = klass(*args) except TypeError as err: raise TypeError("in constructor for %s: %s" % - (klass.__name__, str(err)), sys.exc_info()[2]) + (klass.__name__, str(err)), err.__traceback__) else: value = klass.__new__(klass) self.append(value) @@ -1554,9 +1667,8 @@ def load_ext4(self): dispatch[EXT4[0]] = load_ext4 def get_extension(self, code): - nil = [] - obj = _extension_cache.get(code, nil) - if obj is not nil: + obj = _extension_cache.get(code, _NoValue) + if obj is not _NoValue: self.append(obj) return key = _inverted_registry.get(code) @@ -1578,8 +1690,13 @@ def find_class(self, module, name): elif module in _compat_pickle.IMPORT_MAPPING: module = _compat_pickle.IMPORT_MAPPING[module] __import__(module, level=0) - if self.proto >= 4: - return _getattribute(sys.modules[module], name)[0] + if self.proto >= 4 and '.' in name: + dotted_path = name.split('.') + try: + return _getattribute(sys.modules[module], dotted_path) + except AttributeError: + raise AttributeError( + f"Can't resolve path {name!r} on module {module!r}") else: return getattr(sys.modules[module], name) @@ -1713,8 +1830,8 @@ def load_build(self): stack = self.stack state = stack.pop() inst = stack[-1] - setstate = getattr(inst, "__setstate__", None) - if setstate is not None: + setstate = getattr(inst, "__setstate__", _NoValue) + if setstate is not _NoValue: setstate(state) return slotstate = None @@ -1789,32 +1906,26 @@ def _loads(s, /, *, fix_imports=True, encoding="ASCII", errors="strict", Pickler, Unpickler = _Pickler, _Unpickler dump, dumps, load, loads = _dump, _dumps, _load, _loads -# Doctest -def _test(): - import doctest - return doctest.testmod() -if __name__ == "__main__": +def _main(args=None): import argparse + import pprint parser = argparse.ArgumentParser( - description='display contents of the pickle files') - parser.add_argument( - 'pickle_file', type=argparse.FileType('br'), - nargs='*', help='the pickle file') - parser.add_argument( - '-t', '--test', action='store_true', - help='run self-test suite') + description='display contents of the pickle files', + color=True, + ) parser.add_argument( - '-v', action='store_true', - help='run verbosely; only affects self-test run') - args = parser.parse_args() - if args.test: - _test() - else: - if not args.pickle_file: - parser.print_help() + 'pickle_file', + nargs='+', help='the pickle file') + args = parser.parse_args(args) + for fn in args.pickle_file: + if fn == '-': + obj = load(sys.stdin.buffer) else: - import pprint - for f in args.pickle_file: + with open(fn, 'rb') as f: obj = load(f) - pprint.pprint(obj) + pprint.pprint(obj) + + +if __name__ == "__main__": + _main() diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 95706e746c9..254b6c7fcc9 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -312,7 +312,7 @@ def read_uint8(f): doc="Eight-byte unsigned integer, little-endian.") -def read_stringnl(f, decode=True, stripquotes=True): +def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'): r""" >>> import io >>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n")) @@ -348,7 +348,7 @@ def read_stringnl(f, decode=True, stripquotes=True): for q in (b'"', b"'"): if data.startswith(q): if not data.endswith(q): - raise ValueError("strinq quote %r not found at both " + raise ValueError("string quote %r not found at both " "ends of %r" % (q, data)) data = data[1:-1] break @@ -356,7 +356,7 @@ def read_stringnl(f, decode=True, stripquotes=True): raise ValueError("no string quotes around %r" % data) if decode: - data = codecs.escape_decode(data)[0].decode("ascii") + data = codecs.escape_decode(data)[0].decode(encoding) return data stringnl = ArgumentDescriptor( @@ -370,7 +370,7 @@ def read_stringnl(f, decode=True, stripquotes=True): """) def read_stringnl_noescape(f): - return read_stringnl(f, stripquotes=False) + return read_stringnl(f, stripquotes=False, encoding='utf-8') stringnl_noescape = ArgumentDescriptor( name='stringnl_noescape', @@ -1253,7 +1253,7 @@ def __init__(self, name, code, arg, stack_before=[], stack_after=[pyint], proto=2, - doc="""Long integer using found-byte length. + doc="""Long integer using four-byte length. A more efficient encoding of a Python long; the long4 encoding says it all."""), @@ -2429,8 +2429,6 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): + A memo entry isn't referenced before it's defined. + The markobject isn't stored in the memo. - - + A memo entry isn't redefined. """ # Most of the hair here is for sanity checks, but most of it is needed @@ -2484,7 +2482,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): assert opcode.name == "POP" numtopop = 0 else: - errormsg = markmsg = "no MARK exists on stack" + errormsg = "no MARK exists on stack" # Check for correct memo usage. if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): @@ -2494,9 +2492,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): else: assert arg is not None memo_idx = arg - if memo_idx in memo: - errormsg = "memo key %r already defined" % arg - elif not stack: + if not stack: errormsg = "stack is empty -- can't store into memo" elif stack[-1] is markobject: errormsg = "can't store markobject in the memo" @@ -2513,7 +2509,10 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): # make a mild effort to align arguments line += ' ' * (10 - len(opcode.name)) if arg is not None: - line += ' ' + repr(arg) + if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"): + line += ' ' + ascii(arg) + else: + line += ' ' + repr(arg) if markmsg: line += ' ' + markmsg if annotate: @@ -2839,19 +2838,18 @@ def __init__(self, value): 'disassembler_memo_test': _memo_test, } -def _test(): - import doctest - return doctest.testmod() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser( - description='disassemble one or more pickle files') + description='disassemble one or more pickle files', + color=True, + ) parser.add_argument( - 'pickle_file', type=argparse.FileType('br'), - nargs='*', help='the pickle file') + 'pickle_file', + nargs='+', help='the pickle file') parser.add_argument( - '-o', '--output', default=sys.stdout, type=argparse.FileType('w'), + '-o', '--output', help='the file where the output should be written') parser.add_argument( '-m', '--memo', action='store_true', @@ -2866,25 +2864,24 @@ def _test(): '-p', '--preamble', default="==> {name} <==", help='if more than one pickle file is specified, print this before' ' each disassembly') - parser.add_argument( - '-t', '--test', action='store_true', - help='run self-test suite') - parser.add_argument( - '-v', action='store_true', - help='run verbosely; only affects self-test run') args = parser.parse_args() - if args.test: - _test() + annotate = 30 if args.annotate else 0 + memo = {} if args.memo else None + if args.output is None: + output = sys.stdout else: - annotate = 30 if args.annotate else 0 - if not args.pickle_file: - parser.print_help() - elif len(args.pickle_file) == 1: - dis(args.pickle_file[0], args.output, None, - args.indentlevel, annotate) - else: - memo = {} if args.memo else None - for f in args.pickle_file: - preamble = args.preamble.format(name=f.name) - args.output.write(preamble + '\n') - dis(f, args.output, memo, args.indentlevel, annotate) + output = open(args.output, 'w') + try: + for arg in args.pickle_file: + if len(args.pickle_file) > 1: + name = '' if arg == '-' else arg + preamble = args.preamble.format(name=name) + output.write(preamble + '\n') + if arg == '-': + dis(sys.stdin.buffer, output, memo, args.indentlevel, annotate) + else: + with open(arg, 'rb') as f: + dis(f, output, memo, args.indentlevel, annotate) + finally: + if output is not sys.stdout: + output.close() diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 8e010c79c12..8772a66791a 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -8,13 +8,11 @@ import os import os.path import sys -from types import ModuleType -import warnings __all__ = [ - 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'get_importer', 'iter_importers', 'walk_packages', 'iter_modules', 'get_data', - 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', + 'read_code', 'extend_path', 'ModuleInfo', ] @@ -23,20 +21,6 @@ ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.' -def _get_spec(finder, name): - """Return the finder-specific module spec.""" - # Works with legacy finders. - try: - find_spec = finder.find_spec - except AttributeError: - loader = finder.find_module(name) - if loader is None: - return None - return importlib.util.spec_from_loader(name, loader) - else: - return find_spec(name) - - def read_code(stream): # This helper is needed in order for the PEP 302 emulation to # correctly handle compiled files @@ -185,187 +169,6 @@ def _iter_file_finder_modules(importer, prefix=''): importlib.machinery.FileFinder, _iter_file_finder_modules) -def _import_imp(): - global imp - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - imp = importlib.import_module('imp') - -class ImpImporter: - """PEP 302 Finder that wraps Python's "classic" import algorithm - - ImpImporter(dirname) produces a PEP 302 finder that searches that - directory. ImpImporter(None) produces a PEP 302 finder that searches - the current sys.path, plus any modules that are frozen or built-in. - - Note that ImpImporter does not currently support being used by placement - on sys.meta_path. - """ - - def __init__(self, path=None): - global imp - warnings.warn("This emulation is deprecated and slated for removal " - "in Python 3.12; use 'importlib' instead", - DeprecationWarning) - _import_imp() - self.path = path - - def find_module(self, fullname, path=None): - # Note: we ignore 'path' argument since it is only used via meta_path - subname = fullname.split(".")[-1] - if subname != fullname and self.path is None: - return None - if self.path is None: - path = None - else: - path = [os.path.realpath(self.path)] - try: - file, filename, etc = imp.find_module(subname, path) - except ImportError: - return None - return ImpLoader(fullname, file, filename, etc) - - def iter_modules(self, prefix=''): - if self.path is None or not os.path.isdir(self.path): - return - - yielded = {} - import inspect - try: - filenames = os.listdir(self.path) - except OSError: - # ignore unreadable directories like import does - filenames = [] - filenames.sort() # handle packages before same-named modules - - for fn in filenames: - modname = inspect.getmodulename(fn) - if modname=='__init__' or modname in yielded: - continue - - path = os.path.join(self.path, fn) - ispkg = False - - if not modname and os.path.isdir(path) and '.' not in fn: - modname = fn - try: - dircontents = os.listdir(path) - except OSError: - # ignore unreadable directories like import does - dircontents = [] - for fn in dircontents: - subname = inspect.getmodulename(fn) - if subname=='__init__': - ispkg = True - break - else: - continue # not a package - - if modname and '.' not in modname: - yielded[modname] = 1 - yield prefix + modname, ispkg - - -class ImpLoader: - """PEP 302 Loader that wraps Python's "classic" import algorithm - """ - code = source = None - - def __init__(self, fullname, file, filename, etc): - warnings.warn("This emulation is deprecated and slated for removal in " - "Python 3.12; use 'importlib' instead", - DeprecationWarning) - _import_imp() - self.file = file - self.filename = filename - self.fullname = fullname - self.etc = etc - - def load_module(self, fullname): - self._reopen() - try: - mod = imp.load_module(fullname, self.file, self.filename, self.etc) - finally: - if self.file: - self.file.close() - # Note: we don't set __loader__ because we want the module to look - # normal; i.e. this is just a wrapper for standard import machinery - return mod - - def get_data(self, pathname): - with open(pathname, "rb") as file: - return file.read() - - def _reopen(self): - if self.file and self.file.closed: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self.file = open(self.filename, 'r') - elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): - self.file = open(self.filename, 'rb') - - def _fix_name(self, fullname): - if fullname is None: - fullname = self.fullname - elif fullname != self.fullname: - raise ImportError("Loader for module %s cannot handle " - "module %s" % (self.fullname, fullname)) - return fullname - - def is_package(self, fullname): - fullname = self._fix_name(fullname) - return self.etc[2]==imp.PKG_DIRECTORY - - def get_code(self, fullname=None): - fullname = self._fix_name(fullname) - if self.code is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - source = self.get_source(fullname) - self.code = compile(source, self.filename, 'exec') - elif mod_type==imp.PY_COMPILED: - self._reopen() - try: - self.code = read_code(self.file) - finally: - self.file.close() - elif mod_type==imp.PKG_DIRECTORY: - self.code = self._get_delegate().get_code() - return self.code - - def get_source(self, fullname=None): - fullname = self._fix_name(fullname) - if self.source is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self._reopen() - try: - self.source = self.file.read() - finally: - self.file.close() - elif mod_type==imp.PY_COMPILED: - if os.path.exists(self.filename[:-1]): - with open(self.filename[:-1], 'r') as f: - self.source = f.read() - elif mod_type==imp.PKG_DIRECTORY: - self.source = self._get_delegate().get_source() - return self.source - - def _get_delegate(self): - finder = ImpImporter(self.filename) - spec = _get_spec(finder, '__init__') - return spec.loader - - def get_filename(self, fullname=None): - fullname = self._fix_name(fullname) - mod_type = self.etc[2] - if mod_type==imp.PKG_DIRECTORY: - return self._get_delegate().get_filename() - elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): - return self.filename - return None - - try: import zipimport from zipimport import zipimporter @@ -413,6 +216,7 @@ def get_importer(path_item): The cache (or part of it) can be cleared manually if a rescan of sys.path_hooks is necessary. """ + path_item = os.fsdecode(path_item) try: importer = sys.path_importer_cache[path_item] except KeyError: @@ -457,51 +261,6 @@ def iter_importers(fullname=""): yield get_importer(item) -def get_loader(module_or_name): - """Get a "loader" object for module_or_name - - Returns None if the module cannot be found or imported. - If the named module is not already imported, its containing package - (if any) is imported, in order to establish the package __path__. - """ - if module_or_name in sys.modules: - module_or_name = sys.modules[module_or_name] - if module_or_name is None: - return None - if isinstance(module_or_name, ModuleType): - module = module_or_name - loader = getattr(module, '__loader__', None) - if loader is not None: - return loader - if getattr(module, '__spec__', None) is None: - return None - fullname = module.__name__ - else: - fullname = module_or_name - return find_loader(fullname) - - -def find_loader(fullname): - """Find a "loader" object for fullname - - This is a backwards compatibility wrapper around - importlib.util.find_spec that converts most failures to ImportError - and only returns the loader rather than the full spec - """ - if fullname.startswith('.'): - msg = "Relative module name {!r} not supported".format(fullname) - raise ImportError(msg) - try: - spec = importlib.util.find_spec(fullname) - except (ImportError, AttributeError, TypeError, ValueError) as ex: - # This hack fixes an impedance mismatch between pkgutil and - # importlib, where the latter raises other errors for cases where - # pkgutil previously raised ImportError - msg = "Error while finding loader for {!r} ({}: {})" - raise ImportError(msg.format(fullname, type(ex), ex)) from ex - return spec.loader if spec is not None else None - - def extend_path(path, name): """Extend a package's path. @@ -510,10 +269,10 @@ def extend_path(path, name): from pkgutil import extend_path __path__ = extend_path(__path__, __name__) - This will add to the package's __path__ all subdirectories of - directories on sys.path named after the package. This is useful - if one wants to distribute different parts of a single logical - package as multiple directories. + For each directory on sys.path that has a subdirectory that + matches the package name, add the subdirectory to the package's + __path__. This is useful if one wants to distribute different + parts of a single logical package as multiple directories. It also looks for *.pkg files beginning where * matches the name argument. This feature is similar to *.pth files (see site.py), diff --git a/Lib/platform.py b/Lib/platform.py index 58b66078e1a..1a533688a94 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -10,7 +10,8 @@ """ # This module is maintained by Marc-Andre Lemburg . # If you find problems, please submit bug reports/patches via the -# Python bug tracker (http://bugs.python.org) and assign them to "lemburg". +# Python issue tracker (https://github.com/python/cpython/issues) and +# mention "@malemburg". # # Still needed: # * support for MS-DOS (PythonDX ?) @@ -118,6 +119,10 @@ import sys import functools import itertools +try: + import _wmi +except ImportError: + _wmi = None ### Globals & Constants @@ -136,11 +141,11 @@ 'pl': 200, 'p': 200, } -_component_re = re.compile(r'([0-9]+|[._+-])') def _comparable_version(version): + component_re = re.compile(r'([0-9]+|[._+-])') result = [] - for v in _component_re.split(version): + for v in component_re.split(version): if v not in '._+-': try: v = int(v, 10) @@ -152,11 +157,6 @@ def _comparable_version(version): ### Platform specific APIs -_libc_search = re.compile(b'(__libc_init)' - b'|' - b'(GLIBC_([0-9.]+))' - b'|' - br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) def libc_ver(executable=None, lib='', version='', chunksize=16384): @@ -190,6 +190,12 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): # sys.executable is not set. return lib, version + libc_search = re.compile(b'(__libc_init)' + b'|' + b'(GLIBC_([0-9.]+))' + b'|' + br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) + V = _comparable_version # We use os.path.realpath() # here to work around problems with Cygwin not being @@ -200,7 +206,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): pos = 0 while pos < len(binary): if b'libc' in binary or b'GLIBC' in binary: - m = _libc_search.search(binary, pos) + m = libc_search.search(binary, pos) else: m = None if not m or m.end() == len(binary): @@ -247,9 +253,6 @@ def _norm_version(version, build=''): version = '.'.join(strings[:3]) return version -_ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' - r'.*' - r'\[.* ([\d.]+)\])') # Examples of VER command output: # @@ -295,9 +298,13 @@ def _syscmd_ver(system='', release='', version='', else: return system, release, version + ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' + r'.*' + r'\[.* ([\d.]+)\])') + # Parse the output info = info.strip() - m = _ver_output.match(info) + m = ver_output.match(info) if m is not None: system, release, version = m.groups() # Strip trailing dots from version and release @@ -310,44 +317,62 @@ def _syscmd_ver(system='', release='', version='', version = _norm_version(version) return system, release, version -_WIN32_CLIENT_RELEASES = { - (5, 0): "2000", - (5, 1): "XP", - # Strictly, 5.2 client is XP 64-bit, but platform.py historically - # has always called it 2003 Server - (5, 2): "2003Server", - (5, None): "post2003", - - (6, 0): "Vista", - (6, 1): "7", - (6, 2): "8", - (6, 3): "8.1", - (6, None): "post8.1", - - (10, 0): "10", - (10, None): "post10", -} - -# Server release name lookup will default to client names if necessary -_WIN32_SERVER_RELEASES = { - (5, 2): "2003Server", - (6, 0): "2008Server", - (6, 1): "2008ServerR2", - (6, 2): "2012Server", - (6, 3): "2012ServerR2", - (6, None): "post2012ServerR2", -} +def _wmi_query(table, *keys): + global _wmi + if not _wmi: + raise OSError("not supported") + table = { + "OS": "Win32_OperatingSystem", + "CPU": "Win32_Processor", + }[table] + try: + data = _wmi.exec_query("SELECT {} FROM {}".format( + ",".join(keys), + table, + )).split("\0") + except OSError: + _wmi = None + raise OSError("not supported") + split_data = (i.partition("=") for i in data) + dict_data = {i[0]: i[2] for i in split_data} + return (dict_data[k] for k in keys) + + +_WIN32_CLIENT_RELEASES = [ + ((10, 1, 0), "post11"), + ((10, 0, 22000), "11"), + ((6, 4, 0), "10"), + ((6, 3, 0), "8.1"), + ((6, 2, 0), "8"), + ((6, 1, 0), "7"), + ((6, 0, 0), "Vista"), + ((5, 2, 3790), "XP64"), + ((5, 2, 0), "XPMedia"), + ((5, 1, 0), "XP"), + ((5, 0, 0), "2000"), +] + +_WIN32_SERVER_RELEASES = [ + ((10, 1, 0), "post2025Server"), + ((10, 0, 26100), "2025Server"), + ((10, 0, 20348), "2022Server"), + ((10, 0, 17763), "2019Server"), + ((6, 4, 0), "2016Server"), + ((6, 3, 0), "2012ServerR2"), + ((6, 2, 0), "2012Server"), + ((6, 1, 0), "2008ServerR2"), + ((6, 0, 0), "2008Server"), + ((5, 2, 0), "2003Server"), + ((5, 0, 0), "2000Server"), +] def win32_is_iot(): return win32_edition() in ('IoTUAP', 'NanoServer', 'WindowsCoreHeadless', 'IoTEdgeOS') def win32_edition(): try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -360,22 +385,40 @@ def win32_edition(): return None -def win32_ver(release='', version='', csd='', ptype=''): +def _win32_ver(version, csd, ptype): + # Try using WMI first, as this is the canonical source of data + try: + (version, product_type, ptype, spmajor, spminor) = _wmi_query( + 'OS', + 'Version', + 'ProductType', + 'BuildType', + 'ServicePackMajorVersion', + 'ServicePackMinorVersion', + ) + is_client = (int(product_type) == 1) + if spminor and spminor != '0': + csd = f'SP{spmajor}.{spminor}' + else: + csd = f'SP{spmajor}' + return version, csd, ptype, is_client + except OSError: + pass + + # Fall back to a combination of sys.getwindowsversion and "ver" try: from sys import getwindowsversion except ImportError: - return release, version, csd, ptype + return version, csd, ptype, True winver = getwindowsversion() + is_client = (getattr(winver, 'product_type', 1) == 1) try: - major, minor, build = map(int, _syscmd_ver()[2].split('.')) + version = _syscmd_ver()[2] + major, minor, build = map(int, version.split('.')) except ValueError: major, minor, build = winver.platform_version or winver[:3] - version = '{0}.{1}.{2}'.format(major, minor, build) - - release = (_WIN32_CLIENT_RELEASES.get((major, minor)) or - _WIN32_CLIENT_RELEASES.get((major, None)) or - release) + version = '{0}.{1}.{2}'.format(major, minor, build) # getwindowsversion() reflect the compatibility mode Python is # running under, and so the service pack value is only going to be @@ -387,17 +430,8 @@ def win32_ver(release='', version='', csd='', ptype=''): if csd[:13] == 'Service Pack ': csd = 'SP' + csd[13:] - # VER_NT_SERVER = 3 - if getattr(winver, 'product_type', None) == 3: - release = (_WIN32_SERVER_RELEASES.get((major, minor)) or - _WIN32_SERVER_RELEASES.get((major, None)) or - release) - try: - try: - import winreg - except ImportError: - import _winreg as winreg + import winreg except ImportError: pass else: @@ -408,6 +442,18 @@ def win32_ver(release='', version='', csd='', ptype=''): except OSError: pass + return version, csd, ptype, is_client + +def win32_ver(release='', version='', csd='', ptype=''): + is_client = False + + version, csd, ptype, is_client = _win32_ver(version, csd, ptype) + + if version: + intversion = tuple(map(int, version.split('.'))) + releases = _WIN32_CLIENT_RELEASES if is_client else _WIN32_SERVER_RELEASES + release = next((r for v, r in releases if v <= intversion), release) + return release, version, csd, ptype @@ -452,8 +498,32 @@ def mac_ver(release='', versioninfo=('', '', ''), machine=''): # If that also doesn't work return the default values return release, versioninfo, machine -def _java_getprop(name, default): +# A namedtuple for iOS version information. +IOSVersionInfo = collections.namedtuple( + "IOSVersionInfo", + ["system", "release", "model", "is_simulator"] +) + + +def ios_ver(system="", release="", model="", is_simulator=False): + """Get iOS version information, and return it as a namedtuple: + (system, release, model, is_simulator). + + If values can't be determined, they are set to values provided as + parameters. + """ + if sys.platform == "ios": + import _ios_support + result = _ios_support.get_platform_ios() + if result is not None: + return IOSVersionInfo(*result) + + return IOSVersionInfo(system, release, model, is_simulator) + + +def _java_getprop(name, default): + """This private helper is deprecated in 3.13 and will be removed in 3.15""" from java.lang import System try: value = System.getProperty(name) @@ -475,6 +545,8 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): given as parameters (which all default to ''). """ + import warnings + warnings._deprecated('java_ver', remove=(3, 15)) # Import the needed APIs try: import java.lang @@ -496,6 +568,47 @@ def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')): return release, vendor, vminfo, osinfo + +AndroidVer = collections.namedtuple( + "AndroidVer", "release api_level manufacturer model device is_emulator") + +def android_ver(release="", api_level=0, manufacturer="", model="", device="", + is_emulator=False): + if sys.platform == "android": + try: + from ctypes import CDLL, c_char_p, create_string_buffer + except ImportError: + pass + else: + # An NDK developer confirmed that this is an officially-supported + # API (https://stackoverflow.com/a/28416743). Use `getattr` to avoid + # private name mangling. + system_property_get = getattr(CDLL("libc.so"), "__system_property_get") + system_property_get.argtypes = (c_char_p, c_char_p) + + def getprop(name, default): + # https://android.googlesource.com/platform/bionic/+/refs/tags/android-5.0.0_r1/libc/include/sys/system_properties.h#39 + PROP_VALUE_MAX = 92 + buffer = create_string_buffer(PROP_VALUE_MAX) + length = system_property_get(name.encode("UTF-8"), buffer) + if length == 0: + # This API doesn’t distinguish between an empty property and + # a missing one. + return default + else: + return buffer.value.decode("UTF-8", "backslashreplace") + + release = getprop("ro.build.version.release", release) + api_level = int(getprop("ro.build.version.sdk", api_level)) + manufacturer = getprop("ro.product.manufacturer", manufacturer) + model = getprop("ro.product.model", model) + device = getprop("ro.product.device", device) + is_emulator = getprop("ro.kernel.qemu", "0") == "1" + + return AndroidVer( + release, api_level, manufacturer, model, device, is_emulator) + + ### System name aliasing def system_alias(system, release, version): @@ -562,12 +675,12 @@ def _platform(*args): platform = platform.replace('unknown', '') # Fold '--'s and remove trailing '-' - while 1: + while True: cleaned = platform.replace('--', '-') if cleaned == platform: break platform = cleaned - while platform[-1] == '-': + while platform and platform[-1] == '-': platform = platform[:-1] return platform @@ -608,7 +721,7 @@ def _syscmd_file(target, default=''): default in case the command should fail. """ - if sys.platform in ('dos', 'win32', 'win16'): + if sys.platform in {'dos', 'win32', 'win16', 'ios', 'tvos', 'watchos'}: # XXX Others too ? return default @@ -702,6 +815,8 @@ def architecture(executable=sys.executable, bits='', linkage=''): # Linkage if 'ELF' in fileout: linkage = 'ELF' + elif 'Mach-O' in fileout: + linkage = "Mach-O" elif 'PE' in fileout: # E.g. Windows uses this format if 'Windows' in fileout: @@ -726,6 +841,21 @@ def _get_machine_win32(): # http://www.geocities.com/rick_lively/MANUALS/ENV/MSWIN/PROCESSI.HTM # WOW64 processes mask the native architecture + try: + [arch, *_] = _wmi_query('CPU', 'Architecture') + except OSError: + pass + else: + try: + arch = ['x86', 'MIPS', 'Alpha', 'PowerPC', None, + 'ARM', 'ia64', None, None, + 'AMD64', None, None, 'ARM64', + ][int(arch)] + except (ValueError, IndexError): + pass + else: + if arch: + return arch return ( os.environ.get('PROCESSOR_ARCHITEW6432', '') or os.environ.get('PROCESSOR_ARCHITECTURE', '') @@ -739,7 +869,12 @@ def get(cls): return func() or '' def get_win32(): - return os.environ.get('PROCESSOR_IDENTIFIER', _get_machine_win32()) + try: + manufacturer, caption = _wmi_query('CPU', 'Manufacturer', 'Caption') + except OSError: + return os.environ.get('PROCESSOR_IDENTIFIER', _get_machine_win32()) + else: + return f'{caption}, {manufacturer}' def get_OpenVMS(): try: @@ -750,6 +885,14 @@ def get_OpenVMS(): csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0) return 'Alpha' if cpu_number >= 128 else 'VAX' + # On the iOS simulator, os.uname returns the architecture as uname.machine. + # On device it returns the model name for some reason; but there's only one + # CPU architecture for iOS devices, so we know the right answer. + def get_ios(): + if sys.implementation._multiarch.endswith("simulator"): + return os.uname().machine + return 'arm64' + def from_subprocess(): """ Fall back to `uname -p` @@ -904,6 +1047,15 @@ def uname(): system = 'Windows' release = 'Vista' + # On Android, return the name and version of the OS rather than the kernel. + if sys.platform == 'android': + system = 'Android' + release = android_ver().release + + # Normalize responses on iOS + if sys.platform == 'ios': + system, release, _, _ = ios_ver() + vals = system, node, release, version, machine # Replace 'unknown' values with the more portable '' _uname_cache = uname_result(*map(_unknown_as_blank, vals)) @@ -971,32 +1123,6 @@ def processor(): ### Various APIs for extracting information from sys.version -_sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - -_ironpython_sys_version_parser = re.compile( - r'IronPython\s*' - r'([\d\.]+)' - r'(?: \(([\d\.]+)\))?' - r' on (.NET [\d\.]+)', re.ASCII) - -# IronPython covering 2.6 and 2.7 -_ironpython26_sys_version_parser = re.compile( - r'([\d.]+)\s*' - r'\(IronPython\s*' - r'[\d.]+\s*' - r'\(([\d.]+)\) on ([\w.]+ [\d.]+(?: \(\d+-bit\))?)\)' -) - -_pypy_sys_version_parser = re.compile( - r'([\w.+]+)\s*' - r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' - r'\[PyPy [^\]]+\]?') - _sys_version_cache = {} def _sys_version(sys_version=None): @@ -1028,28 +1154,16 @@ def _sys_version(sys_version=None): if result is not None: return result - # Parse it - if 'IronPython' in sys_version: - # IronPython - name = 'IronPython' - if sys_version.startswith('IronPython'): - match = _ironpython_sys_version_parser.match(sys_version) - else: - match = _ironpython26_sys_version_parser.match(sys_version) - - if match is None: - raise ValueError( - 'failed to parse IronPython sys.version: %s' % - repr(sys_version)) - - version, alt_version, compiler = match.groups() - buildno = '' - builddate = '' - - elif sys.platform.startswith('java'): + if sys.platform.startswith('java'): # Jython + jython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" name = 'Jython' - match = _sys_version_parser.match(sys_version) + match = jython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1061,8 +1175,13 @@ def _sys_version(sys_version=None): elif "PyPy" in sys_version: # PyPy + pypy_sys_version_parser = re.compile( + r'([\w.+]+)\s*' + r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' + r'\[PyPy [^\]]+\]?') + name = "PyPy" - match = _pypy_sys_version_parser.match(sys_version) + match = pypy_sys_version_parser.match(sys_version) if match is None: raise ValueError("failed to parse PyPy sys.version: %s" % repr(sys_version)) @@ -1071,7 +1190,14 @@ def _sys_version(sys_version=None): else: # CPython - match = _sys_version_parser.match(sys_version) + cpython_sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version" + r'(?:experimental free-threading build\s+)?' # "free-threading-build" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + match = cpython_sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % @@ -1080,11 +1206,10 @@ def _sys_version(sys_version=None): match.groups() # XXX: RUSTPYTHON support - if "rustc" in sys_version: + if "RustPython" in sys_version: name = "RustPython" else: name = 'CPython' - if builddate is None: builddate = '' elif buildtime: @@ -1115,7 +1240,6 @@ def python_implementation(): Currently, the following implementations are identified: 'CPython' (C implementation of Python), - 'IronPython' (.NET implementation of Python), 'Jython' (Java implementation of Python), 'PyPy' (Python implementation of Python). @@ -1190,7 +1314,7 @@ def python_compiler(): _platform_cache = {} -def platform(aliased=0, terse=0): +def platform(aliased=False, terse=False): """ Returns a single string identifying the underlying platform with as much useful information as possible (but no more :). @@ -1222,11 +1346,14 @@ def platform(aliased=0, terse=0): system, release, version = system_alias(system, release, version) if system == 'Darwin': - # macOS (darwin kernel) - macos_release = mac_ver()[0] - if macos_release: - system = 'macOS' - release = macos_release + # macOS and iOS both report as a "Darwin" kernel + if sys.platform == "ios": + system, release, _, _ = ios_ver() + else: + macos_release = mac_ver()[0] + if macos_release: + system = 'macOS' + release = macos_release if system == 'Windows': # MS platforms @@ -1236,7 +1363,7 @@ def platform(aliased=0, terse=0): else: platform = _platform(system, release, version, csd) - elif system in ('Linux',): + elif system == 'Linux': # check for libc vs. glibc libcname, libcversion = libc_ver() platform = _platform(system, release, machine, processor, @@ -1267,13 +1394,6 @@ def platform(aliased=0, terse=0): ### freedesktop.org os-release standard # https://www.freedesktop.org/software/systemd/man/os-release.html -# NAME=value with optional quotes (' or "). The regular expression is less -# strict than shell lexer, but that's ok. -_os_release_line = re.compile( - "^(?P[a-zA-Z0-9_]+)=(?P[\"\']?)(?P.*)(?P=quote)$" -) -# unescape five special characters mentioned in the standard -_os_release_unescape = re.compile(r"\\([\\\$\"\'`])") # /etc takes precedence over /usr/lib _os_release_candidates = ("/etc/os-release", "/usr/lib/os-release") _os_release_cache = None @@ -1288,10 +1408,18 @@ def _parse_os_release(lines): "PRETTY_NAME": "Linux", } + # NAME=value with optional quotes (' or "). The regular expression is less + # strict than shell lexer, but that's ok. + os_release_line = re.compile( + "^(?P[a-zA-Z0-9_]+)=(?P[\"\']?)(?P.*)(?P=quote)$" + ) + # unescape five special characters mentioned in the standard + os_release_unescape = re.compile(r"\\([\\\$\"\'`])") + for line in lines: - mo = _os_release_line.match(line) + mo = os_release_line.match(line) if mo is not None: - info[mo.group('name')] = _os_release_unescape.sub( + info[mo.group('name')] = os_release_unescape.sub( r"\1", mo.group('value') ) diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 2eeebe4c9a4..655c51eea3d 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -21,6 +21,9 @@ Generate Plist example: + import datetime + import plistlib + pl = dict( aString = "Doodah", aList = ["A", "B", 12, 32.1, [1, 2, 3]], @@ -28,22 +31,28 @@ anInt = 728, aDict = dict( anotherString = "", - aUnicodeValue = "M\xe4ssig, Ma\xdf", + aThirdString = "M\xe4ssig, Ma\xdf", aTrueValue = True, aFalseValue = False, ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), + aDate = datetime.datetime.now() ) - with open(fileName, 'wb') as fp: - dump(pl, fp) + print(plistlib.dumps(pl).decode()) Parse Plist example: - with open(fileName, 'rb') as fp: - pl = load(fp) - print(pl["aKey"]) + import plistlib + + plist = b''' + + foo + bar + + ''' + pl = plistlib.loads(plist) + print(pl["foo"]) """ __all__ = [ "InvalidFileException", "FMT_XML", "FMT_BINARY", "load", "dump", "loads", "dumps", "UID" @@ -64,6 +73,9 @@ PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) globals().update(PlistFormat.__members__) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 class UID: def __init__(self, data): @@ -131,7 +143,7 @@ def _decode_base64(s): _dateParser = re.compile(r"(?P\d\d\d\d)(?:-(?P\d\d)(?:-(?P\d\d)(?:T(?P\d\d)(?::(?P\d\d)(?::(?P\d\d))?)?)?)?)?Z", re.ASCII) -def _date_from_string(s): +def _date_from_string(s, aware_datetime): order = ('year', 'month', 'day', 'hour', 'minute', 'second') gd = _dateParser.match(s).groupdict() lst = [] @@ -140,10 +152,14 @@ def _date_from_string(s): if val is None: break lst.append(int(val)) + if aware_datetime: + return datetime.datetime(*lst, tzinfo=datetime.UTC) return datetime.datetime(*lst) -def _date_to_string(d): +def _date_to_string(d, aware_datetime): + if aware_datetime: + d = d.astimezone(datetime.UTC) return '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( d.year, d.month, d.day, d.hour, d.minute, d.second @@ -152,7 +168,7 @@ def _date_to_string(d): def _escape(text): m = _controlCharPat.search(text) if m is not None: - raise ValueError("strings can't contains control characters; " + raise ValueError("strings can't contain control characters; " "use bytes instead") text = text.replace("\r\n", "\n") # convert DOS line endings text = text.replace("\r", "\n") # convert Mac line endings @@ -162,11 +178,12 @@ def _escape(text): return text class _PlistParser: - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self.stack = [] self.current_key = None self.root = None self._dict_type = dict_type + self._aware_datetime = aware_datetime def parse(self, fileobj): self.parser = ParserCreate() @@ -178,8 +195,8 @@ def parse(self, fileobj): return self.root def handle_entity_decl(self, entity_name, is_parameter_entity, value, base, system_id, public_id, notation_name): - # Reject plist files with entity declarations to avoid XML vulnerabilies in expat. - # Regular plist files don't contain those declerations, and Apple's plutil tool does not + # Reject plist files with entity declarations to avoid XML vulnerabilities in expat. + # Regular plist files don't contain those declarations, and Apple's plutil tool does not # accept them either. raise InvalidFileException("XML entity declarations are not supported in plist files") @@ -199,7 +216,7 @@ def handle_data(self, data): def add_object(self, value): if self.current_key is not None: - if not isinstance(self.stack[-1], type({})): + if not isinstance(self.stack[-1], dict): raise ValueError("unexpected element at line %d" % self.parser.CurrentLineNumber) self.stack[-1][self.current_key] = value @@ -208,7 +225,7 @@ def add_object(self, value): # this is the root object self.root = value else: - if not isinstance(self.stack[-1], type([])): + if not isinstance(self.stack[-1], list): raise ValueError("unexpected element at line %d" % self.parser.CurrentLineNumber) self.stack[-1].append(value) @@ -232,7 +249,7 @@ def end_dict(self): self.stack.pop() def end_key(self): - if self.current_key or not isinstance(self.stack[-1], type({})): + if self.current_key or not isinstance(self.stack[-1], dict): raise ValueError("unexpected key at line %d" % self.parser.CurrentLineNumber) self.current_key = self.get_data() @@ -268,7 +285,8 @@ def end_data(self): self.add_object(_decode_base64(self.get_data())) def end_date(self): - self.add_object(_date_from_string(self.get_data())) + self.add_object(_date_from_string(self.get_data(), + aware_datetime=self._aware_datetime)) class _DumbXMLWriter: @@ -312,13 +330,14 @@ def writeln(self, line): class _PlistWriter(_DumbXMLWriter): def __init__( self, file, indent_level=0, indent=b"\t", writeHeader=1, - sort_keys=True, skipkeys=False): + sort_keys=True, skipkeys=False, aware_datetime=False): if writeHeader: file.write(PLISTHEADER) _DumbXMLWriter.__init__(self, file, indent_level, indent) self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): self.writeln("") @@ -351,7 +370,8 @@ def write_value(self, value): self.write_bytes(value) elif isinstance(value, datetime.datetime): - self.simple_element("date", _date_to_string(value)) + self.simple_element("date", + _date_to_string(value, self._aware_datetime)) elif isinstance(value, (tuple, list)): self.write_array(value) @@ -452,8 +472,9 @@ class _BinaryPlistParser: see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c """ - def __init__(self, dict_type): + def __init__(self, dict_type, aware_datetime=False): self._dict_type = dict_type + self._aware_datime = aware_datetime def parse(self, fp): try: @@ -490,12 +511,24 @@ def _get_size(self, tokenL): return tokenL + def _read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + data = self._fp.read(cursize) + while True: + if len(data) != cursize: + raise InvalidFileException + if cursize == size: + return data + delta = min(cursize, size - cursize) + data += self._fp.read(delta) + cursize += delta + def _read_ints(self, n, size): - data = self._fp.read(size * n) + data = self._read(size * n) if size in _BINARY_FORMAT: return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: - if not size or len(data) != size * n: + if not size: raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) @@ -547,27 +580,24 @@ def _read_object(self, ref): f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - result = (datetime.datetime(2001, 1, 1) + - datetime.timedelta(seconds=f)) + if self._aware_datime: + epoch = datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + else: + epoch = datetime.datetime(2001, 1, 1) + result = epoch + datetime.timedelta(seconds=f) elif tokenH == 0x40: # data s = self._get_size(tokenL) - result = self._fp.read(s) - if len(result) != s: - raise InvalidFileException() + result = self._read(s) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('ascii') elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) * 2 - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('utf-16be') elif tokenH == 0x80: # UID @@ -579,7 +609,8 @@ def _read_object(self, ref): obj_refs = self._read_refs(s) result = [] self._objects[ref] = result - result.extend(self._read_object(x) for x in obj_refs) + for x in obj_refs: + result.append(self._read_object(x)) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. @@ -620,10 +651,11 @@ def _count_to_size(count): _scalars = (str, int, float, datetime.datetime, bytes) class _BinaryPlistWriter (object): - def __init__(self, fp, sort_keys, skipkeys): + def __init__(self, fp, sort_keys, skipkeys, aware_datetime=False): self._fp = fp self._sort_keys = sort_keys self._skipkeys = skipkeys + self._aware_datetime = aware_datetime def write(self, value): @@ -769,7 +801,12 @@ def _write_object(self, value): self._fp.write(struct.pack('>Bd', 0x23, value)) elif isinstance(value, datetime.datetime): - f = (value - datetime.datetime(2001, 1, 1)).total_seconds() + if self._aware_datetime: + dt = value.astimezone(datetime.UTC) + offset = dt - datetime.datetime(2001, 1, 1, tzinfo=datetime.UTC) + f = offset.total_seconds() + else: + f = (value - datetime.datetime(2001, 1, 1)).total_seconds() self._fp.write(struct.pack('>Bd', 0x33, f)) elif isinstance(value, (bytes, bytearray)): @@ -853,7 +890,7 @@ def _is_fmt_binary(header): } -def load(fp, *, fmt=None, dict_type=dict): +def load(fp, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file. 'fp' should be a readable and binary file object. Return the unpacked root object (which usually is a dictionary). """ @@ -871,32 +908,41 @@ def load(fp, *, fmt=None, dict_type=dict): else: P = _FORMATS[fmt]['parser'] - p = P(dict_type=dict_type) + p = P(dict_type=dict_type, aware_datetime=aware_datetime) return p.parse(fp) -def loads(value, *, fmt=None, dict_type=dict): +def loads(value, *, fmt=None, dict_type=dict, aware_datetime=False): """Read a .plist file from a bytes object. Return the unpacked root object (which usually is a dictionary). """ + if isinstance(value, str): + if fmt == FMT_BINARY: + raise TypeError("value must be bytes-like object when fmt is " + "FMT_BINARY") + value = value.encode() fp = BytesIO(value) - return load(fp, fmt=fmt, dict_type=dict_type) + return load(fp, fmt=fmt, dict_type=dict_type, aware_datetime=aware_datetime) -def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False): +def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False, + aware_datetime=False): """Write 'value' to a .plist file. 'fp' should be a writable, binary file object. """ if fmt not in _FORMATS: raise ValueError("Unsupported format: %r"%(fmt,)) - writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys) + writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys, + aware_datetime=aware_datetime) writer.write(value) -def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True): +def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True, + aware_datetime=False): """Return a bytes object with the contents for a .plist file. """ fp = BytesIO() - dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) + dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys, + aware_datetime=aware_datetime) return fp.getvalue() diff --git a/Lib/poplib.py b/Lib/poplib.py new file mode 100644 index 00000000000..4469bff44b4 --- /dev/null +++ b/Lib/poplib.py @@ -0,0 +1,477 @@ +"""A POP3 client class. + +Based on the J. Myers POP3 draft, Jan. 96 +""" + +# Author: David Ascher +# [heavily stealing from nntplib.py] +# Updated: Piers Lauder [Jul '97] +# String method conversion and test jig improvements by ESR, February 2001. +# Added the POP3_SSL class. Methods loosely based on IMAP_SSL. Hector Urtubia Aug 2003 + +# Example (see the test function at the end of this file) + +# Imports + +import errno +import re +import socket +import sys + +try: + import ssl + HAVE_SSL = True +except ImportError: + HAVE_SSL = False + +__all__ = ["POP3","error_proto"] + +# Exception raised when an error or invalid response is received: + +class error_proto(Exception): pass + +# Standard Port +POP3_PORT = 110 + +# POP SSL PORT +POP3_SSL_PORT = 995 + +# Line terminators (we always output CRLF, but accept any of CRLF, LFCR, LF) +CR = b'\r' +LF = b'\n' +CRLF = CR+LF + +# maximal line length when calling readline(). This is to prevent +# reading arbitrary length lines. RFC 1939 limits POP3 line length to +# 512 characters, including CRLF. We have selected 2048 just to be on +# the safe side. +_MAXLINE = 2048 + + +class POP3: + + """This class supports both the minimal and optional command sets. + Arguments can be strings or integers (where appropriate) + (e.g.: retr(1) and retr('1') both work equally well. + + Minimal Command Set: + USER name user(name) + PASS string pass_(string) + STAT stat() + LIST [msg] list(msg = None) + RETR msg retr(msg) + DELE msg dele(msg) + NOOP noop() + RSET rset() + QUIT quit() + + Optional Commands (some servers support these): + RPOP name rpop(name) + APOP name digest apop(name, digest) + TOP msg n top(msg, n) + UIDL [msg] uidl(msg = None) + CAPA capa() + STLS stls() + UTF8 utf8() + + Raises one exception: 'error_proto'. + + Instantiate with: + POP3(hostname, port=110) + + NB: the POP protocol locks the mailbox from user + authorization until QUIT, so be sure to get in, suck + the messages, and quit, each time you access the + mailbox. + + POP is a line-based protocol, which means large mail + messages consume lots of python cycles reading them + line-by-line. + + If it's available on your mail server, use IMAP4 + instead, it doesn't suffer from the two problems + above. + """ + + encoding = 'UTF-8' + + def __init__(self, host, port=POP3_PORT, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.host = host + self.port = port + self._tls_established = False + sys.audit("poplib.connect", self, host, port) + self.sock = self._create_socket(timeout) + self.file = self.sock.makefile('rb') + self._debugging = 0 + self.welcome = self._getresp() + + def _create_socket(self, timeout): + if timeout is not None and not timeout: + raise ValueError('Non-blocking socket (timeout=0) is not supported') + return socket.create_connection((self.host, self.port), timeout) + + def _putline(self, line): + if self._debugging > 1: print('*put*', repr(line)) + sys.audit("poplib.putline", self, line) + self.sock.sendall(line + CRLF) + + + # Internal: send one command to the server (through _putline()) + + def _putcmd(self, line): + if self._debugging: print('*cmd*', repr(line)) + line = bytes(line, self.encoding) + self._putline(line) + + + # Internal: return one line from the server, stripping CRLF. + # This is where all the CPU time of this module is consumed. + # Raise error_proto('-ERR EOF') if the connection is closed. + + def _getline(self): + line = self.file.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise error_proto('line too long') + + if self._debugging > 1: print('*get*', repr(line)) + if not line: raise error_proto('-ERR EOF') + octets = len(line) + # server can send any combination of CR & LF + # however, 'readline()' returns lines ending in LF + # so only possibilities are ...LF, ...CRLF, CR...LF + if line[-2:] == CRLF: + return line[:-2], octets + if line[:1] == CR: + return line[1:-1], octets + return line[:-1], octets + + + # Internal: get a response from the server. + # Raise 'error_proto' if the response doesn't start with '+'. + + def _getresp(self): + resp, o = self._getline() + if self._debugging > 1: print('*resp*', repr(resp)) + if not resp.startswith(b'+'): + raise error_proto(resp) + return resp + + + # Internal: get a response plus following text from the server. + + def _getlongresp(self): + resp = self._getresp() + list = []; octets = 0 + line, o = self._getline() + while line != b'.': + if line.startswith(b'..'): + o = o-1 + line = line[1:] + octets = octets + o + list.append(line) + line, o = self._getline() + return resp, list, octets + + + # Internal: send a command and get the response + + def _shortcmd(self, line): + self._putcmd(line) + return self._getresp() + + + # Internal: send a command and get the response plus following text + + def _longcmd(self, line): + self._putcmd(line) + return self._getlongresp() + + + # These can be useful: + + def getwelcome(self): + return self.welcome + + + def set_debuglevel(self, level): + self._debugging = level + + + # Here are all the POP commands: + + def user(self, user): + """Send user name, return response + + (should indicate password required). + """ + return self._shortcmd('USER %s' % user) + + + def pass_(self, pswd): + """Send password, return response + + (response includes message count, mailbox size). + + NB: mailbox is locked by server from here to 'quit()' + """ + return self._shortcmd('PASS %s' % pswd) + + + def stat(self): + """Get mailbox status. + + Result is tuple of 2 ints (message count, mailbox size) + """ + retval = self._shortcmd('STAT') + rets = retval.split() + if self._debugging: print('*stat*', repr(rets)) + + # Check if the response has enough elements + # RFC 1939 requires at least 3 elements (+OK, message count, mailbox size) + # but allows additional data after the required fields + if len(rets) < 3: + raise error_proto("Invalid STAT response format") + + try: + numMessages = int(rets[1]) + sizeMessages = int(rets[2]) + except ValueError: + raise error_proto("Invalid STAT response data: non-numeric values") + + return (numMessages, sizeMessages) + + + def list(self, which=None): + """Request listing, return result. + + Result without a message number argument is in form + ['response', ['mesg_num octets', ...], octets]. + + Result when a message number argument is given is a + single response: the "scan listing" for that message. + """ + if which is not None: + return self._shortcmd('LIST %s' % which) + return self._longcmd('LIST') + + + def retr(self, which): + """Retrieve whole message number 'which'. + + Result is in form ['response', ['line', ...], octets]. + """ + return self._longcmd('RETR %s' % which) + + + def dele(self, which): + """Delete message number 'which'. + + Result is 'response'. + """ + return self._shortcmd('DELE %s' % which) + + + def noop(self): + """Does nothing. + + One supposes the response indicates the server is alive. + """ + return self._shortcmd('NOOP') + + + def rset(self): + """Unmark all messages marked for deletion.""" + return self._shortcmd('RSET') + + + def quit(self): + """Signoff: commit changes on server, unlock mailbox, close connection.""" + resp = self._shortcmd('QUIT') + self.close() + return resp + + def close(self): + """Close the connection without assuming anything about it.""" + try: + file = self.file + self.file = None + if file is not None: + file.close() + finally: + sock = self.sock + self.sock = None + if sock is not None: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError as exc: + # The server might already have closed the connection. + # On Windows, this may result in WSAEINVAL (error 10022): + # An invalid operation was attempted. + if (exc.errno != errno.ENOTCONN + and getattr(exc, 'winerror', 0) != 10022): + raise + finally: + sock.close() + + #__del__ = quit + + + # optional commands: + + def rpop(self, user): + """Send RPOP command to access the mailbox with an alternate user.""" + return self._shortcmd('RPOP %s' % user) + + + timestamp = re.compile(br'\+OK.[^<]*(<.*>)') + + def apop(self, user, password): + """Authorisation + + - only possible if server has supplied a timestamp in initial greeting. + + Args: + user - mailbox user; + password - mailbox password. + + NB: mailbox is locked by server from here to 'quit()' + """ + secret = bytes(password, self.encoding) + m = self.timestamp.match(self.welcome) + if not m: + raise error_proto('-ERR APOP not supported by server') + import hashlib + digest = m.group(1)+secret + digest = hashlib.md5(digest).hexdigest() + return self._shortcmd('APOP %s %s' % (user, digest)) + + + def top(self, which, howmuch): + """Retrieve message header of message number 'which' + and first 'howmuch' lines of message body. + + Result is in form ['response', ['line', ...], octets]. + """ + return self._longcmd('TOP %s %s' % (which, howmuch)) + + + def uidl(self, which=None): + """Return message digest (unique id) list. + + If 'which', result contains unique id for that message + in the form 'response mesgnum uid', otherwise result is + the list ['response', ['mesgnum uid', ...], octets] + """ + if which is not None: + return self._shortcmd('UIDL %s' % which) + return self._longcmd('UIDL') + + + def utf8(self): + """Try to enter UTF-8 mode (see RFC 6856). Returns server response. + """ + return self._shortcmd('UTF8') + + + def capa(self): + """Return server capabilities (RFC 2449) as a dictionary + >>> c=poplib.POP3('localhost') + >>> c.capa() + {'IMPLEMENTATION': ['Cyrus', 'POP3', 'server', 'v2.2.12'], + 'TOP': [], 'LOGIN-DELAY': ['0'], 'AUTH-RESP-CODE': [], + 'EXPIRE': ['NEVER'], 'USER': [], 'STLS': [], 'PIPELINING': [], + 'UIDL': [], 'RESP-CODES': []} + >>> + + Really, according to RFC 2449, the cyrus folks should avoid + having the implementation split into multiple arguments... + """ + def _parsecap(line): + lst = line.decode('ascii').split() + return lst[0], lst[1:] + + caps = {} + try: + resp = self._longcmd('CAPA') + rawcaps = resp[1] + for capline in rawcaps: + capnm, capargs = _parsecap(capline) + caps[capnm] = capargs + except error_proto: + raise error_proto('-ERR CAPA not supported by server') + return caps + + + def stls(self, context=None): + """Start a TLS session on the active connection as specified in RFC 2595. + + context - a ssl.SSLContext + """ + if not HAVE_SSL: + raise error_proto('-ERR TLS support missing') + if self._tls_established: + raise error_proto('-ERR TLS session already established') + caps = self.capa() + if not 'STLS' in caps: + raise error_proto('-ERR STLS not supported by server') + if context is None: + context = ssl._create_stdlib_context() + resp = self._shortcmd('STLS') + self.sock = context.wrap_socket(self.sock, + server_hostname=self.host) + self.file = self.sock.makefile('rb') + self._tls_established = True + return resp + + +if HAVE_SSL: + + class POP3_SSL(POP3): + """POP3 client class over SSL connection + + Instantiate with: POP3_SSL(hostname, port=995, context=None) + + hostname - the hostname of the pop3 over ssl server + port - port number + context - a ssl.SSLContext + + See the methods of the parent class POP3 for more documentation. + """ + + def __init__(self, host, port=POP3_SSL_PORT, + *, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, context=None): + if context is None: + context = ssl._create_stdlib_context() + self.context = context + POP3.__init__(self, host, port, timeout) + + def _create_socket(self, timeout): + sock = POP3._create_socket(self, timeout) + sock = self.context.wrap_socket(sock, + server_hostname=self.host) + return sock + + def stls(self, context=None): + """The method unconditionally raises an exception since the + STLS command doesn't make any sense on an already established + SSL/TLS session. + """ + raise error_proto('-ERR TLS session already established') + + __all__.append("POP3_SSL") + +if __name__ == "__main__": + a = POP3(sys.argv[1]) + print(a.getwelcome()) + a.user(sys.argv[2]) + a.pass_(sys.argv[3]) + a.list() + (numMsgs, totalSize) = a.stat() + for i in range(1, numMsgs + 1): + (header, msg, octets) = a.retr(i) + print("Message %d:" % i) + for line in msg: + print(' ' + line) + print('-----------------------') + a.quit() diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 354d7d82d0c..ad86cc06c01 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -22,23 +22,21 @@ altsep = None devnull = '/dev/null' -try: - import os -except ImportError: - import _dummy_os as os +import errno +import os import sys import stat import genericpath from genericpath import * -__all__ = ["normcase","isabs","join","splitdrive","split","splitext", +__all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime","islink","exists","lexists","isdir","isfile", "ismount", "expanduser","expandvars","normpath","abspath", "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", - "commonpath"] + "commonpath", "isjunction","isdevdrive","ALLOW_MISSING"] def _get_sep(path): @@ -80,12 +78,11 @@ def join(a, *p): sep = _get_sep(a) path = a try: - if not p: - path[:0] + sep #23780: Ensure compatible data type even if p is null. - for b in map(os.fspath, p): - if b.startswith(sep): + for b in p: + b = os.fspath(b) + if b.startswith(sep) or not path: path = b - elif not path or path.endswith(sep): + elif path.endswith(sep): path += b else: path += sep + b @@ -138,6 +135,32 @@ def splitdrive(p): return p[:0], p +try: + from posix import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. + + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'/' + empty = b'' + else: + sep = '/' + empty = '' + if p[:1] != sep: + # Relative path, e.g.: 'foo' + return empty, empty, p + elif p[1:2] != sep or p[2:3] == sep: + # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + return empty, sep, p[1:] + else: + # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + return empty, p[:2], p[2:] + + # Return the tail (basename) part of a path, same as split(path)[1]. def basename(p): @@ -161,28 +184,6 @@ def dirname(p): return head -# Is a path a symbolic link? -# This will always return false on systems where os.lstat doesn't exist. - -def islink(path): - """Test whether a path is a symbolic link""" - try: - st = os.lstat(path) - except (OSError, ValueError, AttributeError): - return False - return stat.S_ISLNK(st.st_mode) - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - os.lstat(path) - except (OSError, ValueError): - return False - return True - - # Is a path a mount point? # (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) @@ -198,25 +199,22 @@ def ismount(path): if stat.S_ISLNK(s1.st_mode): return False + path = os.fspath(path) if isinstance(path, bytes): parent = join(path, b'..') else: parent = join(path, '..') - parent = realpath(parent) try: s2 = os.lstat(parent) - except (OSError, ValueError): - return False + except OSError: + parent = realpath(parent) + try: + s2 = os.lstat(parent) + except OSError: + return False - dev1 = s1.st_dev - dev2 = s2.st_dev - if dev1 != dev2: - return True # path/.. on a different device as path - ino1 = s1.st_ino - ino2 = s2.st_ino - if ino1 == ino2: - return True # path/.. is the same i-node as path - return False + # path/.. on a different device as path or the same i-node as path + return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino # Expand paths beginning with '~' or '~user'. @@ -244,7 +242,11 @@ def expanduser(path): i = len(path) if i == 1: if 'HOME' not in os.environ: - import pwd + try: + import pwd + except ImportError: + # pwd module unavailable, return path unchanged + return path try: userhome = pwd.getpwuid(os.getuid()).pw_dir except KeyError: @@ -254,10 +256,14 @@ def expanduser(path): else: userhome = os.environ['HOME'] else: - import pwd + try: + import pwd + except ImportError: + # pwd module unavailable, return path unchanged + return path name = path[1:i] if isinstance(name, bytes): - name = str(name, 'ASCII') + name = os.fsdecode(name) try: pwent = pwd.getpwnam(name) except KeyError: @@ -270,53 +276,49 @@ def expanduser(path): return path if isinstance(path, bytes): userhome = os.fsencode(userhome) - root = b'/' - else: - root = '/' - userhome = userhome.rstrip(root) - return (userhome + path[i:]) or root + userhome = userhome.rstrip(sep) + return (userhome + path[i:]) or sep # Expand paths containing shell variable substitutions. # This expands the forms $variable and ${variable} only. # Non-existent variables are left unchanged. -_varprog = None -_varprogb = None +_varpattern = r'\$(\w+|\{[^}]*\}?)' +_varsub = None +_varsubb = None def expandvars(path): """Expand shell variables of form $var and ${var}. Unknown variables are left unchanged.""" path = os.fspath(path) - global _varprog, _varprogb + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path: return path - if not _varprogb: + if not _varsubb: import re - _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprogb.search + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb start = b'{' end = b'}' environ = getattr(os, 'environb', None) else: if '$' not in path: return path - if not _varprog: + if not _varsub: import re - _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprog.search + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub start = '{' end = '}' environ = os.environ - i = 0 - while True: - m = search(path, i) - if not m: - break - i, j = m.span(0) - name = m.group(1) - if name.startswith(start) and name.endswith(end): + + def repl(m): + name = m[1] + if name.startswith(start): + if not name.endswith(end): + return m[0] name = name[1:-1] try: if environ is None: @@ -324,67 +326,59 @@ def expandvars(path): else: value = environ[name] except KeyError: - i = j + return m[0] else: - tail = path[j:] - path = path[:i] + value - i = len(path) - path += tail - return path + return value + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. # It should be understood that this may change the meaning of the path # if it contains symbolic links! -def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - sep = b'/' - empty = b'' - dot = b'.' - dotdot = b'..' - else: - sep = '/' - empty = '' - dot = '.' - dotdot = '..' - if path == empty: - return dot - initial_slashes = path.startswith(sep) - # POSIX allows one or two initial slashes, but treats three or more - # as single slash. - # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13) - if (initial_slashes and - path.startswith(sep*2) and not path.startswith(sep*3)): - initial_slashes = 2 - comps = path.split(sep) - new_comps = [] - for comp in comps: - if comp in (empty, dot): - continue - if (comp != dotdot or (not initial_slashes and not new_comps) or - (new_comps and new_comps[-1] == dotdot)): - new_comps.append(comp) - elif new_comps: - new_comps.pop() - comps = new_comps - path = sep.join(comps) - if initial_slashes: - path = sep*initial_slashes + path - return path or dot +try: + from posix import _path_normpath as normpath + +except ImportError: + def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'/' + dot = b'.' + dotdot = b'..' + else: + sep = '/' + dot = '.' + dotdot = '..' + if not path: + return dot + _, initial_slashes, path = splitroot(path) + comps = path.split(sep) + new_comps = [] + for comp in comps: + if not comp or comp == dot: + continue + if (comp != dotdot or (not initial_slashes and not new_comps) or + (new_comps and new_comps[-1] == dotdot)): + new_comps.append(comp) + elif new_comps: + new_comps.pop() + comps = new_comps + path = initial_slashes + sep.join(comps) + return path or dot def abspath(path): """Return an absolute path.""" path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) return normpath(path) @@ -395,72 +389,123 @@ def realpath(filename, *, strict=False): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) - path, ok = _joinrealpath(filename[:0], filename, strict, {}) - return abspath(path) - -# Join two paths, normalizing and eliminating any symbolic links -# encountered in the second path. -def _joinrealpath(path, rest, strict, seen): - if isinstance(path, bytes): + if isinstance(filename, bytes): sep = b'/' curdir = b'.' pardir = b'..' + getcwd = os.getcwdb else: sep = '/' curdir = '.' pardir = '..' - - if isabs(rest): - rest = rest[1:] - path = sep - - while rest: - name, _, rest = rest.partition(sep) + getcwd = os.getcwd + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + strict = True + elif strict: + ignored_error = () + else: + ignored_error = OSError + + lstat = os.lstat + readlink = os.readlink + maxlinks = None + + # The stack of unresolved path parts. When popped, a special value of None + # indicates that a symlink target has been resolved, and that the original + # symlink path can be retrieved by popping again. The [::-1] slice is a + # very fast way of spelling list(reversed(...)). + rest = filename.split(sep)[::-1] + + # Number of unprocessed parts in 'rest'. This can differ from len(rest) + # later, because 'rest' might contain markers for unresolved symlinks. + part_count = len(rest) + + # The resolved path, which is absolute throughout this function. + # Note: getcwd() returns a normalized and symlink-free path. + path = sep if filename.startswith(sep) else getcwd() + + # Mapping from symlink paths to *fully resolved* symlink targets. If a + # symlink is encountered but not yet resolved, the value is None. This is + # used both to detect symlink loops and to speed up repeated traversals of + # the same links. + seen = {} + + # Number of symlinks traversed. When the number of traversals is limited + # by *maxlinks*, this is used instead of *seen* to detect symlink loops. + link_count = 0 + + while part_count: + name = rest.pop() + if name is None: + # resolved symlink target + seen[rest.pop()] = path + continue + part_count -= 1 if not name or name == curdir: # current dir continue if name == pardir: # parent dir - if path: - path, name = split(path) - if name == pardir: - path = join(path, pardir, pardir) - else: - path = pardir + path = path[:path.rindex(sep)] or sep continue - newpath = join(path, name) + if path == sep: + newpath = path + name + else: + newpath = path + sep + name try: - st = os.lstat(newpath) - except OSError: - if strict: - raise - is_link = False + st_mode = lstat(newpath).st_mode + if not stat.S_ISLNK(st_mode): + if strict and part_count and not stat.S_ISDIR(st_mode): + raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR), + newpath) + path = newpath + continue + elif maxlinks is not None: + link_count += 1 + if link_count > maxlinks: + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + elif newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + raise OSError(errno.ELOOP, os.strerror(errno.ELOOP), + newpath) + path = newpath + continue + target = readlink(newpath) + except ignored_error: + pass else: - is_link = stat.S_ISLNK(st.st_mode) - if not is_link: - path = newpath + # Resolve the symbolic link + if target.startswith(sep): + # Symlink target is absolute; reset resolved path. + path = sep + if maxlinks is None: + # Mark this symlink as seen but not fully resolved. + seen[newpath] = None + # Push the symlink path onto the stack, and signal its specialness + # by also pushing None. When these entries are popped, we'll + # record the fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) + # Push the unresolved symlink target parts onto the stack. + target_parts = target.split(sep)[::-1] + rest.extend(target_parts) + part_count += len(target_parts) continue - # Resolve the symbolic link - if newpath in seen: - # Already seen this path - path = seen[newpath] - if path is not None: - # use cached value - continue - # The symlink is not resolved, so we must have a symlink loop. - if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) - else: - # Return already resolved part + rest of the path unchanged. - return join(newpath, rest), False - seen[newpath] = None # not resolved symlink - path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen) - if not ok: - return join(path, rest), False - seen[newpath] = path # resolved symlink + # An error occurred and was ignored. + path = newpath - return path, True + return path supports_unicode_filenames = (sys.platform == 'darwin') @@ -468,10 +513,10 @@ def _joinrealpath(path, rest, strict, seen): def relpath(path, start=None): """Return a relative version of a path""" + path = os.fspath(path) if not path: raise ValueError("no path specified") - path = os.fspath(path) if isinstance(path, bytes): curdir = b'.' sep = b'/' @@ -487,15 +532,17 @@ def relpath(path, start=None): start = os.fspath(start) try: - start_list = [x for x in abspath(start).split(sep) if x] - path_list = [x for x in abspath(path).split(sep) if x] + start_tail = abspath(start).lstrip(sep) + path_tail = abspath(path).lstrip(sep) + start_list = start_tail.split(sep) if start_tail else [] + path_list = path_tail.split(sep) if path_tail else [] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise @@ -509,10 +556,11 @@ def relpath(path, start=None): def commonpath(paths): """Given a sequence of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) + if not paths: raise ValueError('commonpath() arg is an empty sequence') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'/' curdir = b'.' @@ -524,7 +572,7 @@ def commonpath(paths): split_paths = [path.split(sep) for path in paths] try: - isabs, = set(p[:1] == sep for p in paths) + isabs, = {p.startswith(sep) for p in paths} except ValueError: raise ValueError("Can't mix absolute and relative paths") from None diff --git a/Lib/pprint.py b/Lib/pprint.py index 575688d8eb6..dc0953cec67 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -35,8 +35,6 @@ """ import collections as _collections -import dataclasses as _dataclasses -import re import sys as _sys import types as _types from io import StringIO as _StringIO @@ -54,6 +52,7 @@ def pprint(object, stream=None, indent=1, width=80, depth=None, *, underscore_numbers=underscore_numbers) printer.pprint(object) + def pformat(object, indent=1, width=80, depth=None, *, compact=False, sort_dicts=True, underscore_numbers=False): """Format a Python object into a pretty-printed representation.""" @@ -61,22 +60,27 @@ def pformat(object, indent=1, width=80, depth=None, *, compact=compact, sort_dicts=sort_dicts, underscore_numbers=underscore_numbers).pformat(object) + def pp(object, *args, sort_dicts=False, **kwargs): """Pretty-print a Python object""" pprint(object, *args, sort_dicts=sort_dicts, **kwargs) + def saferepr(object): """Version of repr() which can handle recursive data structures.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[0] + def isreadable(object): """Determine if saferepr(object) is readable by eval().""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[1] + def isrecursive(object): """Determine if object requires a recursive representation.""" return PrettyPrinter()._safe_repr(object, {}, None, 0)[2] + class _safe_key: """Helper function for key functions when sorting unorderable objects. @@ -99,10 +103,12 @@ def __lt__(self, other): return ((str(type(self.obj)), id(self.obj)) < \ (str(type(other.obj)), id(other.obj))) + def _safe_tuple(t): "Helper function for comparing 2-tuples" return _safe_key(t[0]), _safe_key(t[1]) + class PrettyPrinter: def __init__(self, indent=1, width=80, depth=None, stream=None, *, compact=False, sort_dicts=True, underscore_numbers=False): @@ -128,6 +134,9 @@ def __init__(self, indent=1, width=80, depth=None, stream=None, *, sort_dicts If true, dict keys are sorted. + underscore_numbers + If true, digit groups are separated with underscores. + """ indent = int(indent) width = int(width) @@ -176,12 +185,15 @@ def _format(self, object, stream, indent, allowance, context, level): max_width = self._width - indent - allowance if len(rep) > max_width: p = self._dispatch.get(type(object).__repr__, None) + # Lazy import to improve module import time + from dataclasses import is_dataclass + if p is not None: context[objid] = 1 p(self, object, stream, indent, allowance, context, level + 1) del context[objid] return - elif (_dataclasses.is_dataclass(object) and + elif (is_dataclass(object) and not isinstance(object, type) and object.__dataclass_params__.repr and # Check dataclass has generated repr method. @@ -194,9 +206,12 @@ def _format(self, object, stream, indent, allowance, context, level): stream.write(rep) def _pprint_dataclass(self, object, stream, indent, allowance, context, level): + # Lazy import to improve module import time + from dataclasses import fields as dataclass_fields + cls_name = object.__class__.__name__ indent += len(cls_name) + 1 - items = [(f.name, getattr(object, f.name)) for f in _dataclasses.fields(object) if f.repr] + items = [(f.name, getattr(object, f.name)) for f in dataclass_fields(object) if f.repr] stream.write(cls_name + '(') self._format_namespace_items(items, stream, indent, allowance, context, level) stream.write(')') @@ -288,6 +303,9 @@ def _pprint_str(self, object, stream, indent, allowance, context, level): if len(rep) <= max_width1: chunks.append(rep) else: + # Lazy import to improve module import time + import re + # A list of alternating (non-space, space) strings parts = re.findall(r'\S*\s*', line) assert parts @@ -629,27 +647,16 @@ def _safe_repr(self, object, context, maxlevels, level): rep = repr(object) return rep, (rep and not rep.startswith('<')), False + _builtin_scalars = frozenset({str, bytes, bytearray, float, complex, bool, type(None)}) + def _recursion(object): return ("" % (type(object).__name__, id(object))) -def _perfcheck(object=None): - import time - if object is None: - object = [("string", (1, 2), [3, 4], {5: 6, 7: 8})] * 100000 - p = PrettyPrinter() - t1 = time.perf_counter() - p._safe_repr(object, {}, None, 0, True) - t2 = time.perf_counter() - p.pformat(object) - t3 = time.perf_counter() - print("_safe_repr:", t2 - t1) - print("pformat:", t3 - t2) - def _wrap_bytes_repr(object, width, allowance): current = b'' last = len(object) // 4 * 4 @@ -666,6 +673,3 @@ def _wrap_bytes_repr(object, width, allowance): current = candidate if current: yield repr(current) - -if __name__ == "__main__": - _perfcheck() diff --git a/Lib/pty.py b/Lib/pty.py index 8d8ce40df54..1d97994abef 100644 --- a/Lib/pty.py +++ b/Lib/pty.py @@ -40,6 +40,9 @@ def master_open(): Open a pty master and return the fd, and the filename of the slave end. Deprecated, use openpty() instead.""" + import warnings + warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 + try: master_fd, slave_fd = os.openpty() except (AttributeError, OSError): @@ -69,6 +72,9 @@ def slave_open(tty_name): opened filedescriptor. Deprecated, use openpty() instead.""" + import warnings + warnings.warn("Use pty.openpty() instead.", DeprecationWarning, stacklevel=2) # Remove API in 3.14 + result = os.open(tty_name, os.O_RDWR) try: from fcntl import ioctl, I_PUSH @@ -101,32 +107,14 @@ def fork(): master_fd, slave_fd = openpty() pid = os.fork() if pid == CHILD: - # Establish a new session. - os.setsid() os.close(master_fd) - - # Slave becomes stdin/stdout/stderr of child. - os.dup2(slave_fd, STDIN_FILENO) - os.dup2(slave_fd, STDOUT_FILENO) - os.dup2(slave_fd, STDERR_FILENO) - if slave_fd > STDERR_FILENO: - os.close(slave_fd) - - # Explicitly open the tty to make it become a controlling tty. - tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR) - os.close(tmp_fd) + os.login_tty(slave_fd) else: os.close(slave_fd) # Parent and child process. return pid, master_fd -def _writen(fd, data): - """Write all the data to a descriptor.""" - while data: - n = os.write(fd, data) - data = data[n:] - def _read(fd): """Default read function.""" return os.read(fd, 1024) @@ -136,9 +124,42 @@ def _copy(master_fd, master_read=_read, stdin_read=_read): Copies pty master -> standard output (master_read) standard input -> pty master (stdin_read)""" - fds = [master_fd, STDIN_FILENO] - while fds: - rfds, _wfds, _xfds = select(fds, [], []) + if os.get_blocking(master_fd): + # If we write more than tty/ndisc is willing to buffer, we may block + # indefinitely. So we set master_fd to non-blocking temporarily during + # the copy operation. + os.set_blocking(master_fd, False) + try: + _copy(master_fd, master_read=master_read, stdin_read=stdin_read) + finally: + # restore blocking mode for backwards compatibility + os.set_blocking(master_fd, True) + return + high_waterlevel = 4096 + stdin_avail = master_fd != STDIN_FILENO + stdout_avail = master_fd != STDOUT_FILENO + i_buf = b'' + o_buf = b'' + while 1: + rfds = [] + wfds = [] + if stdin_avail and len(i_buf) < high_waterlevel: + rfds.append(STDIN_FILENO) + if stdout_avail and len(o_buf) < high_waterlevel: + rfds.append(master_fd) + if stdout_avail and len(o_buf) > 0: + wfds.append(STDOUT_FILENO) + if len(i_buf) > 0: + wfds.append(master_fd) + + rfds, wfds, _xfds = select(rfds, wfds, []) + + if STDOUT_FILENO in wfds: + try: + n = os.write(STDOUT_FILENO, o_buf) + o_buf = o_buf[n:] + except OSError: + stdout_avail = False if master_fd in rfds: # Some OSes signal EOF by returning an empty byte string, @@ -150,19 +171,22 @@ def _copy(master_fd, master_read=_read, stdin_read=_read): if not data: # Reached EOF. return # Assume the child process has exited and is # unreachable, so we clean up. - else: - os.write(STDOUT_FILENO, data) + o_buf += data + + if master_fd in wfds: + n = os.write(master_fd, i_buf) + i_buf = i_buf[n:] - if STDIN_FILENO in rfds: + if stdin_avail and STDIN_FILENO in rfds: data = stdin_read(STDIN_FILENO) if not data: - fds.remove(STDIN_FILENO) + stdin_avail = False else: - _writen(master_fd, data) + i_buf += data def spawn(argv, master_read=_read, stdin_read=_read): """Create a spawned process.""" - if type(argv) == type(''): + if isinstance(argv, str): argv = (argv,) sys.audit('pty.spawn', argv) diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 388614e51b1..43d8ec90ffb 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -177,7 +177,7 @@ def main(): import argparse description = 'A simple command-line interface for py_compile module.' - parser = argparse.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description, color=True) parser.add_argument( '-q', '--quiet', action='store_true', diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py new file mode 100644 index 00000000000..37f86995d6c --- /dev/null +++ b/Lib/pyclbr.py @@ -0,0 +1,314 @@ +"""Parse a Python module and describe its classes and functions. + +Parse enough of a Python file to recognize imports and class and +function definitions, and to find out the superclasses of a class. + +The interface consists of a single function: + readmodule_ex(module, path=None) +where module is the name of a Python module, and path is an optional +list of directories where the module is to be searched. If present, +path is prepended to the system search path sys.path. The return value +is a dictionary. The keys of the dictionary are the names of the +classes and functions defined in the module (including classes that are +defined via the from XXX import YYY construct). The values are +instances of classes Class and Function. One special key/value pair is +present for packages: the key '__path__' has a list as its value which +contains the package search path. + +Classes and Functions have a common superclass: _Object. Every instance +has the following attributes: + module -- name of the module; + name -- name of the object; + file -- file in which the object is defined; + lineno -- line in the file where the object's definition starts; + end_lineno -- line in the file where the object's definition ends; + parent -- parent of this object, if any; + children -- nested objects contained in this object. +The 'children' attribute is a dictionary mapping names to objects. + +Instances of Function describe functions with the attributes from _Object, +plus the following: + is_async -- if a function is defined with an 'async' prefix + +Instances of Class describe classes with the attributes from _Object, +plus the following: + super -- list of super classes (Class instances if possible); + methods -- mapping of method names to beginning line numbers. +If the name of a super class is not recognized, the corresponding +entry in the list of super classes is not a class instance but a +string giving the name of the super class. Since import statements +are recognized and imported modules are scanned as well, this +shouldn't happen often. +""" + +import ast +import sys +import importlib.util + +__all__ = ["readmodule", "readmodule_ex", "Class", "Function"] + +_modules = {} # Initialize cache of modules we've seen. + + +class _Object: + "Information about Python class or function." + def __init__(self, module, name, file, lineno, end_lineno, parent): + self.module = module + self.name = name + self.file = file + self.lineno = lineno + self.end_lineno = end_lineno + self.parent = parent + self.children = {} + if parent is not None: + parent.children[name] = self + + +# Odd Function and Class signatures are for back-compatibility. +class Function(_Object): + "Information about a Python function, including methods." + def __init__(self, module, name, file, lineno, + parent=None, is_async=False, *, end_lineno=None): + super().__init__(module, name, file, lineno, end_lineno, parent) + self.is_async = is_async + if isinstance(parent, Class): + parent.methods[name] = lineno + + +class Class(_Object): + "Information about a Python class." + def __init__(self, module, name, super_, file, lineno, + parent=None, *, end_lineno=None): + super().__init__(module, name, file, lineno, end_lineno, parent) + self.super = super_ or [] + self.methods = {} + + +# These 2 functions are used in these tests +# Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py +def _nest_function(ob, func_name, lineno, end_lineno, is_async=False): + "Return a Function after nesting within ob." + return Function(ob.module, func_name, ob.file, lineno, + parent=ob, is_async=is_async, end_lineno=end_lineno) + +def _nest_class(ob, class_name, lineno, end_lineno, super=None): + "Return a Class after nesting within ob." + return Class(ob.module, class_name, super, ob.file, lineno, + parent=ob, end_lineno=end_lineno) + + +def readmodule(module, path=None): + """Return Class objects for the top-level classes in module. + + This is the original interface, before Functions were added. + """ + + res = {} + for key, value in _readmodule(module, path or []).items(): + if isinstance(value, Class): + res[key] = value + return res + +def readmodule_ex(module, path=None): + """Return a dictionary with all functions and classes in module. + + Search for module in PATH + sys.path. + If possible, include imported superclasses. + Do this by reading source, without importing (and executing) it. + """ + return _readmodule(module, path or []) + + +def _readmodule(module, path, inpackage=None): + """Do the hard work for readmodule[_ex]. + + If inpackage is given, it must be the dotted name of the package in + which we are searching for a submodule, and then PATH must be the + package search path; otherwise, we are searching for a top-level + module, and path is combined with sys.path. + """ + # Compute the full module name (prepending inpackage if set). + if inpackage is not None: + fullmodule = "%s.%s" % (inpackage, module) + else: + fullmodule = module + + # Check in the cache. + if fullmodule in _modules: + return _modules[fullmodule] + + # Initialize the dict for this module's contents. + tree = {} + + # Check if it is a built-in module; we don't do much for these. + if module in sys.builtin_module_names and inpackage is None: + _modules[module] = tree + return tree + + # Check for a dotted module name. + i = module.rfind('.') + if i >= 0: + package = module[:i] + submodule = module[i+1:] + parent = _readmodule(package, path, inpackage) + if inpackage is not None: + package = "%s.%s" % (inpackage, package) + if not '__path__' in parent: + raise ImportError('No package named {}'.format(package)) + return _readmodule(submodule, parent['__path__'], package) + + # Search the path for the module. + f = None + if inpackage is not None: + search_path = path + else: + search_path = path + sys.path + spec = importlib.util._find_spec_from_path(fullmodule, search_path) + if spec is None: + raise ModuleNotFoundError(f"no module named {fullmodule!r}", name=fullmodule) + _modules[fullmodule] = tree + # Is module a package? + if spec.submodule_search_locations is not None: + tree['__path__'] = spec.submodule_search_locations + try: + source = spec.loader.get_source(fullmodule) + except (AttributeError, ImportError): + # If module is not Python source, we cannot do anything. + return tree + else: + if source is None: + return tree + + fname = spec.loader.get_filename(fullmodule) + return _create_tree(fullmodule, path, fname, source, tree, inpackage) + + +class _ModuleBrowser(ast.NodeVisitor): + def __init__(self, module, path, file, tree, inpackage): + self.path = path + self.tree = tree + self.file = file + self.module = module + self.inpackage = inpackage + self.stack = [] + + def visit_ClassDef(self, node): + bases = [] + for base in node.bases: + name = ast.unparse(base) + if name in self.tree: + # We know this super class. + bases.append(self.tree[name]) + elif len(names := name.split(".")) > 1: + # Super class form is module.class: + # look in module for class. + *_, module, class_ = names + if module in _modules: + bases.append(_modules[module].get(class_, name)) + else: + bases.append(name) + + parent = self.stack[-1] if self.stack else None + class_ = Class(self.module, node.name, bases, self.file, node.lineno, + parent=parent, end_lineno=node.end_lineno) + if parent is None: + self.tree[node.name] = class_ + self.stack.append(class_) + self.generic_visit(node) + self.stack.pop() + + def visit_FunctionDef(self, node, *, is_async=False): + parent = self.stack[-1] if self.stack else None + function = Function(self.module, node.name, self.file, node.lineno, + parent, is_async, end_lineno=node.end_lineno) + if parent is None: + self.tree[node.name] = function + self.stack.append(function) + self.generic_visit(node) + self.stack.pop() + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node, is_async=True) + + def visit_Import(self, node): + if node.col_offset != 0: + return + + for module in node.names: + try: + try: + _readmodule(module.name, self.path, self.inpackage) + except ImportError: + _readmodule(module.name, []) + except (ImportError, SyntaxError): + # If we can't find or parse the imported module, + # too bad -- don't die here. + continue + + def visit_ImportFrom(self, node): + if node.col_offset != 0: + return + try: + module = "." * node.level + if node.module: + module += node.module + module = _readmodule(module, self.path, self.inpackage) + except (ImportError, SyntaxError): + return + + for name in node.names: + if name.name in module: + self.tree[name.asname or name.name] = module[name.name] + elif name.name == "*": + for import_name, import_value in module.items(): + if import_name.startswith("_"): + continue + self.tree[import_name] = import_value + + +def _create_tree(fullmodule, path, fname, source, tree, inpackage): + mbrowser = _ModuleBrowser(fullmodule, path, fname, tree, inpackage) + mbrowser.visit(ast.parse(source)) + return mbrowser.tree + + +def _main(): + "Print module output (default this file) for quick visual check." + import os + try: + mod = sys.argv[1] + except: + mod = __file__ + if os.path.exists(mod): + path = [os.path.dirname(mod)] + mod = os.path.basename(mod) + if mod.lower().endswith(".py"): + mod = mod[:-3] + else: + path = [] + tree = readmodule_ex(mod, path) + lineno_key = lambda a: getattr(a, 'lineno', 0) + objs = sorted(tree.values(), key=lineno_key, reverse=True) + indent_level = 2 + while objs: + obj = objs.pop() + if isinstance(obj, list): + # Value is a __path__ key. + continue + if not hasattr(obj, 'indent'): + obj.indent = 0 + + if isinstance(obj, _Object): + new_objs = sorted(obj.children.values(), + key=lineno_key, reverse=True) + for ob in new_objs: + ob.indent = obj.indent + indent_level + objs.extend(new_objs) + if isinstance(obj, Class): + print("{}class {} {} {}" + .format(' ' * obj.indent, obj.name, obj.super, obj.lineno)) + elif isinstance(obj, Function): + print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno)) + +if __name__ == "__main__": + _main() diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b521a550472..1f8a6ef3d7c 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Generate Python documentation in HTML or text for interactive use. At the Python interactive prompt, calling help(thing) on a Python object @@ -16,12 +15,15 @@ class or function within a module or module in a package. If the Run "pydoc -k " to search for a keyword in the synopsis lines of all available modules. +Run "pydoc -n " to start an HTTP server with the given +hostname (default: localhost) on the local machine. + Run "pydoc -p " to start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. Run "pydoc -b" to start an HTTP server on an arbitrary unused port and -open a Web browser to interactively browse documentation. The -p option -can be used with the -b option to explicitly specify the server port. +open a web browser to interactively browse documentation. Combine with +the -n and -p options to control the hostname and port used. Run "pydoc -w " to write out the HTML documentation for a module to a file named ".html". @@ -51,6 +53,8 @@ class or function within a module or module in a package. If the # the current directory is changed with os.chdir(), an incorrect # path will be displayed. +import ast +import __future__ import builtins import importlib._bootstrap import importlib._bootstrap_external @@ -63,14 +67,32 @@ class or function within a module or module in a package. If the import platform import re import sys +import sysconfig +import textwrap import time import tokenize import urllib.parse import warnings +from annotationlib import Format from collections import deque from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import (get_pager, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + +# Expose plain() as pydoc.plain() +from _pyrepl.pager import plain # noqa: F401 + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines @@ -86,9 +108,100 @@ def pathdirs(): normdirs.append(normdir) return dirs +def _findclass(func): + cls = sys.modules.get(func.__module__) + if cls is None: + return None + for name in func.__qualname__.split('.')[:-1]: + cls = getattr(cls, name) + if not inspect.isclass(cls): + return None + return cls + +def _finddoc(obj): + if inspect.ismethod(obj): + name = obj.__func__.__name__ + self = obj.__self__ + if (inspect.isclass(self) and + getattr(getattr(self, name, None), '__func__') is obj.__func__): + # classmethod + cls = self + else: + cls = self.__class__ + elif inspect.isfunction(obj): + name = obj.__name__ + cls = _findclass(obj) + if cls is None or getattr(cls, name) is not obj: + return None + elif inspect.isbuiltin(obj): + name = obj.__name__ + self = obj.__self__ + if (inspect.isclass(self) and + self.__qualname__ + '.' + name == obj.__qualname__): + # classmethod + cls = self + else: + cls = self.__class__ + # Should be tested before isdatadescriptor(). + elif isinstance(obj, property): + name = obj.__name__ + cls = _findclass(obj.fget) + if cls is None or getattr(cls, name) is not obj: + return None + elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): + name = obj.__name__ + cls = obj.__objclass__ + if getattr(cls, name) is not obj: + return None + if inspect.ismemberdescriptor(obj): + slots = getattr(cls, '__slots__', None) + if isinstance(slots, dict) and name in slots: + return slots[name] + else: + return None + for base in cls.__mro__: + try: + doc = _getowndoc(getattr(base, name)) + except AttributeError: + continue + if doc is not None: + return doc + return None + +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def _getdoc(object): + """Get the documentation string for an object. + + All tabs are expanded to spaces. To clean up docstrings that are + indented to line up with blocks of code, any whitespace than can be + uniformly removed from the second line onwards is removed.""" + doc = _getowndoc(object) + if doc is None: + try: + doc = _finddoc(object) + except (AttributeError, TypeError): + return None + if not isinstance(doc, str): + return None + return inspect.cleandoc(doc) + def getdoc(object): """Get the doc string or comments for an object.""" - result = inspect.getdoc(object) or inspect.getcomments(object) + result = _getdoc(object) or inspect.getcomments(object) return result and re.sub('^ *\n', '', result.rstrip()) or '' def splitdoc(doc): @@ -100,6 +213,27 @@ def splitdoc(doc): return lines[0], '\n'.join(lines[2:]) return '', '\n'.join(lines) +def _getargspec(object): + try: + signature = inspect.signature(object, annotation_format=Format.STRING) + if signature: + name = getattr(object, '__name__', '') + # function are always single-line and should not be formatted + max_width = (80 - len(name)) if name != '' else None + return signature.format(max_width=max_width, quote_annotation_strings=False) + except (ValueError, TypeError): + argspec = getattr(object, '__text_signature__', None) + if argspec: + if argspec[:2] == '($': + argspec = '(' + argspec[2:] + if getattr(object, '__self__', None) is not None: + # Strip the bound argument. + m = re.match(r'\(\w+(?:(?=\))|,\s*(?:/(?:(?=\))|,\s*))?)', argspec) + if m: + argspec = '(' + argspec[m.end():] + return argspec + return None + def classname(object, modname): """Get a class name and qualify it with a module name if necessary.""" name = object.__name__ @@ -107,6 +241,19 @@ def classname(object, modname): name = object.__module__ + '.' + name return name +def parentname(object, modname): + """Get a name of the enclosing class (qualified it with a module name + if necessary) or module.""" + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname and object.__module__ is not None: + return object.__module__ + '.' + name + else: + return name + else: + if object.__module__ != modname: + return object.__module__ + def isdata(object): """Check if an object is of a type that probably means it's data.""" return not (inspect.ismodule(object) or inspect.isclass(object) or @@ -134,12 +281,6 @@ def stripid(text): # The behaviour of %p is implementation-dependent in terms of case. return _re_stripid.sub(r'\1', text) -def _is_some_method(obj): - return (inspect.isfunction(obj) or - inspect.ismethod(obj) or - inspect.isbuiltin(obj) or - inspect.ismethoddescriptor(obj)) - def _is_bound_method(fn): """ Returns True if fn is a bound method, regardless of whether @@ -155,7 +296,7 @@ def _is_bound_method(fn): def allmethods(cl): methods = {} - for key, value in inspect.getmembers(cl, _is_some_method): + for key, value in inspect.getmembers(cl, inspect.isroutine): methods[key] = 1 for base in cl.__bases__: methods.update(allmethods(base)) # all your base are belong to us @@ -180,6 +321,8 @@ def _split_list(s, predicate): no.append(x) return yes, no +_future_feature_names = set(__future__.all_feature_names) + def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant or internal. @@ -187,13 +330,19 @@ def visiblename(name, all=None, obj=None): if name in {'__author__', '__builtins__', '__cached__', '__credits__', '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', - '__path__', '__qualname__', '__slots__', '__version__'}: + '__path__', '__qualname__', '__slots__', '__version__', + '__static_attributes__', '__firstlineno__', + '__annotate_func__', '__annotations_cache__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 # Namedtuples have public fields and methods with a single leading underscore if name.startswith('_') and hasattr(obj, '_fields'): return True + # Ignore __future__ imports. + if obj is not __future__ and name in _future_feature_names: + if isinstance(getattr(obj, name, None), __future__._Feature): + return False if all is not None: # only document that which the programmer exported in __all__ return name in all @@ -201,11 +350,15 @@ def visiblename(name, all=None, obj=None): return not name.startswith('_') def classify_class_attrs(object): - """Wrap inspect.classify_class_attrs, with fixup for data descriptors.""" + """Wrap inspect.classify_class_attrs, with fixup for data descriptors and bound methods.""" results = [] for (name, kind, cls, value) in inspect.classify_class_attrs(object): if inspect.isdatadescriptor(value): kind = 'data descriptor' + if isinstance(value, property) and value.fset is None: + kind = 'readonly property' + elif kind == 'method' and _is_bound_method(value): + kind = 'static method' results.append((name, kind, cls, value)) return results @@ -225,6 +378,8 @@ def sort_attributes(attrs, object): def ispackage(path): """Guess whether a path refers to a package directory.""" + warnings.warn('The pydoc.ispackage() function is deprecated', + DeprecationWarning, stacklevel=2) if os.path.isdir(path): for ext in ('.py', '.pyc'): if os.path.isfile(os.path.join(path, '__init__' + ext)): @@ -232,21 +387,29 @@ def ispackage(path): return False def source_synopsis(file): - line = file.readline() - while line[:1] == '#' or not line.strip(): - line = file.readline() - if not line: break - line = line.strip() - if line[:4] == 'r"""': line = line[1:] - if line[:3] == '"""': - line = line[3:] - if line[-1:] == '\\': line = line[:-1] - while not line.strip(): - line = file.readline() - if not line: break - result = line.split('"""')[0].strip() - else: result = None - return result + """Return the one-line summary of a file object, if present""" + + string = '' + try: + tokens = tokenize.generate_tokens(file.readline) + for tok_type, tok_string, _, _, _ in tokens: + if tok_type == tokenize.STRING: + string += tok_string + elif tok_type == tokenize.NEWLINE: + with warnings.catch_warnings(): + # Ignore the "invalid escape sequence" warning. + warnings.simplefilter("ignore", SyntaxWarning) + docstring = ast.literal_eval(string) + if not isinstance(docstring, str): + return None + return docstring.strip().split('\n')[0].strip() + elif tok_type == tokenize.OP and tok_string in ('(', ')'): + string += tok_string + elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING): + return None + except (tokenize.TokenError, UnicodeDecodeError, SyntaxError): + return None + return None def synopsis(filename, cache={}): """Get the one-line summary out of a module file.""" @@ -290,8 +453,17 @@ def synopsis(filename, cache={}): class ErrorDuringImport(Exception): """Errors that occurred while trying to import something to document it.""" def __init__(self, filename, exc_info): + if not isinstance(exc_info, tuple): + assert isinstance(exc_info, BaseException) + self.exc = type(exc_info) + self.value = exc_info + self.tb = exc_info.__traceback__ + else: + warnings.warn("A tuple value for exc_info is deprecated, use an exception instance", + DeprecationWarning) + + self.exc, self.value, self.tb = exc_info self.filename = filename - self.exc, self.value, self.tb = exc_info def __str__(self): exc = self.exc.__name__ @@ -312,8 +484,8 @@ def importfile(path): spec = importlib.util.spec_from_file_location(name, path, loader=loader) try: return importlib._bootstrap._load(spec) - except: - raise ErrorDuringImport(path, sys.exc_info()) + except BaseException as err: + raise ErrorDuringImport(path, err) def safeimport(path, forceload=0, cache={}): """Import a module; handle errors; return None if the module isn't found. @@ -340,25 +512,21 @@ def safeimport(path, forceload=0, cache={}): # Prevent garbage collection. cache[key] = sys.modules[key] del sys.modules[key] - module = __import__(path) - except: + module = importlib.import_module(path) + except BaseException as err: # Did the error occur before or after the module was found? - (exc, value, tb) = info = sys.exc_info() if path in sys.modules: # An error occurred while executing the imported module. - raise ErrorDuringImport(sys.modules[path].__file__, info) - elif exc is SyntaxError: + raise ErrorDuringImport(sys.modules[path].__file__, err) + elif type(err) is SyntaxError: # A SyntaxError occurred before we could execute the module. - raise ErrorDuringImport(value.filename, info) - elif issubclass(exc, ImportError) and value.name == path: + raise ErrorDuringImport(err.filename, err) + elif isinstance(err, ImportError) and err.name == path: # No such module in the path. return None else: # Some other error occurred during the importing process. - raise ErrorDuringImport(path, sys.exc_info()) - for part in path.split('.')[1:]: - try: module = getattr(module, part) - except AttributeError: return None + raise ErrorDuringImport(path, err) return module # ---------------------------------------------------- formatter base class @@ -376,15 +544,13 @@ def document(self, object, name=None, *args): # identifies something in a way that pydoc itself has issues handling; # think 'super' and how it is a descriptor (which raises the exception # by lacking a __name__ attribute) and an instance. - if inspect.isgetsetdescriptor(object): return self.docdata(*args) - if inspect.ismemberdescriptor(object): return self.docdata(*args) try: if inspect.ismodule(object): return self.docmodule(*args) if inspect.isclass(object): return self.docclass(*args) if inspect.isroutine(object): return self.docroutine(*args) except AttributeError: pass - if isinstance(object, property): return self.docproperty(*args) + if inspect.isdatadescriptor(object): return self.docdata(*args) return self.docother(*args) def fail(self, object, name=None, *args): @@ -395,9 +561,7 @@ def fail(self, object, name=None, *args): docmodule = docclass = docroutine = docother = docproperty = docdata = fail - def getdocloc(self, object, - basedir=os.path.join(sys.base_exec_prefix, "lib", - "python%d.%d" % sys.version_info[:2])): + def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): """Return the location of module docs or None""" try: @@ -409,16 +573,26 @@ def getdocloc(self, object, basedir = os.path.normcase(basedir) if (isinstance(object, type(os)) and - (object.__name__ in ('errno', 'exceptions', 'gc', 'imp', + (object.__name__ in ('errno', 'exceptions', 'gc', 'marshal', 'posix', 'signal', 'sys', '_thread', 'zipimport') or (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and - object.__name__ not in ('xml.etree', 'test.pydoc_mod')): - if docloc.startswith(("http://", "https://")): - docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower()) + object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): + + try: + from pydoc_data import module_docs + except ImportError: + module_docs = None + + if module_docs and object.__name__ in module_docs.module_docs: + doc_name = module_docs.module_docs[object.__name__] + if docloc.startswith(("http://", "https://")): + docloc = "{}/{}".format(docloc.rstrip("/"), doc_name) + else: + docloc = os.path.join(docloc, doc_name) else: - docloc = os.path.join(docloc, object.__name__.lower() + ".html") + docloc = None else: docloc = None return docloc @@ -454,7 +628,7 @@ def repr_string(self, x, level): # needed to make any special characters, so show a raw string. return 'r' + testrepr[0] + self.escape(test) + testrepr[0] return re.sub(r'((\\[\\abfnrtv\'"]|\\[0-9]..|\\x..|\\u....)+)', - r'\1', + r'\1', self.escape(testrepr)) repr_str = repr_string @@ -479,49 +653,48 @@ class HTMLDoc(Doc): def page(self, title, contents): """Format an HTML page.""" return '''\ - -Python: %s - - + + + + +Python: %s + %s ''' % (title, contents) - def heading(self, title, fgcol, bgcol, extras=''): + def heading(self, title, extras=''): """Format a page heading.""" return ''' -
- -
 
- 
%s		%s
- ''' % (bgcol, fgcol, title, fgcol, extras or ' ') - - def section(self, title, fgcol, bgcol, contents, width=6, + + + +
 
%s		%s
+ ''' % (title, extras or ' ') + + def section(self, title, cls, contents, width=6, prelude='', marginalia=None, gap=' '): """Format a section with a heading.""" if marginalia is None: - marginalia = '' + ' ' * width + '' + marginalia = '' + ' ' * width + '' result = '''

- - - - ''' % (bgcol, fgcol, title) +
 
-%s
+ + + ''' % (cls, title) if prelude: result = result + ''' - - -''' % (bgcol, marginalia, prelude, gap) + + +''' % (cls, marginalia, cls, prelude, gap) else: result = result + ''' -''' % (bgcol, marginalia, gap) +''' % (cls, marginalia, gap) - return result + '\n
 
%s
%s%s
%s
%s%s
%s
%s%s
%s%s%s
' % contents + return result + '\n%s' % contents def bigsection(self, title, *args): """Format a section with a big heading.""" - title = '%s' % title + title = '%s' % title return self.section(title, *args) def preformat(self, text): @@ -530,19 +703,19 @@ def preformat(self, text): return replace(text, '\n\n', '\n \n', '\n\n', '\n \n', ' ', ' ', '\n', '
\n') - def multicolumn(self, list, format, cols=4): + def multicolumn(self, list, format): """Format a list of items into a multi-column list.""" result = '' - rows = (len(list)+cols-1)//cols - for col in range(cols): - result = result + '' % (100//cols) + rows = (len(list) + 3) // 4 + for col in range(4): + result = result + '' for i in range(rows*col, rows*col+rows): if i < len(list): result = result + format(list[i]) + '
\n' result = result + '' - return '%s
' % result + return '%s
' % result - def grey(self, text): return '%s' % text + def grey(self, text): return '%s' % text def namelink(self, name, *dicts): """Make a link for an identifier, given name-to-URL mappings.""" @@ -559,6 +732,25 @@ def classlink(self, object, modname): module.__name__, name, classname(object, modname)) return classname(object, modname) + def parentlink(self, object, modname): + """Make a link for the enclosing class or module.""" + link = None + name, module = object.__name__, sys.modules.get(object.__module__) + if hasattr(module, name) and getattr(module, name) is object: + if '.' in object.__qualname__: + name = object.__qualname__.rpartition('.')[0] + if object.__module__ != modname: + link = '%s.html#%s' % (module.__name__, name) + else: + link = '#%s' % name + else: + if object.__module__ != modname: + link = '%s.html' % module.__name__ + if link: + return '%s' % (link, parentname(object, modname)) + else: + return parentname(object, modname) + def modulelink(self, object): """Make a link for a module.""" return '%s' % (object.__name__, object.__name__) @@ -588,13 +780,11 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): escape = escape or self.escape results = [] here = 0 - pattern = re.compile(r'\b((http|ftp)://\S+[\w/]|' + pattern = re.compile(r'\b((http|https|ftp)://\S+[\w/]|' r'RFC[- ]?(\d+)|' r'PEP[- ]?(\d+)|' r'(self\.)?(\w+))') - while True: - match = pattern.search(text, here) - if not match: break + while match := pattern.search(text, here): start, end = match.span() results.append(escape(text[here:start])) @@ -603,10 +793,10 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): url = escape(all).replace('"', '"') results.append('%s' % (url, url)) elif rfc: - url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) + url = 'https://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('%s' % (url, escape(all))) elif pep: - url = 'http://www.python.org/dev/peps/pep-%04d/' % int(pep) + url = 'https://peps.python.org/pep-%04d/' % int(pep) results.append('%s' % (url, escape(all))) elif selfdot: # Create a link for methods like 'self.method(...)' @@ -629,17 +819,17 @@ def formattree(self, tree, modname, parent=None): """Produce HTML for a class tree as given by inspect.getclasstree().""" result = '' for entry in tree: - if type(entry) is type(()): + if isinstance(entry, tuple): c, bases = entry - result = result + '

' + result = result + '
' result = result + self.classlink(c, modname) if bases and bases != (parent,): parents = [] for base in bases: parents.append(self.classlink(base, modname)) result = result + '(' + ', '.join(parents) + ')' - result = result + '\n
' - elif type(entry) is type([]): + result = result + '\n' + elif isinstance(entry, list): result = result + '
\n%s
\n' % self.formattree( entry, modname, c) return '
\n%s
\n' % result @@ -655,10 +845,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): links = [] for i in range(len(parts)-1): links.append( - '%s' % + '%s' % ('.'.join(parts[:i+1]), parts[i])) linkedname = '.'.join(links + parts[-1:]) - head = '%s' % linkedname + head = '%s' % linkedname try: path = inspect.getabsfile(object) url = urllib.parse.quote(path) @@ -680,9 +870,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): docloc = '
Module Reference' % locals() else: docloc = '' - result = self.heading( - head, '#ffffff', '#7799ee', - 'index
' + filelink + docloc) + result = self.heading(head, 'index
' + filelink + docloc) modules = inspect.getmembers(object, inspect.ismodule) @@ -704,9 +892,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): cdict[key] = cdict[base] = modname + '.html#' + key funcs, fdict = [], {} for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or inspect.isbuiltin(value) + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) fdict[key] = '#-' + key @@ -717,7 +906,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) - doc = doc and '%s' % doc + doc = doc and '%s' % doc result = result + '

%s

\n' % doc if hasattr(object, '__path__'): @@ -727,12 +916,12 @@ def docmodule(self, object, name=None, mod=None, *ignored): modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) result = result + self.bigsection( - 'Package Contents', '#ffffff', '#aa55cc', contents) + 'Package Contents', 'pkg-content', contents) elif modules: contents = self.multicolumn( modules, lambda t: self.modulelink(t[1])) result = result + self.bigsection( - 'Modules', '#ffffff', '#aa55cc', contents) + 'Modules', 'pkg-content', contents) if classes: classlist = [value for (key, value) in classes] @@ -741,27 +930,25 @@ def docmodule(self, object, name=None, mod=None, *ignored): for key, value in classes: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( - 'Classes', '#ffffff', '#ee77aa', ' '.join(contents)) + 'Classes', 'index', ' '.join(contents)) if funcs: contents = [] for key, value in funcs: contents.append(self.document(value, key, name, fdict, cdict)) result = result + self.bigsection( - 'Functions', '#ffffff', '#eeaa77', ' '.join(contents)) + 'Functions', 'functions', ' '.join(contents)) if data: contents = [] for key, value in data: contents.append(self.document(value, key)) result = result + self.bigsection( - 'Data', '#ffffff', '#55aa55', '
\n'.join(contents)) + 'Data', 'data', '
\n'.join(contents)) if hasattr(object, '__author__'): contents = self.markup(str(object.__author__), self.preformat) - result = result + self.bigsection( - 'Author', '#ffffff', '#7799ee', contents) + result = result + self.bigsection('Author', 'author', contents) if hasattr(object, '__credits__'): contents = self.markup(str(object.__credits__), self.preformat) - result = result + self.bigsection( - 'Credits', '#ffffff', '#7799ee', contents) + result = result + self.bigsection('Credits', 'credits', contents) return result @@ -806,10 +993,10 @@ def spill(msg, attrs, predicate): except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) else: push(self.document(value, name, mod, - funcs, classes, mdict, object)) + funcs, classes, mdict, object, homecls)) push('\n') return attrs @@ -819,7 +1006,7 @@ def spilldescriptors(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) return attrs def spilldata(msg, attrs, predicate): @@ -829,16 +1016,13 @@ def spilldata(msg, attrs, predicate): push(msg) for name, kind, homecls, value in ok: base = self.docother(getattr(object, name), name, mod) - if callable(value) or inspect.isdatadescriptor(value): - doc = getattr(value, "__doc__", None) - else: - doc = None - if doc is None: + doc = getdoc(value) + if not doc: push('
%s
\n' % base) else: doc = self.markup(getdoc(value), self.preformat, funcs, classes, mdict) - doc = '
%s' % doc + doc = '
%s' % doc push('
%s%s
\n' % (base, doc)) push('\n') return attrs @@ -870,7 +1054,7 @@ def spilldata(msg, attrs, predicate): thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) - if thisclass is builtins.object: + if object is not builtins.object and thisclass is builtins.object: attrs = inherited continue elif thisclass is object: @@ -889,6 +1073,8 @@ def spilldata(msg, attrs, predicate): lambda t: t[1] == 'class method') attrs = spill('Static methods %s' % tag, attrs, lambda t: t[1] == 'static method') + attrs = spilldescriptors("Readonly properties %s" % tag, attrs, + lambda t: t[1] == 'readonly property') attrs = spilldescriptors('Data descriptors %s' % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata('Data and other attributes %s' % tag, attrs, @@ -909,100 +1095,129 @@ def spilldata(msg, attrs, predicate): for base in bases: parents.append(self.classlink(base, object.__module__)) title = title + '(%s)' % ', '.join(parents) - doc = self.markup(getdoc(object), self.preformat, funcs, classes, mdict) - doc = doc and '%s
 ' % doc - return self.section(title, '#000000', '#ffc8d8', contents, 3, doc) + decl = '' + argspec = _getargspec(object) + if argspec and argspec != '()': + decl = name + self.escape(argspec) + '\n\n' + + doc = getdoc(object) + if decl: + doc = decl + (doc or '') + doc = self.markup(doc, self.preformat, funcs, classes, mdict) + doc = doc and '%s
 ' % doc + + return self.section(title, 'title', contents, 3, doc) def formatvalue(self, object): """Format an argument default value as text.""" return self.grey('=' + self.repr(object)) def docroutine(self, object, name=None, mod=None, - funcs={}, classes={}, methods={}, cl=None): + funcs={}, classes={}, methods={}, cl=None, homecls=None): """Produce HTML documentation for a function or method object.""" realname = object.__name__ name = name or realname - anchor = (cl and cl.__name__ or '') + '-' + name + if homecls is None: + homecls = cl + anchor = ('' if cl is None else cl.__name__) + '-' + name note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + self.classlink(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % self.classlink(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % self.classlink( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % self.classlink(imclass,mod) + note = ' method of %s instance' % self.classlink( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % self.classlink(objclass, mod) + elif objclass is not homecls: + note = ' from ' + self.classlink(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = self.parentlink(imfunc, mod) + if pname: + note = ' from %s' % pname + + if (inspect.iscoroutinefunction(object) or + inspect.isasyncgenfunction(object)): + asyncqualifier = 'async ' + else: + asyncqualifier = '' if name == realname: title = '%s' % (anchor, realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): reallink = '%s' % ( cl.__name__ + '-' + realname, realname) - skipdocs = 1 + skipdocs = True + if note.startswith(' from '): + note = '' else: reallink = realname title = '%s = %s' % ( anchor, name, reallink) argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = '%s lambda ' % name - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. + argspec = _getargspec(object) + if argspec and realname == '': + title = '%s lambda ' % name + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + if not object.__annotations__: argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' - decl = title + self.escape(argspec) + (note and self.grey( - '%s' % note)) + decl = asyncqualifier + title + self.escape(argspec) + (note and + self.grey('%s' % note)) if skipdocs: return '
%s
\n' % decl else: doc = self.markup( getdoc(object), self.preformat, funcs, classes, methods) - doc = doc and '
%s
' % doc + doc = doc and '
%s
' % doc return '
%s
%s
\n' % (decl, doc) - def _docdescriptor(self, name, value, mod): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): + """Produce html documentation for a data descriptor.""" results = [] push = results.append if name: push('
%s
\n' % name) - if value.__doc__ is not None: - doc = self.markup(getdoc(value), self.preformat) - push('
%s
\n' % doc) + doc = self.markup(getdoc(object), self.preformat) + if doc: + push('
%s
\n' % doc) push('
\n') return ''.join(results) - def docproperty(self, object, name=None, mod=None, cl=None): - """Produce html documentation for a property.""" - return self._docdescriptor(name, object, mod) + docproperty = docdata def docother(self, object, name=None, mod=None, *ignored): """Produce HTML documentation for a data object.""" lhs = name and '%s = ' % name or '' return lhs + self.repr(object) - def docdata(self, object, name=None, mod=None, cl=None): - """Produce html documentation for a data descriptor.""" - return self._docdescriptor(name, object, mod) - def index(self, dir, shadowed=None): """Generate an HTML index for a directory of modules.""" modpkgs = [] @@ -1016,7 +1231,7 @@ def index(self, dir, shadowed=None): modpkgs.sort() contents = self.multicolumn(modpkgs, self.modpkglink) - return self.bigsection(dir, '#ffffff', '#ee77aa', contents) + return self.bigsection(dir, 'index', contents) # -------------------------------------------- text documentation generator @@ -1067,8 +1282,7 @@ def bold(self, text): def indent(self, text, prefix=' '): """Indent text by prepending a given prefix to each line.""" if not text: return '' - lines = [prefix + line for line in text.split('\n')] - if lines: lines[-1] = lines[-1].rstrip() + lines = [(prefix + line).rstrip() for line in text.split('\n')] return '\n'.join(lines) def section(self, title, contents): @@ -1082,19 +1296,19 @@ def formattree(self, tree, modname, parent=None, prefix=''): """Render in text a class tree as returned by inspect.getclasstree().""" result = '' for entry in tree: - if type(entry) is type(()): + if isinstance(entry, tuple): c, bases = entry result = result + prefix + classname(c, modname) if bases and bases != (parent,): parents = (classname(c, modname) for c in bases) result = result + '(%s)' % ', '.join(parents) result = result + '\n' - elif type(entry) is type([]): + elif isinstance(entry, list): result = result + self.formattree( entry, modname, c, prefix + ' ') return result - def docmodule(self, object, name=None, mod=None): + def docmodule(self, object, name=None, mod=None, *ignored): """Produce text documentation for a given module object.""" name = object.__name__ # ignore the passed-in name synop, desc = splitdoc(getdoc(object)) @@ -1123,9 +1337,10 @@ def docmodule(self, object, name=None, mod=None): classes.append((key, value)) funcs = [] for key, value in inspect.getmembers(object, inspect.isroutine): - # if __all__ exists, believe it. Otherwise use old heuristic. - if (all is not None or - inspect.isbuiltin(value) or inspect.getmodule(value) is object): + # if __all__ exists, believe it. Otherwise use a heuristic. + if (all is not None + or inspect.isbuiltin(value) + or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) data = [] @@ -1212,10 +1427,17 @@ def makename(c, m=object.__module__): parents = map(makename, bases) title = title + '(%s)' % ', '.join(parents) - doc = getdoc(object) - contents = doc and [doc + '\n'] or [] + contents = [] push = contents.append + argspec = _getargspec(object) + if argspec and argspec != '()': + push(name + argspec + '\n') + + doc = getdoc(object) + if doc: + push(doc + '\n') + # List the mro, if non-trivial. mro = deque(inspect.getmro(object)) if len(mro) > 2: @@ -1224,6 +1446,25 @@ def makename(c, m=object.__module__): push(' ' + makename(base)) push('') + # List the built-in subclasses, if any: + subclasses = sorted( + (str(cls.__name__) for cls in type.__subclasses__(object) + if (not cls.__name__.startswith("_") and + getattr(cls, '__module__', '') == "builtins")), + key=str.lower + ) + no_of_subclasses = len(subclasses) + MAX_SUBCLASSES_TO_DISPLAY = 4 + if subclasses: + push("Built-in subclasses:") + for subclassname in subclasses[:MAX_SUBCLASSES_TO_DISPLAY]: + push(' ' + subclassname) + if no_of_subclasses > MAX_SUBCLASSES_TO_DISPLAY: + push(' ... and ' + + str(no_of_subclasses - MAX_SUBCLASSES_TO_DISPLAY) + + ' other subclasses') + push('') + # Cute little class to pump out a horizontal rule between sections. class HorizontalRule: def __init__(self): @@ -1245,10 +1486,10 @@ def spill(msg, attrs, predicate): except Exception: # Some descriptors may meet a failure in their __get__. # (bug #1785) - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) else: push(self.document(value, - name, mod, object)) + name, mod, object, homecls)) return attrs def spilldescriptors(msg, attrs, predicate): @@ -1257,7 +1498,7 @@ def spilldescriptors(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - push(self._docdescriptor(name, value, mod)) + push(self.docdata(value, name, mod)) return attrs def spilldata(msg, attrs, predicate): @@ -1266,10 +1507,7 @@ def spilldata(msg, attrs, predicate): hr.maybe() push(msg) for name, kind, homecls, value in ok: - if callable(value) or inspect.isdatadescriptor(value): - doc = getdoc(value) - else: - doc = None + doc = getdoc(value) try: obj = getattr(object, name) except AttributeError: @@ -1289,7 +1527,7 @@ def spilldata(msg, attrs, predicate): thisclass = attrs[0][2] attrs, inherited = _split_list(attrs, lambda t: t[2] is thisclass) - if thisclass is builtins.object: + if object is not builtins.object and thisclass is builtins.object: attrs = inherited continue elif thisclass is object: @@ -1307,6 +1545,8 @@ def spilldata(msg, attrs, predicate): lambda t: t[1] == 'class method') attrs = spill("Static methods %s:\n" % tag, attrs, lambda t: t[1] == 'static method') + attrs = spilldescriptors("Readonly properties %s:\n" % tag, attrs, + lambda t: t[1] == 'readonly property') attrs = spilldescriptors("Data descriptors %s:\n" % tag, attrs, lambda t: t[1] == 'data descriptor') attrs = spilldata("Data and other attributes %s:\n" % tag, attrs, @@ -1324,48 +1564,73 @@ def formatvalue(self, object): """Format an argument default value as text.""" return '=' + self.repr(object) - def docroutine(self, object, name=None, mod=None, cl=None): + def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): """Produce text documentation for a function or method object.""" realname = object.__name__ name = name or realname + if homecls is None: + homecls = cl note = '' - skipdocs = 0 + skipdocs = False + imfunc = None if _is_bound_method(object): - imclass = object.__self__.__class__ - if cl: - if imclass is not cl: - note = ' from ' + classname(imclass, mod) + imself = object.__self__ + if imself is cl: + imfunc = getattr(object, '__func__', None) + elif inspect.isclass(imself): + note = ' class method of %s' % classname(imself, mod) else: - if object.__self__ is not None: - note = ' method of %s instance' % classname( - object.__self__.__class__, mod) - else: - note = ' unbound %s method' % classname(imclass,mod) + note = ' method of %s instance' % classname( + imself.__class__, mod) + elif (inspect.ismethoddescriptor(object) or + inspect.ismethodwrapper(object)): + try: + objclass = object.__objclass__ + except AttributeError: + pass + else: + if cl is None: + note = ' unbound %s method' % classname(objclass, mod) + elif objclass is not homecls: + note = ' from ' + classname(objclass, mod) + else: + imfunc = object + if inspect.isfunction(imfunc) and homecls is not None and ( + imfunc.__module__ != homecls.__module__ or + imfunc.__qualname__ != homecls.__qualname__ + '.' + realname): + pname = parentname(imfunc, mod) + if pname: + note = ' from %s' % pname + + if (inspect.iscoroutinefunction(object) or + inspect.isasyncgenfunction(object)): + asyncqualifier = 'async ' + else: + asyncqualifier = '' if name == realname: title = self.bold(realname) else: - if cl and inspect.getattr_static(cl, realname, []) is object: - skipdocs = 1 + if (cl is not None and + inspect.getattr_static(cl, realname, []) is object): + skipdocs = True + if note.startswith(' from '): + note = '' title = self.bold(name) + ' = ' + realname argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = self.bold(name) + ' lambda ' - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + argspec = _getargspec(object) + if argspec and realname == '': + title = self.bold(name) + ' lambda ' + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + if not object.__annotations__: + argspec = argspec[1:-1] if not argspec: argspec = '(...)' - decl = title + argspec + note + decl = asyncqualifier + title + argspec + note if skipdocs: return decl + '\n' @@ -1373,28 +1638,24 @@ def docroutine(self, object, name=None, mod=None, cl=None): doc = getdoc(object) or '' return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') - def _docdescriptor(self, name, value, mod): + def docdata(self, object, name=None, mod=None, cl=None, *ignored): + """Produce text documentation for a data descriptor.""" results = [] push = results.append if name: push(self.bold(name)) push('\n') - doc = getdoc(value) or '' + doc = getdoc(object) or '' if doc: push(self.indent(doc)) push('\n') return ''.join(results) - def docproperty(self, object, name=None, mod=None, cl=None): - """Produce text documentation for a property.""" - return self._docdescriptor(name, object, mod) + docproperty = docdata - def docdata(self, object, name=None, mod=None, cl=None): - """Produce text documentation for a data descriptor.""" - return self._docdescriptor(name, object, mod) - - def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=None): + def docother(self, object, name=None, mod=None, parent=None, *ignored, + maxlen=None, doc=None): """Produce text documentation for a data object.""" repr = self.repr(object) if maxlen: @@ -1402,8 +1663,10 @@ def docother(self, object, name=None, mod=None, parent=None, maxlen=None, doc=No chop = maxlen - len(line) if chop < 0: repr = repr[:chop] + '...' line = (name and self.bold(name) + ' = ' or '') + repr - if doc is not None: - line += '\n' + self.indent(str(doc)) + if not doc: + doc = getdoc(object) + if doc: + line += '\n' + self.indent(str(doc)) + '\n' return line class _PlainTextDoc(TextDoc): @@ -1413,136 +1676,11 @@ def bold(self, text): # --------------------------------------------------------- user interfaces -def pager(text): +def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() - pager(text) - -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text: tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text: pipepager(plain(text), use_pager) - else: - return lambda text: pipepager(text, use_pager) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text: tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text: pipepager(text, 'less') - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text: pipepager(text, 'more') - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def pipepager(text, cmd): - """Page through text by feeding it to another program.""" - import subprocess - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE) - try: - with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd): - """Page through text by invoking a program on a temporary file.""" - import tempfile - filename = tempfile.mktemp() - with open(filename, 'w', errors='backslashreplace') as file: - file.write(text) - try: - os.system(cmd + ' "' + filename + '"') - finally: - os.unlink(filename) - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) + pager = get_pager() + pager(text, title) def describe(thing): """Produce a short description of the given thing.""" @@ -1569,6 +1707,13 @@ def describe(thing): return 'function ' + thing.__name__ if inspect.ismethod(thing): return 'method ' + thing.__name__ + if inspect.ismethodwrapper(thing): + return 'method wrapper ' + thing.__name__ + if inspect.ismethoddescriptor(thing): + try: + return 'method descriptor ' + thing.__name__ + except AttributeError: + pass return type(thing).__name__ def locate(path, forceload=0): @@ -1626,36 +1771,49 @@ def render_doc(thing, title='Python Library Documentation: %s', forceload=0, if not (inspect.ismodule(object) or inspect.isclass(object) or inspect.isroutine(object) or - inspect.isgetsetdescriptor(object) or - inspect.ismemberdescriptor(object) or - isinstance(object, property)): + inspect.isdatadescriptor(object) or + _getdoc(object)): # If the passed object is a piece of data or an instance, # document its available methods instead of its value. - object = type(object) - desc += ' object' + if hasattr(object, '__origin__'): + object = object.__origin__ + else: + object = type(object) + desc += ' object' return title % desc + '\n\n' + renderer.document(object, name) def doc(thing, title='Python Library Documentation: %s', forceload=0, - output=None): + output=None, is_cli=False): """Display text documentation, given an object or a path to an object.""" - try: - if output is None: - pager(render_doc(thing, title, forceload)) - else: - output.write(render_doc(thing, title, forceload, plaintext)) - except (ImportError, ErrorDuringImport) as value: - print(value) + if output is None: + try: + if isinstance(thing, str): + what = thing + else: + what = getattr(thing, '__qualname__', None) + if not isinstance(what, str): + what = getattr(thing, '__name__', None) + if not isinstance(what, str): + what = type(thing).__name__ + ' object' + pager(render_doc(thing, title, forceload), f'Help on {what!s}') + except ImportError as exc: + if is_cli: + raise + print(exc) + else: + try: + s = render_doc(thing, title, forceload, plaintext) + except ImportError as exc: + s = str(exc) + output.write(s) def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" - try: - object, name = resolve(thing, forceload) - page = html.page(describe(object), html.document(object, name)) - with open(name + '.html', 'w', encoding='utf-8') as file: - file.write(page) - print('wrote', name + '.html') - except (ImportError, ErrorDuringImport) as value: - print(value) + object, name = resolve(thing, forceload) + page = html.page(describe(object), html.document(object, name)) + with open(name + '.html', 'w', encoding='utf-8') as file: + file.write(page) + print('wrote', name + '.html') def writedocs(dir, pkgpath='', done=None): """Write out HTML documentation for all modules in a directory tree.""" @@ -1664,6 +1822,37 @@ def writedocs(dir, pkgpath='', done=None): writedoc(modname) return + +def _introdoc(): + import textwrap + ver = '%d.%d' % sys.version_info[:2] + if os.environ.get('PYTHON_BASIC_REPL'): + pyrepl_keys = '' + else: + # Additional help for keyboard shortcuts if enhanced REPL is used. + pyrepl_keys = ''' + You can use the following keyboard shortcuts at the main interpreter prompt. + F1: enter interactive help, F2: enter history browsing mode, F3: enter paste + mode (press again to exit). + ''' + return textwrap.dedent(f'''\ + Welcome to Python {ver}'s help utility! If this is your first time using + Python, you should definitely check out the tutorial at + https://docs.python.org/{ver}/tutorial/. + + Enter the name of any module, keyword, or topic to get help on writing + Python programs and using Python modules. To get a list of available + modules, keywords, symbols, or topics, enter "modules", "keywords", + "symbols", or "topics". + {pyrepl_keys} + Each module also comes with a one-line summary of what it does; to list + the modules whose name or summary contain a given string such as "spam", + enter "modules spam". + + To quit this help utility and return to the interpreter, + enter "q", "quit" or "exit". + ''') + class Helper: # These dictionaries map a topic name to either an alias, or a tuple @@ -1672,7 +1861,7 @@ class Helper: # in pydoc_data/topics.py. # # CAUTION: if you change one of these dictionaries, be sure to adapt the - # list of needed labels in Doc/tools/pyspecific.py and + # list of needed labels in Doc/tools/extensions/pyspecific.py and # regenerate the pydoc_data/topics.py file by running # make pydoc-topics # in Doc/ and copying the output file into the Lib/ directory. @@ -1684,6 +1873,8 @@ class Helper: 'and': 'BOOLEAN', 'as': 'with', 'assert': ('assert', ''), + 'async': ('async', ''), + 'await': ('await', ''), 'break': ('break', 'while for'), 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), @@ -1735,6 +1926,7 @@ class Helper: ':': 'SLICINGS DICTIONARYLITERALS', '@': 'def class', '\\': 'STRINGS', + ':=': 'ASSIGNMENTEXPRESSIONS', '_': 'PRIVATENAMES', '__': 'PRIVATENAMES SPECIALMETHODS', '`': 'BACKQUOTES', @@ -1749,6 +1941,7 @@ class Helper: if topic not in topics: topics = topics + ' ' + topic symbols[symbol] = topics + del topic, symbols_, symbol, topics topics = { 'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS ' @@ -1827,6 +2020,7 @@ class Helper: 'ASSERTION': 'assert', 'ASSIGNMENT': ('assignment', 'AUGMENTEDASSIGNMENT'), 'AUGMENTEDASSIGNMENT': ('augassign', 'NUMBERMETHODS'), + 'ASSIGNMENTEXPRESSIONS': ('assignment-expressions', ''), 'DELETION': 'del', 'RETURNING': 'return', 'IMPORTING': 'import', @@ -1841,8 +2035,13 @@ def __init__(self, input=None, output=None): self._input = input self._output = output - input = property(lambda self: self._input or sys.stdin) - output = property(lambda self: self._output or sys.stdout) + @property + def input(self): + return self._input or sys.stdin + + @property + def output(self): + return self._output or sys.stdout def __repr__(self): if inspect.stack()[1][3] == '?': @@ -1854,7 +2053,10 @@ def __repr__(self): _GoInteractive = object() def __call__(self, request=_GoInteractive): if request is not self._GoInteractive: - self.help(request) + try: + self.help(request) + except ImportError as err: + self.output.write(f'{err}\n') else: self.intro() self.interact() @@ -1870,17 +2072,18 @@ def interact(self): while True: try: request = self.getline('help> ') - if not request: break except (KeyboardInterrupt, EOFError): break request = request.strip() + if not request: + continue # back to the prompt # Make sure significant trailing quoting marks of literals don't # get deleted while cleaning input if (len(request) > 2 and request[0] == request[-1] in ("'", '"') and request[0] not in request[1:-1]): request = request[1:-1] - if request.lower() in ('q', 'quit'): break + if request.lower() in ('q', 'quit', 'exit'): break if request == 'help': self.intro() else: @@ -1895,8 +2098,8 @@ def getline(self, prompt): self.output.flush() return self.input.readline() - def help(self, request): - if type(request) is type(''): + def help(self, request, is_cli=False): + if isinstance(request, str): request = request.strip() if request == 'keywords': self.listkeywords() elif request == 'symbols': self.listsymbols() @@ -1907,34 +2110,20 @@ def help(self, request): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:') + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) - elif request: doc(request, 'Help on %s:', output=self._output) - else: doc(str, 'Help on %s:', output=self._output) + elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) + else: doc(str, 'Help on %s:', output=self._output, is_cli=is_cli) elif isinstance(request, Helper): self() - else: doc(request, 'Help on %s:', output=self._output) + else: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) self.output.write('\n') def intro(self): - self.output.write(''' -Welcome to Python {0}'s help utility! - -If this is your first time using Python, you should definitely check out -the tutorial on the Internet at https://docs.python.org/{0}/tutorial/. - -Enter the name of any module, keyword, or topic to get help on writing -Python programs and using Python modules. To quit this help utility and -return to the interpreter, just type "quit". - -To get a list of available modules, keywords, symbols, or topics, type -"modules", "keywords", "symbols", or "topics". Each module also comes -with a one-line summary of what it does; to list the modules whose name -or summary contain a given string such as "spam", type "modules spam". -'''.format('%d.%d' % sys.version_info[:2])) + self.output.write(_introdoc()) def list(self, items, columns=4, width=80): - items = list(sorted(items)) + items = sorted(items) colw = width // columns rows = (len(items) + columns - 1) // columns for row in range(rows): @@ -1966,7 +2155,7 @@ def listtopics(self): Here is a list of available topics. Enter any topic name to get more help. ''') - self.list(self.topics.keys()) + self.list(self.topics.keys(), columns=3) def showtopic(self, topic, more_xrefs=''): try: @@ -1981,7 +2170,7 @@ def showtopic(self, topic, more_xrefs=''): if not target: self.output.write('no documentation found for %s\n' % repr(topic)) return - if type(target) is type(''): + if isinstance(target, str): return self.showtopic(target, more_xrefs) label, xrefs = target @@ -1998,7 +2187,11 @@ def showtopic(self, topic, more_xrefs=''): text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) - pager(doc) + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). @@ -2090,7 +2283,7 @@ def run(self, callback, key=None, completer=None, onerror=None): callback(None, modname, '') else: try: - spec = pkgutil._get_spec(importer, modname) + spec = importer.find_spec(modname) except SyntaxError: # raised by tests for bad coding cookies or BOM continue @@ -2135,13 +2328,13 @@ def onerror(modname): warnings.filterwarnings('ignore') # ignore problems during import ModuleScanner().run(callback, key, onerror=onerror) -# --------------------------------------- enhanced Web browser interface +# --------------------------------------- enhanced web browser interface -def _start_server(urlhandler, port): +def _start_server(urlhandler, hostname, port): """Start an HTTP server thread on a specific port. Start an HTML/text server thread, so HTML or text documents can be - browsed dynamically and interactively with a Web browser. Example use: + browsed dynamically and interactively with a web browser. Example use: >>> import time >>> import pydoc @@ -2177,14 +2370,14 @@ def _start_server(urlhandler, port): Let the server do its thing. We just need to monitor its status. Use time.sleep so the loop doesn't hog the CPU. - >>> starttime = time.time() + >>> starttime = time.monotonic() >>> timeout = 1 #seconds This is a short timeout for testing purposes. >>> while serverthread.serving: ... time.sleep(.01) - ... if serverthread.serving and time.time() - starttime > timeout: + ... if serverthread.serving and time.monotonic() - starttime > timeout: ... serverthread.stop() ... break @@ -2222,8 +2415,8 @@ def log_message(self, *args): class DocServer(http.server.HTTPServer): - def __init__(self, port, callback): - self.host = 'localhost' + def __init__(self, host, port, callback): + self.host = host self.address = (self.host, port) self.callback = callback self.base.__init__(self, self.address, self.handler) @@ -2243,12 +2436,14 @@ def server_activate(self): class ServerThread(threading.Thread): - def __init__(self, urlhandler, port): + def __init__(self, urlhandler, host, port): self.urlhandler = urlhandler + self.host = host self.port = int(port) threading.Thread.__init__(self) self.serving = False self.error = None + self.docserver = None def run(self): """Start the server.""" @@ -2257,11 +2452,11 @@ def run(self): DocServer.handler = DocHandler DocHandler.MessageClass = email.message.Message DocHandler.urlhandler = staticmethod(self.urlhandler) - docsvr = DocServer(self.port, self.ready) + docsvr = DocServer(self.host, self.port, self.ready) self.docserver = docsvr docsvr.serve_until_quit() - except Exception as e: - self.error = e + except Exception as err: + self.error = err def ready(self, server): self.serving = True @@ -2279,11 +2474,11 @@ def stop(self): self.serving = False self.url = None - thread = ServerThread(urlhandler, port) + thread = ServerThread(urlhandler, hostname, port) thread.start() - # Wait until thread.serving is True to make sure we are - # really up before returning. - while not thread.error and not thread.serving: + # Wait until thread.serving is True and thread.docserver is set + # to make sure we are really up before returning. + while not thread.error and not (thread.serving and thread.docserver): time.sleep(.01) return thread @@ -2306,15 +2501,14 @@ def page(self, title, contents): '' % css_path) return '''\ - -Pydoc: %s - -%s%s
%s
+ + + + +Pydoc: %s +%s%s
%s
''' % (title, css_link, html_navbar(), contents) - def filelink(self, url, path): - return '%s' % (url, path) - html = _HTMLDoc() @@ -2352,22 +2546,21 @@ def bltinlink(name): return '%s' % (name, name) heading = html.heading( - 'Index of Modules', - '#ffffff', '#7799ee') + 'Index of Modules' + ) names = [name for name in sys.builtin_module_names if name != '__main__'] contents = html.multicolumn(names, bltinlink) contents = [heading, '

' + html.bigsection( - 'Built-in Modules', '#ffffff', '#ee77aa', contents)] + 'Built-in Modules', 'index', contents)] seen = {} for dir in sys.path: contents.append(html.index(dir, seen)) contents.append( - '

pydoc by Ka-Ping Yee' - '<ping@lfw.org>') + '

pydoc by Ka-Ping Yee' + '<ping@lfw.org>

') return 'Index of Modules', ''.join(contents) def html_search(key): @@ -2392,27 +2585,14 @@ def bltinlink(name): results = [] heading = html.heading( - 'Search Results', - '#ffffff', '#7799ee') + 'Search Results', + ) for name, desc in search_result: results.append(bltinlink(name) + desc) contents = heading + html.bigsection( - 'key = %s' % key, '#ffffff', '#ee77aa', '
'.join(results)) + 'key = %s' % key, 'index', '
'.join(results)) return 'Search Results', contents - def html_getfile(path): - """Get and display a source file listing safely.""" - path = urllib.parse.unquote(path) - with tokenize.open(path) as fp: - lines = html.escape(fp.read()) - body = '
%s
' % lines - heading = html.heading( - 'File Listing', - '#ffffff', '#7799ee') - contents = heading + html.bigsection( - 'File: %s' % path, '#ffffff', '#ee77aa', body) - return 'getfile %s' % path, contents - def html_topics(): """Index of topic texts available.""" @@ -2420,20 +2600,20 @@ def bltinlink(name): return '%s' % (name, name) heading = html.heading( - 'INDEX', - '#ffffff', '#7799ee') + 'INDEX', + ) names = sorted(Helper.topics.keys()) contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( - 'Topics', '#ffffff', '#ee77aa', contents) + 'Topics', 'index', contents) return 'Topics', contents def html_keywords(): """Index of keywords.""" heading = html.heading( - 'INDEX', - '#ffffff', '#7799ee') + 'INDEX', + ) names = sorted(Helper.keywords.keys()) def bltinlink(name): @@ -2441,7 +2621,7 @@ def bltinlink(name): contents = html.multicolumn(names, bltinlink) contents = heading + html.bigsection( - 'Keywords', '#ffffff', '#ee77aa', contents) + 'Keywords', 'index', contents) return 'Keywords', contents def html_topicpage(topic): @@ -2454,10 +2634,10 @@ def html_topicpage(topic): else: title = 'TOPIC' heading = html.heading( - '%s' % title, - '#ffffff', '#7799ee') + '%s' % title, + ) contents = '
%s
' % html.markup(contents) - contents = html.bigsection(topic , '#ffffff','#ee77aa', contents) + contents = html.bigsection(topic , 'index', contents) if xrefs: xrefs = sorted(xrefs.split()) @@ -2465,8 +2645,7 @@ def bltinlink(name): return '%s' % (name, name) xrefs = html.multicolumn(xrefs, bltinlink) - xrefs = html.section('Related help topics: ', - '#ffffff', '#ee77aa', xrefs) + xrefs = html.section('Related help topics: ', 'index', xrefs) return ('%s %s' % (title, topic), ''.join((heading, contents, xrefs))) @@ -2480,12 +2659,11 @@ def html_getobj(url): def html_error(url, exc): heading = html.heading( - 'Error', - '#ffffff', '#7799ee') + 'Error', + ) contents = '
'.join(html.escape(line) for line in format_exception_only(type(exc), exc)) - contents = heading + html.bigsection(url, '#ffffff', '#bb0000', - contents) + contents = heading + html.bigsection(url, 'error', contents) return "Error - %s" % url, contents def get_html_page(url): @@ -2504,8 +2682,6 @@ def get_html_page(url): op, _, url = url.partition('=') if op == "search?key": title, content = html_search(url) - elif op == "getfile?key": - title, content = html_getfile(url) elif op == "topic?key": # try topics first, then objects. try: @@ -2543,14 +2719,14 @@ def get_html_page(url): raise TypeError('unknown content type %r for url %s' % (content_type, url)) -def browse(port=0, *, open_browser=True): - """Start the enhanced pydoc Web server and open a Web browser. +def browse(port=0, *, open_browser=True, hostname='localhost'): + """Start the enhanced pydoc web server and open a web browser. Use port '0' to start the server on an arbitrary port. Set open_browser to False to suppress opening a browser. """ import webbrowser - serverthread = _start_server(_url_handler, port) + serverthread = _start_server(_url_handler, hostname, port) if serverthread.error: print(serverthread.error) return @@ -2583,25 +2759,58 @@ def browse(port=0, *, open_browser=True): def ispath(x): return isinstance(x, str) and x.find(os.sep) >= 0 +def _get_revised_path(given_path, argv0): + """Ensures current directory is on returned path, and argv0 directory is not + + Exception: argv0 dir is left alone if it's also pydoc's directory. + + Returns a new path entry list, or None if no adjustment is needed. + """ + # Scripts may get the current directory in their path by default if they're + # run with the -m switch, or directly from the current directory. + # The interactive prompt also allows imports from the current directory. + + # Accordingly, if the current directory is already present, don't make + # any changes to the given_path + if '' in given_path or os.curdir in given_path or os.getcwd() in given_path: + return None + + # Otherwise, add the current directory to the given path, and remove the + # script directory (as long as the latter isn't also pydoc's directory. + stdlib_dir = os.path.dirname(__file__) + script_dir = os.path.dirname(argv0) + revised_path = given_path.copy() + if script_dir in given_path and not os.path.samefile(script_dir, stdlib_dir): + revised_path.remove(script_dir) + revised_path.insert(0, os.getcwd()) + return revised_path + + +# Note: the tests only cover _get_revised_path, not _adjust_cli_path itself +def _adjust_cli_sys_path(): + """Ensures current directory is on sys.path, and __main__ directory is not. + + Exception: __main__ dir is left alone if it's also pydoc's directory. + """ + revised_path = _get_revised_path(sys.path, sys.argv[0]) + if revised_path is not None: + sys.path[:] = revised_path + + def cli(): """Command-line interface (looks at sys.argv to decide what to do).""" import getopt class BadUsage(Exception): pass - # Scripts don't get the current directory in their path by default - # unless they are run with the '-m' switch - if '' not in sys.path: - scriptdir = os.path.dirname(sys.argv[0]) - if scriptdir in sys.path: - sys.path.remove(scriptdir) - sys.path.insert(0, '.') + _adjust_cli_sys_path() try: - opts, args = getopt.getopt(sys.argv[1:], 'bk:p:w') + opts, args = getopt.getopt(sys.argv[1:], 'bk:n:p:w') writing = False start_server = False open_browser = False - port = None + port = 0 + hostname = 'localhost' for opt, val in opts: if opt == '-b': start_server = True @@ -2614,18 +2823,19 @@ class BadUsage(Exception): pass port = val if opt == '-w': writing = True + if opt == '-n': + start_server = True + hostname = val if start_server: - if port is None: - port = 0 - browse(port, open_browser=open_browser) + browse(port, hostname=hostname, open_browser=open_browser) return if not args: raise BadUsage for arg in args: if ispath(arg) and not os.path.exists(arg): print('file %r does not exist' % arg) - break + sys.exit(1) try: if ispath(arg) and os.path.isfile(arg): arg = importfile(arg) @@ -2635,9 +2845,10 @@ class BadUsage(Exception): pass else: writedoc(arg) else: - help.help(arg) - except ErrorDuringImport as value: + help.help(arg, is_cli=True) + except (ImportError, ErrorDuringImport) as value: print(value) + sys.exit(1) except (getopt.error, BadUsage): cmd = os.path.splitext(os.path.basename(sys.argv[0]))[0] @@ -2654,14 +2865,17 @@ class BadUsage(Exception): pass {cmd} -k Search for a keyword in the synopsis lines of all available modules. +{cmd} -n + Start an HTTP server with the given hostname (default: localhost). + {cmd} -p Start an HTTP server on the given port on the local machine. Port number 0 can be used to get an arbitrary unused port. {cmd} -b - Start an HTTP server on an arbitrary unused port and open a Web browser - to interactively browse documentation. The -p option can be used with - the -b option to explicitly specify the server port. + Start an HTTP server on an arbitrary unused port and open a web browser + to interactively browse documentation. This option can be used in + combination with -n and/or -p. {cmd} -w ... Write out the HTML documentation for a module to a file in the current diff --git a/Lib/pydoc_data/_pydoc.css b/Lib/pydoc_data/_pydoc.css index f036ef37a5a..a6aa2e4c1a0 100644 --- a/Lib/pydoc_data/_pydoc.css +++ b/Lib/pydoc_data/_pydoc.css @@ -4,3 +4,109 @@ Contents of this file are subject to change without notice. */ + +body { + background-color: #f0f0f8; +} + +table.heading tr { + background-color: #7799ee; +} + +.decor { + color: #ffffff; +} + +.title-decor { + background-color: #ffc8d8; + color: #000000; +} + +.pkg-content-decor { + background-color: #aa55cc; +} + +.index-decor { + background-color: #ee77aa; +} + +.functions-decor { + background-color: #eeaa77; +} + +.data-decor { + background-color: #55aa55; +} + +.author-decor { + background-color: #7799ee; +} + +.credits-decor { + background-color: #7799ee; +} + +.error-decor { + background-color: #bb0000; +} + +.grey { + color: #909090; +} + +.white { + color: #ffffff; +} + +.repr { + color: #c040c0; +} + +table.heading tr td.title { + vertical-align: bottom; +} + +table.heading tr td.extra { + vertical-align: bottom; + text-align: right; +} + +.heading-text { + font-family: helvetica, arial; +} + +.bigsection { + font-size: larger; +} + +.title { + font-size: x-large; +} + +.code { + font-family: monospace; +} + +table { + width: 100%; + border-spacing : 0; + border-collapse : collapse; + border: 0; +} + +td { + padding: 2; +} + +td.section-title { + vertical-align: bottom; +} + +td.multicolumn { + width: 25%; + vertical-align: bottom; +} + +td.singlecolumn { + width: 100%; +} diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py new file mode 100644 index 00000000000..2a6ede3aa14 --- /dev/null +++ b/Lib/pydoc_data/module_docs.py @@ -0,0 +1,320 @@ +# Autogenerated by Sphinx on Tue Feb 3 17:32:13 2026 +# as part of the release process. + +module_docs = { + '__future__': '__future__#module-__future__', + '__main__': '__main__#module-__main__', + '_thread': '_thread#module-_thread', + '_tkinter': 'tkinter#module-_tkinter', + 'abc': 'abc#module-abc', + 'aifc': 'aifc#module-aifc', + 'annotationlib': 'annotationlib#module-annotationlib', + 'argparse': 'argparse#module-argparse', + 'array': 'array#module-array', + 'ast': 'ast#module-ast', + 'asynchat': 'asynchat#module-asynchat', + 'asyncio': 'asyncio#module-asyncio', + 'asyncore': 'asyncore#module-asyncore', + 'atexit': 'atexit#module-atexit', + 'audioop': 'audioop#module-audioop', + 'base64': 'base64#module-base64', + 'bdb': 'bdb#module-bdb', + 'binascii': 'binascii#module-binascii', + 'bisect': 'bisect#module-bisect', + 'builtins': 'builtins#module-builtins', + 'bz2': 'bz2#module-bz2', + 'cProfile': 'profile#module-cProfile', + 'calendar': 'calendar#module-calendar', + 'cgi': 'cgi#module-cgi', + 'cgitb': 'cgitb#module-cgitb', + 'chunk': 'chunk#module-chunk', + 'cmath': 'cmath#module-cmath', + 'cmd': 'cmd#module-cmd', + 'code': 'code#module-code', + 'codecs': 'codecs#module-codecs', + 'codeop': 'codeop#module-codeop', + 'collections': 'collections#module-collections', + 'collections.abc': 'collections.abc#module-collections.abc', + 'colorsys': 'colorsys#module-colorsys', + 'compileall': 'compileall#module-compileall', + 'compression': 'compression#module-compression', + 'compression.zstd': 'compression.zstd#module-compression.zstd', + 'concurrent.futures': 'concurrent.futures#module-concurrent.futures', + 'concurrent.interpreters': 'concurrent.interpreters#module-concurrent.interpreters', + 'configparser': 'configparser#module-configparser', + 'contextlib': 'contextlib#module-contextlib', + 'contextvars': 'contextvars#module-contextvars', + 'copy': 'copy#module-copy', + 'copyreg': 'copyreg#module-copyreg', + 'crypt': 'crypt#module-crypt', + 'csv': 'csv#module-csv', + 'ctypes': 'ctypes#module-ctypes', + 'curses': 'curses#module-curses', + 'curses.ascii': 'curses.ascii#module-curses.ascii', + 'curses.panel': 'curses.panel#module-curses.panel', + 'curses.textpad': 'curses#module-curses.textpad', + 'dataclasses': 'dataclasses#module-dataclasses', + 'datetime': 'datetime#module-datetime', + 'dbm': 'dbm#module-dbm', + 'dbm.dumb': 'dbm#module-dbm.dumb', + 'dbm.gnu': 'dbm#module-dbm.gnu', + 'dbm.ndbm': 'dbm#module-dbm.ndbm', + 'dbm.sqlite3': 'dbm#module-dbm.sqlite3', + 'decimal': 'decimal#module-decimal', + 'difflib': 'difflib#module-difflib', + 'dis': 'dis#module-dis', + 'distutils': 'distutils#module-distutils', + 'doctest': 'doctest#module-doctest', + 'email': 'email#module-email', + 'email.charset': 'email.charset#module-email.charset', + 'email.contentmanager': 'email.contentmanager#module-email.contentmanager', + 'email.encoders': 'email.encoders#module-email.encoders', + 'email.errors': 'email.errors#module-email.errors', + 'email.generator': 'email.generator#module-email.generator', + 'email.header': 'email.header#module-email.header', + 'email.headerregistry': 'email.headerregistry#module-email.headerregistry', + 'email.iterators': 'email.iterators#module-email.iterators', + 'email.message': 'email.message#module-email.message', + 'email.mime': 'email.mime#module-email.mime', + 'email.mime.application': 'email.mime#module-email.mime.application', + 'email.mime.audio': 'email.mime#module-email.mime.audio', + 'email.mime.base': 'email.mime#module-email.mime.base', + 'email.mime.image': 'email.mime#module-email.mime.image', + 'email.mime.message': 'email.mime#module-email.mime.message', + 'email.mime.multipart': 'email.mime#module-email.mime.multipart', + 'email.mime.nonmultipart': 'email.mime#module-email.mime.nonmultipart', + 'email.mime.text': 'email.mime#module-email.mime.text', + 'email.parser': 'email.parser#module-email.parser', + 'email.policy': 'email.policy#module-email.policy', + 'email.utils': 'email.utils#module-email.utils', + 'encodings': 'codecs#module-encodings', + 'encodings.idna': 'codecs#module-encodings.idna', + 'encodings.mbcs': 'codecs#module-encodings.mbcs', + 'encodings.utf_8_sig': 'codecs#module-encodings.utf_8_sig', + 'ensurepip': 'ensurepip#module-ensurepip', + 'enum': 'enum#module-enum', + 'errno': 'errno#module-errno', + 'faulthandler': 'faulthandler#module-faulthandler', + 'fcntl': 'fcntl#module-fcntl', + 'filecmp': 'filecmp#module-filecmp', + 'fileinput': 'fileinput#module-fileinput', + 'fnmatch': 'fnmatch#module-fnmatch', + 'fractions': 'fractions#module-fractions', + 'ftplib': 'ftplib#module-ftplib', + 'functools': 'functools#module-functools', + 'gc': 'gc#module-gc', + 'getopt': 'getopt#module-getopt', + 'getpass': 'getpass#module-getpass', + 'gettext': 'gettext#module-gettext', + 'glob': 'glob#module-glob', + 'graphlib': 'graphlib#module-graphlib', + 'grp': 'grp#module-grp', + 'gzip': 'gzip#module-gzip', + 'hashlib': 'hashlib#module-hashlib', + 'heapq': 'heapq#module-heapq', + 'hmac': 'hmac#module-hmac', + 'html': 'html#module-html', + 'html.entities': 'html.entities#module-html.entities', + 'html.parser': 'html.parser#module-html.parser', + 'http': 'http#module-http', + 'http.client': 'http.client#module-http.client', + 'http.cookiejar': 'http.cookiejar#module-http.cookiejar', + 'http.cookies': 'http.cookies#module-http.cookies', + 'http.server': 'http.server#module-http.server', + 'idlelib': 'idle#module-idlelib', + 'imaplib': 'imaplib#module-imaplib', + 'imghdr': 'imghdr#module-imghdr', + 'imp': 'imp#module-imp', + 'importlib': 'importlib#module-importlib', + 'importlib.abc': 'importlib#module-importlib.abc', + 'importlib.machinery': 'importlib#module-importlib.machinery', + 'importlib.metadata': 'importlib.metadata#module-importlib.metadata', + 'importlib.resources': 'importlib.resources#module-importlib.resources', + 'importlib.resources.abc': 'importlib.resources.abc#module-importlib.resources.abc', + 'importlib.util': 'importlib#module-importlib.util', + 'inspect': 'inspect#module-inspect', + 'io': 'io#module-io', + 'ipaddress': 'ipaddress#module-ipaddress', + 'itertools': 'itertools#module-itertools', + 'json': 'json#module-json', + 'json.tool': 'json#module-json.tool', + 'keyword': 'keyword#module-keyword', + 'linecache': 'linecache#module-linecache', + 'locale': 'locale#module-locale', + 'logging': 'logging#module-logging', + 'logging.config': 'logging.config#module-logging.config', + 'logging.handlers': 'logging.handlers#module-logging.handlers', + 'lzma': 'lzma#module-lzma', + 'mailbox': 'mailbox#module-mailbox', + 'mailcap': 'mailcap#module-mailcap', + 'marshal': 'marshal#module-marshal', + 'math': 'math#module-math', + 'mimetypes': 'mimetypes#module-mimetypes', + 'mmap': 'mmap#module-mmap', + 'modulefinder': 'modulefinder#module-modulefinder', + 'msilib': 'msilib#module-msilib', + 'msvcrt': 'msvcrt#module-msvcrt', + 'multiprocessing': 'multiprocessing#module-multiprocessing', + 'multiprocessing.connection': 'multiprocessing#module-multiprocessing.connection', + 'multiprocessing.dummy': 'multiprocessing#module-multiprocessing.dummy', + 'multiprocessing.managers': 'multiprocessing#module-multiprocessing.managers', + 'multiprocessing.pool': 'multiprocessing#module-multiprocessing.pool', + 'multiprocessing.shared_memory': 'multiprocessing.shared_memory#module-multiprocessing.shared_memory', + 'multiprocessing.sharedctypes': 'multiprocessing#module-multiprocessing.sharedctypes', + 'netrc': 'netrc#module-netrc', + 'nis': 'nis#module-nis', + 'nntplib': 'nntplib#module-nntplib', + 'numbers': 'numbers#module-numbers', + 'operator': 'operator#module-operator', + 'optparse': 'optparse#module-optparse', + 'os': 'os#module-os', + 'os.path': 'os.path#module-os.path', + 'ossaudiodev': 'ossaudiodev#module-ossaudiodev', + 'pathlib': 'pathlib#module-pathlib', + 'pathlib.types': 'pathlib#module-pathlib.types', + 'pdb': 'pdb#module-pdb', + 'pickle': 'pickle#module-pickle', + 'pickletools': 'pickletools#module-pickletools', + 'pipes': 'pipes#module-pipes', + 'pkgutil': 'pkgutil#module-pkgutil', + 'platform': 'platform#module-platform', + 'plistlib': 'plistlib#module-plistlib', + 'poplib': 'poplib#module-poplib', + 'posix': 'posix#module-posix', + 'pprint': 'pprint#module-pprint', + 'profile': 'profile#module-profile', + 'pstats': 'profile#module-pstats', + 'pty': 'pty#module-pty', + 'pwd': 'pwd#module-pwd', + 'py_compile': 'py_compile#module-py_compile', + 'pyclbr': 'pyclbr#module-pyclbr', + 'pydoc': 'pydoc#module-pydoc', + 'queue': 'queue#module-queue', + 'quopri': 'quopri#module-quopri', + 'random': 'random#module-random', + 're': 're#module-re', + 'readline': 'readline#module-readline', + 'reprlib': 'reprlib#module-reprlib', + 'resource': 'resource#module-resource', + 'rlcompleter': 'rlcompleter#module-rlcompleter', + 'runpy': 'runpy#module-runpy', + 'sched': 'sched#module-sched', + 'secrets': 'secrets#module-secrets', + 'select': 'select#module-select', + 'selectors': 'selectors#module-selectors', + 'shelve': 'shelve#module-shelve', + 'shlex': 'shlex#module-shlex', + 'shutil': 'shutil#module-shutil', + 'signal': 'signal#module-signal', + 'site': 'site#module-site', + 'sitecustomize': 'site#module-sitecustomize', + 'smtpd': 'smtpd#module-smtpd', + 'smtplib': 'smtplib#module-smtplib', + 'sndhdr': 'sndhdr#module-sndhdr', + 'socket': 'socket#module-socket', + 'socketserver': 'socketserver#module-socketserver', + 'spwd': 'spwd#module-spwd', + 'sqlite3': 'sqlite3#module-sqlite3', + 'ssl': 'ssl#module-ssl', + 'stat': 'stat#module-stat', + 'statistics': 'statistics#module-statistics', + 'string': 'string#module-string', + 'string.templatelib': 'string.templatelib#module-string.templatelib', + 'stringprep': 'stringprep#module-stringprep', + 'struct': 'struct#module-struct', + 'subprocess': 'subprocess#module-subprocess', + 'sunau': 'sunau#module-sunau', + 'symtable': 'symtable#module-symtable', + 'sys': 'sys#module-sys', + 'sys.monitoring': 'sys.monitoring#module-sys.monitoring', + 'sysconfig': 'sysconfig#module-sysconfig', + 'syslog': 'syslog#module-syslog', + 'tabnanny': 'tabnanny#module-tabnanny', + 'tarfile': 'tarfile#module-tarfile', + 'telnetlib': 'telnetlib#module-telnetlib', + 'tempfile': 'tempfile#module-tempfile', + 'termios': 'termios#module-termios', + 'test': 'test#module-test', + 'test.regrtest': 'test#module-test.regrtest', + 'test.support': 'test#module-test.support', + 'test.support.bytecode_helper': 'test#module-test.support.bytecode_helper', + 'test.support.import_helper': 'test#module-test.support.import_helper', + 'test.support.os_helper': 'test#module-test.support.os_helper', + 'test.support.script_helper': 'test#module-test.support.script_helper', + 'test.support.socket_helper': 'test#module-test.support.socket_helper', + 'test.support.threading_helper': 'test#module-test.support.threading_helper', + 'test.support.warnings_helper': 'test#module-test.support.warnings_helper', + 'textwrap': 'textwrap#module-textwrap', + 'threading': 'threading#module-threading', + 'time': 'time#module-time', + 'timeit': 'timeit#module-timeit', + 'tkinter': 'tkinter#module-tkinter', + 'tkinter.colorchooser': 'tkinter.colorchooser#module-tkinter.colorchooser', + 'tkinter.commondialog': 'dialog#module-tkinter.commondialog', + 'tkinter.dnd': 'tkinter.dnd#module-tkinter.dnd', + 'tkinter.filedialog': 'dialog#module-tkinter.filedialog', + 'tkinter.font': 'tkinter.font#module-tkinter.font', + 'tkinter.messagebox': 'tkinter.messagebox#module-tkinter.messagebox', + 'tkinter.scrolledtext': 'tkinter.scrolledtext#module-tkinter.scrolledtext', + 'tkinter.simpledialog': 'dialog#module-tkinter.simpledialog', + 'tkinter.ttk': 'tkinter.ttk#module-tkinter.ttk', + 'token': 'token#module-token', + 'tokenize': 'tokenize#module-tokenize', + 'tomllib': 'tomllib#module-tomllib', + 'trace': 'trace#module-trace', + 'traceback': 'traceback#module-traceback', + 'tracemalloc': 'tracemalloc#module-tracemalloc', + 'tty': 'tty#module-tty', + 'turtle': 'turtle#module-turtle', + 'turtledemo': 'turtle#module-turtledemo', + 'types': 'types#module-types', + 'typing': 'typing#module-typing', + 'unicodedata': 'unicodedata#module-unicodedata', + 'unittest': 'unittest#module-unittest', + 'unittest.mock': 'unittest.mock#module-unittest.mock', + 'urllib': 'urllib#module-urllib', + 'urllib.error': 'urllib.error#module-urllib.error', + 'urllib.parse': 'urllib.parse#module-urllib.parse', + 'urllib.request': 'urllib.request#module-urllib.request', + 'urllib.response': 'urllib.request#module-urllib.response', + 'urllib.robotparser': 'urllib.robotparser#module-urllib.robotparser', + 'usercustomize': 'site#module-usercustomize', + 'uu': 'uu#module-uu', + 'uuid': 'uuid#module-uuid', + 'venv': 'venv#module-venv', + 'warnings': 'warnings#module-warnings', + 'wave': 'wave#module-wave', + 'weakref': 'weakref#module-weakref', + 'webbrowser': 'webbrowser#module-webbrowser', + 'winreg': 'winreg#module-winreg', + 'winsound': 'winsound#module-winsound', + 'wsgiref': 'wsgiref#module-wsgiref', + 'wsgiref.handlers': 'wsgiref#module-wsgiref.handlers', + 'wsgiref.headers': 'wsgiref#module-wsgiref.headers', + 'wsgiref.simple_server': 'wsgiref#module-wsgiref.simple_server', + 'wsgiref.types': 'wsgiref#module-wsgiref.types', + 'wsgiref.util': 'wsgiref#module-wsgiref.util', + 'wsgiref.validate': 'wsgiref#module-wsgiref.validate', + 'xdrlib': 'xdrlib#module-xdrlib', + 'xml': 'xml#module-xml', + 'xml.dom': 'xml.dom#module-xml.dom', + 'xml.dom.minidom': 'xml.dom.minidom#module-xml.dom.minidom', + 'xml.dom.pulldom': 'xml.dom.pulldom#module-xml.dom.pulldom', + 'xml.etree.ElementInclude': 'xml.etree.elementtree#module-xml.etree.ElementInclude', + 'xml.etree.ElementTree': 'xml.etree.elementtree#module-xml.etree.ElementTree', + 'xml.parsers.expat': 'pyexpat#module-xml.parsers.expat', + 'xml.parsers.expat.errors': 'pyexpat#module-xml.parsers.expat.errors', + 'xml.parsers.expat.model': 'pyexpat#module-xml.parsers.expat.model', + 'xml.sax': 'xml.sax#module-xml.sax', + 'xml.sax.handler': 'xml.sax.handler#module-xml.sax.handler', + 'xml.sax.saxutils': 'xml.sax.utils#module-xml.sax.saxutils', + 'xml.sax.xmlreader': 'xml.sax.reader#module-xml.sax.xmlreader', + 'xmlrpc': 'xmlrpc#module-xmlrpc', + 'xmlrpc.client': 'xmlrpc.client#module-xmlrpc.client', + 'xmlrpc.server': 'xmlrpc.server#module-xmlrpc.server', + 'zipapp': 'zipapp#module-zipapp', + 'zipfile': 'zipfile#module-zipfile', + 'zipimport': 'zipimport#module-zipimport', + 'zlib': 'zlib#module-zlib', + 'zoneinfo': 'zoneinfo#module-zoneinfo', +} diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index e4c63058087..4e31cf08bb5 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,13062 +1,14272 @@ -# -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Sun Dec 23 16:24:21 2018 -topics = {'assert': 'The "assert" statement\n' - '**********************\n' - '\n' - 'Assert statements are a convenient way to insert debugging ' - 'assertions\n' - 'into a program:\n' - '\n' - ' assert_stmt ::= "assert" expression ["," expression]\n' - '\n' - 'The simple form, "assert expression", is equivalent to\n' - '\n' - ' if __debug__:\n' - ' if not expression: raise AssertionError\n' - '\n' - 'The extended form, "assert expression1, expression2", is ' - 'equivalent to\n' - '\n' - ' if __debug__:\n' - ' if not expression1: raise AssertionError(expression2)\n' - '\n' - 'These equivalences assume that "__debug__" and "AssertionError" ' - 'refer\n' - 'to the built-in variables with those names. In the current\n' - 'implementation, the built-in variable "__debug__" is "True" under\n' - 'normal circumstances, "False" when optimization is requested ' - '(command\n' - 'line option "-O"). The current code generator emits no code for ' - 'an\n' - 'assert statement when optimization is requested at compile time. ' - 'Note\n' - 'that it is unnecessary to include the source code for the ' - 'expression\n' - 'that failed in the error message; it will be displayed as part of ' - 'the\n' - 'stack trace.\n' - '\n' - 'Assignments to "__debug__" are illegal. The value for the ' - 'built-in\n' - 'variable is determined when the interpreter starts.\n', - 'assignment': 'Assignment statements\n' - '*********************\n' - '\n' - 'Assignment statements are used to (re)bind names to values and ' - 'to\n' - 'modify attributes or items of mutable objects:\n' - '\n' - ' assignment_stmt ::= (target_list "=")+ (starred_expression ' - '| yield_expression)\n' - ' target_list ::= target ("," target)* [","]\n' - ' target ::= identifier\n' - ' | "(" [target_list] ")"\n' - ' | "[" [target_list] "]"\n' - ' | attributeref\n' - ' | subscription\n' - ' | slicing\n' - ' | "*" target\n' - '\n' - '(See section Primaries for the syntax definitions for ' - '*attributeref*,\n' - '*subscription*, and *slicing*.)\n' - '\n' - 'An assignment statement evaluates the expression list ' - '(remember that\n' - 'this can be a single expression or a comma-separated list, the ' - 'latter\n' - 'yielding a tuple) and assigns the single resulting object to ' - 'each of\n' - 'the target lists, from left to right.\n' - '\n' - 'Assignment is defined recursively depending on the form of the ' - 'target\n' - '(list). When a target is part of a mutable object (an ' - 'attribute\n' - 'reference, subscription or slicing), the mutable object must\n' - 'ultimately perform the assignment and decide about its ' - 'validity, and\n' - 'may raise an exception if the assignment is unacceptable. The ' - 'rules\n' - 'observed by various types and the exceptions raised are given ' - 'with the\n' - 'definition of the object types (see section The standard type\n' - 'hierarchy).\n' - '\n' - 'Assignment of an object to a target list, optionally enclosed ' - 'in\n' - 'parentheses or square brackets, is recursively defined as ' - 'follows.\n' - '\n' - '* If the target list is a single target with no trailing ' - 'comma,\n' - ' optionally in parentheses, the object is assigned to that ' - 'target.\n' - '\n' - '* Else: The object must be an iterable with the same number of ' - 'items\n' - ' as there are targets in the target list, and the items are ' - 'assigned,\n' - ' from left to right, to the corresponding targets.\n' - '\n' - ' * If the target list contains one target prefixed with an\n' - ' asterisk, called a “starred” target: The object must be ' - 'an\n' - ' iterable with at least as many items as there are targets ' - 'in the\n' - ' target list, minus one. The first items of the iterable ' - 'are\n' - ' assigned, from left to right, to the targets before the ' - 'starred\n' - ' target. The final items of the iterable are assigned to ' - 'the\n' - ' targets after the starred target. A list of the remaining ' - 'items\n' - ' in the iterable is then assigned to the starred target ' - '(the list\n' - ' can be empty).\n' - '\n' - ' * Else: The object must be an iterable with the same number ' - 'of\n' - ' items as there are targets in the target list, and the ' - 'items are\n' - ' assigned, from left to right, to the corresponding ' - 'targets.\n' - '\n' - 'Assignment of an object to a single target is recursively ' - 'defined as\n' - 'follows.\n' - '\n' - '* If the target is an identifier (name):\n' - '\n' - ' * If the name does not occur in a "global" or "nonlocal" ' - 'statement\n' - ' in the current code block: the name is bound to the object ' - 'in the\n' - ' current local namespace.\n' - '\n' - ' * Otherwise: the name is bound to the object in the global\n' - ' namespace or the outer namespace determined by ' - '"nonlocal",\n' - ' respectively.\n' - '\n' - ' The name is rebound if it was already bound. This may cause ' - 'the\n' - ' reference count for the object previously bound to the name ' - 'to reach\n' - ' zero, causing the object to be deallocated and its ' - 'destructor (if it\n' - ' has one) to be called.\n' - '\n' - '* If the target is an attribute reference: The primary ' - 'expression in\n' - ' the reference is evaluated. It should yield an object with\n' - ' assignable attributes; if this is not the case, "TypeError" ' - 'is\n' - ' raised. That object is then asked to assign the assigned ' - 'object to\n' - ' the given attribute; if it cannot perform the assignment, it ' - 'raises\n' - ' an exception (usually but not necessarily ' - '"AttributeError").\n' - '\n' - ' Note: If the object is a class instance and the attribute ' - 'reference\n' - ' occurs on both sides of the assignment operator, the RHS ' - 'expression,\n' - ' "a.x" can access either an instance attribute or (if no ' - 'instance\n' - ' attribute exists) a class attribute. The LHS target "a.x" ' - 'is always\n' - ' set as an instance attribute, creating it if necessary. ' - 'Thus, the\n' - ' two occurrences of "a.x" do not necessarily refer to the ' - 'same\n' - ' attribute: if the RHS expression refers to a class ' - 'attribute, the\n' - ' LHS creates a new instance attribute as the target of the\n' - ' assignment:\n' - '\n' - ' class Cls:\n' - ' x = 3 # class variable\n' - ' inst = Cls()\n' - ' inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x ' - 'as 3\n' - '\n' - ' This description does not necessarily apply to descriptor\n' - ' attributes, such as properties created with "property()".\n' - '\n' - '* If the target is a subscription: The primary expression in ' - 'the\n' - ' reference is evaluated. It should yield either a mutable ' - 'sequence\n' - ' object (such as a list) or a mapping object (such as a ' - 'dictionary).\n' - ' Next, the subscript expression is evaluated.\n' - '\n' - ' If the primary is a mutable sequence object (such as a ' - 'list), the\n' - ' subscript must yield an integer. If it is negative, the ' - 'sequence’s\n' - ' length is added to it. The resulting value must be a ' - 'nonnegative\n' - ' integer less than the sequence’s length, and the sequence is ' - 'asked\n' - ' to assign the assigned object to its item with that index. ' - 'If the\n' - ' index is out of range, "IndexError" is raised (assignment to ' - 'a\n' - ' subscripted sequence cannot add new items to a list).\n' - '\n' - ' If the primary is a mapping object (such as a dictionary), ' - 'the\n' - ' subscript must have a type compatible with the mapping’s key ' - 'type,\n' - ' and the mapping is then asked to create a key/datum pair ' - 'which maps\n' - ' the subscript to the assigned object. This can either ' - 'replace an\n' - ' existing key/value pair with the same key value, or insert a ' - 'new\n' - ' key/value pair (if no key with the same value existed).\n' - '\n' - ' For user-defined objects, the "__setitem__()" method is ' - 'called with\n' - ' appropriate arguments.\n' - '\n' - '* If the target is a slicing: The primary expression in the\n' - ' reference is evaluated. It should yield a mutable sequence ' - 'object\n' - ' (such as a list). The assigned object should be a sequence ' - 'object\n' - ' of the same type. Next, the lower and upper bound ' - 'expressions are\n' - ' evaluated, insofar they are present; defaults are zero and ' - 'the\n' - ' sequence’s length. The bounds should evaluate to integers. ' - 'If\n' - ' either bound is negative, the sequence’s length is added to ' - 'it. The\n' - ' resulting bounds are clipped to lie between zero and the ' - 'sequence’s\n' - ' length, inclusive. Finally, the sequence object is asked to ' - 'replace\n' - ' the slice with the items of the assigned sequence. The ' - 'length of\n' - ' the slice may be different from the length of the assigned ' - 'sequence,\n' - ' thus changing the length of the target sequence, if the ' - 'target\n' - ' sequence allows it.\n' - '\n' - '**CPython implementation detail:** In the current ' - 'implementation, the\n' - 'syntax for targets is taken to be the same as for expressions, ' - 'and\n' - 'invalid syntax is rejected during the code generation phase, ' - 'causing\n' - 'less detailed error messages.\n' - '\n' - 'Although the definition of assignment implies that overlaps ' - 'between\n' - 'the left-hand side and the right-hand side are ‘simultaneous’ ' - '(for\n' - 'example "a, b = b, a" swaps two variables), overlaps *within* ' - 'the\n' - 'collection of assigned-to variables occur left-to-right, ' - 'sometimes\n' - 'resulting in confusion. For instance, the following program ' - 'prints\n' - '"[0, 2]":\n' - '\n' - ' x = [0, 1]\n' - ' i = 0\n' - ' i, x[i] = 1, 2 # i is updated, then x[i] is ' - 'updated\n' - ' print(x)\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3132** - Extended Iterable Unpacking\n' - ' The specification for the "*target" feature.\n' - '\n' - '\n' - 'Augmented assignment statements\n' - '===============================\n' - '\n' - 'Augmented assignment is the combination, in a single ' - 'statement, of a\n' - 'binary operation and an assignment statement:\n' - '\n' - ' augmented_assignment_stmt ::= augtarget augop ' - '(expression_list | yield_expression)\n' - ' augtarget ::= identifier | attributeref | ' - 'subscription | slicing\n' - ' augop ::= "+=" | "-=" | "*=" | "@=" | ' - '"/=" | "//=" | "%=" | "**="\n' - ' | ">>=" | "<<=" | "&=" | "^=" | "|="\n' - '\n' - '(See section Primaries for the syntax definitions of the last ' - 'three\n' - 'symbols.)\n' - '\n' - 'An augmented assignment evaluates the target (which, unlike ' - 'normal\n' - 'assignment statements, cannot be an unpacking) and the ' - 'expression\n' - 'list, performs the binary operation specific to the type of ' - 'assignment\n' - 'on the two operands, and assigns the result to the original ' - 'target.\n' - 'The target is only evaluated once.\n' - '\n' - 'An augmented assignment expression like "x += 1" can be ' - 'rewritten as\n' - '"x = x + 1" to achieve a similar, but not exactly equal ' - 'effect. In the\n' - 'augmented version, "x" is only evaluated once. Also, when ' - 'possible,\n' - 'the actual operation is performed *in-place*, meaning that ' - 'rather than\n' - 'creating a new object and assigning that to the target, the ' - 'old object\n' - 'is modified instead.\n' - '\n' - 'Unlike normal assignments, augmented assignments evaluate the ' - 'left-\n' - 'hand side *before* evaluating the right-hand side. For ' - 'example, "a[i]\n' - '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and ' - 'performs\n' - 'the addition, and lastly, it writes the result back to ' - '"a[i]".\n' - '\n' - 'With the exception of assigning to tuples and multiple targets ' - 'in a\n' - 'single statement, the assignment done by augmented assignment\n' - 'statements is handled the same way as normal assignments. ' - 'Similarly,\n' - 'with the exception of the possible *in-place* behavior, the ' - 'binary\n' - 'operation performed by augmented assignment is the same as the ' - 'normal\n' - 'binary operations.\n' - '\n' - 'For targets which are attribute references, the same caveat ' - 'about\n' - 'class and instance attributes applies as for regular ' - 'assignments.\n' - '\n' - '\n' - 'Annotated assignment statements\n' - '===============================\n' - '\n' - 'Annotation assignment is the combination, in a single ' - 'statement, of a\n' - 'variable or attribute annotation and an optional assignment ' - 'statement:\n' - '\n' - ' annotated_assignment_stmt ::= augtarget ":" expression ["=" ' - 'expression]\n' - '\n' - 'The difference from normal Assignment statements is that only ' - 'single\n' - 'target and only single right hand side value is allowed.\n' - '\n' - 'For simple names as assignment targets, if in class or module ' - 'scope,\n' - 'the annotations are evaluated and stored in a special class or ' - 'module\n' - 'attribute "__annotations__" that is a dictionary mapping from ' - 'variable\n' - 'names (mangled if private) to evaluated annotations. This ' - 'attribute is\n' - 'writable and is automatically created at the start of class or ' - 'module\n' - 'body execution, if annotations are found statically.\n' - '\n' - 'For expressions as assignment targets, the annotations are ' - 'evaluated\n' - 'if in class or module scope, but not stored.\n' - '\n' - 'If a name is annotated in a function scope, then this name is ' - 'local\n' - 'for that scope. Annotations are never evaluated and stored in ' - 'function\n' - 'scopes.\n' - '\n' - 'If the right hand side is present, an annotated assignment ' - 'performs\n' - 'the actual assignment before evaluating annotations (where\n' - 'applicable). If the right hand side is not present for an ' - 'expression\n' - 'target, then the interpreter evaluates the target except for ' - 'the last\n' - '"__setitem__()" or "__setattr__()" call.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 526** - Syntax for Variable Annotations\n' - ' The proposal that added syntax for annotating the types ' - 'of\n' - ' variables (including class variables and instance ' - 'variables),\n' - ' instead of expressing them through comments.\n' - '\n' - ' **PEP 484** - Type hints\n' - ' The proposal that added the "typing" module to provide a ' - 'standard\n' - ' syntax for type annotations that can be used in static ' - 'analysis\n' - ' tools and IDEs.\n', - 'atom-identifiers': 'Identifiers (Names)\n' - '*******************\n' - '\n' - 'An identifier occurring as an atom is a name. See ' - 'section Identifiers\n' - 'and keywords for lexical definition and section Naming ' - 'and binding for\n' - 'documentation of naming and binding.\n' - '\n' - 'When the name is bound to an object, evaluation of the ' - 'atom yields\n' - 'that object. When a name is not bound, an attempt to ' - 'evaluate it\n' - 'raises a "NameError" exception.\n' - '\n' - '**Private name mangling:** When an identifier that ' - 'textually occurs in\n' - 'a class definition begins with two or more underscore ' - 'characters and\n' - 'does not end in two or more underscores, it is ' - 'considered a *private\n' - 'name* of that class. Private names are transformed to a ' - 'longer form\n' - 'before code is generated for them. The transformation ' - 'inserts the\n' - 'class name, with leading underscores removed and a ' - 'single underscore\n' - 'inserted, in front of the name. For example, the ' - 'identifier "__spam"\n' - 'occurring in a class named "Ham" will be transformed to ' - '"_Ham__spam".\n' - 'This transformation is independent of the syntactical ' - 'context in which\n' - 'the identifier is used. If the transformed name is ' - 'extremely long\n' - '(longer than 255 characters), implementation defined ' - 'truncation may\n' - 'happen. If the class name consists only of underscores, ' - 'no\n' - 'transformation is done.\n', - 'atom-literals': 'Literals\n' - '********\n' - '\n' - 'Python supports string and bytes literals and various ' - 'numeric\n' - 'literals:\n' - '\n' - ' literal ::= stringliteral | bytesliteral\n' - ' | integer | floatnumber | imagnumber\n' - '\n' - 'Evaluation of a literal yields an object of the given type ' - '(string,\n' - 'bytes, integer, floating point number, complex number) with ' - 'the given\n' - 'value. The value may be approximated in the case of ' - 'floating point\n' - 'and imaginary (complex) literals. See section Literals for ' - 'details.\n' - '\n' - 'All literals correspond to immutable data types, and hence ' - 'the\n' - 'object’s identity is less important than its value. ' - 'Multiple\n' - 'evaluations of literals with the same value (either the ' - 'same\n' - 'occurrence in the program text or a different occurrence) ' - 'may obtain\n' - 'the same object or a different object with the same ' - 'value.\n', - 'attribute-access': 'Customizing attribute access\n' - '****************************\n' - '\n' - 'The following methods can be defined to customize the ' - 'meaning of\n' - 'attribute access (use of, assignment to, or deletion of ' - '"x.name") for\n' - 'class instances.\n' - '\n' - 'object.__getattr__(self, name)\n' - '\n' - ' Called when the default attribute access fails with ' - 'an\n' - ' "AttributeError" (either "__getattribute__()" raises ' - 'an\n' - ' "AttributeError" because *name* is not an instance ' - 'attribute or an\n' - ' attribute in the class tree for "self"; or ' - '"__get__()" of a *name*\n' - ' property raises "AttributeError"). This method ' - 'should either\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - ' Note that if the attribute is found through the ' - 'normal mechanism,\n' - ' "__getattr__()" is not called. (This is an ' - 'intentional asymmetry\n' - ' between "__getattr__()" and "__setattr__()".) This is ' - 'done both for\n' - ' efficiency reasons and because otherwise ' - '"__getattr__()" would have\n' - ' no way to access other attributes of the instance. ' - 'Note that at\n' - ' least for instance variables, you can fake total ' - 'control by not\n' - ' inserting any values in the instance attribute ' - 'dictionary (but\n' - ' instead inserting them in another object). See the\n' - ' "__getattribute__()" method below for a way to ' - 'actually get total\n' - ' control over attribute access.\n' - '\n' - 'object.__getattribute__(self, name)\n' - '\n' - ' Called unconditionally to implement attribute ' - 'accesses for\n' - ' instances of the class. If the class also defines ' - '"__getattr__()",\n' - ' the latter will not be called unless ' - '"__getattribute__()" either\n' - ' calls it explicitly or raises an "AttributeError". ' - 'This method\n' - ' should return the (computed) attribute value or raise ' - 'an\n' - ' "AttributeError" exception. In order to avoid ' - 'infinite recursion in\n' - ' this method, its implementation should always call ' - 'the base class\n' - ' method with the same name to access any attributes it ' - 'needs, for\n' - ' example, "object.__getattribute__(self, name)".\n' - '\n' - ' Note: This method may still be bypassed when looking ' - 'up special\n' - ' methods as the result of implicit invocation via ' - 'language syntax\n' - ' or built-in functions. See Special method lookup.\n' - '\n' - 'object.__setattr__(self, name, value)\n' - '\n' - ' Called when an attribute assignment is attempted. ' - 'This is called\n' - ' instead of the normal mechanism (i.e. store the value ' - 'in the\n' - ' instance dictionary). *name* is the attribute name, ' - '*value* is the\n' - ' value to be assigned to it.\n' - '\n' - ' If "__setattr__()" wants to assign to an instance ' - 'attribute, it\n' - ' should call the base class method with the same name, ' - 'for example,\n' - ' "object.__setattr__(self, name, value)".\n' - '\n' - 'object.__delattr__(self, name)\n' - '\n' - ' Like "__setattr__()" but for attribute deletion ' - 'instead of\n' - ' assignment. This should only be implemented if "del ' - 'obj.name" is\n' - ' meaningful for the object.\n' - '\n' - 'object.__dir__(self)\n' - '\n' - ' Called when "dir()" is called on the object. A ' - 'sequence must be\n' - ' returned. "dir()" converts the returned sequence to a ' - 'list and\n' - ' sorts it.\n' - '\n' - '\n' - 'Customizing module attribute access\n' - '===================================\n' - '\n' - 'For a more fine grained customization of the module ' - 'behavior (setting\n' - 'attributes, properties, etc.), one can set the ' - '"__class__" attribute\n' - 'of a module object to a subclass of "types.ModuleType". ' - 'For example:\n' - '\n' - ' import sys\n' - ' from types import ModuleType\n' - '\n' - ' class VerboseModule(ModuleType):\n' - ' def __repr__(self):\n' - " return f'Verbose {self.__name__}'\n" - '\n' - ' def __setattr__(self, attr, value):\n' - " print(f'Setting {attr}...')\n" - ' setattr(self, attr, value)\n' - '\n' - ' sys.modules[__name__].__class__ = VerboseModule\n' - '\n' - 'Note: Setting module "__class__" only affects lookups ' - 'made using the\n' - ' attribute access syntax – directly accessing the ' - 'module globals\n' - ' (whether by code within the module, or via a reference ' - 'to the\n' - ' module’s globals dictionary) is unaffected.\n' - '\n' - 'Changed in version 3.5: "__class__" module attribute is ' - 'now writable.\n' - '\n' - '\n' - 'Implementing Descriptors\n' - '========================\n' - '\n' - 'The following methods only apply when an instance of the ' - 'class\n' - 'containing the method (a so-called *descriptor* class) ' - 'appears in an\n' - '*owner* class (the descriptor must be in either the ' - 'owner’s class\n' - 'dictionary or in the class dictionary for one of its ' - 'parents). In the\n' - 'examples below, “the attribute” refers to the attribute ' - 'whose name is\n' - 'the key of the property in the owner class’ "__dict__".\n' - '\n' - 'object.__get__(self, instance, owner)\n' - '\n' - ' Called to get the attribute of the owner class (class ' - 'attribute\n' - ' access) or of an instance of that class (instance ' - 'attribute\n' - ' access). *owner* is always the owner class, while ' - '*instance* is the\n' - ' instance that the attribute was accessed through, or ' - '"None" when\n' - ' the attribute is accessed through the *owner*. This ' - 'method should\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - 'object.__set__(self, instance, value)\n' - '\n' - ' Called to set the attribute on an instance *instance* ' - 'of the owner\n' - ' class to a new value, *value*.\n' - '\n' - 'object.__delete__(self, instance)\n' - '\n' - ' Called to delete the attribute on an instance ' - '*instance* of the\n' - ' owner class.\n' - '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Called at the time the owning class *owner* is ' - 'created. The\n' - ' descriptor has been assigned to *name*.\n' - '\n' - ' New in version 3.6.\n' - '\n' - 'The attribute "__objclass__" is interpreted by the ' - '"inspect" module as\n' - 'specifying the class where this object was defined ' - '(setting this\n' - 'appropriately can assist in runtime introspection of ' - 'dynamic class\n' - 'attributes). For callables, it may indicate that an ' - 'instance of the\n' - 'given type (or a subclass) is expected or required as ' - 'the first\n' - 'positional argument (for example, CPython sets this ' - 'attribute for\n' - 'unbound methods that are implemented in C).\n' - '\n' - '\n' - 'Invoking Descriptors\n' - '====================\n' - '\n' - 'In general, a descriptor is an object attribute with ' - '“binding\n' - 'behavior”, one whose attribute access has been ' - 'overridden by methods\n' - 'in the descriptor protocol: "__get__()", "__set__()", ' - 'and\n' - '"__delete__()". If any of those methods are defined for ' - 'an object, it\n' - 'is said to be a descriptor.\n' - '\n' - 'The default behavior for attribute access is to get, ' - 'set, or delete\n' - 'the attribute from an object’s dictionary. For instance, ' - '"a.x" has a\n' - 'lookup chain starting with "a.__dict__[\'x\']", then\n' - '"type(a).__dict__[\'x\']", and continuing through the ' - 'base classes of\n' - '"type(a)" excluding metaclasses.\n' - '\n' - 'However, if the looked-up value is an object defining ' - 'one of the\n' - 'descriptor methods, then Python may override the default ' - 'behavior and\n' - 'invoke the descriptor method instead. Where this occurs ' - 'in the\n' - 'precedence chain depends on which descriptor methods ' - 'were defined and\n' - 'how they were called.\n' - '\n' - 'The starting point for descriptor invocation is a ' - 'binding, "a.x". How\n' - 'the arguments are assembled depends on "a":\n' - '\n' - 'Direct Call\n' - ' The simplest and least common call is when user code ' - 'directly\n' - ' invokes a descriptor method: "x.__get__(a)".\n' - '\n' - 'Instance Binding\n' - ' If binding to an object instance, "a.x" is ' - 'transformed into the\n' - ' call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n' - '\n' - 'Class Binding\n' - ' If binding to a class, "A.x" is transformed into the ' - 'call:\n' - ' "A.__dict__[\'x\'].__get__(None, A)".\n' - '\n' - 'Super Binding\n' - ' If "a" is an instance of "super", then the binding ' - '"super(B,\n' - ' obj).m()" searches "obj.__class__.__mro__" for the ' - 'base class "A"\n' - ' immediately preceding "B" and then invokes the ' - 'descriptor with the\n' - ' call: "A.__dict__[\'m\'].__get__(obj, ' - 'obj.__class__)".\n' - '\n' - 'For instance bindings, the precedence of descriptor ' - 'invocation depends\n' - 'on the which descriptor methods are defined. A ' - 'descriptor can define\n' - 'any combination of "__get__()", "__set__()" and ' - '"__delete__()". If it\n' - 'does not define "__get__()", then accessing the ' - 'attribute will return\n' - 'the descriptor object itself unless there is a value in ' - 'the object’s\n' - 'instance dictionary. If the descriptor defines ' - '"__set__()" and/or\n' - '"__delete__()", it is a data descriptor; if it defines ' - 'neither, it is\n' - 'a non-data descriptor. Normally, data descriptors ' - 'define both\n' - '"__get__()" and "__set__()", while non-data descriptors ' - 'have just the\n' - '"__get__()" method. Data descriptors with "__set__()" ' - 'and "__get__()"\n' - 'defined always override a redefinition in an instance ' - 'dictionary. In\n' - 'contrast, non-data descriptors can be overridden by ' - 'instances.\n' - '\n' - 'Python methods (including "staticmethod()" and ' - '"classmethod()") are\n' - 'implemented as non-data descriptors. Accordingly, ' - 'instances can\n' - 'redefine and override methods. This allows individual ' - 'instances to\n' - 'acquire behaviors that differ from other instances of ' - 'the same class.\n' - '\n' - 'The "property()" function is implemented as a data ' - 'descriptor.\n' - 'Accordingly, instances cannot override the behavior of a ' - 'property.\n' - '\n' - '\n' - '__slots__\n' - '=========\n' - '\n' - '*__slots__* allow us to explicitly declare data members ' - '(like\n' - 'properties) and deny the creation of *__dict__* and ' - '*__weakref__*\n' - '(unless explicitly declared in *__slots__* or available ' - 'in a parent.)\n' - '\n' - 'The space saved over using *__dict__* can be ' - 'significant.\n' - '\n' - 'object.__slots__\n' - '\n' - ' This class variable can be assigned a string, ' - 'iterable, or sequence\n' - ' of strings with variable names used by instances. ' - '*__slots__*\n' - ' reserves space for the declared variables and ' - 'prevents the\n' - ' automatic creation of *__dict__* and *__weakref__* ' - 'for each\n' - ' instance.\n' - '\n' - '\n' - 'Notes on using *__slots__*\n' - '--------------------------\n' - '\n' - '* When inheriting from a class without *__slots__*, the ' - '*__dict__*\n' - ' and *__weakref__* attribute of the instances will ' - 'always be\n' - ' accessible.\n' - '\n' - '* Without a *__dict__* variable, instances cannot be ' - 'assigned new\n' - ' variables not listed in the *__slots__* definition. ' - 'Attempts to\n' - ' assign to an unlisted variable name raises ' - '"AttributeError". If\n' - ' dynamic assignment of new variables is desired, then ' - 'add\n' - ' "\'__dict__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* Without a *__weakref__* variable for each instance, ' - 'classes\n' - ' defining *__slots__* do not support weak references to ' - 'its\n' - ' instances. If weak reference support is needed, then ' - 'add\n' - ' "\'__weakref__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* *__slots__* are implemented at the class level by ' - 'creating\n' - ' descriptors (Implementing Descriptors) for each ' - 'variable name. As a\n' - ' result, class attributes cannot be used to set default ' - 'values for\n' - ' instance variables defined by *__slots__*; otherwise, ' - 'the class\n' - ' attribute would overwrite the descriptor assignment.\n' - '\n' - '* The action of a *__slots__* declaration is not limited ' - 'to the\n' - ' class where it is defined. *__slots__* declared in ' - 'parents are\n' - ' available in child classes. However, child subclasses ' - 'will get a\n' - ' *__dict__* and *__weakref__* unless they also define ' - '*__slots__*\n' - ' (which should only contain names of any *additional* ' - 'slots).\n' - '\n' - '* If a class defines a slot also defined in a base ' - 'class, the\n' - ' instance variable defined by the base class slot is ' - 'inaccessible\n' - ' (except by retrieving its descriptor directly from the ' - 'base class).\n' - ' This renders the meaning of the program undefined. In ' - 'the future, a\n' - ' check may be added to prevent this.\n' - '\n' - '* Nonempty *__slots__* does not work for classes derived ' - 'from\n' - ' “variable-length” built-in types such as "int", ' - '"bytes" and "tuple".\n' - '\n' - '* Any non-string iterable may be assigned to ' - '*__slots__*. Mappings\n' - ' may also be used; however, in the future, special ' - 'meaning may be\n' - ' assigned to the values corresponding to each key.\n' - '\n' - '* *__class__* assignment works only if both classes have ' - 'the same\n' - ' *__slots__*.\n' - '\n' - '* Multiple inheritance with multiple slotted parent ' - 'classes can be\n' - ' used, but only one parent is allowed to have ' - 'attributes created by\n' - ' slots (the other bases must have empty slot layouts) - ' - 'violations\n' - ' raise "TypeError".\n', - 'attribute-references': 'Attribute references\n' - '********************\n' - '\n' - 'An attribute reference is a primary followed by a ' - 'period and a name:\n' - '\n' - ' attributeref ::= primary "." identifier\n' - '\n' - 'The primary must evaluate to an object of a type ' - 'that supports\n' - 'attribute references, which most objects do. This ' - 'object is then\n' - 'asked to produce the attribute whose name is the ' - 'identifier. This\n' - 'production can be customized by overriding the ' - '"__getattr__()" method.\n' - 'If this attribute is not available, the exception ' - '"AttributeError" is\n' - 'raised. Otherwise, the type and value of the object ' - 'produced is\n' - 'determined by the object. Multiple evaluations of ' - 'the same attribute\n' - 'reference may yield different objects.\n', - 'augassign': 'Augmented assignment statements\n' - '*******************************\n' - '\n' - 'Augmented assignment is the combination, in a single statement, ' - 'of a\n' - 'binary operation and an assignment statement:\n' - '\n' - ' augmented_assignment_stmt ::= augtarget augop ' - '(expression_list | yield_expression)\n' - ' augtarget ::= identifier | attributeref | ' - 'subscription | slicing\n' - ' augop ::= "+=" | "-=" | "*=" | "@=" | ' - '"/=" | "//=" | "%=" | "**="\n' - ' | ">>=" | "<<=" | "&=" | "^=" | "|="\n' - '\n' - '(See section Primaries for the syntax definitions of the last ' - 'three\n' - 'symbols.)\n' - '\n' - 'An augmented assignment evaluates the target (which, unlike ' - 'normal\n' - 'assignment statements, cannot be an unpacking) and the ' - 'expression\n' - 'list, performs the binary operation specific to the type of ' - 'assignment\n' - 'on the two operands, and assigns the result to the original ' - 'target.\n' - 'The target is only evaluated once.\n' - '\n' - 'An augmented assignment expression like "x += 1" can be ' - 'rewritten as\n' - '"x = x + 1" to achieve a similar, but not exactly equal effect. ' - 'In the\n' - 'augmented version, "x" is only evaluated once. Also, when ' - 'possible,\n' - 'the actual operation is performed *in-place*, meaning that ' - 'rather than\n' - 'creating a new object and assigning that to the target, the old ' - 'object\n' - 'is modified instead.\n' - '\n' - 'Unlike normal assignments, augmented assignments evaluate the ' - 'left-\n' - 'hand side *before* evaluating the right-hand side. For ' - 'example, "a[i]\n' - '+= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and ' - 'performs\n' - 'the addition, and lastly, it writes the result back to "a[i]".\n' - '\n' - 'With the exception of assigning to tuples and multiple targets ' - 'in a\n' - 'single statement, the assignment done by augmented assignment\n' - 'statements is handled the same way as normal assignments. ' - 'Similarly,\n' - 'with the exception of the possible *in-place* behavior, the ' - 'binary\n' - 'operation performed by augmented assignment is the same as the ' - 'normal\n' - 'binary operations.\n' - '\n' - 'For targets which are attribute references, the same caveat ' - 'about\n' - 'class and instance attributes applies as for regular ' - 'assignments.\n', - 'binary': 'Binary arithmetic operations\n' - '****************************\n' - '\n' - 'The binary arithmetic operations have the conventional priority\n' - 'levels. Note that some of these operations also apply to certain ' - 'non-\n' - 'numeric types. Apart from the power operator, there are only two\n' - 'levels, one for multiplicative operators and one for additive\n' - 'operators:\n' - '\n' - ' m_expr ::= u_expr | m_expr "*" u_expr | m_expr "@" m_expr |\n' - ' m_expr "//" u_expr | m_expr "/" u_expr |\n' - ' m_expr "%" u_expr\n' - ' a_expr ::= m_expr | a_expr "+" m_expr | a_expr "-" m_expr\n' - '\n' - 'The "*" (multiplication) operator yields the product of its ' - 'arguments.\n' - 'The arguments must either both be numbers, or one argument must be ' - 'an\n' - 'integer and the other must be a sequence. In the former case, the\n' - 'numbers are converted to a common type and then multiplied ' - 'together.\n' - 'In the latter case, sequence repetition is performed; a negative\n' - 'repetition factor yields an empty sequence.\n' - '\n' - 'The "@" (at) operator is intended to be used for matrix\n' - 'multiplication. No builtin Python types implement this operator.\n' - '\n' - 'New in version 3.5.\n' - '\n' - 'The "/" (division) and "//" (floor division) operators yield the\n' - 'quotient of their arguments. The numeric arguments are first\n' - 'converted to a common type. Division of integers yields a float, ' - 'while\n' - 'floor division of integers results in an integer; the result is ' - 'that\n' - 'of mathematical division with the ‘floor’ function applied to the\n' - 'result. Division by zero raises the "ZeroDivisionError" ' - 'exception.\n' - '\n' - 'The "%" (modulo) operator yields the remainder from the division ' - 'of\n' - 'the first argument by the second. The numeric arguments are ' - 'first\n' - 'converted to a common type. A zero right argument raises the\n' - '"ZeroDivisionError" exception. The arguments may be floating ' - 'point\n' - 'numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals ' - '"4*0.7 +\n' - '0.34".) The modulo operator always yields a result with the same ' - 'sign\n' - 'as its second operand (or zero); the absolute value of the result ' - 'is\n' - 'strictly smaller than the absolute value of the second operand ' - '[1].\n' - '\n' - 'The floor division and modulo operators are connected by the ' - 'following\n' - 'identity: "x == (x//y)*y + (x%y)". Floor division and modulo are ' - 'also\n' - 'connected with the built-in function "divmod()": "divmod(x, y) ==\n' - '(x//y, x%y)". [2].\n' - '\n' - 'In addition to performing the modulo operation on numbers, the ' - '"%"\n' - 'operator is also overloaded by string objects to perform ' - 'old-style\n' - 'string formatting (also known as interpolation). The syntax for\n' - 'string formatting is described in the Python Library Reference,\n' - 'section printf-style String Formatting.\n' - '\n' - 'The floor division operator, the modulo operator, and the ' - '"divmod()"\n' - 'function are not defined for complex numbers. Instead, convert to ' - 'a\n' - 'floating point number using the "abs()" function if appropriate.\n' - '\n' - 'The "+" (addition) operator yields the sum of its arguments. The\n' - 'arguments must either both be numbers or both be sequences of the ' - 'same\n' - 'type. In the former case, the numbers are converted to a common ' - 'type\n' - 'and then added together. In the latter case, the sequences are\n' - 'concatenated.\n' - '\n' - 'The "-" (subtraction) operator yields the difference of its ' - 'arguments.\n' - 'The numeric arguments are first converted to a common type.\n', - 'bitwise': 'Binary bitwise operations\n' - '*************************\n' - '\n' - 'Each of the three bitwise operations has a different priority ' - 'level:\n' - '\n' - ' and_expr ::= shift_expr | and_expr "&" shift_expr\n' - ' xor_expr ::= and_expr | xor_expr "^" and_expr\n' - ' or_expr ::= xor_expr | or_expr "|" xor_expr\n' - '\n' - 'The "&" operator yields the bitwise AND of its arguments, which ' - 'must\n' - 'be integers.\n' - '\n' - 'The "^" operator yields the bitwise XOR (exclusive OR) of its\n' - 'arguments, which must be integers.\n' - '\n' - 'The "|" operator yields the bitwise (inclusive) OR of its ' - 'arguments,\n' - 'which must be integers.\n', - 'bltin-code-objects': 'Code Objects\n' - '************\n' - '\n' - 'Code objects are used by the implementation to ' - 'represent “pseudo-\n' - 'compiled” executable Python code such as a function ' - 'body. They differ\n' - 'from function objects because they don’t contain a ' - 'reference to their\n' - 'global execution environment. Code objects are ' - 'returned by the built-\n' - 'in "compile()" function and can be extracted from ' - 'function objects\n' - 'through their "__code__" attribute. See also the ' - '"code" module.\n' - '\n' - 'A code object can be executed or evaluated by passing ' - 'it (instead of a\n' - 'source string) to the "exec()" or "eval()" built-in ' - 'functions.\n' - '\n' - 'See The standard type hierarchy for more ' - 'information.\n', - 'bltin-ellipsis-object': 'The Ellipsis Object\n' - '*******************\n' - '\n' - 'This object is commonly used by slicing (see ' - 'Slicings). It supports\n' - 'no special operations. There is exactly one ' - 'ellipsis object, named\n' - '"Ellipsis" (a built-in name). "type(Ellipsis)()" ' - 'produces the\n' - '"Ellipsis" singleton.\n' - '\n' - 'It is written as "Ellipsis" or "...".\n', - 'bltin-null-object': 'The Null Object\n' - '***************\n' - '\n' - 'This object is returned by functions that don’t ' - 'explicitly return a\n' - 'value. It supports no special operations. There is ' - 'exactly one null\n' - 'object, named "None" (a built-in name). "type(None)()" ' - 'produces the\n' - 'same singleton.\n' - '\n' - 'It is written as "None".\n', - 'bltin-type-objects': 'Type Objects\n' - '************\n' - '\n' - 'Type objects represent the various object types. An ' - 'object’s type is\n' - 'accessed by the built-in function "type()". There are ' - 'no special\n' - 'operations on types. The standard module "types" ' - 'defines names for\n' - 'all standard built-in types.\n' - '\n' - 'Types are written like this: "".\n', - 'booleans': 'Boolean operations\n' - '******************\n' - '\n' - ' or_test ::= and_test | or_test "or" and_test\n' - ' and_test ::= not_test | and_test "and" not_test\n' - ' not_test ::= comparison | "not" not_test\n' - '\n' - 'In the context of Boolean operations, and also when expressions ' - 'are\n' - 'used by control flow statements, the following values are ' - 'interpreted\n' - 'as false: "False", "None", numeric zero of all types, and empty\n' - 'strings and containers (including strings, tuples, lists,\n' - 'dictionaries, sets and frozensets). All other values are ' - 'interpreted\n' - 'as true. User-defined objects can customize their truth value ' - 'by\n' - 'providing a "__bool__()" method.\n' - '\n' - 'The operator "not" yields "True" if its argument is false, ' - '"False"\n' - 'otherwise.\n' - '\n' - 'The expression "x and y" first evaluates *x*; if *x* is false, ' - 'its\n' - 'value is returned; otherwise, *y* is evaluated and the resulting ' - 'value\n' - 'is returned.\n' - '\n' - 'The expression "x or y" first evaluates *x*; if *x* is true, its ' - 'value\n' - 'is returned; otherwise, *y* is evaluated and the resulting value ' - 'is\n' - 'returned.\n' - '\n' - 'Note that neither "and" nor "or" restrict the value and type ' - 'they\n' - 'return to "False" and "True", but rather return the last ' - 'evaluated\n' - 'argument. This is sometimes useful, e.g., if "s" is a string ' - 'that\n' - 'should be replaced by a default value if it is empty, the ' - 'expression\n' - '"s or \'foo\'" yields the desired value. Because "not" has to ' - 'create a\n' - 'new value, it returns a boolean value regardless of the type of ' - 'its\n' - 'argument (for example, "not \'foo\'" produces "False" rather ' - 'than "\'\'".)\n', - 'break': 'The "break" statement\n' - '*********************\n' - '\n' - ' break_stmt ::= "break"\n' - '\n' - '"break" may only occur syntactically nested in a "for" or "while"\n' - 'loop, but not nested in a function or class definition within that\n' - 'loop.\n' - '\n' - 'It terminates the nearest enclosing loop, skipping the optional ' - '"else"\n' - 'clause if the loop has one.\n' - '\n' - 'If a "for" loop is terminated by "break", the loop control target\n' - 'keeps its current value.\n' - '\n' - 'When "break" passes control out of a "try" statement with a ' - '"finally"\n' - 'clause, that "finally" clause is executed before really leaving ' - 'the\n' - 'loop.\n', - 'callable-types': 'Emulating callable objects\n' - '**************************\n' - '\n' - 'object.__call__(self[, args...])\n' - '\n' - ' Called when the instance is “called” as a function; if ' - 'this method\n' - ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' - ' "x.__call__(arg1, arg2, ...)".\n', - 'calls': 'Calls\n' - '*****\n' - '\n' - 'A call calls a callable object (e.g., a *function*) with a ' - 'possibly\n' - 'empty series of *arguments*:\n' - '\n' - ' call ::= primary "(" [argument_list [","] | ' - 'comprehension] ")"\n' - ' argument_list ::= positional_arguments ["," ' - 'starred_and_keywords]\n' - ' ["," keywords_arguments]\n' - ' | starred_and_keywords ["," ' - 'keywords_arguments]\n' - ' | keywords_arguments\n' - ' positional_arguments ::= ["*"] expression ("," ["*"] ' - 'expression)*\n' - ' starred_and_keywords ::= ("*" expression | keyword_item)\n' - ' ("," "*" expression | "," ' - 'keyword_item)*\n' - ' keywords_arguments ::= (keyword_item | "**" expression)\n' - ' ("," keyword_item | "," "**" ' - 'expression)*\n' - ' keyword_item ::= identifier "=" expression\n' - '\n' - 'An optional trailing comma may be present after the positional and\n' - 'keyword arguments but does not affect the semantics.\n' - '\n' - 'The primary must evaluate to a callable object (user-defined\n' - 'functions, built-in functions, methods of built-in objects, class\n' - 'objects, methods of class instances, and all objects having a\n' - '"__call__()" method are callable). All argument expressions are\n' - 'evaluated before the call is attempted. Please refer to section\n' - 'Function definitions for the syntax of formal *parameter* lists.\n' - '\n' - 'If keyword arguments are present, they are first converted to\n' - 'positional arguments, as follows. First, a list of unfilled slots ' - 'is\n' - 'created for the formal parameters. If there are N positional\n' - 'arguments, they are placed in the first N slots. Next, for each\n' - 'keyword argument, the identifier is used to determine the\n' - 'corresponding slot (if the identifier is the same as the first ' - 'formal\n' - 'parameter name, the first slot is used, and so on). If the slot ' - 'is\n' - 'already filled, a "TypeError" exception is raised. Otherwise, the\n' - 'value of the argument is placed in the slot, filling it (even if ' - 'the\n' - 'expression is "None", it fills the slot). When all arguments have\n' - 'been processed, the slots that are still unfilled are filled with ' - 'the\n' - 'corresponding default value from the function definition. ' - '(Default\n' - 'values are calculated, once, when the function is defined; thus, a\n' - 'mutable object such as a list or dictionary used as default value ' - 'will\n' - 'be shared by all calls that don’t specify an argument value for ' - 'the\n' - 'corresponding slot; this should usually be avoided.) If there are ' - 'any\n' - 'unfilled slots for which no default value is specified, a ' - '"TypeError"\n' - 'exception is raised. Otherwise, the list of filled slots is used ' - 'as\n' - 'the argument list for the call.\n' - '\n' - '**CPython implementation detail:** An implementation may provide\n' - 'built-in functions whose positional parameters do not have names, ' - 'even\n' - 'if they are ‘named’ for the purpose of documentation, and which\n' - 'therefore cannot be supplied by keyword. In CPython, this is the ' - 'case\n' - 'for functions implemented in C that use "PyArg_ParseTuple()" to ' - 'parse\n' - 'their arguments.\n' - '\n' - 'If there are more positional arguments than there are formal ' - 'parameter\n' - 'slots, a "TypeError" exception is raised, unless a formal ' - 'parameter\n' - 'using the syntax "*identifier" is present; in this case, that ' - 'formal\n' - 'parameter receives a tuple containing the excess positional ' - 'arguments\n' - '(or an empty tuple if there were no excess positional arguments).\n' - '\n' - 'If any keyword argument does not correspond to a formal parameter\n' - 'name, a "TypeError" exception is raised, unless a formal parameter\n' - 'using the syntax "**identifier" is present; in this case, that ' - 'formal\n' - 'parameter receives a dictionary containing the excess keyword\n' - 'arguments (using the keywords as keys and the argument values as\n' - 'corresponding values), or a (new) empty dictionary if there were ' - 'no\n' - 'excess keyword arguments.\n' - '\n' - 'If the syntax "*expression" appears in the function call, ' - '"expression"\n' - 'must evaluate to an *iterable*. Elements from these iterables are\n' - 'treated as if they were additional positional arguments. For the ' - 'call\n' - '"f(x1, x2, *y, x3, x4)", if *y* evaluates to a sequence *y1*, …, ' - '*yM*,\n' - 'this is equivalent to a call with M+4 positional arguments *x1*, ' - '*x2*,\n' - '*y1*, …, *yM*, *x3*, *x4*.\n' - '\n' - 'A consequence of this is that although the "*expression" syntax ' - 'may\n' - 'appear *after* explicit keyword arguments, it is processed ' - '*before*\n' - 'the keyword arguments (and any "**expression" arguments – see ' - 'below).\n' - 'So:\n' - '\n' - ' >>> def f(a, b):\n' - ' ... print(a, b)\n' - ' ...\n' - ' >>> f(b=1, *(2,))\n' - ' 2 1\n' - ' >>> f(a=1, *(2,))\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: f() got multiple values for keyword argument 'a'\n" - ' >>> f(1, *(2,))\n' - ' 1 2\n' - '\n' - 'It is unusual for both keyword arguments and the "*expression" ' - 'syntax\n' - 'to be used in the same call, so in practice this confusion does ' - 'not\n' - 'arise.\n' - '\n' - 'If the syntax "**expression" appears in the function call,\n' - '"expression" must evaluate to a *mapping*, the contents of which ' - 'are\n' - 'treated as additional keyword arguments. If a keyword is already\n' - 'present (as an explicit keyword argument, or from another ' - 'unpacking),\n' - 'a "TypeError" exception is raised.\n' - '\n' - 'Formal parameters using the syntax "*identifier" or "**identifier"\n' - 'cannot be used as positional argument slots or as keyword argument\n' - 'names.\n' - '\n' - 'Changed in version 3.5: Function calls accept any number of "*" ' - 'and\n' - '"**" unpackings, positional arguments may follow iterable ' - 'unpackings\n' - '("*"), and keyword arguments may follow dictionary unpackings ' - '("**").\n' - 'Originally proposed by **PEP 448**.\n' - '\n' - 'A call always returns some value, possibly "None", unless it raises ' - 'an\n' - 'exception. How this value is computed depends on the type of the\n' - 'callable object.\n' - '\n' - 'If it is—\n' - '\n' - 'a user-defined function:\n' - ' The code block for the function is executed, passing it the\n' - ' argument list. The first thing the code block will do is bind ' - 'the\n' - ' formal parameters to the arguments; this is described in ' - 'section\n' - ' Function definitions. When the code block executes a "return"\n' - ' statement, this specifies the return value of the function ' - 'call.\n' - '\n' - 'a built-in function or method:\n' - ' The result is up to the interpreter; see Built-in Functions for ' - 'the\n' - ' descriptions of built-in functions and methods.\n' - '\n' - 'a class object:\n' - ' A new instance of that class is returned.\n' - '\n' - 'a class instance method:\n' - ' The corresponding user-defined function is called, with an ' - 'argument\n' - ' list that is one longer than the argument list of the call: the\n' - ' instance becomes the first argument.\n' - '\n' - 'a class instance:\n' - ' The class must define a "__call__()" method; the effect is then ' - 'the\n' - ' same as if that method was called.\n', - 'class': 'Class definitions\n' - '*****************\n' - '\n' - 'A class definition defines a class object (see section The ' - 'standard\n' - 'type hierarchy):\n' - '\n' - ' classdef ::= [decorators] "class" classname [inheritance] ":" ' - 'suite\n' - ' inheritance ::= "(" [argument_list] ")"\n' - ' classname ::= identifier\n' - '\n' - 'A class definition is an executable statement. The inheritance ' - 'list\n' - 'usually gives a list of base classes (see Metaclasses for more\n' - 'advanced uses), so each item in the list should evaluate to a ' - 'class\n' - 'object which allows subclassing. Classes without an inheritance ' - 'list\n' - 'inherit, by default, from the base class "object"; hence,\n' - '\n' - ' class Foo:\n' - ' pass\n' - '\n' - 'is equivalent to\n' - '\n' - ' class Foo(object):\n' - ' pass\n' - '\n' - 'The class’s suite is then executed in a new execution frame (see\n' - 'Naming and binding), using a newly created local namespace and the\n' - 'original global namespace. (Usually, the suite contains mostly\n' - 'function definitions.) When the class’s suite finishes execution, ' - 'its\n' - 'execution frame is discarded but its local namespace is saved. [3] ' - 'A\n' - 'class object is then created using the inheritance list for the ' - 'base\n' - 'classes and the saved local namespace for the attribute ' - 'dictionary.\n' - 'The class name is bound to this class object in the original local\n' - 'namespace.\n' - '\n' - 'The order in which attributes are defined in the class body is\n' - 'preserved in the new class’s "__dict__". Note that this is ' - 'reliable\n' - 'only right after the class is created and only for classes that ' - 'were\n' - 'defined using the definition syntax.\n' - '\n' - 'Class creation can be customized heavily using metaclasses.\n' - '\n' - 'Classes can also be decorated: just like when decorating ' - 'functions,\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' class Foo: pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' class Foo: pass\n' - ' Foo = f1(arg)(f2(Foo))\n' - '\n' - 'The evaluation rules for the decorator expressions are the same as ' - 'for\n' - 'function decorators. The result is then bound to the class name.\n' - '\n' - '**Programmer’s note:** Variables defined in the class definition ' - 'are\n' - 'class attributes; they are shared by instances. Instance ' - 'attributes\n' - 'can be set in a method with "self.name = value". Both class and\n' - 'instance attributes are accessible through the notation ' - '“"self.name"”,\n' - 'and an instance attribute hides a class attribute with the same ' - 'name\n' - 'when accessed in this way. Class attributes can be used as ' - 'defaults\n' - 'for instance attributes, but using mutable values there can lead ' - 'to\n' - 'unexpected results. Descriptors can be used to create instance\n' - 'variables with different implementation details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' The proposal that changed the declaration of metaclasses to ' - 'the\n' - ' current syntax, and the semantics for how classes with\n' - ' metaclasses are constructed.\n' - '\n' - ' **PEP 3129** - Class Decorators\n' - ' The proposal that added class decorators. Function and ' - 'method\n' - ' decorators were introduced in **PEP 318**.\n', - 'comparisons': 'Comparisons\n' - '***********\n' - '\n' - 'Unlike C, all comparison operations in Python have the same ' - 'priority,\n' - 'which is lower than that of any arithmetic, shifting or ' - 'bitwise\n' - 'operation. Also unlike C, expressions like "a < b < c" have ' - 'the\n' - 'interpretation that is conventional in mathematics:\n' - '\n' - ' comparison ::= or_expr (comp_operator or_expr)*\n' - ' comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n' - ' | "is" ["not"] | ["not"] "in"\n' - '\n' - 'Comparisons yield boolean values: "True" or "False".\n' - '\n' - 'Comparisons can be chained arbitrarily, e.g., "x < y <= z" ' - 'is\n' - 'equivalent to "x < y and y <= z", except that "y" is ' - 'evaluated only\n' - 'once (but in both cases "z" is not evaluated at all when "x < ' - 'y" is\n' - 'found to be false).\n' - '\n' - 'Formally, if *a*, *b*, *c*, …, *y*, *z* are expressions and ' - '*op1*,\n' - '*op2*, …, *opN* are comparison operators, then "a op1 b op2 c ' - '... y\n' - 'opN z" is equivalent to "a op1 b and b op2 c and ... y opN ' - 'z", except\n' - 'that each expression is evaluated at most once.\n' - '\n' - 'Note that "a op1 b op2 c" doesn’t imply any kind of ' - 'comparison between\n' - '*a* and *c*, so that, e.g., "x < y > z" is perfectly legal ' - '(though\n' - 'perhaps not pretty).\n' - '\n' - '\n' - 'Value comparisons\n' - '=================\n' - '\n' - 'The operators "<", ">", "==", ">=", "<=", and "!=" compare ' - 'the values\n' - 'of two objects. The objects do not need to have the same ' - 'type.\n' - '\n' - 'Chapter Objects, values and types states that objects have a ' - 'value (in\n' - 'addition to type and identity). The value of an object is a ' - 'rather\n' - 'abstract notion in Python: For example, there is no canonical ' - 'access\n' - 'method for an object’s value. Also, there is no requirement ' - 'that the\n' - 'value of an object should be constructed in a particular way, ' - 'e.g.\n' - 'comprised of all its data attributes. Comparison operators ' - 'implement a\n' - 'particular notion of what the value of an object is. One can ' - 'think of\n' - 'them as defining the value of an object indirectly, by means ' - 'of their\n' - 'comparison implementation.\n' - '\n' - 'Because all types are (direct or indirect) subtypes of ' - '"object", they\n' - 'inherit the default comparison behavior from "object". Types ' - 'can\n' - 'customize their comparison behavior by implementing *rich ' - 'comparison\n' - 'methods* like "__lt__()", described in Basic customization.\n' - '\n' - 'The default behavior for equality comparison ("==" and "!=") ' - 'is based\n' - 'on the identity of the objects. Hence, equality comparison ' - 'of\n' - 'instances with the same identity results in equality, and ' - 'equality\n' - 'comparison of instances with different identities results in\n' - 'inequality. A motivation for this default behavior is the ' - 'desire that\n' - 'all objects should be reflexive (i.e. "x is y" implies "x == ' - 'y").\n' - '\n' - 'A default order comparison ("<", ">", "<=", and ">=") is not ' - 'provided;\n' - 'an attempt raises "TypeError". A motivation for this default ' - 'behavior\n' - 'is the lack of a similar invariant as for equality.\n' - '\n' - 'The behavior of the default equality comparison, that ' - 'instances with\n' - 'different identities are always unequal, may be in contrast ' - 'to what\n' - 'types will need that have a sensible definition of object ' - 'value and\n' - 'value-based equality. Such types will need to customize ' - 'their\n' - 'comparison behavior, and in fact, a number of built-in types ' - 'have done\n' - 'that.\n' - '\n' - 'The following list describes the comparison behavior of the ' - 'most\n' - 'important built-in types.\n' - '\n' - '* Numbers of built-in numeric types (Numeric Types — int, ' - 'float,\n' - ' complex) and of the standard library types ' - '"fractions.Fraction" and\n' - ' "decimal.Decimal" can be compared within and across their ' - 'types,\n' - ' with the restriction that complex numbers do not support ' - 'order\n' - ' comparison. Within the limits of the types involved, they ' - 'compare\n' - ' mathematically (algorithmically) correct without loss of ' - 'precision.\n' - '\n' - ' The not-a-number values "float(\'NaN\')" and ' - '"Decimal(\'NaN\')" are\n' - ' special. They are identical to themselves ("x is x" is ' - 'true) but\n' - ' are not equal to themselves ("x == x" is false). ' - 'Additionally,\n' - ' comparing any number to a not-a-number value will return ' - '"False".\n' - ' For example, both "3 < float(\'NaN\')" and "float(\'NaN\') ' - '< 3" will\n' - ' return "False".\n' - '\n' - '* Binary sequences (instances of "bytes" or "bytearray") can ' - 'be\n' - ' compared within and across their types. They compare\n' - ' lexicographically using the numeric values of their ' - 'elements.\n' - '\n' - '* Strings (instances of "str") compare lexicographically ' - 'using the\n' - ' numerical Unicode code points (the result of the built-in ' - 'function\n' - ' "ord()") of their characters. [3]\n' - '\n' - ' Strings and binary sequences cannot be directly compared.\n' - '\n' - '* Sequences (instances of "tuple", "list", or "range") can ' - 'be\n' - ' compared only within each of their types, with the ' - 'restriction that\n' - ' ranges do not support order comparison. Equality ' - 'comparison across\n' - ' these types results in inequality, and ordering comparison ' - 'across\n' - ' these types raises "TypeError".\n' - '\n' - ' Sequences compare lexicographically using comparison of\n' - ' corresponding elements, whereby reflexivity of the elements ' - 'is\n' - ' enforced.\n' - '\n' - ' In enforcing reflexivity of elements, the comparison of ' - 'collections\n' - ' assumes that for a collection element "x", "x == x" is ' - 'always true.\n' - ' Based on that assumption, element identity is compared ' - 'first, and\n' - ' element comparison is performed only for distinct ' - 'elements. This\n' - ' approach yields the same result as a strict element ' - 'comparison\n' - ' would, if the compared elements are reflexive. For ' - 'non-reflexive\n' - ' elements, the result is different than for strict element\n' - ' comparison, and may be surprising: The non-reflexive ' - 'not-a-number\n' - ' values for example result in the following comparison ' - 'behavior when\n' - ' used in a list:\n' - '\n' - " >>> nan = float('NaN')\n" - ' >>> nan is nan\n' - ' True\n' - ' >>> nan == nan\n' - ' False <-- the defined non-reflexive ' - 'behavior of NaN\n' - ' >>> [nan] == [nan]\n' - ' True <-- list enforces reflexivity and ' - 'tests identity first\n' - '\n' - ' Lexicographical comparison between built-in collections ' - 'works as\n' - ' follows:\n' - '\n' - ' * For two collections to compare equal, they must be of the ' - 'same\n' - ' type, have the same length, and each pair of ' - 'corresponding\n' - ' elements must compare equal (for example, "[1,2] == ' - '(1,2)" is\n' - ' false because the type is not the same).\n' - '\n' - ' * Collections that support order comparison are ordered the ' - 'same\n' - ' as their first unequal elements (for example, "[1,2,x] <= ' - '[1,2,y]"\n' - ' has the same value as "x <= y"). If a corresponding ' - 'element does\n' - ' not exist, the shorter collection is ordered first (for ' - 'example,\n' - ' "[1,2] < [1,2,3]" is true).\n' - '\n' - '* Mappings (instances of "dict") compare equal if and only if ' - 'they\n' - ' have equal *(key, value)* pairs. Equality comparison of the ' - 'keys and\n' - ' values enforces reflexivity.\n' - '\n' - ' Order comparisons ("<", ">", "<=", and ">=") raise ' - '"TypeError".\n' - '\n' - '* Sets (instances of "set" or "frozenset") can be compared ' - 'within\n' - ' and across their types.\n' - '\n' - ' They define order comparison operators to mean subset and ' - 'superset\n' - ' tests. Those relations do not define total orderings (for ' - 'example,\n' - ' the two sets "{1,2}" and "{2,3}" are not equal, nor subsets ' - 'of one\n' - ' another, nor supersets of one another). Accordingly, sets ' - 'are not\n' - ' appropriate arguments for functions which depend on total ' - 'ordering\n' - ' (for example, "min()", "max()", and "sorted()" produce ' - 'undefined\n' - ' results given a list of sets as inputs).\n' - '\n' - ' Comparison of sets enforces reflexivity of its elements.\n' - '\n' - '* Most other built-in types have no comparison methods ' - 'implemented,\n' - ' so they inherit the default comparison behavior.\n' - '\n' - 'User-defined classes that customize their comparison behavior ' - 'should\n' - 'follow some consistency rules, if possible:\n' - '\n' - '* Equality comparison should be reflexive. In other words, ' - 'identical\n' - ' objects should compare equal:\n' - '\n' - ' "x is y" implies "x == y"\n' - '\n' - '* Comparison should be symmetric. In other words, the ' - 'following\n' - ' expressions should have the same result:\n' - '\n' - ' "x == y" and "y == x"\n' - '\n' - ' "x != y" and "y != x"\n' - '\n' - ' "x < y" and "y > x"\n' - '\n' - ' "x <= y" and "y >= x"\n' - '\n' - '* Comparison should be transitive. The following ' - '(non-exhaustive)\n' - ' examples illustrate that:\n' - '\n' - ' "x > y and y > z" implies "x > z"\n' - '\n' - ' "x < y and y <= z" implies "x < z"\n' - '\n' - '* Inverse comparison should result in the boolean negation. ' - 'In other\n' - ' words, the following expressions should have the same ' - 'result:\n' - '\n' - ' "x == y" and "not x != y"\n' - '\n' - ' "x < y" and "not x >= y" (for total ordering)\n' - '\n' - ' "x > y" and "not x <= y" (for total ordering)\n' - '\n' - ' The last two expressions apply to totally ordered ' - 'collections (e.g.\n' - ' to sequences, but not to sets or mappings). See also the\n' - ' "total_ordering()" decorator.\n' - '\n' - '* The "hash()" result should be consistent with equality. ' - 'Objects\n' - ' that are equal should either have the same hash value, or ' - 'be marked\n' - ' as unhashable.\n' - '\n' - 'Python does not enforce these consistency rules. In fact, ' - 'the\n' - 'not-a-number values are an example for not following these ' - 'rules.\n' - '\n' - '\n' - 'Membership test operations\n' - '==========================\n' - '\n' - 'The operators "in" and "not in" test for membership. "x in ' - 's"\n' - 'evaluates to "True" if *x* is a member of *s*, and "False" ' - 'otherwise.\n' - '"x not in s" returns the negation of "x in s". All built-in ' - 'sequences\n' - 'and set types support this as well as dictionary, for which ' - '"in" tests\n' - 'whether the dictionary has a given key. For container types ' - 'such as\n' - 'list, tuple, set, frozenset, dict, or collections.deque, the\n' - 'expression "x in y" is equivalent to "any(x is e or x == e ' - 'for e in\n' - 'y)".\n' - '\n' - 'For the string and bytes types, "x in y" is "True" if and ' - 'only if *x*\n' - 'is a substring of *y*. An equivalent test is "y.find(x) != ' - '-1".\n' - 'Empty strings are always considered to be a substring of any ' - 'other\n' - 'string, so """ in "abc"" will return "True".\n' - '\n' - 'For user-defined classes which define the "__contains__()" ' - 'method, "x\n' - 'in y" returns "True" if "y.__contains__(x)" returns a true ' - 'value, and\n' - '"False" otherwise.\n' - '\n' - 'For user-defined classes which do not define "__contains__()" ' - 'but do\n' - 'define "__iter__()", "x in y" is "True" if some value "z" ' - 'with "x ==\n' - 'z" is produced while iterating over "y". If an exception is ' - 'raised\n' - 'during the iteration, it is as if "in" raised that ' - 'exception.\n' - '\n' - 'Lastly, the old-style iteration protocol is tried: if a class ' - 'defines\n' - '"__getitem__()", "x in y" is "True" if and only if there is a ' - 'non-\n' - 'negative integer index *i* such that "x == y[i]", and all ' - 'lower\n' - 'integer indices do not raise "IndexError" exception. (If any ' - 'other\n' - 'exception is raised, it is as if "in" raised that ' - 'exception).\n' - '\n' - 'The operator "not in" is defined to have the inverse true ' - 'value of\n' - '"in".\n' - '\n' - '\n' - 'Identity comparisons\n' - '====================\n' - '\n' - 'The operators "is" and "is not" test for object identity: "x ' - 'is y" is\n' - 'true if and only if *x* and *y* are the same object. Object ' - 'identity\n' - 'is determined using the "id()" function. "x is not y" yields ' - 'the\n' - 'inverse truth value. [4]\n', - 'compound': 'Compound statements\n' - '*******************\n' - '\n' - 'Compound statements contain (groups of) other statements; they ' - 'affect\n' - 'or control the execution of those other statements in some way. ' - 'In\n' - 'general, compound statements span multiple lines, although in ' - 'simple\n' - 'incarnations a whole compound statement may be contained in one ' - 'line.\n' - '\n' - 'The "if", "while" and "for" statements implement traditional ' - 'control\n' - 'flow constructs. "try" specifies exception handlers and/or ' - 'cleanup\n' - 'code for a group of statements, while the "with" statement ' - 'allows the\n' - 'execution of initialization and finalization code around a block ' - 'of\n' - 'code. Function and class definitions are also syntactically ' - 'compound\n' - 'statements.\n' - '\n' - 'A compound statement consists of one or more ‘clauses.’ A ' - 'clause\n' - 'consists of a header and a ‘suite.’ The clause headers of a\n' - 'particular compound statement are all at the same indentation ' - 'level.\n' - 'Each clause header begins with a uniquely identifying keyword ' - 'and ends\n' - 'with a colon. A suite is a group of statements controlled by a\n' - 'clause. A suite can be one or more semicolon-separated simple\n' - 'statements on the same line as the header, following the ' - 'header’s\n' - 'colon, or it can be one or more indented statements on ' - 'subsequent\n' - 'lines. Only the latter form of a suite can contain nested ' - 'compound\n' - 'statements; the following is illegal, mostly because it wouldn’t ' - 'be\n' - 'clear to which "if" clause a following "else" clause would ' - 'belong:\n' - '\n' - ' if test1: if test2: print(x)\n' - '\n' - 'Also note that the semicolon binds tighter than the colon in ' - 'this\n' - 'context, so that in the following example, either all or none of ' - 'the\n' - '"print()" calls are executed:\n' - '\n' - ' if x < y < z: print(x); print(y); print(z)\n' - '\n' - 'Summarizing:\n' - '\n' - ' compound_stmt ::= if_stmt\n' - ' | while_stmt\n' - ' | for_stmt\n' - ' | try_stmt\n' - ' | with_stmt\n' - ' | funcdef\n' - ' | classdef\n' - ' | async_with_stmt\n' - ' | async_for_stmt\n' - ' | async_funcdef\n' - ' suite ::= stmt_list NEWLINE | NEWLINE INDENT ' - 'statement+ DEDENT\n' - ' statement ::= stmt_list NEWLINE | compound_stmt\n' - ' stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n' - '\n' - 'Note that statements always end in a "NEWLINE" possibly followed ' - 'by a\n' - '"DEDENT". Also note that optional continuation clauses always ' - 'begin\n' - 'with a keyword that cannot start a statement, thus there are no\n' - 'ambiguities (the ‘dangling "else"’ problem is solved in Python ' - 'by\n' - 'requiring nested "if" statements to be indented).\n' - '\n' - 'The formatting of the grammar rules in the following sections ' - 'places\n' - 'each clause on a separate line for clarity.\n' - '\n' - '\n' - 'The "if" statement\n' - '==================\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the ' - 'expressions one\n' - 'by one until one is found to be true (see section Boolean ' - 'operations\n' - 'for the definition of true and false); then that suite is ' - 'executed\n' - '(and no other part of the "if" statement is executed or ' - 'evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, ' - 'if\n' - 'present, is executed.\n' - '\n' - '\n' - 'The "while" statement\n' - '=====================\n' - '\n' - 'The "while" statement is used for repeated execution as long as ' - 'an\n' - 'expression is true:\n' - '\n' - ' while_stmt ::= "while" expression ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'This repeatedly tests the expression and, if it is true, ' - 'executes the\n' - 'first suite; if the expression is false (which may be the first ' - 'time\n' - 'it is tested) the suite of the "else" clause, if present, is ' - 'executed\n' - 'and the loop terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and goes ' - 'back\n' - 'to testing the expression.\n' - '\n' - '\n' - 'The "for" statement\n' - '===================\n' - '\n' - 'The "for" statement is used to iterate over the elements of a ' - 'sequence\n' - '(such as a string, tuple or list) or other iterable object:\n' - '\n' - ' for_stmt ::= "for" target_list "in" expression_list ":" ' - 'suite\n' - ' ["else" ":" suite]\n' - '\n' - 'The expression list is evaluated once; it should yield an ' - 'iterable\n' - 'object. An iterator is created for the result of the\n' - '"expression_list". The suite is then executed once for each ' - 'item\n' - 'provided by the iterator, in the order returned by the ' - 'iterator. Each\n' - 'item in turn is assigned to the target list using the standard ' - 'rules\n' - 'for assignments (see Assignment statements), and then the suite ' - 'is\n' - 'executed. When the items are exhausted (which is immediately ' - 'when the\n' - 'sequence is empty or an iterator raises a "StopIteration" ' - 'exception),\n' - 'the suite in the "else" clause, if present, is executed, and the ' - 'loop\n' - 'terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and ' - 'continues\n' - 'with the next item, or with the "else" clause if there is no ' - 'next\n' - 'item.\n' - '\n' - 'The for-loop makes assignments to the variables(s) in the target ' - 'list.\n' - 'This overwrites all previous assignments to those variables ' - 'including\n' - 'those made in the suite of the for-loop:\n' - '\n' - ' for i in range(10):\n' - ' print(i)\n' - ' i = 5 # this will not affect the for-loop\n' - ' # because i will be overwritten with ' - 'the next\n' - ' # index in the range\n' - '\n' - 'Names in the target list are not deleted when the loop is ' - 'finished,\n' - 'but if the sequence is empty, they will not have been assigned ' - 'to at\n' - 'all by the loop. Hint: the built-in function "range()" returns ' - 'an\n' - 'iterator of integers suitable to emulate the effect of Pascal’s ' - '"for i\n' - ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, ' - '2]".\n' - '\n' - 'Note: There is a subtlety when the sequence is being modified by ' - 'the\n' - ' loop (this can only occur for mutable sequences, e.g. lists). ' - 'An\n' - ' internal counter is used to keep track of which item is used ' - 'next,\n' - ' and this is incremented on each iteration. When this counter ' - 'has\n' - ' reached the length of the sequence the loop terminates. This ' - 'means\n' - ' that if the suite deletes the current (or a previous) item ' - 'from the\n' - ' sequence, the next item will be skipped (since it gets the ' - 'index of\n' - ' the current item which has already been treated). Likewise, ' - 'if the\n' - ' suite inserts an item in the sequence before the current item, ' - 'the\n' - ' current item will be treated again the next time through the ' - 'loop.\n' - ' This can lead to nasty bugs that can be avoided by making a\n' - ' temporary copy using a slice of the whole sequence, e.g.,\n' - '\n' - ' for x in a[:]:\n' - ' if x < 0: a.remove(x)\n' - '\n' - '\n' - 'The "try" statement\n' - '===================\n' - '\n' - 'The "try" statement specifies exception handlers and/or cleanup ' - 'code\n' - 'for a group of statements:\n' - '\n' - ' try_stmt ::= try1_stmt | try2_stmt\n' - ' try1_stmt ::= "try" ":" suite\n' - ' ("except" [expression ["as" identifier]] ":" ' - 'suite)+\n' - ' ["else" ":" suite]\n' - ' ["finally" ":" suite]\n' - ' try2_stmt ::= "try" ":" suite\n' - ' "finally" ":" suite\n' - '\n' - 'The "except" clause(s) specify one or more exception handlers. ' - 'When no\n' - 'exception occurs in the "try" clause, no exception handler is\n' - 'executed. When an exception occurs in the "try" suite, a search ' - 'for an\n' - 'exception handler is started. This search inspects the except ' - 'clauses\n' - 'in turn until one is found that matches the exception. An ' - 'expression-\n' - 'less except clause, if present, must be last; it matches any\n' - 'exception. For an except clause with an expression, that ' - 'expression\n' - 'is evaluated, and the clause matches the exception if the ' - 'resulting\n' - 'object is “compatible” with the exception. An object is ' - 'compatible\n' - 'with an exception if it is the class or a base class of the ' - 'exception\n' - 'object or a tuple containing an item compatible with the ' - 'exception.\n' - '\n' - 'If no except clause matches the exception, the search for an ' - 'exception\n' - 'handler continues in the surrounding code and on the invocation ' - 'stack.\n' - '[1]\n' - '\n' - 'If the evaluation of an expression in the header of an except ' - 'clause\n' - 'raises an exception, the original search for a handler is ' - 'canceled and\n' - 'a search starts for the new exception in the surrounding code ' - 'and on\n' - 'the call stack (it is treated as if the entire "try" statement ' - 'raised\n' - 'the exception).\n' - '\n' - 'When a matching except clause is found, the exception is ' - 'assigned to\n' - 'the target specified after the "as" keyword in that except ' - 'clause, if\n' - 'present, and the except clause’s suite is executed. All except\n' - 'clauses must have an executable block. When the end of this ' - 'block is\n' - 'reached, execution continues normally after the entire try ' - 'statement.\n' - '(This means that if two nested handlers exist for the same ' - 'exception,\n' - 'and the exception occurs in the try clause of the inner handler, ' - 'the\n' - 'outer handler will not handle the exception.)\n' - '\n' - 'When an exception has been assigned using "as target", it is ' - 'cleared\n' - 'at the end of the except clause. This is as if\n' - '\n' - ' except E as N:\n' - ' foo\n' - '\n' - 'was translated to\n' - '\n' - ' except E as N:\n' - ' try:\n' - ' foo\n' - ' finally:\n' - ' del N\n' - '\n' - 'This means the exception must be assigned to a different name to ' - 'be\n' - 'able to refer to it after the except clause. Exceptions are ' - 'cleared\n' - 'because with the traceback attached to them, they form a ' - 'reference\n' - 'cycle with the stack frame, keeping all locals in that frame ' - 'alive\n' - 'until the next garbage collection occurs.\n' - '\n' - 'Before an except clause’s suite is executed, details about the\n' - 'exception are stored in the "sys" module and can be accessed ' - 'via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting ' - 'of the\n' - 'exception class, the exception instance and a traceback object ' - '(see\n' - 'section The standard type hierarchy) identifying the point in ' - 'the\n' - 'program where the exception occurred. "sys.exc_info()" values ' - 'are\n' - 'restored to their previous values (before the call) when ' - 'returning\n' - 'from a function that handled an exception.\n' - '\n' - 'The optional "else" clause is executed if the control flow ' - 'leaves the\n' - '"try" suite, no exception was raised, and no "return", ' - '"continue", or\n' - '"break" statement was executed. Exceptions in the "else" clause ' - 'are\n' - 'not handled by the preceding "except" clauses.\n' - '\n' - 'If "finally" is present, it specifies a ‘cleanup’ handler. The ' - '"try"\n' - 'clause is executed, including any "except" and "else" clauses. ' - 'If an\n' - 'exception occurs in any of the clauses and is not handled, the\n' - 'exception is temporarily saved. The "finally" clause is ' - 'executed. If\n' - 'there is a saved exception it is re-raised at the end of the ' - '"finally"\n' - 'clause. If the "finally" clause raises another exception, the ' - 'saved\n' - 'exception is set as the context of the new exception. If the ' - '"finally"\n' - 'clause executes a "return" or "break" statement, the saved ' - 'exception\n' - 'is discarded:\n' - '\n' - ' >>> def f():\n' - ' ... try:\n' - ' ... 1/0\n' - ' ... finally:\n' - ' ... return 42\n' - ' ...\n' - ' >>> f()\n' - ' 42\n' - '\n' - 'The exception information is not available to the program ' - 'during\n' - 'execution of the "finally" clause.\n' - '\n' - 'When a "return", "break" or "continue" statement is executed in ' - 'the\n' - '"try" suite of a "try"…"finally" statement, the "finally" clause ' - 'is\n' - 'also executed ‘on the way out.’ A "continue" statement is ' - 'illegal in\n' - 'the "finally" clause. (The reason is a problem with the current\n' - 'implementation — this restriction may be lifted in the future).\n' - '\n' - 'The return value of a function is determined by the last ' - '"return"\n' - 'statement executed. Since the "finally" clause always executes, ' - 'a\n' - '"return" statement executed in the "finally" clause will always ' - 'be the\n' - 'last one executed:\n' - '\n' - ' >>> def foo():\n' - ' ... try:\n' - " ... return 'try'\n" - ' ... finally:\n' - " ... return 'finally'\n" - ' ...\n' - ' >>> foo()\n' - " 'finally'\n" - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information on using the "raise" statement to ' - 'generate\n' - 'exceptions may be found in section The raise statement.\n' - '\n' - '\n' - 'The "with" statement\n' - '====================\n' - '\n' - 'The "with" statement is used to wrap the execution of a block ' - 'with\n' - 'methods defined by a context manager (see section With ' - 'Statement\n' - 'Context Managers). This allows common "try"…"except"…"finally" ' - 'usage\n' - 'patterns to be encapsulated for convenient reuse.\n' - '\n' - ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n' - ' with_item ::= expression ["as" target]\n' - '\n' - 'The execution of the "with" statement with one “item” proceeds ' - 'as\n' - 'follows:\n' - '\n' - '1. The context expression (the expression given in the ' - '"with_item")\n' - ' is evaluated to obtain a context manager.\n' - '\n' - '2. The context manager’s "__exit__()" is loaded for later use.\n' - '\n' - '3. The context manager’s "__enter__()" method is invoked.\n' - '\n' - '4. If a target was included in the "with" statement, the return\n' - ' value from "__enter__()" is assigned to it.\n' - '\n' - ' Note: The "with" statement guarantees that if the ' - '"__enter__()"\n' - ' method returns without an error, then "__exit__()" will ' - 'always be\n' - ' called. Thus, if an error occurs during the assignment to ' - 'the\n' - ' target list, it will be treated the same as an error ' - 'occurring\n' - ' within the suite would be. See step 6 below.\n' - '\n' - '5. The suite is executed.\n' - '\n' - '6. The context manager’s "__exit__()" method is invoked. If an\n' - ' exception caused the suite to be exited, its type, value, ' - 'and\n' - ' traceback are passed as arguments to "__exit__()". Otherwise, ' - 'three\n' - ' "None" arguments are supplied.\n' - '\n' - ' If the suite was exited due to an exception, and the return ' - 'value\n' - ' from the "__exit__()" method was false, the exception is ' - 'reraised.\n' - ' If the return value was true, the exception is suppressed, ' - 'and\n' - ' execution continues with the statement following the "with"\n' - ' statement.\n' - '\n' - ' If the suite was exited for any reason other than an ' - 'exception, the\n' - ' return value from "__exit__()" is ignored, and execution ' - 'proceeds\n' - ' at the normal location for the kind of exit that was taken.\n' - '\n' - 'With more than one item, the context managers are processed as ' - 'if\n' - 'multiple "with" statements were nested:\n' - '\n' - ' with A() as a, B() as b:\n' - ' suite\n' - '\n' - 'is equivalent to\n' - '\n' - ' with A() as a:\n' - ' with B() as b:\n' - ' suite\n' - '\n' - 'Changed in version 3.1: Support for multiple context ' - 'expressions.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the Python ' - '"with"\n' - ' statement.\n' - '\n' - '\n' - 'Function definitions\n' - '====================\n' - '\n' - 'A function definition defines a user-defined function object ' - '(see\n' - 'section The standard type hierarchy):\n' - '\n' - ' funcdef ::= [decorators] "def" funcname "(" ' - '[parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - ' decorators ::= decorator+\n' - ' decorator ::= "@" dotted_name ["(" ' - '[argument_list [","]] ")"] NEWLINE\n' - ' dotted_name ::= identifier ("." identifier)*\n' - ' parameter_list ::= defparameter ("," defparameter)* ' - '["," [parameter_list_starargs]]\n' - ' | parameter_list_starargs\n' - ' parameter_list_starargs ::= "*" [parameter] ("," ' - 'defparameter)* ["," ["**" parameter [","]]]\n' - ' | "**" parameter [","]\n' - ' parameter ::= identifier [":" expression]\n' - ' defparameter ::= parameter ["=" expression]\n' - ' funcname ::= identifier\n' - '\n' - 'A function definition is an executable statement. Its execution ' - 'binds\n' - 'the function name in the current local namespace to a function ' - 'object\n' - '(a wrapper around the executable code for the function). This\n' - 'function object contains a reference to the current global ' - 'namespace\n' - 'as the global namespace to be used when the function is called.\n' - '\n' - 'The function definition does not execute the function body; this ' - 'gets\n' - 'executed only when the function is called. [2]\n' - '\n' - 'A function definition may be wrapped by one or more *decorator*\n' - 'expressions. Decorator expressions are evaluated when the ' - 'function is\n' - 'defined, in the scope that contains the function definition. ' - 'The\n' - 'result must be a callable, which is invoked with the function ' - 'object\n' - 'as the only argument. The returned value is bound to the ' - 'function name\n' - 'instead of the function object. Multiple decorators are applied ' - 'in\n' - 'nested fashion. For example, the following code\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' def func(): pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' def func(): pass\n' - ' func = f1(arg)(f2(func))\n' - '\n' - 'except that the original function is not temporarily bound to ' - 'the name\n' - '"func".\n' - '\n' - 'When one or more *parameters* have the form *parameter* "="\n' - '*expression*, the function is said to have “default parameter ' - 'values.”\n' - 'For a parameter with a default value, the corresponding ' - '*argument* may\n' - 'be omitted from a call, in which case the parameter’s default ' - 'value is\n' - 'substituted. If a parameter has a default value, all following\n' - 'parameters up until the “"*"” must also have a default value — ' - 'this is\n' - 'a syntactic restriction that is not expressed by the grammar.\n' - '\n' - '**Default parameter values are evaluated from left to right when ' - 'the\n' - 'function definition is executed.** This means that the ' - 'expression is\n' - 'evaluated once, when the function is defined, and that the same ' - '“pre-\n' - 'computed” value is used for each call. This is especially ' - 'important\n' - 'to understand when a default parameter is a mutable object, such ' - 'as a\n' - 'list or a dictionary: if the function modifies the object (e.g. ' - 'by\n' - 'appending an item to a list), the default value is in effect ' - 'modified.\n' - 'This is generally not what was intended. A way around this is ' - 'to use\n' - '"None" as the default, and explicitly test for it in the body of ' - 'the\n' - 'function, e.g.:\n' - '\n' - ' def whats_on_the_telly(penguin=None):\n' - ' if penguin is None:\n' - ' penguin = []\n' - ' penguin.append("property of the zoo")\n' - ' return penguin\n' - '\n' - 'Function call semantics are described in more detail in section ' - 'Calls.\n' - 'A function call always assigns values to all parameters ' - 'mentioned in\n' - 'the parameter list, either from position arguments, from ' - 'keyword\n' - 'arguments, or from default values. If the form “"*identifier"” ' - 'is\n' - 'present, it is initialized to a tuple receiving any excess ' - 'positional\n' - 'parameters, defaulting to the empty tuple. If the form\n' - '“"**identifier"” is present, it is initialized to a new ordered\n' - 'mapping receiving any excess keyword arguments, defaulting to a ' - 'new\n' - 'empty mapping of the same type. Parameters after “"*"” or\n' - '“"*identifier"” are keyword-only parameters and may only be ' - 'passed\n' - 'used keyword arguments.\n' - '\n' - 'Parameters may have annotations of the form “": expression"” ' - 'following\n' - 'the parameter name. Any parameter may have an annotation even ' - 'those\n' - 'of the form "*identifier" or "**identifier". Functions may ' - 'have\n' - '“return” annotation of the form “"-> expression"” after the ' - 'parameter\n' - 'list. These annotations can be any valid Python expression and ' - 'are\n' - 'evaluated when the function definition is executed. Annotations ' - 'may\n' - 'be evaluated in a different order than they appear in the source ' - 'code.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - 'dictionary keyed by the parameters’ names in the ' - '"__annotations__"\n' - 'attribute of the function object.\n' - '\n' - 'It is also possible to create anonymous functions (functions not ' - 'bound\n' - 'to a name), for immediate use in expressions. This uses lambda\n' - 'expressions, described in section Lambdas. Note that the ' - 'lambda\n' - 'expression is merely a shorthand for a simplified function ' - 'definition;\n' - 'a function defined in a “"def"” statement can be passed around ' - 'or\n' - 'assigned to another name just like a function defined by a ' - 'lambda\n' - 'expression. The “"def"” form is actually more powerful since ' - 'it\n' - 'allows the execution of multiple statements and annotations.\n' - '\n' - '**Programmer’s note:** Functions are first-class objects. A ' - '“"def"”\n' - 'statement executed inside a function definition defines a local\n' - 'function that can be returned or passed around. Free variables ' - 'used\n' - 'in the nested function can access the local variables of the ' - 'function\n' - 'containing the def. See section Naming and binding for ' - 'details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3107** - Function Annotations\n' - ' The original specification for function annotations.\n' - '\n' - '\n' - 'Class definitions\n' - '=================\n' - '\n' - 'A class definition defines a class object (see section The ' - 'standard\n' - 'type hierarchy):\n' - '\n' - ' classdef ::= [decorators] "class" classname [inheritance] ' - '":" suite\n' - ' inheritance ::= "(" [argument_list] ")"\n' - ' classname ::= identifier\n' - '\n' - 'A class definition is an executable statement. The inheritance ' - 'list\n' - 'usually gives a list of base classes (see Metaclasses for more\n' - 'advanced uses), so each item in the list should evaluate to a ' - 'class\n' - 'object which allows subclassing. Classes without an inheritance ' - 'list\n' - 'inherit, by default, from the base class "object"; hence,\n' - '\n' - ' class Foo:\n' - ' pass\n' - '\n' - 'is equivalent to\n' - '\n' - ' class Foo(object):\n' - ' pass\n' - '\n' - 'The class’s suite is then executed in a new execution frame ' - '(see\n' - 'Naming and binding), using a newly created local namespace and ' - 'the\n' - 'original global namespace. (Usually, the suite contains mostly\n' - 'function definitions.) When the class’s suite finishes ' - 'execution, its\n' - 'execution frame is discarded but its local namespace is saved. ' - '[3] A\n' - 'class object is then created using the inheritance list for the ' - 'base\n' - 'classes and the saved local namespace for the attribute ' - 'dictionary.\n' - 'The class name is bound to this class object in the original ' - 'local\n' - 'namespace.\n' - '\n' - 'The order in which attributes are defined in the class body is\n' - 'preserved in the new class’s "__dict__". Note that this is ' - 'reliable\n' - 'only right after the class is created and only for classes that ' - 'were\n' - 'defined using the definition syntax.\n' - '\n' - 'Class creation can be customized heavily using metaclasses.\n' - '\n' - 'Classes can also be decorated: just like when decorating ' - 'functions,\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' class Foo: pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' class Foo: pass\n' - ' Foo = f1(arg)(f2(Foo))\n' - '\n' - 'The evaluation rules for the decorator expressions are the same ' - 'as for\n' - 'function decorators. The result is then bound to the class ' - 'name.\n' - '\n' - '**Programmer’s note:** Variables defined in the class definition ' - 'are\n' - 'class attributes; they are shared by instances. Instance ' - 'attributes\n' - 'can be set in a method with "self.name = value". Both class ' - 'and\n' - 'instance attributes are accessible through the notation ' - '“"self.name"”,\n' - 'and an instance attribute hides a class attribute with the same ' - 'name\n' - 'when accessed in this way. Class attributes can be used as ' - 'defaults\n' - 'for instance attributes, but using mutable values there can lead ' - 'to\n' - 'unexpected results. Descriptors can be used to create instance\n' - 'variables with different implementation details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' The proposal that changed the declaration of metaclasses to ' - 'the\n' - ' current syntax, and the semantics for how classes with\n' - ' metaclasses are constructed.\n' - '\n' - ' **PEP 3129** - Class Decorators\n' - ' The proposal that added class decorators. Function and ' - 'method\n' - ' decorators were introduced in **PEP 318**.\n' - '\n' - '\n' - 'Coroutines\n' - '==========\n' - '\n' - 'New in version 3.5.\n' - '\n' - '\n' - 'Coroutine function definition\n' - '-----------------------------\n' - '\n' - ' async_funcdef ::= [decorators] "async" "def" funcname "(" ' - '[parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - '\n' - 'Execution of Python coroutines can be suspended and resumed at ' - 'many\n' - 'points (see *coroutine*). In the body of a coroutine, any ' - '"await" and\n' - '"async" identifiers become reserved keywords; "await" ' - 'expressions,\n' - '"async for" and "async with" can only be used in coroutine ' - 'bodies.\n' - '\n' - 'Functions defined with "async def" syntax are always coroutine\n' - 'functions, even if they do not contain "await" or "async" ' - 'keywords.\n' - '\n' - 'It is a "SyntaxError" to use "yield from" expressions in "async ' - 'def"\n' - 'coroutines.\n' - '\n' - 'An example of a coroutine function:\n' - '\n' - ' async def func(param1, param2):\n' - ' do_stuff()\n' - ' await some_coroutine()\n' - '\n' - '\n' - 'The "async for" statement\n' - '-------------------------\n' - '\n' - ' async_for_stmt ::= "async" for_stmt\n' - '\n' - 'An *asynchronous iterable* is able to call asynchronous code in ' - 'its\n' - '*iter* implementation, and *asynchronous iterator* can call\n' - 'asynchronous code in its *next* method.\n' - '\n' - 'The "async for" statement allows convenient iteration over\n' - 'asynchronous iterators.\n' - '\n' - 'The following code:\n' - '\n' - ' async for TARGET in ITER:\n' - ' BLOCK\n' - ' else:\n' - ' BLOCK2\n' - '\n' - 'Is semantically equivalent to:\n' - '\n' - ' iter = (ITER)\n' - ' iter = type(iter).__aiter__(iter)\n' - ' running = True\n' - ' while running:\n' - ' try:\n' - ' TARGET = await type(iter).__anext__(iter)\n' - ' except StopAsyncIteration:\n' - ' running = False\n' - ' else:\n' - ' BLOCK\n' - ' else:\n' - ' BLOCK2\n' - '\n' - 'See also "__aiter__()" and "__anext__()" for details.\n' - '\n' - 'It is a "SyntaxError" to use "async for" statement outside of ' - 'an\n' - '"async def" function.\n' - '\n' - '\n' - 'The "async with" statement\n' - '--------------------------\n' - '\n' - ' async_with_stmt ::= "async" with_stmt\n' - '\n' - 'An *asynchronous context manager* is a *context manager* that is ' - 'able\n' - 'to suspend execution in its *enter* and *exit* methods.\n' - '\n' - 'The following code:\n' - '\n' - ' async with EXPR as VAR:\n' - ' BLOCK\n' - '\n' - 'Is semantically equivalent to:\n' - '\n' - ' mgr = (EXPR)\n' - ' aexit = type(mgr).__aexit__\n' - ' aenter = type(mgr).__aenter__(mgr)\n' - '\n' - ' VAR = await aenter\n' - ' try:\n' - ' BLOCK\n' - ' except:\n' - ' if not await aexit(mgr, *sys.exc_info()):\n' - ' raise\n' - ' else:\n' - ' await aexit(mgr, None, None, None)\n' - '\n' - 'See also "__aenter__()" and "__aexit__()" for details.\n' - '\n' - 'It is a "SyntaxError" to use "async with" statement outside of ' - 'an\n' - '"async def" function.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 492** - Coroutines with async and await syntax\n' - ' The proposal that made coroutines a proper standalone ' - 'concept in\n' - ' Python, and added supporting syntax.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] The exception is propagated to the invocation stack unless\n' - ' there is a "finally" clause which happens to raise another\n' - ' exception. That new exception causes the old one to be ' - 'lost.\n' - '\n' - '[2] A string literal appearing as the first statement in the\n' - ' function body is transformed into the function’s "__doc__"\n' - ' attribute and therefore the function’s *docstring*.\n' - '\n' - '[3] A string literal appearing as the first statement in the ' - 'class\n' - ' body is transformed into the namespace’s "__doc__" item and\n' - ' therefore the class’s *docstring*.\n', - 'context-managers': 'With Statement Context Managers\n' - '*******************************\n' - '\n' - 'A *context manager* is an object that defines the ' - 'runtime context to\n' - 'be established when executing a "with" statement. The ' - 'context manager\n' - 'handles the entry into, and the exit from, the desired ' - 'runtime context\n' - 'for the execution of the block of code. Context ' - 'managers are normally\n' - 'invoked using the "with" statement (described in section ' - 'The with\n' - 'statement), but can also be used by directly invoking ' - 'their methods.\n' - '\n' - 'Typical uses of context managers include saving and ' - 'restoring various\n' - 'kinds of global state, locking and unlocking resources, ' - 'closing opened\n' - 'files, etc.\n' - '\n' - 'For more information on context managers, see Context ' - 'Manager Types.\n' - '\n' - 'object.__enter__(self)\n' - '\n' - ' Enter the runtime context related to this object. The ' - '"with"\n' - ' statement will bind this method’s return value to the ' - 'target(s)\n' - ' specified in the "as" clause of the statement, if ' - 'any.\n' - '\n' - 'object.__exit__(self, exc_type, exc_value, traceback)\n' - '\n' - ' Exit the runtime context related to this object. The ' - 'parameters\n' - ' describe the exception that caused the context to be ' - 'exited. If the\n' - ' context was exited without an exception, all three ' - 'arguments will\n' - ' be "None".\n' - '\n' - ' If an exception is supplied, and the method wishes to ' - 'suppress the\n' - ' exception (i.e., prevent it from being propagated), ' - 'it should\n' - ' return a true value. Otherwise, the exception will be ' - 'processed\n' - ' normally upon exit from this method.\n' - '\n' - ' Note that "__exit__()" methods should not reraise the ' - 'passed-in\n' - ' exception; this is the caller’s responsibility.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the ' - 'Python "with"\n' - ' statement.\n', - 'continue': 'The "continue" statement\n' - '************************\n' - '\n' - ' continue_stmt ::= "continue"\n' - '\n' - '"continue" may only occur syntactically nested in a "for" or ' - '"while"\n' - 'loop, but not nested in a function or class definition or ' - '"finally"\n' - 'clause within that loop. It continues with the next cycle of ' - 'the\n' - 'nearest enclosing loop.\n' - '\n' - 'When "continue" passes control out of a "try" statement with a\n' - '"finally" clause, that "finally" clause is executed before ' - 'really\n' - 'starting the next loop cycle.\n', - 'conversions': 'Arithmetic conversions\n' - '**********************\n' - '\n' - 'When a description of an arithmetic operator below uses the ' - 'phrase\n' - '“the numeric arguments are converted to a common type,” this ' - 'means\n' - 'that the operator implementation for built-in types works as ' - 'follows:\n' - '\n' - '* If either argument is a complex number, the other is ' - 'converted to\n' - ' complex;\n' - '\n' - '* otherwise, if either argument is a floating point number, ' - 'the\n' - ' other is converted to floating point;\n' - '\n' - '* otherwise, both must be integers and no conversion is ' - 'necessary.\n' - '\n' - 'Some additional rules apply for certain operators (e.g., a ' - 'string as a\n' - 'left argument to the ‘%’ operator). Extensions must define ' - 'their own\n' - 'conversion behavior.\n', - 'customization': 'Basic customization\n' - '*******************\n' - '\n' - 'object.__new__(cls[, ...])\n' - '\n' - ' Called to create a new instance of class *cls*. ' - '"__new__()" is a\n' - ' static method (special-cased so you need not declare it ' - 'as such)\n' - ' that takes the class of which an instance was requested ' - 'as its\n' - ' first argument. The remaining arguments are those ' - 'passed to the\n' - ' object constructor expression (the call to the class). ' - 'The return\n' - ' value of "__new__()" should be the new object instance ' - '(usually an\n' - ' instance of *cls*).\n' - '\n' - ' Typical implementations create a new instance of the ' - 'class by\n' - ' invoking the superclass’s "__new__()" method using\n' - ' "super().__new__(cls[, ...])" with appropriate arguments ' - 'and then\n' - ' modifying the newly-created instance as necessary before ' - 'returning\n' - ' it.\n' - '\n' - ' If "__new__()" returns an instance of *cls*, then the ' - 'new\n' - ' instance’s "__init__()" method will be invoked like\n' - ' "__init__(self[, ...])", where *self* is the new ' - 'instance and the\n' - ' remaining arguments are the same as were passed to ' - '"__new__()".\n' - '\n' - ' If "__new__()" does not return an instance of *cls*, ' - 'then the new\n' - ' instance’s "__init__()" method will not be invoked.\n' - '\n' - ' "__new__()" is intended mainly to allow subclasses of ' - 'immutable\n' - ' types (like int, str, or tuple) to customize instance ' - 'creation. It\n' - ' is also commonly overridden in custom metaclasses in ' - 'order to\n' - ' customize class creation.\n' - '\n' - 'object.__init__(self[, ...])\n' - '\n' - ' Called after the instance has been created (by ' - '"__new__()"), but\n' - ' before it is returned to the caller. The arguments are ' - 'those\n' - ' passed to the class constructor expression. If a base ' - 'class has an\n' - ' "__init__()" method, the derived class’s "__init__()" ' - 'method, if\n' - ' any, must explicitly call it to ensure proper ' - 'initialization of the\n' - ' base class part of the instance; for example:\n' - ' "super().__init__([args...])".\n' - '\n' - ' Because "__new__()" and "__init__()" work together in ' - 'constructing\n' - ' objects ("__new__()" to create it, and "__init__()" to ' - 'customize\n' - ' it), no non-"None" value may be returned by ' - '"__init__()"; doing so\n' - ' will cause a "TypeError" to be raised at runtime.\n' - '\n' - 'object.__del__(self)\n' - '\n' - ' Called when the instance is about to be destroyed. This ' - 'is also\n' - ' called a finalizer or (improperly) a destructor. If a ' - 'base class\n' - ' has a "__del__()" method, the derived class’s ' - '"__del__()" method,\n' - ' if any, must explicitly call it to ensure proper ' - 'deletion of the\n' - ' base class part of the instance.\n' - '\n' - ' It is possible (though not recommended!) for the ' - '"__del__()" method\n' - ' to postpone destruction of the instance by creating a ' - 'new reference\n' - ' to it. This is called object *resurrection*. It is\n' - ' implementation-dependent whether "__del__()" is called a ' - 'second\n' - ' time when a resurrected object is about to be destroyed; ' - 'the\n' - ' current *CPython* implementation only calls it once.\n' - '\n' - ' It is not guaranteed that "__del__()" methods are called ' - 'for\n' - ' objects that still exist when the interpreter exits.\n' - '\n' - ' Note: "del x" doesn’t directly call "x.__del__()" — the ' - 'former\n' - ' decrements the reference count for "x" by one, and the ' - 'latter is\n' - ' only called when "x"’s reference count reaches zero.\n' - '\n' - ' **CPython implementation detail:** It is possible for a ' - 'reference\n' - ' cycle to prevent the reference count of an object from ' - 'going to\n' - ' zero. In this case, the cycle will be later detected ' - 'and deleted\n' - ' by the *cyclic garbage collector*. A common cause of ' - 'reference\n' - ' cycles is when an exception has been caught in a local ' - 'variable.\n' - ' The frame’s locals then reference the exception, which ' - 'references\n' - ' its own traceback, which references the locals of all ' - 'frames caught\n' - ' in the traceback.\n' - '\n' - ' See also: Documentation for the "gc" module.\n' - '\n' - ' Warning: Due to the precarious circumstances under ' - 'which\n' - ' "__del__()" methods are invoked, exceptions that occur ' - 'during\n' - ' their execution are ignored, and a warning is printed ' - 'to\n' - ' "sys.stderr" instead. In particular:\n' - '\n' - ' * "__del__()" can be invoked when arbitrary code is ' - 'being\n' - ' executed, including from any arbitrary thread. If ' - '"__del__()"\n' - ' needs to take a lock or invoke any other blocking ' - 'resource, it\n' - ' may deadlock as the resource may already be taken by ' - 'the code\n' - ' that gets interrupted to execute "__del__()".\n' - '\n' - ' * "__del__()" can be executed during interpreter ' - 'shutdown. As\n' - ' a consequence, the global variables it needs to ' - 'access\n' - ' (including other modules) may already have been ' - 'deleted or set\n' - ' to "None". Python guarantees that globals whose name ' - 'begins\n' - ' with a single underscore are deleted from their ' - 'module before\n' - ' other globals are deleted; if no other references to ' - 'such\n' - ' globals exist, this may help in assuring that ' - 'imported modules\n' - ' are still available at the time when the "__del__()" ' - 'method is\n' - ' called.\n' - '\n' - 'object.__repr__(self)\n' - '\n' - ' Called by the "repr()" built-in function to compute the ' - '“official”\n' - ' string representation of an object. If at all possible, ' - 'this\n' - ' should look like a valid Python expression that could be ' - 'used to\n' - ' recreate an object with the same value (given an ' - 'appropriate\n' - ' environment). If this is not possible, a string of the ' - 'form\n' - ' "<...some useful description...>" should be returned. ' - 'The return\n' - ' value must be a string object. If a class defines ' - '"__repr__()" but\n' - ' not "__str__()", then "__repr__()" is also used when an ' - '“informal”\n' - ' string representation of instances of that class is ' - 'required.\n' - '\n' - ' This is typically used for debugging, so it is important ' - 'that the\n' - ' representation is information-rich and unambiguous.\n' - '\n' - 'object.__str__(self)\n' - '\n' - ' Called by "str(object)" and the built-in functions ' - '"format()" and\n' - ' "print()" to compute the “informal” or nicely printable ' - 'string\n' - ' representation of an object. The return value must be a ' - 'string\n' - ' object.\n' - '\n' - ' This method differs from "object.__repr__()" in that ' - 'there is no\n' - ' expectation that "__str__()" return a valid Python ' - 'expression: a\n' - ' more convenient or concise representation can be used.\n' - '\n' - ' The default implementation defined by the built-in type ' - '"object"\n' - ' calls "object.__repr__()".\n' - '\n' - 'object.__bytes__(self)\n' - '\n' - ' Called by bytes to compute a byte-string representation ' - 'of an\n' - ' object. This should return a "bytes" object.\n' - '\n' - 'object.__format__(self, format_spec)\n' - '\n' - ' Called by the "format()" built-in function, and by ' - 'extension,\n' - ' evaluation of formatted string literals and the ' - '"str.format()"\n' - ' method, to produce a “formatted” string representation ' - 'of an\n' - ' object. The "format_spec" argument is a string that ' - 'contains a\n' - ' description of the formatting options desired. The ' - 'interpretation\n' - ' of the "format_spec" argument is up to the type ' - 'implementing\n' - ' "__format__()", however most classes will either ' - 'delegate\n' - ' formatting to one of the built-in types, or use a ' - 'similar\n' - ' formatting option syntax.\n' - '\n' - ' See Format Specification Mini-Language for a description ' - 'of the\n' - ' standard formatting syntax.\n' - '\n' - ' The return value must be a string object.\n' - '\n' - ' Changed in version 3.4: The __format__ method of ' - '"object" itself\n' - ' raises a "TypeError" if passed any non-empty string.\n' - '\n' - 'object.__lt__(self, other)\n' - 'object.__le__(self, other)\n' - 'object.__eq__(self, other)\n' - 'object.__ne__(self, other)\n' - 'object.__gt__(self, other)\n' - 'object.__ge__(self, other)\n' - '\n' - ' These are the so-called “rich comparison” methods. The\n' - ' correspondence between operator symbols and method names ' - 'is as\n' - ' follows: "xy" calls\n' - ' "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n' - '\n' - ' A rich comparison method may return the singleton ' - '"NotImplemented"\n' - ' if it does not implement the operation for a given pair ' - 'of\n' - ' arguments. By convention, "False" and "True" are ' - 'returned for a\n' - ' successful comparison. However, these methods can return ' - 'any value,\n' - ' so if the comparison operator is used in a Boolean ' - 'context (e.g.,\n' - ' in the condition of an "if" statement), Python will call ' - '"bool()"\n' - ' on the value to determine if the result is true or ' - 'false.\n' - '\n' - ' By default, "__ne__()" delegates to "__eq__()" and ' - 'inverts the\n' - ' result unless it is "NotImplemented". There are no ' - 'other implied\n' - ' relationships among the comparison operators, for ' - 'example, the\n' - ' truth of "(x.__hash__".\n' - '\n' - ' If a class that does not override "__eq__()" wishes to ' - 'suppress\n' - ' hash support, it should include "__hash__ = None" in the ' - 'class\n' - ' definition. A class which defines its own "__hash__()" ' - 'that\n' - ' explicitly raises a "TypeError" would be incorrectly ' - 'identified as\n' - ' hashable by an "isinstance(obj, collections.Hashable)" ' - 'call.\n' - '\n' - ' Note: By default, the "__hash__()" values of str, bytes ' - 'and\n' - ' datetime objects are “salted” with an unpredictable ' - 'random value.\n' - ' Although they remain constant within an individual ' - 'Python\n' - ' process, they are not predictable between repeated ' - 'invocations of\n' - ' Python.This is intended to provide protection against ' - 'a denial-\n' - ' of-service caused by carefully-chosen inputs that ' - 'exploit the\n' - ' worst case performance of a dict insertion, O(n^2) ' - 'complexity.\n' - ' See ' - 'http://www.ocert.org/advisories/ocert-2011-003.html for\n' - ' details.Changing hash values affects the iteration ' - 'order of\n' - ' dicts, sets and other mappings. Python has never made ' - 'guarantees\n' - ' about this ordering (and it typically varies between ' - '32-bit and\n' - ' 64-bit builds).See also "PYTHONHASHSEED".\n' - '\n' - ' Changed in version 3.3: Hash randomization is enabled by ' - 'default.\n' - '\n' - 'object.__bool__(self)\n' - '\n' - ' Called to implement truth value testing and the built-in ' - 'operation\n' - ' "bool()"; should return "False" or "True". When this ' - 'method is not\n' - ' defined, "__len__()" is called, if it is defined, and ' - 'the object is\n' - ' considered true if its result is nonzero. If a class ' - 'defines\n' - ' neither "__len__()" nor "__bool__()", all its instances ' - 'are\n' - ' considered true.\n', - 'debugger': '"pdb" — The Python Debugger\n' - '***************************\n' - '\n' - '**Source code:** Lib/pdb.py\n' - '\n' - '======================================================================\n' - '\n' - 'The module "pdb" defines an interactive source code debugger ' - 'for\n' - 'Python programs. It supports setting (conditional) breakpoints ' - 'and\n' - 'single stepping at the source line level, inspection of stack ' - 'frames,\n' - 'source code listing, and evaluation of arbitrary Python code in ' - 'the\n' - 'context of any stack frame. It also supports post-mortem ' - 'debugging\n' - 'and can be called under program control.\n' - '\n' - 'The debugger is extensible – it is actually defined as the ' - 'class\n' - '"Pdb". This is currently undocumented but easily understood by ' - 'reading\n' - 'the source. The extension interface uses the modules "bdb" and ' - '"cmd".\n' - '\n' - 'The debugger’s prompt is "(Pdb)". Typical usage to run a program ' - 'under\n' - 'control of the debugger is:\n' - '\n' - ' >>> import pdb\n' - ' >>> import mymodule\n' - " >>> pdb.run('mymodule.test()')\n" - ' > (0)?()\n' - ' (Pdb) continue\n' - ' > (1)?()\n' - ' (Pdb) continue\n' - " NameError: 'spam'\n" - ' > (1)?()\n' - ' (Pdb)\n' - '\n' - 'Changed in version 3.3: Tab-completion via the "readline" module ' - 'is\n' - 'available for commands and command arguments, e.g. the current ' - 'global\n' - 'and local names are offered as arguments of the "p" command.\n' - '\n' - '"pdb.py" can also be invoked as a script to debug other ' - 'scripts. For\n' - 'example:\n' - '\n' - ' python3 -m pdb myscript.py\n' - '\n' - 'When invoked as a script, pdb will automatically enter ' - 'post-mortem\n' - 'debugging if the program being debugged exits abnormally. After ' - 'post-\n' - 'mortem debugging (or after normal exit of the program), pdb ' - 'will\n' - 'restart the program. Automatic restarting preserves pdb’s state ' - '(such\n' - 'as breakpoints) and in most cases is more useful than quitting ' - 'the\n' - 'debugger upon program’s exit.\n' - '\n' - 'New in version 3.2: "pdb.py" now accepts a "-c" option that ' - 'executes\n' - 'commands as if given in a ".pdbrc" file, see Debugger Commands.\n' - '\n' - 'The typical usage to break into the debugger from a running ' - 'program is\n' - 'to insert\n' - '\n' - ' import pdb; pdb.set_trace()\n' - '\n' - 'at the location you want to break into the debugger. You can ' - 'then\n' - 'step through the code following this statement, and continue ' - 'running\n' - 'without the debugger using the "continue" command.\n' - '\n' - 'The typical usage to inspect a crashed program is:\n' - '\n' - ' >>> import pdb\n' - ' >>> import mymodule\n' - ' >>> mymodule.test()\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - ' File "./mymodule.py", line 4, in test\n' - ' test2()\n' - ' File "./mymodule.py", line 3, in test2\n' - ' print(spam)\n' - ' NameError: spam\n' - ' >>> pdb.pm()\n' - ' > ./mymodule.py(3)test2()\n' - ' -> print(spam)\n' - ' (Pdb)\n' - '\n' - 'The module defines the following functions; each enters the ' - 'debugger\n' - 'in a slightly different way:\n' - '\n' - 'pdb.run(statement, globals=None, locals=None)\n' - '\n' - ' Execute the *statement* (given as a string or a code object) ' - 'under\n' - ' debugger control. The debugger prompt appears before any ' - 'code is\n' - ' executed; you can set breakpoints and type "continue", or you ' - 'can\n' - ' step through the statement using "step" or "next" (all these\n' - ' commands are explained below). The optional *globals* and ' - '*locals*\n' - ' arguments specify the environment in which the code is ' - 'executed; by\n' - ' default the dictionary of the module "__main__" is used. ' - '(See the\n' - ' explanation of the built-in "exec()" or "eval()" functions.)\n' - '\n' - 'pdb.runeval(expression, globals=None, locals=None)\n' - '\n' - ' Evaluate the *expression* (given as a string or a code ' - 'object)\n' - ' under debugger control. When "runeval()" returns, it returns ' - 'the\n' - ' value of the expression. Otherwise this function is similar ' - 'to\n' - ' "run()".\n' - '\n' - 'pdb.runcall(function, *args, **kwds)\n' - '\n' - ' Call the *function* (a function or method object, not a ' - 'string)\n' - ' with the given arguments. When "runcall()" returns, it ' - 'returns\n' - ' whatever the function call returned. The debugger prompt ' - 'appears\n' - ' as soon as the function is entered.\n' - '\n' - 'pdb.set_trace()\n' - '\n' - ' Enter the debugger at the calling stack frame. This is ' - 'useful to\n' - ' hard-code a breakpoint at a given point in a program, even if ' - 'the\n' - ' code is not otherwise being debugged (e.g. when an assertion\n' - ' fails).\n' - '\n' - 'pdb.post_mortem(traceback=None)\n' - '\n' - ' Enter post-mortem debugging of the given *traceback* object. ' - 'If no\n' - ' *traceback* is given, it uses the one of the exception that ' - 'is\n' - ' currently being handled (an exception must be being handled ' - 'if the\n' - ' default is to be used).\n' - '\n' - 'pdb.pm()\n' - '\n' - ' Enter post-mortem debugging of the traceback found in\n' - ' "sys.last_traceback".\n' - '\n' - 'The "run*" functions and "set_trace()" are aliases for ' - 'instantiating\n' - 'the "Pdb" class and calling the method of the same name. If you ' - 'want\n' - 'to access further features, you have to do this yourself:\n' - '\n' - "class pdb.Pdb(completekey='tab', stdin=None, stdout=None, " - 'skip=None, nosigint=False, readrc=True)\n' - '\n' - ' "Pdb" is the debugger class.\n' - '\n' - ' The *completekey*, *stdin* and *stdout* arguments are passed ' - 'to the\n' - ' underlying "cmd.Cmd" class; see the description there.\n' - '\n' - ' The *skip* argument, if given, must be an iterable of ' - 'glob-style\n' - ' module name patterns. The debugger will not step into frames ' - 'that\n' - ' originate in a module that matches one of these patterns. ' - '[1]\n' - '\n' - ' By default, Pdb sets a handler for the SIGINT signal (which ' - 'is sent\n' - ' when the user presses "Ctrl-C" on the console) when you give ' - 'a\n' - ' "continue" command. This allows you to break into the ' - 'debugger\n' - ' again by pressing "Ctrl-C". If you want Pdb not to touch ' - 'the\n' - ' SIGINT handler, set *nosigint* to true.\n' - '\n' - ' The *readrc* argument defaults to true and controls whether ' - 'Pdb\n' - ' will load .pdbrc files from the filesystem.\n' - '\n' - ' Example call to enable tracing with *skip*:\n' - '\n' - " import pdb; pdb.Pdb(skip=['django.*']).set_trace()\n" - '\n' - ' New in version 3.1: The *skip* argument.\n' - '\n' - ' New in version 3.2: The *nosigint* argument. Previously, a ' - 'SIGINT\n' - ' handler was never set by Pdb.\n' - '\n' - ' Changed in version 3.6: The *readrc* argument.\n' - '\n' - ' run(statement, globals=None, locals=None)\n' - ' runeval(expression, globals=None, locals=None)\n' - ' runcall(function, *args, **kwds)\n' - ' set_trace()\n' - '\n' - ' See the documentation for the functions explained above.\n' - '\n' - '\n' - 'Debugger Commands\n' - '=================\n' - '\n' - 'The commands recognized by the debugger are listed below. Most\n' - 'commands can be abbreviated to one or two letters as indicated; ' - 'e.g.\n' - '"h(elp)" means that either "h" or "help" can be used to enter ' - 'the help\n' - 'command (but not "he" or "hel", nor "H" or "Help" or "HELP").\n' - 'Arguments to commands must be separated by whitespace (spaces ' - 'or\n' - 'tabs). Optional arguments are enclosed in square brackets ' - '("[]") in\n' - 'the command syntax; the square brackets must not be typed.\n' - 'Alternatives in the command syntax are separated by a vertical ' - 'bar\n' - '("|").\n' - '\n' - 'Entering a blank line repeats the last command entered. ' - 'Exception: if\n' - 'the last command was a "list" command, the next 11 lines are ' - 'listed.\n' - '\n' - 'Commands that the debugger doesn’t recognize are assumed to be ' - 'Python\n' - 'statements and are executed in the context of the program being\n' - 'debugged. Python statements can also be prefixed with an ' - 'exclamation\n' - 'point ("!"). This is a powerful way to inspect the program ' - 'being\n' - 'debugged; it is even possible to change a variable or call a ' - 'function.\n' - 'When an exception occurs in such a statement, the exception name ' - 'is\n' - 'printed but the debugger’s state is not changed.\n' - '\n' - 'The debugger supports aliases. Aliases can have parameters ' - 'which\n' - 'allows one a certain level of adaptability to the context under\n' - 'examination.\n' - '\n' - 'Multiple commands may be entered on a single line, separated by ' - '";;".\n' - '(A single ";" is not used as it is the separator for multiple ' - 'commands\n' - 'in a line that is passed to the Python parser.) No intelligence ' - 'is\n' - 'applied to separating the commands; the input is split at the ' - 'first\n' - '";;" pair, even if it is in the middle of a quoted string.\n' - '\n' - 'If a file ".pdbrc" exists in the user’s home directory or in ' - 'the\n' - 'current directory, it is read in and executed as if it had been ' - 'typed\n' - 'at the debugger prompt. This is particularly useful for ' - 'aliases. If\n' - 'both files exist, the one in the home directory is read first ' - 'and\n' - 'aliases defined there can be overridden by the local file.\n' - '\n' - 'Changed in version 3.2: ".pdbrc" can now contain commands that\n' - 'continue debugging, such as "continue" or "next". Previously, ' - 'these\n' - 'commands had no effect.\n' - '\n' - 'h(elp) [command]\n' - '\n' - ' Without argument, print the list of available commands. With ' - 'a\n' - ' *command* as argument, print help about that command. "help ' - 'pdb"\n' - ' displays the full documentation (the docstring of the "pdb"\n' - ' module). Since the *command* argument must be an identifier, ' - '"help\n' - ' exec" must be entered to get help on the "!" command.\n' - '\n' - 'w(here)\n' - '\n' - ' Print a stack trace, with the most recent frame at the ' - 'bottom. An\n' - ' arrow indicates the current frame, which determines the ' - 'context of\n' - ' most commands.\n' - '\n' - 'd(own) [count]\n' - '\n' - ' Move the current frame *count* (default one) levels down in ' - 'the\n' - ' stack trace (to a newer frame).\n' - '\n' - 'u(p) [count]\n' - '\n' - ' Move the current frame *count* (default one) levels up in the ' - 'stack\n' - ' trace (to an older frame).\n' - '\n' - 'b(reak) [([filename:]lineno | function) [, condition]]\n' - '\n' - ' With a *lineno* argument, set a break there in the current ' - 'file.\n' - ' With a *function* argument, set a break at the first ' - 'executable\n' - ' statement within that function. The line number may be ' - 'prefixed\n' - ' with a filename and a colon, to specify a breakpoint in ' - 'another\n' - ' file (probably one that hasn’t been loaded yet). The file ' - 'is\n' - ' searched on "sys.path". Note that each breakpoint is ' - 'assigned a\n' - ' number to which all the other breakpoint commands refer.\n' - '\n' - ' If a second argument is present, it is an expression which ' - 'must\n' - ' evaluate to true before the breakpoint is honored.\n' - '\n' - ' Without argument, list all breaks, including for each ' - 'breakpoint,\n' - ' the number of times that breakpoint has been hit, the ' - 'current\n' - ' ignore count, and the associated condition if any.\n' - '\n' - 'tbreak [([filename:]lineno | function) [, condition]]\n' - '\n' - ' Temporary breakpoint, which is removed automatically when it ' - 'is\n' - ' first hit. The arguments are the same as for "break".\n' - '\n' - 'cl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n' - '\n' - ' With a *filename:lineno* argument, clear all the breakpoints ' - 'at\n' - ' this line. With a space separated list of breakpoint numbers, ' - 'clear\n' - ' those breakpoints. Without argument, clear all breaks (but ' - 'first\n' - ' ask confirmation).\n' - '\n' - 'disable [bpnumber [bpnumber ...]]\n' - '\n' - ' Disable the breakpoints given as a space separated list of\n' - ' breakpoint numbers. Disabling a breakpoint means it cannot ' - 'cause\n' - ' the program to stop execution, but unlike clearing a ' - 'breakpoint, it\n' - ' remains in the list of breakpoints and can be (re-)enabled.\n' - '\n' - 'enable [bpnumber [bpnumber ...]]\n' - '\n' - ' Enable the breakpoints specified.\n' - '\n' - 'ignore bpnumber [count]\n' - '\n' - ' Set the ignore count for the given breakpoint number. If ' - 'count is\n' - ' omitted, the ignore count is set to 0. A breakpoint becomes ' - 'active\n' - ' when the ignore count is zero. When non-zero, the count is\n' - ' decremented each time the breakpoint is reached and the ' - 'breakpoint\n' - ' is not disabled and any associated condition evaluates to ' - 'true.\n' - '\n' - 'condition bpnumber [condition]\n' - '\n' - ' Set a new *condition* for the breakpoint, an expression which ' - 'must\n' - ' evaluate to true before the breakpoint is honored. If ' - '*condition*\n' - ' is absent, any existing condition is removed; i.e., the ' - 'breakpoint\n' - ' is made unconditional.\n' - '\n' - 'commands [bpnumber]\n' - '\n' - ' Specify a list of commands for breakpoint number *bpnumber*. ' - 'The\n' - ' commands themselves appear on the following lines. Type a ' - 'line\n' - ' containing just "end" to terminate the commands. An example:\n' - '\n' - ' (Pdb) commands 1\n' - ' (com) p some_variable\n' - ' (com) end\n' - ' (Pdb)\n' - '\n' - ' To remove all commands from a breakpoint, type commands and ' - 'follow\n' - ' it immediately with "end"; that is, give no commands.\n' - '\n' - ' With no *bpnumber* argument, commands refers to the last ' - 'breakpoint\n' - ' set.\n' - '\n' - ' You can use breakpoint commands to start your program up ' - 'again.\n' - ' Simply use the continue command, or step, or any other ' - 'command that\n' - ' resumes execution.\n' - '\n' - ' Specifying any command resuming execution (currently ' - 'continue,\n' - ' step, next, return, jump, quit and their abbreviations) ' - 'terminates\n' - ' the command list (as if that command was immediately followed ' - 'by\n' - ' end). This is because any time you resume execution (even ' - 'with a\n' - ' simple next or step), you may encounter another ' - 'breakpoint—which\n' - ' could have its own command list, leading to ambiguities about ' - 'which\n' - ' list to execute.\n' - '\n' - ' If you use the ‘silent’ command in the command list, the ' - 'usual\n' - ' message about stopping at a breakpoint is not printed. This ' - 'may be\n' - ' desirable for breakpoints that are to print a specific ' - 'message and\n' - ' then continue. If none of the other commands print anything, ' - 'you\n' - ' see no sign that the breakpoint was reached.\n' - '\n' - 's(tep)\n' - '\n' - ' Execute the current line, stop at the first possible ' - 'occasion\n' - ' (either in a function that is called or on the next line in ' - 'the\n' - ' current function).\n' - '\n' - 'n(ext)\n' - '\n' - ' Continue execution until the next line in the current ' - 'function is\n' - ' reached or it returns. (The difference between "next" and ' - '"step"\n' - ' is that "step" stops inside a called function, while "next"\n' - ' executes called functions at (nearly) full speed, only ' - 'stopping at\n' - ' the next line in the current function.)\n' - '\n' - 'unt(il) [lineno]\n' - '\n' - ' Without argument, continue execution until the line with a ' - 'number\n' - ' greater than the current one is reached.\n' - '\n' - ' With a line number, continue execution until a line with a ' - 'number\n' - ' greater or equal to that is reached. In both cases, also ' - 'stop when\n' - ' the current frame returns.\n' - '\n' - ' Changed in version 3.2: Allow giving an explicit line ' - 'number.\n' - '\n' - 'r(eturn)\n' - '\n' - ' Continue execution until the current function returns.\n' - '\n' - 'c(ont(inue))\n' - '\n' - ' Continue execution, only stop when a breakpoint is ' - 'encountered.\n' - '\n' - 'j(ump) lineno\n' - '\n' - ' Set the next line that will be executed. Only available in ' - 'the\n' - ' bottom-most frame. This lets you jump back and execute code ' - 'again,\n' - ' or jump forward to skip code that you don’t want to run.\n' - '\n' - ' It should be noted that not all jumps are allowed – for ' - 'instance it\n' - ' is not possible to jump into the middle of a "for" loop or ' - 'out of a\n' - ' "finally" clause.\n' - '\n' - 'l(ist) [first[, last]]\n' - '\n' - ' List source code for the current file. Without arguments, ' - 'list 11\n' - ' lines around the current line or continue the previous ' - 'listing.\n' - ' With "." as argument, list 11 lines around the current line. ' - 'With\n' - ' one argument, list 11 lines around at that line. With two\n' - ' arguments, list the given range; if the second argument is ' - 'less\n' - ' than the first, it is interpreted as a count.\n' - '\n' - ' The current line in the current frame is indicated by "->". ' - 'If an\n' - ' exception is being debugged, the line where the exception ' - 'was\n' - ' originally raised or propagated is indicated by ">>", if it ' - 'differs\n' - ' from the current line.\n' - '\n' - ' New in version 3.2: The ">>" marker.\n' - '\n' - 'll | longlist\n' - '\n' - ' List all source code for the current function or frame.\n' - ' Interesting lines are marked as for "list".\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'a(rgs)\n' - '\n' - ' Print the argument list of the current function.\n' - '\n' - 'p expression\n' - '\n' - ' Evaluate the *expression* in the current context and print ' - 'its\n' - ' value.\n' - '\n' - ' Note: "print()" can also be used, but is not a debugger ' - 'command —\n' - ' this executes the Python "print()" function.\n' - '\n' - 'pp expression\n' - '\n' - ' Like the "p" command, except the value of the expression is ' - 'pretty-\n' - ' printed using the "pprint" module.\n' - '\n' - 'whatis expression\n' - '\n' - ' Print the type of the *expression*.\n' - '\n' - 'source expression\n' - '\n' - ' Try to get source code for the given object and display it.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'display [expression]\n' - '\n' - ' Display the value of the expression if it changed, each time\n' - ' execution stops in the current frame.\n' - '\n' - ' Without expression, list all display expressions for the ' - 'current\n' - ' frame.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'undisplay [expression]\n' - '\n' - ' Do not display the expression any more in the current frame.\n' - ' Without expression, clear all display expressions for the ' - 'current\n' - ' frame.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'interact\n' - '\n' - ' Start an interactive interpreter (using the "code" module) ' - 'whose\n' - ' global namespace contains all the (global and local) names ' - 'found in\n' - ' the current scope.\n' - '\n' - ' New in version 3.2.\n' - '\n' - 'alias [name [command]]\n' - '\n' - ' Create an alias called *name* that executes *command*. The ' - 'command\n' - ' must *not* be enclosed in quotes. Replaceable parameters can ' - 'be\n' - ' indicated by "%1", "%2", and so on, while "%*" is replaced by ' - 'all\n' - ' the parameters. If no command is given, the current alias ' - 'for\n' - ' *name* is shown. If no arguments are given, all aliases are ' - 'listed.\n' - '\n' - ' Aliases may be nested and can contain anything that can be ' - 'legally\n' - ' typed at the pdb prompt. Note that internal pdb commands ' - '*can* be\n' - ' overridden by aliases. Such a command is then hidden until ' - 'the\n' - ' alias is removed. Aliasing is recursively applied to the ' - 'first\n' - ' word of the command line; all other words in the line are ' - 'left\n' - ' alone.\n' - '\n' - ' As an example, here are two useful aliases (especially when ' - 'placed\n' - ' in the ".pdbrc" file):\n' - '\n' - ' # Print instance variables (usage "pi classInst")\n' - ' alias pi for k in %1.__dict__.keys(): ' - 'print("%1.",k,"=",%1.__dict__[k])\n' - ' # Print instance variables in self\n' - ' alias ps pi self\n' - '\n' - 'unalias name\n' - '\n' - ' Delete the specified alias.\n' - '\n' - '! statement\n' - '\n' - ' Execute the (one-line) *statement* in the context of the ' - 'current\n' - ' stack frame. The exclamation point can be omitted unless the ' - 'first\n' - ' word of the statement resembles a debugger command. To set ' - 'a\n' - ' global variable, you can prefix the assignment command with ' - 'a\n' - ' "global" statement on the same line, e.g.:\n' - '\n' - " (Pdb) global list_options; list_options = ['-l']\n" - ' (Pdb)\n' - '\n' - 'run [args ...]\n' - 'restart [args ...]\n' - '\n' - ' Restart the debugged Python program. If an argument is ' - 'supplied,\n' - ' it is split with "shlex" and the result is used as the new\n' - ' "sys.argv". History, breakpoints, actions and debugger ' - 'options are\n' - ' preserved. "restart" is an alias for "run".\n' - '\n' - 'q(uit)\n' - '\n' - ' Quit from the debugger. The program being executed is ' - 'aborted.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] Whether a frame is considered to originate in a certain ' - 'module\n' - ' is determined by the "__name__" in the frame globals.\n', - 'del': 'The "del" statement\n' - '*******************\n' - '\n' - ' del_stmt ::= "del" target_list\n' - '\n' - 'Deletion is recursively defined very similar to the way assignment ' - 'is\n' - 'defined. Rather than spelling it out in full details, here are some\n' - 'hints.\n' - '\n' - 'Deletion of a target list recursively deletes each target, from left\n' - 'to right.\n' - '\n' - 'Deletion of a name removes the binding of that name from the local ' - 'or\n' - 'global namespace, depending on whether the name occurs in a "global"\n' - 'statement in the same code block. If the name is unbound, a\n' - '"NameError" exception will be raised.\n' - '\n' - 'Deletion of attribute references, subscriptions and slicings is ' - 'passed\n' - 'to the primary object involved; deletion of a slicing is in general\n' - 'equivalent to assignment of an empty slice of the right type (but ' - 'even\n' - 'this is determined by the sliced object).\n' - '\n' - 'Changed in version 3.2: Previously it was illegal to delete a name\n' - 'from the local namespace if it occurs as a free variable in a nested\n' - 'block.\n', - 'dict': 'Dictionary displays\n' - '*******************\n' - '\n' - 'A dictionary display is a possibly empty series of key/datum pairs\n' - 'enclosed in curly braces:\n' - '\n' - ' dict_display ::= "{" [key_datum_list | dict_comprehension] ' - '"}"\n' - ' key_datum_list ::= key_datum ("," key_datum)* [","]\n' - ' key_datum ::= expression ":" expression | "**" or_expr\n' - ' dict_comprehension ::= expression ":" expression comp_for\n' - '\n' - 'A dictionary display yields a new dictionary object.\n' - '\n' - 'If a comma-separated sequence of key/datum pairs is given, they are\n' - 'evaluated from left to right to define the entries of the ' - 'dictionary:\n' - 'each key object is used as a key into the dictionary to store the\n' - 'corresponding datum. This means that you can specify the same key\n' - 'multiple times in the key/datum list, and the final dictionary’s ' - 'value\n' - 'for that key will be the last one given.\n' - '\n' - 'A double asterisk "**" denotes *dictionary unpacking*. Its operand\n' - 'must be a *mapping*. Each mapping item is added to the new\n' - 'dictionary. Later values replace values already set by earlier\n' - 'key/datum pairs and earlier dictionary unpackings.\n' - '\n' - 'New in version 3.5: Unpacking into dictionary displays, originally\n' - 'proposed by **PEP 448**.\n' - '\n' - 'A dict comprehension, in contrast to list and set comprehensions,\n' - 'needs two expressions separated with a colon followed by the usual\n' - '“for” and “if” clauses. When the comprehension is run, the ' - 'resulting\n' - 'key and value elements are inserted in the new dictionary in the ' - 'order\n' - 'they are produced.\n' - '\n' - 'Restrictions on the types of the key values are listed earlier in\n' - 'section The standard type hierarchy. (To summarize, the key type\n' - 'should be *hashable*, which excludes all mutable objects.) Clashes\n' - 'between duplicate keys are not detected; the last datum (textually\n' - 'rightmost in the display) stored for a given key value prevails.\n', - 'dynamic-features': 'Interaction with dynamic features\n' - '*********************************\n' - '\n' - 'Name resolution of free variables occurs at runtime, not ' - 'at compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access ' - 'to the full\n' - 'environment for resolving names. Names may be resolved ' - 'in the local\n' - 'and global namespaces of the caller. Free variables are ' - 'not resolved\n' - 'in the nearest enclosing namespace, but in the global ' - 'namespace. [1]\n' - 'The "exec()" and "eval()" functions have optional ' - 'arguments to\n' - 'override the global and local namespace. If only one ' - 'namespace is\n' - 'specified, it is used for both.\n', - 'else': 'The "if" statement\n' - '******************\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the expressions ' - 'one\n' - 'by one until one is found to be true (see section Boolean ' - 'operations\n' - 'for the definition of true and false); then that suite is executed\n' - '(and no other part of the "if" statement is executed or evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, if\n' - 'present, is executed.\n', - 'exceptions': 'Exceptions\n' - '**********\n' - '\n' - 'Exceptions are a means of breaking out of the normal flow of ' - 'control\n' - 'of a code block in order to handle errors or other ' - 'exceptional\n' - 'conditions. An exception is *raised* at the point where the ' - 'error is\n' - 'detected; it may be *handled* by the surrounding code block or ' - 'by any\n' - 'code block that directly or indirectly invoked the code block ' - 'where\n' - 'the error occurred.\n' - '\n' - 'The Python interpreter raises an exception when it detects a ' - 'run-time\n' - 'error (such as division by zero). A Python program can also\n' - 'explicitly raise an exception with the "raise" statement. ' - 'Exception\n' - 'handlers are specified with the "try" … "except" statement. ' - 'The\n' - '"finally" clause of such a statement can be used to specify ' - 'cleanup\n' - 'code which does not handle the exception, but is executed ' - 'whether an\n' - 'exception occurred or not in the preceding code.\n' - '\n' - 'Python uses the “termination” model of error handling: an ' - 'exception\n' - 'handler can find out what happened and continue execution at ' - 'an outer\n' - 'level, but it cannot repair the cause of the error and retry ' - 'the\n' - 'failing operation (except by re-entering the offending piece ' - 'of code\n' - 'from the top).\n' - '\n' - 'When an exception is not handled at all, the interpreter ' - 'terminates\n' - 'execution of the program, or returns to its interactive main ' - 'loop. In\n' - 'either case, it prints a stack backtrace, except when the ' - 'exception is\n' - '"SystemExit".\n' - '\n' - 'Exceptions are identified by class instances. The "except" ' - 'clause is\n' - 'selected depending on the class of the instance: it must ' - 'reference the\n' - 'class of the instance or a base class thereof. The instance ' - 'can be\n' - 'received by the handler and can carry additional information ' - 'about the\n' - 'exceptional condition.\n' - '\n' - 'Note: Exception messages are not part of the Python API. ' - 'Their\n' - ' contents may change from one version of Python to the next ' - 'without\n' - ' warning and should not be relied on by code which will run ' - 'under\n' - ' multiple versions of the interpreter.\n' - '\n' - 'See also the description of the "try" statement in section The ' - 'try\n' - 'statement and "raise" statement in section The raise ' - 'statement.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] This limitation occurs because the code that is executed ' - 'by\n' - ' these operations is not available at the time the module ' - 'is\n' - ' compiled.\n', - 'execmodel': 'Execution model\n' - '***************\n' - '\n' - '\n' - 'Structure of a program\n' - '======================\n' - '\n' - 'A Python program is constructed from code blocks. A *block* is ' - 'a piece\n' - 'of Python program text that is executed as a unit. The ' - 'following are\n' - 'blocks: a module, a function body, and a class definition. ' - 'Each\n' - 'command typed interactively is a block. A script file (a file ' - 'given\n' - 'as standard input to the interpreter or specified as a command ' - 'line\n' - 'argument to the interpreter) is a code block. A script command ' - '(a\n' - 'command specified on the interpreter command line with the ' - '"-c"\n' - 'option) is a code block. The string argument passed to the ' - 'built-in\n' - 'functions "eval()" and "exec()" is a code block.\n' - '\n' - 'A code block is executed in an *execution frame*. A frame ' - 'contains\n' - 'some administrative information (used for debugging) and ' - 'determines\n' - 'where and how execution continues after the code block’s ' - 'execution has\n' - 'completed.\n' - '\n' - '\n' - 'Naming and binding\n' - '==================\n' - '\n' - '\n' - 'Binding of names\n' - '----------------\n' - '\n' - '*Names* refer to objects. Names are introduced by name ' - 'binding\n' - 'operations.\n' - '\n' - 'The following constructs bind names: formal parameters to ' - 'functions,\n' - '"import" statements, class and function definitions (these bind ' - 'the\n' - 'class or function name in the defining block), and targets that ' - 'are\n' - 'identifiers if occurring in an assignment, "for" loop header, ' - 'or after\n' - '"as" in a "with" statement or "except" clause. The "import" ' - 'statement\n' - 'of the form "from ... import *" binds all names defined in the\n' - 'imported module, except those beginning with an underscore. ' - 'This form\n' - 'may only be used at the module level.\n' - '\n' - 'A target occurring in a "del" statement is also considered ' - 'bound for\n' - 'this purpose (though the actual semantics are to unbind the ' - 'name).\n' - '\n' - 'Each assignment or import statement occurs within a block ' - 'defined by a\n' - 'class or function definition or at the module level (the ' - 'top-level\n' - 'code block).\n' - '\n' - 'If a name is bound in a block, it is a local variable of that ' - 'block,\n' - 'unless declared as "nonlocal" or "global". If a name is bound ' - 'at the\n' - 'module level, it is a global variable. (The variables of the ' - 'module\n' - 'code block are local and global.) If a variable is used in a ' - 'code\n' - 'block but not defined there, it is a *free variable*.\n' - '\n' - 'Each occurrence of a name in the program text refers to the ' - '*binding*\n' - 'of that name established by the following name resolution ' - 'rules.\n' - '\n' - '\n' - 'Resolution of names\n' - '-------------------\n' - '\n' - 'A *scope* defines the visibility of a name within a block. If ' - 'a local\n' - 'variable is defined in a block, its scope includes that block. ' - 'If the\n' - 'definition occurs in a function block, the scope extends to any ' - 'blocks\n' - 'contained within the defining one, unless a contained block ' - 'introduces\n' - 'a different binding for the name.\n' - '\n' - 'When a name is used in a code block, it is resolved using the ' - 'nearest\n' - 'enclosing scope. The set of all such scopes visible to a code ' - 'block\n' - 'is called the block’s *environment*.\n' - '\n' - 'When a name is not found at all, a "NameError" exception is ' - 'raised. If\n' - 'the current scope is a function scope, and the name refers to a ' - 'local\n' - 'variable that has not yet been bound to a value at the point ' - 'where the\n' - 'name is used, an "UnboundLocalError" exception is raised.\n' - '"UnboundLocalError" is a subclass of "NameError".\n' - '\n' - 'If a name binding operation occurs anywhere within a code ' - 'block, all\n' - 'uses of the name within the block are treated as references to ' - 'the\n' - 'current block. This can lead to errors when a name is used ' - 'within a\n' - 'block before it is bound. This rule is subtle. Python lacks\n' - 'declarations and allows name binding operations to occur ' - 'anywhere\n' - 'within a code block. The local variables of a code block can ' - 'be\n' - 'determined by scanning the entire text of the block for name ' - 'binding\n' - 'operations.\n' - '\n' - 'If the "global" statement occurs within a block, all uses of ' - 'the name\n' - 'specified in the statement refer to the binding of that name in ' - 'the\n' - 'top-level namespace. Names are resolved in the top-level ' - 'namespace by\n' - 'searching the global namespace, i.e. the namespace of the ' - 'module\n' - 'containing the code block, and the builtins namespace, the ' - 'namespace\n' - 'of the module "builtins". The global namespace is searched ' - 'first. If\n' - 'the name is not found there, the builtins namespace is ' - 'searched. The\n' - '"global" statement must precede all uses of the name.\n' - '\n' - 'The "global" statement has the same scope as a name binding ' - 'operation\n' - 'in the same block. If the nearest enclosing scope for a free ' - 'variable\n' - 'contains a global statement, the free variable is treated as a ' - 'global.\n' - '\n' - 'The "nonlocal" statement causes corresponding names to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing function ' - 'scope.\n' - '"SyntaxError" is raised at compile time if the given name does ' - 'not\n' - 'exist in any enclosing function scope.\n' - '\n' - 'The namespace for a module is automatically created the first ' - 'time a\n' - 'module is imported. The main module for a script is always ' - 'called\n' - '"__main__".\n' - '\n' - 'Class definition blocks and arguments to "exec()" and "eval()" ' - 'are\n' - 'special in the context of name resolution. A class definition ' - 'is an\n' - 'executable statement that may use and define names. These ' - 'references\n' - 'follow the normal rules for name resolution with an exception ' - 'that\n' - 'unbound local variables are looked up in the global namespace. ' - 'The\n' - 'namespace of the class definition becomes the attribute ' - 'dictionary of\n' - 'the class. The scope of names defined in a class block is ' - 'limited to\n' - 'the class block; it does not extend to the code blocks of ' - 'methods –\n' - 'this includes comprehensions and generator expressions since ' - 'they are\n' - 'implemented using a function scope. This means that the ' - 'following\n' - 'will fail:\n' - '\n' - ' class A:\n' - ' a = 42\n' - ' b = list(a + i for i in range(10))\n' - '\n' - '\n' - 'Builtins and restricted execution\n' - '---------------------------------\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. ' - 'Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' - '\n' - 'The builtins namespace associated with the execution of a code ' - 'block\n' - 'is actually found by looking up the name "__builtins__" in its ' - 'global\n' - 'namespace; this should be a dictionary or a module (in the ' - 'latter case\n' - 'the module’s dictionary is used). By default, when in the ' - '"__main__"\n' - 'module, "__builtins__" is the built-in module "builtins"; when ' - 'in any\n' - 'other module, "__builtins__" is an alias for the dictionary of ' - 'the\n' - '"builtins" module itself.\n' - '\n' - '\n' - 'Interaction with dynamic features\n' - '---------------------------------\n' - '\n' - 'Name resolution of free variables occurs at runtime, not at ' - 'compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access to the ' - 'full\n' - 'environment for resolving names. Names may be resolved in the ' - 'local\n' - 'and global namespaces of the caller. Free variables are not ' - 'resolved\n' - 'in the nearest enclosing namespace, but in the global ' - 'namespace. [1]\n' - 'The "exec()" and "eval()" functions have optional arguments to\n' - 'override the global and local namespace. If only one namespace ' - 'is\n' - 'specified, it is used for both.\n' - '\n' - '\n' - 'Exceptions\n' - '==========\n' - '\n' - 'Exceptions are a means of breaking out of the normal flow of ' - 'control\n' - 'of a code block in order to handle errors or other exceptional\n' - 'conditions. An exception is *raised* at the point where the ' - 'error is\n' - 'detected; it may be *handled* by the surrounding code block or ' - 'by any\n' - 'code block that directly or indirectly invoked the code block ' - 'where\n' - 'the error occurred.\n' - '\n' - 'The Python interpreter raises an exception when it detects a ' - 'run-time\n' - 'error (such as division by zero). A Python program can also\n' - 'explicitly raise an exception with the "raise" statement. ' - 'Exception\n' - 'handlers are specified with the "try" … "except" statement. ' - 'The\n' - '"finally" clause of such a statement can be used to specify ' - 'cleanup\n' - 'code which does not handle the exception, but is executed ' - 'whether an\n' - 'exception occurred or not in the preceding code.\n' - '\n' - 'Python uses the “termination” model of error handling: an ' - 'exception\n' - 'handler can find out what happened and continue execution at an ' - 'outer\n' - 'level, but it cannot repair the cause of the error and retry ' - 'the\n' - 'failing operation (except by re-entering the offending piece of ' - 'code\n' - 'from the top).\n' - '\n' - 'When an exception is not handled at all, the interpreter ' - 'terminates\n' - 'execution of the program, or returns to its interactive main ' - 'loop. In\n' - 'either case, it prints a stack backtrace, except when the ' - 'exception is\n' - '"SystemExit".\n' - '\n' - 'Exceptions are identified by class instances. The "except" ' - 'clause is\n' - 'selected depending on the class of the instance: it must ' - 'reference the\n' - 'class of the instance or a base class thereof. The instance ' - 'can be\n' - 'received by the handler and can carry additional information ' - 'about the\n' - 'exceptional condition.\n' - '\n' - 'Note: Exception messages are not part of the Python API. ' - 'Their\n' - ' contents may change from one version of Python to the next ' - 'without\n' - ' warning and should not be relied on by code which will run ' - 'under\n' - ' multiple versions of the interpreter.\n' - '\n' - 'See also the description of the "try" statement in section The ' - 'try\n' - 'statement and "raise" statement in section The raise ' - 'statement.\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] This limitation occurs because the code that is executed ' - 'by\n' - ' these operations is not available at the time the module ' - 'is\n' - ' compiled.\n', - 'exprlists': 'Expression lists\n' - '****************\n' - '\n' - ' expression_list ::= expression ("," expression)* [","]\n' - ' starred_list ::= starred_item ("," starred_item)* ' - '[","]\n' - ' starred_expression ::= expression | (starred_item ",")* ' - '[starred_item]\n' - ' starred_item ::= expression | "*" or_expr\n' - '\n' - 'Except when part of a list or set display, an expression list\n' - 'containing at least one comma yields a tuple. The length of ' - 'the tuple\n' - 'is the number of expressions in the list. The expressions are\n' - 'evaluated from left to right.\n' - '\n' - 'An asterisk "*" denotes *iterable unpacking*. Its operand must ' - 'be an\n' - '*iterable*. The iterable is expanded into a sequence of items, ' - 'which\n' - 'are included in the new tuple, list, or set, at the site of ' - 'the\n' - 'unpacking.\n' - '\n' - 'New in version 3.5: Iterable unpacking in expression lists, ' - 'originally\n' - 'proposed by **PEP 448**.\n' - '\n' - 'The trailing comma is required only to create a single tuple ' - '(a.k.a. a\n' - '*singleton*); it is optional in all other cases. A single ' - 'expression\n' - 'without a trailing comma doesn’t create a tuple, but rather ' - 'yields the\n' - 'value of that expression. (To create an empty tuple, use an ' - 'empty pair\n' - 'of parentheses: "()".)\n', - 'floating': 'Floating point literals\n' - '***********************\n' - '\n' - 'Floating point literals are described by the following lexical\n' - 'definitions:\n' - '\n' - ' floatnumber ::= pointfloat | exponentfloat\n' - ' pointfloat ::= [digitpart] fraction | digitpart "."\n' - ' exponentfloat ::= (digitpart | pointfloat) exponent\n' - ' digitpart ::= digit (["_"] digit)*\n' - ' fraction ::= "." digitpart\n' - ' exponent ::= ("e" | "E") ["+" | "-"] digitpart\n' - '\n' - 'Note that the integer and exponent parts are always interpreted ' - 'using\n' - 'radix 10. For example, "077e010" is legal, and denotes the same ' - 'number\n' - 'as "77e10". The allowed range of floating point literals is\n' - 'implementation-dependent. As in integer literals, underscores ' - 'are\n' - 'supported for digit grouping.\n' - '\n' - 'Some examples of floating point literals:\n' - '\n' - ' 3.14 10. .001 1e100 3.14e-10 0e0 ' - '3.14_15_93\n' - '\n' - 'Changed in version 3.6: Underscores are now allowed for ' - 'grouping\n' - 'purposes in literals.\n', - 'for': 'The "for" statement\n' - '*******************\n' - '\n' - 'The "for" statement is used to iterate over the elements of a ' - 'sequence\n' - '(such as a string, tuple or list) or other iterable object:\n' - '\n' - ' for_stmt ::= "for" target_list "in" expression_list ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'The expression list is evaluated once; it should yield an iterable\n' - 'object. An iterator is created for the result of the\n' - '"expression_list". The suite is then executed once for each item\n' - 'provided by the iterator, in the order returned by the iterator. ' - 'Each\n' - 'item in turn is assigned to the target list using the standard rules\n' - 'for assignments (see Assignment statements), and then the suite is\n' - 'executed. When the items are exhausted (which is immediately when ' - 'the\n' - 'sequence is empty or an iterator raises a "StopIteration" ' - 'exception),\n' - 'the suite in the "else" clause, if present, is executed, and the ' - 'loop\n' - 'terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the loop\n' - 'without executing the "else" clause’s suite. A "continue" statement\n' - 'executed in the first suite skips the rest of the suite and ' - 'continues\n' - 'with the next item, or with the "else" clause if there is no next\n' - 'item.\n' - '\n' - 'The for-loop makes assignments to the variables(s) in the target ' - 'list.\n' - 'This overwrites all previous assignments to those variables ' - 'including\n' - 'those made in the suite of the for-loop:\n' - '\n' - ' for i in range(10):\n' - ' print(i)\n' - ' i = 5 # this will not affect the for-loop\n' - ' # because i will be overwritten with the ' - 'next\n' - ' # index in the range\n' - '\n' - 'Names in the target list are not deleted when the loop is finished,\n' - 'but if the sequence is empty, they will not have been assigned to at\n' - 'all by the loop. Hint: the built-in function "range()" returns an\n' - 'iterator of integers suitable to emulate the effect of Pascal’s "for ' - 'i\n' - ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n' - '\n' - 'Note: There is a subtlety when the sequence is being modified by the\n' - ' loop (this can only occur for mutable sequences, e.g. lists). An\n' - ' internal counter is used to keep track of which item is used next,\n' - ' and this is incremented on each iteration. When this counter has\n' - ' reached the length of the sequence the loop terminates. This ' - 'means\n' - ' that if the suite deletes the current (or a previous) item from ' - 'the\n' - ' sequence, the next item will be skipped (since it gets the index ' - 'of\n' - ' the current item which has already been treated). Likewise, if ' - 'the\n' - ' suite inserts an item in the sequence before the current item, the\n' - ' current item will be treated again the next time through the loop.\n' - ' This can lead to nasty bugs that can be avoided by making a\n' - ' temporary copy using a slice of the whole sequence, e.g.,\n' - '\n' - ' for x in a[:]:\n' - ' if x < 0: a.remove(x)\n', - 'formatstrings': 'Format String Syntax\n' - '********************\n' - '\n' - 'The "str.format()" method and the "Formatter" class share ' - 'the same\n' - 'syntax for format strings (although in the case of ' - '"Formatter",\n' - 'subclasses can define their own format string syntax). The ' - 'syntax is\n' - 'related to that of formatted string literals, but there ' - 'are\n' - 'differences.\n' - '\n' - 'Format strings contain “replacement fields” surrounded by ' - 'curly braces\n' - '"{}". Anything that is not contained in braces is ' - 'considered literal\n' - 'text, which is copied unchanged to the output. If you need ' - 'to include\n' - 'a brace character in the literal text, it can be escaped by ' - 'doubling:\n' - '"{{" and "}}".\n' - '\n' - 'The grammar for a replacement field is as follows:\n' - '\n' - ' replacement_field ::= "{" [field_name] ["!" ' - 'conversion] [":" format_spec] "}"\n' - ' field_name ::= arg_name ("." attribute_name | ' - '"[" element_index "]")*\n' - ' arg_name ::= [identifier | digit+]\n' - ' attribute_name ::= identifier\n' - ' element_index ::= digit+ | index_string\n' - ' index_string ::= +\n' - ' conversion ::= "r" | "s" | "a"\n' - ' format_spec ::= \n' - '\n' - 'In less formal terms, the replacement field can start with ' - 'a\n' - '*field_name* that specifies the object whose value is to be ' - 'formatted\n' - 'and inserted into the output instead of the replacement ' - 'field. The\n' - '*field_name* is optionally followed by a *conversion* ' - 'field, which is\n' - 'preceded by an exclamation point "\'!\'", and a ' - '*format_spec*, which is\n' - 'preceded by a colon "\':\'". These specify a non-default ' - 'format for the\n' - 'replacement value.\n' - '\n' - 'See also the Format Specification Mini-Language section.\n' - '\n' - 'The *field_name* itself begins with an *arg_name* that is ' - 'either a\n' - 'number or a keyword. If it’s a number, it refers to a ' - 'positional\n' - 'argument, and if it’s a keyword, it refers to a named ' - 'keyword\n' - 'argument. If the numerical arg_names in a format string ' - 'are 0, 1, 2,\n' - '… in sequence, they can all be omitted (not just some) and ' - 'the numbers\n' - '0, 1, 2, … will be automatically inserted in that order. ' - 'Because\n' - '*arg_name* is not quote-delimited, it is not possible to ' - 'specify\n' - 'arbitrary dictionary keys (e.g., the strings "\'10\'" or ' - '"\':-]\'") within\n' - 'a format string. The *arg_name* can be followed by any ' - 'number of index\n' - 'or attribute expressions. An expression of the form ' - '"\'.name\'" selects\n' - 'the named attribute using "getattr()", while an expression ' - 'of the form\n' - '"\'[index]\'" does an index lookup using "__getitem__()".\n' - '\n' - 'Changed in version 3.1: The positional argument specifiers ' - 'can be\n' - 'omitted for "str.format()", so "\'{} {}\'.format(a, b)" is ' - 'equivalent to\n' - '"\'{0} {1}\'.format(a, b)".\n' - '\n' - 'Changed in version 3.4: The positional argument specifiers ' - 'can be\n' - 'omitted for "Formatter".\n' - '\n' - 'Some simple format string examples:\n' - '\n' - ' "First, thou shalt count to {0}" # References first ' - 'positional argument\n' - ' "Bring me a {}" # Implicitly ' - 'references the first positional argument\n' - ' "From {} to {}" # Same as "From {0} to ' - '{1}"\n' - ' "My quest is {name}" # References keyword ' - "argument 'name'\n" - ' "Weight in tons {0.weight}" # \'weight\' attribute ' - 'of first positional arg\n' - ' "Units destroyed: {players[0]}" # First element of ' - "keyword argument 'players'.\n" - '\n' - 'The *conversion* field causes a type coercion before ' - 'formatting.\n' - 'Normally, the job of formatting a value is done by the ' - '"__format__()"\n' - 'method of the value itself. However, in some cases it is ' - 'desirable to\n' - 'force a type to be formatted as a string, overriding its ' - 'own\n' - 'definition of formatting. By converting the value to a ' - 'string before\n' - 'calling "__format__()", the normal formatting logic is ' - 'bypassed.\n' - '\n' - 'Three conversion flags are currently supported: "\'!s\'" ' - 'which calls\n' - '"str()" on the value, "\'!r\'" which calls "repr()" and ' - '"\'!a\'" which\n' - 'calls "ascii()".\n' - '\n' - 'Some examples:\n' - '\n' - ' "Harold\'s a clever {0!s}" # Calls str() on the ' - 'argument first\n' - ' "Bring out the holy {name!r}" # Calls repr() on the ' - 'argument first\n' - ' "More {!a}" # Calls ascii() on the ' - 'argument first\n' - '\n' - 'The *format_spec* field contains a specification of how the ' - 'value\n' - 'should be presented, including such details as field width, ' - 'alignment,\n' - 'padding, decimal precision and so on. Each value type can ' - 'define its\n' - 'own “formatting mini-language” or interpretation of the ' - '*format_spec*.\n' - '\n' - 'Most built-in types support a common formatting ' - 'mini-language, which\n' - 'is described in the next section.\n' - '\n' - 'A *format_spec* field can also include nested replacement ' - 'fields\n' - 'within it. These nested replacement fields may contain a ' - 'field name,\n' - 'conversion flag and format specification, but deeper ' - 'nesting is not\n' - 'allowed. The replacement fields within the format_spec ' - 'are\n' - 'substituted before the *format_spec* string is interpreted. ' - 'This\n' - 'allows the formatting of a value to be dynamically ' - 'specified.\n' - '\n' - 'See the Format examples section for some examples.\n' - '\n' - '\n' - 'Format Specification Mini-Language\n' - '==================================\n' - '\n' - '“Format specifications” are used within replacement fields ' - 'contained\n' - 'within a format string to define how individual values are ' - 'presented\n' - '(see Format String Syntax and Formatted string literals). ' - 'They can\n' - 'also be passed directly to the built-in "format()" ' - 'function. Each\n' - 'formattable type may define how the format specification is ' - 'to be\n' - 'interpreted.\n' - '\n' - 'Most built-in types implement the following options for ' - 'format\n' - 'specifications, although some of the formatting options are ' - 'only\n' - 'supported by the numeric types.\n' - '\n' - 'A general convention is that an empty format string ("""") ' - 'produces\n' - 'the same result as if you had called "str()" on the value. ' - 'A non-empty\n' - 'format string typically modifies the result.\n' - '\n' - 'The general form of a *standard format specifier* is:\n' - '\n' - ' format_spec ::= ' - '[[fill]align][sign][#][0][width][grouping_option][.precision][type]\n' - ' fill ::= \n' - ' align ::= "<" | ">" | "=" | "^"\n' - ' sign ::= "+" | "-" | " "\n' - ' width ::= digit+\n' - ' grouping_option ::= "_" | ","\n' - ' precision ::= digit+\n' - ' type ::= "b" | "c" | "d" | "e" | "E" | "f" | ' - '"F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n' - '\n' - 'If a valid *align* value is specified, it can be preceded ' - 'by a *fill*\n' - 'character that can be any character and defaults to a space ' - 'if\n' - 'omitted. It is not possible to use a literal curly brace ' - '(“"{"” or\n' - '“"}"”) as the *fill* character in a formatted string ' - 'literal or when\n' - 'using the "str.format()" method. However, it is possible ' - 'to insert a\n' - 'curly brace with a nested replacement field. This ' - 'limitation doesn’t\n' - 'affect the "format()" function.\n' - '\n' - 'The meaning of the various alignment options is as ' - 'follows:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Option | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'<\'" | Forces the field to be left-aligned ' - 'within the available |\n' - ' | | space (this is the default for most ' - 'objects). |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'>\'" | Forces the field to be right-aligned ' - 'within the available |\n' - ' | | space (this is the default for ' - 'numbers). |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'=\'" | Forces the padding to be placed after ' - 'the sign (if any) |\n' - ' | | but before the digits. This is used for ' - 'printing fields |\n' - ' | | in the form ‘+000000120’. This alignment ' - 'option is only |\n' - ' | | valid for numeric types. It becomes the ' - 'default when ‘0’ |\n' - ' | | immediately precedes the field ' - 'width. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'^\'" | Forces the field to be centered within ' - 'the available |\n' - ' | | ' - 'space. ' - '|\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'Note that unless a minimum field width is defined, the ' - 'field width\n' - 'will always be the same size as the data to fill it, so ' - 'that the\n' - 'alignment option has no meaning in this case.\n' - '\n' - 'The *sign* option is only valid for number types, and can ' - 'be one of\n' - 'the following:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Option | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'+\'" | indicates that a sign should be used for ' - 'both positive as |\n' - ' | | well as negative ' - 'numbers. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'-\'" | indicates that a sign should be used ' - 'only for negative |\n' - ' | | numbers (this is the default ' - 'behavior). |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | space | indicates that a leading space should be ' - 'used on positive |\n' - ' | | numbers, and a minus sign on negative ' - 'numbers. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'The "\'#\'" option causes the “alternate form” to be used ' - 'for the\n' - 'conversion. The alternate form is defined differently for ' - 'different\n' - 'types. This option is only valid for integer, float, ' - 'complex and\n' - 'Decimal types. For integers, when binary, octal, or ' - 'hexadecimal output\n' - 'is used, this option adds the prefix respective "\'0b\'", ' - '"\'0o\'", or\n' - '"\'0x\'" to the output value. For floats, complex and ' - 'Decimal the\n' - 'alternate form causes the result of the conversion to ' - 'always contain a\n' - 'decimal-point character, even if no digits follow it. ' - 'Normally, a\n' - 'decimal-point character appears in the result of these ' - 'conversions\n' - 'only if a digit follows it. In addition, for "\'g\'" and ' - '"\'G\'"\n' - 'conversions, trailing zeros are not removed from the ' - 'result.\n' - '\n' - 'The "\',\'" option signals the use of a comma for a ' - 'thousands separator.\n' - 'For a locale aware separator, use the "\'n\'" integer ' - 'presentation type\n' - 'instead.\n' - '\n' - 'Changed in version 3.1: Added the "\',\'" option (see also ' - '**PEP 378**).\n' - '\n' - 'The "\'_\'" option signals the use of an underscore for a ' - 'thousands\n' - 'separator for floating point presentation types and for ' - 'integer\n' - 'presentation type "\'d\'". For integer presentation types ' - '"\'b\'", "\'o\'",\n' - '"\'x\'", and "\'X\'", underscores will be inserted every 4 ' - 'digits. For\n' - 'other presentation types, specifying this option is an ' - 'error.\n' - '\n' - 'Changed in version 3.6: Added the "\'_\'" option (see also ' - '**PEP 515**).\n' - '\n' - '*width* is a decimal integer defining the minimum field ' - 'width. If not\n' - 'specified, then the field width will be determined by the ' - 'content.\n' - '\n' - 'When no explicit alignment is given, preceding the *width* ' - 'field by a\n' - 'zero ("\'0\'") character enables sign-aware zero-padding ' - 'for numeric\n' - 'types. This is equivalent to a *fill* character of "\'0\'" ' - 'with an\n' - '*alignment* type of "\'=\'".\n' - '\n' - 'The *precision* is a decimal number indicating how many ' - 'digits should\n' - 'be displayed after the decimal point for a floating point ' - 'value\n' - 'formatted with "\'f\'" and "\'F\'", or before and after the ' - 'decimal point\n' - 'for a floating point value formatted with "\'g\'" or ' - '"\'G\'". For non-\n' - 'number types the field indicates the maximum field size - ' - 'in other\n' - 'words, how many characters will be used from the field ' - 'content. The\n' - '*precision* is not allowed for integer values.\n' - '\n' - 'Finally, the *type* determines how the data should be ' - 'presented.\n' - '\n' - 'The available string presentation types are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'s\'" | String format. This is the default type ' - 'for strings and |\n' - ' | | may be ' - 'omitted. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | The same as ' - '"\'s\'". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'The available integer presentation types are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'b\'" | Binary format. Outputs the number in ' - 'base 2. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'c\'" | Character. Converts the integer to the ' - 'corresponding |\n' - ' | | unicode character before ' - 'printing. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'d\'" | Decimal Integer. Outputs the number in ' - 'base 10. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'o\'" | Octal format. Outputs the number in base ' - '8. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'x\'" | Hex format. Outputs the number in base ' - '16, using lower- |\n' - ' | | case letters for the digits above ' - '9. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'X\'" | Hex format. Outputs the number in base ' - '16, using upper- |\n' - ' | | case letters for the digits above ' - '9. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'n\'" | Number. This is the same as "\'d\'", ' - 'except that it uses the |\n' - ' | | current locale setting to insert the ' - 'appropriate number |\n' - ' | | separator ' - 'characters. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | The same as ' - '"\'d\'". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - 'In addition to the above presentation types, integers can ' - 'be formatted\n' - 'with the floating point presentation types listed below ' - '(except "\'n\'"\n' - 'and "None"). When doing so, "float()" is used to convert ' - 'the integer\n' - 'to a floating point number before formatting.\n' - '\n' - 'The available presentation types for floating point and ' - 'decimal values\n' - 'are:\n' - '\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | Type | ' - 'Meaning ' - '|\n' - ' ' - '+===========+============================================================+\n' - ' | "\'e\'" | Exponent notation. Prints the number in ' - 'scientific |\n' - ' | | notation using the letter ‘e’ to indicate ' - 'the exponent. |\n' - ' | | The default precision is ' - '"6". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'E\'" | Exponent notation. Same as "\'e\'" ' - 'except it uses an upper |\n' - ' | | case ‘E’ as the separator ' - 'character. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'f\'" | Fixed-point notation. Displays the ' - 'number as a fixed-point |\n' - ' | | number. The default precision is ' - '"6". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'F\'" | Fixed-point notation. Same as "\'f\'", ' - 'but converts "nan" to |\n' - ' | | "NAN" and "inf" to ' - '"INF". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'g\'" | General format. For a given precision ' - '"p >= 1", this |\n' - ' | | rounds the number to "p" significant ' - 'digits and then |\n' - ' | | formats the result in either fixed-point ' - 'format or in |\n' - ' | | scientific notation, depending on its ' - 'magnitude. The |\n' - ' | | precise rules are as follows: suppose that ' - 'the result |\n' - ' | | formatted with presentation type "\'e\'" ' - 'and precision "p-1" |\n' - ' | | would have exponent "exp". Then if "-4 <= ' - 'exp < p", the |\n' - ' | | number is formatted with presentation type ' - '"\'f\'" and |\n' - ' | | precision "p-1-exp". Otherwise, the ' - 'number is formatted |\n' - ' | | with presentation type "\'e\'" and ' - 'precision "p-1". In both |\n' - ' | | cases insignificant trailing zeros are ' - 'removed from the |\n' - ' | | significand, and the decimal point is also ' - 'removed if |\n' - ' | | there are no remaining digits following ' - 'it. Positive and |\n' - ' | | negative infinity, positive and negative ' - 'zero, and nans, |\n' - ' | | are formatted as "inf", "-inf", "0", "-0" ' - 'and "nan" |\n' - ' | | respectively, regardless of the ' - 'precision. A precision of |\n' - ' | | "0" is treated as equivalent to a ' - 'precision of "1". The |\n' - ' | | default precision is ' - '"6". |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'G\'" | General format. Same as "\'g\'" except ' - 'switches to "\'E\'" if |\n' - ' | | the number gets too large. The ' - 'representations of infinity |\n' - ' | | and NaN are uppercased, ' - 'too. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'n\'" | Number. This is the same as "\'g\'", ' - 'except that it uses the |\n' - ' | | current locale setting to insert the ' - 'appropriate number |\n' - ' | | separator ' - 'characters. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | "\'%\'" | Percentage. Multiplies the number by 100 ' - 'and displays in |\n' - ' | | fixed ("\'f\'") format, followed by a ' - 'percent sign. |\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - ' | None | Similar to "\'g\'", except that ' - 'fixed-point notation, when |\n' - ' | | used, has at least one digit past the ' - 'decimal point. The |\n' - ' | | default precision is as high as needed to ' - 'represent the |\n' - ' | | particular value. The overall effect is to ' - 'match the |\n' - ' | | output of "str()" as altered by the other ' - 'format |\n' - ' | | ' - 'modifiers. ' - '|\n' - ' ' - '+-----------+------------------------------------------------------------+\n' - '\n' - '\n' - 'Format examples\n' - '===============\n' - '\n' - 'This section contains examples of the "str.format()" syntax ' - 'and\n' - 'comparison with the old "%"-formatting.\n' - '\n' - 'In most of the cases the syntax is similar to the old ' - '"%"-formatting,\n' - 'with the addition of the "{}" and with ":" used instead of ' - '"%". For\n' - 'example, "\'%03.2f\'" can be translated to "\'{:03.2f}\'".\n' - '\n' - 'The new format syntax also supports new and different ' - 'options, shown\n' - 'in the following examples.\n' - '\n' - 'Accessing arguments by position:\n' - '\n' - " >>> '{0}, {1}, {2}'.format('a', 'b', 'c')\n" - " 'a, b, c'\n" - " >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only\n" - " 'a, b, c'\n" - " >>> '{2}, {1}, {0}'.format('a', 'b', 'c')\n" - " 'c, b, a'\n" - " >>> '{2}, {1}, {0}'.format(*'abc') # unpacking " - 'argument sequence\n' - " 'c, b, a'\n" - " >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' " - 'indices can be repeated\n' - " 'abracadabra'\n" - '\n' - 'Accessing arguments by name:\n' - '\n' - " >>> 'Coordinates: {latitude}, " - "{longitude}'.format(latitude='37.24N', " - "longitude='-115.81W')\n" - " 'Coordinates: 37.24N, -115.81W'\n" - " >>> coord = {'latitude': '37.24N', 'longitude': " - "'-115.81W'}\n" - " >>> 'Coordinates: {latitude}, " - "{longitude}'.format(**coord)\n" - " 'Coordinates: 37.24N, -115.81W'\n" - '\n' - 'Accessing arguments’ attributes:\n' - '\n' - ' >>> c = 3-5j\n' - " >>> ('The complex number {0} is formed from the real " - "part {0.real} '\n" - " ... 'and the imaginary part {0.imag}.').format(c)\n" - " 'The complex number (3-5j) is formed from the real part " - "3.0 and the imaginary part -5.0.'\n" - ' >>> class Point:\n' - ' ... def __init__(self, x, y):\n' - ' ... self.x, self.y = x, y\n' - ' ... def __str__(self):\n' - " ... return 'Point({self.x}, " - "{self.y})'.format(self=self)\n" - ' ...\n' - ' >>> str(Point(4, 2))\n' - " 'Point(4, 2)'\n" - '\n' - 'Accessing arguments’ items:\n' - '\n' - ' >>> coord = (3, 5)\n' - " >>> 'X: {0[0]}; Y: {0[1]}'.format(coord)\n" - " 'X: 3; Y: 5'\n" - '\n' - 'Replacing "%s" and "%r":\n' - '\n' - ' >>> "repr() shows quotes: {!r}; str() doesn\'t: ' - '{!s}".format(\'test1\', \'test2\')\n' - ' "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n' - '\n' - 'Aligning the text and specifying a width:\n' - '\n' - " >>> '{:<30}'.format('left aligned')\n" - " 'left aligned '\n" - " >>> '{:>30}'.format('right aligned')\n" - " ' right aligned'\n" - " >>> '{:^30}'.format('centered')\n" - " ' centered '\n" - " >>> '{:*^30}'.format('centered') # use '*' as a fill " - 'char\n' - " '***********centered***********'\n" - '\n' - 'Replacing "%+f", "%-f", and "% f" and specifying a sign:\n' - '\n' - " >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it " - 'always\n' - " '+3.140000; -3.140000'\n" - " >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space " - 'for positive numbers\n' - " ' 3.140000; -3.140000'\n" - " >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the " - "minus -- same as '{:f}; {:f}'\n" - " '3.140000; -3.140000'\n" - '\n' - 'Replacing "%x" and "%o" and converting the value to ' - 'different bases:\n' - '\n' - ' >>> # format also supports binary numbers\n' - ' >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: ' - '{0:b}".format(42)\n' - " 'int: 42; hex: 2a; oct: 52; bin: 101010'\n" - ' >>> # with 0x, 0o, or 0b as prefix:\n' - ' >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: ' - '{0:#b}".format(42)\n' - " 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010'\n" - '\n' - 'Using the comma as a thousands separator:\n' - '\n' - " >>> '{:,}'.format(1234567890)\n" - " '1,234,567,890'\n" - '\n' - 'Expressing a percentage:\n' - '\n' - ' >>> points = 19\n' - ' >>> total = 22\n' - " >>> 'Correct answers: {:.2%}'.format(points/total)\n" - " 'Correct answers: 86.36%'\n" - '\n' - 'Using type-specific formatting:\n' - '\n' - ' >>> import datetime\n' - ' >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n' - " >>> '{:%Y-%m-%d %H:%M:%S}'.format(d)\n" - " '2010-07-04 12:15:58'\n" - '\n' - 'Nesting arguments and more complex examples:\n' - '\n' - " >>> for align, text in zip('<^>', ['left', 'center', " - "'right']):\n" - " ... '{0:{fill}{align}16}'.format(text, fill=align, " - 'align=align)\n' - ' ...\n' - " 'left<<<<<<<<<<<<'\n" - " '^^^^^center^^^^^'\n" - " '>>>>>>>>>>>right'\n" - ' >>>\n' - ' >>> octets = [192, 168, 0, 1]\n' - " >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets)\n" - " 'C0A80001'\n" - ' >>> int(_, 16)\n' - ' 3232235521\n' - ' >>>\n' - ' >>> width = 5\n' - ' >>> for num in range(5,12): \n' - " ... for base in 'dXob':\n" - " ... print('{0:{width}{base}}'.format(num, " - "base=base, width=width), end=' ')\n" - ' ... print()\n' - ' ...\n' - ' 5 5 5 101\n' - ' 6 6 6 110\n' - ' 7 7 7 111\n' - ' 8 8 10 1000\n' - ' 9 9 11 1001\n' - ' 10 A 12 1010\n' - ' 11 B 13 1011\n', - 'function': 'Function definitions\n' - '********************\n' - '\n' - 'A function definition defines a user-defined function object ' - '(see\n' - 'section The standard type hierarchy):\n' - '\n' - ' funcdef ::= [decorators] "def" funcname "(" ' - '[parameter_list] ")"\n' - ' ["->" expression] ":" suite\n' - ' decorators ::= decorator+\n' - ' decorator ::= "@" dotted_name ["(" ' - '[argument_list [","]] ")"] NEWLINE\n' - ' dotted_name ::= identifier ("." identifier)*\n' - ' parameter_list ::= defparameter ("," defparameter)* ' - '["," [parameter_list_starargs]]\n' - ' | parameter_list_starargs\n' - ' parameter_list_starargs ::= "*" [parameter] ("," ' - 'defparameter)* ["," ["**" parameter [","]]]\n' - ' | "**" parameter [","]\n' - ' parameter ::= identifier [":" expression]\n' - ' defparameter ::= parameter ["=" expression]\n' - ' funcname ::= identifier\n' - '\n' - 'A function definition is an executable statement. Its execution ' - 'binds\n' - 'the function name in the current local namespace to a function ' - 'object\n' - '(a wrapper around the executable code for the function). This\n' - 'function object contains a reference to the current global ' - 'namespace\n' - 'as the global namespace to be used when the function is called.\n' - '\n' - 'The function definition does not execute the function body; this ' - 'gets\n' - 'executed only when the function is called. [2]\n' - '\n' - 'A function definition may be wrapped by one or more *decorator*\n' - 'expressions. Decorator expressions are evaluated when the ' - 'function is\n' - 'defined, in the scope that contains the function definition. ' - 'The\n' - 'result must be a callable, which is invoked with the function ' - 'object\n' - 'as the only argument. The returned value is bound to the ' - 'function name\n' - 'instead of the function object. Multiple decorators are applied ' - 'in\n' - 'nested fashion. For example, the following code\n' - '\n' - ' @f1(arg)\n' - ' @f2\n' - ' def func(): pass\n' - '\n' - 'is roughly equivalent to\n' - '\n' - ' def func(): pass\n' - ' func = f1(arg)(f2(func))\n' - '\n' - 'except that the original function is not temporarily bound to ' - 'the name\n' - '"func".\n' - '\n' - 'When one or more *parameters* have the form *parameter* "="\n' - '*expression*, the function is said to have “default parameter ' - 'values.”\n' - 'For a parameter with a default value, the corresponding ' - '*argument* may\n' - 'be omitted from a call, in which case the parameter’s default ' - 'value is\n' - 'substituted. If a parameter has a default value, all following\n' - 'parameters up until the “"*"” must also have a default value — ' - 'this is\n' - 'a syntactic restriction that is not expressed by the grammar.\n' - '\n' - '**Default parameter values are evaluated from left to right when ' - 'the\n' - 'function definition is executed.** This means that the ' - 'expression is\n' - 'evaluated once, when the function is defined, and that the same ' - '“pre-\n' - 'computed” value is used for each call. This is especially ' - 'important\n' - 'to understand when a default parameter is a mutable object, such ' - 'as a\n' - 'list or a dictionary: if the function modifies the object (e.g. ' - 'by\n' - 'appending an item to a list), the default value is in effect ' - 'modified.\n' - 'This is generally not what was intended. A way around this is ' - 'to use\n' - '"None" as the default, and explicitly test for it in the body of ' - 'the\n' - 'function, e.g.:\n' - '\n' - ' def whats_on_the_telly(penguin=None):\n' - ' if penguin is None:\n' - ' penguin = []\n' - ' penguin.append("property of the zoo")\n' - ' return penguin\n' - '\n' - 'Function call semantics are described in more detail in section ' - 'Calls.\n' - 'A function call always assigns values to all parameters ' - 'mentioned in\n' - 'the parameter list, either from position arguments, from ' - 'keyword\n' - 'arguments, or from default values. If the form “"*identifier"” ' - 'is\n' - 'present, it is initialized to a tuple receiving any excess ' - 'positional\n' - 'parameters, defaulting to the empty tuple. If the form\n' - '“"**identifier"” is present, it is initialized to a new ordered\n' - 'mapping receiving any excess keyword arguments, defaulting to a ' - 'new\n' - 'empty mapping of the same type. Parameters after “"*"” or\n' - '“"*identifier"” are keyword-only parameters and may only be ' - 'passed\n' - 'used keyword arguments.\n' - '\n' - 'Parameters may have annotations of the form “": expression"” ' - 'following\n' - 'the parameter name. Any parameter may have an annotation even ' - 'those\n' - 'of the form "*identifier" or "**identifier". Functions may ' - 'have\n' - '“return” annotation of the form “"-> expression"” after the ' - 'parameter\n' - 'list. These annotations can be any valid Python expression and ' - 'are\n' - 'evaluated when the function definition is executed. Annotations ' - 'may\n' - 'be evaluated in a different order than they appear in the source ' - 'code.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - 'dictionary keyed by the parameters’ names in the ' - '"__annotations__"\n' - 'attribute of the function object.\n' - '\n' - 'It is also possible to create anonymous functions (functions not ' - 'bound\n' - 'to a name), for immediate use in expressions. This uses lambda\n' - 'expressions, described in section Lambdas. Note that the ' - 'lambda\n' - 'expression is merely a shorthand for a simplified function ' - 'definition;\n' - 'a function defined in a “"def"” statement can be passed around ' - 'or\n' - 'assigned to another name just like a function defined by a ' - 'lambda\n' - 'expression. The “"def"” form is actually more powerful since ' - 'it\n' - 'allows the execution of multiple statements and annotations.\n' - '\n' - '**Programmer’s note:** Functions are first-class objects. A ' - '“"def"”\n' - 'statement executed inside a function definition defines a local\n' - 'function that can be returned or passed around. Free variables ' - 'used\n' - 'in the nested function can access the local variables of the ' - 'function\n' - 'containing the def. See section Naming and binding for ' - 'details.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3107** - Function Annotations\n' - ' The original specification for function annotations.\n', - 'global': 'The "global" statement\n' - '**********************\n' - '\n' - ' global_stmt ::= "global" identifier ("," identifier)*\n' - '\n' - 'The "global" statement is a declaration which holds for the ' - 'entire\n' - 'current code block. It means that the listed identifiers are to ' - 'be\n' - 'interpreted as globals. It would be impossible to assign to a ' - 'global\n' - 'variable without "global", although free variables may refer to\n' - 'globals without being declared global.\n' - '\n' - 'Names listed in a "global" statement must not be used in the same ' - 'code\n' - 'block textually preceding that "global" statement.\n' - '\n' - 'Names listed in a "global" statement must not be defined as ' - 'formal\n' - 'parameters or in a "for" loop control target, "class" definition,\n' - 'function definition, "import" statement, or variable annotation.\n' - '\n' - '**CPython implementation detail:** The current implementation does ' - 'not\n' - 'enforce some of these restrictions, but programs should not abuse ' - 'this\n' - 'freedom, as future implementations may enforce them or silently ' - 'change\n' - 'the meaning of the program.\n' - '\n' - '**Programmer’s note:** "global" is a directive to the parser. It\n' - 'applies only to code parsed at the same time as the "global"\n' - 'statement. In particular, a "global" statement contained in a ' - 'string\n' - 'or code object supplied to the built-in "exec()" function does ' - 'not\n' - 'affect the code block *containing* the function call, and code\n' - 'contained in such a string is unaffected by "global" statements in ' - 'the\n' - 'code containing the function call. The same applies to the ' - '"eval()"\n' - 'and "compile()" functions.\n', - 'id-classes': 'Reserved classes of identifiers\n' - '*******************************\n' - '\n' - 'Certain classes of identifiers (besides keywords) have ' - 'special\n' - 'meanings. These classes are identified by the patterns of ' - 'leading and\n' - 'trailing underscore characters:\n' - '\n' - '"_*"\n' - ' Not imported by "from module import *". The special ' - 'identifier "_"\n' - ' is used in the interactive interpreter to store the result ' - 'of the\n' - ' last evaluation; it is stored in the "builtins" module. ' - 'When not\n' - ' in interactive mode, "_" has no special meaning and is not ' - 'defined.\n' - ' See section The import statement.\n' - '\n' - ' Note: The name "_" is often used in conjunction with\n' - ' internationalization; refer to the documentation for the\n' - ' "gettext" module for more information on this ' - 'convention.\n' - '\n' - '"__*__"\n' - ' System-defined names. These names are defined by the ' - 'interpreter\n' - ' and its implementation (including the standard library). ' - 'Current\n' - ' system names are discussed in the Special method names ' - 'section and\n' - ' elsewhere. More will likely be defined in future versions ' - 'of\n' - ' Python. *Any* use of "__*__" names, in any context, that ' - 'does not\n' - ' follow explicitly documented use, is subject to breakage ' - 'without\n' - ' warning.\n' - '\n' - '"__*"\n' - ' Class-private names. Names in this category, when used ' - 'within the\n' - ' context of a class definition, are re-written to use a ' - 'mangled form\n' - ' to help avoid name clashes between “private” attributes of ' - 'base and\n' - ' derived classes. See section Identifiers (Names).\n', - 'identifiers': 'Identifiers and keywords\n' - '************************\n' - '\n' - 'Identifiers (also referred to as *names*) are described by ' - 'the\n' - 'following lexical definitions.\n' - '\n' - 'The syntax of identifiers in Python is based on the Unicode ' - 'standard\n' - 'annex UAX-31, with elaboration and changes as defined below; ' - 'see also\n' - '**PEP 3131** for further details.\n' - '\n' - 'Within the ASCII range (U+0001..U+007F), the valid characters ' - 'for\n' - 'identifiers are the same as in Python 2.x: the uppercase and ' - 'lowercase\n' - 'letters "A" through "Z", the underscore "_" and, except for ' - 'the first\n' - 'character, the digits "0" through "9".\n' - '\n' - 'Python 3.0 introduces additional characters from outside the ' - 'ASCII\n' - 'range (see **PEP 3131**). For these characters, the ' - 'classification\n' - 'uses the version of the Unicode Character Database as ' - 'included in the\n' - '"unicodedata" module.\n' - '\n' - 'Identifiers are unlimited in length. Case is significant.\n' - '\n' - ' identifier ::= xid_start xid_continue*\n' - ' id_start ::= \n' - ' id_continue ::= \n' - ' xid_start ::= \n' - ' xid_continue ::= \n' - '\n' - 'The Unicode category codes mentioned above stand for:\n' - '\n' - '* *Lu* - uppercase letters\n' - '\n' - '* *Ll* - lowercase letters\n' - '\n' - '* *Lt* - titlecase letters\n' - '\n' - '* *Lm* - modifier letters\n' - '\n' - '* *Lo* - other letters\n' - '\n' - '* *Nl* - letter numbers\n' - '\n' - '* *Mn* - nonspacing marks\n' - '\n' - '* *Mc* - spacing combining marks\n' - '\n' - '* *Nd* - decimal numbers\n' - '\n' - '* *Pc* - connector punctuations\n' - '\n' - '* *Other_ID_Start* - explicit list of characters in ' - 'PropList.txt to\n' - ' support backwards compatibility\n' - '\n' - '* *Other_ID_Continue* - likewise\n' - '\n' - 'All identifiers are converted into the normal form NFKC while ' - 'parsing;\n' - 'comparison of identifiers is based on NFKC.\n' - '\n' - 'A non-normative HTML file listing all valid identifier ' - 'characters for\n' - 'Unicode 4.1 can be found at https://www.dcl.hpi.uni-\n' - 'potsdam.de/home/loewis/table-3131.html.\n' - '\n' - '\n' - 'Keywords\n' - '========\n' - '\n' - 'The following identifiers are used as reserved words, or ' - '*keywords* of\n' - 'the language, and cannot be used as ordinary identifiers. ' - 'They must\n' - 'be spelled exactly as written here:\n' - '\n' - ' False class finally is return\n' - ' None continue for lambda try\n' - ' True def from nonlocal while\n' - ' and del global not with\n' - ' as elif if or yield\n' - ' assert else import pass\n' - ' break except in raise\n' - '\n' - '\n' - 'Reserved classes of identifiers\n' - '===============================\n' - '\n' - 'Certain classes of identifiers (besides keywords) have ' - 'special\n' - 'meanings. These classes are identified by the patterns of ' - 'leading and\n' - 'trailing underscore characters:\n' - '\n' - '"_*"\n' - ' Not imported by "from module import *". The special ' - 'identifier "_"\n' - ' is used in the interactive interpreter to store the result ' - 'of the\n' - ' last evaluation; it is stored in the "builtins" module. ' - 'When not\n' - ' in interactive mode, "_" has no special meaning and is not ' - 'defined.\n' - ' See section The import statement.\n' - '\n' - ' Note: The name "_" is often used in conjunction with\n' - ' internationalization; refer to the documentation for ' - 'the\n' - ' "gettext" module for more information on this ' - 'convention.\n' - '\n' - '"__*__"\n' - ' System-defined names. These names are defined by the ' - 'interpreter\n' - ' and its implementation (including the standard library). ' - 'Current\n' - ' system names are discussed in the Special method names ' - 'section and\n' - ' elsewhere. More will likely be defined in future versions ' - 'of\n' - ' Python. *Any* use of "__*__" names, in any context, that ' - 'does not\n' - ' follow explicitly documented use, is subject to breakage ' - 'without\n' - ' warning.\n' - '\n' - '"__*"\n' - ' Class-private names. Names in this category, when used ' - 'within the\n' - ' context of a class definition, are re-written to use a ' - 'mangled form\n' - ' to help avoid name clashes between “private” attributes of ' - 'base and\n' - ' derived classes. See section Identifiers (Names).\n', - 'if': 'The "if" statement\n' - '******************\n' - '\n' - 'The "if" statement is used for conditional execution:\n' - '\n' - ' if_stmt ::= "if" expression ":" suite\n' - ' ("elif" expression ":" suite)*\n' - ' ["else" ":" suite]\n' - '\n' - 'It selects exactly one of the suites by evaluating the expressions ' - 'one\n' - 'by one until one is found to be true (see section Boolean operations\n' - 'for the definition of true and false); then that suite is executed\n' - '(and no other part of the "if" statement is executed or evaluated).\n' - 'If all expressions are false, the suite of the "else" clause, if\n' - 'present, is executed.\n', - 'imaginary': 'Imaginary literals\n' - '******************\n' - '\n' - 'Imaginary literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' imagnumber ::= (floatnumber | digitpart) ("j" | "J")\n' - '\n' - 'An imaginary literal yields a complex number with a real part ' - 'of 0.0.\n' - 'Complex numbers are represented as a pair of floating point ' - 'numbers\n' - 'and have the same restrictions on their range. To create a ' - 'complex\n' - 'number with a nonzero real part, add a floating point number to ' - 'it,\n' - 'e.g., "(3+4j)". Some examples of imaginary literals:\n' - '\n' - ' 3.14j 10.j 10j .001j 1e100j 3.14e-10j ' - '3.14_15_93j\n', - 'import': 'The "import" statement\n' - '**********************\n' - '\n' - ' import_stmt ::= "import" module ["as" identifier] ("," ' - 'module ["as" identifier])*\n' - ' | "from" relative_module "import" identifier ' - '["as" identifier]\n' - ' ("," identifier ["as" identifier])*\n' - ' | "from" relative_module "import" "(" ' - 'identifier ["as" identifier]\n' - ' ("," identifier ["as" identifier])* [","] ")"\n' - ' | "from" module "import" "*"\n' - ' module ::= (identifier ".")* identifier\n' - ' relative_module ::= "."* module | "."+\n' - '\n' - 'The basic import statement (no "from" clause) is executed in two\n' - 'steps:\n' - '\n' - '1. find a module, loading and initializing it if necessary\n' - '\n' - '2. define a name or names in the local namespace for the scope\n' - ' where the "import" statement occurs.\n' - '\n' - 'When the statement contains multiple clauses (separated by commas) ' - 'the\n' - 'two steps are carried out separately for each clause, just as ' - 'though\n' - 'the clauses had been separated out into individual import ' - 'statements.\n' - '\n' - 'The details of the first step, finding and loading modules are\n' - 'described in greater detail in the section on the import system, ' - 'which\n' - 'also describes the various types of packages and modules that can ' - 'be\n' - 'imported, as well as all the hooks that can be used to customize ' - 'the\n' - 'import system. Note that failures in this step may indicate ' - 'either\n' - 'that the module could not be located, *or* that an error occurred\n' - 'while initializing the module, which includes execution of the\n' - 'module’s code.\n' - '\n' - 'If the requested module is retrieved successfully, it will be ' - 'made\n' - 'available in the local namespace in one of three ways:\n' - '\n' - '* If the module name is followed by "as", then the name following\n' - ' "as" is bound directly to the imported module.\n' - '\n' - '* If no other name is specified, and the module being imported is ' - 'a\n' - ' top level module, the module’s name is bound in the local ' - 'namespace\n' - ' as a reference to the imported module\n' - '\n' - '* If the module being imported is *not* a top level module, then ' - 'the\n' - ' name of the top level package that contains the module is bound ' - 'in\n' - ' the local namespace as a reference to the top level package. ' - 'The\n' - ' imported module must be accessed using its full qualified name\n' - ' rather than directly\n' - '\n' - 'The "from" form uses a slightly more complex process:\n' - '\n' - '1. find the module specified in the "from" clause, loading and\n' - ' initializing it if necessary;\n' - '\n' - '2. for each of the identifiers specified in the "import" clauses:\n' - '\n' - ' 1. check if the imported module has an attribute by that name\n' - '\n' - ' 2. if not, attempt to import a submodule with that name and ' - 'then\n' - ' check the imported module again for that attribute\n' - '\n' - ' 3. if the attribute is not found, "ImportError" is raised.\n' - '\n' - ' 4. otherwise, a reference to that value is stored in the local\n' - ' namespace, using the name in the "as" clause if it is ' - 'present,\n' - ' otherwise using the attribute name\n' - '\n' - 'Examples:\n' - '\n' - ' import foo # foo imported and bound locally\n' - ' import foo.bar.baz # foo.bar.baz imported, foo bound ' - 'locally\n' - ' import foo.bar.baz as fbb # foo.bar.baz imported and bound as ' - 'fbb\n' - ' from foo.bar import baz # foo.bar.baz imported and bound as ' - 'baz\n' - ' from foo import attr # foo imported and foo.attr bound as ' - 'attr\n' - '\n' - 'If the list of identifiers is replaced by a star ("\'*\'"), all ' - 'public\n' - 'names defined in the module are bound in the local namespace for ' - 'the\n' - 'scope where the "import" statement occurs.\n' - '\n' - 'The *public names* defined by a module are determined by checking ' - 'the\n' - 'module’s namespace for a variable named "__all__"; if defined, it ' - 'must\n' - 'be a sequence of strings which are names defined or imported by ' - 'that\n' - 'module. The names given in "__all__" are all considered public ' - 'and\n' - 'are required to exist. If "__all__" is not defined, the set of ' - 'public\n' - 'names includes all names found in the module’s namespace which do ' - 'not\n' - 'begin with an underscore character ("\'_\'"). "__all__" should ' - 'contain\n' - 'the entire public API. It is intended to avoid accidentally ' - 'exporting\n' - 'items that are not part of the API (such as library modules which ' - 'were\n' - 'imported and used within the module).\n' - '\n' - 'The wild card form of import — "from module import *" — is only\n' - 'allowed at the module level. Attempting to use it in class or\n' - 'function definitions will raise a "SyntaxError".\n' - '\n' - 'When specifying what module to import you do not have to specify ' - 'the\n' - 'absolute name of the module. When a module or package is ' - 'contained\n' - 'within another package it is possible to make a relative import ' - 'within\n' - 'the same top package without having to mention the package name. ' - 'By\n' - 'using leading dots in the specified module or package after "from" ' - 'you\n' - 'can specify how high to traverse up the current package hierarchy\n' - 'without specifying exact names. One leading dot means the current\n' - 'package where the module making the import exists. Two dots means ' - 'up\n' - 'one package level. Three dots is up two levels, etc. So if you ' - 'execute\n' - '"from . import mod" from a module in the "pkg" package then you ' - 'will\n' - 'end up importing "pkg.mod". If you execute "from ..subpkg2 import ' - 'mod"\n' - 'from within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The\n' - 'specification for relative imports is contained within **PEP ' - '328**.\n' - '\n' - '"importlib.import_module()" is provided to support applications ' - 'that\n' - 'determine dynamically the modules to be loaded.\n' - '\n' - '\n' - 'Future statements\n' - '=================\n' - '\n' - 'A *future statement* is a directive to the compiler that a ' - 'particular\n' - 'module should be compiled using syntax or semantics that will be\n' - 'available in a specified future release of Python where the ' - 'feature\n' - 'becomes standard.\n' - '\n' - 'The future statement is intended to ease migration to future ' - 'versions\n' - 'of Python that introduce incompatible changes to the language. ' - 'It\n' - 'allows use of the new features on a per-module basis before the\n' - 'release in which the feature becomes standard.\n' - '\n' - ' future_stmt ::= "from" "__future__" "import" feature ["as" ' - 'identifier]\n' - ' ("," feature ["as" identifier])*\n' - ' | "from" "__future__" "import" "(" feature ' - '["as" identifier]\n' - ' ("," feature ["as" identifier])* [","] ")"\n' - ' feature ::= identifier\n' - '\n' - 'A future statement must appear near the top of the module. The ' - 'only\n' - 'lines that can appear before a future statement are:\n' - '\n' - '* the module docstring (if any),\n' - '\n' - '* comments,\n' - '\n' - '* blank lines, and\n' - '\n' - '* other future statements.\n' - '\n' - 'The features recognized by Python 3.0 are "absolute_import",\n' - '"division", "generators", "unicode_literals", "print_function",\n' - '"nested_scopes" and "with_statement". They are all redundant ' - 'because\n' - 'they are always enabled, and only kept for backwards ' - 'compatibility.\n' - '\n' - 'A future statement is recognized and treated specially at compile\n' - 'time: Changes to the semantics of core constructs are often\n' - 'implemented by generating different code. It may even be the ' - 'case\n' - 'that a new feature introduces new incompatible syntax (such as a ' - 'new\n' - 'reserved word), in which case the compiler may need to parse the\n' - 'module differently. Such decisions cannot be pushed off until\n' - 'runtime.\n' - '\n' - 'For any given release, the compiler knows which feature names ' - 'have\n' - 'been defined, and raises a compile-time error if a future ' - 'statement\n' - 'contains a feature not known to it.\n' - '\n' - 'The direct runtime semantics are the same as for any import ' - 'statement:\n' - 'there is a standard module "__future__", described later, and it ' - 'will\n' - 'be imported in the usual way at the time the future statement is\n' - 'executed.\n' - '\n' - 'The interesting runtime semantics depend on the specific feature\n' - 'enabled by the future statement.\n' - '\n' - 'Note that there is nothing special about the statement:\n' - '\n' - ' import __future__ [as name]\n' - '\n' - 'That is not a future statement; it’s an ordinary import statement ' - 'with\n' - 'no special semantics or syntax restrictions.\n' - '\n' - 'Code compiled by calls to the built-in functions "exec()" and\n' - '"compile()" that occur in a module "M" containing a future ' - 'statement\n' - 'will, by default, use the new syntax or semantics associated with ' - 'the\n' - 'future statement. This can be controlled by optional arguments ' - 'to\n' - '"compile()" — see the documentation of that function for details.\n' - '\n' - 'A future statement typed at an interactive interpreter prompt ' - 'will\n' - 'take effect for the rest of the interpreter session. If an\n' - 'interpreter is started with the "-i" option, is passed a script ' - 'name\n' - 'to execute, and the script includes a future statement, it will be ' - 'in\n' - 'effect in the interactive session started after the script is\n' - 'executed.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 236** - Back to the __future__\n' - ' The original proposal for the __future__ mechanism.\n', - 'in': 'Membership test operations\n' - '**************************\n' - '\n' - 'The operators "in" and "not in" test for membership. "x in s"\n' - 'evaluates to "True" if *x* is a member of *s*, and "False" otherwise.\n' - '"x not in s" returns the negation of "x in s". All built-in ' - 'sequences\n' - 'and set types support this as well as dictionary, for which "in" ' - 'tests\n' - 'whether the dictionary has a given key. For container types such as\n' - 'list, tuple, set, frozenset, dict, or collections.deque, the\n' - 'expression "x in y" is equivalent to "any(x is e or x == e for e in\n' - 'y)".\n' - '\n' - 'For the string and bytes types, "x in y" is "True" if and only if *x*\n' - 'is a substring of *y*. An equivalent test is "y.find(x) != -1".\n' - 'Empty strings are always considered to be a substring of any other\n' - 'string, so """ in "abc"" will return "True".\n' - '\n' - 'For user-defined classes which define the "__contains__()" method, "x\n' - 'in y" returns "True" if "y.__contains__(x)" returns a true value, and\n' - '"False" otherwise.\n' - '\n' - 'For user-defined classes which do not define "__contains__()" but do\n' - 'define "__iter__()", "x in y" is "True" if some value "z" with "x ==\n' - 'z" is produced while iterating over "y". If an exception is raised\n' - 'during the iteration, it is as if "in" raised that exception.\n' - '\n' - 'Lastly, the old-style iteration protocol is tried: if a class defines\n' - '"__getitem__()", "x in y" is "True" if and only if there is a non-\n' - 'negative integer index *i* such that "x == y[i]", and all lower\n' - 'integer indices do not raise "IndexError" exception. (If any other\n' - 'exception is raised, it is as if "in" raised that exception).\n' - '\n' - 'The operator "not in" is defined to have the inverse true value of\n' - '"in".\n', - 'integers': 'Integer literals\n' - '****************\n' - '\n' - 'Integer literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' integer ::= decinteger | bininteger | octinteger | ' - 'hexinteger\n' - ' decinteger ::= nonzerodigit (["_"] digit)* | "0"+ (["_"] ' - '"0")*\n' - ' bininteger ::= "0" ("b" | "B") (["_"] bindigit)+\n' - ' octinteger ::= "0" ("o" | "O") (["_"] octdigit)+\n' - ' hexinteger ::= "0" ("x" | "X") (["_"] hexdigit)+\n' - ' nonzerodigit ::= "1"..."9"\n' - ' digit ::= "0"..."9"\n' - ' bindigit ::= "0" | "1"\n' - ' octdigit ::= "0"..."7"\n' - ' hexdigit ::= digit | "a"..."f" | "A"..."F"\n' - '\n' - 'There is no limit for the length of integer literals apart from ' - 'what\n' - 'can be stored in available memory.\n' - '\n' - 'Underscores are ignored for determining the numeric value of ' - 'the\n' - 'literal. They can be used to group digits for enhanced ' - 'readability.\n' - 'One underscore can occur between digits, and after base ' - 'specifiers\n' - 'like "0x".\n' - '\n' - 'Note that leading zeros in a non-zero decimal number are not ' - 'allowed.\n' - 'This is for disambiguation with C-style octal literals, which ' - 'Python\n' - 'used before version 3.0.\n' - '\n' - 'Some examples of integer literals:\n' - '\n' - ' 7 2147483647 0o177 0b100110111\n' - ' 3 79228162514264337593543950336 0o377 0xdeadbeef\n' - ' 100_000_000_000 0b_1110_0101\n' - '\n' - 'Changed in version 3.6: Underscores are now allowed for ' - 'grouping\n' - 'purposes in literals.\n', - 'lambda': 'Lambdas\n' - '*******\n' - '\n' - ' lambda_expr ::= "lambda" [parameter_list] ":" ' - 'expression\n' - ' lambda_expr_nocond ::= "lambda" [parameter_list] ":" ' - 'expression_nocond\n' - '\n' - 'Lambda expressions (sometimes called lambda forms) are used to ' - 'create\n' - 'anonymous functions. The expression "lambda parameters: ' - 'expression"\n' - 'yields a function object. The unnamed object behaves like a ' - 'function\n' - 'object defined with:\n' - '\n' - ' def (parameters):\n' - ' return expression\n' - '\n' - 'See section Function definitions for the syntax of parameter ' - 'lists.\n' - 'Note that functions created with lambda expressions cannot ' - 'contain\n' - 'statements or annotations.\n', - 'lists': 'List displays\n' - '*************\n' - '\n' - 'A list display is a possibly empty series of expressions enclosed ' - 'in\n' - 'square brackets:\n' - '\n' - ' list_display ::= "[" [starred_list | comprehension] "]"\n' - '\n' - 'A list display yields a new list object, the contents being ' - 'specified\n' - 'by either a list of expressions or a comprehension. When a comma-\n' - 'separated list of expressions is supplied, its elements are ' - 'evaluated\n' - 'from left to right and placed into the list object in that order.\n' - 'When a comprehension is supplied, the list is constructed from the\n' - 'elements resulting from the comprehension.\n', - 'naming': 'Naming and binding\n' - '******************\n' - '\n' - '\n' - 'Binding of names\n' - '================\n' - '\n' - '*Names* refer to objects. Names are introduced by name binding\n' - 'operations.\n' - '\n' - 'The following constructs bind names: formal parameters to ' - 'functions,\n' - '"import" statements, class and function definitions (these bind ' - 'the\n' - 'class or function name in the defining block), and targets that ' - 'are\n' - 'identifiers if occurring in an assignment, "for" loop header, or ' - 'after\n' - '"as" in a "with" statement or "except" clause. The "import" ' - 'statement\n' - 'of the form "from ... import *" binds all names defined in the\n' - 'imported module, except those beginning with an underscore. This ' - 'form\n' - 'may only be used at the module level.\n' - '\n' - 'A target occurring in a "del" statement is also considered bound ' - 'for\n' - 'this purpose (though the actual semantics are to unbind the ' - 'name).\n' - '\n' - 'Each assignment or import statement occurs within a block defined ' - 'by a\n' - 'class or function definition or at the module level (the ' - 'top-level\n' - 'code block).\n' - '\n' - 'If a name is bound in a block, it is a local variable of that ' - 'block,\n' - 'unless declared as "nonlocal" or "global". If a name is bound at ' - 'the\n' - 'module level, it is a global variable. (The variables of the ' - 'module\n' - 'code block are local and global.) If a variable is used in a ' - 'code\n' - 'block but not defined there, it is a *free variable*.\n' - '\n' - 'Each occurrence of a name in the program text refers to the ' - '*binding*\n' - 'of that name established by the following name resolution rules.\n' - '\n' - '\n' - 'Resolution of names\n' - '===================\n' - '\n' - 'A *scope* defines the visibility of a name within a block. If a ' - 'local\n' - 'variable is defined in a block, its scope includes that block. If ' - 'the\n' - 'definition occurs in a function block, the scope extends to any ' - 'blocks\n' - 'contained within the defining one, unless a contained block ' - 'introduces\n' - 'a different binding for the name.\n' - '\n' - 'When a name is used in a code block, it is resolved using the ' - 'nearest\n' - 'enclosing scope. The set of all such scopes visible to a code ' - 'block\n' - 'is called the block’s *environment*.\n' - '\n' - 'When a name is not found at all, a "NameError" exception is ' - 'raised. If\n' - 'the current scope is a function scope, and the name refers to a ' - 'local\n' - 'variable that has not yet been bound to a value at the point where ' - 'the\n' - 'name is used, an "UnboundLocalError" exception is raised.\n' - '"UnboundLocalError" is a subclass of "NameError".\n' - '\n' - 'If a name binding operation occurs anywhere within a code block, ' - 'all\n' - 'uses of the name within the block are treated as references to ' - 'the\n' - 'current block. This can lead to errors when a name is used within ' - 'a\n' - 'block before it is bound. This rule is subtle. Python lacks\n' - 'declarations and allows name binding operations to occur anywhere\n' - 'within a code block. The local variables of a code block can be\n' - 'determined by scanning the entire text of the block for name ' - 'binding\n' - 'operations.\n' - '\n' - 'If the "global" statement occurs within a block, all uses of the ' - 'name\n' - 'specified in the statement refer to the binding of that name in ' - 'the\n' - 'top-level namespace. Names are resolved in the top-level ' - 'namespace by\n' - 'searching the global namespace, i.e. the namespace of the module\n' - 'containing the code block, and the builtins namespace, the ' - 'namespace\n' - 'of the module "builtins". The global namespace is searched ' - 'first. If\n' - 'the name is not found there, the builtins namespace is searched. ' - 'The\n' - '"global" statement must precede all uses of the name.\n' - '\n' - 'The "global" statement has the same scope as a name binding ' - 'operation\n' - 'in the same block. If the nearest enclosing scope for a free ' - 'variable\n' - 'contains a global statement, the free variable is treated as a ' - 'global.\n' - '\n' - 'The "nonlocal" statement causes corresponding names to refer to\n' - 'previously bound variables in the nearest enclosing function ' - 'scope.\n' - '"SyntaxError" is raised at compile time if the given name does ' - 'not\n' - 'exist in any enclosing function scope.\n' - '\n' - 'The namespace for a module is automatically created the first time ' - 'a\n' - 'module is imported. The main module for a script is always ' - 'called\n' - '"__main__".\n' - '\n' - 'Class definition blocks and arguments to "exec()" and "eval()" ' - 'are\n' - 'special in the context of name resolution. A class definition is ' - 'an\n' - 'executable statement that may use and define names. These ' - 'references\n' - 'follow the normal rules for name resolution with an exception ' - 'that\n' - 'unbound local variables are looked up in the global namespace. ' - 'The\n' - 'namespace of the class definition becomes the attribute dictionary ' - 'of\n' - 'the class. The scope of names defined in a class block is limited ' - 'to\n' - 'the class block; it does not extend to the code blocks of methods ' - '–\n' - 'this includes comprehensions and generator expressions since they ' - 'are\n' - 'implemented using a function scope. This means that the ' - 'following\n' - 'will fail:\n' - '\n' - ' class A:\n' - ' a = 42\n' - ' b = list(a + i for i in range(10))\n' - '\n' - '\n' - 'Builtins and restricted execution\n' - '=================================\n' - '\n' - '**CPython implementation detail:** Users should not touch\n' - '"__builtins__"; it is strictly an implementation detail. Users\n' - 'wanting to override values in the builtins namespace should ' - '"import"\n' - 'the "builtins" module and modify its attributes appropriately.\n' - '\n' - 'The builtins namespace associated with the execution of a code ' - 'block\n' - 'is actually found by looking up the name "__builtins__" in its ' - 'global\n' - 'namespace; this should be a dictionary or a module (in the latter ' - 'case\n' - 'the module’s dictionary is used). By default, when in the ' - '"__main__"\n' - 'module, "__builtins__" is the built-in module "builtins"; when in ' - 'any\n' - 'other module, "__builtins__" is an alias for the dictionary of ' - 'the\n' - '"builtins" module itself.\n' - '\n' - '\n' - 'Interaction with dynamic features\n' - '=================================\n' - '\n' - 'Name resolution of free variables occurs at runtime, not at ' - 'compile\n' - 'time. This means that the following code will print 42:\n' - '\n' - ' i = 10\n' - ' def f():\n' - ' print(i)\n' - ' i = 42\n' - ' f()\n' - '\n' - 'The "eval()" and "exec()" functions do not have access to the ' - 'full\n' - 'environment for resolving names. Names may be resolved in the ' - 'local\n' - 'and global namespaces of the caller. Free variables are not ' - 'resolved\n' - 'in the nearest enclosing namespace, but in the global namespace. ' - '[1]\n' - 'The "exec()" and "eval()" functions have optional arguments to\n' - 'override the global and local namespace. If only one namespace ' - 'is\n' - 'specified, it is used for both.\n', - 'nonlocal': 'The "nonlocal" statement\n' - '************************\n' - '\n' - ' nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n' - '\n' - 'The "nonlocal" statement causes the listed identifiers to refer ' - 'to\n' - 'previously bound variables in the nearest enclosing scope ' - 'excluding\n' - 'globals. This is important because the default behavior for ' - 'binding is\n' - 'to search the local namespace first. The statement allows\n' - 'encapsulated code to rebind variables outside of the local ' - 'scope\n' - 'besides the global (module) scope.\n' - '\n' - 'Names listed in a "nonlocal" statement, unlike those listed in ' - 'a\n' - '"global" statement, must refer to pre-existing bindings in an\n' - 'enclosing scope (the scope in which a new binding should be ' - 'created\n' - 'cannot be determined unambiguously).\n' - '\n' - 'Names listed in a "nonlocal" statement must not collide with ' - 'pre-\n' - 'existing bindings in the local scope.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3104** - Access to Names in Outer Scopes\n' - ' The specification for the "nonlocal" statement.\n', - 'numbers': 'Numeric literals\n' - '****************\n' - '\n' - 'There are three types of numeric literals: integers, floating ' - 'point\n' - 'numbers, and imaginary numbers. There are no complex literals\n' - '(complex numbers can be formed by adding a real number and an\n' - 'imaginary number).\n' - '\n' - 'Note that numeric literals do not include a sign; a phrase like ' - '"-1"\n' - 'is actually an expression composed of the unary operator ‘"-"‘ ' - 'and the\n' - 'literal "1".\n', - 'numeric-types': 'Emulating numeric types\n' - '***********************\n' - '\n' - 'The following methods can be defined to emulate numeric ' - 'objects.\n' - 'Methods corresponding to operations that are not supported ' - 'by the\n' - 'particular kind of number implemented (e.g., bitwise ' - 'operations for\n' - 'non-integral numbers) should be left undefined.\n' - '\n' - 'object.__add__(self, other)\n' - 'object.__sub__(self, other)\n' - 'object.__mul__(self, other)\n' - 'object.__matmul__(self, other)\n' - 'object.__truediv__(self, other)\n' - 'object.__floordiv__(self, other)\n' - 'object.__mod__(self, other)\n' - 'object.__divmod__(self, other)\n' - 'object.__pow__(self, other[, modulo])\n' - 'object.__lshift__(self, other)\n' - 'object.__rshift__(self, other)\n' - 'object.__and__(self, other)\n' - 'object.__xor__(self, other)\n' - 'object.__or__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|"). For ' - 'instance, to\n' - ' evaluate the expression "x + y", where *x* is an ' - 'instance of a\n' - ' class that has an "__add__()" method, "x.__add__(y)" is ' - 'called.\n' - ' The "__divmod__()" method should be the equivalent to ' - 'using\n' - ' "__floordiv__()" and "__mod__()"; it should not be ' - 'related to\n' - ' "__truediv__()". Note that "__pow__()" should be ' - 'defined to accept\n' - ' an optional third argument if the ternary version of the ' - 'built-in\n' - ' "pow()" function is to be supported.\n' - '\n' - ' If one of those methods does not support the operation ' - 'with the\n' - ' supplied arguments, it should return "NotImplemented".\n' - '\n' - 'object.__radd__(self, other)\n' - 'object.__rsub__(self, other)\n' - 'object.__rmul__(self, other)\n' - 'object.__rmatmul__(self, other)\n' - 'object.__rtruediv__(self, other)\n' - 'object.__rfloordiv__(self, other)\n' - 'object.__rmod__(self, other)\n' - 'object.__rdivmod__(self, other)\n' - 'object.__rpow__(self, other)\n' - 'object.__rlshift__(self, other)\n' - 'object.__rrshift__(self, other)\n' - 'object.__rand__(self, other)\n' - 'object.__rxor__(self, other)\n' - 'object.__ror__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|") with reflected ' - '(swapped)\n' - ' operands. These functions are only called if the left ' - 'operand does\n' - ' not support the corresponding operation [3] and the ' - 'operands are of\n' - ' different types. [4] For instance, to evaluate the ' - 'expression "x -\n' - ' y", where *y* is an instance of a class that has an ' - '"__rsub__()"\n' - ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" ' - 'returns\n' - ' *NotImplemented*.\n' - '\n' - ' Note that ternary "pow()" will not try calling ' - '"__rpow__()" (the\n' - ' coercion rules would become too complicated).\n' - '\n' - ' Note: If the right operand’s type is a subclass of the ' - 'left\n' - ' operand’s type and that subclass provides the ' - 'reflected method\n' - ' for the operation, this method will be called before ' - 'the left\n' - ' operand’s non-reflected method. This behavior allows ' - 'subclasses\n' - ' to override their ancestors’ operations.\n' - '\n' - 'object.__iadd__(self, other)\n' - 'object.__isub__(self, other)\n' - 'object.__imul__(self, other)\n' - 'object.__imatmul__(self, other)\n' - 'object.__itruediv__(self, other)\n' - 'object.__ifloordiv__(self, other)\n' - 'object.__imod__(self, other)\n' - 'object.__ipow__(self, other[, modulo])\n' - 'object.__ilshift__(self, other)\n' - 'object.__irshift__(self, other)\n' - 'object.__iand__(self, other)\n' - 'object.__ixor__(self, other)\n' - 'object.__ior__(self, other)\n' - '\n' - ' These methods are called to implement the augmented ' - 'arithmetic\n' - ' assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", ' - '"**=",\n' - ' "<<=", ">>=", "&=", "^=", "|="). These methods should ' - 'attempt to\n' - ' do the operation in-place (modifying *self*) and return ' - 'the result\n' - ' (which could be, but does not have to be, *self*). If a ' - 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of ' - 'a class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to ' - '"x =\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' - ' considered, as with the evaluation of "x + y". In ' - 'certain\n' - ' situations, augmented assignment can result in ' - 'unexpected errors\n' - ' (see Why does a_tuple[i] += [‘item’] raise an exception ' - 'when the\n' - ' addition works?), but this behavior is in fact part of ' - 'the data\n' - ' model.\n' - '\n' - 'object.__neg__(self)\n' - 'object.__pos__(self)\n' - 'object.__abs__(self)\n' - 'object.__invert__(self)\n' - '\n' - ' Called to implement the unary arithmetic operations ' - '("-", "+",\n' - ' "abs()" and "~").\n' - '\n' - 'object.__complex__(self)\n' - 'object.__int__(self)\n' - 'object.__float__(self)\n' - '\n' - ' Called to implement the built-in functions "complex()", ' - '"int()" and\n' - ' "float()". Should return a value of the appropriate ' - 'type.\n' - '\n' - 'object.__index__(self)\n' - '\n' - ' Called to implement "operator.index()", and whenever ' - 'Python needs\n' - ' to losslessly convert the numeric object to an integer ' - 'object (such\n' - ' as in slicing, or in the built-in "bin()", "hex()" and ' - '"oct()"\n' - ' functions). Presence of this method indicates that the ' - 'numeric\n' - ' object is an integer type. Must return an integer.\n' - '\n' - ' Note: In order to have a coherent integer type class, ' - 'when\n' - ' "__index__()" is defined "__int__()" should also be ' - 'defined, and\n' - ' both should return the same value.\n' - '\n' - 'object.__round__(self[, ndigits])\n' - 'object.__trunc__(self)\n' - 'object.__floor__(self)\n' - 'object.__ceil__(self)\n' - '\n' - ' Called to implement the built-in function "round()" and ' - '"math"\n' - ' functions "trunc()", "floor()" and "ceil()". Unless ' - '*ndigits* is\n' - ' passed to "__round__()" all these methods should return ' - 'the value\n' - ' of the object truncated to an "Integral" (typically an ' - '"int").\n' - '\n' - ' If "__int__()" is not defined then the built-in function ' - '"int()"\n' - ' falls back to "__trunc__()".\n', - 'objects': 'Objects, values and types\n' - '*************************\n' - '\n' - '*Objects* are Python’s abstraction for data. All data in a ' - 'Python\n' - 'program is represented by objects or by relations between ' - 'objects. (In\n' - 'a sense, and in conformance to Von Neumann’s model of a “stored\n' - 'program computer,” code is also represented by objects.)\n' - '\n' - 'Every object has an identity, a type and a value. An object’s\n' - '*identity* never changes once it has been created; you may think ' - 'of it\n' - 'as the object’s address in memory. The ‘"is"’ operator compares ' - 'the\n' - 'identity of two objects; the "id()" function returns an integer\n' - 'representing its identity.\n' - '\n' - '**CPython implementation detail:** For CPython, "id(x)" is the ' - 'memory\n' - 'address where "x" is stored.\n' - '\n' - 'An object’s type determines the operations that the object ' - 'supports\n' - '(e.g., “does it have a length?”) and also defines the possible ' - 'values\n' - 'for objects of that type. The "type()" function returns an ' - 'object’s\n' - 'type (which is an object itself). Like its identity, an ' - 'object’s\n' - '*type* is also unchangeable. [1]\n' - '\n' - 'The *value* of some objects can change. Objects whose value can\n' - 'change are said to be *mutable*; objects whose value is ' - 'unchangeable\n' - 'once they are created are called *immutable*. (The value of an\n' - 'immutable container object that contains a reference to a ' - 'mutable\n' - 'object can change when the latter’s value is changed; however ' - 'the\n' - 'container is still considered immutable, because the collection ' - 'of\n' - 'objects it contains cannot be changed. So, immutability is not\n' - 'strictly the same as having an unchangeable value, it is more ' - 'subtle.)\n' - 'An object’s mutability is determined by its type; for instance,\n' - 'numbers, strings and tuples are immutable, while dictionaries ' - 'and\n' - 'lists are mutable.\n' - '\n' - 'Objects are never explicitly destroyed; however, when they ' - 'become\n' - 'unreachable they may be garbage-collected. An implementation is\n' - 'allowed to postpone garbage collection or omit it altogether — it ' - 'is a\n' - 'matter of implementation quality how garbage collection is\n' - 'implemented, as long as no objects are collected that are still\n' - 'reachable.\n' - '\n' - '**CPython implementation detail:** CPython currently uses a ' - 'reference-\n' - 'counting scheme with (optional) delayed detection of cyclically ' - 'linked\n' - 'garbage, which collects most objects as soon as they become\n' - 'unreachable, but is not guaranteed to collect garbage containing\n' - 'circular references. See the documentation of the "gc" module ' - 'for\n' - 'information on controlling the collection of cyclic garbage. ' - 'Other\n' - 'implementations act differently and CPython may change. Do not ' - 'depend\n' - 'on immediate finalization of objects when they become unreachable ' - '(so\n' - 'you should always close files explicitly).\n' - '\n' - 'Note that the use of the implementation’s tracing or debugging\n' - 'facilities may keep objects alive that would normally be ' - 'collectable.\n' - 'Also note that catching an exception with a ‘"try"…"except"’ ' - 'statement\n' - 'may keep objects alive.\n' - '\n' - 'Some objects contain references to “external” resources such as ' - 'open\n' - 'files or windows. It is understood that these resources are ' - 'freed\n' - 'when the object is garbage-collected, but since garbage ' - 'collection is\n' - 'not guaranteed to happen, such objects also provide an explicit ' - 'way to\n' - 'release the external resource, usually a "close()" method. ' - 'Programs\n' - 'are strongly recommended to explicitly close such objects. The\n' - '‘"try"…"finally"’ statement and the ‘"with"’ statement provide\n' - 'convenient ways to do this.\n' - '\n' - 'Some objects contain references to other objects; these are ' - 'called\n' - '*containers*. Examples of containers are tuples, lists and\n' - 'dictionaries. The references are part of a container’s value. ' - 'In\n' - 'most cases, when we talk about the value of a container, we imply ' - 'the\n' - 'values, not the identities of the contained objects; however, ' - 'when we\n' - 'talk about the mutability of a container, only the identities of ' - 'the\n' - 'immediately contained objects are implied. So, if an immutable\n' - 'container (like a tuple) contains a reference to a mutable ' - 'object, its\n' - 'value changes if that mutable object is changed.\n' - '\n' - 'Types affect almost all aspects of object behavior. Even the\n' - 'importance of object identity is affected in some sense: for ' - 'immutable\n' - 'types, operations that compute new values may actually return a\n' - 'reference to any existing object with the same type and value, ' - 'while\n' - 'for mutable objects this is not allowed. E.g., after "a = 1; b = ' - '1",\n' - '"a" and "b" may or may not refer to the same object with the ' - 'value\n' - 'one, depending on the implementation, but after "c = []; d = []", ' - '"c"\n' - 'and "d" are guaranteed to refer to two different, unique, newly\n' - 'created empty lists. (Note that "c = d = []" assigns the same ' - 'object\n' - 'to both "c" and "d".)\n', - 'operator-summary': 'Operator precedence\n' - '*******************\n' - '\n' - 'The following table summarizes the operator precedence ' - 'in Python, from\n' - 'lowest precedence (least binding) to highest precedence ' - '(most\n' - 'binding). Operators in the same box have the same ' - 'precedence. Unless\n' - 'the syntax is explicitly given, operators are binary. ' - 'Operators in\n' - 'the same box group left to right (except for ' - 'exponentiation, which\n' - 'groups from right to left).\n' - '\n' - 'Note that comparisons, membership tests, and identity ' - 'tests, all have\n' - 'the same precedence and have a left-to-right chaining ' - 'feature as\n' - 'described in the Comparisons section.\n' - '\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| Operator | ' - 'Description |\n' - '+=================================================+=======================================+\n' - '| "lambda" | ' - 'Lambda expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "if" – "else" | ' - 'Conditional expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "or" | ' - 'Boolean OR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "and" | ' - 'Boolean AND |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "not" "x" | ' - 'Boolean NOT |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "in", "not in", "is", "is not", "<", "<=", ">", | ' - 'Comparisons, including membership |\n' - '| ">=", "!=", "==" | ' - 'tests and identity tests |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "|" | ' - 'Bitwise OR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "^" | ' - 'Bitwise XOR |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "&" | ' - 'Bitwise AND |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "<<", ">>" | ' - 'Shifts |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "+", "-" | ' - 'Addition and subtraction |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "*", "@", "/", "//", "%" | ' - 'Multiplication, matrix |\n' - '| | ' - 'multiplication, division, floor |\n' - '| | ' - 'division, remainder [5] |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "+x", "-x", "~x" | ' - 'Positive, negative, bitwise NOT |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "**" | ' - 'Exponentiation [6] |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "await" "x" | ' - 'Await expression |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "x[index]", "x[index:index]", | ' - 'Subscription, slicing, call, |\n' - '| "x(arguments...)", "x.attribute" | ' - 'attribute reference |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '| "(expressions...)", "[expressions...]", "{key: | ' - 'Binding or tuple display, list |\n' - '| value...}", "{expressions...}" | ' - 'display, dictionary display, set |\n' - '| | ' - 'display |\n' - '+-------------------------------------------------+---------------------------------------+\n' - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] While "abs(x%y) < abs(y)" is true mathematically, ' - 'for floats\n' - ' it may not be true numerically due to roundoff. For ' - 'example, and\n' - ' assuming a platform on which a Python float is an ' - 'IEEE 754 double-\n' - ' precision number, in order that "-1e-100 % 1e100" ' - 'have the same\n' - ' sign as "1e100", the computed result is "-1e-100 + ' - '1e100", which\n' - ' is numerically exactly equal to "1e100". The ' - 'function\n' - ' "math.fmod()" returns a result whose sign matches ' - 'the sign of the\n' - ' first argument instead, and so returns "-1e-100" in ' - 'this case.\n' - ' Which approach is more appropriate depends on the ' - 'application.\n' - '\n' - '[2] If x is very close to an exact integer multiple of ' - 'y, it’s\n' - ' possible for "x//y" to be one larger than ' - '"(x-x%y)//y" due to\n' - ' rounding. In such cases, Python returns the latter ' - 'result, in\n' - ' order to preserve that "divmod(x,y)[0] * y + x % y" ' - 'be very close\n' - ' to "x".\n' - '\n' - '[3] The Unicode standard distinguishes between *code ' - 'points* (e.g.\n' - ' U+0041) and *abstract characters* (e.g. “LATIN ' - 'CAPITAL LETTER A”).\n' - ' While most abstract characters in Unicode are only ' - 'represented\n' - ' using one code point, there is a number of abstract ' - 'characters\n' - ' that can in addition be represented using a sequence ' - 'of more than\n' - ' one code point. For example, the abstract character ' - '“LATIN\n' - ' CAPITAL LETTER C WITH CEDILLA” can be represented as ' - 'a single\n' - ' *precomposed character* at code position U+00C7, or ' - 'as a sequence\n' - ' of a *base character* at code position U+0043 (LATIN ' - 'CAPITAL\n' - ' LETTER C), followed by a *combining character* at ' - 'code position\n' - ' U+0327 (COMBINING CEDILLA).\n' - '\n' - ' The comparison operators on strings compare at the ' - 'level of\n' - ' Unicode code points. This may be counter-intuitive ' - 'to humans. For\n' - ' example, ""\\u00C7" == "\\u0043\\u0327"" is "False", ' - 'even though both\n' - ' strings represent the same abstract character “LATIN ' - 'CAPITAL\n' - ' LETTER C WITH CEDILLA”.\n' - '\n' - ' To compare strings at the level of abstract ' - 'characters (that is,\n' - ' in a way intuitive to humans), use ' - '"unicodedata.normalize()".\n' - '\n' - '[4] Due to automatic garbage-collection, free lists, and ' - 'the\n' - ' dynamic nature of descriptors, you may notice ' - 'seemingly unusual\n' - ' behaviour in certain uses of the "is" operator, like ' - 'those\n' - ' involving comparisons between instance methods, or ' - 'constants.\n' - ' Check their documentation for more info.\n' - '\n' - '[5] The "%" operator is also used for string formatting; ' - 'the same\n' - ' precedence applies.\n' - '\n' - '[6] The power operator "**" binds less tightly than an ' - 'arithmetic\n' - ' or bitwise unary operator on its right, that is, ' - '"2**-1" is "0.5".\n', - 'pass': 'The "pass" statement\n' - '********************\n' - '\n' - ' pass_stmt ::= "pass"\n' - '\n' - '"pass" is a null operation — when it is executed, nothing happens. ' - 'It\n' - 'is useful as a placeholder when a statement is required ' - 'syntactically,\n' - 'but no code needs to be executed, for example:\n' - '\n' - ' def f(arg): pass # a function that does nothing (yet)\n' - '\n' - ' class C: pass # a class with no methods (yet)\n', - 'power': 'The power operator\n' - '******************\n' - '\n' - 'The power operator binds more tightly than unary operators on its\n' - 'left; it binds less tightly than unary operators on its right. ' - 'The\n' - 'syntax is:\n' - '\n' - ' power ::= (await_expr | primary) ["**" u_expr]\n' - '\n' - 'Thus, in an unparenthesized sequence of power and unary operators, ' - 'the\n' - 'operators are evaluated from right to left (this does not ' - 'constrain\n' - 'the evaluation order for the operands): "-1**2" results in "-1".\n' - '\n' - 'The power operator has the same semantics as the built-in "pow()"\n' - 'function, when called with two arguments: it yields its left ' - 'argument\n' - 'raised to the power of its right argument. The numeric arguments ' - 'are\n' - 'first converted to a common type, and the result is of that type.\n' - '\n' - 'For int operands, the result has the same type as the operands ' - 'unless\n' - 'the second argument is negative; in that case, all arguments are\n' - 'converted to float and a float result is delivered. For example,\n' - '"10**2" returns "100", but "10**-2" returns "0.01".\n' - '\n' - 'Raising "0.0" to a negative power results in a ' - '"ZeroDivisionError".\n' - 'Raising a negative number to a fractional power results in a ' - '"complex"\n' - 'number. (In earlier versions it raised a "ValueError".)\n', - 'raise': 'The "raise" statement\n' - '*********************\n' - '\n' - ' raise_stmt ::= "raise" [expression ["from" expression]]\n' - '\n' - 'If no expressions are present, "raise" re-raises the last ' - 'exception\n' - 'that was active in the current scope. If no exception is active ' - 'in\n' - 'the current scope, a "RuntimeError" exception is raised indicating\n' - 'that this is an error.\n' - '\n' - 'Otherwise, "raise" evaluates the first expression as the exception\n' - 'object. It must be either a subclass or an instance of\n' - '"BaseException". If it is a class, the exception instance will be\n' - 'obtained when needed by instantiating the class with no arguments.\n' - '\n' - 'The *type* of the exception is the exception instance’s class, the\n' - '*value* is the instance itself.\n' - '\n' - 'A traceback object is normally created automatically when an ' - 'exception\n' - 'is raised and attached to it as the "__traceback__" attribute, ' - 'which\n' - 'is writable. You can create an exception and set your own traceback ' - 'in\n' - 'one step using the "with_traceback()" exception method (which ' - 'returns\n' - 'the same exception instance, with its traceback set to its ' - 'argument),\n' - 'like so:\n' - '\n' - ' raise Exception("foo occurred").with_traceback(tracebackobj)\n' - '\n' - 'The "from" clause is used for exception chaining: if given, the ' - 'second\n' - '*expression* must be another exception class or instance, which ' - 'will\n' - 'then be attached to the raised exception as the "__cause__" ' - 'attribute\n' - '(which is writable). If the raised exception is not handled, both\n' - 'exceptions will be printed:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except Exception as exc:\n' - ' ... raise RuntimeError("Something bad happened") from exc\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 2, in \n' - ' ZeroDivisionError: division by zero\n' - '\n' - ' The above exception was the direct cause of the following ' - 'exception:\n' - '\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' RuntimeError: Something bad happened\n' - '\n' - 'A similar mechanism works implicitly if an exception is raised ' - 'inside\n' - 'an exception handler or a "finally" clause: the previous exception ' - 'is\n' - 'then attached as the new exception’s "__context__" attribute:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except:\n' - ' ... raise RuntimeError("Something bad happened")\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 2, in \n' - ' ZeroDivisionError: division by zero\n' - '\n' - ' During handling of the above exception, another exception ' - 'occurred:\n' - '\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' RuntimeError: Something bad happened\n' - '\n' - 'Exception chaining can be explicitly suppressed by specifying ' - '"None"\n' - 'in the "from" clause:\n' - '\n' - ' >>> try:\n' - ' ... print(1 / 0)\n' - ' ... except:\n' - ' ... raise RuntimeError("Something bad happened") from None\n' - ' ...\n' - ' Traceback (most recent call last):\n' - ' File "", line 4, in \n' - ' RuntimeError: Something bad happened\n' - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information about handling exceptions is in ' - 'section\n' - 'The try statement.\n' - '\n' - 'Changed in version 3.3: "None" is now permitted as "Y" in "raise X\n' - 'from Y".\n' - '\n' - 'New in version 3.3: The "__suppress_context__" attribute to ' - 'suppress\n' - 'automatic display of the exception context.\n', - 'return': 'The "return" statement\n' - '**********************\n' - '\n' - ' return_stmt ::= "return" [expression_list]\n' - '\n' - '"return" may only occur syntactically nested in a function ' - 'definition,\n' - 'not within a nested class definition.\n' - '\n' - 'If an expression list is present, it is evaluated, else "None" is\n' - 'substituted.\n' - '\n' - '"return" leaves the current function call with the expression list ' - '(or\n' - '"None") as return value.\n' - '\n' - 'When "return" passes control out of a "try" statement with a ' - '"finally"\n' - 'clause, that "finally" clause is executed before really leaving ' - 'the\n' - 'function.\n' - '\n' - 'In a generator function, the "return" statement indicates that ' - 'the\n' - 'generator is done and will cause "StopIteration" to be raised. ' - 'The\n' - 'returned value (if any) is used as an argument to construct\n' - '"StopIteration" and becomes the "StopIteration.value" attribute.\n' - '\n' - 'In an asynchronous generator function, an empty "return" ' - 'statement\n' - 'indicates that the asynchronous generator is done and will cause\n' - '"StopAsyncIteration" to be raised. A non-empty "return" statement ' - 'is\n' - 'a syntax error in an asynchronous generator function.\n', - 'sequence-types': 'Emulating container types\n' - '*************************\n' - '\n' - 'The following methods can be defined to implement ' - 'container objects.\n' - 'Containers usually are sequences (such as lists or tuples) ' - 'or mappings\n' - '(like dictionaries), but can represent other containers as ' - 'well. The\n' - 'first set of methods is used either to emulate a sequence ' - 'or to\n' - 'emulate a mapping; the difference is that for a sequence, ' - 'the\n' - 'allowable keys should be the integers *k* for which "0 <= ' - 'k < N" where\n' - '*N* is the length of the sequence, or slice objects, which ' - 'define a\n' - 'range of items. It is also recommended that mappings ' - 'provide the\n' - 'methods "keys()", "values()", "items()", "get()", ' - '"clear()",\n' - '"setdefault()", "pop()", "popitem()", "copy()", and ' - '"update()"\n' - 'behaving similar to those for Python’s standard dictionary ' - 'objects.\n' - 'The "collections" module provides a "MutableMapping" ' - 'abstract base\n' - 'class to help create those methods from a base set of ' - '"__getitem__()",\n' - '"__setitem__()", "__delitem__()", and "keys()". Mutable ' - 'sequences\n' - 'should provide methods "append()", "count()", "index()", ' - '"extend()",\n' - '"insert()", "pop()", "remove()", "reverse()" and "sort()", ' - 'like Python\n' - 'standard list objects. Finally, sequence types should ' - 'implement\n' - 'addition (meaning concatenation) and multiplication ' - '(meaning\n' - 'repetition) by defining the methods "__add__()", ' - '"__radd__()",\n' - '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" ' - 'described\n' - 'below; they should not define other numerical operators. ' - 'It is\n' - 'recommended that both mappings and sequences implement ' - 'the\n' - '"__contains__()" method to allow efficient use of the "in" ' - 'operator;\n' - 'for mappings, "in" should search the mapping’s keys; for ' - 'sequences, it\n' - 'should search through the values. It is further ' - 'recommended that both\n' - 'mappings and sequences implement the "__iter__()" method ' - 'to allow\n' - 'efficient iteration through the container; for mappings, ' - '"__iter__()"\n' - 'should be the same as "keys()"; for sequences, it should ' - 'iterate\n' - 'through the values.\n' - '\n' - 'object.__len__(self)\n' - '\n' - ' Called to implement the built-in function "len()". ' - 'Should return\n' - ' the length of the object, an integer ">=" 0. Also, an ' - 'object that\n' - ' doesn’t define a "__bool__()" method and whose ' - '"__len__()" method\n' - ' returns zero is considered to be false in a Boolean ' - 'context.\n' - '\n' - ' **CPython implementation detail:** In CPython, the ' - 'length is\n' - ' required to be at most "sys.maxsize". If the length is ' - 'larger than\n' - ' "sys.maxsize" some features (such as "len()") may ' - 'raise\n' - ' "OverflowError". To prevent raising "OverflowError" by ' - 'truth value\n' - ' testing, an object must define a "__bool__()" method.\n' - '\n' - 'object.__length_hint__(self)\n' - '\n' - ' Called to implement "operator.length_hint()". Should ' - 'return an\n' - ' estimated length for the object (which may be greater ' - 'or less than\n' - ' the actual length). The length must be an integer ">=" ' - '0. This\n' - ' method is purely an optimization and is never required ' - 'for\n' - ' correctness.\n' - '\n' - ' New in version 3.4.\n' - '\n' - 'Note: Slicing is done exclusively with the following three ' - 'methods.\n' - ' A call like\n' - '\n' - ' a[1:2] = b\n' - '\n' - ' is translated to\n' - '\n' - ' a[slice(1, 2, None)] = b\n' - '\n' - ' and so forth. Missing slice items are always filled in ' - 'with "None".\n' - '\n' - 'object.__getitem__(self, key)\n' - '\n' - ' Called to implement evaluation of "self[key]". For ' - 'sequence types,\n' - ' the accepted keys should be integers and slice ' - 'objects. Note that\n' - ' the special interpretation of negative indexes (if the ' - 'class wishes\n' - ' to emulate a sequence type) is up to the ' - '"__getitem__()" method. If\n' - ' *key* is of an inappropriate type, "TypeError" may be ' - 'raised; if of\n' - ' a value outside the set of indexes for the sequence ' - '(after any\n' - ' special interpretation of negative values), ' - '"IndexError" should be\n' - ' raised. For mapping types, if *key* is missing (not in ' - 'the\n' - ' container), "KeyError" should be raised.\n' - '\n' - ' Note: "for" loops expect that an "IndexError" will be ' - 'raised for\n' - ' illegal indexes to allow proper detection of the end ' - 'of the\n' - ' sequence.\n' - '\n' - 'object.__setitem__(self, key, value)\n' - '\n' - ' Called to implement assignment to "self[key]". Same ' - 'note as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support changes to the values for keys, or ' - 'if new keys\n' - ' can be added, or for sequences if elements can be ' - 'replaced. The\n' - ' same exceptions should be raised for improper *key* ' - 'values as for\n' - ' the "__getitem__()" method.\n' - '\n' - 'object.__delitem__(self, key)\n' - '\n' - ' Called to implement deletion of "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support removal of keys, or for sequences ' - 'if elements\n' - ' can be removed from the sequence. The same exceptions ' - 'should be\n' - ' raised for improper *key* values as for the ' - '"__getitem__()" method.\n' - '\n' - 'object.__missing__(self, key)\n' - '\n' - ' Called by "dict"."__getitem__()" to implement ' - '"self[key]" for dict\n' - ' subclasses when key is not in the dictionary.\n' - '\n' - 'object.__iter__(self)\n' - '\n' - ' This method is called when an iterator is required for ' - 'a container.\n' - ' This method should return a new iterator object that ' - 'can iterate\n' - ' over all the objects in the container. For mappings, ' - 'it should\n' - ' iterate over the keys of the container.\n' - '\n' - ' Iterator objects also need to implement this method; ' - 'they are\n' - ' required to return themselves. For more information on ' - 'iterator\n' - ' objects, see Iterator Types.\n' - '\n' - 'object.__reversed__(self)\n' - '\n' - ' Called (if present) by the "reversed()" built-in to ' - 'implement\n' - ' reverse iteration. It should return a new iterator ' - 'object that\n' - ' iterates over all the objects in the container in ' - 'reverse order.\n' - '\n' - ' If the "__reversed__()" method is not provided, the ' - '"reversed()"\n' - ' built-in will fall back to using the sequence protocol ' - '("__len__()"\n' - ' and "__getitem__()"). Objects that support the ' - 'sequence protocol\n' - ' should only provide "__reversed__()" if they can ' - 'provide an\n' - ' implementation that is more efficient than the one ' - 'provided by\n' - ' "reversed()".\n' - '\n' - 'The membership test operators ("in" and "not in") are ' - 'normally\n' - 'implemented as an iteration through a sequence. However, ' - 'container\n' - 'objects can supply the following special method with a ' - 'more efficient\n' - 'implementation, which also does not require the object be ' - 'a sequence.\n' - '\n' - 'object.__contains__(self, item)\n' - '\n' - ' Called to implement membership test operators. Should ' - 'return true\n' - ' if *item* is in *self*, false otherwise. For mapping ' - 'objects, this\n' - ' should consider the keys of the mapping rather than the ' - 'values or\n' - ' the key-item pairs.\n' - '\n' - ' For objects that don’t define "__contains__()", the ' - 'membership test\n' - ' first tries iteration via "__iter__()", then the old ' - 'sequence\n' - ' iteration protocol via "__getitem__()", see this ' - 'section in the\n' - ' language reference.\n', - 'shifting': 'Shifting operations\n' - '*******************\n' - '\n' - 'The shifting operations have lower priority than the arithmetic\n' - 'operations:\n' - '\n' - ' shift_expr ::= a_expr | shift_expr ("<<" | ">>") a_expr\n' - '\n' - 'These operators accept integers as arguments. They shift the ' - 'first\n' - 'argument to the left or right by the number of bits given by ' - 'the\n' - 'second argument.\n' - '\n' - 'A right shift by *n* bits is defined as floor division by ' - '"pow(2,n)".\n' - 'A left shift by *n* bits is defined as multiplication with ' - '"pow(2,n)".\n' - '\n' - 'Note: In the current implementation, the right-hand operand is\n' - ' required to be at most "sys.maxsize". If the right-hand ' - 'operand is\n' - ' larger than "sys.maxsize" an "OverflowError" exception is ' - 'raised.\n', - 'slicings': 'Slicings\n' - '********\n' - '\n' - 'A slicing selects a range of items in a sequence object (e.g., ' - 'a\n' - 'string, tuple or list). Slicings may be used as expressions or ' - 'as\n' - 'targets in assignment or "del" statements. The syntax for a ' - 'slicing:\n' - '\n' - ' slicing ::= primary "[" slice_list "]"\n' - ' slice_list ::= slice_item ("," slice_item)* [","]\n' - ' slice_item ::= expression | proper_slice\n' - ' proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" ' - '[stride] ]\n' - ' lower_bound ::= expression\n' - ' upper_bound ::= expression\n' - ' stride ::= expression\n' - '\n' - 'There is ambiguity in the formal syntax here: anything that ' - 'looks like\n' - 'an expression list also looks like a slice list, so any ' - 'subscription\n' - 'can be interpreted as a slicing. Rather than further ' - 'complicating the\n' - 'syntax, this is disambiguated by defining that in this case the\n' - 'interpretation as a subscription takes priority over the\n' - 'interpretation as a slicing (this is the case if the slice list\n' - 'contains no proper slice).\n' - '\n' - 'The semantics for a slicing are as follows. The primary is ' - 'indexed\n' - '(using the same "__getitem__()" method as normal subscription) ' - 'with a\n' - 'key that is constructed from the slice list, as follows. If the ' - 'slice\n' - 'list contains at least one comma, the key is a tuple containing ' - 'the\n' - 'conversion of the slice items; otherwise, the conversion of the ' - 'lone\n' - 'slice item is the key. The conversion of a slice item that is ' - 'an\n' - 'expression is that expression. The conversion of a proper slice ' - 'is a\n' - 'slice object (see section The standard type hierarchy) whose ' - '"start",\n' - '"stop" and "step" attributes are the values of the expressions ' - 'given\n' - 'as lower bound, upper bound and stride, respectively, ' - 'substituting\n' - '"None" for missing expressions.\n', - 'specialattrs': 'Special Attributes\n' - '******************\n' - '\n' - 'The implementation adds a few special read-only attributes ' - 'to several\n' - 'object types, where they are relevant. Some of these are ' - 'not reported\n' - 'by the "dir()" built-in function.\n' - '\n' - 'object.__dict__\n' - '\n' - ' A dictionary or other mapping object used to store an ' - 'object’s\n' - ' (writable) attributes.\n' - '\n' - 'instance.__class__\n' - '\n' - ' The class to which a class instance belongs.\n' - '\n' - 'class.__bases__\n' - '\n' - ' The tuple of base classes of a class object.\n' - '\n' - 'definition.__name__\n' - '\n' - ' The name of the class, function, method, descriptor, or ' - 'generator\n' - ' instance.\n' - '\n' - 'definition.__qualname__\n' - '\n' - ' The *qualified name* of the class, function, method, ' - 'descriptor, or\n' - ' generator instance.\n' - '\n' - ' New in version 3.3.\n' - '\n' - 'class.__mro__\n' - '\n' - ' This attribute is a tuple of classes that are considered ' - 'when\n' - ' looking for base classes during method resolution.\n' - '\n' - 'class.mro()\n' - '\n' - ' This method can be overridden by a metaclass to customize ' - 'the\n' - ' method resolution order for its instances. It is called ' - 'at class\n' - ' instantiation, and its result is stored in "__mro__".\n' - '\n' - 'class.__subclasses__()\n' - '\n' - ' Each class keeps a list of weak references to its ' - 'immediate\n' - ' subclasses. This method returns a list of all those ' - 'references\n' - ' still alive. Example:\n' - '\n' - ' >>> int.__subclasses__()\n' - " []\n" - '\n' - '-[ Footnotes ]-\n' - '\n' - '[1] Additional information on these special methods may be ' - 'found\n' - ' in the Python Reference Manual (Basic customization).\n' - '\n' - '[2] As a consequence, the list "[1, 2]" is considered equal ' - 'to\n' - ' "[1.0, 2.0]", and similarly for tuples.\n' - '\n' - '[3] They must have since the parser can’t tell the type of ' - 'the\n' - ' operands.\n' - '\n' - '[4] Cased characters are those with general category ' - 'property\n' - ' being one of “Lu” (Letter, uppercase), “Ll” (Letter, ' - 'lowercase),\n' - ' or “Lt” (Letter, titlecase).\n' - '\n' - '[5] To format only a tuple you should therefore provide a\n' - ' singleton tuple whose only element is the tuple to be ' - 'formatted.\n', - 'specialnames': 'Special method names\n' - '********************\n' - '\n' - 'A class can implement certain operations that are invoked by ' - 'special\n' - 'syntax (such as arithmetic operations or subscripting and ' - 'slicing) by\n' - 'defining methods with special names. This is Python’s ' - 'approach to\n' - '*operator overloading*, allowing classes to define their own ' - 'behavior\n' - 'with respect to language operators. For instance, if a ' - 'class defines\n' - 'a method named "__getitem__()", and "x" is an instance of ' - 'this class,\n' - 'then "x[i]" is roughly equivalent to "type(x).__getitem__(x, ' - 'i)".\n' - 'Except where mentioned, attempts to execute an operation ' - 'raise an\n' - 'exception when no appropriate method is defined (typically\n' - '"AttributeError" or "TypeError").\n' - '\n' - 'Setting a special method to "None" indicates that the ' - 'corresponding\n' - 'operation is not available. For example, if a class sets ' - '"__iter__()"\n' - 'to "None", the class is not iterable, so calling "iter()" on ' - 'its\n' - 'instances will raise a "TypeError" (without falling back to\n' - '"__getitem__()"). [2]\n' - '\n' - 'When implementing a class that emulates any built-in type, ' - 'it is\n' - 'important that the emulation only be implemented to the ' - 'degree that it\n' - 'makes sense for the object being modelled. For example, ' - 'some\n' - 'sequences may work well with retrieval of individual ' - 'elements, but\n' - 'extracting a slice may not make sense. (One example of this ' - 'is the\n' - '"NodeList" interface in the W3C’s Document Object Model.)\n' - '\n' - '\n' - 'Basic customization\n' - '===================\n' - '\n' - 'object.__new__(cls[, ...])\n' - '\n' - ' Called to create a new instance of class *cls*. ' - '"__new__()" is a\n' - ' static method (special-cased so you need not declare it ' - 'as such)\n' - ' that takes the class of which an instance was requested ' - 'as its\n' - ' first argument. The remaining arguments are those passed ' - 'to the\n' - ' object constructor expression (the call to the class). ' - 'The return\n' - ' value of "__new__()" should be the new object instance ' - '(usually an\n' - ' instance of *cls*).\n' - '\n' - ' Typical implementations create a new instance of the ' - 'class by\n' - ' invoking the superclass’s "__new__()" method using\n' - ' "super().__new__(cls[, ...])" with appropriate arguments ' - 'and then\n' - ' modifying the newly-created instance as necessary before ' - 'returning\n' - ' it.\n' - '\n' - ' If "__new__()" returns an instance of *cls*, then the ' - 'new\n' - ' instance’s "__init__()" method will be invoked like\n' - ' "__init__(self[, ...])", where *self* is the new instance ' - 'and the\n' - ' remaining arguments are the same as were passed to ' - '"__new__()".\n' - '\n' - ' If "__new__()" does not return an instance of *cls*, then ' - 'the new\n' - ' instance’s "__init__()" method will not be invoked.\n' - '\n' - ' "__new__()" is intended mainly to allow subclasses of ' - 'immutable\n' - ' types (like int, str, or tuple) to customize instance ' - 'creation. It\n' - ' is also commonly overridden in custom metaclasses in ' - 'order to\n' - ' customize class creation.\n' - '\n' - 'object.__init__(self[, ...])\n' - '\n' - ' Called after the instance has been created (by ' - '"__new__()"), but\n' - ' before it is returned to the caller. The arguments are ' - 'those\n' - ' passed to the class constructor expression. If a base ' - 'class has an\n' - ' "__init__()" method, the derived class’s "__init__()" ' - 'method, if\n' - ' any, must explicitly call it to ensure proper ' - 'initialization of the\n' - ' base class part of the instance; for example:\n' - ' "super().__init__([args...])".\n' - '\n' - ' Because "__new__()" and "__init__()" work together in ' - 'constructing\n' - ' objects ("__new__()" to create it, and "__init__()" to ' - 'customize\n' - ' it), no non-"None" value may be returned by "__init__()"; ' - 'doing so\n' - ' will cause a "TypeError" to be raised at runtime.\n' - '\n' - 'object.__del__(self)\n' - '\n' - ' Called when the instance is about to be destroyed. This ' - 'is also\n' - ' called a finalizer or (improperly) a destructor. If a ' - 'base class\n' - ' has a "__del__()" method, the derived class’s "__del__()" ' - 'method,\n' - ' if any, must explicitly call it to ensure proper deletion ' - 'of the\n' - ' base class part of the instance.\n' - '\n' - ' It is possible (though not recommended!) for the ' - '"__del__()" method\n' - ' to postpone destruction of the instance by creating a new ' - 'reference\n' - ' to it. This is called object *resurrection*. It is\n' - ' implementation-dependent whether "__del__()" is called a ' - 'second\n' - ' time when a resurrected object is about to be destroyed; ' - 'the\n' - ' current *CPython* implementation only calls it once.\n' - '\n' - ' It is not guaranteed that "__del__()" methods are called ' - 'for\n' - ' objects that still exist when the interpreter exits.\n' - '\n' - ' Note: "del x" doesn’t directly call "x.__del__()" — the ' - 'former\n' - ' decrements the reference count for "x" by one, and the ' - 'latter is\n' - ' only called when "x"’s reference count reaches zero.\n' - '\n' - ' **CPython implementation detail:** It is possible for a ' - 'reference\n' - ' cycle to prevent the reference count of an object from ' - 'going to\n' - ' zero. In this case, the cycle will be later detected and ' - 'deleted\n' - ' by the *cyclic garbage collector*. A common cause of ' - 'reference\n' - ' cycles is when an exception has been caught in a local ' - 'variable.\n' - ' The frame’s locals then reference the exception, which ' - 'references\n' - ' its own traceback, which references the locals of all ' - 'frames caught\n' - ' in the traceback.\n' - '\n' - ' See also: Documentation for the "gc" module.\n' - '\n' - ' Warning: Due to the precarious circumstances under which\n' - ' "__del__()" methods are invoked, exceptions that occur ' - 'during\n' - ' their execution are ignored, and a warning is printed ' - 'to\n' - ' "sys.stderr" instead. In particular:\n' - '\n' - ' * "__del__()" can be invoked when arbitrary code is ' - 'being\n' - ' executed, including from any arbitrary thread. If ' - '"__del__()"\n' - ' needs to take a lock or invoke any other blocking ' - 'resource, it\n' - ' may deadlock as the resource may already be taken by ' - 'the code\n' - ' that gets interrupted to execute "__del__()".\n' - '\n' - ' * "__del__()" can be executed during interpreter ' - 'shutdown. As\n' - ' a consequence, the global variables it needs to ' - 'access\n' - ' (including other modules) may already have been ' - 'deleted or set\n' - ' to "None". Python guarantees that globals whose name ' - 'begins\n' - ' with a single underscore are deleted from their ' - 'module before\n' - ' other globals are deleted; if no other references to ' - 'such\n' - ' globals exist, this may help in assuring that ' - 'imported modules\n' - ' are still available at the time when the "__del__()" ' - 'method is\n' - ' called.\n' - '\n' - 'object.__repr__(self)\n' - '\n' - ' Called by the "repr()" built-in function to compute the ' - '“official”\n' - ' string representation of an object. If at all possible, ' - 'this\n' - ' should look like a valid Python expression that could be ' - 'used to\n' - ' recreate an object with the same value (given an ' - 'appropriate\n' - ' environment). If this is not possible, a string of the ' - 'form\n' - ' "<...some useful description...>" should be returned. The ' - 'return\n' - ' value must be a string object. If a class defines ' - '"__repr__()" but\n' - ' not "__str__()", then "__repr__()" is also used when an ' - '“informal”\n' - ' string representation of instances of that class is ' - 'required.\n' - '\n' - ' This is typically used for debugging, so it is important ' - 'that the\n' - ' representation is information-rich and unambiguous.\n' - '\n' - 'object.__str__(self)\n' - '\n' - ' Called by "str(object)" and the built-in functions ' - '"format()" and\n' - ' "print()" to compute the “informal” or nicely printable ' - 'string\n' - ' representation of an object. The return value must be a ' - 'string\n' - ' object.\n' - '\n' - ' This method differs from "object.__repr__()" in that ' - 'there is no\n' - ' expectation that "__str__()" return a valid Python ' - 'expression: a\n' - ' more convenient or concise representation can be used.\n' - '\n' - ' The default implementation defined by the built-in type ' - '"object"\n' - ' calls "object.__repr__()".\n' - '\n' - 'object.__bytes__(self)\n' - '\n' - ' Called by bytes to compute a byte-string representation ' - 'of an\n' - ' object. This should return a "bytes" object.\n' - '\n' - 'object.__format__(self, format_spec)\n' - '\n' - ' Called by the "format()" built-in function, and by ' - 'extension,\n' - ' evaluation of formatted string literals and the ' - '"str.format()"\n' - ' method, to produce a “formatted” string representation of ' - 'an\n' - ' object. The "format_spec" argument is a string that ' - 'contains a\n' - ' description of the formatting options desired. The ' - 'interpretation\n' - ' of the "format_spec" argument is up to the type ' - 'implementing\n' - ' "__format__()", however most classes will either ' - 'delegate\n' - ' formatting to one of the built-in types, or use a ' - 'similar\n' - ' formatting option syntax.\n' - '\n' - ' See Format Specification Mini-Language for a description ' - 'of the\n' - ' standard formatting syntax.\n' - '\n' - ' The return value must be a string object.\n' - '\n' - ' Changed in version 3.4: The __format__ method of "object" ' - 'itself\n' - ' raises a "TypeError" if passed any non-empty string.\n' - '\n' - 'object.__lt__(self, other)\n' - 'object.__le__(self, other)\n' - 'object.__eq__(self, other)\n' - 'object.__ne__(self, other)\n' - 'object.__gt__(self, other)\n' - 'object.__ge__(self, other)\n' - '\n' - ' These are the so-called “rich comparison” methods. The\n' - ' correspondence between operator symbols and method names ' - 'is as\n' - ' follows: "xy" calls\n' - ' "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n' - '\n' - ' A rich comparison method may return the singleton ' - '"NotImplemented"\n' - ' if it does not implement the operation for a given pair ' - 'of\n' - ' arguments. By convention, "False" and "True" are returned ' - 'for a\n' - ' successful comparison. However, these methods can return ' - 'any value,\n' - ' so if the comparison operator is used in a Boolean ' - 'context (e.g.,\n' - ' in the condition of an "if" statement), Python will call ' - '"bool()"\n' - ' on the value to determine if the result is true or ' - 'false.\n' - '\n' - ' By default, "__ne__()" delegates to "__eq__()" and ' - 'inverts the\n' - ' result unless it is "NotImplemented". There are no other ' - 'implied\n' - ' relationships among the comparison operators, for ' - 'example, the\n' - ' truth of "(x.__hash__".\n' - '\n' - ' If a class that does not override "__eq__()" wishes to ' - 'suppress\n' - ' hash support, it should include "__hash__ = None" in the ' - 'class\n' - ' definition. A class which defines its own "__hash__()" ' - 'that\n' - ' explicitly raises a "TypeError" would be incorrectly ' - 'identified as\n' - ' hashable by an "isinstance(obj, collections.Hashable)" ' - 'call.\n' - '\n' - ' Note: By default, the "__hash__()" values of str, bytes ' - 'and\n' - ' datetime objects are “salted” with an unpredictable ' - 'random value.\n' - ' Although they remain constant within an individual ' - 'Python\n' - ' process, they are not predictable between repeated ' - 'invocations of\n' - ' Python.This is intended to provide protection against a ' - 'denial-\n' - ' of-service caused by carefully-chosen inputs that ' - 'exploit the\n' - ' worst case performance of a dict insertion, O(n^2) ' - 'complexity.\n' - ' See http://www.ocert.org/advisories/ocert-2011-003.html ' - 'for\n' - ' details.Changing hash values affects the iteration ' - 'order of\n' - ' dicts, sets and other mappings. Python has never made ' - 'guarantees\n' - ' about this ordering (and it typically varies between ' - '32-bit and\n' - ' 64-bit builds).See also "PYTHONHASHSEED".\n' - '\n' - ' Changed in version 3.3: Hash randomization is enabled by ' - 'default.\n' - '\n' - 'object.__bool__(self)\n' - '\n' - ' Called to implement truth value testing and the built-in ' - 'operation\n' - ' "bool()"; should return "False" or "True". When this ' - 'method is not\n' - ' defined, "__len__()" is called, if it is defined, and the ' - 'object is\n' - ' considered true if its result is nonzero. If a class ' - 'defines\n' - ' neither "__len__()" nor "__bool__()", all its instances ' - 'are\n' - ' considered true.\n' - '\n' - '\n' - 'Customizing attribute access\n' - '============================\n' - '\n' - 'The following methods can be defined to customize the ' - 'meaning of\n' - 'attribute access (use of, assignment to, or deletion of ' - '"x.name") for\n' - 'class instances.\n' - '\n' - 'object.__getattr__(self, name)\n' - '\n' - ' Called when the default attribute access fails with an\n' - ' "AttributeError" (either "__getattribute__()" raises an\n' - ' "AttributeError" because *name* is not an instance ' - 'attribute or an\n' - ' attribute in the class tree for "self"; or "__get__()" of ' - 'a *name*\n' - ' property raises "AttributeError"). This method should ' - 'either\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - ' Note that if the attribute is found through the normal ' - 'mechanism,\n' - ' "__getattr__()" is not called. (This is an intentional ' - 'asymmetry\n' - ' between "__getattr__()" and "__setattr__()".) This is ' - 'done both for\n' - ' efficiency reasons and because otherwise "__getattr__()" ' - 'would have\n' - ' no way to access other attributes of the instance. Note ' - 'that at\n' - ' least for instance variables, you can fake total control ' - 'by not\n' - ' inserting any values in the instance attribute dictionary ' - '(but\n' - ' instead inserting them in another object). See the\n' - ' "__getattribute__()" method below for a way to actually ' - 'get total\n' - ' control over attribute access.\n' - '\n' - 'object.__getattribute__(self, name)\n' - '\n' - ' Called unconditionally to implement attribute accesses ' - 'for\n' - ' instances of the class. If the class also defines ' - '"__getattr__()",\n' - ' the latter will not be called unless "__getattribute__()" ' - 'either\n' - ' calls it explicitly or raises an "AttributeError". This ' - 'method\n' - ' should return the (computed) attribute value or raise an\n' - ' "AttributeError" exception. In order to avoid infinite ' - 'recursion in\n' - ' this method, its implementation should always call the ' - 'base class\n' - ' method with the same name to access any attributes it ' - 'needs, for\n' - ' example, "object.__getattribute__(self, name)".\n' - '\n' - ' Note: This method may still be bypassed when looking up ' - 'special\n' - ' methods as the result of implicit invocation via ' - 'language syntax\n' - ' or built-in functions. See Special method lookup.\n' - '\n' - 'object.__setattr__(self, name, value)\n' - '\n' - ' Called when an attribute assignment is attempted. This ' - 'is called\n' - ' instead of the normal mechanism (i.e. store the value in ' - 'the\n' - ' instance dictionary). *name* is the attribute name, ' - '*value* is the\n' - ' value to be assigned to it.\n' - '\n' - ' If "__setattr__()" wants to assign to an instance ' - 'attribute, it\n' - ' should call the base class method with the same name, for ' - 'example,\n' - ' "object.__setattr__(self, name, value)".\n' - '\n' - 'object.__delattr__(self, name)\n' - '\n' - ' Like "__setattr__()" but for attribute deletion instead ' - 'of\n' - ' assignment. This should only be implemented if "del ' - 'obj.name" is\n' - ' meaningful for the object.\n' - '\n' - 'object.__dir__(self)\n' - '\n' - ' Called when "dir()" is called on the object. A sequence ' - 'must be\n' - ' returned. "dir()" converts the returned sequence to a ' - 'list and\n' - ' sorts it.\n' - '\n' - '\n' - 'Customizing module attribute access\n' - '-----------------------------------\n' - '\n' - 'For a more fine grained customization of the module behavior ' - '(setting\n' - 'attributes, properties, etc.), one can set the "__class__" ' - 'attribute\n' - 'of a module object to a subclass of "types.ModuleType". For ' - 'example:\n' - '\n' - ' import sys\n' - ' from types import ModuleType\n' - '\n' - ' class VerboseModule(ModuleType):\n' - ' def __repr__(self):\n' - " return f'Verbose {self.__name__}'\n" - '\n' - ' def __setattr__(self, attr, value):\n' - " print(f'Setting {attr}...')\n" - ' setattr(self, attr, value)\n' - '\n' - ' sys.modules[__name__].__class__ = VerboseModule\n' - '\n' - 'Note: Setting module "__class__" only affects lookups made ' - 'using the\n' - ' attribute access syntax – directly accessing the module ' - 'globals\n' - ' (whether by code within the module, or via a reference to ' - 'the\n' - ' module’s globals dictionary) is unaffected.\n' - '\n' - 'Changed in version 3.5: "__class__" module attribute is now ' - 'writable.\n' - '\n' - '\n' - 'Implementing Descriptors\n' - '------------------------\n' - '\n' - 'The following methods only apply when an instance of the ' - 'class\n' - 'containing the method (a so-called *descriptor* class) ' - 'appears in an\n' - '*owner* class (the descriptor must be in either the owner’s ' - 'class\n' - 'dictionary or in the class dictionary for one of its ' - 'parents). In the\n' - 'examples below, “the attribute” refers to the attribute ' - 'whose name is\n' - 'the key of the property in the owner class’ "__dict__".\n' - '\n' - 'object.__get__(self, instance, owner)\n' - '\n' - ' Called to get the attribute of the owner class (class ' - 'attribute\n' - ' access) or of an instance of that class (instance ' - 'attribute\n' - ' access). *owner* is always the owner class, while ' - '*instance* is the\n' - ' instance that the attribute was accessed through, or ' - '"None" when\n' - ' the attribute is accessed through the *owner*. This ' - 'method should\n' - ' return the (computed) attribute value or raise an ' - '"AttributeError"\n' - ' exception.\n' - '\n' - 'object.__set__(self, instance, value)\n' - '\n' - ' Called to set the attribute on an instance *instance* of ' - 'the owner\n' - ' class to a new value, *value*.\n' - '\n' - 'object.__delete__(self, instance)\n' - '\n' - ' Called to delete the attribute on an instance *instance* ' - 'of the\n' - ' owner class.\n' - '\n' - 'object.__set_name__(self, owner, name)\n' - '\n' - ' Called at the time the owning class *owner* is created. ' - 'The\n' - ' descriptor has been assigned to *name*.\n' - '\n' - ' New in version 3.6.\n' - '\n' - 'The attribute "__objclass__" is interpreted by the "inspect" ' - 'module as\n' - 'specifying the class where this object was defined (setting ' - 'this\n' - 'appropriately can assist in runtime introspection of dynamic ' - 'class\n' - 'attributes). For callables, it may indicate that an instance ' - 'of the\n' - 'given type (or a subclass) is expected or required as the ' - 'first\n' - 'positional argument (for example, CPython sets this ' - 'attribute for\n' - 'unbound methods that are implemented in C).\n' - '\n' - '\n' - 'Invoking Descriptors\n' - '--------------------\n' - '\n' - 'In general, a descriptor is an object attribute with ' - '“binding\n' - 'behavior”, one whose attribute access has been overridden by ' - 'methods\n' - 'in the descriptor protocol: "__get__()", "__set__()", and\n' - '"__delete__()". If any of those methods are defined for an ' - 'object, it\n' - 'is said to be a descriptor.\n' - '\n' - 'The default behavior for attribute access is to get, set, or ' - 'delete\n' - 'the attribute from an object’s dictionary. For instance, ' - '"a.x" has a\n' - 'lookup chain starting with "a.__dict__[\'x\']", then\n' - '"type(a).__dict__[\'x\']", and continuing through the base ' - 'classes of\n' - '"type(a)" excluding metaclasses.\n' - '\n' - 'However, if the looked-up value is an object defining one of ' - 'the\n' - 'descriptor methods, then Python may override the default ' - 'behavior and\n' - 'invoke the descriptor method instead. Where this occurs in ' - 'the\n' - 'precedence chain depends on which descriptor methods were ' - 'defined and\n' - 'how they were called.\n' - '\n' - 'The starting point for descriptor invocation is a binding, ' - '"a.x". How\n' - 'the arguments are assembled depends on "a":\n' - '\n' - 'Direct Call\n' - ' The simplest and least common call is when user code ' - 'directly\n' - ' invokes a descriptor method: "x.__get__(a)".\n' - '\n' - 'Instance Binding\n' - ' If binding to an object instance, "a.x" is transformed ' - 'into the\n' - ' call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n' - '\n' - 'Class Binding\n' - ' If binding to a class, "A.x" is transformed into the ' - 'call:\n' - ' "A.__dict__[\'x\'].__get__(None, A)".\n' - '\n' - 'Super Binding\n' - ' If "a" is an instance of "super", then the binding ' - '"super(B,\n' - ' obj).m()" searches "obj.__class__.__mro__" for the base ' - 'class "A"\n' - ' immediately preceding "B" and then invokes the descriptor ' - 'with the\n' - ' call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n' - '\n' - 'For instance bindings, the precedence of descriptor ' - 'invocation depends\n' - 'on the which descriptor methods are defined. A descriptor ' - 'can define\n' - 'any combination of "__get__()", "__set__()" and ' - '"__delete__()". If it\n' - 'does not define "__get__()", then accessing the attribute ' - 'will return\n' - 'the descriptor object itself unless there is a value in the ' - 'object’s\n' - 'instance dictionary. If the descriptor defines "__set__()" ' - 'and/or\n' - '"__delete__()", it is a data descriptor; if it defines ' - 'neither, it is\n' - 'a non-data descriptor. Normally, data descriptors define ' - 'both\n' - '"__get__()" and "__set__()", while non-data descriptors have ' - 'just the\n' - '"__get__()" method. Data descriptors with "__set__()" and ' - '"__get__()"\n' - 'defined always override a redefinition in an instance ' - 'dictionary. In\n' - 'contrast, non-data descriptors can be overridden by ' - 'instances.\n' - '\n' - 'Python methods (including "staticmethod()" and ' - '"classmethod()") are\n' - 'implemented as non-data descriptors. Accordingly, instances ' - 'can\n' - 'redefine and override methods. This allows individual ' - 'instances to\n' - 'acquire behaviors that differ from other instances of the ' - 'same class.\n' - '\n' - 'The "property()" function is implemented as a data ' - 'descriptor.\n' - 'Accordingly, instances cannot override the behavior of a ' - 'property.\n' - '\n' - '\n' - '__slots__\n' - '---------\n' - '\n' - '*__slots__* allow us to explicitly declare data members ' - '(like\n' - 'properties) and deny the creation of *__dict__* and ' - '*__weakref__*\n' - '(unless explicitly declared in *__slots__* or available in a ' - 'parent.)\n' - '\n' - 'The space saved over using *__dict__* can be significant.\n' - '\n' - 'object.__slots__\n' - '\n' - ' This class variable can be assigned a string, iterable, ' - 'or sequence\n' - ' of strings with variable names used by instances. ' - '*__slots__*\n' - ' reserves space for the declared variables and prevents ' - 'the\n' - ' automatic creation of *__dict__* and *__weakref__* for ' - 'each\n' - ' instance.\n' - '\n' - '\n' - 'Notes on using *__slots__*\n' - '~~~~~~~~~~~~~~~~~~~~~~~~~~\n' - '\n' - '* When inheriting from a class without *__slots__*, the ' - '*__dict__*\n' - ' and *__weakref__* attribute of the instances will always ' - 'be\n' - ' accessible.\n' - '\n' - '* Without a *__dict__* variable, instances cannot be ' - 'assigned new\n' - ' variables not listed in the *__slots__* definition. ' - 'Attempts to\n' - ' assign to an unlisted variable name raises ' - '"AttributeError". If\n' - ' dynamic assignment of new variables is desired, then add\n' - ' "\'__dict__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* Without a *__weakref__* variable for each instance, ' - 'classes\n' - ' defining *__slots__* do not support weak references to ' - 'its\n' - ' instances. If weak reference support is needed, then add\n' - ' "\'__weakref__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' - '\n' - '* *__slots__* are implemented at the class level by ' - 'creating\n' - ' descriptors (Implementing Descriptors) for each variable ' - 'name. As a\n' - ' result, class attributes cannot be used to set default ' - 'values for\n' - ' instance variables defined by *__slots__*; otherwise, the ' - 'class\n' - ' attribute would overwrite the descriptor assignment.\n' - '\n' - '* The action of a *__slots__* declaration is not limited to ' - 'the\n' - ' class where it is defined. *__slots__* declared in ' - 'parents are\n' - ' available in child classes. However, child subclasses will ' - 'get a\n' - ' *__dict__* and *__weakref__* unless they also define ' - '*__slots__*\n' - ' (which should only contain names of any *additional* ' - 'slots).\n' - '\n' - '* If a class defines a slot also defined in a base class, ' - 'the\n' - ' instance variable defined by the base class slot is ' - 'inaccessible\n' - ' (except by retrieving its descriptor directly from the ' - 'base class).\n' - ' This renders the meaning of the program undefined. In the ' - 'future, a\n' - ' check may be added to prevent this.\n' - '\n' - '* Nonempty *__slots__* does not work for classes derived ' - 'from\n' - ' “variable-length” built-in types such as "int", "bytes" ' - 'and "tuple".\n' - '\n' - '* Any non-string iterable may be assigned to *__slots__*. ' - 'Mappings\n' - ' may also be used; however, in the future, special meaning ' - 'may be\n' - ' assigned to the values corresponding to each key.\n' - '\n' - '* *__class__* assignment works only if both classes have the ' - 'same\n' - ' *__slots__*.\n' - '\n' - '* Multiple inheritance with multiple slotted parent classes ' - 'can be\n' - ' used, but only one parent is allowed to have attributes ' - 'created by\n' - ' slots (the other bases must have empty slot layouts) - ' - 'violations\n' - ' raise "TypeError".\n' - '\n' - '\n' - 'Customizing class creation\n' - '==========================\n' - '\n' - 'Whenever a class inherits from another class, ' - '*__init_subclass__* is\n' - 'called on that class. This way, it is possible to write ' - 'classes which\n' - 'change the behavior of subclasses. This is closely related ' - 'to class\n' - 'decorators, but where class decorators only affect the ' - 'specific class\n' - 'they’re applied to, "__init_subclass__" solely applies to ' - 'future\n' - 'subclasses of the class defining the method.\n' - '\n' - 'classmethod object.__init_subclass__(cls)\n' - '\n' - ' This method is called whenever the containing class is ' - 'subclassed.\n' - ' *cls* is then the new subclass. If defined as a normal ' - 'instance\n' - ' method, this method is implicitly converted to a class ' - 'method.\n' - '\n' - ' Keyword arguments which are given to a new class are ' - 'passed to the\n' - ' parent’s class "__init_subclass__". For compatibility ' - 'with other\n' - ' classes using "__init_subclass__", one should take out ' - 'the needed\n' - ' keyword arguments and pass the others over to the base ' - 'class, as\n' - ' in:\n' - '\n' - ' class Philosopher:\n' - ' def __init_subclass__(cls, default_name, ' - '**kwargs):\n' - ' super().__init_subclass__(**kwargs)\n' - ' cls.default_name = default_name\n' - '\n' - ' class AustralianPhilosopher(Philosopher, ' - 'default_name="Bruce"):\n' - ' pass\n' - '\n' - ' The default implementation "object.__init_subclass__" ' - 'does nothing,\n' - ' but raises an error if it is called with any arguments.\n' - '\n' - ' Note: The metaclass hint "metaclass" is consumed by the ' - 'rest of\n' - ' the type machinery, and is never passed to ' - '"__init_subclass__"\n' - ' implementations. The actual metaclass (rather than the ' - 'explicit\n' - ' hint) can be accessed as "type(cls)".\n' - '\n' - ' New in version 3.6.\n' - '\n' - '\n' - 'Metaclasses\n' - '-----------\n' - '\n' - 'By default, classes are constructed using "type()". The ' - 'class body is\n' - 'executed in a new namespace and the class name is bound ' - 'locally to the\n' - 'result of "type(name, bases, namespace)".\n' - '\n' - 'The class creation process can be customized by passing the\n' - '"metaclass" keyword argument in the class definition line, ' - 'or by\n' - 'inheriting from an existing class that included such an ' - 'argument. In\n' - 'the following example, both "MyClass" and "MySubclass" are ' - 'instances\n' - 'of "Meta":\n' - '\n' - ' class Meta(type):\n' - ' pass\n' - '\n' - ' class MyClass(metaclass=Meta):\n' - ' pass\n' - '\n' - ' class MySubclass(MyClass):\n' - ' pass\n' - '\n' - 'Any other keyword arguments that are specified in the class ' - 'definition\n' - 'are passed through to all metaclass operations described ' - 'below.\n' - '\n' - 'When a class definition is executed, the following steps ' - 'occur:\n' - '\n' - '* the appropriate metaclass is determined\n' - '\n' - '* the class namespace is prepared\n' - '\n' - '* the class body is executed\n' - '\n' - '* the class object is created\n' - '\n' - '\n' - 'Determining the appropriate metaclass\n' - '-------------------------------------\n' - '\n' - 'The appropriate metaclass for a class definition is ' - 'determined as\n' - 'follows:\n' - '\n' - '* if no bases and no explicit metaclass are given, then ' - '"type()" is\n' - ' used\n' - '\n' - '* if an explicit metaclass is given and it is *not* an ' - 'instance of\n' - ' "type()", then it is used directly as the metaclass\n' - '\n' - '* if an instance of "type()" is given as the explicit ' - 'metaclass, or\n' - ' bases are defined, then the most derived metaclass is ' - 'used\n' - '\n' - 'The most derived metaclass is selected from the explicitly ' - 'specified\n' - 'metaclass (if any) and the metaclasses (i.e. "type(cls)") of ' - 'all\n' - 'specified base classes. The most derived metaclass is one ' - 'which is a\n' - 'subtype of *all* of these candidate metaclasses. If none of ' - 'the\n' - 'candidate metaclasses meets that criterion, then the class ' - 'definition\n' - 'will fail with "TypeError".\n' - '\n' - '\n' - 'Preparing the class namespace\n' - '-----------------------------\n' - '\n' - 'Once the appropriate metaclass has been identified, then the ' - 'class\n' - 'namespace is prepared. If the metaclass has a "__prepare__" ' - 'attribute,\n' - 'it is called as "namespace = metaclass.__prepare__(name, ' - 'bases,\n' - '**kwds)" (where the additional keyword arguments, if any, ' - 'come from\n' - 'the class definition).\n' - '\n' - 'If the metaclass has no "__prepare__" attribute, then the ' - 'class\n' - 'namespace is initialised as an empty ordered mapping.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3115** - Metaclasses in Python 3000\n' - ' Introduced the "__prepare__" namespace hook\n' - '\n' - '\n' - 'Executing the class body\n' - '------------------------\n' - '\n' - 'The class body is executed (approximately) as "exec(body, ' - 'globals(),\n' - 'namespace)". The key difference from a normal call to ' - '"exec()" is that\n' - 'lexical scoping allows the class body (including any ' - 'methods) to\n' - 'reference names from the current and outer scopes when the ' - 'class\n' - 'definition occurs inside a function.\n' - '\n' - 'However, even when the class definition occurs inside the ' - 'function,\n' - 'methods defined inside the class still cannot see names ' - 'defined at the\n' - 'class scope. Class variables must be accessed through the ' - 'first\n' - 'parameter of instance or class methods, or through the ' - 'implicit\n' - 'lexically scoped "__class__" reference described in the next ' - 'section.\n' - '\n' - '\n' - 'Creating the class object\n' - '-------------------------\n' - '\n' - 'Once the class namespace has been populated by executing the ' - 'class\n' - 'body, the class object is created by calling ' - '"metaclass(name, bases,\n' - 'namespace, **kwds)" (the additional keywords passed here are ' - 'the same\n' - 'as those passed to "__prepare__").\n' - '\n' - 'This class object is the one that will be referenced by the ' - 'zero-\n' - 'argument form of "super()". "__class__" is an implicit ' - 'closure\n' - 'reference created by the compiler if any methods in a class ' - 'body refer\n' - 'to either "__class__" or "super". This allows the zero ' - 'argument form\n' - 'of "super()" to correctly identify the class being defined ' - 'based on\n' - 'lexical scoping, while the class or instance that was used ' - 'to make the\n' - 'current call is identified based on the first argument ' - 'passed to the\n' - 'method.\n' - '\n' - '**CPython implementation detail:** In CPython 3.6 and later, ' - 'the\n' - '"__class__" cell is passed to the metaclass as a ' - '"__classcell__" entry\n' - 'in the class namespace. If present, this must be propagated ' - 'up to the\n' - '"type.__new__" call in order for the class to be ' - 'initialised\n' - 'correctly. Failing to do so will result in a ' - '"DeprecationWarning" in\n' - 'Python 3.6, and a "RuntimeError" in Python 3.8.\n' - '\n' - 'When using the default metaclass "type", or any metaclass ' - 'that\n' - 'ultimately calls "type.__new__", the following additional\n' - 'customisation steps are invoked after creating the class ' - 'object:\n' - '\n' - '* first, "type.__new__" collects all of the descriptors in ' - 'the class\n' - ' namespace that define a "__set_name__()" method;\n' - '\n' - '* second, all of these "__set_name__" methods are called ' - 'with the\n' - ' class being defined and the assigned name of that ' - 'particular\n' - ' descriptor; and\n' - '\n' - '* finally, the "__init_subclass__()" hook is called on the ' - 'immediate\n' - ' parent of the new class in its method resolution order.\n' - '\n' - 'After the class object is created, it is passed to the ' - 'class\n' - 'decorators included in the class definition (if any) and the ' - 'resulting\n' - 'object is bound in the local namespace as the defined ' - 'class.\n' - '\n' - 'When a new class is created by "type.__new__", the object ' - 'provided as\n' - 'the namespace parameter is copied to a new ordered mapping ' - 'and the\n' - 'original object is discarded. The new copy is wrapped in a ' - 'read-only\n' - 'proxy, which becomes the "__dict__" attribute of the class ' - 'object.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3135** - New super\n' - ' Describes the implicit "__class__" closure reference\n' - '\n' - '\n' - 'Uses for metaclasses\n' - '--------------------\n' - '\n' - 'The potential uses for metaclasses are boundless. Some ideas ' - 'that have\n' - 'been explored include enum, logging, interface checking, ' - 'automatic\n' - 'delegation, automatic property creation, proxies, ' - 'frameworks, and\n' - 'automatic resource locking/synchronization.\n' - '\n' - '\n' - 'Customizing instance and subclass checks\n' - '========================================\n' - '\n' - 'The following methods are used to override the default ' - 'behavior of the\n' - '"isinstance()" and "issubclass()" built-in functions.\n' - '\n' - 'In particular, the metaclass "abc.ABCMeta" implements these ' - 'methods in\n' - 'order to allow the addition of Abstract Base Classes (ABCs) ' - 'as\n' - '“virtual base classes” to any class or type (including ' - 'built-in\n' - 'types), including other ABCs.\n' - '\n' - 'class.__instancecheck__(self, instance)\n' - '\n' - ' Return true if *instance* should be considered a (direct ' - 'or\n' - ' indirect) instance of *class*. If defined, called to ' - 'implement\n' - ' "isinstance(instance, class)".\n' - '\n' - 'class.__subclasscheck__(self, subclass)\n' - '\n' - ' Return true if *subclass* should be considered a (direct ' - 'or\n' - ' indirect) subclass of *class*. If defined, called to ' - 'implement\n' - ' "issubclass(subclass, class)".\n' - '\n' - 'Note that these methods are looked up on the type ' - '(metaclass) of a\n' - 'class. They cannot be defined as class methods in the ' - 'actual class.\n' - 'This is consistent with the lookup of special methods that ' - 'are called\n' - 'on instances, only in this case the instance is itself a ' - 'class.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 3119** - Introducing Abstract Base Classes\n' - ' Includes the specification for customizing ' - '"isinstance()" and\n' - ' "issubclass()" behavior through "__instancecheck__()" ' - 'and\n' - ' "__subclasscheck__()", with motivation for this ' - 'functionality in\n' - ' the context of adding Abstract Base Classes (see the ' - '"abc"\n' - ' module) to the language.\n' - '\n' - '\n' - 'Emulating callable objects\n' - '==========================\n' - '\n' - 'object.__call__(self[, args...])\n' - '\n' - ' Called when the instance is “called” as a function; if ' - 'this method\n' - ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' - ' "x.__call__(arg1, arg2, ...)".\n' - '\n' - '\n' - 'Emulating container types\n' - '=========================\n' - '\n' - 'The following methods can be defined to implement container ' - 'objects.\n' - 'Containers usually are sequences (such as lists or tuples) ' - 'or mappings\n' - '(like dictionaries), but can represent other containers as ' - 'well. The\n' - 'first set of methods is used either to emulate a sequence or ' - 'to\n' - 'emulate a mapping; the difference is that for a sequence, ' - 'the\n' - 'allowable keys should be the integers *k* for which "0 <= k ' - '< N" where\n' - '*N* is the length of the sequence, or slice objects, which ' - 'define a\n' - 'range of items. It is also recommended that mappings ' - 'provide the\n' - 'methods "keys()", "values()", "items()", "get()", ' - '"clear()",\n' - '"setdefault()", "pop()", "popitem()", "copy()", and ' - '"update()"\n' - 'behaving similar to those for Python’s standard dictionary ' - 'objects.\n' - 'The "collections" module provides a "MutableMapping" ' - 'abstract base\n' - 'class to help create those methods from a base set of ' - '"__getitem__()",\n' - '"__setitem__()", "__delitem__()", and "keys()". Mutable ' - 'sequences\n' - 'should provide methods "append()", "count()", "index()", ' - '"extend()",\n' - '"insert()", "pop()", "remove()", "reverse()" and "sort()", ' - 'like Python\n' - 'standard list objects. Finally, sequence types should ' - 'implement\n' - 'addition (meaning concatenation) and multiplication ' - '(meaning\n' - 'repetition) by defining the methods "__add__()", ' - '"__radd__()",\n' - '"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" ' - 'described\n' - 'below; they should not define other numerical operators. It ' - 'is\n' - 'recommended that both mappings and sequences implement the\n' - '"__contains__()" method to allow efficient use of the "in" ' - 'operator;\n' - 'for mappings, "in" should search the mapping’s keys; for ' - 'sequences, it\n' - 'should search through the values. It is further recommended ' - 'that both\n' - 'mappings and sequences implement the "__iter__()" method to ' - 'allow\n' - 'efficient iteration through the container; for mappings, ' - '"__iter__()"\n' - 'should be the same as "keys()"; for sequences, it should ' - 'iterate\n' - 'through the values.\n' - '\n' - 'object.__len__(self)\n' - '\n' - ' Called to implement the built-in function "len()". ' - 'Should return\n' - ' the length of the object, an integer ">=" 0. Also, an ' - 'object that\n' - ' doesn’t define a "__bool__()" method and whose ' - '"__len__()" method\n' - ' returns zero is considered to be false in a Boolean ' - 'context.\n' - '\n' - ' **CPython implementation detail:** In CPython, the length ' - 'is\n' - ' required to be at most "sys.maxsize". If the length is ' - 'larger than\n' - ' "sys.maxsize" some features (such as "len()") may raise\n' - ' "OverflowError". To prevent raising "OverflowError" by ' - 'truth value\n' - ' testing, an object must define a "__bool__()" method.\n' - '\n' - 'object.__length_hint__(self)\n' - '\n' - ' Called to implement "operator.length_hint()". Should ' - 'return an\n' - ' estimated length for the object (which may be greater or ' - 'less than\n' - ' the actual length). The length must be an integer ">=" 0. ' - 'This\n' - ' method is purely an optimization and is never required ' - 'for\n' - ' correctness.\n' - '\n' - ' New in version 3.4.\n' - '\n' - 'Note: Slicing is done exclusively with the following three ' - 'methods.\n' - ' A call like\n' - '\n' - ' a[1:2] = b\n' - '\n' - ' is translated to\n' - '\n' - ' a[slice(1, 2, None)] = b\n' - '\n' - ' and so forth. Missing slice items are always filled in ' - 'with "None".\n' - '\n' - 'object.__getitem__(self, key)\n' - '\n' - ' Called to implement evaluation of "self[key]". For ' - 'sequence types,\n' - ' the accepted keys should be integers and slice objects. ' - 'Note that\n' - ' the special interpretation of negative indexes (if the ' - 'class wishes\n' - ' to emulate a sequence type) is up to the "__getitem__()" ' - 'method. If\n' - ' *key* is of an inappropriate type, "TypeError" may be ' - 'raised; if of\n' - ' a value outside the set of indexes for the sequence ' - '(after any\n' - ' special interpretation of negative values), "IndexError" ' - 'should be\n' - ' raised. For mapping types, if *key* is missing (not in ' - 'the\n' - ' container), "KeyError" should be raised.\n' - '\n' - ' Note: "for" loops expect that an "IndexError" will be ' - 'raised for\n' - ' illegal indexes to allow proper detection of the end of ' - 'the\n' - ' sequence.\n' - '\n' - 'object.__setitem__(self, key, value)\n' - '\n' - ' Called to implement assignment to "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support changes to the values for keys, or if ' - 'new keys\n' - ' can be added, or for sequences if elements can be ' - 'replaced. The\n' - ' same exceptions should be raised for improper *key* ' - 'values as for\n' - ' the "__getitem__()" method.\n' - '\n' - 'object.__delitem__(self, key)\n' - '\n' - ' Called to implement deletion of "self[key]". Same note ' - 'as for\n' - ' "__getitem__()". This should only be implemented for ' - 'mappings if\n' - ' the objects support removal of keys, or for sequences if ' - 'elements\n' - ' can be removed from the sequence. The same exceptions ' - 'should be\n' - ' raised for improper *key* values as for the ' - '"__getitem__()" method.\n' - '\n' - 'object.__missing__(self, key)\n' - '\n' - ' Called by "dict"."__getitem__()" to implement "self[key]" ' - 'for dict\n' - ' subclasses when key is not in the dictionary.\n' - '\n' - 'object.__iter__(self)\n' - '\n' - ' This method is called when an iterator is required for a ' - 'container.\n' - ' This method should return a new iterator object that can ' - 'iterate\n' - ' over all the objects in the container. For mappings, it ' - 'should\n' - ' iterate over the keys of the container.\n' - '\n' - ' Iterator objects also need to implement this method; they ' - 'are\n' - ' required to return themselves. For more information on ' - 'iterator\n' - ' objects, see Iterator Types.\n' - '\n' - 'object.__reversed__(self)\n' - '\n' - ' Called (if present) by the "reversed()" built-in to ' - 'implement\n' - ' reverse iteration. It should return a new iterator ' - 'object that\n' - ' iterates over all the objects in the container in reverse ' - 'order.\n' - '\n' - ' If the "__reversed__()" method is not provided, the ' - '"reversed()"\n' - ' built-in will fall back to using the sequence protocol ' - '("__len__()"\n' - ' and "__getitem__()"). Objects that support the sequence ' - 'protocol\n' - ' should only provide "__reversed__()" if they can provide ' - 'an\n' - ' implementation that is more efficient than the one ' - 'provided by\n' - ' "reversed()".\n' - '\n' - 'The membership test operators ("in" and "not in") are ' - 'normally\n' - 'implemented as an iteration through a sequence. However, ' - 'container\n' - 'objects can supply the following special method with a more ' - 'efficient\n' - 'implementation, which also does not require the object be a ' - 'sequence.\n' - '\n' - 'object.__contains__(self, item)\n' - '\n' - ' Called to implement membership test operators. Should ' - 'return true\n' - ' if *item* is in *self*, false otherwise. For mapping ' - 'objects, this\n' - ' should consider the keys of the mapping rather than the ' - 'values or\n' - ' the key-item pairs.\n' - '\n' - ' For objects that don’t define "__contains__()", the ' - 'membership test\n' - ' first tries iteration via "__iter__()", then the old ' - 'sequence\n' - ' iteration protocol via "__getitem__()", see this section ' - 'in the\n' - ' language reference.\n' - '\n' - '\n' - 'Emulating numeric types\n' - '=======================\n' - '\n' - 'The following methods can be defined to emulate numeric ' - 'objects.\n' - 'Methods corresponding to operations that are not supported ' - 'by the\n' - 'particular kind of number implemented (e.g., bitwise ' - 'operations for\n' - 'non-integral numbers) should be left undefined.\n' - '\n' - 'object.__add__(self, other)\n' - 'object.__sub__(self, other)\n' - 'object.__mul__(self, other)\n' - 'object.__matmul__(self, other)\n' - 'object.__truediv__(self, other)\n' - 'object.__floordiv__(self, other)\n' - 'object.__mod__(self, other)\n' - 'object.__divmod__(self, other)\n' - 'object.__pow__(self, other[, modulo])\n' - 'object.__lshift__(self, other)\n' - 'object.__rshift__(self, other)\n' - 'object.__and__(self, other)\n' - 'object.__xor__(self, other)\n' - 'object.__or__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, ' - 'to\n' - ' evaluate the expression "x + y", where *x* is an instance ' - 'of a\n' - ' class that has an "__add__()" method, "x.__add__(y)" is ' - 'called.\n' - ' The "__divmod__()" method should be the equivalent to ' - 'using\n' - ' "__floordiv__()" and "__mod__()"; it should not be ' - 'related to\n' - ' "__truediv__()". Note that "__pow__()" should be defined ' - 'to accept\n' - ' an optional third argument if the ternary version of the ' - 'built-in\n' - ' "pow()" function is to be supported.\n' - '\n' - ' If one of those methods does not support the operation ' - 'with the\n' - ' supplied arguments, it should return "NotImplemented".\n' - '\n' - 'object.__radd__(self, other)\n' - 'object.__rsub__(self, other)\n' - 'object.__rmul__(self, other)\n' - 'object.__rmatmul__(self, other)\n' - 'object.__rtruediv__(self, other)\n' - 'object.__rfloordiv__(self, other)\n' - 'object.__rmod__(self, other)\n' - 'object.__rdivmod__(self, other)\n' - 'object.__rpow__(self, other)\n' - 'object.__rlshift__(self, other)\n' - 'object.__rrshift__(self, other)\n' - 'object.__rand__(self, other)\n' - 'object.__rxor__(self, other)\n' - 'object.__ror__(self, other)\n' - '\n' - ' These methods are called to implement the binary ' - 'arithmetic\n' - ' operations ("+", "-", "*", "@", "/", "//", "%", ' - '"divmod()",\n' - ' "pow()", "**", "<<", ">>", "&", "^", "|") with reflected ' - '(swapped)\n' - ' operands. These functions are only called if the left ' - 'operand does\n' - ' not support the corresponding operation [3] and the ' - 'operands are of\n' - ' different types. [4] For instance, to evaluate the ' - 'expression "x -\n' - ' y", where *y* is an instance of a class that has an ' - '"__rsub__()"\n' - ' method, "y.__rsub__(x)" is called if "x.__sub__(y)" ' - 'returns\n' - ' *NotImplemented*.\n' - '\n' - ' Note that ternary "pow()" will not try calling ' - '"__rpow__()" (the\n' - ' coercion rules would become too complicated).\n' - '\n' - ' Note: If the right operand’s type is a subclass of the ' - 'left\n' - ' operand’s type and that subclass provides the reflected ' - 'method\n' - ' for the operation, this method will be called before ' - 'the left\n' - ' operand’s non-reflected method. This behavior allows ' - 'subclasses\n' - ' to override their ancestors’ operations.\n' - '\n' - 'object.__iadd__(self, other)\n' - 'object.__isub__(self, other)\n' - 'object.__imul__(self, other)\n' - 'object.__imatmul__(self, other)\n' - 'object.__itruediv__(self, other)\n' - 'object.__ifloordiv__(self, other)\n' - 'object.__imod__(self, other)\n' - 'object.__ipow__(self, other[, modulo])\n' - 'object.__ilshift__(self, other)\n' - 'object.__irshift__(self, other)\n' - 'object.__iand__(self, other)\n' - 'object.__ixor__(self, other)\n' - 'object.__ior__(self, other)\n' - '\n' - ' These methods are called to implement the augmented ' - 'arithmetic\n' - ' assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", ' - '"**=",\n' - ' "<<=", ">>=", "&=", "^=", "|="). These methods should ' - 'attempt to\n' - ' do the operation in-place (modifying *self*) and return ' - 'the result\n' - ' (which could be, but does not have to be, *self*). If a ' - 'specific\n' - ' method is not defined, the augmented assignment falls ' - 'back to the\n' - ' normal methods. For instance, if *x* is an instance of a ' - 'class\n' - ' with an "__iadd__()" method, "x += y" is equivalent to "x ' - '=\n' - ' x.__iadd__(y)" . Otherwise, "x.__add__(y)" and ' - '"y.__radd__(x)" are\n' - ' considered, as with the evaluation of "x + y". In ' - 'certain\n' - ' situations, augmented assignment can result in unexpected ' - 'errors\n' - ' (see Why does a_tuple[i] += [‘item’] raise an exception ' - 'when the\n' - ' addition works?), but this behavior is in fact part of ' - 'the data\n' - ' model.\n' - '\n' - 'object.__neg__(self)\n' - 'object.__pos__(self)\n' - 'object.__abs__(self)\n' - 'object.__invert__(self)\n' - '\n' - ' Called to implement the unary arithmetic operations ("-", ' - '"+",\n' - ' "abs()" and "~").\n' - '\n' - 'object.__complex__(self)\n' - 'object.__int__(self)\n' - 'object.__float__(self)\n' - '\n' - ' Called to implement the built-in functions "complex()", ' - '"int()" and\n' - ' "float()". Should return a value of the appropriate ' - 'type.\n' - '\n' - 'object.__index__(self)\n' - '\n' - ' Called to implement "operator.index()", and whenever ' - 'Python needs\n' - ' to losslessly convert the numeric object to an integer ' - 'object (such\n' - ' as in slicing, or in the built-in "bin()", "hex()" and ' - '"oct()"\n' - ' functions). Presence of this method indicates that the ' - 'numeric\n' - ' object is an integer type. Must return an integer.\n' - '\n' - ' Note: In order to have a coherent integer type class, ' - 'when\n' - ' "__index__()" is defined "__int__()" should also be ' - 'defined, and\n' - ' both should return the same value.\n' - '\n' - 'object.__round__(self[, ndigits])\n' - 'object.__trunc__(self)\n' - 'object.__floor__(self)\n' - 'object.__ceil__(self)\n' - '\n' - ' Called to implement the built-in function "round()" and ' - '"math"\n' - ' functions "trunc()", "floor()" and "ceil()". Unless ' - '*ndigits* is\n' - ' passed to "__round__()" all these methods should return ' - 'the value\n' - ' of the object truncated to an "Integral" (typically an ' - '"int").\n' - '\n' - ' If "__int__()" is not defined then the built-in function ' - '"int()"\n' - ' falls back to "__trunc__()".\n' - '\n' - '\n' - 'With Statement Context Managers\n' - '===============================\n' - '\n' - 'A *context manager* is an object that defines the runtime ' - 'context to\n' - 'be established when executing a "with" statement. The ' - 'context manager\n' - 'handles the entry into, and the exit from, the desired ' - 'runtime context\n' - 'for the execution of the block of code. Context managers ' - 'are normally\n' - 'invoked using the "with" statement (described in section The ' - 'with\n' - 'statement), but can also be used by directly invoking their ' - 'methods.\n' - '\n' - 'Typical uses of context managers include saving and ' - 'restoring various\n' - 'kinds of global state, locking and unlocking resources, ' - 'closing opened\n' - 'files, etc.\n' - '\n' - 'For more information on context managers, see Context ' - 'Manager Types.\n' - '\n' - 'object.__enter__(self)\n' - '\n' - ' Enter the runtime context related to this object. The ' - '"with"\n' - ' statement will bind this method’s return value to the ' - 'target(s)\n' - ' specified in the "as" clause of the statement, if any.\n' - '\n' - 'object.__exit__(self, exc_type, exc_value, traceback)\n' - '\n' - ' Exit the runtime context related to this object. The ' - 'parameters\n' - ' describe the exception that caused the context to be ' - 'exited. If the\n' - ' context was exited without an exception, all three ' - 'arguments will\n' - ' be "None".\n' - '\n' - ' If an exception is supplied, and the method wishes to ' - 'suppress the\n' - ' exception (i.e., prevent it from being propagated), it ' - 'should\n' - ' return a true value. Otherwise, the exception will be ' - 'processed\n' - ' normally upon exit from this method.\n' - '\n' - ' Note that "__exit__()" methods should not reraise the ' - 'passed-in\n' - ' exception; this is the caller’s responsibility.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the ' - 'Python "with"\n' - ' statement.\n' - '\n' - '\n' - 'Special method lookup\n' - '=====================\n' - '\n' - 'For custom classes, implicit invocations of special methods ' - 'are only\n' - 'guaranteed to work correctly if defined on an object’s type, ' - 'not in\n' - 'the object’s instance dictionary. That behaviour is the ' - 'reason why\n' - 'the following code raises an exception:\n' - '\n' - ' >>> class C:\n' - ' ... pass\n' - ' ...\n' - ' >>> c = C()\n' - ' >>> c.__len__ = lambda: 5\n' - ' >>> len(c)\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: object of type 'C' has no len()\n" - '\n' - 'The rationale behind this behaviour lies with a number of ' - 'special\n' - 'methods such as "__hash__()" and "__repr__()" that are ' - 'implemented by\n' - 'all objects, including type objects. If the implicit lookup ' - 'of these\n' - 'methods used the conventional lookup process, they would ' - 'fail when\n' - 'invoked on the type object itself:\n' - '\n' - ' >>> 1 .__hash__() == hash(1)\n' - ' True\n' - ' >>> int.__hash__() == hash(int)\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " TypeError: descriptor '__hash__' of 'int' object needs an " - 'argument\n' - '\n' - 'Incorrectly attempting to invoke an unbound method of a ' - 'class in this\n' - 'way is sometimes referred to as ‘metaclass confusion’, and ' - 'is avoided\n' - 'by bypassing the instance when looking up special methods:\n' - '\n' - ' >>> type(1).__hash__(1) == hash(1)\n' - ' True\n' - ' >>> type(int).__hash__(int) == hash(int)\n' - ' True\n' - '\n' - 'In addition to bypassing any instance attributes in the ' - 'interest of\n' - 'correctness, implicit special method lookup generally also ' - 'bypasses\n' - 'the "__getattribute__()" method even of the object’s ' - 'metaclass:\n' - '\n' - ' >>> class Meta(type):\n' - ' ... def __getattribute__(*args):\n' - ' ... print("Metaclass getattribute invoked")\n' - ' ... return type.__getattribute__(*args)\n' - ' ...\n' - ' >>> class C(object, metaclass=Meta):\n' - ' ... def __len__(self):\n' - ' ... return 10\n' - ' ... def __getattribute__(*args):\n' - ' ... print("Class getattribute invoked")\n' - ' ... return object.__getattribute__(*args)\n' - ' ...\n' - ' >>> c = C()\n' - ' >>> c.__len__() # Explicit lookup via ' - 'instance\n' - ' Class getattribute invoked\n' - ' 10\n' - ' >>> type(c).__len__(c) # Explicit lookup via ' - 'type\n' - ' Metaclass getattribute invoked\n' - ' 10\n' - ' >>> len(c) # Implicit lookup\n' - ' 10\n' - '\n' - 'Bypassing the "__getattribute__()" machinery in this fashion ' - 'provides\n' - 'significant scope for speed optimisations within the ' - 'interpreter, at\n' - 'the cost of some flexibility in the handling of special ' - 'methods (the\n' - 'special method *must* be set on the class object itself in ' - 'order to be\n' - 'consistently invoked by the interpreter).\n', - 'string-methods': 'String Methods\n' - '**************\n' - '\n' - 'Strings implement all of the common sequence operations, ' - 'along with\n' - 'the additional methods described below.\n' - '\n' - 'Strings also support two styles of string formatting, one ' - 'providing a\n' - 'large degree of flexibility and customization (see ' - '"str.format()",\n' - 'Format String Syntax and Custom String Formatting) and the ' - 'other based\n' - 'on C "printf" style formatting that handles a narrower ' - 'range of types\n' - 'and is slightly harder to use correctly, but is often ' - 'faster for the\n' - 'cases it can handle (printf-style String Formatting).\n' - '\n' - 'The Text Processing Services section of the standard ' - 'library covers a\n' - 'number of other modules that provide various text related ' - 'utilities\n' - '(including regular expression support in the "re" ' - 'module).\n' - '\n' - 'str.capitalize()\n' - '\n' - ' Return a copy of the string with its first character ' - 'capitalized\n' - ' and the rest lowercased.\n' - '\n' - 'str.casefold()\n' - '\n' - ' Return a casefolded copy of the string. Casefolded ' - 'strings may be\n' - ' used for caseless matching.\n' - '\n' - ' Casefolding is similar to lowercasing but more ' - 'aggressive because\n' - ' it is intended to remove all case distinctions in a ' - 'string. For\n' - ' example, the German lowercase letter "\'ß\'" is ' - 'equivalent to ""ss"".\n' - ' Since it is already lowercase, "lower()" would do ' - 'nothing to "\'ß\'";\n' - ' "casefold()" converts it to ""ss"".\n' - '\n' - ' The casefolding algorithm is described in section 3.13 ' - 'of the\n' - ' Unicode Standard.\n' - '\n' - ' New in version 3.3.\n' - '\n' - 'str.center(width[, fillchar])\n' - '\n' - ' Return centered in a string of length *width*. Padding ' - 'is done\n' - ' using the specified *fillchar* (default is an ASCII ' - 'space). The\n' - ' original string is returned if *width* is less than or ' - 'equal to\n' - ' "len(s)".\n' - '\n' - 'str.count(sub[, start[, end]])\n' - '\n' - ' Return the number of non-overlapping occurrences of ' - 'substring *sub*\n' - ' in the range [*start*, *end*]. Optional arguments ' - '*start* and\n' - ' *end* are interpreted as in slice notation.\n' - '\n' - 'str.encode(encoding="utf-8", errors="strict")\n' - '\n' - ' Return an encoded version of the string as a bytes ' - 'object. Default\n' - ' encoding is "\'utf-8\'". *errors* may be given to set a ' - 'different\n' - ' error handling scheme. The default for *errors* is ' - '"\'strict\'",\n' - ' meaning that encoding errors raise a "UnicodeError". ' - 'Other possible\n' - ' values are "\'ignore\'", "\'replace\'", ' - '"\'xmlcharrefreplace\'",\n' - ' "\'backslashreplace\'" and any other name registered ' - 'via\n' - ' "codecs.register_error()", see section Error Handlers. ' - 'For a list\n' - ' of possible encodings, see section Standard Encodings.\n' - '\n' - ' Changed in version 3.1: Support for keyword arguments ' - 'added.\n' - '\n' - 'str.endswith(suffix[, start[, end]])\n' - '\n' - ' Return "True" if the string ends with the specified ' - '*suffix*,\n' - ' otherwise return "False". *suffix* can also be a tuple ' - 'of suffixes\n' - ' to look for. With optional *start*, test beginning at ' - 'that\n' - ' position. With optional *end*, stop comparing at that ' - 'position.\n' - '\n' - 'str.expandtabs(tabsize=8)\n' - '\n' - ' Return a copy of the string where all tab characters ' - 'are replaced\n' - ' by one or more spaces, depending on the current column ' - 'and the\n' - ' given tab size. Tab positions occur every *tabsize* ' - 'characters\n' - ' (default is 8, giving tab positions at columns 0, 8, 16 ' - 'and so on).\n' - ' To expand the string, the current column is set to zero ' - 'and the\n' - ' string is examined character by character. If the ' - 'character is a\n' - ' tab ("\\t"), one or more space characters are inserted ' - 'in the result\n' - ' until the current column is equal to the next tab ' - 'position. (The\n' - ' tab character itself is not copied.) If the character ' - 'is a newline\n' - ' ("\\n") or return ("\\r"), it is copied and the current ' - 'column is\n' - ' reset to zero. Any other character is copied unchanged ' - 'and the\n' - ' current column is incremented by one regardless of how ' - 'the\n' - ' character is represented when printed.\n' - '\n' - " >>> '01\\t012\\t0123\\t01234'.expandtabs()\n" - " '01 012 0123 01234'\n" - " >>> '01\\t012\\t0123\\t01234'.expandtabs(4)\n" - " '01 012 0123 01234'\n" - '\n' - 'str.find(sub[, start[, end]])\n' - '\n' - ' Return the lowest index in the string where substring ' - '*sub* is\n' - ' found within the slice "s[start:end]". Optional ' - 'arguments *start*\n' - ' and *end* are interpreted as in slice notation. Return ' - '"-1" if\n' - ' *sub* is not found.\n' - '\n' - ' Note: The "find()" method should be used only if you ' - 'need to know\n' - ' the position of *sub*. To check if *sub* is a ' - 'substring or not,\n' - ' use the "in" operator:\n' - '\n' - " >>> 'Py' in 'Python'\n" - ' True\n' - '\n' - 'str.format(*args, **kwargs)\n' - '\n' - ' Perform a string formatting operation. The string on ' - 'which this\n' - ' method is called can contain literal text or ' - 'replacement fields\n' - ' delimited by braces "{}". Each replacement field ' - 'contains either\n' - ' the numeric index of a positional argument, or the name ' - 'of a\n' - ' keyword argument. Returns a copy of the string where ' - 'each\n' - ' replacement field is replaced with the string value of ' - 'the\n' - ' corresponding argument.\n' - '\n' - ' >>> "The sum of 1 + 2 is {0}".format(1+2)\n' - " 'The sum of 1 + 2 is 3'\n" - '\n' - ' See Format String Syntax for a description of the ' - 'various\n' - ' formatting options that can be specified in format ' - 'strings.\n' - '\n' - ' Note: When formatting a number ("int", "float", ' - '"complex",\n' - ' "decimal.Decimal" and subclasses) with the "n" type ' - '(ex:\n' - ' "\'{:n}\'.format(1234)"), the function temporarily ' - 'sets the\n' - ' "LC_CTYPE" locale to the "LC_NUMERIC" locale to ' - 'decode\n' - ' "decimal_point" and "thousands_sep" fields of ' - '"localeconv()" if\n' - ' they are non-ASCII or longer than 1 byte, and the ' - '"LC_NUMERIC"\n' - ' locale is different than the "LC_CTYPE" locale. This ' - 'temporary\n' - ' change affects other threads.\n' - '\n' - ' Changed in version 3.6.5: When formatting a number with ' - 'the "n"\n' - ' type, the function sets temporarily the "LC_CTYPE" ' - 'locale to the\n' - ' "LC_NUMERIC" locale in some cases.\n' - '\n' - 'str.format_map(mapping)\n' - '\n' - ' Similar to "str.format(**mapping)", except that ' - '"mapping" is used\n' - ' directly and not copied to a "dict". This is useful if ' - 'for example\n' - ' "mapping" is a dict subclass:\n' - '\n' - ' >>> class Default(dict):\n' - ' ... def __missing__(self, key):\n' - ' ... return key\n' - ' ...\n' - " >>> '{name} was born in " - "{country}'.format_map(Default(name='Guido'))\n" - " 'Guido was born in country'\n" - '\n' - ' New in version 3.2.\n' - '\n' - 'str.index(sub[, start[, end]])\n' - '\n' - ' Like "find()", but raise "ValueError" when the ' - 'substring is not\n' - ' found.\n' - '\n' - 'str.isalnum()\n' - '\n' - ' Return true if all characters in the string are ' - 'alphanumeric and\n' - ' there is at least one character, false otherwise. A ' - 'character "c"\n' - ' is alphanumeric if one of the following returns ' - '"True":\n' - ' "c.isalpha()", "c.isdecimal()", "c.isdigit()", or ' - '"c.isnumeric()".\n' - '\n' - 'str.isalpha()\n' - '\n' - ' Return true if all characters in the string are ' - 'alphabetic and\n' - ' there is at least one character, false otherwise. ' - 'Alphabetic\n' - ' characters are those characters defined in the Unicode ' - 'character\n' - ' database as “Letter”, i.e., those with general category ' - 'property\n' - ' being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note ' - 'that this is\n' - ' different from the “Alphabetic” property defined in the ' - 'Unicode\n' - ' Standard.\n' - '\n' - 'str.isdecimal()\n' - '\n' - ' Return true if all characters in the string are decimal ' - 'characters\n' - ' and there is at least one character, false otherwise. ' - 'Decimal\n' - ' characters are those that can be used to form numbers ' - 'in base 10,\n' - ' e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a ' - 'decimal character\n' - ' is a character in the Unicode General Category “Nd”.\n' - '\n' - 'str.isdigit()\n' - '\n' - ' Return true if all characters in the string are digits ' - 'and there is\n' - ' at least one character, false otherwise. Digits ' - 'include decimal\n' - ' characters and digits that need special handling, such ' - 'as the\n' - ' compatibility superscript digits. This covers digits ' - 'which cannot\n' - ' be used to form numbers in base 10, like the Kharosthi ' - 'numbers.\n' - ' Formally, a digit is a character that has the property ' - 'value\n' - ' Numeric_Type=Digit or Numeric_Type=Decimal.\n' - '\n' - 'str.isidentifier()\n' - '\n' - ' Return true if the string is a valid identifier ' - 'according to the\n' - ' language definition, section Identifiers and keywords.\n' - '\n' - ' Use "keyword.iskeyword()" to test for reserved ' - 'identifiers such as\n' - ' "def" and "class".\n' - '\n' - 'str.islower()\n' - '\n' - ' Return true if all cased characters [4] in the string ' - 'are lowercase\n' - ' and there is at least one cased character, false ' - 'otherwise.\n' - '\n' - 'str.isnumeric()\n' - '\n' - ' Return true if all characters in the string are numeric ' - 'characters,\n' - ' and there is at least one character, false otherwise. ' - 'Numeric\n' - ' characters include digit characters, and all characters ' - 'that have\n' - ' the Unicode numeric value property, e.g. U+2155, VULGAR ' - 'FRACTION\n' - ' ONE FIFTH. Formally, numeric characters are those with ' - 'the\n' - ' property value Numeric_Type=Digit, Numeric_Type=Decimal ' - 'or\n' - ' Numeric_Type=Numeric.\n' - '\n' - 'str.isprintable()\n' - '\n' - ' Return true if all characters in the string are ' - 'printable or the\n' - ' string is empty, false otherwise. Nonprintable ' - 'characters are\n' - ' those characters defined in the Unicode character ' - 'database as\n' - ' “Other” or “Separator”, excepting the ASCII space ' - '(0x20) which is\n' - ' considered printable. (Note that printable characters ' - 'in this\n' - ' context are those which should not be escaped when ' - '"repr()" is\n' - ' invoked on a string. It has no bearing on the handling ' - 'of strings\n' - ' written to "sys.stdout" or "sys.stderr".)\n' - '\n' - 'str.isspace()\n' - '\n' - ' Return true if there are only whitespace characters in ' - 'the string\n' - ' and there is at least one character, false otherwise. ' - 'Whitespace\n' - ' characters are those characters defined in the Unicode ' - 'character\n' - ' database as “Other” or “Separator” and those with ' - 'bidirectional\n' - ' property being one of “WS”, “B”, or “S”.\n' - '\n' - 'str.istitle()\n' - '\n' - ' Return true if the string is a titlecased string and ' - 'there is at\n' - ' least one character, for example uppercase characters ' - 'may only\n' - ' follow uncased characters and lowercase characters only ' - 'cased ones.\n' - ' Return false otherwise.\n' - '\n' - 'str.isupper()\n' - '\n' - ' Return true if all cased characters [4] in the string ' - 'are uppercase\n' - ' and there is at least one cased character, false ' - 'otherwise.\n' - '\n' - 'str.join(iterable)\n' - '\n' - ' Return a string which is the concatenation of the ' - 'strings in\n' - ' *iterable*. A "TypeError" will be raised if there are ' - 'any non-\n' - ' string values in *iterable*, including "bytes" ' - 'objects. The\n' - ' separator between elements is the string providing this ' - 'method.\n' - '\n' - 'str.ljust(width[, fillchar])\n' - '\n' - ' Return the string left justified in a string of length ' - '*width*.\n' - ' Padding is done using the specified *fillchar* (default ' - 'is an ASCII\n' - ' space). The original string is returned if *width* is ' - 'less than or\n' - ' equal to "len(s)".\n' - '\n' - 'str.lower()\n' - '\n' - ' Return a copy of the string with all the cased ' - 'characters [4]\n' - ' converted to lowercase.\n' - '\n' - ' The lowercasing algorithm used is described in section ' - '3.13 of the\n' - ' Unicode Standard.\n' - '\n' - 'str.lstrip([chars])\n' - '\n' - ' Return a copy of the string with leading characters ' - 'removed. The\n' - ' *chars* argument is a string specifying the set of ' - 'characters to be\n' - ' removed. If omitted or "None", the *chars* argument ' - 'defaults to\n' - ' removing whitespace. The *chars* argument is not a ' - 'prefix; rather,\n' - ' all combinations of its values are stripped:\n' - '\n' - " >>> ' spacious '.lstrip()\n" - " 'spacious '\n" - " >>> 'www.example.com'.lstrip('cmowz.')\n" - " 'example.com'\n" - '\n' - 'static str.maketrans(x[, y[, z]])\n' - '\n' - ' This static method returns a translation table usable ' - 'for\n' - ' "str.translate()".\n' - '\n' - ' If there is only one argument, it must be a dictionary ' - 'mapping\n' - ' Unicode ordinals (integers) or characters (strings of ' - 'length 1) to\n' - ' Unicode ordinals, strings (of arbitrary lengths) or ' - '"None".\n' - ' Character keys will then be converted to ordinals.\n' - '\n' - ' If there are two arguments, they must be strings of ' - 'equal length,\n' - ' and in the resulting dictionary, each character in x ' - 'will be mapped\n' - ' to the character at the same position in y. If there ' - 'is a third\n' - ' argument, it must be a string, whose characters will be ' - 'mapped to\n' - ' "None" in the result.\n' - '\n' - 'str.partition(sep)\n' - '\n' - ' Split the string at the first occurrence of *sep*, and ' - 'return a\n' - ' 3-tuple containing the part before the separator, the ' - 'separator\n' - ' itself, and the part after the separator. If the ' - 'separator is not\n' - ' found, return a 3-tuple containing the string itself, ' - 'followed by\n' - ' two empty strings.\n' - '\n' - 'str.replace(old, new[, count])\n' - '\n' - ' Return a copy of the string with all occurrences of ' - 'substring *old*\n' - ' replaced by *new*. If the optional argument *count* is ' - 'given, only\n' - ' the first *count* occurrences are replaced.\n' - '\n' - 'str.rfind(sub[, start[, end]])\n' - '\n' - ' Return the highest index in the string where substring ' - '*sub* is\n' - ' found, such that *sub* is contained within ' - '"s[start:end]".\n' - ' Optional arguments *start* and *end* are interpreted as ' - 'in slice\n' - ' notation. Return "-1" on failure.\n' - '\n' - 'str.rindex(sub[, start[, end]])\n' - '\n' - ' Like "rfind()" but raises "ValueError" when the ' - 'substring *sub* is\n' - ' not found.\n' - '\n' - 'str.rjust(width[, fillchar])\n' - '\n' - ' Return the string right justified in a string of length ' - '*width*.\n' - ' Padding is done using the specified *fillchar* (default ' - 'is an ASCII\n' - ' space). The original string is returned if *width* is ' - 'less than or\n' - ' equal to "len(s)".\n' - '\n' - 'str.rpartition(sep)\n' - '\n' - ' Split the string at the last occurrence of *sep*, and ' - 'return a\n' - ' 3-tuple containing the part before the separator, the ' - 'separator\n' - ' itself, and the part after the separator. If the ' - 'separator is not\n' - ' found, return a 3-tuple containing two empty strings, ' - 'followed by\n' - ' the string itself.\n' - '\n' - 'str.rsplit(sep=None, maxsplit=-1)\n' - '\n' - ' Return a list of the words in the string, using *sep* ' - 'as the\n' - ' delimiter string. If *maxsplit* is given, at most ' - '*maxsplit* splits\n' - ' are done, the *rightmost* ones. If *sep* is not ' - 'specified or\n' - ' "None", any whitespace string is a separator. Except ' - 'for splitting\n' - ' from the right, "rsplit()" behaves like "split()" which ' - 'is\n' - ' described in detail below.\n' - '\n' - 'str.rstrip([chars])\n' - '\n' - ' Return a copy of the string with trailing characters ' - 'removed. The\n' - ' *chars* argument is a string specifying the set of ' - 'characters to be\n' - ' removed. If omitted or "None", the *chars* argument ' - 'defaults to\n' - ' removing whitespace. The *chars* argument is not a ' - 'suffix; rather,\n' - ' all combinations of its values are stripped:\n' - '\n' - " >>> ' spacious '.rstrip()\n" - " ' spacious'\n" - " >>> 'mississippi'.rstrip('ipz')\n" - " 'mississ'\n" - '\n' - 'str.split(sep=None, maxsplit=-1)\n' - '\n' - ' Return a list of the words in the string, using *sep* ' - 'as the\n' - ' delimiter string. If *maxsplit* is given, at most ' - '*maxsplit*\n' - ' splits are done (thus, the list will have at most ' - '"maxsplit+1"\n' - ' elements). If *maxsplit* is not specified or "-1", ' - 'then there is\n' - ' no limit on the number of splits (all possible splits ' - 'are made).\n' - '\n' - ' If *sep* is given, consecutive delimiters are not ' - 'grouped together\n' - ' and are deemed to delimit empty strings (for example,\n' - ' "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', ' - '\'2\']"). The *sep* argument\n' - ' may consist of multiple characters (for example,\n' - ' "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', ' - '\'3\']"). Splitting an\n' - ' empty string with a specified separator returns ' - '"[\'\']".\n' - '\n' - ' For example:\n' - '\n' - " >>> '1,2,3'.split(',')\n" - " ['1', '2', '3']\n" - " >>> '1,2,3'.split(',', maxsplit=1)\n" - " ['1', '2,3']\n" - " >>> '1,2,,3,'.split(',')\n" - " ['1', '2', '', '3', '']\n" - '\n' - ' If *sep* is not specified or is "None", a different ' - 'splitting\n' - ' algorithm is applied: runs of consecutive whitespace ' - 'are regarded\n' - ' as a single separator, and the result will contain no ' - 'empty strings\n' - ' at the start or end if the string has leading or ' - 'trailing\n' - ' whitespace. Consequently, splitting an empty string or ' - 'a string\n' - ' consisting of just whitespace with a "None" separator ' - 'returns "[]".\n' - '\n' - ' For example:\n' - '\n' - " >>> '1 2 3'.split()\n" - " ['1', '2', '3']\n" - " >>> '1 2 3'.split(maxsplit=1)\n" - " ['1', '2 3']\n" - " >>> ' 1 2 3 '.split()\n" - " ['1', '2', '3']\n" - '\n' - 'str.splitlines([keepends])\n' - '\n' - ' Return a list of the lines in the string, breaking at ' - 'line\n' - ' boundaries. Line breaks are not included in the ' - 'resulting list\n' - ' unless *keepends* is given and true.\n' - '\n' - ' This method splits on the following line boundaries. ' - 'In\n' - ' particular, the boundaries are a superset of *universal ' - 'newlines*.\n' - '\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | Representation | ' - 'Description |\n' - ' ' - '+=========================+===============================+\n' - ' | "\\n" | Line ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\r" | Carriage ' - 'Return |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\r\\n" | Carriage Return + Line ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\v" or "\\x0b" | Line ' - 'Tabulation |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\f" or "\\x0c" | Form ' - 'Feed |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1c" | File ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1d" | Group ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x1e" | Record ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\x85" | Next Line (C1 Control ' - 'Code) |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\u2028" | Line ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - ' | "\\u2029" | Paragraph ' - 'Separator |\n' - ' ' - '+-------------------------+-------------------------------+\n' - '\n' - ' Changed in version 3.2: "\\v" and "\\f" added to list ' - 'of line\n' - ' boundaries.\n' - '\n' - ' For example:\n' - '\n' - " >>> 'ab c\\n\\nde fg\\rkl\\r\\n'.splitlines()\n" - " ['ab c', '', 'de fg', 'kl']\n" - " >>> 'ab c\\n\\nde " - "fg\\rkl\\r\\n'.splitlines(keepends=True)\n" - " ['ab c\\n', '\\n', 'de fg\\r', 'kl\\r\\n']\n" - '\n' - ' Unlike "split()" when a delimiter string *sep* is ' - 'given, this\n' - ' method returns an empty list for the empty string, and ' - 'a terminal\n' - ' line break does not result in an extra line:\n' - '\n' - ' >>> "".splitlines()\n' - ' []\n' - ' >>> "One line\\n".splitlines()\n' - " ['One line']\n" - '\n' - ' For comparison, "split(\'\\n\')" gives:\n' - '\n' - " >>> ''.split('\\n')\n" - " ['']\n" - " >>> 'Two lines\\n'.split('\\n')\n" - " ['Two lines', '']\n" - '\n' - 'str.startswith(prefix[, start[, end]])\n' - '\n' - ' Return "True" if string starts with the *prefix*, ' - 'otherwise return\n' - ' "False". *prefix* can also be a tuple of prefixes to ' - 'look for.\n' - ' With optional *start*, test string beginning at that ' - 'position.\n' - ' With optional *end*, stop comparing string at that ' - 'position.\n' - '\n' - 'str.strip([chars])\n' - '\n' - ' Return a copy of the string with the leading and ' - 'trailing\n' - ' characters removed. The *chars* argument is a string ' - 'specifying the\n' - ' set of characters to be removed. If omitted or "None", ' - 'the *chars*\n' - ' argument defaults to removing whitespace. The *chars* ' - 'argument is\n' - ' not a prefix or suffix; rather, all combinations of its ' - 'values are\n' - ' stripped:\n' - '\n' - " >>> ' spacious '.strip()\n" - " 'spacious'\n" - " >>> 'www.example.com'.strip('cmowz.')\n" - " 'example'\n" - '\n' - ' The outermost leading and trailing *chars* argument ' - 'values are\n' - ' stripped from the string. Characters are removed from ' - 'the leading\n' - ' end until reaching a string character that is not ' - 'contained in the\n' - ' set of characters in *chars*. A similar action takes ' - 'place on the\n' - ' trailing end. For example:\n' - '\n' - " >>> comment_string = '#....... Section 3.2.1 Issue " - "#32 .......'\n" - " >>> comment_string.strip('.#! ')\n" - " 'Section 3.2.1 Issue #32'\n" - '\n' - 'str.swapcase()\n' - '\n' - ' Return a copy of the string with uppercase characters ' - 'converted to\n' - ' lowercase and vice versa. Note that it is not ' - 'necessarily true that\n' - ' "s.swapcase().swapcase() == s".\n' - '\n' - 'str.title()\n' - '\n' - ' Return a titlecased version of the string where words ' - 'start with an\n' - ' uppercase character and the remaining characters are ' - 'lowercase.\n' - '\n' - ' For example:\n' - '\n' - " >>> 'Hello world'.title()\n" - " 'Hello World'\n" - '\n' - ' The algorithm uses a simple language-independent ' - 'definition of a\n' - ' word as groups of consecutive letters. The definition ' - 'works in\n' - ' many contexts but it means that apostrophes in ' - 'contractions and\n' - ' possessives form word boundaries, which may not be the ' - 'desired\n' - ' result:\n' - '\n' - ' >>> "they\'re bill\'s friends from the UK".title()\n' - ' "They\'Re Bill\'S Friends From The Uk"\n' - '\n' - ' A workaround for apostrophes can be constructed using ' - 'regular\n' - ' expressions:\n' - '\n' - ' >>> import re\n' - ' >>> def titlecase(s):\n' - ' ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n' - ' ... lambda mo: ' - 'mo.group(0)[0].upper() +\n' - ' ... ' - 'mo.group(0)[1:].lower(),\n' - ' ... s)\n' - ' ...\n' - ' >>> titlecase("they\'re bill\'s friends.")\n' - ' "They\'re Bill\'s Friends."\n' - '\n' - 'str.translate(table)\n' - '\n' - ' Return a copy of the string in which each character has ' - 'been mapped\n' - ' through the given translation table. The table must be ' - 'an object\n' - ' that implements indexing via "__getitem__()", typically ' - 'a *mapping*\n' - ' or *sequence*. When indexed by a Unicode ordinal (an ' - 'integer), the\n' - ' table object can do any of the following: return a ' - 'Unicode ordinal\n' - ' or a string, to map the character to one or more other ' - 'characters;\n' - ' return "None", to delete the character from the return ' - 'string; or\n' - ' raise a "LookupError" exception, to map the character ' - 'to itself.\n' - '\n' - ' You can use "str.maketrans()" to create a translation ' - 'map from\n' - ' character-to-character mappings in different formats.\n' - '\n' - ' See also the "codecs" module for a more flexible ' - 'approach to custom\n' - ' character mappings.\n' - '\n' - 'str.upper()\n' - '\n' - ' Return a copy of the string with all the cased ' - 'characters [4]\n' - ' converted to uppercase. Note that ' - '"s.upper().isupper()" might be\n' - ' "False" if "s" contains uncased characters or if the ' - 'Unicode\n' - ' category of the resulting character(s) is not “Lu” ' - '(Letter,\n' - ' uppercase), but e.g. “Lt” (Letter, titlecase).\n' - '\n' - ' The uppercasing algorithm used is described in section ' - '3.13 of the\n' - ' Unicode Standard.\n' - '\n' - 'str.zfill(width)\n' - '\n' - ' Return a copy of the string left filled with ASCII ' - '"\'0\'" digits to\n' - ' make a string of length *width*. A leading sign prefix\n' - ' ("\'+\'"/"\'-\'") is handled by inserting the padding ' - '*after* the sign\n' - ' character rather than before. The original string is ' - 'returned if\n' - ' *width* is less than or equal to "len(s)".\n' - '\n' - ' For example:\n' - '\n' - ' >>> "42".zfill(5)\n' - " '00042'\n" - ' >>> "-42".zfill(5)\n' - " '-0042'\n", - 'strings': 'String and Bytes literals\n' - '*************************\n' - '\n' - 'String literals are described by the following lexical ' - 'definitions:\n' - '\n' - ' stringliteral ::= [stringprefix](shortstring | longstring)\n' - ' stringprefix ::= "r" | "u" | "R" | "U" | "f" | "F"\n' - ' | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | ' - '"Rf" | "RF"\n' - ' shortstring ::= "\'" shortstringitem* "\'" | \'"\' ' - 'shortstringitem* \'"\'\n' - ' longstring ::= "\'\'\'" longstringitem* "\'\'\'" | ' - '\'"""\' longstringitem* \'"""\'\n' - ' shortstringitem ::= shortstringchar | stringescapeseq\n' - ' longstringitem ::= longstringchar | stringescapeseq\n' - ' shortstringchar ::= \n' - ' longstringchar ::= \n' - ' stringescapeseq ::= "\\" \n' - '\n' - ' bytesliteral ::= bytesprefix(shortbytes | longbytes)\n' - ' bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | ' - '"rb" | "rB" | "Rb" | "RB"\n' - ' shortbytes ::= "\'" shortbytesitem* "\'" | \'"\' ' - 'shortbytesitem* \'"\'\n' - ' longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' ' - 'longbytesitem* \'"""\'\n' - ' shortbytesitem ::= shortbyteschar | bytesescapeseq\n' - ' longbytesitem ::= longbyteschar | bytesescapeseq\n' - ' shortbyteschar ::= \n' - ' longbyteschar ::= \n' - ' bytesescapeseq ::= "\\" \n' - '\n' - 'One syntactic restriction not indicated by these productions is ' - 'that\n' - 'whitespace is not allowed between the "stringprefix" or ' - '"bytesprefix"\n' - 'and the rest of the literal. The source character set is defined ' - 'by\n' - 'the encoding declaration; it is UTF-8 if no encoding declaration ' - 'is\n' - 'given in the source file; see section Encoding declarations.\n' - '\n' - 'In plain English: Both types of literals can be enclosed in ' - 'matching\n' - 'single quotes ("\'") or double quotes ("""). They can also be ' - 'enclosed\n' - 'in matching groups of three single or double quotes (these are\n' - 'generally referred to as *triple-quoted strings*). The ' - 'backslash\n' - '("\\") character is used to escape characters that otherwise have ' - 'a\n' - 'special meaning, such as newline, backslash itself, or the quote\n' - 'character.\n' - '\n' - 'Bytes literals are always prefixed with "\'b\'" or "\'B\'"; they ' - 'produce\n' - 'an instance of the "bytes" type instead of the "str" type. They ' - 'may\n' - 'only contain ASCII characters; bytes with a numeric value of 128 ' - 'or\n' - 'greater must be expressed with escapes.\n' - '\n' - 'Both string and bytes literals may optionally be prefixed with a\n' - 'letter "\'r\'" or "\'R\'"; such strings are called *raw strings* ' - 'and treat\n' - 'backslashes as literal characters. As a result, in string ' - 'literals,\n' - '"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated ' - 'specially.\n' - 'Given that Python 2.x’s raw unicode literals behave differently ' - 'than\n' - 'Python 3.x’s the "\'ur\'" syntax is not supported.\n' - '\n' - 'New in version 3.3: The "\'rb\'" prefix of raw bytes literals has ' - 'been\n' - 'added as a synonym of "\'br\'".\n' - '\n' - 'New in version 3.3: Support for the unicode legacy literal\n' - '("u\'value\'") was reintroduced to simplify the maintenance of ' - 'dual\n' - 'Python 2.x and 3.x codebases. See **PEP 414** for more ' - 'information.\n' - '\n' - 'A string literal with "\'f\'" or "\'F\'" in its prefix is a ' - '*formatted\n' - 'string literal*; see Formatted string literals. The "\'f\'" may ' - 'be\n' - 'combined with "\'r\'", but not with "\'b\'" or "\'u\'", therefore ' - 'raw\n' - 'formatted strings are possible, but formatted bytes literals are ' - 'not.\n' - '\n' - 'In triple-quoted literals, unescaped newlines and quotes are ' - 'allowed\n' - '(and are retained), except that three unescaped quotes in a row\n' - 'terminate the literal. (A “quote” is the character used to open ' - 'the\n' - 'literal, i.e. either "\'" or """.)\n' - '\n' - 'Unless an "\'r\'" or "\'R\'" prefix is present, escape sequences ' - 'in string\n' - 'and bytes literals are interpreted according to rules similar to ' - 'those\n' - 'used by Standard C. The recognized escape sequences are:\n' - '\n' - '+-------------------+-----------------------------------+---------+\n' - '| Escape Sequence | Meaning | Notes ' - '|\n' - '+===================+===================================+=========+\n' - '| "\\newline" | Backslash and newline ignored ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\\\" | Backslash ("\\") ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\\'" | Single quote ("\'") ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\"" | Double quote (""") ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\a" | ASCII Bell (BEL) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\b" | ASCII Backspace (BS) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\f" | ASCII Formfeed (FF) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\n" | ASCII Linefeed (LF) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\r" | ASCII Carriage Return (CR) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\t" | ASCII Horizontal Tab (TAB) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\v" | ASCII Vertical Tab (VT) ' - '| |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\ooo" | Character with octal value *ooo* | ' - '(1,3) |\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\xhh" | Character with hex value *hh* | ' - '(2,3) |\n' - '+-------------------+-----------------------------------+---------+\n' - '\n' - 'Escape sequences only recognized in string literals are:\n' - '\n' - '+-------------------+-----------------------------------+---------+\n' - '| Escape Sequence | Meaning | Notes ' - '|\n' - '+===================+===================================+=========+\n' - '| "\\N{name}" | Character named *name* in the | ' - '(4) |\n' - '| | Unicode database | ' - '|\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\uxxxx" | Character with 16-bit hex value | ' - '(5) |\n' - '| | *xxxx* | ' - '|\n' - '+-------------------+-----------------------------------+---------+\n' - '| "\\Uxxxxxxxx" | Character with 32-bit hex value | ' - '(6) |\n' - '| | *xxxxxxxx* | ' - '|\n' - '+-------------------+-----------------------------------+---------+\n' - '\n' - 'Notes:\n' - '\n' - '1. As in Standard C, up to three octal digits are accepted.\n' - '\n' - '2. Unlike in Standard C, exactly two hex digits are required.\n' - '\n' - '3. In a bytes literal, hexadecimal and octal escapes denote the\n' - ' byte with the given value. In a string literal, these escapes\n' - ' denote a Unicode character with the given value.\n' - '\n' - '4. Changed in version 3.3: Support for name aliases [1] has been\n' - ' added.\n' - '\n' - '5. Exactly four hex digits are required.\n' - '\n' - '6. Any Unicode character can be encoded this way. Exactly eight\n' - ' hex digits are required.\n' - '\n' - 'Unlike Standard C, all unrecognized escape sequences are left in ' - 'the\n' - 'string unchanged, i.e., *the backslash is left in the result*. ' - '(This\n' - 'behavior is useful when debugging: if an escape sequence is ' - 'mistyped,\n' - 'the resulting output is more easily recognized as broken.) It is ' - 'also\n' - 'important to note that the escape sequences only recognized in ' - 'string\n' - 'literals fall into the category of unrecognized escapes for ' - 'bytes\n' - 'literals.\n' - '\n' - ' Changed in version 3.6: Unrecognized escape sequences produce ' - 'a\n' - ' DeprecationWarning. In some future version of Python they ' - 'will be\n' - ' a SyntaxError.\n' - '\n' - 'Even in a raw literal, quotes can be escaped with a backslash, ' - 'but the\n' - 'backslash remains in the result; for example, "r"\\""" is a ' - 'valid\n' - 'string literal consisting of two characters: a backslash and a ' - 'double\n' - 'quote; "r"\\"" is not a valid string literal (even a raw string ' - 'cannot\n' - 'end in an odd number of backslashes). Specifically, *a raw ' - 'literal\n' - 'cannot end in a single backslash* (since the backslash would ' - 'escape\n' - 'the following quote character). Note also that a single ' - 'backslash\n' - 'followed by a newline is interpreted as those two characters as ' - 'part\n' - 'of the literal, *not* as a line continuation.\n', - 'subscriptions': 'Subscriptions\n' - '*************\n' - '\n' - 'A subscription selects an item of a sequence (string, tuple ' - 'or list)\n' - 'or mapping (dictionary) object:\n' - '\n' - ' subscription ::= primary "[" expression_list "]"\n' - '\n' - 'The primary must evaluate to an object that supports ' - 'subscription\n' - '(lists or dictionaries for example). User-defined objects ' - 'can support\n' - 'subscription by defining a "__getitem__()" method.\n' - '\n' - 'For built-in objects, there are two types of objects that ' - 'support\n' - 'subscription:\n' - '\n' - 'If the primary is a mapping, the expression list must ' - 'evaluate to an\n' - 'object whose value is one of the keys of the mapping, and ' - 'the\n' - 'subscription selects the value in the mapping that ' - 'corresponds to that\n' - 'key. (The expression list is a tuple except if it has ' - 'exactly one\n' - 'item.)\n' - '\n' - 'If the primary is a sequence, the expression list must ' - 'evaluate to an\n' - 'integer or a slice (as discussed in the following ' - 'section).\n' - '\n' - 'The formal syntax makes no special provision for negative ' - 'indices in\n' - 'sequences; however, built-in sequences all provide a ' - '"__getitem__()"\n' - 'method that interprets negative indices by adding the ' - 'length of the\n' - 'sequence to the index (so that "x[-1]" selects the last ' - 'item of "x").\n' - 'The resulting value must be a nonnegative integer less than ' - 'the number\n' - 'of items in the sequence, and the subscription selects the ' - 'item whose\n' - 'index is that value (counting from zero). Since the support ' - 'for\n' - 'negative indices and slicing occurs in the object’s ' - '"__getitem__()"\n' - 'method, subclasses overriding this method will need to ' - 'explicitly add\n' - 'that support.\n' - '\n' - 'A string’s items are characters. A character is not a ' - 'separate data\n' - 'type but a string of exactly one character.\n', - 'truth': 'Truth Value Testing\n' - '*******************\n' - '\n' - 'Any object can be tested for truth value, for use in an "if" or\n' - '"while" condition or as operand of the Boolean operations below.\n' - '\n' - 'By default, an object is considered true unless its class defines\n' - 'either a "__bool__()" method that returns "False" or a "__len__()"\n' - 'method that returns zero, when called with the object. [1] Here ' - 'are\n' - 'most of the built-in objects considered false:\n' - '\n' - '* constants defined to be false: "None" and "False".\n' - '\n' - '* zero of any numeric type: "0", "0.0", "0j", "Decimal(0)",\n' - ' "Fraction(0, 1)"\n' - '\n' - '* empty sequences and collections: "\'\'", "()", "[]", "{}", ' - '"set()",\n' - ' "range(0)"\n' - '\n' - 'Operations and built-in functions that have a Boolean result ' - 'always\n' - 'return "0" or "False" for false and "1" or "True" for true, unless\n' - 'otherwise stated. (Important exception: the Boolean operations ' - '"or"\n' - 'and "and" always return one of their operands.)\n', - 'try': 'The "try" statement\n' - '*******************\n' - '\n' - 'The "try" statement specifies exception handlers and/or cleanup code\n' - 'for a group of statements:\n' - '\n' - ' try_stmt ::= try1_stmt | try2_stmt\n' - ' try1_stmt ::= "try" ":" suite\n' - ' ("except" [expression ["as" identifier]] ":" ' - 'suite)+\n' - ' ["else" ":" suite]\n' - ' ["finally" ":" suite]\n' - ' try2_stmt ::= "try" ":" suite\n' - ' "finally" ":" suite\n' - '\n' - 'The "except" clause(s) specify one or more exception handlers. When ' - 'no\n' - 'exception occurs in the "try" clause, no exception handler is\n' - 'executed. When an exception occurs in the "try" suite, a search for ' - 'an\n' - 'exception handler is started. This search inspects the except ' - 'clauses\n' - 'in turn until one is found that matches the exception. An ' - 'expression-\n' - 'less except clause, if present, must be last; it matches any\n' - 'exception. For an except clause with an expression, that expression\n' - 'is evaluated, and the clause matches the exception if the resulting\n' - 'object is “compatible” with the exception. An object is compatible\n' - 'with an exception if it is the class or a base class of the ' - 'exception\n' - 'object or a tuple containing an item compatible with the exception.\n' - '\n' - 'If no except clause matches the exception, the search for an ' - 'exception\n' - 'handler continues in the surrounding code and on the invocation ' - 'stack.\n' - '[1]\n' - '\n' - 'If the evaluation of an expression in the header of an except clause\n' - 'raises an exception, the original search for a handler is canceled ' - 'and\n' - 'a search starts for the new exception in the surrounding code and on\n' - 'the call stack (it is treated as if the entire "try" statement ' - 'raised\n' - 'the exception).\n' - '\n' - 'When a matching except clause is found, the exception is assigned to\n' - 'the target specified after the "as" keyword in that except clause, ' - 'if\n' - 'present, and the except clause’s suite is executed. All except\n' - 'clauses must have an executable block. When the end of this block ' - 'is\n' - 'reached, execution continues normally after the entire try ' - 'statement.\n' - '(This means that if two nested handlers exist for the same ' - 'exception,\n' - 'and the exception occurs in the try clause of the inner handler, the\n' - 'outer handler will not handle the exception.)\n' - '\n' - 'When an exception has been assigned using "as target", it is cleared\n' - 'at the end of the except clause. This is as if\n' - '\n' - ' except E as N:\n' - ' foo\n' - '\n' - 'was translated to\n' - '\n' - ' except E as N:\n' - ' try:\n' - ' foo\n' - ' finally:\n' - ' del N\n' - '\n' - 'This means the exception must be assigned to a different name to be\n' - 'able to refer to it after the except clause. Exceptions are cleared\n' - 'because with the traceback attached to them, they form a reference\n' - 'cycle with the stack frame, keeping all locals in that frame alive\n' - 'until the next garbage collection occurs.\n' - '\n' - 'Before an except clause’s suite is executed, details about the\n' - 'exception are stored in the "sys" module and can be accessed via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of ' - 'the\n' - 'exception class, the exception instance and a traceback object (see\n' - 'section The standard type hierarchy) identifying the point in the\n' - 'program where the exception occurred. "sys.exc_info()" values are\n' - 'restored to their previous values (before the call) when returning\n' - 'from a function that handled an exception.\n' - '\n' - 'The optional "else" clause is executed if the control flow leaves ' - 'the\n' - '"try" suite, no exception was raised, and no "return", "continue", ' - 'or\n' - '"break" statement was executed. Exceptions in the "else" clause are\n' - 'not handled by the preceding "except" clauses.\n' - '\n' - 'If "finally" is present, it specifies a ‘cleanup’ handler. The ' - '"try"\n' - 'clause is executed, including any "except" and "else" clauses. If ' - 'an\n' - 'exception occurs in any of the clauses and is not handled, the\n' - 'exception is temporarily saved. The "finally" clause is executed. ' - 'If\n' - 'there is a saved exception it is re-raised at the end of the ' - '"finally"\n' - 'clause. If the "finally" clause raises another exception, the saved\n' - 'exception is set as the context of the new exception. If the ' - '"finally"\n' - 'clause executes a "return" or "break" statement, the saved exception\n' - 'is discarded:\n' - '\n' - ' >>> def f():\n' - ' ... try:\n' - ' ... 1/0\n' - ' ... finally:\n' - ' ... return 42\n' - ' ...\n' - ' >>> f()\n' - ' 42\n' - '\n' - 'The exception information is not available to the program during\n' - 'execution of the "finally" clause.\n' - '\n' - 'When a "return", "break" or "continue" statement is executed in the\n' - '"try" suite of a "try"…"finally" statement, the "finally" clause is\n' - 'also executed ‘on the way out.’ A "continue" statement is illegal in\n' - 'the "finally" clause. (The reason is a problem with the current\n' - 'implementation — this restriction may be lifted in the future).\n' - '\n' - 'The return value of a function is determined by the last "return"\n' - 'statement executed. Since the "finally" clause always executes, a\n' - '"return" statement executed in the "finally" clause will always be ' - 'the\n' - 'last one executed:\n' - '\n' - ' >>> def foo():\n' - ' ... try:\n' - " ... return 'try'\n" - ' ... finally:\n' - " ... return 'finally'\n" - ' ...\n' - ' >>> foo()\n' - " 'finally'\n" - '\n' - 'Additional information on exceptions can be found in section\n' - 'Exceptions, and information on using the "raise" statement to ' - 'generate\n' - 'exceptions may be found in section The raise statement.\n', - 'types': 'The standard type hierarchy\n' - '***************************\n' - '\n' - 'Below is a list of the types that are built into Python. ' - 'Extension\n' - 'modules (written in C, Java, or other languages, depending on the\n' - 'implementation) can define additional types. Future versions of\n' - 'Python may add types to the type hierarchy (e.g., rational ' - 'numbers,\n' - 'efficiently stored arrays of integers, etc.), although such ' - 'additions\n' - 'will often be provided via the standard library instead.\n' - '\n' - 'Some of the type descriptions below contain a paragraph listing\n' - '‘special attributes.’ These are attributes that provide access to ' - 'the\n' - 'implementation and are not intended for general use. Their ' - 'definition\n' - 'may change in the future.\n' - '\n' - 'None\n' - ' This type has a single value. There is a single object with ' - 'this\n' - ' value. This object is accessed through the built-in name "None". ' - 'It\n' - ' is used to signify the absence of a value in many situations, ' - 'e.g.,\n' - ' it is returned from functions that don’t explicitly return\n' - ' anything. Its truth value is false.\n' - '\n' - 'NotImplemented\n' - ' This type has a single value. There is a single object with ' - 'this\n' - ' value. This object is accessed through the built-in name\n' - ' "NotImplemented". Numeric methods and rich comparison methods\n' - ' should return this value if they do not implement the operation ' - 'for\n' - ' the operands provided. (The interpreter will then try the\n' - ' reflected operation, or some other fallback, depending on the\n' - ' operator.) Its truth value is true.\n' - '\n' - ' See Implementing the arithmetic operations for more details.\n' - '\n' - 'Ellipsis\n' - ' This type has a single value. There is a single object with ' - 'this\n' - ' value. This object is accessed through the literal "..." or the\n' - ' built-in name "Ellipsis". Its truth value is true.\n' - '\n' - '"numbers.Number"\n' - ' These are created by numeric literals and returned as results ' - 'by\n' - ' arithmetic operators and arithmetic built-in functions. ' - 'Numeric\n' - ' objects are immutable; once created their value never changes.\n' - ' Python numbers are of course strongly related to mathematical\n' - ' numbers, but subject to the limitations of numerical ' - 'representation\n' - ' in computers.\n' - '\n' - ' Python distinguishes between integers, floating point numbers, ' - 'and\n' - ' complex numbers:\n' - '\n' - ' "numbers.Integral"\n' - ' These represent elements from the mathematical set of ' - 'integers\n' - ' (positive and negative).\n' - '\n' - ' There are two types of integers:\n' - '\n' - ' Integers ("int")\n' - '\n' - ' These represent numbers in an unlimited range, subject to\n' - ' available (virtual) memory only. For the purpose of ' - 'shift\n' - ' and mask operations, a binary representation is assumed, ' - 'and\n' - ' negative numbers are represented in a variant of 2’s\n' - ' complement which gives the illusion of an infinite string ' - 'of\n' - ' sign bits extending to the left.\n' - '\n' - ' Booleans ("bool")\n' - ' These represent the truth values False and True. The two\n' - ' objects representing the values "False" and "True" are ' - 'the\n' - ' only Boolean objects. The Boolean type is a subtype of ' - 'the\n' - ' integer type, and Boolean values behave like the values 0 ' - 'and\n' - ' 1, respectively, in almost all contexts, the exception ' - 'being\n' - ' that when converted to a string, the strings ""False"" or\n' - ' ""True"" are returned, respectively.\n' - '\n' - ' The rules for integer representation are intended to give ' - 'the\n' - ' most meaningful interpretation of shift and mask operations\n' - ' involving negative integers.\n' - '\n' - ' "numbers.Real" ("float")\n' - ' These represent machine-level double precision floating ' - 'point\n' - ' numbers. You are at the mercy of the underlying machine\n' - ' architecture (and C or Java implementation) for the accepted\n' - ' range and handling of overflow. Python does not support ' - 'single-\n' - ' precision floating point numbers; the savings in processor ' - 'and\n' - ' memory usage that are usually the reason for using these are\n' - ' dwarfed by the overhead of using objects in Python, so there ' - 'is\n' - ' no reason to complicate the language with two kinds of ' - 'floating\n' - ' point numbers.\n' - '\n' - ' "numbers.Complex" ("complex")\n' - ' These represent complex numbers as a pair of machine-level\n' - ' double precision floating point numbers. The same caveats ' - 'apply\n' - ' as for floating point numbers. The real and imaginary parts ' - 'of a\n' - ' complex number "z" can be retrieved through the read-only\n' - ' attributes "z.real" and "z.imag".\n' - '\n' - 'Sequences\n' - ' These represent finite ordered sets indexed by non-negative\n' - ' numbers. The built-in function "len()" returns the number of ' - 'items\n' - ' of a sequence. When the length of a sequence is *n*, the index ' - 'set\n' - ' contains the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* ' - 'is\n' - ' selected by "a[i]".\n' - '\n' - ' Sequences also support slicing: "a[i:j]" selects all items with\n' - ' index *k* such that *i* "<=" *k* "<" *j*. When used as an\n' - ' expression, a slice is a sequence of the same type. This ' - 'implies\n' - ' that the index set is renumbered so that it starts at 0.\n' - '\n' - ' Some sequences also support “extended slicing” with a third ' - '“step”\n' - ' parameter: "a[i:j:k]" selects all items of *a* with index *x* ' - 'where\n' - ' "x = i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n' - '\n' - ' Sequences are distinguished according to their mutability:\n' - '\n' - ' Immutable sequences\n' - ' An object of an immutable sequence type cannot change once it ' - 'is\n' - ' created. (If the object contains references to other ' - 'objects,\n' - ' these other objects may be mutable and may be changed; ' - 'however,\n' - ' the collection of objects directly referenced by an ' - 'immutable\n' - ' object cannot change.)\n' - '\n' - ' The following types are immutable sequences:\n' - '\n' - ' Strings\n' - ' A string is a sequence of values that represent Unicode ' - 'code\n' - ' points. All the code points in the range "U+0000 - ' - 'U+10FFFF"\n' - ' can be represented in a string. Python doesn’t have a ' - '"char"\n' - ' type; instead, every code point in the string is ' - 'represented\n' - ' as a string object with length "1". The built-in ' - 'function\n' - ' "ord()" converts a code point from its string form to an\n' - ' integer in the range "0 - 10FFFF"; "chr()" converts an\n' - ' integer in the range "0 - 10FFFF" to the corresponding ' - 'length\n' - ' "1" string object. "str.encode()" can be used to convert ' - 'a\n' - ' "str" to "bytes" using the given text encoding, and\n' - ' "bytes.decode()" can be used to achieve the opposite.\n' - '\n' - ' Tuples\n' - ' The items of a tuple are arbitrary Python objects. Tuples ' - 'of\n' - ' two or more items are formed by comma-separated lists of\n' - ' expressions. A tuple of one item (a ‘singleton’) can be\n' - ' formed by affixing a comma to an expression (an expression ' - 'by\n' - ' itself does not create a tuple, since parentheses must be\n' - ' usable for grouping of expressions). An empty tuple can ' - 'be\n' - ' formed by an empty pair of parentheses.\n' - '\n' - ' Bytes\n' - ' A bytes object is an immutable array. The items are ' - '8-bit\n' - ' bytes, represented by integers in the range 0 <= x < 256.\n' - ' Bytes literals (like "b\'abc\'") and the built-in ' - '"bytes()"\n' - ' constructor can be used to create bytes objects. Also, ' - 'bytes\n' - ' objects can be decoded to strings via the "decode()" ' - 'method.\n' - '\n' - ' Mutable sequences\n' - ' Mutable sequences can be changed after they are created. ' - 'The\n' - ' subscription and slicing notations can be used as the target ' - 'of\n' - ' assignment and "del" (delete) statements.\n' - '\n' - ' There are currently two intrinsic mutable sequence types:\n' - '\n' - ' Lists\n' - ' The items of a list are arbitrary Python objects. Lists ' - 'are\n' - ' formed by placing a comma-separated list of expressions ' - 'in\n' - ' square brackets. (Note that there are no special cases ' - 'needed\n' - ' to form lists of length 0 or 1.)\n' - '\n' - ' Byte Arrays\n' - ' A bytearray object is a mutable array. They are created ' - 'by\n' - ' the built-in "bytearray()" constructor. Aside from being\n' - ' mutable (and hence unhashable), byte arrays otherwise ' - 'provide\n' - ' the same interface and functionality as immutable "bytes"\n' - ' objects.\n' - '\n' - ' The extension module "array" provides an additional example ' - 'of a\n' - ' mutable sequence type, as does the "collections" module.\n' - '\n' - 'Set types\n' - ' These represent unordered, finite sets of unique, immutable\n' - ' objects. As such, they cannot be indexed by any subscript. ' - 'However,\n' - ' they can be iterated over, and the built-in function "len()"\n' - ' returns the number of items in a set. Common uses for sets are ' - 'fast\n' - ' membership testing, removing duplicates from a sequence, and\n' - ' computing mathematical operations such as intersection, union,\n' - ' difference, and symmetric difference.\n' - '\n' - ' For set elements, the same immutability rules apply as for\n' - ' dictionary keys. Note that numeric types obey the normal rules ' - 'for\n' - ' numeric comparison: if two numbers compare equal (e.g., "1" and\n' - ' "1.0"), only one of them can be contained in a set.\n' - '\n' - ' There are currently two intrinsic set types:\n' - '\n' - ' Sets\n' - ' These represent a mutable set. They are created by the ' - 'built-in\n' - ' "set()" constructor and can be modified afterwards by ' - 'several\n' - ' methods, such as "add()".\n' - '\n' - ' Frozen sets\n' - ' These represent an immutable set. They are created by the\n' - ' built-in "frozenset()" constructor. As a frozenset is ' - 'immutable\n' - ' and *hashable*, it can be used again as an element of ' - 'another\n' - ' set, or as a dictionary key.\n' - '\n' - 'Mappings\n' - ' These represent finite sets of objects indexed by arbitrary ' - 'index\n' - ' sets. The subscript notation "a[k]" selects the item indexed by ' - '"k"\n' - ' from the mapping "a"; this can be used in expressions and as ' - 'the\n' - ' target of assignments or "del" statements. The built-in ' - 'function\n' - ' "len()" returns the number of items in a mapping.\n' - '\n' - ' There is currently a single intrinsic mapping type:\n' - '\n' - ' Dictionaries\n' - ' These represent finite sets of objects indexed by nearly\n' - ' arbitrary values. The only types of values not acceptable ' - 'as\n' - ' keys are values containing lists or dictionaries or other\n' - ' mutable types that are compared by value rather than by ' - 'object\n' - ' identity, the reason being that the efficient implementation ' - 'of\n' - ' dictionaries requires a key’s hash value to remain constant.\n' - ' Numeric types used for keys obey the normal rules for ' - 'numeric\n' - ' comparison: if two numbers compare equal (e.g., "1" and ' - '"1.0")\n' - ' then they can be used interchangeably to index the same\n' - ' dictionary entry.\n' - '\n' - ' Dictionaries are mutable; they can be created by the "{...}"\n' - ' notation (see section Dictionary displays).\n' - '\n' - ' The extension modules "dbm.ndbm" and "dbm.gnu" provide\n' - ' additional examples of mapping types, as does the ' - '"collections"\n' - ' module.\n' - '\n' - 'Callable types\n' - ' These are the types to which the function call operation (see\n' - ' section Calls) can be applied:\n' - '\n' - ' User-defined functions\n' - ' A user-defined function object is created by a function\n' - ' definition (see section Function definitions). It should be\n' - ' called with an argument list containing the same number of ' - 'items\n' - ' as the function’s formal parameter list.\n' - '\n' - ' Special attributes:\n' - '\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | Attribute | Meaning ' - '| |\n' - ' ' - '+===========================+=================================+=============+\n' - ' | "__doc__" | The function’s documentation ' - '| Writable |\n' - ' | | string, or "None" if ' - '| |\n' - ' | | unavailable; not inherited by ' - '| |\n' - ' | | subclasses ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__name__" | The function’s name ' - '| Writable |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__qualname__" | The function’s *qualified name* ' - '| Writable |\n' - ' | | New in version 3.3. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__module__" | The name of the module the ' - '| Writable |\n' - ' | | function was defined in, or ' - '| |\n' - ' | | "None" if unavailable. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__defaults__" | A tuple containing default ' - '| Writable |\n' - ' | | argument values for those ' - '| |\n' - ' | | arguments that have defaults, ' - '| |\n' - ' | | or "None" if no arguments have ' - '| |\n' - ' | | a default value ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__code__" | The code object representing ' - '| Writable |\n' - ' | | the compiled function body. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__globals__" | A reference to the dictionary ' - '| Read-only |\n' - ' | | that holds the function’s ' - '| |\n' - ' | | global variables — the global ' - '| |\n' - ' | | namespace of the module in ' - '| |\n' - ' | | which the function was defined. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__dict__" | The namespace supporting ' - '| Writable |\n' - ' | | arbitrary function attributes. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__closure__" | "None" or a tuple of cells that ' - '| Read-only |\n' - ' | | contain bindings for the ' - '| |\n' - ' | | function’s free variables. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__annotations__" | A dict containing annotations ' - '| Writable |\n' - ' | | of parameters. The keys of the ' - '| |\n' - ' | | dict are the parameter names, ' - '| |\n' - ' | | and "\'return\'" for the ' - 'return | |\n' - ' | | annotation, if provided. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - ' | "__kwdefaults__" | A dict containing defaults for ' - '| Writable |\n' - ' | | keyword-only parameters. ' - '| |\n' - ' ' - '+---------------------------+---------------------------------+-------------+\n' - '\n' - ' Most of the attributes labelled “Writable” check the type of ' - 'the\n' - ' assigned value.\n' - '\n' - ' Function objects also support getting and setting arbitrary\n' - ' attributes, which can be used, for example, to attach ' - 'metadata\n' - ' to functions. Regular attribute dot-notation is used to get ' - 'and\n' - ' set such attributes. *Note that the current implementation ' - 'only\n' - ' supports function attributes on user-defined functions. ' - 'Function\n' - ' attributes on built-in functions may be supported in the\n' - ' future.*\n' - '\n' - ' Additional information about a function’s definition can be\n' - ' retrieved from its code object; see the description of ' - 'internal\n' - ' types below.\n' - '\n' - ' Instance methods\n' - ' An instance method object combines a class, a class instance ' - 'and\n' - ' any callable object (normally a user-defined function).\n' - '\n' - ' Special read-only attributes: "__self__" is the class ' - 'instance\n' - ' object, "__func__" is the function object; "__doc__" is the\n' - ' method’s documentation (same as "__func__.__doc__"); ' - '"__name__"\n' - ' is the method name (same as "__func__.__name__"); ' - '"__module__"\n' - ' is the name of the module the method was defined in, or ' - '"None"\n' - ' if unavailable.\n' - '\n' - ' Methods also support accessing (but not setting) the ' - 'arbitrary\n' - ' function attributes on the underlying function object.\n' - '\n' - ' User-defined method objects may be created when getting an\n' - ' attribute of a class (perhaps via an instance of that class), ' - 'if\n' - ' that attribute is a user-defined function object or a class\n' - ' method object.\n' - '\n' - ' When an instance method object is created by retrieving a ' - 'user-\n' - ' defined function object from a class via one of its ' - 'instances,\n' - ' its "__self__" attribute is the instance, and the method ' - 'object\n' - ' is said to be bound. The new method’s "__func__" attribute ' - 'is\n' - ' the original function object.\n' - '\n' - ' When a user-defined method object is created by retrieving\n' - ' another method object from a class or instance, the behaviour ' - 'is\n' - ' the same as for a function object, except that the ' - '"__func__"\n' - ' attribute of the new instance is not the original method ' - 'object\n' - ' but its "__func__" attribute.\n' - '\n' - ' When an instance method object is created by retrieving a ' - 'class\n' - ' method object from a class or instance, its "__self__" ' - 'attribute\n' - ' is the class itself, and its "__func__" attribute is the\n' - ' function object underlying the class method.\n' - '\n' - ' When an instance method object is called, the underlying\n' - ' function ("__func__") is called, inserting the class ' - 'instance\n' - ' ("__self__") in front of the argument list. For instance, ' - 'when\n' - ' "C" is a class which contains a definition for a function ' - '"f()",\n' - ' and "x" is an instance of "C", calling "x.f(1)" is equivalent ' - 'to\n' - ' calling "C.f(x, 1)".\n' - '\n' - ' When an instance method object is derived from a class ' - 'method\n' - ' object, the “class instance” stored in "__self__" will ' - 'actually\n' - ' be the class itself, so that calling either "x.f(1)" or ' - '"C.f(1)"\n' - ' is equivalent to calling "f(C,1)" where "f" is the ' - 'underlying\n' - ' function.\n' - '\n' - ' Note that the transformation from function object to ' - 'instance\n' - ' method object happens each time the attribute is retrieved ' - 'from\n' - ' the instance. In some cases, a fruitful optimization is to\n' - ' assign the attribute to a local variable and call that local\n' - ' variable. Also notice that this transformation only happens ' - 'for\n' - ' user-defined functions; other callable objects (and all non-\n' - ' callable objects) are retrieved without transformation. It ' - 'is\n' - ' also important to note that user-defined functions which are\n' - ' attributes of a class instance are not converted to bound\n' - ' methods; this *only* happens when the function is an ' - 'attribute\n' - ' of the class.\n' - '\n' - ' Generator functions\n' - ' A function or method which uses the "yield" statement (see\n' - ' section The yield statement) is called a *generator ' - 'function*.\n' - ' Such a function, when called, always returns an iterator ' - 'object\n' - ' which can be used to execute the body of the function: ' - 'calling\n' - ' the iterator’s "iterator.__next__()" method will cause the\n' - ' function to execute until it provides a value using the ' - '"yield"\n' - ' statement. When the function executes a "return" statement ' - 'or\n' - ' falls off the end, a "StopIteration" exception is raised and ' - 'the\n' - ' iterator will have reached the end of the set of values to ' - 'be\n' - ' returned.\n' - '\n' - ' Coroutine functions\n' - ' A function or method which is defined using "async def" is\n' - ' called a *coroutine function*. Such a function, when ' - 'called,\n' - ' returns a *coroutine* object. It may contain "await"\n' - ' expressions, as well as "async with" and "async for" ' - 'statements.\n' - ' See also the Coroutine Objects section.\n' - '\n' - ' Asynchronous generator functions\n' - ' A function or method which is defined using "async def" and\n' - ' which uses the "yield" statement is called a *asynchronous\n' - ' generator function*. Such a function, when called, returns ' - 'an\n' - ' asynchronous iterator object which can be used in an "async ' - 'for"\n' - ' statement to execute the body of the function.\n' - '\n' - ' Calling the asynchronous iterator’s "aiterator.__anext__()"\n' - ' method will return an *awaitable* which when awaited will\n' - ' execute until it provides a value using the "yield" ' - 'expression.\n' - ' When the function executes an empty "return" statement or ' - 'falls\n' - ' off the end, a "StopAsyncIteration" exception is raised and ' - 'the\n' - ' asynchronous iterator will have reached the end of the set ' - 'of\n' - ' values to be yielded.\n' - '\n' - ' Built-in functions\n' - ' A built-in function object is a wrapper around a C function.\n' - ' Examples of built-in functions are "len()" and "math.sin()"\n' - ' ("math" is a standard built-in module). The number and type ' - 'of\n' - ' the arguments are determined by the C function. Special ' - 'read-\n' - ' only attributes: "__doc__" is the function’s documentation\n' - ' string, or "None" if unavailable; "__name__" is the ' - 'function’s\n' - ' name; "__self__" is set to "None" (but see the next item);\n' - ' "__module__" is the name of the module the function was ' - 'defined\n' - ' in or "None" if unavailable.\n' - '\n' - ' Built-in methods\n' - ' This is really a different disguise of a built-in function, ' - 'this\n' - ' time containing an object passed to the C function as an\n' - ' implicit extra argument. An example of a built-in method is\n' - ' "alist.append()", assuming *alist* is a list object. In this\n' - ' case, the special read-only attribute "__self__" is set to ' - 'the\n' - ' object denoted by *alist*.\n' - '\n' - ' Classes\n' - ' Classes are callable. These objects normally act as ' - 'factories\n' - ' for new instances of themselves, but variations are possible ' - 'for\n' - ' class types that override "__new__()". The arguments of the\n' - ' call are passed to "__new__()" and, in the typical case, to\n' - ' "__init__()" to initialize the new instance.\n' - '\n' - ' Class Instances\n' - ' Instances of arbitrary classes can be made callable by ' - 'defining\n' - ' a "__call__()" method in their class.\n' - '\n' - 'Modules\n' - ' Modules are a basic organizational unit of Python code, and are\n' - ' created by the import system as invoked either by the "import"\n' - ' statement (see "import"), or by calling functions such as\n' - ' "importlib.import_module()" and built-in "__import__()". A ' - 'module\n' - ' object has a namespace implemented by a dictionary object (this ' - 'is\n' - ' the dictionary referenced by the "__globals__" attribute of\n' - ' functions defined in the module). Attribute references are\n' - ' translated to lookups in this dictionary, e.g., "m.x" is ' - 'equivalent\n' - ' to "m.__dict__["x"]". A module object does not contain the code\n' - ' object used to initialize the module (since it isn’t needed ' - 'once\n' - ' the initialization is done).\n' - '\n' - ' Attribute assignment updates the module’s namespace dictionary,\n' - ' e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n' - '\n' - ' Predefined (writable) attributes: "__name__" is the module’s ' - 'name;\n' - ' "__doc__" is the module’s documentation string, or "None" if\n' - ' unavailable; "__annotations__" (optional) is a dictionary\n' - ' containing *variable annotations* collected during module body\n' - ' execution; "__file__" is the pathname of the file from which ' - 'the\n' - ' module was loaded, if it was loaded from a file. The "__file__"\n' - ' attribute may be missing for certain types of modules, such as ' - 'C\n' - ' modules that are statically linked into the interpreter; for\n' - ' extension modules loaded dynamically from a shared library, it ' - 'is\n' - ' the pathname of the shared library file.\n' - '\n' - ' Special read-only attribute: "__dict__" is the module’s ' - 'namespace\n' - ' as a dictionary object.\n' - '\n' - ' **CPython implementation detail:** Because of the way CPython\n' - ' clears module dictionaries, the module dictionary will be ' - 'cleared\n' - ' when the module falls out of scope even if the dictionary still ' - 'has\n' - ' live references. To avoid this, copy the dictionary or keep ' - 'the\n' - ' module around while using its dictionary directly.\n' - '\n' - 'Custom classes\n' - ' Custom class types are typically created by class definitions ' - '(see\n' - ' section Class definitions). A class has a namespace implemented ' - 'by\n' - ' a dictionary object. Class attribute references are translated ' - 'to\n' - ' lookups in this dictionary, e.g., "C.x" is translated to\n' - ' "C.__dict__["x"]" (although there are a number of hooks which ' - 'allow\n' - ' for other means of locating attributes). When the attribute name ' - 'is\n' - ' not found there, the attribute search continues in the base\n' - ' classes. This search of the base classes uses the C3 method\n' - ' resolution order which behaves correctly even in the presence ' - 'of\n' - ' ‘diamond’ inheritance structures where there are multiple\n' - ' inheritance paths leading back to a common ancestor. Additional\n' - ' details on the C3 MRO used by Python can be found in the\n' - ' documentation accompanying the 2.3 release at\n' - ' https://www.python.org/download/releases/2.3/mro/.\n' - '\n' - ' When a class attribute reference (for class "C", say) would ' - 'yield a\n' - ' class method object, it is transformed into an instance method\n' - ' object whose "__self__" attribute is "C". When it would yield ' - 'a\n' - ' static method object, it is transformed into the object wrapped ' - 'by\n' - ' the static method object. See section Implementing Descriptors ' - 'for\n' - ' another way in which attributes retrieved from a class may ' - 'differ\n' - ' from those actually contained in its "__dict__".\n' - '\n' - ' Class attribute assignments update the class’s dictionary, ' - 'never\n' - ' the dictionary of a base class.\n' - '\n' - ' A class object can be called (see above) to yield a class ' - 'instance\n' - ' (see below).\n' - '\n' - ' Special attributes: "__name__" is the class name; "__module__" ' - 'is\n' - ' the module name in which the class was defined; "__dict__" is ' - 'the\n' - ' dictionary containing the class’s namespace; "__bases__" is a ' - 'tuple\n' - ' containing the base classes, in the order of their occurrence ' - 'in\n' - ' the base class list; "__doc__" is the class’s documentation ' - 'string,\n' - ' or "None" if undefined; "__annotations__" (optional) is a\n' - ' dictionary containing *variable annotations* collected during ' - 'class\n' - ' body execution.\n' - '\n' - 'Class instances\n' - ' A class instance is created by calling a class object (see ' - 'above).\n' - ' A class instance has a namespace implemented as a dictionary ' - 'which\n' - ' is the first place in which attribute references are searched.\n' - ' When an attribute is not found there, and the instance’s class ' - 'has\n' - ' an attribute by that name, the search continues with the class\n' - ' attributes. If a class attribute is found that is a ' - 'user-defined\n' - ' function object, it is transformed into an instance method ' - 'object\n' - ' whose "__self__" attribute is the instance. Static method and\n' - ' class method objects are also transformed; see above under\n' - ' “Classes”. See section Implementing Descriptors for another way ' - 'in\n' - ' which attributes of a class retrieved via its instances may ' - 'differ\n' - ' from the objects actually stored in the class’s "__dict__". If ' - 'no\n' - ' class attribute is found, and the object’s class has a\n' - ' "__getattr__()" method, that is called to satisfy the lookup.\n' - '\n' - ' Attribute assignments and deletions update the instance’s\n' - ' dictionary, never a class’s dictionary. If the class has a\n' - ' "__setattr__()" or "__delattr__()" method, this is called ' - 'instead\n' - ' of updating the instance dictionary directly.\n' - '\n' - ' Class instances can pretend to be numbers, sequences, or ' - 'mappings\n' - ' if they have methods with certain special names. See section\n' - ' Special method names.\n' - '\n' - ' Special attributes: "__dict__" is the attribute dictionary;\n' - ' "__class__" is the instance’s class.\n' - '\n' - 'I/O objects (also known as file objects)\n' - ' A *file object* represents an open file. Various shortcuts are\n' - ' available to create file objects: the "open()" built-in ' - 'function,\n' - ' and also "os.popen()", "os.fdopen()", and the "makefile()" ' - 'method\n' - ' of socket objects (and perhaps by other functions or methods\n' - ' provided by extension modules).\n' - '\n' - ' The objects "sys.stdin", "sys.stdout" and "sys.stderr" are\n' - ' initialized to file objects corresponding to the interpreter’s\n' - ' standard input, output and error streams; they are all open in ' - 'text\n' - ' mode and therefore follow the interface defined by the\n' - ' "io.TextIOBase" abstract class.\n' - '\n' - 'Internal types\n' - ' A few types used internally by the interpreter are exposed to ' - 'the\n' - ' user. Their definitions may change with future versions of the\n' - ' interpreter, but they are mentioned here for completeness.\n' - '\n' - ' Code objects\n' - ' Code objects represent *byte-compiled* executable Python ' - 'code,\n' - ' or *bytecode*. The difference between a code object and a\n' - ' function object is that the function object contains an ' - 'explicit\n' - ' reference to the function’s globals (the module in which it ' - 'was\n' - ' defined), while a code object contains no context; also the\n' - ' default argument values are stored in the function object, ' - 'not\n' - ' in the code object (because they represent values calculated ' - 'at\n' - ' run-time). Unlike function objects, code objects are ' - 'immutable\n' - ' and contain no references (directly or indirectly) to ' - 'mutable\n' - ' objects.\n' - '\n' - ' Special read-only attributes: "co_name" gives the function ' - 'name;\n' - ' "co_argcount" is the number of positional arguments ' - '(including\n' - ' arguments with default values); "co_nlocals" is the number ' - 'of\n' - ' local variables used by the function (including arguments);\n' - ' "co_varnames" is a tuple containing the names of the local\n' - ' variables (starting with the argument names); "co_cellvars" ' - 'is a\n' - ' tuple containing the names of local variables that are\n' - ' referenced by nested functions; "co_freevars" is a tuple\n' - ' containing the names of free variables; "co_code" is a ' - 'string\n' - ' representing the sequence of bytecode instructions; ' - '"co_consts"\n' - ' is a tuple containing the literals used by the bytecode;\n' - ' "co_names" is a tuple containing the names used by the ' - 'bytecode;\n' - ' "co_filename" is the filename from which the code was ' - 'compiled;\n' - ' "co_firstlineno" is the first line number of the function;\n' - ' "co_lnotab" is a string encoding the mapping from bytecode\n' - ' offsets to line numbers (for details see the source code of ' - 'the\n' - ' interpreter); "co_stacksize" is the required stack size\n' - ' (including local variables); "co_flags" is an integer ' - 'encoding a\n' - ' number of flags for the interpreter.\n' - '\n' - ' The following flag bits are defined for "co_flags": bit ' - '"0x04"\n' - ' is set if the function uses the "*arguments" syntax to accept ' - 'an\n' - ' arbitrary number of positional arguments; bit "0x08" is set ' - 'if\n' - ' the function uses the "**keywords" syntax to accept ' - 'arbitrary\n' - ' keyword arguments; bit "0x20" is set if the function is a\n' - ' generator.\n' - '\n' - ' Future feature declarations ("from __future__ import ' - 'division")\n' - ' also use bits in "co_flags" to indicate whether a code ' - 'object\n' - ' was compiled with a particular feature enabled: bit "0x2000" ' - 'is\n' - ' set if the function was compiled with future division ' - 'enabled;\n' - ' bits "0x10" and "0x1000" were used in earlier versions of\n' - ' Python.\n' - '\n' - ' Other bits in "co_flags" are reserved for internal use.\n' - '\n' - ' If a code object represents a function, the first item in\n' - ' "co_consts" is the documentation string of the function, or\n' - ' "None" if undefined.\n' - '\n' - ' Frame objects\n' - ' Frame objects represent execution frames. They may occur in\n' - ' traceback objects (see below).\n' - '\n' - ' Special read-only attributes: "f_back" is to the previous ' - 'stack\n' - ' frame (towards the caller), or "None" if this is the bottom\n' - ' stack frame; "f_code" is the code object being executed in ' - 'this\n' - ' frame; "f_locals" is the dictionary used to look up local\n' - ' variables; "f_globals" is used for global variables;\n' - ' "f_builtins" is used for built-in (intrinsic) names; ' - '"f_lasti"\n' - ' gives the precise instruction (this is an index into the\n' - ' bytecode string of the code object).\n' - '\n' - ' Special writable attributes: "f_trace", if not "None", is a\n' - ' function called at the start of each source code line (this ' - 'is\n' - ' used by the debugger); "f_lineno" is the current line number ' - 'of\n' - ' the frame — writing to this from within a trace function ' - 'jumps\n' - ' to the given line (only for the bottom-most frame). A ' - 'debugger\n' - ' can implement a Jump command (aka Set Next Statement) by ' - 'writing\n' - ' to f_lineno.\n' - '\n' - ' Frame objects support one method:\n' - '\n' - ' frame.clear()\n' - '\n' - ' This method clears all references to local variables held ' - 'by\n' - ' the frame. Also, if the frame belonged to a generator, ' - 'the\n' - ' generator is finalized. This helps break reference ' - 'cycles\n' - ' involving frame objects (for example when catching an\n' - ' exception and storing its traceback for later use).\n' - '\n' - ' "RuntimeError" is raised if the frame is currently ' - 'executing.\n' - '\n' - ' New in version 3.4.\n' - '\n' - ' Traceback objects\n' - ' Traceback objects represent a stack trace of an exception. ' - 'A\n' - ' traceback object is created when an exception occurs. When ' - 'the\n' - ' search for an exception handler unwinds the execution stack, ' - 'at\n' - ' each unwound level a traceback object is inserted in front ' - 'of\n' - ' the current traceback. When an exception handler is ' - 'entered,\n' - ' the stack trace is made available to the program. (See ' - 'section\n' - ' The try statement.) It is accessible as the third item of ' - 'the\n' - ' tuple returned by "sys.exc_info()". When the program contains ' - 'no\n' - ' suitable handler, the stack trace is written (nicely ' - 'formatted)\n' - ' to the standard error stream; if the interpreter is ' - 'interactive,\n' - ' it is also made available to the user as ' - '"sys.last_traceback".\n' - '\n' - ' Special read-only attributes: "tb_next" is the next level in ' - 'the\n' - ' stack trace (towards the frame where the exception occurred), ' - 'or\n' - ' "None" if there is no next level; "tb_frame" points to the\n' - ' execution frame of the current level; "tb_lineno" gives the ' - 'line\n' - ' number where the exception occurred; "tb_lasti" indicates ' - 'the\n' - ' precise instruction. The line number and last instruction ' - 'in\n' - ' the traceback may differ from the line number of its frame\n' - ' object if the exception occurred in a "try" statement with ' - 'no\n' - ' matching except clause or with a finally clause.\n' - '\n' - ' Slice objects\n' - ' Slice objects are used to represent slices for ' - '"__getitem__()"\n' - ' methods. They are also created by the built-in "slice()"\n' - ' function.\n' - '\n' - ' Special read-only attributes: "start" is the lower bound; ' - '"stop"\n' - ' is the upper bound; "step" is the step value; each is "None" ' - 'if\n' - ' omitted. These attributes can have any type.\n' - '\n' - ' Slice objects support one method:\n' - '\n' - ' slice.indices(self, length)\n' - '\n' - ' This method takes a single integer argument *length* and\n' - ' computes information about the slice that the slice ' - 'object\n' - ' would describe if applied to a sequence of *length* ' - 'items.\n' - ' It returns a tuple of three integers; respectively these ' - 'are\n' - ' the *start* and *stop* indices and the *step* or stride\n' - ' length of the slice. Missing or out-of-bounds indices are\n' - ' handled in a manner consistent with regular slices.\n' - '\n' - ' Static method objects\n' - ' Static method objects provide a way of defeating the\n' - ' transformation of function objects to method objects ' - 'described\n' - ' above. A static method object is a wrapper around any other\n' - ' object, usually a user-defined method object. When a static\n' - ' method object is retrieved from a class or a class instance, ' - 'the\n' - ' object actually returned is the wrapped object, which is not\n' - ' subject to any further transformation. Static method objects ' - 'are\n' - ' not themselves callable, although the objects they wrap ' - 'usually\n' - ' are. Static method objects are created by the built-in\n' - ' "staticmethod()" constructor.\n' - '\n' - ' Class method objects\n' - ' A class method object, like a static method object, is a ' - 'wrapper\n' - ' around another object that alters the way in which that ' - 'object\n' - ' is retrieved from classes and class instances. The behaviour ' - 'of\n' - ' class method objects upon such retrieval is described above,\n' - ' under “User-defined methods”. Class method objects are ' - 'created\n' - ' by the built-in "classmethod()" constructor.\n', - 'typesfunctions': 'Functions\n' - '*********\n' - '\n' - 'Function objects are created by function definitions. The ' - 'only\n' - 'operation on a function object is to call it: ' - '"func(argument-list)".\n' - '\n' - 'There are really two flavors of function objects: built-in ' - 'functions\n' - 'and user-defined functions. Both support the same ' - 'operation (to call\n' - 'the function), but the implementation is different, hence ' - 'the\n' - 'different object types.\n' - '\n' - 'See Function definitions for more information.\n', - 'typesmapping': 'Mapping Types — "dict"\n' - '**********************\n' - '\n' - 'A *mapping* object maps *hashable* values to arbitrary ' - 'objects.\n' - 'Mappings are mutable objects. There is currently only one ' - 'standard\n' - 'mapping type, the *dictionary*. (For other containers see ' - 'the built-\n' - 'in "list", "set", and "tuple" classes, and the "collections" ' - 'module.)\n' - '\n' - 'A dictionary’s keys are *almost* arbitrary values. Values ' - 'that are\n' - 'not *hashable*, that is, values containing lists, ' - 'dictionaries or\n' - 'other mutable types (that are compared by value rather than ' - 'by object\n' - 'identity) may not be used as keys. Numeric types used for ' - 'keys obey\n' - 'the normal rules for numeric comparison: if two numbers ' - 'compare equal\n' - '(such as "1" and "1.0") then they can be used ' - 'interchangeably to index\n' - 'the same dictionary entry. (Note however, that since ' - 'computers store\n' - 'floating-point numbers as approximations it is usually ' - 'unwise to use\n' - 'them as dictionary keys.)\n' - '\n' - 'Dictionaries can be created by placing a comma-separated ' - 'list of "key:\n' - 'value" pairs within braces, for example: "{\'jack\': 4098, ' - "'sjoerd':\n" - '4127}" or "{4098: \'jack\', 4127: \'sjoerd\'}", or by the ' - '"dict"\n' - 'constructor.\n' - '\n' - 'class dict(**kwarg)\n' - 'class dict(mapping, **kwarg)\n' - 'class dict(iterable, **kwarg)\n' - '\n' - ' Return a new dictionary initialized from an optional ' - 'positional\n' - ' argument and a possibly empty set of keyword arguments.\n' - '\n' - ' If no positional argument is given, an empty dictionary ' - 'is created.\n' - ' If a positional argument is given and it is a mapping ' - 'object, a\n' - ' dictionary is created with the same key-value pairs as ' - 'the mapping\n' - ' object. Otherwise, the positional argument must be an ' - '*iterable*\n' - ' object. Each item in the iterable must itself be an ' - 'iterable with\n' - ' exactly two objects. The first object of each item ' - 'becomes a key\n' - ' in the new dictionary, and the second object the ' - 'corresponding\n' - ' value. If a key occurs more than once, the last value ' - 'for that key\n' - ' becomes the corresponding value in the new dictionary.\n' - '\n' - ' If keyword arguments are given, the keyword arguments and ' - 'their\n' - ' values are added to the dictionary created from the ' - 'positional\n' - ' argument. If a key being added is already present, the ' - 'value from\n' - ' the keyword argument replaces the value from the ' - 'positional\n' - ' argument.\n' - '\n' - ' To illustrate, the following examples all return a ' - 'dictionary equal\n' - ' to "{"one": 1, "two": 2, "three": 3}":\n' - '\n' - ' >>> a = dict(one=1, two=2, three=3)\n' - " >>> b = {'one': 1, 'two': 2, 'three': 3}\n" - " >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3]))\n" - " >>> d = dict([('two', 2), ('one', 1), ('three', 3)])\n" - " >>> e = dict({'three': 3, 'one': 1, 'two': 2})\n" - ' >>> a == b == c == d == e\n' - ' True\n' - '\n' - ' Providing keyword arguments as in the first example only ' - 'works for\n' - ' keys that are valid Python identifiers. Otherwise, any ' - 'valid keys\n' - ' can be used.\n' - '\n' - ' These are the operations that dictionaries support (and ' - 'therefore,\n' - ' custom mapping types should support too):\n' - '\n' - ' len(d)\n' - '\n' - ' Return the number of items in the dictionary *d*.\n' - '\n' - ' d[key]\n' - '\n' - ' Return the item of *d* with key *key*. Raises a ' - '"KeyError" if\n' - ' *key* is not in the map.\n' - '\n' - ' If a subclass of dict defines a method "__missing__()" ' - 'and *key*\n' - ' is not present, the "d[key]" operation calls that ' - 'method with\n' - ' the key *key* as argument. The "d[key]" operation ' - 'then returns\n' - ' or raises whatever is returned or raised by the\n' - ' "__missing__(key)" call. No other operations or ' - 'methods invoke\n' - ' "__missing__()". If "__missing__()" is not defined, ' - '"KeyError"\n' - ' is raised. "__missing__()" must be a method; it cannot ' - 'be an\n' - ' instance variable:\n' - '\n' - ' >>> class Counter(dict):\n' - ' ... def __missing__(self, key):\n' - ' ... return 0\n' - ' >>> c = Counter()\n' - " >>> c['red']\n" - ' 0\n' - " >>> c['red'] += 1\n" - " >>> c['red']\n" - ' 1\n' - '\n' - ' The example above shows part of the implementation of\n' - ' "collections.Counter". A different "__missing__" ' - 'method is used\n' - ' by "collections.defaultdict".\n' - '\n' - ' d[key] = value\n' - '\n' - ' Set "d[key]" to *value*.\n' - '\n' - ' del d[key]\n' - '\n' - ' Remove "d[key]" from *d*. Raises a "KeyError" if ' - '*key* is not\n' - ' in the map.\n' - '\n' - ' key in d\n' - '\n' - ' Return "True" if *d* has a key *key*, else "False".\n' - '\n' - ' key not in d\n' - '\n' - ' Equivalent to "not key in d".\n' - '\n' - ' iter(d)\n' - '\n' - ' Return an iterator over the keys of the dictionary. ' - 'This is a\n' - ' shortcut for "iter(d.keys())".\n' - '\n' - ' clear()\n' - '\n' - ' Remove all items from the dictionary.\n' - '\n' - ' copy()\n' - '\n' - ' Return a shallow copy of the dictionary.\n' - '\n' - ' classmethod fromkeys(seq[, value])\n' - '\n' - ' Create a new dictionary with keys from *seq* and ' - 'values set to\n' - ' *value*.\n' - '\n' - ' "fromkeys()" is a class method that returns a new ' - 'dictionary.\n' - ' *value* defaults to "None".\n' - '\n' - ' get(key[, default])\n' - '\n' - ' Return the value for *key* if *key* is in the ' - 'dictionary, else\n' - ' *default*. If *default* is not given, it defaults to ' - '"None", so\n' - ' that this method never raises a "KeyError".\n' - '\n' - ' items()\n' - '\n' - ' Return a new view of the dictionary’s items ("(key, ' - 'value)"\n' - ' pairs). See the documentation of view objects.\n' - '\n' - ' keys()\n' - '\n' - ' Return a new view of the dictionary’s keys. See the\n' - ' documentation of view objects.\n' - '\n' - ' pop(key[, default])\n' - '\n' - ' If *key* is in the dictionary, remove it and return ' - 'its value,\n' - ' else return *default*. If *default* is not given and ' - '*key* is\n' - ' not in the dictionary, a "KeyError" is raised.\n' - '\n' - ' popitem()\n' - '\n' - ' Remove and return an arbitrary "(key, value)" pair ' - 'from the\n' - ' dictionary.\n' - '\n' - ' "popitem()" is useful to destructively iterate over a\n' - ' dictionary, as often used in set algorithms. If the ' - 'dictionary\n' - ' is empty, calling "popitem()" raises a "KeyError".\n' - '\n' - ' setdefault(key[, default])\n' - '\n' - ' If *key* is in the dictionary, return its value. If ' - 'not, insert\n' - ' *key* with a value of *default* and return *default*. ' - '*default*\n' - ' defaults to "None".\n' - '\n' - ' update([other])\n' - '\n' - ' Update the dictionary with the key/value pairs from ' - '*other*,\n' - ' overwriting existing keys. Return "None".\n' - '\n' - ' "update()" accepts either another dictionary object or ' - 'an\n' - ' iterable of key/value pairs (as tuples or other ' - 'iterables of\n' - ' length two). If keyword arguments are specified, the ' - 'dictionary\n' - ' is then updated with those key/value pairs: ' - '"d.update(red=1,\n' - ' blue=2)".\n' - '\n' - ' values()\n' - '\n' - ' Return a new view of the dictionary’s values. See ' - 'the\n' - ' documentation of view objects.\n' - '\n' - ' Dictionaries compare equal if and only if they have the ' - 'same "(key,\n' - ' value)" pairs. Order comparisons (‘<’, ‘<=’, ‘>=’, ‘>’) ' - 'raise\n' - ' "TypeError".\n' - '\n' - 'See also: "types.MappingProxyType" can be used to create a ' - 'read-only\n' - ' view of a "dict".\n' - '\n' - '\n' - 'Dictionary view objects\n' - '=======================\n' - '\n' - 'The objects returned by "dict.keys()", "dict.values()" and\n' - '"dict.items()" are *view objects*. They provide a dynamic ' - 'view on the\n' - 'dictionary’s entries, which means that when the dictionary ' - 'changes,\n' - 'the view reflects these changes.\n' - '\n' - 'Dictionary views can be iterated over to yield their ' - 'respective data,\n' - 'and support membership tests:\n' - '\n' - 'len(dictview)\n' - '\n' - ' Return the number of entries in the dictionary.\n' - '\n' - 'iter(dictview)\n' - '\n' - ' Return an iterator over the keys, values or items ' - '(represented as\n' - ' tuples of "(key, value)") in the dictionary.\n' - '\n' - ' Keys and values are iterated over in an arbitrary order ' - 'which is\n' - ' non-random, varies across Python implementations, and ' - 'depends on\n' - ' the dictionary’s history of insertions and deletions. If ' - 'keys,\n' - ' values and items views are iterated over with no ' - 'intervening\n' - ' modifications to the dictionary, the order of items will ' - 'directly\n' - ' correspond. This allows the creation of "(value, key)" ' - 'pairs using\n' - ' "zip()": "pairs = zip(d.values(), d.keys())". Another ' - 'way to\n' - ' create the same list is "pairs = [(v, k) for (k, v) in ' - 'd.items()]".\n' - '\n' - ' Iterating views while adding or deleting entries in the ' - 'dictionary\n' - ' may raise a "RuntimeError" or fail to iterate over all ' - 'entries.\n' - '\n' - 'x in dictview\n' - '\n' - ' Return "True" if *x* is in the underlying dictionary’s ' - 'keys, values\n' - ' or items (in the latter case, *x* should be a "(key, ' - 'value)"\n' - ' tuple).\n' - '\n' - 'Keys views are set-like since their entries are unique and ' - 'hashable.\n' - 'If all values are hashable, so that "(key, value)" pairs are ' - 'unique\n' - 'and hashable, then the items view is also set-like. (Values ' - 'views are\n' - 'not treated as set-like since the entries are generally not ' - 'unique.)\n' - 'For set-like views, all of the operations defined for the ' - 'abstract\n' - 'base class "collections.abc.Set" are available (for example, ' - '"==",\n' - '"<", or "^").\n' - '\n' - 'An example of dictionary view usage:\n' - '\n' - " >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, " - "'spam': 500}\n" - ' >>> keys = dishes.keys()\n' - ' >>> values = dishes.values()\n' - '\n' - ' >>> # iteration\n' - ' >>> n = 0\n' - ' >>> for val in values:\n' - ' ... n += val\n' - ' >>> print(n)\n' - ' 504\n' - '\n' - ' >>> # keys and values are iterated over in the same ' - 'order\n' - ' >>> list(keys)\n' - " ['eggs', 'bacon', 'sausage', 'spam']\n" - ' >>> list(values)\n' - ' [2, 1, 1, 500]\n' - '\n' - ' >>> # view objects are dynamic and reflect dict changes\n' - " >>> del dishes['eggs']\n" - " >>> del dishes['sausage']\n" - ' >>> list(keys)\n' - " ['spam', 'bacon']\n" - '\n' - ' >>> # set operations\n' - " >>> keys & {'eggs', 'bacon', 'salad'}\n" - " {'bacon'}\n" - " >>> keys ^ {'sausage', 'juice'}\n" - " {'juice', 'sausage', 'bacon', 'spam'}\n", - 'typesmethods': 'Methods\n' - '*******\n' - '\n' - 'Methods are functions that are called using the attribute ' - 'notation.\n' - 'There are two flavors: built-in methods (such as "append()" ' - 'on lists)\n' - 'and class instance methods. Built-in methods are described ' - 'with the\n' - 'types that support them.\n' - '\n' - 'If you access a method (a function defined in a class ' - 'namespace)\n' - 'through an instance, you get a special object: a *bound ' - 'method* (also\n' - 'called *instance method*) object. When called, it will add ' - 'the "self"\n' - 'argument to the argument list. Bound methods have two ' - 'special read-\n' - 'only attributes: "m.__self__" is the object on which the ' - 'method\n' - 'operates, and "m.__func__" is the function implementing the ' - 'method.\n' - 'Calling "m(arg-1, arg-2, ..., arg-n)" is completely ' - 'equivalent to\n' - 'calling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)".\n' - '\n' - 'Like function objects, bound method objects support getting ' - 'arbitrary\n' - 'attributes. However, since method attributes are actually ' - 'stored on\n' - 'the underlying function object ("meth.__func__"), setting ' - 'method\n' - 'attributes on bound methods is disallowed. Attempting to ' - 'set an\n' - 'attribute on a method results in an "AttributeError" being ' - 'raised. In\n' - 'order to set a method attribute, you need to explicitly set ' - 'it on the\n' - 'underlying function object:\n' - '\n' - ' >>> class C:\n' - ' ... def method(self):\n' - ' ... pass\n' - ' ...\n' - ' >>> c = C()\n' - " >>> c.method.whoami = 'my name is method' # can't set on " - 'the method\n' - ' Traceback (most recent call last):\n' - ' File "", line 1, in \n' - " AttributeError: 'method' object has no attribute " - "'whoami'\n" - " >>> c.method.__func__.whoami = 'my name is method'\n" - ' >>> c.method.whoami\n' - " 'my name is method'\n" - '\n' - 'See The standard type hierarchy for more information.\n', - 'typesmodules': 'Modules\n' - '*******\n' - '\n' - 'The only special operation on a module is attribute access: ' - '"m.name",\n' - 'where *m* is a module and *name* accesses a name defined in ' - '*m*’s\n' - 'symbol table. Module attributes can be assigned to. (Note ' - 'that the\n' - '"import" statement is not, strictly speaking, an operation ' - 'on a module\n' - 'object; "import foo" does not require a module object named ' - '*foo* to\n' - 'exist, rather it requires an (external) *definition* for a ' - 'module\n' - 'named *foo* somewhere.)\n' - '\n' - 'A special attribute of every module is "__dict__". This is ' - 'the\n' - 'dictionary containing the module’s symbol table. Modifying ' - 'this\n' - 'dictionary will actually change the module’s symbol table, ' - 'but direct\n' - 'assignment to the "__dict__" attribute is not possible (you ' - 'can write\n' - '"m.__dict__[\'a\'] = 1", which defines "m.a" to be "1", but ' - 'you can’t\n' - 'write "m.__dict__ = {}"). Modifying "__dict__" directly is ' - 'not\n' - 'recommended.\n' - '\n' - 'Modules built into the interpreter are written like this: ' - '"". If loaded from a file, they are ' - 'written as\n' - '"".\n', - 'typesseq': 'Sequence Types — "list", "tuple", "range"\n' - '*****************************************\n' - '\n' - 'There are three basic sequence types: lists, tuples, and range\n' - 'objects. Additional sequence types tailored for processing of ' - 'binary\n' - 'data and text strings are described in dedicated sections.\n' - '\n' - '\n' - 'Common Sequence Operations\n' - '==========================\n' - '\n' - 'The operations in the following table are supported by most ' - 'sequence\n' - 'types, both mutable and immutable. The ' - '"collections.abc.Sequence" ABC\n' - 'is provided to make it easier to correctly implement these ' - 'operations\n' - 'on custom sequence types.\n' - '\n' - 'This table lists the sequence operations sorted in ascending ' - 'priority.\n' - 'In the table, *s* and *t* are sequences of the same type, *n*, ' - '*i*,\n' - '*j* and *k* are integers and *x* is an arbitrary object that ' - 'meets any\n' - 'type and value restrictions imposed by *s*.\n' - '\n' - 'The "in" and "not in" operations have the same priorities as ' - 'the\n' - 'comparison operations. The "+" (concatenation) and "*" ' - '(repetition)\n' - 'operations have the same priority as the corresponding numeric\n' - 'operations. [3]\n' - '\n' - '+----------------------------+----------------------------------+------------+\n' - '| Operation | Result ' - '| Notes |\n' - '+============================+==================================+============+\n' - '| "x in s" | "True" if an item of *s* is ' - '| (1) |\n' - '| | equal to *x*, else "False" ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "x not in s" | "False" if an item of *s* is ' - '| (1) |\n' - '| | equal to *x*, else "True" ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s + t" | the concatenation of *s* and *t* ' - '| (6)(7) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s * n" or "n * s" | equivalent to adding *s* to ' - '| (2)(7) |\n' - '| | itself *n* times ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i]" | *i*th item of *s*, origin 0 ' - '| (3) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i:j]" | slice of *s* from *i* to *j* ' - '| (3)(4) |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s[i:j:k]" | slice of *s* from *i* to *j* ' - '| (3)(5) |\n' - '| | with step *k* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "len(s)" | length of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "min(s)" | smallest item of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "max(s)" | largest item of *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s.index(x[, i[, j]])" | index of the first occurrence of ' - '| (8) |\n' - '| | *x* in *s* (at or after index ' - '| |\n' - '| | *i* and before index *j*) ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '| "s.count(x)" | total number of occurrences of ' - '| |\n' - '| | *x* in *s* ' - '| |\n' - '+----------------------------+----------------------------------+------------+\n' - '\n' - 'Sequences of the same type also support comparisons. In ' - 'particular,\n' - 'tuples and lists are compared lexicographically by comparing\n' - 'corresponding elements. This means that to compare equal, every\n' - 'element must compare equal and the two sequences must be of the ' - 'same\n' - 'type and have the same length. (For full details see ' - 'Comparisons in\n' - 'the language reference.)\n' - '\n' - 'Notes:\n' - '\n' - '1. While the "in" and "not in" operations are used only for ' - 'simple\n' - ' containment testing in the general case, some specialised ' - 'sequences\n' - ' (such as "str", "bytes" and "bytearray") also use them for\n' - ' subsequence testing:\n' - '\n' - ' >>> "gg" in "eggs"\n' - ' True\n' - '\n' - '2. Values of *n* less than "0" are treated as "0" (which yields ' - 'an\n' - ' empty sequence of the same type as *s*). Note that items in ' - 'the\n' - ' sequence *s* are not copied; they are referenced multiple ' - 'times.\n' - ' This often haunts new Python programmers; consider:\n' - '\n' - ' >>> lists = [[]] * 3\n' - ' >>> lists\n' - ' [[], [], []]\n' - ' >>> lists[0].append(3)\n' - ' >>> lists\n' - ' [[3], [3], [3]]\n' - '\n' - ' What has happened is that "[[]]" is a one-element list ' - 'containing\n' - ' an empty list, so all three elements of "[[]] * 3" are ' - 'references\n' - ' to this single empty list. Modifying any of the elements of\n' - ' "lists" modifies this single list. You can create a list of\n' - ' different lists this way:\n' - '\n' - ' >>> lists = [[] for i in range(3)]\n' - ' >>> lists[0].append(3)\n' - ' >>> lists[1].append(5)\n' - ' >>> lists[2].append(7)\n' - ' >>> lists\n' - ' [[3], [5], [7]]\n' - '\n' - ' Further explanation is available in the FAQ entry How do I ' - 'create a\n' - ' multidimensional list?.\n' - '\n' - '3. If *i* or *j* is negative, the index is relative to the end ' - 'of\n' - ' sequence *s*: "len(s) + i" or "len(s) + j" is substituted. ' - 'But\n' - ' note that "-0" is still "0".\n' - '\n' - '4. The slice of *s* from *i* to *j* is defined as the sequence ' - 'of\n' - ' items with index *k* such that "i <= k < j". If *i* or *j* ' - 'is\n' - ' greater than "len(s)", use "len(s)". If *i* is omitted or ' - '"None",\n' - ' use "0". If *j* is omitted or "None", use "len(s)". If *i* ' - 'is\n' - ' greater than or equal to *j*, the slice is empty.\n' - '\n' - '5. The slice of *s* from *i* to *j* with step *k* is defined as ' - 'the\n' - ' sequence of items with index "x = i + n*k" such that "0 <= n ' - '<\n' - ' (j-i)/k". In other words, the indices are "i", "i+k", ' - '"i+2*k",\n' - ' "i+3*k" and so on, stopping when *j* is reached (but never\n' - ' including *j*). When *k* is positive, *i* and *j* are ' - 'reduced to\n' - ' "len(s)" if they are greater. When *k* is negative, *i* and ' - '*j* are\n' - ' reduced to "len(s) - 1" if they are greater. If *i* or *j* ' - 'are\n' - ' omitted or "None", they become “end” values (which end ' - 'depends on\n' - ' the sign of *k*). Note, *k* cannot be zero. If *k* is ' - '"None", it\n' - ' is treated like "1".\n' - '\n' - '6. Concatenating immutable sequences always results in a new\n' - ' object. This means that building up a sequence by repeated\n' - ' concatenation will have a quadratic runtime cost in the ' - 'total\n' - ' sequence length. To get a linear runtime cost, you must ' - 'switch to\n' - ' one of the alternatives below:\n' - '\n' - ' * if concatenating "str" objects, you can build a list and ' - 'use\n' - ' "str.join()" at the end or else write to an "io.StringIO"\n' - ' instance and retrieve its value when complete\n' - '\n' - ' * if concatenating "bytes" objects, you can similarly use\n' - ' "bytes.join()" or "io.BytesIO", or you can do in-place\n' - ' concatenation with a "bytearray" object. "bytearray" ' - 'objects are\n' - ' mutable and have an efficient overallocation mechanism\n' - '\n' - ' * if concatenating "tuple" objects, extend a "list" instead\n' - '\n' - ' * for other types, investigate the relevant class ' - 'documentation\n' - '\n' - '7. Some sequence types (such as "range") only support item\n' - ' sequences that follow specific patterns, and hence don’t ' - 'support\n' - ' sequence concatenation or repetition.\n' - '\n' - '8. "index" raises "ValueError" when *x* is not found in *s*. ' - 'Not\n' - ' all implementations support passing the additional arguments ' - '*i*\n' - ' and *j*. These arguments allow efficient searching of ' - 'subsections\n' - ' of the sequence. Passing the extra arguments is roughly ' - 'equivalent\n' - ' to using "s[i:j].index(x)", only without copying any data and ' - 'with\n' - ' the returned index being relative to the start of the ' - 'sequence\n' - ' rather than the start of the slice.\n' - '\n' - '\n' - 'Immutable Sequence Types\n' - '========================\n' - '\n' - 'The only operation that immutable sequence types generally ' - 'implement\n' - 'that is not also implemented by mutable sequence types is ' - 'support for\n' - 'the "hash()" built-in.\n' - '\n' - 'This support allows immutable sequences, such as "tuple" ' - 'instances, to\n' - 'be used as "dict" keys and stored in "set" and "frozenset" ' - 'instances.\n' - '\n' - 'Attempting to hash an immutable sequence that contains ' - 'unhashable\n' - 'values will result in "TypeError".\n' - '\n' - '\n' - 'Mutable Sequence Types\n' - '======================\n' - '\n' - 'The operations in the following table are defined on mutable ' - 'sequence\n' - 'types. The "collections.abc.MutableSequence" ABC is provided to ' - 'make\n' - 'it easier to correctly implement these operations on custom ' - 'sequence\n' - 'types.\n' - '\n' - 'In the table *s* is an instance of a mutable sequence type, *t* ' - 'is any\n' - 'iterable object and *x* is an arbitrary object that meets any ' - 'type and\n' - 'value restrictions imposed by *s* (for example, "bytearray" ' - 'only\n' - 'accepts integers that meet the value restriction "0 <= x <= ' - '255").\n' - '\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| Operation | ' - 'Result | Notes |\n' - '+================================+==================================+=======================+\n' - '| "s[i] = x" | item *i* of *s* is replaced ' - 'by | |\n' - '| | ' - '*x* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j] = t" | slice of *s* from *i* to *j* ' - 'is | |\n' - '| | replaced by the contents of ' - 'the | |\n' - '| | iterable ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j]" | same as "s[i:j] = ' - '[]" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j:k] = t" | the elements of "s[i:j:k]" ' - 'are | (1) |\n' - '| | replaced by those of ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j:k]" | removes the elements ' - 'of | |\n' - '| | "s[i:j:k]" from the ' - 'list | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.append(x)" | appends *x* to the end of ' - 'the | |\n' - '| | sequence (same ' - 'as | |\n' - '| | "s[len(s):len(s)] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.clear()" | removes all items from *s* ' - '(same | (5) |\n' - '| | as "del ' - 's[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.copy()" | creates a shallow copy of ' - '*s* | (5) |\n' - '| | (same as ' - '"s[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.extend(t)" or "s += t" | extends *s* with the contents ' - 'of | |\n' - '| | *t* (for the most part the ' - 'same | |\n' - '| | as "s[len(s):len(s)] = ' - 't") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s *= n" | updates *s* with its ' - 'contents | (6) |\n' - '| | repeated *n* ' - 'times | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.insert(i, x)" | inserts *x* into *s* at ' - 'the | |\n' - '| | index given by *i* (same ' - 'as | |\n' - '| | "s[i:i] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at *i* ' - 'and | (2) |\n' - '| | also removes it from ' - '*s* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.remove(x)" | remove the first item from ' - '*s* | (3) |\n' - '| | where "s[i] == ' - 'x" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.reverse()" | reverses the items of *s* ' - 'in | (4) |\n' - '| | ' - 'place | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '\n' - 'Notes:\n' - '\n' - '1. *t* must have the same length as the slice it is replacing.\n' - '\n' - '2. The optional argument *i* defaults to "-1", so that by ' - 'default\n' - ' the last item is removed and returned.\n' - '\n' - '3. "remove" raises "ValueError" when *x* is not found in *s*.\n' - '\n' - '4. The "reverse()" method modifies the sequence in place for\n' - ' economy of space when reversing a large sequence. To remind ' - 'users\n' - ' that it operates by side effect, it does not return the ' - 'reversed\n' - ' sequence.\n' - '\n' - '5. "clear()" and "copy()" are included for consistency with the\n' - ' interfaces of mutable containers that don’t support slicing\n' - ' operations (such as "dict" and "set")\n' - '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' - '\n' - '6. The value *n* is an integer, or an object implementing\n' - ' "__index__()". Zero and negative values of *n* clear the ' - 'sequence.\n' - ' Items in the sequence are not copied; they are referenced ' - 'multiple\n' - ' times, as explained for "s * n" under Common Sequence ' - 'Operations.\n' - '\n' - '\n' - 'Lists\n' - '=====\n' - '\n' - 'Lists are mutable sequences, typically used to store collections ' - 'of\n' - 'homogeneous items (where the precise degree of similarity will ' - 'vary by\n' - 'application).\n' - '\n' - 'class list([iterable])\n' - '\n' - ' Lists may be constructed in several ways:\n' - '\n' - ' * Using a pair of square brackets to denote the empty list: ' - '"[]"\n' - '\n' - ' * Using square brackets, separating items with commas: ' - '"[a]",\n' - ' "[a, b, c]"\n' - '\n' - ' * Using a list comprehension: "[x for x in iterable]"\n' - '\n' - ' * Using the type constructor: "list()" or "list(iterable)"\n' - '\n' - ' The constructor builds a list whose items are the same and in ' - 'the\n' - ' same order as *iterable*’s items. *iterable* may be either ' - 'a\n' - ' sequence, a container that supports iteration, or an ' - 'iterator\n' - ' object. If *iterable* is already a list, a copy is made and\n' - ' returned, similar to "iterable[:]". For example, ' - '"list(\'abc\')"\n' - ' returns "[\'a\', \'b\', \'c\']" and "list( (1, 2, 3) )" ' - 'returns "[1, 2,\n' - ' 3]". If no argument is given, the constructor creates a new ' - 'empty\n' - ' list, "[]".\n' - '\n' - ' Many other operations also produce lists, including the ' - '"sorted()"\n' - ' built-in.\n' - '\n' - ' Lists implement all of the common and mutable sequence ' - 'operations.\n' - ' Lists also provide the following additional method:\n' - '\n' - ' sort(*, key=None, reverse=False)\n' - '\n' - ' This method sorts the list in place, using only "<" ' - 'comparisons\n' - ' between items. Exceptions are not suppressed - if any ' - 'comparison\n' - ' operations fail, the entire sort operation will fail (and ' - 'the\n' - ' list will likely be left in a partially modified state).\n' - '\n' - ' "sort()" accepts two arguments that can only be passed by\n' - ' keyword (keyword-only arguments):\n' - '\n' - ' *key* specifies a function of one argument that is used ' - 'to\n' - ' extract a comparison key from each list element (for ' - 'example,\n' - ' "key=str.lower"). The key corresponding to each item in ' - 'the list\n' - ' is calculated once and then used for the entire sorting ' - 'process.\n' - ' The default value of "None" means that list items are ' - 'sorted\n' - ' directly without calculating a separate key value.\n' - '\n' - ' The "functools.cmp_to_key()" utility is available to ' - 'convert a\n' - ' 2.x style *cmp* function to a *key* function.\n' - '\n' - ' *reverse* is a boolean value. If set to "True", then the ' - 'list\n' - ' elements are sorted as if each comparison were reversed.\n' - '\n' - ' This method modifies the sequence in place for economy of ' - 'space\n' - ' when sorting a large sequence. To remind users that it ' - 'operates\n' - ' by side effect, it does not return the sorted sequence ' - '(use\n' - ' "sorted()" to explicitly request a new sorted list ' - 'instance).\n' - '\n' - ' The "sort()" method is guaranteed to be stable. A sort ' - 'is\n' - ' stable if it guarantees not to change the relative order ' - 'of\n' - ' elements that compare equal — this is helpful for sorting ' - 'in\n' - ' multiple passes (for example, sort by department, then by ' - 'salary\n' - ' grade).\n' - '\n' - ' **CPython implementation detail:** While a list is being ' - 'sorted,\n' - ' the effect of attempting to mutate, or even inspect, the ' - 'list is\n' - ' undefined. The C implementation of Python makes the list ' - 'appear\n' - ' empty for the duration, and raises "ValueError" if it can ' - 'detect\n' - ' that the list has been mutated during a sort.\n' - '\n' - '\n' - 'Tuples\n' - '======\n' - '\n' - 'Tuples are immutable sequences, typically used to store ' - 'collections of\n' - 'heterogeneous data (such as the 2-tuples produced by the ' - '"enumerate()"\n' - 'built-in). Tuples are also used for cases where an immutable ' - 'sequence\n' - 'of homogeneous data is needed (such as allowing storage in a ' - '"set" or\n' - '"dict" instance).\n' - '\n' - 'class tuple([iterable])\n' - '\n' - ' Tuples may be constructed in a number of ways:\n' - '\n' - ' * Using a pair of parentheses to denote the empty tuple: ' - '"()"\n' - '\n' - ' * Using a trailing comma for a singleton tuple: "a," or ' - '"(a,)"\n' - '\n' - ' * Separating items with commas: "a, b, c" or "(a, b, c)"\n' - '\n' - ' * Using the "tuple()" built-in: "tuple()" or ' - '"tuple(iterable)"\n' - '\n' - ' The constructor builds a tuple whose items are the same and ' - 'in the\n' - ' same order as *iterable*’s items. *iterable* may be either ' - 'a\n' - ' sequence, a container that supports iteration, or an ' - 'iterator\n' - ' object. If *iterable* is already a tuple, it is returned\n' - ' unchanged. For example, "tuple(\'abc\')" returns "(\'a\', ' - '\'b\', \'c\')"\n' - ' and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument ' - 'is\n' - ' given, the constructor creates a new empty tuple, "()".\n' - '\n' - ' Note that it is actually the comma which makes a tuple, not ' - 'the\n' - ' parentheses. The parentheses are optional, except in the ' - 'empty\n' - ' tuple case, or when they are needed to avoid syntactic ' - 'ambiguity.\n' - ' For example, "f(a, b, c)" is a function call with three ' - 'arguments,\n' - ' while "f((a, b, c))" is a function call with a 3-tuple as the ' - 'sole\n' - ' argument.\n' - '\n' - ' Tuples implement all of the common sequence operations.\n' - '\n' - 'For heterogeneous collections of data where access by name is ' - 'clearer\n' - 'than access by index, "collections.namedtuple()" may be a more\n' - 'appropriate choice than a simple tuple object.\n' - '\n' - '\n' - 'Ranges\n' - '======\n' - '\n' - 'The "range" type represents an immutable sequence of numbers and ' - 'is\n' - 'commonly used for looping a specific number of times in "for" ' - 'loops.\n' - '\n' - 'class range(stop)\n' - 'class range(start, stop[, step])\n' - '\n' - ' The arguments to the range constructor must be integers ' - '(either\n' - ' built-in "int" or any object that implements the "__index__"\n' - ' special method). If the *step* argument is omitted, it ' - 'defaults to\n' - ' "1". If the *start* argument is omitted, it defaults to "0". ' - 'If\n' - ' *step* is zero, "ValueError" is raised.\n' - '\n' - ' For a positive *step*, the contents of a range "r" are ' - 'determined\n' - ' by the formula "r[i] = start + step*i" where "i >= 0" and ' - '"r[i] <\n' - ' stop".\n' - '\n' - ' For a negative *step*, the contents of the range are still\n' - ' determined by the formula "r[i] = start + step*i", but the\n' - ' constraints are "i >= 0" and "r[i] > stop".\n' - '\n' - ' A range object will be empty if "r[0]" does not meet the ' - 'value\n' - ' constraint. Ranges do support negative indices, but these ' - 'are\n' - ' interpreted as indexing from the end of the sequence ' - 'determined by\n' - ' the positive indices.\n' - '\n' - ' Ranges containing absolute values larger than "sys.maxsize" ' - 'are\n' - ' permitted but some features (such as "len()") may raise\n' - ' "OverflowError".\n' - '\n' - ' Range examples:\n' - '\n' - ' >>> list(range(10))\n' - ' [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n' - ' >>> list(range(1, 11))\n' - ' [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n' - ' >>> list(range(0, 30, 5))\n' - ' [0, 5, 10, 15, 20, 25]\n' - ' >>> list(range(0, 10, 3))\n' - ' [0, 3, 6, 9]\n' - ' >>> list(range(0, -10, -1))\n' - ' [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n' - ' >>> list(range(0))\n' - ' []\n' - ' >>> list(range(1, 0))\n' - ' []\n' - '\n' - ' Ranges implement all of the common sequence operations ' - 'except\n' - ' concatenation and repetition (due to the fact that range ' - 'objects\n' - ' can only represent sequences that follow a strict pattern ' - 'and\n' - ' repetition and concatenation will usually violate that ' - 'pattern).\n' - '\n' - ' start\n' - '\n' - ' The value of the *start* parameter (or "0" if the ' - 'parameter was\n' - ' not supplied)\n' - '\n' - ' stop\n' - '\n' - ' The value of the *stop* parameter\n' - '\n' - ' step\n' - '\n' - ' The value of the *step* parameter (or "1" if the parameter ' - 'was\n' - ' not supplied)\n' - '\n' - 'The advantage of the "range" type over a regular "list" or ' - '"tuple" is\n' - 'that a "range" object will always take the same (small) amount ' - 'of\n' - 'memory, no matter the size of the range it represents (as it ' - 'only\n' - 'stores the "start", "stop" and "step" values, calculating ' - 'individual\n' - 'items and subranges as needed).\n' - '\n' - 'Range objects implement the "collections.abc.Sequence" ABC, and\n' - 'provide features such as containment tests, element index ' - 'lookup,\n' - 'slicing and support for negative indices (see Sequence Types — ' - 'list,\n' - 'tuple, range):\n' - '\n' - '>>> r = range(0, 20, 2)\n' - '>>> r\n' - 'range(0, 20, 2)\n' - '>>> 11 in r\n' - 'False\n' - '>>> 10 in r\n' - 'True\n' - '>>> r.index(10)\n' - '5\n' - '>>> r[5]\n' - '10\n' - '>>> r[:5]\n' - 'range(0, 10, 2)\n' - '>>> r[-1]\n' - '18\n' - '\n' - 'Testing range objects for equality with "==" and "!=" compares ' - 'them as\n' - 'sequences. That is, two range objects are considered equal if ' - 'they\n' - 'represent the same sequence of values. (Note that two range ' - 'objects\n' - 'that compare equal might have different "start", "stop" and ' - '"step"\n' - 'attributes, for example "range(0) == range(2, 1, 3)" or ' - '"range(0, 3,\n' - '2) == range(0, 4, 2)".)\n' - '\n' - 'Changed in version 3.2: Implement the Sequence ABC. Support ' - 'slicing\n' - 'and negative indices. Test "int" objects for membership in ' - 'constant\n' - 'time instead of iterating through all items.\n' - '\n' - 'Changed in version 3.3: Define ‘==’ and ‘!=’ to compare range ' - 'objects\n' - 'based on the sequence of values they define (instead of ' - 'comparing\n' - 'based on object identity).\n' - '\n' - 'New in version 3.3: The "start", "stop" and "step" attributes.\n' - '\n' - 'See also:\n' - '\n' - ' * The linspace recipe shows how to implement a lazy version ' - 'of\n' - ' range suitable for floating point applications.\n', - 'typesseq-mutable': 'Mutable Sequence Types\n' - '**********************\n' - '\n' - 'The operations in the following table are defined on ' - 'mutable sequence\n' - 'types. The "collections.abc.MutableSequence" ABC is ' - 'provided to make\n' - 'it easier to correctly implement these operations on ' - 'custom sequence\n' - 'types.\n' - '\n' - 'In the table *s* is an instance of a mutable sequence ' - 'type, *t* is any\n' - 'iterable object and *x* is an arbitrary object that ' - 'meets any type and\n' - 'value restrictions imposed by *s* (for example, ' - '"bytearray" only\n' - 'accepts integers that meet the value restriction "0 <= x ' - '<= 255").\n' - '\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| Operation | ' - 'Result | Notes ' - '|\n' - '+================================+==================================+=======================+\n' - '| "s[i] = x" | item *i* of *s* is ' - 'replaced by | |\n' - '| | ' - '*x* | ' - '|\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j] = t" | slice of *s* from *i* ' - 'to *j* is | |\n' - '| | replaced by the ' - 'contents of the | |\n' - '| | iterable ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j]" | same as "s[i:j] = ' - '[]" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s[i:j:k] = t" | the elements of ' - '"s[i:j:k]" are | (1) |\n' - '| | replaced by those of ' - '*t* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "del s[i:j:k]" | removes the elements ' - 'of | |\n' - '| | "s[i:j:k]" from the ' - 'list | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.append(x)" | appends *x* to the ' - 'end of the | |\n' - '| | sequence (same ' - 'as | |\n' - '| | "s[len(s):len(s)] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.clear()" | removes all items ' - 'from *s* (same | (5) |\n' - '| | as "del ' - 's[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.copy()" | creates a shallow ' - 'copy of *s* | (5) |\n' - '| | (same as ' - '"s[:]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.extend(t)" or "s += t" | extends *s* with the ' - 'contents of | |\n' - '| | *t* (for the most ' - 'part the same | |\n' - '| | as "s[len(s):len(s)] ' - '= t") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s *= n" | updates *s* with its ' - 'contents | (6) |\n' - '| | repeated *n* ' - 'times | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.insert(i, x)" | inserts *x* into *s* ' - 'at the | |\n' - '| | index given by *i* ' - '(same as | |\n' - '| | "s[i:i] = ' - '[x]") | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.pop([i])" | retrieves the item at ' - '*i* and | (2) |\n' - '| | also removes it from ' - '*s* | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.remove(x)" | remove the first item ' - 'from *s* | (3) |\n' - '| | where "s[i] == ' - 'x" | |\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '| "s.reverse()" | reverses the items of ' - '*s* in | (4) |\n' - '| | ' - 'place | ' - '|\n' - '+--------------------------------+----------------------------------+-----------------------+\n' - '\n' - 'Notes:\n' - '\n' - '1. *t* must have the same length as the slice it is ' - 'replacing.\n' - '\n' - '2. The optional argument *i* defaults to "-1", so that ' - 'by default\n' - ' the last item is removed and returned.\n' - '\n' - '3. "remove" raises "ValueError" when *x* is not found in ' - '*s*.\n' - '\n' - '4. The "reverse()" method modifies the sequence in place ' - 'for\n' - ' economy of space when reversing a large sequence. To ' - 'remind users\n' - ' that it operates by side effect, it does not return ' - 'the reversed\n' - ' sequence.\n' - '\n' - '5. "clear()" and "copy()" are included for consistency ' - 'with the\n' - ' interfaces of mutable containers that don’t support ' - 'slicing\n' - ' operations (such as "dict" and "set")\n' - '\n' - ' New in version 3.3: "clear()" and "copy()" methods.\n' - '\n' - '6. The value *n* is an integer, or an object ' - 'implementing\n' - ' "__index__()". Zero and negative values of *n* clear ' - 'the sequence.\n' - ' Items in the sequence are not copied; they are ' - 'referenced multiple\n' - ' times, as explained for "s * n" under Common Sequence ' - 'Operations.\n', - 'unary': 'Unary arithmetic and bitwise operations\n' - '***************************************\n' - '\n' - 'All unary arithmetic and bitwise operations have the same ' - 'priority:\n' - '\n' - ' u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n' - '\n' - 'The unary "-" (minus) operator yields the negation of its numeric\n' - 'argument.\n' - '\n' - 'The unary "+" (plus) operator yields its numeric argument ' - 'unchanged.\n' - '\n' - 'The unary "~" (invert) operator yields the bitwise inversion of ' - 'its\n' - 'integer argument. The bitwise inversion of "x" is defined as\n' - '"-(x+1)". It only applies to integral numbers.\n' - '\n' - 'In all three cases, if the argument does not have the proper type, ' - 'a\n' - '"TypeError" exception is raised.\n', - 'while': 'The "while" statement\n' - '*********************\n' - '\n' - 'The "while" statement is used for repeated execution as long as an\n' - 'expression is true:\n' - '\n' - ' while_stmt ::= "while" expression ":" suite\n' - ' ["else" ":" suite]\n' - '\n' - 'This repeatedly tests the expression and, if it is true, executes ' - 'the\n' - 'first suite; if the expression is false (which may be the first ' - 'time\n' - 'it is tested) the suite of the "else" clause, if present, is ' - 'executed\n' - 'and the loop terminates.\n' - '\n' - 'A "break" statement executed in the first suite terminates the ' - 'loop\n' - 'without executing the "else" clause’s suite. A "continue" ' - 'statement\n' - 'executed in the first suite skips the rest of the suite and goes ' - 'back\n' - 'to testing the expression.\n', - 'with': 'The "with" statement\n' - '********************\n' - '\n' - 'The "with" statement is used to wrap the execution of a block with\n' - 'methods defined by a context manager (see section With Statement\n' - 'Context Managers). This allows common "try"…"except"…"finally" ' - 'usage\n' - 'patterns to be encapsulated for convenient reuse.\n' - '\n' - ' with_stmt ::= "with" with_item ("," with_item)* ":" suite\n' - ' with_item ::= expression ["as" target]\n' - '\n' - 'The execution of the "with" statement with one “item” proceeds as\n' - 'follows:\n' - '\n' - '1. The context expression (the expression given in the "with_item")\n' - ' is evaluated to obtain a context manager.\n' - '\n' - '2. The context manager’s "__exit__()" is loaded for later use.\n' - '\n' - '3. The context manager’s "__enter__()" method is invoked.\n' - '\n' - '4. If a target was included in the "with" statement, the return\n' - ' value from "__enter__()" is assigned to it.\n' - '\n' - ' Note: The "with" statement guarantees that if the "__enter__()"\n' - ' method returns without an error, then "__exit__()" will always ' - 'be\n' - ' called. Thus, if an error occurs during the assignment to the\n' - ' target list, it will be treated the same as an error occurring\n' - ' within the suite would be. See step 6 below.\n' - '\n' - '5. The suite is executed.\n' - '\n' - '6. The context manager’s "__exit__()" method is invoked. If an\n' - ' exception caused the suite to be exited, its type, value, and\n' - ' traceback are passed as arguments to "__exit__()". Otherwise, ' - 'three\n' - ' "None" arguments are supplied.\n' - '\n' - ' If the suite was exited due to an exception, and the return ' - 'value\n' - ' from the "__exit__()" method was false, the exception is ' - 'reraised.\n' - ' If the return value was true, the exception is suppressed, and\n' - ' execution continues with the statement following the "with"\n' - ' statement.\n' - '\n' - ' If the suite was exited for any reason other than an exception, ' - 'the\n' - ' return value from "__exit__()" is ignored, and execution ' - 'proceeds\n' - ' at the normal location for the kind of exit that was taken.\n' - '\n' - 'With more than one item, the context managers are processed as if\n' - 'multiple "with" statements were nested:\n' - '\n' - ' with A() as a, B() as b:\n' - ' suite\n' - '\n' - 'is equivalent to\n' - '\n' - ' with A() as a:\n' - ' with B() as b:\n' - ' suite\n' - '\n' - 'Changed in version 3.1: Support for multiple context expressions.\n' - '\n' - 'See also:\n' - '\n' - ' **PEP 343** - The “with” statement\n' - ' The specification, background, and examples for the Python ' - '"with"\n' - ' statement.\n', - 'yield': 'The "yield" statement\n' - '*********************\n' - '\n' - ' yield_stmt ::= yield_expression\n' - '\n' - 'A "yield" statement is semantically equivalent to a yield ' - 'expression.\n' - 'The yield statement can be used to omit the parentheses that would\n' - 'otherwise be required in the equivalent yield expression ' - 'statement.\n' - 'For example, the yield statements\n' - '\n' - ' yield \n' - ' yield from \n' - '\n' - 'are equivalent to the yield expression statements\n' - '\n' - ' (yield )\n' - ' (yield from )\n' - '\n' - 'Yield expressions and statements are only used when defining a\n' - '*generator* function, and are only used in the body of the ' - 'generator\n' - 'function. Using yield in a function definition is sufficient to ' - 'cause\n' - 'that definition to create a generator function instead of a normal\n' - 'function.\n' - '\n' - 'For full details of "yield" semantics, refer to the Yield ' - 'expressions\n' - 'section.\n'} +# Autogenerated by Sphinx on Tue Feb 3 17:32:13 2026 +# as part of the release process. + +topics = { + 'assert': r'''The "assert" statement +********************** + +Assert statements are a convenient way to insert debugging assertions +into a program: + + assert_stmt: "assert" expression ["," expression] + +The simple form, "assert expression", is equivalent to + + if __debug__: + if not expression: raise AssertionError + +The extended form, "assert expression1, expression2", is equivalent to + + if __debug__: + if not expression1: raise AssertionError(expression2) + +These equivalences assume that "__debug__" and "AssertionError" refer +to the built-in variables with those names. In the current +implementation, the built-in variable "__debug__" is "True" under +normal circumstances, "False" when optimization is requested (command +line option "-O"). The current code generator emits no code for an +"assert" statement when optimization is requested at compile time. +Note that it is unnecessary to include the source code for the +expression that failed in the error message; it will be displayed as +part of the stack trace. + +Assignments to "__debug__" are illegal. The value for the built-in +variable is determined when the interpreter starts. +''', + 'assignment': r'''Assignment statements +********************* + +Assignment statements are used to (re)bind names to values and to +modify attributes or items of mutable objects: + + assignment_stmt: (target_list "=")+ (starred_expression | yield_expression) + target_list: target ("," target)* [","] + target: identifier + | "(" [target_list] ")" + | "[" [target_list] "]" + | attributeref + | subscription + | slicing + | "*" target + +(See section Primaries for the syntax definitions for *attributeref*, +*subscription*, and *slicing*.) + +An assignment statement evaluates the expression list (remember that +this can be a single expression or a comma-separated list, the latter +yielding a tuple) and assigns the single resulting object to each of +the target lists, from left to right. + +Assignment is defined recursively depending on the form of the target +(list). When a target is part of a mutable object (an attribute +reference, subscription or slicing), the mutable object must +ultimately perform the assignment and decide about its validity, and +may raise an exception if the assignment is unacceptable. The rules +observed by various types and the exceptions raised are given with the +definition of the object types (see section The standard type +hierarchy). + +Assignment of an object to a target list, optionally enclosed in +parentheses or square brackets, is recursively defined as follows. + +* If the target list is a single target with no trailing comma, + optionally in parentheses, the object is assigned to that target. + +* Else: + + * If the target list contains one target prefixed with an asterisk, + called a “starred” target: The object must be an iterable with at + least as many items as there are targets in the target list, minus + one. The first items of the iterable are assigned, from left to + right, to the targets before the starred target. The final items + of the iterable are assigned to the targets after the starred + target. A list of the remaining items in the iterable is then + assigned to the starred target (the list can be empty). + + * Else: The object must be an iterable with the same number of items + as there are targets in the target list, and the items are + assigned, from left to right, to the corresponding targets. + +Assignment of an object to a single target is recursively defined as +follows. + +* If the target is an identifier (name): + + * If the name does not occur in a "global" or "nonlocal" statement + in the current code block: the name is bound to the object in the + current local namespace. + + * Otherwise: the name is bound to the object in the global namespace + or the outer namespace determined by "nonlocal", respectively. + + The name is rebound if it was already bound. This may cause the + reference count for the object previously bound to the name to reach + zero, causing the object to be deallocated and its destructor (if it + has one) to be called. + +* If the target is an attribute reference: The primary expression in + the reference is evaluated. It should yield an object with + assignable attributes; if this is not the case, "TypeError" is + raised. That object is then asked to assign the assigned object to + the given attribute; if it cannot perform the assignment, it raises + an exception (usually but not necessarily "AttributeError"). + + Note: If the object is a class instance and the attribute reference + occurs on both sides of the assignment operator, the right-hand side + expression, "a.x" can access either an instance attribute or (if no + instance attribute exists) a class attribute. The left-hand side + target "a.x" is always set as an instance attribute, creating it if + necessary. Thus, the two occurrences of "a.x" do not necessarily + refer to the same attribute: if the right-hand side expression + refers to a class attribute, the left-hand side creates a new + instance attribute as the target of the assignment: + + class Cls: + x = 3 # class variable + inst = Cls() + inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3 + + This description does not necessarily apply to descriptor + attributes, such as properties created with "property()". + +* If the target is a subscription: The primary expression in the + reference is evaluated. It should yield either a mutable sequence + object (such as a list) or a mapping object (such as a dictionary). + Next, the subscript expression is evaluated. + + If the primary is a mutable sequence object (such as a list), the + subscript must yield an integer. If it is negative, the sequence’s + length is added to it. The resulting value must be a nonnegative + integer less than the sequence’s length, and the sequence is asked + to assign the assigned object to its item with that index. If the + index is out of range, "IndexError" is raised (assignment to a + subscripted sequence cannot add new items to a list). + + If the primary is a mapping object (such as a dictionary), the + subscript must have a type compatible with the mapping’s key type, + and the mapping is then asked to create a key/value pair which maps + the subscript to the assigned object. This can either replace an + existing key/value pair with the same key value, or insert a new + key/value pair (if no key with the same value existed). + + For user-defined objects, the "__setitem__()" method is called with + appropriate arguments. + +* If the target is a slicing: The primary expression in the reference + is evaluated. It should yield a mutable sequence object (such as a + list). The assigned object should be a sequence object of the same + type. Next, the lower and upper bound expressions are evaluated, + insofar they are present; defaults are zero and the sequence’s + length. The bounds should evaluate to integers. If either bound is + negative, the sequence’s length is added to it. The resulting + bounds are clipped to lie between zero and the sequence’s length, + inclusive. Finally, the sequence object is asked to replace the + slice with the items of the assigned sequence. The length of the + slice may be different from the length of the assigned sequence, + thus changing the length of the target sequence, if the target + sequence allows it. + +**CPython implementation detail:** In the current implementation, the +syntax for targets is taken to be the same as for expressions, and +invalid syntax is rejected during the code generation phase, causing +less detailed error messages. + +Although the definition of assignment implies that overlaps between +the left-hand side and the right-hand side are ‘simultaneous’ (for +example "a, b = b, a" swaps two variables), overlaps *within* the +collection of assigned-to variables occur left-to-right, sometimes +resulting in confusion. For instance, the following program prints +"[0, 2]": + + x = [0, 1] + i = 0 + i, x[i] = 1, 2 # i is updated, then x[i] is updated + print(x) + +See also: + + **PEP 3132** - Extended Iterable Unpacking + The specification for the "*target" feature. + + +Augmented assignment statements +=============================== + +Augmented assignment is the combination, in a single statement, of a +binary operation and an assignment statement: + + augmented_assignment_stmt: augtarget augop (expression_list | yield_expression) + augtarget: identifier | attributeref | subscription | slicing + augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" + | ">>=" | "<<=" | "&=" | "^=" | "|=" + +(See section Primaries for the syntax definitions of the last three +symbols.) + +An augmented assignment evaluates the target (which, unlike normal +assignment statements, cannot be an unpacking) and the expression +list, performs the binary operation specific to the type of assignment +on the two operands, and assigns the result to the original target. +The target is only evaluated once. + +An augmented assignment statement like "x += 1" can be rewritten as "x += x + 1" to achieve a similar, but not exactly equal effect. In the +augmented version, "x" is only evaluated once. Also, when possible, +the actual operation is performed *in-place*, meaning that rather than +creating a new object and assigning that to the target, the old object +is modified instead. + +Unlike normal assignments, augmented assignments evaluate the left- +hand side *before* evaluating the right-hand side. For example, "a[i] ++= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs +the addition, and lastly, it writes the result back to "a[i]". + +With the exception of assigning to tuples and multiple targets in a +single statement, the assignment done by augmented assignment +statements is handled the same way as normal assignments. Similarly, +with the exception of the possible *in-place* behavior, the binary +operation performed by augmented assignment is the same as the normal +binary operations. + +For targets which are attribute references, the same caveat about +class and instance attributes applies as for regular assignments. + + +Annotated assignment statements +=============================== + +*Annotation* assignment is the combination, in a single statement, of +a variable or attribute annotation and an optional assignment +statement: + + annotated_assignment_stmt: augtarget ":" expression + ["=" (starred_expression | yield_expression)] + +The difference from normal Assignment statements is that only a single +target is allowed. + +The assignment target is considered “simple” if it consists of a +single name that is not enclosed in parentheses. For simple assignment +targets, if in class or module scope, the annotations are gathered in +a lazily evaluated annotation scope. The annotations can be evaluated +using the "__annotations__" attribute of a class or module, or using +the facilities in the "annotationlib" module. + +If the assignment target is not simple (an attribute, subscript node, +or parenthesized name), the annotation is never evaluated. + +If a name is annotated in a function scope, then this name is local +for that scope. Annotations are never evaluated and stored in function +scopes. + +If the right hand side is present, an annotated assignment performs +the actual assignment as if there was no annotation present. If the +right hand side is not present for an expression target, then the +interpreter evaluates the target except for the last "__setitem__()" +or "__setattr__()" call. + +See also: + + **PEP 526** - Syntax for Variable Annotations + The proposal that added syntax for annotating the types of + variables (including class variables and instance variables), + instead of expressing them through comments. + + **PEP 484** - Type hints + The proposal that added the "typing" module to provide a standard + syntax for type annotations that can be used in static analysis + tools and IDEs. + +Changed in version 3.8: Now annotated assignments allow the same +expressions in the right hand side as regular assignments. Previously, +some expressions (like un-parenthesized tuple expressions) caused a +syntax error. + +Changed in version 3.14: Annotations are now lazily evaluated in a +separate annotation scope. If the assignment target is not simple, +annotations are never evaluated. +''', + 'assignment-expressions': r'''Assignment expressions +********************** + + assignment_expression: [identifier ":="] expression + +An assignment expression (sometimes also called a “named expression” +or “walrus”) assigns an "expression" to an "identifier", while also +returning the value of the "expression". + +One common use case is when handling matched regular expressions: + + if matching := pattern.search(data): + do_something(matching) + +Or, when processing a file stream in chunks: + + while chunk := file.read(9000): + process(chunk) + +Assignment expressions must be surrounded by parentheses when used as +expression statements and when used as sub-expressions in slicing, +conditional, lambda, keyword-argument, and comprehension-if +expressions and in "assert", "with", and "assignment" statements. In +all other places where they can be used, parentheses are not required, +including in "if" and "while" statements. + +Added in version 3.8: See **PEP 572** for more details about +assignment expressions. +''', + 'async': r'''Coroutines +********** + +Added in version 3.5. + + +Coroutine function definition +============================= + + async_funcdef: [decorators] "async" "def" funcname "(" [parameter_list] ")" + ["->" expression] ":" suite + +Execution of Python coroutines can be suspended and resumed at many +points (see *coroutine*). "await" expressions, "async for" and "async +with" can only be used in the body of a coroutine function. + +Functions defined with "async def" syntax are always coroutine +functions, even if they do not contain "await" or "async" keywords. + +It is a "SyntaxError" to use a "yield from" expression inside the body +of a coroutine function. + +An example of a coroutine function: + + async def func(param1, param2): + do_stuff() + await some_coroutine() + +Changed in version 3.7: "await" and "async" are now keywords; +previously they were only treated as such inside the body of a +coroutine function. + + +The "async for" statement +========================= + + async_for_stmt: "async" for_stmt + +An *asynchronous iterable* provides an "__aiter__" method that +directly returns an *asynchronous iterator*, which can call +asynchronous code in its "__anext__" method. + +The "async for" statement allows convenient iteration over +asynchronous iterables. + +The following code: + + async for TARGET in ITER: + SUITE + else: + SUITE2 + +Is semantically equivalent to: + + iter = (ITER) + iter = type(iter).__aiter__(iter) + running = True + + while running: + try: + TARGET = await type(iter).__anext__(iter) + except StopAsyncIteration: + running = False + else: + SUITE + else: + SUITE2 + +See also "__aiter__()" and "__anext__()" for details. + +It is a "SyntaxError" to use an "async for" statement outside the body +of a coroutine function. + + +The "async with" statement +========================== + + async_with_stmt: "async" with_stmt + +An *asynchronous context manager* is a *context manager* that is able +to suspend execution in its *enter* and *exit* methods. + +The following code: + + async with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + aenter = type(manager).__aenter__ + aexit = type(manager).__aexit__ + value = await aenter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not await aexit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + await aexit(manager, None, None, None) + +See also "__aenter__()" and "__aexit__()" for details. + +It is a "SyntaxError" to use an "async with" statement outside the +body of a coroutine function. + +See also: + + **PEP 492** - Coroutines with async and await syntax + The proposal that made coroutines a proper standalone concept in + Python, and added supporting syntax. +''', + 'atom-identifiers': r'''Identifiers (Names) +******************* + +An identifier occurring as an atom is a name. See section Names +(identifiers and keywords) for lexical definition and section Naming +and binding for documentation of naming and binding. + +When the name is bound to an object, evaluation of the atom yields +that object. When a name is not bound, an attempt to evaluate it +raises a "NameError" exception. + + +Private name mangling +===================== + +When an identifier that textually occurs in a class definition begins +with two or more underscore characters and does not end in two or more +underscores, it is considered a *private name* of that class. + +See also: The class specifications. + +More precisely, private names are transformed to a longer form before +code is generated for them. If the transformed name is longer than +255 characters, implementation-defined truncation may happen. + +The transformation is independent of the syntactical context in which +the identifier is used but only the following private identifiers are +mangled: + +* Any name used as the name of a variable that is assigned or read or + any name of an attribute being accessed. + + The "__name__" attribute of nested functions, classes, and type + aliases is however not mangled. + +* The name of imported modules, e.g., "__spam" in "import __spam". If + the module is part of a package (i.e., its name contains a dot), the + name is *not* mangled, e.g., the "__foo" in "import __foo.bar" is + not mangled. + +* The name of an imported member, e.g., "__f" in "from spam import + __f". + +The transformation rule is defined as follows: + +* The class name, with leading underscores removed and a single + leading underscore inserted, is inserted in front of the identifier, + e.g., the identifier "__spam" occurring in a class named "Foo", + "_Foo" or "__Foo" is transformed to "_Foo__spam". + +* If the class name consists only of underscores, the transformation + is the identity, e.g., the identifier "__spam" occurring in a class + named "_" or "__" is left as is. +''', + 'atom-literals': r'''Literals +******** + +Python supports string and bytes literals and various numeric +literals: + + literal: strings | NUMBER + +Evaluation of a literal yields an object of the given type (string, +bytes, integer, floating-point number, complex number) with the given +value. The value may be approximated in the case of floating-point +and imaginary (complex) literals. See section Literals for details. +See section String literal concatenation for details on "strings". + +All literals correspond to immutable data types, and hence the +object’s identity is less important than its value. Multiple +evaluations of literals with the same value (either the same +occurrence in the program text or a different occurrence) may obtain +the same object or a different object with the same value. + + +String literal concatenation +============================ + +Multiple adjacent string or bytes literals (delimited by whitespace), +possibly using different quoting conventions, are allowed, and their +meaning is the same as their concatenation: + + >>> "hello" 'world' + "helloworld" + +Formally: + + strings: ( STRING | fstring)+ | tstring+ + +This feature is defined at the syntactical level, so it only works +with literals. To concatenate string expressions at run time, the ‘+’ +operator may be used: + + >>> greeting = "Hello" + >>> space = " " + >>> name = "Blaise" + >>> print(greeting + space + name) # not: print(greeting space name) + Hello Blaise + +Literal concatenation can freely mix raw strings, triple-quoted +strings, and formatted string literals. For example: + + >>> "Hello" r', ' f"{name}!" + "Hello, Blaise!" + +This feature can be used to reduce the number of backslashes needed, +to split long strings conveniently across long lines, or even to add +comments to parts of strings. For example: + + re.compile("[A-Za-z_]" # letter or underscore + "[A-Za-z0-9_]*" # letter, digit or underscore + ) + +However, bytes literals may only be combined with other byte literals; +not with string literals of any kind. Also, template string literals +may only be combined with other template string literals: + + >>> t"Hello" t"{name}!" + Template(strings=('Hello', '!'), interpolations=(...)) +''', + 'attribute-access': r'''Customizing attribute access +**************************** + +The following methods can be defined to customize the meaning of +attribute access (use of, assignment to, or deletion of "x.name") for +class instances. + +object.__getattr__(self, name) + + Called when the default attribute access fails with an + "AttributeError" (either "__getattribute__()" raises an + "AttributeError" because *name* is not an instance attribute or an + attribute in the class tree for "self"; or "__get__()" of a *name* + property raises "AttributeError"). This method should either + return the (computed) attribute value or raise an "AttributeError" + exception. The "object" class itself does not provide this method. + + Note that if the attribute is found through the normal mechanism, + "__getattr__()" is not called. (This is an intentional asymmetry + between "__getattr__()" and "__setattr__()".) This is done both for + efficiency reasons and because otherwise "__getattr__()" would have + no way to access other attributes of the instance. Note that at + least for instance variables, you can take total control by not + inserting any values in the instance attribute dictionary (but + instead inserting them in another object). See the + "__getattribute__()" method below for a way to actually get total + control over attribute access. + +object.__getattribute__(self, name) + + Called unconditionally to implement attribute accesses for + instances of the class. If the class also defines "__getattr__()", + the latter will not be called unless "__getattribute__()" either + calls it explicitly or raises an "AttributeError". This method + should return the (computed) attribute value or raise an + "AttributeError" exception. In order to avoid infinite recursion in + this method, its implementation should always call the base class + method with the same name to access any attributes it needs, for + example, "object.__getattribute__(self, name)". + + Note: + + This method may still be bypassed when looking up special methods + as the result of implicit invocation via language syntax or + built-in functions. See Special method lookup. + + For certain sensitive attribute accesses, raises an auditing event + "object.__getattr__" with arguments "obj" and "name". + +object.__setattr__(self, name, value) + + Called when an attribute assignment is attempted. This is called + instead of the normal mechanism (i.e. store the value in the + instance dictionary). *name* is the attribute name, *value* is the + value to be assigned to it. + + If "__setattr__()" wants to assign to an instance attribute, it + should call the base class method with the same name, for example, + "object.__setattr__(self, name, value)". + + For certain sensitive attribute assignments, raises an auditing + event "object.__setattr__" with arguments "obj", "name", "value". + +object.__delattr__(self, name) + + Like "__setattr__()" but for attribute deletion instead of + assignment. This should only be implemented if "del obj.name" is + meaningful for the object. + + For certain sensitive attribute deletions, raises an auditing event + "object.__delattr__" with arguments "obj" and "name". + +object.__dir__(self) + + Called when "dir()" is called on the object. An iterable must be + returned. "dir()" converts the returned iterable to a list and + sorts it. + + +Customizing module attribute access +=================================== + +module.__getattr__() +module.__dir__() + +Special names "__getattr__" and "__dir__" can be also used to +customize access to module attributes. The "__getattr__" function at +the module level should accept one argument which is the name of an +attribute and return the computed value or raise an "AttributeError". +If an attribute is not found on a module object through the normal +lookup, i.e. "object.__getattribute__()", then "__getattr__" is +searched in the module "__dict__" before raising an "AttributeError". +If found, it is called with the attribute name and the result is +returned. + +The "__dir__" function should accept no arguments, and return an +iterable of strings that represents the names accessible on module. If +present, this function overrides the standard "dir()" search on a +module. + +module.__class__ + +For a more fine grained customization of the module behavior (setting +attributes, properties, etc.), one can set the "__class__" attribute +of a module object to a subclass of "types.ModuleType". For example: + + import sys + from types import ModuleType + + class VerboseModule(ModuleType): + def __repr__(self): + return f'Verbose {self.__name__}' + + def __setattr__(self, attr, value): + print(f'Setting {attr}...') + super().__setattr__(attr, value) + + sys.modules[__name__].__class__ = VerboseModule + +Note: + + Defining module "__getattr__" and setting module "__class__" only + affect lookups made using the attribute access syntax – directly + accessing the module globals (whether by code within the module, or + via a reference to the module’s globals dictionary) is unaffected. + +Changed in version 3.5: "__class__" module attribute is now writable. + +Added in version 3.7: "__getattr__" and "__dir__" module attributes. + +See also: + + **PEP 562** - Module __getattr__ and __dir__ + Describes the "__getattr__" and "__dir__" functions on modules. + + +Implementing Descriptors +======================== + +The following methods only apply when an instance of the class +containing the method (a so-called *descriptor* class) appears in an +*owner* class (the descriptor must be in either the owner’s class +dictionary or in the class dictionary for one of its parents). In the +examples below, “the attribute” refers to the attribute whose name is +the key of the property in the owner class’ "__dict__". The "object" +class itself does not implement any of these protocols. + +object.__get__(self, instance, owner=None) + + Called to get the attribute of the owner class (class attribute + access) or of an instance of that class (instance attribute + access). The optional *owner* argument is the owner class, while + *instance* is the instance that the attribute was accessed through, + or "None" when the attribute is accessed through the *owner*. + + This method should return the computed attribute value or raise an + "AttributeError" exception. + + **PEP 252** specifies that "__get__()" is callable with one or two + arguments. Python’s own built-in descriptors support this + specification; however, it is likely that some third-party tools + have descriptors that require both arguments. Python’s own + "__getattribute__()" implementation always passes in both arguments + whether they are required or not. + +object.__set__(self, instance, value) + + Called to set the attribute on an instance *instance* of the owner + class to a new value, *value*. + + Note, adding "__set__()" or "__delete__()" changes the kind of + descriptor to a “data descriptor”. See Invoking Descriptors for + more details. + +object.__delete__(self, instance) + + Called to delete the attribute on an instance *instance* of the + owner class. + +Instances of descriptors may also have the "__objclass__" attribute +present: + +object.__objclass__ + + The attribute "__objclass__" is interpreted by the "inspect" module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class + attributes). For callables, it may indicate that an instance of the + given type (or a subclass) is expected or required as the first + positional argument (for example, CPython sets this attribute for + unbound methods that are implemented in C). + + +Invoking Descriptors +==================== + +In general, a descriptor is an object attribute with “binding +behavior”, one whose attribute access has been overridden by methods +in the descriptor protocol: "__get__()", "__set__()", and +"__delete__()". If any of those methods are defined for an object, it +is said to be a descriptor. + +The default behavior for attribute access is to get, set, or delete +the attribute from an object’s dictionary. For instance, "a.x" has a +lookup chain starting with "a.__dict__['x']", then +"type(a).__dict__['x']", and continuing through the base classes of +"type(a)" excluding metaclasses. + +However, if the looked-up value is an object defining one of the +descriptor methods, then Python may override the default behavior and +invoke the descriptor method instead. Where this occurs in the +precedence chain depends on which descriptor methods were defined and +how they were called. + +The starting point for descriptor invocation is a binding, "a.x". How +the arguments are assembled depends on "a": + +Direct Call + The simplest and least common call is when user code directly + invokes a descriptor method: "x.__get__(a)". + +Instance Binding + If binding to an object instance, "a.x" is transformed into the + call: "type(a).__dict__['x'].__get__(a, type(a))". + +Class Binding + If binding to a class, "A.x" is transformed into the call: + "A.__dict__['x'].__get__(None, A)". + +Super Binding + A dotted lookup such as "super(A, a).x" searches + "a.__class__.__mro__" for a base class "B" following "A" and then + returns "B.__dict__['x'].__get__(a, A)". If not a descriptor, "x" + is returned unchanged. + +For instance bindings, the precedence of descriptor invocation depends +on which descriptor methods are defined. A descriptor can define any +combination of "__get__()", "__set__()" and "__delete__()". If it +does not define "__get__()", then accessing the attribute will return +the descriptor object itself unless there is a value in the object’s +instance dictionary. If the descriptor defines "__set__()" and/or +"__delete__()", it is a data descriptor; if it defines neither, it is +a non-data descriptor. Normally, data descriptors define both +"__get__()" and "__set__()", while non-data descriptors have just the +"__get__()" method. Data descriptors with "__get__()" and "__set__()" +(and/or "__delete__()") defined always override a redefinition in an +instance dictionary. In contrast, non-data descriptors can be +overridden by instances. + +Python methods (including those decorated with "@staticmethod" and +"@classmethod") are implemented as non-data descriptors. Accordingly, +instances can redefine and override methods. This allows individual +instances to acquire behaviors that differ from other instances of the +same class. + +The "property()" function is implemented as a data descriptor. +Accordingly, instances cannot override the behavior of a property. + + +__slots__ +========= + +*__slots__* allow us to explicitly declare data members (like +properties) and deny the creation of "__dict__" and *__weakref__* +(unless explicitly declared in *__slots__* or available in a parent.) + +The space saved over using "__dict__" can be significant. Attribute +lookup speed can be significantly improved as well. + +object.__slots__ + + This class variable can be assigned a string, iterable, or sequence + of strings with variable names used by instances. *__slots__* + reserves space for the declared variables and prevents the + automatic creation of "__dict__" and *__weakref__* for each + instance. + +Notes on using *__slots__*: + +* When inheriting from a class without *__slots__*, the "__dict__" and + *__weakref__* attribute of the instances will always be accessible. + +* Without a "__dict__" variable, instances cannot be assigned new + variables not listed in the *__slots__* definition. Attempts to + assign to an unlisted variable name raises "AttributeError". If + dynamic assignment of new variables is desired, then add + "'__dict__'" to the sequence of strings in the *__slots__* + declaration. + +* Without a *__weakref__* variable for each instance, classes defining + *__slots__* do not support "weak references" to its instances. If + weak reference support is needed, then add "'__weakref__'" to the + sequence of strings in the *__slots__* declaration. + +* *__slots__* are implemented at the class level by creating + descriptors for each variable name. As a result, class attributes + cannot be used to set default values for instance variables defined + by *__slots__*; otherwise, the class attribute would overwrite the + descriptor assignment. + +* The action of a *__slots__* declaration is not limited to the class + where it is defined. *__slots__* declared in parents are available + in child classes. However, instances of a child subclass will get a + "__dict__" and *__weakref__* unless the subclass also defines + *__slots__* (which should only contain names of any *additional* + slots). + +* If a class defines a slot also defined in a base class, the instance + variable defined by the base class slot is inaccessible (except by + retrieving its descriptor directly from the base class). This + renders the meaning of the program undefined. In the future, a + check may be added to prevent this. + +* "TypeError" will be raised if nonempty *__slots__* are defined for a + class derived from a ""variable-length" built-in type" such as + "int", "bytes", and "tuple". + +* Any non-string *iterable* may be assigned to *__slots__*. + +* If a "dictionary" is used to assign *__slots__*, the dictionary keys + will be used as the slot names. The values of the dictionary can be + used to provide per-attribute docstrings that will be recognised by + "inspect.getdoc()" and displayed in the output of "help()". + +* "__class__" assignment works only if both classes have the same + *__slots__*. + +* Multiple inheritance with multiple slotted parent classes can be + used, but only one parent is allowed to have attributes created by + slots (the other bases must have empty slot layouts) - violations + raise "TypeError". + +* If an *iterator* is used for *__slots__* then a *descriptor* is + created for each of the iterator’s values. However, the *__slots__* + attribute will be an empty iterator. +''', + 'attribute-references': r'''Attribute references +******************** + +An attribute reference is a primary followed by a period and a name: + + attributeref: primary "." identifier + +The primary must evaluate to an object of a type that supports +attribute references, which most objects do. This object is then +asked to produce the attribute whose name is the identifier. The type +and value produced is determined by the object. Multiple evaluations +of the same attribute reference may yield different objects. + +This production can be customized by overriding the +"__getattribute__()" method or the "__getattr__()" method. The +"__getattribute__()" method is called first and either returns a value +or raises "AttributeError" if the attribute is not available. + +If an "AttributeError" is raised and the object has a "__getattr__()" +method, that method is called as a fallback. +''', + 'augassign': r'''Augmented assignment statements +******************************* + +Augmented assignment is the combination, in a single statement, of a +binary operation and an assignment statement: + + augmented_assignment_stmt: augtarget augop (expression_list | yield_expression) + augtarget: identifier | attributeref | subscription | slicing + augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" + | ">>=" | "<<=" | "&=" | "^=" | "|=" + +(See section Primaries for the syntax definitions of the last three +symbols.) + +An augmented assignment evaluates the target (which, unlike normal +assignment statements, cannot be an unpacking) and the expression +list, performs the binary operation specific to the type of assignment +on the two operands, and assigns the result to the original target. +The target is only evaluated once. + +An augmented assignment statement like "x += 1" can be rewritten as "x += x + 1" to achieve a similar, but not exactly equal effect. In the +augmented version, "x" is only evaluated once. Also, when possible, +the actual operation is performed *in-place*, meaning that rather than +creating a new object and assigning that to the target, the old object +is modified instead. + +Unlike normal assignments, augmented assignments evaluate the left- +hand side *before* evaluating the right-hand side. For example, "a[i] ++= f(x)" first looks-up "a[i]", then it evaluates "f(x)" and performs +the addition, and lastly, it writes the result back to "a[i]". + +With the exception of assigning to tuples and multiple targets in a +single statement, the assignment done by augmented assignment +statements is handled the same way as normal assignments. Similarly, +with the exception of the possible *in-place* behavior, the binary +operation performed by augmented assignment is the same as the normal +binary operations. + +For targets which are attribute references, the same caveat about +class and instance attributes applies as for regular assignments. +''', + 'await': r'''Await expression +**************** + +Suspend the execution of *coroutine* on an *awaitable* object. Can +only be used inside a *coroutine function*. + + await_expr: "await" primary + +Added in version 3.5. +''', + 'binary': r'''Binary arithmetic operations +**************************** + +The binary arithmetic operations have the conventional priority +levels. Note that some of these operations also apply to certain non- +numeric types. Apart from the power operator, there are only two +levels, one for multiplicative operators and one for additive +operators: + + m_expr: u_expr | m_expr "*" u_expr | m_expr "@" m_expr | + m_expr "//" u_expr | m_expr "/" u_expr | + m_expr "%" u_expr + a_expr: m_expr | a_expr "+" m_expr | a_expr "-" m_expr + +The "*" (multiplication) operator yields the product of its arguments. +The arguments must either both be numbers, or one argument must be an +integer and the other must be a sequence. In the former case, the +numbers are converted to a common real type and then multiplied +together. In the latter case, sequence repetition is performed; a +negative repetition factor yields an empty sequence. + +This operation can be customized using the special "__mul__()" and +"__rmul__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. + +The "@" (at) operator is intended to be used for matrix +multiplication. No builtin Python types implement this operator. + +This operation can be customized using the special "__matmul__()" and +"__rmatmul__()" methods. + +Added in version 3.5. + +The "/" (division) and "//" (floor division) operators yield the +quotient of their arguments. The numeric arguments are first +converted to a common type. Division of integers yields a float, while +floor division of integers results in an integer; the result is that +of mathematical division with the ‘floor’ function applied to the +result. Division by zero raises the "ZeroDivisionError" exception. + +The division operation can be customized using the special +"__truediv__()" and "__rtruediv__()" methods. The floor division +operation can be customized using the special "__floordiv__()" and +"__rfloordiv__()" methods. + +The "%" (modulo) operator yields the remainder from the division of +the first argument by the second. The numeric arguments are first +converted to a common type. A zero right argument raises the +"ZeroDivisionError" exception. The arguments may be floating-point +numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals "4*0.7 + +0.34".) The modulo operator always yields a result with the same sign +as its second operand (or zero); the absolute value of the result is +strictly smaller than the absolute value of the second operand [1]. + +The floor division and modulo operators are connected by the following +identity: "x == (x//y)*y + (x%y)". Floor division and modulo are also +connected with the built-in function "divmod()": "divmod(x, y) == +(x//y, x%y)". [2]. + +In addition to performing the modulo operation on numbers, the "%" +operator is also overloaded by string objects to perform old-style +string formatting (also known as interpolation). The syntax for +string formatting is described in the Python Library Reference, +section printf-style String Formatting. + +The *modulo* operation can be customized using the special "__mod__()" +and "__rmod__()" methods. + +The floor division operator, the modulo operator, and the "divmod()" +function are not defined for complex numbers. Instead, convert to a +floating-point number using the "abs()" function if appropriate. + +The "+" (addition) operator yields the sum of its arguments. The +arguments must either both be numbers or both be sequences of the same +type. In the former case, the numbers are converted to a common real +type and then added together. In the latter case, the sequences are +concatenated. + +This operation can be customized using the special "__add__()" and +"__radd__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. + +The "-" (subtraction) operator yields the difference of its arguments. +The numeric arguments are first converted to a common real type. + +This operation can be customized using the special "__sub__()" and +"__rsub__()" methods. + +Changed in version 3.14: If only one operand is a complex number, the +other operand is converted to a floating-point number. +''', + 'bitwise': r'''Binary bitwise operations +************************* + +Each of the three bitwise operations has a different priority level: + + and_expr: shift_expr | and_expr "&" shift_expr + xor_expr: and_expr | xor_expr "^" and_expr + or_expr: xor_expr | or_expr "|" xor_expr + +The "&" operator yields the bitwise AND of its arguments, which must +be integers or one of them must be a custom object overriding +"__and__()" or "__rand__()" special methods. + +The "^" operator yields the bitwise XOR (exclusive OR) of its +arguments, which must be integers or one of them must be a custom +object overriding "__xor__()" or "__rxor__()" special methods. + +The "|" operator yields the bitwise (inclusive) OR of its arguments, +which must be integers or one of them must be a custom object +overriding "__or__()" or "__ror__()" special methods. +''', + 'bltin-code-objects': r'''Code Objects +************ + +Code objects are used by the implementation to represent “pseudo- +compiled” executable Python code such as a function body. They differ +from function objects because they don’t contain a reference to their +global execution environment. Code objects are returned by the built- +in "compile()" function and can be extracted from function objects +through their "__code__" attribute. See also the "code" module. + +Accessing "__code__" raises an auditing event "object.__getattr__" +with arguments "obj" and ""__code__"". + +A code object can be executed or evaluated by passing it (instead of a +source string) to the "exec()" or "eval()" built-in functions. + +See The standard type hierarchy for more information. +''', + 'bltin-ellipsis-object': r'''The Ellipsis Object +******************* + +This object is commonly used to indicate that something is omitted. It +supports no special operations. There is exactly one ellipsis object, +named "Ellipsis" (a built-in name). "type(Ellipsis)()" produces the +"Ellipsis" singleton. + +It is written as "Ellipsis" or "...". + +In typical use, "..." as the "Ellipsis" object appears in a few +different places, for instance: + +* In type annotations, such as callable arguments or tuple elements. + +* As the body of a function instead of a pass statement. + +* In third-party libraries, such as Numpy’s slicing and striding. + +Python also uses three dots in ways that are not "Ellipsis" objects, +for instance: + +* Doctest’s "ELLIPSIS", as a pattern for missing content. + +* The default Python prompt of the *interactive* shell when partial + input is incomplete. + +Lastly, the Python documentation often uses three dots in conventional +English usage to mean omitted content, even in code examples that also +use them as the "Ellipsis". +''', + 'bltin-null-object': r'''The Null Object +*************** + +This object is returned by functions that don’t explicitly return a +value. It supports no special operations. There is exactly one null +object, named "None" (a built-in name). "type(None)()" produces the +same singleton. + +It is written as "None". +''', + 'bltin-type-objects': r'''Type Objects +************ + +Type objects represent the various object types. An object’s type is +accessed by the built-in function "type()". There are no special +operations on types. The standard module "types" defines names for +all standard built-in types. + +Types are written like this: "". +''', + 'booleans': r'''Boolean operations +****************** + + or_test: and_test | or_test "or" and_test + and_test: not_test | and_test "and" not_test + not_test: comparison | "not" not_test + +In the context of Boolean operations, and also when expressions are +used by control flow statements, the following values are interpreted +as false: "False", "None", numeric zero of all types, and empty +strings and containers (including strings, tuples, lists, +dictionaries, sets and frozensets). All other values are interpreted +as true. User-defined objects can customize their truth value by +providing a "__bool__()" method. + +The operator "not" yields "True" if its argument is false, "False" +otherwise. + +The expression "x and y" first evaluates *x*; if *x* is false, its +value is returned; otherwise, *y* is evaluated and the resulting value +is returned. + +The expression "x or y" first evaluates *x*; if *x* is true, its value +is returned; otherwise, *y* is evaluated and the resulting value is +returned. + +Note that neither "and" nor "or" restrict the value and type they +return to "False" and "True", but rather return the last evaluated +argument. This is sometimes useful, e.g., if "s" is a string that +should be replaced by a default value if it is empty, the expression +"s or 'foo'" yields the desired value. Because "not" has to create a +new value, it returns a boolean value regardless of the type of its +argument (for example, "not 'foo'" produces "False" rather than "''".) +''', + 'break': r'''The "break" statement +********************* + + break_stmt: "break" + +"break" may only occur syntactically nested in a "for" or "while" +loop, but not nested in a function or class definition within that +loop. + +It terminates the nearest enclosing loop, skipping the optional "else" +clause if the loop has one. + +If a "for" loop is terminated by "break", the loop control target +keeps its current value. + +When "break" passes control out of a "try" statement with a "finally" +clause, that "finally" clause is executed before really leaving the +loop. +''', + 'callable-types': r'''Emulating callable objects +************************** + +object.__call__(self[, args...]) + + Called when the instance is “called” as a function; if this method + is defined, "x(arg1, arg2, ...)" roughly translates to + "type(x).__call__(x, arg1, ...)". The "object" class itself does + not provide this method. +''', + 'calls': r'''Calls +***** + +A call calls a callable object (e.g., a *function*) with a possibly +empty series of *arguments*: + + call: primary "(" [argument_list [","] | comprehension] ")" + argument_list: positional_arguments ["," starred_and_keywords] + ["," keywords_arguments] + | starred_and_keywords ["," keywords_arguments] + | keywords_arguments + positional_arguments: positional_item ("," positional_item)* + positional_item: assignment_expression | "*" expression + starred_and_keywords: ("*" expression | keyword_item) + ("," "*" expression | "," keyword_item)* + keywords_arguments: (keyword_item | "**" expression) + ("," keyword_item | "," "**" expression)* + keyword_item: identifier "=" expression + +An optional trailing comma may be present after the positional and +keyword arguments but does not affect the semantics. + +The primary must evaluate to a callable object (user-defined +functions, built-in functions, methods of built-in objects, class +objects, methods of class instances, and all objects having a +"__call__()" method are callable). All argument expressions are +evaluated before the call is attempted. Please refer to section +Function definitions for the syntax of formal *parameter* lists. + +If keyword arguments are present, they are first converted to +positional arguments, as follows. First, a list of unfilled slots is +created for the formal parameters. If there are N positional +arguments, they are placed in the first N slots. Next, for each +keyword argument, the identifier is used to determine the +corresponding slot (if the identifier is the same as the first formal +parameter name, the first slot is used, and so on). If the slot is +already filled, a "TypeError" exception is raised. Otherwise, the +argument is placed in the slot, filling it (even if the expression is +"None", it fills the slot). When all arguments have been processed, +the slots that are still unfilled are filled with the corresponding +default value from the function definition. (Default values are +calculated, once, when the function is defined; thus, a mutable object +such as a list or dictionary used as default value will be shared by +all calls that don’t specify an argument value for the corresponding +slot; this should usually be avoided.) If there are any unfilled +slots for which no default value is specified, a "TypeError" exception +is raised. Otherwise, the list of filled slots is used as the +argument list for the call. + +**CPython implementation detail:** An implementation may provide +built-in functions whose positional parameters do not have names, even +if they are ‘named’ for the purpose of documentation, and which +therefore cannot be supplied by keyword. In CPython, this is the case +for functions implemented in C that use "PyArg_ParseTuple()" to parse +their arguments. + +If there are more positional arguments than there are formal parameter +slots, a "TypeError" exception is raised, unless a formal parameter +using the syntax "*identifier" is present; in this case, that formal +parameter receives a tuple containing the excess positional arguments +(or an empty tuple if there were no excess positional arguments). + +If any keyword argument does not correspond to a formal parameter +name, a "TypeError" exception is raised, unless a formal parameter +using the syntax "**identifier" is present; in this case, that formal +parameter receives a dictionary containing the excess keyword +arguments (using the keywords as keys and the argument values as +corresponding values), or a (new) empty dictionary if there were no +excess keyword arguments. + +If the syntax "*expression" appears in the function call, "expression" +must evaluate to an *iterable*. Elements from these iterables are +treated as if they were additional positional arguments. For the call +"f(x1, x2, *y, x3, x4)", if *y* evaluates to a sequence *y1*, …, *yM*, +this is equivalent to a call with M+4 positional arguments *x1*, *x2*, +*y1*, …, *yM*, *x3*, *x4*. + +A consequence of this is that although the "*expression" syntax may +appear *after* explicit keyword arguments, it is processed *before* +the keyword arguments (and any "**expression" arguments – see below). +So: + + >>> def f(a, b): + ... print(a, b) + ... + >>> f(b=1, *(2,)) + 2 1 + >>> f(a=1, *(2,)) + Traceback (most recent call last): + File "", line 1, in + TypeError: f() got multiple values for keyword argument 'a' + >>> f(1, *(2,)) + 1 2 + +It is unusual for both keyword arguments and the "*expression" syntax +to be used in the same call, so in practice this confusion does not +often arise. + +If the syntax "**expression" appears in the function call, +"expression" must evaluate to a *mapping*, the contents of which are +treated as additional keyword arguments. If a parameter matching a key +has already been given a value (by an explicit keyword argument, or +from another unpacking), a "TypeError" exception is raised. + +When "**expression" is used, each key in this mapping must be a +string. Each value from the mapping is assigned to the first formal +parameter eligible for keyword assignment whose name is equal to the +key. A key need not be a Python identifier (e.g. ""max-temp °F"" is +acceptable, although it will not match any formal parameter that could +be declared). If there is no match to a formal parameter the key-value +pair is collected by the "**" parameter, if there is one, or if there +is not, a "TypeError" exception is raised. + +Formal parameters using the syntax "*identifier" or "**identifier" +cannot be used as positional argument slots or as keyword argument +names. + +Changed in version 3.5: Function calls accept any number of "*" and +"**" unpackings, positional arguments may follow iterable unpackings +("*"), and keyword arguments may follow dictionary unpackings ("**"). +Originally proposed by **PEP 448**. + +A call always returns some value, possibly "None", unless it raises an +exception. How this value is computed depends on the type of the +callable object. + +If it is— + +a user-defined function: + The code block for the function is executed, passing it the + argument list. The first thing the code block will do is bind the + formal parameters to the arguments; this is described in section + Function definitions. When the code block executes a "return" + statement, this specifies the return value of the function call. + If execution reaches the end of the code block without executing a + "return" statement, the return value is "None". + +a built-in function or method: + The result is up to the interpreter; see Built-in Functions for the + descriptions of built-in functions and methods. + +a class object: + A new instance of that class is returned. + +a class instance method: + The corresponding user-defined function is called, with an argument + list that is one longer than the argument list of the call: the + instance becomes the first argument. + +a class instance: + The class must define a "__call__()" method; the effect is then the + same as if that method was called. +''', + 'class': r'''Class definitions +***************** + +A class definition defines a class object (see section The standard +type hierarchy): + + classdef: [decorators] "class" classname [type_params] [inheritance] ":" suite + inheritance: "(" [argument_list] ")" + classname: identifier + +A class definition is an executable statement. The inheritance list +usually gives a list of base classes (see Metaclasses for more +advanced uses), so each item in the list should evaluate to a class +object which allows subclassing. Classes without an inheritance list +inherit, by default, from the base class "object"; hence, + + class Foo: + pass + +is equivalent to + + class Foo(object): + pass + +The class’s suite is then executed in a new execution frame (see +Naming and binding), using a newly created local namespace and the +original global namespace. (Usually, the suite contains mostly +function definitions.) When the class’s suite finishes execution, its +execution frame is discarded but its local namespace is saved. [5] A +class object is then created using the inheritance list for the base +classes and the saved local namespace for the attribute dictionary. +The class name is bound to this class object in the original local +namespace. + +The order in which attributes are defined in the class body is +preserved in the new class’s "__dict__". Note that this is reliable +only right after the class is created and only for classes that were +defined using the definition syntax. + +Class creation can be customized heavily using metaclasses. + +Classes can also be decorated: just like when decorating functions, + + @f1(arg) + @f2 + class Foo: pass + +is roughly equivalent to + + class Foo: pass + Foo = f1(arg)(f2(Foo)) + +The evaluation rules for the decorator expressions are the same as for +function decorators. The result is then bound to the class name. + +Changed in version 3.9: Classes may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets immediately +after the class’s name. This indicates to static type checkers that +the class is generic. At runtime, the type parameters can be retrieved +from the class’s "__type_params__" attribute. See Generic classes for +more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +**Programmer’s note:** Variables defined in the class definition are +class attributes; they are shared by instances. Instance attributes +can be set in a method with "self.name = value". Both class and +instance attributes are accessible through the notation “"self.name"”, +and an instance attribute hides a class attribute with the same name +when accessed in this way. Class attributes can be used as defaults +for instance attributes, but using mutable values there can lead to +unexpected results. Descriptors can be used to create instance +variables with different implementation details. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + The proposal that changed the declaration of metaclasses to the + current syntax, and the semantics for how classes with + metaclasses are constructed. + + **PEP 3129** - Class Decorators + The proposal that added class decorators. Function and method + decorators were introduced in **PEP 318**. +''', + 'comparisons': r'''Comparisons +*********** + +Unlike C, all comparison operations in Python have the same priority, +which is lower than that of any arithmetic, shifting or bitwise +operation. Also unlike C, expressions like "a < b < c" have the +interpretation that is conventional in mathematics: + + comparison: or_expr (comp_operator or_expr)* + comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" + | "is" ["not"] | ["not"] "in" + +Comparisons yield boolean values: "True" or "False". Custom *rich +comparison methods* may return non-boolean values. In this case Python +will call "bool()" on such value in boolean contexts. + +Comparisons can be chained arbitrarily, e.g., "x < y <= z" is +equivalent to "x < y and y <= z", except that "y" is evaluated only +once (but in both cases "z" is not evaluated at all when "x < y" is +found to be false). + +Formally, if *a*, *b*, *c*, …, *y*, *z* are expressions and *op1*, +*op2*, …, *opN* are comparison operators, then "a op1 b op2 c ... y +opN z" is equivalent to "a op1 b and b op2 c and ... y opN z", except +that each expression is evaluated at most once. + +Note that "a op1 b op2 c" doesn’t imply any kind of comparison between +*a* and *c*, so that, e.g., "x < y > z" is perfectly legal (though +perhaps not pretty). + + +Value comparisons +================= + +The operators "<", ">", "==", ">=", "<=", and "!=" compare the values +of two objects. The objects do not need to have the same type. + +Chapter Objects, values and types states that objects have a value (in +addition to type and identity). The value of an object is a rather +abstract notion in Python: For example, there is no canonical access +method for an object’s value. Also, there is no requirement that the +value of an object should be constructed in a particular way, e.g. +comprised of all its data attributes. Comparison operators implement a +particular notion of what the value of an object is. One can think of +them as defining the value of an object indirectly, by means of their +comparison implementation. + +Because all types are (direct or indirect) subtypes of "object", they +inherit the default comparison behavior from "object". Types can +customize their comparison behavior by implementing *rich comparison +methods* like "__lt__()", described in Basic customization. + +The default behavior for equality comparison ("==" and "!=") is based +on the identity of the objects. Hence, equality comparison of +instances with the same identity results in equality, and equality +comparison of instances with different identities results in +inequality. A motivation for this default behavior is the desire that +all objects should be reflexive (i.e. "x is y" implies "x == y"). + +A default order comparison ("<", ">", "<=", and ">=") is not provided; +an attempt raises "TypeError". A motivation for this default behavior +is the lack of a similar invariant as for equality. + +The behavior of the default equality comparison, that instances with +different identities are always unequal, may be in contrast to what +types will need that have a sensible definition of object value and +value-based equality. Such types will need to customize their +comparison behavior, and in fact, a number of built-in types have done +that. + +The following list describes the comparison behavior of the most +important built-in types. + +* Numbers of built-in numeric types (Numeric Types — int, float, + complex) and of the standard library types "fractions.Fraction" and + "decimal.Decimal" can be compared within and across their types, + with the restriction that complex numbers do not support order + comparison. Within the limits of the types involved, they compare + mathematically (algorithmically) correct without loss of precision. + + The not-a-number values "float('NaN')" and "decimal.Decimal('NaN')" + are special. Any ordered comparison of a number to a not-a-number + value is false. A counter-intuitive implication is that not-a-number + values are not equal to themselves. For example, if "x = + float('NaN')", "3 < x", "x < 3" and "x == x" are all false, while "x + != x" is true. This behavior is compliant with IEEE 754. + +* "None" and "NotImplemented" are singletons. **PEP 8** advises that + comparisons for singletons should always be done with "is" or "is + not", never the equality operators. + +* Binary sequences (instances of "bytes" or "bytearray") can be + compared within and across their types. They compare + lexicographically using the numeric values of their elements. + +* Strings (instances of "str") compare lexicographically using the + numerical Unicode code points (the result of the built-in function + "ord()") of their characters. [3] + + Strings and binary sequences cannot be directly compared. + +* Sequences (instances of "tuple", "list", or "range") can be compared + only within each of their types, with the restriction that ranges do + not support order comparison. Equality comparison across these + types results in inequality, and ordering comparison across these + types raises "TypeError". + + Sequences compare lexicographically using comparison of + corresponding elements. The built-in containers typically assume + identical objects are equal to themselves. That lets them bypass + equality tests for identical objects to improve performance and to + maintain their internal invariants. + + Lexicographical comparison between built-in collections works as + follows: + + * For two collections to compare equal, they must be of the same + type, have the same length, and each pair of corresponding + elements must compare equal (for example, "[1,2] == (1,2)" is + false because the type is not the same). + + * Collections that support order comparison are ordered the same as + their first unequal elements (for example, "[1,2,x] <= [1,2,y]" + has the same value as "x <= y"). If a corresponding element does + not exist, the shorter collection is ordered first (for example, + "[1,2] < [1,2,3]" is true). + +* Mappings (instances of "dict") compare equal if and only if they + have equal "(key, value)" pairs. Equality comparison of the keys and + values enforces reflexivity. + + Order comparisons ("<", ">", "<=", and ">=") raise "TypeError". + +* Sets (instances of "set" or "frozenset") can be compared within and + across their types. + + They define order comparison operators to mean subset and superset + tests. Those relations do not define total orderings (for example, + the two sets "{1,2}" and "{2,3}" are not equal, nor subsets of one + another, nor supersets of one another). Accordingly, sets are not + appropriate arguments for functions which depend on total ordering + (for example, "min()", "max()", and "sorted()" produce undefined + results given a list of sets as inputs). + + Comparison of sets enforces reflexivity of its elements. + +* Most other built-in types have no comparison methods implemented, so + they inherit the default comparison behavior. + +User-defined classes that customize their comparison behavior should +follow some consistency rules, if possible: + +* Equality comparison should be reflexive. In other words, identical + objects should compare equal: + + "x is y" implies "x == y" + +* Comparison should be symmetric. In other words, the following + expressions should have the same result: + + "x == y" and "y == x" + + "x != y" and "y != x" + + "x < y" and "y > x" + + "x <= y" and "y >= x" + +* Comparison should be transitive. The following (non-exhaustive) + examples illustrate that: + + "x > y and y > z" implies "x > z" + + "x < y and y <= z" implies "x < z" + +* Inverse comparison should result in the boolean negation. In other + words, the following expressions should have the same result: + + "x == y" and "not x != y" + + "x < y" and "not x >= y" (for total ordering) + + "x > y" and "not x <= y" (for total ordering) + + The last two expressions apply to totally ordered collections (e.g. + to sequences, but not to sets or mappings). See also the + "total_ordering()" decorator. + +* The "hash()" result should be consistent with equality. Objects that + are equal should either have the same hash value, or be marked as + unhashable. + +Python does not enforce these consistency rules. In fact, the +not-a-number values are an example for not following these rules. + + +Membership test operations +========================== + +The operators "in" and "not in" test for membership. "x in s" +evaluates to "True" if *x* is a member of *s*, and "False" otherwise. +"x not in s" returns the negation of "x in s". All built-in sequences +and set types support this as well as dictionary, for which "in" tests +whether the dictionary has a given key. For container types such as +list, tuple, set, frozenset, dict, or collections.deque, the +expression "x in y" is equivalent to "any(x is e or x == e for e in +y)". + +For the string and bytes types, "x in y" is "True" if and only if *x* +is a substring of *y*. An equivalent test is "y.find(x) != -1". +Empty strings are always considered to be a substring of any other +string, so """ in "abc"" will return "True". + +For user-defined classes which define the "__contains__()" method, "x +in y" returns "True" if "y.__contains__(x)" returns a true value, and +"False" otherwise. + +For user-defined classes which do not define "__contains__()" but do +define "__iter__()", "x in y" is "True" if some value "z", for which +the expression "x is z or x == z" is true, is produced while iterating +over "y". If an exception is raised during the iteration, it is as if +"in" raised that exception. + +Lastly, the old-style iteration protocol is tried: if a class defines +"__getitem__()", "x in y" is "True" if and only if there is a non- +negative integer index *i* such that "x is y[i] or x == y[i]", and no +lower integer index raises the "IndexError" exception. (If any other +exception is raised, it is as if "in" raised that exception). + +The operator "not in" is defined to have the inverse truth value of +"in". + + +Identity comparisons +==================== + +The operators "is" and "is not" test for an object’s identity: "x is +y" is true if and only if *x* and *y* are the same object. An +Object’s identity is determined using the "id()" function. "x is not +y" yields the inverse truth value. [4] +''', + 'compound': r'''Compound statements +******************* + +Compound statements contain (groups of) other statements; they affect +or control the execution of those other statements in some way. In +general, compound statements span multiple lines, although in simple +incarnations a whole compound statement may be contained in one line. + +The "if", "while" and "for" statements implement traditional control +flow constructs. "try" specifies exception handlers and/or cleanup +code for a group of statements, while the "with" statement allows the +execution of initialization and finalization code around a block of +code. Function and class definitions are also syntactically compound +statements. + +A compound statement consists of one or more ‘clauses.’ A clause +consists of a header and a ‘suite.’ The clause headers of a +particular compound statement are all at the same indentation level. +Each clause header begins with a uniquely identifying keyword and ends +with a colon. A suite is a group of statements controlled by a +clause. A suite can be one or more semicolon-separated simple +statements on the same line as the header, following the header’s +colon, or it can be one or more indented statements on subsequent +lines. Only the latter form of a suite can contain nested compound +statements; the following is illegal, mostly because it wouldn’t be +clear to which "if" clause a following "else" clause would belong: + + if test1: if test2: print(x) + +Also note that the semicolon binds tighter than the colon in this +context, so that in the following example, either all or none of the +"print()" calls are executed: + + if x < y < z: print(x); print(y); print(z) + +Summarizing: + + compound_stmt: if_stmt + | while_stmt + | for_stmt + | try_stmt + | with_stmt + | match_stmt + | funcdef + | classdef + | async_with_stmt + | async_for_stmt + | async_funcdef + suite: stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT + statement: stmt_list NEWLINE | compound_stmt + stmt_list: simple_stmt (";" simple_stmt)* [";"] + +Note that statements always end in a "NEWLINE" possibly followed by a +"DEDENT". Also note that optional continuation clauses always begin +with a keyword that cannot start a statement, thus there are no +ambiguities (the ‘dangling "else"’ problem is solved in Python by +requiring nested "if" statements to be indented). + +The formatting of the grammar rules in the following sections places +each clause on a separate line for clarity. + + +The "if" statement +================== + +The "if" statement is used for conditional execution: + + if_stmt: "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. + + +The "while" statement +===================== + +The "while" statement is used for repeated execution as long as an +expression is true: + + while_stmt: "while" assignment_expression ":" suite + ["else" ":" suite] + +This repeatedly tests the expression and, if it is true, executes the +first suite; if the expression is false (which may be the first time +it is tested) the suite of the "else" clause, if present, is executed +and the loop terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and goes back +to testing the expression. + + +The "for" statement +=================== + +The "for" statement is used to iterate over the elements of a sequence +(such as a string, tuple or list) or other iterable object: + + for_stmt: "for" target_list "in" starred_expression_list ":" suite + ["else" ":" suite] + +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop +terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and continues +with the next item, or with the "else" clause if there is no next +item. + +The for-loop makes assignments to the variables in the target list. +This overwrites all previous assignments to those variables including +those made in the suite of the for-loop: + + for i in range(10): + print(i) + i = 5 # this will not affect the for-loop + # because i will be overwritten with the next + # index in the range + +Names in the target list are not deleted when the loop is finished, +but if the sequence is empty, they will not have been assigned to at +all by the loop. Hint: the built-in type "range()" represents +immutable arithmetic sequences of integers. For instance, iterating +"range(3)" successively yields 0, 1, and then 2. + +Changed in version 3.11: Starred elements are now allowed in the +expression list. + + +The "try" statement +=================== + +The "try" statement specifies exception handlers and/or cleanup code +for a group of statements: + + try_stmt: try1_stmt | try2_stmt | try3_stmt + try1_stmt: "try" ":" suite + ("except" [expression ["as" identifier]] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try2_stmt: "try" ":" suite + ("except" "*" expression ["as" identifier] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try3_stmt: "try" ":" suite + "finally" ":" suite + +Additional information on exceptions can be found in section +Exceptions, and information on using the "raise" statement to generate +exceptions may be found in section The raise statement. + +Changed in version 3.14: Support for optionally dropping grouping +parentheses when using multiple exception types. See **PEP 758**. + + +"except" clause +--------------- + +The "except" clause(s) specify one or more exception handlers. When no +exception occurs in the "try" clause, no exception handler is +executed. When an exception occurs in the "try" suite, a search for an +exception handler is started. This search inspects the "except" +clauses in turn until one is found that matches the exception. An +expression-less "except" clause, if present, must be last; it matches +any exception. + +For an "except" clause with an expression, the expression must +evaluate to an exception type or a tuple of exception types. +Parentheses can be dropped if multiple exception types are provided +and the "as" clause is not used. The raised exception matches an +"except" clause whose expression evaluates to the class or a *non- +virtual base class* of the exception object, or to a tuple that +contains such a class. + +If no "except" clause matches the exception, the search for an +exception handler continues in the surrounding code and on the +invocation stack. [1] + +If the evaluation of an expression in the header of an "except" clause +raises an exception, the original search for a handler is canceled and +a search starts for the new exception in the surrounding code and on +the call stack (it is treated as if the entire "try" statement raised +the exception). + +When a matching "except" clause is found, the exception is assigned to +the target specified after the "as" keyword in that "except" clause, +if present, and the "except" clause’s suite is executed. All "except" +clauses must have an executable block. When the end of this block is +reached, execution continues normally after the entire "try" +statement. (This means that if two nested handlers exist for the same +exception, and the exception occurs in the "try" clause of the inner +handler, the outer handler will not handle the exception.) + +When an exception has been assigned using "as target", it is cleared +at the end of the "except" clause. This is as if + + except E as N: + foo + +was translated to + + except E as N: + try: + foo + finally: + del N + +This means the exception must be assigned to a different name to be +able to refer to it after the "except" clause. Exceptions are cleared +because with the traceback attached to them, they form a reference +cycle with the stack frame, keeping all locals in that frame alive +until the next garbage collection occurs. + +Before an "except" clause’s suite is executed, the exception is stored +in the "sys" module, where it can be accessed from within the body of +the "except" clause by calling "sys.exception()". When leaving an +exception handler, the exception stored in the "sys" module is reset +to its previous value: + + >>> print(sys.exception()) + None + >>> try: + ... raise TypeError + ... except: + ... print(repr(sys.exception())) + ... try: + ... raise ValueError + ... except: + ... print(repr(sys.exception())) + ... print(repr(sys.exception())) + ... + TypeError() + ValueError() + TypeError() + >>> print(sys.exception()) + None + + +"except*" clause +---------------- + +The "except*" clause(s) specify one or more handlers for groups of +exceptions ("BaseExceptionGroup" instances). A "try" statement can +have either "except" or "except*" clauses, but not both. The exception +type for matching is mandatory in the case of "except*", so "except*:" +is a syntax error. The type is interpreted as in the case of "except", +but matching is performed on the exceptions contained in the group +that is being handled. An "TypeError" is raised if a matching type is +a subclass of "BaseExceptionGroup", because that would have ambiguous +semantics. + +When an exception group is raised in the try block, each "except*" +clause splits (see "split()") it into the subgroups of matching and +non-matching exceptions. If the matching subgroup is not empty, it +becomes the handled exception (the value returned from +"sys.exception()") and assigned to the target of the "except*" clause +(if there is one). Then, the body of the "except*" clause executes. If +the non-matching subgroup is not empty, it is processed by the next +"except*" in the same manner. This continues until all exceptions in +the group have been matched, or the last "except*" clause has run. + +After all "except*" clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from +within "except*" clauses. This merged exception group propagates on.: + + >>> try: + ... raise ExceptionGroup("eg", + ... [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + ... except* TypeError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... except* OSError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... + caught with nested (TypeError(2),) + caught with nested (OSError(3), OSError(4)) + + Exception Group Traceback (most recent call last): + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: 1 + +------------------------------------ + +If the exception raised from the "try" block is not an exception group +and its type matches one of the "except*" clauses, it is caught and +wrapped by an exception group with an empty message string. This +ensures that the type of the target "e" is consistently +"BaseExceptionGroup": + + >>> try: + ... raise BlockingIOError + ... except* BlockingIOError as e: + ... print(repr(e)) + ... + ExceptionGroup('', (BlockingIOError(),)) + +"break", "continue" and "return" cannot appear in an "except*" clause. + + +"else" clause +------------- + +The optional "else" clause is executed if the control flow leaves the +"try" suite, no exception was raised, and no "return", "continue", or +"break" statement was executed. Exceptions in the "else" clause are +not handled by the preceding "except" clauses. + + +"finally" clause +---------------- + +If "finally" is present, it specifies a ‘cleanup’ handler. The "try" +clause is executed, including any "except" and "else" clauses. If an +exception occurs in any of the clauses and is not handled, the +exception is temporarily saved. The "finally" clause is executed. If +there is a saved exception it is re-raised at the end of the "finally" +clause. If the "finally" clause raises another exception, the saved +exception is set as the context of the new exception. If the "finally" +clause executes a "return", "break" or "continue" statement, the saved +exception is discarded. For example, this function returns 42. + + def f(): + try: + 1/0 + finally: + return 42 + +The exception information is not available to the program during +execution of the "finally" clause. + +When a "return", "break" or "continue" statement is executed in the +"try" suite of a "try"…"finally" statement, the "finally" clause is +also executed ‘on the way out.’ + +The return value of a function is determined by the last "return" +statement executed. Since the "finally" clause always executes, a +"return" statement executed in the "finally" clause will always be the +last one executed. The following function returns ‘finally’. + + def foo(): + try: + return 'try' + finally: + return 'finally' + +Changed in version 3.8: Prior to Python 3.8, a "continue" statement +was illegal in the "finally" clause due to a problem with the +implementation. + +Changed in version 3.14: The compiler emits a "SyntaxWarning" when a +"return", "break" or "continue" appears in a "finally" block (see +**PEP 765**). + + +The "with" statement +==================== + +The "with" statement is used to wrap the execution of a block with +methods defined by a context manager (see section With Statement +Context Managers). This allows common "try"…"except"…"finally" usage +patterns to be encapsulated for convenient reuse. + + with_stmt: "with" ( "(" with_stmt_contents ","? ")" | with_stmt_contents ) ":" suite + with_stmt_contents: with_item ("," with_item)* + with_item: expression ["as" target] + +The execution of the "with" statement with one “item” proceeds as +follows: + +1. The context expression (the expression given in the "with_item") is + evaluated to obtain a context manager. + +2. The context manager’s "__enter__()" is loaded for later use. + +3. The context manager’s "__exit__()" is loaded for later use. + +4. The context manager’s "__enter__()" method is invoked. + +5. If a target was included in the "with" statement, the return value + from "__enter__()" is assigned to it. + + Note: + + The "with" statement guarantees that if the "__enter__()" method + returns without an error, then "__exit__()" will always be + called. Thus, if an error occurs during the assignment to the + target list, it will be treated the same as an error occurring + within the suite would be. See step 7 below. + +6. The suite is executed. + +7. The context manager’s "__exit__()" method is invoked. If an + exception caused the suite to be exited, its type, value, and + traceback are passed as arguments to "__exit__()". Otherwise, three + "None" arguments are supplied. + + If the suite was exited due to an exception, and the return value + from the "__exit__()" method was false, the exception is reraised. + If the return value was true, the exception is suppressed, and + execution continues with the statement following the "with" + statement. + + If the suite was exited for any reason other than an exception, the + return value from "__exit__()" is ignored, and execution proceeds + at the normal location for the kind of exit that was taken. + +The following code: + + with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + enter = type(manager).__enter__ + exit = type(manager).__exit__ + value = enter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not exit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + exit(manager, None, None, None) + +With more than one item, the context managers are processed as if +multiple "with" statements were nested: + + with A() as a, B() as b: + SUITE + +is semantically equivalent to: + + with A() as a: + with B() as b: + SUITE + +You can also write multi-item context managers in multiple lines if +the items are surrounded by parentheses. For example: + + with ( + A() as a, + B() as b, + ): + SUITE + +Changed in version 3.1: Support for multiple context expressions. + +Changed in version 3.10: Support for using grouping parentheses to +break the statement in multiple lines. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. + + +The "match" statement +===================== + +Added in version 3.10. + +The match statement is used for pattern matching. Syntax: + + match_stmt: 'match' subject_expr ":" NEWLINE INDENT case_block+ DEDENT + subject_expr: `!star_named_expression` "," `!star_named_expressions`? + | `!named_expression` + case_block: 'case' patterns [guard] ":" `!block` + +Note: + + This section uses single quotes to denote soft keywords. + +Pattern matching takes a pattern as input (following "case") and a +subject value (following "match"). The pattern (which may contain +subpatterns) is matched against the subject value. The outcomes are: + +* A match success or failure (also termed a pattern success or + failure). + +* Possible binding of matched values to a name. The prerequisites for + this are further discussed below. + +The "match" and "case" keywords are soft keywords. + +See also: + + * **PEP 634** – Structural Pattern Matching: Specification + + * **PEP 636** – Structural Pattern Matching: Tutorial + + +Overview +-------- + +Here’s an overview of the logical flow of a match statement: + +1. The subject expression "subject_expr" is evaluated and a resulting + subject value obtained. If the subject expression contains a comma, + a tuple is constructed using the standard rules. + +2. Each pattern in a "case_block" is attempted to match with the + subject value. The specific rules for success or failure are + described below. The match attempt can also bind some or all of the + standalone names within the pattern. The precise pattern binding + rules vary per pattern type and are specified below. **Name + bindings made during a successful pattern match outlive the + executed block and can be used after the match statement**. + + Note: + + During failed pattern matches, some subpatterns may succeed. Do + not rely on bindings being made for a failed match. Conversely, + do not rely on variables remaining unchanged after a failed + match. The exact behavior is dependent on implementation and may + vary. This is an intentional decision made to allow different + implementations to add optimizations. + +3. If the pattern succeeds, the corresponding guard (if present) is + evaluated. In this case all name bindings are guaranteed to have + happened. + + * If the guard evaluates as true or is missing, the "block" inside + "case_block" is executed. + + * Otherwise, the next "case_block" is attempted as described above. + + * If there are no further case blocks, the match statement is + completed. + +Note: + + Users should generally never rely on a pattern being evaluated. + Depending on implementation, the interpreter may cache values or use + other optimizations which skip repeated evaluations. + +A sample match statement: + + >>> flag = False + >>> match (100, 200): + ... case (100, 300): # Mismatch: 200 != 300 + ... print('Case 1') + ... case (100, 200) if flag: # Successful match, but guard fails + ... print('Case 2') + ... case (100, y): # Matches and binds y to 200 + ... print(f'Case 3, y: {y}') + ... case _: # Pattern not attempted + ... print('Case 4, I match anything!') + ... + Case 3, y: 200 + +In this case, "if flag" is a guard. Read more about that in the next +section. + + +Guards +------ + + guard: "if" `!named_expression` + +A "guard" (which is part of the "case") must succeed for code inside +the "case" block to execute. It takes the form: "if" followed by an +expression. + +The logical flow of a "case" block with a "guard" follows: + +1. Check that the pattern in the "case" block succeeded. If the + pattern failed, the "guard" is not evaluated and the next "case" + block is checked. + +2. If the pattern succeeded, evaluate the "guard". + + * If the "guard" condition evaluates as true, the case block is + selected. + + * If the "guard" condition evaluates as false, the case block is + not selected. + + * If the "guard" raises an exception during evaluation, the + exception bubbles up. + +Guards are allowed to have side effects as they are expressions. +Guard evaluation must proceed from the first to the last case block, +one at a time, skipping case blocks whose pattern(s) don’t all +succeed. (I.e., guard evaluation must happen in order.) Guard +evaluation must stop once a case block is selected. + + +Irrefutable Case Blocks +----------------------- + +An irrefutable case block is a match-all case block. A match +statement may have at most one irrefutable case block, and it must be +last. + +A case block is considered irrefutable if it has no guard and its +pattern is irrefutable. A pattern is considered irrefutable if we can +prove from its syntax alone that it will always succeed. Only the +following patterns are irrefutable: + +* AS Patterns whose left-hand side is irrefutable + +* OR Patterns containing at least one irrefutable pattern + +* Capture Patterns + +* Wildcard Patterns + +* parenthesized irrefutable patterns + + +Patterns +-------- + +Note: + + This section uses grammar notations beyond standard EBNF: + + * the notation "SEP.RULE+" is shorthand for "RULE (SEP RULE)*" + + * the notation "!RULE" is shorthand for a negative lookahead + assertion + +The top-level syntax for "patterns" is: + + patterns: open_sequence_pattern | pattern + pattern: as_pattern | or_pattern + closed_pattern: | literal_pattern + | capture_pattern + | wildcard_pattern + | value_pattern + | group_pattern + | sequence_pattern + | mapping_pattern + | class_pattern + +The descriptions below will include a description “in simple terms” of +what a pattern does for illustration purposes (credits to Raymond +Hettinger for a document that inspired most of the descriptions). Note +that these descriptions are purely for illustration purposes and **may +not** reflect the underlying implementation. Furthermore, they do not +cover all valid forms. + + +OR Patterns +~~~~~~~~~~~ + +An OR pattern is two or more patterns separated by vertical bars "|". +Syntax: + + or_pattern: "|".closed_pattern+ + +Only the final subpattern may be irrefutable, and each subpattern must +bind the same set of names to avoid ambiguity. + +An OR pattern matches each of its subpatterns in turn to the subject +value, until one succeeds. The OR pattern is then considered +successful. Otherwise, if none of the subpatterns succeed, the OR +pattern fails. + +In simple terms, "P1 | P2 | ..." will try to match "P1", if it fails +it will try to match "P2", succeeding immediately if any succeeds, +failing otherwise. + + +AS Patterns +~~~~~~~~~~~ + +An AS pattern matches an OR pattern on the left of the "as" keyword +against a subject. Syntax: + + as_pattern: or_pattern "as" capture_pattern + +If the OR pattern fails, the AS pattern fails. Otherwise, the AS +pattern binds the subject to the name on the right of the as keyword +and succeeds. "capture_pattern" cannot be a "_". + +In simple terms "P as NAME" will match with "P", and on success it +will set "NAME = ". + + +Literal Patterns +~~~~~~~~~~~~~~~~ + +A literal pattern corresponds to most literals in Python. Syntax: + + literal_pattern: signed_number + | signed_number "+" NUMBER + | signed_number "-" NUMBER + | strings + | "None" + | "True" + | "False" + signed_number: ["-"] NUMBER + +The rule "strings" and the token "NUMBER" are defined in the standard +Python grammar. Triple-quoted strings are supported. Raw strings and +byte strings are supported. f-strings and t-strings are not +supported. + +The forms "signed_number '+' NUMBER" and "signed_number '-' NUMBER" +are for expressing complex numbers; they require a real number on the +left and an imaginary number on the right. E.g. "3 + 4j". + +In simple terms, "LITERAL" will succeed only if " == +LITERAL". For the singletons "None", "True" and "False", the "is" +operator is used. + + +Capture Patterns +~~~~~~~~~~~~~~~~ + +A capture pattern binds the subject value to a name. Syntax: + + capture_pattern: !'_' NAME + +A single underscore "_" is not a capture pattern (this is what "!'_'" +expresses). It is instead treated as a "wildcard_pattern". + +In a given pattern, a given name can only be bound once. E.g. "case +x, x: ..." is invalid while "case [x] | x: ..." is allowed. + +Capture patterns always succeed. The binding follows scoping rules +established by the assignment expression operator in **PEP 572**; the +name becomes a local variable in the closest containing function scope +unless there’s an applicable "global" or "nonlocal" statement. + +In simple terms "NAME" will always succeed and it will set "NAME = +". + + +Wildcard Patterns +~~~~~~~~~~~~~~~~~ + +A wildcard pattern always succeeds (matches anything) and binds no +name. Syntax: + + wildcard_pattern: '_' + +"_" is a soft keyword within any pattern, but only within patterns. +It is an identifier, as usual, even within "match" subject +expressions, "guard"s, and "case" blocks. + +In simple terms, "_" will always succeed. + + +Value Patterns +~~~~~~~~~~~~~~ + +A value pattern represents a named value in Python. Syntax: + + value_pattern: attr + attr: name_or_attr "." NAME + name_or_attr: attr | NAME + +The dotted name in the pattern is looked up using standard Python name +resolution rules. The pattern succeeds if the value found compares +equal to the subject value (using the "==" equality operator). + +In simple terms "NAME1.NAME2" will succeed only if " == +NAME1.NAME2" + +Note: + + If the same value occurs multiple times in the same match statement, + the interpreter may cache the first value found and reuse it rather + than repeat the same lookup. This cache is strictly tied to a given + execution of a given match statement. + + +Group Patterns +~~~~~~~~~~~~~~ + +A group pattern allows users to add parentheses around patterns to +emphasize the intended grouping. Otherwise, it has no additional +syntax. Syntax: + + group_pattern: "(" pattern ")" + +In simple terms "(P)" has the same effect as "P". + + +Sequence Patterns +~~~~~~~~~~~~~~~~~ + +A sequence pattern contains several subpatterns to be matched against +sequence elements. The syntax is similar to the unpacking of a list or +tuple. + + sequence_pattern: "[" [maybe_sequence_pattern] "]" + | "(" [open_sequence_pattern] ")" + open_sequence_pattern: maybe_star_pattern "," [maybe_sequence_pattern] + maybe_sequence_pattern: ",".maybe_star_pattern+ ","? + maybe_star_pattern: star_pattern | pattern + star_pattern: "*" (capture_pattern | wildcard_pattern) + +There is no difference if parentheses or square brackets are used for +sequence patterns (i.e. "(...)" vs "[...]" ). + +Note: + + A single pattern enclosed in parentheses without a trailing comma + (e.g. "(3 | 4)") is a group pattern. While a single pattern enclosed + in square brackets (e.g. "[3 | 4]") is still a sequence pattern. + +At most one star subpattern may be in a sequence pattern. The star +subpattern may occur in any position. If no star subpattern is +present, the sequence pattern is a fixed-length sequence pattern; +otherwise it is a variable-length sequence pattern. + +The following is the logical flow for matching a sequence pattern +against a subject value: + +1. If the subject value is not a sequence [2], the sequence pattern + fails. + +2. If the subject value is an instance of "str", "bytes" or + "bytearray" the sequence pattern fails. + +3. The subsequent steps depend on whether the sequence pattern is + fixed or variable-length. + + If the sequence pattern is fixed-length: + + 1. If the length of the subject sequence is not equal to the number + of subpatterns, the sequence pattern fails + + 2. Subpatterns in the sequence pattern are matched to their + corresponding items in the subject sequence from left to right. + Matching stops as soon as a subpattern fails. If all + subpatterns succeed in matching their corresponding item, the + sequence pattern succeeds. + + Otherwise, if the sequence pattern is variable-length: + + 1. If the length of the subject sequence is less than the number of + non-star subpatterns, the sequence pattern fails. + + 2. The leading non-star subpatterns are matched to their + corresponding items as for fixed-length sequences. + + 3. If the previous step succeeds, the star subpattern matches a + list formed of the remaining subject items, excluding the + remaining items corresponding to non-star subpatterns following + the star subpattern. + + 4. Remaining non-star subpatterns are matched to their + corresponding subject items, as for a fixed-length sequence. + + Note: + + The length of the subject sequence is obtained via "len()" (i.e. + via the "__len__()" protocol). This length may be cached by the + interpreter in a similar manner as value patterns. + +In simple terms "[P1, P2, P3," … ", P]" matches only if all the +following happens: + +* check "" is a sequence + +* "len(subject) == " + +* "P1" matches "[0]" (note that this match can also bind + names) + +* "P2" matches "[1]" (note that this match can also bind + names) + +* … and so on for the corresponding pattern/element. + + +Mapping Patterns +~~~~~~~~~~~~~~~~ + +A mapping pattern contains one or more key-value patterns. The syntax +is similar to the construction of a dictionary. Syntax: + + mapping_pattern: "{" [items_pattern] "}" + items_pattern: ",".key_value_pattern+ ","? + key_value_pattern: (literal_pattern | value_pattern) ":" pattern + | double_star_pattern + double_star_pattern: "**" capture_pattern + +At most one double star pattern may be in a mapping pattern. The +double star pattern must be the last subpattern in the mapping +pattern. + +Duplicate keys in mapping patterns are disallowed. Duplicate literal +keys will raise a "SyntaxError". Two keys that otherwise have the same +value will raise a "ValueError" at runtime. + +The following is the logical flow for matching a mapping pattern +against a subject value: + +1. If the subject value is not a mapping [3],the mapping pattern + fails. + +2. If every key given in the mapping pattern is present in the subject + mapping, and the pattern for each key matches the corresponding + item of the subject mapping, the mapping pattern succeeds. + +3. If duplicate keys are detected in the mapping pattern, the pattern + is considered invalid. A "SyntaxError" is raised for duplicate + literal values; or a "ValueError" for named keys of the same value. + +Note: + + Key-value pairs are matched using the two-argument form of the + mapping subject’s "get()" method. Matched key-value pairs must + already be present in the mapping, and not created on-the-fly via + "__missing__()" or "__getitem__()". + +In simple terms "{KEY1: P1, KEY2: P2, ... }" matches only if all the +following happens: + +* check "" is a mapping + +* "KEY1 in " + +* "P1" matches "[KEY1]" + +* … and so on for the corresponding KEY/pattern pair. + + +Class Patterns +~~~~~~~~~~~~~~ + +A class pattern represents a class and its positional and keyword +arguments (if any). Syntax: + + class_pattern: name_or_attr "(" [pattern_arguments ","?] ")" + pattern_arguments: positional_patterns ["," keyword_patterns] + | keyword_patterns + positional_patterns: ",".pattern+ + keyword_patterns: ",".keyword_pattern+ + keyword_pattern: NAME "=" pattern + +The same keyword should not be repeated in class patterns. + +The following is the logical flow for matching a class pattern against +a subject value: + +1. If "name_or_attr" is not an instance of the builtin "type" , raise + "TypeError". + +2. If the subject value is not an instance of "name_or_attr" (tested + via "isinstance()"), the class pattern fails. + +3. If no pattern arguments are present, the pattern succeeds. + Otherwise, the subsequent steps depend on whether keyword or + positional argument patterns are present. + + For a number of built-in types (specified below), a single + positional subpattern is accepted which will match the entire + subject; for these types keyword patterns also work as for other + types. + + If only keyword patterns are present, they are processed as + follows, one by one: + + 1. The keyword is looked up as an attribute on the subject. + + * If this raises an exception other than "AttributeError", the + exception bubbles up. + + * If this raises "AttributeError", the class pattern has failed. + + * Else, the subpattern associated with the keyword pattern is + matched against the subject’s attribute value. If this fails, + the class pattern fails; if this succeeds, the match proceeds + to the next keyword. + + 2. If all keyword patterns succeed, the class pattern succeeds. + + If any positional patterns are present, they are converted to + keyword patterns using the "__match_args__" attribute on the class + "name_or_attr" before matching: + + 1. The equivalent of "getattr(cls, "__match_args__", ())" is + called. + + * If this raises an exception, the exception bubbles up. + + * If the returned value is not a tuple, the conversion fails and + "TypeError" is raised. + + * If there are more positional patterns than + "len(cls.__match_args__)", "TypeError" is raised. + + * Otherwise, positional pattern "i" is converted to a keyword + pattern using "__match_args__[i]" as the keyword. + "__match_args__[i]" must be a string; if not "TypeError" is + raised. + + * If there are duplicate keywords, "TypeError" is raised. + + See also: + + Customizing positional arguments in class pattern matching + + 2. Once all positional patterns have been converted to keyword + patterns, the match proceeds as if there were only keyword + patterns. + + For the following built-in types the handling of positional + subpatterns is different: + + * "bool" + + * "bytearray" + + * "bytes" + + * "dict" + + * "float" + + * "frozenset" + + * "int" + + * "list" + + * "set" + + * "str" + + * "tuple" + + These classes accept a single positional argument, and the pattern + there is matched against the whole object rather than an attribute. + For example "int(0|1)" matches the value "0", but not the value + "0.0". + +In simple terms "CLS(P1, attr=P2)" matches only if the following +happens: + +* "isinstance(, CLS)" + +* convert "P1" to a keyword pattern using "CLS.__match_args__" + +* For each keyword argument "attr=P2": + + * "hasattr(, "attr")" + + * "P2" matches ".attr" + +* … and so on for the corresponding keyword argument/pattern pair. + +See also: + + * **PEP 634** – Structural Pattern Matching: Specification + + * **PEP 636** – Structural Pattern Matching: Tutorial + + +Function definitions +==================== + +A function definition defines a user-defined function object (see +section The standard type hierarchy): + + funcdef: [decorators] "def" funcname [type_params] "(" [parameter_list] ")" + ["->" expression] ":" suite + decorators: decorator+ + decorator: "@" assignment_expression NEWLINE + parameter_list: defparameter ("," defparameter)* "," "/" ["," [parameter_list_no_posonly]] + | parameter_list_no_posonly + parameter_list_no_posonly: defparameter ("," defparameter)* ["," [parameter_list_starargs]] + | parameter_list_starargs + parameter_list_starargs: "*" [star_parameter] ("," defparameter)* ["," [parameter_star_kwargs]] + | "*" ("," defparameter)+ ["," [parameter_star_kwargs]] + | parameter_star_kwargs + parameter_star_kwargs: "**" parameter [","] + parameter: identifier [":" expression] + star_parameter: identifier [":" ["*"] expression] + defparameter: parameter ["=" expression] + funcname: identifier + +A function definition is an executable statement. Its execution binds +the function name in the current local namespace to a function object +(a wrapper around the executable code for the function). This +function object contains a reference to the current global namespace +as the global namespace to be used when the function is called. + +The function definition does not execute the function body; this gets +executed only when the function is called. [4] + +A function definition may be wrapped by one or more *decorator* +expressions. Decorator expressions are evaluated when the function is +defined, in the scope that contains the function definition. The +result must be a callable, which is invoked with the function object +as the only argument. The returned value is bound to the function name +instead of the function object. Multiple decorators are applied in +nested fashion. For example, the following code + + @f1(arg) + @f2 + def func(): pass + +is roughly equivalent to + + def func(): pass + func = f1(arg)(f2(func)) + +except that the original function is not temporarily bound to the name +"func". + +Changed in version 3.9: Functions may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets between the +function’s name and the opening parenthesis for its parameter list. +This indicates to static type checkers that the function is generic. +At runtime, the type parameters can be retrieved from the function’s +"__type_params__" attribute. See Generic functions for more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +When one or more *parameters* have the form *parameter* "=" +*expression*, the function is said to have “default parameter values.” +For a parameter with a default value, the corresponding *argument* may +be omitted from a call, in which case the parameter’s default value is +substituted. If a parameter has a default value, all following +parameters up until the “"*"” must also have a default value — this is +a syntactic restriction that is not expressed by the grammar. + +**Default parameter values are evaluated from left to right when the +function definition is executed.** This means that the expression is +evaluated once, when the function is defined, and that the same “pre- +computed” value is used for each call. This is especially important +to understand when a default parameter value is a mutable object, such +as a list or a dictionary: if the function modifies the object (e.g. +by appending an item to a list), the default parameter value is in +effect modified. This is generally not what was intended. A way +around this is to use "None" as the default, and explicitly test for +it in the body of the function, e.g.: + + def whats_on_the_telly(penguin=None): + if penguin is None: + penguin = [] + penguin.append("property of the zoo") + return penguin + +Function call semantics are described in more detail in section Calls. +A function call always assigns values to all parameters mentioned in +the parameter list, either from positional arguments, from keyword +arguments, or from default values. If the form “"*identifier"” is +present, it is initialized to a tuple receiving any excess positional +parameters, defaulting to the empty tuple. If the form +“"**identifier"” is present, it is initialized to a new ordered +mapping receiving any excess keyword arguments, defaulting to a new +empty mapping of the same type. Parameters after “"*"” or +“"*identifier"” are keyword-only parameters and may only be passed by +keyword arguments. Parameters before “"/"” are positional-only +parameters and may only be passed by positional arguments. + +Changed in version 3.8: The "/" function parameter syntax may be used +to indicate positional-only parameters. See **PEP 570** for details. + +Parameters may have an *annotation* of the form “": expression"” +following the parameter name. Any parameter may have an annotation, +even those of the form "*identifier" or "**identifier". (As a special +case, parameters of the form "*identifier" may have an annotation “": +*expression"”.) Functions may have “return” annotation of the form +“"-> expression"” after the parameter list. These annotations can be +any valid Python expression. The presence of annotations does not +change the semantics of a function. See Annotations for more +information on annotations. + +Changed in version 3.11: Parameters of the form “"*identifier"” may +have an annotation “": *expression"”. See **PEP 646**. + +It is also possible to create anonymous functions (functions not bound +to a name), for immediate use in expressions. This uses lambda +expressions, described in section Lambdas. Note that the lambda +expression is merely a shorthand for a simplified function definition; +a function defined in a “"def"” statement can be passed around or +assigned to another name just like a function defined by a lambda +expression. The “"def"” form is actually more powerful since it +allows the execution of multiple statements and annotations. + +**Programmer’s note:** Functions are first-class objects. A “"def"” +statement executed inside a function definition defines a local +function that can be returned or passed around. Free variables used +in the nested function can access the local variables of the function +containing the def. See section Naming and binding for details. + +See also: + + **PEP 3107** - Function Annotations + The original specification for function annotations. + + **PEP 484** - Type Hints + Definition of a standard meaning for annotations: type hints. + + **PEP 526** - Syntax for Variable Annotations + Ability to type hint variable declarations, including class + variables and instance variables. + + **PEP 563** - Postponed Evaluation of Annotations + Support for forward references within annotations by preserving + annotations in a string form at runtime instead of eager + evaluation. + + **PEP 318** - Decorators for Functions and Methods + Function and method decorators were introduced. Class decorators + were introduced in **PEP 3129**. + + +Class definitions +================= + +A class definition defines a class object (see section The standard +type hierarchy): + + classdef: [decorators] "class" classname [type_params] [inheritance] ":" suite + inheritance: "(" [argument_list] ")" + classname: identifier + +A class definition is an executable statement. The inheritance list +usually gives a list of base classes (see Metaclasses for more +advanced uses), so each item in the list should evaluate to a class +object which allows subclassing. Classes without an inheritance list +inherit, by default, from the base class "object"; hence, + + class Foo: + pass + +is equivalent to + + class Foo(object): + pass + +The class’s suite is then executed in a new execution frame (see +Naming and binding), using a newly created local namespace and the +original global namespace. (Usually, the suite contains mostly +function definitions.) When the class’s suite finishes execution, its +execution frame is discarded but its local namespace is saved. [5] A +class object is then created using the inheritance list for the base +classes and the saved local namespace for the attribute dictionary. +The class name is bound to this class object in the original local +namespace. + +The order in which attributes are defined in the class body is +preserved in the new class’s "__dict__". Note that this is reliable +only right after the class is created and only for classes that were +defined using the definition syntax. + +Class creation can be customized heavily using metaclasses. + +Classes can also be decorated: just like when decorating functions, + + @f1(arg) + @f2 + class Foo: pass + +is roughly equivalent to + + class Foo: pass + Foo = f1(arg)(f2(Foo)) + +The evaluation rules for the decorator expressions are the same as for +function decorators. The result is then bound to the class name. + +Changed in version 3.9: Classes may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets immediately +after the class’s name. This indicates to static type checkers that +the class is generic. At runtime, the type parameters can be retrieved +from the class’s "__type_params__" attribute. See Generic classes for +more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +**Programmer’s note:** Variables defined in the class definition are +class attributes; they are shared by instances. Instance attributes +can be set in a method with "self.name = value". Both class and +instance attributes are accessible through the notation “"self.name"”, +and an instance attribute hides a class attribute with the same name +when accessed in this way. Class attributes can be used as defaults +for instance attributes, but using mutable values there can lead to +unexpected results. Descriptors can be used to create instance +variables with different implementation details. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + The proposal that changed the declaration of metaclasses to the + current syntax, and the semantics for how classes with + metaclasses are constructed. + + **PEP 3129** - Class Decorators + The proposal that added class decorators. Function and method + decorators were introduced in **PEP 318**. + + +Coroutines +========== + +Added in version 3.5. + + +Coroutine function definition +----------------------------- + + async_funcdef: [decorators] "async" "def" funcname "(" [parameter_list] ")" + ["->" expression] ":" suite + +Execution of Python coroutines can be suspended and resumed at many +points (see *coroutine*). "await" expressions, "async for" and "async +with" can only be used in the body of a coroutine function. + +Functions defined with "async def" syntax are always coroutine +functions, even if they do not contain "await" or "async" keywords. + +It is a "SyntaxError" to use a "yield from" expression inside the body +of a coroutine function. + +An example of a coroutine function: + + async def func(param1, param2): + do_stuff() + await some_coroutine() + +Changed in version 3.7: "await" and "async" are now keywords; +previously they were only treated as such inside the body of a +coroutine function. + + +The "async for" statement +------------------------- + + async_for_stmt: "async" for_stmt + +An *asynchronous iterable* provides an "__aiter__" method that +directly returns an *asynchronous iterator*, which can call +asynchronous code in its "__anext__" method. + +The "async for" statement allows convenient iteration over +asynchronous iterables. + +The following code: + + async for TARGET in ITER: + SUITE + else: + SUITE2 + +Is semantically equivalent to: + + iter = (ITER) + iter = type(iter).__aiter__(iter) + running = True + + while running: + try: + TARGET = await type(iter).__anext__(iter) + except StopAsyncIteration: + running = False + else: + SUITE + else: + SUITE2 + +See also "__aiter__()" and "__anext__()" for details. + +It is a "SyntaxError" to use an "async for" statement outside the body +of a coroutine function. + + +The "async with" statement +-------------------------- + + async_with_stmt: "async" with_stmt + +An *asynchronous context manager* is a *context manager* that is able +to suspend execution in its *enter* and *exit* methods. + +The following code: + + async with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + aenter = type(manager).__aenter__ + aexit = type(manager).__aexit__ + value = await aenter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not await aexit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + await aexit(manager, None, None, None) + +See also "__aenter__()" and "__aexit__()" for details. + +It is a "SyntaxError" to use an "async with" statement outside the +body of a coroutine function. + +See also: + + **PEP 492** - Coroutines with async and await syntax + The proposal that made coroutines a proper standalone concept in + Python, and added supporting syntax. + + +Type parameter lists +==================== + +Added in version 3.12. + +Changed in version 3.13: Support for default values was added (see +**PEP 696**). + + type_params: "[" type_param ("," type_param)* "]" + type_param: typevar | typevartuple | paramspec + typevar: identifier (":" expression)? ("=" expression)? + typevartuple: "*" identifier ("=" expression)? + paramspec: "**" identifier ("=" expression)? + +Functions (including coroutines), classes and type aliases may contain +a type parameter list: + + def max[T](args: list[T]) -> T: + ... + + async def amax[T](args: list[T]) -> T: + ... + + class Bag[T]: + def __iter__(self) -> Iterator[T]: + ... + + def add(self, arg: T) -> None: + ... + + type ListOrSet[T] = list[T] | set[T] + +Semantically, this indicates that the function, class, or type alias +is generic over a type variable. This information is primarily used by +static type checkers, and at runtime, generic objects behave much like +their non-generic counterparts. + +Type parameters are declared in square brackets ("[]") immediately +after the name of the function, class, or type alias. The type +parameters are accessible within the scope of the generic object, but +not elsewhere. Thus, after a declaration "def func[T](): pass", the +name "T" is not available in the module scope. Below, the semantics of +generic objects are described with more precision. The scope of type +parameters is modeled with a special function (technically, an +annotation scope) that wraps the creation of the generic object. + +Generic functions, classes, and type aliases have a "__type_params__" +attribute listing their type parameters. + +Type parameters come in three kinds: + +* "typing.TypeVar", introduced by a plain name (e.g., "T"). + Semantically, this represents a single type to a type checker. + +* "typing.TypeVarTuple", introduced by a name prefixed with a single + asterisk (e.g., "*Ts"). Semantically, this stands for a tuple of any + number of types. + +* "typing.ParamSpec", introduced by a name prefixed with two asterisks + (e.g., "**P"). Semantically, this stands for the parameters of a + callable. + +"typing.TypeVar" declarations can define *bounds* and *constraints* +with a colon (":") followed by an expression. A single expression +after the colon indicates a bound (e.g. "T: int"). Semantically, this +means that the "typing.TypeVar" can only represent types that are a +subtype of this bound. A parenthesized tuple of expressions after the +colon indicates a set of constraints (e.g. "T: (str, bytes)"). Each +member of the tuple should be a type (again, this is not enforced at +runtime). Constrained type variables can only take on one of the types +in the list of constraints. + +For "typing.TypeVar"s declared using the type parameter list syntax, +the bound and constraints are not evaluated when the generic object is +created, but only when the value is explicitly accessed through the +attributes "__bound__" and "__constraints__". To accomplish this, the +bounds or constraints are evaluated in a separate annotation scope. + +"typing.TypeVarTuple"s and "typing.ParamSpec"s cannot have bounds or +constraints. + +All three flavors of type parameters can also have a *default value*, +which is used when the type parameter is not explicitly provided. This +is added by appending a single equals sign ("=") followed by an +expression. Like the bounds and constraints of type variables, the +default value is not evaluated when the object is created, but only +when the type parameter’s "__default__" attribute is accessed. To this +end, the default value is evaluated in a separate annotation scope. If +no default value is specified for a type parameter, the "__default__" +attribute is set to the special sentinel object "typing.NoDefault". + +The following example indicates the full set of allowed type parameter +declarations: + + def overly_generic[ + SimpleTypeVar, + TypeVarWithDefault = int, + TypeVarWithBound: int, + TypeVarWithConstraints: (str, bytes), + *SimpleTypeVarTuple = (int, float), + **SimpleParamSpec = (str, bytearray), + ]( + a: SimpleTypeVar, + b: TypeVarWithDefault, + c: TypeVarWithBound, + d: Callable[SimpleParamSpec, TypeVarWithConstraints], + *e: SimpleTypeVarTuple, + ): ... + + +Generic functions +----------------- + +Generic functions are declared as follows: + + def func[T](arg: T): ... + +This syntax is equivalent to: + + annotation-def TYPE_PARAMS_OF_func(): + T = typing.TypeVar("T") + def func(arg: T): ... + func.__type_params__ = (T,) + return func + func = TYPE_PARAMS_OF_func() + +Here "annotation-def" indicates an annotation scope, which is not +actually bound to any name at runtime. (One other liberty is taken in +the translation: the syntax does not go through attribute access on +the "typing" module, but creates an instance of "typing.TypeVar" +directly.) + +The annotations of generic functions are evaluated within the +annotation scope used for declaring the type parameters, but the +function’s defaults and decorators are not. + +The following example illustrates the scoping rules for these cases, +as well as for additional flavors of type parameters: + + @decorator + def func[T: int, *Ts, **P](*args: *Ts, arg: Callable[P, T] = some_default): + ... + +Except for the lazy evaluation of the "TypeVar" bound, this is +equivalent to: + + DEFAULT_OF_arg = some_default + + annotation-def TYPE_PARAMS_OF_func(): + + annotation-def BOUND_OF_T(): + return int + # In reality, BOUND_OF_T() is evaluated only on demand. + T = typing.TypeVar("T", bound=BOUND_OF_T()) + + Ts = typing.TypeVarTuple("Ts") + P = typing.ParamSpec("P") + + def func(*args: *Ts, arg: Callable[P, T] = DEFAULT_OF_arg): + ... + + func.__type_params__ = (T, Ts, P) + return func + func = decorator(TYPE_PARAMS_OF_func()) + +The capitalized names like "DEFAULT_OF_arg" are not actually bound at +runtime. + + +Generic classes +--------------- + +Generic classes are declared as follows: + + class Bag[T]: ... + +This syntax is equivalent to: + + annotation-def TYPE_PARAMS_OF_Bag(): + T = typing.TypeVar("T") + class Bag(typing.Generic[T]): + __type_params__ = (T,) + ... + return Bag + Bag = TYPE_PARAMS_OF_Bag() + +Here again "annotation-def" (not a real keyword) indicates an +annotation scope, and the name "TYPE_PARAMS_OF_Bag" is not actually +bound at runtime. + +Generic classes implicitly inherit from "typing.Generic". The base +classes and keyword arguments of generic classes are evaluated within +the type scope for the type parameters, and decorators are evaluated +outside that scope. This is illustrated by this example: + + @decorator + class Bag(Base[T], arg=T): ... + +This is equivalent to: + + annotation-def TYPE_PARAMS_OF_Bag(): + T = typing.TypeVar("T") + class Bag(Base[T], typing.Generic[T], arg=T): + __type_params__ = (T,) + ... + return Bag + Bag = decorator(TYPE_PARAMS_OF_Bag()) + + +Generic type aliases +-------------------- + +The "type" statement can also be used to create a generic type alias: + + type ListOrSet[T] = list[T] | set[T] + +Except for the lazy evaluation of the value, this is equivalent to: + + annotation-def TYPE_PARAMS_OF_ListOrSet(): + T = typing.TypeVar("T") + + annotation-def VALUE_OF_ListOrSet(): + return list[T] | set[T] + # In reality, the value is lazily evaluated + return typing.TypeAliasType("ListOrSet", VALUE_OF_ListOrSet(), type_params=(T,)) + ListOrSet = TYPE_PARAMS_OF_ListOrSet() + +Here, "annotation-def" (not a real keyword) indicates an annotation +scope. The capitalized names like "TYPE_PARAMS_OF_ListOrSet" are not +actually bound at runtime. + + +Annotations +=========== + +Changed in version 3.14: Annotations are now lazily evaluated by +default. + +Variables and function parameters may carry *annotations*, created by +adding a colon after the name, followed by an expression: + + x: annotation = 1 + def f(param: annotation): ... + +Functions may also carry a return annotation following an arrow: + + def f() -> annotation: ... + +Annotations are conventionally used for *type hints*, but this is not +enforced by the language, and in general annotations may contain +arbitrary expressions. The presence of annotations does not change the +runtime semantics of the code, except if some mechanism is used that +introspects and uses the annotations (such as "dataclasses" or +"functools.singledispatch()"). + +By default, annotations are lazily evaluated in an annotation scope. +This means that they are not evaluated when the code containing the +annotation is evaluated. Instead, the interpreter saves information +that can be used to evaluate the annotation later if requested. The +"annotationlib" module provides tools for evaluating annotations. + +If the future statement "from __future__ import annotations" is +present, all annotations are instead stored as strings: + + >>> from __future__ import annotations + >>> def f(param: annotation): ... + >>> f.__annotations__ + {'param': 'annotation'} + +This future statement will be deprecated and removed in a future +version of Python, but not before Python 3.13 reaches its end of life +(see **PEP 749**). When it is used, introspection tools like +"annotationlib.get_annotations()" and "typing.get_type_hints()" are +less likely to be able to resolve annotations at runtime. + +-[ Footnotes ]- + +[1] The exception is propagated to the invocation stack unless there + is a "finally" clause which happens to raise another exception. + That new exception causes the old one to be lost. + +[2] In pattern matching, a sequence is defined as one of the + following: + + * a class that inherits from "collections.abc.Sequence" + + * a Python class that has been registered as + "collections.abc.Sequence" + + * a builtin class that has its (CPython) "Py_TPFLAGS_SEQUENCE" bit + set + + * a class that inherits from any of the above + + The following standard library classes are sequences: + + * "array.array" + + * "collections.deque" + + * "list" + + * "memoryview" + + * "range" + + * "tuple" + + Note: + + Subject values of type "str", "bytes", and "bytearray" do not + match sequence patterns. + +[3] In pattern matching, a mapping is defined as one of the following: + + * a class that inherits from "collections.abc.Mapping" + + * a Python class that has been registered as + "collections.abc.Mapping" + + * a builtin class that has its (CPython) "Py_TPFLAGS_MAPPING" bit + set + + * a class that inherits from any of the above + + The standard library classes "dict" and "types.MappingProxyType" + are mappings. + +[4] A string literal appearing as the first statement in the function + body is transformed into the function’s "__doc__" attribute and + therefore the function’s *docstring*. + +[5] A string literal appearing as the first statement in the class + body is transformed into the namespace’s "__doc__" item and + therefore the class’s *docstring*. +''', + 'context-managers': r'''With Statement Context Managers +******************************* + +A *context manager* is an object that defines the runtime context to +be established when executing a "with" statement. The context manager +handles the entry into, and the exit from, the desired runtime context +for the execution of the block of code. Context managers are normally +invoked using the "with" statement (described in section The with +statement), but can also be used by directly invoking their methods. + +Typical uses of context managers include saving and restoring various +kinds of global state, locking and unlocking resources, closing opened +files, etc. + +For more information on context managers, see Context Manager Types. +The "object" class itself does not provide the context manager +methods. + +object.__enter__(self) + + Enter the runtime context related to this object. The "with" + statement will bind this method’s return value to the target(s) + specified in the "as" clause of the statement, if any. + +object.__exit__(self, exc_type, exc_value, traceback) + + Exit the runtime context related to this object. The parameters + describe the exception that caused the context to be exited. If the + context was exited without an exception, all three arguments will + be "None". + + If an exception is supplied, and the method wishes to suppress the + exception (i.e., prevent it from being propagated), it should + return a true value. Otherwise, the exception will be processed + normally upon exit from this method. + + Note that "__exit__()" methods should not reraise the passed-in + exception; this is the caller’s responsibility. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. +''', + 'continue': r'''The "continue" statement +************************ + + continue_stmt: "continue" + +"continue" may only occur syntactically nested in a "for" or "while" +loop, but not nested in a function or class definition within that +loop. It continues with the next cycle of the nearest enclosing loop. + +When "continue" passes control out of a "try" statement with a +"finally" clause, that "finally" clause is executed before really +starting the next loop cycle. +''', + 'conversions': r'''Arithmetic conversions +********************** + +When a description of an arithmetic operator below uses the phrase +“the numeric arguments are converted to a common real type”, this +means that the operator implementation for built-in types works as +follows: + +* If both arguments are complex numbers, no conversion is performed; + +* if either argument is a complex or a floating-point number, the + other is converted to a floating-point number; + +* otherwise, both must be integers and no conversion is necessary. + +Some additional rules apply for certain operators (e.g., a string as a +left argument to the ‘%’ operator). Extensions must define their own +conversion behavior. +''', + 'customization': r'''Basic customization +******************* + +object.__new__(cls[, ...]) + + Called to create a new instance of class *cls*. "__new__()" is a + static method (special-cased so you need not declare it as such) + that takes the class of which an instance was requested as its + first argument. The remaining arguments are those passed to the + object constructor expression (the call to the class). The return + value of "__new__()" should be the new object instance (usually an + instance of *cls*). + + Typical implementations create a new instance of the class by + invoking the superclass’s "__new__()" method using + "super().__new__(cls[, ...])" with appropriate arguments and then + modifying the newly created instance as necessary before returning + it. + + If "__new__()" is invoked during object construction and it returns + an instance of *cls*, then the new instance’s "__init__()" method + will be invoked like "__init__(self[, ...])", where *self* is the + new instance and the remaining arguments are the same as were + passed to the object constructor. + + If "__new__()" does not return an instance of *cls*, then the new + instance’s "__init__()" method will not be invoked. + + "__new__()" is intended mainly to allow subclasses of immutable + types (like int, str, or tuple) to customize instance creation. It + is also commonly overridden in custom metaclasses in order to + customize class creation. + +object.__init__(self[, ...]) + + Called after the instance has been created (by "__new__()"), but + before it is returned to the caller. The arguments are those + passed to the class constructor expression. If a base class has an + "__init__()" method, the derived class’s "__init__()" method, if + any, must explicitly call it to ensure proper initialization of the + base class part of the instance; for example: + "super().__init__([args...])". + + Because "__new__()" and "__init__()" work together in constructing + objects ("__new__()" to create it, and "__init__()" to customize + it), no non-"None" value may be returned by "__init__()"; doing so + will cause a "TypeError" to be raised at runtime. + +object.__del__(self) + + Called when the instance is about to be destroyed. This is also + called a finalizer or (improperly) a destructor. If a base class + has a "__del__()" method, the derived class’s "__del__()" method, + if any, must explicitly call it to ensure proper deletion of the + base class part of the instance. + + It is possible (though not recommended!) for the "__del__()" method + to postpone destruction of the instance by creating a new reference + to it. This is called object *resurrection*. It is + implementation-dependent whether "__del__()" is called a second + time when a resurrected object is about to be destroyed; the + current *CPython* implementation only calls it once. + + It is not guaranteed that "__del__()" methods are called for + objects that still exist when the interpreter exits. + "weakref.finalize" provides a straightforward way to register a + cleanup function to be called when an object is garbage collected. + + Note: + + "del x" doesn’t directly call "x.__del__()" — the former + decrements the reference count for "x" by one, and the latter is + only called when "x"’s reference count reaches zero. + + **CPython implementation detail:** It is possible for a reference + cycle to prevent the reference count of an object from going to + zero. In this case, the cycle will be later detected and deleted + by the *cyclic garbage collector*. A common cause of reference + cycles is when an exception has been caught in a local variable. + The frame’s locals then reference the exception, which references + its own traceback, which references the locals of all frames caught + in the traceback. + + See also: Documentation for the "gc" module. + + Warning: + + Due to the precarious circumstances under which "__del__()" + methods are invoked, exceptions that occur during their execution + are ignored, and a warning is printed to "sys.stderr" instead. + In particular: + + * "__del__()" can be invoked when arbitrary code is being + executed, including from any arbitrary thread. If "__del__()" + needs to take a lock or invoke any other blocking resource, it + may deadlock as the resource may already be taken by the code + that gets interrupted to execute "__del__()". + + * "__del__()" can be executed during interpreter shutdown. As a + consequence, the global variables it needs to access (including + other modules) may already have been deleted or set to "None". + Python guarantees that globals whose name begins with a single + underscore are deleted from their module before other globals + are deleted; if no other references to such globals exist, this + may help in assuring that imported modules are still available + at the time when the "__del__()" method is called. + +object.__repr__(self) + + Called by the "repr()" built-in function to compute the “official” + string representation of an object. If at all possible, this + should look like a valid Python expression that could be used to + recreate an object with the same value (given an appropriate + environment). If this is not possible, a string of the form + "<...some useful description...>" should be returned. The return + value must be a string object. If a class defines "__repr__()" but + not "__str__()", then "__repr__()" is also used when an “informal” + string representation of instances of that class is required. + + This is typically used for debugging, so it is important that the + representation is information-rich and unambiguous. A default + implementation is provided by the "object" class itself. + +object.__str__(self) + + Called by "str(object)", the default "__format__()" implementation, + and the built-in function "print()", to compute the “informal” or + nicely printable string representation of an object. The return + value must be a str object. + + This method differs from "object.__repr__()" in that there is no + expectation that "__str__()" return a valid Python expression: a + more convenient or concise representation can be used. + + The default implementation defined by the built-in type "object" + calls "object.__repr__()". + +object.__bytes__(self) + + Called by bytes to compute a byte-string representation of an + object. This should return a "bytes" object. The "object" class + itself does not provide this method. + +object.__format__(self, format_spec) + + Called by the "format()" built-in function, and by extension, + evaluation of formatted string literals and the "str.format()" + method, to produce a “formatted” string representation of an + object. The *format_spec* argument is a string that contains a + description of the formatting options desired. The interpretation + of the *format_spec* argument is up to the type implementing + "__format__()", however most classes will either delegate + formatting to one of the built-in types, or use a similar + formatting option syntax. + + See Format Specification Mini-Language for a description of the + standard formatting syntax. + + The return value must be a string object. + + The default implementation by the "object" class should be given an + empty *format_spec* string. It delegates to "__str__()". + + Changed in version 3.4: The __format__ method of "object" itself + raises a "TypeError" if passed any non-empty string. + + Changed in version 3.7: "object.__format__(x, '')" is now + equivalent to "str(x)" rather than "format(str(x), '')". + +object.__lt__(self, other) +object.__le__(self, other) +object.__eq__(self, other) +object.__ne__(self, other) +object.__gt__(self, other) +object.__ge__(self, other) + + These are the so-called “rich comparison” methods. The + correspondence between operator symbols and method names is as + follows: "xy" calls + "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)". + + A rich comparison method may return the singleton "NotImplemented" + if it does not implement the operation for a given pair of + arguments. By convention, "False" and "True" are returned for a + successful comparison. However, these methods can return any value, + so if the comparison operator is used in a Boolean context (e.g., + in the condition of an "if" statement), Python will call "bool()" + on the value to determine if the result is true or false. + + By default, "object" implements "__eq__()" by using "is", returning + "NotImplemented" in the case of a false comparison: "True if x is y + else NotImplemented". For "__ne__()", by default it delegates to + "__eq__()" and inverts the result unless it is "NotImplemented". + There are no other implied relationships among the comparison + operators or default implementations; for example, the truth of + "(x.__hash__". + + If a class that does not override "__eq__()" wishes to suppress + hash support, it should include "__hash__ = None" in the class + definition. A class which defines its own "__hash__()" that + explicitly raises a "TypeError" would be incorrectly identified as + hashable by an "isinstance(obj, collections.abc.Hashable)" call. + + Note: + + By default, the "__hash__()" values of str and bytes objects are + “salted” with an unpredictable random value. Although they + remain constant within an individual Python process, they are not + predictable between repeated invocations of Python.This is + intended to provide protection against a denial-of-service caused + by carefully chosen inputs that exploit the worst case + performance of a dict insertion, *O*(*n*^2) complexity. See + http://ocert.org/advisories/ocert-2011-003.html for + details.Changing hash values affects the iteration order of sets. + Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds).See also + "PYTHONHASHSEED". + + Changed in version 3.3: Hash randomization is enabled by default. + +object.__bool__(self) + + Called to implement truth value testing and the built-in operation + "bool()"; should return "False" or "True". When this method is not + defined, "__len__()" is called, if it is defined, and the object is + considered true if its result is nonzero. If a class defines + neither "__len__()" nor "__bool__()" (which is true of the "object" + class itself), all its instances are considered true. +''', + 'debugger': r'''"pdb" — The Python Debugger +*************************** + +**Source code:** Lib/pdb.py + +====================================================================== + +The module "pdb" defines an interactive source code debugger for +Python programs. It supports setting (conditional) breakpoints and +single stepping at the source line level, inspection of stack frames, +source code listing, and evaluation of arbitrary Python code in the +context of any stack frame. It also supports post-mortem debugging +and can be called under program control. + +The debugger is extensible – it is actually defined as the class +"Pdb". This is currently undocumented but easily understood by reading +the source. The extension interface uses the modules "bdb" and "cmd". + +See also: + + Module "faulthandler" + Used to dump Python tracebacks explicitly, on a fault, after a + timeout, or on a user signal. + + Module "traceback" + Standard interface to extract, format and print stack traces of + Python programs. + +The typical usage to break into the debugger is to insert: + + import pdb; pdb.set_trace() + +Or: + + breakpoint() + +at the location you want to break into the debugger, and then run the +program. You can then step through the code following this statement, +and continue running without the debugger using the "continue" +command. + +Changed in version 3.7: The built-in "breakpoint()", when called with +defaults, can be used instead of "import pdb; pdb.set_trace()". + + def double(x): + breakpoint() + return x * 2 + val = 3 + print(f"{val} * 2 is {double(val)}") + +The debugger’s prompt is "(Pdb)", which is the indicator that you are +in debug mode: + + > ...(2)double() + -> breakpoint() + (Pdb) p x + 3 + (Pdb) continue + 3 * 2 is 6 + +Changed in version 3.3: Tab-completion via the "readline" module is +available for commands and command arguments, e.g. the current global +and local names are offered as arguments of the "p" command. + + +Command-line interface +====================== + +You can also invoke "pdb" from the command line to debug other +scripts. For example: + + python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] + +When invoked as a module, pdb will automatically enter post-mortem +debugging if the program being debugged exits abnormally. After post- +mortem debugging (or after normal exit of the program), pdb will +restart the program. Automatic restarting preserves pdb’s state (such +as breakpoints) and in most cases is more useful than quitting the +debugger upon program’s exit. + +-c, --command + + To execute commands as if given in a ".pdbrc" file; see Debugger + commands. + + Changed in version 3.2: Added the "-c" option. + +-m + + To execute modules similar to the way "python -m" does. As with a + script, the debugger will pause execution just before the first + line of the module. + + Changed in version 3.7: Added the "-m" option. + +-p, --pid + + Attach to the process with the specified PID. + + Added in version 3.14. + +To attach to a running Python process for remote debugging, use the +"-p" or "--pid" option with the target process’s PID: + + python -m pdb -p 1234 + +Note: + + Attaching to a process that is blocked in a system call or waiting + for I/O will only work once the next bytecode instruction is + executed or when the process receives a signal. + +Typical usage to execute a statement under control of the debugger is: + + >>> import pdb + >>> def f(x): + ... print(1 / x) + >>> pdb.run("f(2)") + > (1)() + (Pdb) continue + 0.5 + >>> + +The typical usage to inspect a crashed program is: + + >>> import pdb + >>> def f(x): + ... print(1 / x) + ... + >>> f(0) + Traceback (most recent call last): + File "", line 1, in + File "", line 2, in f + ZeroDivisionError: division by zero + >>> pdb.pm() + > (2)f() + (Pdb) p x + 0 + (Pdb) + +Changed in version 3.13: The implementation of **PEP 667** means that +name assignments made via "pdb" will immediately affect the active +scope, even when running inside an *optimized scope*. + +The module defines the following functions; each enters the debugger +in a slightly different way: + +pdb.run(statement, globals=None, locals=None) + + Execute the *statement* (given as a string or a code object) under + debugger control. The debugger prompt appears before any code is + executed; you can set breakpoints and type "continue", or you can + step through the statement using "step" or "next" (all these + commands are explained below). The optional *globals* and *locals* + arguments specify the environment in which the code is executed; by + default the dictionary of the module "__main__" is used. (See the + explanation of the built-in "exec()" or "eval()" functions.) + +pdb.runeval(expression, globals=None, locals=None) + + Evaluate the *expression* (given as a string or a code object) + under debugger control. When "runeval()" returns, it returns the + value of the *expression*. Otherwise this function is similar to + "run()". + +pdb.runcall(function, *args, **kwds) + + Call the *function* (a function or method object, not a string) + with the given arguments. When "runcall()" returns, it returns + whatever the function call returned. The debugger prompt appears + as soon as the function is entered. + +pdb.set_trace(*, header=None, commands=None) + + Enter the debugger at the calling stack frame. This is useful to + hard-code a breakpoint at a given point in a program, even if the + code is not otherwise being debugged (e.g. when an assertion + fails). If given, *header* is printed to the console just before + debugging begins. The *commands* argument, if given, is a list of + commands to execute when the debugger starts. + + Changed in version 3.7: The keyword-only argument *header*. + + Changed in version 3.13: "set_trace()" will enter the debugger + immediately, rather than on the next line of code to be executed. + + Added in version 3.14: The *commands* argument. + +awaitable pdb.set_trace_async(*, header=None, commands=None) + + async version of "set_trace()". This function should be used inside + an async function with "await". + + async def f(): + await pdb.set_trace_async() + + "await" statements are supported if the debugger is invoked by this + function. + + Added in version 3.14. + +pdb.post_mortem(t=None) + + Enter post-mortem debugging of the given exception or traceback + object. If no value is given, it uses the exception that is + currently being handled, or raises "ValueError" if there isn’t one. + + Changed in version 3.13: Support for exception objects was added. + +pdb.pm() + + Enter post-mortem debugging of the exception found in + "sys.last_exc". + +pdb.set_default_backend(backend) + + There are two supported backends for pdb: "'settrace'" and + "'monitoring'". See "bdb.Bdb" for details. The user can set the + default backend to use if none is specified when instantiating + "Pdb". If no backend is specified, the default is "'settrace'". + + Note: + + "breakpoint()" and "set_trace()" will not be affected by this + function. They always use "'monitoring'" backend. + + Added in version 3.14. + +pdb.get_default_backend() + + Returns the default backend for pdb. + + Added in version 3.14. + +The "run*" functions and "set_trace()" are aliases for instantiating +the "Pdb" class and calling the method of the same name. If you want +to access further features, you have to do this yourself: + +class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False, readrc=True, mode=None, backend=None, colorize=False) + + "Pdb" is the debugger class. + + The *completekey*, *stdin* and *stdout* arguments are passed to the + underlying "cmd.Cmd" class; see the description there. + + The *skip* argument, if given, must be an iterable of glob-style + module name patterns. The debugger will not step into frames that + originate in a module that matches one of these patterns. [1] + + By default, Pdb sets a handler for the SIGINT signal (which is sent + when the user presses "Ctrl"-"C" on the console) when you give a + "continue" command. This allows you to break into the debugger + again by pressing "Ctrl"-"C". If you want Pdb not to touch the + SIGINT handler, set *nosigint* to true. + + The *readrc* argument defaults to true and controls whether Pdb + will load .pdbrc files from the filesystem. + + The *mode* argument specifies how the debugger was invoked. It + impacts the workings of some debugger commands. Valid values are + "'inline'" (used by the breakpoint() builtin), "'cli'" (used by the + command line invocation) or "None" (for backwards compatible + behaviour, as before the *mode* argument was added). + + The *backend* argument specifies the backend to use for the + debugger. If "None" is passed, the default backend will be used. + See "set_default_backend()". Otherwise the supported backends are + "'settrace'" and "'monitoring'". + + The *colorize* argument, if set to "True", will enable colorized + output in the debugger, if color is supported. This will highlight + source code displayed in pdb. + + Example call to enable tracing with *skip*: + + import pdb; pdb.Pdb(skip=['django.*']).set_trace() + + Raises an auditing event "pdb.Pdb" with no arguments. + + Changed in version 3.1: Added the *skip* parameter. + + Changed in version 3.2: Added the *nosigint* parameter. Previously, + a SIGINT handler was never set by Pdb. + + Changed in version 3.6: The *readrc* argument. + + Added in version 3.14: Added the *mode* argument. + + Added in version 3.14: Added the *backend* argument. + + Added in version 3.14: Added the *colorize* argument. + + Changed in version 3.14: Inline breakpoints like "breakpoint()" or + "pdb.set_trace()" will always stop the program at calling frame, + ignoring the *skip* pattern (if any). + + run(statement, globals=None, locals=None) + runeval(expression, globals=None, locals=None) + runcall(function, *args, **kwds) + set_trace() + + See the documentation for the functions explained above. + + +Debugger commands +================= + +The commands recognized by the debugger are listed below. Most +commands can be abbreviated to one or two letters as indicated; e.g. +"h(elp)" means that either "h" or "help" can be used to enter the help +command (but not "he" or "hel", nor "H" or "Help" or "HELP"). +Arguments to commands must be separated by whitespace (spaces or +tabs). Optional arguments are enclosed in square brackets ("[]") in +the command syntax; the square brackets must not be typed. +Alternatives in the command syntax are separated by a vertical bar +("|"). + +Entering a blank line repeats the last command entered. Exception: if +the last command was a "list" command, the next 11 lines are listed. + +Commands that the debugger doesn’t recognize are assumed to be Python +statements and are executed in the context of the program being +debugged. Python statements can also be prefixed with an exclamation +point ("!"). This is a powerful way to inspect the program being +debugged; it is even possible to change a variable or call a function. +When an exception occurs in such a statement, the exception name is +printed but the debugger’s state is not changed. + +Changed in version 3.13: Expressions/Statements whose prefix is a pdb +command are now correctly identified and executed. + +The debugger supports aliases. Aliases can have parameters which +allows one a certain level of adaptability to the context under +examination. + +Multiple commands may be entered on a single line, separated by ";;". +(A single ";" is not used as it is the separator for multiple commands +in a line that is passed to the Python parser.) No intelligence is +applied to separating the commands; the input is split at the first +";;" pair, even if it is in the middle of a quoted string. A +workaround for strings with double semicolons is to use implicit +string concatenation "';'';'" or "";"";"". + +To set a temporary global variable, use a *convenience variable*. A +*convenience variable* is a variable whose name starts with "$". For +example, "$foo = 1" sets a global variable "$foo" which you can use in +the debugger session. The *convenience variables* are cleared when +the program resumes execution so it’s less likely to interfere with +your program compared to using normal variables like "foo = 1". + +There are four preset *convenience variables*: + +* "$_frame": the current frame you are debugging + +* "$_retval": the return value if the frame is returning + +* "$_exception": the exception if the frame is raising an exception + +* "$_asynctask": the asyncio task if pdb stops in an async function + +Added in version 3.12: Added the *convenience variable* feature. + +Added in version 3.14: Added the "$_asynctask" convenience variable. + +If a file ".pdbrc" exists in the user’s home directory or in the +current directory, it is read with "'utf-8'" encoding and executed as +if it had been typed at the debugger prompt, with the exception that +empty lines and lines starting with "#" are ignored. This is +particularly useful for aliases. If both files exist, the one in the +home directory is read first and aliases defined there can be +overridden by the local file. + +Changed in version 3.2: ".pdbrc" can now contain commands that +continue debugging, such as "continue" or "next". Previously, these +commands had no effect. + +Changed in version 3.11: ".pdbrc" is now read with "'utf-8'" encoding. +Previously, it was read with the system locale encoding. + +h(elp) [command] + + Without argument, print the list of available commands. With a + *command* as argument, print help about that command. "help pdb" + displays the full documentation (the docstring of the "pdb" + module). Since the *command* argument must be an identifier, "help + exec" must be entered to get help on the "!" command. + +w(here) [count] + + Print a stack trace, with the most recent frame at the bottom. if + *count* is 0, print the current frame entry. If *count* is + negative, print the least recent - *count* frames. If *count* is + positive, print the most recent *count* frames. An arrow (">") + indicates the current frame, which determines the context of most + commands. + + Changed in version 3.14: *count* argument is added. + +d(own) [count] + + Move the current frame *count* (default one) levels down in the + stack trace (to a newer frame). + +u(p) [count] + + Move the current frame *count* (default one) levels up in the stack + trace (to an older frame). + +b(reak) [([filename:]lineno | function) [, condition]] + + With a *lineno* argument, set a break at line *lineno* in the + current file. The line number may be prefixed with a *filename* and + a colon, to specify a breakpoint in another file (possibly one that + hasn’t been loaded yet). The file is searched on "sys.path". + Acceptable forms of *filename* are "/abspath/to/file.py", + "relpath/file.py", "module" and "package.module". + + With a *function* argument, set a break at the first executable + statement within that function. *function* can be any expression + that evaluates to a function in the current namespace. + + If a second argument is present, it is an expression which must + evaluate to true before the breakpoint is honored. + + Without argument, list all breaks, including for each breakpoint, + the number of times that breakpoint has been hit, the current + ignore count, and the associated condition if any. + + Each breakpoint is assigned a number to which all the other + breakpoint commands refer. + +tbreak [([filename:]lineno | function) [, condition]] + + Temporary breakpoint, which is removed automatically when it is + first hit. The arguments are the same as for "break". + +cl(ear) [filename:lineno | bpnumber ...] + + With a *filename:lineno* argument, clear all the breakpoints at + this line. With a space separated list of breakpoint numbers, clear + those breakpoints. Without argument, clear all breaks (but first + ask confirmation). + +disable bpnumber [bpnumber ...] + + Disable the breakpoints given as a space separated list of + breakpoint numbers. Disabling a breakpoint means it cannot cause + the program to stop execution, but unlike clearing a breakpoint, it + remains in the list of breakpoints and can be (re-)enabled. + +enable bpnumber [bpnumber ...] + + Enable the breakpoints specified. + +ignore bpnumber [count] + + Set the ignore count for the given breakpoint number. If *count* + is omitted, the ignore count is set to 0. A breakpoint becomes + active when the ignore count is zero. When non-zero, the *count* + is decremented each time the breakpoint is reached and the + breakpoint is not disabled and any associated condition evaluates + to true. + +condition bpnumber [condition] + + Set a new *condition* for the breakpoint, an expression which must + evaluate to true before the breakpoint is honored. If *condition* + is absent, any existing condition is removed; i.e., the breakpoint + is made unconditional. + +commands [bpnumber] + + Specify a list of commands for breakpoint number *bpnumber*. The + commands themselves appear on the following lines. Type a line + containing just "end" to terminate the commands. An example: + + (Pdb) commands 1 + (com) p some_variable + (com) end + (Pdb) + + To remove all commands from a breakpoint, type "commands" and + follow it immediately with "end"; that is, give no commands. + + With no *bpnumber* argument, "commands" refers to the last + breakpoint set. + + You can use breakpoint commands to start your program up again. + Simply use the "continue" command, or "step", or any other command + that resumes execution. + + Specifying any command resuming execution (currently "continue", + "step", "next", "return", "until", "jump", "quit" and their + abbreviations) terminates the command list (as if that command was + immediately followed by end). This is because any time you resume + execution (even with a simple next or step), you may encounter + another breakpoint—which could have its own command list, leading + to ambiguities about which list to execute. + + If the list of commands contains the "silent" command, or a command + that resumes execution, then the breakpoint message containing + information about the frame is not displayed. + + Changed in version 3.14: Frame information will not be displayed if + a command that resumes execution is present in the command list. + +s(tep) + + Execute the current line, stop at the first possible occasion + (either in a function that is called or on the next line in the + current function). + +n(ext) + + Continue execution until the next line in the current function is + reached or it returns. (The difference between "next" and "step" + is that "step" stops inside a called function, while "next" + executes called functions at (nearly) full speed, only stopping at + the next line in the current function.) + +unt(il) [lineno] + + Without argument, continue execution until the line with a number + greater than the current one is reached. + + With *lineno*, continue execution until a line with a number + greater or equal to *lineno* is reached. In both cases, also stop + when the current frame returns. + + Changed in version 3.2: Allow giving an explicit line number. + +r(eturn) + + Continue execution until the current function returns. + +c(ont(inue)) + + Continue execution, only stop when a breakpoint is encountered. + +j(ump) lineno + + Set the next line that will be executed. Only available in the + bottom-most frame. This lets you jump back and execute code again, + or jump forward to skip code that you don’t want to run. + + It should be noted that not all jumps are allowed – for instance it + is not possible to jump into the middle of a "for" loop or out of a + "finally" clause. + +l(ist) [first[, last]] + + List source code for the current file. Without arguments, list 11 + lines around the current line or continue the previous listing. + With "." as argument, list 11 lines around the current line. With + one argument, list 11 lines around at that line. With two + arguments, list the given range; if the second argument is less + than the first, it is interpreted as a count. + + The current line in the current frame is indicated by "->". If an + exception is being debugged, the line where the exception was + originally raised or propagated is indicated by ">>", if it differs + from the current line. + + Changed in version 3.2: Added the ">>" marker. + +ll | longlist + + List all source code for the current function or frame. + Interesting lines are marked as for "list". + + Added in version 3.2. + +a(rgs) + + Print the arguments of the current function and their current + values. + +p expression + + Evaluate *expression* in the current context and print its value. + + Note: + + "print()" can also be used, but is not a debugger command — this + executes the Python "print()" function. + +pp expression + + Like the "p" command, except the value of *expression* is pretty- + printed using the "pprint" module. + +whatis expression + + Print the type of *expression*. + +source expression + + Try to get source code of *expression* and display it. + + Added in version 3.2. + +display [expression] + + Display the value of *expression* if it changed, each time + execution stops in the current frame. + + Without *expression*, list all display expressions for the current + frame. + + Note: + + Display evaluates *expression* and compares to the result of the + previous evaluation of *expression*, so when the result is + mutable, display may not be able to pick up the changes. + + Example: + + lst = [] + breakpoint() + pass + lst.append(1) + print(lst) + + Display won’t realize "lst" has been changed because the result of + evaluation is modified in place by "lst.append(1)" before being + compared: + + > example.py(3)() + -> pass + (Pdb) display lst + display lst: [] + (Pdb) n + > example.py(4)() + -> lst.append(1) + (Pdb) n + > example.py(5)() + -> print(lst) + (Pdb) + + You can do some tricks with copy mechanism to make it work: + + > example.py(3)() + -> pass + (Pdb) display lst[:] + display lst[:]: [] + (Pdb) n + > example.py(4)() + -> lst.append(1) + (Pdb) n + > example.py(5)() + -> print(lst) + display lst[:]: [1] [old: []] + (Pdb) + + Added in version 3.2. + +undisplay [expression] + + Do not display *expression* anymore in the current frame. Without + *expression*, clear all display expressions for the current frame. + + Added in version 3.2. + +interact + + Start an interactive interpreter (using the "code" module) in a new + global namespace initialised from the local and global namespaces + for the current scope. Use "exit()" or "quit()" to exit the + interpreter and return to the debugger. + + Note: + + As "interact" creates a new dedicated namespace for code + execution, assignments to variables will not affect the original + namespaces. However, modifications to any referenced mutable + objects will be reflected in the original namespaces as usual. + + Added in version 3.2. + + Changed in version 3.13: "exit()" and "quit()" can be used to exit + the "interact" command. + + Changed in version 3.13: "interact" directs its output to the + debugger’s output channel rather than "sys.stderr". + +alias [name [command]] + + Create an alias called *name* that executes *command*. The + *command* must *not* be enclosed in quotes. Replaceable parameters + can be indicated by "%1", "%2", … and "%9", while "%*" is replaced + by all the parameters. If *command* is omitted, the current alias + for *name* is shown. If no arguments are given, all aliases are + listed. + + Aliases may be nested and can contain anything that can be legally + typed at the pdb prompt. Note that internal pdb commands *can* be + overridden by aliases. Such a command is then hidden until the + alias is removed. Aliasing is recursively applied to the first + word of the command line; all other words in the line are left + alone. + + As an example, here are two useful aliases (especially when placed + in the ".pdbrc" file): + + # Print instance variables (usage "pi classInst") + alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") + # Print instance variables in self + alias ps pi self + +unalias name + + Delete the specified alias *name*. + +! statement + + Execute the (one-line) *statement* in the context of the current + stack frame. The exclamation point can be omitted unless the first + word of the statement resembles a debugger command, e.g.: + + (Pdb) ! n=42 + (Pdb) + + To set a global variable, you can prefix the assignment command + with a "global" statement on the same line, e.g.: + + (Pdb) global list_options; list_options = ['-l'] + (Pdb) + +run [args ...] +restart [args ...] + + Restart the debugged Python program. If *args* is supplied, it is + split with "shlex" and the result is used as the new "sys.argv". + History, breakpoints, actions and debugger options are preserved. + "restart" is an alias for "run". + + Changed in version 3.14: "run" and "restart" commands are disabled + when the debugger is invoked in "'inline'" mode. + +q(uit) + + Quit from the debugger. The program being executed is aborted. An + end-of-file input is equivalent to "quit". + + A confirmation prompt will be shown if the debugger is invoked in + "'inline'" mode. Either "y", "Y", "" or "EOF" will confirm + the quit. + + Changed in version 3.14: A confirmation prompt will be shown if the + debugger is invoked in "'inline'" mode. After the confirmation, the + debugger will call "sys.exit()" immediately, instead of raising + "bdb.BdbQuit" in the next trace event. + +debug code + + Enter a recursive debugger that steps through *code* (which is an + arbitrary expression or statement to be executed in the current + environment). + +retval + + Print the return value for the last return of the current function. + +exceptions [excnumber] + + List or jump between chained exceptions. + + When using "pdb.pm()" or "Pdb.post_mortem(...)" with a chained + exception instead of a traceback, it allows the user to move + between the chained exceptions using "exceptions" command to list + exceptions, and "exceptions " to switch to that exception. + + Example: + + def out(): + try: + middle() + except Exception as e: + raise ValueError("reraise middle() error") from e + + def middle(): + try: + return inner(0) + except Exception as e: + raise ValueError("Middle fail") + + def inner(x): + 1 / x + + out() + + calling "pdb.pm()" will allow to move between exceptions: + + > example.py(5)out() + -> raise ValueError("reraise middle() error") from e + + (Pdb) exceptions + 0 ZeroDivisionError('division by zero') + 1 ValueError('Middle fail') + > 2 ValueError('reraise middle() error') + + (Pdb) exceptions 0 + > example.py(16)inner() + -> 1 / x + + (Pdb) up + > example.py(10)middle() + -> return inner(0) + + Added in version 3.13. + +-[ Footnotes ]- + +[1] Whether a frame is considered to originate in a certain module is + determined by the "__name__" in the frame globals. +''', + 'del': r'''The "del" statement +******************* + + del_stmt: "del" target_list + +Deletion is recursively defined very similar to the way assignment is +defined. Rather than spelling it out in full details, here are some +hints. + +Deletion of a target list recursively deletes each target, from left +to right. + +Deletion of a name removes the binding of that name from the local or +global namespace, depending on whether the name occurs in a "global" +statement in the same code block. Trying to delete an unbound name +raises a "NameError" exception. + +Deletion of attribute references, subscriptions and slicings is passed +to the primary object involved; deletion of a slicing is in general +equivalent to assignment of an empty slice of the right type (but even +this is determined by the sliced object). + +Changed in version 3.2: Previously it was illegal to delete a name +from the local namespace if it occurs as a free variable in a nested +block. +''', + 'dict': r'''Dictionary displays +******************* + +A dictionary display is a possibly empty series of dict items +(key/value pairs) enclosed in curly braces: + + dict_display: "{" [dict_item_list | dict_comprehension] "}" + dict_item_list: dict_item ("," dict_item)* [","] + dict_item: expression ":" expression | "**" or_expr + dict_comprehension: expression ":" expression comp_for + +A dictionary display yields a new dictionary object. + +If a comma-separated sequence of dict items is given, they are +evaluated from left to right to define the entries of the dictionary: +each key object is used as a key into the dictionary to store the +corresponding value. This means that you can specify the same key +multiple times in the dict item list, and the final dictionary’s value +for that key will be the last one given. + +A double asterisk "**" denotes *dictionary unpacking*. Its operand +must be a *mapping*. Each mapping item is added to the new +dictionary. Later values replace values already set by earlier dict +items and earlier dictionary unpackings. + +Added in version 3.5: Unpacking into dictionary displays, originally +proposed by **PEP 448**. + +A dict comprehension, in contrast to list and set comprehensions, +needs two expressions separated with a colon followed by the usual +“for” and “if” clauses. When the comprehension is run, the resulting +key and value elements are inserted in the new dictionary in the order +they are produced. + +Restrictions on the types of the key values are listed earlier in +section The standard type hierarchy. (To summarize, the key type +should be *hashable*, which excludes all mutable objects.) Clashes +between duplicate keys are not detected; the last value (textually +rightmost in the display) stored for a given key value prevails. + +Changed in version 3.8: Prior to Python 3.8, in dict comprehensions, +the evaluation order of key and value was not well-defined. In +CPython, the value was evaluated before the key. Starting with 3.8, +the key is evaluated before the value, as proposed by **PEP 572**. +''', + 'dynamic-features': r'''Interaction with dynamic features +********************************* + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. +''', + 'else': r'''The "if" statement +****************** + +The "if" statement is used for conditional execution: + + if_stmt: "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. +''', + 'exceptions': r'''Exceptions +********** + +Exceptions are a means of breaking out of the normal flow of control +of a code block in order to handle errors or other exceptional +conditions. An exception is *raised* at the point where the error is +detected; it may be *handled* by the surrounding code block or by any +code block that directly or indirectly invoked the code block where +the error occurred. + +The Python interpreter raises an exception when it detects a run-time +error (such as division by zero). A Python program can also +explicitly raise an exception with the "raise" statement. Exception +handlers are specified with the "try" … "except" statement. The +"finally" clause of such a statement can be used to specify cleanup +code which does not handle the exception, but is executed whether an +exception occurred or not in the preceding code. + +Python uses the “termination” model of error handling: an exception +handler can find out what happened and continue execution at an outer +level, but it cannot repair the cause of the error and retry the +failing operation (except by re-entering the offending piece of code +from the top). + +When an exception is not handled at all, the interpreter terminates +execution of the program, or returns to its interactive main loop. In +either case, it prints a stack traceback, except when the exception is +"SystemExit". + +Exceptions are identified by class instances. The "except" clause is +selected depending on the class of the instance: it must reference the +class of the instance or a *non-virtual base class* thereof. The +instance can be received by the handler and can carry additional +information about the exceptional condition. + +Note: + + Exception messages are not part of the Python API. Their contents + may change from one version of Python to the next without warning + and should not be relied on by code which will run under multiple + versions of the interpreter. + +See also the description of the "try" statement in section The try +statement and "raise" statement in section The raise statement. +''', + 'execmodel': r'''Execution model +*************** + + +Structure of a program +====================== + +A Python program is constructed from code blocks. A *block* is a piece +of Python program text that is executed as a unit. The following are +blocks: a module, a function body, and a class definition. Each +command typed interactively is a block. A script file (a file given +as standard input to the interpreter or specified as a command line +argument to the interpreter) is a code block. A script command (a +command specified on the interpreter command line with the "-c" +option) is a code block. A module run as a top level script (as module +"__main__") from the command line using a "-m" argument is also a code +block. The string argument passed to the built-in functions "eval()" +and "exec()" is a code block. + +A code block is executed in an *execution frame*. A frame contains +some administrative information (used for debugging) and determines +where and how execution continues after the code block’s execution has +completed. + + +Naming and binding +================== + + +Binding of names +---------------- + +*Names* refer to objects. Names are introduced by name binding +operations. + +The following constructs bind names: + +* formal parameters to functions, + +* class definitions, + +* function definitions, + +* assignment expressions, + +* targets that are identifiers if occurring in an assignment: + + * "for" loop header, + + * after "as" in a "with" statement, "except" clause, "except*" + clause, or in the as-pattern in structural pattern matching, + + * in a capture pattern in structural pattern matching + +* "import" statements. + +* "type" statements. + +* type parameter lists. + +The "import" statement of the form "from ... import *" binds all names +defined in the imported module, except those beginning with an +underscore. This form may only be used at the module level. + +A target occurring in a "del" statement is also considered bound for +this purpose (though the actual semantics are to unbind the name). + +Each assignment or import statement occurs within a block defined by a +class or function definition or at the module level (the top-level +code block). + +If a name is bound in a block, it is a local variable of that block, +unless declared as "nonlocal" or "global". If a name is bound at the +module level, it is a global variable. (The variables of the module +code block are local and global.) If a variable is used in a code +block but not defined there, it is a *free variable*. + +Each occurrence of a name in the program text refers to the *binding* +of that name established by the following name resolution rules. + + +Resolution of names +------------------- + +A *scope* defines the visibility of a name within a block. If a local +variable is defined in a block, its scope includes that block. If the +definition occurs in a function block, the scope extends to any blocks +contained within the defining one, unless a contained block introduces +a different binding for the name. + +When a name is used in a code block, it is resolved using the nearest +enclosing scope. The set of all such scopes visible to a code block +is called the block’s *environment*. + +When a name is not found at all, a "NameError" exception is raised. If +the current scope is a function scope, and the name refers to a local +variable that has not yet been bound to a value at the point where the +name is used, an "UnboundLocalError" exception is raised. +"UnboundLocalError" is a subclass of "NameError". + +If a name binding operation occurs anywhere within a code block, all +uses of the name within the block are treated as references to the +current block. This can lead to errors when a name is used within a +block before it is bound. This rule is subtle. Python lacks +declarations and allows name binding operations to occur anywhere +within a code block. The local variables of a code block can be +determined by scanning the entire text of the block for name binding +operations. See the FAQ entry on UnboundLocalError for examples. + +If the "global" statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the +top-level namespace. Names are resolved in the top-level namespace by +searching the global namespace, i.e. the namespace of the module +containing the code block, and the builtins namespace, the namespace +of the module "builtins". The global namespace is searched first. If +the names are not found there, the builtins namespace is searched +next. If the names are also not found in the builtins namespace, new +variables are created in the global namespace. The global statement +must precede all uses of the listed names. + +The "global" statement has the same scope as a name binding operation +in the same block. If the nearest enclosing scope for a free variable +contains a global statement, the free variable is treated as a global. + +The "nonlocal" statement causes corresponding names to refer to +previously bound variables in the nearest enclosing function scope. +"SyntaxError" is raised at compile time if the given name does not +exist in any enclosing function scope. Type parameters cannot be +rebound with the "nonlocal" statement. + +The namespace for a module is automatically created the first time a +module is imported. The main module for a script is always called +"__main__". + +Class definition blocks and arguments to "exec()" and "eval()" are +special in the context of name resolution. A class definition is an +executable statement that may use and define names. These references +follow the normal rules for name resolution with an exception that +unbound local variables are looked up in the global namespace. The +namespace of the class definition becomes the attribute dictionary of +the class. The scope of names defined in a class block is limited to +the class block; it does not extend to the code blocks of methods. +This includes comprehensions and generator expressions, but it does +not include annotation scopes, which have access to their enclosing +class scopes. This means that the following will fail: + + class A: + a = 42 + b = list(a + i for i in range(10)) + +However, the following will succeed: + + class A: + type Alias = Nested + class Nested: pass + + print(A.Alias.__value__) # + + +Annotation scopes +----------------- + +*Annotations*, type parameter lists and "type" statements introduce +*annotation scopes*, which behave mostly like function scopes, but +with some exceptions discussed below. + +Annotation scopes are used in the following contexts: + +* *Function annotations*. + +* *Variable annotations*. + +* Type parameter lists for generic type aliases. + +* Type parameter lists for generic functions. A generic function’s + annotations are executed within the annotation scope, but its + defaults and decorators are not. + +* Type parameter lists for generic classes. A generic class’s base + classes and keyword arguments are executed within the annotation + scope, but its decorators are not. + +* The bounds, constraints, and default values for type parameters + (lazily evaluated). + +* The value of type aliases (lazily evaluated). + +Annotation scopes differ from function scopes in the following ways: + +* Annotation scopes have access to their enclosing class namespace. If + an annotation scope is immediately within a class scope, or within + another annotation scope that is immediately within a class scope, + the code in the annotation scope can use names defined in the class + scope as if it were executed directly within the class body. This + contrasts with regular functions defined within classes, which + cannot access names defined in the class scope. + +* Expressions in annotation scopes cannot contain "yield", "yield + from", "await", or ":=" expressions. (These expressions are allowed + in other scopes contained within the annotation scope.) + +* Names defined in annotation scopes cannot be rebound with "nonlocal" + statements in inner scopes. This includes only type parameters, as + no other syntactic elements that can appear within annotation scopes + can introduce new names. + +* While annotation scopes have an internal name, that name is not + reflected in the *qualified name* of objects defined within the + scope. Instead, the "__qualname__" of such objects is as if the + object were defined in the enclosing scope. + +Added in version 3.12: Annotation scopes were introduced in Python +3.12 as part of **PEP 695**. + +Changed in version 3.13: Annotation scopes are also used for type +parameter defaults, as introduced by **PEP 696**. + +Changed in version 3.14: Annotation scopes are now also used for +annotations, as specified in **PEP 649** and **PEP 749**. + + +Lazy evaluation +--------------- + +Most annotation scopes are *lazily evaluated*. This includes +annotations, the values of type aliases created through the "type" +statement, and the bounds, constraints, and default values of type +variables created through the type parameter syntax. This means that +they are not evaluated when the type alias or type variable is +created, or when the object carrying annotations is created. Instead, +they are only evaluated when necessary, for example when the +"__value__" attribute on a type alias is accessed. + +Example: + + >>> type Alias = 1/0 + >>> Alias.__value__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def func[T: 1/0](): pass + >>> T = func.__type_params__[0] + >>> T.__bound__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + +Here the exception is raised only when the "__value__" attribute of +the type alias or the "__bound__" attribute of the type variable is +accessed. + +This behavior is primarily useful for references to types that have +not yet been defined when the type alias or type variable is created. +For example, lazy evaluation enables creation of mutually recursive +type aliases: + + from typing import Literal + + type SimpleExpr = int | Parenthesized + type Parenthesized = tuple[Literal["("], Expr, Literal[")"]] + type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", "-"], Expr] + +Lazily evaluated values are evaluated in annotation scope, which means +that names that appear inside the lazily evaluated value are looked up +as if they were used in the immediately enclosing scope. + +Added in version 3.12. + + +Builtins and restricted execution +--------------------------------- + +**CPython implementation detail:** Users should not touch +"__builtins__"; it is strictly an implementation detail. Users +wanting to override values in the builtins namespace should "import" +the "builtins" module and modify its attributes appropriately. + +The builtins namespace associated with the execution of a code block +is actually found by looking up the name "__builtins__" in its global +namespace; this should be a dictionary or a module (in the latter case +the module’s dictionary is used). By default, when in the "__main__" +module, "__builtins__" is the built-in module "builtins"; when in any +other module, "__builtins__" is an alias for the dictionary of the +"builtins" module itself. + + +Interaction with dynamic features +--------------------------------- + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. + + +Exceptions +========== + +Exceptions are a means of breaking out of the normal flow of control +of a code block in order to handle errors or other exceptional +conditions. An exception is *raised* at the point where the error is +detected; it may be *handled* by the surrounding code block or by any +code block that directly or indirectly invoked the code block where +the error occurred. + +The Python interpreter raises an exception when it detects a run-time +error (such as division by zero). A Python program can also +explicitly raise an exception with the "raise" statement. Exception +handlers are specified with the "try" … "except" statement. The +"finally" clause of such a statement can be used to specify cleanup +code which does not handle the exception, but is executed whether an +exception occurred or not in the preceding code. + +Python uses the “termination” model of error handling: an exception +handler can find out what happened and continue execution at an outer +level, but it cannot repair the cause of the error and retry the +failing operation (except by re-entering the offending piece of code +from the top). + +When an exception is not handled at all, the interpreter terminates +execution of the program, or returns to its interactive main loop. In +either case, it prints a stack traceback, except when the exception is +"SystemExit". + +Exceptions are identified by class instances. The "except" clause is +selected depending on the class of the instance: it must reference the +class of the instance or a *non-virtual base class* thereof. The +instance can be received by the handler and can carry additional +information about the exceptional condition. + +Note: + + Exception messages are not part of the Python API. Their contents + may change from one version of Python to the next without warning + and should not be relied on by code which will run under multiple + versions of the interpreter. + +See also the description of the "try" statement in section The try +statement and "raise" statement in section The raise statement. + + +Runtime Components +================== + + +General Computing Model +----------------------- + +Python’s execution model does not operate in a vacuum. It runs on a +host machine and through that host’s runtime environment, including +its operating system (OS), if there is one. When a program runs, the +conceptual layers of how it runs on the host look something like this: + + **host machine** + **process** (global resources) + **thread** (runs machine code) + +Each process represents a program running on the host. Think of each +process itself as the data part of its program. Think of the process’ +threads as the execution part of the program. This distinction will +be important to understand the conceptual Python runtime. + +The process, as the data part, is the execution context in which the +program runs. It mostly consists of the set of resources assigned to +the program by the host, including memory, signals, file handles, +sockets, and environment variables. + +Processes are isolated and independent from one another. (The same is +true for hosts.) The host manages the process’ access to its assigned +resources, in addition to coordinating between processes. + +Each thread represents the actual execution of the program’s machine +code, running relative to the resources assigned to the program’s +process. It’s strictly up to the host how and when that execution +takes place. + +From the point of view of Python, a program always starts with exactly +one thread. However, the program may grow to run in multiple +simultaneous threads. Not all hosts support multiple threads per +process, but most do. Unlike processes, threads in a process are not +isolated and independent from one another. Specifically, all threads +in a process share all of the process’ resources. + +The fundamental point of threads is that each one does *run* +independently, at the same time as the others. That may be only +conceptually at the same time (“concurrently”) or physically (“in +parallel”). Either way, the threads effectively run at a non- +synchronized rate. + +Note: + + That non-synchronized rate means none of the process’ memory is + guaranteed to stay consistent for the code running in any given + thread. Thus multi-threaded programs must take care to coordinate + access to intentionally shared resources. Likewise, they must take + care to be absolutely diligent about not accessing any *other* + resources in multiple threads; otherwise two threads running at the + same time might accidentally interfere with each other’s use of some + shared data. All this is true for both Python programs and the + Python runtime.The cost of this broad, unstructured requirement is + the tradeoff for the kind of raw concurrency that threads provide. + The alternative to the required discipline generally means dealing + with non-deterministic bugs and data corruption. + + +Python Runtime Model +-------------------- + +The same conceptual layers apply to each Python program, with some +extra data layers specific to Python: + + **host machine** + **process** (global resources) + Python global runtime (*state*) + Python interpreter (*state*) + **thread** (runs Python bytecode and “C-API”) + Python thread *state* + +At the conceptual level: when a Python program starts, it looks +exactly like that diagram, with one of each. The runtime may grow to +include multiple interpreters, and each interpreter may grow to +include multiple thread states. + +Note: + + A Python implementation won’t necessarily implement the runtime + layers distinctly or even concretely. The only exception is places + where distinct layers are directly specified or exposed to users, + like through the "threading" module. + +Note: + + The initial interpreter is typically called the “main” interpreter. + Some Python implementations, like CPython, assign special roles to + the main interpreter.Likewise, the host thread where the runtime was + initialized is known as the “main” thread. It may be different from + the process’ initial thread, though they are often the same. In + some cases “main thread” may be even more specific and refer to the + initial thread state. A Python runtime might assign specific + responsibilities to the main thread, such as handling signals. + +As a whole, the Python runtime consists of the global runtime state, +interpreters, and thread states. The runtime ensures all that state +stays consistent over its lifetime, particularly when used with +multiple host threads. + +The global runtime, at the conceptual level, is just a set of +interpreters. While those interpreters are otherwise isolated and +independent from one another, they may share some data or other +resources. The runtime is responsible for managing these global +resources safely. The actual nature and management of these resources +is implementation-specific. Ultimately, the external utility of the +global runtime is limited to managing interpreters. + +In contrast, an “interpreter” is conceptually what we would normally +think of as the (full-featured) “Python runtime”. When machine code +executing in a host thread interacts with the Python runtime, it calls +into Python in the context of a specific interpreter. + +Note: + + The term “interpreter” here is not the same as the “bytecode + interpreter”, which is what regularly runs in threads, executing + compiled Python code.In an ideal world, “Python runtime” would refer + to what we currently call “interpreter”. However, it’s been called + “interpreter” at least since introduced in 1997 (CPython:a027efa5b). + +Each interpreter completely encapsulates all of the non-process- +global, non-thread-specific state needed for the Python runtime to +work. Notably, the interpreter’s state persists between uses. It +includes fundamental data like "sys.modules". The runtime ensures +multiple threads using the same interpreter will safely share it +between them. + +A Python implementation may support using multiple interpreters at the +same time in the same process. They are independent and isolated from +one another. For example, each interpreter has its own "sys.modules". + +For thread-specific runtime state, each interpreter has a set of +thread states, which it manages, in the same way the global runtime +contains a set of interpreters. It can have thread states for as many +host threads as it needs. It may even have multiple thread states for +the same host thread, though that isn’t as common. + +Each thread state, conceptually, has all the thread-specific runtime +data an interpreter needs to operate in one host thread. The thread +state includes the current raised exception and the thread’s Python +call stack. It may include other thread-specific resources. + +Note: + + The term “Python thread” can sometimes refer to a thread state, but + normally it means a thread created using the "threading" module. + +Each thread state, over its lifetime, is always tied to exactly one +interpreter and exactly one host thread. It will only ever be used in +that thread and with that interpreter. + +Multiple thread states may be tied to the same host thread, whether +for different interpreters or even the same interpreter. However, for +any given host thread, only one of the thread states tied to it can be +used by the thread at a time. + +Thread states are isolated and independent from one another and don’t +share any data, except for possibly sharing an interpreter and objects +or other resources belonging to that interpreter. + +Once a program is running, new Python threads can be created using the +"threading" module (on platforms and Python implementations that +support threads). Additional processes can be created using the "os", +"subprocess", and "multiprocessing" modules. Interpreters can be +created and used with the "interpreters" module. Coroutines (async) +can be run using "asyncio" in each interpreter, typically only in a +single thread (often the main thread). + +-[ Footnotes ]- + +[1] This limitation occurs because the code that is executed by these + operations is not available at the time the module is compiled. +''', + 'exprlists': r'''Expression lists +**************** + + starred_expression: "*" or_expr | expression + flexible_expression: assignment_expression | starred_expression + flexible_expression_list: flexible_expression ("," flexible_expression)* [","] + starred_expression_list: starred_expression ("," starred_expression)* [","] + expression_list: expression ("," expression)* [","] + yield_list: expression_list | starred_expression "," [starred_expression_list] + +Except when part of a list or set display, an expression list +containing at least one comma yields a tuple. The length of the tuple +is the number of expressions in the list. The expressions are +evaluated from left to right. + +An asterisk "*" denotes *iterable unpacking*. Its operand must be an +*iterable*. The iterable is expanded into a sequence of items, which +are included in the new tuple, list, or set, at the site of the +unpacking. + +Added in version 3.5: Iterable unpacking in expression lists, +originally proposed by **PEP 448**. + +Added in version 3.11: Any item in an expression list may be starred. +See **PEP 646**. + +A trailing comma is required only to create a one-item tuple, such as +"1,"; it is optional in all other cases. A single expression without a +trailing comma doesn’t create a tuple, but rather yields the value of +that expression. (To create an empty tuple, use an empty pair of +parentheses: "()".) +''', + 'floating': r'''Floating-point literals +*********************** + +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. +''', + 'for': r'''The "for" statement +******************* + +The "for" statement is used to iterate over the elements of a sequence +(such as a string, tuple or list) or other iterable object: + + for_stmt: "for" target_list "in" starred_expression_list ":" suite + ["else" ":" suite] + +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop +terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and continues +with the next item, or with the "else" clause if there is no next +item. + +The for-loop makes assignments to the variables in the target list. +This overwrites all previous assignments to those variables including +those made in the suite of the for-loop: + + for i in range(10): + print(i) + i = 5 # this will not affect the for-loop + # because i will be overwritten with the next + # index in the range + +Names in the target list are not deleted when the loop is finished, +but if the sequence is empty, they will not have been assigned to at +all by the loop. Hint: the built-in type "range()" represents +immutable arithmetic sequences of integers. For instance, iterating +"range(3)" successively yields 0, 1, and then 2. + +Changed in version 3.11: Starred elements are now allowed in the +expression list. +''', + 'formatstrings': r'''Format String Syntax +******************** + +The "str.format()" method and the "Formatter" class share the same +syntax for format strings (although in the case of "Formatter", +subclasses can define their own format string syntax). The syntax is +related to that of formatted string literals and template string +literals, but it is less sophisticated and, in particular, does not +support arbitrary expressions in interpolations. + +Format strings contain “replacement fields” surrounded by curly braces +"{}". Anything that is not contained in braces is considered literal +text, which is copied unchanged to the output. If you need to include +a brace character in the literal text, it can be escaped by doubling: +"{{" and "}}". + +The grammar for a replacement field is as follows: + + replacement_field: "{" [field_name] ["!" conversion] [":" format_spec] "}" + field_name: arg_name ("." attribute_name | "[" element_index "]")* + arg_name: [identifier | digit+] + attribute_name: identifier + element_index: digit+ | index_string + index_string: + + conversion: "r" | "s" | "a" + format_spec: format-spec:format_spec + +In less formal terms, the replacement field can start with a +*field_name* that specifies the object whose value is to be formatted +and inserted into the output instead of the replacement field. The +*field_name* is optionally followed by a *conversion* field, which is +preceded by an exclamation point "'!'", and a *format_spec*, which is +preceded by a colon "':'". These specify a non-default format for the +replacement value. + +See also the Format Specification Mini-Language section. + +The *field_name* itself begins with an *arg_name* that is either a +number or a keyword. If it’s a number, it refers to a positional +argument, and if it’s a keyword, it refers to a named keyword +argument. An *arg_name* is treated as a number if a call to +"str.isdecimal()" on the string would return true. If the numerical +arg_names in a format string are 0, 1, 2, … in sequence, they can all +be omitted (not just some) and the numbers 0, 1, 2, … will be +automatically inserted in that order. Because *arg_name* is not quote- +delimited, it is not possible to specify arbitrary dictionary keys +(e.g., the strings "'10'" or "':-]'") within a format string. The +*arg_name* can be followed by any number of index or attribute +expressions. An expression of the form "'.name'" selects the named +attribute using "getattr()", while an expression of the form +"'[index]'" does an index lookup using "__getitem__()". + +Changed in version 3.1: The positional argument specifiers can be +omitted for "str.format()", so "'{} {}'.format(a, b)" is equivalent to +"'{0} {1}'.format(a, b)". + +Changed in version 3.4: The positional argument specifiers can be +omitted for "Formatter". + +Some simple format string examples: + + "First, thou shalt count to {0}" # References first positional argument + "Bring me a {}" # Implicitly references the first positional argument + "From {} to {}" # Same as "From {0} to {1}" + "My quest is {name}" # References keyword argument 'name' + "Weight in tons {0.weight}" # 'weight' attribute of first positional arg + "Units destroyed: {players[0]}" # First element of keyword argument 'players'. + +The *conversion* field causes a type coercion before formatting. +Normally, the job of formatting a value is done by the "__format__()" +method of the value itself. However, in some cases it is desirable to +force a type to be formatted as a string, overriding its own +definition of formatting. By converting the value to a string before +calling "__format__()", the normal formatting logic is bypassed. + +Three conversion flags are currently supported: "'!s'" which calls +"str()" on the value, "'!r'" which calls "repr()" and "'!a'" which +calls "ascii()". + +Some examples: + + "Harold's a clever {0!s}" # Calls str() on the argument first + "Bring out the holy {name!r}" # Calls repr() on the argument first + "More {!a}" # Calls ascii() on the argument first + +The *format_spec* field contains a specification of how the value +should be presented, including such details as field width, alignment, +padding, decimal precision and so on. Each value type can define its +own “formatting mini-language” or interpretation of the *format_spec*. + +Most built-in types support a common formatting mini-language, which +is described in the next section. + +A *format_spec* field can also include nested replacement fields +within it. These nested replacement fields may contain a field name, +conversion flag and format specification, but deeper nesting is not +allowed. The replacement fields within the format_spec are +substituted before the *format_spec* string is interpreted. This +allows the formatting of a value to be dynamically specified. + +See the Format examples section for some examples. + + +Format Specification Mini-Language +================================== + +“Format specifications” are used within replacement fields contained +within a format string to define how individual values are presented +(see Format String Syntax, f-strings, and t-strings). They can also be +passed directly to the built-in "format()" function. Each formattable +type may define how the format specification is to be interpreted. + +Most built-in types implement the following options for format +specifications, although some of the formatting options are only +supported by the numeric types. + +A general convention is that an empty format specification produces +the same result as if you had called "str()" on the value. A non-empty +format specification typically modifies the result. + +The general form of a *standard format specifier* is: + + format_spec: [options][width_and_precision][type] + options: [[fill]align][sign]["z"]["#"]["0"] + fill: + align: "<" | ">" | "=" | "^" + sign: "+" | "-" | " " + width_and_precision: [width_with_grouping][precision_with_grouping] + width_with_grouping: [width][grouping] + precision_with_grouping: "." [precision][grouping] | "." grouping + width: digit+ + precision: digit+ + grouping: "," | "_" + type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" + | "G" | "n" | "o" | "s" | "x" | "X" | "%" + +If a valid *align* value is specified, it can be preceded by a *fill* +character that can be any character and defaults to a space if +omitted. It is not possible to use a literal curly brace (”"{"” or +“"}"”) as the *fill* character in a formatted string literal or when +using the "str.format()" method. However, it is possible to insert a +curly brace with a nested replacement field. This limitation doesn’t +affect the "format()" function. + +The meaning of the various alignment options is as follows: + ++-----------+------------------------------------------------------------+ +| Option | Meaning | +|===========|============================================================| +| "'<'" | Forces the field to be left-aligned within the available | +| | space (this is the default for most objects). | ++-----------+------------------------------------------------------------+ +| "'>'" | Forces the field to be right-aligned within the available | +| | space (this is the default for numbers). | ++-----------+------------------------------------------------------------+ +| "'='" | Forces the padding to be placed after the sign (if any) | +| | but before the digits. This is used for printing fields | +| | in the form ‘+000000120’. This alignment option is only | +| | valid for numeric types, excluding "complex". It becomes | +| | the default for numbers when ‘0’ immediately precedes the | +| | field width. | ++-----------+------------------------------------------------------------+ +| "'^'" | Forces the field to be centered within the available | +| | space. | ++-----------+------------------------------------------------------------+ + +Note that unless a minimum field width is defined, the field width +will always be the same size as the data to fill it, so that the +alignment option has no meaning in this case. + +The *sign* option is only valid for number types, and can be one of +the following: + ++-----------+------------------------------------------------------------+ +| Option | Meaning | +|===========|============================================================| +| "'+'" | Indicates that a sign should be used for both positive as | +| | well as negative numbers. | ++-----------+------------------------------------------------------------+ +| "'-'" | Indicates that a sign should be used only for negative | +| | numbers (this is the default behavior). | ++-----------+------------------------------------------------------------+ +| space | Indicates that a leading space should be used on positive | +| | numbers, and a minus sign on negative numbers. | ++-----------+------------------------------------------------------------+ + +The "'z'" option coerces negative zero floating-point values to +positive zero after rounding to the format precision. This option is +only valid for floating-point presentation types. + +Changed in version 3.11: Added the "'z'" option (see also **PEP +682**). + +The "'#'" option causes the “alternate form” to be used for the +conversion. The alternate form is defined differently for different +types. This option is only valid for integer, float and complex +types. For integers, when binary, octal, or hexadecimal output is +used, this option adds the respective prefix "'0b'", "'0o'", "'0x'", +or "'0X'" to the output value. For float and complex the alternate +form causes the result of the conversion to always contain a decimal- +point character, even if no digits follow it. Normally, a decimal- +point character appears in the result of these conversions only if a +digit follows it. In addition, for "'g'" and "'G'" conversions, +trailing zeros are not removed from the result. + +The *width* is a decimal integer defining the minimum total field +width, including any prefixes, separators, and other formatting +characters. If not specified, then the field width will be determined +by the content. + +When no explicit alignment is given, preceding the *width* field by a +zero ("'0'") character enables sign-aware zero-padding for numeric +types, excluding "complex". This is equivalent to a *fill* character +of "'0'" with an *alignment* type of "'='". + +Changed in version 3.10: Preceding the *width* field by "'0'" no +longer affects the default alignment for strings. + +The *precision* is a decimal integer indicating how many digits should +be displayed after the decimal point for presentation types "'f'" and +"'F'", or before and after the decimal point for presentation types +"'g'" or "'G'". For string presentation types the field indicates the +maximum field size - in other words, how many characters will be used +from the field content. The *precision* is not allowed for integer +presentation types. + +The *grouping* option after *width* and *precision* fields specifies a +digit group separator for the integral and fractional parts of a +number respectively. It can be one of the following: + ++-----------+------------------------------------------------------------+ +| Option | Meaning | +|===========|============================================================| +| "','" | Inserts a comma every 3 digits for integer presentation | +| | type "'d'" and floating-point presentation types, | +| | excluding "'n'". For other presentation types, this option | +| | is not supported. | ++-----------+------------------------------------------------------------+ +| "'_'" | Inserts an underscore every 3 digits for integer | +| | presentation type "'d'" and floating-point presentation | +| | types, excluding "'n'". For integer presentation types | +| | "'b'", "'o'", "'x'", and "'X'", underscores are inserted | +| | every 4 digits. For other presentation types, this option | +| | is not supported. | ++-----------+------------------------------------------------------------+ + +For a locale aware separator, use the "'n'" presentation type instead. + +Changed in version 3.1: Added the "','" option (see also **PEP 378**). + +Changed in version 3.6: Added the "'_'" option (see also **PEP 515**). + +Changed in version 3.14: Support the *grouping* option for the +fractional part. + +Finally, the *type* determines how the data should be presented. + +The available string presentation types are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'s'" | String format. This is the default type for strings and | + | | may be omitted. | + +-----------+------------------------------------------------------------+ + | None | The same as "'s'". | + +-----------+------------------------------------------------------------+ + +The available integer presentation types are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'b'" | Binary format. Outputs the number in base 2. | + +-----------+------------------------------------------------------------+ + | "'c'" | Character. Converts the integer to the corresponding | + | | unicode character before printing. | + +-----------+------------------------------------------------------------+ + | "'d'" | Decimal Integer. Outputs the number in base 10. | + +-----------+------------------------------------------------------------+ + | "'o'" | Octal format. Outputs the number in base 8. | + +-----------+------------------------------------------------------------+ + | "'x'" | Hex format. Outputs the number in base 16, using lower- | + | | case letters for the digits above 9. | + +-----------+------------------------------------------------------------+ + | "'X'" | Hex format. Outputs the number in base 16, using upper- | + | | case letters for the digits above 9. In case "'#'" is | + | | specified, the prefix "'0x'" will be upper-cased to "'0X'" | + | | as well. | + +-----------+------------------------------------------------------------+ + | "'n'" | Number. This is the same as "'d'", except that it uses the | + | | current locale setting to insert the appropriate digit | + | | group separators. | + +-----------+------------------------------------------------------------+ + | None | The same as "'d'". | + +-----------+------------------------------------------------------------+ + +In addition to the above presentation types, integers can be formatted +with the floating-point presentation types listed below (except "'n'" +and "None"). When doing so, "float()" is used to convert the integer +to a floating-point number before formatting. + +The available presentation types for "float" and "Decimal" values are: + + +-----------+------------------------------------------------------------+ + | Type | Meaning | + |===========|============================================================| + | "'e'" | Scientific notation. For a given precision "p", formats | + | | the number in scientific notation with the letter ‘e’ | + | | separating the coefficient from the exponent. The | + | | coefficient has one digit before and "p" digits after the | + | | decimal point, for a total of "p + 1" significant digits. | + | | With no precision given, uses a precision of "6" digits | + | | after the decimal point for "float", and shows all | + | | coefficient digits for "Decimal". If "p=0", the decimal | + | | point is omitted unless the "#" option is used. For | + | | "float", the exponent always contains at least two digits, | + | | and is zero if the value is zero. | + +-----------+------------------------------------------------------------+ + | "'E'" | Scientific notation. Same as "'e'" except it uses an upper | + | | case ‘E’ as the separator character. | + +-----------+------------------------------------------------------------+ + | "'f'" | Fixed-point notation. For a given precision "p", formats | + | | the number as a decimal number with exactly "p" digits | + | | following the decimal point. With no precision given, uses | + | | a precision of "6" digits after the decimal point for | + | | "float", and uses a precision large enough to show all | + | | coefficient digits for "Decimal". If "p=0", the decimal | + | | point is omitted unless the "#" option is used. | + +-----------+------------------------------------------------------------+ + | "'F'" | Fixed-point notation. Same as "'f'", but converts "nan" to | + | | "NAN" and "inf" to "INF". | + +-----------+------------------------------------------------------------+ + | "'g'" | General format. For a given precision "p >= 1", this | + | | rounds the number to "p" significant digits and then | + | | formats the result in either fixed-point format or in | + | | scientific notation, depending on its magnitude. A | + | | precision of "0" is treated as equivalent to a precision | + | | of "1". The precise rules are as follows: suppose that | + | | the result formatted with presentation type "'e'" and | + | | precision "p-1" would have exponent "exp". Then, if "m <= | + | | exp < p", where "m" is -4 for floats and -6 for | + | | "Decimals", the number is formatted with presentation type | + | | "'f'" and precision "p-1-exp". Otherwise, the number is | + | | formatted with presentation type "'e'" and precision | + | | "p-1". In both cases insignificant trailing zeros are | + | | removed from the significand, and the decimal point is | + | | also removed if there are no remaining digits following | + | | it, unless the "'#'" option is used. With no precision | + | | given, uses a precision of "6" significant digits for | + | | "float". For "Decimal", the coefficient of the result is | + | | formed from the coefficient digits of the value; | + | | scientific notation is used for values smaller than "1e-6" | + | | in absolute value and values where the place value of the | + | | least significant digit is larger than 1, and fixed-point | + | | notation is used otherwise. Positive and negative | + | | infinity, positive and negative zero, and nans, are | + | | formatted as "inf", "-inf", "0", "-0" and "nan" | + | | respectively, regardless of the precision. | + +-----------+------------------------------------------------------------+ + | "'G'" | General format. Same as "'g'" except switches to "'E'" if | + | | the number gets too large. The representations of infinity | + | | and NaN are uppercased, too. | + +-----------+------------------------------------------------------------+ + | "'n'" | Number. This is the same as "'g'", except that it uses the | + | | current locale setting to insert the appropriate digit | + | | group separators for the integral part of a number. | + +-----------+------------------------------------------------------------+ + | "'%'" | Percentage. Multiplies the number by 100 and displays in | + | | fixed ("'f'") format, followed by a percent sign. | + +-----------+------------------------------------------------------------+ + | None | For "float" this is like the "'g'" type, except that when | + | | fixed- point notation is used to format the result, it | + | | always includes at least one digit past the decimal point, | + | | and switches to the scientific notation when "exp >= p - | + | | 1". When the precision is not specified, the latter will | + | | be as large as needed to represent the given value | + | | faithfully. For "Decimal", this is the same as either | + | | "'g'" or "'G'" depending on the value of | + | | "context.capitals" for the current decimal context. The | + | | overall effect is to match the output of "str()" as | + | | altered by the other format modifiers. | + +-----------+------------------------------------------------------------+ + +The result should be correctly rounded to a given precision "p" of +digits after the decimal point. The rounding mode for "float" matches +that of the "round()" builtin. For "Decimal", the rounding mode of +the current context will be used. + +The available presentation types for "complex" are the same as those +for "float" ("'%'" is not allowed). Both the real and imaginary +components of a complex number are formatted as floating-point +numbers, according to the specified presentation type. They are +separated by the mandatory sign of the imaginary part, the latter +being terminated by a "j" suffix. If the presentation type is +missing, the result will match the output of "str()" (complex numbers +with a non-zero real part are also surrounded by parentheses), +possibly altered by other format modifiers. + + +Format examples +=============== + +This section contains examples of the "str.format()" syntax and +comparison with the old "%"-formatting. + +In most of the cases the syntax is similar to the old "%"-formatting, +with the addition of the "{}" and with ":" used instead of "%". For +example, "'%03.2f'" can be translated to "'{:03.2f}'". + +The new format syntax also supports new and different options, shown +in the following examples. + +Accessing arguments by position: + + >>> '{0}, {1}, {2}'.format('a', 'b', 'c') + 'a, b, c' + >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only + 'a, b, c' + >>> '{2}, {1}, {0}'.format('a', 'b', 'c') + 'c, b, a' + >>> '{2}, {1}, {0}'.format(*'abc') # unpacking argument sequence + 'c, b, a' + >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' indices can be repeated + 'abracadabra' + +Accessing arguments by name: + + >>> 'Coordinates: {latitude}, {longitude}'.format(latitude='37.24N', longitude='-115.81W') + 'Coordinates: 37.24N, -115.81W' + >>> coord = {'latitude': '37.24N', 'longitude': '-115.81W'} + >>> 'Coordinates: {latitude}, {longitude}'.format(**coord) + 'Coordinates: 37.24N, -115.81W' + +Accessing arguments’ attributes: + + >>> c = 3-5j + >>> ('The complex number {0} is formed from the real part {0.real} ' + ... 'and the imaginary part {0.imag}.').format(c) + 'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.' + >>> class Point: + ... def __init__(self, x, y): + ... self.x, self.y = x, y + ... def __str__(self): + ... return 'Point({self.x}, {self.y})'.format(self=self) + ... + >>> str(Point(4, 2)) + 'Point(4, 2)' + +Accessing arguments’ items: + + >>> coord = (3, 5) + >>> 'X: {0[0]}; Y: {0[1]}'.format(coord) + 'X: 3; Y: 5' + +Replacing "%s" and "%r": + + >>> "repr() shows quotes: {!r}; str() doesn't: {!s}".format('test1', 'test2') + "repr() shows quotes: 'test1'; str() doesn't: test2" + +Aligning the text and specifying a width: + + >>> '{:<30}'.format('left aligned') + 'left aligned ' + >>> '{:>30}'.format('right aligned') + ' right aligned' + >>> '{:^30}'.format('centered') + ' centered ' + >>> '{:*^30}'.format('centered') # use '*' as a fill char + '***********centered***********' + +Replacing "%+f", "%-f", and "% f" and specifying a sign: + + >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it always + '+3.140000; -3.140000' + >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space for positive numbers + ' 3.140000; -3.140000' + >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the minus -- same as '{:f}; {:f}' + '3.140000; -3.140000' + +Replacing "%x" and "%o" and converting the value to different bases: + + >>> # format also supports binary numbers + >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42) + 'int: 42; hex: 2a; oct: 52; bin: 101010' + >>> # with 0x, 0o, or 0b as prefix: + >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42) + 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010' + +Using the comma or the underscore as a digit group separator: + + >>> '{:,}'.format(1234567890) + '1,234,567,890' + >>> '{:_}'.format(1234567890) + '1_234_567_890' + >>> '{:_b}'.format(1234567890) + '100_1001_1001_0110_0000_0010_1101_0010' + >>> '{:_x}'.format(1234567890) + '4996_02d2' + >>> '{:_}'.format(123456789.123456789) + '123_456_789.12345679' + >>> '{:.,}'.format(123456789.123456789) + '123456789.123,456,79' + >>> '{:,._}'.format(123456789.123456789) + '123,456,789.123_456_79' + +Expressing a percentage: + + >>> points = 19 + >>> total = 22 + >>> 'Correct answers: {:.2%}'.format(points/total) + 'Correct answers: 86.36%' + +Using type-specific formatting: + + >>> import datetime + >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) + '2010-07-04 12:15:58' + +Nesting arguments and more complex examples: + + >>> for align, text in zip('<^>', ['left', 'center', 'right']): + ... '{0:{fill}{align}16}'.format(text, fill=align, align=align) + ... + 'left<<<<<<<<<<<<' + '^^^^^center^^^^^' + '>>>>>>>>>>>right' + >>> + >>> octets = [192, 168, 0, 1] + >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets) + 'C0A80001' + >>> int(_, 16) + 3232235521 + >>> + >>> width = 5 + >>> for num in range(5,12): + ... for base in 'dXob': + ... print('{0:{width}{base}}'.format(num, base=base, width=width), end=' ') + ... print() + ... + 5 5 5 101 + 6 6 6 110 + 7 7 7 111 + 8 8 10 1000 + 9 9 11 1001 + 10 A 12 1010 + 11 B 13 1011 +''', + 'function': r'''Function definitions +******************** + +A function definition defines a user-defined function object (see +section The standard type hierarchy): + + funcdef: [decorators] "def" funcname [type_params] "(" [parameter_list] ")" + ["->" expression] ":" suite + decorators: decorator+ + decorator: "@" assignment_expression NEWLINE + parameter_list: defparameter ("," defparameter)* "," "/" ["," [parameter_list_no_posonly]] + | parameter_list_no_posonly + parameter_list_no_posonly: defparameter ("," defparameter)* ["," [parameter_list_starargs]] + | parameter_list_starargs + parameter_list_starargs: "*" [star_parameter] ("," defparameter)* ["," [parameter_star_kwargs]] + | "*" ("," defparameter)+ ["," [parameter_star_kwargs]] + | parameter_star_kwargs + parameter_star_kwargs: "**" parameter [","] + parameter: identifier [":" expression] + star_parameter: identifier [":" ["*"] expression] + defparameter: parameter ["=" expression] + funcname: identifier + +A function definition is an executable statement. Its execution binds +the function name in the current local namespace to a function object +(a wrapper around the executable code for the function). This +function object contains a reference to the current global namespace +as the global namespace to be used when the function is called. + +The function definition does not execute the function body; this gets +executed only when the function is called. [4] + +A function definition may be wrapped by one or more *decorator* +expressions. Decorator expressions are evaluated when the function is +defined, in the scope that contains the function definition. The +result must be a callable, which is invoked with the function object +as the only argument. The returned value is bound to the function name +instead of the function object. Multiple decorators are applied in +nested fashion. For example, the following code + + @f1(arg) + @f2 + def func(): pass + +is roughly equivalent to + + def func(): pass + func = f1(arg)(f2(func)) + +except that the original function is not temporarily bound to the name +"func". + +Changed in version 3.9: Functions may be decorated with any valid +"assignment_expression". Previously, the grammar was much more +restrictive; see **PEP 614** for details. + +A list of type parameters may be given in square brackets between the +function’s name and the opening parenthesis for its parameter list. +This indicates to static type checkers that the function is generic. +At runtime, the type parameters can be retrieved from the function’s +"__type_params__" attribute. See Generic functions for more. + +Changed in version 3.12: Type parameter lists are new in Python 3.12. + +When one or more *parameters* have the form *parameter* "=" +*expression*, the function is said to have “default parameter values.” +For a parameter with a default value, the corresponding *argument* may +be omitted from a call, in which case the parameter’s default value is +substituted. If a parameter has a default value, all following +parameters up until the “"*"” must also have a default value — this is +a syntactic restriction that is not expressed by the grammar. + +**Default parameter values are evaluated from left to right when the +function definition is executed.** This means that the expression is +evaluated once, when the function is defined, and that the same “pre- +computed” value is used for each call. This is especially important +to understand when a default parameter value is a mutable object, such +as a list or a dictionary: if the function modifies the object (e.g. +by appending an item to a list), the default parameter value is in +effect modified. This is generally not what was intended. A way +around this is to use "None" as the default, and explicitly test for +it in the body of the function, e.g.: + + def whats_on_the_telly(penguin=None): + if penguin is None: + penguin = [] + penguin.append("property of the zoo") + return penguin + +Function call semantics are described in more detail in section Calls. +A function call always assigns values to all parameters mentioned in +the parameter list, either from positional arguments, from keyword +arguments, or from default values. If the form “"*identifier"” is +present, it is initialized to a tuple receiving any excess positional +parameters, defaulting to the empty tuple. If the form +“"**identifier"” is present, it is initialized to a new ordered +mapping receiving any excess keyword arguments, defaulting to a new +empty mapping of the same type. Parameters after “"*"” or +“"*identifier"” are keyword-only parameters and may only be passed by +keyword arguments. Parameters before “"/"” are positional-only +parameters and may only be passed by positional arguments. + +Changed in version 3.8: The "/" function parameter syntax may be used +to indicate positional-only parameters. See **PEP 570** for details. + +Parameters may have an *annotation* of the form “": expression"” +following the parameter name. Any parameter may have an annotation, +even those of the form "*identifier" or "**identifier". (As a special +case, parameters of the form "*identifier" may have an annotation “": +*expression"”.) Functions may have “return” annotation of the form +“"-> expression"” after the parameter list. These annotations can be +any valid Python expression. The presence of annotations does not +change the semantics of a function. See Annotations for more +information on annotations. + +Changed in version 3.11: Parameters of the form “"*identifier"” may +have an annotation “": *expression"”. See **PEP 646**. + +It is also possible to create anonymous functions (functions not bound +to a name), for immediate use in expressions. This uses lambda +expressions, described in section Lambdas. Note that the lambda +expression is merely a shorthand for a simplified function definition; +a function defined in a “"def"” statement can be passed around or +assigned to another name just like a function defined by a lambda +expression. The “"def"” form is actually more powerful since it +allows the execution of multiple statements and annotations. + +**Programmer’s note:** Functions are first-class objects. A “"def"” +statement executed inside a function definition defines a local +function that can be returned or passed around. Free variables used +in the nested function can access the local variables of the function +containing the def. See section Naming and binding for details. + +See also: + + **PEP 3107** - Function Annotations + The original specification for function annotations. + + **PEP 484** - Type Hints + Definition of a standard meaning for annotations: type hints. + + **PEP 526** - Syntax for Variable Annotations + Ability to type hint variable declarations, including class + variables and instance variables. + + **PEP 563** - Postponed Evaluation of Annotations + Support for forward references within annotations by preserving + annotations in a string form at runtime instead of eager + evaluation. + + **PEP 318** - Decorators for Functions and Methods + Function and method decorators were introduced. Class decorators + were introduced in **PEP 3129**. +''', + 'global': r'''The "global" statement +********************** + + global_stmt: "global" identifier ("," identifier)* + +The "global" statement causes the listed identifiers to be interpreted +as globals. It would be impossible to assign to a global variable +without "global", although free variables may refer to globals without +being declared global. + +The "global" statement applies to the entire current scope (module, +function body or class definition). A "SyntaxError" is raised if a +variable is used or assigned to prior to its global declaration in the +scope. + +At the module level, all variables are global, so a "global" statement +has no effect. However, variables must still not be used or assigned +to prior to their "global" declaration. This requirement is relaxed in +the interactive prompt (*REPL*). + +**Programmer’s note:** "global" is a directive to the parser. It +applies only to code parsed at the same time as the "global" +statement. In particular, a "global" statement contained in a string +or code object supplied to the built-in "exec()" function does not +affect the code block *containing* the function call, and code +contained in such a string is unaffected by "global" statements in the +code containing the function call. The same applies to the "eval()" +and "compile()" functions. +''', + 'id-classes': r'''Reserved classes of identifiers +******************************* + +Certain classes of identifiers (besides keywords) have special +meanings. These classes are identified by the patterns of leading and +trailing underscore characters: + +"_*" + Not imported by "from module import *". + +"_" + In a "case" pattern within a "match" statement, "_" is a soft + keyword that denotes a wildcard. + + Separately, the interactive interpreter makes the result of the + last evaluation available in the variable "_". (It is stored in the + "builtins" module, alongside built-in functions like "print".) + + Elsewhere, "_" is a regular identifier. It is often used to name + “special” items, but it is not special to Python itself. + + Note: + + The name "_" is often used in conjunction with + internationalization; refer to the documentation for the + "gettext" module for more information on this convention.It is + also commonly used for unused variables. + +"__*__" + System-defined names, informally known as “dunder” names. These + names are defined by the interpreter and its implementation + (including the standard library). Current system names are + discussed in the Special method names section and elsewhere. More + will likely be defined in future versions of Python. *Any* use of + "__*__" names, in any context, that does not follow explicitly + documented use, is subject to breakage without warning. + +"__*" + Class-private names. Names in this category, when used within the + context of a class definition, are re-written to use a mangled form + to help avoid name clashes between “private” attributes of base and + derived classes. See section Identifiers (Names). +''', + 'identifiers': r'''Names (identifiers and keywords) +******************************** + +"NAME" tokens represent *identifiers*, *keywords*, and *soft +keywords*. + +Names are composed of the following characters: + +* uppercase and lowercase letters ("A-Z" and "a-z"), + +* the underscore ("_"), + +* digits ("0" through "9"), which cannot appear as the first + character, and + +* non-ASCII characters. Valid names may only contain “letter-like” and + “digit-like” characters; see Non-ASCII characters in names for + details. + +Names must contain at least one character, but have no upper length +limit. Case is significant. + +Formally, names are described by the following lexical definitions: + + NAME: name_start name_continue* + name_start: "a"..."z" | "A"..."Z" | "_" | + name_continue: name_start | "0"..."9" + identifier: + +Note that not all names matched by this grammar are valid; see Non- +ASCII characters in names for details. + + +Keywords +======== + +The following names are used as reserved words, or *keywords* of the +language, and cannot be used as ordinary identifiers. They must be +spelled exactly as written here: + + False await else import pass + None break except in raise + True class finally is return + and continue for lambda try + as def from nonlocal while + assert del global not with + async elif if or yield + + +Soft Keywords +============= + +Added in version 3.10. + +Some names are only reserved under specific contexts. These are known +as *soft keywords*: + +* "match", "case", and "_", when used in the "match" statement. + +* "type", when used in the "type" statement. + +These syntactically act as keywords in their specific contexts, but +this distinction is done at the parser level, not when tokenizing. + +As soft keywords, their use in the grammar is possible while still +preserving compatibility with existing code that uses these names as +identifier names. + +Changed in version 3.12: "type" is now a soft keyword. + + +Reserved classes of identifiers +=============================== + +Certain classes of identifiers (besides keywords) have special +meanings. These classes are identified by the patterns of leading and +trailing underscore characters: + +"_*" + Not imported by "from module import *". + +"_" + In a "case" pattern within a "match" statement, "_" is a soft + keyword that denotes a wildcard. + + Separately, the interactive interpreter makes the result of the + last evaluation available in the variable "_". (It is stored in the + "builtins" module, alongside built-in functions like "print".) + + Elsewhere, "_" is a regular identifier. It is often used to name + “special” items, but it is not special to Python itself. + + Note: + + The name "_" is often used in conjunction with + internationalization; refer to the documentation for the + "gettext" module for more information on this convention.It is + also commonly used for unused variables. + +"__*__" + System-defined names, informally known as “dunder” names. These + names are defined by the interpreter and its implementation + (including the standard library). Current system names are + discussed in the Special method names section and elsewhere. More + will likely be defined in future versions of Python. *Any* use of + "__*__" names, in any context, that does not follow explicitly + documented use, is subject to breakage without warning. + +"__*" + Class-private names. Names in this category, when used within the + context of a class definition, are re-written to use a mangled form + to help avoid name clashes between “private” attributes of base and + derived classes. See section Identifiers (Names). + + +Non-ASCII characters in names +============================= + +Names that contain non-ASCII characters need additional normalization +and validation beyond the rules and grammar explained above. For +example, "ř_1", "蛇", or "साँप" are valid names, but "r〰2", "€", or +"🐍" are not. + +This section explains the exact rules. + +All names are converted into the normalization form NFKC while +parsing. This means that, for example, some typographic variants of +characters are converted to their “basic” form. For example, +"fiⁿₐˡᵢᶻₐᵗᵢᵒₙ" normalizes to "finalization", so Python treats them as +the same name: + + >>> fiⁿₐˡᵢᶻₐᵗᵢᵒₙ = 3 + >>> finalization + 3 + +Note: + + Normalization is done at the lexical level only. Run-time functions + that take names as *strings* generally do not normalize their + arguments. For example, the variable defined above is accessible at + run time in the "globals()" dictionary as + "globals()["finalization"]" but not "globals()["fiⁿₐˡᵢᶻₐᵗᵢᵒₙ"]". + +Similarly to how ASCII-only names must contain only letters, digits +and the underscore, and cannot start with a digit, a valid name must +start with a character in the “letter-like” set "xid_start", and the +remaining characters must be in the “letter- and digit-like” set +"xid_continue". + +These sets based on the *XID_Start* and *XID_Continue* sets as defined +by the Unicode standard annex UAX-31. Python’s "xid_start" +additionally includes the underscore ("_"). Note that Python does not +necessarily conform to UAX-31. + +A non-normative listing of characters in the *XID_Start* and +*XID_Continue* sets as defined by Unicode is available in the +DerivedCoreProperties.txt file in the Unicode Character Database. For +reference, the construction rules for the "xid_*" sets are given +below. + +The set "id_start" is defined as the union of: + +* Unicode category "" - uppercase letters (includes "A" to "Z") + +* Unicode category "" - lowercase letters (includes "a" to "z") + +* Unicode category "" - titlecase letters + +* Unicode category "" - modifier letters + +* Unicode category "" - other letters + +* Unicode category "" - letter numbers + +* {""_""} - the underscore + +* "" - an explicit set of characters in PropList.txt + to support backwards compatibility + +The set "xid_start" then closes this set under NFKC normalization, by +removing all characters whose normalization is not of the form +"id_start id_continue*". + +The set "id_continue" is defined as the union of: + +* "id_start" (see above) + +* Unicode category "" - decimal numbers (includes "0" to "9") + +* Unicode category "" - connector punctuations + +* Unicode category "" - nonspacing marks + +* Unicode category "" - spacing combining marks + +* "" - another explicit set of characters in + PropList.txt to support backwards compatibility + +Again, "xid_continue" closes this set under NFKC normalization. + +Unicode categories use the version of the Unicode Character Database +as included in the "unicodedata" module. + +See also: + + * **PEP 3131** – Supporting Non-ASCII Identifiers + + * **PEP 672** – Unicode-related Security Considerations for Python +''', + 'if': r'''The "if" statement +****************** + +The "if" statement is used for conditional execution: + + if_stmt: "if" assignment_expression ":" suite + ("elif" assignment_expression ":" suite)* + ["else" ":" suite] + +It selects exactly one of the suites by evaluating the expressions one +by one until one is found to be true (see section Boolean operations +for the definition of true and false); then that suite is executed +(and no other part of the "if" statement is executed or evaluated). +If all expressions are false, the suite of the "else" clause, if +present, is executed. +''', + 'imaginary': r'''Imaginary literals +****************** + +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. + +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") +''', + 'import': r'''The "import" statement +********************** + + import_stmt: "import" module ["as" identifier] ("," module ["as" identifier])* + | "from" relative_module "import" identifier ["as" identifier] + ("," identifier ["as" identifier])* + | "from" relative_module "import" "(" identifier ["as" identifier] + ("," identifier ["as" identifier])* [","] ")" + | "from" relative_module "import" "*" + module: (identifier ".")* identifier + relative_module: "."* module | "."+ + +The basic import statement (no "from" clause) is executed in two +steps: + +1. find a module, loading and initializing it if necessary + +2. define a name or names in the local namespace for the scope where + the "import" statement occurs. + +When the statement contains multiple clauses (separated by commas) the +two steps are carried out separately for each clause, just as though +the clauses had been separated out into individual import statements. + +The details of the first step, finding and loading modules, are +described in greater detail in the section on the import system, which +also describes the various types of packages and modules that can be +imported, as well as all the hooks that can be used to customize the +import system. Note that failures in this step may indicate either +that the module could not be located, *or* that an error occurred +while initializing the module, which includes execution of the +module’s code. + +If the requested module is retrieved successfully, it will be made +available in the local namespace in one of three ways: + +* If the module name is followed by "as", then the name following "as" + is bound directly to the imported module. + +* If no other name is specified, and the module being imported is a + top level module, the module’s name is bound in the local namespace + as a reference to the imported module + +* If the module being imported is *not* a top level module, then the + name of the top level package that contains the module is bound in + the local namespace as a reference to the top level package. The + imported module must be accessed using its full qualified name + rather than directly + +The "from" form uses a slightly more complex process: + +1. find the module specified in the "from" clause, loading and + initializing it if necessary; + +2. for each of the identifiers specified in the "import" clauses: + + 1. check if the imported module has an attribute by that name + + 2. if not, attempt to import a submodule with that name and then + check the imported module again for that attribute + + 3. if the attribute is not found, "ImportError" is raised. + + 4. otherwise, a reference to that value is stored in the local + namespace, using the name in the "as" clause if it is present, + otherwise using the attribute name + +Examples: + + import foo # foo imported and bound locally + import foo.bar.baz # foo, foo.bar, and foo.bar.baz imported, foo bound locally + import foo.bar.baz as fbb # foo, foo.bar, and foo.bar.baz imported, foo.bar.baz bound as fbb + from foo.bar import baz # foo, foo.bar, and foo.bar.baz imported, foo.bar.baz bound as baz + from foo import attr # foo imported and foo.attr bound as attr + +If the list of identifiers is replaced by a star ("'*'"), all public +names defined in the module are bound in the local namespace for the +scope where the "import" statement occurs. + +The *public names* defined by a module are determined by checking the +module’s namespace for a variable named "__all__"; if defined, it must +be a sequence of strings which are names defined or imported by that +module. The names given in "__all__" are all considered public and +are required to exist. If "__all__" is not defined, the set of public +names includes all names found in the module’s namespace which do not +begin with an underscore character ("'_'"). "__all__" should contain +the entire public API. It is intended to avoid accidentally exporting +items that are not part of the API (such as library modules which were +imported and used within the module). + +The wild card form of import — "from module import *" — is only +allowed at the module level. Attempting to use it in class or +function definitions will raise a "SyntaxError". + +When specifying what module to import you do not have to specify the +absolute name of the module. When a module or package is contained +within another package it is possible to make a relative import within +the same top package without having to mention the package name. By +using leading dots in the specified module or package after "from" you +can specify how high to traverse up the current package hierarchy +without specifying exact names. One leading dot means the current +package where the module making the import exists. Two dots means up +one package level. Three dots is up two levels, etc. So if you execute +"from . import mod" from a module in the "pkg" package then you will +end up importing "pkg.mod". If you execute "from ..subpkg2 import mod" +from within "pkg.subpkg1" you will import "pkg.subpkg2.mod". The +specification for relative imports is contained in the Package +Relative Imports section. + +"importlib.import_module()" is provided to support applications that +determine dynamically the modules to be loaded. + +Raises an auditing event "import" with arguments "module", "filename", +"sys.path", "sys.meta_path", "sys.path_hooks". + + +Future statements +================= + +A *future statement* is a directive to the compiler that a particular +module should be compiled using syntax or semantics that will be +available in a specified future release of Python where the feature +becomes standard. + +The future statement is intended to ease migration to future versions +of Python that introduce incompatible changes to the language. It +allows use of the new features on a per-module basis before the +release in which the feature becomes standard. + + future_stmt: "from" "__future__" "import" feature ["as" identifier] + ("," feature ["as" identifier])* + | "from" "__future__" "import" "(" feature ["as" identifier] + ("," feature ["as" identifier])* [","] ")" + feature: identifier + +A future statement must appear near the top of the module. The only +lines that can appear before a future statement are: + +* the module docstring (if any), + +* comments, + +* blank lines, and + +* other future statements. + +The only feature that requires using the future statement is +"annotations" (see **PEP 563**). + +All historical features enabled by the future statement are still +recognized by Python 3. The list includes "absolute_import", +"division", "generators", "generator_stop", "unicode_literals", +"print_function", "nested_scopes" and "with_statement". They are all +redundant because they are always enabled, and only kept for backwards +compatibility. + +A future statement is recognized and treated specially at compile +time: Changes to the semantics of core constructs are often +implemented by generating different code. It may even be the case +that a new feature introduces new incompatible syntax (such as a new +reserved word), in which case the compiler may need to parse the +module differently. Such decisions cannot be pushed off until +runtime. + +For any given release, the compiler knows which feature names have +been defined, and raises a compile-time error if a future statement +contains a feature not known to it. + +The direct runtime semantics are the same as for any import statement: +there is a standard module "__future__", described later, and it will +be imported in the usual way at the time the future statement is +executed. + +The interesting runtime semantics depend on the specific feature +enabled by the future statement. + +Note that there is nothing special about the statement: + + import __future__ [as name] + +That is not a future statement; it’s an ordinary import statement with +no special semantics or syntax restrictions. + +Code compiled by calls to the built-in functions "exec()" and +"compile()" that occur in a module "M" containing a future statement +will, by default, use the new syntax or semantics associated with the +future statement. This can be controlled by optional arguments to +"compile()" — see the documentation of that function for details. + +A future statement typed at an interactive interpreter prompt will +take effect for the rest of the interpreter session. If an +interpreter is started with the "-i" option, is passed a script name +to execute, and the script includes a future statement, it will be in +effect in the interactive session started after the script is +executed. + +See also: + + **PEP 236** - Back to the __future__ + The original proposal for the __future__ mechanism. +''', + 'in': r'''Membership test operations +************************** + +The operators "in" and "not in" test for membership. "x in s" +evaluates to "True" if *x* is a member of *s*, and "False" otherwise. +"x not in s" returns the negation of "x in s". All built-in sequences +and set types support this as well as dictionary, for which "in" tests +whether the dictionary has a given key. For container types such as +list, tuple, set, frozenset, dict, or collections.deque, the +expression "x in y" is equivalent to "any(x is e or x == e for e in +y)". + +For the string and bytes types, "x in y" is "True" if and only if *x* +is a substring of *y*. An equivalent test is "y.find(x) != -1". +Empty strings are always considered to be a substring of any other +string, so """ in "abc"" will return "True". + +For user-defined classes which define the "__contains__()" method, "x +in y" returns "True" if "y.__contains__(x)" returns a true value, and +"False" otherwise. + +For user-defined classes which do not define "__contains__()" but do +define "__iter__()", "x in y" is "True" if some value "z", for which +the expression "x is z or x == z" is true, is produced while iterating +over "y". If an exception is raised during the iteration, it is as if +"in" raised that exception. + +Lastly, the old-style iteration protocol is tried: if a class defines +"__getitem__()", "x in y" is "True" if and only if there is a non- +negative integer index *i* such that "x is y[i] or x == y[i]", and no +lower integer index raises the "IndexError" exception. (If any other +exception is raised, it is as if "in" raised that exception). + +The operator "not in" is defined to have the inverse truth value of +"in". +''', + 'integers': r'''Integer literals +**************** + +Integer literals denote whole numbers. For example: + + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* + bininteger: "0" ("b" | "B") (["_"] bindigit)+ + octinteger: "0" ("o" | "O") (["_"] octdigit)+ + hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* + nonzerodigit: "1"..."9" + digit: "0"..."9" + bindigit: "0" | "1" + octdigit: "0"..."7" + hexdigit: digit | "a"..."f" | "A"..."F" + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. +''', + 'lambda': r'''Lambdas +******* + + lambda_expr: "lambda" [parameter_list] ":" expression + +Lambda expressions (sometimes called lambda forms) are used to create +anonymous functions. The expression "lambda parameters: expression" +yields a function object. The unnamed object behaves like a function +object defined with: + + def (parameters): + return expression + +See section Function definitions for the syntax of parameter lists. +Note that functions created with lambda expressions cannot contain +statements or annotations. +''', + 'lists': r'''List displays +************* + +A list display is a possibly empty series of expressions enclosed in +square brackets: + + list_display: "[" [flexible_expression_list | comprehension] "]" + +A list display yields a new list object, the contents being specified +by either a list of expressions or a comprehension. When a comma- +separated list of expressions is supplied, its elements are evaluated +from left to right and placed into the list object in that order. +When a comprehension is supplied, the list is constructed from the +elements resulting from the comprehension. +''', + 'naming': r'''Naming and binding +****************** + + +Binding of names +================ + +*Names* refer to objects. Names are introduced by name binding +operations. + +The following constructs bind names: + +* formal parameters to functions, + +* class definitions, + +* function definitions, + +* assignment expressions, + +* targets that are identifiers if occurring in an assignment: + + * "for" loop header, + + * after "as" in a "with" statement, "except" clause, "except*" + clause, or in the as-pattern in structural pattern matching, + + * in a capture pattern in structural pattern matching + +* "import" statements. + +* "type" statements. + +* type parameter lists. + +The "import" statement of the form "from ... import *" binds all names +defined in the imported module, except those beginning with an +underscore. This form may only be used at the module level. + +A target occurring in a "del" statement is also considered bound for +this purpose (though the actual semantics are to unbind the name). + +Each assignment or import statement occurs within a block defined by a +class or function definition or at the module level (the top-level +code block). + +If a name is bound in a block, it is a local variable of that block, +unless declared as "nonlocal" or "global". If a name is bound at the +module level, it is a global variable. (The variables of the module +code block are local and global.) If a variable is used in a code +block but not defined there, it is a *free variable*. + +Each occurrence of a name in the program text refers to the *binding* +of that name established by the following name resolution rules. + + +Resolution of names +=================== + +A *scope* defines the visibility of a name within a block. If a local +variable is defined in a block, its scope includes that block. If the +definition occurs in a function block, the scope extends to any blocks +contained within the defining one, unless a contained block introduces +a different binding for the name. + +When a name is used in a code block, it is resolved using the nearest +enclosing scope. The set of all such scopes visible to a code block +is called the block’s *environment*. + +When a name is not found at all, a "NameError" exception is raised. If +the current scope is a function scope, and the name refers to a local +variable that has not yet been bound to a value at the point where the +name is used, an "UnboundLocalError" exception is raised. +"UnboundLocalError" is a subclass of "NameError". + +If a name binding operation occurs anywhere within a code block, all +uses of the name within the block are treated as references to the +current block. This can lead to errors when a name is used within a +block before it is bound. This rule is subtle. Python lacks +declarations and allows name binding operations to occur anywhere +within a code block. The local variables of a code block can be +determined by scanning the entire text of the block for name binding +operations. See the FAQ entry on UnboundLocalError for examples. + +If the "global" statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the +top-level namespace. Names are resolved in the top-level namespace by +searching the global namespace, i.e. the namespace of the module +containing the code block, and the builtins namespace, the namespace +of the module "builtins". The global namespace is searched first. If +the names are not found there, the builtins namespace is searched +next. If the names are also not found in the builtins namespace, new +variables are created in the global namespace. The global statement +must precede all uses of the listed names. + +The "global" statement has the same scope as a name binding operation +in the same block. If the nearest enclosing scope for a free variable +contains a global statement, the free variable is treated as a global. + +The "nonlocal" statement causes corresponding names to refer to +previously bound variables in the nearest enclosing function scope. +"SyntaxError" is raised at compile time if the given name does not +exist in any enclosing function scope. Type parameters cannot be +rebound with the "nonlocal" statement. + +The namespace for a module is automatically created the first time a +module is imported. The main module for a script is always called +"__main__". + +Class definition blocks and arguments to "exec()" and "eval()" are +special in the context of name resolution. A class definition is an +executable statement that may use and define names. These references +follow the normal rules for name resolution with an exception that +unbound local variables are looked up in the global namespace. The +namespace of the class definition becomes the attribute dictionary of +the class. The scope of names defined in a class block is limited to +the class block; it does not extend to the code blocks of methods. +This includes comprehensions and generator expressions, but it does +not include annotation scopes, which have access to their enclosing +class scopes. This means that the following will fail: + + class A: + a = 42 + b = list(a + i for i in range(10)) + +However, the following will succeed: + + class A: + type Alias = Nested + class Nested: pass + + print(A.Alias.__value__) # + + +Annotation scopes +================= + +*Annotations*, type parameter lists and "type" statements introduce +*annotation scopes*, which behave mostly like function scopes, but +with some exceptions discussed below. + +Annotation scopes are used in the following contexts: + +* *Function annotations*. + +* *Variable annotations*. + +* Type parameter lists for generic type aliases. + +* Type parameter lists for generic functions. A generic function’s + annotations are executed within the annotation scope, but its + defaults and decorators are not. + +* Type parameter lists for generic classes. A generic class’s base + classes and keyword arguments are executed within the annotation + scope, but its decorators are not. + +* The bounds, constraints, and default values for type parameters + (lazily evaluated). + +* The value of type aliases (lazily evaluated). + +Annotation scopes differ from function scopes in the following ways: + +* Annotation scopes have access to their enclosing class namespace. If + an annotation scope is immediately within a class scope, or within + another annotation scope that is immediately within a class scope, + the code in the annotation scope can use names defined in the class + scope as if it were executed directly within the class body. This + contrasts with regular functions defined within classes, which + cannot access names defined in the class scope. + +* Expressions in annotation scopes cannot contain "yield", "yield + from", "await", or ":=" expressions. (These expressions are allowed + in other scopes contained within the annotation scope.) + +* Names defined in annotation scopes cannot be rebound with "nonlocal" + statements in inner scopes. This includes only type parameters, as + no other syntactic elements that can appear within annotation scopes + can introduce new names. + +* While annotation scopes have an internal name, that name is not + reflected in the *qualified name* of objects defined within the + scope. Instead, the "__qualname__" of such objects is as if the + object were defined in the enclosing scope. + +Added in version 3.12: Annotation scopes were introduced in Python +3.12 as part of **PEP 695**. + +Changed in version 3.13: Annotation scopes are also used for type +parameter defaults, as introduced by **PEP 696**. + +Changed in version 3.14: Annotation scopes are now also used for +annotations, as specified in **PEP 649** and **PEP 749**. + + +Lazy evaluation +=============== + +Most annotation scopes are *lazily evaluated*. This includes +annotations, the values of type aliases created through the "type" +statement, and the bounds, constraints, and default values of type +variables created through the type parameter syntax. This means that +they are not evaluated when the type alias or type variable is +created, or when the object carrying annotations is created. Instead, +they are only evaluated when necessary, for example when the +"__value__" attribute on a type alias is accessed. + +Example: + + >>> type Alias = 1/0 + >>> Alias.__value__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def func[T: 1/0](): pass + >>> T = func.__type_params__[0] + >>> T.__bound__ + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + +Here the exception is raised only when the "__value__" attribute of +the type alias or the "__bound__" attribute of the type variable is +accessed. + +This behavior is primarily useful for references to types that have +not yet been defined when the type alias or type variable is created. +For example, lazy evaluation enables creation of mutually recursive +type aliases: + + from typing import Literal + + type SimpleExpr = int | Parenthesized + type Parenthesized = tuple[Literal["("], Expr, Literal[")"]] + type Expr = SimpleExpr | tuple[SimpleExpr, Literal["+", "-"], Expr] + +Lazily evaluated values are evaluated in annotation scope, which means +that names that appear inside the lazily evaluated value are looked up +as if they were used in the immediately enclosing scope. + +Added in version 3.12. + + +Builtins and restricted execution +================================= + +**CPython implementation detail:** Users should not touch +"__builtins__"; it is strictly an implementation detail. Users +wanting to override values in the builtins namespace should "import" +the "builtins" module and modify its attributes appropriately. + +The builtins namespace associated with the execution of a code block +is actually found by looking up the name "__builtins__" in its global +namespace; this should be a dictionary or a module (in the latter case +the module’s dictionary is used). By default, when in the "__main__" +module, "__builtins__" is the built-in module "builtins"; when in any +other module, "__builtins__" is an alias for the dictionary of the +"builtins" module itself. + + +Interaction with dynamic features +================================= + +Name resolution of free variables occurs at runtime, not at compile +time. This means that the following code will print 42: + + i = 10 + def f(): + print(i) + i = 42 + f() + +The "eval()" and "exec()" functions do not have access to the full +environment for resolving names. Names may be resolved in the local +and global namespaces of the caller. Free variables are not resolved +in the nearest enclosing namespace, but in the global namespace. [1] +The "exec()" and "eval()" functions have optional arguments to +override the global and local namespace. If only one namespace is +specified, it is used for both. +''', + 'nonlocal': r'''The "nonlocal" statement +************************ + + nonlocal_stmt: "nonlocal" identifier ("," identifier)* + +When the definition of a function or class is nested (enclosed) within +the definitions of other functions, its nonlocal scopes are the local +scopes of the enclosing functions. The "nonlocal" statement causes the +listed identifiers to refer to names previously bound in nonlocal +scopes. It allows encapsulated code to rebind such nonlocal +identifiers. If a name is bound in more than one nonlocal scope, the +nearest binding is used. If a name is not bound in any nonlocal scope, +or if there is no nonlocal scope, a "SyntaxError" is raised. + +The "nonlocal" statement applies to the entire scope of a function or +class body. A "SyntaxError" is raised if a variable is used or +assigned to prior to its nonlocal declaration in the scope. + +See also: + + **PEP 3104** - Access to Names in Outer Scopes + The specification for the "nonlocal" statement. + +**Programmer’s note:** "nonlocal" is a directive to the parser and +applies only to code parsed along with it. See the note for the +"global" statement. +''', + 'numbers': r'''Numeric literals +**************** + +"NUMBER" tokens represent numeric literals, of which there are three +types: integers, floating-point numbers, and imaginary numbers. + + NUMBER: integer | floatnumber | imagnumber + +The numeric value of a numeric literal is the same as if it were +passed as a string to the "int", "float" or "complex" class +constructor, respectively. Note that not all valid inputs for those +constructors are also valid literals. + +Numeric literals do not include a sign; a phrase like "-1" is actually +an expression composed of the unary operator ‘"-"’ and the literal +"1". + + +Integer literals +================ + +Integer literals denote whole numbers. For example: + + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* + bininteger: "0" ("b" | "B") (["_"] bindigit)+ + octinteger: "0" ("o" | "O") (["_"] octdigit)+ + hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* + nonzerodigit: "1"..."9" + digit: "0"..."9" + bindigit: "0" | "1" + octdigit: "0"..."7" + hexdigit: digit | "a"..."f" | "A"..."F" + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Floating-point literals +======================= + +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Imaginary literals +================== + +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. + +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") +''', + 'numeric-types': r'''Emulating numeric types +*********************** + +The following methods can be defined to emulate numeric objects. +Methods corresponding to operations that are not supported by the +particular kind of number implemented (e.g., bitwise operations for +non-integral numbers) should be left undefined. + +object.__add__(self, other) +object.__sub__(self, other) +object.__mul__(self, other) +object.__matmul__(self, other) +object.__truediv__(self, other) +object.__floordiv__(self, other) +object.__mod__(self, other) +object.__divmod__(self, other) +object.__pow__(self, other[, modulo]) +object.__lshift__(self, other) +object.__rshift__(self, other) +object.__and__(self, other) +object.__xor__(self, other) +object.__or__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to + evaluate the expression "x + y", where *x* is an instance of a + class that has an "__add__()" method, "type(x).__add__(x, y)" is + called. The "__divmod__()" method should be the equivalent to + using "__floordiv__()" and "__mod__()"; it should not be related to + "__truediv__()". Note that "__pow__()" should be defined to accept + an optional third argument if the three-argument version of the + built-in "pow()" function is to be supported. + + If one of those methods does not support the operation with the + supplied arguments, it should return "NotImplemented". + +object.__radd__(self, other) +object.__rsub__(self, other) +object.__rmul__(self, other) +object.__rmatmul__(self, other) +object.__rtruediv__(self, other) +object.__rfloordiv__(self, other) +object.__rmod__(self, other) +object.__rdivmod__(self, other) +object.__rpow__(self, other[, modulo]) +object.__rlshift__(self, other) +object.__rrshift__(self, other) +object.__rand__(self, other) +object.__rxor__(self, other) +object.__ror__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped) + operands. These functions are only called if the operands are of + different types, when the left operand does not support the + corresponding operation [3], or the right operand’s class is + derived from the left operand’s class. [4] For instance, to + evaluate the expression "x - y", where *y* is an instance of a + class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is + called if "type(x).__sub__(x, y)" returns "NotImplemented" or + "type(y)" is a subclass of "type(x)". [5] + + Note that "__rpow__()" should be defined to accept an optional + third argument if the three-argument version of the built-in + "pow()" function is to be supported. + + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. + + Note: + + If the right operand’s type is a subclass of the left operand’s + type and that subclass provides a different implementation of the + reflected method for the operation, this method will be called + before the left operand’s non-reflected method. This behavior + allows subclasses to override their ancestors’ operations. + +object.__iadd__(self, other) +object.__isub__(self, other) +object.__imul__(self, other) +object.__imatmul__(self, other) +object.__itruediv__(self, other) +object.__ifloordiv__(self, other) +object.__imod__(self, other) +object.__ipow__(self, other[, modulo]) +object.__ilshift__(self, other) +object.__irshift__(self, other) +object.__iand__(self, other) +object.__ixor__(self, other) +object.__ior__(self, other) + + These methods are called to implement the augmented arithmetic + assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=", + "<<=", ">>=", "&=", "^=", "|="). These methods should attempt to + do the operation in-place (modifying *self*) and return the result + (which could be, but does not have to be, *self*). If a specific + method is not defined, or if that method returns "NotImplemented", + the augmented assignment falls back to the normal methods. For + instance, if *x* is an instance of a class with an "__iadd__()" + method, "x += y" is equivalent to "x = x.__iadd__(y)" . If + "__iadd__()" does not exist, or if "x.__iadd__(y)" returns + "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are + considered, as with the evaluation of "x + y". In certain + situations, augmented assignment can result in unexpected errors + (see Why does a_tuple[i] += [‘item’] raise an exception when the + addition works?), but this behavior is in fact part of the data + model. + +object.__neg__(self) +object.__pos__(self) +object.__abs__(self) +object.__invert__(self) + + Called to implement the unary arithmetic operations ("-", "+", + "abs()" and "~"). + +object.__complex__(self) +object.__int__(self) +object.__float__(self) + + Called to implement the built-in functions "complex()", "int()" and + "float()". Should return a value of the appropriate type. + +object.__index__(self) + + Called to implement "operator.index()", and whenever Python needs + to losslessly convert the numeric object to an integer object (such + as in slicing, or in the built-in "bin()", "hex()" and "oct()" + functions). Presence of this method indicates that the numeric + object is an integer type. Must return an integer. + + If "__int__()", "__float__()" and "__complex__()" are not defined + then corresponding built-in functions "int()", "float()" and + "complex()" fall back to "__index__()". + +object.__round__(self[, ndigits]) +object.__trunc__(self) +object.__floor__(self) +object.__ceil__(self) + + Called to implement the built-in function "round()" and "math" + functions "trunc()", "floor()" and "ceil()". Unless *ndigits* is + passed to "__round__()" all these methods should return the value + of the object truncated to an "Integral" (typically an "int"). + + Changed in version 3.14: "int()" no longer delegates to the + "__trunc__()" method. +''', + 'objects': r'''Objects, values and types +************************* + +*Objects* are Python’s abstraction for data. All data in a Python +program is represented by objects or by relations between objects. +Even code is represented by objects. + +Every object has an identity, a type and a value. An object’s +*identity* never changes once it has been created; you may think of it +as the object’s address in memory. The "is" operator compares the +identity of two objects; the "id()" function returns an integer +representing its identity. + +**CPython implementation detail:** For CPython, "id(x)" is the memory +address where "x" is stored. + +An object’s type determines the operations that the object supports +(e.g., “does it have a length?”) and also defines the possible values +for objects of that type. The "type()" function returns an object’s +type (which is an object itself). Like its identity, an object’s +*type* is also unchangeable. [1] + +The *value* of some objects can change. Objects whose value can +change are said to be *mutable*; objects whose value is unchangeable +once they are created are called *immutable*. (The value of an +immutable container object that contains a reference to a mutable +object can change when the latter’s value is changed; however the +container is still considered immutable, because the collection of +objects it contains cannot be changed. So, immutability is not +strictly the same as having an unchangeable value, it is more subtle.) +An object’s mutability is determined by its type; for instance, +numbers, strings and tuples are immutable, while dictionaries and +lists are mutable. + +Objects are never explicitly destroyed; however, when they become +unreachable they may be garbage-collected. An implementation is +allowed to postpone garbage collection or omit it altogether — it is a +matter of implementation quality how garbage collection is +implemented, as long as no objects are collected that are still +reachable. + +**CPython implementation detail:** CPython currently uses a reference- +counting scheme with (optional) delayed detection of cyclically linked +garbage, which collects most objects as soon as they become +unreachable, but is not guaranteed to collect garbage containing +circular references. See the documentation of the "gc" module for +information on controlling the collection of cyclic garbage. Other +implementations act differently and CPython may change. Do not depend +on immediate finalization of objects when they become unreachable (so +you should always close files explicitly). + +Note that the use of the implementation’s tracing or debugging +facilities may keep objects alive that would normally be collectable. +Also note that catching an exception with a "try"…"except" statement +may keep objects alive. + +Some objects contain references to “external” resources such as open +files or windows. It is understood that these resources are freed +when the object is garbage-collected, but since garbage collection is +not guaranteed to happen, such objects also provide an explicit way to +release the external resource, usually a "close()" method. Programs +are strongly recommended to explicitly close such objects. The +"try"…"finally" statement and the "with" statement provide convenient +ways to do this. + +Some objects contain references to other objects; these are called +*containers*. Examples of containers are tuples, lists and +dictionaries. The references are part of a container’s value. In +most cases, when we talk about the value of a container, we imply the +values, not the identities of the contained objects; however, when we +talk about the mutability of a container, only the identities of the +immediately contained objects are implied. So, if an immutable +container (like a tuple) contains a reference to a mutable object, its +value changes if that mutable object is changed. + +Types affect almost all aspects of object behavior. Even the +importance of object identity is affected in some sense: for immutable +types, operations that compute new values may actually return a +reference to any existing object with the same type and value, while +for mutable objects this is not allowed. For example, after "a = 1; b += 1", *a* and *b* may or may not refer to the same object with the +value one, depending on the implementation. This is because "int" is +an immutable type, so the reference to "1" can be reused. This +behaviour depends on the implementation used, so should not be relied +upon, but is something to be aware of when making use of object +identity tests. However, after "c = []; d = []", *c* and *d* are +guaranteed to refer to two different, unique, newly created empty +lists. (Note that "e = f = []" assigns the *same* object to both *e* +and *f*.) +''', + 'operator-summary': r'''Operator precedence +******************* + +The following table summarizes the operator precedence in Python, from +highest precedence (most binding) to lowest precedence (least +binding). Operators in the same box have the same precedence. Unless +the syntax is explicitly given, operators are binary. Operators in +the same box group left to right (except for exponentiation and +conditional expressions, which group from right to left). + +Note that comparisons, membership tests, and identity tests, all have +the same precedence and have a left-to-right chaining feature as +described in the Comparisons section. + ++-------------------------------------------------+---------------------------------------+ +| Operator | Description | +|=================================================|=======================================| +| "(expressions...)", "[expressions...]", "{key: | Binding or parenthesized expression, | +| value...}", "{expressions...}" | list display, dictionary display, set | +| | display | ++-------------------------------------------------+---------------------------------------+ +| "x[index]", "x[index:index]", | Subscription, slicing, call, | +| "x(arguments...)", "x.attribute" | attribute reference | ++-------------------------------------------------+---------------------------------------+ +| "await x" | Await expression | ++-------------------------------------------------+---------------------------------------+ +| "**" | Exponentiation [5] | ++-------------------------------------------------+---------------------------------------+ +| "+x", "-x", "~x" | Positive, negative, bitwise NOT | ++-------------------------------------------------+---------------------------------------+ +| "*", "@", "/", "//", "%" | Multiplication, matrix | +| | multiplication, division, floor | +| | division, remainder [6] | ++-------------------------------------------------+---------------------------------------+ +| "+", "-" | Addition and subtraction | ++-------------------------------------------------+---------------------------------------+ +| "<<", ">>" | Shifts | ++-------------------------------------------------+---------------------------------------+ +| "&" | Bitwise AND | ++-------------------------------------------------+---------------------------------------+ +| "^" | Bitwise XOR | ++-------------------------------------------------+---------------------------------------+ +| "|" | Bitwise OR | ++-------------------------------------------------+---------------------------------------+ +| "in", "not in", "is", "is not", "<", "<=", ">", | Comparisons, including membership | +| ">=", "!=", "==" | tests and identity tests | ++-------------------------------------------------+---------------------------------------+ +| "not x" | Boolean NOT | ++-------------------------------------------------+---------------------------------------+ +| "and" | Boolean AND | ++-------------------------------------------------+---------------------------------------+ +| "or" | Boolean OR | ++-------------------------------------------------+---------------------------------------+ +| "if" – "else" | Conditional expression | ++-------------------------------------------------+---------------------------------------+ +| "lambda" | Lambda expression | ++-------------------------------------------------+---------------------------------------+ +| ":=" | Assignment expression | ++-------------------------------------------------+---------------------------------------+ + +-[ Footnotes ]- + +[1] While "abs(x%y) < abs(y)" is true mathematically, for floats it + may not be true numerically due to roundoff. For example, and + assuming a platform on which a Python float is an IEEE 754 double- + precision number, in order that "-1e-100 % 1e100" have the same + sign as "1e100", the computed result is "-1e-100 + 1e100", which + is numerically exactly equal to "1e100". The function + "math.fmod()" returns a result whose sign matches the sign of the + first argument instead, and so returns "-1e-100" in this case. + Which approach is more appropriate depends on the application. + +[2] If x is very close to an exact integer multiple of y, it’s + possible for "x//y" to be one larger than "(x-x%y)//y" due to + rounding. In such cases, Python returns the latter result, in + order to preserve that "divmod(x,y)[0] * y + x % y" be very close + to "x". + +[3] The Unicode standard distinguishes between *code points* (e.g. + U+0041) and *abstract characters* (e.g. “LATIN CAPITAL LETTER A”). + While most abstract characters in Unicode are only represented + using one code point, there is a number of abstract characters + that can in addition be represented using a sequence of more than + one code point. For example, the abstract character “LATIN + CAPITAL LETTER C WITH CEDILLA” can be represented as a single + *precomposed character* at code position U+00C7, or as a sequence + of a *base character* at code position U+0043 (LATIN CAPITAL + LETTER C), followed by a *combining character* at code position + U+0327 (COMBINING CEDILLA). + + The comparison operators on strings compare at the level of + Unicode code points. This may be counter-intuitive to humans. For + example, ""\u00C7" == "\u0043\u0327"" is "False", even though both + strings represent the same abstract character “LATIN CAPITAL + LETTER C WITH CEDILLA”. + + To compare strings at the level of abstract characters (that is, + in a way intuitive to humans), use "unicodedata.normalize()". + +[4] Due to automatic garbage-collection, free lists, and the dynamic + nature of descriptors, you may notice seemingly unusual behaviour + in certain uses of the "is" operator, like those involving + comparisons between instance methods, or constants. Check their + documentation for more info. + +[5] The power operator "**" binds less tightly than an arithmetic or + bitwise unary operator on its right, that is, "2**-1" is "0.5". + +[6] The "%" operator is also used for string formatting; the same + precedence applies. +''', + 'pass': r'''The "pass" statement +******************** + + pass_stmt: "pass" + +"pass" is a null operation — when it is executed, nothing happens. It +is useful as a placeholder when a statement is required syntactically, +but no code needs to be executed, for example: + + def f(arg): pass # a function that does nothing (yet) + + class C: pass # a class with no methods (yet) +''', + 'power': r'''The power operator +****************** + +The power operator binds more tightly than unary operators on its +left; it binds less tightly than unary operators on its right. The +syntax is: + + power: (await_expr | primary) ["**" u_expr] + +Thus, in an unparenthesized sequence of power and unary operators, the +operators are evaluated from right to left (this does not constrain +the evaluation order for the operands): "-1**2" results in "-1". + +The power operator has the same semantics as the built-in "pow()" +function, when called with two arguments: it yields its left argument +raised to the power of its right argument. The numeric arguments are +first converted to a common type, and the result is of that type. + +For int operands, the result has the same type as the operands unless +the second argument is negative; in that case, all arguments are +converted to float and a float result is delivered. For example, +"10**2" returns "100", but "10**-2" returns "0.01". + +Raising "0.0" to a negative power results in a "ZeroDivisionError". +Raising a negative number to a fractional power results in a "complex" +number. (In earlier versions it raised a "ValueError".) + +This operation can be customized using the special "__pow__()" and +"__rpow__()" methods. +''', + 'raise': r'''The "raise" statement +********************* + + raise_stmt: "raise" [expression ["from" expression]] + +If no expressions are present, "raise" re-raises the exception that is +currently being handled, which is also known as the *active +exception*. If there isn’t currently an active exception, a +"RuntimeError" exception is raised indicating that this is an error. + +Otherwise, "raise" evaluates the first expression as the exception +object. It must be either a subclass or an instance of +"BaseException". If it is a class, the exception instance will be +obtained when needed by instantiating the class with no arguments. + +The *type* of the exception is the exception instance’s class, the +*value* is the instance itself. + +A traceback object is normally created automatically when an exception +is raised and attached to it as the "__traceback__" attribute. You can +create an exception and set your own traceback in one step using the +"with_traceback()" exception method (which returns the same exception +instance, with its traceback set to its argument), like so: + + raise Exception("foo occurred").with_traceback(tracebackobj) + +The "from" clause is used for exception chaining: if given, the second +*expression* must be another exception class or instance. If the +second expression is an exception instance, it will be attached to the +raised exception as the "__cause__" attribute (which is writable). If +the expression is an exception class, the class will be instantiated +and the resulting exception instance will be attached to the raised +exception as the "__cause__" attribute. If the raised exception is not +handled, both exceptions will be printed: + + >>> try: + ... print(1 / 0) + ... except Exception as exc: + ... raise RuntimeError("Something bad happened") from exc + ... + Traceback (most recent call last): + File "", line 2, in + print(1 / 0) + ~~^~~ + ZeroDivisionError: division by zero + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "", line 4, in + raise RuntimeError("Something bad happened") from exc + RuntimeError: Something bad happened + +A similar mechanism works implicitly if a new exception is raised when +an exception is already being handled. An exception may be handled +when an "except" or "finally" clause, or a "with" statement, is used. +The previous exception is then attached as the new exception’s +"__context__" attribute: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") + ... + Traceback (most recent call last): + File "", line 2, in + print(1 / 0) + ~~^~~ + ZeroDivisionError: division by zero + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "", line 4, in + raise RuntimeError("Something bad happened") + RuntimeError: Something bad happened + +Exception chaining can be explicitly suppressed by specifying "None" +in the "from" clause: + + >>> try: + ... print(1 / 0) + ... except: + ... raise RuntimeError("Something bad happened") from None + ... + Traceback (most recent call last): + File "", line 4, in + RuntimeError: Something bad happened + +Additional information on exceptions can be found in section +Exceptions, and information about handling exceptions is in section +The try statement. + +Changed in version 3.3: "None" is now permitted as "Y" in "raise X +from Y".Added the "__suppress_context__" attribute to suppress +automatic display of the exception context. + +Changed in version 3.11: If the traceback of the active exception is +modified in an "except" clause, a subsequent "raise" statement re- +raises the exception with the modified traceback. Previously, the +exception was re-raised with the traceback it had when it was caught. +''', + 'return': r'''The "return" statement +********************** + + return_stmt: "return" [expression_list] + +"return" may only occur syntactically nested in a function definition, +not within a nested class definition. + +If an expression list is present, it is evaluated, else "None" is +substituted. + +"return" leaves the current function call with the expression list (or +"None") as return value. + +When "return" passes control out of a "try" statement with a "finally" +clause, that "finally" clause is executed before really leaving the +function. + +In a generator function, the "return" statement indicates that the +generator is done and will cause "StopIteration" to be raised. The +returned value (if any) is used as an argument to construct +"StopIteration" and becomes the "StopIteration.value" attribute. + +In an asynchronous generator function, an empty "return" statement +indicates that the asynchronous generator is done and will cause +"StopAsyncIteration" to be raised. A non-empty "return" statement is +a syntax error in an asynchronous generator function. +''', + 'sequence-types': r'''Emulating container types +************************* + +The following methods can be defined to implement container objects. +None of them are provided by the "object" class itself. Containers +usually are *sequences* (such as "lists" or "tuples") or *mappings* +(like *dictionaries*), but can represent other containers as well. +The first set of methods is used either to emulate a sequence or to +emulate a mapping; the difference is that for a sequence, the +allowable keys should be the integers *k* for which "0 <= k < N" where +*N* is the length of the sequence, or "slice" objects, which define a +range of items. It is also recommended that mappings provide the +methods "keys()", "values()", "items()", "get()", "clear()", +"setdefault()", "pop()", "popitem()", "copy()", and "update()" +behaving similar to those for Python’s standard "dictionary" objects. +The "collections.abc" module provides a "MutableMapping" *abstract +base class* to help create those methods from a base set of +"__getitem__()", "__setitem__()", "__delitem__()", and "keys()". + +Mutable sequences should provide methods "append()", "clear()", +"count()", "extend()", "index()", "insert()", "pop()", "remove()", and +"reverse()", like Python standard "list" objects. Finally, sequence +types should implement addition (meaning concatenation) and +multiplication (meaning repetition) by defining the methods +"__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and +"__imul__()" described below; they should not define other numerical +operators. + +It is recommended that both mappings and sequences implement the +"__contains__()" method to allow efficient use of the "in" operator; +for mappings, "in" should search the mapping’s keys; for sequences, it +should search through the values. It is further recommended that both +mappings and sequences implement the "__iter__()" method to allow +efficient iteration through the container; for mappings, "__iter__()" +should iterate through the object’s keys; for sequences, it should +iterate through the values. + +object.__len__(self) + + Called to implement the built-in function "len()". Should return + the length of the object, an integer ">=" 0. Also, an object that + doesn’t define a "__bool__()" method and whose "__len__()" method + returns zero is considered to be false in a Boolean context. + + **CPython implementation detail:** In CPython, the length is + required to be at most "sys.maxsize". If the length is larger than + "sys.maxsize" some features (such as "len()") may raise + "OverflowError". To prevent raising "OverflowError" by truth value + testing, an object must define a "__bool__()" method. + +object.__length_hint__(self) + + Called to implement "operator.length_hint()". Should return an + estimated length for the object (which may be greater or less than + the actual length). The length must be an integer ">=" 0. The + return value may also be "NotImplemented", which is treated the + same as if the "__length_hint__" method didn’t exist at all. This + method is purely an optimization and is never required for + correctness. + + Added in version 3.4. + +Note: + + Slicing is done exclusively with the following three methods. A + call like + + a[1:2] = b + + is translated to + + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with "None". + +object.__getitem__(self, key) + + Called to implement evaluation of "self[key]". For *sequence* + types, the accepted keys should be integers. Optionally, they may + support "slice" objects as well. Negative index support is also + optional. If *key* is of an inappropriate type, "TypeError" may be + raised; if *key* is a value outside the set of indexes for the + sequence (after any special interpretation of negative values), + "IndexError" should be raised. For *mapping* types, if *key* is + missing (not in the container), "KeyError" should be raised. + + Note: + + "for" loops expect that an "IndexError" will be raised for + illegal indexes to allow proper detection of the end of the + sequence. + + Note: + + When subscripting a *class*, the special class method + "__class_getitem__()" may be called instead of "__getitem__()". + See __class_getitem__ versus __getitem__ for more details. + +object.__setitem__(self, key, value) + + Called to implement assignment to "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support changes to the values for keys, or if new keys + can be added, or for sequences if elements can be replaced. The + same exceptions should be raised for improper *key* values as for + the "__getitem__()" method. + +object.__delitem__(self, key) + + Called to implement deletion of "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support removal of keys, or for sequences if elements + can be removed from the sequence. The same exceptions should be + raised for improper *key* values as for the "__getitem__()" method. + +object.__missing__(self, key) + + Called by "dict"."__getitem__()" to implement "self[key]" for dict + subclasses when key is not in the dictionary. + +object.__iter__(self) + + This method is called when an *iterator* is required for a + container. This method should return a new iterator object that can + iterate over all the objects in the container. For mappings, it + should iterate over the keys of the container. + +object.__reversed__(self) + + Called (if present) by the "reversed()" built-in to implement + reverse iteration. It should return a new iterator object that + iterates over all the objects in the container in reverse order. + + If the "__reversed__()" method is not provided, the "reversed()" + built-in will fall back to using the sequence protocol ("__len__()" + and "__getitem__()"). Objects that support the sequence protocol + should only provide "__reversed__()" if they can provide an + implementation that is more efficient than the one provided by + "reversed()". + +The membership test operators ("in" and "not in") are normally +implemented as an iteration through a container. However, container +objects can supply the following special method with a more efficient +implementation, which also does not require the object be iterable. + +object.__contains__(self, item) + + Called to implement membership test operators. Should return true + if *item* is in *self*, false otherwise. For mapping objects, this + should consider the keys of the mapping rather than the values or + the key-item pairs. + + For objects that don’t define "__contains__()", the membership test + first tries iteration via "__iter__()", then the old sequence + iteration protocol via "__getitem__()", see this section in the + language reference. +''', + 'shifting': r'''Shifting operations +******************* + +The shifting operations have lower priority than the arithmetic +operations: + + shift_expr: a_expr | shift_expr ("<<" | ">>") a_expr + +These operators accept integers as arguments. They shift the first +argument to the left or right by the number of bits given by the +second argument. + +The left shift operation can be customized using the special +"__lshift__()" and "__rlshift__()" methods. The right shift operation +can be customized using the special "__rshift__()" and "__rrshift__()" +methods. + +A right shift by *n* bits is defined as floor division by "pow(2,n)". +A left shift by *n* bits is defined as multiplication with "pow(2,n)". +''', + 'slicings': r'''Slicings +******** + +A slicing selects a range of items in a sequence object (e.g., a +string, tuple or list). Slicings may be used as expressions or as +targets in assignment or "del" statements. The syntax for a slicing: + + slicing: primary "[" slice_list "]" + slice_list: slice_item ("," slice_item)* [","] + slice_item: expression | proper_slice + proper_slice: [lower_bound] ":" [upper_bound] [ ":" [stride] ] + lower_bound: expression + upper_bound: expression + stride: expression + +There is ambiguity in the formal syntax here: anything that looks like +an expression list also looks like a slice list, so any subscription +can be interpreted as a slicing. Rather than further complicating the +syntax, this is disambiguated by defining that in this case the +interpretation as a subscription takes priority over the +interpretation as a slicing (this is the case if the slice list +contains no proper slice). + +The semantics for a slicing are as follows. The primary is indexed +(using the same "__getitem__()" method as normal subscription) with a +key that is constructed from the slice list, as follows. If the slice +list contains at least one comma, the key is a tuple containing the +conversion of the slice items; otherwise, the conversion of the lone +slice item is the key. The conversion of a slice item that is an +expression is that expression. The conversion of a proper slice is a +slice object (see section The standard type hierarchy) whose "start", +"stop" and "step" attributes are the values of the expressions given +as lower bound, upper bound and stride, respectively, substituting +"None" for missing expressions. +''', + 'specialattrs': r'''Special Attributes +****************** + +The implementation adds a few special read-only attributes to several +object types, where they are relevant. Some of these are not reported +by the "dir()" built-in function. + +definition.__name__ + + The name of the class, function, method, descriptor, or generator + instance. + +definition.__qualname__ + + The *qualified name* of the class, function, method, descriptor, or + generator instance. + + Added in version 3.3. + +definition.__module__ + + The name of the module in which a class or function was defined. + +definition.__doc__ + + The documentation string of a class or function, or "None" if + undefined. + +definition.__type_params__ + + The type parameters of generic classes, functions, and type + aliases. For classes and functions that are not generic, this will + be an empty tuple. + + Added in version 3.12. +''', + 'specialnames': r'''Special method names +******************** + +A class can implement certain operations that are invoked by special +syntax (such as arithmetic operations or subscripting and slicing) by +defining methods with special names. This is Python’s approach to +*operator overloading*, allowing classes to define their own behavior +with respect to language operators. For instance, if a class defines +a method named "__getitem__()", and "x" is an instance of this class, +then "x[i]" is roughly equivalent to "type(x).__getitem__(x, i)". +Except where mentioned, attempts to execute an operation raise an +exception when no appropriate method is defined (typically +"AttributeError" or "TypeError"). + +Setting a special method to "None" indicates that the corresponding +operation is not available. For example, if a class sets "__iter__()" +to "None", the class is not iterable, so calling "iter()" on its +instances will raise a "TypeError" (without falling back to +"__getitem__()"). [2] + +When implementing a class that emulates any built-in type, it is +important that the emulation only be implemented to the degree that it +makes sense for the object being modelled. For example, some +sequences may work well with retrieval of individual elements, but +extracting a slice may not make sense. (One example of this is the +NodeList interface in the W3C’s Document Object Model.) + + +Basic customization +=================== + +object.__new__(cls[, ...]) + + Called to create a new instance of class *cls*. "__new__()" is a + static method (special-cased so you need not declare it as such) + that takes the class of which an instance was requested as its + first argument. The remaining arguments are those passed to the + object constructor expression (the call to the class). The return + value of "__new__()" should be the new object instance (usually an + instance of *cls*). + + Typical implementations create a new instance of the class by + invoking the superclass’s "__new__()" method using + "super().__new__(cls[, ...])" with appropriate arguments and then + modifying the newly created instance as necessary before returning + it. + + If "__new__()" is invoked during object construction and it returns + an instance of *cls*, then the new instance’s "__init__()" method + will be invoked like "__init__(self[, ...])", where *self* is the + new instance and the remaining arguments are the same as were + passed to the object constructor. + + If "__new__()" does not return an instance of *cls*, then the new + instance’s "__init__()" method will not be invoked. + + "__new__()" is intended mainly to allow subclasses of immutable + types (like int, str, or tuple) to customize instance creation. It + is also commonly overridden in custom metaclasses in order to + customize class creation. + +object.__init__(self[, ...]) + + Called after the instance has been created (by "__new__()"), but + before it is returned to the caller. The arguments are those + passed to the class constructor expression. If a base class has an + "__init__()" method, the derived class’s "__init__()" method, if + any, must explicitly call it to ensure proper initialization of the + base class part of the instance; for example: + "super().__init__([args...])". + + Because "__new__()" and "__init__()" work together in constructing + objects ("__new__()" to create it, and "__init__()" to customize + it), no non-"None" value may be returned by "__init__()"; doing so + will cause a "TypeError" to be raised at runtime. + +object.__del__(self) + + Called when the instance is about to be destroyed. This is also + called a finalizer or (improperly) a destructor. If a base class + has a "__del__()" method, the derived class’s "__del__()" method, + if any, must explicitly call it to ensure proper deletion of the + base class part of the instance. + + It is possible (though not recommended!) for the "__del__()" method + to postpone destruction of the instance by creating a new reference + to it. This is called object *resurrection*. It is + implementation-dependent whether "__del__()" is called a second + time when a resurrected object is about to be destroyed; the + current *CPython* implementation only calls it once. + + It is not guaranteed that "__del__()" methods are called for + objects that still exist when the interpreter exits. + "weakref.finalize" provides a straightforward way to register a + cleanup function to be called when an object is garbage collected. + + Note: + + "del x" doesn’t directly call "x.__del__()" — the former + decrements the reference count for "x" by one, and the latter is + only called when "x"’s reference count reaches zero. + + **CPython implementation detail:** It is possible for a reference + cycle to prevent the reference count of an object from going to + zero. In this case, the cycle will be later detected and deleted + by the *cyclic garbage collector*. A common cause of reference + cycles is when an exception has been caught in a local variable. + The frame’s locals then reference the exception, which references + its own traceback, which references the locals of all frames caught + in the traceback. + + See also: Documentation for the "gc" module. + + Warning: + + Due to the precarious circumstances under which "__del__()" + methods are invoked, exceptions that occur during their execution + are ignored, and a warning is printed to "sys.stderr" instead. + In particular: + + * "__del__()" can be invoked when arbitrary code is being + executed, including from any arbitrary thread. If "__del__()" + needs to take a lock or invoke any other blocking resource, it + may deadlock as the resource may already be taken by the code + that gets interrupted to execute "__del__()". + + * "__del__()" can be executed during interpreter shutdown. As a + consequence, the global variables it needs to access (including + other modules) may already have been deleted or set to "None". + Python guarantees that globals whose name begins with a single + underscore are deleted from their module before other globals + are deleted; if no other references to such globals exist, this + may help in assuring that imported modules are still available + at the time when the "__del__()" method is called. + +object.__repr__(self) + + Called by the "repr()" built-in function to compute the “official” + string representation of an object. If at all possible, this + should look like a valid Python expression that could be used to + recreate an object with the same value (given an appropriate + environment). If this is not possible, a string of the form + "<...some useful description...>" should be returned. The return + value must be a string object. If a class defines "__repr__()" but + not "__str__()", then "__repr__()" is also used when an “informal” + string representation of instances of that class is required. + + This is typically used for debugging, so it is important that the + representation is information-rich and unambiguous. A default + implementation is provided by the "object" class itself. + +object.__str__(self) + + Called by "str(object)", the default "__format__()" implementation, + and the built-in function "print()", to compute the “informal” or + nicely printable string representation of an object. The return + value must be a str object. + + This method differs from "object.__repr__()" in that there is no + expectation that "__str__()" return a valid Python expression: a + more convenient or concise representation can be used. + + The default implementation defined by the built-in type "object" + calls "object.__repr__()". + +object.__bytes__(self) + + Called by bytes to compute a byte-string representation of an + object. This should return a "bytes" object. The "object" class + itself does not provide this method. + +object.__format__(self, format_spec) + + Called by the "format()" built-in function, and by extension, + evaluation of formatted string literals and the "str.format()" + method, to produce a “formatted” string representation of an + object. The *format_spec* argument is a string that contains a + description of the formatting options desired. The interpretation + of the *format_spec* argument is up to the type implementing + "__format__()", however most classes will either delegate + formatting to one of the built-in types, or use a similar + formatting option syntax. + + See Format Specification Mini-Language for a description of the + standard formatting syntax. + + The return value must be a string object. + + The default implementation by the "object" class should be given an + empty *format_spec* string. It delegates to "__str__()". + + Changed in version 3.4: The __format__ method of "object" itself + raises a "TypeError" if passed any non-empty string. + + Changed in version 3.7: "object.__format__(x, '')" is now + equivalent to "str(x)" rather than "format(str(x), '')". + +object.__lt__(self, other) +object.__le__(self, other) +object.__eq__(self, other) +object.__ne__(self, other) +object.__gt__(self, other) +object.__ge__(self, other) + + These are the so-called “rich comparison” methods. The + correspondence between operator symbols and method names is as + follows: "xy" calls + "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)". + + A rich comparison method may return the singleton "NotImplemented" + if it does not implement the operation for a given pair of + arguments. By convention, "False" and "True" are returned for a + successful comparison. However, these methods can return any value, + so if the comparison operator is used in a Boolean context (e.g., + in the condition of an "if" statement), Python will call "bool()" + on the value to determine if the result is true or false. + + By default, "object" implements "__eq__()" by using "is", returning + "NotImplemented" in the case of a false comparison: "True if x is y + else NotImplemented". For "__ne__()", by default it delegates to + "__eq__()" and inverts the result unless it is "NotImplemented". + There are no other implied relationships among the comparison + operators or default implementations; for example, the truth of + "(x.__hash__". + + If a class that does not override "__eq__()" wishes to suppress + hash support, it should include "__hash__ = None" in the class + definition. A class which defines its own "__hash__()" that + explicitly raises a "TypeError" would be incorrectly identified as + hashable by an "isinstance(obj, collections.abc.Hashable)" call. + + Note: + + By default, the "__hash__()" values of str and bytes objects are + “salted” with an unpredictable random value. Although they + remain constant within an individual Python process, they are not + predictable between repeated invocations of Python.This is + intended to provide protection against a denial-of-service caused + by carefully chosen inputs that exploit the worst case + performance of a dict insertion, *O*(*n*^2) complexity. See + http://ocert.org/advisories/ocert-2011-003.html for + details.Changing hash values affects the iteration order of sets. + Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds).See also + "PYTHONHASHSEED". + + Changed in version 3.3: Hash randomization is enabled by default. + +object.__bool__(self) + + Called to implement truth value testing and the built-in operation + "bool()"; should return "False" or "True". When this method is not + defined, "__len__()" is called, if it is defined, and the object is + considered true if its result is nonzero. If a class defines + neither "__len__()" nor "__bool__()" (which is true of the "object" + class itself), all its instances are considered true. + + +Customizing attribute access +============================ + +The following methods can be defined to customize the meaning of +attribute access (use of, assignment to, or deletion of "x.name") for +class instances. + +object.__getattr__(self, name) + + Called when the default attribute access fails with an + "AttributeError" (either "__getattribute__()" raises an + "AttributeError" because *name* is not an instance attribute or an + attribute in the class tree for "self"; or "__get__()" of a *name* + property raises "AttributeError"). This method should either + return the (computed) attribute value or raise an "AttributeError" + exception. The "object" class itself does not provide this method. + + Note that if the attribute is found through the normal mechanism, + "__getattr__()" is not called. (This is an intentional asymmetry + between "__getattr__()" and "__setattr__()".) This is done both for + efficiency reasons and because otherwise "__getattr__()" would have + no way to access other attributes of the instance. Note that at + least for instance variables, you can take total control by not + inserting any values in the instance attribute dictionary (but + instead inserting them in another object). See the + "__getattribute__()" method below for a way to actually get total + control over attribute access. + +object.__getattribute__(self, name) + + Called unconditionally to implement attribute accesses for + instances of the class. If the class also defines "__getattr__()", + the latter will not be called unless "__getattribute__()" either + calls it explicitly or raises an "AttributeError". This method + should return the (computed) attribute value or raise an + "AttributeError" exception. In order to avoid infinite recursion in + this method, its implementation should always call the base class + method with the same name to access any attributes it needs, for + example, "object.__getattribute__(self, name)". + + Note: + + This method may still be bypassed when looking up special methods + as the result of implicit invocation via language syntax or + built-in functions. See Special method lookup. + + For certain sensitive attribute accesses, raises an auditing event + "object.__getattr__" with arguments "obj" and "name". + +object.__setattr__(self, name, value) + + Called when an attribute assignment is attempted. This is called + instead of the normal mechanism (i.e. store the value in the + instance dictionary). *name* is the attribute name, *value* is the + value to be assigned to it. + + If "__setattr__()" wants to assign to an instance attribute, it + should call the base class method with the same name, for example, + "object.__setattr__(self, name, value)". + + For certain sensitive attribute assignments, raises an auditing + event "object.__setattr__" with arguments "obj", "name", "value". + +object.__delattr__(self, name) + + Like "__setattr__()" but for attribute deletion instead of + assignment. This should only be implemented if "del obj.name" is + meaningful for the object. + + For certain sensitive attribute deletions, raises an auditing event + "object.__delattr__" with arguments "obj" and "name". + +object.__dir__(self) + + Called when "dir()" is called on the object. An iterable must be + returned. "dir()" converts the returned iterable to a list and + sorts it. + + +Customizing module attribute access +----------------------------------- + +module.__getattr__() +module.__dir__() + +Special names "__getattr__" and "__dir__" can be also used to +customize access to module attributes. The "__getattr__" function at +the module level should accept one argument which is the name of an +attribute and return the computed value or raise an "AttributeError". +If an attribute is not found on a module object through the normal +lookup, i.e. "object.__getattribute__()", then "__getattr__" is +searched in the module "__dict__" before raising an "AttributeError". +If found, it is called with the attribute name and the result is +returned. + +The "__dir__" function should accept no arguments, and return an +iterable of strings that represents the names accessible on module. If +present, this function overrides the standard "dir()" search on a +module. + +module.__class__ + +For a more fine grained customization of the module behavior (setting +attributes, properties, etc.), one can set the "__class__" attribute +of a module object to a subclass of "types.ModuleType". For example: + + import sys + from types import ModuleType + + class VerboseModule(ModuleType): + def __repr__(self): + return f'Verbose {self.__name__}' + + def __setattr__(self, attr, value): + print(f'Setting {attr}...') + super().__setattr__(attr, value) + + sys.modules[__name__].__class__ = VerboseModule + +Note: + + Defining module "__getattr__" and setting module "__class__" only + affect lookups made using the attribute access syntax – directly + accessing the module globals (whether by code within the module, or + via a reference to the module’s globals dictionary) is unaffected. + +Changed in version 3.5: "__class__" module attribute is now writable. + +Added in version 3.7: "__getattr__" and "__dir__" module attributes. + +See also: + + **PEP 562** - Module __getattr__ and __dir__ + Describes the "__getattr__" and "__dir__" functions on modules. + + +Implementing Descriptors +------------------------ + +The following methods only apply when an instance of the class +containing the method (a so-called *descriptor* class) appears in an +*owner* class (the descriptor must be in either the owner’s class +dictionary or in the class dictionary for one of its parents). In the +examples below, “the attribute” refers to the attribute whose name is +the key of the property in the owner class’ "__dict__". The "object" +class itself does not implement any of these protocols. + +object.__get__(self, instance, owner=None) + + Called to get the attribute of the owner class (class attribute + access) or of an instance of that class (instance attribute + access). The optional *owner* argument is the owner class, while + *instance* is the instance that the attribute was accessed through, + or "None" when the attribute is accessed through the *owner*. + + This method should return the computed attribute value or raise an + "AttributeError" exception. + + **PEP 252** specifies that "__get__()" is callable with one or two + arguments. Python’s own built-in descriptors support this + specification; however, it is likely that some third-party tools + have descriptors that require both arguments. Python’s own + "__getattribute__()" implementation always passes in both arguments + whether they are required or not. + +object.__set__(self, instance, value) + + Called to set the attribute on an instance *instance* of the owner + class to a new value, *value*. + + Note, adding "__set__()" or "__delete__()" changes the kind of + descriptor to a “data descriptor”. See Invoking Descriptors for + more details. + +object.__delete__(self, instance) + + Called to delete the attribute on an instance *instance* of the + owner class. + +Instances of descriptors may also have the "__objclass__" attribute +present: + +object.__objclass__ + + The attribute "__objclass__" is interpreted by the "inspect" module + as specifying the class where this object was defined (setting this + appropriately can assist in runtime introspection of dynamic class + attributes). For callables, it may indicate that an instance of the + given type (or a subclass) is expected or required as the first + positional argument (for example, CPython sets this attribute for + unbound methods that are implemented in C). + + +Invoking Descriptors +-------------------- + +In general, a descriptor is an object attribute with “binding +behavior”, one whose attribute access has been overridden by methods +in the descriptor protocol: "__get__()", "__set__()", and +"__delete__()". If any of those methods are defined for an object, it +is said to be a descriptor. + +The default behavior for attribute access is to get, set, or delete +the attribute from an object’s dictionary. For instance, "a.x" has a +lookup chain starting with "a.__dict__['x']", then +"type(a).__dict__['x']", and continuing through the base classes of +"type(a)" excluding metaclasses. + +However, if the looked-up value is an object defining one of the +descriptor methods, then Python may override the default behavior and +invoke the descriptor method instead. Where this occurs in the +precedence chain depends on which descriptor methods were defined and +how they were called. + +The starting point for descriptor invocation is a binding, "a.x". How +the arguments are assembled depends on "a": + +Direct Call + The simplest and least common call is when user code directly + invokes a descriptor method: "x.__get__(a)". + +Instance Binding + If binding to an object instance, "a.x" is transformed into the + call: "type(a).__dict__['x'].__get__(a, type(a))". + +Class Binding + If binding to a class, "A.x" is transformed into the call: + "A.__dict__['x'].__get__(None, A)". + +Super Binding + A dotted lookup such as "super(A, a).x" searches + "a.__class__.__mro__" for a base class "B" following "A" and then + returns "B.__dict__['x'].__get__(a, A)". If not a descriptor, "x" + is returned unchanged. + +For instance bindings, the precedence of descriptor invocation depends +on which descriptor methods are defined. A descriptor can define any +combination of "__get__()", "__set__()" and "__delete__()". If it +does not define "__get__()", then accessing the attribute will return +the descriptor object itself unless there is a value in the object’s +instance dictionary. If the descriptor defines "__set__()" and/or +"__delete__()", it is a data descriptor; if it defines neither, it is +a non-data descriptor. Normally, data descriptors define both +"__get__()" and "__set__()", while non-data descriptors have just the +"__get__()" method. Data descriptors with "__get__()" and "__set__()" +(and/or "__delete__()") defined always override a redefinition in an +instance dictionary. In contrast, non-data descriptors can be +overridden by instances. + +Python methods (including those decorated with "@staticmethod" and +"@classmethod") are implemented as non-data descriptors. Accordingly, +instances can redefine and override methods. This allows individual +instances to acquire behaviors that differ from other instances of the +same class. + +The "property()" function is implemented as a data descriptor. +Accordingly, instances cannot override the behavior of a property. + + +__slots__ +--------- + +*__slots__* allow us to explicitly declare data members (like +properties) and deny the creation of "__dict__" and *__weakref__* +(unless explicitly declared in *__slots__* or available in a parent.) + +The space saved over using "__dict__" can be significant. Attribute +lookup speed can be significantly improved as well. + +object.__slots__ + + This class variable can be assigned a string, iterable, or sequence + of strings with variable names used by instances. *__slots__* + reserves space for the declared variables and prevents the + automatic creation of "__dict__" and *__weakref__* for each + instance. + +Notes on using *__slots__*: + +* When inheriting from a class without *__slots__*, the "__dict__" and + *__weakref__* attribute of the instances will always be accessible. + +* Without a "__dict__" variable, instances cannot be assigned new + variables not listed in the *__slots__* definition. Attempts to + assign to an unlisted variable name raises "AttributeError". If + dynamic assignment of new variables is desired, then add + "'__dict__'" to the sequence of strings in the *__slots__* + declaration. + +* Without a *__weakref__* variable for each instance, classes defining + *__slots__* do not support "weak references" to its instances. If + weak reference support is needed, then add "'__weakref__'" to the + sequence of strings in the *__slots__* declaration. + +* *__slots__* are implemented at the class level by creating + descriptors for each variable name. As a result, class attributes + cannot be used to set default values for instance variables defined + by *__slots__*; otherwise, the class attribute would overwrite the + descriptor assignment. + +* The action of a *__slots__* declaration is not limited to the class + where it is defined. *__slots__* declared in parents are available + in child classes. However, instances of a child subclass will get a + "__dict__" and *__weakref__* unless the subclass also defines + *__slots__* (which should only contain names of any *additional* + slots). + +* If a class defines a slot also defined in a base class, the instance + variable defined by the base class slot is inaccessible (except by + retrieving its descriptor directly from the base class). This + renders the meaning of the program undefined. In the future, a + check may be added to prevent this. + +* "TypeError" will be raised if nonempty *__slots__* are defined for a + class derived from a ""variable-length" built-in type" such as + "int", "bytes", and "tuple". + +* Any non-string *iterable* may be assigned to *__slots__*. + +* If a "dictionary" is used to assign *__slots__*, the dictionary keys + will be used as the slot names. The values of the dictionary can be + used to provide per-attribute docstrings that will be recognised by + "inspect.getdoc()" and displayed in the output of "help()". + +* "__class__" assignment works only if both classes have the same + *__slots__*. + +* Multiple inheritance with multiple slotted parent classes can be + used, but only one parent is allowed to have attributes created by + slots (the other bases must have empty slot layouts) - violations + raise "TypeError". + +* If an *iterator* is used for *__slots__* then a *descriptor* is + created for each of the iterator’s values. However, the *__slots__* + attribute will be an empty iterator. + + +Customizing class creation +========================== + +Whenever a class inherits from another class, "__init_subclass__()" is +called on the parent class. This way, it is possible to write classes +which change the behavior of subclasses. This is closely related to +class decorators, but where class decorators only affect the specific +class they’re applied to, "__init_subclass__" solely applies to future +subclasses of the class defining the method. + +classmethod object.__init_subclass__(cls) + + This method is called whenever the containing class is subclassed. + *cls* is then the new subclass. If defined as a normal instance + method, this method is implicitly converted to a class method. + + Keyword arguments which are given to a new class are passed to the + parent class’s "__init_subclass__". For compatibility with other + classes using "__init_subclass__", one should take out the needed + keyword arguments and pass the others over to the base class, as + in: + + class Philosopher: + def __init_subclass__(cls, /, default_name, **kwargs): + super().__init_subclass__(**kwargs) + cls.default_name = default_name + + class AustralianPhilosopher(Philosopher, default_name="Bruce"): + pass + + The default implementation "object.__init_subclass__" does nothing, + but raises an error if it is called with any arguments. + + Note: + + The metaclass hint "metaclass" is consumed by the rest of the + type machinery, and is never passed to "__init_subclass__" + implementations. The actual metaclass (rather than the explicit + hint) can be accessed as "type(cls)". + + Added in version 3.6. + +When a class is created, "type.__new__()" scans the class variables +and makes callbacks to those with a "__set_name__()" hook. + +object.__set_name__(self, owner, name) + + Automatically called at the time the owning class *owner* is + created. The object has been assigned to *name* in that class: + + class A: + x = C() # Automatically calls: x.__set_name__(A, 'x') + + If the class variable is assigned after the class is created, + "__set_name__()" will not be called automatically. If needed, + "__set_name__()" can be called directly: + + class A: + pass + + c = C() + A.x = c # The hook is not called + c.__set_name__(A, 'x') # Manually invoke the hook + + See Creating the class object for more details. + + Added in version 3.6. + + +Metaclasses +----------- + +By default, classes are constructed using "type()". The class body is +executed in a new namespace and the class name is bound locally to the +result of "type(name, bases, namespace)". + +The class creation process can be customized by passing the +"metaclass" keyword argument in the class definition line, or by +inheriting from an existing class that included such an argument. In +the following example, both "MyClass" and "MySubclass" are instances +of "Meta": + + class Meta(type): + pass + + class MyClass(metaclass=Meta): + pass + + class MySubclass(MyClass): + pass + +Any other keyword arguments that are specified in the class definition +are passed through to all metaclass operations described below. + +When a class definition is executed, the following steps occur: + +* MRO entries are resolved; + +* the appropriate metaclass is determined; + +* the class namespace is prepared; + +* the class body is executed; + +* the class object is created. + + +Resolving MRO entries +--------------------- + +object.__mro_entries__(self, bases) + + If a base that appears in a class definition is not an instance of + "type", then an "__mro_entries__()" method is searched on the base. + If an "__mro_entries__()" method is found, the base is substituted + with the result of a call to "__mro_entries__()" when creating the + class. The method is called with the original bases tuple passed to + the *bases* parameter, and must return a tuple of classes that will + be used instead of the base. The returned tuple may be empty: in + these cases, the original base is ignored. + +See also: + + "types.resolve_bases()" + Dynamically resolve bases that are not instances of "type". + + "types.get_original_bases()" + Retrieve a class’s “original bases” prior to modifications by + "__mro_entries__()". + + **PEP 560** + Core support for typing module and generic types. + + +Determining the appropriate metaclass +------------------------------------- + +The appropriate metaclass for a class definition is determined as +follows: + +* if no bases and no explicit metaclass are given, then "type()" is + used; + +* if an explicit metaclass is given and it is *not* an instance of + "type()", then it is used directly as the metaclass; + +* if an instance of "type()" is given as the explicit metaclass, or + bases are defined, then the most derived metaclass is used. + +The most derived metaclass is selected from the explicitly specified +metaclass (if any) and the metaclasses (i.e. "type(cls)") of all +specified base classes. The most derived metaclass is one which is a +subtype of *all* of these candidate metaclasses. If none of the +candidate metaclasses meets that criterion, then the class definition +will fail with "TypeError". + + +Preparing the class namespace +----------------------------- + +Once the appropriate metaclass has been identified, then the class +namespace is prepared. If the metaclass has a "__prepare__" attribute, +it is called as "namespace = metaclass.__prepare__(name, bases, +**kwds)" (where the additional keyword arguments, if any, come from +the class definition). The "__prepare__" method should be implemented +as a "classmethod". The namespace returned by "__prepare__" is passed +in to "__new__", but when the final class object is created the +namespace is copied into a new "dict". + +If the metaclass has no "__prepare__" attribute, then the class +namespace is initialised as an empty ordered mapping. + +See also: + + **PEP 3115** - Metaclasses in Python 3000 + Introduced the "__prepare__" namespace hook + + +Executing the class body +------------------------ + +The class body is executed (approximately) as "exec(body, globals(), +namespace)". The key difference from a normal call to "exec()" is that +lexical scoping allows the class body (including any methods) to +reference names from the current and outer scopes when the class +definition occurs inside a function. + +However, even when the class definition occurs inside the function, +methods defined inside the class still cannot see names defined at the +class scope. Class variables must be accessed through the first +parameter of instance or class methods, or through the implicit +lexically scoped "__class__" reference described in the next section. + + +Creating the class object +------------------------- + +Once the class namespace has been populated by executing the class +body, the class object is created by calling "metaclass(name, bases, +namespace, **kwds)" (the additional keywords passed here are the same +as those passed to "__prepare__"). + +This class object is the one that will be referenced by the zero- +argument form of "super()". "__class__" is an implicit closure +reference created by the compiler if any methods in a class body refer +to either "__class__" or "super". This allows the zero argument form +of "super()" to correctly identify the class being defined based on +lexical scoping, while the class or instance that was used to make the +current call is identified based on the first argument passed to the +method. + +**CPython implementation detail:** In CPython 3.6 and later, the +"__class__" cell is passed to the metaclass as a "__classcell__" entry +in the class namespace. If present, this must be propagated up to the +"type.__new__" call in order for the class to be initialised +correctly. Failing to do so will result in a "RuntimeError" in Python +3.8. + +When using the default metaclass "type", or any metaclass that +ultimately calls "type.__new__", the following additional +customization steps are invoked after creating the class object: + +1. The "type.__new__" method collects all of the attributes in the + class namespace that define a "__set_name__()" method; + +2. Those "__set_name__" methods are called with the class being + defined and the assigned name of that particular attribute; + +3. The "__init_subclass__()" hook is called on the immediate parent of + the new class in its method resolution order. + +After the class object is created, it is passed to the class +decorators included in the class definition (if any) and the resulting +object is bound in the local namespace as the defined class. + +When a new class is created by "type.__new__", the object provided as +the namespace parameter is copied to a new ordered mapping and the +original object is discarded. The new copy is wrapped in a read-only +proxy, which becomes the "__dict__" attribute of the class object. + +See also: + + **PEP 3135** - New super + Describes the implicit "__class__" closure reference + + +Uses for metaclasses +-------------------- + +The potential uses for metaclasses are boundless. Some ideas that have +been explored include enum, logging, interface checking, automatic +delegation, automatic property creation, proxies, frameworks, and +automatic resource locking/synchronization. + + +Customizing instance and subclass checks +======================================== + +The following methods are used to override the default behavior of the +"isinstance()" and "issubclass()" built-in functions. + +In particular, the metaclass "abc.ABCMeta" implements these methods in +order to allow the addition of Abstract Base Classes (ABCs) as +“virtual base classes” to any class or type (including built-in +types), including other ABCs. + +type.__instancecheck__(self, instance) + + Return true if *instance* should be considered a (direct or + indirect) instance of *class*. If defined, called to implement + "isinstance(instance, class)". + +type.__subclasscheck__(self, subclass) + + Return true if *subclass* should be considered a (direct or + indirect) subclass of *class*. If defined, called to implement + "issubclass(subclass, class)". + +Note that these methods are looked up on the type (metaclass) of a +class. They cannot be defined as class methods in the actual class. +This is consistent with the lookup of special methods that are called +on instances, only in this case the instance is itself a class. + +See also: + + **PEP 3119** - Introducing Abstract Base Classes + Includes the specification for customizing "isinstance()" and + "issubclass()" behavior through "__instancecheck__()" and + "__subclasscheck__()", with motivation for this functionality in + the context of adding Abstract Base Classes (see the "abc" + module) to the language. + + +Emulating generic types +======================= + +When using *type annotations*, it is often useful to *parameterize* a +*generic type* using Python’s square-brackets notation. For example, +the annotation "list[int]" might be used to signify a "list" in which +all the elements are of type "int". + +See also: + + **PEP 484** - Type Hints + Introducing Python’s framework for type annotations + + Generic Alias Types + Documentation for objects representing parameterized generic + classes + + Generics, user-defined generics and "typing.Generic" + Documentation on how to implement generic classes that can be + parameterized at runtime and understood by static type-checkers. + +A class can *generally* only be parameterized if it defines the +special class method "__class_getitem__()". + +classmethod object.__class_getitem__(cls, key) + + Return an object representing the specialization of a generic class + by type arguments found in *key*. + + When defined on a class, "__class_getitem__()" is automatically a + class method. As such, there is no need for it to be decorated with + "@classmethod" when it is defined. + + +The purpose of *__class_getitem__* +---------------------------------- + +The purpose of "__class_getitem__()" is to allow runtime +parameterization of standard-library generic classes in order to more +easily apply *type hints* to these classes. + +To implement custom generic classes that can be parameterized at +runtime and understood by static type-checkers, users should either +inherit from a standard library class that already implements +"__class_getitem__()", or inherit from "typing.Generic", which has its +own implementation of "__class_getitem__()". + +Custom implementations of "__class_getitem__()" on classes defined +outside of the standard library may not be understood by third-party +type-checkers such as mypy. Using "__class_getitem__()" on any class +for purposes other than type hinting is discouraged. + + +*__class_getitem__* versus *__getitem__* +---------------------------------------- + +Usually, the subscription of an object using square brackets will call +the "__getitem__()" instance method defined on the object’s class. +However, if the object being subscribed is itself a class, the class +method "__class_getitem__()" may be called instead. +"__class_getitem__()" should return a GenericAlias object if it is +properly defined. + +Presented with the *expression* "obj[x]", the Python interpreter +follows something like the following process to decide whether +"__getitem__()" or "__class_getitem__()" should be called: + + from inspect import isclass + + def subscribe(obj, x): + """Return the result of the expression 'obj[x]'""" + + class_of_obj = type(obj) + + # If the class of obj defines __getitem__, + # call class_of_obj.__getitem__(obj, x) + if hasattr(class_of_obj, '__getitem__'): + return class_of_obj.__getitem__(obj, x) + + # Else, if obj is a class and defines __class_getitem__, + # call obj.__class_getitem__(x) + elif isclass(obj) and hasattr(obj, '__class_getitem__'): + return obj.__class_getitem__(x) + + # Else, raise an exception + else: + raise TypeError( + f"'{class_of_obj.__name__}' object is not subscriptable" + ) + +In Python, all classes are themselves instances of other classes. The +class of a class is known as that class’s *metaclass*, and most +classes have the "type" class as their metaclass. "type" does not +define "__getitem__()", meaning that expressions such as "list[int]", +"dict[str, float]" and "tuple[str, bytes]" all result in +"__class_getitem__()" being called: + + >>> # list has class "type" as its metaclass, like most classes: + >>> type(list) + + >>> type(dict) == type(list) == type(tuple) == type(str) == type(bytes) + True + >>> # "list[int]" calls "list.__class_getitem__(int)" + >>> list[int] + list[int] + >>> # list.__class_getitem__ returns a GenericAlias object: + >>> type(list[int]) + + +However, if a class has a custom metaclass that defines +"__getitem__()", subscribing the class may result in different +behaviour. An example of this can be found in the "enum" module: + + >>> from enum import Enum + >>> class Menu(Enum): + ... """A breakfast menu""" + ... SPAM = 'spam' + ... BACON = 'bacon' + ... + >>> # Enum classes have a custom metaclass: + >>> type(Menu) + + >>> # EnumMeta defines __getitem__, + >>> # so __class_getitem__ is not called, + >>> # and the result is not a GenericAlias object: + >>> Menu['SPAM'] + + >>> type(Menu['SPAM']) + + +See also: + + **PEP 560** - Core Support for typing module and generic types + Introducing "__class_getitem__()", and outlining when a + subscription results in "__class_getitem__()" being called + instead of "__getitem__()" + + +Emulating callable objects +========================== + +object.__call__(self[, args...]) + + Called when the instance is “called” as a function; if this method + is defined, "x(arg1, arg2, ...)" roughly translates to + "type(x).__call__(x, arg1, ...)". The "object" class itself does + not provide this method. + + +Emulating container types +========================= + +The following methods can be defined to implement container objects. +None of them are provided by the "object" class itself. Containers +usually are *sequences* (such as "lists" or "tuples") or *mappings* +(like *dictionaries*), but can represent other containers as well. +The first set of methods is used either to emulate a sequence or to +emulate a mapping; the difference is that for a sequence, the +allowable keys should be the integers *k* for which "0 <= k < N" where +*N* is the length of the sequence, or "slice" objects, which define a +range of items. It is also recommended that mappings provide the +methods "keys()", "values()", "items()", "get()", "clear()", +"setdefault()", "pop()", "popitem()", "copy()", and "update()" +behaving similar to those for Python’s standard "dictionary" objects. +The "collections.abc" module provides a "MutableMapping" *abstract +base class* to help create those methods from a base set of +"__getitem__()", "__setitem__()", "__delitem__()", and "keys()". + +Mutable sequences should provide methods "append()", "clear()", +"count()", "extend()", "index()", "insert()", "pop()", "remove()", and +"reverse()", like Python standard "list" objects. Finally, sequence +types should implement addition (meaning concatenation) and +multiplication (meaning repetition) by defining the methods +"__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and +"__imul__()" described below; they should not define other numerical +operators. + +It is recommended that both mappings and sequences implement the +"__contains__()" method to allow efficient use of the "in" operator; +for mappings, "in" should search the mapping’s keys; for sequences, it +should search through the values. It is further recommended that both +mappings and sequences implement the "__iter__()" method to allow +efficient iteration through the container; for mappings, "__iter__()" +should iterate through the object’s keys; for sequences, it should +iterate through the values. + +object.__len__(self) + + Called to implement the built-in function "len()". Should return + the length of the object, an integer ">=" 0. Also, an object that + doesn’t define a "__bool__()" method and whose "__len__()" method + returns zero is considered to be false in a Boolean context. + + **CPython implementation detail:** In CPython, the length is + required to be at most "sys.maxsize". If the length is larger than + "sys.maxsize" some features (such as "len()") may raise + "OverflowError". To prevent raising "OverflowError" by truth value + testing, an object must define a "__bool__()" method. + +object.__length_hint__(self) + + Called to implement "operator.length_hint()". Should return an + estimated length for the object (which may be greater or less than + the actual length). The length must be an integer ">=" 0. The + return value may also be "NotImplemented", which is treated the + same as if the "__length_hint__" method didn’t exist at all. This + method is purely an optimization and is never required for + correctness. + + Added in version 3.4. + +Note: + + Slicing is done exclusively with the following three methods. A + call like + + a[1:2] = b + + is translated to + + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with "None". + +object.__getitem__(self, key) + + Called to implement evaluation of "self[key]". For *sequence* + types, the accepted keys should be integers. Optionally, they may + support "slice" objects as well. Negative index support is also + optional. If *key* is of an inappropriate type, "TypeError" may be + raised; if *key* is a value outside the set of indexes for the + sequence (after any special interpretation of negative values), + "IndexError" should be raised. For *mapping* types, if *key* is + missing (not in the container), "KeyError" should be raised. + + Note: + + "for" loops expect that an "IndexError" will be raised for + illegal indexes to allow proper detection of the end of the + sequence. + + Note: + + When subscripting a *class*, the special class method + "__class_getitem__()" may be called instead of "__getitem__()". + See __class_getitem__ versus __getitem__ for more details. + +object.__setitem__(self, key, value) + + Called to implement assignment to "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support changes to the values for keys, or if new keys + can be added, or for sequences if elements can be replaced. The + same exceptions should be raised for improper *key* values as for + the "__getitem__()" method. + +object.__delitem__(self, key) + + Called to implement deletion of "self[key]". Same note as for + "__getitem__()". This should only be implemented for mappings if + the objects support removal of keys, or for sequences if elements + can be removed from the sequence. The same exceptions should be + raised for improper *key* values as for the "__getitem__()" method. + +object.__missing__(self, key) + + Called by "dict"."__getitem__()" to implement "self[key]" for dict + subclasses when key is not in the dictionary. + +object.__iter__(self) + + This method is called when an *iterator* is required for a + container. This method should return a new iterator object that can + iterate over all the objects in the container. For mappings, it + should iterate over the keys of the container. + +object.__reversed__(self) + + Called (if present) by the "reversed()" built-in to implement + reverse iteration. It should return a new iterator object that + iterates over all the objects in the container in reverse order. + + If the "__reversed__()" method is not provided, the "reversed()" + built-in will fall back to using the sequence protocol ("__len__()" + and "__getitem__()"). Objects that support the sequence protocol + should only provide "__reversed__()" if they can provide an + implementation that is more efficient than the one provided by + "reversed()". + +The membership test operators ("in" and "not in") are normally +implemented as an iteration through a container. However, container +objects can supply the following special method with a more efficient +implementation, which also does not require the object be iterable. + +object.__contains__(self, item) + + Called to implement membership test operators. Should return true + if *item* is in *self*, false otherwise. For mapping objects, this + should consider the keys of the mapping rather than the values or + the key-item pairs. + + For objects that don’t define "__contains__()", the membership test + first tries iteration via "__iter__()", then the old sequence + iteration protocol via "__getitem__()", see this section in the + language reference. + + +Emulating numeric types +======================= + +The following methods can be defined to emulate numeric objects. +Methods corresponding to operations that are not supported by the +particular kind of number implemented (e.g., bitwise operations for +non-integral numbers) should be left undefined. + +object.__add__(self, other) +object.__sub__(self, other) +object.__mul__(self, other) +object.__matmul__(self, other) +object.__truediv__(self, other) +object.__floordiv__(self, other) +object.__mod__(self, other) +object.__divmod__(self, other) +object.__pow__(self, other[, modulo]) +object.__lshift__(self, other) +object.__rshift__(self, other) +object.__and__(self, other) +object.__xor__(self, other) +object.__or__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to + evaluate the expression "x + y", where *x* is an instance of a + class that has an "__add__()" method, "type(x).__add__(x, y)" is + called. The "__divmod__()" method should be the equivalent to + using "__floordiv__()" and "__mod__()"; it should not be related to + "__truediv__()". Note that "__pow__()" should be defined to accept + an optional third argument if the three-argument version of the + built-in "pow()" function is to be supported. + + If one of those methods does not support the operation with the + supplied arguments, it should return "NotImplemented". + +object.__radd__(self, other) +object.__rsub__(self, other) +object.__rmul__(self, other) +object.__rmatmul__(self, other) +object.__rtruediv__(self, other) +object.__rfloordiv__(self, other) +object.__rmod__(self, other) +object.__rdivmod__(self, other) +object.__rpow__(self, other[, modulo]) +object.__rlshift__(self, other) +object.__rrshift__(self, other) +object.__rand__(self, other) +object.__rxor__(self, other) +object.__ror__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped) + operands. These functions are only called if the operands are of + different types, when the left operand does not support the + corresponding operation [3], or the right operand’s class is + derived from the left operand’s class. [4] For instance, to + evaluate the expression "x - y", where *y* is an instance of a + class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is + called if "type(x).__sub__(x, y)" returns "NotImplemented" or + "type(y)" is a subclass of "type(x)". [5] + + Note that "__rpow__()" should be defined to accept an optional + third argument if the three-argument version of the built-in + "pow()" function is to be supported. + + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. + + Note: + + If the right operand’s type is a subclass of the left operand’s + type and that subclass provides a different implementation of the + reflected method for the operation, this method will be called + before the left operand’s non-reflected method. This behavior + allows subclasses to override their ancestors’ operations. + +object.__iadd__(self, other) +object.__isub__(self, other) +object.__imul__(self, other) +object.__imatmul__(self, other) +object.__itruediv__(self, other) +object.__ifloordiv__(self, other) +object.__imod__(self, other) +object.__ipow__(self, other[, modulo]) +object.__ilshift__(self, other) +object.__irshift__(self, other) +object.__iand__(self, other) +object.__ixor__(self, other) +object.__ior__(self, other) + + These methods are called to implement the augmented arithmetic + assignments ("+=", "-=", "*=", "@=", "/=", "//=", "%=", "**=", + "<<=", ">>=", "&=", "^=", "|="). These methods should attempt to + do the operation in-place (modifying *self*) and return the result + (which could be, but does not have to be, *self*). If a specific + method is not defined, or if that method returns "NotImplemented", + the augmented assignment falls back to the normal methods. For + instance, if *x* is an instance of a class with an "__iadd__()" + method, "x += y" is equivalent to "x = x.__iadd__(y)" . If + "__iadd__()" does not exist, or if "x.__iadd__(y)" returns + "NotImplemented", "x.__add__(y)" and "y.__radd__(x)" are + considered, as with the evaluation of "x + y". In certain + situations, augmented assignment can result in unexpected errors + (see Why does a_tuple[i] += [‘item’] raise an exception when the + addition works?), but this behavior is in fact part of the data + model. + +object.__neg__(self) +object.__pos__(self) +object.__abs__(self) +object.__invert__(self) + + Called to implement the unary arithmetic operations ("-", "+", + "abs()" and "~"). + +object.__complex__(self) +object.__int__(self) +object.__float__(self) + + Called to implement the built-in functions "complex()", "int()" and + "float()". Should return a value of the appropriate type. + +object.__index__(self) + + Called to implement "operator.index()", and whenever Python needs + to losslessly convert the numeric object to an integer object (such + as in slicing, or in the built-in "bin()", "hex()" and "oct()" + functions). Presence of this method indicates that the numeric + object is an integer type. Must return an integer. + + If "__int__()", "__float__()" and "__complex__()" are not defined + then corresponding built-in functions "int()", "float()" and + "complex()" fall back to "__index__()". + +object.__round__(self[, ndigits]) +object.__trunc__(self) +object.__floor__(self) +object.__ceil__(self) + + Called to implement the built-in function "round()" and "math" + functions "trunc()", "floor()" and "ceil()". Unless *ndigits* is + passed to "__round__()" all these methods should return the value + of the object truncated to an "Integral" (typically an "int"). + + Changed in version 3.14: "int()" no longer delegates to the + "__trunc__()" method. + + +With Statement Context Managers +=============================== + +A *context manager* is an object that defines the runtime context to +be established when executing a "with" statement. The context manager +handles the entry into, and the exit from, the desired runtime context +for the execution of the block of code. Context managers are normally +invoked using the "with" statement (described in section The with +statement), but can also be used by directly invoking their methods. + +Typical uses of context managers include saving and restoring various +kinds of global state, locking and unlocking resources, closing opened +files, etc. + +For more information on context managers, see Context Manager Types. +The "object" class itself does not provide the context manager +methods. + +object.__enter__(self) + + Enter the runtime context related to this object. The "with" + statement will bind this method’s return value to the target(s) + specified in the "as" clause of the statement, if any. + +object.__exit__(self, exc_type, exc_value, traceback) + + Exit the runtime context related to this object. The parameters + describe the exception that caused the context to be exited. If the + context was exited without an exception, all three arguments will + be "None". + + If an exception is supplied, and the method wishes to suppress the + exception (i.e., prevent it from being propagated), it should + return a true value. Otherwise, the exception will be processed + normally upon exit from this method. + + Note that "__exit__()" methods should not reraise the passed-in + exception; this is the caller’s responsibility. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. + + +Customizing positional arguments in class pattern matching +========================================================== + +When using a class name in a pattern, positional arguments in the +pattern are not allowed by default, i.e. "case MyClass(x, y)" is +typically invalid without special support in "MyClass". To be able to +use that kind of pattern, the class needs to define a *__match_args__* +attribute. + +object.__match_args__ + + This class variable can be assigned a tuple of strings. When this + class is used in a class pattern with positional arguments, each + positional argument will be converted into a keyword argument, + using the corresponding value in *__match_args__* as the keyword. + The absence of this attribute is equivalent to setting it to "()". + +For example, if "MyClass.__match_args__" is "("left", "center", +"right")" that means that "case MyClass(x, y)" is equivalent to "case +MyClass(left=x, center=y)". Note that the number of arguments in the +pattern must be smaller than or equal to the number of elements in +*__match_args__*; if it is larger, the pattern match attempt will +raise a "TypeError". + +Added in version 3.10. + +See also: + + **PEP 634** - Structural Pattern Matching + The specification for the Python "match" statement. + + +Emulating buffer types +====================== + +The buffer protocol provides a way for Python objects to expose +efficient access to a low-level memory array. This protocol is +implemented by builtin types such as "bytes" and "memoryview", and +third-party libraries may define additional buffer types. + +While buffer types are usually implemented in C, it is also possible +to implement the protocol in Python. + +object.__buffer__(self, flags) + + Called when a buffer is requested from *self* (for example, by the + "memoryview" constructor). The *flags* argument is an integer + representing the kind of buffer requested, affecting for example + whether the returned buffer is read-only or writable. + "inspect.BufferFlags" provides a convenient way to interpret the + flags. The method must return a "memoryview" object. + +object.__release_buffer__(self, buffer) + + Called when a buffer is no longer needed. The *buffer* argument is + a "memoryview" object that was previously returned by + "__buffer__()". The method must release any resources associated + with the buffer. This method should return "None". Buffer objects + that do not need to perform any cleanup are not required to + implement this method. + +Added in version 3.12. + +See also: + + **PEP 688** - Making the buffer protocol accessible in Python + Introduces the Python "__buffer__" and "__release_buffer__" + methods. + + "collections.abc.Buffer" + ABC for buffer types. + + +Annotations +=========== + +Functions, classes, and modules may contain *annotations*, which are a +way to associate information (usually *type hints*) with a symbol. + +object.__annotations__ + + This attribute contains the annotations for an object. It is lazily + evaluated, so accessing the attribute may execute arbitrary code + and raise exceptions. If evaluation is successful, the attribute is + set to a dictionary mapping from variable names to annotations. + + Changed in version 3.14: Annotations are now lazily evaluated. + +object.__annotate__(format) + + An *annotate function*. Returns a new dictionary object mapping + attribute/parameter names to their annotation values. + + Takes a format parameter specifying the format in which annotations + values should be provided. It must be a member of the + "annotationlib.Format" enum, or an integer with a value + corresponding to a member of the enum. + + If an annotate function doesn’t support the requested format, it + must raise "NotImplementedError". Annotate functions must always + support "VALUE" format; they must not raise "NotImplementedError()" + when called with this format. + + When called with "VALUE" format, an annotate function may raise + "NameError"; it must not raise "NameError" when called requesting + any other format. + + If an object does not have any annotations, "__annotate__" should + preferably be set to "None" (it can’t be deleted), rather than set + to a function that returns an empty dict. + + Added in version 3.14. + +See also: + + **PEP 649** — Deferred evaluation of annotation using descriptors + Introduces lazy evaluation of annotations and the "__annotate__" + function. + + +Special method lookup +===================== + +For custom classes, implicit invocations of special methods are only +guaranteed to work correctly if defined on an object’s type, not in +the object’s instance dictionary. That behaviour is the reason why +the following code raises an exception: + + >>> class C: + ... pass + ... + >>> c = C() + >>> c.__len__ = lambda: 5 + >>> len(c) + Traceback (most recent call last): + File "", line 1, in + TypeError: object of type 'C' has no len() + +The rationale behind this behaviour lies with a number of special +methods such as "__hash__()" and "__repr__()" that are implemented by +all objects, including type objects. If the implicit lookup of these +methods used the conventional lookup process, they would fail when +invoked on the type object itself: + + >>> 1 .__hash__() == hash(1) + True + >>> int.__hash__() == hash(int) + Traceback (most recent call last): + File "", line 1, in + TypeError: descriptor '__hash__' of 'int' object needs an argument + +Incorrectly attempting to invoke an unbound method of a class in this +way is sometimes referred to as ‘metaclass confusion’, and is avoided +by bypassing the instance when looking up special methods: + + >>> type(1).__hash__(1) == hash(1) + True + >>> type(int).__hash__(int) == hash(int) + True + +In addition to bypassing any instance attributes in the interest of +correctness, implicit special method lookup generally also bypasses +the "__getattribute__()" method even of the object’s metaclass: + + >>> class Meta(type): + ... def __getattribute__(*args): + ... print("Metaclass getattribute invoked") + ... return type.__getattribute__(*args) + ... + >>> class C(object, metaclass=Meta): + ... def __len__(self): + ... return 10 + ... def __getattribute__(*args): + ... print("Class getattribute invoked") + ... return object.__getattribute__(*args) + ... + >>> c = C() + >>> c.__len__() # Explicit lookup via instance + Class getattribute invoked + 10 + >>> type(c).__len__(c) # Explicit lookup via type + Metaclass getattribute invoked + 10 + >>> len(c) # Implicit lookup + 10 + +Bypassing the "__getattribute__()" machinery in this fashion provides +significant scope for speed optimisations within the interpreter, at +the cost of some flexibility in the handling of special methods (the +special method *must* be set on the class object itself in order to be +consistently invoked by the interpreter). +''', + 'string-methods': r'''String Methods +************** + +Strings implement all of the common sequence operations, along with +the additional methods described below. + +Strings also support two styles of string formatting, one providing a +large degree of flexibility and customization (see "str.format()", +Format String Syntax and Custom String Formatting) and the other based +on C "printf" style formatting that handles a narrower range of types +and is slightly harder to use correctly, but is often faster for the +cases it can handle (printf-style String Formatting). + +The Text Processing Services section of the standard library covers a +number of other modules that provide various text related utilities +(including regular expression support in the "re" module). + +str.capitalize() + + Return a copy of the string with its first character capitalized + and the rest lowercased. + + Changed in version 3.8: The first character is now put into + titlecase rather than uppercase. This means that characters like + digraphs will only have their first letter capitalized, instead of + the full character. + +str.casefold() + + Return a casefolded copy of the string. Casefolded strings may be + used for caseless matching. + + Casefolding is similar to lowercasing but more aggressive because + it is intended to remove all case distinctions in a string. For + example, the German lowercase letter "'ß'" is equivalent to ""ss"". + Since it is already lowercase, "lower()" would do nothing to "'ß'"; + "casefold()" converts it to ""ss"". For example: + + >>> 'straße'.lower() + 'straße' + >>> 'straße'.casefold() + 'strasse' + + The casefolding algorithm is described in section 3.13 ‘Default + Case Folding’ of the Unicode Standard. + + Added in version 3.3. + +str.center(width, fillchar=' ', /) + + Return centered in a string of length *width*. Padding is done + using the specified *fillchar* (default is an ASCII space). The + original string is returned if *width* is less than or equal to + "len(s)". For example: + + >>> 'Python'.center(10) + ' Python ' + >>> 'Python'.center(10, '-') + '--Python--' + >>> 'Python'.center(4) + 'Python' + +str.count(sub[, start[, end]]) + + Return the number of non-overlapping occurrences of substring *sub* + in the range [*start*, *end*]. Optional arguments *start* and + *end* are interpreted as in slice notation. + + If *sub* is empty, returns the number of empty strings between + characters which is the length of the string plus one. For example: + + >>> 'spam, spam, spam'.count('spam') + 3 + >>> 'spam, spam, spam'.count('spam', 5) + 2 + >>> 'spam, spam, spam'.count('spam', 5, 10) + 1 + >>> 'spam, spam, spam'.count('eggs') + 0 + >>> 'spam, spam, spam'.count('') + 17 + +str.encode(encoding='utf-8', errors='strict') + + Return the string encoded to "bytes". + + *encoding* defaults to "'utf-8'"; see Standard Encodings for + possible values. + + *errors* controls how encoding errors are handled. If "'strict'" + (the default), a "UnicodeError" exception is raised. Other possible + values are "'ignore'", "'replace'", "'xmlcharrefreplace'", + "'backslashreplace'" and any other name registered via + "codecs.register_error()". See Error Handlers for details. + + For performance reasons, the value of *errors* is not checked for + validity unless an encoding error actually occurs, Python + Development Mode is enabled or a debug build is used. For example: + + >>> encoded_str_to_bytes = 'Python'.encode() + >>> type(encoded_str_to_bytes) + + >>> encoded_str_to_bytes + b'Python' + + Changed in version 3.1: Added support for keyword arguments. + + Changed in version 3.9: The value of the *errors* argument is now + checked in Python Development Mode and in debug mode. + +str.endswith(suffix[, start[, end]]) + + Return "True" if the string ends with the specified *suffix*, + otherwise return "False". *suffix* can also be a tuple of suffixes + to look for. With optional *start*, test beginning at that + position. With optional *end*, stop comparing at that position. + Using *start* and *end* is equivalent to + "str[start:end].endswith(suffix)". For example: + + >>> 'Python'.endswith('on') + True + >>> 'a tuple of suffixes'.endswith(('at', 'in')) + False + >>> 'a tuple of suffixes'.endswith(('at', 'es')) + True + >>> 'Python is amazing'.endswith('is', 0, 9) + True + + See also "startswith()" and "removesuffix()". + +str.expandtabs(tabsize=8) + + Return a copy of the string where all tab characters are replaced + by one or more spaces, depending on the current column and the + given tab size. Tab positions occur every *tabsize* characters + (default is 8, giving tab positions at columns 0, 8, 16 and so on). + To expand the string, the current column is set to zero and the + string is examined character by character. If the character is a + tab ("\t"), one or more space characters are inserted in the result + until the current column is equal to the next tab position. (The + tab character itself is not copied.) If the character is a newline + ("\n") or return ("\r"), it is copied and the current column is + reset to zero. Any other character is copied unchanged and the + current column is incremented by one regardless of how the + character is represented when printed. For example: + + >>> '01\t012\t0123\t01234'.expandtabs() + '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs(4) + '01 012 0123 01234' + >>> print('01\t012\n0123\t01234'.expandtabs(4)) + 01 012 + 0123 01234 + +str.find(sub[, start[, end]]) + + Return the lowest index in the string where substring *sub* is + found within the slice "s[start:end]". Optional arguments *start* + and *end* are interpreted as in slice notation. Return "-1" if + *sub* is not found. For example: + + >>> 'spam, spam, spam'.find('sp') + 0 + >>> 'spam, spam, spam'.find('sp', 5) + 6 + + See also "rfind()" and "index()". + + Note: + + The "find()" method should be used only if you need to know the + position of *sub*. To check if *sub* is a substring or not, use + the "in" operator: + + >>> 'Py' in 'Python' + True + +str.format(*args, **kwargs) + + Perform a string formatting operation. The string on which this + method is called can contain literal text or replacement fields + delimited by braces "{}". Each replacement field contains either + the numeric index of a positional argument, or the name of a + keyword argument. Returns a copy of the string where each + replacement field is replaced with the string value of the + corresponding argument. For example: + + >>> "The sum of 1 + 2 is {0}".format(1+2) + 'The sum of 1 + 2 is 3' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' + >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") + 'Nobody expects the Spanish Inquisition!' + + See Format String Syntax for a description of the various + formatting options that can be specified in format strings. + + Note: + + When formatting a number ("int", "float", "complex", + "decimal.Decimal" and subclasses) with the "n" type (ex: + "'{:n}'.format(1234)"), the function temporarily sets the + "LC_CTYPE" locale to the "LC_NUMERIC" locale to decode + "decimal_point" and "thousands_sep" fields of "localeconv()" if + they are non-ASCII or longer than 1 byte, and the "LC_NUMERIC" + locale is different than the "LC_CTYPE" locale. This temporary + change affects other threads. + + Changed in version 3.7: When formatting a number with the "n" type, + the function sets temporarily the "LC_CTYPE" locale to the + "LC_NUMERIC" locale in some cases. + +str.format_map(mapping, /) + + Similar to "str.format(**mapping)", except that "mapping" is used + directly and not copied to a "dict". This is useful if for example + "mapping" is a dict subclass: + + >>> class Default(dict): + ... def __missing__(self, key): + ... return key + ... + >>> '{name} was born in {country}'.format_map(Default(name='Guido')) + 'Guido was born in country' + + Added in version 3.2. + +str.index(sub[, start[, end]]) + + Like "find()", but raise "ValueError" when the substring is not + found. For example: + + >>> 'spam, spam, spam'.index('spam') + 0 + >>> 'spam, spam, spam'.index('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.index('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also "rindex()". + +str.isalnum() + + Return "True" if all characters in the string are alphanumeric and + there is at least one character, "False" otherwise. A character + "c" is alphanumeric if one of the following returns "True": + "c.isalpha()", "c.isdecimal()", "c.isdigit()", or "c.isnumeric()". + +str.isalpha() + + Return "True" if all characters in the string are alphabetic and + there is at least one character, "False" otherwise. Alphabetic + characters are those characters defined in the Unicode character + database as “Letter”, i.e., those with general category property + being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note that this is + different from the Alphabetic property defined in the section 4.10 + ‘Letters, Alphabetic, and Ideographic’ of the Unicode Standard. For + example: + + >>> 'Letters and spaces'.isalpha() + False + >>> 'LettersOnly'.isalpha() + True + >>> 'µ'.isalpha() # non-ASCII characters can be considered alphabetical too + True + + See Unicode Properties. + +str.isascii() + + Return "True" if the string is empty or all characters in the + string are ASCII, "False" otherwise. ASCII characters have code + points in the range U+0000-U+007F. For example: + + >>> 'ASCII characters'.isascii() + True + >>> 'µ'.isascii() + False + + Added in version 3.7. + +str.isdecimal() + + Return "True" if all characters in the string are decimal + characters and there is at least one character, "False" otherwise. + Decimal characters are those that can be used to form numbers in + base 10, such as U+0660, ARABIC-INDIC DIGIT ZERO. Formally a + decimal character is a character in the Unicode General Category + “Nd”. For example: + + >>> '0123456789'.isdecimal() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdecimal() # Arabic-Indic digits zero to nine + True + >>> 'alphabetic'.isdecimal() + False + +str.isdigit() + + Return "True" if all characters in the string are digits and there + is at least one character, "False" otherwise. Digits include + decimal characters and digits that need special handling, such as + the compatibility superscript digits. This covers digits which + cannot be used to form numbers in base 10, like the Kharosthi + numbers. Formally, a digit is a character that has the property + value Numeric_Type=Digit or Numeric_Type=Decimal. + +str.isidentifier() + + Return "True" if the string is a valid identifier according to the + language definition, section Names (identifiers and keywords). + + "keyword.iskeyword()" can be used to test whether string "s" is a + reserved identifier, such as "def" and "class". + + Example: + + >>> from keyword import iskeyword + + >>> 'hello'.isidentifier(), iskeyword('hello') + (True, False) + >>> 'def'.isidentifier(), iskeyword('def') + (True, True) + +str.islower() + + Return "True" if all cased characters [4] in the string are + lowercase and there is at least one cased character, "False" + otherwise. + +str.isnumeric() + + Return "True" if all characters in the string are numeric + characters, and there is at least one character, "False" otherwise. + Numeric characters include digit characters, and all characters + that have the Unicode numeric value property, e.g. U+2155, VULGAR + FRACTION ONE FIFTH. Formally, numeric characters are those with + the property value Numeric_Type=Digit, Numeric_Type=Decimal or + Numeric_Type=Numeric. For example: + + >>> '0123456789'.isnumeric() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + True + >>> '⅕'.isnumeric() # Vulgar fraction one fifth + True + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also "isdecimal()" and "isdigit()". Numeric characters are a + superset of decimal numbers. + +str.isprintable() + + Return "True" if all characters in the string are printable, + "False" if it contains at least one non-printable character. + + Here “printable” means the character is suitable for "repr()" to + use in its output; “non-printable” means that "repr()" on built-in + types will hex-escape the character. It has no bearing on the + handling of strings written to "sys.stdout" or "sys.stderr". + + The printable characters are those which in the Unicode character + database (see "unicodedata") have a general category in group + Letter, Mark, Number, Punctuation, or Symbol (L, M, N, P, or S); + plus the ASCII space 0x20. Nonprintable characters are those in + group Separator or Other (Z or C), except the ASCII space. + + For example: + + >>> ''.isprintable(), ' '.isprintable() + (True, True) + >>> '\t'.isprintable(), '\n'.isprintable() + (False, False) + +str.isspace() + + Return "True" if there are only whitespace characters in the string + and there is at least one character, "False" otherwise. + + A character is *whitespace* if in the Unicode character database + (see "unicodedata"), either its general category is "Zs" + (“Separator, space”), or its bidirectional class is one of "WS", + "B", or "S". + +str.istitle() + + Return "True" if the string is a titlecased string and there is at + least one character, for example uppercase characters may only + follow uncased characters and lowercase characters only cased ones. + Return "False" otherwise. + + For example: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also "title()". + +str.isupper() + + Return "True" if all cased characters [4] in the string are + uppercase and there is at least one cased character, "False" + otherwise. + + >>> 'BANANA'.isupper() + True + >>> 'banana'.isupper() + False + >>> 'baNana'.isupper() + False + >>> ' '.isupper() + False + +str.join(iterable, /) + + Return a string which is the concatenation of the strings in + *iterable*. A "TypeError" will be raised if there are any non- + string values in *iterable*, including "bytes" objects. The + separator between elements is the string providing this method. For + example: + + >>> ', '.join(['spam', 'spam', 'spam']) + 'spam, spam, spam' + >>> '-'.join('Python') + 'P-y-t-h-o-n' + + See also "split()". + +str.ljust(width, fillchar=' ', /) + + Return the string left justified in a string of length *width*. + Padding is done using the specified *fillchar* (default is an ASCII + space). The original string is returned if *width* is less than or + equal to "len(s)". + + For example: + + >>> 'Python'.ljust(10) + 'Python ' + >>> 'Python'.ljust(10, '.') + 'Python....' + >>> 'Monty Python'.ljust(10, '.') + 'Monty Python' + + See also "rjust()". + +str.lower() + + Return a copy of the string with all the cased characters [4] + converted to lowercase. For example: + + >>> 'Lower Method Example'.lower() + 'lower method example' + + The lowercasing algorithm used is described in section 3.13 + ‘Default Case Folding’ of the Unicode Standard. + +str.lstrip(chars=None, /) + + Return a copy of the string with leading characters removed. The + *chars* argument is a string specifying the set of characters to be + removed. If omitted or "None", the *chars* argument defaults to + removing whitespace. The *chars* argument is not a prefix; rather, + all combinations of its values are stripped: + + >>> ' spacious '.lstrip() + 'spacious ' + >>> 'www.example.com'.lstrip('cmowz.') + 'example.com' + + See "str.removeprefix()" for a method that will remove a single + prefix string rather than all of a set of characters. For example: + + >>> 'Arthur: three!'.lstrip('Arthur: ') + 'ee!' + >>> 'Arthur: three!'.removeprefix('Arthur: ') + 'three!' + +static str.maketrans(dict, /) +static str.maketrans(from, to, remove='', /) + + This static method returns a translation table usable for + "str.translate()". + + If there is only one argument, it must be a dictionary mapping + Unicode ordinals (integers) or characters (strings of length 1) to + Unicode ordinals, strings (of arbitrary lengths) or "None". + Character keys will then be converted to ordinals. + + If there are two arguments, they must be strings of equal length, + and in the resulting dictionary, each character in *from* will be + mapped to the character at the same position in *to*. If there is + a third argument, it must be a string, whose characters will be + mapped to "None" in the result. + +str.partition(sep, /) + + Split the string at the first occurrence of *sep*, and return a + 3-tuple containing the part before the separator, the separator + itself, and the part after the separator. If the separator is not + found, return a 3-tuple containing the string itself, followed by + two empty strings. + +str.removeprefix(prefix, /) + + If the string starts with the *prefix* string, return + "string[len(prefix):]". Otherwise, return a copy of the original + string: + + >>> 'TestHook'.removeprefix('Test') + 'Hook' + >>> 'BaseTestCase'.removeprefix('Test') + 'BaseTestCase' + + Added in version 3.9. + + See also "removesuffix()" and "startswith()". + +str.removesuffix(suffix, /) + + If the string ends with the *suffix* string and that *suffix* is + not empty, return "string[:-len(suffix)]". Otherwise, return a copy + of the original string: + + >>> 'MiscTests'.removesuffix('Tests') + 'Misc' + >>> 'TmpDirMixin'.removesuffix('Tests') + 'TmpDirMixin' + + Added in version 3.9. + + See also "removeprefix()" and "endswith()". + +str.replace(old, new, /, count=-1) + + Return a copy of the string with all occurrences of substring *old* + replaced by *new*. If *count* is given, only the first *count* + occurrences are replaced. If *count* is not specified or "-1", then + all occurrences are replaced. For example: + + >>> 'spam, spam, spam'.replace('spam', 'eggs') + 'eggs, eggs, eggs' + >>> 'spam, spam, spam'.replace('spam', 'eggs', 1) + 'eggs, spam, spam' + + Changed in version 3.13: *count* is now supported as a keyword + argument. + +str.rfind(sub[, start[, end]]) + + Return the highest index in the string where substring *sub* is + found, such that *sub* is contained within "s[start:end]". + Optional arguments *start* and *end* are interpreted as in slice + notation. Return "-1" on failure. For example: + + >>> 'spam, spam, spam'.rfind('sp') + 12 + >>> 'spam, spam, spam'.rfind('sp', 0, 10) + 6 + + See also "find()" and "rindex()". + +str.rindex(sub[, start[, end]]) + + Like "rfind()" but raises "ValueError" when the substring *sub* is + not found. For example: + + >>> 'spam, spam, spam'.rindex('spam') + 12 + >>> 'spam, spam, spam'.rindex('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.rindex('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also "index()" and "find()". + +str.rjust(width, fillchar=' ', /) + + Return the string right justified in a string of length *width*. + Padding is done using the specified *fillchar* (default is an ASCII + space). The original string is returned if *width* is less than or + equal to "len(s)". + +str.rpartition(sep, /) + + Split the string at the last occurrence of *sep*, and return a + 3-tuple containing the part before the separator, the separator + itself, and the part after the separator. If the separator is not + found, return a 3-tuple containing two empty strings, followed by + the string itself. + + For example: + + >>> 'Monty Python'.rpartition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".rpartition(' ') + ("Monty Python's Flying", ' ', 'Circus') + >>> 'Monty Python'.rpartition('-') + ('', '', 'Monty Python') + + See also "partition()". + +str.rsplit(sep=None, maxsplit=-1) + + Return a list of the words in the string, using *sep* as the + delimiter string. If *maxsplit* is given, at most *maxsplit* splits + are done, the *rightmost* ones. If *sep* is not specified or + "None", any whitespace string is a separator. Except for splitting + from the right, "rsplit()" behaves like "split()" which is + described in detail below. + +str.rstrip(chars=None, /) + + Return a copy of the string with trailing characters removed. The + *chars* argument is a string specifying the set of characters to be + removed. If omitted or "None", the *chars* argument defaults to + removing whitespace. The *chars* argument is not a suffix; rather, + all combinations of its values are stripped: + + >>> ' spacious '.rstrip() + ' spacious' + >>> 'mississippi'.rstrip('ipz') + 'mississ' + + See "str.removesuffix()" for a method that will remove a single + suffix string rather than all of a set of characters. For example: + + >>> 'Monty Python'.rstrip(' Python') + 'M' + >>> 'Monty Python'.removesuffix(' Python') + 'Monty' + +str.split(sep=None, maxsplit=-1) + + Return a list of the words in the string, using *sep* as the + delimiter string. If *maxsplit* is given, at most *maxsplit* + splits are done (thus, the list will have at most "maxsplit+1" + elements). If *maxsplit* is not specified or "-1", then there is + no limit on the number of splits (all possible splits are made). + + If *sep* is given, consecutive delimiters are not grouped together + and are deemed to delimit empty strings (for example, + "'1,,2'.split(',')" returns "['1', '', '2']"). The *sep* argument + may consist of multiple characters as a single delimiter (to split + with multiple delimiters, use "re.split()"). Splitting an empty + string with a specified separator returns "['']". + + For example: + + >>> '1,2,3'.split(',') + ['1', '2', '3'] + >>> '1,2,3'.split(',', maxsplit=1) + ['1', '2,3'] + >>> '1,2,,3,'.split(',') + ['1', '2', '', '3', ''] + >>> '1<>2<>3<4'.split('<>') + ['1', '2', '3<4'] + + If *sep* is not specified or is "None", a different splitting + algorithm is applied: runs of consecutive whitespace are regarded + as a single separator, and the result will contain no empty strings + at the start or end if the string has leading or trailing + whitespace. Consequently, splitting an empty string or a string + consisting of just whitespace with a "None" separator returns "[]". + + For example: + + >>> '1 2 3'.split() + ['1', '2', '3'] + >>> '1 2 3'.split(maxsplit=1) + ['1', '2 3'] + >>> ' 1 2 3 '.split() + ['1', '2', '3'] + + If *sep* is not specified or is "None" and *maxsplit* is "0", only + leading runs of consecutive whitespace are considered. + + For example: + + >>> "".split(None, 0) + [] + >>> " ".split(None, 0) + [] + >>> " foo ".split(maxsplit=0) + ['foo '] + + See also "join()". + +str.splitlines(keepends=False) + + Return a list of the lines in the string, breaking at line + boundaries. Line breaks are not included in the resulting list + unless *keepends* is given and true. + + This method splits on the following line boundaries. In + particular, the boundaries are a superset of *universal newlines*. + + +-------------------------+-------------------------------+ + | Representation | Description | + |=========================|===============================| + | "\n" | Line Feed | + +-------------------------+-------------------------------+ + | "\r" | Carriage Return | + +-------------------------+-------------------------------+ + | "\r\n" | Carriage Return + Line Feed | + +-------------------------+-------------------------------+ + | "\v" or "\x0b" | Line Tabulation | + +-------------------------+-------------------------------+ + | "\f" or "\x0c" | Form Feed | + +-------------------------+-------------------------------+ + | "\x1c" | File Separator | + +-------------------------+-------------------------------+ + | "\x1d" | Group Separator | + +-------------------------+-------------------------------+ + | "\x1e" | Record Separator | + +-------------------------+-------------------------------+ + | "\x85" | Next Line (C1 Control Code) | + +-------------------------+-------------------------------+ + | "\u2028" | Line Separator | + +-------------------------+-------------------------------+ + | "\u2029" | Paragraph Separator | + +-------------------------+-------------------------------+ + + Changed in version 3.2: "\v" and "\f" added to list of line + boundaries. + + For example: + + >>> 'ab c\n\nde fg\rkl\r\n'.splitlines() + ['ab c', '', 'de fg', 'kl'] + >>> 'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) + ['ab c\n', '\n', 'de fg\r', 'kl\r\n'] + + Unlike "split()" when a delimiter string *sep* is given, this + method returns an empty list for the empty string, and a terminal + line break does not result in an extra line: + + >>> "".splitlines() + [] + >>> "One line\n".splitlines() + ['One line'] + + For comparison, "split('\n')" gives: + + >>> ''.split('\n') + [''] + >>> 'Two lines\n'.split('\n') + ['Two lines', ''] + +str.startswith(prefix[, start[, end]]) + + Return "True" if string starts with the *prefix*, otherwise return + "False". *prefix* can also be a tuple of prefixes to look for. + With optional *start*, test string beginning at that position. + With optional *end*, stop comparing string at that position. + +str.strip(chars=None, /) + + Return a copy of the string with the leading and trailing + characters removed. The *chars* argument is a string specifying the + set of characters to be removed. If omitted or "None", the *chars* + argument defaults to removing whitespace. The *chars* argument is + not a prefix or suffix; rather, all combinations of its values are + stripped: + + >>> ' spacious '.strip() + 'spacious' + >>> 'www.example.com'.strip('cmowz.') + 'example' + + The outermost leading and trailing *chars* argument values are + stripped from the string. Characters are removed from the leading + end until reaching a string character that is not contained in the + set of characters in *chars*. A similar action takes place on the + trailing end. For example: + + >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' + >>> comment_string.strip('.#! ') + 'Section 3.2.1 Issue #32' + +str.swapcase() + + Return a copy of the string with uppercase characters converted to + lowercase and vice versa. Note that it is not necessarily true that + "s.swapcase().swapcase() == s". + +str.title() + + Return a titlecased version of the string where words start with an + uppercase character and the remaining characters are lowercase. + + For example: + + >>> 'Hello world'.title() + 'Hello World' + + The algorithm uses a simple language-independent definition of a + word as groups of consecutive letters. The definition works in + many contexts but it means that apostrophes in contractions and + possessives form word boundaries, which may not be the desired + result: + + >>> "they're bill's friends from the UK".title() + "They'Re Bill'S Friends From The Uk" + + The "string.capwords()" function does not have this problem, as it + splits words on spaces only. + + Alternatively, a workaround for apostrophes can be constructed + using regular expressions: + + >>> import re + >>> def titlecase(s): + ... return re.sub(r"[A-Za-z]+('[A-Za-z]+)?", + ... lambda mo: mo.group(0).capitalize(), + ... s) + ... + >>> titlecase("they're bill's friends.") + "They're Bill's Friends." + + See also "istitle()". + +str.translate(table, /) + + Return a copy of the string in which each character has been mapped + through the given translation table. The table must be an object + that implements indexing via "__getitem__()", typically a *mapping* + or *sequence*. When indexed by a Unicode ordinal (an integer), the + table object can do any of the following: return a Unicode ordinal + or a string, to map the character to one or more other characters; + return "None", to delete the character from the return string; or + raise a "LookupError" exception, to map the character to itself. + + You can use "str.maketrans()" to create a translation map from + character-to-character mappings in different formats. + + See also the "codecs" module for a more flexible approach to custom + character mappings. + +str.upper() + + Return a copy of the string with all the cased characters [4] + converted to uppercase. Note that "s.upper().isupper()" might be + "False" if "s" contains uncased characters or if the Unicode + category of the resulting character(s) is not “Lu” (Letter, + uppercase), but e.g. “Lt” (Letter, titlecase). + + The uppercasing algorithm used is described in section 3.13 + ‘Default Case Folding’ of the Unicode Standard. + +str.zfill(width, /) + + Return a copy of the string left filled with ASCII "'0'" digits to + make a string of length *width*. A leading sign prefix + ("'+'"/"'-'") is handled by inserting the padding *after* the sign + character rather than before. The original string is returned if + *width* is less than or equal to "len(s)". + + For example: + + >>> "42".zfill(5) + '00042' + >>> "-42".zfill(5) + '-0042' +''', + 'strings': '''String and Bytes literals +************************* + +String literals are text enclosed in single quotes ("'") or double +quotes ("""). For example: + + "spam" + 'eggs' + +The quote used to start the literal also terminates it, so a string +literal can only contain the other quote (except with escape +sequences, see below). For example: + + 'Say "Hello", please.' + "Don't do that!" + +Except for this limitation, the choice of quote character ("'" or """) +does not affect how the literal is parsed. + +Inside a string literal, the backslash ("\\") character introduces an +*escape sequence*, which has special meaning depending on the +character after the backslash. For example, "\\"" denotes the double +quote character, and does *not* end the string: + + >>> print("Say \\"Hello\\" to everyone!") + Say "Hello" to everyone! + +See escape sequences below for a full list of such sequences, and more +details. + + +Triple-quoted strings +===================== + +Strings can also be enclosed in matching groups of three single or +double quotes. These are generally referred to as *triple-quoted +strings*: + + """This is a triple-quoted string.""" + +In triple-quoted literals, unescaped quotes are allowed (and are +retained), except that three unescaped quotes in a row terminate the +literal, if they are of the same kind ("'" or """) used at the start: + + """This string has "quotes" inside.""" + +Unescaped newlines are also allowed and retained: + + \'\'\'This triple-quoted string + continues on the next line.\'\'\' + + +String prefixes +=============== + +String literals can have an optional *prefix* that influences how the +content of the literal is parsed, for example: + + b"data" + f'{result=}' + +The allowed prefixes are: + +* "b": Bytes literal + +* "r": Raw string + +* "f": Formatted string literal (“f-string”) + +* "t": Template string literal (“t-string”) + +* "u": No effect (allowed for backwards compatibility) + +See the linked sections for details on each type. + +Prefixes are case-insensitive (for example, ‘"B"’ works the same as +‘"b"’). The ‘"r"’ prefix can be combined with ‘"f"’, ‘"t"’ or ‘"b"’, +so ‘"fr"’, ‘"rf"’, ‘"tr"’, ‘"rt"’, ‘"br"’, and ‘"rb"’ are also valid +prefixes. + +Added in version 3.3: The "'rb'" prefix of raw bytes literals has been +added as a synonym of "'br'".Support for the unicode legacy literal +("u'value'") was reintroduced to simplify the maintenance of dual +Python 2.x and 3.x codebases. See **PEP 414** for more information. + + +Formal grammar +============== + +String literals, except “f-strings” and “t-strings”, are described by +the following lexical definitions. + +These definitions use negative lookaheads ("!") to indicate that an +ending quote ends the literal. + + STRING: [stringprefix] (stringcontent) + stringprefix: <("r" | "u" | "b" | "br" | "rb"), case-insensitive> + stringcontent: + | "\'\'\'" ( !"\'\'\'" longstringitem)* "\'\'\'" + | '"""' ( !'"""' longstringitem)* '"""' + | "'" ( !"'" stringitem)* "'" + | '"' ( !'"' stringitem)* '"' + stringitem: stringchar | stringescapeseq + stringchar: + longstringitem: stringitem | newline + stringescapeseq: "\\" + +Note that as in all lexical definitions, whitespace is significant. In +particular, the prefix (if any) must be immediately followed by the +starting quote. + + +Escape sequences +================ + +Unless an ‘"r"’ or ‘"R"’ prefix is present, escape sequences in string +and bytes literals are interpreted according to rules similar to those +used by Standard C. The recognized escape sequences are: + ++----------------------------------------------------+----------------------------------------------------+ +| Escape Sequence | Meaning | +|====================================================|====================================================| +| "\\" | Ignored end of line | ++----------------------------------------------------+----------------------------------------------------+ +| "\\\\" | Backslash | ++----------------------------------------------------+----------------------------------------------------+ +| "\\'" | Single quote | ++----------------------------------------------------+----------------------------------------------------+ +| "\\"" | Double quote | ++----------------------------------------------------+----------------------------------------------------+ +| "\\a" | ASCII Bell (BEL) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\b" | ASCII Backspace (BS) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\f" | ASCII Formfeed (FF) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\n" | ASCII Linefeed (LF) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\r" | ASCII Carriage Return (CR) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\t" | ASCII Horizontal Tab (TAB) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\v" | ASCII Vertical Tab (VT) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\*ooo*" | Octal character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\x*hh*" | Hexadecimal character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\N{*name*}" | Named Unicode character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\u*xxxx*" | Hexadecimal Unicode character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\U*xxxxxxxx*" | Hexadecimal Unicode character | ++----------------------------------------------------+----------------------------------------------------+ + + +Ignored end of line +------------------- + +A backslash can be added at the end of a line to ignore the newline: + + >>> 'This string will not include \\ + ... backslashes or newline characters.' + 'This string will not include backslashes or newline characters.' + +The same result can be achieved using triple-quoted strings, or +parentheses and string literal concatenation. + + +Escaped characters +------------------ + +To include a backslash in a non-raw Python string literal, it must be +doubled. The "\\\\" escape sequence denotes a single backslash +character: + + >>> print('C:\\\\Program Files') + C:\\Program Files + +Similarly, the "\\'" and "\\"" sequences denote the single and double +quote character, respectively: + + >>> print('\\' and \\"') + ' and " + + +Octal character +--------------- + +The sequence "\\*ooo*" denotes a *character* with the octal (base 8) +value *ooo*: + + >>> '\\120' + 'P' + +Up to three octal digits (0 through 7) are accepted. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given +value. + +Changed in version 3.11: Octal escapes with value larger than "0o377" +(255) produce a "DeprecationWarning". + +Changed in version 3.12: Octal escapes with value larger than "0o377" +(255) produce a "SyntaxWarning". In a future Python version they will +raise a "SyntaxError". + + +Hexadecimal character +--------------------- + +The sequence "\\x*hh*" denotes a *character* with the hex (base 16) +value *hh*: + + >>> '\\x50' + 'P' + +Unlike in Standard C, exactly two hex digits are required. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given +value. + + +Named Unicode character +----------------------- + +The sequence "\\N{*name*}" denotes a Unicode character with the given +*name*: + + >>> '\\N{LATIN CAPITAL LETTER P}' + 'P' + >>> '\\N{SNAKE}' + '🐍' + +This sequence cannot appear in bytes literals. + +Changed in version 3.3: Support for name aliases has been added. + + +Hexadecimal Unicode characters +------------------------------ + +These sequences "\\u*xxxx*" and "\\U*xxxxxxxx*" denote the Unicode +character with the given hex (base 16) value. Exactly four digits are +required for "\\u"; exactly eight digits are required for "\\U". The +latter can encode any Unicode character. + + >>> '\\u1234' + 'ሴ' + >>> '\\U0001f40d' + '🐍' + +These sequences cannot appear in bytes literals. + + +Unrecognized escape sequences +----------------------------- + +Unlike in Standard C, all unrecognized escape sequences are left in +the string unchanged, that is, *the backslash is left in the result*: + + >>> print('\\q') + \\q + >>> list('\\q') + ['\\\\', 'q'] + +Note that for bytes literals, the escape sequences only recognized in +string literals ("\\N...", "\\u...", "\\U...") fall into the category of +unrecognized escapes. + +Changed in version 3.6: Unrecognized escape sequences produce a +"DeprecationWarning". + +Changed in version 3.12: Unrecognized escape sequences produce a +"SyntaxWarning". In a future Python version they will raise a +"SyntaxError". + + +Bytes literals +============== + +*Bytes literals* are always prefixed with ‘"b"’ or ‘"B"’; they produce +an instance of the "bytes" type instead of the "str" type. They may +only contain ASCII characters; bytes with a numeric value of 128 or +greater must be expressed with escape sequences (typically Hexadecimal +character or Octal character): + + >>> b'\\x89PNG\\r\\n\\x1a\\n' + b'\\x89PNG\\r\\n\\x1a\\n' + >>> list(b'\\x89PNG\\r\\n\\x1a\\n') + [137, 80, 78, 71, 13, 10, 26, 10] + +Similarly, a zero byte must be expressed using an escape sequence +(typically "\\0" or "\\x00"). + + +Raw string literals +=================== + +Both string and bytes literals may optionally be prefixed with a +letter ‘"r"’ or ‘"R"’; such constructs are called *raw string +literals* and *raw bytes literals* respectively and treat backslashes +as literal characters. As a result, in raw string literals, escape +sequences are not treated specially: + + >>> r'\\d{4}-\\d{2}-\\d{2}' + '\\\\d{4}-\\\\d{2}-\\\\d{2}' + +Even in a raw literal, quotes can be escaped with a backslash, but the +backslash remains in the result; for example, "r"\\""" is a valid +string literal consisting of two characters: a backslash and a double +quote; "r"\\"" is not a valid string literal (even a raw string cannot +end in an odd number of backslashes). Specifically, *a raw literal +cannot end in a single backslash* (since the backslash would escape +the following quote character). Note also that a single backslash +followed by a newline is interpreted as those two characters as part +of the literal, *not* as a line continuation. + + +f-strings +========= + +Added in version 3.6. + +Changed in version 3.7: The "await" and "async for" can be used in +expressions within f-strings. + +Changed in version 3.8: Added the debug specifier ("=") + +Changed in version 3.12: Many restrictions on expressions within +f-strings have been removed. Notably, nested strings, comments, and +backslashes are now permitted. + +A *formatted string literal* or *f-string* is a string literal that is +prefixed with ‘"f"’ or ‘"F"’. Unlike other string literals, f-strings +do not have a constant value. They may contain *replacement fields* +delimited by curly braces "{}". Replacement fields contain expressions +which are evaluated at run time. For example: + + >>> who = 'nobody' + >>> nationality = 'Spanish' + >>> f'{who.title()} expects the {nationality} Inquisition!' + 'Nobody expects the Spanish Inquisition!' + +Any doubled curly braces ("{{" or "}}") outside replacement fields are +replaced with the corresponding single curly brace: + + >>> print(f'{{...}}') + {...} + +Other characters outside replacement fields are treated like in +ordinary string literals. This means that escape sequences are decoded +(except when a literal is also marked as a raw string), and newlines +are possible in triple-quoted f-strings: + + >>> name = 'Galahad' + >>> favorite_color = 'blue' + >>> print(f'{name}:\\t{favorite_color}') + Galahad: blue + >>> print(rf"C:\\Users\\{name}") + C:\\Users\\Galahad + >>> print(f\'\'\'Three shall be the number of the counting + ... and the number of the counting shall be three.\'\'\') + Three shall be the number of the counting + and the number of the counting shall be three. + +Expressions in formatted string literals are treated like regular +Python expressions. Each expression is evaluated in the context where +the formatted string literal appears, in order from left to right. An +empty expression is not allowed, and both "lambda" and assignment +expressions ":=" must be surrounded by explicit parentheses: + + >>> f'{(half := 1/2)}, {half * 42}' + '0.5, 21.0' + +Reusing the outer f-string quoting type inside a replacement field is +permitted: + + >>> a = dict(x=2) + >>> f"abc {a["x"]} def" + 'abc 2 def' + +Backslashes are also allowed in replacement fields and are evaluated +the same way as in any other context: + + >>> a = ["a", "b", "c"] + >>> print(f"List a contains:\\n{"\\n".join(a)}") + List a contains: + a + b + c + +It is possible to nest f-strings: + + >>> name = 'world' + >>> f'Repeated:{f' hello {name}' * 3}' + 'Repeated: hello world hello world hello world' + +Portable Python programs should not use more than 5 levels of nesting. + +**CPython implementation detail:** CPython does not limit nesting of +f-strings. + +Replacement expressions can contain newlines in both single-quoted and +triple-quoted f-strings and they can contain comments. Everything that +comes after a "#" inside a replacement field is a comment (even +closing braces and quotes). This means that replacement fields with +comments must be closed in a different line: + + >>> a = 2 + >>> f"abc{a # This comment }" continues until the end of the line + ... + 3}" + 'abc5' + +After the expression, replacement fields may optionally contain: + +* a *debug specifier* – an equal sign ("="), optionally surrounded by + whitespace on one or both sides; + +* a *conversion specifier* – "!s", "!r" or "!a"; and/or + +* a *format specifier* prefixed with a colon (":"). + +See the Standard Library section on f-strings for details on how these +fields are evaluated. + +As that section explains, *format specifiers* are passed as the second +argument to the "format()" function to format a replacement field +value. For example, they can be used to specify a field width and +padding characters using the Format Specification Mini-Language: + + >>> number = 14.3 + >>> f'{number:20.7f}' + ' 14.3000000' + +Top-level format specifiers may include nested replacement fields: + + >>> field_size = 20 + >>> precision = 7 + >>> f'{number:{field_size}.{precision}f}' + ' 14.3000000' + +These nested fields may include their own conversion fields and format +specifiers: + + >>> number = 3 + >>> f'{number:{field_size}}' + ' 3' + >>> f'{number:{field_size:05}}' + '00000000000000000003' + +However, these nested fields may not include more deeply nested +replacement fields. + +Formatted string literals cannot be used as *docstrings*, even if they +do not include expressions: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> print(foo.__doc__) + None + +See also: + + * **PEP 498** – Literal String Interpolation + + * **PEP 701** – Syntactic formalization of f-strings + + * "str.format()", which uses a related format string mechanism. + + +t-strings +========= + +Added in version 3.14. + +A *template string literal* or *t-string* is a string literal that is +prefixed with ‘"t"’ or ‘"T"’. These strings follow the same syntax +rules as formatted string literals. For differences in evaluation +rules, see the Standard Library section on t-strings + + +Formal grammar for f-strings +============================ + +F-strings are handled partly by the *lexical analyzer*, which produces +the tokens "FSTRING_START", "FSTRING_MIDDLE" and "FSTRING_END", and +partly by the parser, which handles expressions in the replacement +field. The exact way the work is split is a CPython implementation +detail. + +Correspondingly, the f-string grammar is a mix of lexical and +syntactic definitions. + +Whitespace is significant in these situations: + +* There may be no whitespace in "FSTRING_START" (between the prefix + and quote). + +* Whitespace in "FSTRING_MIDDLE" is part of the literal string + contents. + +* In "fstring_replacement_field", if "f_debug_specifier" is present, + all whitespace after the opening brace until the + "f_debug_specifier", as well as whitespace immediately following + "f_debug_specifier", is retained as part of the expression. + + **CPython implementation detail:** The expression is not handled in + the tokenization phase; it is retrieved from the source code using + locations of the "{" token and the token after "=". + +The "FSTRING_MIDDLE" definition uses negative lookaheads ("!") to +indicate special characters (backslash, newline, "{", "}") and +sequences ("f_quote"). + + fstring: FSTRING_START fstring_middle* FSTRING_END + + FSTRING_START: fstringprefix ("'" | '"' | "\'\'\'" | '"""') + FSTRING_END: f_quote + fstringprefix: <("f" | "fr" | "rf"), case-insensitive> + f_debug_specifier: '=' + f_quote: + + fstring_middle: + | fstring_replacement_field + | FSTRING_MIDDLE + FSTRING_MIDDLE: + | (!"\\" !newline !'{' !'}' !f_quote) source_character + | stringescapeseq + | "{{" + | "}}" + | + fstring_replacement_field: + | '{' f_expression [f_debug_specifier] [fstring_conversion] + [fstring_full_format_spec] '}' + fstring_conversion: + | "!" ("s" | "r" | "a") + fstring_full_format_spec: + | ':' fstring_format_spec* + fstring_format_spec: + | FSTRING_MIDDLE + | fstring_replacement_field + f_expression: + | ','.(conditional_expression | "*" or_expr)+ [","] + | yield_expression + +Note: + + In the above grammar snippet, the "f_quote" and "FSTRING_MIDDLE" + rules are context-sensitive – they depend on the contents of + "FSTRING_START" of the nearest enclosing "fstring".Constructing a + more traditional formal grammar from this template is left as an + exercise for the reader. + +The grammar for t-strings is identical to the one for f-strings, with +*t* instead of *f* at the beginning of rule and token names and in the +prefix. + + tstring: TSTRING_START tstring_middle* TSTRING_END + + +''', + 'subscriptions': r'''Subscriptions +************* + +The subscription of an instance of a container class will generally +select an element from the container. The subscription of a *generic +class* will generally return a GenericAlias object. + + subscription: primary "[" flexible_expression_list "]" + +When an object is subscripted, the interpreter will evaluate the +primary and the expression list. + +The primary must evaluate to an object that supports subscription. An +object may support subscription through defining one or both of +"__getitem__()" and "__class_getitem__()". When the primary is +subscripted, the evaluated result of the expression list will be +passed to one of these methods. For more details on when +"__class_getitem__" is called instead of "__getitem__", see +__class_getitem__ versus __getitem__. + +If the expression list contains at least one comma, or if any of the +expressions are starred, the expression list will evaluate to a +"tuple" containing the items of the expression list. Otherwise, the +expression list will evaluate to the value of the list’s sole member. + +Changed in version 3.11: Expressions in an expression list may be +starred. See **PEP 646**. + +For built-in objects, there are two types of objects that support +subscription via "__getitem__()": + +1. Mappings. If the primary is a *mapping*, the expression list must + evaluate to an object whose value is one of the keys of the + mapping, and the subscription selects the value in the mapping that + corresponds to that key. An example of a builtin mapping class is + the "dict" class. + +2. Sequences. If the primary is a *sequence*, the expression list must + evaluate to an "int" or a "slice" (as discussed in the following + section). Examples of builtin sequence classes include the "str", + "list" and "tuple" classes. + +The formal syntax makes no special provision for negative indices in +*sequences*. However, built-in sequences all provide a "__getitem__()" +method that interprets negative indices by adding the length of the +sequence to the index so that, for example, "x[-1]" selects the last +item of "x". The resulting value must be a nonnegative integer less +than the number of items in the sequence, and the subscription selects +the item whose index is that value (counting from zero). Since the +support for negative indices and slicing occurs in the object’s +"__getitem__()" method, subclasses overriding this method will need to +explicitly add that support. + +A "string" is a special kind of sequence whose items are *characters*. +A character is not a separate data type but a string of exactly one +character. +''', + 'truth': r'''Truth Value Testing +******************* + +Any object can be tested for truth value, for use in an "if" or +"while" condition or as operand of the Boolean operations below. + +By default, an object is considered true unless its class defines +either a "__bool__()" method that returns "False" or a "__len__()" +method that returns zero, when called with the object. [1] If one of +the methods raises an exception when called, the exception is +propagated and the object does not have a truth value (for example, +"NotImplemented"). Here are most of the built-in objects considered +false: + +* constants defined to be false: "None" and "False" + +* zero of any numeric type: "0", "0.0", "0j", "Decimal(0)", + "Fraction(0, 1)" + +* empty sequences and collections: "''", "()", "[]", "{}", "set()", + "range(0)" + +Operations and built-in functions that have a Boolean result always +return "0" or "False" for false and "1" or "True" for true, unless +otherwise stated. (Important exception: the Boolean operations "or" +and "and" always return one of their operands.) +''', + 'try': r'''The "try" statement +******************* + +The "try" statement specifies exception handlers and/or cleanup code +for a group of statements: + + try_stmt: try1_stmt | try2_stmt | try3_stmt + try1_stmt: "try" ":" suite + ("except" [expression ["as" identifier]] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try2_stmt: "try" ":" suite + ("except" "*" expression ["as" identifier] ":" suite)+ + ["else" ":" suite] + ["finally" ":" suite] + try3_stmt: "try" ":" suite + "finally" ":" suite + +Additional information on exceptions can be found in section +Exceptions, and information on using the "raise" statement to generate +exceptions may be found in section The raise statement. + +Changed in version 3.14: Support for optionally dropping grouping +parentheses when using multiple exception types. See **PEP 758**. + + +"except" clause +=============== + +The "except" clause(s) specify one or more exception handlers. When no +exception occurs in the "try" clause, no exception handler is +executed. When an exception occurs in the "try" suite, a search for an +exception handler is started. This search inspects the "except" +clauses in turn until one is found that matches the exception. An +expression-less "except" clause, if present, must be last; it matches +any exception. + +For an "except" clause with an expression, the expression must +evaluate to an exception type or a tuple of exception types. +Parentheses can be dropped if multiple exception types are provided +and the "as" clause is not used. The raised exception matches an +"except" clause whose expression evaluates to the class or a *non- +virtual base class* of the exception object, or to a tuple that +contains such a class. + +If no "except" clause matches the exception, the search for an +exception handler continues in the surrounding code and on the +invocation stack. [1] + +If the evaluation of an expression in the header of an "except" clause +raises an exception, the original search for a handler is canceled and +a search starts for the new exception in the surrounding code and on +the call stack (it is treated as if the entire "try" statement raised +the exception). + +When a matching "except" clause is found, the exception is assigned to +the target specified after the "as" keyword in that "except" clause, +if present, and the "except" clause’s suite is executed. All "except" +clauses must have an executable block. When the end of this block is +reached, execution continues normally after the entire "try" +statement. (This means that if two nested handlers exist for the same +exception, and the exception occurs in the "try" clause of the inner +handler, the outer handler will not handle the exception.) + +When an exception has been assigned using "as target", it is cleared +at the end of the "except" clause. This is as if + + except E as N: + foo + +was translated to + + except E as N: + try: + foo + finally: + del N + +This means the exception must be assigned to a different name to be +able to refer to it after the "except" clause. Exceptions are cleared +because with the traceback attached to them, they form a reference +cycle with the stack frame, keeping all locals in that frame alive +until the next garbage collection occurs. + +Before an "except" clause’s suite is executed, the exception is stored +in the "sys" module, where it can be accessed from within the body of +the "except" clause by calling "sys.exception()". When leaving an +exception handler, the exception stored in the "sys" module is reset +to its previous value: + + >>> print(sys.exception()) + None + >>> try: + ... raise TypeError + ... except: + ... print(repr(sys.exception())) + ... try: + ... raise ValueError + ... except: + ... print(repr(sys.exception())) + ... print(repr(sys.exception())) + ... + TypeError() + ValueError() + TypeError() + >>> print(sys.exception()) + None + + +"except*" clause +================ + +The "except*" clause(s) specify one or more handlers for groups of +exceptions ("BaseExceptionGroup" instances). A "try" statement can +have either "except" or "except*" clauses, but not both. The exception +type for matching is mandatory in the case of "except*", so "except*:" +is a syntax error. The type is interpreted as in the case of "except", +but matching is performed on the exceptions contained in the group +that is being handled. An "TypeError" is raised if a matching type is +a subclass of "BaseExceptionGroup", because that would have ambiguous +semantics. + +When an exception group is raised in the try block, each "except*" +clause splits (see "split()") it into the subgroups of matching and +non-matching exceptions. If the matching subgroup is not empty, it +becomes the handled exception (the value returned from +"sys.exception()") and assigned to the target of the "except*" clause +(if there is one). Then, the body of the "except*" clause executes. If +the non-matching subgroup is not empty, it is processed by the next +"except*" in the same manner. This continues until all exceptions in +the group have been matched, or the last "except*" clause has run. + +After all "except*" clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from +within "except*" clauses. This merged exception group propagates on.: + + >>> try: + ... raise ExceptionGroup("eg", + ... [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + ... except* TypeError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... except* OSError as e: + ... print(f'caught {type(e)} with nested {e.exceptions}') + ... + caught with nested (TypeError(2),) + caught with nested (OSError(3), OSError(4)) + + Exception Group Traceback (most recent call last): + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) + +-+---------------- 1 ---------------- + | ValueError: 1 + +------------------------------------ + +If the exception raised from the "try" block is not an exception group +and its type matches one of the "except*" clauses, it is caught and +wrapped by an exception group with an empty message string. This +ensures that the type of the target "e" is consistently +"BaseExceptionGroup": + + >>> try: + ... raise BlockingIOError + ... except* BlockingIOError as e: + ... print(repr(e)) + ... + ExceptionGroup('', (BlockingIOError(),)) + +"break", "continue" and "return" cannot appear in an "except*" clause. + + +"else" clause +============= + +The optional "else" clause is executed if the control flow leaves the +"try" suite, no exception was raised, and no "return", "continue", or +"break" statement was executed. Exceptions in the "else" clause are +not handled by the preceding "except" clauses. + + +"finally" clause +================ + +If "finally" is present, it specifies a ‘cleanup’ handler. The "try" +clause is executed, including any "except" and "else" clauses. If an +exception occurs in any of the clauses and is not handled, the +exception is temporarily saved. The "finally" clause is executed. If +there is a saved exception it is re-raised at the end of the "finally" +clause. If the "finally" clause raises another exception, the saved +exception is set as the context of the new exception. If the "finally" +clause executes a "return", "break" or "continue" statement, the saved +exception is discarded. For example, this function returns 42. + + def f(): + try: + 1/0 + finally: + return 42 + +The exception information is not available to the program during +execution of the "finally" clause. + +When a "return", "break" or "continue" statement is executed in the +"try" suite of a "try"…"finally" statement, the "finally" clause is +also executed ‘on the way out.’ + +The return value of a function is determined by the last "return" +statement executed. Since the "finally" clause always executes, a +"return" statement executed in the "finally" clause will always be the +last one executed. The following function returns ‘finally’. + + def foo(): + try: + return 'try' + finally: + return 'finally' + +Changed in version 3.8: Prior to Python 3.8, a "continue" statement +was illegal in the "finally" clause due to a problem with the +implementation. + +Changed in version 3.14: The compiler emits a "SyntaxWarning" when a +"return", "break" or "continue" appears in a "finally" block (see +**PEP 765**). +''', + 'types': r'''The standard type hierarchy +*************************** + +Below is a list of the types that are built into Python. Extension +modules (written in C, Java, or other languages, depending on the +implementation) can define additional types. Future versions of +Python may add types to the type hierarchy (e.g., rational numbers, +efficiently stored arrays of integers, etc.), although such additions +will often be provided via the standard library instead. + +Some of the type descriptions below contain a paragraph listing +‘special attributes.’ These are attributes that provide access to the +implementation and are not intended for general use. Their definition +may change in the future. + + +None +==== + +This type has a single value. There is a single object with this +value. This object is accessed through the built-in name "None". It is +used to signify the absence of a value in many situations, e.g., it is +returned from functions that don’t explicitly return anything. Its +truth value is false. + + +NotImplemented +============== + +This type has a single value. There is a single object with this +value. This object is accessed through the built-in name +"NotImplemented". Numeric methods and rich comparison methods should +return this value if they do not implement the operation for the +operands provided. (The interpreter will then try the reflected +operation, or some other fallback, depending on the operator.) It +should not be evaluated in a boolean context. + +See Implementing the arithmetic operations for more details. + +Changed in version 3.9: Evaluating "NotImplemented" in a boolean +context was deprecated. + +Changed in version 3.14: Evaluating "NotImplemented" in a boolean +context now raises a "TypeError". It previously evaluated to "True" +and emitted a "DeprecationWarning" since Python 3.9. + + +Ellipsis +======== + +This type has a single value. There is a single object with this +value. This object is accessed through the literal "..." or the built- +in name "Ellipsis". Its truth value is true. + + +"numbers.Number" +================ + +These are created by numeric literals and returned as results by +arithmetic operators and arithmetic built-in functions. Numeric +objects are immutable; once created their value never changes. Python +numbers are of course strongly related to mathematical numbers, but +subject to the limitations of numerical representation in computers. + +The string representations of the numeric classes, computed by +"__repr__()" and "__str__()", have the following properties: + +* They are valid numeric literals which, when passed to their class + constructor, produce an object having the value of the original + numeric. + +* The representation is in base 10, when possible. + +* Leading zeros, possibly excepting a single zero before a decimal + point, are not shown. + +* Trailing zeros, possibly excepting a single zero after a decimal + point, are not shown. + +* A sign is shown only when the number is negative. + +Python distinguishes between integers, floating-point numbers, and +complex numbers: + + +"numbers.Integral" +------------------ + +These represent elements from the mathematical set of integers +(positive and negative). + +Note: + + The rules for integer representation are intended to give the most + meaningful interpretation of shift and mask operations involving + negative integers. + +There are two types of integers: + +Integers ("int") + These represent numbers in an unlimited range, subject to available + (virtual) memory only. For the purpose of shift and mask + operations, a binary representation is assumed, and negative + numbers are represented in a variant of 2’s complement which gives + the illusion of an infinite string of sign bits extending to the + left. + +Booleans ("bool") + These represent the truth values False and True. The two objects + representing the values "False" and "True" are the only Boolean + objects. The Boolean type is a subtype of the integer type, and + Boolean values behave like the values 0 and 1, respectively, in + almost all contexts, the exception being that when converted to a + string, the strings ""False"" or ""True"" are returned, + respectively. + + +"numbers.Real" ("float") +------------------------ + +These represent machine-level double precision floating-point numbers. +You are at the mercy of the underlying machine architecture (and C or +Java implementation) for the accepted range and handling of overflow. +Python does not support single-precision floating-point numbers; the +savings in processor and memory usage that are usually the reason for +using these are dwarfed by the overhead of using objects in Python, so +there is no reason to complicate the language with two kinds of +floating-point numbers. + + +"numbers.Complex" ("complex") +----------------------------- + +These represent complex numbers as a pair of machine-level double +precision floating-point numbers. The same caveats apply as for +floating-point numbers. The real and imaginary parts of a complex +number "z" can be retrieved through the read-only attributes "z.real" +and "z.imag". + + +Sequences +========= + +These represent finite ordered sets indexed by non-negative numbers. +The built-in function "len()" returns the number of items of a +sequence. When the length of a sequence is *n*, the index set contains +the numbers 0, 1, …, *n*-1. Item *i* of sequence *a* is selected by +"a[i]". Some sequences, including built-in sequences, interpret +negative subscripts by adding the sequence length. For example, +"a[-2]" equals "a[n-2]", the second to last item of sequence a with +length "n". + +Sequences also support slicing: "a[i:j]" selects all items with index +*k* such that *i* "<=" *k* "<" *j*. When used as an expression, a +slice is a sequence of the same type. The comment above about negative +indexes also applies to negative slice positions. + +Some sequences also support “extended slicing” with a third “step” +parameter: "a[i:j:k]" selects all items of *a* with index *x* where "x += i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*. + +Sequences are distinguished according to their mutability: + + +Immutable sequences +------------------- + +An object of an immutable sequence type cannot change once it is +created. (If the object contains references to other objects, these +other objects may be mutable and may be changed; however, the +collection of objects directly referenced by an immutable object +cannot change.) + +The following types are immutable sequences: + +Strings + A string is a sequence of values that represent Unicode code + points. All the code points in the range "U+0000 - U+10FFFF" can be + represented in a string. Python doesn’t have a char type; instead, + every code point in the string is represented as a string object + with length "1". The built-in function "ord()" converts a code + point from its string form to an integer in the range "0 - 10FFFF"; + "chr()" converts an integer in the range "0 - 10FFFF" to the + corresponding length "1" string object. "str.encode()" can be used + to convert a "str" to "bytes" using the given text encoding, and + "bytes.decode()" can be used to achieve the opposite. + +Tuples + The items of a tuple are arbitrary Python objects. Tuples of two or + more items are formed by comma-separated lists of expressions. A + tuple of one item (a ‘singleton’) can be formed by affixing a comma + to an expression (an expression by itself does not create a tuple, + since parentheses must be usable for grouping of expressions). An + empty tuple can be formed by an empty pair of parentheses. + +Bytes + A bytes object is an immutable array. The items are 8-bit bytes, + represented by integers in the range 0 <= x < 256. Bytes literals + (like "b'abc'") and the built-in "bytes()" constructor can be used + to create bytes objects. Also, bytes objects can be decoded to + strings via the "decode()" method. + + +Mutable sequences +----------------- + +Mutable sequences can be changed after they are created. The +subscription and slicing notations can be used as the target of +assignment and "del" (delete) statements. + +Note: + + The "collections" and "array" module provide additional examples of + mutable sequence types. + +There are currently two intrinsic mutable sequence types: + +Lists + The items of a list are arbitrary Python objects. Lists are formed + by placing a comma-separated list of expressions in square + brackets. (Note that there are no special cases needed to form + lists of length 0 or 1.) + +Byte Arrays + A bytearray object is a mutable array. They are created by the + built-in "bytearray()" constructor. Aside from being mutable (and + hence unhashable), byte arrays otherwise provide the same interface + and functionality as immutable "bytes" objects. + + +Set types +========= + +These represent unordered, finite sets of unique, immutable objects. +As such, they cannot be indexed by any subscript. However, they can be +iterated over, and the built-in function "len()" returns the number of +items in a set. Common uses for sets are fast membership testing, +removing duplicates from a sequence, and computing mathematical +operations such as intersection, union, difference, and symmetric +difference. + +For set elements, the same immutability rules apply as for dictionary +keys. Note that numeric types obey the normal rules for numeric +comparison: if two numbers compare equal (e.g., "1" and "1.0"), only +one of them can be contained in a set. + +There are currently two intrinsic set types: + +Sets + These represent a mutable set. They are created by the built-in + "set()" constructor and can be modified afterwards by several + methods, such as "add()". + +Frozen sets + These represent an immutable set. They are created by the built-in + "frozenset()" constructor. As a frozenset is immutable and + *hashable*, it can be used again as an element of another set, or + as a dictionary key. + + +Mappings +======== + +These represent finite sets of objects indexed by arbitrary index +sets. The subscript notation "a[k]" selects the item indexed by "k" +from the mapping "a"; this can be used in expressions and as the +target of assignments or "del" statements. The built-in function +"len()" returns the number of items in a mapping. + +There is currently a single intrinsic mapping type: + + +Dictionaries +------------ + +These represent finite sets of objects indexed by nearly arbitrary +values. The only types of values not acceptable as keys are values +containing lists or dictionaries or other mutable types that are +compared by value rather than by object identity, the reason being +that the efficient implementation of dictionaries requires a key’s +hash value to remain constant. Numeric types used for keys obey the +normal rules for numeric comparison: if two numbers compare equal +(e.g., "1" and "1.0") then they can be used interchangeably to index +the same dictionary entry. + +Dictionaries preserve insertion order, meaning that keys will be +produced in the same order they were added sequentially over the +dictionary. Replacing an existing key does not change the order, +however removing a key and re-inserting it will add it to the end +instead of keeping its old place. + +Dictionaries are mutable; they can be created by the "{}" notation +(see section Dictionary displays). + +The extension modules "dbm.ndbm" and "dbm.gnu" provide additional +examples of mapping types, as does the "collections" module. + +Changed in version 3.7: Dictionaries did not preserve insertion order +in versions of Python before 3.6. In CPython 3.6, insertion order was +preserved, but it was considered an implementation detail at that time +rather than a language guarantee. + + +Callable types +============== + +These are the types to which the function call operation (see section +Calls) can be applied: + + +User-defined functions +---------------------- + +A user-defined function object is created by a function definition +(see section Function definitions). It should be called with an +argument list containing the same number of items as the function’s +formal parameter list. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| function.__builtins__ | A reference to the "dictionary" that holds the | +| | function’s builtins namespace. Added in version | +| | 3.10. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__globals__ | A reference to the "dictionary" that holds the | +| | function’s global variables – the global namespace | +| | of the module in which the function was defined. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__closure__ | "None" or a "tuple" of cells that contain bindings | +| | for the names specified in the "co_freevars" | +| | attribute of the function’s "code object". A cell | +| | object has the attribute "cell_contents". This can | +| | be used to get the value of the cell, as well as | +| | set the value. | ++----------------------------------------------------+----------------------------------------------------+ + + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Most of these attributes check the type of the assigned value: + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| function.__doc__ | The function’s documentation string, or "None" if | +| | unavailable. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__name__ | The function’s name. See also: "__name__ | +| | attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| function.__qualname__ | The function’s *qualified name*. See also: | +| | "__qualname__ attributes". Added in version 3.3. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__module__ | The name of the module the function was defined | +| | in, or "None" if unavailable. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__defaults__ | A "tuple" containing default *parameter* values | +| | for those parameters that have defaults, or "None" | +| | if no parameters have a default value. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__code__ | The code object representing the compiled function | +| | body. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__dict__ | The namespace supporting arbitrary function | +| | attributes. See also: "__dict__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| function.__annotations__ | A "dictionary" containing annotations of | +| | *parameters*. The keys of the dictionary are the | +| | parameter names, and "'return'" for the return | +| | annotation, if provided. See also: | +| | "object.__annotations__". Changed in version | +| | 3.14: Annotations are now lazily evaluated. See | +| | **PEP 649**. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__annotate__ | The *annotate function* for this function, or | +| | "None" if the function has no annotations. See | +| | "object.__annotate__". Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__kwdefaults__ | A "dictionary" containing defaults for keyword- | +| | only *parameters*. | ++----------------------------------------------------+----------------------------------------------------+ +| function.__type_params__ | A "tuple" containing the type parameters of a | +| | generic function. Added in version 3.12. | ++----------------------------------------------------+----------------------------------------------------+ + +Function objects also support getting and setting arbitrary +attributes, which can be used, for example, to attach metadata to +functions. Regular attribute dot-notation is used to get and set such +attributes. + +**CPython implementation detail:** CPython’s current implementation +only supports function attributes on user-defined functions. Function +attributes on built-in functions may be supported in the future. + +Additional information about a function’s definition can be retrieved +from its code object (accessible via the "__code__" attribute). + + +Instance methods +---------------- + +An instance method object combines a class, a class instance and any +callable object (normally a user-defined function). + +Special read-only attributes: + ++----------------------------------------------------+----------------------------------------------------+ +| method.__self__ | Refers to the class instance object to which the | +| | method is bound | ++----------------------------------------------------+----------------------------------------------------+ +| method.__func__ | Refers to the original function object | ++----------------------------------------------------+----------------------------------------------------+ +| method.__doc__ | The method’s documentation (same as | +| | "method.__func__.__doc__"). A "string" if the | +| | original function had a docstring, else "None". | ++----------------------------------------------------+----------------------------------------------------+ +| method.__name__ | The name of the method (same as | +| | "method.__func__.__name__") | ++----------------------------------------------------+----------------------------------------------------+ +| method.__module__ | The name of the module the method was defined in, | +| | or "None" if unavailable. | ++----------------------------------------------------+----------------------------------------------------+ + +Methods also support accessing (but not setting) the arbitrary +function attributes on the underlying function object. + +User-defined method objects may be created when getting an attribute +of a class (perhaps via an instance of that class), if that attribute +is a user-defined function object or a "classmethod" object. + +When an instance method object is created by retrieving a user-defined +function object from a class via one of its instances, its "__self__" +attribute is the instance, and the method object is said to be +*bound*. The new method’s "__func__" attribute is the original +function object. + +When an instance method object is created by retrieving a +"classmethod" object from a class or instance, its "__self__" +attribute is the class itself, and its "__func__" attribute is the +function object underlying the class method. + +When an instance method object is called, the underlying function +("__func__") is called, inserting the class instance ("__self__") in +front of the argument list. For instance, when "C" is a class which +contains a definition for a function "f()", and "x" is an instance of +"C", calling "x.f(1)" is equivalent to calling "C.f(x, 1)". + +When an instance method object is derived from a "classmethod" object, +the “class instance” stored in "__self__" will actually be the class +itself, so that calling either "x.f(1)" or "C.f(1)" is equivalent to +calling "f(C,1)" where "f" is the underlying function. + +It is important to note that user-defined functions which are +attributes of a class instance are not converted to bound methods; +this *only* happens when the function is an attribute of the class. + + +Generator functions +------------------- + +A function or method which uses the "yield" statement (see section The +yield statement) is called a *generator function*. Such a function, +when called, always returns an *iterator* object which can be used to +execute the body of the function: calling the iterator’s +"iterator.__next__()" method will cause the function to execute until +it provides a value using the "yield" statement. When the function +executes a "return" statement or falls off the end, a "StopIteration" +exception is raised and the iterator will have reached the end of the +set of values to be returned. + + +Coroutine functions +------------------- + +A function or method which is defined using "async def" is called a +*coroutine function*. Such a function, when called, returns a +*coroutine* object. It may contain "await" expressions, as well as +"async with" and "async for" statements. See also the Coroutine +Objects section. + + +Asynchronous generator functions +-------------------------------- + +A function or method which is defined using "async def" and which uses +the "yield" statement is called a *asynchronous generator function*. +Such a function, when called, returns an *asynchronous iterator* +object which can be used in an "async for" statement to execute the +body of the function. + +Calling the asynchronous iterator’s "aiterator.__anext__" method will +return an *awaitable* which when awaited will execute until it +provides a value using the "yield" expression. When the function +executes an empty "return" statement or falls off the end, a +"StopAsyncIteration" exception is raised and the asynchronous iterator +will have reached the end of the set of values to be yielded. + + +Built-in functions +------------------ + +A built-in function object is a wrapper around a C function. Examples +of built-in functions are "len()" and "math.sin()" ("math" is a +standard built-in module). The number and type of the arguments are +determined by the C function. Special read-only attributes: + +* "__doc__" is the function’s documentation string, or "None" if + unavailable. See "function.__doc__". + +* "__name__" is the function’s name. See "function.__name__". + +* "__self__" is set to "None" (but see the next item). + +* "__module__" is the name of the module the function was defined in + or "None" if unavailable. See "function.__module__". + + +Built-in methods +---------------- + +This is really a different disguise of a built-in function, this time +containing an object passed to the C function as an implicit extra +argument. An example of a built-in method is "alist.append()", +assuming *alist* is a list object. In this case, the special read-only +attribute "__self__" is set to the object denoted by *alist*. (The +attribute has the same semantics as it does with "other instance +methods".) + + +Classes +------- + +Classes are callable. These objects normally act as factories for new +instances of themselves, but variations are possible for class types +that override "__new__()". The arguments of the call are passed to +"__new__()" and, in the typical case, to "__init__()" to initialize +the new instance. + + +Class Instances +--------------- + +Instances of arbitrary classes can be made callable by defining a +"__call__()" method in their class. + + +Modules +======= + +Modules are a basic organizational unit of Python code, and are +created by the import system as invoked either by the "import" +statement, or by calling functions such as "importlib.import_module()" +and built-in "__import__()". A module object has a namespace +implemented by a "dictionary" object (this is the dictionary +referenced by the "__globals__" attribute of functions defined in the +module). Attribute references are translated to lookups in this +dictionary, e.g., "m.x" is equivalent to "m.__dict__["x"]". A module +object does not contain the code object used to initialize the module +(since it isn’t needed once the initialization is done). + +Attribute assignment updates the module’s namespace dictionary, e.g., +"m.x = 1" is equivalent to "m.__dict__["x"] = 1". + + +Import-related attributes on module objects +------------------------------------------- + +Module objects have the following attributes that relate to the import +system. When a module is created using the machinery associated with +the import system, these attributes are filled in based on the +module’s *spec*, before the *loader* executes and loads the module. + +To create a module dynamically rather than using the import system, +it’s recommended to use "importlib.util.module_from_spec()", which +will set the various import-controlled attributes to appropriate +values. It’s also possible to use the "types.ModuleType" constructor +to create modules directly, but this technique is more error-prone, as +most attributes must be manually set on the module object after it has +been created when using this approach. + +Caution: + + With the exception of "__name__", it is **strongly** recommended + that you rely on "__spec__" and its attributes instead of any of the + other individual attributes listed in this subsection. Note that + updating an attribute on "__spec__" will not update the + corresponding attribute on the module itself: + + >>> import typing + >>> typing.__name__, typing.__spec__.name + ('typing', 'typing') + >>> typing.__spec__.name = 'spelling' + >>> typing.__name__, typing.__spec__.name + ('typing', 'spelling') + >>> typing.__name__ = 'keyboard_smashing' + >>> typing.__name__, typing.__spec__.name + ('keyboard_smashing', 'spelling') + +module.__name__ + + The name used to uniquely identify the module in the import system. + For a directly executed module, this will be set to ""__main__"". + + This attribute must be set to the fully qualified name of the + module. It is expected to match the value of + "module.__spec__.name". + +module.__spec__ + + A record of the module’s import-system-related state. + + Set to the "module spec" that was used when importing the module. + See Module specs for more details. + + Added in version 3.4. + +module.__package__ + + The *package* a module belongs to. + + If the module is top-level (that is, not a part of any specific + package) then the attribute should be set to "''" (the empty + string). Otherwise, it should be set to the name of the module’s + package (which can be equal to "module.__name__" if the module + itself is a package). See **PEP 366** for further details. + + This attribute is used instead of "__name__" to calculate explicit + relative imports for main modules. It defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor; use "importlib.util.module_from_spec()" instead to + ensure the attribute is set to a "str". + + It is **strongly** recommended that you use + "module.__spec__.parent" instead of "module.__package__". + "__package__" is now only used as a fallback if "__spec__.parent" + is not set, and this fallback path is deprecated. + + Changed in version 3.4: This attribute now defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor. Previously the attribute was optional. + + Changed in version 3.6: The value of "__package__" is expected to + be the same as "__spec__.parent". "__package__" is now only used as + a fallback during import resolution if "__spec__.parent" is not + defined. + + Changed in version 3.10: "ImportWarning" is raised if an import + resolution falls back to "__package__" instead of + "__spec__.parent". + + Changed in version 3.12: Raise "DeprecationWarning" instead of + "ImportWarning" when falling back to "__package__" during import + resolution. + + Deprecated since version 3.13, will be removed in version 3.15: + "__package__" will cease to be set or taken into consideration by + the import system or standard library. + +module.__loader__ + + The *loader* object that the import machinery used to load the + module. + + This attribute is mostly useful for introspection, but can be used + for additional loader-specific functionality, for example getting + data associated with a loader. + + "__loader__" defaults to "None" for modules created dynamically + using the "types.ModuleType" constructor; use + "importlib.util.module_from_spec()" instead to ensure the attribute + is set to a *loader* object. + + It is **strongly** recommended that you use + "module.__spec__.loader" instead of "module.__loader__". + + Changed in version 3.4: This attribute now defaults to "None" for + modules created dynamically using the "types.ModuleType" + constructor. Previously the attribute was optional. + + Deprecated since version 3.12, will be removed in version 3.16: + Setting "__loader__" on a module while failing to set + "__spec__.loader" is deprecated. In Python 3.16, "__loader__" will + cease to be set or taken into consideration by the import system or + the standard library. + +module.__path__ + + A (possibly empty) *sequence* of strings enumerating the locations + where the package’s submodules will be found. Non-package modules + should not have a "__path__" attribute. See __path__ attributes on + modules for more details. + + It is **strongly** recommended that you use + "module.__spec__.submodule_search_locations" instead of + "module.__path__". + +module.__file__ + +module.__cached__ + + "__file__" and "__cached__" are both optional attributes that may + or may not be set. Both attributes should be a "str" when they are + available. + + "__file__" indicates the pathname of the file from which the module + was loaded (if loaded from a file), or the pathname of the shared + library file for extension modules loaded dynamically from a shared + library. It might be missing for certain types of modules, such as + C modules that are statically linked into the interpreter, and the + import system may opt to leave it unset if it has no semantic + meaning (for example, a module loaded from a database). + + If "__file__" is set then the "__cached__" attribute might also be + set, which is the path to any compiled version of the code (for + example, a byte-compiled file). The file does not need to exist to + set this attribute; the path can simply point to where the compiled + file *would* exist (see **PEP 3147**). + + Note that "__cached__" may be set even if "__file__" is not set. + However, that scenario is quite atypical. Ultimately, the *loader* + is what makes use of the module spec provided by the *finder* (from + which "__file__" and "__cached__" are derived). So if a loader can + load from a cached module but otherwise does not load from a file, + that atypical scenario may be appropriate. + + It is **strongly** recommended that you use + "module.__spec__.cached" instead of "module.__cached__". + + Deprecated since version 3.13, will be removed in version 3.15: + Setting "__cached__" on a module while failing to set + "__spec__.cached" is deprecated. In Python 3.15, "__cached__" will + cease to be set or taken into consideration by the import system or + standard library. + + +Other writable attributes on module objects +------------------------------------------- + +As well as the import-related attributes listed above, module objects +also have the following writable attributes: + +module.__doc__ + + The module’s documentation string, or "None" if unavailable. See + also: "__doc__ attributes". + +module.__annotations__ + + A dictionary containing *variable annotations* collected during + module body execution. For best practices on working with + "__annotations__", see "annotationlib". + + Changed in version 3.14: Annotations are now lazily evaluated. See + **PEP 649**. + +module.__annotate__ + + The *annotate function* for this module, or "None" if the module + has no annotations. See also: "__annotate__" attributes. + + Added in version 3.14. + + +Module dictionaries +------------------- + +Module objects also have the following special read-only attribute: + +module.__dict__ + + The module’s namespace as a dictionary object. Uniquely among the + attributes listed here, "__dict__" cannot be accessed as a global + variable from within a module; it can only be accessed as an + attribute on module objects. + + **CPython implementation detail:** Because of the way CPython + clears module dictionaries, the module dictionary will be cleared + when the module falls out of scope even if the dictionary still has + live references. To avoid this, copy the dictionary or keep the + module around while using its dictionary directly. + + +Custom classes +============== + +Custom class types are typically created by class definitions (see +section Class definitions). A class has a namespace implemented by a +dictionary object. Class attribute references are translated to +lookups in this dictionary, e.g., "C.x" is translated to +"C.__dict__["x"]" (although there are a number of hooks which allow +for other means of locating attributes). When the attribute name is +not found there, the attribute search continues in the base classes. +This search of the base classes uses the C3 method resolution order +which behaves correctly even in the presence of ‘diamond’ inheritance +structures where there are multiple inheritance paths leading back to +a common ancestor. Additional details on the C3 MRO used by Python can +be found at The Python 2.3 Method Resolution Order. + +When a class attribute reference (for class "C", say) would yield a +class method object, it is transformed into an instance method object +whose "__self__" attribute is "C". When it would yield a +"staticmethod" object, it is transformed into the object wrapped by +the static method object. See section Implementing Descriptors for +another way in which attributes retrieved from a class may differ from +those actually contained in its "__dict__". + +Class attribute assignments update the class’s dictionary, never the +dictionary of a base class. + +A class object can be called (see above) to yield a class instance +(see below). + + +Special attributes +------------------ + ++----------------------------------------------------+----------------------------------------------------+ +| Attribute | Meaning | +|====================================================|====================================================| +| type.__name__ | The class’s name. See also: "__name__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__qualname__ | The class’s *qualified name*. See also: | +| | "__qualname__ attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__module__ | The name of the module in which the class was | +| | defined. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__dict__ | A "mapping proxy" providing a read-only view of | +| | the class’s namespace. See also: "__dict__ | +| | attributes". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__bases__ | A "tuple" containing the class’s bases. In most | +| | cases, for a class defined as "class X(A, B, C)", | +| | "X.__bases__" will be exactly equal to "(A, B, | +| | C)". | ++----------------------------------------------------+----------------------------------------------------+ +| type.__base__ | **CPython implementation detail:** The single base | +| | class in the inheritance chain that is responsible | +| | for the memory layout of instances. This attribute | +| | corresponds to "tp_base" at the C level. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__doc__ | The class’s documentation string, or "None" if | +| | undefined. Not inherited by subclasses. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__annotations__ | A dictionary containing *variable annotations* | +| | collected during class body execution. See also: | +| | "__annotations__ attributes". For best practices | +| | on working with "__annotations__", please see | +| | "annotationlib". Use | +| | "annotationlib.get_annotations()" instead of | +| | accessing this attribute directly. Warning: | +| | Accessing the "__annotations__" attribute directly | +| | on a class object may return annotations for the | +| | wrong class, specifically in certain cases where | +| | the class, its base class, or a metaclass is | +| | defined under "from __future__ import | +| | annotations". See **749** for details.This | +| | attribute does not exist on certain builtin | +| | classes. On user-defined classes without | +| | "__annotations__", it is an empty dictionary. | +| | Changed in version 3.14: Annotations are now | +| | lazily evaluated. See **PEP 649**. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__annotate__() | The *annotate function* for this class, or "None" | +| | if the class has no annotations. See also: | +| | "__annotate__ attributes". Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__type_params__ | A "tuple" containing the type parameters of a | +| | generic class. Added in version 3.12. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__static_attributes__ | A "tuple" containing names of attributes of this | +| | class which are assigned through "self.X" from any | +| | function in its body. Added in version 3.13. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__firstlineno__ | The line number of the first line of the class | +| | definition, including decorators. Setting the | +| | "__module__" attribute removes the | +| | "__firstlineno__" item from the type’s dictionary. | +| | Added in version 3.13. | ++----------------------------------------------------+----------------------------------------------------+ +| type.__mro__ | The "tuple" of classes that are considered when | +| | looking for base classes during method resolution. | ++----------------------------------------------------+----------------------------------------------------+ + + +Special methods +--------------- + +In addition to the special attributes described above, all Python +classes also have the following two methods available: + +type.mro() + + This method can be overridden by a metaclass to customize the + method resolution order for its instances. It is called at class + instantiation, and its result is stored in "__mro__". + +type.__subclasses__() + + Each class keeps a list of weak references to its immediate + subclasses. This method returns a list of all those references + still alive. The list is in definition order. Example: + + >>> class A: pass + >>> class B(A): pass + >>> A.__subclasses__() + [] + + +Class instances +=============== + +A class instance is created by calling a class object (see above). A +class instance has a namespace implemented as a dictionary which is +the first place in which attribute references are searched. When an +attribute is not found there, and the instance’s class has an +attribute by that name, the search continues with the class +attributes. If a class attribute is found that is a user-defined +function object, it is transformed into an instance method object +whose "__self__" attribute is the instance. Static method and class +method objects are also transformed; see above under “Classes”. See +section Implementing Descriptors for another way in which attributes +of a class retrieved via its instances may differ from the objects +actually stored in the class’s "__dict__". If no class attribute is +found, and the object’s class has a "__getattr__()" method, that is +called to satisfy the lookup. + +Attribute assignments and deletions update the instance’s dictionary, +never a class’s dictionary. If the class has a "__setattr__()" or +"__delattr__()" method, this is called instead of updating the +instance dictionary directly. + +Class instances can pretend to be numbers, sequences, or mappings if +they have methods with certain special names. See section Special +method names. + + +Special attributes +------------------ + +object.__class__ + + The class to which a class instance belongs. + +object.__dict__ + + A dictionary or other mapping object used to store an object’s + (writable) attributes. Not all instances have a "__dict__" + attribute; see the section on __slots__ for more details. + + +I/O objects (also known as file objects) +======================================== + +A *file object* represents an open file. Various shortcuts are +available to create file objects: the "open()" built-in function, and +also "os.popen()", "os.fdopen()", and the "makefile()" method of +socket objects (and perhaps by other functions or methods provided by +extension modules). + +The objects "sys.stdin", "sys.stdout" and "sys.stderr" are initialized +to file objects corresponding to the interpreter’s standard input, +output and error streams; they are all open in text mode and therefore +follow the interface defined by the "io.TextIOBase" abstract class. + + +Internal types +============== + +A few types used internally by the interpreter are exposed to the +user. Their definitions may change with future versions of the +interpreter, but they are mentioned here for completeness. + + +Code objects +------------ + +Code objects represent *byte-compiled* executable Python code, or +*bytecode*. The difference between a code object and a function object +is that the function object contains an explicit reference to the +function’s globals (the module in which it was defined), while a code +object contains no context; also the default argument values are +stored in the function object, not in the code object (because they +represent values calculated at run-time). Unlike function objects, +code objects are immutable and contain no references (directly or +indirectly) to mutable objects. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_name | The function name | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_qualname | The fully qualified function name Added in | +| | version 3.11. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_argcount | The total number of positional *parameters* | +| | (including positional-only parameters and | +| | parameters with default values) that the function | +| | has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_posonlyargcount | The number of positional-only *parameters* | +| | (including arguments with default values) that the | +| | function has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_kwonlyargcount | The number of keyword-only *parameters* (including | +| | arguments with default values) that the function | +| | has | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_nlocals | The number of local variables used by the function | +| | (including parameters) | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_varnames | A "tuple" containing the names of the local | +| | variables in the function (starting with the | +| | parameter names) | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_cellvars | A "tuple" containing the names of local variables | +| | that are referenced from at least one *nested | +| | scope* inside the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_freevars | A "tuple" containing the names of *free (closure) | +| | variables* that a *nested scope* references in an | +| | outer scope. See also "function.__closure__". | +| | Note: references to global and builtin names are | +| | *not* included. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_code | A string representing the sequence of *bytecode* | +| | instructions in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_consts | A "tuple" containing the literals used by the | +| | *bytecode* in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_names | A "tuple" containing the names used by the | +| | *bytecode* in the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_filename | The name of the file from which the code was | +| | compiled | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_firstlineno | The line number of the first line of the function | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_lnotab | A string encoding the mapping from *bytecode* | +| | offsets to line numbers. For details, see the | +| | source code of the interpreter. Deprecated since | +| | version 3.12: This attribute of code objects is | +| | deprecated, and may be removed in Python 3.15. | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_stacksize | The required stack size of the code object | ++----------------------------------------------------+----------------------------------------------------+ +| codeobject.co_flags | An "integer" encoding a number of flags for the | +| | interpreter. | ++----------------------------------------------------+----------------------------------------------------+ + +The following flag bits are defined for "co_flags": bit "0x04" is set +if the function uses the "*arguments" syntax to accept an arbitrary +number of positional arguments; bit "0x08" is set if the function uses +the "**keywords" syntax to accept arbitrary keyword arguments; bit +"0x20" is set if the function is a generator. See Code Objects Bit +Flags for details on the semantics of each flags that might be +present. + +Future feature declarations (for example, "from __future__ import +division") also use bits in "co_flags" to indicate whether a code +object was compiled with a particular feature enabled. See +"compiler_flag". + +Other bits in "co_flags" are reserved for internal use. + +If a code object represents a function and has a docstring, the +"CO_HAS_DOCSTRING" bit is set in "co_flags" and the first item in +"co_consts" is the docstring of the function. + + +Methods on code objects +~~~~~~~~~~~~~~~~~~~~~~~ + +codeobject.co_positions() + + Returns an iterable over the source code positions of each + *bytecode* instruction in the code object. + + The iterator returns "tuple"s containing the "(start_line, + end_line, start_column, end_column)". The *i-th* tuple corresponds + to the position of the source code that compiled to the *i-th* code + unit. Column information is 0-indexed utf-8 byte offsets on the + given source line. + + This positional information can be missing. A non-exhaustive lists + of cases where this may happen: + + * Running the interpreter with "-X" "no_debug_ranges". + + * Loading a pyc file compiled while using "-X" "no_debug_ranges". + + * Position tuples corresponding to artificial instructions. + + * Line and column numbers that can’t be represented due to + implementation specific limitations. + + When this occurs, some or all of the tuple elements can be "None". + + Added in version 3.11. + + Note: + + This feature requires storing column positions in code objects + which may result in a small increase of disk usage of compiled + Python files or interpreter memory usage. To avoid storing the + extra information and/or deactivate printing the extra traceback + information, the "-X" "no_debug_ranges" command line flag or the + "PYTHONNODEBUGRANGES" environment variable can be used. + +codeobject.co_lines() + + Returns an iterator that yields information about successive ranges + of *bytecode*s. Each item yielded is a "(start, end, lineno)" + "tuple": + + * "start" (an "int") represents the offset (inclusive) of the start + of the *bytecode* range + + * "end" (an "int") represents the offset (exclusive) of the end of + the *bytecode* range + + * "lineno" is an "int" representing the line number of the + *bytecode* range, or "None" if the bytecodes in the given range + have no line number + + The items yielded will have the following properties: + + * The first range yielded will have a "start" of 0. + + * The "(start, end)" ranges will be non-decreasing and consecutive. + That is, for any pair of "tuple"s, the "start" of the second will + be equal to the "end" of the first. + + * No range will be backwards: "end >= start" for all triples. + + * The last "tuple" yielded will have "end" equal to the size of the + *bytecode*. + + Zero-width ranges, where "start == end", are allowed. Zero-width + ranges are used for lines that are present in the source code, but + have been eliminated by the *bytecode* compiler. + + Added in version 3.10. + + See also: + + **PEP 626** - Precise line numbers for debugging and other tools. + The PEP that introduced the "co_lines()" method. + +codeobject.replace(**kwargs) + + Return a copy of the code object with new values for the specified + fields. + + Code objects are also supported by the generic function + "copy.replace()". + + Added in version 3.8. + + +Frame objects +------------- + +Frame objects represent execution frames. They may occur in traceback +objects, and are also passed to registered trace functions. + + +Special read-only attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_back | Points to the previous stack frame (towards the | +| | caller), or "None" if this is the bottom stack | +| | frame | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_code | The code object being executed in this frame. | +| | Accessing this attribute raises an auditing event | +| | "object.__getattr__" with arguments "obj" and | +| | ""f_code"". | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_locals | The mapping used by the frame to look up local | +| | variables. If the frame refers to an *optimized | +| | scope*, this may return a write-through proxy | +| | object. Changed in version 3.13: Return a proxy | +| | for optimized scopes. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_globals | The dictionary used by the frame to look up global | +| | variables | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_builtins | The dictionary used by the frame to look up built- | +| | in (intrinsic) names | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_lasti | The “precise instruction” of the frame object | +| | (this is an index into the *bytecode* string of | +| | the code object) | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_generator | The *generator* or *coroutine* object that owns | +| | this frame, or "None" if the frame is a normal | +| | function. Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ + + +Special writable attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace | If not "None", this is a function called for | +| | various events during code execution (this is used | +| | by debuggers). Normally an event is triggered for | +| | each new source line (see "f_trace_lines"). | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace_lines | Set this attribute to "False" to disable | +| | triggering a tracing event for each source line. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_trace_opcodes | Set this attribute to "True" to allow per-opcode | +| | events to be requested. Note that this may lead to | +| | undefined interpreter behaviour if exceptions | +| | raised by the trace function escape to the | +| | function being traced. | ++----------------------------------------------------+----------------------------------------------------+ +| frame.f_lineno | The current line number of the frame – writing to | +| | this from within a trace function jumps to the | +| | given line (only for the bottom-most frame). A | +| | debugger can implement a Jump command (aka Set | +| | Next Statement) by writing to this attribute. | ++----------------------------------------------------+----------------------------------------------------+ + + +Frame object methods +~~~~~~~~~~~~~~~~~~~~ + +Frame objects support one method: + +frame.clear() + + This method clears all references to local variables held by the + frame. Also, if the frame belonged to a *generator*, the generator + is finalized. This helps break reference cycles involving frame + objects (for example when catching an exception and storing its + traceback for later use). + + "RuntimeError" is raised if the frame is currently executing or + suspended. + + Added in version 3.4. + + Changed in version 3.13: Attempting to clear a suspended frame + raises "RuntimeError" (as has always been the case for executing + frames). + + +Traceback objects +----------------- + +Traceback objects represent the stack trace of an exception. A +traceback object is implicitly created when an exception occurs, and +may also be explicitly created by calling "types.TracebackType". + +Changed in version 3.7: Traceback objects can now be explicitly +instantiated from Python code. + +For implicitly created tracebacks, when the search for an exception +handler unwinds the execution stack, at each unwound level a traceback +object is inserted in front of the current traceback. When an +exception handler is entered, the stack trace is made available to the +program. (See section The try statement.) It is accessible as the +third item of the tuple returned by "sys.exc_info()", and as the +"__traceback__" attribute of the caught exception. + +When the program contains no suitable handler, the stack trace is +written (nicely formatted) to the standard error stream; if the +interpreter is interactive, it is also made available to the user as +"sys.last_traceback". + +For explicitly created tracebacks, it is up to the creator of the +traceback to determine how the "tb_next" attributes should be linked +to form a full stack trace. + +Special read-only attributes: + ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_frame | Points to the execution frame of the current | +| | level. Accessing this attribute raises an | +| | auditing event "object.__getattr__" with arguments | +| | "obj" and ""tb_frame"". | ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_lineno | Gives the line number where the exception occurred | ++----------------------------------------------------+----------------------------------------------------+ +| traceback.tb_lasti | Indicates the “precise instruction”. | ++----------------------------------------------------+----------------------------------------------------+ + +The line number and last instruction in the traceback may differ from +the line number of its frame object if the exception occurred in a +"try" statement with no matching except clause or with a "finally" +clause. + +traceback.tb_next + + The special writable attribute "tb_next" is the next level in the + stack trace (towards the frame where the exception occurred), or + "None" if there is no next level. + + Changed in version 3.7: This attribute is now writable + + +Slice objects +------------- + +Slice objects are used to represent slices for "__getitem__()" +methods. They are also created by the built-in "slice()" function. + +Special read-only attributes: "start" is the lower bound; "stop" is +the upper bound; "step" is the step value; each is "None" if omitted. +These attributes can have any type. + +Slice objects support one method: + +slice.indices(self, length) + + This method takes a single integer argument *length* and computes + information about the slice that the slice object would describe if + applied to a sequence of *length* items. It returns a tuple of + three integers; respectively these are the *start* and *stop* + indices and the *step* or stride length of the slice. Missing or + out-of-bounds indices are handled in a manner consistent with + regular slices. + + +Static method objects +--------------------- + +Static method objects provide a way of defeating the transformation of +function objects to method objects described above. A static method +object is a wrapper around any other object, usually a user-defined +method object. When a static method object is retrieved from a class +or a class instance, the object actually returned is the wrapped +object, which is not subject to any further transformation. Static +method objects are also callable. Static method objects are created by +the built-in "staticmethod()" constructor. + + +Class method objects +-------------------- + +A class method object, like a static method object, is a wrapper +around another object that alters the way in which that object is +retrieved from classes and class instances. The behaviour of class +method objects upon such retrieval is described above, under “instance +methods”. Class method objects are created by the built-in +"classmethod()" constructor. +''', + 'typesfunctions': r'''Functions +********* + +Function objects are created by function definitions. The only +operation on a function object is to call it: "func(argument-list)". + +There are really two flavors of function objects: built-in functions +and user-defined functions. Both support the same operation (to call +the function), but the implementation is different, hence the +different object types. + +See Function definitions for more information. +''', + 'typesmapping': r'''Mapping Types — "dict" +********************** + +A *mapping* object maps *hashable* values to arbitrary objects. +Mappings are mutable objects. There is currently only one standard +mapping type, the *dictionary*. (For other containers see the built- +in "list", "set", and "tuple" classes, and the "collections" module.) + +A dictionary’s keys are *almost* arbitrary values. Values that are +not *hashable*, that is, values containing lists, dictionaries or +other mutable types (that are compared by value rather than by object +identity) may not be used as keys. Values that compare equal (such as +"1", "1.0", and "True") can be used interchangeably to index the same +dictionary entry. + +class dict(**kwargs) +class dict(mapping, /, **kwargs) +class dict(iterable, /, **kwargs) + + Return a new dictionary initialized from an optional positional + argument and a possibly empty set of keyword arguments. + + Dictionaries can be created by several means: + + * Use a comma-separated list of "key: value" pairs within braces: + "{'jack': 4098, 'sjoerd': 4127}" or "{4098: 'jack', 4127: + 'sjoerd'}" + + * Use a dict comprehension: "{}", "{x: x ** 2 for x in range(10)}" + + * Use the type constructor: "dict()", "dict([('foo', 100), ('bar', + 200)])", "dict(foo=100, bar=200)" + + If no positional argument is given, an empty dictionary is created. + If a positional argument is given and it defines a "keys()" method, + a dictionary is created by calling "__getitem__()" on the argument + with each returned key from the method. Otherwise, the positional + argument must be an *iterable* object. Each item in the iterable + must itself be an iterable with exactly two elements. The first + element of each item becomes a key in the new dictionary, and the + second element the corresponding value. If a key occurs more than + once, the last value for that key becomes the corresponding value + in the new dictionary. + + If keyword arguments are given, the keyword arguments and their + values are added to the dictionary created from the positional + argument. If a key being added is already present, the value from + the keyword argument replaces the value from the positional + argument. + + Dictionaries compare equal if and only if they have the same "(key, + value)" pairs (regardless of ordering). Order comparisons (‘<’, + ‘<=’, ‘>=’, ‘>’) raise "TypeError". To illustrate dictionary + creation and equality, the following examples all return a + dictionary equal to "{"one": 1, "two": 2, "three": 3}": + + >>> a = dict(one=1, two=2, three=3) + >>> b = {'one': 1, 'two': 2, 'three': 3} + >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3])) + >>> d = dict([('two', 2), ('one', 1), ('three', 3)]) + >>> e = dict({'three': 3, 'one': 1, 'two': 2}) + >>> f = dict({'one': 1, 'three': 3}, two=2) + >>> a == b == c == d == e == f + True + + Providing keyword arguments as in the first example only works for + keys that are valid Python identifiers. Otherwise, any valid keys + can be used. + + Dictionaries preserve insertion order. Note that updating a key + does not affect the order. Keys added after deletion are inserted + at the end. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. This behavior was an implementation detail of + CPython from 3.6. + + These are the operations that dictionaries support (and therefore, + custom mapping types should support too): + + list(d) + + Return a list of all the keys used in the dictionary *d*. + + len(d) + + Return the number of items in the dictionary *d*. + + d[key] + + Return the item of *d* with key *key*. Raises a "KeyError" if + *key* is not in the map. + + If a subclass of dict defines a method "__missing__()" and *key* + is not present, the "d[key]" operation calls that method with + the key *key* as argument. The "d[key]" operation then returns + or raises whatever is returned or raised by the + "__missing__(key)" call. No other operations or methods invoke + "__missing__()". If "__missing__()" is not defined, "KeyError" + is raised. "__missing__()" must be a method; it cannot be an + instance variable: + + >>> class Counter(dict): + ... def __missing__(self, key): + ... return 0 + ... + >>> c = Counter() + >>> c['red'] + 0 + >>> c['red'] += 1 + >>> c['red'] + 1 + + The example above shows part of the implementation of + "collections.Counter". A different "__missing__()" method is + used by "collections.defaultdict". + + d[key] = value + + Set "d[key]" to *value*. + + del d[key] + + Remove "d[key]" from *d*. Raises a "KeyError" if *key* is not + in the map. + + key in d + + Return "True" if *d* has a key *key*, else "False". + + key not in d + + Equivalent to "not key in d". + + iter(d) + + Return an iterator over the keys of the dictionary. This is a + shortcut for "iter(d.keys())". + + clear() + + Remove all items from the dictionary. + + copy() + + Return a shallow copy of the dictionary. + + classmethod fromkeys(iterable, value=None, /) + + Create a new dictionary with keys from *iterable* and values set + to *value*. + + "fromkeys()" is a class method that returns a new dictionary. + *value* defaults to "None". All of the values refer to just a + single instance, so it generally doesn’t make sense for *value* + to be a mutable object such as an empty list. To get distinct + values, use a dict comprehension instead. + + get(key, default=None, /) + + Return the value for *key* if *key* is in the dictionary, else + *default*. If *default* is not given, it defaults to "None", so + that this method never raises a "KeyError". + + items() + + Return a new view of the dictionary’s items ("(key, value)" + pairs). See the documentation of view objects. + + keys() + + Return a new view of the dictionary’s keys. See the + documentation of view objects. + + pop(key, /) + pop(key, default, /) + + If *key* is in the dictionary, remove it and return its value, + else return *default*. If *default* is not given and *key* is + not in the dictionary, a "KeyError" is raised. + + popitem() + + Remove and return a "(key, value)" pair from the dictionary. + Pairs are returned in LIFO (last-in, first-out) order. + + "popitem()" is useful to destructively iterate over a + dictionary, as often used in set algorithms. If the dictionary + is empty, calling "popitem()" raises a "KeyError". + + Changed in version 3.7: LIFO order is now guaranteed. In prior + versions, "popitem()" would return an arbitrary key/value pair. + + reversed(d) + + Return a reverse iterator over the keys of the dictionary. This + is a shortcut for "reversed(d.keys())". + + Added in version 3.8. + + setdefault(key, default=None, /) + + If *key* is in the dictionary, return its value. If not, insert + *key* with a value of *default* and return *default*. *default* + defaults to "None". + + update(**kwargs) + update(mapping, /, **kwargs) + update(iterable, /, **kwargs) + + Update the dictionary with the key/value pairs from *mapping* or + *iterable* and *kwargs*, overwriting existing keys. Return + "None". + + "update()" accepts either another object with a "keys()" method + (in which case "__getitem__()" is called with every key returned + from the method) or an iterable of key/value pairs (as tuples or + other iterables of length two). If keyword arguments are + specified, the dictionary is then updated with those key/value + pairs: "d.update(red=1, blue=2)". + + values() + + Return a new view of the dictionary’s values. See the + documentation of view objects. + + An equality comparison between one "dict.values()" view and + another will always return "False". This also applies when + comparing "dict.values()" to itself: + + >>> d = {'a': 1} + >>> d.values() == d.values() + False + + d | other + + Create a new dictionary with the merged keys and values of *d* + and *other*, which must both be dictionaries. The values of + *other* take priority when *d* and *other* share keys. + + Added in version 3.9. + + d |= other + + Update the dictionary *d* with keys and values from *other*, + which may be either a *mapping* or an *iterable* of key/value + pairs. The values of *other* take priority when *d* and *other* + share keys. + + Added in version 3.9. + + Dictionaries and dictionary views are reversible. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(reversed(d)) + ['four', 'three', 'two', 'one'] + >>> list(reversed(d.values())) + [4, 3, 2, 1] + >>> list(reversed(d.items())) + [('four', 4), ('three', 3), ('two', 2), ('one', 1)] + + Changed in version 3.8: Dictionaries are now reversible. + +See also: + + "types.MappingProxyType" can be used to create a read-only view of a + "dict". + + +Dictionary view objects +======================= + +The objects returned by "dict.keys()", "dict.values()" and +"dict.items()" are *view objects*. They provide a dynamic view on the +dictionary’s entries, which means that when the dictionary changes, +the view reflects these changes. + +Dictionary views can be iterated over to yield their respective data, +and support membership tests: + +len(dictview) + + Return the number of entries in the dictionary. + +iter(dictview) + + Return an iterator over the keys, values or items (represented as + tuples of "(key, value)") in the dictionary. + + Keys and values are iterated over in insertion order. This allows + the creation of "(value, key)" pairs using "zip()": "pairs = + zip(d.values(), d.keys())". Another way to create the same list is + "pairs = [(v, k) for (k, v) in d.items()]". + + Iterating views while adding or deleting entries in the dictionary + may raise a "RuntimeError" or fail to iterate over all entries. + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. + +x in dictview + + Return "True" if *x* is in the underlying dictionary’s keys, values + or items (in the latter case, *x* should be a "(key, value)" + tuple). + +reversed(dictview) + + Return a reverse iterator over the keys, values or items of the + dictionary. The view will be iterated in reverse order of the + insertion. + + Changed in version 3.8: Dictionary views are now reversible. + +dictview.mapping + + Return a "types.MappingProxyType" that wraps the original + dictionary to which the view refers. + + Added in version 3.10. + +Keys views are set-like since their entries are unique and *hashable*. +Items views also have set-like operations since the (key, value) pairs +are unique and the keys are hashable. If all values in an items view +are hashable as well, then the items view can interoperate with other +sets. (Values views are not treated as set-like since the entries are +generally not unique.) For set-like views, all of the operations +defined for the abstract base class "collections.abc.Set" are +available (for example, "==", "<", or "^"). While using set +operators, set-like views accept any iterable as the other operand, +unlike sets which only accept sets as the input. + +An example of dictionary view usage: + + >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500} + >>> keys = dishes.keys() + >>> values = dishes.values() + + >>> # iteration + >>> n = 0 + >>> for val in values: + ... n += val + ... + >>> print(n) + 504 + + >>> # keys and values are iterated over in the same order (insertion order) + >>> list(keys) + ['eggs', 'sausage', 'bacon', 'spam'] + >>> list(values) + [2, 1, 1, 500] + + >>> # view objects are dynamic and reflect dict changes + >>> del dishes['eggs'] + >>> del dishes['sausage'] + >>> list(keys) + ['bacon', 'spam'] + + >>> # set operations + >>> keys & {'eggs', 'bacon', 'salad'} + {'bacon'} + >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', 'bacon', 'spam'} + True + >>> keys | ['juice', 'juice', 'juice'] == {'bacon', 'spam', 'juice'} + True + + >>> # get back a read-only proxy for the original dictionary + >>> values.mapping + mappingproxy({'bacon': 1, 'spam': 500}) + >>> values.mapping['spam'] + 500 +''', + 'typesmethods': r'''Methods +******* + +Methods are functions that are called using the attribute notation. +There are two flavors: built-in methods (such as "append()" on lists) +and class instance method. Built-in methods are described with the +types that support them. + +If you access a method (a function defined in a class namespace) +through an instance, you get a special object: a *bound method* (also +called instance method) object. When called, it will add the "self" +argument to the argument list. Bound methods have two special read- +only attributes: "m.__self__" is the object on which the method +operates, and "m.__func__" is the function implementing the method. +Calling "m(arg-1, arg-2, ..., arg-n)" is completely equivalent to +calling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)". + +Like function objects, bound method objects support getting arbitrary +attributes. However, since method attributes are actually stored on +the underlying function object ("method.__func__"), setting method +attributes on bound methods is disallowed. Attempting to set an +attribute on a method results in an "AttributeError" being raised. In +order to set a method attribute, you need to explicitly set it on the +underlying function object: + + >>> class C: + ... def method(self): + ... pass + ... + >>> c = C() + >>> c.method.whoami = 'my name is method' # can't set on the method + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'method' object has no attribute 'whoami' + >>> c.method.__func__.whoami = 'my name is method' + >>> c.method.whoami + 'my name is method' + +See Instance methods for more information. +''', + 'typesmodules': r'''Modules +******* + +The only special operation on a module is attribute access: "m.name", +where *m* is a module and *name* accesses a name defined in *m*’s +symbol table. Module attributes can be assigned to. (Note that the +"import" statement is not, strictly speaking, an operation on a module +object; "import foo" does not require a module object named *foo* to +exist, rather it requires an (external) *definition* for a module +named *foo* somewhere.) + +A special attribute of every module is "__dict__". This is the +dictionary containing the module’s symbol table. Modifying this +dictionary will actually change the module’s symbol table, but direct +assignment to the "__dict__" attribute is not possible (you can write +"m.__dict__['a'] = 1", which defines "m.a" to be "1", but you can’t +write "m.__dict__ = {}"). Modifying "__dict__" directly is not +recommended. + +Modules built into the interpreter are written like this: "". If loaded from a file, they are written as +"". +''', + 'typesseq': r'''Sequence Types — "list", "tuple", "range" +***************************************** + +There are three basic sequence types: lists, tuples, and range +objects. Additional sequence types tailored for processing of binary +data and text strings are described in dedicated sections. + + +Common Sequence Operations +========================== + +The operations in the following table are supported by most sequence +types, both mutable and immutable. The "collections.abc.Sequence" ABC +is provided to make it easier to correctly implement these operations +on custom sequence types. + +This table lists the sequence operations sorted in ascending priority. +In the table, *s* and *t* are sequences of the same type, *n*, *i*, +*j* and *k* are integers and *x* is an arbitrary object that meets any +type and value restrictions imposed by *s*. + +The "in" and "not in" operations have the same priorities as the +comparison operations. The "+" (concatenation) and "*" (repetition) +operations have the same priority as the corresponding numeric +operations. [3] + ++----------------------------+----------------------------------+------------+ +| Operation | Result | Notes | +|============================|==================================|============| +| "x in s" | "True" if an item of *s* is | (1) | +| | equal to *x*, else "False" | | ++----------------------------+----------------------------------+------------+ +| "x not in s" | "False" if an item of *s* is | (1) | +| | equal to *x*, else "True" | | ++----------------------------+----------------------------------+------------+ +| "s + t" | the concatenation of *s* and *t* | (6)(7) | ++----------------------------+----------------------------------+------------+ +| "s * n" or "n * s" | equivalent to adding *s* to | (2)(7) | +| | itself *n* times | | ++----------------------------+----------------------------------+------------+ +| "s[i]" | *i*th item of *s*, origin 0 | (3)(8) | ++----------------------------+----------------------------------+------------+ +| "s[i:j]" | slice of *s* from *i* to *j* | (3)(4) | ++----------------------------+----------------------------------+------------+ +| "s[i:j:k]" | slice of *s* from *i* to *j* | (3)(5) | +| | with step *k* | | ++----------------------------+----------------------------------+------------+ +| "len(s)" | length of *s* | | ++----------------------------+----------------------------------+------------+ +| "min(s)" | smallest item of *s* | | ++----------------------------+----------------------------------+------------+ +| "max(s)" | largest item of *s* | | ++----------------------------+----------------------------------+------------+ + +Sequences of the same type also support comparisons. In particular, +tuples and lists are compared lexicographically by comparing +corresponding elements. This means that to compare equal, every +element must compare equal and the two sequences must be of the same +type and have the same length. (For full details see Comparisons in +the language reference.) + +Forward and reversed iterators over mutable sequences access values +using an index. That index will continue to march forward (or +backward) even if the underlying sequence is mutated. The iterator +terminates only when an "IndexError" or a "StopIteration" is +encountered (or when the index drops below zero). + +Notes: + +1. While the "in" and "not in" operations are used only for simple + containment testing in the general case, some specialised sequences + (such as "str", "bytes" and "bytearray") also use them for + subsequence testing: + + >>> "gg" in "eggs" + True + +2. Values of *n* less than "0" are treated as "0" (which yields an + empty sequence of the same type as *s*). Note that items in the + sequence *s* are not copied; they are referenced multiple times. + This often haunts new Python programmers; consider: + + >>> lists = [[]] * 3 + >>> lists + [[], [], []] + >>> lists[0].append(3) + >>> lists + [[3], [3], [3]] + + What has happened is that "[[]]" is a one-element list containing + an empty list, so all three elements of "[[]] * 3" are references + to this single empty list. Modifying any of the elements of + "lists" modifies this single list. You can create a list of + different lists this way: + + >>> lists = [[] for i in range(3)] + >>> lists[0].append(3) + >>> lists[1].append(5) + >>> lists[2].append(7) + >>> lists + [[3], [5], [7]] + + Further explanation is available in the FAQ entry How do I create a + multidimensional list?. + +3. If *i* or *j* is negative, the index is relative to the end of + sequence *s*: "len(s) + i" or "len(s) + j" is substituted. But + note that "-0" is still "0". + +4. The slice of *s* from *i* to *j* is defined as the sequence of + items with index *k* such that "i <= k < j". + + * If *i* is omitted or "None", use "0". + + * If *j* is omitted or "None", use "len(s)". + + * If *i* or *j* is less than "-len(s)", use "0". + + * If *i* or *j* is greater than "len(s)", use "len(s)". + + * If *i* is greater than or equal to *j*, the slice is empty. + +5. The slice of *s* from *i* to *j* with step *k* is defined as the + sequence of items with index "x = i + n*k" such that "0 <= n < + (j-i)/k". In other words, the indices are "i", "i+k", "i+2*k", + "i+3*k" and so on, stopping when *j* is reached (but never + including *j*). When *k* is positive, *i* and *j* are reduced to + "len(s)" if they are greater. When *k* is negative, *i* and *j* are + reduced to "len(s) - 1" if they are greater. If *i* or *j* are + omitted or "None", they become “end” values (which end depends on + the sign of *k*). Note, *k* cannot be zero. If *k* is "None", it + is treated like "1". + +6. Concatenating immutable sequences always results in a new object. + This means that building up a sequence by repeated concatenation + will have a quadratic runtime cost in the total sequence length. + To get a linear runtime cost, you must switch to one of the + alternatives below: + + * if concatenating "str" objects, you can build a list and use + "str.join()" at the end or else write to an "io.StringIO" + instance and retrieve its value when complete + + * if concatenating "bytes" objects, you can similarly use + "bytes.join()" or "io.BytesIO", or you can do in-place + concatenation with a "bytearray" object. "bytearray" objects are + mutable and have an efficient overallocation mechanism + + * if concatenating "tuple" objects, extend a "list" instead + + * for other types, investigate the relevant class documentation + +7. Some sequence types (such as "range") only support item sequences + that follow specific patterns, and hence don’t support sequence + concatenation or repetition. + +8. An "IndexError" is raised if *i* is outside the sequence range. + +-[ Sequence Methods ]- + +Sequence types also support the following methods: + +sequence.count(value, /) + + Return the total number of occurrences of *value* in *sequence*. + +sequence.index(value[, start[, stop]) + + Return the index of the first occurrence of *value* in *sequence*. + + Raises "ValueError" if *value* is not found in *sequence*. + + The *start* or *stop* arguments allow for efficient searching of + subsections of the sequence, beginning at *start* and ending at + *stop*. This is roughly equivalent to "start + + sequence[start:stop].index(value)", only without copying any data. + + Caution: + + Not all sequence types support passing the *start* and *stop* + arguments. + + +Immutable Sequence Types +======================== + +The only operation that immutable sequence types generally implement +that is not also implemented by mutable sequence types is support for +the "hash()" built-in. + +This support allows immutable sequences, such as "tuple" instances, to +be used as "dict" keys and stored in "set" and "frozenset" instances. + +Attempting to hash an immutable sequence that contains unhashable +values will result in "TypeError". + + +Mutable Sequence Types +====================== + +The operations in the following table are defined on mutable sequence +types. The "collections.abc.MutableSequence" ABC is provided to make +it easier to correctly implement these operations on custom sequence +types. + +In the table *s* is an instance of a mutable sequence type, *t* is any +iterable object and *x* is an arbitrary object that meets any type and +value restrictions imposed by *s* (for example, "bytearray" only +accepts integers that meet the value restriction "0 <= x <= 255"). + ++--------------------------------+----------------------------------+-----------------------+ +| Operation | Result | Notes | +|================================|==================================|=======================| +| "s[i] = x" | item *i* of *s* is replaced by | | +| | *x* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i]" | removes item *i* of *s* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j] = t" | slice of *s* from *i* to *j* is | | +| | replaced by the contents of the | | +| | iterable *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | +| | replaced by those of *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j:k]" | removes the elements of | | +| | "s[i:j:k]" from the list | | ++--------------------------------+----------------------------------+-----------------------+ +| "s += t" | extends *s* with the contents of | | +| | *t* (for the most part the same | | +| | as "s[len(s):len(s)] = t") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s *= n" | updates *s* with its contents | (2) | +| | repeated *n* times | | ++--------------------------------+----------------------------------+-----------------------+ + +Notes: + +1. If *k* is not equal to "1", *t* must have the same length as the + slice it is replacing. + +2. The value *n* is an integer, or an object implementing + "__index__()". Zero and negative values of *n* clear the sequence. + Items in the sequence are not copied; they are referenced multiple + times, as explained for "s * n" under Common Sequence Operations. + +-[ Mutable Sequence Methods ]- + +Mutable sequence types also support the following methods: + +sequence.append(value, /) + + Append *value* to the end of the sequence This is equivalent to + writing "seq[len(seq):len(seq)] = [value]". + +sequence.clear() + + Added in version 3.3. + + Remove all items from *sequence*. This is equivalent to writing + "del sequence[:]". + +sequence.copy() + + Added in version 3.3. + + Create a shallow copy of *sequence*. This is equivalent to writing + "sequence[:]". + + Hint: + + The "copy()" method is not part of the "MutableSequence" "ABC", + but most concrete mutable sequence types provide it. + +sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. For the most + part, this is the same as writing "seq[len(seq):len(seq)] = + iterable". + +sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. This is + equivalent to writing "sequence[index:index] = [value]". + +sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +sequence.remove(value, /) + + Remove the first item from *sequence* where "sequence[i] == value". + + Raises "ValueError" if *value* is not found in *sequence*. + +sequence.reverse() + + Reverse the items of *sequence* in place. This method maintains + economy of space when reversing a large sequence. To remind users + that it operates by side-effect, it returns "None". + + +Lists +===== + +Lists are mutable sequences, typically used to store collections of +homogeneous items (where the precise degree of similarity will vary by +application). + +class list(iterable=(), /) + + Lists may be constructed in several ways: + + * Using a pair of square brackets to denote the empty list: "[]" + + * Using square brackets, separating items with commas: "[a]", "[a, + b, c]" + + * Using a list comprehension: "[x for x in iterable]" + + * Using the type constructor: "list()" or "list(iterable)" + + The constructor builds a list whose items are the same and in the + same order as *iterable*’s items. *iterable* may be either a + sequence, a container that supports iteration, or an iterator + object. If *iterable* is already a list, a copy is made and + returned, similar to "iterable[:]". For example, "list('abc')" + returns "['a', 'b', 'c']" and "list( (1, 2, 3) )" returns "[1, 2, + 3]". If no argument is given, the constructor creates a new empty + list, "[]". + + Many other operations also produce lists, including the "sorted()" + built-in. + + Lists implement all of the common and mutable sequence operations. + Lists also provide the following additional method: + + sort(*, key=None, reverse=False) + + This method sorts the list in place, using only "<" comparisons + between items. Exceptions are not suppressed - if any comparison + operations fail, the entire sort operation will fail (and the + list will likely be left in a partially modified state). + + "sort()" accepts two arguments that can only be passed by + keyword (keyword-only arguments): + + *key* specifies a function of one argument that is used to + extract a comparison key from each list element (for example, + "key=str.lower"). The key corresponding to each item in the list + is calculated once and then used for the entire sorting process. + The default value of "None" means that list items are sorted + directly without calculating a separate key value. + + The "functools.cmp_to_key()" utility is available to convert a + 2.x style *cmp* function to a *key* function. + + *reverse* is a boolean value. If set to "True", then the list + elements are sorted as if each comparison were reversed. + + This method modifies the sequence in place for economy of space + when sorting a large sequence. To remind users that it operates + by side effect, it does not return the sorted sequence (use + "sorted()" to explicitly request a new sorted list instance). + + The "sort()" method is guaranteed to be stable. A sort is + stable if it guarantees not to change the relative order of + elements that compare equal — this is helpful for sorting in + multiple passes (for example, sort by department, then by salary + grade). + + For sorting examples and a brief sorting tutorial, see Sorting + Techniques. + + **CPython implementation detail:** While a list is being sorted, + the effect of attempting to mutate, or even inspect, the list is + undefined. The C implementation of Python makes the list appear + empty for the duration, and raises "ValueError" if it can detect + that the list has been mutated during a sort. + +Thread safety: Reading a single element from a "list" is *atomic*: + + lst[i] # list.__getitem__ + +The following methods traverse the list and use *atomic* reads of each +item to perform their function. That means that they may return +results affected by concurrent modifications: + + item in lst + lst.index(item) + lst.count(item) + +All of the above methods/operations are also lock-free. They do not +block concurrent modifications. Other operations that hold a lock will +not block these from observing intermediate states.All other +operations from here on block using the per-object lock.Writing a +single item via "lst[i] = x" is safe to call from multiple threads and +will not corrupt the list.The following operations return new objects +and appear *atomic* to other threads: + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + +Methods that only operate on a single elements with no shifting +required are *atomic*: + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + +The "clear()" method is also *atomic*. Other threads cannot observe +elements being removed.The "sort()" method is not *atomic*. Other +threads cannot observe intermediate states during sorting, but the +list appears empty for the duration of the sort.The following +operations may allow lock-free operations to observe intermediate +states since they modify multiple elements in place: + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + +The "remove()" method may allow concurrent modifications since element +comparison may execute arbitrary Python code (via +"__eq__()")."extend()" is safe to call from multiple threads. +However, its guarantees depend on the iterable passed to it. If it is +a "list", a "tuple", a "set", a "frozenset", a "dict" or a dictionary +view object (but not their subclasses), the "extend" operation is safe +from concurrent modifications to the iterable. Otherwise, an iterator +is created which can be concurrently modified by another thread. The +same applies to inplace concatenation of a list with other iterables +when using "lst += iterable".Similarly, assigning to a list slice with +"lst[i:j] = iterable" is safe to call from multiple threads, but +"iterable" is only locked when it is also a "list" (but not its +subclasses).Operations that involve multiple accesses, as well as +iteration, are never atomic. For example: + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + +Consider external synchronization when sharing "list" instances across +threads. See Python support for free threading for more information. + + +Tuples +====== + +Tuples are immutable sequences, typically used to store collections of +heterogeneous data (such as the 2-tuples produced by the "enumerate()" +built-in). Tuples are also used for cases where an immutable sequence +of homogeneous data is needed (such as allowing storage in a "set" or +"dict" instance). + +class tuple(iterable=(), /) + + Tuples may be constructed in a number of ways: + + * Using a pair of parentheses to denote the empty tuple: "()" + + * Using a trailing comma for a singleton tuple: "a," or "(a,)" + + * Separating items with commas: "a, b, c" or "(a, b, c)" + + * Using the "tuple()" built-in: "tuple()" or "tuple(iterable)" + + The constructor builds a tuple whose items are the same and in the + same order as *iterable*’s items. *iterable* may be either a + sequence, a container that supports iteration, or an iterator + object. If *iterable* is already a tuple, it is returned + unchanged. For example, "tuple('abc')" returns "('a', 'b', 'c')" + and "tuple( [1, 2, 3] )" returns "(1, 2, 3)". If no argument is + given, the constructor creates a new empty tuple, "()". + + Note that it is actually the comma which makes a tuple, not the + parentheses. The parentheses are optional, except in the empty + tuple case, or when they are needed to avoid syntactic ambiguity. + For example, "f(a, b, c)" is a function call with three arguments, + while "f((a, b, c))" is a function call with a 3-tuple as the sole + argument. + + Tuples implement all of the common sequence operations. + +For heterogeneous collections of data where access by name is clearer +than access by index, "collections.namedtuple()" may be a more +appropriate choice than a simple tuple object. + + +Ranges +====== + +The "range" type represents an immutable sequence of numbers and is +commonly used for looping a specific number of times in "for" loops. + +class range(stop, /) +class range(start, stop, step=1, /) + + The arguments to the range constructor must be integers (either + built-in "int" or any object that implements the "__index__()" + special method). If the *step* argument is omitted, it defaults to + "1". If the *start* argument is omitted, it defaults to "0". If + *step* is zero, "ValueError" is raised. + + For a positive *step*, the contents of a range "r" are determined + by the formula "r[i] = start + step*i" where "i >= 0" and "r[i] < + stop". + + For a negative *step*, the contents of the range are still + determined by the formula "r[i] = start + step*i", but the + constraints are "i >= 0" and "r[i] > stop". + + A range object will be empty if "r[0]" does not meet the value + constraint. Ranges do support negative indices, but these are + interpreted as indexing from the end of the sequence determined by + the positive indices. + + Ranges containing absolute values larger than "sys.maxsize" are + permitted but some features (such as "len()") may raise + "OverflowError". + + Range examples: + + >>> list(range(10)) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> list(range(1, 11)) + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + >>> list(range(0, 30, 5)) + [0, 5, 10, 15, 20, 25] + >>> list(range(0, 10, 3)) + [0, 3, 6, 9] + >>> list(range(0, -10, -1)) + [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] + >>> list(range(0)) + [] + >>> list(range(1, 0)) + [] + + Ranges implement all of the common sequence operations except + concatenation and repetition (due to the fact that range objects + can only represent sequences that follow a strict pattern and + repetition and concatenation will usually violate that pattern). + + start + + The value of the *start* parameter (or "0" if the parameter was + not supplied) + + stop + + The value of the *stop* parameter + + step + + The value of the *step* parameter (or "1" if the parameter was + not supplied) + +The advantage of the "range" type over a regular "list" or "tuple" is +that a "range" object will always take the same (small) amount of +memory, no matter the size of the range it represents (as it only +stores the "start", "stop" and "step" values, calculating individual +items and subranges as needed). + +Range objects implement the "collections.abc.Sequence" ABC, and +provide features such as containment tests, element index lookup, +slicing and support for negative indices (see Sequence Types — list, +tuple, range): + +>>> r = range(0, 20, 2) +>>> r +range(0, 20, 2) +>>> 11 in r +False +>>> 10 in r +True +>>> r.index(10) +5 +>>> r[5] +10 +>>> r[:5] +range(0, 10, 2) +>>> r[-1] +18 + +Testing range objects for equality with "==" and "!=" compares them as +sequences. That is, two range objects are considered equal if they +represent the same sequence of values. (Note that two range objects +that compare equal might have different "start", "stop" and "step" +attributes, for example "range(0) == range(2, 1, 3)" or "range(0, 3, +2) == range(0, 4, 2)".) + +Changed in version 3.2: Implement the Sequence ABC. Support slicing +and negative indices. Test "int" objects for membership in constant +time instead of iterating through all items. + +Changed in version 3.3: Define ‘==’ and ‘!=’ to compare range objects +based on the sequence of values they define (instead of comparing +based on object identity).Added the "start", "stop" and "step" +attributes. + +See also: + + * The linspace recipe shows how to implement a lazy version of range + suitable for floating-point applications. +''', + 'typesseq-mutable': r'''Mutable Sequence Types +********************** + +The operations in the following table are defined on mutable sequence +types. The "collections.abc.MutableSequence" ABC is provided to make +it easier to correctly implement these operations on custom sequence +types. + +In the table *s* is an instance of a mutable sequence type, *t* is any +iterable object and *x* is an arbitrary object that meets any type and +value restrictions imposed by *s* (for example, "bytearray" only +accepts integers that meet the value restriction "0 <= x <= 255"). + ++--------------------------------+----------------------------------+-----------------------+ +| Operation | Result | Notes | +|================================|==================================|=======================| +| "s[i] = x" | item *i* of *s* is replaced by | | +| | *x* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i]" | removes item *i* of *s* | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j] = t" | slice of *s* from *i* to *j* is | | +| | replaced by the contents of the | | +| | iterable *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | +| | replaced by those of *t* | | ++--------------------------------+----------------------------------+-----------------------+ +| "del s[i:j:k]" | removes the elements of | | +| | "s[i:j:k]" from the list | | ++--------------------------------+----------------------------------+-----------------------+ +| "s += t" | extends *s* with the contents of | | +| | *t* (for the most part the same | | +| | as "s[len(s):len(s)] = t") | | ++--------------------------------+----------------------------------+-----------------------+ +| "s *= n" | updates *s* with its contents | (2) | +| | repeated *n* times | | ++--------------------------------+----------------------------------+-----------------------+ + +Notes: + +1. If *k* is not equal to "1", *t* must have the same length as the + slice it is replacing. + +2. The value *n* is an integer, or an object implementing + "__index__()". Zero and negative values of *n* clear the sequence. + Items in the sequence are not copied; they are referenced multiple + times, as explained for "s * n" under Common Sequence Operations. + +-[ Mutable Sequence Methods ]- + +Mutable sequence types also support the following methods: + +sequence.append(value, /) + + Append *value* to the end of the sequence This is equivalent to + writing "seq[len(seq):len(seq)] = [value]". + +sequence.clear() + + Added in version 3.3. + + Remove all items from *sequence*. This is equivalent to writing + "del sequence[:]". + +sequence.copy() + + Added in version 3.3. + + Create a shallow copy of *sequence*. This is equivalent to writing + "sequence[:]". + + Hint: + + The "copy()" method is not part of the "MutableSequence" "ABC", + but most concrete mutable sequence types provide it. + +sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. For the most + part, this is the same as writing "seq[len(seq):len(seq)] = + iterable". + +sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. This is + equivalent to writing "sequence[index:index] = [value]". + +sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +sequence.remove(value, /) + + Remove the first item from *sequence* where "sequence[i] == value". + + Raises "ValueError" if *value* is not found in *sequence*. + +sequence.reverse() + + Reverse the items of *sequence* in place. This method maintains + economy of space when reversing a large sequence. To remind users + that it operates by side-effect, it returns "None". +''', + 'unary': r'''Unary arithmetic and bitwise operations +*************************************** + +All unary arithmetic and bitwise operations have the same priority: + + u_expr: power | "-" u_expr | "+" u_expr | "~" u_expr + +The unary "-" (minus) operator yields the negation of its numeric +argument; the operation can be overridden with the "__neg__()" special +method. + +The unary "+" (plus) operator yields its numeric argument unchanged; +the operation can be overridden with the "__pos__()" special method. + +The unary "~" (invert) operator yields the bitwise inversion of its +integer argument. The bitwise inversion of "x" is defined as +"-(x+1)". It only applies to integral numbers or to custom objects +that override the "__invert__()" special method. + +In all three cases, if the argument does not have the proper type, a +"TypeError" exception is raised. +''', + 'while': r'''The "while" statement +********************* + +The "while" statement is used for repeated execution as long as an +expression is true: + + while_stmt: "while" assignment_expression ":" suite + ["else" ":" suite] + +This repeatedly tests the expression and, if it is true, executes the +first suite; if the expression is false (which may be the first time +it is tested) the suite of the "else" clause, if present, is executed +and the loop terminates. + +A "break" statement executed in the first suite terminates the loop +without executing the "else" clause’s suite. A "continue" statement +executed in the first suite skips the rest of the suite and goes back +to testing the expression. +''', + 'with': r'''The "with" statement +******************** + +The "with" statement is used to wrap the execution of a block with +methods defined by a context manager (see section With Statement +Context Managers). This allows common "try"…"except"…"finally" usage +patterns to be encapsulated for convenient reuse. + + with_stmt: "with" ( "(" with_stmt_contents ","? ")" | with_stmt_contents ) ":" suite + with_stmt_contents: with_item ("," with_item)* + with_item: expression ["as" target] + +The execution of the "with" statement with one “item” proceeds as +follows: + +1. The context expression (the expression given in the "with_item") is + evaluated to obtain a context manager. + +2. The context manager’s "__enter__()" is loaded for later use. + +3. The context manager’s "__exit__()" is loaded for later use. + +4. The context manager’s "__enter__()" method is invoked. + +5. If a target was included in the "with" statement, the return value + from "__enter__()" is assigned to it. + + Note: + + The "with" statement guarantees that if the "__enter__()" method + returns without an error, then "__exit__()" will always be + called. Thus, if an error occurs during the assignment to the + target list, it will be treated the same as an error occurring + within the suite would be. See step 7 below. + +6. The suite is executed. + +7. The context manager’s "__exit__()" method is invoked. If an + exception caused the suite to be exited, its type, value, and + traceback are passed as arguments to "__exit__()". Otherwise, three + "None" arguments are supplied. + + If the suite was exited due to an exception, and the return value + from the "__exit__()" method was false, the exception is reraised. + If the return value was true, the exception is suppressed, and + execution continues with the statement following the "with" + statement. + + If the suite was exited for any reason other than an exception, the + return value from "__exit__()" is ignored, and execution proceeds + at the normal location for the kind of exit that was taken. + +The following code: + + with EXPRESSION as TARGET: + SUITE + +is semantically equivalent to: + + manager = (EXPRESSION) + enter = type(manager).__enter__ + exit = type(manager).__exit__ + value = enter(manager) + hit_except = False + + try: + TARGET = value + SUITE + except: + hit_except = True + if not exit(manager, *sys.exc_info()): + raise + finally: + if not hit_except: + exit(manager, None, None, None) + +With more than one item, the context managers are processed as if +multiple "with" statements were nested: + + with A() as a, B() as b: + SUITE + +is semantically equivalent to: + + with A() as a: + with B() as b: + SUITE + +You can also write multi-item context managers in multiple lines if +the items are surrounded by parentheses. For example: + + with ( + A() as a, + B() as b, + ): + SUITE + +Changed in version 3.1: Support for multiple context expressions. + +Changed in version 3.10: Support for using grouping parentheses to +break the statement in multiple lines. + +See also: + + **PEP 343** - The “with” statement + The specification, background, and examples for the Python "with" + statement. +''', + 'yield': r'''The "yield" statement +********************* + + yield_stmt: yield_expression + +A "yield" statement is semantically equivalent to a yield expression. +The "yield" statement can be used to omit the parentheses that would +otherwise be required in the equivalent yield expression statement. +For example, the yield statements + + yield + yield from + +are equivalent to the yield expression statements + + (yield ) + (yield from ) + +Yield expressions and statements are only used when defining a +*generator* function, and are only used in the body of the generator +function. Using "yield" in a function definition is sufficient to +cause that definition to create a generator function instead of a +normal function. + +For full details of "yield" semantics, refer to the Yield expressions +section. +''', +} diff --git a/Lib/queue.py b/Lib/queue.py index 55f50088460..c0b35987654 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -10,7 +10,15 @@ except ImportError: SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] +__all__ = [ + 'Empty', + 'Full', + 'ShutDown', + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'SimpleQueue', +] try: @@ -25,6 +33,10 @@ class Full(Exception): pass +class ShutDown(Exception): + '''Raised when put/get with shut-down queue.''' + + class Queue: '''Create a queue object with a given maximum size. @@ -54,6 +66,9 @@ def __init__(self, maxsize=0): self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 + # Queue shutdown state + self.is_shutdown = False + def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -129,8 +144,12 @@ def put(self, item, block=True, timeout=None): Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down. ''' with self.not_full: + if self.is_shutdown: + raise ShutDown if self.maxsize > 0: if not block: if self._qsize() >= self.maxsize: @@ -138,6 +157,8 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() + if self.is_shutdown: + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -147,6 +168,8 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) + if self.is_shutdown: + raise ShutDown self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() @@ -161,14 +184,21 @@ def get(self, block=True, timeout=None): Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. ''' with self.not_empty: + if self.is_shutdown and not self._qsize(): + raise ShutDown if not block: if not self._qsize(): raise Empty elif timeout is None: while not self._qsize(): self.not_empty.wait() + if self.is_shutdown and not self._qsize(): + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -178,6 +208,8 @@ def get(self, block=True, timeout=None): if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) + if self.is_shutdown and not self._qsize(): + raise ShutDown item = self._get() self.not_full.notify() return item @@ -198,6 +230,31 @@ def get_nowait(self): ''' return self.get(block=False) + def shutdown(self, immediate=False): + '''Shut-down the queue, making queue gets and puts raise ShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. + ''' + with self.mutex: + self.is_shutdown = True + if immediate: + while self._qsize(): + self._get() + if self.unfinished_tasks > 0: + self.unfinished_tasks -= 1 + # release all blocked threads in `join()` + self.all_tasks_done.notify_all() + # All getters need to re-check queue-empty to raise ShutDown + self.not_empty.notify_all() + self.not_full.notify_all() + # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held diff --git a/Lib/quopri.py b/Lib/quopri.py old mode 100755 new mode 100644 index 08899c5cb73..129fd2f5c7c --- a/Lib/quopri.py +++ b/Lib/quopri.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Conversions to/from quoted-printable transport encoding as per RFC 1521.""" # (Dec 1991 version). @@ -67,10 +65,7 @@ def write(s, output=output, lineEnd=b'\n'): output.write(s + lineEnd) prevline = None - while 1: - line = input.readline() - if not line: - break + while line := input.readline(): outline = [] # Strip off any readline induced trailing newline stripped = b'' @@ -126,9 +121,7 @@ def decode(input, output, header=False): return new = b'' - while 1: - line = input.readline() - if not line: break + while line := input.readline(): i, n = 0, len(line) if n > 0 and line[n-1:n] == b'\n': partial = 0; n = n-1 diff --git a/Lib/random.py b/Lib/random.py index 85bad08d57e..86d562f0b8a 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -32,6 +32,11 @@ circular uniform von Mises + discrete distributions + ---------------------- + binomial + + General notes on the underlying Mersenne Twister core generator: * The period is 2**19937-1. @@ -45,42 +50,23 @@ # Adrian Baddeley. Adapted by Raymond Hettinger for use with # the Mersenne Twister and os.urandom() core generators. -from warnings import warn as _warn from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin from math import tau as TWOPI, floor as _floor, isfinite as _isfinite -try: - from os import urandom as _urandom -except ImportError: - # XXX RUSTPYTHON - # On wasm, _random.Random.random() does give a proper random value, but - # we don't have the os module - def _urandom(*args, **kwargs): - raise NotImplementedError("urandom") - _os = None -from _collections_abc import Set as _Set, Sequence as _Sequence +from math import lgamma as _lgamma, fabs as _fabs, log2 as _log2 +from os import urandom as _urandom +from _collections_abc import Sequence as _Sequence from operator import index as _index from itertools import accumulate as _accumulate, repeat as _repeat from bisect import bisect as _bisect -try: - import os as _os -except ImportError: - # XXX RUSTPYTHON - # On wasm, we don't have the os module - _os = None +import os as _os import _random -try: - # hashlib is pretty heavy to load, try lean internal module first - from _sha512 import sha512 as _sha512 -except ImportError: - # fallback to official implementation - from hashlib import sha512 as _sha512 - __all__ = [ "Random", "SystemRandom", "betavariate", + "binomialvariate", "choice", "choices", "expovariate", @@ -111,6 +97,7 @@ def _urandom(*args, **kwargs): BPF = 53 # Number of bits in a float RECIP_BPF = 2 ** -BPF _ONE = 1 +_sha512 = None class Random(_random.Random): @@ -165,13 +152,23 @@ def seed(self, a=None, version=2): a = -2 if x == -1 else x elif version == 2 and isinstance(a, (str, bytes, bytearray)): + global _sha512 + if _sha512 is None: + try: + # hashlib is pretty heavy to load, try lean internal + # module first + from _sha2 import sha512 as _sha512 + except ImportError: + # fallback to official implementation + from hashlib import sha512 as _sha512 + if isinstance(a, str): a = a.encode() a = int.from_bytes(a + _sha512(a).digest()) elif not isinstance(a, (type(None), int, float, str, bytes, bytearray)): - raise TypeError('The only supported seed types are: None,\n' - 'int, float, str, bytes, and bytearray.') + raise TypeError('The only supported seed types are:\n' + 'None, int, float, str, bytes, and bytearray.') super().seed(a) self.gauss_next = None @@ -248,11 +245,10 @@ def __init_subclass__(cls, /, **kwargs): def _randbelow_with_getrandbits(self, n): "Return a random int in the range [0,n). Defined for n > 0." - getrandbits = self.getrandbits - k = n.bit_length() # don't use (n-1) here because n can be 1 - r = getrandbits(k) # 0 <= r < 2**k + k = n.bit_length() + r = self.getrandbits(k) # 0 <= r < 2**k while r >= n: - r = getrandbits(k) + r = self.getrandbits(k) return r def _randbelow_without_getrandbits(self, n, maxsize=1<= maxsize: - _warn("Underlying random() generator does not supply \n" - "enough bits to choose from a population range this large.\n" - "To remove the range limitation, add a getrandbits() method.") + from warnings import warn + warn("Underlying random() generator does not supply \n" + "enough bits to choose from a population range this large.\n" + "To remove the range limitation, add a getrandbits() method.") return _floor(random() * n) rem = maxsize % n limit = (maxsize - rem) / maxsize # int(limit * maxsize) % n == 0 @@ -304,58 +301,25 @@ def randrange(self, start, stop=None, step=_ONE): # This code is a bit messy to make it fast for the # common case while still doing adequate error checking. - try: - istart = _index(start) - except TypeError: - istart = int(start) - if istart != start: - _warn('randrange() will raise TypeError in the future', - DeprecationWarning, 2) - raise ValueError("non-integer arg 1 for randrange()") - _warn('non-integer arguments to randrange() have been deprecated ' - 'since Python 3.10 and will be removed in a subsequent ' - 'version', - DeprecationWarning, 2) + istart = _index(start) if stop is None: # We don't check for "step != 1" because it hasn't been # type checked and converted to an integer yet. if step is not _ONE: - raise TypeError('Missing a non-None stop argument') + raise TypeError("Missing a non-None stop argument") if istart > 0: return self._randbelow(istart) raise ValueError("empty range for randrange()") - # stop argument supplied. - try: - istop = _index(stop) - except TypeError: - istop = int(stop) - if istop != stop: - _warn('randrange() will raise TypeError in the future', - DeprecationWarning, 2) - raise ValueError("non-integer stop for randrange()") - _warn('non-integer arguments to randrange() have been deprecated ' - 'since Python 3.10 and will be removed in a subsequent ' - 'version', - DeprecationWarning, 2) + # Stop argument supplied. + istop = _index(stop) width = istop - istart - try: - istep = _index(step) - except TypeError: - istep = int(step) - if istep != step: - _warn('randrange() will raise TypeError in the future', - DeprecationWarning, 2) - raise ValueError("non-integer step for randrange()") - _warn('non-integer arguments to randrange() have been deprecated ' - 'since Python 3.10 and will be removed in a subsequent ' - 'version', - DeprecationWarning, 2) + istep = _index(step) # Fast path. if istep == 1: if width > 0: return istart + self._randbelow(width) - raise ValueError("empty range for randrange() (%d, %d, %d)" % (istart, istop, width)) + raise ValueError(f"empty range in randrange({start}, {stop})") # Non-unit step argument supplied. if istep > 0: @@ -365,14 +329,17 @@ def randrange(self, start, stop=None, step=_ONE): else: raise ValueError("zero step for randrange()") if n <= 0: - raise ValueError("empty range for randrange()") + raise ValueError(f"empty range in randrange({start}, {stop}, {step})") return istart + istep * self._randbelow(n) def randint(self, a, b): """Return random integer in range [a, b], including both end points. """ - - return self.randrange(a, b+1) + a = _index(a) + b = _index(b) + if b < a: + raise ValueError(f"empty range in randint({a}, {b})") + return a + self._randbelow(b - a + 1) ## -------------------- sequence methods ------------------- @@ -456,11 +423,11 @@ def sample(self, population, k, *, counts=None): cum_counts = list(_accumulate(counts)) if len(cum_counts) != n: raise ValueError('The number of counts does not match the population') - total = cum_counts.pop() + total = cum_counts.pop() if cum_counts else 0 if not isinstance(total, int): raise TypeError('Counts must be integers') - if total <= 0: - raise ValueError('Total of counts must be greater than zero') + if total < 0: + raise ValueError('Counts must be non-negative') selections = self.sample(range(total), k=k) bisect = _bisect return [population[bisect(cum_counts, s)] for s in selections] @@ -531,7 +498,14 @@ def choices(self, population, weights=None, *, cum_weights=None, k=1): ## -------------------- real-valued distributions ------------------- def uniform(self, a, b): - "Get a random number in the range [a, b) or [a, b] depending on rounding." + """Get a random number in the range [a, b) or [a, b] depending on rounding. + + The mean (expected value) and variance of the random variable are: + + E[X] = (a + b) / 2 + Var[X] = (b - a) ** 2 / 12 + + """ return a + (b - a) * self.random() def triangular(self, low=0.0, high=1.0, mode=None): @@ -542,6 +516,11 @@ def triangular(self, low=0.0, high=1.0, mode=None): http://en.wikipedia.org/wiki/Triangular_distribution + The mean (expected value) and variance of the random variable are: + + E[X] = (low + high + mode) / 3 + Var[X] = (low**2 + high**2 + mode**2 - low*high - low*mode - high*mode) / 18 + """ u = self.random() try: @@ -623,7 +602,7 @@ def lognormvariate(self, mu, sigma): """ return _exp(self.normalvariate(mu, sigma)) - def expovariate(self, lambd): + def expovariate(self, lambd=1.0): """Exponential distribution. lambd is 1.0 divided by the desired mean. It should be @@ -632,12 +611,15 @@ def expovariate(self, lambd): positive infinity if lambd is positive, and from negative infinity to 0 if lambd is negative. - """ - # lambd: rate lambd = 1/mean - # ('lambda' is a Python reserved word) + The mean (expected value) and variance of the random variable are: + E[X] = 1 / lambd + Var[X] = 1 / lambd ** 2 + + """ # we use 1-random() instead of random() to preclude the # possibility of taking the log of zero. + return -_log(1.0 - self.random()) / lambd def vonmisesvariate(self, mu, kappa): @@ -693,8 +675,12 @@ def gammavariate(self, alpha, beta): pdf(x) = -------------------------------------- math.gamma(alpha) * beta ** alpha + The mean (expected value) and variance of the random variable are: + + E[X] = alpha * beta + Var[X] = alpha * beta ** 2 + """ - # alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 # Warning: a few older sources define the gamma distribution in terms # of alpha > -1.0 @@ -753,6 +739,11 @@ def betavariate(self, alpha, beta): Conditions on the parameters are alpha > 0 and beta > 0. Returned values range between 0 and 1. + The mean (expected value) and variance of the random variable are: + + E[X] = alpha / (alpha + beta) + Var[X] = alpha * beta / ((alpha + beta)**2 * (alpha + beta + 1)) + """ ## See ## http://mail.python.org/pipermail/python-bugs-list/2001-January/003752.html @@ -793,6 +784,103 @@ def weibullvariate(self, alpha, beta): return alpha * (-_log(u)) ** (1.0 / beta) + ## -------------------- discrete distributions --------------------- + + def binomialvariate(self, n=1, p=0.5): + """Binomial random variable. + + Gives the number of successes for *n* independent trials + with the probability of success in each trial being *p*: + + sum(random() < p for i in range(n)) + + Returns an integer in the range: + + 0 <= X <= n + + The integer is chosen with the probability: + + P(X == k) = math.comb(n, k) * p ** k * (1 - p) ** (n - k) + + The mean (expected value) and variance of the random variable are: + + E[X] = n * p + Var[X] = n * p * (1 - p) + + """ + # Error check inputs and handle edge cases + if n < 0: + raise ValueError("n must be non-negative") + if p <= 0.0 or p >= 1.0: + if p == 0.0: + return 0 + if p == 1.0: + return n + raise ValueError("p must be in the range 0.0 <= p <= 1.0") + + random = self.random + + # Fast path for a common case + if n == 1: + return _index(random() < p) + + # Exploit symmetry to establish: p <= 0.5 + if p > 0.5: + return n - self.binomialvariate(n, 1.0 - p) + + if n * p < 10.0: + # BG: Geometric method by Devroye with running time of O(np). + # https://dl.acm.org/doi/pdf/10.1145/42372.42381 + x = y = 0 + c = _log2(1.0 - p) + if not c: + return x + while True: + y += _floor(_log2(random()) / c) + 1 + if y > n: + return x + x += 1 + + # BTRS: Transformed rejection with squeeze method by Wolfgang Hörmann + # https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.47.8407&rep=rep1&type=pdf + assert n*p >= 10.0 and p <= 0.5 + setup_complete = False + + spq = _sqrt(n * p * (1.0 - p)) # Standard deviation of the distribution + b = 1.15 + 2.53 * spq + a = -0.0873 + 0.0248 * b + 0.01 * p + c = n * p + 0.5 + vr = 0.92 - 4.2 / b + + while True: + + u = random() + u -= 0.5 + us = 0.5 - _fabs(u) + k = _floor((2.0 * a / us + b) * u + c) + if k < 0 or k > n: + continue + + # The early-out "squeeze" test substantially reduces + # the number of acceptance condition evaluations. + v = random() + if us >= 0.07 and v <= vr: + return k + + # Acceptance-rejection test. + # Note, the original paper erroneously omits the call to log(v) + # when comparing to the log of the rescaled binomial distribution. + if not setup_complete: + alpha = (2.83 + 5.1 / b) * spq + lpq = _log(p / (1.0 - p)) + m = _floor((n + 1) * p) # Mode of the distribution + h = _lgamma(m + 1) + _lgamma(n - m + 1) + setup_complete = True # Only needs to be done once + v *= alpha / (a / (us * us) + b) + if _log(v) <= h - _lgamma(k + 1) - _lgamma(n - k + 1) + (k - m) * lpq: + return k + + ## ------------------------------------------------------------------ ## --------------- Operating System Random Source ------------------ @@ -859,6 +947,7 @@ def _notimplemented(self, *args, **kwds): gammavariate = _inst.gammavariate gauss = _inst.gauss betavariate = _inst.betavariate +binomialvariate = _inst.binomialvariate paretovariate = _inst.paretovariate weibullvariate = _inst.weibullvariate getstate = _inst.getstate @@ -883,15 +972,17 @@ def _test_generator(n, func, args): low = min(data) high = max(data) - print(f'{t1 - t0:.3f} sec, {n} times {func.__name__}') + print(f'{t1 - t0:.3f} sec, {n} times {func.__name__}{args!r}') print('avg %g, stddev %g, min %g, max %g\n' % (xbar, sigma, low, high)) -def _test(N=2000): +def _test(N=10_000): _test_generator(N, random, ()) _test_generator(N, normalvariate, (0.0, 1.0)) _test_generator(N, lognormvariate, (0.0, 1.0)) _test_generator(N, vonmisesvariate, (0.0, 1.0)) + _test_generator(N, binomialvariate, (15, 0.60)) + _test_generator(N, binomialvariate, (100, 0.75)) _test_generator(N, gammavariate, (0.01, 1.0)) _test_generator(N, gammavariate, (0.1, 1.0)) _test_generator(N, gammavariate, (0.1, 2.0)) @@ -913,5 +1004,75 @@ def _test(N=2000): _os.register_at_fork(after_in_child=_inst.seed) +# ------------------------------------------------------ +# -------------- command-line interface ---------------- + + +def _parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, color=True) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-c", "--choice", nargs="+", + help="print a random choice") + group.add_argument( + "-i", "--integer", type=int, metavar="N", + help="print a random integer between 1 and N inclusive") + group.add_argument( + "-f", "--float", type=float, metavar="N", + help="print a random floating-point number between 0 and N inclusive") + group.add_argument( + "--test", type=int, const=10_000, nargs="?", + help=argparse.SUPPRESS) + parser.add_argument("input", nargs="*", + help="""\ +if no options given, output depends on the input + string or multiple: same as --choice + integer: same as --integer + float: same as --float""") + args = parser.parse_args(arg_list) + return args, parser.format_help() + + +def main(arg_list: list[str] | None = None) -> int | str: + args, help_text = _parse_args(arg_list) + + # Explicit arguments + if args.choice: + return choice(args.choice) + + if args.integer is not None: + return randint(1, args.integer) + + if args.float is not None: + return uniform(0, args.float) + + if args.test: + _test(args.test) + return "" + + # No explicit argument, select based on input + if len(args.input) == 1: + val = args.input[0] + try: + # Is it an integer? + val = int(val) + return randint(1, val) + except ValueError: + try: + # Is it a float? + val = float(val) + return uniform(0, val) + except ValueError: + # Split in case of space-separated string: "a b c" + return choice(val.split()) + + if len(args.input) >= 2: + return choice(args.input) + + return help_text + + if __name__ == '__main__': - _test() + print(main()) diff --git a/Lib/re.py b/Lib/re.py deleted file mode 100644 index bfb7b1ccd93..00000000000 --- a/Lib/re.py +++ /dev/null @@ -1,384 +0,0 @@ -# -# Secret Labs' Regular Expression Engine -# -# re-compatible interface for the sre matching engine -# -# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. -# -# This version of the SRE library can be redistributed under CNRI's -# Python 1.6 license. For any other use, please contact Secret Labs -# AB (info@pythonware.com). -# -# Portions of this engine have been developed in cooperation with -# CNRI. Hewlett-Packard provided funding for 1.6 integration and -# other compatibility work. -# - -r"""Support for regular expressions (RE). - -This module provides regular expression matching operations similar to -those found in Perl. It supports both 8-bit and Unicode strings; both -the pattern and the strings being processed can contain null bytes and -characters outside the US ASCII range. - -Regular expressions can contain both special and ordinary characters. -Most ordinary characters, like "A", "a", or "0", are the simplest -regular expressions; they simply match themselves. You can -concatenate ordinary characters, so last matches the string 'last'. - -The special characters are: - "." Matches any character except a newline. - "^" Matches the start of the string. - "$" Matches the end of the string or just before the newline at - the end of the string. - "*" Matches 0 or more (greedy) repetitions of the preceding RE. - Greedy means that it will match as many repetitions as possible. - "+" Matches 1 or more (greedy) repetitions of the preceding RE. - "?" Matches 0 or 1 (greedy) of the preceding RE. - *?,+?,?? Non-greedy versions of the previous three special characters. - {m,n} Matches from m to n repetitions of the preceding RE. - {m,n}? Non-greedy version of the above. - "\\" Either escapes special characters or signals a special sequence. - [] Indicates a set of characters. - A "^" as the first character indicates a complementing set. - "|" A|B, creates an RE that will match either A or B. - (...) Matches the RE inside the parentheses. - The contents can be retrieved or matched later in the string. - (?aiLmsux) The letters set the corresponding flags defined below. - (?:...) Non-grouping version of regular parentheses. - (?P...) The substring matched by the group is accessible by name. - (?P=name) Matches the text matched earlier by the group named name. - (?#...) A comment; ignored. - (?=...) Matches if ... matches next, but doesn't consume the string. - (?!...) Matches if ... doesn't match next. - (?<=...) Matches if preceded by ... (must be fixed length). - (? 1: - res = f'~({res})' - else: - res = f'~{res}' - return res - __str__ = object.__str__ - -globals().update(RegexFlag.__members__) - -# sre exception -error = sre_compile.error - -# -------------------------------------------------------------------- -# public interface - -def match(pattern, string, flags=0): - """Try to apply the pattern at the start of the string, returning - a Match object, or None if no match was found.""" - return _compile(pattern, flags).match(string) - -def fullmatch(pattern, string, flags=0): - """Try to apply the pattern to all of the string, returning - a Match object, or None if no match was found.""" - return _compile(pattern, flags).fullmatch(string) - -def search(pattern, string, flags=0): - """Scan through string looking for a match to the pattern, returning - a Match object, or None if no match was found.""" - return _compile(pattern, flags).search(string) - -def sub(pattern, repl, string, count=0, flags=0): - """Return the string obtained by replacing the leftmost - non-overlapping occurrences of the pattern in string by the - replacement repl. repl can be either a string or a callable; - if a string, backslash escapes in it are processed. If it is - a callable, it's passed the Match object and must return - a replacement string to be used.""" - return _compile(pattern, flags).sub(repl, string, count) - -def subn(pattern, repl, string, count=0, flags=0): - """Return a 2-tuple containing (new_string, number). - new_string is the string obtained by replacing the leftmost - non-overlapping occurrences of the pattern in the source - string by the replacement repl. number is the number of - substitutions that were made. repl can be either a string or a - callable; if a string, backslash escapes in it are processed. - If it is a callable, it's passed the Match object and must - return a replacement string to be used.""" - return _compile(pattern, flags).subn(repl, string, count) - -def split(pattern, string, maxsplit=0, flags=0): - """Split the source string by the occurrences of the pattern, - returning a list containing the resulting substrings. If - capturing parentheses are used in pattern, then the text of all - groups in the pattern are also returned as part of the resulting - list. If maxsplit is nonzero, at most maxsplit splits occur, - and the remainder of the string is returned as the final element - of the list.""" - return _compile(pattern, flags).split(string, maxsplit) - -def findall(pattern, string, flags=0): - """Return a list of all non-overlapping matches in the string. - - If one or more capturing groups are present in the pattern, return - a list of groups; this will be a list of tuples if the pattern - has more than one group. - - Empty matches are included in the result.""" - return _compile(pattern, flags).findall(string) - -def finditer(pattern, string, flags=0): - """Return an iterator over all non-overlapping matches in the - string. For each match, the iterator returns a Match object. - - Empty matches are included in the result.""" - return _compile(pattern, flags).finditer(string) - -def compile(pattern, flags=0): - "Compile a regular expression pattern, returning a Pattern object." - return _compile(pattern, flags) - -def purge(): - "Clear the regular expression caches" - _cache.clear() - _compile_repl.cache_clear() - -def template(pattern, flags=0): - "Compile a template pattern, returning a Pattern object" - return _compile(pattern, flags|T) - -# SPECIAL_CHARS -# closing ')', '}' and ']' -# '-' (a range in character set) -# '&', '~', (extended character set operations) -# '#' (comment) and WHITESPACE (ignored) in verbose mode -_special_chars_map = {i: '\\' + chr(i) for i in b'()[]{}?*+-|^$\\.&~# \t\n\r\v\f'} - -def escape(pattern): - """ - Escape special characters in a string. - """ - if isinstance(pattern, str): - return pattern.translate(_special_chars_map) - else: - pattern = str(pattern, 'latin1') - return pattern.translate(_special_chars_map).encode('latin1') - -Pattern = type(sre_compile.compile('', 0)) -Match = type(sre_compile.compile('', 0).match('')) - -# -------------------------------------------------------------------- -# internals - -_cache = {} # ordered! - -_MAXCACHE = 512 -def _compile(pattern, flags): - # internal: compile pattern - if isinstance(flags, RegexFlag): - flags = flags.value - try: - return _cache[type(pattern), pattern, flags] - except KeyError: - pass - if isinstance(pattern, Pattern): - if flags: - raise ValueError( - "cannot process flags argument with a compiled pattern") - return pattern - if not sre_compile.isstring(pattern): - raise TypeError("first argument must be string or compiled pattern") - p = sre_compile.compile(pattern, flags) - if not (flags & DEBUG): - if len(_cache) >= _MAXCACHE: - # Drop the oldest item - try: - del _cache[next(iter(_cache))] - except (StopIteration, RuntimeError, KeyError): - pass - _cache[type(pattern), pattern, flags] = p - return p - -@functools.lru_cache(_MAXCACHE) -def _compile_repl(repl, pattern): - # internal: compile replacement pattern - return sre_parse.parse_template(repl, pattern) - -def _expand(pattern, match, template): - # internal: Match.expand implementation hook - template = sre_parse.parse_template(template, pattern) - return sre_parse.expand_template(template, match) - -def _subx(pattern, template): - # internal: Pattern.sub/subn implementation helper - template = _compile_repl(template, pattern) - if not template[0] and len(template[1]) == 1: - # literal replacement - return template[1][0] - def filter(match, template=template): - return sre_parse.expand_template(template, match) - return filter - -# register myself for pickling - -import copyreg - -def _pickle(p): - return _compile, (p.pattern, p.flags) - -copyreg.pickle(Pattern, _pickle, _compile) - -# -------------------------------------------------------------------- -# experimental stuff (see python-dev discussions for details) - -class Scanner: - def __init__(self, lexicon, flags=0): - from sre_constants import BRANCH, SUBPATTERN - if isinstance(flags, RegexFlag): - flags = flags.value - self.lexicon = lexicon - # combine phrases into a compound pattern - p = [] - s = sre_parse.State() - s.flags = flags - for phrase, action in lexicon: - gid = s.opengroup() - p.append(sre_parse.SubPattern(s, [ - (SUBPATTERN, (gid, 0, 0, sre_parse.parse(phrase, flags))), - ])) - s.closegroup(gid, p[-1]) - p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) - self.scanner = sre_compile.compile(p) - def scan(self, string): - result = [] - append = result.append - match = self.scanner.scanner(string).match - i = 0 - while True: - m = match() - if not m: - break - j = m.end() - if i == j: - break - action = self.lexicon[m.lastindex-1][1] - if callable(action): - self.match = m - action = action(self, m.group()) - if action is not None: - append(action) - i = j - return result, string[i:] diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py new file mode 100644 index 00000000000..af2808a77da --- /dev/null +++ b/Lib/re/__init__.py @@ -0,0 +1,428 @@ +# +# Secret Labs' Regular Expression Engine +# +# re-compatible interface for the sre matching engine +# +# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. +# +# This version of the SRE library can be redistributed under CNRI's +# Python 1.6 license. For any other use, please contact Secret Labs +# AB (info@pythonware.com). +# +# Portions of this engine have been developed in cooperation with +# CNRI. Hewlett-Packard provided funding for 1.6 integration and +# other compatibility work. +# + +r"""Support for regular expressions (RE). + +This module provides regular expression matching operations similar to +those found in Perl. It supports both 8-bit and Unicode strings; both +the pattern and the strings being processed can contain null bytes and +characters outside the US ASCII range. + +Regular expressions can contain both special and ordinary characters. +Most ordinary characters, like "A", "a", or "0", are the simplest +regular expressions; they simply match themselves. You can +concatenate ordinary characters, so last matches the string 'last'. + +The special characters are: + "." Matches any character except a newline. + "^" Matches the start of the string. + "$" Matches the end of the string or just before the newline at + the end of the string. + "*" Matches 0 or more (greedy) repetitions of the preceding RE. + Greedy means that it will match as many repetitions as possible. + "+" Matches 1 or more (greedy) repetitions of the preceding RE. + "?" Matches 0 or 1 (greedy) of the preceding RE. + *?,+?,?? Non-greedy versions of the previous three special characters. + {m,n} Matches from m to n repetitions of the preceding RE. + {m,n}? Non-greedy version of the above. + "\\" Either escapes special characters or signals a special sequence. + [] Indicates a set of characters. + A "^" as the first character indicates a complementing set. + "|" A|B, creates an RE that will match either A or B. + (...) Matches the RE inside the parentheses. + The contents can be retrieved or matched later in the string. + (?aiLmsux) The letters set the corresponding flags defined below. + (?:...) Non-grouping version of regular parentheses. + (?P...) The substring matched by the group is accessible by name. + (?P=name) Matches the text matched earlier by the group named name. + (?#...) A comment; ignored. + (?=...) Matches if ... matches next, but doesn't consume the string. + (?!...) Matches if ... doesn't match next. + (?<=...) Matches if preceded by ... (must be fixed length). + (?= _MAXCACHE: + # Drop the least recently used item. + # next(iter(_cache)) is known to have linear amortized time, + # but it is used here to avoid a dependency from using OrderedDict. + # For the small _MAXCACHE value it doesn't make much of a difference. + try: + del _cache[next(iter(_cache))] + except (StopIteration, RuntimeError, KeyError): + pass + # Append to the end. + _cache[key] = p + + if len(_cache2) >= _MAXCACHE2: + # Drop the oldest item. + try: + del _cache2[next(iter(_cache2))] + except (StopIteration, RuntimeError, KeyError): + pass + _cache2[key] = p + return p + +@functools.lru_cache(_MAXCACHE) +def _compile_template(pattern, repl): + # internal: compile replacement pattern + return _sre.template(pattern, _parser.parse_template(repl, pattern)) + +# register myself for pickling + +import copyreg + +def _pickle(p): + return _compile, (p.pattern, p.flags) + +copyreg.pickle(Pattern, _pickle, _compile) + +# -------------------------------------------------------------------- +# experimental stuff (see python-dev discussions for details) + +class Scanner: + def __init__(self, lexicon, flags=0): + from ._constants import BRANCH, SUBPATTERN + if isinstance(flags, RegexFlag): + flags = flags.value + self.lexicon = lexicon + # combine phrases into a compound pattern + p = [] + s = _parser.State() + s.flags = flags + for phrase, action in lexicon: + gid = s.opengroup() + p.append(_parser.SubPattern(s, [ + (SUBPATTERN, (gid, 0, 0, _parser.parse(phrase, flags))), + ])) + s.closegroup(gid, p[-1]) + p = _parser.SubPattern(s, [(BRANCH, (None, p))]) + self.scanner = _compiler.compile(p) + def scan(self, string): + result = [] + append = result.append + match = self.scanner.scanner(string).match + i = 0 + while True: + m = match() + if not m: + break + j = m.end() + if i == j: + break + action = self.lexicon[m.lastindex-1][1] + if callable(action): + self.match = m + action = action(self, m.group()) + if action is not None: + append(action) + i = j + return result, string[i:] diff --git a/Lib/re/_casefix.py b/Lib/re/_casefix.py new file mode 100644 index 00000000000..fed2d84fc01 --- /dev/null +++ b/Lib/re/_casefix.py @@ -0,0 +1,106 @@ +# Auto-generated by Tools/build/generate_re_casefix.py. + +# Maps the code of lowercased character to codes of different lowercased +# characters which have the same uppercase. +_EXTRA_CASES = { + # LATIN SMALL LETTER I: LATIN SMALL LETTER DOTLESS I + 0x0069: (0x0131,), # 'i': 'ı' + # LATIN SMALL LETTER S: LATIN SMALL LETTER LONG S + 0x0073: (0x017f,), # 's': 'ſ' + # MICRO SIGN: GREEK SMALL LETTER MU + 0x00b5: (0x03bc,), # 'µ': 'μ' + # LATIN SMALL LETTER DOTLESS I: LATIN SMALL LETTER I + 0x0131: (0x0069,), # 'ı': 'i' + # LATIN SMALL LETTER LONG S: LATIN SMALL LETTER S + 0x017f: (0x0073,), # 'ſ': 's' + # COMBINING GREEK YPOGEGRAMMENI: GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI + 0x0345: (0x03b9, 0x1fbe), # '\u0345': 'ιι' + # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + 0x0390: (0x1fd3,), # 'ΐ': 'ΐ' + # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + 0x03b0: (0x1fe3,), # 'ΰ': 'ΰ' + # GREEK SMALL LETTER BETA: GREEK BETA SYMBOL + 0x03b2: (0x03d0,), # 'β': 'ϐ' + # GREEK SMALL LETTER EPSILON: GREEK LUNATE EPSILON SYMBOL + 0x03b5: (0x03f5,), # 'ε': 'ϵ' + # GREEK SMALL LETTER THETA: GREEK THETA SYMBOL + 0x03b8: (0x03d1,), # 'θ': 'ϑ' + # GREEK SMALL LETTER IOTA: COMBINING GREEK YPOGEGRAMMENI, GREEK PROSGEGRAMMENI + 0x03b9: (0x0345, 0x1fbe), # 'ι': '\u0345ι' + # GREEK SMALL LETTER KAPPA: GREEK KAPPA SYMBOL + 0x03ba: (0x03f0,), # 'κ': 'ϰ' + # GREEK SMALL LETTER MU: MICRO SIGN + 0x03bc: (0x00b5,), # 'μ': 'µ' + # GREEK SMALL LETTER PI: GREEK PI SYMBOL + 0x03c0: (0x03d6,), # 'π': 'ϖ' + # GREEK SMALL LETTER RHO: GREEK RHO SYMBOL + 0x03c1: (0x03f1,), # 'ρ': 'ϱ' + # GREEK SMALL LETTER FINAL SIGMA: GREEK SMALL LETTER SIGMA + 0x03c2: (0x03c3,), # 'ς': 'σ' + # GREEK SMALL LETTER SIGMA: GREEK SMALL LETTER FINAL SIGMA + 0x03c3: (0x03c2,), # 'σ': 'ς' + # GREEK SMALL LETTER PHI: GREEK PHI SYMBOL + 0x03c6: (0x03d5,), # 'φ': 'ϕ' + # GREEK BETA SYMBOL: GREEK SMALL LETTER BETA + 0x03d0: (0x03b2,), # 'ϐ': 'β' + # GREEK THETA SYMBOL: GREEK SMALL LETTER THETA + 0x03d1: (0x03b8,), # 'ϑ': 'θ' + # GREEK PHI SYMBOL: GREEK SMALL LETTER PHI + 0x03d5: (0x03c6,), # 'ϕ': 'φ' + # GREEK PI SYMBOL: GREEK SMALL LETTER PI + 0x03d6: (0x03c0,), # 'ϖ': 'π' + # GREEK KAPPA SYMBOL: GREEK SMALL LETTER KAPPA + 0x03f0: (0x03ba,), # 'ϰ': 'κ' + # GREEK RHO SYMBOL: GREEK SMALL LETTER RHO + 0x03f1: (0x03c1,), # 'ϱ': 'ρ' + # GREEK LUNATE EPSILON SYMBOL: GREEK SMALL LETTER EPSILON + 0x03f5: (0x03b5,), # 'ϵ': 'ε' + # CYRILLIC SMALL LETTER VE: CYRILLIC SMALL LETTER ROUNDED VE + 0x0432: (0x1c80,), # 'в': 'ᲀ' + # CYRILLIC SMALL LETTER DE: CYRILLIC SMALL LETTER LONG-LEGGED DE + 0x0434: (0x1c81,), # 'д': 'ᲁ' + # CYRILLIC SMALL LETTER O: CYRILLIC SMALL LETTER NARROW O + 0x043e: (0x1c82,), # 'о': 'ᲂ' + # CYRILLIC SMALL LETTER ES: CYRILLIC SMALL LETTER WIDE ES + 0x0441: (0x1c83,), # 'с': 'ᲃ' + # CYRILLIC SMALL LETTER TE: CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE + 0x0442: (0x1c84, 0x1c85), # 'т': 'ᲄᲅ' + # CYRILLIC SMALL LETTER HARD SIGN: CYRILLIC SMALL LETTER TALL HARD SIGN + 0x044a: (0x1c86,), # 'ъ': 'ᲆ' + # CYRILLIC SMALL LETTER YAT: CYRILLIC SMALL LETTER TALL YAT + 0x0463: (0x1c87,), # 'ѣ': 'ᲇ' + # CYRILLIC SMALL LETTER ROUNDED VE: CYRILLIC SMALL LETTER VE + 0x1c80: (0x0432,), # 'ᲀ': 'в' + # CYRILLIC SMALL LETTER LONG-LEGGED DE: CYRILLIC SMALL LETTER DE + 0x1c81: (0x0434,), # 'ᲁ': 'д' + # CYRILLIC SMALL LETTER NARROW O: CYRILLIC SMALL LETTER O + 0x1c82: (0x043e,), # 'ᲂ': 'о' + # CYRILLIC SMALL LETTER WIDE ES: CYRILLIC SMALL LETTER ES + 0x1c83: (0x0441,), # 'ᲃ': 'с' + # CYRILLIC SMALL LETTER TALL TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER THREE-LEGGED TE + 0x1c84: (0x0442, 0x1c85), # 'ᲄ': 'тᲅ' + # CYRILLIC SMALL LETTER THREE-LEGGED TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE + 0x1c85: (0x0442, 0x1c84), # 'ᲅ': 'тᲄ' + # CYRILLIC SMALL LETTER TALL HARD SIGN: CYRILLIC SMALL LETTER HARD SIGN + 0x1c86: (0x044a,), # 'ᲆ': 'ъ' + # CYRILLIC SMALL LETTER TALL YAT: CYRILLIC SMALL LETTER YAT + 0x1c87: (0x0463,), # 'ᲇ': 'ѣ' + # CYRILLIC SMALL LETTER UNBLENDED UK: CYRILLIC SMALL LETTER MONOGRAPH UK + 0x1c88: (0xa64b,), # 'ᲈ': 'ꙋ' + # LATIN SMALL LETTER S WITH DOT ABOVE: LATIN SMALL LETTER LONG S WITH DOT ABOVE + 0x1e61: (0x1e9b,), # 'ṡ': 'ẛ' + # LATIN SMALL LETTER LONG S WITH DOT ABOVE: LATIN SMALL LETTER S WITH DOT ABOVE + 0x1e9b: (0x1e61,), # 'ẛ': 'ṡ' + # GREEK PROSGEGRAMMENI: COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA + 0x1fbe: (0x0345, 0x03b9), # 'ι': '\u0345ι' + # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + 0x1fd3: (0x0390,), # 'ΐ': 'ΐ' + # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + 0x1fe3: (0x03b0,), # 'ΰ': 'ΰ' + # CYRILLIC SMALL LETTER MONOGRAPH UK: CYRILLIC SMALL LETTER UNBLENDED UK + 0xa64b: (0x1c88,), # 'ꙋ': 'ᲈ' + # LATIN SMALL LIGATURE LONG S T: LATIN SMALL LIGATURE ST + 0xfb05: (0xfb06,), # 'ſt': 'st' + # LATIN SMALL LIGATURE ST: LATIN SMALL LIGATURE LONG S T + 0xfb06: (0xfb05,), # 'st': 'ſt' +} diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py new file mode 100644 index 00000000000..20dd561d1c1 --- /dev/null +++ b/Lib/re/_compiler.py @@ -0,0 +1,782 @@ +# +# Secret Labs' Regular Expression Engine +# +# convert template to internal format +# +# Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. +# +# See the __init__.py file for information on usage and redistribution. +# + +"""Internal support module for sre""" + +import _sre +from . import _parser +from ._constants import * +from ._casefix import _EXTRA_CASES + +assert _sre.MAGIC == MAGIC, "SRE module mismatch" + +_LITERAL_CODES = {LITERAL, NOT_LITERAL} +_SUCCESS_CODES = {SUCCESS, FAILURE} +_ASSERT_CODES = {ASSERT, ASSERT_NOT} +_UNIT_CODES = _LITERAL_CODES | {ANY, IN} + +_REPEATING_CODES = { + MIN_REPEAT: (REPEAT, MIN_UNTIL, MIN_REPEAT_ONE), + MAX_REPEAT: (REPEAT, MAX_UNTIL, REPEAT_ONE), + POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE), +} + +_CHARSET_ALL = [(NEGATE, None)] + +def _combine_flags(flags, add_flags, del_flags, + TYPE_FLAGS=_parser.TYPE_FLAGS): + if add_flags & TYPE_FLAGS: + flags &= ~TYPE_FLAGS + return (flags | add_flags) & ~del_flags + +def _compile(code, pattern, flags): + # internal: compile a (sub)pattern + emit = code.append + _len = len + LITERAL_CODES = _LITERAL_CODES + REPEATING_CODES = _REPEATING_CODES + SUCCESS_CODES = _SUCCESS_CODES + ASSERT_CODES = _ASSERT_CODES + iscased = None + tolower = None + fixes = None + if flags & SRE_FLAG_IGNORECASE and not flags & SRE_FLAG_LOCALE: + if flags & SRE_FLAG_UNICODE: + iscased = _sre.unicode_iscased + tolower = _sre.unicode_tolower + fixes = _EXTRA_CASES + else: + iscased = _sre.ascii_iscased + tolower = _sre.ascii_tolower + for op, av in pattern: + if op in LITERAL_CODES: + if not flags & SRE_FLAG_IGNORECASE: + emit(op) + emit(av) + elif flags & SRE_FLAG_LOCALE: + emit(OP_LOCALE_IGNORE[op]) + emit(av) + elif not iscased(av): + emit(op) + emit(av) + else: + lo = tolower(av) + if not fixes: # ascii + emit(OP_IGNORE[op]) + emit(lo) + elif lo not in fixes: + emit(OP_UNICODE_IGNORE[op]) + emit(lo) + else: + emit(IN_UNI_IGNORE) + skip = _len(code); emit(0) + if op is NOT_LITERAL: + emit(NEGATE) + for k in (lo,) + fixes[lo]: + emit(LITERAL) + emit(k) + emit(FAILURE) + code[skip] = _len(code) - skip + elif op is IN: + charset, hascased = _optimize_charset(av, iscased, tolower, fixes) + if not charset: + emit(FAILURE) + elif charset == _CHARSET_ALL: + emit(ANY_ALL) + else: + if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: + emit(IN_LOC_IGNORE) + elif not hascased: + emit(IN) + elif not fixes: # ascii + emit(IN_IGNORE) + else: + emit(IN_UNI_IGNORE) + skip = _len(code); emit(0) + _compile_charset(charset, flags, code) + code[skip] = _len(code) - skip + elif op is ANY: + if flags & SRE_FLAG_DOTALL: + emit(ANY_ALL) + else: + emit(ANY) + elif op in REPEATING_CODES: + if _simple(av[2]): + emit(REPEATING_CODES[op][2]) + skip = _len(code); emit(0) + emit(av[0]) + emit(av[1]) + _compile(code, av[2], flags) + emit(SUCCESS) + code[skip] = _len(code) - skip + else: + emit(REPEATING_CODES[op][0]) + skip = _len(code); emit(0) + emit(av[0]) + emit(av[1]) + _compile(code, av[2], flags) + code[skip] = _len(code) - skip + emit(REPEATING_CODES[op][1]) + elif op is SUBPATTERN: + group, add_flags, del_flags, p = av + if group: + emit(MARK) + emit((group-1)*2) + # _compile_info(code, p, _combine_flags(flags, add_flags, del_flags)) + _compile(code, p, _combine_flags(flags, add_flags, del_flags)) + if group: + emit(MARK) + emit((group-1)*2+1) + elif op is ATOMIC_GROUP: + # Atomic Groups are handled by starting with an Atomic + # Group op code, then putting in the atomic group pattern + # and finally a success op code to tell any repeat + # operations within the Atomic Group to stop eating and + # pop their stack if they reach it + emit(ATOMIC_GROUP) + skip = _len(code); emit(0) + _compile(code, av, flags) + emit(SUCCESS) + code[skip] = _len(code) - skip + elif op in SUCCESS_CODES: + emit(op) + elif op in ASSERT_CODES: + emit(op) + skip = _len(code); emit(0) + if av[0] >= 0: + emit(0) # look ahead + else: + lo, hi = av[1].getwidth() + if lo > MAXCODE: + raise error("looks too much behind") + if lo != hi: + raise PatternError("look-behind requires fixed-width pattern") + emit(lo) # look behind + _compile(code, av[1], flags) + emit(SUCCESS) + code[skip] = _len(code) - skip + elif op is AT: + emit(op) + if flags & SRE_FLAG_MULTILINE: + av = AT_MULTILINE.get(av, av) + if flags & SRE_FLAG_LOCALE: + av = AT_LOCALE.get(av, av) + elif flags & SRE_FLAG_UNICODE: + av = AT_UNICODE.get(av, av) + emit(av) + elif op is BRANCH: + emit(op) + tail = [] + tailappend = tail.append + for av in av[1]: + skip = _len(code); emit(0) + # _compile_info(code, av, flags) + _compile(code, av, flags) + emit(JUMP) + tailappend(_len(code)); emit(0) + code[skip] = _len(code) - skip + emit(FAILURE) # end of branch + for tail in tail: + code[tail] = _len(code) - tail + elif op is CATEGORY: + emit(op) + if flags & SRE_FLAG_LOCALE: + av = CH_LOCALE[av] + elif flags & SRE_FLAG_UNICODE: + av = CH_UNICODE[av] + emit(av) + elif op is GROUPREF: + if not flags & SRE_FLAG_IGNORECASE: + emit(op) + elif flags & SRE_FLAG_LOCALE: + emit(GROUPREF_LOC_IGNORE) + elif not fixes: # ascii + emit(GROUPREF_IGNORE) + else: + emit(GROUPREF_UNI_IGNORE) + emit(av-1) + elif op is GROUPREF_EXISTS: + emit(op) + emit(av[0]-1) + skipyes = _len(code); emit(0) + _compile(code, av[1], flags) + if av[2]: + emit(JUMP) + skipno = _len(code); emit(0) + code[skipyes] = _len(code) - skipyes + 1 + _compile(code, av[2], flags) + code[skipno] = _len(code) - skipno + else: + code[skipyes] = _len(code) - skipyes + 1 + else: + raise PatternError(f"internal: unsupported operand type {op!r}") + +def _compile_charset(charset, flags, code): + # compile charset subprogram + emit = code.append + for op, av in charset: + emit(op) + if op is NEGATE: + pass + elif op is LITERAL: + emit(av) + elif op is RANGE or op is RANGE_UNI_IGNORE: + emit(av[0]) + emit(av[1]) + elif op is CHARSET: + code.extend(av) + elif op is BIGCHARSET: + code.extend(av) + elif op is CATEGORY: + if flags & SRE_FLAG_LOCALE: + emit(CH_LOCALE[av]) + elif flags & SRE_FLAG_UNICODE: + emit(CH_UNICODE[av]) + else: + emit(av) + else: + raise PatternError(f"internal: unsupported set operator {op!r}") + emit(FAILURE) + +def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): + # internal: optimize character set + out = [] + tail = [] + charmap = bytearray(256) + hascased = False + for op, av in charset: + while True: + try: + if op is LITERAL: + if fixup: # IGNORECASE and not LOCALE + av = fixup(av) + charmap[av] = 1 + if fixes and av in fixes: + for k in fixes[av]: + charmap[k] = 1 + if not hascased and iscased(av): + hascased = True + else: + charmap[av] = 1 + elif op is RANGE: + r = range(av[0], av[1]+1) + if fixup: # IGNORECASE and not LOCALE + if fixes: + for i in map(fixup, r): + charmap[i] = 1 + if i in fixes: + for k in fixes[i]: + charmap[k] = 1 + else: + for i in map(fixup, r): + charmap[i] = 1 + if not hascased: + hascased = any(map(iscased, r)) + else: + for i in r: + charmap[i] = 1 + elif op is NEGATE: + out.append((op, av)) + elif op is CATEGORY and tail and (CATEGORY, CH_NEGATE[av]) in tail: + # Optimize [\s\S] etc. + out = [] if out else _CHARSET_ALL + return out, False + else: + tail.append((op, av)) + except IndexError: + if len(charmap) == 256: + # character set contains non-UCS1 character codes + charmap += b'\0' * 0xff00 + continue + # Character set contains non-BMP character codes. + # For range, all BMP characters in the range are already + # proceeded. + if fixup: # IGNORECASE and not LOCALE + # For now, IN_UNI_IGNORE+LITERAL and + # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP + # characters, because two characters (at least one of + # which is not in the BMP) match case-insensitively + # if and only if: + # 1) c1.lower() == c2.lower() + # 2) c1.lower() == c2 or c1.lower().upper() == c2 + # Also, both c.lower() and c.lower().upper() are single + # characters for every non-BMP character. + if op is RANGE: + if fixes: # not ASCII + op = RANGE_UNI_IGNORE + hascased = True + else: + assert op is LITERAL + if not hascased and iscased(av): + hascased = True + tail.append((op, av)) + break + + # compress character map + runs = [] + q = 0 + while True: + p = charmap.find(1, q) + if p < 0: + break + if len(runs) >= 2: + runs = None + break + q = charmap.find(0, p) + if q < 0: + runs.append((p, len(charmap))) + break + runs.append((p, q)) + if runs is not None: + # use literal/range + for p, q in runs: + if q - p == 1: + out.append((LITERAL, p)) + else: + out.append((RANGE, (p, q - 1))) + out += tail + # if the case was changed or new representation is more compact + if hascased or len(out) < len(charset): + return out, hascased + # else original character set is good enough + return charset, hascased + + # use bitmap + if len(charmap) == 256: + data = _mk_bitmap(charmap) + out.append((CHARSET, data)) + out += tail + return out, hascased + + # To represent a big charset, first a bitmap of all characters in the + # set is constructed. Then, this bitmap is sliced into chunks of 256 + # characters, duplicate chunks are eliminated, and each chunk is + # given a number. In the compiled expression, the charset is + # represented by a 32-bit word sequence, consisting of one word for + # the number of different chunks, a sequence of 256 bytes (64 words) + # of chunk numbers indexed by their original chunk position, and a + # sequence of 256-bit chunks (8 words each). + + # Compression is normally good: in a typical charset, large ranges of + # Unicode will be either completely excluded (e.g. if only cyrillic + # letters are to be matched), or completely included (e.g. if large + # subranges of Kanji match). These ranges will be represented by + # chunks of all one-bits or all zero-bits. + + # Matching can be also done efficiently: the more significant byte of + # the Unicode character is an index into the chunk number, and the + # less significant byte is a bit index in the chunk (just like the + # CHARSET matching). + + charmap = bytes(charmap) # should be hashable + comps = {} + mapping = bytearray(256) + block = 0 + data = bytearray() + for i in range(0, 65536, 256): + chunk = charmap[i: i + 256] + if chunk in comps: + mapping[i // 256] = comps[chunk] + else: + mapping[i // 256] = comps[chunk] = block + block += 1 + data += chunk + data = _mk_bitmap(data) + data[0:0] = [block] + _bytes_to_codes(mapping) + out.append((BIGCHARSET, data)) + out += tail + return out, hascased + +_CODEBITS = _sre.CODESIZE * 8 +MAXCODE = (1 << _CODEBITS) - 1 +_BITS_TRANS = b'0' + b'1' * 255 +def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): + s = bits.translate(_BITS_TRANS)[::-1] + return [_int(s[i - _CODEBITS: i], 2) + for i in range(len(s), 0, -_CODEBITS)] + +def _bytes_to_codes(b): + # Convert block indices to word array + a = memoryview(b).cast('I') + assert a.itemsize == _sre.CODESIZE + assert len(a) * a.itemsize == len(b) + return a.tolist() + +def _simple(p): + # check if this subpattern is a "simple" operator + if len(p) != 1: + return False + op, av = p[0] + if op is SUBPATTERN: + return av[0] is None and _simple(av[-1]) + return op in _UNIT_CODES + +def _generate_overlap_table(prefix): + """ + Generate an overlap table for the following prefix. + An overlap table is a table of the same size as the prefix which + informs about the potential self-overlap for each index in the prefix: + - if overlap[i] == 0, prefix[i:] can't overlap prefix[0:...] + - if overlap[i] == k with 0 < k <= i, prefix[i-k+1:i+1] overlaps with + prefix[0:k] + """ + table = [0] * len(prefix) + for i in range(1, len(prefix)): + idx = table[i - 1] + while prefix[i] != prefix[idx]: + if idx == 0: + table[i] = 0 + break + idx = table[idx - 1] + else: + table[i] = idx + 1 + return table + +def _get_iscased(flags): + if not flags & SRE_FLAG_IGNORECASE: + return None + elif flags & SRE_FLAG_UNICODE: + return _sre.unicode_iscased + else: + return _sre.ascii_iscased + +def _get_literal_prefix(pattern, flags): + # look for literal prefix + prefix = [] + prefixappend = prefix.append + prefix_skip = None + iscased = _get_iscased(flags) + for op, av in pattern.data: + if op is LITERAL: + if iscased and iscased(av): + break + prefixappend(av) + elif op is SUBPATTERN: + group, add_flags, del_flags, p = av + flags1 = _combine_flags(flags, add_flags, del_flags) + if flags1 & SRE_FLAG_IGNORECASE and flags1 & SRE_FLAG_LOCALE: + break + prefix1, prefix_skip1, got_all = _get_literal_prefix(p, flags1) + if prefix_skip is None: + if group is not None: + prefix_skip = len(prefix) + elif prefix_skip1 is not None: + prefix_skip = len(prefix) + prefix_skip1 + prefix.extend(prefix1) + if not got_all: + break + else: + break + else: + return prefix, prefix_skip, True + return prefix, prefix_skip, False + +def _get_charset_prefix(pattern, flags): + while True: + if not pattern.data: + return None + op, av = pattern.data[0] + if op is not SUBPATTERN: + break + group, add_flags, del_flags, pattern = av + flags = _combine_flags(flags, add_flags, del_flags) + if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: + return None + + iscased = _get_iscased(flags) + if op is LITERAL: + if iscased and iscased(av): + return None + return [(op, av)] + elif op is BRANCH: + charset = [] + charsetappend = charset.append + for p in av[1]: + if not p: + return None + op, av = p[0] + if op is LITERAL and not (iscased and iscased(av)): + charsetappend((op, av)) + else: + return None + return charset + elif op is IN: + charset = av + if iscased: + for op, av in charset: + if op is LITERAL: + if iscased(av): + return None + elif op is RANGE: + if av[1] > 0xffff: + return None + if any(map(iscased, range(av[0], av[1]+1))): + return None + return charset + return None + +def _compile_info(code, pattern, flags): + # internal: compile an info block. in the current version, + # this contains min/max pattern width, and an optional literal + # prefix or a character map + lo, hi = pattern.getwidth() + if hi > MAXCODE: + hi = MAXCODE + if lo == 0: + code.extend([INFO, 4, 0, lo, hi]) + return + # look for a literal prefix + prefix = [] + prefix_skip = 0 + charset = None # not used + if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): + # look for literal prefix + prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) + # if no prefix, look for charset prefix + if not prefix: + charset = _get_charset_prefix(pattern, flags) + if charset: + charset, hascased = _optimize_charset(charset) + assert not hascased + if charset == _CHARSET_ALL: + charset = None +## if prefix: +## print("*** PREFIX", prefix, prefix_skip) +## if charset: +## print("*** CHARSET", charset) + # add an info block + emit = code.append + emit(INFO) + skip = len(code); emit(0) + # literal flag + mask = 0 + if prefix: + mask = SRE_INFO_PREFIX + if prefix_skip is None and got_all: + mask = mask | SRE_INFO_LITERAL + elif charset: + mask = mask | SRE_INFO_CHARSET + emit(mask) + # pattern length + if lo < MAXCODE: + emit(lo) + else: + emit(MAXCODE) + prefix = prefix[:MAXCODE] + emit(hi) + # add literal prefix + if prefix: + emit(len(prefix)) # length + if prefix_skip is None: + prefix_skip = len(prefix) + emit(prefix_skip) # skip + code.extend(prefix) + # generate overlap table + code.extend(_generate_overlap_table(prefix)) + elif charset: + _compile_charset(charset, flags, code) + code[skip] = len(code) - skip + +def isstring(obj): + return isinstance(obj, (str, bytes)) + +def _code(p, flags): + + flags = p.state.flags | flags + code = [] + + # compile info block + _compile_info(code, p, flags) + + # compile the pattern + _compile(code, p.data, flags) + + code.append(SUCCESS) + + return code + +def _hex_code(code): + return '[%s]' % ', '.join('%#0*x' % (_sre.CODESIZE*2+2, x) for x in code) + +def dis(code): + import sys + + labels = set() + level = 0 + offset_width = len(str(len(code) - 1)) + + def dis_(start, end): + def print_(*args, to=None): + if to is not None: + labels.add(to) + args += ('(to %d)' % (to,),) + print('%*d%s ' % (offset_width, start, ':' if start in labels else '.'), + end=' '*(level-1)) + print(*args) + + def print_2(*args): + print(end=' '*(offset_width + 2*level)) + print(*args) + + nonlocal level + level += 1 + i = start + while i < end: + start = i + op = code[i] + i += 1 + op = OPCODES[op] + if op in (SUCCESS, FAILURE, ANY, ANY_ALL, + MAX_UNTIL, MIN_UNTIL, NEGATE): + print_(op) + elif op in (LITERAL, NOT_LITERAL, + LITERAL_IGNORE, NOT_LITERAL_IGNORE, + LITERAL_UNI_IGNORE, NOT_LITERAL_UNI_IGNORE, + LITERAL_LOC_IGNORE, NOT_LITERAL_LOC_IGNORE): + arg = code[i] + i += 1 + print_(op, '%#02x (%r)' % (arg, chr(arg))) + elif op is AT: + arg = code[i] + i += 1 + arg = str(ATCODES[arg]) + assert arg[:3] == 'AT_' + print_(op, arg[3:]) + elif op is CATEGORY: + arg = code[i] + i += 1 + arg = str(CHCODES[arg]) + assert arg[:9] == 'CATEGORY_' + print_(op, arg[9:]) + elif op in (IN, IN_IGNORE, IN_UNI_IGNORE, IN_LOC_IGNORE): + skip = code[i] + print_(op, skip, to=i+skip) + dis_(i+1, i+skip) + i += skip + elif op in (RANGE, RANGE_UNI_IGNORE): + lo, hi = code[i: i+2] + i += 2 + print_(op, '%#02x %#02x (%r-%r)' % (lo, hi, chr(lo), chr(hi))) + elif op is CHARSET: + print_(op, _hex_code(code[i: i + 256//_CODEBITS])) + i += 256//_CODEBITS + elif op is BIGCHARSET: + arg = code[i] + i += 1 + mapping = list(b''.join(x.to_bytes(_sre.CODESIZE, sys.byteorder) + for x in code[i: i + 256//_sre.CODESIZE])) + print_(op, arg, mapping) + i += 256//_sre.CODESIZE + level += 1 + for j in range(arg): + print_2(_hex_code(code[i: i + 256//_CODEBITS])) + i += 256//_CODEBITS + level -= 1 + elif op in (MARK, GROUPREF, GROUPREF_IGNORE, GROUPREF_UNI_IGNORE, + GROUPREF_LOC_IGNORE): + arg = code[i] + i += 1 + print_(op, arg) + elif op is JUMP: + skip = code[i] + print_(op, skip, to=i+skip) + i += 1 + elif op is BRANCH: + skip = code[i] + print_(op, skip, to=i+skip) + while skip: + dis_(i+1, i+skip) + i += skip + start = i + skip = code[i] + if skip: + print_('branch', skip, to=i+skip) + else: + print_(FAILURE) + i += 1 + elif op in (REPEAT, REPEAT_ONE, MIN_REPEAT_ONE, + POSSESSIVE_REPEAT, POSSESSIVE_REPEAT_ONE): + skip, min, max = code[i: i+3] + if max == MAXREPEAT: + max = 'MAXREPEAT' + print_(op, skip, min, max, to=i+skip) + dis_(i+3, i+skip) + i += skip + elif op is GROUPREF_EXISTS: + arg, skip = code[i: i+2] + print_(op, arg, skip, to=i+skip) + i += 2 + elif op in (ASSERT, ASSERT_NOT): + skip, arg = code[i: i+2] + print_(op, skip, arg, to=i+skip) + dis_(i+2, i+skip) + i += skip + elif op is ATOMIC_GROUP: + skip = code[i] + print_(op, skip, to=i+skip) + dis_(i+1, i+skip) + i += skip + elif op is INFO: + skip, flags, min, max = code[i: i+4] + if max == MAXREPEAT: + max = 'MAXREPEAT' + print_(op, skip, bin(flags), min, max, to=i+skip) + start = i+4 + if flags & SRE_INFO_PREFIX: + prefix_len, prefix_skip = code[i+4: i+6] + print_2(' prefix_skip', prefix_skip) + start = i + 6 + prefix = code[start: start+prefix_len] + print_2(' prefix', + '[%s]' % ', '.join('%#02x' % x for x in prefix), + '(%r)' % ''.join(map(chr, prefix))) + start += prefix_len + print_2(' overlap', code[start: start+prefix_len]) + start += prefix_len + if flags & SRE_INFO_CHARSET: + level += 1 + print_2('in') + dis_(start, i+skip) + level -= 1 + i += skip + else: + raise ValueError(op) + + level -= 1 + + dis_(0, len(code)) + + +def compile(p, flags=0): + # internal: convert pattern list to internal format + + if isstring(p): + pattern = p + p = _parser.parse(p, flags) + else: + pattern = None + + code = _code(p, flags) + + if flags & SRE_FLAG_DEBUG: + print() + dis(code) + + # map in either direction + groupindex = p.state.groupdict + indexgroup = [None] * p.state.groups + for k, i in groupindex.items(): + indexgroup[i] = k + + return _sre.compile( + pattern, flags | p.state.flags, code, + p.state.groups-1, + groupindex, tuple(indexgroup) + ) diff --git a/Lib/re/_constants.py b/Lib/re/_constants.py new file mode 100644 index 00000000000..d6f32302d37 --- /dev/null +++ b/Lib/re/_constants.py @@ -0,0 +1,224 @@ +# +# Secret Labs' Regular Expression Engine +# +# various symbols used by the regular expression engine. +# run this script to update the _sre include files! +# +# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. +# +# See the __init__.py file for information on usage and redistribution. +# + +"""Internal support module for sre""" + +# update when constants are added or removed + +MAGIC = 20230612 + +from _sre import MAXREPEAT, MAXGROUPS # noqa: F401 + +# SRE standard exception (access as sre.error) +# should this really be here? + +class PatternError(Exception): + """Exception raised for invalid regular expressions. + + Attributes: + + msg: The unformatted error message + pattern: The regular expression pattern + pos: The index in the pattern where compilation failed (may be None) + lineno: The line corresponding to pos (may be None) + colno: The column corresponding to pos (may be None) + """ + + __module__ = 're' + + def __init__(self, msg, pattern=None, pos=None): + self.msg = msg + self.pattern = pattern + self.pos = pos + if pattern is not None and pos is not None: + msg = '%s at position %d' % (msg, pos) + if isinstance(pattern, str): + newline = '\n' + else: + newline = b'\n' + self.lineno = pattern.count(newline, 0, pos) + 1 + self.colno = pos - pattern.rfind(newline, 0, pos) + if newline in pattern: + msg = '%s (line %d, column %d)' % (msg, self.lineno, self.colno) + else: + self.lineno = self.colno = None + super().__init__(msg) + + +# Backward compatibility after renaming in 3.13 +error = PatternError + +class _NamedIntConstant(int): + def __new__(cls, value, name): + self = super(_NamedIntConstant, cls).__new__(cls, value) + self.name = name + return self + + def __repr__(self): + return self.name + + __reduce__ = None + +MAXREPEAT = _NamedIntConstant(MAXREPEAT, 'MAXREPEAT') + +def _makecodes(*names): + items = [_NamedIntConstant(i, name) for i, name in enumerate(names)] + globals().update({item.name: item for item in items}) + return items + +# operators +OPCODES = _makecodes( + # failure=0 success=1 (just because it looks better that way :-) + 'FAILURE', 'SUCCESS', + + 'ANY', 'ANY_ALL', + 'ASSERT', 'ASSERT_NOT', + 'AT', + 'BRANCH', + 'CATEGORY', + 'CHARSET', 'BIGCHARSET', + 'GROUPREF', 'GROUPREF_EXISTS', + 'IN', + 'INFO', + 'JUMP', + 'LITERAL', + 'MARK', + 'MAX_UNTIL', + 'MIN_UNTIL', + 'NOT_LITERAL', + 'NEGATE', + 'RANGE', + 'REPEAT', + 'REPEAT_ONE', + 'SUBPATTERN', + 'MIN_REPEAT_ONE', + 'ATOMIC_GROUP', + 'POSSESSIVE_REPEAT', + 'POSSESSIVE_REPEAT_ONE', + + 'GROUPREF_IGNORE', + 'IN_IGNORE', + 'LITERAL_IGNORE', + 'NOT_LITERAL_IGNORE', + + 'GROUPREF_LOC_IGNORE', + 'IN_LOC_IGNORE', + 'LITERAL_LOC_IGNORE', + 'NOT_LITERAL_LOC_IGNORE', + + 'GROUPREF_UNI_IGNORE', + 'IN_UNI_IGNORE', + 'LITERAL_UNI_IGNORE', + 'NOT_LITERAL_UNI_IGNORE', + 'RANGE_UNI_IGNORE', + + # The following opcodes are only occurred in the parser output, + # but not in the compiled code. + 'MIN_REPEAT', 'MAX_REPEAT', +) +del OPCODES[-2:] # remove MIN_REPEAT and MAX_REPEAT + +# positions +ATCODES = _makecodes( + 'AT_BEGINNING', 'AT_BEGINNING_LINE', 'AT_BEGINNING_STRING', + 'AT_BOUNDARY', 'AT_NON_BOUNDARY', + 'AT_END', 'AT_END_LINE', 'AT_END_STRING', + + 'AT_LOC_BOUNDARY', 'AT_LOC_NON_BOUNDARY', + + 'AT_UNI_BOUNDARY', 'AT_UNI_NON_BOUNDARY', +) + +# categories +CHCODES = _makecodes( + 'CATEGORY_DIGIT', 'CATEGORY_NOT_DIGIT', + 'CATEGORY_SPACE', 'CATEGORY_NOT_SPACE', + 'CATEGORY_WORD', 'CATEGORY_NOT_WORD', + 'CATEGORY_LINEBREAK', 'CATEGORY_NOT_LINEBREAK', + + 'CATEGORY_LOC_WORD', 'CATEGORY_LOC_NOT_WORD', + + 'CATEGORY_UNI_DIGIT', 'CATEGORY_UNI_NOT_DIGIT', + 'CATEGORY_UNI_SPACE', 'CATEGORY_UNI_NOT_SPACE', + 'CATEGORY_UNI_WORD', 'CATEGORY_UNI_NOT_WORD', + 'CATEGORY_UNI_LINEBREAK', 'CATEGORY_UNI_NOT_LINEBREAK', +) + + +# replacement operations for "ignore case" mode +OP_IGNORE = { + LITERAL: LITERAL_IGNORE, + NOT_LITERAL: NOT_LITERAL_IGNORE, +} + +OP_LOCALE_IGNORE = { + LITERAL: LITERAL_LOC_IGNORE, + NOT_LITERAL: NOT_LITERAL_LOC_IGNORE, +} + +OP_UNICODE_IGNORE = { + LITERAL: LITERAL_UNI_IGNORE, + NOT_LITERAL: NOT_LITERAL_UNI_IGNORE, +} + +AT_MULTILINE = { + AT_BEGINNING: AT_BEGINNING_LINE, + AT_END: AT_END_LINE +} + +AT_LOCALE = { + AT_BOUNDARY: AT_LOC_BOUNDARY, + AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY +} + +AT_UNICODE = { + AT_BOUNDARY: AT_UNI_BOUNDARY, + AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY +} + +CH_LOCALE = { + CATEGORY_DIGIT: CATEGORY_DIGIT, + CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT, + CATEGORY_SPACE: CATEGORY_SPACE, + CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE, + CATEGORY_WORD: CATEGORY_LOC_WORD, + CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD, + CATEGORY_LINEBREAK: CATEGORY_LINEBREAK, + CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK +} + +CH_UNICODE = { + CATEGORY_DIGIT: CATEGORY_UNI_DIGIT, + CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT, + CATEGORY_SPACE: CATEGORY_UNI_SPACE, + CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE, + CATEGORY_WORD: CATEGORY_UNI_WORD, + CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD, + CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK, + CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK +} + +CH_NEGATE = dict(zip(CHCODES[::2] + CHCODES[1::2], CHCODES[1::2] + CHCODES[::2])) + +# flags +SRE_FLAG_IGNORECASE = 2 # case insensitive +SRE_FLAG_LOCALE = 4 # honour system locale +SRE_FLAG_MULTILINE = 8 # treat target as multiline string +SRE_FLAG_DOTALL = 16 # treat target as a single string +SRE_FLAG_UNICODE = 32 # use unicode "locale" +SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments +SRE_FLAG_DEBUG = 128 # debugging +SRE_FLAG_ASCII = 256 # use ascii "locale" + +# flags for INFO primitive +SRE_INFO_PREFIX = 1 # has prefix +SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix) +SRE_INFO_CHARSET = 4 # pattern starts with character from given set diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py new file mode 100644 index 00000000000..35ab7ede2a7 --- /dev/null +++ b/Lib/re/_parser.py @@ -0,0 +1,1066 @@ +# +# Secret Labs' Regular Expression Engine +# +# convert re-style regular expression to sre pattern +# +# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. +# +# See the __init__.py file for information on usage and redistribution. +# + +"""Internal support module for sre""" + +# XXX: show string offset and offending character for all errors + +from ._constants import * + +SPECIAL_CHARS = ".\\[{()*+?^$|" +REPEAT_CHARS = "*+?{" + +DIGITS = frozenset("0123456789") + +OCTDIGITS = frozenset("01234567") +HEXDIGITS = frozenset("0123456789abcdefABCDEF") +ASCIILETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +WHITESPACE = frozenset(" \t\n\r\v\f") + +_REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT, POSSESSIVE_REPEAT}) +_UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY}) + +ESCAPES = { + r"\a": (LITERAL, ord("\a")), + r"\b": (LITERAL, ord("\b")), + r"\f": (LITERAL, ord("\f")), + r"\n": (LITERAL, ord("\n")), + r"\r": (LITERAL, ord("\r")), + r"\t": (LITERAL, ord("\t")), + r"\v": (LITERAL, ord("\v")), + r"\\": (LITERAL, ord("\\")) +} + +CATEGORIES = { + r"\A": (AT, AT_BEGINNING_STRING), # start of string + r"\b": (AT, AT_BOUNDARY), + r"\B": (AT, AT_NON_BOUNDARY), + r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]), + r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]), + r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]), + r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), + r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), + r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), + r"\z": (AT, AT_END_STRING), # end of string + r"\Z": (AT, AT_END_STRING), # end of string (obsolete) +} + +FLAGS = { + # standard flags + "i": SRE_FLAG_IGNORECASE, + "L": SRE_FLAG_LOCALE, + "m": SRE_FLAG_MULTILINE, + "s": SRE_FLAG_DOTALL, + "x": SRE_FLAG_VERBOSE, + # extensions + "a": SRE_FLAG_ASCII, + "u": SRE_FLAG_UNICODE, +} + +TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE +GLOBAL_FLAGS = SRE_FLAG_DEBUG + +# Maximal value returned by SubPattern.getwidth(). +# Must be larger than MAXREPEAT, MAXCODE and sys.maxsize. +MAXWIDTH = 1 << 64 + +class State: + # keeps track of state for parsing + def __init__(self): + self.flags = 0 + self.groupdict = {} + self.groupwidths = [None] # group 0 + self.lookbehindgroups = None + self.grouprefpos = {} + @property + def groups(self): + return len(self.groupwidths) + def opengroup(self, name=None): + gid = self.groups + self.groupwidths.append(None) + if self.groups > MAXGROUPS: + raise error("too many groups") + if name is not None: + ogid = self.groupdict.get(name, None) + if ogid is not None: + raise error("redefinition of group name %r as group %d; " + "was group %d" % (name, gid, ogid)) + self.groupdict[name] = gid + return gid + def closegroup(self, gid, p): + self.groupwidths[gid] = p.getwidth() + def checkgroup(self, gid): + return gid < self.groups and self.groupwidths[gid] is not None + + def checklookbehindgroup(self, gid, source): + if self.lookbehindgroups is not None: + if not self.checkgroup(gid): + raise source.error('cannot refer to an open group') + if gid >= self.lookbehindgroups: + raise source.error('cannot refer to group defined in the same ' + 'lookbehind subpattern') + +class SubPattern: + # a subpattern, in intermediate form + def __init__(self, state, data=None): + self.state = state + if data is None: + data = [] + self.data = data + self.width = None + + def dump(self, level=0): + seqtypes = (tuple, list) + for op, av in self.data: + print(level*" " + str(op), end='') + if op is IN: + # member sublanguage + print() + for op, a in av: + print((level+1)*" " + str(op), a) + elif op is BRANCH: + print() + for i, a in enumerate(av[1]): + if i: + print(level*" " + "OR") + a.dump(level+1) + elif op is GROUPREF_EXISTS: + condgroup, item_yes, item_no = av + print('', condgroup) + item_yes.dump(level+1) + if item_no: + print(level*" " + "ELSE") + item_no.dump(level+1) + elif isinstance(av, SubPattern): + print() + av.dump(level+1) + elif isinstance(av, seqtypes): + nl = False + for a in av: + if isinstance(a, SubPattern): + if not nl: + print() + a.dump(level+1) + nl = True + else: + if not nl: + print(' ', end='') + print(a, end='') + nl = False + if not nl: + print() + else: + print('', av) + def __repr__(self): + return repr(self.data) + def __len__(self): + return len(self.data) + def __delitem__(self, index): + del self.data[index] + def __getitem__(self, index): + if isinstance(index, slice): + return SubPattern(self.state, self.data[index]) + return self.data[index] + def __setitem__(self, index, code): + self.data[index] = code + def insert(self, index, code): + self.data.insert(index, code) + def append(self, code): + self.data.append(code) + def getwidth(self): + # determine the width (min, max) for this subpattern + if self.width is not None: + return self.width + lo = hi = 0 + for op, av in self.data: + if op is BRANCH: + i = MAXWIDTH + j = 0 + for av in av[1]: + l, h = av.getwidth() + i = min(i, l) + j = max(j, h) + lo = lo + i + hi = hi + j + elif op is ATOMIC_GROUP: + i, j = av.getwidth() + lo = lo + i + hi = hi + j + elif op is SUBPATTERN: + i, j = av[-1].getwidth() + lo = lo + i + hi = hi + j + elif op in _REPEATCODES: + i, j = av[2].getwidth() + lo = lo + i * av[0] + if av[1] == MAXREPEAT and j: + hi = MAXWIDTH + else: + hi = hi + j * av[1] + elif op in _UNITCODES: + lo = lo + 1 + hi = hi + 1 + elif op is GROUPREF: + i, j = self.state.groupwidths[av] + lo = lo + i + hi = hi + j + elif op is GROUPREF_EXISTS: + i, j = av[1].getwidth() + if av[2] is not None: + l, h = av[2].getwidth() + i = min(i, l) + j = max(j, h) + else: + i = 0 + lo = lo + i + hi = hi + j + elif op is SUCCESS: + break + self.width = min(lo, MAXWIDTH), min(hi, MAXWIDTH) + return self.width + +class Tokenizer: + def __init__(self, string): + self.istext = isinstance(string, str) + self.string = string + if not self.istext: + string = str(string, 'latin1') + self.decoded_string = string + self.index = 0 + self.next = None + self.__next() + def __next(self): + index = self.index + try: + char = self.decoded_string[index] + except IndexError: + self.next = None + return + if char == "\\": + index += 1 + try: + char += self.decoded_string[index] + except IndexError: + raise error("bad escape (end of pattern)", + self.string, len(self.string) - 1) from None + self.index = index + 1 + self.next = char + def match(self, char): + if char == self.next: + self.__next() + return True + return False + def get(self): + this = self.next + self.__next() + return this + def getwhile(self, n, charset): + result = '' + for _ in range(n): + c = self.next + if c not in charset: + break + result += c + self.__next() + return result + def getuntil(self, terminator, name): + result = '' + while True: + c = self.next + self.__next() + if c is None: + if not result: + raise self.error("missing " + name) + raise self.error("missing %s, unterminated name" % terminator, + len(result)) + if c == terminator: + if not result: + raise self.error("missing " + name, 1) + break + result += c + return result + @property + def pos(self): + return self.index - len(self.next or '') + def tell(self): + return self.index - len(self.next or '') + def seek(self, index): + self.index = index + self.__next() + + def error(self, msg, offset=0): + if not self.istext: + msg = msg.encode('ascii', 'backslashreplace').decode('ascii') + return error(msg, self.string, self.tell() - offset) + + def checkgroupname(self, name, offset): + if not (self.istext or name.isascii()): + msg = "bad character in group name %a" % name + raise self.error(msg, len(name) + offset) + if not name.isidentifier(): + msg = "bad character in group name %r" % name + raise self.error(msg, len(name) + offset) + +def _class_escape(source, escape): + # handle escape code inside character class + code = ESCAPES.get(escape) + if code: + return code + code = CATEGORIES.get(escape) + if code and code[0] is IN: + return code + try: + c = escape[1:2] + if c == "x": + # hexadecimal escape (exactly two digits) + escape += source.getwhile(2, HEXDIGITS) + if len(escape) != 4: + raise source.error("incomplete escape %s" % escape, len(escape)) + return LITERAL, int(escape[2:], 16) + elif c == "u" and source.istext: + # unicode escape (exactly four digits) + escape += source.getwhile(4, HEXDIGITS) + if len(escape) != 6: + raise source.error("incomplete escape %s" % escape, len(escape)) + return LITERAL, int(escape[2:], 16) + elif c == "U" and source.istext: + # unicode escape (exactly eight digits) + escape += source.getwhile(8, HEXDIGITS) + if len(escape) != 10: + raise source.error("incomplete escape %s" % escape, len(escape)) + c = int(escape[2:], 16) + chr(c) # raise ValueError for invalid code + return LITERAL, c + elif c == "N" and source.istext: + import unicodedata + # named unicode escape e.g. \N{EM DASH} + if not source.match('{'): + raise source.error("missing {") + charname = source.getuntil('}', 'character name') + try: + c = ord(unicodedata.lookup(charname)) + except (KeyError, TypeError): + raise source.error("undefined character name %r" % charname, + len(charname) + len(r'\N{}')) from None + return LITERAL, c + elif c in OCTDIGITS: + # octal escape (up to three digits) + escape += source.getwhile(2, OCTDIGITS) + c = int(escape[1:], 8) + if c > 0o377: + raise source.error('octal escape value %s outside of ' + 'range 0-0o377' % escape, len(escape)) + return LITERAL, c + elif c in DIGITS: + raise ValueError + if len(escape) == 2: + if c in ASCIILETTERS: + raise source.error('bad escape %s' % escape, len(escape)) + return LITERAL, ord(escape[1]) + except ValueError: + pass + raise source.error("bad escape %s" % escape, len(escape)) + +def _escape(source, escape, state): + # handle escape code in expression + code = CATEGORIES.get(escape) + if code: + return code + code = ESCAPES.get(escape) + if code: + return code + try: + c = escape[1:2] + if c == "x": + # hexadecimal escape + escape += source.getwhile(2, HEXDIGITS) + if len(escape) != 4: + raise source.error("incomplete escape %s" % escape, len(escape)) + return LITERAL, int(escape[2:], 16) + elif c == "u" and source.istext: + # unicode escape (exactly four digits) + escape += source.getwhile(4, HEXDIGITS) + if len(escape) != 6: + raise source.error("incomplete escape %s" % escape, len(escape)) + return LITERAL, int(escape[2:], 16) + elif c == "U" and source.istext: + # unicode escape (exactly eight digits) + escape += source.getwhile(8, HEXDIGITS) + if len(escape) != 10: + raise source.error("incomplete escape %s" % escape, len(escape)) + c = int(escape[2:], 16) + chr(c) # raise ValueError for invalid code + return LITERAL, c + elif c == "N" and source.istext: + import unicodedata + # named unicode escape e.g. \N{EM DASH} + if not source.match('{'): + raise source.error("missing {") + charname = source.getuntil('}', 'character name') + try: + c = ord(unicodedata.lookup(charname)) + except (KeyError, TypeError): + raise source.error("undefined character name %r" % charname, + len(charname) + len(r'\N{}')) from None + return LITERAL, c + elif c == "0": + # octal escape + escape += source.getwhile(2, OCTDIGITS) + return LITERAL, int(escape[1:], 8) + elif c in DIGITS: + # octal escape *or* decimal group reference (sigh) + if source.next in DIGITS: + escape += source.get() + if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and + source.next in OCTDIGITS): + # got three octal digits; this is an octal escape + escape += source.get() + c = int(escape[1:], 8) + if c > 0o377: + raise source.error('octal escape value %s outside of ' + 'range 0-0o377' % escape, + len(escape)) + return LITERAL, c + # not an octal escape, so this is a group reference + group = int(escape[1:]) + if group < state.groups: + if not state.checkgroup(group): + raise source.error("cannot refer to an open group", + len(escape)) + state.checklookbehindgroup(group, source) + return GROUPREF, group + raise source.error("invalid group reference %d" % group, len(escape) - 1) + if len(escape) == 2: + if c in ASCIILETTERS: + raise source.error("bad escape %s" % escape, len(escape)) + return LITERAL, ord(escape[1]) + except ValueError: + pass + raise source.error("bad escape %s" % escape, len(escape)) + +def _uniq(items): + return list(dict.fromkeys(items)) + +def _parse_sub(source, state, verbose, nested): + # parse an alternation: a|b|c + + items = [] + itemsappend = items.append + sourcematch = source.match + start = source.tell() + while True: + itemsappend(_parse(source, state, verbose, nested + 1, + not nested and not items)) + if not sourcematch("|"): + break + if not nested: + verbose = state.flags & SRE_FLAG_VERBOSE + + if len(items) == 1: + return items[0] + + subpattern = SubPattern(state) + + # check if all items share a common prefix + while True: + prefix = None + for item in items: + if not item: + break + if prefix is None: + prefix = item[0] + elif item[0] != prefix: + break + else: + # all subitems start with a common "prefix". + # move it out of the branch + for item in items: + del item[0] + subpattern.append(prefix) + continue # check next one + break + + # check if the branch can be replaced by a character set + set = [] + for item in items: + if len(item) != 1: + break + op, av = item[0] + if op is LITERAL: + set.append((op, av)) + elif op is IN and av[0][0] is not NEGATE: + set.extend(av) + else: + break + else: + # we can store this as a character set instead of a + # branch (the compiler may optimize this even more) + subpattern.append((IN, _uniq(set))) + return subpattern + + subpattern.append((BRANCH, (None, items))) + return subpattern + +def _parse(source, state, verbose, nested, first=False): + # parse a simple pattern + subpattern = SubPattern(state) + + # precompute constants into local variables + subpatternappend = subpattern.append + sourceget = source.get + sourcematch = source.match + _len = len + _ord = ord + + while True: + + this = source.next + if this is None: + break # end of pattern + if this in "|)": + break # end of subpattern + sourceget() + + if verbose: + # skip whitespace and comments + if this in WHITESPACE: + continue + if this == "#": + while True: + this = sourceget() + if this is None or this == "\n": + break + continue + + if this[0] == "\\": + code = _escape(source, this, state) + subpatternappend(code) + + elif this not in SPECIAL_CHARS: + subpatternappend((LITERAL, _ord(this))) + + elif this == "[": + here = source.tell() - 1 + # character set + set = [] + setappend = set.append +## if sourcematch(":"): +## pass # handle character classes + if source.next == '[': + import warnings + warnings.warn( + 'Possible nested set at position %d' % source.tell(), + FutureWarning, stacklevel=nested + 6 + ) + negate = sourcematch("^") + # check remaining characters + while True: + this = sourceget() + if this is None: + raise source.error("unterminated character set", + source.tell() - here) + if this == "]" and set: + break + elif this[0] == "\\": + code1 = _class_escape(source, this) + else: + if set and this in '-&~|' and source.next == this: + import warnings + warnings.warn( + 'Possible set %s at position %d' % ( + 'difference' if this == '-' else + 'intersection' if this == '&' else + 'symmetric difference' if this == '~' else + 'union', + source.tell() - 1), + FutureWarning, stacklevel=nested + 6 + ) + code1 = LITERAL, _ord(this) + if sourcematch("-"): + # potential range + that = sourceget() + if that is None: + raise source.error("unterminated character set", + source.tell() - here) + if that == "]": + if code1[0] is IN: + code1 = code1[1][0] + setappend(code1) + setappend((LITERAL, _ord("-"))) + break + if that[0] == "\\": + code2 = _class_escape(source, that) + else: + if that == '-': + import warnings + warnings.warn( + 'Possible set difference at position %d' % ( + source.tell() - 2), + FutureWarning, stacklevel=nested + 6 + ) + code2 = LITERAL, _ord(that) + if code1[0] != LITERAL or code2[0] != LITERAL: + msg = "bad character range %s-%s" % (this, that) + raise source.error(msg, len(this) + 1 + len(that)) + lo = code1[1] + hi = code2[1] + if hi < lo: + msg = "bad character range %s-%s" % (this, that) + raise source.error(msg, len(this) + 1 + len(that)) + setappend((RANGE, (lo, hi))) + else: + if code1[0] is IN: + code1 = code1[1][0] + setappend(code1) + + set = _uniq(set) + # XXX: should move set optimization to compiler! + if _len(set) == 1 and set[0][0] is LITERAL: + # optimization + if negate: + subpatternappend((NOT_LITERAL, set[0][1])) + else: + subpatternappend(set[0]) + else: + if negate: + set.insert(0, (NEGATE, None)) + # charmap optimization can't be added here because + # global flags still are not known + subpatternappend((IN, set)) + + elif this in REPEAT_CHARS: + # repeat previous item + here = source.tell() + if this == "?": + min, max = 0, 1 + elif this == "*": + min, max = 0, MAXREPEAT + + elif this == "+": + min, max = 1, MAXREPEAT + elif this == "{": + if source.next == "}": + subpatternappend((LITERAL, _ord(this))) + continue + + min, max = 0, MAXREPEAT + lo = hi = "" + while source.next in DIGITS: + lo += sourceget() + if sourcematch(","): + while source.next in DIGITS: + hi += sourceget() + else: + hi = lo + if not sourcematch("}"): + subpatternappend((LITERAL, _ord(this))) + source.seek(here) + continue + + if lo: + min = int(lo) + if min >= MAXREPEAT: + raise OverflowError("the repetition number is too large") + if hi: + max = int(hi) + if max >= MAXREPEAT: + raise OverflowError("the repetition number is too large") + if max < min: + raise source.error("min repeat greater than max repeat", + source.tell() - here) + else: + raise AssertionError("unsupported quantifier %r" % (char,)) + # figure out which item to repeat + if subpattern: + item = subpattern[-1:] + else: + item = None + if not item or item[0][0] is AT: + raise source.error("nothing to repeat", + source.tell() - here + len(this)) + if item[0][0] in _REPEATCODES: + raise source.error("multiple repeat", + source.tell() - here + len(this)) + if item[0][0] is SUBPATTERN: + group, add_flags, del_flags, p = item[0][1] + if group is None and not add_flags and not del_flags: + item = p + if sourcematch("?"): + # Non-Greedy Match + subpattern[-1] = (MIN_REPEAT, (min, max, item)) + elif sourcematch("+"): + # Possessive Match (Always Greedy) + subpattern[-1] = (POSSESSIVE_REPEAT, (min, max, item)) + else: + # Greedy Match + subpattern[-1] = (MAX_REPEAT, (min, max, item)) + + elif this == ".": + subpatternappend((ANY, None)) + + elif this == "(": + start = source.tell() - 1 + capture = True + atomic = False + name = None + add_flags = 0 + del_flags = 0 + if sourcematch("?"): + # options + char = sourceget() + if char is None: + raise source.error("unexpected end of pattern") + if char == "P": + # python extensions + if sourcematch("<"): + # named group: skip forward to end of name + name = source.getuntil(">", "group name") + source.checkgroupname(name, 1) + elif sourcematch("="): + # named backreference + name = source.getuntil(")", "group name") + source.checkgroupname(name, 1) + gid = state.groupdict.get(name) + if gid is None: + msg = "unknown group name %r" % name + raise source.error(msg, len(name) + 1) + if not state.checkgroup(gid): + raise source.error("cannot refer to an open group", + len(name) + 1) + state.checklookbehindgroup(gid, source) + subpatternappend((GROUPREF, gid)) + continue + + else: + char = sourceget() + if char is None: + raise source.error("unexpected end of pattern") + raise source.error("unknown extension ?P" + char, + len(char) + 2) + elif char == ":": + # non-capturing group + capture = False + elif char == "#": + # comment + while True: + if source.next is None: + raise source.error("missing ), unterminated comment", + source.tell() - start) + if sourceget() == ")": + break + continue + + elif char in "=!<": + # lookahead assertions + dir = 1 + if char == "<": + char = sourceget() + if char is None: + raise source.error("unexpected end of pattern") + if char not in "=!": + raise source.error("unknown extension ?<" + char, + len(char) + 2) + dir = -1 # lookbehind + lookbehindgroups = state.lookbehindgroups + if lookbehindgroups is None: + state.lookbehindgroups = state.groups + p = _parse_sub(source, state, verbose, nested + 1) + if dir < 0: + if lookbehindgroups is None: + state.lookbehindgroups = None + if not sourcematch(")"): + raise source.error("missing ), unterminated subpattern", + source.tell() - start) + if char == "=": + subpatternappend((ASSERT, (dir, p))) + elif p: + subpatternappend((ASSERT_NOT, (dir, p))) + else: + subpatternappend((FAILURE, ())) + continue + + elif char == "(": + # conditional backreference group + condname = source.getuntil(")", "group name") + if not (condname.isdecimal() and condname.isascii()): + source.checkgroupname(condname, 1) + condgroup = state.groupdict.get(condname) + if condgroup is None: + msg = "unknown group name %r" % condname + raise source.error(msg, len(condname) + 1) + else: + condgroup = int(condname) + if not condgroup: + raise source.error("bad group number", + len(condname) + 1) + if condgroup >= MAXGROUPS: + msg = "invalid group reference %d" % condgroup + raise source.error(msg, len(condname) + 1) + if condgroup not in state.grouprefpos: + state.grouprefpos[condgroup] = ( + source.tell() - len(condname) - 1 + ) + state.checklookbehindgroup(condgroup, source) + item_yes = _parse(source, state, verbose, nested + 1) + if source.match("|"): + item_no = _parse(source, state, verbose, nested + 1) + if source.next == "|": + raise source.error("conditional backref with more than two branches") + else: + item_no = None + if not source.match(")"): + raise source.error("missing ), unterminated subpattern", + source.tell() - start) + subpatternappend((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) + continue + + elif char == ">": + # non-capturing, atomic group + capture = False + atomic = True + elif char in FLAGS or char == "-": + # flags + flags = _parse_flags(source, state, char) + if flags is None: # global flags + if not first or subpattern: + raise source.error('global flags not at the start ' + 'of the expression', + source.tell() - start) + verbose = state.flags & SRE_FLAG_VERBOSE + continue + + add_flags, del_flags = flags + capture = False + else: + raise source.error("unknown extension ?" + char, + len(char) + 1) + + # parse group contents + if capture: + try: + group = state.opengroup(name) + except error as err: + raise source.error(err.msg, len(name) + 1) from None + else: + group = None + sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and + not (del_flags & SRE_FLAG_VERBOSE)) + p = _parse_sub(source, state, sub_verbose, nested + 1) + if not source.match(")"): + raise source.error("missing ), unterminated subpattern", + source.tell() - start) + if group is not None: + state.closegroup(group, p) + if atomic: + assert group is None + subpatternappend((ATOMIC_GROUP, p)) + else: + subpatternappend((SUBPATTERN, (group, add_flags, del_flags, p))) + + elif this == "^": + subpatternappend((AT, AT_BEGINNING)) + + elif this == "$": + subpatternappend((AT, AT_END)) + + else: + raise AssertionError("unsupported special character %r" % (char,)) + + # unpack non-capturing groups + for i in range(len(subpattern))[::-1]: + op, av = subpattern[i] + if op is SUBPATTERN: + group, add_flags, del_flags, p = av + if group is None and not add_flags and not del_flags: + subpattern[i: i+1] = p + + return subpattern + +def _parse_flags(source, state, char): + sourceget = source.get + add_flags = 0 + del_flags = 0 + if char != "-": + while True: + flag = FLAGS[char] + if source.istext: + if char == 'L': + msg = "bad inline flags: cannot use 'L' flag with a str pattern" + raise source.error(msg) + else: + if char == 'u': + msg = "bad inline flags: cannot use 'u' flag with a bytes pattern" + raise source.error(msg) + add_flags |= flag + if (flag & TYPE_FLAGS) and (add_flags & TYPE_FLAGS) != flag: + msg = "bad inline flags: flags 'a', 'u' and 'L' are incompatible" + raise source.error(msg) + char = sourceget() + if char is None: + raise source.error("missing -, : or )") + if char in ")-:": + break + if char not in FLAGS: + msg = "unknown flag" if char.isalpha() else "missing -, : or )" + raise source.error(msg, len(char)) + if char == ")": + state.flags |= add_flags + return None + if add_flags & GLOBAL_FLAGS: + raise source.error("bad inline flags: cannot turn on global flag", 1) + if char == "-": + char = sourceget() + if char is None: + raise source.error("missing flag") + if char not in FLAGS: + msg = "unknown flag" if char.isalpha() else "missing flag" + raise source.error(msg, len(char)) + while True: + flag = FLAGS[char] + if flag & TYPE_FLAGS: + msg = "bad inline flags: cannot turn off flags 'a', 'u' and 'L'" + raise source.error(msg) + del_flags |= flag + char = sourceget() + if char is None: + raise source.error("missing :") + if char == ":": + break + if char not in FLAGS: + msg = "unknown flag" if char.isalpha() else "missing :" + raise source.error(msg, len(char)) + assert char == ":" + if del_flags & GLOBAL_FLAGS: + raise source.error("bad inline flags: cannot turn off global flag", 1) + if add_flags & del_flags: + raise source.error("bad inline flags: flag turned on and off", 1) + return add_flags, del_flags + +def fix_flags(src, flags): + # Check and fix flags according to the type of pattern (str or bytes) + if isinstance(src, str): + if flags & SRE_FLAG_LOCALE: + raise ValueError("cannot use LOCALE flag with a str pattern") + if not flags & SRE_FLAG_ASCII: + flags |= SRE_FLAG_UNICODE + elif flags & SRE_FLAG_UNICODE: + raise ValueError("ASCII and UNICODE flags are incompatible") + else: + if flags & SRE_FLAG_UNICODE: + raise ValueError("cannot use UNICODE flag with a bytes pattern") + if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII: + raise ValueError("ASCII and LOCALE flags are incompatible") + return flags + +def parse(str, flags=0, state=None): + # parse 're' pattern into list of (opcode, argument) tuples + + source = Tokenizer(str) + + if state is None: + state = State() + state.flags = flags + state.str = str + + p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0) + p.state.flags = fix_flags(str, p.state.flags) + + if source.next is not None: + assert source.next == ")" + raise source.error("unbalanced parenthesis") + + for g in p.state.grouprefpos: + if g >= p.state.groups: + msg = "invalid group reference %d" % g + raise error(msg, str, p.state.grouprefpos[g]) + + if flags & SRE_FLAG_DEBUG: + p.dump() + + return p + +def parse_template(source, pattern): + # parse 're' replacement string into list of literals and + # group references + s = Tokenizer(source) + sget = s.get + result = [] + literal = [] + lappend = literal.append + def addliteral(): + if s.istext: + result.append(''.join(literal)) + else: + # The tokenizer implicitly decodes bytes objects as latin-1, we must + # therefore re-encode the final representation. + result.append(''.join(literal).encode('latin-1')) + del literal[:] + def addgroup(index, pos): + if index > pattern.groups: + raise s.error("invalid group reference %d" % index, pos) + addliteral() + result.append(index) + groupindex = pattern.groupindex + while True: + this = sget() + if this is None: + break # end of replacement string + if this[0] == "\\": + # group + c = this[1] + if c == "g": + if not s.match("<"): + raise s.error("missing <") + name = s.getuntil(">", "group name") + if not (name.isdecimal() and name.isascii()): + s.checkgroupname(name, 1) + try: + index = groupindex[name] + except KeyError: + raise IndexError("unknown group name %r" % name) from None + else: + index = int(name) + if index >= MAXGROUPS: + raise s.error("invalid group reference %d" % index, + len(name) + 1) + addgroup(index, len(name) + 1) + elif c == "0": + if s.next in OCTDIGITS: + this += sget() + if s.next in OCTDIGITS: + this += sget() + lappend(chr(int(this[1:], 8) & 0xff)) + elif c in DIGITS: + isoctal = False + if s.next in DIGITS: + this += sget() + if (c in OCTDIGITS and this[2] in OCTDIGITS and + s.next in OCTDIGITS): + this += sget() + isoctal = True + c = int(this[1:], 8) + if c > 0o377: + raise s.error('octal escape value %s outside of ' + 'range 0-0o377' % this, len(this)) + lappend(chr(c)) + if not isoctal: + addgroup(int(this[1:]), len(this) - 1) + else: + try: + this = chr(ESCAPES[this][1]) + except KeyError: + if c in ASCIILETTERS: + raise s.error('bad escape %s' % this, len(this)) from None + lappend(this) + else: + lappend(this) + addliteral() + return result diff --git a/Lib/reprlib.py b/Lib/reprlib.py index 616b3439b5d..ab18247682b 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -28,50 +28,101 @@ def wrapper(self): wrapper.__doc__ = getattr(user_function, '__doc__') wrapper.__name__ = getattr(user_function, '__name__') wrapper.__qualname__ = getattr(user_function, '__qualname__') - wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + wrapper.__annotate__ = getattr(user_function, '__annotate__', None) + wrapper.__type_params__ = getattr(user_function, '__type_params__', ()) + wrapper.__wrapped__ = user_function return wrapper return decorating_function class Repr: - - def __init__(self): - self.maxlevel = 6 - self.maxtuple = 6 - self.maxlist = 6 - self.maxarray = 5 - self.maxdict = 4 - self.maxset = 6 - self.maxfrozenset = 6 - self.maxdeque = 6 - self.maxstring = 30 - self.maxlong = 40 - self.maxother = 30 + _lookup = { + 'tuple': 'builtins', + 'list': 'builtins', + 'array': 'array', + 'set': 'builtins', + 'frozenset': 'builtins', + 'deque': 'collections', + 'dict': 'builtins', + 'str': 'builtins', + 'int': 'builtins' + } + + def __init__( + self, *, maxlevel=6, maxtuple=6, maxlist=6, maxarray=5, maxdict=4, + maxset=6, maxfrozenset=6, maxdeque=6, maxstring=30, maxlong=40, + maxother=30, fillvalue='...', indent=None, + ): + self.maxlevel = maxlevel + self.maxtuple = maxtuple + self.maxlist = maxlist + self.maxarray = maxarray + self.maxdict = maxdict + self.maxset = maxset + self.maxfrozenset = maxfrozenset + self.maxdeque = maxdeque + self.maxstring = maxstring + self.maxlong = maxlong + self.maxother = maxother + self.fillvalue = fillvalue + self.indent = indent def repr(self, x): return self.repr1(x, self.maxlevel) def repr1(self, x, level): - typename = type(x).__name__ + cls = type(x) + typename = cls.__name__ + if ' ' in typename: parts = typename.split() typename = '_'.join(parts) - if hasattr(self, 'repr_' + typename): - return getattr(self, 'repr_' + typename)(x, level) - else: - return self.repr_instance(x, level) + + method = getattr(self, 'repr_' + typename, None) + if method: + # not defined in this class + if typename not in self._lookup: + return method(x, level) + module = getattr(cls, '__module__', None) + # defined in this class and is the module intended + if module == self._lookup[typename]: + return method(x, level) + + return self.repr_instance(x, level) + + def _join(self, pieces, level): + if self.indent is None: + return ', '.join(pieces) + if not pieces: + return '' + indent = self.indent + if isinstance(indent, int): + if indent < 0: + raise ValueError( + f'Repr.indent cannot be negative int (was {indent!r})' + ) + indent *= ' ' + try: + sep = ',\n' + (self.maxlevel - level + 1) * indent + except TypeError as error: + raise TypeError( + f'Repr.indent must be a str, int or None, not {type(indent)}' + ) from error + return sep.join(('', *pieces, ''))[1:-len(indent) or None] def _repr_iterable(self, x, level, left, right, maxiter, trail=''): n = len(x) if level <= 0 and n: - s = '...' + s = self.fillvalue else: newlevel = level - 1 repr1 = self.repr1 pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)] - if n > maxiter: pieces.append('...') - s = ', '.join(pieces) - if n == 1 and trail: right = trail + right + if n > maxiter: + pieces.append(self.fillvalue) + s = self._join(pieces, level) + if n == 1 and trail and self.indent is None: + right = trail + right return '%s%s%s' % (left, s, right) def repr_tuple(self, x, level): @@ -104,8 +155,10 @@ def repr_deque(self, x, level): def repr_dict(self, x, level): n = len(x) - if n == 0: return '{}' - if level <= 0: return '{...}' + if n == 0: + return '{}' + if level <= 0: + return '{' + self.fillvalue + '}' newlevel = level - 1 repr1 = self.repr1 pieces = [] @@ -113,8 +166,9 @@ def repr_dict(self, x, level): keyrepr = repr1(key, newlevel) valrepr = repr1(x[key], newlevel) pieces.append('%s: %s' % (keyrepr, valrepr)) - if n > self.maxdict: pieces.append('...') - s = ', '.join(pieces) + if n > self.maxdict: + pieces.append(self.fillvalue) + s = self._join(pieces, level) return '{%s}' % (s,) def repr_str(self, x, level): @@ -123,15 +177,30 @@ def repr_str(self, x, level): i = max(0, (self.maxstring-3)//2) j = max(0, self.maxstring-3-i) s = builtins.repr(x[:i] + x[len(x)-j:]) - s = s[:i] + '...' + s[len(s)-j:] + s = s[:i] + self.fillvalue + s[len(s)-j:] return s def repr_int(self, x, level): - s = builtins.repr(x) # XXX Hope this isn't too slow... + try: + s = builtins.repr(x) + except ValueError as exc: + assert 'sys.set_int_max_str_digits()' in str(exc) + # Those imports must be deferred due to Python's build system + # where the reprlib module is imported before the math module. + import math, sys + # Integers with more than sys.get_int_max_str_digits() digits + # are rendered differently as their repr() raises a ValueError. + # See https://github.com/python/cpython/issues/135487. + k = 1 + int(math.log10(abs(x))) + # Note: math.log10(abs(x)) may be overestimated or underestimated, + # but for simplicity, we do not compute the exact number of digits. + max_digits = sys.get_int_max_str_digits() + return (f'<{x.__class__.__name__} instance with roughly {k} ' + f'digits (limit at {max_digits}) at 0x{id(x):x}>') if len(s) > self.maxlong: i = max(0, (self.maxlong-3)//2) j = max(0, self.maxlong-3-i) - s = s[:i] + '...' + s[len(s)-j:] + s = s[:i] + self.fillvalue + s[len(s)-j:] return s def repr_instance(self, x, level): @@ -144,7 +213,7 @@ def repr_instance(self, x, level): if len(s) > self.maxother: i = max(0, (self.maxother-3)//2) j = max(0, self.maxother-3-i) - s = s[:i] + '...' + s[len(s)-j:] + s = s[:i] + self.fillvalue + s[len(s)-j:] return s diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index bca4a7bc521..23eb0020f42 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -31,7 +31,11 @@ import atexit import builtins +import inspect +import keyword +import re import __main__ +import warnings __all__ = ["Completer"] @@ -85,10 +89,11 @@ def complete(self, text, state): return None if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) + with warnings.catch_warnings(action="ignore"): + if "." in text: + self.matches = self.attr_matches(text) + else: + self.matches = self.global_matches(text) try: return self.matches[state] except IndexError: @@ -96,7 +101,13 @@ def complete(self, text, state): def _callable_postfix(self, val, word): if callable(val): - word = word + "(" + word += "(" + try: + if not inspect.signature(val).parameters: + word += ")" + except ValueError: + pass + return word def global_matches(self, text): @@ -106,18 +117,17 @@ def global_matches(self, text): defined in self.namespace that match. """ - import keyword matches = [] seen = {"__builtins__"} n = len(text) - for word in keyword.kwlist: + for word in keyword.kwlist + keyword.softkwlist: if word[:n] == text: seen.add(word) if word in {'finally', 'try'}: word = word + ':' elif word not in {'False', 'None', 'True', 'break', 'continue', 'pass', - 'else'}: + 'else', '_'}: word = word + ' ' matches.append(word) for nspace in [self.namespace, builtins.__dict__]: @@ -139,7 +149,6 @@ def attr_matches(self, text): with a __getattr__ hook is evaluated. """ - import re m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text) if not m: return [] @@ -169,13 +178,20 @@ def attr_matches(self, text): if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - try: - val = getattr(thisobject, word) - except Exception: - pass # Include even if attribute not set + if isinstance(getattr(type(thisobject), word, None), + property): + # bpo-44752: thisobject.word is a method decorated by + # `@property`. What follows applies a postfix if + # thisobject.word is callable, but know we know that + # this is not callable (because it is a property). + # Also, getattr(thisobject, word) will evaluate the + # property method, which is not desirable. + matches.append(match) + continue + if (value := getattr(thisobject, word, None)) is not None: + matches.append(self._callable_postfix(value, match)) else: - match = self._callable_postfix(val, match) - matches.append(match) + matches.append(match) if matches or not noprefix: break if noprefix == '_': diff --git a/Lib/runpy.py b/Lib/runpy.py index c7d3d8caad1..ef54d3282ee 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -14,18 +14,20 @@ import importlib.machinery # importlib first so we can test #15386 via -m import importlib.util import io -import types import os __all__ = [ "run_module", "run_path", ] +# avoid 'import types' just for ModuleType +ModuleType = type(sys) + class _TempModule(object): """Temporarily replace a module in sys.modules with an empty namespace""" def __init__(self, mod_name): self.mod_name = mod_name - self.module = types.ModuleType(mod_name) + self.module = ModuleType(mod_name) self._saved_module = [] def __enter__(self): @@ -245,17 +247,17 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(run_name, fname): +def _get_code_from_file(fname): # Check for a compiled file first from pkgutil import read_code - decoded_path = os.path.abspath(os.fsdecode(fname)) - with io.open_code(decoded_path) as f: + code_path = os.path.abspath(fname) + with io.open_code(code_path) as f: code = read_code(f) if code is None: # That didn't work, so try it as normal source code - with io.open_code(decoded_path) as f: + with io.open_code(code_path) as f: code = compile(f.read(), fname, 'exec') - return code, fname + return code def run_path(path_name, init_globals=None, run_name=None): """Execute code located at the specified filesystem location. @@ -277,17 +279,13 @@ def run_path(path_name, init_globals=None, run_name=None): pkg_name = run_name.rpartition(".")[0] from pkgutil import get_importer importer = get_importer(path_name) - # Trying to avoid importing imp so as to not consume the deprecation warning. - is_NullImporter = False - if type(importer).__module__ == 'imp': - if type(importer).__name__ == 'NullImporter': - is_NullImporter = True - if isinstance(importer, type(None)) or is_NullImporter: + path_name = os.fsdecode(path_name) + if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code, fname = _get_code_from_file(run_name, path_name) + code = _get_code_from_file(path_name) return _run_module_code(code, init_globals, run_name, - pkg_name=pkg_name, script_name=fname) + pkg_name=pkg_name, script_name=path_name) else: # Finder is defined for path, so add it to # the start of sys.path diff --git a/Lib/sched.py b/Lib/sched.py index 14613cf2987..fb20639d459 100644 --- a/Lib/sched.py +++ b/Lib/sched.py @@ -11,7 +11,7 @@ implement simulated time by writing your own functions. This can also be used to integrate scheduling with STDWIN events; the delay function is allowed to modify the queue. Time can be expressed as -integers or floating point numbers, as long as it is consistent. +integers or floating-point numbers, as long as it is consistent. Events are specified by tuples (time, priority, action, argument, kwargs). As in UNIX, lower priority numbers mean higher priority; in this diff --git a/Lib/secrets.py b/Lib/secrets.py index a546efbdd42..566a09b7311 100644 --- a/Lib/secrets.py +++ b/Lib/secrets.py @@ -2,7 +2,7 @@ managing secrets such as account authentication, tokens, and similar. See PEP 506 for more information. -https://www.python.org/dev/peps/pep-0506/ +https://peps.python.org/pep-0506/ """ @@ -13,7 +13,6 @@ import base64 -import binascii from hmac import compare_digest from random import SystemRandom @@ -56,7 +55,7 @@ def token_hex(nbytes=None): 'f9bf78b9a18ce6d46a0cd2b0b86df9da' """ - return binascii.hexlify(token_bytes(nbytes)).decode('ascii') + return token_bytes(nbytes).hex() def token_urlsafe(nbytes=None): """Return a random URL-safe text string, in Base64 encoding. diff --git a/Lib/selectors.py b/Lib/selectors.py index bb15a1cb1ba..b8e5f6a4f77 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -50,12 +50,11 @@ def _fileobj_to_fd(fileobj): Object used to associate a file object to its backing file descriptor, selected event mask, and attached data. """ -if sys.version_info >= (3, 5): - SelectorKey.fileobj.__doc__ = 'File object registered.' - SelectorKey.fd.__doc__ = 'Underlying file descriptor.' - SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' - SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. - For example, this could be used to store a per-client session ID.''') +SelectorKey.fileobj.__doc__ = 'File object registered.' +SelectorKey.fd.__doc__ = 'Underlying file descriptor.' +SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' +SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. +For example, this could be used to store a per-client session ID.''') class _SelectorMapping(Mapping): @@ -67,12 +66,16 @@ def __init__(self, selector): def __len__(self): return len(self._selector._fd_to_key) + def get(self, fileobj, default=None): + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key.get(fd, default) + def __getitem__(self, fileobj): - try: - fd = self._selector._fileobj_lookup(fileobj) - return self._selector._fd_to_key[fd] - except KeyError: - raise KeyError("{!r} is not registered".format(fileobj)) from None + fd = self._selector._fileobj_lookup(fileobj) + key = self._selector._fd_to_key.get(fd) + if key is None: + raise KeyError("{!r} is not registered".format(fileobj)) + return key def __iter__(self): return iter(self._selector._fd_to_key) @@ -273,19 +276,6 @@ def close(self): def get_map(self): return self._map - def _key_from_fd(self, fd): - """Return the key associated to a given file descriptor. - - Parameters: - fd -- file descriptor - - Returns: - corresponding key, or None if not found - """ - try: - return self._fd_to_key[fd] - except KeyError: - return None class SelectSelector(_BaseSelectorImpl): @@ -324,17 +314,15 @@ def select(self, timeout=None): r, w, _ = self._select(self._readers, self._writers, [], timeout) except InterruptedError: return ready - r = set(r) - w = set(w) - for fd in r | w: - events = 0 - if fd in r: - events |= EVENT_READ - if fd in w: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) + r = frozenset(r) + w = frozenset(w) + rw = r | w + fd_to_key_get = self._fd_to_key.get + for fd in rw: + key = fd_to_key_get(fd) if key: + events = ((fd in r and EVENT_READ) + | (fd in w and EVENT_WRITE)) ready.append((key, events & key.events)) return ready @@ -351,11 +339,8 @@ def __init__(self): def register(self, fileobj, events, data=None): key = super().register(fileobj, events, data) - poller_events = 0 - if events & EVENT_READ: - poller_events |= self._EVENT_READ - if events & EVENT_WRITE: - poller_events |= self._EVENT_WRITE + poller_events = ((events & EVENT_READ and self._EVENT_READ) + | (events & EVENT_WRITE and self._EVENT_WRITE) ) try: self._selector.register(key.fd, poller_events) except: @@ -381,11 +366,8 @@ def modify(self, fileobj, events, data=None): changed = False if events != key.events: - selector_events = 0 - if events & EVENT_READ: - selector_events |= self._EVENT_READ - if events & EVENT_WRITE: - selector_events |= self._EVENT_WRITE + selector_events = ((events & EVENT_READ and self._EVENT_READ) + | (events & EVENT_WRITE and self._EVENT_WRITE)) try: self._selector.modify(key.fd, selector_events) except: @@ -416,15 +398,13 @@ def select(self, timeout=None): fd_event_list = self._selector.poll(timeout) except InterruptedError: return ready - for fd, event in fd_event_list: - events = 0 - if event & ~self._EVENT_READ: - events |= EVENT_WRITE - if event & ~self._EVENT_WRITE: - events |= EVENT_READ - key = self._key_from_fd(fd) + fd_to_key_get = self._fd_to_key.get + for fd, event in fd_event_list: + key = fd_to_key_get(fd) if key: + events = ((event & ~self._EVENT_READ and EVENT_WRITE) + | (event & ~self._EVENT_WRITE and EVENT_READ)) ready.append((key, events & key.events)) return ready @@ -440,6 +420,9 @@ class PollSelector(_PollLikeSelector): if hasattr(select, 'epoll'): + _NOT_EPOLLIN = ~select.EPOLLIN + _NOT_EPOLLOUT = ~select.EPOLLOUT + class EpollSelector(_PollLikeSelector): """Epoll-based selector.""" _selector_cls = select.epoll @@ -462,22 +445,20 @@ def select(self, timeout=None): # epoll_wait() expects `maxevents` to be greater than zero; # we want to make sure that `select()` can be called when no # FD is registered. - max_ev = max(len(self._fd_to_key), 1) + max_ev = len(self._fd_to_key) or 1 ready = [] try: fd_event_list = self._selector.poll(timeout, max_ev) except InterruptedError: return ready - for fd, event in fd_event_list: - events = 0 - if event & ~select.EPOLLIN: - events |= EVENT_WRITE - if event & ~select.EPOLLOUT: - events |= EVENT_READ - key = self._key_from_fd(fd) + fd_to_key = self._fd_to_key + for fd, event in fd_event_list: + key = fd_to_key.get(fd) if key: + events = ((event & _NOT_EPOLLIN and EVENT_WRITE) + | (event & _NOT_EPOLLOUT and EVENT_READ)) ready.append((key, events & key.events)) return ready @@ -510,6 +491,7 @@ class KqueueSelector(_BaseSelectorImpl): def __init__(self): super().__init__() self._selector = select.kqueue() + self._max_events = 0 def fileno(self): return self._selector.fileno() @@ -521,10 +503,12 @@ def register(self, fileobj, events, data=None): kev = select.kevent(key.fd, select.KQ_FILTER_READ, select.KQ_EV_ADD) self._selector.control([kev], 0, 0) + self._max_events += 1 if events & EVENT_WRITE: kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD) self._selector.control([kev], 0, 0) + self._max_events += 1 except: super().unregister(fileobj) raise @@ -535,6 +519,7 @@ def unregister(self, fileobj): if key.events & EVENT_READ: kev = select.kevent(key.fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE) + self._max_events -= 1 try: self._selector.control([kev], 0, 0) except OSError: @@ -544,6 +529,7 @@ def unregister(self, fileobj): if key.events & EVENT_WRITE: kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE) + self._max_events -= 1 try: self._selector.control([kev], 0, 0) except OSError: @@ -556,23 +542,21 @@ def select(self, timeout=None): # If max_ev is 0, kqueue will ignore the timeout. For consistent # behavior with the other selector classes, we prevent that here # (using max). See https://bugs.python.org/issue29255 - max_ev = max(len(self._fd_to_key), 1) + max_ev = self._max_events or 1 ready = [] try: kev_list = self._selector.control(None, max_ev, timeout) except InterruptedError: return ready + + fd_to_key_get = self._fd_to_key.get for kev in kev_list: fd = kev.ident flag = kev.filter - events = 0 - if flag == select.KQ_FILTER_READ: - events |= EVENT_READ - if flag == select.KQ_FILTER_WRITE: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) + key = fd_to_key_get(fd) if key: + events = ((flag == select.KQ_FILTER_READ and EVENT_READ) + | (flag == select.KQ_FILTER_WRITE and EVENT_WRITE)) ready.append((key, events & key.events)) return ready diff --git a/Lib/shelve.py b/Lib/shelve.py index 5d443a0fa8d..50584716e9e 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -56,7 +56,7 @@ the persistent dictionary on disk, if feasible). """ -from pickle import Pickler, Unpickler +from pickle import DEFAULT_PROTOCOL, Pickler, Unpickler from io import BytesIO import collections.abc @@ -85,7 +85,7 @@ def __init__(self, dict, protocol=None, writeback=False, keyencoding="utf-8"): self.dict = dict if protocol is None: - protocol = 3 + protocol = DEFAULT_PROTOCOL self._protocol = protocol self.writeback = writeback self.cache = {} @@ -226,6 +226,13 @@ def __init__(self, filename, flag='c', protocol=None, writeback=False): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) + def clear(self): + """Remove all items from the shelf.""" + # Call through to the clear method on dbm-backed shelves. + # see https://github.com/python/cpython/issues/107089 + self.cache.clear() + self.dict.clear() + def open(filename, flag='c', protocol=None, writeback=False): """Open a persistent dictionary for reading and writing. diff --git a/Lib/shlex.py b/Lib/shlex.py index 4801a6c1d47..5959f52dd12 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -7,11 +7,7 @@ # iterator interface by Gustavo Niemeyer, April 2003. # changes to tokenize more like Posix shells by Vinay Sajip, July 2016. -import os -import re import sys -from collections import deque - from io import StringIO __all__ = ["shlex", "split", "quote", "join"] @@ -20,6 +16,8 @@ class shlex: "A lexical analyzer class for simple shell-like syntaxes." def __init__(self, instream=None, infile=None, posix=False, punctuation_chars=False): + from collections import deque # deferred import for performance + if isinstance(instream, str): instream = StringIO(instream) if instream is not None: @@ -278,6 +276,7 @@ def read_token(self): def sourcehook(self, newfile): "Hook called on a filename to be sourced." + import os.path if newfile[0] == '"': newfile = newfile[1:-1] # This implements cpp-like semantics for relative-path inclusion. @@ -305,9 +304,7 @@ def __next__(self): def split(s, comments=False, posix=True): """Split the string *s* using shell-like syntax.""" if s is None: - import warnings - warnings.warn("Passing None for 's' to shlex.split() is deprecated.", - DeprecationWarning, stacklevel=2) + raise ValueError("s argument must not be None") lex = shlex(s, posix=posix) lex.whitespace_split = True if not comments: @@ -320,13 +317,20 @@ def join(split_command): return ' '.join(quote(arg) for arg in split_command) -_find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search - def quote(s): """Return a shell-escaped version of the string *s*.""" if not s: return "''" - if _find_unsafe(s) is None: + + if not isinstance(s, str): + raise TypeError(f"expected string object, got {type(s).__name__!r}") + + # Use bytes.translate() for performance + safe_chars = (b'%+,-./0123456789:=@' + b'ABCDEFGHIJKLMNOPQRSTUVWXYZ_' + b'abcdefghijklmnopqrstuvwxyz') + # No quoting is needed if `s` is an ASCII string consisting only of `safe_chars` + if s.isascii() and not s.encode().translate(None, delete=safe_chars): return s # use single quotes, and put single quotes into double quotes @@ -335,10 +339,7 @@ def quote(s): def _print_tokens(lexer): - while 1: - tt = lexer.get_token() - if not tt: - break + while tt := lexer.get_token(): print("Token: " + repr(tt)) if __name__ == '__main__': diff --git a/Lib/shutil.py b/Lib/shutil.py index 31336e08e81..8d8fe145567 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -33,14 +33,11 @@ _LZMA_SUPPORTED = False try: - from pwd import getpwnam + from compression import zstd + del zstd + _ZSTD_SUPPORTED = True except ImportError: - getpwnam = None - -try: - from grp import getgrnam -except ImportError: - getgrnam = None + _ZSTD_SUPPORTED = False _WINDOWS = os.name == 'nt' posix = nt = None @@ -49,13 +46,25 @@ elif _WINDOWS: import nt -COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 -_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") +if sys.platform == 'win32': + import _winapi +else: + _winapi = None + +COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 256 * 1024 +# This should never be removed, see rationale in: +# https://bugs.python.org/issue43743#msg393429 +_USE_CP_SENDFILE = (hasattr(os, "sendfile") + and sys.platform.startswith(("linux", "android", "sunos"))) +_USE_CP_COPY_FILE_RANGE = hasattr(os, "copy_file_range") _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS +# CMD defaults in Windows 10 +_WIN_DEFAULT_PATHEXT = ".COM;.EXE;.BAT;.CMD;.VBS;.JS;.WS;.MSC" + __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "copytree", "move", "rmtree", "Error", "SpecialFileError", - "ExecError", "make_archive", "get_archive_formats", + "make_archive", "get_archive_formats", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", "unregister_unpack_format", "unpack_archive", @@ -73,8 +82,6 @@ class SpecialFileError(OSError): """Raised when trying to do a kind of operation (e.g. copying) which is not supported on a special file (e.g. a named pipe)""" -class ExecError(OSError): - """Raised when a command could not be executed""" class ReadError(OSError): """Raised when an archive cannot be read""" @@ -108,10 +115,70 @@ def _fastcopy_fcopyfile(fsrc, fdst, flags): else: raise err from None +def _determine_linux_fastcopy_blocksize(infd): + """Determine blocksize for fastcopying on Linux. + + Hopefully the whole file will be copied in a single call. + The copying itself should be performed in a loop 'till EOF is + reached (0 return) so a blocksize smaller or bigger than the actual + file size should not make any difference, also in case the file + content changes while being copied. + """ + try: + blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB + except OSError: + blocksize = 2 ** 27 # 128 MiB + # On 32-bit architectures truncate to 1 GiB to avoid OverflowError, + # see gh-82500. + if sys.maxsize < 2 ** 32: + blocksize = min(blocksize, 2 ** 30) + return blocksize + +def _fastcopy_copy_file_range(fsrc, fdst): + """Copy data from one regular mmap-like fd to another by using + a high-performance copy_file_range(2) syscall that gives filesystems + an opportunity to implement the use of reflinks or server-side copy. + + This should work on Linux >= 4.5 only. + """ + try: + infd = fsrc.fileno() + outfd = fdst.fileno() + except Exception as err: + raise _GiveupOnFastCopy(err) # not a regular file + + blocksize = _determine_linux_fastcopy_blocksize(infd) + offset = 0 + while True: + try: + n_copied = os.copy_file_range(infd, outfd, blocksize, offset_dst=offset) + except OSError as err: + # ...in oder to have a more informative exception. + err.filename = fsrc.name + err.filename2 = fdst.name + + if err.errno == errno.ENOSPC: # filesystem is full + raise err from None + + # Give up on first call and if no data was copied. + if offset == 0 and os.lseek(outfd, 0, os.SEEK_CUR) == 0: + raise _GiveupOnFastCopy(err) + + raise err + else: + if n_copied == 0: + # If no bytes have been copied yet, copy_file_range + # might silently fail. + # https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0 + if offset == 0: + raise _GiveupOnFastCopy() + break + offset += n_copied + def _fastcopy_sendfile(fsrc, fdst): """Copy data from one regular mmap-like fd to another by using high-performance sendfile(2) syscall. - This should work on Linux >= 2.6.33 only. + This should work on Linux >= 2.6.33, Android and Solaris. """ # Note: copyfileobj() is left alone in order to not introduce any # unexpected breakage. Possible risks by using zero-copy calls @@ -129,39 +196,24 @@ def _fastcopy_sendfile(fsrc, fdst): except Exception as err: raise _GiveupOnFastCopy(err) # not a regular file - # Hopefully the whole file will be copied in a single call. - # sendfile() is called in a loop 'till EOF is reached (0 return) - # so a bufsize smaller or bigger than the actual file size - # should not make any difference, also in case the file content - # changes while being copied. - try: - blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8MiB - except OSError: - blocksize = 2 ** 27 # 128MiB - # On 32-bit architectures truncate to 1GiB to avoid OverflowError, - # see bpo-38319. - if sys.maxsize < 2 ** 32: - blocksize = min(blocksize, 2 ** 30) - + blocksize = _determine_linux_fastcopy_blocksize(infd) offset = 0 while True: try: sent = os.sendfile(outfd, infd, offset, blocksize) except OSError as err: - # ...in oder to have a more informative exception. + # ...in order to have a more informative exception. err.filename = fsrc.name err.filename2 = fdst.name - # XXX RUSTPYTHON TODO: consistent OSError.errno - if hasattr(err, "errno") and err.errno == errno.ENOTSOCK: + if err.errno == errno.ENOTSOCK: # sendfile() on this platform (probably Linux < 2.6.33) # does not support copies between regular files (only # sockets). _USE_CP_SENDFILE = False raise _GiveupOnFastCopy(err) - # XXX RUSTPYTHON TODO: consistent OSError.errno - if hasattr(err, "errno") and err.errno == errno.ENOSPC: # filesystem is full + if err.errno == errno.ENOSPC: # filesystem is full raise err from None # Give up on first call and if no data was copied. @@ -189,21 +241,19 @@ def _copyfileobj_readinto(fsrc, fdst, length=COPY_BUFSIZE): break elif n < length: with mv[:n] as smv: - fdst.write(smv) + fdst_write(smv) + break else: fdst_write(mv) def copyfileobj(fsrc, fdst, length=0): """copy data from file-like object fsrc to file-like object fdst""" - # Localize variable access to minimize overhead. if not length: length = COPY_BUFSIZE + # Localize variable access to minimize overhead. fsrc_read = fsrc.read fdst_write = fdst.write - while True: - buf = fsrc_read(length) - if not buf: - break + while buf := fsrc_read(length): fdst_write(buf) def _samefile(src, dst): @@ -260,28 +310,45 @@ def copyfile(src, dst, *, follow_symlinks=True): if not follow_symlinks and _islink(src): os.symlink(os.readlink(src), dst) else: - with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: - # macOS - if _HAS_FCOPYFILE: - try: - _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA) - return dst - except _GiveupOnFastCopy: - pass - # Linux - elif _USE_CP_SENDFILE: - try: - _fastcopy_sendfile(fsrc, fdst) - return dst - except _GiveupOnFastCopy: - pass - # Windows, see: - # https://github.com/python/cpython/pull/7160#discussion_r195405230 - elif _WINDOWS and file_size > 0: - _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE)) - return dst - - copyfileobj(fsrc, fdst) + with open(src, 'rb') as fsrc: + try: + with open(dst, 'wb') as fdst: + # macOS + if _HAS_FCOPYFILE: + try: + _fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA) + return dst + except _GiveupOnFastCopy: + pass + # Linux / Android / Solaris + elif _USE_CP_SENDFILE or _USE_CP_COPY_FILE_RANGE: + # reflink may be implicit in copy_file_range. + if _USE_CP_COPY_FILE_RANGE: + try: + _fastcopy_copy_file_range(fsrc, fdst) + return dst + except _GiveupOnFastCopy: + pass + if _USE_CP_SENDFILE: + try: + _fastcopy_sendfile(fsrc, fdst) + return dst + except _GiveupOnFastCopy: + pass + # Windows, see: + # https://github.com/python/cpython/pull/7160#discussion_r195405230 + elif _WINDOWS and file_size > 0: + _copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE)) + return dst + + copyfileobj(fsrc, fdst) + + # Issue 43219, raise a less confusing exception + except IsADirectoryError as e: + if not os.path.exists(dst): + raise FileNotFoundError(f'Directory does not exist: {dst}') from e + else: + raise return dst @@ -301,7 +368,12 @@ def copymode(src, dst, *, follow_symlinks=True): else: return else: - stat_func, chmod_func = _stat, os.chmod + stat_func = _stat + if os.name == 'nt' and os.path.islink(dst): + def chmod_func(*args): + os.chmod(*args, follow_symlinks=True) + else: + chmod_func = os.chmod st = stat_func(src) chmod_func(dst, stat.S_IMODE(st.st_mode)) @@ -328,7 +400,7 @@ def _copyxattr(src, dst, *, follow_symlinks=True): os.setxattr(dst, name, value, follow_symlinks=follow_symlinks) except OSError as e: if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA, - errno.EINVAL): + errno.EINVAL, errno.EACCES): raise else: def _copyxattr(*args, **kwargs): @@ -431,6 +503,29 @@ def copy2(src, dst, *, follow_symlinks=True): """ if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) + + if hasattr(_winapi, "CopyFile2"): + src_ = os.fsdecode(src) + dst_ = os.fsdecode(dst) + flags = _winapi.COPY_FILE_ALLOW_DECRYPTED_DESTINATION # for compat + if not follow_symlinks: + flags |= _winapi.COPY_FILE_COPY_SYMLINK + try: + _winapi.CopyFile2(src_, dst_, flags) + return dst + except OSError as exc: + if (exc.winerror == _winapi.ERROR_PRIVILEGE_NOT_HELD + and not follow_symlinks): + # Likely encountered a symlink we aren't allowed to create. + # Fall back on the old code + pass + elif exc.winerror == _winapi.ERROR_ACCESS_DENIED: + # Possibly encountered a hidden or readonly file we can't + # overwrite. Fall back on old code + pass + else: + raise + copyfile(src, dst, follow_symlinks=follow_symlinks) copystat(src, dst, follow_symlinks=follow_symlinks) return dst @@ -452,7 +547,7 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, if ignore is not None: ignored_names = ignore(os.fspath(src), [x.name for x in entries]) else: - ignored_names = set() + ignored_names = () os.makedirs(dst, exist_ok=dirs_exist_ok) errors = [] @@ -487,12 +582,13 @@ def _copytree(entries, src, dst, symlinks, ignore, copy_function, # otherwise let the copy occur. copy2 will raise an error if srcentry.is_dir(): copytree(srcobj, dstname, symlinks, ignore, - copy_function, dirs_exist_ok=dirs_exist_ok) + copy_function, ignore_dangling_symlinks, + dirs_exist_ok) else: copy_function(srcobj, dstname) elif srcentry.is_dir(): copytree(srcobj, dstname, symlinks, ignore, copy_function, - dirs_exist_ok=dirs_exist_ok) + ignore_dangling_symlinks, dirs_exist_ok) else: # Will raise a SpecialFileError for unsupported file types copy_function(srcobj, dstname) @@ -516,15 +612,12 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, ignore_dangling_symlinks=False, dirs_exist_ok=False): """Recursively copy a directory tree and return the destination directory. - dirs_exist_ok dictates whether to raise an exception in case dst or any - missing parent directory already exists. - If exception(s) occur, an Error is raised with a list of reasons. If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't + links are copied. If the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. @@ -549,6 +642,11 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, destination path as arguments. By default, copy2() is used, but any function that supports the same signature (like copy()) can be used. + If dirs_exist_ok is false (the default) and `dst` already exists, a + `FileExistsError` is raised. If `dirs_exist_ok` is true, the copying + operation will continue if it encounters existing directories, and files + within the `dst` tree will be overwritten by corresponding files from the + `src` tree. """ sys.audit("shutil.copytree", src, dst) with os.scandir(src) as itr: @@ -559,192 +657,219 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, dirs_exist_ok=dirs_exist_ok) if hasattr(os.stat_result, 'st_file_attributes'): - # Special handling for directory junctions to make them behave like - # symlinks for shutil.rmtree, since in general they do not appear as - # regular links. - def _rmtree_isdir(entry): - try: - st = entry.stat(follow_symlinks=False) - return (stat.S_ISDIR(st.st_mode) and not - (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT - and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) - except OSError: - return False - - def _rmtree_islink(path): - try: - st = os.lstat(path) - return (stat.S_ISLNK(st.st_mode) or - (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT - and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) - except OSError: - return False + def _rmtree_islink(st): + return (stat.S_ISLNK(st.st_mode) or + (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT + and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) else: - def _rmtree_isdir(entry): - try: - return entry.is_dir(follow_symlinks=False) - except OSError: - return False - - def _rmtree_islink(path): - return os.path.islink(path) + def _rmtree_islink(st): + return stat.S_ISLNK(st.st_mode) # version vulnerable to race conditions -def _rmtree_unsafe(path, onerror): +def _rmtree_unsafe(path, dir_fd, onexc): + if dir_fd is not None: + raise NotImplementedError("dir_fd unavailable on this platform") try: - with os.scandir(path) as scandir_it: - entries = list(scandir_it) - except OSError: - onerror(os.scandir, path, sys.exc_info()) - entries = [] - for entry in entries: - fullname = entry.path - if _rmtree_isdir(entry): + st = os.lstat(path) + except OSError as err: + onexc(os.lstat, path, err) + return + try: + if _rmtree_islink(st): + # symlinks to directories are forbidden, see bug #1669 + raise OSError("Cannot call rmtree on a symbolic link") + except OSError as err: + onexc(os.path.islink, path, err) + # can't continue even if onexc hook returns + return + def onerror(err): + if not isinstance(err, FileNotFoundError): + onexc(os.scandir, err.filename, err) + results = os.walk(path, topdown=False, onerror=onerror, followlinks=os._walk_symlinks_as_files) + for dirpath, dirnames, filenames in results: + for name in dirnames: + fullname = os.path.join(dirpath, name) try: - if entry.is_symlink(): - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or entry.is_dir above. - raise OSError("Cannot call rmtree on a symbolic link") - except OSError: - onerror(os.path.islink, fullname, sys.exc_info()) + os.rmdir(fullname) + except FileNotFoundError: continue - _rmtree_unsafe(fullname, onerror) - else: + except OSError as err: + onexc(os.rmdir, fullname, err) + for name in filenames: + fullname = os.path.join(dirpath, name) try: os.unlink(fullname) - except OSError: - onerror(os.unlink, fullname, sys.exc_info()) + except FileNotFoundError: + continue + except OSError as err: + onexc(os.unlink, fullname, err) try: os.rmdir(path) - except OSError: - onerror(os.rmdir, path, sys.exc_info()) + except FileNotFoundError: + pass + except OSError as err: + onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(topfd, path, onerror): +def _rmtree_safe_fd(path, dir_fd, onexc): + # While the unsafe rmtree works fine on bytes, the fd based does not. + if isinstance(path, bytes): + path = os.fsdecode(path) + stack = [(os.lstat, dir_fd, path, None)] + try: + while stack: + _rmtree_safe_fd_step(stack, onexc) + finally: + # Close any file descriptors still on the stack. + while stack: + func, fd, path, entry = stack.pop() + if func is not os.close: + continue + try: + os.close(fd) + except OSError as err: + onexc(os.close, path, err) + +def _rmtree_safe_fd_step(stack, onexc): + # Each stack item has four elements: + # * func: The first operation to perform: os.lstat, os.close or os.rmdir. + # Walking a directory starts with an os.lstat() to detect symlinks; in + # this case, func is updated before subsequent operations and passed to + # onexc() if an error occurs. + # * dirfd: Open file descriptor, or None if we're processing the top-level + # directory given to rmtree() and the user didn't supply dir_fd. + # * path: Path of file to operate upon. This is passed to onexc() if an + # error occurs. + # * orig_entry: os.DirEntry, or None if we're processing the top-level + # directory given to rmtree(). We used the cached stat() of the entry to + # save a call to os.lstat() when walking subdirectories. + func, dirfd, path, orig_entry = stack.pop() + name = path if orig_entry is None else orig_entry.name try: + if func is os.close: + os.close(dirfd) + return + if func is os.rmdir: + os.rmdir(name, dir_fd=dirfd) + return + + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + assert func is os.lstat + if orig_entry is None: + orig_st = os.lstat(name, dir_fd=dirfd) + else: + orig_st = orig_entry.stat(follow_symlinks=False) + + func = os.open # For error reporting. + topfd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dirfd) + + func = os.path.islink # For error reporting. + try: + if not os.path.samestat(orig_st, os.fstat(topfd)): + # Symlinks to directories are forbidden, see GH-46010. + raise OSError("Cannot call rmtree on a symbolic link") + stack.append((os.rmdir, dirfd, path, orig_entry)) + finally: + stack.append((os.close, topfd, path, orig_entry)) + + func = os.scandir # For error reporting. with os.scandir(topfd) as scandir_it: entries = list(scandir_it) - except OSError as err: - err.filename = path - onerror(os.scandir, path, sys.exc_info()) - return - for entry in entries: - fullname = os.path.join(path, entry.name) - try: - is_dir = entry.is_dir(follow_symlinks=False) - except OSError: - is_dir = False - else: - if is_dir: - try: - orig_st = entry.stat(follow_symlinks=False) - is_dir = stat.S_ISDIR(orig_st.st_mode) - except OSError: - onerror(os.lstat, fullname, sys.exc_info()) - continue - if is_dir: + for entry in entries: + fullname = os.path.join(path, entry.name) try: - dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd) + if entry.is_dir(follow_symlinks=False): + # Traverse into sub-directory. + stack.append((os.lstat, topfd, fullname, entry)) + continue + except FileNotFoundError: + continue except OSError: - onerror(os.open, fullname, sys.exc_info()) - else: - try: - if os.path.samestat(orig_st, os.fstat(dirfd)): - _rmtree_safe_fd(dirfd, fullname, onerror) - try: - os.rmdir(entry.name, dir_fd=topfd) - except OSError: - onerror(os.rmdir, fullname, sys.exc_info()) - else: - try: - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or stat.S_ISDIR above. - raise OSError("Cannot call rmtree on a symbolic " - "link") - except OSError: - onerror(os.path.islink, fullname, sys.exc_info()) - finally: - os.close(dirfd) - else: + pass try: os.unlink(entry.name, dir_fd=topfd) - except OSError: - onerror(os.unlink, fullname, sys.exc_info()) + except FileNotFoundError: + continue + except OSError as err: + onexc(os.unlink, fullname, err) + except FileNotFoundError as err: + if orig_entry is None or func is os.close: + err.filename = path + onexc(func, path, err) + except OSError as err: + err.filename = path + onexc(func, path, err) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and os.scandir in os.supports_fd and os.stat in os.supports_follow_symlinks) +_rmtree_impl = _rmtree_safe_fd if _use_fd_functions else _rmtree_unsafe -def rmtree(path, ignore_errors=False, onerror=None): +def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None): """Recursively delete a directory tree. - If ignore_errors is set, errors are ignored; otherwise, if onerror - is set, it is called to handle the error with arguments (func, + If dir_fd is not None, it should be a file descriptor open to a directory; + path will then be relative to that directory. + dir_fd may not be implemented on your platform. + If it is unavailable, using it will raise a NotImplementedError. + + If ignore_errors is set, errors are ignored; otherwise, if onexc or + onerror is set, it is called to handle the error with arguments (func, path, exc_info) where func is platform and implementation dependent; path is the argument to that function that caused it to fail; and - exc_info is a tuple returned by sys.exc_info(). If ignore_errors - is false and onerror is None, an exception is raised. + the value of exc_info describes the exception. For onexc it is the + exception instance, and for onerror it is a tuple as returned by + sys.exc_info(). If ignore_errors is false and both onexc and + onerror are None, the exception is reraised. + onerror is deprecated and only remains for backwards compatibility. + If both onerror and onexc are set, onerror is ignored and onexc is used. """ - sys.audit("shutil.rmtree", path) + + sys.audit("shutil.rmtree", path, dir_fd) if ignore_errors: - def onerror(*args): + def onexc(*args): pass - elif onerror is None: - def onerror(*args): + elif onerror is None and onexc is None: + def onexc(*args): raise - if _use_fd_functions: - # While the unsafe rmtree works fine on bytes, the fd based does not. - if isinstance(path, bytes): - path = os.fsdecode(path) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - try: - orig_st = os.lstat(path) - except Exception: - onerror(os.lstat, path, sys.exc_info()) - return - try: - fd = os.open(path, os.O_RDONLY) - except Exception: - onerror(os.lstat, path, sys.exc_info()) - return - try: - if os.path.samestat(orig_st, os.fstat(fd)): - _rmtree_safe_fd(fd, path, onerror) - try: - os.rmdir(path) - except OSError: - onerror(os.rmdir, path, sys.exc_info()) - else: - try: - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError: - onerror(os.path.islink, path, sys.exc_info()) - finally: - os.close(fd) - else: - try: - if _rmtree_islink(path): - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError: - onerror(os.path.islink, path, sys.exc_info()) - # can't continue even if onerror hook returns - return - return _rmtree_unsafe(path, onerror) + elif onexc is None: + if onerror is None: + def onexc(*args): + raise + else: + # delegate to onerror + def onexc(*args): + func, path, exc = args + if exc is None: + exc_info = None, None, None + else: + exc_info = type(exc), exc, exc.__traceback__ + return onerror(func, path, exc_info) + + _rmtree_impl(path, dir_fd, onexc) # Allow introspection of whether or not the hardening against symlink # attacks is supported on the current platform rmtree.avoids_symlink_attacks = _use_fd_functions def _basename(path): - # A basename() variant which first strips the trailing slash, if present. - # Thus we always get the last component of the path, even for directories. + """A basename() variant which first strips the trailing slash, if present. + Thus we always get the last component of the path, even for directories. + + path: Union[PathLike, str] + + e.g. + >>> os.path.basename('/bar/foo') + 'foo' + >>> os.path.basename('/bar/foo/') + '' + >>> _basename('/bar/foo/') + 'foo' + """ + path = os.fspath(path) sep = os.path.sep + (os.path.altsep or '') return os.path.basename(path.rstrip(sep)) @@ -753,12 +878,12 @@ def move(src, dst, copy_function=copy2): similar to the Unix "mv" command. Return the file or directory's destination. - If the destination is a directory or a symlink to a directory, the source - is moved inside the directory. The destination path must not already - exist. + If dst is an existing directory or a symlink to a directory, then src is + moved inside that directory. The destination path in that directory must + not already exist. - If the destination already exists but is not a directory, it may be - overwritten depending on os.rename() semantics. + If dst already exists but is not a directory, it may be overwritten + depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. Symlinks are @@ -777,13 +902,16 @@ def move(src, dst, copy_function=copy2): sys.audit("shutil.move", src, dst) real_dst = dst if os.path.isdir(dst): - if _samefile(src, dst): + if _samefile(src, dst) and not os.path.islink(src): # We might be on a case insensitive filesystem, # perform the rename anyway. os.rename(src, dst) return + # Using _basename instead of os.path.basename is important, as we must + # ignore any trailing slash to avoid the basename returning '' real_dst = os.path.join(dst, _basename(src)) + if os.path.exists(real_dst): raise Error("Destination path '%s' already exists" % real_dst) try: @@ -797,6 +925,12 @@ def move(src, dst, copy_function=copy2): if _destinsrc(src, dst): raise Error("Cannot move a directory '%s' into itself" " '%s'." % (src, dst)) + if (_is_immutable(src) + or (not os.access(src, os.W_OK) and os.listdir(src) + and sys.platform == 'darwin')): + raise PermissionError("Cannot move the non-empty directory " + "'%s': Lacking write permission to '%s'." + % (src, src)) copytree(src, real_dst, copy_function=copy_function, symlinks=True) rmtree(src) @@ -814,10 +948,21 @@ def _destinsrc(src, dst): dst += os.path.sep return dst.startswith(src) +def _is_immutable(src): + st = _stat(src) + immutable_states = [stat.UF_IMMUTABLE, stat.SF_IMMUTABLE] + return hasattr(st, 'st_flags') and st.st_flags in immutable_states + def _get_gid(name): """Returns a gid, given a group name.""" - if getgrnam is None or name is None: + if name is None: return None + + try: + from grp import getgrnam + except ImportError: + return None + try: result = getgrnam(name) except KeyError: @@ -828,8 +973,14 @@ def _get_gid(name): def _get_uid(name): """Returns an uid, given a user name.""" - if getpwnam is None or name is None: + if name is None: + return None + + try: + from pwd import getpwnam + except ImportError: return None + try: result = getpwnam(name) except KeyError: @@ -839,18 +990,18 @@ def _get_uid(name): return None def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, - owner=None, group=None, logger=None): + owner=None, group=None, logger=None, root_dir=None): """Create a (possibly compressed) tar file from all the files under 'base_dir'. - 'compress' must be "gzip" (the default), "bzip2", "xz", or None. + 'compress' must be "gzip" (the default), "bzip2", "xz", "zst", or None. 'owner' and 'group' can be used to define an owner and a group for the archive that is being built. If not provided, the current owner and group will be used. The output tar file will be named 'base_name' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", or ".xz"). + the appropriate compression extension (".gz", ".bz2", ".xz", or ".zst"). Returns the output filename. """ @@ -862,6 +1013,8 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, tar_compression = 'bz2' elif _LZMA_SUPPORTED and compress == 'xz': tar_compression = 'xz' + elif _ZSTD_SUPPORTED and compress == 'zst': + tar_compression = 'zst' else: raise ValueError("bad value for 'compress', or compression format not " "supported : {0}".format(compress)) @@ -896,14 +1049,20 @@ def _set_uid_gid(tarinfo): if not dry_run: tar = tarfile.open(archive_name, 'w|%s' % tar_compression) + arcname = base_dir + if root_dir is not None: + base_dir = os.path.join(root_dir, base_dir) try: - tar.add(base_dir, filter=_set_uid_gid) + tar.add(base_dir, arcname, filter=_set_uid_gid) finally: tar.close() + if root_dir is not None: + archive_name = os.path.abspath(archive_name) return archive_name -def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): +def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, + logger=None, owner=None, group=None, root_dir=None): """Create a zip file from all the files under 'base_dir'. The output zip file will be named 'base_name' + ".zip". Returns the @@ -927,28 +1086,48 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): if not dry_run: with zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_DEFLATED) as zf: - path = os.path.normpath(base_dir) - if path != os.curdir: - zf.write(path, path) + arcname = os.path.normpath(base_dir) + if root_dir is not None: + base_dir = os.path.join(root_dir, base_dir) + base_dir = os.path.normpath(base_dir) + if arcname != os.curdir: + zf.write(base_dir, arcname) if logger is not None: - logger.info("adding '%s'", path) + logger.info("adding '%s'", base_dir) for dirpath, dirnames, filenames in os.walk(base_dir): + arcdirpath = dirpath + if root_dir is not None: + arcdirpath = os.path.relpath(arcdirpath, root_dir) + arcdirpath = os.path.normpath(arcdirpath) for name in sorted(dirnames): - path = os.path.normpath(os.path.join(dirpath, name)) - zf.write(path, path) + path = os.path.join(dirpath, name) + arcname = os.path.join(arcdirpath, name) + zf.write(path, arcname) if logger is not None: logger.info("adding '%s'", path) for name in filenames: - path = os.path.normpath(os.path.join(dirpath, name)) + path = os.path.join(dirpath, name) + path = os.path.normpath(path) if os.path.isfile(path): - zf.write(path, path) + arcname = os.path.join(arcdirpath, name) + zf.write(path, arcname) if logger is not None: logger.info("adding '%s'", path) + if root_dir is not None: + zip_filename = os.path.abspath(zip_filename) return zip_filename +_make_tarball.supports_root_dir = True +_make_zipfile.supports_root_dir = True + +# Maps the name of the archive format to a tuple containing: +# * the archiving function +# * extra keyword arguments +# * description _ARCHIVE_FORMATS = { - 'tar': (_make_tarball, [('compress', None)], "uncompressed tar file"), + 'tar': (_make_tarball, [('compress', None)], + "uncompressed tar file"), } if _ZLIB_SUPPORTED: @@ -964,6 +1143,10 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): _ARCHIVE_FORMATS['xztar'] = (_make_tarball, [('compress', 'xz')], "xz'ed tar-file") +if _ZSTD_SUPPORTED: + _ARCHIVE_FORMATS['zstdtar'] = (_make_tarball, [('compress', 'zst')], + "zstd'ed tar-file") + def get_archive_formats(): """Returns a list of supported formats for archiving and unarchiving. @@ -1004,7 +1187,7 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, 'base_name' is the name of the file to create, minus any format-specific extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", or "xztar". Or any other registered format. + "bztar", "xztar", or "zstdtar". Or any other registered format. 'root_dir' is a directory that will be the root directory of the archive; ie. we typically chdir into 'root_dir' before creating the @@ -1017,36 +1200,44 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, uses the current owner and group. """ sys.audit("shutil.make_archive", base_name, format, root_dir, base_dir) - save_cwd = os.getcwd() - if root_dir is not None: - if logger is not None: - logger.debug("changing into '%s'", root_dir) - base_name = os.path.abspath(base_name) - if not dry_run: - os.chdir(root_dir) - - if base_dir is None: - base_dir = os.curdir - - kwargs = {'dry_run': dry_run, 'logger': logger} - try: format_info = _ARCHIVE_FORMATS[format] except KeyError: raise ValueError("unknown archive format '%s'" % format) from None + kwargs = {'dry_run': dry_run, 'logger': logger, + 'owner': owner, 'group': group} + func = format_info[0] for arg, val in format_info[1]: kwargs[arg] = val - if format != 'zip': - kwargs['owner'] = owner - kwargs['group'] = group + if base_dir is None: + base_dir = os.curdir + + supports_root_dir = getattr(func, 'supports_root_dir', False) + save_cwd = None + if root_dir is not None: + stmd = os.stat(root_dir).st_mode + if not stat.S_ISDIR(stmd): + raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', root_dir) + + if supports_root_dir: + # Support path-like base_name here for backwards-compatibility. + base_name = os.fspath(base_name) + kwargs['root_dir'] = root_dir + else: + save_cwd = os.getcwd() + if logger is not None: + logger.debug("changing into '%s'", root_dir) + base_name = os.path.abspath(base_name) + if not dry_run: + os.chdir(root_dir) try: filename = func(base_name, base_dir, **kwargs) finally: - if root_dir is not None: + if save_cwd is not None: if logger is not None: logger.debug("changing back to '%s'", save_cwd) os.chdir(save_cwd) @@ -1132,25 +1323,21 @@ def _unpack_zipfile(filename, extract_dir): if name.startswith('/') or '..' in name: continue - target = os.path.join(extract_dir, *name.split('/')) - if not target: + targetpath = os.path.join(extract_dir, *name.split('/')) + if not targetpath: continue - _ensure_directory(target) + _ensure_directory(targetpath) if not name.endswith('/'): # file - data = zip.read(info.filename) - f = open(target, 'wb') - try: - f.write(data) - finally: - f.close() - del data + with zip.open(name, 'r') as source, \ + open(targetpath, 'wb') as target: + copyfileobj(source, target) finally: zip.close() -def _unpack_tarfile(filename, extract_dir): - """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` +def _unpack_tarfile(filename, extract_dir, *, filter=None): + """Unpack tar/tar.gz/tar.bz2/tar.xz/tar.zst `filename` to `extract_dir` """ import tarfile # late import for breaking circular dependency try: @@ -1159,10 +1346,15 @@ def _unpack_tarfile(filename, extract_dir): raise ReadError( "%s is not a compressed or uncompressed tar file" % filename) try: - tarobj.extractall(extract_dir) + tarobj.extractall(extract_dir, filter=filter) finally: tarobj.close() +# Maps the name of the unpack format to a tuple containing: +# * extensions +# * the unpacking function +# * extra keyword arguments +# * description _UNPACK_FORMATS = { 'tar': (['.tar'], _unpack_tarfile, [], "uncompressed tar file"), 'zip': (['.zip'], _unpack_zipfile, [], "ZIP file"), @@ -1180,6 +1372,10 @@ def _unpack_tarfile(filename, extract_dir): _UNPACK_FORMATS['xztar'] = (['.tar.xz', '.txz'], _unpack_tarfile, [], "xz'ed tar-file") +if _ZSTD_SUPPORTED: + _UNPACK_FORMATS['zstdtar'] = (['.tar.zst', '.tzst'], _unpack_tarfile, [], + "zstd'ed tar-file") + def _find_unpack_format(filename): for name, info in _UNPACK_FORMATS.items(): for extension in info[0]: @@ -1187,7 +1383,7 @@ def _find_unpack_format(filename): return name return None -def unpack_archive(filename, extract_dir=None, format=None): +def unpack_archive(filename, extract_dir=None, format=None, *, filter=None): """Unpack an archive. `filename` is the name of the archive. @@ -1196,11 +1392,14 @@ def unpack_archive(filename, extract_dir=None, format=None): is unpacked. If not provided, the current working directory is used. `format` is the archive format: one of "zip", "tar", "gztar", "bztar", - or "xztar". Or any other registered format. If not provided, + "xztar", or "zstdtar". Or any other registered format. If not provided, unpack_archive will use the filename extension and see if an unpacker was registered for that extension. In case none is found, a ValueError is raised. + + If `filter` is given, it is passed to the underlying + extraction function. """ sys.audit("shutil.unpack_archive", filename, extract_dir, format) @@ -1210,6 +1409,10 @@ def unpack_archive(filename, extract_dir=None, format=None): extract_dir = os.fspath(extract_dir) filename = os.fspath(filename) + if filter is None: + filter_kwargs = {} + else: + filter_kwargs = {'filter': filter} if format is not None: try: format_info = _UNPACK_FORMATS[format] @@ -1217,7 +1420,7 @@ def unpack_archive(filename, extract_dir=None, format=None): raise ValueError("Unknown unpack format '{0}'".format(format)) from None func = format_info[1] - func(filename, extract_dir, **dict(format_info[2])) + func(filename, extract_dir, **dict(format_info[2]), **filter_kwargs) else: # we need to look at the registered unpackers supported extensions format = _find_unpack_format(filename) @@ -1225,7 +1428,7 @@ def unpack_archive(filename, extract_dir=None, format=None): raise ReadError("Unknown archive format '{0}'".format(filename)) func = _UNPACK_FORMATS[format][1] - kwargs = dict(_UNPACK_FORMATS[format][2]) + kwargs = dict(_UNPACK_FORMATS[format][2]) | filter_kwargs func(filename, extract_dir, **kwargs) @@ -1265,11 +1468,18 @@ def disk_usage(path): return _ntuple_diskusage(total, used, free) -def chown(path, user=None, group=None): +def chown(path, user=None, group=None, *, dir_fd=None, follow_symlinks=True): """Change owner user and group of the given path. user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. + + If dir_fd is set, it should be an open file descriptor to the directory to + be used as the root of *path* if it is relative. + + If follow_symlinks is set to False and the last element of the path is a + symbolic link, chown will modify the link itself and not the file being + referenced by the link. """ sys.audit('shutil.chown', path, user, group) @@ -1295,7 +1505,8 @@ def chown(path, user=None, group=None): if _group is None: raise LookupError("no such group: {!r}".format(group)) - os.chown(path, _user, _group) + os.chown(path, _user, _group, dir_fd=dir_fd, + follow_symlinks=follow_symlinks) def get_terminal_size(fallback=(80, 24)): """Get the size of the terminal window. @@ -1336,9 +1547,9 @@ def get_terminal_size(fallback=(80, 24)): # os.get_terminal_size() is unsupported size = os.terminal_size(fallback) if columns <= 0: - columns = size.columns + columns = size.columns or fallback[0] if lines <= 0: - lines = size.lines + lines = size.lines or fallback[1] return os.terminal_size((columns, lines)) @@ -1351,6 +1562,16 @@ def _access_check(fn, mode): and not os.path.isdir(fn)) +def _win_path_needs_curdir(cmd, mode): + """ + On Windows, we can use NeedCurrentDirectoryForExePath to figure out + if we should add the cwd to PATH when searching for executables if + the mode is executable. + """ + return (not (mode & os.X_OK)) or _winapi.NeedCurrentDirectoryForExePath( + os.fsdecode(cmd)) + + def which(cmd, mode=os.F_OK | os.X_OK, path=None): """Given a command, mode, and a PATH string, return the path which conforms to the given mode on the PATH, or None if there is no such @@ -1361,58 +1582,62 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): path. """ - # If we're given a path with a directory part, look it up directly rather - # than referring to PATH directories. This includes checking relative to the - # current directory, e.g. ./script - if os.path.dirname(cmd): - if _access_check(cmd, mode): - return cmd - return None - use_bytes = isinstance(cmd, bytes) - if path is None: - path = os.environ.get("PATH", None) - if path is None: - try: - path = os.confstr("CS_PATH") - except (AttributeError, ValueError): - # os.confstr() or CS_PATH is not available - path = os.defpath - # bpo-35755: Don't use os.defpath if the PATH environment variable is - # set to an empty string - - # PATH='' doesn't match, whereas PATH=':' looks in the current directory - if not path: - return None - - if use_bytes: - path = os.fsencode(path) - path = path.split(os.fsencode(os.pathsep)) + # If we're given a path with a directory part, look it up directly rather + # than referring to PATH directories. This includes checking relative to + # the current directory, e.g. ./script + dirname, cmd = os.path.split(cmd) + if dirname: + path = [dirname] else: - path = os.fsdecode(path) - path = path.split(os.pathsep) + if path is None: + path = os.environ.get("PATH", None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable + # is set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current + # directory + if not path: + return None - if sys.platform == "win32": - # The current directory takes precedence on Windows. - curdir = os.curdir if use_bytes: - curdir = os.fsencode(curdir) - if curdir not in path: + path = os.fsencode(path) + path = path.split(os.fsencode(os.pathsep)) + else: + path = os.fsdecode(path) + path = path.split(os.pathsep) + + if sys.platform == "win32" and _win_path_needs_curdir(cmd, mode): + curdir = os.curdir + if use_bytes: + curdir = os.fsencode(curdir) path.insert(0, curdir) + if sys.platform == "win32": # PATHEXT is necessary to check on Windows. - pathext = os.environ.get("PATHEXT", "").split(os.pathsep) + pathext_source = os.getenv("PATHEXT") or _WIN_DEFAULT_PATHEXT + pathext = pathext_source.split(os.pathsep) + pathext = [ext.rstrip('.') for ext in pathext if ext] + if use_bytes: pathext = [os.fsencode(ext) for ext in pathext] - # See if the given file matches any of the expected path extensions. - # This will allow us to short circuit when given "python.exe". - # If it does match, only test that one, otherwise we have to try - # others. - if any(cmd.lower().endswith(ext.lower()) for ext in pathext): - files = [cmd] - else: - files = [cmd + ext for ext in pathext] + + files = [cmd + ext for ext in pathext] + + # If X_OK in mode, simulate the cmd.exe behavior: look at direct + # match if and only if the extension is in PATHEXT. + # If X_OK not in mode, simulate the first result of where.exe: + # always look at direct match before a PATHEXT match. + normcmd = cmd.upper() + if not (mode & os.X_OK) or any(normcmd.endswith(ext.upper()) for ext in pathext): + files.insert(0, cmd) else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. @@ -1421,10 +1646,22 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): seen = set() for dir in path: normdir = os.path.normcase(dir) - if not normdir in seen: + if normdir not in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) if _access_check(name, mode): return name return None + +def __getattr__(name): + if name == "ExecError": + import warnings + warnings._deprecated( + "shutil.ExecError", + f"{warnings._DEPRECATED_MSG}; it " + "isn't raised by any shutil function.", + remove=(3, 16) + ) + return RuntimeError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/signal.py b/Lib/signal.py index 50b215b29d2..c8cd3d4f597 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -22,9 +22,11 @@ def _int_to_enum(value, enum_klass): - """Convert a numeric value to an IntEnum member. - If it's not a known member, return the numeric value itself. + """Convert a possible numeric value to an IntEnum member. + If it's not a known member, return the value itself. """ + if not isinstance(value, int): + return value try: return enum_klass(value) except ValueError: diff --git a/Lib/site.py b/Lib/site.py index 6d5dca641ab..5305d67b3b8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -73,7 +73,9 @@ import os import builtins import _sitebuiltins -import io +import _io as io +import stat +import errno # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -93,6 +95,12 @@ def _trace(message): print(message, file=sys.stderr) +def _warn(*args, **kwargs): + import warnings + + warnings.warn(*args, **kwargs) + + def makepath(*paths): dir = os.path.join(*paths) try: @@ -168,37 +176,56 @@ def addpackage(sitedir, name, known_paths): else: reset = False fullname = os.path.join(sitedir, name) + try: + st = os.lstat(fullname) + except OSError: + return + if ((getattr(st, 'st_flags', 0) & stat.UF_HIDDEN) or + (getattr(st, 'st_file_attributes', 0) & stat.FILE_ATTRIBUTE_HIDDEN)): + _trace(f"Skipping hidden .pth file: {fullname!r}") + return _trace(f"Processing .pth file: {fullname!r}") try: - # locale encoding is not ideal especially on Windows. But we have used - # it for a long time. setuptools uses the locale encoding too. - f = io.TextIOWrapper(io.open_code(fullname), encoding="locale") + with io.open_code(fullname) as f: + pth_content = f.read() except OSError: return - with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": + + try: + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") + except UnicodeDecodeError: + # Fallback to locale encoding for backward compatibility. + # We will deprecate this fallback in the future. + import locale + pth_content = pth_content.decode(locale.getencoding()) + _trace(f"Cannot read {fullname!r} as UTF-8. " + f"Using fallback encoding {locale.getencoding()!r}") + + for n, line in enumerate(pth_content.splitlines(), 1): + if line.startswith("#"): + continue + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) continue - try: - if line.startswith(("import ", "import\t")): - exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(*sys.exc_info()): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if dircase not in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print(f"Error processing line {n:d} of {fullname}:\n", + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths @@ -221,7 +248,8 @@ def addsitedir(sitedir, known_paths=None): names = os.listdir(sitedir) except OSError: return - names = [name for name in names if name.endswith(".pth")] + names = [name for name in names + if name.endswith(".pth") and not name.startswith(".")] for name in sorted(names): addpackage(sitedir, name, known_paths) if reset: @@ -260,14 +288,18 @@ def check_enableusersite(): # # See https://bugs.python.org/issue29585 +# Copy of sysconfig._get_implementation() +def _get_implementation(): + return 'RustPython' # XXX: RustPython; for site-packages + # Copy of sysconfig._getuserbase() def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -275,8 +307,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - # XXX: RUSTPYTHON; please keep this change for site-packages - return joinuser(base, "RustPython") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -288,15 +319,22 @@ def joinuser(*args): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' + implementation = _get_implementation() + implementation_lower = implementation.lower() if os.name == 'nt': ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\RustPython{ver_nodot}\\site-packages' + return f'{userbase}\\{implementation}{ver_nodot}\\site-packages' if sys.platform == 'darwin' and sys._framework: - return f'{userbase}/lib/rustpython/site-packages' + return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/rustpython{version[0]}.{version[1]}/site-packages' + # XXX: RUSTPYTHON + return f'{userbase}/lib/rustpython{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -362,6 +400,12 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) + implementation = _get_implementation().lower() + ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -369,8 +413,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - # XXX: RUSTPYTHON; please keep this change for site-packages - "rustpython%d.%d" % sys.version_info[:2], + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: @@ -406,14 +449,10 @@ def setquit(): def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) - if sys.platform[:4] == 'java': - builtins.credits = _sitebuiltins._Printer( - "credits", - "Jython is maintained by the Jython developers (www.jython.org).") - else: - builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands - for supporting Python development. See www.python.org for more information.""") + builtins.credits = _sitebuiltins._Printer("credits", """\ +Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software +Foundation, and a cast of thousands for supporting Python +development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -432,27 +471,76 @@ def setcopyright(): def sethelper(): builtins.help = _sitebuiltins._Helper() + +def gethistoryfile(): + """Check if the PYTHON_HISTORY environment variable is set and define + it as the .python_history file. If PYTHON_HISTORY is not set, use the + default .python_history file. + """ + if not sys.flags.ignore_environment: + history = os.environ.get("PYTHON_HISTORY") + if history: + return history + return os.path.join(os.path.expanduser('~'), + '.python_history') + + def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. + """ + sys.__interactivehook__ = register_readline + + +def register_readline(): + """Configure readline completion on interactive prompts. If the readline module can be imported, the hook will set the Tab key as completion key and register ~/.python_history as history file. This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ - def register_readline(): - import atexit + if not sys.flags.ignore_environment: + PYTHON_BASIC_REPL = os.getenv("PYTHON_BASIC_REPL") + else: + PYTHON_BASIC_REPL = False + + import atexit + + try: try: import readline - import rlcompleter except ImportError: - return + readline = None + else: + import rlcompleter # noqa: F401 + except ImportError: + return + try: + if PYTHON_BASIC_REPL: + CAN_USE_PYREPL = False + else: + original_path = sys.path + sys.path = [p for p in original_path if p != ''] + try: + import _pyrepl.readline + if os.name == "nt": + import _pyrepl.windows_console + console_errors = (_pyrepl.windows_console._error,) + else: + import _pyrepl.unix_console + console_errors = _pyrepl.unix_console._error + from _pyrepl.main import CAN_USE_PYREPL + finally: + sys.path = original_path + except ImportError: + return + + if readline is not None: # Reading the initialization (config) file may not be enough to set a # completion key, so we set one first and then read the file. - readline_doc = getattr(readline, '__doc__', '') - if readline_doc is not None and 'libedit' in readline_doc: + if readline.backend == 'editline': readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') @@ -466,30 +554,44 @@ def register_readline(): # want to ignore the exception. pass - if readline.get_current_history_length() == 0: - # If no history was loaded, default to .python_history. - # The guard is necessary to avoid doubling history size at - # each interpreter exit when readline was already configured - # through a PYTHONSTARTUP hook, see: - # http://bugs.python.org/issue5845#msg198636 - history = os.path.join(os.path.expanduser('~'), - '.python_history') + if readline is None or readline.get_current_history_length() == 0: + # If no history was loaded, default to .python_history, + # or PYTHON_HISTORY. + # The guard is necessary to avoid doubling history size at + # each interpreter exit when readline was already configured + # through a PYTHONSTARTUP hook, see: + # http://bugs.python.org/issue5845#msg198636 + history = gethistoryfile() + + if CAN_USE_PYREPL: + readline_module = _pyrepl.readline + exceptions = (OSError, *console_errors) + else: + if readline is None: + return + readline_module = readline + exceptions = OSError + + try: + readline_module.read_history_file(history) + except exceptions: + pass + + def write_history(): try: - readline.read_history_file(history) - except OSError: + readline_module.write_history_file(history) + except FileNotFoundError, PermissionError: + # home directory does not exist or is not writable + # https://bugs.python.org/issue19891 pass + except OSError: + if errno.EROFS: + pass # gh-128066: read-only file system + else: + raise - def write_history(): - try: - readline.write_history_file(history) - except OSError: - # bpo-19891, bpo-41193: Home directory does not exist - # or is not writable, or the filesystem is read-only. - pass - - atexit.register(write_history) + atexit.register(write_history) - sys.__interactivehook__ = register_readline def venv(known_paths): global PREFIXES, ENABLE_USER_SITE @@ -499,20 +601,23 @@ def venv(known_paths): executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__'] else: executable = sys.executable - exe_dir, _ = os.path.split(os.path.abspath(executable)) + exe_dir = os.path.dirname(os.path.abspath(executable)) site_prefix = os.path.dirname(exe_dir) sys._home = None conf_basename = 'pyvenv.cfg' - candidate_confs = [ - conffile for conffile in ( - os.path.join(exe_dir, conf_basename), - os.path.join(site_prefix, conf_basename) + candidate_conf = next( + ( + conffile for conffile in ( + os.path.join(exe_dir, conf_basename), + os.path.join(site_prefix, conf_basename) ) - if os.path.isfile(conffile) - ] + if os.path.isfile(conffile) + ), + None + ) - if candidate_confs: - virtual_conf = candidate_confs[0] + if candidate_conf: + virtual_conf = candidate_conf system_site = "true" # Issue 25185: Use UTF-8, as that's what the venv module uses when # writing the file. @@ -527,17 +632,17 @@ def venv(known_paths): elif key == 'home': sys._home = value - sys.prefix = sys.exec_prefix = site_prefix + if sys.prefix != site_prefix: + _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + if sys.exec_prefix != site_prefix: + _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) - # addsitepackages will process site_prefix again if its in PREFIXES, - # but that's ok; known_paths will prevent anything being added twice if system_site == "true": - PREFIXES.insert(0, sys.prefix) + PREFIXES += [sys.base_prefix, sys.base_exec_prefix] else: - PREFIXES = [sys.prefix] ENABLE_USER_SITE = False return known_paths @@ -547,7 +652,7 @@ def execsitecustomize(): """Run custom site specific code, if available.""" try: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError as exc: if exc.name == 'sitecustomize': pass @@ -567,7 +672,7 @@ def execusercustomize(): """Run custom user specific code, if available.""" try: try: - import usercustomize + import usercustomize # noqa: F401 except ImportError as exc: if exc.name == 'usercustomize': pass diff --git a/Lib/smtplib.py b/Lib/smtplib.py new file mode 100644 index 00000000000..72093f7f8b0 --- /dev/null +++ b/Lib/smtplib.py @@ -0,0 +1,1121 @@ +'''SMTP/ESMTP client class. + +This should follow RFC 821 (SMTP), RFC 1869 (ESMTP), RFC 2554 (SMTP +Authentication) and RFC 2487 (Secure SMTP over TLS). + +Notes: + +Please remember, when doing ESMTP, that the names of the SMTP service +extensions are NOT the same thing as the option keywords for the RCPT +and MAIL commands! + +Example: + + >>> import smtplib + >>> s=smtplib.SMTP("localhost") + >>> print(s.help()) + This is Sendmail version 8.8.4 + Topics: + HELO EHLO MAIL RCPT DATA + RSET NOOP QUIT HELP VRFY + EXPN VERB ETRN DSN + For more info use "HELP ". + To report bugs in the implementation send email to + sendmail-bugs@sendmail.org. + For local information send email to Postmaster at your site. + End of HELP info + >>> s.putcmd("vrfy","someone@here") + >>> s.getreply() + (250, "Somebody OverHere ") + >>> s.quit() +''' + +# Author: The Dragon De Monsyne +# ESMTP support, test code and doc fixes added by +# Eric S. Raymond +# Better RFC 821 compliance (MAIL and RCPT, and CRLF in data) +# by Carey Evans , for picky mail servers. +# RFC 2554 (authentication) support by Gerhard Haering . +# +# This was modified from the Python 1.5 library HTTP lib. + +import socket +import io +import re +import email.utils +import email.message +import email.generator +import base64 +import hmac +import copy +import datetime +import sys +from email.base64mime import body_encode as encode_base64 + +__all__ = ["SMTPException", "SMTPNotSupportedError", "SMTPServerDisconnected", "SMTPResponseException", + "SMTPSenderRefused", "SMTPRecipientsRefused", "SMTPDataError", + "SMTPConnectError", "SMTPHeloError", "SMTPAuthenticationError", + "quoteaddr", "quotedata", "SMTP"] + +SMTP_PORT = 25 +SMTP_SSL_PORT = 465 +CRLF = "\r\n" +bCRLF = b"\r\n" +_MAXLINE = 8192 # more than 8 times larger than RFC 821, 4.5.3 +_MAXCHALLENGE = 5 # Maximum number of AUTH challenges sent + +OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I) + +# Exception classes used by this module. +class SMTPException(OSError): + """Base class for all exceptions raised by this module.""" + +class SMTPNotSupportedError(SMTPException): + """The command or option is not supported by the SMTP server. + + This exception is raised when an attempt is made to run a command or a + command with an option which is not supported by the server. + """ + +class SMTPServerDisconnected(SMTPException): + """Not connected to any SMTP server. + + This exception is raised when the server unexpectedly disconnects, + or when an attempt is made to use the SMTP instance before + connecting it to a server. + """ + +class SMTPResponseException(SMTPException): + """Base class for all exceptions that include an SMTP error code. + + These exceptions are generated in some instances when the SMTP + server returns an error code. The error code is stored in the + `smtp_code' attribute of the error, and the `smtp_error' attribute + is set to the error message. + """ + + def __init__(self, code, msg): + self.smtp_code = code + self.smtp_error = msg + self.args = (code, msg) + +class SMTPSenderRefused(SMTPResponseException): + """Sender address refused. + + In addition to the attributes set by on all SMTPResponseException + exceptions, this sets 'sender' to the string that the SMTP refused. + """ + + def __init__(self, code, msg, sender): + self.smtp_code = code + self.smtp_error = msg + self.sender = sender + self.args = (code, msg, sender) + +class SMTPRecipientsRefused(SMTPException): + """All recipient addresses refused. + + The errors for each recipient are accessible through the attribute + 'recipients', which is a dictionary of exactly the same sort as + SMTP.sendmail() returns. + """ + + def __init__(self, recipients): + self.recipients = recipients + self.args = (recipients,) + + +class SMTPDataError(SMTPResponseException): + """The SMTP server didn't accept the data.""" + +class SMTPConnectError(SMTPResponseException): + """Error during connection establishment.""" + +class SMTPHeloError(SMTPResponseException): + """The server refused our HELO reply.""" + +class SMTPAuthenticationError(SMTPResponseException): + """Authentication error. + + Most probably the server didn't accept the username/password + combination provided. + """ + +def quoteaddr(addrstring): + """Quote a subset of the email addresses defined by RFC 821. + + Should be able to handle anything email.utils.parseaddr can handle. + """ + displayname, addr = email.utils.parseaddr(addrstring) + if (displayname, addr) == ('', ''): + # parseaddr couldn't parse it, use it as is and hope for the best. + if addrstring.strip().startswith('<'): + return addrstring + return "<%s>" % addrstring + return "<%s>" % addr + +def _addr_only(addrstring): + displayname, addr = email.utils.parseaddr(addrstring) + if (displayname, addr) == ('', ''): + # parseaddr couldn't parse it, so use it as is. + return addrstring + return addr + +# Legacy method kept for backward compatibility. +def quotedata(data): + """Quote data for email. + + Double leading '.', and change Unix newline '\\n', or Mac '\\r' into + internet CRLF end-of-line. + """ + return re.sub(r'(?m)^\.', '..', + re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data)) + +def _quote_periods(bindata): + return re.sub(br'(?m)^\.', b'..', bindata) + +def _fix_eols(data): + return re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data) + + +try: + hmac.digest(b'', b'', 'md5') +except ValueError: + _have_cram_md5_support = False +else: + _have_cram_md5_support = True + + +try: + import ssl +except ImportError: + _have_ssl = False +else: + _have_ssl = True + + +class SMTP: + """This class manages a connection to an SMTP or ESMTP server. + SMTP Objects: + SMTP objects have the following attributes: + helo_resp + This is the message given by the server in response to the + most recent HELO command. + + ehlo_resp + This is the message given by the server in response to the + most recent EHLO command. This is usually multiline. + + does_esmtp + This is a True value _after you do an EHLO command_, if the + server supports ESMTP. + + esmtp_features + This is a dictionary, which, if the server supports ESMTP, + will _after you do an EHLO command_, contain the names of the + SMTP service extensions this server supports, and their + parameters (if any). + + Note, all extension names are mapped to lower case in the + dictionary. + + See each method's docstrings for details. In general, there is a + method of the same name to perform each SMTP command. There is also a + method called 'sendmail' that will do an entire mail transaction. + """ + debuglevel = 0 + + sock = None + file = None + helo_resp = None + ehlo_msg = "ehlo" + ehlo_resp = None + does_esmtp = False + default_port = SMTP_PORT + + def __init__(self, host='', port=0, local_hostname=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None): + """Initialize a new instance. + + If specified, `host` is the name of the remote host to which to + connect. If specified, `port` specifies the port to which to connect. + By default, smtplib.SMTP_PORT is used. If a host is specified the + connect method is called, and if it returns anything other than a + success code an SMTPConnectError is raised. If specified, + `local_hostname` is used as the FQDN of the local host in the HELO/EHLO + command. Otherwise, the local hostname is found using + socket.getfqdn(). The `source_address` parameter takes a 2-tuple (host, + port) for the socket to bind to as its source address before + connecting. If the host is '' and port is 0, the OS default behavior + will be used. + + """ + self._host = host + self.timeout = timeout + self.esmtp_features = {} + self.command_encoding = 'ascii' + self.source_address = source_address + self._auth_challenge_count = 0 + + if host: + (code, msg) = self.connect(host, port) + if code != 220: + self.close() + raise SMTPConnectError(code, msg) + if local_hostname is not None: + self.local_hostname = local_hostname + else: + # RFC 2821 says we should use the fqdn in the EHLO/HELO verb, and + # if that can't be calculated, that we should use a domain literal + # instead (essentially an encoded IP address like [A.B.C.D]). + fqdn = socket.getfqdn() + if '.' in fqdn: + self.local_hostname = fqdn + else: + # We can't find an fqdn hostname, so use a domain literal + addr = '127.0.0.1' + try: + addr = socket.gethostbyname(socket.gethostname()) + except socket.gaierror: + pass + self.local_hostname = '[%s]' % addr + + def __enter__(self): + return self + + def __exit__(self, *args): + try: + code, message = self.docmd("QUIT") + if code != 221: + raise SMTPResponseException(code, message) + except SMTPServerDisconnected: + pass + finally: + self.close() + + def set_debuglevel(self, debuglevel): + """Set the debug output level. + + A non-false value results in debug messages for connection and for all + messages sent to and received from the server. + + """ + self.debuglevel = debuglevel + + def _print_debug(self, *args): + if self.debuglevel > 1: + print(datetime.datetime.now().time(), *args, file=sys.stderr) + else: + print(*args, file=sys.stderr) + + def _get_socket(self, host, port, timeout): + # This makes it simpler for SMTP_SSL to use the SMTP connect code + # and just alter the socket connection bit. + if timeout is not None and not timeout: + raise ValueError('Non-blocking socket (timeout=0) is not supported') + if self.debuglevel > 0: + self._print_debug('connect: to', (host, port), self.source_address) + return socket.create_connection((host, port), timeout, + self.source_address) + + def connect(self, host='localhost', port=0, source_address=None): + """Connect to a host on a given port. + + If the hostname ends with a colon (':') followed by a number, and + there is no port specified, that suffix will be stripped off and the + number interpreted as the port number to use. + + Note: This method is automatically invoked by __init__, if a host is + specified during instantiation. + + """ + + if source_address: + self.source_address = source_address + + if not port and (host.find(':') == host.rfind(':')): + i = host.rfind(':') + if i >= 0: + host, port = host[:i], host[i + 1:] + try: + port = int(port) + except ValueError: + raise OSError("nonnumeric port") + if not port: + port = self.default_port + sys.audit("smtplib.connect", self, host, port) + self.sock = self._get_socket(host, port, self.timeout) + self.file = None + (code, msg) = self.getreply() + if self.debuglevel > 0: + self._print_debug('connect:', repr(msg)) + return (code, msg) + + def send(self, s): + """Send 's' to the server.""" + if self.debuglevel > 0: + self._print_debug('send:', repr(s)) + if self.sock: + if isinstance(s, str): + # send is used by the 'data' command, where command_encoding + # should not be used, but 'data' needs to convert the string to + # binary itself anyway, so that's not a problem. + s = s.encode(self.command_encoding) + sys.audit("smtplib.send", self, s) + try: + self.sock.sendall(s) + except OSError: + self.close() + raise SMTPServerDisconnected('Server not connected') + else: + raise SMTPServerDisconnected('please run connect() first') + + def putcmd(self, cmd, args=""): + """Send a command to the server.""" + if args == "": + s = cmd + else: + s = f'{cmd} {args}' + if '\r' in s or '\n' in s: + s = s.replace('\n', '\\n').replace('\r', '\\r') + raise ValueError( + f'command and arguments contain prohibited newline characters: {s}' + ) + self.send(f'{s}{CRLF}') + + def getreply(self): + """Get a reply from the server. + + Returns a tuple consisting of: + + - server response code (e.g. '250', or such, if all goes well) + Note: returns -1 if it can't read response code. + + - server response string corresponding to response code (multiline + responses are converted to a single, multiline string). + + Raises SMTPServerDisconnected if end-of-file is reached. + """ + resp = [] + if self.file is None: + self.file = self.sock.makefile('rb') + while 1: + try: + line = self.file.readline(_MAXLINE + 1) + except OSError as e: + self.close() + raise SMTPServerDisconnected("Connection unexpectedly closed: " + + str(e)) + if not line: + self.close() + raise SMTPServerDisconnected("Connection unexpectedly closed") + if self.debuglevel > 0: + self._print_debug('reply:', repr(line)) + if len(line) > _MAXLINE: + self.close() + raise SMTPResponseException(500, "Line too long.") + resp.append(line[4:].strip(b' \t\r\n')) + code = line[:3] + # Check that the error code is syntactically correct. + # Don't attempt to read a continuation line if it is broken. + try: + errcode = int(code) + except ValueError: + errcode = -1 + break + # Check if multiline response. + if line[3:4] != b"-": + break + + errmsg = b"\n".join(resp) + if self.debuglevel > 0: + self._print_debug('reply: retcode (%s); Msg: %a' % (errcode, errmsg)) + return errcode, errmsg + + def docmd(self, cmd, args=""): + """Send a command, and return its response code.""" + self.putcmd(cmd, args) + return self.getreply() + + # std smtp commands + def helo(self, name=''): + """SMTP 'helo' command. + Hostname to send for this command defaults to the FQDN of the local + host. + """ + self.putcmd("helo", name or self.local_hostname) + (code, msg) = self.getreply() + self.helo_resp = msg + return (code, msg) + + def ehlo(self, name=''): + """ SMTP 'ehlo' command. + Hostname to send for this command defaults to the FQDN of the local + host. + """ + self.esmtp_features = {} + self.putcmd(self.ehlo_msg, name or self.local_hostname) + (code, msg) = self.getreply() + # According to RFC1869 some (badly written) + # MTA's will disconnect on an ehlo. Toss an exception if + # that happens -ddm + if code == -1 and len(msg) == 0: + self.close() + raise SMTPServerDisconnected("Server not connected") + self.ehlo_resp = msg + if code != 250: + return (code, msg) + self.does_esmtp = True + #parse the ehlo response -ddm + assert isinstance(self.ehlo_resp, bytes), repr(self.ehlo_resp) + resp = self.ehlo_resp.decode("latin-1").split('\n') + del resp[0] + for each in resp: + # To be able to communicate with as many SMTP servers as possible, + # we have to take the old-style auth advertisement into account, + # because: + # 1) Else our SMTP feature parser gets confused. + # 2) There are some servers that only advertise the auth methods we + # support using the old style. + auth_match = OLDSTYLE_AUTH.match(each) + if auth_match: + # This doesn't remove duplicates, but that's no problem + self.esmtp_features["auth"] = self.esmtp_features.get("auth", "") \ + + " " + auth_match.groups(0)[0] + continue + + # RFC 1869 requires a space between ehlo keyword and parameters. + # It's actually stricter, in that only spaces are allowed between + # parameters, but were not going to check for that here. Note + # that the space isn't present if there are no parameters. + m = re.match(r'(?P[A-Za-z0-9][A-Za-z0-9\-]*) ?', each) + if m: + feature = m.group("feature").lower() + params = m.string[m.end("feature"):].strip() + if feature == "auth": + self.esmtp_features[feature] = self.esmtp_features.get(feature, "") \ + + " " + params + else: + self.esmtp_features[feature] = params + return (code, msg) + + def has_extn(self, opt): + """Does the server support a given SMTP service extension?""" + return opt.lower() in self.esmtp_features + + def help(self, args=''): + """SMTP 'help' command. + Returns help text from server.""" + self.putcmd("help", args) + return self.getreply()[1] + + def rset(self): + """SMTP 'rset' command -- resets session.""" + self.command_encoding = 'ascii' + return self.docmd("rset") + + def _rset(self): + """Internal 'rset' command which ignores any SMTPServerDisconnected error. + + Used internally in the library, since the server disconnected error + should appear to the application when the *next* command is issued, if + we are doing an internal "safety" reset. + """ + try: + self.rset() + except SMTPServerDisconnected: + pass + + def noop(self): + """SMTP 'noop' command -- doesn't do anything :>""" + return self.docmd("noop") + + def mail(self, sender, options=()): + """SMTP 'mail' command -- begins mail xfer session. + + This method may raise the following exceptions: + + SMTPNotSupportedError The options parameter includes 'SMTPUTF8' + but the SMTPUTF8 extension is not supported by + the server. + """ + optionlist = '' + if options and self.does_esmtp: + if any(x.lower()=='smtputf8' for x in options): + if self.has_extn('smtputf8'): + self.command_encoding = 'utf-8' + else: + raise SMTPNotSupportedError( + 'SMTPUTF8 not supported by server') + optionlist = ' ' + ' '.join(options) + self.putcmd("mail", "from:%s%s" % (quoteaddr(sender), optionlist)) + return self.getreply() + + def rcpt(self, recip, options=()): + """SMTP 'rcpt' command -- indicates 1 recipient for this mail.""" + optionlist = '' + if options and self.does_esmtp: + optionlist = ' ' + ' '.join(options) + self.putcmd("rcpt", "to:%s%s" % (quoteaddr(recip), optionlist)) + return self.getreply() + + def data(self, msg): + """SMTP 'DATA' command -- sends message data to server. + + Automatically quotes lines beginning with a period per rfc821. + Raises SMTPDataError if there is an unexpected reply to the + DATA command; the return value from this method is the final + response code received when the all data is sent. If msg + is a string, lone '\\r' and '\\n' characters are converted to + '\\r\\n' characters. If msg is bytes, it is transmitted as is. + """ + self.putcmd("data") + (code, repl) = self.getreply() + if self.debuglevel > 0: + self._print_debug('data:', (code, repl)) + if code != 354: + raise SMTPDataError(code, repl) + else: + if isinstance(msg, str): + msg = _fix_eols(msg).encode('ascii') + q = _quote_periods(msg) + if q[-2:] != bCRLF: + q = q + bCRLF + q = q + b"." + bCRLF + self.send(q) + (code, msg) = self.getreply() + if self.debuglevel > 0: + self._print_debug('data:', (code, msg)) + return (code, msg) + + def verify(self, address): + """SMTP 'verify' command -- checks for address validity.""" + self.putcmd("vrfy", _addr_only(address)) + return self.getreply() + # a.k.a. + vrfy = verify + + def expn(self, address): + """SMTP 'expn' command -- expands a mailing list.""" + self.putcmd("expn", _addr_only(address)) + return self.getreply() + + # some useful methods + + def ehlo_or_helo_if_needed(self): + """Call self.ehlo() and/or self.helo() if needed. + + If there has been no previous EHLO or HELO command this session, this + method tries ESMTP EHLO first. + + This method may raise the following exceptions: + + SMTPHeloError The server didn't reply properly to + the helo greeting. + """ + if self.helo_resp is None and self.ehlo_resp is None: + if not (200 <= self.ehlo()[0] <= 299): + (code, resp) = self.helo() + if not (200 <= code <= 299): + raise SMTPHeloError(code, resp) + + def auth(self, mechanism, authobject, *, initial_response_ok=True): + """Authentication command - requires response processing. + + 'mechanism' specifies which authentication mechanism is to + be used - the valid values are those listed in the 'auth' + element of 'esmtp_features'. + + 'authobject' must be a callable object taking a single argument: + + data = authobject(challenge) + + It will be called to process the server's challenge response; the + challenge argument it is passed will be a bytes. It should return + an ASCII string that will be base64 encoded and sent to the server. + + Keyword arguments: + - initial_response_ok: Allow sending the RFC 4954 initial-response + to the AUTH command, if the authentication methods supports it. + """ + # RFC 4954 allows auth methods to provide an initial response. Not all + # methods support it. By definition, if they return something other + # than None when challenge is None, then they do. See issue #15014. + mechanism = mechanism.upper() + initial_response = (authobject() if initial_response_ok else None) + if initial_response is not None: + response = encode_base64(initial_response.encode('ascii'), eol='') + (code, resp) = self.docmd("AUTH", mechanism + " " + response) + self._auth_challenge_count = 1 + else: + (code, resp) = self.docmd("AUTH", mechanism) + self._auth_challenge_count = 0 + # If server responds with a challenge, send the response. + while code == 334: + self._auth_challenge_count += 1 + challenge = base64.decodebytes(resp) + response = encode_base64( + authobject(challenge).encode('ascii'), eol='') + (code, resp) = self.docmd(response) + # If server keeps sending challenges, something is wrong. + if self._auth_challenge_count > _MAXCHALLENGE: + raise SMTPException( + "Server AUTH mechanism infinite loop. Last response: " + + repr((code, resp)) + ) + if code in (235, 503): + return (code, resp) + raise SMTPAuthenticationError(code, resp) + + def auth_cram_md5(self, challenge=None): + """ Authobject to use with CRAM-MD5 authentication. Requires self.user + and self.password to be set.""" + # CRAM-MD5 does not support initial-response. + if challenge is None: + return None + if not _have_cram_md5_support: + raise SMTPException("CRAM-MD5 is not supported") + password = self.password.encode('ascii') + authcode = hmac.HMAC(password, challenge, 'md5') + return f"{self.user} {authcode.hexdigest()}" + + def auth_plain(self, challenge=None): + """ Authobject to use with PLAIN authentication. Requires self.user and + self.password to be set.""" + return "\0%s\0%s" % (self.user, self.password) + + def auth_login(self, challenge=None): + """ Authobject to use with LOGIN authentication. Requires self.user and + self.password to be set.""" + if challenge is None or self._auth_challenge_count < 2: + return self.user + else: + return self.password + + def login(self, user, password, *, initial_response_ok=True): + """Log in on an SMTP server that requires authentication. + + The arguments are: + - user: The user name to authenticate with. + - password: The password for the authentication. + + Keyword arguments: + - initial_response_ok: Allow sending the RFC 4954 initial-response + to the AUTH command, if the authentication methods supports it. + + If there has been no previous EHLO or HELO command this session, this + method tries ESMTP EHLO first. + + This method will return normally if the authentication was successful. + + This method may raise the following exceptions: + + SMTPHeloError The server didn't reply properly to + the helo greeting. + SMTPAuthenticationError The server didn't accept the username/ + password combination. + SMTPNotSupportedError The AUTH command is not supported by the + server. + SMTPException No suitable authentication method was + found. + """ + + self.ehlo_or_helo_if_needed() + if not self.has_extn("auth"): + raise SMTPNotSupportedError( + "SMTP AUTH extension not supported by server.") + + # Authentication methods the server claims to support + advertised_authlist = self.esmtp_features["auth"].split() + + # Authentication methods we can handle in our preferred order: + if _have_cram_md5_support: + preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] + else: + preferred_auths = ['PLAIN', 'LOGIN'] + # We try the supported authentications in our preferred order, if + # the server supports them. + authlist = [auth for auth in preferred_auths + if auth in advertised_authlist] + if not authlist: + raise SMTPException("No suitable authentication method found.") + + # Some servers advertise authentication methods they don't really + # support, so if authentication fails, we continue until we've tried + # all methods. + self.user, self.password = user, password + for authmethod in authlist: + method_name = 'auth_' + authmethod.lower().replace('-', '_') + try: + (code, resp) = self.auth( + authmethod, getattr(self, method_name), + initial_response_ok=initial_response_ok) + # 235 == 'Authentication successful' + # 503 == 'Error: already authenticated' + if code in (235, 503): + return (code, resp) + except SMTPAuthenticationError as e: + last_exception = e + + # We could not login successfully. Return result of last attempt. + raise last_exception + + def starttls(self, *, context=None): + """Puts the connection to the SMTP server into TLS mode. + + If there has been no previous EHLO or HELO command this session, this + method tries ESMTP EHLO first. + + If the server supports TLS, this will encrypt the rest of the SMTP + session. If you provide the context parameter, + the identity of the SMTP server and client can be checked. This, + however, depends on whether the socket module really checks the + certificates. + + This method may raise the following exceptions: + + SMTPHeloError The server didn't reply properly to + the helo greeting. + """ + self.ehlo_or_helo_if_needed() + if not self.has_extn("starttls"): + raise SMTPNotSupportedError( + "STARTTLS extension not supported by server.") + (resp, reply) = self.docmd("STARTTLS") + if resp == 220: + if not _have_ssl: + raise RuntimeError("No SSL support included in this Python") + if context is None: + context = ssl._create_stdlib_context() + self.sock = context.wrap_socket(self.sock, + server_hostname=self._host) + self.file = None + # RFC 3207: + # The client MUST discard any knowledge obtained from + # the server, such as the list of SMTP service extensions, + # which was not obtained from the TLS negotiation itself. + self.helo_resp = None + self.ehlo_resp = None + self.esmtp_features = {} + self.does_esmtp = False + else: + # RFC 3207: + # 501 Syntax error (no parameters allowed) + # 454 TLS not available due to temporary reason + raise SMTPResponseException(resp, reply) + return (resp, reply) + + def sendmail(self, from_addr, to_addrs, msg, mail_options=(), + rcpt_options=()): + """This command performs an entire mail transaction. + + The arguments are: + - from_addr : The address sending this mail. + - to_addrs : A list of addresses to send this mail to. A bare + string will be treated as a list with 1 address. + - msg : The message to send. + - mail_options : List of ESMTP options (such as 8bitmime) for the + mail command. + - rcpt_options : List of ESMTP options (such as DSN commands) for + all the rcpt commands. + + msg may be a string containing characters in the ASCII range, or a byte + string. A string is encoded to bytes using the ascii codec, and lone + \\r and \\n characters are converted to \\r\\n characters. + + If there has been no previous EHLO or HELO command this session, this + method tries ESMTP EHLO first. If the server does ESMTP, message size + and each of the specified options will be passed to it. If EHLO + fails, HELO will be tried and ESMTP options suppressed. + + This method will return normally if the mail is accepted for at least + one recipient. It returns a dictionary, with one entry for each + recipient that was refused. Each entry contains a tuple of the SMTP + error code and the accompanying error message sent by the server. + + This method may raise the following exceptions: + + SMTPHeloError The server didn't reply properly to + the helo greeting. + SMTPRecipientsRefused The server rejected ALL recipients + (no mail was sent). + SMTPSenderRefused The server didn't accept the from_addr. + SMTPDataError The server replied with an unexpected + error code (other than a refusal of + a recipient). + SMTPNotSupportedError The mail_options parameter includes 'SMTPUTF8' + but the SMTPUTF8 extension is not supported by + the server. + + Note: the connection will be open even after an exception is raised. + + Example: + + >>> import smtplib + >>> s=smtplib.SMTP("localhost") + >>> tolist=["one@one.org","two@two.org","three@three.org","four@four.org"] + >>> msg = '''\\ + ... From: Me@my.org + ... Subject: testin'... + ... + ... This is a test ''' + >>> s.sendmail("me@my.org",tolist,msg) + { "three@three.org" : ( 550 ,"User unknown" ) } + >>> s.quit() + + In the above example, the message was accepted for delivery to three + of the four addresses, and one was rejected, with the error code + 550. If all addresses are accepted, then the method will return an + empty dictionary. + + """ + self.ehlo_or_helo_if_needed() + esmtp_opts = [] + if isinstance(msg, str): + msg = _fix_eols(msg).encode('ascii') + if self.does_esmtp: + if self.has_extn('size'): + esmtp_opts.append("size=%d" % len(msg)) + for option in mail_options: + esmtp_opts.append(option) + (code, resp) = self.mail(from_addr, esmtp_opts) + if code != 250: + if code == 421: + self.close() + else: + self._rset() + raise SMTPSenderRefused(code, resp, from_addr) + senderrs = {} + if isinstance(to_addrs, str): + to_addrs = [to_addrs] + for each in to_addrs: + (code, resp) = self.rcpt(each, rcpt_options) + if (code != 250) and (code != 251): + senderrs[each] = (code, resp) + if code == 421: + self.close() + raise SMTPRecipientsRefused(senderrs) + if len(senderrs) == len(to_addrs): + # the server refused all our recipients + self._rset() + raise SMTPRecipientsRefused(senderrs) + (code, resp) = self.data(msg) + if code != 250: + if code == 421: + self.close() + else: + self._rset() + raise SMTPDataError(code, resp) + #if we got here then somebody got our mail + return senderrs + + def send_message(self, msg, from_addr=None, to_addrs=None, + mail_options=(), rcpt_options=()): + """Converts message to a bytestring and passes it to sendmail. + + The arguments are as for sendmail, except that msg is an + email.message.Message object. If from_addr is None or to_addrs is + None, these arguments are taken from the headers of the Message as + described in RFC 5322 (a ValueError is raised if there is more than + one set of 'Resent-' headers). Regardless of the values of from_addr and + to_addr, any Bcc field (or Resent-Bcc field, when the Message is a + resent) of the Message object won't be transmitted. The Message + object is then serialized using email.generator.BytesGenerator and + sendmail is called to transmit the message. If the sender or any of + the recipient addresses contain non-ASCII and the server advertises the + SMTPUTF8 capability, the policy is cloned with utf8 set to True for the + serialization, and SMTPUTF8 and BODY=8BITMIME are asserted on the send. + If the server does not support SMTPUTF8, an SMTPNotSupported error is + raised. Otherwise the generator is called without modifying the + policy. + + """ + # 'Resent-Date' is a mandatory field if the Message is resent (RFC 5322 + # Section 3.6.6). In such a case, we use the 'Resent-*' fields. However, + # if there is more than one 'Resent-' block there's no way to + # unambiguously determine which one is the most recent in all cases, + # so rather than guess we raise a ValueError in that case. + # + # TODO implement heuristics to guess the correct Resent-* block with an + # option allowing the user to enable the heuristics. (It should be + # possible to guess correctly almost all of the time.) + + self.ehlo_or_helo_if_needed() + resent = msg.get_all('Resent-Date') + if resent is None: + header_prefix = '' + elif len(resent) == 1: + header_prefix = 'Resent-' + else: + raise ValueError("message has more than one 'Resent-' header block") + if from_addr is None: + # Prefer the sender field per RFC 5322 section 3.6.2. + from_addr = (msg[header_prefix + 'Sender'] + if (header_prefix + 'Sender') in msg + else msg[header_prefix + 'From']) + from_addr = email.utils.getaddresses([from_addr])[0][1] + if to_addrs is None: + addr_fields = [f for f in (msg[header_prefix + 'To'], + msg[header_prefix + 'Bcc'], + msg[header_prefix + 'Cc']) + if f is not None] + to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)] + # Make a local copy so we can delete the bcc headers. + msg_copy = copy.copy(msg) + del msg_copy['Bcc'] + del msg_copy['Resent-Bcc'] + international = False + try: + ''.join([from_addr, *to_addrs]).encode('ascii') + except UnicodeEncodeError: + if not self.has_extn('smtputf8'): + raise SMTPNotSupportedError( + "One or more source or delivery addresses require" + " internationalized email support, but the server" + " does not advertise the required SMTPUTF8 capability") + international = True + with io.BytesIO() as bytesmsg: + if international: + g = email.generator.BytesGenerator( + bytesmsg, policy=msg.policy.clone(utf8=True)) + mail_options = (*mail_options, 'SMTPUTF8', 'BODY=8BITMIME') + else: + g = email.generator.BytesGenerator(bytesmsg) + g.flatten(msg_copy, linesep='\r\n') + flatmsg = bytesmsg.getvalue() + return self.sendmail(from_addr, to_addrs, flatmsg, mail_options, + rcpt_options) + + def close(self): + """Close the connection to the SMTP server.""" + try: + file = self.file + self.file = None + if file: + file.close() + finally: + sock = self.sock + self.sock = None + if sock: + sock.close() + + def quit(self): + """Terminate the SMTP session.""" + res = self.docmd("quit") + # A new EHLO is required after reconnecting with connect() + self.ehlo_resp = self.helo_resp = None + self.esmtp_features = {} + self.does_esmtp = False + self.close() + return res + +if _have_ssl: + + class SMTP_SSL(SMTP): + """ This is a subclass derived from SMTP that connects over an SSL + encrypted socket (to use this class you need a socket module that was + compiled with SSL support). If host is not specified, '' (the local + host) is used. If port is omitted, the standard SMTP-over-SSL port + (465) is used. local_hostname and source_address have the same meaning + as they do in the SMTP class. context also optional, can contain a + SSLContext. + + """ + + default_port = SMTP_SSL_PORT + + def __init__(self, host='', port=0, local_hostname=None, + *, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, context=None): + if context is None: + context = ssl._create_stdlib_context() + self.context = context + SMTP.__init__(self, host, port, local_hostname, timeout, + source_address) + + def _get_socket(self, host, port, timeout): + if self.debuglevel > 0: + self._print_debug('connect:', (host, port)) + new_socket = super()._get_socket(host, port, timeout) + new_socket = self.context.wrap_socket(new_socket, + server_hostname=self._host) + return new_socket + + __all__.append("SMTP_SSL") + +# +# LMTP extension +# +LMTP_PORT = 2003 + +class LMTP(SMTP): + """LMTP - Local Mail Transfer Protocol + + The LMTP protocol, which is very similar to ESMTP, is heavily based + on the standard SMTP client. It's common to use Unix sockets for + LMTP, so our connect() method must support that as well as a regular + host:port server. local_hostname and source_address have the same + meaning as they do in the SMTP class. To specify a Unix socket, + you must use an absolute path as the host, starting with a '/'. + + Authentication is supported, using the regular SMTP mechanism. When + using a Unix socket, LMTP generally don't support or require any + authentication, but your mileage might vary.""" + + ehlo_msg = "lhlo" + + def __init__(self, host='', port=LMTP_PORT, local_hostname=None, + source_address=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + """Initialize a new instance.""" + super().__init__(host, port, local_hostname=local_hostname, + source_address=source_address, timeout=timeout) + + def connect(self, host='localhost', port=0, source_address=None): + """Connect to the LMTP daemon, on either a Unix or a TCP socket.""" + if host[0] != '/': + return super().connect(host, port, source_address=source_address) + + if self.timeout is not None and not self.timeout: + raise ValueError('Non-blocking socket (timeout=0) is not supported') + + # Handle Unix-domain sockets. + try: + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + self.sock.settimeout(self.timeout) + self.file = None + self.sock.connect(host) + except OSError: + if self.debuglevel > 0: + self._print_debug('connect fail:', host) + if self.sock: + self.sock.close() + self.sock = None + raise + (code, msg) = self.getreply() + if self.debuglevel > 0: + self._print_debug('connect:', msg) + return (code, msg) + + +# Test the sendmail method, which tests most of the others. +# Note: This always sends to localhost. +if __name__ == '__main__': + def prompt(prompt): + sys.stdout.write(prompt + ": ") + sys.stdout.flush() + return sys.stdin.readline().strip() + + fromaddr = prompt("From") + toaddrs = prompt("To").split(',') + print("Enter message, end with ^D:") + msg = '' + while line := sys.stdin.readline(): + msg = msg + line + print("Message length is %d" % len(msg)) + + server = SMTP('localhost') + server.set_debuglevel(1) + server.sendmail(fromaddr, toaddrs, msg) + server.quit() diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py deleted file mode 100644 index 594353136f5..00000000000 --- a/Lib/sndhdr.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Routines to help recognizing sound files. - -Function whathdr() recognizes various types of sound file headers. -It understands almost all headers that SOX can decode. - -The return tuple contains the following items, in this order: -- file type (as SOX understands it) -- sampling rate (0 if unknown or hard to decode) -- number of channels (0 if unknown or hard to decode) -- number of frames in the file (-1 if unknown or hard to decode) -- number of bits/sample, or 'U' for U-LAW, or 'A' for A-LAW - -If the file doesn't have a recognizable type, it returns None. -If the file can't be opened, OSError is raised. - -To compute the total time, divide the number of frames by the -sampling rate (a frame contains a sample for each channel). - -Function what() calls whathdr(). (It used to also use some -heuristics for raw data, but this doesn't work very well.) - -Finally, the function test() is a simple main program that calls -what() for all files mentioned on the argument list. For directory -arguments it calls what() for all files in that directory. Default -argument is "." (testing all files in the current directory). The -option -r tells it to recurse down directories found inside -explicitly given directories. -""" - -# The file structure is top-down except that the test program and its -# subroutine come last. - -__all__ = ['what', 'whathdr'] - -from collections import namedtuple - -SndHeaders = namedtuple('SndHeaders', - 'filetype framerate nchannels nframes sampwidth') - -SndHeaders.filetype.__doc__ = ("""The value for type indicates the data type -and will be one of the strings 'aifc', 'aiff', 'au','hcom', -'sndr', 'sndt', 'voc', 'wav', '8svx', 'sb', 'ub', or 'ul'.""") -SndHeaders.framerate.__doc__ = ("""The sampling_rate will be either the actual -value or 0 if unknown or difficult to decode.""") -SndHeaders.nchannels.__doc__ = ("""The number of channels or 0 if it cannot be -determined or if the value is difficult to decode.""") -SndHeaders.nframes.__doc__ = ("""The value for frames will be either the number -of frames or -1.""") -SndHeaders.sampwidth.__doc__ = ("""Either the sample size in bits or -'A' for A-LAW or 'U' for u-LAW.""") - -def what(filename): - """Guess the type of a sound file.""" - res = whathdr(filename) - return res - - -def whathdr(filename): - """Recognize sound headers.""" - with open(filename, 'rb') as f: - h = f.read(512) - for tf in tests: - res = tf(h, f) - if res: - return SndHeaders(*res) - return None - - -#-----------------------------------# -# Subroutines per sound header type # -#-----------------------------------# - -tests = [] - -def test_aifc(h, f): - import aifc - if not h.startswith(b'FORM'): - return None - if h[8:12] == b'AIFC': - fmt = 'aifc' - elif h[8:12] == b'AIFF': - fmt = 'aiff' - else: - return None - f.seek(0) - try: - a = aifc.open(f, 'r') - except (EOFError, aifc.Error): - return None - return (fmt, a.getframerate(), a.getnchannels(), - a.getnframes(), 8 * a.getsampwidth()) - -tests.append(test_aifc) - - -def test_au(h, f): - if h.startswith(b'.snd'): - func = get_long_be - elif h[:4] in (b'\0ds.', b'dns.'): - func = get_long_le - else: - return None - filetype = 'au' - hdr_size = func(h[4:8]) - data_size = func(h[8:12]) - encoding = func(h[12:16]) - rate = func(h[16:20]) - nchannels = func(h[20:24]) - sample_size = 1 # default - if encoding == 1: - sample_bits = 'U' - elif encoding == 2: - sample_bits = 8 - elif encoding == 3: - sample_bits = 16 - sample_size = 2 - else: - sample_bits = '?' - frame_size = sample_size * nchannels - if frame_size: - nframe = data_size / frame_size - else: - nframe = -1 - return filetype, rate, nchannels, nframe, sample_bits - -tests.append(test_au) - - -def test_hcom(h, f): - if h[65:69] != b'FSSD' or h[128:132] != b'HCOM': - return None - divisor = get_long_be(h[144:148]) - if divisor: - rate = 22050 / divisor - else: - rate = 0 - return 'hcom', rate, 1, -1, 8 - -tests.append(test_hcom) - - -def test_voc(h, f): - if not h.startswith(b'Creative Voice File\032'): - return None - sbseek = get_short_le(h[20:22]) - rate = 0 - if 0 <= sbseek < 500 and h[sbseek] == 1: - ratecode = 256 - h[sbseek+4] - if ratecode: - rate = int(1000000.0 / ratecode) - return 'voc', rate, 1, -1, 8 - -tests.append(test_voc) - - -def test_wav(h, f): - import wave - # 'RIFF' 'WAVE' 'fmt ' - if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ': - return None - f.seek(0) - try: - w = wave.open(f, 'r') - except (EOFError, wave.Error): - return None - return ('wav', w.getframerate(), w.getnchannels(), - w.getnframes(), 8*w.getsampwidth()) - -tests.append(test_wav) - - -def test_8svx(h, f): - if not h.startswith(b'FORM') or h[8:12] != b'8SVX': - return None - # Should decode it to get #channels -- assume always 1 - return '8svx', 0, 1, 0, 8 - -tests.append(test_8svx) - - -def test_sndt(h, f): - if h.startswith(b'SOUND'): - nsamples = get_long_le(h[8:12]) - rate = get_short_le(h[20:22]) - return 'sndt', rate, 1, nsamples, 8 - -tests.append(test_sndt) - - -def test_sndr(h, f): - if h.startswith(b'\0\0'): - rate = get_short_le(h[2:4]) - if 4000 <= rate <= 25000: - return 'sndr', rate, 1, -1, 8 - -tests.append(test_sndr) - - -#-------------------------------------------# -# Subroutines to extract numbers from bytes # -#-------------------------------------------# - -def get_long_be(b): - return (b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3] - -def get_long_le(b): - return (b[3] << 24) | (b[2] << 16) | (b[1] << 8) | b[0] - -def get_short_be(b): - return (b[0] << 8) | b[1] - -def get_short_le(b): - return (b[1] << 8) | b[0] - - -#--------------------# -# Small test program # -#--------------------# - -def test(): - import sys - recursive = 0 - if sys.argv[1:] and sys.argv[1] == '-r': - del sys.argv[1:2] - recursive = 1 - try: - if sys.argv[1:]: - testall(sys.argv[1:], recursive, 1) - else: - testall(['.'], recursive, 1) - except KeyboardInterrupt: - sys.stderr.write('\n[Interrupted]\n') - sys.exit(1) - -def testall(list, recursive, toplevel): - import sys - import os - for filename in list: - if os.path.isdir(filename): - print(filename + '/:', end=' ') - if recursive or toplevel: - print('recursing down:') - import glob - names = glob.glob(os.path.join(filename, '*')) - testall(names, recursive, 0) - else: - print('*** directory (use -r) ***') - else: - print(filename + ':', end=' ') - sys.stdout.flush() - try: - print(what(filename)) - except OSError: - print('*** not found ***') - -if __name__ == '__main__': - test() diff --git a/Lib/socket.py b/Lib/socket.py index 63ba0acc908..727b0e75f03 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -13,7 +13,7 @@ socketpair() -- create a pair of new socket objects [*] fromfd() -- create a socket object from an open file descriptor [*] send_fds() -- Send file descriptor to the socket. -recv_fds() -- Recieve file descriptors from the socket. +recv_fds() -- Receive file descriptors from the socket. fromshare() -- create a socket object from data received from socket.share() [*] gethostname() -- return the current hostname gethostbyname() -- map a hostname to its IP number @@ -28,6 +28,7 @@ socket.setdefaulttimeout() -- set the default timeout value create_connection() -- connects to an address, with an optional timeout and optional source address. +create_server() -- create a TCP socket and bind it to a specified address. [*] not available on all platforms! @@ -51,7 +52,9 @@ import _socket from _socket import * -import os, sys, io, selectors +import io +import os +import sys from enum import IntEnum, IntFlag try: @@ -109,102 +112,103 @@ def _intenum_converter(value, enum_klass): # WSA error codes if sys.platform.lower().startswith("win"): - errorTab = {} - errorTab[6] = "Specified event object handle is invalid." - errorTab[8] = "Insufficient memory available." - errorTab[87] = "One or more parameters are invalid." - errorTab[995] = "Overlapped operation aborted." - errorTab[996] = "Overlapped I/O event object not in signaled state." - errorTab[997] = "Overlapped operation will complete later." - errorTab[10004] = "The operation was interrupted." - errorTab[10009] = "A bad file handle was passed." - errorTab[10013] = "Permission denied." - errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT - errorTab[10022] = "An invalid operation was attempted." - errorTab[10024] = "Too many open files." - errorTab[10035] = "The socket operation would block" - errorTab[10036] = "A blocking operation is already in progress." - errorTab[10037] = "Operation already in progress." - errorTab[10038] = "Socket operation on nonsocket." - errorTab[10039] = "Destination address required." - errorTab[10040] = "Message too long." - errorTab[10041] = "Protocol wrong type for socket." - errorTab[10042] = "Bad protocol option." - errorTab[10043] = "Protocol not supported." - errorTab[10044] = "Socket type not supported." - errorTab[10045] = "Operation not supported." - errorTab[10046] = "Protocol family not supported." - errorTab[10047] = "Address family not supported by protocol family." - errorTab[10048] = "The network address is in use." - errorTab[10049] = "Cannot assign requested address." - errorTab[10050] = "Network is down." - errorTab[10051] = "Network is unreachable." - errorTab[10052] = "Network dropped connection on reset." - errorTab[10053] = "Software caused connection abort." - errorTab[10054] = "The connection has been reset." - errorTab[10055] = "No buffer space available." - errorTab[10056] = "Socket is already connected." - errorTab[10057] = "Socket is not connected." - errorTab[10058] = "The network has been shut down." - errorTab[10059] = "Too many references." - errorTab[10060] = "The operation timed out." - errorTab[10061] = "Connection refused." - errorTab[10062] = "Cannot translate name." - errorTab[10063] = "The name is too long." - errorTab[10064] = "The host is down." - errorTab[10065] = "The host is unreachable." - errorTab[10066] = "Directory not empty." - errorTab[10067] = "Too many processes." - errorTab[10068] = "User quota exceeded." - errorTab[10069] = "Disk quota exceeded." - errorTab[10070] = "Stale file handle reference." - errorTab[10071] = "Item is remote." - errorTab[10091] = "Network subsystem is unavailable." - errorTab[10092] = "Winsock.dll version out of range." - errorTab[10093] = "Successful WSAStartup not yet performed." - errorTab[10101] = "Graceful shutdown in progress." - errorTab[10102] = "No more results from WSALookupServiceNext." - errorTab[10103] = "Call has been canceled." - errorTab[10104] = "Procedure call table is invalid." - errorTab[10105] = "Service provider is invalid." - errorTab[10106] = "Service provider failed to initialize." - errorTab[10107] = "System call failure." - errorTab[10108] = "Service not found." - errorTab[10109] = "Class type not found." - errorTab[10110] = "No more results from WSALookupServiceNext." - errorTab[10111] = "Call was canceled." - errorTab[10112] = "Database query was refused." - errorTab[11001] = "Host not found." - errorTab[11002] = "Nonauthoritative host not found." - errorTab[11003] = "This is a nonrecoverable error." - errorTab[11004] = "Valid name, no data record requested type." - errorTab[11005] = "QoS receivers." - errorTab[11006] = "QoS senders." - errorTab[11007] = "No QoS senders." - errorTab[11008] = "QoS no receivers." - errorTab[11009] = "QoS request confirmed." - errorTab[11010] = "QoS admission error." - errorTab[11011] = "QoS policy failure." - errorTab[11012] = "QoS bad style." - errorTab[11013] = "QoS bad object." - errorTab[11014] = "QoS traffic control error." - errorTab[11015] = "QoS generic error." - errorTab[11016] = "QoS service type error." - errorTab[11017] = "QoS flowspec error." - errorTab[11018] = "Invalid QoS provider buffer." - errorTab[11019] = "Invalid QoS filter style." - errorTab[11020] = "Invalid QoS filter style." - errorTab[11021] = "Incorrect QoS filter count." - errorTab[11022] = "Invalid QoS object length." - errorTab[11023] = "Incorrect QoS flow count." - errorTab[11024] = "Unrecognized QoS object." - errorTab[11025] = "Invalid QoS policy object." - errorTab[11026] = "Invalid QoS flow descriptor." - errorTab[11027] = "Invalid QoS provider-specific flowspec." - errorTab[11028] = "Invalid QoS provider-specific filterspec." - errorTab[11029] = "Invalid QoS shape discard mode object." - errorTab[11030] = "Invalid QoS shaping rate object." - errorTab[11031] = "Reserved policy QoS element type." + errorTab = { + 6: "Specified event object handle is invalid.", + 8: "Insufficient memory available.", + 87: "One or more parameters are invalid.", + 995: "Overlapped operation aborted.", + 996: "Overlapped I/O event object not in signaled state.", + 997: "Overlapped operation will complete later.", + 10004: "The operation was interrupted.", + 10009: "A bad file handle was passed.", + 10013: "Permission denied.", + 10014: "A fault occurred on the network??", + 10022: "An invalid operation was attempted.", + 10024: "Too many open files.", + 10035: "The socket operation would block.", + 10036: "A blocking operation is already in progress.", + 10037: "Operation already in progress.", + 10038: "Socket operation on nonsocket.", + 10039: "Destination address required.", + 10040: "Message too long.", + 10041: "Protocol wrong type for socket.", + 10042: "Bad protocol option.", + 10043: "Protocol not supported.", + 10044: "Socket type not supported.", + 10045: "Operation not supported.", + 10046: "Protocol family not supported.", + 10047: "Address family not supported by protocol family.", + 10048: "The network address is in use.", + 10049: "Cannot assign requested address.", + 10050: "Network is down.", + 10051: "Network is unreachable.", + 10052: "Network dropped connection on reset.", + 10053: "Software caused connection abort.", + 10054: "The connection has been reset.", + 10055: "No buffer space available.", + 10056: "Socket is already connected.", + 10057: "Socket is not connected.", + 10058: "The network has been shut down.", + 10059: "Too many references.", + 10060: "The operation timed out.", + 10061: "Connection refused.", + 10062: "Cannot translate name.", + 10063: "The name is too long.", + 10064: "The host is down.", + 10065: "The host is unreachable.", + 10066: "Directory not empty.", + 10067: "Too many processes.", + 10068: "User quota exceeded.", + 10069: "Disk quota exceeded.", + 10070: "Stale file handle reference.", + 10071: "Item is remote.", + 10091: "Network subsystem is unavailable.", + 10092: "Winsock.dll version out of range.", + 10093: "Successful WSAStartup not yet performed.", + 10101: "Graceful shutdown in progress.", + 10102: "No more results from WSALookupServiceNext.", + 10103: "Call has been canceled.", + 10104: "Procedure call table is invalid.", + 10105: "Service provider is invalid.", + 10106: "Service provider failed to initialize.", + 10107: "System call failure.", + 10108: "Service not found.", + 10109: "Class type not found.", + 10110: "No more results from WSALookupServiceNext.", + 10111: "Call was canceled.", + 10112: "Database query was refused.", + 11001: "Host not found.", + 11002: "Nonauthoritative host not found.", + 11003: "This is a nonrecoverable error.", + 11004: "Valid name, no data record requested type.", + 11005: "QoS receivers.", + 11006: "QoS senders.", + 11007: "No QoS senders.", + 11008: "QoS no receivers.", + 11009: "QoS request confirmed.", + 11010: "QoS admission error.", + 11011: "QoS policy failure.", + 11012: "QoS bad style.", + 11013: "QoS bad object.", + 11014: "QoS traffic control error.", + 11015: "QoS generic error.", + 11016: "QoS service type error.", + 11017: "QoS flowspec error.", + 11018: "Invalid QoS provider buffer.", + 11019: "Invalid QoS filter style.", + 11020: "Invalid QoS filter style.", + 11021: "Incorrect QoS filter count.", + 11022: "Invalid QoS object length.", + 11023: "Incorrect QoS flow count.", + 11024: "Unrecognized QoS object.", + 11025: "Invalid QoS policy object.", + 11026: "Invalid QoS flow descriptor.", + 11027: "Invalid QoS provider-specific flowspec.", + 11028: "Invalid QoS provider-specific filterspec.", + 11029: "Invalid QoS shape discard mode object.", + 11030: "Invalid QoS shaping rate object.", + 11031: "Reserved policy QoS element type." + } __all__.append("errorTab") @@ -254,17 +258,18 @@ def __repr__(self): self.type, self.proto) if not closed: + # getsockname and getpeername may not be available on WASI. try: laddr = self.getsockname() if laddr: s += ", laddr=%s" % str(laddr) - except error: + except (error, AttributeError): pass try: raddr = self.getpeername() if raddr: s += ", raddr=%s" % str(raddr) - except error: + except (error, AttributeError): pass s += '>' return s @@ -304,7 +309,8 @@ def makefile(self, mode="r", buffering=None, *, """makefile(...) -> an I/O stream connected to the socket The arguments are as for io.open() after the filename, except the only - supported mode values are 'r' (default), 'w' and 'b'. + supported mode values are 'r' (default), 'w', 'b', or a combination of + those. """ # XXX refactor to share code? if not set(mode) <= {"r", "w", "b"}: @@ -345,6 +351,9 @@ def makefile(self, mode="r", buffering=None, *, if hasattr(os, 'sendfile'): def _sendfile_use_sendfile(self, file, offset=0, count=None): + # Lazy import to improve module import time + import selectors + self._check_sendfile_params(file, offset, count) sockno = self.fileno() try: @@ -380,7 +389,7 @@ def _sendfile_use_sendfile(self, file, offset=0, count=None): if timeout and not selector_select(timeout): raise TimeoutError('timed out') if count: - blocksize = count - total_sent + blocksize = min(count - total_sent, blocksize) if blocksize <= 0: break try: @@ -546,20 +555,18 @@ def fromfd(fd, family, type, proto=0): return socket(family, type, proto, nfd) if hasattr(_socket.socket, "sendmsg"): - import array - def send_fds(sock, buffers, fds, flags=0, address=None): """ send_fds(sock, buffers, fds[, flags[, address]]) -> integer Send the list of file descriptors fds over an AF_UNIX socket. """ + import array + return sock.sendmsg(buffers, [(_socket.SOL_SOCKET, _socket.SCM_RIGHTS, array.array("i", fds))]) __all__.append("send_fds") if hasattr(_socket.socket, "recvmsg"): - import array - def recv_fds(sock, bufsize, maxfds, flags=0): """ recv_fds(sock, bufsize, maxfds[, flags]) -> (data, list of file descriptors, msg_flags, address) @@ -567,6 +574,8 @@ def recv_fds(sock, bufsize, maxfds, flags=0): Receive up to maxfds file descriptors returning the message data and a list containing the descriptors. """ + import array + # Array of ints fds = array.array("i") msg, ancdata, flags, addr = sock.recvmsg(bufsize, @@ -589,16 +598,65 @@ def fromshare(info): return socket(0, 0, 0, info) __all__.append("fromshare") -if hasattr(_socket, "socketpair"): +# Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. +# This is used if _socket doesn't natively provide socketpair. It's +# always defined so that it can be patched in for testing purposes. +def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): + if family == AF_INET: + host = _LOCALHOST + elif family == AF_INET6: + host = _LOCALHOST_V6 + else: + raise ValueError("Only AF_INET and AF_INET6 socket address families " + "are supported") + if type != SOCK_STREAM: + raise ValueError("Only SOCK_STREAM socket type is supported") + if proto != 0: + raise ValueError("Only protocol zero is supported") + + # We create a connected TCP socket. Note the trick with + # setblocking(False) that prevents us from having to create a thread. + lsock = socket(family, type, proto) + try: + lsock.bind((host, 0)) + lsock.listen() + # On IPv6, ignore flow_info and scope_id + addr, port = lsock.getsockname()[:2] + csock = socket(family, type, proto) + try: + csock.setblocking(False) + try: + csock.connect((addr, port)) + except (BlockingIOError, InterruptedError): + pass + csock.setblocking(True) + ssock, _ = lsock.accept() + except: + csock.close() + raise + finally: + lsock.close() - def socketpair(family=None, type=SOCK_STREAM, proto=0): - """socketpair([family[, type[, proto]]]) -> (socket object, socket object) + # Authenticating avoids using a connection from something else + # able to connect to {host}:{port} instead of us. + # We expect only AF_INET and AF_INET6 families. + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise - Create a pair of socket objects from the sockets returned by the platform - socketpair() function. - The arguments are the same as for socket() except the default family is - AF_UNIX if defined on the platform; otherwise, the default is AF_INET. - """ + return (ssock, csock) + +if hasattr(_socket, "socketpair"): + def socketpair(family=None, type=SOCK_STREAM, proto=0): if family is None: try: family = AF_UNIX @@ -610,44 +668,7 @@ def socketpair(family=None, type=SOCK_STREAM, proto=0): return a, b else: - - # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. - def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): - if family == AF_INET: - host = _LOCALHOST - elif family == AF_INET6: - host = _LOCALHOST_V6 - else: - raise ValueError("Only AF_INET and AF_INET6 socket address families " - "are supported") - if type != SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with - # setblocking(False) that prevents us from having to create a thread. - lsock = socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen() - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) + socketpair = _fallback_socketpair __all__.append("socketpair") socketpair.__doc__ = """socketpair([family[, type[, proto]]]) -> (socket object, socket object) @@ -700,16 +721,15 @@ def readinto(self, b): self._checkReadable() if self._timeout_occurred: raise OSError("cannot read from timed out object") - while True: - try: - return self._sock.recv_into(b) - except timeout: - self._timeout_occurred = True - raise - except error as e: - if e.errno in _blocking_errnos: - return None - raise + try: + return self._sock.recv_into(b) + except timeout: + self._timeout_occurred = True + raise + except error as e: + if e.errno in _blocking_errnos: + return None + raise def write(self, b): """Write the given bytes or bytearray object *b* to the socket @@ -783,11 +803,11 @@ def getfqdn(name=''): First the hostname returned by gethostbyaddr() is checked, then possibly existing aliases. In case no FQDN is available and `name` - was given, it is returned unchanged. If `name` was empty or '0.0.0.0', + was given, it is returned unchanged. If `name` was empty, '0.0.0.0' or '::', hostname from gethostname() is returned. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name in ('0.0.0.0', '::'): name = gethostname() try: hostname, aliases, ipaddrs = gethostbyaddr(name) @@ -806,7 +826,7 @@ def getfqdn(name=''): _GLOBAL_DEFAULT_TIMEOUT = object() def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + source_address=None, *, all_errors=False): """Connect to *address* and return the socket object. Convenience function. Connect to *address* (a 2-tuple ``(host, @@ -816,11 +836,13 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, global default timeout setting returned by :func:`getdefaulttimeout` is used. If *source_address* is set it must be a tuple of (host, port) for the socket to bind as a source address before making the connection. - A host of '' or port 0 tells the OS to use the default. + A host of '' or port 0 tells the OS to use the default. When a connection + cannot be created, raises the last error if *all_errors* is False, + and an ExceptionGroup of all errors if *all_errors* is True. """ host, port = address - err = None + exceptions = [] for res in getaddrinfo(host, port, 0, SOCK_STREAM): af, socktype, proto, canonname, sa = res sock = None @@ -832,20 +854,24 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, sock.bind(source_address) sock.connect(sa) # Break explicitly a reference cycle - err = None + exceptions.clear() return sock - except error as _: - err = _ + except error as exc: + if not all_errors: + exceptions.clear() # raise only the last error + exceptions.append(exc) if sock is not None: sock.close() - if err is not None: + if len(exceptions): try: - raise err + if not all_errors: + raise exceptions[0] + raise ExceptionGroup("create_connection failed", exceptions) finally: # Break explicitly a reference cycle - err = None + exceptions.clear() else: raise error("getaddrinfo returns an empty list") @@ -902,7 +928,7 @@ def create_server(address, *, family=AF_INET, backlog=None, reuse_port=False, # address, effectively preventing this one from accepting # connections. Also, it may set the process in a state where # it'll no longer respond to any signals or graceful kills. - # See: msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx + # See: https://learn.microsoft.com/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse if os.name not in ('nt', 'cygwin') and \ hasattr(_socket, 'SO_REUSEADDR'): try: @@ -911,7 +937,9 @@ def create_server(address, *, family=AF_INET, backlog=None, reuse_port=False, # Fail later on bind(), for platforms which may not # support this option. pass - if reuse_port: + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if reuse_port and family in (AF_INET, AF_INET6): sock.setsockopt(SOL_SOCKET, SO_REUSEPORT, 1) if has_ipv6 and family == AF_INET6: if dualstack_ipv6: diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 2905e3eac36..35b2723de3b 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -127,10 +127,7 @@ class will essentially render the service "deaf" while one request is import selectors import os import sys -try: - import threading -except ImportError: - import dummy_threading as threading +import threading from io import BufferedIOBase from time import monotonic as time @@ -144,6 +141,8 @@ class will essentially render the service "deaf" while one request is __all__.extend(["UnixStreamServer","UnixDatagramServer", "ThreadingUnixStreamServer", "ThreadingUnixDatagramServer"]) + if hasattr(os, "fork"): + __all__.extend(["ForkingUnixStreamServer", "ForkingUnixDatagramServer"]) # poll/select have the advantage of not requiring any extra file descriptor, # contrarily to epoll/kqueue (also, they require a single syscall). @@ -190,6 +189,7 @@ class BaseServer: - address_family - socket_type - allow_reuse_address + - allow_reuse_port Instance variables: @@ -294,8 +294,7 @@ def handle_request(self): selector.register(self, selectors.EVENT_READ) while True: - ready = selector.select(timeout) - if ready: + if selector.select(timeout): return self._handle_request_noblock() else: if timeout is not None: @@ -428,6 +427,7 @@ class TCPServer(BaseServer): - socket_type - request_queue_size (only for stream sockets) - allow_reuse_address + - allow_reuse_port Instance variables: @@ -445,6 +445,8 @@ class TCPServer(BaseServer): allow_reuse_address = False + allow_reuse_port = False + def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True): """Constructor. May be extended, do not override.""" BaseServer.__init__(self, server_address, RequestHandlerClass) @@ -464,8 +466,15 @@ def server_bind(self): May be overridden. """ - if self.allow_reuse_address: + if self.allow_reuse_address and hasattr(socket, "SO_REUSEADDR"): self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # Since Linux 6.12.9, SO_REUSEPORT is not allowed + # on other address families than AF_INET/AF_INET6. + if ( + self.allow_reuse_port and hasattr(socket, "SO_REUSEPORT") + and self.address_family in (socket.AF_INET, socket.AF_INET6) + ): + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) self.socket.bind(self.server_address) self.server_address = self.socket.getsockname() @@ -522,6 +531,8 @@ class UDPServer(TCPServer): allow_reuse_address = False + allow_reuse_port = False + socket_type = socket.SOCK_DGRAM max_packet_size = 8192 @@ -723,6 +734,11 @@ class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): pass class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): pass + if hasattr(os, "fork"): + class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer): pass + + class ForkingUnixDatagramServer(ForkingMixIn, UnixDatagramServer): pass + class BaseRequestHandler: """Base class for request handler classes. diff --git a/Lib/sqlite3/__init__.py b/Lib/sqlite3/__init__.py index 927267cf0b9..ed727fae609 100644 --- a/Lib/sqlite3/__init__.py +++ b/Lib/sqlite3/__init__.py @@ -22,7 +22,7 @@ """ The sqlite3 extension module provides a DB-API 2.0 (PEP 249) compliant -interface to the SQLite library, and requires SQLite 3.7.15 or newer. +interface to the SQLite library, and requires SQLite 3.15.2 or newer. To use the module, start by creating a database Connection object: @@ -55,16 +55,3 @@ """ from sqlite3.dbapi2 import * -from sqlite3.dbapi2 import (_deprecated_names, - _deprecated_version_info, - _deprecated_version) - - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 1832fc13080..4ccf292ddf2 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -46,38 +46,34 @@ def runsource(self, source, filename="", symbol="single"): """Override runsource, the core of the InteractiveConsole REPL. Return True if more input is needed; buffering is done automatically. - Return False is input is a complete statement ready for execution. + Return False if input is a complete statement ready for execution. """ - if source == ".version": - print(f"{sqlite3.sqlite_version}") - elif source == ".help": - print("Enter SQL code and press enter.") - elif source == ".quit": - sys.exit(0) - elif not sqlite3.complete_statement(source): - return True + if not source or source.isspace(): + return False + if source[0] == ".": + match source[1:].strip(): + case "version": + print(f"{sqlite3.sqlite_version}") + case "help": + print("Enter SQL code and press enter.") + case "quit": + sys.exit(0) + case "": + pass + case _ as unknown: + self.write("Error: unknown command or invalid arguments:" + f' "{unknown}".\n') else: + if not sqlite3.complete_statement(source): + return True execute(self._cur, source) - return False - # TODO: RUSTPYTHON match statement supporting - # match source: - # case ".version": - # print(f"{sqlite3.sqlite_version}") - # case ".help": - # print("Enter SQL code and press enter.") - # case ".quit": - # sys.exit(0) - # case _: - # if not sqlite3.complete_statement(source): - # return True - # execute(self._cur, source) - # return False - - -def main(): + return False + + +def main(*args): parser = ArgumentParser( description="Python sqlite3 CLI", - prog="python -m sqlite3", + color=True, ) parser.add_argument( "filename", type=str, default=":memory:", nargs="?", @@ -98,7 +94,7 @@ def main(): version=f"SQLite version {sqlite3.sqlite_version}", help="Print underlying SQLite library version", ) - args = parser.parse_args() + args = parser.parse_args(*args) if args.filename == ":memory:": db_name = "a transient in-memory database" @@ -106,12 +102,16 @@ def main(): db_name = repr(args.filename) # Prepare REPL banner and prompts. + if sys.platform == "win32" and "idlelib.run" not in sys.modules: + eofkey = "CTRL-Z" + else: + eofkey = "CTRL-D" banner = dedent(f""" sqlite3 shell, running on SQLite version {sqlite3.sqlite_version} Connected to {db_name} Each command will be run using execute() on the cursor. - Type ".help" for more information; type ".quit" or CTRL-D to quit. + Type ".help" for more information; type ".quit" or {eofkey} to quit. """).strip() sys.ps1 = "sqlite> " sys.ps2 = " ... " @@ -124,9 +124,16 @@ def main(): else: # No SQL provided; start the REPL. console = SqliteInteractiveConsole(con) + try: + import readline # noqa: F401 + except ImportError: + pass console.interact(banner, exitmsg="") finally: con.close() + sys.exit(0) + -main() +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 56fc0461e6c..0315760516e 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -25,9 +25,6 @@ import collections.abc from _sqlite3 import * -from _sqlite3 import _deprecated_version - -_deprecated_names = frozenset({"version", "version_info"}) paramstyle = "qmark" @@ -48,7 +45,7 @@ def TimeFromTicks(ticks): def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) -_deprecated_version_info = tuple(map(int, _deprecated_version.split("."))) + sqlite_version_info = tuple([int(x) for x in sqlite_version.split(".")]) Binary = memoryview @@ -97,12 +94,3 @@ def convert_timestamp(val): # Clean up namespace del(register_adapters_and_converters) - -def __getattr__(name): - if name in _deprecated_names: - from warnings import warn - - warn(f"{name} is deprecated and will be removed in Python 3.14", - DeprecationWarning, stacklevel=2) - return globals()[f"_deprecated_{name}"] - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 07b9da10b92..57e6a3b4f1e 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -7,7 +7,15 @@ # future enhancements, you should normally quote any identifier that # is an English language word, even if you do not have to." -def _iterdump(connection): +def _quote_name(name): + return '"{0}"'.format(name.replace('"', '""')) + + +def _quote_value(value): + return "'{0}'".format(value.replace("'", "''")) + + +def _iterdump(connection, *, filter=None): """ Returns an iterator to the dump of the database in an SQL text format. @@ -16,64 +24,87 @@ def _iterdump(connection): directly but instead called from the Connection method, iterdump(). """ + writeable_schema = False cu = connection.cursor() + cu.row_factory = None # Make sure we get predictable results. + # Disable foreign key constraints, if there is any foreign key violation. + violations = cu.execute("PRAGMA foreign_key_check").fetchall() + if violations: + yield('PRAGMA foreign_keys=OFF;') yield('BEGIN TRANSACTION;') + if filter: + # Return database objects which match the filter pattern. + filter_name_clause = 'AND "name" LIKE ?' + params = [filter] + else: + filter_name_clause = "" + params = [] # sqlite_master table contains the SQL CREATE statements for the database. - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" == 'table' + {filter_name_clause} ORDER BY "name" """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) sqlite_sequence = [] for table_name, type, sql in schema_res.fetchall(): if table_name == 'sqlite_sequence': - rows = cu.execute('SELECT * FROM "sqlite_sequence";').fetchall() + rows = cu.execute('SELECT * FROM "sqlite_sequence";') sqlite_sequence = ['DELETE FROM "sqlite_sequence"'] sqlite_sequence += [ - f'INSERT INTO "sqlite_sequence" VALUES(\'{row[0]}\',{row[1]})' - for row in rows + f'INSERT INTO "sqlite_sequence" VALUES({_quote_value(table_name)},{seq_value})' + for table_name, seq_value in rows.fetchall() ] continue elif table_name == 'sqlite_stat1': yield('ANALYZE "sqlite_master";') elif table_name.startswith('sqlite_'): continue - # NOTE: Virtual table support not implemented - #elif sql.startswith('CREATE VIRTUAL TABLE'): - # qtable = table_name.replace("'", "''") - # yield("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)"\ - # "VALUES('table','{0}','{0}',0,'{1}');".format( - # qtable, - # sql.replace("''"))) + elif sql.startswith('CREATE VIRTUAL TABLE'): + if not writeable_schema: + writeable_schema = True + yield('PRAGMA writable_schema=ON;') + yield("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table',{0},{0},0,{1});".format( + _quote_value(table_name), + _quote_value(sql), + )) else: yield('{0};'.format(sql)) # Build the insert statement for each row of the current table - table_name_ident = table_name.replace('"', '""') - res = cu.execute('PRAGMA table_info("{0}")'.format(table_name_ident)) + table_name_ident = _quote_name(table_name) + res = cu.execute(f'PRAGMA table_info({table_name_ident})') column_names = [str(table_info[1]) for table_info in res.fetchall()] - q = """SELECT 'INSERT INTO "{0}" VALUES({1})' FROM "{0}";""".format( + q = "SELECT 'INSERT INTO {0} VALUES('{1}')' FROM {0};".format( table_name_ident, - ",".join("""'||quote("{0}")||'""".format(col.replace('"', '""')) for col in column_names)) + "','".join( + "||quote({0})||".format(_quote_name(col)) for col in column_names + ) + ) query_res = cu.execute(q) for row in query_res: yield("{0};".format(row[0])) # Now when the type is 'index', 'trigger', or 'view' - q = """ + q = f""" SELECT "name", "type", "sql" FROM "sqlite_master" WHERE "sql" NOT NULL AND "type" IN ('index', 'trigger', 'view') + {filter_name_clause} """ - schema_res = cu.execute(q) + schema_res = cu.execute(q, params) for name, type, sql in schema_res.fetchall(): yield('{0};'.format(sql)) + if writeable_schema: + yield('PRAGMA writable_schema=OFF;') + # gh-79009: Yield statements concerning the sqlite_sequence table at the # end of the transaction. for row in sqlite_sequence: diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py index c6398bfb83a..f9da61e6487 100644 --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -1,784 +1,7 @@ -# -# Secret Labs' Regular Expression Engine -# -# convert template to internal format -# -# Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. -# -# See the sre.py file for information on usage and redistribution. -# +import warnings +warnings.warn(f"module {__name__!r} is deprecated", + DeprecationWarning, + stacklevel=2) -"""Internal support module for sre""" - -import _sre -import sre_parse -from sre_constants import * - -assert _sre.MAGIC == MAGIC, "SRE module mismatch" - -_LITERAL_CODES = {LITERAL, NOT_LITERAL} -_REPEATING_CODES = {REPEAT, MIN_REPEAT, MAX_REPEAT} -_SUCCESS_CODES = {SUCCESS, FAILURE} -_ASSERT_CODES = {ASSERT, ASSERT_NOT} -_UNIT_CODES = _LITERAL_CODES | {ANY, IN} - -# Sets of lowercase characters which have the same uppercase. -_equivalences = ( - # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I - (0x69, 0x131), # iı - # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S - (0x73, 0x17f), # sſ - # MICRO SIGN, GREEK SMALL LETTER MU - (0xb5, 0x3bc), # µμ - # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI - (0x345, 0x3b9, 0x1fbe), # \u0345ιι - # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA - (0x390, 0x1fd3), # ΐΐ - # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA - (0x3b0, 0x1fe3), # ΰΰ - # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL - (0x3b2, 0x3d0), # βϐ - # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL - (0x3b5, 0x3f5), # εϵ - # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL - (0x3b8, 0x3d1), # θϑ - # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL - (0x3ba, 0x3f0), # κϰ - # GREEK SMALL LETTER PI, GREEK PI SYMBOL - (0x3c0, 0x3d6), # πϖ - # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL - (0x3c1, 0x3f1), # ρϱ - # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA - (0x3c2, 0x3c3), # ςσ - # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL - (0x3c6, 0x3d5), # φϕ - # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE - (0x1e61, 0x1e9b), # ṡẛ - # LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST - (0xfb05, 0xfb06), # ſtst -) - -# Maps the lowercase code to lowercase codes which have the same uppercase. -_ignorecase_fixes = {i: tuple(j for j in t if i != j) - for t in _equivalences for i in t} - -def _combine_flags(flags, add_flags, del_flags, - TYPE_FLAGS=sre_parse.TYPE_FLAGS): - if add_flags & TYPE_FLAGS: - flags &= ~TYPE_FLAGS - return (flags | add_flags) & ~del_flags - -def _compile(code, pattern, flags): - # internal: compile a (sub)pattern - emit = code.append - _len = len - LITERAL_CODES = _LITERAL_CODES - REPEATING_CODES = _REPEATING_CODES - SUCCESS_CODES = _SUCCESS_CODES - ASSERT_CODES = _ASSERT_CODES - iscased = None - tolower = None - fixes = None - if flags & SRE_FLAG_IGNORECASE and not flags & SRE_FLAG_LOCALE: - if flags & SRE_FLAG_UNICODE: - iscased = _sre.unicode_iscased - tolower = _sre.unicode_tolower - fixes = _ignorecase_fixes - else: - iscased = _sre.ascii_iscased - tolower = _sre.ascii_tolower - for op, av in pattern: - if op in LITERAL_CODES: - if not flags & SRE_FLAG_IGNORECASE: - emit(op) - emit(av) - elif flags & SRE_FLAG_LOCALE: - emit(OP_LOCALE_IGNORE[op]) - emit(av) - elif not iscased(av): - emit(op) - emit(av) - else: - lo = tolower(av) - if not fixes: # ascii - emit(OP_IGNORE[op]) - emit(lo) - elif lo not in fixes: - emit(OP_UNICODE_IGNORE[op]) - emit(lo) - else: - emit(IN_UNI_IGNORE) - skip = _len(code); emit(0) - if op is NOT_LITERAL: - emit(NEGATE) - for k in (lo,) + fixes[lo]: - emit(LITERAL) - emit(k) - emit(FAILURE) - code[skip] = _len(code) - skip - elif op is IN: - charset, hascased = _optimize_charset(av, iscased, tolower, fixes) - if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: - emit(IN_LOC_IGNORE) - elif not hascased: - emit(IN) - elif not fixes: # ascii - emit(IN_IGNORE) - else: - emit(IN_UNI_IGNORE) - skip = _len(code); emit(0) - _compile_charset(charset, flags, code) - code[skip] = _len(code) - skip - elif op is ANY: - if flags & SRE_FLAG_DOTALL: - emit(ANY_ALL) - else: - emit(ANY) - elif op in REPEATING_CODES: - if flags & SRE_FLAG_TEMPLATE: - raise error("internal: unsupported template operator %r" % (op,)) - if _simple(av[2]): - if op is MAX_REPEAT: - emit(REPEAT_ONE) - else: - emit(MIN_REPEAT_ONE) - skip = _len(code); emit(0) - emit(av[0]) - emit(av[1]) - _compile(code, av[2], flags) - emit(SUCCESS) - code[skip] = _len(code) - skip - else: - emit(REPEAT) - skip = _len(code); emit(0) - emit(av[0]) - emit(av[1]) - _compile(code, av[2], flags) - code[skip] = _len(code) - skip - if op is MAX_REPEAT: - emit(MAX_UNTIL) - else: - emit(MIN_UNTIL) - elif op is SUBPATTERN: - group, add_flags, del_flags, p = av - if group: - emit(MARK) - emit((group-1)*2) - # _compile_info(code, p, _combine_flags(flags, add_flags, del_flags)) - _compile(code, p, _combine_flags(flags, add_flags, del_flags)) - if group: - emit(MARK) - emit((group-1)*2+1) - elif op in SUCCESS_CODES: - emit(op) - elif op in ASSERT_CODES: - emit(op) - skip = _len(code); emit(0) - if av[0] >= 0: - emit(0) # look ahead - else: - lo, hi = av[1].getwidth() - if lo != hi: - raise error("look-behind requires fixed-width pattern") - emit(lo) # look behind - _compile(code, av[1], flags) - emit(SUCCESS) - code[skip] = _len(code) - skip - elif op is CALL: - emit(op) - skip = _len(code); emit(0) - _compile(code, av, flags) - emit(SUCCESS) - code[skip] = _len(code) - skip - elif op is AT: - emit(op) - if flags & SRE_FLAG_MULTILINE: - av = AT_MULTILINE.get(av, av) - if flags & SRE_FLAG_LOCALE: - av = AT_LOCALE.get(av, av) - elif flags & SRE_FLAG_UNICODE: - av = AT_UNICODE.get(av, av) - emit(av) - elif op is BRANCH: - emit(op) - tail = [] - tailappend = tail.append - for av in av[1]: - skip = _len(code); emit(0) - # _compile_info(code, av, flags) - _compile(code, av, flags) - emit(JUMP) - tailappend(_len(code)); emit(0) - code[skip] = _len(code) - skip - emit(FAILURE) # end of branch - for tail in tail: - code[tail] = _len(code) - tail - elif op is CATEGORY: - emit(op) - if flags & SRE_FLAG_LOCALE: - av = CH_LOCALE[av] - elif flags & SRE_FLAG_UNICODE: - av = CH_UNICODE[av] - emit(av) - elif op is GROUPREF: - if not flags & SRE_FLAG_IGNORECASE: - emit(op) - elif flags & SRE_FLAG_LOCALE: - emit(GROUPREF_LOC_IGNORE) - elif not fixes: # ascii - emit(GROUPREF_IGNORE) - else: - emit(GROUPREF_UNI_IGNORE) - emit(av-1) - elif op is GROUPREF_EXISTS: - emit(op) - emit(av[0]-1) - skipyes = _len(code); emit(0) - _compile(code, av[1], flags) - if av[2]: - emit(JUMP) - skipno = _len(code); emit(0) - code[skipyes] = _len(code) - skipyes + 1 - _compile(code, av[2], flags) - code[skipno] = _len(code) - skipno - else: - code[skipyes] = _len(code) - skipyes + 1 - else: - raise error("internal: unsupported operand type %r" % (op,)) - -def _compile_charset(charset, flags, code): - # compile charset subprogram - emit = code.append - for op, av in charset: - emit(op) - if op is NEGATE: - pass - elif op is LITERAL: - emit(av) - elif op is RANGE or op is RANGE_UNI_IGNORE: - emit(av[0]) - emit(av[1]) - elif op is CHARSET: - code.extend(av) - elif op is BIGCHARSET: - code.extend(av) - elif op is CATEGORY: - if flags & SRE_FLAG_LOCALE: - emit(CH_LOCALE[av]) - elif flags & SRE_FLAG_UNICODE: - emit(CH_UNICODE[av]) - else: - emit(av) - else: - raise error("internal: unsupported set operator %r" % (op,)) - emit(FAILURE) - -def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): - # internal: optimize character set - out = [] - tail = [] - charmap = bytearray(256) - hascased = False - for op, av in charset: - while True: - try: - if op is LITERAL: - if fixup: - lo = fixup(av) - charmap[lo] = 1 - if fixes and lo in fixes: - for k in fixes[lo]: - charmap[k] = 1 - if not hascased and iscased(av): - hascased = True - else: - charmap[av] = 1 - elif op is RANGE: - r = range(av[0], av[1]+1) - if fixup: - if fixes: - for i in map(fixup, r): - charmap[i] = 1 - if i in fixes: - for k in fixes[i]: - charmap[k] = 1 - else: - for i in map(fixup, r): - charmap[i] = 1 - if not hascased: - hascased = any(map(iscased, r)) - else: - for i in r: - charmap[i] = 1 - elif op is NEGATE: - out.append((op, av)) - else: - tail.append((op, av)) - except IndexError: - if len(charmap) == 256: - # character set contains non-UCS1 character codes - charmap += b'\0' * 0xff00 - continue - # Character set contains non-BMP character codes. - if fixup: - hascased = True - # There are only two ranges of cased non-BMP characters: - # 10400-1044F (Deseret) and 118A0-118DF (Warang Citi), - # and for both ranges RANGE_UNI_IGNORE works. - if op is RANGE: - op = RANGE_UNI_IGNORE - tail.append((op, av)) - break - - # compress character map - runs = [] - q = 0 - while True: - p = charmap.find(1, q) - if p < 0: - break - if len(runs) >= 2: - runs = None - break - q = charmap.find(0, p) - if q < 0: - runs.append((p, len(charmap))) - break - runs.append((p, q)) - if runs is not None: - # use literal/range - for p, q in runs: - if q - p == 1: - out.append((LITERAL, p)) - else: - out.append((RANGE, (p, q - 1))) - out += tail - # if the case was changed or new representation is more compact - if hascased or len(out) < len(charset): - return out, hascased - # else original character set is good enough - return charset, hascased - - # use bitmap - if len(charmap) == 256: - data = _mk_bitmap(charmap) - out.append((CHARSET, data)) - out += tail - return out, hascased - - # To represent a big charset, first a bitmap of all characters in the - # set is constructed. Then, this bitmap is sliced into chunks of 256 - # characters, duplicate chunks are eliminated, and each chunk is - # given a number. In the compiled expression, the charset is - # represented by a 32-bit word sequence, consisting of one word for - # the number of different chunks, a sequence of 256 bytes (64 words) - # of chunk numbers indexed by their original chunk position, and a - # sequence of 256-bit chunks (8 words each). - - # Compression is normally good: in a typical charset, large ranges of - # Unicode will be either completely excluded (e.g. if only cyrillic - # letters are to be matched), or completely included (e.g. if large - # subranges of Kanji match). These ranges will be represented by - # chunks of all one-bits or all zero-bits. - - # Matching can be also done efficiently: the more significant byte of - # the Unicode character is an index into the chunk number, and the - # less significant byte is a bit index in the chunk (just like the - # CHARSET matching). - - charmap = bytes(charmap) # should be hashable - comps = {} - mapping = bytearray(256) - block = 0 - data = bytearray() - for i in range(0, 65536, 256): - chunk = charmap[i: i + 256] - if chunk in comps: - mapping[i // 256] = comps[chunk] - else: - mapping[i // 256] = comps[chunk] = block - block += 1 - data += chunk - data = _mk_bitmap(data) - data[0:0] = [block] + _bytes_to_codes(mapping) - out.append((BIGCHARSET, data)) - out += tail - return out, hascased - -_CODEBITS = _sre.CODESIZE * 8 -MAXCODE = (1 << _CODEBITS) - 1 -_BITS_TRANS = b'0' + b'1' * 255 -def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): - s = bits.translate(_BITS_TRANS)[::-1] - return [_int(s[i - _CODEBITS: i], 2) - for i in range(len(s), 0, -_CODEBITS)] - -def _bytes_to_codes(b): - # Convert block indices to word array - a = memoryview(b).cast('I') - assert a.itemsize == _sre.CODESIZE - assert len(a) * a.itemsize == len(b) - return a.tolist() - -def _simple(p): - # check if this subpattern is a "simple" operator - if len(p) != 1: - return False - op, av = p[0] - if op is SUBPATTERN: - return av[0] is None and _simple(av[-1]) - return op in _UNIT_CODES - -def _generate_overlap_table(prefix): - """ - Generate an overlap table for the following prefix. - An overlap table is a table of the same size as the prefix which - informs about the potential self-overlap for each index in the prefix: - - if overlap[i] == 0, prefix[i:] can't overlap prefix[0:...] - - if overlap[i] == k with 0 < k <= i, prefix[i-k+1:i+1] overlaps with - prefix[0:k] - """ - table = [0] * len(prefix) - for i in range(1, len(prefix)): - idx = table[i - 1] - while prefix[i] != prefix[idx]: - if idx == 0: - table[i] = 0 - break - idx = table[idx - 1] - else: - table[i] = idx + 1 - return table - -def _get_iscased(flags): - if not flags & SRE_FLAG_IGNORECASE: - return None - elif flags & SRE_FLAG_UNICODE: - return _sre.unicode_iscased - else: - return _sre.ascii_iscased - -def _get_literal_prefix(pattern, flags): - # look for literal prefix - prefix = [] - prefixappend = prefix.append - prefix_skip = None - iscased = _get_iscased(flags) - for op, av in pattern.data: - if op is LITERAL: - if iscased and iscased(av): - break - prefixappend(av) - elif op is SUBPATTERN: - group, add_flags, del_flags, p = av - flags1 = _combine_flags(flags, add_flags, del_flags) - if flags1 & SRE_FLAG_IGNORECASE and flags1 & SRE_FLAG_LOCALE: - break - prefix1, prefix_skip1, got_all = _get_literal_prefix(p, flags1) - if prefix_skip is None: - if group is not None: - prefix_skip = len(prefix) - elif prefix_skip1 is not None: - prefix_skip = len(prefix) + prefix_skip1 - prefix.extend(prefix1) - if not got_all: - break - else: - break - else: - return prefix, prefix_skip, True - return prefix, prefix_skip, False - -def _get_charset_prefix(pattern, flags): - while True: - if not pattern.data: - return None - op, av = pattern.data[0] - if op is not SUBPATTERN: - break - group, add_flags, del_flags, pattern = av - flags = _combine_flags(flags, add_flags, del_flags) - if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: - return None - - iscased = _get_iscased(flags) - if op is LITERAL: - if iscased and iscased(av): - return None - return [(op, av)] - elif op is BRANCH: - charset = [] - charsetappend = charset.append - for p in av[1]: - if not p: - return None - op, av = p[0] - if op is LITERAL and not (iscased and iscased(av)): - charsetappend((op, av)) - else: - return None - return charset - elif op is IN: - charset = av - if iscased: - for op, av in charset: - if op is LITERAL: - if iscased(av): - return None - elif op is RANGE: - if av[1] > 0xffff: - return None - if any(map(iscased, range(av[0], av[1]+1))): - return None - return charset - return None - -def _compile_info(code, pattern, flags): - # internal: compile an info block. in the current version, - # this contains min/max pattern width, and an optional literal - # prefix or a character map - lo, hi = pattern.getwidth() - if hi > MAXCODE: - hi = MAXCODE - if lo == 0: - code.extend([INFO, 4, 0, lo, hi]) - return - # look for a literal prefix - prefix = [] - prefix_skip = 0 - charset = [] # not used - if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): - # look for literal prefix - prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) - # if no prefix, look for charset prefix - if not prefix: - charset = _get_charset_prefix(pattern, flags) -## if prefix: -## print("*** PREFIX", prefix, prefix_skip) -## if charset: -## print("*** CHARSET", charset) - # add an info block - emit = code.append - emit(INFO) - skip = len(code); emit(0) - # literal flag - mask = 0 - if prefix: - mask = SRE_INFO_PREFIX - if prefix_skip is None and got_all: - mask = mask | SRE_INFO_LITERAL - elif charset: - mask = mask | SRE_INFO_CHARSET - emit(mask) - # pattern length - if lo < MAXCODE: - emit(lo) - else: - emit(MAXCODE) - prefix = prefix[:MAXCODE] - emit(min(hi, MAXCODE)) - # add literal prefix - if prefix: - emit(len(prefix)) # length - if prefix_skip is None: - prefix_skip = len(prefix) - emit(prefix_skip) # skip - code.extend(prefix) - # generate overlap table - code.extend(_generate_overlap_table(prefix)) - elif charset: - charset, hascased = _optimize_charset(charset) - assert not hascased - _compile_charset(charset, flags, code) - code[skip] = len(code) - skip - -def isstring(obj): - return isinstance(obj, (str, bytes)) - -def _code(p, flags): - - flags = p.state.flags | flags - code = [] - - # compile info block - _compile_info(code, p, flags) - - # compile the pattern - _compile(code, p.data, flags) - - code.append(SUCCESS) - - return code - -def _hex_code(code): - return '[%s]' % ', '.join('%#0*x' % (_sre.CODESIZE*2+2, x) for x in code) - -def dis(code): - import sys - - labels = set() - level = 0 - offset_width = len(str(len(code) - 1)) - - def dis_(start, end): - def print_(*args, to=None): - if to is not None: - labels.add(to) - args += ('(to %d)' % (to,),) - print('%*d%s ' % (offset_width, start, ':' if start in labels else '.'), - end=' '*(level-1)) - print(*args) - - def print_2(*args): - print(end=' '*(offset_width + 2*level)) - print(*args) - - nonlocal level - level += 1 - i = start - while i < end: - start = i - op = code[i] - i += 1 - op = OPCODES[op] - if op in (SUCCESS, FAILURE, ANY, ANY_ALL, - MAX_UNTIL, MIN_UNTIL, NEGATE): - print_(op) - elif op in (LITERAL, NOT_LITERAL, - LITERAL_IGNORE, NOT_LITERAL_IGNORE, - LITERAL_UNI_IGNORE, NOT_LITERAL_UNI_IGNORE, - LITERAL_LOC_IGNORE, NOT_LITERAL_LOC_IGNORE): - arg = code[i] - i += 1 - print_(op, '%#02x (%r)' % (arg, chr(arg))) - elif op is AT: - arg = code[i] - i += 1 - arg = str(ATCODES[arg]) - assert arg[:3] == 'AT_' - print_(op, arg[3:]) - elif op is CATEGORY: - arg = code[i] - i += 1 - arg = str(CHCODES[arg]) - assert arg[:9] == 'CATEGORY_' - print_(op, arg[9:]) - elif op in (IN, IN_IGNORE, IN_UNI_IGNORE, IN_LOC_IGNORE): - skip = code[i] - print_(op, skip, to=i+skip) - dis_(i+1, i+skip) - i += skip - elif op in (RANGE, RANGE_UNI_IGNORE): - lo, hi = code[i: i+2] - i += 2 - print_(op, '%#02x %#02x (%r-%r)' % (lo, hi, chr(lo), chr(hi))) - elif op is CHARSET: - print_(op, _hex_code(code[i: i + 256//_CODEBITS])) - i += 256//_CODEBITS - elif op is BIGCHARSET: - arg = code[i] - i += 1 - mapping = list(b''.join(x.to_bytes(_sre.CODESIZE, sys.byteorder) - for x in code[i: i + 256//_sre.CODESIZE])) - print_(op, arg, mapping) - i += 256//_sre.CODESIZE - level += 1 - for j in range(arg): - print_2(_hex_code(code[i: i + 256//_CODEBITS])) - i += 256//_CODEBITS - level -= 1 - elif op in (MARK, GROUPREF, GROUPREF_IGNORE, GROUPREF_UNI_IGNORE, - GROUPREF_LOC_IGNORE): - arg = code[i] - i += 1 - print_(op, arg) - elif op is JUMP: - skip = code[i] - print_(op, skip, to=i+skip) - i += 1 - elif op is BRANCH: - skip = code[i] - print_(op, skip, to=i+skip) - while skip: - dis_(i+1, i+skip) - i += skip - start = i - skip = code[i] - if skip: - print_('branch', skip, to=i+skip) - else: - print_(FAILURE) - i += 1 - elif op in (REPEAT, REPEAT_ONE, MIN_REPEAT_ONE): - skip, min, max = code[i: i+3] - if max == MAXREPEAT: - max = 'MAXREPEAT' - print_(op, skip, min, max, to=i+skip) - dis_(i+3, i+skip) - i += skip - elif op is GROUPREF_EXISTS: - arg, skip = code[i: i+2] - print_(op, arg, skip, to=i+skip) - i += 2 - elif op in (ASSERT, ASSERT_NOT): - skip, arg = code[i: i+2] - print_(op, skip, arg, to=i+skip) - dis_(i+2, i+skip) - i += skip - elif op is INFO: - skip, flags, min, max = code[i: i+4] - if max == MAXREPEAT: - max = 'MAXREPEAT' - print_(op, skip, bin(flags), min, max, to=i+skip) - start = i+4 - if flags & SRE_INFO_PREFIX: - prefix_len, prefix_skip = code[i+4: i+6] - print_2(' prefix_skip', prefix_skip) - start = i + 6 - prefix = code[start: start+prefix_len] - print_2(' prefix', - '[%s]' % ', '.join('%#02x' % x for x in prefix), - '(%r)' % ''.join(map(chr, prefix))) - start += prefix_len - print_2(' overlap', code[start: start+prefix_len]) - start += prefix_len - if flags & SRE_INFO_CHARSET: - level += 1 - print_2('in') - dis_(start, i+skip) - level -= 1 - i += skip - else: - raise ValueError(op) - - level -= 1 - - dis_(0, len(code)) - - -def compile(p, flags=0): - # internal: convert pattern list to internal format - - if isstring(p): - pattern = p - p = sre_parse.parse(p, flags) - else: - pattern = None - - code = _code(p, flags) - - if flags & SRE_FLAG_DEBUG: - print() - dis(code) - - # map in either direction - groupindex = p.state.groupdict - indexgroup = [None] * p.state.groups - for k, i in groupindex.items(): - indexgroup[i] = k - - return _sre.compile( - pattern, flags | p.state.flags, code, - p.state.groups-1, - groupindex, tuple(indexgroup) - ) +from re import _compiler as _ +globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'}) diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py index 8360acb6957..fa09d044292 100644 --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -1,293 +1,7 @@ -# -# Secret Labs' Regular Expression Engine -# -# various symbols used by the regular expression engine. -# run this script to update the _sre include files! -# -# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. -# -# See the sre.py file for information on usage and redistribution. -# +import warnings +warnings.warn(f"module {__name__!r} is deprecated", + DeprecationWarning, + stacklevel=2) -"""Internal support module for sre""" - -# update when constants are added or removed - -MAGIC = 20171005 - -from _sre import MAXREPEAT, MAXGROUPS - -# SRE standard exception (access as sre.error) -# should this really be here? - -class error(Exception): - """Exception raised for invalid regular expressions. - - Attributes: - - msg: The unformatted error message - pattern: The regular expression pattern - pos: The index in the pattern where compilation failed (may be None) - lineno: The line corresponding to pos (may be None) - colno: The column corresponding to pos (may be None) - """ - - __module__ = 're' - - def __init__(self, msg, pattern=None, pos=None): - self.msg = msg - self.pattern = pattern - self.pos = pos - if pattern is not None and pos is not None: - msg = '%s at position %d' % (msg, pos) - if isinstance(pattern, str): - newline = '\n' - else: - newline = b'\n' - self.lineno = pattern.count(newline, 0, pos) + 1 - self.colno = pos - pattern.rfind(newline, 0, pos) - if newline in pattern: - msg = '%s (line %d, column %d)' % (msg, self.lineno, self.colno) - else: - self.lineno = self.colno = None - super().__init__(msg) - - -class _NamedIntConstant(int): - def __new__(cls, value, name): - self = super(_NamedIntConstant, cls).__new__(cls, value) - self.name = name - return self - - def __repr__(self): - return self.name - -MAXREPEAT = _NamedIntConstant(MAXREPEAT, 'MAXREPEAT') - -def _makecodes(names): - names = names.strip().split() - items = [_NamedIntConstant(i, name) for i, name in enumerate(names)] - globals().update({item.name: item for item in items}) - return items - -# operators -# failure=0 success=1 (just because it looks better that way :-) -OPCODES = _makecodes(""" - FAILURE SUCCESS - - ANY ANY_ALL - ASSERT ASSERT_NOT - AT - BRANCH - CALL - CATEGORY - CHARSET BIGCHARSET - GROUPREF GROUPREF_EXISTS - IN - INFO - JUMP - LITERAL - MARK - MAX_UNTIL - MIN_UNTIL - NOT_LITERAL - NEGATE - RANGE - REPEAT - REPEAT_ONE - SUBPATTERN - MIN_REPEAT_ONE - - GROUPREF_IGNORE - IN_IGNORE - LITERAL_IGNORE - NOT_LITERAL_IGNORE - - GROUPREF_LOC_IGNORE - IN_LOC_IGNORE - LITERAL_LOC_IGNORE - NOT_LITERAL_LOC_IGNORE - - GROUPREF_UNI_IGNORE - IN_UNI_IGNORE - LITERAL_UNI_IGNORE - NOT_LITERAL_UNI_IGNORE - RANGE_UNI_IGNORE - - MIN_REPEAT MAX_REPEAT -""") -del OPCODES[-2:] # remove MIN_REPEAT and MAX_REPEAT - -# positions -ATCODES = _makecodes(""" - AT_BEGINNING AT_BEGINNING_LINE AT_BEGINNING_STRING - AT_BOUNDARY AT_NON_BOUNDARY - AT_END AT_END_LINE AT_END_STRING - - AT_LOC_BOUNDARY AT_LOC_NON_BOUNDARY - - AT_UNI_BOUNDARY AT_UNI_NON_BOUNDARY -""") - -# categories -CHCODES = _makecodes(""" - CATEGORY_DIGIT CATEGORY_NOT_DIGIT - CATEGORY_SPACE CATEGORY_NOT_SPACE - CATEGORY_WORD CATEGORY_NOT_WORD - CATEGORY_LINEBREAK CATEGORY_NOT_LINEBREAK - - CATEGORY_LOC_WORD CATEGORY_LOC_NOT_WORD - - CATEGORY_UNI_DIGIT CATEGORY_UNI_NOT_DIGIT - CATEGORY_UNI_SPACE CATEGORY_UNI_NOT_SPACE - CATEGORY_UNI_WORD CATEGORY_UNI_NOT_WORD - CATEGORY_UNI_LINEBREAK CATEGORY_UNI_NOT_LINEBREAK -""") - - -# replacement operations for "ignore case" mode -OP_IGNORE = { - LITERAL: LITERAL_IGNORE, - NOT_LITERAL: NOT_LITERAL_IGNORE, -} - -OP_LOCALE_IGNORE = { - LITERAL: LITERAL_LOC_IGNORE, - NOT_LITERAL: NOT_LITERAL_LOC_IGNORE, -} - -OP_UNICODE_IGNORE = { - LITERAL: LITERAL_UNI_IGNORE, - NOT_LITERAL: NOT_LITERAL_UNI_IGNORE, -} - -AT_MULTILINE = { - AT_BEGINNING: AT_BEGINNING_LINE, - AT_END: AT_END_LINE -} - -AT_LOCALE = { - AT_BOUNDARY: AT_LOC_BOUNDARY, - AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY -} - -AT_UNICODE = { - AT_BOUNDARY: AT_UNI_BOUNDARY, - AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY -} - -CH_LOCALE = { - CATEGORY_DIGIT: CATEGORY_DIGIT, - CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT, - CATEGORY_SPACE: CATEGORY_SPACE, - CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE, - CATEGORY_WORD: CATEGORY_LOC_WORD, - CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD, - CATEGORY_LINEBREAK: CATEGORY_LINEBREAK, - CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK -} - -CH_UNICODE = { - CATEGORY_DIGIT: CATEGORY_UNI_DIGIT, - CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT, - CATEGORY_SPACE: CATEGORY_UNI_SPACE, - CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE, - CATEGORY_WORD: CATEGORY_UNI_WORD, - CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD, - CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK, - CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK -} - -# flags -SRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking) -SRE_FLAG_IGNORECASE = 2 # case insensitive -SRE_FLAG_LOCALE = 4 # honour system locale -SRE_FLAG_MULTILINE = 8 # treat target as multiline string -SRE_FLAG_DOTALL = 16 # treat target as a single string -SRE_FLAG_UNICODE = 32 # use unicode "locale" -SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments -SRE_FLAG_DEBUG = 128 # debugging -SRE_FLAG_ASCII = 256 # use ascii "locale" - -# flags for INFO primitive -SRE_INFO_PREFIX = 1 # has prefix -SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix) -SRE_INFO_CHARSET = 4 # pattern starts with character from given set - -if __name__ == "__main__": - def dump(f, d, typ, int_t, prefix): - items = sorted(d) - f.write(f"""\ -#[derive(num_enum::TryFromPrimitive, Debug)] -#[repr({int_t})] -#[allow(non_camel_case_types, clippy::upper_case_acronyms)] -pub enum {typ} {{ -""") - for item in items: - name = str(item).removeprefix(prefix) - val = int(item) - f.write(f" {name} = {val},\n") - f.write("""\ -} -""") - import sys - if len(sys.argv) > 1: - constants_file = sys.argv[1] - else: - import os - constants_file = os.path.join(os.path.dirname(__file__), "../../sre-engine/src/constants.rs") - with open(constants_file, "w") as f: - f.write("""\ -/* - * Secret Labs' Regular Expression Engine - * - * regular expression matching engine - * - * NOTE: This file is generated by sre_constants.py. If you need - * to change anything in here, edit sre_constants.py and run it. - * - * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. - * - * See the _sre.c file for information on usage and redistribution. - */ - -""") - - f.write("use bitflags::bitflags;\n\n"); - - f.write("pub const SRE_MAGIC: usize = %d;\n" % MAGIC) - - dump(f, OPCODES, "SreOpcode", "u32", "") - dump(f, ATCODES, "SreAtCode", "u32", "AT_") - dump(f, CHCODES, "SreCatCode", "u32", "CATEGORY_") - - def bitflags(typ, int_t, prefix, flags): - f.write(f"""\ -bitflags! {{ - pub struct {typ}: {int_t} {{ -""") - for name in flags: - val = globals()[prefix + name] - f.write(f" const {name} = {val};\n") - f.write("""\ - } -} -""") - - bitflags("SreFlag", "u16", "SRE_FLAG_", [ - "TEMPLATE", - "IGNORECASE", - "LOCALE", - "MULTILINE", - "DOTALL", - "UNICODE", - "VERBOSE", - "DEBUG", - "ASCII", - ]) - - bitflags("SreInfo", "u32", "SRE_INFO_", [ - "PREFIX", "LITERAL", "CHARSET", - ]) - - print("done") +from re import _constants as _ +globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'}) diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 83119168e63..25a3f557d44 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -1,1064 +1,7 @@ -# -# Secret Labs' Regular Expression Engine -# -# convert re-style regular expression to sre pattern -# -# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. -# -# See the sre.py file for information on usage and redistribution. -# +import warnings +warnings.warn(f"module {__name__!r} is deprecated", + DeprecationWarning, + stacklevel=2) -"""Internal support module for sre""" - -# XXX: show string offset and offending character for all errors - -from sre_constants import * - -SPECIAL_CHARS = ".\\[{()*+?^$|" -REPEAT_CHARS = "*+?{" - -DIGITS = frozenset("0123456789") - -OCTDIGITS = frozenset("01234567") -HEXDIGITS = frozenset("0123456789abcdefABCDEF") -ASCIILETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -WHITESPACE = frozenset(" \t\n\r\v\f") - -_REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT}) -_UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY}) - -ESCAPES = { - r"\a": (LITERAL, ord("\a")), - r"\b": (LITERAL, ord("\b")), - r"\f": (LITERAL, ord("\f")), - r"\n": (LITERAL, ord("\n")), - r"\r": (LITERAL, ord("\r")), - r"\t": (LITERAL, ord("\t")), - r"\v": (LITERAL, ord("\v")), - r"\\": (LITERAL, ord("\\")) -} - -CATEGORIES = { - r"\A": (AT, AT_BEGINNING_STRING), # start of string - r"\b": (AT, AT_BOUNDARY), - r"\B": (AT, AT_NON_BOUNDARY), - r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]), - r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]), - r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]), - r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), - r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), - r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), - r"\Z": (AT, AT_END_STRING), # end of string -} - -FLAGS = { - # standard flags - "i": SRE_FLAG_IGNORECASE, - "L": SRE_FLAG_LOCALE, - "m": SRE_FLAG_MULTILINE, - "s": SRE_FLAG_DOTALL, - "x": SRE_FLAG_VERBOSE, - # extensions - "a": SRE_FLAG_ASCII, - "t": SRE_FLAG_TEMPLATE, - "u": SRE_FLAG_UNICODE, -} - -TYPE_FLAGS = SRE_FLAG_ASCII | SRE_FLAG_LOCALE | SRE_FLAG_UNICODE -GLOBAL_FLAGS = SRE_FLAG_DEBUG | SRE_FLAG_TEMPLATE - -class Verbose(Exception): - pass - -class State: - # keeps track of state for parsing - def __init__(self): - self.flags = 0 - self.groupdict = {} - self.groupwidths = [None] # group 0 - self.lookbehindgroups = None - @property - def groups(self): - return len(self.groupwidths) - def opengroup(self, name=None): - gid = self.groups - self.groupwidths.append(None) - if self.groups > MAXGROUPS: - raise error("too many groups") - if name is not None: - ogid = self.groupdict.get(name, None) - if ogid is not None: - raise error("redefinition of group name %r as group %d; " - "was group %d" % (name, gid, ogid)) - self.groupdict[name] = gid - return gid - def closegroup(self, gid, p): - self.groupwidths[gid] = p.getwidth() - def checkgroup(self, gid): - return gid < self.groups and self.groupwidths[gid] is not None - - def checklookbehindgroup(self, gid, source): - if self.lookbehindgroups is not None: - if not self.checkgroup(gid): - raise source.error('cannot refer to an open group') - if gid >= self.lookbehindgroups: - raise source.error('cannot refer to group defined in the same ' - 'lookbehind subpattern') - -class SubPattern: - # a subpattern, in intermediate form - def __init__(self, state, data=None): - self.state = state - if data is None: - data = [] - self.data = data - self.width = None - - def dump(self, level=0): - nl = True - seqtypes = (tuple, list) - for op, av in self.data: - print(level*" " + str(op), end='') - if op is IN: - # member sublanguage - print() - for op, a in av: - print((level+1)*" " + str(op), a) - elif op is BRANCH: - print() - for i, a in enumerate(av[1]): - if i: - print(level*" " + "OR") - a.dump(level+1) - elif op is GROUPREF_EXISTS: - condgroup, item_yes, item_no = av - print('', condgroup) - item_yes.dump(level+1) - if item_no: - print(level*" " + "ELSE") - item_no.dump(level+1) - elif isinstance(av, seqtypes): - nl = False - for a in av: - if isinstance(a, SubPattern): - if not nl: - print() - a.dump(level+1) - nl = True - else: - if not nl: - print(' ', end='') - print(a, end='') - nl = False - if not nl: - print() - else: - print('', av) - def __repr__(self): - return repr(self.data) - def __len__(self): - return len(self.data) - def __delitem__(self, index): - del self.data[index] - def __getitem__(self, index): - if isinstance(index, slice): - return SubPattern(self.state, self.data[index]) - return self.data[index] - def __setitem__(self, index, code): - self.data[index] = code - def insert(self, index, code): - self.data.insert(index, code) - def append(self, code): - self.data.append(code) - def getwidth(self): - # determine the width (min, max) for this subpattern - if self.width is not None: - return self.width - lo = hi = 0 - for op, av in self.data: - if op is BRANCH: - i = MAXREPEAT - 1 - j = 0 - for av in av[1]: - l, h = av.getwidth() - i = min(i, l) - j = max(j, h) - lo = lo + i - hi = hi + j - elif op is CALL: - i, j = av.getwidth() - lo = lo + i - hi = hi + j - elif op is SUBPATTERN: - i, j = av[-1].getwidth() - lo = lo + i - hi = hi + j - elif op in _REPEATCODES: - i, j = av[2].getwidth() - lo = lo + i * av[0] - hi = hi + j * av[1] - elif op in _UNITCODES: - lo = lo + 1 - hi = hi + 1 - elif op is GROUPREF: - i, j = self.state.groupwidths[av] - lo = lo + i - hi = hi + j - elif op is GROUPREF_EXISTS: - i, j = av[1].getwidth() - if av[2] is not None: - l, h = av[2].getwidth() - i = min(i, l) - j = max(j, h) - else: - i = 0 - lo = lo + i - hi = hi + j - elif op is SUCCESS: - break - self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT) - return self.width - -class Tokenizer: - def __init__(self, string): - self.istext = isinstance(string, str) - self.string = string - if not self.istext: - string = str(string, 'latin1') - self.decoded_string = string - self.index = 0 - self.next = None - self.__next() - def __next(self): - index = self.index - try: - char = self.decoded_string[index] - except IndexError: - self.next = None - return - if char == "\\": - index += 1 - try: - char += self.decoded_string[index] - except IndexError: - raise error("bad escape (end of pattern)", - self.string, len(self.string) - 1) from None - self.index = index + 1 - self.next = char - def match(self, char): - if char == self.next: - self.__next() - return True - return False - def get(self): - this = self.next - self.__next() - return this - def getwhile(self, n, charset): - result = '' - for _ in range(n): - c = self.next - if c not in charset: - break - result += c - self.__next() - return result - def getuntil(self, terminator, name): - result = '' - while True: - c = self.next - self.__next() - if c is None: - if not result: - raise self.error("missing " + name) - raise self.error("missing %s, unterminated name" % terminator, - len(result)) - if c == terminator: - if not result: - raise self.error("missing " + name, 1) - break - result += c - return result - @property - def pos(self): - return self.index - len(self.next or '') - def tell(self): - return self.index - len(self.next or '') - def seek(self, index): - self.index = index - self.__next() - - def error(self, msg, offset=0): - return error(msg, self.string, self.tell() - offset) - -def _class_escape(source, escape): - # handle escape code inside character class - code = ESCAPES.get(escape) - if code: - return code - code = CATEGORIES.get(escape) - if code and code[0] is IN: - return code - try: - c = escape[1:2] - if c == "x": - # hexadecimal escape (exactly two digits) - escape += source.getwhile(2, HEXDIGITS) - if len(escape) != 4: - raise source.error("incomplete escape %s" % escape, len(escape)) - return LITERAL, int(escape[2:], 16) - elif c == "u" and source.istext: - # unicode escape (exactly four digits) - escape += source.getwhile(4, HEXDIGITS) - if len(escape) != 6: - raise source.error("incomplete escape %s" % escape, len(escape)) - return LITERAL, int(escape[2:], 16) - elif c == "U" and source.istext: - # unicode escape (exactly eight digits) - escape += source.getwhile(8, HEXDIGITS) - if len(escape) != 10: - raise source.error("incomplete escape %s" % escape, len(escape)) - c = int(escape[2:], 16) - chr(c) # raise ValueError for invalid code - return LITERAL, c - elif c == "N" and source.istext: - import unicodedata - # named unicode escape e.g. \N{EM DASH} - if not source.match('{'): - raise source.error("missing {") - charname = source.getuntil('}', 'character name') - try: - c = ord(unicodedata.lookup(charname)) - except KeyError: - raise source.error("undefined character name %r" % charname, - len(charname) + len(r'\N{}')) - return LITERAL, c - elif c in OCTDIGITS: - # octal escape (up to three digits) - escape += source.getwhile(2, OCTDIGITS) - c = int(escape[1:], 8) - if c > 0o377: - raise source.error('octal escape value %s outside of ' - 'range 0-0o377' % escape, len(escape)) - return LITERAL, c - elif c in DIGITS: - raise ValueError - if len(escape) == 2: - if c in ASCIILETTERS: - raise source.error('bad escape %s' % escape, len(escape)) - return LITERAL, ord(escape[1]) - except ValueError: - pass - raise source.error("bad escape %s" % escape, len(escape)) - -def _escape(source, escape, state): - # handle escape code in expression - code = CATEGORIES.get(escape) - if code: - return code - code = ESCAPES.get(escape) - if code: - return code - try: - c = escape[1:2] - if c == "x": - # hexadecimal escape - escape += source.getwhile(2, HEXDIGITS) - if len(escape) != 4: - raise source.error("incomplete escape %s" % escape, len(escape)) - return LITERAL, int(escape[2:], 16) - elif c == "u" and source.istext: - # unicode escape (exactly four digits) - escape += source.getwhile(4, HEXDIGITS) - if len(escape) != 6: - raise source.error("incomplete escape %s" % escape, len(escape)) - return LITERAL, int(escape[2:], 16) - elif c == "U" and source.istext: - # unicode escape (exactly eight digits) - escape += source.getwhile(8, HEXDIGITS) - if len(escape) != 10: - raise source.error("incomplete escape %s" % escape, len(escape)) - c = int(escape[2:], 16) - chr(c) # raise ValueError for invalid code - return LITERAL, c - elif c == "N" and source.istext: - import unicodedata - # named unicode escape e.g. \N{EM DASH} - if not source.match('{'): - raise source.error("missing {") - charname = source.getuntil('}', 'character name') - try: - c = ord(unicodedata.lookup(charname)) - except KeyError: - raise source.error("undefined character name %r" % charname, - len(charname) + len(r'\N{}')) - return LITERAL, c - elif c == "0": - # octal escape - escape += source.getwhile(2, OCTDIGITS) - return LITERAL, int(escape[1:], 8) - elif c in DIGITS: - # octal escape *or* decimal group reference (sigh) - if source.next in DIGITS: - escape += source.get() - if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and - source.next in OCTDIGITS): - # got three octal digits; this is an octal escape - escape += source.get() - c = int(escape[1:], 8) - if c > 0o377: - raise source.error('octal escape value %s outside of ' - 'range 0-0o377' % escape, - len(escape)) - return LITERAL, c - # not an octal escape, so this is a group reference - group = int(escape[1:]) - if group < state.groups: - if not state.checkgroup(group): - raise source.error("cannot refer to an open group", - len(escape)) - state.checklookbehindgroup(group, source) - return GROUPREF, group - raise source.error("invalid group reference %d" % group, len(escape) - 1) - if len(escape) == 2: - if c in ASCIILETTERS: - raise source.error("bad escape %s" % escape, len(escape)) - return LITERAL, ord(escape[1]) - except ValueError: - pass - raise source.error("bad escape %s" % escape, len(escape)) - -def _uniq(items): - return list(dict.fromkeys(items)) - -def _parse_sub(source, state, verbose, nested): - # parse an alternation: a|b|c - - items = [] - itemsappend = items.append - sourcematch = source.match - start = source.tell() - while True: - itemsappend(_parse(source, state, verbose, nested + 1, - not nested and not items)) - if not sourcematch("|"): - break - - if len(items) == 1: - return items[0] - - subpattern = SubPattern(state) - - # check if all items share a common prefix - while True: - prefix = None - for item in items: - if not item: - break - if prefix is None: - prefix = item[0] - elif item[0] != prefix: - break - else: - # all subitems start with a common "prefix". - # move it out of the branch - for item in items: - del item[0] - subpattern.append(prefix) - continue # check next one - break - - # check if the branch can be replaced by a character set - set = [] - for item in items: - if len(item) != 1: - break - op, av = item[0] - if op is LITERAL: - set.append((op, av)) - elif op is IN and av[0][0] is not NEGATE: - set.extend(av) - else: - break - else: - # we can store this as a character set instead of a - # branch (the compiler may optimize this even more) - subpattern.append((IN, _uniq(set))) - return subpattern - - subpattern.append((BRANCH, (None, items))) - return subpattern - -def _parse(source, state, verbose, nested, first=False): - # parse a simple pattern - subpattern = SubPattern(state) - - # precompute constants into local variables - subpatternappend = subpattern.append - sourceget = source.get - sourcematch = source.match - _len = len - _ord = ord - - while True: - - this = source.next - if this is None: - break # end of pattern - if this in "|)": - break # end of subpattern - sourceget() - - if verbose: - # skip whitespace and comments - if this in WHITESPACE: - continue - if this == "#": - while True: - this = sourceget() - if this is None or this == "\n": - break - continue - - if this[0] == "\\": - code = _escape(source, this, state) - subpatternappend(code) - - elif this not in SPECIAL_CHARS: - subpatternappend((LITERAL, _ord(this))) - - elif this == "[": - here = source.tell() - 1 - # character set - set = [] - setappend = set.append -## if sourcematch(":"): -## pass # handle character classes - if source.next == '[': - import warnings - warnings.warn( - 'Possible nested set at position %d' % source.tell(), - FutureWarning, stacklevel=nested + 6 - ) - negate = sourcematch("^") - # check remaining characters - while True: - this = sourceget() - if this is None: - raise source.error("unterminated character set", - source.tell() - here) - if this == "]" and set: - break - elif this[0] == "\\": - code1 = _class_escape(source, this) - else: - if set and this in '-&~|' and source.next == this: - import warnings - warnings.warn( - 'Possible set %s at position %d' % ( - 'difference' if this == '-' else - 'intersection' if this == '&' else - 'symmetric difference' if this == '~' else - 'union', - source.tell() - 1), - FutureWarning, stacklevel=nested + 6 - ) - code1 = LITERAL, _ord(this) - if sourcematch("-"): - # potential range - that = sourceget() - if that is None: - raise source.error("unterminated character set", - source.tell() - here) - if that == "]": - if code1[0] is IN: - code1 = code1[1][0] - setappend(code1) - setappend((LITERAL, _ord("-"))) - break - if that[0] == "\\": - code2 = _class_escape(source, that) - else: - if that == '-': - import warnings - warnings.warn( - 'Possible set difference at position %d' % ( - source.tell() - 2), - FutureWarning, stacklevel=nested + 6 - ) - code2 = LITERAL, _ord(that) - if code1[0] != LITERAL or code2[0] != LITERAL: - msg = "bad character range %s-%s" % (this, that) - raise source.error(msg, len(this) + 1 + len(that)) - lo = code1[1] - hi = code2[1] - if hi < lo: - msg = "bad character range %s-%s" % (this, that) - raise source.error(msg, len(this) + 1 + len(that)) - setappend((RANGE, (lo, hi))) - else: - if code1[0] is IN: - code1 = code1[1][0] - setappend(code1) - - set = _uniq(set) - # XXX: should move set optimization to compiler! - if _len(set) == 1 and set[0][0] is LITERAL: - # optimization - if negate: - subpatternappend((NOT_LITERAL, set[0][1])) - else: - subpatternappend(set[0]) - else: - if negate: - set.insert(0, (NEGATE, None)) - # charmap optimization can't be added here because - # global flags still are not known - subpatternappend((IN, set)) - - elif this in REPEAT_CHARS: - # repeat previous item - here = source.tell() - if this == "?": - min, max = 0, 1 - elif this == "*": - min, max = 0, MAXREPEAT - - elif this == "+": - min, max = 1, MAXREPEAT - elif this == "{": - if source.next == "}": - subpatternappend((LITERAL, _ord(this))) - continue - - min, max = 0, MAXREPEAT - lo = hi = "" - while source.next in DIGITS: - lo += sourceget() - if sourcematch(","): - while source.next in DIGITS: - hi += sourceget() - else: - hi = lo - if not sourcematch("}"): - subpatternappend((LITERAL, _ord(this))) - source.seek(here) - continue - - if lo: - min = int(lo) - if min >= MAXREPEAT: - raise OverflowError("the repetition number is too large") - if hi: - max = int(hi) - if max >= MAXREPEAT: - raise OverflowError("the repetition number is too large") - if max < min: - raise source.error("min repeat greater than max repeat", - source.tell() - here) - else: - raise AssertionError("unsupported quantifier %r" % (char,)) - # figure out which item to repeat - if subpattern: - item = subpattern[-1:] - else: - item = None - if not item or item[0][0] is AT: - raise source.error("nothing to repeat", - source.tell() - here + len(this)) - if item[0][0] in _REPEATCODES: - raise source.error("multiple repeat", - source.tell() - here + len(this)) - if item[0][0] is SUBPATTERN: - group, add_flags, del_flags, p = item[0][1] - if group is None and not add_flags and not del_flags: - item = p - if sourcematch("?"): - subpattern[-1] = (MIN_REPEAT, (min, max, item)) - else: - subpattern[-1] = (MAX_REPEAT, (min, max, item)) - - elif this == ".": - subpatternappend((ANY, None)) - - elif this == "(": - start = source.tell() - 1 - group = True - name = None - add_flags = 0 - del_flags = 0 - if sourcematch("?"): - # options - char = sourceget() - if char is None: - raise source.error("unexpected end of pattern") - if char == "P": - # python extensions - if sourcematch("<"): - # named group: skip forward to end of name - name = source.getuntil(">", "group name") - if not name.isidentifier(): - msg = "bad character in group name %r" % name - raise source.error(msg, len(name) + 1) - elif sourcematch("="): - # named backreference - name = source.getuntil(")", "group name") - if not name.isidentifier(): - msg = "bad character in group name %r" % name - raise source.error(msg, len(name) + 1) - gid = state.groupdict.get(name) - if gid is None: - msg = "unknown group name %r" % name - raise source.error(msg, len(name) + 1) - if not state.checkgroup(gid): - raise source.error("cannot refer to an open group", - len(name) + 1) - state.checklookbehindgroup(gid, source) - subpatternappend((GROUPREF, gid)) - continue - - else: - char = sourceget() - if char is None: - raise source.error("unexpected end of pattern") - raise source.error("unknown extension ?P" + char, - len(char) + 2) - elif char == ":": - # non-capturing group - group = None - elif char == "#": - # comment - while True: - if source.next is None: - raise source.error("missing ), unterminated comment", - source.tell() - start) - if sourceget() == ")": - break - continue - - elif char in "=!<": - # lookahead assertions - dir = 1 - if char == "<": - char = sourceget() - if char is None: - raise source.error("unexpected end of pattern") - if char not in "=!": - raise source.error("unknown extension ?<" + char, - len(char) + 2) - dir = -1 # lookbehind - lookbehindgroups = state.lookbehindgroups - if lookbehindgroups is None: - state.lookbehindgroups = state.groups - p = _parse_sub(source, state, verbose, nested + 1) - if dir < 0: - if lookbehindgroups is None: - state.lookbehindgroups = None - if not sourcematch(")"): - raise source.error("missing ), unterminated subpattern", - source.tell() - start) - if char == "=": - subpatternappend((ASSERT, (dir, p))) - else: - subpatternappend((ASSERT_NOT, (dir, p))) - continue - - elif char == "(": - # conditional backreference group - condname = source.getuntil(")", "group name") - if condname.isidentifier(): - condgroup = state.groupdict.get(condname) - if condgroup is None: - msg = "unknown group name %r" % condname - raise source.error(msg, len(condname) + 1) - else: - try: - condgroup = int(condname) - if condgroup < 0: - raise ValueError - except ValueError: - msg = "bad character in group name %r" % condname - raise source.error(msg, len(condname) + 1) from None - if not condgroup: - raise source.error("bad group number", - len(condname) + 1) - if condgroup >= MAXGROUPS: - msg = "invalid group reference %d" % condgroup - raise source.error(msg, len(condname) + 1) - state.checklookbehindgroup(condgroup, source) - item_yes = _parse(source, state, verbose, nested + 1) - if source.match("|"): - item_no = _parse(source, state, verbose, nested + 1) - if source.next == "|": - raise source.error("conditional backref with more than two branches") - else: - item_no = None - if not source.match(")"): - raise source.error("missing ), unterminated subpattern", - source.tell() - start) - subpatternappend((GROUPREF_EXISTS, (condgroup, item_yes, item_no))) - continue - - elif char in FLAGS or char == "-": - # flags - flags = _parse_flags(source, state, char) - if flags is None: # global flags - if not first or subpattern: - import warnings - warnings.warn( - 'Flags not at the start of the expression %r%s' % ( - source.string[:20], # truncate long regexes - ' (truncated)' if len(source.string) > 20 else '', - ), - DeprecationWarning, stacklevel=nested + 6 - ) - if (state.flags & SRE_FLAG_VERBOSE) and not verbose: - raise Verbose - continue - - add_flags, del_flags = flags - group = None - else: - raise source.error("unknown extension ?" + char, - len(char) + 1) - - # parse group contents - if group is not None: - try: - group = state.opengroup(name) - except error as err: - raise source.error(err.msg, len(name) + 1) from None - sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and - not (del_flags & SRE_FLAG_VERBOSE)) - p = _parse_sub(source, state, sub_verbose, nested + 1) - if not source.match(")"): - raise source.error("missing ), unterminated subpattern", - source.tell() - start) - if group is not None: - state.closegroup(group, p) - subpatternappend((SUBPATTERN, (group, add_flags, del_flags, p))) - - elif this == "^": - subpatternappend((AT, AT_BEGINNING)) - - elif this == "$": - subpatternappend((AT, AT_END)) - - else: - raise AssertionError("unsupported special character %r" % (char,)) - - # unpack non-capturing groups - for i in range(len(subpattern))[::-1]: - op, av = subpattern[i] - if op is SUBPATTERN: - group, add_flags, del_flags, p = av - if group is None and not add_flags and not del_flags: - subpattern[i: i+1] = p - - return subpattern - -def _parse_flags(source, state, char): - sourceget = source.get - add_flags = 0 - del_flags = 0 - if char != "-": - while True: - flag = FLAGS[char] - if source.istext: - if char == 'L': - msg = "bad inline flags: cannot use 'L' flag with a str pattern" - raise source.error(msg) - else: - if char == 'u': - msg = "bad inline flags: cannot use 'u' flag with a bytes pattern" - raise source.error(msg) - add_flags |= flag - if (flag & TYPE_FLAGS) and (add_flags & TYPE_FLAGS) != flag: - msg = "bad inline flags: flags 'a', 'u' and 'L' are incompatible" - raise source.error(msg) - char = sourceget() - if char is None: - raise source.error("missing -, : or )") - if char in ")-:": - break - if char not in FLAGS: - msg = "unknown flag" if char.isalpha() else "missing -, : or )" - raise source.error(msg, len(char)) - if char == ")": - state.flags |= add_flags - return None - if add_flags & GLOBAL_FLAGS: - raise source.error("bad inline flags: cannot turn on global flag", 1) - if char == "-": - char = sourceget() - if char is None: - raise source.error("missing flag") - if char not in FLAGS: - msg = "unknown flag" if char.isalpha() else "missing flag" - raise source.error(msg, len(char)) - while True: - flag = FLAGS[char] - if flag & TYPE_FLAGS: - msg = "bad inline flags: cannot turn off flags 'a', 'u' and 'L'" - raise source.error(msg) - del_flags |= flag - char = sourceget() - if char is None: - raise source.error("missing :") - if char == ":": - break - if char not in FLAGS: - msg = "unknown flag" if char.isalpha() else "missing :" - raise source.error(msg, len(char)) - assert char == ":" - if del_flags & GLOBAL_FLAGS: - raise source.error("bad inline flags: cannot turn off global flag", 1) - if add_flags & del_flags: - raise source.error("bad inline flags: flag turned on and off", 1) - return add_flags, del_flags - -def fix_flags(src, flags): - # Check and fix flags according to the type of pattern (str or bytes) - if isinstance(src, str): - if flags & SRE_FLAG_LOCALE: - raise ValueError("cannot use LOCALE flag with a str pattern") - if not flags & SRE_FLAG_ASCII: - flags |= SRE_FLAG_UNICODE - elif flags & SRE_FLAG_UNICODE: - raise ValueError("ASCII and UNICODE flags are incompatible") - else: - if flags & SRE_FLAG_UNICODE: - raise ValueError("cannot use UNICODE flag with a bytes pattern") - if flags & SRE_FLAG_LOCALE and flags & SRE_FLAG_ASCII: - raise ValueError("ASCII and LOCALE flags are incompatible") - return flags - -def parse(str, flags=0, state=None): - # parse 're' pattern into list of (opcode, argument) tuples - - source = Tokenizer(str) - - if state is None: - state = State() - state.flags = flags - state.str = str - - try: - p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0) - except Verbose: - # the VERBOSE flag was switched on inside the pattern. to be - # on the safe side, we'll parse the whole thing again... - state = State() - state.flags = flags | SRE_FLAG_VERBOSE - state.str = str - source.seek(0) - p = _parse_sub(source, state, True, 0) - - p.state.flags = fix_flags(str, p.state.flags) - - if source.next is not None: - assert source.next == ")" - raise source.error("unbalanced parenthesis") - - if flags & SRE_FLAG_DEBUG: - p.dump() - - return p - -def parse_template(source, state): - # parse 're' replacement string into list of literals and - # group references - s = Tokenizer(source) - sget = s.get - groups = [] - literals = [] - literal = [] - lappend = literal.append - def addgroup(index, pos): - if index > state.groups: - raise s.error("invalid group reference %d" % index, pos) - if literal: - literals.append(''.join(literal)) - del literal[:] - groups.append((len(literals), index)) - literals.append(None) - groupindex = state.groupindex - while True: - this = sget() - if this is None: - break # end of replacement string - if this[0] == "\\": - # group - c = this[1] - if c == "g": - name = "" - if not s.match("<"): - raise s.error("missing <") - name = s.getuntil(">", "group name") - if name.isidentifier(): - try: - index = groupindex[name] - except KeyError: - raise IndexError("unknown group name %r" % name) - else: - try: - index = int(name) - if index < 0: - raise ValueError - except ValueError: - raise s.error("bad character in group name %r" % name, - len(name) + 1) from None - if index >= MAXGROUPS: - raise s.error("invalid group reference %d" % index, - len(name) + 1) - addgroup(index, len(name) + 1) - elif c == "0": - if s.next in OCTDIGITS: - this += sget() - if s.next in OCTDIGITS: - this += sget() - lappend(chr(int(this[1:], 8) & 0xff)) - elif c in DIGITS: - isoctal = False - if s.next in DIGITS: - this += sget() - if (c in OCTDIGITS and this[2] in OCTDIGITS and - s.next in OCTDIGITS): - this += sget() - isoctal = True - c = int(this[1:], 8) - if c > 0o377: - raise s.error('octal escape value %s outside of ' - 'range 0-0o377' % this, len(this)) - lappend(chr(c)) - if not isoctal: - addgroup(int(this[1:]), len(this) - 1) - else: - try: - this = chr(ESCAPES[this][1]) - except KeyError: - if c in ASCIILETTERS: - raise s.error('bad escape %s' % this, len(this)) - lappend(this) - else: - lappend(this) - if literal: - literals.append(''.join(literal)) - if not isinstance(source, str): - # The tokenizer implicitly decodes bytes objects as latin-1, we must - # therefore re-encode the final representation. - literals = [None if s is None else s.encode('latin-1') for s in literals] - return groups, literals - -def expand_template(template, match): - g = match.group - empty = match.string[:0] - groups, literals = template - literals = literals[:] - try: - for index, group in groups: - literals[index] = g(group) or empty - except IndexError: - raise error("invalid group reference %d" % index) - return empty.join(literals) +from re import _parser as _ +globals().update({k: v for k, v in vars(_).items() if k[:2] != '__'}) diff --git a/Lib/ssl.py b/Lib/ssl.py index 1200d7d9939..8889aff92fa 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -18,9 +18,10 @@ seconds past the Epoch (the time values returned from time.time()) - fetch_server_certificate (HOST, PORT) -- fetch the certificate provided - by the server running on HOST at port PORT. No - validation of the certificate is performed. + get_server_certificate (addr, ssl_version, ca_certs, timeout) -- Retrieve the + certificate from the server at the specified + address and return it as a PEM-encoded string + Integer constants: @@ -94,31 +95,31 @@ import os from collections import namedtuple from enum import Enum as _Enum, IntEnum as _IntEnum, IntFlag as _IntFlag +from enum import _simple_enum import _ssl # if we can't import it, let the error propagate from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION -from _ssl import _SSLContext#, MemoryBIO, SSLSession +from _ssl import _SSLContext, MemoryBIO, SSLSession from _ssl import ( SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError, SSLSyscallError, SSLEOFError, SSLCertVerificationError ) from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj -from _ssl import RAND_status, RAND_add, RAND_bytes, RAND_pseudo_bytes +from _ssl import RAND_status, RAND_add, RAND_bytes try: from _ssl import RAND_egd except ImportError: - # LibreSSL does not provide RAND_egd + # RAND_egd is not supported on some platforms pass from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3 + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK, HAS_PHA ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION - _IntEnum._convert_( '_SSLMethod', __name__, lambda name: name.startswith('PROTOCOL_') and name != 'PROTOCOL_SSLv23', @@ -155,7 +156,8 @@ _SSLv2_IF_EXISTS = getattr(_SSLMethod, 'PROTOCOL_SSLv2', None) -class TLSVersion(_IntEnum): +@_simple_enum(_IntEnum) +class TLSVersion: MINIMUM_SUPPORTED = _ssl.PROTO_MINIMUM_SUPPORTED SSLv3 = _ssl.PROTO_SSLv3 TLSv1 = _ssl.PROTO_TLSv1 @@ -165,7 +167,8 @@ class TLSVersion(_IntEnum): MAXIMUM_SUPPORTED = _ssl.PROTO_MAXIMUM_SUPPORTED -class _TLSContentType(_IntEnum): +@_simple_enum(_IntEnum) +class _TLSContentType: """Content types (record layer) See RFC 8446, section B.1 @@ -179,10 +182,11 @@ class _TLSContentType(_IntEnum): INNER_CONTENT_TYPE = 0x101 -class _TLSAlertType(_IntEnum): +@_simple_enum(_IntEnum) +class _TLSAlertType: """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 @@ -220,7 +224,8 @@ class _TLSAlertType(_IntEnum): NO_APPLICATION_PROTOCOL = 120 -class _TLSMessageType(_IntEnum): +@_simple_enum(_IntEnum) +class _TLSMessageType: """Message types (handshake protocol) See RFC 8446, section B.3 @@ -250,10 +255,10 @@ class _TLSMessageType(_IntEnum): if sys.platform == "win32": - from _ssl import enum_certificates #, enum_crls + from _ssl import enum_certificates, enum_crls -from socket import socket, AF_INET, SOCK_STREAM, create_connection -from socket import SOL_SOCKET, SO_TYPE +from socket import socket, SOCK_STREAM, create_connection +from socket import SOL_SOCKET, SO_TYPE, _GLOBAL_DEFAULT_TIMEOUT import socket as _socket import base64 # for DER-to-PEM translation import errno @@ -275,7 +280,7 @@ class _TLSMessageType(_IntEnum): def _dnsname_match(dn, hostname): """Matching according to RFC 6125, section 6.4.3 - - Hostnames are compared lower case. + - Hostnames are compared lower-case. - For IDNA, both dn and hostname must be encoded as IDN A-label (ACE). - Partial wildcards like 'www*.example.org', multiple wildcards, sole wildcard or wildcards in labels other then the left-most label are not @@ -363,68 +368,11 @@ def _ipaddress_match(cert_ipaddress, host_ip): (section 1.7.2 - "Out of Scope"). """ # OpenSSL may add a trailing newline to a subjectAltName's IP address, - # commonly woth IPv6 addresses. Strip off trailing \n. + # commonly with IPv6 addresses. Strip off trailing \n. ip = _inet_paton(cert_ipaddress.rstrip()) return ip == host_ip -def match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by - SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 - rules are followed. - - The function matches IP addresses rather than dNSNames if hostname is a - valid ipaddress string. IPv4 addresses are supported on all platforms. - IPv6 addresses are supported on platforms with IPv6 support (AF_INET6 - and inet_pton). - - CertificateError is raised on failure. On success, the function - returns nothing. - """ - if not cert: - raise ValueError("empty or no certificate, match_hostname needs a " - "SSL socket or SSL context with either " - "CERT_OPTIONAL or CERT_REQUIRED") - try: - host_ip = _inet_paton(hostname) - except ValueError: - # Not an IP address (common case) - host_ip = None - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if host_ip is None and _dnsname_match(value, hostname): - return - dnsnames.append(value) - elif key == 'IP Address': - if host_ip is not None and _ipaddress_match(value, host_ip): - return - dnsnames.append(value) - if not dnsnames: - # The subject is only checked when there is no dNSName entry - # in subjectAltName - for sub in cert.get('subject', ()): - for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. - if key == 'commonName': - if _dnsname_match(value, hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise CertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise CertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) - else: - raise CertificateError("no appropriate commonName or " - "subjectAltName fields were found") - - DefaultVerifyPaths = namedtuple("DefaultVerifyPaths", "cafile capath openssl_cafile_env openssl_cafile openssl_capath_env " "openssl_capath") @@ -479,7 +427,14 @@ class SSLContext(_SSLContext): sslsocket_class = None # SSLSocket is assigned later. sslobject_class = None # SSLObject is assigned later. - def __new__(cls, protocol=PROTOCOL_TLS, *args, **kwargs): + def __new__(cls, protocol=None, *args, **kwargs): + if protocol is None: + warnings.warn( + "ssl.SSLContext() without protocol argument is deprecated.", + category=DeprecationWarning, + stacklevel=2 + ) + protocol = PROTOCOL_TLS self = _SSLContext.__new__(cls, protocol) return self @@ -518,6 +473,11 @@ def wrap_bio(self, incoming, outgoing, server_side=False, ) def set_npn_protocols(self, npn_protocols): + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) protos = bytearray() for protocol in npn_protocols: b = bytes(protocol, 'ascii') @@ -553,18 +513,17 @@ def set_alpn_protocols(self, alpn_protocols): self._set_alpn_protocols(protos) def _load_windows_store_certs(self, storename, purpose): - certs = bytearray() try: for cert, encoding, trust in enum_certificates(storename): # CA certs are never PKCS#7 encoded if encoding == "x509_asn": if trust is True or purpose.oid in trust: - certs.extend(cert) + try: + self.load_verify_locations(cadata=cert) + except SSLError as exc: + warnings.warn(f"Bad certificate in Windows certificate store: {exc!s}") except PermissionError: warnings.warn("unable to enumerate Windows certificate store") - if certs: - self.load_verify_locations(cadata=certs) - return certs def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): @@ -734,12 +693,25 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, # SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. - context = SSLContext(PROTOCOL_TLS) - if purpose == Purpose.SERVER_AUTH: # verify certs and host name in client mode + context = SSLContext(PROTOCOL_TLS_CLIENT) context.verify_mode = CERT_REQUIRED context.check_hostname = True + elif purpose == Purpose.CLIENT_AUTH: + context = SSLContext(PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) + + # `VERIFY_X509_PARTIAL_CHAIN` makes OpenSSL's chain building behave more + # like RFC 3280 and 5280, which specify that chain building stops with the + # first trust anchor, even if that anchor is not self-signed. + # + # `VERIFY_X509_STRICT` makes OpenSSL more conservative about the + # certificates it accepts, including "disabling workarounds for + # some broken certificates." + context.verify_flags |= (_ssl.VERIFY_X509_PARTIAL_CHAIN | + _ssl.VERIFY_X509_STRICT) if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) @@ -755,7 +727,7 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, context.keylog_filename = keylogfile return context -def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=CERT_NONE, +def _create_unverified_context(protocol=None, *, cert_reqs=CERT_NONE, check_hostname=False, purpose=Purpose.SERVER_AUTH, certfile=None, keyfile=None, cafile=None, capath=None, cadata=None): @@ -772,10 +744,18 @@ def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=CERT_NONE, # SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. - context = SSLContext(protocol) + if purpose == Purpose.SERVER_AUTH: + # verify certs and host name in client mode + if protocol is None: + protocol = PROTOCOL_TLS_CLIENT + elif purpose == Purpose.CLIENT_AUTH: + if protocol is None: + protocol = PROTOCOL_TLS_SERVER + else: + raise ValueError(purpose) - if not check_hostname: - context.check_hostname = False + context = SSLContext(protocol) + context.check_hostname = check_hostname if cert_reqs is not None: context.verify_mode = cert_reqs if check_hostname: @@ -905,19 +885,46 @@ def getpeercert(self, binary_form=False): """ return self._sslobj.getpeercert(binary_form) + def get_verified_chain(self): + """Returns verified certificate chain provided by the other + end of the SSL channel as a list of DER-encoded bytes. + + If certificate verification was disabled method acts the same as + ``SSLSocket.get_unverified_chain``. + """ + chain = self._sslobj.get_verified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] + + def get_unverified_chain(self): + """Returns raw certificate chain provided by the other + end of the SSL channel as a list of DER-encoded bytes. + """ + chain = self._sslobj.get_unverified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] + def selected_npn_protocol(self): """Return the currently selected NPN protocol as a string, or ``None`` if a next protocol was not negotiated or if NPN is not supported by one of the peers.""" - if _ssl.HAS_NPN: - return self._sslobj.selected_npn_protocol() + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) def selected_alpn_protocol(self): """Return the currently selected ALPN protocol as a string, or ``None`` if a next protocol was not negotiated or if ALPN is not supported by one of the peers.""" - if _ssl.HAS_ALPN: - return self._sslobj.selected_alpn_protocol() + return self._sslobj.selected_alpn_protocol() def cipher(self): """Return the currently selected cipher as a 3-tuple ``(name, @@ -996,38 +1003,67 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if context.check_hostname and not server_hostname: raise ValueError("check_hostname requires server_hostname") + sock_timeout = sock.gettimeout() kwargs = dict( family=sock.family, type=sock.type, proto=sock.proto, fileno=sock.fileno() ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) sock.detach() - - self._context = context - self._session = session - self._closed = False - self._sslobj = None - self.server_side = server_side - self.server_hostname = context._encode_hostname(server_hostname) - self.do_handshake_on_connect = do_handshake_on_connect - self.suppress_ragged_eofs = suppress_ragged_eofs - - # See if we are connected + # Now SSLSocket is responsible for closing the file descriptor. try: - self.getpeername() - except OSError as e: - if e.errno != errno.ENOTCONN: - raise - connected = False - else: - connected = True + self._context = context + self._session = session + self._closed = False + self._sslobj = None + self.server_side = server_side + self.server_hostname = context._encode_hostname(server_hostname) + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs - self._connected = connected - if connected: - # create the SSL object + # See if we are connected try: + self.getpeername() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None + else: + connected = True + + self.settimeout(sock_timeout) # Must come after setblocking() calls. + self._connected = connected + if connected: + # create the SSL object self._sslobj = self._context._wrap_socket( self, server_side, self.server_hostname, owner=self, session=self._session, @@ -1038,9 +1074,12 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, # non-blocking raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") self.do_handshake() - except (OSError, ValueError): + except: + try: self.close() - raise + except OSError: + pass + raise return self @property @@ -1123,13 +1162,33 @@ def getpeercert(self, binary_form=False): self._check_connected() return self._sslobj.getpeercert(binary_form) + @_sslcopydoc + def get_verified_chain(self): + chain = self._sslobj.get_verified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] + + @_sslcopydoc + def get_unverified_chain(self): + chain = self._sslobj.get_unverified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] + @_sslcopydoc def selected_npn_protocol(self): self._checkClosed() - if self._sslobj is None or not _ssl.HAS_NPN: - return None - else: - return self._sslobj.selected_npn_protocol() + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) + return None @_sslcopydoc def selected_alpn_protocol(self): @@ -1229,10 +1288,14 @@ def recv(self, buflen=1024, flags=0): def recv_into(self, buffer, nbytes=None, flags=0): self._checkClosed() - if buffer and (nbytes is None): - nbytes = len(buffer) - elif nbytes is None: - nbytes = 1024 + if nbytes is None: + if buffer is not None: + with memoryview(buffer) as view: + nbytes = view.nbytes + if not nbytes: + nbytes = 1024 + else: + nbytes = 1024 if self._sslobj is not None: if flags != 0: raise ValueError( @@ -1382,32 +1445,6 @@ def version(self): SSLContext.sslobject_class = SSLObject -def wrap_socket(sock, keyfile=None, certfile=None, - server_side=False, cert_reqs=CERT_NONE, - ssl_version=PROTOCOL_TLS, ca_certs=None, - do_handshake_on_connect=True, - suppress_ragged_eofs=True, - ciphers=None): - - if server_side and not certfile: - raise ValueError("certfile must be specified for server-side " - "operations") - if keyfile and not certfile: - raise ValueError("certfile must be specified") - context = SSLContext(ssl_version) - context.verify_mode = cert_reqs - if ca_certs: - context.load_verify_locations(ca_certs) - if certfile: - context.load_cert_chain(certfile, keyfile) - if ciphers: - context.set_ciphers(ciphers) - return context.wrap_socket( - sock=sock, server_side=server_side, - do_handshake_on_connect=do_handshake_on_connect, - suppress_ragged_eofs=suppress_ragged_eofs - ) - # some utility functions def cert_time_to_seconds(cert_time): @@ -1466,11 +1503,14 @@ def PEM_cert_to_DER_cert(pem_cert_string): d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)] return base64.decodebytes(d.encode('ASCII', 'strict')) -def get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None): +def get_server_certificate(addr, ssl_version=PROTOCOL_TLS_CLIENT, + ca_certs=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Retrieve the certificate from the server at the specified address, and return it as a PEM-encoded string. If 'ca_certs' is specified, validate the server cert against it. - If 'ssl_version' is specified, use it in the connection attempt.""" + If 'ssl_version' is specified, use it in the connection attempt. + If 'timeout' is specified, use it in the connection attempt. + """ host, port = addr if ca_certs is not None: @@ -1480,8 +1520,8 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None): context = _create_stdlib_context(ssl_version, cert_reqs=cert_reqs, cafile=ca_certs) - with create_connection(addr) as sock: - with context.wrap_socket(sock) as sslsock: + with create_connection(addr, timeout=timeout) as sock: + with context.wrap_socket(sock, server_hostname=host) as sslsock: dercert = sslsock.getpeercert(True) return DER_cert_to_PEM_cert(dercert) diff --git a/Lib/stat.py b/Lib/stat.py index fc024db3f4f..81f694329bf 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -110,22 +110,30 @@ def S_ISWHT(mode): S_IXOTH = 0o0001 # execute by others # Names for file flags - +UF_SETTABLE = 0x0000ffff # owner settable flags UF_NODUMP = 0x00000001 # do not dump file UF_IMMUTABLE = 0x00000002 # file may not be changed UF_APPEND = 0x00000004 # file may only be appended to UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted -UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed -UF_HIDDEN = 0x00008000 # OS X: file should not be displayed +UF_COMPRESSED = 0x00000020 # macOS: file is compressed +UF_TRACKED = 0x00000040 # macOS: used for handling document IDs +UF_DATAVAULT = 0x00000080 # macOS: entitlement needed for I/O +UF_HIDDEN = 0x00008000 # macOS: file should not be displayed +SF_SETTABLE = 0xffff0000 # superuser settable flags SF_ARCHIVED = 0x00010000 # file may be archived SF_IMMUTABLE = 0x00020000 # file may not be changed SF_APPEND = 0x00040000 # file may only be appended to +SF_RESTRICTED = 0x00080000 # macOS: entitlement needed for writing SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted SF_SNAPSHOT = 0x00200000 # file is a snapshot file +SF_FIRMLINK = 0x00800000 # macOS: file is a firmlink +SF_DATALESS = 0x40000000 # macOS: file is a dataless object _filemode_table = ( + # File type chars according to: + # http://en.wikibooks.org/wiki/C_Programming/POSIX_Reference/sys/stat.h ((S_IFLNK, "l"), (S_IFSOCK, "s"), # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR (S_IFREG, "-"), @@ -156,13 +164,22 @@ def S_ISWHT(mode): def filemode(mode): """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" perm = [] - for table in _filemode_table: + for index, table in enumerate(_filemode_table): for bit, char in table: - if mode & bit == bit: - perm.append(char) - break + if index == 0: + if S_IFMT(mode) == bit: + perm.append(char) + break + else: + if mode & bit == bit: + perm.append(char) + break else: - perm.append("-") + if index == 0: + # Unknown filetype + perm.append("?") + else: + perm.append("-") return "".join(perm) diff --git a/Lib/statistics.py b/Lib/statistics.py index f66245380ab..26cf925529e 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -11,7 +11,7 @@ Function Description ================== ================================================== mean Arithmetic mean (average) of data. -fmean Fast, floating point arithmetic mean. +fmean Fast, floating-point arithmetic mean. geometric_mean Geometric mean of data. harmonic_mean Harmonic mean of data. median Median (middle value) of data. @@ -112,6 +112,8 @@ 'fmean', 'geometric_mean', 'harmonic_mean', + 'kde', + 'kde_random', 'linear_regression', 'mean', 'median', @@ -130,180 +132,28 @@ import math import numbers import random +import sys from fractions import Fraction from decimal import Decimal -from itertools import groupby, repeat +from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right -from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum +from math import hypot, sqrt, fabs, exp, erfc, tau, log, fsum, sumprod +from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos +from functools import reduce from operator import itemgetter -from collections import Counter, namedtuple +from collections import Counter, namedtuple, defaultdict -# === Exceptions === +_SQRT2 = sqrt(2.0) +_random = random + +## Exceptions ############################################################## class StatisticsError(ValueError): pass -# === Private utilities === - -def _sum(data): - """_sum(data) -> (type, sum, count) - - Return a high-precision sum of the given numeric data as a fraction, - together with the type to be converted to and the count of items. - - Examples - -------- - - >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) - (, Fraction(19, 2), 5) - - Some sources of round-off error will be avoided: - - # Built-in sum returns zero. - >>> _sum([1e50, 1, -1e50] * 1000) - (, Fraction(1000, 1), 3000) - - Fractions and Decimals are also supported: - - >>> from fractions import Fraction as F - >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - (, Fraction(63, 20), 4) - - >>> from decimal import Decimal as D - >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] - >>> _sum(data) - (, Fraction(6963, 10000), 4) - - Mixed types are currently treated as an error, except that int is - allowed. - """ - count = 0 - partials = {} - partials_get = partials.get - T = int - for typ, values in groupby(data, type): - T = _coerce(T, typ) # or raise TypeError - for n, d in map(_exact_ratio, values): - count += 1 - partials[d] = partials_get(d, 0) + n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - # Sum all the partial sums using builtin sum. - total = sum(Fraction(n, d) for d, n in partials.items()) - return (T, total, count) - - -def _isfinite(x): - try: - return x.is_finite() # Likely a Decimal. - except AttributeError: - return math.isfinite(x) # Coerces to float first. - - -def _coerce(T, S): - """Coerce types T and S to a common type, or raise TypeError. - - Coercion rules are currently an implementation detail. See the CoerceTest - test class in test_statistics for details. - """ - # See http://bugs.python.org/issue24068. - assert T is not bool, "initial type T is bool" - # If the types are the same, no need to coerce anything. Put this - # first, so that the usual case (no coercion needed) happens as soon - # as possible. - if T is S: return T - # Mixed int & other coerce to the other type. - if S is int or S is bool: return T - if T is int: return S - # If one is a (strict) subclass of the other, coerce to the subclass. - if issubclass(S, T): return S - if issubclass(T, S): return T - # Ints coerce to the other type. - if issubclass(T, int): return S - if issubclass(S, int): return T - # Mixed fraction & float coerces to float (or float subclass). - if issubclass(T, Fraction) and issubclass(S, float): - return S - if issubclass(T, float) and issubclass(S, Fraction): - return T - # Any other combination is disallowed. - msg = "don't know how to coerce %s and %s" - raise TypeError(msg % (T.__name__, S.__name__)) - - -def _exact_ratio(x): - """Return Real number x to exact (numerator, denominator) pair. - - >>> _exact_ratio(0.25) - (1, 4) - - x is expected to be an int, Fraction, Decimal or float. - """ - try: - return x.as_integer_ratio() - except AttributeError: - pass - except (OverflowError, ValueError): - # float NAN or INF. - assert not _isfinite(x) - return (x, None) - try: - # x may be an Integral ABC. - return (x.numerator, x.denominator) - except AttributeError: - msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" - raise TypeError(msg) - - -def _convert(value, T): - """Convert value to given numeric type T.""" - if type(value) is T: - # This covers the cases where T is Fraction, or where value is - # a NAN or INF (Decimal or float). - return value - if issubclass(T, int) and value.denominator != 1: - T = float - try: - # FIXME: what do we do if this overflows? - return T(value) - except TypeError: - if issubclass(T, Decimal): - return T(value.numerator) / T(value.denominator) - else: - raise - - -def _find_lteq(a, x): - 'Locate the leftmost value exactly equal to x' - i = bisect_left(a, x) - if i != len(a) and a[i] == x: - return i - raise ValueError - - -def _find_rteq(a, l, x): - 'Locate the rightmost value exactly equal to x' - i = bisect_right(a, x, lo=l) - if i != (len(a) + 1) and a[i - 1] == x: - return i - 1 - raise ValueError - - -def _fail_neg(values, errmsg='negative value'): - """Iterate over values, failing if any are less than zero.""" - for x in values: - if x < 0: - raise StatisticsError(errmsg) - yield x - - -# === Measures of central tendency (averages) === +## Measures of central tendency (averages) ################################# def mean(data): """Return the sample arithmetic mean of data. @@ -320,18 +170,15 @@ def mean(data): Decimal('0.5625') If ``data`` is empty, StatisticsError will be raised. + """ - if iter(data) is data: - data = list(data) - n = len(data) + T, total, n = _sum(data) if n < 1: raise StatisticsError('mean requires at least one data point') - T, total, count = _sum(data) - assert count == n return _convert(total / n, T) -def fmean(data): +def fmean(data, weights=None): """Convert data to floats and compute the arithmetic mean. This runs faster than the mean() function and it always returns a float. @@ -339,42 +186,79 @@ def fmean(data): >>> fmean([3.5, 4.0, 5.25]) 4.25 + """ - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - total = fsum(count(data)) - else: - total = fsum(data) - try: + if weights is None: + + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + counter = count() + total = fsum(map(itemgetter(0), zip(data, counter))) + n = next(counter) + else: + total = fsum(data) + + if not n: + raise StatisticsError('fmean requires at least one data point') + return total / n - except ZeroDivisionError: - raise StatisticsError('fmean requires at least one data point') from None + + if not isinstance(weights, (list, tuple)): + weights = list(weights) + + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + + den = fsum(weights) + + if not den: + raise StatisticsError('sum of weights must be non-zero') + + return num / den def geometric_mean(data): """Convert data to floats and compute the geometric mean. - Raises a StatisticsError if the input dataset is empty, - if it contains a zero, or if it contains a negative value. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. + + Returns zero if the product of inputs is zero. No special efforts are made to achieve exact results. (However, this may change in the future.) >>> round(geometric_mean([54, 24, 36]), 9) 36.0 + """ - try: - return exp(fmean(map(log, data))) - except ValueError: - raise StatisticsError('geometric mean requires a non-empty dataset ' - 'containing positive numbers') from None + n = 0 + found_zero = False + + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + + total = fsum(map(log, count_positive(data))) + + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + + return exp(total / n) def harmonic_mean(data, weights=None): @@ -399,10 +283,13 @@ def harmonic_mean(data, weights=None): If ``data`` is empty, or any element is less than zero, ``harmonic_mean`` will raise ``StatisticsError``. + """ if iter(data) is data: data = list(data) + errmsg = 'harmonic mean does not support negative values' + n = len(data) if n < 1: raise StatisticsError('harmonic_mean requires at least one data point') @@ -414,6 +301,7 @@ def harmonic_mean(data, weights=None): return x else: raise TypeError('unsupported type') + if weights is None: weights = repeat(1, n) sum_weights = n @@ -423,16 +311,19 @@ def harmonic_mean(data, weights=None): if len(weights) != n: raise StatisticsError('Number of weights does not match data size') _, sum_weights, _ = _sum(w for w in _fail_neg(weights, errmsg)) + try: data = _fail_neg(data, errmsg) T, total, count = _sum(w / x if w else 0 for w, x in zip(weights, data)) except ZeroDivisionError: return 0 + if total <= 0: raise StatisticsError('Weighted sum must be positive') + return _convert(sum_weights / total, T) -# FIXME: investigate ways to calculate medians without sorting? Quickselect? + def median(data): """Return the median (middle value) of numeric data. @@ -469,6 +360,9 @@ def median_low(data): 3 """ + # Potentially the sorting step could be replaced with a quickselect. + # However, it would require an excellent implementation to beat our + # highly optimized builtin sort. data = sorted(data) n = len(data) if n == 0: @@ -498,58 +392,75 @@ def median_high(data): return data[n // 2] -def median_grouped(data, interval=1): - """Return the 50th percentile (median) of grouped continuous data. +def median_grouped(data, interval=1.0): + """Estimates the median for numeric data binned around the midpoints + of consecutive, fixed-width intervals. - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - >>> median_grouped([52, 52, 53, 54]) - 52.5 + The *data* can be any iterable of numeric data with each value being + exactly the midpoint of a bin. At least one value must be present. - This calculates the median as the 50th percentile, and should be - used when your data is continuous and grouped. In the above example, - the values 1, 2, 3, etc. actually represent the midpoint of classes - 0.5-1.5, 1.5-2.5, 2.5-3.5, etc. The middle value falls somewhere in - class 3.5-4.5, and interpolation is used to estimate it. + The *interval* is width of each bin. - Optional argument ``interval`` represents the class interval, and - defaults to 1. Changing the class interval naturally will change the - interpolated 50th percentile value: + For example, demographic information may have been summarized into + consecutive ten-year age groups with each group being represented + by the 5-year midpoints of the intervals: - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 + >>> demographics = Counter({ + ... 25: 172, # 20 to 30 years old + ... 35: 484, # 30 to 40 years old + ... 45: 387, # 40 to 50 years old + ... 55: 22, # 50 to 60 years old + ... 65: 6, # 60 to 70 years old + ... }) + + The 50th percentile (median) is the 536th person out of the 1071 + member cohort. That person is in the 30 to 40 year old age group. + + The regular median() function would assume that everyone in the + tricenarian age group was exactly 35 years old. A more tenable + assumption is that the 484 members of that age group are evenly + distributed between 30 and 40. For that, we use median_grouped(). + + >>> data = list(demographics.elements()) + >>> median(data) + 35 + >>> round(median_grouped(data, interval=10), 1) + 37.5 + + The caller is responsible for making sure the data points are separated + by exact multiples of *interval*. This is essential for getting a + correct result. The function does not check this precondition. + + Inputs may be any numeric type that can be coerced to a float during + the interpolation step. - This function does not check whether the data points are at least - ``interval`` apart. """ data = sorted(data) n = len(data) - if n == 0: + if not n: raise StatisticsError("no median for empty data") - elif n == 1: - return data[0] + # Find the value at the midpoint. Remember this corresponds to the - # centre of the class interval. + # midpoint of the class interval. x = data[n // 2] - for obj in (x, interval): - if isinstance(obj, (str, bytes)): - raise TypeError('expected number but got %r' % obj) + + # Using O(log n) bisection, find where all the x values occur in the data. + # All x will lie within data[i:j]. + i = bisect_left(data, x) + j = bisect_right(data, x, lo=i) + + # Coerce to floats, raising a TypeError if not possible try: - L = x - interval / 2 # The lower limit of the median interval. - except TypeError: - # Mixed type. For now we just coerce to float. - L = float(x) - float(interval) / 2 - - # Uses bisection search to search for x in data with log(n) time complexity - # Find the position of leftmost occurrence of x in data - l1 = _find_lteq(data, x) - # Find the position of rightmost occurrence of x in data[l1...len(data)] - # Assuming always l1 <= l2 - l2 = _find_rteq(data, l1, x) - cf = l1 - f = l2 - l1 + 1 + interval = float(interval) + x = float(x) + except ValueError: + raise TypeError(f'Value cannot be converted to a float') + + # Interpolate the median using the formula found at: + # https://www.cuemath.com/data/median-of-grouped-data/ + L = x - interval / 2.0 # Lower limit of the median interval + cf = i # Cumulative frequency of the preceding interval + f = j - i # Number of elements in the median internal return L + interval * (n / 2 - cf) / f @@ -595,159 +506,43 @@ def multimode(data): ['b', 'd', 'f'] >>> multimode('') [] + """ - counts = Counter(iter(data)).most_common() - maxcount, mode_items = next(groupby(counts, key=itemgetter(1)), (0, [])) - return list(map(itemgetter(0), mode_items)) + counts = Counter(iter(data)) + if not counts: + return [] + maxcount = max(counts.values()) + return [value for value, count in counts.items() if count == maxcount] -# Notes on methods for computing quantiles -# ---------------------------------------- -# -# There is no one perfect way to compute quantiles. Here we offer -# two methods that serve common needs. Most other packages -# surveyed offered at least one or both of these two, making them -# "standard" in the sense of "widely-adopted and reproducible". -# They are also easy to explain, easy to compute manually, and have -# straight-forward interpretations that aren't surprising. +## Measures of spread ###################################################### -# The default method is known as "R6", "PERCENTILE.EXC", or "expected -# value of rank order statistics". The alternative method is known as -# "R7", "PERCENTILE.INC", or "mode of rank order statistics". +def variance(data, xbar=None): + """Return the sample variance of data. -# For sample data where there is a positive probability for values -# beyond the range of the data, the R6 exclusive method is a -# reasonable choice. Consider a random sample of nine values from a -# population with a uniform distribution from 0.0 to 1.0. The -# distribution of the third ranked sample point is described by -# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and -# mean=0.300. Only the latter (which corresponds with R6) gives the -# desired cut point with 30% of the population falling below that -# value, making it comparable to a result from an inv_cdf() function. -# The R6 exclusive method is also idempotent. + data should be an iterable of Real-valued numbers, with at least two + values. The optional argument xbar, if given, should be the mean of + the data. If it is missing or None, the mean is automatically calculated. -# For describing population data where the end points are known to -# be included in the data, the R7 inclusive method is a reasonable -# choice. Instead of the mean, it uses the mode of the beta -# distribution for the interior points. Per Hyndman & Fan, "One nice -# property is that the vertices of Q7(p) divide the range into n - 1 -# intervals, and exactly 100p% of the intervals lie to the left of -# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + Use this function when your data is a sample from a population. To + calculate the variance from the entire population, see ``pvariance``. -# If needed, other methods could be added. However, for now, the -# position is that fewer options make for easier choices and that -# external packages can be used for anything more advanced. + Examples: -def quantiles(data, *, n=4, method='exclusive'): - """Divide *data* into *n* continuous intervals with equal probability. + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 - Returns a list of (n - 1) cut points separating the intervals. + If you have already calculated the mean of your data, you can pass it as + the optional second argument ``xbar`` to avoid recalculating it: - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate *data* in to 100 equal sized groups. + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 - The *data* can be any iterable containing sample. - The cut points are linearly interpolated between data points. - - If *method* is set to *inclusive*, *data* is treated as population - data. The minimum value is treated as the 0th percentile and the - maximum value is treated as the 100th percentile. - """ - if n < 1: - raise StatisticsError('n must be at least 1') - data = sorted(data) - ld = len(data) - if ld < 2: - raise StatisticsError('must have at least two data points') - if method == 'inclusive': - m = ld - 1 - result = [] - for i in range(1, n): - j, delta = divmod(i * m, n) - interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n - result.append(interpolated) - return result - if method == 'exclusive': - m = ld + 1 - result = [] - for i in range(1, n): - j = i * m // n # rescale i to m/n - j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 - delta = i*m - j*n # exact integer math - interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n - result.append(interpolated) - return result - raise ValueError(f'Unknown method: {method!r}') - - -# === Measures of spread === - -# See http://mathworld.wolfram.com/Variance.html -# http://mathworld.wolfram.com/SampleVariance.html -# http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance -# -# Under no circumstances use the so-called "computational formula for -# variance", as that is only suitable for hand calculations with a small -# amount of low-precision data. It has terrible numeric properties. -# -# See a comparison of three computational methods here: -# http://www.johndcook.com/blog/2008/09/26/comparing-three-methods-of-computing-standard-deviation/ - -def _ss(data, c=None): - """Return sum of square deviations of sequence data. - - If ``c`` is None, the mean is calculated in one pass, and the deviations - from the mean are calculated in a second pass. Otherwise, deviations are - calculated from ``c`` as given. Use the second case with care, as it can - lead to garbage results. - """ - if c is not None: - T, total, count = _sum((x-c)**2 for x in data) - return (T, total) - T, total, count = _sum(data) - mean_n, mean_d = (total / count).as_integer_ratio() - partials = Counter() - for n, d in map(_exact_ratio, data): - diff_n = n * mean_d - d * mean_n - diff_d = d * mean_d - partials[diff_d * diff_d] += diff_n * diff_n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - total = sum(Fraction(n, d) for d, n in partials.items()) - return (T, total) - - -def variance(data, xbar=None): - """Return the sample variance of data. - - data should be an iterable of Real-valued numbers, with at least two - values. The optional argument xbar, if given, should be the mean of - the data. If it is missing or None, the mean is automatically calculated. - - Use this function when your data is a sample from a population. To - calculate the variance from the entire population, see ``pvariance``. - - Examples: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as - the optional second argument ``xbar`` to avoid recalculating it: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not check that ``xbar`` is actually the mean of - ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or - impossible results. + This function does not check that ``xbar`` is actually the mean of + ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or + impossible results. Decimals and Fractions are supported: @@ -760,12 +555,11 @@ def variance(data, xbar=None): Fraction(67, 108) """ - if iter(data) is data: - data = list(data) - n = len(data) + # http://mathworld.wolfram.com/SampleVariance.html + + T, ss, c, n = _ss(data, xbar) if n < 2: raise StatisticsError('variance requires at least two data points') - T, ss = _ss(data, xbar) return _convert(ss / (n - 1), T) @@ -804,12 +598,11 @@ def pvariance(data, mu=None): Fraction(13, 72) """ - if iter(data) is data: - data = list(data) - n = len(data) + # http://mathworld.wolfram.com/Variance.html + + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') - T, ss = _ss(data, mu) return _convert(ss / n, T) @@ -822,14 +615,18 @@ def stdev(data, xbar=None): 1.0810874155219827 """ - # Fixme: Despite the exact sum of squared deviations, some inaccuracy - # remain because there are two rounding steps. The first occurs in - # the _convert() step for variance(), the second occurs in math.sqrt(). - var = variance(data, xbar) + T, ss, c, n = _ss(data, xbar) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) try: - return var.sqrt() + mss_numerator = mss.numerator + mss_denominator = mss.denominator except AttributeError: - return math.sqrt(var) + raise ValueError('inf or nan encountered in data') + if issubclass(T, Decimal): + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) def pstdev(data, mu=None): @@ -841,22 +638,21 @@ def pstdev(data, mu=None): 0.986893273527251 """ - # Fixme: Despite the exact sum of squared deviations, some inaccuracy - # remain because there are two rounding steps. The first occurs in - # the _convert() step for pvariance(), the second occurs in math.sqrt(). - var = pvariance(data, mu) + T, ss, c, n = _ss(data, mu) + if n < 1: + raise StatisticsError('pstdev requires at least one data point') + mss = ss / n try: - return var.sqrt() + mss_numerator = mss.numerator + mss_denominator = mss.denominator except AttributeError: - return math.sqrt(var) - + raise ValueError('inf or nan encountered in data') + if issubclass(T, Decimal): + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) -# === Statistics for relations between two inputs === - -# See https://en.wikipedia.org/wiki/Covariance -# https://en.wikipedia.org/wiki/Pearson_correlation_coefficient -# https://en.wikipedia.org/wiki/Simple_linear_regression +## Statistics for relations between two inputs ############################# def covariance(x, y, /): """Covariance @@ -875,6 +671,7 @@ def covariance(x, y, /): -7.5 """ + # https://en.wikipedia.org/wiki/Covariance n = len(x) if len(y) != n: raise StatisticsError('covariance requires that both inputs have same number of data points') @@ -882,18 +679,16 @@ def covariance(x, y, /): raise StatisticsError('covariance requires at least two data points') xbar = fsum(x) / n ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxy = sumprod((xi - xbar for xi in x), (yi - ybar for yi in y)) return sxy / (n - 1) -def correlation(x, y, /): +def correlation(x, y, /, *, method='linear'): """Pearson's correlation coefficient Return the Pearson's correlation coefficient for two inputs. Pearson's - correlation coefficient *r* takes values between -1 and +1. It measures the - strength and direction of the linear relationship, where +1 means very - strong, positive linear relationship, -1 very strong, negative linear - relationship, and 0 no linear relationship. + correlation coefficient *r* takes values between -1 and +1. It measures + the strength and direction of a linear relationship. >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] >>> y = [9, 8, 7, 6, 5, 4, 3, 2, 1] @@ -902,19 +697,43 @@ def correlation(x, y, /): >>> correlation(x, y) -1.0 + If *method* is "ranked", computes Spearman's rank correlation coefficient + for two inputs. The data is replaced by ranks. Ties are averaged + so that equal values receive the same rank. The resulting coefficient + measures the strength of a monotonic relationship. + + Spearman's rank correlation coefficient is appropriate for ordinal + data or for continuous data that doesn't meet the linear proportion + requirement for Pearson's correlation coefficient. + """ + # https://en.wikipedia.org/wiki/Pearson_correlation_coefficient + # https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') if n < 2: raise StatisticsError('correlation requires at least two data points') - xbar = fsum(x) / n - ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((xi - xbar) ** 2.0 for xi in x) - syy = fsum((yi - ybar) ** 2.0 for yi in y) + if method not in {'linear', 'ranked'}: + raise ValueError(f'Unknown method: {method!r}') + + if method == 'ranked': + start = (n - 1) / -2 # Center rankings around zero + x = _rank(x, start=start) + y = _rank(y, start=start) + + else: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] + y = [yi - ybar for yi in y] + + sxy = sumprod(x, y) + sxx = sumprod(x, x) + syy = sumprod(y, y) + try: - return sxy / sqrt(sxx * syy) + return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: raise StatisticsError('at least one of the inputs is constant') @@ -922,13 +741,13 @@ def correlation(x, y, /): LinearRegression = namedtuple('LinearRegression', ('slope', 'intercept')) -def linear_regression(x, y, /): +def linear_regression(x, y, /, *, proportional=False): """Slope and intercept for simple linear regression. Return the slope and intercept of simple linear regression parameters estimated using ordinary least squares. Simple linear regression describes relationship between an independent variable - *x* and a dependent variable *y* in terms of linear function: + *x* and a dependent variable *y* in terms of a linear function: y = slope * x + intercept + noise @@ -944,201 +763,557 @@ def linear_regression(x, y, /): >>> noise = NormalDist().samples(5, seed=42) >>> y = [3 * x[i] + 2 + noise[i] for i in range(5)] >>> linear_regression(x, y) #doctest: +ELLIPSIS - LinearRegression(slope=3.09078914170..., intercept=1.75684970486...) + LinearRegression(slope=3.17495..., intercept=1.00925...) + + If *proportional* is true, the independent variable *x* and the + dependent variable *y* are assumed to be directly proportional. + The data is fit to a line passing through the origin. + + Since the *intercept* will always be 0.0, the underlying linear + function simplifies to: + + y = slope * x + noise + + >>> y = [3 * x[i] + noise[i] for i in range(5)] + >>> linear_regression(x, y, proportional=True) #doctest: +ELLIPSIS + LinearRegression(slope=2.90475..., intercept=0.0) """ + # https://en.wikipedia.org/wiki/Simple_linear_regression n = len(x) if len(y) != n: raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') - xbar = fsum(x) / n - ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((xi - xbar) ** 2.0 for xi in x) + + if not proportional: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] # List because used three times below + y = (yi - ybar for yi in y) # Generator because only used once below + + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float + sxx = sumprod(x, x) + try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') - intercept = ybar - slope * xbar + + intercept = 0.0 if proportional else ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) -## Normal Distribution ##################################################### +## Kernel Density Estimation ############################################### + +_kernel_specs = {} + +def register(*kernels): + "Load the kernel's pdf, cdf, invcdf, and support into _kernel_specs." + def deco(builder): + spec = dict(zip(('pdf', 'cdf', 'invcdf', 'support'), builder())) + for kernel in kernels: + _kernel_specs[kernel] = spec + return builder + return deco + +@register('normal', 'gauss') +def normal_kernel(): + sqrt2pi = sqrt(2 * pi) + neg_sqrt2 = -sqrt(2) + pdf = lambda t: exp(-1/2 * t * t) / sqrt2pi + cdf = lambda t: 1/2 * erfc(t / neg_sqrt2) + invcdf = lambda t: _normal_dist_inv_cdf(t, 0.0, 1.0) + support = None + return pdf, cdf, invcdf, support + +@register('logistic') +def logistic_kernel(): + # 1.0 / (exp(t) + 2.0 + exp(-t)) + pdf = lambda t: 1/2 / (1.0 + cosh(t)) + cdf = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + invcdf = lambda p: log(p / (1.0 - p)) + support = None + return pdf, cdf, invcdf, support + +@register('sigmoid') +def sigmoid_kernel(): + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + c3 = pi / 2 + pdf = lambda t: c1 / cosh(t) + cdf = lambda t: c2 * atan(exp(t)) + invcdf = lambda p: log(tan(p * c3)) + support = None + return pdf, cdf, invcdf, support + +@register('rectangular', 'uniform') +def rectangular_kernel(): + pdf = lambda t: 1/2 + cdf = lambda t: 1/2 * t + 1/2 + invcdf = lambda p: 2.0 * p - 1.0 + support = 1.0 + return pdf, cdf, invcdf, support + +@register('triangular') +def triangular_kernel(): + pdf = lambda t: 1.0 - abs(t) + cdf = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + invcdf = lambda p: sqrt(2.0*p) - 1.0 if p < 1/2 else 1.0 - sqrt(2.0 - 2.0*p) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('parabolic', 'epanechnikov') +def parabolic_kernel(): + pdf = lambda t: 3/4 * (1.0 - t * t) + cdf = lambda t: sumprod((-1/4, 3/4, 1/2), (t**3, t, 1.0)) + invcdf = lambda p: 2.0 * cos((acos(2.0*p - 1.0) + pi) / 3.0) + support = 1.0 + return pdf, cdf, invcdf, support + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + if p < 0.0106: + return ((2.0 * p) ** 0.3838 - 1.0) * sign + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +@register('quartic', 'biweight') +def quartic_kernel(): + pdf = lambda t: 15/16 * (1.0 - t * t) ** 2 + cdf = lambda t: sumprod((3/16, -5/8, 15/16, 1/2), + (t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_quartic_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +def _triweight_invcdf_estimate(p): + # A handrolled piecewise approximation. There is no magic here. + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + if 0.00001 < p < 0.499: + x -= 0.033 * sin(1.07 * tau * (p - 0.035)) + return x * sign + +@register('triweight') +def triweight_kernel(): + pdf = lambda t: 35/32 * (1.0 - t * t) ** 3 + cdf = lambda t: sumprod((-5/32, 21/32, -35/32, 35/32, 1/2), + (t**7, t**5, t**3, t, 1.0)) + invcdf = _newton_raphson(_triweight_invcdf_estimate, f=cdf, f_prime=pdf) + support = 1.0 + return pdf, cdf, invcdf, support + +@register('cosine') +def cosine_kernel(): + c1 = pi / 4 + c2 = pi / 2 + pdf = lambda t: c1 * cos(c2 * t) + cdf = lambda t: 1/2 * sin(c2 * t) + 1/2 + invcdf = lambda p: 2.0 * asin(2.0 * p - 1.0) / pi + support = 1.0 + return pdf, cdf, invcdf, support + +del register, normal_kernel, logistic_kernel, sigmoid_kernel +del rectangular_kernel, triangular_kernel, parabolic_kernel +del quartic_kernel, triweight_kernel, cosine_kernel + + +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal (gauss) + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine + + If *cumulative* is true, will return a cumulative distribution function. + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: + + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + K = kernel_spec['pdf'] + W = kernel_spec['cdf'] + support = kernel_spec['support'] + + if support is None: + + def pdf(x): + return sum(K((x - x_i) / h) for x_i in data) / (len(data) * h) + + def cdf(x): + return sum(W((x - x_i) / h) for x_i in data) / len(data) -def _normal_dist_inv_cdf(p, mu, sigma): - # There is no closed-form solution to the inverse CDF for the normal - # distribution, so we use a rational approximation instead: - # Wichura, M.J. (1988). "Algorithm AS241: The Percentage Points of the - # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 - # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. - q = p - 0.5 - if fabs(q) <= 0.425: - r = 0.180625 - q * q - # Hash sum: 55.88319_28806_14901_4439 - num = (((((((2.50908_09287_30122_6727e+3 * r + - 3.34305_75583_58812_8105e+4) * r + - 6.72657_70927_00870_0853e+4) * r + - 4.59219_53931_54987_1457e+4) * r + - 1.37316_93765_50946_1125e+4) * r + - 1.97159_09503_06551_4427e+3) * r + - 1.33141_66789_17843_7745e+2) * r + - 3.38713_28727_96366_6080e+0) * q - den = (((((((5.22649_52788_52854_5610e+3 * r + - 2.87290_85735_72194_2674e+4) * r + - 3.93078_95800_09271_0610e+4) * r + - 2.12137_94301_58659_5867e+4) * r + - 5.39419_60214_24751_1077e+3) * r + - 6.87187_00749_20579_0830e+2) * r + - 4.23133_30701_60091_1252e+1) * r + - 1.0) - x = num / den - return mu + (x * sigma) - r = p if q <= 0.0 else 1.0 - p - r = sqrt(-log(r)) - if r <= 5.0: - r = r - 1.6 - # Hash sum: 49.33206_50330_16102_89036 - num = (((((((7.74545_01427_83414_07640e-4 * r + - 2.27238_44989_26918_45833e-2) * r + - 2.41780_72517_74506_11770e-1) * r + - 1.27045_82524_52368_38258e+0) * r + - 3.64784_83247_63204_60504e+0) * r + - 5.76949_72214_60691_40550e+0) * r + - 4.63033_78461_56545_29590e+0) * r + - 1.42343_71107_49683_57734e+0) - den = (((((((1.05075_00716_44416_84324e-9 * r + - 5.47593_80849_95344_94600e-4) * r + - 1.51986_66563_61645_71966e-2) * r + - 1.48103_97642_74800_74590e-1) * r + - 6.89767_33498_51000_04550e-1) * r + - 1.67638_48301_83803_84940e+0) * r + - 2.05319_16266_37758_82187e+0) * r + - 1.0) else: - r = r - 5.0 - # Hash sum: 47.52583_31754_92896_71629 - num = (((((((2.01033_43992_92288_13265e-7 * r + - 2.71155_55687_43487_57815e-5) * r + - 1.24266_09473_88078_43860e-3) * r + - 2.65321_89526_57612_30930e-2) * r + - 2.96560_57182_85048_91230e-1) * r + - 1.78482_65399_17291_33580e+0) * r + - 5.46378_49111_64114_36990e+0) * r + - 6.65790_46435_01103_77720e+0) - den = (((((((2.04426_31033_89939_78564e-15 * r + - 1.42151_17583_16445_88870e-7) * r + - 1.84631_83175_10054_68180e-5) * r + - 7.86869_13114_56132_59100e-4) * r + - 1.48753_61290_85061_48525e-2) * r + - 1.36929_88092_27358_05310e-1) * r + - 5.99832_20655_58879_37690e-1) * r + - 1.0) - x = num / den - if q < 0.0: - x = -x - return mu + (x * sigma) + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n + + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf -# If available, use C implementation -try: - from _statistics import _normal_dist_inv_cdf -except ImportError: - pass + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf -class NormalDist: - "Normal distribution of a random variable" - # https://en.wikipedia.org/wiki/Normal_distribution - # https://en.wikipedia.org/wiki/Variance#Properties +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). - __slots__ = { - '_mu': 'Arithmetic mean of a normal distribution', - '_sigma': 'Standard deviation of a normal distribution', - } + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. - def __init__(self, mu=0.0, sigma=1.0): - "NormalDist where mu is the mean and sigma is the standard deviation." - if sigma < 0.0: - raise StatisticsError('sigma must be non-negative') - self._mu = float(mu) - self._sigma = float(sigma) + A StatisticsError will be raised if the *data* sequence is empty. - @classmethod - def from_samples(cls, data): - "Make a normal distribution instance from sample data." - if not isinstance(data, (list, tuple)): - data = list(data) - xbar = fmean(data) - return cls(xbar, stdev(data, xbar)) + Example: - def samples(self, n, *, seed=None): - "Generate *n* samples for a given mean and standard deviation." - gauss = random.gauss if seed is None else random.Random(seed).gauss - mu, sigma = self._mu, self._sigma - return [gauss(mu, sigma) for i in range(n)] + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] - def pdf(self, x): - "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma ** 2.0 - if not variance: - raise StatisticsError('pdf() not defined when sigma is zero') - return exp((x - self._mu)**2.0 / (-2.0*variance)) / sqrt(tau*variance) + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') - def cdf(self, x): - "Cumulative distribution function. P(X <= x)" - if not self._sigma: - raise StatisticsError('cdf() not defined when sigma is zero') - return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * sqrt(2.0)))) + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') - def inv_cdf(self, p): - """Inverse cumulative distribution function. x : P(X <= x) = p + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - Finds the value of the random variable such that the probability of - the variable being less than or equal to that value equals the given - probability. + kernel_spec = _kernel_specs.get(kernel) + if kernel_spec is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + invcdf = kernel_spec['invcdf'] - This function is also called the percent point function or quantile - function. - """ - if p <= 0.0 or p >= 1.0: - raise StatisticsError('p must be in the range 0.0 < p < 1.0') - if self._sigma <= 0.0: - raise StatisticsError('cdf() not defined when sigma at or below zero') - return _normal_dist_inv_cdf(p, self._mu, self._sigma) + prng = _random.Random(seed) + random = prng.random + choice = prng.choice - def quantiles(self, n=4): - """Divide into *n* continuous intervals with equal probability. + def rand(): + return choice(data) + h * invcdf(random()) - Returns a list of (n - 1) cut points separating the intervals. + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution in to 100 equal sized groups. - """ - return [self.inv_cdf(i / n) for i in range(1, n)] + return rand - def overlap(self, other): - """Compute the overlapping coefficient (OVL) between two normal distributions. - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving the overlapping area in - the two underlying probability density functions. +## Quantiles ############################################################### - >>> N1 = NormalDist(2.4, 1.6) - >>> N2 = NormalDist(3.2, 2.0) - >>> N1.overlap(N2) - 0.8035050657330205 - """ - # See: "The overlapping coefficient as a measure of agreement between - # probability distributions and point estimation of the overlap of two - # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr - # http://dx.doi.org/10.1080/03610928908830127 - if not isinstance(other, NormalDist): - raise TypeError('Expected another NormalDist instance') - X, Y = self, other - if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity +# There is no one perfect way to compute quantiles. Here we offer +# two methods that serve common needs. Most other packages +# surveyed offered at least one or both of these two, making them +# "standard" in the sense of "widely-adopted and reproducible". +# They are also easy to explain, easy to compute manually, and have +# straight-forward interpretations that aren't surprising. + +# The default method is known as "R6", "PERCENTILE.EXC", or "expected +# value of rank order statistics". The alternative method is known as +# "R7", "PERCENTILE.INC", or "mode of rank order statistics". + +# For sample data where there is a positive probability for values +# beyond the range of the data, the R6 exclusive method is a +# reasonable choice. Consider a random sample of nine values from a +# population with a uniform distribution from 0.0 to 1.0. The +# distribution of the third ranked sample point is described by +# betavariate(alpha=3, beta=7) which has mode=0.250, median=0.286, and +# mean=0.300. Only the latter (which corresponds with R6) gives the +# desired cut point with 30% of the population falling below that +# value, making it comparable to a result from an inv_cdf() function. +# The R6 exclusive method is also idempotent. + +# For describing population data where the end points are known to +# be included in the data, the R7 inclusive method is a reasonable +# choice. Instead of the mean, it uses the mode of the beta +# distribution for the interior points. Per Hyndman & Fan, "One nice +# property is that the vertices of Q7(p) divide the range into n - 1 +# intervals, and exactly 100p% of the intervals lie to the left of +# Q7(p) and 100(1 - p)% of the intervals lie to the right of Q7(p)." + +# If needed, other methods could be added. However, for now, the +# position is that fewer options make for easier choices and that +# external packages can be used for anything more advanced. + +def quantiles(data, *, n=4, method='exclusive'): + """Divide *data* into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate *data* in to 100 equal sized groups. + + The *data* can be any iterable containing sample. + The cut points are linearly interpolated between data points. + + If *method* is set to *inclusive*, *data* is treated as population + data. The minimum value is treated as the 0th percentile and the + maximum value is treated as the 100th percentile. + + """ + if n < 1: + raise StatisticsError('n must be at least 1') + + data = sorted(data) + + ld = len(data) + if ld < 2: + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + + if method == 'inclusive': + m = ld - 1 + result = [] + for i in range(1, n): + j, delta = divmod(i * m, n) + interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n + result.append(interpolated) + return result + + if method == 'exclusive': + m = ld + 1 + result = [] + for i in range(1, n): + j = i * m // n # rescale i to m/n + j = 1 if j < 1 else ld-1 if j > ld-1 else j # clamp to 1 .. ld-1 + delta = i*m - j*n # exact integer math + interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n + result.append(interpolated) + return result + + raise ValueError(f'Unknown method: {method!r}') + + +## Normal Distribution ##################################################### + +class NormalDist: + "Normal distribution of a random variable" + # https://en.wikipedia.org/wiki/Normal_distribution + # https://en.wikipedia.org/wiki/Variance#Properties + + __slots__ = { + '_mu': 'Arithmetic mean of a normal distribution', + '_sigma': 'Standard deviation of a normal distribution', + } + + def __init__(self, mu=0.0, sigma=1.0): + "NormalDist where mu is the mean and sigma is the standard deviation." + if sigma < 0.0: + raise StatisticsError('sigma must be non-negative') + self._mu = float(mu) + self._sigma = float(sigma) + + @classmethod + def from_samples(cls, data): + "Make a normal distribution instance from sample data." + return cls(*_mean_stdev(data)) + + def samples(self, n, *, seed=None): + "Generate *n* samples for a given mean and standard deviation." + rnd = random.random if seed is None else random.Random(seed).random + inv_cdf = _normal_dist_inv_cdf + mu = self._mu + sigma = self._sigma + return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] + + def pdf(self, x): + "Probability density function. P(x <= X < x+dx) / dx" + variance = self._sigma * self._sigma + if not variance: + raise StatisticsError('pdf() not defined when sigma is zero') + diff = x - self._mu + return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) + + def cdf(self, x): + "Cumulative distribution function. P(X <= x)" + if not self._sigma: + raise StatisticsError('cdf() not defined when sigma is zero') + return 0.5 * erfc((self._mu - x) / (self._sigma * _SQRT2)) + + def inv_cdf(self, p): + """Inverse cumulative distribution function. x : P(X <= x) = p + + Finds the value of the random variable such that the probability of + the variable being less than or equal to that value equals the given + probability. + + This function is also called the percent point function or quantile + function. + """ + if p <= 0.0 or p >= 1.0: + raise StatisticsError('p must be in the range 0.0 < p < 1.0') + return _normal_dist_inv_cdf(p, self._mu, self._sigma) + + def quantiles(self, n=4): + """Divide into *n* continuous intervals with equal probability. + + Returns a list of (n - 1) cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution in to 100 equal sized groups. + """ + return [self.inv_cdf(i / n) for i in range(1, n)] + + def overlap(self, other): + """Compute the overlapping coefficient (OVL) between two normal distributions. + + Measures the agreement between two normal probability distributions. + Returns a value between 0.0 and 1.0 giving the overlapping area in + the two underlying probability density functions. + + >>> N1 = NormalDist(2.4, 1.6) + >>> N2 = NormalDist(3.2, 2.0) + >>> N1.overlap(N2) + 0.8035050657330205 + """ + # See: "The overlapping coefficient as a measure of agreement between + # probability distributions and point estimation of the overlap of two + # normal densities" -- Henry F. Inman and Edwin L. Bradley Jr + # http://dx.doi.org/10.1080/03610928908830127 + if not isinstance(other, NormalDist): + raise TypeError('Expected another NormalDist instance') + X, Y = self, other + if (Y._sigma, Y._mu) < (X._sigma, X._mu): # sort to assure commutativity X, Y = Y, X X_var, Y_var = X.variance, Y.variance if not X_var or not Y_var: @@ -1146,9 +1321,9 @@ def overlap(self, other): dv = Y_var - X_var dm = fabs(Y._mu - X._mu) if not dv: - return 1.0 - erf(dm / (2.0 * X._sigma * sqrt(2.0))) + return erfc(dm / (2.0 * X._sigma * _SQRT2)) a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm**2.0 + dv * log(Y_var / X_var)) + b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) x1 = (a + b) / dv x2 = (a - b) / dv return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) @@ -1191,7 +1366,7 @@ def stdev(self): @property def variance(self): "Square of the standard deviation." - return self._sigma ** 2.0 + return self._sigma * self._sigma def __add__(x1, x2): """Add a constant or another NormalDist instance. @@ -1265,3 +1440,440 @@ def __hash__(self): def __repr__(self): return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state + + +## Private utilities ####################################################### + +def _sum(data): + """_sum(data) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. + + Examples + -------- + + >>> _sum([3, 2.25, 4.5, -0.5, 0.25]) + (, Fraction(19, 2), 5) + + Some sources of round-off error will be avoided: + + # Built-in sum returns zero. + >>> _sum([1e50, 1, -1e50] * 1000) + (, Fraction(1000, 1), 3000) + + Fractions and Decimals are also supported: + + >>> from fractions import Fraction as F + >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) + (, Fraction(63, 20), 4) + + >>> from decimal import Decimal as D + >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] + >>> _sum(data) + (, Fraction(6963, 10000), 4) + + Mixed types are currently treated as an error, except that int is + allowed. + + """ + count = 0 + types = set() + types_add = types.add + partials = {} + partials_get = partials.get + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n + + if None in partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + total = sum(Fraction(n, d) for d, n in partials.items()) + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, total, count) + + +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + + if not count: + ssd = c = Fraction(0) + + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) + + +def _exact_ratio(x): + """Return Real number x to exact (numerator, denominator) pair. + + >>> _exact_ratio(0.25) + (1, 4) + + x is expected to be an int, Fraction, Decimal or float. + + """ + try: + return x.as_integer_ratio() + except AttributeError: + pass + except (OverflowError, ValueError): + # float NAN or INF. + assert not _isfinite(x) + return (x, None) + + try: + # x may be an Integral ABC. + return (x.numerator, x.denominator) + except AttributeError: + msg = f"can't convert type '{type(x).__name__}' to numerator/denominator" + raise TypeError(msg) + + +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + + if issubclass(T, int) and value.denominator != 1: + T = float + + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator) / T(value.denominator) + else: + raise + + +def _fail_neg(values, errmsg='negative value'): + """Iterate over values, failing if any are less than zero.""" + for x in values: + if x < 0: + raise StatisticsError(errmsg) + yield x + + +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. + + Ties are averaged so that equal values receive the same rank: + + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + The operation is idempotent: + + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: + + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] + + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: + + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] + + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. + + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result + + +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) + + +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 + + +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float + + +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m + + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() + + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus + + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus + + return root + + +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + try: + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) + except AttributeError: + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) + + +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." + + h = sqrt(x * y) + + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h + + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) + + +def _normal_dist_inv_cdf(p, mu, sigma): + # There is no closed-form solution to the inverse CDF for the normal + # distribution, so we use a rational approximation instead: + # Wichura, M.J. (1988). "Algorithm AS241: The Percentage Points of the + # Normal Distribution". Applied Statistics. Blackwell Publishing. 37 + # (3): 477–484. doi:10.2307/2347330. JSTOR 2347330. + q = p - 0.5 + + if fabs(q) <= 0.425: + r = 0.180625 - q * q + # Hash sum: 55.88319_28806_14901_4439 + num = (((((((2.50908_09287_30122_6727e+3 * r + + 3.34305_75583_58812_8105e+4) * r + + 6.72657_70927_00870_0853e+4) * r + + 4.59219_53931_54987_1457e+4) * r + + 1.37316_93765_50946_1125e+4) * r + + 1.97159_09503_06551_4427e+3) * r + + 1.33141_66789_17843_7745e+2) * r + + 3.38713_28727_96366_6080e+0) * q + den = (((((((5.22649_52788_52854_5610e+3 * r + + 2.87290_85735_72194_2674e+4) * r + + 3.93078_95800_09271_0610e+4) * r + + 2.12137_94301_58659_5867e+4) * r + + 5.39419_60214_24751_1077e+3) * r + + 6.87187_00749_20579_0830e+2) * r + + 4.23133_30701_60091_1252e+1) * r + + 1.0) + x = num / den + return mu + (x * sigma) + + r = p if q <= 0.0 else 1.0 - p + r = sqrt(-log(r)) + if r <= 5.0: + r = r - 1.6 + # Hash sum: 49.33206_50330_16102_89036 + num = (((((((7.74545_01427_83414_07640e-4 * r + + 2.27238_44989_26918_45833e-2) * r + + 2.41780_72517_74506_11770e-1) * r + + 1.27045_82524_52368_38258e+0) * r + + 3.64784_83247_63204_60504e+0) * r + + 5.76949_72214_60691_40550e+0) * r + + 4.63033_78461_56545_29590e+0) * r + + 1.42343_71107_49683_57734e+0) + den = (((((((1.05075_00716_44416_84324e-9 * r + + 5.47593_80849_95344_94600e-4) * r + + 1.51986_66563_61645_71966e-2) * r + + 1.48103_97642_74800_74590e-1) * r + + 6.89767_33498_51000_04550e-1) * r + + 1.67638_48301_83803_84940e+0) * r + + 2.05319_16266_37758_82187e+0) * r + + 1.0) + else: + r = r - 5.0 + # Hash sum: 47.52583_31754_92896_71629 + num = (((((((2.01033_43992_92288_13265e-7 * r + + 2.71155_55687_43487_57815e-5) * r + + 1.24266_09473_88078_43860e-3) * r + + 2.65321_89526_57612_30930e-2) * r + + 2.96560_57182_85048_91230e-1) * r + + 1.78482_65399_17291_33580e+0) * r + + 5.46378_49111_64114_36990e+0) * r + + 6.65790_46435_01103_77720e+0) + den = (((((((2.04426_31033_89939_78564e-15 * r + + 1.42151_17583_16445_88870e-7) * r + + 1.84631_83175_10054_68180e-5) * r + + 7.86869_13114_56132_59100e-4) * r + + 1.48753_61290_85061_48525e-2) * r + + 1.36929_88092_27358_05310e-1) * r + + 5.99832_20655_58879_37690e-1) * r + + 1.0) + + x = num / den + if q < 0.0: + x = -x + + return mu + (x * sigma) + + +# If available, use C implementation +try: + from _statistics import _normal_dist_inv_cdf +except ImportError: + pass diff --git a/Lib/string.py b/Lib/string.py deleted file mode 100644 index 489777b10c2..00000000000 --- a/Lib/string.py +++ /dev/null @@ -1,280 +0,0 @@ -"""A collection of string constants. - -Public module variables: - -whitespace -- a string containing all ASCII whitespace -ascii_lowercase -- a string containing all ASCII lowercase letters -ascii_uppercase -- a string containing all ASCII uppercase letters -ascii_letters -- a string containing all ASCII letters -digits -- a string containing all ASCII decimal digits -hexdigits -- a string containing all ASCII hexadecimal digits -octdigits -- a string containing all ASCII octal digits -punctuation -- a string containing all ASCII punctuation characters -printable -- a string containing all ASCII characters considered printable - -""" - -__all__ = ["ascii_letters", "ascii_lowercase", "ascii_uppercase", "capwords", - "digits", "hexdigits", "octdigits", "printable", "punctuation", - "whitespace", "Formatter", "Template"] - -import _string - -# Some strings for ctype-style character classification -whitespace = ' \t\n\r\v\f' -ascii_lowercase = 'abcdefghijklmnopqrstuvwxyz' -ascii_uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' -ascii_letters = ascii_lowercase + ascii_uppercase -digits = '0123456789' -hexdigits = digits + 'abcdef' + 'ABCDEF' -octdigits = '01234567' -punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" -printable = digits + ascii_letters + punctuation + whitespace - -# Functions which aren't available as string methods. - -# Capitalize the words in a string, e.g. " aBc dEf " -> "Abc Def". -def capwords(s, sep=None): - """capwords(s [,sep]) -> string - - Split the argument into words using split, capitalize each - word using capitalize, and join the capitalized words using - join. If the optional second argument sep is absent or None, - runs of whitespace characters are replaced by a single space - and leading and trailing whitespace are removed, otherwise - sep is used to split and join the words. - - """ - return (sep or ' ').join(x.capitalize() for x in s.split(sep)) - - -#################################################################### -import re as _re -from collections import ChainMap as _ChainMap - -_sentinel_dict = {} - -class Template: - """A string class for supporting $-substitutions.""" - - delimiter = '$' - # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE, but - # without the ASCII flag. We can't add re.ASCII to flags because of - # backward compatibility. So we use the ?a local flag and [a-z] pattern. - # See https://bugs.python.org/issue31672 - idpattern = r'(?a:[_a-z][_a-z0-9]*)' - braceidpattern = None - flags = _re.IGNORECASE - - def __init_subclass__(cls): - super().__init_subclass__() - if 'pattern' in cls.__dict__: - pattern = cls.pattern - else: - delim = _re.escape(cls.delimiter) - id = cls.idpattern - bid = cls.braceidpattern or cls.idpattern - pattern = fr""" - {delim}(?: - (?P{delim}) | # Escape sequence of two delimiters - (?P{id}) | # delimiter and a Python identifier - {{(?P{bid})}} | # delimiter and a braced identifier - (?P) # Other ill-formed delimiter exprs - ) - """ - cls.pattern = _re.compile(pattern, cls.flags | _re.VERBOSE) - - def __init__(self, template): - self.template = template - - # Search for $$, $identifier, ${identifier}, and any bare $'s - - def _invalid(self, mo): - i = mo.start('invalid') - lines = self.template[:i].splitlines(keepends=True) - if not lines: - colno = 1 - lineno = 1 - else: - colno = i - len(''.join(lines[:-1])) - lineno = len(lines) - raise ValueError('Invalid placeholder in string: line %d, col %d' % - (lineno, colno)) - - def substitute(self, mapping=_sentinel_dict, /, **kws): - if mapping is _sentinel_dict: - mapping = kws - elif kws: - mapping = _ChainMap(kws, mapping) - # Helper function for .sub() - def convert(mo): - # Check the most common path first. - named = mo.group('named') or mo.group('braced') - if named is not None: - return str(mapping[named]) - if mo.group('escaped') is not None: - return self.delimiter - if mo.group('invalid') is not None: - self._invalid(mo) - raise ValueError('Unrecognized named group in pattern', - self.pattern) - return self.pattern.sub(convert, self.template) - - def safe_substitute(self, mapping=_sentinel_dict, /, **kws): - if mapping is _sentinel_dict: - mapping = kws - elif kws: - mapping = _ChainMap(kws, mapping) - # Helper function for .sub() - def convert(mo): - named = mo.group('named') or mo.group('braced') - if named is not None: - try: - return str(mapping[named]) - except KeyError: - return mo.group() - if mo.group('escaped') is not None: - return self.delimiter - if mo.group('invalid') is not None: - return mo.group() - raise ValueError('Unrecognized named group in pattern', - self.pattern) - return self.pattern.sub(convert, self.template) - -# Initialize Template.pattern. __init_subclass__() is automatically called -# only for subclasses, not for the Template class itself. -Template.__init_subclass__() - - -######################################################################## -# the Formatter class -# see PEP 3101 for details and purpose of this class - -# The hard parts are reused from the C implementation. They're exposed as "_" -# prefixed methods of str. - -# The overall parser is implemented in _string.formatter_parser. -# The field name parser is implemented in _string.formatter_field_name_split - -class Formatter: - def format(self, format_string, /, *args, **kwargs): - return self.vformat(format_string, args, kwargs) - - def vformat(self, format_string, args, kwargs): - used_args = set() - result, _ = self._vformat(format_string, args, kwargs, used_args, 2) - self.check_unused_args(used_args, args, kwargs) - return result - - def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, - auto_arg_index=0): - if recursion_depth < 0: - raise ValueError('Max string recursion exceeded') - result = [] - for literal_text, field_name, format_spec, conversion in \ - self.parse(format_string): - - # output the literal text - if literal_text: - result.append(literal_text) - - # if there's a field, output it - if field_name is not None: - # this is some markup, find the object and do - # the formatting - - # handle arg indexing when empty field_names are given. - if field_name == '': - if auto_arg_index is False: - raise ValueError('cannot switch from manual field ' - 'specification to automatic field ' - 'numbering') - field_name = str(auto_arg_index) - auto_arg_index += 1 - elif field_name.isdigit(): - if auto_arg_index: - raise ValueError('cannot switch from manual field ' - 'specification to automatic field ' - 'numbering') - # disable auto arg incrementing, if it gets - # used later on, then an exception will be raised - auto_arg_index = False - - # given the field_name, find the object it references - # and the argument it came from - obj, arg_used = self.get_field(field_name, args, kwargs) - used_args.add(arg_used) - - # do any conversion on the resulting object - obj = self.convert_field(obj, conversion) - - # expand the format spec, if needed - format_spec, auto_arg_index = self._vformat( - format_spec, args, kwargs, - used_args, recursion_depth-1, - auto_arg_index=auto_arg_index) - - # format the object and append to the result - result.append(self.format_field(obj, format_spec)) - - return ''.join(result), auto_arg_index - - - def get_value(self, key, args, kwargs): - if isinstance(key, int): - return args[key] - else: - return kwargs[key] - - - def check_unused_args(self, used_args, args, kwargs): - pass - - - def format_field(self, value, format_spec): - return format(value, format_spec) - - - def convert_field(self, value, conversion): - # do any conversion on the resulting object - if conversion is None: - return value - elif conversion == 's': - return str(value) - elif conversion == 'r': - return repr(value) - elif conversion == 'a': - return ascii(value) - raise ValueError("Unknown conversion specifier {0!s}".format(conversion)) - - - # returns an iterable that contains tuples of the form: - # (literal_text, field_name, format_spec, conversion) - # literal_text can be zero length - # field_name can be None, in which case there's no - # object to format and output - # if field_name is not None, it is looked up, formatted - # with format_spec and conversion and then used - def parse(self, format_string): - return _string.formatter_parser(format_string) - - - # given a field_name, find the object it references. - # field_name: the field being looked up, e.g. "0.name" - # or "lookup[3]" - # used_args: a set of which args have been used - # args, kwargs: as passed in to vformat - def get_field(self, field_name, args, kwargs): - first, rest = _string.formatter_field_name_split(field_name) - - obj = self.get_value(first, args, kwargs) - - # loop through the rest of the field_name, doing - # getattr or getitem as needed - for is_attr, i in rest: - if is_attr: - obj = getattr(obj, i) - else: - obj = obj[i] - - return obj, first diff --git a/Lib/string/__init__.py b/Lib/string/__init__.py new file mode 100644 index 00000000000..eab5067c9b1 --- /dev/null +++ b/Lib/string/__init__.py @@ -0,0 +1,325 @@ +"""A collection of string constants. + +Public module variables: + +whitespace -- a string containing all ASCII whitespace +ascii_lowercase -- a string containing all ASCII lowercase letters +ascii_uppercase -- a string containing all ASCII uppercase letters +ascii_letters -- a string containing all ASCII letters +digits -- a string containing all ASCII decimal digits +hexdigits -- a string containing all ASCII hexadecimal digits +octdigits -- a string containing all ASCII octal digits +punctuation -- a string containing all ASCII punctuation characters +printable -- a string containing all ASCII characters considered printable + +""" + +__all__ = ["ascii_letters", "ascii_lowercase", "ascii_uppercase", "capwords", + "digits", "hexdigits", "octdigits", "printable", "punctuation", + "whitespace", "Formatter", "Template"] + +import _string + +# Some strings for ctype-style character classification +whitespace = ' \t\n\r\v\f' +ascii_lowercase = 'abcdefghijklmnopqrstuvwxyz' +ascii_uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' +ascii_letters = ascii_lowercase + ascii_uppercase +digits = '0123456789' +hexdigits = digits + 'abcdef' + 'ABCDEF' +octdigits = '01234567' +punctuation = r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" +printable = digits + ascii_letters + punctuation + whitespace + +# Functions which aren't available as string methods. + +# Capitalize the words in a string, e.g. " aBc dEf " -> "Abc Def". +def capwords(s, sep=None): + """capwords(s [,sep]) -> string + + Split the argument into words using split, capitalize each + word using capitalize, and join the capitalized words using + join. If the optional second argument sep is absent or None, + runs of whitespace characters are replaced by a single space + and leading and trailing whitespace are removed, otherwise + sep is used to split and join the words. + + """ + return (sep or ' ').join(map(str.capitalize, s.split(sep))) + + +#################################################################### +_sentinel_dict = {} + + +class _TemplatePattern: + # This descriptor is overwritten in ``Template._compile_pattern()``. + def __get__(self, instance, cls=None): + if cls is None: + return self + return cls._compile_pattern() +_TemplatePattern = _TemplatePattern() + + +class Template: + """A string class for supporting $-substitutions.""" + + delimiter = '$' + # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE, but + # without the ASCII flag. We can't add re.ASCII to flags because of + # backward compatibility. So we use the ?a local flag and [a-z] pattern. + # See https://bugs.python.org/issue31672 + idpattern = r'(?a:[_a-z][_a-z0-9]*)' + braceidpattern = None + flags = None # default: re.IGNORECASE + + pattern = _TemplatePattern # use a descriptor to compile the pattern + + def __init_subclass__(cls): + super().__init_subclass__() + cls._compile_pattern() + + @classmethod + def _compile_pattern(cls): + import re # deferred import, for performance + + pattern = cls.__dict__.get('pattern', _TemplatePattern) + if pattern is _TemplatePattern: + delim = re.escape(cls.delimiter) + id = cls.idpattern + bid = cls.braceidpattern or cls.idpattern + pattern = fr""" + {delim}(?: + (?P{delim}) | # Escape sequence of two delimiters + (?P{id}) | # delimiter and a Python identifier + {{(?P{bid})}} | # delimiter and a braced identifier + (?P) # Other ill-formed delimiter exprs + ) + """ + if cls.flags is None: + cls.flags = re.IGNORECASE + pat = cls.pattern = re.compile(pattern, cls.flags | re.VERBOSE) + return pat + + def __init__(self, template): + self.template = template + + # Search for $$, $identifier, ${identifier}, and any bare $'s + + def _invalid(self, mo): + i = mo.start('invalid') + lines = self.template[:i].splitlines(keepends=True) + if not lines: + colno = 1 + lineno = 1 + else: + colno = i - len(''.join(lines[:-1])) + lineno = len(lines) + raise ValueError('Invalid placeholder in string: line %d, col %d' % + (lineno, colno)) + + def substitute(self, mapping=_sentinel_dict, /, **kws): + if mapping is _sentinel_dict: + mapping = kws + elif kws: + from collections import ChainMap + mapping = ChainMap(kws, mapping) + # Helper function for .sub() + def convert(mo): + # Check the most common path first. + named = mo.group('named') or mo.group('braced') + if named is not None: + return str(mapping[named]) + if mo.group('escaped') is not None: + return self.delimiter + if mo.group('invalid') is not None: + self._invalid(mo) + raise ValueError('Unrecognized named group in pattern', + self.pattern) + return self.pattern.sub(convert, self.template) + + def safe_substitute(self, mapping=_sentinel_dict, /, **kws): + if mapping is _sentinel_dict: + mapping = kws + elif kws: + from collections import ChainMap + mapping = ChainMap(kws, mapping) + # Helper function for .sub() + def convert(mo): + named = mo.group('named') or mo.group('braced') + if named is not None: + try: + return str(mapping[named]) + except KeyError: + return mo.group() + if mo.group('escaped') is not None: + return self.delimiter + if mo.group('invalid') is not None: + return mo.group() + raise ValueError('Unrecognized named group in pattern', + self.pattern) + return self.pattern.sub(convert, self.template) + + def is_valid(self): + for mo in self.pattern.finditer(self.template): + if mo.group('invalid') is not None: + return False + if (mo.group('named') is None + and mo.group('braced') is None + and mo.group('escaped') is None): + # If all the groups are None, there must be + # another group we're not expecting + raise ValueError('Unrecognized named group in pattern', + self.pattern) + return True + + def get_identifiers(self): + ids = [] + for mo in self.pattern.finditer(self.template): + named = mo.group('named') or mo.group('braced') + if named is not None and named not in ids: + # add a named group only the first time it appears + ids.append(named) + elif (named is None + and mo.group('invalid') is None + and mo.group('escaped') is None): + # If all the groups are None, there must be + # another group we're not expecting + raise ValueError('Unrecognized named group in pattern', + self.pattern) + return ids + + +######################################################################## +# the Formatter class +# see PEP 3101 for details and purpose of this class + +# The hard parts are reused from the C implementation. They're exposed as "_" +# prefixed methods of str. + +# The overall parser is implemented in _string.formatter_parser. +# The field name parser is implemented in _string.formatter_field_name_split + +class Formatter: + def format(self, format_string, /, *args, **kwargs): + return self.vformat(format_string, args, kwargs) + + def vformat(self, format_string, args, kwargs): + used_args = set() + result, _ = self._vformat(format_string, args, kwargs, used_args, 2) + self.check_unused_args(used_args, args, kwargs) + return result + + def _vformat(self, format_string, args, kwargs, used_args, recursion_depth, + auto_arg_index=0): + if recursion_depth < 0: + raise ValueError('Max string recursion exceeded') + result = [] + for literal_text, field_name, format_spec, conversion in \ + self.parse(format_string): + + # output the literal text + if literal_text: + result.append(literal_text) + + # if there's a field, output it + if field_name is not None: + # this is some markup, find the object and do + # the formatting + + # handle arg indexing when empty field first parts are given. + field_first, _ = _string.formatter_field_name_split(field_name) + if field_first == '': + if auto_arg_index is False: + raise ValueError('cannot switch from manual field ' + 'specification to automatic field ' + 'numbering') + field_name = str(auto_arg_index) + field_name + auto_arg_index += 1 + elif isinstance(field_first, int): + if auto_arg_index: + raise ValueError('cannot switch from automatic field ' + 'numbering to manual field ' + 'specification') + # disable auto arg incrementing, if it gets + # used later on, then an exception will be raised + auto_arg_index = False + + # given the field_name, find the object it references + # and the argument it came from + obj, arg_used = self.get_field(field_name, args, kwargs) + used_args.add(arg_used) + + # do any conversion on the resulting object + obj = self.convert_field(obj, conversion) + + # expand the format spec, if needed + format_spec, auto_arg_index = self._vformat( + format_spec, args, kwargs, + used_args, recursion_depth-1, + auto_arg_index=auto_arg_index) + + # format the object and append to the result + result.append(self.format_field(obj, format_spec)) + + return ''.join(result), auto_arg_index + + + def get_value(self, key, args, kwargs): + if isinstance(key, int): + return args[key] + else: + return kwargs[key] + + + def check_unused_args(self, used_args, args, kwargs): + pass + + + def format_field(self, value, format_spec): + return format(value, format_spec) + + + def convert_field(self, value, conversion): + # do any conversion on the resulting object + if conversion is None: + return value + elif conversion == 's': + return str(value) + elif conversion == 'r': + return repr(value) + elif conversion == 'a': + return ascii(value) + raise ValueError("Unknown conversion specifier {0!s}".format(conversion)) + + + # returns an iterable that contains tuples of the form: + # (literal_text, field_name, format_spec, conversion) + # literal_text can be zero length + # field_name can be None, in which case there's no + # object to format and output + # if field_name is not None, it is looked up, formatted + # with format_spec and conversion and then used + def parse(self, format_string): + return _string.formatter_parser(format_string) + + + # given a field_name, find the object it references. + # field_name: the field being looked up, e.g. "0.name" + # or "lookup[3]" + # used_args: a set of which args have been used + # args, kwargs: as passed in to vformat + def get_field(self, field_name, args, kwargs): + first, rest = _string.formatter_field_name_split(field_name) + + obj = self.get_value(first, args, kwargs) + + # loop through the rest of the field_name, doing + # getattr or getitem as needed + for is_attr, i in rest: + if is_attr: + obj = getattr(obj, i) + else: + obj = obj[i] + + return obj, first diff --git a/Lib/string/templatelib.py b/Lib/string/templatelib.py new file mode 100644 index 00000000000..8164872432a --- /dev/null +++ b/Lib/string/templatelib.py @@ -0,0 +1,33 @@ +"""Support for template string literals (t-strings).""" + +t = t"{0}" +Template = type(t) +Interpolation = type(t.interpolations[0]) +del t + +def convert(obj, /, conversion): + """Convert *obj* using formatted string literal semantics.""" + if conversion is None: + return obj + if conversion == 'r': + return repr(obj) + if conversion == 's': + return str(obj) + if conversion == 'a': + return ascii(obj) + raise ValueError(f'invalid conversion specifier: {conversion}') + +def _template_unpickle(*args): + import itertools + + if len(args) != 2: + raise ValueError('Template expects tuple of length 2 to unpickle') + + strings, interpolations = args + parts = [] + for string, interpolation in itertools.zip_longest(strings, interpolations): + if string is not None: + parts.append(string) + if interpolation is not None: + parts.append(interpolation) + return Template(*parts) diff --git a/Lib/struct.py b/Lib/struct.py index d6bba588636..ff98e8c4cb3 100644 --- a/Lib/struct.py +++ b/Lib/struct.py @@ -11,5 +11,5 @@ ] from _struct import * -from _struct import _clearcache -from _struct import __doc__ +from _struct import _clearcache # noqa: F401 +from _struct import __doc__ # noqa: F401 diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 9cadd1bf8e6..578d7b95d05 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -74,15 +74,16 @@ else: _mswindows = True -# wasm32-emscripten and wasm32-wasi do not support processes -_can_fork_exec = sys.platform not in {"emscripten", "wasi"} +# some platforms do not support subprocesses +_can_fork_exec = sys.platform not in {"emscripten", "wasi", "ios", "tvos", "watchos"} if _mswindows: import _winapi - from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, + from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, # noqa: F401 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE, SW_HIDE, STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW, + STARTF_FORCEONFEEDBACK, STARTF_FORCEOFFFEEDBACK, ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, HIGH_PRIORITY_CLASS, IDLE_PRIORITY_CLASS, NORMAL_PRIORITY_CLASS, REALTIME_PRIORITY_CLASS, @@ -93,6 +94,7 @@ "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", "STD_ERROR_HANDLE", "SW_HIDE", "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW", + "STARTF_FORCEONFEEDBACK", "STARTF_FORCEOFFFEEDBACK", "STARTUPINFO", "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", @@ -103,18 +105,22 @@ if _can_fork_exec: from _posixsubprocess import fork_exec as _fork_exec # used in methods that are called by __del__ - _waitpid = os.waitpid - _waitstatus_to_exitcode = os.waitstatus_to_exitcode - _WIFSTOPPED = os.WIFSTOPPED - _WSTOPSIG = os.WSTOPSIG - _WNOHANG = os.WNOHANG + class _del_safe: + waitpid = os.waitpid + waitstatus_to_exitcode = os.waitstatus_to_exitcode + WIFSTOPPED = os.WIFSTOPPED + WSTOPSIG = os.WSTOPSIG + WNOHANG = os.WNOHANG + ECHILD = errno.ECHILD else: - _fork_exec = None - _waitpid = None - _waitstatus_to_exitcode = None - _WIFSTOPPED = None - _WSTOPSIG = None - _WNOHANG = None + class _del_safe: + waitpid = None + waitstatus_to_exitcode = None + WIFSTOPPED = None + WSTOPSIG = None + WNOHANG = None + ECHILD = errno.ECHILD + import select import selectors @@ -346,7 +352,7 @@ def _args_from_interpreter_flags(): if dev_mode: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'showrefcount', 'utf8'): + 'frozen_modules', 'showrefcount', 'utf8', 'gil'): if opt in xoptions: value = xoptions[opt] if value is True: @@ -380,7 +386,7 @@ def _text_encoding(): def call(*popenargs, timeout=None, **kwargs): """Run command with arguments. Wait for command to complete or - timeout, then return the returncode attribute. + for timeout seconds, then return the returncode attribute. The arguments are the same as for the Popen constructor. Example: @@ -517,8 +523,8 @@ def run(*popenargs, in the returncode attribute, and output & stderr attributes if those streams were captured. - If timeout is given, and the process takes too long, a TimeoutExpired - exception will be raised. + If timeout (seconds) is given and the process takes too long, + a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument @@ -709,6 +715,9 @@ def _use_posix_spawn(): # os.posix_spawn() is not available return False + if ((_env := os.environ.get('_PYTHON_SUBPROCESS_USE_POSIX_SPAWN')) in ('0', '1')): + return bool(int(_env)) + if sys.platform in ('darwin', 'sunos5'): # posix_spawn() is a syscall on both macOS and Solaris, # and properly reports errors @@ -743,7 +752,7 @@ def _use_posix_spawn(): # These are primarily fail-safe knobs for negatives. A True value does not # guarantee the given libc/syscall API will be used. _USE_POSIX_SPAWN = _use_posix_spawn() -_USE_VFORK = True +_HAVE_POSIX_SPAWN_CLOSEFROM = hasattr(os, 'POSIX_SPAWN_CLOSEFROM') class Popen: @@ -834,6 +843,9 @@ def __init__(self, args, bufsize=-1, executable=None, if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") + if stdout is STDOUT: + raise ValueError("STDOUT can only be used for stderr") + if pipesize is None: pipesize = -1 # Restore default if not isinstance(pipesize, int): @@ -872,37 +884,6 @@ def __init__(self, args, bufsize=-1, executable=None, 'and universal_newlines are supplied but ' 'different. Pass one or the other.') - # Input and output objects. The general principle is like - # this: - # - # Parent Child - # ------ ----- - # p2cwrite ---stdin---> p2cread - # c2pread <--stdout--- c2pwrite - # errread <--stderr--- errwrite - # - # On POSIX, the child objects are file descriptors. On - # Windows, these are Windows file handles. The parent objects - # are file descriptors on both platforms. The parent objects - # are -1 when not using PIPEs. The child objects are -1 - # when not redirecting. - - (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) - - # We wrap OS handles *before* launching the child, otherwise a - # quickly terminating child could make our fds unwrappable - # (see #8458). - - if _mswindows: - if p2cwrite != -1: - p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) - if c2pread != -1: - c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) - if errread != -1: - errread = msvcrt.open_osfhandle(errread.Detach(), 0) - self.text_mode = encoding or errors or text or universal_newlines if self.text_mode and encoding is None: self.encoding = encoding = _text_encoding() @@ -1003,6 +984,39 @@ def __init__(self, args, bufsize=-1, executable=None, if uid < 0: raise ValueError(f"User ID cannot be negative, got {uid}") + # Input and output objects. The general principle is like + # this: + # + # Parent Child + # ------ ----- + # p2cwrite ---stdin---> p2cread + # c2pread <--stdout--- c2pwrite + # errread <--stderr--- errwrite + # + # On POSIX, the child objects are file descriptors. On + # Windows, these are Windows file handles. The parent objects + # are file descriptors on both platforms. The parent objects + # are -1 when not using PIPEs. The child objects are -1 + # when not redirecting. + + (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) = self._get_handles(stdin, stdout, stderr) + + # From here on, raising exceptions may cause file descriptor leakage + + # We wrap OS handles *before* launching the child, otherwise a + # quickly terminating child could make our fds unwrappable + # (see #8458). + + if _mswindows: + if p2cwrite != -1: + p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) + if c2pread != -1: + c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) + if errread != -1: + errread = msvcrt.open_osfhandle(errread.Detach(), 0) + try: if p2cwrite != -1: self.stdin = io.open(p2cwrite, 'wb', bufsize) @@ -1110,10 +1124,9 @@ def __exit__(self, exc_type, value, traceback): except TimeoutExpired: pass self._sigint_wait_secs = 0 # Note that this has been done. - return # resume the KeyboardInterrupt - - # Wait for the process to terminate, to avoid zombies. - self.wait() + else: + # Wait for the process to terminate, to avoid zombies. + self.wait() def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): if not self._child_created: @@ -1222,8 +1235,11 @@ def communicate(self, input=None, timeout=None): finally: self._communication_started = True - - sts = self.wait(timeout=self._remaining_time(endtime)) + try: + sts = self.wait(timeout=self._remaining_time(endtime)) + except TimeoutExpired as exc: + exc.timeout = timeout + raise return (stdout, stderr) @@ -1306,6 +1322,26 @@ def _close_pipe_fds(self, # Prevent a double close of these handles/fds from __init__ on error. self._closed_child_pipe_fds = True + @contextlib.contextmanager + def _on_error_fd_closer(self): + """Helper to ensure file descriptors opened in _get_handles are closed""" + to_close = [] + try: + yield to_close + except: + if hasattr(self, '_devnull'): + to_close.append(self._devnull) + del self._devnull + for fd in to_close: + try: + if _mswindows and isinstance(fd, Handle): + fd.Close() + else: + os.close(fd) + except OSError: + pass + raise + if _mswindows: # # Windows methods @@ -1321,61 +1357,68 @@ def _get_handles(self, stdin, stdout, stderr): c2pread, c2pwrite = -1, -1 errread, errwrite = -1, -1 - if stdin is None: - p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) - if p2cread is None: - p2cread, _ = _winapi.CreatePipe(None, 0) - p2cread = Handle(p2cread) - _winapi.CloseHandle(_) - elif stdin == PIPE: - p2cread, p2cwrite = _winapi.CreatePipe(None, 0) - p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) - elif stdin == DEVNULL: - p2cread = msvcrt.get_osfhandle(self._get_devnull()) - elif isinstance(stdin, int): - p2cread = msvcrt.get_osfhandle(stdin) - else: - # Assuming file-like object - p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) - - if stdout is None: - c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) - if c2pwrite is None: - _, c2pwrite = _winapi.CreatePipe(None, 0) - c2pwrite = Handle(c2pwrite) - _winapi.CloseHandle(_) - elif stdout == PIPE: - c2pread, c2pwrite = _winapi.CreatePipe(None, 0) - c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) - elif stdout == DEVNULL: - c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) - elif isinstance(stdout, int): - c2pwrite = msvcrt.get_osfhandle(stdout) - else: - # Assuming file-like object - c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) - - if stderr is None: - errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) - if errwrite is None: - _, errwrite = _winapi.CreatePipe(None, 0) - errwrite = Handle(errwrite) - _winapi.CloseHandle(_) - elif stderr == PIPE: - errread, errwrite = _winapi.CreatePipe(None, 0) - errread, errwrite = Handle(errread), Handle(errwrite) - elif stderr == STDOUT: - errwrite = c2pwrite - elif stderr == DEVNULL: - errwrite = msvcrt.get_osfhandle(self._get_devnull()) - elif isinstance(stderr, int): - errwrite = msvcrt.get_osfhandle(stderr) - else: - # Assuming file-like object - errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) + with self._on_error_fd_closer() as err_close_fds: + if stdin is None: + p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE) + if p2cread is None: + p2cread, _ = _winapi.CreatePipe(None, 0) + p2cread = Handle(p2cread) + err_close_fds.append(p2cread) + _winapi.CloseHandle(_) + elif stdin == PIPE: + p2cread, p2cwrite = _winapi.CreatePipe(None, 0) + p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) + err_close_fds.extend((p2cread, p2cwrite)) + elif stdin == DEVNULL: + p2cread = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stdin, int): + p2cread = msvcrt.get_osfhandle(stdin) + else: + # Assuming file-like object + p2cread = msvcrt.get_osfhandle(stdin.fileno()) + p2cread = self._make_inheritable(p2cread) + + if stdout is None: + c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE) + if c2pwrite is None: + _, c2pwrite = _winapi.CreatePipe(None, 0) + c2pwrite = Handle(c2pwrite) + err_close_fds.append(c2pwrite) + _winapi.CloseHandle(_) + elif stdout == PIPE: + c2pread, c2pwrite = _winapi.CreatePipe(None, 0) + c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) + err_close_fds.extend((c2pread, c2pwrite)) + elif stdout == DEVNULL: + c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stdout, int): + c2pwrite = msvcrt.get_osfhandle(stdout) + else: + # Assuming file-like object + c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) + c2pwrite = self._make_inheritable(c2pwrite) + + if stderr is None: + errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE) + if errwrite is None: + _, errwrite = _winapi.CreatePipe(None, 0) + errwrite = Handle(errwrite) + err_close_fds.append(errwrite) + _winapi.CloseHandle(_) + elif stderr == PIPE: + errread, errwrite = _winapi.CreatePipe(None, 0) + errread, errwrite = Handle(errread), Handle(errwrite) + err_close_fds.extend((errread, errwrite)) + elif stderr == STDOUT: + errwrite = c2pwrite + elif stderr == DEVNULL: + errwrite = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stderr, int): + errwrite = msvcrt.get_osfhandle(stderr) + else: + # Assuming file-like object + errwrite = msvcrt.get_osfhandle(stderr.fileno()) + errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, @@ -1480,7 +1523,23 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, if shell: startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW startupinfo.wShowWindow = _winapi.SW_HIDE - comspec = os.environ.get("COMSPEC", "cmd.exe") + if not executable: + # gh-101283: without a fully-qualified path, before Windows + # checks the system directories, it first looks in the + # application directory, and also the current directory if + # NeedCurrentDirectoryForExePathW(ExeName) is true, so try + # to avoid executing unqualified "cmd.exe". + comspec = os.environ.get('ComSpec') + if not comspec: + system_root = os.environ.get('SystemRoot', '') + comspec = os.path.join(system_root, 'System32', 'cmd.exe') + if not os.path.isabs(comspec): + raise FileNotFoundError('shell not found: neither %ComSpec% nor %SystemRoot% is set') + if os.path.isabs(comspec): + executable = comspec + else: + comspec = executable + args = '{} /c "{}"'.format (comspec, args) if cwd is not None: @@ -1536,6 +1595,8 @@ def _wait(self, timeout): """Internal implementation of wait() on Windows.""" if timeout is None: timeout_millis = _winapi.INFINITE + elif timeout <= 0: + timeout_millis = 0 else: timeout_millis = int(timeout * 1000) if self.returncode is None: @@ -1553,6 +1614,10 @@ def _readerthread(self, fh, buffer): fh.close() + def _writerthread(self, input): + self._stdin_write(input) + + def _communicate(self, input, endtime, orig_timeout): # Start reader threads feeding into a list hanging off of this # object, unless they've already been started. @@ -1571,8 +1636,23 @@ def _communicate(self, input, endtime, orig_timeout): self.stderr_thread.daemon = True self.stderr_thread.start() - if self.stdin: - self._stdin_write(input) + # Start writer thread to send input to stdin, unless already + # started. The thread writes input and closes stdin when done, + # or continues in the background on timeout. + if self.stdin and not hasattr(self, "_stdin_thread"): + self._stdin_thread = \ + threading.Thread(target=self._writerthread, + args=(input,)) + self._stdin_thread.daemon = True + self._stdin_thread.start() + + # Wait for the writer thread, or time out. If we time out, the + # thread remains writing and the fd left open in case the user + # calls communicate again. + if hasattr(self, "_stdin_thread"): + self._stdin_thread.join(self._remaining_time(endtime)) + if self._stdin_thread.is_alive(): + raise TimeoutExpired(self.args, orig_timeout) # Wait for the reader threads, or time out. If we time out, the # threads remain reading and the fds left open in case the user @@ -1646,66 +1726,67 @@ def _get_handles(self, stdin, stdout, stderr): c2pread, c2pwrite = -1, -1 errread, errwrite = -1, -1 - if stdin is None: - pass - elif stdin == PIPE: - p2cread, p2cwrite = os.pipe() - if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): - fcntl.fcntl(p2cwrite, fcntl.F_SETPIPE_SZ, self.pipesize) - elif stdin == DEVNULL: - p2cread = self._get_devnull() - elif isinstance(stdin, int): - p2cread = stdin - else: - # Assuming file-like object - p2cread = stdin.fileno() + with self._on_error_fd_closer() as err_close_fds: + if stdin is None: + pass + elif stdin == PIPE: + p2cread, p2cwrite = os.pipe() + err_close_fds.extend((p2cread, p2cwrite)) + if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(p2cwrite, fcntl.F_SETPIPE_SZ, self.pipesize) + elif stdin == DEVNULL: + p2cread = self._get_devnull() + elif isinstance(stdin, int): + p2cread = stdin + else: + # Assuming file-like object + p2cread = stdin.fileno() - if stdout is None: - pass - elif stdout == PIPE: - c2pread, c2pwrite = os.pipe() - if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): - fcntl.fcntl(c2pwrite, fcntl.F_SETPIPE_SZ, self.pipesize) - elif stdout == DEVNULL: - c2pwrite = self._get_devnull() - elif isinstance(stdout, int): - c2pwrite = stdout - else: - # Assuming file-like object - c2pwrite = stdout.fileno() + if stdout is None: + pass + elif stdout == PIPE: + c2pread, c2pwrite = os.pipe() + err_close_fds.extend((c2pread, c2pwrite)) + if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(c2pwrite, fcntl.F_SETPIPE_SZ, self.pipesize) + elif stdout == DEVNULL: + c2pwrite = self._get_devnull() + elif isinstance(stdout, int): + c2pwrite = stdout + else: + # Assuming file-like object + c2pwrite = stdout.fileno() - if stderr is None: - pass - elif stderr == PIPE: - errread, errwrite = os.pipe() - if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): - fcntl.fcntl(errwrite, fcntl.F_SETPIPE_SZ, self.pipesize) - elif stderr == STDOUT: - if c2pwrite != -1: - errwrite = c2pwrite - else: # child's stdout is not set, use parent's stdout - errwrite = sys.__stdout__.fileno() - elif stderr == DEVNULL: - errwrite = self._get_devnull() - elif isinstance(stderr, int): - errwrite = stderr - else: - # Assuming file-like object - errwrite = stderr.fileno() + if stderr is None: + pass + elif stderr == PIPE: + errread, errwrite = os.pipe() + err_close_fds.extend((errread, errwrite)) + if self.pipesize > 0 and hasattr(fcntl, "F_SETPIPE_SZ"): + fcntl.fcntl(errwrite, fcntl.F_SETPIPE_SZ, self.pipesize) + elif stderr == STDOUT: + if c2pwrite != -1: + errwrite = c2pwrite + else: # child's stdout is not set, use parent's stdout + errwrite = sys.__stdout__.fileno() + elif stderr == DEVNULL: + errwrite = self._get_devnull() + elif isinstance(stderr, int): + errwrite = stderr + else: + # Assuming file-like object + errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) - def _posix_spawn(self, args, executable, env, restore_signals, + def _posix_spawn(self, args, executable, env, restore_signals, close_fds, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program using os.posix_spawn().""" - if env is None: - env = os.environ - kwargs = {} if restore_signals: # See _Py_RestoreSignals() in Python/pylifecycle.c @@ -1727,6 +1808,10 @@ def _posix_spawn(self, args, executable, env, restore_signals, ): if fd != -1: file_actions.append((os.POSIX_SPAWN_DUP2, fd, fd2)) + + if close_fds: + file_actions.append((os.POSIX_SPAWN_CLOSEFROM, 3)) + if file_actions: kwargs['file_actions'] = file_actions @@ -1774,7 +1859,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, if (_USE_POSIX_SPAWN and os.path.dirname(executable) and preexec_fn is None - and not close_fds + and (not close_fds or _HAVE_POSIX_SPAWN_CLOSEFROM) and not pass_fds and cwd is None and (p2cread == -1 or p2cread > 2) @@ -1786,7 +1871,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, and gids is None and uid is None and umask < 0): - self._posix_spawn(args, executable, env, restore_signals, + self._posix_spawn(args, executable, env, restore_signals, close_fds, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) @@ -1840,7 +1925,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, errpipe_read, errpipe_write, restore_signals, start_new_session, process_group, gid, gids, uid, umask, - preexec_fn, _USE_VFORK) + preexec_fn) self._child_created = True finally: # be sure the FD is closed no matter what @@ -1889,33 +1974,34 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, SubprocessError) if issubclass(child_exception_type, OSError) and hex_errno: errno_num = int(hex_errno, 16) - child_exec_never_called = (err_msg == "noexec") - if child_exec_never_called: + if err_msg == "noexec:chdir": err_msg = "" # The error must be from chdir(cwd). err_filename = cwd + elif err_msg == "noexec": + err_msg = "" + err_filename = None else: err_filename = orig_executable if errno_num != 0: err_msg = os.strerror(errno_num) - raise child_exception_type(errno_num, err_msg, err_filename) + if err_filename is not None: + raise child_exception_type(errno_num, err_msg, err_filename) + else: + raise child_exception_type(errno_num, err_msg) raise child_exception_type(err_msg) - def _handle_exitstatus(self, sts, - _waitstatus_to_exitcode=_waitstatus_to_exitcode, - _WIFSTOPPED=_WIFSTOPPED, - _WSTOPSIG=_WSTOPSIG): + def _handle_exitstatus(self, sts, _del_safe=_del_safe): """All callers to this function MUST hold self._waitpid_lock.""" # This method is called (indirectly) by __del__, so it cannot # refer to anything outside of its local scope. - if _WIFSTOPPED(sts): - self.returncode = -_WSTOPSIG(sts) + if _del_safe.WIFSTOPPED(sts): + self.returncode = -_del_safe.WSTOPSIG(sts) else: - self.returncode = _waitstatus_to_exitcode(sts) + self.returncode = _del_safe.waitstatus_to_exitcode(sts) - def _internal_poll(self, _deadstate=None, _waitpid=_waitpid, - _WNOHANG=_WNOHANG, _ECHILD=errno.ECHILD): + def _internal_poll(self, _deadstate=None, _del_safe=_del_safe): """Check if child process has terminated. Returns returncode attribute. @@ -1931,13 +2017,13 @@ def _internal_poll(self, _deadstate=None, _waitpid=_waitpid, try: if self.returncode is not None: return self.returncode # Another thread waited. - pid, sts = _waitpid(self.pid, _WNOHANG) + pid, sts = _del_safe.waitpid(self.pid, _del_safe.WNOHANG) if pid == self.pid: self._handle_exitstatus(sts) except OSError as e: if _deadstate is not None: self.returncode = _deadstate - elif e.errno == _ECHILD: + elif e.errno == _del_safe.ECHILD: # This happens if SIGCLD is set to be ignored or # waiting for child processes has otherwise been # disabled for our process. This child is dead, we @@ -2011,6 +2097,10 @@ def _communicate(self, input, endtime, orig_timeout): self.stdin.flush() except BrokenPipeError: pass # communicate() must ignore BrokenPipeError. + except ValueError: + # ignore ValueError: I/O operation on closed file. + if not self.stdin.closed: + raise if not input: try: self.stdin.close() @@ -2036,10 +2126,13 @@ def _communicate(self, input, endtime, orig_timeout): self._save_input(input) if self._input: - input_view = memoryview(self._input) + if not isinstance(self._input, memoryview): + input_view = memoryview(self._input) + else: + input_view = self._input.cast("b") # byte input required with _PopenSelector() as selector: - if self.stdin and input: + if self.stdin and not self.stdin.closed and self._input: selector.register(self.stdin, selectors.EVENT_WRITE) if self.stdout and not self.stdout.closed: selector.register(self.stdout, selectors.EVENT_READ) @@ -2048,7 +2141,7 @@ def _communicate(self, input, endtime, orig_timeout): while selector.get_map(): timeout = self._remaining_time(endtime) - if timeout is not None and timeout < 0: + if timeout is not None and timeout <= 0: self._check_timeout(endtime, orig_timeout, stdout, stderr, skip_check_and_raise=True) @@ -2072,7 +2165,7 @@ def _communicate(self, input, endtime, orig_timeout): selector.unregister(key.fileobj) key.fileobj.close() else: - if self._input_offset >= len(self._input): + if self._input_offset >= len(input_view): selector.unregister(key.fileobj) key.fileobj.close() elif key.fileobj in (self.stdout, self.stderr): @@ -2081,8 +2174,11 @@ def _communicate(self, input, endtime, orig_timeout): selector.unregister(key.fileobj) key.fileobj.close() self._fileobj2output[key.fileobj].append(data) - - self.wait(timeout=self._remaining_time(endtime)) + try: + self.wait(timeout=self._remaining_time(endtime)) + except TimeoutExpired as exc: + exc.timeout = orig_timeout + raise # All data exchanged. Translate lists into strings. if stdout is not None: diff --git a/Lib/sunau.py b/Lib/sunau.py deleted file mode 100644 index 129502b0b41..00000000000 --- a/Lib/sunau.py +++ /dev/null @@ -1,531 +0,0 @@ -"""Stuff to parse Sun and NeXT audio files. - -An audio file consists of a header followed by the data. The structure -of the header is as follows. - - +---------------+ - | magic word | - +---------------+ - | header size | - +---------------+ - | data size | - +---------------+ - | encoding | - +---------------+ - | sample rate | - +---------------+ - | # of channels | - +---------------+ - | info | - | | - +---------------+ - -The magic word consists of the 4 characters '.snd'. Apart from the -info field, all header fields are 4 bytes in size. They are all -32-bit unsigned integers encoded in big-endian byte order. - -The header size really gives the start of the data. -The data size is the physical size of the data. From the other -parameters the number of frames can be calculated. -The encoding gives the way in which audio samples are encoded. -Possible values are listed below. -The info field currently consists of an ASCII string giving a -human-readable description of the audio file. The info field is -padded with NUL bytes to the header size. - -Usage. - -Reading audio files: - f = sunau.open(file, 'r') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods read(), seek(), and close(). -When the setpos() and rewind() methods are not used, the seek() -method is not necessary. - -This returns an instance of a class with the following public methods: - getnchannels() -- returns number of audio channels (1 for - mono, 2 for stereo) - getsampwidth() -- returns sample width in bytes - getframerate() -- returns sampling frequency - getnframes() -- returns number of audio frames - getcomptype() -- returns compression type ('NONE' or 'ULAW') - getcompname() -- returns human-readable version of - compression type ('not compressed' matches 'NONE') - getparams() -- returns a namedtuple consisting of all of the - above in the above order - getmarkers() -- returns None (for compatibility with the - aifc module) - getmark(id) -- raises an error since the mark does not - exist (for compatibility with the aifc module) - readframes(n) -- returns at most n frames of audio - rewind() -- rewind to the beginning of the audio stream - setpos(pos) -- seek to the specified position - tell() -- return the current position - close() -- close the instance (make it unusable) -The position returned by tell() and the position given to setpos() -are compatible and have nothing to do with the actual position in the -file. -The close() method is called automatically when the class instance -is destroyed. - -Writing audio files: - f = sunau.open(file, 'w') -where file is either the name of a file or an open file pointer. -The open file pointer must have methods write(), tell(), seek(), and -close(). - -This returns an instance of a class with the following public methods: - setnchannels(n) -- set the number of channels - setsampwidth(n) -- set the sample width - setframerate(n) -- set the frame rate - setnframes(n) -- set the number of frames - setcomptype(type, name) - -- set the compression type and the - human-readable compression type - setparams(tuple)-- set all parameters at once - tell() -- return current position in output file - writeframesraw(data) - -- write audio frames without pathing up the - file header - writeframes(data) - -- write audio frames and patch up the file header - close() -- patch up the file header and close the - output file -You should set the parameters before the first writeframesraw or -writeframes. The total number of frames does not need to be set, -but when it is set to the correct value, the header does not have to -be patched up. -It is best to first set all parameters, perhaps possibly the -compression type, and then write audio frames using writeframesraw. -When all frames have been written, either call writeframes(b'') or -close() to patch up the sizes in the header. -The close() method is called automatically when the class instance -is destroyed. -""" - -from collections import namedtuple -import warnings - -_sunau_params = namedtuple('_sunau_params', - 'nchannels sampwidth framerate nframes comptype compname') - -# from -AUDIO_FILE_MAGIC = 0x2e736e64 -AUDIO_FILE_ENCODING_MULAW_8 = 1 -AUDIO_FILE_ENCODING_LINEAR_8 = 2 -AUDIO_FILE_ENCODING_LINEAR_16 = 3 -AUDIO_FILE_ENCODING_LINEAR_24 = 4 -AUDIO_FILE_ENCODING_LINEAR_32 = 5 -AUDIO_FILE_ENCODING_FLOAT = 6 -AUDIO_FILE_ENCODING_DOUBLE = 7 -AUDIO_FILE_ENCODING_ADPCM_G721 = 23 -AUDIO_FILE_ENCODING_ADPCM_G722 = 24 -AUDIO_FILE_ENCODING_ADPCM_G723_3 = 25 -AUDIO_FILE_ENCODING_ADPCM_G723_5 = 26 -AUDIO_FILE_ENCODING_ALAW_8 = 27 - -# from -AUDIO_UNKNOWN_SIZE = 0xFFFFFFFF # ((unsigned)(~0)) - -_simple_encodings = [AUDIO_FILE_ENCODING_MULAW_8, - AUDIO_FILE_ENCODING_LINEAR_8, - AUDIO_FILE_ENCODING_LINEAR_16, - AUDIO_FILE_ENCODING_LINEAR_24, - AUDIO_FILE_ENCODING_LINEAR_32, - AUDIO_FILE_ENCODING_ALAW_8] - -class Error(Exception): - pass - -def _read_u32(file): - x = 0 - for i in range(4): - byte = file.read(1) - if not byte: - raise EOFError - x = x*256 + ord(byte) - return x - -def _write_u32(file, x): - data = [] - for i in range(4): - d, m = divmod(x, 256) - data.insert(0, int(m)) - x = d - file.write(bytes(data)) - -class Au_read: - - def __init__(self, f): - if type(f) == type(''): - import builtins - f = builtins.open(f, 'rb') - self._opened = True - else: - self._opened = False - self.initfp(f) - - def __del__(self): - if self._file: - self.close() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def initfp(self, file): - self._file = file - self._soundpos = 0 - magic = int(_read_u32(file)) - if magic != AUDIO_FILE_MAGIC: - raise Error('bad magic number') - self._hdr_size = int(_read_u32(file)) - if self._hdr_size < 24: - raise Error('header size too small') - if self._hdr_size > 100: - raise Error('header size ridiculously large') - self._data_size = _read_u32(file) - if self._data_size != AUDIO_UNKNOWN_SIZE: - self._data_size = int(self._data_size) - self._encoding = int(_read_u32(file)) - if self._encoding not in _simple_encodings: - raise Error('encoding not (yet) supported') - if self._encoding in (AUDIO_FILE_ENCODING_MULAW_8, - AUDIO_FILE_ENCODING_ALAW_8): - self._sampwidth = 2 - self._framesize = 1 - elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_8: - self._framesize = self._sampwidth = 1 - elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_16: - self._framesize = self._sampwidth = 2 - elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_24: - self._framesize = self._sampwidth = 3 - elif self._encoding == AUDIO_FILE_ENCODING_LINEAR_32: - self._framesize = self._sampwidth = 4 - else: - raise Error('unknown encoding') - self._framerate = int(_read_u32(file)) - self._nchannels = int(_read_u32(file)) - if not self._nchannels: - raise Error('bad # of channels') - self._framesize = self._framesize * self._nchannels - if self._hdr_size > 24: - self._info = file.read(self._hdr_size - 24) - self._info, _, _ = self._info.partition(b'\0') - else: - self._info = b'' - try: - self._data_pos = file.tell() - except (AttributeError, OSError): - self._data_pos = None - - def getfp(self): - return self._file - - def getnchannels(self): - return self._nchannels - - def getsampwidth(self): - return self._sampwidth - - def getframerate(self): - return self._framerate - - def getnframes(self): - if self._data_size == AUDIO_UNKNOWN_SIZE: - return AUDIO_UNKNOWN_SIZE - if self._encoding in _simple_encodings: - return self._data_size // self._framesize - return 0 # XXX--must do some arithmetic here - - def getcomptype(self): - if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: - return 'ULAW' - elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8: - return 'ALAW' - else: - return 'NONE' - - def getcompname(self): - if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: - return 'CCITT G.711 u-law' - elif self._encoding == AUDIO_FILE_ENCODING_ALAW_8: - return 'CCITT G.711 A-law' - else: - return 'not compressed' - - def getparams(self): - return _sunau_params(self.getnchannels(), self.getsampwidth(), - self.getframerate(), self.getnframes(), - self.getcomptype(), self.getcompname()) - - def getmarkers(self): - return None - - def getmark(self, id): - raise Error('no marks') - - def readframes(self, nframes): - if self._encoding in _simple_encodings: - if nframes == AUDIO_UNKNOWN_SIZE: - data = self._file.read() - else: - data = self._file.read(nframes * self._framesize) - self._soundpos += len(data) // self._framesize - if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: - import audioop - data = audioop.ulaw2lin(data, self._sampwidth) - return data - return None # XXX--not implemented yet - - def rewind(self): - if self._data_pos is None: - raise OSError('cannot seek') - self._file.seek(self._data_pos) - self._soundpos = 0 - - def tell(self): - return self._soundpos - - def setpos(self, pos): - if pos < 0 or pos > self.getnframes(): - raise Error('position not in range') - if self._data_pos is None: - raise OSError('cannot seek') - self._file.seek(self._data_pos + pos * self._framesize) - self._soundpos = pos - - def close(self): - file = self._file - if file: - self._file = None - if self._opened: - file.close() - -class Au_write: - - def __init__(self, f): - if type(f) == type(''): - import builtins - f = builtins.open(f, 'wb') - self._opened = True - else: - self._opened = False - self.initfp(f) - - def __del__(self): - if self._file: - self.close() - self._file = None - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def initfp(self, file): - self._file = file - self._framerate = 0 - self._nchannels = 0 - self._sampwidth = 0 - self._framesize = 0 - self._nframes = AUDIO_UNKNOWN_SIZE - self._nframeswritten = 0 - self._datawritten = 0 - self._datalength = 0 - self._info = b'' - self._comptype = 'ULAW' # default is U-law - - def setnchannels(self, nchannels): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if nchannels not in (1, 2, 4): - raise Error('only 1, 2, or 4 channels supported') - self._nchannels = nchannels - - def getnchannels(self): - if not self._nchannels: - raise Error('number of channels not set') - return self._nchannels - - def setsampwidth(self, sampwidth): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if sampwidth not in (1, 2, 3, 4): - raise Error('bad sample width') - self._sampwidth = sampwidth - - def getsampwidth(self): - if not self._framerate: - raise Error('sample width not specified') - return self._sampwidth - - def setframerate(self, framerate): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - self._framerate = framerate - - def getframerate(self): - if not self._framerate: - raise Error('frame rate not set') - return self._framerate - - def setnframes(self, nframes): - if self._nframeswritten: - raise Error('cannot change parameters after starting to write') - if nframes < 0: - raise Error('# of frames cannot be negative') - self._nframes = nframes - - def getnframes(self): - return self._nframeswritten - - def setcomptype(self, type, name): - if type in ('NONE', 'ULAW'): - self._comptype = type - else: - raise Error('unknown compression type') - - def getcomptype(self): - return self._comptype - - def getcompname(self): - if self._comptype == 'ULAW': - return 'CCITT G.711 u-law' - elif self._comptype == 'ALAW': - return 'CCITT G.711 A-law' - else: - return 'not compressed' - - def setparams(self, params): - nchannels, sampwidth, framerate, nframes, comptype, compname = params - self.setnchannels(nchannels) - self.setsampwidth(sampwidth) - self.setframerate(framerate) - self.setnframes(nframes) - self.setcomptype(comptype, compname) - - def getparams(self): - return _sunau_params(self.getnchannels(), self.getsampwidth(), - self.getframerate(), self.getnframes(), - self.getcomptype(), self.getcompname()) - - def tell(self): - return self._nframeswritten - - def writeframesraw(self, data): - if not isinstance(data, (bytes, bytearray)): - data = memoryview(data).cast('B') - self._ensure_header_written() - if self._comptype == 'ULAW': - import audioop - data = audioop.lin2ulaw(data, self._sampwidth) - nframes = len(data) // self._framesize - self._file.write(data) - self._nframeswritten = self._nframeswritten + nframes - self._datawritten = self._datawritten + len(data) - - def writeframes(self, data): - self.writeframesraw(data) - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - - def close(self): - if self._file: - try: - self._ensure_header_written() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten: - self._patchheader() - self._file.flush() - finally: - file = self._file - self._file = None - if self._opened: - file.close() - - # - # private methods - # - - def _ensure_header_written(self): - if not self._nframeswritten: - if not self._nchannels: - raise Error('# of channels not specified') - if not self._sampwidth: - raise Error('sample width not specified') - if not self._framerate: - raise Error('frame rate not specified') - self._write_header() - - def _write_header(self): - if self._comptype == 'NONE': - if self._sampwidth == 1: - encoding = AUDIO_FILE_ENCODING_LINEAR_8 - self._framesize = 1 - elif self._sampwidth == 2: - encoding = AUDIO_FILE_ENCODING_LINEAR_16 - self._framesize = 2 - elif self._sampwidth == 3: - encoding = AUDIO_FILE_ENCODING_LINEAR_24 - self._framesize = 3 - elif self._sampwidth == 4: - encoding = AUDIO_FILE_ENCODING_LINEAR_32 - self._framesize = 4 - else: - raise Error('internal error') - elif self._comptype == 'ULAW': - encoding = AUDIO_FILE_ENCODING_MULAW_8 - self._framesize = 1 - else: - raise Error('internal error') - self._framesize = self._framesize * self._nchannels - _write_u32(self._file, AUDIO_FILE_MAGIC) - header_size = 25 + len(self._info) - header_size = (header_size + 7) & ~7 - _write_u32(self._file, header_size) - if self._nframes == AUDIO_UNKNOWN_SIZE: - length = AUDIO_UNKNOWN_SIZE - else: - length = self._nframes * self._framesize - try: - self._form_length_pos = self._file.tell() - except (AttributeError, OSError): - self._form_length_pos = None - _write_u32(self._file, length) - self._datalength = length - _write_u32(self._file, encoding) - _write_u32(self._file, self._framerate) - _write_u32(self._file, self._nchannels) - self._file.write(self._info) - self._file.write(b'\0'*(header_size - len(self._info) - 24)) - - def _patchheader(self): - if self._form_length_pos is None: - raise OSError('cannot seek') - self._file.seek(self._form_length_pos) - _write_u32(self._file, self._datawritten) - self._datalength = self._datawritten - self._file.seek(0, 2) - -def open(f, mode=None): - if mode is None: - if hasattr(f, 'mode'): - mode = f.mode - else: - mode = 'rb' - if mode in ('r', 'rb'): - return Au_read(f) - elif mode in ('w', 'wb'): - return Au_write(f) - else: - raise Error("mode must be 'r', 'rb', 'w', or 'wb'") - -def openfp(f, mode=None): - warnings.warn("sunau.openfp is deprecated since Python 3.7. " - "Use sunau.open instead.", DeprecationWarning, stacklevel=2) - return open(f, mode=mode) diff --git a/Lib/symtable.py b/Lib/symtable.py new file mode 100644 index 00000000000..7a30e1ac4ca --- /dev/null +++ b/Lib/symtable.py @@ -0,0 +1,451 @@ +"""Interface to the compiler's internal symbol tables""" + +import _symtable +from _symtable import ( + USE, + DEF_GLOBAL, # noqa: F401 + DEF_NONLOCAL, DEF_LOCAL, + DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, + DEF_IMPORT, DEF_BOUND, DEF_ANNOT, + DEF_COMP_ITER, DEF_COMP_CELL, + SCOPE_OFF, SCOPE_MASK, + FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL +) + +import weakref +from enum import StrEnum + +__all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] + +def symtable(code, filename, compile_type): + """ Return the toplevel *SymbolTable* for the source code. + + *filename* is the name of the file with the code + and *compile_type* is the *compile()* mode argument. + """ + top = _symtable.symtable(code, filename, compile_type) + return _newSymbolTable(top, filename) + +class SymbolTableFactory: + def __init__(self): + self.__memo = weakref.WeakValueDictionary() + + def new(self, table, filename): + if table.type == _symtable.TYPE_FUNCTION: + return Function(table, filename) + if table.type == _symtable.TYPE_CLASS: + return Class(table, filename) + return SymbolTable(table, filename) + + def __call__(self, table, filename): + key = table, filename + obj = self.__memo.get(key, None) + if obj is None: + obj = self.__memo[key] = self.new(table, filename) + return obj + +_newSymbolTable = SymbolTableFactory() + + +class SymbolTableType(StrEnum): + MODULE = "module" + FUNCTION = "function" + CLASS = "class" + ANNOTATION = "annotation" + TYPE_ALIAS = "type alias" + TYPE_PARAMETERS = "type parameters" + TYPE_VARIABLE = "type variable" + + +class SymbolTable: + + def __init__(self, raw_table, filename): + self._table = raw_table + self._filename = filename + self._symbols = {} + + def __repr__(self): + if self.__class__ == SymbolTable: + kind = "" + else: + kind = "%s " % self.__class__.__name__ + + if self._table.name == "top": + return "<{0}SymbolTable for module {1}>".format(kind, self._filename) + else: + return "<{0}SymbolTable for {1} in {2}>".format(kind, + self._table.name, + self._filename) + + def get_type(self): + """Return the type of the symbol table. + + The value returned is one of the values in + the ``SymbolTableType`` enumeration. + """ + if self._table.type == _symtable.TYPE_MODULE: + return SymbolTableType.MODULE + if self._table.type == _symtable.TYPE_FUNCTION: + return SymbolTableType.FUNCTION + if self._table.type == _symtable.TYPE_CLASS: + return SymbolTableType.CLASS + if self._table.type == _symtable.TYPE_ANNOTATION: + return SymbolTableType.ANNOTATION + if self._table.type == _symtable.TYPE_TYPE_ALIAS: + return SymbolTableType.TYPE_ALIAS + if self._table.type == _symtable.TYPE_TYPE_PARAMETERS: + return SymbolTableType.TYPE_PARAMETERS + if self._table.type == _symtable.TYPE_TYPE_VARIABLE: + return SymbolTableType.TYPE_VARIABLE + assert False, f"unexpected type: {self._table.type}" + + def get_id(self): + """Return an identifier for the table. + """ + return self._table.id + + def get_name(self): + """Return the table's name. + + This corresponds to the name of the class, function + or 'top' if the table is for a class, function or + global respectively. + """ + return self._table.name + + def get_lineno(self): + """Return the number of the first line in the + block for the table. + """ + return self._table.lineno + + def is_optimized(self): + """Return *True* if the locals in the table + are optimizable. + """ + return bool(self._table.type == _symtable.TYPE_FUNCTION) + + def is_nested(self): + """Return *True* if the block is a nested class + or function.""" + return bool(self._table.nested) + + def has_children(self): + """Return *True* if the block has nested namespaces. + """ + return bool(self._table.children) + + def get_identifiers(self): + """Return a view object containing the names of symbols in the table. + """ + return self._table.symbols.keys() + + def lookup(self, name): + """Lookup a *name* in the table. + + Returns a *Symbol* instance. + """ + sym = self._symbols.get(name) + if sym is None: + flags = self._table.symbols[name] + namespaces = self.__check_children(name) + module_scope = (self._table.name == "top") + sym = self._symbols[name] = Symbol(name, flags, namespaces, + module_scope=module_scope) + return sym + + def get_symbols(self): + """Return a list of *Symbol* instances for + names in the table. + """ + return [self.lookup(ident) for ident in self.get_identifiers()] + + def __check_children(self, name): + return [_newSymbolTable(st, self._filename) + for st in self._table.children + if st.name == name] + + def get_children(self): + """Return a list of the nested symbol tables. + """ + return [_newSymbolTable(st, self._filename) + for st in self._table.children] + + +def _get_scope(flags): # like _PyST_GetScope() + return (flags >> SCOPE_OFF) & SCOPE_MASK + + +class Function(SymbolTable): + + # Default values for instance variables + __params = None + __locals = None + __frees = None + __globals = None + __nonlocals = None + + def __idents_matching(self, test_func): + return tuple(ident for ident in self.get_identifiers() + if test_func(self._table.symbols[ident])) + + def get_parameters(self): + """Return a tuple of parameters to the function. + """ + if self.__params is None: + self.__params = self.__idents_matching(lambda x:x & DEF_PARAM) + return self.__params + + def get_locals(self): + """Return a tuple of locals in the function. + """ + if self.__locals is None: + locs = (LOCAL, CELL) + test = lambda x: _get_scope(x) in locs + self.__locals = self.__idents_matching(test) + return self.__locals + + def get_globals(self): + """Return a tuple of globals in the function. + """ + if self.__globals is None: + glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) + test = lambda x: _get_scope(x) in glob + self.__globals = self.__idents_matching(test) + return self.__globals + + def get_nonlocals(self): + """Return a tuple of nonlocals in the function. + """ + if self.__nonlocals is None: + self.__nonlocals = self.__idents_matching(lambda x:x & DEF_NONLOCAL) + return self.__nonlocals + + def get_frees(self): + """Return a tuple of free variables in the function. + """ + if self.__frees is None: + is_free = lambda x: _get_scope(x) == FREE + self.__frees = self.__idents_matching(is_free) + return self.__frees + + +class Class(SymbolTable): + + __methods = None + + def get_methods(self): + """Return a tuple of methods declared in the class. + """ + import warnings + typename = f'{self.__class__.__module__}.{self.__class__.__name__}' + warnings.warn(f'{typename}.get_methods() is deprecated ' + f'and will be removed in Python 3.16.', + DeprecationWarning, stacklevel=2) + + if self.__methods is None: + d = {} + + def is_local_symbol(ident): + flags = self._table.symbols.get(ident, 0) + return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL + + for st in self._table.children: + # pick the function-like symbols that are local identifiers + if is_local_symbol(st.name): + match st.type: + case _symtable.TYPE_FUNCTION: + # generators are of type TYPE_FUNCTION with a ".0" + # parameter as a first parameter (which makes them + # distinguishable from a function named 'genexpr') + if st.name == 'genexpr' and '.0' in st.varnames: + continue + d[st.name] = 1 + case _symtable.TYPE_TYPE_PARAMETERS: + # Get the function-def block in the annotation + # scope 'st' with the same identifier, if any. + scope_name = st.name + for c in st.children: + if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: + # A generic generator of type TYPE_FUNCTION + # cannot be a direct child of 'st' (but it + # can be a descendant), e.g.: + # + # class A: + # type genexpr[genexpr] = (x for x in []) + assert scope_name != 'genexpr' or '.0' not in c.varnames + d[scope_name] = 1 + break + self.__methods = tuple(d) + return self.__methods + + +class Symbol: + + def __init__(self, name, flags, namespaces=None, *, module_scope=False): + self.__name = name + self.__flags = flags + self.__scope = _get_scope(flags) + self.__namespaces = namespaces or () + self.__module_scope = module_scope + + def __repr__(self): + flags_str = '|'.join(self._flags_str()) + return f'' + + def _scope_str(self): + return _scopes_value_to_name.get(self.__scope) or str(self.__scope) + + def _flags_str(self): + for flagname, flagvalue in _flags: + if self.__flags & flagvalue == flagvalue: + yield flagname + + def get_name(self): + """Return a name of a symbol. + """ + return self.__name + + def is_referenced(self): + """Return *True* if the symbol is used in + its block. + """ + return bool(self.__flags & USE) + + def is_parameter(self): + """Return *True* if the symbol is a parameter. + """ + return bool(self.__flags & DEF_PARAM) + + def is_type_parameter(self): + """Return *True* if the symbol is a type parameter. + """ + return bool(self.__flags & DEF_TYPE_PARAM) + + def is_global(self): + """Return *True* if the symbol is global. + """ + return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) + or (self.__module_scope and self.__flags & DEF_BOUND)) + + def is_nonlocal(self): + """Return *True* if the symbol is nonlocal.""" + return bool(self.__flags & DEF_NONLOCAL) + + def is_declared_global(self): + """Return *True* if the symbol is declared global + with a global statement.""" + return bool(self.__scope == GLOBAL_EXPLICIT) + + def is_local(self): + """Return *True* if the symbol is local. + """ + return bool(self.__scope in (LOCAL, CELL) + or (self.__module_scope and self.__flags & DEF_BOUND)) + + def is_annotated(self): + """Return *True* if the symbol is annotated. + """ + return bool(self.__flags & DEF_ANNOT) + + def is_free(self): + """Return *True* if a referenced symbol is + not assigned to. + """ + return bool(self.__scope == FREE) + + def is_free_class(self): + """Return *True* if a class-scoped symbol is free from + the perspective of a method.""" + return bool(self.__flags & DEF_FREE_CLASS) + + def is_imported(self): + """Return *True* if the symbol is created from + an import statement. + """ + return bool(self.__flags & DEF_IMPORT) + + def is_assigned(self): + """Return *True* if a symbol is assigned to.""" + return bool(self.__flags & DEF_LOCAL) + + def is_comp_iter(self): + """Return *True* if the symbol is a comprehension iteration variable. + """ + return bool(self.__flags & DEF_COMP_ITER) + + def is_comp_cell(self): + """Return *True* if the symbol is a cell in an inlined comprehension. + """ + return bool(self.__flags & DEF_COMP_CELL) + + def is_namespace(self): + """Returns *True* if name binding introduces new namespace. + + If the name is used as the target of a function or class + statement, this will be true. + + Note that a single name can be bound to multiple objects. If + is_namespace() is true, the name may also be bound to other + objects, like an int or list, that does not introduce a new + namespace. + """ + return bool(self.__namespaces) + + def get_namespaces(self): + """Return a list of namespaces bound to this name""" + return self.__namespaces + + def get_namespace(self): + """Return the single namespace bound to this name. + + Raises ValueError if the name is bound to multiple namespaces + or no namespace. + """ + if len(self.__namespaces) == 0: + raise ValueError("name is not bound to any namespaces") + elif len(self.__namespaces) > 1: + raise ValueError("name is bound to multiple namespaces") + else: + return self.__namespaces[0] + + +_flags = [('USE', USE)] +_flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_')) +_scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL') +_scopes_value_to_name = {globals()[n]: n for n in _scopes_names} + + +def main(args): + import sys + def print_symbols(table, level=0): + indent = ' ' * level + nested = "nested " if table.is_nested() else "" + if table.get_type() == 'module': + what = f'from file {table._filename!r}' + else: + what = f'{table.get_name()!r}' + print(f'{indent}symbol table for {nested}{table.get_type()} {what}:') + for ident in table.get_identifiers(): + symbol = table.lookup(ident) + flags = ', '.join(symbol._flags_str()).lower() + print(f' {indent}{symbol._scope_str().lower()} symbol {symbol.get_name()!r}: {flags}') + print() + + for table2 in table.get_children(): + print_symbols(table2, level + 1) + + for filename in args or ['-']: + if filename == '-': + src = sys.stdin.read() + filename = '' + else: + with open(filename, 'rb') as f: + src = f.read() + mod = symtable(src, filename, 'exec') + print_symbols(mod) + + +if __name__ == "__main__": + import sys + main(sys.argv[1:]) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py deleted file mode 100644 index 9999d6bbd5d..00000000000 --- a/Lib/sysconfig.py +++ /dev/null @@ -1,858 +0,0 @@ -# XXX: RUSTPYTHON; Trick to make sysconfig work as RustPython -exec(r''' -"""Access to Python's configuration information.""" - -import os -import sys -from os.path import pardir, realpath - -__all__ = [ - 'get_config_h_filename', - 'get_config_var', - 'get_config_vars', - 'get_makefile_filename', - 'get_path', - 'get_path_names', - 'get_paths', - 'get_platform', - 'get_python_version', - 'get_scheme_names', - 'parse_config_h', -] - -# Keys for get_config_var() that are never converted to Python integers. -_ALWAYS_STR = { - 'MACOSX_DEPLOYMENT_TARGET', -} - -_INSTALL_SCHEMES = { - 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', - 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', - 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', - 'scripts': '{base}/bin', - 'data': '{base}', - }, - 'posix_home': { - 'stdlib': '{installed_base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{installed_base}/include/python', - 'platinclude': '{installed_base}/include/python', - 'scripts': '{base}/bin', - 'data': '{base}', - }, - 'nt': { - 'stdlib': '{installed_base}/Lib', - 'platstdlib': '{base}/Lib', - 'purelib': '{base}/Lib/site-packages', - 'platlib': '{base}/Lib/site-packages', - 'include': '{installed_base}/Include', - 'platinclude': '{installed_base}/Include', - 'scripts': '{base}/Scripts', - 'data': '{base}', - }, - # Downstream distributors can overwrite the default install scheme. - # This is done to support downstream modifications where distributors change - # the installation layout (eg. different site-packages directory). - # So, distributors will change the default scheme to one that correctly - # represents their layout. - # This presents an issue for projects/people that need to bootstrap virtual - # environments, like virtualenv. As distributors might now be customizing - # the default install scheme, there is no guarantee that the information - # returned by sysconfig.get_default_scheme/get_paths is correct for - # a virtual environment, the only guarantee we have is that it is correct - # for the *current* environment. When bootstrapping a virtual environment, - # we need to know its layout, so that we can place the files in the - # correct locations. - # The "*_venv" install scheme is a scheme to bootstrap virtual environments, - # essentially identical to the default posix_prefix/nt schemes. - # Downstream distributors who patch posix_prefix/nt scheme are encouraged to - # leave the following schemes unchanged - 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', - 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', - 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', - 'scripts': '{base}/bin', - 'data': '{base}', - }, - 'nt_venv': { - 'stdlib': '{installed_base}/Lib', - 'platstdlib': '{base}/Lib', - 'purelib': '{base}/Lib/site-packages', - 'platlib': '{base}/Lib/site-packages', - 'include': '{installed_base}/Include', - 'platinclude': '{installed_base}/Include', - 'scripts': '{base}/Scripts', - 'data': '{base}', - }, - } - -# For the OS-native venv scheme, we essentially provide an alias: -if os.name == 'nt': - _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['nt_venv'] -else: - _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] - - -# NOTE: site.py has copy of this function. -# Sync it when modify this function. -def _getuserbase(): - env_base = os.environ.get("PYTHONUSERBASE", None) - if env_base: - return env_base - - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: - return None - - def joinuser(*args): - return os.path.expanduser(os.path.join(*args)) - - if os.name == "nt": - base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") - - if sys.platform == "darwin" and sys._framework: - return joinuser("~", "Library", sys._framework, - f"{sys.version_info[0]}.{sys.version_info[1]}") - - return joinuser("~", ".local") - -_HAS_USER_BASE = (_getuserbase() is not None) - -if _HAS_USER_BASE: - _INSTALL_SCHEMES |= { - # NOTE: When modifying "purelib" scheme, update site._get_path() too. - 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', - 'data': '{userbase}', - }, - 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data': '{userbase}', - }, - 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data': '{userbase}', - }, - } - -_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', - 'scripts', 'data') - -_PY_VERSION = sys.version.split()[0] -_PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' -_PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) -_BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -_BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) -_CONFIG_VARS = None -_USER_BASE = None - -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" -_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" -_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" - - -def _safe_realpath(path): - try: - return realpath(path) - except OSError: - return path - -if sys.executable: - _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) -else: - # sys.executable can be empty if argv[0] has been changed and Python is - # unable to retrieve the real program name - _PROJECT_BASE = _safe_realpath(os.getcwd()) - -# In a virtual environment, `sys._home` gives us the target directory -# `_PROJECT_BASE` for the executable that created it when the virtual -# python is an actual executable ('venv --copies' or Windows). -_sys_home = getattr(sys, '_home', None) -if _sys_home: - _PROJECT_BASE = _sys_home - -if os.name == 'nt': - # In a source build, the executable is in a subdirectory of the root - # that we want (\PCbuild\). - # `_BASE_PREFIX` is used as the base installation is where the source - # will be. The realpath is needed to prevent mount point confusion - # that can occur with just string comparisons. - if _safe_realpath(_PROJECT_BASE).startswith( - _safe_realpath(f'{_BASE_PREFIX}\\PCbuild')): - _PROJECT_BASE = _BASE_PREFIX - -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) - -def is_python_build(check_home=None): - if check_home is not None: - import warnings - warnings.warn("check_home argument is deprecated and ignored.", - DeprecationWarning, stacklevel=2) - for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): - return True - return False - -_PYTHON_BUILD = is_python_build() - -if _PYTHON_BUILD: - for scheme in ('posix_prefix', 'posix_home'): - # On POSIX-y platforms, Python will: - # - Build from .h files in 'headers' (which is only added to the - # scheme when building CPython) - # - Install .h files to 'include' - scheme = _INSTALL_SCHEMES[scheme] - scheme['headers'] = scheme['include'] - scheme['include'] = '{srcdir}/Include' - scheme['platinclude'] = '{projectbase}/.' - del scheme - - -def _subst_vars(s, local_vars): - try: - return s.format(**local_vars) - except KeyError as var: - try: - return s.format(**os.environ) - except KeyError: - raise AttributeError(f'{var}') from None - -def _extend_dict(target_dict, other_dict): - target_keys = target_dict.keys() - for key, value in other_dict.items(): - if key in target_keys: - continue - target_dict[key] = value - - -def _expand_vars(scheme, vars): - res = {} - if vars is None: - vars = {} - _extend_dict(vars, get_config_vars()) - if os.name == 'nt': - # On Windows we want to substitute 'lib' for schemes rather - # than the native value (without modifying vars, in case it - # was passed in) - vars = vars | {'platlibdir': 'lib'} - - for key, value in _INSTALL_SCHEMES[scheme].items(): - if os.name in ('posix', 'nt'): - value = os.path.expanduser(value) - res[key] = os.path.normpath(_subst_vars(value, vars)) - return res - - -def _get_preferred_schemes(): - if os.name == 'nt': - return { - 'prefix': 'nt', - 'home': 'posix_home', - 'user': 'nt_user', - } - if sys.platform == 'darwin' and sys._framework: - return { - 'prefix': 'posix_prefix', - 'home': 'posix_home', - 'user': 'osx_framework_user', - } - return { - 'prefix': 'posix_prefix', - 'home': 'posix_home', - 'user': 'posix_user', - } - - -def get_preferred_scheme(key): - if key == 'prefix' and sys.prefix != sys.base_prefix: - return 'venv' - scheme = _get_preferred_schemes()[key] - if scheme not in _INSTALL_SCHEMES: - raise ValueError( - f"{key!r} returned {scheme!r}, which is not a valid scheme " - f"on this platform" - ) - return scheme - - -def get_default_scheme(): - return get_preferred_scheme('prefix') - - -def _parse_makefile(filename, vars=None, keep_unresolved=True): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - import re - - if vars is None: - vars = {} - done = {} - notdone = {} - - with open(filename, encoding=sys.getfilesystemencoding(), - errors="surrogateescape") as f: - lines = f.readlines() - - for line in lines: - if line.startswith('#') or line.strip() == '': - continue - m = re.match(_variable_rx, line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - if n in _ALWAYS_STR: - raise ValueError - - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # do variable interpolation here - variables = list(notdone.keys()) - - # Variables with a 'PY_' prefix in the makefile. These need to - # be made available without that prefix through sysconfig. - # Special care is needed to ensure that variable expansion works, even - # if the expansion uses the name without a prefix. - renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - - while len(variables) > 0: - for name in tuple(variables): - value = notdone[name] - m1 = re.search(_findvar1_rx, value) - m2 = re.search(_findvar2_rx, value) - if m1 and m2: - m = m1 if m1.start() < m2.start() else m2 - else: - m = m1 if m1 else m2 - if m is not None: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if (name.startswith('PY_') and - name[3:] in renamed_variables): - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - - else: - done[n] = item = "" - - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: - if name in _ALWAYS_STR: - raise ValueError - value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - variables.remove(name) - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - - else: - # Adds unresolved variables to the done dict. - # This is disabled when called from distutils.sysconfig - if keep_unresolved: - done[name] = value - # bogus variable reference (e.g. "prefix=$/opt/python"); - # just drop it since we can't deal - variables.remove(name) - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - vars.update(done) - return vars - - -def get_makefile_filename(): - """Return the path of the Makefile.""" - if _PYTHON_BUILD: - return os.path.join(_PROJECT_BASE, "Makefile") - if hasattr(sys, 'abiflags'): - config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' - else: - config_dir_name = 'config' - if hasattr(sys.implementation, '_multiarch'): - config_dir_name += f'-{sys.implementation._multiarch}' - return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') - - -def _get_sysconfigdata_name(): - multiarch = getattr(sys.implementation, '_multiarch', '') - return os.environ.get( - '_PYTHON_SYSCONFIGDATA_NAME', - f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', - ) - - -def _generate_posix_vars(): - """Generate the Python module containing build-time variables.""" - import pprint - vars = {} - # load the installed Makefile: - makefile = get_makefile_filename() - try: - _parse_makefile(makefile, vars) - except OSError as e: - msg = f"invalid Python installation: unable to open {makefile}" - if hasattr(e, "strerror"): - msg = f"{msg} ({e.strerror})" - raise OSError(msg) - # load the installed pyconfig.h: - config_h = get_config_h_filename() - try: - with open(config_h, encoding="utf-8") as f: - parse_config_h(f, vars) - except OSError as e: - msg = f"invalid Python installation: unable to open {config_h}" - if hasattr(e, "strerror"): - msg = f"{msg} ({e.strerror})" - raise OSError(msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if _PYTHON_BUILD: - vars['BLDSHARED'] = vars['LDSHARED'] - - # There's a chicken-and-egg situation on OS X with regards to the - # _sysconfigdata module after the changes introduced by #15298: - # get_config_vars() is called by get_platform() as part of the - # `make pybuilddir.txt` target -- which is a precursor to the - # _sysconfigdata.py module being constructed. Unfortunately, - # get_config_vars() eventually calls _init_posix(), which attempts - # to import _sysconfigdata, which we won't have built yet. In order - # for _init_posix() to work, if we're on Darwin, just mock up the - # _sysconfigdata module manually and populate it with the build vars. - # This is more than sufficient for ensuring the subsequent call to - # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module - - pybuilddir = f'build/lib.{get_platform()}-{_PY_VERSION_SHORT}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' - os.makedirs(pybuilddir, exist_ok=True) - destfile = os.path.join(pybuilddir, name + '.py') - - with open(destfile, 'w', encoding='utf8') as f: - f.write('# system configuration generated and used by' - ' the sysconfig module\n') - f.write('build_time_vars = ') - pprint.pprint(vars, stream=f) - - # Create file used for sys.path fixup -- see Modules/getpath.c - with open('pybuilddir.txt', 'w', encoding='utf8') as f: - f.write(pybuilddir) - -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see _generate_posix_vars() - name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars - vars.update(build_time_vars) - -def _init_non_posix(vars): - """Initialize the module as appropriate for NT""" - # set basic install directories - import _imp - vars['LIBDEST'] = get_path('stdlib') - vars['BINLIBDEST'] = get_path('platstdlib') - vars['INCLUDEPY'] = get_path('include') - try: - # GH-99201: _imp.extension_suffixes may be empty when - # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX. - vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - except IndexError: - pass - vars['EXE'] = '.exe' - vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT - vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) - vars['TZPATH'] = '' - -# -# public APIs -# - - -def parse_config_h(fp, vars=None): - """Parse a config.h-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - if vars is None: - vars = {} - import re - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - - while True: - line = fp.readline() - if not line: - break - m = define_rx.match(line) - if m: - n, v = m.group(1, 2) - try: - if n in _ALWAYS_STR: - raise ValueError - v = int(v) - except ValueError: - pass - vars[n] = v - else: - m = undef_rx.match(line) - if m: - vars[m.group(1)] = 0 - return vars - - -def get_config_h_filename(): - """Return the path of pyconfig.h.""" - if _PYTHON_BUILD: - if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") - else: - inc_dir = _PROJECT_BASE - else: - inc_dir = get_path('platinclude') - return os.path.join(inc_dir, 'pyconfig.h') - - -def get_scheme_names(): - """Return a tuple containing the schemes names.""" - return tuple(sorted(_INSTALL_SCHEMES)) - - -def get_path_names(): - """Return a tuple containing the paths names.""" - return _SCHEME_KEYS - - -def get_paths(scheme=get_default_scheme(), vars=None, expand=True): - """Return a mapping containing an install scheme. - - ``scheme`` is the install scheme name. If not provided, it will - return the default scheme for the current platform. - """ - if expand: - return _expand_vars(scheme, vars) - else: - return _INSTALL_SCHEMES[scheme] - - -def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): - """Return a path corresponding to the scheme. - - ``scheme`` is the install scheme name. - """ - return get_paths(scheme, vars, expand)[name] - - -def get_config_vars(*args): - """With no arguments, return a dictionary of all configuration - variables relevant for the current platform. - - On Unix, this means every variable defined in Python's installed Makefile; - On Windows it's a much smaller set. - - With arguments, return a list of values that result from looking up - each argument in the configuration variable dictionary. - """ - global _CONFIG_VARS - if _CONFIG_VARS is None: - _CONFIG_VARS = {} - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX - _CONFIG_VARS['py_version'] = _PY_VERSION - _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT - _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX - _CONFIG_VARS['projectbase'] = _PROJECT_BASE - _CONFIG_VARS['platlibdir'] = sys.platlibdir - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' - try: - _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') - except AttributeError: - _CONFIG_VARS['py_version_nodot_plat'] = '' - - if os.name == 'nt': - _init_non_posix(_CONFIG_VARS) - _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) - if _HAS_USER_BASE: - # Setting 'userbase' is done below the call to the - # init function to enable using 'get_config_var' in - # the init-function. - _CONFIG_VARS['userbase'] = _getuserbase() - - # Always convert srcdir to an absolute path - srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) - if os.name == 'posix': - if _PYTHON_BUILD: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_CONFIG_VARS) - - if args: - vals = [] - for name in args: - vals.append(_CONFIG_VARS.get(name)) - return vals - else: - return _CONFIG_VARS - - -def get_config_var(name): - """Return the value of a single variable using the dictionary returned by - 'get_config_vars()'. - - Equivalent to get_config_vars().get(name) - """ - return get_config_vars().get(name) - - -def get_platform(): - """Return a string that identifies the current platform. - - This is used mainly to distinguish platform-specific build directories and - platform-specific built distributions. Typically includes the OS name and - version and the architecture (as supplied by 'os.uname()'), although the - exact information included depends on the OS; on Linux, the kernel version - isn't particularly important. - - Examples of returned values: - linux-i586 - linux-alpha (?) - solaris-2.6-sun4u - - Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) - win32 (all others - specifically, sys.platform is returned) - - For other non-POSIX platforms, currently just returns 'sys.platform'. - - """ - if os.name == 'nt': - if 'amd64' in sys.version.lower(): - return 'win-amd64' - if '(arm)' in sys.version.lower(): - return 'win-arm32' - if '(arm64)' in sys.version.lower(): - return 'win-arm64' - return sys.platform - - if os.name != "posix" or not hasattr(os, 'uname'): - # XXX what about the architecture? NT is Intel or Alpha - return sys.platform - - # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - - # Try to distinguish various flavours of Unix - osname, host, release, version, machine = os.uname() - - # Convert the OS name to lowercase, remove '/' characters, and translate - # spaces (for "Power Macintosh") - osname = osname.lower().replace('/', '') - machine = machine.replace(' ', '_') - machine = machine.replace('/', '-') - - if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" - elif osname[:5] == "sunos": - if release[0] >= "5": # SunOS 5 == Solaris 2 - osname = "solaris" - release = f"{int(release[0]) - 3}.{release[2:]}" - # We can't use "platform.architecture()[0]" because a - # bootstrap problem. We use a dict to get an error - # if some suspicious happens. - bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} - machine += f".{bitness[sys.maxsize]}" - # fall through to standard osname-release-machine representation - elif osname[:3] == "aix": - from _aix_support import aix_platform - return aix_platform() - elif osname[:6] == "cygwin": - osname = "cygwin" - import re - rel_re = re.compile(r'[\d.]+') - m = rel_re.match(release) - if m: - release = m.group() - elif osname[:6] == "darwin": - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) - - return f"{osname}-{release}-{machine}" - - -def get_python_version(): - return _PY_VERSION_SHORT - - -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. - """ - import re - - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. - - while True: - m = re.search(_findvar1_rx, s) or re.search(_findvar2_rx, s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] - else: - break - return s - - -def _print_dict(title, data): - for index, (key, value) in enumerate(sorted(data.items())): - if index == 0: - print(f'{title}: ') - print(f'\t{key} = "{value}"') - - -def _main(): - """Display all information sysconfig detains.""" - if '--generate-posix-vars' in sys.argv: - _generate_posix_vars() - return - print(f'Platform: "{get_platform()}"') - print(f'Python version: "{get_python_version()}"') - print(f'Current installation scheme: "{get_default_scheme()}"') - print() - _print_dict('Paths', get_paths()) - print() - _print_dict('Variables', get_config_vars()) - -if __name__ == '__main__': - _main() -'''.replace("Python", "RustPython").replace("/python", "/rustpython")) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py new file mode 100644 index 00000000000..ae83243fa3e --- /dev/null +++ b/Lib/sysconfig/__init__.py @@ -0,0 +1,789 @@ +"""Access to Python's configuration information.""" + +import os +import sys +import threading +from os.path import realpath + +__all__ = [ + 'get_config_h_filename', + 'get_config_var', + 'get_config_vars', + 'get_makefile_filename', + 'get_path', + 'get_path_names', + 'get_paths', + 'get_platform', + 'get_python_version', + 'get_scheme_names', + 'parse_config_h', +] + +# Keys for get_config_var() that are never converted to Python integers. +_ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', + 'MACOSX_DEPLOYMENT_TARGET', +} + +_INSTALL_SCHEMES = { + 'posix_prefix': { + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', + 'platinclude': + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', + 'scripts': '{base}/bin', + 'data': '{base}', + }, + 'posix_home': { + 'stdlib': '{installed_base}/lib/{implementation_lower}', + 'platstdlib': '{base}/lib/{implementation_lower}', + 'purelib': '{base}/lib/{implementation_lower}', + 'platlib': '{base}/lib/{implementation_lower}', + 'include': '{installed_base}/include/{implementation_lower}', + 'platinclude': '{installed_base}/include/{implementation_lower}', + 'scripts': '{base}/bin', + 'data': '{base}', + }, + 'nt': { + 'stdlib': '{installed_base}/Lib', + 'platstdlib': '{base}/Lib', + 'purelib': '{base}/Lib/site-packages', + 'platlib': '{base}/Lib/site-packages', + 'include': '{installed_base}/Include', + 'platinclude': '{installed_base}/Include', + 'scripts': '{base}/Scripts', + 'data': '{base}', + }, + + # Downstream distributors can overwrite the default install scheme. + # This is done to support downstream modifications where distributors change + # the installation layout (eg. different site-packages directory). + # So, distributors will change the default scheme to one that correctly + # represents their layout. + # This presents an issue for projects/people that need to bootstrap virtual + # environments, like virtualenv. As distributors might now be customizing + # the default install scheme, there is no guarantee that the information + # returned by sysconfig.get_default_scheme/get_paths is correct for + # a virtual environment, the only guarantee we have is that it is correct + # for the *current* environment. When bootstrapping a virtual environment, + # we need to know its layout, so that we can place the files in the + # correct locations. + # The "*_venv" install scheme is a scheme to bootstrap virtual environments, + # essentially identical to the default posix_prefix/nt schemes. + # Downstream distributors who patch posix_prefix/nt scheme are encouraged to + # leave the following schemes unchanged + 'posix_venv': { + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', + 'platinclude': + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', + 'scripts': '{base}/bin', + 'data': '{base}', + }, + 'nt_venv': { + 'stdlib': '{installed_base}/Lib', + 'platstdlib': '{base}/Lib', + 'purelib': '{base}/Lib/site-packages', + 'platlib': '{base}/Lib/site-packages', + 'include': '{installed_base}/Include', + 'platinclude': '{installed_base}/Include', + 'scripts': '{base}/Scripts', + 'data': '{base}', + }, + } + +# For the OS-native venv scheme, we essentially provide an alias: +if os.name == 'nt': + _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['nt_venv'] +else: + _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] + +def _get_implementation(): + return 'RustPython' # XXX: For site-packages + +# NOTE: site.py has copy of this function. +# Sync it when modify this function. +def _getuserbase(): + env_base = os.environ.get("PYTHONUSERBASE", None) + if env_base: + return env_base + + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories. + # Use _PYTHON_HOST_PLATFORM to get the correct platform when cross-compiling. + system_name = os.environ.get('_PYTHON_HOST_PLATFORM', sys.platform).split('-')[0] + if system_name in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: + return None + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + return joinuser(base, _get_implementation()) + + if sys.platform == "darwin" and sys._framework: + return joinuser("~", "Library", sys._framework, + f"{sys.version_info[0]}.{sys.version_info[1]}") + + return joinuser("~", ".local") + +_HAS_USER_BASE = (_getuserbase() is not None) + +if _HAS_USER_BASE: + _INSTALL_SCHEMES |= { + # NOTE: When modifying "purelib" scheme, update site._get_path() too. + 'nt_user': { + 'stdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'platstdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'purelib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/{implementation}{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts', + 'data': '{userbase}', + }, + 'posix_user': { + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', + 'scripts': '{userbase}/bin', + 'data': '{userbase}', + }, + 'osx_framework_user': { + 'stdlib': '{userbase}/lib/{implementation_lower}', + 'platstdlib': '{userbase}/lib/{implementation_lower}', + 'purelib': '{userbase}/lib/{implementation_lower}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', + 'scripts': '{userbase}/bin', + 'data': '{userbase}', + }, + } + +_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', + 'scripts', 'data') + +_PY_VERSION = sys.version.split()[0] +_PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' +_PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' +_BASE_PREFIX = os.path.normpath(sys.base_prefix) +_BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) +# Mutex guarding initialization of _CONFIG_VARS. +_CONFIG_VARS_LOCK = threading.RLock() +_CONFIG_VARS = None +# True iff _CONFIG_VARS has been fully initialized. +_CONFIG_VARS_INITIALIZED = False +_USER_BASE = None + + +def _safe_realpath(path): + try: + return realpath(path) + except OSError: + return path + +if sys.executable: + _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) +else: + # sys.executable can be empty if argv[0] has been changed and Python is + # unable to retrieve the real program name + _PROJECT_BASE = _safe_realpath(os.getcwd()) + +# In a virtual environment, `sys._home` gives us the target directory +# `_PROJECT_BASE` for the executable that created it when the virtual +# python is an actual executable ('venv --copies' or Windows). +_sys_home = getattr(sys, '_home', None) +if _sys_home: + _PROJECT_BASE = _sys_home + +if os.name == 'nt': + # In a source build, the executable is in a subdirectory of the root + # that we want (\PCbuild\). + # `_BASE_PREFIX` is used as the base installation is where the source + # will be. The realpath is needed to prevent mount point confusion + # that can occur with just string comparisons. + if _safe_realpath(_PROJECT_BASE).startswith( + _safe_realpath(f'{_BASE_PREFIX}\\PCbuild')): + _PROJECT_BASE = _BASE_PREFIX + +# set for cross builds +if "_PYTHON_PROJECT_BASE" in os.environ: + _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) + +def is_python_build(check_home=None): + if check_home is not None: + import warnings + warnings.warn( + ( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + DeprecationWarning, + stacklevel=2, + ) + for fn in ("Setup", "Setup.local"): + if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): + return True + return False + +_PYTHON_BUILD = is_python_build() + +if _PYTHON_BUILD: + for scheme in ('posix_prefix', 'posix_home'): + # On POSIX-y platforms, Python will: + # - Build from .h files in 'headers' (which is only added to the + # scheme when building CPython) + # - Install .h files to 'include' + scheme = _INSTALL_SCHEMES[scheme] + scheme['headers'] = scheme['include'] + scheme['include'] = '{srcdir}/Include' + scheme['platinclude'] = '{projectbase}/.' + del scheme + + +def _subst_vars(s, local_vars): + try: + return s.format(**local_vars) + except KeyError as var: + try: + return s.format(**os.environ) + except KeyError: + raise AttributeError(f'{var}') from None + +def _extend_dict(target_dict, other_dict): + target_keys = target_dict.keys() + for key, value in other_dict.items(): + if key in target_keys: + continue + target_dict[key] = value + + +def _expand_vars(scheme, vars): + res = {} + if vars is None: + vars = {} + _extend_dict(vars, get_config_vars()) + if os.name == 'nt': + # On Windows we want to substitute 'lib' for schemes rather + # than the native value (without modifying vars, in case it + # was passed in) + vars = vars | {'platlibdir': 'lib'} + + for key, value in _INSTALL_SCHEMES[scheme].items(): + if os.name in ('posix', 'nt'): + value = os.path.expanduser(value) + res[key] = os.path.normpath(_subst_vars(value, vars)) + return res + + +def _get_preferred_schemes(): + if os.name == 'nt': + return { + 'prefix': 'nt', + 'home': 'posix_home', + 'user': 'nt_user', + } + if sys.platform == 'darwin' and sys._framework: + return { + 'prefix': 'posix_prefix', + 'home': 'posix_home', + 'user': 'osx_framework_user', + } + + return { + 'prefix': 'posix_prefix', + 'home': 'posix_home', + 'user': 'posix_user', + } + + +def get_preferred_scheme(key): + if key == 'prefix' and sys.prefix != sys.base_prefix: + return 'venv' + scheme = _get_preferred_schemes()[key] + if scheme not in _INSTALL_SCHEMES: + raise ValueError( + f"{key!r} returned {scheme!r}, which is not a valid scheme " + f"on this platform" + ) + return scheme + + +def get_default_scheme(): + return get_preferred_scheme('prefix') + + +def get_makefile_filename(): + """Return the path of the Makefile.""" + + # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python. + if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'): + return os.path.join(cross_base, 'Makefile') + + if _PYTHON_BUILD: + return os.path.join(_PROJECT_BASE, "Makefile") + + if hasattr(sys, 'abiflags'): + config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' + else: + config_dir_name = 'config' + + if hasattr(sys.implementation, '_multiarch'): + config_dir_name += f'-{sys.implementation._multiarch}' + + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') + + +def _import_from_directory(path, name): + if name not in sys.modules: + import importlib.machinery + import importlib.util + + spec = importlib.machinery.PathFinder.find_spec(name, [path]) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[name] = module + return sys.modules[name] + + +def _get_sysconfigdata_name(): + multiarch = getattr(sys.implementation, '_multiarch', '') + return os.environ.get( + '_PYTHON_SYSCONFIGDATA_NAME', + f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', + ) + + +def _get_sysconfigdata(): + import importlib + + name = _get_sysconfigdata_name() + path = os.environ.get('_PYTHON_SYSCONFIGDATA_PATH') + module = _import_from_directory(path, name) if path else importlib.import_module(name) + + return module.build_time_vars + + +def _installation_is_relocated(): + """Is the Python installation running from a different prefix than what was targetted when building?""" + if os.name != 'posix': + raise NotImplementedError('sysconfig._installation_is_relocated() is currently only supported on POSIX') + + data = _get_sysconfigdata() + return ( + data['prefix'] != getattr(sys, 'base_prefix', '') + or data['exec_prefix'] != getattr(sys, 'base_exec_prefix', '') + ) + + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" + # GH-126920: Make sure we don't overwrite any of the keys already set + vars.update(_get_sysconfigdata() | vars) + + +def _init_non_posix(vars): + """Initialize the module as appropriate for NT""" + # set basic install directories + import _winapi + import _sysconfig + vars['LIBDEST'] = get_path('stdlib') + vars['BINLIBDEST'] = get_path('platstdlib') + vars['INCLUDEPY'] = get_path('include') + + # Add EXT_SUFFIX, SOABI, Py_DEBUG, and Py_GIL_DISABLED + vars.update(_sysconfig.config_vars()) + + # NOTE: ABIFLAGS is only an emulated value. It is not present during build + # on Windows. sys.abiflags is absent on Windows and vars['abiflags'] + # is already widely used to calculate paths, so it should remain an + # empty string. + vars['ABIFLAGS'] = ''.join( + ( + 't' if vars['Py_GIL_DISABLED'] else '', + '_d' if vars['Py_DEBUG'] else '', + ), + ) + + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) + if hasattr(sys, 'dllhandle'): + dllhandle = _winapi.GetModuleFileName(sys.dllhandle) + vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle)) + vars['LDLIBRARY'] = vars['LIBRARY'] + vars['EXE'] = '.exe' + vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT + vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) + vars['TZPATH'] = '' + +# +# public APIs +# + + +def parse_config_h(fp, vars=None): + """Parse a config.h-style file. + + A dictionary containing name/value pairs is returned. If an + optional dictionary is passed in as the second argument, it is + used instead of a new dictionary. + """ + if vars is None: + vars = {} + import re + define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") + undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") + + while True: + line = fp.readline() + if not line: + break + m = define_rx.match(line) + if m: + n, v = m.group(1, 2) + try: + if n in _ALWAYS_STR: + raise ValueError + v = int(v) + except ValueError: + pass + vars[n] = v + else: + m = undef_rx.match(line) + if m: + vars[m.group(1)] = 0 + return vars + + +def get_config_h_filename(): + """Return the path of pyconfig.h.""" + if _PYTHON_BUILD: + if os.name == "nt": + inc_dir = os.path.join(_PROJECT_BASE, 'PC') + else: + inc_dir = _PROJECT_BASE + else: + inc_dir = get_path('platinclude') + return os.path.join(inc_dir, 'pyconfig.h') + + +def get_scheme_names(): + """Return a tuple containing the schemes names.""" + return tuple(sorted(_INSTALL_SCHEMES)) + + +def get_path_names(): + """Return a tuple containing the paths names.""" + return _SCHEME_KEYS + + +def get_paths(scheme=get_default_scheme(), vars=None, expand=True): + """Return a mapping containing an install scheme. + + ``scheme`` is the install scheme name. If not provided, it will + return the default scheme for the current platform. + """ + if expand: + return _expand_vars(scheme, vars) + else: + return _INSTALL_SCHEMES[scheme] + + +def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): + """Return a path corresponding to the scheme. + + ``scheme`` is the install scheme name. + """ + return get_paths(scheme, vars, expand)[name] + + +def _init_config_vars(): + global _CONFIG_VARS + _CONFIG_VARS = {} + + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + + try: + abiflags = sys.abiflags + except AttributeError: + abiflags = '' + + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + # If we are cross-compiling, load the prefixes from the Makefile instead. + if '_PYTHON_PROJECT_BASE' in os.environ: + prefix = _CONFIG_VARS['host_prefix'] + exec_prefix = _CONFIG_VARS['host_exec_prefix'] + base_prefix = _CONFIG_VARS['host_prefix'] + base_exec_prefix = _CONFIG_VARS['host_exec_prefix'] + abiflags = _CONFIG_VARS['ABIFLAGS'] + + # Normalized versions of prefix and exec_prefix are handy to have; + # in fact, these are the standard versions used most places in the + # Distutils. + _CONFIG_VARS['prefix'] = prefix + _CONFIG_VARS['exec_prefix'] = exec_prefix + _CONFIG_VARS['py_version'] = _PY_VERSION + _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT + _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT + _CONFIG_VARS['installed_base'] = base_prefix + _CONFIG_VARS['base'] = prefix + _CONFIG_VARS['installed_platbase'] = base_exec_prefix + _CONFIG_VARS['platbase'] = exec_prefix + _CONFIG_VARS['projectbase'] = _PROJECT_BASE + _CONFIG_VARS['platlibdir'] = sys.platlibdir + _CONFIG_VARS['implementation'] = _get_implementation() + _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() + _CONFIG_VARS['abiflags'] = abiflags + try: + _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') + except AttributeError: + _CONFIG_VARS['py_version_nodot_plat'] = '' + + if os.name == 'nt': + _init_non_posix(_CONFIG_VARS) + _CONFIG_VARS['VPATH'] = sys._vpath + if _HAS_USER_BASE: + # Setting 'userbase' is done below the call to the + # init function to enable using 'get_config_var' in + # the init-function. + _CONFIG_VARS['userbase'] = _getuserbase() + + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + + # Always convert srcdir to an absolute path + srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) + if os.name == 'posix': + if _PYTHON_BUILD: + # If srcdir is a relative path (typically '.' or '..') + # then it should be interpreted relative to the directory + # containing Makefile. + base = os.path.dirname(get_makefile_filename()) + srcdir = os.path.join(base, srcdir) + else: + # srcdir is not meaningful since the installation is + # spread about the filesystem. We choose the + # directory containing the Makefile since we know it + # exists. + srcdir = os.path.dirname(get_makefile_filename()) + _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) + + # OS X platforms require special customization to handle + # multi-architecture, multi-os-version installers + if sys.platform == 'darwin': + import _osx_support + _osx_support.customize_config_vars(_CONFIG_VARS) + + global _CONFIG_VARS_INITIALIZED + _CONFIG_VARS_INITIALIZED = True + + +def get_config_vars(*args): + """With no arguments, return a dictionary of all configuration + variables relevant for the current platform. + + On Unix, this means every variable defined in Python's installed Makefile; + On Windows it's a much smaller set. + + With arguments, return a list of values that result from looking up + each argument in the configuration variable dictionary. + """ + global _CONFIG_VARS_INITIALIZED + + # Avoid claiming the lock once initialization is complete. + if _CONFIG_VARS_INITIALIZED: + # GH-126789: If sys.prefix or sys.exec_prefix were updated, invalidate the cache. + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + if _CONFIG_VARS['prefix'] != prefix or _CONFIG_VARS['exec_prefix'] != exec_prefix: + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() + else: + # Initialize the config_vars cache. + with _CONFIG_VARS_LOCK: + # Test again with the lock held to avoid races. Note that + # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, + # to ensure that recursive calls to get_config_vars() + # don't re-enter init_config_vars(). + if _CONFIG_VARS is None: + _init_config_vars() + + if args: + vals = [] + for name in args: + vals.append(_CONFIG_VARS.get(name)) + return vals + else: + return _CONFIG_VARS + + +def get_config_var(name): + """Return the value of a single variable using the dictionary returned by + 'get_config_vars()'. + + Equivalent to get_config_vars().get(name) + """ + return get_config_vars().get(name) + + +def get_platform(): + """Return a string that identifies the current platform. + + This is used mainly to distinguish platform-specific build directories and + platform-specific built distributions. Typically includes the OS name and + version and the architecture (as supplied by 'os.uname()'), although the + exact information included depends on the OS; on Linux, the kernel version + isn't particularly important. + + Examples of returned values: + + + Windows: + + - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) + - win-arm64 (64-bit Windows on ARM64, aka AArch64) + - win32 (all others - specifically, sys.platform is returned) + + POSIX based OS: + + - linux-x86_64 + - macosx-15.5-arm64 + - macosx-26.0-universal2 (macOS on Apple Silicon or Intel) + - android-24-arm64_v8a + + For other non-POSIX platforms, currently just returns :data:`sys.platform`.""" + if os.name == 'nt': + if 'amd64' in sys.version.lower(): + return 'win-amd64' + if '(arm)' in sys.version.lower(): + return 'win-arm32' + if '(arm64)' in sys.version.lower(): + return 'win-arm64' + return sys.platform + + if os.name != "posix" or not hasattr(os, 'uname'): + # XXX what about the architecture? NT is Intel or Alpha + return sys.platform + + # Set for cross builds explicitly + if "_PYTHON_HOST_PLATFORM" in os.environ: + osname, _, machine = os.environ["_PYTHON_HOST_PLATFORM"].partition('-') + release = None + else: + # Try to distinguish various flavours of Unix + osname, host, release, version, machine = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname == "android" or sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + elif osname == "linux": + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" + elif osname[:5] == "sunos": + if release[0] >= "5": # SunOS 5 == Solaris 2 + osname = "solaris" + release = f"{int(release[0]) - 3}.{release[2:]}" + # We can't use "platform.architecture()[0]" because a + # bootstrap problem. We use a dict to get an error + # if some suspicious happens. + bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} + machine += f".{bitness[sys.maxsize]}" + # fall through to standard osname-release-machine representation + elif osname[:3] == "aix": + from _aix_support import aix_platform + return aix_platform() + elif osname[:6] == "cygwin": + osname = "cygwin" + import re + rel_re = re.compile(r'[\d.]+') + m = rel_re.match(release) + if m: + release = m.group() + elif osname[:6] == "darwin": + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) + + return '-'.join(map(str, filter(None, (osname, release, machine)))) + + +def get_python_version(): + return _PY_VERSION_SHORT + + +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + +def expand_makefile_vars(s, vars): + """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in + 'string' according to 'vars' (a dictionary mapping variable names to + values). Variables not present in 'vars' are silently expanded to the + empty string. The variable values in 'vars' should not contain further + variable expansions; if 'vars' is the output of 'parse_makefile()', + you're fine. Returns a variable-expanded version of 's'. + """ + + import warnings + warnings.warn( + 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' + 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', + DeprecationWarning, + stacklevel=2, + ) + + import re + + _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" + _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + + # This algorithm does multiple expansion, so if vars['foo'] contains + # "${bar}", it will expand ${foo} to ${bar}, and then expand + # ${bar}... and so forth. This is fine as long as 'vars' comes from + # 'parse_makefile()', which takes care of such expansions eagerly, + # according to make's variable expansion semantics. + + while True: + m = re.search(_findvar1_rx, s) or re.search(_findvar2_rx, s) + if m: + (beg, end) = m.span() + s = s[0:beg] + vars.get(m.group(1)) + s[end:] + else: + break + return s diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py new file mode 100644 index 00000000000..bc2197cfe79 --- /dev/null +++ b/Lib/sysconfig/__main__.py @@ -0,0 +1,276 @@ +import json +import os +import sys +import types +from sysconfig import ( + _ALWAYS_STR, + _PYTHON_BUILD, + _get_sysconfigdata_name, + get_config_h_filename, + get_config_var, + get_config_vars, + get_default_scheme, + get_makefile_filename, + get_paths, + get_platform, + get_python_version, + parse_config_h, +) + + +# Regexes needed for parsing Makefile (and similar syntaxes, +# like old-style Setup files). +_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" +_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" +_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + + +def _parse_makefile(filename, vars=None, keep_unresolved=True): + """Parse a Makefile-style file. + + A dictionary containing name/value pairs is returned. If an + optional dictionary is passed in as the second argument, it is + used instead of a new dictionary. + """ + import re + + if vars is None: + vars = {} + done = {} + notdone = {} + + with open(filename, encoding=sys.getfilesystemencoding(), + errors="surrogateescape") as f: + lines = f.readlines() + + for line in lines: + if line.startswith('#') or line.strip() == '': + continue + m = re.match(_variable_rx, line) + if m: + n, v = m.group(1, 2) + v = v.strip() + # `$$' is a literal `$' in make + tmpv = v.replace('$$', '') + + if "$" in tmpv: + notdone[n] = v + else: + try: + if n in _ALWAYS_STR: + raise ValueError + + v = int(v) + except ValueError: + # insert literal `$' + done[n] = v.replace('$$', '$') + else: + done[n] = v + + # do variable interpolation here + variables = list(notdone.keys()) + + # Variables with a 'PY_' prefix in the makefile. These need to + # be made available without that prefix through sysconfig. + # Special care is needed to ensure that variable expansion works, even + # if the expansion uses the name without a prefix. + renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') + + while len(variables) > 0: + for name in tuple(variables): + value = notdone[name] + m1 = re.search(_findvar1_rx, value) + m2 = re.search(_findvar2_rx, value) + if m1 and m2: + m = m1 if m1.start() < m2.start() else m2 + else: + m = m1 if m1 else m2 + if m is not None: + n = m.group(1) + found = True + if n in done: + item = str(done[n]) + elif n in notdone: + # get it on a subsequent round + found = False + elif n in os.environ: + # do it like make: fall back to environment + item = os.environ[n] + + elif n in renamed_variables: + if (name.startswith('PY_') and + name[3:] in renamed_variables): + item = "" + + elif 'PY_' + n in notdone: + found = False + + else: + item = str(done['PY_' + n]) + + else: + done[n] = item = "" + + if found: + after = value[m.end():] + value = value[:m.start()] + item + after + if "$" in after: + notdone[name] = value + else: + try: + if name in _ALWAYS_STR: + raise ValueError + value = int(value) + except ValueError: + done[name] = value.strip() + else: + done[name] = value + variables.remove(name) + + if name.startswith('PY_') \ + and name[3:] in renamed_variables: + + name = name[3:] + if name not in done: + done[name] = value + + else: + # Adds unresolved variables to the done dict. + # This is disabled when called from distutils.sysconfig + if keep_unresolved: + done[name] = value + # bogus variable reference (e.g. "prefix=$/opt/python"); + # just drop it since we can't deal + variables.remove(name) + + # strip spurious spaces + for k, v in done.items(): + if isinstance(v, str): + done[k] = v.strip() + + # save the results in the global dictionary + vars.update(done) + return vars + + +def _print_config_dict(d, stream): + print ("{", file=stream) + for k, v in sorted(d.items()): + print(f" {k!r}: {v!r},", file=stream) + print ("}", file=stream) + + +def _get_pybuilddir(): + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if get_config_var('Py_DEBUG') == '1': + pybuilddir += '-pydebug' + return pybuilddir + + +def _get_json_data_name(): + name = _get_sysconfigdata_name() + assert name.startswith('_sysconfigdata') + return name.replace('_sysconfigdata', '_sysconfig_vars') + '.json' + + +def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" + vars = {} + # load the installed Makefile: + makefile = get_makefile_filename() + try: + _parse_makefile(makefile, vars) + except OSError as e: + msg = f"invalid Python installation: unable to open {makefile}" + if hasattr(e, "strerror"): + msg = f"{msg} ({e.strerror})" + raise OSError(msg) + # load the installed pyconfig.h: + config_h = get_config_h_filename() + try: + with open(config_h, encoding="utf-8") as f: + parse_config_h(f, vars) + except OSError as e: + msg = f"invalid Python installation: unable to open {config_h}" + if hasattr(e, "strerror"): + msg = f"{msg} ({e.strerror})" + raise OSError(msg) + # On AIX, there are wrong paths to the linker scripts in the Makefile + # -- these paths are relative to the Python source, but when installed + # the scripts are in another directory. + if _PYTHON_BUILD: + vars['BLDSHARED'] = vars['LDSHARED'] + + name = _get_sysconfigdata_name() + + # There's a chicken-and-egg situation on OS X with regards to the + # _sysconfigdata module after the changes introduced by #15298: + # get_config_vars() is called by get_platform() as part of the + # `make pybuilddir.txt` target -- which is a precursor to the + # _sysconfigdata.py module being constructed. Unfortunately, + # get_config_vars() eventually calls _init_posix(), which attempts + # to import _sysconfigdata, which we won't have built yet. In order + # for _init_posix() to work, if we're on Darwin, just mock up the + # _sysconfigdata module manually and populate it with the build vars. + # This is more than sufficient for ensuring the subsequent call to + # get_platform() succeeds. + # GH-127178: Since we started generating a .json file, we also need this to + # be able to run sysconfig.get_config_vars(). + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module + + pybuilddir = _get_pybuilddir() + os.makedirs(pybuilddir, exist_ok=True) + destfile = os.path.join(pybuilddir, name + '.py') + + with open(destfile, 'w', encoding='utf8') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') + f.write('build_time_vars = ') + _print_config_dict(vars, stream=f) + + print(f'Written {destfile}') + + install_vars = get_config_vars() + # Fix config vars to match the values after install (of the default environment) + install_vars['projectbase'] = install_vars['BINDIR'] + install_vars['srcdir'] = install_vars['LIBPL'] + # Write a JSON file with the output of sysconfig.get_config_vars + jsonfile = os.path.join(pybuilddir, _get_json_data_name()) + with open(jsonfile, 'w') as f: + json.dump(install_vars, f, indent=2) + + print(f'Written {jsonfile}') + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w', encoding='utf8') as f: + f.write(pybuilddir) + + +def _print_dict(title, data): + for index, (key, value) in enumerate(sorted(data.items())): + if index == 0: + print(f'{title}: ') + print(f'\t{key} = "{value}"') + + +def _main(): + """Display all information sysconfig detains.""" + if '--generate-posix-vars' in sys.argv: + _generate_posix_vars() + return + print(f'Platform: "{get_platform()}"') + print(f'Python version: "{get_python_version()}"') + print(f'Current installation scheme: "{get_default_scheme()}"') + print() + _print_dict('Paths', get_paths()) + print() + _print_dict('Variables', get_config_vars()) + + +if __name__ == '__main__': + try: + _main() + except BrokenPipeError: + pass diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py old mode 100755 new mode 100644 index 7973f26f98b..c0097351b26 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """The Tab Nanny despises ambiguous indentation. She knows no mercy. tabnanny -- Detection of ambiguous indentation @@ -23,8 +21,6 @@ import os import sys import tokenize -if not hasattr(tokenize, 'NL'): - raise ValueError("tokenize.NL doesn't exist -- tokenize module too old") __all__ = ["check", "NannyNag", "process_tokens"] @@ -37,6 +33,7 @@ def errprint(*args): sys.stderr.write(sep + str(arg)) sep = " " sys.stderr.write("\n") + sys.exit(1) def main(): import getopt @@ -46,7 +43,6 @@ def main(): opts, args = getopt.getopt(sys.argv[1:], "qv") except getopt.error as msg: errprint(msg) - return for o, a in opts: if o == '-q': filename_only = filename_only + 1 @@ -54,7 +50,6 @@ def main(): verbose = verbose + 1 if not args: errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...") - return for arg in args: check(arg) @@ -114,6 +109,10 @@ def check(file): errprint("%r: Indentation Error: %s" % (file, msg)) return + except SyntaxError as msg: + errprint("%r: Syntax Error: %s" % (file, msg)) + return + except NannyNag as nag: badline = nag.get_lineno() line = nag.get_line() @@ -275,6 +274,12 @@ def format_witnesses(w): return prefix + " " + ', '.join(firsts) def process_tokens(tokens): + try: + _process_tokens(tokens) + except TabError as e: + raise NannyNag(e.lineno, e.msg, e.text) + +def _process_tokens(tokens): INDENT = tokenize.INDENT DEDENT = tokenize.DEDENT NEWLINE = tokenize.NEWLINE diff --git a/Lib/tarfile.py b/Lib/tarfile.py old mode 100755 new mode 100644 index dea150e8dbb..c7e9f7d681a --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 #------------------------------------------------------------------- # tarfile.py #------------------------------------------------------------------- @@ -57,19 +56,19 @@ grp = None # os.symlink on Windows prior to 6.0 raises NotImplementedError -symlink_exception = (AttributeError, NotImplementedError) -try: - # OSError (winerror=1314) will be raised if the caller does not hold the - # SeCreateSymbolicLinkPrivilege privilege - symlink_exception += (OSError,) -except NameError: - pass +# OSError (winerror=1314) will be raised if the caller does not hold the +# SeCreateSymbolicLinkPrivilege privilege +symlink_exception = (AttributeError, NotImplementedError, OSError) # from tarfile import * __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", "CompressionError", "StreamError", "ExtractError", "HeaderError", "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", - "DEFAULT_FORMAT", "open"] + "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", + "tar_filter", "FilterError", "AbsoluteLinkError", + "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", + "LinkOutsideDestinationError", "LinkFallbackError"] + #--------------------------------------------------------- # tar constants @@ -158,6 +157,8 @@ def stn(s, length, encoding, errors): """Convert a string to a null-terminated bytes object. """ + if s is None: + raise ValueError("metadata cannot contain None") s = s.encode(encoding, errors) return s[:length] + (length - len(s)) * NUL @@ -328,15 +329,17 @@ def write(self, s): class _Stream: """Class that serves as an adapter between TarFile and a stream-like object. The stream-like object only - needs to have a read() or write() method and is accessed - blockwise. Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin, - sys.stdout, a socket, a tape device etc. + needs to have a read() or write() method that works with bytes, + and the method is accessed blockwise. + Use of gzip or bzip2 compression is possible. + A stream-like object could be for example: sys.stdin.buffer, + sys.stdout.buffer, a socket, a tape device etc. _Stream is intended to be used only internally. """ - def __init__(self, name, mode, comptype, fileobj, bufsize): + def __init__(self, name, mode, comptype, fileobj, bufsize, + compresslevel, preset): """Construct a _Stream object. """ self._extfileobj = True @@ -350,7 +353,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): fileobj = _StreamProxy(fileobj) comptype = fileobj.getcomptype() - self.name = name or "" + self.name = os.fspath(name) if name is not None else "" self.mode = mode self.comptype = comptype self.fileobj = fileobj @@ -368,10 +371,10 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": - self._init_read_gz() self.exception = zlib.error + self._init_read_gz() else: - self._init_write_gz() + self._init_write_gz(compresslevel) elif comptype == "bz2": try: @@ -383,7 +386,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.cmp = bz2.BZ2Decompressor() self.exception = OSError else: - self.cmp = bz2.BZ2Compressor() + self.cmp = bz2.BZ2Compressor(compresslevel) elif comptype == "xz": try: @@ -395,8 +398,18 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.cmp = lzma.LZMADecompressor() self.exception = lzma.LZMAError else: - self.cmp = lzma.LZMACompressor() - + self.cmp = lzma.LZMACompressor(preset=preset) + elif comptype == "zst": + try: + from compression import zstd + except ImportError: + raise CompressionError("compression.zstd module is not available") from None + if mode == "r": + self.dbuf = b"" + self.cmp = zstd.ZstdDecompressor() + self.exception = zstd.ZstdError + else: + self.cmp = zstd.ZstdCompressor() elif comptype != "tar": raise CompressionError("unknown compression type %r" % comptype) @@ -410,13 +423,14 @@ def __del__(self): if hasattr(self, "closed") and not self.closed: self.close() - def _init_write_gz(self): + def _init_write_gz(self, compresslevel): """Initialize for writing with gzip compression. """ - self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED, - -self.zlib.MAX_WBITS, - self.zlib.DEF_MEM_LEVEL, - 0) + self.cmp = self.zlib.compressobj(compresslevel, + self.zlib.DEFLATED, + -self.zlib.MAX_WBITS, + self.zlib.DEF_MEM_LEVEL, + 0) timestamp = struct.pack("" % (self.__class__.__name__,self.name,id(self)) + def replace(self, *, + name=_KEEP, mtime=_KEEP, mode=_KEEP, linkname=_KEEP, + uid=_KEEP, gid=_KEEP, uname=_KEEP, gname=_KEEP, + deep=True, _KEEP=_KEEP): + """Return a deep copy of self with the given attributes replaced. + """ + if deep: + result = copy.deepcopy(self) + else: + result = copy.copy(self) + if name is not _KEEP: + result.name = name + if mtime is not _KEEP: + result.mtime = mtime + if mode is not _KEEP: + result.mode = mode + if linkname is not _KEEP: + result.linkname = linkname + if uid is not _KEEP: + result.uid = uid + if gid is not _KEEP: + result.gid = gid + if uname is not _KEEP: + result.uname = uname + if gname is not _KEEP: + result.gname = gname + return result + def get_info(self): """Return the TarInfo's attributes as a dictionary. """ + if self.mode is None: + mode = None + else: + mode = self.mode & 0o7777 info = { "name": self.name, - "mode": self.mode & 0o7777, + "mode": mode, "uid": self.uid, "gid": self.gid, "size": self.size, @@ -820,6 +1035,9 @@ def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescap """Return a tar header as a string of 512 byte blocks. """ info = self.get_info() + for name, value in info.items(): + if value is None: + raise ValueError("%s may not be None" % name) if format == USTAR_FORMAT: return self.create_ustar_header(info, encoding, errors) @@ -950,6 +1168,12 @@ def _create_header(info, format, encoding, errors): devmajor = stn("", 8, encoding, errors) devminor = stn("", 8, encoding, errors) + # None values in metadata should cause ValueError. + # itn()/stn() do this for all fields except type. + filetype = info.get("type", REGTYPE) + if filetype is None: + raise ValueError("TarInfo.type must not be None") + parts = [ stn(info.get("name", ""), 100, encoding, errors), itn(info.get("mode", 0) & 0o7777, 8, format), @@ -958,7 +1182,7 @@ def _create_header(info, format, encoding, errors): itn(info.get("size", 0), 12, format), itn(info.get("mtime", 0), 12, format), b" ", # checksum field - info.get("type", REGTYPE), + filetype, stn(info.get("linkname", ""), 100, encoding, errors), info.get("magic", POSIX_MAGIC), stn(info.get("uname", ""), 32, encoding, errors), @@ -1024,7 +1248,7 @@ def _create_pax_generic_header(cls, pax_headers, type, encoding): for keyword, value in pax_headers.items(): keyword = keyword.encode("utf-8") if binary: - # Try to restore the original byte representation of `value'. + # Try to restore the original byte representation of 'value'. # Needless to say, that the encoding must match the string. value = value.encode(encoding, "surrogateescape") else: @@ -1240,41 +1464,59 @@ def _proc_pax(self, tarfile): else: pax_headers = tarfile.pax_headers.copy() - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - # Parse pax header information. A record looks like that: # "%d %s=%s\n" % (length, keyword, value). length is the size # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") + # the newline. pos = 0 - while True: - match = regex.match(buf, pos) - if not match: - break + encoding = None + raw_headers = [] + while len(buf) > pos and buf[pos] != 0x00: + if not (match := _header_length_prefix_re.match(buf, pos)): + raise InvalidHeaderError("invalid header") + try: + length = int(match.group(1)) + except ValueError: + raise InvalidHeaderError("invalid header") + # Headers must be at least 5 bytes, shortest being '5 x=\n'. + # Value is allowed to be empty. + if length < 5: + raise InvalidHeaderError("invalid header") + if pos + length > len(buf): + raise InvalidHeaderError("invalid header") + + header_value_end_offset = match.start(1) + length - 1 # Last byte of the header + keyword_and_value = buf[match.end(1) + 1:header_value_end_offset] + raw_keyword, equals, raw_value = keyword_and_value.partition(b"=") - length, keyword = match.groups() - length = int(length) - if length == 0: + # Check the framing of the header. The last character must be '\n' (0x0A) + if not raw_keyword or equals != b"=" or buf[header_value_end_offset] != 0x0A: raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] + raw_headers.append((length, raw_keyword, raw_value)) + + # Check if the pax header contains a hdrcharset field. This tells us + # the encoding of the path, linkpath, uname and gname fields. Normally, + # these fields are UTF-8 encoded but since POSIX.1-2008 tar + # implementations are allowed to store them as raw binary strings if + # the translation to UTF-8 fails. For the time being, we don't care about + # anything other than "BINARY". The only other value that is currently + # allowed by the standard is "ISO-IR 10646 2000 UTF-8" in other words UTF-8. + # Note that we only follow the initial 'hdrcharset' setting to preserve + # the initial behavior of the 'tarfile' module. + if raw_keyword == b"hdrcharset" and encoding is None: + if raw_value == b"BINARY": + encoding = tarfile.encoding + else: # This branch ensures only the first 'hdrcharset' header is used. + encoding = "utf-8" + + pos += length + + # If no explicit hdrcharset is set, we use UTF-8 as a default. + if encoding is None: + encoding = "utf-8" + # After parsing the raw headers we can decode them to text. + for length, raw_keyword, raw_value in raw_headers: # Normally, we could just use "utf-8" as the encoding and "strict" # as the error handler, but we better not take the risk. For # example, GNU tar <= 1.23 is known to store filenames it cannot @@ -1282,17 +1524,16 @@ def _proc_pax(self, tarfile): # hdrcharset=BINARY header). # We first try the strict standard encoding, and if that fails we # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", + keyword = self._decode_pax_field(raw_keyword, "utf-8", "utf-8", tarfile.errors) if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, + value = self._decode_pax_field(raw_value, encoding, tarfile.encoding, tarfile.errors) else: - value = self._decode_pax_field(value, "utf-8", "utf-8", + value = self._decode_pax_field(raw_value, "utf-8", "utf-8", tarfile.errors) pax_headers[keyword] = value - pos += length # Fetch the next header. try: @@ -1307,7 +1548,7 @@ def _proc_pax(self, tarfile): elif "GNU.sparse.size" in pax_headers: # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) + self._proc_gnusparse_00(next, raw_headers) elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": # GNU extended sparse format version 1.0. @@ -1329,15 +1570,24 @@ def _proc_pax(self, tarfile): return next - def _proc_gnusparse_00(self, next, pax_headers, buf): + def _proc_gnusparse_00(self, next, raw_headers): """Process a GNU tar extended sparse header, version 0.0. """ offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) + for _, keyword, value in raw_headers: + if keyword == b"GNU.sparse.offset": + try: + offsets.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + + elif keyword == b"GNU.sparse.numbytes": + try: + numbytes.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + next.sparse = list(zip(offsets, numbytes)) def _proc_gnusparse_01(self, next, pax_headers): @@ -1397,6 +1647,9 @@ def _block(self, count): """Round up a byte count by BLOCKSIZE and return it, e.g. _block(834) => 1024. """ + # Only non-negative offsets are allowed + if count < 0: + raise InvalidHeaderError("invalid offset") blocks, remainder = divmod(count, BLOCKSIZE) if remainder: blocks += 1 @@ -1468,17 +1721,19 @@ class TarFile(object): fileobject = ExFileObject # The file-object for extractfile(). + extraction_filter = None # The default filter for extraction. + def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, - errorlevel=None, copybufsize=None): - """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to + errorlevel=None, copybufsize=None, stream=False): + """Open an (uncompressed) tar archive 'name'. 'mode' is either 'r' to read from an existing archive, 'a' to append data to an existing - file or 'w' to create a new file overwriting an existing one. `mode' + file or 'w' to create a new file overwriting an existing one. 'mode' defaults to 'r'. - If `fileobj' is given, it is used for reading or writing data. If it - can be determined, `mode' is overridden by `fileobj's mode. - `fileobj' is not closed, when TarFile is closed. + If 'fileobj' is given, it is used for reading or writing data. If it + can be determined, 'mode' is overridden by 'fileobj's mode. + 'fileobj' is not closed, when TarFile is closed. """ modes = {"r": "rb", "a": "r+b", "w": "wb", "x": "xb"} if mode not in modes: @@ -1503,6 +1758,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, self.name = os.path.abspath(name) if name else None self.fileobj = fileobj + self.stream = stream + # Init attributes. if format is not None: self.format = format @@ -1535,6 +1792,8 @@ def __init__(self, name=None, mode="r", fileobj=None, format=None, # current position in the archive file self.inodes = {} # dictionary caching the inodes of # archive members already added + self._unames = {} # Cached mappings of uid -> uname + self._gnames = {} # Cached mappings of gid -> gname try: if self.mode == "r": @@ -1590,11 +1849,13 @@ def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): 'r:gz' open for reading with gzip compression 'r:bz2' open for reading with bzip2 compression 'r:xz' open for reading with lzma compression + 'r:zst' open for reading with zstd compression 'a' or 'a:' open for appending, creating the file if necessary 'w' or 'w:' open for writing without compression 'w:gz' open for writing with gzip compression 'w:bz2' open for writing with bzip2 compression 'w:xz' open for writing with lzma compression + 'w:zst' open for writing with zstd compression 'x' or 'x:' create a tarfile exclusively without compression, raise an exception if the file is already created @@ -1604,16 +1865,20 @@ def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs): if the file is already created 'x:xz' create an lzma compressed tarfile, raise an exception if the file is already created + 'x:zst' create a zstd compressed tarfile, raise an exception + if the file is already created 'r|*' open a stream of tar blocks with transparent compression 'r|' open an uncompressed stream of tar blocks for reading 'r|gz' open a gzip compressed stream of tar blocks 'r|bz2' open a bzip2 compressed stream of tar blocks 'r|xz' open an lzma compressed stream of tar blocks + 'r|zst' open a zstd compressed stream of tar blocks 'w|' open an uncompressed stream for writing 'w|gz' open a gzip compressed stream for writing 'w|bz2' open a bzip2 compressed stream for writing 'w|xz' open an lzma compressed stream for writing + 'w|zst' open a zstd compressed stream for writing """ if not name and not fileobj: @@ -1658,8 +1923,17 @@ def not_compressed(comptype): if filemode not in ("r", "w"): raise ValueError("mode must be 'r' or 'w'") - - stream = _Stream(name, filemode, comptype, fileobj, bufsize) + if "compresslevel" in kwargs and comptype not in ("gz", "bz2"): + raise ValueError( + "compresslevel is only valid for w|gz and w|bz2 modes" + ) + if "preset" in kwargs and comptype not in ("xz",): + raise ValueError("preset is only valid for w|xz mode") + + compresslevel = kwargs.pop("compresslevel", 9) + preset = kwargs.pop("preset", None) + stream = _Stream(name, filemode, comptype, fileobj, bufsize, + compresslevel, preset) try: t = cls(name, filemode, stream, **kwargs) except: @@ -1770,12 +2044,48 @@ def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): t._extfileobj = False return t + @classmethod + def zstopen(cls, name, mode="r", fileobj=None, level=None, options=None, + zstd_dict=None, **kwargs): + """Open zstd compressed tar archive name for reading or writing. + Appending is not allowed. + """ + if mode not in ("r", "w", "x"): + raise ValueError("mode must be 'r', 'w' or 'x'") + + try: + from compression.zstd import ZstdFile, ZstdError + except ImportError: + raise CompressionError("compression.zstd module is not available") from None + + fileobj = ZstdFile( + fileobj or name, + mode, + level=level, + options=options, + zstd_dict=zstd_dict + ) + + try: + t = cls.taropen(name, mode, fileobj, **kwargs) + except (ZstdError, EOFError) as e: + fileobj.close() + if mode == 'r': + raise ReadError("not a zstd file") from e + raise + except Exception: + fileobj.close() + raise + t._extfileobj = False + return t + # All *open() methods are registered here. OPEN_METH = { "tar": "taropen", # uncompressed tar "gz": "gzopen", # gzip compressed tar "bz2": "bz2open", # bzip2 compressed tar - "xz": "xzopen" # lzma compressed tar + "xz": "xzopen", # lzma compressed tar + "zst": "zstopen", # zstd compressed tar } #-------------------------------------------------------------------------- @@ -1803,7 +2113,7 @@ def close(self): self.fileobj.close() def getmember(self, name): - """Return a TarInfo object for member `name'. If `name' can not be + """Return a TarInfo object for member 'name'. If 'name' can not be found in the archive, KeyError is raised. If a member occurs more than once in the archive, its last occurrence is assumed to be the most up-to-date version. @@ -1831,9 +2141,9 @@ def getnames(self): def gettarinfo(self, name=None, arcname=None, fileobj=None): """Create a TarInfo object from the result of os.stat or equivalent - on an existing file. The file is either named by `name', or - specified as a file object `fileobj' with a file descriptor. If - given, `arcname' specifies an alternative name for the file in the + on an existing file. The file is either named by 'name', or + specified as a file object 'fileobj' with a file descriptor. If + given, 'arcname' specifies an alternative name for the file in the archive, otherwise, the name is taken from the 'name' attribute of 'fileobj', or the 'name' argument. The name should be a text string. @@ -1857,7 +2167,7 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): # Now, fill the TarInfo object with # information specific for the file. tarinfo = self.tarinfo() - tarinfo.tarfile = self # Not needed + tarinfo._tarfile = self # To be removed in 3.16. # Use os.stat or os.lstat, depending on if symlinks shall be resolved. if fileobj is None: @@ -1911,16 +2221,23 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): tarinfo.mtime = statres.st_mtime tarinfo.type = type tarinfo.linkname = linkname + + # Calls to pwd.getpwuid() and grp.getgrgid() tend to be expensive. To + # speed things up, cache the resolved usernames and group names. if pwd: - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass + if tarinfo.uid not in self._unames: + try: + self._unames[tarinfo.uid] = pwd.getpwuid(tarinfo.uid)[0] + except KeyError: + self._unames[tarinfo.uid] = '' + tarinfo.uname = self._unames[tarinfo.uid] if grp: - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass + if tarinfo.gid not in self._gnames: + try: + self._gnames[tarinfo.gid] = grp.getgrgid(tarinfo.gid)[0] + except KeyError: + self._gnames[tarinfo.gid] = '' + tarinfo.gname = self._gnames[tarinfo.gid] if type in (CHRTYPE, BLKTYPE): if hasattr(os, "major") and hasattr(os, "minor"): @@ -1929,18 +2246,26 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If `verbose' is False, only - the names of the members are printed. If it is True, an `ls -l'-like - output is produced. `members' is optional and must be a subset of the + """Print a table of contents to sys.stdout. If 'verbose' is False, only + the names of the members are printed. If it is True, an 'ls -l'-like + output is produced. 'members' is optional and must be a subset of the list returned by getmembers(). """ + # Convert tarinfo type to stat type. + type2mode = {REGTYPE: stat.S_IFREG, SYMTYPE: stat.S_IFLNK, + FIFOTYPE: stat.S_IFIFO, CHRTYPE: stat.S_IFCHR, + DIRTYPE: stat.S_IFDIR, BLKTYPE: stat.S_IFBLK} self._check() if members is None: members = self for tarinfo in members: if verbose: - _safe_print(stat.filemode(tarinfo.mode)) + if tarinfo.mode is None: + _safe_print("??????????") + else: + modetype = type2mode.get(tarinfo.type, 0) + _safe_print(stat.filemode(modetype | tarinfo.mode)) _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, tarinfo.gname or tarinfo.gid)) if tarinfo.ischr() or tarinfo.isblk(): @@ -1948,8 +2273,11 @@ def list(self, verbose=True, *, members=None): ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) else: _safe_print("%10d" % tarinfo.size) - _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(tarinfo.mtime)[:6]) + if tarinfo.mtime is None: + _safe_print("????-??-?? ??:??:??") + else: + _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ + % time.localtime(tarinfo.mtime)[:6]) _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) @@ -1961,11 +2289,11 @@ def list(self, verbose=True, *, members=None): print() def add(self, name, arcname=None, recursive=True, *, filter=None): - """Add the file `name' to the archive. `name' may be any type of file - (directory, fifo, symbolic link, etc.). If given, `arcname' + """Add the file 'name' to the archive. 'name' may be any type of file + (directory, fifo, symbolic link, etc.). If given, 'arcname' specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by - setting `recursive' to False. `filter' is a function + setting 'recursive' to False. 'filter' is a function that expects a TarInfo object argument and returns the changed TarInfo object, if it returns None the TarInfo object will be excluded from the archive. @@ -2012,13 +2340,16 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is - given, it should be a binary file, and tarinfo.size bytes are read - from it and added to the archive. You can create TarInfo objects - directly, or by using gettarinfo(). + """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents + a non zero-size regular file, the 'fileobj' argument should be a binary file, + and tarinfo.size bytes are read from it and added to the archive. + You can create TarInfo objects directly, or by using gettarinfo(). """ self._check("awx") + if fileobj is None and tarinfo.isreg() and tarinfo.size != 0: + raise ValueError("fileobj not provided for non zero-size regular file") + tarinfo = copy.copy(tarinfo) buf = tarinfo.tobuf(self.format, self.encoding, self.errors) @@ -2036,89 +2367,191 @@ def addfile(self, tarinfo, fileobj=None): self.members.append(tarinfo) - def extractall(self, path=".", members=None, *, numeric_owner=False): + def _get_filter_function(self, filter): + if filter is None: + filter = self.extraction_filter + if filter is None: + return data_filter + if isinstance(filter, str): + raise TypeError( + 'String names are not supported for ' + + 'TarFile.extraction_filter. Use a function such as ' + + 'tarfile.data_filter directly.') + return filter + if callable(filter): + return filter + try: + return _NAMED_FILTERS[filter] + except KeyError: + raise ValueError(f"filter {filter!r} not found") from None + + def extractall(self, path=".", members=None, *, numeric_owner=False, + filter=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on - directories afterwards. `path' specifies a different directory - to extract to. `members' is optional and must be a subset of the - list returned by getmembers(). If `numeric_owner` is True, only + directories afterwards. 'path' specifies a different directory + to extract to. 'members' is optional and must be a subset of the + list returned by getmembers(). If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. + + The 'filter' function will be called on each member just + before extraction. + It can return a changed TarInfo or None to skip the member. + String names of common filters are accepted. """ directories = [] + filter_function = self._get_filter_function(filter) if members is None: members = self - for tarinfo in members: + for member in members: + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) + if tarinfo is None: + continue if tarinfo.isdir(): - # Extract directories with a safe mode. - directories.append(tarinfo) - tarinfo = copy.copy(tarinfo) - tarinfo.mode = 0o700 - # Do not set_attrs directories, as we will do that further down - self.extract(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) + # For directories, delay setting attributes until later, + # since permissions can interfere with extraction and + # extracting contents can reset mtime. + directories.append(unfiltered) + self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), + numeric_owner=numeric_owner, + filter_function=filter_function) # Reverse sort directories. - directories.sort(key=lambda a: a.name) - directories.reverse() + directories.sort(key=lambda a: a.name, reverse=True) + # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) + for unfiltered in directories: try: + # Need to re-apply any filter, to take the *current* filesystem + # state into account. + try: + tarinfo = filter_function(unfiltered, path) + except _FILTER_ERRORS as exc: + self._log_no_directory_fixup(unfiltered, repr(exc)) + continue + if tarinfo is None: + self._log_no_directory_fixup(unfiltered, + 'excluded by filter') + continue + dirpath = os.path.join(path, tarinfo.name) + try: + lstat = os.lstat(dirpath) + except FileNotFoundError: + self._log_no_directory_fixup(tarinfo, 'missing') + continue + if not stat.S_ISDIR(lstat.st_mode): + # This is no longer a directory; presumably a later + # member overwrote the entry. + self._log_no_directory_fixup(tarinfo, 'not a directory') + continue self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) + self._handle_nonfatal_error(e) + + def _log_no_directory_fixup(self, member, reason): + self._dbg(2, "tarfile: Not fixing up directory %r (%s)" % + (member.name, reason)) - def extract(self, member, path="", set_attrs=True, *, numeric_owner=False): + def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, + filter=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a TarInfo object. You can - specify a different directory using `path'. File attributes (owner, - mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` + as possible. 'member' may be a filename or a TarInfo object. You can + specify a different directory using 'path'. File attributes (owner, + mtime, mode) are set unless 'set_attrs' is False. If 'numeric_owner' is True, only the numbers for user/group names are used and not the names. + + The 'filter' function will be called before extraction. + It can return a changed TarInfo or None to skip the member. + String names of common filters are accepted. + """ + filter_function = self._get_filter_function(filter) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) + if tarinfo is not None: + self._extract_one(tarinfo, path, set_attrs, numeric_owner) + + def _get_extract_tarinfo(self, member, filter_function, path): + """Get (filtered, unfiltered) TarInfos from *member* + + *member* might be a string. + + Return (None, None) if not found. """ - self._check("r") if isinstance(member, str): - tarinfo = self.getmember(member) + unfiltered = self.getmember(member) else: - tarinfo = member + unfiltered = member + + filtered = None + try: + filtered = filter_function(unfiltered, path) + except (OSError, UnicodeEncodeError, FilterError) as e: + self._handle_fatal_error(e) + except ExtractError as e: + self._handle_nonfatal_error(e) + if filtered is None: + self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) + return None, None # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo._link_target = os.path.join(path, tarinfo.linkname) + if filtered.islnk(): + filtered = copy.copy(filtered) + filtered._link_target = os.path.join(path, filtered.linkname) + return filtered, unfiltered + + def _extract_one(self, tarinfo, path, set_attrs, numeric_owner, + filter_function=None): + """Extract from filtered tarinfo to disk. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) + """ + self._check("r") try: self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, - numeric_owner=numeric_owner) - except OSError as e: - if self.errorlevel > 0: - raise - else: - if e.filename is None: - self._dbg(1, "tarfile: %s" % e.strerror) - else: - self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) + numeric_owner=numeric_owner, + filter_function=filter_function, + extraction_root=path) + except (OSError, UnicodeEncodeError) as e: + self._handle_fatal_error(e) except ExtractError as e: - if self.errorlevel > 1: - raise + self._handle_nonfatal_error(e) + + def _handle_nonfatal_error(self, e): + """Handle non-fatal error (ExtractError) according to errorlevel""" + if self.errorlevel > 1: + raise + else: + self._dbg(1, "tarfile: %s" % e) + + def _handle_fatal_error(self, e): + """Handle "fatal" error according to self.errorlevel""" + if self.errorlevel > 0: + raise + elif isinstance(e, OSError): + if e.filename is None: + self._dbg(1, "tarfile: %s" % e.strerror) else: - self._dbg(1, "tarfile: %s" % e) + self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) + else: + self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) def extractfile(self, member): - """Extract a member from the archive as a file object. `member' may be - a filename or a TarInfo object. If `member' is a regular file or + """Extract a member from the archive as a file object. 'member' may be + a filename or a TarInfo object. If 'member' is a regular file or a link, an io.BufferedReader object is returned. For all other - existing members, None is returned. If `member' does not appear + existing members, None is returned. If 'member' does not appear in the archive, KeyError is raised. """ self._check("r") @@ -2147,9 +2580,13 @@ def extractfile(self, member): return None def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical + numeric_owner=False, *, filter_function=None, + extraction_root=None): + """Extract the filtered TarInfo object tarinfo to a physical file called targetpath. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) """ # Fetch the TarInfo object for the given name # and build the destination pathname, replacing @@ -2162,7 +2599,7 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, if upperdirs and not os.path.exists(upperdirs): # Create directories that are not part of the archive with # default permissions. - os.makedirs(upperdirs) + os.makedirs(upperdirs, exist_ok=True) if tarinfo.islnk() or tarinfo.issym(): self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname)) @@ -2178,7 +2615,10 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, elif tarinfo.ischr() or tarinfo.isblk(): self.makedev(tarinfo, targetpath) elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) + self.makelink_with_filter( + tarinfo, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) elif tarinfo.type not in SUPPORTED_TYPES: self.makeunknown(tarinfo, targetpath) else: @@ -2199,11 +2639,16 @@ def makedir(self, tarinfo, targetpath): """Make a directory called targetpath. """ try: - # Use a safe mode for the directory, the real mode is set - # later in _extract_member(). - os.mkdir(targetpath, 0o700) + if tarinfo.mode is None: + # Use the system's default mode + os.mkdir(targetpath) + else: + # Use a safe mode for the directory, the real mode is set + # later in _extract_member(). + os.mkdir(targetpath, 0o700) except FileExistsError: - pass + if not os.path.isdir(targetpath): + raise def makefile(self, tarinfo, targetpath): """Make a file called targetpath. @@ -2244,6 +2689,9 @@ def makedev(self, tarinfo, targetpath): raise ExtractError("special devices not supported by system") mode = tarinfo.mode + if mode is None: + # Use mknod's default + mode = 0o600 if tarinfo.isblk(): mode |= stat.S_IFBLK else: @@ -2253,10 +2701,18 @@ def makedev(self, tarinfo, targetpath): os.makedev(tarinfo.devmajor, tarinfo.devminor)) def makelink(self, tarinfo, targetpath): + return self.makelink_with_filter(tarinfo, targetpath, None, None) + + def makelink_with_filter(self, tarinfo, targetpath, + filter_function, extraction_root): """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a link). """ + keyerror_to_extracterror = False try: # For systems that support symbolic and hard links. if tarinfo.issym(): @@ -2264,19 +2720,41 @@ def makelink(self, tarinfo, targetpath): # Avoid FileExistsError on following os.symlink. os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) + return else: - # See extract(). if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + # Avoid FileExistsError on following os.link. + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) + return except symlink_exception: + keyerror_to_extracterror = True + + try: + unfiltered = self._find_link_target(tarinfo) + except KeyError: + if keyerror_to_extracterror: + raise ExtractError( + "unable to resolve link inside archive") from None + else: + raise + + if filter_function is None: + filtered = unfiltered + else: + if extraction_root is None: + raise ExtractError( + "makelink_with_filter: if filter_function is not None, " + + "extraction_root must also not be None") try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None + filtered = filter_function(unfiltered, extraction_root) + except _FILTER_ERRORS as cause: + raise LinkFallbackError(tarinfo, unfiltered.name) from cause + if filtered is not None: + self._extract_member(filtered, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) def chown(self, tarinfo, targetpath, numeric_owner): """Set owner of targetpath according to tarinfo. If numeric_owner @@ -2290,26 +2768,33 @@ def chown(self, tarinfo, targetpath, numeric_owner): u = tarinfo.uid if not numeric_owner: try: - if grp: + if grp and tarinfo.gname: g = grp.getgrnam(tarinfo.gname)[2] except KeyError: pass try: - if pwd: + if pwd and tarinfo.uname: u = pwd.getpwnam(tarinfo.uname)[2] except KeyError: pass + if g is None: + g = -1 + if u is None: + u = -1 try: if tarinfo.issym() and hasattr(os, "lchown"): os.lchown(targetpath, u, g) else: os.chown(targetpath, u, g) - except OSError as e: + except (OSError, OverflowError) as e: + # OverflowError can be raised if an ID doesn't fit in 'id_t' raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): """Set file permissions of targetpath according to tarinfo. """ + if tarinfo.mode is None: + return try: os.chmod(targetpath, tarinfo.mode) except OSError as e: @@ -2318,10 +2803,13 @@ def chmod(self, tarinfo, targetpath): def utime(self, tarinfo, targetpath): """Set modification time of targetpath according to tarinfo. """ + mtime = tarinfo.mtime + if mtime is None: + return if not hasattr(os, 'utime'): return try: - os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime)) + os.utime(targetpath, (mtime, mtime)) except OSError as e: raise ExtractError("could not change modification time") from e @@ -2339,6 +2827,8 @@ def next(self): # Advance the file pointer. if self.offset != self.fileobj.tell(): + if self.offset == 0: + return None self.fileobj.seek(self.offset - 1) if not self.fileobj.read(1): raise ReadError("unexpected end of data") @@ -2380,7 +2870,9 @@ def next(self): break if tarinfo is not None: - self.members.append(tarinfo) + # if streaming the file we do not want to cache the tarinfo + if not self.stream: + self.members.append(tarinfo) else: self._loaded = True @@ -2397,13 +2889,26 @@ def _getmember(self, name, tarinfo=None, normalize=False): members = self.getmembers() # Limit the member search list up to tarinfo. + skipping = False if tarinfo is not None: - members = members[:members.index(tarinfo)] + try: + index = members.index(tarinfo) + except ValueError: + # The given starting point might be a (modified) copy. + # We'll later skip members until we find an equivalent. + skipping = True + else: + # Happy fast path + members = members[:index] if normalize: name = os.path.normpath(name) for member in reversed(members): + if skipping: + if tarinfo.offset == member.offset: + skipping = False + continue if normalize: member_name = os.path.normpath(member.name) else: @@ -2412,15 +2917,18 @@ def _getmember(self, name, tarinfo=None, normalize=False): if name == member_name: return member + if skipping: + # Starting point was not found + raise ValueError(tarinfo) + def _load(self): """Read through the entire archive file and look for readable - members. + members. This should not run if the file is set to stream. """ - while True: - tarinfo = self.next() - if tarinfo is None: - break - self._loaded = True + if not self.stream: + while self.next() is not None: + pass + self._loaded = True def _check(self, mode=None): """Check if TarFile is still open, and if the operation's mode @@ -2504,6 +3012,7 @@ def __exit__(self, type, value, traceback): #-------------------- # exported functions #-------------------- + def is_tarfile(name): """Return True if name points to a tar archive that we are able to handle, else return False. @@ -2512,7 +3021,9 @@ def is_tarfile(name): """ try: if hasattr(name, "read"): + pos = name.tell() t = open(fileobj=name) + name.seek(pos) else: t = open(name) t.close() @@ -2527,9 +3038,13 @@ def main(): import argparse description = 'A simple command-line interface for tarfile module.' - parser = argparse.ArgumentParser(description=description) + parser = argparse.ArgumentParser(description=description, color=True) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose output') + parser.add_argument('--filter', metavar='', + choices=_NAMED_FILTERS, + help='Filter for extraction') + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-l', '--list', metavar='', help='Show listing of a tarfile') @@ -2541,8 +3056,12 @@ def main(): help='Create tarfile from sources') group.add_argument('-t', '--test', metavar='', help='Test if a tarfile is valid') + args = parser.parse_args() + if args.filter and args.extract is None: + parser.exit(1, '--filter is only valid for extraction\n') + if args.test is not None: src = args.test if is_tarfile(src): @@ -2573,7 +3092,7 @@ def main(): if is_tarfile(src): with TarFile.open(src, 'r:*') as tf: - tf.extractall(path=curdir) + tf.extractall(path=curdir, filter=args.filter) if args.verbose: if curdir == '.': msg = '{!r} file is extracted.'.format(src) @@ -2599,6 +3118,9 @@ def main(): '.tbz': 'bz2', '.tbz2': 'bz2', '.tb2': 'bz2', + # zstd + '.zst': 'zst', + '.tzst': 'zst', } tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w' tar_files = args.create diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py deleted file mode 100644 index 8ce053e881a..00000000000 --- a/Lib/telnetlib.py +++ /dev/null @@ -1,677 +0,0 @@ -r"""TELNET client class. - -Based on RFC 854: TELNET Protocol Specification, by J. Postel and -J. Reynolds - -Example: - ->>> from telnetlib import Telnet ->>> tn = Telnet('www.python.org', 79) # connect to finger port ->>> tn.write(b'guido\r\n') ->>> print(tn.read_all()) -Login Name TTY Idle When Where -guido Guido van Rossum pts/2 snag.cnri.reston.. - ->>> - -Note that read_all() won't read until eof -- it just reads some data --- but it guarantees to read at least one byte unless EOF is hit. - -It is possible to pass a Telnet object to a selector in order to wait until -more data is available. Note that in this case, read_eager() may return b'' -even if there was data on the socket, because the protocol negotiation may have -eaten the data. This is why EOFError is needed in some cases to distinguish -between "no data" and "connection closed" (since the socket also appears ready -for reading when it is closed). - -To do: -- option negotiation -- timeout should be intrinsic to the connection object instead of an - option on one of the read calls only - -""" - - -# Imported modules -import sys -import socket -import selectors -from time import monotonic as _time - -__all__ = ["Telnet"] - -# Tunable parameters -DEBUGLEVEL = 0 - -# Telnet protocol defaults -TELNET_PORT = 23 - -# Telnet protocol characters (don't change) -IAC = bytes([255]) # "Interpret As Command" -DONT = bytes([254]) -DO = bytes([253]) -WONT = bytes([252]) -WILL = bytes([251]) -theNULL = bytes([0]) - -SE = bytes([240]) # Subnegotiation End -NOP = bytes([241]) # No Operation -DM = bytes([242]) # Data Mark -BRK = bytes([243]) # Break -IP = bytes([244]) # Interrupt process -AO = bytes([245]) # Abort output -AYT = bytes([246]) # Are You There -EC = bytes([247]) # Erase Character -EL = bytes([248]) # Erase Line -GA = bytes([249]) # Go Ahead -SB = bytes([250]) # Subnegotiation Begin - - -# Telnet protocol options code (don't change) -# These ones all come from arpa/telnet.h -BINARY = bytes([0]) # 8-bit data path -ECHO = bytes([1]) # echo -RCP = bytes([2]) # prepare to reconnect -SGA = bytes([3]) # suppress go ahead -NAMS = bytes([4]) # approximate message size -STATUS = bytes([5]) # give status -TM = bytes([6]) # timing mark -RCTE = bytes([7]) # remote controlled transmission and echo -NAOL = bytes([8]) # negotiate about output line width -NAOP = bytes([9]) # negotiate about output page size -NAOCRD = bytes([10]) # negotiate about CR disposition -NAOHTS = bytes([11]) # negotiate about horizontal tabstops -NAOHTD = bytes([12]) # negotiate about horizontal tab disposition -NAOFFD = bytes([13]) # negotiate about formfeed disposition -NAOVTS = bytes([14]) # negotiate about vertical tab stops -NAOVTD = bytes([15]) # negotiate about vertical tab disposition -NAOLFD = bytes([16]) # negotiate about output LF disposition -XASCII = bytes([17]) # extended ascii character set -LOGOUT = bytes([18]) # force logout -BM = bytes([19]) # byte macro -DET = bytes([20]) # data entry terminal -SUPDUP = bytes([21]) # supdup protocol -SUPDUPOUTPUT = bytes([22]) # supdup output -SNDLOC = bytes([23]) # send location -TTYPE = bytes([24]) # terminal type -EOR = bytes([25]) # end or record -TUID = bytes([26]) # TACACS user identification -OUTMRK = bytes([27]) # output marking -TTYLOC = bytes([28]) # terminal location number -VT3270REGIME = bytes([29]) # 3270 regime -X3PAD = bytes([30]) # X.3 PAD -NAWS = bytes([31]) # window size -TSPEED = bytes([32]) # terminal speed -LFLOW = bytes([33]) # remote flow control -LINEMODE = bytes([34]) # Linemode option -XDISPLOC = bytes([35]) # X Display Location -OLD_ENVIRON = bytes([36]) # Old - Environment variables -AUTHENTICATION = bytes([37]) # Authenticate -ENCRYPT = bytes([38]) # Encryption option -NEW_ENVIRON = bytes([39]) # New - Environment variables -# the following ones come from -# http://www.iana.org/assignments/telnet-options -# Unfortunately, that document does not assign identifiers -# to all of them, so we are making them up -TN3270E = bytes([40]) # TN3270E -XAUTH = bytes([41]) # XAUTH -CHARSET = bytes([42]) # CHARSET -RSP = bytes([43]) # Telnet Remote Serial Port -COM_PORT_OPTION = bytes([44]) # Com Port Control Option -SUPPRESS_LOCAL_ECHO = bytes([45]) # Telnet Suppress Local Echo -TLS = bytes([46]) # Telnet Start TLS -KERMIT = bytes([47]) # KERMIT -SEND_URL = bytes([48]) # SEND-URL -FORWARD_X = bytes([49]) # FORWARD_X -PRAGMA_LOGON = bytes([138]) # TELOPT PRAGMA LOGON -SSPI_LOGON = bytes([139]) # TELOPT SSPI LOGON -PRAGMA_HEARTBEAT = bytes([140]) # TELOPT PRAGMA HEARTBEAT -EXOPL = bytes([255]) # Extended-Options-List -NOOPT = bytes([0]) - - -# poll/select have the advantage of not requiring any extra file descriptor, -# contrarily to epoll/kqueue (also, they require a single syscall). -if hasattr(selectors, 'PollSelector'): - _TelnetSelector = selectors.PollSelector -else: - _TelnetSelector = selectors.SelectSelector - - -class Telnet: - - """Telnet interface class. - - An instance of this class represents a connection to a telnet - server. The instance is initially not connected; the open() - method must be used to establish a connection. Alternatively, the - host name and optional port number can be passed to the - constructor, too. - - Don't try to reopen an already connected instance. - - This class has many read_*() methods. Note that some of them - raise EOFError when the end of the connection is read, because - they can return an empty string for other reasons. See the - individual doc strings. - - read_until(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. - - read_all() - Read all data until EOF; may block. - - read_some() - Read at least one byte or EOF; may block. - - read_very_eager() - Read all data available already queued or on the socket, - without blocking. - - read_eager() - Read either data already queued or some data available on the - socket, without blocking. - - read_lazy() - Read all data in the raw queue (processing it first), without - doing any socket I/O. - - read_very_lazy() - Reads all data in the cooked queue, without doing any socket - I/O. - - read_sb_data() - Reads available data between SB ... SE sequence. Don't block. - - set_option_negotiation_callback(callback) - Each time a telnet option is read on the input flow, this callback - (if set) is called with the following parameters : - callback(telnet socket, command, option) - option will be chr(0) when there is no option. - No other action is done afterwards by telnetlib. - - """ - - def __init__(self, host=None, port=0, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Constructor. - - When called without arguments, create an unconnected instance. - With a hostname argument, it connects the instance; port number - and timeout are optional. - """ - self.debuglevel = DEBUGLEVEL - self.host = host - self.port = port - self.timeout = timeout - self.sock = None - self.rawq = b'' - self.irawq = 0 - self.cookedq = b'' - self.eof = 0 - self.iacseq = b'' # Buffer for IAC sequence. - self.sb = 0 # flag for SB and SE sequence. - self.sbdataq = b'' - self.option_callback = None - if host is not None: - self.open(host, port, timeout) - - def open(self, host, port=0, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): - """Connect to a host. - - The optional second argument is the port number, which - defaults to the standard telnet port (23). - - Don't try to reopen an already connected instance. - """ - self.eof = 0 - if not port: - port = TELNET_PORT - self.host = host - self.port = port - self.timeout = timeout - sys.audit("telnetlib.Telnet.open", self, host, port) - self.sock = socket.create_connection((host, port), timeout) - - def __del__(self): - """Destructor -- close the connection.""" - self.close() - - def msg(self, msg, *args): - """Print a debug message, when the debug level is > 0. - - If extra arguments are present, they are substituted in the - message using the standard string formatting operator. - - """ - if self.debuglevel > 0: - print('Telnet(%s,%s):' % (self.host, self.port), end=' ') - if args: - print(msg % args) - else: - print(msg) - - def set_debuglevel(self, debuglevel): - """Set the debug level. - - The higher it is, the more debug output you get (on sys.stdout). - - """ - self.debuglevel = debuglevel - - def close(self): - """Close the connection.""" - sock = self.sock - self.sock = None - self.eof = True - self.iacseq = b'' - self.sb = 0 - if sock: - sock.close() - - def get_socket(self): - """Return the socket object used internally.""" - return self.sock - - def fileno(self): - """Return the fileno() of the socket object used internally.""" - return self.sock.fileno() - - def write(self, buffer): - """Write a string to the socket, doubling any IAC characters. - - Can block if the connection is blocked. May raise - OSError if the connection is closed. - - """ - if IAC in buffer: - buffer = buffer.replace(IAC, IAC+IAC) - sys.audit("telnetlib.Telnet.write", self, buffer) - self.msg("send %r", buffer) - self.sock.sendall(buffer) - - def read_until(self, match, timeout=None): - """Read until a given string is encountered or until timeout. - - When no match is found, return whatever is available instead, - possibly the empty string. Raise EOFError if the connection - is closed and no cooked data is available. - - """ - n = len(match) - self.process_rawq() - i = self.cookedq.find(match) - if i >= 0: - i = i+n - buf = self.cookedq[:i] - self.cookedq = self.cookedq[i:] - return buf - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - if selector.select(timeout): - i = max(0, len(self.cookedq)-n) - self.fill_rawq() - self.process_rawq() - i = self.cookedq.find(match, i) - if i >= 0: - i = i+n - buf = self.cookedq[:i] - self.cookedq = self.cookedq[i:] - return buf - if timeout is not None: - timeout = deadline - _time() - if timeout < 0: - break - return self.read_very_lazy() - - def read_all(self): - """Read all data until EOF; block until connection closed.""" - self.process_rawq() - while not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - def read_some(self): - """Read at least one byte of cooked data unless EOF is hit. - - Return b'' if EOF is hit. Block if no data is immediately - available. - - """ - self.process_rawq() - while not self.cookedq and not self.eof: - self.fill_rawq() - self.process_rawq() - buf = self.cookedq - self.cookedq = b'' - return buf - - def read_very_eager(self): - """Read everything that's possible without blocking in I/O (eager). - - Raise EOFError if connection closed and no cooked data - available. Return b'' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_eager(self): - """Read readily available data. - - Raise EOFError if connection closed and no cooked data - available. Return b'' if no cooked data available otherwise. - Don't block unless in the midst of an IAC sequence. - - """ - self.process_rawq() - while not self.cookedq and not self.eof and self.sock_avail(): - self.fill_rawq() - self.process_rawq() - return self.read_very_lazy() - - def read_lazy(self): - """Process and return data that's already in the queues (lazy). - - Raise EOFError if connection closed and no data available. - Return b'' if no cooked data available otherwise. Don't block - unless in the midst of an IAC sequence. - - """ - self.process_rawq() - return self.read_very_lazy() - - def read_very_lazy(self): - """Return any data available in the cooked queue (very lazy). - - Raise EOFError if connection closed and no data available. - Return b'' if no cooked data available otherwise. Don't block. - - """ - buf = self.cookedq - self.cookedq = b'' - if not buf and self.eof and not self.rawq: - raise EOFError('telnet connection closed') - return buf - - def read_sb_data(self): - """Return any data available in the SB ... SE queue. - - Return b'' if no SB ... SE available. Should only be called - after seeing a SB or SE command. When a new SB command is - found, old unread SB data will be discarded. Don't block. - - """ - buf = self.sbdataq - self.sbdataq = b'' - return buf - - def set_option_negotiation_callback(self, callback): - """Provide a callback function called after each receipt of a telnet option.""" - self.option_callback = callback - - def process_rawq(self): - """Transfer from raw queue to cooked queue. - - Set self.eof when connection is closed. Don't block unless in - the midst of an IAC sequence. - - """ - buf = [b'', b''] - try: - while self.rawq: - c = self.rawq_getchar() - if not self.iacseq: - if c == theNULL: - continue - if c == b"\021": - continue - if c != IAC: - buf[self.sb] = buf[self.sb] + c - continue - else: - self.iacseq += c - elif len(self.iacseq) == 1: - # 'IAC: IAC CMD [OPTION only for WILL/WONT/DO/DONT]' - if c in (DO, DONT, WILL, WONT): - self.iacseq += c - continue - - self.iacseq = b'' - if c == IAC: - buf[self.sb] = buf[self.sb] + c - else: - if c == SB: # SB ... SE start. - self.sb = 1 - self.sbdataq = b'' - elif c == SE: - self.sb = 0 - self.sbdataq = self.sbdataq + buf[1] - buf[1] = b'' - if self.option_callback: - # Callback is supposed to look into - # the sbdataq - self.option_callback(self.sock, c, NOOPT) - else: - # We can't offer automatic processing of - # suboptions. Alas, we should not get any - # unless we did a WILL/DO before. - self.msg('IAC %d not recognized' % ord(c)) - elif len(self.iacseq) == 2: - cmd = self.iacseq[1:2] - self.iacseq = b'' - opt = c - if cmd in (DO, DONT): - self.msg('IAC %s %d', - cmd == DO and 'DO' or 'DONT', ord(opt)) - if self.option_callback: - self.option_callback(self.sock, cmd, opt) - else: - self.sock.sendall(IAC + WONT + opt) - elif cmd in (WILL, WONT): - self.msg('IAC %s %d', - cmd == WILL and 'WILL' or 'WONT', ord(opt)) - if self.option_callback: - self.option_callback(self.sock, cmd, opt) - else: - self.sock.sendall(IAC + DONT + opt) - except EOFError: # raised by self.rawq_getchar() - self.iacseq = b'' # Reset on EOF - self.sb = 0 - pass - self.cookedq = self.cookedq + buf[0] - self.sbdataq = self.sbdataq + buf[1] - - def rawq_getchar(self): - """Get next char from raw queue. - - Block if no data is immediately available. Raise EOFError - when connection is closed. - - """ - if not self.rawq: - self.fill_rawq() - if self.eof: - raise EOFError - c = self.rawq[self.irawq:self.irawq+1] - self.irawq = self.irawq + 1 - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - return c - - def fill_rawq(self): - """Fill raw queue from exactly one recv() system call. - - Block if no data is immediately available. Set self.eof when - connection is closed. - - """ - if self.irawq >= len(self.rawq): - self.rawq = b'' - self.irawq = 0 - # The buffer size should be fairly small so as to avoid quadratic - # behavior in process_rawq() above - buf = self.sock.recv(50) - self.msg("recv %r", buf) - self.eof = (not buf) - self.rawq = self.rawq + buf - - def sock_avail(self): - """Test whether data is available on the socket.""" - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - return bool(selector.select(0)) - - def interact(self): - """Interaction function, emulates a very dumb telnet client.""" - if sys.platform == "win32": - self.mt_interact() - return - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - selector.register(sys.stdin, selectors.EVENT_READ) - - while True: - for key, events in selector.select(): - if key.fileobj is self: - try: - text = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if text: - sys.stdout.write(text.decode('ascii')) - sys.stdout.flush() - elif key.fileobj is sys.stdin: - line = sys.stdin.readline().encode('ascii') - if not line: - return - self.write(line) - - def mt_interact(self): - """Multithreaded version of interact().""" - import _thread - _thread.start_new_thread(self.listener, ()) - while 1: - line = sys.stdin.readline() - if not line: - break - self.write(line.encode('ascii')) - - def listener(self): - """Helper for mt_interact() -- this executes in the other thread.""" - while 1: - try: - data = self.read_eager() - except EOFError: - print('*** Connection closed by remote host ***') - return - if data: - sys.stdout.write(data.decode('ascii')) - else: - sys.stdout.flush() - - def expect(self, list, timeout=None): - """Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either - compiled (re.Pattern instances) or uncompiled (strings). - The optional second argument is a timeout, in seconds; default - is no timeout. - - Return a tuple of three items: the index in the list of the - first regular expression that matches; the re.Match object - returned; and the text read up till and including the match. - - If EOF is read and no text was read, raise EOFError. - Otherwise, when nothing matches, return (-1, None, text) where - text is the text received so far (may be the empty string if a - timeout happened). - - If a regular expression ends with a greedy match (e.g. '.*') - or if more than one expression can match the same input, the - results are undeterministic, and may depend on the I/O timing. - - """ - re = None - list = list[:] - indices = range(len(list)) - for i in indices: - if not hasattr(list[i], "search"): - if not re: import re - list[i] = re.compile(list[i]) - if timeout is not None: - deadline = _time() + timeout - with _TelnetSelector() as selector: - selector.register(self, selectors.EVENT_READ) - while not self.eof: - self.process_rawq() - for i in indices: - m = list[i].search(self.cookedq) - if m: - e = m.end() - text = self.cookedq[:e] - self.cookedq = self.cookedq[e:] - return (i, m, text) - if timeout is not None: - ready = selector.select(timeout) - timeout = deadline - _time() - if not ready: - if timeout < 0: - break - else: - continue - self.fill_rawq() - text = self.read_very_lazy() - if not text and self.eof: - raise EOFError - return (-1, None, text) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - -def test(): - """Test program for telnetlib. - - Usage: python telnetlib.py [-d] ... [host [port]] - - Default host is localhost; default port is 23. - - """ - debuglevel = 0 - while sys.argv[1:] and sys.argv[1] == '-d': - debuglevel = debuglevel+1 - del sys.argv[1] - host = 'localhost' - if sys.argv[1:]: - host = sys.argv[1] - port = 0 - if sys.argv[2:]: - portstr = sys.argv[2] - try: - port = int(portstr) - except ValueError: - port = socket.getservbyname(portstr, 'tcp') - with Telnet() as tn: - tn.set_debuglevel(debuglevel) - tn.open(host, port, timeout=0.5) - tn.interact() - -if __name__ == '__main__': - test() diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 3aceb3f70fd..5e3ccab5f48 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -46,7 +46,6 @@ import sys as _sys import types as _types import weakref as _weakref - import _thread _allocate_lock = _thread.allocate_lock @@ -181,7 +180,7 @@ def _candidate_tempdir_list(): return dirlist -def _get_default_tempdir(): +def _get_default_tempdir(dirlist=None): """Calculate the default directory to use for temporary files. This routine should be called exactly once. @@ -191,7 +190,8 @@ def _get_default_tempdir(): service, the name of the test file must be randomized.""" namer = _RandomNameSequence() - dirlist = _candidate_tempdir_list() + if dirlist is None: + dirlist = _candidate_tempdir_list() for dir in dirlist: if dir != _os.curdir: @@ -204,8 +204,7 @@ def _get_default_tempdir(): fd = _os.open(filename, _bin_openflags, 0o600) try: try: - with _io.open(fd, 'wb', closefd=False) as fp: - fp.write(b'blat') + _os.write(fd, b'blat') finally: _os.close(fd) finally: @@ -245,6 +244,7 @@ def _get_candidate_names(): def _mkstemp_inner(dir, pre, suf, flags, output_type): """Code common to mkstemp, TemporaryFile, and NamedTemporaryFile.""" + dir = _os.path.abspath(dir) names = _get_candidate_names() if output_type is bytes: names = map(_os.fsencode, names) @@ -265,11 +265,27 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type): continue else: raise - return (fd, _os.path.abspath(file)) + return fd, file raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") +def _dont_follow_symlinks(func, path, *args): + # Pass follow_symlinks=False, unless not supported on this platform. + if func in _os.supports_follow_symlinks: + func(path, *args, follow_symlinks=False) + elif not _os.path.islink(path): + func(path, *args) + +def _resetperms(path): + try: + chflags = _os.chflags + except AttributeError: + pass + else: + _dont_follow_symlinks(chflags, path, 0) + _dont_follow_symlinks(_os.chmod, path, 0o700) + # User visible interfaces. @@ -377,7 +393,7 @@ def mkdtemp(suffix=None, prefix=None, dir=None): continue else: raise - return file + return _os.path.abspath(file) raise FileExistsError(_errno.EEXIST, "No usable temporary directory name found") @@ -419,42 +435,53 @@ class _TemporaryFileCloser: underlying file object, without adding a __del__ method to the temporary file.""" - file = None # Set here since __del__ checks it + cleanup_called = False close_called = False - def __init__(self, file, name, delete=True): + def __init__( + self, + file, + name, + delete=True, + delete_on_close=True, + warn_message="Implicitly cleaning up unknown file", + ): self.file = file self.name = name self.delete = delete + self.delete_on_close = delete_on_close + self.warn_message = warn_message - # NT provides delete-on-close as a primitive, so we don't need - # the wrapper to do anything special. We still use it so that - # file.name is useful (i.e. not "(fdopen)") with NamedTemporaryFile. - if _os.name != 'nt': - # Cache the unlinker so we don't get spurious errors at - # shutdown when the module-level "os" is None'd out. Note - # that this must be referenced as self.unlink, because the - # name TemporaryFileWrapper may also get None'd out before - # __del__ is called. - - def close(self, unlink=_os.unlink): - if not self.close_called and self.file is not None: - self.close_called = True - try: + def cleanup(self, windows=(_os.name == 'nt'), unlink=_os.unlink): + if not self.cleanup_called: + self.cleanup_called = True + try: + if not self.close_called: + self.close_called = True self.file.close() - finally: - if self.delete: + finally: + # Windows provides delete-on-close as a primitive, in which + # case the file was deleted by self.file.close(). + if self.delete and not (windows and self.delete_on_close): + try: unlink(self.name) + except FileNotFoundError: + pass - # Need to ensure the file is deleted on __del__ - def __del__(self): - self.close() - - else: - def close(self): - if not self.close_called: - self.close_called = True + def close(self): + if not self.close_called: + self.close_called = True + try: self.file.close() + finally: + if self.delete and self.delete_on_close: + self.cleanup() + + def __del__(self): + close_called = self.close_called + self.cleanup() + if not close_called: + _warnings.warn(self.warn_message, ResourceWarning) class _TemporaryFileWrapper: @@ -465,11 +492,20 @@ class _TemporaryFileWrapper: remove the file when it is no longer needed. """ - def __init__(self, file, name, delete=True): + def __init__(self, file, name, delete=True, delete_on_close=True): self.file = file self.name = name - self.delete = delete - self._closer = _TemporaryFileCloser(file, name, delete) + self._closer = _TemporaryFileCloser( + file, + name, + delete, + delete_on_close, + warn_message=f"Implicitly cleaning up {self!r}", + ) + + def __repr__(self): + file = self.__dict__['file'] + return f"<{type(self).__name__} {file=}>" def __getattr__(self, name): # Attribute lookups are delegated to the underlying file @@ -500,7 +536,7 @@ def __enter__(self): # deleted when used in a with statement def __exit__(self, exc, value, tb): result = self.file.__exit__(exc, value, tb) - self.close() + self._closer.cleanup() return result def close(self): @@ -519,10 +555,10 @@ def __iter__(self): for line in self.file: yield line - def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, - dir=None, delete=True, *, errors=None): + dir=None, delete=True, *, errors=None, + delete_on_close=True): """Create and return a temporary file. Arguments: 'prefix', 'suffix', 'dir' -- as for mkstemp. @@ -530,13 +566,20 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, 'buffering' -- the buffer size argument to io.open (default -1). 'encoding' -- the encoding argument to io.open (default None) 'newline' -- the newline argument to io.open (default None) - 'delete' -- whether the file is deleted on close (default True). + 'delete' -- whether the file is automatically deleted (default True). + 'delete_on_close' -- if 'delete', whether the file is deleted on close + (default True) or otherwise either on context manager exit + (if context manager was used) or on object finalization. . 'errors' -- the errors argument to io.open (default None) The file is created as mkstemp() would do it. Returns an object with a file-like interface; the name of the file is accessible as its 'name' attribute. The file will be automatically deleted when it is closed unless the 'delete' argument is set to False. + + On POSIX, NamedTemporaryFiles cannot be automatically deleted if + the creating process is terminated abruptly with a SIGKILL signal. + Windows can delete the file even in this case. """ prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) @@ -545,21 +588,33 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, # Setting O_TEMPORARY in the flags causes the OS to delete # the file when it is closed. This is only supported by Windows. - if _os.name == 'nt' and delete: + if _os.name == 'nt' and delete and delete_on_close: flags |= _os.O_TEMPORARY if "b" not in mode: encoding = _io.text_encoding(encoding) - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) + name = None + def opener(*args): + nonlocal name + fd, name = _mkstemp_inner(dir, prefix, suffix, flags, output_type) + return fd try: - file = _io.open(fd, mode, buffering=buffering, - newline=newline, encoding=encoding, errors=errors) - - return _TemporaryFileWrapper(file, name, delete) - except BaseException: - _os.unlink(name) - _os.close(fd) + file = _io.open(dir, mode, buffering=buffering, + newline=newline, encoding=encoding, errors=errors, + opener=opener) + try: + raw = getattr(file, 'buffer', file) + raw = getattr(raw, 'raw', raw) + raw.name = name + return _TemporaryFileWrapper(file, name, delete, delete_on_close) + except: + file.close() + raise + except: + if name is not None and not ( + _os.name == 'nt' and delete and delete_on_close): + _os.unlink(name) raise if _os.name != 'posix' or _sys.platform == 'cygwin': @@ -598,9 +653,20 @@ def TemporaryFile(mode='w+b', buffering=-1, encoding=None, flags = _bin_openflags if _O_TMPFILE_WORKS: - try: + fd = None + def opener(*args): + nonlocal fd flags2 = (flags | _os.O_TMPFILE) & ~_os.O_CREAT fd = _os.open(dir, flags2, 0o600) + return fd + try: + file = _io.open(dir, mode, buffering=buffering, + newline=newline, encoding=encoding, + errors=errors, opener=opener) + raw = getattr(file, 'buffer', file) + raw = getattr(raw, 'raw', raw) + raw.name = fd + return file except IsADirectoryError: # Linux kernel older than 3.11 ignores the O_TMPFILE flag: # O_TMPFILE is read as O_DIRECTORY. Trying to open a directory @@ -617,26 +683,27 @@ def TemporaryFile(mode='w+b', buffering=-1, encoding=None, # fails with NotADirectoryError, because O_TMPFILE is read as # O_DIRECTORY. pass - else: - try: - return _io.open(fd, mode, buffering=buffering, - newline=newline, encoding=encoding, - errors=errors) - except: - _os.close(fd) - raise # Fallback to _mkstemp_inner(). - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) - try: - _os.unlink(name) - return _io.open(fd, mode, buffering=buffering, - newline=newline, encoding=encoding, errors=errors) - except: - _os.close(fd) - raise + fd = None + def opener(*args): + nonlocal fd + fd, name = _mkstemp_inner(dir, prefix, suffix, flags, output_type) + try: + _os.unlink(name) + except BaseException as e: + _os.close(fd) + raise + return fd + file = _io.open(dir, mode, buffering=buffering, + newline=newline, encoding=encoding, errors=errors, + opener=opener) + raw = getattr(file, 'buffer', file) + raw = getattr(raw, 'raw', raw) + raw.name = fd + return file -class SpooledTemporaryFile: +class SpooledTemporaryFile(_io.IOBase): """Temporary file wrapper, specialized to switch from BytesIO or StringIO to a real file when it exceeds a certain size or when a fileno is needed. @@ -701,6 +768,16 @@ def __exit__(self, exc, value, tb): def __iter__(self): return self._file.__iter__() + def __del__(self): + if not self.closed: + _warnings.warn( + "Unclosed file {!r}".format(self), + ResourceWarning, + stacklevel=2, + source=self + ) + self.close() + def close(self): self._file.close() @@ -744,15 +821,30 @@ def name(self): def newlines(self): return self._file.newlines + def readable(self): + return self._file.readable() + def read(self, *args): return self._file.read(*args) + def read1(self, *args): + return self._file.read1(*args) + + def readinto(self, b): + return self._file.readinto(b) + + def readinto1(self, b): + return self._file.readinto1(b) + def readline(self, *args): return self._file.readline(*args) def readlines(self, *args): return self._file.readlines(*args) + def seekable(self): + return self._file.seekable() + def seek(self, *args): return self._file.seek(*args) @@ -761,11 +853,14 @@ def tell(self): def truncate(self, size=None): if size is None: - self._file.truncate() + return self._file.truncate() else: if size > self._max_size: self.rollover() - self._file.truncate(size) + return self._file.truncate(size) + + def writable(self): + return self._file.writable() def write(self, s): file = self._file @@ -774,10 +869,17 @@ def write(self, s): return rv def writelines(self, iterable): - file = self._file - rv = file.writelines(iterable) - self._check(file) - return rv + if self._max_size == 0 or self._rolled: + return self._file.writelines(iterable) + + it = iter(iterable) + for line in it: + self.write(line) + if self._rolled: + return self._file.writelines(it) + + def detach(self): + return self._file.detach() class TemporaryDirectory: @@ -789,53 +891,74 @@ class TemporaryDirectory: ... Upon exiting the context, the directory and everything contained - in it are removed. + in it are removed (unless delete=False is passed or an exception + is raised during cleanup and ignore_cleanup_errors is not True). + + Optional Arguments: + suffix - A str suffix for the directory name. (see mkdtemp) + prefix - A str prefix for the directory name. (see mkdtemp) + dir - A directory to create this temp dir in. (see mkdtemp) + ignore_cleanup_errors - False; ignore exceptions during cleanup? + delete - True; whether the directory is automatically deleted. """ def __init__(self, suffix=None, prefix=None, dir=None, - ignore_cleanup_errors=False): + ignore_cleanup_errors=False, *, delete=True): self.name = mkdtemp(suffix, prefix, dir) self._ignore_cleanup_errors = ignore_cleanup_errors + self._delete = delete self._finalizer = _weakref.finalize( self, self._cleanup, self.name, warn_message="Implicitly cleaning up {!r}".format(self), - ignore_errors=self._ignore_cleanup_errors) + ignore_errors=self._ignore_cleanup_errors, delete=self._delete) @classmethod - def _rmtree(cls, name, ignore_errors=False): - def onerror(func, path, exc_info): - if issubclass(exc_info[0], PermissionError): - def resetperms(path): - try: - _os.chflags(path, 0) - except AttributeError: - pass - _os.chmod(path, 0o700) + def _rmtree(cls, name, ignore_errors=False, repeated=False): + def onexc(func, path, exc): + if isinstance(exc, PermissionError): + if repeated and path == name: + if ignore_errors: + return + raise try: if path != name: - resetperms(_os.path.dirname(path)) - resetperms(path) + _resetperms(_os.path.dirname(path)) + _resetperms(path) try: _os.unlink(path) - # PermissionError is raised on FreeBSD for directories - except (IsADirectoryError, PermissionError): + except IsADirectoryError: cls._rmtree(path, ignore_errors=ignore_errors) + except PermissionError: + # The PermissionError handler was originally added for + # FreeBSD in directories, but it seems that it is raised + # on Windows too. + # bpo-43153: Calling _rmtree again may + # raise NotADirectoryError and mask the PermissionError. + # So we must re-raise the current PermissionError if + # path is not a directory. + if not _os.path.isdir(path) or _os.path.isjunction(path): + if ignore_errors: + return + raise + cls._rmtree(path, ignore_errors=ignore_errors, + repeated=(path == name)) except FileNotFoundError: pass - elif issubclass(exc_info[0], FileNotFoundError): + elif isinstance(exc, FileNotFoundError): pass else: if not ignore_errors: raise - _shutil.rmtree(name, onerror=onerror) + _shutil.rmtree(name, onexc=onexc) @classmethod - def _cleanup(cls, name, warn_message, ignore_errors=False): - cls._rmtree(name, ignore_errors=ignore_errors) - _warnings.warn(warn_message, ResourceWarning) + def _cleanup(cls, name, warn_message, ignore_errors=False, delete=True): + if delete: + cls._rmtree(name, ignore_errors=ignore_errors) + _warnings.warn(warn_message, ResourceWarning) def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.name) @@ -844,7 +967,8 @@ def __enter__(self): return self.name def __exit__(self, exc, value, tb): - self.cleanup() + if self._delete: + self.cleanup() def cleanup(self): if self._finalizer.detach() or _os.path.exists(self.name): diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index 19a6b2b8904..82b50ad2c6e 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,2 +1,2 @@ -from test.libregrtest import main -main() +from test.libregrtest.main import main +main(_add_python_opts=True) diff --git a/Lib/test/_test_atexit.py b/Lib/test/_test_atexit.py index 55d28083349..2e961d6a485 100644 --- a/Lib/test/_test_atexit.py +++ b/Lib/test/_test_atexit.py @@ -19,7 +19,9 @@ def assert_raises_unraisable(self, exc_type, func, *args): atexit.register(func, *args) atexit._run_exitfuncs() - self.assertEqual(cm.unraisable.object, func) + self.assertIsNone(cm.unraisable.object) + self.assertEqual(cm.unraisable.err_msg, + f'Exception ignored in atexit callback {func!r}') self.assertEqual(cm.unraisable.exc_type, exc_type) self.assertEqual(type(cm.unraisable.exc_value), exc_type) @@ -125,12 +127,61 @@ def func(): try: with support.catch_unraisable_exception() as cm: atexit._run_exitfuncs() - self.assertEqual(cm.unraisable.object, func) + self.assertIsNone(cm.unraisable.object) + self.assertEqual(cm.unraisable.err_msg, + f'Exception ignored in atexit callback {func!r}') self.assertEqual(cm.unraisable.exc_type, ZeroDivisionError) self.assertEqual(type(cm.unraisable.exc_value), ZeroDivisionError) finally: atexit.unregister(func) + def test_eq_unregister_clear(self): + # Issue #112127: callback's __eq__ may call unregister or _clear + class Evil: + def __eq__(self, other): + action(other) + return NotImplemented + + for action in atexit.unregister, lambda o: atexit._clear(): + with self.subTest(action=action): + atexit.register(lambda: None) + atexit.unregister(Evil()) + atexit._clear() + + def test_eq_unregister(self): + # Issue #112127: callback's __eq__ may call unregister + def f1(): + log.append(1) + def f2(): + log.append(2) + def f3(): + log.append(3) + + class Pred: + def __eq__(self, other): + nonlocal cnt + cnt += 1 + if cnt == when: + atexit.unregister(what) + if other is f2: + return True + return False + + for what, expected in ( + (f1, [3]), + (f2, [3, 1]), + (f3, [1]), + ): + for when in range(1, 4): + with self.subTest(what=what.__name__, when=when): + cnt = 0 + log = [] + for f in (f1, f2, f3): + atexit.register(f) + atexit.unregister(Pred()) + atexit._run_exitfuncs() + self.assertEqual(log, expected) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py new file mode 100644 index 00000000000..a826d31bcd6 --- /dev/null +++ b/Lib/test/_test_eintr.py @@ -0,0 +1,560 @@ +""" +This test suite exercises some system calls subject to interruption with EINTR, +to check that it is actually handled transparently. +It is intended to be run by the main test suite within a child process, to +ensure there is no background thread running (so that signals are delivered to +the correct thread). +Signals are generated in-process using setitimer(ITIMER_REAL), which allows +sub-second periodicity (contrarily to signal()). +""" + +import contextlib +import faulthandler +import fcntl +import os +import platform +import select +import signal +import socket +import subprocess +import sys +import textwrap +import time +import unittest + +from test import support +from test.support import os_helper +from test.support import socket_helper + + +# gh-109592: Tolerate a difference of 20 ms when comparing timings +# (clock resolution) +CLOCK_RES = 0.020 + + +@contextlib.contextmanager +def kill_on_error(proc): + """Context manager killing the subprocess if a Python exception is raised.""" + with proc: + try: + yield proc + except: + proc.kill() + raise + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class EINTRBaseTest(unittest.TestCase): + """ Base class for EINTR tests. """ + + # delay for initial signal delivery + signal_delay = 0.1 + # signal delivery periodicity + signal_period = 0.1 + # default sleep time for tests - should obviously have: + # sleep_time > signal_period + sleep_time = 0.2 + + def sighandler(self, signum, frame): + self.signals += 1 + + def setUp(self): + self.signals = 0 + self.orig_handler = signal.signal(signal.SIGALRM, self.sighandler) + signal.setitimer(signal.ITIMER_REAL, self.signal_delay, + self.signal_period) + + # Use faulthandler as watchdog to debug when a test hangs + # (timeout of 10 minutes) + faulthandler.dump_traceback_later(10 * 60, exit=True, + file=sys.__stderr__) + + @staticmethod + def stop_alarm(): + signal.setitimer(signal.ITIMER_REAL, 0, 0) + + def tearDown(self): + self.stop_alarm() + signal.signal(signal.SIGALRM, self.orig_handler) + faulthandler.cancel_dump_traceback_later() + + def subprocess(self, *args, **kw): + cmd_args = (sys.executable, '-c') + args + return subprocess.Popen(cmd_args, **kw) + + def check_elapsed_time(self, elapsed): + self.assertGreaterEqual(elapsed, self.sleep_time - CLOCK_RES) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class OSEINTRTest(EINTRBaseTest): + """ EINTR tests for the os module. """ + + def new_sleep_process(self): + code = f'import time; time.sleep({self.sleep_time!r})' + return self.subprocess(code) + + def _test_wait_multiple(self, wait_func): + num = 3 + processes = [self.new_sleep_process() for _ in range(num)] + for _ in range(num): + wait_func() + # Call the Popen method to avoid a ResourceWarning + for proc in processes: + proc.wait() + + def test_wait(self): + self._test_wait_multiple(os.wait) + + @unittest.skipUnless(hasattr(os, 'wait3'), 'requires wait3()') + def test_wait3(self): + self._test_wait_multiple(lambda: os.wait3(0)) + + def _test_wait_single(self, wait_func): + proc = self.new_sleep_process() + wait_func(proc.pid) + # Call the Popen method to avoid a ResourceWarning + proc.wait() + + def test_waitpid(self): + self._test_wait_single(lambda pid: os.waitpid(pid, 0)) + + @unittest.skipUnless(hasattr(os, 'wait4'), 'requires wait4()') + def test_wait4(self): + self._test_wait_single(lambda pid: os.wait4(pid, 0)) + + def _interrupted_reads(self): + """Make a fd which will force block on read of expected bytes.""" + rd, wr = os.pipe() + self.addCleanup(os.close, rd) + # wr closed explicitly by parent + + # the payload below are smaller than PIPE_BUF, hence the writes will be + # atomic + data = [b"hello", b"world", b"spam"] + + code = '\n'.join(( + 'import os, sys, time', + '', + 'wr = int(sys.argv[1])', + f'data = {data!r}', + f'sleep_time = {self.sleep_time!r}', + '', + 'for item in data:', + ' # let the parent block on read()', + ' time.sleep(sleep_time)', + ' os.write(wr, item)', + )) + + proc = self.subprocess(code, str(wr), pass_fds=[wr]) + with kill_on_error(proc): + os.close(wr) + for datum in data: + yield rd, datum + self.assertEqual(proc.wait(), 0) + + def test_read(self): + for fd, expected in self._interrupted_reads(): + self.assertEqual(expected, os.read(fd, len(expected))) + + def test_readinto(self): + for fd, expected in self._interrupted_reads(): + buffer = bytearray(len(expected)) + self.assertEqual(os.readinto(fd, buffer), len(expected)) + self.assertEqual(buffer, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON; InterruptedError: [Errno 4] Interrupted system call + def test_write(self): + rd, wr = os.pipe() + self.addCleanup(os.close, wr) + # rd closed explicitly by parent + + # we must write enough data for the write() to block + data = b"x" * support.PIPE_MAX_SIZE + + code = '\n'.join(( + 'import io, os, sys, time', + '', + 'rd = int(sys.argv[1])', + f'sleep_time = {self.sleep_time!r}', + f'data = b"x" * {support.PIPE_MAX_SIZE}', + 'data_len = len(data)', + '', + '# let the parent block on write()', + 'time.sleep(sleep_time)', + '', + 'read_data = io.BytesIO()', + 'while len(read_data.getvalue()) < data_len:', + ' chunk = os.read(rd, 2 * data_len)', + ' read_data.write(chunk)', + '', + 'value = read_data.getvalue()', + 'if value != data:', + ' raise Exception(f"read error: {len(value)}' + ' vs {data_len} bytes")', + )) + + proc = self.subprocess(code, str(rd), pass_fds=[rd]) + with kill_on_error(proc): + os.close(rd) + written = 0 + while written < len(data): + written += os.write(wr, memoryview(data)[written:]) + self.assertEqual(proc.wait(), 0) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class SocketEINTRTest(EINTRBaseTest): + """ EINTR tests for the socket module. """ + + @unittest.skipUnless(hasattr(socket, 'socketpair'), 'needs socketpair()') + def _test_recv(self, recv_func): + rd, wr = socket.socketpair() + self.addCleanup(rd.close) + # wr closed explicitly by parent + + # single-byte payload guard us against partial recv + data = [b"x", b"y", b"z"] + + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + f'family = {int(wr.family)}', + f'sock_type = {int(wr.type)}', + f'data = {data!r}', + f'sleep_time = {self.sleep_time!r}', + '', + 'wr = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with wr:', + ' for item in data:', + ' # let the parent block on recv()', + ' time.sleep(sleep_time)', + ' wr.sendall(item)', + )) + + fd = wr.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with kill_on_error(proc): + wr.close() + for item in data: + self.assertEqual(item, recv_func(rd, len(item))) + self.assertEqual(proc.wait(), 0) + + def test_recv(self): + self._test_recv(socket.socket.recv) + + @unittest.skipUnless(hasattr(socket.socket, 'recvmsg'), 'needs recvmsg()') + def test_recvmsg(self): + self._test_recv(lambda sock, data: sock.recvmsg(data)[0]) + + def _test_send(self, send_func): + rd, wr = socket.socketpair() + self.addCleanup(wr.close) + # rd closed explicitly by parent + + # we must send enough data for the send() to block + data = b"xyz" * (support.SOCK_MAX_SIZE // 3) + + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + f'family = {int(rd.family)}', + f'sock_type = {int(rd.type)}', + f'sleep_time = {self.sleep_time!r}', + f'data = b"xyz" * {support.SOCK_MAX_SIZE // 3}', + 'data_len = len(data)', + '', + 'rd = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with rd:', + ' # let the parent block on send()', + ' time.sleep(sleep_time)', + '', + ' received_data = bytearray(data_len)', + ' n = 0', + ' while n < data_len:', + ' n += rd.recv_into(memoryview(received_data)[n:])', + '', + 'if received_data != data:', + ' raise Exception(f"recv error: {len(received_data)}' + ' vs {data_len} bytes")', + )) + + fd = rd.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with kill_on_error(proc): + rd.close() + written = 0 + while written < len(data): + sent = send_func(wr, memoryview(data)[written:]) + # sendall() returns None + written += len(data) if sent is None else sent + self.assertEqual(proc.wait(), 0) + + def test_send(self): + self._test_send(socket.socket.send) + + def test_sendall(self): + self._test_send(socket.socket.sendall) + + @unittest.skipUnless(hasattr(socket.socket, 'sendmsg'), 'needs sendmsg()') + def test_sendmsg(self): + self._test_send(lambda sock, data: sock.sendmsg([data])) + + def test_accept(self): + sock = socket.create_server((socket_helper.HOST, 0)) + self.addCleanup(sock.close) + port = sock.getsockname()[1] + + code = '\n'.join(( + 'import socket, time', + '', + f'host = {socket_helper.HOST!r}', + f'port = {port}', + f'sleep_time = {self.sleep_time!r}', + '', + '# let parent block on accept()', + 'time.sleep(sleep_time)', + 'with socket.create_connection((host, port)):', + ' time.sleep(sleep_time)', + )) + + proc = self.subprocess(code) + with kill_on_error(proc): + client_sock, _ = sock.accept() + client_sock.close() + self.assertEqual(proc.wait(), 0) + + # Issue #25122: There is a race condition in the FreeBSD kernel on + # handling signals in the FIFO device. Skip the test until the bug is + # fixed in the kernel. + # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=203162 + @support.requires_freebsd_version(10, 3) + @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') + def _test_open(self, do_open_close_reader, do_open_close_writer): + filename = os_helper.TESTFN + + # Use a fifo: until the child opens it for reading, the parent will + # block when trying to open it for writing. + os_helper.unlink(filename) + try: + os.mkfifo(filename) + except PermissionError as exc: + self.skipTest(f'os.mkfifo(): {exc!r}') + self.addCleanup(os_helper.unlink, filename) + + code = '\n'.join(( + 'import os, time', + '', + f'path = {filename!a}', + f'sleep_time = {self.sleep_time!r}', + '', + '# let the parent block', + 'time.sleep(sleep_time)', + '', + do_open_close_reader, + )) + + proc = self.subprocess(code) + with kill_on_error(proc): + do_open_close_writer(filename) + self.assertEqual(proc.wait(), 0) + + def python_open(self, path): + fp = open(path, 'w') + fp.close() + + @unittest.skipIf(sys.platform == "darwin", + "hangs under macOS; see bpo-25234, bpo-35363") + def test_open(self): + self._test_open("fp = open(path, 'r')\nfp.close()", + self.python_open) + + def os_open(self, path): + fd = os.open(path, os.O_WRONLY) + os.close(fd) + + @unittest.skipIf(sys.platform == "darwin", + "hangs under macOS; see bpo-25234, bpo-35363") + @unittest.skipIf(sys.platform.startswith('netbsd'), + "hangs on NetBSD; see gh-137397") + def test_os_open(self): + self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", + self.os_open) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class TimeEINTRTest(EINTRBaseTest): + """ EINTR tests for the time module. """ + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_sleep(self): + t0 = time.monotonic() + time.sleep(self.sleep_time) + self.stop_alarm() + dt = time.monotonic() - t0 + self.check_elapsed_time(dt) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +# bpo-30320: Need pthread_sigmask() to block the signal, otherwise the test +# is vulnerable to a race condition between the child and the parent processes. +@unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') +class SignalEINTRTest(EINTRBaseTest): + """ EINTR tests for the signal module. """ + + def check_sigwait(self, wait_func): + signum = signal.SIGUSR1 + + old_handler = signal.signal(signum, lambda *args: None) + self.addCleanup(signal.signal, signum, old_handler) + + code = '\n'.join(( + 'import os, time', + f'pid = {os.getpid()}', + f'signum = {int(signum)}', + f'sleep_time = {self.sleep_time!r}', + 'time.sleep(sleep_time)', + 'os.kill(pid, signum)', + )) + + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + self.addCleanup(signal.pthread_sigmask, signal.SIG_UNBLOCK, [signum]) + + proc = self.subprocess(code) + with kill_on_error(proc): + wait_func(signum) + + self.assertEqual(proc.wait(), 0) + + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo(self): + def wait_func(signum): + signal.sigwaitinfo([signum]) + + self.check_sigwait(wait_func) + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigwaitinfo()') + def test_sigtimedwait(self): + def wait_func(signum): + signal.sigtimedwait([signum], 120.0) + + self.check_sigwait(wait_func) + + +@unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") +class SelectEINTRTest(EINTRBaseTest): + """ EINTR tests for the select module. """ + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_select(self): + t0 = time.monotonic() + select.select([], [], [], self.sleep_time) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + @unittest.skip("TODO: RUSTPYTHON; timed out at the 10 minute mark") + @unittest.skipIf(sys.platform == "darwin", + "poll may fail on macOS; see issue #28087") + @unittest.skipUnless(hasattr(select, 'poll'), 'need select.poll') + def test_poll(self): + poller = select.poll() + + t0 = time.monotonic() + poller.poll(self.sleep_time * 1e3) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + @unittest.skipUnless(hasattr(select, 'epoll'), 'need select.epoll') + def test_epoll(self): + poller = select.epoll() + self.addCleanup(poller.close) + + t0 = time.monotonic() + poller.poll(self.sleep_time) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + @unittest.skipUnless(hasattr(select, 'kqueue'), 'need select.kqueue') + def test_kqueue(self): + kqueue = select.kqueue() + self.addCleanup(kqueue.close) + + t0 = time.monotonic() + kqueue.control(None, 1, self.sleep_time) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + @unittest.skipUnless(hasattr(select, 'devpoll'), 'need select.devpoll') + def test_devpoll(self): + poller = select.devpoll() + self.addCleanup(poller.close) + + t0 = time.monotonic() + poller.poll(self.sleep_time * 1e3) + dt = time.monotonic() - t0 + self.stop_alarm() + self.check_elapsed_time(dt) + + +class FCNTLEINTRTest(EINTRBaseTest): + def _lock(self, lock_func, lock_name): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + rd1, wr1 = os.pipe() + rd2, wr2 = os.pipe() + for fd in (rd1, wr1, rd2, wr2): + self.addCleanup(os.close, fd) + code = textwrap.dedent(f""" + import fcntl, os, time + with open('{os_helper.TESTFN}', 'wb') as f: + fcntl.{lock_name}(f, fcntl.LOCK_EX) + os.write({wr1}, b"ok") + _ = os.read({rd2}, 2) # wait for parent process + time.sleep({self.sleep_time}) + """) + proc = self.subprocess(code, pass_fds=[wr1, rd2]) + with kill_on_error(proc): + with open(os_helper.TESTFN, 'wb') as f: + # synchronize the subprocess + ok = os.read(rd1, 2) + self.assertEqual(ok, b"ok") + + # notify the child that the parent is ready + start_time = time.monotonic() + os.write(wr2, b"go") + + # the child locked the file just a moment ago for 'sleep_time' seconds + # that means that the lock below will block for 'sleep_time' minus some + # potential context switch delay + lock_func(f, fcntl.LOCK_EX) + dt = time.monotonic() - start_time + self.stop_alarm() + self.check_elapsed_time(dt) + proc.wait() + + # Issue 35633: See https://bugs.python.org/issue35633#msg333662 + # skip test rather than accept PermissionError from all platforms + @unittest.expectedFailure # TODO: RUSTPYTHON; InterruptedError: [Errno 4] Interrupted system call + @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + def test_lockf(self): + self._lock(fcntl.lockf, "lockf") + + @unittest.expectedFailure # TODO: RUSTPYTHON; InterruptedError: [Errno 4] Interrupted system call + def test_flock(self): + self._lock(fcntl.flock, "flock") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/_test_gc_fast_cycles.py b/Lib/test/_test_gc_fast_cycles.py new file mode 100644 index 00000000000..4e2c7d72a02 --- /dev/null +++ b/Lib/test/_test_gc_fast_cycles.py @@ -0,0 +1,48 @@ +# Run by test_gc. +from test import support +import _testinternalcapi +import gc +import unittest + +class IncrementalGCTests(unittest.TestCase): + + # Use small increments to emulate longer running process in a shorter time + @support.gc_threshold(200, 10) + def test_incremental_gc_handles_fast_cycle_creation(self): + + class LinkedList: + + #Use slots to reduce number of implicit objects + __slots__ = "next", "prev", "surprise" + + def __init__(self, next=None, prev=None): + self.next = next + if next is not None: + next.prev = self + self.prev = prev + if prev is not None: + prev.next = self + + def make_ll(depth): + head = LinkedList() + for i in range(depth): + head = LinkedList(head, head.prev) + return head + + head = make_ll(1000) + + assert(gc.isenabled()) + olds = [] + initial_heap_size = _testinternalcapi.get_tracked_heap_size() + for i in range(20_000): + newhead = make_ll(20) + newhead.surprise = head + olds.append(newhead) + if len(olds) == 20: + new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size + self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") + del olds[:] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py new file mode 100644 index 00000000000..965c0c0f493 --- /dev/null +++ b/Lib/test/_test_multiprocessing.py @@ -0,0 +1,6872 @@ +# +# Unit tests for the multiprocessing package +# + +import unittest +import unittest.mock +import queue as pyqueue +import textwrap +import time +import io +import itertools +import sys +import os +import gc +import importlib +import errno +import functools +import signal +import array +import socket +import random +import logging +import shutil +import subprocess +import struct +import tempfile +import operator +import pickle +import weakref +import warnings +import test.support +import test.support.script_helper +from test import support +from test.support import hashlib_helper +from test.support import import_helper +from test.support import os_helper +from test.support import script_helper +from test.support import socket_helper +from test.support import threading_helper +from test.support import warnings_helper +from test.support import subTests + +# Skip tests if _multiprocessing wasn't built. +_multiprocessing = import_helper.import_module('_multiprocessing') +# Skip tests if sem_open implementation is broken. +support.skip_if_broken_multiprocessing_synchronize() +import threading + +import multiprocessing.connection +import multiprocessing.dummy +import multiprocessing.heap +import multiprocessing.managers +import multiprocessing.pool +import multiprocessing.queues +from multiprocessing.connection import wait + +from multiprocessing import util + +try: + from multiprocessing import reduction + HAS_REDUCTION = reduction.HAVE_SEND_HANDLE +except ImportError: + HAS_REDUCTION = False + +try: + from multiprocessing.sharedctypes import Value, copy + HAS_SHAREDCTYPES = True +except ImportError: + HAS_SHAREDCTYPES = False + +try: + from multiprocessing import shared_memory + HAS_SHMEM = True +except ImportError: + HAS_SHMEM = False + +try: + import msvcrt +except ImportError: + msvcrt = None + + +if support.HAVE_ASAN_FORK_BUG: + # gh-89363: Skip multiprocessing tests if Python is built with ASAN to + # work around a libasan race condition: dead lock in pthread_create(). + raise unittest.SkipTest("libasan has a pthread_create() dead lock related to thread+fork") + + +# gh-110666: Tolerate a difference of 100 ms when comparing timings +# (clock resolution) +CLOCK_RES = 0.100 + + +def latin(s): + return s.encode('latin') + + +def close_queue(queue): + if isinstance(queue, multiprocessing.queues.Queue): + queue.close() + queue.join_thread() + + +def join_process(process): + # Since multiprocessing.Process has the same API than threading.Thread + # (join() and is_alive(), the support function can be reused + threading_helper.join_thread(process) + + +if os.name == "posix": + from multiprocessing import resource_tracker + + def _resource_unlink(name, rtype): + resource_tracker._CLEANUP_FUNCS[rtype](name) + + +# +# Constants +# + +LOG_LEVEL = util.SUBWARNING +#LOG_LEVEL = logging.DEBUG + +DELTA = 0.1 +CHECK_TIMINGS = False # making true makes tests take a lot longer + # and can sometimes cause some non-serious + # failures because some calls block a bit + # longer than expected +if CHECK_TIMINGS: + TIMEOUT1, TIMEOUT2, TIMEOUT3 = 0.82, 0.35, 1.4 +else: + TIMEOUT1, TIMEOUT2, TIMEOUT3 = 0.1, 0.1, 0.1 + +# BaseManager.shutdown_timeout +SHUTDOWN_TIMEOUT = support.SHORT_TIMEOUT + +WAIT_ACTIVE_CHILDREN_TIMEOUT = 5.0 + +HAVE_GETVALUE = not getattr(_multiprocessing, + 'HAVE_BROKEN_SEM_GETVALUE', False) + +WIN32 = (sys.platform == "win32") + +def wait_for_handle(handle, timeout): + if timeout is not None and timeout < 0.0: + timeout = None + return wait([handle], timeout) + +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except: + MAXFD = 256 + +# To speed up tests when using the forkserver, we can preload these: +PRELOAD = ['__main__', 'test.test_multiprocessing_forkserver'] + +# +# Some tests require ctypes +# + +try: + from ctypes import Structure, c_int, c_double, c_longlong +except ImportError: + Structure = object + c_int = c_double = c_longlong = None + + +def check_enough_semaphores(): + """Check that the system supports enough semaphores to run the test.""" + # minimum number of semaphores available according to POSIX + nsems_min = 256 + try: + nsems = os.sysconf("SC_SEM_NSEMS_MAX") + except (AttributeError, ValueError): + # sysconf not available or setting not available + return + if nsems == -1 or nsems >= nsems_min: + return + raise unittest.SkipTest("The OS doesn't support enough semaphores " + "to run the test (required: %d)." % nsems_min) + + +def only_run_in_spawn_testsuite(reason): + """Returns a decorator: raises SkipTest when SM != spawn at test time. + + This can be useful to save overall Python test suite execution time. + "spawn" is the universal mode available on all platforms so this limits the + decorated test to only execute within test_multiprocessing_spawn. + + This would not be necessary if we refactored our test suite to split things + into other test files when they are not start method specific to be rerun + under all start methods. + """ + + def decorator(test_item): + + @functools.wraps(test_item) + def spawn_check_wrapper(*args, **kwargs): + if (start_method := multiprocessing.get_start_method()) != "spawn": + raise unittest.SkipTest(f"{start_method=}, not 'spawn'; {reason}") + return test_item(*args, **kwargs) + + return spawn_check_wrapper + + return decorator + + +class TestInternalDecorators(unittest.TestCase): + """Logic within a test suite that could errantly skip tests? Test it!""" + + @unittest.skipIf(sys.platform == "win32", "test requires that fork exists.") + def test_only_run_in_spawn_testsuite(self): + if multiprocessing.get_start_method() != "spawn": + raise unittest.SkipTest("only run in test_multiprocessing_spawn.") + + try: + @only_run_in_spawn_testsuite("testing this decorator") + def return_four_if_spawn(): + return 4 + except Exception as err: + self.fail(f"expected decorated `def` not to raise; caught {err}") + + orig_start_method = multiprocessing.get_start_method(allow_none=True) + try: + multiprocessing.set_start_method("spawn", force=True) + self.assertEqual(return_four_if_spawn(), 4) + multiprocessing.set_start_method("fork", force=True) + with self.assertRaises(unittest.SkipTest) as ctx: + return_four_if_spawn() + self.assertIn("testing this decorator", str(ctx.exception)) + self.assertIn("start_method=", str(ctx.exception)) + finally: + multiprocessing.set_start_method(orig_start_method, force=True) + + +# +# Creates a wrapper for a function which records the time it takes to finish +# + +class TimingWrapper(object): + + def __init__(self, func): + self.func = func + self.elapsed = None + + def __call__(self, *args, **kwds): + t = time.monotonic() + try: + return self.func(*args, **kwds) + finally: + self.elapsed = time.monotonic() - t + +# +# Base class for test cases +# + +class BaseTestCase(object): + + ALLOWED_TYPES = ('processes', 'manager', 'threads') + # If not empty, limit which start method suites run this class. + START_METHODS: set[str] = set() + start_method = None # set by install_tests_in_module_dict() + + def assertTimingAlmostEqual(self, a, b): + if CHECK_TIMINGS: + self.assertAlmostEqual(a, b, 1) + + def assertReturnsIfImplemented(self, value, func, *args): + try: + res = func(*args) + except NotImplementedError: + pass + else: + return self.assertEqual(value, res) + + # For the sanity of Windows users, rather than crashing or freezing in + # multiple ways. + def __reduce__(self, *args): + raise NotImplementedError("shouldn't try to pickle a test case") + + __reduce_ex__ = __reduce__ + +# +# Return the value of a semaphore +# + +def get_value(self): + try: + return self.get_value() + except AttributeError: + try: + return self._Semaphore__value + except AttributeError: + try: + return self._value + except AttributeError: + raise NotImplementedError + +# +# Testcases +# + +class DummyCallable: + def __call__(self, q, c): + assert isinstance(c, DummyCallable) + q.put(5) + + +class _TestProcess(BaseTestCase): + + ALLOWED_TYPES = ('processes', 'threads') + + def test_current(self): + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + current = self.current_process() + authkey = current.authkey + + self.assertTrue(current.is_alive()) + self.assertTrue(not current.daemon) + self.assertIsInstance(authkey, bytes) + self.assertTrue(len(authkey) > 0) + self.assertEqual(current.ident, os.getpid()) + self.assertEqual(current.exitcode, None) + + def test_set_executable(self): + if self.TYPE == 'threads': + self.skipTest(f'test not appropriate for {self.TYPE}') + paths = [ + sys.executable, # str + os.fsencode(sys.executable), # bytes + os_helper.FakePath(sys.executable), # os.PathLike + os_helper.FakePath(os.fsencode(sys.executable)), # os.PathLike bytes + ] + for path in paths: + self.set_executable(path) + p = self.Process() + p.start() + p.join() + self.assertEqual(p.exitcode, 0) + + @support.requires_resource('cpu') + def test_args_argument(self): + # bpo-45735: Using list or tuple as *args* in constructor could + # achieve the same effect. + args_cases = (1, "str", [1], (1,)) + args_types = (list, tuple) + + test_cases = itertools.product(args_cases, args_types) + + for args, args_type in test_cases: + with self.subTest(args=args, args_type=args_type): + q = self.Queue(1) + # pass a tuple or list as args + p = self.Process(target=self._test_args, args=args_type((q, args))) + p.daemon = True + p.start() + child_args = q.get() + self.assertEqual(child_args, args) + p.join() + close_queue(q) + + @classmethod + def _test_args(cls, q, arg): + q.put(arg) + + def test_daemon_argument(self): + if self.TYPE == "threads": + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + # By default uses the current process's daemon flag. + proc0 = self.Process(target=self._test) + self.assertEqual(proc0.daemon, self.current_process().daemon) + proc1 = self.Process(target=self._test, daemon=True) + self.assertTrue(proc1.daemon) + proc2 = self.Process(target=self._test, daemon=False) + self.assertFalse(proc2.daemon) + + @classmethod + def _test(cls, q, *args, **kwds): + current = cls.current_process() + q.put(args) + q.put(kwds) + q.put(current.name) + if cls.TYPE != 'threads': + q.put(bytes(current.authkey)) + q.put(current.pid) + + def test_parent_process_attributes(self): + if self.TYPE == "threads": + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + self.assertIsNone(self.parent_process()) + + rconn, wconn = self.Pipe(duplex=False) + p = self.Process(target=self._test_send_parent_process, args=(wconn,)) + p.start() + p.join() + parent_pid, parent_name = rconn.recv() + self.assertEqual(parent_pid, self.current_process().pid) + self.assertEqual(parent_pid, os.getpid()) + self.assertEqual(parent_name, self.current_process().name) + + @classmethod + def _test_send_parent_process(cls, wconn): + from multiprocessing.process import parent_process + wconn.send([parent_process().pid, parent_process().name]) + + def test_parent_process(self): + if self.TYPE == "threads": + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + # Launch a child process. Make it launch a grandchild process. Kill the + # child process and make sure that the grandchild notices the death of + # its parent (a.k.a the child process). + rconn, wconn = self.Pipe(duplex=False) + p = self.Process( + target=self._test_create_grandchild_process, args=(wconn, )) + p.start() + + if not rconn.poll(timeout=support.LONG_TIMEOUT): + raise AssertionError("Could not communicate with child process") + parent_process_status = rconn.recv() + self.assertEqual(parent_process_status, "alive") + + p.terminate() + p.join() + + if not rconn.poll(timeout=support.LONG_TIMEOUT): + raise AssertionError("Could not communicate with child process") + parent_process_status = rconn.recv() + self.assertEqual(parent_process_status, "not alive") + + @classmethod + def _test_create_grandchild_process(cls, wconn): + p = cls.Process(target=cls._test_report_parent_status, args=(wconn, )) + p.start() + time.sleep(300) + + @classmethod + def _test_report_parent_status(cls, wconn): + from multiprocessing.process import parent_process + wconn.send("alive" if parent_process().is_alive() else "not alive") + parent_process().join(timeout=support.SHORT_TIMEOUT) + wconn.send("alive" if parent_process().is_alive() else "not alive") + + def test_process(self): + q = self.Queue(1) + e = self.Event() + args = (q, 1, 2) + kwargs = {'hello':23, 'bye':2.54} + name = 'SomeProcess' + p = self.Process( + target=self._test, args=args, kwargs=kwargs, name=name + ) + p.daemon = True + current = self.current_process() + + if self.TYPE != 'threads': + self.assertEqual(p.authkey, current.authkey) + self.assertEqual(p.is_alive(), False) + self.assertEqual(p.daemon, True) + self.assertNotIn(p, self.active_children()) + self.assertTrue(type(self.active_children()) is list) + self.assertEqual(p.exitcode, None) + + p.start() + + self.assertEqual(p.exitcode, None) + self.assertEqual(p.is_alive(), True) + self.assertIn(p, self.active_children()) + + self.assertEqual(q.get(), args[1:]) + self.assertEqual(q.get(), kwargs) + self.assertEqual(q.get(), p.name) + if self.TYPE != 'threads': + self.assertEqual(q.get(), current.authkey) + self.assertEqual(q.get(), p.pid) + + p.join() + + self.assertEqual(p.exitcode, 0) + self.assertEqual(p.is_alive(), False) + self.assertNotIn(p, self.active_children()) + close_queue(q) + + @unittest.skipUnless(threading._HAVE_THREAD_NATIVE_ID, "needs native_id") + def test_process_mainthread_native_id(self): + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + current_mainthread_native_id = threading.main_thread().native_id + + q = self.Queue(1) + p = self.Process(target=self._test_process_mainthread_native_id, args=(q,)) + p.start() + + child_mainthread_native_id = q.get() + p.join() + close_queue(q) + + self.assertNotEqual(current_mainthread_native_id, child_mainthread_native_id) + + @classmethod + def _test_process_mainthread_native_id(cls, q): + mainthread_native_id = threading.main_thread().native_id + q.put(mainthread_native_id) + + @classmethod + def _sleep_some(cls): + time.sleep(100) + + @classmethod + def _sleep_some_event(cls, event): + event.set() + time.sleep(100) + + @classmethod + def _test_sleep(cls, delay): + time.sleep(delay) + + def _kill_process(self, meth): + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + event = self.Event() + p = self.Process(target=self._sleep_some_event, args=(event,)) + p.daemon = True + p.start() + + self.assertEqual(p.is_alive(), True) + self.assertIn(p, self.active_children()) + self.assertEqual(p.exitcode, None) + + join = TimingWrapper(p.join) + + self.assertEqual(join(0), None) + self.assertTimingAlmostEqual(join.elapsed, 0.0) + self.assertEqual(p.is_alive(), True) + + self.assertEqual(join(-1), None) + self.assertTimingAlmostEqual(join.elapsed, 0.0) + self.assertEqual(p.is_alive(), True) + + timeout = support.SHORT_TIMEOUT + if not event.wait(timeout): + p.terminate() + p.join() + self.fail(f"event not signaled in {timeout} seconds") + + meth(p) + + if hasattr(signal, 'alarm'): + # On the Gentoo buildbot waitpid() often seems to block forever. + # We use alarm() to interrupt it if it blocks for too long. + def handler(*args): + raise RuntimeError('join took too long: %s' % p) + old_handler = signal.signal(signal.SIGALRM, handler) + try: + signal.alarm(10) + self.assertEqual(join(), None) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + else: + self.assertEqual(join(), None) + + self.assertTimingAlmostEqual(join.elapsed, 0.0) + + self.assertEqual(p.is_alive(), False) + self.assertNotIn(p, self.active_children()) + + p.join() + + return p.exitcode + + def test_terminate(self): + exitcode = self._kill_process(multiprocessing.Process.terminate) + self.assertEqual(exitcode, -signal.SIGTERM) + + def test_kill(self): + exitcode = self._kill_process(multiprocessing.Process.kill) + if os.name != 'nt': + self.assertEqual(exitcode, -signal.SIGKILL) + else: + self.assertEqual(exitcode, -signal.SIGTERM) + + def test_cpu_count(self): + try: + cpus = multiprocessing.cpu_count() + except NotImplementedError: + cpus = 1 + self.assertTrue(type(cpus) is int) + self.assertTrue(cpus >= 1) + + def test_active_children(self): + self.assertEqual(type(self.active_children()), list) + + event = self.Event() + p = self.Process(target=event.wait, args=()) + self.assertNotIn(p, self.active_children()) + + try: + p.daemon = True + p.start() + self.assertIn(p, self.active_children()) + finally: + event.set() + + p.join() + self.assertNotIn(p, self.active_children()) + + @classmethod + def _test_recursion(cls, wconn, id): + wconn.send(id) + if len(id) < 2: + for i in range(2): + p = cls.Process( + target=cls._test_recursion, args=(wconn, id+[i]) + ) + p.start() + p.join() + + def test_recursion(self): + rconn, wconn = self.Pipe(duplex=False) + self._test_recursion(wconn, []) + + time.sleep(DELTA) + result = [] + while rconn.poll(): + result.append(rconn.recv()) + + expected = [ + [], + [0], + [0, 0], + [0, 1], + [1], + [1, 0], + [1, 1] + ] + self.assertEqual(result, expected) + + @classmethod + def _test_sentinel(cls, event): + event.wait(10.0) + + def test_sentinel(self): + if self.TYPE == "threads": + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + event = self.Event() + p = self.Process(target=self._test_sentinel, args=(event,)) + with self.assertRaises(ValueError): + p.sentinel + p.start() + self.addCleanup(p.join) + sentinel = p.sentinel + self.assertIsInstance(sentinel, int) + self.assertFalse(wait_for_handle(sentinel, timeout=0.0)) + event.set() + p.join() + self.assertTrue(wait_for_handle(sentinel, timeout=1)) + + @classmethod + def _test_close(cls, rc=0, q=None): + if q is not None: + q.get() + sys.exit(rc) + + def test_close(self): + if self.TYPE == "threads": + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + q = self.Queue() + p = self.Process(target=self._test_close, kwargs={'q': q}) + p.daemon = True + p.start() + self.assertEqual(p.is_alive(), True) + # Child is still alive, cannot close + with self.assertRaises(ValueError): + p.close() + + q.put(None) + p.join() + self.assertEqual(p.is_alive(), False) + self.assertEqual(p.exitcode, 0) + p.close() + with self.assertRaises(ValueError): + p.is_alive() + with self.assertRaises(ValueError): + p.join() + with self.assertRaises(ValueError): + p.terminate() + p.close() + + wr = weakref.ref(p) + del p + gc.collect() + self.assertIs(wr(), None) + + close_queue(q) + + @support.requires_resource('walltime') + def test_many_processes(self): + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + sm = multiprocessing.get_start_method() + N = 5 if sm == 'spawn' else 100 + + # Try to overwhelm the forkserver loop with events + procs = [self.Process(target=self._test_sleep, args=(0.01,)) + for i in range(N)] + for p in procs: + p.start() + for p in procs: + join_process(p) + for p in procs: + self.assertEqual(p.exitcode, 0) + + procs = [self.Process(target=self._sleep_some) + for i in range(N)] + for p in procs: + p.start() + time.sleep(0.001) # let the children start... + for p in procs: + p.terminate() + for p in procs: + join_process(p) + if os.name != 'nt': + exitcodes = [-signal.SIGTERM] + if sys.platform == 'darwin': + # bpo-31510: On macOS, killing a freshly started process with + # SIGTERM sometimes kills the process with SIGKILL. + exitcodes.append(-signal.SIGKILL) + for p in procs: + self.assertIn(p.exitcode, exitcodes) + + def test_lose_target_ref(self): + c = DummyCallable() + wr = weakref.ref(c) + q = self.Queue() + p = self.Process(target=c, args=(q, c)) + del c + p.start() + p.join() + gc.collect() # For PyPy or other GCs. + self.assertIs(wr(), None) + self.assertEqual(q.get(), 5) + close_queue(q) + + @classmethod + def _test_child_fd_inflation(self, evt, q): + q.put(os_helper.fd_count()) + evt.wait() + + def test_child_fd_inflation(self): + # Number of fds in child processes should not grow with the + # number of running children. + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + sm = multiprocessing.get_start_method() + if sm == 'fork': + # The fork method by design inherits all fds from the parent, + # trying to go against it is a lost battle + self.skipTest('test not appropriate for {}'.format(sm)) + + N = 5 + evt = self.Event() + q = self.Queue() + + procs = [self.Process(target=self._test_child_fd_inflation, args=(evt, q)) + for i in range(N)] + for p in procs: + p.start() + + try: + fd_counts = [q.get() for i in range(N)] + self.assertEqual(len(set(fd_counts)), 1, fd_counts) + + finally: + evt.set() + for p in procs: + p.join() + close_queue(q) + + @classmethod + def _test_wait_for_threads(self, evt): + def func1(): + time.sleep(0.5) + evt.set() + + def func2(): + time.sleep(20) + evt.clear() + + threading.Thread(target=func1).start() + threading.Thread(target=func2, daemon=True).start() + + def test_wait_for_threads(self): + # A child process should wait for non-daemonic threads to end + # before exiting + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + evt = self.Event() + proc = self.Process(target=self._test_wait_for_threads, args=(evt,)) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + + @classmethod + def _test_error_on_stdio_flush(self, evt, break_std_streams={}): + for stream_name, action in break_std_streams.items(): + if action == 'close': + stream = io.StringIO() + stream.close() + else: + assert action == 'remove' + stream = None + setattr(sys, stream_name, None) + evt.set() + + def test_error_on_stdio_flush_1(self): + # Check that Process works with broken standard streams + streams = [io.StringIO(), None] + streams[0].close() + for stream_name in ('stdout', 'stderr'): + for stream in streams: + old_stream = getattr(sys, stream_name) + setattr(sys, stream_name, stream) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt,)) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) + finally: + setattr(sys, stream_name, old_stream) + + def test_error_on_stdio_flush_2(self): + # Same as test_error_on_stdio_flush_1(), but standard streams are + # broken by the child process + for stream_name in ('stdout', 'stderr'): + for action in ('close', 'remove'): + old_stream = getattr(sys, stream_name) + try: + evt = self.Event() + proc = self.Process(target=self._test_error_on_stdio_flush, + args=(evt, {stream_name: action})) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) + finally: + setattr(sys, stream_name, old_stream) + + @classmethod + def _sleep_and_set_event(self, evt, delay=0.0): + time.sleep(delay) + evt.set() + + def check_forkserver_death(self, signum): + # bpo-31308: if the forkserver process has died, we should still + # be able to create and run new Process instances (the forkserver + # is implicitly restarted). + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + sm = multiprocessing.get_start_method() + if sm != 'forkserver': + # The fork method by design inherits all fds from the parent, + # trying to go against it is a lost battle + self.skipTest('test not appropriate for {}'.format(sm)) + + from multiprocessing.forkserver import _forkserver + _forkserver.ensure_running() + + # First process sleeps 500 ms + delay = 0.5 + + evt = self.Event() + proc = self.Process(target=self._sleep_and_set_event, args=(evt, delay)) + proc.start() + + pid = _forkserver._forkserver_pid + os.kill(pid, signum) + # give time to the fork server to die and time to proc to complete + time.sleep(delay * 2.0) + + evt2 = self.Event() + proc2 = self.Process(target=self._sleep_and_set_event, args=(evt2,)) + proc2.start() + proc2.join() + self.assertTrue(evt2.is_set()) + self.assertEqual(proc2.exitcode, 0) + + proc.join() + self.assertTrue(evt.is_set()) + self.assertIn(proc.exitcode, (0, 255)) + + def test_forkserver_sigint(self): + # Catchable signal + self.check_forkserver_death(signal.SIGINT) + + def test_forkserver_sigkill(self): + # Uncatchable signal + if os.name != 'nt': + self.check_forkserver_death(signal.SIGKILL) + + +# +# +# + +class _UpperCaser(multiprocessing.Process): + + def __init__(self): + multiprocessing.Process.__init__(self) + self.child_conn, self.parent_conn = multiprocessing.Pipe() + + def run(self): + self.parent_conn.close() + for s in iter(self.child_conn.recv, None): + self.child_conn.send(s.upper()) + self.child_conn.close() + + def submit(self, s): + assert type(s) is str + self.parent_conn.send(s) + return self.parent_conn.recv() + + def stop(self): + self.parent_conn.send(None) + self.parent_conn.close() + self.child_conn.close() + +class _TestSubclassingProcess(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + def test_subclassing(self): + uppercaser = _UpperCaser() + uppercaser.daemon = True + uppercaser.start() + self.assertEqual(uppercaser.submit('hello'), 'HELLO') + self.assertEqual(uppercaser.submit('world'), 'WORLD') + uppercaser.stop() + uppercaser.join() + + def test_stderr_flush(self): + # sys.stderr is flushed at process shutdown (issue #13812) + if self.TYPE == "threads": + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + testfn = os_helper.TESTFN + self.addCleanup(os_helper.unlink, testfn) + proc = self.Process(target=self._test_stderr_flush, args=(testfn,)) + proc.start() + proc.join() + with open(testfn, encoding="utf-8") as f: + err = f.read() + # The whole traceback was printed + self.assertIn("ZeroDivisionError", err) + self.assertIn("test_multiprocessing.py", err) + self.assertIn("1/0 # MARKER", err) + + @classmethod + def _test_stderr_flush(cls, testfn): + fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + sys.stderr = open(fd, 'w', encoding="utf-8", closefd=False) + 1/0 # MARKER + + + @classmethod + def _test_sys_exit(cls, reason, testfn): + fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + sys.stderr = open(fd, 'w', encoding="utf-8", closefd=False) + sys.exit(reason) + + def test_sys_exit(self): + # See Issue 13854 + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + testfn = os_helper.TESTFN + self.addCleanup(os_helper.unlink, testfn) + + for reason in ( + [1, 2, 3], + 'ignore this', + ): + p = self.Process(target=self._test_sys_exit, args=(reason, testfn)) + p.daemon = True + p.start() + join_process(p) + self.assertEqual(p.exitcode, 1) + + with open(testfn, encoding="utf-8") as f: + content = f.read() + self.assertEqual(content.rstrip(), str(reason)) + + os.unlink(testfn) + + cases = [ + ((True,), 1), + ((False,), 0), + ((8,), 8), + ((None,), 0), + ((), 0), + ] + + for args, expected in cases: + with self.subTest(args=args): + p = self.Process(target=sys.exit, args=args) + p.daemon = True + p.start() + join_process(p) + self.assertEqual(p.exitcode, expected) + +# +# +# + +def queue_empty(q): + if hasattr(q, 'empty'): + return q.empty() + else: + return q.qsize() == 0 + +def queue_full(q, maxsize): + if hasattr(q, 'full'): + return q.full() + else: + return q.qsize() == maxsize + + +class _TestQueue(BaseTestCase): + + + @classmethod + def _test_put(cls, queue, child_can_start, parent_can_continue): + child_can_start.wait() + for i in range(6): + queue.get() + parent_can_continue.set() + + def test_put(self): + MAXSIZE = 6 + queue = self.Queue(maxsize=MAXSIZE) + child_can_start = self.Event() + parent_can_continue = self.Event() + + proc = self.Process( + target=self._test_put, + args=(queue, child_can_start, parent_can_continue) + ) + proc.daemon = True + proc.start() + + self.assertEqual(queue_empty(queue), True) + self.assertEqual(queue_full(queue, MAXSIZE), False) + + queue.put(1) + queue.put(2, True) + queue.put(3, True, None) + queue.put(4, False) + queue.put(5, False, None) + queue.put_nowait(6) + + # the values may be in buffer but not yet in pipe so sleep a bit + time.sleep(DELTA) + + self.assertEqual(queue_empty(queue), False) + self.assertEqual(queue_full(queue, MAXSIZE), True) + + put = TimingWrapper(queue.put) + put_nowait = TimingWrapper(queue.put_nowait) + + self.assertRaises(pyqueue.Full, put, 7, False) + self.assertTimingAlmostEqual(put.elapsed, 0) + + self.assertRaises(pyqueue.Full, put, 7, False, None) + self.assertTimingAlmostEqual(put.elapsed, 0) + + self.assertRaises(pyqueue.Full, put_nowait, 7) + self.assertTimingAlmostEqual(put_nowait.elapsed, 0) + + self.assertRaises(pyqueue.Full, put, 7, True, TIMEOUT1) + self.assertTimingAlmostEqual(put.elapsed, TIMEOUT1) + + self.assertRaises(pyqueue.Full, put, 7, False, TIMEOUT2) + self.assertTimingAlmostEqual(put.elapsed, 0) + + self.assertRaises(pyqueue.Full, put, 7, True, timeout=TIMEOUT3) + self.assertTimingAlmostEqual(put.elapsed, TIMEOUT3) + + child_can_start.set() + parent_can_continue.wait() + + self.assertEqual(queue_empty(queue), True) + self.assertEqual(queue_full(queue, MAXSIZE), False) + + proc.join() + close_queue(queue) + + @classmethod + def _test_get(cls, queue, child_can_start, parent_can_continue): + child_can_start.wait() + queue.put(1) + queue.put(2) + queue.put(3) + queue.put(4) + queue.put(5) + parent_can_continue.set() + + def test_get(self): + queue = self.Queue() + child_can_start = self.Event() + parent_can_continue = self.Event() + + proc = self.Process( + target=self._test_get, + args=(queue, child_can_start, parent_can_continue) + ) + proc.daemon = True + proc.start() + + self.assertEqual(queue_empty(queue), True) + + child_can_start.set() + parent_can_continue.wait() + + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if not queue_empty(queue): + break + self.assertEqual(queue_empty(queue), False) + + self.assertEqual(queue.get_nowait(), 1) + self.assertEqual(queue.get(True, None), 2) + self.assertEqual(queue.get(True), 3) + self.assertEqual(queue.get(timeout=1), 4) + self.assertEqual(queue.get(), 5) + + self.assertEqual(queue_empty(queue), True) + + get = TimingWrapper(queue.get) + get_nowait = TimingWrapper(queue.get_nowait) + + self.assertRaises(pyqueue.Empty, get, False) + self.assertTimingAlmostEqual(get.elapsed, 0) + + self.assertRaises(pyqueue.Empty, get, False, None) + self.assertTimingAlmostEqual(get.elapsed, 0) + + self.assertRaises(pyqueue.Empty, get_nowait) + self.assertTimingAlmostEqual(get_nowait.elapsed, 0) + + self.assertRaises(pyqueue.Empty, get, True, TIMEOUT1) + self.assertTimingAlmostEqual(get.elapsed, TIMEOUT1) + + self.assertRaises(pyqueue.Empty, get, False, TIMEOUT2) + self.assertTimingAlmostEqual(get.elapsed, 0) + + self.assertRaises(pyqueue.Empty, get, timeout=TIMEOUT3) + self.assertTimingAlmostEqual(get.elapsed, TIMEOUT3) + + proc.join() + close_queue(queue) + + @classmethod + def _test_fork(cls, queue): + for i in range(10, 20): + queue.put(i) + # note that at this point the items may only be buffered, so the + # process cannot shutdown until the feeder thread has finished + # pushing items onto the pipe. + + def test_fork(self): + # Old versions of Queue would fail to create a new feeder + # thread for a forked process if the original process had its + # own feeder thread. This test checks that this no longer + # happens. + + queue = self.Queue() + + # put items on queue so that main process starts a feeder thread + for i in range(10): + queue.put(i) + + # wait to make sure thread starts before we fork a new process + time.sleep(DELTA) + + # fork process + p = self.Process(target=self._test_fork, args=(queue,)) + p.daemon = True + p.start() + + # check that all expected items are in the queue + for i in range(20): + self.assertEqual(queue.get(), i) + self.assertRaises(pyqueue.Empty, queue.get, False) + + p.join() + close_queue(queue) + + def test_qsize(self): + q = self.Queue() + try: + self.assertEqual(q.qsize(), 0) + except NotImplementedError: + self.skipTest('qsize method not implemented') + q.put(1) + self.assertEqual(q.qsize(), 1) + q.put(5) + self.assertEqual(q.qsize(), 2) + q.get() + self.assertEqual(q.qsize(), 1) + q.get() + self.assertEqual(q.qsize(), 0) + close_queue(q) + + @classmethod + def _test_task_done(cls, q): + for obj in iter(q.get, None): + time.sleep(DELTA) + q.task_done() + + def test_task_done(self): + queue = self.JoinableQueue() + + workers = [self.Process(target=self._test_task_done, args=(queue,)) + for i in range(4)] + + for p in workers: + p.daemon = True + p.start() + + for i in range(10): + queue.put(i) + + queue.join() + + for p in workers: + queue.put(None) + + for p in workers: + p.join() + close_queue(queue) + + def test_no_import_lock_contention(self): + with os_helper.temp_cwd(): + module_name = 'imported_by_an_imported_module' + with open(module_name + '.py', 'w', encoding="utf-8") as f: + f.write("""if 1: + import multiprocessing + + q = multiprocessing.Queue() + q.put('knock knock') + q.get(timeout=3) + q.close() + del q + """) + + with import_helper.DirsOnSysPath(os.getcwd()): + try: + __import__(module_name) + except pyqueue.Empty: + self.fail("Probable regression on import lock contention;" + " see Issue #22853") + + def test_timeout(self): + q = multiprocessing.Queue() + start = time.monotonic() + self.assertRaises(pyqueue.Empty, q.get, True, 0.200) + delta = time.monotonic() - start + # bpo-30317: Tolerate a delta of 100 ms because of the bad clock + # resolution on Windows (usually 15.6 ms). x86 Windows7 3.x once + # failed because the delta was only 135.8 ms. + self.assertGreaterEqual(delta, 0.100) + close_queue(q) + + def test_queue_feeder_donot_stop_onexc(self): + # bpo-30414: verify feeder handles exceptions correctly + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + class NotSerializable(object): + def __reduce__(self): + raise AttributeError + with test.support.captured_stderr(): + q = self.Queue() + q.put(NotSerializable()) + q.put(True) + self.assertTrue(q.get(timeout=support.SHORT_TIMEOUT)) + close_queue(q) + + with test.support.captured_stderr(): + # bpo-33078: verify that the queue size is correctly handled + # on errors. + q = self.Queue(maxsize=1) + q.put(NotSerializable()) + q.put(True) + try: + self.assertEqual(q.qsize(), 1) + except NotImplementedError: + # qsize is not available on all platform as it + # relies on sem_getvalue + pass + self.assertTrue(q.get(timeout=support.SHORT_TIMEOUT)) + # Check that the size of the queue is correct + self.assertTrue(q.empty()) + close_queue(q) + + def test_queue_feeder_on_queue_feeder_error(self): + # bpo-30006: verify feeder handles exceptions using the + # _on_queue_feeder_error hook. + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + class NotSerializable(object): + """Mock unserializable object""" + def __init__(self): + self.reduce_was_called = False + self.on_queue_feeder_error_was_called = False + + def __reduce__(self): + self.reduce_was_called = True + raise AttributeError + + class SafeQueue(multiprocessing.queues.Queue): + """Queue with overloaded _on_queue_feeder_error hook""" + @staticmethod + def _on_queue_feeder_error(e, obj): + if (isinstance(e, AttributeError) and + isinstance(obj, NotSerializable)): + obj.on_queue_feeder_error_was_called = True + + not_serializable_obj = NotSerializable() + # The captured_stderr reduces the noise in the test report + with test.support.captured_stderr(): + q = SafeQueue(ctx=multiprocessing.get_context()) + q.put(not_serializable_obj) + + # Verify that q is still functioning correctly + q.put(True) + self.assertTrue(q.get(timeout=support.SHORT_TIMEOUT)) + + # Assert that the serialization and the hook have been called correctly + self.assertTrue(not_serializable_obj.reduce_was_called) + self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called) + + def test_closed_queue_empty_exceptions(self): + # Assert that checking the emptiness of an unused closed queue + # does not raise an OSError. The rationale is that q.close() is + # a no-op upon construction and becomes effective once the queue + # has been used (e.g., by calling q.put()). + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.close() # this is a no-op since the feeder thread is None + q.join_thread() # this is also a no-op + self.assertTrue(q.empty()) + + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.put('foo') # make sure that the queue is 'used' + q.close() # close the feeder thread + q.join_thread() # make sure to join the feeder thread + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + + def test_closed_queue_put_get_exceptions(self): + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.close() + with self.assertRaisesRegex(ValueError, 'is closed'): + q.put('foo') + with self.assertRaisesRegex(ValueError, 'is closed'): + q.get() +# +# +# + +class _TestLock(BaseTestCase): + + @staticmethod + def _acquire(lock, l=None): + lock.acquire() + if l is not None: + l.append(repr(lock)) + + @staticmethod + def _acquire_event(lock, event): + lock.acquire() + event.set() + time.sleep(1.0) + + def test_repr_lock(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + lock = self.Lock() + self.assertEqual(f'', repr(lock)) + + lock.acquire() + self.assertEqual(f'', repr(lock)) + lock.release() + + tname = 'T1' + l = [] + t = threading.Thread(target=self._acquire, + args=(lock, l), + name=tname) + t.start() + time.sleep(0.1) + self.assertEqual(f'', l[0]) + lock.release() + + t = threading.Thread(target=self._acquire, + args=(lock,), + name=tname) + t.start() + time.sleep(0.1) + self.assertEqual('', repr(lock)) + lock.release() + + pname = 'P1' + l = multiprocessing.Manager().list() + p = self.Process(target=self._acquire, + args=(lock, l), + name=pname) + p.start() + p.join() + self.assertEqual(f'', l[0]) + + lock = self.Lock() + event = self.Event() + p = self.Process(target=self._acquire_event, + args=(lock, event), + name='P2') + p.start() + event.wait() + self.assertEqual(f'', repr(lock)) + p.terminate() + + def test_lock(self): + lock = self.Lock() + self.assertEqual(lock.acquire(), True) + self.assertEqual(lock.acquire(False), False) + self.assertEqual(lock.release(), None) + self.assertRaises((ValueError, threading.ThreadError), lock.release) + + @staticmethod + def _acquire_release(lock, timeout, l=None, n=1): + for _ in range(n): + lock.acquire() + if l is not None: + l.append(repr(lock)) + time.sleep(timeout) + for _ in range(n): + lock.release() + + @unittest.skip("TODO: RUSTPYTHON; flaky timeout - thread start latency") + def test_repr_rlock(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + lock = self.RLock() + self.assertEqual('', repr(lock)) + + n = 3 + for _ in range(n): + lock.acquire() + self.assertEqual(f'', repr(lock)) + for _ in range(n): + lock.release() + + t, l = [], [] + for i in range(n): + t.append(threading.Thread(target=self._acquire_release, + args=(lock, 0.1, l, i+1), + name=f'T{i+1}')) + t[-1].start() + for t_ in t: + t_.join() + for i in range(n): + self.assertIn(f'', l) + + + t = threading.Thread(target=self._acquire_release, + args=(lock, 0.2), + name=f'T1') + t.start() + time.sleep(0.1) + self.assertEqual('', repr(lock)) + time.sleep(0.2) + + pname = 'P1' + l = multiprocessing.Manager().list() + p = self.Process(target=self._acquire_release, + args=(lock, 0.1, l), + name=pname) + p.start() + p.join() + self.assertEqual(f'', l[0]) + + event = self.Event() + lock = self.RLock() + p = self.Process(target=self._acquire_event, + args=(lock, event)) + p.start() + event.wait() + self.assertEqual('', repr(lock)) + p.join() + + def test_rlock(self): + lock = self.RLock() + self.assertEqual(lock.acquire(), True) + self.assertEqual(lock.acquire(), True) + self.assertEqual(lock.acquire(), True) + self.assertEqual(lock.release(), None) + self.assertEqual(lock.release(), None) + self.assertEqual(lock.release(), None) + self.assertRaises((AssertionError, RuntimeError), lock.release) + + def test_lock_context(self): + with self.Lock(): + pass + + +class _TestSemaphore(BaseTestCase): + + def _test_semaphore(self, sem): + self.assertReturnsIfImplemented(2, get_value, sem) + self.assertEqual(sem.acquire(), True) + self.assertReturnsIfImplemented(1, get_value, sem) + self.assertEqual(sem.acquire(), True) + self.assertReturnsIfImplemented(0, get_value, sem) + self.assertEqual(sem.acquire(False), False) + self.assertReturnsIfImplemented(0, get_value, sem) + self.assertEqual(sem.release(), None) + self.assertReturnsIfImplemented(1, get_value, sem) + self.assertEqual(sem.release(), None) + self.assertReturnsIfImplemented(2, get_value, sem) + + def test_semaphore(self): + sem = self.Semaphore(2) + self._test_semaphore(sem) + self.assertEqual(sem.release(), None) + self.assertReturnsIfImplemented(3, get_value, sem) + self.assertEqual(sem.release(), None) + self.assertReturnsIfImplemented(4, get_value, sem) + + def test_bounded_semaphore(self): + sem = self.BoundedSemaphore(2) + self._test_semaphore(sem) + # Currently fails on OS/X + #if HAVE_GETVALUE: + # self.assertRaises(ValueError, sem.release) + # self.assertReturnsIfImplemented(2, get_value, sem) + + def test_timeout(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + sem = self.Semaphore(0) + acquire = TimingWrapper(sem.acquire) + + self.assertEqual(acquire(False), False) + self.assertTimingAlmostEqual(acquire.elapsed, 0.0) + + self.assertEqual(acquire(False, None), False) + self.assertTimingAlmostEqual(acquire.elapsed, 0.0) + + self.assertEqual(acquire(False, TIMEOUT1), False) + self.assertTimingAlmostEqual(acquire.elapsed, 0) + + self.assertEqual(acquire(True, TIMEOUT2), False) + self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT2) + + self.assertEqual(acquire(timeout=TIMEOUT3), False) + self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT3) + + +class _TestCondition(BaseTestCase): + + @classmethod + def f(cls, cond, sleeping, woken, timeout=None): + cond.acquire() + sleeping.release() + cond.wait(timeout) + woken.release() + cond.release() + + def assertReachesEventually(self, func, value): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + try: + if func() == value: + break + except NotImplementedError: + break + + self.assertReturnsIfImplemented(value, func) + + def check_invariant(self, cond): + # this is only supposed to succeed when there are no sleepers + if self.TYPE == 'processes': + try: + sleepers = (cond._sleeping_count.get_value() - + cond._woken_count.get_value()) + self.assertEqual(sleepers, 0) + self.assertEqual(cond._wait_semaphore.get_value(), 0) + except NotImplementedError: + pass + + def test_notify(self): + cond = self.Condition() + sleeping = self.Semaphore(0) + woken = self.Semaphore(0) + + p = self.Process(target=self.f, args=(cond, sleeping, woken)) + p.daemon = True + p.start() + + t = threading.Thread(target=self.f, args=(cond, sleeping, woken)) + t.daemon = True + t.start() + + # wait for both children to start sleeping + sleeping.acquire() + sleeping.acquire() + + # check no process/thread has woken up + self.assertReachesEventually(lambda: get_value(woken), 0) + + # wake up one process/thread + cond.acquire() + cond.notify() + cond.release() + + # check one process/thread has woken up + self.assertReachesEventually(lambda: get_value(woken), 1) + + # wake up another + cond.acquire() + cond.notify() + cond.release() + + # check other has woken up + self.assertReachesEventually(lambda: get_value(woken), 2) + + # check state is not mucked up + self.check_invariant(cond) + + threading_helper.join_thread(t) + join_process(p) + + def test_notify_all(self): + cond = self.Condition() + sleeping = self.Semaphore(0) + woken = self.Semaphore(0) + + # start some threads/processes which will timeout + workers = [] + for i in range(3): + p = self.Process(target=self.f, + args=(cond, sleeping, woken, TIMEOUT1)) + p.daemon = True + p.start() + workers.append(p) + + t = threading.Thread(target=self.f, + args=(cond, sleeping, woken, TIMEOUT1)) + t.daemon = True + t.start() + workers.append(t) + + # wait for them all to sleep + for i in range(6): + sleeping.acquire() + + # check they have all timed out + for i in range(6): + woken.acquire() + self.assertReturnsIfImplemented(0, get_value, woken) + + # check state is not mucked up + self.check_invariant(cond) + + # start some more threads/processes + for i in range(3): + p = self.Process(target=self.f, args=(cond, sleeping, woken)) + p.daemon = True + p.start() + workers.append(p) + + t = threading.Thread(target=self.f, args=(cond, sleeping, woken)) + t.daemon = True + t.start() + workers.append(t) + + # wait for them to all sleep + for i in range(6): + sleeping.acquire() + + # check no process/thread has woken up + time.sleep(DELTA) + self.assertReturnsIfImplemented(0, get_value, woken) + + # wake them all up + cond.acquire() + cond.notify_all() + cond.release() + + # check they have all woken + for i in range(6): + woken.acquire() + self.assertReturnsIfImplemented(0, get_value, woken) + + # check state is not mucked up + self.check_invariant(cond) + + for w in workers: + # NOTE: join_process and join_thread are the same + threading_helper.join_thread(w) + + def test_notify_n(self): + cond = self.Condition() + sleeping = self.Semaphore(0) + woken = self.Semaphore(0) + + # start some threads/processes + workers = [] + for i in range(3): + p = self.Process(target=self.f, args=(cond, sleeping, woken)) + p.daemon = True + p.start() + workers.append(p) + + t = threading.Thread(target=self.f, args=(cond, sleeping, woken)) + t.daemon = True + t.start() + workers.append(t) + + # wait for them to all sleep + for i in range(6): + sleeping.acquire() + + # check no process/thread has woken up + time.sleep(DELTA) + self.assertReturnsIfImplemented(0, get_value, woken) + + # wake some of them up + cond.acquire() + cond.notify(n=2) + cond.release() + + # check 2 have woken + self.assertReachesEventually(lambda: get_value(woken), 2) + + # wake the rest of them + cond.acquire() + cond.notify(n=4) + cond.release() + + self.assertReachesEventually(lambda: get_value(woken), 6) + + # doesn't do anything more + cond.acquire() + cond.notify(n=3) + cond.release() + + self.assertReturnsIfImplemented(6, get_value, woken) + + # check state is not mucked up + self.check_invariant(cond) + + for w in workers: + # NOTE: join_process and join_thread are the same + threading_helper.join_thread(w) + + def test_timeout(self): + cond = self.Condition() + wait = TimingWrapper(cond.wait) + cond.acquire() + res = wait(TIMEOUT1) + cond.release() + self.assertEqual(res, False) + self.assertTimingAlmostEqual(wait.elapsed, TIMEOUT1) + + @classmethod + def _test_waitfor_f(cls, cond, state): + with cond: + state.value = 0 + cond.notify() + result = cond.wait_for(lambda : state.value==4) + if not result or state.value != 4: + sys.exit(1) + + @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') + def test_waitfor(self): + # based on test in test/lock_tests.py + cond = self.Condition() + state = self.Value('i', -1) + + p = self.Process(target=self._test_waitfor_f, args=(cond, state)) + p.daemon = True + p.start() + + with cond: + result = cond.wait_for(lambda : state.value==0) + self.assertTrue(result) + self.assertEqual(state.value, 0) + + for i in range(4): + time.sleep(0.01) + with cond: + state.value += 1 + cond.notify() + + join_process(p) + self.assertEqual(p.exitcode, 0) + + @classmethod + def _test_waitfor_timeout_f(cls, cond, state, success, sem): + sem.release() + with cond: + expected = 0.100 + dt = time.monotonic() + result = cond.wait_for(lambda : state.value==4, timeout=expected) + dt = time.monotonic() - dt + if not result and (expected - CLOCK_RES) <= dt: + success.value = True + + @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') + def test_waitfor_timeout(self): + # based on test in test/lock_tests.py + cond = self.Condition() + state = self.Value('i', 0) + success = self.Value('i', False) + sem = self.Semaphore(0) + + p = self.Process(target=self._test_waitfor_timeout_f, + args=(cond, state, success, sem)) + p.daemon = True + p.start() + self.assertTrue(sem.acquire(timeout=support.LONG_TIMEOUT)) + + # Only increment 3 times, so state == 4 is never reached. + for i in range(3): + time.sleep(0.010) + with cond: + state.value += 1 + cond.notify() + + join_process(p) + self.assertTrue(success.value) + + @classmethod + def _test_wait_result(cls, c, pid): + with c: + c.notify() + time.sleep(1) + if pid is not None: + os.kill(pid, signal.SIGINT) + + def test_wait_result(self): + if isinstance(self, ProcessesMixin) and sys.platform != 'win32': + pid = os.getpid() + else: + pid = None + + c = self.Condition() + with c: + self.assertFalse(c.wait(0)) + self.assertFalse(c.wait(0.1)) + + p = self.Process(target=self._test_wait_result, args=(c, pid)) + p.start() + + self.assertTrue(c.wait(60)) + if pid is not None: + self.assertRaises(KeyboardInterrupt, c.wait, 60) + + p.join() + + +class _TestEvent(BaseTestCase): + + @classmethod + def _test_event(cls, event): + time.sleep(TIMEOUT2) + event.set() + + def test_event(self): + event = self.Event() + wait = TimingWrapper(event.wait) + + # Removed temporarily, due to API shear, this does not + # work with threading._Event objects. is_set == isSet + self.assertEqual(event.is_set(), False) + + # Removed, threading.Event.wait() will return the value of the __flag + # instead of None. API Shear with the semaphore backed mp.Event + self.assertEqual(wait(0.0), False) + self.assertTimingAlmostEqual(wait.elapsed, 0.0) + self.assertEqual(wait(TIMEOUT1), False) + self.assertTimingAlmostEqual(wait.elapsed, TIMEOUT1) + + event.set() + + # See note above on the API differences + self.assertEqual(event.is_set(), True) + self.assertEqual(wait(), True) + self.assertTimingAlmostEqual(wait.elapsed, 0.0) + self.assertEqual(wait(TIMEOUT1), True) + self.assertTimingAlmostEqual(wait.elapsed, 0.0) + # self.assertEqual(event.is_set(), True) + + event.clear() + + #self.assertEqual(event.is_set(), False) + + p = self.Process(target=self._test_event, args=(event,)) + p.daemon = True + p.start() + self.assertEqual(wait(), True) + p.join() + + def test_repr(self) -> None: + event = self.Event() + if self.TYPE == 'processes': + self.assertRegex(repr(event), r"") + event.set() + self.assertRegex(repr(event), r"") + event.clear() + self.assertRegex(repr(event), r"") + elif self.TYPE == 'manager': + self.assertRegex(repr(event), r" 256 (issue #11657) + if self.TYPE != 'processes': + self.skipTest("only makes sense with processes") + conn, child_conn = self.Pipe(duplex=True) + + p = self.Process(target=self._writefd, args=(child_conn, b"bar", True)) + p.daemon = True + p.start() + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + fd = f.fileno() + for newfd in range(256, MAXFD): + if not self._is_fd_assigned(newfd): + break + else: + self.fail("could not find an unassigned large file descriptor") + os.dup2(fd, newfd) + try: + reduction.send_handle(conn, newfd, p.pid) + finally: + os.close(newfd) + p.join() + with open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"bar") + + @classmethod + def _send_data_without_fd(self, conn): + os.write(conn.fileno(), b"\0") + + @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction") + @unittest.skipIf(sys.platform == "win32", "doesn't make sense on Windows") + def test_missing_fd_transfer(self): + # Check that exception is raised when received data is not + # accompanied by a file descriptor in ancillary data. + if self.TYPE != 'processes': + self.skipTest("only makes sense with processes") + conn, child_conn = self.Pipe(duplex=True) + + p = self.Process(target=self._send_data_without_fd, args=(child_conn,)) + p.daemon = True + p.start() + self.assertRaises(RuntimeError, reduction.recv_handle, conn) + p.join() + + def test_context(self): + a, b = self.Pipe() + + with a, b: + a.send(1729) + self.assertEqual(b.recv(), 1729) + if self.TYPE == 'processes': + self.assertFalse(a.closed) + self.assertFalse(b.closed) + + if self.TYPE == 'processes': + self.assertTrue(a.closed) + self.assertTrue(b.closed) + self.assertRaises(OSError, a.recv) + self.assertRaises(OSError, b.recv) + +class _TestListener(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + def test_multiple_bind(self): + for family in self.connection.families: + l = self.connection.Listener(family=family) + self.addCleanup(l.close) + self.assertRaises(OSError, self.connection.Listener, + l.address, family) + + def test_context(self): + with self.connection.Listener() as l: + with self.connection.Client(l.address) as c: + with l.accept() as d: + c.send(1729) + self.assertEqual(d.recv(), 1729) + + if self.TYPE == 'processes': + self.assertRaises(OSError, l.accept) + + def test_empty_authkey(self): + # bpo-43952: allow empty bytes as authkey + def handler(*args): + raise RuntimeError('Connection took too long...') + + def run(addr, authkey): + client = self.connection.Client(addr, authkey=authkey) + client.send(1729) + + key = b'' + + with self.connection.Listener(authkey=key) as listener: + thread = threading.Thread(target=run, args=(listener.address, key)) + thread.start() + try: + with listener.accept() as d: + self.assertEqual(d.recv(), 1729) + finally: + thread.join() + + if self.TYPE == 'processes': + with self.assertRaises(OSError): + listener.accept() + + @unittest.skipUnless(util.abstract_sockets_supported, + "test needs abstract socket support") + def test_abstract_socket(self): + with self.connection.Listener("\0something") as listener: + with self.connection.Client(listener.address) as client: + with listener.accept() as d: + client.send(1729) + self.assertEqual(d.recv(), 1729) + + if self.TYPE == 'processes': + self.assertRaises(OSError, listener.accept) + + +class _TestListenerClient(BaseTestCase): + + ALLOWED_TYPES = ('processes', 'threads') + + @classmethod + def _test(cls, address): + conn = cls.connection.Client(address) + conn.send('hello') + conn.close() + + def test_listener_client(self): + for family in self.connection.families: + l = self.connection.Listener(family=family) + p = self.Process(target=self._test, args=(l.address,)) + p.daemon = True + p.start() + conn = l.accept() + self.assertEqual(conn.recv(), 'hello') + p.join() + l.close() + + def test_issue14725(self): + l = self.connection.Listener() + p = self.Process(target=self._test, args=(l.address,)) + p.daemon = True + p.start() + time.sleep(1) + # On Windows the client process should by now have connected, + # written data and closed the pipe handle by now. This causes + # ConnectNamdedPipe() to fail with ERROR_NO_DATA. See Issue + # 14725. + conn = l.accept() + self.assertEqual(conn.recv(), 'hello') + conn.close() + p.join() + l.close() + + def test_issue16955(self): + for fam in self.connection.families: + l = self.connection.Listener(family=fam) + c = self.connection.Client(l.address) + a = l.accept() + a.send_bytes(b"hello") + self.assertTrue(c.poll(1)) + a.close() + c.close() + l.close() + +class _TestPoll(BaseTestCase): + + ALLOWED_TYPES = ('processes', 'threads') + + def test_empty_string(self): + a, b = self.Pipe() + self.assertEqual(a.poll(), False) + b.send_bytes(b'') + self.assertEqual(a.poll(), True) + self.assertEqual(a.poll(), True) + + @classmethod + def _child_strings(cls, conn, strings): + for s in strings: + time.sleep(0.1) + conn.send_bytes(s) + conn.close() + + def test_strings(self): + strings = (b'hello', b'', b'a', b'b', b'', b'bye', b'', b'lop') + a, b = self.Pipe() + p = self.Process(target=self._child_strings, args=(b, strings)) + p.start() + + for s in strings: + for i in range(200): + if a.poll(0.01): + break + x = a.recv_bytes() + self.assertEqual(s, x) + + p.join() + + @classmethod + def _child_boundaries(cls, r): + # Polling may "pull" a message in to the child process, but we + # don't want it to pull only part of a message, as that would + # corrupt the pipe for any other processes which might later + # read from it. + r.poll(5) + + def test_boundaries(self): + r, w = self.Pipe(False) + p = self.Process(target=self._child_boundaries, args=(r,)) + p.start() + time.sleep(2) + L = [b"first", b"second"] + for obj in L: + w.send_bytes(obj) + w.close() + p.join() + self.assertIn(r.recv_bytes(), L) + + @classmethod + def _child_dont_merge(cls, b): + b.send_bytes(b'a') + b.send_bytes(b'b') + b.send_bytes(b'cd') + + def test_dont_merge(self): + a, b = self.Pipe() + self.assertEqual(a.poll(0.0), False) + self.assertEqual(a.poll(0.1), False) + + p = self.Process(target=self._child_dont_merge, args=(b,)) + p.start() + + self.assertEqual(a.recv_bytes(), b'a') + self.assertEqual(a.poll(1.0), True) + self.assertEqual(a.poll(1.0), True) + self.assertEqual(a.recv_bytes(), b'b') + self.assertEqual(a.poll(1.0), True) + self.assertEqual(a.poll(1.0), True) + self.assertEqual(a.poll(0.0), True) + self.assertEqual(a.recv_bytes(), b'cd') + + p.join() + +# +# Test of sending connection and socket objects between processes +# + +@unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction") +@hashlib_helper.requires_hashdigest('sha256') +class _TestPicklingConnections(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def tearDownClass(cls): + from multiprocessing import resource_sharer + resource_sharer.stop(timeout=support.LONG_TIMEOUT) + + @classmethod + def _listener(cls, conn, families): + for fam in families: + l = cls.connection.Listener(family=fam) + conn.send(l.address) + new_conn = l.accept() + conn.send(new_conn) + new_conn.close() + l.close() + + l = socket.create_server((socket_helper.HOST, 0)) + conn.send(l.getsockname()) + new_conn, addr = l.accept() + conn.send(new_conn) + new_conn.close() + l.close() + + conn.recv() + + @classmethod + def _remote(cls, conn): + for (address, msg) in iter(conn.recv, None): + client = cls.connection.Client(address) + client.send(msg.upper()) + client.close() + + address, msg = conn.recv() + client = socket.socket() + client.connect(address) + client.sendall(msg.upper()) + client.close() + + conn.close() + + # TODO: RUSTPYTHON - hangs + @unittest.skip("TODO: RUSTPYTHON") + def test_pickling(self): + families = self.connection.families + + lconn, lconn0 = self.Pipe() + lp = self.Process(target=self._listener, args=(lconn0, families)) + lp.daemon = True + lp.start() + lconn0.close() + + rconn, rconn0 = self.Pipe() + rp = self.Process(target=self._remote, args=(rconn0,)) + rp.daemon = True + rp.start() + rconn0.close() + + for fam in families: + msg = ('This connection uses family %s' % fam).encode('ascii') + address = lconn.recv() + rconn.send((address, msg)) + new_conn = lconn.recv() + self.assertEqual(new_conn.recv(), msg.upper()) + + rconn.send(None) + + msg = latin('This connection uses a normal socket') + address = lconn.recv() + rconn.send((address, msg)) + new_conn = lconn.recv() + buf = [] + while True: + s = new_conn.recv(100) + if not s: + break + buf.append(s) + buf = b''.join(buf) + self.assertEqual(buf, msg.upper()) + new_conn.close() + + lconn.send(None) + + rconn.close() + lconn.close() + + lp.join() + rp.join() + + @classmethod + def child_access(cls, conn): + w = conn.recv() + w.send('all is well') + w.close() + + r = conn.recv() + msg = r.recv() + conn.send(msg*2) + + conn.close() + + def test_access(self): + # On Windows, if we do not specify a destination pid when + # using DupHandle then we need to be careful to use the + # correct access flags for DuplicateHandle(), or else + # DupHandle.detach() will raise PermissionError. For example, + # for a read only pipe handle we should use + # access=FILE_GENERIC_READ. (Unfortunately + # DUPLICATE_SAME_ACCESS does not work.) + conn, child_conn = self.Pipe() + p = self.Process(target=self.child_access, args=(child_conn,)) + p.daemon = True + p.start() + child_conn.close() + + r, w = self.Pipe(duplex=False) + conn.send(w) + w.close() + self.assertEqual(r.recv(), 'all is well') + r.close() + + r, w = self.Pipe(duplex=False) + conn.send(r) + r.close() + w.send('foobar') + w.close() + self.assertEqual(conn.recv(), 'foobar'*2) + + p.join() + +# +# +# + +class _TestHeap(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + def setUp(self): + super().setUp() + # Make pristine heap for these tests + self.old_heap = multiprocessing.heap.BufferWrapper._heap + multiprocessing.heap.BufferWrapper._heap = multiprocessing.heap.Heap() + + def tearDown(self): + multiprocessing.heap.BufferWrapper._heap = self.old_heap + super().tearDown() + + def test_heap(self): + iterations = 5000 + maxblocks = 50 + blocks = [] + + # get the heap object + heap = multiprocessing.heap.BufferWrapper._heap + heap._DISCARD_FREE_SPACE_LARGER_THAN = 0 + + # create and destroy lots of blocks of different sizes + for i in range(iterations): + size = int(random.lognormvariate(0, 1) * 1000) + b = multiprocessing.heap.BufferWrapper(size) + blocks.append(b) + if len(blocks) > maxblocks: + i = random.randrange(maxblocks) + del blocks[i] + del b + + # verify the state of the heap + with heap._lock: + all = [] + free = 0 + occupied = 0 + for L in list(heap._len_to_seq.values()): + # count all free blocks in arenas + for arena, start, stop in L: + all.append((heap._arenas.index(arena), start, stop, + stop-start, 'free')) + free += (stop-start) + for arena, arena_blocks in heap._allocated_blocks.items(): + # count all allocated blocks in arenas + for start, stop in arena_blocks: + all.append((heap._arenas.index(arena), start, stop, + stop-start, 'occupied')) + occupied += (stop-start) + + self.assertEqual(free + occupied, + sum(arena.size for arena in heap._arenas)) + + all.sort() + + for i in range(len(all)-1): + (arena, start, stop) = all[i][:3] + (narena, nstart, nstop) = all[i+1][:3] + if arena != narena: + # Two different arenas + self.assertEqual(stop, heap._arenas[arena].size) # last block + self.assertEqual(nstart, 0) # first block + else: + # Same arena: two adjacent blocks + self.assertEqual(stop, nstart) + + # test free'ing all blocks + random.shuffle(blocks) + while blocks: + blocks.pop() + + self.assertEqual(heap._n_frees, heap._n_mallocs) + self.assertEqual(len(heap._pending_free_blocks), 0) + self.assertEqual(len(heap._arenas), 0) + self.assertEqual(len(heap._allocated_blocks), 0, heap._allocated_blocks) + self.assertEqual(len(heap._len_to_seq), 0) + + def test_free_from_gc(self): + # Check that freeing of blocks by the garbage collector doesn't deadlock + # (issue #12352). + # Make sure the GC is enabled, and set lower collection thresholds to + # make collections more frequent (and increase the probability of + # deadlock). + if not gc.isenabled(): + gc.enable() + self.addCleanup(gc.disable) + thresholds = gc.get_threshold() + self.addCleanup(gc.set_threshold, *thresholds) + gc.set_threshold(10) + + # perform numerous block allocations, with cyclic references to make + # sure objects are collected asynchronously by the gc + for i in range(5000): + a = multiprocessing.heap.BufferWrapper(1) + b = multiprocessing.heap.BufferWrapper(1) + # circular references + a.buddy = b + b.buddy = a + +# +# +# + +class _Foo(Structure): + _fields_ = [ + ('x', c_int), + ('y', c_double), + ('z', c_longlong,) + ] + +class _TestSharedCTypes(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + def setUp(self): + if not HAS_SHAREDCTYPES: + self.skipTest("requires multiprocessing.sharedctypes") + + @classmethod + def _double(cls, x, y, z, foo, arr, string): + x.value *= 2 + y.value *= 2 + z.value *= 2 + foo.x *= 2 + foo.y *= 2 + string.value *= 2 + for i in range(len(arr)): + arr[i] *= 2 + + # TODO: RUSTPYTHON - ctypes Structure shared memory not working + @unittest.expectedFailure + def test_sharedctypes(self, lock=False): + x = Value('i', 7, lock=lock) + y = Value(c_double, 1.0/3.0, lock=lock) + z = Value(c_longlong, 2 ** 33, lock=lock) + foo = Value(_Foo, 3, 2, lock=lock) + arr = self.Array('d', list(range(10)), lock=lock) + string = self.Array('c', 20, lock=lock) + string.value = latin('hello') + + p = self.Process(target=self._double, args=(x, y, z, foo, arr, string)) + p.daemon = True + p.start() + p.join() + + self.assertEqual(x.value, 14) + self.assertAlmostEqual(y.value, 2.0/3.0) + self.assertEqual(z.value, 2 ** 34) + self.assertEqual(foo.x, 6) + self.assertAlmostEqual(foo.y, 4.0) + for i in range(10): + self.assertAlmostEqual(arr[i], i*2) + self.assertEqual(string.value, latin('hellohello')) + + # TODO: RUSTPYTHON - calls test_sharedctypes which fails + @unittest.expectedFailure + def test_synchronize(self): + self.test_sharedctypes(lock=True) + + def test_copy(self): + foo = _Foo(2, 5.0, 2 ** 33) + bar = copy(foo) + foo.x = 0 + foo.y = 0 + foo.z = 0 + self.assertEqual(bar.x, 2) + self.assertAlmostEqual(bar.y, 5.0) + self.assertEqual(bar.z, 2 ** 33) + + +def resource_tracker_format_subtests(func): + """Run given test using both resource tracker communication formats""" + def _inner(self, *args, **kwargs): + tracker = resource_tracker._resource_tracker + for use_simple_format in False, True: + with ( + self.subTest(use_simple_format=use_simple_format), + unittest.mock.patch.object( + tracker, '_use_simple_format', use_simple_format) + ): + func(self, *args, **kwargs) + return _inner + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +@hashlib_helper.requires_hashdigest('sha256') +class _TestSharedMemory(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @staticmethod + def _attach_existing_shmem_then_write(shmem_name_or_obj, binary_data): + if isinstance(shmem_name_or_obj, str): + local_sms = shared_memory.SharedMemory(shmem_name_or_obj) + else: + local_sms = shmem_name_or_obj + local_sms.buf[:len(binary_data)] = binary_data + local_sms.close() + + def _new_shm_name(self, prefix): + # Add a PID to the name of a POSIX shared memory object to allow + # running multiprocessing tests (test_multiprocessing_fork, + # test_multiprocessing_spawn, etc) in parallel. + return prefix + str(os.getpid()) + + def test_shared_memory_name_with_embedded_null(self): + name_tsmb = self._new_shm_name('test01_null') + sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) + self.addCleanup(sms.unlink) + with self.assertRaises(ValueError): + shared_memory.SharedMemory(name_tsmb + '\0a', create=False, size=512) + if shared_memory._USE_POSIX: + orig_name = sms._name + try: + sms._name = orig_name + '\0a' + with self.assertRaises(ValueError): + sms.unlink() + finally: + sms._name = orig_name + + def test_shared_memory_basics(self): + name_tsmb = self._new_shm_name('test01_tsmb') + sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512) + self.addCleanup(sms.unlink) + + # Verify attributes are readable. + self.assertEqual(sms.name, name_tsmb) + self.assertGreaterEqual(sms.size, 512) + self.assertGreaterEqual(len(sms.buf), sms.size) + + # Verify __repr__ + self.assertIn(sms.name, str(sms)) + self.assertIn(str(sms.size), str(sms)) + + # Modify contents of shared memory segment through memoryview. + sms.buf[0] = 42 + self.assertEqual(sms.buf[0], 42) + + # Attach to existing shared memory segment. + also_sms = shared_memory.SharedMemory(name_tsmb) + self.assertEqual(also_sms.buf[0], 42) + also_sms.close() + + # Attach to existing shared memory segment but specify a new size. + same_sms = shared_memory.SharedMemory(name_tsmb, size=20*sms.size) + self.assertLess(same_sms.size, 20*sms.size) # Size was ignored. + same_sms.close() + + # Creating Shared Memory Segment with -ve size + with self.assertRaises(ValueError): + shared_memory.SharedMemory(create=True, size=-2) + + # Attaching Shared Memory Segment without a name + with self.assertRaises(ValueError): + shared_memory.SharedMemory(create=False) + + # Test if shared memory segment is created properly, + # when _make_filename returns an existing shared memory segment name + with unittest.mock.patch( + 'multiprocessing.shared_memory._make_filename') as mock_make_filename: + + NAME_PREFIX = shared_memory._SHM_NAME_PREFIX + names = [self._new_shm_name('test01_fn'), self._new_shm_name('test02_fn')] + # Prepend NAME_PREFIX which can be '/psm_' or 'wnsm_', necessary + # because some POSIX compliant systems require name to start with / + names = [NAME_PREFIX + name for name in names] + + mock_make_filename.side_effect = names + shm1 = shared_memory.SharedMemory(create=True, size=1) + self.addCleanup(shm1.unlink) + self.assertEqual(shm1._name, names[0]) + + mock_make_filename.side_effect = names + shm2 = shared_memory.SharedMemory(create=True, size=1) + self.addCleanup(shm2.unlink) + self.assertEqual(shm2._name, names[1]) + + if shared_memory._USE_POSIX: + # Posix Shared Memory can only be unlinked once. Here we + # test an implementation detail that is not observed across + # all supported platforms (since WindowsNamedSharedMemory + # manages unlinking on its own and unlink() does nothing). + # True release of shared memory segment does not necessarily + # happen until process exits, depending on the OS platform. + name_dblunlink = self._new_shm_name('test01_dblunlink') + sms_uno = shared_memory.SharedMemory( + name_dblunlink, + create=True, + size=5000 + ) + with self.assertRaises(FileNotFoundError): + try: + self.assertGreaterEqual(sms_uno.size, 5000) + + sms_duo = shared_memory.SharedMemory(name_dblunlink) + sms_duo.unlink() # First shm_unlink() call. + sms_duo.close() + sms_uno.close() + + finally: + sms_uno.unlink() # A second shm_unlink() call is bad. + + with self.assertRaises(FileExistsError): + # Attempting to create a new shared memory segment with a + # name that is already in use triggers an exception. + there_can_only_be_one_sms = shared_memory.SharedMemory( + name_tsmb, + create=True, + size=512 + ) + + if shared_memory._USE_POSIX: + # Requesting creation of a shared memory segment with the option + # to attach to an existing segment, if that name is currently in + # use, should not trigger an exception. + # Note: Using a smaller size could possibly cause truncation of + # the existing segment but is OS platform dependent. In the + # case of MacOS/darwin, requesting a smaller size is disallowed. + class OptionalAttachSharedMemory(shared_memory.SharedMemory): + _flags = os.O_CREAT | os.O_RDWR + ok_if_exists_sms = OptionalAttachSharedMemory(name_tsmb) + self.assertEqual(ok_if_exists_sms.size, sms.size) + ok_if_exists_sms.close() + + # Attempting to attach to an existing shared memory segment when + # no segment exists with the supplied name triggers an exception. + with self.assertRaises(FileNotFoundError): + nonexisting_sms = shared_memory.SharedMemory('test01_notthere') + nonexisting_sms.unlink() # Error should occur on prior line. + + sms.close() + + def test_shared_memory_recreate(self): + # Test if shared memory segment is created properly, + # when _make_filename returns an existing shared memory segment name + with unittest.mock.patch( + 'multiprocessing.shared_memory._make_filename') as mock_make_filename: + + NAME_PREFIX = shared_memory._SHM_NAME_PREFIX + names = [self._new_shm_name('test03_fn'), self._new_shm_name('test04_fn')] + # Prepend NAME_PREFIX which can be '/psm_' or 'wnsm_', necessary + # because some POSIX compliant systems require name to start with / + names = [NAME_PREFIX + name for name in names] + + mock_make_filename.side_effect = names + shm1 = shared_memory.SharedMemory(create=True, size=1) + self.addCleanup(shm1.unlink) + self.assertEqual(shm1._name, names[0]) + + mock_make_filename.side_effect = names + shm2 = shared_memory.SharedMemory(create=True, size=1) + self.addCleanup(shm2.unlink) + self.assertEqual(shm2._name, names[1]) + + def test_invalid_shared_memory_creation(self): + # Test creating a shared memory segment with negative size + with self.assertRaises(ValueError): + sms_invalid = shared_memory.SharedMemory(create=True, size=-1) + + # Test creating a shared memory segment with size 0 + with self.assertRaises(ValueError): + sms_invalid = shared_memory.SharedMemory(create=True, size=0) + + # Test creating a shared memory segment without size argument + with self.assertRaises(ValueError): + sms_invalid = shared_memory.SharedMemory(create=True) + + def test_shared_memory_pickle_unpickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + sms = shared_memory.SharedMemory(create=True, size=512) + self.addCleanup(sms.unlink) + sms.buf[0:6] = b'pickle' + + # Test pickling + pickled_sms = pickle.dumps(sms, protocol=proto) + + # Test unpickling + sms2 = pickle.loads(pickled_sms) + self.assertIsInstance(sms2, shared_memory.SharedMemory) + self.assertEqual(sms.name, sms2.name) + self.assertEqual(bytes(sms.buf[0:6]), b'pickle') + self.assertEqual(bytes(sms2.buf[0:6]), b'pickle') + + # Test that unpickled version is still the same SharedMemory + sms.buf[0:6] = b'newval' + self.assertEqual(bytes(sms.buf[0:6]), b'newval') + self.assertEqual(bytes(sms2.buf[0:6]), b'newval') + + sms2.buf[0:6] = b'oldval' + self.assertEqual(bytes(sms.buf[0:6]), b'oldval') + self.assertEqual(bytes(sms2.buf[0:6]), b'oldval') + + def test_shared_memory_pickle_unpickle_dead_object(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + sms = shared_memory.SharedMemory(create=True, size=512) + sms.buf[0:6] = b'pickle' + pickled_sms = pickle.dumps(sms, protocol=proto) + + # Now, we are going to kill the original object. + # So, unpickled one won't be able to attach to it. + sms.close() + sms.unlink() + + with self.assertRaises(FileNotFoundError): + pickle.loads(pickled_sms) + + def test_shared_memory_across_processes(self): + # bpo-40135: don't define shared memory block's name in case of + # the failure when we run multiprocessing tests in parallel. + sms = shared_memory.SharedMemory(create=True, size=512) + self.addCleanup(sms.unlink) + + # Verify remote attachment to existing block by name is working. + p = self.Process( + target=self._attach_existing_shmem_then_write, + args=(sms.name, b'howdy') + ) + p.daemon = True + p.start() + p.join() + self.assertEqual(bytes(sms.buf[:5]), b'howdy') + + # Verify pickling of SharedMemory instance also works. + p = self.Process( + target=self._attach_existing_shmem_then_write, + args=(sms, b'HELLO') + ) + p.daemon = True + p.start() + p.join() + self.assertEqual(bytes(sms.buf[:5]), b'HELLO') + + sms.close() + + @unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms") + def test_shared_memory_SharedMemoryServer_ignores_sigint(self): + # bpo-36368: protect SharedMemoryManager server process from + # KeyboardInterrupt signals. + smm = multiprocessing.managers.SharedMemoryManager() + smm.start() + + # make sure the manager works properly at the beginning + sl = smm.ShareableList(range(10)) + + # the manager's server should ignore KeyboardInterrupt signals, and + # maintain its connection with the current process, and success when + # asked to deliver memory segments. + os.kill(smm._process.pid, signal.SIGINT) + + sl2 = smm.ShareableList(range(10)) + + # test that the custom signal handler registered in the Manager does + # not affect signal handling in the parent process. + with self.assertRaises(KeyboardInterrupt): + os.kill(os.getpid(), signal.SIGINT) + + smm.shutdown() + + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests + def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): + # bpo-36867: test that a SharedMemoryManager uses the + # same resource_tracker process as its parent. + cmd = '''if 1: + from multiprocessing.managers import SharedMemoryManager + + + smm = SharedMemoryManager() + smm.start() + sl = smm.ShareableList(range(10)) + smm.shutdown() + ''' + rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd) + + # Before bpo-36867 was fixed, a SharedMemoryManager not using the same + # resource_tracker process as its parent would make the parent's + # tracker complain about sl being leaked even though smm.shutdown() + # properly released sl. + self.assertFalse(err) + + def test_shared_memory_SharedMemoryManager_basics(self): + smm1 = multiprocessing.managers.SharedMemoryManager() + with self.assertRaises(ValueError): + smm1.SharedMemory(size=9) # Fails if SharedMemoryServer not started + smm1.start() + lol = [ smm1.ShareableList(range(i)) for i in range(5, 10) ] + lom = [ smm1.SharedMemory(size=j) for j in range(32, 128, 16) ] + doppleganger_list0 = shared_memory.ShareableList(name=lol[0].shm.name) + self.assertEqual(len(doppleganger_list0), 5) + doppleganger_shm0 = shared_memory.SharedMemory(name=lom[0].name) + self.assertGreaterEqual(len(doppleganger_shm0.buf), 32) + held_name = lom[0].name + smm1.shutdown() + if sys.platform != "win32": + # Calls to unlink() have no effect on Windows platform; shared + # memory will only be released once final process exits. + with self.assertRaises(FileNotFoundError): + # No longer there to be attached to again. + absent_shm = shared_memory.SharedMemory(name=held_name) + + with multiprocessing.managers.SharedMemoryManager() as smm2: + sl = smm2.ShareableList("howdy") + shm = smm2.SharedMemory(size=128) + held_name = sl.shm.name + if sys.platform != "win32": + with self.assertRaises(FileNotFoundError): + # No longer there to be attached to again. + absent_sl = shared_memory.ShareableList(name=held_name) + + + def test_shared_memory_ShareableList_basics(self): + sl = shared_memory.ShareableList( + ['howdy', b'HoWdY', -273.154, 100, None, True, 42] + ) + self.addCleanup(sl.shm.unlink) + + # Verify __repr__ + self.assertIn(sl.shm.name, str(sl)) + self.assertIn(str(list(sl)), str(sl)) + + # Index Out of Range (get) + with self.assertRaises(IndexError): + sl[7] + + # Index Out of Range (set) + with self.assertRaises(IndexError): + sl[7] = 2 + + # Assign value without format change (str -> str) + current_format = sl._get_packing_format(0) + sl[0] = 'howdy' + self.assertEqual(current_format, sl._get_packing_format(0)) + + # Verify attributes are readable. + self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q') + + # Exercise len(). + self.assertEqual(len(sl), 7) + + # Exercise index(). + with warnings.catch_warnings(): + # Suppress BytesWarning when comparing against b'HoWdY'. + warnings.simplefilter('ignore') + with self.assertRaises(ValueError): + sl.index('100') + self.assertEqual(sl.index(100), 3) + + # Exercise retrieving individual values. + self.assertEqual(sl[0], 'howdy') + self.assertEqual(sl[-2], True) + + # Exercise iterability. + self.assertEqual( + tuple(sl), + ('howdy', b'HoWdY', -273.154, 100, None, True, 42) + ) + + # Exercise modifying individual values. + sl[3] = 42 + self.assertEqual(sl[3], 42) + sl[4] = 'some' # Change type at a given position. + self.assertEqual(sl[4], 'some') + self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q') + with self.assertRaisesRegex(ValueError, + "exceeds available storage"): + sl[4] = 'far too many' + self.assertEqual(sl[4], 'some') + sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data + self.assertEqual(sl[0], 'encodés') + self.assertEqual(sl[1], b'HoWdY') # no spillage + with self.assertRaisesRegex(ValueError, + "exceeds available storage"): + sl[0] = 'encodées' # Exactly 9 bytes of UTF-8 data + self.assertEqual(sl[1], b'HoWdY') + with self.assertRaisesRegex(ValueError, + "exceeds available storage"): + sl[1] = b'123456789' + self.assertEqual(sl[1], b'HoWdY') + + # Exercise count(). + with warnings.catch_warnings(): + # Suppress BytesWarning when comparing against b'HoWdY'. + warnings.simplefilter('ignore') + self.assertEqual(sl.count(42), 2) + self.assertEqual(sl.count(b'HoWdY'), 1) + self.assertEqual(sl.count(b'adios'), 0) + + # Exercise creating a duplicate. + name_duplicate = self._new_shm_name('test03_duplicate') + sl_copy = shared_memory.ShareableList(sl, name=name_duplicate) + try: + self.assertNotEqual(sl.shm.name, sl_copy.shm.name) + self.assertEqual(name_duplicate, sl_copy.shm.name) + self.assertEqual(list(sl), list(sl_copy)) + self.assertEqual(sl.format, sl_copy.format) + sl_copy[-1] = 77 + self.assertEqual(sl_copy[-1], 77) + self.assertNotEqual(sl[-1], 77) + sl_copy.shm.close() + finally: + sl_copy.shm.unlink() + + # Obtain a second handle on the same ShareableList. + sl_tethered = shared_memory.ShareableList(name=sl.shm.name) + self.assertEqual(sl.shm.name, sl_tethered.shm.name) + sl_tethered[-1] = 880 + self.assertEqual(sl[-1], 880) + sl_tethered.shm.close() + + sl.shm.close() + + # Exercise creating an empty ShareableList. + empty_sl = shared_memory.ShareableList() + try: + self.assertEqual(len(empty_sl), 0) + self.assertEqual(empty_sl.format, '') + self.assertEqual(empty_sl.count('any'), 0) + with self.assertRaises(ValueError): + empty_sl.index(None) + empty_sl.shm.close() + finally: + empty_sl.shm.unlink() + + def test_shared_memory_ShareableList_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + sl = shared_memory.ShareableList(range(10)) + self.addCleanup(sl.shm.unlink) + + serialized_sl = pickle.dumps(sl, protocol=proto) + deserialized_sl = pickle.loads(serialized_sl) + self.assertIsInstance( + deserialized_sl, shared_memory.ShareableList) + self.assertEqual(deserialized_sl[-1], 9) + self.assertIsNot(sl, deserialized_sl) + + deserialized_sl[4] = "changed" + self.assertEqual(sl[4], "changed") + sl[3] = "newvalue" + self.assertEqual(deserialized_sl[3], "newvalue") + + larger_sl = shared_memory.ShareableList(range(400)) + self.addCleanup(larger_sl.shm.unlink) + serialized_larger_sl = pickle.dumps(larger_sl, protocol=proto) + self.assertEqual(len(serialized_sl), len(serialized_larger_sl)) + larger_sl.shm.close() + + deserialized_sl.shm.close() + sl.shm.close() + + def test_shared_memory_ShareableList_pickling_dead_object(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + sl = shared_memory.ShareableList(range(10)) + serialized_sl = pickle.dumps(sl, protocol=proto) + + # Now, we are going to kill the original object. + # So, unpickled one won't be able to attach to it. + sl.shm.close() + sl.shm.unlink() + + with self.assertRaises(FileNotFoundError): + pickle.loads(serialized_sl) + + def test_shared_memory_cleaned_after_process_termination(self): + cmd = '''if 1: + import os, time, sys + from multiprocessing import shared_memory + + # Create a shared_memory segment, and send the segment name + sm = shared_memory.SharedMemory(create=True, size=10) + sys.stdout.write(sm.name + '\\n') + sys.stdout.flush() + time.sleep(100) + ''' + with subprocess.Popen([sys.executable, '-E', '-c', cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + name = p.stdout.readline().strip().decode() + + # killing abruptly processes holding reference to a shared memory + # segment should not leak the given memory segment. + p.terminate() + p.wait() + + err_msg = ("A SharedMemory segment was leaked after " + "a process was abruptly terminated") + for _ in support.sleeping_retry(support.LONG_TIMEOUT, err_msg): + try: + smm = shared_memory.SharedMemory(name, create=False) + except FileNotFoundError: + break + + if os.name == 'posix': + # Without this line it was raising warnings like: + # UserWarning: resource_tracker: + # There appear to be 1 leaked shared_memory + # objects to clean up at shutdown + # See: https://bugs.python.org/issue45209 + resource_tracker.unregister(f"/{name}", "shared_memory") + + # A warning was emitted by the subprocess' own + # resource_tracker (on Windows, shared memory segments + # are released automatically by the OS). + err = p.stderr.read().decode() + self.assertIn( + "resource_tracker: There appear to be 1 leaked " + "shared_memory objects to clean up at shutdown", err) + + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests + def test_shared_memory_untracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=False, it must not cause the memory to be deleted + # when terminating. + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=False) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + # The resource tracker shares pipes with the subprocess, and so + # err existing means that the tracker process has terminated now. + try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertNotIn(b"resource_tracker", err) + self.assertEqual(rc, 0) + mem2 = shared_memory.SharedMemory(create=False, name=mem.name) + mem2.close() + finally: + try: + mem.unlink() + except OSError: + pass + mem.close() + + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests + def test_shared_memory_tracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=True, it must cause the memory to be deleted when + # terminating. + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=True) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertEqual(rc, 0) + self.assertIn( + b"resource_tracker: There appear to be 1 leaked " + b"shared_memory objects to clean up at shutdown", err) + finally: + try: + mem.unlink() + except OSError: + pass + resource_tracker.unregister(mem._name, "shared_memory") + mem.close() + +# +# Test to verify that `Finalize` works. +# + +class _TestFinalize(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + def setUp(self): + self.registry_backup = util._finalizer_registry.copy() + util._finalizer_registry.clear() + + def tearDown(self): + gc.collect() # For PyPy or other GCs. + self.assertFalse(util._finalizer_registry) + util._finalizer_registry.update(self.registry_backup) + + @classmethod + def _test_finalize(cls, conn): + class Foo(object): + pass + + a = Foo() + util.Finalize(a, conn.send, args=('a',)) + del a # triggers callback for a + gc.collect() # For PyPy or other GCs. + + b = Foo() + close_b = util.Finalize(b, conn.send, args=('b',)) + close_b() # triggers callback for b + close_b() # does nothing because callback has already been called + del b # does nothing because callback has already been called + gc.collect() # For PyPy or other GCs. + + c = Foo() + util.Finalize(c, conn.send, args=('c',)) + + d10 = Foo() + util.Finalize(d10, conn.send, args=('d10',), exitpriority=1) + + d01 = Foo() + util.Finalize(d01, conn.send, args=('d01',), exitpriority=0) + d02 = Foo() + util.Finalize(d02, conn.send, args=('d02',), exitpriority=0) + d03 = Foo() + util.Finalize(d03, conn.send, args=('d03',), exitpriority=0) + + util.Finalize(None, conn.send, args=('e',), exitpriority=-10) + + util.Finalize(None, conn.send, args=('STOP',), exitpriority=-100) + + # call multiprocessing's cleanup function then exit process without + # garbage collecting locals + util._exit_function() + conn.close() + os._exit(0) + + def test_finalize(self): + conn, child_conn = self.Pipe() + + p = self.Process(target=self._test_finalize, args=(child_conn,)) + p.daemon = True + p.start() + p.join() + + result = [obj for obj in iter(conn.recv, 'STOP')] + self.assertEqual(result, ['a', 'b', 'd10', 'd03', 'd02', 'd01', 'e']) + + @support.requires_resource('cpu') + # TODO: RUSTPYTHON; dict iteration races with concurrent GC mutations + @unittest.expectedFailure + def test_thread_safety(self): + # bpo-24484: _run_finalizers() should be thread-safe + def cb(): + pass + + class Foo(object): + def __init__(self): + self.ref = self # create reference cycle + # insert finalizer at random key + util.Finalize(self, cb, exitpriority=random.randint(1, 100)) + + finish = False + exc = None + + def run_finalizers(): + nonlocal exc + while not finish: + time.sleep(random.random() * 1e-1) + try: + # A GC run will eventually happen during this, + # collecting stale Foo's and mutating the registry + util._run_finalizers() + except Exception as e: + exc = e + + def make_finalizers(): + nonlocal exc + d = {} + while not finish: + try: + # Old Foo's get gradually replaced and later + # collected by the GC (because of the cyclic ref) + d[random.getrandbits(5)] = {Foo() for i in range(10)} + except Exception as e: + exc = e + d.clear() + + old_interval = sys.getswitchinterval() + old_threshold = gc.get_threshold() + try: + support.setswitchinterval(1e-6) + gc.set_threshold(5, 5, 5) + threads = [threading.Thread(target=run_finalizers), + threading.Thread(target=make_finalizers)] + with threading_helper.start_threads(threads): + time.sleep(4.0) # Wait a bit to trigger race condition + finish = True + if exc is not None: + raise exc + finally: + sys.setswitchinterval(old_interval) + gc.set_threshold(*old_threshold) + gc.collect() # Collect remaining Foo's + + +# +# Test that from ... import * works for each module +# + +class _TestImportStar(unittest.TestCase): + + def get_module_names(self): + import glob + folder = os.path.dirname(multiprocessing.__file__) + pattern = os.path.join(glob.escape(folder), '*.py') + files = glob.glob(pattern) + modules = [os.path.splitext(os.path.split(f)[1])[0] for f in files] + modules = ['multiprocessing.' + m for m in modules] + modules.remove('multiprocessing.__init__') + modules.append('multiprocessing') + return modules + + def test_import(self): + modules = self.get_module_names() + if sys.platform == 'win32': + modules.remove('multiprocessing.popen_fork') + modules.remove('multiprocessing.popen_forkserver') + modules.remove('multiprocessing.popen_spawn_posix') + else: + modules.remove('multiprocessing.popen_spawn_win32') + if not HAS_REDUCTION: + modules.remove('multiprocessing.popen_forkserver') + + if c_int is None: + # This module requires _ctypes + modules.remove('multiprocessing.sharedctypes') + + for name in modules: + __import__(name) + mod = sys.modules[name] + self.assertTrue(hasattr(mod, '__all__'), name) + + for attr in mod.__all__: + self.assertTrue( + hasattr(mod, attr), + '%r does not have attribute %r' % (mod, attr) + ) + +# +# Quick test that logging works -- does not test logging output +# + +class _TestLogging(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + def test_enable_logging(self): + logger = multiprocessing.get_logger() + logger.setLevel(util.SUBWARNING) + self.assertTrue(logger is not None) + logger.debug('this will not be printed') + logger.info('nor will this') + logger.setLevel(LOG_LEVEL) + + @classmethod + def _test_level(cls, conn): + logger = multiprocessing.get_logger() + conn.send(logger.getEffectiveLevel()) + + def test_level(self): + LEVEL1 = 32 + LEVEL2 = 37 + + logger = multiprocessing.get_logger() + root_logger = logging.getLogger() + root_level = root_logger.level + + reader, writer = multiprocessing.Pipe(duplex=False) + + logger.setLevel(LEVEL1) + p = self.Process(target=self._test_level, args=(writer,)) + p.start() + self.assertEqual(LEVEL1, reader.recv()) + p.join() + p.close() + + logger.setLevel(logging.NOTSET) + root_logger.setLevel(LEVEL2) + p = self.Process(target=self._test_level, args=(writer,)) + p.start() + self.assertEqual(LEVEL2, reader.recv()) + p.join() + p.close() + + root_logger.setLevel(root_level) + logger.setLevel(level=LOG_LEVEL) + + def test_filename(self): + logger = multiprocessing.get_logger() + original_level = logger.level + try: + logger.setLevel(util.DEBUG) + stream = io.StringIO() + handler = logging.StreamHandler(stream) + logging_format = '[%(levelname)s] [%(filename)s] %(message)s' + handler.setFormatter(logging.Formatter(logging_format)) + logger.addHandler(handler) + logger.info('1') + util.info('2') + logger.debug('3') + filename = os.path.basename(__file__) + log_record = stream.getvalue() + self.assertIn(f'[INFO] [{filename}] 1', log_record) + self.assertIn(f'[INFO] [{filename}] 2', log_record) + self.assertIn(f'[DEBUG] [{filename}] 3', log_record) + finally: + logger.setLevel(original_level) + logger.removeHandler(handler) + handler.close() + + +# class _TestLoggingProcessName(BaseTestCase): +# +# def handle(self, record): +# assert record.processName == multiprocessing.current_process().name +# self.__handled = True +# +# def test_logging(self): +# handler = logging.Handler() +# handler.handle = self.handle +# self.__handled = False +# # Bypass getLogger() and side-effects +# logger = logging.getLoggerClass()( +# 'multiprocessing.test.TestLoggingProcessName') +# logger.addHandler(handler) +# logger.propagate = False +# +# logger.warn('foo') +# assert self.__handled + +# +# Check that Process.join() retries if os.waitpid() fails with EINTR +# + +class _TestPollEintr(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def _killer(cls, pid): + time.sleep(0.1) + os.kill(pid, signal.SIGUSR1) + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') + def test_poll_eintr(self): + got_signal = [False] + def record(*args): + got_signal[0] = True + pid = os.getpid() + oldhandler = signal.signal(signal.SIGUSR1, record) + try: + killer = self.Process(target=self._killer, args=(pid,)) + killer.start() + try: + p = self.Process(target=time.sleep, args=(2,)) + p.start() + p.join() + finally: + killer.join() + self.assertTrue(got_signal[0]) + self.assertEqual(p.exitcode, 0) + finally: + signal.signal(signal.SIGUSR1, oldhandler) + +# +# Test to verify handle verification, see issue 3321 +# + +class TestInvalidHandle(unittest.TestCase): + + @unittest.skipIf(WIN32, "skipped on Windows") + def test_invalid_handles(self): + conn = multiprocessing.connection.Connection(44977608) + # check that poll() doesn't crash + try: + conn.poll() + except (ValueError, OSError): + pass + finally: + # Hack private attribute _handle to avoid printing an error + # in conn.__del__ + conn._handle = None + self.assertRaises((ValueError, OSError), + multiprocessing.connection.Connection, -1) + + + +@hashlib_helper.requires_hashdigest('sha256') +class OtherTest(unittest.TestCase): + # TODO: add more tests for deliver/answer challenge. + def test_deliver_challenge_auth_failure(self): + class _FakeConnection(object): + def recv_bytes(self, size): + return b'something bogus' + def send_bytes(self, data): + pass + self.assertRaises(multiprocessing.AuthenticationError, + multiprocessing.connection.deliver_challenge, + _FakeConnection(), b'abc') + + def test_answer_challenge_auth_failure(self): + class _FakeConnection(object): + def __init__(self): + self.count = 0 + def recv_bytes(self, size): + self.count += 1 + if self.count == 1: + return multiprocessing.connection._CHALLENGE + elif self.count == 2: + return b'something bogus' + return b'' + def send_bytes(self, data): + pass + self.assertRaises(multiprocessing.AuthenticationError, + multiprocessing.connection.answer_challenge, + _FakeConnection(), b'abc') + + +@hashlib_helper.requires_hashdigest('md5') +@hashlib_helper.requires_hashdigest('sha256') +class ChallengeResponseTest(unittest.TestCase): + authkey = b'supadupasecretkey' + + def create_response(self, message): + return multiprocessing.connection._create_response( + self.authkey, message + ) + + def verify_challenge(self, message, response): + return multiprocessing.connection._verify_challenge( + self.authkey, message, response + ) + + def test_challengeresponse(self): + for algo in [None, "md5", "sha256"]: + with self.subTest(f"{algo=}"): + msg = b'is-twenty-bytes-long' # The length of a legacy message. + if algo: + prefix = b'{%s}' % algo.encode("ascii") + else: + prefix = b'' + msg = prefix + msg + response = self.create_response(msg) + if not response.startswith(prefix): + self.fail(response) + self.verify_challenge(msg, response) + + # TODO(gpshead): We need integration tests for handshakes between modern + # deliver_challenge() and verify_response() code and connections running a + # test-local copy of the legacy Python <=3.11 implementations. + + # TODO(gpshead): properly annotate tests for requires_hashdigest rather than + # only running these on a platform supporting everything. otherwise logic + # issues preventing it from working on FIPS mode setups will be hidden. + +# +# Test Manager.start()/Pool.__init__() initializer feature - see issue 5585 +# + +def initializer(ns): + ns.test += 1 + +@hashlib_helper.requires_hashdigest('sha256') +class TestInitializers(unittest.TestCase): + def setUp(self): + self.mgr = multiprocessing.Manager() + self.ns = self.mgr.Namespace() + self.ns.test = 0 + + def tearDown(self): + self.mgr.shutdown() + self.mgr.join() + + def test_manager_initializer(self): + m = multiprocessing.managers.SyncManager() + self.assertRaises(TypeError, m.start, 1) + m.start(initializer, (self.ns,)) + self.assertEqual(self.ns.test, 1) + m.shutdown() + m.join() + + def test_pool_initializer(self): + self.assertRaises(TypeError, multiprocessing.Pool, initializer=1) + p = multiprocessing.Pool(1, initializer, (self.ns,)) + p.close() + p.join() + self.assertEqual(self.ns.test, 1) + +# +# Issue 5155, 5313, 5331: Test process in processes +# Verifies os.close(sys.stdin.fileno) vs. sys.stdin.close() behavior +# + +def _this_sub_process(q): + try: + item = q.get(block=False) + except pyqueue.Empty: + pass + +def _test_process(): + queue = multiprocessing.Queue() + subProc = multiprocessing.Process(target=_this_sub_process, args=(queue,)) + subProc.daemon = True + subProc.start() + subProc.join() + +def _afunc(x): + return x*x + +def pool_in_process(): + pool = multiprocessing.Pool(processes=4) + x = pool.map(_afunc, [1, 2, 3, 4, 5, 6, 7]) + pool.close() + pool.join() + +class _file_like(object): + def __init__(self, delegate): + self._delegate = delegate + self._pid = None + + @property + def cache(self): + pid = os.getpid() + # There are no race conditions since fork keeps only the running thread + if pid != self._pid: + self._pid = pid + self._cache = [] + return self._cache + + def write(self, data): + self.cache.append(data) + + def flush(self): + self._delegate.write(''.join(self.cache)) + self._cache = [] + +class TestStdinBadfiledescriptor(unittest.TestCase): + + def test_queue_in_process(self): + proc = multiprocessing.Process(target=_test_process) + proc.start() + proc.join() + + def test_pool_in_process(self): + p = multiprocessing.Process(target=pool_in_process) + p.start() + p.join() + + def test_flushing(self): + sio = io.StringIO() + flike = _file_like(sio) + flike.write('foo') + proc = multiprocessing.Process(target=lambda: flike.flush()) + flike.flush() + assert sio.getvalue() == 'foo' + + +class TestWait(unittest.TestCase): + + @classmethod + def _child_test_wait(cls, w, slow): + for i in range(10): + if slow: + time.sleep(random.random() * 0.100) + w.send((i, os.getpid())) + w.close() + + def test_wait(self, slow=False): + from multiprocessing.connection import wait + readers = [] + procs = [] + messages = [] + + for i in range(4): + r, w = multiprocessing.Pipe(duplex=False) + p = multiprocessing.Process(target=self._child_test_wait, args=(w, slow)) + p.daemon = True + p.start() + w.close() + readers.append(r) + procs.append(p) + self.addCleanup(p.join) + + while readers: + for r in wait(readers): + try: + msg = r.recv() + except EOFError: + readers.remove(r) + r.close() + else: + messages.append(msg) + + messages.sort() + expected = sorted((i, p.pid) for i in range(10) for p in procs) + self.assertEqual(messages, expected) + + @classmethod + def _child_test_wait_socket(cls, address, slow): + s = socket.socket() + s.connect(address) + for i in range(10): + if slow: + time.sleep(random.random() * 0.100) + s.sendall(('%s\n' % i).encode('ascii')) + s.close() + + def test_wait_socket(self, slow=False): + from multiprocessing.connection import wait + l = socket.create_server((socket_helper.HOST, 0)) + addr = l.getsockname() + readers = [] + procs = [] + dic = {} + + for i in range(4): + p = multiprocessing.Process(target=self._child_test_wait_socket, + args=(addr, slow)) + p.daemon = True + p.start() + procs.append(p) + self.addCleanup(p.join) + + for i in range(4): + r, _ = l.accept() + readers.append(r) + dic[r] = [] + l.close() + + while readers: + for r in wait(readers): + msg = r.recv(32) + if not msg: + readers.remove(r) + r.close() + else: + dic[r].append(msg) + + expected = ''.join('%s\n' % i for i in range(10)).encode('ascii') + for v in dic.values(): + self.assertEqual(b''.join(v), expected) + + def test_wait_slow(self): + self.test_wait(True) + + def test_wait_socket_slow(self): + self.test_wait_socket(True) + + @support.requires_resource('walltime') + def test_wait_timeout(self): + from multiprocessing.connection import wait + + timeout = 5.0 # seconds + a, b = multiprocessing.Pipe() + + start = time.monotonic() + res = wait([a, b], timeout) + delta = time.monotonic() - start + + self.assertEqual(res, []) + self.assertGreater(delta, timeout - CLOCK_RES) + + b.send(None) + res = wait([a, b], 20) + self.assertEqual(res, [a]) + + @classmethod + def signal_and_sleep(cls, sem, period): + sem.release() + time.sleep(period) + + @support.requires_resource('walltime') + def test_wait_integer(self): + from multiprocessing.connection import wait + + expected = 3 + sorted_ = lambda l: sorted(l, key=lambda x: id(x)) + sem = multiprocessing.Semaphore(0) + a, b = multiprocessing.Pipe() + p = multiprocessing.Process(target=self.signal_and_sleep, + args=(sem, expected)) + + p.start() + self.assertIsInstance(p.sentinel, int) + self.assertTrue(sem.acquire(timeout=20)) + + start = time.monotonic() + res = wait([a, p.sentinel, b], expected + 20) + delta = time.monotonic() - start + + self.assertEqual(res, [p.sentinel]) + self.assertLess(delta, expected + 2) + self.assertGreater(delta, expected - 2) + + a.send(None) + + start = time.monotonic() + res = wait([a, p.sentinel, b], 20) + delta = time.monotonic() - start + + self.assertEqual(sorted_(res), sorted_([p.sentinel, b])) + self.assertLess(delta, 0.4) + + b.send(None) + + start = time.monotonic() + res = wait([a, p.sentinel, b], 20) + delta = time.monotonic() - start + + self.assertEqual(sorted_(res), sorted_([a, p.sentinel, b])) + self.assertLess(delta, 0.4) + + p.terminate() + p.join() + + def test_neg_timeout(self): + from multiprocessing.connection import wait + a, b = multiprocessing.Pipe() + t = time.monotonic() + res = wait([a], timeout=-1) + t = time.monotonic() - t + self.assertEqual(res, []) + self.assertLess(t, 1) + a.close() + b.close() + +# +# Issue 14151: Test invalid family on invalid environment +# + +class TestInvalidFamily(unittest.TestCase): + + @unittest.skipIf(WIN32, "skipped on Windows") + def test_invalid_family(self): + with self.assertRaises(ValueError): + multiprocessing.connection.Listener(r'\\.\test') + + @unittest.skipUnless(WIN32, "skipped on non-Windows platforms") + def test_invalid_family_win32(self): + with self.assertRaises(ValueError): + multiprocessing.connection.Listener('/var/test.pipe') + +# +# Issue 12098: check sys.flags of child matches that for parent +# + +class TestFlags(unittest.TestCase): + @classmethod + def run_in_grandchild(cls, conn): + conn.send(tuple(sys.flags)) + + @classmethod + def run_in_child(cls, start_method): + import json + mp = multiprocessing.get_context(start_method) + r, w = mp.Pipe(duplex=False) + p = mp.Process(target=cls.run_in_grandchild, args=(w,)) + with warnings.catch_warnings(category=DeprecationWarning): + p.start() + grandchild_flags = r.recv() + p.join() + r.close() + w.close() + flags = (tuple(sys.flags), grandchild_flags) + print(json.dumps(flags)) + + def test_flags(self): + import json + # start child process using unusual flags + prog = ( + 'from test._test_multiprocessing import TestFlags; ' + f'TestFlags.run_in_child({multiprocessing.get_start_method()!r})' + ) + data = subprocess.check_output( + [sys.executable, '-E', '-S', '-O', '-c', prog]) + child_flags, grandchild_flags = json.loads(data.decode('ascii')) + self.assertEqual(child_flags, grandchild_flags) + +# +# Test interaction with socket timeouts - see Issue #6056 +# + +class TestTimeouts(unittest.TestCase): + @classmethod + def _test_timeout(cls, child, address): + time.sleep(1) + child.send(123) + child.close() + conn = multiprocessing.connection.Client(address) + conn.send(456) + conn.close() + + def test_timeout(self): + old_timeout = socket.getdefaulttimeout() + try: + socket.setdefaulttimeout(0.1) + parent, child = multiprocessing.Pipe(duplex=True) + l = multiprocessing.connection.Listener(family='AF_INET') + p = multiprocessing.Process(target=self._test_timeout, + args=(child, l.address)) + p.start() + child.close() + self.assertEqual(parent.recv(), 123) + parent.close() + conn = l.accept() + self.assertEqual(conn.recv(), 456) + conn.close() + l.close() + join_process(p) + finally: + socket.setdefaulttimeout(old_timeout) + +# +# Test what happens with no "if __name__ == '__main__'" +# + +class TestNoForkBomb(unittest.TestCase): + def test_noforkbomb(self): + sm = multiprocessing.get_start_method() + name = os.path.join(os.path.dirname(__file__), 'mp_fork_bomb.py') + if sm != 'fork': + rc, out, err = test.support.script_helper.assert_python_failure(name, sm) + self.assertEqual(out, b'') + self.assertIn(b'RuntimeError', err) + else: + rc, out, err = test.support.script_helper.assert_python_ok(name, sm) + self.assertEqual(out.rstrip(), b'123') + self.assertEqual(err, b'') + +# +# Issue #17555: ForkAwareThreadLock +# + +class TestForkAwareThreadLock(unittest.TestCase): + # We recursively start processes. Issue #17555 meant that the + # after fork registry would get duplicate entries for the same + # lock. The size of the registry at generation n was ~2**n. + + @classmethod + def child(cls, n, conn): + if n > 1: + p = multiprocessing.Process(target=cls.child, args=(n-1, conn)) + p.start() + conn.close() + join_process(p) + else: + conn.send(len(util._afterfork_registry)) + conn.close() + + def test_lock(self): + r, w = multiprocessing.Pipe(False) + l = util.ForkAwareThreadLock() + old_size = len(util._afterfork_registry) + p = multiprocessing.Process(target=self.child, args=(5, w)) + p.start() + w.close() + new_size = r.recv() + join_process(p) + self.assertLessEqual(new_size, old_size) + +# +# Check that non-forked child processes do not inherit unneeded fds/handles +# + +class TestCloseFds(unittest.TestCase): + + def get_high_socket_fd(self): + if WIN32: + # The child process will not have any socket handles, so + # calling socket.fromfd() should produce WSAENOTSOCK even + # if there is a handle of the same number. + return socket.socket().detach() + else: + # We want to produce a socket with an fd high enough that a + # freshly created child process will not have any fds as high. + fd = socket.socket().detach() + to_close = [] + while fd < 50: + to_close.append(fd) + fd = os.dup(fd) + for x in to_close: + os.close(x) + return fd + + def close(self, fd): + if WIN32: + socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd).close() + else: + os.close(fd) + + @classmethod + def _test_closefds(cls, conn, fd): + try: + s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) + except Exception as e: + conn.send(e) + else: + s.close() + conn.send(None) + + def test_closefd(self): + if not HAS_REDUCTION: + raise unittest.SkipTest('requires fd pickling') + + reader, writer = multiprocessing.Pipe() + fd = self.get_high_socket_fd() + try: + p = multiprocessing.Process(target=self._test_closefds, + args=(writer, fd)) + p.start() + writer.close() + e = reader.recv() + join_process(p) + finally: + self.close(fd) + writer.close() + reader.close() + + if multiprocessing.get_start_method() == 'fork': + self.assertIs(e, None) + else: + WSAENOTSOCK = 10038 + self.assertIsInstance(e, OSError) + self.assertTrue(e.errno == errno.EBADF or + e.winerror == WSAENOTSOCK, e) + +# +# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc +# + +class TestIgnoreEINTR(unittest.TestCase): + + # Sending CONN_MAX_SIZE bytes into a multiprocessing pipe must block + CONN_MAX_SIZE = max(support.PIPE_MAX_SIZE, support.SOCK_MAX_SIZE) + + @classmethod + def _test_ignore(cls, conn): + def handler(signum, frame): + pass + signal.signal(signal.SIGUSR1, handler) + conn.send('ready') + x = conn.recv() + conn.send(x) + conn.send_bytes(b'x' * cls.CONN_MAX_SIZE) + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') + def test_ignore(self): + conn, child_conn = multiprocessing.Pipe() + try: + p = multiprocessing.Process(target=self._test_ignore, + args=(child_conn,)) + p.daemon = True + p.start() + child_conn.close() + self.assertEqual(conn.recv(), 'ready') + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + time.sleep(0.1) + conn.send(1234) + self.assertEqual(conn.recv(), 1234) + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + self.assertEqual(conn.recv_bytes(), b'x' * self.CONN_MAX_SIZE) + time.sleep(0.1) + p.join() + finally: + conn.close() + + @classmethod + def _test_ignore_listener(cls, conn): + def handler(signum, frame): + pass + signal.signal(signal.SIGUSR1, handler) + with multiprocessing.connection.Listener() as l: + conn.send(l.address) + a = l.accept() + a.send('welcome') + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') + def test_ignore_listener(self): + conn, child_conn = multiprocessing.Pipe() + try: + p = multiprocessing.Process(target=self._test_ignore_listener, + args=(child_conn,)) + p.daemon = True + p.start() + child_conn.close() + address = conn.recv() + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + time.sleep(0.1) + client = multiprocessing.connection.Client(address) + self.assertEqual(client.recv(), 'welcome') + p.join() + finally: + conn.close() + +class TestStartMethod(unittest.TestCase): + @classmethod + def _check_context(cls, conn): + conn.send(multiprocessing.get_start_method()) + + def check_context(self, ctx): + r, w = ctx.Pipe(duplex=False) + p = ctx.Process(target=self._check_context, args=(w,)) + p.start() + w.close() + child_method = r.recv() + r.close() + p.join() + self.assertEqual(child_method, ctx.get_start_method()) + + def test_context(self): + for method in ('fork', 'spawn', 'forkserver'): + try: + ctx = multiprocessing.get_context(method) + except ValueError: + continue + self.assertEqual(ctx.get_start_method(), method) + self.assertIs(ctx.get_context(), ctx) + self.assertRaises(ValueError, ctx.set_start_method, 'spawn') + self.assertRaises(ValueError, ctx.set_start_method, None) + self.check_context(ctx) + + def test_context_check_module_types(self): + try: + ctx = multiprocessing.get_context('forkserver') + except ValueError: + raise unittest.SkipTest('forkserver should be available') + with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'): + ctx.set_forkserver_preload([1, 2, 3]) + + def test_set_get(self): + multiprocessing.set_forkserver_preload(PRELOAD) + count = 0 + old_method = multiprocessing.get_start_method() + try: + for method in ('fork', 'spawn', 'forkserver'): + try: + multiprocessing.set_start_method(method, force=True) + except ValueError: + continue + self.assertEqual(multiprocessing.get_start_method(), method) + ctx = multiprocessing.get_context() + self.assertEqual(ctx.get_start_method(), method) + self.assertTrue(type(ctx).__name__.lower().startswith(method)) + self.assertTrue( + ctx.Process.__name__.lower().startswith(method)) + self.check_context(multiprocessing) + count += 1 + finally: + multiprocessing.set_start_method(old_method, force=True) + self.assertGreaterEqual(count, 1) + + def test_get_all(self): + methods = multiprocessing.get_all_start_methods() + if sys.platform == 'win32': + self.assertEqual(methods, ['spawn']) + else: + self.assertTrue(methods == ['fork', 'spawn'] or + methods == ['spawn', 'fork'] or + methods == ['fork', 'spawn', 'forkserver'] or + methods == ['spawn', 'fork', 'forkserver']) + + def test_preload_resources(self): + if multiprocessing.get_start_method() != 'forkserver': + self.skipTest("test only relevant for 'forkserver' method") + name = os.path.join(os.path.dirname(__file__), 'mp_preload.py') + rc, out, err = test.support.script_helper.assert_python_ok(name) + out = out.decode() + err = err.decode() + if out.rstrip() != 'ok' or err != '': + print(out) + print(err) + self.fail("failed spawning forkserver or grandchild") + + @unittest.skipIf(sys.platform == "win32", + "Only Spawn on windows so no risk of mixing") + @only_run_in_spawn_testsuite("avoids redundant testing.") + def test_mixed_startmethod(self): + # Fork-based locks cannot be used with spawned process + for process_method in ["spawn", "forkserver"]: + queue = multiprocessing.get_context("fork").Queue() + process_ctx = multiprocessing.get_context(process_method) + p = process_ctx.Process(target=close_queue, args=(queue,)) + err_msg = "A SemLock created in a fork" + with self.assertRaisesRegex(RuntimeError, err_msg): + p.start() + + # non-fork-based locks can be used with all other start methods + for queue_method in ["spawn", "forkserver"]: + for process_method in multiprocessing.get_all_start_methods(): + queue = multiprocessing.get_context(queue_method).Queue() + process_ctx = multiprocessing.get_context(process_method) + p = process_ctx.Process(target=close_queue, args=(queue,)) + p.start() + p.join() + + @classmethod + def _put_one_in_queue(cls, queue): + queue.put(1) + + @classmethod + def _put_two_and_nest_once(cls, queue): + queue.put(2) + process = multiprocessing.Process(target=cls._put_one_in_queue, args=(queue,)) + process.start() + process.join() + + def test_nested_startmethod(self): + # gh-108520: Regression test to ensure that child process can send its + # arguments to another process + queue = multiprocessing.Queue() + + process = multiprocessing.Process(target=self._put_two_and_nest_once, args=(queue,)) + process.start() + process.join() + + results = [] + while not queue.empty(): + results.append(queue.get()) + + # gh-109706: queue.put(1) can write into the queue before queue.put(2), + # there is no synchronization in the test. + self.assertSetEqual(set(results), set([2, 1])) + + +@unittest.skipIf(sys.platform == "win32", + "test semantics don't make sense on Windows") +class TestResourceTracker(unittest.TestCase): + + def test_resource_tracker(self): + # + # Check that killing process does not leak named semaphores + # + cmd = '''if 1: + import time, os + import multiprocessing as mp + from multiprocessing import resource_tracker + from multiprocessing.shared_memory import SharedMemory + + mp.set_start_method("spawn") + + + def create_and_register_resource(rtype): + if rtype == "semaphore": + lock = mp.Lock() + return lock, lock._semlock.name + elif rtype == "shared_memory": + sm = SharedMemory(create=True, size=10) + return sm, sm._name + else: + raise ValueError( + "Resource type {{}} not understood".format(rtype)) + + + resource1, rname1 = create_and_register_resource("{rtype}") + resource2, rname2 = create_and_register_resource("{rtype}") + + os.write({w}, rname1.encode("ascii") + b"\\n") + os.write({w}, rname2.encode("ascii") + b"\\n") + + time.sleep(10) + ''' + for rtype in resource_tracker._CLEANUP_FUNCS: + with self.subTest(rtype=rtype): + if rtype in ("noop", "dummy"): + # Artefact resource type used by the resource_tracker + # or tests + continue + r, w = os.pipe() + p = subprocess.Popen([sys.executable, + '-E', '-c', cmd.format(w=w, rtype=rtype)], + pass_fds=[w], + stderr=subprocess.PIPE) + os.close(w) + with open(r, 'rb', closefd=True) as f: + name1 = f.readline().rstrip().decode('ascii') + name2 = f.readline().rstrip().decode('ascii') + _resource_unlink(name1, rtype) + p.terminate() + p.wait() + + err_msg = (f"A {rtype} resource was leaked after a process was " + f"abruptly terminated") + for _ in support.sleeping_retry(support.SHORT_TIMEOUT, + err_msg): + try: + _resource_unlink(name2, rtype) + except OSError as e: + # docs say it should be ENOENT, but OSX seems to give + # EINVAL + self.assertIn(e.errno, (errno.ENOENT, errno.EINVAL)) + break + + err = p.stderr.read().decode('utf-8') + p.stderr.close() + expected = ('resource_tracker: There appear to be 2 leaked {} ' + 'objects'.format( + rtype)) + self.assertRegex(err, expected) + self.assertRegex(err, r'resource_tracker: %r: \[Errno' % name1) + + def check_resource_tracker_death(self, signum, should_die): + # bpo-31310: if the semaphore tracker process has died, it should + # be restarted implicitly. + from multiprocessing.resource_tracker import _resource_tracker + pid = _resource_tracker._pid + if pid is not None: + os.kill(pid, signal.SIGKILL) + support.wait_process(pid, exitcode=-signal.SIGKILL) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + _resource_tracker.ensure_running() + pid = _resource_tracker._pid + + os.kill(pid, signum) + time.sleep(1.0) # give it time to die + + ctx = multiprocessing.get_context("spawn") + with warnings.catch_warnings(record=True) as all_warn: + warnings.simplefilter("always") + sem = ctx.Semaphore() + sem.acquire() + sem.release() + wr = weakref.ref(sem) + # ensure `sem` gets collected, which triggers communication with + # the semaphore tracker + del sem + gc.collect() + self.assertIsNone(wr()) + if should_die: + self.assertEqual(len(all_warn), 1) + the_warn = all_warn[0] + self.assertTrue(issubclass(the_warn.category, UserWarning)) + self.assertTrue("resource_tracker: process died" + in str(the_warn.message)) + else: + self.assertEqual(len(all_warn), 0) + + def test_resource_tracker_sigint(self): + # Catchable signal (ignored by semaphore tracker) + self.check_resource_tracker_death(signal.SIGINT, False) + + def test_resource_tracker_sigterm(self): + # Catchable signal (ignored by semaphore tracker) + self.check_resource_tracker_death(signal.SIGTERM, False) + + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.") + def test_resource_tracker_sigkill(self): + # Uncatchable signal. + self.check_resource_tracker_death(signal.SIGKILL, True) + + @staticmethod + def _is_resource_tracker_reused(conn, pid): + from multiprocessing.resource_tracker import _resource_tracker + _resource_tracker.ensure_running() + # The pid should be None in the child process, expect for the fork + # context. It should not be a new value. + reused = _resource_tracker._pid in (None, pid) + reused &= _resource_tracker._check_alive() + conn.send(reused) + + def test_resource_tracker_reused(self): + from multiprocessing.resource_tracker import _resource_tracker + _resource_tracker.ensure_running() + pid = _resource_tracker._pid + + r, w = multiprocessing.Pipe(duplex=False) + p = multiprocessing.Process(target=self._is_resource_tracker_reused, + args=(w, pid)) + p.start() + is_resource_tracker_reused = r.recv() + + # Clean up + p.join() + w.close() + r.close() + + self.assertTrue(is_resource_tracker_reused) + + def test_too_long_name_resource(self): + # gh-96819: Resource names that will make the length of a write to a pipe + # greater than PIPE_BUF are not allowed + rtype = "shared_memory" + too_long_name_resource = "a" * (512 - len(rtype)) + with self.assertRaises(ValueError): + resource_tracker.register(too_long_name_resource, rtype) + + def _test_resource_tracker_leak_resources(self, cleanup): + # We use a separate instance for testing, since the main global + # _resource_tracker may be used to watch test infrastructure. + from multiprocessing.resource_tracker import ResourceTracker + tracker = ResourceTracker() + tracker.ensure_running() + self.assertTrue(tracker._check_alive()) + + self.assertIsNone(tracker._exitcode) + tracker.register('somename', 'dummy') + if cleanup: + tracker.unregister('somename', 'dummy') + expected_exit_code = 0 + else: + expected_exit_code = 1 + + self.assertTrue(tracker._check_alive()) + self.assertIsNone(tracker._exitcode) + tracker._stop() + self.assertEqual(tracker._exitcode, expected_exit_code) + + def test_resource_tracker_exit_code(self): + """ + Test the exit code of the resource tracker. + + If no leaked resources were found, exit code should be 0, otherwise 1 + """ + for cleanup in [True, False]: + with self.subTest(cleanup=cleanup): + self._test_resource_tracker_leak_resources( + cleanup=cleanup, + ) + + @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available") + def test_resource_tracker_blocked_signals(self): + # + # gh-127586: Check that resource_tracker does not override blocked signals of caller. + # + from multiprocessing.resource_tracker import ResourceTracker + orig_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) + signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1} + + try: + for sig in signals: + signal.pthread_sigmask(signal.SIG_SETMASK, {sig}) + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker = ResourceTracker() + tracker.ensure_running() + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker._stop() + finally: + # restore sigmask to what it was before executing test + signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask) + +class TestSimpleQueue(unittest.TestCase): + + @classmethod + def _test_empty(cls, queue, child_can_start, parent_can_continue): + child_can_start.wait() + # issue 30301, could fail under spawn and forkserver + try: + queue.put(queue.empty()) + queue.put(queue.empty()) + finally: + parent_can_continue.set() + + def test_empty_exceptions(self): + # Assert that checking emptiness of a closed queue raises + # an OSError, independently of whether the queue was used + # or not. This differs from Queue and JoinableQueue. + q = multiprocessing.SimpleQueue() + q.close() # close the pipe + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + + def test_empty(self): + queue = multiprocessing.SimpleQueue() + child_can_start = multiprocessing.Event() + parent_can_continue = multiprocessing.Event() + + proc = multiprocessing.Process( + target=self._test_empty, + args=(queue, child_can_start, parent_can_continue) + ) + proc.daemon = True + proc.start() + + self.assertTrue(queue.empty()) + + child_can_start.set() + parent_can_continue.wait() + + self.assertFalse(queue.empty()) + self.assertEqual(queue.get(), True) + self.assertEqual(queue.get(), False) + self.assertTrue(queue.empty()) + + proc.join() + + def test_close(self): + queue = multiprocessing.SimpleQueue() + queue.close() + # closing a queue twice should not fail + queue.close() + + # Test specific to CPython since it tests private attributes + @test.support.cpython_only + def test_closed(self): + queue = multiprocessing.SimpleQueue() + queue.close() + self.assertTrue(queue._reader.closed) + self.assertTrue(queue._writer.closed) + + +class TestPoolNotLeakOnFailure(unittest.TestCase): + + def test_release_unused_processes(self): + # Issue #19675: During pool creation, if we can't create a process, + # don't leak already created ones. + will_fail_in = 3 + forked_processes = [] + + class FailingForkProcess: + def __init__(self, **kwargs): + self.name = 'Fake Process' + self.exitcode = None + self.state = None + forked_processes.append(self) + + def start(self): + nonlocal will_fail_in + if will_fail_in <= 0: + raise OSError("Manually induced OSError") + will_fail_in -= 1 + self.state = 'started' + + def terminate(self): + self.state = 'stopping' + + def join(self): + if self.state == 'stopping': + self.state = 'stopped' + + def is_alive(self): + return self.state == 'started' or self.state == 'stopping' + + with self.assertRaisesRegex(OSError, 'Manually induced OSError'): + p = multiprocessing.pool.Pool(5, context=unittest.mock.MagicMock( + Process=FailingForkProcess)) + p.close() + p.join() + self.assertFalse( + any(process.is_alive() for process in forked_processes)) + + +@hashlib_helper.requires_hashdigest('sha256') +class TestSyncManagerTypes(unittest.TestCase): + """Test all the types which can be shared between a parent and a + child process by using a manager which acts as an intermediary + between them. + + In the following unit-tests the base type is created in the parent + process, the @classmethod represents the worker process and the + shared object is readable and editable between the two. + + # The child. + @classmethod + def _test_list(cls, obj): + assert obj[0] == 5 + assert obj.append(6) + + # The parent. + def test_list(self): + o = self.manager.list() + o.append(5) + self.run_worker(self._test_list, o) + assert o[1] == 6 + """ + manager_class = multiprocessing.managers.SyncManager + + def setUp(self): + self.manager = self.manager_class() + self.manager.start() + self.proc = None + + def tearDown(self): + if self.proc is not None and self.proc.is_alive(): + self.proc.terminate() + self.proc.join() + self.manager.shutdown() + self.manager = None + self.proc = None + + @classmethod + def setUpClass(cls): + support.reap_children() + + tearDownClass = setUpClass + + def wait_proc_exit(self): + # Only the manager process should be returned by active_children() + # but this can take a bit on slow machines, so wait a few seconds + # if there are other children too (see #17395). + join_process(self.proc) + + timeout = WAIT_ACTIVE_CHILDREN_TIMEOUT + start_time = time.monotonic() + for _ in support.sleeping_retry(timeout, error=False): + if len(multiprocessing.active_children()) <= 1: + break + else: + dt = time.monotonic() - start_time + support.environment_altered = True + support.print_warning(f"multiprocessing.Manager still has " + f"{multiprocessing.active_children()} " + f"active children after {dt:.1f} seconds") + + def run_worker(self, worker, obj): + self.proc = multiprocessing.Process(target=worker, args=(obj, )) + self.proc.daemon = True + self.proc.start() + self.wait_proc_exit() + self.assertEqual(self.proc.exitcode, 0) + + @classmethod + def _test_event(cls, obj): + assert obj.is_set() + obj.wait() + obj.clear() + obj.wait(0.001) + + def test_event(self): + o = self.manager.Event() + o.set() + self.run_worker(self._test_event, o) + assert not o.is_set() + o.wait(0.001) + + @classmethod + def _test_lock(cls, obj): + obj.acquire() + + def test_lock(self, lname="Lock"): + o = getattr(self.manager, lname)() + self.run_worker(self._test_lock, o) + o.release() + self.assertRaises(RuntimeError, o.release) # already released + + @classmethod + def _test_rlock(cls, obj): + obj.acquire() + obj.release() + + def test_rlock(self, lname="Lock"): + o = getattr(self.manager, lname)() + self.run_worker(self._test_rlock, o) + + @classmethod + def _test_semaphore(cls, obj): + obj.acquire() + + def test_semaphore(self, sname="Semaphore"): + o = getattr(self.manager, sname)() + self.run_worker(self._test_semaphore, o) + o.release() + + def test_bounded_semaphore(self): + self.test_semaphore(sname="BoundedSemaphore") + + @classmethod + def _test_condition(cls, obj): + obj.acquire() + obj.release() + + def test_condition(self): + o = self.manager.Condition() + self.run_worker(self._test_condition, o) + + @classmethod + def _test_barrier(cls, obj): + assert obj.parties == 5 + obj.reset() + + def test_barrier(self): + o = self.manager.Barrier(5) + self.run_worker(self._test_barrier, o) + + @classmethod + def _test_pool(cls, obj): + # TODO: fix https://bugs.python.org/issue35919 + with obj: + pass + + def test_pool(self): + o = self.manager.Pool(processes=4) + self.run_worker(self._test_pool, o) + + @classmethod + def _test_queue(cls, obj): + assert obj.qsize() == 2 + assert obj.full() + assert not obj.empty() + assert obj.get() == 5 + assert not obj.empty() + assert obj.get() == 6 + assert obj.empty() + + def test_queue(self, qname="Queue"): + o = getattr(self.manager, qname)(2) + o.put(5) + o.put(6) + self.run_worker(self._test_queue, o) + assert o.empty() + assert not o.full() + + def test_joinable_queue(self): + self.test_queue("JoinableQueue") + + @classmethod + def _test_list(cls, obj): + case = unittest.TestCase() + case.assertEqual(obj[0], 5) + case.assertEqual(obj.count(5), 1) + case.assertEqual(obj.index(5), 0) + obj.sort() + obj.reverse() + for x in obj: + pass + case.assertEqual(len(obj), 1) + case.assertEqual(obj.pop(0), 5) + + def test_list(self): + o = self.manager.list() + o.append(5) + self.run_worker(self._test_list, o) + self.assertIsNotNone(o) + self.assertEqual(len(o), 0) + + @classmethod + def _test_dict(cls, obj): + case = unittest.TestCase() + case.assertEqual(len(obj), 1) + case.assertEqual(obj['foo'], 5) + case.assertEqual(obj.get('foo'), 5) + case.assertListEqual(list(obj.items()), [('foo', 5)]) + case.assertListEqual(list(obj.keys()), ['foo']) + case.assertListEqual(list(obj.values()), [5]) + case.assertDictEqual(obj.copy(), {'foo': 5}) + case.assertTupleEqual(obj.popitem(), ('foo', 5)) + + def test_dict(self): + o = self.manager.dict() + o['foo'] = 5 + self.run_worker(self._test_dict, o) + self.assertIsNotNone(o) + self.assertEqual(len(o), 0) + + @classmethod + def _test_value(cls, obj): + case = unittest.TestCase() + case.assertEqual(obj.value, 1) + case.assertEqual(obj.get(), 1) + obj.set(2) + + def test_value(self): + o = self.manager.Value('i', 1) + self.run_worker(self._test_value, o) + self.assertEqual(o.value, 2) + self.assertEqual(o.get(), 2) + + @classmethod + def _test_array(cls, obj): + case = unittest.TestCase() + case.assertEqual(obj[0], 0) + case.assertEqual(obj[1], 1) + case.assertEqual(len(obj), 2) + case.assertListEqual(list(obj), [0, 1]) + + def test_array(self): + o = self.manager.Array('i', [0, 1]) + self.run_worker(self._test_array, o) + + @classmethod + def _test_namespace(cls, obj): + case = unittest.TestCase() + case.assertEqual(obj.x, 0) + case.assertEqual(obj.y, 1) + + def test_namespace(self): + o = self.manager.Namespace() + o.x = 0 + o.y = 1 + self.run_worker(self._test_namespace, o) + + +class TestNamedResource(unittest.TestCase): + @only_run_in_spawn_testsuite("spawn specific test.") + def test_global_named_resource_spawn(self): + # + # gh-90549: Check that global named resources in main module + # will not leak by a subprocess, in spawn context. + # + testfn = os_helper.TESTFN + self.addCleanup(os_helper.unlink, testfn) + with open(testfn, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + import multiprocessing as mp + ctx = mp.get_context('spawn') + global_resource = ctx.Semaphore() + def submain(): pass + if __name__ == '__main__': + p = ctx.Process(target=submain) + p.start() + p.join() + ''')) + rc, out, err = script_helper.assert_python_ok(testfn) + # on error, err = 'UserWarning: resource_tracker: There appear to + # be 1 leaked semaphore objects to clean up at shutdown' + self.assertFalse(err, msg=err.decode('utf-8')) + + +class _TestAtExit(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def _write_file_at_exit(self, output_path): + import atexit + def exit_handler(): + with open(output_path, 'w') as f: + f.write("deadbeef") + atexit.register(exit_handler) + + def test_atexit(self): + # gh-83856 + with os_helper.temp_dir() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + p = self.Process(target=self._write_file_at_exit, args=(output_path,)) + p.start() + p.join() + with open(output_path) as f: + self.assertEqual(f.read(), 'deadbeef') + + +class _TestSpawnedSysPath(BaseTestCase): + """Test that sys.path is setup in forkserver and spawn processes.""" + + ALLOWED_TYPES = {'processes'} + # Not applicable to fork which inherits everything from the process as is. + START_METHODS = {"forkserver", "spawn"} + + def setUp(self): + self._orig_sys_path = list(sys.path) + self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-") + self._mod_name = "unique_test_mod" + module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py") + with open(module_path, "w", encoding="utf-8") as mod: + mod.write("# A simple test module\n") + sys.path[:] = [p for p in sys.path if p] # remove any existing ""s + sys.path.insert(0, self._temp_dir) + sys.path.insert(0, "") # Replaced with an abspath in child. + self.assertIn(self.start_method, self.START_METHODS) + self._ctx = multiprocessing.get_context(self.start_method) + + def tearDown(self): + sys.path[:] = self._orig_sys_path + shutil.rmtree(self._temp_dir, ignore_errors=True) + + @staticmethod + def enq_imported_module_names(queue): + queue.put(tuple(sys.modules)) + + def test_forkserver_preload_imports_sys_path(self): + if self._ctx.get_start_method() != "forkserver": + self.skipTest("forkserver specific test.") + self.assertNotIn(self._mod_name, sys.modules) + multiprocessing.forkserver._forkserver._stop() # Must be fresh. + self._ctx.set_forkserver_preload( + ["test.test_multiprocessing_forkserver", self._mod_name]) + q = self._ctx.Queue() + proc = self._ctx.Process( + target=self.enq_imported_module_names, args=(q,)) + proc.start() + proc.join() + child_imported_modules = q.get() + q.close() + self.assertIn(self._mod_name, child_imported_modules) + + @staticmethod + def enq_sys_path_and_import(queue, mod_name): + queue.put(sys.path) + try: + importlib.import_module(mod_name) + except ImportError as exc: + queue.put(exc) + else: + queue.put(None) + + def test_child_sys_path(self): + q = self._ctx.Queue() + proc = self._ctx.Process( + target=self.enq_sys_path_and_import, args=(q, self._mod_name)) + proc.start() + proc.join() + child_sys_path = q.get() + import_error = q.get() + q.close() + self.assertNotIn("", child_sys_path) # replaced by an abspath + self.assertIn(self._temp_dir, child_sys_path) # our addition + # ignore the first element, it is the absolute "" replacement + self.assertEqual(child_sys_path[1:], sys.path[1:]) + self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}") + + def test_std_streams_flushed_after_preload(self): + # gh-135335: Check fork server flushes standard streams after + # preloading modules + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_flush.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + + # Check stderr first, as it is more likely to be useful to see in the + # event of a failure. + self.assertEqual(err.decode().rstrip(), '__main____mp_main__') + self.assertEqual(out.decode().rstrip(), '__main____mp_main__') + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + # Just make sure names in not_exported are excluded + support.check__all__(self, multiprocessing, extra=multiprocessing.__all__, + not_exported=['SUBDEBUG', 'SUBWARNING']) + + @only_run_in_spawn_testsuite("avoids redundant testing.") + def test_spawn_sys_executable_none_allows_import(self): + # Regression test for a bug introduced in + # https://github.com/python/cpython/issues/90876 that caused an + # ImportError in multiprocessing when sys.executable was None. + # This can be true in embedded environments. + rc, out, err = script_helper.assert_python_ok( + "-c", + """if 1: + import sys + sys.executable = None + assert "multiprocessing" not in sys.modules, "already imported!" + import multiprocessing + import multiprocessing.spawn # This should not fail\n""", + ) + self.assertEqual(rc, 0) + self.assertFalse(err, msg=err.decode('utf-8')) + + def test_large_pool(self): + # + # gh-89240: Check that large pools are always okay + # + testfn = os_helper.TESTFN + self.addCleanup(os_helper.unlink, testfn) + with open(testfn, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + import multiprocessing + def f(x): return x*x + if __name__ == '__main__': + with multiprocessing.Pool(200) as p: + print(sum(p.map(f, range(1000)))) + ''')) + rc, out, err = script_helper.assert_python_ok(testfn) + self.assertEqual("332833500", out.decode('utf-8').strip()) + self.assertFalse(err, msg=err.decode('utf-8')) + + def test_forked_thread_not_started(self): + # gh-134381: Ensure that a thread that has not been started yet in + # the parent process can be started within a forked child process. + + if multiprocessing.get_start_method() != "fork": + self.skipTest("fork specific test") + + q = multiprocessing.Queue() + t = threading.Thread(target=lambda: q.put("done"), daemon=True) + + def child(): + t.start() + t.join() + + p = multiprocessing.Process(target=child) + p.start() + p.join(support.SHORT_TIMEOUT) + + self.assertEqual(p.exitcode, 0) + self.assertEqual(q.get_nowait(), "done") + close_queue(q) + + def test_preload_main(self): + # gh-126631: Check that __main__ can be pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_main.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + self.assertEqual(err, b'') + + # The trailing empty string comes from split() on output ending with \n + out = out.decode().split("\n") + self.assertEqual(out, ['__main__', '__mp_main__', 'f', 'f', '']) + +# +# Mixins +# + +class BaseMixin(object): + @classmethod + def setUpClass(cls): + cls.dangling = (multiprocessing.process._dangling.copy(), + threading._dangling.copy()) + + @classmethod + def tearDownClass(cls): + # bpo-26762: Some multiprocessing objects like Pool create reference + # cycles. Trigger a garbage collection to break these cycles. + test.support.gc_collect() + + # TODO: RUSTPYTHON: Filter out stopped processes since gc.collect() is a no-op + processes = {p for p in multiprocessing.process._dangling if p.is_alive()} - {p for p in cls.dangling[0] if p.is_alive()} + if processes: + test.support.environment_altered = True + support.print_warning(f'Dangling processes: {processes}') + processes = None + + # TODO: RUSTPYTHON: Filter out stopped threads since gc.collect() is a no-op + threads = {t for t in threading._dangling if t.is_alive()} - {t for t in cls.dangling[1] if t.is_alive()} + if threads: + test.support.environment_altered = True + support.print_warning(f'Dangling threads: {threads}') + threads = None + + +class ProcessesMixin(BaseMixin): + TYPE = 'processes' + Process = multiprocessing.Process + connection = multiprocessing.connection + current_process = staticmethod(multiprocessing.current_process) + parent_process = staticmethod(multiprocessing.parent_process) + active_children = staticmethod(multiprocessing.active_children) + set_executable = staticmethod(multiprocessing.set_executable) + Pool = staticmethod(multiprocessing.Pool) + Pipe = staticmethod(multiprocessing.Pipe) + Queue = staticmethod(multiprocessing.Queue) + JoinableQueue = staticmethod(multiprocessing.JoinableQueue) + Lock = staticmethod(multiprocessing.Lock) + RLock = staticmethod(multiprocessing.RLock) + Semaphore = staticmethod(multiprocessing.Semaphore) + BoundedSemaphore = staticmethod(multiprocessing.BoundedSemaphore) + Condition = staticmethod(multiprocessing.Condition) + Event = staticmethod(multiprocessing.Event) + Barrier = staticmethod(multiprocessing.Barrier) + Value = staticmethod(multiprocessing.Value) + Array = staticmethod(multiprocessing.Array) + RawValue = staticmethod(multiprocessing.RawValue) + RawArray = staticmethod(multiprocessing.RawArray) + + +class ManagerMixin(BaseMixin): + TYPE = 'manager' + Process = multiprocessing.Process + Queue = property(operator.attrgetter('manager.Queue')) + JoinableQueue = property(operator.attrgetter('manager.JoinableQueue')) + Lock = property(operator.attrgetter('manager.Lock')) + RLock = property(operator.attrgetter('manager.RLock')) + Semaphore = property(operator.attrgetter('manager.Semaphore')) + BoundedSemaphore = property(operator.attrgetter('manager.BoundedSemaphore')) + Condition = property(operator.attrgetter('manager.Condition')) + Event = property(operator.attrgetter('manager.Event')) + Barrier = property(operator.attrgetter('manager.Barrier')) + Value = property(operator.attrgetter('manager.Value')) + Array = property(operator.attrgetter('manager.Array')) + list = property(operator.attrgetter('manager.list')) + dict = property(operator.attrgetter('manager.dict')) + Namespace = property(operator.attrgetter('manager.Namespace')) + + @classmethod + def Pool(cls, *args, **kwds): + return cls.manager.Pool(*args, **kwds) + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.manager = multiprocessing.Manager() + + @classmethod + def tearDownClass(cls): + # only the manager process should be returned by active_children() + # but this can take a bit on slow machines, so wait a few seconds + # if there are other children too (see #17395) + timeout = WAIT_ACTIVE_CHILDREN_TIMEOUT + start_time = time.monotonic() + for _ in support.sleeping_retry(timeout, error=False): + if len(multiprocessing.active_children()) <= 1: + break + else: + dt = time.monotonic() - start_time + support.environment_altered = True + support.print_warning(f"multiprocessing.Manager still has " + f"{multiprocessing.active_children()} " + f"active children after {dt:.1f} seconds") + + gc.collect() # do garbage collection + if cls.manager._number_of_objects() != 0: + # This is not really an error since some tests do not + # ensure that all processes which hold a reference to a + # managed object have been joined. + test.support.environment_altered = True + support.print_warning('Shared objects which still exist ' + 'at manager shutdown:') + support.print_warning(cls.manager._debug_info()) + cls.manager.shutdown() + cls.manager.join() + cls.manager = None + + super().tearDownClass() + + +class ThreadsMixin(BaseMixin): + TYPE = 'threads' + Process = multiprocessing.dummy.Process + connection = multiprocessing.dummy.connection + current_process = staticmethod(multiprocessing.dummy.current_process) + active_children = staticmethod(multiprocessing.dummy.active_children) + Pool = staticmethod(multiprocessing.dummy.Pool) + Pipe = staticmethod(multiprocessing.dummy.Pipe) + Queue = staticmethod(multiprocessing.dummy.Queue) + JoinableQueue = staticmethod(multiprocessing.dummy.JoinableQueue) + Lock = staticmethod(multiprocessing.dummy.Lock) + RLock = staticmethod(multiprocessing.dummy.RLock) + Semaphore = staticmethod(multiprocessing.dummy.Semaphore) + BoundedSemaphore = staticmethod(multiprocessing.dummy.BoundedSemaphore) + Condition = staticmethod(multiprocessing.dummy.Condition) + Event = staticmethod(multiprocessing.dummy.Event) + Barrier = staticmethod(multiprocessing.dummy.Barrier) + Value = staticmethod(multiprocessing.dummy.Value) + Array = staticmethod(multiprocessing.dummy.Array) + +# +# Functions used to create test cases from the base ones in this module +# + +def install_tests_in_module_dict(remote_globs, start_method, + only_type=None, exclude_types=False): + __module__ = remote_globs['__name__'] + local_globs = globals() + ALL_TYPES = {'processes', 'threads', 'manager'} + + for name, base in local_globs.items(): + if not isinstance(base, type): + continue + if issubclass(base, BaseTestCase): + if base is BaseTestCase: + continue + assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES + if base.START_METHODS and start_method not in base.START_METHODS: + continue # class not intended for this start method. + for type_ in base.ALLOWED_TYPES: + if only_type and type_ != only_type: + continue + if exclude_types: + continue + newname = 'With' + type_.capitalize() + name[1:] + Mixin = local_globs[type_.capitalize() + 'Mixin'] + class Temp(base, Mixin, unittest.TestCase): + pass + if type_ == 'manager': + Temp = hashlib_helper.requires_hashdigest('sha256')(Temp) + Temp.__name__ = Temp.__qualname__ = newname + Temp.__module__ = __module__ + Temp.start_method = start_method + remote_globs[newname] = Temp + elif issubclass(base, unittest.TestCase): + if only_type: + continue + + class Temp(base, object): + pass + Temp.__name__ = Temp.__qualname__ = name + Temp.__module__ = __module__ + remote_globs[name] = Temp + + dangling = [None, None] + old_start_method = [None] + + def setUpModule(): + multiprocessing.set_forkserver_preload(PRELOAD) + multiprocessing.process._cleanup() + dangling[0] = multiprocessing.process._dangling.copy() + dangling[1] = threading._dangling.copy() + old_start_method[0] = multiprocessing.get_start_method(allow_none=True) + try: + multiprocessing.set_start_method(start_method, force=True) + except ValueError: + raise unittest.SkipTest(start_method + + ' start method not supported') + + if sys.platform.startswith("linux"): + try: + lock = multiprocessing.RLock() + except OSError: + raise unittest.SkipTest("OSError raises on RLock creation, " + "see issue 3111!") + check_enough_semaphores() + util.get_temp_dir() # creates temp directory + multiprocessing.get_logger().setLevel(LOG_LEVEL) + + def tearDownModule(): + need_sleep = False + + # bpo-26762: Some multiprocessing objects like Pool create reference + # cycles. Trigger a garbage collection to break these cycles. + test.support.gc_collect() + + multiprocessing.set_start_method(old_start_method[0], force=True) + # pause a bit so we don't get warning about dangling threads/processes + # TODO: RUSTPYTHON: Filter out stopped processes since gc.collect() is a no-op + processes = {p for p in multiprocessing.process._dangling if p.is_alive()} - {p for p in dangling[0] if p.is_alive()} + if processes: + need_sleep = True + test.support.environment_altered = True + support.print_warning(f'Dangling processes: {processes}') + processes = None + + # TODO: RUSTPYTHON: Filter out stopped threads since gc.collect() is a no-op + threads = {t for t in threading._dangling if t.is_alive()} - {t for t in dangling[1] if t.is_alive()} + if threads: + need_sleep = True + test.support.environment_altered = True + support.print_warning(f'Dangling threads: {threads}') + threads = None + + # Sleep 500 ms to give time to child processes to complete. + if need_sleep: + time.sleep(0.5) + + multiprocessing.util._cleanup_tests() + + remote_globs['setUpModule'] = setUpModule + remote_globs['tearDownModule'] = tearDownModule + + +@unittest.skipIf(not hasattr(_multiprocessing, 'SemLock'), 'SemLock not available') +@unittest.skipIf(sys.platform != "linux", "Linux only") +class SemLockTests(unittest.TestCase): + + def test_semlock_subclass(self): + class SemLock(_multiprocessing.SemLock): + pass + name = f'test_semlock_subclass-{os.getpid()}' + s = SemLock(1, 0, 10, name, False) + _multiprocessing.sem_unlink(name) + + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +class TestSharedMemoryNames(unittest.TestCase): + @subTests('use_simple_format', (True, False)) + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors( + self, use_simple_format): + # Test script that creates and cleans up shared memory with colon in name + test_script = textwrap.dedent(""" + import sys + from multiprocessing import shared_memory + from multiprocessing import resource_tracker + import time + + resource_tracker._resource_tracker._use_simple_format = %s + + # Test various patterns of colons in names + test_names = [ + "a:b", + "a:b:c", + "test:name:with:many:colons", + ":starts:with:colon", + "ends:with:colon:", + "::double::colons::", + "name\\nwithnewline", + "name-with-trailing-newline\\n", + "\\nname-starts-with-newline", + "colons:and\\nnewlines:mix", + "multi\\nline\\nname", + ] + + for name in test_names: + try: + shm = shared_memory.SharedMemory(create=True, size=100, name=name) + shm.buf[:5] = b'hello' # Write something to the shared memory + shm.close() + shm.unlink() + + except Exception as e: + print(f"Error with name '{name}': {e}", file=sys.stderr) + sys.exit(1) + + print("SUCCESS") + """ % use_simple_format) + + rc, out, err = script_helper.assert_python_ok("-c", test_script) + self.assertIn(b"SUCCESS", out) + self.assertNotIn(b"traceback", err.lower(), err) + self.assertNotIn(b"resource_tracker.py", err, err) diff --git a/Lib/test/_test_venv_multiprocessing.py b/Lib/test/_test_venv_multiprocessing.py new file mode 100644 index 00000000000..ad985dd8d56 --- /dev/null +++ b/Lib/test/_test_venv_multiprocessing.py @@ -0,0 +1,40 @@ +import multiprocessing +import random +import sys + +def fill_queue(queue, code): + queue.put(code) + + +def drain_queue(queue, code): + if code != queue.get(): + sys.exit(1) + + +def test_func(): + code = random.randrange(0, 1000) + queue = multiprocessing.Queue() + fill_pool = multiprocessing.Process( + target=fill_queue, + args=(queue, code) + ) + drain_pool = multiprocessing.Process( + target=drain_queue, + args=(queue, code) + ) + drain_pool.start() + fill_pool.start() + fill_pool.join() + drain_pool.join() + + +def main(): + multiprocessing.set_start_method('spawn') + test_pool = multiprocessing.Process(target=test_func) + test_pool.start() + test_pool.join() + sys.exit(test_pool.exitcode) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/_typed_dict_helper.py b/Lib/test/_typed_dict_helper.py deleted file mode 100644 index d333db19318..00000000000 --- a/Lib/test/_typed_dict_helper.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Used to test `get_type_hints()` on a cross-module inherited `TypedDict` class - -This script uses future annotations to postpone a type that won't be available -on the module inheriting from to `Foo`. The subclass in the other module should -look something like this: - - class Bar(_typed_dict_helper.Foo, total=False): - b: int -""" - -from __future__ import annotations - -from typing import Optional, TypedDict - -OptionalIntType = Optional[int] - -class Foo(TypedDict): - a: OptionalIntType diff --git a/Lib/test/ann_module.py b/Lib/test/ann_module.py deleted file mode 100644 index 5081e6b5834..00000000000 --- a/Lib/test/ann_module.py +++ /dev/null @@ -1,62 +0,0 @@ - - -""" -The module for testing variable annotations. -Empty lines above are for good reason (testing for correct line numbers) -""" - -from typing import Optional -from functools import wraps - -__annotations__[1] = 2 - -class C: - - x = 5; y: Optional['C'] = None - -from typing import Tuple -x: int = 5; y: str = x; f: Tuple[int, int] - -class M(type): - - __annotations__['123'] = 123 - o: type = object - -(pars): bool = True - -class D(C): - j: str = 'hi'; k: str= 'bye' - -from types import new_class -h_class = new_class('H', (C,)) -j_class = new_class('J') - -class F(): - z: int = 5 - def __init__(self, x): - pass - -class Y(F): - def __init__(self): - super(F, self).__init__(123) - -class Meta(type): - def __new__(meta, name, bases, namespace): - return super().__new__(meta, name, bases, namespace) - -class S(metaclass = Meta): - x: str = 'something' - y: str = 'something else' - -def foo(x: int = 10): - def bar(y: List[str]): - x: str = 'yes' - bar() - -def dec(func): - @wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - return wrapper - -u: int | float diff --git a/Lib/test/ann_module2.py b/Lib/test/ann_module2.py deleted file mode 100644 index 76cf5b3ad97..00000000000 --- a/Lib/test/ann_module2.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Some correct syntax for variable annotation here. -More examples are in test_grammar and test_parser. -""" - -from typing import no_type_check, ClassVar - -i: int = 1 -j: int -x: float = i/10 - -def f(): - class C: ... - return C() - -f().new_attr: object = object() - -class C: - def __init__(self, x: int) -> None: - self.x = x - -c = C(5) -c.new_attr: int = 10 - -__annotations__ = {} - - -@no_type_check -class NTC: - def meth(self, param: complex) -> None: - ... - -class CV: - var: ClassVar['CV'] - -CV.var = CV() diff --git a/Lib/test/ann_module3.py b/Lib/test/ann_module3.py deleted file mode 100644 index eccd7be22dd..00000000000 --- a/Lib/test/ann_module3.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Correct syntax for variable annotation that should fail at runtime -in a certain manner. More examples are in test_grammar and test_parser. -""" - -def f_bad_ann(): - __annotations__[1] = 2 - -class C_OK: - def __init__(self, x: int) -> None: - self.x: no_such_name = x # This one is OK as proposed by Guido - -class D_bad_ann: - def __init__(self, x: int) -> None: - sfel.y: int = 0 - -def g_bad_ann(): - no_such_name.attr: int = 0 diff --git a/Lib/test/ann_module4.py b/Lib/test/ann_module4.py deleted file mode 100644 index 13e9aee54c9..00000000000 --- a/Lib/test/ann_module4.py +++ /dev/null @@ -1,5 +0,0 @@ -# This ann_module isn't for test_typing, -# it's for test_module - -a:int=3 -b:str=4 diff --git a/Lib/test/ann_module5.py b/Lib/test/ann_module5.py deleted file mode 100644 index 837041e121f..00000000000 --- a/Lib/test/ann_module5.py +++ /dev/null @@ -1,10 +0,0 @@ -# Used by test_typing to verify that Final wrapped in ForwardRef works. - -from __future__ import annotations - -from typing import Final - -name: Final[str] = "final" - -class MyClass: - value: Final = 3000 diff --git a/Lib/test/ann_module6.py b/Lib/test/ann_module6.py deleted file mode 100644 index 679175669bc..00000000000 --- a/Lib/test/ann_module6.py +++ /dev/null @@ -1,7 +0,0 @@ -# Tests that top-level ClassVar is not allowed - -from __future__ import annotations - -from typing import ClassVar - -wrong: ClassVar[int] = 1 diff --git a/Lib/test/ann_module7.py b/Lib/test/ann_module7.py deleted file mode 100644 index 8f890cd2802..00000000000 --- a/Lib/test/ann_module7.py +++ /dev/null @@ -1,11 +0,0 @@ -# Tests class have ``__text_signature__`` - -from __future__ import annotations - -DEFAULT_BUFFER_SIZE = 8192 - -class BufferedReader(object): - """BufferedReader(raw, buffer_size=DEFAULT_BUFFER_SIZE)\n--\n\n - Create a new buffered reader using the given readable raw IO object. - """ - pass diff --git a/Lib/test/archiver_tests.py b/Lib/test/archiver_tests.py new file mode 100644 index 00000000000..24745941b08 --- /dev/null +++ b/Lib/test/archiver_tests.py @@ -0,0 +1,177 @@ +"""Tests common to tarfile and zipfile.""" + +import os +import sys + +from test.support import swap_attr +from test.support import os_helper + +class OverwriteTests: + + def setUp(self): + os.makedirs(self.testdir) + self.addCleanup(os_helper.rmtree, self.testdir) + + def create_file(self, path, content=b''): + with open(path, 'wb') as f: + f.write(content) + + def open(self, path): + raise NotImplementedError + + def extractall(self, ar): + raise NotImplementedError + + + def test_overwrite_file_as_file(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_dir_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) + with open(os.path.join(target, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + def test_overwrite_dir_as_file(self): + target = os.path.join(self.testdir, 'test') + os.mkdir(target) + with self.open(self.ar_with_file) as ar: + with self.assertRaises(PermissionError if sys.platform == 'win32' + else IsADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_overwrite_file_as_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + def test_overwrite_file_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + self.create_file(target, b'content') + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileNotFoundError if sys.platform == 'win32' + else NotADirectoryError): + self.extractall(ar) + self.assertTrue(os.path.isfile(target)) + with open(target, 'rb') as f: + self.assertEqual(f.read(), b'content') + + @os_helper.skip_unless_symlink + def test_overwrite_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + self.create_file(target2, b'content') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_file_symlink_as_file(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target) + with self.open(self.ar_with_file) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isfile(target2)) + with open(target2, 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_dir_symlink_as_implicit_dir(self): + # XXX: It is potential security vulnerability. + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.mkdir(target2) + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertTrue(os.path.isdir(target2)) + self.assertTrue(os.path.isfile(os.path.join(target2, 'file'))) + with open(os.path.join(target2, 'file'), 'rb') as f: + self.assertEqual(f.read(), b'newcontent') + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + @os_helper.skip_unless_symlink + def test_overwrite_broken_dir_symlink_as_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + target2 = os.path.join(self.testdir, 'test2') + os.symlink('test2', target, target_is_directory=True) + with self.open(self.ar_with_implicit_dir) as ar: + with self.assertRaises(FileExistsError): + self.extractall(ar) + self.assertTrue(os.path.islink(target)) + self.assertFalse(os.path.exists(target2)) + + def test_concurrent_extract_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + + def test_concurrent_extract_implicit_dir(self): + target = os.path.join(self.testdir, 'test') + def concurrent_mkdir(*args, **kwargs): + orig_mkdir(*args, **kwargs) + orig_mkdir(*args, **kwargs) + with swap_attr(os, 'mkdir', concurrent_mkdir) as orig_mkdir: + with self.open(self.ar_with_implicit_dir) as ar: + self.extractall(ar) + self.assertTrue(os.path.isdir(target)) + self.assertTrue(os.path.isfile(os.path.join(target, 'file'))) diff --git a/Lib/test/archivetestdata/README.md b/Lib/test/archivetestdata/README.md new file mode 100644 index 00000000000..7b555fa3276 --- /dev/null +++ b/Lib/test/archivetestdata/README.md @@ -0,0 +1,36 @@ +# Test data for `test_zipfile`, `test_tarfile` (and even some others) + +## `test_zipfile` + +The test executables in this directory are created manually from `header.sh` and +the `testdata_module_inside_zip.py` file. You must have Info-ZIP's zip utility +installed (`apt install zip` on Debian). + +### Purpose of `exe_with_zip` and `exe_with_z64` + +These are used to test executable files with an appended zipfile, in a scenario +where the executable is _not_ a Python interpreter itself so our automatic +zipimport machinery (that'd look for `__main__.py`) is not being used. + +### Updating the test executables + +If you update header.sh or the testdata_module_inside_zip.py file, rerun the +commands below. These are expected to be rarely changed, if ever. + +#### Standard old format (2.0) zip file + +``` +zip -0 zip2.zip testdata_module_inside_zip.py +cat header.sh zip2.zip >exe_with_zip +rm zip2.zip +``` + +#### Modern format (4.5) zip64 file + +Redirecting from stdin forces Info-ZIP's zip tool to create a zip64. + +``` +zip -0 zip64.zip +cat header.sh zip64.zip >exe_with_z64 +rm zip64.zip +``` diff --git a/Lib/test/archivetestdata/exe_with_z64 b/Lib/test/archivetestdata/exe_with_z64 new file mode 100755 index 00000000000..82b03cf39d9 Binary files /dev/null and b/Lib/test/archivetestdata/exe_with_z64 differ diff --git a/Lib/test/archivetestdata/exe_with_zip b/Lib/test/archivetestdata/exe_with_zip new file mode 100755 index 00000000000..c833cdf9f93 Binary files /dev/null and b/Lib/test/archivetestdata/exe_with_zip differ diff --git a/Lib/test/archivetestdata/header.sh b/Lib/test/archivetestdata/header.sh new file mode 100755 index 00000000000..52dc91acf74 --- /dev/null +++ b/Lib/test/archivetestdata/header.sh @@ -0,0 +1,24 @@ +#!/bin/bash +INTERPRETER_UNDER_TEST="$1" +if [[ ! -x "${INTERPRETER_UNDER_TEST}" ]]; then + echo "Interpreter must be the command line argument." + exit 4 +fi +EXECUTABLE="$0" exec "${INTERPRETER_UNDER_TEST}" -E - < 0: + i += 1 + fout.writeframes(f.readframes(i)) + n -= i + fout.close() + fout = self.fout = self.module.open(TESTFN, 'rb') + f.rewind() + self.assertEqual(f.getparams(), fout.getparams()) + self.assertEqual(f.readframes(f.getnframes()), + fout.readframes(fout.getnframes())) + + def test_read_not_from_start(self): + with open(TESTFN, 'wb') as testfile: + testfile.write(b'ababagalamaga') + with open(self.sndfilepath, 'rb') as f: + testfile.write(f.read()) + + with open(TESTFN, 'rb') as testfile: + self.assertEqual(testfile.read(13), b'ababagalamaga') + with self.module.open(testfile, 'rb') as f: + self.assertEqual(f.getnchannels(), self.nchannels) + self.assertEqual(f.getsampwidth(), self.sampwidth) + self.assertEqual(f.getframerate(), self.framerate) + self.assertEqual(f.getnframes(), self.sndfilenframes) + self.assertEqual(f.readframes(self.nframes), self.frames) diff --git a/Lib/test/autotest.py b/Lib/test/autotest.py new file mode 100644 index 00000000000..b5a1fab404c --- /dev/null +++ b/Lib/test/autotest.py @@ -0,0 +1,5 @@ +# This should be equivalent to running regrtest.py from the cmdline. +# It can be especially handy if you're in an interactive shell, e.g., +# from test import autotest. +from test.libregrtest.main import main +main() diff --git a/Lib/test/badsyntax_future3.py b/Lib/test/badsyntax_future3.py deleted file mode 100644 index f1c8417edaa..00000000000 --- a/Lib/test/badsyntax_future3.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -from __future__ import nested_scopes -from __future__ import rested_snopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/badsyntax_future4.py b/Lib/test/badsyntax_future4.py deleted file mode 100644 index b5f4c98e922..00000000000 --- a/Lib/test/badsyntax_future4.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -import __future__ -from __future__ import nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/badsyntax_future5.py b/Lib/test/badsyntax_future5.py deleted file mode 100644 index 8a7e5fcb70f..00000000000 --- a/Lib/test/badsyntax_future5.py +++ /dev/null @@ -1,12 +0,0 @@ -"""This is a test""" -from __future__ import nested_scopes -import foo -from __future__ import nested_scopes - - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/badsyntax_future6.py b/Lib/test/badsyntax_future6.py deleted file mode 100644 index 5a8b55a02c4..00000000000 --- a/Lib/test/badsyntax_future6.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" -"this isn't a doc string" -from __future__ import nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/badsyntax_future7.py b/Lib/test/badsyntax_future7.py deleted file mode 100644 index 131db2c2164..00000000000 --- a/Lib/test/badsyntax_future7.py +++ /dev/null @@ -1,11 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes; import string; from __future__ import \ - nested_scopes - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/badsyntax_future8.py b/Lib/test/badsyntax_future8.py deleted file mode 100644 index ca45289e2e5..00000000000 --- a/Lib/test/badsyntax_future8.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import * - -def f(x): - def g(y): - return x + y - return g - -print(f(2)(4)) diff --git a/Lib/test/badsyntax_future9.py b/Lib/test/badsyntax_future9.py deleted file mode 100644 index 916de06ab71..00000000000 --- a/Lib/test/badsyntax_future9.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes, braces - -def f(x): - def g(y): - return x + y - return g - -print(f(2)(4)) diff --git a/Lib/test/certdata/allsans.pem b/Lib/test/certdata/allsans.pem new file mode 100644 index 00000000000..f6a63b9219d --- /dev/null +++ b/Lib/test/certdata/allsans.pem @@ -0,0 +1,167 @@ +-----BEGIN PRIVATE KEY----- +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQCYEY+lIwS/w+gA +U5PfxSeL4qMF5TtFkrifL0oQaCjWFb8jeX2Rb1tKaObyrYmTURVWbMkI1LJOQWYt +GlANo2mvf4VD1wZZ/x/Tmylxe3A7itSTZgZuGXLHtFP9wk7yS46BlVcNq3ZoOi7k +k8NxTiUqgsmLg3Nj+Ifk8OI7c1BcP2pa1wZO/l3PTWqtFSLsFEvOK5N6ZPqILAUi +5vEkOiFefxVHHvD4N3IWSFgSwiDi9KhntdRwa+axsZxjq/o70FjQO5q5Quym9aDq +qo47PMykEb1gdTBQK1mPliMKsLi5zycI/ypm1PEcH53+9/F+U2B1bmvA/9PBx8T7 +hUyclBbQHCmdcXIYCgFr5AbGS3jFeVPiF7hCPj1k/jcpayXZAtWmqJTlkXKGbM68 +AgM/DHO0fGCYGRUw2vhV3ZIRCm0BqwforeFh+2oMyZjDjvo/uZjZNazdpcwvOy0c +lFkORwVI0zTIixXYSV4xN1BX821JCuhYubro89m+MSjxO14qgs0CAwEAAQKCAYAZ +txqF7+qNLSlN6NRASnw6BQzRYebIiJus25fYN2z0awAEFTbdQan75sprLlpt7Y/A +qivC0QkH/7oyFVmFPOWR1mwoQTPjQyfUJlB3Tsr3Xq488MtUkfwddkqfxlyIT6ud +ES6D8sNWs8QbRjuOLQtO6YgAji2UscH1PqDbMdBckSLAks0Pzab6d9p6w3DA4FvD +VQ4e6/WL0nnZ4ZjUqfnbm3zzJnHUX7fsubYfEfHyvzG9O/vdOPntgZ3zIvFxbPVc +/Dp8MRNgfQoRgAsc7VBQQifFNvva9SjvRsXwE7zQLQh9JszFpHWqIMevGwjdH8dO +1hHESqkRHL0sYwsqoGqbaHFrJVEA2Ljuxs/UsH6G2U6NuDVaYkqM8UwH0+EqGFtF +9qP8oh6Yvp+szI5yET4brW0Rlk8r2UqvQ5iSSl8SSWsNKR2wRs0PM0aNm13TDJmt +NOuxd/0ranG+NvBMdtLCwsoEplYDaaczYv9gbKHT64OlFAdGBkOytbS7eqOojmcC +gcEA0uQb9bPKbiMYSEJMAalOlruzvMiHBRW420IEuGyHJtocn/RK804Jj6gzu+WD +GT81rB9NXAt/3s6WDMSdrGrrBh/CQZJs50ok9hej0f6gsq1JNsWJ/c10rU/mqrTn +s++pdaRqbd+bgM/N6h79tJi3j4SgB9rJqjC+7qWlDJFWZOoDR59elc10xDA0AE0a +K6UvsJJGk4/v715K1Uh2qU5HYupuijNc0m3z2OMZ6zxTNNUdbSL4XF5X9e2+BwMQ +YrRzAoHBALiYeYWsmHGRdg++/p/9SE86YwyEYr+zrUiWenIPC1W5WQlkdFQqALmg +kjcXAXFFouC/DnKG65YOfGq772ekFuw06Lm4DEotQP02pESfAP9ulVufK2yJtbxi +m0PeN3JRrs/H72VW9hUSxCafNRyBeZ6nsfzEfEkPOdc+lRGeMHT9C7r7Y1asNk8A +Zl7wKZgv+Yvn8xhHWQj5z/T6K2z5CXFn6FERc0qV8+DggE9m3CjaaNxft6IodfHD +h1iakNtbvwKBwQCpWr/NRy131r0IQh0xdFoFGAUVxF8RSUli4hhSVe0O2TcFiLOr +wW5SK/wnlv75hlY+vABuu1lbfsDmzfnk3RORnm1sJP9JmbQm4AMRfw5jjl7uGiJf +a9+X0kNlsNMlH4ARVhCV3WzOO5KbwXlxzvYRzaqJxDwQbQbXNLRfbFNZxMcPfD8D +w7NSXXdVCpXKmOO8QytkEsHWkv07W+7WtWMEX0iXuPmAjwW0lWNaEd6r3by8yMlz +u9udRedFUEOXUFsCgcAX0g00c75EQXoTtBjVenC/UJCBh//aLwx4Znqsh0Z2LHHR +5XWher4XNiJIG57jCBJpoB30J3b1KS9i8peFL0aJ+pXhiV+EnuxZAJkYBdCyJYn+ +hb6rxeV+xta0XlOXW/UL+QfqcttUgtRvC3JmGEsibw9nx88l+mIDZZ8E4/3qytCd +s1zxTU3AyhNrwuALNH2mUSssgeB6aQot2a6K5GQUj00KUQ8om8sZxL6qAGL+npiT +f4KJ2WDG7u1jQKbat68CgcBONTGp3M3pdwc3OV/PMFq1+h1/BuyW/5qUtbsvXkrD +DZyXGY2SxJFMipakzOppusaQOYmO+VXKVDRJm8UG6fa2cdXjwm2kDxHB5E+v59VI ++uM8ErX6L52I0+rdoOU6AwpXyiVW5AXzMInOGHF4B7zJ1SA25RwVG44LbDw1cehW +MALXUdVCaegmvCCFq5TQlrlaEktxWk6Th92mXWbNbwUHlIGpcjEkASFvt90aBXWZ +w0YCQFVxx/K95nzKyRTjHTQ= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5f + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2525 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=allsans + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:98:11:8f:a5:23:04:bf:c3:e8:00:53:93:df:c5: + 27:8b:e2:a3:05:e5:3b:45:92:b8:9f:2f:4a:10:68: + 28:d6:15:bf:23:79:7d:91:6f:5b:4a:68:e6:f2:ad: + 89:93:51:15:56:6c:c9:08:d4:b2:4e:41:66:2d:1a: + 50:0d:a3:69:af:7f:85:43:d7:06:59:ff:1f:d3:9b: + 29:71:7b:70:3b:8a:d4:93:66:06:6e:19:72:c7:b4: + 53:fd:c2:4e:f2:4b:8e:81:95:57:0d:ab:76:68:3a: + 2e:e4:93:c3:71:4e:25:2a:82:c9:8b:83:73:63:f8: + 87:e4:f0:e2:3b:73:50:5c:3f:6a:5a:d7:06:4e:fe: + 5d:cf:4d:6a:ad:15:22:ec:14:4b:ce:2b:93:7a:64: + fa:88:2c:05:22:e6:f1:24:3a:21:5e:7f:15:47:1e: + f0:f8:37:72:16:48:58:12:c2:20:e2:f4:a8:67:b5: + d4:70:6b:e6:b1:b1:9c:63:ab:fa:3b:d0:58:d0:3b: + 9a:b9:42:ec:a6:f5:a0:ea:aa:8e:3b:3c:cc:a4:11: + bd:60:75:30:50:2b:59:8f:96:23:0a:b0:b8:b9:cf: + 27:08:ff:2a:66:d4:f1:1c:1f:9d:fe:f7:f1:7e:53: + 60:75:6e:6b:c0:ff:d3:c1:c7:c4:fb:85:4c:9c:94: + 16:d0:1c:29:9d:71:72:18:0a:01:6b:e4:06:c6:4b: + 78:c5:79:53:e2:17:b8:42:3e:3d:64:fe:37:29:6b: + 25:d9:02:d5:a6:a8:94:e5:91:72:86:6c:ce:bc:02: + 03:3f:0c:73:b4:7c:60:98:19:15:30:da:f8:55:dd: + 92:11:0a:6d:01:ab:07:e8:ad:e1:61:fb:6a:0c:c9: + 98:c3:8e:fa:3f:b9:98:d9:35:ac:dd:a5:cc:2f:3b: + 2d:1c:94:59:0e:47:05:48:d3:34:c8:8b:15:d8:49: + 5e:31:37:50:57:f3:6d:49:0a:e8:58:b9:ba:e8:f3: + d9:be:31:28:f1:3b:5e:2a:82:cd + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:allsans, othername: 1.2.3.4::some other identifier, othername: 1.3.6.1.5.2.2::, email:user@example.org, DNS:www.example.org, DirName:/C=XY/L=Castle Anthrax/O=Python Software Foundation/CN=dirname example, URI:https://www.python.org/, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, Registered ID:1.2.3.4.5 + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 73:0D:4E:1F:4A:EC:F2:53:0F:53:FC:85:6F:CF:82:CD:A3:E8:11:8D + X509v3 Authority Key Identifier: + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + a5:3d:12:87:b9:3b:3e:9c:1c:59:fb:d5:38:22:49:61:f3:c3: + 11:53:4b:4e:63:af:f2:3d:ef:24:67:45:bc:74:5c:4a:65:c5: + b4:bb:fe:84:b8:b6:ca:7d:fc:aa:ff:80:ae:67:1f:cb:c3:cd: + 8f:f9:75:8c:f9:d3:3f:db:f6:81:d8:06:42:c3:5d:a9:1e:a3: + 81:7d:57:ac:97:d9:bd:c8:ce:1e:ec:74:d7:94:d5:df:b1:ad: + ce:84:14:5c:8c:45:a4:a8:eb:67:ab:16:57:61:15:86:ae:11: + 1e:b5:10:42:de:84:76:88:9b:37:12:aa:a6:77:42:75:b4:c0: + 04:b3:75:45:e0:d7:aa:34:e3:07:c5:ed:f8:4e:f0:39:99:1f: + 5b:d8:4e:0c:ad:64:6d:09:07:3f:e3:e1:9f:1b:65:07:96:59: + 9a:b5:f1:4d:c3:ec:f7:32:a1:05:94:d1:0b:18:54:3c:67:cf: + 38:f5:2b:ec:cb:bd:79:be:f7:1b:b7:71:3f:c6:44:80:7f:00: + dc:3d:a0:07:c0:b5:1c:fb:52:f6:a0:f8:92:c6:c6:73:07:c5: + ca:0b:04:7c:55:51:e3:ba:93:32:17:bd:61:ae:cf:13:e4:5e: + 03:b2:51:11:c7:68:f5:08:b6:0e:57:49:11:3c:e3:f4:0e:e1: + 96:20:44:28:31:94:11:44:50:cf:17:70:8d:9c:14:c5:ed:94: + 4d:ba:94:9b:db:8b:9f:55:a1:5e:0a:90:bb:a0:0e:0d:3b:a0: + dd:4d:47:d1:cf:d0:47:6b:ff:6f:af:e4:83:40:73:e6:3a:59: + 40:bd:4f:3a:21:22:63:27:5d:02:26:67:89:1d:2f:19:c5:77: + e6:b5:67:4a:dd:b5:0e:de:46:34:57:c7:03:84:5d:cd:34:08: + 8f:47:c9:4d:7f:04:c0:4f:ff:68:52:bb:ae:84:0e:54:ce:5c: + 27:71:0f:a2:3f:9f:7a:49:a8:fa:0a:45:cf:96:42:a7:65:23: + b9:3e:40:eb:46:7b +-----BEGIN CERTIFICATE----- +MIIHDzCCBXegAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +YWxsc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAJgRj6UjBL/D +6ABTk9/FJ4viowXlO0WSuJ8vShBoKNYVvyN5fZFvW0po5vKtiZNRFVZsyQjUsk5B +Zi0aUA2jaa9/hUPXBln/H9ObKXF7cDuK1JNmBm4Zcse0U/3CTvJLjoGVVw2rdmg6 +LuSTw3FOJSqCyYuDc2P4h+Tw4jtzUFw/alrXBk7+Xc9Naq0VIuwUS84rk3pk+ogs +BSLm8SQ6IV5/FUce8Pg3chZIWBLCIOL0qGe11HBr5rGxnGOr+jvQWNA7mrlC7Kb1 +oOqqjjs8zKQRvWB1MFArWY+WIwqwuLnPJwj/KmbU8Rwfnf738X5TYHVua8D/08HH +xPuFTJyUFtAcKZ1xchgKAWvkBsZLeMV5U+IXuEI+PWT+NylrJdkC1aaolOWRcoZs +zrwCAz8Mc7R8YJgZFTDa+FXdkhEKbQGrB+it4WH7agzJmMOO+j+5mNk1rN2lzC87 +LRyUWQ5HBUjTNMiLFdhJXjE3UFfzbUkK6Fi5uujz2b4xKPE7XiqCzQIDAQABo4IC +3jCCAtowggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBv +dGhlciBpZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2h +FTAToAMCAQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4 +YW1wbGUub3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRo +cmF4MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UE +AwwPZGlybmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAA +AYcQAAAAAAAAAAAAAAAAAAAAAYgEKgMEBTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0l +BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYE +FHMNTh9K7PJTD1P8hW/Pgs2j6BGNMH0GA1UdIwR2MHSAFMAKKyhD3l/JfUflR5s2 +8mWMZzvioVGkTzBNMQswCQYDVQQGEwJYWTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3 +YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXKCCQDLLYCZ +WmlSWzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUFBzAChjBodHRwOi8vdGVzdGNh +LnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2VydC5jZXIwNQYIKwYBBQUHMAGG +KWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL29jc3AvMEMGA1Ud +HwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNh +L3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBCwUAA4IBgQClPRKHuTs+nBxZ+9U4 +Iklh88MRU0tOY6/yPe8kZ0W8dFxKZcW0u/6EuLbKffyq/4CuZx/Lw82P+XWM+dM/ +2/aB2AZCw12pHqOBfVesl9m9yM4e7HTXlNXfsa3OhBRcjEWkqOtnqxZXYRWGrhEe +tRBC3oR2iJs3Eqqmd0J1tMAEs3VF4NeqNOMHxe34TvA5mR9b2E4MrWRtCQc/4+Gf +G2UHllmatfFNw+z3MqEFlNELGFQ8Z8849Svsy715vvcbt3E/xkSAfwDcPaAHwLUc ++1L2oPiSxsZzB8XKCwR8VVHjupMyF71hrs8T5F4DslERx2j1CLYOV0kRPOP0DuGW +IEQoMZQRRFDPF3CNnBTF7ZRNupSb24ufVaFeCpC7oA4NO6DdTUfRz9BHa/9vr+SD +QHPmOllAvU86ISJjJ10CJmeJHS8ZxXfmtWdK3bUO3kY0V8cDhF3NNAiPR8lNfwTA +T/9oUruuhA5UzlwncQ+iP596Saj6CkXPlkKnZSO5PkDrRns= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/badcert.pem b/Lib/test/certdata/badcert.pem new file mode 100644 index 00000000000..c4191460f9e --- /dev/null +++ b/Lib/test/certdata/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/badkey.pem b/Lib/test/certdata/badkey.pem new file mode 100644 index 00000000000..1c8a9557193 --- /dev/null +++ b/Lib/test/certdata/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/4e1295a3.0 b/Lib/test/certdata/capath/4e1295a3.0 new file mode 100644 index 00000000000..9d7ac238d86 --- /dev/null +++ b/Lib/test/certdata/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/5ed36f99.0 b/Lib/test/certdata/capath/5ed36f99.0 new file mode 100644 index 00000000000..e7dfc82947e --- /dev/null +++ b/Lib/test/certdata/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/6e88d7b8.0 b/Lib/test/certdata/capath/6e88d7b8.0 new file mode 100644 index 00000000000..9d7ac238d86 --- /dev/null +++ b/Lib/test/certdata/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/99d0fa06.0 b/Lib/test/certdata/capath/99d0fa06.0 new file mode 100644 index 00000000000..e7dfc82947e --- /dev/null +++ b/Lib/test/certdata/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/b1930218.0 b/Lib/test/certdata/capath/b1930218.0 new file mode 100644 index 00000000000..6d773ee5b32 --- /dev/null +++ b/Lib/test/certdata/capath/b1930218.0 @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/ceff1710.0 b/Lib/test/certdata/capath/ceff1710.0 new file mode 100644 index 00000000000..6d773ee5b32 --- /dev/null +++ b/Lib/test/certdata/capath/ceff1710.0 @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/cert3.pem b/Lib/test/certdata/cert3.pem new file mode 100644 index 00000000000..a11dc614657 --- /dev/null +++ b/Lib/test/certdata/cert3.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/ffdh3072.pem b/Lib/test/certdata/ffdh3072.pem new file mode 100644 index 00000000000..ad69bac8d00 --- /dev/null +++ b/Lib/test/certdata/ffdh3072.pem @@ -0,0 +1,41 @@ + DH Parameters: (3072 bit) + prime: + 00:ff:ff:ff:ff:ff:ff:ff:ff:ad:f8:54:58:a2:bb: + 4a:9a:af:dc:56:20:27:3d:3c:f1:d8:b9:c5:83:ce: + 2d:36:95:a9:e1:36:41:14:64:33:fb:cc:93:9d:ce: + 24:9b:3e:f9:7d:2f:e3:63:63:0c:75:d8:f6:81:b2: + 02:ae:c4:61:7a:d3:df:1e:d5:d5:fd:65:61:24:33: + f5:1f:5f:06:6e:d0:85:63:65:55:3d:ed:1a:f3:b5: + 57:13:5e:7f:57:c9:35:98:4f:0c:70:e0:e6:8b:77: + e2:a6:89:da:f3:ef:e8:72:1d:f1:58:a1:36:ad:e7: + 35:30:ac:ca:4f:48:3a:79:7a:bc:0a:b1:82:b3:24: + fb:61:d1:08:a9:4b:b2:c8:e3:fb:b9:6a:da:b7:60: + d7:f4:68:1d:4f:42:a3:de:39:4d:f4:ae:56:ed:e7: + 63:72:bb:19:0b:07:a7:c8:ee:0a:6d:70:9e:02:fc: + e1:cd:f7:e2:ec:c0:34:04:cd:28:34:2f:61:91:72: + fe:9c:e9:85:83:ff:8e:4f:12:32:ee:f2:81:83:c3: + fe:3b:1b:4c:6f:ad:73:3b:b5:fc:bc:2e:c2:20:05: + c5:8e:f1:83:7d:16:83:b2:c6:f3:4a:26:c1:b2:ef: + fa:88:6b:42:38:61:1f:cf:dc:de:35:5b:3b:65:19: + 03:5b:bc:34:f4:de:f9:9c:02:38:61:b4:6f:c9:d6: + e6:c9:07:7a:d9:1d:26:91:f7:f7:ee:59:8c:b0:fa: + c1:86:d9:1c:ae:fe:13:09:85:13:92:70:b4:13:0c: + 93:bc:43:79:44:f4:fd:44:52:e2:d7:4d:d3:64:f2: + e2:1e:71:f5:4b:ff:5c:ae:82:ab:9c:9d:f6:9e:e8: + 6d:2b:c5:22:36:3a:0d:ab:c5:21:97:9b:0d:ea:da: + 1d:bf:9a:42:d5:c4:48:4e:0a:bc:d0:6b:fa:53:dd: + ef:3c:1b:20:ee:3f:d5:9d:7c:25:e4:1d:2b:66:c6: + 2e:37:ff:ff:ff:ff:ff:ff:ff:ff + generator: 2 (0x2) + recommended-private-length: 276 bits +-----BEGIN DH PARAMETERS----- +MIIBjAKCAYEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEfz9zeNVs7ZRkDW7w09N75nAI4YbRvydbmyQd62R0mkff3 +7lmMsPrBhtkcrv4TCYUTknC0EwyTvEN5RPT9RFLi103TZPLiHnH1S/9croKrnJ32 +nuhtK8UiNjoNq8Uhl5sN6todv5pC1cRITgq80Gv6U93vPBsg7j/VnXwl5B0rZsYu +N///////////AgECAgIBFA== +-----END DH PARAMETERS----- diff --git a/Lib/test/certdata/idnsans.pem b/Lib/test/certdata/idnsans.pem new file mode 100644 index 00000000000..ebd7250cb4e --- /dev/null +++ b/Lib/test/certdata/idnsans.pem @@ -0,0 +1,166 @@ +-----BEGIN PRIVATE KEY----- +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDeGGysqNqsz64+ +xsN+CUZDfCgPPhzUK0lDUlkEkp0ZeFeQm3+po2Y82gvJIValO5ChiXjPxSgJwoto +SHe4QZ8fWAlkX1d19A8xjjetx1K3dqZstqe3mPP7HQvfJlRkmAtVhFaUj1+SUx9K +s4HB4Lk2hqKUoq3TlRMpeSfJTXaHThVP7KZPWBzfM7pdpGYnznBpxgfymNrpTcXN +CjeR6lUlj/TExHiYLj4E0aODR/SUz0tBNdCQRcRDw+OyREe6Rk32Krl34Pr1dNKe +ER0tulhM2wRFsqw0a8vNs13I30BH4aXAr5bGgLCF2tXbP3KoeoR/GkQSWV14KrhT +xhKD3QEJ8gHjoHYKr4IdcFYcTakbI9E3THsQjQTUMJ/Xygza7OOr6pNPKY10tYfp +7WhyyY7cB/C/rHnUY0f58IZlRdMYgcLCUl7wzUxSVEMyr9Ary8XZZlxuO629k2ud +AIGwIi2TVq+UK0kNPZyGicGHMeEhoNc3YDMB/zYeqMr4KryI+mkCAwEAAQKCAYA4 +EENSnHdC+1P5bdRIew/dFjjIjD3bwyeF0oI9GMOGe+3ix5YE3QYAY2xpM7y7Dhu2 +40x3akXunMjzJKPwA8SmtWL9juG1mUvCjyt39yJmxJFDTSJuQrKIF694/6R7FjR6 +PGNcsgqGlewGv+SH6/HlFTxyN9SYXf/NztMfyimbAzd3Cv56dfwnzdeELu1IrCCN +WtuDvlk4XpUJasRXVadzyXCYwR3OEJJARik4CRBxBhjxl6OT38Co+IiAZiMTHw6z +jXeRuvyxXyYB76BQs9uCt05c5JV1I2OT1aKdpC0EbHlJEzJhmxrpdfBeoVdJUVVg +4b0QoocZE89KXOVmjmIKExxmjOjxH6o6qfxvDQWLspADD4D+69Zqjj6hFAfn3EUW +hcq3RHwuUheCE6i2xRtWPnA1ygqWN9mdNojX1+EXIv0Dh29kFWHpMUC29WWm9JA6 +Arx2UrtvoWGYotKxTDxZ0SV5ws/LzGjvMaT/dbXZHjywnB7sPfpcE9b91p8Q6VcC +gcEA+Wr44ZzyMirF9FYYA+KEBL91khmffhXzCDkRXqd1j2BCQWETWqvrUuAx+kwh +FZ5AABdip96w8Eaj8XBWb1c81mJUbOimbcqiV6B4GokCI/7u3FkOMPFRNBq41R/M +43JH6QLHNxhXI4OlZsxv5WHmR5N88kXEHhofBoSn14EHtdkK/Hu8ly1Wux4/da83 +2wanwlysmQkk1CnpJVT0st7HrKHP8YWDbcBvH+xysHOIOs22q3gvAF2rsdORidpR +fZnHAoHBAOP03brYpLU1xa5LHQz/B7uWfgJm3l7wRQ627YOfUBe4IjnH2y/kNKjv +ROVjaooB2gtya6Z6xVuY1m4LzUfYuRN1FdX1Np4NjvfgPNFViX+0HoxyAytXn++V +Iy1tCpL6X43Iqrj/VxBcX09vIpbsprW2d3dcIo/45abkAO6Uoa6HUmZoqrqrBA7b +LnuhEfBAyyN42OiXbHDqif7XAWC30yY6k0FfDaOSLiBkAOKZ9lkf7ei3eNr4SdOa +Tf5a/RVKTwKBwB7YfOkiCM3tfkfGcffhBqSzrO2hn5jvS/wjWqOTIDXYGLmPMN6Q +zmyUb3nd+mV7Cb05JylNoCJHCjVsyDPC3TJCPOCvMQ349nTR0qitcwdSmuXDWb7x +yTIhb+Rjp2olkwEdJ9gHeZdZy5XYCKqcnecSNWyc9jEm19lthHhha7uwmOw6vUsQ +/13q0rxSLB05SHwADBRtDhHzEPNd+1k3tggChv3+ng9vsg6HpnNuBlYHZOT12xI3 +g2ldme0rg9J9twKBwA4R1gGrT3czy3C3iCJ+Ny732e0yBjWb5NdEqSI/mgTsw4gH +ctrg3fMzWXBDE5dTB+8+77AF0dqWc121csUldj7iMifTi7xzn8hi2b4d5m+wYVZP +zyxEq0VxUguCuG1b8Lvij879S5Vh7iwL8vmXv65lhbgjQqraNOp5FimjmNsZ1Rcn +DKqa1ZRJKPROe7n1ddRJqDGq7vGFOGE3Sgl7Lxgj82TMhh37bsdnBLr3v8G+e8Oq +V1ZEjuH1myzA1vASdwKBwQDcMHbUKeoJUxyIlABFsWxhm0MKwL5ZRgo7rhH01rF2 +TF3mbAEHHsAEfkHgBVRooifbxglxUDy1olIyRk+kqs1gmXZA0UmE9lUqoX5+n/j8 +pgbCc3sV53x7JJ4BCnAn49XTOnTi2ILeQ3MvTqcj+sDKo+0T4Zq0r5d8ZrMikKbO +HJ4MMBGth645QKhgpb0XgltvSn8aceS4uTxIKrbNpZVnWj/VzuOYoXQmKnZko9p1 +dbyjt6PMeVXWj0tz8FB2DGE= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:60 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2525 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:de:18:6c:ac:a8:da:ac:cf:ae:3e:c6:c3:7e:09: + 46:43:7c:28:0f:3e:1c:d4:2b:49:43:52:59:04:92: + 9d:19:78:57:90:9b:7f:a9:a3:66:3c:da:0b:c9:21: + 56:a5:3b:90:a1:89:78:cf:c5:28:09:c2:8b:68:48: + 77:b8:41:9f:1f:58:09:64:5f:57:75:f4:0f:31:8e: + 37:ad:c7:52:b7:76:a6:6c:b6:a7:b7:98:f3:fb:1d: + 0b:df:26:54:64:98:0b:55:84:56:94:8f:5f:92:53: + 1f:4a:b3:81:c1:e0:b9:36:86:a2:94:a2:ad:d3:95: + 13:29:79:27:c9:4d:76:87:4e:15:4f:ec:a6:4f:58: + 1c:df:33:ba:5d:a4:66:27:ce:70:69:c6:07:f2:98: + da:e9:4d:c5:cd:0a:37:91:ea:55:25:8f:f4:c4:c4: + 78:98:2e:3e:04:d1:a3:83:47:f4:94:cf:4b:41:35: + d0:90:45:c4:43:c3:e3:b2:44:47:ba:46:4d:f6:2a: + b9:77:e0:fa:f5:74:d2:9e:11:1d:2d:ba:58:4c:db: + 04:45:b2:ac:34:6b:cb:cd:b3:5d:c8:df:40:47:e1: + a5:c0:af:96:c6:80:b0:85:da:d5:db:3f:72:a8:7a: + 84:7f:1a:44:12:59:5d:78:2a:b8:53:c6:12:83:dd: + 01:09:f2:01:e3:a0:76:0a:af:82:1d:70:56:1c:4d: + a9:1b:23:d1:37:4c:7b:10:8d:04:d4:30:9f:d7:ca: + 0c:da:ec:e3:ab:ea:93:4f:29:8d:74:b5:87:e9:ed: + 68:72:c9:8e:dc:07:f0:bf:ac:79:d4:63:47:f9:f0: + 86:65:45:d3:18:81:c2:c2:52:5e:f0:cd:4c:52:54: + 43:32:af:d0:2b:cb:c5:d9:66:5c:6e:3b:ad:bd:93: + 6b:9d:00:81:b0:22:2d:93:56:af:94:2b:49:0d:3d: + 9c:86:89:c1:87:31:e1:21:a0:d7:37:60:33:01:ff: + 36:1e:a8:ca:f8:2a:bc:88:fa:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:idnsans, DNS:xn--knig-5qa.idn.pythontest.net, DNS:xn--knigsgsschen-lcb0w.idna2003.pythontest.net, DNS:xn--knigsgchen-b4a3dun.idna2008.pythontest.net, DNS:xn--nxasmq6b.idna2003.pythontest.net, DNS:xn--nxasmm1c.idna2008.pythontest.net + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + C8:06:99:B7:E8:8F:EC:4F:3D:5C:89:6A:06:F5:77:2E:B0:E0:6A:9E + X509v3 Authority Key Identifier: + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 40:d1:6d:8e:a2:0b:91:4b:a8:c4:08:d0:f3:f9:8b:d0:a3:0b: + dc:00:22:8c:f1:2e:2b:e5:e6:b4:6e:ce:9d:cf:59:32:66:6c: + bb:0e:3b:1d:9c:05:d2:eb:a6:29:f8:74:4f:dc:83:3b:32:5a: + 2c:67:86:61:25:bc:bd:19:eb:20:c6:30:69:0e:4c:b2:e3:18: + ca:9e:fe:40:bc:1c:ad:8b:03:f5:04:be:90:ce:27:27:2f:83: + 14:57:8d:4f:a0:db:46:ce:e0:7d:e2:cf:7d:ea:0c:fd:8d:00: + 27:0a:db:0d:5f:e7:1e:52:25:1f:64:b9:30:5f:07:1a:10:a3: + 69:35:0e:dc:f8:23:f7:34:07:ce:c8:92:94:39:4d:d5:c3:ab: + 33:aa:f9:67:be:66:18:ac:67:14:5f:93:5f:68:48:04:ed:1e: + c9:74:28:b2:47:34:49:11:e4:7b:38:32:e5:dc:40:13:b4:69: + 75:39:43:db:7c:4a:f0:2b:94:cd:01:ba:4d:9b:9e:68:b3:ee: + 03:9e:7f:9c:0c:cf:9c:5c:cb:d4:33:d5:f0:e3:21:54:9a:13: + 6f:eb:1a:0f:f3:8b:e8:ef:eb:34:ba:09:77:39:2a:8a:4b:e1: + 7e:9f:b5:05:be:95:b6:92:5d:4c:35:47:38:64:38:5e:27:b8: + f9:34:94:2f:57:16:b0:f5:6a:21:3f:09:34:b9:dd:f8:d1:47: + 2c:c7:5e:7f:63:49:f4:5b:f4:d9:ea:66:fc:aa:64:27:f0:72: + d7:94:6f:86:0f:e7:3b:b3:d4:d9:30:67:b8:a2:c3:f7:4d:07: + 44:b3:70:67:dd:b1:21:ac:7c:2a:04:7b:2c:1d:df:0b:82:a9: + fb:df:88:72:47:1c:f5:5d:a3:f7:52:22:2d:ea:f4:2a:45:4f: + 9b:9d:63:95:59:f3:79:05:2b:f1:5b:3b:62:71:69:90:30:d7: + 7a:b2:c8:ec:68:e5:94:bb:97:00:d0:95:a7:fd:04:6c:f7:8b: + 28:c1:96:9b:6a:94 +-----BEGIN CERTIFICATE----- +MIIGvzCCBSegAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEQMA4GA1UEAwwH +aWRuc2FuczCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAN4YbKyo2qzP +rj7Gw34JRkN8KA8+HNQrSUNSWQSSnRl4V5Cbf6mjZjzaC8khVqU7kKGJeM/FKAnC +i2hId7hBnx9YCWRfV3X0DzGON63HUrd2pmy2p7eY8/sdC98mVGSYC1WEVpSPX5JT +H0qzgcHguTaGopSirdOVEyl5J8lNdodOFU/spk9YHN8zul2kZifOcGnGB/KY2ulN +xc0KN5HqVSWP9MTEeJguPgTRo4NH9JTPS0E10JBFxEPD47JER7pGTfYquXfg+vV0 +0p4RHS26WEzbBEWyrDRry82zXcjfQEfhpcCvlsaAsIXa1ds/cqh6hH8aRBJZXXgq +uFPGEoPdAQnyAeOgdgqvgh1wVhxNqRsj0TdMexCNBNQwn9fKDNrs46vqk08pjXS1 +h+ntaHLJjtwH8L+sedRjR/nwhmVF0xiBwsJSXvDNTFJUQzKv0CvLxdlmXG47rb2T +a50AgbAiLZNWr5QrSQ09nIaJwYcx4SGg1zdgMwH/Nh6oyvgqvIj6aQIDAQABo4IC +jjCCAoowgeEGA1UdEQSB2TCB1oIHaWRuc2Fuc4IfeG4tLWtuaWctNXFhLmlkbi5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ3NzY2hlbi1sY2Iwdy5pZG5hMjAwMy5w +eXRob250ZXN0Lm5ldIIueG4tLWtuaWdzZ2NoZW4tYjRhM2R1bi5pZG5hMjAwOC5w +eXRob250ZXN0Lm5ldIIkeG4tLW54YXNtcTZiLmlkbmEyMDAzLnB5dGhvbnRlc3Qu +bmV0giR4bi0tbnhhc21tMWMuaWRuYTIwMDgucHl0aG9udGVzdC5uZXQwDgYDVR0P +AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB +Af8EAjAAMB0GA1UdDgQWBBTIBpm36I/sTz1ciWoG9XcusOBqnjB9BgNVHSMEdjB0 +gBTACisoQ95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNV +BAoMHVB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXIt +Y2Etc2VydmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcw +AoYwaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQu +Y2VyMDUGCCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9vY3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhv +bnRlc3QubmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOC +AYEAQNFtjqILkUuoxAjQ8/mL0KML3AAijPEuK+XmtG7Onc9ZMmZsuw47HZwF0uum +Kfh0T9yDOzJaLGeGYSW8vRnrIMYwaQ5MsuMYyp7+QLwcrYsD9QS+kM4nJy+DFFeN +T6DbRs7gfeLPfeoM/Y0AJwrbDV/nHlIlH2S5MF8HGhCjaTUO3Pgj9zQHzsiSlDlN +1cOrM6r5Z75mGKxnFF+TX2hIBO0eyXQoskc0SRHkezgy5dxAE7RpdTlD23xK8CuU +zQG6TZueaLPuA55/nAzPnFzL1DPV8OMhVJoTb+saD/OL6O/rNLoJdzkqikvhfp+1 +Bb6VtpJdTDVHOGQ4Xie4+TSUL1cWsPVqIT8JNLnd+NFHLMdef2NJ9Fv02epm/Kpk +J/By15Rvhg/nO7PU2TBnuKLD900HRLNwZ92xIax8KgR7LB3fC4Kp+9+Ickcc9V2j +91IiLer0KkVPm51jlVnzeQUr8Vs7YnFpkDDXerLI7GjllLuXANCVp/0EbPeLKMGW +m2qU +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.passwd.pem b/Lib/test/certdata/keycert.passwd.pem new file mode 100644 index 00000000000..1739a3525fe --- /dev/null +++ b/Lib/test/certdata/keycert.passwd.pem @@ -0,0 +1,69 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQEKGM7Z40p3TpyCvl +LTPLsQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEJAwQy7HESbbXMoF +KXadaz8EggcQBlxqFXmWQ3YlUuWTTc+dUcmikIUmI8gUSB2doBxQWLkTzaNuQ/hj +uV/eU7VkNFswxrU1j+EEkoCAtClWrQ2aKTyQSM4YfhgN1vQJdOD8DypHtd0TaRS3 +VCSkJODqcWaCEc/Ypgb1KnB6UWhit1waPq4NVCZdiSF/ueRK9/iMlIRaejg61Qn+ +LVLcegWw47zTtCZKuZqFAO3PcvfC0cresGc6hrQrb9tjUyI9qCR6MeZSrHUJQ/Pb +T9e7OREQxCLxWEUb0/tMJH5k9HoG7waU5lELOyjGKXsflEIs+uBaWIssWp69KGLi +yas7BHZc8kxJNtigFQlmECZjoKvRxJWN4fOknxI8T+ttBovOtsIlVvocBfmH+Z2m +7a5VMBxD8Phk6zU4Pk0L2S6EIUCIYGS+kgHibgxZ58/QcoDZBrKOaJ/Md+Lhk2Rn +2JCuPe8CBTkB3V+xAJdaz7JAcgxz4TqwXFXA249lfYb8qY9XXcr67O83Cxwb95lu +skmxnhAKOwKIrhS1nKiefuwN5qOiA+nuPbJFadCOO5cdOSriG3s80ugDDvlPN8k6 +4b7XLulB38R2PdH/OHuF05QGQ3kOUbJWatMig+/09LNo9FiMaCczdIXQ6LGxcHc1 +G1Y3BqK17z3yULqHvxU4tIbj5Elt+X1mfKIfifmXBUujbz5rR1pBTfpnk+DFGcHZ +B/cXWFP0tDJAE7mvna3HimNDQMP58NbhZwFtFdU3H9R5n4R5MsYQ6+a+amQ0pZ1Y +XAaQ/iEg0WY8CEFO8zEvB9R05zMm6vycNtN8CbfBq9CJ0OaR1ymMW8pRVagsLDuh +D0T5ZtWZctE2+ImlwlpGDs91CX3zDxvW/3Bwf40PF4x7LJMt/tFzqQEovokvzhUw +0jX/kf42QhvydnDoxqczdVZOfbjHELA21U+JAeAAj9jhcCEYd50c2BVt4jhST6dP +pzNAqxc1RGwTU3K3nZkWKt4hMXPDGjb0pVqiOuPz/718Nsd6ck+Ko90gFX13mPLC +6FLPGjNUmE6f3TKjbRrQz3+IWMncCyo/JSWA85rldxD0XAem64h+2XppFWJsxJD5 +nH2prLckQUWBLljWNIQQyVuAWo0TVq3NCXDFGaP6GHNrZo9zZxDhxcp6v2Me3Mir +few9aQNb59Q1/A0qwiCf26Oe8JYTD14UCez41imxD+w721SO8jzLR80+ReemdwzE +B9fXh3T/lrCIDNB+F+GWA9wQUz8mzArKbhqQhl5gk39dnLahc4aN7GrkwL4hjhKW +O3X5bBdhhRlcub9SbXFC+tx/w6G/0roGMSvD7h8G/mw3QcUUwwDvOuI7qAqQ8FC+ +xcGWeHUiTGxp2t2gz8DSaeL2TbBMbHe+tRBmKlmmIrAq3AEllWzyze9ta5ZYt46P +njZzS+vbbA0mfcFAId7tQOjQ2/ygZePs5hg91revLzNMDRJ1IeBHnrVUuDA5D1HR +iFW+hhj1Lx1s0XAJBJcDos8xy/LZlRDLxJSHrMktZOELTYh7UZw4S2PNkOPms4kP +V4G6D/0wZh0t+T9zpH57ivLdyBQ2fsFwx3X3JfQkYmoFlwCz+uUb368TunQ/yZZY +fVh9Q9ndk6y6aor0wkORsBefVv3eN+2BDt6ZUpYu3Liim+W4C/SiKVfQ+5yDb/qe +KmX7kcmr/f13BWvEOUN4fzCMiw/N0aAgEejZQTwMgivvGSMcNaBiLfdvM6hmy7NE +uvgpLtK5v+YRcOLwiaVH7UL6Q61CJ+pcDNRvbJT/iVwidYUO16zTVR6NrtPhM62a +Ziq3Q77HD4D2KCRPEo3Xivc5RwITHeDgKHwpLRB00jiZ3iR0Wm5lI5VbyWLnmQ4x +CDPAZetJZGBkbhTmUVbL0m3HNLHfF9g+LWA23L9mRNCKRemL0c0UJRRJPaNDIx8p +8h27al4xI8nFav0TZLxmSw7mqnzyE7YAoe4EsmedOSXktHWN06dnczHu8AWrgNjY +Izv9XUmy9V5vsRG/lg09FyG/eZSgfs9rdmL/qeFXbbl2J33YTqrt6O3ZaEsgg42r +piZztung8Xro3VlFSAXZGrl7Z0AWwzaOvwreCxbAG1WtBmGgLrs98z627XgfBFYA +BPvJwn9f2GZzixiN6M8c2M5XueDE/Vpn4A/GKLJ8RXXxKtbGRXFfdF/M6mEgcpa1 +9pxAhzNTPaOp+SFbm/cFa43HZYZgDg1D9zth2ZII0ZITd5OpEaSNOsrOrhqXeVQU +isBpybgPqVQ50xUuRUyoHYAZClVe6+PBYVbvlXBlTWhSPY3leUm4AYt+fjZ4VFlb +p4I2KRmd68zXXwl2spkEsC96e9k28kHta55dO32gSwaXUUHRKnFmKB24jsleF6n3 +BOKxzsO9jr6vi0jUqZXvaCnmYWqS84wZH/1S1pgotaWdMQ/KT27xwHk= +-----END ENCRYPTED PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem b/Lib/test/certdata/keycert.pem new file mode 100644 index 00000000000..43ad52c2af5 --- /dev/null +++ b/Lib/test/certdata/keycert.pem @@ -0,0 +1,67 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem.reference b/Lib/test/certdata/keycert.pem.reference new file mode 100644 index 00000000000..5528b98fff9 --- /dev/null +++ b/Lib/test/certdata/keycert.pem.reference @@ -0,0 +1,13 @@ +{'issuer': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'notAfter': 'Jan 29 11:51:12 2408 GMT', + 'notBefore': 'Oct 8 11:51:12 2024 GMT', + 'serialNumber': '4F1870457EB3050737FA5DE298392511BA9C0E92', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert2.pem b/Lib/test/certdata/keycert2.pem new file mode 100644 index 00000000000..63e2181ed83 --- /dev/null +++ b/Lib/test/certdata/keycert2.pem @@ -0,0 +1,67 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCtFmmyQ+HjjeBX +UE61fTenp2C9cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4v +ZADh/eIFOONvPebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5b +yhhpcOH1AmPCsiLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKTh +J+X8+PCY+FRObAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpO +G35/bW9jsQRpbeGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJ +izTuFV91pGmsXSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyU +Vr+odU0t7+YExBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp ++5cGWLOkcs8Xt0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAQKCAYAU +x+HCxFb6LLjS4tJrBHCf0VRSX9rf/7Ri2oYRXRUN1B3cTrZFeZyLdJyKggNiSy3E +GcF4lrxl2Xgaz8/cp3rxE5EYHv93hCQPZUbu4r2kZCLJsRPcGn3D/dyEKhCGyAi/ +fgsTkxyxnyK/9HXqhYyrZNu4rrh0SH63GKJ4GZsIFhZME95b0f6/e/3YJpSJFxKK +LF+xd/rjoWNOS62s1JqA/czpaXDqnJ0frNX5aQixLHnWUy/5qgRGNnY3/LOB0MuU +EjFizRSJp6gQQ64dIFgiMyUMi+Z6fw6xkTpN5EbmgFCwmaja1/hp9UZ+atlVpJsO +SB+OhF2wxB+bX2phGHJ9DD4RY/di1YKPvmH0+LITq4JiJWwJLgoCm7krzfVxZFZN +lgZPvdNw/Yt/Y3bN31FMw8DwRCThnL+MH83E58iCWQq2zKS1RssBBwFacI6Wf1ed +442NPIH47oi5qRKUxYHze6kFssSs4TkbR++VYHXLbTLjqfePLP92ZFxdOstx5jEC +gcEA48mx39dwr1oSk4uR+q8aDZIjPT+9D0IxcdAmL968QgG98hF2Tx7s5alR4y+9 +2FueR5fSqRweJfaDn77j5DVNXLD5jBz73pdwIkfqJHLfcsN7NnzCz3mxU9WP2F7R +W8KD3C2MtzGwQwOI3GT20+wJaA7p5n5zKR7AC0RxhC3U4zwH5nSjtY70A3dWXkn3 +F0j+B6ak0P0oN9VQUuvk5zw0IZ32uxef7poiHvFZpl1DQSgX/edT1CIT1s0nmVFl +w1ojAoHBAMKGX+TobGZfZFj/I7oIVQ7023pi516fZx/R0Y0FTmZzIKsdt3wrPmar +3K5dZGasNd0wzUsxX+vSjMtI1oERuPd4Rs6YqQHpX892uJ7L41/oBcFCc+Fk7jBY +TiRg/8/gOTaQe1nDR52K/R0sqBatRqInNMBDcBsVguBC4dTqP8CQ0CnEjAOYsk1s +1nYPKcfM36BUXBmSlnvUPWMm8KCT9Kko9ylHVq+d+ziDebfYpy6fTvti2NM31wTM +1prasEUaqQKBwQDQX14+5MapMd1SaVelmW5cwbVIvzjEb4npkj6MhdVzEELg4IZ4 +hFKzGDvXdoHVHKJi3YiQuC8ADUyE4kt4JCZbx2zQdmcVTTT/tweCRi8PvbDFvEU9 +JBZKGU+X38zmgr66uFRD9MlH1EDrU9TTMdW9Af+HoV7ZW87Tv82T25UmNXEIqORl +HpsrXIx+fmzxOQ1glFmq8BpNUO5EnJPtz43kvqrIpSjhTNAvvBqFbEUsom+oDWgK +4w2A7nTt9J8BoD8CgcBkYKa3HmBhazQC4JV097u8nglrXAH8R9EVEFZLqMNOBnaD +FjCKeF4Y6PJVX4fhm1eoLfihpnbS37EbbRiTPavutzgCf7AmdmCkU6Ts/FT2NmpR +0ZKuakCm3cpk51DZ2eBsEZ41MZmQ6Bm4pkSOfxeFsSl9VM9SioUgaCLUlZQUMCXa +h7ugV3kajuETxrtOiJ+UwjNMVuIkP971fTCKDA8iAyuXN2K5+JGcFewHPFr4qeg9 +vEIarCPeLD1JZzOyVRECgcEAhqeXfmFezNKGvr7qCflJe92LF9MStgJ7yE0OfADy +B+RZKeOwoqOO3VFR1piAn/DzrC2K9q+Of61gw+KWQOgMnsigqZ2mFLGChRjWhb7S +3G0DGOb4+DD9RR6wlFPFXSwVxSWGKrqNhOJik/IzVWvYxOOFjVt1adXZ3GftnYsv +nZCsS94H4kMiXr6UkbkjjxnZ1WkE9DaQJU27Mw1dtwb1ECzO0rty9B9L6b80MYW/ +nMhJV6sanCUex4nAChHD+VR7 +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIEjjCCAvagAwIBAgIUCP/QP57z62jCsjf6l0ft/yzDxOkwDQYJKoZIhvcNAQEL +BQAwYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhv +c3RuYW1lMCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBiMQswCQYD +VQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhv +biBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDDAxmYWtlaG9zdG5hbWUwggGi +MA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCtFmmyQ+HjjeBXUE61fTenp2C9 +cvHzZSTFvKNeKZ2ZVZzH1HFnMBlTXGlXybSi2PhfxNYo5JrNGW4vZADh/eIFOONv +PebQBONi4B1188Mb1RHgyIMtl/gOLF+L9DkStFb2eJ3fFInjNh5byhhpcOH1AmPC +siLeCAO3qc9qeChmJ4hAy1xmx74yQkOGzu0hKZ9d5QNNemCpPKThJ+X8+PCY+FRO +bAdKJ56er2BjkdaUB/Mp6BBLokYs917VdnH6TiicqfrUsjh36XpOG35/bW9jsQRp +beGkMIW6vn9GEYT79d4+FWdl9SxTfcSy7tn35/SvtIYDByWLc4IJizTuFV91pGms +XSav4YVF1hLC5cy8iEfKrdAl1kzamHfHKuY+P+sbwSQHPn5Z5xyUVr+odU0t7+YE +xBKLG8zHkQ+h2bwEeFy6H9uYQZFhBLOHrkkWSPYLKjzch6DyNkTp+5cGWLOkcs8X +t0mjG2gzUlgwIYWHkq7wE9jxz3/tJ3MGf2CNDxsCAwEAAaM6MDgwFwYDVR0RBBAw +DoIMZmFrZWhvc3RuYW1lMB0GA1UdDgQWBBSAf/Z+TUHMOcr020NGG4geSnlNhTAN +BgkqhkiG9w0BAQsFAAOCAYEAIia0ULQmC510p78HZbiOb2BV8q+0AY0l6Cn/FRIL +Voqy7uB5oEYdDisla8epmBTU35+JmfrRHbP6IuSqKdcGj8VqsjZljhntSrB7rw4O +IqjbnxnfEuREjY2w+WGLvvdtVGXCBgfRmIItORFKpoOvLzLIi1lXeDq8QL97K4nM +tPsZpzILCBCoXmhh0MweCCNe1HSD8q6EbSDjVoxraBl0YK/l1ID08Fi5fwdddRX7 +txfx5NtoWWbsLZY0GdBtQmRCifs7P9lyhjUd87bJGd8WCBE4IzLwJiHEyNPI4WHe +8jmvKD/zO7mKxT9/jF7IBqwTHgQQ1uaRGytiXjRlfPu4Ez8ASR6rP7Hlua7h0Ba7 +OcDPcgM8rzN4zcfsvZ+8Cd84HXFjHhV6ZdRPbBWFWd8TlDufrWNuY4SyOtkyfE5E +lWcoRnkBGBaooZNcd72ZwZVO0gAS31bgs8ju+k02GIDLkuZSRNPeFsRQbcMu1Mhc +cFteFsqniLhzpkWJVQQgeGlV +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem b/Lib/test/certdata/keycert3.pem new file mode 100644 index 00000000000..097dc9f8e88 --- /dev/null +++ b/Lib/test/certdata/keycert3.pem @@ -0,0 +1,161 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDuNnCqYWoguzKI +B3PwOsoRzI8wgNdxx3RiIJSItFISlAgB4XkYzPBpi5nFR8GCqq7IUQQ+iYwdZa3y +H6X11IdlrCx4gDdQFSFOegnG41oqv7GwBEFvq6xlowpoaGw+5/8/sJX/VU44rL9/ +fiMVTbehMhMS/z5nTSYSACV7sRXnwW4T6l2Od77FzPBjnaFFDNr5GIoiD4mNSJ35 +t05tliGcz0IyE5TGR6kIEFNIH5theHoJJ4JC4uLyWJweJpxDx10njWRwKN/tXm8F +nHQPMRAUl2wM07QZEI1yfzUr00/oG2dpPnPyNh+6p2ld8JS0g2/Pd2pQL8OtmM2c +DcN5E+qCEW1UVIJ5//ad8pUcoQwyRIBROJbakqv6FcVTdgTogoq5Vu+T6TVFLSIS +vVvpsV0MtT/H9/9TtCnqOwliE8s8kb3NjQT8e72VHNd43bVPE4DeQ+XlbMoKzV+G +OmSHSfzSOZB9kScSmGd5YD8783TbL8/7Nn1ERQdWqaixGI36+BECAwEAAQKCAYBV +Ubhevg9V89ZwdELpSxUu9NZgZ/VCck7UCplIsVUoBE8t5UULRfPhybdkuoOrulhp +tOLRR1ChAtcffohhmSJ5nwY6jqnBDCBmzD0OOEYGQ6xvv8Z0KcfQi2nh5WzHxy5b +8HJ5BmPC1tSr5FDKg5B+ssG0Lyl5tF8rWVQTjmSrIlO73Fhv+6GflPyQoVeutKEF +UO3Ar1H0AYtbcnUruPcHBBDQgMTrk6UOF1LM5U0wxwbmmnkEXeEtLeGUxv13JUMe +QTrSw0P+hn5uiDMwY5lI212ayorbwyNuzU0hGW7j98qa/S9MVSeRwv4bsbtBqwtT +ZVGL2TogjeJ+8+qDlv5tf3crD1pV0yu7uVUeIGrJZJdiiMM6N7/x3cIJtrLDaF1w +9kUqdQGWDWwRV3w0FaeXDVlUTJ3VFTZIinLo1Vmd3AXLn97eVy+68DWD6ELlMkfF +ro3bQbQu59DGswDF9+mMauhgvljnxY/wUnkyp8LGZt8w/RU/op8LcvPKFjQfX6UC +gcEA/RClguU/JTWQSF4Vjy5+DmrzSCXabrO7986lkOwPXEaXLsF2pGOU+sbxffXZ +PCQ4zU6aNYBMtNWLrVqBrPnI78XZQbmSG7zUDxWwmr1ttuSHGthdaZUVTOsFsnpE +R+1lj20G2T2ankzv0ICFz7BWyaYwyzDrABwuP189b3juuX6xGLlCO25/AnK+iTXQ +WF4ZURZiwDg4WpxvPsE2FFglN09TtDDCeQw/zv7Jh6jWtHZfivfv/wR8ILu72gde +LdinAoHBAPD5slojp2St6iWTaxcYYBc+4x+Rv7oyxeAyM/00iIAnf19oaAw3jki1 +yz6zeNnhaZuwa6ruvhJAESPc+pXkmoPpQ1yIRjKZc3jPVoaxYuSPgUEbHV6BQbjD +clIoGkC6ISsjanMq8Z3pK7e2bxY4mxGRQIFSFnTeK8m8nzq5dYisVtxxJFnV2fDG +NWDMdDYBh52vpkPjnm5OoOuilkEH2ygtZlT6bKAbnhi/FVjYCHk46inaYx8PakMu +LnvZI0OIhwKBwBGzYGBPeKM5o+Xr7sYdEmQfxvR88VJc8ADdS2dfm5NwvJJgpdPJ +w1nnIG0XDSLPxclWfiLP3o2ngiWV9wwKTKu4wwF94WJfStXjRn8MUOhCA9E04RPJ +gbvnlHZvZudBC6GElr4LOQ1phDypQLLOOsPQBAmyWj2fuvxjxQBPDSOcYPbBvog5 +qliZfgpK4U/NBShO0IlxZT+xQXa6PPYfVDsSKWCpKHEfEjeASshaXuowfW5S+U51 +GdmQSAtwCH5ccQKBwQCOLgW5gYfms2aPvSdWfR9VF9nSaqCBMCvoWDasky5mzucs +V+HsM2tUI09EM4h+pa02GyWruSmUgxCZ5GxFvJgedKc2FYG1oSysf0lCN69tw+4z +h9gQRpuMdGUjbF3xCuE/HqpUQWZGEamlv5JTvhpghx9ULibp1Zxob05Ty9E5TtYB +QxB7oN3yXkBoWLnIk6Z8t4KWU9rKosH3xfp5bDU2w3K5ePhWj3T8jOH/hZeaTqZ7 +A0uwq9u6v6jVkgxocEkCgcEAhyMPyiMqqq55qD6JlPY4zf4mWQJS4MnmWexH/5N1 +a6uogerwZnIjuH0/wNc2qzQSOwVJRqjlxRZbEkWST0JK/meXZ+e0nqaROWICutO6 +ciIf2wVjTzn1f85DHwPhUf3P2zIfpVFh+XuvSb32J+pzNwK40gMhirYMDSbsm/RQ +JrJHQwEX9BvI6IP5kaMOMlSwiG3Soo3MxeJWmozkz09+cs7BwMFsUKOdqruJHSaw +/Q06DQ9u9UyjmLyKv3IkN/cK +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2525 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:ee:36:70:aa:61:6a:20:bb:32:88:07:73:f0:3a: + ca:11:cc:8f:30:80:d7:71:c7:74:62:20:94:88:b4: + 52:12:94:08:01:e1:79:18:cc:f0:69:8b:99:c5:47: + c1:82:aa:ae:c8:51:04:3e:89:8c:1d:65:ad:f2:1f: + a5:f5:d4:87:65:ac:2c:78:80:37:50:15:21:4e:7a: + 09:c6:e3:5a:2a:bf:b1:b0:04:41:6f:ab:ac:65:a3: + 0a:68:68:6c:3e:e7:ff:3f:b0:95:ff:55:4e:38:ac: + bf:7f:7e:23:15:4d:b7:a1:32:13:12:ff:3e:67:4d: + 26:12:00:25:7b:b1:15:e7:c1:6e:13:ea:5d:8e:77: + be:c5:cc:f0:63:9d:a1:45:0c:da:f9:18:8a:22:0f: + 89:8d:48:9d:f9:b7:4e:6d:96:21:9c:cf:42:32:13: + 94:c6:47:a9:08:10:53:48:1f:9b:61:78:7a:09:27: + 82:42:e2:e2:f2:58:9c:1e:26:9c:43:c7:5d:27:8d: + 64:70:28:df:ed:5e:6f:05:9c:74:0f:31:10:14:97: + 6c:0c:d3:b4:19:10:8d:72:7f:35:2b:d3:4f:e8:1b: + 67:69:3e:73:f2:36:1f:ba:a7:69:5d:f0:94:b4:83: + 6f:cf:77:6a:50:2f:c3:ad:98:cd:9c:0d:c3:79:13: + ea:82:11:6d:54:54:82:79:ff:f6:9d:f2:95:1c:a1: + 0c:32:44:80:51:38:96:da:92:ab:fa:15:c5:53:76: + 04:e8:82:8a:b9:56:ef:93:e9:35:45:2d:22:12:bd: + 5b:e9:b1:5d:0c:b5:3f:c7:f7:ff:53:b4:29:ea:3b: + 09:62:13:cb:3c:91:bd:cd:8d:04:fc:7b:bd:95:1c: + d7:78:dd:b5:4f:13:80:de:43:e5:e5:6c:ca:0a:cd: + 5f:86:3a:64:87:49:fc:d2:39:90:7d:91:27:12:98: + 67:79:60:3f:3b:f3:74:db:2f:cf:fb:36:7d:44:45: + 07:56:a9:a8:b1:18:8d:fa:f8:11 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 16:22:C9:AA:20:D4:F4:8E:ED:9C:A6:BB:23:F5:F2:29:44:ED:F2:1E + X509v3 Authority Key Identifier: + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 43:82:1f:18:b4:cb:7a:b7:4e:ac:c3:65:2e:b7:57:3a:f0:78: + b3:b2:20:3e:bd:7b:24:f7:e7:17:cc:62:c0:b8:63:f0:2e:30: + 0c:54:be:c9:94:7c:54:d4:40:06:6e:92:9d:72:2c:44:f5:e3: + c1:4a:3f:26:e4:21:ca:ba:ef:4f:a3:50:56:ee:89:5c:dc:5b: + 98:bc:0c:e9:db:16:98:f0:4c:3f:65:b9:d9:e0:2e:bd:83:9e: + c7:1b:f7:4f:87:5f:75:ff:67:5d:0a:ff:ce:fa:0b:f1:d5:8b: + 58:23:2b:92:27:b8:15:ff:66:c2:f9:68:e2:ed:64:a0:4a:e0: + 42:8b:ac:35:6a:67:64:f5:cb:a9:00:37:20:56:dd:3a:02:6e: + d8:0f:b4:5d:ff:92:a3:0d:65:fc:2e:ab:4e:37:bc:7e:d6:5b: + 1c:55:78:a0:6e:a9:2e:55:31:cf:61:f0:1b:4c:fe:fe:60:75: + 76:e8:d9:a1:f5:14:f0:43:33:4d:52:05:be:33:ff:da:31:94: + bd:0d:e5:4d:a8:ee:d1:f3:92:34:c2:36:91:37:3a:6d:ec:20: + df:b3:9e:a2:5d:d0:f0:a1:e2:ff:b7:85:ce:09:e3:c8:4a:6a: + 93:a4:ae:ec:c1:67:42:f7:8e:2f:64:ca:d7:76:72:bf:3f:ef: + 64:ff:10:dd:a2:3e:2d:f5:aa:02:cf:31:f0:6d:93:6b:56:4c: + b0:88:95:54:dc:d4:ac:dc:c1:88:27:8b:d9:9c:4e:66:93:f8: + dd:5e:74:41:26:9a:ef:e8:c8:46:5c:d5:f4:fc:cd:cc:67:c7: + 33:62:2e:70:9b:d1:39:de:34:fd:ad:9d:b9:48:61:e7:89:53: + 62:62:f0:40:6c:68:2f:e1:da:40:61:2a:50:af:c9:41:6d:98: + 16:af:cb:63:03:ec:04:99:e0:39:98:60:ae:aa:6a:57:9b:25: + 73:03:16:bf:78:c8:d7:2f:eb:fc:64:7f:c7:78:8c:34:15:5b: + 66:40:98:86:e6:0a +-----BEGIN CERTIFICATE----- +MIIF8zCCBFugAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJ +bG9jYWxob3N0MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA7jZwqmFq +ILsyiAdz8DrKEcyPMIDXccd0YiCUiLRSEpQIAeF5GMzwaYuZxUfBgqquyFEEPomM +HWWt8h+l9dSHZawseIA3UBUhTnoJxuNaKr+xsARBb6usZaMKaGhsPuf/P7CV/1VO +OKy/f34jFU23oTITEv8+Z00mEgAle7EV58FuE+pdjne+xczwY52hRQza+RiKIg+J +jUid+bdObZYhnM9CMhOUxkepCBBTSB+bYXh6CSeCQuLi8licHiacQ8ddJ41kcCjf +7V5vBZx0DzEQFJdsDNO0GRCNcn81K9NP6BtnaT5z8jYfuqdpXfCUtINvz3dqUC/D +rZjNnA3DeRPqghFtVFSCef/2nfKVHKEMMkSAUTiW2pKr+hXFU3YE6IKKuVbvk+k1 +RS0iEr1b6bFdDLU/x/f/U7Qp6jsJYhPLPJG9zY0E/Hu9lRzXeN21TxOA3kPl5WzK +Cs1fhjpkh0n80jmQfZEnEphneWA/O/N02y/P+zZ9REUHVqmosRiN+vgRAgMBAAGj +ggHAMIIBvDAUBgNVHREEDTALgglsb2NhbGhvc3QwDgYDVR0PAQH/BAQDAgWgMB0G +A1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1Ud +DgQWBBQWIsmqINT0ju2cprsj9fIpRO3yHjB9BgNVHSMEdjB0gBTACisoQ95fyX1H +5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkA +yy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rl +c3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUF +BzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBD +BgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAQ4IfGLTLerdO +rMNlLrdXOvB4s7IgPr17JPfnF8xiwLhj8C4wDFS+yZR8VNRABm6SnXIsRPXjwUo/ +JuQhyrrvT6NQVu6JXNxbmLwM6dsWmPBMP2W52eAuvYOexxv3T4dfdf9nXQr/zvoL +8dWLWCMrkie4Ff9mwvlo4u1koErgQousNWpnZPXLqQA3IFbdOgJu2A+0Xf+Sow1l +/C6rTje8ftZbHFV4oG6pLlUxz2HwG0z+/mB1dujZofUU8EMzTVIFvjP/2jGUvQ3l +Taju0fOSNMI2kTc6bewg37Oeol3Q8KHi/7eFzgnjyEpqk6Su7MFnQveOL2TK13Zy +vz/vZP8Q3aI+LfWqAs8x8G2Ta1ZMsIiVVNzUrNzBiCeL2ZxOZpP43V50QSaa7+jI +RlzV9PzNzGfHM2IucJvROd40/a2duUhh54lTYmLwQGxoL+HaQGEqUK/JQW2YFq/L +YwPsBJngOZhgrqpqV5slcwMWv3jI1y/r/GR/x3iMNBVbZkCYhuYK +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem.reference b/Lib/test/certdata/keycert3.pem.reference new file mode 100644 index 00000000000..84d2ca29953 --- /dev/null +++ b/Lib/test/certdata/keycert3.pem.reference @@ -0,0 +1,15 @@ +{'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Oct 28 14:23:16 2525 GMT', + 'notBefore': 'Aug 29 14:23:16 2018 GMT', + 'serialNumber': 'CB2D80995A69525C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3} diff --git a/Lib/test/certdata/keycert4.pem b/Lib/test/certdata/keycert4.pem new file mode 100644 index 00000000000..fcaa3a1656d --- /dev/null +++ b/Lib/test/certdata/keycert4.pem @@ -0,0 +1,161 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC/uCI/4eVSCIrx +//GcCCxeQP16Mtf0wAuINBR9uTySF8sEVoS6rNBwLeudZXaHmcp3jM7N2e9GjI8d +dQccPApKBwjAxhdT4XeuUTep4VraKWjLTWDvABHfCVFB1tX3HusEWaS8n/pH+N4L +UfnxS3HnFqGC5ib2DSgFTCnLQSj82iUgACdf87iZTons+kRvqYsdwpsgBUnO+w+o +c5o7Nj1bJJlTREsY6kQGY94as4pCkmqhoBTC7/le9wFGmXipRYi3pfSTOqAQJfqW +1VlQ4Bbc3lN5L+ONTmFXOgTJ5qhaS8cglnYElJI1QQTagyWQMHuBDIMpr3Iv/5qU +UzdIMzoMrpX6ien3Gan/jyjjzvP95jJvGujQudw/myXK8VOUThVtstRDFA3cIoV4 +UqMODz8w6qYU0GPYFBq//1UdpDrk/fBPBe3MLJT2+sB/kTzwxxYsKqcdjPG2Bt3Y +2nk7hCzE0KuueqCQPk/3fomY1TDyTlWOriAS40EYue5Rhx9qr8kCAwEAAQKCAYAL +X7EoeQ5Rv4/+q2B6EUIQlWp0RW/qZTpJ6k0M6GBfimnQ6BEXwgjbnt3LiKlvggAw +93mNXNCFLvGOndK+KhGyMpiiVx4rK8Ud4lObEHODXdGJvh1yEF7/DF51uXkYIA1x +RKAxUIxYmLsTkNlzJzaqrv0F9wF4t28YYVxZYpQ76/Un4Np1JtBcx/wGwxIsTbKj +IVhynd2zGdHj/He5643YSmOOPQ73e25tsp8KlnwtVuJclFKm/fWdXKoGtjQiVq2b +bViG7WTqb+a+R3IrO8CE2vAQ1ZqklP4Pg33AU1XFogRweqSkXvBGy1GHSSXCO4uX +gORj/bo/gAZ/eeufwqxBxJMS4Kbmxes/oYEgJidoaeA6DZjcKr+J3rFUt9j6rpxt +abeKBiK6wTHkKAdQwoccDdKFROvL4ZEu4zpjkmiqDrtIZEqtB8YCvoMpteIqrBhr +stLCDrftsysBTNGNn0THb4H5wx8nkZPCVyvaBD4N4Skp5rapP2bbrPhigcOoR8EC +gcEA4ToVg2b09tLWpmCg6XXssT1SvvFcSaJ3jjRKMfpmP5epWFiZ21qAOCbYhMNu +2G21AyNd1W7oVi6epXsB2YBOlSI0Fr6RuN0fDwg5jZLxEkEqDsL2QKsgoKHtGfBn +gX9jLQ5Lsy5/0CRXYCfPjdccbr2J5Gm2fU/jhhCi9ji/6b2RbgrRkIkr2I/J6KbS +sO+p1d93CFNagLMbdDspugDlIgj52OBMYO60nDlNU8O2pkAcLQTpWVqEjmoo4Q4K +oyOXAoHBANnqB93sLCndEiZNth7cOOEhNKCWRDkpGwDBbn2Dj3kBaXg8eCTG7tiF +FPyNVydYg+u0vcPIjj9URsOfpHc9MLSYtKAgCL3rSk+OQnROeg9hCjmB5Ucg7NTR +gSh/flfWp/q1EDeOVAu8B9DuBX1209br4xi78LnBP5/UzRjhuFEmvCvOqVCMCI7Y +I1k3l4NhOk87fKq5dbtxvNTU2eFTtt1+YjYwZFY+AUmx3WBz+aFwygks3S4SvN1J +LtLVYhOznwKBwQDC047WixIuDLX3WDD5orOroeNZHsn5PFv1HBBuaS9XpSatMH9u ++ztc12WGetQAze2+GDLMNNMv8cX0WZKBBfd0FBFA93pwkn6Sb0fxyoFUjCAIguen +iyB/M3M5c/blUz+EMxCSoA+aCkW2/NkS1lhXBwgoGLXuclPbnbqKCQ8h74TEzwD2 +6WGPRNqgsOYifj7IrjR2dDwehlCiW6c9qhaLOX5+94+6beK4HO1iHzN5Xo3A97Wv +QJjX5McV3yKeemMCgcA4N3reEo52IlULUqL4JSH7WkCkaP+iq2sO79fcQ3Ng6S9X +WGo6OqPlcbevS5s/SEOILDGEb5na1pgG4YlhRYTqIjb+1CTNMgUSrwWP0asFiqhD +m7IVfnX6lS23z+Q9LuBY+hr76hjeihyOFsmNy3jtCh+lAt8gXK1YQ2LB14FgVhjX +SFI/uFCA4VuFKaVJvGx5gkQwGvY3bCkl0t9+lMUpMPCPQD6yTP6yD1OoDWNJ9bn5 +UfyhZS4Z/EY7F9dcc8sCgcEAi8yW4gNCZSsGQRLh+fyDCSdak4ahATAeiI24231u +jK5+ncK9ZoZvzO/h+V5bw7ha59rD8aGmrLqzBVKEMi0PAT38psNH9ZtRt9y29YQ8 +WpyHtODMbphfbH4SFH3nj3PldRtBcOt0EDdp/4s64A9qdOWRx/vlJhdg0vtWTy5s +koyyOHhTcplpyPPq0BetYKl9UlXPVjus/RT1XV/lz2YrE8E9UAq+/r7ECoMExaZt +8j8J8O3lTWPxNTaQuiPiHA7E +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5d + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2525 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:bf:b8:22:3f:e1:e5:52:08:8a:f1:ff:f1:9c:08: + 2c:5e:40:fd:7a:32:d7:f4:c0:0b:88:34:14:7d:b9: + 3c:92:17:cb:04:56:84:ba:ac:d0:70:2d:eb:9d:65: + 76:87:99:ca:77:8c:ce:cd:d9:ef:46:8c:8f:1d:75: + 07:1c:3c:0a:4a:07:08:c0:c6:17:53:e1:77:ae:51: + 37:a9:e1:5a:da:29:68:cb:4d:60:ef:00:11:df:09: + 51:41:d6:d5:f7:1e:eb:04:59:a4:bc:9f:fa:47:f8: + de:0b:51:f9:f1:4b:71:e7:16:a1:82:e6:26:f6:0d: + 28:05:4c:29:cb:41:28:fc:da:25:20:00:27:5f:f3: + b8:99:4e:89:ec:fa:44:6f:a9:8b:1d:c2:9b:20:05: + 49:ce:fb:0f:a8:73:9a:3b:36:3d:5b:24:99:53:44: + 4b:18:ea:44:06:63:de:1a:b3:8a:42:92:6a:a1:a0: + 14:c2:ef:f9:5e:f7:01:46:99:78:a9:45:88:b7:a5: + f4:93:3a:a0:10:25:fa:96:d5:59:50:e0:16:dc:de: + 53:79:2f:e3:8d:4e:61:57:3a:04:c9:e6:a8:5a:4b: + c7:20:96:76:04:94:92:35:41:04:da:83:25:90:30: + 7b:81:0c:83:29:af:72:2f:ff:9a:94:53:37:48:33: + 3a:0c:ae:95:fa:89:e9:f7:19:a9:ff:8f:28:e3:ce: + f3:fd:e6:32:6f:1a:e8:d0:b9:dc:3f:9b:25:ca:f1: + 53:94:4e:15:6d:b2:d4:43:14:0d:dc:22:85:78:52: + a3:0e:0f:3f:30:ea:a6:14:d0:63:d8:14:1a:bf:ff: + 55:1d:a4:3a:e4:fd:f0:4f:05:ed:cc:2c:94:f6:fa: + c0:7f:91:3c:f0:c7:16:2c:2a:a7:1d:8c:f1:b6:06: + dd:d8:da:79:3b:84:2c:c4:d0:ab:ae:7a:a0:90:3e: + 4f:f7:7e:89:98:d5:30:f2:4e:55:8e:ae:20:12:e3: + 41:18:b9:ee:51:87:1f:6a:af:c9 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:fakehostname + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + E4:3D:4C:DB:96:85:CC:90:ED:0F:58:1E:8D:4D:7D:34:6C:4E:11:10 + X509v3 Authority Key Identifier: + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 63:e9:cd:0f:ec:dd:27:a0:fc:76:16:a1:f2:5f:d3:07:9a:3f: + 97:f4:11:e8:20:e9:3c:dd:16:4f:20:62:71:73:a9:46:3e:85: + 6e:44:be:9d:08:4b:eb:80:08:d5:1b:93:5f:4a:0f:df:b7:1b: + 38:e2:2b:5c:68:48:ea:0a:58:5c:36:6c:79:a4:1e:b5:7e:ef: + cb:8d:5d:bd:7d:4a:e6:4e:dd:0b:87:10:ff:01:0e:9b:8b:bd: + de:1c:9a:25:fb:a2:e1:52:7b:8a:aa:08:37:b2:87:f7:45:9d: + 0b:ab:11:6b:0c:7f:db:ed:de:cc:1e:86:f0:be:30:25:6b:4b: + ff:f5:f2:99:ed:b0:b1:68:c1:44:0b:79:7b:81:95:b5:36:37: + 12:c4:99:9e:85:e8:2f:2d:cc:cd:d8:c6:f2:20:ec:6a:06:fd: + b8:fc:ff:02:11:95:4d:38:0d:42:8b:b2:43:eb:c2:b9:a4:e0: + 33:f1:da:25:2f:11:cf:57:b9:25:e5:ab:92:f2:b2:b5:11:2f: + bd:31:f7:55:eb:96:94:78:5a:ad:9c:8f:56:ec:34:a0:73:9c: + 01:c2:76:24:e0:f8:b3:c9:23:b9:ea:ab:c3:0a:d3:f8:53:44: + c7:12:37:76:44:d7:ee:25:93:f4:1c:1d:7c:fe:06:2c:fa:c5: + bb:c6:90:a4:57:fd:09:2e:da:af:f3:ac:e6:6d:7f:03:a1:26: + 36:9a:51:7d:8d:28:a7:5d:5d:37:57:cc:6a:11:2a:98:1d:57: + 71:32:6e:98:7c:12:28:ef:5c:2f:26:29:d4:56:0a:b1:23:d9: + 9d:35:20:1a:ec:cc:51:53:6e:ef:1c:e7:bc:3c:21:e7:64:31: + b1:4c:f3:55:b5:c4:93:17:d8:72:5a:05:2e:4a:e5:33:07:0e: + 7d:cf:22:73:0b:67:9f:b4:82:60:cd:71:f8:76:0c:c4:dc:98: + ee:49:f9:03:7f:0d:d8:c6:76:79:3c:28:ed:77:77:d3:7e:d5: + aa:1e:6e:2e:df:a5 +-----BEGIN CERTIFICATE----- +MIIF+TCCBGGgAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwM +ZmFrZWhvc3RuYW1lMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAv7gi +P+HlUgiK8f/xnAgsXkD9ejLX9MALiDQUfbk8khfLBFaEuqzQcC3rnWV2h5nKd4zO +zdnvRoyPHXUHHDwKSgcIwMYXU+F3rlE3qeFa2iloy01g7wAR3wlRQdbV9x7rBFmk +vJ/6R/jeC1H58Utx5xahguYm9g0oBUwpy0Eo/NolIAAnX/O4mU6J7PpEb6mLHcKb +IAVJzvsPqHOaOzY9WySZU0RLGOpEBmPeGrOKQpJqoaAUwu/5XvcBRpl4qUWIt6X0 +kzqgECX6ltVZUOAW3N5TeS/jjU5hVzoEyeaoWkvHIJZ2BJSSNUEE2oMlkDB7gQyD +Ka9yL/+alFM3SDM6DK6V+onp9xmp/48o487z/eYybxro0LncP5slyvFTlE4VbbLU +QxQN3CKFeFKjDg8/MOqmFNBj2BQav/9VHaQ65P3wTwXtzCyU9vrAf5E88McWLCqn +HYzxtgbd2Np5O4QsxNCrrnqgkD5P936JmNUw8k5Vjq4gEuNBGLnuUYcfaq/JAgMB +AAGjggHDMIIBvzAXBgNVHREEEDAOggxmYWtlaG9zdG5hbWUwDgYDVR0PAQH/BAQD +AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAA +MB0GA1UdDgQWBBTkPUzbloXMkO0PWB6NTX00bE4REDB9BgNVHSMEdjB0gBTACiso +Q95fyX1H5UebNvJljGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5 +dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2Vy +dmVyggkAyy2AmVppUlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0 +cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUG +CCsGAQUFBzABhilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9v +Y3NwLzBDBgNVHR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3Qu +bmV0L3Rlc3RjYS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAY+nN +D+zdJ6D8dhah8l/TB5o/l/QR6CDpPN0WTyBicXOpRj6FbkS+nQhL64AI1RuTX0oP +37cbOOIrXGhI6gpYXDZseaQetX7vy41dvX1K5k7dC4cQ/wEOm4u93hyaJfui4VJ7 +iqoIN7KH90WdC6sRawx/2+3ezB6G8L4wJWtL//Xyme2wsWjBRAt5e4GVtTY3EsSZ +noXoLy3MzdjG8iDsagb9uPz/AhGVTTgNQouyQ+vCuaTgM/HaJS8Rz1e5JeWrkvKy +tREvvTH3VeuWlHharZyPVuw0oHOcAcJ2JOD4s8kjueqrwwrT+FNExxI3dkTX7iWT +9BwdfP4GLPrFu8aQpFf9CS7ar/Os5m1/A6EmNppRfY0op11dN1fMahEqmB1XcTJu +mHwSKO9cLyYp1FYKsSPZnTUgGuzMUVNu7xznvDwh52QxsUzzVbXEkxfYcloFLkrl +MwcOfc8icwtnn7SCYM1x+HYMxNyY7kn5A38N2MZ2eTwo7Xd3037Vqh5uLt+l +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycertecc.pem b/Lib/test/certdata/keycertecc.pem new file mode 100644 index 00000000000..d503d49dc85 --- /dev/null +++ b/Lib/test/certdata/keycertecc.pem @@ -0,0 +1,103 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDBuhPMBtySJ8hFhn5hU +1yvdDujmB8ajjXNlkQ9GhGdghQIAfKmP9vWtSY+1+BRjpAahZANiAAQKLvQoetPb +S5tbgExB5vSDUtgc7hK1j6AN++En6AkW8KkK+O/31jkItqBoSQz8ts+VCU5cloTf +H6i5AS9zUkHr4DEyn+XxfiutzcNQPdQIBmygWAsUpa1XlZf8ExMFf/A= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5e + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2525 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc + Subject Public Key Info: + Public Key Algorithm: id-ecPublicKey + Public-Key: (384 bit) + pub: + 04:0a:2e:f4:28:7a:d3:db:4b:9b:5b:80:4c:41:e6: + f4:83:52:d8:1c:ee:12:b5:8f:a0:0d:fb:e1:27:e8: + 09:16:f0:a9:0a:f8:ef:f7:d6:39:08:b6:a0:68:49: + 0c:fc:b6:cf:95:09:4e:5c:96:84:df:1f:a8:b9:01: + 2f:73:52:41:eb:e0:31:32:9f:e5:f1:7e:2b:ad:cd: + c3:50:3d:d4:08:06:6c:a0:58:0b:14:a5:ad:57:95: + 97:fc:13:13:05:7f:f0 + ASN1 OID: secp384r1 + NIST CURVE: P-384 + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost-ecc + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + F3:39:E5:90:3F:58:6F:55:9D:91:D4:86:B8:1B:14:35:09:AC:06:97 + X509v3 Authority Key Identifier: + keyid:C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + b8:d9:bf:b8:88:0a:01:2b:aa:64:32:e8:97:e1:0f:ee:34:40: + ef:71:fc:e5:f4:a2:3b:26:00:e2:19:3b:3e:cb:8e:2b:51:4c: + 30:ff:ab:44:9d:28:8d:d2:9c:32:e3:6b:96:73:1c:9d:55:76: + b2:bf:af:b8:51:ed:fd:04:e7:0d:fa:8c:2a:68:01:cb:92:90: + 7d:d1:31:d8:6c:f4:b8:4b:ea:62:59:1e:31:4e:f7:17:ae:3f: + f0:b6:ff:f8:6e:64:e3:6e:e9:19:4f:d0:1e:84:1e:df:56:49: + ae:90:a9:e0:7b:70:e6:97:7f:08:f2:03:49:82:7d:58:8e:a5: + a0:88:59:f3:a1:ab:b7:dd:6b:9a:2b:79:64:9c:d3:07:7c:ec: + a3:8d:56:77:28:10:c9:42:a5:fa:81:8e:35:ad:f7:28:da:58: + cc:a0:61:fa:0d:75:fd:26:ab:07:88:c5:64:21:f1:98:1c:e3: + 13:51:38:a7:59:a5:59:88:b9:10:4d:c1:79:70:3f:36:9e:08: + 75:53:7f:77:a3:3d:5d:16:b7:3b:a2:f6:eb:41:9b:56:16:80: + 34:96:57:f0:3a:3d:91:43:17:51:2b:64:2c:d7:e2:25:45:85: + f3:5f:73:f8:7d:aa:a3:6d:61:e8:ba:c5:90:5e:16:50:bc:79: + b9:4e:df:66:db:ae:95:80:15:da:4e:bf:8f:4e:9d:e3:7d:b0: + ab:8b:72:93:3c:56:1b:92:d3:67:07:d5:5a:ad:98:16:69:ea: + 46:fe:e0:d0:f4:ae:95:d3:b5:80:7d:f2:64:4d:14:c0:97:d2: + f3:91:bb:e6:43:d0:7e:39:14:0b:95:ab:c1:56:fd:26:79:f8: + 8c:69:eb:bc:74:0e:ea:cd:94:62:7e:30:58:01:7f:84:ee:9d: + 58:9c:fc:8f:75:1e:d9:86:cb:f4:fd:4c:af:b2:f7:69:ea:58: + cb:7c:93:ce:3d:86:f0:46:01:df:86:02:a1:6c:0f:fb:13:84: + d6:75:23:f9:17:21 +-----BEGIN CERTIFICATE----- +MIIEzTCCAzWgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowYzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEWMBQGA1UEAwwN +bG9jYWxob3N0LWVjYzB2MBAGByqGSM49AgEGBSuBBAAiA2IABAou9Ch609tLm1uA +TEHm9INS2BzuErWPoA374SfoCRbwqQr47/fWOQi2oGhJDPy2z5UJTlyWhN8fqLkB +L3NSQevgMTKf5fF+K63Nw1A91AgGbKBYCxSlrVeVl/wTEwV/8KOCAcQwggHAMBgG +A1UdEQQRMA+CDWxvY2FsaG9zdC1lY2MwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTz +OeWQP1hvVZ2R1Ia4GxQ1CawGlzB9BgNVHSMEdjB0gBTACisoQ95fyX1H5UebNvJl +jGc74qFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp +UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w +eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo +dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E +PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y +ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAuNm/uIgKASuqZDLol+EP +7jRA73H85fSiOyYA4hk7PsuOK1FMMP+rRJ0ojdKcMuNrlnMcnVV2sr+vuFHt/QTn +DfqMKmgBy5KQfdEx2Gz0uEvqYlkeMU73F64/8Lb/+G5k427pGU/QHoQe31ZJrpCp +4Htw5pd/CPIDSYJ9WI6loIhZ86Grt91rmit5ZJzTB3zso41WdygQyUKl+oGONa33 +KNpYzKBh+g11/SarB4jFZCHxmBzjE1E4p1mlWYi5EE3BeXA/Np4IdVN/d6M9XRa3 +O6L260GbVhaANJZX8Do9kUMXUStkLNfiJUWF819z+H2qo21h6LrFkF4WULx5uU7f +ZtuulYAV2k6/j06d432wq4tykzxWG5LTZwfVWq2YFmnqRv7g0PSuldO1gH3yZE0U +wJfS85G75kPQfjkUC5WrwVb9Jnn4jGnrvHQO6s2UYn4wWAF/hO6dWJz8j3Ue2YbL +9P1Mr7L3aepYy3yTzj2G8EYB34YCoWwP+xOE1nUj+Rch +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.ca.pem b/Lib/test/certdata/leaf-missing-aki.ca.pem new file mode 100644 index 00000000000..36b202ae02e --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.ca.pem @@ -0,0 +1,13 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN CERTIFICATE----- +MIIBkDCCATWgAwIBAgIUGjIb/aYm9u9fBh2o4GAYRJwk5XIwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAaMRgwFgYDVQQDDA94NTA5LWxpbWJvLXJvb3QwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAARUzBhjMOkO911U65Fvs4YmL1YPNj63P9Fa+g9U +KrUqiIy8WjaDXdIe8g8Zj0TalpbU1gYCs3atteMxgIp6qxwHo1cwVTAPBgNVHRMB +Af8EBTADAQH/MAsGA1UdDwQEAwICBDAWBgNVHREEDzANggtleGFtcGxlLmNvbTAd +BgNVHQ4EFgQUcv1fyqgezMGzmo+lhmUkdUuAbIowCgYIKoZIzj0EAwIDSQAwRgIh +AIOErPSRlWpnyMub9UgtPF/lSzdvnD4Q8KjLQppHx6oPAiEA373p4L/HvUbs0xg8 +6/pLyn0RT02toKKJcMV3ChohLtM= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.keycert.pem b/Lib/test/certdata/leaf-missing-aki.keycert.pem new file mode 100644 index 00000000000..0fd2ab39bf2 --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.keycert.pem @@ -0,0 +1,18 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIF5Re+/FP3rg+7c1odKEQPXhb9V65kXnlZIWHDG9gKrLoAoGCCqGSM49 +AwEHoUQDQgAE1WAQMdC7ims7T9lpK9uzaCuKqHb/oNMbGjh1f10pOHv3Z+oAvsqF +Sv3hGzreu69YLy01afA6sUCf1AA/95dKkg== +-----END EC PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIBjjCCATWgAwIBAgIUVlBgclml+OXlrWzZfcgYCiNm96UwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAWMRQwEgYDVQQDDAtleGFtcGxlLmNvbTBZMBMGByqGSM49 +AgEGCCqGSM49AwEHA0IABNVgEDHQu4prO0/ZaSvbs2griqh2/6DTGxo4dX9dKTh7 +92fqAL7KhUr94Rs63ruvWC8tNWnwOrFAn9QAP/eXSpKjWzBZMB0GA1UdDgQWBBS3 +yYRQQwo3syjGVQ8Yw7/XRZHbpzALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwFgYDVR0RBA8wDYILZXhhbXBsZS5jb20wCgYIKoZIzj0EAwIDRwAwRAIg +BVq7lw4Y5MPEyisPhowMWd4KnERupdM5qeImDO+dD7ICIE/ksd6Wz1b8rMAfllNV +yiYst9lfwTd2SkFgdDNUDFud +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/make_ssl_certs.py b/Lib/test/certdata/make_ssl_certs.py new file mode 100644 index 00000000000..18e61449638 --- /dev/null +++ b/Lib/test/certdata/make_ssl_certs.py @@ -0,0 +1,326 @@ +"""Make the custom certificate and private key files used by test_ssl +and friends.""" + +import argparse +import os +import pprint +import shutil +import tempfile +from subprocess import * + +startdate = "20180829142316Z" +enddate_default = "25251028142316Z" +days_default = "140000" + +req_template = """ + [ default ] + base_url = http://testca.pythontest.net/testca + + [req] + distinguished_name = req_distinguished_name + prompt = no + + [req_distinguished_name] + C = XY + L = Castle Anthrax + O = Python Software Foundation + CN = {hostname} + + [req_x509_extensions_nosan] + + [req_x509_extensions_simple] + subjectAltName = @san + + [req_x509_extensions_full] + subjectAltName = @san + keyUsage = critical,keyEncipherment,digitalSignature + extendedKeyUsage = serverAuth,clientAuth + basicConstraints = critical,CA:false + subjectKeyIdentifier = hash + authorityKeyIdentifier = keyid:always,issuer:always + authorityInfoAccess = @issuer_ocsp_info + crlDistributionPoints = @crl_info + + [ issuer_ocsp_info ] + caIssuers;URI.0 = $base_url/pycacert.cer + OCSP;URI.0 = $base_url/ocsp/ + + [ crl_info ] + URI.0 = $base_url/revocation.crl + + [san] + DNS.1 = {hostname} + {extra_san} + + [dir_sect] + C = XY + L = Castle Anthrax + O = Python Software Foundation + CN = dirname example + + [princ_name] + realm = EXP:0, GeneralString:KERBEROS.REALM + principal_name = EXP:1, SEQUENCE:principal_seq + + [principal_seq] + name_type = EXP:0, INTEGER:1 + name_string = EXP:1, SEQUENCE:principals + + [principals] + princ1 = GeneralString:username + + [ ca ] + default_ca = CA_default + + [ CA_default ] + dir = cadir + database = $dir/index.txt + crlnumber = $dir/crl.txt + default_md = sha256 + startdate = {startdate} + default_startdate = {startdate} + enddate = {enddate} + default_enddate = {enddate} + default_days = {days} + default_crl_days = {days} + certificate = pycacert.pem + private_key = pycakey.pem + serial = $dir/serial + RANDFILE = $dir/.rand + policy = policy_match + + [ policy_match ] + countryName = match + stateOrProvinceName = optional + organizationName = match + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + [ policy_anything ] + countryName = optional + stateOrProvinceName = optional + localityName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + + [ v3_ca ] + + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid:always,issuer + basicConstraints = critical, CA:true + keyUsage = critical, digitalSignature, keyCertSign, cRLSign + + """ + +here = os.path.abspath(os.path.dirname(__file__)) + + +def make_cert_key(cmdlineargs, hostname, sign=False, extra_san='', + ext='req_x509_extensions_full', key='rsa:3072'): + print("creating cert for " + hostname) + tempnames = [] + for i in range(3): + with tempfile.NamedTemporaryFile(delete=False) as f: + tempnames.append(f.name) + req_file, cert_file, key_file = tempnames + try: + req = req_template.format( + hostname=hostname, + extra_san=extra_san, + startdate=startdate, + enddate=cmdlineargs.enddate, + days=cmdlineargs.days + ) + with open(req_file, 'w') as f: + f.write(req) + args = ['req', '-new', '-nodes', '-days', cmdlineargs.days, + '-newkey', key, '-keyout', key_file, + '-config', req_file] + if sign: + with tempfile.NamedTemporaryFile(delete=False) as f: + tempnames.append(f.name) + reqfile = f.name + args += ['-out', reqfile ] + + else: + args += ['-extensions', ext, '-x509', '-out', cert_file ] + check_call(['openssl'] + args) + + if sign: + args = [ + 'ca', + '-config', req_file, + '-extensions', ext, + '-out', cert_file, + '-outdir', 'cadir', + '-policy', 'policy_anything', + '-batch', '-infiles', reqfile + ] + check_call(['openssl'] + args) + + + with open(cert_file, 'r') as f: + cert = f.read() + with open(key_file, 'r') as f: + key = f.read() + return cert, key + finally: + for name in tempnames: + os.remove(name) + +TMP_CADIR = 'cadir' + +def unmake_ca(): + shutil.rmtree(TMP_CADIR) + +def make_ca(cmdlineargs): + os.mkdir(TMP_CADIR) + with open(os.path.join('cadir','index.txt'),'a+') as f: + pass # empty file + with open(os.path.join('cadir','crl.txt'),'a+') as f: + f.write("00") + with open(os.path.join('cadir','index.txt.attr'),'w+') as f: + f.write('unique_subject = no') + # random start value for serial numbers + with open(os.path.join('cadir','serial'), 'w') as f: + f.write('CB2D80995A69525B\n') + + with tempfile.NamedTemporaryFile("w") as t: + req = req_template.format( + hostname='our-ca-server', + extra_san='', + startdate=startdate, + enddate=cmdlineargs.enddate, + days=cmdlineargs.days + ) + t.write(req) + t.flush() + with tempfile.NamedTemporaryFile() as f: + args = ['req', '-config', t.name, '-new', + '-nodes', + '-newkey', 'rsa:3072', + '-keyout', 'pycakey.pem', + '-out', f.name, + '-subj', '/C=XY/L=Castle Anthrax/O=Python Software Foundation CA/CN=our-ca-server'] + check_call(['openssl'] + args) + args = ['ca', '-config', t.name, + '-out', 'pycacert.pem', '-batch', '-outdir', TMP_CADIR, + '-keyfile', 'pycakey.pem', + '-selfsign', '-extensions', 'v3_ca', '-infiles', f.name ] + check_call(['openssl'] + args) + args = ['ca', '-config', t.name, '-gencrl', '-out', 'revocation.crl'] + check_call(['openssl'] + args) + + # capath hashes depend on subject! + check_call([ + 'openssl', 'x509', '-in', 'pycacert.pem', '-out', 'capath/ceff1710.0' + ]) + shutil.copy('capath/ceff1710.0', 'capath/b1930218.0') + + +def write_cert_reference(path): + import _ssl + refdata = pprint.pformat(_ssl._test_decode_cert(path)) + print(refdata) + with open(path + '.reference', 'w') as f: + print(refdata, file=f) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Make the custom certificate and private key files used by test_ssl and friends.') + parser.add_argument('--days', default=days_default) + parser.add_argument('--enddate', default=enddate_default) + cmdlineargs = parser.parse_args() + + os.chdir(here) + cert, key = make_cert_key(cmdlineargs, 'localhost', ext='req_x509_extensions_simple') + with open('ssl_cert.pem', 'w') as f: + f.write(cert) + with open('ssl_key.pem', 'w') as f: + f.write(key) + print("password protecting ssl_key.pem in ssl_key.passwd.pem") + check_call(['openssl','pkey','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-aes256','-passout','pass:somepass']) + check_call(['openssl','pkey','-in','ssl_key.pem','-out','keycert.passwd.pem','-aes256','-passout','pass:somepass']) + + with open('keycert.pem', 'w') as f: + f.write(key) + f.write(cert) + + with open('keycert.passwd.pem', 'a+') as f: + f.write(cert) + + # For certificate matching tests + make_ca(cmdlineargs) + cert, key = make_cert_key(cmdlineargs, 'fakehostname', ext='req_x509_extensions_simple') + with open('keycert2.pem', 'w') as f: + f.write(key) + f.write(cert) + + cert, key = make_cert_key(cmdlineargs, 'localhost', sign=True) + with open('keycert3.pem', 'w') as f: + f.write(key) + f.write(cert) + + check_call(['openssl', 'x509', '-outform', 'pem', '-in', 'keycert3.pem', '-out', 'cert3.pem']) + + cert, key = make_cert_key(cmdlineargs, 'fakehostname', sign=True) + with open('keycert4.pem', 'w') as f: + f.write(key) + f.write(cert) + + cert, key = make_cert_key( + cmdlineargs, 'localhost-ecc', sign=True, key='param:secp384r1.pem' + ) + with open('keycertecc.pem', 'w') as f: + f.write(key) + f.write(cert) + + extra_san = [ + 'otherName.1 = 1.2.3.4;UTF8:some other identifier', + 'otherName.2 = 1.3.6.1.5.2.2;SEQUENCE:princ_name', + 'email.1 = user@example.org', + 'DNS.2 = www.example.org', + # GEN_X400 + 'dirName.1 = dir_sect', + # GEN_EDIPARTY + 'URI.1 = https://www.python.org/', + 'IP.1 = 127.0.0.1', + 'IP.2 = ::1', + 'RID.1 = 1.2.3.4.5', + ] + + cert, key = make_cert_key(cmdlineargs, 'allsans', sign=True, extra_san='\n'.join(extra_san)) + with open('allsans.pem', 'w') as f: + f.write(key) + f.write(cert) + + extra_san = [ + # könig (king) + 'DNS.2 = xn--knig-5qa.idn.pythontest.net', + # königsgäßchen (king's alleyway) + 'DNS.3 = xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'DNS.4 = xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + # βόλοσ (marble) + 'DNS.5 = xn--nxasmq6b.idna2003.pythontest.net', + 'DNS.6 = xn--nxasmm1c.idna2008.pythontest.net', + ] + + # IDN SANS, signed + cert, key = make_cert_key(cmdlineargs, 'idnsans', sign=True, extra_san='\n'.join(extra_san)) + with open('idnsans.pem', 'w') as f: + f.write(key) + f.write(cert) + + cert, key = make_cert_key(cmdlineargs, 'nosan', sign=True, ext='req_x509_extensions_nosan') + with open('nosan.pem', 'w') as f: + f.write(key) + f.write(cert) + + unmake_ca() + print("Writing out reference data for Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") + write_cert_reference('keycert.pem') + write_cert_reference('keycert3.pem') diff --git a/Lib/test/certdata/nokia.pem b/Lib/test/certdata/nokia.pem new file mode 100644 index 00000000000..0d044df4367 --- /dev/null +++ b/Lib/test/certdata/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/nosan.pem b/Lib/test/certdata/nosan.pem new file mode 100644 index 00000000000..c397342fa86 --- /dev/null +++ b/Lib/test/certdata/nosan.pem @@ -0,0 +1,137 @@ +-----BEGIN PRIVATE KEY----- +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAQKCAYAE +A1FQwmMxHOhUbuAIRoUInwhMeLs7ZKJx8OsGPDSCTGRGGnAFD/rSP63cNy71T+AF +gggpiGQegz9KkeGfhQwM1GssLGk8WnCp2fnqbAwyhkZzpOs+WEJSCsKMFDc3S0GU +hChIYAJFtgRLJnAFM+b2V0uIfHCz26hGqKOdDb/Wumzd8zPhH+/SNI5hdM7xStez +2d/dlqVZ21c0ghiMLZ/M2qrz8EEvWccc4BEuYtqJYNpdgAauLIGF0SxByPt/oZMf +4fa0zOVfAL6F5cPFkycRzuz97Lx+ZdtLuUBx/3y8Sk24TR1QutwaFhbwpRSQSUrg +xT7ZmS24rQhaP8i7+ROLGy0CPlLMc/D3WvvtdLDWOUF30azURvAFcqKA87Bp05bR +HI/ewXaSB26J34MUmBHy5dBjld+HX6hQInPhCTZu7gNDOrPaEc8Ug50c1r7HkYRl +VBv/4BQo3YtNdoKhL08MVuNbcxrzge3kI0mbQu7qDSyrve02WnQ1zijJ916NbgEC +gcEA6oFNNtWzVw/7/MBxUZ1tE5VW5U2SjRvQZlcPTJ9Ub16byhOCdNvE17uyYLt5 +diqoVAvSGWiJ+OKsmw3XTLdhxzWeZ1a9ZiCozsmIfHMQmL9t3MxzTnMEZCfcvxz0 +DPvAN0REUtwRkMbt7vC+g01Eom9UpAH7hKXkc3pTgtkoyFeebC/YneGtdOTVATKU +WMfsY6Biye7P1D8V7btjLI8LTnv90ygoP0uSrX53hp7k4EFAeXyT9VN5wMXLw81a +5K69AoHBANXZsvSSIwe7fjspNUU9yELnEokDIdBn4ARX6x5PonL2KB0iVUzWgjHb +vs23GAzAgKKY+S8VLYptBLUBlsZ/pSLfZLWd1ywxeHmk/nkzuqlAw1jwuUlMq6Vr +BJmfhpOoXeSrQwvkL/uOejdrDSmLAFiIV3l9LtkmIJzfQSSy2QB8mfWwni0vxOJT +mIqZE01rfRTNLIqxO3+0d7pDqArVvWxYqF4C8kihUhNjkB5NaURUTzDQ3Y33aAvT +zAm697nGSQKBwQDEQ8ed9ykL2sLpfR7aUclytHBvpYbcNsUqgf66ADeopiP48m8i +4rRSYjMepok3jugmv2XuAgJHnV8cvm7NNEXPdl7G2l/V08u0lhN3JM5lKQIH4801 +gSnRsVMdWFwhaaosFySfvLOu2e9VJYQtXEPvNwI96bLaCAW1aFHwl1N8qWhb34eK +S9DinopvYCesTlbX4uoLW6XxW4M83rJYHrg1zaxYR6m3n8Z5EflzYBTqY3JUuyES +F/U0k9bAX2SNNHkCgcALk8mYbADxfjkLQuPbZ8jbtl7OhBjki3sZQRk9ftowlxr8 +2Mr9ae+Ke3cM9AidSB6urtFutxrMD7LdicR74pUyGh39pxnrDpKTI1eTgDVuzE7H +FeEyErCIOA77siM7AzZyFsN+dVATslbzgRwpT5kpMdhqf1h18RZ656tDLVuKJzS+ +lF073QYvqo7rkfX1jwgqhCERMR8jfsWsk9UZIREsOHCFBmvPesxSuGUo/s/gHyBa +aDRWZzp+yWyWakTXDeECgcBaPIxLK0ky89Mv8jTSyIJIMhQj8+z8wFLJFQG5vF/Z +1taLIDY0stU8HzKflTJ5v72r2jpIvu5YgDEgl+kKjyqLMWSu4nBZ2OoJet49S6d3 +X0YL2VzsIiSG+8cmVvBBf7iZCHDKiCM20dzzhZq8BCLiaPq+Zu2No2xEJyE0m8rc +I66z6fV1BWQXLe6vPC8EVvBWz3Ybje0czUdZnZ44qV6qyZzREXuqbC0qS5RanE4Z ++Ps9b3TezE7gDII6SbUwhZo= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:61 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2525 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=nosan + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:c3:e5:00:c7:04:14:ea:1b:98:3f:39:6e:a4:55: + be:a8:4c:72:cc:e2:ae:b0:5f:8d:98:a1:f3:c6:06: + 35:85:94:87:8c:5f:bf:00:d4:fc:d1:be:02:8b:a3: + d4:57:6c:13:9b:fd:0f:3d:ad:bb:6c:2f:d7:5d:f8: + 57:14:68:d3:fb:18:d2:e7:5f:e3:29:78:e9:cc:38: + be:8c:ec:bc:e2:84:58:b4:3b:24:66:bc:2c:ad:58: + 85:0c:62:41:fc:96:a1:ee:c9:dc:1d:51:13:ac:6e: + a1:a4:da:be:d2:6d:ef:17:cc:36:df:f5:d3:67:09: + 86:67:a0:e2:4c:88:6b:e9:d3:c0:f9:14:51:89:2f: + e2:bb:37:dc:42:10:70:a9:1c:9e:66:17:2c:b0:ce: + e9:e8:47:7b:ac:12:05:7e:d1:bf:27:b7:6f:3b:45: + c2:e8:04:d6:8e:44:22:ed:e8:bd:45:4a:c4:9b:ea: + 0d:cd:4b:f0:14:34:d0:db:d5:52:e8:6b:de:87:14: + b2:f7:ac:10:aa:4a:dd:f2:48:21:ad:cf:79:67:70: + 41:b9:1d:70:51:34:45:a8:91:ae:12:2a:a1:c3:38: + ff:33:0d:96:71:b6:63:70:5a:9e:90:b4:76:85:8f: + 8a:d8:fd:79:cc:1a:27:a4:26:02:06:72:bc:de:07: + c8:c0:83:01:82:f6:ed:b7:62:8e:7f:21:7e:4e:86: + 04:88:6e:a8:f4:9a:d6:15:47:06:70:25:bb:33:5d: + 54:27:ec:13:5b:43:33:45:36:09:e9:24:fa:49:c4: + 0e:31:0d:1a:f8:d7:92:2c:01:01:e6:ee:e8:2d:dc: + 6d:1a:5f:63:db:6d:76:31:3a:91:7f:51:df:b7:25: + 0b:b9:bd:81:9b:70:c5:31:a8:e3:da:1e:2d:c2:f1: + 05:13:ad:cb:c8:44:19:b8:5b:62:f6:7d:dd:f1:00: + da:42:e9:16:9f:45:55:e3:7d:e1:85:7e:4d:29:c8: + 0a:e5:bc:8c:98:08:6b:f1:01:e5 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 27:04:5B:37:51:0D:92:FD:2C:60:61:FE:4F:9C:66:20:41:2C:08:45 + X509v3 Authority Key Identifier: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + a0:b7:48:56:c6:b3:92:26:3f:d8:92:3c:ed:72:b5:89:ea:fd: + c9:66:da:ba:6a:8e:0d:04:ea:b2:fd:1f:e4:29:da:1e:c7:8f: + 5a:f0:88:74:dd:b3:f0:5e:a7:c4:77:13:cf:a8:19:fb:f2:2d: + ee:47:b4:0c:7c:5b:d3:dc:2f:2a:5c:bf:43:22:1c:91:d8:03: + 7d:44:90:c0:2d:fe:9e:7c:8b:ef:39:4e:b3:87:99:c8:eb:c2: + b7:cf:86:65:05:52:8c:15:b9:6a:8d:cd:e3:2a:29:d1:f5:87: + 42:11:c3:2e:42:ec:ed:26:55:8d:f3:ad:66:f4:79:72:f7:9e: + ed:bc:0c:5a:a7:74:ab:dc:57:e8:5c:99:b6:32:8f:7e:58:6e: + 70:48:ea:5d:a7:fa:b1:fc:c0:e9:50:3a:a4:53:21:e7:8b:77: + 54:2d:bb:64:1e:fc:88:86:90:c1:03:90:17:bb:3c:cf:ce:e8: + 48:32:c2:07:e5:8e:4f:93:a9:1a:e2:f5:93:a8:01:9f:30:26: + 1e:ed:b5:62:e9:25:c5:b0:32:e1:fc:bd:d6:48:b4:70:a9:e2: + cd:f6:a9:42:cb:bb:24:39:b9:34:fc:b9:cb:09:01:f0:5e:7e: + ef:b5:59:d6:88:31:a9:4c:be:7d:5b:de:4d:ec:84:1b:a1:1b: + d8:7d:83:cb:f1:04:c9:f1:f3:a4:08:05:3c:b5:96:13:1d:37: + 8a:23:83:22:86:72:17:13:5e:e8:89:06:58:cd:89:42:71:12: + e5:47:fc:f7:6e:96:28:8f:19:b9:d7:86:5b:c5:62:14:e1:5b: + 06:e7:e0:66:7e:fc:b7:9e:a9:99:14:e5:0a:d6:df:8f:b5:a2: + 1a:74:54:30:f6:f4:bf:1b:43:1d:be:4f:38:92:55:10:7b:d8: + 4f:e0:33:0f:40:2e:58:ec:9c:78:1b:43:17:b3:cb:0b:f5:34: + e2:7e:11:a1:90:b6:3c:79:6a:0b:91:ce:0b:8d:d5:60:e4:6d: + c8:2a:3d:40:6d:17 +-----BEGIN CERTIFICATE----- +MIIEbzCCAtegAwIBAgIJAMstgJlaaVJhMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowWzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4 +MSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEOMAwGA1UEAwwF +bm9zYW4wggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDD5QDHBBTqG5g/ +OW6kVb6oTHLM4q6wX42YofPGBjWFlIeMX78A1PzRvgKLo9RXbBOb/Q89rbtsL9dd ++FcUaNP7GNLnX+MpeOnMOL6M7LzihFi0OyRmvCytWIUMYkH8lqHuydwdUROsbqGk +2r7Sbe8XzDbf9dNnCYZnoOJMiGvp08D5FFGJL+K7N9xCEHCpHJ5mFyywzunoR3us +EgV+0b8nt287RcLoBNaORCLt6L1FSsSb6g3NS/AUNNDb1VLoa96HFLL3rBCqSt3y +SCGtz3lncEG5HXBRNEWoka4SKqHDOP8zDZZxtmNwWp6QtHaFj4rY/XnMGiekJgIG +crzeB8jAgwGC9u23Yo5/IX5OhgSIbqj0mtYVRwZwJbszXVQn7BNbQzNFNgnpJPpJ +xA4xDRr415IsAQHm7ugt3G0aX2PbbXYxOpF/Ud+3JQu5vYGbcMUxqOPaHi3C8QUT +rcvIRBm4W2L2fd3xANpC6RafRVXjfeGFfk0pyArlvIyYCGvxAeUCAwEAAaNCMEAw +HQYDVR0OBBYEFCcEWzdRDZL9LGBh/k+cZiBBLAhFMB8GA1UdIwQYMBaAFMAKKyhD +3l/JfUflR5s28mWMZzviMA0GCSqGSIb3DQEBCwUAA4IBgQCgt0hWxrOSJj/Ykjzt +crWJ6v3JZtq6ao4NBOqy/R/kKdoex49a8Ih03bPwXqfEdxPPqBn78i3uR7QMfFvT +3C8qXL9DIhyR2AN9RJDALf6efIvvOU6zh5nI68K3z4ZlBVKMFblqjc3jKinR9YdC +EcMuQuztJlWN861m9Hly957tvAxap3Sr3FfoXJm2Mo9+WG5wSOpdp/qx/MDpUDqk +UyHni3dULbtkHvyIhpDBA5AXuzzPzuhIMsIH5Y5Pk6ka4vWTqAGfMCYe7bVi6SXF +sDLh/L3WSLRwqeLN9qlCy7skObk0/LnLCQHwXn7vtVnWiDGpTL59W95N7IQboRvY +fYPL8QTJ8fOkCAU8tZYTHTeKI4MihnIXE17oiQZYzYlCcRLlR/z3bpYojxm514Zb +xWIU4VsG5+Bmfvy3nqmZFOUK1t+PtaIadFQw9vS/G0Mdvk84klUQe9hP4DMPQC5Y +7Jx4G0MXs8sL9TTifhGhkLY8eWoLkc4LjdVg5G3IKj1AbRc= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/nullbytecert.pem b/Lib/test/certdata/nullbytecert.pem new file mode 100644 index 00000000000..447186c9508 --- /dev/null +++ b/Lib/test/certdata/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/nullcert.pem b/Lib/test/certdata/nullcert.pem new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/certdata/pycacert.pem b/Lib/test/certdata/pycacert.pem new file mode 100644 index 00000000000..c2c8b1ecdcf --- /dev/null +++ b/Lib/test/certdata/pycacert.pem @@ -0,0 +1,102 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5b + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2525 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:e6:ba:e2:e4:c1:c2:0c:1c:3e:62:d8:b9:5c:57: + 2e:52:b8:83:c5:88:3a:e6:9a:7a:f5:64:16:33:eb: + 37:6e:2f:7b:f3:68:03:45:65:47:5d:71:10:59:ca: + 2b:1b:00:6c:81:14:61:f4:86:59:3f:ea:fd:78:37: + 16:9d:43:f1:c4:f6:69:8c:c5:29:06:88:9e:26:22: + 04:ac:04:d8:87:34:48:39:eb:6b:f2:0b:92:aa:c3: + 6e:63:51:51:6b:c2:ad:ff:5c:c8:2f:b2:1b:9c:20: + 8a:40:3e:a2:2f:6a:ea:c8:d9:37:43:5c:dc:ed:92: + e2:d9:40:d2:61:9f:71:8a:f5:ed:39:ba:a8:5e:3e: + b5:21:63:10:d8:6f:b4:e2:11:01:0b:10:e8:bb:fb: + 62:ef:48:55:bc:f5:d2:9c:ab:68:ae:95:25:19:f2: + 97:7d:1a:dc:66:ea:88:5e:86:e4:cb:cb:69:4d:5e: + b0:a3:fb:6c:31:e4:28:60:5e:90:f1:d4:2e:10:50: + e1:85:f0:0d:5c:bd:dd:45:24:08:19:3e:1c:93:66: + 8f:2b:da:53:7d:04:1c:0e:42:c4:68:5e:a6:cd:a9: + 18:ed:a7:cd:6a:d0:d1:86:ba:90:ff:b7:4c:de:c7: + 43:24:6d:c7:1c:6b:9c:81:e7:e1:1b:57:25:90:a9: + 0e:c9:56:f3:f6:6b:5e:2d:b4:2e:40:50:9b:42:63: + d2:d6:99:1c:38:dc:cf:2b:2c:a7:72:f1:c7:5e:63: + 34:76:48:f4:3e:88:13:9e:86:16:53:2f:74:fb:87: + 01:8d:22:a4:68:33:ee:13:6c:7a:06:14:54:56:17: + 57:57:98:34:d0:0b:66:09:e3:88:09:f8:a5:15:1c: + 10:73:d0:88:50:99:5e:18:65:3b:ff:31:27:1b:5e: + c6:aa:41:fd:2d:2f:18:a7:c0:f2:ab:c7:22:b5:0b: + 69:d2:73:d1:bb:d0:1c:3d:fa:a4:35:62:cd:33:86: + c7:a0:23:0f:b9:6a:d5:d2:6d:8d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + X509v3 Authority Key Identifier: + C0:0A:2B:28:43:DE:5F:C9:7D:47:E5:47:9B:36:F2:65:8C:67:3B:E2 + X509v3 Basic Constraints: critical + CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + bc:e3:56:22:03:e4:5c:b9:67:ff:94:bc:75:9c:00:85:b0:d5: + 9c:c4:c3:29:66:5e:8b:b2:a9:a6:30:86:71:1a:6b:f2:00:c5: + 82:ab:5f:50:04:2a:fb:ed:b8:4c:b9:00:1b:49:57:92:11:cd: + a2:bc:cb:0f:a8:b4:61:f8:14:ca:a0:ec:40:17:ba:55:a1:c4: + bc:a6:b2:5a:ef:f4:20:10:77:47:d0:a0:c5:58:b9:6c:b5:10: + 7b:85:4a:43:a3:fb:2c:01:b9:77:17:b0:be:a0:ee:ae:ae:4d: + 67:86:48:89:57:86:78:ea:3c:ed:f0:41:35:8d:71:68:55:f9: + f2:e9:ac:32:d4:c6:a2:ef:ec:54:e6:c4:8e:2c:fd:bd:aa:60: + 56:65:33:95:ea:10:c6:74:04:eb:2a:6e:9b:11:f6:61:00:aa: + fd:ec:f2:0b:b1:4b:11:cd:93:eb:df:98:ae:4c:b4:07:04:4a: + e5:ef:ff:52:58:75:f5:3e:a4:71:e1:4a:72:5c:a9:8f:d4:aa: + 88:f0:6a:71:b4:c3:00:5f:99:6e:d7:91:af:6c:98:0d:64:c2: + 24:c7:9e:05:11:68:5e:24:62:e3:2e:45:ec:a3:34:f2:a3:9d: + 4d:e5:32:18:2f:74:fc:11:f1:36:50:4f:a0:40:29:68:5c:43: + 4c:23:6c:5d:72:c4:ec:52:76:eb:dc:b2:bc:1f:a6:c4:06:66: + 9b:5c:c7:cc:ca:f2:d1:25:4f:de:a5:1f:8d:e4:0c:49:b6:cf: + 85:40:a1:b9:1f:c6:c7:19:15:07:63:34:93:d0:57:a0:5a:70: + ec:af:4a:1c:72:17:1d:74:a3:6c:31:45:0b:33:7a:a1:b8:46: + db:c7:0e:64:4c:6f:b7:99:04:82:43:1f:e0:59:d6:99:21:27: + 28:09:40:ae:fc:c4:23:aa:a0:0c:08:05:2a:92:1c:db:23:9e: + d1:d5:63:ae:39:13:a3:12:88:5a:43:3c:4a:6e:32:f0:84:9f: + f9:09:0c:91:e7:b8 +-----BEGIN CERTIFICATE----- +MIIEgjCCAuqgAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAgFw0xODA4MjkxNDIzMTZaGA8yNTI1MTAy +ODE0MjMxNlowTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyMIIBojANBgkq +hkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA5rri5MHCDBw+Yti5XFcuUriDxYg65pp6 +9WQWM+s3bi9782gDRWVHXXEQWcorGwBsgRRh9IZZP+r9eDcWnUPxxPZpjMUpBoie +JiIErATYhzRIOetr8guSqsNuY1FRa8Kt/1zIL7IbnCCKQD6iL2rqyNk3Q1zc7ZLi +2UDSYZ9xivXtObqoXj61IWMQ2G+04hEBCxDou/ti70hVvPXSnKtorpUlGfKXfRrc +ZuqIXobky8tpTV6wo/tsMeQoYF6Q8dQuEFDhhfANXL3dRSQIGT4ck2aPK9pTfQQc +DkLEaF6mzakY7afNatDRhrqQ/7dM3sdDJG3HHGucgefhG1clkKkOyVbz9mteLbQu +QFCbQmPS1pkcONzPKyyncvHHXmM0dkj0PogTnoYWUy90+4cBjSKkaDPuE2x6BhRU +VhdXV5g00AtmCeOICfilFRwQc9CIUJleGGU7/zEnG17GqkH9LS8Yp8Dyq8citQtp +0nPRu9AcPfqkNWLNM4bHoCMPuWrV0m2NAgMBAAGjYzBhMB0GA1UdDgQWBBTACiso +Q95fyX1H5UebNvJljGc74jAfBgNVHSMEGDAWgBTACisoQ95fyX1H5UebNvJljGc7 +4jAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAYEAvONWIgPkXLln/5S8dZwAhbDVnMTDKWZei7KppjCGcRpr8gDFgqtfUAQq +++24TLkAG0lXkhHNorzLD6i0YfgUyqDsQBe6VaHEvKayWu/0IBB3R9CgxVi5bLUQ +e4VKQ6P7LAG5dxewvqDurq5NZ4ZIiVeGeOo87fBBNY1xaFX58umsMtTGou/sVObE +jiz9vapgVmUzleoQxnQE6ypumxH2YQCq/ezyC7FLEc2T69+Yrky0BwRK5e//Ulh1 +9T6kceFKclypj9SqiPBqcbTDAF+ZbteRr2yYDWTCJMeeBRFoXiRi4y5F7KM08qOd +TeUyGC90/BHxNlBPoEApaFxDTCNsXXLE7FJ269yyvB+mxAZmm1zHzMry0SVP3qUf +jeQMSbbPhUChuR/GxxkVB2M0k9BXoFpw7K9KHHIXHXSjbDFFCzN6obhG28cOZExv +t5kEgkMf4FnWmSEnKAlArvzEI6qgDAgFKpIc2yOe0dVjrjkToxKIWkM8Sm4y8ISf ++QkMkee4 +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycakey.pem b/Lib/test/certdata/pycakey.pem new file mode 100644 index 00000000000..0248f985545 --- /dev/null +++ b/Lib/test/certdata/pycakey.pem @@ -0,0 +1,40 @@ +-----BEGIN PRIVATE KEY----- +MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQDmuuLkwcIMHD5i +2LlcVy5SuIPFiDrmmnr1ZBYz6zduL3vzaANFZUddcRBZyisbAGyBFGH0hlk/6v14 +NxadQ/HE9mmMxSkGiJ4mIgSsBNiHNEg562vyC5Kqw25jUVFrwq3/XMgvshucIIpA +PqIvaurI2TdDXNztkuLZQNJhn3GK9e05uqhePrUhYxDYb7TiEQELEOi7+2LvSFW8 +9dKcq2iulSUZ8pd9Gtxm6ohehuTLy2lNXrCj+2wx5ChgXpDx1C4QUOGF8A1cvd1F +JAgZPhyTZo8r2lN9BBwOQsRoXqbNqRjtp81q0NGGupD/t0zex0Mkbccca5yB5+Eb +VyWQqQ7JVvP2a14ttC5AUJtCY9LWmRw43M8rLKdy8cdeYzR2SPQ+iBOehhZTL3T7 +hwGNIqRoM+4TbHoGFFRWF1dXmDTQC2YJ44gJ+KUVHBBz0IhQmV4YZTv/MScbXsaq +Qf0tLxinwPKrxyK1C2nSc9G70Bw9+qQ1Ys0zhsegIw+5atXSbY0CAwEAAQKCAYBf +4VWcPjBHHA2Iwgr1Jn1nfqmzklL3tUZXZwoa9SoJrc3Sbmy9j8LCP9PNnEehZuGw +GipClPnNp/dA15OcMrnrYYKnLt9Hico+inBqk3Dvbnh9KSmoYcrHD4N13jr5juMD +dSjzOQ5kKNmKrPx0u/dpE2r1oUdlql5+bYN/ceSbHGtCTCDfWSun/iTn7DO8pdhL +IvGz/Fk2mlaWuYiV9lz//5Z1W+w73sesNNYKgf/d+F9/+VNqMXbanLdypJmTBNp9 +eoS3eLk3ycoiGHCnVNm28IjeElFkUOJKVXY39BoMbS/x1MUHjuZDxdOcEaCV9iOr +adg7d91srwJlHGGX3IkC+08J5OJZtfTEMcB3i9g2omOxLAFfapeRxZdD6zMdxWUG +PUNIiy7UKaHe7JJ+fftnk0QBu5DzyrdP0fryFNz9nySot+gzVlU9idTMnIyE73UJ +foPowKpmxDOYNw0ZGXBIP/lMRNXkQM765P1NJpCN0Ud+kBl7GdFMkI++GFjiH3kC +gcEA+VIsOliXn5xfU2zXZb2q/bPPlCBDw/EzIelRTFVqUC2kL3oh4NlDNWAFgcfX +rQUCCoY9rEKYdTQI0w47K4AkJYajnDG2zHj6fkaPcLweyEDa9Hw2ezDDiwz2uwH2 +4HW+JBZ0I7rRuSRXTSJrzFYYVU6tsr54Kl85l2bmL8V2XVYj4ko9uddsyH0F1Ejt +BYPBghr01uEHsO3nIcNMl10fS9eXdhMTr/haXdDxpFXjR5AO/ouQEyGrXk6LPJpn +rUE7AoHBAOzpN8lQDQlf0JcYFBvFd7JdrhpbaIdATmT0NM/Mh+4B8qy5qnIJBdeQ +9T46sHmHzvhtGHmnP+dckKJi1Y/8Lt6noWNYDRnmrDol046ce2Nj8gxDGBT1kG4n +GBoKkA1Gj5N5yWMB8cGTVzQ+tViRMkn9+jmxRaNsW1Y0j7lMtVz0hJOUscbUALV1 +fYGH7zPiqfHoKoz7UuB2OlCKOH0+V96lAO4EgKMQNcEKMs9+Tg13fHTAyodxy/CY +tjUcKpef1wKBwBE24EDjDw0BMf/Dmxe2QdEkkieLFsK3q60iu+9GUoHYtOZmS2KH +/cD4sUilsLmMh/iMDkQPkRE+l4FjESjOvzAsHK3TLOjvTXRckNja1FFFURjiXqyg +0E+QhJSi7RXQa2F4f2pcItDitnhn8QN5ylJRjWKzDf729jYC78/KlYKaSP393Ecx +nZw2LanboynnT/wYumD/xpUrx/Kn1mj5EAkfiKCpbomO30Zs/9I17+xoAPEIV9lK +UNfBGpIDozbuMwKBwFlZmAWf4FrRvSzPEv5qWjt2I2yjXufrs+VVSPm6LOXx7CGC +oKsDhiWH8UZ4AgjD1KZTFvECyBItEgt8dQkp1k95L1/1XHORURFZJNHbaJnSnv5K +67Ez8DXrHqbrpuqq2wmG3BIwMIqOVExK/kAZ+rp3REEv/5CkFEqN5kq/iIM3YSz7 +3pSbbm0Bk8UfjHKoIOowYqPrQZWQYWvwxV9O/PrmhlQ+dHmLaoqUmxcwjqV7k//A +mmG85GqoXcfoCJRI3wKBwBdnxBzg17TMFufuvX2Bc/M9MqL3+vlwH6SDdr+2yYKA +hiD8Ur2OwtDGHnV4m3NeA/Guyz32H4CzvFAnpzlvMow/dvfp9JUcpdeidhIBZy8V +D7VODIiCyyTAb3g1LK0+HTEHAVRFihbNXhub/P6NckFXw0MJJQOvNNsYQnJTUngY +oxqdt1HeAujEwBrRSfrOGE2K8FVJ/MYf4PmxTIocICBk1/BmNHsUeJ5yFUDBweh5 +UJN6yp5PiGwvW8WFl4waXw== +-----END PRIVATE KEY----- diff --git a/Lib/test/certdata/revocation.crl b/Lib/test/certdata/revocation.crl new file mode 100644 index 00000000000..a10586d1f79 --- /dev/null +++ b/Lib/test/certdata/revocation.crl @@ -0,0 +1,14 @@ +-----BEGIN X509 CRL----- +MIICKDCBkQIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWqAOMAwwCgYD +VR0UBAMCAQAwDQYJKoZIhvcNAQELBQADggGBAKN4S2g4wCeU1fO5TSckAwxgdzdh +pY28f4musnQt7l37MzB2gmJVDSCZfQyrUnfSEST15WEY7CZVyTlbsu6gYKK53Yej +j3ORBfGUgzaz62Hs8di7SrHDzWUlNCFa47YFWDmtj96KTX1AItnpkCCE58Wfpivp +Hu+YINFpi/2vI2nvP/xcfvgT3dXek9kyz+2jHmadxcn2VerSBZZ9fiZk/k4NzgoI +JdiSswtN1c5GelHQfftwRXbWqsp6TvgHC5MagDuHh5Bj7/DftI7nCy0IT5GnP8lS +ZqmXUMpa8zbtSNSTIk0XepmypNW8HHMQbfJp0y7yOQ4pPyXICrjYTg7wKpODRcm3 +BRN89vvNfCszMU41glVfQG+2Po5uAMTl1hX8WYSj0+Xxrdg+wgJead4S5Sq3CgMT +bKsH2Dqh43L8BTxuxzLQyduK0gKSl8vlN7a9Bzm3IXYlyk+kKSyo4jP8XK79pj1k +1JglMFM9jpoMF2VmNjiROtVEl2tbDGwlvpjWYQ== +-----END X509 CRL----- diff --git a/Lib/test/certdata/secp384r1.pem b/Lib/test/certdata/secp384r1.pem new file mode 100644 index 00000000000..eef7117af7a --- /dev/null +++ b/Lib/test/certdata/secp384r1.pem @@ -0,0 +1,7 @@ +$ openssl genpkey -genparam -algorithm EC -pkeyopt ec_paramgen_curve:secp384r1 -pkeyopt ec_param_enc:named_curve -text +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +ECDSA-Parameters: (384 bit) +ASN1 OID: secp384r1 +NIST CURVE: P-384 diff --git a/Lib/test/certdata/selfsigned_pythontestdotnet.pem b/Lib/test/certdata/selfsigned_pythontestdotnet.pem new file mode 100644 index 00000000000..2b1760747bc --- /dev/null +++ b/Lib/test/certdata/selfsigned_pythontestdotnet.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL +BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG +A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw +HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx +FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu +aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD +DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2 +RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix +IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx +jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK +DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3 +TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI +aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU +OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH +OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg +7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ +8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB +AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW +gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP +ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN +9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9 +1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH +AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e +W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk +BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8 +XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku +B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL +Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV +J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_cert.pem b/Lib/test/certdata/ssl_cert.pem new file mode 100644 index 00000000000..6db52404942 --- /dev/null +++ b/Lib/test/certdata/ssl_cert.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEhTCCAu2gAwIBAgIUTxhwRX6zBQc3+l3imDklEbqcDpIwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MCAXDTI0MTAwODExNTExMloYDzI0MDgwMTI5MTE1MTEyWjBfMQswCQYDVQQG +EwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBT +b2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDDAlsb2NhbGhvc3QwggGiMA0GCSqG +SIb3DQEBAQUAA4IBjwAwggGKAoIBgQDwcIAYm12nmQTGB3caFn7alDe3LSliEfNC +2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3zmrI/FAiQI/RrUHyBZiEt +nFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVwg9cR0khTqD5cg2jvTB05 +yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYliGx5qqCQmsXeTsxCpvQD7 +u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAaF48xEnokxenzBIqZ82BR +kFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLYsFXUp8L7ZMVE27Bej+Sq +4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11tgtctsYbwgDSUYGA3w+DC +KD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3RPJKV+Db8E1V2mXmNE0M +Lg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAaM3MDUwFAYDVR0RBA0wC4IJbG9j +YWxob3N0MB0GA1UdDgQWBBR459BlAel5MqCtG0DrvVtMZlQfuTANBgkqhkiG9w0B +AQsFAAOCAYEAaTFWjK/LFzOo+0TWqTTj4WC4N3I8JFrHnlqFJlpchYTW2z92SU1G +iEzFjWzuDNjp5KM9BqlmGtzXZvy6MItGkYsjPRdPVU0rbCmyTho6y77kTyiEG12V +UAJ1in3FOQfDwLPcp7wQRgCQq3iZlv7pwXp2Lm5fzu8kZPnxmTVdiKQun9Ps7uKq +BoM0fM2K14MxVO4Wc0SERnaPszE7xAhkIcs+NRT/gHYdTBlPhao83S3LOOdtCqCP +pNUOEaShlwI5bVsDPUXNX/eS0MYFNlsYTb5rCxK8jf3W3KNjKTOzN94pHiQOhpkg +xMPPi3m03/9oKTVXBtHI2A+u3ukheKE6sBXCLdv/GEs9zYI49zmpQxNWz5EOumWL +k+W/vPv7cD6LeHxxp+nCbEJi1gZtYb3gMY1sLkMNxcOu0QHTqHfme+k7VKWm8anO +3ogGdGtPuPAD/qjMwg3ChSDLl5Ur/E9UPlD4yM/7KtUD7mLv+jbddA62EiA9MxVB +t+yt7pOwOA66 +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_key.passwd.pem b/Lib/test/certdata/ssl_key.passwd.pem new file mode 100644 index 00000000000..e60a66c2798 --- /dev/null +++ b/Lib/test/certdata/ssl_key.passwd.pem @@ -0,0 +1,42 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHdTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQMMUN+qWFiwTbIjjb +cLEtYQICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEGGzvsc6e5Zn2M9S +sdoMbaIEggcQ0mQYuMcujHtFfRKLmfclrf/dHOYgnVYc2c9MSR3jfoOwnTdBR1Op +MUuO4ukqrxGkrJBy+MLb1oPEGrnvn39pdBMZx4CBmdwX0jDq36UL9EU5/m071mT+ +Xoe/z/RHtrtmbpA9XKu7n/5gLi9jg57qorss8r5MaPHghm2ZOqaQZIOOMCF+4gNU +Cmp/NHL9TUM/sfQpaw9YM4VuTRyHNO3OiDKaCCz2romFrgM1TYz/I5/n5MMR543J +dsyIFzef3jHYfaXTl97v9ibGVpIUCKfHgYlQf41H+Me890AP4HaS0e2y+73/UwjB +4YSzPVm+lCiBWhFDbMuRmPY3YDQOA+TV06oCO8/aMamPRsryl8g7iOQAIz4hQdoP +ZTXtSZ+F68YmfPjKAj3NNQosvLzfuSfxZ9HqyvZ/0w1eead7pdSs7uvCEvN2Zcvk +a9EWy1bM895zD3DwrxqGnSYbitXouIOf9zHsl2lRUK44XmvgTHoNaJFIVyzVNgQr +e1YJM7LbEvErspa6cZTz7DCR44g9cYBi393XEt5yl4NCtq2PFq762pNqpsQqt5/Y +pLX5w12npf+OZ1WqJla89FabqLaJ1blmDEgfko0XriofW5gWLByvhYINtt+u5x/5 +QdTya2gWBxI2K+sAeGIt913RBAus8hLEDnVGDX/FEthz1OOKLXp0G6HQmqUom5PU +/O36GszhWIz1Q/SpMfZkdHfEfbJFCx0mpaJZTlC01hg+OpnwWxUvO2dxrK46xAzd +/obsVQRLOuyTxjIrQd+vVZRIcOroTHVUbh7qV2NV1sdQbsKnc4mYK8jjHsb/v/O3 +lcOFJX2V55z7FdEOi7qKKn3l5j6SNmdMGlGde16692VCdYo4Wjs2ufkYryJoJQ7L +O3AeoXVrpc2JwKj6GFKN3Bw6A3XEXTShWdHS8DAuHdbLBhHESeb/qA76qNInw/od +PXtsRNBWaS87rh7GGWrO2ULHBk0k5hLa9puXgAZVh5NcrVuujWM2spGPHGJWhvDt +ffOJAe0Kra8b1CHOC9aRUuHnapuu/mAD+XZTAt/UUNq0hdTDO1VTxL4lHpZjmWJ3 +OvmTp1HpA8FpSmzSBw6H0beVT9LXDbkSax86E2YQJPcC9IjQARoaL+7ba1XdXeSQ +k796zrq4YD1SB09qRVVar8nBSHUYAzG2gbAWgAQgOjwtC5nT1Que3kI7weG2nFZ4 +B1MfBmpEqPwMs8O5hurdYmfpBRoudivKrK5si+LB7og1IwVD2IS8mm55/Kmnugf4 +YCEiGu5s9F+njqEZBV7UPAabaFph6Iw3Jfmcg0YUZV60hY6O3EieonfwVBBKWBQp +K0SUs24Ld80B3oYWuqYI+MQlSprskRKFBfx7hp2PsQU9jAVPc8NohmfM5IJAFBwo +rCqAOsRsrRZnVoaRP89X50VuBg6ROiu/cmI5sqou3u30Pndzmlbcg8ekWj+1qrur +prva6uPc0oiCgFTMksFnhKvlHErIm+ceGvNSib+rWNomaL5cjpgnqAG6lZMP7N+p +QvH+ABchP+fJyyl/XuaOLmcXDSkIcadfjJSHACwmGRas3unfOYEyBg29oPu6PNlW +jXFJb7OzWaQfFNcCnHzvYGTvmrbg7VqBVqmVRYX3r6WNDJeaSwDJpxYNNQlTugbm +FOlD9JTeHyZ17rrqiCitbQwBVBpWOhBEIkFD+3JL1ZdGjyc3rcvbERLr/UN649vS +FcMOLiEvFjFdirq6Pe9fx/VD1GrIzXaEhvyCePSeoV6eILm7SSFTVCm+JMtiBlBi +ZjDxlUhQGuMg3IwZRqjfUR90wo3QWe5XOgM9mJa5qY/Yaa9YpFEJcd4mYnXcPLIz +eY8lqGAhROwJhGRhQoTQXucsSEhAkUBpRiE7UCN8OjezvAeFn0+6XyHeST+QZenX +ixAKJ27lFn/njvL4sDocd7ZvXpb3P5ZxCRakMnjunQQyjtUlJmPrNjs+ZlPenLuh +UA3Oj5d96dDqzgZNLxbDHKr6B+CMApBrwUDcum09PYgJ5xZ/Hrct4iSa57Gi528L +l0dcVgPHd80oIn/vyhjVYkkXNMrRhTJIgLuh0KsddZ7/8Xvxma9W8hNQ1sYwE0Yp +RqLgMRdFpTSN8hKUTfRvjzbrW4nQ2XVqAdyM6PEkdrPCBZczdGisp9oMF6woI/pA +ZKNxdUr0DG3tVBJ7z6qfdp2j+yHvhqQt+ohmk3YldPKpXfYz20ZJ52URE8vbCj2K +NZ/MVEACbg6FDgNQ1bIKD61pKL72+SjyKW1wQfIkeqMfOBI8BGclp7BL6dGRrVHN +PLg2j9gsgZ3XsiJOFtJ6Q3UABkeUrbRvAFQXPM8+keWU4VP/99dCJTvnEQTM6O8U +gz2NA4j1ZEOmB5L+hmf2gW+xgW6eeMjylj5JdeAliNaUqIhOju30vo4= +-----END ENCRYPTED PRIVATE KEY----- diff --git a/Lib/test/certdata/ssl_key.pem b/Lib/test/certdata/ssl_key.pem new file mode 100644 index 00000000000..1ad9e5e2b83 --- /dev/null +++ b/Lib/test/certdata/ssl_key.pem @@ -0,0 +1,40 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDwcIAYm12nmQTG +B3caFn7alDe3LSliEfNC2ZTR+5sh1eucQPbzgFM5SR4soKGElwI68Eg7g1kwqu3z +mrI/FAiQI/RrUHyBZiEtnFBPM44vY02XjUlJ9nBtgP7QjpRz6ZP0z1DrrojpYLVw +g9cR0khTqD5cg2jvTB05yL9lQNk295/rMuueC9FaAQ+Y5la0ub7Lbe8gkYPotYli +Gx5qqCQmsXeTsxCpvQD7u0vBF9/nxlwywwzKGXabcN9YQhSECCC4c+eYjqoRgvAa +F48xEnokxenzBIqZ82BRkFo+zfNR+VhJRctNiZ6Ppa1Ise1H3LjZMDfY1S89QOLY +sFXUp8L7ZMVE27Bej+Sq4wEJ3soK/k1kr0YauqJ0lCEKkUPD9OeNHmczeakjj11t +gtctsYbwgDSUYGA3w+DCKD1aSfeuR3Hj89cSsVRrRrPFFih46Tr2PpTNoK6MtPH3 +RPJKV+Db8E1V2mXmNE0MLg6ramSHsD9iZXTLhG2JO+/876k3N3kCAwEAAQKCAYAK +Ap0KqTlCd4fv2LK4NtSMNByHt00gRKAMmfNstJ12UKoxBLFjXOXaHjWv5PYkh4bz +vjo7pBHMCWnDuR6Pqr1ahuyvpRex6XcbJ4VebsaOKYO6+gphlm2C2ZqCQ1Vh6Akd +aZ40Wb1gfgK/zvVezBLvzLLf9iahw9j5pWZ2iDci5zdUuvd9Sn+qUB3+nyRf/NW5 +MXgBsp07zIcOOxPOm/Z5V+0jDJL2hiRq1pbmUKClTShcgqtfJKU//niF+6ZQAuiJ +LBPaKIdPXyxLYnkyq2IgjPU0ZwxzdP0a2k72kvImd25Daj7elhGr3++IR+nFzt6h +vqflOfmKDF3zZPyUVI3YXjxo/FrOwGbLMHuuHBgE9txH/mOl1gByrxP+Ax18i3Bf +spSLeUvtaf/w/MopyspPoJBRbAM06PUHQI2v9xq3BZL/gHe2CdJPds2WzpaaVFG4 +oJWNrE3s6CowLqUkqzB7LqJ4ReZ6xe6SpkRotdmVknlIKgDenTFeEUEEVyBiFQEC +gcEA/F1GAaBG0e9vB+AOHZ96SLlZVzObSBYq2kVwUhhGItNnyU9c3fWPIrGREKQa +lw5dsvjl58ij5uEtJoPZf5BsJ0q6xHAs/kKxfpNfZAeoKAV96Z6MVcY+6WOyGjPF +aQo+GgSrCPIciW//WXZrWI1t0M2G0vZ5CFNohnKod+cSgV03PAActlyM2H+r7dtm +MpAD3EPWeeA75saKj/x0SOzuL/wzXKR8BZ6CINZ6r61Tcbk2mDwOHPhUrHeCwjoU +nhy5AoHBAPPnP2FSXFCPXD1Z1hFInCFgm41j7LEyBTVLlnqUrRk7i18fi/WcwwLH ++XvM5DcONY/V3sh7a3tZeHN1P70tRxLE0oO51D4tP5im/oZ6L+hszSYXX7lCbJSR +tni6nU1dssW3nmswfUn01Oh+B0rBGon3RQB6x4beTAW0piVxg9Ic2HYucS1Scrqw +afiFQ5KWklnMYJKInPFzlCwMdgBCuue1dZoJstU9nLQALNNSpGXB2X0+7j9D/qkz +Caw5MfgQwQKBwQDzdCvP78XCSuBq0XvsmefG9n+4fwGDFld6v9gualpmyFjsPJKT +UYwm5PPUAOvh46sCt9hatRVg6sO6zyFoTXP4p7/rN2hAVSiTuiog/r369elVEW3C +ZYBVeKbdXipIPehRA0XYWHCtKY1Fydae07kn4M37AGkcXhKM+VmKajFQ+RMK3/TS +/A+n3+qFiM1bY9FFkW/7nRVMeSY850dq/p59TihibA91AEf6084BYg0IvatsSys2 +SV6uDpDnPE6dhYkCgcBECtAwq1RbmRLnfqdsnPAJk7Txhd3jNQwk6RhqzA1aS7U+ +7UMTWw9AOF+OPQOxpEInBUgob931RGmI9D263eXFA6mi2/Ws/tyODpBVHcM9uRSm +OsEWosQ90kSwe4ckrS4RYH9OcfGR7z5yOa55GVP5B0V1s8r0AhH9SX9MVNWsiSWO +GriyJx0gndSCY1MNkvnzGUQbvQbjiRXeD//fZL5Vo9bSCUCdopmT0bSvo49/X8v3 +19WJSsPBmh5psG8TQEECgcEA64CqZpPux35LeLQsKe0fYeNfAncqiaIoRbAvxKCi +SQf27SD8HK+sfvhvYY7bP7TMEeM7B/O2/AqBQQP0UARIGJg2AknBQT0A7//yJu+o +v4FHy2XKh+RMAx7QrdvnQ4CfrjvjQIaAcN1HrdTKWwgQZZImRf57nUCMm82ktZ2k +vYEJTXMkT8CY0DSeGtPmX5ynk7cauHTdZrkPGhZ3Hr6GAFomOammnnytv2wc+5FA +Ap+d65UgF4KjGY4rtsS+jOHn +-----END PRIVATE KEY----- diff --git a/Lib/test/certdata/talos-2019-0758.pem b/Lib/test/certdata/talos-2019-0758.pem new file mode 100644 index 00000000000..13b95a77fd8 --- /dev/null +++ b/Lib/test/certdata/talos-2019-0758.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDqDCCApKgAwIBAgIBAjALBgkqhkiG9w0BAQswHzELMAkGA1UEBhMCVUsxEDAO +BgNVBAMTB2NvZHktY2EwHhcNMTgwNjE4MTgwMDU4WhcNMjgwNjE0MTgwMDU4WjA7 +MQswCQYDVQQGEwJVSzEsMCoGA1UEAxMjY29kZW5vbWljb24tdm0tMi50ZXN0Lmxh +bC5jaXNjby5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC63fGB +J80A9Av1GB0bptslKRIUtJm8EeEu34HkDWbL6AJY0P8WfDtlXjlPaLqFa6sqH6ES +V48prSm1ZUbDSVL8R6BYVYpOlK8/48xk4pGTgRzv69gf5SGtQLwHy8UPBKgjSZoD +5a5k5wJXGswhKFFNqyyxqCvWmMnJWxXTt2XDCiWc4g4YAWi4O4+6SeeHVAV9rV7C +1wxqjzKovVe2uZOHjKEzJbbIU6JBPb6TRfMdRdYOw98n1VXDcKVgdX2DuuqjCzHP +WhU4Tw050M9NaK3eXp4Mh69VuiKoBGOLSOcS8reqHIU46Reg0hqeL8LIL6OhFHIF +j7HR6V1X6F+BfRS/AgMBAAGjgdYwgdMwCQYDVR0TBAIwADAdBgNVHQ4EFgQUOktp +HQjxDXXUg8prleY9jeLKeQ4wTwYDVR0jBEgwRoAUx6zgPygZ0ZErF9sPC4+5e2Io +UU+hI6QhMB8xCzAJBgNVBAYTAlVLMRAwDgYDVQQDEwdjb2R5LWNhggkA1QEAuwb7 +2s0wCQYDVR0SBAIwADAuBgNVHREEJzAlgiNjb2Rlbm9taWNvbi12bS0yLnRlc3Qu +bGFsLmNpc2NvLmNvbTAOBgNVHQ8BAf8EBAMCBaAwCwYDVR0fBAQwAjAAMAsGCSqG +SIb3DQEBCwOCAQEAvqantx2yBlM11RoFiCfi+AfSblXPdrIrHvccepV4pYc/yO6p +t1f2dxHQb8rWH3i6cWag/EgIZx+HJQvo0rgPY1BFJsX1WnYf1/znZpkUBGbVmlJr +t/dW1gSkNS6sPsM0Q+7HPgEv8CPDNK5eo7vU2seE0iWOkxSyVUuiCEY9ZVGaLVit +p0C78nZ35Pdv4I+1cosmHl28+es1WI22rrnmdBpH8J1eY6WvUw2xuZHLeNVN0TzV +Q3qq53AaCWuLOD1AjESWuUCxMZTK9DPS4JKXTK8RLyDeqOvJGjsSWp3kL0y3GaQ+ +10T1rfkKJub2+m9A9duin1fn6tHc2wSvB7m3DA== +-----END CERTIFICATE----- diff --git a/Lib/test/cjkencodings/big5-utf8.txt b/Lib/test/cjkencodings/big5-utf8.txt new file mode 100644 index 00000000000..a0a534a964d --- /dev/null +++ b/Lib/test/cjkencodings/big5-utf8.txt @@ -0,0 +1,9 @@ +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/Lib/test/cjkencodings/big5.txt b/Lib/test/cjkencodings/big5.txt new file mode 100644 index 00000000000..f4424959e91 --- /dev/null +++ b/Lib/test/cjkencodings/big5.txt @@ -0,0 +1,9 @@ +pb Python ϥάJ C library? +@bTާֳtoi, }oδճn骺t׬Oe +D. [ֶ}oδժt, ڭ̫K`ƱQΤ@Ǥw}on +library, æ@ fast prototyping programming language i +Ѩϥ. ثe\\hh library OH C g, Python O@ +fast prototyping programming language. Gڭ̧ƱNJ +C library Python ҤդξX. 䤤̥Dn]Oڭ̩ +nQתDNO: + diff --git a/Lib/test/cjkencodings/big5hkscs-utf8.txt b/Lib/test/cjkencodings/big5hkscs-utf8.txt new file mode 100644 index 00000000000..f744ce9ae08 --- /dev/null +++ b/Lib/test/cjkencodings/big5hkscs-utf8.txt @@ -0,0 +1,2 @@ +𠄌Ě鵮罓洆 +ÊÊ̄ê êê̄ diff --git a/Lib/test/cjkencodings/big5hkscs.txt b/Lib/test/cjkencodings/big5hkscs.txt new file mode 100644 index 00000000000..81c42b3503d --- /dev/null +++ b/Lib/test/cjkencodings/big5hkscs.txt @@ -0,0 +1,2 @@ +E\sڍ +fb diff --git a/Lib/test/cjkencodings/cp949-utf8.txt b/Lib/test/cjkencodings/cp949-utf8.txt new file mode 100644 index 00000000000..5655e385176 --- /dev/null +++ b/Lib/test/cjkencodings/cp949-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙 家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙 ㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/Lib/test/cjkencodings/cp949.txt b/Lib/test/cjkencodings/cp949.txt new file mode 100644 index 00000000000..16549aa5e49 --- /dev/null +++ b/Lib/test/cjkencodings/cp949.txt @@ -0,0 +1,9 @@ +c氢 ݶ + +!! Вp ިR ѵ . . +䬿Ѵ . . . . ʫR ! ! !. + ٤_  O h O +j ʫR . . . . ֚f ѱ ސtƒO  +;R ! ! 䬿 ʫɱ ߾ ɱŴ 䬴ɵR ۾֊ +޷ ǴR  Ĩ!! ҡ* + diff --git a/Lib/test/cjkencodings/euc_jisx0213-utf8.txt b/Lib/test/cjkencodings/euc_jisx0213-utf8.txt new file mode 100644 index 00000000000..9a56a2e18bd --- /dev/null +++ b/Lib/test/cjkencodings/euc_jisx0213-utf8.txt @@ -0,0 +1,8 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + +ノか゚ ト゚ トキ喝塀 𡚴𪎌 麀齁𩛰 diff --git a/Lib/test/cjkencodings/euc_jisx0213.txt b/Lib/test/cjkencodings/euc_jisx0213.txt new file mode 100644 index 00000000000..51e9268ca98 --- /dev/null +++ b/Lib/test/cjkencodings/euc_jisx0213.txt @@ -0,0 +1,8 @@ +Python γȯϡ1990 ǯ鳫ϤƤޤ +ȯԤ Guido van Rossum ϶ѤΥץߥ󥰸ABCפγȯ˻äƤޤABC ϼѾŪˤϤޤŬƤޤǤ +ΤᡢGuido ϤŪʥץߥ󥰸γȯ򳫻Ϥѹ BBS Υǥȡ֥ƥ ѥפΥեǤ Guido ϤθPythonפ̾Ťޤ +Τ褦طʤޤ줿 Python θ߷פϡ֥ץפǡֽưספȤɸ˽֤Ƥޤ +¿ΥץȷϸǤϥ桼ͥ褷ƿʵǽǤȤƼ礬¿ΤǤPython ǤϤäٹɲä뤳ȤϤޤꤢޤ +켫ΤεǽϺǾ¤˲ɬפʵǽϳĥ⥸塼Ȥɲä롢ȤΤ Python ΥݥꥷǤ + +Τ ȥ ԏ diff --git a/Lib/test/cjkencodings/euc_jp-utf8.txt b/Lib/test/cjkencodings/euc_jp-utf8.txt new file mode 100644 index 00000000000..7763250ebbe --- /dev/null +++ b/Lib/test/cjkencodings/euc_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/Lib/test/cjkencodings/euc_jp.txt b/Lib/test/cjkencodings/euc_jp.txt new file mode 100644 index 00000000000..9da6b5d83da --- /dev/null +++ b/Lib/test/cjkencodings/euc_jp.txt @@ -0,0 +1,7 @@ +Python γȯϡ1990 ǯ鳫ϤƤޤ +ȯԤ Guido van Rossum ϶ѤΥץߥ󥰸ABCפγȯ˻äƤޤABC ϼѾŪˤϤޤŬƤޤǤ +ΤᡢGuido ϤŪʥץߥ󥰸γȯ򳫻Ϥѹ BBS Υǥȡ֥ƥ ѥפΥեǤ Guido ϤθPythonפ̾Ťޤ +Τ褦طʤޤ줿 Python θ߷פϡ֥ץפǡֽưספȤɸ˽֤Ƥޤ +¿ΥץȷϸǤϥ桼ͥ褷ƿʵǽǤȤƼ礬¿ΤǤPython ǤϤäٹɲä뤳ȤϤޤꤢޤ +켫ΤεǽϺǾ¤˲ɬפʵǽϳĥ⥸塼Ȥɲä롢ȤΤ Python ΥݥꥷǤ + diff --git a/Lib/test/cjkencodings/euc_kr-utf8.txt b/Lib/test/cjkencodings/euc_kr-utf8.txt new file mode 100644 index 00000000000..16c37412b6a --- /dev/null +++ b/Lib/test/cjkencodings/euc_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓔쓔쓩~ 닁큼! 뜽금없이 전홥니다. 뷁. 그런거 읎다. diff --git a/Lib/test/cjkencodings/euc_kr.txt b/Lib/test/cjkencodings/euc_kr.txt new file mode 100644 index 00000000000..f68dd350289 --- /dev/null +++ b/Lib/test/cjkencodings/euc_kr.txt @@ -0,0 +1,7 @@ + ̽(Python) , α׷ Դϴ. ̽ +ȿ ȿ üα׷ +մϴ. ̽ () Ÿ, ׸ +ȯ ̽ ũð о߿ κ ÷ +ø̼ ִ ̻ ݴϴ. + +ù: ƶ ԤФԤԤФԾ~ ԤҤŭ! ԤѤݾ ԤȤϴ. ԤΤ. ׷ ԤѤ. diff --git a/Lib/test/cjkencodings/gb18030-utf8.txt b/Lib/test/cjkencodings/gb18030-utf8.txt new file mode 100644 index 00000000000..2060d2593eb --- /dev/null +++ b/Lib/test/cjkencodings/gb18030-utf8.txt @@ -0,0 +1,15 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: +파이썬은 강력한 기능을 지닌 범용 컴퓨터 프로그래밍 언어다. + diff --git a/Lib/test/cjkencodings/gb18030.txt b/Lib/test/cjkencodings/gb18030.txt new file mode 100644 index 00000000000..5d1f6dca232 --- /dev/null +++ b/Lib/test/cjkencodings/gb18030.txt @@ -0,0 +1,15 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + Python ʹüе C library? +YӍƼٰlչĽ, _lyԇܛwٶDzݺҕ +n}. ӿ_lyԇٶ, ҂㳣ϣһЩ_lõ +library, Kһ fast prototyping programming language +ʹ. ĿǰSS library C , Python һ +fast prototyping programming language. ҂ϣ܌е +C library õ Python ĭhМyԇ. ҪҲ҂ +ҪӑՓĆ}: +51332131 760463 858635 3195 0930 435755 5509899304 292599. + diff --git a/Lib/test/cjkencodings/gb2312-utf8.txt b/Lib/test/cjkencodings/gb2312-utf8.txt new file mode 100644 index 00000000000..efb7d8f95cd --- /dev/null +++ b/Lib/test/cjkencodings/gb2312-utf8.txt @@ -0,0 +1,6 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 + diff --git a/Lib/test/cjkencodings/gb2312.txt b/Lib/test/cjkencodings/gb2312.txt new file mode 100644 index 00000000000..1536ac10b9e --- /dev/null +++ b/Lib/test/cjkencodings/gb2312.txt @@ -0,0 +1,6 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + diff --git a/Lib/test/cjkencodings/gbk-utf8.txt b/Lib/test/cjkencodings/gbk-utf8.txt new file mode 100644 index 00000000000..75bbd31ec5a --- /dev/null +++ b/Lib/test/cjkencodings/gbk-utf8.txt @@ -0,0 +1,14 @@ +Python（派森）语言是一种功能强大而完善的通用型计算机程序设计语言， +已经具有十多年的发展历史，成熟且稳定。这种语言具有非常简捷而清晰 +的语法特点，适合完成各种高层任务，几乎可以在所有的操作系统中 +运行。这种语言简单而强大，适合各种人士学习使用。目前，基于这 +种语言的相关技术正在飞速的发展，用户数量急剧扩大，相关的资源非常多。 +如何在 Python 中使用既有的 C library? +　在資訊科技快速發展的今天, 開發及測試軟體的速度是不容忽視的 +課題. 為加快開發及測試的速度, 我們便常希望能利用一些已開發好的 +library, 並有一個 fast prototyping 的 programming language 可 +供使用. 目前有許許多多的 library 是以 C 寫成, 而 Python 是一個 +fast prototyping 的 programming language. 故我們希望能將既有的 +C library 拿到 Python 的環境中測試及整合. 其中最主要也是我們所 +要討論的問題就是: + diff --git a/Lib/test/cjkencodings/gbk.txt b/Lib/test/cjkencodings/gbk.txt new file mode 100644 index 00000000000..8788f8a2dc4 --- /dev/null +++ b/Lib/test/cjkencodings/gbk.txt @@ -0,0 +1,14 @@ +PythonɭһֹǿƵͨͼԣ +ѾʮķչʷȶԾзdzݶ +﷨ص㣬ʺɸָ߲񣬼еIJϵͳ +СԼ򵥶ǿʺϸʿѧϰʹáĿǰ +ԵؼڷٵķչûصԴdzࡣ + Python ʹüе C library? +YӍƼٰlչĽ, _lyԇܛwٶDzݺҕ +n}. ӿ_lyԇٶ, ҂㳣ϣһЩ_lõ +library, Kһ fast prototyping programming language +ʹ. ĿǰSS library C , Python һ +fast prototyping programming language. ҂ϣ܌е +C library õ Python ĭhМyԇ. ҪҲ҂ +ҪӑՓĆ}: + diff --git a/Lib/test/cjkencodings/hz-utf8.txt b/Lib/test/cjkencodings/hz-utf8.txt new file mode 100644 index 00000000000..7c11735c1f1 --- /dev/null +++ b/Lib/test/cjkencodings/hz-utf8.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.己所不欲，勿施於人。Bye. diff --git a/Lib/test/cjkencodings/hz.txt b/Lib/test/cjkencodings/hz.txt new file mode 100644 index 00000000000..f882d463447 --- /dev/null +++ b/Lib/test/cjkencodings/hz.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye. diff --git a/Lib/test/cjkencodings/iso2022_jp-utf8.txt b/Lib/test/cjkencodings/iso2022_jp-utf8.txt new file mode 100644 index 00000000000..7763250ebbe --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/Lib/test/cjkencodings/iso2022_jp.txt b/Lib/test/cjkencodings/iso2022_jp.txt new file mode 100644 index 00000000000..fc398d64ad2 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_jp.txt @@ -0,0 +1,7 @@ +Python $B$N3+H/$O!"(B1990 $BG/$4$m$+$i3+;O$5$l$F$$$^$9!#(B +$B3+H/e$NL\E*$K$O$"$^$jE,$7$F$$$^$;$s$G$7$?!#(B +$B$3$N$?$a!"(BGuido $B$O$h$j$E$1$^$7$?!#(B +$B$3$N$h$&$JGX7J$+$i@8$^$l$?(B Python $B$N8@8l@_7W$O!"!V%7%s%W%k!W$G!V=,F@$,MF0W!W$H$$$&L\I8$K=EE@$,CV$+$l$F$$$^$9!#(B +$BB?$/$N%9%/%j%W%H7O8@8l$G$O%f!<%6$NL\@h$NMxJX@-$rM%@h$7$F?'!9$J5!G=$r8@8lMWAG$H$7$Fl9g$,B?$$$N$G$9$,!"(BPython $B$G$O$=$&$$$C$?>.:Y9)$,DI2C$5$l$k$3$H$O$"$^$j$"$j$^$;$s!#(B +$B8@8l<+BN$N5!G=$O:G>.8B$K2!$5$(!"I,MW$J5!G=$O3HD%%b%8%e!<%k$H$7$FDI2C$9$k!"$H$$$&$N$,(B Python $B$N%]%j%7!<$G$9!#(B + diff --git a/Lib/test/cjkencodings/iso2022_kr-utf8.txt b/Lib/test/cjkencodings/iso2022_kr-utf8.txt new file mode 100644 index 00000000000..d5c9d6eeeb2 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓩~ 큼! 금없이 전니다. 그런거 다. diff --git a/Lib/test/cjkencodings/iso2022_kr.txt b/Lib/test/cjkencodings/iso2022_kr.txt new file mode 100644 index 00000000000..2cece21c5dd --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_kr.txt @@ -0,0 +1,7 @@ +$)C!] FD@L=c(Python)@: 9h?l1b =10m, 0-7BGQ GA7N1W7!9V >p>n@T4O4Y. FD@L=c@: +H?@2@{@N 0mF(iPd:)GQ 9.9}0z 5?@{ E8@LGN, 1W8.0m @NEMGA8.FC +H/0f@: FD@L=c@; =:E)83FC0z ?)7/ :P>_?!<-?M 4k:N:P@G GC7'F{?!<-@G :|8% +>VGC8.DI@Lp>n7N 885i>nA]4O4Y. + +!YC90!3!: 3/>F6s >1~ E-! 1]>x@L @|4O4Y. 1W710E 4Y. diff --git a/Lib/test/cjkencodings/johab-utf8.txt b/Lib/test/cjkencodings/johab-utf8.txt new file mode 100644 index 00000000000..5655e385176 --- /dev/null +++ b/Lib/test/cjkencodings/johab-utf8.txt @@ -0,0 +1,9 @@ +똠방각하 펲시콜라 + +㉯㉯납!! 因九月패믤릔궈 ⓡⓖ훀¿¿¿ 긍뒙 ⓔ뎨 ㉯. . +亞영ⓔ능횹 . . . . 서울뤄 뎐학乙 家훀 ! ! !ㅠ.ㅠ +흐흐흐 ㄱㄱㄱ☆ㅠ_ㅠ 어릨 탸콰긐 뎌응 칑九들乙 ㉯드긐 +설릌 家훀 . . . . 굴애쉌 ⓔ궈 ⓡ릘㉱긐 因仁川女中까즼 +와쒀훀 ! ! 亞영ⓔ 家능궈 ☆上관 없능궈능 亞능뒈훀 글애듴 +ⓡ려듀九 싀풔숴훀 어릨 因仁川女中싁⑨들앜!! ㉯㉯납♡ ⌒⌒* + diff --git a/Lib/test/cjkencodings/johab.txt b/Lib/test/cjkencodings/johab.txt new file mode 100644 index 00000000000..067781b785a --- /dev/null +++ b/Lib/test/cjkencodings/johab.txt @@ -0,0 +1,9 @@ +wba \ũa + +s!! gÚ zٯٯٯ w ѕ . . + LARGEST) + self.assertTrue(tz <= LARGEST) + self.assertFalse(tz >= LARGEST) + self.assertFalse(tz < SMALLEST) + self.assertTrue(tz > SMALLEST) + self.assertFalse(tz <= SMALLEST) + self.assertTrue(tz >= SMALLEST) + + def test_aware_datetime(self): + # test that timezone instances can be used by datetime + t = datetime(1, 1, 1) + for tz in [timezone.min, timezone.max, timezone.utc]: + self.assertEqual(tz.tzname(t), + t.replace(tzinfo=tz).tzname()) + self.assertEqual(tz.utcoffset(t), + t.replace(tzinfo=tz).utcoffset()) + self.assertEqual(tz.dst(t), + t.replace(tzinfo=tz).dst()) + + def test_pickle(self): + for tz in self.ACDT, self.EST, timezone.min, timezone.max: + for pickler, unpickler, proto in pickle_choices: + tz_copy = unpickler.loads(pickler.dumps(tz, proto)) + self.assertEqual(tz_copy, tz) + tz = timezone.utc + for pickler, unpickler, proto in pickle_choices: + tz_copy = unpickler.loads(pickler.dumps(tz, proto)) + self.assertIs(tz_copy, tz) + + def test_copy(self): + for tz in self.ACDT, self.EST, timezone.min, timezone.max: + tz_copy = copy.copy(tz) + self.assertEqual(tz_copy, tz) + tz = timezone.utc + tz_copy = copy.copy(tz) + self.assertIs(tz_copy, tz) + + def test_deepcopy(self): + for tz in self.ACDT, self.EST, timezone.min, timezone.max: + tz_copy = copy.deepcopy(tz) + self.assertEqual(tz_copy, tz) + tz = timezone.utc + tz_copy = copy.deepcopy(tz) + self.assertIs(tz_copy, tz) + + def test_offset_boundaries(self): + # Test timedeltas close to the boundaries + time_deltas = [ + timedelta(hours=23, minutes=59), + timedelta(hours=23, minutes=59, seconds=59), + timedelta(hours=23, minutes=59, seconds=59, microseconds=999999), + ] + time_deltas.extend([-delta for delta in time_deltas]) + + for delta in time_deltas: + with self.subTest(test_type='good', delta=delta): + timezone(delta) + + # Test timedeltas on and outside the boundaries + bad_time_deltas = [ + timedelta(hours=24), + timedelta(hours=24, microseconds=1), + ] + bad_time_deltas.extend([-delta for delta in bad_time_deltas]) + + for delta in bad_time_deltas: + with self.subTest(test_type='bad', delta=delta): + with self.assertRaises(ValueError): + timezone(delta) + + def test_comparison_with_tzinfo(self): + # Constructing tzinfo objects directly should not be done by users + # and serves only to check the bug described in bpo-37915 + self.assertNotEqual(timezone.utc, tzinfo()) + self.assertNotEqual(timezone(timedelta(hours=1)), tzinfo()) + +############################################################################# +# Base class for testing a particular aspect of timedelta, time, date and +# datetime comparisons. + +class HarmlessMixedComparison: + # Test that __eq__ and __ne__ don't complain for mixed-type comparisons. + + # Subclasses must define 'theclass', and theclass(1, 1, 1) must be a + # legit constructor. + + def test_harmless_mixed_comparison(self): + me = self.theclass(1, 1, 1) + + self.assertFalse(me == ()) + self.assertTrue(me != ()) + self.assertFalse(() == me) + self.assertTrue(() != me) + + self.assertIn(me, [1, 20, [], me]) + self.assertIn([], [me, 1, 20, []]) + + # Comparison to objects of unsupported types should return + # NotImplemented which falls back to the right hand side's __eq__ + # method. In this case, ALWAYS_EQ.__eq__ always returns True. + # ALWAYS_EQ.__ne__ always returns False. + self.assertTrue(me == ALWAYS_EQ) + self.assertFalse(me != ALWAYS_EQ) + + # If the other class explicitly defines ordering + # relative to our class, it is allowed to do so + self.assertTrue(me < LARGEST) + self.assertFalse(me > LARGEST) + self.assertTrue(me <= LARGEST) + self.assertFalse(me >= LARGEST) + self.assertFalse(me < SMALLEST) + self.assertTrue(me > SMALLEST) + self.assertFalse(me <= SMALLEST) + self.assertTrue(me >= SMALLEST) + + def test_harmful_mixed_comparison(self): + me = self.theclass(1, 1, 1) + + self.assertRaises(TypeError, lambda: me < ()) + self.assertRaises(TypeError, lambda: me <= ()) + self.assertRaises(TypeError, lambda: me > ()) + self.assertRaises(TypeError, lambda: me >= ()) + + self.assertRaises(TypeError, lambda: () < me) + self.assertRaises(TypeError, lambda: () <= me) + self.assertRaises(TypeError, lambda: () > me) + self.assertRaises(TypeError, lambda: () >= me) + +############################################################################# +# timedelta tests + +class SubclassTimeDelta(timedelta): + sub_var = 1 + +class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): + + theclass = timedelta + + def test_constructor(self): + eq = self.assertEqual + ra = self.assertRaises + td = timedelta + + # Check keyword args to constructor + eq(td(), td(weeks=0, days=0, hours=0, minutes=0, seconds=0, + milliseconds=0, microseconds=0)) + eq(td(1), td(days=1)) + eq(td(0, 1), td(seconds=1)) + eq(td(0, 0, 1), td(microseconds=1)) + eq(td(weeks=1), td(days=7)) + eq(td(days=1), td(hours=24)) + eq(td(hours=1), td(minutes=60)) + eq(td(minutes=1), td(seconds=60)) + eq(td(seconds=1), td(milliseconds=1000)) + eq(td(milliseconds=1), td(microseconds=1000)) + + # Check float args to constructor + eq(td(weeks=1.0/7), td(days=1)) + eq(td(days=1.0/24), td(hours=1)) + eq(td(hours=1.0/60), td(minutes=1)) + eq(td(minutes=1.0/60), td(seconds=1)) + eq(td(seconds=0.001), td(milliseconds=1)) + eq(td(milliseconds=0.001), td(microseconds=1)) + + # Check type of args to constructor + ra(TypeError, lambda: td(weeks='1')) + ra(TypeError, lambda: td(days='1')) + ra(TypeError, lambda: td(hours='1')) + ra(TypeError, lambda: td(minutes='1')) + ra(TypeError, lambda: td(seconds='1')) + ra(TypeError, lambda: td(milliseconds='1')) + ra(TypeError, lambda: td(microseconds='1')) + + def test_computations(self): + eq = self.assertEqual + td = timedelta + + a = td(7) # One week + b = td(0, 60) # One minute + c = td(0, 0, 1000) # One millisecond + eq(a+b+c, td(7, 60, 1000)) + eq(a-b, td(6, 24*3600 - 60)) + eq(b.__rsub__(a), td(6, 24*3600 - 60)) + eq(-a, td(-7)) + eq(+a, td(7)) + eq(-b, td(-1, 24*3600 - 60)) + eq(-c, td(-1, 24*3600 - 1, 999000)) + eq(abs(a), a) + eq(abs(-a), a) + eq(td(6, 24*3600), a) + eq(td(0, 0, 60*1000000), b) + eq(a*10, td(70)) + eq(a*10, 10*a) + eq(a*10, 10*a) + eq(b*10, td(0, 600)) + eq(10*b, td(0, 600)) + eq(b*10, td(0, 600)) + eq(c*10, td(0, 0, 10000)) + eq(10*c, td(0, 0, 10000)) + eq(c*10, td(0, 0, 10000)) + eq(a*-1, -a) + eq(b*-2, -b-b) + eq(c*-2, -c+-c) + eq(b*(60*24), (b*60)*24) + eq(b*(60*24), (60*b)*24) + eq(c*1000, td(0, 1)) + eq(1000*c, td(0, 1)) + eq(a//7, td(1)) + eq(b//10, td(0, 6)) + eq(c//1000, td(0, 0, 1)) + eq(a//10, td(0, 7*24*360)) + eq(a//3600000, td(0, 0, 7*24*1000)) + eq(a/0.5, td(14)) + eq(b/0.5, td(0, 120)) + eq(a/7, td(1)) + eq(b/10, td(0, 6)) + eq(c/1000, td(0, 0, 1)) + eq(a/10, td(0, 7*24*360)) + eq(a/3600000, td(0, 0, 7*24*1000)) + + # Multiplication by float + us = td(microseconds=1) + eq((3*us) * 0.5, 2*us) + eq((5*us) * 0.5, 2*us) + eq(0.5 * (3*us), 2*us) + eq(0.5 * (5*us), 2*us) + eq((-3*us) * 0.5, -2*us) + eq((-5*us) * 0.5, -2*us) + + # Issue #23521 + eq(td(seconds=1) * 0.123456, td(microseconds=123456)) + eq(td(seconds=1) * 0.6112295, td(microseconds=611229)) + + # Division by int and float + eq((3*us) / 2, 2*us) + eq((5*us) / 2, 2*us) + eq((-3*us) / 2.0, -2*us) + eq((-5*us) / 2.0, -2*us) + eq((3*us) / -2, -2*us) + eq((5*us) / -2, -2*us) + eq((3*us) / -2.0, -2*us) + eq((5*us) / -2.0, -2*us) + for i in range(-10, 10): + eq((i*us/3)//us, round(i/3)) + for i in range(-10, 10): + eq((i*us/-3)//us, round(i/-3)) + + # Issue #23521 + eq(td(seconds=1) / (1 / 0.6112295), td(microseconds=611229)) + + # Issue #11576 + eq(td(999999999, 86399, 999999) - td(999999999, 86399, 999998), + td(0, 0, 1)) + eq(td(999999999, 1, 1) - td(999999999, 1, 0), + td(0, 0, 1)) + + def test_disallowed_computations(self): + a = timedelta(42) + + # Add/sub ints or floats should be illegal + for i in 1, 1.0: + self.assertRaises(TypeError, lambda: a+i) + self.assertRaises(TypeError, lambda: a-i) + self.assertRaises(TypeError, lambda: i+a) + self.assertRaises(TypeError, lambda: i-a) + + # Division of int by timedelta doesn't make sense. + # Division by zero doesn't make sense. + zero = 0 + self.assertRaises(TypeError, lambda: zero // a) + self.assertRaises(ZeroDivisionError, lambda: a // zero) + self.assertRaises(ZeroDivisionError, lambda: a / zero) + self.assertRaises(ZeroDivisionError, lambda: a / 0.0) + self.assertRaises(TypeError, lambda: a / '') + + @support.requires_IEEE_754 + def test_disallowed_special(self): + a = timedelta(42) + self.assertRaises(ValueError, a.__mul__, NAN) + self.assertRaises(ValueError, a.__truediv__, NAN) + + def test_basic_attributes(self): + days, seconds, us = 1, 7, 31 + td = timedelta(days, seconds, us) + self.assertEqual(td.days, days) + self.assertEqual(td.seconds, seconds) + self.assertEqual(td.microseconds, us) + + def test_total_seconds(self): + td = timedelta(days=365) + self.assertEqual(td.total_seconds(), 31536000.0) + for total_seconds in [123456.789012, -123456.789012, 0.123456, 0, 1e6]: + td = timedelta(seconds=total_seconds) + self.assertEqual(td.total_seconds(), total_seconds) + # Issue8644: Test that td.total_seconds() has the same + # accuracy as td / timedelta(seconds=1). + for ms in [-1, -2, -123]: + td = timedelta(microseconds=ms) + self.assertEqual(td.total_seconds(), td / timedelta(seconds=1)) + + def test_carries(self): + t1 = timedelta(days=100, + weeks=-7, + hours=-24*(100-49), + minutes=-3, + seconds=12, + microseconds=(3*60 - 12) * 1e6 + 1) + t2 = timedelta(microseconds=1) + self.assertEqual(t1, t2) + + def test_hash_equality(self): + t1 = timedelta(days=100, + weeks=-7, + hours=-24*(100-49), + minutes=-3, + seconds=12, + microseconds=(3*60 - 12) * 1000000) + t2 = timedelta() + self.assertEqual(hash(t1), hash(t2)) + + t1 += timedelta(weeks=7) + t2 += timedelta(days=7*7) + self.assertEqual(t1, t2) + self.assertEqual(hash(t1), hash(t2)) + + d = {t1: 1} + d[t2] = 2 + self.assertEqual(len(d), 1) + self.assertEqual(d[t1], 2) + + def test_pickling(self): + args = 12, 34, 56 + orig = timedelta(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + + def test_compare(self): + t1 = timedelta(2, 3, 4) + t2 = timedelta(2, 3, 4) + self.assertEqual(t1, t2) + self.assertTrue(t1 <= t2) + self.assertTrue(t1 >= t2) + self.assertFalse(t1 != t2) + self.assertFalse(t1 < t2) + self.assertFalse(t1 > t2) + + for args in (3, 3, 3), (2, 4, 4), (2, 3, 5): + t2 = timedelta(*args) # this is larger than t1 + self.assertTrue(t1 < t2) + self.assertTrue(t2 > t1) + self.assertTrue(t1 <= t2) + self.assertTrue(t2 >= t1) + self.assertTrue(t1 != t2) + self.assertTrue(t2 != t1) + self.assertFalse(t1 == t2) + self.assertFalse(t2 == t1) + self.assertFalse(t1 > t2) + self.assertFalse(t2 < t1) + self.assertFalse(t1 >= t2) + self.assertFalse(t2 <= t1) + + for badarg in OTHERSTUFF: + self.assertEqual(t1 == badarg, False) + self.assertEqual(t1 != badarg, True) + self.assertEqual(badarg == t1, False) + self.assertEqual(badarg != t1, True) + + self.assertRaises(TypeError, lambda: t1 <= badarg) + self.assertRaises(TypeError, lambda: t1 < badarg) + self.assertRaises(TypeError, lambda: t1 > badarg) + self.assertRaises(TypeError, lambda: t1 >= badarg) + self.assertRaises(TypeError, lambda: badarg <= t1) + self.assertRaises(TypeError, lambda: badarg < t1) + self.assertRaises(TypeError, lambda: badarg > t1) + self.assertRaises(TypeError, lambda: badarg >= t1) + + def test_str(self): + td = timedelta + eq = self.assertEqual + + eq(str(td(1)), "1 day, 0:00:00") + eq(str(td(-1)), "-1 day, 0:00:00") + eq(str(td(2)), "2 days, 0:00:00") + eq(str(td(-2)), "-2 days, 0:00:00") + + eq(str(td(hours=12, minutes=58, seconds=59)), "12:58:59") + eq(str(td(hours=2, minutes=3, seconds=4)), "2:03:04") + eq(str(td(weeks=-30, hours=23, minutes=12, seconds=34)), + "-210 days, 23:12:34") + + eq(str(td(milliseconds=1)), "0:00:00.001000") + eq(str(td(microseconds=3)), "0:00:00.000003") + + eq(str(td(days=999999999, hours=23, minutes=59, seconds=59, + microseconds=999999)), + "999999999 days, 23:59:59.999999") + + # test the Doc/library/datetime.rst recipe + eq(f'-({-td(hours=-1)!s})', "-(1:00:00)") + + def test_repr(self): + name = 'datetime.' + self.theclass.__name__ + self.assertEqual(repr(self.theclass(1)), + "%s(days=1)" % name) + self.assertEqual(repr(self.theclass(10, 2)), + "%s(days=10, seconds=2)" % name) + self.assertEqual(repr(self.theclass(-10, 2, 400000)), + "%s(days=-10, seconds=2, microseconds=400000)" % name) + self.assertEqual(repr(self.theclass(seconds=60)), + "%s(seconds=60)" % name) + self.assertEqual(repr(self.theclass()), + "%s(0)" % name) + self.assertEqual(repr(self.theclass(microseconds=100)), + "%s(microseconds=100)" % name) + self.assertEqual(repr(self.theclass(days=1, microseconds=100)), + "%s(days=1, microseconds=100)" % name) + self.assertEqual(repr(self.theclass(seconds=1, microseconds=100)), + "%s(seconds=1, microseconds=100)" % name) + + def test_repr_subclass(self): + """Subclasses should have bare names in the repr (gh-107773).""" + td = SubclassTimeDelta(days=1) + self.assertEqual(repr(td), "SubclassTimeDelta(days=1)") + td = SubclassTimeDelta(seconds=30) + self.assertEqual(repr(td), "SubclassTimeDelta(seconds=30)") + td = SubclassTimeDelta(weeks=2) + self.assertEqual(repr(td), "SubclassTimeDelta(days=14)") + + def test_roundtrip(self): + for td in (timedelta(days=999999999, hours=23, minutes=59, + seconds=59, microseconds=999999), + timedelta(days=-999999999), + timedelta(days=-999999999, seconds=1), + timedelta(days=1, seconds=2, microseconds=3)): + + # Verify td -> string -> td identity. + s = repr(td) + self.assertStartsWith(s, 'datetime.') + s = s[9:] + td2 = eval(s) + self.assertEqual(td, td2) + + # Verify identity via reconstructing from pieces. + td2 = timedelta(td.days, td.seconds, td.microseconds) + self.assertEqual(td, td2) + + def test_resolution_info(self): + self.assertIsInstance(timedelta.min, timedelta) + self.assertIsInstance(timedelta.max, timedelta) + self.assertIsInstance(timedelta.resolution, timedelta) + self.assertTrue(timedelta.max > timedelta.min) + self.assertEqual(timedelta.min, timedelta(-999999999)) + self.assertEqual(timedelta.max, timedelta(999999999, 24*3600-1, 1e6-1)) + self.assertEqual(timedelta.resolution, timedelta(0, 0, 1)) + + def test_overflow(self): + tiny = timedelta.resolution + + td = timedelta.min + tiny + td -= tiny # no problem + self.assertRaises(OverflowError, td.__sub__, tiny) + self.assertRaises(OverflowError, td.__add__, -tiny) + + td = timedelta.max - tiny + td += tiny # no problem + self.assertRaises(OverflowError, td.__add__, tiny) + self.assertRaises(OverflowError, td.__sub__, -tiny) + + self.assertRaises(OverflowError, lambda: -timedelta.max) + + day = timedelta(1) + self.assertRaises(OverflowError, day.__mul__, 10**9) + self.assertRaises(OverflowError, day.__mul__, 1e9) + self.assertRaises(OverflowError, day.__truediv__, 1e-20) + self.assertRaises(OverflowError, day.__truediv__, 1e-10) + self.assertRaises(OverflowError, day.__truediv__, 9e-10) + + @support.requires_IEEE_754 + def _test_overflow_special(self): + day = timedelta(1) + self.assertRaises(OverflowError, day.__mul__, INF) + self.assertRaises(OverflowError, day.__mul__, -INF) + + def test_microsecond_rounding(self): + td = timedelta + eq = self.assertEqual + + # Single-field rounding. + eq(td(milliseconds=0.4/1000), td(0)) # rounds to 0 + eq(td(milliseconds=-0.4/1000), td(0)) # rounds to 0 + eq(td(milliseconds=0.5/1000), td(microseconds=0)) + eq(td(milliseconds=-0.5/1000), td(microseconds=-0)) + eq(td(milliseconds=0.6/1000), td(microseconds=1)) + eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) + eq(td(milliseconds=1.5/1000), td(microseconds=2)) + eq(td(milliseconds=-1.5/1000), td(microseconds=-2)) + eq(td(seconds=0.5/10**6), td(microseconds=0)) + eq(td(seconds=-0.5/10**6), td(microseconds=-0)) + eq(td(seconds=1/2**7), td(microseconds=7812)) + eq(td(seconds=-1/2**7), td(microseconds=-7812)) + + # Rounding due to contributions from more than one field. + us_per_hour = 3600e6 + us_per_day = us_per_hour * 24 + eq(td(days=.4/us_per_day), td(0)) + eq(td(hours=.2/us_per_hour), td(0)) + eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1)) + + eq(td(days=-.4/us_per_day), td(0)) + eq(td(hours=-.2/us_per_hour), td(0)) + eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) + + # Test for a patch in Issue 8860 + eq(td(microseconds=0.5), 0.5*td(microseconds=1.0)) + eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution) + + def test_massive_normalization(self): + td = timedelta(microseconds=-1) + self.assertEqual((td.days, td.seconds, td.microseconds), + (-1, 24*3600-1, 999999)) + + def test_bool(self): + self.assertTrue(timedelta(1)) + self.assertTrue(timedelta(0, 1)) + self.assertTrue(timedelta(0, 0, 1)) + self.assertTrue(timedelta(microseconds=1)) + self.assertFalse(timedelta(0)) + + def test_subclass_timedelta(self): + + class T(timedelta): + @staticmethod + def from_td(td): + return T(td.days, td.seconds, td.microseconds) + + def as_hours(self): + sum = (self.days * 24 + + self.seconds / 3600.0 + + self.microseconds / 3600e6) + return round(sum) + + t1 = T(days=1) + self.assertIs(type(t1), T) + self.assertEqual(t1.as_hours(), 24) + + t2 = T(days=-1, seconds=-3600) + self.assertIs(type(t2), T) + self.assertEqual(t2.as_hours(), -25) + + t3 = t1 + t2 + self.assertIs(type(t3), timedelta) + t4 = T.from_td(t3) + self.assertIs(type(t4), T) + self.assertEqual(t3.days, t4.days) + self.assertEqual(t3.seconds, t4.seconds) + self.assertEqual(t3.microseconds, t4.microseconds) + self.assertEqual(str(t3), str(t4)) + self.assertEqual(t4.as_hours(), -1) + + def test_subclass_date(self): + class DateSubclass(date): + pass + + d1 = DateSubclass(2018, 1, 5) + td = timedelta(days=1) + + tests = [ + ('add', lambda d, t: d + t, DateSubclass(2018, 1, 6)), + ('radd', lambda d, t: t + d, DateSubclass(2018, 1, 6)), + ('sub', lambda d, t: d - t, DateSubclass(2018, 1, 4)), + ] + + for name, func, expected in tests: + with self.subTest(name): + act = func(d1, td) + self.assertEqual(act, expected) + self.assertIsInstance(act, DateSubclass) + + def test_subclass_datetime(self): + class DateTimeSubclass(datetime): + pass + + d1 = DateTimeSubclass(2018, 1, 5, 12, 30) + td = timedelta(days=1, minutes=30) + + tests = [ + ('add', lambda d, t: d + t, DateTimeSubclass(2018, 1, 6, 13)), + ('radd', lambda d, t: t + d, DateTimeSubclass(2018, 1, 6, 13)), + ('sub', lambda d, t: d - t, DateTimeSubclass(2018, 1, 4, 12)), + ] + + for name, func, expected in tests: + with self.subTest(name): + act = func(d1, td) + self.assertEqual(act, expected) + self.assertIsInstance(act, DateTimeSubclass) + + def test_division(self): + t = timedelta(hours=1, minutes=24, seconds=19) + second = timedelta(seconds=1) + self.assertEqual(t / second, 5059.0) + self.assertEqual(t // second, 5059) + + t = timedelta(minutes=2, seconds=30) + minute = timedelta(minutes=1) + self.assertEqual(t / minute, 2.5) + self.assertEqual(t // minute, 2) + + zerotd = timedelta(0) + self.assertRaises(ZeroDivisionError, truediv, t, zerotd) + self.assertRaises(ZeroDivisionError, floordiv, t, zerotd) + + # self.assertRaises(TypeError, truediv, t, 2) + # note: floor division of a timedelta by an integer *is* + # currently permitted. + + def test_remainder(self): + t = timedelta(minutes=2, seconds=30) + minute = timedelta(minutes=1) + r = t % minute + self.assertEqual(r, timedelta(seconds=30)) + + t = timedelta(minutes=-2, seconds=30) + r = t % minute + self.assertEqual(r, timedelta(seconds=30)) + + zerotd = timedelta(0) + self.assertRaises(ZeroDivisionError, mod, t, zerotd) + + self.assertRaises(TypeError, mod, t, 10) + + def test_divmod(self): + t = timedelta(minutes=2, seconds=30) + minute = timedelta(minutes=1) + q, r = divmod(t, minute) + self.assertEqual(q, 2) + self.assertEqual(r, timedelta(seconds=30)) + + t = timedelta(minutes=-2, seconds=30) + q, r = divmod(t, minute) + self.assertEqual(q, -2) + self.assertEqual(r, timedelta(seconds=30)) + + zerotd = timedelta(0) + self.assertRaises(ZeroDivisionError, divmod, t, zerotd) + + self.assertRaises(TypeError, divmod, t, 10) + + def test_issue31293(self): + # The interpreter shouldn't crash in case a timedelta is divided or + # multiplied by a float with a bad as_integer_ratio() method. + def get_bad_float(bad_ratio): + class BadFloat(float): + def as_integer_ratio(self): + return bad_ratio + return BadFloat() + + with self.assertRaises(TypeError): + timedelta() / get_bad_float(1 << 1000) + with self.assertRaises(TypeError): + timedelta() * get_bad_float(1 << 1000) + + for bad_ratio in [(), (42, ), (1, 2, 3)]: + with self.assertRaises(ValueError): + timedelta() / get_bad_float(bad_ratio) + with self.assertRaises(ValueError): + timedelta() * get_bad_float(bad_ratio) + + def test_issue31752(self): + # The interpreter shouldn't crash because divmod() returns negative + # remainder. + class BadInt(int): + def __mul__(self, other): + return Prod() + def __rmul__(self, other): + return Prod() + def __floordiv__(self, other): + return Prod() + def __rfloordiv__(self, other): + return Prod() + + class Prod: + def __add__(self, other): + return Sum() + def __radd__(self, other): + return Sum() + + class Sum(int): + def __divmod__(self, other): + return divmodresult + + for divmodresult in [None, (), (0, 1, 2), (0, -1)]: + with self.subTest(divmodresult=divmodresult): + # The following examples should not crash. + try: + timedelta(microseconds=BadInt(1)) + except TypeError: + pass + try: + timedelta(hours=BadInt(1)) + except TypeError: + pass + try: + timedelta(weeks=BadInt(1)) + except (TypeError, ValueError): + pass + try: + timedelta(1) * BadInt(1) + except (TypeError, ValueError): + pass + try: + BadInt(1) * timedelta(1) + except TypeError: + pass + try: + timedelta(1) // BadInt(1) + except TypeError: + pass + + +############################################################################# +# date tests + +class TestDateOnly(unittest.TestCase): + # Tests here won't pass if also run on datetime objects, so don't + # subclass this to test datetimes too. + + def test_delta_non_days_ignored(self): + dt = date(2000, 1, 2) + delta = timedelta(days=1, hours=2, minutes=3, seconds=4, + microseconds=5) + days = timedelta(delta.days) + self.assertEqual(days, timedelta(1)) + + dt2 = dt + delta + self.assertEqual(dt2, dt + days) + + dt2 = delta + dt + self.assertEqual(dt2, dt + days) + + dt2 = dt - delta + self.assertEqual(dt2, dt - days) + + delta = -delta + days = timedelta(delta.days) + self.assertEqual(days, timedelta(-2)) + + dt2 = dt + delta + self.assertEqual(dt2, dt + days) + + dt2 = delta + dt + self.assertEqual(dt2, dt + days) + + dt2 = dt - delta + self.assertEqual(dt2, dt - days) + + def test_strptime(self): + inputs = [ + # Basic valid cases + (date(1998, 2, 3), '1998-02-03', '%Y-%m-%d'), + (date(2004, 12, 2), '2004-12-02', '%Y-%m-%d'), + + # Edge cases: Leap year + (date(2020, 2, 29), '2020-02-29', '%Y-%m-%d'), # Valid leap year date + + # bpo-34482: Handle surrogate pairs + (date(2004, 12, 2), '2004-12\ud80002', '%Y-%m\ud800%d'), + (date(2004, 12, 2), '2004\ud80012-02', '%Y\ud800%m-%d'), + + # Month/day variations + (date(2004, 2, 1), '2004-02', '%Y-%m'), # No day provided + (date(2004, 2, 1), '02-2004', '%m-%Y'), # Month and year swapped + + # Different day-month-year formats + (date(2004, 12, 2), '02/12/2004', '%d/%m/%Y'), # Day/Month/Year + (date(2004, 12, 2), '12/02/2004', '%m/%d/%Y'), # Month/Day/Year + + # Different separators + (date(2023, 9, 24), '24.09.2023', '%d.%m.%Y'), # Dots as separators + (date(2023, 9, 24), '24-09-2023', '%d-%m-%Y'), # Dashes + (date(2023, 9, 24), '2023/09/24', '%Y/%m/%d'), # Slashes + + # Handling years with fewer digits + (date(127, 2, 3), '0127-02-03', '%Y-%m-%d'), + (date(99, 2, 3), '0099-02-03', '%Y-%m-%d'), + (date(5, 2, 3), '0005-02-03', '%Y-%m-%d'), + + # Variations on ISO 8601 format + (date(2023, 9, 25), '2023-W39-1', '%G-W%V-%u'), # ISO week date (Week 39, Monday) + (date(2023, 9, 25), '2023-268', '%Y-%j'), # Year and day of the year (Julian) + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = date.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), date) + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit dates are allowed. + strptime = date.strptime + with self.assertRaises(ValueError): + # %y does require two digits. + newdate = strptime('01/02/3', '%d/%m/%y') + + d1 = date(2003, 2, 1) + d2 = date(2003, 1, 2) + d3 = date(2003, 1, 25) + inputs = [ + ('%d', '1/02/03', '%d/%m/%y', d1), + ('%m', '01/2/03', '%d/%m/%y', d1), + ('%j', '2/03', '%j/%y', d2), + ('%w', '6/04/03', '%w/%U/%y', d1), + # %u requires a single digit. + ('%W', '6/4/2003', '%u/%W/%Y', d1), + ('%V', '6/4/2003', '%u/%V/%G', d3), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + date.strptime('02-29', '%m-%d') + with self._assertNotWarns(DeprecationWarning): + date.strptime('20-03-14', '%y-%m-%d') + date.strptime('02-29,2024', '%m-%d,%Y') + +class SubclassDate(date): + sub_var = 1 + +class TestDate(HarmlessMixedComparison, unittest.TestCase): + # Tests here should pass for both dates and datetimes, except for a + # few tests that TestDateTime overrides. + + theclass = date + + def test_basic_attributes(self): + dt = self.theclass(2002, 3, 1) + self.assertEqual(dt.year, 2002) + self.assertEqual(dt.month, 3) + self.assertEqual(dt.day, 1) + + def test_roundtrip(self): + for dt in (self.theclass(1, 2, 3), + self.theclass.today()): + # Verify dt -> string -> date identity. + s = repr(dt) + self.assertStartsWith(s, 'datetime.') + s = s[9:] + dt2 = eval(s) + self.assertEqual(dt, dt2) + + # Verify identity via reconstructing from pieces. + dt2 = self.theclass(dt.year, dt.month, dt.day) + self.assertEqual(dt, dt2) + + def test_repr_subclass(self): + """Subclasses should have bare names in the repr (gh-107773).""" + td = SubclassDate(1, 2, 3) + self.assertEqual(repr(td), "SubclassDate(1, 2, 3)") + td = SubclassDate(2014, 1, 1) + self.assertEqual(repr(td), "SubclassDate(2014, 1, 1)") + td = SubclassDate(2010, 10, day=10) + self.assertEqual(repr(td), "SubclassDate(2010, 10, 10)") + + def test_ordinal_conversions(self): + # Check some fixed values. + for y, m, d, n in [(1, 1, 1, 1), # calendar origin + (1, 12, 31, 365), + (2, 1, 1, 366), + # first example from "Calendrical Calculations" + (1945, 11, 12, 710347)]: + d = self.theclass(y, m, d) + self.assertEqual(n, d.toordinal()) + fromord = self.theclass.fromordinal(n) + self.assertEqual(d, fromord) + if hasattr(fromord, "hour"): + # if we're checking something fancier than a date, verify + # the extra fields have been zeroed out + self.assertEqual(fromord.hour, 0) + self.assertEqual(fromord.minute, 0) + self.assertEqual(fromord.second, 0) + self.assertEqual(fromord.microsecond, 0) + + # Check first and last days of year spottily across the whole + # range of years supported. + for year in range(MINYEAR, MAXYEAR+1, 7): + # Verify (year, 1, 1) -> ordinal -> y, m, d is identity. + d = self.theclass(year, 1, 1) + n = d.toordinal() + d2 = self.theclass.fromordinal(n) + self.assertEqual(d, d2) + # Verify that moving back a day gets to the end of year-1. + if year > 1: + d = self.theclass.fromordinal(n-1) + d2 = self.theclass(year-1, 12, 31) + self.assertEqual(d, d2) + self.assertEqual(d2.toordinal(), n-1) + + # Test every day in a leap-year and a non-leap year. + dim = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + for year, isleap in (2000, True), (2002, False): + n = self.theclass(year, 1, 1).toordinal() + for month, maxday in zip(range(1, 13), dim): + if month == 2 and isleap: + maxday += 1 + for day in range(1, maxday+1): + d = self.theclass(year, month, day) + self.assertEqual(d.toordinal(), n) + self.assertEqual(d, self.theclass.fromordinal(n)) + n += 1 + + def test_extreme_ordinals(self): + a = self.theclass.min + a = self.theclass(a.year, a.month, a.day) # get rid of time parts + aord = a.toordinal() + b = a.fromordinal(aord) + self.assertEqual(a, b) + + self.assertRaises(ValueError, lambda: a.fromordinal(aord - 1)) + + b = a + timedelta(days=1) + self.assertEqual(b.toordinal(), aord + 1) + self.assertEqual(b, self.theclass.fromordinal(aord + 1)) + + a = self.theclass.max + a = self.theclass(a.year, a.month, a.day) # get rid of time parts + aord = a.toordinal() + b = a.fromordinal(aord) + self.assertEqual(a, b) + + self.assertRaises(ValueError, lambda: a.fromordinal(aord + 1)) + + b = a - timedelta(days=1) + self.assertEqual(b.toordinal(), aord - 1) + self.assertEqual(b, self.theclass.fromordinal(aord - 1)) + + def test_bad_constructor_arguments(self): + # bad years + self.theclass(MINYEAR, 1, 1) # no exception + self.theclass(MAXYEAR, 1, 1) # no exception + self.assertRaises(ValueError, self.theclass, MINYEAR-1, 1, 1) + self.assertRaises(ValueError, self.theclass, MAXYEAR+1, 1, 1) + # bad months + self.theclass(2000, 1, 1) # no exception + self.theclass(2000, 12, 1) # no exception + self.assertRaises(ValueError, self.theclass, 2000, 0, 1) + self.assertRaises(ValueError, self.theclass, 2000, 13, 1) + # bad days + self.theclass(2000, 2, 29) # no exception + self.theclass(2004, 2, 29) # no exception + self.theclass(2400, 2, 29) # no exception + self.assertRaises(ValueError, self.theclass, 2000, 2, 30) + self.assertRaises(ValueError, self.theclass, 2001, 2, 29) + self.assertRaises(ValueError, self.theclass, 2100, 2, 29) + self.assertRaises(ValueError, self.theclass, 1900, 2, 29) + self.assertRaises(ValueError, self.theclass, 2000, 1, 0) + self.assertRaises(ValueError, self.theclass, 2000, 1, 32) + + def test_hash_equality(self): + d = self.theclass(2000, 12, 31) + # same thing + e = self.theclass(2000, 12, 31) + self.assertEqual(d, e) + self.assertEqual(hash(d), hash(e)) + + dic = {d: 1} + dic[e] = 2 + self.assertEqual(len(dic), 1) + self.assertEqual(dic[d], 2) + self.assertEqual(dic[e], 2) + + d = self.theclass(2001, 1, 1) + # same thing + e = self.theclass(2001, 1, 1) + self.assertEqual(d, e) + self.assertEqual(hash(d), hash(e)) + + dic = {d: 1} + dic[e] = 2 + self.assertEqual(len(dic), 1) + self.assertEqual(dic[d], 2) + self.assertEqual(dic[e], 2) + + def test_computations(self): + a = self.theclass(2002, 1, 31) + b = self.theclass(1956, 1, 31) + c = self.theclass(2001,2,1) + + diff = a-b + self.assertEqual(diff.days, 46*365 + len(range(1956, 2002, 4))) + self.assertEqual(diff.seconds, 0) + self.assertEqual(diff.microseconds, 0) + + day = timedelta(1) + week = timedelta(7) + a = self.theclass(2002, 3, 2) + self.assertEqual(a + day, self.theclass(2002, 3, 3)) + self.assertEqual(day + a, self.theclass(2002, 3, 3)) + self.assertEqual(a - day, self.theclass(2002, 3, 1)) + self.assertEqual(-day + a, self.theclass(2002, 3, 1)) + self.assertEqual(a + week, self.theclass(2002, 3, 9)) + self.assertEqual(a - week, self.theclass(2002, 2, 23)) + self.assertEqual(a + 52*week, self.theclass(2003, 3, 1)) + self.assertEqual(a - 52*week, self.theclass(2001, 3, 3)) + self.assertEqual((a + week) - a, week) + self.assertEqual((a + day) - a, day) + self.assertEqual((a - week) - a, -week) + self.assertEqual((a - day) - a, -day) + self.assertEqual(a - (a + week), -week) + self.assertEqual(a - (a + day), -day) + self.assertEqual(a - (a - week), week) + self.assertEqual(a - (a - day), day) + self.assertEqual(c - (c - day), day) + + # Add/sub ints or floats should be illegal + for i in 1, 1.0: + self.assertRaises(TypeError, lambda: a+i) + self.assertRaises(TypeError, lambda: a-i) + self.assertRaises(TypeError, lambda: i+a) + self.assertRaises(TypeError, lambda: i-a) + + # delta - date is senseless. + self.assertRaises(TypeError, lambda: day - a) + # mixing date and (delta or date) via * or // is senseless + self.assertRaises(TypeError, lambda: day * a) + self.assertRaises(TypeError, lambda: a * day) + self.assertRaises(TypeError, lambda: day // a) + self.assertRaises(TypeError, lambda: a // day) + self.assertRaises(TypeError, lambda: a * a) + self.assertRaises(TypeError, lambda: a // a) + # date + date is senseless + self.assertRaises(TypeError, lambda: a + a) + + def test_overflow(self): + tiny = self.theclass.resolution + + for delta in [tiny, timedelta(1), timedelta(2)]: + dt = self.theclass.min + delta + dt -= delta # no problem + self.assertRaises(OverflowError, dt.__sub__, delta) + self.assertRaises(OverflowError, dt.__add__, -delta) + + dt = self.theclass.max - delta + dt += delta # no problem + self.assertRaises(OverflowError, dt.__add__, delta) + self.assertRaises(OverflowError, dt.__sub__, -delta) + + def test_fromtimestamp(self): + import time + + # Try an arbitrary fixed value. + year, month, day = 1999, 9, 19 + ts = time.mktime((year, month, day, 0, 0, 0, 0, 0, -1)) + d = self.theclass.fromtimestamp(ts) + self.assertEqual(d.year, year) + self.assertEqual(d.month, month) + self.assertEqual(d.day, day) + + def test_insane_fromtimestamp(self): + # It's possible that some platform maps time_t to double, + # and that this test will fail there. This test should + # exempt such platforms (provided they return reasonable + # results!). + for insane in -1e200, 1e200: + self.assertRaises(OverflowError, self.theclass.fromtimestamp, + insane) + + def test_fromtimestamp_with_none_arg(self): + # See gh-120268 for more details + with self.assertRaises(TypeError): + self.theclass.fromtimestamp(None) + + def test_today(self): + import time + + # We claim that today() is like fromtimestamp(time.time()), so + # prove it. + for dummy in range(3): + today = self.theclass.today() + ts = time.time() + todayagain = self.theclass.fromtimestamp(ts) + if today == todayagain: + break + # There are several legit reasons that could fail: + # 1. It recently became midnight, between the today() and the + # time() calls. + # 2. The platform time() has such fine resolution that we'll + # never get the same value twice. + # 3. The platform time() has poor resolution, and we just + # happened to call today() right before a resolution quantum + # boundary. + # 4. The system clock got fiddled between calls. + # In any case, wait a little while and try again. + time.sleep(0.1) + + # It worked or it didn't. If it didn't, assume it's reason #2, and + # let the test pass if they're within half a second of each other. + if today != todayagain: + self.assertAlmostEqual(todayagain, today, + delta=timedelta(seconds=0.5)) + + def test_weekday(self): + for i in range(7): + # March 4, 2002 is a Monday + self.assertEqual(self.theclass(2002, 3, 4+i).weekday(), i) + self.assertEqual(self.theclass(2002, 3, 4+i).isoweekday(), i+1) + # January 2, 1956 is a Monday + self.assertEqual(self.theclass(1956, 1, 2+i).weekday(), i) + self.assertEqual(self.theclass(1956, 1, 2+i).isoweekday(), i+1) + + def test_isocalendar(self): + # Check examples from + # http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm + week_mondays = [ + ((2003, 12, 22), (2003, 52, 1)), + ((2003, 12, 29), (2004, 1, 1)), + ((2004, 1, 5), (2004, 2, 1)), + ((2009, 12, 21), (2009, 52, 1)), + ((2009, 12, 28), (2009, 53, 1)), + ((2010, 1, 4), (2010, 1, 1)), + ] + + test_cases = [] + for cal_date, iso_date in week_mondays: + base_date = self.theclass(*cal_date) + # Adds one test case for every day of the specified weeks + for i in range(7): + new_date = base_date + timedelta(i) + new_iso = iso_date[0:2] + (iso_date[2] + i,) + test_cases.append((new_date, new_iso)) + + for d, exp_iso in test_cases: + with self.subTest(d=d, comparison="tuple"): + self.assertEqual(d.isocalendar(), exp_iso) + + # Check that the tuple contents are accessible by field name + with self.subTest(d=d, comparison="fields"): + t = d.isocalendar() + self.assertEqual((t.year, t.week, t.weekday), exp_iso) + + def test_isocalendar_pickling(self): + """Test that the result of datetime.isocalendar() can be pickled. + + The result of a round trip should be a plain tuple. + """ + d = self.theclass(2019, 1, 1) + p = pickle.dumps(d.isocalendar()) + res = pickle.loads(p) + self.assertEqual(type(res), tuple) + self.assertEqual(res, (2019, 1, 2)) + + def test_iso_long_years(self): + # Calculate long ISO years and compare to table from + # http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm + ISO_LONG_YEARS_TABLE = """ + 4 32 60 88 + 9 37 65 93 + 15 43 71 99 + 20 48 76 + 26 54 82 + + 105 133 161 189 + 111 139 167 195 + 116 144 172 + 122 150 178 + 128 156 184 + + 201 229 257 285 + 207 235 263 291 + 212 240 268 296 + 218 246 274 + 224 252 280 + + 303 331 359 387 + 308 336 364 392 + 314 342 370 398 + 320 348 376 + 325 353 381 + """ + iso_long_years = sorted(map(int, ISO_LONG_YEARS_TABLE.split())) + L = [] + for i in range(400): + d = self.theclass(2000+i, 12, 31) + d1 = self.theclass(1600+i, 12, 31) + self.assertEqual(d.isocalendar()[1:], d1.isocalendar()[1:]) + if d.isocalendar()[1] == 53: + L.append(i) + self.assertEqual(L, iso_long_years) + + def test_isoformat(self): + t = self.theclass(2, 3, 2) + self.assertEqual(t.isoformat(), "0002-03-02") + + def test_ctime(self): + t = self.theclass(2002, 3, 2) + self.assertEqual(t.ctime(), "Sat Mar 2 00:00:00 2002") + + def test_strftime(self): + t = self.theclass(2005, 3, 2) + self.assertEqual(t.strftime("m:%m d:%d y:%y"), "m:03 d:02 y:05") + self.assertEqual(t.strftime(""), "") # SF bug #761337 + self.assertEqual(t.strftime('x'*1000), 'x'*1000) # SF bug #1556784 + + self.assertRaises(TypeError, t.strftime) # needs an arg + self.assertRaises(TypeError, t.strftime, "one", "two") # too many args + self.assertRaises(TypeError, t.strftime, 42) # arg wrong type + + # test that unicode input is allowed (issue 2782) + self.assertEqual(t.strftime("%m"), "03") + + # A naive object replaces %z, %:z and %Z w/ empty strings. + self.assertEqual(t.strftime("'%z' '%:z' '%Z'"), "'' '' ''") + + #make sure that invalid format specifiers are handled correctly + #self.assertRaises(ValueError, t.strftime, "%e") + #self.assertRaises(ValueError, t.strftime, "%") + #self.assertRaises(ValueError, t.strftime, "%#") + + #oh well, some systems just ignore those invalid ones. + #at least, exercise them to make sure that no crashes + #are generated + for f in ["%e", "%", "%#"]: + try: + t.strftime(f) + except ValueError: + pass + + # bpo-34482: Check that surrogates don't cause a crash. + try: + t.strftime('%y\ud800%m') + except UnicodeEncodeError: + pass + + #check that this standard extension works + t.strftime("%f") + + # bpo-41260: The parameter was named "fmt" in the pure python impl. + t.strftime(format="%f") + + def test_strftime_trailing_percent(self): + # bpo-35066: Make sure trailing '%' doesn't cause datetime's strftime to + # complain. Different libcs have different handling of trailing + # percents, so we simply check datetime's strftime acts the same as + # time.strftime. + t = self.theclass(2005, 3, 2) + try: + _time.strftime('%') + except ValueError: + self.skipTest('time module does not support trailing %') + self.assertEqual(t.strftime('%'), _time.strftime('%', t.timetuple())) + self.assertEqual( + t.strftime("m:%m d:%d y:%y %"), + _time.strftime("m:03 d:02 y:05 %", t.timetuple()), + ) + + def test_format(self): + dt = self.theclass(2007, 9, 10) + self.assertEqual(dt.__format__(''), str(dt)) + + with self.assertRaisesRegex(TypeError, 'must be str, not int'): + dt.__format__(123) + + # check that a derived class's __str__() gets called + class A(self.theclass): + def __str__(self): + return 'A' + a = A(2007, 9, 10) + self.assertEqual(a.__format__(''), 'A') + + # check that a derived class's strftime gets called + class B(self.theclass): + def strftime(self, format_spec): + return 'B' + b = B(2007, 9, 10) + self.assertEqual(b.__format__(''), str(dt)) + + for fmt in ["m:%m d:%d y:%y", + "m:%m d:%d y:%y H:%H M:%M S:%S", + "%z %:z %Z", + ]: + self.assertEqual(dt.__format__(fmt), dt.strftime(fmt)) + self.assertEqual(a.__format__(fmt), dt.strftime(fmt)) + self.assertEqual(b.__format__(fmt), 'B') + + def test_resolution_info(self): + # XXX: Should min and max respect subclassing? + if issubclass(self.theclass, datetime): + expected_class = datetime + else: + expected_class = date + self.assertIsInstance(self.theclass.min, expected_class) + self.assertIsInstance(self.theclass.max, expected_class) + self.assertIsInstance(self.theclass.resolution, timedelta) + self.assertTrue(self.theclass.max > self.theclass.min) + + def test_extreme_timedelta(self): + big = self.theclass.max - self.theclass.min + # 3652058 days, 23 hours, 59 minutes, 59 seconds, 999999 microseconds + n = (big.days*24*3600 + big.seconds)*1000000 + big.microseconds + # n == 315537897599999999 ~= 2**58.13 + justasbig = timedelta(0, 0, n) + self.assertEqual(big, justasbig) + self.assertEqual(self.theclass.min + big, self.theclass.max) + self.assertEqual(self.theclass.max - big, self.theclass.min) + + def test_timetuple(self): + for i in range(7): + # January 2, 1956 is a Monday (0) + d = self.theclass(1956, 1, 2+i) + t = d.timetuple() + self.assertEqual(t, (1956, 1, 2+i, 0, 0, 0, i, 2+i, -1)) + # February 1, 1956 is a Wednesday (2) + d = self.theclass(1956, 2, 1+i) + t = d.timetuple() + self.assertEqual(t, (1956, 2, 1+i, 0, 0, 0, (2+i)%7, 32+i, -1)) + # March 1, 1956 is a Thursday (3), and is the 31+29+1 = 61st day + # of the year. + d = self.theclass(1956, 3, 1+i) + t = d.timetuple() + self.assertEqual(t, (1956, 3, 1+i, 0, 0, 0, (3+i)%7, 61+i, -1)) + self.assertEqual(t.tm_year, 1956) + self.assertEqual(t.tm_mon, 3) + self.assertEqual(t.tm_mday, 1+i) + self.assertEqual(t.tm_hour, 0) + self.assertEqual(t.tm_min, 0) + self.assertEqual(t.tm_sec, 0) + self.assertEqual(t.tm_wday, (3+i)%7) + self.assertEqual(t.tm_yday, 61+i) + self.assertEqual(t.tm_isdst, -1) + + def test_pickling(self): + args = 6, 7, 23 + orig = self.theclass(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) + + def test_compat_unpickle(self): + tests = [ + b"cdatetime\ndate\n(S'\\x07\\xdf\\x0b\\x1b'\ntR.", + b'cdatetime\ndate\n(U\x04\x07\xdf\x0b\x1btR.', + b'\x80\x02cdatetime\ndate\nU\x04\x07\xdf\x0b\x1b\x85R.', + ] + args = 2015, 11, 27 + expected = self.theclass(*args) + for data in tests: + for loads in pickle_loads: + derived = loads(data, encoding='latin1') + self.assertEqual(derived, expected) + + def test_compare(self): + t1 = self.theclass(2, 3, 4) + t2 = self.theclass(2, 3, 4) + self.assertEqual(t1, t2) + self.assertTrue(t1 <= t2) + self.assertTrue(t1 >= t2) + self.assertFalse(t1 != t2) + self.assertFalse(t1 < t2) + self.assertFalse(t1 > t2) + + for args in (3, 3, 3), (2, 4, 4), (2, 3, 5): + t2 = self.theclass(*args) # this is larger than t1 + self.assertTrue(t1 < t2) + self.assertTrue(t2 > t1) + self.assertTrue(t1 <= t2) + self.assertTrue(t2 >= t1) + self.assertTrue(t1 != t2) + self.assertTrue(t2 != t1) + self.assertFalse(t1 == t2) + self.assertFalse(t2 == t1) + self.assertFalse(t1 > t2) + self.assertFalse(t2 < t1) + self.assertFalse(t1 >= t2) + self.assertFalse(t2 <= t1) + + for badarg in OTHERSTUFF: + self.assertEqual(t1 == badarg, False) + self.assertEqual(t1 != badarg, True) + self.assertEqual(badarg == t1, False) + self.assertEqual(badarg != t1, True) + + self.assertRaises(TypeError, lambda: t1 < badarg) + self.assertRaises(TypeError, lambda: t1 > badarg) + self.assertRaises(TypeError, lambda: t1 >= badarg) + self.assertRaises(TypeError, lambda: badarg <= t1) + self.assertRaises(TypeError, lambda: badarg < t1) + self.assertRaises(TypeError, lambda: badarg > t1) + self.assertRaises(TypeError, lambda: badarg >= t1) + + def test_mixed_compare(self): + our = self.theclass(2000, 4, 5) + + # Our class can be compared for equality to other classes + self.assertEqual(our == 1, False) + self.assertEqual(1 == our, False) + self.assertEqual(our != 1, True) + self.assertEqual(1 != our, True) + + # But the ordering is undefined + self.assertRaises(TypeError, lambda: our < 1) + self.assertRaises(TypeError, lambda: 1 < our) + + # Repeat those tests with a different class + + class SomeClass: + pass + + their = SomeClass() + self.assertEqual(our == their, False) + self.assertEqual(their == our, False) + self.assertEqual(our != their, True) + self.assertEqual(their != our, True) + self.assertRaises(TypeError, lambda: our < their) + self.assertRaises(TypeError, lambda: their < our) + + def test_bool(self): + # All dates are considered true. + self.assertTrue(self.theclass.min) + self.assertTrue(self.theclass.max) + + def check_strftime_y2k(self, specifier): + # Test that years less than 1000 are 0-padded; note that the beginning + # of an ISO 8601 year may fall in an ISO week of the year before, and + # therefore needs an offset of -1 when formatting with '%G'. + dataset = ( + (1, 0), + (49, -1), + (70, 0), + (99, 0), + (100, -1), + (999, 0), + (1000, 0), + (1970, 0), + ) + for year, g_offset in dataset: + with self.subTest(year=year, specifier=specifier): + d = self.theclass(year, 1, 1) + if specifier == 'G': + year += g_offset + if specifier == 'C': + expected = f"{year // 100:02d}" + else: + expected = f"{year:04d}" + if specifier == 'F': + expected += f"-01-01" + self.assertEqual(d.strftime(f"%{specifier}"), expected) + + def test_strftime_y2k(self): + self.check_strftime_y2k('Y') + self.check_strftime_y2k('G') + + def test_strftime_y2k_c99(self): + # CPython requires C11; specifiers new in C99 must work. + # (Other implementations may want to disable this test.) + self.check_strftime_y2k('F') + self.check_strftime_y2k('C') + + def test_replace(self): + cls = self.theclass + args = [1, 2, 3] + base = cls(*args) + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("year", 2), + ("month", 3), + ("day", 4)) + for i, (name, newval) in enumerate(changes): + newargs = args[:] + newargs[i] = newval + expected = cls(*newargs) + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) + + # Out of bounds. + base = cls(2000, 2, 29) + self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) + + def test_subclass_replace(self): + class DateSubclass(self.theclass): + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result + + dt = DateSubclass(2012, 1, 1) + + test_cases = [ + ('self.replace', dt.replace(year=2013)), + ('copy.replace', copy.replace(dt, year=2013)), + ] + + for name, res in test_cases: + with self.subTest(name): + self.assertIs(type(res), DateSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.month, 1) + self.assertEqual(res.extra, 7) + + def test_subclass_date(self): + + class C(self.theclass): + theAnswer = 42 + + def __new__(cls, *args, **kws): + temp = kws.copy() + extra = temp.pop('extra') + result = self.theclass.__new__(cls, *args, **temp) + result.extra = extra + return result + + def newmeth(self, start): + return start + self.year + self.month + + args = 2003, 4, 14 + + dt1 = self.theclass(*args) + dt2 = C(*args, **{'extra': 7}) + + self.assertEqual(dt2.__class__, C) + self.assertEqual(dt2.theAnswer, 42) + self.assertEqual(dt2.extra, 7) + self.assertEqual(dt1.toordinal(), dt2.toordinal()) + self.assertEqual(dt2.newmeth(-7), dt1.year + dt1.month - 7) + + def test_subclass_alternate_constructors(self): + # Test that alternate constructors call the constructor + class DateSubclass(self.theclass): + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + + return result + + args = (2003, 4, 14) + d_ord = 731319 # Equivalent ordinal date + d_isoformat = '2003-04-14' # Equivalent isoformat() + + base_d = DateSubclass(*args) + self.assertIsInstance(base_d, DateSubclass) + self.assertEqual(base_d.extra, 7) + + # Timestamp depends on time zone, so we'll calculate the equivalent here + ts = datetime.combine(base_d, time(0)).timestamp() + + test_cases = [ + ('fromordinal', (d_ord,)), + ('fromtimestamp', (ts,)), + ('fromisoformat', (d_isoformat,)), + ] + + for constr_name, constr_args in test_cases: + for base_obj in (DateSubclass, base_d): + # Test both the classmethod and method + with self.subTest(base_obj_type=type(base_obj), + constr_name=constr_name): + constr = getattr(base_obj, constr_name) + + dt = constr(*constr_args) + + # Test that it creates the right subclass + self.assertIsInstance(dt, DateSubclass) + + # Test that it's equal to the base object + self.assertEqual(dt, base_d) + + # Test that it called the constructor + self.assertEqual(dt.extra, 7) + + def test_pickling_subclass_date(self): + + args = 6, 7, 23 + orig = SubclassDate(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertTrue(isinstance(derived, SubclassDate)) + + def test_backdoor_resistance(self): + # For fast unpickling, the constructor accepts a pickle byte string. + # This is a low-overhead backdoor. A user can (by intent or + # mistake) pass a string directly, which (if it's the right length) + # will get treated like a pickle, and bypass the normal sanity + # checks in the constructor. This can create insane objects. + # The constructor doesn't want to burn the time to validate all + # fields, but does check the month field. This stops, e.g., + # datetime.datetime('1995-03-25') from yielding an insane object. + base = b'1995-03-25' + if not issubclass(self.theclass, datetime): + base = base[:4] + for month_byte in b'9', b'\0', b'\r', b'\xff': + self.assertRaises(TypeError, self.theclass, + base[:2] + month_byte + base[3:]) + if issubclass(self.theclass, datetime): + # Good bytes, but bad tzinfo: + with self.assertRaisesRegex(TypeError, '^bad tzinfo state arg$'): + self.theclass(bytes([1] * len(base)), 'EST') + + for ord_byte in range(1, 13): + # This shouldn't blow up because of the month byte alone. If + # the implementation changes to do more-careful checking, it may + # blow up because other fields are insane. + self.theclass(base[:2] + bytes([ord_byte]) + base[3:]) + + def test_valuerror_messages(self): + pattern = re.compile( + r"(year|month|day) must be in \d+\.\.\d+, not \d+" + ) + test_cases = [ + (2009, 13, 1), # Month out of range + (2009, 0, 1), # Month out of range + (10000, 12, 31), # Year out of range + (0, 12, 31), # Year out of range + ] + for case in test_cases: + with self.subTest(case): + with self.assertRaisesRegex(ValueError, pattern): + self.theclass(*case) + + # days out of range have their own error message, see issue 70647 + with self.assertRaises(ValueError) as msg: + self.theclass(2009, 1, 32) + self.assertIn(f"day 32 must be in range 1..31 for month 1 in year 2009", str(msg.exception)) + + def test_fromisoformat(self): + # Test that isoformat() is reversible + base_dates = [ + (1, 1, 1), + (1000, 2, 14), + (1900, 1, 1), + (2000, 2, 29), + (2004, 11, 12), + (2004, 4, 3), + (2017, 5, 30) + ] + + for dt_tuple in base_dates: + dt = self.theclass(*dt_tuple) + dt_str = dt.isoformat() + with self.subTest(dt_str=dt_str): + dt_rt = self.theclass.fromisoformat(dt.isoformat()) + + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_date_examples(self): + examples = [ + ('00010101', self.theclass(1, 1, 1)), + ('20000101', self.theclass(2000, 1, 1)), + ('20250102', self.theclass(2025, 1, 2)), + ('99991231', self.theclass(9999, 12, 31)), + ('0001-01-01', self.theclass(1, 1, 1)), + ('2000-01-01', self.theclass(2000, 1, 1)), + ('2025-01-02', self.theclass(2025, 1, 2)), + ('9999-12-31', self.theclass(9999, 12, 31)), + ('2025W01', self.theclass(2024, 12, 30)), + ('2025-W01', self.theclass(2024, 12, 30)), + ('2025W014', self.theclass(2025, 1, 2)), + ('2025-W01-4', self.theclass(2025, 1, 2)), + ('2026W01', self.theclass(2025, 12, 29)), + ('2026-W01', self.theclass(2025, 12, 29)), + ('2026W013', self.theclass(2025, 12, 31)), + ('2026-W01-3', self.theclass(2025, 12, 31)), + ('2022W52', self.theclass(2022, 12, 26)), + ('2022-W52', self.theclass(2022, 12, 26)), + ('2022W527', self.theclass(2023, 1, 1)), + ('2022-W52-7', self.theclass(2023, 1, 1)), + ('2015W534', self.theclass(2015, 12, 31)), # Has week 53 + ('2015-W53-4', self.theclass(2015, 12, 31)), # Has week 53 + ('2015-W53-5', self.theclass(2016, 1, 1)), + ('2020W531', self.theclass(2020, 12, 28)), # Leap year + ('2020-W53-1', self.theclass(2020, 12, 28)), # Leap year + ('2020-W53-6', self.theclass(2021, 1, 2)), + ] + + for input_str, expected in examples: + with self.subTest(input_str=input_str): + actual = self.theclass.fromisoformat(input_str) + self.assertEqual(actual, expected) + + def test_fromisoformat_subclass(self): + class DateSubclass(self.theclass): + pass + + dt = DateSubclass(2014, 12, 14) + + dt_rt = DateSubclass.fromisoformat(dt.isoformat()) + + self.assertIsInstance(dt_rt, DateSubclass) + + def test_fromisoformat_fails(self): + # Test that fromisoformat() fails on invalid values + bad_strs = [ + '', # Empty string + '\ud800', # bpo-34454: Surrogate code point + '009-03-04', # Not 10 characters + '123456789', # Not a date + '200a-12-04', # Invalid character in year + '2009-1a-04', # Invalid character in month + '2009-12-0a', # Invalid character in day + '2009-01-32', # Invalid day + '2009-02-29', # Invalid leap day + '2019-W53-1', # No week 53 in 2019 + '2020-W54-1', # No week 54 + '0000-W25-1', # Invalid year + '10000-W25-1', # Invalid year + '2020-W25-0', # Invalid day-of-week + '2020-W25-8', # Invalid day-of-week + '٢025-03-09' # Unicode characters + '2009\ud80002\ud80028', # Separators are surrogate codepoints + ] + + for bad_str in bad_strs: + with self.assertRaises(ValueError): + self.theclass.fromisoformat(bad_str) + + def test_fromisoformat_fails_typeerror(self): + # Test that fromisoformat fails when passed the wrong type + bad_types = [b'2009-03-01', None, io.StringIO('2009-03-01')] + for bad_type in bad_types: + with self.assertRaises(TypeError): + self.theclass.fromisoformat(bad_type) + + def test_fromisocalendar(self): + # For each test case, assert that fromisocalendar is the + # inverse of the isocalendar function + dates = [ + (2016, 4, 3), + (2005, 1, 2), # (2004, 53, 7) + (2008, 12, 30), # (2009, 1, 2) + (2010, 1, 2), # (2009, 53, 6) + (2009, 12, 31), # (2009, 53, 4) + (1900, 1, 1), # Unusual non-leap year (year % 100 == 0) + (1900, 12, 31), + (2000, 1, 1), # Unusual leap year (year % 400 == 0) + (2000, 12, 31), + (2004, 1, 1), # Leap year + (2004, 12, 31), + (1, 1, 1), + (9999, 12, 31), + (MINYEAR, 1, 1), + (MAXYEAR, 12, 31), + ] + + for datecomps in dates: + with self.subTest(datecomps=datecomps): + dobj = self.theclass(*datecomps) + isocal = dobj.isocalendar() + + d_roundtrip = self.theclass.fromisocalendar(*isocal) + + self.assertEqual(dobj, d_roundtrip) + + def test_fromisocalendar_value_errors(self): + isocals = [ + (2019, 0, 1), + (2019, -1, 1), + (2019, 54, 1), + (2019, 1, 0), + (2019, 1, -1), + (2019, 1, 8), + (2019, 53, 1), + (10000, 1, 1), + (0, 1, 1), + (9999999, 1, 1), + (2<<32, 1, 1), + (2019, 2<<32, 1), + (2019, 1, 2<<32), + ] + + for isocal in isocals: + with self.subTest(isocal=isocal): + with self.assertRaises(ValueError): + self.theclass.fromisocalendar(*isocal) + + def test_fromisocalendar_type_errors(self): + err_txformers = [ + str, + float, + lambda x: None, + ] + + # Take a valid base tuple and transform it to contain one argument + # with the wrong type. Repeat this for each argument, e.g. + # [("2019", 1, 1), (2019, "1", 1), (2019, 1, "1"), ...] + isocals = [] + base = (2019, 1, 1) + for i in range(3): + for txformer in err_txformers: + err_val = list(base) + err_val[i] = txformer(err_val[i]) + isocals.append(tuple(err_val)) + + for isocal in isocals: + with self.subTest(isocal=isocal): + with self.assertRaises(TypeError): + self.theclass.fromisocalendar(*isocal) + + +############################################################################# +# datetime tests + +class SubclassDatetime(datetime): + sub_var = 1 + +class TestDateTime(TestDate): + + theclass = datetime + + def test_basic_attributes(self): + dt = self.theclass(2002, 3, 1, 12, 0) + self.assertEqual(dt.year, 2002) + self.assertEqual(dt.month, 3) + self.assertEqual(dt.day, 1) + self.assertEqual(dt.hour, 12) + self.assertEqual(dt.minute, 0) + self.assertEqual(dt.second, 0) + self.assertEqual(dt.microsecond, 0) + + def test_basic_attributes_nonzero(self): + # Make sure all attributes are non-zero so bugs in + # bit-shifting access show up. + dt = self.theclass(2002, 3, 1, 12, 59, 59, 8000) + self.assertEqual(dt.year, 2002) + self.assertEqual(dt.month, 3) + self.assertEqual(dt.day, 1) + self.assertEqual(dt.hour, 12) + self.assertEqual(dt.minute, 59) + self.assertEqual(dt.second, 59) + self.assertEqual(dt.microsecond, 8000) + + def test_roundtrip(self): + for dt in (self.theclass(1, 2, 3, 4, 5, 6, 7), + self.theclass.now()): + # Verify dt -> string -> datetime identity. + s = repr(dt) + self.assertStartsWith(s, 'datetime.') + s = s[9:] + dt2 = eval(s) + self.assertEqual(dt, dt2) + + # Verify identity via reconstructing from pieces. + dt2 = self.theclass(dt.year, dt.month, dt.day, + dt.hour, dt.minute, dt.second, + dt.microsecond) + self.assertEqual(dt, dt2) + + def test_isoformat(self): + t = self.theclass(1, 2, 3, 4, 5, 1, 123) + self.assertEqual(t.isoformat(), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat('T'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(' '), "0001-02-03 04:05:01.000123") + self.assertEqual(t.isoformat('\x00'), "0001-02-03\x0004:05:01.000123") + # bpo-34482: Check that surrogates are handled properly. + self.assertEqual(t.isoformat('\ud800'), + "0001-02-03\ud80004:05:01.000123") + self.assertEqual(t.isoformat(timespec='hours'), "0001-02-03T04") + self.assertEqual(t.isoformat(timespec='minutes'), "0001-02-03T04:05") + self.assertEqual(t.isoformat(timespec='seconds'), "0001-02-03T04:05:01") + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(timespec='auto'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(sep=' ', timespec='minutes'), "0001-02-03 04:05") + self.assertRaises(ValueError, t.isoformat, timespec='foo') + # bpo-34482: Check that surrogates are handled properly. + self.assertRaises(ValueError, t.isoformat, timespec='\ud800') + # str is ISO format with the separator forced to a blank. + self.assertEqual(str(t), "0001-02-03 04:05:01.000123") + + t = self.theclass(1, 2, 3, 4, 5, 1, 999500, tzinfo=timezone.utc) + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.999+00:00") + + t = self.theclass(1, 2, 3, 4, 5, 1, 999500) + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.999") + + t = self.theclass(1, 2, 3, 4, 5, 1) + self.assertEqual(t.isoformat(timespec='auto'), "0001-02-03T04:05:01") + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "0001-02-03T04:05:01.000000") + + t = self.theclass(2, 3, 2) + self.assertEqual(t.isoformat(), "0002-03-02T00:00:00") + self.assertEqual(t.isoformat('T'), "0002-03-02T00:00:00") + self.assertEqual(t.isoformat(' '), "0002-03-02 00:00:00") + # str is ISO format with the separator forced to a blank. + self.assertEqual(str(t), "0002-03-02 00:00:00") + # ISO format with timezone + tz = FixedOffset(timedelta(seconds=16), 'XXX') + t = self.theclass(2, 3, 2, tzinfo=tz) + self.assertEqual(t.isoformat(), "0002-03-02T00:00:00+00:00:16") + + def test_isoformat_timezone(self): + tzoffsets = [ + ('05:00', timedelta(hours=5)), + ('02:00', timedelta(hours=2)), + ('06:27', timedelta(hours=6, minutes=27)), + ('12:32:30', timedelta(hours=12, minutes=32, seconds=30)), + ('02:04:09.123456', timedelta(hours=2, minutes=4, seconds=9, microseconds=123456)) + ] + + tzinfos = [ + ('', None), + ('+00:00', timezone.utc), + ('+00:00', timezone(timedelta(0))), + ] + + tzinfos += [ + (prefix + expected, timezone(sign * td)) + for expected, td in tzoffsets + for prefix, sign in [('-', -1), ('+', 1)] + ] + + dt_base = self.theclass(2016, 4, 1, 12, 37, 9) + exp_base = '2016-04-01T12:37:09' + + for exp_tz, tzi in tzinfos: + dt = dt_base.replace(tzinfo=tzi) + exp = exp_base + exp_tz + with self.subTest(tzi=tzi): + assert dt.isoformat() == exp + + def test_format(self): + dt = self.theclass(2007, 9, 10, 4, 5, 1, 123) + self.assertEqual(dt.__format__(''), str(dt)) + + with self.assertRaisesRegex(TypeError, 'must be str, not int'): + dt.__format__(123) + + # check that a derived class's __str__() gets called + class A(self.theclass): + def __str__(self): + return 'A' + a = A(2007, 9, 10, 4, 5, 1, 123) + self.assertEqual(a.__format__(''), 'A') + + # check that a derived class's strftime gets called + class B(self.theclass): + def strftime(self, format_spec): + return 'B' + b = B(2007, 9, 10, 4, 5, 1, 123) + self.assertEqual(b.__format__(''), str(dt)) + + for fmt in ["m:%m d:%d y:%y", + "m:%m d:%d y:%y H:%H M:%M S:%S", + "%z %:z %Z", + ]: + self.assertEqual(dt.__format__(fmt), dt.strftime(fmt)) + self.assertEqual(a.__format__(fmt), dt.strftime(fmt)) + self.assertEqual(b.__format__(fmt), 'B') + + def test_more_ctime(self): + # Test fields that TestDate doesn't touch. + import time + + t = self.theclass(2002, 3, 2, 18, 3, 5, 123) + self.assertEqual(t.ctime(), "Sat Mar 2 18:03:05 2002") + # Oops! The next line fails on Win2K under MSVC 6, so it's commented + # out. The difference is that t.ctime() produces " 2" for the day, + # but platform ctime() produces "02" for the day. According to + # C99, t.ctime() is correct here. + # self.assertEqual(t.ctime(), time.ctime(time.mktime(t.timetuple()))) + + # So test a case where that difference doesn't matter. + t = self.theclass(2002, 3, 22, 18, 3, 5, 123) + self.assertEqual(t.ctime(), time.ctime(time.mktime(t.timetuple()))) + + def test_tz_independent_comparing(self): + dt1 = self.theclass(2002, 3, 1, 9, 0, 0) + dt2 = self.theclass(2002, 3, 1, 10, 0, 0) + dt3 = self.theclass(2002, 3, 1, 9, 0, 0) + self.assertEqual(dt1, dt3) + self.assertTrue(dt2 > dt3) + + # Make sure comparison doesn't forget microseconds, and isn't done + # via comparing a float timestamp (an IEEE double doesn't have enough + # precision to span microsecond resolution across years 1 through 9999, + # so comparing via timestamp necessarily calls some distinct values + # equal). + dt1 = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999998) + us = timedelta(microseconds=1) + dt2 = dt1 + us + self.assertEqual(dt2 - dt1, us) + self.assertTrue(dt1 < dt2) + + def test_strftime_with_bad_tzname_replace(self): + # verify ok if tzinfo.tzname().replace() returns a non-string + class MyTzInfo(FixedOffset): + def tzname(self, dt): + class MyStr(str): + def replace(self, *args): + return None + return MyStr('name') + t = self.theclass(2005, 3, 2, 0, 0, 0, 0, MyTzInfo(3, 'name')) + self.assertRaises(TypeError, t.strftime, '%Z') + + def test_bad_constructor_arguments(self): + # bad years + self.theclass(MINYEAR, 1, 1) # no exception + self.theclass(MAXYEAR, 1, 1) # no exception + self.assertRaises(ValueError, self.theclass, MINYEAR-1, 1, 1) + self.assertRaises(ValueError, self.theclass, MAXYEAR+1, 1, 1) + # bad months + self.theclass(2000, 1, 1) # no exception + self.theclass(2000, 12, 1) # no exception + self.assertRaises(ValueError, self.theclass, 2000, 0, 1) + self.assertRaises(ValueError, self.theclass, 2000, 13, 1) + # bad days + self.theclass(2000, 2, 29) # no exception + self.theclass(2004, 2, 29) # no exception + self.theclass(2400, 2, 29) # no exception + self.assertRaises(ValueError, self.theclass, 2000, 2, 30) + self.assertRaises(ValueError, self.theclass, 2001, 2, 29) + self.assertRaises(ValueError, self.theclass, 2100, 2, 29) + self.assertRaises(ValueError, self.theclass, 1900, 2, 29) + self.assertRaises(ValueError, self.theclass, 2000, 1, 0) + self.assertRaises(ValueError, self.theclass, 2000, 1, 32) + # bad hours + self.theclass(2000, 1, 31, 0) # no exception + self.theclass(2000, 1, 31, 23) # no exception + self.assertRaises(ValueError, self.theclass, 2000, 1, 31, -1) + self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 24) + # bad minutes + self.theclass(2000, 1, 31, 23, 0) # no exception + self.theclass(2000, 1, 31, 23, 59) # no exception + self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, -1) + self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 60) + # bad seconds + self.theclass(2000, 1, 31, 23, 59, 0) # no exception + self.theclass(2000, 1, 31, 23, 59, 59) # no exception + self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 59, -1) + self.assertRaises(ValueError, self.theclass, 2000, 1, 31, 23, 59, 60) + # bad microseconds + self.theclass(2000, 1, 31, 23, 59, 59, 0) # no exception + self.theclass(2000, 1, 31, 23, 59, 59, 999999) # no exception + self.assertRaises(ValueError, self.theclass, + 2000, 1, 31, 23, 59, 59, -1) + self.assertRaises(ValueError, self.theclass, + 2000, 1, 31, 23, 59, 59, + 1000000) + # bad fold + self.assertRaises(ValueError, self.theclass, + 2000, 1, 31, fold=-1) + self.assertRaises(ValueError, self.theclass, + 2000, 1, 31, fold=2) + # Positional fold: + self.assertRaises(TypeError, self.theclass, + 2000, 1, 31, 23, 59, 59, 0, None, 1) + + def test_hash_equality(self): + d = self.theclass(2000, 12, 31, 23, 30, 17) + e = self.theclass(2000, 12, 31, 23, 30, 17) + self.assertEqual(d, e) + self.assertEqual(hash(d), hash(e)) + + dic = {d: 1} + dic[e] = 2 + self.assertEqual(len(dic), 1) + self.assertEqual(dic[d], 2) + self.assertEqual(dic[e], 2) + + d = self.theclass(2001, 1, 1, 0, 5, 17) + e = self.theclass(2001, 1, 1, 0, 5, 17) + self.assertEqual(d, e) + self.assertEqual(hash(d), hash(e)) + + dic = {d: 1} + dic[e] = 2 + self.assertEqual(len(dic), 1) + self.assertEqual(dic[d], 2) + self.assertEqual(dic[e], 2) + + def test_computations(self): + a = self.theclass(2002, 1, 31) + b = self.theclass(1956, 1, 31) + diff = a-b + self.assertEqual(diff.days, 46*365 + len(range(1956, 2002, 4))) + self.assertEqual(diff.seconds, 0) + self.assertEqual(diff.microseconds, 0) + a = self.theclass(2002, 3, 2, 17, 6) + millisec = timedelta(0, 0, 1000) + hour = timedelta(0, 3600) + day = timedelta(1) + week = timedelta(7) + self.assertEqual(a + hour, self.theclass(2002, 3, 2, 18, 6)) + self.assertEqual(hour + a, self.theclass(2002, 3, 2, 18, 6)) + self.assertEqual(a + 10*hour, self.theclass(2002, 3, 3, 3, 6)) + self.assertEqual(a - hour, self.theclass(2002, 3, 2, 16, 6)) + self.assertEqual(-hour + a, self.theclass(2002, 3, 2, 16, 6)) + self.assertEqual(a - hour, a + -hour) + self.assertEqual(a - 20*hour, self.theclass(2002, 3, 1, 21, 6)) + self.assertEqual(a + day, self.theclass(2002, 3, 3, 17, 6)) + self.assertEqual(a - day, self.theclass(2002, 3, 1, 17, 6)) + self.assertEqual(a + week, self.theclass(2002, 3, 9, 17, 6)) + self.assertEqual(a - week, self.theclass(2002, 2, 23, 17, 6)) + self.assertEqual(a + 52*week, self.theclass(2003, 3, 1, 17, 6)) + self.assertEqual(a - 52*week, self.theclass(2001, 3, 3, 17, 6)) + self.assertEqual((a + week) - a, week) + self.assertEqual((a + day) - a, day) + self.assertEqual((a + hour) - a, hour) + self.assertEqual((a + millisec) - a, millisec) + self.assertEqual((a - week) - a, -week) + self.assertEqual((a - day) - a, -day) + self.assertEqual((a - hour) - a, -hour) + self.assertEqual((a - millisec) - a, -millisec) + self.assertEqual(a - (a + week), -week) + self.assertEqual(a - (a + day), -day) + self.assertEqual(a - (a + hour), -hour) + self.assertEqual(a - (a + millisec), -millisec) + self.assertEqual(a - (a - week), week) + self.assertEqual(a - (a - day), day) + self.assertEqual(a - (a - hour), hour) + self.assertEqual(a - (a - millisec), millisec) + self.assertEqual(a + (week + day + hour + millisec), + self.theclass(2002, 3, 10, 18, 6, 0, 1000)) + self.assertEqual(a + (week + day + hour + millisec), + (((a + week) + day) + hour) + millisec) + self.assertEqual(a - (week + day + hour + millisec), + self.theclass(2002, 2, 22, 16, 5, 59, 999000)) + self.assertEqual(a - (week + day + hour + millisec), + (((a - week) - day) - hour) - millisec) + # Add/sub ints or floats should be illegal + for i in 1, 1.0: + self.assertRaises(TypeError, lambda: a+i) + self.assertRaises(TypeError, lambda: a-i) + self.assertRaises(TypeError, lambda: i+a) + self.assertRaises(TypeError, lambda: i-a) + + # delta - datetime is senseless. + self.assertRaises(TypeError, lambda: day - a) + # mixing datetime and (delta or datetime) via * or // is senseless + self.assertRaises(TypeError, lambda: day * a) + self.assertRaises(TypeError, lambda: a * day) + self.assertRaises(TypeError, lambda: day // a) + self.assertRaises(TypeError, lambda: a // day) + self.assertRaises(TypeError, lambda: a * a) + self.assertRaises(TypeError, lambda: a // a) + # datetime + datetime is senseless + self.assertRaises(TypeError, lambda: a + a) + + def test_pickling(self): + args = 6, 7, 23, 20, 59, 1, 64**2 + orig = self.theclass(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) + + def test_more_pickling(self): + a = self.theclass(2003, 2, 7, 16, 48, 37, 444116) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(a, proto) + b = pickle.loads(s) + self.assertEqual(b.year, 2003) + self.assertEqual(b.month, 2) + self.assertEqual(b.day, 7) + + def test_pickling_subclass_datetime(self): + args = 6, 7, 23, 20, 59, 1, 64**2 + orig = SubclassDatetime(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertTrue(isinstance(derived, SubclassDatetime)) + + def test_compat_unpickle(self): + tests = [ + b'cdatetime\ndatetime\n(' + b"S'\\x07\\xdf\\x0b\\x1b\\x14;\\x01\\x00\\x10\\x00'\ntR.", + + b'cdatetime\ndatetime\n(' + b'U\n\x07\xdf\x0b\x1b\x14;\x01\x00\x10\x00tR.', + + b'\x80\x02cdatetime\ndatetime\n' + b'U\n\x07\xdf\x0b\x1b\x14;\x01\x00\x10\x00\x85R.', + ] + args = 2015, 11, 27, 20, 59, 1, 64**2 + expected = self.theclass(*args) + for data in tests: + for loads in pickle_loads: + derived = loads(data, encoding='latin1') + self.assertEqual(derived, expected) + + def test_more_compare(self): + # The test_compare() inherited from TestDate covers the error cases. + # We just want to test lexicographic ordering on the members datetime + # has that date lacks. + args = [2000, 11, 29, 20, 58, 16, 999998] + t1 = self.theclass(*args) + t2 = self.theclass(*args) + self.assertEqual(t1, t2) + self.assertTrue(t1 <= t2) + self.assertTrue(t1 >= t2) + self.assertFalse(t1 != t2) + self.assertFalse(t1 < t2) + self.assertFalse(t1 > t2) + + for i in range(len(args)): + newargs = args[:] + newargs[i] = args[i] + 1 + t2 = self.theclass(*newargs) # this is larger than t1 + self.assertTrue(t1 < t2) + self.assertTrue(t2 > t1) + self.assertTrue(t1 <= t2) + self.assertTrue(t2 >= t1) + self.assertTrue(t1 != t2) + self.assertTrue(t2 != t1) + self.assertFalse(t1 == t2) + self.assertFalse(t2 == t1) + self.assertFalse(t1 > t2) + self.assertFalse(t2 < t1) + self.assertFalse(t1 >= t2) + self.assertFalse(t2 <= t1) + + + # A helper for timestamp constructor tests. + def verify_field_equality(self, expected, got): + self.assertEqual(expected.tm_year, got.year) + self.assertEqual(expected.tm_mon, got.month) + self.assertEqual(expected.tm_mday, got.day) + self.assertEqual(expected.tm_hour, got.hour) + self.assertEqual(expected.tm_min, got.minute) + self.assertEqual(expected.tm_sec, got.second) + + def test_fromtimestamp(self): + import time + + ts = time.time() + expected = time.localtime(ts) + got = self.theclass.fromtimestamp(ts) + self.verify_field_equality(expected, got) + + def test_fromtimestamp_keyword_arg(self): + import time + + # gh-85432: The parameter was named "t" in the pure-Python impl. + self.theclass.fromtimestamp(timestamp=time.time()) + + def test_utcfromtimestamp(self): + import time + + ts = time.time() + expected = time.gmtime(ts) + with self.assertWarns(DeprecationWarning): + got = self.theclass.utcfromtimestamp(ts) + self.verify_field_equality(expected, got) + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_timestamp_naive(self): + t = self.theclass(1970, 1, 1) + self.assertEqual(t.timestamp(), 18000.0) + t = self.theclass(1970, 1, 1, 1, 2, 3, 4) + self.assertEqual(t.timestamp(), + 18000.0 + 3600 + 2*60 + 3 + 4*1e-6) + # Missing hour + t0 = self.theclass(2012, 3, 11, 2, 30) + t1 = t0.replace(fold=1) + self.assertEqual(self.theclass.fromtimestamp(t1.timestamp()), + t0 - timedelta(hours=1)) + self.assertEqual(self.theclass.fromtimestamp(t0.timestamp()), + t1 + timedelta(hours=1)) + # Ambiguous hour defaults to DST + t = self.theclass(2012, 11, 4, 1, 30) + self.assertEqual(self.theclass.fromtimestamp(t.timestamp()), t) + + # Timestamp may raise an overflow error on some platforms + # XXX: Do we care to support the first and last year? + for t in [self.theclass(2,1,1), self.theclass(9998,12,12)]: + try: + s = t.timestamp() + except OverflowError: + pass + else: + self.assertEqual(self.theclass.fromtimestamp(s), t) + + def test_timestamp_aware(self): + t = self.theclass(1970, 1, 1, tzinfo=timezone.utc) + self.assertEqual(t.timestamp(), 0.0) + t = self.theclass(1970, 1, 1, 1, 2, 3, 4, tzinfo=timezone.utc) + self.assertEqual(t.timestamp(), + 3600 + 2*60 + 3 + 4*1e-6) + t = self.theclass(1970, 1, 1, 1, 2, 3, 4, + tzinfo=timezone(timedelta(hours=-5), 'EST')) + self.assertEqual(t.timestamp(), + 18000 + 3600 + 2*60 + 3 + 4*1e-6) + + @support.run_with_tz('MSK-03') # Something east of Greenwich + def test_microsecond_rounding(self): + def utcfromtimestamp(*args, **kwargs): + with self.assertWarns(DeprecationWarning): + return self.theclass.utcfromtimestamp(*args, **kwargs) + + for fts in [self.theclass.fromtimestamp, + utcfromtimestamp]: + zero = fts(0) + self.assertEqual(zero.second, 0) + self.assertEqual(zero.microsecond, 0) + one = fts(1e-6) + try: + minus_one = fts(-1e-6) + except OSError: + # localtime(-1) and gmtime(-1) is not supported on Windows + pass + else: + self.assertEqual(minus_one.second, 59) + self.assertEqual(minus_one.microsecond, 999999) + + t = fts(-1e-8) + self.assertEqual(t, zero) + t = fts(-9e-7) + self.assertEqual(t, minus_one) + t = fts(-1e-7) + self.assertEqual(t, zero) + t = fts(-1/2**7) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992188) + + t = fts(1e-7) + self.assertEqual(t, zero) + t = fts(9e-7) + self.assertEqual(t, one) + t = fts(0.99999949) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 999999) + t = fts(0.9999999) + self.assertEqual(t.second, 1) + self.assertEqual(t.microsecond, 0) + t = fts(1/2**7) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 7812) + + def test_timestamp_limits(self): + with self.subTest("minimum UTC"): + min_dt = self.theclass.min.replace(tzinfo=timezone.utc) + min_ts = min_dt.timestamp() + + # This test assumes that datetime.min == 0000-01-01T00:00:00.00 + # If that assumption changes, this value can change as well + self.assertEqual(min_ts, -62135596800) + + with self.subTest("maximum UTC"): + # Zero out microseconds to avoid rounding issues + max_dt = self.theclass.max.replace(tzinfo=timezone.utc, + microsecond=0) + max_ts = max_dt.timestamp() + + # This test assumes that datetime.max == 9999-12-31T23:59:59.999999 + # If that assumption changes, this value can change as well + self.assertEqual(max_ts, 253402300799.0) + + def test_fromtimestamp_limits(self): + try: + self.theclass.fromtimestamp(-2**32 - 1) + except (OSError, OverflowError): + self.skipTest("Test not valid on this platform") + + # XXX: Replace these with datetime.{min,max}.timestamp() when we solve + # the issue with gh-91012 + min_dt = self.theclass.min + timedelta(days=1) + min_ts = min_dt.timestamp() + + max_dt = self.theclass.max.replace(microsecond=0) + max_ts = ((self.theclass.max - timedelta(hours=23)).timestamp() + + timedelta(hours=22, minutes=59, seconds=59).total_seconds()) + + for (test_name, ts, expected) in [ + ("minimum", min_ts, min_dt), + ("maximum", max_ts, max_dt), + ]: + with self.subTest(test_name, ts=ts, expected=expected): + actual = self.theclass.fromtimestamp(ts) + + self.assertEqual(actual, expected) + + # Test error conditions + test_cases = [ + ("Too small by a little", min_ts - timedelta(days=1, hours=12).total_seconds()), + ("Too small by a lot", min_ts - timedelta(days=400).total_seconds()), + ("Too big by a little", max_ts + timedelta(days=1).total_seconds()), + ("Too big by a lot", max_ts + timedelta(days=400).total_seconds()), + ] + + for test_name, ts in test_cases: + with self.subTest(test_name, ts=ts): + with self.assertRaises((ValueError, OverflowError)): + # converting a Python int to C time_t can raise a + # OverflowError, especially on 32-bit platforms. + self.theclass.fromtimestamp(ts) + + def test_utcfromtimestamp_limits(self): + with self.assertWarns(DeprecationWarning): + try: + self.theclass.utcfromtimestamp(-2**32 - 1) + except (OSError, OverflowError): + self.skipTest("Test not valid on this platform") + + min_dt = self.theclass.min.replace(tzinfo=timezone.utc) + min_ts = min_dt.timestamp() + + max_dt = self.theclass.max.replace(microsecond=0, tzinfo=timezone.utc) + max_ts = max_dt.timestamp() + + for (test_name, ts, expected) in [ + ("minimum", min_ts, min_dt.replace(tzinfo=None)), + ("maximum", max_ts, max_dt.replace(tzinfo=None)), + ]: + with self.subTest(test_name, ts=ts, expected=expected): + with self.assertWarns(DeprecationWarning): + try: + actual = self.theclass.utcfromtimestamp(ts) + except (OSError, OverflowError) as exc: + self.skipTest(str(exc)) + + self.assertEqual(actual, expected) + + # Test error conditions + test_cases = [ + ("Too small by a little", min_ts - 1), + ("Too small by a lot", min_ts - timedelta(days=400).total_seconds()), + ("Too big by a little", max_ts + 1), + ("Too big by a lot", max_ts + timedelta(days=400).total_seconds()), + ] + + for test_name, ts in test_cases: + with self.subTest(test_name, ts=ts): + with self.assertRaises((ValueError, OverflowError)): + with self.assertWarns(DeprecationWarning): + # converting a Python int to C time_t can raise a + # OverflowError, especially on 32-bit platforms. + self.theclass.utcfromtimestamp(ts) + + def test_insane_fromtimestamp(self): + # It's possible that some platform maps time_t to double, + # and that this test will fail there. This test should + # exempt such platforms (provided they return reasonable + # results!). + for insane in -1e200, 1e200: + self.assertRaises(OverflowError, self.theclass.fromtimestamp, + insane) + + def test_insane_utcfromtimestamp(self): + # It's possible that some platform maps time_t to double, + # and that this test will fail there. This test should + # exempt such platforms (provided they return reasonable + # results!). + for insane in -1e200, 1e200: + with self.assertWarns(DeprecationWarning): + self.assertRaises(OverflowError, self.theclass.utcfromtimestamp, + insane) + + @unittest.skipIf(sys.platform == "win32", "Windows doesn't accept negative timestamps") + def test_negative_float_fromtimestamp(self): + # The result is tz-dependent; at least test that this doesn't + # fail (like it did before bug 1646728 was fixed). + self.theclass.fromtimestamp(-1.05) + + @unittest.skipIf(sys.platform == "win32", "Windows doesn't accept negative timestamps") + def test_negative_float_utcfromtimestamp(self): + with self.assertWarns(DeprecationWarning): + d = self.theclass.utcfromtimestamp(-1.05) + self.assertEqual(d, self.theclass(1969, 12, 31, 23, 59, 58, 950000)) + + def test_utcnow(self): + import time + + # Call it a success if utcnow() and utcfromtimestamp() are within + # a second of each other. + tolerance = timedelta(seconds=1) + for dummy in range(3): + with self.assertWarns(DeprecationWarning): + from_now = self.theclass.utcnow() + + with self.assertWarns(DeprecationWarning): + from_timestamp = self.theclass.utcfromtimestamp(time.time()) + if abs(from_timestamp - from_now) <= tolerance: + break + # Else try again a few times. + self.assertLessEqual(abs(from_timestamp - from_now), tolerance) + + def test_strptime(self): + string = '2004-12-01 13:02:47.197' + format = '%Y-%m-%d %H:%M:%S.%f' + expected = _strptime._strptime_datetime_datetime(self.theclass, string, + format) + got = self.theclass.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(expected), self.theclass) + self.assertIs(type(got), self.theclass) + + # bpo-34482: Check that surrogates are handled properly. + inputs = [ + ('2004-12-01\ud80013:02:47.197', '%Y-%m-%d\ud800%H:%M:%S.%f'), + ('2004\ud80012-01 13:02:47.197', '%Y\ud800%m-%d %H:%M:%S.%f'), + ('2004-12-01 13:02\ud80047.197', '%Y-%m-%d %H:%M\ud800%S.%f'), + ] + for string, format in inputs: + with self.subTest(string=string, format=format): + expected = _strptime._strptime_datetime_datetime(self.theclass, + string, format) + got = self.theclass.strptime(string, format) + self.assertEqual(expected, got) + + strptime = self.theclass.strptime + + self.assertEqual(strptime("+0002", "%z").utcoffset(), 2 * MINUTE) + self.assertEqual(strptime("-0002", "%z").utcoffset(), -2 * MINUTE) + self.assertEqual( + strptime("-00:02:01.000003", "%z").utcoffset(), + -timedelta(minutes=2, seconds=1, microseconds=3) + ) + # Only local timezone and UTC are supported + for tzseconds, tzname in ((0, 'UTC'), (0, 'GMT'), + (-_time.timezone, _time.tzname[0])): + if tzseconds < 0: + sign = '-' + seconds = -tzseconds + else: + sign ='+' + seconds = tzseconds + hours, minutes = divmod(seconds//60, 60) + dtstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) + dt = strptime(dtstr, "%z %Z") + self.assertEqual(dt.utcoffset(), timedelta(seconds=tzseconds)) + self.assertEqual(dt.tzname(), tzname) + # Can produce inconsistent datetime + dtstr, fmt = "+1234 UTC", "%z %Z" + dt = strptime(dtstr, fmt) + self.assertEqual(dt.utcoffset(), 12 * HOUR + 34 * MINUTE) + self.assertEqual(dt.tzname(), 'UTC') + # yet will roundtrip + self.assertEqual(dt.strftime(fmt), dtstr) + + # Produce naive datetime if no %z is provided + self.assertEqual(strptime("UTC", "%Z").tzinfo, None) + + with self.assertRaises(ValueError): strptime("-2400", "%z") + with self.assertRaises(ValueError): strptime("-000", "%z") + with self.assertRaises(ValueError): strptime("z", "%z") + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit dates and times are allowed. + + strptime = self.theclass.strptime + + with self.assertRaises(ValueError): + # %y does require two digits. + newdate = strptime('01/02/3 04:05:06', '%d/%m/%y %H:%M:%S') + dt1 = self.theclass(2003, 2, 1, 4, 5, 6) + dt2 = self.theclass(2003, 1, 2, 4, 5, 6) + dt3 = self.theclass(2003, 2, 1, 0, 0, 0) + dt4 = self.theclass(2003, 1, 25, 0, 0, 0) + inputs = [ + ('%d', '1/02/03 4:5:6', '%d/%m/%y %H:%M:%S', dt1), + ('%m', '01/2/03 4:5:6', '%d/%m/%y %H:%M:%S', dt1), + ('%H', '01/02/03 4:05:06', '%d/%m/%y %H:%M:%S', dt1), + ('%M', '01/02/03 04:5:06', '%d/%m/%y %H:%M:%S', dt1), + ('%S', '01/02/03 04:05:6', '%d/%m/%y %H:%M:%S', dt1), + ('%j', '2/03 04am:05:06', '%j/%y %I%p:%M:%S',dt2), + ('%I', '02/03 4am:05:06', '%j/%y %I%p:%M:%S',dt2), + ('%w', '6/04/03', '%w/%U/%y', dt3), + # %u requires a single digit. + ('%W', '6/4/2003', '%u/%W/%Y', dt3), + ('%V', '6/4/2003', '%u/%V/%G', dt4), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertRaises(ValueError): + # The existing behavior that GH-70647 seeks to change. + self.theclass.strptime('02-29', '%m-%d') + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + self.theclass.strptime('03-14.159265', '%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('20-03-14.159265', '%y-%m-%d.%f') + with self._assertNotWarns(DeprecationWarning): + self.theclass.strptime('02-29,2024', '%m-%d,%Y') + + def test_more_timetuple(self): + # This tests fields beyond those tested by the TestDate.test_timetuple. + t = self.theclass(2004, 12, 31, 6, 22, 33) + self.assertEqual(t.timetuple(), (2004, 12, 31, 6, 22, 33, 4, 366, -1)) + self.assertEqual(t.timetuple(), + (t.year, t.month, t.day, + t.hour, t.minute, t.second, + t.weekday(), + t.toordinal() - date(t.year, 1, 1).toordinal() + 1, + -1)) + tt = t.timetuple() + self.assertEqual(tt.tm_year, t.year) + self.assertEqual(tt.tm_mon, t.month) + self.assertEqual(tt.tm_mday, t.day) + self.assertEqual(tt.tm_hour, t.hour) + self.assertEqual(tt.tm_min, t.minute) + self.assertEqual(tt.tm_sec, t.second) + self.assertEqual(tt.tm_wday, t.weekday()) + self.assertEqual(tt.tm_yday, t.toordinal() - + date(t.year, 1, 1).toordinal() + 1) + self.assertEqual(tt.tm_isdst, -1) + + def test_more_strftime(self): + # This tests fields beyond those tested by the TestDate.test_strftime. + t = self.theclass(2004, 12, 31, 6, 22, 33, 47) + self.assertEqual(t.strftime("%m %d %y %f %S %M %H %j"), + "12 31 04 000047 33 22 06 366") + for (s, us), z in [((33, 123), "33.000123"), ((33, 0), "33"),]: + tz = timezone(-timedelta(hours=2, seconds=s, microseconds=us)) + t = t.replace(tzinfo=tz) + self.assertEqual(t.strftime("%z"), "-0200" + z) + self.assertEqual(t.strftime("%:z"), "-02:00:" + z) + + @unittest.skip("TODO: RUSTPYTHON") + def test_strftime_special(self): + t = self.theclass(2004, 12, 31, 6, 22, 33, 47) + s1 = t.strftime('%c') + s2 = t.strftime('%B') + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d') + self.assertEqual(t.strftime('\U0001f4bb%c\U0001f40d%B'), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(t.strftime('%c\U0001f4bb%B\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(t.strftime('\ud83d'), '\ud83d') + self.assertEqual(t.strftime('\udc0d'), '\udc0d') + self.assertEqual(t.strftime('\ud83d%c\udc0d%B'), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(t.strftime('%c\ud83d%B\udc0d'), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(t.strftime('%c\udc0d%B\ud83d'), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d') + self.assertEqual(t.strftime('%c\ud83d\udc0d%B'), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(t.strftime('%c\udcf0\udc9f\udc90\udc8d%B'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(t.strftime('\0'), '\0') + self.assertEqual(t.strftime('\0'*1000), '\0'*1000) + self.assertEqual(t.strftime('\0%c\0%B'), f'\0{s1}\0{s2}') + self.assertEqual(t.strftime('%c\0%B\0'), f'{s1}\0{s2}\0') + + def test_extract(self): + dt = self.theclass(2002, 3, 4, 18, 45, 3, 1234) + self.assertEqual(dt.date(), date(2002, 3, 4)) + self.assertEqual(dt.time(), time(18, 45, 3, 1234)) + + def test_combine(self): + d = date(2002, 3, 4) + t = time(18, 45, 3, 1234) + expected = self.theclass(2002, 3, 4, 18, 45, 3, 1234) + combine = self.theclass.combine + dt = combine(d, t) + self.assertEqual(dt, expected) + + dt = combine(time=t, date=d) + self.assertEqual(dt, expected) + + self.assertEqual(d, dt.date()) + self.assertEqual(t, dt.time()) + self.assertEqual(dt, combine(dt.date(), dt.time())) + + self.assertRaises(TypeError, combine) # need an arg + self.assertRaises(TypeError, combine, d) # need two args + self.assertRaises(TypeError, combine, t, d) # args reversed + self.assertRaises(TypeError, combine, d, t, 1) # wrong tzinfo type + self.assertRaises(TypeError, combine, d, t, 1, 2) # too many args + self.assertRaises(TypeError, combine, "date", "time") # wrong types + self.assertRaises(TypeError, combine, d, "time") # wrong type + self.assertRaises(TypeError, combine, "date", t) # wrong type + + # tzinfo= argument + dt = combine(d, t, timezone.utc) + self.assertIs(dt.tzinfo, timezone.utc) + dt = combine(d, t, tzinfo=timezone.utc) + self.assertIs(dt.tzinfo, timezone.utc) + t = time() + dt = combine(dt, t) + self.assertEqual(dt.date(), d) + self.assertEqual(dt.time(), t) + + def test_replace(self): + cls = self.theclass + args = [1, 2, 3, 4, 5, 6, 7] + base = cls(*args) + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("year", 2), + ("month", 3), + ("day", 4), + ("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8)) + for i, (name, newval) in enumerate(changes): + newargs = args[:] + newargs[i] = newval + expected = cls(*newargs) + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) + + # Out of bounds. + base = cls(2000, 2, 29) + self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) + + @support.run_with_tz('EDT4') + def test_astimezone(self): + dt = self.theclass.now() + f = FixedOffset(44, "0044") + dt_utc = dt.replace(tzinfo=timezone(timedelta(hours=-4), 'EDT')) + self.assertEqual(dt.astimezone(), dt_utc) # naive + self.assertRaises(TypeError, dt.astimezone, f, f) # too many args + self.assertRaises(TypeError, dt.astimezone, dt) # arg wrong type + dt_f = dt.replace(tzinfo=f) + timedelta(hours=4, minutes=44) + self.assertEqual(dt.astimezone(f), dt_f) # naive + self.assertEqual(dt.astimezone(tz=f), dt_f) # naive + + class Bogus(tzinfo): + def utcoffset(self, dt): return None + def dst(self, dt): return timedelta(0) + bog = Bogus() + self.assertRaises(ValueError, dt.astimezone, bog) # naive + self.assertEqual(dt.replace(tzinfo=bog).astimezone(f), dt_f) + + class AlsoBogus(tzinfo): + def utcoffset(self, dt): return timedelta(0) + def dst(self, dt): return None + alsobog = AlsoBogus() + self.assertRaises(ValueError, dt.astimezone, alsobog) # also naive + + class Broken(tzinfo): + def utcoffset(self, dt): return 1 + def dst(self, dt): return 1 + broken = Broken() + dt_broken = dt.replace(tzinfo=broken) + with self.assertRaises(TypeError): + dt_broken.astimezone() + + def test_subclass_datetime(self): + + class C(self.theclass): + theAnswer = 42 + + def __new__(cls, *args, **kws): + temp = kws.copy() + extra = temp.pop('extra') + result = self.theclass.__new__(cls, *args, **temp) + result.extra = extra + return result + + def newmeth(self, start): + return start + self.year + self.month + self.second + + args = 2003, 4, 14, 12, 13, 41 + + dt1 = self.theclass(*args) + dt2 = C(*args, **{'extra': 7}) + + self.assertEqual(dt2.__class__, C) + self.assertEqual(dt2.theAnswer, 42) + self.assertEqual(dt2.extra, 7) + self.assertEqual(dt1.toordinal(), dt2.toordinal()) + self.assertEqual(dt2.newmeth(-7), dt1.year + dt1.month + + dt1.second - 7) + + def test_subclass_alternate_constructors_datetime(self): + # Test that alternate constructors call the constructor + class DateTimeSubclass(self.theclass): + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + + return result + + args = (2003, 4, 14, 12, 30, 15, 123456) + d_isoformat = '2003-04-14T12:30:15.123456' # Equivalent isoformat() + utc_ts = 1050323415.123456 # UTC timestamp + + base_d = DateTimeSubclass(*args) + self.assertIsInstance(base_d, DateTimeSubclass) + self.assertEqual(base_d.extra, 7) + + # Timestamp depends on time zone, so we'll calculate the equivalent here + ts = base_d.timestamp() + + test_cases = [ + ('fromtimestamp', (ts,), base_d), + # See https://bugs.python.org/issue32417 + ('fromtimestamp', (ts, timezone.utc), + base_d.astimezone(timezone.utc)), + ('utcfromtimestamp', (utc_ts,), base_d), + ('fromisoformat', (d_isoformat,), base_d), + ('strptime', (d_isoformat, '%Y-%m-%dT%H:%M:%S.%f'), base_d), + ('combine', (date(*args[0:3]), time(*args[3:])), base_d), + ] + + for constr_name, constr_args, expected in test_cases: + for base_obj in (DateTimeSubclass, base_d): + # Test both the classmethod and method + with self.subTest(base_obj_type=type(base_obj), + constr_name=constr_name): + constructor = getattr(base_obj, constr_name) + + if constr_name == "utcfromtimestamp": + with self.assertWarns(DeprecationWarning): + dt = constructor(*constr_args) + else: + dt = constructor(*constr_args) + + # Test that it creates the right subclass + self.assertIsInstance(dt, DateTimeSubclass) + + # Test that it's equal to the base object + self.assertEqual(dt, expected) + + # Test that it called the constructor + self.assertEqual(dt.extra, 7) + + def test_subclass_now(self): + # Test that alternate constructors call the constructor + class DateTimeSubclass(self.theclass): + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + + return result + + test_cases = [ + ('now', 'now', {}), + ('utcnow', 'utcnow', {}), + ('now_utc', 'now', {'tz': timezone.utc}), + ('now_fixed', 'now', {'tz': timezone(timedelta(hours=-5), "EST")}), + ] + + for name, meth_name, kwargs in test_cases: + with self.subTest(name): + constr = getattr(DateTimeSubclass, meth_name) + if meth_name == "utcnow": + with self.assertWarns(DeprecationWarning): + dt = constr(**kwargs) + else: + dt = constr(**kwargs) + + self.assertIsInstance(dt, DateTimeSubclass) + self.assertEqual(dt.extra, 7) + + def test_subclass_replace_fold(self): + class DateTimeSubclass(self.theclass): + pass + + dt = DateTimeSubclass(2012, 1, 1) + dt2 = DateTimeSubclass(2012, 1, 1, fold=1) + + test_cases = [ + ('self.replace', dt.replace(year=2013), 0), + ('self.replace', dt2.replace(year=2013), 1), + ('copy.replace', copy.replace(dt, year=2013), 0), + ('copy.replace', copy.replace(dt2, year=2013), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), DateTimeSubclass) + self.assertEqual(res.year, 2013) + self.assertEqual(res.fold, fold) + + def test_valuerror_messages(self): + pattern = re.compile( + r"(year|month|day|hour|minute|second) must " + r"be in \d+\.\.\d+, not \d+" + ) + test_cases = [ + (2009, 4, 1, 12, 30, 90), # Second out of range + (2009, 4, 1, 12, 90, 45), # Minute out of range + (2009, 4, 1, 25, 30, 45), # Hour out of range + (2009, 13, 1, 24, 0, 0), # Month out of range + (9999, 12, 31, 24, 0, 0), # Year out of range + ] + for case in test_cases: + with self.subTest(case): + with self.assertRaisesRegex(ValueError, pattern): + self.theclass(*case) + + # days out of range have their own error message, see issue 70647 + with self.assertRaises(ValueError) as msg: + self.theclass(2009, 4, 32, 24, 0, 0) + self.assertIn(f"day 32 must be in range 1..30 for month 4 in year 2009", str(msg.exception)) + + def test_fromisoformat_datetime(self): + # Test that isoformat() is reversible + base_dates = [ + (1, 1, 1), + (1900, 1, 1), + (2004, 11, 12), + (2017, 5, 30) + ] + + base_times = [ + (0, 0, 0, 0), + (0, 0, 0, 241000), + (0, 0, 0, 234567), + (12, 30, 45, 234567) + ] + + separators = [' ', 'T'] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=-5)), + timezone(timedelta(hours=2))] + + dts = [self.theclass(*date_tuple, *time_tuple, tzinfo=tzi) + for date_tuple in base_dates + for time_tuple in base_times + for tzi in tzinfos] + + for dt in dts: + for sep in separators: + dtstr = dt.isoformat(sep=sep) + + with self.subTest(dtstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_timezone(self): + base_dt = self.theclass(2014, 12, 30, 12, 30, 45, 217456) + + tzoffsets = [ + timedelta(hours=5), timedelta(hours=2), + timedelta(hours=6, minutes=27), + timedelta(hours=12, minutes=32, seconds=30), + timedelta(hours=2, minutes=4, seconds=9, microseconds=123456) + ] + + tzoffsets += [-1 * td for td in tzoffsets] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=0))] + + tzinfos += [timezone(td) for td in tzoffsets] + + for tzi in tzinfos: + dt = base_dt.replace(tzinfo=tzi) + dtstr = dt.isoformat() + + with self.subTest(tstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + assert dt == dt_rt, dt_rt + + def test_fromisoformat_separators(self): + separators = [ + ' ', 'T', '\u007f', # 1-bit widths + '\u0080', 'ʁ', # 2-bit widths + 'ᛇ', '時', # 3-bit widths + '🐍', # 4-bit widths + '\ud800', # bpo-34454: Surrogate code point + ] + + for sep in separators: + dt = self.theclass(2018, 1, 31, 23, 59, 47, 124789) + dtstr = dt.isoformat(sep=sep) + + with self.subTest(dtstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_ambiguous(self): + # Test strings like 2018-01-31+12:15 (where +12:15 is not a time zone) + separators = ['+', '-'] + for sep in separators: + dt = self.theclass(2018, 1, 31, 12, 15) + dtstr = dt.isoformat(sep=sep) + + with self.subTest(dtstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_timespecs(self): + datetime_bases = [ + (2009, 12, 4, 8, 17, 45, 123456), + (2009, 12, 4, 8, 17, 45, 0)] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=-5)), + timezone(timedelta(hours=2)), + timezone(timedelta(hours=6, minutes=27))] + + timespecs = ['hours', 'minutes', 'seconds', + 'milliseconds', 'microseconds'] + + for ip, ts in enumerate(timespecs): + for tzi in tzinfos: + for dt_tuple in datetime_bases: + if ts == 'milliseconds': + new_microseconds = 1000 * (dt_tuple[6] // 1000) + dt_tuple = dt_tuple[0:6] + (new_microseconds,) + + dt = self.theclass(*(dt_tuple[0:(4 + ip)]), tzinfo=tzi) + dtstr = dt.isoformat(timespec=ts) + with self.subTest(dtstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_datetime_examples(self): + UTC = timezone.utc + BST = timezone(timedelta(hours=1), 'BST') + EST = timezone(timedelta(hours=-5), 'EST') + EDT = timezone(timedelta(hours=-4), 'EDT') + examples = [ + ('2025-01-02', self.theclass(2025, 1, 2, 0, 0)), + ('2025-01-02T03', self.theclass(2025, 1, 2, 3, 0)), + ('2025-01-02T03:04', self.theclass(2025, 1, 2, 3, 4)), + ('2025-01-02T0304', self.theclass(2025, 1, 2, 3, 4)), + ('2025-01-02T03:04:05', self.theclass(2025, 1, 2, 3, 4, 5)), + ('2025-01-02T030405', self.theclass(2025, 1, 2, 3, 4, 5)), + ('2025-01-02T03:04:05.6', + self.theclass(2025, 1, 2, 3, 4, 5, 600000)), + ('2025-01-02T03:04:05,6', + self.theclass(2025, 1, 2, 3, 4, 5, 600000)), + ('2025-01-02T03:04:05.678', + self.theclass(2025, 1, 2, 3, 4, 5, 678000)), + ('2025-01-02T03:04:05.678901', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('2025-01-02T03:04:05,678901', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('2025-01-02T030405.678901', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('2025-01-02T030405,678901', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('2025-01-02T03:04:05.6789010', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('2009-04-19T03:15:45.2345', + self.theclass(2009, 4, 19, 3, 15, 45, 234500)), + ('2009-04-19T03:15:45.1234567', + self.theclass(2009, 4, 19, 3, 15, 45, 123456)), + ('2025-01-02T03:04:05,678', + self.theclass(2025, 1, 2, 3, 4, 5, 678000)), + ('20250102', self.theclass(2025, 1, 2, 0, 0)), + ('20250102T03', self.theclass(2025, 1, 2, 3, 0)), + ('20250102T03:04', self.theclass(2025, 1, 2, 3, 4)), + ('20250102T03:04:05', self.theclass(2025, 1, 2, 3, 4, 5)), + ('20250102T030405', self.theclass(2025, 1, 2, 3, 4, 5)), + ('20250102T03:04:05.6', + self.theclass(2025, 1, 2, 3, 4, 5, 600000)), + ('20250102T03:04:05,6', + self.theclass(2025, 1, 2, 3, 4, 5, 600000)), + ('20250102T03:04:05.678', + self.theclass(2025, 1, 2, 3, 4, 5, 678000)), + ('20250102T03:04:05,678', + self.theclass(2025, 1, 2, 3, 4, 5, 678000)), + ('20250102T03:04:05.678901', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('20250102T030405.678901', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('20250102T030405,678901', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('20250102T030405.6789010', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('2022W01', self.theclass(2022, 1, 3)), + ('2022W52520', self.theclass(2022, 12, 26, 20, 0)), + ('2022W527520', self.theclass(2023, 1, 1, 20, 0)), + ('2026W01516', self.theclass(2025, 12, 29, 16, 0)), + ('2026W013516', self.theclass(2025, 12, 31, 16, 0)), + ('2025W01503', self.theclass(2024, 12, 30, 3, 0)), + ('2025W014503', self.theclass(2025, 1, 2, 3, 0)), + ('2025W01512', self.theclass(2024, 12, 30, 12, 0)), + ('2025W014512', self.theclass(2025, 1, 2, 12, 0)), + ('2025W014T121431', self.theclass(2025, 1, 2, 12, 14, 31)), + ('2026W013T162100', self.theclass(2025, 12, 31, 16, 21)), + ('2026W013 162100', self.theclass(2025, 12, 31, 16, 21)), + ('2022W527T202159', self.theclass(2023, 1, 1, 20, 21, 59)), + ('2022W527 202159', self.theclass(2023, 1, 1, 20, 21, 59)), + ('2025W014 121431', self.theclass(2025, 1, 2, 12, 14, 31)), + ('2025W014T030405', self.theclass(2025, 1, 2, 3, 4, 5)), + ('2025W014 030405', self.theclass(2025, 1, 2, 3, 4, 5)), + ('2020-W53-6T03:04:05', self.theclass(2021, 1, 2, 3, 4, 5)), + ('2020W537 03:04:05', self.theclass(2021, 1, 3, 3, 4, 5)), + ('2025-W01-4T03:04:05', self.theclass(2025, 1, 2, 3, 4, 5)), + ('2025-W01-4T03:04:05.678901', + self.theclass(2025, 1, 2, 3, 4, 5, 678901)), + ('2025-W01-4T12:14:31', self.theclass(2025, 1, 2, 12, 14, 31)), + ('2025-W01-4T12:14:31.012345', + self.theclass(2025, 1, 2, 12, 14, 31, 12345)), + ('2026-W01-3T16:21:00', self.theclass(2025, 12, 31, 16, 21)), + ('2026-W01-3T16:21:00.000000', self.theclass(2025, 12, 31, 16, 21)), + ('2022-W52-7T20:21:59', + self.theclass(2023, 1, 1, 20, 21, 59)), + ('2022-W52-7T20:21:59.999999', + self.theclass(2023, 1, 1, 20, 21, 59, 999999)), + ('2025-W01003+00', + self.theclass(2024, 12, 30, 3, 0, tzinfo=UTC)), + ('2025-01-02T03:04:05+00', + self.theclass(2025, 1, 2, 3, 4, 5, tzinfo=UTC)), + ('2025-01-02T03:04:05Z', + self.theclass(2025, 1, 2, 3, 4, 5, tzinfo=UTC)), + ('2025-01-02003:04:05,6+00:00:00.00', + self.theclass(2025, 1, 2, 3, 4, 5, 600000, tzinfo=UTC)), + ('2000-01-01T00+21', + self.theclass(2000, 1, 1, 0, 0, tzinfo=timezone(timedelta(hours=21)))), + ('2025-01-02T03:05:06+0300', + self.theclass(2025, 1, 2, 3, 5, 6, + tzinfo=timezone(timedelta(hours=3)))), + ('2025-01-02T03:05:06-0300', + self.theclass(2025, 1, 2, 3, 5, 6, + tzinfo=timezone(timedelta(hours=-3)))), + ('2025-01-02T03:04:05+0000', + self.theclass(2025, 1, 2, 3, 4, 5, tzinfo=UTC)), + ('2025-01-02T03:05:06+03', + self.theclass(2025, 1, 2, 3, 5, 6, + tzinfo=timezone(timedelta(hours=3)))), + ('2025-01-02T03:05:06-03', + self.theclass(2025, 1, 2, 3, 5, 6, + tzinfo=timezone(timedelta(hours=-3)))), + ('2020-01-01T03:05:07.123457-05:00', + self.theclass(2020, 1, 1, 3, 5, 7, 123457, tzinfo=EST)), + ('2020-01-01T03:05:07.123457-0500', + self.theclass(2020, 1, 1, 3, 5, 7, 123457, tzinfo=EST)), + ('2020-06-01T04:05:06.111111-04:00', + self.theclass(2020, 6, 1, 4, 5, 6, 111111, tzinfo=EDT)), + ('2020-06-01T04:05:06.111111-0400', + self.theclass(2020, 6, 1, 4, 5, 6, 111111, tzinfo=EDT)), + ('2021-10-31T01:30:00.000000+01:00', + self.theclass(2021, 10, 31, 1, 30, tzinfo=BST)), + ('2021-10-31T01:30:00.000000+0100', + self.theclass(2021, 10, 31, 1, 30, tzinfo=BST)), + ('2025-01-02T03:04:05,6+000000.00', + self.theclass(2025, 1, 2, 3, 4, 5, 600000, tzinfo=UTC)), + ('2025-01-02T03:04:05,678+00:00:10', + self.theclass(2025, 1, 2, 3, 4, 5, 678000, + tzinfo=timezone(timedelta(seconds=10)))), + ('2025-01-02T24:00:00', self.theclass(2025, 1, 3, 0, 0, 0)), + ('2025-01-31T24:00:00', self.theclass(2025, 2, 1, 0, 0, 0)), + ('2025-12-31T24:00:00', self.theclass(2026, 1, 1, 0, 0, 0)) + ] + + for input_str, expected in examples: + with self.subTest(input_str=input_str): + actual = self.theclass.fromisoformat(input_str) + self.assertEqual(actual, expected) + + def test_fromisoformat_fails_datetime(self): + # Test that fromisoformat() fails on invalid values + bad_strs = [ + '', # Empty string + '\ud800', # bpo-34454: Surrogate code point + '2009.04-19T03', # Wrong first separator + '2009-04.19T03', # Wrong second separator + '2009-04-19T0a', # Invalid hours + '2009-04-19T03:1a:45', # Invalid minutes + '2009-04-19T03:15:4a', # Invalid seconds + '2009-04-19T03;15:45', # Bad first time separator + '2009-04-19T03:15;45', # Bad second time separator + '2009-04-19T03:15:4500:00', # Bad time zone separator + '2009-04-19T03:15:45.123456+24:30', # Invalid time zone offset + '2009-04-19T03:15:45.123456-24:30', # Invalid negative offset + '2009-04-10ᛇᛇᛇᛇᛇ12:15', # Unicode chars + '2009-04\ud80010T12:15', # Surrogate char in date + '2009-04-10T12\ud80015', # Surrogate char in time + '2009-04-19T1', # Incomplete hours + '2009-04-19T12:3', # Incomplete minutes + '2009-04-19T12:30:4', # Incomplete seconds + '2009-04-19T12:', # Ends with time separator + '2009-04-19T12:30:', # Ends with time separator + '2009-04-19T12:30:45.', # Ends with time separator + '2009-04-19T12:30:45.123456+', # Ends with timezone separator + '2009-04-19T12:30:45.123456-', # Ends with timezone separator + '2009-04-19T12:30:45.123456-05:00a', # Extra text + '2009-04-19T12:30:45.123-05:00a', # Extra text + '2009-04-19T12:30:45-05:00a', # Extra text + '2009-04-19T24:00:00.000001', # Has non-zero microseconds on 24:00 + '2009-04-19T24:00:01.000000', # Has non-zero seconds on 24:00 + '2009-04-19T24:01:00.000000', # Has non-zero minutes on 24:00 + '2009-04-32T24:00:00.000000', # Day is invalid before wrapping due to 24:00 + '2009-13-01T24:00:00.000000', # Month is invalid before wrapping due to 24:00 + '9999-12-31T24:00:00.000000', # Year is invalid after wrapping due to 24:00 + '2009-04-19T12:30Z12:00', # Extra time zone info after Z + '2009-04-19T12:30:45:334034', # Invalid microsecond separator + '2009-04-19T12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) + '2009-04-19T12:30:45.400 ', # Trailing space (gh-130959) + '2009-04-19T12:30:45. 400', # Space before fraction (gh-130959) + ] + + for bad_str in bad_strs: + with self.subTest(bad_str=bad_str): + with self.assertRaises(ValueError): + self.theclass.fromisoformat(bad_str) + + def test_fromisoformat_fails_datetime_valueerror(self): + pattern = re.compile( + r"(year|month|day|hour|minute|second) must " + r"be in \d+\.\.\d+, not \d+" + ) + bad_strs = [ + "2009-04-01T12:30:90", # Second out of range + "2009-04-01T12:90:45", # Minute out of range + "2009-04-01T25:30:45", # Hour out of range + "2009-13-01T24:00:00", # Month out of range + "9999-12-31T24:00:00", # Year out of range + ] + + for bad_str in bad_strs: + with self.subTest(bad_str=bad_str): + with self.assertRaisesRegex(ValueError, pattern): + self.theclass.fromisoformat(bad_str) + + # days out of range have their own error message, see issue 70647 + with self.assertRaises(ValueError) as msg: + self.theclass.fromisoformat("2009-04-32T24:00:00") + self.assertIn(f"day 32 must be in range 1..30 for month 4 in year 2009", str(msg.exception)) + + def test_fromisoformat_fails_surrogate(self): + # Test that when fromisoformat() fails with a surrogate character as + # the separator, the error message contains the original string + dtstr = "2018-01-03\ud80001:0113" + + with self.assertRaisesRegex(ValueError, re.escape(repr(dtstr))): + self.theclass.fromisoformat(dtstr) + + def test_fromisoformat_utc(self): + dt_str = '2014-04-19T13:21:13+00:00' + dt = self.theclass.fromisoformat(dt_str) + + self.assertIs(dt.tzinfo, timezone.utc) + + def test_fromisoformat_subclass(self): + class DateTimeSubclass(self.theclass): + pass + + dt = DateTimeSubclass(2014, 12, 14, 9, 30, 45, 457390, + tzinfo=timezone(timedelta(hours=10, minutes=45))) + + dt_rt = DateTimeSubclass.fromisoformat(dt.isoformat()) + + self.assertEqual(dt, dt_rt) + self.assertIsInstance(dt_rt, DateTimeSubclass) + + def test_repr_subclass(self): + """Subclasses should have bare names in the repr (gh-107773).""" + td = SubclassDatetime(2014, 1, 1) + self.assertEqual(repr(td), "SubclassDatetime(2014, 1, 1, 0, 0)") + td = SubclassDatetime(2010, 10, day=10) + self.assertEqual(repr(td), "SubclassDatetime(2010, 10, 10, 0, 0)") + td = SubclassDatetime(2010, 10, 2, second=3) + self.assertEqual(repr(td), "SubclassDatetime(2010, 10, 2, 0, 0, 3)") + + +class TestSubclassDateTime(TestDateTime): + theclass = SubclassDatetime + # Override tests not designed for subclass + @unittest.skip('not appropriate for subclasses') + def test_roundtrip(self): + pass + +class SubclassTime(time): + sub_var = 1 + +class TestTime(HarmlessMixedComparison, unittest.TestCase): + + theclass = time + + def test_basic_attributes(self): + t = self.theclass(12, 0) + self.assertEqual(t.hour, 12) + self.assertEqual(t.minute, 0) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 0) + + def test_basic_attributes_nonzero(self): + # Make sure all attributes are non-zero so bugs in + # bit-shifting access show up. + t = self.theclass(12, 59, 59, 8000) + self.assertEqual(t.hour, 12) + self.assertEqual(t.minute, 59) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 8000) + + def test_roundtrip(self): + t = self.theclass(1, 2, 3, 4) + + # Verify t -> string -> time identity. + s = repr(t) + self.assertStartsWith(s, 'datetime.') + s = s[9:] + t2 = eval(s) + self.assertEqual(t, t2) + + # Verify identity via reconstructing from pieces. + t2 = self.theclass(t.hour, t.minute, t.second, + t.microsecond) + self.assertEqual(t, t2) + + def test_comparing(self): + args = [1, 2, 3, 4] + t1 = self.theclass(*args) + t2 = self.theclass(*args) + self.assertEqual(t1, t2) + self.assertTrue(t1 <= t2) + self.assertTrue(t1 >= t2) + self.assertFalse(t1 != t2) + self.assertFalse(t1 < t2) + self.assertFalse(t1 > t2) + + for i in range(len(args)): + newargs = args[:] + newargs[i] = args[i] + 1 + t2 = self.theclass(*newargs) # this is larger than t1 + self.assertTrue(t1 < t2) + self.assertTrue(t2 > t1) + self.assertTrue(t1 <= t2) + self.assertTrue(t2 >= t1) + self.assertTrue(t1 != t2) + self.assertTrue(t2 != t1) + self.assertFalse(t1 == t2) + self.assertFalse(t2 == t1) + self.assertFalse(t1 > t2) + self.assertFalse(t2 < t1) + self.assertFalse(t1 >= t2) + self.assertFalse(t2 <= t1) + + for badarg in OTHERSTUFF: + self.assertEqual(t1 == badarg, False) + self.assertEqual(t1 != badarg, True) + self.assertEqual(badarg == t1, False) + self.assertEqual(badarg != t1, True) + + self.assertRaises(TypeError, lambda: t1 <= badarg) + self.assertRaises(TypeError, lambda: t1 < badarg) + self.assertRaises(TypeError, lambda: t1 > badarg) + self.assertRaises(TypeError, lambda: t1 >= badarg) + self.assertRaises(TypeError, lambda: badarg <= t1) + self.assertRaises(TypeError, lambda: badarg < t1) + self.assertRaises(TypeError, lambda: badarg > t1) + self.assertRaises(TypeError, lambda: badarg >= t1) + + def test_bad_constructor_arguments(self): + # bad hours + self.theclass(0, 0) # no exception + self.theclass(23, 0) # no exception + self.assertRaises(ValueError, self.theclass, -1, 0) + self.assertRaises(ValueError, self.theclass, 24, 0) + # bad minutes + self.theclass(23, 0) # no exception + self.theclass(23, 59) # no exception + self.assertRaises(ValueError, self.theclass, 23, -1) + self.assertRaises(ValueError, self.theclass, 23, 60) + # bad seconds + self.theclass(23, 59, 0) # no exception + self.theclass(23, 59, 59) # no exception + self.assertRaises(ValueError, self.theclass, 23, 59, -1) + self.assertRaises(ValueError, self.theclass, 23, 59, 60) + # bad microseconds + self.theclass(23, 59, 59, 0) # no exception + self.theclass(23, 59, 59, 999999) # no exception + self.assertRaises(ValueError, self.theclass, 23, 59, 59, -1) + self.assertRaises(ValueError, self.theclass, 23, 59, 59, 1000000) + + def test_hash_equality(self): + d = self.theclass(23, 30, 17) + e = self.theclass(23, 30, 17) + self.assertEqual(d, e) + self.assertEqual(hash(d), hash(e)) + + dic = {d: 1} + dic[e] = 2 + self.assertEqual(len(dic), 1) + self.assertEqual(dic[d], 2) + self.assertEqual(dic[e], 2) + + d = self.theclass(0, 5, 17) + e = self.theclass(0, 5, 17) + self.assertEqual(d, e) + self.assertEqual(hash(d), hash(e)) + + dic = {d: 1} + dic[e] = 2 + self.assertEqual(len(dic), 1) + self.assertEqual(dic[d], 2) + self.assertEqual(dic[e], 2) + + def test_isoformat(self): + t = self.theclass(4, 5, 1, 123) + self.assertEqual(t.isoformat(), "04:05:01.000123") + self.assertEqual(t.isoformat(), str(t)) + + t = self.theclass() + self.assertEqual(t.isoformat(), "00:00:00") + self.assertEqual(t.isoformat(), str(t)) + + t = self.theclass(microsecond=1) + self.assertEqual(t.isoformat(), "00:00:00.000001") + self.assertEqual(t.isoformat(), str(t)) + + t = self.theclass(microsecond=10) + self.assertEqual(t.isoformat(), "00:00:00.000010") + self.assertEqual(t.isoformat(), str(t)) + + t = self.theclass(microsecond=100) + self.assertEqual(t.isoformat(), "00:00:00.000100") + self.assertEqual(t.isoformat(), str(t)) + + t = self.theclass(microsecond=1000) + self.assertEqual(t.isoformat(), "00:00:00.001000") + self.assertEqual(t.isoformat(), str(t)) + + t = self.theclass(microsecond=10000) + self.assertEqual(t.isoformat(), "00:00:00.010000") + self.assertEqual(t.isoformat(), str(t)) + + t = self.theclass(microsecond=100000) + self.assertEqual(t.isoformat(), "00:00:00.100000") + self.assertEqual(t.isoformat(), str(t)) + + t = self.theclass(hour=12, minute=34, second=56, microsecond=123456) + self.assertEqual(t.isoformat(timespec='hours'), "12") + self.assertEqual(t.isoformat(timespec='minutes'), "12:34") + self.assertEqual(t.isoformat(timespec='seconds'), "12:34:56") + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.123") + self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.123456") + self.assertEqual(t.isoformat(timespec='auto'), "12:34:56.123456") + self.assertRaises(ValueError, t.isoformat, timespec='monkey') + # bpo-34482: Check that surrogates are handled properly. + self.assertRaises(ValueError, t.isoformat, timespec='\ud800') + + t = self.theclass(hour=12, minute=34, second=56, microsecond=999500) + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.999") + + t = self.theclass(hour=12, minute=34, second=56, microsecond=0) + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.000000") + self.assertEqual(t.isoformat(timespec='auto'), "12:34:56") + + def test_isoformat_timezone(self): + tzoffsets = [ + ('05:00', timedelta(hours=5)), + ('02:00', timedelta(hours=2)), + ('06:27', timedelta(hours=6, minutes=27)), + ('12:32:30', timedelta(hours=12, minutes=32, seconds=30)), + ('02:04:09.123456', timedelta(hours=2, minutes=4, seconds=9, microseconds=123456)) + ] + + tzinfos = [ + ('', None), + ('+00:00', timezone.utc), + ('+00:00', timezone(timedelta(0))), + ] + + tzinfos += [ + (prefix + expected, timezone(sign * td)) + for expected, td in tzoffsets + for prefix, sign in [('-', -1), ('+', 1)] + ] + + t_base = self.theclass(12, 37, 9) + exp_base = '12:37:09' + + for exp_tz, tzi in tzinfos: + t = t_base.replace(tzinfo=tzi) + exp = exp_base + exp_tz + with self.subTest(tzi=tzi): + assert t.isoformat() == exp + + def test_1653736(self): + # verify it doesn't accept extra keyword arguments + t = self.theclass(second=1) + self.assertRaises(TypeError, t.isoformat, foo=3) + + def test_strftime(self): + t = self.theclass(1, 2, 3, 4) + self.assertEqual(t.strftime('%H %M %S %f'), "01 02 03 000004") + # A naive object replaces %z, %:z and %Z with empty strings. + self.assertEqual(t.strftime("'%z' '%:z' '%Z'"), "'' '' ''") + + # bpo-34482: Check that surrogates don't cause a crash. + try: + t.strftime('%H\ud800%M') + except UnicodeEncodeError: + pass + + # gh-85432: The parameter was named "fmt" in the pure-Python impl. + t.strftime(format="%f") + + @unittest.skip("TODO: RUSTPYTHON") + def test_strftime_special(self): + t = self.theclass(1, 2, 3, 4) + s1 = t.strftime('%I%p%Z') + s2 = t.strftime('%X') + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(t.strftime('\U0001f40d'), '\U0001f40d') + self.assertEqual(t.strftime('\U0001f4bb%I%p%Z\U0001f40d%X'), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(t.strftime('%I%p%Z\U0001f4bb%X\U0001f40d'), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(t.strftime('\ud83d'), '\ud83d') + self.assertEqual(t.strftime('\udc0d'), '\udc0d') + self.assertEqual(t.strftime('\ud83d%I%p%Z\udc0d%X'), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(t.strftime('%I%p%Z\ud83d%X\udc0d'), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(t.strftime('%I%p%Z\udc0d%X\ud83d'), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(t.strftime('\ud83d\udc0d'), '\ud83d\udc0d') + self.assertEqual(t.strftime('%I%p%Z\ud83d\udc0d%X'), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(t.strftime('\udcf0\udc9f\udc90\udc8d'), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(t.strftime('%I%p%Z\udcf0\udc9f\udc90\udc8d%X'), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(t.strftime('\0'), '\0') + self.assertEqual(t.strftime('\0'*1000), '\0'*1000) + self.assertEqual(t.strftime('\0%I%p%Z\0%X'), f'\0{s1}\0{s2}') + self.assertEqual(t.strftime('%I%p%Z\0%X\0'), f'{s1}\0{s2}\0') + + def test_format(self): + t = self.theclass(1, 2, 3, 4) + self.assertEqual(t.__format__(''), str(t)) + + with self.assertRaisesRegex(TypeError, 'must be str, not int'): + t.__format__(123) + + # check that a derived class's __str__() gets called + class A(self.theclass): + def __str__(self): + return 'A' + a = A(1, 2, 3, 4) + self.assertEqual(a.__format__(''), 'A') + + # check that a derived class's strftime gets called + class B(self.theclass): + def strftime(self, format_spec): + return 'B' + b = B(1, 2, 3, 4) + self.assertEqual(b.__format__(''), str(t)) + + for fmt in ['%H %M %S', + ]: + self.assertEqual(t.__format__(fmt), t.strftime(fmt)) + self.assertEqual(a.__format__(fmt), t.strftime(fmt)) + self.assertEqual(b.__format__(fmt), 'B') + + def test_str(self): + self.assertEqual(str(self.theclass(1, 2, 3, 4)), "01:02:03.000004") + self.assertEqual(str(self.theclass(10, 2, 3, 4000)), "10:02:03.004000") + self.assertEqual(str(self.theclass(0, 2, 3, 400000)), "00:02:03.400000") + self.assertEqual(str(self.theclass(12, 2, 3, 0)), "12:02:03") + self.assertEqual(str(self.theclass(23, 15, 0, 0)), "23:15:00") + + def test_repr(self): + name = 'datetime.' + self.theclass.__name__ + self.assertEqual(repr(self.theclass(1, 2, 3, 4)), + "%s(1, 2, 3, 4)" % name) + self.assertEqual(repr(self.theclass(10, 2, 3, 4000)), + "%s(10, 2, 3, 4000)" % name) + self.assertEqual(repr(self.theclass(0, 2, 3, 400000)), + "%s(0, 2, 3, 400000)" % name) + self.assertEqual(repr(self.theclass(12, 2, 3, 0)), + "%s(12, 2, 3)" % name) + self.assertEqual(repr(self.theclass(23, 15, 0, 0)), + "%s(23, 15)" % name) + + def test_repr_subclass(self): + """Subclasses should have bare names in the repr (gh-107773).""" + td = SubclassTime(hour=1) + self.assertEqual(repr(td), "SubclassTime(1, 0)") + td = SubclassTime(hour=2, minute=30) + self.assertEqual(repr(td), "SubclassTime(2, 30)") + td = SubclassTime(hour=2, minute=30, second=11) + self.assertEqual(repr(td), "SubclassTime(2, 30, 11)") + td = SubclassTime(minute=30, second=11, fold=0) + self.assertEqual(repr(td), "SubclassTime(0, 30, 11)") + td = SubclassTime(minute=30, second=11, fold=1) + self.assertEqual(repr(td), "SubclassTime(0, 30, 11, fold=1)") + + def test_resolution_info(self): + self.assertIsInstance(self.theclass.min, self.theclass) + self.assertIsInstance(self.theclass.max, self.theclass) + self.assertIsInstance(self.theclass.resolution, timedelta) + self.assertTrue(self.theclass.max > self.theclass.min) + + def test_pickling(self): + args = 20, 59, 16, 64**2 + orig = self.theclass(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) + + def test_pickling_subclass_time(self): + args = 20, 59, 16, 64**2 + orig = SubclassTime(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertTrue(isinstance(derived, SubclassTime)) + + def test_compat_unpickle(self): + tests = [ + (b"cdatetime\ntime\n(S'\\x14;\\x10\\x00\\x10\\x00'\ntR.", + (20, 59, 16, 64**2)), + (b'cdatetime\ntime\n(U\x06\x14;\x10\x00\x10\x00tR.', + (20, 59, 16, 64**2)), + (b'\x80\x02cdatetime\ntime\nU\x06\x14;\x10\x00\x10\x00\x85R.', + (20, 59, 16, 64**2)), + (b"cdatetime\ntime\n(S'\\x14;\\x19\\x00\\x10\\x00'\ntR.", + (20, 59, 25, 64**2)), + (b'cdatetime\ntime\n(U\x06\x14;\x19\x00\x10\x00tR.', + (20, 59, 25, 64**2)), + (b'\x80\x02cdatetime\ntime\nU\x06\x14;\x19\x00\x10\x00\x85R.', + (20, 59, 25, 64**2)), + ] + for i, (data, args) in enumerate(tests): + with self.subTest(i=i): + expected = self.theclass(*args) + for loads in pickle_loads: + derived = loads(data, encoding='latin1') + self.assertEqual(derived, expected) + + def test_strptime(self): + # bpo-34482: Check that surrogates are handled properly. + inputs = [ + (self.theclass(13, 2, 47, 197000), '13:02:47.197', '%H:%M:%S.%f'), + (self.theclass(13, 2, 47, 197000), '13:02\ud80047.197', '%H:%M\ud800%S.%f'), + (self.theclass(13, 2, 47, 197000), '13\ud80002:47.197', '%H\ud800%M:%S.%f'), + ] + for expected, string, format in inputs: + with self.subTest(string=string, format=format): + got = self.theclass.strptime(string, format) + self.assertEqual(expected, got) + self.assertIs(type(got), self.theclass) + + def test_strptime_tz(self): + strptime = self.theclass.strptime + self.assertEqual(strptime("+0002", "%z").utcoffset(), 2 * MINUTE) + self.assertEqual(strptime("-0002", "%z").utcoffset(), -2 * MINUTE) + self.assertEqual( + strptime("-00:02:01.000003", "%z").utcoffset(), + -timedelta(minutes=2, seconds=1, microseconds=3) + ) + # Only local timezone and UTC are supported + for tzseconds, tzname in ((0, 'UTC'), (0, 'GMT'), + (-_time.timezone, _time.tzname[0])): + if tzseconds < 0: + sign = '-' + seconds = -tzseconds + else: + sign ='+' + seconds = tzseconds + hours, minutes = divmod(seconds//60, 60) + tstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) + with self.subTest(tstr=tstr): + t = strptime(tstr, "%z %Z") + self.assertEqual(t.utcoffset(), timedelta(seconds=tzseconds)) + self.assertEqual(t.tzname(), tzname) + self.assertIs(type(t), self.theclass) + + # Can produce inconsistent time + tstr, fmt = "+1234 UTC", "%z %Z" + t = strptime(tstr, fmt) + self.assertEqual(t.utcoffset(), 12 * HOUR + 34 * MINUTE) + self.assertEqual(t.tzname(), 'UTC') + # yet will roundtrip + self.assertEqual(t.strftime(fmt), tstr) + + # Produce naive time if no %z is provided + self.assertEqual(strptime("UTC", "%Z").tzinfo, None) + + def test_strptime_errors(self): + for tzstr in ("-2400", "-000", "z"): + with self.assertRaises(ValueError): + self.theclass.strptime(tzstr, "%z") + + def test_strptime_single_digit(self): + # bpo-34903: Check that single digit times are allowed. + t = self.theclass(4, 5, 6) + inputs = [ + ('%H', '4:05:06', '%H:%M:%S', t), + ('%M', '04:5:06', '%H:%M:%S', t), + ('%S', '04:05:6', '%H:%M:%S', t), + ('%I', '4am:05:06', '%I%p:%M:%S', t), + ] + for reason, string, format, target in inputs: + reason = 'test single digit ' + reason + with self.subTest(reason=reason, + string=string, + format=format, + target=target): + newdate = self.theclass.strptime(string, format) + self.assertEqual(newdate, target, msg=reason) + + def test_bool(self): + # time is always True. + cls = self.theclass + self.assertTrue(cls(1)) + self.assertTrue(cls(0, 1)) + self.assertTrue(cls(0, 0, 1)) + self.assertTrue(cls(0, 0, 0, 1)) + self.assertTrue(cls(0)) + self.assertTrue(cls()) + + def test_replace(self): + cls = self.theclass + args = [1, 2, 3, 4] + base = cls(*args) + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8)) + for i, (name, newval) in enumerate(changes): + newargs = args[:] + newargs[i] = newval + expected = cls(*newargs) + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) + + # Out of bounds. + base = cls(1) + self.assertRaises(ValueError, base.replace, hour=24) + self.assertRaises(ValueError, base.replace, minute=-1) + self.assertRaises(ValueError, base.replace, second=100) + self.assertRaises(ValueError, base.replace, microsecond=1000000) + self.assertRaises(ValueError, copy.replace, base, hour=24) + self.assertRaises(ValueError, copy.replace, base, minute=-1) + self.assertRaises(ValueError, copy.replace, base, second=100) + self.assertRaises(ValueError, copy.replace, base, microsecond=1000000) + + def test_subclass_replace(self): + class TimeSubclass(self.theclass): + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + return result + + ctime = TimeSubclass(12, 30) + ctime2 = TimeSubclass(12, 30, fold=1) + + test_cases = [ + ('self.replace', ctime.replace(hour=10), 0), + ('self.replace', ctime2.replace(hour=10), 1), + ('copy.replace', copy.replace(ctime, hour=10), 0), + ('copy.replace', copy.replace(ctime2, hour=10), 1), + ] + + for name, res, fold in test_cases: + with self.subTest(name, fold=fold): + self.assertIs(type(res), TimeSubclass) + self.assertEqual(res.hour, 10) + self.assertEqual(res.minute, 30) + self.assertEqual(res.extra, 7) + self.assertEqual(res.fold, fold) + + def test_subclass_time(self): + + class C(self.theclass): + theAnswer = 42 + + def __new__(cls, *args, **kws): + temp = kws.copy() + extra = temp.pop('extra') + result = self.theclass.__new__(cls, *args, **temp) + result.extra = extra + return result + + def newmeth(self, start): + return start + self.hour + self.second + + args = 4, 5, 6 + + dt1 = self.theclass(*args) + dt2 = C(*args, **{'extra': 7}) + + self.assertEqual(dt2.__class__, C) + self.assertEqual(dt2.theAnswer, 42) + self.assertEqual(dt2.extra, 7) + self.assertEqual(dt1.isoformat(), dt2.isoformat()) + self.assertEqual(dt2.newmeth(-7), dt1.hour + dt1.second - 7) + + def test_backdoor_resistance(self): + # see TestDate.test_backdoor_resistance(). + base = '2:59.0' + for hour_byte in ' ', '9', chr(24), '\xff': + self.assertRaises(TypeError, self.theclass, + hour_byte + base[1:]) + # Good bytes, but bad tzinfo: + with self.assertRaisesRegex(TypeError, '^bad tzinfo state arg$'): + self.theclass(bytes([1] * len(base)), 'EST') + +# A mixin for classes with a tzinfo= argument. Subclasses must define +# theclass as a class attribute, and theclass(1, 1, 1, tzinfo=whatever) +# must be legit (which is true for time and datetime). +class TZInfoBase: + + def test_argument_passing(self): + cls = self.theclass + # A datetime passes itself on, a time passes None. + class introspective(tzinfo): + def tzname(self, dt): return dt and "real" or "none" + def utcoffset(self, dt): + return timedelta(minutes = dt and 42 or -42) + dst = utcoffset + + obj = cls(1, 2, 3, tzinfo=introspective()) + + expected = cls is time and "none" or "real" + self.assertEqual(obj.tzname(), expected) + + expected = timedelta(minutes=(cls is time and -42 or 42)) + self.assertEqual(obj.utcoffset(), expected) + self.assertEqual(obj.dst(), expected) + + def test_bad_tzinfo_classes(self): + cls = self.theclass + self.assertRaises(TypeError, cls, 1, 1, 1, tzinfo=12) + + class NiceTry(object): + def __init__(self): pass + def utcoffset(self, dt): pass + self.assertRaises(TypeError, cls, 1, 1, 1, tzinfo=NiceTry) + + class BetterTry(tzinfo): + def __init__(self): pass + def utcoffset(self, dt): pass + b = BetterTry() + t = cls(1, 1, 1, tzinfo=b) + self.assertIs(t.tzinfo, b) + + def test_utc_offset_out_of_bounds(self): + class Edgy(tzinfo): + def __init__(self, offset): + self.offset = timedelta(minutes=offset) + def utcoffset(self, dt): + return self.offset + + cls = self.theclass + for offset, legit in ((-1440, False), + (-1439, True), + (1439, True), + (1440, False)): + if cls is time: + t = cls(1, 2, 3, tzinfo=Edgy(offset)) + elif cls is datetime: + t = cls(6, 6, 6, 1, 2, 3, tzinfo=Edgy(offset)) + else: + assert 0, "impossible" + if legit: + aofs = abs(offset) + h, m = divmod(aofs, 60) + tag = "%c%02d:%02d" % (offset < 0 and '-' or '+', h, m) + if isinstance(t, datetime): + t = t.timetz() + self.assertEqual(str(t), "01:02:03" + tag) + else: + self.assertRaises(ValueError, str, t) + + def test_tzinfo_classes(self): + cls = self.theclass + class C1(tzinfo): + def utcoffset(self, dt): return None + def dst(self, dt): return None + def tzname(self, dt): return None + for t in (cls(1, 1, 1), + cls(1, 1, 1, tzinfo=None), + cls(1, 1, 1, tzinfo=C1())): + self.assertIsNone(t.utcoffset()) + self.assertIsNone(t.dst()) + self.assertIsNone(t.tzname()) + + class C3(tzinfo): + def utcoffset(self, dt): return timedelta(minutes=-1439) + def dst(self, dt): return timedelta(minutes=1439) + def tzname(self, dt): return "aname" + t = cls(1, 1, 1, tzinfo=C3()) + self.assertEqual(t.utcoffset(), timedelta(minutes=-1439)) + self.assertEqual(t.dst(), timedelta(minutes=1439)) + self.assertEqual(t.tzname(), "aname") + + # Wrong types. + class C4(tzinfo): + def utcoffset(self, dt): return "aname" + def dst(self, dt): return 7 + def tzname(self, dt): return 0 + t = cls(1, 1, 1, tzinfo=C4()) + self.assertRaises(TypeError, t.utcoffset) + self.assertRaises(TypeError, t.dst) + self.assertRaises(TypeError, t.tzname) + + # Offset out of range. + class C6(tzinfo): + def utcoffset(self, dt): return timedelta(hours=-24) + def dst(self, dt): return timedelta(hours=24) + t = cls(1, 1, 1, tzinfo=C6()) + self.assertRaises(ValueError, t.utcoffset) + self.assertRaises(ValueError, t.dst) + + # Not a whole number of seconds. + class C7(tzinfo): + def utcoffset(self, dt): return timedelta(microseconds=61) + def dst(self, dt): return timedelta(microseconds=-81) + t = cls(1, 1, 1, tzinfo=C7()) + self.assertEqual(t.utcoffset(), timedelta(microseconds=61)) + self.assertEqual(t.dst(), timedelta(microseconds=-81)) + + def test_aware_compare(self): + cls = self.theclass + + # Ensure that utcoffset() gets ignored if the comparands have + # the same tzinfo member. + class OperandDependentOffset(tzinfo): + def utcoffset(self, t): + if t.minute < 10: + # d0 and d1 equal after adjustment + return timedelta(minutes=t.minute) + else: + # d2 off in the weeds + return timedelta(minutes=59) + + base = cls(8, 9, 10, tzinfo=OperandDependentOffset()) + d0 = base.replace(minute=3) + d1 = base.replace(minute=9) + d2 = base.replace(minute=11) + for x in d0, d1, d2: + for y in d0, d1, d2: + for op in lt, le, gt, ge, eq, ne: + got = op(x, y) + expected = op(x.minute, y.minute) + self.assertEqual(got, expected) + + # However, if they're different members, uctoffset is not ignored. + # Note that a time can't actually have an operand-dependent offset, + # though (and time.utcoffset() passes None to tzinfo.utcoffset()), + # so skip this test for time. + if cls is not time: + d0 = base.replace(minute=3, tzinfo=OperandDependentOffset()) + d1 = base.replace(minute=9, tzinfo=OperandDependentOffset()) + d2 = base.replace(minute=11, tzinfo=OperandDependentOffset()) + for x in d0, d1, d2: + for y in d0, d1, d2: + got = (x > y) - (x < y) + if (x is d0 or x is d1) and (y is d0 or y is d1): + expected = 0 + elif x is y is d2: + expected = 0 + elif x is d2: + expected = -1 + else: + assert y is d2 + expected = 1 + self.assertEqual(got, expected) + + +# Testing time objects with a non-None tzinfo. +class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase): + theclass = time + + def test_empty(self): + t = self.theclass() + self.assertEqual(t.hour, 0) + self.assertEqual(t.minute, 0) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 0) + self.assertIsNone(t.tzinfo) + + @unittest.skip("TODO: RUSTPYTHON") + def test_zones(self): + est = FixedOffset(-300, "EST", 1) + utc = FixedOffset(0, "UTC", -2) + met = FixedOffset(60, "MET", 3) + t1 = time( 7, 47, tzinfo=est) + t2 = time(12, 47, tzinfo=utc) + t3 = time(13, 47, tzinfo=met) + t4 = time(microsecond=40) + t5 = time(microsecond=40, tzinfo=utc) + + self.assertEqual(t1.tzinfo, est) + self.assertEqual(t2.tzinfo, utc) + self.assertEqual(t3.tzinfo, met) + self.assertIsNone(t4.tzinfo) + self.assertEqual(t5.tzinfo, utc) + + self.assertEqual(t1.utcoffset(), timedelta(minutes=-300)) + self.assertEqual(t2.utcoffset(), timedelta(minutes=0)) + self.assertEqual(t3.utcoffset(), timedelta(minutes=60)) + self.assertIsNone(t4.utcoffset()) + self.assertRaises(TypeError, t1.utcoffset, "no args") + + self.assertEqual(t1.tzname(), "EST") + self.assertEqual(t2.tzname(), "UTC") + self.assertEqual(t3.tzname(), "MET") + self.assertIsNone(t4.tzname()) + self.assertRaises(TypeError, t1.tzname, "no args") + + self.assertEqual(t1.dst(), timedelta(minutes=1)) + self.assertEqual(t2.dst(), timedelta(minutes=-2)) + self.assertEqual(t3.dst(), timedelta(minutes=3)) + self.assertIsNone(t4.dst()) + self.assertRaises(TypeError, t1.dst, "no args") + + self.assertEqual(hash(t1), hash(t2)) + self.assertEqual(hash(t1), hash(t3)) + self.assertEqual(hash(t2), hash(t3)) + + self.assertEqual(t1, t2) + self.assertEqual(t1, t3) + self.assertEqual(t2, t3) + self.assertNotEqual(t4, t5) # mixed tz-aware & naive + self.assertRaises(TypeError, lambda: t4 < t5) # mixed tz-aware & naive + self.assertRaises(TypeError, lambda: t5 < t4) # mixed tz-aware & naive + + self.assertEqual(str(t1), "07:47:00-05:00") + self.assertEqual(str(t2), "12:47:00+00:00") + self.assertEqual(str(t3), "13:47:00+01:00") + self.assertEqual(str(t4), "00:00:00.000040") + self.assertEqual(str(t5), "00:00:00.000040+00:00") + + self.assertEqual(t1.isoformat(), "07:47:00-05:00") + self.assertEqual(t2.isoformat(), "12:47:00+00:00") + self.assertEqual(t3.isoformat(), "13:47:00+01:00") + self.assertEqual(t4.isoformat(), "00:00:00.000040") + self.assertEqual(t5.isoformat(), "00:00:00.000040+00:00") + + d = 'datetime.time' + self.assertEqual(repr(t1), d + "(7, 47, tzinfo=est)") + self.assertEqual(repr(t2), d + "(12, 47, tzinfo=utc)") + self.assertEqual(repr(t3), d + "(13, 47, tzinfo=met)") + self.assertEqual(repr(t4), d + "(0, 0, 0, 40)") + self.assertEqual(repr(t5), d + "(0, 0, 0, 40, tzinfo=utc)") + + self.assertEqual(t1.strftime("%H:%M:%S %%Z=%Z %%z=%z %%:z=%:z"), + "07:47:00 %Z=EST %z=-0500 %:z=-05:00") + self.assertEqual(t2.strftime("%H:%M:%S %Z %z %:z"), "12:47:00 UTC +0000 +00:00") + self.assertEqual(t3.strftime("%H:%M:%S %Z %z %:z"), "13:47:00 MET +0100 +01:00") + + yuck = FixedOffset(-1439, "%z %Z %%z%%Z") + t1 = time(23, 59, tzinfo=yuck) + self.assertEqual(t1.strftime("%H:%M %%Z='%Z' %%z='%z'"), + "23:59 %Z='%z %Z %%z%%Z' %z='-2359'") + + # Check that an invalid tzname result raises an exception. + class Badtzname(tzinfo): + tz = 42 + def tzname(self, dt): return self.tz + t = time(2, 3, 4, tzinfo=Badtzname()) + self.assertEqual(t.strftime("%H:%M:%S"), "02:03:04") + self.assertRaises(TypeError, t.strftime, "%Z") + + # Issue #6697: + Badtzname.tz = '\ud800' + self.assertEqual(t.strftime("%Z"), '\ud800') + + def test_hash_edge_cases(self): + # Offsets that overflow a basic time. + t1 = self.theclass(0, 1, 2, 3, tzinfo=FixedOffset(1439, "")) + t2 = self.theclass(0, 0, 2, 3, tzinfo=FixedOffset(1438, "")) + self.assertEqual(hash(t1), hash(t2)) + + t1 = self.theclass(23, 58, 6, 100, tzinfo=FixedOffset(-1000, "")) + t2 = self.theclass(23, 48, 6, 100, tzinfo=FixedOffset(-1010, "")) + self.assertEqual(hash(t1), hash(t2)) + + def test_pickling(self): + # Try one without a tzinfo. + args = 20, 59, 16, 64**2 + orig = self.theclass(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) + + # Try one with a tzinfo. + tinfo = PicklableFixedOffset(-300, 'cookie') + orig = self.theclass(5, 6, 7, tzinfo=tinfo) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertIsInstance(derived.tzinfo, PicklableFixedOffset) + self.assertEqual(derived.utcoffset(), timedelta(minutes=-300)) + self.assertEqual(derived.tzname(), 'cookie') + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) + + def test_compat_unpickle(self): + tests = [ + b"cdatetime\ntime\n(S'\\x05\\x06\\x07\\x01\\xe2@'\n" + b"ctest.datetimetester\nPicklableFixedOffset\n(tR" + b"(dS'_FixedOffset__offset'\ncdatetime\ntimedelta\n" + b"(I-1\nI68400\nI0\ntRs" + b"S'_FixedOffset__dstoffset'\nNs" + b"S'_FixedOffset__name'\nS'cookie'\nsbtR.", + + b'cdatetime\ntime\n(U\x06\x05\x06\x07\x01\xe2@' + b'ctest.datetimetester\nPicklableFixedOffset\n)R' + b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n' + b'(J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00tR' + b'U\x17_FixedOffset__dstoffsetN' + b'U\x12_FixedOffset__nameU\x06cookieubtR.', + + b'\x80\x02cdatetime\ntime\nU\x06\x05\x06\x07\x01\xe2@' + b'ctest.datetimetester\nPicklableFixedOffset\n)R' + b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n' + b'J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00\x87R' + b'U\x17_FixedOffset__dstoffsetN' + b'U\x12_FixedOffset__nameU\x06cookieub\x86R.', + ] + + tinfo = PicklableFixedOffset(-300, 'cookie') + expected = self.theclass(5, 6, 7, 123456, tzinfo=tinfo) + for data in tests: + for loads in pickle_loads: + derived = loads(data, encoding='latin1') + self.assertEqual(derived, expected, repr(data)) + self.assertIsInstance(derived.tzinfo, PicklableFixedOffset) + self.assertEqual(derived.utcoffset(), timedelta(minutes=-300)) + self.assertEqual(derived.tzname(), 'cookie') + + def test_more_bool(self): + # time is always True. + cls = self.theclass + + t = cls(0, tzinfo=FixedOffset(-300, "")) + self.assertTrue(t) + + t = cls(5, tzinfo=FixedOffset(-300, "")) + self.assertTrue(t) + + t = cls(5, tzinfo=FixedOffset(300, "")) + self.assertTrue(t) + + t = cls(23, 59, tzinfo=FixedOffset(23*60 + 59, "")) + self.assertTrue(t) + + def test_replace(self): + cls = self.theclass + z100 = FixedOffset(100, "+100") + zm200 = FixedOffset(timedelta(minutes=-200), "-200") + args = [1, 2, 3, 4, z100] + base = cls(*args) + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8), + ("tzinfo", zm200)) + for i, (name, newval) in enumerate(changes): + newargs = args[:] + newargs[i] = newval + expected = cls(*newargs) + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) + + # Ensure we can get rid of a tzinfo. + self.assertEqual(base.tzname(), "+100") + base2 = base.replace(tzinfo=None) + self.assertIsNone(base2.tzinfo) + self.assertIsNone(base2.tzname()) + base22 = copy.replace(base, tzinfo=None) + self.assertIsNone(base22.tzinfo) + self.assertIsNone(base22.tzname()) + + # Ensure we can add one. + base3 = base2.replace(tzinfo=z100) + self.assertEqual(base, base3) + self.assertIs(base.tzinfo, base3.tzinfo) + base32 = copy.replace(base22, tzinfo=z100) + self.assertEqual(base, base32) + self.assertIs(base.tzinfo, base32.tzinfo) + + # Out of bounds. + base = cls(1) + self.assertRaises(ValueError, base.replace, hour=24) + self.assertRaises(ValueError, base.replace, minute=-1) + self.assertRaises(ValueError, base.replace, second=100) + self.assertRaises(ValueError, base.replace, microsecond=1000000) + self.assertRaises(ValueError, copy.replace, base, hour=24) + self.assertRaises(ValueError, copy.replace, base, minute=-1) + self.assertRaises(ValueError, copy.replace, base, second=100) + self.assertRaises(ValueError, copy.replace, base, microsecond=1000000) + + def test_mixed_compare(self): + t1 = self.theclass(1, 2, 3) + t2 = self.theclass(1, 2, 3) + self.assertEqual(t1, t2) + t2 = t2.replace(tzinfo=None) + self.assertEqual(t1, t2) + t2 = t2.replace(tzinfo=FixedOffset(None, "")) + self.assertEqual(t1, t2) + t2 = t2.replace(tzinfo=FixedOffset(0, "")) + self.assertNotEqual(t1, t2) + + # In time w/ identical tzinfo objects, utcoffset is ignored. + class Varies(tzinfo): + def __init__(self): + self.offset = timedelta(minutes=22) + def utcoffset(self, t): + self.offset += timedelta(minutes=1) + return self.offset + + v = Varies() + t1 = t2.replace(tzinfo=v) + t2 = t2.replace(tzinfo=v) + self.assertEqual(t1.utcoffset(), timedelta(minutes=23)) + self.assertEqual(t2.utcoffset(), timedelta(minutes=24)) + self.assertEqual(t1, t2) + + # But if they're not identical, it isn't ignored. + t2 = t2.replace(tzinfo=Varies()) + self.assertTrue(t1 < t2) # t1's offset counter still going up + + def test_valuerror_messages(self): + pattern = re.compile( + r"(hour|minute|second|microsecond) must be in \d+\.\.\d+, not \d+" + ) + test_cases = [ + (12, 30, 90, 9999991), # Microsecond out of range + (12, 30, 90, 000000), # Second out of range + (25, 30, 45, 000000), # Hour out of range + (12, 90, 45, 000000), # Minute out of range + ] + for case in test_cases: + with self.subTest(case): + with self.assertRaisesRegex(ValueError, pattern): + self.theclass(*case) + + def test_fromisoformat(self): + time_examples = [ + (0, 0, 0, 0), + (23, 59, 59, 999999), + ] + + hh = (9, 12, 20) + mm = (5, 30) + ss = (4, 45) + usec = (0, 245000, 678901) + + time_examples += list(itertools.product(hh, mm, ss, usec)) + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=2)), + timezone(timedelta(hours=6, minutes=27))] + + for ttup in time_examples: + for tzi in tzinfos: + t = self.theclass(*ttup, tzinfo=tzi) + tstr = t.isoformat() + + with self.subTest(tstr=tstr): + t_rt = self.theclass.fromisoformat(tstr) + self.assertEqual(t, t_rt) + + def test_fromisoformat_timezone(self): + base_time = self.theclass(12, 30, 45, 217456) + + tzoffsets = [ + timedelta(hours=5), timedelta(hours=2), + timedelta(hours=6, minutes=27), + timedelta(hours=12, minutes=32, seconds=30), + timedelta(hours=2, minutes=4, seconds=9, microseconds=123456) + ] + + tzoffsets += [-1 * td for td in tzoffsets] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=0))] + + tzinfos += [timezone(td) for td in tzoffsets] + + for tzi in tzinfos: + t = base_time.replace(tzinfo=tzi) + tstr = t.isoformat() + + with self.subTest(tstr=tstr): + t_rt = self.theclass.fromisoformat(tstr) + assert t == t_rt + + def test_fromisoformat_timespecs(self): + time_bases = [ + (8, 17, 45, 123456), + (8, 17, 45, 0) + ] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=-5)), + timezone(timedelta(hours=2)), + timezone(timedelta(hours=6, minutes=27))] + + timespecs = ['hours', 'minutes', 'seconds', + 'milliseconds', 'microseconds'] + + for ip, ts in enumerate(timespecs): + for tzi in tzinfos: + for t_tuple in time_bases: + if ts == 'milliseconds': + new_microseconds = 1000 * (t_tuple[-1] // 1000) + t_tuple = t_tuple[0:-1] + (new_microseconds,) + + t = self.theclass(*(t_tuple[0:(1 + ip)]), tzinfo=tzi) + tstr = t.isoformat(timespec=ts) + with self.subTest(tstr=tstr): + t_rt = self.theclass.fromisoformat(tstr) + self.assertEqual(t, t_rt) + + def test_fromisoformat_fractions(self): + strs = [ + ('12:30:45.1', (12, 30, 45, 100000)), + ('12:30:45.12', (12, 30, 45, 120000)), + ('12:30:45.123', (12, 30, 45, 123000)), + ('12:30:45.1234', (12, 30, 45, 123400)), + ('12:30:45.12345', (12, 30, 45, 123450)), + ('12:30:45.123456', (12, 30, 45, 123456)), + ('12:30:45.1234567', (12, 30, 45, 123456)), + ('12:30:45.12345678', (12, 30, 45, 123456)), + ] + + for time_str, time_comps in strs: + expected = self.theclass(*time_comps) + actual = self.theclass.fromisoformat(time_str) + + self.assertEqual(actual, expected) + + def test_fromisoformat_time_examples(self): + examples = [ + ('0000', self.theclass(0, 0)), + ('00:00', self.theclass(0, 0)), + ('000000', self.theclass(0, 0)), + ('00:00:00', self.theclass(0, 0)), + ('000000.0', self.theclass(0, 0)), + ('00:00:00.0', self.theclass(0, 0)), + ('000000.000', self.theclass(0, 0)), + ('00:00:00.000', self.theclass(0, 0)), + ('000000.000000', self.theclass(0, 0)), + ('00:00:00.000000', self.theclass(0, 0)), + ('00:00:00,100000', self.theclass(0, 0, 0, 100000)), + ('1200', self.theclass(12, 0)), + ('12:00', self.theclass(12, 0)), + ('120000', self.theclass(12, 0)), + ('12:00:00', self.theclass(12, 0)), + ('120000.0', self.theclass(12, 0)), + ('12:00:00.0', self.theclass(12, 0)), + ('120000.000', self.theclass(12, 0)), + ('12:00:00.000', self.theclass(12, 0)), + ('120000.000000', self.theclass(12, 0)), + ('12:00:00.000000', self.theclass(12, 0)), + ('2359', self.theclass(23, 59)), + ('23:59', self.theclass(23, 59)), + ('235959', self.theclass(23, 59, 59)), + ('23:59:59', self.theclass(23, 59, 59)), + ('235959.9', self.theclass(23, 59, 59, 900000)), + ('23:59:59.9', self.theclass(23, 59, 59, 900000)), + ('235959.999', self.theclass(23, 59, 59, 999000)), + ('23:59:59.999', self.theclass(23, 59, 59, 999000)), + ('235959.999999', self.theclass(23, 59, 59, 999999)), + ('23:59:59.999999', self.theclass(23, 59, 59, 999999)), + ('00:00:00Z', self.theclass(0, 0, tzinfo=timezone.utc)), + ('12:00:00+0000', self.theclass(12, 0, tzinfo=timezone.utc)), + ('12:00:00+00:00', self.theclass(12, 0, tzinfo=timezone.utc)), + ('00:00:00+05', + self.theclass(0, 0, tzinfo=timezone(timedelta(hours=5)))), + ('00:00:00+05:30', + self.theclass(0, 0, tzinfo=timezone(timedelta(hours=5, minutes=30)))), + ('12:00:00-05:00', + self.theclass(12, 0, tzinfo=timezone(timedelta(hours=-5)))), + ('12:00:00-0500', + self.theclass(12, 0, tzinfo=timezone(timedelta(hours=-5)))), + ('00:00:00,000-23:59:59.999999', + self.theclass(0, 0, tzinfo=timezone(-timedelta(hours=23, minutes=59, seconds=59, microseconds=999999)))), + ] + + for input_str, expected in examples: + with self.subTest(input_str=input_str): + actual = self.theclass.fromisoformat(input_str) + self.assertEqual(actual, expected) + + def test_fromisoformat_fails(self): + bad_strs = [ + '', # Empty string + '12\ud80000', # Invalid separator - surrogate char + '12:', # Ends on a separator + '12:30:', # Ends on a separator + '12:30:15.', # Ends on a separator + '1', # Incomplete hours + '12:3', # Incomplete minutes + '12:30:1', # Incomplete seconds + '1a:30:45.334034', # Invalid character in hours + '12:a0:45.334034', # Invalid character in minutes + '12:30:a5.334034', # Invalid character in seconds + '12:30:45.123456+24:30', # Invalid time zone offset + '12:30:45.123456-24:30', # Invalid negative offset + '12：30：45', # Uses full-width unicode colons + '12:30:45.123456a', # Non-numeric data after 6 components + '12:30:45.123456789a', # Non-numeric data after 9 components + '12:30:45․123456', # Uses \u2024 in place of decimal point + '12:30:45a', # Extra at tend of basic time + '12:30:45.123a', # Extra at end of millisecond time + '12:30:45.123456a', # Extra at end of microsecond time + '12:30:45.123456-', # Extra at end of microsecond time + '12:30:45.123456+', # Extra at end of microsecond time + '12:30:45.123456+12:00:30a', # Extra at end of full time + '12.5', # Decimal mark at end of hour + '12:30,5', # Decimal mark at end of minute + '12:30:45.123456Z12:00', # Extra time zone info after Z + '12:30:45:334034', # Invalid microsecond separator + '12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) + '12:30:45.400 ', # Trailing space (gh-130959) + '12:30:45. 400', # Space before fraction (gh-130959) + ] + + for bad_str in bad_strs: + with self.subTest(bad_str=bad_str): + with self.assertRaises(ValueError): + self.theclass.fromisoformat(bad_str) + + def test_fromisoformat_fails_typeerror(self): + # Test the fromisoformat fails when passed the wrong type + bad_types = [b'12:30:45', None, io.StringIO('12:30:45')] + + for bad_type in bad_types: + with self.assertRaises(TypeError): + self.theclass.fromisoformat(bad_type) + + def test_fromisoformat_subclass(self): + class TimeSubclass(self.theclass): + pass + + tsc = TimeSubclass(12, 14, 45, 203745, tzinfo=timezone.utc) + tsc_rt = TimeSubclass.fromisoformat(tsc.isoformat()) + + self.assertEqual(tsc, tsc_rt) + self.assertIsInstance(tsc_rt, TimeSubclass) + + def test_subclass_timetz(self): + + class C(self.theclass): + theAnswer = 42 + + def __new__(cls, *args, **kws): + temp = kws.copy() + extra = temp.pop('extra') + result = self.theclass.__new__(cls, *args, **temp) + result.extra = extra + return result + + def newmeth(self, start): + return start + self.hour + self.second + + args = 4, 5, 6, 500, FixedOffset(-300, "EST", 1) + + dt1 = self.theclass(*args) + dt2 = C(*args, **{'extra': 7}) + + self.assertEqual(dt2.__class__, C) + self.assertEqual(dt2.theAnswer, 42) + self.assertEqual(dt2.extra, 7) + self.assertEqual(dt1.utcoffset(), dt2.utcoffset()) + self.assertEqual(dt2.newmeth(-7), dt1.hour + dt1.second - 7) + + +# Testing datetime objects with a non-None tzinfo. + +class TestDateTimeTZ(TestDateTime, TZInfoBase, unittest.TestCase): + theclass = datetime + + def test_trivial(self): + dt = self.theclass(1, 2, 3, 4, 5, 6, 7) + self.assertEqual(dt.year, 1) + self.assertEqual(dt.month, 2) + self.assertEqual(dt.day, 3) + self.assertEqual(dt.hour, 4) + self.assertEqual(dt.minute, 5) + self.assertEqual(dt.second, 6) + self.assertEqual(dt.microsecond, 7) + self.assertEqual(dt.tzinfo, None) + + def test_even_more_compare(self): + # The test_compare() and test_more_compare() inherited from TestDate + # and TestDateTime covered non-tzinfo cases. + + # Smallest possible after UTC adjustment. + t1 = self.theclass(1, 1, 1, tzinfo=FixedOffset(1439, "")) + # Largest possible after UTC adjustment. + t2 = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999999, + tzinfo=FixedOffset(-1439, "")) + + # Make sure those compare correctly, and w/o overflow. + self.assertTrue(t1 < t2) + self.assertTrue(t1 != t2) + self.assertTrue(t2 > t1) + + self.assertEqual(t1, t1) + self.assertEqual(t2, t2) + + # Equal after adjustment. + t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(1, "")) + t2 = self.theclass(2, 1, 1, 3, 13, tzinfo=FixedOffset(3*60+13+2, "")) + self.assertEqual(t1, t2) + + # Change t1 not to subtract a minute, and t1 should be larger. + t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(0, "")) + self.assertTrue(t1 > t2) + + # Change t1 to subtract 2 minutes, and t1 should be smaller. + t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(2, "")) + self.assertTrue(t1 < t2) + + # Back to the original t1, but make seconds resolve it. + t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(1, ""), + second=1) + self.assertTrue(t1 > t2) + + # Likewise, but make microseconds resolve it. + t1 = self.theclass(1, 12, 31, 23, 59, tzinfo=FixedOffset(1, ""), + microsecond=1) + self.assertTrue(t1 > t2) + + # Make t2 naive and it should differ. + t2 = self.theclass.min + self.assertNotEqual(t1, t2) + self.assertEqual(t2, t2) + # and > comparison should fail + with self.assertRaises(TypeError): + t1 > t2 + + # It's also naive if it has tzinfo but tzinfo.utcoffset() is None. + class Naive(tzinfo): + def utcoffset(self, dt): return None + t2 = self.theclass(5, 6, 7, tzinfo=Naive()) + self.assertNotEqual(t1, t2) + self.assertEqual(t2, t2) + + # OTOH, it's OK to compare two of these mixing the two ways of being + # naive. + t1 = self.theclass(5, 6, 7) + self.assertEqual(t1, t2) + + # Try a bogus uctoffset. + class Bogus(tzinfo): + def utcoffset(self, dt): + return timedelta(minutes=1440) # out of bounds + t1 = self.theclass(2, 2, 2, tzinfo=Bogus()) + t2 = self.theclass(2, 2, 2, tzinfo=FixedOffset(0, "")) + self.assertRaises(ValueError, lambda: t1 == t2) + + def test_pickling(self): + # Try one without a tzinfo. + args = 6, 7, 23, 20, 59, 1, 64**2 + orig = self.theclass(*args) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) + + # Try one with a tzinfo. + tinfo = PicklableFixedOffset(-300, 'cookie') + orig = self.theclass(*args, **{'tzinfo': tinfo}) + derived = self.theclass(1, 1, 1, tzinfo=FixedOffset(0, "", 0)) + for pickler, unpickler, proto in pickle_choices: + green = pickler.dumps(orig, proto) + derived = unpickler.loads(green) + self.assertEqual(orig, derived) + self.assertIsInstance(derived.tzinfo, PicklableFixedOffset) + self.assertEqual(derived.utcoffset(), timedelta(minutes=-300)) + self.assertEqual(derived.tzname(), 'cookie') + self.assertEqual(orig.__reduce__(), orig.__reduce_ex__(2)) + + def test_compat_unpickle(self): + tests = [ + b'cdatetime\ndatetime\n' + b"(S'\\x07\\xdf\\x0b\\x1b\\x14;\\x01\\x01\\xe2@'\n" + b'ctest.datetimetester\nPicklableFixedOffset\n(tR' + b"(dS'_FixedOffset__offset'\ncdatetime\ntimedelta\n" + b'(I-1\nI68400\nI0\ntRs' + b"S'_FixedOffset__dstoffset'\nNs" + b"S'_FixedOffset__name'\nS'cookie'\nsbtR.", + + b'cdatetime\ndatetime\n' + b'(U\n\x07\xdf\x0b\x1b\x14;\x01\x01\xe2@' + b'ctest.datetimetester\nPicklableFixedOffset\n)R' + b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n' + b'(J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00tR' + b'U\x17_FixedOffset__dstoffsetN' + b'U\x12_FixedOffset__nameU\x06cookieubtR.', + + b'\x80\x02cdatetime\ndatetime\n' + b'U\n\x07\xdf\x0b\x1b\x14;\x01\x01\xe2@' + b'ctest.datetimetester\nPicklableFixedOffset\n)R' + b'}(U\x14_FixedOffset__offsetcdatetime\ntimedelta\n' + b'J\xff\xff\xff\xffJ0\x0b\x01\x00K\x00\x87R' + b'U\x17_FixedOffset__dstoffsetN' + b'U\x12_FixedOffset__nameU\x06cookieub\x86R.', + ] + args = 2015, 11, 27, 20, 59, 1, 123456 + tinfo = PicklableFixedOffset(-300, 'cookie') + expected = self.theclass(*args, **{'tzinfo': tinfo}) + for data in tests: + for loads in pickle_loads: + derived = loads(data, encoding='latin1') + self.assertEqual(derived, expected) + self.assertIsInstance(derived.tzinfo, PicklableFixedOffset) + self.assertEqual(derived.utcoffset(), timedelta(minutes=-300)) + self.assertEqual(derived.tzname(), 'cookie') + + def test_extreme_hashes(self): + # If an attempt is made to hash these via subtracting the offset + # then hashing a datetime object, OverflowError results. The + # Python implementation used to blow up here. + t = self.theclass(1, 1, 1, tzinfo=FixedOffset(1439, "")) + hash(t) + t = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999999, + tzinfo=FixedOffset(-1439, "")) + hash(t) + + # OTOH, an OOB offset should blow up. + t = self.theclass(5, 5, 5, tzinfo=FixedOffset(-1440, "")) + self.assertRaises(ValueError, hash, t) + + def test_zones(self): + est = FixedOffset(-300, "EST") + utc = FixedOffset(0, "UTC") + met = FixedOffset(60, "MET") + t1 = datetime(2002, 3, 19, 7, 47, tzinfo=est) + t2 = datetime(2002, 3, 19, 12, 47, tzinfo=utc) + t3 = datetime(2002, 3, 19, 13, 47, tzinfo=met) + self.assertEqual(t1.tzinfo, est) + self.assertEqual(t2.tzinfo, utc) + self.assertEqual(t3.tzinfo, met) + self.assertEqual(t1.utcoffset(), timedelta(minutes=-300)) + self.assertEqual(t2.utcoffset(), timedelta(minutes=0)) + self.assertEqual(t3.utcoffset(), timedelta(minutes=60)) + self.assertEqual(t1.tzname(), "EST") + self.assertEqual(t2.tzname(), "UTC") + self.assertEqual(t3.tzname(), "MET") + self.assertEqual(hash(t1), hash(t2)) + self.assertEqual(hash(t1), hash(t3)) + self.assertEqual(hash(t2), hash(t3)) + self.assertEqual(t1, t2) + self.assertEqual(t1, t3) + self.assertEqual(t2, t3) + self.assertEqual(str(t1), "2002-03-19 07:47:00-05:00") + self.assertEqual(str(t2), "2002-03-19 12:47:00+00:00") + self.assertEqual(str(t3), "2002-03-19 13:47:00+01:00") + d = 'datetime.datetime(2002, 3, 19, ' + self.assertEqual(repr(t1), d + "7, 47, tzinfo=est)") + self.assertEqual(repr(t2), d + "12, 47, tzinfo=utc)") + self.assertEqual(repr(t3), d + "13, 47, tzinfo=met)") + + def test_combine(self): + met = FixedOffset(60, "MET") + d = date(2002, 3, 4) + tz = time(18, 45, 3, 1234, tzinfo=met) + dt = datetime.combine(d, tz) + self.assertEqual(dt, datetime(2002, 3, 4, 18, 45, 3, 1234, + tzinfo=met)) + + def test_extract(self): + met = FixedOffset(60, "MET") + dt = self.theclass(2002, 3, 4, 18, 45, 3, 1234, tzinfo=met) + self.assertEqual(dt.date(), date(2002, 3, 4)) + self.assertEqual(dt.time(), time(18, 45, 3, 1234)) + self.assertEqual(dt.timetz(), time(18, 45, 3, 1234, tzinfo=met)) + + def test_tz_aware_arithmetic(self): + now = self.theclass.now() + tz55 = FixedOffset(-330, "west 5:30") + timeaware = now.time().replace(tzinfo=tz55) + nowaware = self.theclass.combine(now.date(), timeaware) + self.assertIs(nowaware.tzinfo, tz55) + self.assertEqual(nowaware.timetz(), timeaware) + + # Can't mix aware and non-aware. + self.assertRaises(TypeError, lambda: now - nowaware) + self.assertRaises(TypeError, lambda: nowaware - now) + + # And adding datetime's doesn't make sense, aware or not. + self.assertRaises(TypeError, lambda: now + nowaware) + self.assertRaises(TypeError, lambda: nowaware + now) + self.assertRaises(TypeError, lambda: nowaware + nowaware) + + # Subtracting should yield 0. + self.assertEqual(now - now, timedelta(0)) + self.assertEqual(nowaware - nowaware, timedelta(0)) + + # Adding a delta should preserve tzinfo. + delta = timedelta(weeks=1, minutes=12, microseconds=5678) + nowawareplus = nowaware + delta + self.assertIs(nowaware.tzinfo, tz55) + nowawareplus2 = delta + nowaware + self.assertIs(nowawareplus2.tzinfo, tz55) + self.assertEqual(nowawareplus, nowawareplus2) + + # that - delta should be what we started with, and that - what we + # started with should be delta. + diff = nowawareplus - delta + self.assertIs(diff.tzinfo, tz55) + self.assertEqual(nowaware, diff) + self.assertRaises(TypeError, lambda: delta - nowawareplus) + self.assertEqual(nowawareplus - nowaware, delta) + + # Make up a random timezone. + tzr = FixedOffset(random.randrange(-1439, 1440), "randomtimezone") + # Attach it to nowawareplus. + nowawareplus = nowawareplus.replace(tzinfo=tzr) + self.assertIs(nowawareplus.tzinfo, tzr) + # Make sure the difference takes the timezone adjustments into account. + got = nowaware - nowawareplus + # Expected: (nowaware base - nowaware offset) - + # (nowawareplus base - nowawareplus offset) = + # (nowaware base - nowawareplus base) + + # (nowawareplus offset - nowaware offset) = + # -delta + nowawareplus offset - nowaware offset + expected = nowawareplus.utcoffset() - nowaware.utcoffset() - delta + self.assertEqual(got, expected) + + # Try max possible difference. + min = self.theclass(1, 1, 1, tzinfo=FixedOffset(1439, "min")) + max = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999999, + tzinfo=FixedOffset(-1439, "max")) + maxdiff = max - min + self.assertEqual(maxdiff, self.theclass.max - self.theclass.min + + timedelta(minutes=2*1439)) + # Different tzinfo, but the same offset + tza = timezone(HOUR, 'A') + tzb = timezone(HOUR, 'B') + delta = min.replace(tzinfo=tza) - max.replace(tzinfo=tzb) + self.assertEqual(delta, self.theclass.min - self.theclass.max) + + def test_tzinfo_now(self): + meth = self.theclass.now + # Ensure it doesn't require tzinfo (i.e., that this doesn't blow up). + base = meth() + # Try with and without naming the keyword. + off42 = FixedOffset(42, "42") + another = meth(off42) + again = meth(tz=off42) + self.assertIs(another.tzinfo, again.tzinfo) + self.assertEqual(another.utcoffset(), timedelta(minutes=42)) + # Bad argument with and w/o naming the keyword. + self.assertRaises(TypeError, meth, 16) + self.assertRaises(TypeError, meth, tzinfo=16) + # Bad keyword name. + self.assertRaises(TypeError, meth, tinfo=off42) + # Too many args. + self.assertRaises(TypeError, meth, off42, off42) + + # We don't know which time zone we're in, and don't have a tzinfo + # class to represent it, so seeing whether a tz argument actually + # does a conversion is tricky. + utc = FixedOffset(0, "utc", 0) + for weirdtz in [FixedOffset(timedelta(hours=15, minutes=58), "weirdtz", 0), + timezone(timedelta(hours=15, minutes=58), "weirdtz"),]: + for dummy in range(3): + now = datetime.now(weirdtz) + self.assertIs(now.tzinfo, weirdtz) + with self.assertWarns(DeprecationWarning): + utcnow = datetime.utcnow().replace(tzinfo=utc) + now2 = utcnow.astimezone(weirdtz) + if abs(now - now2) < timedelta(seconds=30): + break + # Else the code is broken, or more than 30 seconds passed between + # calls; assuming the latter, just try again. + else: + # Three strikes and we're out. + self.fail("utcnow(), now(tz), or astimezone() may be broken") + + def test_tzinfo_fromtimestamp(self): + import time + meth = self.theclass.fromtimestamp + ts = time.time() + # Ensure it doesn't require tzinfo (i.e., that this doesn't blow up). + base = meth(ts) + # Try with and without naming the keyword. + off42 = FixedOffset(42, "42") + another = meth(ts, off42) + again = meth(ts, tz=off42) + self.assertIs(another.tzinfo, again.tzinfo) + self.assertEqual(another.utcoffset(), timedelta(minutes=42)) + # Bad argument with and w/o naming the keyword. + self.assertRaises(TypeError, meth, ts, 16) + self.assertRaises(TypeError, meth, ts, tzinfo=16) + # Bad keyword name. + self.assertRaises(TypeError, meth, ts, tinfo=off42) + # Too many args. + self.assertRaises(TypeError, meth, ts, off42, off42) + # Too few args. + self.assertRaises(TypeError, meth) + + # Try to make sure tz= actually does some conversion. + timestamp = 1000000000 + with self.assertWarns(DeprecationWarning): + utcdatetime = datetime.utcfromtimestamp(timestamp) + # In POSIX (epoch 1970), that's 2001-09-09 01:46:40 UTC, give or take. + # But on some flavor of Mac, it's nowhere near that. So we can't have + # any idea here what time that actually is, we can only test that + # relative changes match. + utcoffset = timedelta(hours=-15, minutes=39) # arbitrary, but not zero + tz = FixedOffset(utcoffset, "tz", 0) + expected = utcdatetime + utcoffset + got = datetime.fromtimestamp(timestamp, tz) + self.assertEqual(expected, got.replace(tzinfo=None)) + + def test_tzinfo_utcnow(self): + meth = self.theclass.utcnow + # Ensure it doesn't require tzinfo (i.e., that this doesn't blow up). + with self.assertWarns(DeprecationWarning): + base = meth() + # Try with and without naming the keyword; for whatever reason, + # utcnow() doesn't accept a tzinfo argument. + off42 = FixedOffset(42, "42") + self.assertRaises(TypeError, meth, off42) + self.assertRaises(TypeError, meth, tzinfo=off42) + + def test_tzinfo_utcfromtimestamp(self): + import time + meth = self.theclass.utcfromtimestamp + ts = time.time() + # Ensure it doesn't require tzinfo (i.e., that this doesn't blow up). + with self.assertWarns(DeprecationWarning): + base = meth(ts) + # Try with and without naming the keyword; for whatever reason, + # utcfromtimestamp() doesn't accept a tzinfo argument. + off42 = FixedOffset(42, "42") + with warnings.catch_warnings(category=DeprecationWarning): + warnings.simplefilter("ignore", category=DeprecationWarning) + self.assertRaises(TypeError, meth, ts, off42) + self.assertRaises(TypeError, meth, ts, tzinfo=off42) + + def test_tzinfo_timetuple(self): + # TestDateTime tested most of this. datetime adds a twist to the + # DST flag. + class DST(tzinfo): + def __init__(self, dstvalue): + if isinstance(dstvalue, int): + dstvalue = timedelta(minutes=dstvalue) + self.dstvalue = dstvalue + def dst(self, dt): + return self.dstvalue + + cls = self.theclass + for dstvalue, flag in (-33, 1), (33, 1), (0, 0), (None, -1): + d = cls(1, 1, 1, 10, 20, 30, 40, tzinfo=DST(dstvalue)) + t = d.timetuple() + self.assertEqual(1, t.tm_year) + self.assertEqual(1, t.tm_mon) + self.assertEqual(1, t.tm_mday) + self.assertEqual(10, t.tm_hour) + self.assertEqual(20, t.tm_min) + self.assertEqual(30, t.tm_sec) + self.assertEqual(0, t.tm_wday) + self.assertEqual(1, t.tm_yday) + self.assertEqual(flag, t.tm_isdst) + + # dst() returns wrong type. + self.assertRaises(TypeError, cls(1, 1, 1, tzinfo=DST("x")).timetuple) + + # dst() at the edge. + self.assertEqual(cls(1,1,1, tzinfo=DST(1439)).timetuple().tm_isdst, 1) + self.assertEqual(cls(1,1,1, tzinfo=DST(-1439)).timetuple().tm_isdst, 1) + + # dst() out of range. + self.assertRaises(ValueError, cls(1,1,1, tzinfo=DST(1440)).timetuple) + self.assertRaises(ValueError, cls(1,1,1, tzinfo=DST(-1440)).timetuple) + + def test_utctimetuple(self): + class DST(tzinfo): + def __init__(self, dstvalue=0): + if isinstance(dstvalue, int): + dstvalue = timedelta(minutes=dstvalue) + self.dstvalue = dstvalue + def dst(self, dt): + return self.dstvalue + + cls = self.theclass + # This can't work: DST didn't implement utcoffset. + self.assertRaises(NotImplementedError, + cls(1, 1, 1, tzinfo=DST(0)).utcoffset) + + class UOFS(DST): + def __init__(self, uofs, dofs=None): + DST.__init__(self, dofs) + self.uofs = timedelta(minutes=uofs) + def utcoffset(self, dt): + return self.uofs + + for dstvalue in -33, 33, 0, None: + d = cls(1, 2, 3, 10, 20, 30, 40, tzinfo=UOFS(-53, dstvalue)) + t = d.utctimetuple() + self.assertEqual(d.year, t.tm_year) + self.assertEqual(d.month, t.tm_mon) + self.assertEqual(d.day, t.tm_mday) + self.assertEqual(11, t.tm_hour) # 20mm + 53mm = 1hn + 13mm + self.assertEqual(13, t.tm_min) + self.assertEqual(d.second, t.tm_sec) + self.assertEqual(d.weekday(), t.tm_wday) + self.assertEqual(d.toordinal() - date(1, 1, 1).toordinal() + 1, + t.tm_yday) + # Ensure tm_isdst is 0 regardless of what dst() says: DST + # is never in effect for a UTC time. + self.assertEqual(0, t.tm_isdst) + + # For naive datetime, utctimetuple == timetuple except for isdst + d = cls(1, 2, 3, 10, 20, 30, 40) + t = d.utctimetuple() + self.assertEqual(t[:-1], d.timetuple()[:-1]) + self.assertEqual(0, t.tm_isdst) + # Same if utcoffset is None + class NOFS(DST): + def utcoffset(self, dt): + return None + d = cls(1, 2, 3, 10, 20, 30, 40, tzinfo=NOFS()) + t = d.utctimetuple() + self.assertEqual(t[:-1], d.timetuple()[:-1]) + self.assertEqual(0, t.tm_isdst) + # Check that bad tzinfo is detected + class BOFS(DST): + def utcoffset(self, dt): + return "EST" + d = cls(1, 2, 3, 10, 20, 30, 40, tzinfo=BOFS()) + self.assertRaises(TypeError, d.utctimetuple) + + # Check that utctimetuple() is the same as + # astimezone(utc).timetuple() + d = cls(2010, 11, 13, 14, 15, 16, 171819) + for tz in [timezone.min, timezone.utc, timezone.max]: + dtz = d.replace(tzinfo=tz) + self.assertEqual(dtz.utctimetuple()[:-1], + dtz.astimezone(timezone.utc).timetuple()[:-1]) + # At the edges, UTC adjustment can produce years out-of-range + # for a datetime object. Ensure that an OverflowError is + # raised. + tiny = cls(MINYEAR, 1, 1, 0, 0, 37, tzinfo=UOFS(1439)) + # That goes back 1 minute less than a full day. + self.assertRaises(OverflowError, tiny.utctimetuple) + + huge = cls(MAXYEAR, 12, 31, 23, 59, 37, 999999, tzinfo=UOFS(-1439)) + # That goes forward 1 minute less than a full day. + self.assertRaises(OverflowError, huge.utctimetuple) + # More overflow cases + tiny = cls.min.replace(tzinfo=timezone(MINUTE)) + self.assertRaises(OverflowError, tiny.utctimetuple) + huge = cls.max.replace(tzinfo=timezone(-MINUTE)) + self.assertRaises(OverflowError, huge.utctimetuple) + + def test_tzinfo_isoformat(self): + zero = FixedOffset(0, "+00:00") + plus = FixedOffset(220, "+03:40") + minus = FixedOffset(-231, "-03:51") + unknown = FixedOffset(None, "") + + cls = self.theclass + datestr = '0001-02-03' + for ofs in None, zero, plus, minus, unknown: + for us in 0, 987001: + d = cls(1, 2, 3, 4, 5, 59, us, tzinfo=ofs) + timestr = '04:05:59' + (us and '.987001' or '') + ofsstr = ofs is not None and d.tzname() or '' + tailstr = timestr + ofsstr + iso = d.isoformat() + self.assertEqual(iso, datestr + 'T' + tailstr) + self.assertEqual(iso, d.isoformat('T')) + self.assertEqual(d.isoformat('k'), datestr + 'k' + tailstr) + self.assertEqual(d.isoformat('\u1234'), datestr + '\u1234' + tailstr) + self.assertEqual(str(d), datestr + ' ' + tailstr) + + def test_replace(self): + cls = self.theclass + z100 = FixedOffset(100, "+100") + zm200 = FixedOffset(timedelta(minutes=-200), "-200") + args = [1, 2, 3, 4, 5, 6, 7, z100] + base = cls(*args) + self.assertEqual(base.replace(), base) + self.assertEqual(copy.replace(base), base) + + changes = (("year", 2), + ("month", 3), + ("day", 4), + ("hour", 5), + ("minute", 6), + ("second", 7), + ("microsecond", 8), + ("tzinfo", zm200)) + for i, (name, newval) in enumerate(changes): + newargs = args[:] + newargs[i] = newval + expected = cls(*newargs) + self.assertEqual(base.replace(**{name: newval}), expected) + self.assertEqual(copy.replace(base, **{name: newval}), expected) + + # Ensure we can get rid of a tzinfo. + self.assertEqual(base.tzname(), "+100") + base2 = base.replace(tzinfo=None) + self.assertIsNone(base2.tzinfo) + self.assertIsNone(base2.tzname()) + base22 = copy.replace(base, tzinfo=None) + self.assertIsNone(base22.tzinfo) + self.assertIsNone(base22.tzname()) + + # Ensure we can add one. + base3 = base2.replace(tzinfo=z100) + self.assertEqual(base, base3) + self.assertIs(base.tzinfo, base3.tzinfo) + base32 = copy.replace(base22, tzinfo=z100) + self.assertEqual(base, base32) + self.assertIs(base.tzinfo, base32.tzinfo) + + # Out of bounds. + base = cls(2000, 2, 29) + self.assertRaises(ValueError, base.replace, year=2001) + self.assertRaises(ValueError, copy.replace, base, year=2001) + + def test_more_astimezone(self): + # The inherited test_astimezone covered some trivial and error cases. + fnone = FixedOffset(None, "None") + f44m = FixedOffset(44, "44") + fm5h = FixedOffset(-timedelta(hours=5), "m300") + + dt = self.theclass.now(tz=f44m) + self.assertIs(dt.tzinfo, f44m) + # Replacing with degenerate tzinfo raises an exception. + self.assertRaises(ValueError, dt.astimezone, fnone) + # Replacing with same tzinfo makes no change. + x = dt.astimezone(dt.tzinfo) + self.assertIs(x.tzinfo, f44m) + self.assertEqual(x.date(), dt.date()) + self.assertEqual(x.time(), dt.time()) + + # Replacing with different tzinfo does adjust. + got = dt.astimezone(fm5h) + self.assertIs(got.tzinfo, fm5h) + self.assertEqual(got.utcoffset(), timedelta(hours=-5)) + expected = dt - dt.utcoffset() # in effect, convert to UTC + expected += fm5h.utcoffset(dt) # and from there to local time + expected = expected.replace(tzinfo=fm5h) # and attach new tzinfo + self.assertEqual(got.date(), expected.date()) + self.assertEqual(got.time(), expected.time()) + self.assertEqual(got.timetz(), expected.timetz()) + self.assertIs(got.tzinfo, expected.tzinfo) + self.assertEqual(got, expected) + + @support.run_with_tz('UTC') + def test_astimezone_default_utc(self): + dt = self.theclass.now(timezone.utc) + self.assertEqual(dt.astimezone(None), dt) + self.assertEqual(dt.astimezone(), dt) + + # Note that offset in TZ variable has the opposite sign to that + # produced by %z directive. + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_astimezone_default_eastern(self): + dt = self.theclass(2012, 11, 4, 6, 30, tzinfo=timezone.utc) + local = dt.astimezone() + self.assertEqual(dt, local) + self.assertEqual(local.strftime("%z %Z"), "-0500 EST") + dt = self.theclass(2012, 11, 4, 5, 30, tzinfo=timezone.utc) + local = dt.astimezone() + self.assertEqual(dt, local) + self.assertEqual(local.strftime("%z %Z"), "-0400 EDT") + + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_astimezone_default_near_fold(self): + # Issue #26616. + u = datetime(2015, 11, 1, 5, tzinfo=timezone.utc) + t = u.astimezone() + s = t.astimezone() + self.assertEqual(t.tzinfo, s.tzinfo) + + def test_aware_subtract(self): + cls = self.theclass + + # Ensure that utcoffset() is ignored when the operands have the + # same tzinfo member. + class OperandDependentOffset(tzinfo): + def utcoffset(self, t): + if t.minute < 10: + # d0 and d1 equal after adjustment + return timedelta(minutes=t.minute) + else: + # d2 off in the weeds + return timedelta(minutes=59) + + base = cls(8, 9, 10, 11, 12, 13, 14, tzinfo=OperandDependentOffset()) + d0 = base.replace(minute=3) + d1 = base.replace(minute=9) + d2 = base.replace(minute=11) + for x in d0, d1, d2: + for y in d0, d1, d2: + got = x - y + expected = timedelta(minutes=x.minute - y.minute) + self.assertEqual(got, expected) + + # OTOH, if the tzinfo members are distinct, utcoffsets aren't + # ignored. + base = cls(8, 9, 10, 11, 12, 13, 14) + d0 = base.replace(minute=3, tzinfo=OperandDependentOffset()) + d1 = base.replace(minute=9, tzinfo=OperandDependentOffset()) + d2 = base.replace(minute=11, tzinfo=OperandDependentOffset()) + for x in d0, d1, d2: + for y in d0, d1, d2: + got = x - y + if (x is d0 or x is d1) and (y is d0 or y is d1): + expected = timedelta(0) + elif x is y is d2: + expected = timedelta(0) + elif x is d2: + expected = timedelta(minutes=(11-59)-0) + else: + assert y is d2 + expected = timedelta(minutes=0-(11-59)) + self.assertEqual(got, expected) + + def test_mixed_compare(self): + t1 = datetime(1, 2, 3, 4, 5, 6, 7) + t2 = datetime(1, 2, 3, 4, 5, 6, 7) + self.assertEqual(t1, t2) + t2 = t2.replace(tzinfo=None) + self.assertEqual(t1, t2) + t2 = t2.replace(tzinfo=FixedOffset(None, "")) + self.assertEqual(t1, t2) + t2 = t2.replace(tzinfo=FixedOffset(0, "")) + self.assertNotEqual(t1, t2) + + # In datetime w/ identical tzinfo objects, utcoffset is ignored. + class Varies(tzinfo): + def __init__(self): + self.offset = timedelta(minutes=22) + def utcoffset(self, t): + self.offset += timedelta(minutes=1) + return self.offset + + v = Varies() + t1 = t2.replace(tzinfo=v) + t2 = t2.replace(tzinfo=v) + self.assertEqual(t1.utcoffset(), timedelta(minutes=23)) + self.assertEqual(t2.utcoffset(), timedelta(minutes=24)) + self.assertEqual(t1, t2) + + # But if they're not identical, it isn't ignored. + t2 = t2.replace(tzinfo=Varies()) + self.assertTrue(t1 < t2) # t1's offset counter still going up + + def test_subclass_datetimetz(self): + + class C(self.theclass): + theAnswer = 42 + + def __new__(cls, *args, **kws): + temp = kws.copy() + extra = temp.pop('extra') + result = self.theclass.__new__(cls, *args, **temp) + result.extra = extra + return result + + def newmeth(self, start): + return start + self.hour + self.year + + args = 2002, 12, 31, 4, 5, 6, 500, FixedOffset(-300, "EST", 1) + + dt1 = self.theclass(*args) + dt2 = C(*args, **{'extra': 7}) + + self.assertEqual(dt2.__class__, C) + self.assertEqual(dt2.theAnswer, 42) + self.assertEqual(dt2.extra, 7) + self.assertEqual(dt1.utcoffset(), dt2.utcoffset()) + self.assertEqual(dt2.newmeth(-7), dt1.hour + dt1.year - 7) + +# Pain to set up DST-aware tzinfo classes. + +def first_sunday_on_or_after(dt): + days_to_go = 6 - dt.weekday() + if days_to_go: + dt += timedelta(days_to_go) + return dt + +ZERO = timedelta(0) +MINUTE = timedelta(minutes=1) +HOUR = timedelta(hours=1) +DAY = timedelta(days=1) +# In the US, DST starts at 2am (standard time) on the first Sunday in April. +DSTSTART = datetime(1, 4, 1, 2) +# and ends at 2am (DST time; 1am standard time) on the last Sunday of Oct, +# which is the first Sunday on or after Oct 25. Because we view 1:MM as +# being standard time on that day, there is no spelling in local time of +# the last hour of DST (that's 1:MM DST, but 1:MM is taken as standard time). +DSTEND = datetime(1, 10, 25, 1) + +class USTimeZone(tzinfo): + + def __init__(self, hours, reprname, stdname, dstname): + self.stdoffset = timedelta(hours=hours) + self.reprname = reprname + self.stdname = stdname + self.dstname = dstname + + def __repr__(self): + return self.reprname + + def tzname(self, dt): + if self.dst(dt): + return self.dstname + else: + return self.stdname + + def utcoffset(self, dt): + return self.stdoffset + self.dst(dt) + + def dst(self, dt): + if dt is None or dt.tzinfo is None: + # An exception instead may be sensible here, in one or more of + # the cases. + return ZERO + assert dt.tzinfo is self + + # Find first Sunday in April. + start = first_sunday_on_or_after(DSTSTART.replace(year=dt.year)) + assert start.weekday() == 6 and start.month == 4 and start.day <= 7 + + # Find last Sunday in October. + end = first_sunday_on_or_after(DSTEND.replace(year=dt.year)) + assert end.weekday() == 6 and end.month == 10 and end.day >= 25 + + # Can't compare naive to aware objects, so strip the timezone from + # dt first. + if start <= dt.replace(tzinfo=None) < end: + return HOUR + else: + return ZERO + +Eastern = USTimeZone(-5, "Eastern", "EST", "EDT") +Central = USTimeZone(-6, "Central", "CST", "CDT") +Mountain = USTimeZone(-7, "Mountain", "MST", "MDT") +Pacific = USTimeZone(-8, "Pacific", "PST", "PDT") +utc_real = FixedOffset(0, "UTC", 0) +# For better test coverage, we want another flavor of UTC that's west of +# the Eastern and Pacific timezones. +utc_fake = FixedOffset(-12*60, "UTCfake", 0) + +class TestTimezoneConversions(unittest.TestCase): + # The DST switch times for 2002, in std time. + dston = datetime(2002, 4, 7, 2) + dstoff = datetime(2002, 10, 27, 1) + + theclass = datetime + + # Check a time that's inside DST. + def checkinside(self, dt, tz, utc, dston, dstoff): + self.assertEqual(dt.dst(), HOUR) + + # Conversion to our own timezone is always an identity. + self.assertEqual(dt.astimezone(tz), dt) + + asutc = dt.astimezone(utc) + there_and_back = asutc.astimezone(tz) + + # Conversion to UTC and back isn't always an identity here, + # because there are redundant spellings (in local time) of + # UTC time when DST begins: the clock jumps from 1:59:59 + # to 3:00:00, and a local time of 2:MM:SS doesn't really + # make sense then. The classes above treat 2:MM:SS as + # daylight time then (it's "after 2am"), really an alias + # for 1:MM:SS standard time. The latter form is what + # conversion back from UTC produces. + if dt.date() == dston.date() and dt.hour == 2: + # We're in the redundant hour, and coming back from + # UTC gives the 1:MM:SS standard-time spelling. + self.assertEqual(there_and_back + HOUR, dt) + # Although during was considered to be in daylight + # time, there_and_back is not. + self.assertEqual(there_and_back.dst(), ZERO) + # They're the same times in UTC. + self.assertEqual(there_and_back.astimezone(utc), + dt.astimezone(utc)) + else: + # We're not in the redundant hour. + self.assertEqual(dt, there_and_back) + + # Because we have a redundant spelling when DST begins, there is + # (unfortunately) an hour when DST ends that can't be spelled at all in + # local time. When DST ends, the clock jumps from 1:59 back to 1:00 + # again. The hour 1:MM DST has no spelling then: 1:MM is taken to be + # standard time. 1:MM DST == 0:MM EST, but 0:MM is taken to be + # daylight time. The hour 1:MM daylight == 0:MM standard can't be + # expressed in local time. Nevertheless, we want conversion back + # from UTC to mimic the local clock's "repeat an hour" behavior. + nexthour_utc = asutc + HOUR + nexthour_tz = nexthour_utc.astimezone(tz) + if dt.date() == dstoff.date() and dt.hour == 0: + # We're in the hour before the last DST hour. The last DST hour + # is ineffable. We want the conversion back to repeat 1:MM. + self.assertEqual(nexthour_tz, dt.replace(hour=1)) + nexthour_utc += HOUR + nexthour_tz = nexthour_utc.astimezone(tz) + self.assertEqual(nexthour_tz, dt.replace(hour=1)) + else: + self.assertEqual(nexthour_tz - dt, HOUR) + + # Check a time that's outside DST. + def checkoutside(self, dt, tz, utc): + self.assertEqual(dt.dst(), ZERO) + + # Conversion to our own timezone is always an identity. + self.assertEqual(dt.astimezone(tz), dt) + + # Converting to UTC and back is an identity too. + asutc = dt.astimezone(utc) + there_and_back = asutc.astimezone(tz) + self.assertEqual(dt, there_and_back) + + def convert_between_tz_and_utc(self, tz, utc): + dston = self.dston.replace(tzinfo=tz) + # Because 1:MM on the day DST ends is taken as being standard time, + # there is no spelling in tz for the last hour of daylight time. + # For purposes of the test, the last hour of DST is 0:MM, which is + # taken as being daylight time (and 1:MM is taken as being standard + # time). + dstoff = self.dstoff.replace(tzinfo=tz) + for delta in (timedelta(weeks=13), + DAY, + HOUR, + timedelta(minutes=1), + timedelta(microseconds=1)): + + self.checkinside(dston, tz, utc, dston, dstoff) + for during in dston + delta, dstoff - delta: + self.checkinside(during, tz, utc, dston, dstoff) + + self.checkoutside(dstoff, tz, utc) + for outside in dston - delta, dstoff + delta: + self.checkoutside(outside, tz, utc) + + def test_easy(self): + # Despite the name of this test, the endcases are excruciating. + self.convert_between_tz_and_utc(Eastern, utc_real) + self.convert_between_tz_and_utc(Pacific, utc_real) + self.convert_between_tz_and_utc(Eastern, utc_fake) + self.convert_between_tz_and_utc(Pacific, utc_fake) + # The next is really dancing near the edge. It works because + # Pacific and Eastern are far enough apart that their "problem + # hours" don't overlap. + self.convert_between_tz_and_utc(Eastern, Pacific) + self.convert_between_tz_and_utc(Pacific, Eastern) + # OTOH, these fail! Don't enable them. The difficulty is that + # the edge case tests assume that every hour is representable in + # the "utc" class. This is always true for a fixed-offset tzinfo + # class (like utc_real and utc_fake), but not for Eastern or Central. + # For these adjacent DST-aware time zones, the range of time offsets + # tested ends up creating hours in the one that aren't representable + # in the other. For the same reason, we would see failures in the + # Eastern vs Pacific tests too if we added 3*HOUR to the list of + # offset deltas in convert_between_tz_and_utc(). + # + # self.convert_between_tz_and_utc(Eastern, Central) # can't work + # self.convert_between_tz_and_utc(Central, Eastern) # can't work + + def test_tricky(self): + # 22:00 on day before daylight starts. + fourback = self.dston - timedelta(hours=4) + ninewest = FixedOffset(-9*60, "-0900", 0) + fourback = fourback.replace(tzinfo=ninewest) + # 22:00-0900 is 7:00 UTC == 2:00 EST == 3:00 DST. Since it's "after + # 2", we should get the 3 spelling. + # If we plug 22:00 the day before into Eastern, it "looks like std + # time", so its offset is returned as -5, and -5 - -9 = 4. Adding 4 + # to 22:00 lands on 2:00, which makes no sense in local time (the + # local clock jumps from 1 to 3). The point here is to make sure we + # get the 3 spelling. + expected = self.dston.replace(hour=3) + got = fourback.astimezone(Eastern).replace(tzinfo=None) + self.assertEqual(expected, got) + + # Similar, but map to 6:00 UTC == 1:00 EST == 2:00 DST. In that + # case we want the 1:00 spelling. + sixutc = self.dston.replace(hour=6, tzinfo=utc_real) + # Now 6:00 "looks like daylight", so the offset wrt Eastern is -4, + # and adding -4-0 == -4 gives the 2:00 spelling. We want the 1:00 EST + # spelling. + expected = self.dston.replace(hour=1) + got = sixutc.astimezone(Eastern).replace(tzinfo=None) + self.assertEqual(expected, got) + + # Now on the day DST ends, we want "repeat an hour" behavior. + # UTC 4:MM 5:MM 6:MM 7:MM checking these + # EST 23:MM 0:MM 1:MM 2:MM + # EDT 0:MM 1:MM 2:MM 3:MM + # wall 0:MM 1:MM 1:MM 2:MM against these + for utc in utc_real, utc_fake: + for tz in Eastern, Pacific: + first_std_hour = self.dstoff - timedelta(hours=2) # 23:MM + # Convert that to UTC. + first_std_hour -= tz.utcoffset(None) + # Adjust for possibly fake UTC. + asutc = first_std_hour + utc.utcoffset(None) + # First UTC hour to convert; this is 4:00 when utc=utc_real & + # tz=Eastern. + asutcbase = asutc.replace(tzinfo=utc) + for tzhour in (0, 1, 1, 2): + expectedbase = self.dstoff.replace(hour=tzhour) + for minute in 0, 30, 59: + expected = expectedbase.replace(minute=minute) + asutc = asutcbase.replace(minute=minute) + astz = asutc.astimezone(tz) + self.assertEqual(astz.replace(tzinfo=None), expected) + asutcbase += HOUR + + + def test_bogus_dst(self): + class ok(tzinfo): + def utcoffset(self, dt): return HOUR + def dst(self, dt): return HOUR + + now = self.theclass.now().replace(tzinfo=utc_real) + # Doesn't blow up. + now.astimezone(ok()) + + # Does blow up. + class notok(ok): + def dst(self, dt): return None + self.assertRaises(ValueError, now.astimezone, notok()) + + # Sometimes blow up. In the following, tzinfo.dst() + # implementation may return None or not None depending on + # whether DST is assumed to be in effect. In this situation, + # a ValueError should be raised by astimezone(). + class tricky_notok(ok): + def dst(self, dt): + if dt.year == 2000: + return None + else: + return 10*HOUR + dt = self.theclass(2001, 1, 1).replace(tzinfo=utc_real) + self.assertRaises(ValueError, dt.astimezone, tricky_notok()) + + def test_fromutc(self): + self.assertRaises(TypeError, Eastern.fromutc) # not enough args + now = datetime.now(tz=utc_real) + self.assertRaises(ValueError, Eastern.fromutc, now) # wrong tzinfo + now = now.replace(tzinfo=Eastern) # insert correct tzinfo + enow = Eastern.fromutc(now) # doesn't blow up + self.assertEqual(enow.tzinfo, Eastern) # has right tzinfo member + self.assertRaises(TypeError, Eastern.fromutc, now, now) # too many args + self.assertRaises(TypeError, Eastern.fromutc, date.today()) # wrong type + + # Always converts UTC to standard time. + class FauxUSTimeZone(USTimeZone): + def fromutc(self, dt): + return dt + self.stdoffset + FEastern = FauxUSTimeZone(-5, "FEastern", "FEST", "FEDT") + + # UTC 4:MM 5:MM 6:MM 7:MM 8:MM 9:MM + # EST 23:MM 0:MM 1:MM 2:MM 3:MM 4:MM + # EDT 0:MM 1:MM 2:MM 3:MM 4:MM 5:MM + + # Check around DST start. + start = self.dston.replace(hour=4, tzinfo=Eastern) + fstart = start.replace(tzinfo=FEastern) + for wall in 23, 0, 1, 3, 4, 5: + expected = start.replace(hour=wall) + if wall == 23: + expected -= timedelta(days=1) + got = Eastern.fromutc(start) + self.assertEqual(expected, got) + + expected = fstart + FEastern.stdoffset + got = FEastern.fromutc(fstart) + self.assertEqual(expected, got) + + # Ensure astimezone() calls fromutc() too. + got = fstart.replace(tzinfo=utc_real).astimezone(FEastern) + self.assertEqual(expected, got) + + start += HOUR + fstart += HOUR + + # Check around DST end. + start = self.dstoff.replace(hour=4, tzinfo=Eastern) + fstart = start.replace(tzinfo=FEastern) + for wall in 0, 1, 1, 2, 3, 4: + expected = start.replace(hour=wall) + got = Eastern.fromutc(start) + self.assertEqual(expected, got) + + expected = fstart + FEastern.stdoffset + got = FEastern.fromutc(fstart) + self.assertEqual(expected, got) + + # Ensure astimezone() calls fromutc() too. + got = fstart.replace(tzinfo=utc_real).astimezone(FEastern) + self.assertEqual(expected, got) + + start += HOUR + fstart += HOUR + + +############################################################################# +# oddballs + +class Oddballs(unittest.TestCase): + + def test_date_datetime_comparison(self): + # bpo-1028306, bpo-5516 (gh-49766) + # Trying to compare a date to a datetime should act like a mixed- + # type comparison, despite that datetime is a subclass of date. + as_date = date.today() + as_datetime = datetime.combine(as_date, time()) + date_sc = SubclassDate(as_date.year, as_date.month, as_date.day) + datetime_sc = SubclassDatetime(as_date.year, as_date.month, + as_date.day, 0, 0, 0) + for d in (as_date, date_sc): + for dt in (as_datetime, datetime_sc): + for x, y in (d, dt), (dt, d): + self.assertTrue(x != y) + self.assertFalse(x == y) + self.assertRaises(TypeError, lambda: x < y) + self.assertRaises(TypeError, lambda: x <= y) + self.assertRaises(TypeError, lambda: x > y) + self.assertRaises(TypeError, lambda: x >= y) + + # And date should compare with other subclasses of date. If a + # subclass wants to stop this, it's up to the subclass to do so. + # Ditto for datetimes. + for x, y in ((as_date, date_sc), + (date_sc, as_date), + (as_datetime, datetime_sc), + (datetime_sc, as_datetime)): + self.assertTrue(x == y) + self.assertFalse(x != y) + self.assertFalse(x < y) + self.assertFalse(x > y) + self.assertTrue(x <= y) + self.assertTrue(x >= y) + + # Nevertheless, comparison should work if other object is an instance + # of date or datetime class with overridden comparison operators. + # So special methods should return NotImplemented, as if + # date and datetime were independent classes. + for x, y in (as_date, as_datetime), (as_datetime, as_date): + self.assertEqual(x.__eq__(y), NotImplemented) + self.assertEqual(x.__ne__(y), NotImplemented) + self.assertEqual(x.__lt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__gt__(y), NotImplemented) + self.assertEqual(x.__ge__(y), NotImplemented) + + def test_extra_attributes(self): + with self.assertWarns(DeprecationWarning): + utcnow = datetime.utcnow() + for x in [date.today(), + time(), + utcnow, + timedelta(), + tzinfo(), + timezone(timedelta())]: + with self.assertRaises(AttributeError): + x.abc = 1 + + def test_check_arg_types(self): + class Number: + def __init__(self, value): + self.value = value + def __int__(self): + return self.value + + class Float(float): + pass + + for xx in [10.0, Float(10.9), + decimal.Decimal(10), decimal.Decimal('10.9'), + Number(10), Number(10.9), + '10']: + self.assertRaises(TypeError, datetime, xx, 10, 10, 10, 10, 10, 10) + self.assertRaises(TypeError, datetime, 10, xx, 10, 10, 10, 10, 10) + self.assertRaises(TypeError, datetime, 10, 10, xx, 10, 10, 10, 10) + self.assertRaises(TypeError, datetime, 10, 10, 10, xx, 10, 10, 10) + self.assertRaises(TypeError, datetime, 10, 10, 10, 10, xx, 10, 10) + self.assertRaises(TypeError, datetime, 10, 10, 10, 10, 10, xx, 10) + self.assertRaises(TypeError, datetime, 10, 10, 10, 10, 10, 10, xx) + + +############################################################################# +# Local Time Disambiguation + +# An experimental reimplementation of fromutc that respects the "fold" flag. + +class tzinfo2(tzinfo): + + def fromutc(self, dt): + "datetime in UTC -> datetime in local time." + + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + # Returned value satisfies + # dt + ldt.utcoffset() = ldt + off0 = dt.replace(fold=0).utcoffset() + off1 = dt.replace(fold=1).utcoffset() + if off0 is None or off1 is None or dt.dst() is None: + raise ValueError + if off0 == off1: + ldt = dt + off0 + off1 = ldt.utcoffset() + if off0 == off1: + return ldt + # Now, we discovered both possible offsets, so + # we can just try four possible solutions: + for off in [off0, off1]: + ldt = dt + off + if ldt.utcoffset() == off: + return ldt + ldt = ldt.replace(fold=1) + if ldt.utcoffset() == off: + return ldt + + raise ValueError("No suitable local time found") + +# Reimplementing simplified US timezones to respect the "fold" flag: + +class USTimeZone2(tzinfo2): + + def __init__(self, hours, reprname, stdname, dstname): + self.stdoffset = timedelta(hours=hours) + self.reprname = reprname + self.stdname = stdname + self.dstname = dstname + + def __repr__(self): + return self.reprname + + def tzname(self, dt): + if self.dst(dt): + return self.dstname + else: + return self.stdname + + def utcoffset(self, dt): + return self.stdoffset + self.dst(dt) + + def dst(self, dt): + if dt is None or dt.tzinfo is None: + # An exception instead may be sensible here, in one or more of + # the cases. + return ZERO + assert dt.tzinfo is self + + # Find first Sunday in April. + start = first_sunday_on_or_after(DSTSTART.replace(year=dt.year)) + assert start.weekday() == 6 and start.month == 4 and start.day <= 7 + + # Find last Sunday in October. + end = first_sunday_on_or_after(DSTEND.replace(year=dt.year)) + assert end.weekday() == 6 and end.month == 10 and end.day >= 25 + + # Can't compare naive to aware objects, so strip the timezone from + # dt first. + dt = dt.replace(tzinfo=None) + if start + HOUR <= dt < end: + # DST is in effect. + return HOUR + elif end <= dt < end + HOUR: + # Fold (an ambiguous hour): use dt.fold to disambiguate. + return ZERO if dt.fold else HOUR + elif start <= dt < start + HOUR: + # Gap (a non-existent hour): reverse the fold rule. + return HOUR if dt.fold else ZERO + else: + # DST is off. + return ZERO + +Eastern2 = USTimeZone2(-5, "Eastern2", "EST", "EDT") +Central2 = USTimeZone2(-6, "Central2", "CST", "CDT") +Mountain2 = USTimeZone2(-7, "Mountain2", "MST", "MDT") +Pacific2 = USTimeZone2(-8, "Pacific2", "PST", "PDT") + +# Europe_Vilnius_1941 tzinfo implementation reproduces the following +# 1941 transition from Olson's tzdist: +# +# Zone NAME GMTOFF RULES FORMAT [UNTIL] +# ZoneEurope/Vilnius 1:00 - CET 1940 Aug 3 +# 3:00 - MSK 1941 Jun 24 +# 1:00 C-Eur CE%sT 1944 Aug +# +# $ zdump -v Europe/Vilnius | grep 1941 +# Europe/Vilnius Mon Jun 23 20:59:59 1941 UTC = Mon Jun 23 23:59:59 1941 MSK isdst=0 gmtoff=10800 +# Europe/Vilnius Mon Jun 23 21:00:00 1941 UTC = Mon Jun 23 23:00:00 1941 CEST isdst=1 gmtoff=7200 + +class Europe_Vilnius_1941(tzinfo): + def _utc_fold(self): + return [datetime(1941, 6, 23, 21, tzinfo=self), # Mon Jun 23 21:00:00 1941 UTC + datetime(1941, 6, 23, 22, tzinfo=self)] # Mon Jun 23 22:00:00 1941 UTC + + def _loc_fold(self): + return [datetime(1941, 6, 23, 23, tzinfo=self), # Mon Jun 23 23:00:00 1941 MSK / CEST + datetime(1941, 6, 24, 0, tzinfo=self)] # Mon Jun 24 00:00:00 1941 CEST + + def utcoffset(self, dt): + fold_start, fold_stop = self._loc_fold() + if dt < fold_start: + return 3 * HOUR + if dt < fold_stop: + return (2 if dt.fold else 3) * HOUR + # if dt >= fold_stop + return 2 * HOUR + + def dst(self, dt): + fold_start, fold_stop = self._loc_fold() + if dt < fold_start: + return 0 * HOUR + if dt < fold_stop: + return (1 if dt.fold else 0) * HOUR + # if dt >= fold_stop + return 1 * HOUR + + def tzname(self, dt): + fold_start, fold_stop = self._loc_fold() + if dt < fold_start: + return 'MSK' + if dt < fold_stop: + return ('MSK', 'CEST')[dt.fold] + # if dt >= fold_stop + return 'CEST' + + def fromutc(self, dt): + assert dt.fold == 0 + assert dt.tzinfo is self + if dt.year != 1941: + raise NotImplementedError + fold_start, fold_stop = self._utc_fold() + if dt < fold_start: + return dt + 3 * HOUR + if dt < fold_stop: + return (dt + 2 * HOUR).replace(fold=1) + # if dt >= fold_stop + return dt + 2 * HOUR + + +class TestLocalTimeDisambiguation(unittest.TestCase): + + def test_vilnius_1941_fromutc(self): + Vilnius = Europe_Vilnius_1941() + + gdt = datetime(1941, 6, 23, 20, 59, 59, tzinfo=timezone.utc) + ldt = gdt.astimezone(Vilnius) + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), + 'Mon Jun 23 23:59:59 1941 MSK+0300') + self.assertEqual(ldt.fold, 0) + self.assertFalse(ldt.dst()) + + gdt = datetime(1941, 6, 23, 21, tzinfo=timezone.utc) + ldt = gdt.astimezone(Vilnius) + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), + 'Mon Jun 23 23:00:00 1941 CEST+0200') + self.assertEqual(ldt.fold, 1) + self.assertTrue(ldt.dst()) + + gdt = datetime(1941, 6, 23, 22, tzinfo=timezone.utc) + ldt = gdt.astimezone(Vilnius) + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), + 'Tue Jun 24 00:00:00 1941 CEST+0200') + self.assertEqual(ldt.fold, 0) + self.assertTrue(ldt.dst()) + + def test_vilnius_1941_toutc(self): + Vilnius = Europe_Vilnius_1941() + + ldt = datetime(1941, 6, 23, 22, 59, 59, tzinfo=Vilnius) + gdt = ldt.astimezone(timezone.utc) + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), + 'Mon Jun 23 19:59:59 1941 UTC') + + ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius) + gdt = ldt.astimezone(timezone.utc) + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), + 'Mon Jun 23 20:59:59 1941 UTC') + + ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius, fold=1) + gdt = ldt.astimezone(timezone.utc) + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), + 'Mon Jun 23 21:59:59 1941 UTC') + + ldt = datetime(1941, 6, 24, 0, tzinfo=Vilnius) + gdt = ldt.astimezone(timezone.utc) + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), + 'Mon Jun 23 22:00:00 1941 UTC') + + def test_constructors(self): + t = time(0, fold=1) + dt = datetime(1, 1, 1, fold=1) + self.assertEqual(t.fold, 1) + self.assertEqual(dt.fold, 1) + with self.assertRaises(TypeError): + time(0, 0, 0, 0, None, 0) + + def test_member(self): + dt = datetime(1, 1, 1, fold=1) + t = dt.time() + self.assertEqual(t.fold, 1) + t = dt.timetz() + self.assertEqual(t.fold, 1) + + def test_replace(self): + t = time(0) + dt = datetime(1, 1, 1) + self.assertEqual(t.replace(fold=1).fold, 1) + self.assertEqual(dt.replace(fold=1).fold, 1) + self.assertEqual(t.replace(fold=0).fold, 0) + self.assertEqual(dt.replace(fold=0).fold, 0) + # Check that replacement of other fields does not change "fold". + t = t.replace(fold=1, tzinfo=Eastern) + dt = dt.replace(fold=1, tzinfo=Eastern) + self.assertEqual(t.replace(tzinfo=None).fold, 1) + self.assertEqual(dt.replace(tzinfo=None).fold, 1) + # Out of bounds. + with self.assertRaises(ValueError): + t.replace(fold=2) + with self.assertRaises(ValueError): + dt.replace(fold=2) + # Check that fold is a keyword-only argument + with self.assertRaises(TypeError): + t.replace(1, 1, 1, None, 1) + with self.assertRaises(TypeError): + dt.replace(1, 1, 1, 1, 1, 1, 1, None, 1) + + def test_comparison(self): + t = time(0) + dt = datetime(1, 1, 1) + self.assertEqual(t, t.replace(fold=1)) + self.assertEqual(dt, dt.replace(fold=1)) + + def test_hash(self): + t = time(0) + dt = datetime(1, 1, 1) + self.assertEqual(hash(t), hash(t.replace(fold=1))) + self.assertEqual(hash(dt), hash(dt.replace(fold=1))) + + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_fromtimestamp(self): + s = 1414906200 + dt0 = datetime.fromtimestamp(s) + dt1 = datetime.fromtimestamp(s + 3600) + self.assertEqual(dt0.fold, 0) + self.assertEqual(dt1.fold, 1) + + @support.run_with_tz('Australia/Lord_Howe') + def test_fromtimestamp_lord_howe(self): + tm = _time.localtime(1.4e9) + if _time.strftime('%Z%z', tm) != 'LHST+1030': + self.skipTest('Australia/Lord_Howe timezone is not supported on this platform') + # $ TZ=Australia/Lord_Howe date -r 1428158700 + # Sun Apr 5 01:45:00 LHDT 2015 + # $ TZ=Australia/Lord_Howe date -r 1428160500 + # Sun Apr 5 01:45:00 LHST 2015 + s = 1428158700 + t0 = datetime.fromtimestamp(s) + t1 = datetime.fromtimestamp(s + 1800) + self.assertEqual(t0, t1) + self.assertEqual(t0.fold, 0) + self.assertEqual(t1.fold, 1) + + def test_fromtimestamp_low_fold_detection(self): + # Ensure that fold detection doesn't cause an + # OSError for really low values, see bpo-29097 + self.assertEqual(datetime.fromtimestamp(0).fold, 0) + + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_timestamp(self): + dt0 = datetime(2014, 11, 2, 1, 30) + dt1 = dt0.replace(fold=1) + self.assertEqual(dt0.timestamp() + 3600, + dt1.timestamp()) + + @support.run_with_tz('Australia/Lord_Howe') + def test_timestamp_lord_howe(self): + tm = _time.localtime(1.4e9) + if _time.strftime('%Z%z', tm) != 'LHST+1030': + self.skipTest('Australia/Lord_Howe timezone is not supported on this platform') + t = datetime(2015, 4, 5, 1, 45) + s0 = t.replace(fold=0).timestamp() + s1 = t.replace(fold=1).timestamp() + self.assertEqual(s0 + 1800, s1) + + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_astimezone(self): + dt0 = datetime(2014, 11, 2, 1, 30) + dt1 = dt0.replace(fold=1) + # Convert both naive instances to aware. + adt0 = dt0.astimezone() + adt1 = dt1.astimezone() + # Check that the first instance in DST zone and the second in STD + self.assertEqual(adt0.tzname(), 'EDT') + self.assertEqual(adt1.tzname(), 'EST') + self.assertEqual(adt0 + HOUR, adt1) + # Aware instances with fixed offset tzinfo's always have fold=0 + self.assertEqual(adt0.fold, 0) + self.assertEqual(adt1.fold, 0) + + def test_pickle_fold(self): + t = time(fold=1) + dt = datetime(1, 1, 1, fold=1) + for pickler, unpickler, proto in pickle_choices: + for x in [t, dt]: + s = pickler.dumps(x, proto) + y = unpickler.loads(s) + self.assertEqual(x, y) + self.assertEqual((0 if proto < 4 else x.fold), y.fold) + + def test_repr(self): + t = time(fold=1) + dt = datetime(1, 1, 1, fold=1) + self.assertEqual(repr(t), 'datetime.time(0, 0, fold=1)') + self.assertEqual(repr(dt), + 'datetime.datetime(1, 1, 1, 0, 0, fold=1)') + + def test_dst(self): + # Let's first establish that things work in regular times. + dt_summer = datetime(2002, 10, 27, 1, tzinfo=Eastern2) - timedelta.resolution + dt_winter = datetime(2002, 10, 27, 2, tzinfo=Eastern2) + self.assertEqual(dt_summer.dst(), HOUR) + self.assertEqual(dt_winter.dst(), ZERO) + # The disambiguation flag is ignored + self.assertEqual(dt_summer.replace(fold=1).dst(), HOUR) + self.assertEqual(dt_winter.replace(fold=1).dst(), ZERO) + + # Pick local time in the fold. + for minute in [0, 30, 59]: + dt = datetime(2002, 10, 27, 1, minute, tzinfo=Eastern2) + # With fold=0 (the default) it is in DST. + self.assertEqual(dt.dst(), HOUR) + # With fold=1 it is in STD. + self.assertEqual(dt.replace(fold=1).dst(), ZERO) + + # Pick local time in the gap. + for minute in [0, 30, 59]: + dt = datetime(2002, 4, 7, 2, minute, tzinfo=Eastern2) + # With fold=0 (the default) it is in STD. + self.assertEqual(dt.dst(), ZERO) + # With fold=1 it is in DST. + self.assertEqual(dt.replace(fold=1).dst(), HOUR) + + + def test_utcoffset(self): + # Let's first establish that things work in regular times. + dt_summer = datetime(2002, 10, 27, 1, tzinfo=Eastern2) - timedelta.resolution + dt_winter = datetime(2002, 10, 27, 2, tzinfo=Eastern2) + self.assertEqual(dt_summer.utcoffset(), -4 * HOUR) + self.assertEqual(dt_winter.utcoffset(), -5 * HOUR) + # The disambiguation flag is ignored + self.assertEqual(dt_summer.replace(fold=1).utcoffset(), -4 * HOUR) + self.assertEqual(dt_winter.replace(fold=1).utcoffset(), -5 * HOUR) + + def test_fromutc(self): + # Let's first establish that things work in regular times. + u_summer = datetime(2002, 10, 27, 6, tzinfo=Eastern2) - timedelta.resolution + u_winter = datetime(2002, 10, 27, 7, tzinfo=Eastern2) + t_summer = Eastern2.fromutc(u_summer) + t_winter = Eastern2.fromutc(u_winter) + self.assertEqual(t_summer, u_summer - 4 * HOUR) + self.assertEqual(t_winter, u_winter - 5 * HOUR) + self.assertEqual(t_summer.fold, 0) + self.assertEqual(t_winter.fold, 0) + + # What happens in the fall-back fold? + u = datetime(2002, 10, 27, 5, 30, tzinfo=Eastern2) + t0 = Eastern2.fromutc(u) + u += HOUR + t1 = Eastern2.fromutc(u) + self.assertEqual(t0, t1) + self.assertEqual(t0.fold, 0) + self.assertEqual(t1.fold, 1) + # The tricky part is when u is in the local fold: + u = datetime(2002, 10, 27, 1, 30, tzinfo=Eastern2) + t = Eastern2.fromutc(u) + self.assertEqual((t.day, t.hour), (26, 21)) + # .. or gets into the local fold after a standard time adjustment + u = datetime(2002, 10, 27, 6, 30, tzinfo=Eastern2) + t = Eastern2.fromutc(u) + self.assertEqual((t.day, t.hour), (27, 1)) + + # What happens in the spring-forward gap? + u = datetime(2002, 4, 7, 2, 0, tzinfo=Eastern2) + t = Eastern2.fromutc(u) + self.assertEqual((t.day, t.hour), (6, 21)) + + def test_mixed_compare_regular(self): + t = datetime(2000, 1, 1, tzinfo=Eastern2) + self.assertEqual(t, t.astimezone(timezone.utc)) + t = datetime(2000, 6, 1, tzinfo=Eastern2) + self.assertEqual(t, t.astimezone(timezone.utc)) + + def test_mixed_compare_fold(self): + t_fold = datetime(2002, 10, 27, 1, 45, tzinfo=Eastern2) + t_fold_utc = t_fold.astimezone(timezone.utc) + self.assertNotEqual(t_fold, t_fold_utc) + self.assertNotEqual(t_fold_utc, t_fold) + + def test_mixed_compare_gap(self): + t_gap = datetime(2002, 4, 7, 2, 45, tzinfo=Eastern2) + t_gap_utc = t_gap.astimezone(timezone.utc) + self.assertNotEqual(t_gap, t_gap_utc) + self.assertNotEqual(t_gap_utc, t_gap) + + def test_hash_aware(self): + t = datetime(2000, 1, 1, tzinfo=Eastern2) + self.assertEqual(hash(t), hash(t.replace(fold=1))) + t_fold = datetime(2002, 10, 27, 1, 45, tzinfo=Eastern2) + t_gap = datetime(2002, 4, 7, 2, 45, tzinfo=Eastern2) + self.assertEqual(hash(t_fold), hash(t_fold.replace(fold=1))) + self.assertEqual(hash(t_gap), hash(t_gap.replace(fold=1))) + +SEC = timedelta(0, 1) + +def pairs(iterable): + a, b = itertools.tee(iterable) + next(b, None) + return zip(a, b) + +class ZoneInfo(tzinfo): + zoneroot = '/usr/share/zoneinfo' + def __init__(self, ut, ti): + """ + + :param ut: array + Array of transition point timestamps + :param ti: list + A list of (offset, isdst, abbr) tuples + :return: None + """ + self.ut = ut + self.ti = ti + self.lt = self.invert(ut, ti) + + @staticmethod + def invert(ut, ti): + lt = (array('q', ut), array('q', ut)) + if ut: + offset = ti[0][0] // SEC + lt[0][0] += offset + lt[1][0] += offset + for i in range(1, len(ut)): + lt[0][i] += ti[i-1][0] // SEC + lt[1][i] += ti[i][0] // SEC + return lt + + @classmethod + def fromfile(cls, fileobj): + if fileobj.read(4).decode() != "TZif": + raise ValueError("not a zoneinfo file") + fileobj.seek(32) + counts = array('i') + counts.fromfile(fileobj, 3) + if sys.byteorder != 'big': + counts.byteswap() + + ut = array('i') + ut.fromfile(fileobj, counts[0]) + if sys.byteorder != 'big': + ut.byteswap() + + type_indices = array('B') + type_indices.fromfile(fileobj, counts[0]) + + ttis = [] + for i in range(counts[1]): + ttis.append(struct.unpack(">lbb", fileobj.read(6))) + + abbrs = fileobj.read(counts[2]) + + # Convert ttis + for i, (gmtoff, isdst, abbrind) in enumerate(ttis): + abbr = abbrs[abbrind:abbrs.find(0, abbrind)].decode() + ttis[i] = (timedelta(0, gmtoff), isdst, abbr) + + ti = [None] * len(ut) + for i, idx in enumerate(type_indices): + ti[i] = ttis[idx] + + self = cls(ut, ti) + + return self + + @classmethod + def fromname(cls, name): + path = os.path.join(cls.zoneroot, name) + with open(path, 'rb') as f: + return cls.fromfile(f) + + EPOCHORDINAL = date(1970, 1, 1).toordinal() + + def fromutc(self, dt): + """datetime in UTC -> datetime in local time.""" + + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + + timestamp = ((dt.toordinal() - self.EPOCHORDINAL) * 86400 + + dt.hour * 3600 + + dt.minute * 60 + + dt.second) + + if timestamp < self.ut[1]: + tti = self.ti[0] + fold = 0 + else: + idx = bisect.bisect_right(self.ut, timestamp) + assert self.ut[idx-1] <= timestamp + assert idx == len(self.ut) or timestamp < self.ut[idx] + tti_prev, tti = self.ti[idx-2:idx] + # Detect fold + shift = tti_prev[0] - tti[0] + fold = (shift > timedelta(0, timestamp - self.ut[idx-1])) + dt += tti[0] + if fold: + return dt.replace(fold=1) + else: + return dt + + def _find_ti(self, dt, i): + timestamp = ((dt.toordinal() - self.EPOCHORDINAL) * 86400 + + dt.hour * 3600 + + dt.minute * 60 + + dt.second) + lt = self.lt[dt.fold] + idx = bisect.bisect_right(lt, timestamp) + + return self.ti[max(0, idx - 1)][i] + + def utcoffset(self, dt): + return self._find_ti(dt, 0) + + def dst(self, dt): + isdst = self._find_ti(dt, 1) + # XXX: We cannot accurately determine the "save" value, + # so let's return 1h whenever DST is in effect. Since + # we don't use dst() in fromutc(), it is unlikely that + # it will be needed for anything more than bool(dst()). + return ZERO if isdst else HOUR + + def tzname(self, dt): + return self._find_ti(dt, 2) + + @classmethod + def zonenames(cls, zonedir=None): + if zonedir is None: + zonedir = cls.zoneroot + zone_tab = os.path.join(zonedir, 'zone.tab') + try: + f = open(zone_tab) + except OSError: + return + with f: + for line in f: + line = line.strip() + if line and not line.startswith('#'): + yield line.split()[2] + + @classmethod + def stats(cls, start_year=1): + count = gap_count = fold_count = zeros_count = 0 + min_gap = min_fold = timedelta.max + max_gap = max_fold = ZERO + min_gap_datetime = max_gap_datetime = datetime.min + min_gap_zone = max_gap_zone = None + min_fold_datetime = max_fold_datetime = datetime.min + min_fold_zone = max_fold_zone = None + stats_since = datetime(start_year, 1, 1) # Starting from 1970 eliminates a lot of noise + for zonename in cls.zonenames(): + count += 1 + tz = cls.fromname(zonename) + for dt, shift in tz.transitions(): + if dt < stats_since: + continue + if shift > ZERO: + gap_count += 1 + if (shift, dt) > (max_gap, max_gap_datetime): + max_gap = shift + max_gap_zone = zonename + max_gap_datetime = dt + if (shift, datetime.max - dt) < (min_gap, datetime.max - min_gap_datetime): + min_gap = shift + min_gap_zone = zonename + min_gap_datetime = dt + elif shift < ZERO: + fold_count += 1 + shift = -shift + if (shift, dt) > (max_fold, max_fold_datetime): + max_fold = shift + max_fold_zone = zonename + max_fold_datetime = dt + if (shift, datetime.max - dt) < (min_fold, datetime.max - min_fold_datetime): + min_fold = shift + min_fold_zone = zonename + min_fold_datetime = dt + else: + zeros_count += 1 + trans_counts = (gap_count, fold_count, zeros_count) + print("Number of zones: %5d" % count) + print("Number of transitions: %5d = %d (gaps) + %d (folds) + %d (zeros)" % + ((sum(trans_counts),) + trans_counts)) + print("Min gap: %16s at %s in %s" % (min_gap, min_gap_datetime, min_gap_zone)) + print("Max gap: %16s at %s in %s" % (max_gap, max_gap_datetime, max_gap_zone)) + print("Min fold: %16s at %s in %s" % (min_fold, min_fold_datetime, min_fold_zone)) + print("Max fold: %16s at %s in %s" % (max_fold, max_fold_datetime, max_fold_zone)) + + + def transitions(self): + for (_, prev_ti), (t, ti) in pairs(zip(self.ut, self.ti)): + shift = ti[0] - prev_ti[0] + yield (EPOCH_NAIVE + timedelta(seconds=t)), shift + + def nondst_folds(self): + """Find all folds with the same value of isdst on both sides of the transition.""" + for (_, prev_ti), (t, ti) in pairs(zip(self.ut, self.ti)): + shift = ti[0] - prev_ti[0] + if shift < ZERO and ti[1] == prev_ti[1]: + yield _utcfromtimestamp(datetime, t,), -shift, prev_ti[2], ti[2] + + @classmethod + def print_all_nondst_folds(cls, same_abbr=False, start_year=1): + count = 0 + for zonename in cls.zonenames(): + tz = cls.fromname(zonename) + for dt, shift, prev_abbr, abbr in tz.nondst_folds(): + if dt.year < start_year or same_abbr and prev_abbr != abbr: + continue + count += 1 + print("%3d) %-30s %s %10s %5s -> %s" % + (count, zonename, dt, shift, prev_abbr, abbr)) + + def folds(self): + for t, shift in self.transitions(): + if shift < ZERO: + yield t, -shift + + def gaps(self): + for t, shift in self.transitions(): + if shift > ZERO: + yield t, shift + + def zeros(self): + for t, shift in self.transitions(): + if not shift: + yield t + + +class ZoneInfoTest(unittest.TestCase): + zonename = 'America/New_York' + + def setUp(self): + if sys.platform == "vxworks": + self.skipTest("Skipping zoneinfo tests on VxWorks") + if sys.platform == "win32": + self.skipTest("Skipping zoneinfo tests on Windows") + try: + self.tz = ZoneInfo.fromname(self.zonename) + except FileNotFoundError as err: + self.skipTest("Skipping %s: %s" % (self.zonename, err)) + + def assertEquivDatetimes(self, a, b): + self.assertEqual((a.replace(tzinfo=None), a.fold, id(a.tzinfo)), + (b.replace(tzinfo=None), b.fold, id(b.tzinfo))) + + def test_folds(self): + tz = self.tz + for dt, shift in tz.folds(): + for x in [0 * shift, 0.5 * shift, shift - timedelta.resolution]: + udt = dt + x + ldt = tz.fromutc(udt.replace(tzinfo=tz)) + self.assertEqual(ldt.fold, 1) + adt = udt.replace(tzinfo=timezone.utc).astimezone(tz) + self.assertEquivDatetimes(adt, ldt) + utcoffset = ldt.utcoffset() + self.assertEqual(ldt.replace(tzinfo=None), udt + utcoffset) + # Round trip + self.assertEquivDatetimes(ldt.astimezone(timezone.utc), + udt.replace(tzinfo=timezone.utc)) + + + for x in [-timedelta.resolution, shift]: + udt = dt + x + udt = udt.replace(tzinfo=tz) + ldt = tz.fromutc(udt) + self.assertEqual(ldt.fold, 0) + + def test_gaps(self): + tz = self.tz + for dt, shift in tz.gaps(): + for x in [0 * shift, 0.5 * shift, shift - timedelta.resolution]: + udt = dt + x + udt = udt.replace(tzinfo=tz) + ldt = tz.fromutc(udt) + self.assertEqual(ldt.fold, 0) + adt = udt.replace(tzinfo=timezone.utc).astimezone(tz) + self.assertEquivDatetimes(adt, ldt) + utcoffset = ldt.utcoffset() + self.assertEqual(ldt.replace(tzinfo=None), udt.replace(tzinfo=None) + utcoffset) + # Create a local time inside the gap + ldt = tz.fromutc(dt.replace(tzinfo=tz)) - shift + x + self.assertLess(ldt.replace(fold=1).utcoffset(), + ldt.replace(fold=0).utcoffset(), + "At %s." % ldt) + + for x in [-timedelta.resolution, shift]: + udt = dt + x + ldt = tz.fromutc(udt.replace(tzinfo=tz)) + self.assertEqual(ldt.fold, 0) + + @classmethod + @contextlib.contextmanager + def _change_tz(cls, new_tzinfo): + try: + with os_helper.EnvironmentVarGuard() as env: + env["TZ"] = new_tzinfo + _time.tzset() + yield + finally: + _time.tzset() + + @unittest.skipUnless( + hasattr(_time, "tzset"), "time module has no attribute tzset" + ) + def test_system_transitions(self): + if ('Riyadh8' in self.zonename or + # From tzdata NEWS file: + # The files solar87, solar88, and solar89 are no longer distributed. + # They were a negative experiment - that is, a demonstration that + # tz data can represent solar time only with some difficulty and error. + # Their presence in the distribution caused confusion, as Riyadh + # civil time was generally not solar time in those years. + self.zonename.startswith('right/')): + self.skipTest("Skipping %s" % self.zonename) + tz = self.tz + with self._change_tz(self.zonename): + for udt, shift in tz.transitions(): + if udt.year >= 2037: + # System support for times around the end of 32-bit time_t + # and later is flaky on many systems. + break + s0 = (udt - datetime(1970, 1, 1)) // SEC + ss = shift // SEC # shift seconds + for x in [-40 * 3600, -20 * 3600, -1, 0, + ss - 1, ss + 20 * 3600, ss + 40 * 3600]: + s = s0 + x + sdt = datetime.fromtimestamp(s) + tzdt = datetime.fromtimestamp(s, tz).replace(tzinfo=None) + self.assertEquivDatetimes(sdt, tzdt) + s1 = sdt.timestamp() + self.assertEqual(s, s1) + if ss > 0: # gap + # Create local time inside the gap + dt = datetime.fromtimestamp(s0) - shift / 2 + ts0 = dt.timestamp() + ts1 = dt.replace(fold=1).timestamp() + self.assertEqual(ts0, s0 + ss / 2) + self.assertEqual(ts1, s0 - ss / 2) + # gh-83861 + utc0 = dt.astimezone(timezone.utc) + utc1 = dt.replace(fold=1).astimezone(timezone.utc) + self.assertEqual(utc0, utc1 + timedelta(0, ss)) + + +class ZoneInfoCompleteTest(unittest.TestSuite): + def __init__(self): + tests = [] + if is_resource_enabled('tzdata'): + for name in ZoneInfo.zonenames(): + Test = type('ZoneInfoTest[%s]' % name, (ZoneInfoTest,), {}) + Test.zonename = name + for method in dir(Test): + if method.startswith('test_'): + tests.append(Test(method)) + super().__init__(tests) + +# Iran had a sub-minute UTC offset before 1946. +class IranTest(ZoneInfoTest): + zonename = 'Asia/Tehran' + + +@unittest.skipIf(_testcapi is None, 'need _testcapi module') +class CapiTest(unittest.TestCase): + def setUp(self): + # Since the C API is not present in the _Pure tests, skip all tests + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + # This *must* be called, and it must be called first, so until either + # restriction is loosened, we'll call it as part of test setup + _testcapi.test_datetime_capi() + + def test_utc_capi(self): + for use_macro in (True, False): + capi_utc = _testcapi.get_timezone_utc_capi(use_macro) + + with self.subTest(use_macro=use_macro): + self.assertIs(capi_utc, timezone.utc) + + def test_timezones_capi(self): + est_capi, est_macro, est_macro_nn = _testcapi.make_timezones_capi() + + exp_named = timezone(timedelta(hours=-5), "EST") + exp_unnamed = timezone(timedelta(hours=-5)) + + cases = [ + ('est_capi', est_capi, exp_named), + ('est_macro', est_macro, exp_named), + ('est_macro_nn', est_macro_nn, exp_unnamed) + ] + + for name, tz_act, tz_exp in cases: + with self.subTest(name=name): + self.assertEqual(tz_act, tz_exp) + + dt1 = datetime(2000, 2, 4, tzinfo=tz_act) + dt2 = datetime(2000, 2, 4, tzinfo=tz_exp) + + self.assertEqual(dt1, dt2) + self.assertEqual(dt1.tzname(), dt2.tzname()) + + dt_utc = datetime(2000, 2, 4, 5, tzinfo=timezone.utc) + + self.assertEqual(dt1.astimezone(timezone.utc), dt_utc) + + def test_PyDateTime_DELTA_GET(self): + class TimeDeltaSubclass(timedelta): + pass + + for klass in [timedelta, TimeDeltaSubclass]: + for args in [(26, 55, 99999), (26, 55, 99999)]: + d = klass(*args) + with self.subTest(cls=klass, date=args): + days, seconds, microseconds = _testcapi.PyDateTime_DELTA_GET(d) + + self.assertEqual(days, d.days) + self.assertEqual(seconds, d.seconds) + self.assertEqual(microseconds, d.microseconds) + + def test_PyDateTime_GET(self): + class DateSubclass(date): + pass + + for klass in [date, DateSubclass]: + for args in [(2000, 1, 2), (2012, 2, 29)]: + d = klass(*args) + with self.subTest(cls=klass, date=args): + year, month, day = _testcapi.PyDateTime_GET(d) + + self.assertEqual(year, d.year) + self.assertEqual(month, d.month) + self.assertEqual(day, d.day) + + def test_PyDateTime_DATE_GET(self): + class DateTimeSubclass(datetime): + pass + + for klass in [datetime, DateTimeSubclass]: + for args in [(1993, 8, 26, 22, 12, 55, 99999), + (1993, 8, 26, 22, 12, 55, 99999, + timezone.utc)]: + d = klass(*args) + with self.subTest(cls=klass, date=args): + hour, minute, second, microsecond, tzinfo = \ + _testcapi.PyDateTime_DATE_GET(d) + + self.assertEqual(hour, d.hour) + self.assertEqual(minute, d.minute) + self.assertEqual(second, d.second) + self.assertEqual(microsecond, d.microsecond) + self.assertIs(tzinfo, d.tzinfo) + + def test_PyDateTime_TIME_GET(self): + class TimeSubclass(time): + pass + + for klass in [time, TimeSubclass]: + for args in [(12, 30, 20, 10), + (12, 30, 20, 10, timezone.utc)]: + d = klass(*args) + with self.subTest(cls=klass, date=args): + hour, minute, second, microsecond, tzinfo = \ + _testcapi.PyDateTime_TIME_GET(d) + + self.assertEqual(hour, d.hour) + self.assertEqual(minute, d.minute) + self.assertEqual(second, d.second) + self.assertEqual(microsecond, d.microsecond) + self.assertIs(tzinfo, d.tzinfo) + + def test_timezones_offset_zero(self): + utc0, utc1, non_utc = _testcapi.get_timezones_offset_zero() + + with self.subTest(testname="utc0"): + self.assertIs(utc0, timezone.utc) + + with self.subTest(testname="utc1"): + self.assertIs(utc1, timezone.utc) + + with self.subTest(testname="non_utc"): + self.assertIsNot(non_utc, timezone.utc) + + non_utc_exp = timezone(timedelta(hours=0), "") + + self.assertEqual(non_utc, non_utc_exp) + + dt1 = datetime(2000, 2, 4, tzinfo=non_utc) + dt2 = datetime(2000, 2, 4, tzinfo=non_utc_exp) + + self.assertEqual(dt1, dt2) + self.assertEqual(dt1.tzname(), dt2.tzname()) + + def test_check_date(self): + class DateSubclass(date): + pass + + d = date(2011, 1, 1) + ds = DateSubclass(2011, 1, 1) + dt = datetime(2011, 1, 1) + + is_date = _testcapi.datetime_check_date + + # Check the ones that should be valid + self.assertTrue(is_date(d)) + self.assertTrue(is_date(dt)) + self.assertTrue(is_date(ds)) + self.assertTrue(is_date(d, True)) + + # Check that the subclasses do not match exactly + self.assertFalse(is_date(dt, True)) + self.assertFalse(is_date(ds, True)) + + # Check that various other things are not dates at all + args = [tuple(), list(), 1, '2011-01-01', + timedelta(1), timezone.utc, time(12, 00)] + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_date(arg, exact)) + + def test_check_time(self): + class TimeSubclass(time): + pass + + t = time(12, 30) + ts = TimeSubclass(12, 30) + + is_time = _testcapi.datetime_check_time + + # Check the ones that should be valid + self.assertTrue(is_time(t)) + self.assertTrue(is_time(ts)) + self.assertTrue(is_time(t, True)) + + # Check that the subclass does not match exactly + self.assertFalse(is_time(ts, True)) + + # Check that various other things are not times + args = [tuple(), list(), 1, '2011-01-01', + timedelta(1), timezone.utc, date(2011, 1, 1)] + + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_time(arg, exact)) + + def test_check_datetime(self): + class DateTimeSubclass(datetime): + pass + + dt = datetime(2011, 1, 1, 12, 30) + dts = DateTimeSubclass(2011, 1, 1, 12, 30) + + is_datetime = _testcapi.datetime_check_datetime + + # Check the ones that should be valid + self.assertTrue(is_datetime(dt)) + self.assertTrue(is_datetime(dts)) + self.assertTrue(is_datetime(dt, True)) + + # Check that the subclass does not match exactly + self.assertFalse(is_datetime(dts, True)) + + # Check that various other things are not datetimes + args = [tuple(), list(), 1, '2011-01-01', + timedelta(1), timezone.utc, date(2011, 1, 1)] + + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_datetime(arg, exact)) + + def test_check_delta(self): + class TimeDeltaSubclass(timedelta): + pass + + td = timedelta(1) + tds = TimeDeltaSubclass(1) + + is_timedelta = _testcapi.datetime_check_delta + + # Check the ones that should be valid + self.assertTrue(is_timedelta(td)) + self.assertTrue(is_timedelta(tds)) + self.assertTrue(is_timedelta(td, True)) + + # Check that the subclass does not match exactly + self.assertFalse(is_timedelta(tds, True)) + + # Check that various other things are not timedeltas + args = [tuple(), list(), 1, '2011-01-01', + timezone.utc, date(2011, 1, 1), datetime(2011, 1, 1)] + + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_timedelta(arg, exact)) + + def test_check_tzinfo(self): + class TZInfoSubclass(tzinfo): + pass + + tzi = tzinfo() + tzis = TZInfoSubclass() + tz = timezone(timedelta(hours=-5)) + + is_tzinfo = _testcapi.datetime_check_tzinfo + + # Check the ones that should be valid + self.assertTrue(is_tzinfo(tzi)) + self.assertTrue(is_tzinfo(tz)) + self.assertTrue(is_tzinfo(tzis)) + self.assertTrue(is_tzinfo(tzi, True)) + + # Check that the subclasses do not match exactly + self.assertFalse(is_tzinfo(tz, True)) + self.assertFalse(is_tzinfo(tzis, True)) + + # Check that various other things are not tzinfos + args = [tuple(), list(), 1, '2011-01-01', + date(2011, 1, 1), datetime(2011, 1, 1)] + + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_tzinfo(arg, exact)) + + def test_date_from_date(self): + exp_date = date(1993, 8, 26) + + for macro in False, True: + with self.subTest(macro=macro): + c_api_date = _testcapi.get_date_fromdate( + macro, + exp_date.year, + exp_date.month, + exp_date.day) + + self.assertEqual(c_api_date, exp_date) + + def test_datetime_from_dateandtime(self): + exp_date = datetime(1993, 8, 26, 22, 12, 55, 99999) + + for macro in False, True: + with self.subTest(macro=macro): + c_api_date = _testcapi.get_datetime_fromdateandtime( + macro, + exp_date.year, + exp_date.month, + exp_date.day, + exp_date.hour, + exp_date.minute, + exp_date.second, + exp_date.microsecond) + + self.assertEqual(c_api_date, exp_date) + + def test_datetime_from_dateandtimeandfold(self): + exp_date = datetime(1993, 8, 26, 22, 12, 55, 99999) + + for fold in [0, 1]: + for macro in False, True: + with self.subTest(macro=macro, fold=fold): + c_api_date = _testcapi.get_datetime_fromdateandtimeandfold( + macro, + exp_date.year, + exp_date.month, + exp_date.day, + exp_date.hour, + exp_date.minute, + exp_date.second, + exp_date.microsecond, + exp_date.fold) + + self.assertEqual(c_api_date, exp_date) + self.assertEqual(c_api_date.fold, exp_date.fold) + + def test_time_from_time(self): + exp_time = time(22, 12, 55, 99999) + + for macro in False, True: + with self.subTest(macro=macro): + c_api_time = _testcapi.get_time_fromtime( + macro, + exp_time.hour, + exp_time.minute, + exp_time.second, + exp_time.microsecond) + + self.assertEqual(c_api_time, exp_time) + + def test_time_from_timeandfold(self): + exp_time = time(22, 12, 55, 99999) + + for fold in [0, 1]: + for macro in False, True: + with self.subTest(macro=macro, fold=fold): + c_api_time = _testcapi.get_time_fromtimeandfold( + macro, + exp_time.hour, + exp_time.minute, + exp_time.second, + exp_time.microsecond, + exp_time.fold) + + self.assertEqual(c_api_time, exp_time) + self.assertEqual(c_api_time.fold, exp_time.fold) + + def test_delta_from_dsu(self): + exp_delta = timedelta(26, 55, 99999) + + for macro in False, True: + with self.subTest(macro=macro): + c_api_delta = _testcapi.get_delta_fromdsu( + macro, + exp_delta.days, + exp_delta.seconds, + exp_delta.microseconds) + + self.assertEqual(c_api_delta, exp_delta) + + def test_date_from_timestamp(self): + ts = datetime(1995, 4, 12).timestamp() + + for macro in False, True: + with self.subTest(macro=macro): + d = _testcapi.get_date_fromtimestamp(int(ts), macro) + + self.assertEqual(d, date(1995, 4, 12)) + + def test_datetime_from_timestamp(self): + cases = [ + ((1995, 4, 12), None, False), + ((1995, 4, 12), None, True), + ((1995, 4, 12), timezone(timedelta(hours=1)), True), + ((1995, 4, 12, 14, 30), None, False), + ((1995, 4, 12, 14, 30), None, True), + ((1995, 4, 12, 14, 30), timezone(timedelta(hours=1)), True), + ] + + from_timestamp = _testcapi.get_datetime_fromtimestamp + for case in cases: + for macro in False, True: + with self.subTest(case=case, macro=macro): + dtup, tzinfo, usetz = case + dt_orig = datetime(*dtup, tzinfo=tzinfo) + ts = int(dt_orig.timestamp()) + + dt_rt = from_timestamp(ts, tzinfo, usetz, macro) + + self.assertEqual(dt_orig, dt_rt) + + def test_type_check_in_subinterp(self): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + + script = textwrap.dedent(f""" + if {_interpreters is None}: + import _testcapi as module + module.test_datetime_capi() + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + def run(type_checker, obj): + if not type_checker(obj, True): + raise TypeError(f'{{type(obj)}} is not C API type') + + import _datetime + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) + + +class ExtensionModuleTests(unittest.TestCase): + + def setUp(self): + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + @support.cpython_only + def test_gh_120161(self): + with self.subTest('simple'): + script = textwrap.dedent(""" + import datetime + from _ast import Tuple + f = lambda: None + Tuple.dims = property(f, f) + + class tzutc(datetime.tzinfo): + pass + """) + script_helper.assert_python_ok('-c', script) + + with self.subTest('complex'): + script = textwrap.dedent(""" + import asyncio + import datetime + from typing import Type + + class tzutc(datetime.tzinfo): + pass + _EPOCHTZ = datetime.datetime(1970, 1, 1, tzinfo=tzutc()) + + class FakeDateMeta(type): + def __instancecheck__(self, obj): + return True + class FakeDate(datetime.date, metaclass=FakeDateMeta): + pass + def pickle_fake_date(datetime_) -> Type[FakeDate]: + # A pickle function for FakeDate + return FakeDate + """) + script_helper.assert_python_ok('-c', script) + + def test_update_type_cache(self): + # gh-120782 + script = textwrap.dedent(""" + import sys + for i in range(5): + import _datetime + assert _datetime.date.max > _datetime.date.min + assert _datetime.time.max > _datetime.time.min + assert _datetime.datetime.max > _datetime.datetime.min + assert _datetime.timedelta.max > _datetime.timedelta.min + assert _datetime.date.__dict__["min"] is _datetime.date.min + assert _datetime.date.__dict__["max"] is _datetime.date.max + assert _datetime.date.__dict__["resolution"] is _datetime.date.resolution + assert _datetime.time.__dict__["min"] is _datetime.time.min + assert _datetime.time.__dict__["max"] is _datetime.time.max + assert _datetime.time.__dict__["resolution"] is _datetime.time.resolution + assert _datetime.datetime.__dict__["min"] is _datetime.datetime.min + assert _datetime.datetime.__dict__["max"] is _datetime.datetime.max + assert _datetime.datetime.__dict__["resolution"] is _datetime.datetime.resolution + assert _datetime.timedelta.__dict__["min"] is _datetime.timedelta.min + assert _datetime.timedelta.__dict__["max"] is _datetime.timedelta.max + assert _datetime.timedelta.__dict__["resolution"] is _datetime.timedelta.resolution + assert _datetime.timezone.__dict__["min"] is _datetime.timezone.min + assert _datetime.timezone.__dict__["max"] is _datetime.timezone.max + assert _datetime.timezone.__dict__["utc"] is _datetime.timezone.utc + assert isinstance(_datetime.timezone.min, _datetime.tzinfo) + assert isinstance(_datetime.timezone.max, _datetime.tzinfo) + assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) + del sys.modules['_datetime'] + """) + script_helper.assert_python_ok('-c', script) + + def test_concurrent_initialization_subinterpreter(self): + # gh-136421: Concurrent initialization of _datetime across multiple + # interpreters wasn't thread-safe due to its static types. + + # Run in a subprocess to ensure we get a clean version of _datetime + script = """if True: + from concurrent.futures import InterpreterPoolExecutor + + def func(): + import _datetime + print('a', end='') + + with InterpreterPoolExecutor() as executor: + for _ in range(8): + executor.submit(func) + """ + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + + # Now test against concurrent reinitialization + script = "import _datetime\n" + script + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + + +def load_tests(loader, standard_tests, pattern): + standard_tests.addTest(ZoneInfoCompleteTest()) + return standard_tests + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/decimaltestdata/abs.decTest b/Lib/test/decimaltestdata/abs.decTest index 01f73d77666..569b8fcd84a 100644 --- a/Lib/test/decimaltestdata/abs.decTest +++ b/Lib/test/decimaltestdata/abs.decTest @@ -20,7 +20,7 @@ version: 2.59 -- This set of tests primarily tests the existence of the operator. --- Additon, subtraction, rounding, and more overflows are tested +-- Addition, subtraction, rounding, and more overflows are tested -- elsewhere. precision: 9 diff --git a/Lib/test/decimaltestdata/ddFMA.decTest b/Lib/test/decimaltestdata/ddFMA.decTest index 9094fc015bd..7f2e5230374 100644 --- a/Lib/test/decimaltestdata/ddFMA.decTest +++ b/Lib/test/decimaltestdata/ddFMA.decTest @@ -1663,7 +1663,7 @@ ddfma375087 fma 1 12345678 1E-33 -> 12345678.00000001 Inexac ddfma375088 fma 1 12345678 1E-34 -> 12345678.00000001 Inexact Rounded ddfma375089 fma 1 12345678 1E-35 -> 12345678.00000001 Inexact Rounded --- desctructive subtraction (from remainder tests) +-- destructive subtraction (from remainder tests) -- +++ some of these will be off-by-one remainder vs remainderNear diff --git a/Lib/test/decimaltestdata/ddQuantize.decTest b/Lib/test/decimaltestdata/ddQuantize.decTest index 91776201694..e1c5674d9ac 100644 --- a/Lib/test/decimaltestdata/ddQuantize.decTest +++ b/Lib/test/decimaltestdata/ddQuantize.decTest @@ -462,7 +462,7 @@ ddqua520 quantize 1.234 1e359 -> 0E+359 Inexact Rounded ddqua521 quantize 123.456 1e359 -> 0E+359 Inexact Rounded ddqua522 quantize 1.234 1e359 -> 0E+359 Inexact Rounded ddqua523 quantize 123.456 1e359 -> 0E+359 Inexact Rounded --- next four are "won't fit" overfl +-- next four are "won't fit" overflow ddqua526 quantize 1.234 1e-299 -> NaN Invalid_operation ddqua527 quantize 123.456 1e-299 -> NaN Invalid_operation ddqua528 quantize 1.234 1e-299 -> NaN Invalid_operation diff --git a/Lib/test/decimaltestdata/ddRemainder.decTest b/Lib/test/decimaltestdata/ddRemainder.decTest index 5bd1e32d01e..b1866d39a28 100644 --- a/Lib/test/decimaltestdata/ddRemainder.decTest +++ b/Lib/test/decimaltestdata/ddRemainder.decTest @@ -422,7 +422,7 @@ ddrem757 remainder 1 sNaN -> NaN Invalid_operation ddrem758 remainder 1000 sNaN -> NaN Invalid_operation ddrem759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs ddrem760 remainder NaN1 NaN7 -> NaN1 ddrem761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation ddrem762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/ddRemainderNear.decTest b/Lib/test/decimaltestdata/ddRemainderNear.decTest index 6ba64ebafe9..bbe82ea3747 100644 --- a/Lib/test/decimaltestdata/ddRemainderNear.decTest +++ b/Lib/test/decimaltestdata/ddRemainderNear.decTest @@ -450,7 +450,7 @@ ddrmn757 remaindernear 1 sNaN -> NaN Invalid_operation ddrmn758 remaindernear 1000 sNaN -> NaN Invalid_operation ddrmn759 remaindernear Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs ddrmn760 remaindernear NaN1 NaN7 -> NaN1 ddrmn761 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation ddrmn762 remaindernear NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/dqRemainder.decTest b/Lib/test/decimaltestdata/dqRemainder.decTest index bae8eae5269..e0aaca3747e 100644 --- a/Lib/test/decimaltestdata/dqRemainder.decTest +++ b/Lib/test/decimaltestdata/dqRemainder.decTest @@ -418,7 +418,7 @@ dqrem757 remainder 1 sNaN -> NaN Invalid_operation dqrem758 remainder 1000 sNaN -> NaN Invalid_operation dqrem759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs dqrem760 remainder NaN1 NaN7 -> NaN1 dqrem761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation dqrem762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/dqRemainderNear.decTest b/Lib/test/decimaltestdata/dqRemainderNear.decTest index b850626fe4e..2c5c3f5074e 100644 --- a/Lib/test/decimaltestdata/dqRemainderNear.decTest +++ b/Lib/test/decimaltestdata/dqRemainderNear.decTest @@ -450,7 +450,7 @@ dqrmn757 remaindernear 1 sNaN -> NaN Invalid_operation dqrmn758 remaindernear 1000 sNaN -> NaN Invalid_operation dqrmn759 remaindernear Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs dqrmn760 remaindernear NaN1 NaN7 -> NaN1 dqrmn761 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation dqrmn762 remaindernear NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/exp.decTest b/Lib/test/decimaltestdata/exp.decTest index 6a7af23b625..e01d7a8f92d 100644 --- a/Lib/test/decimaltestdata/exp.decTest +++ b/Lib/test/decimaltestdata/exp.decTest @@ -28,7 +28,7 @@ rounding: half_even maxExponent: 384 minexponent: -383 --- basics (examples in specificiation, etc.) +-- basics (examples in specification, etc.) expx001 exp -Infinity -> 0 expx002 exp -10 -> 0.0000453999298 Inexact Rounded expx003 exp -1 -> 0.367879441 Inexact Rounded diff --git a/Lib/test/decimaltestdata/extra.decTest b/Lib/test/decimaltestdata/extra.decTest index b630d8e3f9d..31291202a35 100644 --- a/Lib/test/decimaltestdata/extra.decTest +++ b/Lib/test/decimaltestdata/extra.decTest @@ -156,7 +156,7 @@ extr1302 fma -Inf 0E-456 sNaN148 -> NaN Invalid_operation -- max/min/max_mag/min_mag bug in 2.5.2/2.6/3.0: max(NaN, finite) gave -- incorrect answers when the finite number required rounding; similarly --- for the other thre functions +-- for the other three functions maxexponent: 999 minexponent: -999 precision: 6 diff --git a/Lib/test/decimaltestdata/remainder.decTest b/Lib/test/decimaltestdata/remainder.decTest index 7a1061b1e6f..4f59b332877 100644 --- a/Lib/test/decimaltestdata/remainder.decTest +++ b/Lib/test/decimaltestdata/remainder.decTest @@ -435,7 +435,7 @@ remx757 remainder 1 sNaN -> NaN Invalid_operation remx758 remainder 1000 sNaN -> NaN Invalid_operation remx759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs remx760 remainder NaN1 NaN7 -> NaN1 remx761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation remx762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/remainderNear.decTest b/Lib/test/decimaltestdata/remainderNear.decTest index b768b9e0cf6..000b1424d8a 100644 --- a/Lib/test/decimaltestdata/remainderNear.decTest +++ b/Lib/test/decimaltestdata/remainderNear.decTest @@ -498,7 +498,7 @@ rmnx758 remaindernear 1000 sNaN -> NaN Invalid_operation rmnx759 remaindernear Inf sNaN -> NaN Invalid_operation rmnx760 remaindernear NaN sNaN -> NaN Invalid_operation --- propaging NaNs +-- propagating NaNs rmnx761 remaindernear NaN1 NaN7 -> NaN1 rmnx762 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation rmnx763 remaindernear NaN3 -sNaN9 -> -NaN9 Invalid_operation diff --git a/Lib/test/dis_module.py b/Lib/test/dis_module.py new file mode 100644 index 00000000000..afbf600fdee --- /dev/null +++ b/Lib/test/dis_module.py @@ -0,0 +1,5 @@ + +# A simple module for testing the dis module. + +def f(): pass +def g(): pass diff --git a/Lib/test/dtracedata/assert_usable.d b/Lib/test/dtracedata/assert_usable.d new file mode 100644 index 00000000000..0b2d4da66e0 --- /dev/null +++ b/Lib/test/dtracedata/assert_usable.d @@ -0,0 +1,5 @@ +BEGIN +{ + printf("probe: success\n"); + exit(0); +} diff --git a/Lib/test/dtracedata/assert_usable.stp b/Lib/test/dtracedata/assert_usable.stp new file mode 100644 index 00000000000..88e7e68e2cc --- /dev/null +++ b/Lib/test/dtracedata/assert_usable.stp @@ -0,0 +1,5 @@ +probe begin +{ + println("probe: success") + exit () +} diff --git a/Lib/test/dtracedata/call_stack.d b/Lib/test/dtracedata/call_stack.d new file mode 100644 index 00000000000..761d30fd559 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.d @@ -0,0 +1,31 @@ +self int indent; + +python$target:::function-entry +/copyinstr(arg1) == "start"/ +{ + self->trace = 1; +} + +python$target:::function-entry +/self->trace/ +{ + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); + self->indent++; +} + +python$target:::function-return +/self->trace/ +{ + self->indent--; + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); +} + +python$target:::function-return +/copyinstr(arg1) == "start"/ +{ + self->trace = 0; +} diff --git a/Lib/test/dtracedata/call_stack.d.expected b/Lib/test/dtracedata/call_stack.d.expected new file mode 100644 index 00000000000..27849d15495 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.d.expected @@ -0,0 +1,18 @@ + function-entry:call_stack.py:start:23 + function-entry: call_stack.py:function_1:1 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 +function-return: call_stack.py:function_1:2 + function-entry: call_stack.py:function_2:5 + function-entry: call_stack.py:function_1:1 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 +function-return: call_stack.py:function_1:2 +function-return: call_stack.py:function_2:6 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 + function-entry: call_stack.py:function_4:13 +function-return: call_stack.py:function_4:14 + function-entry: call_stack.py:function_5:18 +function-return: call_stack.py:function_5:21 +function-return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/call_stack.py b/Lib/test/dtracedata/call_stack.py new file mode 100644 index 00000000000..ee9f3ae8d6c --- /dev/null +++ b/Lib/test/dtracedata/call_stack.py @@ -0,0 +1,30 @@ +def function_1(): + function_3(1, 2) + +# Check stacktrace +def function_2(): + function_1() + +# CALL_FUNCTION_VAR +def function_3(dummy, dummy2): + pass + +# CALL_FUNCTION_KW +def function_4(**dummy): + return 1 + return 2 # unreachable + +# CALL_FUNCTION_VAR_KW +def function_5(dummy, dummy2, **dummy3): + if False: + return 7 + return 8 + +def start(): + function_1() + function_2() + function_3(1, 2) + function_4(test=42) + function_5(*(1, 2), **{"test": 42}) + +start() diff --git a/Lib/test/dtracedata/call_stack.stp b/Lib/test/dtracedata/call_stack.stp new file mode 100644 index 00000000000..54082c202f6 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.stp @@ -0,0 +1,41 @@ +global tracing + +function basename:string(path:string) +{ + last_token = token = tokenize(path, "/"); + while (token != "") { + last_token = token; + token = tokenize("", "/"); + } + return last_token; +} + +probe process.mark("function__entry") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 1; + } +} + +probe process.mark("function__entry"), process.mark("function__return") +{ + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + + if (tracing) { + printf("%d\t%s:%s:%s:%d\n", gettimeofday_us(), $$name, + basename(filename), funcname, lineno); + } +} + +probe process.mark("function__return") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 0; + } +} diff --git a/Lib/test/dtracedata/call_stack.stp.expected b/Lib/test/dtracedata/call_stack.stp.expected new file mode 100644 index 00000000000..32cf396f820 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.stp.expected @@ -0,0 +1,14 @@ +function__entry:call_stack.py:start:23 +function__entry:call_stack.py:function_1:1 +function__return:call_stack.py:function_1:2 +function__entry:call_stack.py:function_2:5 +function__entry:call_stack.py:function_1:1 +function__return:call_stack.py:function_1:2 +function__return:call_stack.py:function_2:6 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__entry:call_stack.py:function_4:13 +function__return:call_stack.py:function_4:14 +function__entry:call_stack.py:function_5:18 +function__return:call_stack.py:function_5:21 +function__return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/gc.d b/Lib/test/dtracedata/gc.d new file mode 100644 index 00000000000..4d91487b7e9 --- /dev/null +++ b/Lib/test/dtracedata/gc.d @@ -0,0 +1,18 @@ +python$target:::function-entry +/copyinstr(arg1) == "start"/ +{ + self->trace = 1; +} + +python$target:::gc-start, +python$target:::gc-done +/self->trace/ +{ + printf("%d\t%s:%ld\n", timestamp, probename, arg0); +} + +python$target:::function-return +/copyinstr(arg1) == "start"/ +{ + self->trace = 0; +} diff --git a/Lib/test/dtracedata/gc.d.expected b/Lib/test/dtracedata/gc.d.expected new file mode 100644 index 00000000000..8e5ac2a6d56 --- /dev/null +++ b/Lib/test/dtracedata/gc.d.expected @@ -0,0 +1,8 @@ +gc-start:0 +gc-done:0 +gc-start:1 +gc-done:0 +gc-start:2 +gc-done:0 +gc-start:2 +gc-done:1 diff --git a/Lib/test/dtracedata/gc.py b/Lib/test/dtracedata/gc.py new file mode 100644 index 00000000000..144a783b7b7 --- /dev/null +++ b/Lib/test/dtracedata/gc.py @@ -0,0 +1,13 @@ +import gc + +def start(): + gc.collect(0) + gc.collect(1) + gc.collect(2) + l = [] + l.append(l) + del l + gc.collect(2) + +gc.collect() +start() diff --git a/Lib/test/dtracedata/gc.stp b/Lib/test/dtracedata/gc.stp new file mode 100644 index 00000000000..162c6d3a220 --- /dev/null +++ b/Lib/test/dtracedata/gc.stp @@ -0,0 +1,26 @@ +global tracing + +probe process.mark("function__entry") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 1; + } +} + +probe process.mark("gc__start"), process.mark("gc__done") +{ + if (tracing) { + printf("%d\t%s:%ld\n", gettimeofday_us(), $$name, $arg1); + } +} + +probe process.mark("function__return") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 0; + } +} diff --git a/Lib/test/dtracedata/gc.stp.expected b/Lib/test/dtracedata/gc.stp.expected new file mode 100644 index 00000000000..7e6e6227fba --- /dev/null +++ b/Lib/test/dtracedata/gc.stp.expected @@ -0,0 +1,8 @@ +gc__start:0 +gc__done:0 +gc__start:1 +gc__done:0 +gc__start:2 +gc__done:0 +gc__start:2 +gc__done:1 diff --git a/Lib/test/dtracedata/instance.py b/Lib/test/dtracedata/instance.py new file mode 100644 index 00000000000..f1421378b00 --- /dev/null +++ b/Lib/test/dtracedata/instance.py @@ -0,0 +1,24 @@ +import gc + +class old_style_class(): + pass +class new_style_class(object): + pass + +a = old_style_class() +del a +gc.collect() +b = new_style_class() +del b +gc.collect() + +a = old_style_class() +del old_style_class +gc.collect() +b = new_style_class() +del new_style_class +gc.collect() +del a +gc.collect() +del b +gc.collect() diff --git a/Lib/test/dtracedata/line.d b/Lib/test/dtracedata/line.d new file mode 100644 index 00000000000..03f22db6fcc --- /dev/null +++ b/Lib/test/dtracedata/line.d @@ -0,0 +1,7 @@ +python$target:::line +/(copyinstr(arg1)=="test_line")/ +{ + printf("%d\t%s:%s:%s:%d\n", timestamp, + probename, basename(copyinstr(arg0)), + copyinstr(arg1), arg2); +} diff --git a/Lib/test/dtracedata/line.d.expected b/Lib/test/dtracedata/line.d.expected new file mode 100644 index 00000000000..9b16ce76ee6 --- /dev/null +++ b/Lib/test/dtracedata/line.d.expected @@ -0,0 +1,20 @@ +line:line.py:test_line:2 +line:line.py:test_line:3 +line:line.py:test_line:4 +line:line.py:test_line:5 +line:line.py:test_line:6 +line:line.py:test_line:7 +line:line.py:test_line:8 +line:line.py:test_line:9 +line:line.py:test_line:10 +line:line.py:test_line:11 +line:line.py:test_line:4 +line:line.py:test_line:5 +line:line.py:test_line:6 +line:line.py:test_line:7 +line:line.py:test_line:8 +line:line.py:test_line:10 +line:line.py:test_line:11 +line:line.py:test_line:4 +line:line.py:test_line:12 +line:line.py:test_line:13 diff --git a/Lib/test/dtracedata/line.py b/Lib/test/dtracedata/line.py new file mode 100644 index 00000000000..0930ff391f7 --- /dev/null +++ b/Lib/test/dtracedata/line.py @@ -0,0 +1,17 @@ +def test_line(): + a = 1 + print('# Preamble', a) + for i in range(2): + a = i + b = i+2 + c = i+3 + if c < 4: + a = c + d = a + b +c + print('#', a, b, c, d) + a = 1 + print('# Epilogue', a) + + +if __name__ == '__main__': + test_line() diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py new file mode 100644 index 00000000000..8c32895f5e0 --- /dev/null +++ b/Lib/test/fork_wait.py @@ -0,0 +1,80 @@ +"""This test case provides support for checking forking and wait behavior. + +To test different wait behavior, override the wait_impl method. + +We want fork1() semantics -- only the forking thread survives in the +child after a fork(). + +On some systems (e.g. Solaris without posix threads) we find that all +active threads survive in the child after a fork(); this is an error. +""" + +import os, time, unittest +import threading +from test import support +from test.support import threading_helper +import warnings + + +LONGSLEEP = 2 +SHORTSLEEP = 0.5 +NUM_THREADS = 4 + +class ForkWait(unittest.TestCase): + + def setUp(self): + self._threading_key = threading_helper.threading_setup() + self.alive = {} + self.stop = 0 + self.threads = [] + + def tearDown(self): + # Stop threads + self.stop = 1 + for thread in self.threads: + thread.join() + thread = None + self.threads.clear() + threading_helper.threading_cleanup(*self._threading_key) + + def f(self, id): + while not self.stop: + self.alive[id] = os.getpid() + try: + time.sleep(SHORTSLEEP) + except OSError: + pass + + def wait_impl(self, cpid, *, exitcode): + support.wait_process(cpid, exitcode=exitcode) + + def test_wait(self): + for i in range(NUM_THREADS): + thread = threading.Thread(target=self.f, args=(i,)) + thread.start() + self.threads.append(thread) + + # busy-loop to wait for threads + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(self.alive) >= NUM_THREADS: + break + + a = sorted(self.alive.keys()) + self.assertEqual(a, list(range(NUM_THREADS))) + + prefork_lives = self.alive.copy() + + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + if (cpid := os.fork()) == 0: + # Child + time.sleep(LONGSLEEP) + n = 0 + for key in self.alive: + if self.alive[key] != prefork_lives[key]: + n += 1 + os._exit(n) + else: + # Parent + self.wait_impl(cpid, exitcode=0) diff --git a/Lib/test/future_test2.py b/Lib/test/future_test2.py deleted file mode 100644 index 3d7fc860a37..00000000000 --- a/Lib/test/future_test2.py +++ /dev/null @@ -1,10 +0,0 @@ -"""This is a test""" - -from __future__ import nested_scopes; import site - -def f(x): - def g(y): - return x + y - return g - -result = f(2)(4) diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem deleted file mode 100644 index 0ad69605519..00000000000 --- a/Lib/test/keycert.passwd.pem +++ /dev/null @@ -1,50 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,E74528136B90D2DD - -WRHVD2PJXPqjFSHg92HURIsUzvsTE4a9oi0SC5yMBFKNWA5Z933gK3XTifp6jul5 -zpNYi8jBXZ2EqJJBxCuVcefmXSxL0q7CMej25TdIC4BVAFJVveeprHPUFkNB0IM1 -go5Lg4YofYqTCg3OE3k7WvfR3Zg1cRYxksDKO+WNZgWyKBex5X4vjOiyUqDl3GKt -kQXnkg1VgPV2Vrx93S9XNdELNRTguwf+XG0fkhtYhp/zCto8uKTgy5elK2P/ulGp -7fe6uj7h/uN9L7EOC6CjRkitywfeBUER739mOcGT4imSFJ9G27TCqPzj2ea3uuaf -/v1xhkQ4M6lNY/gcRfgVpCXhW43aAQV8XXQRMJTqLmz5Y5hYTKn+Ugq5vJ/ngyRM -lu1gUJnYYaemBTb4hbm6aBvnYK9mORa891Pmf+vxU9rYuQIdVAhvvXh4KBreSEBI -1AFy6dFKXl8ZKs6Wrq5wPefmFFkRmZ8OBiiq0fp2ApCRGZw6LsjCgwrRM38JiY7d -3OdsJpKvRYufgUyuuzUE0xA+E4yMvD48M9pPq2fC8O5giuGL1uEekQWXJuq+6ZRI -XYKIeSkuQALbX3RAzCPXTUEMtCYXKm/gxrrwJ+Bet4ob2amf3MX0uvWwOuAq++Fk -J0HFSBxrwvIWOhyQXOPuJdAN8PXA7dWOXfOgOMF0hQYqZCl3T4TiVZJbwVQtg1sN -dO7oAD5ZPHiKzvveZuB6k1FlBG8j0TyAC+44ChxkPDD3jF4dd6zGe62sDf85p4/d -W80gxJeD3xnDxG0ePPns+GuKUpUaWS7WvHtDpeFW1JEhvOqf8p1Li9a7RzWVo8ML -mGTdQgBIYIf6/fk69pFKl0nKtBU75KaunZz4nAmd9bNED4naDurMBg44u5TvODbJ -vgYIYXIYjNvONbskJatVrrTS8zch2NwVIjCi8L/hecwBXbIXzo1pECpc6BU7sQT8 -+i9sDKBeJcRipzfKZNHvnO19mUZaPCY8+a/f9c21DgKXz+bgLcJbohpSaeGM8Gfc -aZd3Vp9n3OJ3g2zQR1++HO9v1vR/wLELu6MeydkvMduHLmOPCn54gZ9z51ZNPAwa -qfFIsH+mLh9ks0H74ssF59uIlstkgB9zmZHv/Q0dK9ZfG/VEH6rSgdETWhZxhoMQ -Z92jXBEFT0zhI3rrIPNY+XS7eJCQIc1wc84Ea3cRk7SP+S1og3JtAxX56ykUwtkM -LQ/Dwwa6h1aqD0l2d5x1/BSdavtTuSegISRWQ4iOmSvEdlFP7H4g6RZk/okbLzMD -Evq5gNc7vlXhVawoQU8JCanJ5ZbbWnIRZfiXxBQS4lpYPKvJt4ML9z/x+82XxcXv -Z93N2Wep7wWW5OwS2LcQcOgZRDSIPompwo/0pMFGOS+5oort0ZDRHdmmGLjvBcCb -1KQmKQ4+8brI/3rjRzts6uDLjTGNxSCieNsnqhwHUv9Mg9WDSWupcGa+x27L89x3 -rObf6+3umcFLSjIzU8wuv1hx/e/y98Kv7BDBNYpAr6kVMrLnzYjAfJbBmqlxkzkQ -IgQzgrk2QZoTdgwR+S374NAMO0AE5IlO+/qa6qp2SORGTDX64I3UNw== ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx -MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 -RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj -9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj -SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN -sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn -96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ -bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK -Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk -ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 -qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu -4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS -dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF ------END CERTIFICATE----- diff --git a/Lib/test/keycert.pem b/Lib/test/keycert.pem deleted file mode 100644 index 9545dcf4b94..00000000000 --- a/Lib/test/keycert.pem +++ /dev/null @@ -1,48 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCr77F9oBLYuLIb -3t4mDHPoxOEnu1h7NQaJdn9tR/KjW6AzhXXm9USh45qt3sR6Wo8sDlBIJ3vZyhyW -P939qYUeGOZJahupYi4IkqVckZXARm3k9qhAI/aC4ixVFyxnxg3bNpk8Ir0AyyQV -oudY+33+ZNj6+fHzeboGDJ9uE/RTcp9JqE/qo0haATZufJfY63ZCUpYFn6j5W4jG -vpqg5/0hba8Cxdk62387uNknfVHSMzGkkq82zbBpud6TYQofCp3VlEPBjz9iLCz6 -FFOKgLZmbk8QsdktXF6zfRJJk+vbZTh/OGH0p/eiIfW1kXOzOcuW31XRFTPnykJd -4QUX9OajAgMBAAECggEAHppmXDbuw9Z0FVPg9KLIysioTtsgz6VLiZIm8juZK4x2 -glUh/D7xvWL2uDXrgN+3lh7iGUW13LkFx5SMncbbo9TIwI57Z/XKvcnkVwquve+L -RfLFVc1Q5lD9lROv2rS86KTaN4LzYz3FKXi6dvMkpPAsUtfEQhMLkmISypQQq/1z -EJaqo7r85OjN7e0wKazlKZpOzJEa5FQLMVRjTRFhLFNbHXX/tAet2jw+umATKbw8 -hYgiuZ44TwSEd9JeIV/oSYWfI/3HetuYW0ru3caiztRF2NySNu8lcsWgNC7fIku9 -mcHjtSNzs91QN1Qlu7GQvvhpt6OWDirNDCW+49WGaQKBgQDg9SDhfF0jRYslgYbH -cqO4ggaFdHjrAAYpwnAgvanhFZL/zEqm5G1E7l/e2fCkJ9VOSFO0A208chvwMcr+ -dCjHE2tVdE81aQ2v/Eo83VdS1RcOV4Y75yPH48rMhxPaHvxWD/FFDbf0/P2mtPB7 -SZ3kIeZMkE1wxdaO3AKUbQoozwKBgQDDqYgg7kVtygyICE1mB8Hwp6nUxFTczG7y -4XcsDqMIrKmw+PbQluvkoHoStxeVrsTloDhkTjIrpmYLyAiazg+PUJdkd6xrfLSj -VV6X93W0S/1egEb1F1CGFxtk8v/PWH4K76EPL2vxXdxjywz3GWlrL9yDYaB2szzS -DqgwVMqx7QKBgDCD7UF0Bsoyl13RX3XoPXLvZ+SkR+e2q52Z94C4JskKVBeiwX7Y -yNAS8M4pBoMArDoj0xmBm69rlKbqtjLGbnzwrTdSzDpim7cWnBQgUFLm7gAD1Elb -AhZ8BCK0Bw4FnLoa2hfga4oEfdfUMgEE0W5/+SEOBgWKRUmuHUhRc911AoGAY2EN -YmSDYSM5wDIvVb5k9B3EtevOiqNPSw/XnsoEZtiEC/44JnQxdltIBY93bDBrk5IQ -cmoBM4h91kgQjshQwOMXMhFSwvmBKmCm/hrTbvMVytTutXfVD3ZXFKwT4DW7N0TF -ElhsxBh/YzRz7mG62JVjtFt2zDN3ld2Z8YpvtXUCgYEA4EJ4ObS5YyvcXAKHJFo6 -Fxmavyrf8LSm3MFA65uSnFvWukMVqqRMReQc5jvpxHKCis+XvnHzyOfL0gW9ZTi7 -tWGGbBi0TRJCa8BkvgngUZxOxUlMfg/7cVxOIB0TPoUSgxFd/+qVz4GZMvr0dPu7 -eAF7J/8ECVvb0wSPTUI1N3c= ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx -MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 -RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj -9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj -SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN -sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn -96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ -bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK -Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk -ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 -qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu -4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS -dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF ------END CERTIFICATE----- diff --git a/Lib/test/keycert2.pem b/Lib/test/keycert2.pem deleted file mode 100644 index bb5fa65a8ac..00000000000 --- a/Lib/test/keycert2.pem +++ /dev/null @@ -1,49 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC3ulRNfhbOAey/ -B+wIVYx+d5az7EV4riR6yi/qE6G+bxbTvay2pqySHtDweuaYSh2cVmcasBKKIFJm -rCD1zR8UmLb5i2XFIina1t3eePCuBZMrvZZwkzlQUSM1AZtjGOO/W0I3FwO6y645 -9xA5PduKI7SMYkH/VL3zE5W1JwMovv6bvNiT+GU5l6mB9ylCTgLpmUqoQhRqz/35 -zCzVyoh+ppDvVcpWYfvXywsXsgQwbAF0QJm8SSFi0TZm5ykv4WE16afQp08yuZS0 -3U4K3MJCa4rxO58edcxBopWYfQ29K3iINM8enRfr5q+u5mAAbALAEEvyFjgLWl/u -7arxn7bJAgMBAAECggEBAJfMt8KfHzBunrDnVrk8FayYGkfmOzAOkc1yKEx6k/TH -zFB+Mqlm5MaF95P5t3S0J+r36JBAUdEWC38RUNpF9BwMYYGlDxzlsTdCuGYL/q+J -o6NMLXQt7/jQUQqGnWAvPFzqhbcGqOo5R2ZVH25sEWv9PDuRI35XAepIkDTwWsfa -P6UcJJoP+4v9B++fb3sSL4zNwp1BqS4wxR8YTR0t1zQqOxJ5BGPw1J8aBMs1sq5t -qyosAQAT63kLrdqWotHaM26QxjqEQUMlh12XMWb5GdBXUxbvyGtEabsqskGa/f8B -RdHE437J8D8l+jxb2mZLzrlaH3dq2tbFGCe1rT8qLRECgYEA5CWIvoD/YnQydLGA -OlEhCSocqURuqcotg9Ev0nt/C60jkr/NHFLGppz9lhqjIDjixt3sIMGZMFzxRtwM -pSYal3XiR7rZuHau9iM35yDhpuytEiGbYy1ADakJRzY5jq/Qa8RfPP9Atua5xAeP -q6DiSnq9vhHv9G+O4MxzHBmrw9sCgYEAziiJWFthcwvuXn3Jv9xFYKEb/06puZAx -EgQCz/3rPzv5fmGD/sKVo1U/K4z/eA82DNeKG8QRTFJCxT8TCNRxOmGV7HdCYo/B -4BTNNvbKcdi3l0j75kKoADg+nt5CD5lz6gLG0GrUEnVO1y5HVfCTb3BEAfa36C85 -9i0sfQGiwysCgYEAuus9k8cgdct5oz3iLuVVSark/JGCkT2B+OOkaLChsDFUWeEm -7TOsaclpwldkmvvAYOplkZjMJ2GelE2pVo1XcAw3LkmaI5WpVyQXoxe/iQGT8qzy -IFlsh0Scw2lb0tmcyw6CcPk4TiHOxRrkzNrtS9QwLM+JZx0XVHptPPKTVc0CgYAu -j/VFYY5G/8Dc0qhIjyWUR48dQNUQtkJ/ASzpcT46z/7vznKTjbtiYpSb74KbyUO5 -7sygrM4DYOj3x+Eys1jHiNbly6HQxQtS4x/edCsRP5NntfI+9XsgYZOzKhvdjhki -F3J0DEzNxnUCIM+311hVaRPTJbgv1srOkTFlIoNydQKBgQC6/OHGaC/OewQqRlRK -Mg5KZm01/pk4iKrpA5nG7OTAeoa70NzXNtG8J3WnaJ4mWanNwNUOyRMAMrsUAy9q -EeGqHM5mMFpY4TeVuNLL21lu/x3KYw6mKL3Ctinn+JLAoYoqEy8deZnEA5/tjYlz -YhFBchnUicjoUN1chdpM6SpV2Q== ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDYjCCAkqgAwIBAgIJALJXRr8qF6oIMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTAeFw0x -ODAxMTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMGIxCzAJBgNVBAYTAlhZMRcwFQYD -VQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZv -dW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBALe6VE1+Fs4B7L8H7AhVjH53lrPsRXiuJHrKL+oTob5v -FtO9rLamrJIe0PB65phKHZxWZxqwEoogUmasIPXNHxSYtvmLZcUiKdrW3d548K4F -kyu9lnCTOVBRIzUBm2MY479bQjcXA7rLrjn3EDk924ojtIxiQf9UvfMTlbUnAyi+ -/pu82JP4ZTmXqYH3KUJOAumZSqhCFGrP/fnMLNXKiH6mkO9VylZh+9fLCxeyBDBs -AXRAmbxJIWLRNmbnKS/hYTXpp9CnTzK5lLTdTgrcwkJrivE7nx51zEGilZh9Db0r -eIg0zx6dF+vmr67mYABsAsAQS/IWOAtaX+7tqvGftskCAwEAAaMbMBkwFwYDVR0R -BBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBCwUAA4IBAQCZhHhGItpkqhEq -ntMRd6Hv0GoOJixNvgeMwK4NJSRT/no3OirtUTzccn46h+SWibSa2eVssAV+pAVJ -HbzkN/DH27A1mMx1zJL1ekcOKA1AF6MXhUnrUGXMqW36YNtzHfXJLrwvpLJ13OQg -/Kxo4Nw68bGzM+PyRtKU/mpgYyfcvwR+ZSeIDh1fvUZK/IEVCf8ub42GPVs5wPfv -M+k5aHxWTxeif3K1byTRzxHupYNG2yWO4XEdnBGOuOwzzN4/iQyNcsuQKeuKHGrt -YvIlG/ri04CQ7xISZCj74yjTZ+/A2bXre2mQXAHqKPumHL7cl34+erzbUaxYxbTE -u5FcOmLQ ------END CERTIFICATE----- diff --git a/Lib/test/keycert3.pem b/Lib/test/keycert3.pem deleted file mode 100644 index 621eb08bb0c..00000000000 --- a/Lib/test/keycert3.pem +++ /dev/null @@ -1,132 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDgV4G+Zzf2DT5n -oAisIGFhn/bz7Vn5WiXUqbDsxROJOh/7BtOlduZka0pPhFylGbnxS8l1kEWHRI2T -6hOoWzumB6ItKiN+T5J30lAvSyo7iwdFoAQ/S5nPXQfhNARQe/NEOhRtpcuNdyx4 -BWdPdPuJQA1ASNJCLwcLOoRxaLbKLvb2V5T5FCAkeNPtRvPuT4gKQItMmiHfAhoV -C8MZWF/GC0RukHINys5MwqeFexam8CznmQPMYrLdhmKTj3DTivCPoh97EDIFGlgZ -SCaaYDVQA+aqlo/q2pi52PFwC1KzhNEA7EeOqSwC1NQjwjHuhcnf9WbxrgTq2zh3 -rv5YEW2ZAgMBAAECggEAPfSMtTumPcJskIumuXp7yk02EyliZrWZqwBuBwVqHsS5 -nkbFXnXWrLbgn9MrDsFrE5NdgKUmPnQVMVs8sIr5jyGejSCNCs4I4iRn1pfIgwcj -K/xEEALd6GGF0pDd/CgvB5GOoLVf4KKf2kmLvWrOKJpSzoUN5A8+v8AaYYOMr4sC -czbvfGomzEIewEG+Rw9zOVUDlmwyEKPQZ47E7PQ+EEA7oeFdR+1Zj6eT9ndegf8B -54frySYCLRUCk/sHCpWhaJBtBrcpht7Y8CfY7hiH/7x866fvuLnYPz4YALtUb0wN -7zUCNS9ol3n4LbjFFKfZtiRjKaCBRzMjK0rz6ydFcQKBgQDyLI3oGbnW73vqcDe/ -6eR0w++fiCAVhfMs3AO/gOaJi2la2JHlJ5u+cIHQIOFwEhn6Zq0AtdmnFx1TS5IQ -C0MdXI0XoQQw7rEF8EJcvfe85Z0QxENVhzydtdb8QpJfnQGfBfLyQlaaRYzRRHB6 -VdYUHF3EIPVIhbjbghal+Qep/QKBgQDtJlRPHkWwTMevu0J0fYbWN1ywtVTFUR// -k7VyORSf8yuuSnaQRop4cbcqONxmDKH6Or1fl3NYBsAxtXkkOK1E2OZNo2sfQdRa -wpA7o7mPHRhztQFpT5vflp+8P6+PEFat8D04eBOhNwrwwfhiPjD4gv5KvN4XutRW -VWv/2pnmzQKBgHPvHGg2mJ7quvm6ixXW1MWJX1eSBToIjCe3lBvDi5nhIaiZ8Q4w -7gA3QA3xD7tlDwauzLeAVxgEmsdbcCs6GQEfY3QiYy1Bt4FOSZa4YrcNfSmfq1Rw -j3Y4rRjKjeQz96i3YlzToT3tecJc7zPBj+DEy6au2H3Fdn+vQURneWHJAoGBANG7 -XES8mRVaUh/wlM1BVsaNH8SIGfiHzqzRjV7/bGYpQTBbWpAuUrhCmaMVtpXqBjav -TFwGLVRkZAWSYRjPpy2ERenT5SE3rv61o6mbGrifGsj6A82HQmtzYsGx8SmtYXtj -REF0sKebbmmOooUAS379GrguYJzL9o6D7YfRZNrhAoGAVfb/tiFU4S67DSpYpQey -ULhgfsFpDByICY6Potsg67gVFf9jIaB83NPTx3u/r6sHFgxFw7lQsuZcgSuWMu7t -glzOXVIP11Y5sl5CJ5OsfeK1/0umMZF5MWPyAQCx/qrPlZL86vXjt24Y/VaOxsAi -CZYdyJsjgOrJrWoMbo5ta54= ------END PRIVATE KEY----- -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - 82:ed:bf:41:c8:80:91:9c - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server - Validity - Not Before: Jan 19 19:09:06 2018 GMT - Not After : Nov 28 19:09:06 2027 GMT - Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:e0:57:81:be:67:37:f6:0d:3e:67:a0:08:ac:20: - 61:61:9f:f6:f3:ed:59:f9:5a:25:d4:a9:b0:ec:c5: - 13:89:3a:1f:fb:06:d3:a5:76:e6:64:6b:4a:4f:84: - 5c:a5:19:b9:f1:4b:c9:75:90:45:87:44:8d:93:ea: - 13:a8:5b:3b:a6:07:a2:2d:2a:23:7e:4f:92:77:d2: - 50:2f:4b:2a:3b:8b:07:45:a0:04:3f:4b:99:cf:5d: - 07:e1:34:04:50:7b:f3:44:3a:14:6d:a5:cb:8d:77: - 2c:78:05:67:4f:74:fb:89:40:0d:40:48:d2:42:2f: - 07:0b:3a:84:71:68:b6:ca:2e:f6:f6:57:94:f9:14: - 20:24:78:d3:ed:46:f3:ee:4f:88:0a:40:8b:4c:9a: - 21:df:02:1a:15:0b:c3:19:58:5f:c6:0b:44:6e:90: - 72:0d:ca:ce:4c:c2:a7:85:7b:16:a6:f0:2c:e7:99: - 03:cc:62:b2:dd:86:62:93:8f:70:d3:8a:f0:8f:a2: - 1f:7b:10:32:05:1a:58:19:48:26:9a:60:35:50:03: - e6:aa:96:8f:ea:da:98:b9:d8:f1:70:0b:52:b3:84: - d1:00:ec:47:8e:a9:2c:02:d4:d4:23:c2:31:ee:85: - c9:df:f5:66:f1:ae:04:ea:db:38:77:ae:fe:58:11: - 6d:99 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Alternative Name: - DNS:localhost - X509v3 Key Usage: critical - Digital Signature, Key Encipherment - X509v3 Extended Key Usage: - TLS Web Server Authentication, TLS Web Client Authentication - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Subject Key Identifier: - 85:11:BE:16:47:04:D1:30:EE:86:8A:18:70:BE:A8:28:6F:82:3D:CE - X509v3 Authority Key Identifier: - keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 - DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server - serial:82:ED:BF:41:C8:80:91:9B - - Authority Information Access: - CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer - OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - - X509v3 CRL Distribution Points: - - Full Name: - URI:http://testca.pythontest.net/testca/revocation.crl - - Signature Algorithm: sha1WithRSAEncryption - 7f:a1:7e:3e:68:01:b0:32:b8:57:b8:03:68:13:13:b3:e3:f4: - 70:2f:15:e5:0f:87:b9:fd:e0:12:e3:16:f2:91:53:c7:4e:25: - af:ca:cb:a7:d9:9d:57:4d:bf:a2:80:d4:78:aa:04:31:fd:6d: - cc:6d:82:43:e9:62:16:0d:0e:26:8b:e7:f1:3d:57:5c:68:02: - 9c:2b:b6:c9:fd:62:2f:10:85:88:cc:44:a5:e7:a2:3e:89:f2: - 1f:02:6a:3f:d0:3c:6c:24:2d:bc:51:62:7a:ec:25:c5:86:87: - 77:35:8f:f9:7e:d0:17:3d:77:56:bf:1a:0c:be:09:78:ee:ea: - 73:97:65:60:94:91:35:b3:5c:46:8a:5e:6d:94:52:de:48:b7: - 1f:6c:28:79:7f:ff:08:8d:e4:7d:d0:b9:0b:7c:ae:c4:1d:2a: - a1:b3:50:11:82:03:5e:6c:e7:26:fa:05:32:39:07:83:49:b9: - a2:fa:04:da:0d:e5:ff:4c:db:97:d0:c3:a7:43:37:4c:16:de: - 3c:b5:e9:7e:82:d4:b3:10:df:d1:c1:66:72:9c:15:67:19:3b: - 7b:91:0a:82:07:67:c5:06:03:5f:80:54:08:81:8a:b1:5c:7c: - 4c:d2:07:38:92:eb:12:f5:71:ae:de:05:15:c8:e1:33:f0:e4: - 96:0f:0f:1e ------BEGIN CERTIFICATE----- -MIIE8TCCA9mgAwIBAgIJAILtv0HIgJGcMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV -BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx -OTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOBXgb5nN/YN -PmegCKwgYWGf9vPtWflaJdSpsOzFE4k6H/sG06V25mRrSk+EXKUZufFLyXWQRYdE -jZPqE6hbO6YHoi0qI35PknfSUC9LKjuLB0WgBD9Lmc9dB+E0BFB780Q6FG2ly413 -LHgFZ090+4lADUBI0kIvBws6hHFotsou9vZXlPkUICR40+1G8+5PiApAi0yaId8C -GhULwxlYX8YLRG6Qcg3KzkzCp4V7FqbwLOeZA8xist2GYpOPcNOK8I+iH3sQMgUa -WBlIJppgNVAD5qqWj+ramLnY8XALUrOE0QDsR46pLALU1CPCMe6Fyd/1ZvGuBOrb -OHeu/lgRbZkCAwEAAaOCAcAwggG8MBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAOBgNV -HQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1Ud -EwEB/wQCMAAwHQYDVR0OBBYEFIURvhZHBNEw7oaKGHC+qChvgj3OMH0GA1UdIwR2 -MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYDVQQGEwJYWTEmMCQG -A1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91 -ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUF -BzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2Vy -dC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQv -dGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0 -aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBBQUA -A4IBAQB/oX4+aAGwMrhXuANoExOz4/RwLxXlD4e5/eAS4xbykVPHTiWvysun2Z1X -Tb+igNR4qgQx/W3MbYJD6WIWDQ4mi+fxPVdcaAKcK7bJ/WIvEIWIzESl56I+ifIf -Amo/0DxsJC28UWJ67CXFhod3NY/5ftAXPXdWvxoMvgl47upzl2VglJE1s1xGil5t -lFLeSLcfbCh5f/8IjeR90LkLfK7EHSqhs1ARggNebOcm+gUyOQeDSbmi+gTaDeX/ -TNuX0MOnQzdMFt48tel+gtSzEN/RwWZynBVnGTt7kQqCB2fFBgNfgFQIgYqxXHxM -0gc4kusS9XGu3gUVyOEz8OSWDw8e ------END CERTIFICATE----- diff --git a/Lib/test/keycert4.pem b/Lib/test/keycert4.pem deleted file mode 100644 index b7df7f3f2c7..00000000000 --- a/Lib/test/keycert4.pem +++ /dev/null @@ -1,132 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDH/76hZAZH4cSV -CmVZa5HEqKCjCKrcPwBECs9BS+3ibwN4x9NnFNP+tCeFGgJXl7WGFoeXgg3oK+1p -FsOWpsRHuF3BdqkCnShSydmT8bLaGHwKeL0cPxJP5T/uW7ezPKW2VWXGMwmwRaRJ -9dj2VCUu20vDZWSGFr9zjnjoJczBtH3RsVUgpK7euEHuQ5pIM9QSOaCo+5FPR7s7 -1nU7YqbFWtd+NhC8Og1G497B31DQlHciF6BRm6/cNGAmHaAErKUGBFdkGtFPHBn4 -vktoEg9fwxJAZLvGpoTZWrB4HRsRwVTmFdGvK+JXK225xF23AXRXp/snhSuSFeLj -E5cpyJJ7AgMBAAECggEAQOv527X2e/sDr0XSpHZQuT/r9UBpBlnFIlFH+fBF5k0X -GWv0ae/O6U1dzs0kmX57xG0n0ry6+vTXeleTYiH8cTOd66EzN9AAOO+hG29IGZf9 -HAEZkkO/FARc/mjzdtFnEYsjIHWM3ZWdwQx3Q28JKu6w51rQiN51g3NqOCGdF/uF -rE5XPKsKndn+nLHvsNuApFgUYZEwdrozgUueEgRaPTUCNhzotcA9eWoBdA24XNhk -x8Cm/bZWabXm7gBO75zl3Cu2F21ay+EuwyOZTsx6lZi6YX9/zo1mkO81Zi3tQk50 -NMEI0feLNwsdxTbmOcVJadjOgd+QVghlFyr5HGBWMQKBgQD3AH3rhnAo6tOyNkGN -+IzIU1MhUS452O7IavykUYO9sM24BVChpRtlI9Dpev4yE/q3BAO3+oWT3cJrN7/3 -iyo1dzAkpGvI65XWfElXFM4nLjEiZzx4W9fiPN91Oucpr0ED6+BZXTtz4gVm0TP/ -TUc2xvTB6EKvIyWmKOYEi0snxQKBgQDPSOjbz9jWOrC9XY7PmtLB6QJDDz7XSGVK -wzD+gDAPpAwhk58BEokdOhBx2Lwl8zMJi0CRHgH2vNvkRyhvUQ4UFzisrqann/Tw -klp5sw3iWC6ERC8z9zL7GfHs7sK3mOVeAdK6ffowPM3JrZ2vPusVBdr0MN3oZwki -CtNXqbY1PwKBgGheQNbAW6wubX0kB9chavtKmhm937Z5v4vYCSC1gOEqUAKt3EAx -L74wwBmn6rjmUE382EVpCgBM99WuHONQXmlxD1qsTw763LlgkuzE0cckcYaD8L06 -saHa7uDuHrcyYlpx1L5t8q0ol/e19i6uTKUMtGcq6OJwC3yGU4sgAIWxAoGBAMVq -qiQXm2vFL+jafxYoXUvDMJ1PmskMsTP4HOR2j8+FrOwZnVk3HxGP6HOVOPRn4JbZ -YiAT1Uj6a+7I+rCyINdvmlGUcTK6fFzW9oZryvBkjcD483/pkktmVWwTpa2YV/Ml -h16IdsyUTGYlDUYHhXtbPUJOfDpIT4F1j/0wrFGfAoGAO82BcUsehEUQE0xvQLIn -7QaFtUI5z19WW730jVuEobiYlh9Ka4DPbKMvka8MwyOxEwhk39gZQavmfG6+wZm+ -kjERU23LhHziJGWS2Um4yIhC7myKbWaLzjHEq72dszLpQku4BzE5fT60fxI7cURD -WGm/Z3Q2weS3ZGIoMj1RNPI= ------END PRIVATE KEY----- -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - 82:ed:bf:41:c8:80:91:9d - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server - Validity - Not Before: Jan 19 19:09:06 2018 GMT - Not After : Nov 28 19:09:06 2027 GMT - Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:c7:ff:be:a1:64:06:47:e1:c4:95:0a:65:59:6b: - 91:c4:a8:a0:a3:08:aa:dc:3f:00:44:0a:cf:41:4b: - ed:e2:6f:03:78:c7:d3:67:14:d3:fe:b4:27:85:1a: - 02:57:97:b5:86:16:87:97:82:0d:e8:2b:ed:69:16: - c3:96:a6:c4:47:b8:5d:c1:76:a9:02:9d:28:52:c9: - d9:93:f1:b2:da:18:7c:0a:78:bd:1c:3f:12:4f:e5: - 3f:ee:5b:b7:b3:3c:a5:b6:55:65:c6:33:09:b0:45: - a4:49:f5:d8:f6:54:25:2e:db:4b:c3:65:64:86:16: - bf:73:8e:78:e8:25:cc:c1:b4:7d:d1:b1:55:20:a4: - ae:de:b8:41:ee:43:9a:48:33:d4:12:39:a0:a8:fb: - 91:4f:47:bb:3b:d6:75:3b:62:a6:c5:5a:d7:7e:36: - 10:bc:3a:0d:46:e3:de:c1:df:50:d0:94:77:22:17: - a0:51:9b:af:dc:34:60:26:1d:a0:04:ac:a5:06:04: - 57:64:1a:d1:4f:1c:19:f8:be:4b:68:12:0f:5f:c3: - 12:40:64:bb:c6:a6:84:d9:5a:b0:78:1d:1b:11:c1: - 54:e6:15:d1:af:2b:e2:57:2b:6d:b9:c4:5d:b7:01: - 74:57:a7:fb:27:85:2b:92:15:e2:e3:13:97:29:c8: - 92:7b - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Alternative Name: - DNS:fakehostname - X509v3 Key Usage: critical - Digital Signature, Key Encipherment - X509v3 Extended Key Usage: - TLS Web Server Authentication, TLS Web Client Authentication - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Subject Key Identifier: - F8:76:79:CB:11:85:F0:46:E5:95:E6:7E:69:CB:12:5E:4E:AA:EC:4D - X509v3 Authority Key Identifier: - keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 - DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server - serial:82:ED:BF:41:C8:80:91:9B - - Authority Information Access: - CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer - OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - - X509v3 CRL Distribution Points: - - Full Name: - URI:http://testca.pythontest.net/testca/revocation.crl - - Signature Algorithm: sha1WithRSAEncryption - 6d:50:8d:fb:ee:4e:93:8b:eb:47:56:ba:38:cc:80:e1:9d:c7: - e1:9e:1f:9c:22:0c:d2:08:9b:ed:bf:31:d9:00:ee:af:8c:56: - 78:92:d1:7c:ba:4e:81:7f:82:1f:f4:68:99:86:91:c6:cb:57: - d3:b9:41:12:fa:75:53:fd:22:32:21:50:af:6b:4c:b1:34:36: - d1:a8:25:0a:d0:f0:f8:81:7d:69:58:6e:af:e3:d2:c4:32:87: - 79:d7:cd:ad:0c:56:f3:15:27:10:0c:f9:57:59:53:00:ed:af: - 5d:4d:07:86:7a:e5:f3:97:88:bc:86:b4:f1:17:46:33:55:28: - 66:7b:70:d3:a5:12:b9:4f:c7:ed:e6:13:20:2d:f0:9e:ec:17: - 64:cf:fd:13:14:1b:76:ba:64:ac:c5:51:b6:cd:13:0a:93:b1: - fd:43:09:a0:0b:44:6c:77:45:43:0b:e5:ed:70:b2:76:dc:08: - 4a:5b:73:5f:c1:fc:7f:63:70:f8:b9:ca:3c:98:06:5f:fd:98: - d1:e4:e6:61:5f:09:8f:6c:18:86:98:9c:cb:3f:73:7b:3f:38: - f5:a7:09:20:ee:a5:63:1c:ff:8b:a6:d1:8c:e8:f4:84:3d:99: - 38:0f:cc:e0:52:03:f9:18:05:23:76:39:de:52:ce:8e:fb:a6: - 6e:f5:4f:c3 ------BEGIN CERTIFICATE----- -MIIE9zCCA9+gAwIBAgIJAILtv0HIgJGdMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV -BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx -OTA5MDZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh -a2Vob3N0bmFtZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMf/vqFk -BkfhxJUKZVlrkcSooKMIqtw/AEQKz0FL7eJvA3jH02cU0/60J4UaAleXtYYWh5eC -Degr7WkWw5amxEe4XcF2qQKdKFLJ2ZPxstoYfAp4vRw/Ek/lP+5bt7M8pbZVZcYz -CbBFpEn12PZUJS7bS8NlZIYWv3OOeOglzMG0fdGxVSCkrt64Qe5Dmkgz1BI5oKj7 -kU9HuzvWdTtipsVa1342ELw6DUbj3sHfUNCUdyIXoFGbr9w0YCYdoASspQYEV2Qa -0U8cGfi+S2gSD1/DEkBku8amhNlasHgdGxHBVOYV0a8r4lcrbbnEXbcBdFen+yeF -K5IV4uMTlynIknsCAwEAAaOCAcMwggG/MBcGA1UdEQQQMA6CDGZha2Vob3N0bmFt -ZTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC -MAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPh2ecsRhfBG5ZXmfmnLEl5OquxNMH0G -A1UdIwR2MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYDVQQGEwJY -WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV -BAMMDW91ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEEdzB1MDwG -CCsGAQUFBzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9w -eWNhY2VydC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVz -dC5uZXQvdGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0 -Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3 -DQEBBQUAA4IBAQBtUI377k6Ti+tHVro4zIDhncfhnh+cIgzSCJvtvzHZAO6vjFZ4 -ktF8uk6Bf4If9GiZhpHGy1fTuUES+nVT/SIyIVCva0yxNDbRqCUK0PD4gX1pWG6v -49LEMod5182tDFbzFScQDPlXWVMA7a9dTQeGeuXzl4i8hrTxF0YzVShme3DTpRK5 -T8ft5hMgLfCe7Bdkz/0TFBt2umSsxVG2zRMKk7H9QwmgC0Rsd0VDC+XtcLJ23AhK -W3Nfwfx/Y3D4uco8mAZf/ZjR5OZhXwmPbBiGmJzLP3N7Pzj1pwkg7qVjHP+LptGM -6PSEPZk4D8zgUgP5GAUjdjneUs6O+6Zu9U/D ------END CERTIFICATE----- diff --git a/Lib/test/keycertecc.pem b/Lib/test/keycertecc.pem deleted file mode 100644 index deb484f9920..00000000000 --- a/Lib/test/keycertecc.pem +++ /dev/null @@ -1,96 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDe3QWmhZX07HZbntz4 -CFqAOaoYMdYwD7Z3WPNIc2zR7p4D6BMOa7NAWjLV5A7CUw6hZANiAAQ5IVKzLLz4 -LCfcpy6fMOp+jk5KwywsU3upPtjA6E3UetxPcfnnv+gghRyDAYLN2OVqZgLMEmUo -F1j1SM1QrbhHIuNcVxI9gPPMdumcNFSz/hqxrBRtA/8Z2gywczdNLjc= ------END PRIVATE KEY----- -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - 82:ed:bf:41:c8:80:91:9e - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server - Validity - Not Before: Jan 19 19:09:06 2018 GMT - Not After : Nov 28 19:09:06 2027 GMT - Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc - Subject Public Key Info: - Public Key Algorithm: id-ecPublicKey - Public-Key: (384 bit) - pub: - 04:39:21:52:b3:2c:bc:f8:2c:27:dc:a7:2e:9f:30: - ea:7e:8e:4e:4a:c3:2c:2c:53:7b:a9:3e:d8:c0:e8: - 4d:d4:7a:dc:4f:71:f9:e7:bf:e8:20:85:1c:83:01: - 82:cd:d8:e5:6a:66:02:cc:12:65:28:17:58:f5:48: - cd:50:ad:b8:47:22:e3:5c:57:12:3d:80:f3:cc:76: - e9:9c:34:54:b3:fe:1a:b1:ac:14:6d:03:ff:19:da: - 0c:b0:73:37:4d:2e:37 - ASN1 OID: secp384r1 - NIST CURVE: P-384 - X509v3 extensions: - X509v3 Subject Alternative Name: - DNS:localhost-ecc - X509v3 Key Usage: critical - Digital Signature, Key Encipherment - X509v3 Extended Key Usage: - TLS Web Server Authentication, TLS Web Client Authentication - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Subject Key Identifier: - 33:23:0E:15:04:83:2E:3D:BF:DA:81:6D:10:38:80:C3:C2:B0:A4:74 - X509v3 Authority Key Identifier: - keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 - DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server - serial:82:ED:BF:41:C8:80:91:9B - - Authority Information Access: - CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer - OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - - X509v3 CRL Distribution Points: - - Full Name: - URI:http://testca.pythontest.net/testca/revocation.crl - - Signature Algorithm: sha1WithRSAEncryption - 3b:6f:97:af:7e:5f:e0:14:34:ed:57:7e:de:ce:c4:85:1e:aa: - 84:52:94:7c:e5:ce:e9:9c:88:8b:ad:b5:4d:16:ac:af:81:ea: - b8:a2:e2:50:2e:cb:e9:11:bd:1b:a6:3f:0c:a2:d7:7b:67:72: - b3:43:16:ad:c6:87:ac:6e:ac:47:78:ef:2f:8c:86:e8:9b:d1: - 43:8c:c1:7a:91:30:e9:14:d6:9f:41:8b:9b:0b:24:9b:78:86: - 11:8a:fc:2b:cd:c9:13:ee:90:4f:14:33:51:a3:c4:9e:d6:06: - 48:f5:41:12:af:f0:f2:71:40:78:f5:96:c2:5d:cf:e1:38:ff: - bf:10:eb:74:2f:c2:23:21:3e:27:f5:f1:f2:af:2c:62:82:31: - 00:c8:96:1b:c3:7e:8d:71:89:e7:40:b5:67:1a:33:fb:c0:8b: - 96:0c:36:78:25:27:82:d8:27:27:52:0f:f7:69:cd:ff:2b:92: - 10:d3:d2:0a:db:65:ed:af:90:eb:db:76:f3:8a:7a:13:9e:c6: - 33:57:15:42:06:13:d6:54:49:fa:84:a7:0e:1d:14:72:ca:19: - 8e:2b:aa:a4:02:54:3c:f6:1c:23:81:7a:59:54:b0:92:65:72: - c8:e5:ba:9f:03:4e:30:f2:4d:45:85:e3:35:a8:b1:68:58:b9: - 3b:20:a3:eb ------BEGIN CERTIFICATE----- -MIIESzCCAzOgAwIBAgIJAILtv0HIgJGeMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV -BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx -OTA5MDZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv -Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQ5IVKzLLz4LCfcpy6f -MOp+jk5KwywsU3upPtjA6E3UetxPcfnnv+gghRyDAYLN2OVqZgLMEmUoF1j1SM1Q -rbhHIuNcVxI9gPPMdumcNFSz/hqxrBRtA/8Z2gywczdNLjejggHEMIIBwDAYBgNV -HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU -BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUMyMO -FQSDLj2/2oFtEDiAw8KwpHQwfQYDVR0jBHYwdIAUms/PbutxPds88a6Ia1ZyA8sI -p0ihUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAILtv0HIgJGb -MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 -aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 -cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww -OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 -b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQEFBQADggEBADtvl69+X+AUNO1Xft7OxIUe -qoRSlHzlzumciIuttU0WrK+B6rii4lAuy+kRvRumPwyi13tncrNDFq3Gh6xurEd4 -7y+Mhuib0UOMwXqRMOkU1p9Bi5sLJJt4hhGK/CvNyRPukE8UM1GjxJ7WBkj1QRKv -8PJxQHj1lsJdz+E4/78Q63QvwiMhPif18fKvLGKCMQDIlhvDfo1xiedAtWcaM/vA -i5YMNnglJ4LYJydSD/dpzf8rkhDT0grbZe2vkOvbdvOKehOexjNXFUIGE9ZUSfqE -pw4dFHLKGY4rqqQCVDz2HCOBellUsJJlcsjlup8DTjDyTUWF4zWosWhYuTsgo+s= ------END CERTIFICATE----- diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 3427b51b60a..8b137891791 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,5 +1 @@ -# We import importlib *ASAP* in order to test #15386 -import importlib -from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES -from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 0a97c8c19b1..e7a12e4d0b6 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,8 +1,9 @@ import argparse -import os +import os.path +import shlex import sys -from test import support -from test.support import os_helper +from test.support import os_helper, Py_DEBUG +from .utils import ALL_RESOURCES, RESOURCE_NAMES, TestFilter USAGE = """\ @@ -27,8 +28,10 @@ Additional option details: -r randomizes test execution order. You can use --randseed=int to provide an -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. +int seed value for the randomizer. The randseed value will be used +to set seeds for all random usages in tests +(including randomizing the tests order if -r is set). +By default we always set random seed, but do not randomize test order. -s On the first invocation of regrtest using -s, the first test file found or the first test file given on the command line is run, and the name of @@ -41,11 +44,19 @@ doing memory analysis on the Python interpreter, which process tends to consume too many resources to run the full regression test non-stop. --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). +-S is used to resume running tests after an interrupted run. It will +maintain the order a standard run (i.e. it assumes -r is not used). This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. +reason and you want to resume the run from where you left off rather +than starting from the beginning. Note: this is different from --prioritize. + +--prioritize is used to influence the order of selected tests, such that +the tests listed as an argument are executed first. This is especially +useful when combined with -j and -r to pin the longest-running tests +to start at the beginning of a test run. Pass --prioritize=test_a,test_b +to make test_a run first, followed by test_b, and then the other tests. +If test_a wasn't selected for execution by regular means, --prioritize will +not make it execute. -f reads the names of tests from the file given as f's argument, one or more test names per line. Whitespace is ignored. Blank lines and @@ -84,36 +95,40 @@ The argument is a comma-separated list of words indicating the resources to test. Currently only the following are defined: - all - Enable all special resources. + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) - none - Disable all special resources (this is the default). + curses - Tests that use curses and will modify the terminal's + state and output modes. - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2 GiB of disk space temporarily. - curses - Tests that use curses and will modify the terminal's - state and output modes. + extralargefile - Like 'largefile', but even larger (and slower). - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2 GiB of disk space temporarily. + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. + decimal - Test the decimal module against a large suite that + verifies compliance with standards. - decimal - Test the decimal module against a large suite that - verifies compliance with standards. + cpu - Used for certain CPU-heavy tests. - cpu - Used for certain CPU-heavy tests. + walltime - Long running but not CPU-bound tests. - subprocess Run all tests for the subprocess module. + subprocess Run all tests for the subprocess module. - urlfetch - It is okay to download files required on testing. + urlfetch - It is okay to download files required on testing. - gui - Run tests that require a running GUI. + gui - Run tests that require a running GUI. - tzdata - Run tests that require timezone data. + tzdata - Run tests that require timezone data. To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the @@ -128,17 +143,53 @@ """ -ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') +class Namespace(argparse.Namespace): + def __init__(self, **kwargs) -> None: + self.ci = False + self.testdir = None + self.verbose = 0 + self.quiet = False + self.exclude = False + self.cleanup = False + self.wait = False + self.list_cases = False + self.list_tests = False + self.single = False + self.randomize = False + self.fromfile = None + self.fail_env_changed = False + self.use_resources: list[str] = [] + self.trace = False + self.coverdir = 'coverage' + self.runleaks = False + self.huntrleaks: tuple[int, int, str] | None = None + self.rerun = False + self.verbose3 = False + self.print_slow = False + self.random_seed = None + self.use_mp = None + self.parallel_threads = None + self.forever = False + self.header = False + self.failfast = False + self.match_tests: TestFilter = [] + self.pgo = False + self.pgo_extended = False + self.tsan = False + self.tsan_parallel = False + self.worker_json = None + self.start = None + self.timeout = None + self.memlimit = None + self.threshold = None + self.fail_rerun = False + self.tempdir = None + self._add_python_opts = True + self.xmlpath = None + self.single_process = False + + super().__init__(**kwargs) -# Other resources excluded from --use=all: -# -# - extralagefile (ex: test_zipfile64): really too slow to be enabled -# "by default" -# - tzdata: while needed to validate fully test_datetime, it makes -# test_datetime too slow (15-20 min on some buildbots) and so is disabled by -# default (see bpo-30822). -RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') class _ArgParser(argparse.ArgumentParser): @@ -146,6 +197,20 @@ def error(self, message): super().error(message + "\nPass -h or --help for complete help.") +class FilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + items.append((value, self.const)) + + +class FromFileFilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + with open(value, encoding='utf-8') as fp: + for line in fp: + items.append((line.strip(), self.const)) + + def _create_parser(): # Set prog to prevent the uninformative "__main__.py" from displaying in # error messages when using "python -m test ...". @@ -155,6 +220,7 @@ def _create_parser(): epilog=EPILOG, add_help=False, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.set_defaults(match_tests=[]) # Arguments with this clause added to its help are described further in # the epilog's "Additional option details" section. @@ -164,23 +230,35 @@ def _create_parser(): # We add help explicitly to control what argument group it renders under. group.add_argument('-h', '--help', action='help', help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, + group.add_argument('--fast-ci', action='store_true', + help='Fast Continuous Integration (CI) mode used by ' + 'GitHub Actions') + group.add_argument('--slow-ci', action='store_true', + help='Slow Continuous Integration (CI) mode used by ' + 'buildbot workers') + group.add_argument('--timeout', metavar='TIMEOUT', help='dump the traceback and exit if a test takes ' 'more than TIMEOUT seconds; disabled if TIMEOUT ' 'is negative or equals to zero') group.add_argument('--wait', action='store_true', help='wait for user input, e.g., allow a debugger ' 'to be attached') - group.add_argument('--worker-args', metavar='ARGS') group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + + help='resume an interrupted run at the following test.' + more_details) + group.add_argument('-p', '--python', metavar='PYTHON', + help='Command to run Python test subprocesses with.') + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a global random seed') group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', + group.add_argument('-w', '--rerun', action='store_true', help='re-run failed tests in verbose mode') + group.add_argument('--verbose2', action='store_true', dest='rerun', + help='deprecated alias to --rerun') group.add_argument('-W', '--verbose3', action='store_true', help='display test output on failure') group.add_argument('-q', '--quiet', action='store_true', @@ -193,10 +271,13 @@ def _create_parser(): group = parser.add_argument_group('Selecting tests') group.add_argument('-r', '--randomize', action='store_true', help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') + group.add_argument('--no-randomize', dest='no_randomize', action='store_true', + help='do not randomize test execution order, even if ' + 'it would be implied by another option') + group.add_argument('--prioritize', metavar='TEST1,TEST2,...', + action='append', type=priority_list, + help='select these tests first, even if the order is' + ' randomized.' + more_details) group.add_argument('-f', '--fromfile', metavar='FILE', help='read names of tests to run from a file.' + more_details) @@ -206,12 +287,21 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', action='append', + dest='match_tests', action=FilterAction, const=True, help='match test cases and methods with glob pattern PAT') + group.add_argument('-i', '--ignore', metavar='PAT', + dest='match_tests', action=FilterAction, const=False, + help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', - dest='match_filename', + dest='match_tests', + action=FromFileFilterAction, const=True, help='similar to --match but get patterns from a ' 'text file, one pattern per line') + group.add_argument('--ignorefile', metavar='FILENAME', + dest='match_tests', + action=FromFileFilterAction, const=False, + help='similar to --matchfile but it receives patterns ' + 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', @@ -227,9 +317,6 @@ def _create_parser(): '(instead of the Python stdlib test suite)') group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_const', const=2, - default=1, - help='deprecated alias to --fail-env-changed') group.add_argument('-L', '--runleaks', action='store_true', help='run the leaks(1) command just before exit.' + more_details) @@ -240,6 +327,16 @@ def _create_parser(): group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, help='run PROCESSES processes at once') + group.add_argument('--single-process', action='store_true', + dest='single_process', + help='always run all tests sequentially in ' + 'a single process, ignore -jN option, ' + 'and failed tests are also rerun sequentially ' + 'in the same process') + group.add_argument('--parallel-threads', metavar='PARALLEL_THREADS', + type=int, + help='run copies of each test in PARALLEL_THREADS at ' + 'once') group.add_argument('-T', '--coverage', action='store_true', dest='trace', help='turn on code coverage tracing using the trace ' @@ -257,7 +354,7 @@ def _create_parser(): help='suppress error message boxes on Windows') group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' - 'error happens') + 'error happens; imply --failfast') group.add_argument('--list-tests', action='store_true', help="only write the name of tests that will be run, " "don't execute them") @@ -265,16 +362,33 @@ def _create_parser(): help='only write the name of test cases that will be run' ' , don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', - help='enable Profile Guided Optimization training') + help='enable Profile Guided Optimization (PGO) training') + group.add_argument('--pgo-extended', action='store_true', + help='enable extended PGO training (slower training)') + group.add_argument('--tsan', dest='tsan', action='store_true', + help='run a subset of test cases that are proper for the TSAN test') + group.add_argument('--tsan-parallel', action='store_true', + help='run a subset of test cases that are appropriate ' + 'for TSAN with `--parallel-threads=N`') group.add_argument('--fail-env-changed', action='store_true', help='if a test file alters the environment, mark ' 'the test as failed') + group.add_argument('--fail-rerun', action='store_true', + help='if a test failed and then passed when re-run, ' + 'mark the tests as failed') group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME', help='writes JUnit-style XML results to the specified ' 'file') - group.add_argument('--tempdir', dest='tempdir', metavar='PATH', + group.add_argument('--tempdir', metavar='PATH', help='override the working directory for the test run') + group.add_argument('--cleanup', action='store_true', + help='remove old test_python_* directories') + group.add_argument('--bisect', action='store_true', + help='if some tests fail, run test.bisect_cmd on them') + group.add_argument('--dont-add-python-opts', dest='_add_python_opts', + action='store_false', + help="internal option, don't use it") return parser @@ -307,21 +421,18 @@ def resources_list(string): return u +def priority_list(string): + return string.split(",") + + def _parse_args(args, **kwargs): # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=1, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None, pgo=False) + ns = Namespace() for k, v in kwargs.items(): if not hasattr(ns, k): raise TypeError('%r is an invalid keyword argument ' 'for this function' % k) setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] parser = _create_parser() # Issue #14191: argparse doesn't support "intermixed" positional and @@ -330,19 +441,81 @@ def _parse_args(args, **kwargs): for arg in ns.args: if arg.startswith('-'): parser.error("unrecognized arguments: %s" % arg) - sys.exit(1) - if ns.findleaks > 1: - # --findleaks implies --fail-env-changed + if ns.timeout is not None: + # Support "--timeout=" (no value) so Makefile.pre.pre TESTTIMEOUT + # can be used by "make buildbottest" and "make test". + if ns.timeout != "": + try: + ns.timeout = float(ns.timeout) + except ValueError: + parser.error(f"invalid timeout value: {ns.timeout!r}") + else: + ns.timeout = None + + # Continuous Integration (CI): common options for fast/slow CI modes + if ns.slow_ci or ns.fast_ci: + # Similar to options: + # -j0 --randomize --fail-env-changed --rerun --slowest --verbose3 + if ns.use_mp is None: + ns.use_mp = 0 + ns.randomize = True ns.fail_env_changed = True + if ns.python is None: + ns.rerun = True + ns.print_slow = True + if not ns.verbose: + ns.verbose3 = True + else: + # --verbose has the priority over --verbose3 + pass + else: + ns._add_python_opts = False + + # --singleprocess overrides -jN option + if ns.single_process: + ns.use_mp = None + + # When both --slow-ci and --fast-ci options are present, + # --slow-ci has the priority + if ns.slow_ci: + # Similar to: -u "all" --timeout=1200 + if ns.use is None: + ns.use = [] + ns.use.insert(0, ['all']) + if ns.timeout is None: + ns.timeout = 1200 # 20 minutes + elif ns.fast_ci: + # Similar to: -u "all,-cpu" --timeout=600 + if ns.use is None: + ns.use = [] + ns.use.insert(0, ['all', '-cpu']) + if ns.timeout is None: + ns.timeout = 600 # 10 minutes + if ns.single and ns.fromfile: parser.error("-s and -f don't go together!") - if ns.use_mp is not None and ns.trace: - parser.error("-T and -j don't go together!") + if ns.trace: + if ns.use_mp is not None: + if not Py_DEBUG: + parser.error("need --with-pydebug to use -T and -j together") + else: + print( + "Warning: collecting coverage without -j is imprecise. Configure" + " --with-pydebug and run -m test -T -j for best results.", + file=sys.stderr + ) + if ns.python is not None: + if ns.use_mp is None: + parser.error("-p requires -j!") + # The "executable" may be two or more parts, e.g. "node python.js" + ns.python = shlex.split(ns.python) if ns.failfast and not (ns.verbose or ns.verbose3): parser.error("-G/--failfast needs either -v or -W") - if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): + if ns.pgo and (ns.verbose or ns.rerun or ns.verbose3): parser.error("--pgo/-v don't go together!") + if ns.pgo_extended: + ns.pgo = True # pgo_extended implies pgo if ns.nowindows: print("Warning: the --nowindows (-n) option is deprecated. " @@ -353,10 +526,6 @@ def _parse_args(args, **kwargs): if ns.timeout is not None: if ns.timeout <= 0: ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) if ns.use: for a in ns.use: for r in a: @@ -377,18 +546,40 @@ def _parse_args(args, **kwargs): ns.use_resources.append(r) if ns.random_seed is not None: ns.randomize = True + if ns.no_randomize: + ns.randomize = False if ns.verbose: ns.header = True - if ns.huntrleaks and ns.verbose3: + + # When -jN option is used, a worker process does not use --verbose3 + # and so -R 3:3 -jN --verbose3 just works as expected: there is no false + # alarm about memory leak. + if ns.huntrleaks and ns.verbose3 and ns.use_mp is None: + # run_single_test() replaces sys.stdout with io.StringIO if verbose3 + # is true. In this case, huntrleaks sees an write into StringIO as + # a memory leak, whereas it is not (gh-71290). ns.verbose3 = False print("WARNING: Disable --verbose3 because it's incompatible with " - "--huntrleaks: see http://bugs.python.org/issue27103", + "--huntrleaks without -jN option", file=sys.stderr) - if ns.match_filename: - if ns.match_tests is None: - ns.match_tests = [] - with open(ns.match_filename) as fp: - for line in fp: - ns.match_tests.append(line.strip()) + + if ns.forever: + # --forever implies --failfast + ns.failfast = True + + if ns.huntrleaks: + warmup, repetitions, _ = ns.huntrleaks + if warmup < 1 or repetitions < 1: + msg = ("Invalid values for the --huntrleaks/-R parameters. The " + "number of warmups and repetitions must be at least 1 " + "each (1:1).") + print(msg, file=sys.stderr, flush=True) + sys.exit(2) + + ns.prioritize = [ + test + for test_list in (ns.prioritize or ()) + for test in test_list + ] return ns diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py new file mode 100644 index 00000000000..41372e427ff --- /dev/null +++ b/Lib/test/libregrtest/filter.py @@ -0,0 +1,77 @@ +import itertools +import operator +import re + + +# By default, don't filter tests +_test_matchers = () +_test_patterns = () + + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + result = False + for matcher, result in reversed(_test_matchers): + if matcher(test.id()): + return result + return not result + + +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, ignore 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def get_match_tests(): + global _test_patterns + return _test_patterns + + +def set_match_tests(patterns): + global _test_matchers, _test_patterns + + if not patterns: + _test_matchers = () + _test_patterns = () + else: + itemgetter = operator.itemgetter + patterns = tuple(patterns) + if patterns != _test_patterns: + _test_matchers = [ + (_compile_match_function(map(itemgetter(0), it)), result) + for result, it in itertools.groupby(patterns, itemgetter(1)) + ] + _test_patterns = patterns + + +def _compile_match_function(patterns): + patterns = list(patterns) + + if all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect_cmd utility only uses such full test identifiers. + return set(patterns).__contains__ + else: + import fnmatch + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id, regex_match=regex_match): + if regex_match(test_id): + # The regex matches the whole identifier, for example + # 'test.test_os.FileTests.test_access'. + return True + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + return match_test_regex diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py new file mode 100644 index 00000000000..f01c1240774 --- /dev/null +++ b/Lib/test/libregrtest/findtests.py @@ -0,0 +1,110 @@ +import os +import sys +import unittest +from collections.abc import Container + +from test import support + +from .filter import match_test, set_match_tests +from .utils import ( + StrPath, TestName, TestTuple, TestList, TestFilter, + abs_module_name, count, printlist) + + +# If these test directories are encountered recurse into them and treat each +# "test_*.py" file or each sub-directory as a separate test module. This can +# increase parallelism. +# +# Beware this can't generally be done for any directory with sub-tests as the +# __init__.py may do things which alter what tests are to be run. +SPLITTESTDIRS: set[TestName] = { + "test_asyncio", + "test_concurrent_futures", + "test_doctests", + "test_future_stmt", + "test_gdb", + "test_inspect", + "test_pydoc", + "test_multiprocessing_fork", + "test_multiprocessing_forkserver", + "test_multiprocessing_spawn", +} + + +def findtestdir(path: StrPath | None = None) -> StrPath: + return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir + + +def findtests(*, testdir: StrPath | None = None, exclude: Container[str] = (), + split_test_dirs: set[TestName] = SPLITTESTDIRS, + base_mod: str = "") -> TestList: + """Return a list of all applicable test modules.""" + testdir = findtestdir(testdir) + tests = [] + for name in os.listdir(testdir): + mod, ext = os.path.splitext(name) + if (not mod.startswith("test_")) or (mod in exclude): + continue + if base_mod: + fullname = f"{base_mod}.{mod}" + else: + fullname = mod + if fullname in split_test_dirs: + subdir = os.path.join(testdir, mod) + if not base_mod: + fullname = f"test.{mod}" + tests.extend(findtests(testdir=subdir, exclude=exclude, + split_test_dirs=split_test_dirs, + base_mod=fullname)) + elif ext in (".py", ""): + tests.append(fullname) + return sorted(tests) + + +def split_test_packages(tests, *, testdir: StrPath | None = None, + exclude: Container[str] = (), + split_test_dirs=SPLITTESTDIRS) -> list[TestName]: + testdir = findtestdir(testdir) + splitted = [] + for name in tests: + if name in split_test_dirs: + subdir = os.path.join(testdir, name) + splitted.extend(findtests(testdir=subdir, exclude=exclude, + split_test_dirs=split_test_dirs, + base_mod=name)) + else: + splitted.append(name) + return splitted + + +def _list_cases(suite: unittest.TestSuite) -> None: + for test in suite: + if isinstance(test, unittest.loader._FailedTest): # type: ignore[attr-defined] + continue + if isinstance(test, unittest.TestSuite): + _list_cases(test) + elif isinstance(test, unittest.TestCase): + if match_test(test): + print(test.id()) + +def list_cases(tests: TestTuple, *, + match_tests: TestFilter | None = None, + test_dir: StrPath | None = None) -> None: + support.verbose = False + set_match_tests(match_tests) + + skipped = [] + for test_name in tests: + module_name = abs_module_name(test_name, test_dir) + try: + suite = unittest.defaultTestLoader.loadTestsFromName(module_name) + _list_cases(suite) + except unittest.SkipTest: + skipped.append(test_name) + + if skipped: + sys.stdout.flush() + stderr = sys.stderr + print(file=stderr) + print(count(len(skipped), "test"), "skipped:", file=stderr) + printlist(skipped, file=stderr) diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py new file mode 100644 index 00000000000..fa1d4d575c8 --- /dev/null +++ b/Lib/test/libregrtest/logger.py @@ -0,0 +1,89 @@ +import os +import time + +from test.support import MS_WINDOWS +from .results import TestResults +from .runtests import RunTests +from .utils import print_warning + +if MS_WINDOWS: + from .win_utils import WindowsLoadTracker + + +class Logger: + def __init__(self, results: TestResults, quiet: bool, pgo: bool): + self.start_time = time.perf_counter() + self.test_count_text = '' + self.test_count_width = 3 + self.win_load_tracker: WindowsLoadTracker | None = None + self._results: TestResults = results + self._quiet: bool = quiet + self._pgo: bool = pgo + + def log(self, line: str = '') -> None: + empty = not line + + # add the system load prefix: "load avg: 1.80 " + load_avg = self.get_load_avg() + if load_avg is not None: + line = f"load avg: {load_avg:.2f} {line}" + + # add the timestamp prefix: "0:01:05 " + log_time = time.perf_counter() - self.start_time + + mins, secs = divmod(int(log_time), 60) + hours, mins = divmod(mins, 60) + formatted_log_time = "%d:%02d:%02d" % (hours, mins, secs) + + line = f"{formatted_log_time} {line}" + if empty: + line = line[:-1] + + print(line, flush=True) + + def get_load_avg(self) -> float | None: + if hasattr(os, 'getloadavg'): + try: + return os.getloadavg()[0] + except OSError: + pass + if self.win_load_tracker is not None: + return self.win_load_tracker.getloadavg() + return None + + def display_progress(self, test_index: int, text: str) -> None: + if self._quiet: + return + results = self._results + + # "[ 51/405/1] test_tcl passed" + line = f"{test_index:{self.test_count_width}}{self.test_count_text}" + fails = len(results.bad) + len(results.env_changed) + if fails and not self._pgo: + line = f"{line}/{fails}" + self.log(f"[{line}] {text}") + + def set_tests(self, runtests: RunTests) -> None: + if runtests.forever: + self.test_count_text = '' + self.test_count_width = 3 + else: + self.test_count_text = '/{}'.format(len(runtests.tests)) + self.test_count_width = len(self.test_count_text) - 1 + + def start_load_tracker(self) -> None: + if not MS_WINDOWS: + return + + try: + self.win_load_tracker = WindowsLoadTracker() + except PermissionError as error: + # Standard accounts may not have access to the performance + # counters. + print_warning(f'Failed to create WindowsLoadTracker: {error}') + + def stop_load_tracker(self) -> None: + if self.win_load_tracker is None: + return + self.win_load_tracker.close() + self.win_load_tracker = None diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fba24e4f323..0fc2548789e 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,42 +1,33 @@ -import datetime -import faulthandler -import json -import locale import os -import platform import random import re +import shlex import sys import sysconfig -import tempfile import time -import unittest -from test.libregrtest.cmdline import _parse_args -from test.libregrtest.runtest import ( - findtests, runtest, get_abs_module, - STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, - INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN, - PROGRESS_MIN_TIME, format_test_result) -from test.libregrtest.setup import setup_tests -from test.libregrtest.utils import removepy, count, format_duration, printlist -from test import support -from test.support import os_helper, import_helper - - -# When tests are run from the Python build directory, it is best practice -# to keep the test files in a subfolder. This eases the cleanup of leftover -# files using the "make distclean" command. -if sysconfig.is_python_build(): - TEMPDIR = sysconfig.get_config_var('abs_builddir') - if TEMPDIR is None: - # bpo-30284: On Windows, only srcdir is available. Using abs_builddir - # mostly matters on UNIX when building Python out of the source tree, - # especially when the source tree is read only. - TEMPDIR = sysconfig.get_config_var('srcdir') - TEMPDIR = os.path.join(TEMPDIR, 'build') -else: - TEMPDIR = tempfile.gettempdir() -TEMPDIR = os.path.abspath(TEMPDIR) +import trace +from _colorize import get_colors # type: ignore[import-not-found] +from typing import NoReturn + +from test.support import os_helper, MS_WINDOWS, flush_std_streams + +from .cmdline import _parse_args, Namespace +from .findtests import findtests, split_test_packages, list_cases +from .logger import Logger +from .pgo import setup_pgo_tests +from .result import TestResult +from .results import TestResults, EXITCODE_INTERRUPTED +from .runtests import RunTests, HuntRefleak +from .setup import setup_process, setup_test_dir +from .single import run_single_test, PROGRESS_MIN_TIME +from .tsan import setup_tsan_tests, setup_tsan_parallel_tests +from .utils import ( + StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, + strip_py_suffix, count, format_duration, + printlist, get_temp_dir, get_work_dir, exit_timeout, + display_header, cleanup_temp_dir, print_warning, + is_cross_compiled, get_host_runner, + EXIT_TIMEOUT) class Regrtest: @@ -57,452 +48,413 @@ class Regrtest: files beginning with test_ will be used. The other default arguments (verbose, quiet, exclude, - single, randomize, findleaks, use_resources, trace, coverdir, + single, randomize, use_resources, trace, coverdir, print_slow, and random_seed) allow programmers calling main() directly to set the values that would normally be set by flags on the command line. """ - def __init__(self): - # Namespace of command line options - self.ns = None - - # tests - self.tests = [] - self.selected = [] - - # test results - self.good = [] - self.bad = [] - self.skipped = [] - self.resource_denieds = [] - self.environment_changed = [] - self.run_no_tests = [] - self.rerun = [] - self.first_result = None - self.interrupted = False - - # used by --slow - self.test_times = [] - - # used by --coverage, trace.Trace instance - self.tracer = None - - # used to display the progress bar "[ 3/100]" - self.start_time = time.monotonic() - self.test_count = '' - self.test_count_width = 1 - - # used by --single - self.next_single_test = None - self.next_single_filename = None - - # used by --junit-xml - self.testsuite_xml = None - - self.win_load_tracker = None - - def get_executed(self): - return (set(self.good) | set(self.bad) | set(self.skipped) - | set(self.resource_denieds) | set(self.environment_changed) - | set(self.run_no_tests)) - - def accumulate_result(self, result, rerun=False): - test_name = result.test_name - ok = result.result - - if ok not in (CHILD_ERROR, INTERRUPTED) and not rerun: - self.test_times.append((result.test_time, test_name)) - - if ok == PASSED: - self.good.append(test_name) - elif ok in (FAILED, CHILD_ERROR): - if not rerun: - self.bad.append(test_name) - elif ok == ENV_CHANGED: - self.environment_changed.append(test_name) - elif ok == SKIPPED: - self.skipped.append(test_name) - elif ok == RESOURCE_DENIED: - self.skipped.append(test_name) - self.resource_denieds.append(test_name) - elif ok == TEST_DID_NOT_RUN: - self.run_no_tests.append(test_name) - elif ok == INTERRUPTED: - self.interrupted = True + def __init__(self, ns: Namespace, _add_python_opts: bool = False): + # Log verbosity + self.verbose: int = int(ns.verbose) + self.quiet: bool = ns.quiet + self.pgo: bool = ns.pgo + self.pgo_extended: bool = ns.pgo_extended + self.tsan: bool = ns.tsan + self.tsan_parallel: bool = ns.tsan_parallel + + # Test results + self.results: TestResults = TestResults() + self.first_state: str | None = None + + # Logger + self.logger = Logger(self.results, self.quiet, self.pgo) + + # Actions + self.want_header: bool = ns.header + self.want_list_tests: bool = ns.list_tests + self.want_list_cases: bool = ns.list_cases + self.want_wait: bool = ns.wait + self.want_cleanup: bool = ns.cleanup + self.want_rerun: bool = ns.rerun + self.want_run_leaks: bool = ns.runleaks + self.want_bisect: bool = ns.bisect + + self.ci_mode: bool = (ns.fast_ci or ns.slow_ci) + self.want_add_python_opts: bool = (_add_python_opts + and ns._add_python_opts) + + # Select tests + self.match_tests: TestFilter = ns.match_tests + self.exclude: bool = ns.exclude + self.fromfile: StrPath | None = ns.fromfile + self.starting_test: TestName | None = ns.start + self.cmdline_args: TestList = ns.args + + # Workers + self.single_process: bool = ns.single_process + if self.single_process or ns.use_mp is None: + num_workers = 0 # run sequentially in a single process + elif ns.use_mp <= 0: + num_workers = -1 # run in parallel, use the number of CPUs else: - raise ValueError("invalid test result: %r" % ok) - - if rerun and ok not in {FAILED, CHILD_ERROR, INTERRUPTED}: - self.bad.remove(test_name) - - xml_data = result.xml_data - if xml_data: - import xml.etree.ElementTree as ET - for e in xml_data: - try: - self.testsuite_xml.append(ET.fromstring(e)) - except ET.ParseError: - print(xml_data, file=sys.__stderr__) - raise - - def display_progress(self, test_index, text): - if self.ns.quiet: - return - - # "[ 51/405/1] test_tcl passed" - line = f"{test_index:{self.test_count_width}}{self.test_count}" - fails = len(self.bad) + len(self.environment_changed) - if fails and not self.ns.pgo: - line = f"{line}/{fails}" - line = f"[{line}] {text}" - - # add the system load prefix: "load avg: 1.80 " - load_avg = self.getloadavg() - if load_avg is not None: - line = f"load avg: {load_avg:.2f} {line}" - - # add the timestamp prefix: "0:01:05 " - test_time = time.monotonic() - self.start_time - test_time = datetime.timedelta(seconds=int(test_time)) - line = f"{test_time} {line}" - print(line, flush=True) - - def parse_args(self, kwargs): - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'): - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later", file=sys.stderr) - ns.timeout = None + num_workers = ns.use_mp # run in parallel + self.num_workers: int = num_workers + self.worker_json: StrJSON | None = ns.worker_json + + # Options to run tests + self.fail_fast: bool = ns.failfast + self.fail_env_changed: bool = ns.fail_env_changed + self.fail_rerun: bool = ns.fail_rerun + self.forever: bool = ns.forever + self.output_on_failure: bool = ns.verbose3 + self.timeout: float | None = ns.timeout + if ns.huntrleaks: + warmups, runs, filename = ns.huntrleaks + filename = os.path.abspath(filename) + self.hunt_refleak: HuntRefleak | None = HuntRefleak(warmups, runs, filename) + else: + self.hunt_refleak = None + self.test_dir: StrPath | None = ns.testdir + self.junit_filename: StrPath | None = ns.xmlpath + self.memory_limit: str | None = ns.memlimit + self.gc_threshold: int | None = ns.threshold + self.use_resources: tuple[str, ...] = tuple(ns.use_resources) + if ns.python: + self.python_cmd: tuple[str, ...] | None = tuple(ns.python) + else: + self.python_cmd = None + self.coverage: bool = ns.trace + self.coverage_dir: StrPath | None = ns.coverdir + self._tmp_dir: StrPath | None = ns.tempdir + + # Randomize + self.randomize: bool = ns.randomize + if ('SOURCE_DATE_EPOCH' in os.environ + # don't use the variable if empty + and os.environ['SOURCE_DATE_EPOCH'] + ): + self.randomize = False + # SOURCE_DATE_EPOCH should be an integer, but use a string to not + # fail if it's not integer. random.seed() accepts a string. + # https://reproducible-builds.org/docs/source-date-epoch/ + self.random_seed: int | str = os.environ['SOURCE_DATE_EPOCH'] + elif ns.random_seed is None: + self.random_seed = random.getrandbits(32) + else: + self.random_seed = ns.random_seed + self.prioritize_tests: tuple[str, ...] = tuple(ns.prioritize) - if ns.xmlpath: - support.junit_xml_list = self.testsuite_xml = [] + self.parallel_threads = ns.parallel_threads - # Strip .py extensions. - removepy(ns.args) + # tests + self.first_runtests: RunTests | None = None - return ns + # used by --slowest + self.print_slowest: bool = ns.print_slow - def find_tests(self, tests): - self.tests = tests + # used to display the progress bar "[ 3/100]" + self.start_time = time.perf_counter() - if self.ns.single: - self.next_single_filename = os.path.join(TEMPDIR, 'pynexttest') + # used by --single + self.single_test_run: bool = ns.single + self.next_single_test: TestName | None = None + self.next_single_filename: StrPath | None = None + + def log(self, line: str = '') -> None: + self.logger.log(line) + + def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList | None]: + if tests is None: + tests = [] + if self.single_test_run: + self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest') try: with open(self.next_single_filename, 'r') as fp: next_test = fp.read().strip() - self.tests = [next_test] + tests = [next_test] except OSError: pass - if self.ns.fromfile: - self.tests = [] + if self.fromfile: + tests = [] # regex to match 'test_builtin' in line: # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' regex = re.compile(r'\btest_[a-zA-Z0-9_]+\b') - with open(os.path.join(os_helper.SAVEDCWD, self.ns.fromfile)) as fp: + with open(os.path.join(os_helper.SAVEDCWD, self.fromfile)) as fp: for line in fp: line = line.split('#', 1)[0] line = line.strip() match = regex.search(line) if match is not None: - self.tests.append(match.group()) - - removepy(self.tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if self.ns.exclude: - for arg in self.ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - self.ns.args = [] - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if self.ns.testdir: - alltests = findtests(self.ns.testdir, list(), set()) - else: - alltests = findtests(self.ns.testdir, stdtests, nottests) + tests.append(match.group()) + + strip_py_suffix(tests) + + exclude_tests = set() + if self.exclude: + for arg in self.cmdline_args: + exclude_tests.add(arg) + self.cmdline_args = [] + + if self.pgo: + # add default PGO tests if no tests are specified + setup_pgo_tests(self.cmdline_args, self.pgo_extended) + + if self.tsan: + setup_tsan_tests(self.cmdline_args) + + if self.tsan_parallel: + setup_tsan_parallel_tests(self.cmdline_args) - if not self.ns.fromfile: - self.selected = self.tests or self.ns.args or alltests + alltests = findtests(testdir=self.test_dir, + exclude=exclude_tests) + + if not self.fromfile: + selected = tests or self.cmdline_args + if exclude_tests: + # Support "--pgo/--tsan -x test_xxx" command + selected = [name for name in selected + if name not in exclude_tests] + if selected: + selected = split_test_packages(selected) + else: + selected = alltests else: - self.selected = self.tests - if self.ns.single: - self.selected = self.selected[:1] + selected = tests + + if self.single_test_run: + selected = selected[:1] try: - pos = alltests.index(self.selected[0]) + pos = alltests.index(selected[0]) self.next_single_test = alltests[pos + 1] except IndexError: pass # Remove all the selected tests that precede start if it's set. - if self.ns.start: + if self.starting_test: try: - del self.selected[:self.selected.index(self.ns.start)] + del selected[:selected.index(self.starting_test)] except ValueError: - print("Couldn't find starting test (%s), using all tests" - % self.ns.start, file=sys.stderr) + print(f"Cannot find starting test: {self.starting_test}") + sys.exit(1) - if self.ns.randomize: - if self.ns.random_seed is None: - self.ns.random_seed = random.randrange(10000000) - random.seed(self.ns.random_seed) - random.shuffle(self.selected) - - def list_tests(self): - for name in self.selected: - print(name) + random.seed(self.random_seed) + if self.randomize: + random.shuffle(selected) - def _list_cases(self, suite): - for test in suite: - if isinstance(test, unittest.loader._FailedTest): - continue - if isinstance(test, unittest.TestSuite): - self._list_cases(test) - elif isinstance(test, unittest.TestCase): - if support.match_test(test): - print(test.id()) - - def list_cases(self): - support.verbose = False - support.set_match_tests(self.ns.match_tests) - - for test_name in self.selected: - abstest = get_abs_module(self.ns, test_name) + for priority_test in reversed(self.prioritize_tests): try: - suite = unittest.defaultTestLoader.loadTestsFromName(abstest) - self._list_cases(suite) - except unittest.SkipTest: - self.skipped.append(test_name) + selected.remove(priority_test) + except ValueError: + print(f"warning: --prioritize={priority_test} used" + f" but test not actually selected") + continue + else: + selected.insert(0, priority_test) + + return (tuple(selected), tests) - if self.skipped: - print(file=sys.stderr) - print(count(len(self.skipped), "test"), "skipped:", file=sys.stderr) - printlist(self.skipped, file=sys.stderr) + @staticmethod + def list_tests(tests: TestTuple) -> None: + for name in tests: + print(name) - def rerun_failed_tests(self): - self.ns.verbose = True - self.ns.failfast = False - self.ns.verbose3 = False + def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: + # Configure the runner to re-run tests + if self.num_workers == 0 and not self.single_process: + # Always run tests in fresh processes to have more deterministic + # initial state. Don't re-run tests in parallel but limit to a + # single worker process to have side effects (on the system load + # and timings) between tests. + self.num_workers = 1 + + tests, match_tests_dict = self.results.prepare_rerun() + + # Re-run failed tests + runtests = runtests.copy( + tests=tests, + rerun=True, + verbose=True, + forever=False, + fail_fast=False, + match_tests_dict=match_tests_dict, + output_on_failure=False) + self.logger.set_tests(runtests) + + msg = f"Re-running {len(tests)} failed tests in verbose mode" + if not self.single_process: + msg = f"{msg} in subprocesses" + self.log(msg) + self._run_tests_mp(runtests, self.num_workers) + else: + self.log(msg) + self.run_tests_sequentially(runtests) + return runtests + + def rerun_failed_tests(self, runtests: RunTests) -> None: + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET + + if self.python_cmd: + # Temp patch for https://github.com/python/cpython/issues/94052 + self.log( + "Re-running failed tests is not supported with --python " + "host runner option." + ) + return - self.first_result = self.get_tests_result() + self.first_state = self.get_state() print() - print("Re-running failed tests in verbose mode") - self.rerun = self.bad[:] - for test_name in self.rerun: - print(f"Re-running {test_name} in verbose mode", flush=True) - self.ns.verbose = True - result = runtest(self.ns, test_name) + rerun_runtests = self._rerun_failed_tests(runtests) - self.accumulate_result(result, rerun=True) + if self.results.bad: + print( + f"{red}{count(len(self.results.bad), 'test')} " + f"failed again:{reset}" + ) + printlist(self.results.bad) - if result.result == INTERRUPTED: - break + self.display_result(rerun_runtests) - if self.bad: - print(count(len(self.bad), 'test'), "failed again:") - printlist(self.bad) + def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: + print() + title = f"Bisect {test}" + if progress: + title = f"{title} ({progress})" + print(title) + print("#" * len(title)) + print() - self.display_result() + cmd = runtests.create_python_cmd() + cmd.extend([ + "-u", "-m", "test.bisect_cmd", + # Limit to 25 iterations (instead of 100) to not abuse CI resources + "--max-iter", "25", + "-v", + # runtests.match_tests is not used (yet) for bisect_cmd -i arg + ]) + cmd.extend(runtests.bisect_cmd_args()) + cmd.append(test) + print("+", shlex.join(cmd), flush=True) + + flush_std_streams() + + import subprocess + proc = subprocess.run(cmd, timeout=runtests.timeout) + exitcode = proc.returncode + + title = f"{title}: exit code {exitcode}" + print(title) + print("#" * len(title)) + print(flush=True) + + if exitcode: + print(f"Bisect failed with exit code {exitcode}") + return False + + return True + + def run_bisect(self, runtests: RunTests) -> None: + tests, _ = self.results.prepare_rerun(clear=False) + + for index, name in enumerate(tests, 1): + if len(tests) > 1: + progress = f"{index}/{len(tests)}" + else: + progress = "" + if not self._run_bisect(runtests, name, progress): + return - def display_result(self): + def display_result(self, runtests: RunTests) -> None: # If running the test suite for PGO then no one cares about results. - if self.ns.pgo: + if runtests.pgo: return + state = self.get_state() print() - print("== Tests result: %s ==" % self.get_tests_result()) - - if self.interrupted: - print("Test suite interrupted by signal SIGINT.") - - omitted = set(self.selected) - self.get_executed() - if omitted: - print() - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - - if self.good and not self.ns.quiet: - print() - if (not self.bad - and not self.skipped - and not self.interrupted - and len(self.good) > 1): - print("All", end=' ') - print(count(len(self.good), "test"), "OK.") - - if self.ns.print_slow: - self.test_times.sort(reverse=True) - print() - print("10 slowest tests:") - for test_time, test in self.test_times[:10]: - print("- %s: %s" % (test, format_duration(test_time))) - - if self.bad: - print() - print(count(len(self.bad), "test"), "failed:") - printlist(self.bad) - - if self.environment_changed: - print() - print("{} altered the execution environment:".format( - count(len(self.environment_changed), "test"))) - printlist(self.environment_changed) - - if self.skipped and not self.ns.quiet: - print() - print(count(len(self.skipped), "test"), "skipped:") - printlist(self.skipped) - - if self.rerun: - print() - print("%s:" % count(len(self.rerun), "re-run test")) - printlist(self.rerun) - - if self.run_no_tests: - print() - print(count(len(self.run_no_tests), "test"), "run no tests:") - printlist(self.run_no_tests) - - def run_tests_sequential(self): - if self.ns.trace: - import trace - self.tracer = trace.Trace(trace=False, count=True) - - save_modules = sys.modules.keys() - - print("Run tests sequentially") - - previous_test = None - for test_index, test_name in enumerate(self.tests, 1): - start_time = time.monotonic() - - text = test_name - if previous_test: - text = '%s -- %s' % (text, previous_test) - self.display_progress(test_index, text) - - if self.tracer: - # If we're tracing code coverage, then we don't exit with status - # if on a false return value from main. - cmd = ('result = runtest(self.ns, test_name); ' - 'self.accumulate_result(result)') - ns = dict(locals()) - self.tracer.runctx(cmd, globals=globals(), locals=ns) - result = ns['result'] - else: - result = runtest(self.ns, test_name) - self.accumulate_result(result) + print(f"== Tests result: {state} ==") + + self.results.display_result(runtests.tests, + self.quiet, self.print_slowest) + + def run_test( + self, test_name: TestName, runtests: RunTests, tracer: trace.Trace | None + ) -> TestResult: + if tracer is not None: + # If we're tracing code coverage, then we don't exit with status + # if on a false return value from main. + cmd = ('result = run_single_test(test_name, runtests)') + namespace = dict(locals()) + tracer.runctx(cmd, globals=globals(), locals=namespace) + result = namespace['result'] + result.covered_lines = list(tracer.counts) + else: + result = run_single_test(test_name, runtests) - if result.result == INTERRUPTED: - break + self.results.accumulate_result(result, runtests) - previous_test = format_test_result(result) - test_time = time.monotonic() - start_time - if test_time >= PROGRESS_MIN_TIME: - previous_test = "%s in %s" % (previous_test, format_duration(test_time)) - elif result[0] == PASSED: - # be quiet: say nothing if the test passed shortly - previous_test = None - - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - import_helper.unload(module) - - if previous_test: - print(previous_test) - - def _test_forever(self, tests): - while True: - for test_name in tests: - yield test_name - if self.bad: - return - if self.ns.fail_env_changed and self.environment_changed: - return - - def display_header(self): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - try: - print("==", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - except: - print("== RustPython: Need to fix platform.platform") - print("== cwd:", os.getcwd()) - cpu_count = os.cpu_count() - if cpu_count: - print("== CPU count:", cpu_count) - try: - print("== encodings: locale=%s, FS=%s" - % (locale.getpreferredencoding(False), - sys.getfilesystemencoding())) - except: - print("== RustPython: Need to fix encoding stuff") - - def get_tests_result(self): - result = [] - if self.bad: - result.append("FAILURE") - elif self.ns.fail_env_changed and self.environment_changed: - result.append("ENV CHANGED") - elif not any((self.good, self.bad, self.skipped, self.interrupted, - self.environment_changed)): - result.append("NO TEST RUN") - - if self.interrupted: - result.append("INTERRUPTED") - - if not result: - result.append("SUCCESS") - - result = ', '.join(result) - if self.first_result: - result = '%s then %s' % (self.first_result, result) return result - def run_tests(self): - # For a partial run, we do not need to clutter the output. - if (self.ns.header - or not(self.ns.pgo or self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - self.display_header() - - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 3: - msg = ("WARNING: Running tests with --huntrleaks/-R and less than " - "3 warmup repetitions can give false positives!") - print(msg, file=sys.stdout, flush=True) - - if self.ns.randomize: - print("Using random seed", self.ns.random_seed) - - if self.ns.forever: - self.tests = self._test_forever(list(self.selected)) - self.test_count = '' - self.test_count_width = 3 + def run_tests_sequentially(self, runtests: RunTests) -> None: + if self.coverage: + tracer = trace.Trace(trace=False, count=True) else: - self.tests = iter(self.selected) - self.test_count = '/{}'.format(len(self.selected)) - self.test_count_width = len(self.test_count) - 1 + tracer = None - if self.ns.use_mp: - from test.libregrtest.runtest_mp import run_tests_multiprocess - run_tests_multiprocess(self) + save_modules = set(sys.modules) + + jobs = runtests.get_jobs() + if jobs is not None: + tests = count(jobs, 'test') else: - self.run_tests_sequential() + tests = 'tests' + msg = f"Run {tests} sequentially in a single process" + if runtests.timeout: + msg += " (timeout: %s)" % format_duration(runtests.timeout) + self.log(msg) + + tests_iter = runtests.iter_tests() + for test_index, test_name in enumerate(tests_iter, 1): + start_time = time.perf_counter() + + self.logger.display_progress(test_index, test_name) + + result = self.run_test(test_name, runtests, tracer) + + # Unload the newly imported test modules (best effort finalization) + new_modules = [module for module in sys.modules + if module not in save_modules and + module.startswith(("test.", "test_"))] + for module in new_modules: + sys.modules.pop(module, None) + # Remove the attribute of the parent module. + parent, _, name = module.rpartition('.') + try: + delattr(sys.modules[parent], name) + except (KeyError, AttributeError): + pass + + text = str(result) + test_time = time.perf_counter() - start_time + if test_time >= PROGRESS_MIN_TIME: + text = f"{text} in {format_duration(test_time)}" + self.logger.display_progress(test_index, text) + + if result.must_stop(self.fail_fast, self.fail_env_changed): + break - def finalize(self): - if self.win_load_tracker is not None: - self.win_load_tracker.close() - self.win_load_tracker = None + def get_state(self) -> str: + state = self.results.get_state(self.fail_env_changed) + if self.first_state: + state = f'{self.first_state} then {state}' + return state + def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None: + from .run_workers import RunWorkers + RunWorkers(num_workers, runtests, self.logger, self.results).run() + + def finalize_tests(self, coverage: trace.CoverageResults | None) -> None: if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -510,141 +462,326 @@ def finalize(self): else: os.unlink(self.next_single_filename) - if self.tracer: - r = self.tracer.results() - r.write_results(show_missing=True, summary=True, - coverdir=self.ns.coverdir) + if coverage is not None: + # uses a new-in-Python 3.13 keyword argument that mypy doesn't know about yet: + coverage.write_results(show_missing=True, summary=True, # type: ignore[call-arg] + coverdir=self.coverage_dir, + ignore_missing_files=True) + + if self.want_run_leaks: + os.system("leaks %d" % os.getpid()) + + if self.junit_filename: + self.results.write_junit(self.junit_filename) + + def display_summary(self) -> None: + if self.first_runtests is None: + raise ValueError( + "Should never call `display_summary()` before calling `_run_test()`" + ) + duration = time.perf_counter() - self.logger.start_time + filtered = bool(self.match_tests) + + # Total duration print() - duration = time.monotonic() - self.start_time print("Total duration: %s" % format_duration(duration)) - print("Tests result: %s" % self.get_tests_result()) - if self.ns.runleaks: - os.system("leaks %d" % os.getpid()) + self.results.display_summary(self.first_runtests, filtered) + + # Result + state = self.get_state() + print(f"Result: {state}") + + def create_run_tests(self, tests: TestTuple) -> RunTests: + return RunTests( + tests, + fail_fast=self.fail_fast, + fail_env_changed=self.fail_env_changed, + match_tests=self.match_tests, + match_tests_dict=None, + rerun=False, + forever=self.forever, + pgo=self.pgo, + pgo_extended=self.pgo_extended, + output_on_failure=self.output_on_failure, + timeout=self.timeout, + verbose=self.verbose, + quiet=self.quiet, + hunt_refleak=self.hunt_refleak, + test_dir=self.test_dir, + use_junit=(self.junit_filename is not None), + coverage=self.coverage, + memory_limit=self.memory_limit, + gc_threshold=self.gc_threshold, + use_resources=self.use_resources, + python_cmd=self.python_cmd, + randomize=self.randomize, + random_seed=self.random_seed, + parallel_threads=self.parallel_threads, + ) + + def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: + if self.hunt_refleak and self.hunt_refleak.warmups < 3: + msg = ("WARNING: Running tests with --huntrleaks/-R and " + "less than 3 warmup repetitions can give false positives!") + print(msg, file=sys.stdout, flush=True) + + if self.num_workers < 0: + # Use all CPUs + 2 extra worker processes for tests + # that like to sleep + # + # os.process.cpu_count() is new in Python 3.13; + # mypy doesn't know about it yet + self.num_workers = (os.process_cpu_count() or 1) + 2 # type: ignore[attr-defined] - def save_xml_result(self): - if not self.ns.xmlpath and not self.testsuite_xml: - return + # For a partial run, we do not need to clutter the output. + if (self.want_header + or not(self.pgo or self.quiet or self.single_test_run + or tests or self.cmdline_args)): + display_header(self.use_resources, self.python_cmd) - import xml.etree.ElementTree as ET - root = ET.Element("testsuites") + print("Using random seed:", self.random_seed) - # Manually count the totals for the overall summary - totals = {'tests': 0, 'errors': 0, 'failures': 0} - for suite in self.testsuite_xml: - root.append(suite) - for k in totals: - try: - totals[k] += int(suite.get(k, 0)) - except ValueError: - pass + runtests = self.create_run_tests(selected) + self.first_runtests = runtests + self.logger.set_tests(runtests) - for k, v in totals.items(): - root.set(k, str(v)) - - xmlpath = os.path.join(os_helper.SAVEDCWD, self.ns.xmlpath) - with open(xmlpath, 'wb') as f: - for s in ET.tostringlist(root): - f.write(s) - - def main(self, tests=None, **kwargs): - global TEMPDIR - self.ns = self.parse_args(kwargs) - - if self.ns.tempdir: - TEMPDIR = self.ns.tempdir - elif self.ns.worker_args: - ns_dict, _ = json.loads(self.ns.worker_args) - TEMPDIR = ns_dict.get("tempdir") or TEMPDIR - - os.makedirs(TEMPDIR, exist_ok=True) - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - test_cwd = 'test_python_{}'.format(os.getpid()) - test_cwd = os.path.join(TEMPDIR, test_cwd) - - # Run the tests in a context manager that temporarily changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from os_helper.SAVEDCWD. - with os_helper.temp_cwd(test_cwd, quiet=True): - self._main(tests, kwargs) - - def getloadavg(self): - if self.win_load_tracker is not None: - return self.win_load_tracker.getloadavg() - - if hasattr(os, 'getloadavg'): - return os.getloadavg()[0] - - return None - - def _main(self, tests, kwargs): - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 1 or repetitions < 1: - msg = ("Invalid values for the --huntrleaks/-R parameters. The " - "number of warmups and repetitions must be at least 1 " - "each (1:1).") - print(msg, file=sys.stderr, flush=True) - sys.exit(2) - - if self.ns.worker_args is not None: - from test.libregrtest.runtest_mp import run_tests_worker - run_tests_worker(self.ns.worker_args) - - if self.ns.wait: - input("Press any key to continue...") + if (runtests.hunt_refleak is not None) and (not self.num_workers): + # gh-109739: WindowsLoadTracker thread interferes with refleak check + use_load_tracker = False + else: + # WindowsLoadTracker is only needed on Windows + use_load_tracker = MS_WINDOWS - support.PGO = self.ns.pgo + if use_load_tracker: + self.logger.start_load_tracker() + try: + if self.num_workers: + self._run_tests_mp(runtests, self.num_workers) + else: + self.run_tests_sequentially(runtests) + + coverage = self.results.get_coverage_results() + self.display_result(runtests) + + if self.want_rerun and self.results.need_rerun(): + self.rerun_failed_tests(runtests) + + if self.want_bisect and self.results.need_rerun(): + self.run_bisect(runtests) + finally: + if use_load_tracker: + self.logger.stop_load_tracker() + + self.display_summary() + self.finalize_tests(coverage) + + return self.results.get_exitcode(self.fail_env_changed, + self.fail_rerun) + + def run_tests(self, selected: TestTuple, tests: TestList | None) -> int: + os.makedirs(self.tmp_dir, exist_ok=True) + work_dir = get_work_dir(self.tmp_dir) + + # Put a timeout on Python exit + with exit_timeout(): + # Run the tests in a context manager that temporarily changes the + # CWD to a temporary and writable directory. If it's not possible + # to create or change the CWD, the original CWD will be used. + # The original CWD is available from os_helper.SAVEDCWD. + with os_helper.temp_cwd(work_dir, quiet=True): + # When using multiprocessing, worker processes will use + # work_dir as their parent temporary directory. So when the + # main process exit, it removes also subdirectories of worker + # processes. + return self._run_tests(selected, tests) + + def _add_cross_compile_opts(self, regrtest_opts): + # WASM/WASI buildbot builders pass multiple PYTHON environment + # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER. + keep_environ = bool(self.python_cmd) + environ = None + + # Are we using cross-compilation? + cross_compile = is_cross_compiled() + + # Get HOSTRUNNER + hostrunner = get_host_runner() + + if cross_compile: + # emulate -E, but keep PYTHONPATH + cross compile env vars, + # so test executable can load correct sysconfigdata file. + keep = { + '_PYTHON_PROJECT_BASE', + '_PYTHON_HOST_PLATFORM', + '_PYTHON_SYSCONFIGDATA_NAME', + "_PYTHON_SYSCONFIGDATA_PATH", + 'PYTHONPATH' + } + old_environ = os.environ + new_environ = { + name: value for name, value in os.environ.items() + if not name.startswith(('PYTHON', '_PYTHON')) or name in keep + } + # Only set environ if at least one variable was removed + if new_environ != old_environ: + environ = new_environ + keep_environ = True + + if cross_compile and hostrunner: + if self.num_workers == 0 and not self.single_process: + # For now use only two cores for cross-compiled builds; + # hostrunner can be expensive. + regrtest_opts.extend(['-j', '2']) + + # If HOSTRUNNER is set and -p/--python option is not given, then + # use hostrunner to execute python binary for tests. + if not self.python_cmd: + buildpython = sysconfig.get_config_var("BUILDPYTHON") + python_cmd = f"{hostrunner} {buildpython}" + regrtest_opts.extend(["--python", python_cmd]) + keep_environ = True + + return (environ, keep_environ) + + def _add_ci_python_opts(self, python_opts, keep_environ): + # --fast-ci and --slow-ci add options to Python. + # + # Some platforms run tests in embedded mode and cannot change options + # after startup, so if this function changes, consider also updating: + # * gradle_task in Android/android.py + + # Unbuffered stdout and stderr. This isn't helpful on Android, because + # it would cause lines to be split into multiple log messages. + if not sys.stdout.write_through and sys.platform != "android": + python_opts.append('-u') + + # Add warnings filter 'error', unless the user specified a different + # filter. Ignore BytesWarning since it's controlled by '-b' below. + if not [ + opt for opt in sys.warnoptions + if not opt.endswith("::BytesWarning") + ]: + python_opts.extend(('-W', 'error')) + + # Error on bytes/str comparison + if sys.flags.bytes_warning < 2: + python_opts.append('-bb') + + if not keep_environ: + # Ignore PYTHON* environment variables + if not sys.flags.ignore_environment: + python_opts.append('-E') + + def _execute_python(self, cmd, environ): + # Make sure that messages before execv() are logged + sys.stdout.flush() + sys.stderr.flush() + + cmd_text = shlex.join(cmd) + try: + # Android and iOS run tests in embedded mode. To update their + # Python options, see the comment in _add_ci_python_opts. + if not cmd[0]: + raise ValueError("No Python executable is present") + + print(f"+ {cmd_text}", flush=True) + if hasattr(os, 'execv') and not MS_WINDOWS: + os.execv(cmd[0], cmd) + # On success, execv() do no return. + # On error, it raises an OSError. + else: + import subprocess + with subprocess.Popen(cmd, env=environ) as proc: + try: + proc.wait() + except KeyboardInterrupt: + # There is no need to call proc.terminate(): on CTRL+C, + # SIGTERM is also sent to the child process. + try: + proc.wait(timeout=EXIT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + sys.exit(EXITCODE_INTERRUPTED) + + sys.exit(proc.returncode) + except Exception as exc: + print_warning(f"Failed to change Python options: {exc!r}\n" + f"Command: {cmd_text}") + # continue executing main() + + def _add_python_opts(self) -> None: + python_opts: list[str] = [] + regrtest_opts: list[str] = [] + + environ, keep_environ = self._add_cross_compile_opts(regrtest_opts) + if self.ci_mode: + self._add_ci_python_opts(python_opts, keep_environ) + + if (not python_opts) and (not regrtest_opts) and (environ is None): + # Nothing changed: nothing to do + return - setup_tests(self.ns) + # Create new command line + cmd = list(sys.orig_argv) + if python_opts: + cmd[1:1] = python_opts + if regrtest_opts: + cmd.extend(regrtest_opts) + cmd.append("--dont-add-python-opts") - self.find_tests(tests) + self._execute_python(cmd, environ) - if self.ns.list_tests: - self.list_tests() - sys.exit(0) + def _init(self): + setup_process() - if self.ns.list_cases: - self.list_cases() - sys.exit(0) + if self.junit_filename and not os.path.isabs(self.junit_filename): + self.junit_filename = os.path.abspath(self.junit_filename) + + strip_py_suffix(self.cmdline_args) - # If we're on windows and this is the parent runner (not a worker), - # track the load average. - # TODO: RUSTPYTHON - # if sys.platform == 'win32' and (self.ns.worker_args is None): - # from test.libregrtest.win_utils import WindowsLoadTracker + self._tmp_dir = get_temp_dir(self._tmp_dir) - # try: - # self.win_load_tracker = WindowsLoadTracker() - # except FileNotFoundError as error: - # # Windows IoT Core and Windows Nano Server do not provide - # # typeperf.exe for x64, x86 or ARM - # print(f'Failed to create WindowsLoadTracker: {error}') + @property + def tmp_dir(self) -> StrPath: + if self._tmp_dir is None: + raise ValueError( + "Should never use `.tmp_dir` before calling `.main()`" + ) + return self._tmp_dir - self.run_tests() - self.display_result() + def main(self, tests: TestList | None = None) -> NoReturn: + if self.want_add_python_opts: + self._add_python_opts() - if self.ns.verbose2 and self.bad: - self.rerun_failed_tests() + self._init() - self.finalize() + if self.want_cleanup: + cleanup_temp_dir(self.tmp_dir) + sys.exit(0) + + if self.want_wait: + input("Press any key to continue...") - self.save_xml_result() + setup_test_dir(self.test_dir) + selected, tests = self.find_tests(tests) + + exitcode = 0 + if self.want_list_tests: + self.list_tests(selected) + elif self.want_list_cases: + list_cases(selected, + match_tests=self.match_tests, + test_dir=self.test_dir) + else: + exitcode = self.run_tests(selected, tests) - if self.bad: - sys.exit(2) - if self.interrupted: - sys.exit(130) - if self.ns.fail_env_changed and self.environment_changed: - sys.exit(3) - sys.exit(0) + sys.exit(exitcode) -def main(tests=None, **kwargs): +def main(tests=None, _add_python_opts=False, **kwargs) -> NoReturn: """Run the Python suite.""" - Regrtest().main(tests=tests, **kwargs) + ns = _parse_args(sys.argv[1:], **kwargs) + Regrtest(ns, _add_python_opts=_add_python_opts).main(tests=tests) diff --git a/Lib/test/libregrtest/mypy.ini b/Lib/test/libregrtest/mypy.ini new file mode 100644 index 00000000000..3fa9afcb7a4 --- /dev/null +++ b/Lib/test/libregrtest/mypy.ini @@ -0,0 +1,26 @@ +# Config file for running mypy on libregrtest. +# Run mypy by invoking `mypy --config-file Lib/test/libregrtest/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/test/libregrtest +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_incomplete_defs = False +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False +warn_return_any = False + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/test/libregrtest/parallel_case.py b/Lib/test/libregrtest/parallel_case.py new file mode 100644 index 00000000000..8eb3c314916 --- /dev/null +++ b/Lib/test/libregrtest/parallel_case.py @@ -0,0 +1,78 @@ +"""Run a test case multiple times in parallel threads.""" + +import copy +import threading +import unittest + +from unittest import TestCase + + +class ParallelTestCase(TestCase): + def __init__(self, test_case: TestCase, num_threads: int): + self.test_case = test_case + self.num_threads = num_threads + self._testMethodName = test_case._testMethodName + self._testMethodDoc = test_case._testMethodDoc + + def __str__(self): + return f"{str(self.test_case)} [threads={self.num_threads}]" + + def run_worker(self, test_case: TestCase, result: unittest.TestResult, + barrier: threading.Barrier): + barrier.wait() + test_case.run(result) + + def run(self, result=None): + if result is None: + result = test_case.defaultTestResult() + startTestRun = getattr(result, 'startTestRun', None) + stopTestRun = getattr(result, 'stopTestRun', None) + if startTestRun is not None: + startTestRun() + else: + stopTestRun = None + + # Called at the beginning of each test. See TestCase.run. + result.startTest(self) + + cases = [copy.copy(self.test_case) for _ in range(self.num_threads)] + results = [unittest.TestResult() for _ in range(self.num_threads)] + + barrier = threading.Barrier(self.num_threads) + threads = [] + for i, (case, r) in enumerate(zip(cases, results)): + thread = threading.Thread(target=self.run_worker, + args=(case, r, barrier), + name=f"{str(self.test_case)}-{i}", + daemon=True) + threads.append(thread) + + for thread in threads: + thread.start() + + for threads in threads: + threads.join() + + # Aggregate test results + if all(r.wasSuccessful() for r in results): + result.addSuccess(self) + + # Note: We can't call result.addError, result.addFailure, etc. because + # we no longer have the original exception, just the string format. + for r in results: + if len(r.errors) > 0 or len(r.failures) > 0: + result._mirrorOutput = True + result.errors.extend(r.errors) + result.failures.extend(r.failures) + result.skipped.extend(r.skipped) + result.expectedFailures.extend(r.expectedFailures) + result.unexpectedSuccesses.extend(r.unexpectedSuccesses) + result.collectedDurations.extend(r.collectedDurations) + + if any(r.shouldStop for r in results): + result.stop() + + # Test has finished running + result.stopTest(self) + if stopTestRun is not None: + stopTestRun() diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py new file mode 100644 index 00000000000..04803ddf644 --- /dev/null +++ b/Lib/test/libregrtest/pgo.py @@ -0,0 +1,55 @@ +# Set of tests run by default if --pgo is specified. The tests below were +# chosen based on the following criteria: either they exercise a commonly used +# C extension module or type, or they run some relatively typical Python code. +# Long running tests should be avoided because the PGO instrumented executable +# runs slowly. +PGO_TESTS = [ + 'test_array', + 'test_base64', + 'test_binascii', + 'test_binop', + 'test_bisect', + 'test_bytes', + 'test_bz2', + 'test_cmath', + 'test_codecs', + 'test_collections', + 'test_complex', + 'test_dataclasses', + 'test_datetime', + 'test_decimal', + 'test_difflib', + 'test_float', + 'test_fstring', + 'test_functools', + 'test_generators', + 'test_hashlib', + 'test_heapq', + 'test_int', + 'test_itertools', + 'test_json', + 'test_long', + 'test_lzma', + 'test_math', + 'test_memoryview', + 'test_operator', + 'test_ordered_dict', + 'test_patma', + 'test_pickle', + 'test_pprint', + 'test_re', + 'test_set', + 'test_sqlite3', + 'test_statistics', + 'test_str', + 'test_struct', + 'test_tabnanny', + 'test_time', + 'test_xml_etree', + 'test_xml_etree_c', +] + +def setup_pgo_tests(cmdline_args, pgo_extended: bool) -> None: + if not cmdline_args and not pgo_extended: + # run default set of tests for PGO training + cmdline_args[:] = PGO_TESTS[:] diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 03747f7f757..5c78515506d 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,9 +1,17 @@ import os -import re import sys import warnings from inspect import isabstract +from typing import Any +import linecache + from test import support +from test.support import os_helper +from test.support import refleak_helper + +from .runtests import HuntRefleak +from .utils import clear_caches + try: from _abc import _get_dump except ImportError: @@ -17,7 +25,33 @@ def _get_dump(cls): cls._abc_negative_cache, cls._abc_negative_cache_version) -def dash_R(ns, test_name, test_func): +def save_support_xml(filename): + if support.junit_xml_list is None: + return + + import pickle + with open(filename, 'xb') as fp: + pickle.dump(support.junit_xml_list, fp) + support.junit_xml_list = None + + +def restore_support_xml(filename): + try: + fp = open(filename, 'rb') + except FileNotFoundError: + return + + import pickle + with fp: + xml_list = pickle.load(fp) + os.unlink(filename) + + support.junit_xml_list = xml_list + + +def runtest_refleak(test_name, test_func, + hunt_refleak: HuntRefleak, + quiet: bool): """Run a test multiple times, looking for reference leaks. Returns: @@ -39,12 +73,19 @@ def dash_R(ns, test_name, test_func): fs = warnings.filters[:] ps = copyreg.dispatch_table.copy() pic = sys.path_importer_cache.copy() + zdc: dict[str, Any] | None + # Linecache holds a cache with the source of interactive code snippets + # (e.g. code typed in the REPL). This cache is not cleared by + # linecache.clearcache(). We need to save and restore it to avoid false + # positives. + linecache_data = linecache.cache.copy(), linecache._interactive_cache.copy() # type: ignore[attr-defined] try: import zipimport except ImportError: zdc = None # Run unmodified on platforms without zipimport support else: - zdc = zipimport._zip_directory_cache.copy() + # private attribute that mypy doesn't know about: + zdc = zipimport._zip_directory_cache.copy() # type: ignore[attr-defined] abcs = {} for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): @@ -60,9 +101,10 @@ def dash_R(ns, test_name, test_func): def get_pooled_int(value): return int_pool.setdefault(value, value) - nwarmup, ntracked, fname = ns.huntrleaks - fname = os.path.join(os_helper.SAVEDCWD, fname) - repcount = nwarmup + ntracked + warmups = hunt_refleak.warmups + runs = hunt_refleak.runs + filename = hunt_refleak.filename + repcount = warmups + runs # Pre-allocate to ensure that the loop doesn't allocate anything new rep_range = list(range(repcount)) @@ -71,45 +113,81 @@ def get_pooled_int(value): fd_deltas = [0] * repcount getallocatedblocks = sys.getallocatedblocks gettotalrefcount = sys.gettotalrefcount - fd_count = support.fd_count - + getunicodeinternedsize = sys.getunicodeinternedsize + fd_count = os_helper.fd_count # initialize variables to make pyflakes quiet - rc_before = alloc_before = fd_before = 0 + rc_before = alloc_before = fd_before = interned_immortal_before = 0 - if not ns.quiet: - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, - flush=True) + if not quiet: + print("beginning", repcount, "repetitions. Showing number of leaks " + "(. for 0 or less, X for 10 or more)", + file=sys.stderr) + numbers = ("1234567890"*(repcount//10 + 1))[:repcount] + numbers = numbers[:warmups] + ':' + numbers[warmups:] + print(numbers, file=sys.stderr, flush=True) - dash_R_cleanup(fs, ps, pic, zdc, abcs) + xml_filename = 'refleak-xml.tmp' + result = None + dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data) for i in rep_range: - test_func() - dash_R_cleanup(fs, ps, pic, zdc, abcs) - - # dash_R_cleanup() ends with collecting cyclic trash: - # read memory statistics immediately after. - alloc_after = getallocatedblocks() + support.gc_collect() + current = refleak_helper._hunting_for_refleaks + refleak_helper._hunting_for_refleaks = True + try: + result = test_func() + finally: + refleak_helper._hunting_for_refleaks = current + + save_support_xml(xml_filename) + dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data) + support.gc_collect() + + # Read memory statistics immediately after the garbage collection. + # Also, readjust the reference counts and alloc blocks by ignoring + # any strings that might have been interned during test_func. These + # strings will be deallocated at runtime shutdown + interned_immortal_after = getunicodeinternedsize( + # Use an internal-only keyword argument that mypy doesn't know yet + _only_immortal=True) # type: ignore[call-arg] + alloc_after = getallocatedblocks() - interned_immortal_after rc_after = gettotalrefcount() fd_after = fd_count() - if not ns.quiet: - print('.', end='', file=sys.stderr, flush=True) - rc_deltas[i] = get_pooled_int(rc_after - rc_before) alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before) fd_deltas[i] = get_pooled_int(fd_after - fd_before) + if not quiet: + # use max, not sum, so total_leaks is one of the pooled ints + total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i]) + if total_leaks <= 0: + symbol = '.' + elif total_leaks < 10: + symbol = ( + '.', '1', '2', '3', '4', '5', '6', '7', '8', '9', + )[total_leaks] + else: + symbol = 'X' + if i == warmups: + print(' ', end='', file=sys.stderr, flush=True) + print(symbol, end='', file=sys.stderr, flush=True) + del total_leaks + del symbol + alloc_before = alloc_after rc_before = rc_after fd_before = fd_after + interned_immortal_before = interned_immortal_after + + restore_support_xml(xml_filename) - if not ns.quiet: + if not quiet: print(file=sys.stderr) # These checkers return False on success, True on failure def check_rc_deltas(deltas): - # Checker for reference counters and memomry blocks. + # Checker for reference counters and memory blocks. # # bpo-30776: Try to ignore false positives: # @@ -133,19 +211,25 @@ def check_fd_deltas(deltas): (fd_deltas, 'file descriptors', check_fd_deltas) ]: # ignore warmup runs - deltas = deltas[nwarmup:] - if checker(deltas): + deltas = deltas[warmups:] + failing = checker(deltas) + suspicious = any(deltas) + if failing or suspicious: msg = '%s leaked %s %s, sum=%s' % ( test_name, deltas, item_name, sum(deltas)) - print(msg, file=sys.stderr, flush=True) - with open(fname, "a") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True - return failed - - -def dash_R_cleanup(fs, ps, pic, zdc, abcs): + print(msg, end='', file=sys.stderr) + if failing: + print(file=sys.stderr, flush=True) + with open(filename, "a", encoding="utf-8") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + else: + print(' (this is fine)', file=sys.stderr, flush=True) + return (failed, result) + + +def dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data): import copyreg import collections.abc @@ -155,6 +239,11 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): copyreg.dispatch_table.update(ps) sys.path_importer_cache.clear() sys.path_importer_cache.update(pic) + lcache, linteractive = linecache_data + linecache._interactive_cache.clear() + linecache._interactive_cache.update(linteractive) + linecache.cache.clear() + linecache.cache.update(lcache) try: import zipimport except ImportError: @@ -163,121 +252,28 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.clear() zipimport._zip_directory_cache.update(zdc) - # clear type cache - sys._clear_type_cache() - # Clear ABC registries, restoring previously saved ABC registries. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: - for ref in abcs.get(obj, set()): - if ref() is not None: - obj.register(ref()) + refs = abcs.get(obj, None) + if refs is not None: + obj._abc_registry_clear() + for ref in refs: + subclass = ref() + if subclass is not None: + obj.register(subclass) obj._abc_caches_clear() + # Clear caches clear_caches() - -def clear_caches(): - # Clear the warnings registry, so they can be displayed again - for mod in sys.modules.values(): - if hasattr(mod, '__warningregistry__'): - del mod.__warningregistry__ - - # Flush standard output, so that buffered data is sent to the OS and - # associated Python objects are reclaimed. - for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): - if stream is not None: - stream.flush() - - # Clear assorted module caches. - # Don't worry about resetting the cache if the module is not loaded - try: - distutils_dir_util = sys.modules['distutils.dir_util'] - except KeyError: - pass - else: - distutils_dir_util._path_created.clear() - re.purge() - - try: - _strptime = sys.modules['_strptime'] - except KeyError: - pass - else: - _strptime._regex_cache.clear() - - try: - urllib_parse = sys.modules['urllib.parse'] - except KeyError: - pass - else: - urllib_parse.clear_cache() - - try: - urllib_request = sys.modules['urllib.request'] - except KeyError: - pass - else: - urllib_request.urlcleanup() - - try: - linecache = sys.modules['linecache'] - except KeyError: - pass - else: - linecache.clearcache() - - try: - mimetypes = sys.modules['mimetypes'] - except KeyError: - pass - else: - mimetypes._default_mime_types() - - try: - filecmp = sys.modules['filecmp'] - except KeyError: - pass - else: - filecmp._cache.clear() - - try: - struct = sys.modules['struct'] - except KeyError: - pass - else: - # TODO: fix - # struct._clearcache() - pass - - try: - doctest = sys.modules['doctest'] - except KeyError: - pass - else: - doctest.master = None - - try: - ctypes = sys.modules['ctypes'] - except KeyError: - pass - else: - ctypes._reset_cache() - - try: - typing = sys.modules['typing'] - except KeyError: - pass - else: - for f in typing._cleanups: - f() - - support.gc_collect() + # Clear other caches last (previous function calls can re-populate them): + sys._clear_internal_caches() -def warm_caches(): +def warm_caches() -> None: # char cache s = bytes(range(256)) for i in range(256): diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py new file mode 100644 index 00000000000..daf7624366e --- /dev/null +++ b/Lib/test/libregrtest/result.py @@ -0,0 +1,243 @@ +import dataclasses +import json +from _colorize import get_colors # type: ignore[import-not-found] +from typing import Any + +from .utils import ( + StrJSON, TestName, FilterTuple, + format_duration, normalize_test_name, print_warning) + + +@dataclasses.dataclass(slots=True) +class TestStats: + tests_run: int = 0 + failures: int = 0 + skipped: int = 0 + + @staticmethod + def from_unittest(result): + return TestStats(result.testsRun, + len(result.failures), + len(result.skipped)) + + @staticmethod + def from_doctest(results): + return TestStats(results.attempted, + results.failed, + results.skipped) + + def accumulate(self, stats): + self.tests_run += stats.tests_run + self.failures += stats.failures + self.skipped += stats.skipped + + +# Avoid enum.Enum to reduce the number of imports when tests are run +class State: + PASSED = "PASSED" + FAILED = "FAILED" + SKIPPED = "SKIPPED" + UNCAUGHT_EXC = "UNCAUGHT_EXC" + REFLEAK = "REFLEAK" + ENV_CHANGED = "ENV_CHANGED" + RESOURCE_DENIED = "RESOURCE_DENIED" + INTERRUPTED = "INTERRUPTED" + WORKER_FAILED = "WORKER_FAILED" # non-zero worker process exit code + WORKER_BUG = "WORKER_BUG" # exception when running a worker + DID_NOT_RUN = "DID_NOT_RUN" + TIMEOUT = "TIMEOUT" + + @staticmethod + def is_failed(state): + return state in { + State.FAILED, + State.UNCAUGHT_EXC, + State.REFLEAK, + State.WORKER_FAILED, + State.WORKER_BUG, + State.TIMEOUT} + + @staticmethod + def has_meaningful_duration(state): + # Consider that the duration is meaningless for these cases. + # For example, if a whole test file is skipped, its duration + # is unlikely to be the duration of executing its tests, + # but just the duration to execute code which skips the test. + return state not in { + State.SKIPPED, + State.RESOURCE_DENIED, + State.INTERRUPTED, + State.WORKER_FAILED, + State.WORKER_BUG, + State.DID_NOT_RUN} + + @staticmethod + def must_stop(state): + return state in { + State.INTERRUPTED, + State.WORKER_BUG, + } + + +FileName = str +LineNo = int +Location = tuple[FileName, LineNo] + + +@dataclasses.dataclass(slots=True) +class TestResult: + test_name: TestName + state: str | None = None + # Test duration in seconds + duration: float | None = None + xml_data: list[str] | None = None + stats: TestStats | None = None + + # errors and failures copied from support.TestFailedWithDetails + errors: list[tuple[str, str]] | None = None + failures: list[tuple[str, str]] | None = None + + # partial coverage in a worker run; not used by sequential in-process runs + covered_lines: list[Location] | None = None + + def is_failed(self, fail_env_changed: bool) -> bool: + if self.state == State.ENV_CHANGED: + return fail_env_changed + return State.is_failed(self.state) + + def _format_failed(self): + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET + if self.errors and self.failures: + le = len(self.errors) + lf = len(self.failures) + error_s = "error" + ("s" if le > 1 else "") + failure_s = "failure" + ("s" if lf > 1 else "") + return ( + f"{red}{self.test_name} failed " + f"({le} {error_s}, {lf} {failure_s}){reset}" + ) + + if self.errors: + le = len(self.errors) + error_s = "error" + ("s" if le > 1 else "") + return f"{red}{self.test_name} failed ({le} {error_s}){reset}" + + if self.failures: + lf = len(self.failures) + failure_s = "failure" + ("s" if lf > 1 else "") + return f"{red}{self.test_name} failed ({lf} {failure_s}){reset}" + + return f"{red}{self.test_name} failed{reset}" + + def __str__(self) -> str: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + + match self.state: + case State.PASSED: + return f"{green}{self.test_name} passed{reset}" + case State.FAILED: + return f"{red}{self._format_failed()}{reset}" + case State.SKIPPED: + return f"{yellow}{self.test_name} skipped{reset}" + case State.UNCAUGHT_EXC: + return ( + f"{red}{self.test_name} failed (uncaught exception){reset}" + ) + case State.REFLEAK: + return f"{red}{self.test_name} failed (reference leak){reset}" + case State.ENV_CHANGED: + return f"{red}{self.test_name} failed (env changed){reset}" + case State.RESOURCE_DENIED: + return f"{yellow}{self.test_name} skipped (resource denied){reset}" + case State.INTERRUPTED: + return f"{yellow}{self.test_name} interrupted{reset}" + case State.WORKER_FAILED: + return ( + f"{red}{self.test_name} worker non-zero exit code{reset}" + ) + case State.WORKER_BUG: + return f"{red}{self.test_name} worker bug{reset}" + case State.DID_NOT_RUN: + return f"{yellow}{self.test_name} ran no tests{reset}" + case State.TIMEOUT: + assert self.duration is not None, "self.duration is None" + return f"{self.test_name} timed out ({format_duration(self.duration)})" + case _: + raise ValueError( + f"{red}unknown result state: {{state!r}}{reset}" + ) + + def has_meaningful_duration(self): + return State.has_meaningful_duration(self.state) + + def set_env_changed(self): + if self.state is None or self.state == State.PASSED: + self.state = State.ENV_CHANGED + + def must_stop(self, fail_fast: bool, fail_env_changed: bool) -> bool: + if State.must_stop(self.state): + return True + if fail_fast and self.is_failed(fail_env_changed): + return True + return False + + def get_rerun_match_tests(self) -> FilterTuple | None: + match_tests = [] + + errors = self.errors or [] + failures = self.failures or [] + for error_list, is_error in ( + (errors, True), + (failures, False), + ): + for full_name, *_ in error_list: + match_name = normalize_test_name(full_name, is_error=is_error) + if match_name is None: + # 'setUpModule (test.test_sys)': don't filter tests + return None + if not match_name: + error_type = "ERROR" if is_error else "FAIL" + print_warning(f"rerun failed to parse {error_type} test name: " + f"{full_name!r}: don't filter tests") + return None + match_tests.append(match_name) + + if not match_tests: + return None + return tuple(match_tests) + + def write_json_into(self, file) -> None: + json.dump(self, file, cls=_EncodeTestResult) + + @staticmethod + def from_json(worker_json: StrJSON) -> 'TestResult': + return json.loads(worker_json, object_hook=_decode_test_result) + + +class _EncodeTestResult(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, TestResult): + result = dataclasses.asdict(o) + result["__test_result__"] = o.__class__.__name__ + return result + else: + return super().default(o) + + +def _decode_test_result(data: dict[str, Any]) -> TestResult | dict[str, Any]: + if "__test_result__" in data: + data.pop('__test_result__') + if data['stats'] is not None: + data['stats'] = TestStats(**data['stats']) + if data['covered_lines'] is not None: + data['covered_lines'] = [ + tuple(loc) for loc in data['covered_lines'] + ] + return TestResult(**data) + else: + return data diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py new file mode 100644 index 00000000000..a35934fc2c9 --- /dev/null +++ b/Lib/test/libregrtest/results.py @@ -0,0 +1,309 @@ +import sys +import trace +from _colorize import get_colors # type: ignore[import-not-found] +from typing import TYPE_CHECKING + +from .runtests import RunTests +from .result import State, TestResult, TestStats, Location +from .utils import ( + StrPath, TestName, TestTuple, TestList, FilterDict, + printlist, count, format_duration) + +if TYPE_CHECKING: + from xml.etree.ElementTree import Element + + +# Python uses exit code 1 when an exception is not caught +# argparse.ArgumentParser.error() uses exit code 2 +EXITCODE_BAD_TEST = 2 +EXITCODE_ENV_CHANGED = 3 +EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_RERUN_FAIL = 5 +EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 + + +class TestResults: + def __init__(self) -> None: + self.bad: TestList = [] + self.good: TestList = [] + self.rerun_bad: TestList = [] + self.skipped: TestList = [] + self.resource_denied: TestList = [] + self.env_changed: TestList = [] + self.run_no_tests: TestList = [] + self.rerun: TestList = [] + self.rerun_results: list[TestResult] = [] + + self.interrupted: bool = False + self.worker_bug: bool = False + self.test_times: list[tuple[float, TestName]] = [] + self.stats = TestStats() + # used by --junit-xml + self.testsuite_xml: list['Element'] = [] + # used by -T with -j + self.covered_lines: set[Location] = set() + + def is_all_good(self) -> bool: + return (not self.bad + and not self.skipped + and not self.interrupted + and not self.worker_bug) + + def get_executed(self) -> set[TestName]: + return (set(self.good) | set(self.bad) | set(self.skipped) + | set(self.resource_denied) | set(self.env_changed) + | set(self.run_no_tests)) + + def no_tests_run(self) -> bool: + return not any((self.good, self.bad, self.skipped, self.interrupted, + self.env_changed)) + + def get_state(self, fail_env_changed: bool) -> str: + state = [] + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + if self.bad: + state.append(f"{red}FAILURE{reset}") + elif fail_env_changed and self.env_changed: + state.append(f"{yellow}ENV CHANGED{reset}") + elif self.no_tests_run(): + state.append(f"{yellow}NO TESTS RAN{reset}") + + if self.interrupted: + state.append(f"{yellow}INTERRUPTED{reset}") + if self.worker_bug: + state.append(f"{red}WORKER BUG{reset}") + if not state: + state.append(f"{green}SUCCESS{reset}") + + return ', '.join(state) + + def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int: + exitcode = 0 + if self.bad: + exitcode = EXITCODE_BAD_TEST + elif self.interrupted: + exitcode = EXITCODE_INTERRUPTED + elif fail_env_changed and self.env_changed: + exitcode = EXITCODE_ENV_CHANGED + elif self.no_tests_run(): + exitcode = EXITCODE_NO_TESTS_RAN + elif fail_rerun and self.rerun: + exitcode = EXITCODE_RERUN_FAIL + elif self.worker_bug: + exitcode = EXITCODE_BAD_TEST + return exitcode + + def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: + test_name = result.test_name + rerun = runtests.rerun + fail_env_changed = runtests.fail_env_changed + + match result.state: + case State.PASSED: + self.good.append(test_name) + case State.ENV_CHANGED: + self.env_changed.append(test_name) + self.rerun_results.append(result) + case State.SKIPPED: + self.skipped.append(test_name) + case State.RESOURCE_DENIED: + self.resource_denied.append(test_name) + case State.INTERRUPTED: + self.interrupted = True + case State.DID_NOT_RUN: + self.run_no_tests.append(test_name) + case _: + if result.is_failed(fail_env_changed): + self.bad.append(test_name) + self.rerun_results.append(result) + else: + raise ValueError(f"invalid test state: {result.state!r}") + + if result.state == State.WORKER_BUG: + self.worker_bug = True + + if result.has_meaningful_duration() and not rerun: + if result.duration is None: + raise ValueError("result.duration is None") + self.test_times.append((result.duration, test_name)) + if result.stats is not None: + self.stats.accumulate(result.stats) + if rerun: + self.rerun.append(test_name) + if result.covered_lines: + # we don't care about trace counts so we don't have to sum them up + self.covered_lines.update(result.covered_lines) + xml_data = result.xml_data + if xml_data: + self.add_junit(xml_data) + + def get_coverage_results(self) -> trace.CoverageResults: + counts = {loc: 1 for loc in self.covered_lines} + return trace.CoverageResults(counts=counts) + + def need_rerun(self) -> bool: + return bool(self.rerun_results) + + def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: + tests: TestList = [] + match_tests_dict = {} + for result in self.rerun_results: + tests.append(result.test_name) + + match_tests = result.get_rerun_match_tests() + # ignore empty match list + if match_tests: + match_tests_dict[result.test_name] = match_tests + + if clear: + # Clear previously failed tests + self.rerun_bad.extend(self.bad) + self.bad.clear() + self.env_changed.clear() + self.rerun_results.clear() + + return (tuple(tests), match_tests_dict) + + def add_junit(self, xml_data: list[str]) -> None: + import xml.etree.ElementTree as ET + for e in xml_data: + try: + self.testsuite_xml.append(ET.fromstring(e)) + except ET.ParseError: + print(xml_data, file=sys.__stderr__) + raise + + def write_junit(self, filename: StrPath) -> None: + if not self.testsuite_xml: + # Don't create empty XML file + return + + import xml.etree.ElementTree as ET + root = ET.Element("testsuites") + + # Manually count the totals for the overall summary + totals = {'tests': 0, 'errors': 0, 'failures': 0} + for suite in self.testsuite_xml: + root.append(suite) + for k in totals: + try: + totals[k] += int(suite.get(k, 0)) + except ValueError: + pass + + for k, v in totals.items(): + root.set(k, str(v)) + + with open(filename, 'wb') as f: + for s in ET.tostringlist(root): + f.write(s) + + def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + + if print_slowest: + self.test_times.sort(reverse=True) + print() + print(f"{yellow}10 slowest tests:{reset}") + for test_time, test in self.test_times[:10]: + print(f"- {test}: {format_duration(test_time)}") + + all_tests = [] + omitted = set(tests) - self.get_executed() + + # less important + all_tests.append( + (sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}") + ) + if not quiet: + all_tests.append( + (self.skipped, "test", f"{yellow}{{}} skipped:{reset}") + ) + all_tests.append( + ( + self.resource_denied, + "test", + f"{yellow}{{}} skipped (resource denied):{reset}", + ) + ) + all_tests.append( + (self.run_no_tests, "test", f"{yellow}{{}} run no tests:{reset}") + ) + + # more important + all_tests.append( + ( + self.env_changed, + "test", + f"{yellow}{{}} altered the execution environment (env changed):{reset}", + ) + ) + all_tests.append((self.rerun, "re-run test", f"{yellow}{{}}:{reset}")) + all_tests.append((self.bad, "test", f"{red}{{}} failed:{reset}")) + + for tests_list, count_text, title_format in all_tests: + if tests_list: + print() + count_text = count(len(tests_list), count_text) + print(title_format.format(count_text)) + printlist(tests_list) + + if self.good and not quiet: + print() + text = count(len(self.good), "test") + text = f"{green}{text} OK.{reset}" + if self.is_all_good() and len(self.good) > 1: + text = f"All {text}" + print(text) + + if self.interrupted: + print() + print(f"{yellow}Test suite interrupted by signal SIGINT.{reset}") + + def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: + # Total tests + ansi = get_colors() + red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW + + stats = self.stats + text = f'run={stats.tests_run:,}' + if filtered: + text = f"{text} (filtered)" + report = [text] + if stats.failures: + report.append(f'{red}failures={stats.failures:,}{reset}') + if stats.skipped: + report.append(f'{yellow}skipped={stats.skipped:,}{reset}') + print(f"Total tests: {' '.join(report)}") + + # Total test files + all_tests = [self.good, self.bad, self.rerun, + self.skipped, + self.env_changed, self.run_no_tests] + run = sum(map(len, all_tests)) + text = f'run={run}' + if not first_runtests.forever: + ntest = len(first_runtests.tests) + text = f"{text}/{ntest}" + if filtered: + text = f"{text} (filtered)" + report = [text] + for name, tests, color in ( + ('failed', self.bad, red), + ('env_changed', self.env_changed, yellow), + ('skipped', self.skipped, yellow), + ('resource_denied', self.resource_denied, yellow), + ('rerun', self.rerun, yellow), + ('run_no_tests', self.run_no_tests, yellow), + ): + if tests: + report.append(f'{color}{name}={len(tests)}{reset}') + print(f"Total test files: {' '.join(report)}") diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py new file mode 100644 index 00000000000..424085a0050 --- /dev/null +++ b/Lib/test/libregrtest/run_workers.py @@ -0,0 +1,627 @@ +import contextlib +import dataclasses +import faulthandler +import os.path +import queue +import signal +import subprocess +import sys +import tempfile +import threading +import time +import traceback +from typing import Any, Literal, TextIO + +from test import support +from test.support import os_helper, MS_WINDOWS + +from .logger import Logger +from .result import TestResult, State +from .results import TestResults +from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType +from .single import PROGRESS_MIN_TIME +from .utils import ( + StrPath, TestName, + format_duration, print_warning, count, plural) +from .worker import create_worker_process, USE_PROCESS_GROUP + +if MS_WINDOWS: + import locale + import msvcrt + + + +# Display the running tests if nothing happened last N seconds +PROGRESS_UPDATE = 30.0 # seconds +assert PROGRESS_UPDATE >= PROGRESS_MIN_TIME + +# Kill the main process after 5 minutes. It is supposed to write an update +# every PROGRESS_UPDATE seconds. Tolerate 5 minutes for Python slowest +# buildbot workers. +MAIN_PROCESS_TIMEOUT = 5 * 60.0 +assert MAIN_PROCESS_TIMEOUT >= PROGRESS_UPDATE + +# Time to wait until a worker completes: should be immediate +WAIT_COMPLETED_TIMEOUT = 30.0 # seconds + +# Time to wait a killed process (in seconds) +WAIT_KILLED_TIMEOUT = 60.0 + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessIterator: + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests_iter): + self.lock = threading.Lock() + self.tests_iter = tests_iter + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.tests_iter is None: + raise StopIteration + return next(self.tests_iter) + + def stop(self): + with self.lock: + self.tests_iter = None + + +@dataclasses.dataclass(slots=True, frozen=True) +class MultiprocessResult: + result: TestResult + # bpo-45410: stderr is written into stdout to keep messages order + worker_stdout: str | None = None + err_msg: str | None = None + + +class WorkerThreadExited: + """Indicates that a worker thread has exited""" + +ExcStr = str +QueueOutput = tuple[Literal[False], MultiprocessResult] | tuple[Literal[True], ExcStr] +QueueContent = QueueOutput | WorkerThreadExited + + +class ExitThread(Exception): + pass + + +class WorkerError(Exception): + def __init__(self, + test_name: TestName, + err_msg: str | None, + stdout: str | None, + state: str): + result = TestResult(test_name, state=state) + self.mp_result = MultiprocessResult(result, stdout, err_msg) + super().__init__() + + +_NOT_RUNNING = "" + + +class WorkerThread(threading.Thread): + def __init__(self, worker_id: int, runner: "RunWorkers") -> None: + super().__init__() + self.worker_id = worker_id + self.runtests = runner.runtests + self.pending = runner.pending + self.output = runner.output + self.timeout = runner.worker_timeout + self.log = runner.log + self.test_name = _NOT_RUNNING + self.start_time = time.monotonic() + self._popen: subprocess.Popen[str] | None = None + self._killed = False + self._stopped = False + + def __repr__(self) -> str: + info = [f'WorkerThread #{self.worker_id}'] + if self.is_alive(): + info.append("running") + else: + info.append('stopped') + test = self.test_name + if test: + info.append(f'test={test}') + popen = self._popen + if popen is not None: + dt = time.monotonic() - self.start_time + info.extend((f'pid={popen.pid}', + f'time={format_duration(dt)}')) + return '<%s>' % ' '.join(info) + + def _kill(self) -> None: + popen = self._popen + if popen is None: + return + + if self._killed: + return + self._killed = True + + use_killpg = USE_PROCESS_GROUP + if use_killpg: + parent_sid = os.getsid(0) + sid = os.getsid(popen.pid) + use_killpg = (sid != parent_sid) + + if use_killpg: + what = f"{self} process group" + else: + what = f"{self} process" + + print(f"Kill {what}", file=sys.stderr, flush=True) + try: + if use_killpg: + os.killpg(popen.pid, signal.SIGKILL) + else: + popen.kill() + except ProcessLookupError: + # popen.kill(): the process completed, the WorkerThread thread + # read its exit status, but Popen.send_signal() read the returncode + # just before Popen.wait() set returncode. + pass + except OSError as exc: + print_warning(f"Failed to kill {what}: {exc!r}") + + def stop(self) -> None: + # Method called from a different thread to stop this thread + self._stopped = True + self._kill() + + def _run_process(self, runtests: WorkerRunTests, output_fd: int, + tmp_dir: StrPath | None = None) -> int | None: + popen = create_worker_process(runtests, output_fd, tmp_dir) + self._popen = popen + self._killed = False + + try: + if self._stopped: + # If kill() has been called before self._popen is set, + # self._popen is still running. Call again kill() + # to ensure that the process is killed. + self._kill() + raise ExitThread + + try: + # gh-94026: stdout+stderr are written to tempfile + retcode = popen.wait(timeout=self.timeout) + assert retcode is not None + return retcode + except subprocess.TimeoutExpired: + if self._stopped: + # kill() has been called: communicate() fails on reading + # closed stdout + raise ExitThread + + # On timeout, kill the process + self._kill() + + # None means TIMEOUT for the caller + retcode = None + # bpo-38207: Don't attempt to call communicate() again: on it + # can hang until all child processes using stdout + # pipes completes. + except OSError: + if self._stopped: + # kill() has been called: communicate() fails + # on reading closed stdout + raise ExitThread + raise + return None + except: + self._kill() + raise + finally: + self._wait_completed() + self._popen = None + + def create_stdout(self, stack: contextlib.ExitStack) -> TextIO: + """Create stdout temporary file (file descriptor).""" + + if MS_WINDOWS: + # gh-95027: When stdout is not a TTY, Python uses the ANSI code + # page for the sys.stdout encoding. If the main process runs in a + # terminal, sys.stdout uses WindowsConsoleIO with UTF-8 encoding. + encoding = locale.getencoding() + else: + encoding = sys.stdout.encoding + + # gh-94026: Write stdout+stderr to a tempfile as workaround for + # non-blocking pipes on Emscripten with NodeJS. + # gh-109425: Use "backslashreplace" error handler: log corrupted + # stdout+stderr, instead of failing with a UnicodeDecodeError and not + # logging stdout+stderr at all. + stdout_file = tempfile.TemporaryFile('w+', + encoding=encoding, + errors='backslashreplace') + stack.enter_context(stdout_file) + return stdout_file + + def create_json_file(self, stack: contextlib.ExitStack) -> tuple[JsonFile, TextIO | None]: + """Create JSON file.""" + + json_file_use_stdout = self.runtests.json_file_use_stdout() + if json_file_use_stdout: + json_file = JsonFile(None, JsonFileType.STDOUT) + json_tmpfile = None + else: + json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8') + stack.enter_context(json_tmpfile) + + json_fd = json_tmpfile.fileno() + if MS_WINDOWS: + # The msvcrt module is only available on Windows; + # we run mypy with `--platform=linux` in CI + json_handle: int = msvcrt.get_osfhandle(json_fd) # type: ignore[attr-defined] + json_file = JsonFile(json_handle, + JsonFileType.WINDOWS_HANDLE) + else: + json_file = JsonFile(json_fd, JsonFileType.UNIX_FD) + return (json_file, json_tmpfile) + + def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> WorkerRunTests: + tests = (test_name,) + if self.runtests.rerun: + match_tests = self.runtests.get_match_tests(test_name) + else: + match_tests = None + + kwargs: dict[str, Any] = {} + if match_tests: + kwargs['match_tests'] = [(test, True) for test in match_tests] + if self.runtests.output_on_failure: + kwargs['verbose'] = True + kwargs['output_on_failure'] = False + return self.runtests.create_worker_runtests( + tests=tests, + json_file=json_file, + **kwargs) + + def run_tmp_files(self, worker_runtests: WorkerRunTests, + stdout_fd: int) -> tuple[int | None, list[StrPath]]: + # gh-93353: Check for leaked temporary files in the parent process, + # since the deletion of temporary files can happen late during + # Python finalization: too late for libregrtest. + if not support.is_wasi: + # Don't check for leaked temporary files and directories if Python is + # run on WASI. WASI doesn't pass environment variables like TMPDIR to + # worker processes. + tmp_dir = tempfile.mkdtemp(prefix="test_python_") + tmp_dir = os.path.abspath(tmp_dir) + try: + retcode = self._run_process(worker_runtests, + stdout_fd, tmp_dir) + finally: + tmp_files = os.listdir(tmp_dir) + os_helper.rmtree(tmp_dir) + else: + retcode = self._run_process(worker_runtests, stdout_fd) + tmp_files = [] + + return (retcode, tmp_files) + + def read_stdout(self, stdout_file: TextIO) -> str: + stdout_file.seek(0) + try: + return stdout_file.read().strip() + except Exception as exc: + # gh-101634: Catch UnicodeDecodeError if stdout cannot be + # decoded from encoding + raise WorkerError(self.test_name, + f"Cannot read process stdout: {exc}", + stdout=None, + state=State.WORKER_BUG) + + def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None, + stdout: str) -> tuple[TestResult, str]: + try: + if json_tmpfile is not None: + json_tmpfile.seek(0) + worker_json = json_tmpfile.read() + elif json_file.file_type == JsonFileType.STDOUT: + stdout, _, worker_json = stdout.rpartition("\n") + stdout = stdout.rstrip() + else: + with json_file.open(encoding='utf8') as json_fp: + worker_json = json_fp.read() + except Exception as exc: + # gh-101634: Catch UnicodeDecodeError if stdout cannot be + # decoded from encoding + err_msg = f"Failed to read worker process JSON: {exc}" + raise WorkerError(self.test_name, err_msg, stdout, + state=State.WORKER_BUG) + + if not worker_json: + raise WorkerError(self.test_name, "empty JSON", stdout, + state=State.WORKER_BUG) + + try: + result = TestResult.from_json(worker_json) + except Exception as exc: + # gh-101634: Catch UnicodeDecodeError if stdout cannot be + # decoded from encoding + err_msg = f"Failed to parse worker process JSON: {exc}" + raise WorkerError(self.test_name, err_msg, stdout, + state=State.WORKER_BUG) + + return (result, stdout) + + def _runtest(self, test_name: TestName) -> MultiprocessResult: + with contextlib.ExitStack() as stack: + stdout_file = self.create_stdout(stack) + json_file, json_tmpfile = self.create_json_file(stack) + worker_runtests = self.create_worker_runtests(test_name, json_file) + + retcode: str | int | None + retcode, tmp_files = self.run_tmp_files(worker_runtests, + stdout_file.fileno()) + + stdout = self.read_stdout(stdout_file) + + if retcode is None: + raise WorkerError(self.test_name, stdout=stdout, + err_msg=None, + state=State.TIMEOUT) + if retcode != 0: + name = support.get_signal_name(retcode) + if name: + retcode = f"{retcode} ({name})" + raise WorkerError(self.test_name, f"Exit code {retcode}", stdout, + state=State.WORKER_FAILED) + + result, stdout = self.read_json(json_file, json_tmpfile, stdout) + + if tmp_files: + msg = (f'\n\n' + f'Warning -- {test_name} leaked temporary files ' + f'({len(tmp_files)}): {", ".join(sorted(tmp_files))}') + stdout += msg + result.set_env_changed() + + return MultiprocessResult(result, stdout) + + def run(self) -> None: + fail_fast = self.runtests.fail_fast + fail_env_changed = self.runtests.fail_env_changed + try: + while not self._stopped: + try: + test_name = next(self.pending) + except StopIteration: + break + + self.start_time = time.monotonic() + self.test_name = test_name + try: + mp_result = self._runtest(test_name) + except WorkerError as exc: + mp_result = exc.mp_result + finally: + self.test_name = _NOT_RUNNING + mp_result.result.duration = time.monotonic() - self.start_time + self.output.put((False, mp_result)) + + if mp_result.result.must_stop(fail_fast, fail_env_changed): + break + except ExitThread: + pass + except BaseException: + self.output.put((True, traceback.format_exc())) + finally: + self.output.put(WorkerThreadExited()) + + def _wait_completed(self) -> None: + popen = self._popen + # only needed for mypy: + if popen is None: + raise ValueError("Should never access `._popen` before calling `.run()`") + + try: + popen.wait(WAIT_COMPLETED_TIMEOUT) + except (subprocess.TimeoutExpired, OSError) as exc: + print_warning(f"Failed to wait for {self} completion " + f"(timeout={format_duration(WAIT_COMPLETED_TIMEOUT)}): " + f"{exc!r}") + + def wait_stopped(self, start_time: float) -> None: + # bpo-38207: RunWorkers.stop_workers() called self.stop() + # which killed the process. Sometimes, killing the process from the + # main thread does not interrupt popen.communicate() in + # WorkerThread thread. This loop with a timeout is a workaround + # for that. + # + # Moreover, if this method fails to join the thread, it is likely + # that Python will hang at exit while calling threading._shutdown() + # which tries again to join the blocked thread. Regrtest.main() + # uses EXIT_TIMEOUT to workaround this second bug. + while True: + # Write a message every second + self.join(1.0) + if not self.is_alive(): + break + dt = time.monotonic() - start_time + self.log(f"Waiting for {self} thread for {format_duration(dt)}") + if dt > WAIT_KILLED_TIMEOUT: + print_warning(f"Failed to join {self} in {format_duration(dt)}") + break + + +def get_running(workers: list[WorkerThread]) -> str | None: + running: list[str] = [] + for worker in workers: + test_name = worker.test_name + if test_name == _NOT_RUNNING: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + text = f'{test_name} ({format_duration(dt)})' + running.append(text) + if not running: + return None + return f"running ({len(running)}): {', '.join(running)}" + + +class RunWorkers: + def __init__(self, num_workers: int, runtests: RunTests, + logger: Logger, results: TestResults) -> None: + self.num_workers = num_workers + self.runtests = runtests + self.log = logger.log + self.display_progress = logger.display_progress + self.results: TestResults = results + self.live_worker_count = 0 + + self.output: queue.Queue[QueueContent] = queue.Queue() + tests_iter = runtests.iter_tests() + self.pending = MultiprocessIterator(tests_iter) + self.timeout = runtests.timeout + if self.timeout is not None: + # Rely on faulthandler to kill a worker process. This timouet is + # when faulthandler fails to kill a worker process. Give a maximum + # of 5 minutes to faulthandler to kill the worker. + self.worker_timeout: float | None = min(self.timeout * 1.5, self.timeout + 5 * 60) + else: + self.worker_timeout = None + self.workers: list[WorkerThread] = [] + + jobs = self.runtests.get_jobs() + if jobs is not None: + # Don't spawn more threads than the number of jobs: + # these worker threads would never get anything to do. + self.num_workers = min(self.num_workers, jobs) + + def start_workers(self) -> None: + self.workers = [WorkerThread(index, self) + for index in range(1, self.num_workers + 1)] + jobs = self.runtests.get_jobs() + if jobs is not None: + tests = count(jobs, 'test') + else: + tests = 'tests' + nworkers = len(self.workers) + processes = plural(nworkers, "process", "processes") + msg = (f"Run {tests} in parallel using " + f"{nworkers} worker {processes}") + if self.timeout and self.worker_timeout is not None: + msg += (" (timeout: %s, worker timeout: %s)" + % (format_duration(self.timeout), + format_duration(self.worker_timeout))) + self.log(msg) + for worker in self.workers: + worker.start() + self.live_worker_count += 1 + + def stop_workers(self) -> None: + start_time = time.monotonic() + for worker in self.workers: + worker.stop() + for worker in self.workers: + worker.wait_stopped(start_time) + + def _get_result(self) -> QueueOutput | None: + pgo = self.runtests.pgo + use_faulthandler = (self.timeout is not None) + + # bpo-46205: check the status of workers every iteration to avoid + # waiting forever on an empty queue. + while self.live_worker_count > 0: + if use_faulthandler: + faulthandler.dump_traceback_later(MAIN_PROCESS_TIMEOUT, + exit=True) + + # wait for a thread + try: + result = self.output.get(timeout=PROGRESS_UPDATE) + if isinstance(result, WorkerThreadExited): + self.live_worker_count -= 1 + continue + return result + except queue.Empty: + pass + + if not pgo: + # display progress + running = get_running(self.workers) + if running: + self.log(running) + return None + + def display_result(self, mp_result: MultiprocessResult) -> None: + result = mp_result.result + pgo = self.runtests.pgo + + text = str(result) + if mp_result.err_msg: + # WORKER_BUG + text += ' (%s)' % mp_result.err_msg + elif (result.duration and result.duration >= PROGRESS_MIN_TIME and not pgo): + text += ' (%s)' % format_duration(result.duration) + if not pgo: + running = get_running(self.workers) + if running: + text += f' -- {running}' + self.display_progress(self.test_index, text) + + def _process_result(self, item: QueueOutput) -> TestResult: + """Returns True if test runner must stop.""" + if item[0]: + # Thread got an exception + format_exc = item[1] + print_warning(f"regrtest worker thread failed: {format_exc}") + result = TestResult("", state=State.WORKER_BUG) + self.results.accumulate_result(result, self.runtests) + return result + + self.test_index += 1 + mp_result = item[1] + result = mp_result.result + self.results.accumulate_result(result, self.runtests) + self.display_result(mp_result) + + # Display worker stdout + if not self.runtests.output_on_failure: + show_stdout = True + else: + # --verbose3 ignores stdout on success + show_stdout = (result.state != State.PASSED) + if show_stdout: + stdout = mp_result.worker_stdout + if stdout: + print(stdout, flush=True) + + return result + + def run(self) -> None: + fail_fast = self.runtests.fail_fast + fail_env_changed = self.runtests.fail_env_changed + + self.start_workers() + + self.test_index = 0 + try: + while True: + item = self._get_result() + if item is None: + break + + result = self._process_result(item) + if result.must_stop(fail_fast, fail_env_changed): + break + except KeyboardInterrupt: + print() + self.results.interrupted = True + finally: + if self.timeout is not None: + faulthandler.cancel_dump_traceback_later() + + # Always ensure that all worker processes are no longer + # worker when we exit this function + self.pending.stop() + self.stop_workers() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py deleted file mode 100644 index e2af18f3499..00000000000 --- a/Lib/test/libregrtest/runtest.py +++ /dev/null @@ -1,328 +0,0 @@ -import collections -import faulthandler -import functools -# import gc -import importlib -import io -import os -import sys -import time -import traceback -import unittest - -from test import support -from test.support import os_helper, import_helper -from test.libregrtest.refleak import dash_R, clear_caches -from test.libregrtest.save_env import saved_test_environment -from test.libregrtest.utils import print_warning - - -# Test result constants. -PASSED = 1 -FAILED = 0 -ENV_CHANGED = -1 -SKIPPED = -2 -RESOURCE_DENIED = -3 -INTERRUPTED = -4 -CHILD_ERROR = -5 # error in a child process -TEST_DID_NOT_RUN = -6 # error in a child process - -_FORMAT_TEST_RESULT = { - PASSED: '%s passed', - FAILED: '%s failed', - ENV_CHANGED: '%s failed (env changed)', - SKIPPED: '%s skipped', - RESOURCE_DENIED: '%s skipped (resource denied)', - INTERRUPTED: '%s interrupted', - CHILD_ERROR: '%s crashed', - TEST_DID_NOT_RUN: '%s run no tests', -} - -# Minimum duration of a test to display its duration or to mention that -# the test is running in background -PROGRESS_MIN_TIME = 30.0 # seconds - -# small set of tests to determine if we have a basically functioning interpreter -# (i.e. if any of these fail, then anything else is likely to follow) -STDTESTS = [ - # 'test_grammar', - # 'test_opcodes', - # 'test_dict', - # 'test_builtin', - # 'test_exceptions', - # 'test_types', - # 'test_unittest', - # 'test_doctest', - # 'test_doctest2', - # 'test_support' -] - -# set of tests that we don't want to be executed when using regrtest -NOTTESTS = set() - - -# used by --findleaks, store for gc.garbage -FOUND_GARBAGE = [] - - -def format_test_result(result): - fmt = _FORMAT_TEST_RESULT.get(result.result, "%s") - return fmt % result.test_name - - -def findtestdir(path=None): - return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir - - -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): - """Return a list of all applicable test modules.""" - testdir = findtestdir(testdir) - names = os.listdir(testdir) - tests = [] - others = set(stdtests) | nottests - for name in names: - mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) - return stdtests + sorted(tests) - - -def get_abs_module(ns, test_name): - if test_name.startswith('test.') or ns.testdir: - return test_name - else: - # Import it from the test package - return 'test.' + test_name - - -TestResult = collections.namedtuple('TestResult', - 'test_name result test_time xml_data') - -def _runtest(ns, test_name): - # Handle faulthandler timeout, capture stdout+stderr, XML serialization - # and measure time. - - output_on_failure = ns.verbose3 - - use_timeout = (ns.timeout is not None) - if use_timeout: - faulthandler.dump_traceback_later(ns.timeout, exit=True) - - start_time = time.perf_counter() - try: - support.set_match_tests(ns.match_tests) - support.junit_xml_list = xml_list = [] if ns.xmlpath else None - if ns.failfast: - support.failfast = True - - if output_on_failure: - support.verbose = True - - stream = io.StringIO() - orig_stdout = sys.stdout - orig_stderr = sys.stderr - try: - sys.stdout = stream - sys.stderr = stream - result = _runtest_inner(ns, test_name, - display_failure=False) - if result != PASSED: - output = stream.getvalue() - orig_stderr.write(output) - orig_stderr.flush() - finally: - sys.stdout = orig_stdout - sys.stderr = orig_stderr - else: - # Tell tests to be moderately quiet - support.verbose = ns.verbose - - result = _runtest_inner(ns, test_name, - display_failure=not ns.verbose) - - if xml_list: - import xml.etree.ElementTree as ET - xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list] - else: - xml_data = None - - test_time = time.perf_counter() - start_time - - return TestResult(test_name, result, test_time, xml_data) - finally: - if use_timeout: - faulthandler.cancel_dump_traceback_later() - support.junit_xml_list = None - - -def runtest(ns, test_name): - """Run a single test. - - ns -- regrtest namespace of options - test_name -- the name of the test - - Returns the tuple (result, test_time, xml_data), where result is one - of the constants: - - INTERRUPTED KeyboardInterrupt - RESOURCE_DENIED test skipped because resource denied - SKIPPED test skipped for some other reason - ENV_CHANGED test failed because it changed the execution environment - FAILED test failed - PASSED test passed - EMPTY_TEST_SUITE test ran no subtests. - - If ns.xmlpath is not None, xml_data is a list containing each - generated testsuite element. - """ - try: - return _runtest(ns, test_name) - except: - if not ns.pgo: - msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) - return TestResult(test_name, FAILED, 0.0, None) - - -def _test_module(the_module): - loader = unittest.TestLoader() - tests = loader.loadTestsFromModule(the_module) - for error in loader.errors: - print(error, file=sys.stderr) - if loader.errors: - raise Exception("errors while loading tests") - support.run_unittest(tests) - - -def _runtest_inner2(ns, test_name): - # Load the test function, run the test function, handle huntrleaks - # and findleaks to detect leaks - - abstest = get_abs_module(ns, test_name) - - # remove the module from sys.module to reload it if it was already imported - import_helper.unload(abstest) - - the_module = importlib.import_module(abstest) - - # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test loading. - test_runner = getattr(the_module, "test_main", None) - if test_runner is None: - test_runner = functools.partial(_test_module, the_module) - - try: - if ns.huntrleaks: - # Return True if the test leaked references - refleak = dash_R(ns, test_name, test_runner) - else: - test_runner() - refleak = False - finally: - cleanup_test_droppings(test_name, ns.verbose) - - support.gc_collect() - - # if gc.garbage: - # support.environment_altered = True - # print_warning(f"{test_name} created {len(gc.garbage)} " - # f"uncollectable object(s).") - - # # move the uncollectable objects somewhere, - # # so we don't see them again - # FOUND_GARBAGE.extend(gc.garbage) - # gc.garbage.clear() - - support.reap_children() - - return refleak - - -def _runtest_inner(ns, test_name, display_failure=True): - # Detect environment changes, handle exceptions. - - # Reset the environment_altered flag to detect if a test altered - # the environment - support.environment_altered = False - - if ns.pgo: - display_failure = False - - try: - clear_caches() - - # with saved_test_environment(test_name, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: - refleak = _runtest_inner2(ns, test_name) - except support.ResourceDenied as msg: - if not ns.quiet and not ns.pgo: - print(f"{test_name} skipped -- {msg}", flush=True) - return RESOURCE_DENIED - except unittest.SkipTest as msg: - if not ns.quiet and not ns.pgo: - print(f"{test_name} skipped -- {msg}", flush=True) - return SKIPPED - except support.TestFailed as exc: - msg = f"test {test_name} failed" - if display_failure: - msg = f"{msg} -- {exc}" - print(msg, file=sys.stderr, flush=True) - return FAILED - except support.TestDidNotRun: - return TEST_DID_NOT_RUN - except KeyboardInterrupt: - print() - return INTERRUPTED - except: - if not ns.pgo: - msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", - file=sys.stderr, flush=True) - return FAILED - - if refleak: - return FAILED - # if environment.changed: - # return ENV_CHANGED - return PASSED - - -def cleanup_test_droppings(test_name, verbose): - # First kill any dangling references to open files etc. - # This can also issue some ResourceWarnings which would otherwise get - # triggered during the following test run, and possibly produce failures. - support.gc_collect() - - # Try to clean up junk commonly left behind. While tests shouldn't leave - # any files or directories behind, when a test fails that can be tedious - # for it to arrange. The consequences can be especially nasty on Windows, - # since if a test leaves a file open, it cannot be deleted by name (while - # there's nothing we can do about that here either, we can display the - # name of the offending test, which is a real help). - for name in (os_helper.TESTFN, - "db_home", - ): - if not os.path.exists(name): - continue - - if os.path.isdir(name): - import shutil - kind, nuker = "directory", shutil.rmtree - elif os.path.isfile(name): - kind, nuker = "file", os.unlink - else: - raise RuntimeError(f"os.path says {name!r} exists but is neither " - f"directory nor file") - - if verbose: - print_warning("%r left behind %s %r" % (test_name, kind, name)) - support.environment_altered = True - - try: - import stat - # fix possible permissions problems that might prevent cleanup - os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - nuker(name) - except Exception as exc: - print_warning(f"{test_name} left behind {kind} {name!r} " - f"and it couldn't be removed: {exc}") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py deleted file mode 100644 index c2177d99955..00000000000 --- a/Lib/test/libregrtest/runtest_mp.py +++ /dev/null @@ -1,288 +0,0 @@ -import collections -import faulthandler -import json -import os -import queue -import subprocess -import sys -import threading -import time -import traceback -import types -from test import support - -from test.libregrtest.runtest import ( - runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, - format_test_result, TestResult) -from test.libregrtest.setup import setup_tests -from test.libregrtest.utils import format_duration -from test.support import os_helper - - -# Display the running tests if nothing happened last N seconds -PROGRESS_UPDATE = 30.0 # seconds - - -def must_stop(result): - return result.result in (INTERRUPTED, CHILD_ERROR) - - -def run_test_in_subprocess(testname, ns): - ns_dict = vars(ns) - worker_args = (ns_dict, testname) - worker_args = json.dumps(worker_args) - - cmd = [sys.executable, *support.args_from_interpreter_flags(), - '-u', # Unbuffered stdout and stderr - '-m', 'test.regrtest', - '--worker-args', worker_args] - if ns.pgo: - cmd += ['--pgo'] - - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - return subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=os_helper.SAVEDCWD) - - -def run_tests_worker(worker_args): - ns_dict, testname = json.loads(worker_args) - ns = types.SimpleNamespace(**ns_dict) - - setup_tests(ns) - - result = runtest(ns, testname) - print() # Force a newline (just in case) - print(json.dumps(result), flush=True) - sys.exit(0) - - -# We do not use a generator so multiple threads can call next(). -class MultiprocessIterator: - - """A thread-safe iterator over tests for multiprocess mode.""" - - def __init__(self, tests): - self.lock = threading.Lock() - self.tests = tests - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - return next(self.tests) - - -MultiprocessResult = collections.namedtuple('MultiprocessResult', - 'result stdout stderr error_msg') - -class MultiprocessThread(threading.Thread): - def __init__(self, pending, output, ns): - super().__init__() - self.pending = pending - self.output = output - self.ns = ns - self.current_test_name = None - self.start_time = None - self._popen = None - - def kill(self): - if not self.is_alive(): - return - if self._popen is not None: - self._popen.kill() - - def _runtest(self, test_name): - try: - self.start_time = time.monotonic() - self.current_test_name = test_name - - popen = run_test_in_subprocess(test_name, self.ns) - self._popen = popen - with popen: - try: - stdout, stderr = popen.communicate() - except: - popen.kill() - popen.wait() - raise - - retcode = popen.wait() - finally: - self.current_test_name = None - self._popen = None - - stdout = stdout.strip() - stderr = stderr.rstrip() - - err_msg = None - if retcode != 0: - err_msg = "Exit code %s" % retcode - else: - stdout, _, result = stdout.rpartition("\n") - stdout = stdout.rstrip() - if not result: - err_msg = "Failed to parse worker stdout" - else: - try: - # deserialize run_tests_worker() output - result = json.loads(result) - result = TestResult(*result) - except Exception as exc: - err_msg = "Failed to parse worker JSON: %s" % exc - - if err_msg is not None: - test_time = time.monotonic() - self.start_time - result = TestResult(test_name, CHILD_ERROR, test_time, None) - - return MultiprocessResult(result, stdout, stderr, err_msg) - - def run(self): - while True: - try: - try: - test_name = next(self.pending) - except StopIteration: - break - - mp_result = self._runtest(test_name) - self.output.put((False, mp_result)) - - if must_stop(mp_result.result): - break - except BaseException: - self.output.put((True, traceback.format_exc())) - break - - -def get_running(workers): - running = [] - for worker in workers: - current_test_name = worker.current_test_name - if not current_test_name: - continue - dt = time.monotonic() - worker.start_time - if dt >= PROGRESS_MIN_TIME: - text = '%s (%s)' % (current_test_name, format_duration(dt)) - running.append(text) - return running - - -class MultiprocessRunner: - def __init__(self, regrtest): - self.regrtest = regrtest - self.ns = regrtest.ns - self.output = queue.Queue() - self.pending = MultiprocessIterator(self.regrtest.tests) - if self.ns.timeout is not None: - self.test_timeout = self.ns.timeout * 1.5 - else: - self.test_timeout = None - self.workers = None - - def start_workers(self): - self.workers = [MultiprocessThread(self.pending, self.output, self.ns) - for _ in range(self.ns.use_mp)] - print("Run tests in parallel using %s child processes" - % len(self.workers)) - for worker in self.workers: - worker.start() - - def wait_workers(self): - for worker in self.workers: - worker.kill() - for worker in self.workers: - worker.join() - - def _get_result(self): - if not any(worker.is_alive() for worker in self.workers): - # all worker threads are done: consume pending results - try: - return self.output.get(timeout=0) - except queue.Empty: - return None - - while True: - if self.test_timeout is not None: - faulthandler.dump_traceback_later(self.test_timeout, exit=True) - - # wait for a thread - timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) - try: - return self.output.get(timeout=timeout) - except queue.Empty: - pass - - # display progress - running = get_running(self.workers) - if running and not self.ns.pgo: - print('running: %s' % ', '.join(running), flush=True) - - def display_result(self, mp_result): - result = mp_result.result - - text = format_test_result(result) - if mp_result.error_msg is not None: - # CHILD_ERROR - text += ' (%s)' % mp_result.error_msg - elif (result.test_time >= PROGRESS_MIN_TIME and not self.ns.pgo): - text += ' (%s)' % format_duration(result.test_time) - running = get_running(self.workers) - if running and not self.ns.pgo: - text += ' -- running: %s' % ', '.join(running) - self.regrtest.display_progress(self.test_index, text) - - def _process_result(self, item): - if item[0]: - # Thread got an exception - format_exc = item[1] - print(f"regrtest worker thread failed: {format_exc}", - file=sys.stderr, flush=True) - return True - - self.test_index += 1 - mp_result = item[1] - self.regrtest.accumulate_result(mp_result.result) - self.display_result(mp_result) - - if mp_result.stdout: - print(mp_result.stdout, flush=True) - if mp_result.stderr and not self.ns.pgo: - print(mp_result.stderr, file=sys.stderr, flush=True) - - if must_stop(mp_result.result): - return True - - return False - - def run_tests(self): - self.start_workers() - - self.test_index = 0 - try: - while True: - item = self._get_result() - if item is None: - break - - stop = self._process_result(item) - if stop: - break - except KeyboardInterrupt: - print() - self.regrtest.interrupted = True - finally: - if self.test_timeout is not None: - faulthandler.cancel_dump_traceback_later() - - self.wait_workers() - - -def run_tests_multiprocess(regrtest): - MultiprocessRunner(regrtest).run_tests() diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py new file mode 100644 index 00000000000..759f24fc25e --- /dev/null +++ b/Lib/test/libregrtest/runtests.py @@ -0,0 +1,225 @@ +import contextlib +import dataclasses +import json +import os +import shlex +import subprocess +import sys +from typing import Any, Iterator + +from test import support + +from .utils import ( + StrPath, StrJSON, TestTuple, TestName, TestFilter, FilterTuple, FilterDict) + + +class JsonFileType: + UNIX_FD = "UNIX_FD" + WINDOWS_HANDLE = "WINDOWS_HANDLE" + STDOUT = "STDOUT" + + +@dataclasses.dataclass(slots=True, frozen=True) +class JsonFile: + # file type depends on file_type: + # - UNIX_FD: file descriptor (int) + # - WINDOWS_HANDLE: handle (int) + # - STDOUT: use process stdout (None) + file: int | None + file_type: str + + def configure_subprocess(self, popen_kwargs: dict[str, Any]) -> None: + match self.file_type: + case JsonFileType.UNIX_FD: + # Unix file descriptor + popen_kwargs['pass_fds'] = [self.file] + case JsonFileType.WINDOWS_HANDLE: + # Windows handle + # We run mypy with `--platform=linux` so it complains about this: + startupinfo = subprocess.STARTUPINFO() # type: ignore[attr-defined] + startupinfo.lpAttributeList = {"handle_list": [self.file]} + popen_kwargs['startupinfo'] = startupinfo + + @contextlib.contextmanager + def inherit_subprocess(self) -> Iterator[None]: + if sys.platform == 'win32' and self.file_type == JsonFileType.WINDOWS_HANDLE: + os.set_handle_inheritable(self.file, True) + try: + yield + finally: + os.set_handle_inheritable(self.file, False) + else: + yield + + def open(self, mode='r', *, encoding): + if self.file_type == JsonFileType.STDOUT: + raise ValueError("for STDOUT file type, just use sys.stdout") + + file = self.file + if self.file_type == JsonFileType.WINDOWS_HANDLE: + import msvcrt + # Create a file descriptor from the handle + file = msvcrt.open_osfhandle(file, os.O_WRONLY) + return open(file, mode, encoding=encoding) + + +@dataclasses.dataclass(slots=True, frozen=True) +class HuntRefleak: + warmups: int + runs: int + filename: StrPath + + def bisect_cmd_args(self) -> list[str]: + # Ignore filename since it can contain colon (":"), + # and usually it's not used. Use the default filename. + return ["-R", f"{self.warmups}:{self.runs}:"] + + +@dataclasses.dataclass(slots=True, frozen=True) +class RunTests: + tests: TestTuple + fail_fast: bool + fail_env_changed: bool + match_tests: TestFilter + match_tests_dict: FilterDict | None + rerun: bool + forever: bool + pgo: bool + pgo_extended: bool + output_on_failure: bool + timeout: float | None + verbose: int + quiet: bool + hunt_refleak: HuntRefleak | None + test_dir: StrPath | None + use_junit: bool + coverage: bool + memory_limit: str | None + gc_threshold: int | None + use_resources: tuple[str, ...] + python_cmd: tuple[str, ...] | None + randomize: bool + random_seed: int | str + parallel_threads: int | None + + def copy(self, **override) -> 'RunTests': + state = dataclasses.asdict(self) + state.update(override) + return RunTests(**state) + + def create_worker_runtests(self, **override) -> WorkerRunTests: + state = dataclasses.asdict(self) + state.update(override) + return WorkerRunTests(**state) + + def get_match_tests(self, test_name: TestName) -> FilterTuple | None: + if self.match_tests_dict is not None: + return self.match_tests_dict.get(test_name, None) + else: + return None + + def get_jobs(self) -> int | None: + # Number of run_single_test() calls needed to run all tests. + # None means that there is not bound limit (--forever option). + if self.forever: + return None + return len(self.tests) + + def iter_tests(self) -> Iterator[TestName]: + if self.forever: + while True: + yield from self.tests + else: + yield from self.tests + + def json_file_use_stdout(self) -> bool: + # Use STDOUT in two cases: + # + # - If --python command line option is used; + # - On Emscripten and WASI. + # + # On other platforms, UNIX_FD or WINDOWS_HANDLE can be used. + return ( + bool(self.python_cmd) + or support.is_emscripten + or support.is_wasi + ) + + def create_python_cmd(self) -> list[str]: + python_opts = support.args_from_interpreter_flags() + if self.python_cmd is not None: + executable = self.python_cmd + # Remove -E option, since --python=COMMAND can set PYTHON + # environment variables, such as PYTHONPATH, in the worker + # process. + python_opts = [opt for opt in python_opts if opt != "-E"] + else: + executable = (sys.executable,) + cmd = [*executable, *python_opts] + if '-u' not in python_opts: + cmd.append('-u') # Unbuffered stdout and stderr + if self.coverage: + cmd.append("-Xpresite=test.cov") + return cmd + + def bisect_cmd_args(self) -> list[str]: + args = [] + if self.fail_fast: + args.append("--failfast") + if self.fail_env_changed: + args.append("--fail-env-changed") + if self.timeout: + args.append(f"--timeout={self.timeout}") + if self.hunt_refleak is not None: + args.extend(self.hunt_refleak.bisect_cmd_args()) + if self.test_dir: + args.extend(("--testdir", self.test_dir)) + if self.memory_limit: + args.extend(("--memlimit", self.memory_limit)) + if self.gc_threshold: + args.append(f"--threshold={self.gc_threshold}") + if self.use_resources: + args.extend(("-u", ','.join(self.use_resources))) + if self.python_cmd: + cmd = shlex.join(self.python_cmd) + args.extend(("--python", cmd)) + if self.randomize: + args.append(f"--randomize") + if self.parallel_threads: + args.append(f"--parallel-threads={self.parallel_threads}") + args.append(f"--randseed={self.random_seed}") + return args + + +@dataclasses.dataclass(slots=True, frozen=True) +class WorkerRunTests(RunTests): + json_file: JsonFile + + def as_json(self) -> StrJSON: + return json.dumps(self, cls=_EncodeRunTests) + + @staticmethod + def from_json(worker_json: StrJSON) -> 'WorkerRunTests': + return json.loads(worker_json, object_hook=_decode_runtests) + + +class _EncodeRunTests(json.JSONEncoder): + def default(self, o: Any) -> dict[str, Any]: + if isinstance(o, WorkerRunTests): + result = dataclasses.asdict(o) + result["__runtests__"] = True + return result + else: + return super().default(o) + + +def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: + if "__runtests__" in data: + data.pop('__runtests__') + if data['hunt_refleak']: + data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) + if data['json_file']: + data['json_file'] = JsonFile(**data['json_file']) + return WorkerRunTests(**data) + else: + return data diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index b9a1c0b3926..eeb1f17d702 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -1,20 +1,24 @@ -import asyncio import builtins import locale -import logging import os -import shutil import sys -import sysconfig import threading -import warnings + from test import support from test.support import os_helper -from test.libregrtest.utils import print_warning + +from .utils import print_warning + +# Import termios to save and restore terminal echo. This is only available on +# Unix, and it's fine if the module can't be found. try: - import _multiprocessing, multiprocessing.process -except ImportError: - multiprocessing = None + import termios # noqa: F401 +except ModuleNotFoundError: + pass + + +class SkipTestEnvironment(Exception): + pass # Unit tests are supposed to leave the execution environment unchanged @@ -28,21 +32,19 @@ class saved_test_environment: """Save bits of the test environment and restore them at block exit. - with saved_test_environment(testname, verbose, quiet): + with saved_test_environment(test_name, verbose, quiet): #stuff Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. + the saved items was changed by the test. The support.environment_altered + attribute is set to True if a change is detected. If verbose is more than 1, the before and after state of changed items is also printed. """ - changed = False - - def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): - self.testname = testname + def __init__(self, test_name, verbose, quiet, *, pgo): + self.test_name = test_name self.verbose = verbose self.quiet = quiet self.pgo = pgo @@ -69,12 +71,41 @@ def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): 'files', 'locale', 'warnings.showwarning', 'shutil_archive_formats', 'shutil_unpack_formats', 'asyncio.events._event_loop_policy', + 'urllib.requests._url_tempfiles', 'urllib.requests._opener', + 'stty_echo', ) + def get_module(self, name): + # function for restore() methods + return sys.modules[name] + + def try_get_module(self, name): + # function for get() methods + try: + return self.get_module(name) + except KeyError: + raise SkipTestEnvironment + + def get_urllib_requests__url_tempfiles(self): + urllib_request = self.try_get_module('urllib.request') + return list(urllib_request._url_tempfiles) + def restore_urllib_requests__url_tempfiles(self, tempfiles): + for filename in tempfiles: + os_helper.unlink(filename) + + def get_urllib_requests__opener(self): + urllib_request = self.try_get_module('urllib.request') + return urllib_request._opener + def restore_urllib_requests__opener(self, opener): + urllib_request = self.get_module('urllib.request') + urllib_request._opener = opener + def get_asyncio_events__event_loop_policy(self): + self.try_get_module('asyncio') return support.maybe_get_event_loop_policy() def restore_asyncio_events__event_loop_policy(self, policy): - asyncio.set_event_loop_policy(policy) + asyncio = self.get_module('asyncio') + asyncio.events._set_event_loop_policy(policy) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] @@ -132,39 +163,46 @@ def restore___import__(self, import_): builtins.__import__ = import_ def get_warnings_filters(self): + warnings = self.try_get_module('warnings') return id(warnings.filters), warnings.filters, warnings.filters[:] def restore_warnings_filters(self, saved_filters): + warnings = self.get_module('warnings') warnings.filters = saved_filters[1] warnings.filters[:] = saved_filters[2] def get_asyncore_socket_map(self): - asyncore = sys.modules.get('asyncore') + asyncore = sys.modules.get('test.support.asyncore') # XXX Making a copy keeps objects alive until __exit__ gets called. return asyncore and asyncore.socket_map.copy() or {} def restore_asyncore_socket_map(self, saved_map): - asyncore = sys.modules.get('asyncore') + asyncore = sys.modules.get('test.support.asyncore') if asyncore is not None: asyncore.close_all(ignore_all=True) asyncore.socket_map.update(saved_map) def get_shutil_archive_formats(self): + shutil = self.try_get_module('shutil') # we could call get_archives_formats() but that only returns the # registry keys; we want to check the values too (the functions that # are registered) return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() def restore_shutil_archive_formats(self, saved): + shutil = self.get_module('shutil') shutil._ARCHIVE_FORMATS = saved[0] shutil._ARCHIVE_FORMATS.clear() shutil._ARCHIVE_FORMATS.update(saved[1]) def get_shutil_unpack_formats(self): + shutil = self.try_get_module('shutil') return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() def restore_shutil_unpack_formats(self, saved): + shutil = self.get_module('shutil') shutil._UNPACK_FORMATS = saved[0] shutil._UNPACK_FORMATS.clear() shutil._UNPACK_FORMATS.update(saved[1]) def get_logging__handlers(self): + logging = self.try_get_module('logging') # _handlers is a WeakValueDictionary return id(logging._handlers), logging._handlers, logging._handlers.copy() def restore_logging__handlers(self, saved_handlers): @@ -172,6 +210,7 @@ def restore_logging__handlers(self, saved_handlers): pass def get_logging__handlerList(self): + logging = self.try_get_module('logging') # _handlerList is a list of weakrefs to handlers return id(logging._handlerList), logging._handlerList, logging._handlerList[:] def restore_logging__handlerList(self, saved_handlerList): @@ -188,46 +227,54 @@ def restore_sys_warnoptions(self, saved_options): # to track reference leaks. def get_threading__dangling(self): # This copies the weakrefs without making any strong reference - return threading._dangling.copy() + # XXX: RUSTPYTHON - filter out dead threads since gc doesn't clean WeakSet. Revert this line when we have a GC + # return threading._dangling.copy() + return {t for t in threading._dangling if t.is_alive()} def restore_threading__dangling(self, saved): threading._dangling.clear() threading._dangling.update(saved) # Same for Process objects def get_multiprocessing_process__dangling(self): - if not multiprocessing: - return None + multiprocessing_process = self.try_get_module('multiprocessing.process') # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() + multiprocessing_process._cleanup() # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() + # TODO: RUSTPYTHON - filter out dead processes since gc doesn't clean WeakSet. Revert this line when we have a GC + # return multiprocessing_process._dangling.copy() + return {p for p in multiprocessing_process._dangling if p.is_alive()} def restore_multiprocessing_process__dangling(self, saved): - if not multiprocessing: - return - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(saved) + multiprocessing_process = self.get_module('multiprocessing.process') + multiprocessing_process._dangling.clear() + multiprocessing_process._dangling.update(saved) def get_sysconfig__CONFIG_VARS(self): # make sure the dict is initialized + sysconfig = self.try_get_module('sysconfig') sysconfig.get_config_var('prefix') return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, dict(sysconfig._CONFIG_VARS)) def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig = self.get_module('sysconfig') sysconfig._CONFIG_VARS = saved[1] sysconfig._CONFIG_VARS.clear() sysconfig._CONFIG_VARS.update(saved[2]) def get_sysconfig__INSTALL_SCHEMES(self): + sysconfig = self.try_get_module('sysconfig') return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, sysconfig._INSTALL_SCHEMES.copy()) def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig = self.get_module('sysconfig') sysconfig._INSTALL_SCHEMES = saved[1] sysconfig._INSTALL_SCHEMES.clear() sysconfig._INSTALL_SCHEMES.update(saved[2]) def get_files(self): + # XXX: Maybe add an allow-list here? return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) + for fn in os.listdir() + if not fn.startswith(".hypothesis")) def restore_files(self, saved_value): fn = os_helper.TESTFN if fn not in saved_value and (fn + '/') not in saved_value: @@ -251,10 +298,30 @@ def restore_locale(self, saved): locale.setlocale(lc, setting) def get_warnings_showwarning(self): + warnings = self.try_get_module('warnings') return warnings.showwarning def restore_warnings_showwarning(self, fxn): + warnings = self.get_module('warnings') warnings.showwarning = fxn + def get_stty_echo(self): + termios = self.try_get_module('termios') + if not os.isatty(fd := sys.__stdin__.fileno()): + return None + attrs = termios.tcgetattr(fd) + lflags = attrs[3] + return bool(lflags & termios.ECHO) + def restore_stty_echo(self, echo): + termios = self.get_module('termios') + attrs = termios.tcgetattr(fd := sys.__stdin__.fileno()) + if echo: + # Turn echo on. + attrs[3] |= termios.ECHO + else: + # Turn echo off. + attrs[3] &= ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, attrs) + def resource_info(self): for name in self.resources: method_suffix = name.replace('.', '_') @@ -263,29 +330,32 @@ def resource_info(self): yield name, getattr(self, get_name), getattr(self, restore_name) def __enter__(self): - self.saved_values = dict((name, get()) for name, get, restore - in self.resource_info()) + self.saved_values = [] + for name, get, restore in self.resource_info(): + try: + original = get() + except SkipTestEnvironment: + continue + + self.saved_values.append((name, get, restore, original)) return self def __exit__(self, exc_type, exc_val, exc_tb): saved_values = self.saved_values - del self.saved_values + self.saved_values = None # Some resources use weak references support.gc_collect() - # Read support.environment_altered, set by support helper functions - self.changed |= support.environment_altered - - for name, get, restore in self.resource_info(): + for name, get, restore, original in saved_values: current = get() - original = saved_values.pop(name) # Check for changes to the resource's value if current != original: - self.changed = True + support.environment_altered = True restore(original) if not self.quiet and not self.pgo: - print_warning(f"{name} was modified by {self.testname}") - print(f" Before: {original}\n After: {current} ", - file=sys.stderr, flush=True) + print_warning( + f"{name} was modified by {self.test_name}\n" + f" Before: {original}\n" + f" After: {current} ") return False diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index b1a5ded5254..b9b76a44e3b 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -1,17 +1,34 @@ -import atexit import faulthandler +import gc +import io import os +import random import signal import sys import unittest from test import support -try: - import gc -except ImportError: - gc = None +from test.support.os_helper import TESTFN_UNDECODABLE, FS_NONASCII +from _colorize import can_colorize # type: ignore[import-not-found] +from .filter import set_match_tests +from .runtests import RunTests +from .utils import ( + setup_unraisable_hook, setup_threading_excepthook, + adjust_rlimit_nofile) -def setup_tests(ns): + +UNICODE_GUARD_ENV = "PYTHONREGRTEST_UNICODE_GUARD" + + +def setup_test_dir(testdir: str | None) -> None: + if testdir: + # Prepend test directory to sys.path, so runtest() will be able + # to locate tests + sys.path.insert(0, os.path.abspath(testdir)) + + +def setup_process() -> None: + assert sys.__stderr__ is not None, "sys.__stderr__ is None" try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): @@ -19,13 +36,13 @@ def setup_tests(ns): # and ValueError on a closed stream. # # Catch AttributeError for stderr being None. - stderr_fd = None + pass else: # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True, file=stderr_fd) # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] + signals: list[signal.Signals] = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): @@ -33,13 +50,17 @@ def setup_tests(ns): for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) - # replace_stdout() - # support.record_original_stdout(sys.stdout) + adjust_rlimit_nofile() - if ns.testdir: - # Prepend test directory to sys.path, so runtest() will be able - # to locate tests - sys.path.insert(0, os.path.abspath(ns.testdir)) + support.record_original_stdout(sys.stdout) + + # Set sys.stdout encoder error handler to backslashreplace, + # similar to sys.stderr error handler, to avoid UnicodeEncodeError + # when printing a traceback or any other non-encodable character. + # + # Use an assertion to fix mypy error. + assert isinstance(sys.stdout, io.TextIOWrapper) + sys.stdout.reconfigure(errors="backslashreplace") # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -56,79 +77,73 @@ def setup_tests(ns): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if getattr(module, '__file__', None): - module.__file__ = os.path.abspath(module.__file__) - - # MacOSX (a.k.a. Darwin) has a default stack size that is too small - # for deeply recursive regular expressions. We see this as crashes in - # the Python test suite when running test_re.py and test_sre.py. The - # fix is to set the stack limit to 2048. - # This approach may also be useful for other Unixy platforms that - # suffer from small default stack limits. - if sys.platform == 'darwin': - try: - import resource - except ImportError: + module.__file__ = os.path.abspath(module.__file__) # type: ignore[type-var] + + if hasattr(sys, 'addaudithook'): + # Add an auditing hook for all tests to ensure PySys_Audit is tested + def _test_audit_hook(name, args): pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + sys.addaudithook(_test_audit_hook) - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + setup_unraisable_hook() + setup_threading_excepthook() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) + # Ensure there's a non-ASCII character in env vars at all times to force + # tests consider this case. See BPO-44647 for details. + if TESTFN_UNDECODABLE and os.supports_bytes_environ: + os.environb.setdefault(UNICODE_GUARD_ENV.encode(), TESTFN_UNDECODABLE) + elif FS_NONASCII: + os.environ.setdefault(UNICODE_GUARD_ENV, FS_NONASCII) - if ns.threshold is not None: - gc.set_threshold(ns.threshold) - try: - import msvcrt - except ImportError: - pass +def setup_tests(runtests: RunTests) -> None: + support.verbose = runtests.verbose + support.failfast = runtests.fail_fast + support.PGO = runtests.pgo + support.PGO_EXTENDED = runtests.pgo_extended + + set_match_tests(runtests.match_tests) + + if runtests.use_junit: + support.junit_xml_list = [] + from .testresult import RegressionTestResult + RegressionTestResult.USE_XML = True else: - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - if ns.verbose and ns.verbose >= 2: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - else: - msvcrt.CrtSetReportMode(m, 0) + support.junit_xml_list = None - support.use_resources = ns.use_resources + if runtests.memory_limit is not None: + support.set_memlimit(runtests.memory_limit) + support.suppress_msvcrt_asserts(runtests.verbose >= 2) -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - stdout = sys.stdout - try: - fd = stdout.fileno() - except ValueError: - # On IDLE, sys.stdout has no file descriptor and is not a TextIOWrapper - # object. Leaving sys.stdout unchanged. - # - # Catch ValueError to catch io.UnsupportedOperation on TextIOBase - # and ValueError on a closed stream. - return - - sys.stdout = open(fd, 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) + support.use_resources = runtests.use_resources + + timeout = runtests.timeout + if timeout is not None: + # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT + support.LOOPBACK_TIMEOUT = max(support.LOOPBACK_TIMEOUT, timeout / 120) + # don't increase INTERNET_TIMEOUT + support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, timeout / 40) + support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, timeout / 4) + + # If --timeout is short: reduce timeouts + support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, timeout) + support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, timeout) + support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, timeout) + support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, timeout) + + if runtests.hunt_refleak: + # private attribute that mypy doesn't know about: + unittest.BaseTestSuite._cleanup = False # type: ignore[attr-defined] + + if runtests.gc_threshold is not None: + gc.set_threshold(runtests.gc_threshold) + + random.seed(runtests.random_seed) + + # sys.stdout is redirected to a StringIO in single process mode on which + # color auto-detect fails as StringIO is not a TTY. If the original + # sys.stdout supports color pass that through with FORCE_COLOR so that when + # results are printed, such as with -W, they get color. + if can_colorize(file=sys.stdout): + os.environ['FORCE_COLOR'] = "1" diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py new file mode 100644 index 00000000000..3dfb0b01dc1 --- /dev/null +++ b/Lib/test/libregrtest/single.py @@ -0,0 +1,361 @@ +import faulthandler +import gc +import importlib +import io +import sys +import time +import traceback +import unittest + +from _colorize import get_colors # type: ignore[import-not-found] +from test import support +from test.support import threading_helper + +from .filter import match_test +from .result import State, TestResult, TestStats +from .runtests import RunTests +from .save_env import saved_test_environment +from .setup import setup_tests +from .testresult import get_test_runner +from .parallel_case import ParallelTestCase +from .utils import ( + TestName, + clear_caches, remove_testfn, abs_module_name, print_warning) + + +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + +def run_unittest(test_mod, runtests: RunTests): + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(test_mod) + + for error in loader.errors: + print(error, file=sys.stderr) + if loader.errors: + raise Exception("errors while loading tests") + _filter_suite(tests, match_test) + if runtests.parallel_threads: + _parallelize_tests(tests, runtests.parallel_threads) + return _run_suite(tests) + +def _filter_suite(suite, pred): + """Recursively filter test cases in a suite based on a predicate.""" + newtests = [] + for test in suite._tests: + if isinstance(test, unittest.TestSuite): + _filter_suite(test, pred) + newtests.append(test) + else: + if pred(test): + newtests.append(test) + suite._tests = newtests + +def _parallelize_tests(suite, parallel_threads: int): + def is_thread_unsafe(test): + test_method = getattr(test, test._testMethodName) + instance = test_method.__self__ + return (getattr(test_method, "__unittest_thread_unsafe__", False) or + getattr(instance, "__unittest_thread_unsafe__", False)) + + newtests: list[object] = [] + for test in suite._tests: + if isinstance(test, unittest.TestSuite): + _parallelize_tests(test, parallel_threads) + newtests.append(test) + continue + + if is_thread_unsafe(test): + # Don't parallelize thread-unsafe tests + newtests.append(test) + continue + + newtests.append(ParallelTestCase(test, parallel_threads)) + suite._tests = newtests + +def _run_suite(suite): + """Run tests from a unittest.TestSuite-derived class.""" + runner = get_test_runner(sys.stdout, + verbosity=support.verbose, + capture_output=(support.junit_xml_list is not None)) + + result = runner.run(suite) + + if support.junit_xml_list is not None: + import xml.etree.ElementTree as ET + xml_elem = result.get_xml_element() + xml_str = ET.tostring(xml_elem).decode('ascii') + support.junit_xml_list.append(xml_str) + + if not result.testsRun and not result.skipped and not result.errors: + raise support.TestDidNotRun + if not result.wasSuccessful(): + stats = TestStats.from_unittest(result) + if len(result.errors) == 1 and not result.failures: + err = result.errors[0][1] + elif len(result.failures) == 1 and not result.errors: + err = result.failures[0][1] + else: + err = "multiple errors occurred" + if not support.verbose: err += "; run in verbose mode for details" + errors = [(str(tc), exc_str) for tc, exc_str in result.errors] + failures = [(str(tc), exc_str) for tc, exc_str in result.failures] + raise support.TestFailedWithDetails(err, errors, failures, stats=stats) + return result + + +def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: + # Run test_func(), collect statistics, and detect reference and memory + # leaks. + if runtests.hunt_refleak: + from .refleak import runtest_refleak + refleak, test_result = runtest_refleak(result.test_name, test_func, + runtests.hunt_refleak, + runtests.quiet) + else: + test_result = test_func() + refleak = False + + if refleak: + result.state = State.REFLEAK + + stats: TestStats | None + + match test_result: + case TestStats(): + stats = test_result + case unittest.TestResult(): + stats = TestStats.from_unittest(test_result) + case None: + print_warning(f"{result.test_name} test runner returned None: {test_func}") + stats = None + case _: + # Don't import doctest at top level since only few tests return + # a doctest.TestResult instance. + import doctest + if isinstance(test_result, doctest.TestResults): + stats = TestStats.from_doctest(test_result) + else: + print_warning(f"Unknown test result type: {type(test_result)}") + stats = None + + result.stats = stats + + +# Storage of uncollectable GC objects (gc.garbage) +GC_GARBAGE = [] + + +def _load_run_test(result: TestResult, runtests: RunTests) -> None: + # Load the test module and run the tests. + test_name = result.test_name + module_name = abs_module_name(test_name, runtests.test_dir) + test_mod = importlib.import_module(module_name) + + if hasattr(test_mod, "test_main"): + # https://github.com/python/cpython/issues/89392 + raise Exception(f"Module {test_name} defines test_main() which " + f"is no longer supported by regrtest") + def test_func(): + return run_unittest(test_mod, runtests) + + try: + regrtest_runner(result, test_func, runtests) + finally: + # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce + # failures. + support.gc_collect() + + remove_testfn(test_name, runtests.verbose) + + # XXX: RUSTPYTHON, build a functional garbage collector into the interpreter + # if gc.garbage: + # support.environment_altered = True + # print_warning(f"{test_name} created {len(gc.garbage)} " + # f"uncollectable object(s)") + + # # move the uncollectable objects somewhere, + # # so we don't see them again + # GC_GARBAGE.extend(gc.garbage) + # gc.garbage.clear() + + support.reap_children() + + +def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, + display_failure: bool = True) -> None: + # Handle exceptions, detect environment changes. + stdout = get_colors(file=sys.stdout) + stderr = get_colors(file=sys.stderr) + + # Reset the environment_altered flag to detect if a test altered + # the environment + support.environment_altered = False + + pgo = runtests.pgo + if pgo: + display_failure = False + quiet = runtests.quiet + + test_name = result.test_name + try: + clear_caches() + support.gc_collect() + + with saved_test_environment(test_name, + runtests.verbose, quiet, pgo=pgo): + _load_run_test(result, runtests) + except support.ResourceDenied as exc: + if not quiet and not pgo: + print( + f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}", + flush=True, + ) + result.state = State.RESOURCE_DENIED + return + except unittest.SkipTest as exc: + if not quiet and not pgo: + print( + f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}", + flush=True, + ) + result.state = State.SKIPPED + return + except support.TestFailedWithDetails as exc: + msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}" + if display_failure: + msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}" + print(msg, file=sys.stderr, flush=True) + result.state = State.FAILED + result.errors = exc.errors + result.failures = exc.failures + result.stats = exc.stats + return + except support.TestFailed as exc: + msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}" + if display_failure: + msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}" + print(msg, file=sys.stderr, flush=True) + result.state = State.FAILED + result.stats = exc.stats + return + except support.TestDidNotRun: + result.state = State.DID_NOT_RUN + return + except KeyboardInterrupt: + print() + result.state = State.INTERRUPTED + return + except: + if not pgo: + msg = traceback.format_exc() + print( + f"{stderr.RED}test {test_name} crashed -- {msg}{stderr.RESET}", + file=sys.stderr, + flush=True, + ) + result.state = State.UNCAUGHT_EXC + return + + if support.environment_altered: + result.set_env_changed() + # Don't override the state if it was already set (REFLEAK or ENV_CHANGED) + if result.state is None: + result.state = State.PASSED + + +def _runtest(result: TestResult, runtests: RunTests) -> None: + # Capture stdout and stderr, set faulthandler timeout, + # and create JUnit XML report. + verbose = runtests.verbose + output_on_failure = runtests.output_on_failure + timeout = runtests.timeout + + if timeout is not None and threading_helper.can_start_thread: + use_timeout = True + faulthandler.dump_traceback_later(timeout, exit=True) + else: + use_timeout = False + + try: + setup_tests(runtests) + + if output_on_failure or runtests.pgo: + support.verbose = True + + stream = io.StringIO() + orig_stdout = sys.stdout + orig_stderr = sys.stderr + print_warning = support.print_warning + orig_print_warnings_stderr = print_warning.orig_stderr + + output = None + try: + sys.stdout = stream + sys.stderr = stream + # print_warning() writes into the temporary stream to preserve + # messages order. If support.environment_altered becomes true, + # warnings will be written to sys.stderr below. + print_warning.orig_stderr = stream + + _runtest_env_changed_exc(result, runtests, display_failure=False) + # Ignore output if the test passed successfully + if result.state != State.PASSED: + output = stream.getvalue() + finally: + sys.stdout = orig_stdout + sys.stderr = orig_stderr + print_warning.orig_stderr = orig_print_warnings_stderr + + if output is not None: + sys.stderr.write(output) + sys.stderr.flush() + else: + # Tell tests to be moderately quiet + support.verbose = verbose + _runtest_env_changed_exc(result, runtests, + display_failure=not verbose) + + xml_list = support.junit_xml_list + if xml_list: + result.xml_data = xml_list + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + support.junit_xml_list = None + + +def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: + """Run a single test. + + test_name -- the name of the test + + Returns a TestResult. + + If runtests.use_junit, xml_data is a list containing each generated + testsuite element. + """ + ansi = get_colors(file=sys.stderr) + red, reset, yellow = ansi.BOLD_RED, ansi.RESET, ansi.YELLOW + + start_time = time.perf_counter() + result = TestResult(test_name) + pgo = runtests.pgo + try: + _runtest(result, runtests) + except: + if not pgo: + msg = traceback.format_exc() + print(f"{red}test {test_name} crashed -- {msg}{reset}", + file=sys.stderr, flush=True) + result.state = State.UNCAUGHT_EXC + + sys.stdout.flush() + sys.stderr.flush() + + result.duration = time.perf_counter() - start_time + return result diff --git a/Lib/test/support/testresult.py b/Lib/test/libregrtest/testresult.py similarity index 94% rename from Lib/test/support/testresult.py rename to Lib/test/libregrtest/testresult.py index de23fdd59de..1820f354572 100644 --- a/Lib/test/support/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -9,6 +9,7 @@ import traceback import unittest from test import support +from test.libregrtest.utils import sanitize_xml class RegressionTestResult(unittest.TextTestResult): USE_XML = False @@ -65,23 +66,24 @@ def _add_result(self, test, capture=False, **args): if capture: if self._stdout_buffer is not None: stdout = self._stdout_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-out').text = stdout + ET.SubElement(e, 'system-out').text = sanitize_xml(stdout) if self._stderr_buffer is not None: stderr = self._stderr_buffer.getvalue().rstrip() - ET.SubElement(e, 'system-err').text = stderr + ET.SubElement(e, 'system-err').text = sanitize_xml(stderr) for k, v in args.items(): if not k or not v: continue + e2 = ET.SubElement(e, k) if hasattr(v, 'items'): for k2, v2 in v.items(): if k2: - e2.set(k2, str(v2)) + e2.set(k2, sanitize_xml(str(v2))) else: - e2.text = str(v2) + e2.text = sanitize_xml(str(v2)) else: - e2.text = str(v) + e2.text = sanitize_xml(str(v)) @classmethod def __makeErrorDict(cls, err_type, err_value, err_tb): diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py new file mode 100644 index 00000000000..d984a735bdf --- /dev/null +++ b/Lib/test/libregrtest/tsan.py @@ -0,0 +1,51 @@ +# Set of tests run by default if --tsan is specified. The tests below were +# chosen because they use threads and run in a reasonable amount of time. + +TSAN_TESTS = [ + 'test_asyncio', + # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). + 'test_capi.test_mem', + 'test_capi.test_pyatomic', + 'test_code', + 'test_ctypes', + # 'test_concurrent_futures', # gh-130605: too many data races + 'test_enum', + 'test_functools', + 'test_httpservers', + 'test_imaplib', + 'test_importlib', + 'test_io', + 'test_logging', + 'test_opcache', + 'test_queue', + 'test_signal', + 'test_socket', + 'test_sqlite3', + 'test_ssl', + 'test_syslog', + 'test_thread', + 'test_thread_local_bytecode', + 'test_threadedtempfile', + 'test_threading', + 'test_threading_local', + 'test_threadsignals', + 'test_weakref', + 'test_free_threading', +] + +# Tests that should be run with `--parallel-threads=N` under TSAN. These tests +# typically do not use threads, but are run multiple times in parallel by +# the regression test runner with the `--parallel-threads` option enabled. +TSAN_PARALLEL_TESTS = [ + 'test_abc', + 'test_hashlib', +] + + +def setup_tsan_tests(cmdline_args) -> None: + if not cmdline_args: + cmdline_args[:] = TSAN_TESTS[:] + +def setup_tsan_parallel_tests(cmdline_args) -> None: + if not cmdline_args: + cmdline_args[:] = TSAN_PARALLEL_TESTS[:] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index fb9971a64f6..d94fb84a743 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -1,10 +1,63 @@ +import contextlib +import faulthandler +import locale import math import os.path +import platform +import random +import re +import shlex +import subprocess import sys +import sysconfig +import tempfile import textwrap +from collections.abc import Callable, Iterable +from test import support +from test.support import os_helper +from test.support import threading_helper -def format_duration(seconds): + +# All temporary files and temporary directories created by libregrtest should +# use TMP_PREFIX so cleanup_temp_dir() can remove them all. +TMP_PREFIX = 'test_python_' +WORK_DIR_PREFIX = TMP_PREFIX +WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_' + +# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). +# Used to protect against threading._shutdown() hang. +# Must be smaller than buildbot "1200 seconds without output" limit. +EXIT_TIMEOUT = 120.0 + + +ALL_RESOURCES = ('audio', 'console', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') + +# Other resources excluded from --use=all: +# +# - extralagefile (ex: test_zipfile64): really too slow to be enabled +# "by default" +# - tzdata: while needed to validate fully test_datetime, it makes +# test_datetime too slow (15-20 min on some buildbots) and so is disabled by +# default (see bpo-30822). +RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') + + +# Types for types hints +StrPath = str +TestName = str +StrJSON = str +TestTuple = tuple[TestName, ...] +TestList = list[TestName] +# --match and --ignore options: list of patterns +# ('*' joker character can be used) +TestFilter = list[tuple[TestName, bool]] +FilterTuple = tuple[TestName, ...] +FilterDict = dict[TestName, FilterTuple] + + +def format_duration(seconds: float) -> str: ms = math.ceil(seconds * 1e3) seconds, ms = divmod(ms, 1000) minutes, seconds = divmod(seconds, 60) @@ -16,17 +69,20 @@ def format_duration(seconds): if minutes: parts.append('%s min' % minutes) if seconds: - parts.append('%s sec' % seconds) - if ms: - parts.append('%s ms' % ms) + if parts: + # 2 min 1 sec + parts.append('%s sec' % seconds) + else: + # 1.0 sec + parts.append('%.1f sec' % (seconds + ms / 1000)) if not parts: - return '0 ms' + return '%s ms' % ms parts = parts[:2] return ' '.join(parts) -def removepy(names): +def strip_py_suffix(names: list[str] | None) -> None: if not names: return for idx, name in enumerate(names): @@ -35,11 +91,20 @@ def removepy(names): names[idx] = basename -def count(n, word): +def plural(n: int, singular: str, plural: str | None = None) -> str: if n == 1: - return "%d %s" % (n, word) + return singular + elif plural is not None: + return plural else: - return "%d %ss" % (n, word) + return singular + 's' + + +def count(n: int, word: str) -> str: + if n == 1: + return f"{n} {word}" + else: + return f"{n} {word}s" def printlist(x, width=70, indent=4, file=None): @@ -57,5 +122,605 @@ def printlist(x, width=70, indent=4, file=None): file=file) -def print_warning(msg): - print(f"Warning -- {msg}", file=sys.stderr, flush=True) +def print_warning(msg: str) -> None: + support.print_warning(msg) + + +orig_unraisablehook: Callable[..., None] | None = None + + +def regrtest_unraisable_hook(unraisable) -> None: + global orig_unraisablehook + support.environment_altered = True + support.print_warning("Unraisable exception") + old_stderr = sys.stderr + try: + support.flush_std_streams() + sys.stderr = support.print_warning.orig_stderr + assert orig_unraisablehook is not None, "orig_unraisablehook not set" + orig_unraisablehook(unraisable) + sys.stderr.flush() + finally: + sys.stderr = old_stderr + + +def setup_unraisable_hook() -> None: + global orig_unraisablehook + orig_unraisablehook = sys.unraisablehook + sys.unraisablehook = regrtest_unraisable_hook + + +orig_threading_excepthook: Callable[..., None] | None = None + + +def regrtest_threading_excepthook(args) -> None: + global orig_threading_excepthook + support.environment_altered = True + support.print_warning(f"Uncaught thread exception: {args.exc_type.__name__}") + old_stderr = sys.stderr + try: + support.flush_std_streams() + sys.stderr = support.print_warning.orig_stderr + assert orig_threading_excepthook is not None, "orig_threading_excepthook not set" + orig_threading_excepthook(args) + sys.stderr.flush() + finally: + sys.stderr = old_stderr + + +def setup_threading_excepthook() -> None: + global orig_threading_excepthook + import threading + orig_threading_excepthook = threading.excepthook + threading.excepthook = regrtest_threading_excepthook + + +def clear_caches(): + # Clear the warnings registry, so they can be displayed again + for mod in sys.modules.values(): + if hasattr(mod, '__warningregistry__'): + del mod.__warningregistry__ + + # Flush standard output, so that buffered data is sent to the OS and + # associated Python objects are reclaimed. + for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): + if stream is not None: + stream.flush() + + try: + re = sys.modules['re'] + except KeyError: + pass + else: + re.purge() + + try: + _strptime = sys.modules['_strptime'] + except KeyError: + pass + else: + _strptime._regex_cache.clear() + + try: + urllib_parse = sys.modules['urllib.parse'] + except KeyError: + pass + else: + urllib_parse.clear_cache() + + try: + urllib_request = sys.modules['urllib.request'] + except KeyError: + pass + else: + urllib_request.urlcleanup() + + try: + linecache = sys.modules['linecache'] + except KeyError: + pass + else: + linecache.clearcache() + + try: + mimetypes = sys.modules['mimetypes'] + except KeyError: + pass + else: + mimetypes._default_mime_types() + + try: + filecmp = sys.modules['filecmp'] + except KeyError: + pass + else: + filecmp._cache.clear() + + try: + struct = sys.modules['struct'] + except KeyError: + pass + else: + struct._clearcache() + + try: + doctest = sys.modules['doctest'] + except KeyError: + pass + else: + doctest.master = None + + try: + ctypes = sys.modules['ctypes'] + except KeyError: + pass + else: + ctypes._reset_cache() + + try: + typing = sys.modules['typing'] + except KeyError: + pass + else: + for f in typing._cleanups: + f() + + import inspect + abs_classes = filter(inspect.isabstract, typing.__dict__.values()) + for abc in abs_classes: + for obj in abc.__subclasses__() + [abc]: + obj._abc_caches_clear() + + try: + fractions = sys.modules['fractions'] + except KeyError: + pass + else: + fractions._hash_algorithm.cache_clear() + + try: + inspect = sys.modules['inspect'] + except KeyError: + pass + else: + inspect._shadowed_dict_from_weakref_mro_tuple.cache_clear() + inspect._filesbymodname.clear() + inspect.modulesbyfile.clear() + + try: + importlib_metadata = sys.modules['importlib.metadata'] + except KeyError: + pass + else: + importlib_metadata.FastPath.__new__.cache_clear() + + +def get_build_info(): + # Get most important configure and build options as a list of strings. + # Example: ['debug', 'ASAN+MSAN'] or ['release', 'LTO+PGO']. + + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + cflags = sysconfig.get_config_var('PY_CFLAGS') or '' + cflags += ' ' + (sysconfig.get_config_var('PY_CFLAGS_NODIST') or '') + ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' + + build = [] + + # --disable-gil + if sysconfig.get_config_var('Py_GIL_DISABLED'): + if not sys.flags.ignore_environment: + PYTHON_GIL = os.environ.get('PYTHON_GIL', None) + if PYTHON_GIL: + PYTHON_GIL = (PYTHON_GIL == '1') + else: + PYTHON_GIL = None + + free_threading = "free_threading" + if PYTHON_GIL is not None: + free_threading = f"{free_threading} GIL={int(PYTHON_GIL)}" + build.append(free_threading) + + if hasattr(sys, 'gettotalrefcount'): + # --with-pydebug + build.append('debug') + + if '-DNDEBUG' in cflags: + build.append('without_assert') + else: + build.append('release') + + if '--with-assertions' in config_args: + build.append('with_assert') + elif '-DNDEBUG' not in cflags: + build.append('with_assert') + + # --enable-experimental-jit + if sys._jit.is_available(): + if sys._jit.is_enabled(): + build.append("JIT") + else: + build.append("JIT (disabled)") + + # --enable-framework=name + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + if framework: + build.append(f'framework={framework}') + + # --enable-shared + shared = int(sysconfig.get_config_var('PY_ENABLE_SHARED') or '0') + if shared: + build.append('shared') + + # --with-lto + optimizations = [] + if '-flto=thin' in ldflags_nodist: + optimizations.append('ThinLTO') + elif '-flto' in ldflags_nodist: + optimizations.append('LTO') + + if support.check_cflags_pgo(): + # PGO (--enable-optimizations) + optimizations.append('PGO') + + if support.check_bolt_optimized(): + # BOLT (--enable-bolt) + optimizations.append('BOLT') + + if optimizations: + build.append('+'.join(optimizations)) + + # --with-address-sanitizer + sanitizers = [] + if support.check_sanitizer(address=True): + sanitizers.append("ASAN") + # --with-memory-sanitizer + if support.check_sanitizer(memory=True): + sanitizers.append("MSAN") + # --with-undefined-behavior-sanitizer + if support.check_sanitizer(ub=True): + sanitizers.append("UBSAN") + # --with-thread-sanitizer + if support.check_sanitizer(thread=True): + sanitizers.append("TSAN") + if sanitizers: + build.append('+'.join(sanitizers)) + + # --with-trace-refs + if hasattr(sys, 'getobjects'): + build.append("TraceRefs") + # --enable-pystats + if hasattr(sys, '_stats_on'): + build.append("pystats") + # --with-valgrind + if sysconfig.get_config_var('WITH_VALGRIND'): + build.append("valgrind") + # --with-dtrace + if sysconfig.get_config_var('WITH_DTRACE'): + build.append("dtrace") + + return build + + +def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: + if tmp_dir: + tmp_dir = os.path.expanduser(tmp_dir) + else: + # When tests are run from the Python build directory, it is best practice + # to keep the test files in a subfolder. This eases the cleanup of leftover + # files using the "make distclean" command. + if sysconfig.is_python_build(): + if not support.is_wasi: + tmp_dir = sysconfig.get_config_var('abs_builddir') + if tmp_dir is None: + tmp_dir = sysconfig.get_config_var('abs_srcdir') + if not tmp_dir: + # gh-74470: On Windows, only srcdir is available. Using + # abs_builddir mostly matters on UNIX when building + # Python out of the source tree, especially when the + # source tree is read only. + tmp_dir = sysconfig.get_config_var('srcdir') + if not tmp_dir: + raise RuntimeError( + "Could not determine the correct value for tmp_dir" + ) + tmp_dir = os.path.join(tmp_dir, 'build') + else: + # WASI platform + tmp_dir = sysconfig.get_config_var('projectbase') + if not tmp_dir: + raise RuntimeError( + "sysconfig.get_config_var('projectbase') " + f"unexpectedly returned {tmp_dir!r} on WASI" + ) + tmp_dir = os.path.join(tmp_dir, 'build') + + # When get_temp_dir() is called in a worker process, + # get_temp_dir() path is different than in the parent process + # which is not a WASI process. So the parent does not create + # the same "tmp_dir" than the test worker process. + os.makedirs(tmp_dir, exist_ok=True) + else: + tmp_dir = tempfile.gettempdir() + + return os.path.abspath(tmp_dir) + + +def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + # Emscripten and WASI have stubbed getpid(), Emscripten has only + # millisecond clock resolution. Use randint() instead. + if support.is_emscripten or support.is_wasi: + nounce = random.randint(0, 1_000_000) + else: + nounce = os.getpid() + + if worker: + work_dir = WORK_DIR_PREFIX + str(nounce) + else: + work_dir = WORKER_WORK_DIR_PREFIX + str(nounce) + work_dir += os_helper.FS_NONASCII + work_dir = os.path.join(parent_dir, work_dir) + return work_dir + + +@contextlib.contextmanager +def exit_timeout(): + try: + yield + except SystemExit as exc: + # bpo-38203: Python can hang at exit in Py_Finalize(), especially + # on threading._shutdown() call: put a timeout + if threading_helper.can_start_thread: + faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True) + sys.exit(exc.code) + + +def remove_testfn(test_name: TestName, verbose: int) -> None: + # Try to clean up os_helper.TESTFN if left behind. + # + # While tests shouldn't leave any files or directories behind, when a test + # fails that can be tedious for it to arrange. The consequences can be + # especially nasty on Windows, since if a test leaves a file open, it + # cannot be deleted by name (while there's nothing we can do about that + # here either, we can display the name of the offending test, which is a + # real help). + name = os_helper.TESTFN + if not os.path.exists(name): + return + + nuker: Callable[[str], None] + if os.path.isdir(name): + import shutil + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise RuntimeError(f"os.path says {name!r} exists but is neither " + f"directory nor file") + + if verbose: + print_warning(f"{test_name} left behind {kind} {name!r}") + support.environment_altered = True + + try: + import stat + # fix possible permissions problems that might prevent cleanup + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as exc: + print_warning(f"{test_name} left behind {kind} {name!r} " + f"and it couldn't be removed: {exc}") + + +def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: + if test_name.startswith('test.') or test_dir: + return test_name + else: + # Import it from the test package + return 'test.' + test_name + + +# gh-90681: When rerunning tests, we might need to rerun the whole +# class or module suite if some its life-cycle hooks fail. +# Test level hooks are not affected. +_TEST_LIFECYCLE_HOOKS = frozenset(( + 'setUpClass', 'tearDownClass', + 'setUpModule', 'tearDownModule', +)) + +def normalize_test_name(test_full_name: str, *, + is_error: bool = False) -> str | None: + short_name = test_full_name.split(" ")[0] + if is_error and short_name in _TEST_LIFECYCLE_HOOKS: + if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): + # if setUpModule() or tearDownModule() failed, don't filter + # tests with the test file name, don't use filters. + return None + + # This means that we have a failure in a life-cycle hook, + # we need to rerun the whole module or class suite. + # Basically the error looks like this: + # ERROR: setUpClass (test.test_reg_ex.RegTest) + # or + # ERROR: setUpModule (test.test_reg_ex) + # So, we need to parse the class / module name. + lpar = test_full_name.index('(') + rpar = test_full_name.index(')') + return test_full_name[lpar + 1: rpar].split('.')[-1] + return short_name + + +def adjust_rlimit_nofile() -> None: + """ + On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256) + for our test suite to succeed. Raise it to something more reasonable. 1024 + is a common Linux default. + """ + try: + import resource + except ImportError: + return + + fd_limit, max_fds = resource.getrlimit(resource.RLIMIT_NOFILE) + + desired_fds = 1024 + + if fd_limit < desired_fds and fd_limit < max_fds: + new_fd_limit = min(desired_fds, max_fds) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, + (new_fd_limit, max_fds)) + print(f"Raised RLIMIT_NOFILE: {fd_limit} -> {new_fd_limit}") + except (ValueError, OSError) as err: + print_warning(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " + f"{new_fd_limit}: {err}.") + + +def get_host_runner() -> str: + if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: + hostrunner = sysconfig.get_config_var("HOSTRUNNER") + return hostrunner + + +def is_cross_compiled() -> bool: + return ('_PYTHON_HOST_PLATFORM' in os.environ) + + +def format_resources(use_resources: Iterable[str]) -> str: + use_resources = set(use_resources) + all_resources = set(ALL_RESOURCES) + + # Express resources relative to "all" + relative_all = ['all'] + for name in sorted(all_resources - use_resources): + relative_all.append(f'-{name}') + for name in sorted(use_resources - all_resources): + relative_all.append(f'{name}') + all_text = ','.join(relative_all) + all_text = f"resources: {all_text}" + + # List of enabled resources + text = ','.join(sorted(use_resources)) + text = f"resources ({len(use_resources)}): {text}" + + # Pick the shortest string (prefer relative to all if lengths are equal) + if len(all_text) <= len(text): + return all_text + else: + return text + + +def display_header(use_resources: tuple[str, ...], + python_cmd: tuple[str, ...] | None) -> None: + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("==", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== Python build:", ' '.join(get_build_info())) + print("== cwd:", os.getcwd()) + + cpu_count: object = os.cpu_count() + if cpu_count: + # The function is new in Python 3.13; mypy doesn't know about it yet: + process_cpu_count = os.process_cpu_count() # type: ignore[attr-defined] + if process_cpu_count and process_cpu_count != cpu_count: + cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)" + print("== CPU count:", cpu_count) + print("== encodings: locale=%s FS=%s" + % (locale.getencoding(), sys.getfilesystemencoding())) + + if use_resources: + text = format_resources(use_resources) + print(f"== {text}") + else: + print("== resources: all test resources are disabled, " + "use -u option to unskip tests") + + cross_compile = is_cross_compiled() + if cross_compile: + print("== cross compiled: Yes") + if python_cmd: + cmd = shlex.join(python_cmd) + print(f"== host python: {cmd}") + + get_cmd = [*python_cmd, '-m', 'platform'] + proc = subprocess.run( + get_cmd, + stdout=subprocess.PIPE, + text=True, + cwd=os_helper.SAVEDCWD) + stdout = proc.stdout.replace('\n', ' ').strip() + if stdout: + print(f"== host platform: {stdout}") + elif proc.returncode: + print(f"== host platform: ") + else: + hostrunner = get_host_runner() + if hostrunner: + print(f"== host runner: {hostrunner}") + + # This makes it easier to remember what to set in your local + # environment when trying to reproduce a sanitizer failure. + asan = support.check_sanitizer(address=True) + msan = support.check_sanitizer(memory=True) + ubsan = support.check_sanitizer(ub=True) + tsan = support.check_sanitizer(thread=True) + sanitizers = [] + if asan: + sanitizers.append("address") + if msan: + sanitizers.append("memory") + if ubsan: + sanitizers.append("undefined behavior") + if tsan: + sanitizers.append("thread") + if sanitizers: + print(f"== sanitizers: {', '.join(sanitizers)}") + for sanitizer, env_var in ( + (asan, "ASAN_OPTIONS"), + (msan, "MSAN_OPTIONS"), + (ubsan, "UBSAN_OPTIONS"), + (tsan, "TSAN_OPTIONS"), + ): + options= os.environ.get(env_var) + if sanitizer and options is not None: + print(f"== {env_var}={options!r}") + + print(flush=True) + + +def cleanup_temp_dir(tmp_dir: StrPath) -> None: + import glob + + path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*') + print("Cleanup %s directory" % tmp_dir) + for name in glob.glob(path): + if os.path.isdir(name): + print("Remove directory: %s" % name) + os_helper.rmtree(name) + else: + print("Remove file: %s" % name) + os_helper.unlink(name) + + +ILLEGAL_XML_CHARS_RE = re.compile( + '[' + # Control characters; newline (\x0A and \x0D) and TAB (\x09) are legal + '\x00-\x08\x0B\x0C\x0E-\x1F' + # Surrogate characters + '\uD800-\uDFFF' + # Special Unicode characters + '\uFFFE' + '\uFFFF' + # Match multiple sequential invalid characters for better efficiency + ']+') + +def _sanitize_xml_replace(regs): + text = regs[0] + return ''.join(f'\\x{ord(ch):02x}' if ch <= '\xff' else ascii(ch)[1:-1] + for ch in text) + +def sanitize_xml(text: str) -> str: + return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) diff --git a/Lib/test/libregrtest/win_utils.py b/Lib/test/libregrtest/win_utils.py index 95db3def36f..b51fde0af57 100644 --- a/Lib/test/libregrtest/win_utils.py +++ b/Lib/test/libregrtest/win_utils.py @@ -1,105 +1,128 @@ +import _overlapped +import _thread import _winapi -import msvcrt -import os -import subprocess -import uuid -from test import support +import math +import struct +import winreg -# Max size of asynchronous reads -BUFSIZE = 8192 -# Exponential damping factor (see below) -LOAD_FACTOR_1 = 0.9200444146293232478931553241 # Seconds per measurement -SAMPLING_INTERVAL = 5 -COUNTER_NAME = r'\System\Processor Queue Length' +SAMPLING_INTERVAL = 1 +# Exponential damping factor to compute exponentially weighted moving average +# on 1 minute (60 seconds) +LOAD_FACTOR_1 = 1 / math.exp(SAMPLING_INTERVAL / 60) +# Initialize the load using the arithmetic mean of the first NVALUE values +# of the Processor Queue Length +NVALUE = 5 class WindowsLoadTracker(): """ - This class asynchronously interacts with the `typeperf` command to read - the system load on Windows. Mulitprocessing and threads can't be used - here because they interfere with the test suite's cases for those - modules. + This class asynchronously reads the performance counters to calculate + the system load on Windows. A "raw" thread is used here to prevent + interference with the test suite's cases for the threading module. """ def __init__(self): - self.load = 0.0 - self.start() - - def start(self): - # Create a named pipe which allows for asynchronous IO in Windows - pipe_name = r'\\.\pipe\typeperf_output_' + str(uuid.uuid4()) - - open_mode = _winapi.PIPE_ACCESS_INBOUND - open_mode |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE - open_mode |= _winapi.FILE_FLAG_OVERLAPPED - - # This is the read end of the pipe, where we will be grabbing output - self.pipe = _winapi.CreateNamedPipe( - pipe_name, open_mode, _winapi.PIPE_WAIT, - 1, BUFSIZE, BUFSIZE, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL - ) - # The write end of the pipe which is passed to the created process - pipe_write_end = _winapi.CreateFile( - pipe_name, _winapi.GENERIC_WRITE, 0, _winapi.NULL, - _winapi.OPEN_EXISTING, 0, _winapi.NULL - ) - # Open up the handle as a python file object so we can pass it to - # subprocess - command_stdout = msvcrt.open_osfhandle(pipe_write_end, 0) - - # Connect to the read end of the pipe in overlap/async mode - overlap = _winapi.ConnectNamedPipe(self.pipe, overlapped=True) - overlap.GetOverlappedResult(True) - - # Spawn off the load monitor - command = ['typeperf', COUNTER_NAME, '-si', str(SAMPLING_INTERVAL)] - self.p = subprocess.Popen(command, stdout=command_stdout, cwd=os_helper.SAVEDCWD) - - # Close our copy of the write end of the pipe - os.close(command_stdout) - - def close(self): - if self.p is None: + # make __del__ not fail if pre-flight test fails + self._running = None + self._stopped = None + + # Pre-flight test for access to the performance data; + # `PermissionError` will be raised if not allowed + winreg.QueryInfoKey(winreg.HKEY_PERFORMANCE_DATA) + + self._values = [] + self._load = None + self._running = _overlapped.CreateEvent(None, True, False, None) + self._stopped = _overlapped.CreateEvent(None, True, False, None) + + _thread.start_new_thread(self._update_load, (), {}) + + def _update_load(self, + # localize module access to prevent shutdown errors + _wait=_winapi.WaitForSingleObject, + _signal=_overlapped.SetEvent): + # run until signaled to stop + while _wait(self._running, 1000): + self._calculate_load() + # notify stopped + _signal(self._stopped) + + def _calculate_load(self, + # localize module access to prevent shutdown errors + _query=winreg.QueryValueEx, + _hkey=winreg.HKEY_PERFORMANCE_DATA, + _unpack=struct.unpack_from): + # get the 'System' object + data, _ = _query(_hkey, '2') + # PERF_DATA_BLOCK { + # WCHAR Signature[4] 8 + + # DWOWD LittleEndian 4 + + # DWORD Version 4 + + # DWORD Revision 4 + + # DWORD TotalByteLength 4 + + # DWORD HeaderLength = 24 byte offset + # ... + # } + obj_start, = _unpack('L', data, 24) + # PERF_OBJECT_TYPE { + # DWORD TotalByteLength + # DWORD DefinitionLength + # DWORD HeaderLength + # ... + # } + data_start, defn_start = _unpack('4xLL', data, obj_start) + data_base = obj_start + data_start + defn_base = obj_start + defn_start + # find the 'Processor Queue Length' counter (index=44) + while defn_base < data_base: + # PERF_COUNTER_DEFINITION { + # DWORD ByteLength + # DWORD CounterNameTitleIndex + # ... [7 DWORDs/28 bytes] + # DWORD CounterOffset + # } + size, idx, offset = _unpack('LL28xL', data, defn_base) + defn_base += size + if idx == 44: + counter_offset = data_base + offset + # the counter is known to be PERF_COUNTER_RAWCOUNT (DWORD) + processor_queue_length, = _unpack('L', data, counter_offset) + break + else: return - self.p.kill() - self.p.wait() - self.p = None - def __del__(self): - self.close() - - def read_output(self): - import _winapi - - overlapped, _ = _winapi.ReadFile(self.pipe, BUFSIZE, True) - bytes_read, res = overlapped.GetOverlappedResult(False) - if res != 0: - return - - return overlapped.getbuffer().decode() + # We use an exponentially weighted moving average, imitating the + # load calculation on Unix systems. + # https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation + # https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + if self._load is not None: + self._load = (self._load * LOAD_FACTOR_1 + + processor_queue_length * (1.0 - LOAD_FACTOR_1)) + elif len(self._values) < NVALUE: + self._values.append(processor_queue_length) + else: + self._load = sum(self._values) / len(self._values) + + def close(self, kill=True): + self.__del__() + return + + def __del__(self, + # localize module access to prevent shutdown errors + _wait=_winapi.WaitForSingleObject, + _close=_winapi.CloseHandle, + _signal=_overlapped.SetEvent): + if self._running is not None: + # tell the update thread to quit + _signal(self._running) + # wait for the update thread to signal done + _wait(self._stopped, -1) + # cleanup events + _close(self._running) + _close(self._stopped) + self._running = self._stopped = None def getloadavg(self): - typeperf_output = self.read_output() - # Nothing to update, just return the current load - if not typeperf_output: - return self.load - - # Process the backlog of load values - for line in typeperf_output.splitlines(): - # typeperf outputs in a CSV format like this: - # "07/19/2018 01:32:26.605","3.000000" - toks = line.split(',') - # Ignore blank lines and the initial header - if line.strip() == '' or (COUNTER_NAME in line) or len(toks) != 2: - continue - - load = float(toks[1].replace('"', '')) - # We use an exponentially weighted moving average, imitating the - # load calculation on Unix systems. - # https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation - new_load = self.load * LOAD_FACTOR_1 + load * (1.0 - LOAD_FACTOR_1) - self.load = new_load - - return self.load + return self._load diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py new file mode 100644 index 00000000000..1ad67e1cebf --- /dev/null +++ b/Lib/test/libregrtest/worker.py @@ -0,0 +1,138 @@ +import subprocess +import sys +import os +from _colorize import can_colorize # type: ignore[import-not-found] +from typing import Any, NoReturn + +from test.support import os_helper, Py_DEBUG + +from .setup import setup_process, setup_test_dir +from .runtests import WorkerRunTests, JsonFile, JsonFileType +from .single import run_single_test +from .utils import ( + StrPath, StrJSON, TestFilter, + get_temp_dir, get_work_dir, exit_timeout) + + +USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +NEED_TTY = { + 'test_ioctl', +} + + +def create_worker_process(runtests: WorkerRunTests, output_fd: int, + tmp_dir: StrPath | None = None) -> subprocess.Popen[str]: + worker_json = runtests.as_json() + + cmd = runtests.create_python_cmd() + cmd.extend(['-m', 'test.libregrtest.worker', worker_json]) + + env = dict(os.environ) + if tmp_dir is not None: + env['TMPDIR'] = tmp_dir + env['TEMP'] = tmp_dir + env['TMP'] = tmp_dir + + # The subcommand is run with a temporary output which means it is not a TTY + # and won't auto-color. The test results are printed to stdout so if we can + # color that have the subprocess use color. + if can_colorize(file=sys.stdout): + env['FORCE_COLOR'] = '1' + + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + # + # Emscripten and WASI Python must start in the Python source code directory + # to get 'python.js' or 'python.wasm' file. Then worker_process() changes + # to a temporary directory created to run tests. + work_dir = os_helper.SAVEDCWD + + kwargs: dict[str, Any] = dict( + env=env, + stdout=output_fd, + # bpo-45410: Write stderr into stdout to keep messages order + stderr=output_fd, + text=True, + close_fds=True, + cwd=work_dir, + ) + + # Don't use setsid() in tests using TTY + test_name = runtests.tests[0] + if USE_PROCESS_GROUP and test_name not in NEED_TTY: + kwargs['start_new_session'] = True + + # Include the test name in the TSAN log file name + if 'TSAN_OPTIONS' in env: + parts = env['TSAN_OPTIONS'].split(' ') + for i, part in enumerate(parts): + if part.startswith('log_path='): + parts[i] = f'{part}.{test_name}' + break + env['TSAN_OPTIONS'] = ' '.join(parts) + + # Pass json_file to the worker process + json_file = runtests.json_file + json_file.configure_subprocess(kwargs) + + with json_file.inherit_subprocess(): + return subprocess.Popen(cmd, **kwargs) + + +def worker_process(worker_json: StrJSON) -> NoReturn: + runtests = WorkerRunTests.from_json(worker_json) + test_name = runtests.tests[0] + match_tests: TestFilter = runtests.match_tests + json_file: JsonFile = runtests.json_file + + setup_test_dir(runtests.test_dir) + setup_process() + + if runtests.rerun: + if match_tests: + matching = "matching: " + ", ".join(pattern for pattern, result in match_tests if result) + print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) + else: + print(f"Re-running {test_name} in verbose mode", flush=True) + + result = run_single_test(test_name, runtests) + if runtests.coverage: + if "test.cov" in sys.modules: # imported by -Xpresite= + result.covered_lines = list(sys.modules["test.cov"].coverage) + elif not Py_DEBUG: + print( + "Gathering coverage in worker processes requires --with-pydebug", + flush=True, + ) + else: + raise LookupError( + "`test.cov` not found in sys.modules but coverage wanted" + ) + + if json_file.file_type == JsonFileType.STDOUT: + print() + result.write_json_into(sys.stdout) + else: + with json_file.open('w', encoding='utf-8') as json_fp: + result.write_json_into(json_fp) + + sys.exit(0) + + +def main() -> NoReturn: + if len(sys.argv) != 2: + print("usage: python -m test.libregrtest.worker JSON") + sys.exit(1) + worker_json = sys.argv[1] + + tmp_dir = get_temp_dir() + work_dir = get_work_dir(tmp_dir, worker=True) + + with exit_timeout(): + with os_helper.temp_cwd(work_dir, quiet=True): + worker_process(worker_json) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 0e11af6f36e..e76f79c274e 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -2,13 +2,12 @@ Tests common to list and UserList.UserList """ -import unittest import sys -import os from functools import cmp_to_key -from test import support, seq_tests +from test import seq_tests from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow class CommonTest(seq_tests.CommonTest): @@ -33,13 +32,13 @@ def test_init(self): self.assertEqual(a, b) def test_getitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] def test_setitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] = "python" @@ -61,9 +60,11 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(200_000): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) @@ -193,6 +194,14 @@ def test_setslice(self): self.assertRaises(TypeError, a.__setitem__) + def test_slice_assign_iterator(self): + x = self.type2test(range(5)) + x[0:3] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0, 3, 4])) + + x[:] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0])) + def test_delslice(self): a = self.type2test([0, 1]) del a[1:2] @@ -552,7 +561,7 @@ def test_constructor_exception_handling(self): class F(object): def __iter__(self): raise KeyboardInterrupt - self.assertRaises(KeyboardInterrupt, list, F()) + self.assertRaises(KeyboardInterrupt, self.type2test, F()) def test_exhausted_iterator(self): a = self.type2test([1, 2, 3]) @@ -564,3 +573,8 @@ def test_exhausted_iterator(self): self.assertEqual(list(exhit), []) self.assertEqual(list(empit), [9]) self.assertEqual(a, self.type2test([1, 2, 3, 9])) + + # gh-115733: Crash when iterating over exhausted iterator + exhit = iter(self.type2test([1, 2, 3])) + for _ in exhit: + next(exhit, 1) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 09b91147801..8c7a4f76563 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -2,6 +2,7 @@ Various tests for synchronization primitives. """ +import gc import sys import time from _thread import start_new_thread, TIMEOUT_MAX @@ -13,54 +14,79 @@ from test.support import threading_helper -def _wait(): - # A crude wait/yield function not relying on synchronization primitives. - time.sleep(0.01) +requires_fork = unittest.skipUnless(support.has_fork_support, + "platform doesn't support fork " + "(no _at_fork_reinit method)") + + +def wait_threads_blocked(nthread): + # Arbitrary sleep to wait until N threads are blocked, + # like waiting for a lock. + time.sleep(0.010 * nthread) + class Bunch(object): """ A bunch of threads. """ - def __init__(self, f, n, wait_before_exit=False): + def __init__(self, func, nthread, wait_before_exit=False): """ - Construct a bunch of `n` threads running the same function `f`. + Construct a bunch of `nthread` threads running the same function `func`. If `wait_before_exit` is True, the threads won't terminate until do_finish() is called. """ - self.f = f - self.n = n + self.func = func + self.nthread = nthread self.started = [] self.finished = [] + self.exceptions = [] self._can_exit = not wait_before_exit - self.wait_thread = threading_helper.wait_threads_exit() - self.wait_thread.__enter__() + self._wait_thread = None - def task(): - tid = threading.get_ident() - self.started.append(tid) - try: - f() - finally: - self.finished.append(tid) - while not self._can_exit: - _wait() + def task(self): + tid = threading.get_ident() + self.started.append(tid) + try: + self.func() + except BaseException as exc: + self.exceptions.append(exc) + finally: + self.finished.append(tid) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if self._can_exit: + break + + def __enter__(self): + self._wait_thread = threading_helper.wait_threads_exit(support.SHORT_TIMEOUT) + self._wait_thread.__enter__() try: - for i in range(n): - start_new_thread(task, ()) + for _ in range(self.nthread): + start_new_thread(self.task, ()) except: self._can_exit = True raise - def wait_for_started(self): - while len(self.started) < self.n: - _wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(self.started) >= self.nthread: + break + + return self - def wait_for_finished(self): - while len(self.finished) < self.n: - _wait() - # Wait for threads exit - self.wait_thread.__exit__(None, None, None) + def __exit__(self, exc_type, exc_value, traceback): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(self.finished) >= self.nthread: + break + + # Wait until threads completely exit according to _thread._count() + self._wait_thread.__exit__(None, None, None) + + # Break reference cycle + exceptions = self.exceptions + self.exceptions = None + if exceptions: + raise ExceptionGroup(f"{self.func} threads raised exceptions", + exceptions) def do_finish(self): self._can_exit = True @@ -88,6 +114,12 @@ class BaseLockTests(BaseTestCase): Tests for both recursive and non-recursive locks. """ + def wait_phase(self, phase, expected): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(phase) >= expected: + break + self.assertEqual(len(phase), expected) + def test_constructor(self): lock = self.locktype() del lock @@ -125,44 +157,60 @@ def test_try_acquire_contended(self): result = [] def f(): result.append(lock.acquire(False)) - Bunch(f, 1).wait_for_finished() + with Bunch(f, 1): + pass self.assertFalse(result[0]) lock.release() - @unittest.skip("TODO: RUSTPYTHON, sometimes hangs") + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_acquire_contended(self): lock = self.locktype() lock.acquire() - N = 5 def f(): lock.acquire() lock.release() - b = Bunch(f, N) - b.wait_for_started() - _wait() - self.assertEqual(len(b.finished), 0) - lock.release() - b.wait_for_finished() - self.assertEqual(len(b.finished), N) + N = 5 + with Bunch(f, N) as bunch: + # Threads block on lock.acquire() + wait_threads_blocked(N) + self.assertEqual(len(bunch.finished), 0) + + # Threads unblocked + lock.release() + + self.assertEqual(len(bunch.finished), N) def test_with(self): lock = self.locktype() def f(): lock.acquire() lock.release() - def _with(err=None): + + def with_lock(err=None): with lock: if err is not None: raise err - _with() - # Check the lock is unacquired - Bunch(f, 1).wait_for_finished() - self.assertRaises(TypeError, _with, TypeError) - # Check the lock is unacquired - Bunch(f, 1).wait_for_finished() - @unittest.skip("TODO: RUSTPYTHON, sometimes hangs") + # Acquire the lock, do nothing, with releases the lock + with lock: + pass + + # Check that the lock is unacquired + with Bunch(f, 1): + pass + + # Acquire the lock, raise an exception, with releases the lock + with self.assertRaises(TypeError): + with lock: + raise TypeError + + # Check that the lock is unacquired even if after an exception + # was raised in the previous "with lock:" block + with Bunch(f, 1): + pass + + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_thread_leak(self): # The lock shouldn't leak a Thread instance when used from a foreign # (non-threading) thread. @@ -170,22 +218,16 @@ def test_thread_leak(self): def f(): lock.acquire() lock.release() - n = len(threading.enumerate()) + # We run many threads in the hope that existing threads ids won't # be recycled. - Bunch(f, 15).wait_for_finished() - if len(threading.enumerate()) != n: - # There is a small window during which a Thread instance's - # target function has finished running, but the Thread is still - # alive and registered. Avoid spurious failures by waiting a - # bit more (seen on a buildbot). - time.sleep(0.4) - self.assertEqual(n, len(threading.enumerate())) + with Bunch(f, 15): + pass def test_timeout(self): lock = self.locktype() # Can't set timeout if not blocking - self.assertRaises(ValueError, lock.acquire, 0, 1) + self.assertRaises(ValueError, lock.acquire, False, 1) # Invalid timeout values self.assertRaises(ValueError, lock.acquire, timeout=-100) self.assertRaises(OverflowError, lock.acquire, timeout=1e100) @@ -204,7 +246,8 @@ def f(): results.append(lock.acquire(timeout=0.5)) t2 = time.monotonic() results.append(t2 - t1) - Bunch(f, 1).wait_for_finished() + with Bunch(f, 1): + pass self.assertFalse(results[0]) self.assertTimeout(results[1], 0.5) @@ -217,6 +260,7 @@ def test_weakref_deleted(self): lock = self.locktype() ref = weakref.ref(lock) del lock + gc.collect() # For PyPy or other GCs. self.assertIsNone(ref()) @@ -237,15 +281,13 @@ def f(): phase.append(None) with threading_helper.wait_threads_exit(): + # Thread blocked on lock.acquire() start_new_thread(f, ()) - while len(phase) == 0: - _wait() - _wait() - self.assertEqual(len(phase), 1) + self.wait_phase(phase, 1) + + # Thread unblocked lock.release() - while len(phase) == 1: - _wait() - self.assertEqual(len(phase), 2) + self.wait_phase(phase, 2) def test_different_thread(self): # Lock can be released from a different thread. @@ -253,8 +295,8 @@ def test_different_thread(self): lock.acquire() def f(): lock.release() - b = Bunch(f, 1) - b.wait_for_finished() + with Bunch(f, 1): + pass lock.acquire() lock.release() @@ -268,11 +310,50 @@ def test_state_after_timeout(self): self.assertFalse(lock.locked()) self.assertTrue(lock.acquire(blocking=False)) + @requires_fork + def test_at_fork_reinit(self): + def use_lock(lock): + # make sure that the lock still works normally + # after _at_fork_reinit() + lock.acquire() + lock.release() + + # unlocked + lock = self.locktype() + lock._at_fork_reinit() + use_lock(lock) + + # locked: _at_fork_reinit() resets the lock to the unlocked state + lock2 = self.locktype() + lock2.acquire() + lock2._at_fork_reinit() + use_lock(lock2) + class RLockTests(BaseLockTests): """ Tests for recursive locks. """ + def test_repr_count(self): + # see gh-134322: check that count values are correct: + # when a rlock is just created, + # in a second thread when rlock is acquired in the main thread. + lock = self.locktype() + self.assertIn("count=0", repr(lock)) + self.assertIn("") + evt.set() + self.assertRegex(repr(evt), r"<\w+\.Event at .*: set>") + class ConditionTests(BaseTestCase): """ @@ -466,15 +631,14 @@ def _check_notify(self, cond): # Note that this test is sensitive to timing. If the worker threads # don't execute in a timely fashion, the main thread may think they # are further along then they are. The main thread therefore issues - # _wait() statements to try to make sure that it doesn't race ahead - # of the workers. + # wait_threads_blocked() statements to try to make sure that it doesn't + # race ahead of the workers. # Secondly, this test assumes that condition variables are not subject # to spurious wakeups. The absence of spurious wakeups is an implementation # detail of Condition Variables in current CPython, but in general, not # a guaranteed property of condition variables as a programming # construct. In particular, it is possible that this can no longer # be conveniently guaranteed should their implementation ever change. - N = 5 ready = [] results1 = [] results2 = [] @@ -483,58 +647,83 @@ def f(): cond.acquire() ready.append(phase_num) result = cond.wait() + cond.release() results1.append((result, phase_num)) + cond.acquire() ready.append(phase_num) + result = cond.wait() cond.release() results2.append((result, phase_num)) - b = Bunch(f, N) - b.wait_for_started() - # first wait, to ensure all workers settle into cond.wait() before - # we continue. See issues #8799 and #30727. - while len(ready) < 5: - _wait() - ready.clear() - self.assertEqual(results1, []) - # Notify 3 threads at first - cond.acquire() - cond.notify(3) - _wait() - phase_num = 1 - cond.release() - while len(results1) < 3: - _wait() - self.assertEqual(results1, [(True, 1)] * 3) - self.assertEqual(results2, []) - # make sure all awaken workers settle into cond.wait() - while len(ready) < 3: - _wait() - # Notify 5 threads: they might be in their first or second wait - cond.acquire() - cond.notify(5) - _wait() - phase_num = 2 - cond.release() - while len(results1) + len(results2) < 8: - _wait() - self.assertEqual(results1, [(True, 1)] * 3 + [(True, 2)] * 2) - self.assertEqual(results2, [(True, 2)] * 3) - # make sure all workers settle into cond.wait() - while len(ready) < 5: - _wait() - # Notify all threads: they are all in their second wait - cond.acquire() - cond.notify_all() - _wait() - phase_num = 3 - cond.release() - while len(results2) < 5: - _wait() - self.assertEqual(results1, [(True, 1)] * 3 + [(True,2)] * 2) - self.assertEqual(results2, [(True, 2)] * 3 + [(True, 3)] * 2) - b.wait_for_finished() + + N = 5 + with Bunch(f, N): + # first wait, to ensure all workers settle into cond.wait() before + # we continue. See issues #8799 and #30727. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= N: + break + + ready.clear() + self.assertEqual(results1, []) + + # Notify 3 threads at first + count1 = 3 + cond.acquire() + cond.notify(count1) + wait_threads_blocked(count1) + + # Phase 1 + phase_num = 1 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) >= count1: + break + + self.assertEqual(results1, [(True, 1)] * count1) + self.assertEqual(results2, []) + + # Wait until awaken workers are blocked on cond.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= count1 : + break + + # Notify 5 threads: they might be in their first or second wait + cond.acquire() + cond.notify(5) + wait_threads_blocked(N) + + # Phase 2 + phase_num = 2 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= (N + count1): + break + + count2 = N - count1 + self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) + self.assertEqual(results2, [(True, 2)] * count1) + + # Make sure all workers settle into cond.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(ready) >= N: + break + + # Notify all threads: they are all in their second wait + cond.acquire() + cond.notify_all() + wait_threads_blocked(N) + + # Phase 3 + phase_num = 3 + cond.release() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results2) >= N: + break + self.assertEqual(results1, [(True, 1)] * count1 + [(True, 2)] * count2) + self.assertEqual(results2, [(True, 2)] * count1 + [(True, 3)] * count2) def test_notify(self): cond = self.condtype() @@ -544,19 +733,23 @@ def test_notify(self): def test_timeout(self): cond = self.condtype() + timeout = 0.5 results = [] - N = 5 def f(): cond.acquire() t1 = time.monotonic() - result = cond.wait(0.5) + result = cond.wait(timeout) t2 = time.monotonic() cond.release() results.append((t2 - t1, result)) - Bunch(f, N).wait_for_finished() + + N = 5 + with Bunch(f, N): + pass self.assertEqual(len(results), N) + for dt, result in results: - self.assertTimeout(dt, 0.5) + self.assertTimeout(dt, timeout) # Note that conceptually (that"s the condition variable protocol) # a wait() may succeed even if no one notifies us and before any # timeout occurs. Spurious wakeups can occur. @@ -569,17 +762,16 @@ def test_waitfor(self): state = 0 def f(): with cond: - result = cond.wait_for(lambda : state==4) + result = cond.wait_for(lambda: state == 4) self.assertTrue(result) self.assertEqual(state, 4) - b = Bunch(f, 1) - b.wait_for_started() - for i in range(4): - time.sleep(0.01) - with cond: - state += 1 - cond.notify() - b.wait_for_finished() + + with Bunch(f, 1): + for i in range(4): + time.sleep(0.010) + with cond: + state += 1 + cond.notify() def test_waitfor_timeout(self): cond = self.condtype() @@ -593,15 +785,15 @@ def f(): self.assertFalse(result) self.assertTimeout(dt, 0.1) success.append(None) - b = Bunch(f, 1) - b.wait_for_started() - # Only increment 3 times, so state == 4 is never reached. - for i in range(3): - time.sleep(0.01) - with cond: - state += 1 - cond.notify() - b.wait_for_finished() + + with Bunch(f, 1): + # Only increment 3 times, so state == 4 is never reached. + for i in range(3): + time.sleep(0.010) + with cond: + state += 1 + cond.notify() + self.assertEqual(len(success), 1) @@ -630,41 +822,107 @@ def test_acquire_destroy(self): del sem def test_acquire_contended(self): - sem = self.semtype(7) + sem_value = 7 + sem = self.semtype(sem_value) sem.acquire() - N = 10 + sem_results = [] results1 = [] results2 = [] phase_num = 0 - def f(): + + def func(): sem_results.append(sem.acquire()) results1.append(phase_num) + sem_results.append(sem.acquire()) results2.append(phase_num) - b = Bunch(f, 10) - b.wait_for_started() - while len(results1) + len(results2) < 6: - _wait() - self.assertEqual(results1 + results2, [0] * 6) - phase_num = 1 - for i in range(7): + + def wait_count(count): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= count: + break + + N = 10 + with Bunch(func, N): + # Phase 0 + count1 = sem_value - 1 + wait_count(count1) + self.assertEqual(results1 + results2, [0] * count1) + + # Phase 1 + phase_num = 1 + for i in range(sem_value): + sem.release() + count2 = sem_value + wait_count(count1 + count2) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2) + + # Phase 2 + phase_num = 2 + count3 = (sem_value - 1) + for i in range(count3): + sem.release() + wait_count(count1 + count2 + count3) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2 + [2] * count3) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + + # Final release, to let the last thread finish + count4 = 1 sem.release() - while len(results1) + len(results2) < 13: - _wait() - self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) - phase_num = 2 - for i in range(6): + + self.assertEqual(sem_results, + [True] * (count1 + count2 + count3 + count4)) + + def test_multirelease(self): + sem_value = 7 + sem = self.semtype(sem_value) + sem.acquire() + + results1 = [] + results2 = [] + phase_num = 0 + def func(): + sem.acquire() + results1.append(phase_num) + + sem.acquire() + results2.append(phase_num) + + def wait_count(count): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(results1) + len(results2) >= count: + break + + with Bunch(func, 10): + # Phase 0 + count1 = sem_value - 1 + wait_count(count1) + self.assertEqual(results1 + results2, [0] * count1) + + # Phase 1 + phase_num = 1 + count2 = sem_value + sem.release(count2) + wait_count(count1 + count2) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2) + + # Phase 2 + phase_num = 2 + count3 = sem_value - 1 + sem.release(count3) + wait_count(count1 + count2 + count3) + self.assertEqual(sorted(results1 + results2), + [0] * count1 + [1] * count2 + [2] * count3) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + + # Final release, to let the last thread finish sem.release() - while len(results1) + len(results2) < 19: - _wait() - self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) - # The semaphore is still locked - self.assertFalse(sem.acquire(False)) - # Final release, to let the last thread finish - sem.release() - b.wait_for_finished() - self.assertEqual(sem_results, [True] * (6 + 7 + 6 + 1)) def test_try_acquire(self): sem = self.semtype(2) @@ -681,7 +939,8 @@ def test_try_acquire_contended(self): def f(): results.append(sem.acquire(False)) results.append(sem.acquire(False)) - Bunch(f, 5).wait_for_finished() + with Bunch(f, 5): + pass # There can be a thread switch between acquiring the semaphore and # appending the result, therefore results will not necessarily be # ordered. @@ -707,12 +966,14 @@ def test_default_value(self): def f(): sem.acquire() sem.release() - b = Bunch(f, 1) - b.wait_for_started() - _wait() - self.assertFalse(b.finished) - sem.release() - b.wait_for_finished() + + with Bunch(f, 1) as bunch: + # Thread blocked on sem.acquire() + wait_threads_blocked(1) + self.assertFalse(bunch.finished) + + # Thread unblocked + sem.release() def test_with(self): sem = self.semtype(2) @@ -744,6 +1005,15 @@ def test_release_unacquired(self): sem.acquire() sem.release() + def test_repr(self): + sem = self.semtype(3) + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=3>") + sem.acquire() + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=2>") + sem.release() + sem.release() + self.assertRegex(repr(sem), r"<\w+\.Semaphore at .*: value=4>") + class BoundedSemaphoreTests(BaseSemaphoreTests): """ @@ -758,6 +1028,12 @@ def test_release_unacquired(self): sem.release() self.assertRaises(ValueError, sem.release) + def test_repr(self): + sem = self.semtype(3) + self.assertRegex(repr(sem), r"<\w+\.BoundedSemaphore at .*: value=3/3>") + sem.acquire() + self.assertRegex(repr(sem), r"<\w+\.BoundedSemaphore at .*: value=2/3>") + class BarrierTests(BaseTestCase): """ @@ -768,13 +1044,13 @@ class BarrierTests(BaseTestCase): def setUp(self): self.barrier = self.barriertype(self.N, timeout=self.defaultTimeout) + def tearDown(self): self.barrier.abort() def run_threads(self, f): - b = Bunch(f, self.N-1) - f() - b.wait_for_finished() + with Bunch(f, self.N): + pass def multipass(self, results, n): m = self.barrier.parties @@ -789,6 +1065,10 @@ def multipass(self, results, n): self.assertEqual(self.barrier.n_waiting, 0) self.assertFalse(self.barrier.broken) + def test_constructor(self): + self.assertRaises(ValueError, self.barriertype, parties=0) + self.assertRaises(ValueError, self.barriertype, parties=-1) + def test_barrier(self, passes=1): """ Test that a barrier is passed in lockstep @@ -865,8 +1145,9 @@ def f(): i = self.barrier.wait() if i == self.N//2: # Wait until the other threads are all in the barrier. - while self.barrier.n_waiting < self.N-1: - time.sleep(0.001) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if self.barrier.n_waiting >= (self.N - 1): + break self.barrier.reset() else: try: @@ -926,27 +1207,56 @@ def f(): i = self.barrier.wait() if i == self.N // 2: # One thread is late! - time.sleep(1.0) + time.sleep(self.defaultTimeout / 2) # Default timeout is 2.0, so this is shorter. self.assertRaises(threading.BrokenBarrierError, - self.barrier.wait, 0.5) + self.barrier.wait, self.defaultTimeout / 4) self.run_threads(f) def test_default_timeout(self): """ Test the barrier's default timeout """ - # create a barrier with a low default timeout - barrier = self.barriertype(self.N, timeout=0.3) + timeout = 0.100 + barrier = self.barriertype(2, timeout=timeout) def f(): - i = barrier.wait() - if i == self.N // 2: - # One thread is later than the default timeout of 0.3s. - time.sleep(1.0) - self.assertRaises(threading.BrokenBarrierError, barrier.wait) - self.run_threads(f) + self.assertRaises(threading.BrokenBarrierError, + barrier.wait) + + start_time = time.monotonic() + with Bunch(f, 1): + pass + dt = time.monotonic() - start_time + self.assertGreaterEqual(dt, timeout) def test_single_thread(self): b = self.barriertype(1) b.wait() b.wait() + + def test_repr(self): + barrier = self.barriertype(3) + timeout = support.LONG_TIMEOUT + self.assertRegex(repr(barrier), r"<\w+\.Barrier at .*: waiters=0/3>") + def f(): + barrier.wait(timeout) + + N = 2 + with Bunch(f, N): + # Threads blocked on barrier.wait() + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if barrier.n_waiting >= N: + break + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: waiters=2/3>") + + # Threads unblocked + barrier.wait(timeout) + + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: waiters=0/3>") + + # Abort the barrier + barrier.abort() + self.assertRegex(repr(barrier), + r"<\w+\.Barrier at .*: broken>") diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 53f29f60538..20306e1526d 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,7 +1,7 @@ # tests common to dict and UserDict import unittest import collections -import sys +from test import support class BasicTestMappingProtocol(unittest.TestCase): @@ -70,8 +70,8 @@ def test_read(self): if not d: self.fail("Full mapping must compare to True") # keys(), items(), iterkeys() ... def check_iterandlist(iter, lst, ref): - self.assertTrue(hasattr(iter, '__next__')) - self.assertTrue(hasattr(iter, '__iter__')) + self.assertHasAttr(iter, '__next__') + self.assertHasAttr(iter, '__iter__') x = list(iter) self.assertTrue(set(x)==set(lst)==set(ref)) check_iterandlist(iter(d.keys()), list(d.keys()), @@ -448,7 +448,7 @@ def __new__(cls): class Exc(Exception): pass class baddict1(self.type2test): - def __init__(self): + def __init__(self, *args, **kwargs): raise Exc() self.assertRaises(Exc, baddict1.fromkeys, [1]) @@ -595,12 +595,14 @@ def test_mutatingiteration(self): d = self._empty_mapping() d[1] = 1 try: + count = 0 for i in d: d[i+1] = 1 + if count >= 1: + self.fail("changing dict size during iteration doesn't raise Error") + count += 1 except RuntimeError: pass - else: - self.fail("changing dict size during iteration doesn't raise Error") def test_repr(self): d = self._empty_mapping() @@ -620,9 +622,12 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() + @support.skip_if_sanitizer("requires deep stack", ub=True) def test_repr_deep(self): d = self._empty_mapping() - for i in range(sys.getrecursionlimit() + 100): + for i in range(support.exceeds_recursion_limit()): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/cmath_testcases.txt b/Lib/test/mathdata/cmath_testcases.txt similarity index 99% rename from Lib/test/cmath_testcases.txt rename to Lib/test/mathdata/cmath_testcases.txt index dd7e458ddcb..7b98b5a2998 100644 --- a/Lib/test/cmath_testcases.txt +++ b/Lib/test/mathdata/cmath_testcases.txt @@ -371,9 +371,9 @@ acosh1002 acosh 0.0 inf -> inf 1.5707963267948966 acosh1003 acosh 2.3 inf -> inf 1.5707963267948966 acosh1004 acosh -0.0 inf -> inf 1.5707963267948966 acosh1005 acosh -2.3 inf -> inf 1.5707963267948966 -acosh1006 acosh 0.0 nan -> nan nan +acosh1006 acosh 0.0 nan -> nan 1.5707963267948966 ignore-imag-sign acosh1007 acosh 2.3 nan -> nan nan -acosh1008 acosh -0.0 nan -> nan nan +acosh1008 acosh -0.0 nan -> nan 1.5707963267948966 ignore-imag-sign acosh1009 acosh -2.3 nan -> nan nan acosh1010 acosh -inf 0.0 -> inf 3.1415926535897931 acosh1011 acosh -inf 2.3 -> inf 3.1415926535897931 @@ -1536,6 +1536,7 @@ sqrt0141 sqrt -1.797e+308 -9.9999999999999999e+306 -> 3.7284476432057307e+152 -1 sqrt0150 sqrt 1.7976931348623157e+308 0.0 -> 1.3407807929942596355e+154 0.0 sqrt0151 sqrt 2.2250738585072014e-308 0.0 -> 1.4916681462400413487e-154 0.0 sqrt0152 sqrt 5e-324 0.0 -> 2.2227587494850774834e-162 0.0 +sqrt0153 sqrt 5e-324 1.0 -> 0.7071067811865476 0.7071067811865476 -- special values sqrt1000 sqrt 0.0 0.0 -> 0.0 0.0 @@ -1744,6 +1745,7 @@ cosh0023 cosh 2.218885944363501 2.0015727395883687 -> -1.94294321081968 4.129026 -- large real part cosh0030 cosh 710.5 2.3519999999999999 -> -1.2967465239355998e+308 1.3076707908857333e+308 cosh0031 cosh -710.5 0.69999999999999996 -> 1.4085466381392499e+308 -1.1864024666450239e+308 +cosh0032 cosh 720.0 0.0 -> inf 0.0 overflow -- Additional real values (mpmath) cosh0050 cosh 1e-150 0.0 -> 1.0 0.0 @@ -1853,6 +1855,7 @@ sinh0023 sinh 0.043713693678420068 0.22512549887532657 -> 0.042624198673416713 0 -- large real part sinh0030 sinh 710.5 -2.3999999999999999 -> -1.3579970564885919e+308 -1.24394470907798e+308 sinh0031 sinh -710.5 0.80000000000000004 -> -1.2830671601735164e+308 1.3210954193997678e+308 +sinh0032 sinh 720.0 0.0 -> inf 0.0 overflow -- Additional real values (mpmath) sinh0050 sinh 1e-100 0.0 -> 1.00000000000000002e-100 0.0 @@ -1989,9 +1992,9 @@ tanh0065 tanh 1.797e+308 0.0 -> 1.0 0.0 --special values tanh1000 tanh 0.0 0.0 -> 0.0 0.0 -tanh1001 tanh 0.0 inf -> nan nan invalid +tanh1001 tanh 0.0 inf -> 0.0 nan invalid tanh1002 tanh 2.3 inf -> nan nan invalid -tanh1003 tanh 0.0 nan -> nan nan +tanh1003 tanh 0.0 nan -> 0.0 nan tanh1004 tanh 2.3 nan -> nan nan tanh1005 tanh inf 0.0 -> 1.0 0.0 tanh1006 tanh inf 0.7 -> 1.0 0.0 @@ -2006,7 +2009,7 @@ tanh1014 tanh nan 2.3 -> nan nan tanh1015 tanh nan inf -> nan nan tanh1016 tanh nan nan -> nan nan tanh1017 tanh 0.0 -0.0 -> 0.0 -0.0 -tanh1018 tanh 0.0 -inf -> nan nan invalid +tanh1018 tanh 0.0 -inf -> 0.0 nan invalid tanh1019 tanh 2.3 -inf -> nan nan invalid tanh1020 tanh inf -0.0 -> 1.0 -0.0 tanh1021 tanh inf -0.7 -> 1.0 -0.0 @@ -2019,9 +2022,9 @@ tanh1027 tanh nan -0.0 -> nan -0.0 tanh1028 tanh nan -2.3 -> nan nan tanh1029 tanh nan -inf -> nan nan tanh1030 tanh -0.0 -0.0 -> -0.0 -0.0 -tanh1031 tanh -0.0 -inf -> nan nan invalid +tanh1031 tanh -0.0 -inf -> -0.0 nan invalid tanh1032 tanh -2.3 -inf -> nan nan invalid -tanh1033 tanh -0.0 nan -> nan nan +tanh1033 tanh -0.0 nan -> -0.0 nan tanh1034 tanh -2.3 nan -> nan nan tanh1035 tanh -inf -0.0 -> -1.0 -0.0 tanh1036 tanh -inf -0.7 -> -1.0 -0.0 @@ -2032,7 +2035,7 @@ tanh1040 tanh -inf -3.5 -> -1.0 -0.0 tanh1041 tanh -inf -inf -> -1.0 0.0 ignore-imag-sign tanh1042 tanh -inf nan -> -1.0 0.0 ignore-imag-sign tanh1043 tanh -0.0 0.0 -> -0.0 0.0 -tanh1044 tanh -0.0 inf -> nan nan invalid +tanh1044 tanh -0.0 inf -> -0.0 nan invalid tanh1045 tanh -2.3 inf -> nan nan invalid tanh1046 tanh -inf 0.0 -> -1.0 0.0 tanh1047 tanh -inf 0.7 -> -1.0 0.0 @@ -2304,9 +2307,9 @@ tan0066 tan -8.79645943005142 0.0 -> 0.7265425280053614098 0.0 -- special values tan1000 tan -0.0 0.0 -> -0.0 0.0 -tan1001 tan -inf 0.0 -> nan nan invalid +tan1001 tan -inf 0.0 -> nan 0.0 invalid tan1002 tan -inf 2.2999999999999998 -> nan nan invalid -tan1003 tan nan 0.0 -> nan nan +tan1003 tan nan 0.0 -> nan 0.0 tan1004 tan nan 2.2999999999999998 -> nan nan tan1005 tan -0.0 inf -> -0.0 1.0 tan1006 tan -0.69999999999999996 inf -> -0.0 1.0 @@ -2321,7 +2324,7 @@ tan1014 tan -2.2999999999999998 nan -> nan nan tan1015 tan -inf nan -> nan nan tan1016 tan nan nan -> nan nan tan1017 tan 0.0 0.0 -> 0.0 0.0 -tan1018 tan inf 0.0 -> nan nan invalid +tan1018 tan inf 0.0 -> nan 0.0 invalid tan1019 tan inf 2.2999999999999998 -> nan nan invalid tan1020 tan 0.0 inf -> 0.0 1.0 tan1021 tan 0.69999999999999996 inf -> 0.0 1.0 @@ -2334,9 +2337,9 @@ tan1027 tan 0.0 nan -> 0.0 nan tan1028 tan 2.2999999999999998 nan -> nan nan tan1029 tan inf nan -> nan nan tan1030 tan 0.0 -0.0 -> 0.0 -0.0 -tan1031 tan inf -0.0 -> nan nan invalid +tan1031 tan inf -0.0 -> nan -0.0 invalid tan1032 tan inf -2.2999999999999998 -> nan nan invalid -tan1033 tan nan -0.0 -> nan nan +tan1033 tan nan -0.0 -> nan -0.0 tan1034 tan nan -2.2999999999999998 -> nan nan tan1035 tan 0.0 -inf -> 0.0 -1.0 tan1036 tan 0.69999999999999996 -inf -> 0.0 -1.0 @@ -2347,7 +2350,7 @@ tan1040 tan 3.5 -inf -> 0.0 -1.0 tan1041 tan inf -inf -> -0.0 -1.0 ignore-real-sign tan1042 tan nan -inf -> -0.0 -1.0 ignore-real-sign tan1043 tan -0.0 -0.0 -> -0.0 -0.0 -tan1044 tan -inf -0.0 -> nan nan invalid +tan1044 tan -inf -0.0 -> nan -0.0 invalid tan1045 tan -inf -2.2999999999999998 -> nan nan invalid tan1046 tan -0.0 -inf -> -0.0 -1.0 tan1047 tan -0.69999999999999996 -inf -> -0.0 -1.0 diff --git a/Lib/test/mathdata/floating_points.txt b/Lib/test/mathdata/floating_points.txt new file mode 100644 index 00000000000..539073d19d8 --- /dev/null +++ b/Lib/test/mathdata/floating_points.txt @@ -0,0 +1,1028 @@ +# These numbers are used to test floating point binary-to-decimal conversion. +# They are based on the TCL test suite (tests/expr.test), which is based on +# test data from: +# Brigitte Verdonk, Annie Cuyt, Dennis Verschaeren, A precision and range +# independent tool for testing floating-point arithmetic II: Conversions, +# ACM Transactions on Mathematical Software 27:2 (March 2001), pp. 119-140. + +0E0 +-0E0 +1E0 +15E-1 +125E-2 +1125E-3 +10625E-4 +103125E-5 +1015625E-6 +10078125E-7 +100390625E-8 +1001953125E-9 +10009765625E-10 +100048828125E-11 +1000244140625E-12 +10001220703125E-13 +100006103515625E-14 +1000030517578125E-15 +10000152587890625E-16 ++8E153 +-1E153 ++9E306 +-2E153 ++7E-304 +-3E-49 ++7E-303 +-6E-49 ++9E43 +-9E44 ++8E303 +-1E303 ++7E-287 +-2E-204 ++2E-205 +-9E-47 ++34E195 +-68E195 ++85E194 +-67E97 ++93E-234 +-19E-87 ++38E-87 +-38E-88 +-69E220 ++18E43 +-36E43 ++61E-99 +-43E-92 ++86E-92 +-51E-74 ++283E85 +-566E85 ++589E187 +-839E143 +-744E-234 ++930E-235 +-186E-234 ++604E175 +-302E175 ++755E174 +-151E175 ++662E-213 +-408E-74 ++510E-75 ++6782E55 +-2309E92 ++7963E34 +-3391E55 ++7903E-96 +-7611E-226 ++4907E-196 +-5547E-311 ++5311E241 +-5311E243 ++5311E242 ++9269E-45 +-8559E-289 ++8699E-276 +-8085E-64 ++74819E201 +-82081E41 ++51881E37 +-55061E157 ++77402E-215 +-33891E-92 ++38701E-215 +-82139E-76 ++75859E25 ++89509E140 +-57533E287 ++46073E-32 +-92146E-32 ++83771E-74 +-34796E-276 ++584169E229 ++164162E41 +-328324E41 ++209901E-11 +-419802E-11 ++940189E-112 +-892771E-213 ++757803E120 +-252601E120 ++252601E121 +-505202E120 ++970811E-264 +-654839E-60 ++289767E-178 +-579534E-178 +-8823691E130 ++9346704E229 +-1168338E229 +-6063369E-136 ++3865421E-225 +-5783893E-127 ++2572231E223 +-5144462E223 ++1817623E109 ++6431543E-97 +-5444097E-21 ++8076999E-121 +-9997649E-270 ++50609263E157 ++70589528E130 +-88236910E129 ++87575437E-310 +-23135572E-127 ++85900881E177 +-84863171E113 ++68761586E232 +-50464069E286 ++27869147E-248 +-55738294E-248 ++70176353E-53 +-80555086E-32 +-491080654E121 ++526250918E287 +-245540327E121 +-175150874E-310 ++350301748E-310 +-437877185E-311 ++458117166E52 +-916234332E52 ++229058583E52 +-525789935E98 ++282926897E-227 +-565853794E-227 ++667284113E-240 +-971212611E-126 ++9981396317E-182 +-5035231965E-156 ++8336960483E-153 +-8056371144E-155 ++6418488827E79 +-3981006983E252 ++7962013966E252 +-4713898551E261 ++8715380633E-58 +-9078555839E-109 ++9712126110E-127 ++42333842451E201 +-84667684902E201 ++23792120709E-315 +-78564021519E-227 ++71812054883E-188 +-30311163631E-116 ++71803914657E292 ++36314223356E-109 ++18157111678E-109 +-45392779195E-110 ++778380362293E218 +-685763015669E280 ++952918668151E70 +-548357443505E32 ++384865004907E-285 +-769730009814E-285 ++697015418417E-93 +-915654049301E-28 ++178548656339E169 +-742522891517E259 ++742522891517E258 +-357097312678E169 +-3113521449172E218 ++3891901811465E217 +-1556760724586E218 ++9997878507563E-195 +-7247563029154E-319 ++3623781514577E-319 +-3092446298323E-200 ++6363857920591E145 +-8233559360849E94 ++2689845954547E49 +-5379691909094E49 ++5560322501926E-301 +-7812878489261E-179 ++8439398533053E-256 +-2780161250963E-301 +-87605699161665E155 +-17521139832333E156 +-88218101363513E-170 ++38639244311627E-115 ++35593959807306E261 +-53390939710959E260 ++71187919614612E261 +-88984899518265E260 ++77003665618895E-73 +-15400733123779E-72 ++61602932495116E-72 +-30801466247558E-72 ++834735494917063E-300 +-589795149206434E-151 ++475603213226859E-42 +-294897574603217E-151 ++850813008001913E93 +-203449172043339E185 ++406898344086678E185 +-813796688173356E185 ++6045338514609393E244 +-5145963778954906E142 ++2572981889477453E142 +-6965949469487146E74 ++6182410494241627E-119 +-8510309498186985E-277 ++6647704637273331E-212 +-2215901545757777E-212 ++3771476185376383E276 +-3729901848043846E212 ++3771476185376383E277 +-9977830465649166E119 ++8439928496349319E-142 +-8204230082070882E-59 ++8853686434843997E-244 +-5553274272288559E-104 ++36149023611096162E144 +-36149023611096162E147 ++18074511805548081E146 +-18074511805548081E147 ++97338774138954421E-290 +-88133809804950961E-308 ++94080055902682397E-243 +-24691002732654881E-115 ++52306490527514614E49 +-26153245263757307E49 ++55188692254193604E165 +-68985865317742005E164 ++27176258005319167E-261 +-73169230107256116E-248 ++91461537634070145E-249 +-54352516010638334E-261 ++586144289638535878E280 +-601117006785295431E245 ++293072144819267939E280 +-953184713238516652E272 ++902042358290366539E-281 +-557035730189854663E-294 ++902042358290366539E-280 +-354944100507554393E-238 ++272104041512242479E199 +-816312124536727437E199 ++544208083024484958E199 +-792644927852378159E78 +-679406450132979175E-263 ++543525160106383340E-262 ++7400253695682920196E215 +-1850063423920730049E215 ++3700126847841460098E215 +-9250317119603650245E214 ++8396094300569779681E-252 +-3507665085003296281E-75 ++7015330170006592562E-75 +-7015330170006592562E-74 ++7185620434951919351E205 +-1360520207561212395E198 ++2178999185345151731E-184 +-8691089486201567102E-218 ++4345544743100783551E-218 +-4357998370690303462E-184 ++59825267349106892461E177 +-62259110684423957791E47 ++58380168477038565599E265 +-62259110684423957791E48 +-33584377202279118724E-252 +-57484963479615354808E205 ++71856204349519193510E204 +-14371240869903838702E205 ++36992084760177624177E-318 +-73984169520355248354E-318 ++99257763227713890244E-115 +-87336362425182547697E-280 ++7E289 +-3E153 ++6E153 +-5E243 ++7E-161 +-7E-172 ++8E-63 +-7E-113 ++8E126 +-4E126 ++5E125 +-1E126 ++8E-163 +-1E-163 ++2E-163 +-4E-163 ++51E195 +-37E46 ++74E46 +-56E289 ++69E-145 +-70E-162 ++56E-161 +-21E-303 ++34E-276 +-68E-276 ++85E-277 +-87E-274 ++829E102 +-623E100 ++723E-162 +-457E-102 ++914E-102 +-323E-135 ++151E176 +-302E176 ++921E90 +-604E176 ++823E-206 +-463E-114 ++348E-274 ++9968E100 +-6230E99 ++1246E100 ++6676E-296 +-8345E-297 ++1669E-296 +-3338E-296 ++3257E58 +-6514E58 ++2416E176 ++8085E-63 +-3234E-62 ++1617E-62 +-6468E-62 ++53418E111 +-60513E160 ++26709E111 +-99447E166 ++12549E48 +-25098E48 ++50196E48 +-62745E47 ++83771E-73 +-97451E-167 ++86637E-203 +-75569E-254 ++473806E83 +-947612E83 ++292369E76 +-584738E76 ++933587E-140 +-720919E-14 ++535001E-149 +-890521E-235 ++548057E81 +-706181E88 ++820997E106 +-320681E63 ++928609E-261 +-302276E-254 ++151138E-254 ++4691773E45 +-9383546E45 ++3059949E-243 +-6119898E-243 ++5356626E-213 +-4877378E-199 ++7716693E223 +-5452869E109 ++4590831E156 +-9181662E156 +-3714436E-261 ++4643045E-262 +-7428872E-261 ++52942146E130 +-27966061E145 ++26471073E130 +-55932122E145 ++95412548E-99 +-47706274E-99 ++23853137E-99 +-78493654E-301 ++65346417E29 +-51083099E167 ++89396333E264 +-84863171E114 ++59540836E-251 +-74426045E-252 ++14885209E-251 +-29770418E-251 ++982161308E122 +-245540327E122 ++491080654E122 ++525452622E-310 +-771837113E-134 ++820858081E-150 +-262726311E-310 ++923091487E209 +-653777767E273 ++842116236E-53 +-741111169E-202 ++839507247E-284 +-951487269E-264 +-9821613080E121 ++6677856011E-31 +-3573796826E-266 ++7147593652E-266 +-9981396317E-181 ++3268888835E272 +-2615111068E273 ++1307555534E273 ++2990671154E-190 +-1495335577E-190 ++5981342308E-190 +-7476677885E-191 ++82259684194E-202 +-93227267727E-49 ++41129842097E-202 +-47584241418E-314 +-79360293406E92 ++57332259349E225 +-57202326162E111 ++86860597053E-206 +-53827010643E-200 ++53587107423E-61 ++635007636765E200 ++508006109412E201 +-254003054706E201 ++561029718715E-72 +-897647549944E-71 ++112205943743E-71 +-873947086081E-236 ++809184709177E116 +-573112917422E81 ++286556458711E81 ++952805821491E-259 +-132189992873E-44 +-173696038493E-144 ++1831132757599E-107 +-9155663787995E-108 ++7324531030396E-107 +-9277338894969E-200 ++8188292423973E287 +-5672557437938E59 ++2836278718969E59 +-9995153153494E54 ++9224786422069E-291 +-3142213164987E-294 ++6284426329974E-294 +-8340483752889E-301 ++67039371486466E89 +-62150786615239E197 ++33519685743233E89 +-52563419496999E156 ++32599460466991E-65 +-41010988798007E-133 ++65198920933982E-65 +-82021977596014E-133 ++80527976643809E61 +-74712611505209E158 ++53390939710959E261 +-69277302659155E225 ++46202199371337E-72 +-23438635467783E-179 ++41921560615349E-67 +-92404398742674E-72 ++738545606647197E124 +-972708181182949E117 +-837992143580825E87 ++609610927149051E-255 +-475603213226859E-41 ++563002800671023E-177 +-951206426453718E-41 ++805416432656519E202 +-530658674694337E159 ++946574173863918E208 +-318329953318553E113 +-462021993713370E-73 ++369617594970696E-72 ++3666156212014994E233 +-1833078106007497E233 ++8301790508624232E174 +-1037723813578029E174 ++7297662880581139E-286 +-5106185698912191E-276 ++7487252720986826E-165 +-3743626360493413E-165 ++3773057430100257E230 +-7546114860200514E230 ++4321222892463822E58 +-7793560217139653E51 ++26525993941010681E112 +-53051987882021362E112 ++72844871414247907E77 +-88839359596763261E105 ++18718131802467065E-166 +-14974505441973652E-165 ++73429396004640239E106 +-58483921078398283E57 ++41391519190645203E165 +-82783038381290406E165 ++58767043776702677E-163 +-90506231831231999E-129 ++64409240769861689E-159 +-77305427432277771E-190 ++476592356619258326E273 +-953184713238516652E273 ++899810892172646163E283 +-929167076892018333E187 ++647761278967534239E-312 +-644290479820542942E-180 ++926145344610700019E-225 +-958507931896511964E-246 ++272104041512242479E200 +-792644927852378159E79 ++544208083024484958E200 +-929963218616126365E290 ++305574339166810102E-219 +-152787169583405051E-219 ++611148678333620204E-219 +-763935847917025255E-220 ++7439550220920798612E158 +-3719775110460399306E158 ++9299437776150998265E157 +-7120190517612959703E120 ++3507665085003296281E-73 +-7015330170006592562E-73 +-6684428762278255956E-294 +-1088416166048969916E200 +-8707329328391759328E200 ++4439021781608558002E-65 +-8878043563217116004E-65 ++2219510890804279001E-65 ++33051223951904955802E55 +-56961524140903677624E120 ++71201905176129597030E119 ++14030660340013185124E-73 +-17538325425016481405E-74 ++67536228609141569109E-133 +-35620497849450218807E-306 ++66550376797582521751E-126 +-71240995698900437614E-306 ++3E24 +-6E24 ++6E26 +-7E25 ++1E-14 +-2E-14 ++4E-14 +-8E-14 ++5E26 +-8E27 ++1E27 +-4E27 ++9E-13 +-7E-20 ++56E25 +-70E24 ++51E26 ++71E-17 +-31E-5 ++62E-5 +-94E-8 ++67E27 +-81E24 ++54E23 +-54E25 ++63E-22 +-63E-23 ++43E-4 +-86E-4 ++942E26 +-471E25 ++803E24 +-471E26 +-409E-21 ++818E-21 +-867E-8 ++538E27 +-857E24 ++269E27 +-403E26 ++959E-7 +-959E-6 ++373E-27 +-746E-27 ++4069E24 +-4069E23 +-8138E24 ++8294E-15 +-4147E-14 ++4147E-15 +-8294E-14 ++538E27 +-2690E26 ++269E27 +-2152E27 ++1721E-17 +-7979E-27 ++6884E-17 +-8605E-18 ++82854E27 +-55684E24 ++27842E24 +-48959E25 ++81921E-17 +-76207E-8 ++4147E-15 +-41470E-16 ++89309E24 ++75859E26 +-75859E25 ++14257E-23 +-28514E-23 ++57028E-23 +-71285E-24 ++344863E27 +-951735E27 ++200677E23 +-401354E24 ++839604E-11 +-209901E-11 ++419802E-11 +-537734E-24 ++910308E26 +-227577E26 ++455154E26 +-531013E25 ++963019E-21 +-519827E-13 ++623402E-27 +-311701E-27 ++9613651E26 +-9191316E23 ++4595658E23 +-2297829E23 +-1679208E-11 ++3379223E27 +-6758446E27 ++5444097E-21 +-8399969E-27 ++8366487E-16 +-8366487E-15 ++65060671E25 ++65212389E23 ++55544957E-13 +-51040905E-20 ++99585767E-22 +-99585767E-23 ++40978393E26 +-67488159E24 ++69005339E23 +-81956786E26 +-87105552E-21 ++10888194E-21 +-21776388E-21 ++635806667E27 +-670026614E25 ++335013307E26 +-335013307E25 ++371790617E-24 +-371790617E-25 ++743581234E-24 +-743581234E-25 ++202464477E24 +-404928954E24 ++997853758E27 +-997853758E26 ++405498418E-17 +-582579084E-14 ++608247627E-18 +-291289542E-14 +-9537100005E26 ++6358066670E27 +-1271613334E27 ++5229646999E-16 ++5229646999E-17 ++4429943614E24 +-8859887228E24 ++2214971807E24 +-4176887093E26 ++4003495257E-20 +-4361901637E-23 ++8723803274E-23 +-8006990514E-20 ++72835110098E27 +-36417555049E27 ++84279630104E25 +-84279630104E24 ++21206176437E-27 +-66461566917E-22 ++64808355539E-16 +-84932679673E-19 ++65205430094E26 +-68384463429E25 ++32602715047E26 +-62662203426E27 ++58784444678E-18 +-50980203373E-21 ++29392222339E-18 +-75529940323E-27 +-937495906299E26 ++842642485799E-20 +-387824150699E-23 ++924948814726E-27 +-775648301398E-23 ++547075707432E25 ++683844634290E24 +-136768926858E25 ++509802033730E-22 ++101960406746E-21 +-815683253968E-21 ++7344124123524E24 +-9180155154405E23 ++6479463327323E27 +-1836031030881E24 ++4337269293039E-19 +-4599163554373E-23 ++9198327108746E-23 ++4812803938347E27 +-8412030890011E23 ++9625607876694E27 +-4739968828249E24 ++9697183891673E-23 +-7368108517543E-20 ++51461358161422E25 +-77192037242133E26 ++77192037242133E25 +-51461358161422E27 ++43999661561541E-21 +-87999323123082E-21 ++48374886826137E-26 +-57684246567111E-23 ++87192805957686E23 +-75108713005913E24 ++64233110587487E27 +-77577471133384E-23 ++48485919458365E-24 +-56908598265713E-26 ++589722294620133E23 ++652835804449289E-22 +-656415363936202E-23 ++579336749585745E-25 +-381292764980839E-26 ++965265859649698E23 +-848925235434882E27 ++536177612222491E23 +-424462617717441E27 ++276009279888989E-27 +-608927158043691E-26 ++552018559777978E-27 +-425678377667758E-22 ++8013702726927119E26 ++8862627962362001E27 +-5068007907757162E26 +-7379714799828406E-23 ++4114538064016107E-27 +-3689857399914203E-23 ++5575954851815478E23 ++3395700941739528E27 ++4115535777581961E-23 +-8231071555163922E-23 ++6550246696190871E-26 +-68083046403986701E27 ++43566388595783643E27 +-87132777191567286E27 ++59644881059342141E25 +-83852770718576667E23 ++99482967418206961E-25 +-99482967418206961E-26 ++87446669969994614E-27 +-43723334984997307E-27 ++5E24 +-8E25 ++1E25 +-4E25 ++2E-5 +-5E-6 ++4E-5 +-3E-20 ++3E27 +-9E26 ++7E25 +-6E27 ++2E-21 +-5E-22 +-4E-21 ++87E25 +-97E24 ++82E-24 +-41E-24 ++76E-23 ++83E25 +-50E27 ++25E27 +-99E27 ++97E-10 +-57E-20 ++997E23 ++776E24 +-388E24 ++521E-10 +-506E-26 ++739E-10 +-867E-7 +-415E24 ++332E25 +-664E25 ++291E-13 +-982E-8 ++582E-13 +-491E-8 ++4574E26 +-8609E26 ++2287E26 +-4818E24 ++6529E-8 +-8151E-21 ++1557E-12 +-2573E-18 ++4929E-16 +-3053E-22 ++9858E-16 +-7767E-11 ++54339E26 +-62409E25 ++32819E27 +-89849E27 ++63876E-20 +-15969E-20 ++31938E-20 +-79845E-21 ++89306E27 +-25487E24 ++79889E24 +-97379E26 ++81002E-8 +-43149E-25 ++40501E-8 +-60318E-10 +-648299E27 ++780649E24 ++720919E-14 +-629703E-11 ++557913E24 +-847899E23 ++565445E27 +-736531E24 ++680013E-19 +-529981E-10 ++382923E-23 +-633614E-18 ++2165479E27 +-8661916E27 ++4330958E27 +-9391993E22 +-5767352E-14 ++7209190E-15 +-1441838E-14 ++8478990E22 ++1473062E24 ++8366487E-14 +-8399969E-25 ++9366737E-12 +-9406141E-13 ++65970979E24 +-65060671E26 ++54923002E27 +-63846927E25 ++99585767E-21 ++67488159E25 +-69005339E24 ++81956786E27 +-40978393E27 ++77505754E-12 +-38752877E-12 ++82772981E-15 +-95593517E-25 ++200036989E25 +-772686455E27 ++859139907E23 +-400073978E25 ++569014327E-14 +-794263862E-15 ++397131931E-15 +-380398957E-16 ++567366773E27 +-337440795E24 ++134976318E25 +-269952636E25 ++932080597E-20 +-331091924E-15 +-413864905E-16 ++8539246247E26 +-5859139791E26 ++6105010149E24 +-3090745820E27 ++3470877773E-20 +-6136309089E-27 ++8917758713E-19 +-6941755546E-20 ++9194900535E25 +-1838980107E26 ++7355920428E26 +-3677960214E26 ++8473634343E-17 +-8870766274E-16 ++4435383137E-16 +-9598990129E-15 ++71563496764E26 +-89454370955E25 ++17890874191E26 +-35781748382E26 ++57973447842E-19 +-28986723921E-19 ++76822711313E-19 +-97699466874E-20 ++67748656762E27 +-19394840991E24 ++38789681982E24 +-33874328381E27 ++54323763886E-27 +-58987193887E-20 ++27161881943E-27 +-93042648033E-19 ++520831059055E27 +-768124264394E25 ++384062132197E25 ++765337749889E-25 ++794368912771E25 +-994162090146E23 ++781652779431E26 ++910077190046E-26 +-455038595023E-26 ++471897551096E-20 +-906698409911E-21 ++8854128003935E25 +-8146122716299E27 ++7083302403148E26 +-3541651201574E26 ++8394920649291E-25 +-7657975756753E-22 ++5473834002228E-20 +-6842292502785E-21 +-2109568884597E25 ++8438275538388E25 +-4219137769194E25 ++3200141789841E-25 +-8655689322607E-22 ++6400283579682E-25 +-8837719634493E-21 ++19428217075297E24 +-38856434150594E24 ++77712868301188E24 +-77192037242133E27 ++76579757567530E-23 ++15315951513506E-22 +-38289878783765E-23 ++49378033925202E25 +-50940527102367E24 ++98756067850404E25 +-99589397544892E26 +-56908598265713E-25 ++97470695699657E-22 +-35851901247343E-25 ++154384074484266E27 +-308768148968532E27 ++910990389005985E23 ++271742424169201E-27 +-543484848338402E-27 ++162192083357563E-26 +-869254552770081E-23 ++664831007626046E24 +-332415503813023E24 ++943701829041427E24 +-101881054204734E24 ++828027839666967E-27 +-280276135608777E-27 ++212839188833879E-21 +-113817196531426E-25 ++9711553197796883E27 +-2739849386524269E26 ++5479698773048538E26 ++6124568318523113E-25 +-1139777988171071E-24 ++6322612303128019E-27 +-2955864564844617E-25 +-9994029144998961E25 +-2971238324022087E27 +-1656055679333934E-27 +-1445488709150234E-26 ++55824717499885172E27 +-69780896874856465E26 ++84161538867545199E25 +-27912358749942586E27 ++24711112462926331E-25 +-12645224606256038E-27 +-12249136637046226E-25 ++74874448287465757E27 +-35642836832753303E24 +-71285673665506606E24 ++43723334984997307E-26 ++10182419849537963E-24 +-93501703572661982E-26 + +# A value that caused a crash in debug builds for Python >= 2.7, 3.1 +# See http://bugs.python.org/issue7632 +2183167012312112312312.23538020374420446192e-370 + +# Another value designed to test a corner case of Python's strtod code. +0.99999999999999999999999999999999999999999e+23 diff --git a/Lib/test/mathdata/formatfloat_testcases.txt b/Lib/test/mathdata/formatfloat_testcases.txt new file mode 100644 index 00000000000..25c07ba2939 --- /dev/null +++ b/Lib/test/mathdata/formatfloat_testcases.txt @@ -0,0 +1,355 @@ +-- 'f' code formatting, with explicit precision (>= 0). Output always +-- has the given number of places after the point; zeros are added if +-- necessary to make this true. + +-- zeros +%.0f 0 -> 0 +%.1f 0 -> 0.0 +%.2f 0 -> 0.00 +%.3f 0 -> 0.000 +%.50f 0 -> 0.00000000000000000000000000000000000000000000000000 + +-- precision 0; result should never include a . +%.0f 1.5 -> 2 +%.0f 2.5 -> 2 +%.0f 3.5 -> 4 +%.0f 0.0 -> 0 +%.0f 0.1 -> 0 +%.0f 0.001 -> 0 +%.0f 10.0 -> 10 +%.0f 10.1 -> 10 +%.0f 10.01 -> 10 +%.0f 123.456 -> 123 +%.0f 1234.56 -> 1235 +%.0f 1e49 -> 9999999999999999464902769475481793196872414789632 +%.0f 9.9999999999999987e+49 -> 99999999999999986860582406952576489172979654066176 +%.0f 1e50 -> 100000000000000007629769841091887003294964970946560 + +-- precision 1 +%.1f 0.0001 -> 0.0 +%.1f 0.001 -> 0.0 +%.1f 0.01 -> 0.0 +%.1f 0.04 -> 0.0 +%.1f 0.06 -> 0.1 +%.1f 0.25 -> 0.2 +%.1f 0.75 -> 0.8 +%.1f 1.4 -> 1.4 +%.1f 1.5 -> 1.5 +%.1f 10.0 -> 10.0 +%.1f 1000.03 -> 1000.0 +%.1f 1234.5678 -> 1234.6 +%.1f 1234.7499 -> 1234.7 +%.1f 1234.75 -> 1234.8 + +-- precision 2 +%.2f 0.0001 -> 0.00 +%.2f 0.001 -> 0.00 +%.2f 0.004999 -> 0.00 +%.2f 0.005001 -> 0.01 +%.2f 0.01 -> 0.01 +%.2f 0.125 -> 0.12 +%.2f 0.375 -> 0.38 +%.2f 1234500 -> 1234500.00 +%.2f 1234560 -> 1234560.00 +%.2f 1234567 -> 1234567.00 +%.2f 1234567.8 -> 1234567.80 +%.2f 1234567.89 -> 1234567.89 +%.2f 1234567.891 -> 1234567.89 +%.2f 1234567.8912 -> 1234567.89 + +-- alternate form always includes a decimal point. This only +-- makes a difference when the precision is 0. +%#.0f 0 -> 0. +%#.1f 0 -> 0.0 +%#.0f 1.5 -> 2. +%#.0f 2.5 -> 2. +%#.0f 10.1 -> 10. +%#.0f 1234.56 -> 1235. +%#.1f 1.4 -> 1.4 +%#.2f 0.375 -> 0.38 + +-- if precision is omitted it defaults to 6 +%f 0 -> 0.000000 +%f 1230000 -> 1230000.000000 +%f 1234567 -> 1234567.000000 +%f 123.4567 -> 123.456700 +%f 1.23456789 -> 1.234568 +%f 0.00012 -> 0.000120 +%f 0.000123 -> 0.000123 +%f 0.00012345 -> 0.000123 +%f 0.000001 -> 0.000001 +%f 0.0000005001 -> 0.000001 +%f 0.0000004999 -> 0.000000 + +-- 'e' code formatting with explicit precision (>= 0). Output should +-- always have exactly the number of places after the point that were +-- requested. + +-- zeros +%.0e 0 -> 0e+00 +%.1e 0 -> 0.0e+00 +%.2e 0 -> 0.00e+00 +%.10e 0 -> 0.0000000000e+00 +%.50e 0 -> 0.00000000000000000000000000000000000000000000000000e+00 + +-- precision 0. no decimal point in the output +%.0e 0.01 -> 1e-02 +%.0e 0.1 -> 1e-01 +%.0e 1 -> 1e+00 +%.0e 10 -> 1e+01 +%.0e 100 -> 1e+02 +%.0e 0.012 -> 1e-02 +%.0e 0.12 -> 1e-01 +%.0e 1.2 -> 1e+00 +%.0e 12 -> 1e+01 +%.0e 120 -> 1e+02 +%.0e 123.456 -> 1e+02 +%.0e 0.000123456 -> 1e-04 +%.0e 123456000 -> 1e+08 +%.0e 0.5 -> 5e-01 +%.0e 1.4 -> 1e+00 +%.0e 1.5 -> 2e+00 +%.0e 1.6 -> 2e+00 +%.0e 2.4999999 -> 2e+00 +%.0e 2.5 -> 2e+00 +%.0e 2.5000001 -> 3e+00 +%.0e 3.499999999999 -> 3e+00 +%.0e 3.5 -> 4e+00 +%.0e 4.5 -> 4e+00 +%.0e 5.5 -> 6e+00 +%.0e 6.5 -> 6e+00 +%.0e 7.5 -> 8e+00 +%.0e 8.5 -> 8e+00 +%.0e 9.4999 -> 9e+00 +%.0e 9.5 -> 1e+01 +%.0e 10.5 -> 1e+01 +%.0e 14.999 -> 1e+01 +%.0e 15 -> 2e+01 + +-- precision 1 +%.1e 0.0001 -> 1.0e-04 +%.1e 0.001 -> 1.0e-03 +%.1e 0.01 -> 1.0e-02 +%.1e 0.1 -> 1.0e-01 +%.1e 1 -> 1.0e+00 +%.1e 10 -> 1.0e+01 +%.1e 100 -> 1.0e+02 +%.1e 120 -> 1.2e+02 +%.1e 123 -> 1.2e+02 +%.1e 123.4 -> 1.2e+02 + +-- precision 2 +%.2e 0.00013 -> 1.30e-04 +%.2e 0.000135 -> 1.35e-04 +%.2e 0.0001357 -> 1.36e-04 +%.2e 0.0001 -> 1.00e-04 +%.2e 0.001 -> 1.00e-03 +%.2e 0.01 -> 1.00e-02 +%.2e 0.1 -> 1.00e-01 +%.2e 1 -> 1.00e+00 +%.2e 10 -> 1.00e+01 +%.2e 100 -> 1.00e+02 +%.2e 1000 -> 1.00e+03 +%.2e 1500 -> 1.50e+03 +%.2e 1590 -> 1.59e+03 +%.2e 1598 -> 1.60e+03 +%.2e 1598.7 -> 1.60e+03 +%.2e 1598.76 -> 1.60e+03 +%.2e 9999 -> 1.00e+04 + +-- omitted precision defaults to 6 +%e 0 -> 0.000000e+00 +%e 165 -> 1.650000e+02 +%e 1234567 -> 1.234567e+06 +%e 12345678 -> 1.234568e+07 +%e 1.1 -> 1.100000e+00 + +-- alternate form always contains a decimal point. This only makes +-- a difference when precision is 0. + +%#.0e 0.01 -> 1.e-02 +%#.0e 0.1 -> 1.e-01 +%#.0e 1 -> 1.e+00 +%#.0e 10 -> 1.e+01 +%#.0e 100 -> 1.e+02 +%#.0e 0.012 -> 1.e-02 +%#.0e 0.12 -> 1.e-01 +%#.0e 1.2 -> 1.e+00 +%#.0e 12 -> 1.e+01 +%#.0e 120 -> 1.e+02 +%#.0e 123.456 -> 1.e+02 +%#.0e 0.000123456 -> 1.e-04 +%#.0e 123456000 -> 1.e+08 +%#.0e 0.5 -> 5.e-01 +%#.0e 1.4 -> 1.e+00 +%#.0e 1.5 -> 2.e+00 +%#.0e 1.6 -> 2.e+00 +%#.0e 2.4999999 -> 2.e+00 +%#.0e 2.5 -> 2.e+00 +%#.0e 2.5000001 -> 3.e+00 +%#.0e 3.499999999999 -> 3.e+00 +%#.0e 3.5 -> 4.e+00 +%#.0e 4.5 -> 4.e+00 +%#.0e 5.5 -> 6.e+00 +%#.0e 6.5 -> 6.e+00 +%#.0e 7.5 -> 8.e+00 +%#.0e 8.5 -> 8.e+00 +%#.0e 9.4999 -> 9.e+00 +%#.0e 9.5 -> 1.e+01 +%#.0e 10.5 -> 1.e+01 +%#.0e 14.999 -> 1.e+01 +%#.0e 15 -> 2.e+01 +%#.1e 123.4 -> 1.2e+02 +%#.2e 0.0001357 -> 1.36e-04 + +-- 'g' code formatting. + +-- zeros +%.0g 0 -> 0 +%.1g 0 -> 0 +%.2g 0 -> 0 +%.3g 0 -> 0 +%.4g 0 -> 0 +%.10g 0 -> 0 +%.50g 0 -> 0 +%.100g 0 -> 0 + +-- precision 0 doesn't make a lot of sense for the 'g' code (what does +-- it mean to have no significant digits?); in practice, it's interpreted +-- as identical to precision 1 +%.0g 1000 -> 1e+03 +%.0g 100 -> 1e+02 +%.0g 10 -> 1e+01 +%.0g 1 -> 1 +%.0g 0.1 -> 0.1 +%.0g 0.01 -> 0.01 +%.0g 1e-3 -> 0.001 +%.0g 1e-4 -> 0.0001 +%.0g 1e-5 -> 1e-05 +%.0g 1e-6 -> 1e-06 +%.0g 12 -> 1e+01 +%.0g 120 -> 1e+02 +%.0g 1.2 -> 1 +%.0g 0.12 -> 0.1 +%.0g 0.012 -> 0.01 +%.0g 0.0012 -> 0.001 +%.0g 0.00012 -> 0.0001 +%.0g 0.000012 -> 1e-05 +%.0g 0.0000012 -> 1e-06 + +-- precision 1 identical to precision 0 +%.1g 1000 -> 1e+03 +%.1g 100 -> 1e+02 +%.1g 10 -> 1e+01 +%.1g 1 -> 1 +%.1g 0.1 -> 0.1 +%.1g 0.01 -> 0.01 +%.1g 1e-3 -> 0.001 +%.1g 1e-4 -> 0.0001 +%.1g 1e-5 -> 1e-05 +%.1g 1e-6 -> 1e-06 +%.1g 12 -> 1e+01 +%.1g 120 -> 1e+02 +%.1g 1.2 -> 1 +%.1g 0.12 -> 0.1 +%.1g 0.012 -> 0.01 +%.1g 0.0012 -> 0.001 +%.1g 0.00012 -> 0.0001 +%.1g 0.000012 -> 1e-05 +%.1g 0.0000012 -> 1e-06 + +-- precision 2 +%.2g 1000 -> 1e+03 +%.2g 100 -> 1e+02 +%.2g 10 -> 10 +%.2g 1 -> 1 +%.2g 0.1 -> 0.1 +%.2g 0.01 -> 0.01 +%.2g 0.001 -> 0.001 +%.2g 1e-4 -> 0.0001 +%.2g 1e-5 -> 1e-05 +%.2g 1e-6 -> 1e-06 +%.2g 1234 -> 1.2e+03 +%.2g 123 -> 1.2e+02 +%.2g 12.3 -> 12 +%.2g 1.23 -> 1.2 +%.2g 0.123 -> 0.12 +%.2g 0.0123 -> 0.012 +%.2g 0.00123 -> 0.0012 +%.2g 0.000123 -> 0.00012 +%.2g 0.0000123 -> 1.2e-05 + +-- bad cases from http://bugs.python.org/issue9980 +%.12g 38210.0 -> 38210 +%.12g 37210.0 -> 37210 +%.12g 36210.0 -> 36210 + +-- alternate g formatting: always include decimal point and +-- exactly significant digits. +%#.0g 0 -> 0. +%#.1g 0 -> 0. +%#.2g 0 -> 0.0 +%#.3g 0 -> 0.00 +%#.4g 0 -> 0.000 + +%#.0g 0.2 -> 0.2 +%#.1g 0.2 -> 0.2 +%#.2g 0.2 -> 0.20 +%#.3g 0.2 -> 0.200 +%#.4g 0.2 -> 0.2000 +%#.10g 0.2 -> 0.2000000000 + +%#.0g 2 -> 2. +%#.1g 2 -> 2. +%#.2g 2 -> 2.0 +%#.3g 2 -> 2.00 +%#.4g 2 -> 2.000 + +%#.0g 20 -> 2.e+01 +%#.1g 20 -> 2.e+01 +%#.2g 20 -> 20. +%#.3g 20 -> 20.0 +%#.4g 20 -> 20.00 + +%#.0g 234.56 -> 2.e+02 +%#.1g 234.56 -> 2.e+02 +%#.2g 234.56 -> 2.3e+02 +%#.3g 234.56 -> 235. +%#.4g 234.56 -> 234.6 +%#.5g 234.56 -> 234.56 +%#.6g 234.56 -> 234.560 + +-- repr formatting. Result always includes decimal point and at +-- least one digit after the point, or an exponent. +%r 0 -> 0.0 +%r 1 -> 1.0 + +%r 0.01 -> 0.01 +%r 0.02 -> 0.02 +%r 0.03 -> 0.03 +%r 0.04 -> 0.04 +%r 0.05 -> 0.05 + +-- values >= 1e16 get an exponent +%r 10 -> 10.0 +%r 100 -> 100.0 +%r 1e15 -> 1000000000000000.0 +%r 9.999e15 -> 9999000000000000.0 +%r 9999999999999998 -> 9999999999999998.0 +%r 9999999999999999 -> 1e+16 +%r 1e16 -> 1e+16 +%r 1e17 -> 1e+17 + +-- as do values < 1e-4 +%r 1e-3 -> 0.001 +%r 1.001e-4 -> 0.0001001 +%r 1.0000000000000001e-4 -> 0.0001 +%r 1.000000000000001e-4 -> 0.0001000000000000001 +%r 1.00000000001e-4 -> 0.000100000000001 +%r 1.0000000001e-4 -> 0.00010000000001 +%r 1e-4 -> 0.0001 +%r 0.99999999999999999e-4 -> 0.0001 +%r 0.9999999999999999e-4 -> 9.999999999999999e-05 +%r 0.999999999999e-4 -> 9.99999999999e-05 +%r 0.999e-4 -> 9.99e-05 +%r 1e-5 -> 1e-05 diff --git a/Lib/test/mathdata/ieee754.txt b/Lib/test/mathdata/ieee754.txt new file mode 100644 index 00000000000..9be667826a6 --- /dev/null +++ b/Lib/test/mathdata/ieee754.txt @@ -0,0 +1,183 @@ +====================================== +Python IEEE 754 floating point support +====================================== + +>>> from sys import float_info as FI +>>> from math import * +>>> PI = pi +>>> E = e + +You must never compare two floats with == because you are not going to get +what you expect. We treat two floats as equal if the difference between them +is small than epsilon. +>>> EPS = 1E-15 +>>> def equal(x, y): +... """Almost equal helper for floats""" +... return abs(x - y) < EPS + + +NaNs and INFs +============= + +In Python 2.6 and newer NaNs (not a number) and infinity can be constructed +from the strings 'inf' and 'nan'. + +>>> INF = float('inf') +>>> NINF = float('-inf') +>>> NAN = float('nan') + +>>> INF +inf +>>> NINF +-inf +>>> NAN +nan + +The math module's ``isnan`` and ``isinf`` functions can be used to detect INF +and NAN: +>>> isinf(INF), isinf(NINF), isnan(NAN) +(True, True, True) +>>> INF == -NINF +True + +Infinity +-------- + +Ambiguous operations like ``0 * inf`` or ``inf - inf`` result in NaN. +>>> INF * 0 +nan +>>> INF - INF +nan +>>> INF / INF +nan + +However unambiguous operations with inf return inf: +>>> INF * INF +inf +>>> 1.5 * INF +inf +>>> 0.5 * INF +inf +>>> INF / 1000 +inf + +Not a Number +------------ + +NaNs are never equal to another number, even itself +>>> NAN == NAN +False +>>> NAN < 0 +False +>>> NAN >= 0 +False + +All operations involving a NaN return a NaN except for nan**0 and 1**nan. +>>> 1 + NAN +nan +>>> 1 * NAN +nan +>>> 0 * NAN +nan +>>> 1 ** NAN +1.0 +>>> NAN ** 0 +1.0 +>>> 0 ** NAN +nan +>>> (1.0 + FI.epsilon) * NAN +nan + +Misc Functions +============== + +The power of 1 raised to x is always 1.0, even for special values like 0, +infinity and NaN. + +>>> pow(1, 0) +1.0 +>>> pow(1, INF) +1.0 +>>> pow(1, -INF) +1.0 +>>> pow(1, NAN) +1.0 + +The power of 0 raised to x is defined as 0, if x is positive. Negative +finite values are a domain error or zero division error and NaN result in a +silent NaN. + +>>> pow(0, 0) +1.0 +>>> pow(0, INF) +0.0 +>>> pow(0, -INF) +inf +>>> 0 ** -1 +Traceback (most recent call last): +... +ZeroDivisionError: zero to a negative power +>>> pow(0, NAN) +nan + + +Trigonometric Functions +======================= + +>>> sin(INF) +Traceback (most recent call last): +... +ValueError: expected a finite input, got inf +>>> sin(NINF) +Traceback (most recent call last): +... +ValueError: expected a finite input, got -inf +>>> sin(NAN) +nan +>>> cos(INF) +Traceback (most recent call last): +... +ValueError: expected a finite input, got inf +>>> cos(NINF) +Traceback (most recent call last): +... +ValueError: expected a finite input, got -inf +>>> cos(NAN) +nan +>>> tan(INF) +Traceback (most recent call last): +... +ValueError: expected a finite input, got inf +>>> tan(NINF) +Traceback (most recent call last): +... +ValueError: expected a finite input, got -inf +>>> tan(NAN) +nan + +Neither pi nor tan are exact, but you can assume that tan(pi/2) is a large value +and tan(pi) is a very small value: +>>> tan(PI/2) > 1E10 +True +>>> -tan(-PI/2) > 1E10 +True +>>> tan(PI) < 1E-15 +True + +>>> asin(NAN), acos(NAN), atan(NAN) +(nan, nan, nan) +>>> asin(INF), asin(NINF) +Traceback (most recent call last): +... +ValueError: expected a number in range from -1 up to 1, got inf +>>> acos(INF), acos(NINF) +Traceback (most recent call last): +... +ValueError: expected a number in range from -1 up to 1, got inf +>>> equal(atan(INF), PI/2), equal(atan(NINF), -PI/2) +(True, True) + + +Hyberbolic Functions +==================== + diff --git a/Lib/test/math_testcases.txt b/Lib/test/mathdata/math_testcases.txt similarity index 100% rename from Lib/test/math_testcases.txt rename to Lib/test/mathdata/math_testcases.txt diff --git a/Lib/test/mod_generics_cache.py b/Lib/test/mod_generics_cache.py deleted file mode 100644 index 6d35c58396d..00000000000 --- a/Lib/test/mod_generics_cache.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Module for testing the behavior of generics across different modules.""" - -import sys -from textwrap import dedent -from typing import TypeVar, Generic, Optional - - -if sys.version_info[:2] >= (3, 6): - exec(dedent(""" - default_a: Optional['A'] = None - default_b: Optional['B'] = None - - T = TypeVar('T') - - - class A(Generic[T]): - some_b: 'B' - - - class B(Generic[T]): - class A(Generic[T]): - pass - - my_inner_a1: 'B.A' - my_inner_a2: A - my_outer_a: 'A' # unless somebody calls get_type_hints with localns=B.__dict__ - """)) -else: # This should stay in sync with the syntax above. - __annotations__ = dict( - default_a=Optional['A'], - default_b=Optional['B'], - ) - default_a = None - default_b = None - - T = TypeVar('T') - - - class A(Generic[T]): - __annotations__ = dict( - some_b='B' - ) - - - class B(Generic[T]): - class A(Generic[T]): - pass - - __annotations__ = dict( - my_inner_a1='B.A', - my_inner_a2=A, - my_outer_a='A' # unless somebody calls get_type_hints with localns=B.__dict__ - ) diff --git a/Lib/test/mp_fork_bomb.py b/Lib/test/mp_fork_bomb.py new file mode 100644 index 00000000000..017e010ba0e --- /dev/null +++ b/Lib/test/mp_fork_bomb.py @@ -0,0 +1,18 @@ +import multiprocessing, sys + +def foo(): + print("123") + +# Because "if __name__ == '__main__'" is missing this will not work +# correctly on Windows. However, we should get a RuntimeError rather +# than the Windows equivalent of a fork bomb. + +if len(sys.argv) > 1: + multiprocessing.set_start_method(sys.argv[1]) +else: + multiprocessing.set_start_method('spawn') + +p = multiprocessing.Process(target=foo) +p.start() +p.join() +sys.exit(p.exitcode) diff --git a/Lib/test/mp_preload.py b/Lib/test/mp_preload.py new file mode 100644 index 00000000000..5314e8f0b21 --- /dev/null +++ b/Lib/test/mp_preload.py @@ -0,0 +1,18 @@ +import multiprocessing + +multiprocessing.Lock() + + +def f(): + print("ok") + + +if __name__ == "__main__": + ctx = multiprocessing.get_context("forkserver") + modname = "test.mp_preload" + # Make sure it's importable + __import__(modname) + ctx.set_forkserver_preload([modname]) + proc = ctx.Process(target=f) + proc.start() + proc.join() diff --git a/Lib/test/mp_preload_flush.py b/Lib/test/mp_preload_flush.py new file mode 100644 index 00000000000..c195a9ef6b2 --- /dev/null +++ b/Lib/test/mp_preload_flush.py @@ -0,0 +1,11 @@ +import multiprocessing +import sys + +print(__name__, end='', file=sys.stderr) +print(__name__, end='', file=sys.stdout) +if __name__ == '__main__': + multiprocessing.set_start_method('forkserver') + for _ in range(2): + p = multiprocessing.Process() + p.start() + p.join() diff --git a/Lib/test/mp_preload_main.py b/Lib/test/mp_preload_main.py new file mode 100644 index 00000000000..acb342822ec --- /dev/null +++ b/Lib/test/mp_preload_main.py @@ -0,0 +1,14 @@ +import multiprocessing + +print(f"{__name__}") + +def f(): + print("f") + +if __name__ == "__main__": + ctx = multiprocessing.get_context("forkserver") + ctx.set_forkserver_preload(['__main__']) + for _ in range(2): + p = ctx.Process(target=f) + p.start() + p.join() diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py new file mode 100644 index 00000000000..6b4c57d0b4b --- /dev/null +++ b/Lib/test/multibytecodec_support.py @@ -0,0 +1,400 @@ +# +# multibytecodec_support.py +# Common Unittest Routines for CJK codecs +# + +import codecs +import os +import re +import sys +import unittest +from http.client import HTTPException +from test import support +from io import BytesIO + +class TestBase: + encoding = '' # codec name + codec = None # codec tuple (with 4 elements) + tstring = None # must set. 2 strings to test StreamReader + + codectests = None # must set. codec test tuple + roundtriptest = 1 # set if roundtrip is possible with unicode + has_iso10646 = 0 # set if this encoding contains whole iso10646 map + xmlcharnametest = None # string to test xmlcharrefreplace + unmappedunicode = '\udeee' # a unicode code point that is not mapped. + + def setUp(self): + if self.codec is None: + self.codec = codecs.lookup(self.encoding) + self.encode = self.codec.encode + self.decode = self.codec.decode + self.reader = self.codec.streamreader + self.writer = self.codec.streamwriter + self.incrementalencoder = self.codec.incrementalencoder + self.incrementaldecoder = self.codec.incrementaldecoder + + def test_chunkcoding(self): + tstring_lines = [] + for b in self.tstring: + lines = b.split(b"\n") + last = lines.pop() + assert last == b"" + lines = [line + b"\n" for line in lines] + tstring_lines.append(lines) + for native, utf8 in zip(*tstring_lines): + u = self.decode(native)[0] + self.assertEqual(u, utf8.decode('utf-8')) + if self.roundtriptest: + self.assertEqual(native, self.encode(u)[0]) + + def test_errorhandle(self): + for source, scheme, expected in self.codectests: + if isinstance(source, bytes): + func = self.decode + else: + func = self.encode + if expected: + result = func(source, scheme)[0] + if func is self.decode: + self.assertTrue(type(result) is str, type(result)) + self.assertEqual(result, expected, + '%a.decode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertTrue(type(result) is bytes, type(result)) + self.assertEqual(result, expected, + '%a.encode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertRaises(UnicodeError, func, source, scheme) + + def test_xmlcharrefreplace(self): + if self.has_iso10646: + self.skipTest('encoding contains full ISO 10646 map') + + s = "\u0b13\u0b23\u0b60 nd eggs" + self.assertEqual( + self.encode(s, "xmlcharrefreplace")[0], + b"ଓଣୠ nd eggs" + ) + + def test_customreplace_encode(self): + if self.has_iso10646: + self.skipTest('encoding contains full ISO 10646 map') + + from html.entities import codepoint2name + + def xmlcharnamereplace(exc): + if not isinstance(exc, UnicodeEncodeError): + raise TypeError("don't know how to handle %r" % exc) + l = [] + for c in exc.object[exc.start:exc.end]: + if ord(c) in codepoint2name: + l.append("&%s;" % codepoint2name[ord(c)]) + else: + l.append("&#%d;" % ord(c)) + return ("".join(l), exc.end) + + codecs.register_error("test.xmlcharnamereplace", xmlcharnamereplace) + + if self.xmlcharnametest: + sin, sout = self.xmlcharnametest + else: + sin = "\xab\u211c\xbb = \u2329\u1234\u232a" + sout = b"«ℜ» = ⟨ሴ⟩" + self.assertEqual(self.encode(sin, + "test.xmlcharnamereplace")[0], sout) + + def test_callback_returns_bytes(self): + def myreplace(exc): + return (b"1234", exc.end) + codecs.register_error("test.cjktest", myreplace) + enc = self.encode("abc" + self.unmappedunicode + "def", "test.cjktest")[0] + self.assertEqual(enc, b"abc1234def") + + def test_callback_wrong_objects(self): + def myreplace(exc): + return (ret, exc.end) + codecs.register_error("test.cjktest", myreplace) + + for ret in ([1, 2, 3], [], None, object()): + self.assertRaises(TypeError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_long_index(self): + def myreplace(exc): + return ('x', int(exc.end)) + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), (b'abcdxefgh', 9)) + + def myreplace(exc): + return ('x', sys.maxsize + 1) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(IndexError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_None_index(self): + def myreplace(exc): + return ('x', None) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(TypeError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_callback_backward_index(self): + def myreplace(exc): + if myreplace.limit > 0: + myreplace.limit -= 1 + return ('REPLACED', 0) + else: + return ('TERMINAL', exc.end) + myreplace.limit = 3 + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), + (b'abcdREPLACEDabcdREPLACEDabcdREPLACEDabcdTERMINALefgh', 9)) + + def test_callback_forward_index(self): + def myreplace(exc): + return ('REPLACED', exc.end + 2) + codecs.register_error("test.cjktest", myreplace) + self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh', + 'test.cjktest'), (b'abcdREPLACEDgh', 9)) + + def test_callback_index_outofbound(self): + def myreplace(exc): + return ('TERM', 100) + codecs.register_error("test.cjktest", myreplace) + self.assertRaises(IndexError, self.encode, self.unmappedunicode, + 'test.cjktest') + + def test_incrementalencoder(self): + UTF8Reader = codecs.getreader('utf-8') + for sizehint in [None] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = UTF8Reader(BytesIO(self.tstring[1])) + ostream = BytesIO() + encoder = self.incrementalencoder() + while 1: + if sizehint is not None: + data = istream.read(sizehint) + else: + data = istream.read() + + if not data: + break + e = encoder.encode(data) + ostream.write(e) + + self.assertEqual(ostream.getvalue(), self.tstring[0]) + + def test_incrementaldecoder(self): + UTF8Writer = codecs.getwriter('utf-8') + for sizehint in [None, -1] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = BytesIO(self.tstring[0]) + ostream = UTF8Writer(BytesIO()) + decoder = self.incrementaldecoder() + while 1: + data = istream.read(sizehint) + if not data: + break + else: + u = decoder.decode(data) + ostream.write(u) + + self.assertEqual(ostream.getvalue(), self.tstring[1]) + + def test_incrementalencoder_error_callback(self): + inv = self.unmappedunicode + + e = self.incrementalencoder() + self.assertRaises(UnicodeEncodeError, e.encode, inv, True) + + e.errors = 'ignore' + self.assertEqual(e.encode(inv, True), b'') + + e.reset() + def tempreplace(exc): + return ('called', exc.end) + codecs.register_error('test.incremental_error_callback', tempreplace) + e.errors = 'test.incremental_error_callback' + self.assertEqual(e.encode(inv, True), b'called') + + # again + e.errors = 'ignore' + self.assertEqual(e.encode(inv, True), b'') + + def test_streamreader(self): + UTF8Writer = codecs.getwriter('utf-8') + for name in ["read", "readline", "readlines"]: + for sizehint in [None, -1] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = self.reader(BytesIO(self.tstring[0])) + ostream = UTF8Writer(BytesIO()) + func = getattr(istream, name) + while 1: + data = func(sizehint) + if not data: + break + if name == "readlines": + ostream.writelines(data) + else: + ostream.write(data) + + self.assertEqual(ostream.getvalue(), self.tstring[1]) + + def test_streamwriter(self): + readfuncs = ('read', 'readline', 'readlines') + UTF8Reader = codecs.getreader('utf-8') + for name in readfuncs: + for sizehint in [None] + list(range(1, 33)) + \ + [64, 128, 256, 512, 1024]: + istream = UTF8Reader(BytesIO(self.tstring[1])) + ostream = self.writer(BytesIO()) + func = getattr(istream, name) + while 1: + if sizehint is not None: + data = func(sizehint) + else: + data = func() + + if not data: + break + if name == "readlines": + ostream.writelines(data) + else: + ostream.write(data) + + self.assertEqual(ostream.getvalue(), self.tstring[0]) + + def test_streamwriter_reset_no_pending(self): + # Issue #23247: Calling reset() on a fresh StreamWriter instance + # (without pending data) must not crash + stream = BytesIO() + writer = self.writer(stream) + writer.reset() + + def test_incrementalencoder_del_segfault(self): + e = self.incrementalencoder() + with self.assertRaises(AttributeError): + del e.errors + + def test_null_terminator(self): + # see gh-101828 + text = "フルーツ" + try: + text.encode(self.encoding) + except UnicodeEncodeError: + text = "Python is cool" + encode_w_null = (text + "\0").encode(self.encoding) + encode_plus_null = text.encode(self.encoding) + "\0".encode(self.encoding) + self.assertTrue(encode_w_null.endswith(b'\x00')) + self.assertEqual(encode_w_null, encode_plus_null) + + encode_w_null_2 = (text + "\0" + text + "\0").encode(self.encoding) + encode_plus_null_2 = encode_plus_null + encode_plus_null + self.assertEqual(encode_w_null_2.count(b'\x00'), 2) + self.assertEqual(encode_w_null_2, encode_plus_null_2) + + +class TestBase_Mapping(unittest.TestCase): + pass_enctest = [] + pass_dectest = [] + supmaps = [] + codectests = [] + + def setUp(self): + try: + self.open_mapping_file().close() # test it to report the error early + except (OSError, HTTPException): + self.skipTest("Could not retrieve "+self.mapfileurl) + + def open_mapping_file(self): + return support.open_urlresource(self.mapfileurl, encoding="utf-8") + + def test_mapping_file(self): + if self.mapfileurl.endswith('.xml'): + self._test_mapping_file_ucm() + else: + self._test_mapping_file_plain() + + def _test_mapping_file_plain(self): + def unichrs(s): + return ''.join(chr(int(x, 16)) for x in s.split('+')) + + urt_wa = {} + + with self.open_mapping_file() as f: + for line in f: + if not line: + break + data = line.split('#')[0].split() + if len(data) != 2: + continue + + if data[0][:2] != '0x': + self.fail(f"Invalid line: {line!r}") + csetch = bytes.fromhex(data[0][2:]) + if len(csetch) == 1 and 0x80 <= csetch[0]: + continue + + unich = unichrs(data[1]) + if ord(unich) == 0xfffd or unich in urt_wa: + continue + urt_wa[unich] = csetch + + self._testpoint(csetch, unich) + + def _test_mapping_file_ucm(self): + with self.open_mapping_file() as f: + ucmdata = f.read() + uc = re.findall('', ucmdata) + for uni, coded in uc: + unich = chr(int(uni, 16)) + codech = bytes.fromhex(coded) + self._testpoint(codech, unich) + + def test_mapping_supplemental(self): + for mapping in self.supmaps: + self._testpoint(*mapping) + + def _testpoint(self, csetch, unich): + if (csetch, unich) not in self.pass_enctest: + self.assertEqual(unich.encode(self.encoding), csetch) + if (csetch, unich) not in self.pass_dectest: + self.assertEqual(str(csetch, self.encoding), unich) + + def test_errorhandle(self): + for source, scheme, expected in self.codectests: + if isinstance(source, bytes): + func = source.decode + else: + func = source.encode + if expected: + if isinstance(source, bytes): + result = func(self.encoding, scheme) + self.assertTrue(type(result) is str, type(result)) + self.assertEqual(result, expected, + '%a.decode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + result = func(self.encoding, scheme) + self.assertTrue(type(result) is bytes, type(result)) + self.assertEqual(result, expected, + '%a.encode(%r, %r)=%a != %a' + % (source, self.encoding, scheme, result, + expected)) + else: + self.assertRaises(UnicodeError, func, self.encoding, scheme) + +def load_teststring(name): + dir = os.path.join(os.path.dirname(__file__), 'cjkencodings') + with open(os.path.join(dir, name + '.txt'), 'rb') as f: + encoded = f.read() + with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f: + utf8 = f.read() + return encoded, utf8 diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 8a4de7a4fdf..9a3a26a8400 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1,3 +1,4 @@ +import builtins import collections import copyreg import dbm @@ -11,6 +12,7 @@ import struct import sys import threading +import types import unittest import weakref from textwrap import dedent @@ -24,7 +26,7 @@ from test import support from test.support import os_helper from test.support import ( - TestFailed, run_with_locale, no_tracing, + TestFailed, run_with_locales, no_tracing, _2G, _4G, bigmemtest ) from test.support.import_helper import forget @@ -142,6 +144,14 @@ class E(C): def __getinitargs__(self): return () +import __main__ +__main__.C = C +C.__module__ = "__main__" +__main__.D = D +D.__module__ = "__main__" +__main__.E = E +E.__module__ = "__main__" + # Simple mutable object. class Object: pass @@ -155,14 +165,6 @@ def __reduce__(self): # Shouldn't support the recursion itself return K, (self.value,) -import __main__ -__main__.C = C -C.__module__ = "__main__" -__main__.D = D -D.__module__ = "__main__" -__main__.E = E -E.__module__ = "__main__" - class myint(int): def __init__(self, x): self.str = str(x) @@ -1010,6 +1012,26 @@ def test_constants(self): self.assertIs(self.loads(b'I01\n.'), True) self.assertIs(self.loads(b'I00\n.'), False) + def test_zero_padded_integers(self): + self.assertEqual(self.loads(b'I010\n.'), 10) + self.assertEqual(self.loads(b'I-010\n.'), -10) + self.assertEqual(self.loads(b'I0010\n.'), 10) + self.assertEqual(self.loads(b'I-0010\n.'), -10) + self.assertEqual(self.loads(b'L010\n.'), 10) + self.assertEqual(self.loads(b'L-010\n.'), -10) + self.assertEqual(self.loads(b'L0010\n.'), 10) + self.assertEqual(self.loads(b'L-0010\n.'), -10) + self.assertEqual(self.loads(b'L010L\n.'), 10) + self.assertEqual(self.loads(b'L-010L\n.'), -10) + + def test_nondecimal_integers(self): + self.assertRaises(ValueError, self.loads, b'I0b10\n.') + self.assertRaises(ValueError, self.loads, b'I0o10\n.') + self.assertRaises(ValueError, self.loads, b'I0x10\n.') + self.assertRaises(ValueError, self.loads, b'L0b10L\n.') + self.assertRaises(ValueError, self.loads, b'L0o10L\n.') + self.assertRaises(ValueError, self.loads, b'L0x10L\n.') + def test_empty_bytestring(self): # issue 11286 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') @@ -1078,6 +1100,11 @@ def test_large_32b_binunicode8(self): self.check_unpickling_error((pickle.UnpicklingError, OverflowError), dumped) + def test_large_binstring(self): + errmsg = 'BINSTRING pickle has negative byte count' + with self.assertRaisesRegex(pickle.UnpicklingError, errmsg): + self.loads(b'T\0\0\0\x80') + def test_get(self): pickled = b'((lp100000\ng100000\nt.' unpickled = self.loads(pickled) @@ -1177,6 +1204,166 @@ def test_compat_unpickle(self): self.assertIs(type(unpickled), collections.UserDict) self.assertEqual(unpickled, collections.UserDict({1: 2})) + def test_load_global(self): + self.assertIs(self.loads(b'cbuiltins\nstr\n.'), str) + self.assertIs(self.loads(b'cmath\nlog\n.'), math.log) + self.assertIs(self.loads(b'cos.path\njoin\n.'), os.path.join) + self.assertIs(self.loads(b'\x80\x04cbuiltins\nstr.upper\n.'), str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04cm\xc3\xb6dule\ngl\xc3\xb6bal\n.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'c\xff\nlog\n.') + self.assertRaises(UnicodeDecodeError, self.loads, b'cmath\n\xff\n.') + self.assertRaises(self.truncated_errors, self.loads, b'c\nlog\n.') + self.assertRaises(self.truncated_errors, self.loads, b'cmath\n\n.') + self.assertRaises(self.truncated_errors, self.loads, b'\x80\x04cmath\n\n.') + + def test_load_stack_global(self): + self.assertIs(self.loads(b'\x8c\x08builtins\x8c\x03str\x93.'), str) + self.assertIs(self.loads(b'\x8c\x04math\x8c\x03log\x93.'), math.log) + self.assertIs(self.loads(b'\x8c\x07os.path\x8c\x04join\x93.'), + os.path.join) + self.assertIs(self.loads(b'\x80\x04\x8c\x08builtins\x8c\x09str.upper\x93.'), + str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04\x8c\x07m\xc3\xb6dule\x8c\x07gl\xc3\xb6bal\x93.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x01\xff\x8c\x03log\x93.') + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x04math\x8c\x01\xff\x93.') + self.assertRaises(ValueError, self.loads, b'\x8c\x00\x8c\x03log\x93.') + self.assertRaises(AttributeError, self.loads, b'\x8c\x04math\x8c\x00\x93.') + self.assertRaises(AttributeError, self.loads, b'\x80\x04\x8c\x04math\x8c\x00\x93.') + + self.assertRaises(pickle.UnpicklingError, self.loads, b'N\x8c\x03log\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x8c\x04mathN\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x80\x04\x8c\x04mathN\x93.') + + def test_find_class(self): + unpickler = self.unpickler(io.BytesIO()) + unpickler_nofix = self.unpickler(io.BytesIO(), fix_imports=False) + unpickler4 = self.unpickler(io.BytesIO(b'\x80\x04N.')) + unpickler4.load() + + self.assertIs(unpickler.find_class('__builtin__', 'str'), str) + self.assertRaises(ModuleNotFoundError, + unpickler_nofix.find_class, '__builtin__', 'str') + self.assertIs(unpickler.find_class('builtins', 'str'), str) + self.assertIs(unpickler_nofix.find_class('builtins', 'str'), str) + self.assertIs(unpickler.find_class('math', 'log'), math.log) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + + self.assertIs(unpickler4.find_class('builtins', 'str.upper'), str.upper) + with self.assertRaisesRegex(AttributeError, + r"module 'builtins' has no attribute 'str\.upper'"): + unpickler.find_class('builtins', 'str.upper') + + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): + unpickler.find_class('math', 'spam') + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute 'spam'"): + unpickler4.find_class('math', 'spam') + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.spam'"): + unpickler.find_class('math', 'log.spam') + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.spam' on module 'math'") as cm: + unpickler4.find_class('math', 'log.spam') + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute 'spam'") + with self.assertRaisesRegex(AttributeError, + r"module 'math' has no attribute 'log\.\.spam'"): + unpickler.find_class('math', 'log..spam') + with self.assertRaisesRegex(AttributeError, + r"Can't resolve path 'log\.\.spam' on module 'math'") as cm: + unpickler4.find_class('math', 'log..spam') + self.assertEqual(str(cm.exception.__context__), + "'builtin_function_or_method' object has no attribute ''") + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): + unpickler.find_class('math', '') + with self.assertRaisesRegex(AttributeError, + "module 'math' has no attribute ''"): + unpickler4.find_class('math', '') + self.assertRaises(ModuleNotFoundError, unpickler.find_class, 'spam', 'log') + self.assertRaises(ValueError, unpickler.find_class, '', 'log') + + self.assertRaises(TypeError, unpickler.find_class, None, 'log') + self.assertRaises(TypeError, unpickler.find_class, 'math', None) + self.assertRaises((TypeError, AttributeError), unpickler4.find_class, 'math', None) + + def test_custom_find_class(self): + def loads(data): + class Unpickler(self.unpickler): + def find_class(self, module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + @staticmethod + def find_class(module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + @classmethod + def find_class(cls, module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + pass + def find_class(module_name, global_name): + return (module_name, global_name) + unpickler = Unpickler(io.BytesIO(data)) + unpickler.find_class = find_class + return unpickler.load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def test_bad_ext_code(self): + # unregistered extension code + self.check_unpickling_error(ValueError, b'\x82\x01.') + self.check_unpickling_error(ValueError, b'\x82\xff.') + self.check_unpickling_error(ValueError, b'\x83\x01\x00.') + self.check_unpickling_error(ValueError, b'\x83\xff\xff.') + self.check_unpickling_error(ValueError, b'\x84\x01\x00\x00\x00.') + self.check_unpickling_error(ValueError, b'\x84\xff\xff\xff\x7f.') + # EXT specifies code <= 0 + self.check_unpickling_error(pickle.UnpicklingError, b'\x82\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x83\x00\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x80.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\xff\xff\xff\xff.') + + @support.cpython_only + def test_bad_ext_inverted_registry(self): + code = 1 + def check(key, exc): + with support.swap_item(copyreg._inverted_registry, code, key): + with self.assertRaises(exc): + self.loads(b'\x82\x01.') + check(None, ValueError) + check((), ValueError) + check((__name__,), (TypeError, ValueError)) + check((__name__, "MyList", "x"), (TypeError, ValueError)) + check((__name__, None), (TypeError, ValueError)) + check((None, "MyList"), (TypeError, ValueError)) + def test_bad_reduce(self): self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) self.check_unpickling_error(TypeError, b'N)R.') @@ -1195,6 +1382,41 @@ def test_bad_newobj_ex(self): self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.') self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.') + def test_bad_state(self): + c = C() + c.x = None + base = b'c__main__\nC\n)\x81' + self.assertEqual(self.loads(base + b'}X\x01\x00\x00\x00xNsb.'), c) + self.assertEqual(self.loads(base + b'N}X\x01\x00\x00\x00xNs\x86b.'), c) + # non-hashable dict key + self.check_unpickling_error(TypeError, base + b'}]Nsb.') + # state = list + error = (pickle.UnpicklingError, AttributeError) + self.check_unpickling_error(error, base + b'](}}eb.') + # state = 1-tuple + self.check_unpickling_error(error, base + b'}\x85b.') + # state = 3-tuple + self.check_unpickling_error(error, base + b'}}}\x87b.') + # non-hashable slot name + self.check_unpickling_error(TypeError, base + b'}}]Ns\x86b.') + # non-string slot name + self.check_unpickling_error(TypeError, base + b'}}NNs\x86b.') + # dict = True + self.check_unpickling_error(error, base + b'\x88}\x86b.') + # slots dict = True + self.check_unpickling_error(error, base + b'}\x88\x86b.') + + class BadKey1: + count = 1 + def __hash__(self): + if not self.count: + raise CustomError + self.count -= 1 + return 42 + __main__.BadKey1 = BadKey1 + # bad hashable dict key + self.check_unpickling_error(CustomError, base + b'}c__main__\nBadKey1\n)\x81Nsb.') + def test_bad_stack(self): badpickles = [ b'.', # STOP @@ -1380,6 +1602,7 @@ def test_truncated_data(self): self.check_unpickling_error(self.truncated_errors, p) @threading_helper.reap_threads + @threading_helper.requires_working_threading() def test_unpickle_module_race(self): # https://bugs.python.org/issue34572 locker_module = dedent(""" @@ -1440,6 +1663,798 @@ def t(): [ToBeUnpickled] * 2) +class AbstractPicklingErrorTests: + # Subclass must define self.dumps, self.pickler. + + def test_bad_reduce_result(self): + obj = REX([print, ()]) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + '__reduce__ must return a string or tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((print,)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((print, (), None, None, None, None, None)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'tuple returned by __reduce__ must contain 2 through 6 elements') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_reconstructor(self): + obj = REX((42, ())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'first item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_reconstructor(self): + obj = REX((UnpickleableCallable(), ())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + + def test_bad_reconstructor_args(self): + obj = REX((print, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_reconstructor_args(self): + obj = REX((print, (1, 2, UNPICKLEABLE))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_args(self): + obj = REX((copyreg.__newobj__, ())) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((IndexError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'tuple index out of range', + '__newobj__ expected at least 1 argument, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj__, [REX])) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_class(self): + obj = REX((copyreg.__newobj__, (NoNew(),))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj__() has no __new__', + f'first argument to __newobj__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_wrong_newobj_class(self): + obj = REX((copyreg.__newobj__, (str,))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj__() must be {REX!r}, not {str!r}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_class(self): + class LocalREX(REX): pass + obj = LocalREX((copyreg.__newobj__, (LocalREX,))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + + def test_unpickleable_newobj_args(self): + obj = REX((copyreg.__newobj__, (REX, 1, 2, UNPICKLEABLE))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_ex_args(self): + obj = REX((copyreg.__newobj_ex__, ())) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((ValueError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 3, got 0)', + '__newobj_ex__ expected 3 arguments, got 0'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, 42)) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second item of the tuple returned by __reduce__ ' + 'must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, (REX, 42, {}))) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'Value after * must be an iterable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'second argument to __newobj_ex__() must be a tuple, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + obj = REX((copyreg.__newobj_ex__, (REX, (), []))) + if self.pickler is pickle._Pickler: + for proto in protocols[2:4]: + with self.subTest(proto=proto): + with self.assertRaises(TypeError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'functools.partial() argument after ** must be a mapping, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + else: + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'third argument to __newobj_ex__() must be a dict, not list') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_bad_newobj_ex__class(self): + obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {}))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'first argument to __newobj_ex__() has no __new__', + f'first argument to __newobj_ex__() must be a class, not {__name__}.NoNew'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_wrong_newobj_ex_class(self): + if self.pickler is not pickle._Pickler: + self.skipTest('only verified in the Python implementation') + obj = REX((copyreg.__newobj_ex__, (str, (), {}))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f'first argument to __newobj_ex__() must be {REX}, not {str}') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_ex_class(self): + class LocalREX(REX): pass + obj = LocalREX((copyreg.__newobj_ex__, (LocalREX, (), {}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} class', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 0', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} reconstructor arguments', + f'when serializing {LocalREX.__module__}.{LocalREX.__qualname__} object']) + + def test_unpickleable_newobj_ex_args(self): + obj = REX((copyreg.__newobj_ex__, (REX, (1, 2, UNPICKLEABLE), {}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 3', + 'when serializing tuple item 1', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_newobj_ex_kwargs(self): + obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing test.pickletester.REX __new__ arguments', + 'when serializing test.pickletester.REX object']) + elif proto >= 2: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing functools.partial state', + 'when serializing functools.partial object', + 'when serializing test.pickletester.REX reconstructor', + 'when serializing test.pickletester.REX object']) + else: + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'", + 'when serializing tuple item 2', + 'when serializing test.pickletester.REX reconstructor arguments', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state(self): + obj = REX_state(UNPICKLEABLE) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_state state', + 'when serializing test.pickletester.REX_state object']) + + def test_bad_state_setter(self): + if self.pickler is pickle._Pickler: + self.skipTest('only verified in the C implementation') + obj = REX((print, (), 'state', None, None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'sixth item of the tuple returned by __reduce__ ' + 'must be callable, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state_setter(self): + obj = REX((print, (), 'state', None, None, UnpickleableCallable())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state setter', + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_state_with_state_setter(self): + obj = REX((print, (), UNPICKLEABLE, None, None, print)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX state', + 'when serializing test.pickletester.REX object']) + + def test_bad_object_list_items(self): + # Issue4176: crash when 4th and 5th items of __reduce__() + # are not iterators + obj = REX((list, (), None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + if self.pickler is not pickle._Pickler: + # Python implementation is less strict and also accepts iterables. + obj = REX((list, (), None, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'fourth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_object_list_items(self): + obj = REX_six([1, 2, UNPICKLEABLE]) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX_six item 2', + 'when serializing test.pickletester.REX_six object']) + + def test_bad_object_dict_items(self): + # Issue4176: crash when 4th and 5th items of __reduce__() + # are not iterators + obj = REX((dict, (), None, None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + "'int' object is not iterable", + 'fifth item of the tuple returned by __reduce__ ' + 'must be an iterator, not int'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + for proto in protocols: + obj = REX((dict, (), None, None, iter([('a',)]))) + with self.subTest(proto=proto): + with self.assertRaises((ValueError, TypeError)) as cm: + self.dumps(obj, proto) + self.assertIn(str(cm.exception), { + 'not enough values to unpack (expected 2, got 1)', + 'dict items iterator must return 2-tuples'}) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + if self.pickler is not pickle._Pickler: + # Python implementation is less strict and also accepts iterables. + obj = REX((dict, (), None, None, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + 'dict items iterator must return 2-tuples') + self.assertEqual(cm.exception.__notes__, [ + 'when serializing test.pickletester.REX object']) + + def test_unpickleable_object_dict_items(self): + obj = REX_seven({'a': UNPICKLEABLE}) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing test.pickletester.REX_seven item 'a'", + 'when serializing test.pickletester.REX_seven object']) + + def test_unpickleable_list_items(self): + obj = [1, [2, 3, UNPICKLEABLE]] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 2', + 'when serializing list item 1']) + for n in [0, 1, 1000, 1005]: + obj = [*range(n), UNPICKLEABLE] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + f'when serializing list item {n}']) + + def test_unpickleable_tuple_items(self): + obj = (1, (2, 3, UNPICKLEABLE)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 2', + 'when serializing tuple item 1']) + obj = (*range(10), UNPICKLEABLE) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + 'when serializing tuple item 10']) + + def test_unpickleable_dict_items(self): + obj = {'a': {'b': UNPICKLEABLE}} + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'b'", + "when serializing dict item 'a'"]) + for n in [0, 1, 1000, 1005]: + obj = dict.fromkeys(range(n)) + obj['a'] = UNPICKLEABLE + for proto in protocols: + with self.subTest(proto=proto, n=n): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + self.assertEqual(cm.exception.__notes__, [ + "when serializing dict item 'a'"]) + + def test_unpickleable_set_items(self): + obj = {UNPICKLEABLE} + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing set element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing set reconstructor arguments']) + + def test_unpickleable_frozenset_items(self): + obj = frozenset({frozenset({UNPICKLEABLE})}) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError) as cm: + self.dumps(obj, proto) + if proto >= 4: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing frozenset element', + 'when serializing frozenset element']) + else: + self.assertEqual(cm.exception.__notes__, [ + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments', + 'when serializing list item 0', + 'when serializing tuple item 0', + 'when serializing frozenset reconstructor arguments']) + + def test_global_lookup_error(self): + # Global name does not exist + obj = REX('spam') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as {__name__}.spam") + self.assertEqual(str(cm.exception.__context__), + f"module '{__name__}' has no attribute 'spam'") + + obj.__module__ = 'nonexisting' + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: No module named 'nonexisting'") + self.assertEqual(str(cm.exception.__context__), + "No module named 'nonexisting'") + + obj.__module__ = '' + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: Empty module name") + self.assertEqual(str(cm.exception.__context__), + "Empty module name") + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: it's not found as __main__.spam") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'spam'") + + def test_nonencodable_global_name_error(self): + for proto in protocols[:4]: + with self.subTest(proto=proto): + name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff' + obj = REX(name) + obj.__module__ = __name__ + with support.swap_item(globals(), name, obj): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle global identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) + + def test_nonencodable_module_name_error(self): + for proto in protocols[:4]: + with self.subTest(proto=proto): + name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff' + obj = REX('test') + obj.__module__ = name + mod = types.SimpleNamespace(test=obj) + with support.swap_item(sys.modules, name, mod): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"can't pickle module identifier {name!r} using pickle protocol {proto}") + self.assertIsInstance(cm.exception.__context__, UnicodeEncodeError) + + def test_nested_lookup_error(self): + # Nested name does not exist + global TestGlobal + class TestGlobal: + class A: + pass + obj = REX('TestGlobal.A.B.C') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as {__name__}.TestGlobal.A.B.C") + self.assertEqual(str(cm.exception.__context__), + "type object 'A' has no attribute 'B'") + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal.A.B.C") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'TestGlobal'") + + def test_wrong_object_lookup_error(self): + # Name is bound to different object + global TestGlobal + class TestGlobal: + pass + obj = REX('TestGlobal') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not the same object as {__name__}.TestGlobal") + self.assertIsNone(cm.exception.__context__) + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(obj, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal") + self.assertEqual(str(cm.exception.__context__), + "module '__main__' has no attribute 'TestGlobal'") + + def test_local_lookup_error(self): + # Test that whichmodule() errors out cleanly when looking up + # an assumed globally-reachable object fails. + def f(): + pass + # Since the function is local, lookup will fail + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + # Same without a __module__ attribute (exercises a different path + # in _pickle.c). + del f.__module__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + # Yet a different path. + f.__name__ = f.__qualname__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(f, proto) + self.assertEqual(str(cm.exception), + f"Can't pickle local object {f!r}") + + def test_reduce_ex_None(self): + c = REX_None() + with self.assertRaises(TypeError): + self.dumps(c) + + def test_reduce_None(self): + c = R_None() + with self.assertRaises(TypeError): + self.dumps(c) + + @no_tracing + def test_bad_getattr(self): + # Issue #3514: crash when there is an infinite loop in __getattr__ + x = BadGetattr() + for proto in range(2): + with support.infinite_recursion(25): + self.assertRaises(RuntimeError, self.dumps, x, proto) + for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): + s = self.dumps(x, proto) + + def test_picklebuffer_error(self): + # PickleBuffer forbidden with protocol < 5 + pb = pickle.PickleBuffer(b"foobar") + for proto in range(0, 5): + with self.subTest(proto=proto): + with self.assertRaises(pickle.PickleError) as cm: + self.dumps(pb, proto) + self.assertEqual(str(cm.exception), + 'PickleBuffer can only be pickled with protocol >= 5') + + def test_non_continuous_buffer(self): + for proto in protocols[5:]: + with self.subTest(proto=proto): + pb = pickle.PickleBuffer(memoryview(b"foobar")[::2]) + with self.assertRaises((pickle.PicklingError, BufferError)): + self.dumps(pb, proto) + + def test_buffer_callback_error(self): + def buffer_callback(buffers): + raise CustomError + pb = pickle.PickleBuffer(b"foobar") + with self.assertRaises(CustomError): + self.dumps(pb, 5, buffer_callback=buffer_callback) + + def test_evil_pickler_mutating_collection(self): + # https://github.com/python/cpython/issues/92930 + global Clearer + class Clearer: + pass + + def check(collection): + class EvilPickler(self.pickler): + def persistent_id(self, obj): + if isinstance(obj, Clearer): + collection.clear() + return None + pickler = EvilPickler(io.BytesIO(), proto) + try: + pickler.dump(collection) + except RuntimeError as e: + expected = "changed size during iteration" + self.assertIn(expected, str(e)) + + for proto in protocols: + check([Clearer()]) + check([Clearer(), Clearer()]) + check({Clearer()}) + check({Clearer(), Clearer()}) + check({Clearer(): 1}) + check({Clearer(): 1, Clearer(): 2}) + check({1: Clearer(), 2: Clearer()}) + + @support.cpython_only + def test_bad_ext_code(self): + # This should never happen in normal circumstances, because the type + # and the value of the extension code is checked in copyreg.add_extension(). + key = (__name__, 'MyList') + def check(code, exc): + assert key not in copyreg._extension_registry + assert code not in copyreg._inverted_registry + with (support.swap_item(copyreg._extension_registry, key, code), + support.swap_item(copyreg._inverted_registry, code, key)): + for proto in protocols[2:]: + with self.assertRaises(exc): + self.dumps(MyList, proto) + + check(object(), TypeError) + check(None, TypeError) + check(-1, (RuntimeError, struct.error)) + check(0, RuntimeError) + check(2**31, (RuntimeError, OverflowError, struct.error)) + check(2**1000, (OverflowError, struct.error)) + check(-2**1000, (OverflowError, struct.error)) + class AbstractPickleTests: # Subclass must define self.dumps, self.loads. @@ -1822,6 +2837,14 @@ def test_unicode_high_plane(self): t2 = self.loads(p) self.assert_is_copy(t, t2) + def test_unicode_memoization(self): + # Repeated str is re-used (even when escapes added). + for proto in protocols: + for s in '', 'xyz', 'xyz\n', 'x\\yz', 'x\xa1yz\r': + p = self.dumps((s, s), proto) + s1, s2 = self.loads(p) + self.assertIs(s1, s2) + def test_bytes(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -1834,6 +2857,25 @@ def test_bytes(self): p = self.dumps(s, proto) self.assert_is_copy(s, self.loads(p)) + def test_bytes_memoization(self): + for proto in protocols: + for array_type in [bytes, ZeroCopyBytes]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + x, y = self.loads(p) + self.assertIs(x, y) + self.assert_is_copy((b, b), (x, y)) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1, b2 = array_type(s), array_type(s) + p = self.dumps((b1, b2), proto) + # Note that (b1, b2) = self.loads(p) might have identical + # components, i.e., b1 is b2, but this is not always the + # case if the content is large (equality still holds). + self.assert_is_copy((b1, b2), self.loads(p)) + def test_bytearray(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -1853,6 +2895,32 @@ def test_bytearray(self): self.assertNotIn(b'bytearray', p) self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) + def test_bytearray_memoization(self): + for proto in protocols: + for array_type in [bytearray, ZeroCopyBytearray]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + b1, b2 = self.loads(p) + self.assertIs(b1, b2) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1a, b2a = array_type(s), array_type(s) + # Unlike bytes, equal but independent bytearray objects are + # never identical. + self.assertIsNot(b1a, b2a) + + p = self.dumps((b1a, b2a), proto) + b1b, b2b = self.loads(p) + self.assertIsNot(b1b, b2b) + + self.assertIsNot(b1a, b1b) + self.assert_is_copy(b1a, b1b) + + self.assertIsNot(b2a, b2b) + self.assert_is_copy(b2a, b2b) + def test_ints(self): for proto in protocols: n = sys.maxsize @@ -1896,7 +2964,7 @@ def test_float(self): got = self.loads(pickle) self.assert_is_copy(value, got) - @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @run_with_locales('LC_ALL', 'de_DE', 'fr_FR', '') def test_float_format(self): # make sure that floats are formatted locale independent with proto 0 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') @@ -1971,6 +3039,33 @@ def test_singleton_types(self): u = self.loads(s) self.assertIs(type(singleton), u) + def test_builtin_types(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and not issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + + def test_builtin_exceptions(self): + for t in builtins.__dict__.values(): + if isinstance(t, type) and issubclass(t, BaseException): + for proto in protocols: + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) + + def test_builtin_functions(self): + for t in builtins.__dict__.values(): + if isinstance(t, types.BuiltinFunctionType): + for proto in protocols: + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) + # Tests for protocol 2 def test_proto(self): @@ -1978,7 +3073,7 @@ def test_proto(self): pickled = self.dumps(None, proto) if proto >= 2: proto_header = pickle.PROTO + bytes([proto]) - self.assertTrue(pickled.startswith(proto_header)) + self.assertStartsWith(pickled, proto_header) else: self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) @@ -2370,36 +3465,11 @@ def test_reduce_calls_base(self): y = self.loads(s) self.assertEqual(y._reduce_called, 1) - @no_tracing - def test_bad_getattr(self): - # Issue #3514: crash when there is an infinite loop in __getattr__ - x = BadGetattr() - for proto in protocols: - with support.infinite_recursion(): - self.assertRaises(RuntimeError, self.dumps, x, proto) - - def test_reduce_bad_iterator(self): - # Issue4176: crash when 4th and 5th items of __reduce__() - # are not iterators - class C(object): - def __reduce__(self): - # 4th item is not an iterator - return list, (), None, [], None - class D(object): - def __reduce__(self): - # 5th item is not an iterator - return dict, (), None, None, [] - - # Python implementation is less strict and also accepts iterables. - for proto in protocols: - try: - self.dumps(C(), proto) - except pickle.PicklingError: - pass - try: - self.dumps(D(), proto) - except pickle.PicklingError: - pass + def test_pickle_setstate_None(self): + c = C_None_setstate() + p = self.dumps(c) + with self.assertRaises(TypeError): + self.loads(p) def test_many_puts_and_gets(self): # Test that internal data structures correctly deal with lots of @@ -2536,6 +3606,7 @@ def check_frame_opcodes(self, pickled): self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN) @support.skip_if_pgo_task + @support.requires_resource('cpu') def test_framing_many_objects(self): obj = list(range(10**5)) for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): @@ -2716,6 +3787,18 @@ class Recursive: self.assertIs(unpickled, Recursive) del Recursive.mod # break reference loop + def test_recursive_nested_names2(self): + global Recursive + class Recursive: + pass + Recursive.ref = Recursive + Recursive.__qualname__ = 'Recursive.ref' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(Recursive, proto)) + self.assertIs(unpickled, Recursive) + del Recursive.ref # break reference loop + def test_py_methods(self): global PyMethodsTest class PyMethodsTest: @@ -2765,6 +3848,15 @@ def pie(self): unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(obj), unpickled(obj)) + descriptors = ( + PyMethodsTest.__dict__['cheese'], # static method descriptor + PyMethodsTest.__dict__['wine'], # class method descriptor + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for descr in descriptors: + with self.subTest(proto=proto, descr=descr): + self.assertRaises(TypeError, self.dumps, descr, proto) + def test_c_methods(self): global Subclass class Subclass(tuple): @@ -2800,6 +3892,15 @@ class Nested(str): unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(*args), unpickled(*args)) + descriptors = ( + bytearray.__dict__['maketrans'], # built-in static method descriptor + dict.__dict__['fromkeys'], # built-in class method descriptor + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for descr in descriptors: + with self.subTest(proto=proto, descr=descr): + self.assertRaises(TypeError, self.dumps, descr, proto) + def test_compat_pickle(self): tests = [ (range(1, 7), '__builtin__', 'xrange'), @@ -2818,27 +3919,6 @@ def test_compat_pickle(self): self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) self.assertIs(type(self.loads(pickled)), type(val)) - def test_local_lookup_error(self): - # Test that whichmodule() errors out cleanly when looking up - # an assumed globally-reachable object fails. - def f(): - pass - # Since the function is local, lookup will fail - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Same without a __module__ attribute (exercises a different path - # in _pickle.c). - del f.__module__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Yet a different path. - f.__name__ = f.__qualname__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # # PEP 574 tests below # @@ -2949,20 +4029,6 @@ def test_oob_buffers_writable_to_readonly(self): self.assertIs(type(new), type(obj)) self.assertEqual(new, obj) - def test_picklebuffer_error(self): - # PickleBuffer forbidden with protocol < 5 - pb = pickle.PickleBuffer(b"foobar") - for proto in range(0, 5): - with self.assertRaises(pickle.PickleError): - self.dumps(pb, proto) - - def test_buffer_callback_error(self): - def buffer_callback(buffers): - 1/0 - pb = pickle.PickleBuffer(b"foobar") - with self.assertRaises(ZeroDivisionError): - self.dumps(pb, 5, buffer_callback=buffer_callback) - def test_buffers_error(self): pb = pickle.PickleBuffer(b"foobar") for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): @@ -3024,6 +4090,36 @@ def check_array(arr): # 2-D, non-contiguous check_array(arr[::2]) + def test_evil_class_mutating_dict(self): + # https://github.com/python/cpython/issues/92930 + from random import getrandbits + + global Bad + class Bad: + def __eq__(self, other): + return ENABLED + def __hash__(self): + return 42 + def __reduce__(self): + if getrandbits(6) == 0: + collection.clear() + return (Bad, ()) + + for proto in protocols: + for _ in range(20): + ENABLED = False + collection = {Bad(): Bad() for _ in range(20)} + for bad in collection: + bad.bad = bad + bad.collection = collection + ENABLED = True + try: + data = self.dumps(collection, proto) + self.loads(data) + except RuntimeError as e: + expected = "changed size during iteration" + self.assertIn(expected, str(e)) + class BigmemPickleTests: @@ -3154,6 +4250,18 @@ def test_huge_str_64b(self, size): # Test classes for reduce_ex +class R: + def __init__(self, reduce=None): + self.reduce = reduce + def __reduce__(self, proto): + return self.reduce + +class REX: + def __init__(self, reduce_ex=None): + self.reduce_ex = reduce_ex + def __reduce_ex__(self, proto): + return self.reduce_ex + class REX_one(object): """No __reduce_ex__ here, but inheriting it from object""" _reduce_called = 0 @@ -3229,6 +4337,34 @@ def __setstate__(self, state): def __reduce__(self): return type(self), (), self.state +class REX_None: + """ Setting __reduce_ex__ to None should fail """ + __reduce_ex__ = None + +class R_None: + """ Setting __reduce__ to None should fail """ + __reduce__ = None + +class C_None_setstate: + """ Setting __setstate__ to None should fail """ + def __getstate__(self): + return 1 + + __setstate__ = None + +class CustomError(Exception): + pass + +class Unpickleable: + def __reduce__(self): + raise CustomError + +UNPICKLEABLE = Unpickleable() + +class UnpickleableCallable(Unpickleable): + def __call__(self, *args, **kwargs): + pass + # Test classes for newobj @@ -3278,7 +4414,9 @@ class MyIntWithNew2(MyIntWithNew): class SlotList(MyList): __slots__ = ["foo"] -class SimpleNewObj(int): +# Ruff "redefined while unused" false positive here due to `global` variables +# being assigned (and then restored) from within test methods earlier in the file +class SimpleNewObj(int): # noqa: F811 def __init__(self, *args, **kwargs): # raise an error, to make sure this isn't called raise TypeError("SimpleNewObj.__init__() didn't expect to get called") @@ -3297,6 +4435,12 @@ class BadGetattr: def __getattr__(self, key): self.foo +class NoNew: + def __getattribute__(self, name): + if name == '__new__': + raise AttributeError + return super().__getattribute__(name) + class AbstractPickleModuleTests: @@ -3363,6 +4507,84 @@ def __init__(self): pass self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) + def test_unpickler_bad_file(self): + # bpo-38384: Crash in _pickle if the read attribute raises an error. + def raises_oserror(self, *args, **kwargs): + raise OSError + @property + def bad_property(self): + raise CustomError + + # File without read and readline + class F: + pass + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File without read + class F: + readline = raises_oserror + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File without readline + class F: + read = raises_oserror + self.assertRaises((AttributeError, TypeError), self.Unpickler, F()) + + # File with bad read + class F: + read = bad_property + readline = raises_oserror + self.assertRaises(CustomError, self.Unpickler, F()) + + # File with bad readline + class F: + readline = bad_property + read = raises_oserror + self.assertRaises(CustomError, self.Unpickler, F()) + + # File with bad readline, no read + class F: + readline = bad_property + self.assertRaises(CustomError, self.Unpickler, F()) + + # File with bad read, no readline + class F: + read = bad_property + self.assertRaises((AttributeError, CustomError), self.Unpickler, F()) + + # File with bad peek + class F: + peek = bad_property + read = raises_oserror + readline = raises_oserror + try: + self.Unpickler(F()) + except CustomError: + pass + + # File with bad readinto + class F: + readinto = bad_property + read = raises_oserror + readline = raises_oserror + try: + self.Unpickler(F()) + except CustomError: + pass + + def test_pickler_bad_file(self): + # File without write + class F: + pass + self.assertRaises(TypeError, self.Pickler, F()) + + # File with bad write + class F: + @property + def write(self): + raise CustomError + self.assertRaises(CustomError, self.Pickler, F()) + def check_dumps_loads_oob_buffers(self, dumps, loads): # No need to do the full gamut of tests here, just enough to # check that dumps() and loads() redirect their arguments @@ -3469,9 +4691,15 @@ def test_return_correct_type(self): def test_protocol0_is_ascii_only(self): non_ascii_str = "\N{EMPTY SET}" - self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(non_ascii_str, 0) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' - self.assertRaises(pickle.UnpicklingError, self.loads, pickled) + with self.assertRaises(pickle.UnpicklingError) as cm: + self.loads(pickled) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') class AbstractPicklerUnpicklerObjectTests: @@ -3632,6 +4860,25 @@ def test_unpickling_buffering_readline(self): unpickler = self.unpickler_class(f) self.assertEqual(unpickler.load(), data) + def test_pickle_invalid_reducer_override(self): + # gh-103035 + obj = object() + + f = io.BytesIO() + class MyPickler(self.pickler_class): + pass + pickler = MyPickler(f) + pickler.dump(obj) + + pickler.clear_memo() + pickler.reducer_override = None + with self.assertRaises(TypeError): + pickler.dump(obj) + + pickler.clear_memo() + pickler.reducer_override = 10 + with self.assertRaises(TypeError): + pickler.dump(obj) # Tests for dispatch_table attribute @@ -3722,8 +4969,11 @@ class MyClass: # NotImplemented self.assertIs(math_log, math.log) - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError) as cm: p.dump(g) + self.assertRegex(str(cm.exception), + r'(__reduce__|)' + r' must return (a )?string or tuple') with self.assertRaisesRegex( ValueError, 'The reducer just failed'): @@ -3762,7 +5012,7 @@ def test_default_dispatch_table(self): p = self.pickler_class(f, 0) with self.assertRaises(AttributeError): p.dispatch_table - self.assertFalse(hasattr(p, 'dispatch_table')) + self.assertNotHasAttr(p, 'dispatch_table') def test_class_dispatch_table(self): # A dispatch_table attribute can be specified class-wide @@ -3794,6 +5044,15 @@ def dumps(obj, protocol=None): self._test_dispatch_table(dumps, dt) + def test_dispatch_table_None_item(self): + # gh-93627 + obj = object() + f = io.BytesIO() + pickler = self.pickler_class(f) + pickler.dispatch_table = {type(obj): None} + with self.assertRaises(TypeError): + pickler.dump(obj) + def _test_dispatch_table(self, dumps, dispatch_table): def custom_load_dump(obj): return pickle.loads(dumps(obj, 0)) diff --git a/Lib/test/profilee.py b/Lib/test/profilee.py new file mode 100644 index 00000000000..b6a090a2e34 --- /dev/null +++ b/Lib/test/profilee.py @@ -0,0 +1,115 @@ +""" +Input for test_profile.py and test_cprofile.py. + +IMPORTANT: This stuff is touchy. If you modify anything above the +test class you'll have to regenerate the stats by running the two +test files. + +*ALL* NUMBERS in the expected output are relevant. If you change +the formatting of pstats, please don't just regenerate the expected +output without checking very carefully that not a single number has +changed. +""" + +import sys + +# In order to have reproducible time, we simulate a timer in the global +# variable 'TICKS', which represents simulated time in milliseconds. +# (We can't use a helper function increment the timer since it would be +# included in the profile and would appear to consume all the time.) +TICKS = 42000 + +def timer(): + return TICKS + +def testfunc(): + # 1 call + # 1000 ticks total: 270 ticks local, 730 ticks in subfunctions + global TICKS + TICKS += 99 + helper() # 300 + helper() # 300 + TICKS += 171 + factorial(14) # 130 + +def factorial(n): + # 23 calls total + # 170 ticks total, 150 ticks local + # 3 primitive calls, 130, 20 and 20 ticks total + # including 116, 17, 17 ticks local + global TICKS + if n > 0: + TICKS += n + return mul(n, factorial(n-1)) + else: + TICKS += 11 + return 1 + +def mul(a, b): + # 20 calls + # 1 tick, local + global TICKS + TICKS += 1 + return a * b + +def helper(): + # 2 calls + # 300 ticks total: 20 ticks local, 260 ticks in subfunctions + global TICKS + TICKS += 1 + helper1() # 30 + TICKS += 2 + helper1() # 30 + TICKS += 6 + helper2() # 50 + TICKS += 3 + helper2() # 50 + TICKS += 2 + helper2() # 50 + TICKS += 5 + helper2_indirect() # 70 + TICKS += 1 + +def helper1(): + # 4 calls + # 30 ticks total: 29 ticks local, 1 tick in subfunctions + global TICKS + TICKS += 10 + hasattr(C(), "foo") # 1 + TICKS += 19 + lst = [] + lst.append(42) # 0 + sys.exception() # 0 + +def helper2_indirect(): + helper2() # 50 + factorial(3) # 20 + +def helper2(): + # 8 calls + # 50 ticks local: 39 ticks local, 11 ticks in subfunctions + global TICKS + TICKS += 11 + hasattr(C(), "bar") # 1 + TICKS += 13 + subhelper() # 10 + TICKS += 15 + +def subhelper(): + # 8 calls + # 10 ticks total: 8 ticks local, 2 ticks in subfunctions + global TICKS + TICKS += 2 + for i in range(2): # 0 + try: + C().foo # 1 x 2 + except AttributeError: + TICKS += 3 # 3 x 2 + +class C: + def __getattr__(self, name): + # 28 calls + # 1 tick, local + global TICKS + TICKS += 1 + raise AttributeError diff --git a/Lib/test/pyclbr_input.py b/Lib/test/pyclbr_input.py new file mode 100644 index 00000000000..5535edbfa77 --- /dev/null +++ b/Lib/test/pyclbr_input.py @@ -0,0 +1,85 @@ +"""Test cases for test_pyclbr.py""" + +def f(): pass + +class Other(object): + @classmethod + def foo(c): pass + + def om(self): pass + +class B (object): + def bm(self): pass + +class C (B): + d = 10 + + # This one is correctly considered by both test_pyclbr.py and pyclbr.py + # as a non-method of C. + foo = Other().foo + + # This causes test_pyclbr.py to fail, but only because the + # introspection-based is_method() code in the test can't + # distinguish between this and a genuine method function like m(). + # + # The pyclbr.py module gets this right as it parses the text. + om = Other.om + f = f + + def m(self): pass + + @staticmethod + def sm(self): pass + + @classmethod + def cm(self): pass + +# Check that mangling is correctly handled + +class a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class ___: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py new file mode 100644 index 00000000000..19b336ba96f --- /dev/null +++ b/Lib/test/pythoninfo.py @@ -0,0 +1,1100 @@ +""" +Collect various information about Python to help debugging test failures. +""" +import errno +import re +import sys +import traceback +import warnings + + +def normalize_text(text): + if text is None: + return None + text = str(text) + text = re.sub(r'\s+', ' ', text) + return text.strip() + + +class PythonInfo: + def __init__(self): + self.info = {} + + def add(self, key, value): + if key in self.info: + raise ValueError("duplicate key: %r" % key) + + if value is None: + return + + if not isinstance(value, int): + if not isinstance(value, str): + # convert other objects like sys.flags to string + value = str(value) + + value = value.strip() + if not value: + return + + self.info[key] = value + + def get_infos(self): + """ + Get information as a key:value dictionary where values are strings. + """ + return {key: str(value) for key, value in self.info.items()} + + +def copy_attributes(info_add, obj, name_fmt, attributes, *, formatter=None): + for attr in attributes: + value = getattr(obj, attr, None) + if value is None: + continue + name = name_fmt % attr + if formatter is not None: + value = formatter(attr, value) + info_add(name, value) + + +def copy_attr(info_add, name, mod, attr_name): + try: + value = getattr(mod, attr_name) + except AttributeError: + return + info_add(name, value) + + +def call_func(info_add, name, mod, func_name, *, formatter=None): + try: + func = getattr(mod, func_name) + except AttributeError: + return + value = func() + if formatter is not None: + value = formatter(value) + info_add(name, value) + + +def collect_sys(info_add): + attributes = ( + '_emscripten_info', + '_framework', + 'abiflags', + 'api_version', + 'builtin_module_names', + 'byteorder', + 'dont_write_bytecode', + 'executable', + 'flags', + 'float_info', + 'float_repr_style', + 'hash_info', + 'hexversion', + 'implementation', + 'int_info', + 'maxsize', + 'maxunicode', + 'path', + 'platform', + 'platlibdir', + 'prefix', + 'thread_info', + 'version', + 'version_info', + 'winver', + ) + copy_attributes(info_add, sys, 'sys.%s', attributes) + + for func in ( + '_is_gil_enabled', + 'getandroidapilevel', + 'getrecursionlimit', + 'getwindowsversion', + ): + call_func(info_add, f'sys.{func}', sys, func) + + encoding = sys.getfilesystemencoding() + if hasattr(sys, 'getfilesystemencodeerrors'): + encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors()) + info_add('sys.filesystem_encoding', encoding) + + for name in ('stdin', 'stdout', 'stderr'): + stream = getattr(sys, name) + if stream is None: + continue + encoding = getattr(stream, 'encoding', None) + if not encoding: + continue + errors = getattr(stream, 'errors', None) + if errors: + encoding = '%s/%s' % (encoding, errors) + info_add('sys.%s.encoding' % name, encoding) + + # Were we compiled --with-pydebug? + Py_DEBUG = hasattr(sys, 'gettotalrefcount') + if Py_DEBUG: + text = 'Yes (sys.gettotalrefcount() present)' + else: + text = 'No (sys.gettotalrefcount() missing)' + info_add('build.Py_DEBUG', text) + + # Were we compiled --with-trace-refs? + Py_TRACE_REFS = hasattr(sys, 'getobjects') + if Py_TRACE_REFS: + text = 'Yes (sys.getobjects() present)' + else: + text = 'No (sys.getobjects() missing)' + info_add('build.Py_TRACE_REFS', text) + + +def collect_platform(info_add): + import platform + + arch = platform.architecture() + arch = ' '.join(filter(bool, arch)) + info_add('platform.architecture', arch) + + info_add('platform.python_implementation', + platform.python_implementation()) + info_add('platform.platform', + platform.platform(aliased=True)) + + libc_ver = ('%s %s' % platform.libc_ver()).strip() + if libc_ver: + info_add('platform.libc_ver', libc_ver) + + try: + os_release = platform.freedesktop_os_release() + except OSError: + pass + else: + for key in ( + 'ID', + 'NAME', + 'PRETTY_NAME' + 'VARIANT', + 'VARIANT_ID', + 'VERSION', + 'VERSION_CODENAME', + 'VERSION_ID', + ): + if key not in os_release: + continue + info_add(f'platform.freedesktop_os_release[{key}]', + os_release[key]) + + if sys.platform == 'android': + call_func(info_add, 'platform.android_ver', platform, 'android_ver') + + +def collect_locale(info_add): + import locale + + info_add('locale.getencoding', locale.getencoding()) + + +def collect_builtins(info_add): + info_add('builtins.float.float_format', float.__getformat__("float")) + info_add('builtins.float.double_format', float.__getformat__("double")) + + +def collect_urandom(info_add): + import os + + if hasattr(os, 'getrandom'): + # PEP 524: Check if system urandom is initialized + try: + try: + os.getrandom(1, os.GRND_NONBLOCK) + state = 'ready (initialized)' + except BlockingIOError as exc: + state = 'not seeded yet (%s)' % exc + info_add('os.getrandom', state) + except OSError as exc: + # Python was compiled on a more recent Linux version + # than the current Linux kernel: ignore OSError(ENOSYS) + if exc.errno != errno.ENOSYS: + raise + + +def collect_os(info_add): + import os + + def format_attr(attr, value): + if attr in ('supports_follow_symlinks', 'supports_fd', + 'supports_effective_ids'): + return str(sorted(func.__name__ for func in value)) + else: + return value + + attributes = ( + 'name', + 'supports_bytes_environ', + 'supports_effective_ids', + 'supports_fd', + 'supports_follow_symlinks', + ) + copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr) + + for func in ( + 'cpu_count', + 'getcwd', + 'getegid', + 'geteuid', + 'getgid', + 'getloadavg', + 'getresgid', + 'getresuid', + 'getuid', + 'process_cpu_count', + 'uname', + ): + call_func(info_add, 'os.%s' % func, os, func) + + def format_groups(groups): + return ', '.join(map(str, groups)) + + call_func(info_add, 'os.getgroups', os, 'getgroups', formatter=format_groups) + + if hasattr(os, 'getlogin'): + try: + login = os.getlogin() + except OSError: + # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl + # for device" on Travis CI + pass + else: + info_add("os.login", login) + + # Environment variables used by the stdlib and tests. Don't log the full + # environment: filter to list to not leak sensitive information. + # + # HTTP_PROXY is not logged because it can contain a password. + ENV_VARS = frozenset(( + "APPDATA", + "AR", + "ARCHFLAGS", + "ARFLAGS", + "AUDIODEV", + "BUILDPYTHON", + "CC", + "CFLAGS", + "COLUMNS", + "COMPUTERNAME", + "COMSPEC", + "CPP", + "CPPFLAGS", + "DISPLAY", + "DISTUTILS_DEBUG", + "DISTUTILS_USE_SDK", + "DYLD_LIBRARY_PATH", + "ENSUREPIP_OPTIONS", + "HISTORY_FILE", + "HOME", + "HOMEDRIVE", + "HOMEPATH", + "IDLESTARTUP", + "IPHONEOS_DEPLOYMENT_TARGET", + "LANG", + "LDFLAGS", + "LDSHARED", + "LD_LIBRARY_PATH", + "LINES", + "MACOSX_DEPLOYMENT_TARGET", + "MAILCAPS", + "MAKEFLAGS", + "MIXERDEV", + "MSSDK", + "PATH", + "PATHEXT", + "PIP_CONFIG_FILE", + "PLAT", + "POSIXLY_CORRECT", + "PY_SAX_PARSER", + "ProgramFiles", + "ProgramFiles(x86)", + "RUNNING_ON_VALGRIND", + "SDK_TOOLS_BIN", + "SERVER_SOFTWARE", + "SHELL", + "SOURCE_DATE_EPOCH", + "SYSTEMROOT", + "TEMP", + "TERM", + "TILE_LIBRARY", + "TMP", + "TMPDIR", + "TRAVIS", + "TZ", + "USERPROFILE", + "VIRTUAL_ENV", + "WAYLAND_DISPLAY", + "WINDIR", + "_PYTHON_HOSTRUNNER", + "_PYTHON_HOST_PLATFORM", + "_PYTHON_PROJECT_BASE", + "_PYTHON_SYSCONFIGDATA_NAME", + "_PYTHON_SYSCONFIGDATA_PATH", + "__PYVENV_LAUNCHER__", + + # Sanitizer options + "ASAN_OPTIONS", + "LSAN_OPTIONS", + "MSAN_OPTIONS", + "TSAN_OPTIONS", + "UBSAN_OPTIONS", + )) + for name, value in os.environ.items(): + uname = name.upper() + if (uname in ENV_VARS + # Copy PYTHON* variables like PYTHONPATH + # Copy LC_* variables like LC_ALL + or uname.startswith(("PYTHON", "LC_")) + # Visual Studio: VS140COMNTOOLS + or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))): + info_add('os.environ[%s]' % name, value) + + if hasattr(os, 'umask'): + mask = os.umask(0) + os.umask(mask) + info_add("os.umask", '0o%03o' % mask) + + +def collect_pwd(info_add): + try: + import pwd + except ImportError: + return + import os + + uid = os.getuid() + try: + entry = pwd.getpwuid(uid) + except KeyError: + entry = None + + info_add('pwd.getpwuid(%s)'% uid, + entry if entry is not None else '') + + if entry is None: + # there is nothing interesting to read if the current user identifier + # is not the password database + return + + if hasattr(os, 'getgrouplist'): + groups = os.getgrouplist(entry.pw_name, entry.pw_gid) + groups = ', '.join(map(str, groups)) + info_add('os.getgrouplist', groups) + + +def collect_readline(info_add): + try: + import readline + except ImportError: + return + + def format_attr(attr, value): + if isinstance(value, int): + return "%#x" % value + else: + return value + + attributes = ( + "_READLINE_VERSION", + "_READLINE_RUNTIME_VERSION", + "_READLINE_LIBRARY_VERSION", + ) + copy_attributes(info_add, readline, 'readline.%s', attributes, + formatter=format_attr) + + if not hasattr(readline, "_READLINE_LIBRARY_VERSION"): + # _READLINE_LIBRARY_VERSION has been added to CPython 3.7 + doc = getattr(readline, '__doc__', '') + if 'libedit readline' in doc: + info_add('readline.library', 'libedit readline') + elif 'GNU readline' in doc: + info_add('readline.library', 'GNU readline') + + +def collect_gdb(info_add): + import subprocess + + try: + proc = subprocess.Popen(["gdb", "-nx", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + version = proc.communicate()[0] + if proc.returncode: + # ignore gdb failure: test_gdb will log the error + return + except OSError: + return + + # Only keep the first line + version = version.splitlines()[0] + info_add('gdb_version', version) + + +def collect_tkinter(info_add): + try: + import _tkinter + except ImportError: + pass + else: + attributes = ('TK_VERSION', 'TCL_VERSION') + copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes) + + try: + import tkinter + except ImportError: + pass + else: + tcl = tkinter.Tcl() + patchlevel = tcl.call('info', 'patchlevel') + info_add('tkinter.info_patchlevel', patchlevel) + + +def collect_time(info_add): + import time + + info_add('time.time', time.time()) + + attributes = ( + 'altzone', + 'daylight', + 'timezone', + 'tzname', + ) + copy_attributes(info_add, time, 'time.%s', attributes) + + if hasattr(time, 'get_clock_info'): + for clock in ('clock', 'monotonic', 'perf_counter', + 'process_time', 'thread_time', 'time'): + try: + # prevent DeprecatingWarning on get_clock_info('clock') + with warnings.catch_warnings(record=True): + clock_info = time.get_clock_info(clock) + except ValueError: + # missing clock like time.thread_time() + pass + else: + info_add('time.get_clock_info(%s)' % clock, clock_info) + + +def collect_curses(info_add): + try: + import curses + except ImportError: + return + + copy_attr(info_add, 'curses.ncurses_version', curses, 'ncurses_version') + + +def collect_datetime(info_add): + try: + import datetime + except ImportError: + return + + info_add('datetime.datetime.now', datetime.datetime.now()) + + +def collect_sysconfig(info_add): + import sysconfig + + info_add('sysconfig.is_python_build', sysconfig.is_python_build()) + + for name in ( + 'ABIFLAGS', + 'ANDROID_API_LEVEL', + 'CC', + 'CCSHARED', + 'CFLAGS', + 'CFLAGSFORSHARED', + 'CONFIG_ARGS', + 'HOSTRUNNER', + 'HOST_GNU_TYPE', + 'MACHDEP', + 'MULTIARCH', + 'OPT', + 'PGO_PROF_USE_FLAG', + 'PY_CFLAGS', + 'PY_CFLAGS_NODIST', + 'PY_CORE_LDFLAGS', + 'PY_LDFLAGS', + 'PY_LDFLAGS_NODIST', + 'PY_STDMODULE_CFLAGS', + 'Py_DEBUG', + 'Py_ENABLE_SHARED', + 'Py_GIL_DISABLED', + 'SHELL', + 'SOABI', + 'TEST_MODULES', + 'abs_builddir', + 'abs_srcdir', + 'prefix', + 'srcdir', + ): + value = sysconfig.get_config_var(name) + if name == 'ANDROID_API_LEVEL' and not value: + # skip ANDROID_API_LEVEL=0 + continue + value = normalize_text(value) + info_add('sysconfig[%s]' % name, value) + + PY_CFLAGS = sysconfig.get_config_var('PY_CFLAGS') + NDEBUG = (PY_CFLAGS and '-DNDEBUG' in PY_CFLAGS) + if NDEBUG: + text = 'ignore assertions (macro defined)' + else: + text= 'build assertions (macro not defined)' + info_add('build.NDEBUG',text) + + for name in ( + 'WITH_DOC_STRINGS', + 'WITH_DTRACE', + 'WITH_FREELISTS', + 'WITH_MIMALLOC', + 'WITH_PYMALLOC', + 'WITH_VALGRIND', + ): + value = sysconfig.get_config_var(name) + if value: + text = 'Yes' + else: + text = 'No' + info_add(f'build.{name}', text) + + +def collect_ssl(info_add): + import os + try: + import ssl + except ImportError: + return + try: + import _ssl + except ImportError: + _ssl = None + + def format_attr(attr, value): + if attr.startswith('OP_'): + return '%#8x' % value + else: + return value + + attributes = ( + 'OPENSSL_VERSION', + 'OPENSSL_VERSION_INFO', + 'HAS_SNI', + 'OP_ALL', + 'OP_NO_TLSv1_1', + ) + copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr) + + for name, ctx in ( + ('SSLContext', ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)), + ('default_https_context', ssl._create_default_https_context()), + ('stdlib_context', ssl._create_stdlib_context()), + ): + attributes = ( + 'minimum_version', + 'maximum_version', + 'protocol', + 'options', + 'verify_mode', + ) + copy_attributes(info_add, ctx, f'ssl.{name}.%s', attributes) + + env_names = ["OPENSSL_CONF", "SSLKEYLOGFILE"] + if _ssl is not None and hasattr(_ssl, 'get_default_verify_paths'): + parts = _ssl.get_default_verify_paths() + env_names.extend((parts[0], parts[2])) + + for name in env_names: + try: + value = os.environ[name] + except KeyError: + continue + info_add('ssl.environ[%s]' % name, value) + + +def collect_socket(info_add): + try: + import socket + except ImportError: + return + + try: + hostname = socket.gethostname() + except (OSError, AttributeError): + # WASI SDK 16.0 does not have gethostname(2). + if sys.platform != "wasi": + raise + else: + info_add('socket.hostname', hostname) + + +def collect_sqlite(info_add): + try: + import sqlite3 + except ImportError: + return + + attributes = ('sqlite_version',) + copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes) + + +def collect_zlib(info_add): + try: + import zlib + except ImportError: + return + + attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION') + copy_attributes(info_add, zlib, 'zlib.%s', attributes) + + +def collect_expat(info_add): + try: + from xml.parsers import expat + except ImportError: + return + + attributes = ('EXPAT_VERSION',) + copy_attributes(info_add, expat, 'expat.%s', attributes) + + +def collect_decimal(info_add): + try: + import _decimal + except ImportError: + return + + attributes = ('__libmpdec_version__',) + copy_attributes(info_add, _decimal, '_decimal.%s', attributes) + + +def collect_testcapi(info_add): + try: + import _testcapi + except ImportError: + return + + for name in ( + 'LONG_MAX', # always 32-bit on Windows, 64-bit on 64-bit Unix + 'PY_SSIZE_T_MAX', + 'Py_C_RECURSION_LIMIT', + 'SIZEOF_TIME_T', # 32-bit or 64-bit depending on the platform + 'SIZEOF_WCHAR_T', # 16-bit or 32-bit depending on the platform + ): + copy_attr(info_add, f'_testcapi.{name}', _testcapi, name) + + +def collect_testinternalcapi(info_add): + try: + import _testinternalcapi + except ImportError: + return + + call_func(info_add, 'pymem.allocator', _testinternalcapi, 'pymem_getallocatorsname') + + for name in ( + 'SIZEOF_PYGC_HEAD', + 'SIZEOF_PYOBJECT', + ): + copy_attr(info_add, f'_testinternalcapi.{name}', _testinternalcapi, name) + + +def collect_resource(info_add): + try: + import resource + except ImportError: + return + + limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')] + for name in limits: + key = getattr(resource, name) + value = resource.getrlimit(key) + info_add('resource.%s' % name, value) + + call_func(info_add, 'resource.pagesize', resource, 'getpagesize') + + +def collect_test_socket(info_add): + import unittest + try: + from test import test_socket + except (ImportError, unittest.SkipTest): + return + + # all check attributes like HAVE_SOCKET_CAN + attributes = [name for name in dir(test_socket) + if name.startswith('HAVE_')] + copy_attributes(info_add, test_socket, 'test_socket.%s', attributes) + + +def collect_support(info_add): + try: + from test import support + except ImportError: + return + + attributes = ( + 'MS_WINDOWS', + 'has_fork_support', + 'has_socket_support', + 'has_strftime_extensions', + 'has_subprocess_support', + 'is_android', + 'is_emscripten', + 'is_jython', + 'is_wasi', + ) + copy_attributes(info_add, support, 'support.%s', attributes) + + call_func(info_add, 'support._is_gui_available', support, '_is_gui_available') + call_func(info_add, 'support.python_is_optimized', support, 'python_is_optimized') + + info_add('support.check_sanitizer(address=True)', + support.check_sanitizer(address=True)) + info_add('support.check_sanitizer(memory=True)', + support.check_sanitizer(memory=True)) + info_add('support.check_sanitizer(ub=True)', + support.check_sanitizer(ub=True)) + + +def collect_support_os_helper(info_add): + try: + from test.support import os_helper + except ImportError: + return + + for name in ( + 'can_symlink', + 'can_xattr', + 'can_chmod', + 'can_dac_override', + ): + func = getattr(os_helper, name) + info_add(f'support_os_helper.{name}', func()) + + +def collect_support_socket_helper(info_add): + try: + from test.support import socket_helper + except ImportError: + return + + attributes = ( + 'IPV6_ENABLED', + 'has_gethostname', + ) + copy_attributes(info_add, socket_helper, 'support_socket_helper.%s', attributes) + + for name in ( + 'tcp_blackhole', + ): + func = getattr(socket_helper, name) + info_add(f'support_socket_helper.{name}', func()) + + +def collect_support_threading_helper(info_add): + try: + from test.support import threading_helper + except ImportError: + return + + attributes = ( + 'can_start_thread', + ) + copy_attributes(info_add, threading_helper, 'support_threading_helper.%s', attributes) + + +def collect_cc(info_add): + import subprocess + import sysconfig + + CC = sysconfig.get_config_var('CC') + if not CC: + return + + try: + import shlex + args = shlex.split(CC) + except ImportError: + args = CC.split() + args.append('--version') + try: + proc = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + except OSError: + # Cannot run the compiler, for example when Python has been + # cross-compiled and installed on the target platform where the + # compiler is missing. + return + + stdout = proc.communicate()[0] + if proc.returncode: + # CC --version failed: ignore error + return + + text = stdout.splitlines()[0] + text = normalize_text(text) + info_add('CC.version', text) + + +def collect_gdbm(info_add): + try: + from _gdbm import _GDBM_VERSION + except ImportError: + return + + info_add('gdbm.GDBM_VERSION', '.'.join(map(str, _GDBM_VERSION))) + + +def collect_get_config(info_add): + # Get global configuration variables, _PyPreConfig and _PyCoreConfig + try: + from _testinternalcapi import get_configs + except ImportError: + return + + all_configs = get_configs() + for config_type in sorted(all_configs): + config = all_configs[config_type] + for key in sorted(config): + info_add('%s[%s]' % (config_type, key), repr(config[key])) + + +def collect_subprocess(info_add): + import subprocess + copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',)) + + +def collect_windows(info_add): + if sys.platform != "win32": + # Code specific to Windows + return + + # windows.RtlAreLongPathsEnabled: RtlAreLongPathsEnabled() + # windows.is_admin: IsUserAnAdmin() + try: + import ctypes + if not hasattr(ctypes, 'WinDLL'): + raise ImportError + except ImportError: + pass + else: + ntdll = ctypes.WinDLL('ntdll') + BOOLEAN = ctypes.c_ubyte + try: + RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled + except AttributeError: + res = '' + else: + RtlAreLongPathsEnabled.restype = BOOLEAN + RtlAreLongPathsEnabled.argtypes = () + res = bool(RtlAreLongPathsEnabled()) + info_add('windows.RtlAreLongPathsEnabled', res) + + shell32 = ctypes.windll.shell32 + IsUserAnAdmin = shell32.IsUserAnAdmin + IsUserAnAdmin.restype = BOOLEAN + IsUserAnAdmin.argtypes = () + info_add('windows.is_admin', IsUserAnAdmin()) + + try: + import _winapi + dll_path = _winapi.GetModuleFileName(sys.dllhandle) + info_add('windows.dll_path', dll_path) + except (ImportError, AttributeError): + pass + + # windows.version_caption: "wmic os get Caption,Version /value" command + import subprocess + try: + # When wmic.exe output is redirected to a pipe, + # it uses the OEM code page + proc = subprocess.Popen(["wmic", "os", "get", "Caption,Version", "/value"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="oem", + text=True) + output, stderr = proc.communicate() + if proc.returncode: + output = "" + except OSError: + pass + else: + for line in output.splitlines(): + line = line.strip() + if line.startswith('Caption='): + line = line.removeprefix('Caption=').strip() + if line: + info_add('windows.version_caption', line) + elif line.startswith('Version='): + line = line.removeprefix('Version=').strip() + if line: + info_add('windows.version', line) + + # windows.ver: "ver" command + try: + proc = subprocess.Popen(["ver"], shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True) + output = proc.communicate()[0] + if proc.returncode == 0xc0000142: + return + if proc.returncode: + output = "" + except OSError: + return + else: + output = output.strip() + line = output.splitlines()[0] + if line: + info_add('windows.ver', line) + + # windows.developer_mode: get AllowDevelopmentWithoutDevLicense registry + import winreg + try: + key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock") + subkey = "AllowDevelopmentWithoutDevLicense" + try: + value, value_type = winreg.QueryValueEx(key, subkey) + finally: + winreg.CloseKey(key) + except OSError: + pass + else: + info_add('windows.developer_mode', "enabled" if value else "disabled") + + +def collect_fips(info_add): + try: + import _hashlib + except ImportError: + _hashlib = None + + if _hashlib is not None: + call_func(info_add, 'fips.openssl_fips_mode', _hashlib, 'get_fips_mode') + + try: + with open("/proc/sys/crypto/fips_enabled", encoding="utf-8") as fp: + line = fp.readline().rstrip() + + if line: + info_add('fips.linux_crypto_fips_enabled', line) + except OSError: + pass + + +def collect_tempfile(info_add): + import tempfile + + info_add('tempfile.gettempdir', tempfile.gettempdir()) + + +def collect_libregrtest_utils(info_add): + try: + from test.libregrtest import utils + except ImportError: + return + + info_add('libregrtests.build_info', ' '.join(utils.get_build_info())) + + +def collect_info(info): + error = False + info_add = info.add + + for collect_func in ( + # collect_urandom() must be the first, to check the getrandom() status. + # Other functions may block on os.urandom() indirectly and so change + # its state. + collect_urandom, + + collect_builtins, + collect_cc, + collect_curses, + collect_datetime, + collect_decimal, + collect_expat, + collect_fips, + collect_gdb, + collect_gdbm, + collect_get_config, + collect_locale, + collect_os, + collect_platform, + collect_pwd, + collect_readline, + collect_resource, + collect_socket, + collect_sqlite, + collect_ssl, + collect_subprocess, + collect_sys, + collect_sysconfig, + collect_testcapi, + collect_testinternalcapi, + collect_tempfile, + collect_time, + collect_tkinter, + collect_windows, + collect_zlib, + collect_libregrtest_utils, + + # Collecting from tests should be last as they have side effects. + collect_test_socket, + collect_support, + collect_support_os_helper, + collect_support_socket_helper, + collect_support_threading_helper, + ): + try: + collect_func(info_add) + except Exception: + error = True + print("ERROR: %s() failed" % (collect_func.__name__), + file=sys.stderr) + traceback.print_exc(file=sys.stderr) + print(file=sys.stderr) + sys.stderr.flush() + + return error + + +def dump_info(info, file=None): + title = "Python debug information" + print(title) + print("=" * len(title)) + print() + + infos = info.get_infos() + infos = sorted(infos.items()) + for key, value in infos: + value = value.replace("\n", " ") + print("%s: %s" % (key, value)) + + +def main(): + info = PythonInfo() + error = collect_info(info) + dump_info(info) + + if error: + print() + print("Collection failed: exit with error", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 21b0edfd073..dd61b051354 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,13 +6,10 @@ Run this script with -h or --help for documentation. """ -# We import importlib *ASAP* in order to test #15386 -import importlib - import os import sys -from test.libregrtest import main +from test.libregrtest.main import main # Alias for backward compatibility (just in case) main_in_temp_cwd = main diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py new file mode 100644 index 00000000000..9c3d0c7cf4b --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_a.py @@ -0,0 +1,11 @@ +import sys +import unittest +import test_regrtest_b.util + +class Test(unittest.TestCase): + def test(self): + test_regrtest_b.util # does not fail + self.assertIn('test_regrtest_a', sys.modules) + self.assertIs(sys.modules['test_regrtest_b'], test_regrtest_b) + self.assertIs(sys.modules['test_regrtest_b.util'], test_regrtest_b.util) + self.assertNotIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py new file mode 100644 index 00000000000..3dfba253455 --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/__init__.py @@ -0,0 +1,9 @@ +import sys +import unittest + +class Test(unittest.TestCase): + def test(self): + self.assertNotIn('test_regrtest_a', sys.modules) + self.assertIn('test_regrtest_b', sys.modules) + self.assertNotIn('test_regrtest_b.util', sys.modules) + self.assertNotIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/util.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_b/util.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py b/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py new file mode 100644 index 00000000000..de80769118d --- /dev/null +++ b/Lib/test/regrtestdata/import_from_tests/test_regrtest_c.py @@ -0,0 +1,11 @@ +import sys +import unittest +import test_regrtest_b.util + +class Test(unittest.TestCase): + def test(self): + test_regrtest_b.util # does not fail + self.assertNotIn('test_regrtest_a', sys.modules) + self.assertIs(sys.modules['test_regrtest_b'], test_regrtest_b) + self.assertIs(sys.modules['test_regrtest_b.util'], test_regrtest_b.util) + self.assertIn('test_regrtest_c', sys.modules) diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index 59db2664a02..c7497d09f64 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -145,6 +145,9 @@ def __getitem__(self, i): self.assertEqual(self.type2test(LyingTuple((2,))), self.type2test((1,))) self.assertEqual(self.type2test(LyingList([2])), self.type2test([1])) + with self.assertRaises(TypeError): + self.type2test(unsupported_arg=[]) + def test_truth(self): self.assertFalse(self.type2test()) self.assertTrue(self.type2test([42])) @@ -423,8 +426,7 @@ def test_pickle(self): self.assertEqual(lst2, lst) self.assertNotEqual(id(lst2), id(lst)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.type2test) support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/Lib/test/site-packages/README.txt b/Lib/test/site-packages/README.txt new file mode 100644 index 00000000000..273f6251a7f --- /dev/null +++ b/Lib/test/site-packages/README.txt @@ -0,0 +1,2 @@ +This directory exists so that 3rd party packages can be installed +here. Read the source for site.py for more details. diff --git a/Lib/test/ssl_servers.py b/Lib/test/ssl_servers.py index a4bd7455d47..15b071e04dd 100644 --- a/Lib/test/ssl_servers.py +++ b/Lib/test/ssl_servers.py @@ -14,7 +14,7 @@ here = os.path.dirname(__file__) HOST = socket_helper.HOST -CERTFILE = os.path.join(here, 'keycert.pem') +CERTFILE = os.path.join(here, 'certdata', 'keycert.pem') # This one's based on HTTPServer, which is based on socketserver diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index ea82b0166cd..c5831c47fcf 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -8,18 +8,12 @@ from collections import UserList import random + class Sequence: def __init__(self, seq='wxyz'): self.seq = seq def __len__(self): return len(self.seq) def __getitem__(self, i): return self.seq[i] -class BadSeq1(Sequence): - def __init__(self): self.seq = [7, 'hello', 123] - def __str__(self): return '{0} {1} {2}'.format(*self.seq) - -class BadSeq2(Sequence): - def __init__(self): self.seq = ['a', 'b', 'c'] - def __len__(self): return 8 class BaseTest: # These tests are for buffers of values (bytes) and not @@ -27,7 +21,7 @@ class BaseTest: # and various string implementations # The type to be tested - # Change in subclasses to change the behaviour of fixtesttype() + # Change in subclasses to change the behaviour of fixtype() type2test = None # Whether the "contained items" of the container are integers in @@ -36,7 +30,7 @@ class BaseTest: contains_bytes = False # All tests pass their arguments to the testing methods - # as str objects. fixtesttype() can be used to propagate + # as str objects. fixtype() can be used to propagate # these arguments to the appropriate type def fixtype(self, obj): if isinstance(obj, str): @@ -160,6 +154,12 @@ def test_count(self): self.assertEqual(rem, 0, '%s != 0 for %s' % (rem, i)) self.assertEqual(r1, r2, '%s != %s for %s' % (r1, r2, i)) + def test_count_keyword(self): + self.assertEqual('aa'.replace('a', 'b', 0), 'aa'.replace('a', 'b', count=0)) + self.assertEqual('aa'.replace('a', 'b', 1), 'aa'.replace('a', 'b', count=1)) + self.assertEqual('aa'.replace('a', 'b', 2), 'aa'.replace('a', 'b', count=2)) + self.assertEqual('aa'.replace('a', 'b', 3), 'aa'.replace('a', 'b', count=3)) + def test_find(self): self.checkequal(0, 'abcdefghiabc', 'find', 'abc') self.checkequal(9, 'abcdefghiabc', 'find', 'abc', 1) @@ -327,11 +327,12 @@ def reference_find(p, s): for i in range(len(s)): if s.startswith(p, i): return i + if p == '' and s == '': + return 0 return -1 - rr = random.randrange - choices = random.choices - for _ in range(1000): + def check_pattern(rr): + choices = random.choices p0 = ''.join(choices('abcde', k=rr(10))) * rr(10, 20) p = p0[:len(p0) - rr(10)] # pop off some characters left = ''.join(choices('abcdef', k=rr(2000))) @@ -341,6 +342,49 @@ def reference_find(p, s): self.checkequal(reference_find(p, text), text, 'find', p) + rr = random.randrange + for _ in range(1000): + check_pattern(rr) + + # Test that empty string always work: + check_pattern(lambda *args: 0) + + def test_find_many_lengths(self): + haystack_repeats = [a * 10**e for e in range(6) for a in (1,2,5)] + haystacks = [(n, self.fixtype("abcab"*n + "da")) for n in haystack_repeats] + + needle_repeats = [a * 10**e for e in range(6) for a in (1, 3)] + needles = [(m, self.fixtype("abcab"*m + "da")) for m in needle_repeats] + + for n, haystack1 in haystacks: + haystack2 = haystack1[:-1] + for m, needle in needles: + answer1 = 5 * (n - m) if m <= n else -1 + self.assertEqual(haystack1.find(needle), answer1, msg=(n,m)) + self.assertEqual(haystack2.find(needle), -1, msg=(n,m)) + + def test_adaptive_find(self): + # This would be very slow for the naive algorithm, + # but str.find() should be O(n + m). + for N in 1000, 10_000, 100_000, 1_000_000: + A, B = 'a' * N, 'b' * N + haystack = A + A + B + A + A + needle = A + B + B + A + self.checkequal(-1, haystack, 'find', needle) + self.checkequal(0, haystack, 'count', needle) + self.checkequal(len(haystack), haystack + needle, 'find', needle) + self.checkequal(1, haystack + needle, 'count', needle) + + def test_find_with_memory(self): + # Test the "Skip with memory" path in the two-way algorithm. + for N in 1000, 3000, 10_000, 30_000: + needle = 'ab' * N + haystack = ('ab'*(N-1) + 'b') * 2 + self.checkequal(-1, haystack, 'find', needle) + self.checkequal(0, haystack, 'count', needle) + self.checkequal(len(haystack), haystack + needle, 'find', needle) + self.checkequal(1, haystack + needle, 'count', needle) + def test_find_shift_table_overflow(self): """When the table of 8-bit shifts overflows.""" N = 2**8 + 100 @@ -724,6 +768,18 @@ def test_replace(self): self.checkraises(TypeError, 'hello', 'replace', 42, 'h') self.checkraises(TypeError, 'hello', 'replace', 'h', 42) + def test_replace_uses_two_way_maxcount(self): + # Test that maxcount works in _two_way_count in fastsearch.h + A, B = "A"*1000, "B"*1000 + AABAA = A + A + B + A + A + ABBA = A + B + B + A + self.checkequal(AABAA + ABBA, + AABAA + ABBA, 'replace', ABBA, "ccc", 0) + self.checkequal(AABAA + "ccc", + AABAA + ABBA, 'replace', ABBA, "ccc", 1) + self.checkequal(AABAA + "ccc", + AABAA + ABBA, 'replace', ABBA, "ccc", 2) + @unittest.skip("TODO: RUSTPYTHON, may only apply to 32-bit platforms") @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4, 'only applies to 32-bit platforms') @@ -734,8 +790,6 @@ def test_replace_overflow(self): self.checkraises(OverflowError, A2_16, "replace", "A", A2_16) self.checkraises(OverflowError, A2_16, "replace", "AA", A2_16+A2_16) - - # Python 3.9 def test_removeprefix(self): self.checkequal('am', 'spam', 'removeprefix', 'sp') self.checkequal('spamspam', 'spamspamspam', 'removeprefix', 'spam') @@ -754,7 +808,6 @@ def test_removeprefix(self): self.checkraises(TypeError, 'hello', 'removeprefix', 'h', 42) self.checkraises(TypeError, 'hello', 'removeprefix', ("he", "l")) - # Python 3.9 def test_removesuffix(self): self.checkequal('sp', 'spam', 'removesuffix', 'am') self.checkequal('spamspam', 'spamspamspam', 'removesuffix', 'spam') @@ -1053,7 +1106,7 @@ def test_splitlines(self): self.checkraises(TypeError, 'abc', 'splitlines', 42, 42) -class CommonTest(BaseTest): +class StringLikeTest(BaseTest): # This testcase contains tests that can be used in all # stringlike classes. Currently this is str and UserString. @@ -1066,8 +1119,6 @@ def test_hash(self): hash(b) self.assertEqual(hash(a), hash(b)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_capitalize_nonascii(self): # check that titlecased chars are lowered correctly # \u1ffc is the titlecased char @@ -1086,11 +1137,6 @@ def test_capitalize_nonascii(self): self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7', '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize') - -class MixinStrUnicodeUserStringTest: - # additional tests that only work for - # stringlike objects, i.e. str, UserString - def test_startswith(self): self.checkequal(True, 'hello', 'startswith', 'he') self.checkequal(True, 'hello', 'startswith', 'hello') @@ -1275,8 +1321,11 @@ def test_join(self): self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join', ('a' * i,) * i) - #self.checkequal(str(BadSeq1()), ' ', 'join', BadSeq1()) - self.checkequal('a b c', ' ', 'join', BadSeq2()) + class LiesAboutLengthSeq(Sequence): + def __init__(self): self.seq = ['a', 'b', 'c'] + def __len__(self): return 8 + + self.checkequal('a b c', ' ', 'join', LiesAboutLengthSeq()) self.checkraises(TypeError, ' ', 'join') self.checkraises(TypeError, ' ', 'join', None) @@ -1461,19 +1510,19 @@ def test_find_etc_raise_correct_error_messages(self): # issue 11828 s = 'hello' x = 'x' - self.assertRaisesRegex(TypeError, r'^find\(', s.find, + self.assertRaisesRegex(TypeError, r'^find\b', s.find, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rfind\(', s.rfind, + self.assertRaisesRegex(TypeError, r'^rfind\b', s.rfind, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^index\(', s.index, + self.assertRaisesRegex(TypeError, r'^index\b', s.index, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rindex\(', s.rindex, + self.assertRaisesRegex(TypeError, r'^rindex\b', s.rindex, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^count\(', s.count, + self.assertRaisesRegex(TypeError, r'^count\b', s.count, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^startswith\(', s.startswith, + self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith, + self.assertRaisesRegex(TypeError, r'^endswith\b', s.endswith, x, None, None, None) # issue #15534 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 975ff211011..6b3f2c447e8 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3,11 +3,12 @@ if __name__ != 'test.support': raise ImportError('support must be imported from the test package') +import annotationlib import contextlib -import dataclasses import functools -import getpass -import opcode +import inspect +import logging +import _opcode import os import re import stat @@ -19,8 +20,6 @@ import unittest import warnings -from .testresult import get_test_runner - __all__ = [ # globals @@ -29,13 +28,13 @@ "Error", "TestFailed", "TestDidNotRun", "ResourceDenied", # io "record_original_stdout", "get_original_stdout", "captured_stdout", - "captured_stdin", "captured_stderr", + "captured_stdin", "captured_stderr", "captured_output", # unittest - "is_resource_enabled", "requires", "requires_freebsd_version", - "requires_linux_version", "requires_mac_ver", + "is_resource_enabled", "get_resource_value", "requires", "requires_resource", + "requires_freebsd_version", + "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", - "run_unittest", "run_doctest", - "requires_gzip", "requires_bz2", "requires_lzma", + "requires_gzip", "requires_bz2", "requires_lzma", "requires_zstd", "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "requires_zlib", "has_fork_support", "requires_fork", @@ -44,10 +43,12 @@ "anticipate_failure", "load_package_tests", "detect_api_mismatch", "check__all__", "skip_if_buggy_ucrt_strfptime", "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", - "requires_limited_api", "requires_specialization", + "requires_limited_api", "requires_specialization", "thread_unsafe", + "skip_if_unlimited_stack_size", # sys - "is_jython", "is_android", "is_emscripten", "is_wasi", - "check_impl_detail", "unix_shell", "setswitchinterval", + "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", + "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", + "support_remote_exec_only", # os "get_pagesize", # network @@ -60,8 +61,16 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", - "skip_on_s390x", + "Py_DEBUG", "exceeds_recursion_limit", "skip_on_s390x", + "requires_jit_enabled", + "requires_jit_disabled", + "force_not_colorized", + "force_not_colorized_test_class", + "make_clean_env", + "BrokenIter", + "in_systemd_nspawn_sync_suppressed", + "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", + "reset_code", "on_github_actions" ] @@ -74,13 +83,7 @@ # # The timeout should be long enough for connect(), recv() and send() methods # of socket.socket. -LOOPBACK_TIMEOUT = 5.0 -if sys.platform == 'win32' and ' 32 bit (ARM)' in sys.version: - # bpo-37553: test_socket.SendfileUsingSendTest is taking longer than 2 - # seconds on Windows ARM32 buildbot - LOOPBACK_TIMEOUT = 10 -elif sys.platform == 'vxworks': - LOOPBACK_TIMEOUT = 10 +LOOPBACK_TIMEOUT = 10.0 # Timeout in seconds for network requests going to the internet. The timeout is # short enough to prevent a test to wait for too long if the internet request @@ -183,7 +186,7 @@ def get_attribute(obj, name): return attribute verbose = 1 # Flag set to 0 by regrtest.py -use_resources = None # Flag set to [] by regrtest.py +use_resources = None # Flag set to {} by regrtest.py max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 @@ -259,22 +262,16 @@ class USEROBJECTFLAGS(ctypes.Structure): # process not running under the same user id as the current console # user. To avoid that, raise an exception if the window manager # connection is not available. - from ctypes import cdll, c_int, pointer, Structure - from ctypes.util import find_library - - app_services = cdll.LoadLibrary(find_library("ApplicationServices")) - - if app_services.CGMainDisplayID() == 0: - reason = "gui tests cannot run without OS X window manager" + import subprocess + try: + rc = subprocess.run(["launchctl", "managername"], + capture_output=True, check=True) + managername = rc.stdout.decode("utf-8").strip() + except subprocess.CalledProcessError: + reason = "unable to detect macOS launchd job manager" else: - class ProcessSerialNumber(Structure): - _fields_ = [("highLongOfPSN", c_int), - ("lowLongOfPSN", c_int)] - psn = ProcessSerialNumber() - psn_p = pointer(psn) - if ( (app_services.GetCurrentProcess(psn_p) < 0) or - (app_services.SetFrontProcess(psn_p) < 0) ): - reason = "cannot run without OS X gui process" + if managername != "Aqua": + reason = f"{managername=} -- can only run in a macOS GUI session" # check on every platform whether tkinter can actually do anything if not reason: @@ -304,6 +301,16 @@ def is_resource_enabled(resource): """ return use_resources is None or resource in use_resources +def get_resource_value(resource): + """Test whether a resource is enabled. + + Known resources are set by regrtest.py. If not running under regrtest.py, + all resources are assumed enabled unless use_resources has been set. + """ + if use_resources is None: + return None + return use_resources.get(resource) + def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available.""" if not is_resource_enabled(resource): @@ -315,6 +322,16 @@ def requires(resource, msg=None): if resource == 'gui' and not _is_gui_available(): raise ResourceDenied(_is_gui_available.reason) +def _get_kernel_version(sysname="Linux"): + import platform + if platform.system() != sysname: + return None + version_txt = platform.release().split('-', 1)[0] + try: + return tuple(map(int, version_txt.split('.'))) + except ValueError: + return None + def _requires_unix_version(sysname, min_version): """Decorator raising SkipTest if the OS is `sysname` and the version is less than `min_version`. @@ -389,51 +406,89 @@ def wrapper(*args, **kw): return decorator +def thread_unsafe(reason): + """Mark a test as not thread safe. When the test runner is run with + --parallel-threads=N, the test will be run in a single thread.""" + def decorator(test_item): + test_item.__unittest_thread_unsafe__ = True + # the reason is not currently used + test_item.__unittest_thread_unsafe__why__ = reason + return test_item + if isinstance(reason, types.FunctionType): + test_item = reason + reason = '' + return decorator(test_item) + return decorator + + def skip_if_buildbot(reason=None): """Decorator raising SkipTest if running on a buildbot.""" + import getpass if not reason: reason = 'not suitable for buildbots' try: isbuildbot = getpass.getuser().lower() == 'buildbot' - except (KeyError, EnvironmentError) as err: - warnings.warn(f'getpass.getuser() failed {err}.', RuntimeWarning) + except (KeyError, OSError) as err: + logging.getLogger(__name__).warning('getpass.getuser() failed %s.', err, exc_info=err) isbuildbot = False return unittest.skipIf(isbuildbot, reason) -def check_sanitizer(*, address=False, memory=False, ub=False): +def check_sanitizer(*, address=False, memory=False, ub=False, thread=False, + function=True): """Returns True if Python is compiled with sanitizer support""" - if not (address or memory or ub): - raise ValueError('At least one of address, memory, or ub must be True') + if not (address or memory or ub or thread): + raise ValueError('At least one of address, memory, ub or thread must be True') - _cflags = sysconfig.get_config_var('CFLAGS') or '' - _config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + cflags = sysconfig.get_config_var('CFLAGS') or '' + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' memory_sanitizer = ( - '-fsanitize=memory' in _cflags or - '--with-memory-sanitizer' in _config_args + '-fsanitize=memory' in cflags or + '--with-memory-sanitizer' in config_args ) address_sanitizer = ( - '-fsanitize=address' in _cflags or - '--with-address-sanitizer' in _config_args + '-fsanitize=address' in cflags or + '--with-address-sanitizer' in config_args ) ub_sanitizer = ( - '-fsanitize=undefined' in _cflags or - '--with-undefined-behavior-sanitizer' in _config_args + '-fsanitize=undefined' in cflags or + '--with-undefined-behavior-sanitizer' in config_args + ) + thread_sanitizer = ( + '-fsanitize=thread' in cflags or + '--with-thread-sanitizer' in config_args + ) + function_sanitizer = ( + '-fsanitize=function' in cflags ) return ( (memory and memory_sanitizer) or (address and address_sanitizer) or - (ub and ub_sanitizer) + (ub and ub_sanitizer) or + (thread and thread_sanitizer) or + (function and function_sanitizer) ) -def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): +def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False, thread=False): """Decorator raising SkipTest if running with a sanitizer active.""" if not reason: reason = 'not working with sanitizers active' - skip = check_sanitizer(address=address, memory=memory, ub=ub) + skip = check_sanitizer(address=address, memory=memory, ub=ub, thread=thread) return unittest.skipIf(skip, reason) +# gh-89363: True if fork() can hang if Python is built with Address Sanitizer +# (ASAN): libasan race condition, dead lock in pthread_create(). +HAVE_ASAN_FORK_BUG = check_sanitizer(address=True) + + +def set_sanitizer_env_var(env, option): + for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS', 'TSAN_OPTIONS'): + if name in env: + env[name] += f':{option}' + else: + env[name] = option + def system_must_validate_cert(f): """Skip the test on TLS certificate validation failures.""" @@ -493,34 +548,45 @@ def requires_lzma(reason='requires lzma'): import lzma except ImportError: lzma = None + lzma = None # XXX: RUSTPYTHON; xz is not supported yet return unittest.skipUnless(lzma, reason) +def requires_zstd(reason='requires zstd'): + try: + from compression import zstd + except ImportError: + zstd = None + return unittest.skipUnless(zstd, reason) + def has_no_debug_ranges(): try: - import _testinternalcapi + import _testcapi except ImportError: raise unittest.SkipTest("_testinternalcapi required") - config = _testinternalcapi.get_config() - return not bool(config['code_debug_ranges']) + return not _testcapi.config_get('code_debug_ranges') def requires_debug_ranges(reason='requires co_positions / debug_ranges'): - return unittest.skipIf(has_no_debug_ranges(), reason) - -def requires_legacy_unicode_capi(): try: - from _testcapi import unicode_legacy_string - except ImportError: - unicode_legacy_string = None + skip = has_no_debug_ranges() + except unittest.SkipTest as e: + skip = True + reason = e.args[0] if e.args else reason + return unittest.skipIf(skip, reason) - return unittest.skipUnless(unicode_legacy_string, - 'requires legacy Unicode C API') + +MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') -is_android = hasattr(sys, 'getandroidapilevel') +is_android = sys.platform == "android" -if sys.platform not in ('win32', 'vxworks'): +def skip_android_selinux(name): + return unittest.skipIf( + sys.platform == "android", f"Android blocks {name} with SELinux" + ) + +if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' else: unix_shell = None @@ -530,19 +596,53 @@ def requires_legacy_unicode_capi(): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi +# Use is_wasm32 as a generic check for WebAssembly platforms. +is_wasm32 = is_emscripten or is_wasi + +def skip_emscripten_stack_overflow(): + return unittest.skipIf(is_emscripten, "Exhausts stack on Emscripten") + +def skip_wasi_stack_overflow(): + return unittest.skipIf(is_wasi, "Exhausts stack on WASI") + +is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} +is_apple = is_apple_mobile or sys.platform == "darwin" + +has_fork_support = hasattr(os, "fork") and not ( + # WASM and Apple mobile platforms do not support subprocesses. + is_emscripten + or is_wasi + or is_apple_mobile + + # Although Android supports fork, it's unsafe to call it from Python because + # all Android apps are multi-threaded. + or is_android +) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") -has_subprocess_support = not is_emscripten and not is_wasi +has_subprocess_support = not ( + # WASM and Apple mobile platforms do not support subprocesses. + is_emscripten + or is_wasi + or is_apple_mobile + + # Although Android supports subproceses, they're almost never useful in + # practice (see PEP 738). And most of the tests that use them are calling + # sys.executable, which won't work when Python is embedded in an Android app. + or is_android +) def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" return unittest.skipUnless(has_subprocess_support, "requires subprocess support") # Emscripten's socket emulation and WASI sockets have limitations. -has_socket_support = not is_emscripten and not is_wasi +has_socket_support = not ( + is_emscripten + or is_wasi +) def requires_working_socket(*, module=False): """Skip tests or modules that require working sockets @@ -627,9 +727,11 @@ def sortdict(dict): return "{%s}" % withcommas -def run_code(code: str) -> dict[str, object]: +def run_code(code: str, extra_names: dict[str, object] | None = None) -> dict[str, object]: """Run a piece of code after dedenting it, and return its global namespace.""" ns = {} + if extra_names: + ns.update(extra_names) exec(textwrap.dedent(code), ns) return ns @@ -754,28 +856,40 @@ def gc_collect(): longer than expected. This function tries its best to force all garbage objects to disappear. """ - # TODO: RUSTPYTHON (comment out before) - # import gc - # gc.collect() - # if is_jython: - # time.sleep(0.1) - # gc.collect() - # gc.collect() - pass + return # TODO: RUSTPYTHON + + import gc + gc.collect() + gc.collect() + gc.collect() @contextlib.contextmanager def disable_gc(): - # TODO: RUSTPYTHON (comment out before) - # import gc - # have_gc = gc.isenabled() - # gc.disable() - # try: - # yield - # finally: - # if have_gc: - # gc.enable() - yield + # TODO: RUSTPYTHON; GC is not supported yet + try: + yield + finally: + pass + return + import gc + have_gc = gc.isenabled() + gc.disable() + try: + yield + finally: + if have_gc: + gc.enable() + +@contextlib.contextmanager +def gc_threshold(*args): + import gc + old_threshold = gc.get_threshold() + gc.set_threshold(*args) + try: + yield + finally: + gc.set_threshold(*old_threshold) def python_is_optimized(): """Find if Python was built with optimizations.""" @@ -784,14 +898,57 @@ def python_is_optimized(): for opt in cflags.split(): if opt.startswith('-O'): final_opt = opt - return final_opt not in ('', '-O0', '-Og') + if sysconfig.get_config_var("CC") == "gcc": + non_opts = ('', '-O0', '-Og') + else: + non_opts = ('', '-O0') + return final_opt not in non_opts + + +def check_cflags_pgo(): + # Check if Python was built with ./configure --enable-optimizations: + # with Profile Guided Optimization (PGO). + cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' + pgo_options = [ + # GCC + '-fprofile-use', + # clang: -fprofile-instr-use=code.profclangd + '-fprofile-instr-use', + # ICC + "-prof-use", + ] + PGO_PROF_USE_FLAG = sysconfig.get_config_var('PGO_PROF_USE_FLAG') + if PGO_PROF_USE_FLAG: + pgo_options.append(PGO_PROF_USE_FLAG) + return any(option in cflags_nodist for option in pgo_options) -_header = 'nP' +def check_bolt_optimized(): + # Always return false, if the platform is WASI, + # because BOLT optimization does not support WASM binary. + if is_wasi: + return False + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + return '--enable-bolt' in config_args + + +Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) + +def requires_gil_enabled(msg="needs the GIL enabled"): + """Decorator for skipping tests on the free-threaded build.""" + return unittest.skipIf(Py_GIL_DISABLED, msg) + +def expected_failure_if_gil_disabled(): + """Expect test failure if the GIL is disabled.""" + if Py_GIL_DISABLED: + return unittest.expectedFailure + return lambda test_case: test_case + +if Py_GIL_DISABLED: + _header = 'PHBBInP' +else: + _header = 'nP' _align = '0n' -if hasattr(sys, "getobjects"): - _header = '2P' + _header - _align = '0P' _vheader = _header + 'n' def calcobjsize(fmt): @@ -803,8 +960,16 @@ def calcvobjsize(fmt): return struct.calcsize(_vheader + fmt + _align) -_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_STATIC_BUILTIN = 1<<1 +_TPFLAGS_DISALLOW_INSTANTIATION = 1<<7 +_TPFLAGS_IMMUTABLETYPE = 1<<8 _TPFLAGS_HEAPTYPE = 1<<9 +_TPFLAGS_BASETYPE = 1<<10 +_TPFLAGS_READY = 1<<12 +_TPFLAGS_READYING = 1<<13 +_TPFLAGS_HAVE_GC = 1<<14 +_TPFLAGS_BASE_EXC_SUBCLASS = 1<<30 +_TPFLAGS_TYPE_SUBCLASS = 1<<31 def check_sizeof(test, o, size): try: @@ -820,9 +985,34 @@ def check_sizeof(test, o, size): % (type(o), result, size) test.assertEqual(result, size, msg) +def subTests(arg_names, arg_values, /, *, _do_cleanups=False): + """Run multiple subtests with different parameters. + """ + single_param = False + if isinstance(arg_names, str): + arg_names = arg_names.replace(',',' ').split() + if len(arg_names) == 1: + single_param = True + arg_values = tuple(arg_values) + def decorator(func): + if isinstance(func, type): + raise TypeError('subTests() can only decorate methods, not classes') + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + for values in arg_values: + if single_param: + values = (values,) + subtest_kwargs = dict(zip(arg_names, values)) + with self.subTest(**subtest_kwargs): + func(self, *args, **kwargs, **subtest_kwargs) + if _do_cleanups: + self.doCleanups() + return wrapper + return decorator + #======================================================================= -# Decorator for running a function in a different locale, correctly resetting -# it afterwards. +# Decorator/context manager for running a code in a different locale, +# correctly resetting it afterwards. @contextlib.contextmanager def run_with_locale(catstr, *locales): @@ -833,16 +1023,21 @@ def run_with_locale(catstr, *locales): except AttributeError: # if the test author gives us an invalid category string raise - except: + except Exception: # cannot retrieve original locale, so do nothing locale = orig_locale = None + if '' not in locales: + raise unittest.SkipTest('no locales') else: for loc in locales: try: locale.setlocale(category, loc) break - except: + except locale.Error: pass + else: + if '' not in locales: + raise unittest.SkipTest(f'no locales {locales}') try: yield @@ -850,6 +1045,46 @@ def run_with_locale(catstr, *locales): if locale and orig_locale: locale.setlocale(category, orig_locale) +#======================================================================= +# Decorator for running a function in multiple locales (if they are +# availasble) and resetting the original locale afterwards. + +def run_with_locales(catstr, *locales): + def deco(func): + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + dry_run = '' in locales + try: + import locale + category = getattr(locale, catstr) + orig_locale = locale.setlocale(category) + except AttributeError: + # if the test author gives us an invalid category string + raise + except Exception: + # cannot retrieve original locale, so do nothing + pass + else: + try: + for loc in locales: + with self.subTest(locale=loc): + try: + locale.setlocale(category, loc) + except locale.Error: + self.skipTest(f'no locale {loc!r}') + else: + dry_run = False + func(self, *args, **kwargs) + finally: + locale.setlocale(category, orig_locale) + if dry_run: + # no locales available, so just run the test + # with the current locale + with self.subTest(locale=None): + func(self, *args, **kwargs) + return wrapper + return deco + #======================================================================= # Decorator for running a function in a specific timezone, correctly # resetting it afterwards. @@ -896,27 +1131,31 @@ def inner(*args, **kwds): MAX_Py_ssize_t = sys.maxsize -def set_memlimit(limit): - global max_memuse - global real_max_memuse +def _parse_memlimit(limit: str) -> int: sizes = { 'k': 1024, 'm': _1M, 'g': _1G, 't': 1024*_1G, } - m = re.match(r'(\d+(\.\d+)?) (K|M|G|T)b?$', limit, + m = re.match(r'(\d+(?:\.\d+)?) (K|M|G|T)b?$', limit, re.IGNORECASE | re.VERBOSE) if m is None: - raise ValueError('Invalid memory limit %r' % (limit,)) - memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()]) - real_max_memuse = memlimit - if memlimit > MAX_Py_ssize_t: - memlimit = MAX_Py_ssize_t + raise ValueError(f'Invalid memory limit: {limit!r}') + return int(float(m.group(1)) * sizes[m.group(2).lower()]) + +def set_memlimit(limit: str) -> None: + global max_memuse + global real_max_memuse + memlimit = _parse_memlimit(limit) if memlimit < _2G - 1: - raise ValueError('Memory limit %r too low to be useful' % (limit,)) + raise ValueError(f'Memory limit {limit!r} too low to be useful') + + real_max_memuse = memlimit + memlimit = min(memlimit, MAX_Py_ssize_t) max_memuse = memlimit + class _MemoryWatchdog: """An object which periodically watches the process' memory consumption and prints it out. @@ -931,8 +1170,7 @@ def start(self): try: f = open(self.procfile, 'r') except OSError as e: - warnings.warn('/proc not available for stats: {}'.format(e), - RuntimeWarning) + logging.getLogger(__name__).warning('/proc not available for stats: %s', e, exc_info=e) sys.stderr.flush() return @@ -1069,18 +1307,50 @@ def check_impl_detail(**guards): def no_tracing(func): """Decorator to temporarily turn off tracing for the duration of a test.""" - if not hasattr(sys, 'gettrace'): - return func - else: + trace_wrapper = func + if hasattr(sys, 'gettrace'): @functools.wraps(func) - def wrapper(*args, **kwargs): + def trace_wrapper(*args, **kwargs): original_trace = sys.gettrace() try: sys.settrace(None) return func(*args, **kwargs) finally: sys.settrace(original_trace) + + coverage_wrapper = trace_wrapper + if 'test.cov' in sys.modules: # -Xpresite=test.cov used + cov = sys.monitoring.COVERAGE_ID + @functools.wraps(func) + def coverage_wrapper(*args, **kwargs): + original_events = sys.monitoring.get_events(cov) + try: + sys.monitoring.set_events(cov, 0) + return trace_wrapper(*args, **kwargs) + finally: + sys.monitoring.set_events(cov, original_events) + + return coverage_wrapper + + +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + assert not isinstance(func, type), func + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True return wrapper + return deco def refcount_test(test): @@ -1096,185 +1366,33 @@ def refcount_test(test): def requires_limited_api(test): try: - import _testcapi + import _testcapi # noqa: F401 + import _testlimitedcapi # noqa: F401 except ImportError: - return unittest.skip('needs _testcapi module')(test) - return unittest.skipUnless( - _testcapi.LIMITED_API_AVAILABLE, 'needs Limited API support')(test) - -def requires_specialization(test): - return unittest.skipUnless( - opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) - -def _filter_suite(suite, pred): - """Recursively filter test cases in a suite based on a predicate.""" - newtests = [] - for test in suite._tests: - if isinstance(test, unittest.TestSuite): - _filter_suite(test, pred) - newtests.append(test) - else: - if pred(test): - newtests.append(test) - suite._tests = newtests - -@dataclasses.dataclass(slots=True) -class TestStats: - tests_run: int = 0 - failures: int = 0 - skipped: int = 0 - - @staticmethod - def from_unittest(result): - return TestStats(result.testsRun, - len(result.failures), - len(result.skipped)) - - @staticmethod - def from_doctest(results): - return TestStats(results.attempted, - results.failed) - - def accumulate(self, stats): - self.tests_run += stats.tests_run - self.failures += stats.failures - self.skipped += stats.skipped - - -def _run_suite(suite): - """Run tests from a unittest.TestSuite-derived class.""" - runner = get_test_runner(sys.stdout, - verbosity=verbose, - capture_output=(junit_xml_list is not None)) - - result = runner.run(suite) - - if junit_xml_list is not None: - junit_xml_list.append(result.get_xml_element()) - - if not result.testsRun and not result.skipped and not result.errors: - raise TestDidNotRun - if not result.wasSuccessful(): - stats = TestStats.from_unittest(result) - if len(result.errors) == 1 and not result.failures: - err = result.errors[0][1] - elif len(result.failures) == 1 and not result.errors: - err = result.failures[0][1] - else: - err = "multiple errors occurred" - if not verbose: err += "; run in verbose mode for details" - errors = [(str(tc), exc_str) for tc, exc_str in result.errors] - failures = [(str(tc), exc_str) for tc, exc_str in result.failures] - raise TestFailedWithDetails(err, errors, failures, stats=stats) - return result - - -# By default, don't filter tests -_match_test_func = None - -_accept_test_patterns = None -_ignore_test_patterns = None + return unittest.skip('needs _testcapi and _testlimitedcapi modules')(test) + return test -def match_test(test): - # Function used by support.run_unittest() and regrtest --list-cases - if _match_test_func is None: - return True - else: - return _match_test_func(test.id()) +# Windows build doesn't support --disable-test-modules feature, so there's no +# 'TEST_MODULES' var in config +TEST_MODULES_ENABLED = (sysconfig.get_config_var('TEST_MODULES') or 'yes') == 'yes' +def requires_specialization(test): + return unittest.skipUnless( + _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) -def _is_full_match_test(pattern): - # If a pattern contains at least one dot, it's considered - # as a full test identifier. - # Example: 'test.test_os.FileTests.test_access'. - # - # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' - # or '[!...]'. For example, ignore 'test_access*'. - return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) - - -def set_match_tests(accept_patterns=None, ignore_patterns=None): - global _match_test_func, _accept_test_patterns, _ignore_test_patterns - - if accept_patterns is None: - accept_patterns = () - if ignore_patterns is None: - ignore_patterns = () - - accept_func = ignore_func = None - - if accept_patterns != _accept_test_patterns: - accept_patterns, accept_func = _compile_match_function(accept_patterns) - if ignore_patterns != _ignore_test_patterns: - ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) - - # Create a copy since patterns can be mutable and so modified later - _accept_test_patterns = tuple(accept_patterns) - _ignore_test_patterns = tuple(ignore_patterns) - - if accept_func is not None or ignore_func is not None: - def match_function(test_id): - accept = True - ignore = False - if accept_func: - accept = accept_func(test_id) - if ignore_func: - ignore = ignore_func(test_id) - return accept and not ignore - - _match_test_func = match_function - - -def _compile_match_function(patterns): - if not patterns: - func = None - # set_match_tests(None) behaves as set_match_tests(()) - patterns = () - elif all(map(_is_full_match_test, patterns)): - # Simple case: all patterns are full test identifier. - # The test.bisect_cmd utility only uses such full test identifiers. - func = set(patterns).__contains__ - else: - import fnmatch - regex = '|'.join(map(fnmatch.translate, patterns)) - # The search *is* case sensitive on purpose: - # don't use flags=re.IGNORECASE - regex_match = re.compile(regex).match - - def match_test_regex(test_id): - if regex_match(test_id): - # The regex matches the whole identifier, for example - # 'test.test_os.FileTests.test_access'. - return True - else: - # Try to match parts of the test identifier. - # For example, split 'test.test_os.FileTests.test_access' - # into: 'test', 'test_os', 'FileTests' and 'test_access'. - return any(map(regex_match, test_id.split("."))) - func = match_test_regex +def requires_specialization_ft(test): + return unittest.skipUnless( + _opcode.ENABLE_SPECIALIZATION_FT, "requires specialization")(test) - return patterns, func +def reset_code(f: types.FunctionType) -> types.FunctionType: + """Clear all specializations, local instrumentation, and JIT code for the given function.""" + f.__code__ = f.__code__.replace() + return f -def run_unittest(*classes): - """Run tests from unittest.TestCase-derived classes.""" - valid_types = (unittest.TestSuite, unittest.TestCase) - loader = unittest.TestLoader() - suite = unittest.TestSuite() - for cls in classes: - if isinstance(cls, str): - if cls in sys.modules: - suite.addTest(loader.loadTestsFromModule(sys.modules[cls])) - else: - raise ValueError("str arguments must be keys in sys.modules") - elif isinstance(cls, valid_types): - suite.addTest(cls) - else: - suite.addTest(loader.loadTestsFromTestCase(cls)) - _filter_suite(suite, match_test) - return _run_suite(suite) +on_github_actions = "GITHUB_ACTIONS" in os.environ #======================================================================= # Check for the presence of docstrings. @@ -1289,45 +1407,13 @@ def _check_docstrings(): sys.platform != 'win32' and not sysconfig.get_config_var('WITH_DOC_STRINGS')) -HAVE_DOCSTRINGS = (_check_docstrings.__doc__ is not None and - not MISSING_C_DOCSTRINGS) +HAVE_PY_DOCSTRINGS = _check_docstrings.__doc__ is not None +HAVE_DOCSTRINGS = (HAVE_PY_DOCSTRINGS and not MISSING_C_DOCSTRINGS) requires_docstrings = unittest.skipUnless(HAVE_DOCSTRINGS, "test requires docstrings") -#======================================================================= -# doctest driver. - -def run_doctest(module, verbosity=None, optionflags=0): - """Run doctest on the given module. Return (#failures, #tests). - - If optional argument verbosity is not specified (or is None), pass - support's belief about verbosity on to doctest. Else doctest's - usual behavior is used (it searches sys.argv for -v). - """ - - import doctest - - if verbosity is None: - verbosity = verbose - else: - verbosity = None - - results = doctest.testmod(module, - verbose=verbosity, - optionflags=optionflags) - if results.failed: - stats = TestStats.from_doctest(results) - raise TestFailed(f"{results.failed} of {results.attempted} " - f"doctests failed", - stats=stats) - if verbose: - print('doctest (%s) ... %d tests with zero failures' % - (module.__name__, results.attempted)) - return results - - #======================================================================= # Support for saving and restoring the imported modules. @@ -1609,6 +1695,25 @@ def skip_if_pgo_task(test): return test if ok else unittest.skip(msg)(test) +def skip_if_unlimited_stack_size(test): + """Skip decorator for tests not run when an unlimited stack size is configured. + + Tests using support.infinite_recursion([...]) may otherwise run into + an infinite loop, running until the memory on the system is filled and + crashing due to OOM. + + See https://github.com/python/cpython/issues/143460. + """ + if is_emscripten or is_wasi or os.name == "nt": + return test + + import resource + curlim, maxlim = resource.getrlimit(resource.RLIMIT_STACK) + unlimited_stack_size_cond = curlim == maxlim and curlim in (-1, 0xFFFF_FFFF_FFFF_FFFF) + reason = "Not run due to unlimited stack size" + return unittest.skipIf(unlimited_stack_size_cond, reason)(test) + + def detect_api_mismatch(ref_api, other_api, *, ignore=()): """Returns the set of items in ref_api not in other_api, except for a defined list of items to be ignored in this check. @@ -1633,7 +1738,7 @@ def check__all__(test_case, module, name_of_module=None, extra=(), 'module'. The 'name_of_module' argument can specify (as a string or tuple thereof) - what module(s) an API could be defined in in order to be detected as a + what module(s) an API could be defined in order to be detected as a public API. One case for this is when 'module' imports part of its public API from other modules, possibly a C backend (like 'csv' and its '_csv'). @@ -1850,7 +1955,10 @@ def run_in_subinterp(code): module is enabled. """ _check_tracemalloc() - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.run_in_subinterp(code) @@ -1860,11 +1968,25 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): module is enabled. """ _check_tracemalloc() - import _testcapi + try: + import _testinternalcapi + except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + elif not isinstance(gil, str): + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): @@ -1881,24 +2003,30 @@ def _check_tracemalloc(): "memory allocations") -# TODO: RUSTPYTHON (comment out before) -# def check_free_after_iterating(test, iter, cls, args=()): -# class A(cls): -# def __del__(self): -# nonlocal done -# done = True -# try: -# next(it) -# except StopIteration: -# pass +def check_free_after_iterating(test, iter, cls, args=()): + # TODO: RUSTPYTHON; GC is not supported yet + test.assertTrue(False) + return + + done = False + def wrapper(): + class A(cls): + def __del__(self): + nonlocal done + done = True + try: + next(it) + except StopIteration: + pass + + it = iter(A(*args)) + # Issue 26494: Shouldn't crash + test.assertRaises(StopIteration, next, it) -# done = False -# it = iter(A(*args)) -# # Issue 26494: Shouldn't crash -# test.assertRaises(StopIteration, next, it) -# # The sequence should be deallocated just after the end of iterating -# gc_collect() -# test.assertTrue(done) + wrapper() + # The sequence should be deallocated just after the end of iterating + gc_collect() + test.assertTrue(done) def missing_compiler_executable(cmd_names=[]): @@ -1910,8 +2038,9 @@ def missing_compiler_executable(cmd_names=[]): missing. """ - from setuptools._distutils import ccompiler, sysconfig, spawn + from setuptools._distutils import ccompiler, sysconfig from setuptools import errors + import shutil compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) @@ -1930,22 +2059,22 @@ def missing_compiler_executable(cmd_names=[]): "the '%s' executable is not configured" % name elif not cmd: continue - if spawn.find_executable(cmd[0]) is None: + if shutil.which(cmd[0]) is None: return cmd[0] -_is_android_emulator = None +_old_android_emulator = None def setswitchinterval(interval): # Setting a very low gil interval on the Android emulator causes python # to hang (issue #26939). - minimum_interval = 1e-5 + minimum_interval = 1e-4 # 100 us if is_android and interval < minimum_interval: - global _is_android_emulator - if _is_android_emulator is None: - import subprocess - _is_android_emulator = (subprocess.check_output( - ['getprop', 'ro.kernel.qemu']).strip() == b'1') - if _is_android_emulator: + global _old_android_emulator + if _old_android_emulator is None: + import platform + av = platform.android_ver() + _old_android_emulator = av.is_emulator and av.api_level < 24 + if _old_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) @@ -2020,8 +2149,19 @@ def restore(self): def with_pymalloc(): - import _testcapi - return _testcapi.WITH_PYMALLOC + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") + return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED + + +def with_mimalloc(): + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") + return _testcapi.WITH_MIMALLOC class _ALWAYS_EQ: @@ -2230,7 +2370,15 @@ def skip_if_broken_multiprocessing_synchronize(): # bpo-38377: On Linux, creating a semaphore fails with OSError # if the current user does not have the permission to create # a file in /dev/shm/ directory. - synchronize.Lock(ctx=None) + import multiprocessing + synchronize.Lock(ctx=multiprocessing.get_context('fork')) + # The explicit fork mp context is required in order for + # TestResourceTracker.test_resource_tracker_reused to work. + # synchronize creates a new multiprocessing.resource_tracker + # process at module import time via the above call in that + # scenario. Awkward. This enables gh-84559. No code involved + # should have threads at that point so fork() should be safe. + except OSError as exc: raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}") @@ -2249,6 +2397,7 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): qualname = f"{name}" msg = f"cannot create '{re.escape(qualname)}' instances" testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) + testcase.assertRaisesRegex(TypeError, msg, tp.__new__, tp, *args, **kwds) def get_recursion_depth(): """Get the recursion depth of the caller function. @@ -2293,14 +2442,14 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) -def infinite_recursion(max_depth=100): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" - if max_depth < 3: - raise ValueError("max_depth must be at least 3, got {max_depth}") +def infinite_recursion(max_depth=None): + if max_depth is None: + # Pick a number large enough to cause problems + # but not take too long for code that can handle + # very deep recursion. + max_depth = 20_000 + elif max_depth < 3: + raise ValueError(f"max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. limit = depth + max_depth @@ -2321,8 +2470,9 @@ def clear_ignored_deprecations(*tokens: object) -> None: raise ValueError("Provide token or tokens returned by ignore_deprecations_from") new_filters = [] + old_filters = warnings._get_filters() endswith = tuple(rf"(?#support{id(token)})" for token in tokens) - for action, message, category, module, lineno in warnings.filters: + for action, message, category, module, lineno in old_filters: if action == "ignore" and category is DeprecationWarning: if isinstance(message, re.Pattern): msg = message.pattern @@ -2331,8 +2481,8 @@ def clear_ignored_deprecations(*tokens: object) -> None: if msg.endswith(endswith): continue new_filters.append((action, message, category, module, lineno)) - if warnings.filters != new_filters: - warnings.filters[:] = new_filters + if old_filters != new_filters: + old_filters[:] = new_filters warnings._filters_mutated() @@ -2340,7 +2490,7 @@ def clear_ignored_deprecations(*tokens: object) -> None: def requires_venv_with_pip(): # ensurepip requires zlib to open ZIP archives (.whl binary wheel packages) try: - import zlib + import zlib # noqa: F401 except ImportError: return unittest.skipIf(True, "venv: ensurepip requires zlib") @@ -2360,11 +2510,13 @@ def _findwheel(pkgname): If set, the wheels are searched for in WHEEL_PKG_DIR (see ensurepip). Otherwise, they are searched for in the test directory. """ - wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or TEST_HOME_DIR + wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or os.path.join( + TEST_HOME_DIR, 'wheeldata', + ) filenames = os.listdir(wheel_dir) filenames = sorted(filenames, reverse=True) # approximate "newest" first for filename in filenames: - # filename is like 'setuptools-67.6.1-py3-none-any.whl' + # filename is like 'setuptools-{version}-py3-none-any.whl' if not filename.endswith(".whl"): continue prefix = pkgname + '-' @@ -2373,20 +2525,29 @@ def _findwheel(pkgname): raise FileNotFoundError(f"No wheel for {pkgname} found in {wheel_dir}") -# Context manager that creates a virtual environment, install setuptools and wheel in it -# and returns the path to the venv directory and the path to the python executable +# Context manager that creates a virtual environment, install setuptools in it, +# and returns the paths to the venv directory and the python executable @contextlib.contextmanager -def setup_venv_with_pip_setuptools_wheel(venv_dir): +def setup_venv_with_pip_setuptools(venv_dir): import subprocess from .os_helper import temp_cwd + def run_command(cmd): + if verbose: + import shlex + print() + print('Run:', ' '.join(map(shlex.quote, cmd))) + subprocess.run(cmd, check=True) + else: + subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True) + with temp_cwd() as temp_dir: # Create virtual environment to get setuptools cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) venv = os.path.join(temp_dir, venv_dir) @@ -2397,14 +2558,11 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): else: python = os.path.join(venv, 'bin', python_exe) - cmd = [python, '-X', 'dev', + cmd = (python, '-X', 'dev', '-m', 'pip', 'install', _findwheel('setuptools'), - _findwheel('wheel')] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + ) + run_command(cmd) yield python @@ -2527,6 +2685,46 @@ def sleeping_retry(timeout, err_msg=None, /, delay = min(delay * 2, max_delay) +class Stopwatch: + """Context manager to roughly time a CPU-bound operation. + + Disables GC. Uses perf_counter, which is a clock with the highest + available resolution. It is chosen even though it does include + time elapsed during sleep and is system-wide, because the + resolution of process_time is too coarse on Windows and + process_time does not exist everywhere (for example, WASM). + + Note: + - This *includes* time spent in other threads/processes. + - Some systems only have a coarse resolution; check + stopwatch.clock_info.resolution when using the results. + + Usage: + + with Stopwatch() as stopwatch: + ... + elapsed = stopwatch.seconds + resolution = stopwatch.clock_info.resolution + """ + def __enter__(self): + get_time = time.perf_counter + clock_info = time.get_clock_info('perf_counter') + self.context = disable_gc() + self.context.__enter__() + self.get_time = get_time + self.clock_info = clock_info + self.start_time = get_time() + return self + + def __exit__(self, *exc): + try: + end_time = self.get_time() + finally: + result = self.context.__exit__(*exc) + self.seconds = end_time - self.start_time + return result + + @contextlib.contextmanager def adjust_int_max_str_digits(max_digits): """Temporarily change the integer string conversion length limit.""" @@ -2537,12 +2735,505 @@ def adjust_int_max_str_digits(max_digits): finally: sys.set_int_max_str_digits(current) -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = 5000 -# The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 1500 +def exceeds_recursion_limit(): + """For recursion tests, easily exceeds default recursion limit.""" + return 150_000 + + +# Windows doesn't have os.uname() but it doesn't support s390x. +is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' +skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') + +Py_TRACE_REFS = hasattr(sys, 'getobjects') + +_JIT_ENABLED = sys._jit.is_enabled() +requires_jit_enabled = unittest.skipUnless(_JIT_ENABLED, "requires JIT enabled") +requires_jit_disabled = unittest.skipIf(_JIT_ENABLED, "requires JIT disabled") + + +_BASE_COPY_SRC_DIR_IGNORED_NAMES = frozenset({ + # SRC_DIR/.git + '.git', + # ignore all __pycache__/ sub-directories + '__pycache__', +}) + +# Ignore function for shutil.copytree() to copy the Python source code. +def copy_python_src_ignore(path, names): + ignored = _BASE_COPY_SRC_DIR_IGNORED_NAMES + if os.path.basename(path) == 'Doc': + ignored |= { + # SRC_DIR/Doc/build/ + 'build', + # SRC_DIR/Doc/venv/ + 'venv', + } + + # check if we are at the root of the source code + elif 'Modules' in names: + ignored |= { + # SRC_DIR/build/ + 'build', + } + return ignored + + +# XXX Move this to the inspect module? +def walk_class_hierarchy(top, *, topdown=True): + # This is based on the logic in os.walk(). + assert isinstance(top, type), repr(top) + stack = [top] + while stack: + top = stack.pop() + if isinstance(top, tuple): + yield top + continue + + subs = type(top).__subclasses__(top) + if topdown: + # Yield before subclass traversal if going top down. + yield top, subs + # Traverse into subclasses. + for sub in reversed(subs): + stack.append(sub) + else: + # Yield after subclass traversal if going bottom up. + stack.append((top, subs)) + # Traverse into subclasses. + for sub in reversed(subs): + stack.append(sub) + + +def iter_builtin_types(): + # First try the explicit route. + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + if _testinternalcapi is not None: + yield from _testinternalcapi.get_static_builtin_types() + return + + # Fall back to making a best-effort guess. + if hasattr(object, '__flags__'): + # Look for any type object with the Py_TPFLAGS_STATIC_BUILTIN flag set. + import datetime + seen = set() + for cls, subs in walk_class_hierarchy(object): + if cls in seen: + continue + seen.add(cls) + if not (cls.__flags__ & _TPFLAGS_STATIC_BUILTIN): + # Do not walk its subclasses. + subs[:] = [] + continue + yield cls + else: + # Fall back to a naive approach. + seen = set() + for obj in __builtins__.values(): + if not isinstance(obj, type): + continue + cls = obj + # XXX? + if cls.__module__ != 'builtins': + continue + if cls == ExceptionGroup: + # It's a heap type. + continue + if cls in seen: + continue + seen.add(cls) + yield cls + + +# XXX Move this to the inspect module? +def iter_name_in_mro(cls, name): + """Yield matching items found in base.__dict__ across the MRO. + + The descriptor protocol is not invoked. + + list(iter_name_in_mro(cls, name))[0] is roughly equivalent to + find_name_in_mro() in Objects/typeobject.c (AKA PyType_Lookup()). + + inspect.getattr_static() is similar. + """ + # This can fail if "cls" is weird. + for base in inspect._static_getmro(cls): + # This can fail if "base" is weird. + ns = inspect._get_dunder_dict_of_class(base) + try: + obj = ns[name] + except KeyError: + continue + yield obj, base + + +# XXX Move this to the inspect module? +def find_name_in_mro(cls, name, default=inspect._sentinel): + for res in iter_name_in_mro(cls, name): + # Return the first one. + return res + if default is not inspect._sentinel: + return default, None + raise AttributeError(name) -#Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') + +# XXX The return value should always be exactly the same... +def identify_type_slot_wrappers(): + try: + import _testinternalcapi + except ImportError: + _testinternalcapi = None + if _testinternalcapi is not None: + names = {n: None for n in _testinternalcapi.identify_type_slot_wrappers()} + return list(names) + else: + raise NotImplementedError + + +def iter_slot_wrappers(cls): + def is_slot_wrapper(name, value): + if not isinstance(value, types.WrapperDescriptorType): + assert not repr(value).startswith(' dict[str, str]: + clean_env = os.environ.copy() + for k in clean_env.copy(): + if k.startswith("PYTHON"): + clean_env.pop(k) + clean_env.pop("FORCE_COLOR", None) + clean_env.pop("NO_COLOR", None) + return clean_env + + +WINDOWS_STATUS = { + 0xC0000005: "STATUS_ACCESS_VIOLATION", + 0xC00000FD: "STATUS_STACK_OVERFLOW", + 0xC000013A: "STATUS_CONTROL_C_EXIT", +} + +def get_signal_name(exitcode): + import signal + + if exitcode < 0: + signum = -exitcode + try: + return signal.Signals(signum).name + except ValueError: + pass + + # Shell exit code (ex: WASI build) + if 128 < exitcode < 256: + signum = exitcode - 128 + try: + return signal.Signals(signum).name + except ValueError: + pass + + try: + return WINDOWS_STATUS[exitcode] + except KeyError: + pass + + return None + +class BrokenIter: + def __init__(self, init_raises=False, next_raises=False, iter_raises=False): + if init_raises: + 1/0 + self.next_raises = next_raises + self.iter_raises = iter_raises + + def __next__(self): + if self.next_raises: + 1/0 + + def __iter__(self): + if self.iter_raises: + 1/0 + return self + + +def in_systemd_nspawn_sync_suppressed() -> bool: + """ + Test whether the test suite is runing in systemd-nspawn + with ``--suppress-sync=true``. + + This can be used to skip tests that rely on ``fsync()`` calls + and similar not being intercepted. + """ + + if not hasattr(os, "O_SYNC"): + return False + + try: + with open("/run/systemd/container", "rb") as fp: + if fp.read().rstrip() != b"systemd-nspawn": + return False + except FileNotFoundError: + return False + + # If systemd-nspawn is used, O_SYNC flag will immediately + # trigger EINVAL. Otherwise, ENOENT will be given instead. + import errno + try: + fd = os.open(__file__, os.O_RDONLY | os.O_SYNC) + except OSError as err: + if err.errno == errno.EINVAL: + return True + else: + os.close(fd) + + return False + +def run_no_yield_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + coro.send(None) + except StopIteration as e: + return e.value + else: + raise AssertionError("coroutine did not complete") + finally: + coro.close() + + +@types.coroutine +def async_yield(v): + return (yield v) + + +def run_yielding_async_fn(async_fn, /, *args, **kwargs): + coro = async_fn(*args, **kwargs) + try: + while True: + try: + coro.send(None) + except StopIteration as e: + return e.value + finally: + coro.close() + + +def is_libssl_fips_mode(): + try: + from _hashlib import get_fips_mode # ask _hashopenssl.c + except ImportError: + return False # more of a maybe, unless we add this to the _ssl module. + return get_fips_mode() != 0 + +def _supports_remote_attaching(): + PROCESS_VM_READV_SUPPORTED = False + + try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED + except ImportError: + pass + + return PROCESS_VM_READV_SUPPORTED + +def _support_remote_exec_only_impl(): + if not sys.is_remote_debug_enabled(): + return unittest.skip("Remote debugging is not enabled") + if sys.platform not in ("darwin", "linux", "win32"): + return unittest.skip("Test only runs on Linux, Windows and macOS") + if sys.platform == "linux" and not _supports_remote_attaching(): + return unittest.skip("Test only runs on Linux with process_vm_readv support") + return _id + +def support_remote_exec_only(test): + return _support_remote_exec_only_impl()(test) + +class EqualToForwardRef: + """Helper to ease use of annotationlib.ForwardRef in tests. + + This checks only attributes that can be set using the constructor. + + """ + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_class=False, + ): + self.__forward_arg__ = arg + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__owner__ = owner + + def __eq__(self, other): + if not isinstance(other, (EqualToForwardRef, annotationlib.ForwardRef)): + return NotImplemented + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + and self.__forward_is_class__ == other.__forward_is_class__ + and self.__owner__ == other.__owner__ + ) + + def __repr__(self): + extra = [] + if self.__forward_module__ is not None: + extra.append(f", module={self.__forward_module__!r}") + if self.__forward_is_class__: + extra.append(", is_class=True") + if self.__owner__ is not None: + extra.append(f", owner={self.__owner__!r}") + return f"EqualToForwardRef({self.__forward_arg__!r}{''.join(extra)})" + + +_linked_to_musl = None +def linked_to_musl(): + """ + Report if the Python executable is linked to the musl C library. + + Return False if we don't think it is, or a version triple otherwise. + """ + # This is can be a relatively expensive check, so we use a cache. + global _linked_to_musl + if _linked_to_musl is not None: + return _linked_to_musl + + # emscripten (at least as far as we're concerned) and wasi use musl, + # but platform doesn't know how to get the version, so set it to zero. + if is_wasm32: + _linked_to_musl = (0, 0, 0) + return _linked_to_musl + + # On all other non-linux platforms assume no musl. + if sys.platform != 'linux': + _linked_to_musl = False + return _linked_to_musl + + # On linux, we'll depend on the platform module to do the check, so new + # musl platforms should add support in that module if possible. + import platform + lib, version = platform.libc_ver() + if lib != 'musl': + _linked_to_musl = False + return _linked_to_musl + _linked_to_musl = tuple(map(int, version.split('.'))) + return _linked_to_musl + + +def control_characters_c0() -> list[str]: + """Returns a list of C0 control characters as strings. + C0 control characters defined as the byte range 0x00-0x1F, and 0x7F. + """ + return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"] diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py new file mode 100644 index 00000000000..9a57c309616 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -0,0 +1,111 @@ +import functools +import unittest +from enum import Enum + +__all__ = [ + "given", + "example", + "assume", + "reject", + "register_random", + "strategies", + "HealthCheck", + "settings", + "Verbosity", +] + +from . import strategies + + +def given(*_args, **_kwargs): + def decorator(f): + if examples := getattr(f, "_examples", []): + + @functools.wraps(f) + def test_function(self): + for example_args, example_kwargs in examples: + with self.subTest(*example_args, **example_kwargs): + f(self, *example_args, **example_kwargs) + + else: + # If we have found no examples, we must skip the test. If @example + # is applied after @given, it will re-wrap the test to remove the + # skip decorator. + test_function = unittest.skip( + "Hypothesis required for property test with no " + + "specified examples" + )(f) + + test_function._given = True + return test_function + + return decorator + + +def example(*args, **kwargs): + if bool(args) == bool(kwargs): + raise ValueError("Must specify exactly one of *args or **kwargs") + + def decorator(f): + base_func = getattr(f, "__wrapped__", f) + if not hasattr(base_func, "_examples"): + base_func._examples = [] + + base_func._examples.append((args, kwargs)) + + if getattr(f, "_given", False): + # If the given decorator is below all the example decorators, + # it would be erroneously skipped, so we need to re-wrap the new + # base function. + f = given()(base_func) + + return f + + return decorator + + +def assume(condition): + if not condition: + raise unittest.SkipTest("Unsatisfied assumption") + return True + + +def reject(): + assume(False) + + +def register_random(*args, **kwargs): + pass # pragma: no cover + + +def settings(*args, **kwargs): + return lambda f: f # pragma: nocover + + +class HealthCheck(Enum): + data_too_large = 1 + filter_too_much = 2 + too_slow = 3 + return_value = 5 + large_base_example = 7 + not_a_test_method = 8 + + @classmethod + def all(cls): + return list(cls) + + +class Verbosity(Enum): + quiet = 0 + normal = 1 + verbose = 2 + debug = 3 + + +class Phase(Enum): + explicit = 0 + reuse = 1 + generate = 2 + target = 3 + shrink = 4 + explain = 5 diff --git a/Lib/test/support/_hypothesis_stubs/_helpers.py b/Lib/test/support/_hypothesis_stubs/_helpers.py new file mode 100644 index 00000000000..3f6244e4dbc --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/_helpers.py @@ -0,0 +1,43 @@ +# Stub out only the subset of the interface that we actually use in our tests. +class StubClass: + def __init__(self, *args, **kwargs): + self.__stub_args = args + self.__stub_kwargs = kwargs + self.__repr = None + + def _with_repr(self, new_repr): + new_obj = self.__class__(*self.__stub_args, **self.__stub_kwargs) + new_obj.__repr = new_repr + return new_obj + + def __repr__(self): + if self.__repr is not None: + return self.__repr + + argstr = ", ".join(self.__stub_args) + kwargstr = ", ".join(f"{kw}={val}" for kw, val in self.__stub_kwargs.items()) + + in_parens = argstr + if kwargstr: + in_parens += ", " + kwargstr + + return f"{self.__class__.__qualname__}({in_parens})" + + +def stub_factory(klass, name, *, with_repr=None, _seen={}): + if (klass, name) not in _seen: + + class Stub(klass): + def __init__(self, *args, **kwargs): + super().__init__() + self.__stub_args = args + self.__stub_kwargs = kwargs + + Stub.__name__ = name + Stub.__qualname__ = name + if with_repr is not None: + Stub._repr = None + + _seen.setdefault((klass, name, with_repr), Stub) + + return _seen[(klass, name, with_repr)] diff --git a/Lib/test/support/_hypothesis_stubs/strategies.py b/Lib/test/support/_hypothesis_stubs/strategies.py new file mode 100644 index 00000000000..d2b885d41e1 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/strategies.py @@ -0,0 +1,91 @@ +import functools + +from ._helpers import StubClass, stub_factory + + +class StubStrategy(StubClass): + def __make_trailing_repr(self, transformation_name, func): + func_name = func.__name__ or repr(func) + return f"{self!r}.{transformation_name}({func_name})" + + def map(self, pack): + return self._with_repr(self.__make_trailing_repr("map", pack)) + + def flatmap(self, expand): + return self._with_repr(self.__make_trailing_repr("flatmap", expand)) + + def filter(self, condition): + return self._with_repr(self.__make_trailing_repr("filter", condition)) + + def __or__(self, other): + new_repr = f"one_of({self!r}, {other!r})" + return self._with_repr(new_repr) + + +_STRATEGIES = { + "binary", + "booleans", + "builds", + "characters", + "complex_numbers", + "composite", + "data", + "dates", + "datetimes", + "decimals", + "deferred", + "dictionaries", + "emails", + "fixed_dictionaries", + "floats", + "fractions", + "from_regex", + "from_type", + "frozensets", + "functions", + "integers", + "iterables", + "just", + "lists", + "none", + "nothing", + "one_of", + "permutations", + "random_module", + "randoms", + "recursive", + "register_type_strategy", + "runner", + "sampled_from", + "sets", + "shared", + "slices", + "timedeltas", + "times", + "text", + "tuples", + "uuids", +} + +__all__ = sorted(_STRATEGIES) + + +def composite(f): + strategy = stub_factory(StubStrategy, f.__name__) + + @functools.wraps(f) + def inner(*args, **kwargs): + return strategy(*args, **kwargs) + + return inner + + +def __getattr__(name): + if name not in _STRATEGIES: + raise AttributeError(f"Unknown attribute {name}") + + return stub_factory(StubStrategy, f"hypothesis.strategies.{name}") + + +def __dir__(): + return __all__ diff --git a/Lib/test/support/ast_helper.py b/Lib/test/support/ast_helper.py index 8a0415b6aae..98eaf0b2721 100644 --- a/Lib/test/support/ast_helper.py +++ b/Lib/test/support/ast_helper.py @@ -1,5 +1,6 @@ import ast + class ASTTestMixin: """Test mixing to have basic assertions for AST nodes.""" @@ -16,6 +17,9 @@ def traverse_compare(a, b, missing=object()): self.fail(f"{type(a)!r} is not {type(b)!r}") if isinstance(a, ast.AST): for field in a._fields: + if isinstance(a, ast.Constant) and field == "kind": + # Skip the 'kind' field for ast.Constant + continue value1 = getattr(a, field, missing) value2 = getattr(b, field, missing) # Singletons are equal by definition, so further diff --git a/Lib/test/support/asynchat.py b/Lib/test/support/asynchat.py index 38c47a1fda6..a8c6b28a9e1 100644 --- a/Lib/test/support/asynchat.py +++ b/Lib/test/support/asynchat.py @@ -1,5 +1,5 @@ # TODO: This module was deprecated and removed from CPython 3.12 -# Now it is a test-only helper. Any attempts to rewrite exising tests that +# Now it is a test-only helper. Any attempts to rewrite existing tests that # are using this module and remove it completely are appreciated! # See: https://github.com/python/cpython/issues/72719 diff --git a/Lib/test/support/asyncore.py b/Lib/test/support/asyncore.py index b397aca5568..658c22fdcee 100644 --- a/Lib/test/support/asyncore.py +++ b/Lib/test/support/asyncore.py @@ -1,5 +1,5 @@ # TODO: This module was deprecated and removed from CPython 3.12 -# Now it is a test-only helper. Any attempts to rewrite exising tests that +# Now it is a test-only helper. Any attempts to rewrite existing tests that # are using this module and remove it completely are appreciated! # See: https://github.com/python/cpython/issues/72719 @@ -51,17 +51,27 @@ sophisticated high-performance network servers and clients a snap. """ +import os import select import socket import sys import time import warnings - -import os -from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ - ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ - errorcode - +from errno import ( + EAGAIN, + EALREADY, + EBADF, + ECONNABORTED, + ECONNRESET, + EINPROGRESS, + EINVAL, + EISCONN, + ENOTCONN, + EPIPE, + ESHUTDOWN, + EWOULDBLOCK, + errorcode, +) _DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, EBADF}) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 388d1266773..4a3c8c2c4f1 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -1,12 +1,29 @@ """bytecode_helper - support tools for testing correct bytecode generation""" -import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object +import opcode +import unittest + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None _UNSPECIFIED = object() +def instructions_with_positions(instrs, co_positions): + # Return (instr, positions) pairs from the instrs list and co_positions + # iterator. The latter contains items for cache lines and the former + # doesn't, so those need to be skipped. + + co_positions = co_positions or iter(()) + for instr in instrs: + yield instr, next(co_positions, ()) + for _, size, _ in (instr.cache_info or ()): + for i in range(size): + next(co_positions, ()) + class BytecodeTestCase(unittest.TestCase): """Custom assertion methods for inspecting bytecode.""" @@ -53,16 +70,14 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Normalize the labels to the - # instruction count of the target, and compare the lists. - - self.assertIsInstance(actual_, list) - self.assertIsInstance(expected_, list) + def assertInstructionsMatch(self, actual_seq, expected): + # get an InstructionSequence and an expected list, where each + # entry is a label or an instruction tuple. Construct an expected + # instruction sequence and compare with the one given. - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) + self.assertIsInstance(expected, list) + actual = actual_seq.get_instructions() + expected = self.seq_from_insts(expected).get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -72,10 +87,8 @@ def assertInstructionsMatch(self, actual_, expected_): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - # crop comparison to the provided expected values - if len(act) > len(exp): - act = act[:len(exp)] - self.assertEqual(exp, act) + idx = max([p[0] for p in enumerate(exp) if p[1] != -1]) + self.assertEqual(exp[:idx], act[:idx]) def resolveAndRemoveLabels(self, insts): idx = 0 @@ -90,54 +103,57 @@ def resolveAndRemoveLabels(self, insts): return res - def normalize_insts(self, insts): - """ Map labels to instruction index. - Map opcodes to opnames. - """ - insts = self.resolveAndRemoveLabels(insts) - res = [] - for item in insts: - assert isinstance(item, tuple) - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if isinstance(oparg, self.Label): - arg = oparg.value - else: - arg = oparg if opcode in self.HAS_ARG else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) - return res + def seq_from_insts(self, insts): + labels = {item for item in insts if isinstance(item, self.Label)} + for i, lbl in enumerate(labels): + lbl.value = i - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] + seq = _testinternalcapi.new_instruction_sequence() for item in insts: - assert isinstance(item, tuple) - inst = list(item) - opcode = dis.opmap[inst[0]] - oparg = inst[1] - loc = inst[2:] + [-1] * (6 - len(inst)) - res.append((opcode, oparg, *loc)) - return res + if isinstance(item, self.Label): + seq.use_label(item.value) + else: + op = item[0] + if isinstance(op, str): + op = opcode.opmap[op] + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value + loc = loc + [-1] * (4 - len(loc)) + seq.addop(op, arg or 0, *loc) + return seq + + def check_instructions(self, insts): + for inst in insts: + if isinstance(inst, self.Label): + continue + op, arg, *loc = inst + if isinstance(op, str): + op = opcode.opmap[op] + self.assertEqual(op in opcode.hasarg, + arg is not None, + f"{opcode.opname[op]=} {arg=}") + self.assertTrue(all(isinstance(l, int) for l in loc)) +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CodegenTestCase(CompilationStepTestCase): def generate_code(self, ast): - insts, _ = compiler_codegen(ast, "my_file.py", 0) + insts, _ = _testinternalcapi.compiler_codegen(ast, "my_file.py", 0) return insts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CfgOptimizationTestCase(CompilationStepTestCase): - def get_optimized(self, insts, consts, nlocals=0): - insts = self.normalize_insts(insts) - insts = self.complete_insts_info(insts) - insts = optimize_cfg(insts, consts, nlocals) + def get_optimized(self, seq, consts, nlocals=0): + insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals) return insts, consts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class AssemblerTestCase(CompilationStepTestCase): def get_code_object(self, filename, insts, metadata): - co = assemble_code_object(filename, insts, metadata) + co = _testinternalcapi.assemble_code_object(filename, insts, metadata) return co diff --git a/Lib/test/support/channels.py b/Lib/test/support/channels.py new file mode 100644 index 00000000000..3f7b46030fd --- /dev/null +++ b/Lib/test/support/channels.py @@ -0,0 +1,282 @@ +"""Cross-interpreter Channels High Level Module.""" + +import time +from concurrent.interpreters import _crossinterp +from concurrent.interpreters._crossinterp import ( + UNBOUND_ERROR, + UNBOUND_REMOVE, +) + +import _interpchannels as _channels + +# aliases: +from _interpchannels import ( + ChannelClosedError, + ChannelEmptyError, + ChannelError, + ChannelNotEmptyError, + ChannelNotFoundError, +) + +__all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', + 'create', 'list_all', + 'SendChannel', 'RecvChannel', + 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', + 'ItemInterpreterDestroyed', +] + + +class ItemInterpreterDestroyed(ChannelError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(*, unbounditems=UNBOUND): + """Return (recv, send) for a new cross-interpreter channel. + + The channel may be used to pass data safely between interpreters. + + "unbounditems" sets the default for the send end of the channel. + See SendChannel.send() for supported values. The default value + is UNBOUND, which replaces the unbound item when received. + """ + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + cid = _channels.create(unboundop, -1) + recv, send = RecvChannel(cid), SendChannel(cid) + send._set_unbound(unboundop, unbounditems) + return recv, send + + +def list_all(): + """Return a list of (recv, send) for all open channels.""" + channels = [] + for cid, unboundop, _ in _channels.list_all(): + chan = _, send = RecvChannel(cid), SendChannel(cid) + if not hasattr(send, '_unboundop'): + send._set_unbound(unboundop) + else: + assert send._unbound[0] == unboundop + channels.append(chan) + return channels + + +class _ChannelEnd: + """The base class for RecvChannel and SendChannel.""" + + _end = None + + def __new__(cls, cid): + self = super().__new__(cls) + if self._end == 'send': + cid = _channels._channel_id(cid, send=True, force=True) + elif self._end == 'recv': + cid = _channels._channel_id(cid, recv=True, force=True) + else: + raise NotImplementedError(self._end) + self._id = cid + return self + + def __repr__(self): + return f'{type(self).__name__}(id={int(self._id)})' + + def __hash__(self): + return hash(self._id) + + def __eq__(self, other): + if isinstance(self, RecvChannel): + if not isinstance(other, RecvChannel): + return NotImplemented + elif not isinstance(other, SendChannel): + return NotImplemented + return other._id == self._id + + # for pickling: + def __reduce__(self): + return (type(self), (int(self._id),)) + + @property + def id(self): + return self._id + + @property + def _info(self): + return _channels.get_info(self._id) + + @property + def is_closed(self): + return self._info.closed + + +_NOT_SET = object() + + +class RecvChannel(_ChannelEnd): + """The receiving end of a cross-interpreter channel.""" + + _end = 'recv' + + def recv(self, timeout=None, *, + _sentinel=object(), + _delay=10 / 1000, # 10 milliseconds + ): + """Return the next object from the channel. + + This blocks until an object has been sent, if none have been + sent already. + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + obj, unboundop = _channels.recv(self._id, _sentinel) + while obj is _sentinel: + time.sleep(_delay) + if timeout is not None and time.time() >= end: + raise TimeoutError + obj, unboundop = _channels.recv(self._id, _sentinel) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj + + def recv_nowait(self, default=_NOT_SET): + """Return the next object from the channel. + + If none have been sent then return the default if one + is provided or fail with ChannelEmptyError. Otherwise this + is the same as recv(). + """ + if default is _NOT_SET: + obj, unboundop = _channels.recv(self._id) + else: + obj, unboundop = _channels.recv(self._id, default) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj + + def close(self): + _channels.close(self._id, recv=True) + + +class SendChannel(_ChannelEnd): + """The sending end of a cross-interpreter channel.""" + + _end = 'send' + +# def __new__(cls, cid, *, _unbound=None): +# if _unbound is None: +# try: +# op = _channels.get_channel_defaults(cid) +# _unbound = (op,) +# except ChannelNotFoundError: +# _unbound = _serialize_unbound(UNBOUND) +# self = super().__new__(cls, cid) +# self._unbound = _unbound +# return self + + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound + + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _channels.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items + + @property + def is_closed(self): + info = self._info + return info.closed or info.closing + + def send(self, obj, timeout=None, *, + unbounditems=None, + ): + """Send the object (i.e. its data) to the channel's receiving end. + + This blocks until the object is received. + """ + if unbounditems is None: + unboundop = -1 + else: + unboundop, = _serialize_unbound(unbounditems) + _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) + + def send_nowait(self, obj, *, + unbounditems=None, + ): + """Send the object to the channel's receiving end. + + If the object is immediately received then return True + (else False). Otherwise this is the same as send(). + """ + if unbounditems is None: + unboundop = -1 + else: + unboundop, = _serialize_unbound(unbounditems) + # XXX Note that at the moment channel_send() only ever returns + # None. This should be fixed when channel_send_wait() is added. + # See bpo-32604 and gh-19829. + return _channels.send(self._id, obj, unboundop, blocking=False) + + def send_buffer(self, obj, timeout=None, *, + unbounditems=None, + ): + """Send the object's buffer to the channel's receiving end. + + This blocks until the object is received. + """ + if unbounditems is None: + unboundop = -1 + else: + unboundop, = _serialize_unbound(unbounditems) + _channels.send_buffer(self._id, obj, unboundop, + timeout=timeout, blocking=True) + + def send_buffer_nowait(self, obj, *, + unbounditems=None, + ): + """Send the object's buffer to the channel's receiving end. + + If the object is immediately received then return True + (else False). Otherwise this is the same as send(). + """ + if unbounditems is None: + unboundop = -1 + else: + unboundop, = _serialize_unbound(unbounditems) + return _channels.send_buffer(self._id, obj, unboundop, blocking=False) + + def close(self): + _channels.close(self._id, send=True) + + +# XXX This is causing leaks (gh-110318): +_channels._register_end_types(SendChannel, RecvChannel) diff --git a/Lib/test/support/hashlib_helper.py b/Lib/test/support/hashlib_helper.py index a4e6c92203a..75dc2ba7506 100644 --- a/Lib/test/support/hashlib_helper.py +++ b/Lib/test/support/hashlib_helper.py @@ -1,51 +1,330 @@ import functools import hashlib +import importlib import unittest +from test.support.import_helper import import_module + try: import _hashlib except ImportError: _hashlib = None +try: + import _hmac +except ImportError: + _hmac = None + + +def requires_hashlib(): + return unittest.skipIf(_hashlib is None, "requires _hashlib") + + +def requires_builtin_hmac(): + return unittest.skipIf(_hmac is None, "requires _hmac") + + +def _missing_hash(digestname, implementation=None, *, exc=None): + parts = ["missing", implementation, f"hash algorithm: {digestname!r}"] + msg = " ".join(filter(None, parts)) + raise unittest.SkipTest(msg) from exc + + +def _openssl_availabillity(digestname, *, usedforsecurity): + try: + _hashlib.new(digestname, usedforsecurity=usedforsecurity) + except AttributeError: + assert _hashlib is None + _missing_hash(digestname, "OpenSSL") + except ValueError as exc: + _missing_hash(digestname, "OpenSSL", exc=exc) + + +def _decorate_func_or_class(func_or_class, decorator_func): + if not isinstance(func_or_class, type): + return decorator_func(func_or_class) + + decorated_class = func_or_class + setUpClass = decorated_class.__dict__.get('setUpClass') + if setUpClass is None: + def setUpClass(cls): + super(decorated_class, cls).setUpClass() + setUpClass.__qualname__ = decorated_class.__qualname__ + '.setUpClass' + setUpClass.__module__ = decorated_class.__module__ + else: + setUpClass = setUpClass.__func__ + setUpClass = classmethod(decorator_func(setUpClass)) + decorated_class.setUpClass = setUpClass + return decorated_class + def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): - """Decorator raising SkipTest if a hashing algorithm is not available + """Decorator raising SkipTest if a hashing algorithm is not available. - The hashing algorithm could be missing or blocked by a strict crypto - policy. + The hashing algorithm may be missing, blocked by a strict crypto policy, + or Python may be configured with `--with-builtin-hashlib-hashes=no`. If 'openssl' is True, then the decorator checks that OpenSSL provides - the algorithm. Otherwise the check falls back to built-in - implementations. The usedforsecurity flag is passed to the constructor. + the algorithm. Otherwise the check falls back to (optional) built-in + HACL* implementations. + The usedforsecurity flag is passed to the constructor but has no effect + on HACL* implementations. + + Examples of exceptions being suppressed: ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS ValueError: unsupported hash type md4 """ + if openssl and _hashlib is not None: + def test_availability(): + _hashlib.new(digestname, usedforsecurity=usedforsecurity) + else: + def test_availability(): + hashlib.new(digestname, usedforsecurity=usedforsecurity) + + def decorator_func(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + test_availability() + except ValueError as exc: + _missing_hash(digestname, exc=exc) + return func(*args, **kwargs) + return wrapper + + def decorator(func_or_class): + return _decorate_func_or_class(func_or_class, decorator_func) + return decorator + + +def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): + """Decorator raising SkipTest if an OpenSSL hashing algorithm is missing. + + The hashing algorithm may be missing or blocked by a strict crypto policy. + """ + def decorator_func(func): + @requires_hashlib() # avoid checking at each call + @functools.wraps(func) + def wrapper(*args, **kwargs): + _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) + return func(*args, **kwargs) + return wrapper + def decorator(func_or_class): - if isinstance(func_or_class, type): - setUpClass = func_or_class.__dict__.get('setUpClass') - if setUpClass is None: - def setUpClass(cls): - super(func_or_class, cls).setUpClass() - setUpClass.__qualname__ = func_or_class.__qualname__ + '.setUpClass' - setUpClass.__module__ = func_or_class.__module__ - else: - setUpClass = setUpClass.__func__ - setUpClass = classmethod(decorator(setUpClass)) - func_or_class.setUpClass = setUpClass - return func_or_class - - @functools.wraps(func_or_class) + return _decorate_func_or_class(func_or_class, decorator_func) + return decorator + + +def find_openssl_hashdigest_constructor(digestname, *, usedforsecurity=True): + """Find the OpenSSL hash function constructor by its name.""" + assert isinstance(digestname, str), digestname + _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) + # This returns a function of the form _hashlib.openssl_ and + # not a lambda function as it is rejected by _hashlib.hmac_new(). + return getattr(_hashlib, f"openssl_{digestname}") + + +def requires_builtin_hashdigest( + module_name, digestname, *, usedforsecurity=True +): + """Decorator raising SkipTest if a HACL* hashing algorithm is missing. + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + """ + def decorator_func(func): + @functools.wraps(func) def wrapper(*args, **kwargs): + module = import_module(module_name) try: - if openssl and _hashlib is not None: - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - else: - hashlib.new(digestname, usedforsecurity=usedforsecurity) - except ValueError: - raise unittest.SkipTest( - f"hash digest '{digestname}' is not available." - ) - return func_or_class(*args, **kwargs) + getattr(module, digestname) + except AttributeError: + fullname = f'{module_name}.{digestname}' + _missing_hash(fullname, implementation="HACL") + return func(*args, **kwargs) return wrapper + + def decorator(func_or_class): + return _decorate_func_or_class(func_or_class, decorator_func) return decorator + + +def find_builtin_hashdigest_constructor( + module_name, digestname, *, usedforsecurity=True +): + """Find the HACL* hash function constructor. + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + """ + module = import_module(module_name) + try: + constructor = getattr(module, digestname) + constructor(b'', usedforsecurity=usedforsecurity) + except (AttributeError, TypeError, ValueError): + _missing_hash(f'{module_name}.{digestname}', implementation="HACL") + return constructor + + +class HashFunctionsTrait: + """Mixin trait class containing hash functions. + + This class is assumed to have all unitest.TestCase methods but should + not directly inherit from it to prevent the test suite being run on it. + + Subclasses should implement the hash functions by returning an object + that can be recognized as a valid digestmod parameter for both hashlib + and HMAC. In particular, it cannot be a lambda function as it will not + be recognized by hashlib (it will still be accepted by the pure Python + implementation of HMAC). + """ + + ALGORITHMS = [ + 'md5', 'sha1', + 'sha224', 'sha256', 'sha384', 'sha512', + 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + ] + + # Default 'usedforsecurity' to use when looking up a hash function. + usedforsecurity = True + + def _find_constructor(self, name): + # By default, a missing algorithm skips the test that uses it. + self.assertIn(name, self.ALGORITHMS) + self.skipTest(f"missing hash function: {name}") + + @property + def md5(self): + return self._find_constructor("md5") + + @property + def sha1(self): + return self._find_constructor("sha1") + + @property + def sha224(self): + return self._find_constructor("sha224") + + @property + def sha256(self): + return self._find_constructor("sha256") + + @property + def sha384(self): + return self._find_constructor("sha384") + + @property + def sha512(self): + return self._find_constructor("sha512") + + @property + def sha3_224(self): + return self._find_constructor("sha3_224") + + @property + def sha3_256(self): + return self._find_constructor("sha3_256") + + @property + def sha3_384(self): + return self._find_constructor("sha3_384") + + @property + def sha3_512(self): + return self._find_constructor("sha3_512") + + +class NamedHashFunctionsTrait(HashFunctionsTrait): + """Trait containing named hash functions. + + Hash functions are available if and only if they are available in hashlib. + """ + + def _find_constructor(self, name): + self.assertIn(name, self.ALGORITHMS) + return name + + +class OpenSSLHashFunctionsTrait(HashFunctionsTrait): + """Trait containing OpenSSL hash functions. + + Hash functions are available if and only if they are available in _hashlib. + """ + + def _find_constructor(self, name): + self.assertIn(name, self.ALGORITHMS) + return find_openssl_hashdigest_constructor( + name, usedforsecurity=self.usedforsecurity + ) + + +class BuiltinHashFunctionsTrait(HashFunctionsTrait): + """Trait containing HACL* hash functions. + + Hash functions are available if and only if they are available in C. + In particular, HACL* HMAC-MD5 may be available even though HACL* md5 + is not since the former is unconditionally built. + """ + + def _find_constructor_in(self, module, name): + self.assertIn(name, self.ALGORITHMS) + return find_builtin_hashdigest_constructor(module, name) + + @property + def md5(self): + return self._find_constructor_in("_md5", "md5") + + @property + def sha1(self): + return self._find_constructor_in("_sha1", "sha1") + + @property + def sha224(self): + return self._find_constructor_in("_sha2", "sha224") + + @property + def sha256(self): + return self._find_constructor_in("_sha2", "sha256") + + @property + def sha384(self): + return self._find_constructor_in("_sha2", "sha384") + + @property + def sha512(self): + return self._find_constructor_in("_sha2", "sha512") + + @property + def sha3_224(self): + return self._find_constructor_in("_sha3", "sha3_224") + + @property + def sha3_256(self): + return self._find_constructor_in("_sha3","sha3_256") + + @property + def sha3_384(self): + return self._find_constructor_in("_sha3","sha3_384") + + @property + def sha3_512(self): + return self._find_constructor_in("_sha3","sha3_512") + + +def find_gil_minsize(modules_names, default=2048): + """Get the largest GIL_MINSIZE value for the given cryptographic modules. + + The valid module names are the following: + + - _hashlib + - _md5, _sha1, _sha2, _sha3, _blake2 + - _hmac + """ + sizes = [] + for module_name in modules_names: + try: + module = importlib.import_module(module_name) + except ImportError: + continue + sizes.append(getattr(module, '_GIL_MINSIZE', default)) + return max(sizes, default=default) diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py new file mode 100644 index 00000000000..6e9e168f63a --- /dev/null +++ b/Lib/test/support/hypothesis_helper.py @@ -0,0 +1,54 @@ +import os + +try: + import hypothesis +except ImportError: + from . import _hypothesis_stubs as hypothesis +else: + # Regrtest changes to use a tempdir as the working directory, so we have + # to tell Hypothesis to use the original in order to persist the database. + from hypothesis.configuration import set_hypothesis_home_dir + + from test.support import has_socket_support + from test.support.os_helper import SAVEDCWD + + set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis")) + + # When using the real Hypothesis, we'll configure it to ignore occasional + # slow tests (avoiding flakiness from random VM slowness in CI). + hypothesis.settings.register_profile( + "slow-is-ok", + deadline=None, + suppress_health_check=[ + hypothesis.HealthCheck.too_slow, + hypothesis.HealthCheck.differing_executors, + ], + ) + hypothesis.settings.load_profile("slow-is-ok") + + # For local development, we'll write to the default on-local-disk database + # of failing examples, and also use a pull-through cache to automatically + # replay any failing examples discovered in CI. For details on how this + # works, see https://hypothesis.readthedocs.io/en/latest/database.html + # We only do that if a GITHUB_TOKEN env var is provided, see: + # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + # And Python is built with socket support: + if ( + has_socket_support + and "CI" not in os.environ + and "GITHUB_TOKEN" in os.environ + ): + from hypothesis.database import ( + GitHubArtifactDatabase, + MultiplexedDatabase, + ReadOnlyDatabase, + ) + + hypothesis.settings.register_profile( + "cpython-local-dev", + database=MultiplexedDatabase( + hypothesis.settings.default.database, + ReadOnlyDatabase(GitHubArtifactDatabase("python", "cpython")), + ), + ) + hypothesis.settings.load_profile("cpython-local-dev") diff --git a/Lib/test/support/i18n_helper.py b/Lib/test/support/i18n_helper.py new file mode 100644 index 00000000000..af97cdc9cb5 --- /dev/null +++ b/Lib/test/support/i18n_helper.py @@ -0,0 +1,63 @@ +import re +import subprocess +import sys +import unittest +from pathlib import Path + +from test.support import REPO_ROOT, TEST_HOME_DIR, requires_subprocess +from test.test_tools import skip_if_missing + +pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py' + +msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)', + re.DOTALL) +msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"') + + +def _generate_po_file(path, *, stdout_only=True): + res = subprocess.run([sys.executable, pygettext, + '--no-location', '-o', '-', path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True) + if stdout_only: + return res.stdout + return res + + +def _extract_msgids(po): + msgids = [] + for msgid in msgid_pattern.findall(po): + msgid_string = ''.join(msgid_string_pattern.findall(msgid)) + msgid_string = msgid_string.replace(r'\"', '"') + if msgid_string: + msgids.append(msgid_string) + return sorted(msgids) + + +def _get_snapshot_path(module_name): + return Path(TEST_HOME_DIR) / 'translationdata' / module_name / 'msgids.txt' + + +@requires_subprocess() +class TestTranslationsBase(unittest.TestCase): + + def assertMsgidsEqual(self, module): + '''Assert that msgids extracted from a given module match a + snapshot. + + ''' + skip_if_missing('i18n') + res = _generate_po_file(module.__file__, stdout_only=False) + self.assertEqual(res.returncode, 0) + self.assertEqual(res.stderr, '') + msgids = _extract_msgids(res.stdout) + snapshot_path = _get_snapshot_path(module.__name__) + snapshot = snapshot_path.read_text().splitlines() + self.assertListEqual(msgids, snapshot) + + +def update_translation_snapshots(module): + contents = _generate_po_file(module.__file__) + msgids = _extract_msgids(contents) + snapshot_path = _get_snapshot_path(module.__name__) + snapshot_path.write_text('\n'.join(msgids)) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 67f18e530ed..4c7eac0b7eb 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -1,14 +1,16 @@ import contextlib import _imp import importlib +import importlib.machinery import importlib.util import os import shutil import sys +import textwrap import unittest import warnings -from .os_helper import unlink +from .os_helper import unlink, temp_dir @contextlib.contextmanager @@ -58,8 +60,8 @@ def make_legacy_pyc(source): :return: The file system path to the legacy pyc file. """ pyc_file = importlib.util.cache_from_source(source) - up_one = os.path.dirname(os.path.abspath(source)) - legacy_pyc = os.path.join(up_one, source + 'c') + assert source.endswith('.py') + legacy_pyc = source + 'c' shutil.move(pyc_file, legacy_pyc) return legacy_pyc @@ -114,7 +116,7 @@ def multi_interp_extensions_check(enabled=True): This only applies to modules that haven't been imported yet. It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and - _xxsubinterpreters.create()). + _interpreters.create()). Also see importlib.utils.allowing_all_extensions(). """ @@ -268,9 +270,173 @@ def modules_cleanup(oldmodules): sys.modules.update(oldmodules) +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = modules_setup() + try: + yield + finally: + modules_cleanup(saved) + + def mock_register_at_fork(func): # bpo-30599: Mock os.register_at_fork() when importing the random module, # since this function doesn't allow to unregister callbacks and would leak # memory. from unittest import mock return mock.patch('os.register_at_fork', create=True)(func) + + +@contextlib.contextmanager +def ready_to_import(name=None, source=""): + from test.support import script_helper + + # 1. Sets up a temporary directory and removes it afterwards + # 2. Creates the module file + # 3. Temporarily clears the module from sys.modules (if any) + # 4. Reverts or removes the module when cleaning up + name = name or "spam" + with temp_dir() as tempdir: + path = script_helper.make_script(tempdir, name, source) + old_module = sys.modules.pop(name, None) + try: + sys.path.insert(0, tempdir) + yield name, path + finally: + sys.path.remove(tempdir) + if old_module is not None: + sys.modules[name] = old_module + else: + sys.modules.pop(name, None) + + +def ensure_lazy_imports(imported_module, modules_to_block): + """Test that when imported_module is imported, none of the modules in + modules_to_block are imported as a side effect.""" + modules_to_block = frozenset(modules_to_block) + script = textwrap.dedent( + f""" + import sys + modules_to_block = {modules_to_block} + if unexpected := modules_to_block & sys.modules.keys(): + startup = ", ".join(unexpected) + raise AssertionError(f'unexpectedly imported at startup: {{startup}}') + + import {imported_module} + if unexpected := modules_to_block & sys.modules.keys(): + after = ", ".join(unexpected) + raise AssertionError(f'unexpectedly imported after importing {imported_module}: {{after}}') + """ + ) + from .script_helper import assert_python_ok + assert_python_ok("-S", "-c", script) + + +@contextlib.contextmanager +def module_restored(name): + """A context manager that restores a module to the original state.""" + missing = object() + orig = sys.modules.get(name, missing) + if orig is None: + mod = importlib.import_module(name) + else: + mod = type(sys)(name) + mod.__dict__.update(orig.__dict__) + sys.modules[name] = mod + try: + yield mod + finally: + if orig is missing: + sys.modules.pop(name, None) + else: + sys.modules[name] = orig + + +def create_module(name, loader=None, *, ispkg=False): + """Return a new, empty module.""" + spec = importlib.machinery.ModuleSpec( + name, + loader, + origin='', + is_package=ispkg, + ) + return importlib.util.module_from_spec(spec) + + +def _ensure_module(name, ispkg, addparent, clearnone): + try: + mod = orig = sys.modules[name] + except KeyError: + mod = orig = None + missing = True + else: + missing = False + if mod is not None: + # It was already imported. + return mod, orig, missing + # Otherwise, None means it was explicitly disabled. + + assert name != '__main__' + if not missing: + assert orig is None, (name, sys.modules[name]) + if not clearnone: + raise ModuleNotFoundError(name) + del sys.modules[name] + # Try normal import, then fall back to adding the module. + try: + mod = importlib.import_module(name) + except ModuleNotFoundError: + if addparent and not clearnone: + addparent = None + mod = _add_module(name, ispkg, addparent) + return mod, orig, missing + + +def _add_module(spec, ispkg, addparent): + if isinstance(spec, str): + name = spec + mod = create_module(name, ispkg=ispkg) + spec = mod.__spec__ + else: + name = spec.name + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + if addparent is not False and spec.parent: + _ensure_module(spec.parent, True, addparent, bool(addparent)) + return mod + + +def add_module(spec, *, parents=True): + """Return the module after creating it and adding it to sys.modules. + + If parents is True then also create any missing parents. + """ + return _add_module(spec, False, parents) + + +def add_package(spec, *, parents=True): + """Return the module after creating it and adding it to sys.modules. + + If parents is True then also create any missing parents. + """ + return _add_module(spec, True, parents) + + +def ensure_module_imported(name, *, clearnone=True): + """Return the corresponding module. + + If it was already imported then return that. Otherwise, try + importing it (optionally clear it first if None). If that fails + then create a new empty module. + + It can be helpful to combine this with ready_to_import() and/or + isolated_modules(). + """ + if sys.modules.get(name) is not None: + mod = sys.modules[name] + else: + mod, _, _ = _ensure_module(name, False, True, clearnone) + return mod diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py deleted file mode 100644 index 5c484d1170d..00000000000 --- a/Lib/test/support/interpreters.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Subinterpreters High Level Module.""" - -import time -import _xxsubinterpreters as _interpreters -import _xxinterpchannels as _channels - -# aliases: -from _xxsubinterpreters import is_shareable, RunFailedError -from _xxinterpchannels import ( - ChannelError, ChannelNotFoundError, ChannelEmptyError, -) - - -__all__ = [ - 'Interpreter', 'get_current', 'get_main', 'create', 'list_all', - 'SendChannel', 'RecvChannel', - 'create_channel', 'list_all_channels', 'is_shareable', - 'ChannelError', 'ChannelNotFoundError', - 'ChannelEmptyError', - ] - - -def create(*, isolated=True): - """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=isolated) - return Interpreter(id, isolated=isolated) - - -def list_all(): - """Return all existing interpreters.""" - return [Interpreter(id) for id in _interpreters.list_all()] - - -def get_current(): - """Return the currently running interpreter.""" - id = _interpreters.get_current() - return Interpreter(id) - - -def get_main(): - """Return the main interpreter.""" - id = _interpreters.get_main() - return Interpreter(id) - - -class Interpreter: - """A single Python interpreter.""" - - def __init__(self, id, *, isolated=None): - if not isinstance(id, (int, _interpreters.InterpreterID)): - raise TypeError(f'id must be an int, got {id!r}') - self._id = id - self._isolated = isolated - - def __repr__(self): - data = dict(id=int(self._id), isolated=self._isolated) - kwargs = (f'{k}={v!r}' for k, v in data.items()) - return f'{type(self).__name__}({", ".join(kwargs)})' - - def __hash__(self): - return hash(self._id) - - def __eq__(self, other): - if not isinstance(other, Interpreter): - return NotImplemented - else: - return other._id == self._id - - @property - def id(self): - return self._id - - @property - def isolated(self): - if self._isolated is None: - # XXX The low-level function has not been added yet. - # See bpo-.... - self._isolated = _interpreters.is_isolated(self._id) - return self._isolated - - def is_running(self): - """Return whether or not the identified interpreter is running.""" - return _interpreters.is_running(self._id) - - def close(self): - """Finalize and destroy the interpreter. - - Attempting to destroy the current interpreter results - in a RuntimeError. - """ - return _interpreters.destroy(self._id) - - def run(self, src_str, /, *, channels=None): - """Run the given source code in the interpreter. - - This blocks the current Python thread until done. - """ - _interpreters.run_string(self._id, src_str, channels) - - -def create_channel(): - """Return (recv, send) for a new cross-interpreter channel. - - The channel may be used to pass data safely between interpreters. - """ - cid = _channels.create() - recv, send = RecvChannel(cid), SendChannel(cid) - return recv, send - - -def list_all_channels(): - """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid)) - for cid in _channels.list_all()] - - -class _ChannelEnd: - """The base class for RecvChannel and SendChannel.""" - - def __init__(self, id): - if not isinstance(id, (int, _channels.ChannelID)): - raise TypeError(f'id must be an int, got {id!r}') - self._id = id - - def __repr__(self): - return f'{type(self).__name__}(id={int(self._id)})' - - def __hash__(self): - return hash(self._id) - - def __eq__(self, other): - if isinstance(self, RecvChannel): - if not isinstance(other, RecvChannel): - return NotImplemented - elif not isinstance(other, SendChannel): - return NotImplemented - return other._id == self._id - - @property - def id(self): - return self._id - - -_NOT_SET = object() - - -class RecvChannel(_ChannelEnd): - """The receiving end of a cross-interpreter channel.""" - - def recv(self, *, _sentinel=object(), _delay=10 / 1000): # 10 milliseconds - """Return the next object from the channel. - - This blocks until an object has been sent, if none have been - sent already. - """ - obj = _channels.recv(self._id, _sentinel) - while obj is _sentinel: - time.sleep(_delay) - obj = _channels.recv(self._id, _sentinel) - return obj - - def recv_nowait(self, default=_NOT_SET): - """Return the next object from the channel. - - If none have been sent then return the default if one - is provided or fail with ChannelEmptyError. Otherwise this - is the same as recv(). - """ - if default is _NOT_SET: - return _channels.recv(self._id) - else: - return _channels.recv(self._id, default) - - -class SendChannel(_ChannelEnd): - """The sending end of a cross-interpreter channel.""" - - def send(self, obj): - """Send the object (i.e. its data) to the channel's receiving end. - - This blocks until the object is received. - """ - _channels.send(self._id, obj) - # XXX We are missing a low-level channel_send_wait(). - # See bpo-32604 and gh-19829. - # Until that shows up we fake it: - time.sleep(2) - - def send_nowait(self, obj): - """Send the object to the channel's receiving end. - - If the object is immediately received then return True - (else False). Otherwise this is the same as send(). - """ - # XXX Note that at the moment channel_send() only ever returns - # None. This should be fixed when channel_send_wait() is added. - # See bpo-32604 and gh-19829. - return _channels.send(self._id, obj) diff --git a/Lib/test/support/logging_helper.py b/Lib/test/support/logging_helper.py index 12fcca4f0f0..db556c7f5ad 100644 --- a/Lib/test/support/logging_helper.py +++ b/Lib/test/support/logging_helper.py @@ -1,5 +1,6 @@ import logging.handlers + class TestHandler(logging.handlers.BufferingHandler): def __init__(self, matcher): # BufferingHandler takes a "capacity" argument diff --git a/Lib/test/support/numbers.py b/Lib/test/support/numbers.py new file mode 100644 index 00000000000..d5dbb41aceb --- /dev/null +++ b/Lib/test/support/numbers.py @@ -0,0 +1,80 @@ +# These are shared with test_tokenize and other test modules. +# +# Note: since several test cases filter out floats by looking for "e" and ".", +# don't add hexadecimal literals that contain "e" or "E". +VALID_UNDERSCORE_LITERALS = [ + '0_0_0', + '4_2', + '1_0000_0000', + '0b1001_0100', + '0xffff_ffff', + '0o5_7_7', + '1_00_00.5', + '1_00_00.5e5', + '1_00_00e5_1', + '1e1_0', + '.1_4', + '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', + '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', +] +INVALID_UNDERSCORE_LITERALS = [ + # Trailing underscores: + '0_', + '42_', + '1.4j_', + '0x_', + '0b1_', + '0xf_', + '0o5_', + '0 if 1_Else 1', + # Underscores in the base selector: + '0_b0', + '0_xf', + '0_o5', + # Old-style octal, still disallowed: + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', + # Underscore right before a dot: + '1_.4', + '1_.4j', + # Underscore right after a dot: + '1._4', + '1._4j', + '._5', + '._5j', + # Underscore right after a sign: + '1.0e+_1', + '1.0e+_1j', + # Underscore right before j: + '1.4_j', + '1.4e5_j', + # Underscore right before e: + '1_e1', + '1.4_e1', + '1.4_e1j', + # Underscore right after e: + '1e_1', + '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', +] diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 821a4b1ffd5..d3d6fa632f9 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -1,6 +1,7 @@ import collections.abc import contextlib import errno +import logging import os import re import stat @@ -10,6 +11,7 @@ import unittest import warnings +from test import support # Filename used for testing TESTFN_ASCII = '@test' @@ -20,8 +22,8 @@ # TESTFN_UNICODE is a non-ascii filename TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f" -if sys.platform == 'darwin': - # In Mac OS X's VFS API file names are, by definition, canonically +if support.is_apple: + # On Apple's VFS API file names are, by definition, canonically # decomposed Unicode, encoded using UTF-8. See QA1173: # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html import unicodedata @@ -46,8 +48,8 @@ 'encoding (%s). Unicode filename tests may not be effective' % (TESTFN_UNENCODABLE, sys.getfilesystemencoding())) TESTFN_UNENCODABLE = None -# macOS and Emscripten deny unencodable filenames (invalid utf-8) -elif sys.platform not in {'darwin', 'emscripten', 'wasi'}: +# Apple and Emscripten deny unencodable filenames (invalid utf-8) +elif not support.is_apple and sys.platform not in {"emscripten", "wasi"}: try: # ascii and utf-8 cannot encode the byte 0xff b'\xff'.decode(sys.getfilesystemencoding()) @@ -196,6 +198,23 @@ def skip_unless_symlink(test): return test if ok else unittest.skip(msg)(test) +_can_hardlink = None + +def can_hardlink(): + global _can_hardlink + if _can_hardlink is None: + # Android blocks hard links using SELinux + # (https://stackoverflow.com/q/32365690). + _can_hardlink = hasattr(os, "link") and not support.is_android + return _can_hardlink + + +def skip_unless_hardlink(test): + ok = can_hardlink() + msg = "requires hardlink support" + return test if ok else unittest.skip(msg)(test) + + _can_xattr = None @@ -245,15 +264,15 @@ def can_chmod(): global _can_chmod if _can_chmod is not None: return _can_chmod - if not hasattr(os, "chown"): + if not hasattr(os, "chmod"): _can_chmod = False return _can_chmod try: with open(TESTFN, "wb") as f: try: - os.chmod(TESTFN, 0o777) + os.chmod(TESTFN, 0o555) mode1 = os.stat(TESTFN).st_mode - os.chmod(TESTFN, 0o666) + os.chmod(TESTFN, 0o777) mode2 = os.stat(TESTFN).st_mode except OSError as e: can = False @@ -275,6 +294,33 @@ def skip_unless_working_chmod(test): return test if ok else unittest.skip(msg)(test) +@contextlib.contextmanager +def save_mode(path, *, quiet=False): + """Context manager that restores the mode (permissions) of *path* on exit. + + Arguments: + + path: Path of the file to restore the mode of. + + quiet: if False (the default), the context manager raises an exception + on error. Otherwise, it issues only a warning and keeps the current + working directory the same. + + """ + saved_mode = os.stat(path) + try: + yield + finally: + try: + os.chmod(path, saved_mode.st_mode) + except OSError as exc: + if not quiet: + raise + warnings.warn(f'tests may fail, unable to restore the mode of ' + f'{path!r} to {saved_mode.st_mode}: {exc}', + RuntimeWarning, stacklevel=3) + + # Check whether the current effective user has the capability to override # DAC (discretionary access control). Typically user root is able to # bypass file read, write, and execute permission checks. The capability @@ -300,6 +346,10 @@ def can_dac_override(): else: _can_dac_override = True finally: + try: + os.chmod(TESTFN, 0o700) + except OSError: + pass unlink(TESTFN) return _can_dac_override @@ -355,8 +405,12 @@ def _waitfor(func, pathname, waitall=False): # Increase the timeout and try again time.sleep(timeout) timeout *= 2 - warnings.warn('tests may fail, delete still pending for ' + pathname, - RuntimeWarning, stacklevel=4) + logging.getLogger(__name__).warning( + 'tests may fail, delete still pending for %s', + pathname, + stack_info=True, + stacklevel=4, + ) def _unlink(filename): _waitfor(os.unlink, filename) @@ -471,9 +525,14 @@ def temp_dir(path=None, quiet=False): except OSError as exc: if not quiet: raise - warnings.warn(f'tests may fail, unable to create ' - f'temporary directory {path!r}: {exc}', - RuntimeWarning, stacklevel=3) + logging.getLogger(__name__).warning( + "tests may fail, unable to create temporary directory %r: %s", + path, + exc, + exc_info=exc, + stack_info=True, + stacklevel=3, + ) if dir_created: pid = os.getpid() try: @@ -504,9 +563,15 @@ def change_cwd(path, quiet=False): except OSError as exc: if not quiet: raise - warnings.warn(f'tests may fail, unable to change the current working ' - f'directory to {path!r}: {exc}', - RuntimeWarning, stacklevel=3) + logging.getLogger(__name__).warning( + 'tests may fail, unable to change the current working directory ' + 'to %r: %s', + path, + exc, + exc_info=exc, + stack_info=True, + stacklevel=3, + ) try: yield os.getcwd() finally: @@ -589,11 +654,18 @@ def __fspath__(self): def fd_count(): """Count the number of open file descriptors. """ - if sys.platform.startswith(('linux', 'freebsd', 'emscripten')): + if sys.platform.startswith(('linux', 'android', 'freebsd', 'emscripten')): + fd_path = "/proc/self/fd" + elif support.is_apple: + fd_path = "/dev/fd" + else: + fd_path = None + + if fd_path is not None: try: - names = os.listdir("/proc/self/fd") + names = os.listdir(fd_path) # Subtract one because listdir() internally opens a file - # descriptor to list the content of the /proc/self/fd/ directory. + # descriptor to list the content of the directory. return len(names) - 1 except FileNotFoundError: pass @@ -663,9 +735,10 @@ def temp_umask(umask): class EnvironmentVarGuard(collections.abc.MutableMapping): + """Class to help protect the environment variable properly. - """Class to help protect the environment variable properly. Can be used as - a context manager.""" + Can be used as a context manager. + """ def __init__(self): self._environ = os.environ @@ -699,8 +772,10 @@ def __len__(self): def set(self, envvar, value): self[envvar] = value - def unset(self, envvar): - del self[envvar] + def unset(self, envvar, /, *envvars): + """Unset one or more environment variables.""" + for ev in (envvar, *envvars): + del self[ev] def copy(self): # We do what os.environ.copy() does. @@ -720,13 +795,16 @@ def __exit__(self, *ignore_exc): try: - import ctypes - kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) - - ERROR_FILE_NOT_FOUND = 2 - DDD_REMOVE_DEFINITION = 2 - DDD_EXACT_MATCH_ON_REMOVE = 4 - DDD_NO_BROADCAST_SYSTEM = 8 + if support.MS_WINDOWS: + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + + ERROR_FILE_NOT_FOUND = 2 + DDD_REMOVE_DEFINITION = 2 + DDD_EXACT_MATCH_ON_REMOVE = 4 + DDD_NO_BROADCAST_SYSTEM = 8 + else: + raise AttributeError except (ImportError, AttributeError): def subst_drive(path): raise unittest.SkipTest('ctypes or kernel32 is not available') diff --git a/Lib/test/support/pty_helper.py b/Lib/test/support/pty_helper.py new file mode 100644 index 00000000000..7e1ae9e59b8 --- /dev/null +++ b/Lib/test/support/pty_helper.py @@ -0,0 +1,81 @@ +""" +Helper to run a script in a pseudo-terminal. +""" +import os +import selectors +import subprocess +import sys +from contextlib import ExitStack +from errno import EIO + +from test.support.import_helper import import_module + + +def run_pty(script, input=b"dummy input\r", env=None): + pty = import_module('pty') + output = bytearray() + [master, slave] = pty.openpty() + args = (sys.executable, '-c', script) + proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave, env=env) + os.close(slave) + with ExitStack() as cleanup: + cleanup.enter_context(proc) + def terminate(proc): + try: + proc.terminate() + except ProcessLookupError: + # Workaround for Open/Net BSD bug (Issue 16762) + pass + cleanup.callback(terminate, proc) + cleanup.callback(os.close, master) + # Avoid using DefaultSelector and PollSelector. Kqueue() does not + # work with pseudo-terminals on OS X < 10.9 (Issue 20365) and Open + # BSD (Issue 20667). Poll() does not work with OS X 10.6 or 10.4 + # either (Issue 20472). Hopefully the file descriptor is low enough + # to use with select(). + sel = cleanup.enter_context(selectors.SelectSelector()) + sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE) + os.set_blocking(master, False) + while True: + for [_, events] in sel.select(): + if events & selectors.EVENT_READ: + try: + chunk = os.read(master, 0x10000) + except OSError as err: + # Linux raises EIO when slave is closed (Issue 5380) + if err.errno != EIO: + raise + chunk = b"" + if not chunk: + return output + output.extend(chunk) + if events & selectors.EVENT_WRITE: + try: + input = input[os.write(master, input):] + except OSError as err: + # Apparently EIO means the slave was closed + if err.errno != EIO: + raise + input = b"" # Stop writing + if not input: + sel.modify(master, selectors.EVENT_READ) + + +###################################################################### +## Fake stdin (for testing interactive debugging) +###################################################################### + +class FakeInput: + """ + A fake input stream for pdb's interactive debugger. Whenever a + line is read, print it (to simulate the user typing it), and then + return it. The set of lines to return is specified in the + constructor; they should not have trailing newlines. + """ + def __init__(self, lines): + self.lines = lines + + def readline(self): + line = self.lines.pop(0) + print(line) + return line + '\n' diff --git a/Lib/test/support/refleak_helper.py b/Lib/test/support/refleak_helper.py new file mode 100644 index 00000000000..2f86c93a1e2 --- /dev/null +++ b/Lib/test/support/refleak_helper.py @@ -0,0 +1,8 @@ +""" +Utilities for changing test behaviour while hunting +for refleaks +""" + +_hunting_for_refleaks = False +def hunting_for_refleaks(): + return _hunting_for_refleaks diff --git a/Lib/test/support/rustpython.py b/Lib/test/support/rustpython.py new file mode 100644 index 00000000000..8ed7bc24dcf --- /dev/null +++ b/Lib/test/support/rustpython.py @@ -0,0 +1,24 @@ +""" +RustPython specific helpers. +""" + +import doctest + + +# copied from https://github.com/RustPython/RustPython/pull/6919 +EXPECTED_FAILURE = doctest.register_optionflag("EXPECTED_FAILURE") + + +class DocTestChecker(doctest.OutputChecker): + """ + Custom output checker that lets us add: `+EXPECTED_FAILURE` for doctest tests. + + We want to be able to mark failing doctest test the same way we do with normal + unit test, without this class we would have to skip the doctest for the CI to pass. + """ + + def check_output(self, want, got, optionflags): + res = super().check_output(want, got, optionflags) + if optionflags & EXPECTED_FAILURE: + res = not res + return res diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index c2b43f4060e..a338f484449 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -3,18 +3,16 @@ import collections import importlib -import sys import os import os.path -import subprocess import py_compile -import zipfile - +import subprocess +import sys from importlib.util import source_from_cache + from test import support from test.support.import_helper import make_legacy_pyc - # Cached result of the expensive test performed in the function below. __cached_interp_requires_environment = None @@ -64,42 +62,59 @@ class _PythonRunResult(collections.namedtuple("_PythonRunResult", """Helper for reporting Python subprocess run results""" def fail(self, cmd_line): """Provide helpful details about failed subcommand runs""" - # Limit to 80 lines to ASCII characters - maxlen = 80 * 100 + # Limit to 300 lines of ASCII characters + maxlen = 300 * 100 out, err = self.out, self.err if len(out) > maxlen: out = b'(... truncated stdout ...)' + out[-maxlen:] if len(err) > maxlen: err = b'(... truncated stderr ...)' + err[-maxlen:] - out = out.decode('ascii', 'replace').rstrip() - err = err.decode('ascii', 'replace').rstrip() - raise AssertionError("Process return code is %d\n" - "command line: %r\n" - "\n" - "stdout:\n" - "---\n" - "%s\n" - "---\n" - "\n" - "stderr:\n" - "---\n" - "%s\n" - "---" - % (self.rc, cmd_line, - out, - err)) + out = out.decode('utf8', 'replace').rstrip() + err = err.decode('utf8', 'replace').rstrip() + + exitcode = self.rc + signame = support.get_signal_name(exitcode) + if signame: + exitcode = f"{exitcode} ({signame})" + raise AssertionError(f"Process return code is {exitcode}\n" + f"command line: {cmd_line!r}\n" + f"\n" + f"stdout:\n" + f"---\n" + f"{out}\n" + f"---\n" + f"\n" + f"stderr:\n" + f"---\n" + f"{err}\n" + f"---") # Executing the interpreter in a subprocess @support.requires_subprocess() def run_python_until_end(*args, **env_vars): + """Used to implement assert_python_*. + + *args are the command line flags to pass to the python interpreter. + **env_vars keyword arguments are environment variables to set on the process. + + If __run_using_command= is supplied, it must be a list of + command line arguments to prepend to the command line used. + Useful when you want to run another command that should launch the + python interpreter via its own arguments. ["/bin/echo", "--"] for + example could print the unquoted python command line instead of + run it. + """ env_required = interpreter_requires_environment() + run_using_command = env_vars.pop('__run_using_command', None) cwd = env_vars.pop('__cwd', None) if '__isolated' in env_vars: isolated = env_vars.pop('__isolated') else: isolated = not env_vars and not env_required cmd_line = [sys.executable, '-X', 'faulthandler'] + if run_using_command: + cmd_line = run_using_command + cmd_line if isolated: # isolated mode: ignore Python environment variables, ignore user # site-packages, and don't add the current directory to sys.path @@ -218,14 +233,19 @@ def make_script(script_dir, script_basename, source, omit_suffix=False): if not omit_suffix: script_filename += os.extsep + 'py' script_name = os.path.join(script_dir, script_filename) - # The script should be encoded to UTF-8, the default string encoding - with open(script_name, 'w', encoding='utf-8') as script_file: - script_file.write(source) + if isinstance(source, str): + # The script should be encoded to UTF-8, the default string encoding + with open(script_name, 'w', encoding='utf-8') as script_file: + script_file.write(source) + else: + with open(script_name, 'wb') as script_file: + script_file.write(source) importlib.invalidate_caches() return script_name def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): + import zipfile zip_filename = zip_basename+os.extsep+'zip' zip_name = os.path.join(zip_dir, zip_filename) with zipfile.ZipFile(zip_name, 'w') as zip_file: @@ -252,6 +272,7 @@ def make_pkg(pkg_dir, init_source=''): def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, source, depth=1, compiled=False): + import zipfile unlink = [] init_name = make_script(zip_dir, '__init__', '') unlink.append(init_name) diff --git a/Lib/smtpd.py b/Lib/test/support/smtpd.py similarity index 85% rename from Lib/smtpd.py rename to Lib/test/support/smtpd.py index 963e0a7689c..cf333aaf6b0 100755 --- a/Lib/smtpd.py +++ b/Lib/test/support/smtpd.py @@ -7,7 +7,7 @@ --nosetuid -n - This program generally tries to setuid `nobody', unless this flag is + This program generally tries to setuid 'nobody', unless this flag is set. The setuid call will fail if this program is not run as root (in which case, use this flag). @@ -17,7 +17,7 @@ --class classname -c classname - Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by + Use 'classname' as the concrete SMTP proxy class. Uses 'PureProxy' by default. --size limit @@ -39,8 +39,8 @@ Version: %(__version__)s -If localhost is not given then `localhost' is used, and if localport is not -given then 8025 is used. If remotehost is not given then `localhost' is used, +If localhost is not given then 'localhost' is used, and if localport is not +given then 8025 is used. If remotehost is not given then 'localhost' is used, and if remoteport is not given, then 25 is used. """ @@ -60,13 +60,6 @@ # SMTP errors from the backend server at all. This should be fixed # (contributions are welcome!). # -# MailmanProxy - An experimental hack to work with GNU Mailman -# . Using this server as your real incoming smtpd, your -# mailhost will automatically recognize and accept mail destined to Mailman -# lists when those lists are created. Every message not destined for a list -# gets forwarded to a real backend smtpd, as with PureProxy. Again, errors -# are not handled correctly yet. -# # # Author: Barry Warsaw # @@ -77,35 +70,22 @@ # - Handle more ESMTP extensions # - handle error codes from the backend smtpd -import sys -import os +import collections import errno import getopt -import time +import os import socket -import collections -from warnings import warn +import sys +import time from email._header_value_parser import get_addr_spec, get_angle_addr +from warnings import warn + +from test.support import asynchat, asyncore __all__ = [ "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", - "MailmanProxy", ] -warn( - 'The smtpd module is deprecated and unmaintained and will be removed ' - 'in Python 3.12. Please see aiosmtpd ' - '(https://aiosmtpd.readthedocs.io/) for the recommended replacement.', - DeprecationWarning, - stacklevel=2) - - -# These are imported after the above warning so that users get the correct -# deprecation warning. -import asyncore -import asynchat - - program = sys.argv[0] __version__ = 'Python SMTP proxy version 0.3' @@ -654,7 +634,8 @@ def __init__(self, localaddr, remoteaddr, " be set to True at the same time") asyncore.dispatcher.__init__(self, map=map) try: - gai_results = socket.getaddrinfo(*localaddr, + family = 0 if socket.has_ipv6 else socket.AF_INET + gai_results = socket.getaddrinfo(*localaddr, family=family, type=socket.SOCK_STREAM) self.create_socket(gai_results[0][0], gai_results[0][1]) # try to re-use a server port if possible @@ -693,9 +674,9 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): message to. data is a string containing the entire full text of the message, - headers (if supplied) and all. It has been `de-transparencied' + headers (if supplied) and all. It has been 'de-transparencied' according to RFC 821, Section 4.5.2. In other words, a line - containing a `.' followed by other text has had the leading dot + containing a '.' followed by other text has had the leading dot removed. kwargs is a dictionary containing additional information. It is @@ -706,7 +687,7 @@ def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): ['BODY=8BITMIME', 'SMTPUTF8']. 'rcpt_options': same, for the rcpt command. - This function should return None for a normal `250 Ok' response; + This function should return None for a normal '250 Ok' response; otherwise, it should return the desired response string in RFC 821 format. @@ -789,91 +770,6 @@ def _deliver(self, mailfrom, rcpttos, data): return refused -class MailmanProxy(PureProxy): - def __init__(self, *args, **kwargs): - warn('MailmanProxy is deprecated and will be removed ' - 'in future', DeprecationWarning, 2) - if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: - raise ValueError("MailmanProxy does not support SMTPUTF8.") - super(PureProxy, self).__init__(*args, **kwargs) - - def process_message(self, peer, mailfrom, rcpttos, data): - from io import StringIO - from Mailman import Utils - from Mailman import Message - from Mailman import MailList - # If the message is to a Mailman mailing list, then we'll invoke the - # Mailman script directly, without going through the real smtpd. - # Otherwise we'll forward it to the local proxy for disposition. - listnames = [] - for rcpt in rcpttos: - local = rcpt.lower().split('@')[0] - # We allow the following variations on the theme - # listname - # listname-admin - # listname-owner - # listname-request - # listname-join - # listname-leave - parts = local.split('-') - if len(parts) > 2: - continue - listname = parts[0] - if len(parts) == 2: - command = parts[1] - else: - command = '' - if not Utils.list_exists(listname) or command not in ( - '', 'admin', 'owner', 'request', 'join', 'leave'): - continue - listnames.append((rcpt, listname, command)) - # Remove all list recipients from rcpttos and forward what we're not - # going to take care of ourselves. Linear removal should be fine - # since we don't expect a large number of recipients. - for rcpt, listname, command in listnames: - rcpttos.remove(rcpt) - # If there's any non-list destined recipients left, - print('forwarding recips:', ' '.join(rcpttos), file=DEBUGSTREAM) - if rcpttos: - refused = self._deliver(mailfrom, rcpttos, data) - # TBD: what to do with refused addresses? - print('we got refusals:', refused, file=DEBUGSTREAM) - # Now deliver directly to the list commands - mlists = {} - s = StringIO(data) - msg = Message.Message(s) - # These headers are required for the proper execution of Mailman. All - # MTAs in existence seem to add these if the original message doesn't - # have them. - if not msg.get('from'): - msg['From'] = mailfrom - if not msg.get('date'): - msg['Date'] = time.ctime(time.time()) - for rcpt, listname, command in listnames: - print('sending message to', rcpt, file=DEBUGSTREAM) - mlist = mlists.get(listname) - if not mlist: - mlist = MailList.MailList(listname, lock=0) - mlists[listname] = mlist - # dispatch on the type of command - if command == '': - # post - msg.Enqueue(mlist, tolist=1) - elif command == 'admin': - msg.Enqueue(mlist, toadmin=1) - elif command == 'owner': - msg.Enqueue(mlist, toowner=1) - elif command == 'request': - msg.Enqueue(mlist, torequest=1) - elif command in ('join', 'leave'): - # TBD: this is a hack! - if command == 'join': - msg['Subject'] = 'subscribe' - else: - msg['Subject'] = 'unsubscribe' - msg.Enqueue(mlist, torequest=1) - - class Options: setuid = True classname = 'PureProxy' diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index d9c087c251e..655ffbea0db 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -2,13 +2,12 @@ import errno import os.path import socket -import sys import subprocess +import sys import tempfile import unittest from .. import support -from . import warnings_helper HOST = "localhost" HOSTv4 = "127.0.0.1" @@ -196,7 +195,6 @@ def get_socket_conn_refused_errs(): def transient_internet(resource_name, *, timeout=_NOT_SET, errnos=()): """Return a context manager that raises ResourceDenied when various issues with the internet connection manifest themselves as exceptions.""" - nntplib = warnings_helper.import_deprecated("nntplib") import urllib.error if timeout is _NOT_SET: timeout = support.INTERNET_TIMEOUT @@ -249,10 +247,6 @@ def filter_error(err): if timeout is not None: socket.setdefaulttimeout(timeout) yield - except nntplib.NNTPTemporaryError as err: - if support.verbose: - sys.stderr.write(denied.args[0] + "\n") - raise denied from err except OSError as err: # urllib can wrap original socket errors multiple times (!), we must # unwrap to get at the original error. @@ -265,6 +259,10 @@ def filter_error(err): # raise OSError('socket error', msg) from msg elif len(a) >= 2 and isinstance(a[1], OSError): err = a[1] + # The error can also be wrapped as __cause__: + # raise URLError(f"ftp error: {exp}") from exp + elif isinstance(err, urllib.error.URLError) and err.__cause__: + err = err.__cause__ else: break filter_error(err) @@ -303,7 +301,7 @@ def _get_sysctl(name): stderr=subprocess.STDOUT, text=True) if proc.returncode: - support.print_warning(f'{" ".join(cmd)!r} command failed with ' + support.print_warning(f'{' '.join(cmd)!r} command failed with ' f'exit code {proc.returncode}') # cache the error to only log the warning once _sysctl_cache[name] = None @@ -314,7 +312,7 @@ def _get_sysctl(name): try: value = int(output.strip()) except Exception as exc: - support.print_warning(f'Failed to parse {" ".join(cmd)!r} ' + support.print_warning(f'Failed to parse {' '.join(cmd)!r} ' f'command output {output!r}: {exc!r}') # cache the error to only log the warning once _sysctl_cache[name] = None diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py new file mode 100644 index 00000000000..abc93dee2ce --- /dev/null +++ b/Lib/test/support/strace_helper.py @@ -0,0 +1,210 @@ +import os +import re +import sys +import textwrap +import unittest +from dataclasses import dataclass +from functools import cache + +from test import support +from test.support.script_helper import run_python_until_end + +_strace_binary = "/usr/bin/strace" +_syscall_regex = re.compile( + r"(?P[^(]*)\((?P[^)]*)\)\s*[=]\s*(?P.+)") +_returncode_regex = re.compile( + br"\+\+\+ exited with (?P\d+) \+\+\+") + + +@dataclass +class StraceEvent: + syscall: str + args: list[str] + returncode: str + + +@dataclass +class StraceResult: + strace_returncode: int + python_returncode: int + + """The event messages generated by strace. This is very similar to the + stderr strace produces with returncode marker section removed.""" + event_bytes: bytes + stdout: bytes + stderr: bytes + + def events(self): + """Parse event_bytes data into system calls for easier processing. + + This assumes the program under inspection doesn't print any non-utf8 + strings which would mix into the strace output.""" + decoded_events = self.event_bytes.decode('utf-8', 'surrogateescape') + matches = [ + _syscall_regex.match(event) + for event in decoded_events.splitlines() + ] + return [ + StraceEvent(match["syscall"], + [arg.strip() for arg in (match["args"].split(","))], + match["returncode"]) for match in matches if match + ] + + def sections(self): + """Find all "MARK " writes and use them to make groups of events. + + This is useful to avoid variable / overhead events, like those at + interpreter startup or when opening a file so a test can verify just + the small case under study.""" + current_section = "__startup" + sections = {current_section: []} + for event in self.events(): + if event.syscall == 'write' and len( + event.args) > 2 and event.args[1].startswith("\"MARK "): + # Found a new section, don't include the write in the section + # but all events until next mark should be in that section + current_section = event.args[1].split( + " ", 1)[1].removesuffix('\\n"') + if current_section not in sections: + sections[current_section] = list() + else: + sections[current_section].append(event) + + return sections + +def _filter_memory_call(call): + # mmap can operate on a fd or "MAP_ANONYMOUS" which gives a block of memory. + # Ignore "MAP_ANONYMOUS + the "MAP_ANON" alias. + if call.syscall == "mmap" and "MAP_ANON" in call.args[3]: + return True + + if call.syscall in ("munmap", "mprotect"): + return True + + return False + + +def filter_memory(syscalls): + """Filter out memory allocation calls from File I/O calls. + + Some calls (mmap, munmap, etc) can be used on files or to just get a block + of memory. Use this function to filter out the memory related calls from + other calls.""" + + return [call for call in syscalls if not _filter_memory_call(call)] + + +@support.requires_subprocess() +def strace_python(code, strace_flags, check=True): + """Run strace and return the trace. + + Sets strace_returncode and python_returncode to `-1` on error.""" + res = None + + def _make_error(reason, details): + return StraceResult( + strace_returncode=-1, + python_returncode=-1, + event_bytes= f"error({reason},details={details!r}) = -1".encode('utf-8'), + stdout=res.out if res else b"", + stderr=res.err if res else b"") + + # Run strace, and get out the raw text + try: + res, cmd_line = run_python_until_end( + "-c", + textwrap.dedent(code), + __run_using_command=[_strace_binary] + strace_flags, + ) + except OSError as err: + return _make_error("Caught OSError", err) + + if check and res.rc: + res.fail(cmd_line) + + # Get out program returncode + stripped = res.err.strip() + output = stripped.rsplit(b"\n", 1) + if len(output) != 2: + return _make_error("Expected strace events and exit code line", + stripped[-50:]) + + returncode_match = _returncode_regex.match(output[1]) + if not returncode_match: + return _make_error("Expected to find returncode in last line.", + output[1][:50]) + + python_returncode = int(returncode_match["returncode"]) + if check and python_returncode: + res.fail(cmd_line) + + return StraceResult(strace_returncode=res.rc, + python_returncode=python_returncode, + event_bytes=output[0], + stdout=res.out, + stderr=res.err) + + +def get_events(code, strace_flags, prelude, cleanup): + # NOTE: The flush is currently required to prevent the prints from getting + # buffered and done all at once at exit + prelude = textwrap.dedent(prelude) + code = textwrap.dedent(code) + cleanup = textwrap.dedent(cleanup) + to_run = f""" +print("MARK prelude", flush=True) +{prelude} +print("MARK code", flush=True) +{code} +print("MARK cleanup", flush=True) +{cleanup} +print("MARK __shutdown", flush=True) + """ + trace = strace_python(to_run, strace_flags) + all_sections = trace.sections() + return all_sections['code'] + + +def get_syscalls(code, strace_flags, prelude="", cleanup="", + ignore_memory=True): + """Get the syscalls which a given chunk of python code generates""" + events = get_events(code, strace_flags, prelude=prelude, cleanup=cleanup) + + if ignore_memory: + events = filter_memory(events) + + return [ev.syscall for ev in events] + + +# Moderately expensive (spawns a subprocess), so share results when possible. +@cache +def _can_strace(): + res = strace_python("import sys; sys.exit(0)", + # --trace option needs strace 5.5 (gh-133741) + ["--trace=%process"], + check=False) + if res.strace_returncode == 0 and res.python_returncode == 0: + assert res.events(), "Should have parsed multiple calls" + return True + return False + + +def requires_strace(): + if sys.platform != "linux": + return unittest.skip("Linux only, requires strace.") + + if "LD_PRELOAD" in os.environ: + # Distribution packaging (ex. Debian `fakeroot` and Gentoo `sandbox`) + # use LD_PRELOAD to intercept system calls, which changes the overall + # set of system calls which breaks tests expecting a specific set of + # system calls). + return unittest.skip("Not supported when LD_PRELOAD is intercepting system calls.") + + if support.check_sanitizer(address=True, memory=True): + return unittest.skip("LeakSanitizer does not work under ptrace (strace, gdb, etc)") + + return unittest.skipUnless(_can_strace(), "Requires working strace") + + +__all__ = ["filter_memory", "get_events", "get_syscalls", "requires_strace", + "strace_python", "StraceEvent", "StraceResult"] diff --git a/Lib/test/support/testcase.py b/Lib/test/support/testcase.py new file mode 100644 index 00000000000..e617b19b6ac --- /dev/null +++ b/Lib/test/support/testcase.py @@ -0,0 +1,123 @@ +from math import copysign, isnan + + +# XXX: RUSTPYTHON: removed in 3.14 +class ExtraAssertions: + + def assertIsSubclass(self, cls, superclass, msg=None): + if issubclass(cls, superclass): + return + standardMsg = f'{cls!r} is not a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotIsSubclass(self, cls, superclass, msg=None): + if not issubclass(cls, superclass): + return + standardMsg = f'{cls!r} is a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertHasAttr(self, obj, name, msg=None): + if not hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has no attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has no attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has no attribute {name!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotHasAttr(self, obj, name, msg=None): + if hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has unexpected attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has unexpected attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has unexpected attribute {name!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertStartsWith(self, s, prefix, msg=None): + if s.startswith(prefix): + return + standardMsg = f"{s!r} doesn't start with {prefix!r}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotStartsWith(self, s, prefix, msg=None): + if not s.startswith(prefix): + return + self.fail(self._formatMessage(msg, f"{s!r} starts with {prefix!r}")) + + def assertEndsWith(self, s, suffix, msg=None): + if s.endswith(suffix): + return + standardMsg = f"{s!r} doesn't end with {suffix!r}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotEndsWith(self, s, suffix, msg=None): + if not s.endswith(suffix): + return + self.fail(self._formatMessage(msg, f"{s!r} ends with {suffix!r}")) + + +class ExceptionIsLikeMixin: + def assertExceptionIsLike(self, exc, template): + """ + Passes when the provided `exc` matches the structure of `template`. + Individual exceptions don't have to be the same objects or even pass + an equality test: they only need to be the same type and contain equal + `exc_obj.args`. + """ + if exc is None and template is None: + return + + if template is None: + self.fail(f"unexpected exception: {exc}") + + if exc is None: + self.fail(f"expected an exception like {template!r}, got None") + + if not isinstance(exc, ExceptionGroup): + self.assertEqual(exc.__class__, template.__class__) + self.assertEqual(exc.args[0], template.args[0]) + else: + self.assertEqual(exc.message, template.message) + self.assertEqual(len(exc.exceptions), len(template.exceptions)) + for e, t in zip(exc.exceptions, template.exceptions): + self.assertExceptionIsLike(e, t) + + +class FloatsAreIdenticalMixin: + def assertFloatsAreIdentical(self, x, y): + """Fail unless floats x and y are identical, in the sense that: + (1) both x and y are nans, or + (2) both x and y are infinities, with the same sign, or + (3) both x and y are zeros, with the same sign, or + (4) x and y are both finite and nonzero, and x == y + + """ + msg = 'floats {!r} and {!r} are not identical' + + if isnan(x) or isnan(y): + if isnan(x) and isnan(y): + return + elif x == y: + if x != 0.0: + return + # both zero; check that signs match + elif copysign(1.0, x) == copysign(1.0, y): + return + else: + msg += ': zeros have different signs' + self.fail(msg.format(x, y)) + + +class ComplexesAreIdenticalMixin(FloatsAreIdenticalMixin): + def assertComplexesAreIdentical(self, x, y): + """Fail unless complex numbers x and y have equal values and signs. + + In particular, if x and y both have real (or imaginary) part + zero, but the zeros have different signs, this test will fail. + + """ + self.assertFloatsAreIdentical(x.real, y.real) + self.assertFloatsAreIdentical(x.imag, y.imag) diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index 7f16050f32b..9b2b8f2dff0 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -8,7 +8,6 @@ from test import support - #======================================================================= # Threading support to prevent reporting refleaks when running regrtest.py -R @@ -22,34 +21,37 @@ def threading_setup(): - return _thread._count(), threading._dangling.copy() + return _thread._count(), len(threading._dangling) def threading_cleanup(*original_values): - _MAX_COUNT = 100 - - for count in range(_MAX_COUNT): - values = _thread._count(), threading._dangling - if values == original_values: - break - - if not count: - # Display a warning at the first iteration - support.environment_altered = True - dangling_threads = values[1] - support.print_warning(f"threading_cleanup() failed to cleanup " - f"{values[0] - original_values[0]} threads " - f"(count: {values[0]}, " - f"dangling: {len(dangling_threads)})") - for thread in dangling_threads: - support.print_warning(f"Dangling thread: {thread!r}") - - # Don't hold references to threads - dangling_threads = None - values = None - - time.sleep(0.01) - support.gc_collect() + orig_count, orig_ndangling = original_values + + timeout = 1.0 + for _ in support.sleeping_retry(timeout, error=False): + # Copy the thread list to get a consistent output. threading._dangling + # is a WeakSet, its value changes when it's read. + dangling_threads = list(threading._dangling) + count = _thread._count() + + if count <= orig_count: + return + + # Timeout! + support.environment_altered = True + support.print_warning( + f"threading_cleanup() failed to clean up threads " + f"in {timeout:.1f} seconds\n" + f" before: thread count={orig_count}, dangling={orig_ndangling}\n" + f" after: thread count={count}, dangling={len(dangling_threads)}") + for thread in dangling_threads: + support.print_warning(f"Dangling thread: {thread!r}") + + # The warning happens when a test spawns threads and some of these threads + # are still running after the test completes. To fix this warning, join + # threads explicitly to wait until they complete. + # + # To make the warning more likely, reduce the timeout. def reap_threads(func): @@ -245,3 +247,27 @@ def requires_working_threading(*, module=False): raise unittest.SkipTest(msg) else: return unittest.skipUnless(can_start_thread, msg) + + +def run_concurrently(worker_func, nthreads, args=(), kwargs={}): + """ + Run the worker function concurrently in multiple threads. + """ + barrier = threading.Barrier(nthreads) + + def wrapper_func(*args, **kwargs): + # Wait for all threads to reach this point before proceeding. + barrier.wait() + worker_func(*args, **kwargs) + + with catch_threading_exception() as cm: + workers = [ + threading.Thread(target=wrapper_func, args=args, kwargs=kwargs) + for _ in range(nthreads) + ] + with start_threads(workers): + pass + + # If a worker thread raises an exception, re-raise it. + if cm.exc_value is not None: + raise cm.exc_value diff --git a/Lib/test/support/venv.py b/Lib/test/support/venv.py new file mode 100644 index 00000000000..b60f6097e65 --- /dev/null +++ b/Lib/test/support/venv.py @@ -0,0 +1,81 @@ +import contextlib +import logging +import os +import shlex +import subprocess +import sys +import sysconfig +import tempfile +import venv + + +class VirtualEnvironment: + def __init__(self, prefix, **venv_create_args): + self._logger = logging.getLogger(self.__class__.__name__) + venv.create(prefix, **venv_create_args) + self._prefix = prefix + self._paths = sysconfig.get_paths( + scheme='venv', + vars={'base': self.prefix}, + expand=True, + ) + + @classmethod + @contextlib.contextmanager + def from_tmpdir(cls, *, prefix=None, dir=None, **venv_create_args): + delete = not bool(os.environ.get('PYTHON_TESTS_KEEP_VENV')) + with tempfile.TemporaryDirectory(prefix=prefix, dir=dir, delete=delete) as tmpdir: + yield cls(tmpdir, **venv_create_args) + + @property + def prefix(self): + return self._prefix + + @property + def paths(self): + return self._paths + + @property + def interpreter(self): + return os.path.join(self.paths['scripts'], os.path.basename(sys.executable)) + + def _format_output(self, name, data, indent='\t'): + if not data: + return indent + f'{name}: (none)' + if len(data.splitlines()) == 1: + return indent + f'{name}: {data}' + else: + prefixed_lines = '\n'.join(indent + '> ' + line for line in data.splitlines()) + return indent + f'{name}:\n' + prefixed_lines + + def run(self, *args, **subprocess_args): + if subprocess_args.get('shell'): + raise ValueError('Running the subprocess in shell mode is not supported.') + default_args = { + 'capture_output': True, + 'check': True, + } + try: + result = subprocess.run([self.interpreter, *args], **default_args | subprocess_args) + except subprocess.CalledProcessError as e: + if e.returncode != 0: + self._logger.error( + f'Interpreter returned non-zero exit status {e.returncode}.\n' + + self._format_output('COMMAND', shlex.join(e.cmd)) + '\n' + + self._format_output('STDOUT', e.stdout.decode()) + '\n' + + self._format_output('STDERR', e.stderr.decode()) + '\n' + ) + raise + else: + return result + + +class VirtualEnvironmentMixin: + def venv(self, name=None, **venv_create_args): + venv_name = self.id() + if name: + venv_name += f'-{name}' + return VirtualEnvironment.from_tmpdir( + prefix=f'{venv_name}-venv-', + **venv_create_args, + ) diff --git a/Lib/test/support/warnings_helper.py b/Lib/test/support/warnings_helper.py index c1bf0562300..5f6f14afd74 100644 --- a/Lib/test/support/warnings_helper.py +++ b/Lib/test/support/warnings_helper.py @@ -23,8 +23,7 @@ def check_syntax_warning(testcase, statement, errtext='', testcase.assertEqual(len(warns), 1, warns) warn, = warns - testcase.assertTrue(issubclass(warn.category, SyntaxWarning), - warn.category) + testcase.assertIsSubclass(warn.category, SyntaxWarning) if errtext: testcase.assertRegex(str(warn.message), errtext) testcase.assertEqual(warn.filename, '') @@ -160,11 +159,12 @@ def _filterwarnings(filters, quiet=False): registry = frame.f_globals.get('__warningregistry__') if registry: registry.clear() - with warnings.catch_warnings(record=True) as w: - # Set filter "always" to record all warnings. Because - # test_warnings swap the module, we need to look up in - # the sys.modules dictionary. - sys.modules['warnings'].simplefilter("always") + # Because test_warnings swap the module, we need to look up in the + # sys.modules dictionary. + wmod = sys.modules['warnings'] + with wmod.catch_warnings(record=True) as w: + # Set filter "always" to record all warnings. + wmod.simplefilter("always") yield WarningsRecorder(w) # Filter the recorded warnings reraise = list(w) diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index a620dd5b4ce..8ded9f99248 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -3,19 +3,22 @@ from test.support import warnings_helper import os import sys -import types - -try: - import _multiprocessing -except ModuleNotFoundError: - _multiprocessing = None if support.check_sanitizer(address=True, memory=True): - # bpo-46633: test___all__ is skipped because importing some modules - # directly can trigger known problems with ASAN (like tk or crypt). - raise unittest.SkipTest("workaround ASAN build issues on loading tests " - "like tk or crypt") + SKIP_MODULES = frozenset(( + # gh-90791: Tests involving libX11 can SEGFAULT on ASAN/MSAN builds. + # Skip modules, packages and tests using '_tkinter'. + '_tkinter', + 'tkinter', + 'test_tkinter', + 'test_ttk', + 'test_ttk_textonly', + 'idlelib', + 'test_idle', + )) +else: + SKIP_MODULES = () class NoAll(RuntimeError): @@ -27,17 +30,6 @@ class FailedImport(RuntimeError): class AllTest(unittest.TestCase): - def setUp(self): - # concurrent.futures uses a __getattr__ hook. Its __all__ triggers - # import of a submodule, which fails when _multiprocessing is not - # available. - if _multiprocessing is None: - sys.modules["_multiprocessing"] = types.ModuleType("_multiprocessing") - - def tearDown(self): - if _multiprocessing is None: - sys.modules.pop("_multiprocessing") - def check_all(self, modname): names = {} with warnings_helper.check_warnings( @@ -45,6 +37,7 @@ def check_all(self, modname): (".* (module|package)", DeprecationWarning), (".* (module|package)", PendingDeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("import %s" % modname, names) @@ -60,6 +53,7 @@ def check_all(self, modname): with warnings_helper.check_warnings( ("", DeprecationWarning), ("", ResourceWarning), + ("", SyntaxWarning), quiet=True): try: exec("from %s import *" % modname, names) @@ -78,23 +72,31 @@ def check_all(self, modname): all_set = set(all_list) self.assertCountEqual(all_set, all_list, "in module {}".format(modname)) self.assertEqual(keys, all_set, "in module {}".format(modname)) + # Verify __dir__ is non-empty and doesn't produce an error + self.assertTrue(dir(sys.modules[modname])) def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): path = os.path.join(basedir, fn) if os.path.isdir(path): + if fn in SKIP_MODULES: + continue pkg_init = os.path.join(path, '__init__.py') if os.path.exists(pkg_init): yield pkg_init, modpath + fn for p, m in self.walk_modules(path, modpath + fn + "."): yield p, m continue - if not fn.endswith('.py') or fn == '__init__.py': + + if fn == '__init__.py': continue - yield path, modpath + fn[:-3] - - # TODO: RUSTPYTHON - @unittest.expectedFailure + if not fn.endswith('.py'): + continue + modname = fn.removesuffix('.py') + if modname in SKIP_MODULES: + continue + yield path, modpath + modname + def test_all(self): # List of denied modules and packages denylist = set([ @@ -103,8 +105,9 @@ def test_all(self): ]) # In case _socket fails to build, make this test fail more gracefully - # than an AttributeError somewhere deep in CGIHTTPServer. - import _socket + # than an AttributeError somewhere deep in concurrent.futures, email + # or unittest. + import _socket # noqa: F401 ignored = [] failed_imports = [] @@ -120,14 +123,14 @@ def test_all(self): if denied: continue if support.verbose: - print(modname) + print(f"Check {modname}", flush=True) try: # This heuristic speeds up the process by removing, de facto, # most test modules (and avoiding the auto-executing ones). with open(path, "rb") as f: if b"__all__" not in f.read(): raise NoAll(modname) - self.check_all(modname) + self.check_all(modname) except NoAll: ignored.append(modname) except FailedImport: diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py new file mode 100644 index 00000000000..026277267e0 --- /dev/null +++ b/Lib/test/test__colorize.py @@ -0,0 +1,181 @@ +import contextlib +import dataclasses +import io +import sys +import unittest +import unittest.mock +import _colorize +from test.support.os_helper import EnvironmentVarGuard + + +@contextlib.contextmanager +def clear_env(): + with EnvironmentVarGuard() as mock_env: + mock_env.unset("FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS", "TERM") + yield mock_env + + +def supports_virtual_terminal(): + if sys.platform == "win32": + return unittest.mock.patch("nt._supports_virtual_terminal", return_value=True) + else: + return contextlib.nullcontext() + + +class TestTheme(unittest.TestCase): + + def test_attributes(self): + # only theme configurations attributes by default + for field in dataclasses.fields(_colorize.Theme): + with self.subTest(field.name): + self.assertIsSubclass(field.type, _colorize.ThemeSection) + self.assertIsNotNone(field.default_factory) + + def test_copy_with(self): + theme = _colorize.Theme() + + copy = theme.copy_with() + self.assertEqual(theme, copy) + + unittest_no_colors = _colorize.Unittest.no_colors() + copy = theme.copy_with(unittest=unittest_no_colors) + self.assertEqual(copy.argparse, theme.argparse) + self.assertEqual(copy.syntax, theme.syntax) + self.assertEqual(copy.traceback, theme.traceback) + self.assertEqual(copy.unittest, unittest_no_colors) + + def test_no_colors(self): + # idempotence test + theme_no_colors = _colorize.Theme().no_colors() + theme_no_colors_no_colors = theme_no_colors.no_colors() + self.assertEqual(theme_no_colors, theme_no_colors_no_colors) + + # attributes check + for section in dataclasses.fields(_colorize.Theme): + with self.subTest(section.name): + section_theme = getattr(theme_no_colors, section.name) + self.assertEqual(section_theme, section.type.no_colors()) + + +class TestColorizeFunction(unittest.TestCase): + def test_colorized_detection_checks_for_environment_variables(self): + def check(env, fallback, expected): + with (self.subTest(env=env, fallback=fallback), + clear_env() as mock_env): + mock_env.update(env) + isatty_mock.return_value = fallback + stdout_mock.isatty.return_value = fallback + self.assertEqual(_colorize.can_colorize(), expected) + + with (unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + supports_virtual_terminal()): + stdout_mock.fileno.return_value = 1 + + for fallback in False, True: + check({}, fallback, fallback) + check({'TERM': 'dumb'}, fallback, False) + check({'TERM': 'xterm'}, fallback, fallback) + check({'TERM': ''}, fallback, fallback) + check({'FORCE_COLOR': '1'}, fallback, True) + check({'FORCE_COLOR': '0'}, fallback, True) + check({'FORCE_COLOR': ''}, fallback, fallback) + check({'NO_COLOR': '1'}, fallback, False) + check({'NO_COLOR': '0'}, fallback, False) + check({'NO_COLOR': ''}, fallback, fallback) + + check({'TERM': 'dumb', 'FORCE_COLOR': '1'}, False, True) + check({'FORCE_COLOR': '1', 'NO_COLOR': '1'}, True, False) + + for ignore_environment in False, True: + # Simulate running with or without `-E`. + flags = unittest.mock.MagicMock(ignore_environment=ignore_environment) + with unittest.mock.patch("sys.flags", flags): + check({'PYTHON_COLORS': '1'}, True, True) + check({'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'PYTHON_COLORS': '0'}, True, ignore_environment) + check({'PYTHON_COLORS': '0'}, False, False) + for fallback in False, True: + check({'PYTHON_COLORS': 'x'}, fallback, fallback) + check({'PYTHON_COLORS': ''}, fallback, fallback) + + check({'TERM': 'dumb', 'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'NO_COLOR': '1', 'PYTHON_COLORS': '1'}, False, not ignore_environment) + check({'FORCE_COLOR': '1', 'PYTHON_COLORS': '0'}, True, ignore_environment) + + @unittest.skipUnless(sys.platform == "win32", "requires Windows") + def test_colorized_detection_checks_on_windows(self): + with (clear_env(), + unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + supports_virtual_terminal() as vt_mock): + stdout_mock.fileno.return_value = 1 + isatty_mock.return_value = True + stdout_mock.isatty.return_value = True + + vt_mock.return_value = True + self.assertEqual(_colorize.can_colorize(), True) + vt_mock.return_value = False + self.assertEqual(_colorize.can_colorize(), False) + import nt + del nt._supports_virtual_terminal + self.assertEqual(_colorize.can_colorize(), False) + + def test_colorized_detection_checks_for_std_streams(self): + with (clear_env(), + unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.stdout") as stdout_mock, + unittest.mock.patch("sys.stderr") as stderr_mock, + supports_virtual_terminal()): + stdout_mock.fileno.return_value = 1 + stderr_mock.fileno.side_effect = ZeroDivisionError + stderr_mock.isatty.side_effect = ZeroDivisionError + + isatty_mock.return_value = True + stdout_mock.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(), True) + + isatty_mock.return_value = False + stdout_mock.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(), False) + + def test_colorized_detection_checks_for_file(self): + with clear_env(), supports_virtual_terminal(): + + with unittest.mock.patch("os.isatty") as isatty_mock: + file = unittest.mock.MagicMock() + file.fileno.return_value = 1 + isatty_mock.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + isatty_mock.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + + # No file.fileno. + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock(spec=['isatty']) + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), False) + + # file.fileno() raises io.UnsupportedOperation. + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = io.UnsupportedOperation + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + + # The documentation for file.fileno says: + # > An OSError is raised if the IO object does not use a file descriptor. + # gh-141570: Check OSError is caught and handled + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = OSError + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py new file mode 100644 index 00000000000..11b2c9545a1 --- /dev/null +++ b/Lib/test/test__locale.py @@ -0,0 +1,300 @@ +from _locale import (setlocale, LC_ALL, LC_CTYPE, LC_NUMERIC, LC_TIME, localeconv, Error) +try: + from _locale import (RADIXCHAR, THOUSEP, nl_langinfo) +except ImportError: + nl_langinfo = None + +import locale +import sys +import unittest +from platform import uname + +from test import support + +if uname().system == "Darwin": + maj, min, mic = [int(part) for part in uname().release.split(".")] + if (maj, min, mic) < (8, 0, 0): + raise unittest.SkipTest("locale support broken for OS X < 10.4") + +candidate_locales = ['es_UY', 'fr_FR', 'fi_FI', 'es_CO', 'pt_PT', 'it_IT', + 'et_EE', 'es_PY', 'no_NO', 'nl_NL', 'lv_LV', 'el_GR', 'be_BY', 'fr_BE', + 'ro_RO', 'ru_UA', 'ru_RU', 'es_VE', 'ca_ES', 'se_NO', 'es_EC', 'id_ID', + 'ka_GE', 'es_CL', 'wa_BE', 'hu_HU', 'lt_LT', 'sl_SI', 'hr_HR', 'es_AR', + 'es_ES', 'oc_FR', 'gl_ES', 'bg_BG', 'is_IS', 'mk_MK', 'de_AT', 'pt_BR', + 'da_DK', 'nn_NO', 'cs_CZ', 'de_LU', 'es_BO', 'sq_AL', 'sk_SK', 'fr_CH', + 'de_DE', 'sr_YU', 'br_FR', 'nl_BE', 'sv_FI', 'pl_PL', 'fr_CA', 'fo_FO', + 'bs_BA', 'fr_LU', 'kl_GL', 'fa_IR', 'de_BE', 'sv_SE', 'it_CH', 'uk_UA', + 'eu_ES', 'vi_VN', 'af_ZA', 'nb_NO', 'en_DK', 'tg_TJ', 'ps_AF', 'en_US', + 'fr_FR.ISO8859-1', 'fr_FR.UTF-8', 'fr_FR.ISO8859-15@euro', + 'ru_RU.KOI8-R', 'ko_KR.eucKR', + 'ja_JP.UTF-8', 'lzh_TW.UTF-8', 'my_MM.UTF-8', 'or_IN.UTF-8', 'shn_MM.UTF-8', + 'ar_AE.UTF-8', 'bn_IN.UTF-8', 'mr_IN.UTF-8', 'th_TH.TIS620', +] + +def setUpModule(): + global candidate_locales + # Issue #13441: Skip some locales (e.g. cs_CZ and hu_HU) on Solaris to + # workaround a mbstowcs() bug. For example, on Solaris, the hu_HU locale uses + # the locale encoding ISO-8859-2, the thousands separator is b'\xA0' and it is + # decoded as U+30000020 (an invalid character) by mbstowcs(). + if sys.platform == 'sunos5': + old_locale = locale.setlocale(locale.LC_ALL) + try: + locales = [] + for loc in candidate_locales: + try: + locale.setlocale(locale.LC_ALL, loc) + except Error: + continue + encoding = locale.getencoding() + try: + localeconv() + except Exception as err: + print("WARNING: Skip locale %s (encoding %s): [%s] %s" + % (loc, encoding, type(err), err)) + else: + locales.append(loc) + candidate_locales = locales + finally: + locale.setlocale(locale.LC_ALL, old_locale) + + # Workaround for MSVC6(debug) crash bug + if "MSC v.1200" in sys.version: + def accept(loc): + a = loc.split(".") + return not(len(a) == 2 and len(a[-1]) >= 9) + candidate_locales = [loc for loc in candidate_locales if accept(loc)] + +# List known locale values to test against when available. +# Dict formatted as `` : (, )``. If a +# value is not known, use '' . +known_numerics = { + 'en_US': ('.', ','), + 'de_DE' : (',', '.'), + # The French thousands separator may be a breaking or non-breaking space + # depending on the platform, so do not test it + 'fr_FR' : (',', ''), + 'ps_AF': ('\u066b', '\u066c'), +} + +known_alt_digits = { + 'C': (0, {}), + 'en_US': (0, {}), + 'fa_IR': (100, {0: '\u06f0\u06f0', 10: '\u06f1\u06f0', 99: '\u06f9\u06f9'}), + 'ja_JP': (100, {1: '\u4e00', 10: '\u5341', 99: '\u4e5d\u5341\u4e5d'}), + 'lzh_TW': (32, {0: '\u3007', 10: '\u5341', 31: '\u5345\u4e00'}), + 'my_MM': (100, {0: '\u1040\u1040', 10: '\u1041\u1040', 99: '\u1049\u1049'}), + 'or_IN': (100, {0: '\u0b66', 10: '\u0b67\u0b66', 99: '\u0b6f\u0b6f'}), + 'shn_MM': (100, {0: '\u1090\u1090', 10: '\u1091\u1090', 99: '\u1099\u1099'}), + 'ar_AE': (100, {0: '\u0660', 10: '\u0661\u0660', 99: '\u0669\u0669'}), + 'bn_IN': (100, {0: '\u09e6', 10: '\u09e7\u09e6', 99: '\u09ef\u09ef'}), +} + +known_era = { + 'C': (0, ''), + 'en_US': (0, ''), + 'ja_JP': (11, '+:1:2019/05/01:2019/12/31:令和:%EC元年'), + 'zh_TW': (3, '+:1:1912/01/01:1912/12/31:民國:%EC元年'), + 'th_TW': (1, '+:1:-543/01/01:+*:พ.ศ.:%EC %Ey'), +} + +if sys.platform == 'win32': + # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) + del known_numerics['ps_AF'] + +if sys.platform == 'sunos5': + # On Solaris, Japanese ERAs start with the year 1927, + # and thus there's less of them. + known_era['ja_JP'] = (5, '+:1:2019/05/01:2019/12/31:令和:%EC元年') + +class _LocaleTests(unittest.TestCase): + + def setUp(self): + self.oldlocale = setlocale(LC_ALL) + + def tearDown(self): + setlocale(LC_ALL, self.oldlocale) + + # Want to know what value was calculated, what it was compared against, + # what function was used for the calculation, what type of data was used, + # the locale that was supposedly set, and the actual locale that is set. + lc_numeric_err_msg = "%s != %s (%s for %s; set to %s, using %s)" + + def numeric_tester(self, calc_type, calc_value, data_type, used_locale): + """Compare calculation against known value, if available""" + try: + set_locale = setlocale(LC_NUMERIC) + except Error: + set_locale = "" + known_value = known_numerics.get(used_locale, + ('', ''))[data_type == 'thousands_sep'] + if known_value and calc_value: + self.assertEqual(calc_value, known_value, + self.lc_numeric_err_msg % ( + calc_value, known_value, + calc_type, data_type, set_locale, + used_locale)) + return True + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_lc_numeric_nl_langinfo(self): + # Test nl_langinfo against known values + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + for li, lc in ((RADIXCHAR, "decimal_point"), + (THOUSEP, "thousands_sep")): + if self.numeric_tester('nl_langinfo', nl_langinfo(li), lc, loc): + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_lc_numeric_localeconv(self): + # Test localeconv against known values + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + formatting = localeconv() + for lc in ("decimal_point", + "thousands_sep"): + if self.numeric_tester('localeconv', formatting[lc], lc, loc): + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + def test_lc_numeric_basic(self): + # Test nl_langinfo against localeconv + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + for li, lc in ((RADIXCHAR, "decimal_point"), + (THOUSEP, "thousands_sep")): + nl_radixchar = nl_langinfo(li) + li_radixchar = localeconv()[lc] + try: + set_locale = setlocale(LC_NUMERIC) + except Error: + set_locale = "" + self.assertEqual(nl_radixchar, li_radixchar, + "%s (nl_langinfo) != %s (localeconv) " + "(set to %s, using %s)" % ( + nl_radixchar, li_radixchar, + loc, set_locale)) + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ALT_DIGITS'), "requires locale.ALT_DIGITS") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_alt_digits_nl_langinfo(self): + # Test nl_langinfo(ALT_DIGITS) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + alt_digits = nl_langinfo(locale.ALT_DIGITS) + self.assertIsInstance(alt_digits, str) + alt_digits = alt_digits.split(';') if alt_digits else [] + if alt_digits: + self.assertGreaterEqual(len(alt_digits), 10, alt_digits) + loc1 = loc.split('.', 1)[0] + if loc1 in known_alt_digits: + count, samples = known_alt_digits[loc1] + if count and not alt_digits: + self.skipTest(f'ALT_DIGITS is not set for locale {loc!r} on this platform') + self.assertEqual(len(alt_digits), count, alt_digits) + for i in samples: + self.assertEqual(alt_digits[i], samples[i]) + tested = True + if not tested: + self.skipTest('no suitable locales') + + @unittest.skipUnless(nl_langinfo, "nl_langinfo is not available") + @unittest.skipUnless(hasattr(locale, 'ERA'), "requires locale.ERA") + @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") + def test_era_nl_langinfo(self): + # Test nl_langinfo(ERA) + tested = False + for loc in candidate_locales: + with self.subTest(locale=loc): + try: + setlocale(LC_TIME, loc) + except Error: + self.skipTest(f'no locale {loc!r}') + continue + + with self.subTest(locale=loc): + era = nl_langinfo(locale.ERA) + self.assertIsInstance(era, str) + if era: + self.assertEqual(era.count(':'), (era.count(';') + 1) * 5, era) + + loc1 = loc.split('.', 1)[0] + if loc1 in known_era: + count, sample = known_era[loc1] + if count: + if not era: + self.skipTest(f'ERA is not set for locale {loc!r} on this platform') + self.assertGreaterEqual(era.count(';') + 1, count) + self.assertIn(sample, era) + else: + self.assertEqual(era, '') + tested = True + if not tested: + self.skipTest('no suitable locales') + + def test_float_parsing(self): + # Bug #1391872: Test whether float parsing is okay on European + # locales. + tested = False + oldloc = setlocale(LC_CTYPE) + for loc in candidate_locales: + try: + setlocale(LC_NUMERIC, loc) + except Error: + continue + + # Ignore buggy locale databases. (Mac OS 10.4 and some other BSDs) + if loc == 'eu_ES' and localeconv()['decimal_point'] == "' ": + continue + + self.assertEqual(int(eval('3.14') * 100), 314, + "using eval('3.14') failed for %s" % loc) + self.assertEqual(int(float('3.14') * 100), 314, + "using float('3.14') failed for %s" % loc) + if localeconv()['decimal_point'] != '.': + self.assertRaises(ValueError, float, + localeconv()['decimal_point'].join(['1', '23'])) + tested = True + self.assertEqual(setlocale(LC_CTYPE), oldloc) + if not tested: + self.skipTest('no suitable locales') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py new file mode 100644 index 00000000000..43d475baa5d --- /dev/null +++ b/Lib/test/test__opcode.py @@ -0,0 +1,147 @@ +import dis +from test.support.import_helper import import_module +import unittest +import opcode + +_opcode = import_module("_opcode") +from _opcode import stack_effect + + +class OpListTests(unittest.TestCase): + def check_bool_function_result(self, func, ops, expected): + for op in ops: + if isinstance(op, str): + op = dis.opmap[op] + with self.subTest(opcode=op, func=func): + self.assertIsInstance(func(op), bool) + self.assertEqual(func(op), expected) + + def test_invalid_opcodes(self): + invalid = [-100, -1, 512, 513, 1000] + self.check_bool_function_result(_opcode.is_valid, invalid, False) + self.check_bool_function_result(_opcode.has_arg, invalid, False) + self.check_bool_function_result(_opcode.has_const, invalid, False) + self.check_bool_function_result(_opcode.has_name, invalid, False) + self.check_bool_function_result(_opcode.has_jump, invalid, False) + self.check_bool_function_result(_opcode.has_free, invalid, False) + self.check_bool_function_result(_opcode.has_local, invalid, False) + self.check_bool_function_result(_opcode.has_exc, invalid, False) + + def test_is_valid(self): + names = [ + 'CACHE', + 'POP_TOP', + 'IMPORT_NAME', + 'JUMP', + 'INSTRUMENTED_RETURN_VALUE', + ] + opcodes = [dis.opmap[opname] for opname in names] + self.check_bool_function_result(_opcode.is_valid, opcodes, True) + + @unittest.expectedFailure # TODO: RUSTPYTHON; KeyError: 'BINARY_OP_ADD_INT' + def test_opmaps(self): + def check_roundtrip(name, map): + return self.assertEqual(opcode.opname[map[name]], name) + + check_roundtrip('BINARY_OP', opcode.opmap) + check_roundtrip('BINARY_OP_ADD_INT', opcode._specialized_opmap) + + def test_oplists(self): + def check_function(self, func, expected): + for op in [-10, 520]: + with self.subTest(opcode=op, func=func): + res = func(op) + self.assertIsInstance(res, bool) + self.assertEqual(res, op in expected) + + check_function(self, _opcode.has_arg, dis.hasarg) + check_function(self, _opcode.has_const, dis.hasconst) + check_function(self, _opcode.has_name, dis.hasname) + check_function(self, _opcode.has_jump, dis.hasjump) + check_function(self, _opcode.has_free, dis.hasfree) + check_function(self, _opcode.has_local, dis.haslocal) + check_function(self, _opcode.has_exc, dis.hasexc) + + +class StackEffectTests(unittest.TestCase): + def test_stack_effect(self): + self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 2), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 3), -2) + self.assertRaises(ValueError, stack_effect, 30000) + # All defined opcodes + has_arg = dis.hasarg + for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= opcode.MIN_INSTRUMENTED_OPCODE: + continue + with self.subTest(opname=name): + stack_effect(code) + stack_effect(code, 0) + # All not defined opcodes + for code in set(range(256)) - set(dis.opmap.values()): + with self.subTest(opcode=code): + self.assertRaises(ValueError, stack_effect, code) + self.assertRaises(ValueError, stack_effect, code, 0) + + def test_stack_effect_jump(self): + FOR_ITER = dis.opmap['FOR_ITER'] + self.assertEqual(stack_effect(FOR_ITER, 0), 1) + self.assertEqual(stack_effect(FOR_ITER, 0, jump=True), 1) + self.assertEqual(stack_effect(FOR_ITER, 0, jump=False), 1) + JUMP_FORWARD = dis.opmap['JUMP_FORWARD'] + self.assertEqual(stack_effect(JUMP_FORWARD, 0), 0) + self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=True), 0) + self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0) + # All defined opcodes + has_arg = dis.hasarg + has_exc = dis.hasexc + has_jump = dis.hasjabs + dis.hasjrel + for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= opcode.MIN_INSTRUMENTED_OPCODE: + continue + with self.subTest(opname=name): + if code not in has_arg: + common = stack_effect(code) + jump = stack_effect(code, jump=True) + nojump = stack_effect(code, jump=False) + else: + common = stack_effect(code, 0) + jump = stack_effect(code, 0, jump=True) + nojump = stack_effect(code, 0, jump=False) + if code in has_jump or code in has_exc: + self.assertEqual(common, max(jump, nojump)) + else: + self.assertEqual(jump, common) + self.assertEqual(nojump, common) + + +class SpecializationStatsTests(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'load_attr' not found in [] + def test_specialization_stats(self): + stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] + specialized_opcodes = [ + op.lower() + for op in opcode._specializations + if opcode._inline_cache_entries.get(op, 0) + ] + self.assertIn('load_attr', specialized_opcodes) + self.assertIn('binary_op', specialized_opcodes) + + stats = _opcode.get_specialization_stats() + if stats is not None: + self.assertIsInstance(stats, dict) + self.assertCountEqual(stats.keys(), specialized_opcodes) + self.assertCountEqual( + stats['load_attr'].keys(), + stat_names + ['failure_kinds']) + for sn in stat_names: + self.assertIsInstance(stats['load_attr'][sn], int) + self.assertIsInstance( + stats['load_attr']['failure_kinds'], + tuple) + for v in stats['load_attr']['failure_kinds']: + self.assertIsInstance(v, int) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index 4a14cb35213..0813c4804c1 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -20,12 +20,13 @@ def setUp(self): self.prog_name = 'bogus_program_xxxx' self.temp_path_dir = os.path.abspath(os.getcwd()) self.env = self.enterContext(os_helper.EnvironmentVarGuard()) - for cv in ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', - 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', - 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS'): - if cv in self.env: - self.env.unset(cv) + + self.env.unset( + 'CFLAGS', 'LDFLAGS', 'CPPFLAGS', + 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', + 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS' + ) def add_expected_saved_initial_values(self, config_vars, expected_vars): # Ensure that the initial values for all modified config vars @@ -65,8 +66,8 @@ def test__find_build_tool(self): 'cc not found - check xcode-select') def test__get_system_version(self): - self.assertTrue(platform.mac_ver()[0].startswith( - _osx_support._get_system_version())) + self.assertStartsWith(platform.mac_ver()[0], + _osx_support._get_system_version()) def test__remove_original_values(self): config_vars = { diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index d912954a41d..e29fc5a2394 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -20,7 +20,7 @@ def test_abstractproperty_basics(self): def foo(self): pass self.assertTrue(foo.__isabstractmethod__) def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) + self.assertNotHasAttr(bar, "__isabstractmethod__") class C(metaclass=abc_ABCMeta): @abc.abstractproperty @@ -89,7 +89,7 @@ def test_abstractmethod_basics(self): def foo(self): pass self.assertTrue(foo.__isabstractmethod__) def bar(self): pass - self.assertFalse(hasattr(bar, "__isabstractmethod__")) + self.assertNotHasAttr(bar, "__isabstractmethod__") def test_abstractproperty_basics(self): @property @@ -149,8 +149,6 @@ def foo(): return 4 self.assertEqual(D.foo(), 4) self.assertEqual(D().foo(), 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_object_new_with_one_abstractmethod(self): class C(metaclass=abc_ABCMeta): @abc.abstractmethod @@ -159,8 +157,6 @@ def method_one(self): msg = r"class C without an implementation for abstract method 'method_one'" self.assertRaisesRegex(TypeError, msg, C) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_object_new_with_many_abstractmethods(self): class C(metaclass=abc_ABCMeta): @abc.abstractmethod @@ -172,6 +168,7 @@ def method_two(self): msg = r"class C without an implementation for abstract methods 'method_one', 'method_two'" self.assertRaisesRegex(TypeError, msg, C) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_abstractmethod_integration(self): for abstractthing in [abc.abstractmethod, abc.abstractproperty, abc.abstractclassmethod, @@ -280,21 +277,21 @@ class A(metaclass=abc_ABCMeta): class B(object): pass b = B() - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) + self.assertNotIsSubclass(B, A) + self.assertNotIsSubclass(B, (A,)) self.assertNotIsInstance(b, A) self.assertNotIsInstance(b, (A,)) B1 = A.register(B) - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) self.assertIs(B1, B) class C(B): pass c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) @@ -305,16 +302,16 @@ class A(metaclass=abc_ABCMeta): class B(object): pass b = B() - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) @A.register class C(B): pass c = C() - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) self.assertIs(C, A.register(C)) @@ -325,14 +322,14 @@ class A(metaclass=abc_ABCMeta): class B: pass b = B() - self.assertFalse(isinstance(b, A)) - self.assertFalse(isinstance(b, (A,))) + self.assertNotIsInstance(b, A) + self.assertNotIsInstance(b, (A,)) token_old = abc_get_cache_token() A.register(B) token_new = abc_get_cache_token() self.assertGreater(token_new, token_old) - self.assertTrue(isinstance(b, A)) - self.assertTrue(isinstance(b, (A,))) + self.assertIsInstance(b, A) + self.assertIsInstance(b, (A,)) def test_registration_builtins(self): class A(metaclass=abc_ABCMeta): @@ -340,18 +337,18 @@ class A(metaclass=abc_ABCMeta): A.register(int) self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) - self.assertTrue(issubclass(int, A)) - self.assertTrue(issubclass(int, (A,))) + self.assertIsSubclass(int, A) + self.assertIsSubclass(int, (A,)) class B(A): pass B.register(str) class C(str): pass self.assertIsInstance("", A) self.assertIsInstance("", (A,)) - self.assertTrue(issubclass(str, A)) - self.assertTrue(issubclass(str, (A,))) - self.assertTrue(issubclass(C, A)) - self.assertTrue(issubclass(C, (A,))) + self.assertIsSubclass(str, A) + self.assertIsSubclass(str, (A,)) + self.assertIsSubclass(C, A) + self.assertIsSubclass(C, (A,)) def test_registration_edge_cases(self): class A(metaclass=abc_ABCMeta): @@ -379,39 +376,39 @@ class A(metaclass=abc_ABCMeta): def test_registration_transitiveness(self): class A(metaclass=abc_ABCMeta): pass - self.assertTrue(issubclass(A, A)) - self.assertTrue(issubclass(A, (A,))) + self.assertIsSubclass(A, A) + self.assertIsSubclass(A, (A,)) class B(metaclass=abc_ABCMeta): pass - self.assertFalse(issubclass(A, B)) - self.assertFalse(issubclass(A, (B,))) - self.assertFalse(issubclass(B, A)) - self.assertFalse(issubclass(B, (A,))) + self.assertNotIsSubclass(A, B) + self.assertNotIsSubclass(A, (B,)) + self.assertNotIsSubclass(B, A) + self.assertNotIsSubclass(B, (A,)) class C(metaclass=abc_ABCMeta): pass A.register(B) class B1(B): pass - self.assertTrue(issubclass(B1, A)) - self.assertTrue(issubclass(B1, (A,))) + self.assertIsSubclass(B1, A) + self.assertIsSubclass(B1, (A,)) class C1(C): pass B1.register(C1) - self.assertFalse(issubclass(C, B)) - self.assertFalse(issubclass(C, (B,))) - self.assertFalse(issubclass(C, B1)) - self.assertFalse(issubclass(C, (B1,))) - self.assertTrue(issubclass(C1, A)) - self.assertTrue(issubclass(C1, (A,))) - self.assertTrue(issubclass(C1, B)) - self.assertTrue(issubclass(C1, (B,))) - self.assertTrue(issubclass(C1, B1)) - self.assertTrue(issubclass(C1, (B1,))) + self.assertNotIsSubclass(C, B) + self.assertNotIsSubclass(C, (B,)) + self.assertNotIsSubclass(C, B1) + self.assertNotIsSubclass(C, (B1,)) + self.assertIsSubclass(C1, A) + self.assertIsSubclass(C1, (A,)) + self.assertIsSubclass(C1, B) + self.assertIsSubclass(C1, (B,)) + self.assertIsSubclass(C1, B1) + self.assertIsSubclass(C1, (B1,)) C1.register(int) class MyInt(int): pass - self.assertTrue(issubclass(MyInt, A)) - self.assertTrue(issubclass(MyInt, (A,))) + self.assertIsSubclass(MyInt, A) + self.assertIsSubclass(MyInt, (A,)) self.assertIsInstance(42, A) self.assertIsInstance(42, (A,)) @@ -471,16 +468,16 @@ def __subclasshook__(cls, C): if cls is A: return 'foo' in C.__dict__ return NotImplemented - self.assertFalse(issubclass(A, A)) - self.assertFalse(issubclass(A, (A,))) + self.assertNotIsSubclass(A, A) + self.assertNotIsSubclass(A, (A,)) class B: foo = 42 - self.assertTrue(issubclass(B, A)) - self.assertTrue(issubclass(B, (A,))) + self.assertIsSubclass(B, A) + self.assertIsSubclass(B, (A,)) class C: spam = 42 - self.assertFalse(issubclass(C, A)) - self.assertFalse(issubclass(C, (A,))) + self.assertNotIsSubclass(C, A) + self.assertNotIsSubclass(C, (A,)) def test_all_new_methods_are_called(self): class A(metaclass=abc_ABCMeta): @@ -497,7 +494,7 @@ class C(A, B): self.assertEqual(B.counter, 1) def test_ABC_has___slots__(self): - self.assertTrue(hasattr(abc.ABC, '__slots__')) + self.assertHasAttr(abc.ABC, '__slots__') def test_tricky_new_works(self): def with_metaclass(meta, *bases): @@ -519,15 +516,14 @@ def foo(self): del A.foo self.assertEqual(A.__abstractmethods__, {'foo'}) - self.assertFalse(hasattr(A, 'foo')) + self.assertNotHasAttr(A, 'foo') abc.update_abstractmethods(A) self.assertEqual(A.__abstractmethods__, set()) A() - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_update_new_abstractmethods(self): class A(metaclass=abc_ABCMeta): @abc.abstractmethod @@ -544,8 +540,6 @@ def updated_foo(self): msg = "class A without an implementation for abstract methods 'bar', 'foo'" self.assertRaisesRegex(TypeError, msg, A) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_update_implementation(self): class A(metaclass=abc_ABCMeta): @abc.abstractmethod @@ -595,10 +589,8 @@ def updated_foo(self): A.foo = updated_foo abc.update_abstractmethods(A) A() - self.assertFalse(hasattr(A, '__abstractmethods__')) + self.assertNotHasAttr(A, '__abstractmethods__') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_update_del_implementation(self): class A(metaclass=abc_ABCMeta): @abc.abstractmethod @@ -618,8 +610,6 @@ def foo(self): msg = "class B without an implementation for abstract method 'foo'" self.assertRaisesRegex(TypeError, msg, B) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_update_layered_implementation(self): class A(metaclass=abc_ABCMeta): @abc.abstractmethod @@ -695,10 +685,16 @@ class B(A, metaclass=abc_ABCMeta, name="test"): return TestLegacyAPI, TestABC, TestABCWithInitSubclass -TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(abc.ABCMeta, - abc.get_cache_token) -TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(_py_abc.ABCMeta, - _py_abc.get_cache_token) +TestLegacyAPI_Py, TestABC_Py, TestABCWithInitSubclass_Py = test_factory(_py_abc.ABCMeta, + _py_abc.get_cache_token) +TestLegacyAPI_C, TestABC_C, TestABCWithInitSubclass_C = test_factory(abc.ABCMeta, + abc.get_cache_token) + +# gh-130095: The _py_abc tests are not thread-safe when run with +# `--parallel-threads` +TestLegacyAPI_Py.__unittest_thread_unsafe__ = True +TestABC_Py.__unittest_thread_unsafe__ = True +TestABCWithInitSubclass_Py.__unittest_thread_unsafe__ = True if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 2e06f0d16fd..cf071d2c933 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -1,14 +1,34 @@ """Unit tests for numbers.py.""" +import abc import math import operator import unittest -from numbers import Complex, Real, Rational, Integral +from numbers import Complex, Real, Rational, Integral, Number + + +def concretize(cls): + def not_implemented(*args, **kwargs): + raise NotImplementedError() + + for name in dir(cls): + try: + value = getattr(cls, name) + if value.__isabstractmethod__: + setattr(cls, name, not_implemented) + except AttributeError: + pass + abc.update_abstractmethods(cls) + return cls + class TestNumbers(unittest.TestCase): def test_int(self): - self.assertTrue(issubclass(int, Integral)) - self.assertTrue(issubclass(int, Complex)) + self.assertIsSubclass(int, Integral) + self.assertIsSubclass(int, Rational) + self.assertIsSubclass(int, Real) + self.assertIsSubclass(int, Complex) + self.assertIsSubclass(int, Number) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -18,8 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): - self.assertFalse(issubclass(float, Rational)) - self.assertTrue(issubclass(float, Real)) + self.assertNotIsSubclass(float, Integral) + self.assertNotIsSubclass(float, Rational) + self.assertIsSubclass(float, Real) + self.assertIsSubclass(float, Complex) + self.assertIsSubclass(float, Number) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -27,8 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): - self.assertFalse(issubclass(complex, Real)) - self.assertTrue(issubclass(complex, Complex)) + self.assertNotIsSubclass(complex, Integral) + self.assertNotIsSubclass(complex, Rational) + self.assertNotIsSubclass(complex, Real) + self.assertIsSubclass(complex, Complex) + self.assertIsSubclass(complex, Number) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). @@ -40,5 +66,135 @@ def test_complex(self): self.assertRaises(TypeError, int, c1) +class TestNumbersDefaultMethods(unittest.TestCase): + def test_complex(self): + @concretize + class MyComplex(Complex): + def __init__(self, real, imag): + self.r = real + self.i = imag + + @property + def real(self): + return self.r + + @property + def imag(self): + return self.i + + def __add__(self, other): + if isinstance(other, Complex): + return MyComplex(self.imag + other.imag, + self.real + other.real) + raise NotImplementedError + + def __neg__(self): + return MyComplex(-self.real, -self.imag) + + def __eq__(self, other): + if isinstance(other, Complex): + return self.imag == other.imag and self.real == other.real + if isinstance(other, Number): + return self.imag == 0 and self.real == other.real + + # test __bool__ + self.assertTrue(bool(MyComplex(1, 1))) + self.assertTrue(bool(MyComplex(0, 1))) + self.assertTrue(bool(MyComplex(1, 0))) + self.assertFalse(bool(MyComplex(0, 0))) + + # test __sub__ + self.assertEqual(MyComplex(2, 3) - complex(1, 2), MyComplex(1, 1)) + + # test __rsub__ + self.assertEqual(complex(2, 3) - MyComplex(1, 2), MyComplex(1, 1)) + + def test_real(self): + @concretize + class MyReal(Real): + def __init__(self, n): + self.n = n + + def __pos__(self): + return self.n + + def __float__(self): + return float(self.n) + + def __floordiv__(self, other): + return self.n // other + + def __rfloordiv__(self, other): + return other // self.n + + def __mod__(self, other): + return self.n % other + + def __rmod__(self, other): + return other % self.n + + # test __divmod__ + self.assertEqual(divmod(MyReal(3), 2), (1, 1)) + + # test __rdivmod__ + self.assertEqual(divmod(3, MyReal(2)), (1, 1)) + + # test __complex__ + self.assertEqual(complex(MyReal(1)), 1+0j) + + # test real + self.assertEqual(MyReal(3).real, 3) + + # test imag + self.assertEqual(MyReal(3).imag, 0) + + # test conjugate + self.assertEqual(MyReal(123).conjugate(), 123) + + + def test_rational(self): + @concretize + class MyRational(Rational): + def __init__(self, numerator, denominator): + self.n = numerator + self.d = denominator + + @property + def numerator(self): + return self.n + + @property + def denominator(self): + return self.d + + # test__float__ + self.assertEqual(float(MyRational(5, 2)), 2.5) + + + def test_integral(self): + @concretize + class MyIntegral(Integral): + def __init__(self, n): + self.n = n + + def __pos__(self): + return self.n + + def __int__(self): + return self.n + + # test __index__ + self.assertEqual(operator.index(MyIntegral(123)), 123) + + # test __float__ + self.assertEqual(float(MyIntegral(123)), 123.0) + + # test numerator + self.assertEqual(MyIntegral(123).numerator, 123) + + # test denominator + self.assertEqual(MyIntegral(123).denominator, 1) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 00000000000..076190f7572 --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,448 @@ +import io +import platform +import queue +import re +import subprocess +import sys +import unittest +from _android_support import TextLogStream +from array import array +from contextlib import ExitStack, contextmanager +from threading import Thread +from test.support import LOOPBACK_TIMEOUT +from time import time +from unittest.mock import patch + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + +# (name, level, fileno) +STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] + + +# Test redirection of stdout and stderr to the Android log. +@unittest.skipIf( + api_level < 23 and platform.machine() == "aarch64", + "SELinux blocks reading logs on older ARM64 emulators" +) +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + self.logcat_thread = Thread(target=logcat_thread) + self.logcat_thread.start() + + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(LOOPBACK_TIMEOUT) + self.logcat_thread.join(LOOPBACK_TIMEOUT) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. + def stream_context(self, stream_name, level): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] + + stack = ExitStack() + stack.enter_context(self.subTest(stream_name)) + stream = getattr(sys, stream_name) + native_stream = getattr(sys, f"__{stream_name}__") + if isinstance(stream, io.StringIO): + stack.enter_context( + patch( + f"sys.{stream_name}", + TextLogStream( + prio, f"python.{stream_name}", native_stream.fileno(), + errors="backslashreplace" + ), + ) + ) + return stack + + def test_str(self): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + self.assertEqual("UTF-8", stream.encoding) + self.assertEqual("backslashreplace", stream.errors) + self.assertIs(stream.line_buffering, True) + self.assertIs(stream.write_through, False) + + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600") + + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + # Since this is a line-based logging system, line buffering + # cannot be turned off, i.e. a newline always causes a flush. + stream.reconfigure(line_buffering=False) + self.assertIs(stream.line_buffering, True) + + # However, buffering can be turned off completely if you want a + # flush after every write. + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", # 4200 bytes in + [("foobar" * 666) + "foob", # 4000 bytes out + "ar" + ("foobar" * 33)]) # 200 bytes out + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", # 4500 bytes in + [s * 100, # 3000 bytes out + s * 50]) # 1500 bytes out + + s = "0123456789" + write(s * 200, []) # 2000 bytes in + write(s * 150, []) # 1500 bytes in + write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out + write("\n", [s * 51]) # 0 bytes in, 510 bytes out + + def test_bytes(self): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80") + + # Null bytes are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + +class TestAndroidRateLimit(unittest.TestCase): + def test_rate_limit(self): + # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 + PER_MESSAGE_OVERHEAD = 28 + + # https://developer.android.com/ndk/reference/group/logging + ANDROID_LOG_DEBUG = 3 + + # To avoid flooding the test script output, use a different tag rather + # than stdout or stderr. + tag = "python.rate_limit" + stream = TextLogStream(ANDROID_LOG_DEBUG, tag) + + # Make a test message which consumes 1 KB of the logcat buffer. + message = "Line {:03d} " + message += "." * ( + 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) + ) + "\n" + + # To avoid depending on the performance of the test device, we mock the + # passage of time. + mock_now = time() + + def mock_time(): + # Avoid division by zero by simulating a small delay. + mock_sleep(0.0001) + return mock_now + + def mock_sleep(duration): + nonlocal mock_now + mock_now += duration + + # See _android_support.py. The default values of these parameters work + # well across a wide range of devices, but we'll use smaller values to + # ensure a quick and reliable test that doesn't flood the log too much. + MAX_KB_PER_SECOND = 100 + BUCKET_KB = 10 + with ( + patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), + patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024), + patch("_android_support.sleep", mock_sleep), + patch("_android_support.time", mock_time), + ): + # Make sure the token bucket is full. + stream.write("Initial message to reset _prev_write_time") + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + line_num = 0 + + # Write BUCKET_KB messages, and return the rate at which they were + # accepted in KB per second. + def write_bucketful(): + nonlocal line_num + start = mock_time() + max_line_num = line_num + BUCKET_KB + while line_num < max_line_num: + stream.write(message.format(line_num)) + line_num += 1 + return BUCKET_KB / (mock_time() - start) + + # The first bucketful should be written with minimal delay. The + # factor of 2 here is not arbitrary: it verifies that the system can + # write fast enough to empty the bucket within two bucketfuls, which + # the next part of the test depends on. + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) + + # Write another bucketful to empty the token bucket completely. + write_bucketful() + + # The next bucketful should be written at the rate limit. + self.assertAlmostEqual( + write_bucketful(), MAX_KB_PER_SECOND, + delta=MAX_KB_PER_SECOND * 0.1 + ) + + # Once the token bucket refills, we should go back to full speed. + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py new file mode 100644 index 00000000000..e89d6c0b161 --- /dev/null +++ b/Lib/test/test_annotationlib.py @@ -0,0 +1,2252 @@ +"""Tests for the annotations module.""" + +import textwrap +import annotationlib +import builtins +import collections +import functools +import itertools +import pickle +from string.templatelib import Template, Interpolation +import types +import typing +import sys +import unittest +from annotationlib import ( + Format, + ForwardRef, + get_annotations, + annotations_to_string, + type_repr, +) +from typing import Unpack, get_type_hints, List, Union + +from test import support +from test.support import import_helper +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 + + +def times_three(fn): + @functools.wraps(fn) + def wrapper(a, b): + return fn(a * 3, b * 3) + + return wrapper + + +class MyClass: + def __repr__(self): + return "my repr" + + +class TestFormat(unittest.TestCase): + def test_enum(self): + self.assertEqual(Format.VALUE.value, 1) + self.assertEqual(Format.VALUE, 1) + + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS.value, 2) + self.assertEqual(Format.VALUE_WITH_FAKE_GLOBALS, 2) + + self.assertEqual(Format.FORWARDREF.value, 3) + self.assertEqual(Format.FORWARDREF, 3) + + self.assertEqual(Format.STRING.value, 4) + self.assertEqual(Format.STRING, 4) + + +class TestForwardRefFormat(unittest.TestCase): + def test_closure(self): + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.FORWARDREF) + fwdref = anno["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = 1 + self.assertEqual(fwdref.evaluate(), x) + + anno = get_annotations(inner, format=Format.FORWARDREF) + self.assertEqual(anno["arg"], x) + + def test_multiple_closure(self): + def inner(arg: x[y]): + pass + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x[y]") + with self.assertRaises(NameError): + fwdref.evaluate() + + y = str + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + extra_name, extra_val = next(iter(fwdref.__extra_names__.items())) + self.assertEqual(fwdref.__forward_arg__.replace(extra_name, extra_val.__name__), "x[str]") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = list + self.assertEqual(fwdref.evaluate(), x[y]) + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertEqual(fwdref, x[y]) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = get_annotations(f, format=Format.FORWARDREF) + self.assertIs(anno["x"], int) + fwdref = anno["y"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "doesntexist") + with self.assertRaises(NameError): + fwdref.evaluate() + self.assertEqual(fwdref.evaluate(globals={"doesntexist": 1}), 1) + + def test_nonexistent_attribute(self): + def f( + x: some.module, + y: some[module], + z: some(module), + alpha: some | obj, + beta: +some, + gamma: some < obj, + delta: some | {obj: module}, + epsilon: some | {obj}, + zeta: some | [obj, module], + eta: some | (), + ): + pass + + anno = get_annotations(f, format=Format.FORWARDREF) + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno, support.EqualToForwardRef("some.module", owner=f)) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno, support.EqualToForwardRef("some[module]", owner=f)) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno, support.EqualToForwardRef("some(module)", owner=f)) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno, support.EqualToForwardRef("some | obj", owner=f)) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno, support.EqualToForwardRef("+some", owner=f)) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + + delta_anno = anno["delta"] + self.assertIsInstance(delta_anno, ForwardRef) + self.assertEqual(delta_anno, support.EqualToForwardRef("some | {obj: module}", owner=f)) + + epsilon_anno = anno["epsilon"] + self.assertIsInstance(epsilon_anno, ForwardRef) + self.assertEqual(epsilon_anno, support.EqualToForwardRef("some | {obj}", owner=f)) + + zeta_anno = anno["zeta"] + self.assertIsInstance(zeta_anno, ForwardRef) + self.assertEqual(zeta_anno, support.EqualToForwardRef("some | [obj, module]", owner=f)) + + eta_anno = anno["eta"] + self.assertIsInstance(eta_anno, ForwardRef) + self.assertEqual(eta_anno, support.EqualToForwardRef("some | ()", owner=f)) + + def test_partially_nonexistent(self): + # These annotations start with a non-existent variable and then use + # global types with defined values. This partially evaluates by putting + # those globals into `fwdref.__extra_names__`. + def f( + x: obj | int, + y: container[int:obj, int], + z: dict_val | {str: int}, + alpha: set_val | {str, int}, + beta: obj | bool | int, + gamma: obj | call_func(int, kwd=bool), + ): + pass + + def func(*args, **kwargs): + return Union[*args, *(kwargs.values())] + + anno = get_annotations(f, format=Format.FORWARDREF) + globals_ = { + "obj": str, "container": list, "dict_val": {1: 2}, "set_val": {1, 2}, + "call_func": func + } + + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno.evaluate(globals=globals_), str | int) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno.evaluate(globals=globals_), list[int:str, int]) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno.evaluate(globals=globals_), {1: 2} | {str: int}) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno.evaluate(globals=globals_), {1, 2} | {str, int}) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno.evaluate(globals=globals_), str | bool | int) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno.evaluate(globals=globals_), str | func(int, kwd=bool)) + + def test_partially_nonexistent_union(self): + # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs + class UnionForwardrefs: + pipe: str | undefined + union: Union[str, undefined] + + annos = get_annotations(UnionForwardrefs, format=Format.FORWARDREF) + + pipe = annos["pipe"] + self.assertIsInstance(pipe, ForwardRef) + self.assertEqual( + pipe.evaluate(globals={"undefined": int}), + str | int, + ) + union = annos["union"] + self.assertIsInstance(union, Union) + arg1, arg2 = typing.get_args(union) + self.assertIs(arg1, str) + self.assertEqual( + arg2, support.EqualToForwardRef("undefined", is_class=True, owner=UnionForwardrefs) + ) + + +class TestStringFormat(unittest.TestCase): + def test_closure(self): + x = 0 + + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_closure_undefined(self): + if False: + x = 0 + + def inner(arg: x): + pass + + anno = get_annotations(inner, format=Format.STRING) + self.assertEqual(anno, {"arg": "x"}) + + def test_function(self): + def f(x: int, y: doesntexist): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "int", "y": "doesntexist"}) + + def test_expressions(self): + def f( + add: a + b, + sub: a - b, + mul: a * b, + matmul: a @ b, + truediv: a / b, + mod: a % b, + lshift: a << b, + rshift: a >> b, + or_: a | b, + xor: a ^ b, + and_: a & b, + floordiv: a // b, + pow_: a**b, + lt: a < b, + le: a <= b, + eq: a == b, + ne: a != b, + gt: a > b, + ge: a >= b, + invert: ~a, + neg: -a, + pos: +a, + getitem: a[b], + getattr: a.b, + call: a(b, *c, d=e), # **kwargs are not supported + *args: *a, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "add": "a + b", + "sub": "a - b", + "mul": "a * b", + "matmul": "a @ b", + "truediv": "a / b", + "mod": "a % b", + "lshift": "a << b", + "rshift": "a >> b", + "or_": "a | b", + "xor": "a ^ b", + "and_": "a & b", + "floordiv": "a // b", + "pow_": "a ** b", + "lt": "a < b", + "le": "a <= b", + "eq": "a == b", + "ne": "a != b", + "gt": "a > b", + "ge": "a >= b", + "invert": "~a", + "neg": "-a", + "pos": "+a", + "getitem": "a[b]", + "getattr": "a.b", + "call": "a(b, *c, d=e)", + "args": "*a", + }, + ) + + def test_reverse_ops(self): + def f( + radd: 1 + a, + rsub: 1 - a, + rmul: 1 * a, + rmatmul: 1 @ a, + rtruediv: 1 / a, + rmod: 1 % a, + rlshift: 1 << a, + rrshift: 1 >> a, + ror: 1 | a, + rxor: 1 ^ a, + rand: 1 & a, + rfloordiv: 1 // a, + rpow: 1**a, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "radd": "1 + a", + "rsub": "1 - a", + "rmul": "1 * a", + "rmatmul": "1 @ a", + "rtruediv": "1 / a", + "rmod": "1 % a", + "rlshift": "1 << a", + "rrshift": "1 >> a", + "ror": "1 | a", + "rxor": "1 ^ a", + "rand": "1 & a", + "rfloordiv": "1 // a", + "rpow": "1 ** a", + }, + ) + + def test_template_str(self): + def f( + x: t"{a}", + y: list[t"{a}"], + z: t"{a:b} {c!r} {d!s:t}", + a: t"a{b}c{d}e{f}g", + b: t"{a:{1}}", + c: t"{a | b * c}", + gh138558: t"{ 0}", + ): pass + + annos = get_annotations(f, format=Format.STRING) + self.assertEqual(annos, { + "x": "t'{a}'", + "y": "list[t'{a}']", + "z": "t'{a:b} {c!r} {d!s:t}'", + "a": "t'a{b}c{d}e{f}g'", + # interpolations in the format spec are eagerly evaluated so we can't recover the source + "b": "t'{a:1}'", + "c": "t'{a | b * c}'", + "gh138558": "t'{ 0}'", + }) + + def g( + x: t"{a}", + ): ... + + annos = get_annotations(g, format=Format.FORWARDREF) + templ = annos["x"] + # Template and Interpolation don't have __eq__ so we have to compare manually + self.assertIsInstance(templ, Template) + self.assertEqual(templ.strings, ("", "")) + self.assertEqual(len(templ.interpolations), 1) + interp = templ.interpolations[0] + self.assertEqual(interp.value, support.EqualToForwardRef("a", owner=g)) + self.assertEqual(interp.expression, "a") + self.assertIsNone(interp.conversion) + self.assertEqual(interp.format_spec, "") + + def test_getitem(self): + def f(x: undef1[str, undef2]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "undef1[str, undef2]"}) + + anno = get_annotations(f, format=Format.FORWARDREF) + fwdref = anno["x"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual( + fwdref.evaluate(globals={"undef1": dict, "undef2": float}), dict[str, float] + ) + + def test_slice(self): + def f(x: a[b:c]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c]"}) + + def f(x: a[b:c, d:e]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[b:c, d:e]"}) + + obj = slice(1, 1, 1) + def f(x: obj): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "obj"}) + + def test_literals(self): + def f( + a: 1, + b: 1.0, + c: "hello", + d: b"hello", + e: True, + f: None, + g: ..., + h: 1j, + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "a": "1", + "b": "1.0", + "c": 'hello', + "d": "b'hello'", + "e": "True", + "f": "None", + "g": "...", + "h": "1j", + }, + ) + + def test_displays(self): + # Simple case first + def f(x: a[[int, str], float]): + pass + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "a[[int, str], float]"}) + + def g( + w: a[[int, str], float], + x: a[{int}, 3], + y: a[{int: str}, 4], + z: a[(int, str), 5], + ): + pass + anno = get_annotations(g, format=Format.STRING) + self.assertEqual( + anno, + { + "w": "a[[int, str], float]", + "x": "a[{int}, 3]", + "y": "a[{int: str}, 4]", + "z": "a[(int, str), 5]", + }, + ) + + def test_nested_expressions(self): + def f( + nested: list[Annotated[set[int], "set of ints", 4j]], + set: {a + b}, # single element because order is not guaranteed + dict: {a + b: c + d, "key": e + g}, + list: [a, b, c], + tuple: (a, b, c), + slice: (a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d]), + extended_slice: a[:, :, c:d], + unpack1: [*a], + unpack2: [*a, b, c], + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "nested": "list[Annotated[set[int], 'set of ints', 4j]]", + "set": "{a + b}", + "dict": "{a + b: c + d, 'key': e + g}", + "list": "[a, b, c]", + "tuple": "(a, b, c)", + "slice": "(a[b:c], a[b:c:d], a[:c], a[b:], a[:], a[::d], a[b::d])", + "extended_slice": "a[:, :, c:d]", + "unpack1": "[*a]", + "unpack2": "[*a, b, c]", + }, + ) + + def test_unsupported_operations(self): + format_msg = "Cannot stringify annotation containing string formatting" + + def f(fstring: f"{a}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + get_annotations(f, format=Format.STRING) + + def f(fstring_format: f"{a:02d}"): + pass + + with self.assertRaisesRegex(TypeError, format_msg): + get_annotations(f, format=Format.STRING) + + def test_shenanigans(self): + # In cases like this we can't reconstruct the source; test that we do something + # halfway reasonable. + def f(x: x | (1).__class__, y: (1).__class__): + pass + + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"x": "x | ", "y": ""}, + ) + + +class TestGetAnnotations(unittest.TestCase): + def test_builtin_type(self): + self.assertEqual(get_annotations(int), {}) + self.assertEqual(get_annotations(object), {}) + + def test_custom_metaclass(self): + class Meta(type): + pass + + class C(metaclass=Meta): + x: int + + self.assertEqual(get_annotations(C), {"x": int}) + + def test_missing_dunder_dict(self): + class NoDict(type): + @property + def __dict__(cls): + raise AttributeError + + b: str + + class C1(metaclass=NoDict): + a: int + + self.assertEqual(get_annotations(C1), {"a": int}) + self.assertEqual( + get_annotations(C1, format=Format.FORWARDREF), + {"a": int}, + ) + self.assertEqual( + get_annotations(C1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(get_annotations(NoDict), {"b": str}) + self.assertEqual( + get_annotations(NoDict, format=Format.FORWARDREF), + {"b": str}, + ) + self.assertEqual( + get_annotations(NoDict, format=Format.STRING), + {"b": "str"}, + ) + + def test_format(self): + def f1(a: int): + pass + + def f2(a: undefined): + pass + + self.assertEqual( + get_annotations(f1, format=Format.VALUE), + {"a": int}, + ) + self.assertEqual(get_annotations(f1, format=1), {"a": int}) + + fwd = support.EqualToForwardRef("undefined", owner=f2) + self.assertEqual( + get_annotations(f2, format=Format.FORWARDREF), + {"a": fwd}, + ) + self.assertEqual(get_annotations(f2, format=3), {"a": fwd}) + + self.assertEqual( + get_annotations(f1, format=Format.STRING), + {"a": "int"}, + ) + self.assertEqual(get_annotations(f1, format=4), {"a": "int"}) + + with self.assertRaises(ValueError): + get_annotations(f1, format=42) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + get_annotations(f1, format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaisesRegex( + ValueError, + r"The VALUE_WITH_FAKE_GLOBALS format is for internal use only", + ): + get_annotations(f1, format=2) + + def test_custom_object_with_annotations(self): + class C: + def __init__(self): + self.__annotations__ = {"x": int, "y": str} + + self.assertEqual(get_annotations(C()), {"x": int, "y": str}) + + def test_custom_format_eval_str(self): + def foo(): + pass + + with self.assertRaises(ValueError): + get_annotations(foo, format=Format.FORWARDREF, eval_str=True) + get_annotations(foo, format=Format.STRING, eval_str=True) + + def test_stock_annotations(self): + def foo(a: int, b: str): + pass + + for format in (Format.VALUE, Format.FORWARDREF): + with self.subTest(format=format): + self.assertEqual( + get_annotations(foo, format=format), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(foo, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + + foo.__annotations__ = {"a": "foo", "b": "str"} + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with self.subTest(format=format): + self.assertEqual( + get_annotations(foo, format=format), + {"a": "foo", "b": "str"}, + ) + + self.assertEqual( + get_annotations(foo, eval_str=True, locals=locals()), + {"a": foo, "b": str}, + ) + self.assertEqual( + get_annotations(foo, eval_str=True, globals=locals()), + {"a": foo, "b": str}, + ) + + def test_stock_annotations_in_module(self): + isa = inspect_stock_annotations + + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + get_annotations(annotationlib, **kwargs), {} + ) # annotations module has no annotations + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": str, "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": int, "b": str, "c": isa.MyClass}, + ) + self.assertEqual(get_annotations(annotationlib, **kwargs), {}) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + self.assertEqual( + get_annotations(isa, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.MyClass, format=Format.STRING), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.function, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function2, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function3, format=Format.STRING), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual( + get_annotations(annotationlib, format=Format.STRING), + {}, + ) + self.assertEqual( + get_annotations(isa.UnannotatedClass, format=Format.STRING), + {}, + ) + self.assertEqual( + get_annotations(isa.unannotated_function, format=Format.STRING), + {}, + ) + + def test_stock_annotations_on_wrapper(self): + isa = inspect_stock_annotations + + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, format=Format.FORWARDREF), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, format=Format.STRING), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"a": int, "b": str, "return": isa.MyClass}, + ) + + def test_stringized_annotations_in_module(self): + isa = inspect_stringized_annotations + for kwargs in [ + {}, + {"eval_str": False}, + {"format": Format.VALUE}, + {"format": Format.FORWARDREF}, + {"format": Format.STRING}, + {"format": Format.VALUE, "eval_str": False}, + {"format": Format.FORWARDREF, "eval_str": False}, + {"format": Format.STRING, "eval_str": False}, + ]: + with self.subTest(**kwargs): + self.assertEqual( + get_annotations(isa, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": "int", "b": "str"}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": "int", "b": "'str'", "c": "MyClass", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "'int'", "b": "'str'", "c": "'MyClass'"}, + ) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + for kwargs in [ + {"eval_str": True}, + {"eval_str": True, "globals": isa.__dict__, "locals": {}}, + {"eval_str": True, "globals": {}, "locals": isa.__dict__}, + {"format": Format.VALUE, "eval_str": True}, + ]: + with self.subTest(**kwargs): + self.assertEqual(get_annotations(isa, **kwargs), {"a": int, "b": str}) + self.assertEqual( + get_annotations(isa.MyClass, **kwargs), + {"a": int, "b": str}, + ) + self.assertEqual( + get_annotations(isa.function, **kwargs), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function2, **kwargs), + {"a": int, "b": "str", "c": isa.MyClass, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(isa.function3, **kwargs), + {"a": "int", "b": "str", "c": "MyClass"}, + ) + self.assertEqual(get_annotations(isa.UnannotatedClass, **kwargs), {}) + self.assertEqual( + get_annotations(isa.unannotated_function, **kwargs), + {}, + ) + + def test_stringized_annotations_in_empty_module(self): + isa2 = inspect_stringized_annotations_2 + self.assertEqual(get_annotations(isa2), {}) + self.assertEqual(get_annotations(isa2, eval_str=True), {}) + self.assertEqual(get_annotations(isa2, eval_str=False), {}) + + def test_stringized_annotations_with_star_unpack(self): + def f(*args: "*tuple[int, ...]"): ... + self.assertEqual(get_annotations(f, eval_str=True), + {'args': (*tuple[int, ...],)[0]}) + def f(*args: " *tuple[int, ...]"): ... + self.assertEqual(get_annotations(f, eval_str=True), + {'args': (*tuple[int, ...],)[0]}) + + + def test_stringized_annotations_on_wrapper(self): + isa = inspect_stringized_annotations + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, "x"), isa.MyClass(3, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"a": int, "b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"a": "int", "b": "str", "return": "MyClass"}, + ) + + def test_stringized_annotations_on_partial_wrapper(self): + isa = inspect_stringized_annotations + + def times_three_str(fn: typing.Callable[[str], isa.MyClass]): + @functools.wraps(fn) + def wrapper(b: "str") -> "MyClass": + return fn(b * 3) + + return wrapper + + wrapped = times_three_str(functools.partial(isa.function, 1)) + self.assertEqual(wrapped("x"), isa.MyClass(1, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + # If functools is not loaded, names will be evaluated in the current + # module instead of being unwrapped to the original. + functools_mod = sys.modules["functools"] + del sys.modules["functools"] + + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + sys.modules["functools"] = functools_mod + + def test_stringized_annotations_on_class(self): + isa = inspect_stringized_annotations + # test that local namespace lookups work + self.assertEqual( + get_annotations(isa.MyClassWithLocalAnnotations), + {"x": "mytype"}, + ) + self.assertEqual( + get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), + {"x": int}, + ) + + def test_stringized_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": "int"} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha), {"x": "int"}) + self.assertEqual(get_annotations(ha, eval_str=True), {"x": int}) + + def test_stringized_annotation_permutations(self): + def define_class(name, has_future, has_annos, base_text, extra_names=None): + lines = [] + if has_future: + lines.append("from __future__ import annotations") + lines.append(f"class {name}({base_text}):") + if has_annos: + lines.append(f" {name}_attr: int") + else: + lines.append(" pass") + code = "\n".join(lines) + ns = support.run_code(code, extra_names=extra_names) + return ns[name] + + def check_annotations(cls, has_future, has_annos): + if has_annos: + if has_future: + anno = "int" + else: + anno = int + self.assertEqual(get_annotations(cls), {f"{cls.__name__}_attr": anno}) + else: + self.assertEqual(get_annotations(cls), {}) + + for meta_future, base_future, child_future, meta_has_annos, base_has_annos, child_has_annos in itertools.product( + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + ): + with self.subTest( + meta_future=meta_future, + base_future=base_future, + child_future=child_future, + meta_has_annos=meta_has_annos, + base_has_annos=base_has_annos, + child_has_annos=child_has_annos, + ): + meta = define_class( + "Meta", + has_future=meta_future, + has_annos=meta_has_annos, + base_text="type", + ) + base = define_class( + "Base", + has_future=base_future, + has_annos=base_has_annos, + base_text="metaclass=Meta", + extra_names={"Meta": meta}, + ) + child = define_class( + "Child", + has_future=child_future, + has_annos=child_has_annos, + base_text="Base", + extra_names={"Base": base}, + ) + check_annotations(meta, meta_future, meta_has_annos) + check_annotations(base, base_future, base_has_annos) + check_annotations(child, child_future, child_has_annos) + + def test_modify_annotations(self): + def f(x: int): + pass + + self.assertEqual(get_annotations(f), {"x": int}) + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + {"x": int}, + ) + + f.__annotations__["x"] = str + # The modification is reflected in VALUE (the default) + self.assertEqual(get_annotations(f), {"x": str}) + # ... and also in FORWARDREF, which tries __annotations__ if available + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + {"x": str}, + ) + # ... but not in STRING which always uses __annotate__ + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotations(self): + class WeirdAnnotations: + @property + def __annotations__(self): + return "not a dict" + + wa = WeirdAnnotations() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotations__ is neither a dict nor None" + ), + ): + get_annotations(wa, format=format) + + def test_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": int} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(ha, format=Format.FORWARDREF), {"x": int}) + + self.assertEqual(get_annotations(ha, format=Format.STRING), {"x": "int"}) + + def test_raising_annotations_on_custom_object(self): + class HasRaisingAnnotations: + @property + def __annotations__(self): + return {"x": undefined} + + hra = HasRaisingAnnotations() + + with self.assertRaises(NameError): + get_annotations(hra, format=Format.VALUE) + + with self.assertRaises(NameError): + get_annotations(hra, format=Format.FORWARDREF) + + undefined = float + self.assertEqual(get_annotations(hra, format=Format.VALUE), {"x": float}) + + def test_forwardref_prefers_annotations(self): + class HasBoth: + @property + def __annotations__(self): + return {"x": int} + + @property + def __annotate__(self): + return lambda format: {"x": str} + + hb = HasBoth() + self.assertEqual(get_annotations(hb, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(hb, format=Format.FORWARDREF), {"x": int}) + self.assertEqual(get_annotations(hb, format=Format.STRING), {"x": str}) + + def test_only_annotate(self): + def f(x: int): + pass + + class OnlyAnnotate: + @property + def __annotate__(self): + return f.__annotate__ + + oa = OnlyAnnotate() + self.assertEqual(get_annotations(oa, format=Format.VALUE), {"x": int}) + self.assertEqual(get_annotations(oa, format=Format.FORWARDREF), {"x": int}) + self.assertEqual( + get_annotations(oa, format=Format.STRING), + {"x": "int"}, + ) + + def test_non_dict_annotate(self): + class WeirdAnnotate: + def __annotate__(self, *args, **kwargs): + return "not a dict" + + wa = WeirdAnnotate() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotate__ returned a non-dict" + ), + ): + get_annotations(wa, format=format) + + def test_no_annotations(self): + class CustomClass: + pass + + class MyCallable: + def __call__(self): + pass + + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + for obj in (None, 1, object(), CustomClass()): + with self.subTest(format=format, obj=obj): + with self.assertRaises(TypeError): + get_annotations(obj, format=format) + + # Callables and types with no annotations return an empty dict + for obj in (int, len, MyCallable()): + with self.subTest(format=format, obj=obj): + self.assertEqual(get_annotations(obj, format=format), {}) + + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars( + self, + ): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None}, + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars( + self, + ): + self.assertEqual( + set( + get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True, + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ), + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars( + self, + ): + self.assertEqual( + get_annotations(inspect_stringized_annotations_pep695.E, eval_str=True), + {"x": str}, + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.F_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__), + ) + self.assertNotEqual( + set(results.F_meth_annotations.values()), set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()).intersection( + results.F.__type_params__ + ), + set(), + ) + + self.assertEqual(results.G_annotations, {"x": str}) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__), + ) + + def test_partial_evaluation(self): + def f( + x: builtins.undef, + y: list[int], + z: 1 + int, + a: builtins.int, + b: [builtins.undef, builtins.int], + ): + pass + + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + { + "x": support.EqualToForwardRef("builtins.undef", owner=f), + "y": list[int], + "z": support.EqualToForwardRef("1 + int", owner=f), + "a": int, + "b": [ + support.EqualToForwardRef("builtins.undef", owner=f), + # We can't resolve this because we have to evaluate the whole annotation + support.EqualToForwardRef("builtins.int", owner=f), + ], + }, + ) + + self.assertEqual( + get_annotations(f, format=Format.STRING), + { + "x": "builtins.undef", + "y": "list[int]", + "z": "1 + int", + "a": "builtins.int", + "b": "[builtins.undef, builtins.int]", + }, + ) + + def test_partial_evaluation_error(self): + def f(x: range[1]): + pass + with self.assertRaisesRegex( + TypeError, "type 'range' is not subscriptable" + ): + f.__annotations__ + + self.assertEqual( + get_annotations(f, format=Format.FORWARDREF), + { + "x": support.EqualToForwardRef("range[1]", owner=f), + }, + ) + + def test_partial_evaluation_cell(self): + obj = object() + + class RaisesAttributeError: + attriberr: obj.missing + + anno = get_annotations(RaisesAttributeError, format=Format.FORWARDREF) + self.assertEqual( + anno, + { + "attriberr": support.EqualToForwardRef( + "obj.missing", is_class=True, owner=RaisesAttributeError + ) + }, + ) + + def test_nonlocal_in_annotation_scope(self): + class Demo: + nonlocal sequence_b + x: sequence_b + y: sequence_b[int] + + fwdrefs = get_annotations(Demo, format=Format.FORWARDREF) + + self.assertIsInstance(fwdrefs["x"], ForwardRef) + self.assertIsInstance(fwdrefs["y"], ForwardRef) + + sequence_b = list + self.assertIs(fwdrefs["x"].evaluate(), list) + self.assertEqual(fwdrefs["y"].evaluate(), list[int]) + + def test_raises_error_from_value(self): + # test that if VALUE is the only supported format, but raises an error + # that error is propagated from get_annotations + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + def f(): ... + + f.__annotate__ = annotate + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + get_annotations(f, format=fmt) + + +class TestCallEvaluateFunction(unittest.TestCase): + def test_evaluation(self): + def evaluate(format, exc=NotImplementedError): + if format > 2: + raise exc + return undefined + + with self.assertRaises(NameError): + annotationlib.call_evaluate_function(evaluate, Format.VALUE) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.FORWARDREF), + support.EqualToForwardRef("undefined"), + ) + self.assertEqual( + annotationlib.call_evaluate_function(evaluate, Format.STRING), + "undefined", + ) + + def test_fake_global_evaluation(self): + # This will raise an AttributeError + def evaluate_union(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + # Return a ForwardRef + return builtins.undefined | list[int] + raise exc + + self.assertEqual( + annotationlib.call_evaluate_function(evaluate_union, Format.FORWARDREF), + support.EqualToForwardRef("builtins.undefined | list[int]"), + ) + + # This will raise an AttributeError + def evaluate_intermediate(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + intermediate = builtins.undefined + # Return a literal + return intermediate is None + raise exc + + self.assertIs( + annotationlib.call_evaluate_function(evaluate_intermediate, Format.FORWARDREF), + False, + ) + + +class TestCallAnnotateFunction(unittest.TestCase): + # Tests for user defined annotate functions. + + # Format and NotImplementedError are provided as arguments so they exist in + # the fake globals namespace. + # This avoids non-matching conditions passing by being converted to stringifiers. + # See: https://github.com/python/cpython/issues/138764 + + def test_user_annotate_value(self): + def annotate(format, /): + if format == Format.VALUE: + return {"x": str} + else: + raise NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_forwardref_supported(self): + # If Format.FORWARDREF is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.FORWARDREF: + return {'x': float} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": float}) + + def test_user_annotate_forwardref_fakeglobals(self): + # If Format.FORWARDREF is not supported, use Format.VALUE_WITH_FAKE_GLOBALS + # before falling back to Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": int}) + + def test_user_annotate_forwardref_value_fallback(self): + # If Format.FORWARDREF and Format.VALUE_WITH_FAKE_GLOBALS are not supported + # use Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_string_supported(self): + # If Format.STRING is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "float"}) + + def test_user_annotate_string_fakeglobals(self): + # If Format.STRING is not supported but Format.VALUE_WITH_FAKE_GLOBALS is + # prefer that over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "int"}) + + def test_user_annotate_string_value_fallback(self): + # If Format.STRING and Format.VALUE_WITH_FAKE_GLOBALS are not + # supported fall back to Format.VALUE and convert to strings + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "str"}) + + def test_condition_not_stringified(self): + # Make sure the first condition isn't evaluated as True by being converted + # to a _Stringifier + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(NotImplementedError): + annotationlib.call_annotate_function(annotate, Format.STRING) + + def test_unsupported_formats(self): + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(ValueError): + annotationlib.call_annotate_function(annotate, Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(RuntimeError): + annotationlib.call_annotate_function(annotate, Format.VALUE) + + with self.assertRaises(ValueError): + # Some non-Format value + annotationlib.call_annotate_function(annotate, 7) + + def test_error_from_value_raised(self): + # Test that the error from format.VALUE is raised + # if all formats fail + + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + annotationlib.call_annotate_function(annotate, format=fmt) + + +class MetaclassTests(unittest.TestCase): + def test_annotated_meta(self): + class Meta(type): + a: int + + class X(metaclass=Meta): + pass + + class Y(metaclass=Meta): + b: float + + self.assertEqual(get_annotations(Meta), {"a": int}) + self.assertEqual(Meta.__annotate__(Format.VALUE), {"a": int}) + + self.assertEqual(get_annotations(X), {}) + self.assertIs(X.__annotate__, None) + + self.assertEqual(get_annotations(Y), {"b": float}) + self.assertEqual(Y.__annotate__(Format.VALUE), {"b": float}) + + def test_unannotated_meta(self): + class Meta(type): + pass + + class X(metaclass=Meta): + a: str + + class Y(X): + pass + + self.assertEqual(get_annotations(Meta), {}) + self.assertIs(Meta.__annotate__, None) + + self.assertEqual(get_annotations(Y), {}) + self.assertIs(Y.__annotate__, None) + + self.assertEqual(get_annotations(X), {"a": str}) + self.assertEqual(X.__annotate__(Format.VALUE), {"a": str}) + + def test_ordering(self): + # Based on a sample by David Ellis + # https://discuss.python.org/t/pep-749-implementing-pep-649/54974/38 + + def make_classes(): + class Meta(type): + a: int + expected_annotations = {"a": int} + + class A(type, metaclass=Meta): + b: float + expected_annotations = {"b": float} + + class B(metaclass=A): + c: str + expected_annotations = {"c": str} + + class C(B): + expected_annotations = {} + + class D(metaclass=Meta): + expected_annotations = {} + + return Meta, A, B, C, D + + classes = make_classes() + class_count = len(classes) + for order in itertools.permutations(range(class_count), class_count): + names = ", ".join(classes[i].__name__ for i in order) + with self.subTest(names=names): + classes = make_classes() # Regenerate classes + for i in order: + get_annotations(classes[i]) + for c in classes: + with self.subTest(c=c): + self.assertEqual(get_annotations(c), c.expected_annotations) + annotate_func = getattr(c, "__annotate__", None) + if c.expected_annotations: + self.assertEqual( + annotate_func(Format.VALUE), c.expected_annotations + ) + else: + self.assertIs(annotate_func, None) + + +class TestGetAnnotateFromClassNamespace(unittest.TestCase): + def test_with_metaclass(self): + class Meta(type): + def __new__(mcls, name, bases, ns): + annotate = annotationlib.get_annotate_from_class_namespace(ns) + expected = ns["expected_annotate"] + with self.subTest(name=name): + if expected: + self.assertIsNotNone(annotate) + else: + self.assertIsNone(annotate) + return super().__new__(mcls, name, bases, ns) + + class HasAnnotations(metaclass=Meta): + expected_annotate = True + a: int + + class NoAnnotations(metaclass=Meta): + expected_annotate = False + + class CustomAnnotate(metaclass=Meta): + expected_annotate = True + def __annotate__(format): + return {} + + code = """ + from __future__ import annotations + + class HasFutureAnnotations(metaclass=Meta): + expected_annotate = False + a: int + """ + exec(textwrap.dedent(code), {"Meta": Meta}) + + +class TestTypeRepr(unittest.TestCase): + def test_type_repr(self): + class Nested: + pass + + def nested(): + pass + + self.assertEqual(type_repr(int), "int") + self.assertEqual(type_repr(MyClass), f"{__name__}.MyClass") + self.assertEqual( + type_repr(Nested), f"{__name__}.TestTypeRepr.test_type_repr..Nested" + ) + self.assertEqual( + type_repr(nested), f"{__name__}.TestTypeRepr.test_type_repr..nested" + ) + self.assertEqual(type_repr(len), "len") + self.assertEqual(type_repr(type_repr), "annotationlib.type_repr") + self.assertEqual(type_repr(times_three), f"{__name__}.times_three") + self.assertEqual(type_repr(...), "...") + self.assertEqual(type_repr(None), "None") + self.assertEqual(type_repr(1), "1") + self.assertEqual(type_repr("1"), "'1'") + self.assertEqual(type_repr(Format.VALUE), repr(Format.VALUE)) + self.assertEqual(type_repr(MyClass()), "my repr") + # gh138558 tests + self.assertEqual(type_repr(t'''{ 0 + & 1 + | 2 + }'''), 't"""{ 0\n & 1\n | 2}"""') + self.assertEqual( + type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'" + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42))), + "Template('hi', Interpolation(42, '', None, ''))", + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42, " "))), + "Template('hi', Interpolation(42, ' ', None, ''))", + ) + # gh138558: perhaps in the future, we can improve this behavior: + self.assertEqual(type_repr(Template(Interpolation(42, "99"))), "t'{99}'") + + +class TestAnnotationsToString(unittest.TestCase): + def test_annotations_to_string(self): + self.assertEqual(annotations_to_string({}), {}) + self.assertEqual(annotations_to_string({"x": int}), {"x": "int"}) + self.assertEqual(annotations_to_string({"x": "int"}), {"x": "int"}) + self.assertEqual( + annotations_to_string({"x": int, "y": str}), {"x": "int", "y": "str"} + ) + + +class A: + pass + +TypeParamsAlias1 = int + +class TypeParamsSample[TypeParamsAlias1, TypeParamsAlias2]: + TypeParamsAlias2 = str + + +class TestForwardRefClass(unittest.TestCase): + def test_forwardref_instance_type_error(self): + fr = ForwardRef("int") + with self.assertRaises(TypeError): + isinstance(42, fr) + + def test_forwardref_subclass_type_error(self): + fr = ForwardRef("int") + with self.assertRaises(TypeError): + issubclass(int, fr) + + def test_forwardref_only_str_arg(self): + with self.assertRaises(TypeError): + ForwardRef(1) # only `str` type is allowed + + def test_forward_equality(self): + fr = ForwardRef("int") + self.assertEqual(fr, ForwardRef("int")) + self.assertNotEqual(List["int"], List[int]) + self.assertNotEqual(fr, ForwardRef("int", module=__name__)) + frm = ForwardRef("int", module=__name__) + self.assertEqual(frm, ForwardRef("int", module=__name__)) + self.assertNotEqual(frm, ForwardRef("int", module="__other_name__")) + + def test_forward_equality_get_type_hints(self): + c1 = ForwardRef("C") + c1_gth = ForwardRef("C") + c2 = ForwardRef("C") + c2_gth = ForwardRef("C") + + class C: + pass + + def foo(a: c1_gth, b: c2_gth): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), {"a": C, "b": C}) + self.assertEqual(c1, c2) + self.assertEqual(c1, c1_gth) + self.assertEqual(c1_gth, c2_gth) + self.assertEqual(List[c1], List[c1_gth]) + self.assertNotEqual(List[c1], List[C]) + self.assertNotEqual(List[c1_gth], List[C]) + self.assertEqual(Union[c1, c1_gth], Union[c1]) + self.assertEqual(Union[c1, c1_gth, int], Union[c1, int]) + + def test_forward_equality_hash(self): + c1 = ForwardRef("int") + c1_gth = ForwardRef("int") + c2 = ForwardRef("int") + c2_gth = ForwardRef("int") + + def foo(a: c1_gth, b: c2_gth): + pass + + get_type_hints(foo, globals(), locals()) + + self.assertEqual(hash(c1), hash(c2)) + self.assertEqual(hash(c1_gth), hash(c2_gth)) + self.assertEqual(hash(c1), hash(c1_gth)) + + c3 = ForwardRef("int", module=__name__) + c4 = ForwardRef("int", module="__other_name__") + + self.assertNotEqual(hash(c3), hash(c1)) + self.assertNotEqual(hash(c3), hash(c1_gth)) + self.assertNotEqual(hash(c3), hash(c4)) + self.assertEqual(hash(c3), hash(ForwardRef("int", module=__name__))) + + def test_forward_equality_and_hash_with_cells(self): + """Regression test for GH-143831.""" + class A: + def one(_) -> C1: + """One cell.""" + + one_f = ForwardRef("C1", owner=one) + one_f_ga1 = get_annotations(one, format=Format.FORWARDREF)["return"] + one_f_ga2 = get_annotations(one, format=Format.FORWARDREF)["return"] + self.assertIsInstance(one_f_ga1.__cell__, types.CellType) + self.assertIs(one_f_ga1.__cell__, one_f_ga2.__cell__) + + def two(_) -> C1 | C2: + """Two cells.""" + + two_f_ga1 = get_annotations(two, format=Format.FORWARDREF)["return"] + two_f_ga2 = get_annotations(two, format=Format.FORWARDREF)["return"] + self.assertIsNot(two_f_ga1.__cell__, two_f_ga2.__cell__) + self.assertIsInstance(two_f_ga1.__cell__, dict) + self.assertIsInstance(two_f_ga2.__cell__, dict) + + type C1 = None + type C2 = None + + self.assertNotEqual(A.one_f, A.one_f_ga1) + self.assertNotEqual(hash(A.one_f), hash(A.one_f_ga1)) + + self.assertEqual(A.one_f_ga1, A.one_f_ga2) + self.assertEqual(hash(A.one_f_ga1), hash(A.one_f_ga2)) + + self.assertEqual(A.two_f_ga1, A.two_f_ga2) + self.assertEqual(hash(A.two_f_ga1), hash(A.two_f_ga2)) + + def test_forward_equality_namespace(self): + def namespace1(): + a = ForwardRef("A") + + def fun(x: a): + pass + + get_type_hints(fun, globals(), locals()) + return a + + def namespace2(): + a = ForwardRef("A") + + class A: + pass + + def fun(x: a): + pass + + get_type_hints(fun, globals(), locals()) + return a + + self.assertEqual(namespace1(), namespace1()) + self.assertEqual(namespace1(), namespace2()) + + def test_forward_repr(self): + self.assertEqual(repr(List["int"]), "typing.List[ForwardRef('int')]") + self.assertEqual( + repr(List[ForwardRef("int", module="mod")]), + "typing.List[ForwardRef('int', module='mod')]", + ) + self.assertEqual( + repr(List[ForwardRef("int", module="mod", is_class=True)]), + "typing.List[ForwardRef('int', module='mod', is_class=True)]", + ) + self.assertEqual( + repr(List[ForwardRef("int", owner="class")]), + "typing.List[ForwardRef('int', owner='class')]", + ) + + def test_forward_recursion_actually(self): + def namespace1(): + a = ForwardRef("A") + A = a + + def fun(x: a): + pass + + ret = get_type_hints(fun, globals(), locals()) + return a + + def namespace2(): + a = ForwardRef("A") + A = a + + def fun(x: a): + pass + + ret = get_type_hints(fun, globals(), locals()) + return a + + r1 = namespace1() + r2 = namespace2() + self.assertIsNot(r1, r2) + self.assertEqual(r1, r2) + + def test_syntax_error(self): + + with self.assertRaises(SyntaxError): + typing.Generic["/T"] + + def test_delayed_syntax_error(self): + + def foo(a: "Node[T"): + pass + + with self.assertRaises(SyntaxError): + get_type_hints(foo) + + def test_syntax_error_empty_string(self): + for form in [typing.List, typing.Set, typing.Type, typing.Deque]: + with self.subTest(form=form): + with self.assertRaises(SyntaxError): + form[""] + + def test_or(self): + X = ForwardRef("X") + # __or__/__ror__ itself + self.assertEqual(X | "x", Union[X, "x"]) + self.assertEqual("x" | X, Union["x", X]) + + def test_multiple_ways_to_create(self): + X1 = Union["X"] + self.assertIsInstance(X1, ForwardRef) + X2 = ForwardRef("X") + self.assertIsInstance(X2, ForwardRef) + self.assertEqual(X1, X2) + + def test_special_attrs(self): + # Forward refs provide a different introspection API. __name__ and + # __qualname__ make little sense for forward refs as they can store + # complex typing expressions. + fr = ForwardRef("set[Any]") + self.assertNotHasAttr(fr, "__name__") + self.assertNotHasAttr(fr, "__qualname__") + self.assertEqual(fr.__module__, "annotationlib") + # Forward refs are currently unpicklable once they contain a code object. + fr.__forward_code__ # fill the cache + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(fr, proto) + + def test_evaluate_string_format(self): + fr = ForwardRef("set[Any]") + self.assertEqual(fr.evaluate(format=Format.STRING), "set[Any]") + + def test_evaluate_forwardref_format(self): + fr = ForwardRef("undef") + evaluated = fr.evaluate(format=Format.FORWARDREF) + self.assertIs(fr, evaluated) + + fr = ForwardRef("set[undefined]") + evaluated = fr.evaluate(format=Format.FORWARDREF) + self.assertEqual( + evaluated, + set[support.EqualToForwardRef("undefined")], + ) + + fr = ForwardRef("a + b") + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF), + support.EqualToForwardRef("a + b"), + ) + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF, locals={"a": 1, "b": 2}), + 3, + ) + + fr = ForwardRef('"a" + 1') + self.assertEqual( + fr.evaluate(format=Format.FORWARDREF), + support.EqualToForwardRef('"a" + 1'), + ) + + def test_evaluate_notimplemented_format(self): + class C: + x: alias + + fwdref = get_annotations(C, format=Format.FORWARDREF)["x"] + + with self.assertRaises(NotImplementedError): + fwdref.evaluate(format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(NotImplementedError): + # Some other unsupported value + fwdref.evaluate(format=7) + + def test_evaluate_with_type_params(self): + class Gen[T]: + alias = int + + with self.assertRaises(NameError): + ForwardRef("T").evaluate() + with self.assertRaises(NameError): + ForwardRef("T").evaluate(type_params=()) + with self.assertRaises(NameError): + ForwardRef("T").evaluate(owner=int) + + (T,) = Gen.__type_params__ + self.assertIs(ForwardRef("T").evaluate(type_params=Gen.__type_params__), T) + self.assertIs(ForwardRef("T").evaluate(owner=Gen), T) + + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(type_params=Gen.__type_params__) + self.assertIs(ForwardRef("alias").evaluate(owner=Gen), int) + # If you pass custom locals, we don't look at the owner's locals + with self.assertRaises(NameError): + ForwardRef("alias").evaluate(owner=Gen, locals={}) + # But if the name exists in the locals, it works + self.assertIs( + ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str + ) + + def test_evaluate_with_type_params_and_scope_conflict(self): + for is_class in (False, True): + with self.subTest(is_class=is_class): + fwdref1 = ForwardRef("TypeParamsAlias1", owner=TypeParamsSample, is_class=is_class) + fwdref2 = ForwardRef("TypeParamsAlias2", owner=TypeParamsSample, is_class=is_class) + + self.assertIs( + fwdref1.evaluate(), + TypeParamsSample.__type_params__[0], + ) + self.assertIs( + fwdref2.evaluate(), + TypeParamsSample.TypeParamsAlias2, + ) + + def test_fwdref_with_module(self): + self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) + self.assertIs( + ForwardRef("Counter", module="collections").evaluate(), collections.Counter + ) + self.assertEqual( + ForwardRef("Counter[int]", module="collections").evaluate(), + collections.Counter[int], + ) + + with self.assertRaises(NameError): + # If globals are passed explicitly, we don't look at the module dict + ForwardRef("Format", module="annotationlib").evaluate(globals={}) + + def test_fwdref_to_builtin(self): + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int", module="collections").evaluate(), int) + self.assertIs(ForwardRef("int", owner=str).evaluate(), int) + + # builtins are still searched with explicit globals + self.assertIs(ForwardRef("int").evaluate(globals={}), int) + + # explicit values in globals have precedence + obj = object() + self.assertIs(ForwardRef("int").evaluate(globals={"int": obj}), obj) + + def test_fwdref_value_is_not_cached(self): + fr = ForwardRef("hello") + with self.assertRaises(NameError): + fr.evaluate() + self.assertIs(fr.evaluate(globals={"hello": str}), str) + with self.assertRaises(NameError): + fr.evaluate() + + def test_fwdref_with_owner(self): + self.assertEqual( + ForwardRef("Counter[int]", owner=collections).evaluate(), + collections.Counter[int], + ) + + def test_name_lookup_without_eval(self): + # test the codepath where we look up simple names directly in the + # namespaces without going through eval() + self.assertIs(ForwardRef("int").evaluate(), int) + self.assertIs(ForwardRef("int").evaluate(locals={"int": str}), str) + self.assertIs( + ForwardRef("int").evaluate(locals={"int": float}, globals={"int": str}), + float, + ) + self.assertIs(ForwardRef("int").evaluate(globals={"int": str}), str) + with support.swap_attr(builtins, "int", dict): + self.assertIs(ForwardRef("int").evaluate(), dict) + + with self.assertRaises(NameError, msg="name 'doesntexist' is not defined") as exc: + ForwardRef("doesntexist").evaluate() + + self.assertEqual(exc.exception.name, "doesntexist") + + def test_evaluate_undefined_generic(self): + # Test the codepath where have to eval() with undefined variables. + class C: + x: alias[int, undef] + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": Union}, + locals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + def test_fwdref_invalid_syntax(self): + fr = ForwardRef("if") + with self.assertRaises(SyntaxError): + fr.evaluate() + fr = ForwardRef("1+") + with self.assertRaises(SyntaxError): + fr.evaluate() + + def test_re_evaluate_generics(self): + global global_alias + + # If we've already run this test before, + # ensure the variable is still undefined + if "global_alias" in globals(): + del global_alias + + class C: + x: global_alias[int] + + # Evaluate the ForwardRef once + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF + ) + + # Now define the global and ensure that the ForwardRef evaluates + global_alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + + def test_fwdref_evaluate_argument_mutation(self): + class C[T]: + nonlocal alias + x: alias[T] + + # Mutable arguments + globals_ = globals() + globals_copy = globals_.copy() + locals_ = locals() + locals_copy = locals_.copy() + + # Evaluate the ForwardRef, ensuring we use __cell__ and type params + get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + globals=globals_, + locals=locals_, + type_params=C.__type_params__, + format=Format.FORWARDREF, + ) + + # Check if the passed in mutable arguments equal the originals + self.assertEqual(globals_, globals_copy) + self.assertEqual(locals_, locals_copy) + + alias = list + + def test_fwdref_final_class(self): + with self.assertRaises(TypeError): + class C(ForwardRef): + pass + + +class TestAnnotationLib(unittest.TestCase): + def test__all__(self): + support.check__all__(self, annotationlib) + + @support.cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports( + "annotationlib", + { + "typing", + "warnings", + }, + ) diff --git a/Lib/test/test_apple.py b/Lib/test/test_apple.py new file mode 100644 index 00000000000..ab5296afad1 --- /dev/null +++ b/Lib/test/test_apple.py @@ -0,0 +1,155 @@ +import unittest +from _apple_support import SystemLog +from test.support import is_apple +from unittest.mock import Mock, call + +if not is_apple: + raise unittest.SkipTest("Apple-specific") + + +# Test redirection of stdout and stderr to the Apple system log. +class TestAppleSystemLogOutput(unittest.TestCase): + maxDiff = None + + def assert_writes(self, output): + self.assertEqual( + self.log_write.mock_calls, + [ + call(self.log_level, line) + for line in output + ] + ) + + self.log_write.reset_mock() + + def setUp(self): + self.log_write = Mock() + self.log_level = 42 + self.log = SystemLog(self.log_write, self.log_level, errors="replace") + + def test_repr(self): + self.assertEqual(repr(self.log), "") + self.assertEqual(repr(self.log.buffer), "") + + def test_log_config(self): + self.assertIs(self.log.writable(), True) + self.assertIs(self.log.readable(), False) + + self.assertEqual("UTF-8", self.log.encoding) + self.assertEqual("replace", self.log.errors) + + self.assertIs(self.log.line_buffering, True) + self.assertIs(self.log.write_through, False) + + def test_empty_str(self): + self.log.write("") + self.log.flush() + + self.assert_writes([]) + + def test_simple_str(self): + self.log.write("hello world\n") + + self.assert_writes([b"hello world\n"]) + + def test_buffered_str(self): + self.log.write("h") + self.log.write("ello") + self.log.write(" ") + self.log.write("world\n") + self.log.write("goodbye.") + self.log.flush() + + self.assert_writes([b"hello world\n", b"goodbye."]) + + def test_manual_flush(self): + self.log.write("Hello") + + self.assert_writes([]) + + self.log.write(" world\nHere for a while...\nGoodbye") + self.assert_writes([b"Hello world\n", b"Here for a while...\n"]) + + self.log.write(" world\nHello again") + self.assert_writes([b"Goodbye world\n"]) + + self.log.flush() + self.assert_writes([b"Hello again"]) + + def test_non_ascii(self): + # Spanish + self.log.write("ol\u00e9\n") + self.assert_writes([b"ol\xc3\xa9\n"]) + + # Chinese + self.log.write("\u4e2d\u6587\n") + self.assert_writes([b"\xe4\xb8\xad\xe6\x96\x87\n"]) + + # Printing Non-BMP emoji + self.log.write("\U0001f600\n") + self.assert_writes([b"\xf0\x9f\x98\x80\n"]) + + # Non-encodable surrogates are replaced + self.log.write("\ud800\udc00\n") + self.assert_writes([b"??\n"]) + + def test_modified_null(self): + # Null characters are logged using "modified UTF-8". + self.log.write("\u0000\n") + self.assert_writes([b"\xc0\x80\n"]) + self.log.write("a\u0000\n") + self.assert_writes([b"a\xc0\x80\n"]) + self.log.write("\u0000b\n") + self.assert_writes([b"\xc0\x80b\n"]) + self.log.write("a\u0000b\n") + self.assert_writes([b"a\xc0\x80b\n"]) + + def test_nonstandard_str(self): + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + self.log.write(CustomStr("custom\n")) + self.assert_writes([b"custom\n"]) + + def test_non_str(self): + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + self.log.write(obj) + + def test_byteslike_in_buffer(self): + # The underlying buffer *can* accept bytes-like objects + self.log.buffer.write(bytearray(b"hello")) + self.log.flush() + + self.log.buffer.write(b"") + self.log.flush() + + self.log.buffer.write(b"goodbye") + self.log.flush() + + self.assert_writes([b"hello", b"goodbye"]) + + def test_non_byteslike_in_buffer(self): + for obj in ["hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + self.log.buffer.write(obj) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 3a62a16cee3..f48fb765bb3 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1,11 +1,13 @@ # Author: Steven J. Bethard . +import _colorize import contextlib import functools import inspect import io import operator import os +import py_compile import shutil import stat import sys @@ -15,10 +17,23 @@ import argparse import warnings +from enum import StrEnum +from test.support import ( + captured_stderr, + force_not_colorized, + force_not_colorized_test_class, + swap_attr, +) +from test.support import import_helper from test.support import os_helper +from test.support import script_helper +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots from unittest import mock +py = os.path.basename(sys.executable) + + class StdIOBuffer(io.TextIOWrapper): '''Replacement for writable io.StringIO that behaves more like real file @@ -280,16 +295,18 @@ def test_failures(self, tester): parser = self._get_parser(tester) for args_str in tester.failures: args = args_str.split() - with tester.assertRaises(ArgumentParserError, msg=args): - parser.parse_args(args) + with tester.subTest(args=args): + with tester.assertRaises(ArgumentParserError, msg=args): + parser.parse_args(args) def test_successes(self, tester): parser = self._get_parser(tester) for args, expected_ns in tester.successes: if isinstance(args, str): args = args.split() - result_ns = self._parse_args(parser, args) - tester.assertEqual(expected_ns, result_ns) + with tester.subTest(args=args): + result_ns = self._parse_args(parser, args) + tester.assertEqual(expected_ns, result_ns) # add tests for each combination of an optionals adding method # and an arg parsing method @@ -378,15 +395,22 @@ class TestOptionalsSingleDashAmbiguous(ParserTestCase): """Test Optionals that partially match but are not subsets""" argument_signatures = [Sig('-foobar'), Sig('-foorab')] - failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b'] + failures = ['-f', '-f a', '-fa', '-foa', '-foo', '-fo', '-foo b', + '-f=a', '-foo=b'] successes = [ ('', NS(foobar=None, foorab=None)), ('-foob a', NS(foobar='a', foorab=None)), + ('-foob=a', NS(foobar='a', foorab=None)), ('-foor a', NS(foobar=None, foorab='a')), + ('-foor=a', NS(foobar=None, foorab='a')), ('-fooba a', NS(foobar='a', foorab=None)), + ('-fooba=a', NS(foobar='a', foorab=None)), ('-foora a', NS(foobar=None, foorab='a')), + ('-foora=a', NS(foobar=None, foorab='a')), ('-foobar a', NS(foobar='a', foorab=None)), + ('-foobar=a', NS(foobar='a', foorab=None)), ('-foorab a', NS(foobar=None, foorab='a')), + ('-foorab=a', NS(foobar=None, foorab='a')), ] @@ -619,9 +643,9 @@ class TestOptionalsNargsOptional(ParserTestCase): Sig('-w', nargs='?'), Sig('-x', nargs='?', const=42), Sig('-y', nargs='?', default='spam'), - Sig('-z', nargs='?', type=int, const='42', default='84'), + Sig('-z', nargs='?', type=int, const='42', default='84', choices=[1, 2]), ] - failures = ['2'] + failures = ['2', '-z a', '-z 42', '-z 84'] successes = [ ('', NS(w=None, x=None, y='spam', z=84)), ('-w', NS(w=None, x=None, y='spam', z=84)), @@ -677,7 +701,7 @@ class TestOptionalsChoices(ParserTestCase): argument_signatures = [ Sig('-f', choices='abc'), Sig('-g', type=int, choices=range(5))] - failures = ['a', '-f d', '-fad', '-ga', '-g 6'] + failures = ['a', '-f d', '-f ab', '-fad', '-ga', '-g 6'] successes = [ ('', NS(f=None, g=None)), ('-f a', NS(f='a', g=None)), @@ -765,48 +789,12 @@ def test_const(self): self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) - def test_deprecated_init_kw(self): - # See gh-92248 + def test_invalid_name(self): parser = argparse.ArgumentParser() - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-a', - action=argparse.BooleanOptionalAction, - type=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-b', - action=argparse.BooleanOptionalAction, - type=bool, - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-c', - action=argparse.BooleanOptionalAction, - metavar=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-d', - action=argparse.BooleanOptionalAction, - metavar='d', - ) - - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-e', - action=argparse.BooleanOptionalAction, - choices=None, - ) - with self.assertWarns(DeprecationWarning): - parser.add_argument( - '-f', - action=argparse.BooleanOptionalAction, - choices=(), - ) + with self.assertRaises(ValueError) as cm: + parser.add_argument('--no-foo', action=argparse.BooleanOptionalAction) + self.assertEqual(str(cm.exception), + "invalid option name '--no-foo' for BooleanOptionalAction") class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" @@ -916,7 +904,9 @@ class TestOptionalsAllowLongAbbreviation(ParserTestCase): successes = [ ('', NS(foo=None, foobaz=None, fooble=False)), ('--foo 7', NS(foo='7', foobaz=None, fooble=False)), + ('--foo=7', NS(foo='7', foobaz=None, fooble=False)), ('--fooba a', NS(foo=None, foobaz='a', fooble=False)), + ('--fooba=a', NS(foo=None, foobaz='a', fooble=False)), ('--foobl --foo g', NS(foo='g', foobaz=None, fooble=True)), ] @@ -955,6 +945,23 @@ class TestOptionalsDisallowLongAbbreviationPrefixChars(ParserTestCase): ] +class TestOptionalsDisallowSingleDashLongAbbreviation(ParserTestCase): + """Do not allow abbreviations of long options at all""" + + parser_signature = Sig(allow_abbrev=False) + argument_signatures = [ + Sig('-foo'), + Sig('-foodle', action='store_true'), + Sig('-foonly'), + ] + failures = ['-foon 3', '-food', '-food -foo 2'] + successes = [ + ('', NS(foo=None, foodle=False, foonly=None)), + ('-foo 3', NS(foo='3', foodle=False, foonly=None)), + ('-foonly 7 -foodle -foo 2', NS(foo='2', foodle=True, foonly='7')), + ] + + class TestDisallowLongAbbreviationAllowsShortGrouping(ParserTestCase): """Do not allow abbreviations of long options at all""" @@ -993,6 +1000,35 @@ class TestDisallowLongAbbreviationAllowsShortGroupingPrefix(ParserTestCase): ] +class TestStrEnumChoices(TestCase): + class Color(StrEnum): + RED = "red" + GREEN = "green" + BLUE = "blue" + + def test_parse_enum_value(self): + parser = argparse.ArgumentParser() + parser.add_argument('--color', choices=self.Color) + args = parser.parse_args(['--color', 'red']) + self.assertEqual(args.color, self.Color.RED) + + @force_not_colorized + def test_help_message_contains_enum_choices(self): + parser = argparse.ArgumentParser() + parser.add_argument('--color', choices=self.Color, help='Choose a color') + self.assertIn('[--color {red,green,blue}]', parser.format_usage()) + self.assertIn(' --color {red,green,blue}', parser.format_help()) + + def test_invalid_enum_value_raises_error(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--color', choices=self.Color) + self.assertRaisesRegex( + argparse.ArgumentError, + r"invalid choice: 'yellow' \(choose from red, green, blue\)", + parser.parse_args, + ['--color', 'yellow'], + ) + # ================ # Positional tests # ================ @@ -1042,8 +1078,8 @@ class TestPositionalsNargsZeroOrMore(ParserTestCase): class TestPositionalsNargsZeroOrMoreDefault(ParserTestCase): """Test a Positional that specifies unlimited nargs and a default""" - argument_signatures = [Sig('foo', nargs='*', default='bar')] - failures = ['-x'] + argument_signatures = [Sig('foo', nargs='*', default='bar', choices=['a', 'b'])] + failures = ['-x', 'bar', 'a c'] successes = [ ('', NS(foo='bar')), ('a', NS(foo=['a'])), @@ -1076,8 +1112,8 @@ class TestPositionalsNargsOptional(ParserTestCase): class TestPositionalsNargsOptionalDefault(ParserTestCase): """Tests an Optional Positional with a default value""" - argument_signatures = [Sig('foo', nargs='?', default=42)] - failures = ['-x', 'a b'] + argument_signatures = [Sig('foo', nargs='?', default=42, choices=['a', 'b'])] + failures = ['-x', 'a b', '42'] successes = [ ('', NS(foo=42)), ('a', NS(foo='a')), @@ -1090,9 +1126,9 @@ class TestPositionalsNargsOptionalConvertedDefault(ParserTestCase): """ argument_signatures = [ - Sig('foo', nargs='?', type=int, default='42'), + Sig('foo', nargs='?', type=int, default='42', choices=[1, 2]), ] - failures = ['-x', 'a b', '1 2'] + failures = ['-x', 'a b', '1 2', '42'] successes = [ ('', NS(foo=42)), ('1', NS(foo=1)), @@ -1132,57 +1168,87 @@ class TestPositionalsNargs2None(ParserTestCase): class TestPositionalsNargsNoneZeroOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with unlimited""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='*')] - failures = ['', '--foo'] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='*')] + failures = ['', '--foo', 'a b -x X c'] successes = [ - ('a', NS(foo='a', bar=[])), - ('a b', NS(foo='a', bar=['b'])), - ('a b c', NS(foo='a', bar=['b', 'c'])), + ('a', NS(x=None, foo='a', bar=[])), + ('a b', NS(x=None, foo='a', bar=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b', 'c'])), + ('-x X a', NS(x='X', foo='a', bar=[])), + ('a -x X', NS(x='X', foo='a', bar=[])), + ('-x X a b', NS(x='X', foo='a', bar=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOneOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with one or more""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='+')] - failures = ['', '--foo', 'a'] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='+')] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=['b'])), - ('a b c', NS(foo='a', bar=['b', 'c'])), + ('a b', NS(x=None, foo='a', bar=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b', 'c'])), + ('-x X a b', NS(x='X', foo='a', bar=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOptional(ParserTestCase): """Test a Positional with no nargs followed by one with an Optional""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='?')] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='?')] failures = ['', '--foo', 'a b c'] successes = [ - ('a', NS(foo='a', bar=None)), - ('a b', NS(foo='a', bar='b')), + ('a', NS(x=None, foo='a', bar=None)), + ('a b', NS(x=None, foo='a', bar='b')), + ('-x X a', NS(x='X', foo='a', bar=None)), + ('a -x X', NS(x='X', foo='a', bar=None)), + ('-x X a b', NS(x='X', foo='a', bar='b')), + ('a -x X b', NS(x='X', foo='a', bar='b')), + ('a b -x X', NS(x='X', foo='a', bar='b')), ] class TestPositionalsNargsZeroOrMoreNone(ParserTestCase): """Test a Positional with unlimited nargs followed by one with none""" - argument_signatures = [Sig('foo', nargs='*'), Sig('bar')] - failures = ['', '--foo'] + argument_signatures = [Sig('-x'), Sig('foo', nargs='*'), Sig('bar')] + failures = ['', '--foo', 'a -x X b', 'a -x X b c', 'a b -x X c'] successes = [ - ('a', NS(foo=[], bar='a')), - ('a b', NS(foo=['a'], bar='b')), - ('a b c', NS(foo=['a', 'b'], bar='c')), + ('a', NS(x=None, foo=[], bar='a')), + ('a b', NS(x=None, foo=['a'], bar='b')), + ('a b c', NS(x=None, foo=['a', 'b'], bar='c')), + ('-x X a', NS(x='X', foo=[], bar='a')), + ('a -x X', NS(x='X', foo=[], bar='a')), + ('-x X a b', NS(x='X', foo=['a'], bar='b')), + ('a b -x X', NS(x='X', foo=['a'], bar='b')), + ('-x X a b c', NS(x='X', foo=['a', 'b'], bar='c')), + ('a b c -x X', NS(x='X', foo=['a', 'b'], bar='c')), ] class TestPositionalsNargsOneOrMoreNone(ParserTestCase): """Test a Positional with one or more nargs followed by one with none""" - argument_signatures = [Sig('foo', nargs='+'), Sig('bar')] - failures = ['', '--foo', 'a'] + argument_signatures = [Sig('-x'), Sig('foo', nargs='+'), Sig('bar')] + failures = ['', '--foo', 'a', 'a -x X b c', 'a b -x X c'] successes = [ - ('a b', NS(foo=['a'], bar='b')), - ('a b c', NS(foo=['a', 'b'], bar='c')), + ('a b', NS(x=None, foo=['a'], bar='b')), + ('a b c', NS(x=None, foo=['a', 'b'], bar='c')), + ('-x X a b', NS(x='X', foo=['a'], bar='b')), + ('a -x X b', NS(x='X', foo=['a'], bar='b')), + ('a b -x X', NS(x='X', foo=['a'], bar='b')), + ('-x X a b c', NS(x='X', foo=['a', 'b'], bar='c')), + ('a b c -x X', NS(x='X', foo=['a', 'b'], bar='c')), ] @@ -1267,14 +1333,21 @@ class TestPositionalsNargsNoneZeroOrMore1(ParserTestCase): """Test three Positionals: no nargs, unlimited nargs and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='*'), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a'] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=[], baz=['b'])), - ('a b c', NS(foo='a', bar=['b'], baz=['c'])), + ('a b', NS(x=None, foo='a', bar=[], baz=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b'], baz=['c'])), + ('-x X a b', NS(x='X', foo='a', bar=[], baz=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=[], baz=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=[], baz=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b'], baz=['c'])), ] @@ -1282,14 +1355,22 @@ class TestPositionalsNargsNoneOneOrMore1(ParserTestCase): """Test three Positionals: no nargs, one or more nargs and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='+'), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a', 'b'] + failures = ['', '--foo', 'a', 'b', 'a b -x X c d', 'a b c -x X d'] successes = [ - ('a b c', NS(foo='a', bar=['b'], baz=['c'])), - ('a b c d', NS(foo='a', bar=['b', 'c'], baz=['d'])), + ('a b c', NS(x=None, foo='a', bar=['b'], baz=['c'])), + ('a b c d', NS(x=None, foo='a', bar=['b', 'c'], baz=['d'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b -x X c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('-x X a b c d', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), + ('a -x X b c d', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), + ('a b c d -x X', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), ] @@ -1297,14 +1378,21 @@ class TestPositionalsNargsNoneOptional1(ParserTestCase): """Test three Positionals: no nargs, optional narg and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='?', default=0.625), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a'] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=0.625, baz=['b'])), - ('a b c', NS(foo='a', bar='b', baz=['c'])), + ('a b', NS(x=None, foo='a', bar=0.625, baz=['b'])), + ('a b c', NS(x=None, foo='a', bar='b', baz=['c'])), + ('-x X a b', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar='b', baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar='b', baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar='b', baz=['c'])), ] @@ -1382,6 +1470,19 @@ class TestPositionalsActionAppend(ParserTestCase): ('a b c', NS(spam=['a', ['b', 'c']])), ] + +class TestPositionalsActionExtend(ParserTestCase): + """Test the 'extend' action""" + + argument_signatures = [ + Sig('spam', action='extend'), + Sig('spam', action='extend', nargs=2), + ] + failures = ['', '--foo', 'a', 'a b', 'a b c d'] + successes = [ + ('a b c', NS(spam=['a', 'b', 'c'])), + ] + # ======================================== # Combined optionals and positionals tests # ======================================== @@ -1419,6 +1520,32 @@ class TestOptionalsAlmostNumericAndPositionals(ParserTestCase): ] +class TestOptionalsAndPositionalsAppend(ParserTestCase): + argument_signatures = [ + Sig('foo', nargs='*', action='append'), + Sig('--bar'), + ] + failures = ['-foo'] + successes = [ + ('a b', NS(foo=[['a', 'b']], bar=None)), + ('--bar a b', NS(foo=[['b']], bar='a')), + ('a b --bar c', NS(foo=[['a', 'b']], bar='c')), + ] + + +class TestOptionalsAndPositionalsExtend(ParserTestCase): + argument_signatures = [ + Sig('foo', nargs='*', action='extend'), + Sig('--bar'), + ] + failures = ['-foo'] + successes = [ + ('a b', NS(foo=['a', 'b'], bar=None)), + ('--bar a b', NS(foo=['b'], bar='a')), + ('a b --bar c', NS(foo=['a', 'b'], bar='c')), + ] + + class TestEmptyAndSpaceContainingArguments(ParserTestCase): argument_signatures = [ @@ -1481,6 +1608,9 @@ class TestNargsRemainder(ParserTestCase): successes = [ ('X', NS(x='X', y=[], z=None)), ('-z Z X', NS(x='X', y=[], z='Z')), + ('-z Z X A B', NS(x='X', y=['A', 'B'], z='Z')), + ('X -z Z A B', NS(x='X', y=['-z', 'Z', 'A', 'B'], z=None)), + ('X A -z Z B', NS(x='X', y=['A', '-z', 'Z', 'B'], z=None)), ('X A B -z Z', NS(x='X', y=['A', 'B', '-z', 'Z'], z=None)), ('X Y --foo', NS(x='X', y=['Y', '--foo'], z=None)), ] @@ -1517,18 +1647,24 @@ class TestDefaultSuppress(ParserTestCase): """Test actions with suppressed defaults""" argument_signatures = [ - Sig('foo', nargs='?', default=argparse.SUPPRESS), - Sig('bar', nargs='*', default=argparse.SUPPRESS), + Sig('foo', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('bar', nargs='*', type=int, default=argparse.SUPPRESS), Sig('--baz', action='store_true', default=argparse.SUPPRESS), + Sig('--qux', nargs='?', type=int, default=argparse.SUPPRESS), + Sig('--quux', nargs='*', type=int, default=argparse.SUPPRESS), ] - failures = ['-x'] + failures = ['-x', 'a', '1 a'] successes = [ ('', NS()), - ('a', NS(foo='a')), - ('a b', NS(foo='a', bar=['b'])), + ('1', NS(foo=1)), + ('1 2', NS(foo=1, bar=[2])), ('--baz', NS(baz=True)), - ('a --baz', NS(foo='a', baz=True)), - ('--baz a b', NS(foo='a', bar=['b'], baz=True)), + ('1 --baz', NS(foo=1, baz=True)), + ('--baz 1 2', NS(foo=1, bar=[2], baz=True)), + ('--qux', NS(qux=None)), + ('--qux 1', NS(qux=1)), + ('--quux', NS(quux=[])), + ('--quux 1 2', NS(quux=[1, 2])), ] @@ -1652,27 +1788,43 @@ def convert_arg_line_to_args(self, arg_line): # Type conversion tests # ===================== +def FileType(*args, **kwargs): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', 'FileType is deprecated', + PendingDeprecationWarning, __name__) + return argparse.FileType(*args, **kwargs) + + +class TestFileTypeDeprecation(TestCase): + + def test(self): + with self.assertWarns(PendingDeprecationWarning) as cm: + argparse.FileType() + self.assertIn('FileType is deprecated', str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + class TestFileTypeRepr(TestCase): def test_r(self): - type = argparse.FileType('r') + type = FileType('r') self.assertEqual("FileType('r')", repr(type)) def test_wb_1(self): - type = argparse.FileType('wb', 1) + type = FileType('wb', 1) self.assertEqual("FileType('wb', 1)", repr(type)) def test_r_latin(self): - type = argparse.FileType('r', encoding='latin_1') + type = FileType('r', encoding='latin_1') self.assertEqual("FileType('r', encoding='latin_1')", repr(type)) def test_w_big5_ignore(self): - type = argparse.FileType('w', encoding='big5', errors='ignore') + type = FileType('w', encoding='big5', errors='ignore') self.assertEqual("FileType('w', encoding='big5', errors='ignore')", repr(type)) def test_r_1_replace(self): - type = argparse.FileType('r', 1, errors='replace') + type = FileType('r', 1, errors='replace') self.assertEqual("FileType('r', 1, errors='replace')", repr(type)) @@ -1726,7 +1878,6 @@ def __eq__(self, other): text = text.decode('ascii') return self.name == other.name == text - class TestFileTypeR(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for reading files""" @@ -1739,8 +1890,8 @@ def setUp(self): self.create_readonly_file('readonly') argument_signatures = [ - Sig('-x', type=argparse.FileType()), - Sig('spam', type=argparse.FileType('r')), + Sig('-x', type=FileType()), + Sig('spam', type=FileType('r')), ] failures = ['-x', '', 'non-existent-file.txt'] successes = [ @@ -1760,7 +1911,7 @@ def setUp(self): file.close() argument_signatures = [ - Sig('-c', type=argparse.FileType('r'), default='no-file.txt'), + Sig('-c', type=FileType('r'), default='no-file.txt'), ] # should provoke no such file error failures = [''] @@ -1779,8 +1930,8 @@ def setUp(self): file.write(file_name) argument_signatures = [ - Sig('-x', type=argparse.FileType('rb')), - Sig('spam', type=argparse.FileType('rb')), + Sig('-x', type=FileType('rb')), + Sig('spam', type=FileType('rb')), ] failures = ['-x', ''] successes = [ @@ -1818,8 +1969,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('w')), - Sig('spam', type=argparse.FileType('w')), + Sig('-x', type=FileType('w')), + Sig('spam', type=FileType('w')), ] failures = ['-x', '', 'readonly'] successes = [ @@ -1841,8 +1992,8 @@ def setUp(self): self.create_writable_file('writable') argument_signatures = [ - Sig('-x', type=argparse.FileType('x')), - Sig('spam', type=argparse.FileType('x')), + Sig('-x', type=FileType('x')), + Sig('spam', type=FileType('x')), ] failures = ['-x', '', 'readonly', 'writable'] successes = [ @@ -1856,8 +2007,8 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase): """Test the FileType option/argument type for writing binary files""" argument_signatures = [ - Sig('-x', type=argparse.FileType('wb')), - Sig('spam', type=argparse.FileType('wb')), + Sig('-x', type=FileType('wb')), + Sig('spam', type=FileType('wb')), ] failures = ['-x', ''] successes = [ @@ -1873,8 +2024,8 @@ class TestFileTypeXB(TestFileTypeX): "Test the FileType option/argument type for writing new binary files only" argument_signatures = [ - Sig('-x', type=argparse.FileType('xb')), - Sig('spam', type=argparse.FileType('xb')), + Sig('-x', type=FileType('xb')), + Sig('spam', type=FileType('xb')), ] successes = [ ('-x foo bar', NS(x=WFile('foo'), spam=WFile('bar'))), @@ -1886,7 +2037,7 @@ class TestFileTypeOpenArgs(TestCase): """Test that open (the builtin) is correctly called""" def test_open_args(self): - FT = argparse.FileType + FT = FileType cases = [ (FT('rb'), ('rb', -1, None, None)), (FT('w', 1), ('w', 1, None, None)), @@ -1899,6 +2050,10 @@ def test_open_args(self): type('foo') m.assert_called_with('foo', *args) + def test_invalid_file_type(self): + with self.assertRaises(ValueError): + FileType('b')('-test') + class TestFileTypeMissingInitialization(TestCase): """ @@ -1908,7 +2063,7 @@ class TestFileTypeMissingInitialization(TestCase): def test(self): parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('-x', type=argparse.FileType) self.assertEqual( @@ -2092,10 +2247,169 @@ class TestActionExtend(ParserTestCase): ('--foo f1 --foo f2 f3 f4', NS(foo=['f1', 'f2', 'f3', 'f4'])), ] + +class TestNegativeNumber(ParserTestCase): + """Test parsing negative numbers""" + + argument_signatures = [ + Sig('--int', type=int), + Sig('--float', type=float), + Sig('--complex', type=complex), + ] + failures = [ + '--float -_.45', + '--float -1__000.0', + '--float -1.0.0', + '--int -1__000', + '--int -1.0', + '--complex -1__000.0j', + '--complex -1.0jj', + '--complex -_.45j', + ] + successes = [ + ('--int -1000 --float -1000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000 --float -1_000.0', NS(int=-1000, float=-1000.0, complex=None)), + ('--int -1_000_000 --float -1_000_000.0', NS(int=-1000000, float=-1000000.0, complex=None)), + ('--float -1_000.0', NS(int=None, float=-1000.0, complex=None)), + ('--float -1_000_000.0_0', NS(int=None, float=-1000000.0, complex=None)), + ('--float -.5', NS(int=None, float=-0.5, complex=None)), + ('--float -.5_000', NS(int=None, float=-0.5, complex=None)), + ('--float -1e3', NS(int=None, float=-1000, complex=None)), + ('--float -1e-3', NS(int=None, float=-0.001, complex=None)), + ('--complex -1j', NS(int=None, float=None, complex=-1j)), + ('--complex -1_000j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1_000.0j', NS(int=None, float=None, complex=-1000.0j)), + ('--complex -1e3j', NS(int=None, float=None, complex=-1000j)), + ('--complex -1e-3j', NS(int=None, float=None, complex=-0.001j)), + ] + +class TestArgumentAndSubparserSuggestions(TestCase): + """Test error handling and suggestion when a user makes a typo""" + + def test_wrong_argument_error_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + excinfo.exception.stderr + ) + + def test_wrong_argument_error_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_with_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" + " 'bar'? (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_subparsers_no_suggestions(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=False) + subparsers = parser.add_subparsers(required=True) + subparsers.add_parser('foo') + subparsers.add_parser('bar') + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('baz',)) + self.assertIn( + "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + excinfo.exception.stderr, + ) + + def test_wrong_argument_no_suggestion_implicit(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('foo', choices=['bar', 'baz']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_empty(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('bazz',)) + self.assertIn( + "error: argument foo: invalid choice: 'bazz' (choose from )", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_int(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, 2]) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + def test_suggestions_choices_mixed_types(self): + parser = ErrorRaisingArgumentParser(suggest_on_error=True) + parser.add_argument('foo', choices=[1, '2']) + with self.assertRaises(ArgumentParserError) as excinfo: + parser.parse_args(('3',)) + self.assertIn( + "error: argument foo: invalid choice: '3' (choose from 1, 2)", + excinfo.exception.stderr, + ) + + +class TestInvalidAction(TestCase): + """Test invalid user defined Action""" + + class ActionWithoutCall(argparse.Action): + pass + + def test_invalid_type(self): + parser = argparse.ArgumentParser() + + parser.add_argument('--foo', action=self.ActionWithoutCall) + self.assertRaises(NotImplementedError, parser.parse_args, ['--foo', 'bar']) + + def test_modified_invalid_action(self): + parser = argparse.ArgumentParser(exit_on_error=False) + action = parser.add_argument('--foo') + # Someone got crazy and did this + action.type = 1 + self.assertRaisesRegex(TypeError, '1 is not callable', + parser.parse_args, ['--foo', 'bar']) + action.type = () + self.assertRaisesRegex(TypeError, r'\(\) is not callable', + parser.parse_args, ['--foo', 'bar']) + # It is impossible to distinguish a TypeError raised due to a mismatch + # of the required function arguments from a TypeError raised for an incorrect + # argument value, and using the heavy inspection machinery is not worthwhile + # as it does not reliably work in all cases. + # Therefore, a generic ArgumentError is raised to handle this logical error. + action.type = pow + self.assertRaisesRegex(argparse.ArgumentError, + "argument --foo: invalid pow value: 'bar'", + parser.parse_args, ['--foo', 'bar']) + + # ================ # Subparsers tests # ================ +@force_not_colorized_test_class class TestAddSubparsers(TestCase): """Test the add_subparsers method""" @@ -2103,16 +2417,17 @@ def assertArgumentParserError(self, *args, **kwargs): self.assertRaises(ArgumentParserError, *args, **kwargs) def _get_parser(self, subparser_help=False, prefix_chars=None, - aliases=False): + aliases=False, usage=None): # create a parser with a subparsers argument if prefix_chars: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description', prefix_chars=prefix_chars) + prog='PROG', description='main description', usage=usage, + prefix_chars=prefix_chars) parser.add_argument( prefix_chars[0] * 2 + 'foo', action='store_true', help='foo help') else: parser = ErrorRaisingArgumentParser( - prog='PROG', description='main description') + prog='PROG', description='main description', usage=usage) parser.add_argument( '--foo', action='store_true', help='foo help') parser.add_argument( @@ -2126,7 +2441,9 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, else: subparsers_kwargs['help'] = 'command help' subparsers = parser.add_subparsers(**subparsers_kwargs) - self.assertArgumentParserError(parser.add_subparsers) + self.assertRaisesRegex(ValueError, + 'cannot have multiple subparser arguments', + parser.add_subparsers) # add first sub-parser parser1_kwargs = dict(description='1 description') @@ -2136,18 +2453,19 @@ def _get_parser(self, subparser_help=False, prefix_chars=None, parser1_kwargs['aliases'] = ['1alias1', '1alias2'] parser1 = subparsers.add_parser('1', **parser1_kwargs) parser1.add_argument('-w', type=int, help='w help') - parser1.add_argument('x', choices='abc', help='x help') + parser1.add_argument('x', choices=['a', 'b', 'c'], help='x help') # add second sub-parser parser2_kwargs = dict(description='2 description') if subparser_help: parser2_kwargs['help'] = '2 help' parser2 = subparsers.add_parser('2', **parser2_kwargs) - parser2.add_argument('-y', choices='123', help='y help') + parser2.add_argument('-y', choices=['1', '2', '3'], help='y help') parser2.add_argument('z', type=complex, nargs='*', help='z help') # add third sub-parser - parser3_kwargs = dict(description='3 description') + parser3_kwargs = dict(description='3 description', + usage='PROG --foo bar 3 t ...') if subparser_help: parser3_kwargs['help'] = '3 help' parser3 = subparsers.add_parser('3', **parser3_kwargs) @@ -2169,6 +2487,47 @@ def test_parse_args_failures(self): args = args_str.split() self.assertArgumentParserError(self.parser.parse_args, args) + def test_parse_args_failures_details(self): + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [-h] [--foo] bar {1,2,3} ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + self.parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + + def test_parse_args_failures_details_custom_usage(self): + parser = self._get_parser(usage='PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...') + for args_str, usage_str, error_str in [ + ('', + 'usage: PROG [--foo] bar 1 [-w W] {a,b,c}\n' + ' PROG --foo bar 3 t ...', + 'PROG: error: the following arguments are required: bar'), + ('0.5 1 -y', + 'usage: PROG bar 1 [-h] [-w W] {a,b,c}', + 'PROG bar 1: error: the following arguments are required: x'), + ('0.5 3', + 'usage: PROG --foo bar 3 t ...', + 'PROG bar 3: error: the following arguments are required: t'), + ]: + with self.subTest(args_str): + args = args_str.split() + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(args) + self.assertEqual(cm.exception.args[0], 'SystemExit') + self.assertEqual(cm.exception.args[2], f'{usage_str}\n{error_str}\n') + def test_parse_args(self): # check some non-failure cases: self.assertEqual( @@ -2210,6 +2569,68 @@ def test_parse_known_args(self): (NS(foo=False, bar=0.5, w=7, x='b'), ['-W', '-X', 'Y', 'Z']), ) + def test_parse_known_args_to_class_namespace(self): + class C: + pass + self.assertEqual( + self.parser.parse_known_args('0.5 1 b -w 7 -p'.split(), namespace=C), + (C, ['-p']), + ) + self.assertIs(C.foo, False) + self.assertEqual(C.bar, 0.5) + self.assertEqual(C.w, 7) + self.assertEqual(C.x, 'b') + + def test_abbreviation(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('--foodle') + parser.add_argument('--foonly') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('bar') + parser1.add_argument('--fo') + parser1.add_argument('--foonew') + + self.assertEqual(parser.parse_args(['--food', 'baz', 'bar']), + NS(foodle='baz', foonly=None, fo=None, foonew=None)) + self.assertEqual(parser.parse_args(['--foon', 'baz', 'bar']), + NS(foodle=None, foonly='baz', fo=None, foonew=None)) + self.assertArgumentParserError(parser.parse_args, ['--fo', 'baz', 'bar']) + self.assertEqual(parser.parse_args(['bar', '--fo', 'baz']), + NS(foodle=None, foonly=None, fo='baz', foonew=None)) + self.assertEqual(parser.parse_args(['bar', '--foo', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertEqual(parser.parse_args(['bar', '--foon', 'baz']), + NS(foodle=None, foonly=None, fo=None, foonew='baz')) + self.assertArgumentParserError(parser.parse_args, ['bar', '--food', 'baz']) + + def test_parse_known_args_with_single_dash_option(self): + parser = ErrorRaisingArgumentParser() + parser.add_argument('-k', '--known', action='count', default=0) + parser.add_argument('-n', '--new', action='count', default=0) + self.assertEqual(parser.parse_known_args(['-k', '-u']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-u', '-k']), + (NS(known=1, new=0), ['-u'])) + self.assertEqual(parser.parse_known_args(['-ku']), + (NS(known=1, new=0), ['-u'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=u']) + self.assertEqual(parser.parse_known_args(['-uk']), + (NS(known=0, new=0), ['-uk'])) + self.assertEqual(parser.parse_known_args(['-u=k']), + (NS(known=0, new=0), ['-u=k'])) + self.assertEqual(parser.parse_known_args(['-kunknown']), + (NS(known=1, new=0), ['-unknown'])) + self.assertArgumentParserError(parser.parse_known_args, ['-k=unknown']) + self.assertEqual(parser.parse_known_args(['-ku=nknown']), + (NS(known=1, new=0), ['-u=nknown'])) + self.assertEqual(parser.parse_known_args(['-knew']), + (NS(known=1, new=1), ['-ew'])) + self.assertArgumentParserError(parser.parse_known_args, ['-kn=ew']) + self.assertArgumentParserError(parser.parse_known_args, ['-k-new']) + self.assertArgumentParserError(parser.parse_known_args, ['-kn-ew']) + self.assertEqual(parser.parse_known_args(['-kne-w']), + (NS(known=1, new=1), ['-e-w'])) + def test_dest(self): parser = ErrorRaisingArgumentParser() parser.add_argument('--foo', action='store_true') @@ -2260,18 +2681,6 @@ def test_required_subparsers_no_destination_error(self): 'error: the following arguments are required: {foo,bar}\n$' ) - def test_wrong_argument_subparsers_no_destination_error(self): - parser = ErrorRaisingArgumentParser() - subparsers = parser.add_subparsers(required=True) - subparsers.add_parser('foo') - subparsers.add_parser('bar') - with self.assertRaises(ArgumentParserError) as excinfo: - parser.parse_args(('baz',)) - self.assertRegex( - excinfo.exception.stderr, - r"error: argument {foo,bar}: invalid choice: 'baz' \(choose from 'foo', 'bar'\)\n$" - ) - def test_optional_subparsers(self): parser = ErrorRaisingArgumentParser() subparsers = parser.add_subparsers(dest='command', required=False) @@ -2407,6 +2816,29 @@ def test_parser_command_help(self): --foo foo help ''')) + def assert_bad_help(self, context_type, func, *args, **kwargs): + with self.assertRaisesRegex(ValueError, 'badly formed help string') as cm: + func(*args, **kwargs) + self.assertIsInstance(cm.exception.__context__, context_type) + + def test_invalid_subparsers_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(ValueError, parser.add_subparsers, help='%Y-%m-%d') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(KeyError, parser.add_subparsers, help='%(spam)s') + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assert_bad_help(TypeError, parser.add_subparsers, help='%(prog)d') + + def test_invalid_subparser_help(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + subparsers = parser.add_subparsers() + self.assert_bad_help(ValueError, subparsers.add_parser, '1', + help='%Y-%m-%d') + self.assert_bad_help(KeyError, subparsers.add_parser, '1', + help='%(spam)s') + self.assert_bad_help(TypeError, subparsers.add_parser, '1', + help='%(prog)d') + def test_subparser_title_help(self): parser = ErrorRaisingArgumentParser(prog='PROG', description='main description') @@ -2548,10 +2980,43 @@ def test_interleaved_groups(self): result = parser.parse_args('1 2 3 4'.split()) self.assertEqual(expected, result) +class TestGroupConstructor(TestCase): + def test_group_prefix_chars(self): + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-+') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_group_prefix_chars_default(self): + # "default" isn't quite the right word here, but it's the same as + # the parser's default prefix so it's a good test + parser = ErrorRaisingArgumentParser() + msg = ( + "The use of the undocumented 'prefix_chars' parameter in " + "ArgumentParser.add_argument_group() is deprecated." + ) + with self.assertWarns(DeprecationWarning) as cm: + parser.add_argument_group(prefix_chars='-') + self.assertEqual(msg, str(cm.warning)) + self.assertEqual(cm.filename, __file__) + + def test_nested_argument_group(self): + parser = argparse.ArgumentParser() + g = parser.add_argument_group() + self.assertRaisesRegex(ValueError, + 'argument groups cannot be nested', + g.add_argument_group) + # =================== # Parent parser tests # =================== +@force_not_colorized_test_class class TestParentParsers(TestCase): """Tests that parsers can be created with parent parsers""" @@ -2584,8 +3049,6 @@ def setUp(self): group.add_argument('-a', action='store_true') group.add_argument('-b', action='store_true') - self.main_program = os.path.basename(sys.argv[0]) - def test_single_parent(self): parser = ErrorRaisingArgumentParser(parents=[self.wxyz_parent]) self.assertEqual(parser.parse_args('-y 1 2 --w 3'.split()), @@ -2596,7 +3059,7 @@ def test_single_parent_mutex(self): parser = ErrorRaisingArgumentParser(parents=[self.ab_mutex_parent]) self._test_mutex_ab(parser.parse_args) - def test_single_granparent_mutex(self): + def test_single_grandparent_mutex(self): parents = [self.ab_mutex_parent] parser = ErrorRaisingArgumentParser(add_help=False, parents=parents) parser = ErrorRaisingArgumentParser(parents=[parser]) @@ -2675,11 +3138,10 @@ def test_subparser_parents_mutex(self): def test_parent_help(self): parents = [self.abcd_parent, self.wxyz_parent] - parser = ErrorRaisingArgumentParser(parents=parents) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=parents) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-b B] [--d D] [--w W] [-y Y] a z + usage: PROG [-h] [-b B] [--d D] [--w W] [-y Y] a z positional arguments: a @@ -2695,7 +3157,7 @@ def test_parent_help(self): x: -y Y - '''.format(progname, ' ' if progname else '' ))) + ''')) def test_groups_parents(self): parent = ErrorRaisingArgumentParser(add_help=False) @@ -2705,15 +3167,14 @@ def test_groups_parents(self): m = parent.add_mutually_exclusive_group() m.add_argument('-y') m.add_argument('-z') - parser = ErrorRaisingArgumentParser(parents=[parent]) + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) self.assertRaises(ArgumentParserError, parser.parse_args, ['-y', 'Y', '-z', 'Z']) parser_help = parser.format_help() - progname = self.main_program self.assertEqual(parser_help, textwrap.dedent('''\ - usage: {}{}[-h] [-w W] [-x X] [-y Y | -z Z] + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] options: -h, --help show this help message and exit @@ -2725,12 +3186,45 @@ def test_groups_parents(self): -w W -x X - '''.format(progname, ' ' if progname else '' ))) + ''')) + + def test_wrong_type_parents(self): + self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) + + def test_mutex_groups_parents(self): + parent = ErrorRaisingArgumentParser(add_help=False) + g = parent.add_argument_group(title='g', description='gd') + g.add_argument('-w') + g.add_argument('-x') + m = g.add_mutually_exclusive_group() + m.add_argument('-y') + m.add_argument('-z') + parser = ErrorRaisingArgumentParser(prog='PROG', parents=[parent]) + + self.assertRaises(ArgumentParserError, parser.parse_args, + ['-y', 'Y', '-z', 'Z']) + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [-w W] [-x X] [-y Y | -z Z] + + options: + -h, --help show this help message and exit + + g: + gd + + -w W + -x X + -y Y + -z Z + ''')) # ============================== # Mutually exclusive group tests # ============================== +@force_not_colorized_test_class class TestMutuallyExclusiveGroupErrors(TestCase): def test_invalid_add_argument_group(self): @@ -2769,12 +3263,63 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) - def test_empty_group(self): + def test_optional_order(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--foo') + group.add_argument('bar', nargs='?') + expected = '''\ + usage: PROG [-h] (--foo FOO | bar) + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('bar', nargs='?') + group.add_argument('--foo') + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): + self.maxDiff = None + parser = ErrorRaisingArgumentParser(prog='PROG') + commands = parser.add_subparsers(title="commands", dest="command") + cmd_foo = commands.add_parser("foo") + group = cmd_foo.add_mutually_exclusive_group() + group.add_argument('--verbose', action='store_true', help=argparse.SUPPRESS) + group.add_argument('--quiet', action='store_true', help=argparse.SUPPRESS) + longopt = '--' + 'long'*32 + longmeta = 'LONG'*32 + cmd_foo.add_argument(longopt) + expected = f'''\ + usage: PROG foo [-h] + [{longopt} {longmeta}] + + options: + -h, --help show this help message and exit + {longopt} {longmeta} + ''' + self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) + + def test_usage_empty_group(self): # See issue 26952 - parser = argparse.ArgumentParser() + parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group() - with self.assertRaises(ValueError): - parser.parse_args(['-h']) + self.assertEqual(parser.format_usage(), 'usage: PROG [-h]\n') + + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + self.assertRaisesRegex(ValueError, + 'mutually exclusive groups cannot be nested', + g.add_mutually_exclusive_group) class MEMixin(object): @@ -2782,42 +3327,50 @@ def test_failures_when_not_required(self): parse_args = self.get_parser(required=False).parse_args error = ArgumentParserError for args_string in self.failures: - self.assertRaises(error, parse_args, args_string.split()) + with self.subTest(args=args_string): + self.assertRaises(error, parse_args, args_string.split()) def test_failures_when_required(self): parse_args = self.get_parser(required=True).parse_args error = ArgumentParserError for args_string in self.failures + ['']: - self.assertRaises(error, parse_args, args_string.split()) + with self.subTest(args=args_string): + self.assertRaises(error, parse_args, args_string.split()) def test_successes_when_not_required(self): parse_args = self.get_parser(required=False).parse_args successes = self.successes + self.successes_when_not_required for args_string, expected_ns in successes: - actual_ns = parse_args(args_string.split()) - self.assertEqual(actual_ns, expected_ns) + with self.subTest(args=args_string): + actual_ns = parse_args(args_string.split()) + self.assertEqual(actual_ns, expected_ns) def test_successes_when_required(self): parse_args = self.get_parser(required=True).parse_args for args_string, expected_ns in self.successes: - actual_ns = parse_args(args_string.split()) - self.assertEqual(actual_ns, expected_ns) + with self.subTest(args=args_string): + actual_ns = parse_args(args_string.split()) + self.assertEqual(actual_ns, expected_ns) + @force_not_colorized def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage expected_usage = self.usage_when_not_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_usage_when_required(self): format_usage = self.get_parser(required=True).format_usage expected_usage = self.usage_when_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_help_when_not_required(self): format_help = self.get_parser(required=False).format_help help = self.usage_when_not_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) + @force_not_colorized def test_help_when_required(self): format_help = self.get_parser(required=True).format_help help = self.usage_when_required + self.help @@ -2884,12 +3437,12 @@ def get_parser(self, required=None): ] usage_when_not_required = '''\ - usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] - [--klmno KLMNO | --pqrst PQRST] + usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] [--klmno KLMNO | + --pqrst PQRST] ''' usage_when_required = '''\ - usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] - (--klmno KLMNO | --pqrst PQRST) + usage: PROG [-h] [--abcde ABCDE] [--fghij FGHIJ] (--klmno KLMNO | + --pqrst PQRST) ''' help = '''\ @@ -2978,7 +3531,7 @@ def get_parser(self, required): group = parser.add_mutually_exclusive_group(required=required) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') - group.add_argument('badger', nargs='*', default='X', help='BADGER') + group.add_argument('badger', nargs='*', help='BADGER') return parser failures = [ @@ -2989,13 +3542,13 @@ def get_parser(self, required): '--foo X Y', ] successes = [ - ('--foo', NS(foo=True, spam=None, badger='X')), - ('--spam S', NS(foo=False, spam='S', badger='X')), + ('--foo', NS(foo=True, spam=None, badger=[])), + ('--spam S', NS(foo=False, spam='S', badger=[])), ('X', NS(foo=False, spam=None, badger=['X'])), ('X Y Z', NS(foo=False, spam=None, badger=['X', 'Y', 'Z'])), ] successes_when_not_required = [ - ('', NS(foo=False, spam=None, badger='X')), + ('', NS(foo=False, spam=None, badger=[])), ] usage_when_not_required = '''\ @@ -3026,25 +3579,29 @@ def get_parser(self, required): group.add_argument('-b', action='store_true', help='b help') parser.add_argument('-y', action='store_true', help='y help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['-a -b', '-b -c', '-a -c', '-a -b -c'] successes = [ - ('-a', NS(a=True, b=False, c=False, x=False, y=False)), - ('-b', NS(a=False, b=True, c=False, x=False, y=False)), - ('-c', NS(a=False, b=False, c=True, x=False, y=False)), - ('-a -x', NS(a=True, b=False, c=False, x=True, y=False)), - ('-y -b', NS(a=False, b=True, c=False, x=False, y=True)), - ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True)), + ('-a', NS(a=True, b=False, c=False, x=False, y=False, z=False)), + ('-b', NS(a=False, b=True, c=False, x=False, y=False, z=False)), + ('-c', NS(a=False, b=False, c=True, x=False, y=False, z=False)), + ('-a -x', NS(a=True, b=False, c=False, x=True, y=False, z=False)), + ('-y -b', NS(a=False, b=True, c=False, x=False, y=True, z=False)), + ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True, z=False)), ] successes_when_not_required = [ - ('', NS(a=False, b=False, c=False, x=False, y=False)), - ('-x', NS(a=False, b=False, c=False, x=True, y=False)), - ('-y', NS(a=False, b=False, c=False, x=False, y=True)), + ('', NS(a=False, b=False, c=False, x=False, y=False, z=False)), + ('-x', NS(a=False, b=False, c=False, x=True, y=False, z=False)), + ('-y', NS(a=False, b=False, c=False, x=False, y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-x] [-a] [-b] [-y] [-c] + usage_when_not_required = '''\ + usage: PROG [-h] [-x] [-a | -b | -c] [-y] [-z] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-x] (-a | -b | -c) [-y] [-z] ''' help = '''\ @@ -3055,6 +3612,7 @@ def get_parser(self, required): -b b help -y y help -c c help + -z z help ''' @@ -3108,23 +3666,27 @@ def get_parser(self, required): group.add_argument('a', nargs='?', help='a help') group.add_argument('-b', action='store_true', help='b help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['X A -b', '-b -c', '-c X A'] successes = [ - ('X A', NS(a='A', b=False, c=False, x='X', y=False)), - ('X -b', NS(a=None, b=True, c=False, x='X', y=False)), - ('X -c', NS(a=None, b=False, c=True, x='X', y=False)), - ('X A -y', NS(a='A', b=False, c=False, x='X', y=True)), - ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True)), + ('X A', NS(a='A', b=False, c=False, x='X', y=False, z=False)), + ('X -b', NS(a=None, b=True, c=False, x='X', y=False, z=False)), + ('X -c', NS(a=None, b=False, c=True, x='X', y=False, z=False)), + ('X A -y', NS(a='A', b=False, c=False, x='X', y=True, z=False)), + ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True, z=False)), ] successes_when_not_required = [ - ('X', NS(a=None, b=False, c=False, x='X', y=False)), - ('X -y', NS(a=None, b=False, c=False, x='X', y=True)), + ('X', NS(a=None, b=False, c=False, x='X', y=False, z=False)), + ('X -y', NS(a=None, b=False, c=False, x='X', y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-y] [-b] [-c] x [a] + usage_when_not_required = '''\ + usage: PROG [-h] [-y] [-z] x [-b | -c | a] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-y] [-z] x (-b | -c | a) ''' help = '''\ @@ -3137,56 +3699,113 @@ def get_parser(self, required): -y y help -b b help -c c help + -z z help ''' -class TestMutuallyExclusiveNested(MEMixin, TestCase): - # Nesting mutually exclusive groups is an undocumented feature - # that came about by accident through inheritance and has been - # the source of many bugs. It is deprecated and this test should - # eventually be removed along with it. - - def get_parser(self, required): +class TestMutuallyExclusiveOptionalOptional(MEMixin, TestCase): + def get_parser(self, required=None): parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group(required=required) - group.add_argument('-a') - group.add_argument('-b') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group2 = group.add_mutually_exclusive_group(required=required) - group2.add_argument('-c') - group2.add_argument('-d') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - group3 = group2.add_mutually_exclusive_group(required=required) - group3.add_argument('-e') - group3.add_argument('-f') + group.add_argument('--foo') + group.add_argument('--bar', nargs='?') return parser - usage_when_not_required = '''\ - usage: PROG [-h] [-a A | -b B | [-c C | -d D | [-e E | -f F]]] - ''' - usage_when_required = '''\ - usage: PROG [-h] (-a A | -b B | (-c C | -d D | (-e E | -f F))) - ''' - - help = '''\ - + failures = [ + '--foo X --bar Y', + '--foo X --bar', + ] + successes = [ + ('--foo X', NS(foo='X', bar=None)), + ('--bar X', NS(foo=None, bar='X')), + ('--bar', NS(foo=None, bar=None)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=None)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | --bar [BAR]) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | --bar [BAR]] + ''' + help = '''\ + + options: + -h, --help show this help message and exit + --foo FOO + --bar [BAR] + ''' + + +class TestMutuallyExclusiveOptionalWithDefault(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('--bar', type=bool, default=True) + return parser + + failures = [ + '--foo X --bar Y', + '--foo X --bar=', + ] + successes = [ + ('--foo X', NS(foo='X', bar=True)), + ('--bar X', NS(foo=None, bar=True)), + ('--bar=', NS(foo=None, bar=False)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=True)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | --bar BAR) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | --bar BAR] + ''' + help = '''\ + options: -h, --help show this help message and exit - -a A - -b B - -c C - -d D - -e E - -f F + --foo FOO + --bar BAR ''' - # We are only interested in testing the behavior of format_usage(). - test_failures_when_not_required = None - test_failures_when_required = None - test_successes_when_not_required = None - test_successes_when_required = None + +class TestMutuallyExclusivePositionalWithDefault(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('bar', nargs='?', type=bool, default=True) + return parser + + failures = [ + '--foo X Y', + ] + successes = [ + ('--foo X', NS(foo='X', bar=True)), + ('X', NS(foo=None, bar=True)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=True)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | bar) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | bar] + ''' + help = '''\ + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' # ================================================= # Mutually exclusive group in parent parser tests @@ -3433,11 +4052,13 @@ def _test(self, tester, parser_text): tester.maxDiff = None tester.assertEqual(expected_text, parser_text) + @force_not_colorized def test_format(self, tester): parser = self._get_parser(tester) format = getattr(parser, 'format_%s' % self.func_suffix) self._test(tester, format()) + @force_not_colorized def test_print(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -3450,6 +4071,7 @@ def test_print(self, tester): setattr(sys, self.std_name, old_stream) self._test(tester, parser_text) + @force_not_colorized def test_print_file(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -3855,7 +4477,7 @@ class TestHelpUsageWithParentheses(HelpTestCase): options: -h, --help show this help message and exit - -p {1 (option A), 2 (option B)}, --optional {1 (option A), 2 (option B)} + -p, --optional {1 (option A), 2 (option B)} ''' version = '' @@ -4139,6 +4761,159 @@ class TestHelpUsagePositionalsOnlyWrap(HelpTestCase): version = '' +class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): + # https://github.com/python/cpython/issues/62549 + # https://github.com/python/cpython/issues/89743 + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('-n1', metavar='()', help='n1'), + Sig('-o1', metavar='(1, 2)', help='o1'), + Sig('-u1', metavar=' (uu) ', help='u1'), + Sig('-v1', metavar='( vv )', help='v1'), + Sig('-w1', metavar='(w)w', help='w1'), + Sig('-x1', metavar='x(x)', help='x1'), + Sig('-y1', metavar='yy)', help='y1'), + Sig('-z1', metavar='(zz', help='z1'), + Sig('-n2', metavar='[]', help='n2'), + Sig('-o2', metavar='[1, 2]', help='o2'), + Sig('-u2', metavar=' [uu] ', help='u2'), + Sig('-v2', metavar='[ vv ]', help='v2'), + Sig('-w2', metavar='[w]w', help='w2'), + Sig('-x2', metavar='x[x]', help='x2'), + Sig('-y2', metavar='yy]', help='y2'), + Sig('-z2', metavar='[zz', help='z2'), + ] + + usage = '''\ + usage: PROG [-h] [-n1 ()] [-o1 (1, 2)] [-u1 (uu) ] [-v1 ( vv )] [-w1 (w)w] + [-x1 x(x)] [-y1 yy)] [-z1 (zz] [-n2 []] [-o2 [1, 2]] [-u2 [uu] ] + [-v2 [ vv ]] [-w2 [w]w] [-x2 x[x]] [-y2 yy]] [-z2 [zz] + ''' + help = usage + '''\ + + options: + -h, --help show this help message and exit + -n1 () n1 + -o1 (1, 2) o1 + -u1 (uu) u1 + -v1 ( vv ) v1 + -w1 (w)w w1 + -x1 x(x) x1 + -y1 yy) y1 + -z1 (zz z1 + -n2 [] n2 + -o2 [1, 2] o2 + -u2 [uu] u2 + -v2 [ vv ] v2 + -w2 [w]w w2 + -x2 x[x] x2 + -y2 yy] y2 + -z2 [zz z2 + ''' + version = '' + + +@force_not_colorized_test_class +class TestHelpUsageNoWhitespaceCrash(TestCase): + + def test_all_suppressed_mutex_followed_by_long_arg(self): + # https://github.com/python/cpython/issues/62090 + # https://github.com/python/cpython/issues/96310 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument('--spam', help=argparse.SUPPRESS) + parser.add_argument('--eggs-eggs-eggs-eggs-eggs-eggs') + usage = textwrap.dedent('''\ + usage: PROG [-h] + [--eggs-eggs-eggs-eggs-eggs-eggs EGGS_EGGS_EGGS_EGGS_EGGS_EGGS] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_newline_in_metavar(self): + # https://github.com/python/cpython/issues/77048 + mapping = ['123456', '12345', '12345', '123'] + parser = argparse.ArgumentParser('11111111111111') + parser.add_argument('-v', '--verbose', + help='verbose mode', action='store_true') + parser.add_argument('targets', + help='installation targets', + nargs='+', + metavar='\n'.join(mapping)) + usage = textwrap.dedent('''\ + usage: 11111111111111 [-h] [-v] + 123456 + 12345 + 12345 + 123 [123456 + 12345 + 12345 + 123 ...] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_empty_metavar_required_arg(self): + # https://github.com/python/cpython/issues/82091 + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('--nil', metavar='', required=True) + parser.add_argument('--a', metavar='A' * 70) + usage = ( + 'usage: PROG [-h] --nil \n' + ' [--a AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + 'AAAAAAAAAAAAAAAAAAAAAAA]\n' + ) + self.assertEqual(parser.format_usage(), usage) + + def test_all_suppressed_mutex_with_optional_nargs(self): + # https://github.com/python/cpython/issues/98666 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument( + '--param1', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + mutex.add_argument( + '--param2', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + usage = 'usage: PROG [-h]\n' + self.assertEqual(parser.format_usage(), usage) + + def test_long_mutex_groups_wrap(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('--op1', metavar='MET', nargs='?') + g.add_argument('--op2', metavar=('MET1', 'MET2'), nargs='*') + g.add_argument('--op3', nargs='*') + g.add_argument('--op4', metavar=('MET1', 'MET2'), nargs='+') + g.add_argument('--op5', nargs='+') + g.add_argument('--op6', nargs=3) + g.add_argument('--op7', metavar=('MET1', 'MET2', 'MET3'), nargs=3) + + usage = textwrap.dedent('''\ + usage: PROG [-h] [--op1 [MET] | --op2 [MET1 [MET2 ...]] | --op3 [OP3 ...] | + --op4 MET1 [MET2 ...] | --op5 OP5 [OP5 ...] | --op6 OP6 OP6 OP6 | + --op7 MET1 MET2 MET3] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_mutex_groups_with_mixed_optionals_positionals_wrap(self): + # https://github.com/python/cpython/issues/75949 + # Mutually exclusive groups containing both optionals and positionals + # should preserve pipe separators when the usage line wraps. + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('-v', '--verbose', action='store_true') + g.add_argument('-q', '--quiet', action='store_true') + g.add_argument('-x', '--extra-long-option-name', nargs='?') + g.add_argument('-y', '--yet-another-long-option', nargs='?') + g.add_argument('positional', nargs='?') + + usage = textwrap.dedent('''\ + usage: PROG [-h] + [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | + -y [YET_ANOTHER_LONG_OPTION] | positional] + ''') + self.assertEqual(parser.format_usage(), usage) + + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" @@ -4148,7 +4923,7 @@ class TestHelpVariableExpansion(HelpTestCase): help='x %(prog)s %(default)s %(type)s %%'), Sig('-y', action='store_const', default=42, const='XXX', help='y %(prog)s %(default)s %(const)s'), - Sig('--foo', choices='abc', + Sig('--foo', choices=['a', 'b', 'c'], help='foo %(prog)s %(default)s %(choices)s'), Sig('--bar', default='baz', choices=[1, 2], metavar='BBB', help='bar %(prog)s %(default)s %(dest)s'), @@ -4338,8 +5113,8 @@ class TestHelpAlternatePrefixChars(HelpTestCase): help = usage + '''\ options: - ^^foo foo help - ;b BAR, ;;bar BAR bar help + ^^foo foo help + ;b, ;;bar BAR bar help ''' version = '' @@ -4391,7 +5166,7 @@ class TestHelpNone(HelpTestCase): version = '' -class TestHelpTupleMetavar(HelpTestCase): +class TestHelpTupleMetavarOptional(HelpTestCase): """Test specifying metavar as a tuple""" parser_signature = Sig(prog='PROG') @@ -4418,6 +5193,34 @@ class TestHelpTupleMetavar(HelpTestCase): version = '' +class TestHelpTupleMetavarPositional(HelpTestCase): + """Test specifying metavar on a Positional as a tuple""" + + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('w', help='w help', nargs='+', metavar=('W1', 'W2')), + Sig('x', help='x help', nargs='*', metavar=('X1', 'X2')), + Sig('y', help='y help', nargs=3, metavar=('Y1', 'Y2', 'Y3')), + Sig('z', help='z help', nargs='?', metavar=('Z1',)), + ] + argument_group_signatures = [] + usage = '''\ + usage: PROG [-h] W1 [W2 ...] [X1 [X2 ...]] Y1 Y2 Y3 [Z1] + ''' + help = usage + '''\ + + positional arguments: + W1 W2 w help + X1 X2 x help + Y1 Y2 Y3 y help + Z1 z help + + options: + -h, --help show this help message and exit + ''' + version = '' + + class TestHelpRawText(HelpTestCase): """Test the RawTextHelpFormatter""" @@ -4525,6 +5328,7 @@ class TestHelpArgumentDefaults(HelpTestCase): argument_signatures = [ Sig('--foo', help='foo help - oh and by the way, %(default)s'), Sig('--bar', action='store_true', help='bar help'), + Sig('--required', required=True, help='some help'), Sig('--taz', action=argparse.BooleanOptionalAction, help='Whether to taz it', default=True), Sig('--corge', action=argparse.BooleanOptionalAction, @@ -4538,8 +5342,8 @@ class TestHelpArgumentDefaults(HelpTestCase): [Sig('--baz', type=int, default=42, help='baz help')]), ] usage = '''\ - usage: PROG [-h] [--foo FOO] [--bar] [--taz | --no-taz] [--corge | --no-corge] - [--quux QUUX] [--baz BAZ] + usage: PROG [-h] [--foo FOO] [--bar] --required REQUIRED [--taz | --no-taz] + [--corge | --no-corge] [--quux QUUX] [--baz BAZ] spam [badger] ''' help = usage + '''\ @@ -4554,6 +5358,7 @@ class TestHelpArgumentDefaults(HelpTestCase): -h, --help show this help message and exit --foo FOO foo help - oh and by the way, None --bar bar help (default: False) + --required REQUIRED some help --taz, --no-taz Whether to taz it (default: True) --corge, --no-corge Whether to corge it --quux QUUX Set the quux (default: 42) @@ -4711,6 +5516,96 @@ def custom_type(string): version = '' +@force_not_colorized_test_class +class TestHelpCustomHelpFormatter(TestCase): + maxDiff = None + + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=custom_formatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=CustomFormatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_usage_long_subparser_command(self): + """Test that subparser commands are formatted correctly in help""" + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, max_help_position=50) + + parent_parser = argparse.ArgumentParser( + prog='PROG', + formatter_class=custom_formatter + ) + + cmd_subparsers = parent_parser.add_subparsers(title="commands", + metavar='CMD', + help='command to use') + cmd_subparsers.add_parser("add", + help="add something") + + cmd_subparsers.add_parser("remove", + help="remove something") + + cmd_subparsers.add_parser("a-very-long-command", + help="command that does something") + + parser_help = parent_parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] CMD ... + + options: + -h, --help show this help message and exit + + commands: + CMD command to use + add add something + remove remove something + a-very-long-command command that does something + ''')) + + # ===================================== # Optional/Positional constructor tests # ===================================== @@ -4718,15 +5613,15 @@ def custom_type(string): class TestInvalidArgumentConstructors(TestCase): """Test a bunch of invalid Argument constructors""" - def assertTypeError(self, *args, **kwargs): + def assertTypeError(self, *args, errmsg=None, **kwargs): parser = argparse.ArgumentParser() - self.assertRaises(TypeError, parser.add_argument, - *args, **kwargs) + self.assertRaisesRegex(TypeError, errmsg, parser.add_argument, + *args, **kwargs) - def assertValueError(self, *args, **kwargs): + def assertValueError(self, *args, errmsg=None, **kwargs): parser = argparse.ArgumentParser() - self.assertRaises(ValueError, parser.add_argument, - *args, **kwargs) + self.assertRaisesRegex(ValueError, errmsg, parser.add_argument, + *args, **kwargs) def test_invalid_keyword_arguments(self): self.assertTypeError('-x', bar=None) @@ -4736,70 +5631,99 @@ def test_invalid_keyword_arguments(self): def test_missing_destination(self): self.assertTypeError() - for action in ['append', 'store']: - self.assertTypeError(action=action) + for action in ['store', 'append', 'extend']: + with self.subTest(action=action): + self.assertTypeError(action=action) def test_invalid_option_strings(self): - self.assertValueError('--') - self.assertValueError('---') + self.assertTypeError('-', errmsg='dest= is required') + self.assertTypeError('--', errmsg='dest= is required') + self.assertTypeError('---', errmsg='dest= is required') + + def test_invalid_prefix(self): + self.assertValueError('--foo', '+foo', + errmsg='must start with a character') def test_invalid_type(self): - self.assertValueError('--foo', type='int') - self.assertValueError('--foo', type=(int, float)) + self.assertTypeError('--foo', type='int', + errmsg="'int' is not callable") + self.assertTypeError('--foo', type=(int, float), + errmsg='is not callable') def test_invalid_action(self): - self.assertValueError('-x', action='foo') - self.assertValueError('foo', action='baz') - self.assertValueError('--foo', action=('store', 'append')) - parser = argparse.ArgumentParser() - with self.assertRaises(ValueError) as cm: - parser.add_argument("--foo", action="store-true") - self.assertIn('unknown action', str(cm.exception)) + self.assertValueError('-x', action='foo', + errmsg='unknown action') + self.assertValueError('foo', action='baz', + errmsg='unknown action') + self.assertValueError('--foo', action=('store', 'append'), + errmsg='unknown action') + self.assertValueError('--foo', action="store-true", + errmsg='unknown action') + + def test_invalid_help(self): + self.assertValueError('--foo', help='%Y-%m-%d', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(spam)s', + errmsg='badly formed help string') + self.assertValueError('--foo', help='%(prog)d', + errmsg='badly formed help string') def test_multiple_dest(self): parser = argparse.ArgumentParser() parser.add_argument(dest='foo') - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError) as cm: parser.add_argument('bar', dest='baz') - self.assertIn('dest supplied twice for positional argument', + self.assertIn('dest supplied twice for positional argument,' + ' did you mean metavar?', str(cm.exception)) def test_no_argument_actions(self): for action in ['store_const', 'store_true', 'store_false', 'append_const', 'count']: - for attrs in [dict(type=int), dict(nargs='+'), - dict(choices='ab')]: - self.assertTypeError('-x', action=action, **attrs) + with self.subTest(action=action): + for attrs in [dict(type=int), dict(nargs='+'), + dict(choices=['a', 'b'])]: + with self.subTest(attrs=attrs): + self.assertTypeError('-x', action=action, **attrs) + self.assertTypeError('x', action=action, **attrs) + self.assertValueError('x', action=action, + errmsg=f"action '{action}' is not valid for positional arguments") + self.assertTypeError('-x', action=action, nargs=0) + self.assertValueError('x', action=action, nargs=0, + errmsg='nargs for positionals must be != 0') def test_no_argument_no_const_actions(self): # options with zero arguments for action in ['store_true', 'store_false', 'count']: + with self.subTest(action=action): + # const is always disallowed + self.assertTypeError('-x', const='foo', action=action) - # const is always disallowed - self.assertTypeError('-x', const='foo', action=action) - - # nargs is always disallowed - self.assertTypeError('-x', nargs='*', action=action) + # nargs is always disallowed + self.assertTypeError('-x', nargs='*', action=action) def test_more_than_one_argument_actions(self): - for action in ['store', 'append']: - - # nargs=0 is disallowed - self.assertValueError('-x', nargs=0, action=action) - self.assertValueError('spam', nargs=0, action=action) - - # const is disallowed with non-optional arguments - for nargs in [1, '*', '+']: - self.assertValueError('-x', const='foo', - nargs=nargs, action=action) - self.assertValueError('spam', const='foo', - nargs=nargs, action=action) + for action in ['store', 'append', 'extend']: + with self.subTest(action=action): + # nargs=0 is disallowed + action_name = 'append' if action == 'extend' else action + self.assertValueError('-x', nargs=0, action=action, + errmsg=f'nargs for {action_name} actions must be != 0') + self.assertValueError('spam', nargs=0, action=action, + errmsg='nargs for positionals must be != 0') + + # const is disallowed with non-optional arguments + for nargs in [1, '*', '+']: + self.assertValueError('-x', const='foo', + nargs=nargs, action=action) + self.assertValueError('spam', const='foo', + nargs=nargs, action=action) def test_required_const_actions(self): for action in ['store_const', 'append_const']: - - # nargs is always disallowed - self.assertTypeError('-x', nargs='+', action=action) + with self.subTest(action=action): + # nargs is always disallowed + self.assertTypeError('-x', nargs='+', action=action) def test_parsers_action_missing_params(self): self.assertTypeError('command', action='parsers') @@ -4807,6 +5731,9 @@ def test_parsers_action_missing_params(self): self.assertTypeError('command', action='parsers', parser_class=argparse.ArgumentParser) + def test_version_missing_params(self): + self.assertTypeError('command', action='version') + def test_required_positional(self): self.assertTypeError('foo', required=True) @@ -4886,6 +5813,7 @@ def test_conflict_error(self): self.assertRaises(argparse.ArgumentError, parser.add_argument, '--spam') + @force_not_colorized def test_resolve_error(self): get_parser = argparse.ArgumentParser parser = get_parser(prog='PROG', conflict_handler='resolve') @@ -4915,20 +5843,25 @@ def test_subparser_conflict(self): parser = argparse.ArgumentParser() sp = parser.add_subparsers() sp.add_parser('fullname', aliases=['alias']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'fullname') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'alias') - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['fullname']) - self.assertRaises(argparse.ArgumentError, - sp.add_parser, 'other', aliases=['alias']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser: fullname', + sp.add_parser, 'fullname') + self.assertRaisesRegex(ValueError, + 'conflicting subparser: alias', + sp.add_parser, 'alias') + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: fullname', + sp.add_parser, 'other', aliases=['fullname']) + self.assertRaisesRegex(ValueError, + 'conflicting subparser alias: alias', + sp.add_parser, 'other', aliases=['alias']) # ============================= # Help and Version option tests # ============================= +@force_not_colorized_test_class class TestOptionalsHelpVersionActions(TestCase): """Test the help and version actions""" @@ -5026,7 +5959,8 @@ def test_optional(self): string = ( "Action(option_strings=['--foo', '-a', '-b'], dest='b', " "nargs='+', const=None, default=42, type='int', " - "choices=[1, 2, 3], required=False, help='HELP', metavar='METAVAR')") + "choices=[1, 2, 3], required=False, help='HELP', " + "metavar='METAVAR', deprecated=False)") self.assertStringEqual(option, string) def test_argument(self): @@ -5043,7 +5977,8 @@ def test_argument(self): string = ( "Action(option_strings=[], dest='x', nargs='?', " "const=None, default=2.5, type=%r, choices=[0.5, 1.5, 2.5], " - "required=True, help='H HH H', metavar='MV MV MV')" % float) + "required=True, help='H HH H', metavar='MV MV MV', " + "deprecated=False)" % float) self.assertStringEqual(argument, string) def test_namespace(self): @@ -5146,6 +6081,7 @@ def test_argument_error(self): class TestArgumentTypeError(TestCase): + @force_not_colorized def test_argument_type_error(self): def spam(string): @@ -5235,6 +6171,139 @@ def spam(string_to_convert): args = parser.parse_args('--foo spam!'.split()) self.assertEqual(NS(foo='foo_converted'), args) + +# ============================================== +# Check that deprecated arguments output warning +# ============================================== + +class TestDeprecatedArguments(TestCase): + + def test_deprecated_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f', 'spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '-f', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', 'spam', '--foo', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_boolean_option(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--foo', action=argparse.BooleanOptionalAction, deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args(['--foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['-f']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '-f' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['--foo', '--no-foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: option '--foo' is deprecated") + self.assertRegex(stderr, "warning: option '--no-foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_arguments(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='?', deprecated=True) + parser.add_argument('bar', nargs='?', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertRegex(stderr, "warning: argument 'bar' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 2) + + def test_deprecated_varargument(self): + parser = argparse.ArgumentParser() + parser.add_argument('foo', nargs='*', deprecated=True) + + with captured_stderr() as stderr: + parser.parse_args([]) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['spam']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['spam', 'ham']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: argument 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + def test_deprecated_subparser(self): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + subparsers.add_parser('foo', aliases=['baz'], deprecated=True) + subparsers.add_parser('bar') + + with captured_stderr() as stderr: + parser.parse_args(['bar']) + stderr = stderr.getvalue() + self.assertEqual(stderr.count('is deprecated'), 0) + + with captured_stderr() as stderr: + parser.parse_args(['foo']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'foo' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + with captured_stderr() as stderr: + parser.parse_args(['baz']) + stderr = stderr.getvalue() + self.assertRegex(stderr, "warning: command 'baz' is deprecated") + self.assertEqual(stderr.count('is deprecated'), 1) + + # ================================================================== # Check semantics regarding the default argument and type conversion # ================================================================== @@ -5333,6 +6402,133 @@ def test_zero_or_more_optional(self): self.assertEqual(NS(x=[]), args) +class TestDoubleDash(TestCase): + def test_single_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('-f', '--foo') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo='--', bar=[]), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument -f/--foo: expected one argument', + parser.parse_args, ['--foo', '--']) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo='--', bar=[]), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument -f/--foo: expected one argument', + parser.parse_args, ['-f', '--']) + args = parser.parse_args(['--foo', 'a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--foo', 'c']) + self.assertEqual(NS(foo='c', bar=['a', 'b']), args) + args = parser.parse_args(['a', '--', 'b', '--foo', 'c']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--foo', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c', '--foo', 'd']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--', 'c', '--foo', 'd']), args) + + def test_multiple_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('-f', '--foo', nargs='*') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo=--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['--foo', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['-f--']) + self.assertEqual(NS(foo=['--'], bar=[]), args) + args = parser.parse_args(['-f', '--']) + self.assertEqual(NS(foo=[], bar=[]), args) + args = parser.parse_args(['--foo', 'a', 'b', '--', 'c', 'd']) + self.assertEqual(NS(foo=['a', 'b'], bar=['c', 'd']), args) + args = parser.parse_args(['a', 'b', '--foo', 'c', 'd']) + self.assertEqual(NS(foo=['c', 'd'], bar=['a', 'b']), args) + args = parser.parse_args(['a', '--', 'b', '--foo', 'c', 'd']) + self.assertEqual(NS(foo=None, bar=['a', 'b', '--foo', 'c', 'd']), args) + args, argv = parser.parse_known_args(['a', 'b', '--foo', 'c', '--', 'd']) + self.assertEqual(NS(foo=['c'], bar=['a', 'b']), args) + self.assertEqual(argv, ['--', 'd']) + + def test_multiple_double_dashes(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + parser.add_argument('bar', nargs='*') + + args = parser.parse_args(['--', 'a', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + args = parser.parse_args(['--', '--', 'a', '--', 'b', 'c']) + self.assertEqual(NS(foo='--', bar=['a', '--', 'b', 'c']), args) + + def test_remainder(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + parser.add_argument('bar', nargs='...') + + args = parser.parse_args(['--', 'a', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', '--', 'b', 'c']) + self.assertEqual(NS(foo='a', bar=['b', 'c']), args) + args = parser.parse_args(['a', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + args = parser.parse_args(['a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['b', '--', 'c']), args) + + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo') + parser.add_argument('bar', nargs='...') + args = parser.parse_args(['--foo', 'a', '--', 'b', '--', 'c']) + self.assertEqual(NS(foo='a', bar=['--', 'b', '--', 'c']), args) + + def test_subparser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('foo') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('run') + parser1.add_argument('-f') + parser1.add_argument('bar', nargs='*') + + args = parser.parse_args(['x', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['x', 'run', 'a', 'b', '--', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', 'run', 'a', '--', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', 'run', '--', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f=None, bar=['a', 'b', '-f', 'c']), args) + args = parser.parse_args(['x', '--', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['--', 'x', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo='x', f='c', bar=['a', 'b']), args) + args = parser.parse_args(['x', 'run', '--', 'a', '--', 'b']) + self.assertEqual(NS(foo='x', f=None, bar=['a', '--', 'b']), args) + args = parser.parse_args(['x', '--', 'run', '--', 'a', '--', 'b']) + self.assertEqual(NS(foo='x', f=None, bar=['a', '--', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + "invalid choice: '--'", + parser.parse_args, ['--', 'x', '--', 'run', 'a', 'b']) + + def test_subparser_after_multiple_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', nargs='*') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('run') + parser1.add_argument('-f') + parser1.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo', 'x', 'y', '--', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo=['x', 'y'], f='c', bar=['a', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + "invalid choice: '--'", + parser.parse_args, ['--foo', 'x', '--', '--', 'run', 'a', 'b']) + + # =========================== # parse_intermixed_args tests # =========================== @@ -5352,14 +6548,25 @@ def test_basic(self): args, extras = parser.parse_known_args(argv) # cannot parse the '1,2,3' - self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[]), args) - self.assertEqual(["1", "2", "3"], extras) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args) + self.assertEqual(["2", "3"], extras) + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual([], extras) + # unknown optionals go into extras + argv = 'cmd --foo x --error 1 2 --bar y 3'.split() + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) argv = 'cmd --foo x 1 --error 2 --bar y 3'.split() args, extras = parser.parse_known_intermixed_args(argv) - # unknown optionals go into extras - self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args) - self.assertEqual(['--error', '2', '3'], extras) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) + argv = 'cmd --foo x 1 2 --error --bar y 3'.split() + args, extras = parser.parse_known_intermixed_args(argv) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1, 2, 3]), args) + self.assertEqual(['--error'], extras) # restores attributes that were temporarily changed self.assertIsNone(parser.usage) @@ -5378,28 +6585,49 @@ def test_remainder(self): parser.parse_intermixed_args(argv) self.assertRegex(str(cm.exception), r'\.\.\.') - def test_exclusive(self): - # mutually exclusive group; intermixed works fine - parser = ErrorRaisingArgumentParser(prog='PROG') + def test_required_exclusive(self): + # required mutually exclusive group; intermixed works fine + parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') parser.add_argument('badger', nargs='*', default='X', help='BADGER') + args = parser.parse_intermixed_args('--foo 1 2'.split()) + self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args) args = parser.parse_intermixed_args('1 --foo 2'.split()) self.assertEqual(NS(badger=['1', '2'], foo=True, spam=None), args) - self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, '1 2'.split()) + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --foo --spam is required', + parser.parse_intermixed_args, '1 2'.split()) self.assertEqual(group.required, True) - def test_exclusive_incompatible(self): - # mutually exclusive group including positional - fail - parser = ErrorRaisingArgumentParser(prog='PROG') + def test_required_exclusive_with_positional(self): + # required mutually exclusive group with positional argument + parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') group.add_argument('badger', nargs='*', default='X', help='BADGER') - self.assertRaises(TypeError, parser.parse_intermixed_args, []) + args = parser.parse_intermixed_args(['--foo']) + self.assertEqual(NS(foo=True, spam=None, badger='X'), args) + args = parser.parse_intermixed_args(['a', 'b']) + self.assertEqual(NS(foo=False, spam=None, badger=['a', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --foo --spam badger is required', + parser.parse_intermixed_args, []) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument badger: not allowed with argument --foo', + parser.parse_intermixed_args, ['--foo', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument badger: not allowed with argument --foo', + parser.parse_intermixed_args, ['a', '--foo', 'b']) self.assertEqual(group.required, True) + def test_invalid_args(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, ['a']) + + class TestIntermixedMessageContentError(TestCase): # case where Intermixed gives different error message # error is raised by 1st parsing step @@ -5417,7 +6645,7 @@ def test_missing_argument_name_in_message(self): with self.assertRaises(ArgumentParserError) as cm: parser.parse_intermixed_args([]) msg = str(cm.exception) - self.assertNotRegex(msg, 'req_pos') + self.assertRegex(msg, 'req_pos') self.assertRegex(msg, 'req_opt') # ========================== @@ -5628,7 +6856,7 @@ class TestImportStar(TestCase): def test(self): for name in argparse.__all__: - self.assertTrue(hasattr(argparse, name)) + self.assertHasAttr(argparse, name) def test_all_exports_everything_but_modules(self): items = [ @@ -5652,6 +6880,7 @@ def setUp(self): metavar = '' self.parser.add_argument('--proxy', metavar=metavar) + @force_not_colorized def test_help_with_metavar(self): help_text = self.parser.format_help() self.assertEqual(help_text, textwrap.dedent('''\ @@ -5667,7 +6896,8 @@ def test_help_with_metavar(self): class TestExitOnError(TestCase): def setUp(self): - self.parser = argparse.ArgumentParser(exit_on_error=False) + self.parser = argparse.ArgumentParser(exit_on_error=False, + fromfile_prefix_chars='@') self.parser.add_argument('--integers', metavar='N', type=int) def test_exit_on_error_with_good_args(self): @@ -5678,6 +6908,518 @@ def test_exit_on_error_with_bad_args(self): with self.assertRaises(argparse.ArgumentError): self.parser.parse_args('--integers a'.split()) + def test_unrecognized_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_args, '--foo bar'.split()) + + def test_unrecognized_intermixed_args(self): + self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --foo bar', + self.parser.parse_intermixed_args, '--foo bar'.split()) + + def test_required_args(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', metavar='BaZ') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, BaZ$', + self.parser.parse_args, []) + + def test_required_args_n(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs=3) + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_n_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs=3, metavar=('B', 'A', 'Z')) + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, B, A, Z$', + self.parser.parse_args, []) + + def test_required_args_optional(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='?') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_args_zero_or_more(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='*') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_args_one_or_more(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='+') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_one_or_more_with_metavar(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='+', metavar=('BaZ1', 'BaZ2')) + self.assertRaisesRegex(argparse.ArgumentError, + r'the following arguments are required: bar, BaZ1\[, BaZ2]$', + self.parser.parse_args, []) + + def test_required_args_remainder(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='...') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_mutually_exclusive_args(self): + group = self.parser.add_mutually_exclusive_group(required=True) + group.add_argument('--bar') + group.add_argument('--baz') + self.assertRaisesRegex(argparse.ArgumentError, + 'one of the arguments --bar --baz is required', + self.parser.parse_args, []) + + def test_conflicting_mutually_exclusive_args_optional_with_metavar(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='?', metavar='BaZ') + self.assertRaisesRegex(argparse.ArgumentError, + 'argument BaZ: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument --bar: not allowed with argument BaZ$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar1(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='*', metavar=('BAZ1',)) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument BAZ1: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + 'argument --bar: not allowed with argument BAZ1$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_conflicting_mutually_exclusive_args_zero_or_more_with_metavar2(self): + group = self.parser.add_mutually_exclusive_group() + group.add_argument('--bar') + group.add_argument('baz', nargs='*', metavar=('BAZ1', 'BAZ2')) + self.assertRaisesRegex(argparse.ArgumentError, + r'argument BAZ1\[, BAZ2]: not allowed with argument --bar$', + self.parser.parse_args, ['--bar', 'a', 'b']) + self.assertRaisesRegex(argparse.ArgumentError, + r'argument --bar: not allowed with argument BAZ1\[, BAZ2]$', + self.parser.parse_args, ['a', '--bar', 'b']) + + def test_ambiguous_option(self): + self.parser.add_argument('--foobaz') + self.parser.add_argument('--fooble', action='store_true') + self.parser.add_argument('--foogle') + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble", + self.parser.parse_args, ['--foob']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob=1 could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob=1']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob', '1', '--foogle', '2']) + self.assertRaisesRegex(argparse.ArgumentError, + "ambiguous option: --foob=1 could match --foobaz, --fooble$", + self.parser.parse_args, ['--foob=1', '--foogle', '2']) + + def test_os_error(self): + self.parser.add_argument('file') + self.assertRaisesRegex(argparse.ArgumentError, + "No such file or directory: 'no-such-file'", + self.parser.parse_args, ['@no-such-file']) + + +@force_not_colorized_test_class +class TestProgName(TestCase): + source = textwrap.dedent('''\ + import argparse + parser = argparse.ArgumentParser() + parser.parse_args() + ''') + + def setUp(self): + self.dirname = 'package' + os_helper.FS_NONASCII + self.addCleanup(os_helper.rmtree, self.dirname) + os.mkdir(self.dirname) + + def make_script(self, dirname, basename, *, compiled=False): + script_name = script_helper.make_script(dirname, basename, self.source) + if not compiled: + return script_name + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + return pyc_file + + def make_zip_script(self, script_name, name_in_zip=None): + zip_name, _ = script_helper.make_zip_script(self.dirname, 'test_zip', + script_name, name_in_zip) + return zip_name + + def check_usage(self, expected, *args, **kwargs): + res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) + self.assertEqual(os.fsdecode(res.out.splitlines()[0]), + f'usage: {expected} [-h]') + + def test_script(self, compiled=False): + basename = os_helper.TESTFN + script_name = self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(os.path.basename(script_name), script_name, '-h') + + def test_script_compiled(self): + self.test_script(compiled=True) + + def test_directory(self, compiled=False): + dirname = os.path.join(self.dirname, os_helper.TESTFN) + os.mkdir(dirname) + self.make_script(dirname, '__main__', compiled=compiled) + self.check_usage(f'{py} {dirname}', dirname) + dirname2 = os.path.join(os.curdir, dirname) + self.check_usage(f'{py} {dirname2}', dirname2) + + def test_directory_compiled(self): + self.test_directory(compiled=True) + + def test_module(self, compiled=False): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename, compiled=compiled) + self.check_usage(f'{py} -m {modulename}', + '-m', modulename, PYTHONPATH=os.curdir) + + def test_module_compiled(self): + self.test_module(compiled=True) + + def test_package(self, compiled=False): + basename = 'subpackage' + os_helper.FS_NONASCII + packagename = f'{self.dirname}.{basename}' + subdirname = os.path.join(self.dirname, basename) + os.mkdir(subdirname) + self.make_script(subdirname, '__main__', compiled=compiled) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename, PYTHONPATH=os.curdir) + self.check_usage(f'{py} -m {packagename}', + '-m', packagename + '.__main__', PYTHONPATH=os.curdir) + + def test_package_compiled(self): + self.test_package(compiled=True) + + def test_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + zip_name = self.make_zip_script(script_name) + self.check_usage(f'{py} {zip_name}', zip_name) + + def test_zipfile_compiled(self): + self.test_zipfile(compiled=True) + + def test_directory_in_zipfile(self, compiled=False): + script_name = self.make_script(self.dirname, '__main__', compiled=compiled) + name_in_zip = 'package/subpackage/__main__' + ('.py', '.pyc')[compiled] + zip_name = self.make_zip_script(script_name, name_in_zip) + dirname = os.path.join(zip_name, 'package', 'subpackage') + self.check_usage(f'{py} {dirname}', dirname) + + def test_directory_in_zipfile_compiled(self): + self.test_directory_in_zipfile(compiled=True) + +# ================= +# Translation tests +# ================= + +class TestTranslations(TestTranslationsBase): + + def test_translations(self): + self.assertMsgidsEqual(argparse) + + +# =========== +# Color tests +# =========== + + +class TestColorized(TestCase): + maxDiff = None + + def setUp(self): + super().setUp() + # Ensure color even if ran with NO_COLOR=1 + self.enterContext(swap_attr(_colorize, 'can_colorize', + lambda *args, **kwargs: True)) + self.theme = _colorize.get_theme(force_color=True).argparse + + def test_argparse_color(self): + # Arrange: create a parser with a bit of everything + parser = argparse.ArgumentParser( + color=True, + description="Colorful help", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prefix_chars="-+", + prog="PROG", + ) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-v", "--verbose", action="store_true", help="more spam" + ) + group.add_argument( + "-q", "--quiet", action="store_true", help="less spam" + ) + parser.add_argument("x", type=int, help="the base") + parser.add_argument( + "y", type=int, help="the exponent", deprecated=True + ) + parser.add_argument( + "this_indeed_is_a_very_long_action_name", + type=int, + help="the exponent", + ) + parser.add_argument( + "-o", "--optional1", action="store_true", deprecated=True + ) + parser.add_argument("--optional2", help="pick one") + parser.add_argument("--optional3", choices=("X", "Y", "Z")) + parser.add_argument( + "--optional4", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "--optional5", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "--optional6", choices=("X", "Y", "Z"), help="pick one" + ) + parser.add_argument( + "-p", + "--optional7", + choices=("Aaaaa", "Bbbbb", "Ccccc", "Ddddd"), + help="pick one", + ) + + parser.add_argument("+f") + parser.add_argument("++bar") + parser.add_argument("-+baz") + parser.add_argument("-c", "--count") + + subparsers = parser.add_subparsers( + title="subcommands", + description="valid subcommands", + help="additional help", + ) + subparsers.add_parser("sub1", deprecated=True, help="sub1 help") + sub2 = subparsers.add_parser("sub2", deprecated=True, help="sub2 help") + sub2.add_argument("--baz", choices=("X", "Y", "Z"), help="baz help") + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + # Act + help_text = parser.format_help() + + # Assert + self.assertEqual( + help_text, + textwrap.dedent( + f"""\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}-v{reset} | {short}-q{reset}] [{short}-o{reset}] [{long}--optional2 {label}OPTIONAL2{reset}] [{long}--optional3 {label}{{X,Y,Z}}{reset}] + [{long}--optional4 {label}{{X,Y,Z}}{reset}] [{long}--optional5 {label}{{X,Y,Z}}{reset}] [{long}--optional6 {label}{{X,Y,Z}}{reset}] + [{short}-p {label}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset}] [{short}+f {label}F{reset}] [{long}++bar {label}BAR{reset}] [{long}-+baz {label}BAZ{reset}] + [{short}-c {label}COUNT{reset}] + {pos}x{reset} {pos}y{reset} {pos}this_indeed_is_a_very_long_action_name{reset} {pos}{{sub1,sub2}} ...{reset} + + Colorful help + + {heading}positional arguments:{reset} + {pos_b}x{reset} the base + {pos_b}y{reset} the exponent + {pos_b}this_indeed_is_a_very_long_action_name{reset} + the exponent + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}-v{reset}, {long_b}--verbose{reset} more spam (default: False) + {short_b}-q{reset}, {long_b}--quiet{reset} less spam (default: False) + {short_b}-o{reset}, {long_b}--optional1{reset} + {long_b}--optional2{reset} {label_b}OPTIONAL2{reset} + pick one (default: None) + {long_b}--optional3{reset} {label_b}{{X,Y,Z}}{reset} + {long_b}--optional4{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional5{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {long_b}--optional6{reset} {label_b}{{X,Y,Z}}{reset} pick one (default: None) + {short_b}-p{reset}, {long_b}--optional7{reset} {label_b}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset} + pick one (default: None) + {short_b}+f{reset} {label_b}F{reset} + {long_b}++bar{reset} {label_b}BAR{reset} + {long_b}-+baz{reset} {label_b}BAZ{reset} + {short_b}-c{reset}, {long_b}--count{reset} {label_b}COUNT{reset} + + {heading}subcommands:{reset} + valid subcommands + + {pos_b}{{sub1,sub2}}{reset} additional help + {pos_b}sub1{reset} sub1 help + {pos_b}sub2{reset} sub2 help + """ + ), + ) + + def test_argparse_color_mutually_exclusive_group_usage(self): + parser = argparse.ArgumentParser(color=True, prog="PROG") + group = parser.add_mutually_exclusive_group() + group.add_argument('--foo', action='store_true', help='FOO') + group.add_argument('--spam', help='SPAM') + group.add_argument('badger', nargs='*', help='BADGER') + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + reset = self.theme.reset + + self.assertEqual(parser.format_usage(), + f"{heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] " + f"[{long}--foo{reset} | " + f"{long}--spam {label}SPAM{reset} | " + f"{pos}badger ...{reset}]\n") + + def test_argparse_color_custom_usage(self): + # Arrange + parser = argparse.ArgumentParser( + add_help=False, + color=True, + description="Test prog and usage colors", + prog="PROG", + usage="[prefix] %(prog)s [suffix]", + ) + heading = self.theme.heading + prog = self.theme.prog + reset = self.theme.reset + usage = self.theme.prog_extra + + # Act + help_text = parser.format_help() + + # Assert + self.assertEqual( + help_text, + textwrap.dedent( + f"""\ + {heading}usage: {reset}{usage}[prefix] {prog}PROG{reset}{usage} [suffix]{reset} + + Test prog and usage colors + """ + ), + ) + + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=custom_formatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=CustomFormatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_subparser_prog_is_stored_without_color(self): + parser = argparse.ArgumentParser(prog='complex', color=True) + sub = parser.add_subparsers(dest='command') + demo_parser = sub.add_parser('demo') + + self.assertNotIn('\x1b[', demo_parser.prog) + + demo_parser.color = False + help_text = demo_parser.format_help() + self.assertNotIn('\x1b[', help_text) + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. @@ -5686,4 +7428,8 @@ def tearDownModule(): if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(argparse) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index c3250ef72e2..db09e50e8f4 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -8,16 +8,20 @@ from test.support import import_helper from test.support import os_helper from test.support import _2G +from test.support import subTests import weakref import pickle import operator import struct import sys +import warnings import array from array import _array_reconstructor as array_reconstructor -sizeof_wchar = array.array('u').itemsize +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + sizeof_wchar = array.array('u').itemsize class ArraySubclass(array.array): @@ -27,7 +31,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = 'ubBhHiIlLfdqQ' +typecodes = 'uwbBhHiIlLfdqQ' class MiscTest(unittest.TestCase): @@ -93,8 +97,17 @@ def test_empty(self): UTF32_LE = 20 UTF32_BE = 21 + class ArrayReconstructorTest(unittest.TestCase): + def setUp(self): + self.enterContext(warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="The 'u' type code is deprecated and " + "will be removed in Python 3.16", + category=DeprecationWarning) + def test_error(self): self.assertRaises(TypeError, array_reconstructor, "", "b", 0, b"") @@ -176,23 +189,23 @@ def test_numbers(self): self.assertEqual(a, b, msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_unicode(self): teststr = "Bonne Journ\xe9e \U0002030a\U00020347" testcases = ( (UTF16_LE, "UTF-16-LE"), (UTF16_BE, "UTF-16-BE"), - (UTF32_LE, "UTF-32-LE"), # TODO: RUSTPYTHON - (UTF32_BE, "UTF-32-BE") # TODO: RUSTPYTHON + (UTF32_LE, "UTF-32-LE"), + (UTF32_BE, "UTF-32-BE") ) for testcase in testcases: mformat_code, encoding = testcase - a = array.array('u', teststr) - b = array_reconstructor( - array.array, 'u', mformat_code, teststr.encode(encoding)) - self.assertEqual(a, b, - msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) + for c in 'uw': + a = array.array(c, teststr) + b = array_reconstructor( + array.array, c, mformat_code, teststr.encode(encoding)) + self.assertEqual(a, b, + msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) class BaseTest: @@ -204,6 +217,14 @@ class BaseTest: # outside: An entry that is not in example # minitemsize: the minimum guaranteed itemsize + def setUp(self): + self.enterContext(warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="The 'u' type code is deprecated and " + "will be removed in Python 3.16", + category=DeprecationWarning) + def assertEntryEqual(self, entry1, entry2): self.assertEqual(entry1, entry2) @@ -236,7 +257,7 @@ def test_buffer_info(self): self.assertEqual(bi[1], len(a)) def test_byteswap(self): - if self.typecode == 'u': + if self.typecode in ('u', 'w'): example = '\U00100100' else: example = self.example @@ -357,8 +378,6 @@ def test_reverse_iterator(self): self.assertEqual(list(a), list(self.example)) self.assertEqual(list(reversed(a)), list(iter(a))[::-1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_reverse_iterator_picking(self): orig = array.array(self.typecode, self.example) data = list(orig) @@ -997,6 +1016,29 @@ def test_pop(self): array.array(self.typecode, self.example[3:]+self.example[:-1]) ) + def test_clear(self): + a = array.array(self.typecode, self.example) + with self.assertRaises(TypeError): + a.clear(42) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode) + a.clear() + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, self.typecode) + + a = array.array(self.typecode, self.example) + a.clear() + a.append(self.example[2]) + a.append(self.example[3]) + self.assertEqual(a, array.array(self.typecode, self.example[2:4])) + + with memoryview(a): + with self.assertRaises(BufferError): + a.clear() + def test_reverse(self): a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.reverse, 42) @@ -1083,7 +1125,7 @@ def test_buffer(self): self.assertEqual(m.tobytes(), expected) self.assertRaises(BufferError, a.frombytes, a.tobytes()) self.assertEqual(m.tobytes(), expected) - if self.typecode == 'u': + if self.typecode in ('u', 'w'): self.assertRaises(BufferError, a.fromunicode, a.tounicode()) self.assertEqual(m.tobytes(), expected) self.assertRaises(BufferError, operator.imul, a, 2) @@ -1138,17 +1180,19 @@ def test_sizeof_without_buffer(self): basesize = support.calcvobjsize('Pn2Pi') support.check_sizeof(self, a, basesize) + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' def test_initialize_with_unicode(self): - if self.typecode != 'u': + if self.typecode not in ('u', 'w'): with self.assertRaises(TypeError) as cm: a = array.array(self.typecode, 'foo') self.assertIn("cannot use a str", str(cm.exception)) with self.assertRaises(TypeError) as cm: - a = array.array(self.typecode, array.array('u', 'foo')) + a = array.array(self.typecode, array.array('w', 'foo')) self.assertIn("cannot use a unicode array", str(cm.exception)) else: a = array.array(self.typecode, "foo") a = array.array(self.typecode, array.array('u', 'foo')) + a = array.array(self.typecode, array.array('w', 'foo')) @support.cpython_only def test_obsolete_write_lock(self): @@ -1156,8 +1200,7 @@ def test_obsolete_write_lock(self): a = array.array('B', b"") self.assertRaises(BufferError, _testcapi.getbuffer_with_null_view, a) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, array.array, (self.typecode,)) @@ -1177,40 +1220,255 @@ class UnicodeTest(StringTest, unittest.TestCase): smallerexample = '\x01\u263a\x00\ufefe' biggerexample = '\x01\u263a\x01\ufeff' outside = str('\x33') - minitemsize = 2 + minitemsize = sizeof_wchar def test_unicode(self): self.assertRaises(TypeError, array.array, 'b', 'foo') - a = array.array('u', '\xa0\xc2\u1234') + a = array.array(self.typecode, '\xa0\xc2\u1234') a.fromunicode(' ') a.fromunicode('') a.fromunicode('') a.fromunicode('\x11abc\xff\u1234') s = a.tounicode() self.assertEqual(s, '\xa0\xc2\u1234 \x11abc\xff\u1234') - self.assertEqual(a.itemsize, sizeof_wchar) + self.assertEqual(a.itemsize, self.minitemsize) s = '\x00="\'a\\b\x80\xff\u0000\u0001\u1234' - a = array.array('u', s) + a = array.array(self.typecode, s) self.assertEqual( repr(a), - "array('u', '\\x00=\"\\'a\\\\b\\x80\xff\\x00\\x01\u1234')") + f"array('{self.typecode}', '\\x00=\"\\'a\\\\b\\x80\xff\\x00\\x01\u1234')") self.assertRaises(TypeError, a.fromunicode) def test_issue17223(self): - # this used to crash - if sizeof_wchar == 4: - # U+FFFFFFFF is an invalid code point in Unicode 6.0 - invalid_str = b'\xff\xff\xff\xff' - else: + if self.typecode == 'u' and sizeof_wchar == 2: # PyUnicode_FromUnicode() cannot fail with 16-bit wchar_t self.skipTest("specific to 32-bit wchar_t") - a = array.array('u', invalid_str) + + # this used to crash + # U+FFFFFFFF is an invalid code point in Unicode 6.0 + invalid_str = b'\xff\xff\xff\xff' + + a = array.array(self.typecode, invalid_str) self.assertRaises(ValueError, a.tounicode) self.assertRaises(ValueError, str, a) + def test_typecode_u_deprecation(self): + with self.assertWarns(DeprecationWarning): + array.array("u") + + def test_empty_string_mem_leak_gh140474(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + for _ in range(1000): + a = array.array('u', '') + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, 'u') + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_add(self): + return super().test_add() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extend(self): + return super().test_extend() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_iadd(self): + return super().test_iadd() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setiadd(self): + return super().test_setiadd() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setslice(self): + return super().test_setslice() + + +class UCS4Test(UnicodeTest): + typecode = 'w' + minitemsize = 4 + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_buffer(self): + return super().test_buffer() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_buffer_info(self): + return super().test_buffer_info() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_byteswap(self): + return super().test_byteswap() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_clear(self): + return super().test_clear() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_cmp(self): + return super().test_cmp() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_constructor(self): + return super().test_constructor() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_constructor_with_iterable_argument(self): + return super().test_constructor_with_iterable_argument() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_copy(self): + return super().test_copy() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_count(self): + return super().test_count() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_coveritertraverse(self): + return super().test_coveritertraverse() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_deepcopy(self): + return super().test_deepcopy() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_delitem(self): + return super().test_delitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_exhausted_iterator(self): + return super().test_exhausted_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_exhausted_reverse_iterator(self): + return super().test_exhausted_reverse_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extended_getslice(self): + return super().test_extended_getslice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_extended_set_del_slice(self): + return super().test_extended_set_del_slice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_filewrite(self): + return super().test_filewrite() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_fromarray(self): + return super().test_fromarray() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_fromfile_ioerror(self): + return super().test_fromfile_ioerror() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_getitem(self): + return super().test_getitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_getslice(self): + return super().test_getslice() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_imul(self): + return super().test_imul() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_index(self): + return super().test_index() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_insert(self): + return super().test_insert() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_issue17223(self): + return super().test_issue17223() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_iterator_pickle(self): + return super().test_iterator_pickle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_len(self): + return super().test_len() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_mul(self): + return super().test_mul() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pickle(self): + return super().test_pickle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pickle_for_empty_array(self): + return super().test_pickle_for_empty_array() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_pop(self): + return super().test_pop() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reduce_ex(self): + return super().test_reduce_ex() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_remove(self): + return super().test_remove() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_repr(self): + return super().test_repr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse(self): + return super().test_reverse() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse_iterator(self): + return super().test_reverse_iterator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_reverse_iterator_picking(self): + return super().test_reverse_iterator_picking() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_setitem(self): + return super().test_setitem() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_str(self): + return super().test_str() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofrombytes(self): + return super().test_tofrombytes() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofromfile(self): + return super().test_tofromfile() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_tofromlist(self): + return super().test_tofromlist() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_unicode(self): + return super().test_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Add support for 'w' + def test_weakref(self): + return super().test_weakref() + + class NumberTest(BaseTest): def test_extslice(self): @@ -1285,8 +1543,6 @@ def check_overflow(self, lower, upper): self.assertRaises(OverflowError, array.array, self.typecode, [upper+1]) self.assertRaises(OverflowError, a.__setitem__, 0, upper+1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclassing(self): typecode = self.typecode class ExaggeratingArray(array.array): @@ -1444,8 +1700,8 @@ def test_byteswap(self): if a.itemsize==1: self.assertEqual(a, b) else: - # On alphas treating the byte swapped bit patters as - # floats/doubles results in floating point exceptions + # On alphas treating the byte swapped bit patterns as + # floats/doubles results in floating-point exceptions # => compare the 8bit string values instead self.assertNotEqual(a.tobytes(), b.tobytes()) b.byteswap() @@ -1617,5 +1873,55 @@ def test_tolist(self, size): self.assertEqual(ls[:8], list(example[:8])) self.assertEqual(ls[-8:], list(example[-8:])) + def test_gh_128961(self): + a = array.array('i') + it = iter(a) + list(it) + it.__setstate__(0) + self.assertRaises(StopIteration, next, it) + + # Tests for NULL pointer dereference in array.__setitem__ + # when the index conversion mutates the array. + # See: https://github.com/python/cpython/issues/142555. + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + @subTests("dtype", ["b", "B", "h", "H", "i", "l", "q", "I", "L", "Q"]) + def test_setitem_use_after_clear_with_int_data(self, dtype): + victim = array.array(dtype, list(range(64))) + + class Index: + def __index__(self): + victim.clear() + return 0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Index()) + self.assertEqual(len(victim), 0) + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + def test_setitem_use_after_shrink_with_int_data(self): + victim = array.array('b', [1, 2, 3]) + + class Index: + def __index__(self): + victim.pop() + victim.pop() + return 0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Index()) + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + @subTests("dtype", ["f", "d"]) + def test_setitem_use_after_clear_with_float_data(self, dtype): + victim = array.array(dtype, [1.0, 2.0, 3.0]) + + class Float: + def __float__(self): + victim.clear() + return 0.0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Float()) + self.assertEqual(len(victim), 0) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py deleted file mode 100644 index af3e2bb5eb8..00000000000 --- a/Lib/test/test_ast.py +++ /dev/null @@ -1,3231 +0,0 @@ -import ast -import builtins -import dis -import enum -import os -import re -import sys -import textwrap -import types -import unittest -import warnings -import weakref -from functools import partial -from textwrap import dedent - -from test import support -from test.support.import_helper import import_fresh_module -from test.support import os_helper, script_helper -from test.support.ast_helper import ASTTestMixin - -def to_tuple(t): - if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis: - return t - elif isinstance(t, list): - return [to_tuple(e) for e in t] - result = [t.__class__.__name__] - if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): - result.append((t.lineno, t.col_offset)) - if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): - result[-1] += (t.end_lineno, t.end_col_offset) - if t._fields is None: - return tuple(result) - for f in t._fields: - result.append(to_tuple(getattr(t, f))) - return tuple(result) - - -# These tests are compiled through "exec" -# There should be at least one test per statement -exec_tests = [ - # None - "None", - # Module docstring - "'module docstring'", - # FunctionDef - "def f(): pass", - # FunctionDef with docstring - "def f(): 'function docstring'", - # FunctionDef with arg - "def f(a): pass", - # FunctionDef with arg and default value - "def f(a=0): pass", - # FunctionDef with varargs - "def f(*args): pass", - # FunctionDef with varargs as TypeVarTuple - "def f(*args: *Ts): pass", - # FunctionDef with varargs as unpacked Tuple - "def f(*args: *tuple[int, ...]): pass", - # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple - "def f(*args: *tuple[int, *Ts]): pass", - # FunctionDef with kwargs - "def f(**kwargs): pass", - # FunctionDef with all kind of args and docstring - "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", - # FunctionDef with type annotation on return involving unpacking - "def f() -> tuple[*Ts]: pass", - "def f() -> tuple[int, *Ts]: pass", - "def f() -> tuple[int, *tuple[int, ...]]: pass", - # ClassDef - "class C:pass", - # ClassDef with docstring - "class C: 'docstring for class C'", - # ClassDef, new style class - "class C(object): pass", - # Return - "def f():return 1", - # Delete - "del v", - # Assign - "v = 1", - "a,b = c", - "(a,b) = c", - "[a,b] = c", - # AnnAssign with unpacked types - "x: tuple[*Ts]", - "x: tuple[int, *Ts]", - "x: tuple[int, *tuple[str, ...]]", - # AugAssign - "v += 1", - # For - "for v in v:pass", - # While - "while v:pass", - # If - "if v:pass", - # If-Elif - "if a:\n pass\nelif b:\n pass", - # If-Elif-Else - "if a:\n pass\nelif b:\n pass\nelse:\n pass", - # With - "with x as y: pass", - "with x as y, z as q: pass", - # Raise - "raise Exception('string')", - # TryExcept - "try:\n pass\nexcept Exception:\n pass", - # TryFinally - "try:\n pass\nfinally:\n pass", - # TryStarExcept - "try:\n pass\nexcept* Exception:\n pass", - # Assert - "assert v", - # Import - "import sys", - # ImportFrom - "from sys import v", - # Global - "global v", - # Expr - "1", - # Pass, - "pass", - # Break - "for v in v:break", - # Continue - "for v in v:continue", - # for statements with naked tuples (see http://bugs.python.org/issue6704) - "for a,b in c: pass", - "for (a,b) in c: pass", - "for [a,b] in c: pass", - # Multiline generator expression (test for .lineno & .col_offset) - """( - ( - Aa - , - Bb - ) - for - Aa - , - Bb in Cc - )""", - # dictcomp - "{a : b for w in x for m in p if g}", - # dictcomp with naked tuple - "{a : b for v,w in x}", - # setcomp - "{r for l in x if g}", - # setcomp with naked tuple - "{r for l,m in x}", - # AsyncFunctionDef - "async def f():\n 'async function'\n await something()", - # AsyncFor - "async def f():\n async for e in i: 1\n else: 2", - # AsyncWith - "async def f():\n async with a as b: 1", - # PEP 448: Additional Unpacking Generalizations - "{**{1:2}, 2:3}", - "{*{1, 2}, 3}", - # Asynchronous comprehensions - "async def f():\n [i async for b in c]", - # Decorated FunctionDef - "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", - # Decorated AsyncFunctionDef - "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", - # Decorated ClassDef - "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", - # Decorator with generator argument - "@deco(a for a in b)\ndef f(): pass", - # Decorator with attribute - "@a.b.c\ndef f(): pass", - # Simple assignment expression - "(a := 1)", - # Positional-only arguments - "def f(a, /,): pass", - "def f(a, /, c, d, e): pass", - "def f(a, /, c, *, d, e): pass", - "def f(a, /, c, *, d, e, **kwargs): pass", - # Positional-only arguments with defaults - "def f(a=1, /,): pass", - "def f(a=1, /, b=2, c=4): pass", - "def f(a=1, /, b=2, *, c=4): pass", - "def f(a=1, /, b=2, *, c): pass", - "def f(a=1, /, b=2, *, c=4, **kwargs): pass", - "def f(a=1, /, b=2, *, c, **kwargs): pass", - # Type aliases - "type X = int", - "type X[T] = int", - "type X[T, *Ts, **P] = (T, Ts, P)", - "type X[T: int, *Ts, **P] = (T, Ts, P)", - "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", - # Generic classes - "class X[T]: pass", - "class X[T, *Ts, **P]: pass", - "class X[T: int, *Ts, **P]: pass", - "class X[T: (int, str), *Ts, **P]: pass", - # Generic functions - "def f[T](): pass", - "def f[T, *Ts, **P](): pass", - "def f[T: int, *Ts, **P](): pass", - "def f[T: (int, str), *Ts, **P](): pass", -] - -# These are compiled through "single" -# because of overlap with "eval", it just tests what -# can't be tested with "eval" -single_tests = [ - "1+2" -] - -# These are compiled through "eval" -# It should test all expressions -eval_tests = [ - # None - "None", - # BoolOp - "a and b", - # BinOp - "a + b", - # UnaryOp - "not v", - # Lambda - "lambda:None", - # Dict - "{ 1:2 }", - # Empty dict - "{}", - # Set - "{None,}", - # Multiline dict (test for .lineno & .col_offset) - """{ - 1 - : - 2 - }""", - # ListComp - "[a for b in c if d]", - # GeneratorExp - "(a for b in c if d)", - # Comprehensions with multiple for targets - "[(a,b) for a,b in c]", - "[(a,b) for (a,b) in c]", - "[(a,b) for [a,b] in c]", - "{(a,b) for a,b in c}", - "{(a,b) for (a,b) in c}", - "{(a,b) for [a,b] in c}", - "((a,b) for a,b in c)", - "((a,b) for (a,b) in c)", - "((a,b) for [a,b] in c)", - # Yield - yield expressions can't work outside a function - # - # Compare - "1 < 2 < 3", - # Call - "f(1,2,c=3,*d,**e)", - # Call with multi-character starred - "f(*[0, 1])", - # Call with a generator argument - "f(a for a in b)", - # Num - "10", - # Str - "'string'", - # Attribute - "a.b", - # Subscript - "a[b:c]", - # Name - "v", - # List - "[1,2,3]", - # Empty list - "[]", - # Tuple - "1,2,3", - # Tuple - "(1,2,3)", - # Empty tuple - "()", - # Combination - "a.b.c.d(a.b[1:2])", -] - -# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension -# excepthandler, arguments, keywords, alias - -class AST_Tests(unittest.TestCase): - maxDiff = None - - def _is_ast_node(self, name, node): - if not isinstance(node, type): - return False - if "ast" not in node.__module__: - return False - return name != 'AST' and name[0].isupper() - - def _assertTrueorder(self, ast_node, parent_pos): - if not isinstance(ast_node, ast.AST) or ast_node._fields is None: - return - if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)): - node_pos = (ast_node.lineno, ast_node.col_offset) - self.assertGreaterEqual(node_pos, parent_pos) - parent_pos = (ast_node.lineno, ast_node.col_offset) - for name in ast_node._fields: - value = getattr(ast_node, name) - if isinstance(value, list): - first_pos = parent_pos - if value and name == 'decorator_list': - first_pos = (value[0].lineno, value[0].col_offset) - for child in value: - self._assertTrueorder(child, first_pos) - elif value is not None: - self._assertTrueorder(value, parent_pos) - self.assertEqual(ast_node._fields, ast_node.__match_args__) - - def test_AST_objects(self): - x = ast.AST() - self.assertEqual(x._fields, ()) - x.foobar = 42 - self.assertEqual(x.foobar, 42) - self.assertEqual(x.__dict__["foobar"], 42) - - with self.assertRaises(AttributeError): - x.vararg - - with self.assertRaises(TypeError): - # "ast.AST constructor takes 0 positional arguments" - ast.AST(2) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_AST_garbage_collection(self): - class X: - pass - a = ast.AST() - a.x = X() - a.x.a = a - ref = weakref.ref(a.x) - del a - support.gc_collect() - self.assertIsNone(ref()) - - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'not implemented: async for comprehensions'") - def test_snippets(self): - for input, output, kind in ((exec_tests, exec_results, "exec"), - (single_tests, single_results, "single"), - (eval_tests, eval_results, "eval")): - for i, o in zip(input, output): - with self.subTest(action="parsing", input=i): - ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) - self.assertEqual(to_tuple(ast_tree), o) - self._assertTrueorder(ast_tree, (0, 0)) - with self.subTest(action="compiling", input=i, kind=kind): - compile(ast_tree, "?", kind) - - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'not implemented: async for comprehensions'") - def test_ast_validation(self): - # compile() is the only function that calls PyAST_Validate - snippets_to_validate = exec_tests + single_tests + eval_tests - for snippet in snippets_to_validate: - tree = ast.parse(snippet) - compile(tree, '', 'exec') - - @unittest.skip("TODO: RUSTPYTHON, OverflowError: Python int too large to convert to Rust u32") - def test_invalid_position_information(self): - invalid_linenos = [ - (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) - ] - - for lineno, end_lineno in invalid_linenos: - with self.subTest(f"Check invalid linenos {lineno}:{end_lineno}"): - snippet = "a = 1" - tree = ast.parse(snippet) - tree.body[0].lineno = lineno - tree.body[0].end_lineno = end_lineno - with self.assertRaises(ValueError): - compile(tree, '', 'exec') - - invalid_col_offsets = [ - (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) - ] - for col_offset, end_col_offset in invalid_col_offsets: - with self.subTest(f"Check invalid col_offset {col_offset}:{end_col_offset}"): - snippet = "a = 1" - tree = ast.parse(snippet) - tree.body[0].col_offset = col_offset - tree.body[0].end_col_offset = end_col_offset - with self.assertRaises(ValueError): - compile(tree, '', 'exec') - - def test_compilation_of_ast_nodes_with_default_end_position_values(self): - tree = ast.Module(body=[ - ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), - ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) - ], type_ignores=[]) - - # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. - compile(tree, "", "exec") - - def test_slice(self): - slc = ast.parse("x[::]").body[0].value.slice - self.assertIsNone(slc.upper) - self.assertIsNone(slc.lower) - self.assertIsNone(slc.step) - - def test_from_import(self): - im = ast.parse("from . import y").body[0] - self.assertIsNone(im.module) - - def test_non_interned_future_from_ast(self): - mod = ast.parse("from __future__ import division") - self.assertIsInstance(mod.body[0], ast.ImportFrom) - mod.body[0].module = " __future__ ".strip() - compile(mod, "", "exec") - - def test_alias(self): - im = ast.parse("from bar import y").body[0] - self.assertEqual(len(im.names), 1) - alias = im.names[0] - self.assertEqual(alias.name, 'y') - self.assertIsNone(alias.asname) - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 17) - - im = ast.parse("from bar import *").body[0] - alias = im.names[0] - self.assertEqual(alias.name, '*') - self.assertIsNone(alias.asname) - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 17) - - im = ast.parse("from bar import y as z").body[0] - alias = im.names[0] - self.assertEqual(alias.name, "y") - self.assertEqual(alias.asname, "z") - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 22) - - im = ast.parse("import bar as foo").body[0] - alias = im.names[0] - self.assertEqual(alias.name, "bar") - self.assertEqual(alias.asname, "foo") - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 7) - self.assertEqual(alias.end_col_offset, 17) - - def test_base_classes(self): - self.assertTrue(issubclass(ast.For, ast.stmt)) - self.assertTrue(issubclass(ast.Name, ast.expr)) - self.assertTrue(issubclass(ast.stmt, ast.AST)) - self.assertTrue(issubclass(ast.expr, ast.AST)) - self.assertTrue(issubclass(ast.comprehension, ast.AST)) - self.assertTrue(issubclass(ast.Gt, ast.AST)) - - def test_import_deprecated(self): - ast = import_fresh_module('ast') - depr_regex = ( - r'ast\.{} is deprecated and will be removed in Python 3.14; ' - r'use ast\.Constant instead' - ) - for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - - def test_field_attr_existence(self): - for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: - continue - # constructor has a different signature - if name == 'Index': - continue - if self._is_ast_node(name, item): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_arguments(self): - x = ast.arguments() - self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', - 'kw_defaults', 'kwarg', 'defaults')) - - with self.assertRaises(AttributeError): - x.args - self.assertIsNone(x.vararg) - - x = ast.arguments(*range(1, 8)) - self.assertEqual(x.args, 2) - self.assertEqual(x.vararg, 3) - - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - - def test_field_attr_writable(self): - x = ast.Constant() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_classattrs(self): - x = ast.Constant() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - x = ast.Constant(42) - self.assertEqual(x.value, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Constant(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Constant(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - - self.assertRaises(TypeError, ast.Constant, 1, None, 2) - self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): - ast.Constant(1, value=2) - - self.assertEqual(ast.Constant(42).value, 42) - self.assertEqual(ast.Constant(4.25).value, 4.25) - self.assertEqual(ast.Constant(4.25j).value, 4.25j) - self.assertEqual(ast.Constant('42').value, '42') - self.assertEqual(ast.Constant(b'42').value, b'42') - self.assertIs(ast.Constant(True).value, True) - self.assertIs(ast.Constant(False).value, False) - self.assertIs(ast.Constant(None).value, None) - self.assertIs(ast.Constant(...).value, ...) - - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str('42')), ast.Constant) - self.assertIs(type(ast.Bytes(b'42')), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - 'ast.{} is deprecated and will be removed in Python 3.14; ' - 'use ast.Constant instead' - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant") - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str('42') - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b'42') - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant('42'), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b'42'), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str('42') - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in '42', True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) - - class S(str): pass - with assertStrDeprecated(): - self.assertIsInstance(Constant(S('42')), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S('42')), Num) - - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, 'spam') - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual([str(w.message) for w in wlog], [ - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - ]) - - def test_constant_subclasses(self): - class N(ast.Constant): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Constant): - pass - - n = N(42) - self.assertEqual(n.value, 42) - self.assertEqual(n.z, 'spam') - self.assertEqual(type(n), N) - self.assertTrue(isinstance(n, N)) - self.assertTrue(isinstance(n, ast.Constant)) - self.assertFalse(isinstance(n, N2)) - self.assertFalse(isinstance(ast.Constant(42), N)) - n = N(value=42) - self.assertEqual(n.value, 42) - self.assertEqual(type(n), N) - - def test_module(self): - body = [ast.Constant(42)] - x = ast.Module(body, []) - self.assertEqual(x.body, body) - - def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed - x = ast.BinOp() - self.assertEqual(x._fields, ('left', 'op', 'right')) - - # Random attribute allowed too - x.foobarbaz = 5 - self.assertEqual(x.foobarbaz, 5) - - n1 = ast.Constant(1) - n3 = ast.Constant(3) - addop = ast.Add() - x = ast.BinOp(n1, addop, n3) - self.assertEqual(x.left, n1) - self.assertEqual(x.op, addop) - self.assertEqual(x.right, n3) - - x = ast.BinOp(1, 2, 3) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - - x = ast.BinOp(1, 2, 3, lineno=0) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - self.assertEqual(x.lineno, 0) - - # node raises exception when given too many arguments - self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4) - # node raises exception when given too many arguments - self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4, lineno=0) - - # can set attributes through kwargs too - x = ast.BinOp(left=1, op=2, right=3, lineno=0) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - self.assertEqual(x.lineno, 0) - - # Random kwargs also allowed - x = ast.BinOp(1, 2, 3, foobarbaz=42) - self.assertEqual(x.foobarbaz, 42) - - def test_no_fields(self): - # this used to fail because Sub._fields was None - x = ast.Sub() - self.assertEqual(x._fields, ()) - - # TODO: RUSTPYTHON _ast classes should be HEAPTYPES (except for _ast.AST) - @unittest.expectedFailure - def test_pickling(self): - import pickle - - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_invalid_sum(self): - pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) - with self.assertRaises(TypeError) as cm: - compile(m, "", "exec") - self.assertIn("but got ", "exec") - self.assertIn("identifier must be of type str", str(cm.exception)) - - def test_invalid_constant(self): - for invalid_constant in int, (1, 2, int), frozenset((1, 2, int)): - e = ast.Expression(body=ast.Constant(invalid_constant)) - ast.fix_missing_locations(e) - with self.assertRaisesRegex( - TypeError, "invalid type in Constant: type" - ): - compile(e, "", "eval") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_empty_yield_from(self): - # Issue 16546: yield from value is not optional. - empty_yield_from = ast.parse("def f():\n yield from g()") - empty_yield_from.body[0].body[0].value.value = None - with self.assertRaises(ValueError) as cm: - compile(empty_yield_from, "", "exec") - self.assertIn("field 'value' is required", str(cm.exception)) - - @support.cpython_only - def test_issue31592(self): - # There shouldn't be an assertion failure in case of a bad - # unicodedata.normalize(). - import unicodedata - def bad_normalize(*args): - return None - with support.swap_attr(unicodedata, 'normalize', bad_normalize): - self.assertRaises(TypeError, ast.parse, '\u03D5') - - def test_issue18374_binop_col_offset(self): - tree = ast.parse('4+5+6+7') - parent_binop = tree.body[0].value - child_binop = parent_binop.left - grandchild_binop = child_binop.left - self.assertEqual(parent_binop.col_offset, 0) - self.assertEqual(parent_binop.end_col_offset, 7) - self.assertEqual(child_binop.col_offset, 0) - self.assertEqual(child_binop.end_col_offset, 5) - self.assertEqual(grandchild_binop.col_offset, 0) - self.assertEqual(grandchild_binop.end_col_offset, 3) - - tree = ast.parse('4+5-\\\n 6-7') - parent_binop = tree.body[0].value - child_binop = parent_binop.left - grandchild_binop = child_binop.left - self.assertEqual(parent_binop.col_offset, 0) - self.assertEqual(parent_binop.lineno, 1) - self.assertEqual(parent_binop.end_col_offset, 4) - self.assertEqual(parent_binop.end_lineno, 2) - - self.assertEqual(child_binop.col_offset, 0) - self.assertEqual(child_binop.lineno, 1) - self.assertEqual(child_binop.end_col_offset, 2) - self.assertEqual(child_binop.end_lineno, 2) - - self.assertEqual(grandchild_binop.col_offset, 0) - self.assertEqual(grandchild_binop.lineno, 1) - self.assertEqual(grandchild_binop.end_col_offset, 3) - self.assertEqual(grandchild_binop.end_lineno, 1) - - def test_issue39579_dotted_name_end_col_offset(self): - tree = ast.parse('@a.b.c\ndef f(): pass') - attr_b = tree.body[0].decorator_list[0].value - self.assertEqual(attr_b.end_col_offset, 4) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_ast_asdl_signature(self): - self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)") - self.assertEqual(ast.GtE.__doc__, "GtE") - self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)") - self.assertEqual(ast.cmpop.__doc__, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn") - expressions = [f" | {node.__doc__}" for node in ast.expr.__subclasses__()] - expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" - self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_positional_only_feature_version(self): - ast.parse('def foo(x, /): ...', feature_version=(3, 8)) - ast.parse('def bar(x=1, /): ...', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('def foo(x, /): ...', feature_version=(3, 7)) - with self.assertRaises(SyntaxError): - ast.parse('def bar(x=1, /): ...', feature_version=(3, 7)) - - ast.parse('lambda x, /: ...', feature_version=(3, 8)) - ast.parse('lambda x=1, /: ...', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('lambda x, /: ...', feature_version=(3, 7)) - with self.assertRaises(SyntaxError): - ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_parenthesized_with_feature_version(self): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 10)) - # While advertised as a feature in Python 3.10, this was allowed starting 3.9 - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 9)) - with self.assertRaises(SyntaxError): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 8)) - ast.parse('with CtxManager() as example: ...', feature_version=(3, 8)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_assignment_expression_feature_version(self): - ast.parse('(x := 0)', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('(x := 0)', feature_version=(3, 7)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_exception_groups_feature_version(self): - code = dedent(''' - try: ... - except* Exception: ... - ''') - ast.parse(code) - with self.assertRaises(SyntaxError): - ast.parse(code, feature_version=(3, 10)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_type_params_feature_version(self): - samples = [ - "type X = int", - "class X[T]: pass", - "def f[T](): pass", - ] - for sample in samples: - with self.subTest(sample): - ast.parse(sample) - with self.assertRaises(SyntaxError): - ast.parse(sample, feature_version=(3, 11)) - - def test_invalid_major_feature_version(self): - with self.assertRaises(ValueError): - ast.parse('pass', feature_version=(2, 7)) - with self.assertRaises(ValueError): - ast.parse('pass', feature_version=(4, 0)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_constant_as_name(self): - for constant in "True", "False", "None": - expr = ast.Expression(ast.Name(constant, ast.Load())) - ast.fix_missing_locations(expr) - with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): - compile(expr, "", "eval") - - @unittest.skip("TODO: RUSTPYTHON, TypeError: enum mismatch") - def test_precedence_enum(self): - class _Precedence(enum.IntEnum): - """Precedence table that originated from python grammar.""" - NAMED_EXPR = enum.auto() # := - TUPLE = enum.auto() # , - YIELD = enum.auto() # 'yield', 'yield from' - TEST = enum.auto() # 'if'-'else', 'lambda' - OR = enum.auto() # 'or' - AND = enum.auto() # 'and' - NOT = enum.auto() # 'not' - CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', - # 'in', 'not in', 'is', 'is not' - EXPR = enum.auto() - BOR = EXPR # '|' - BXOR = enum.auto() # '^' - BAND = enum.auto() # '&' - SHIFT = enum.auto() # '<<', '>>' - ARITH = enum.auto() # '+', '-' - TERM = enum.auto() # '*', '@', '/', '%', '//' - FACTOR = enum.auto() # unary '+', '-', '~' - POWER = enum.auto() # '**' - AWAIT = enum.auto() # 'await' - ATOM = enum.auto() - def next(self): - try: - return self.__class__(self + 1) - except ValueError: - return self - enum._test_simple_enum(_Precedence, ast._Precedence) - - @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") - @support.cpython_only - def test_ast_recursion_limit(self): - fail_depth = support.EXCEEDS_RECURSION_LIMIT - crash_depth = 100_000 - success_depth = 1200 - - def check_limit(prefix, repeated): - expect_ok = prefix + repeated * success_depth - ast.parse(expect_ok) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format( - prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - with support.infinite_recursion(): - ast.parse(broken) - - check_limit("a", "()") - check_limit("a", ".b") - check_limit("a", "[0]") - check_limit("a", "*a") - - def test_null_bytes(self): - with self.assertRaises(SyntaxError, - msg="source code string cannot contain null bytes"): - ast.parse("a\0b") - - def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None: - with self.subTest(f"{node.__name__}.{attr}"): - tree = ast.parse(source) - found = 0 - for child in ast.walk(tree): - if isinstance(child, node): - setattr(child, attr, None) - found += 1 - self.assertEqual(found, 1) - e = re.escape(f"field '{attr}' is required for {node.__name__}") - with self.assertRaisesRegex(ValueError, f"^{e}$"): - compile(tree, "", "exec") - - @unittest.skip("TODO: RUSTPYTHON, TypeError: Expected type 'str' but 'NoneType' found") - def test_none_checks(self) -> None: - tests = [ - (ast.alias, "name", "import spam as SPAM"), - (ast.arg, "arg", "def spam(SPAM): spam"), - (ast.comprehension, "target", "[spam for SPAM in spam]"), - (ast.comprehension, "iter", "[spam for spam in SPAM]"), - (ast.keyword, "value", "spam(**SPAM)"), - (ast.match_case, "pattern", "match spam:\n case SPAM: spam"), - (ast.withitem, "context_expr", "with SPAM: spam"), - ] - for node, attr, source in tests: - self.assert_none_check(node, attr, source) - -class ASTHelpers_Test(unittest.TestCase): - maxDiff = None - - def test_parse(self): - a = ast.parse('foo(1 + 1)') - b = compile('foo(1 + 1)', '', 'exec', ast.PyCF_ONLY_AST) - self.assertEqual(ast.dump(a), ast.dump(b)) - - def test_parse_in_error(self): - try: - 1/0 - except Exception: - with self.assertRaises(SyntaxError) as e: - ast.literal_eval(r"'\U'") - self.assertIsNotNone(e.exception.__context__) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dump(self): - node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual(ast.dump(node), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " - "keywords=[]))], type_ignores=[])" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))], [])" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " - "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " - "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " - "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" - ) - - # TODO: RUSTPYTHON; redundant kind for Contant node - @unittest.expectedFailure - def test_dump_indent(self): - node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual(ast.dump(node, indent=3), """\ -Module( - body=[ - Expr( - value=Call( - func=Name(id='spam', ctx=Load()), - args=[ - Name(id='eggs', ctx=Load()), - Constant(value='and cheese')], - keywords=[]))], - type_ignores=[])""") - self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ -Module( -\t[ -\t\tExpr( -\t\t\tCall( -\t\t\t\tName('spam', Load()), -\t\t\t\t[ -\t\t\t\t\tName('eggs', Load()), -\t\t\t\t\tConstant('and cheese')], -\t\t\t\t[]))], -\t[])""") - self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ -Module( - body=[ - Expr( - value=Call( - func=Name( - id='spam', - ctx=Load(), - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=4), - args=[ - Name( - id='eggs', - ctx=Load(), - lineno=1, - col_offset=5, - end_lineno=1, - end_col_offset=9), - Constant( - value='and cheese', - lineno=1, - col_offset=11, - end_lineno=1, - end_col_offset=23)], - keywords=[], - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=24), - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=24)], - type_ignores=[])""") - - def test_dump_incomplete(self): - node = ast.Raise(lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), - "Raise()" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(lineno=3, col_offset=4)" - ) - node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), - "Raise(exc=Name(id='e', ctx=Load()))" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(Name('e', Load()))" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" - ) - self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), - "Raise(Name('e', Load()), lineno=3, col_offset=4)" - ) - node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) - self.assertEqual(ast.dump(node), - "Raise(cause=Name(id='e', ctx=Load()))" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(cause=Name('e', Load()))" - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_copy_location(self): - src = ast.parse('1 + 1', mode='eval') - src.body.right = ast.copy_location(ast.Constant(2), src.body.right) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' - 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' - 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' - 'col_offset=0, end_lineno=1, end_col_offset=5))' - ) - src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1) - new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None)) - self.assertIsNone(new.end_lineno) - self.assertIsNone(new.end_col_offset) - self.assertEqual(new.lineno, 1) - self.assertEqual(new.col_offset, 1) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_fix_missing_locations(self): - src = ast.parse('write("spam")') - src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), - [ast.Constant('eggs')], []))) - self.assertEqual(src, ast.fix_missing_locations(src)) - self.maxDiff = None - self.assertEqual(ast.dump(src, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " - "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " - "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " - "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], " - "type_ignores=[])" - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_increment_lineno(self): - src = ast.parse('1 + 1', mode='eval') - self.assertEqual(ast.increment_lineno(src, n=3), src) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' - 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' - 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' - 'col_offset=0, end_lineno=4, end_col_offset=5))' - ) - # issue10869: do not increment lineno of root twice - src = ast.parse('1 + 1', mode='eval') - self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' - 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' - 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' - 'col_offset=0, end_lineno=4, end_col_offset=5))' - ) - src = ast.Call( - func=ast.Name("test", ast.Load()), args=[], keywords=[], lineno=1 - ) - self.assertEqual(ast.increment_lineno(src).lineno, 2) - self.assertIsNone(ast.increment_lineno(src).end_lineno) - - @unittest.skip("TODO: RUSTPYTHON, NameError: name 'PyCF_TYPE_COMMENTS' is not defined") - def test_increment_lineno_on_module(self): - src = ast.parse(dedent("""\ - a = 1 - b = 2 # type: ignore - c = 3 - d = 4 # type: ignore@tag - """), type_comments=True) - ast.increment_lineno(src, n=5) - self.assertEqual(src.type_ignores[0].lineno, 7) - self.assertEqual(src.type_ignores[1].lineno, 9) - self.assertEqual(src.type_ignores[1].tag, '@tag') - - def test_iter_fields(self): - node = ast.parse('foo()', mode='eval') - d = dict(ast.iter_fields(node.body)) - self.assertEqual(d.pop('func').id, 'foo') - self.assertEqual(d, {'keywords': [], 'args': []}) - - # TODO: RUSTPYTHON; redundant kind for Constant node - @unittest.expectedFailure - def test_iter_child_nodes(self): - node = ast.parse("spam(23, 42, eggs='leek')", mode='eval') - self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) - iterator = ast.iter_child_nodes(node.body) - self.assertEqual(next(iterator).id, 'spam') - self.assertEqual(next(iterator).value, 23) - self.assertEqual(next(iterator).value, 42) - self.assertEqual(ast.dump(next(iterator)), - "keyword(arg='eggs', value=Constant(value='leek'))" - ) - - def test_get_docstring(self): - node = ast.parse('"""line one\n line two"""') - self.assertEqual(ast.get_docstring(node), - 'line one\nline two') - - node = ast.parse('class foo:\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), - 'line one\nline two') - - node = ast.parse('def foo():\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), - 'line one\nline two') - - node = ast.parse('async def foo():\n """spam\n ham"""') - self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') - - def test_get_docstring_none(self): - self.assertIsNone(ast.get_docstring(ast.parse(''))) - node = ast.parse('x = "not docstring"') - self.assertIsNone(ast.get_docstring(node)) - node = ast.parse('def foo():\n pass') - self.assertIsNone(ast.get_docstring(node)) - - node = ast.parse('class foo:\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('class foo:\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('class foo:\n def bar(self): pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - - node = ast.parse('def foo():\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('def foo():\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - - node = ast.parse('async def foo():\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('async def foo():\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - - def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): - node = ast.parse( - '"""line one\nline two"""\n\n' - 'def foo():\n """line one\n line two"""\n\n' - ' def bar():\n """line one\n line two"""\n' - ' """line one\n line two"""\n' - '"""line one\nline two"""\n\n' - ) - self.assertEqual(node.body[0].col_offset, 0) - self.assertEqual(node.body[0].lineno, 1) - self.assertEqual(node.body[1].body[0].col_offset, 2) - self.assertEqual(node.body[1].body[0].lineno, 5) - self.assertEqual(node.body[1].body[1].body[0].col_offset, 4) - self.assertEqual(node.body[1].body[1].body[0].lineno, 9) - self.assertEqual(node.body[1].body[2].col_offset, 2) - self.assertEqual(node.body[1].body[2].lineno, 11) - self.assertEqual(node.body[2].col_offset, 0) - self.assertEqual(node.body[2].lineno, 13) - - def test_elif_stmt_start_position(self): - node = ast.parse('if a:\n pass\nelif b:\n pass\n') - elif_stmt = node.body[0].orelse[0] - self.assertEqual(elif_stmt.lineno, 3) - self.assertEqual(elif_stmt.col_offset, 0) - - def test_elif_stmt_start_position_with_else(self): - node = ast.parse('if a:\n pass\nelif b:\n pass\nelse:\n pass\n') - elif_stmt = node.body[0].orelse[0] - self.assertEqual(elif_stmt.lineno, 3) - self.assertEqual(elif_stmt.col_offset, 0) - - def test_starred_expr_end_position_within_call(self): - node = ast.parse('f(*[0, 1])') - starred_expr = node.body[0].value.args[0] - self.assertEqual(starred_expr.end_lineno, 1) - self.assertEqual(starred_expr.end_col_offset, 9) - - def test_literal_eval(self): - self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3]) - self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) - self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None)) - self.assertEqual(ast.literal_eval('{1, 2, 3}'), {1, 2, 3}) - self.assertEqual(ast.literal_eval('b"hi"'), b"hi") - self.assertEqual(ast.literal_eval('set()'), set()) - self.assertRaises(ValueError, ast.literal_eval, 'foo()') - self.assertEqual(ast.literal_eval('6'), 6) - self.assertEqual(ast.literal_eval('+6'), 6) - self.assertEqual(ast.literal_eval('-6'), -6) - self.assertEqual(ast.literal_eval('3.25'), 3.25) - self.assertEqual(ast.literal_eval('+3.25'), 3.25) - self.assertEqual(ast.literal_eval('-3.25'), -3.25) - self.assertEqual(repr(ast.literal_eval('-0.0')), '-0.0') - self.assertRaises(ValueError, ast.literal_eval, '++6') - self.assertRaises(ValueError, ast.literal_eval, '+True') - self.assertRaises(ValueError, ast.literal_eval, '2+3') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_literal_eval_str_int_limit(self): - with support.adjust_int_max_str_digits(4000): - ast.literal_eval('3'*4000) # no error - with self.assertRaises(SyntaxError) as err_ctx: - ast.literal_eval('3'*4001) - self.assertIn('Exceeds the limit ', str(err_ctx.exception)) - self.assertIn(' Consider hexadecimal ', str(err_ctx.exception)) - - def test_literal_eval_complex(self): - # Issue #4907 - self.assertEqual(ast.literal_eval('6j'), 6j) - self.assertEqual(ast.literal_eval('-6j'), -6j) - self.assertEqual(ast.literal_eval('6.75j'), 6.75j) - self.assertEqual(ast.literal_eval('-6.75j'), -6.75j) - self.assertEqual(ast.literal_eval('3+6j'), 3+6j) - self.assertEqual(ast.literal_eval('-3+6j'), -3+6j) - self.assertEqual(ast.literal_eval('3-6j'), 3-6j) - self.assertEqual(ast.literal_eval('-3-6j'), -3-6j) - self.assertEqual(ast.literal_eval('3.25+6.75j'), 3.25+6.75j) - self.assertEqual(ast.literal_eval('-3.25+6.75j'), -3.25+6.75j) - self.assertEqual(ast.literal_eval('3.25-6.75j'), 3.25-6.75j) - self.assertEqual(ast.literal_eval('-3.25-6.75j'), -3.25-6.75j) - self.assertEqual(ast.literal_eval('(3+6j)'), 3+6j) - self.assertRaises(ValueError, ast.literal_eval, '-6j+3') - self.assertRaises(ValueError, ast.literal_eval, '-6j+3j') - self.assertRaises(ValueError, ast.literal_eval, '3+-6j') - self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') - self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') - - def test_literal_eval_malformed_dict_nodes(self): - malformed = ast.Dict(keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)]) - self.assertRaises(ValueError, ast.literal_eval, malformed) - malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) - self.assertRaises(ValueError, ast.literal_eval, malformed) - - def test_literal_eval_trailing_ws(self): - self.assertEqual(ast.literal_eval(" -1"), -1) - self.assertEqual(ast.literal_eval("\t\t-1"), -1) - self.assertEqual(ast.literal_eval(" \t -1"), -1) - self.assertRaises(IndentationError, ast.literal_eval, "\n -1") - - def test_literal_eval_malformed_lineno(self): - msg = r'malformed node or string on line 3:' - with self.assertRaisesRegex(ValueError, msg): - ast.literal_eval("{'a': 1,\n'b':2,\n'c':++3,\n'd':4}") - - node = ast.UnaryOp( - ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) - self.assertIsNone(getattr(node, 'lineno', None)) - msg = r'malformed node or string:' - with self.assertRaisesRegex(ValueError, msg): - ast.literal_eval(node) - - def test_literal_eval_syntax_errors(self): - with self.assertRaisesRegex(SyntaxError, "unexpected indent"): - ast.literal_eval(r''' - \ - (\ - \ ''') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bad_integer(self): - # issue13436: Bad error message with invalid numeric values - body = [ast.ImportFrom(module='time', - names=[ast.alias(name='sleep')], - level=None, - lineno=None, col_offset=None)] - mod = ast.Module(body, []) - with self.assertRaises(ValueError) as cm: - compile(mod, 'test', 'exec') - self.assertIn("invalid integer value: None", str(cm.exception)) - - def test_level_as_none(self): - body = [ast.ImportFrom(module='time', - names=[ast.alias(name='sleep', - lineno=0, col_offset=0)], - level=None, - lineno=0, col_offset=0)] - mod = ast.Module(body, []) - code = compile(mod, 'test', 'exec') - ns = {} - exec(code, ns) - self.assertIn('sleep', ns) - - @unittest.skip("TODO: RUSTPYTHON; crash") - def test_recursion_direct(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - e.operand = e - with self.assertRaises(RecursionError): - with support.infinite_recursion(): - compile(ast.Expression(e), "", "eval") - - @unittest.skip("TODO: RUSTPYTHON; crash") - def test_recursion_indirect(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - e.operand = f - f.operand = e - with self.assertRaises(RecursionError): - with support.infinite_recursion(): - compile(ast.Expression(e), "", "eval") - - -class ASTValidatorTests(unittest.TestCase): - - def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): - mod.lineno = mod.col_offset = 0 - ast.fix_missing_locations(mod) - if msg is None: - compile(mod, "", mode) - else: - with self.assertRaises(exc) as cm: - compile(mod, "", mode) - self.assertIn(msg, str(cm.exception)) - - def expr(self, node, msg=None, *, exc=ValueError): - mod = ast.Module([ast.Expr(node)], []) - self.mod(mod, msg, exc=exc) - - def stmt(self, stmt, msg=None): - mod = ast.Module([stmt], []) - self.mod(mod, msg) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_module(self): - m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) - self.mod(m, "must have Load context", "single") - m = ast.Expression(ast.Name("x", ast.Store())) - self.mod(m, "must have Load context", "eval") - - def _check_arguments(self, fac, check): - def arguments(args=None, posonlyargs=None, vararg=None, - kwonlyargs=None, kwarg=None, - defaults=None, kw_defaults=None): - if args is None: - args = [] - if posonlyargs is None: - posonlyargs = [] - if kwonlyargs is None: - kwonlyargs = [] - if defaults is None: - defaults = [] - if kw_defaults is None: - kw_defaults = [] - args = ast.arguments(args, posonlyargs, vararg, kwonlyargs, - kw_defaults, kwarg, defaults) - return fac(args) - args = [ast.arg("x", ast.Name("x", ast.Store()))] - check(arguments(args=args), "must have Load context") - check(arguments(posonlyargs=args), "must have Load context") - check(arguments(kwonlyargs=args), "must have Load context") - check(arguments(defaults=[ast.Constant(3)]), - "more positional defaults than args") - check(arguments(kw_defaults=[ast.Constant(4)]), - "length of kwonlyargs is not the same as kw_defaults") - args = [ast.arg("x", ast.Name("x", ast.Load()))] - check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), - "must have Load context") - args = [ast.arg("a", ast.Name("x", ast.Load())), - ast.arg("b", ast.Name("y", ast.Load()))] - check(arguments(kwonlyargs=args, - kw_defaults=[None, ast.Name("x", ast.Store())]), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_funcdef(self): - a = ast.arguments([], [], None, [], [], None, []) - f = ast.FunctionDef("x", a, [], [], None, None, []) - self.stmt(f, "empty body on FunctionDef") - f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], None, None, []) - self.stmt(f, "must have Load context") - f = ast.FunctionDef("x", a, [ast.Pass()], [], - ast.Name("x", ast.Store()), None, []) - self.stmt(f, "must have Load context") - f = ast.FunctionDef("x", ast.arguments(), [ast.Pass()]) - self.stmt(f) - def fac(args): - return ast.FunctionDef("x", args, [ast.Pass()], [], None, None, []) - self._check_arguments(fac, self.stmt) - - # TODO: RUSTPYTHON, match expression is not implemented yet - # def test_funcdef_pattern_matching(self): - # # gh-104799: New fields on FunctionDef should be added at the end - # def matcher(node): - # match node: - # case ast.FunctionDef("foo", ast.arguments(args=[ast.arg("bar")]), - # [ast.Pass()], - # [ast.Name("capybara", ast.Load())], - # ast.Name("pacarana", ast.Load())): - # return True - # case _: - # return False - - # code = """ - # @capybara - # def foo(bar) -> pacarana: - # pass - # """ - # source = ast.parse(textwrap.dedent(code)) - # funcdef = source.body[0] - # self.assertIsInstance(funcdef, ast.FunctionDef) - # self.assertTrue(matcher(funcdef)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_classdef(self): - def cls(bases=None, keywords=None, body=None, decorator_list=None, type_params=None): - if bases is None: - bases = [] - if keywords is None: - keywords = [] - if body is None: - body = [ast.Pass()] - if decorator_list is None: - decorator_list = [] - if type_params is None: - type_params = [] - return ast.ClassDef("myclass", bases, keywords, - body, decorator_list, type_params) - self.stmt(cls(bases=[ast.Name("x", ast.Store())]), - "must have Load context") - self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), - "must have Load context") - self.stmt(cls(body=[]), "empty body on ClassDef") - self.stmt(cls(body=[None]), "None disallowed") - self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_delete(self): - self.stmt(ast.Delete([]), "empty targets on Delete") - self.stmt(ast.Delete([None]), "None disallowed") - self.stmt(ast.Delete([ast.Name("x", ast.Load())]), - "must have Del context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_assign(self): - self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") - self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") - self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), - "must have Store context") - self.stmt(ast.Assign([ast.Name("x", ast.Store())], - ast.Name("y", ast.Store())), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_augassign(self): - aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(), - ast.Name("y", ast.Load())) - self.stmt(aug, "must have Store context") - aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(), - ast.Name("y", ast.Store())) - self.stmt(aug, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_for(self): - x = ast.Name("x", ast.Store()) - y = ast.Name("y", ast.Load()) - p = ast.Pass() - self.stmt(ast.For(x, y, [], []), "empty body on For") - self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []), - "must have Store context") - self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []), - "must have Load context") - e = ast.Expr(ast.Name("x", ast.Store())) - self.stmt(ast.For(x, y, [e], []), "must have Load context") - self.stmt(ast.For(x, y, [p], [e]), "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_while(self): - self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") - self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), - "must have Load context") - self.stmt(ast.While(ast.Constant(3), [ast.Pass()], - [ast.Expr(ast.Name("x", ast.Store()))]), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_if(self): - self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") - i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) - self.stmt(i, "must have Load context") - i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) - self.stmt(i, "must have Load context") - i = ast.If(ast.Constant(3), [ast.Pass()], - [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(i, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_with(self): - p = ast.Pass() - self.stmt(ast.With([], [p]), "empty items on With") - i = ast.withitem(ast.Constant(3), None) - self.stmt(ast.With([i], []), "empty body on With") - i = ast.withitem(ast.Name("x", ast.Store()), None) - self.stmt(ast.With([i], [p]), "must have Load context") - i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) - self.stmt(ast.With([i], [p]), "must have Store context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_raise(self): - r = ast.Raise(None, ast.Constant(3)) - self.stmt(r, "Raise with cause but no exception") - r = ast.Raise(ast.Name("x", ast.Store()), None) - self.stmt(r, "must have Load context") - r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) - self.stmt(r, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_try(self): - p = ast.Pass() - t = ast.Try([], [], [], [p]) - self.stmt(t, "empty body on Try") - t = ast.Try([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) - self.stmt(t, "must have Load context") - t = ast.Try([p], [], [], []) - self.stmt(t, "Try has neither except handlers nor finalbody") - t = ast.Try([p], [], [p], [p]) - self.stmt(t, "Try has orelse but no except handlers") - t = ast.Try([p], [ast.ExceptHandler(None, "x", [])], [], []) - self.stmt(t, "empty body on ExceptHandler") - e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] - self.stmt(ast.Try([p], e, [], []), "must have Load context") - e = [ast.ExceptHandler(None, "x", [p])] - t = ast.Try([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) - self.stmt(t, "must have Load context") - t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(t, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON, SyntaxError: RustPython does not implement this feature yet") - def test_try_star(self): - p = ast.Pass() - t = ast.TryStar([], [], [], [p]) - self.stmt(t, "empty body on TryStar") - t = ast.TryStar([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) - self.stmt(t, "must have Load context") - t = ast.TryStar([p], [], [], []) - self.stmt(t, "TryStar has neither except handlers nor finalbody") - t = ast.TryStar([p], [], [p], [p]) - self.stmt(t, "TryStar has orelse but no except handlers") - t = ast.TryStar([p], [ast.ExceptHandler(None, "x", [])], [], []) - self.stmt(t, "empty body on ExceptHandler") - e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] - self.stmt(ast.TryStar([p], e, [], []), "must have Load context") - e = [ast.ExceptHandler(None, "x", [p])] - t = ast.TryStar([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) - self.stmt(t, "must have Load context") - t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(t, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_assert(self): - self.stmt(ast.Assert(ast.Name("x", ast.Store()), None), - "must have Load context") - assrt = ast.Assert(ast.Name("x", ast.Load()), - ast.Name("y", ast.Store())) - self.stmt(assrt, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_import(self): - self.stmt(ast.Import([]), "empty names on Import") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_importfrom(self): - imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) - self.stmt(imp, "Negative ImportFrom level") - self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_global(self): - self.stmt(ast.Global([]), "empty names on Global") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_nonlocal(self): - self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_expr(self): - e = ast.Expr(ast.Name("x", ast.Store())) - self.stmt(e, "must have Load context") - - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'called `Option::unwrap()` on a `None` value'") - def test_boolop(self): - b = ast.BoolOp(ast.And(), []) - self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Constant(3)]) - self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Constant(4), None]) - self.expr(b, "None disallowed") - b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) - self.expr(b, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_unaryop(self): - u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) - self.expr(u, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_lambda(self): - a = ast.arguments([], [], None, [], [], None, []) - self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), - "must have Load context") - def fac(args): - return ast.Lambda(args, ast.Name("x", ast.Load())) - self._check_arguments(fac, self.expr) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_ifexp(self): - l = ast.Name("x", ast.Load()) - s = ast.Name("y", ast.Store()) - for args in (s, l, l), (l, s, l), (l, l, s): - self.expr(ast.IfExp(*args), "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dict(self): - d = ast.Dict([], [ast.Name("x", ast.Load())]) - self.expr(d, "same number of keys as values") - d = ast.Dict([ast.Name("x", ast.Load())], [None]) - self.expr(d, "None disallowed") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_set(self): - self.expr(ast.Set([None]), "None disallowed") - s = ast.Set([ast.Name("x", ast.Store())]) - self.expr(s, "must have Load context") - - def _check_comprehension(self, fac): - self.expr(fac([]), "comprehension with no generators") - g = ast.comprehension(ast.Name("x", ast.Load()), - ast.Name("x", ast.Load()), [], 0) - self.expr(fac([g]), "must have Store context") - g = ast.comprehension(ast.Name("x", ast.Store()), - ast.Name("x", ast.Store()), [], 0) - self.expr(fac([g]), "must have Load context") - x = ast.Name("x", ast.Store()) - y = ast.Name("y", ast.Load()) - g = ast.comprehension(x, y, [None], 0) - self.expr(fac([g]), "None disallowed") - g = ast.comprehension(x, y, [ast.Name("x", ast.Store())], 0) - self.expr(fac([g]), "must have Load context") - - def _simple_comp(self, fac): - g = ast.comprehension(ast.Name("x", ast.Store()), - ast.Name("x", ast.Load()), [], 0) - self.expr(fac(ast.Name("x", ast.Store()), [g]), - "must have Load context") - def wrap(gens): - return fac(ast.Name("x", ast.Store()), gens) - self._check_comprehension(wrap) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_listcomp(self): - self._simple_comp(ast.ListComp) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_setcomp(self): - self._simple_comp(ast.SetComp) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_generatorexp(self): - self._simple_comp(ast.GeneratorExp) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dictcomp(self): - g = ast.comprehension(ast.Name("y", ast.Store()), - ast.Name("p", ast.Load()), [], 0) - c = ast.DictComp(ast.Name("x", ast.Store()), - ast.Name("y", ast.Load()), [g]) - self.expr(c, "must have Load context") - c = ast.DictComp(ast.Name("x", ast.Load()), - ast.Name("y", ast.Store()), [g]) - self.expr(c, "must have Load context") - def factory(comps): - k = ast.Name("x", ast.Load()) - v = ast.Name("y", ast.Load()) - return ast.DictComp(k, v, comps) - self._check_comprehension(factory) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_yield(self): - self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") - self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") - - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'assertion failed: `(left == right)` left: `0`, right: `1`'") - def test_compare(self): - left = ast.Name("x", ast.Load()) - comp = ast.Compare(left, [ast.In()], []) - self.expr(comp, "no comparators") - comp = ast.Compare(left, [ast.In()], [ast.Constant(4), ast.Constant(5)]) - self.expr(comp, "different number of comparators and operands") - comp = ast.Compare(ast.Constant("blah"), [ast.In()], [left]) - self.expr(comp) - comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) - self.expr(comp) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_call(self): - func = ast.Name("x", ast.Load()) - args = [ast.Name("y", ast.Load())] - keywords = [ast.keyword("w", ast.Name("z", ast.Load()))] - call = ast.Call(ast.Name("x", ast.Store()), args, keywords) - self.expr(call, "must have Load context") - call = ast.Call(func, [None], keywords) - self.expr(call, "None disallowed") - bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))] - call = ast.Call(func, args, bad_keywords) - self.expr(call, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_attribute(self): - attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) - self.expr(attr, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), - ast.Load()) - self.expr(sub, "must have Load context") - x = ast.Name("x", ast.Load()) - sub = ast.Subscript(x, ast.Name("y", ast.Store()), - ast.Load()) - self.expr(sub, "must have Load context") - s = ast.Name("x", ast.Store()) - for args in (s, None, None), (None, s, None), (None, None, s): - sl = ast.Slice(*args) - self.expr(ast.Subscript(x, sl, ast.Load()), - "must have Load context") - sl = ast.Tuple([], ast.Load()) - self.expr(ast.Subscript(x, sl, ast.Load())) - sl = ast.Tuple([s], ast.Load()) - self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_starred(self): - left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], - ast.Store()) - assign = ast.Assign([left], ast.Constant(4)) - self.stmt(assign, "must have Store context") - - def _sequence(self, fac): - self.expr(fac([None], ast.Load()), "None disallowed") - self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_list(self): - self._sequence(ast.List) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_tuple(self): - self._sequence(ast.Tuple) - - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @support.requires_resource('cpu') - def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: - with self.subTest(module): - fn = os.path.join(stdlib, module) - with open(fn, "r", encoding="utf-8") as fp: - source = fp.read() - mod = ast.parse(source, fn) - compile(mod, fn, "exec") - - constant_1 = ast.Constant(1) - pattern_1 = ast.MatchValue(constant_1) - - constant_x = ast.Constant('x') - pattern_x = ast.MatchValue(constant_x) - - constant_true = ast.Constant(True) - pattern_true = ast.MatchSingleton(True) - - name_carter = ast.Name('carter', ast.Load()) - - _MATCH_PATTERNS = [ - ast.MatchValue( - ast.Attribute( - ast.Attribute( - ast.Name('x', ast.Store()), - 'y', ast.Load() - ), - 'z', ast.Load() - ) - ), - ast.MatchValue( - ast.Attribute( - ast.Attribute( - ast.Name('x', ast.Load()), - 'y', ast.Store() - ), - 'z', ast.Load() - ) - ), - ast.MatchValue( - ast.Constant(...) - ), - ast.MatchValue( - ast.Constant(True) - ), - ast.MatchValue( - ast.Constant((1,2,3)) - ), - ast.MatchSingleton('string'), - ast.MatchSequence([ - ast.MatchSingleton('string') - ]), - ast.MatchSequence( - [ - ast.MatchSequence( - [ - ast.MatchSingleton('string') - ] - ) - ] - ), - ast.MatchMapping( - [constant_1, constant_true], - [pattern_x] - ), - ast.MatchMapping( - [constant_true, constant_1], - [pattern_x, pattern_1], - rest='True' - ), - ast.MatchMapping( - [constant_true, ast.Starred(ast.Name('lol', ast.Load()), ast.Load())], - [pattern_x, pattern_1], - rest='legit' - ), - ast.MatchClass( - ast.Attribute( - ast.Attribute( - constant_x, - 'y', ast.Load()), - 'z', ast.Load()), - patterns=[], kwd_attrs=[], kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=['True'], - kwd_patterns=[pattern_1] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=[], - kwd_patterns=[pattern_1] - ), - ast.MatchClass( - name_carter, - patterns=[ast.MatchSingleton('string')], - kwd_attrs=[], - kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[ast.MatchStar()], - kwd_attrs=[], - kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=[], - kwd_patterns=[ast.MatchStar()] - ), - ast.MatchClass( - constant_true, # invalid name - patterns=[], - kwd_attrs=['True'], - kwd_patterns=[pattern_1] - ), - ast.MatchSequence( - [ - ast.MatchStar("True") - ] - ), - ast.MatchAs( - name='False' - ), - ast.MatchOr( - [] - ), - ast.MatchOr( - [pattern_1] - ), - ast.MatchOr( - [pattern_1, pattern_x, ast.MatchSingleton('xxx')] - ), - ast.MatchAs(name="_"), - ast.MatchStar(name="x"), - ast.MatchSequence([ast.MatchStar("_")]), - ast.MatchMapping([], [], rest="_"), - ] - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_match_validation_pattern(self): - name_x = ast.Name('x', ast.Load()) - for pattern in self._MATCH_PATTERNS: - with self.subTest(ast.dump(pattern, indent=4)): - node = ast.Match( - subject=name_x, - cases = [ - ast.match_case( - pattern=pattern, - body = [ast.Pass()] - ) - ] - ) - node = ast.fix_missing_locations(node) - module = ast.Module([node], []) - with self.assertRaises(ValueError): - compile(module, "", "exec") - - -class ConstantTests(unittest.TestCase): - """Tests on the ast.Constant node type.""" - - def compile_constant(self, value): - tree = ast.parse("x = 123") - - node = tree.body[0].value - new_node = ast.Constant(value=value) - ast.copy_location(new_node, node) - tree.body[0].value = new_node - - code = compile(tree, "", "exec") - - ns = {} - exec(code, ns) - return ns['x'] - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_validation(self): - with self.assertRaises(TypeError) as cm: - self.compile_constant([1, 2, 3]) - self.assertEqual(str(cm.exception), - "got an invalid type in Constant: list") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_singletons(self): - for const in (None, False, True, Ellipsis, b'', frozenset()): - with self.subTest(const=const): - value = self.compile_constant(const) - self.assertIs(value, const) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_values(self): - nested_tuple = (1,) - nested_frozenset = frozenset({1}) - for level in range(3): - nested_tuple = (nested_tuple, 2) - nested_frozenset = frozenset({nested_frozenset, 2}) - values = (123, 123.0, 123j, - "unicode", b'bytes', - tuple("tuple"), frozenset("frozenset"), - nested_tuple, nested_frozenset) - for value in values: - with self.subTest(value=value): - result = self.compile_constant(value) - self.assertEqual(result, value) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_assign_to_constant(self): - tree = ast.parse("x = 1") - - target = tree.body[0].targets[0] - new_target = ast.Constant(value=1) - ast.copy_location(new_target, target) - tree.body[0].targets[0] = new_target - - with self.assertRaises(ValueError) as cm: - compile(tree, "string", "exec") - self.assertEqual(str(cm.exception), - "expression which can't be assigned " - "to in Store context") - - def test_get_docstring(self): - tree = ast.parse("'docstring'\nx = 1") - self.assertEqual(ast.get_docstring(tree), 'docstring') - - def get_load_const(self, tree): - # Compile to bytecode, disassemble and get parameter of LOAD_CONST - # instructions - co = compile(tree, '', 'exec') - consts = [] - for instr in dis.get_instructions(co): - if instr.opname == 'LOAD_CONST' or instr.opname == 'RETURN_CONST': - consts.append(instr.argval) - return consts - - @support.cpython_only - def test_load_const(self): - consts = [None, - True, False, - 124, - 2.0, - 3j, - "unicode", - b'bytes', - (1, 2, 3)] - - code = '\n'.join(['x={!r}'.format(const) for const in consts]) - code += '\nx = ...' - consts.extend((Ellipsis, None)) - - tree = ast.parse(code) - self.assertEqual(self.get_load_const(tree), - consts) - - # Replace expression nodes with constants - for assign, const in zip(tree.body, consts): - assert isinstance(assign, ast.Assign), ast.dump(assign) - new_node = ast.Constant(value=const) - ast.copy_location(new_node, assign.value) - assign.value = new_node - - self.assertEqual(self.get_load_const(tree), - consts) - - def test_literal_eval(self): - tree = ast.parse("1 + 2") - binop = tree.body[0].value - - new_left = ast.Constant(value=10) - ast.copy_location(new_left, binop.left) - binop.left = new_left - - new_right = ast.Constant(value=20j) - ast.copy_location(new_right, binop.right) - binop.right = new_right - - self.assertEqual(ast.literal_eval(binop), 10+20j) - - def test_string_kind(self): - c = ast.parse('"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, None) - - c = ast.parse('u"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, "u") - - c = ast.parse('r"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, None) - - c = ast.parse('b"x"', mode='eval').body - self.assertEqual(c.value, b"x") - self.assertEqual(c.kind, None) - - -class EndPositionTests(unittest.TestCase): - """Tests for end position of AST nodes. - - Testing end positions of nodes requires a bit of extra care - because of how LL parsers work. - """ - def _check_end_pos(self, ast_node, end_lineno, end_col_offset): - self.assertEqual(ast_node.end_lineno, end_lineno) - self.assertEqual(ast_node.end_col_offset, end_col_offset) - - def _check_content(self, source, ast_node, content): - self.assertEqual(ast.get_source_segment(source, ast_node), content) - - def _parse_value(self, s): - # Use duck-typing to support both single expression - # and a right hand side of an assignment statement. - return ast.parse(s).body[0].value - - def test_lambda(self): - s = 'lambda x, *y: None' - lam = self._parse_value(s) - self._check_content(s, lam.body, 'None') - self._check_content(s, lam.args.args[0], 'x') - self._check_content(s, lam.args.vararg, 'y') - - def test_func_def(self): - s = dedent(''' - def func(x: int, - *args: str, - z: float = 0, - **kwargs: Any) -> bool: - return True - ''').strip() - fdef = ast.parse(s).body[0] - self._check_end_pos(fdef, 5, 15) - self._check_content(s, fdef.body[0], 'return True') - self._check_content(s, fdef.args.args[0], 'x: int') - self._check_content(s, fdef.args.args[0].annotation, 'int') - self._check_content(s, fdef.args.kwarg, 'kwargs: Any') - self._check_content(s, fdef.args.kwarg.annotation, 'Any') - - def test_call(self): - s = 'func(x, y=2, **kw)' - call = self._parse_value(s) - self._check_content(s, call.func, 'func') - self._check_content(s, call.keywords[0].value, '2') - self._check_content(s, call.keywords[1].value, 'kw') - - def test_call_noargs(self): - s = 'x[0]()' - call = self._parse_value(s) - self._check_content(s, call.func, 'x[0]') - self._check_end_pos(call, 1, 6) - - def test_class_def(self): - s = dedent(''' - class C(A, B): - x: int = 0 - ''').strip() - cdef = ast.parse(s).body[0] - self._check_end_pos(cdef, 2, 14) - self._check_content(s, cdef.bases[1], 'B') - self._check_content(s, cdef.body[0], 'x: int = 0') - - def test_class_kw(self): - s = 'class S(metaclass=abc.ABCMeta): pass' - cdef = ast.parse(s).body[0] - self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta') - - def test_multi_line_str(self): - s = dedent(''' - x = """Some multi-line text. - - It goes on starting from same indent.""" - ''').strip() - assign = ast.parse(s).body[0] - self._check_end_pos(assign, 3, 40) - self._check_end_pos(assign.value, 3, 40) - - def test_continued_str(self): - s = dedent(''' - x = "first part" \\ - "second part" - ''').strip() - assign = ast.parse(s).body[0] - self._check_end_pos(assign, 2, 13) - self._check_end_pos(assign.value, 2, 13) - - def test_suites(self): - # We intentionally put these into the same string to check - # that empty lines are not part of the suite. - s = dedent(''' - while True: - pass - - if one(): - x = None - elif other(): - y = None - else: - z = None - - for x, y in stuff: - assert True - - try: - raise RuntimeError - except TypeError as e: - pass - - pass - ''').strip() - mod = ast.parse(s) - while_loop = mod.body[0] - if_stmt = mod.body[1] - for_loop = mod.body[2] - try_stmt = mod.body[3] - pass_stmt = mod.body[4] - - self._check_end_pos(while_loop, 2, 8) - self._check_end_pos(if_stmt, 9, 12) - self._check_end_pos(for_loop, 12, 15) - self._check_end_pos(try_stmt, 17, 8) - self._check_end_pos(pass_stmt, 19, 4) - - self._check_content(s, while_loop.test, 'True') - self._check_content(s, if_stmt.body[0], 'x = None') - self._check_content(s, if_stmt.orelse[0].test, 'other()') - self._check_content(s, for_loop.target, 'x, y') - self._check_content(s, try_stmt.body[0], 'raise RuntimeError') - self._check_content(s, try_stmt.handlers[0].type, 'TypeError') - - def test_fstring(self): - s = 'x = f"abc {x + y} abc"' - fstr = self._parse_value(s) - binop = fstr.values[1].value - self._check_content(s, binop, 'x + y') - - def test_fstring_multi_line(self): - s = dedent(''' - f"""Some multi-line text. - { - arg_one - + - arg_two - } - It goes on...""" - ''').strip() - fstr = self._parse_value(s) - binop = fstr.values[1].value - self._check_end_pos(binop, 5, 7) - self._check_content(s, binop.left, 'arg_one') - self._check_content(s, binop.right, 'arg_two') - - def test_import_from_multi_line(self): - s = dedent(''' - from x.y.z import ( - a, b, c as c - ) - ''').strip() - imp = ast.parse(s).body[0] - self._check_end_pos(imp, 3, 1) - self._check_end_pos(imp.names[2], 2, 16) - - def test_slices(self): - s1 = 'f()[1, 2] [0]' - s2 = 'x[ a.b: c.d]' - sm = dedent(''' - x[ a.b: f () , - g () : c.d - ] - ''').strip() - i1, i2, im = map(self._parse_value, (s1, s2, sm)) - self._check_content(s1, i1.value, 'f()[1, 2]') - self._check_content(s1, i1.value.slice, '1, 2') - self._check_content(s2, i2.slice.lower, 'a.b') - self._check_content(s2, i2.slice.upper, 'c.d') - self._check_content(sm, im.slice.elts[0].upper, 'f ()') - self._check_content(sm, im.slice.elts[1].lower, 'g ()') - self._check_end_pos(im, 3, 3) - - def test_binop(self): - s = dedent(''' - (1 * 2 + (3 ) + - 4 - ) - ''').strip() - binop = self._parse_value(s) - self._check_end_pos(binop, 2, 6) - self._check_content(s, binop.right, '4') - self._check_content(s, binop.left, '1 * 2 + (3 )') - self._check_content(s, binop.left.right, '3') - - def test_boolop(self): - s = dedent(''' - if (one_condition and - (other_condition or yet_another_one)): - pass - ''').strip() - bop = ast.parse(s).body[0].test - self._check_end_pos(bop, 2, 44) - self._check_content(s, bop.values[1], - 'other_condition or yet_another_one') - - def test_tuples(self): - s1 = 'x = () ;' - s2 = 'x = 1 , ;' - s3 = 'x = (1 , 2 ) ;' - sm = dedent(''' - x = ( - a, b, - ) - ''').strip() - t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm)) - self._check_content(s1, t1, '()') - self._check_content(s2, t2, '1 ,') - self._check_content(s3, t3, '(1 , 2 )') - self._check_end_pos(tm, 3, 1) - - def test_attribute_spaces(self): - s = 'func(x. y .z)' - call = self._parse_value(s) - self._check_content(s, call, s) - self._check_content(s, call.args[0], 'x. y .z') - - def test_redundant_parenthesis(self): - s = '( ( ( a + b ) ) )' - v = ast.parse(s).body[0].value - self.assertEqual(type(v).__name__, 'BinOp') - self._check_content(s, v, 'a + b') - s2 = 'await ' + s - v = ast.parse(s2).body[0].value.value - self.assertEqual(type(v).__name__, 'BinOp') - self._check_content(s2, v, 'a + b') - - def test_trailers_with_redundant_parenthesis(self): - tests = ( - ('( ( ( a ) ) ) ( )', 'Call'), - ('( ( ( a ) ) ) ( b )', 'Call'), - ('( ( ( a ) ) ) [ b ]', 'Subscript'), - ('( ( ( a ) ) ) . b', 'Attribute'), - ) - for s, t in tests: - with self.subTest(s): - v = ast.parse(s).body[0].value - self.assertEqual(type(v).__name__, t) - self._check_content(s, v, s) - s2 = 'await ' + s - v = ast.parse(s2).body[0].value.value - self.assertEqual(type(v).__name__, t) - self._check_content(s2, v, s) - - def test_displays(self): - s1 = '[{}, {1, }, {1, 2,} ]' - s2 = '{a: b, f (): g () ,}' - c1 = self._parse_value(s1) - c2 = self._parse_value(s2) - self._check_content(s1, c1.elts[0], '{}') - self._check_content(s1, c1.elts[1], '{1, }') - self._check_content(s1, c1.elts[2], '{1, 2,}') - self._check_content(s2, c2.keys[1], 'f ()') - self._check_content(s2, c2.values[1], 'g ()') - - def test_comprehensions(self): - s = dedent(''' - x = [{x for x, y in stuff - if cond.x} for stuff in things] - ''').strip() - cmp = self._parse_value(s) - self._check_end_pos(cmp, 2, 37) - self._check_content(s, cmp.generators[0].iter, 'things') - self._check_content(s, cmp.elt.generators[0].iter, 'stuff') - self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x') - self._check_content(s, cmp.elt.generators[0].target, 'x, y') - - def test_yield_await(self): - s = dedent(''' - async def f(): - yield x - await y - ''').strip() - fdef = ast.parse(s).body[0] - self._check_content(s, fdef.body[0].value, 'yield x') - self._check_content(s, fdef.body[1].value, 'await y') - - def test_source_segment_multi(self): - s_orig = dedent(''' - x = ( - a, b, - ) + () - ''').strip() - s_tuple = dedent(''' - ( - a, b, - ) - ''').strip() - binop = self._parse_value(s_orig) - self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_source_segment_padded(self): - s_orig = dedent(''' - class C: - def fun(self) -> None: - "ЖЖЖЖЖ" - ''').strip() - s_method = ' def fun(self) -> None:\n' \ - ' "ЖЖЖЖЖ"' - cdef = ast.parse(s_orig).body[0] - self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True), - s_method) - - def test_source_segment_endings(self): - s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n' - v, w, x, y, z = ast.parse(s).body - self._check_content(s, v, 'v = 1') - self._check_content(s, w, 'w = 1') - self._check_content(s, x, 'x = 1') - self._check_content(s, y, 'y = 1') - self._check_content(s, z, 'z = 1') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_source_segment_tabs(self): - s = dedent(''' - class C: - \t\f def fun(self) -> None: - \t\f pass - ''').strip() - s_method = ' \t\f def fun(self) -> None:\n' \ - ' \t\f pass' - - cdef = ast.parse(s).body[0] - self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method) - - def test_source_segment_newlines(self): - s = 'def f():\n pass\ndef g():\r pass\r\ndef h():\r\n pass\r\n' - f, g, h = ast.parse(s).body - self._check_content(s, f, 'def f():\n pass') - self._check_content(s, g, 'def g():\r pass') - self._check_content(s, h, 'def h():\r\n pass') - - s = 'def f():\n a = 1\r b = 2\r\n c = 3\n' - f = ast.parse(s).body[0] - self._check_content(s, f, s.rstrip()) - - def test_source_segment_missing_info(self): - s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n' - v, w, x, y = ast.parse(s).body - del v.lineno - del w.end_lineno - del x.col_offset - del y.end_col_offset - self.assertIsNone(ast.get_source_segment(s, v)) - self.assertIsNone(ast.get_source_segment(s, w)) - self.assertIsNone(ast.get_source_segment(s, x)) - self.assertIsNone(ast.get_source_segment(s, y)) - -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, 'Num', node.n)) - def visit_Str(self, node): - log.append((node.lineno, 'Str', node.s)) - def visit_Bytes(self, node): - log.append((node.lineno, 'Bytes', node.s)) - def visit_NameConstant(self, node): - log.append((node.lineno, 'NameConstant', node.value)) - def visit_Ellipsis(self, node): - log.append((node.lineno, 'Ellipsis', ...)) - mod = ast.parse(dedent('''\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - ''')) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - visitor.visit(mod) - self.assertEqual(log, [ - (1, 'Num', 42), - (2, 'Num', 4.25), - (3, 'Num', 4.25j), - (4, 'Str', 'string'), - (5, 'Bytes', b'bytes'), - (6, 'NameConstant', True), - (7, 'NameConstant', None), - (8, 'Ellipsis', ...), - ]) - self.assertEqual([str(w.message) for w in wlog], [ - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Str is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Bytes is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_Ellipsis is deprecated; add visit_Constant', - ]) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer - - def assertASTTransformation(self, tranformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) - expected_ast = ast.parse(dedent(expected_code)) - - tranformer = tranformer_class() - result_ast = ast.fix_missing_locations(tranformer.visit(initial_ast)) - - self.assertASTEqual(result_ast, expected_ast) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_node_remove_single(self): - code = 'def func(arg) -> SomeType: ...' - expected = 'def func(arg): ...' - - # Since `FunctionDef.returns` is defined as a single value, we test - # the `if isinstance(old_value, AST):` branch here. - class SomeTypeRemover(ast.NodeTransformer): - def visit_Name(self, node: ast.Name): - self.generic_visit(node) - if node.id == 'SomeType': - return None - return node - - self.assertASTTransformation(SomeTypeRemover, code, expected) - - def test_node_remove_from_list(self): - code = """ - def func(arg): - print(arg) - yield arg - """ - expected = """ - def func(arg): - print(arg) - """ - - # Since `FunctionDef.body` is defined as a list, we test - # the `if isinstance(old_value, list):` branch here. - class YieldRemover(ast.NodeTransformer): - def visit_Expr(self, node: ast.Expr): - self.generic_visit(node) - if isinstance(node.value, ast.Yield): - return None # Remove `yield` from a function - return node - - self.assertASTTransformation(YieldRemover, code, expected) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_node_return_list(self): - code = """ - class DSL(Base, kw1=True): ... - """ - expected = """ - class DSL(Base, kw1=True, kw2=True, kw3=False): ... - """ - - class ExtendKeywords(ast.NodeTransformer): - def visit_keyword(self, node: ast.keyword): - self.generic_visit(node) - if node.arg == 'kw1': - return [ - node, - ast.keyword('kw2', ast.Constant(True)), - ast.keyword('kw3', ast.Constant(False)), - ] - return node - - self.assertASTTransformation(ExtendKeywords, code, expected) - - def test_node_mutate(self): - code = """ - def func(arg): - print(arg) - """ - expected = """ - def func(arg): - log(arg) - """ - - class PrintToLog(ast.NodeTransformer): - def visit_Call(self, node: ast.Call): - self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == 'print': - node.func.id = 'log' - return node - - self.assertASTTransformation(PrintToLog, code, expected) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_node_replace(self): - code = """ - def func(arg): - print(arg) - """ - expected = """ - def func(arg): - logger.log(arg, debug=True) - """ - - class PrintToLog(ast.NodeTransformer): - def visit_Call(self, node: ast.Call): - self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == 'print': - return ast.Call( - func=ast.Attribute( - ast.Name('logger', ctx=ast.Load()), - attr='log', - ctx=ast.Load(), - ), - args=node.args, - keywords=[ast.keyword('debug', ast.Constant(True))], - ) - return node - - self.assertASTTransformation(PrintToLog, code, expected) - - -@support.cpython_only -class ModuleStateTests(unittest.TestCase): - # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. - - def check_ast_module(self): - # Check that the _ast module still works as expected - code = 'x + 1' - filename = '' - mode = 'eval' - - # Create _ast.AST subclasses instances - ast_tree = compile(code, filename, mode, flags=ast.PyCF_ONLY_AST) - - # Call PyAST_Check() - code = compile(ast_tree, filename, mode) - self.assertIsInstance(code, types.CodeType) - - def test_reload_module(self): - # bpo-41194: Importing the _ast module twice must not crash. - with support.swap_item(sys.modules, '_ast', None): - del sys.modules['_ast'] - import _ast as ast1 - - del sys.modules['_ast'] - import _ast as ast2 - - self.check_ast_module() - - # Unloading the two _ast module instances must not crash. - del ast1 - del ast2 - support.gc_collect() - - self.check_ast_module() - - def test_sys_modules(self): - # bpo-41631: Test reproducing a Mercurial crash when PyAST_Check() - # imported the _ast module internally. - lazy_mod = object() - - def my_import(name, *args, **kw): - sys.modules[name] = lazy_mod - return lazy_mod - - with support.swap_item(sys.modules, '_ast', None): - del sys.modules['_ast'] - - with support.swap_attr(builtins, '__import__', my_import): - # Test that compile() does not import the _ast module - self.check_ast_module() - self.assertNotIn('_ast', sys.modules) - - # Sanity check of the test itself - import _ast - self.assertIs(_ast, lazy_mod) - - def test_subinterpreter(self): - # bpo-41631: Importing and using the _ast module in a subinterpreter - # must not crash. - code = dedent(''' - import _ast - import ast - import gc - import sys - import types - - # Create _ast.AST subclasses instances and call PyAST_Check() - ast_tree = compile('x+1', '', 'eval', - flags=ast.PyCF_ONLY_AST) - code = compile(ast_tree, 'string', 'eval') - if not isinstance(code, types.CodeType): - raise AssertionError - - # Unloading the _ast module must not crash. - del ast, _ast - del sys.modules['ast'], sys.modules['_ast'] - gc.collect() - ''') - res = support.run_in_subinterp(code) - self.assertEqual(res, 0) - - -class ASTMainTests(unittest.TestCase): - # Tests `ast.main()` function. - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_cli_file_input(self): - code = "print(1, 2, 3)" - expected = ast.dump(ast.parse(code), indent=3) - - with os_helper.temp_dir() as tmp_dir: - filename = os.path.join(tmp_dir, "test_module.py") - with open(filename, 'w', encoding='utf-8') as f: - f.write(code) - res, _ = script_helper.run_python_until_end("-m", "ast", filename) - - self.assertEqual(res.err, b"") - self.assertEqual(expected.splitlines(), - res.out.decode("utf8").splitlines()) - self.assertEqual(res.rc, 0) - - -def main(): - if __name__ != '__main__': - return - if sys.argv[1:] == ['-g']: - for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), - (eval_tests, "eval")): - print(kind+"_results = [") - for statement in statements: - tree = ast.parse(statement, "?", kind) - print("%r," % (to_tuple(tree),)) - print("]") - print("main()") - raise SystemExit - unittest.main() - -#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### -exec_results = [ -('Module', [('Expr', (1, 0, 1, 4), ('Constant', (1, 0, 1, 4), None, None))], []), -('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), -('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), -('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), -('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), -('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), -('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), -('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), -('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), -('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), -('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), -('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), -('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), -('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), -('Module', [('Global', (1, 0, 1, 8), ['v'])], []), -('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), -('Module', [('Pass', (1, 0, 1, 4))], []), -('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), -('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), -('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), -('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), -('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), -('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts'), ('ParamSpec', (1, 15, 1, 18), 'P')], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',))), ('TypeVarTuple', (1, 15, 1, 18), 'Ts'), ('ParamSpec', (1, 20, 1, 23), 'P')], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 22, 1, 25), 'Ts'), ('ParamSpec', (1, 27, 1, 30), 'P')], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), -('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts'), ('ParamSpec', (1, 16, 1, 19), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',))), ('TypeVarTuple', (1, 16, 1, 19), 'Ts'), ('ParamSpec', (1, 21, 1, 24), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 23, 1, 26), 'Ts'), ('ParamSpec', (1, 28, 1, 31), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts'), ('ParamSpec', (1, 14, 1, 17), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',))), ('TypeVarTuple', (1, 14, 1, 17), 'Ts'), ('ParamSpec', (1, 19, 1, 22), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 21, 1, 24), 'Ts'), ('ParamSpec', (1, 26, 1, 29), 'P')])], []), -] -single_results = [ -('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), -] -eval_results = [ -('Expression', ('Constant', (1, 0, 1, 4), None, None)), -('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), -('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), -('Expression', ('Dict', (1, 0, 1, 2), [], [])), -('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), -('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), -('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), -('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), -('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), -('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), -('Expression', ('Constant', (1, 0, 1, 2), 10, None)), -('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), -('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), -('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), -('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), -] -main() diff --git a/Lib/test/test_ast/__init__.py b/Lib/test/test_ast/__init__.py new file mode 100644 index 00000000000..9a89d27ba9f --- /dev/null +++ b/Lib/test/test_ast/__init__.py @@ -0,0 +1,7 @@ +import os + +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_ast/data/ast_repr.txt b/Lib/test/test_ast/data/ast_repr.txt new file mode 100644 index 00000000000..1c1985519cd --- /dev/null +++ b/Lib/test/test_ast/data/ast_repr.txt @@ -0,0 +1,214 @@ +Module(body=[Expr(value=Constant(value='module docstring', kind=None))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=arg(...), kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[arg(...), ..., arg(...)], vararg=arg(...), kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), ..., Dict(...)]), body=[Expr(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Expr(value=Constant(...))], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='object', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[Name(id='A', ctx=Load(...)), Name(id='B', ctx=Load(...))], keywords=[], body=[Pass()], decorator_list=[], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=Constant(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Return(value=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Delete(targets=[Name(id='v', ctx=Del(...))])], type_ignores=[]) +Module(body=[Assign(targets=[Name(id='v', ctx=Store(...))], value=Constant(value=1, kind=None), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Tuple(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[List(elts=[Name(...), Name(...)], ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[Assign(targets=[Subscript(value=Name(...), slice=Name(...), ctx=Store(...))], value=Name(id='c', ctx=Load(...)), type_comment=None)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AnnAssign(target=Name(id='x', ctx=Store(...)), annotation=Subscript(value=Name(...), slice=Tuple(...), ctx=Load(...)), value=None, simple=1)], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Add(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Sub(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=MatMult(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Div(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Mod(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=Pow(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=LShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=RShift(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitOr(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitXor(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=BitAnd(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[AugAssign(target=Name(id='v', ctx=Store(...)), op=FloorDiv(), value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='v', ctx=Load(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[Pass()])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[Pass(...)])])], type_ignores=[]) +Module(body=[If(test=Name(id='a', ctx=Load(...)), body=[Pass()], orelse=[If(test=Name(...), body=[Pass(...)], orelse=[If(...)])])], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...)), withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=Name(...))], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[With(items=[withitem(context_expr=Name(...), optional_vars=None), withitem(context_expr=Name(...), optional_vars=None)], body=[Pass()], type_comment=None)], type_ignores=[]) +Module(body=[Raise(exc=None, cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Name(id='Exception', ctx=Load(...)), cause=None)], type_ignores=[]) +Module(body=[Raise(exc=Call(func=Name(...), args=[Constant(...)], keywords=[]), cause=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[], orelse=[], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[], finalbody=[])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name=None, body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[]) +Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[]) +Module(body=[Global(names=['v'])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[]) +Module(body=[Pass()], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Break()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Name(id='v', ctx=Store(...)), iter=Name(id='v', ctx=Load(...)), body=[Continue()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=Tuple(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[For(target=List(elts=[Name(...), Name(...)], ctx=Store(...)), iter=Name(id='c', ctx=Load(...)), body=[Pass()], orelse=[], type_comment=None)], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...), comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Constant(...)), Expr(value=Await(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncFor(target=Name(...), iter=Name(...), body=[Expr(...)], orelse=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[AsyncWith(items=[withitem(...)], body=[Expr(...)], type_comment=None)], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[None, Constant(...)], values=[Dict(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Starred(...), Constant(...)]))], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=Yield(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=YieldFrom(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Expr(value=ListComp(...))], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[AsyncFunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[ClassDef(name='C', bases=[], keywords=[], body=[Pass()], decorator_list=[Name(id='deco1', ctx=Load(...)), ..., Call(func=Name(...), args=[Constant(...)], keywords=[])], type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Call(func=Name(...), args=[GeneratorExp(...)], keywords=[])], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Attribute(value=Attribute(...), attr='c', ctx=Load(...))], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[Expr(value=NamedExpr(target=Name(...), value=Constant(...)))], type_ignores=[]) +Module(body=[If(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[While(test=NamedExpr(target=Name(...), value=Call(...)), body=[Pass()], orelse=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), ..., arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...), arg(...)], kw_defaults=[None, None], kwarg=arg(...), defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...), arg(...)], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[Constant(...), ..., Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=None, defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[Constant(...)], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[arg(...)], args=[arg(...)], vararg=None, kwonlyargs=[arg(...)], kw_defaults=[None], kwarg=arg(...), defaults=[Constant(...), Constant(...)]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[])], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None)], value=Name(id='int', ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[TypeAlias(name=Name(id='X', ctx=Store(...)), type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))], value=Tuple(elts=[Name(...), ..., Name(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[ClassDef(name='X', bases=[], keywords=[], body=[Pass()], decorator_list=[], type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=None, default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Tuple(...), default_value=None), ..., ParamSpec(name='P', default_value=None)])], type_ignores=[]) +Module(body=[FunctionDef(name='f', args=arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[], returns=None, type_comment=None, type_params=[TypeVar(name='T', bound=Name(...), default_value=Constant(...)), ..., ParamSpec(name='P', default_value=Constant(...))])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Match(subject=Name(id='x', ctx=Load(...)), cases=[match_case(pattern=MatchValue(...), guard=None, body=[Pass(...)]), match_case(pattern=MatchAs(...), guard=None, body=[Pass(...)])])], type_ignores=[]) +Module(body=[Expr(value=Constant(value=None, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=True, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=False, kind=None))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=And(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BoolOp(op=Or(...), values=[Name(...), Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Add(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Sub(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Div(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=MatMult(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=FloorDiv(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Pow(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=Mod(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=RShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=LShift(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitXor(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitOr(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=BinOp(left=Name(...), op=BitAnd(...), right=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Not(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=UAdd(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=USub(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=UnaryOp(op=Invert(...), operand=Name(...)))], type_ignores=[]) +Module(body=[Expr(value=Lambda(args=arguments(...), body=Constant(...)))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[], values=[]))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Dict(keys=[Constant(...)], values=[Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Set(elts=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=DictComp(key=Name(...), value=Name(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=ListComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=SetComp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=GeneratorExp(elt=Tuple(...), generators=[comprehension(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Constant(...), ops=[Lt(...), Lt(...)], comparators=[Constant(...), Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Eq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[LtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[GtE(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotEq(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[Is(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[IsNot(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[In(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Compare(left=Name(...), ops=[NotIn(...)], comparators=[Name(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Constant(...), ..., Starred(...)], keywords=[keyword(...), keyword(...)]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[Starred(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Name(...), args=[GeneratorExp(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=10, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value=1j, kind=None))], type_ignores=[]) +Module(body=[Expr(value=Constant(value='string', kind=None))], type_ignores=[]) +Module(body=[Expr(value=Attribute(value=Name(...), attr='b', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=Name(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Name(id='v', ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=List(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[Constant(...), ..., Constant(...)], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Tuple(elts=[], ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Call(func=Attribute(...), args=[Subscript(...)], keywords=[]))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=Subscript(value=List(...), slice=Slice(...), ctx=Load(...)))], type_ignores=[]) +Module(body=[Expr(value=IfExp(test=Name(...), body=Call(...), orelse=Call(...)))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[FormattedValue(...)]))], type_ignores=[]) +Module(body=[Expr(value=JoinedStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Interpolation(...)]))], type_ignores=[]) +Module(body=[Expr(value=TemplateStr(values=[Constant(...), ..., Constant(...)]))], type_ignores=[]) \ No newline at end of file diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py new file mode 100644 index 00000000000..b76f98901d2 --- /dev/null +++ b/Lib/test/test_ast/snippets.py @@ -0,0 +1,612 @@ +import ast +import sys + +from test.test_ast.utils import to_tuple + + +# These tests are compiled through "exec" +# There should be at least one test per statement +exec_tests = [ + # Module docstring + "'module docstring'", + # FunctionDef + "def f(): pass", + # FunctionDef with docstring + "def f(): 'function docstring'", + # FunctionDef with arg + "def f(a): pass", + # FunctionDef with arg and default value + "def f(a=0): pass", + # FunctionDef with varargs + "def f(*args): pass", + # FunctionDef with varargs as TypeVarTuple + "def f(*args: *Ts): pass", + # FunctionDef with varargs as unpacked Tuple + "def f(*args: *tuple[int, ...]): pass", + # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple + "def f(*args: *tuple[int, *Ts]): pass", + # FunctionDef with kwargs + "def f(**kwargs): pass", + # FunctionDef with all kind of args and docstring + "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", + # FunctionDef with type annotation on return involving unpacking + "def f() -> tuple[*Ts]: pass", + "def f() -> tuple[int, *Ts]: pass", + "def f() -> tuple[int, *tuple[int, ...]]: pass", + # ClassDef + "class C:pass", + # ClassDef with docstring + "class C: 'docstring for class C'", + # ClassDef, new style class + "class C(object): pass", + # Classdef with multiple bases + "class C(A, B): pass", + # Return + "def f():return 1", + "def f():return", + # Delete + "del v", + # Assign + "v = 1", + "a,b = c", + "(a,b) = c", + "[a,b] = c", + "a[b] = c", + # AnnAssign with unpacked types + "x: tuple[*Ts]", + "x: tuple[int, *Ts]", + "x: tuple[int, *tuple[str, ...]]", + # AugAssign + "v += 1", + "v -= 1", + "v *= 1", + "v @= 1", + "v /= 1", + "v %= 1", + "v **= 1", + "v <<= 1", + "v >>= 1", + "v |= 1", + "v ^= 1", + "v &= 1", + "v //= 1", + # For + "for v in v:pass", + # For-Else + "for v in v:\n pass\nelse:\n pass", + # While + "while v:pass", + # While-Else + "while v:\n pass\nelse:\n pass", + # If-Elif-Else + "if v:pass", + "if a:\n pass\nelif b:\n pass", + "if a:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelif b:\n pass\nelif b:\n pass\nelse:\n pass", + # With + "with x: pass", + "with x, y: pass", + "with x as y: pass", + "with x as y, z as q: pass", + "with (x as y): pass", + "with (x, y): pass", + # Raise + "raise", + "raise Exception('string')", + "raise Exception", + "raise Exception('string') from None", + # TryExcept + "try:\n pass\nexcept Exception:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass", + # TryFinally + "try:\n pass\nfinally:\n pass", + # TryStarExcept + "try:\n pass\nexcept* Exception:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass", + # TryExceptFinallyElse + "try:\n pass\nexcept Exception:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + # Assert + "assert v", + # Assert with message + "assert v, 'message'", + # Import + "import sys", + "import foo as bar", + # ImportFrom + "from sys import x as y", + "from sys import v", + # Global + "global v", + # Expr + "1", + # Pass, + "pass", + # Break + "for v in v:break", + # Continue + "for v in v:continue", + # for statements with naked tuples (see http://bugs.python.org/issue6704) + "for a,b in c: pass", + "for (a,b) in c: pass", + "for [a,b] in c: pass", + # Multiline generator expression (test for .lineno & .col_offset) + """( + ( + Aa + , + Bb + ) + for + Aa + , + Bb in Cc + )""", + # dictcomp + "{a : b for w in x for m in p if g}", + # dictcomp with naked tuple + "{a : b for v,w in x}", + # setcomp + "{r for l in x if g}", + # setcomp with naked tuple + "{r for l,m in x}", + # AsyncFunctionDef + "async def f():\n 'async function'\n await something()", + # AsyncFor + "async def f():\n async for e in i: 1\n else: 2", + # AsyncWith + "async def f():\n async with a as b: 1", + # PEP 448: Additional Unpacking Generalizations + "{**{1:2}, 2:3}", + "{*{1, 2}, 3}", + # Function with yield (from) + "def f(): yield 1", + "def f(): yield from []", + # Asynchronous comprehensions + "async def f():\n [i async for b in c]", + # Decorated FunctionDef + "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", + # Decorated AsyncFunctionDef + "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", + # Decorated ClassDef + "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", + # Decorator with generator argument + "@deco(a for a in b)\ndef f(): pass", + # Decorator with attribute + "@a.b.c\ndef f(): pass", + # Simple assignment expression + "(a := 1)", + # Assignment expression in if statement + "if a := foo(): pass", + # Assignment expression in while + "while a := foo(): pass", + # Positional-only arguments + "def f(a, /,): pass", + "def f(a, /, c, d, e): pass", + "def f(a, /, c, *, d, e): pass", + "def f(a, /, c, *, d, e, **kwargs): pass", + # Positional-only arguments with defaults + "def f(a=1, /,): pass", + "def f(a=1, /, b=2, c=4): pass", + "def f(a=1, /, b=2, *, c=4): pass", + "def f(a=1, /, b=2, *, c): pass", + "def f(a=1, /, b=2, *, c=4, **kwargs): pass", + "def f(a=1, /, b=2, *, c, **kwargs): pass", + # Type aliases + "type X = int", + "type X[T] = int", + "type X[T, *Ts, **P] = (T, Ts, P)", + "type X[T: int, *Ts, **P] = (T, Ts, P)", + "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", + "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", + # Generic classes + "class X[T]: pass", + "class X[T, *Ts, **P]: pass", + "class X[T: int, *Ts, **P]: pass", + "class X[T: (int, str), *Ts, **P]: pass", + "class X[T: int = 1, *Ts = 2, **P = 3]: pass", + # Generic functions + "def f[T](): pass", + "def f[T, *Ts, **P](): pass", + "def f[T: int, *Ts, **P](): pass", + "def f[T: (int, str), *Ts, **P](): pass", + "def f[T: int = 1, *Ts = 2, **P = 3](): pass", + # Match + "match x:\n\tcase 1:\n\t\tpass", + # Match with _ + "match x:\n\tcase 1:\n\t\tpass\n\tcase _:\n\t\tpass", +] + +# These are compiled through "single" +# because of overlap with "eval", it just tests what +# can't be tested with "eval" +single_tests = [ + "1+2" +] + +# These are compiled through "eval" +# It should test all expressions +eval_tests = [ + # Constant(value=None) + "None", + # True + "True", + # False + "False", + # BoolOp + "a and b", + "a or b", + # BinOp + "a + b", + "a - b", + "a * b", + "a / b", + "a @ b", + "a // b", + "a ** b", + "a % b", + "a >> b", + "a << b", + "a ^ b", + "a | b", + "a & b", + # UnaryOp + "not v", + "+v", + "-v", + "~v", + # Lambda + "lambda:None", + # Dict + "{ 1:2 }", + # Empty dict + "{}", + # Set + "{None,}", + # Multiline dict (test for .lineno & .col_offset) + """{ + 1 + : + 2 + }""", + # Multiline list + """[ + 1 + , + 1 + ]""", + # Multiline tuple + """( + 1 + , + )""", + # Multiline set + """{ + 1 + , + 1 + }""", + # ListComp + "[a for b in c if d]", + # GeneratorExp + "(a for b in c if d)", + # SetComp + "{a for b in c if d}", + # DictComp + "{k: v for k, v in c if d}", + # Comprehensions with multiple for targets + "[(a,b) for a,b in c]", + "[(a,b) for (a,b) in c]", + "[(a,b) for [a,b] in c]", + "{(a,b) for a,b in c}", + "{(a,b) for (a,b) in c}", + "{(a,b) for [a,b] in c}", + "((a,b) for a,b in c)", + "((a,b) for (a,b) in c)", + "((a,b) for [a,b] in c)", + # Async comprehensions - async comprehensions can't work outside an asynchronous function + # + # Yield - yield expressions can't work outside a function + # + # Compare + "1 < 2 < 3", + "a == b", + "a <= b", + "a >= b", + "a != b", + "a is b", + "a is not b", + "a in b", + "a not in b", + # Call without argument + "f()", + # Call + "f(1,2,c=3,*d,**e)", + # Call with multi-character starred + "f(*[0, 1])", + # Call with a generator argument + "f(a for a in b)", + # Constant(value=int()) + "10", + # Complex num + "1j", + # Constant(value=str()) + "'string'", + # Attribute + "a.b", + # Subscript + "a[b:c]", + # Name + "v", + # List + "[1,2,3]", + # Empty list + "[]", + # Tuple + "1,2,3", + # Tuple + "(1,2,3)", + # Empty tuple + "()", + # Combination + "a.b.c.d(a.b[1:2])", + # Slice + "[5][1:]", + "[5][:1]", + "[5][::1]", + "[5][1:1:1]", + # IfExp + "foo() if x else bar()", + # JoinedStr and FormattedValue + "f'{a}'", + "f'{a:.2f}'", + "f'{a!r}'", + "f'foo({a})'", + # TemplateStr and Interpolation + "t'{a}'", + "t'{a:.2f}'", + "t'{a!r}'", + "t'{a!r:.2f}'", + "t'foo({a})'", +] + + +def main(): + if __name__ != '__main__': + return + if sys.argv[1:] == ['-g']: + for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), + (eval_tests, "eval")): + print(kind+"_results = [") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) + print("]") + print("main()") + raise SystemExit + +#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast/snippets.py -g ##### +exec_results = [ +('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), +('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 19), 'C', [('Name', (1, 8, 1, 9), 'A', ('Load',)), ('Name', (1, 11, 1, 12), 'B', ('Load',))], [], [('Pass', (1, 15, 1, 19))], [], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 14), None)], [], None, None, [])], []), +('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), +('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), +('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 8), [('Subscript', (1, 0, 1, 4), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Store',))], ('Name', (1, 7, 1, 8), 'c', ('Load',)), None)], []), +('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Sub',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('MatMult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Div',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mod',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Pow',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('LShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('RShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitOr',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitXor',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitAnd',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('FloorDiv',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), +('Module', [('For', (1, 0, 4, 6), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))], None)], []), +('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), +('Module', [('While', (1, 0, 4, 6), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), +('Module', [('If', (1, 0, 10, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 10, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('If', (5, 0, 10, 6), ('Name', (5, 5, 5, 6), 'b', ('Load',)), [('Pass', (6, 2, 6, 6))], [('If', (7, 0, 10, 6), ('Name', (7, 5, 7, 6), 'b', ('Load',)), [('Pass', (8, 2, 8, 6))], [('Pass', (10, 2, 10, 6))])])])])], []), +('Module', [('With', (1, 0, 1, 12), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None)], [('Pass', (1, 8, 1, 12))], None)], []), +('Module', [('With', (1, 0, 1, 15), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None), ('withitem', ('Name', (1, 8, 1, 9), 'y', ('Load',)), None)], [('Pass', (1, 11, 1, 15))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), +('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('Raise', (1, 0, 1, 5), None, None)], []), +('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), +('Module', [('Raise', (1, 0, 1, 15), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), None)], []), +('Module', [('Raise', (1, 0, 1, 35), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), ('Constant', (1, 31, 1, 35), None, None))], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), +('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), +('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), +('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), +('Module', [('Global', (1, 0, 1, 8), ['v'])], []), +('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), +('Module', [('Pass', (1, 0, 1, 4))], []), +('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), +('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), +('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), +('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 16), ('Yield', (1, 9, 1, 16), ('Constant', (1, 15, 1, 16), 1, None)))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 22), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 22), ('YieldFrom', (1, 9, 1, 22), ('List', (1, 20, 1, 22), [], ('Load',))))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), +('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), +('Module', [('If', (1, 0, 1, 19), ('NamedExpr', (1, 3, 1, 13), ('Name', (1, 3, 1, 4), 'a', ('Store',)), ('Call', (1, 8, 1, 13), ('Name', (1, 8, 1, 11), 'foo', ('Load',)), [], [])), [('Pass', (1, 15, 1, 19))], [])], []), +('Module', [('While', (1, 0, 1, 22), ('NamedExpr', (1, 6, 1, 16), ('Name', (1, 6, 1, 7), 'a', ('Store',)), ('Call', (1, 11, 1, 16), ('Name', (1, 11, 1, 14), 'foo', ('Load',)), [], [])), [('Pass', (1, 18, 1, 22))], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), +('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), +('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), +('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), +('Module', [('Match', (1, 0, 3, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))])])], []), +('Module', [('Match', (1, 0, 5, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))]), ('match_case', ('MatchAs', (4, 6, 4, 7), None, None), None, [('Pass', (5, 2, 5, 6))])])], []), +] +single_results = [ +('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), +] +eval_results = [ +('Expression', ('Constant', (1, 0, 1, 4), None, None)), +('Expression', ('Constant', (1, 0, 1, 4), True, None)), +('Expression', ('Constant', (1, 0, 1, 5), False, None)), +('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), +('Expression', ('BoolOp', (1, 0, 1, 6), ('Or',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Sub',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Div',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('MatMult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('FloorDiv',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Pow',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mod',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('RShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('LShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitXor',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitOr',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitAnd',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('UAdd',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('USub',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('Invert',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), +('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), +('Expression', ('Dict', (1, 0, 1, 2), [], [])), +('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), +('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), +('Expression', ('List', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 4, 6), [('Constant', (2, 6, 2, 7), 1, None)], ('Load',))), +('Expression', ('Set', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)])), +('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('DictComp', (1, 0, 1, 25), ('Name', (1, 1, 1, 2), 'k', ('Load',)), ('Name', (1, 4, 1, 5), 'v', ('Load',)), [('comprehension', ('Tuple', (1, 10, 1, 14), [('Name', (1, 10, 1, 11), 'k', ('Store',)), ('Name', (1, 13, 1, 14), 'v', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [('Name', (1, 23, 1, 24), 'd', ('Load',))], 0)])), +('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Eq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('LtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('GtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotEq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Is',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('IsNot',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('In',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotIn',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Call', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [], [])), +('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), +('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), +('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), +('Expression', ('Constant', (1, 0, 1, 2), 10, None)), +('Expression', ('Constant', (1, 0, 1, 2), 1j, None)), +('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), +('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), +('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), +('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), ('Constant', (1, 4, 1, 5), 1, None), None, None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), None, ('Constant', (1, 5, 1, 6), 1, None), None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 8), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 7), None, None, ('Constant', (1, 6, 1, 7), 1, None)), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 10), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 9), ('Constant', (1, 4, 1, 5), 1, None), ('Constant', (1, 6, 1, 7), 1, None), ('Constant', (1, 8, 1, 9), 1, None)), ('Load',))), +('Expression', ('IfExp', (1, 0, 1, 21), ('Name', (1, 9, 1, 10), 'x', ('Load',)), ('Call', (1, 0, 1, 5), ('Name', (1, 0, 1, 3), 'foo', ('Load',)), [], []), ('Call', (1, 16, 1, 21), ('Name', (1, 16, 1, 19), 'bar', ('Load',)), [], []))), +('Expression', ('JoinedStr', (1, 0, 1, 6), [('FormattedValue', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), +('Expression', ('TemplateStr', (1, 0, 1, 6), [('Interpolation', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', -1, None)])), +('Expression', ('TemplateStr', (1, 0, 1, 10), [('Interpolation', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('TemplateStr', (1, 0, 1, 8), [('Interpolation', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', 114, None)])), +('Expression', ('TemplateStr', (1, 0, 1, 12), [('Interpolation', (1, 2, 1, 11), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 'a', 114, ('JoinedStr', (1, 6, 1, 10), [('Constant', (1, 7, 1, 10), '.2f', None)]))])), +('Expression', ('TemplateStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('Interpolation', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), 'a', -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), +] +main() diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py new file mode 100644 index 00000000000..e2e619c5a23 --- /dev/null +++ b/Lib/test/test_ast/test_ast.py @@ -0,0 +1,3874 @@ +import _ast_unparse +import ast +import builtins +import contextlib +import copy +import dis +import enum +import itertools +import os +import re +import sys +import tempfile +import textwrap +import types +import unittest +import weakref +from io import StringIO +from pathlib import Path +from textwrap import dedent +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + +from test import support +from test.support import os_helper +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, skip_if_unlimited_stack_size +from test.support.ast_helper import ASTTestMixin +from test.support.import_helper import ensure_lazy_imports +from test.test_ast.utils import to_tuple +from test.test_ast.snippets import ( + eval_tests, eval_results, exec_tests, exec_results, single_tests, single_results +) + + +STDLIB = os.path.dirname(ast.__file__) +STDLIB_FILES = [fn for fn in os.listdir(STDLIB) if fn.endswith(".py")] +STDLIB_FILES.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) + +AST_REPR_DATA_FILE = Path(__file__).parent / "data" / "ast_repr.txt" + +def ast_repr_get_test_cases() -> list[str]: + return exec_tests + eval_tests + + +def ast_repr_update_snapshots() -> None: + data = [repr(ast.parse(test)) for test in ast_repr_get_test_cases()] + AST_REPR_DATA_FILE.write_text("\n".join(data)) + + +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("ast", {"contextlib", "enum", "inspect", "re", "collections", "argparse"}) + + +class AST_Tests(unittest.TestCase): + maxDiff = None + + def _is_ast_node(self, name, node): + if not isinstance(node, type): + return False + if "ast" not in node.__module__: + return False + return name != 'AST' and name[0].isupper() + + def _assertTrueorder(self, ast_node, parent_pos): + if not isinstance(ast_node, ast.AST) or ast_node._fields is None: + return + if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)): + node_pos = (ast_node.lineno, ast_node.col_offset) + self.assertGreaterEqual(node_pos, parent_pos) + parent_pos = (ast_node.lineno, ast_node.col_offset) + for name in ast_node._fields: + value = getattr(ast_node, name) + if isinstance(value, list): + first_pos = parent_pos + if value and name == 'decorator_list': + first_pos = (value[0].lineno, value[0].col_offset) + for child in value: + self._assertTrueorder(child, first_pos) + elif value is not None: + self._assertTrueorder(value, parent_pos) + self.assertEqual(ast_node._fields, ast_node.__match_args__) + + def test_AST_objects(self): + x = ast.AST() + self.assertEqual(x._fields, ()) + x.foobar = 42 + self.assertEqual(x.foobar, 42) + self.assertEqual(x.__dict__["foobar"], 42) + + with self.assertRaises(AttributeError): + x.vararg + + with self.assertRaises(TypeError): + # "ast.AST constructor takes 0 positional arguments" + ast.AST(2) + + def test_AST_fields_NULL_check(self): + # See: https://github.com/python/cpython/issues/126105 + old_value = ast.AST._fields + + def cleanup(): + ast.AST._fields = old_value + self.addCleanup(cleanup) + + del ast.AST._fields + + msg = "type object 'ast.AST' has no attribute '_fields'" + # Both examples used to crash: + with self.assertRaisesRegex(AttributeError, msg): + ast.AST(arg1=123) + with self.assertRaisesRegex(AttributeError, msg): + ast.AST() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: .X object at 0x7e85c3a80> is not None + def test_AST_garbage_collection(self): + class X: + pass + a = ast.AST() + a.x = X() + a.x.a = a + ref = weakref.ref(a.x) + del a + support.gc_collect() + self.assertIsNone(ref()) + + def test_snippets(self): + for input, output, kind in ((exec_tests, exec_results, "exec"), + (single_tests, single_results, "single"), + (eval_tests, eval_results, "eval")): + for i, o in zip(input, output): + with self.subTest(action="parsing", input=i): + ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) + self.assertEqual(to_tuple(ast_tree), o) + self._assertTrueorder(ast_tree, (0, 0)) + with self.subTest(action="compiling", input=i, kind=kind): + compile(ast_tree, "?", kind) + + def test_ast_validation(self): + # compile() is the only function that calls PyAST_Validate + snippets_to_validate = exec_tests + single_tests + eval_tests + for snippet in snippets_to_validate: + tree = ast.parse(snippet) + compile(tree, '', 'exec') + + def test_parse_invalid_ast(self): + # see gh-130139 + for optval in (-1, 0, 1, 2): + self.assertRaises(TypeError, ast.parse, ast.Constant(42), + optimize=optval) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_optimization_levels__debug__(self): + cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] + for (optval, expected) in cases: + with self.subTest(optval=optval, expected=expected): + res1 = ast.parse("__debug__", optimize=optval) + res2 = ast.parse(ast.parse("__debug__"), optimize=optval) + for res in [res1, res2]: + self.assertIsInstance(res.body[0], ast.Expr) + if isinstance(expected, bool): + self.assertIsInstance(res.body[0].value, ast.Constant) + self.assertEqual(res.body[0].value.value, expected) + else: + self.assertIsInstance(res.body[0].value, ast.Name) + self.assertEqual(res.body[0].value.id, expected) + + def test_invalid_position_information(self): + invalid_linenos = [ + (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) + ] + + for lineno, end_lineno in invalid_linenos: + with self.subTest(f"Check invalid linenos {lineno}:{end_lineno}"): + snippet = "a = 1" + tree = ast.parse(snippet) + tree.body[0].lineno = lineno + tree.body[0].end_lineno = end_lineno + with self.assertRaises(ValueError): + compile(tree, '', 'exec') + + invalid_col_offsets = [ + (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) + ] + for col_offset, end_col_offset in invalid_col_offsets: + with self.subTest(f"Check invalid col_offset {col_offset}:{end_col_offset}"): + snippet = "a = 1" + tree = ast.parse(snippet) + tree.body[0].col_offset = col_offset + tree.body[0].end_col_offset = end_col_offset + with self.assertRaises(ValueError): + compile(tree, '', 'exec') + + def test_compilation_of_ast_nodes_with_default_end_position_values(self): + tree = ast.Module(body=[ + ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), + ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) + ], type_ignores=[]) + + # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. + compile(tree, "", "exec") + + def test_negative_locations_for_compile(self): + # See https://github.com/python/cpython/issues/130775 + alias = ast.alias(name='traceback', lineno=0, col_offset=0) + for attrs in ( + {'lineno': -2, 'col_offset': 0}, + {'lineno': 0, 'col_offset': -2}, + {'lineno': 0, 'col_offset': -2, 'end_col_offset': -2}, + {'lineno': -2, 'end_lineno': -2, 'col_offset': 0}, + ): + with self.subTest(attrs=attrs): + tree = ast.Module(body=[ + ast.Import(names=[alias], **attrs) + ], type_ignores=[]) + + # It used to crash on this step: + compile(tree, "", "exec") + + # This also must not crash: + ast.parse(tree, optimize=2) + + def test_docstring_optimization_single_node(self): + # https://github.com/python/cpython/issues/137308 + class_example1 = textwrap.dedent(''' + class A: + """Docstring""" + ''') + class_example2 = textwrap.dedent(''' + class A: + """ + Docstring""" + ''') + def_example1 = textwrap.dedent(''' + def some(): + """Docstring""" + ''') + def_example2 = textwrap.dedent(''' + def some(): + """Docstring + """ + ''') + async_def_example1 = textwrap.dedent(''' + async def some(): + """Docstring""" + ''') + async_def_example2 = textwrap.dedent(''' + async def some(): + """ + Docstring + """ + ''') + for code in [ + class_example1, + class_example2, + def_example1, + def_example2, + async_def_example1, + async_def_example2, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + self.assertEqual(len(mod.body[0].body), 1) + if opt_level == 2: + pass_stmt = mod.body[0].body[0] + self.assertIsInstance(pass_stmt, ast.Pass) + self.assertEqual( + vars(pass_stmt), + { + 'lineno': 3, + 'col_offset': 4, + 'end_lineno': 3, + 'end_col_offset': 8, + }, + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + + def test_docstring_optimization_multiple_nodes(self): + # https://github.com/python/cpython/issues/137308 + class_example = textwrap.dedent( + """ + class A: + ''' + Docstring + ''' + x = 1 + """ + ) + + def_example = textwrap.dedent( + """ + def some(): + ''' + Docstring + + ''' + x = 1 + """ + ) + + async_def_example = textwrap.dedent( + """ + async def some(): + + '''Docstring + + ''' + x = 1 + """ + ) + + for code in [ + class_example, + def_example, + async_def_example, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + if opt_level == 2: + self.assertNotIsInstance( + mod.body[0].body[0], + (ast.Pass, ast.Expr), + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + + def test_slice(self): + slc = ast.parse("x[::]").body[0].value.slice + self.assertIsNone(slc.upper) + self.assertIsNone(slc.lower) + self.assertIsNone(slc.step) + + def test_from_import(self): + im = ast.parse("from . import y").body[0] + self.assertIsNone(im.module) + + def test_non_interned_future_from_ast(self): + mod = ast.parse("from __future__ import division") + self.assertIsInstance(mod.body[0], ast.ImportFrom) + mod.body[0].module = " __future__ ".strip() + compile(mod, "", "exec") + + def test_alias(self): + im = ast.parse("from bar import y").body[0] + self.assertEqual(len(im.names), 1) + alias = im.names[0] + self.assertEqual(alias.name, 'y') + self.assertIsNone(alias.asname) + self.assertEqual(alias.lineno, 1) + self.assertEqual(alias.end_lineno, 1) + self.assertEqual(alias.col_offset, 16) + self.assertEqual(alias.end_col_offset, 17) + + im = ast.parse("from bar import *").body[0] + alias = im.names[0] + self.assertEqual(alias.name, '*') + self.assertIsNone(alias.asname) + self.assertEqual(alias.lineno, 1) + self.assertEqual(alias.end_lineno, 1) + self.assertEqual(alias.col_offset, 16) + self.assertEqual(alias.end_col_offset, 17) + + im = ast.parse("from bar import y as z").body[0] + alias = im.names[0] + self.assertEqual(alias.name, "y") + self.assertEqual(alias.asname, "z") + self.assertEqual(alias.lineno, 1) + self.assertEqual(alias.end_lineno, 1) + self.assertEqual(alias.col_offset, 16) + self.assertEqual(alias.end_col_offset, 22) + + im = ast.parse("import bar as foo").body[0] + alias = im.names[0] + self.assertEqual(alias.name, "bar") + self.assertEqual(alias.asname, "foo") + self.assertEqual(alias.lineno, 1) + self.assertEqual(alias.end_lineno, 1) + self.assertEqual(alias.col_offset, 7) + self.assertEqual(alias.end_col_offset, 17) + + def test_base_classes(self): + self.assertIsSubclass(ast.For, ast.stmt) + self.assertIsSubclass(ast.Name, ast.expr) + self.assertIsSubclass(ast.stmt, ast.AST) + self.assertIsSubclass(ast.expr, ast.AST) + self.assertIsSubclass(ast.comprehension, ast.AST) + self.assertIsSubclass(ast.Gt, ast.AST) + + def test_field_attr_existence(self): + for name, item in ast.__dict__.items(): + # constructor has a different signature + if name == 'Index': + continue + if self._is_ast_node(name, item): + x = self._construct_ast_class(item) + if isinstance(x, ast.AST): + self.assertIs(type(x._fields), tuple) + + def _construct_ast_class(self, cls): + kwargs = {} + for name, typ in cls.__annotations__.items(): + if typ is str: + kwargs[name] = 'capybara' + elif typ is int: + kwargs[name] = 42 + elif typ is object: + kwargs[name] = b'capybara' + elif isinstance(typ, type) and issubclass(typ, ast.AST): + kwargs[name] = self._construct_ast_class(typ) + return cls(**kwargs) + + def test_arguments(self): + x = ast.arguments() + self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', + 'kw_defaults', 'kwarg', 'defaults')) + self.assertEqual(ast.arguments.__annotations__, { + 'posonlyargs': list[ast.arg], + 'args': list[ast.arg], + 'vararg': ast.arg | None, + 'kwonlyargs': list[ast.arg], + 'kw_defaults': list[ast.expr], + 'kwarg': ast.arg | None, + 'defaults': list[ast.expr], + }) + + self.assertEqual(x.args, []) + self.assertIsNone(x.vararg) + + x = ast.arguments(*range(1, 8)) + self.assertEqual(x.args, 2) + self.assertEqual(x.vararg, 3) + + def test_field_attr_writable(self): + x = ast.Constant(1) + # We can assign to _fields + x._fields = 666 + self.assertEqual(x._fields, 666) + + def test_classattrs(self): + with self.assertWarns(DeprecationWarning): + x = ast.Constant() + self.assertEqual(x._fields, ('value', 'kind')) + + with self.assertRaises(AttributeError): + x.value + + x = ast.Constant(42) + self.assertEqual(x.value, 42) + + with self.assertRaises(AttributeError): + x.lineno + + with self.assertRaises(AttributeError): + x.foobar + + x = ast.Constant(lineno=2, value=3) + self.assertEqual(x.lineno, 2) + + x = ast.Constant(42, lineno=0) + self.assertEqual(x.lineno, 0) + self.assertEqual(x._fields, ('value', 'kind')) + self.assertEqual(x.value, 42) + + self.assertRaises(TypeError, ast.Constant, 1, None, 2) + self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) + + # Arbitrary keyword arguments are supported (but deprecated) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') + + with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): + ast.Constant(1, value=2) + + self.assertEqual(ast.Constant(42).value, 42) + self.assertEqual(ast.Constant(4.25).value, 4.25) + self.assertEqual(ast.Constant(4.25j).value, 4.25j) + self.assertEqual(ast.Constant('42').value, '42') + self.assertEqual(ast.Constant(b'42').value, b'42') + self.assertIs(ast.Constant(True).value, True) + self.assertIs(ast.Constant(False).value, False) + self.assertIs(ast.Constant(None).value, None) + self.assertIs(ast.Constant(...).value, ...) + + def test_constant_subclasses(self): + class N(ast.Constant): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.z = 'spam' + class N2(ast.Constant): + pass + + n = N(42) + self.assertEqual(n.value, 42) + self.assertEqual(n.z, 'spam') + self.assertEqual(type(n), N) + self.assertTrue(isinstance(n, N)) + self.assertTrue(isinstance(n, ast.Constant)) + self.assertFalse(isinstance(n, N2)) + self.assertFalse(isinstance(ast.Constant(42), N)) + n = N(value=42) + self.assertEqual(n.value, 42) + self.assertEqual(type(n), N) + + def test_module(self): + body = [ast.Constant(42)] + x = ast.Module(body, []) + self.assertEqual(x.body, body) + + def test_nodeclasses(self): + # Zero arguments constructor explicitly allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp() + self.assertEqual(x._fields, ('left', 'op', 'right')) + + # Random attribute allowed too + x.foobarbaz = 5 + self.assertEqual(x.foobarbaz, 5) + + n1 = ast.Constant(1) + n3 = ast.Constant(3) + addop = ast.Add() + x = ast.BinOp(n1, addop, n3) + self.assertEqual(x.left, n1) + self.assertEqual(x.op, addop) + self.assertEqual(x.right, n3) + + x = ast.BinOp(1, 2, 3) + self.assertEqual(x.left, 1) + self.assertEqual(x.op, 2) + self.assertEqual(x.right, 3) + + x = ast.BinOp(1, 2, 3, lineno=0) + self.assertEqual(x.left, 1) + self.assertEqual(x.op, 2) + self.assertEqual(x.right, 3) + self.assertEqual(x.lineno, 0) + + # node raises exception when given too many arguments + self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4) + # node raises exception when given too many arguments + self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4, lineno=0) + + # can set attributes through kwargs too + x = ast.BinOp(left=1, op=2, right=3, lineno=0) + self.assertEqual(x.left, 1) + self.assertEqual(x.op, 2) + self.assertEqual(x.right, 3) + self.assertEqual(x.lineno, 0) + + # Random kwargs also allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp(1, 2, 3, foobarbaz=42) + self.assertEqual(x.foobarbaz, 42) + + def test_no_fields(self): + # this used to fail because Sub._fields was None + x = ast.Sub() + self.assertEqual(x._fields, ()) + + def test_invalid_sum(self): + pos = dict(lineno=2, col_offset=3) + m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) + with self.assertRaises(TypeError) as cm: + compile(m, "", "exec") + self.assertIn("but got expr()", str(cm.exception)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: expected str for name + def test_invalid_identifier(self): + m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], []) + ast.fix_missing_locations(m) + with self.assertRaises(TypeError) as cm: + compile(m, "", "exec") + self.assertIn("identifier must be of type str", str(cm.exception)) + + def test_invalid_constant(self): + for invalid_constant in int, (1, 2, int), frozenset((1, 2, int)): + e = ast.Expression(body=ast.Constant(invalid_constant)) + ast.fix_missing_locations(e) + with self.assertRaisesRegex( + TypeError, "invalid type in Constant: type" + ): + compile(e, "", "eval") + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None + def test_empty_yield_from(self): + # Issue 16546: yield from value is not optional. + empty_yield_from = ast.parse("def f():\n yield from g()") + empty_yield_from.body[0].body[0].value.value = None + with self.assertRaises(ValueError) as cm: + compile(empty_yield_from, "", "exec") + self.assertIn("field 'value' is required", str(cm.exception)) + + @support.cpython_only + def test_issue31592(self): + # There shouldn't be an assertion failure in case of a bad + # unicodedata.normalize(). + import unicodedata + def bad_normalize(*args): + return None + with support.swap_attr(unicodedata, 'normalize', bad_normalize): + self.assertRaises(TypeError, ast.parse, '\u03D5') + + def test_issue18374_binop_col_offset(self): + tree = ast.parse('4+5+6+7') + parent_binop = tree.body[0].value + child_binop = parent_binop.left + grandchild_binop = child_binop.left + self.assertEqual(parent_binop.col_offset, 0) + self.assertEqual(parent_binop.end_col_offset, 7) + self.assertEqual(child_binop.col_offset, 0) + self.assertEqual(child_binop.end_col_offset, 5) + self.assertEqual(grandchild_binop.col_offset, 0) + self.assertEqual(grandchild_binop.end_col_offset, 3) + + tree = ast.parse('4+5-\\\n 6-7') + parent_binop = tree.body[0].value + child_binop = parent_binop.left + grandchild_binop = child_binop.left + self.assertEqual(parent_binop.col_offset, 0) + self.assertEqual(parent_binop.lineno, 1) + self.assertEqual(parent_binop.end_col_offset, 4) + self.assertEqual(parent_binop.end_lineno, 2) + + self.assertEqual(child_binop.col_offset, 0) + self.assertEqual(child_binop.lineno, 1) + self.assertEqual(child_binop.end_col_offset, 2) + self.assertEqual(child_binop.end_lineno, 2) + + self.assertEqual(grandchild_binop.col_offset, 0) + self.assertEqual(grandchild_binop.lineno, 1) + self.assertEqual(grandchild_binop.end_col_offset, 3) + self.assertEqual(grandchild_binop.end_lineno, 1) + + def test_issue39579_dotted_name_end_col_offset(self): + tree = ast.parse('@a.b.c\ndef f(): pass') + attr_b = tree.body[0].decorator_list[0].value + self.assertEqual(attr_b.end_col_offset, 4) + + def test_ast_asdl_signature(self): + self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)") + self.assertEqual(ast.GtE.__doc__, "GtE") + self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)") + self.assertEqual(ast.cmpop.__doc__, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn") + expressions = [f" | {node.__doc__}" for node in ast.expr.__subclasses__()] + expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" + self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) + + def test_compare_basics(self): + self.assertTrue(ast.compare(ast.parse("x = 10"), ast.parse("x = 10"))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse(""))) + self.assertFalse(ast.compare(ast.parse("x = 10"), ast.parse("x"))) + self.assertFalse( + ast.compare(ast.parse("x = 10;y = 20"), ast.parse("class C:pass")) + ) + + def test_compare_modified_ast(self): + # The ast API is a bit underspecified. The objects are mutable, + # and even _fields and _attributes are mutable. The compare() does + # some simple things to accommodate mutability. + a = ast.parse("m * x + b", mode="eval") + b = ast.parse("m * x + b", mode="eval") + self.assertTrue(ast.compare(a, b)) + + a._fields = a._fields + ("spam",) + a.spam = "Spam" + self.assertNotEqual(a._fields, b._fields) + self.assertFalse(ast.compare(a, b)) + self.assertFalse(ast.compare(b, a)) + + b._fields = a._fields + b.spam = a.spam + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(b, a)) + + b._attributes = b._attributes + ("eggs",) + b.eggs = "eggs" + self.assertNotEqual(a._attributes, b._attributes) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + self.assertFalse(ast.compare(b, a, compare_attributes=True)) + + a._attributes = b._attributes + a.eggs = b.eggs + self.assertTrue(ast.compare(a, b, compare_attributes=True)) + self.assertTrue(ast.compare(b, a, compare_attributes=True)) + + def test_compare_literals(self): + constants = ( + -20, + 20, + 20.0, + 1, + 1.0, + True, + 0, + False, + frozenset(), + tuple(), + "ABCD", + "abcd", + "中文字", + 1e1000, + -1e1000, + ) + for next_index, constant in enumerate(constants[:-1], 1): + next_constant = constants[next_index] + with self.subTest(literal=constant, next_literal=next_constant): + self.assertTrue( + ast.compare(ast.Constant(constant), ast.Constant(constant)) + ) + self.assertFalse( + ast.compare( + ast.Constant(constant), ast.Constant(next_constant) + ) + ) + + same_looking_literal_cases = [ + {1, 1.0, True, 1 + 0j}, + {0, 0.0, False, 0 + 0j}, + ] + for same_looking_literals in same_looking_literal_cases: + for literal in same_looking_literals: + for same_looking_literal in same_looking_literals - {literal}: + self.assertFalse( + ast.compare( + ast.Constant(literal), + ast.Constant(same_looking_literal), + ) + ) + + def test_compare_fieldless(self): + self.assertTrue(ast.compare(ast.Add(), ast.Add())) + self.assertFalse(ast.compare(ast.Sub(), ast.Add())) + + # test that missing runtime fields is handled in ast.compare() + a1, a2 = ast.Name('a'), ast.Name('a') + self.assertTrue(ast.compare(a1, a2)) + self.assertTrue(ast.compare(a1, a2)) + del a1.id + self.assertFalse(ast.compare(a1, a2)) + del a2.id + self.assertTrue(ast.compare(a1, a2)) + + def test_compare_modes(self): + for mode, sources in ( + ("exec", exec_tests), + ("eval", eval_tests), + ("single", single_tests), + ): + for source in sources: + a = ast.parse(source, mode=mode) + b = ast.parse(source, mode=mode) + self.assertTrue( + ast.compare(a, b), f"{ast.dump(a)} != {ast.dump(b)}" + ) + + def test_compare_attributes_option(self): + def parse(a, b): + return ast.parse(a), ast.parse(b) + + a, b = parse("2 + 2", "2+2") + self.assertTrue(ast.compare(a, b)) + self.assertTrue(ast.compare(a, b, compare_attributes=False)) + self.assertFalse(ast.compare(a, b, compare_attributes=True)) + + def test_compare_attributes_option_missing_attribute(self): + # test that missing runtime attributes is handled in ast.compare() + a1, a2 = ast.Name('a', lineno=1), ast.Name('a', lineno=1) + self.assertTrue(ast.compare(a1, a2)) + self.assertTrue(ast.compare(a1, a2, compare_attributes=True)) + del a1.lineno + self.assertFalse(ast.compare(a1, a2, compare_attributes=True)) + del a2.lineno + self.assertTrue(ast.compare(a1, a2, compare_attributes=True)) + + def test_positional_only_feature_version(self): + ast.parse('def foo(x, /): ...', feature_version=(3, 8)) + ast.parse('def bar(x=1, /): ...', feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse('def foo(x, /): ...', feature_version=(3, 7)) + with self.assertRaises(SyntaxError): + ast.parse('def bar(x=1, /): ...', feature_version=(3, 7)) + + ast.parse('lambda x, /: ...', feature_version=(3, 8)) + ast.parse('lambda x=1, /: ...', feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse('lambda x, /: ...', feature_version=(3, 7)) + with self.assertRaises(SyntaxError): + ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) + + def test_assignment_expression_feature_version(self): + ast.parse('(x := 0)', feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse('(x := 0)', feature_version=(3, 7)) + + def test_pep750_tstring(self): + code = 't""' + ast.parse(code, feature_version=(3, 14)) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_without_parens(self): + code = textwrap.dedent(""" + try: + ... + except ValueError, TypeError: + ... + """) + ast.parse(code, feature_version=(3, 14)) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_with_single_expr(self): + single_expr = textwrap.dedent(""" + try: + ... + except{0} TypeError: + ... + """) + + single_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} TypeError as exc: + ... + """) + + single_tuple_expr = textwrap.dedent(""" + try: + ... + except{0} (TypeError,): + ... + """) + + single_tuple_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} (TypeError,) as exc: + ... + """) + + single_parens_expr = textwrap.dedent(""" + try: + ... + except{0} (TypeError): + ... + """) + + single_parens_expr_with_as = textwrap.dedent(""" + try: + ... + except{0} (TypeError) as exc: + ... + """) + + for code in [ + single_expr, + single_expr_with_as, + single_tuple_expr, + single_tuple_expr_with_as, + single_parens_expr, + single_parens_expr_with_as, + ]: + for star in [True, False]: + code = code.format('*' if star else '') + with self.subTest(code=code, star=star): + ast.parse(code, feature_version=(3, 14)) + ast.parse(code, feature_version=(3, 13)) + + def test_pep758_except_star_without_parens(self): + code = textwrap.dedent(""" + try: + ... + except* ValueError, TypeError: + ... + """) + ast.parse(code, feature_version=(3, 14)) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 13)) + + def test_conditional_context_managers_parse_with_low_feature_version(self): + # regression test for gh-115881 + ast.parse('with (x() if y else z()): ...', feature_version=(3, 8)) + + def test_exception_groups_feature_version(self): + code = dedent(''' + try: ... + except* Exception: ... + ''') + ast.parse(code) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 10)) + + def test_type_params_feature_version(self): + samples = [ + "type X = int", + "class X[T]: pass", + "def f[T](): pass", + ] + for sample in samples: + with self.subTest(sample): + ast.parse(sample) + with self.assertRaises(SyntaxError): + ast.parse(sample, feature_version=(3, 11)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_type_params_default_feature_version(self): + samples = [ + "type X[*Ts=int] = int", + "class X[T=int]: pass", + "def f[**P=int](): pass", + ] + for sample in samples: + with self.subTest(sample): + ast.parse(sample) + with self.assertRaises(SyntaxError): + ast.parse(sample, feature_version=(3, 12)) + + def test_invalid_major_feature_version(self): + with self.assertRaises(ValueError): + ast.parse('pass', feature_version=(2, 7)) + with self.assertRaises(ValueError): + ast.parse('pass', feature_version=(4, 0)) + + def test_constant_as_name(self): + for constant in "True", "False", "None": + expr = ast.Expression(ast.Name(constant, ast.Load())) + ast.fix_missing_locations(expr) + with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): + compile(expr, "", "eval") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_constant_as_unicode_name(self): + constants = [ + ("True", b"Tru\xe1\xb5\x89"), + ("False", b"Fal\xc5\xbfe"), + ("None", b"N\xc2\xbane"), + ] + for constant in constants: + with self.assertRaisesRegex(ValueError, + f"identifier field can't represent '{constant[0]}' constant"): + ast.parse(constant[1], mode="eval") + + def test_precedence_enum(self): + class _Precedence(enum.IntEnum): + """Precedence table that originated from python grammar.""" + NAMED_EXPR = enum.auto() # := + TUPLE = enum.auto() # , + YIELD = enum.auto() # 'yield', 'yield from' + TEST = enum.auto() # 'if'-'else', 'lambda' + OR = enum.auto() # 'or' + AND = enum.auto() # 'and' + NOT = enum.auto() # 'not' + CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', + # 'in', 'not in', 'is', 'is not' + EXPR = enum.auto() + BOR = EXPR # '|' + BXOR = enum.auto() # '^' + BAND = enum.auto() # '&' + SHIFT = enum.auto() # '<<', '>>' + ARITH = enum.auto() # '+', '-' + TERM = enum.auto() # '*', '@', '/', '%', '//' + FACTOR = enum.auto() # unary '+', '-', '~' + POWER = enum.auto() # '**' + AWAIT = enum.auto() # 'await' + ATOM = enum.auto() + def next(self): + try: + return self.__class__(self + 1) + except ValueError: + return self + enum._test_simple_enum(_Precedence, _ast_unparse._Precedence) + + @support.cpython_only + @skip_if_unlimited_stack_size + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() + def test_ast_recursion_limit(self): + # Android test devices have less memory. + crash_depth = 100_000 if sys.platform == "android" else 500_000 + success_depth = 200 + if _testinternalcapi is not None: + remaining = _testinternalcapi.get_c_recursion_remaining() + success_depth = min(success_depth, remaining) + + def check_limit(prefix, repeated): + expect_ok = prefix + repeated * success_depth + ast.parse(expect_ok) + + broken = prefix + repeated * crash_depth + details = "Compiling ({!r} + {!r} * {})".format( + prefix, repeated, crash_depth) + with self.assertRaises(RecursionError, msg=details): + with support.infinite_recursion(): + ast.parse(broken) + + check_limit("a", "()") + check_limit("a", ".b") + check_limit("a", "[0]") + check_limit("a", "*a") + + def test_null_bytes(self): + with self.assertRaises(SyntaxError, + msg="source code string cannot contain null bytes"): + ast.parse("a\0b") + + def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None: + with self.subTest(f"{node.__name__}.{attr}"): + tree = ast.parse(source) + found = 0 + for child in ast.walk(tree): + if isinstance(child, node): + setattr(child, attr, None) + found += 1 + self.assertEqual(found, 1) + e = re.escape(f"field '{attr}' is required for {node.__name__}") + with self.assertRaisesRegex(ValueError, f"^{e}$"): + compile(tree, "", "exec") + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None + def test_none_checks(self) -> None: + tests = [ + (ast.alias, "name", "import spam as SPAM"), + (ast.arg, "arg", "def spam(SPAM): spam"), + (ast.comprehension, "target", "[spam for SPAM in spam]"), + (ast.comprehension, "iter", "[spam for spam in SPAM]"), + (ast.keyword, "value", "spam(**SPAM)"), + (ast.match_case, "pattern", "match spam:\n case SPAM: spam"), + (ast.withitem, "context_expr", "with SPAM: spam"), + ] + for node, attr, source in tests: + self.assert_none_check(node, attr, source) + + def test_repr(self) -> None: + snapshots = AST_REPR_DATA_FILE.read_text().split("\n") + for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): + with self.subTest(test_input=test): + self.assertEqual(repr(ast.parse(test)), snapshot) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_repr_large_input_crash(self): + # gh-125010: Fix use-after-free in ast repr() + source = "0x0" + "e" * 10_000 + with self.assertRaisesRegex(ValueError, + r"Exceeds the limit \(\d+ digits\)"): + repr(ast.Constant(value=eval(source))) + + def test_tstring(self): + # Test AST structure for simple t-string + tree = ast.parse('t"Hello"') + self.assertIsInstance(tree.body[0].value, ast.TemplateStr) + self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) + + # Test AST for t-string with interpolation + tree = ast.parse('t"Hello {name}"') + self.assertIsInstance(tree.body[0].value, ast.TemplateStr) + self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) + self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_optimization_levels_const_folding(self): + return super().test_optimization_levels_const_folding() + + @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14 + def test_constant_subclasses_deprecated(self): + return super().test_constant_subclasses_deprecated() + + +class CopyTests(unittest.TestCase): + """Test copying and pickling AST nodes.""" + + @staticmethod + def iter_ast_classes(): + """Iterate over the (native) subclasses of ast.AST recursively. + + This excludes the special class ast.Index since its constructor + returns an integer. + """ + def do(cls): + if cls.__module__ != 'ast': + return + if cls is ast.Index: + return + + yield cls + for sub in cls.__subclasses__(): + yield from do(sub) + + yield from do(ast.AST) + + def test_pickling(self): + import pickle + + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for code in exec_tests: + with self.subTest(code=code, protocol=protocol): + tree = compile(code, "?", "exec", 0x400) + ast2 = pickle.loads(pickle.dumps(tree, protocol)) + self.assertEqual(to_tuple(ast2), to_tuple(tree)) + + @skip_if_unlimited_stack_size + def test_copy_with_parents(self): + # gh-120108 + code = """ + ('',) + while i < n: + if ch == '': + ch = format[i] + if ch == '': + if freplace is None: + '' % getattr(object) + elif ch == '': + if zreplace is None: + if hasattr: + offset = object.utcoffset() + if offset is not None: + if offset.days < 0: + offset = -offset + h = divmod(timedelta(hours=0)) + if u: + zreplace = '' % (sign,) + elif s: + zreplace = '' % (sign,) + else: + zreplace = '' % (sign,) + elif ch == '': + if Zreplace is None: + Zreplace = '' + if hasattr(object): + s = object.tzname() + if s is not None: + Zreplace = s.replace('') + newformat.append(Zreplace) + else: + push('') + else: + push(ch) + + """ + tree = ast.parse(textwrap.dedent(code)) + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + child.parent = node + try: + with support.infinite_recursion(200): + tree2 = copy.deepcopy(tree) + finally: + # Singletons like ast.Load() are shared; make sure we don't + # leave them mutated after this test. + for node in ast.walk(tree): + if hasattr(node, "parent"): + del node.parent + + for node in ast.walk(tree2): + for child in ast.iter_child_nodes(node): + if hasattr(child, "parent") and not isinstance(child, ( + ast.expr_context, ast.boolop, ast.unaryop, ast.cmpop, ast.operator, + )): + self.assertEqual(to_tuple(child.parent), to_tuple(node)) + + def test_replace_interface(self): + for klass in self.iter_ast_classes(): + with self.subTest(klass=klass): + self.assertHasAttr(klass, '__replace__') + + fields = set(klass._fields) + with self.subTest(klass=klass, fields=fields): + node = klass(**dict.fromkeys(fields)) + # forbid positional arguments in replace() + self.assertRaises(TypeError, copy.replace, node, 1) + self.assertRaises(TypeError, node.__replace__, 1) + + def test_replace_native(self): + for klass in self.iter_ast_classes(): + fields = set(klass._fields) + attributes = set(klass._attributes) + + with self.subTest(klass=klass, fields=fields, attributes=attributes): + # use of object() to ensure that '==' and 'is' + # behave similarly in ast.compare(node, repl) + old_fields = {field: object() for field in fields} + old_attrs = {attr: object() for attr in attributes} + + # check shallow copy + node = klass(**old_fields) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + # check when passing using attributes (they may be optional!) + node = klass(**old_fields, **old_attrs) + repl = copy.replace(node) + self.assertTrue(ast.compare(node, repl, compare_attributes=True)) + + for field in fields: + # check when we sometimes have attributes and sometimes not + for init_attrs in [{}, old_attrs]: + node = klass(**old_fields, **init_attrs) + # only change a single field (do not change attributes) + new_value = object() + repl = copy.replace(node, **{field: new_value}) + for f in fields: + old_value = old_fields[f] + # assert that there is no side-effect + self.assertIs(getattr(node, f), old_value) + # check the changes + if f != field: + self.assertIs(getattr(repl, f), old_value) + else: + self.assertIs(getattr(repl, f), new_value) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + for attribute in attributes: + node = klass(**old_fields, **old_attrs) + # only change a single attribute (do not change fields) + new_attr = object() + repl = copy.replace(node, **{attribute: new_attr}) + for a in attributes: + old_attr = old_attrs[a] + # assert that there is no side-effect + self.assertIs(getattr(node, a), old_attr) + # check the changes + if a != attribute: + self.assertIs(getattr(repl, a), old_attr) + else: + self.assertIs(getattr(repl, a), new_attr) + self.assertFalse(ast.compare(node, repl, compare_attributes=True)) + + def test_replace_accept_known_class_fields(self): + nid, ctx = object(), object() + + node = ast.Name(id=nid, ctx=ctx) + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + + new_nid = object() + repl = copy.replace(node, id=new_nid) + # assert that there is no side-effect + self.assertIs(node.id, nid) + self.assertIs(node.ctx, ctx) + # check the changes + self.assertIs(repl.id, new_nid) + self.assertIs(repl.ctx, node.ctx) # no changes + + def test_replace_accept_known_class_attributes(self): + node = ast.parse('x').body[0].value + self.assertEqual(node.id, 'x') + self.assertEqual(node.lineno, 1) + + # constructor allows any type so replace() should do the same + lineno = object() + repl = copy.replace(node, lineno=lineno) + # assert that there is no side-effect + self.assertEqual(node.lineno, 1) + # check the changes + self.assertEqual(repl.id, node.id) + self.assertEqual(repl.ctx, node.ctx) + self.assertEqual(repl.lineno, lineno) + + _, _, state = node.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], 1) + + _, _, state = repl.__reduce__() + self.assertEqual(state['id'], 'x') + self.assertEqual(state['ctx'], node.ctx) + self.assertEqual(state['lineno'], lineno) + + def test_replace_accept_known_custom_class_fields(self): + class MyNode(ast.AST): + _fields = ('name', 'data') + __annotations__ = {'name': str, 'data': object} + __match_args__ = ('name', 'data') + + name, data = 'name', object() + + node = MyNode(name, data) + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check shallow copy + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the shallow copy + self.assertIs(repl.name, name) + self.assertIs(repl.data, data) + + node = MyNode(name, data) + repl_data = object() + # replace custom but known field + repl = copy.replace(node, data=repl_data) + # assert that there is no side-effect + self.assertIs(node.name, name) + self.assertIs(node.data, data) + # check the changes + self.assertIs(repl.name, node.name) + self.assertIs(repl.data, repl_data) + + def test_replace_accept_known_custom_class_attributes(self): + class MyNode(ast.AST): + x = 0 + y = 1 + _attributes = ('x', 'y') + + node = MyNode() + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + + y = object() + repl = copy.replace(node, y=y) + # assert that there is no side-effect + self.assertEqual(node.x, 0) + self.assertEqual(node.y, 1) + # check the changes + self.assertEqual(repl.x, 0) + self.assertEqual(repl.y, y) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'x' is not 'x' + def test_replace_ignore_known_custom_instance_fields(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # assert initial values + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # shallow copy, but drops extra fields + repl = copy.replace(node) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'x') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + # change known native field + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + # verify that the 'extra' field is not kept + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + self.assertRaises(AttributeError, getattr, repl, 'extra') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ missing\ 1\ keyword\ argument:\ 'id'\." does not match "replace() does not support Name objects" + def test_replace_reject_missing_field(self): + # case: warn if deleted field is not replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + msg = "Name.__replace__ missing 1 keyword argument: 'id'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node) + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + + # case: do not raise if deleted field is replaced + node = ast.parse('x').body[0].value + context = node.ctx + del node.id + + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + repl = copy.replace(node, id='y') + # assert that there is no side-effect + self.assertRaises(AttributeError, getattr, node, 'id') + self.assertIs(node.ctx, context) + self.assertIs(repl.id, 'y') + self.assertIs(repl.ctx, context) + + def test_replace_accept_missing_field_with_default(self): + node = ast.FunctionDef(name="foo", args=ast.arguments()) + self.assertIs(node.returns, None) + self.assertEqual(node.decorator_list, []) + node2 = copy.replace(node, name="bar") + self.assertEqual(node2.name, "bar") + self.assertIs(node2.returns, None) + self.assertEqual(node2.decorator_list, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ got\ an\ unexpected\ keyword\ argument\ 'extra'\." does not match "replace() does not support Name objects" + def test_replace_reject_known_custom_instance_fields_commits(self): + node = ast.parse('x').body[0].value + node.extra = extra = object() # add instance 'extra' field + context = node.ctx + + # explicit rejection of known instance fields + self.assertHasAttr(node, 'extra') + msg = "Name.__replace__ got an unexpected keyword argument 'extra'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, extra=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertIs(node.extra, extra) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "Name\.__replace__\ got\ an\ unexpected\ keyword\ argument\ 'unknown'\." does not match "replace() does not support Name objects" + def test_replace_reject_unknown_instance_fields(self): + node = ast.parse('x').body[0].value + context = node.ctx + + # explicit rejection of unknown extra fields + self.assertRaises(AttributeError, getattr, node, 'unknown') + msg = "Name.__replace__ got an unexpected keyword argument 'unknown'." + with self.assertRaisesRegex(TypeError, re.escape(msg)): + copy.replace(node, unknown=1) + # assert that there is no side-effect + self.assertIs(node.id, 'x') + self.assertIs(node.ctx, context) + self.assertRaises(AttributeError, getattr, node, 'unknown') + + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message + def test_replace_non_str_kwarg(self): + node = ast.Name(id="x") + errmsg = "got an unexpected keyword argument ', 'exec', ast.PyCF_ONLY_AST) + self.assertEqual(ast.dump(a), ast.dump(b)) + + def test_parse_in_error(self): + try: + 1/0 + except Exception: + with self.assertRaises(SyntaxError) as e: + ast.literal_eval(r"'\U'") + self.assertIsNotNone(e.exception.__context__) + + def test_dump(self): + node = ast.parse('spam(eggs, "and cheese")') + self.assertEqual(ast.dump(node), + "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " + "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " + "Constant('and cheese')]))])" + ) + self.assertEqual(ast.dump(node, include_attributes=True), + "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " + "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " + "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " + "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])" + ) + + def test_dump_indent(self): + node = ast.parse('spam(eggs, "and cheese")') + self.assertEqual(ast.dump(node, indent=3), """\ +Module( + body=[ + Expr( + value=Call( + func=Name(id='spam', ctx=Load()), + args=[ + Name(id='eggs', ctx=Load()), + Constant(value='and cheese')]))])""") + self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ +Module( +\t[ +\t\tExpr( +\t\t\tCall( +\t\t\t\tName('spam', Load()), +\t\t\t\t[ +\t\t\t\t\tName('eggs', Load()), +\t\t\t\t\tConstant('and cheese')]))])""") + self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ +Module( + body=[ + Expr( + value=Call( + func=Name( + id='spam', + ctx=Load(), + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=4), + args=[ + Name( + id='eggs', + ctx=Load(), + lineno=1, + col_offset=5, + end_lineno=1, + end_col_offset=9), + Constant( + value='and cheese', + lineno=1, + col_offset=11, + end_lineno=1, + end_col_offset=23)], + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=24), + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=24)])""") + + def test_dump_incomplete(self): + node = ast.Raise(lineno=3, col_offset=4) + self.assertEqual(ast.dump(node), + "Raise()" + ) + self.assertEqual(ast.dump(node, include_attributes=True), + "Raise(lineno=3, col_offset=4)" + ) + node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) + self.assertEqual(ast.dump(node), + "Raise(exc=Name(id='e', ctx=Load()))" + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "Raise(Name('e', Load()))" + ) + self.assertEqual(ast.dump(node, include_attributes=True), + "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" + ) + self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), + "Raise(Name('e', Load()), lineno=3, col_offset=4)" + ) + node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) + self.assertEqual(ast.dump(node), + "Raise(cause=Name(id='e', ctx=Load()))" + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "Raise(cause=Name('e', Load()))" + ) + # Arguments: + node = ast.arguments(args=[ast.arg("x")]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg('y')]) + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments() + self.assertEqual(ast.dump(node, annotate_fields=False), + "arguments()", + ) + # Classes: + node = ast.ClassDef( + 'T', + [], + [ast.keyword('a', ast.Constant(None))], + [], + [ast.Name('dataclass', ctx=ast.Load())], + ) + self.assertEqual(ast.dump(node), + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", + ) + + def test_dump_show_empty(self): + def check_node(node, empty, full, **kwargs): + with self.subTest(show_empty=False): + self.assertEqual( + ast.dump(node, show_empty=False, **kwargs), + empty, + ) + with self.subTest(show_empty=True): + self.assertEqual( + ast.dump(node, show_empty=True, **kwargs), + full, + ) + + def check_text(code, empty, full, **kwargs): + check_node(ast.parse(code), empty, full, **kwargs) + + check_node( + ast.arguments(), + empty="arguments()", + full="arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])", + ) + + check_node( + # Corner case: there are no real `Name` instances with `id=''`: + ast.Name(id='', ctx=ast.Load()), + empty="Name(id='', ctx=Load())", + full="Name(id='', ctx=Load())", + ) + + check_node( + ast.MatchSingleton(value=None), + empty="MatchSingleton(value=None)", + full="MatchSingleton(value=None)", + ) + + check_node( + ast.MatchSingleton(value=[]), + empty="MatchSingleton(value=[])", + full="MatchSingleton(value=[])", + ) + + check_node( + ast.Constant(value=None), + empty="Constant(value=None)", + full="Constant(value=None)", + ) + + check_node( + ast.Constant(value=[]), + empty="Constant(value=[])", + full="Constant(value=[])", + ) + + check_node( + ast.Constant(value=''), + empty="Constant(value='')", + full="Constant(value='')", + ) + + check_node( + ast.Interpolation(value=ast.Constant(42), str=None, conversion=-1), + empty="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + full="Interpolation(value=Constant(value=42), str=None, conversion=-1)", + ) + + check_node( + ast.Interpolation(value=ast.Constant(42), str=[], conversion=-1), + empty="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + full="Interpolation(value=Constant(value=42), str=[], conversion=-1)", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))], decorator_list=[], type_params=[])], type_ignores=[])", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], decorator_list=[], type_params=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)], type_ignores=[])", + include_attributes=True, + ) + + check_text( + 'spam(eggs, "and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], keywords=[]))], type_ignores=[])", + ) + + check_text( + 'spam(eggs, text="and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))], type_ignores=[])", + ) + + check_text( + "import _ast as ast; from module import sub", + empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)])", + full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^^^^^ ^^^^^^^^^ + def test_copy_location(self): + src = ast.parse('1 + 1', mode='eval') + src.body.right = ast.copy_location(ast.Constant(2), src.body.right) + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' + 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' + 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' + 'col_offset=0, end_lineno=1, end_col_offset=5))' + ) + func = ast.Name('spam', ast.Load()) + src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func) + new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) + self.assertIsNone(new.end_lineno) + self.assertIsNone(new.end_col_offset) + self.assertEqual(new.lineno, 1) + self.assertEqual(new.col_offset, 1) + + def test_fix_missing_locations(self): + src = ast.parse('write("spam")') + src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), + [ast.Constant('eggs')], []))) + self.assertEqual(src, ast.fix_missing_locations(src)) + self.maxDiff = None + self.assertEqual(ast.dump(src, include_attributes=True), + "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " + "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " + "end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " + "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])" + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + def test_increment_lineno(self): + src = ast.parse('1 + 1', mode='eval') + self.assertEqual(ast.increment_lineno(src, n=3), src) + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' + 'col_offset=0, end_lineno=4, end_col_offset=5))' + ) + # issue10869: do not increment lineno of root twice + src = ast.parse('1 + 1', mode='eval') + self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) + self.assertEqual(ast.dump(src, include_attributes=True), + 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' + 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' + 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' + 'col_offset=0, end_lineno=4, end_col_offset=5))' + ) + src = ast.Call( + func=ast.Name("test", ast.Load()), args=[], keywords=[], lineno=1 + ) + self.assertEqual(ast.increment_lineno(src).lineno, 2) + self.assertIsNone(ast.increment_lineno(src).end_lineno) + + def test_increment_lineno_on_module(self): + src = ast.parse(dedent("""\ + a = 1 + b = 2 # type: ignore + c = 3 + d = 4 # type: ignore@tag + """), type_comments=True) + ast.increment_lineno(src, n=5) + self.assertEqual(src.type_ignores[0].lineno, 7) + self.assertEqual(src.type_ignores[1].lineno, 9) + self.assertEqual(src.type_ignores[1].tag, '@tag') + + def test_iter_fields(self): + node = ast.parse('foo()', mode='eval') + d = dict(ast.iter_fields(node.body)) + self.assertEqual(d.pop('func').id, 'foo') + self.assertEqual(d, {'keywords': [], 'args': []}) + + def test_iter_child_nodes(self): + node = ast.parse("spam(23, 42, eggs='leek')", mode='eval') + self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) + iterator = ast.iter_child_nodes(node.body) + self.assertEqual(next(iterator).id, 'spam') + self.assertEqual(next(iterator).value, 23) + self.assertEqual(next(iterator).value, 42) + self.assertEqual(ast.dump(next(iterator)), + "keyword(arg='eggs', value=Constant(value='leek'))" + ) + + def test_get_docstring(self): + node = ast.parse('"""line one\n line two"""') + self.assertEqual(ast.get_docstring(node), + 'line one\nline two') + + node = ast.parse('class foo:\n """line one\n line two"""') + self.assertEqual(ast.get_docstring(node.body[0]), + 'line one\nline two') + + node = ast.parse('def foo():\n """line one\n line two"""') + self.assertEqual(ast.get_docstring(node.body[0]), + 'line one\nline two') + + node = ast.parse('async def foo():\n """spam\n ham"""') + self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') + + node = ast.parse('async def foo():\n """spam\n ham"""') + self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') + + node = ast.parse('x') + self.assertRaises(TypeError, ast.get_docstring, node.body[0]) + + def test_get_docstring_none(self): + self.assertIsNone(ast.get_docstring(ast.parse(''))) + node = ast.parse('x = "not docstring"') + self.assertIsNone(ast.get_docstring(node)) + node = ast.parse('def foo():\n pass') + self.assertIsNone(ast.get_docstring(node)) + + node = ast.parse('class foo:\n pass') + self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('class foo:\n x = "not docstring"') + self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('class foo:\n def bar(self): pass') + self.assertIsNone(ast.get_docstring(node.body[0])) + + node = ast.parse('def foo():\n pass') + self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('def foo():\n x = "not docstring"') + self.assertIsNone(ast.get_docstring(node.body[0])) + + node = ast.parse('async def foo():\n pass') + self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('async def foo():\n x = "not docstring"') + self.assertIsNone(ast.get_docstring(node.body[0])) + + node = ast.parse('async def foo():\n 42') + self.assertIsNone(ast.get_docstring(node.body[0])) + + def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): + node = ast.parse( + '"""line one\nline two"""\n\n' + 'def foo():\n """line one\n line two"""\n\n' + ' def bar():\n """line one\n line two"""\n' + ' """line one\n line two"""\n' + '"""line one\nline two"""\n\n' + ) + self.assertEqual(node.body[0].col_offset, 0) + self.assertEqual(node.body[0].lineno, 1) + self.assertEqual(node.body[1].body[0].col_offset, 2) + self.assertEqual(node.body[1].body[0].lineno, 5) + self.assertEqual(node.body[1].body[1].body[0].col_offset, 4) + self.assertEqual(node.body[1].body[1].body[0].lineno, 9) + self.assertEqual(node.body[1].body[2].col_offset, 2) + self.assertEqual(node.body[1].body[2].lineno, 11) + self.assertEqual(node.body[2].col_offset, 0) + self.assertEqual(node.body[2].lineno, 13) + + def test_elif_stmt_start_position(self): + node = ast.parse('if a:\n pass\nelif b:\n pass\n') + elif_stmt = node.body[0].orelse[0] + self.assertEqual(elif_stmt.lineno, 3) + self.assertEqual(elif_stmt.col_offset, 0) + + def test_elif_stmt_start_position_with_else(self): + node = ast.parse('if a:\n pass\nelif b:\n pass\nelse:\n pass\n') + elif_stmt = node.body[0].orelse[0] + self.assertEqual(elif_stmt.lineno, 3) + self.assertEqual(elif_stmt.col_offset, 0) + + def test_starred_expr_end_position_within_call(self): + node = ast.parse('f(*[0, 1])') + starred_expr = node.body[0].value.args[0] + self.assertEqual(starred_expr.end_lineno, 1) + self.assertEqual(starred_expr.end_col_offset, 9) + + def test_literal_eval(self): + self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3]) + self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) + self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None)) + self.assertEqual(ast.literal_eval('{1, 2, 3}'), {1, 2, 3}) + self.assertEqual(ast.literal_eval('b"hi"'), b"hi") + self.assertEqual(ast.literal_eval('set()'), set()) + self.assertRaises(ValueError, ast.literal_eval, 'foo()') + self.assertEqual(ast.literal_eval('6'), 6) + self.assertEqual(ast.literal_eval('+6'), 6) + self.assertEqual(ast.literal_eval('-6'), -6) + self.assertEqual(ast.literal_eval('3.25'), 3.25) + self.assertEqual(ast.literal_eval('+3.25'), 3.25) + self.assertEqual(ast.literal_eval('-3.25'), -3.25) + self.assertEqual(repr(ast.literal_eval('-0.0')), '-0.0') + self.assertRaises(ValueError, ast.literal_eval, '++6') + self.assertRaises(ValueError, ast.literal_eval, '+True') + self.assertRaises(ValueError, ast.literal_eval, '2+3') + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_literal_eval_str_int_limit(self): + with support.adjust_int_max_str_digits(4000): + ast.literal_eval('3'*4000) # no error + with self.assertRaises(SyntaxError) as err_ctx: + ast.literal_eval('3'*4001) + self.assertIn('Exceeds the limit ', str(err_ctx.exception)) + self.assertIn(' Consider hexadecimal ', str(err_ctx.exception)) + + def test_literal_eval_complex(self): + # Issue #4907 + self.assertEqual(ast.literal_eval('6j'), 6j) + self.assertEqual(ast.literal_eval('-6j'), -6j) + self.assertEqual(ast.literal_eval('6.75j'), 6.75j) + self.assertEqual(ast.literal_eval('-6.75j'), -6.75j) + self.assertEqual(ast.literal_eval('3+6j'), 3+6j) + self.assertEqual(ast.literal_eval('-3+6j'), -3+6j) + self.assertEqual(ast.literal_eval('3-6j'), 3-6j) + self.assertEqual(ast.literal_eval('-3-6j'), -3-6j) + self.assertEqual(ast.literal_eval('3.25+6.75j'), 3.25+6.75j) + self.assertEqual(ast.literal_eval('-3.25+6.75j'), -3.25+6.75j) + self.assertEqual(ast.literal_eval('3.25-6.75j'), 3.25-6.75j) + self.assertEqual(ast.literal_eval('-3.25-6.75j'), -3.25-6.75j) + self.assertEqual(ast.literal_eval('(3+6j)'), 3+6j) + self.assertRaises(ValueError, ast.literal_eval, '-6j+3') + self.assertRaises(ValueError, ast.literal_eval, '-6j+3j') + self.assertRaises(ValueError, ast.literal_eval, '3+-6j') + self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') + self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') + + def test_literal_eval_malformed_dict_nodes(self): + malformed = ast.Dict(keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)]) + self.assertRaises(ValueError, ast.literal_eval, malformed) + malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) + self.assertRaises(ValueError, ast.literal_eval, malformed) + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: expected an expression + def test_literal_eval_trailing_ws(self): + self.assertEqual(ast.literal_eval(" -1"), -1) + self.assertEqual(ast.literal_eval("\t\t-1"), -1) + self.assertEqual(ast.literal_eval(" \t -1"), -1) + self.assertRaises(IndentationError, ast.literal_eval, "\n -1") + + def test_literal_eval_malformed_lineno(self): + msg = r'malformed node or string on line 3:' + with self.assertRaisesRegex(ValueError, msg): + ast.literal_eval("{'a': 1,\n'b':2,\n'c':++3,\n'd':4}") + + node = ast.UnaryOp( + ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) + self.assertIsNone(getattr(node, 'lineno', None)) + msg = r'malformed node or string:' + with self.assertRaisesRegex(ValueError, msg): + ast.literal_eval(node) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "unexpected indent" does not match "expected an expression (, line 2)" + def test_literal_eval_syntax_errors(self): + with self.assertRaisesRegex(SyntaxError, "unexpected indent"): + ast.literal_eval(r''' + \ + (\ + \ ''') + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: required field "lineno" missing from alias + def test_bad_integer(self): + # issue13436: Bad error message with invalid numeric values + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep')], + level=None, + lineno=None, col_offset=None)] + mod = ast.Module(body, []) + with self.assertRaises(ValueError) as cm: + compile(mod, 'test', 'exec') + self.assertIn("invalid integer value: None", str(cm.exception)) + + def test_level_as_none(self): + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep', + lineno=0, col_offset=0)], + level=None, + lineno=0, col_offset=0)] + mod = ast.Module(body, []) + code = compile(mod, 'test', 'exec') + ns = {} + exec(code, ns) + self.assertIn('sleep', ns) + + @unittest.skip("TODO: RUSTPYTHON; crash") + @skip_if_unlimited_stack_size + @skip_emscripten_stack_overflow() + def test_recursion_direct(self): + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + e.operand = e + with self.assertRaises(RecursionError): + with support.infinite_recursion(): + compile(ast.Expression(e), "", "eval") + + @unittest.skip("TODO: RUSTPYTHON; crash") + @skip_if_unlimited_stack_size + @skip_emscripten_stack_overflow() + def test_recursion_indirect(self): + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + e.operand = f + f.operand = e + with self.assertRaises(RecursionError): + with support.infinite_recursion(): + compile(ast.Expression(e), "", "eval") + + +class ASTValidatorTests(unittest.TestCase): + + def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): + mod.lineno = mod.col_offset = 0 + ast.fix_missing_locations(mod) + if msg is None: + compile(mod, "", mode) + else: + with self.assertRaises(exc) as cm: + compile(mod, "", mode) + self.assertIn(msg, str(cm.exception)) + + def expr(self, node, msg=None, *, exc=ValueError): + mod = ast.Module([ast.Expr(node)], []) + self.mod(mod, msg, exc=exc) + + def stmt(self, stmt, msg=None): + mod = ast.Module([stmt], []) + self.mod(mod, msg) + + def test_module(self): + m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) + self.mod(m, "must have Load context", "single") + m = ast.Expression(ast.Name("x", ast.Store())) + self.mod(m, "must have Load context", "eval") + + def _check_arguments(self, fac, check): + def arguments(args=None, posonlyargs=None, vararg=None, + kwonlyargs=None, kwarg=None, + defaults=None, kw_defaults=None): + if args is None: + args = [] + if posonlyargs is None: + posonlyargs = [] + if kwonlyargs is None: + kwonlyargs = [] + if defaults is None: + defaults = [] + if kw_defaults is None: + kw_defaults = [] + args = ast.arguments(args, posonlyargs, vararg, kwonlyargs, + kw_defaults, kwarg, defaults) + return fac(args) + args = [ast.arg("x", ast.Name("x", ast.Store()))] + check(arguments(args=args), "must have Load context") + check(arguments(posonlyargs=args), "must have Load context") + check(arguments(kwonlyargs=args), "must have Load context") + check(arguments(defaults=[ast.Constant(3)]), + "more positional defaults than args") + check(arguments(kw_defaults=[ast.Constant(4)]), + "length of kwonlyargs is not the same as kw_defaults") + args = [ast.arg("x", ast.Name("x", ast.Load()))] + check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), + "must have Load context") + args = [ast.arg("a", ast.Name("x", ast.Load())), + ast.arg("b", ast.Name("y", ast.Load()))] + check(arguments(kwonlyargs=args, + kw_defaults=[None, ast.Name("x", ast.Store())]), + "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_funcdef(self): + a = ast.arguments([], [], None, [], [], None, []) + f = ast.FunctionDef("x", a, [], [], None, None, []) + self.stmt(f, "empty body on FunctionDef") + f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], None, None, []) + self.stmt(f, "must have Load context") + f = ast.FunctionDef("x", a, [ast.Pass()], [], + ast.Name("x", ast.Store()), None, []) + self.stmt(f, "must have Load context") + f = ast.FunctionDef("x", ast.arguments(), [ast.Pass()]) + self.stmt(f) + def fac(args): + return ast.FunctionDef("x", args, [ast.Pass()], [], None, None, []) + self._check_arguments(fac, self.stmt) + + def test_funcdef_pattern_matching(self): + # gh-104799: New fields on FunctionDef should be added at the end + def matcher(node): + match node: + case ast.FunctionDef("foo", ast.arguments(args=[ast.arg("bar")]), + [ast.Pass()], + [ast.Name("capybara", ast.Load())], + ast.Name("pacarana", ast.Load())): + return True + case _: + return False + + code = """ + @capybara + def foo(bar) -> pacarana: + pass + """ + source = ast.parse(textwrap.dedent(code)) + funcdef = source.body[0] + self.assertIsInstance(funcdef, ast.FunctionDef) + self.assertTrue(matcher(funcdef)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_classdef(self): + def cls(bases=None, keywords=None, body=None, decorator_list=None, type_params=None): + if bases is None: + bases = [] + if keywords is None: + keywords = [] + if body is None: + body = [ast.Pass()] + if decorator_list is None: + decorator_list = [] + if type_params is None: + type_params = [] + return ast.ClassDef("myclass", bases, keywords, + body, decorator_list, type_params) + self.stmt(cls(bases=[ast.Name("x", ast.Store())]), + "must have Load context") + self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), + "must have Load context") + self.stmt(cls(body=[]), "empty body on ClassDef") + self.stmt(cls(body=[None]), "None disallowed") + self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]), + "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_delete(self): + self.stmt(ast.Delete([]), "empty targets on Delete") + self.stmt(ast.Delete([None]), "None disallowed") + self.stmt(ast.Delete([ast.Name("x", ast.Load())]), + "must have Del context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_assign(self): + self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") + self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") + self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), + "must have Store context") + self.stmt(ast.Assign([ast.Name("x", ast.Store())], + ast.Name("y", ast.Store())), + "must have Load context") + + def test_augassign(self): + aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(), + ast.Name("y", ast.Load())) + self.stmt(aug, "must have Store context") + aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(), + ast.Name("y", ast.Store())) + self.stmt(aug, "must have Load context") + + def test_for(self): + x = ast.Name("x", ast.Store()) + y = ast.Name("y", ast.Load()) + p = ast.Pass() + self.stmt(ast.For(x, y, [], []), "empty body on For") + self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []), + "must have Store context") + self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []), + "must have Load context") + e = ast.Expr(ast.Name("x", ast.Store())) + self.stmt(ast.For(x, y, [e], []), "must have Load context") + self.stmt(ast.For(x, y, [p], [e]), "must have Load context") + + def test_while(self): + self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") + self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), + "must have Load context") + self.stmt(ast.While(ast.Constant(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]), + "must have Load context") + + def test_if(self): + self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") + i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) + self.stmt(i, "must have Load context") + i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) + self.stmt(i, "must have Load context") + i = ast.If(ast.Constant(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(i, "must have Load context") + + def test_with(self): + p = ast.Pass() + self.stmt(ast.With([], [p]), "empty items on With") + i = ast.withitem(ast.Constant(3), None) + self.stmt(ast.With([i], []), "empty body on With") + i = ast.withitem(ast.Name("x", ast.Store()), None) + self.stmt(ast.With([i], [p]), "must have Load context") + i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) + self.stmt(ast.With([i], [p]), "must have Store context") + + def test_raise(self): + r = ast.Raise(None, ast.Constant(3)) + self.stmt(r, "Raise with cause but no exception") + r = ast.Raise(ast.Name("x", ast.Store()), None) + self.stmt(r, "must have Load context") + r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) + self.stmt(r, "must have Load context") + + def test_try(self): + p = ast.Pass() + t = ast.Try([], [], [], [p]) + self.stmt(t, "empty body on Try") + t = ast.Try([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) + self.stmt(t, "must have Load context") + t = ast.Try([p], [], [], []) + self.stmt(t, "Try has neither except handlers nor finalbody") + t = ast.Try([p], [], [p], [p]) + self.stmt(t, "Try has orelse but no except handlers") + t = ast.Try([p], [ast.ExceptHandler(None, "x", [])], [], []) + self.stmt(t, "empty body on ExceptHandler") + e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] + self.stmt(ast.Try([p], e, [], []), "must have Load context") + e = [ast.ExceptHandler(None, "x", [p])] + t = ast.Try([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) + self.stmt(t, "must have Load context") + t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(t, "must have Load context") + + def test_try_star(self): + p = ast.Pass() + t = ast.TryStar([], [], [], [p]) + self.stmt(t, "empty body on TryStar") + t = ast.TryStar([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) + self.stmt(t, "must have Load context") + t = ast.TryStar([p], [], [], []) + self.stmt(t, "TryStar has neither except handlers nor finalbody") + t = ast.TryStar([p], [], [p], [p]) + self.stmt(t, "TryStar has orelse but no except handlers") + t = ast.TryStar([p], [ast.ExceptHandler(None, "x", [])], [], []) + self.stmt(t, "empty body on ExceptHandler") + e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] + self.stmt(ast.TryStar([p], e, [], []), "must have Load context") + e = [ast.ExceptHandler(None, "x", [p])] + t = ast.TryStar([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) + self.stmt(t, "must have Load context") + t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(t, "must have Load context") + + def test_assert(self): + self.stmt(ast.Assert(ast.Name("x", ast.Store()), None), + "must have Load context") + assrt = ast.Assert(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store())) + self.stmt(assrt, "must have Load context") + + def test_import(self): + self.stmt(ast.Import([]), "empty names on Import") + + def test_importfrom(self): + imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) + self.stmt(imp, "Negative ImportFrom level") + self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") + + def test_global(self): + self.stmt(ast.Global([]), "empty names on Global") + + def test_nonlocal(self): + self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") + + def test_expr(self): + e = ast.Expr(ast.Name("x", ast.Store())) + self.stmt(e, "must have Load context") + + @unittest.skip("TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value") + def test_boolop(self): + b = ast.BoolOp(ast.And(), []) + self.expr(b, "less than 2 values") + b = ast.BoolOp(ast.And(), [ast.Constant(3)]) + self.expr(b, "less than 2 values") + b = ast.BoolOp(ast.And(), [ast.Constant(4), None]) + self.expr(b, "None disallowed") + b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) + self.expr(b, "must have Load context") + + def test_unaryop(self): + u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) + self.expr(u, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_lambda(self): + a = ast.arguments([], [], None, [], [], None, []) + self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), + "must have Load context") + def fac(args): + return ast.Lambda(args, ast.Name("x", ast.Load())) + self._check_arguments(fac, self.expr) + + def test_ifexp(self): + l = ast.Name("x", ast.Load()) + s = ast.Name("y", ast.Store()) + for args in (s, l, l), (l, s, l), (l, l, s): + self.expr(ast.IfExp(*args), "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_dict(self): + d = ast.Dict([], [ast.Name("x", ast.Load())]) + self.expr(d, "same number of keys as values") + d = ast.Dict([ast.Name("x", ast.Load())], [None]) + self.expr(d, "None disallowed") + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None + def test_set(self): + self.expr(ast.Set([None]), "None disallowed") + s = ast.Set([ast.Name("x", ast.Store())]) + self.expr(s, "must have Load context") + + def _check_comprehension(self, fac): + self.expr(fac([]), "comprehension with no generators") + g = ast.comprehension(ast.Name("x", ast.Load()), + ast.Name("x", ast.Load()), [], 0) + self.expr(fac([g]), "must have Store context") + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Store()), [], 0) + self.expr(fac([g]), "must have Load context") + x = ast.Name("x", ast.Store()) + y = ast.Name("y", ast.Load()) + g = ast.comprehension(x, y, [None], 0) + self.expr(fac([g]), "None disallowed") + g = ast.comprehension(x, y, [ast.Name("x", ast.Store())], 0) + self.expr(fac([g]), "must have Load context") + + def _simple_comp(self, fac): + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Load()), [], 0) + self.expr(fac(ast.Name("x", ast.Store()), [g]), + "must have Load context") + def wrap(gens): + return fac(ast.Name("x", ast.Store()), gens) + self._check_comprehension(wrap) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_listcomp(self): + self._simple_comp(ast.ListComp) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_setcomp(self): + self._simple_comp(ast.SetComp) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_generatorexp(self): + self._simple_comp(ast.GeneratorExp) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_dictcomp(self): + g = ast.comprehension(ast.Name("y", ast.Store()), + ast.Name("p", ast.Load()), [], 0) + c = ast.DictComp(ast.Name("x", ast.Store()), + ast.Name("y", ast.Load()), [g]) + self.expr(c, "must have Load context") + c = ast.DictComp(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store()), [g]) + self.expr(c, "must have Load context") + def factory(comps): + k = ast.Name("x", ast.Load()) + v = ast.Name("y", ast.Load()) + return ast.DictComp(k, v, comps) + self._check_comprehension(factory) + + def test_yield(self): + self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") + self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") + + @unittest.skip("TODO: RUSTPYTHON; thread 'main' panicked") + def test_compare(self): + left = ast.Name("x", ast.Load()) + comp = ast.Compare(left, [ast.In()], []) + self.expr(comp, "no comparators") + comp = ast.Compare(left, [ast.In()], [ast.Constant(4), ast.Constant(5)]) + self.expr(comp, "different number of comparators and operands") + comp = ast.Compare(ast.Constant("blah"), [ast.In()], [left]) + self.expr(comp) + comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) + self.expr(comp) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_call(self): + func = ast.Name("x", ast.Load()) + args = [ast.Name("y", ast.Load())] + keywords = [ast.keyword("w", ast.Name("z", ast.Load()))] + call = ast.Call(ast.Name("x", ast.Store()), args, keywords) + self.expr(call, "must have Load context") + call = ast.Call(func, [None], keywords) + self.expr(call, "None disallowed") + bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))] + call = ast.Call(func, args, bad_keywords) + self.expr(call, "must have Load context") + + def test_attribute(self): + attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) + self.expr(attr, "must have Load context") + + def test_subscript(self): + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), + ast.Load()) + self.expr(sub, "must have Load context") + x = ast.Name("x", ast.Load()) + sub = ast.Subscript(x, ast.Name("y", ast.Store()), + ast.Load()) + self.expr(sub, "must have Load context") + s = ast.Name("x", ast.Store()) + for args in (s, None, None), (None, s, None), (None, None, s): + sl = ast.Slice(*args) + self.expr(ast.Subscript(x, sl, ast.Load()), + "must have Load context") + sl = ast.Tuple([], ast.Load()) + self.expr(ast.Subscript(x, sl, ast.Load())) + sl = ast.Tuple([s], ast.Load()) + self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") + + def test_starred(self): + left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], + ast.Store()) + assign = ast.Assign([left], ast.Constant(4)) + self.stmt(assign, "must have Store context") + + def _sequence(self, fac): + self.expr(fac([None], ast.Load()), "None disallowed") + self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), + "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None + def test_list(self): + self._sequence(ast.List) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None + def test_tuple(self): + self._sequence(ast.Tuple) + + @support.requires_resource('cpu') + def test_stdlib_validates(self): + for module in STDLIB_FILES: + with self.subTest(module): + fn = os.path.join(STDLIB, module) + with open(fn, "r", encoding="utf-8") as fp: + source = fp.read() + mod = ast.parse(source, fn) + compile(mod, fn, "exec") + mod2 = ast.parse(source, fn) + self.assertTrue(ast.compare(mod, mod2)) + + constant_1 = ast.Constant(1) + pattern_1 = ast.MatchValue(constant_1) + + constant_x = ast.Constant('x') + pattern_x = ast.MatchValue(constant_x) + + constant_true = ast.Constant(True) + pattern_true = ast.MatchSingleton(True) + + name_carter = ast.Name('carter', ast.Load()) + + _MATCH_PATTERNS = [ + ast.MatchValue( + ast.Attribute( + ast.Attribute( + ast.Name('x', ast.Store()), + 'y', ast.Load() + ), + 'z', ast.Load() + ) + ), + ast.MatchValue( + ast.Attribute( + ast.Attribute( + ast.Name('x', ast.Load()), + 'y', ast.Store() + ), + 'z', ast.Load() + ) + ), + ast.MatchValue( + ast.Constant(...) + ), + ast.MatchValue( + ast.Constant(True) + ), + ast.MatchValue( + ast.Constant((1,2,3)) + ), + ast.MatchSingleton('string'), + ast.MatchSequence([ + ast.MatchSingleton('string') + ]), + ast.MatchSequence( + [ + ast.MatchSequence( + [ + ast.MatchSingleton('string') + ] + ) + ] + ), + ast.MatchMapping( + [constant_1, constant_true], + [pattern_x] + ), + ast.MatchMapping( + [constant_true, constant_1], + [pattern_x, pattern_1], + rest='True' + ), + ast.MatchMapping( + [constant_true, ast.Starred(ast.Name('lol', ast.Load()), ast.Load())], + [pattern_x, pattern_1], + rest='legit' + ), + ast.MatchClass( + ast.Attribute( + ast.Attribute( + constant_x, + 'y', ast.Load()), + 'z', ast.Load()), + patterns=[], kwd_attrs=[], kwd_patterns=[] + ), + ast.MatchClass( + name_carter, + patterns=[], + kwd_attrs=['True'], + kwd_patterns=[pattern_1] + ), + ast.MatchClass( + name_carter, + patterns=[], + kwd_attrs=[], + kwd_patterns=[pattern_1] + ), + ast.MatchClass( + name_carter, + patterns=[ast.MatchSingleton('string')], + kwd_attrs=[], + kwd_patterns=[] + ), + ast.MatchClass( + name_carter, + patterns=[ast.MatchStar()], + kwd_attrs=[], + kwd_patterns=[] + ), + ast.MatchClass( + name_carter, + patterns=[], + kwd_attrs=[], + kwd_patterns=[ast.MatchStar()] + ), + ast.MatchClass( + constant_true, # invalid name + patterns=[], + kwd_attrs=['True'], + kwd_patterns=[pattern_1] + ), + ast.MatchSequence( + [ + ast.MatchStar("True") + ] + ), + ast.MatchAs( + name='False' + ), + ast.MatchOr( + [] + ), + ast.MatchOr( + [pattern_1] + ), + ast.MatchOr( + [pattern_1, pattern_x, ast.MatchSingleton('xxx')] + ), + ast.MatchAs(name="_"), + ast.MatchStar(name="x"), + ast.MatchSequence([ast.MatchStar("_")]), + ast.MatchMapping([], [], rest="_"), + ] + + def test_match_validation_pattern(self): + name_x = ast.Name('x', ast.Load()) + for pattern in self._MATCH_PATTERNS: + with self.subTest(ast.dump(pattern, indent=4)): + node = ast.Match( + subject=name_x, + cases = [ + ast.match_case( + pattern=pattern, + body = [ast.Pass()] + ) + ] + ) + node = ast.fix_missing_locations(node) + module = ast.Module([node], []) + with self.assertRaises(ValueError): + compile(module, "", "exec") + + +class ConstantTests(unittest.TestCase): + """Tests on the ast.Constant node type.""" + + def compile_constant(self, value): + tree = ast.parse("x = 123") + + node = tree.body[0].value + new_node = ast.Constant(value=value) + ast.copy_location(new_node, node) + tree.body[0].value = new_node + + code = compile(tree, "", "exec") + + ns = {} + exec(code, ns) + return ns['x'] + + def test_validation(self): + with self.assertRaises(TypeError) as cm: + self.compile_constant([1, 2, 3]) + self.assertEqual(str(cm.exception), + "got an invalid type in Constant: list") + + def test_singletons(self): + for const in (None, False, True, Ellipsis, b''): + with self.subTest(const=const): + value = self.compile_constant(const) + self.assertIs(value, const) + + def test_values(self): + nested_tuple = (1,) + nested_frozenset = frozenset({1}) + for level in range(3): + nested_tuple = (nested_tuple, 2) + nested_frozenset = frozenset({nested_frozenset, 2}) + values = (123, 123.0, 123j, + "unicode", b'bytes', + tuple("tuple"), frozenset("frozenset"), + nested_tuple, nested_frozenset) + for value in values: + with self.subTest(value=value): + result = self.compile_constant(value) + self.assertEqual(result, value) + + def test_assign_to_constant(self): + tree = ast.parse("x = 1") + + target = tree.body[0].targets[0] + new_target = ast.Constant(value=1) + ast.copy_location(new_target, target) + tree.body[0].targets[0] = new_target + + with self.assertRaises(ValueError) as cm: + compile(tree, "string", "exec") + self.assertEqual(str(cm.exception), + "expression which can't be assigned " + "to in Store context") + + def test_get_docstring(self): + tree = ast.parse("'docstring'\nx = 1") + self.assertEqual(ast.get_docstring(tree), 'docstring') + + def get_load_const(self, tree): + # Compile to bytecode, disassemble and get parameter of LOAD_CONST + # instructions + co = compile(tree, '', 'exec') + consts = [] + for instr in dis.get_instructions(co): + if instr.opcode in dis.hasconst: + consts.append(instr.argval) + return consts + + @support.cpython_only + def test_load_const(self): + consts = [None, + True, False, + 1000, + 2.0, + 3j, + "unicode", + b'bytes', + (1, 2, 3)] + + code = '\n'.join(['x={!r}'.format(const) for const in consts]) + code += '\nx = ...' + consts.extend((Ellipsis, None)) + + tree = ast.parse(code) + self.assertEqual(self.get_load_const(tree), + consts) + + # Replace expression nodes with constants + for assign, const in zip(tree.body, consts): + assert isinstance(assign, ast.Assign), ast.dump(assign) + new_node = ast.Constant(value=const) + ast.copy_location(new_node, assign.value) + assign.value = new_node + + self.assertEqual(self.get_load_const(tree), + consts) + + def test_literal_eval(self): + tree = ast.parse("1 + 2") + binop = tree.body[0].value + + new_left = ast.Constant(value=10) + ast.copy_location(new_left, binop.left) + binop.left = new_left + + new_right = ast.Constant(value=20j) + ast.copy_location(new_right, binop.right) + binop.right = new_right + + self.assertEqual(ast.literal_eval(binop), 10+20j) + + def test_string_kind(self): + c = ast.parse('"x"', mode='eval').body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, None) + + c = ast.parse('u"x"', mode='eval').body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, "u") + + c = ast.parse('r"x"', mode='eval').body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, None) + + c = ast.parse('b"x"', mode='eval').body + self.assertEqual(c.value, b"x") + self.assertEqual(c.kind, None) + + +class EndPositionTests(unittest.TestCase): + """Tests for end position of AST nodes. + + Testing end positions of nodes requires a bit of extra care + because of how LL parsers work. + """ + def _check_end_pos(self, ast_node, end_lineno, end_col_offset): + self.assertEqual(ast_node.end_lineno, end_lineno) + self.assertEqual(ast_node.end_col_offset, end_col_offset) + + def _check_content(self, source, ast_node, content): + self.assertEqual(ast.get_source_segment(source, ast_node), content) + + def _parse_value(self, s): + # Use duck-typing to support both single expression + # and a right hand side of an assignment statement. + return ast.parse(s).body[0].value + + def test_lambda(self): + s = 'lambda x, *y: None' + lam = self._parse_value(s) + self._check_content(s, lam.body, 'None') + self._check_content(s, lam.args.args[0], 'x') + self._check_content(s, lam.args.vararg, 'y') + + def test_func_def(self): + s = dedent(''' + def func(x: int, + *args: str, + z: float = 0, + **kwargs: Any) -> bool: + return True + ''').strip() + fdef = ast.parse(s).body[0] + self._check_end_pos(fdef, 5, 15) + self._check_content(s, fdef.body[0], 'return True') + self._check_content(s, fdef.args.args[0], 'x: int') + self._check_content(s, fdef.args.args[0].annotation, 'int') + self._check_content(s, fdef.args.kwarg, 'kwargs: Any') + self._check_content(s, fdef.args.kwarg.annotation, 'Any') + + def test_call(self): + s = 'func(x, y=2, **kw)' + call = self._parse_value(s) + self._check_content(s, call.func, 'func') + self._check_content(s, call.keywords[0].value, '2') + self._check_content(s, call.keywords[1].value, 'kw') + + def test_call_noargs(self): + s = 'x[0]()' + call = self._parse_value(s) + self._check_content(s, call.func, 'x[0]') + self._check_end_pos(call, 1, 6) + + def test_class_def(self): + s = dedent(''' + class C(A, B): + x: int = 0 + ''').strip() + cdef = ast.parse(s).body[0] + self._check_end_pos(cdef, 2, 14) + self._check_content(s, cdef.bases[1], 'B') + self._check_content(s, cdef.body[0], 'x: int = 0') + + def test_class_kw(self): + s = 'class S(metaclass=abc.ABCMeta): pass' + cdef = ast.parse(s).body[0] + self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta') + + def test_multi_line_str(self): + s = dedent(''' + x = """Some multi-line text. + + It goes on starting from same indent.""" + ''').strip() + assign = ast.parse(s).body[0] + self._check_end_pos(assign, 3, 40) + self._check_end_pos(assign.value, 3, 40) + + def test_continued_str(self): + s = dedent(''' + x = "first part" \\ + "second part" + ''').strip() + assign = ast.parse(s).body[0] + self._check_end_pos(assign, 2, 13) + self._check_end_pos(assign.value, 2, 13) + + def test_suites(self): + # We intentionally put these into the same string to check + # that empty lines are not part of the suite. + s = dedent(''' + while True: + pass + + if one(): + x = None + elif other(): + y = None + else: + z = None + + for x, y in stuff: + assert True + + try: + raise RuntimeError + except TypeError as e: + pass + + pass + ''').strip() + mod = ast.parse(s) + while_loop = mod.body[0] + if_stmt = mod.body[1] + for_loop = mod.body[2] + try_stmt = mod.body[3] + pass_stmt = mod.body[4] + + self._check_end_pos(while_loop, 2, 8) + self._check_end_pos(if_stmt, 9, 12) + self._check_end_pos(for_loop, 12, 15) + self._check_end_pos(try_stmt, 17, 8) + self._check_end_pos(pass_stmt, 19, 4) + + self._check_content(s, while_loop.test, 'True') + self._check_content(s, if_stmt.body[0], 'x = None') + self._check_content(s, if_stmt.orelse[0].test, 'other()') + self._check_content(s, for_loop.target, 'x, y') + self._check_content(s, try_stmt.body[0], 'raise RuntimeError') + self._check_content(s, try_stmt.handlers[0].type, 'TypeError') + + def test_fstring(self): + s = 'x = f"abc {x + y} abc"' + fstr = self._parse_value(s) + binop = fstr.values[1].value + self._check_content(s, binop, 'x + y') + + def test_fstring_multi_line(self): + s = dedent(''' + f"""Some multi-line text. + { + arg_one + + + arg_two + } + It goes on...""" + ''').strip() + fstr = self._parse_value(s) + binop = fstr.values[1].value + self._check_end_pos(binop, 5, 7) + self._check_content(s, binop.left, 'arg_one') + self._check_content(s, binop.right, 'arg_two') + + def test_import_from_multi_line(self): + s = dedent(''' + from x.y.z import ( + a, b, c as c + ) + ''').strip() + imp = ast.parse(s).body[0] + self._check_end_pos(imp, 3, 1) + self._check_end_pos(imp.names[2], 2, 16) + + def test_slices(self): + s1 = 'f()[1, 2] [0]' + s2 = 'x[ a.b: c.d]' + sm = dedent(''' + x[ a.b: f () , + g () : c.d + ] + ''').strip() + i1, i2, im = map(self._parse_value, (s1, s2, sm)) + self._check_content(s1, i1.value, 'f()[1, 2]') + self._check_content(s1, i1.value.slice, '1, 2') + self._check_content(s2, i2.slice.lower, 'a.b') + self._check_content(s2, i2.slice.upper, 'c.d') + self._check_content(sm, im.slice.elts[0].upper, 'f ()') + self._check_content(sm, im.slice.elts[1].lower, 'g ()') + self._check_end_pos(im, 3, 3) + + def test_binop(self): + s = dedent(''' + (1 * 2 + (3 ) + + 4 + ) + ''').strip() + binop = self._parse_value(s) + self._check_end_pos(binop, 2, 6) + self._check_content(s, binop.right, '4') + self._check_content(s, binop.left, '1 * 2 + (3 )') + self._check_content(s, binop.left.right, '3') + + def test_boolop(self): + s = dedent(''' + if (one_condition and + (other_condition or yet_another_one)): + pass + ''').strip() + bop = ast.parse(s).body[0].test + self._check_end_pos(bop, 2, 44) + self._check_content(s, bop.values[1], + 'other_condition or yet_another_one') + + def test_tuples(self): + s1 = 'x = () ;' + s2 = 'x = 1 , ;' + s3 = 'x = (1 , 2 ) ;' + sm = dedent(''' + x = ( + a, b, + ) + ''').strip() + t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm)) + self._check_content(s1, t1, '()') + self._check_content(s2, t2, '1 ,') + self._check_content(s3, t3, '(1 , 2 )') + self._check_end_pos(tm, 3, 1) + + def test_attribute_spaces(self): + s = 'func(x. y .z)' + call = self._parse_value(s) + self._check_content(s, call, s) + self._check_content(s, call.args[0], 'x. y .z') + + def test_redundant_parenthesis(self): + s = '( ( ( a + b ) ) )' + v = ast.parse(s).body[0].value + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s, v, 'a + b') + s2 = 'await ' + s + v = ast.parse(s2).body[0].value.value + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s2, v, 'a + b') + + def test_trailers_with_redundant_parenthesis(self): + tests = ( + ('( ( ( a ) ) ) ( )', 'Call'), + ('( ( ( a ) ) ) ( b )', 'Call'), + ('( ( ( a ) ) ) [ b ]', 'Subscript'), + ('( ( ( a ) ) ) . b', 'Attribute'), + ) + for s, t in tests: + with self.subTest(s): + v = ast.parse(s).body[0].value + self.assertEqual(type(v).__name__, t) + self._check_content(s, v, s) + s2 = 'await ' + s + v = ast.parse(s2).body[0].value.value + self.assertEqual(type(v).__name__, t) + self._check_content(s2, v, s) + + def test_displays(self): + s1 = '[{}, {1, }, {1, 2,} ]' + s2 = '{a: b, f (): g () ,}' + c1 = self._parse_value(s1) + c2 = self._parse_value(s2) + self._check_content(s1, c1.elts[0], '{}') + self._check_content(s1, c1.elts[1], '{1, }') + self._check_content(s1, c1.elts[2], '{1, 2,}') + self._check_content(s2, c2.keys[1], 'f ()') + self._check_content(s2, c2.values[1], 'g ()') + + def test_comprehensions(self): + s = dedent(''' + x = [{x for x, y in stuff + if cond.x} for stuff in things] + ''').strip() + cmp = self._parse_value(s) + self._check_end_pos(cmp, 2, 37) + self._check_content(s, cmp.generators[0].iter, 'things') + self._check_content(s, cmp.elt.generators[0].iter, 'stuff') + self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x') + self._check_content(s, cmp.elt.generators[0].target, 'x, y') + + def test_yield_await(self): + s = dedent(''' + async def f(): + yield x + await y + ''').strip() + fdef = ast.parse(s).body[0] + self._check_content(s, fdef.body[0].value, 'yield x') + self._check_content(s, fdef.body[1].value, 'await y') + + def test_source_segment_multi(self): + s_orig = dedent(''' + x = ( + a, b, + ) + () + ''').strip() + s_tuple = dedent(''' + ( + a, b, + ) + ''').strip() + binop = self._parse_value(s_orig) + self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) + + def test_source_segment_padded(self): + s_orig = dedent(''' + class C: + def fun(self) -> None: + "ЖЖЖЖЖ" + ''').strip() + s_method = ' def fun(self) -> None:\n' \ + ' "ЖЖЖЖЖ"' + cdef = ast.parse(s_orig).body[0] + self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True), + s_method) + + def test_source_segment_endings(self): + s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n' + v, w, x, y, z = ast.parse(s).body + self._check_content(s, v, 'v = 1') + self._check_content(s, w, 'w = 1') + self._check_content(s, x, 'x = 1') + self._check_content(s, y, 'y = 1') + self._check_content(s, z, 'z = 1') + + def test_source_segment_tabs(self): + s = dedent(''' + class C: + \t\f def fun(self) -> None: + \t\f pass + ''').strip() + s_method = ' \t\f def fun(self) -> None:\n' \ + ' \t\f pass' + + cdef = ast.parse(s).body[0] + self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method) + + def test_source_segment_newlines(self): + s = 'def f():\n pass\ndef g():\r pass\r\ndef h():\r\n pass\r\n' + f, g, h = ast.parse(s).body + self._check_content(s, f, 'def f():\n pass') + self._check_content(s, g, 'def g():\r pass') + self._check_content(s, h, 'def h():\r\n pass') + + s = 'def f():\n a = 1\r b = 2\r\n c = 3\n' + f = ast.parse(s).body[0] + self._check_content(s, f, s.rstrip()) + + def test_source_segment_missing_info(self): + s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n' + v, w, x, y = ast.parse(s).body + del v.lineno + del w.end_lineno + del x.col_offset + del y.end_col_offset + self.assertIsNone(ast.get_source_segment(s, v)) + self.assertIsNone(ast.get_source_segment(s, w)) + self.assertIsNone(ast.get_source_segment(s, x)) + self.assertIsNone(ast.get_source_segment(s, y)) + + +class NodeTransformerTests(ASTTestMixin, unittest.TestCase): + def assertASTTransformation(self, transformer_class, + initial_code, expected_code): + initial_ast = ast.parse(dedent(initial_code)) + expected_ast = ast.parse(dedent(expected_code)) + + transformer = transformer_class() + result_ast = ast.fix_missing_locations(transformer.visit(initial_ast)) + + self.assertASTEqual(result_ast, expected_ast) + + def test_node_remove_single(self): + code = 'def func(arg) -> SomeType: ...' + expected = 'def func(arg): ...' + + # Since `FunctionDef.returns` is defined as a single value, we test + # the `if isinstance(old_value, AST):` branch here. + class SomeTypeRemover(ast.NodeTransformer): + def visit_Name(self, node: ast.Name): + self.generic_visit(node) + if node.id == 'SomeType': + return None + return node + + self.assertASTTransformation(SomeTypeRemover, code, expected) + + def test_node_remove_from_list(self): + code = """ + def func(arg): + print(arg) + yield arg + """ + expected = """ + def func(arg): + print(arg) + """ + + # Since `FunctionDef.body` is defined as a list, we test + # the `if isinstance(old_value, list):` branch here. + class YieldRemover(ast.NodeTransformer): + def visit_Expr(self, node: ast.Expr): + self.generic_visit(node) + if isinstance(node.value, ast.Yield): + return None # Remove `yield` from a function + return node + + self.assertASTTransformation(YieldRemover, code, expected) + + def test_node_return_list(self): + code = """ + class DSL(Base, kw1=True): ... + """ + expected = """ + class DSL(Base, kw1=True, kw2=True, kw3=False): ... + """ + + class ExtendKeywords(ast.NodeTransformer): + def visit_keyword(self, node: ast.keyword): + self.generic_visit(node) + if node.arg == 'kw1': + return [ + node, + ast.keyword('kw2', ast.Constant(True)), + ast.keyword('kw3', ast.Constant(False)), + ] + return node + + self.assertASTTransformation(ExtendKeywords, code, expected) + + def test_node_mutate(self): + code = """ + def func(arg): + print(arg) + """ + expected = """ + def func(arg): + log(arg) + """ + + class PrintToLog(ast.NodeTransformer): + def visit_Call(self, node: ast.Call): + self.generic_visit(node) + if isinstance(node.func, ast.Name) and node.func.id == 'print': + node.func.id = 'log' + return node + + self.assertASTTransformation(PrintToLog, code, expected) + + def test_node_replace(self): + code = """ + def func(arg): + print(arg) + """ + expected = """ + def func(arg): + logger.log(arg, debug=True) + """ + + class PrintToLog(ast.NodeTransformer): + def visit_Call(self, node: ast.Call): + self.generic_visit(node) + if isinstance(node.func, ast.Name) and node.func.id == 'print': + return ast.Call( + func=ast.Attribute( + ast.Name('logger', ctx=ast.Load()), + attr='log', + ctx=ast.Load(), + ), + args=node.args, + keywords=[ast.keyword('debug', ast.Constant(True))], + ) + return node + + self.assertASTTransformation(PrintToLog, code, expected) + + +class ASTConstructorTests(unittest.TestCase): + """Test the autogenerated constructors for AST nodes.""" + + def test_FunctionDef(self): + args = ast.arguments() + self.assertEqual(args.args, []) + self.assertEqual(args.posonlyargs, []) + with self.assertWarnsRegex(DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): + node = ast.FunctionDef(args=args) + self.assertNotHasAttr(node, "name") + self.assertEqual(node.decorator_list, []) + node = ast.FunctionDef(name='foo', args=args) + self.assertEqual(node.name, 'foo') + self.assertEqual(node.decorator_list, []) + + def test_expr_context(self): + name = ast.Name("x") + self.assertEqual(name.id, "x") + self.assertIsInstance(name.ctx, ast.Load) + + name2 = ast.Name("x", ast.Store()) + self.assertEqual(name2.id, "x") + self.assertIsInstance(name2.ctx, ast.Store) + + name3 = ast.Name("x", ctx=ast.Del()) + self.assertEqual(name3.id, "x") + self.assertIsInstance(name3.ctx, ast.Del) + + with self.assertWarnsRegex(DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'"): + name3 = ast.Name() + + def test_custom_subclass_with_no_fields(self): + class NoInit(ast.AST): + pass + + obj = NoInit() + self.assertIsInstance(obj, NoInit) + self.assertEqual(obj.__dict__, {}) + + def test_fields_but_no_field_types(self): + class Fields(ast.AST): + _fields = ('a',) + + obj = Fields() + with self.assertRaises(AttributeError): + obj.a + obj = Fields(a=1) + self.assertEqual(obj.a, 1) + + def test_fields_and_types(self): + class FieldsAndTypes(ast.AST): + _fields = ('a',) + _field_types = {'a': int | None} + a: int | None = None + + obj = FieldsAndTypes() + self.assertIs(obj.a, None) + obj = FieldsAndTypes(a=1) + self.assertEqual(obj.a, 1) + + def test_custom_attributes(self): + class MyAttrs(ast.AST): + _attributes = ("a", "b") + + obj = MyAttrs(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + with self.assertWarnsRegex(DeprecationWarning, + r"MyAttrs.__init__ got an unexpected keyword argument 'c'."): + obj = MyAttrs(c=3) + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_fields_and_types_no_default(self): + class FieldsAndTypesNoDefault(ast.AST): + _fields = ('a',) + _field_types = {'a': int} + + with self.assertWarnsRegex(DeprecationWarning, + r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\."): + obj = FieldsAndTypesNoDefault() + with self.assertRaises(AttributeError): + obj.a + obj = FieldsAndTypesNoDefault(a=1) + self.assertEqual(obj.a, 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_incomplete_field_types(self): + class MoreFieldsThanTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None} + a: int | None = None + b: int | None = None + + with self.assertWarnsRegex( + DeprecationWarning, + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" + ): + obj = MoreFieldsThanTypes() + self.assertIs(obj.a, None) + self.assertIs(obj.b, None) + + obj = MoreFieldsThanTypes(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'str' but 'bytes' found. + def test_malformed_fields_with_bytes(self): + class BadFields(ast.AST): + _fields = (b'\xff'*64,) + _field_types = {'a': int} + + # This should not crash + with self.assertWarnsRegex(DeprecationWarning, r"Field b'\\xff\\xff.*' .*"): + obj = BadFields() + + def test_complete_field_types(self): + class _AllFieldTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None, 'b': list[str]} + # This must be set explicitly + a: int | None = None + # This will add an implicit empty list default + b: list[str] + + obj = _AllFieldTypes() + self.assertIs(obj.a, None) + self.assertEqual(obj.b, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_non_str_kwarg(self): + warn_msg = "got an unexpected keyword argument ', 'eval', + flags=ast.PyCF_ONLY_AST) + code = compile(ast_tree, 'string', 'eval') + if not isinstance(code, types.CodeType): + raise AssertionError + + # Unloading the _ast module must not crash. + del ast, _ast + del sys.modules['ast'], sys.modules['_ast'] + gc.collect() + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + + +class CommandLineTests(unittest.TestCase): + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + return textwrap.dedent(string).strip() + + def set_source(self, content): + Path(self.filename).write_text(self.text_normalize(content)) + + def invoke_ast(self, *flags): + stderr = StringIO() + stdout = StringIO() + with ( + contextlib.redirect_stdout(stdout), + contextlib.redirect_stderr(stderr), + ): + ast.main(args=[*flags, self.filename]) + self.assertEqual(stderr.getvalue(), '') + return stdout.getvalue().strip() + + def check_output(self, source, expect, *flags): + self.set_source(source) + res = self.invoke_ast(*flags) + expect = self.text_normalize(expect) + self.assertEqual(res, expect) + + @support.requires_resource('cpu') + def test_invocation(self): + # test various combinations of parameters + base_flags = ( + ('-m=exec', '--mode=exec'), + ('--no-type-comments', '--no-type-comments'), + ('-a', '--include-attributes'), + ('-i=4', '--indent=4'), + ('--feature-version=3.13', '--feature-version=3.13'), + ('-O=-1', '--optimize=-1'), + ('--show-empty', '--show-empty'), + ) + self.set_source(''' + print(1, 2, 3) + def f(x: int) -> int: + x -= 1 + return x + ''') + + for r in range(1, len(base_flags) + 1): + for choices in itertools.combinations(base_flags, r=r): + for args in itertools.product(*choices): + with self.subTest(flags=args): + self.invoke_ast(*args) + + @support.force_not_colorized + def test_help_message(self): + for flag in ('-h', '--help', '--unknown'): + with self.subTest(flag=flag): + output = StringIO() + with self.assertRaises(SystemExit): + with contextlib.redirect_stderr(output): + ast.main(args=flag) + self.assertStartsWith(output.getvalue(), 'usage: ') + + def test_exec_mode_flag(self): + # test 'python -m ast -m/--mode exec' + source = 'x: bool = 1 # type: ignore[assignment]' + expect = ''' + Module( + body=[ + AnnAssign( + target=Name(id='x', ctx=Store()), + annotation=Name(id='bool', ctx=Load()), + value=Constant(value=1), + simple=1)], + type_ignores=[ + TypeIgnore(lineno=1, tag='[assignment]')]) + ''' + for flag in ('-m=exec', '--mode=exec'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_single_mode_flag(self): + # test 'python -m ast -m/--mode single' + source = 'pass' + expect = ''' + Interactive( + body=[ + Pass()]) + ''' + for flag in ('-m=single', '--mode=single'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_eval_mode_flag(self): + # test 'python -m ast -m/--mode eval' + source = 'print(1, 2, 3)' + expect = ''' + Expression( + body=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)])) + ''' + for flag in ('-m=eval', '--mode=eval'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_func_type_mode_flag(self): + # test 'python -m ast -m/--mode func_type' + source = '(int, str) -> list[int]' + expect = ''' + FunctionType( + argtypes=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + returns=Subscript( + value=Name(id='list', ctx=Load()), + slice=Name(id='int', ctx=Load()), + ctx=Load())) + ''' + for flag in ('-m=func_type', '--mode=func_type'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_no_type_comments_flag(self): + # test 'python -m ast --no-type-comments' + source = 'x: bool = 1 # type: ignore[assignment]' + expect = ''' + Module( + body=[ + AnnAssign( + target=Name(id='x', ctx=Store()), + annotation=Name(id='bool', ctx=Load()), + value=Constant(value=1), + simple=1)]) + ''' + self.check_output(source, expect, '--no-type-comments') + + def test_include_attributes_flag(self): + # test 'python -m ast -a/--include-attributes' + source = 'pass' + expect = ''' + Module( + body=[ + Pass( + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=4)]) + ''' + for flag in ('-a', '--include-attributes'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_indent_flag(self): + # test 'python -m ast -i/--indent 0' + source = 'pass' + expect = ''' + Module( + body=[ + Pass()]) + ''' + for flag in ('-i=0', '--indent=0'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_feature_version_flag(self): + # test 'python -m ast --feature-version 3.9/3.10' + source = ''' + match x: + case 1: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value=1)), + body=[ + Pass()])])]) + ''' + self.check_output(source, expect, '--feature-version=3.10') + with self.assertRaises(SyntaxError): + self.invoke_ast('--feature-version=3.9') + + def test_no_optimize_flag(self): + # test 'python -m ast -O/--optimize -1/0' + source = ''' + match a: + case 1+2j: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='a', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=BinOp( + left=Constant(value=1), + op=Add(), + right=Constant(value=2j))), + body=[ + Pass()])])]) + ''' + for flag in ('-O=-1', '--optimize=-1', '-O=0', '--optimize=0'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_optimize_flag(self): + # test 'python -m ast -O/--optimize 1/2' + source = ''' + match a: + case 1+2j: + pass + ''' + expect = ''' + Module( + body=[ + Match( + subject=Name(id='a', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value=(1+2j))), + body=[ + Pass()])])]) + ''' + for flag in ('-O=1', '--optimize=1', '-O=2', '--optimize=2'): + with self.subTest(flag=flag): + self.check_output(source, expect, flag) + + def test_show_empty_flag(self): + # test 'python -m ast --show-empty' + source = 'print(1, 2, 3)' + expect = ''' + Module( + body=[ + Expr( + value=Call( + func=Name(id='print', ctx=Load()), + args=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + keywords=[]))], + type_ignores=[]) + ''' + self.check_output(source, expect, '--show-empty') + + +class ASTOptimizationTests(unittest.TestCase): + def wrap_expr(self, expr): + return ast.Module(body=[ast.Expr(value=expr)]) + + def wrap_statement(self, statement): + return ast.Module(body=[statement]) + + def assert_ast(self, code, non_optimized_target, optimized_target): + non_optimized_tree = ast.parse(code, optimize=-1) + optimized_tree = ast.parse(code, optimize=1) + + # Is a non-optimized tree equal to a non-optimized target? + self.assertTrue( + ast.compare(non_optimized_tree, non_optimized_target), + f"{ast.dump(non_optimized_target)} must equal " + f"{ast.dump(non_optimized_tree)}", + ) + + # Is a optimized tree equal to a non-optimized target? + self.assertFalse( + ast.compare(optimized_tree, non_optimized_target), + f"{ast.dump(non_optimized_target)} must not equal " + f"{ast.dump(non_optimized_tree)}" + ) + + # Is a optimized tree is equal to an optimized target? + self.assertTrue( + ast.compare(optimized_tree, optimized_target), + f"{ast.dump(optimized_target)} must equal " + f"{ast.dump(optimized_tree)}", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_format(self): + code = "'%s' % (a,)" + + non_optimized_target = self.wrap_expr( + ast.BinOp( + left=ast.Constant(value="%s"), + op=ast.Mod(), + right=ast.Tuple(elts=[ast.Name(id='a')])) + ) + optimized_target = self.wrap_expr( + ast.JoinedStr( + values=[ + ast.FormattedValue(value=ast.Name(id='a'), conversion=115) + ] + ) + ) + + self.assert_ast(code, non_optimized_target, optimized_target) + + def test_folding_match_case_allowed_expressions(self): + def get_match_case_values(node): + result = [] + if isinstance(node, ast.Constant): + result.append(node.value) + elif isinstance(node, ast.MatchValue): + result.extend(get_match_case_values(node.value)) + elif isinstance(node, ast.MatchMapping): + for key in node.keys: + result.extend(get_match_case_values(key)) + elif isinstance(node, ast.MatchSequence): + for pat in node.patterns: + result.extend(get_match_case_values(pat)) + else: + self.fail(f"Unexpected node {node}") + return result + + tests = [ + ("-0", [0]), + ("-0.1", [-0.1]), + ("-0j", [complex(0, 0)]), + ("-0.1j", [complex(0, -0.1)]), + ("1 + 2j", [complex(1, 2)]), + ("1 - 2j", [complex(1, -2)]), + ("1.1 + 2.1j", [complex(1.1, 2.1)]), + ("1.1 - 2.1j", [complex(1.1, -2.1)]), + ("-0 + 1j", [complex(0, 1)]), + ("-0 - 1j", [complex(0, -1)]), + ("-0.1 + 1.1j", [complex(-0.1, 1.1)]), + ("-0.1 - 1.1j", [complex(-0.1, -1.1)]), + ("{-0: 0}", [0]), + ("{-0.1: 0}", [-0.1]), + ("{-0j: 0}", [complex(0, 0)]), + ("{-0.1j: 0}", [complex(0, -0.1)]), + ("{1 + 2j: 0}", [complex(1, 2)]), + ("{1 - 2j: 0}", [complex(1, -2)]), + ("{1.1 + 2.1j: 0}", [complex(1.1, 2.1)]), + ("{1.1 - 2.1j: 0}", [complex(1.1, -2.1)]), + ("{-0 + 1j: 0}", [complex(0, 1)]), + ("{-0 - 1j: 0}", [complex(0, -1)]), + ("{-0.1 + 1.1j: 0}", [complex(-0.1, 1.1)]), + ("{-0.1 - 1.1j: 0}", [complex(-0.1, -1.1)]), + ("{-0: 0, 0 + 1j: 0, 0.1 + 1j: 0}", [0, complex(0, 1), complex(0.1, 1)]), + ("[-0, -0.1, -0j, -0.1j]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[[[-0, -0.1, -0j, -0.1j]]]]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[-0, -0.1], -0j, -0.1j]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("[[-0, -0.1], [-0j, -0.1j]]", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("(-0, -0.1, -0j, -0.1j)", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((((-0, -0.1, -0j, -0.1j))))", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((-0, -0.1), -0j, -0.1j)", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ("((-0, -0.1), (-0j, -0.1j))", [0, -0.1, complex(0, 0), complex(0, -0.1)]), + ] + for match_expr, constants in tests: + with self.subTest(match_expr): + src = f"match 0:\n\t case {match_expr}: pass" + tree = ast.parse(src, optimize=1) + match_stmt = tree.body[0] + case = match_stmt.cases[0] + values = get_match_case_values(case.pattern) + self.assertListEqual(constants, values) + + def test_match_case_not_folded_in_unoptimized_ast(self): + src = textwrap.dedent(""" + match a: + case 1+2j: + pass + """) + + unfolded = "MatchValue(value=BinOp(left=Constant(value=1), op=Add(), right=Constant(value=2j))" + folded = "MatchValue(value=Constant(value=(1+2j)))" + for optval in (0, 1, 2): + self.assertIn(folded if optval else unfolded, ast.dump(ast.parse(src, optimize=optval))) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_binop(self): + return super().test_folding_binop() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_comparator(self): + return super().test_folding_comparator() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_iter(self): + return super().test_folding_iter() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_not(self): + return super().test_folding_not() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_subscript(self): + return super().test_folding_subscript() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_tuple(self): + return super().test_folding_tuple() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_type_param_in_class_def(self): + return super().test_folding_type_param_in_class_def() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_type_param_in_function_def(self): + return super().test_folding_type_param_in_function_def() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_type_param_in_type_alias(self): + return super().test_folding_type_param_in_type_alias() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_unaryop(self): + return super().test_folding_unaryop() + + +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + ast_repr_update_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_ast/utils.py b/Lib/test/test_ast/utils.py new file mode 100644 index 00000000000..e7054f3f710 --- /dev/null +++ b/Lib/test/test_ast/utils.py @@ -0,0 +1,15 @@ +def to_tuple(t): + if t is None or isinstance(t, (str, int, complex, float, bytes, tuple)) or t is Ellipsis: + return t + elif isinstance(t, list): + return [to_tuple(e) for e in t] + result = [t.__class__.__name__] + if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): + result.append((t.lineno, t.col_offset)) + if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): + result[-1] += (t.end_lineno, t.end_col_offset) + if t._fields is None: + return tuple(result) + for f in t._fields: + result.append(to_tuple(getattr(t, f))) + return tuple(result) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index f6d05a61437..181476e0989 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -4,10 +4,12 @@ import contextlib from test.support.import_helper import import_module -from test.support import gc_collect +from test.support import gc_collect, requires_working_socket asyncio = import_module("asyncio") +requires_working_socket(module=True) + _no_default = object() @@ -375,8 +377,178 @@ async def async_gen_wrapper(): self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_async_gen_exception_12(self): + async def gen(): + with self.assertWarnsRegex(RuntimeWarning, + f"coroutine method 'asend' of '{gen.__qualname__}' " + f"was never awaited"): + await anext(me) + yield 123 + + me = gen() + ai = me.__aiter__() + an = ai.__anext__() + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + an.__next__() + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + an.send(None) + + def test_async_gen_asend_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_asend_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.asend(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.athrow(None) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_3_arg_deprecation_warning(self): + async def gen(): + yield 123 + + with self.assertWarns(DeprecationWarning): + x = gen().athrow(GeneratorExit, GeneratorExit(), None) + with self.assertRaises(GeneratorExit): + x.send(None) + del x + gc_collect() + def test_async_gen_api_01(self): async def gen(): yield 123 @@ -395,8 +567,57 @@ async def gen(): self.assertIsInstance(g.ag_frame, types.FrameType) self.assertFalse(g.ag_running) self.assertIsInstance(g.ag_code, types.CodeType) + aclose = g.aclose() + self.assertTrue(inspect.isawaitable(aclose)) + aclose.close() + + def test_async_gen_asend_close_runtime_error(self): + import types - self.assertTrue(inspect.isawaitable(g.aclose())) + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + + def test_async_gen_athrow_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + gen = agen.athrow(MyExc) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() class AsyncGenAsyncioTest(unittest.TestCase): @@ -408,7 +629,7 @@ def setUp(self): def tearDown(self): self.loop.close() self.loop = None - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def check_async_iterator_anext(self, ait_class): with self.subTest(anext="pure-Python"): @@ -467,16 +688,12 @@ async def test_throw(): result = self.loop.run_until_complete(test_throw()) self.assertEqual(result, "completed") - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_async_generator_anext(self): async def agen(): yield 1 yield 2 self.check_async_iterator_anext(agen) - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_python_async_iterator_anext(self): class MyAsyncIter: """Asynchronously yield 1, then 2.""" @@ -492,8 +709,6 @@ async def __anext__(self): return self.yielded self.check_async_iterator_anext(MyAsyncIter) - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_python_async_iterator_types_coroutine_anext(self): import types class MyAsyncIterWithTypesCoro: @@ -513,19 +728,16 @@ def __anext__(self): return self.yielded self.check_async_iterator_anext(MyAsyncIterWithTypesCoro) - # TODO: RUSTPYTHON: async for gen expression compilation - # def test_async_gen_aiter(self): - # async def gen(): - # yield 1 - # yield 2 - # g = gen() - # async def consume(): - # return [i async for i in aiter(g)] - # res = self.loop.run_until_complete(consume()) - # self.assertEqual(res, [1, 2]) - - # TODO: RUSTPYTHON, NameError: name 'aiter' is not defined - @unittest.expectedFailure + def test_async_gen_aiter(self): + async def gen(): + yield 1 + yield 2 + g = gen() + async def consume(): + return [i async for i in aiter(g)] + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2]) + def test_async_gen_aiter_class(self): results = [] class Gen: @@ -550,8 +762,6 @@ async def gen(): applied_twice = aiter(applied_once) self.assertIs(applied_once, applied_twice) - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_bad_args(self): async def gen(): yield 1 @@ -572,8 +782,6 @@ async def call_with_kwarg(): with self.assertRaises(TypeError): self.loop.run_until_complete(call_with_kwarg()) - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_bad_await(self): async def bad_awaitable(): class BadAwaitable: @@ -604,8 +812,6 @@ async def check_anext_returning_iterator(self, aiter_class): await awaitable return "completed" - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_return_iterator(self): class WithIterAnext: def __aiter__(self): @@ -615,8 +821,6 @@ def __anext__(self): result = self.loop.run_until_complete(self.check_anext_returning_iterator(WithIterAnext)) self.assertEqual(result, "completed") - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_return_generator(self): class WithGenAnext: def __aiter__(self): @@ -626,8 +830,6 @@ def __anext__(self): result = self.loop.run_until_complete(self.check_anext_returning_iterator(WithGenAnext)) self.assertEqual(result, "completed") - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_await_raises(self): class RaisingAwaitable: def __await__(self): @@ -649,8 +851,6 @@ async def do_test(): result = self.loop.run_until_complete(do_test()) self.assertEqual(result, "completed") - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_iter(self): @types.coroutine def _async_yield(v): @@ -671,7 +871,7 @@ def test1(anext): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 1) - self.assertEqual(g.throw(MyError, MyError(), None), 2) + self.assertEqual(g.throw(MyError()), 2) try: g.send(None) except StopIteration as e: @@ -684,9 +884,9 @@ def test2(anext): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 1) - self.assertEqual(g.throw(MyError, MyError(), None), 2) + self.assertEqual(g.throw(MyError()), 2) with self.assertRaises(MyError): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def test3(anext): agen = agenfn() @@ -713,9 +913,9 @@ async def agenfn(): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 10) - self.assertEqual(g.throw(MyError, MyError(), None), 20) + self.assertEqual(g.throw(MyError()), 20) with self.assertRaisesRegex(MyError, 'val'): - g.throw(MyError, MyError('val'), None) + g.throw(MyError('val')) def test5(anext): @types.coroutine @@ -734,7 +934,7 @@ async def agenfn(): with contextlib.closing(anext(agen, "default").__await__()) as g: self.assertEqual(g.send(None), 10) with self.assertRaisesRegex(StopIteration, 'default'): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def test6(anext): @types.coroutine @@ -749,7 +949,7 @@ async def agenfn(): agen = agenfn() with contextlib.closing(anext(agen, "default").__await__()) as g: with self.assertRaises(MyError): - g.throw(MyError, MyError(), None) + g.throw(MyError()) def run_test(test): with self.subTest('pure-Python anext()'): @@ -952,6 +1152,43 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_asyncio_anext_tuple_no_exceptions(self): + # StopAsyncIteration exceptions should be cleared. + # See: https://github.com/python/cpython/issues/128078. + + async def foo(): + if False: + yield (1, 2) + + async def run(): + it = foo().__aiter__() + with self.assertRaises(StopAsyncIteration): + await it.__anext__() + res = await anext(it, ('a', 'b')) + self.assertTupleEqual(res, ('a', 'b')) + + self.loop.run_until_complete(run()) + + def test_sync_anext_raises_exception(self): + # See: https://github.com/python/cpython/issues/131670 + msg = 'custom' + for exc_type in [ + StopAsyncIteration, + StopIteration, + ValueError, + Exception, + ]: + exc = exc_type(msg) + with self.subTest(exc=exc): + class A: + def __anext__(self): + raise exc + + with self.assertRaisesRegex(exc_type, msg): + anext(A()) + with self.assertRaisesRegex(exc_type, msg): + anext(A(), 1) + def test_async_gen_asyncio_anext_stopiteration(self): async def foo(): try: @@ -1049,8 +1286,6 @@ async def run(): fut.cancel() self.loop.run_until_complete(asyncio.sleep(0.01)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_async_gen_asyncio_gc_aclose_09(self): DONE = 0 @@ -1060,8 +1295,7 @@ async def gen(): while True: yield 1 finally: - await asyncio.sleep(0.01) - await asyncio.sleep(0.01) + await asyncio.sleep(0) DONE = 1 async def run(): @@ -1071,7 +1305,10 @@ async def run(): del g gc_collect() # For PyPy or other GCs. - await asyncio.sleep(0.1) + # Starts running the aclose task + await asyncio.sleep(0) + # For asyncio.sleep(0) in finally block + await asyncio.sleep(0) self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) @@ -1508,8 +1745,6 @@ async def main(): self.assertEqual(messages, []) - # TODO: RUSTPYTHON, ValueError: not enough values to unpack (expected 1, got 0) - @unittest.expectedFailure def test_async_gen_asyncio_shutdown_exception_01(self): messages = [] @@ -1539,8 +1774,6 @@ async def main(): self.assertIn('an error occurred during closing of asynchronous generator', message['message']) - # TODO: RUSTPYTHON, ValueError: not enough values to unpack (expected 1, got 0) - @unittest.expectedFailure def test_async_gen_asyncio_shutdown_exception_02(self): messages = [] @@ -1568,39 +1801,54 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) + del message, messages + gc_collect() + + def test_async_gen_expression_01(self): + async def arange(n): + for i in range(n): + await asyncio.sleep(0.01) + yield i + + def make_arange(n): + # This syntax is legal starting with Python 3.7 + return (i * 2 async for i in arange(n)) + + async def run(): + return [i async for i in make_arange(10)] + + res = self.loop.run_until_complete(run()) + self.assertEqual(res, [i * 2 for i in range(10)]) - # TODO: RUSTPYTHON: async for gen expression compilation - # def test_async_gen_expression_01(self): - # async def arange(n): - # for i in range(n): - # await asyncio.sleep(0.01) - # yield i + def test_async_gen_expression_02(self): + async def wrap(n): + await asyncio.sleep(0.01) + return n - # def make_arange(n): - # # This syntax is legal starting with Python 3.7 - # return (i * 2 async for i in arange(n)) + def make_arange(n): + # This syntax is legal starting with Python 3.7 + return (i * 2 for i in range(n) if await wrap(i)) - # async def run(): - # return [i async for i in make_arange(10)] + async def run(): + return [i async for i in make_arange(10)] - # res = self.loop.run_until_complete(run()) - # self.assertEqual(res, [i * 2 for i in range(10)]) + res = self.loop.run_until_complete(run()) + self.assertEqual(res, [i * 2 for i in range(1, 10)]) - # TODO: RUSTPYTHON: async for gen expression compilation - # def test_async_gen_expression_02(self): - # async def wrap(n): - # await asyncio.sleep(0.01) - # return n + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: __aiter__ + def test_async_gen_expression_incorrect(self): + async def ag(): + yield 42 - # def make_arange(n): - # # This syntax is legal starting with Python 3.7 - # return (i * 2 for i in range(n) if await wrap(i)) + async def run(arg): + (x async for x in arg) - # async def run(): - # return [i async for i in make_arange(10)] + err_msg_async = "'async for' requires an object with " \ + "__aiter__ method, got .*" - # res = self.loop.run_until_complete(run()) - # self.assertEqual(res, [i * 2 for i in range(1, 10)]) + self.loop.run_until_complete(run(ag())) + with self.assertRaisesRegex(TypeError, err_msg_async): + self.loop.run_until_complete(run(None)) def test_asyncgen_nonstarted_hooks_are_cancellable(self): # See https://bugs.python.org/issue38013 @@ -1623,6 +1871,7 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1660,6 +1909,62 @@ async def run(): self.loop.run_until_complete(run()) + def test_async_gen_throw_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + nxt = it.aclose() + with self.assertRaises(StopIteration): + nxt.throw(GeneratorExit) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(GeneratorExit) + + def test_async_gen_throw_custom_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.aclose() + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + + def test_async_gen_throw_custom_same_athrow_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + it = async_iterate() + + class MyException(Exception): + pass + + nxt = it.athrow(MyException) + with self.assertRaises(MyException): + nxt.throw(MyException) + + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + nxt.throw(MyException) + def test_async_gen_aclose_twice_with_different_coros(self): # Regression test for https://bugs.python.org/issue39606 async def async_iterate(): @@ -1702,5 +2007,109 @@ async def run(): self.loop.run_until_complete(run()) +class TestUnawaitedWarnings(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_asend(self): + async def gen(): + yield 1 + + # gh-113753: asend objects allocated from a free-list should warn. + # Ensure there is a finalized 'asend' object ready to be reused. + try: + g = gen() + g.asend(None).send(None) + except StopIteration: + pass + + msg = f"coroutine method 'asend' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.asend(None) + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_athrow(self): + async def gen(): + yield 1 + + msg = f"coroutine method 'athrow' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.athrow(RuntimeError) + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_aclose(self): + async def gen(): + yield 1 + + msg = f"coroutine method 'aclose' of '{gen.__qualname__}' was never awaited" + with self.assertWarnsRegex(RuntimeWarning, msg): + g = gen() + g.aclose() + gc_collect() + + def test_aclose_throw(self): + async def gen(): + return + yield + + class MyException(Exception): + pass + + g = gen() + with self.assertRaises(MyException): + g.aclose().throw(MyException) + + del g + gc_collect() # does not warn unawaited + + def test_asend_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + + + def test_athrow_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(Exception) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py deleted file mode 100644 index 1fcc882ce63..00000000000 --- a/Lib/test/test_asynchat.py +++ /dev/null @@ -1,290 +0,0 @@ -# test asynchat - -from test import support -from test.support import socket_helper -from test.support import threading_helper - - -import asynchat -import asyncore -import errno -import socket -import sys -import threading -import time -import unittest -import unittest.mock - -HOST = socket_helper.HOST -SERVER_QUIT = b'QUIT\n' -TIMEOUT = 3.0 - - -class echo_server(threading.Thread): - # parameter to determine the number of bytes passed back to the - # client each send - chunk_size = 1 - - def __init__(self, event): - threading.Thread.__init__(self) - self.event = event - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.port = socket_helper.bind_port(self.sock) - # This will be set if the client wants us to wait before echoing - # data back. - self.start_resend_event = None - - def run(self): - self.sock.listen() - self.event.set() - conn, client = self.sock.accept() - self.buffer = b"" - # collect data until quit message is seen - while SERVER_QUIT not in self.buffer: - data = conn.recv(1) - if not data: - break - self.buffer = self.buffer + data - - # remove the SERVER_QUIT message - self.buffer = self.buffer.replace(SERVER_QUIT, b'') - - if self.start_resend_event: - self.start_resend_event.wait() - - # re-send entire set of collected data - try: - # this may fail on some tests, such as test_close_when_done, - # since the client closes the channel when it's done sending - while self.buffer: - n = conn.send(self.buffer[:self.chunk_size]) - time.sleep(0.001) - self.buffer = self.buffer[n:] - except: - pass - - conn.close() - self.sock.close() - -class echo_client(asynchat.async_chat): - - def __init__(self, terminator, server_port): - asynchat.async_chat.__init__(self) - self.contents = [] - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.connect((HOST, server_port)) - self.set_terminator(terminator) - self.buffer = b"" - - def handle_connect(self): - pass - - if sys.platform == 'darwin': - # select.poll returns a select.POLLHUP at the end of the tests - # on darwin, so just ignore it - def handle_expt(self): - pass - - def collect_incoming_data(self, data): - self.buffer += data - - def found_terminator(self): - self.contents.append(self.buffer) - self.buffer = b"" - -def start_echo_server(): - event = threading.Event() - s = echo_server(event) - s.start() - event.wait() - event.clear() - time.sleep(0.01) # Give server time to start accepting. - return s, event - - -class TestAsynchat(unittest.TestCase): - usepoll = False - - def setUp(self): - self._threads = threading_helper.threading_setup() - - def tearDown(self): - threading_helper.threading_cleanup(*self._threads) - - def line_terminator_check(self, term, server_chunk): - event = threading.Event() - s = echo_server(event) - s.chunk_size = server_chunk - s.start() - event.wait() - event.clear() - time.sleep(0.01) # Give server time to start accepting. - c = echo_client(term, s.port) - c.push(b"hello ") - c.push(b"world" + term) - c.push(b"I'm not dead yet!" + term) - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) - - # the line terminator tests below check receiving variously-sized - # chunks back from the server in order to exercise all branches of - # async_chat.handle_read - - def test_line_terminator1(self): - # test one-character terminator - for l in (1, 2, 3): - self.line_terminator_check(b'\n', l) - - def test_line_terminator2(self): - # test two-character terminator - for l in (1, 2, 3): - self.line_terminator_check(b'\r\n', l) - - def test_line_terminator3(self): - # test three-character terminator - for l in (1, 2, 3): - self.line_terminator_check(b'qqq', l) - - def numeric_terminator_check(self, termlen): - # Try reading a fixed number of bytes - s, event = start_echo_server() - c = echo_client(termlen, s.port) - data = b"hello world, I'm not dead yet!\n" - c.push(data) - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, [data[:termlen]]) - - def test_numeric_terminator1(self): - # check that ints & longs both work (since type is - # explicitly checked in async_chat.handle_read) - self.numeric_terminator_check(1) - - def test_numeric_terminator2(self): - self.numeric_terminator_check(6) - - def test_none_terminator(self): - # Try reading a fixed number of bytes - s, event = start_echo_server() - c = echo_client(None, s.port) - data = b"hello world, I'm not dead yet!\n" - c.push(data) - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, []) - self.assertEqual(c.buffer, data) - - def test_simple_producer(self): - s, event = start_echo_server() - c = echo_client(b'\n', s.port) - data = b"hello world\nI'm not dead yet!\n" - p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8) - c.push_with_producer(p) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) - - def test_string_producer(self): - s, event = start_echo_server() - c = echo_client(b'\n', s.port) - data = b"hello world\nI'm not dead yet!\n" - c.push_with_producer(data+SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) - - def test_empty_line(self): - # checks that empty lines are handled correctly - s, event = start_echo_server() - c = echo_client(b'\n', s.port) - c.push(b"hello world\n\nI'm not dead yet!\n") - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, - [b"hello world", b"", b"I'm not dead yet!"]) - - def test_close_when_done(self): - s, event = start_echo_server() - s.start_resend_event = threading.Event() - c = echo_client(b'\n', s.port) - c.push(b"hello world\nI'm not dead yet!\n") - c.push(SERVER_QUIT) - c.close_when_done() - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - - # Only allow the server to start echoing data back to the client after - # the client has closed its connection. This prevents a race condition - # where the server echoes all of its data before we can check that it - # got any down below. - s.start_resend_event.set() - threading_helper.join_thread(s) - - self.assertEqual(c.contents, []) - # the server might have been able to send a byte or two back, but this - # at least checks that it received something and didn't just fail - # (which could still result in the client not having received anything) - self.assertGreater(len(s.buffer), 0) - - def test_push(self): - # Issue #12523: push() should raise a TypeError if it doesn't get - # a bytes string - s, event = start_echo_server() - c = echo_client(b'\n', s.port) - data = b'bytes\n' - c.push(data) - c.push(bytearray(data)) - c.push(memoryview(data)) - self.assertRaises(TypeError, c.push, 10) - self.assertRaises(TypeError, c.push, 'unicode') - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - self.assertEqual(c.contents, [b'bytes', b'bytes', b'bytes']) - - -class TestAsynchat_WithPoll(TestAsynchat): - usepoll = True - - -class TestAsynchatMocked(unittest.TestCase): - def test_blockingioerror(self): - # Issue #16133: handle_read() must ignore BlockingIOError - sock = unittest.mock.Mock() - sock.recv.side_effect = BlockingIOError(errno.EAGAIN) - - dispatcher = asynchat.async_chat() - dispatcher.set_socket(sock) - self.addCleanup(dispatcher.del_channel) - - with unittest.mock.patch.object(dispatcher, 'handle_error') as error: - dispatcher.handle_read() - self.assertFalse(error.called) - - -class TestHelperFunctions(unittest.TestCase): - def test_find_prefix_at_end(self): - self.assertEqual(asynchat.find_prefix_at_end("qwerty\r", "\r\n"), 1) - self.assertEqual(asynchat.find_prefix_at_end("qwertydkjf", "\r\n"), 0) - - -class TestNotConnected(unittest.TestCase): - def test_disallow_negative_terminator(self): - # Issue #11259 - client = asynchat.async_chat() - self.assertRaises(ValueError, client.set_terminator, -1) - - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_asyncio/__init__.py b/Lib/test/test_asyncio/__init__.py new file mode 100644 index 00000000000..ab0b5aa9489 --- /dev/null +++ b/Lib/test/test_asyncio/__init__.py @@ -0,0 +1,12 @@ +import os +from test import support +from test.support import load_package_tests +from test.support import import_helper + +support.requires_working_socket(module=True) + +# Skip tests if we don't have concurrent.futures. +import_helper.import_module('concurrent.futures') + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_asyncio/__main__.py b/Lib/test/test_asyncio/__main__.py new file mode 100644 index 00000000000..40a23a297ec --- /dev/null +++ b/Lib/test/test_asyncio/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_asyncio/echo.py b/Lib/test/test_asyncio/echo.py new file mode 100644 index 00000000000..006364bb007 --- /dev/null +++ b/Lib/test/test_asyncio/echo.py @@ -0,0 +1,8 @@ +import os + +if __name__ == '__main__': + while True: + buf = os.read(0, 1024) + if not buf: + break + os.write(1, buf) diff --git a/Lib/test/test_asyncio/echo2.py b/Lib/test/test_asyncio/echo2.py new file mode 100644 index 00000000000..e83ca09fb7a --- /dev/null +++ b/Lib/test/test_asyncio/echo2.py @@ -0,0 +1,6 @@ +import os + +if __name__ == '__main__': + buf = os.read(0, 1024) + os.write(1, b'OUT:'+buf) + os.write(2, b'ERR:'+buf) diff --git a/Lib/test/test_asyncio/echo3.py b/Lib/test/test_asyncio/echo3.py new file mode 100644 index 00000000000..064496736bf --- /dev/null +++ b/Lib/test/test_asyncio/echo3.py @@ -0,0 +1,11 @@ +import os + +if __name__ == '__main__': + while True: + buf = os.read(0, 1024) + if not buf: + break + try: + os.write(1, b'OUT:'+buf) + except OSError as ex: + os.write(2, b'ERR:' + ex.__class__.__name__.encode('ascii')) diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py new file mode 100644 index 00000000000..96dc9ab4401 --- /dev/null +++ b/Lib/test/test_asyncio/functional.py @@ -0,0 +1,268 @@ +import asyncio +import asyncio.events +import contextlib +import os +import pprint +import select +import socket +import tempfile +import threading +from test import support + + +class FunctionalTestCaseMixin: + + def new_loop(self): + return asyncio.new_event_loop() + + def run_loop_briefly(self, *, delay=0.01): + self.loop.run_until_complete(asyncio.sleep(delay)) + + def loop_exception_handler(self, loop, context): + self.__unhandled_exceptions.append(context) + self.loop.default_exception_handler(context) + + def setUp(self): + self.loop = self.new_loop() + asyncio.set_event_loop(None) + + self.loop.set_exception_handler(self.loop_exception_handler) + self.__unhandled_exceptions = [] + + def tearDown(self): + try: + self.loop.close() + + if self.__unhandled_exceptions: + print('Unexpected calls to loop.call_exception_handler():') + pprint.pprint(self.__unhandled_exceptions) + self.fail('unexpected calls to loop.call_exception_handler()') + + finally: + asyncio.set_event_loop(None) + self.loop = None + + def tcp_server(self, server_prog, *, + family=socket.AF_INET, + addr=None, + timeout=support.LOOPBACK_TIMEOUT, + backlog=1, + max_clients=10): + + if addr is None: + if hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX: + with tempfile.NamedTemporaryFile() as tmp: + addr = tmp.name + else: + addr = ('127.0.0.1', 0) + + sock = socket.create_server(addr, family=family, backlog=backlog) + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + return TestThreadedServer( + self, sock, server_prog, timeout, max_clients) + + def tcp_client(self, client_prog, + family=socket.AF_INET, + timeout=support.LOOPBACK_TIMEOUT): + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + return TestThreadedClient( + self, sock, client_prog, timeout) + + def unix_server(self, *args, **kwargs): + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError + return self.tcp_server(*args, family=socket.AF_UNIX, **kwargs) + + def unix_client(self, *args, **kwargs): + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError + return self.tcp_client(*args, family=socket.AF_UNIX, **kwargs) + + @contextlib.contextmanager + def unix_sock_name(self): + with tempfile.TemporaryDirectory() as td: + fn = os.path.join(td, 'sock') + try: + yield fn + finally: + try: + os.unlink(fn) + except OSError: + pass + + def _abort_socket_test(self, ex): + try: + self.loop.stop() + finally: + self.fail(ex) + + +############################################################################## +# Socket Testing Utilities +############################################################################## + + +class TestSocketWrapper: + + def __init__(self, sock): + self.__sock = sock + + def recv_all(self, n): + buf = b'' + while len(buf) < n: + data = self.recv(n - len(buf)) + if data == b'': + raise ConnectionAbortedError + buf += data + return buf + + def start_tls(self, ssl_context, *, + server_side=False, + server_hostname=None): + + ssl_sock = ssl_context.wrap_socket( + self.__sock, server_side=server_side, + server_hostname=server_hostname, + do_handshake_on_connect=False) + + try: + ssl_sock.do_handshake() + except: + ssl_sock.close() + raise + finally: + self.__sock.close() + + self.__sock = ssl_sock + + def __getattr__(self, name): + return getattr(self.__sock, name) + + def __repr__(self): + return '<{} {!r}>'.format(type(self).__name__, self.__sock) + + +class SocketThread(threading.Thread): + + def stop(self): + self._active = False + self.join() + + def __enter__(self): + self.start() + return self + + def __exit__(self, *exc): + self.stop() + + +class TestThreadedClient(SocketThread): + + def __init__(self, test, sock, prog, timeout): + threading.Thread.__init__(self, None, None, 'test-client') + self.daemon = True + + self._timeout = timeout + self._sock = sock + self._active = True + self._prog = prog + self._test = test + + def run(self): + try: + self._prog(TestSocketWrapper(self._sock)) + except Exception as ex: + self._test._abort_socket_test(ex) + + +class TestThreadedServer(SocketThread): + + def __init__(self, test, sock, prog, timeout, max_clients): + threading.Thread.__init__(self, None, None, 'test-server') + self.daemon = True + + self._clients = 0 + self._finished_clients = 0 + self._max_clients = max_clients + self._timeout = timeout + self._sock = sock + self._active = True + + self._prog = prog + + self._s1, self._s2 = socket.socketpair() + self._s1.setblocking(False) + + self._test = test + + def stop(self): + try: + if self._s2 and self._s2.fileno() != -1: + try: + self._s2.send(b'stop') + except OSError: + pass + finally: + super().stop() + self._sock.close() + self._s1.close() + self._s2.close() + + + def run(self): + self._sock.setblocking(False) + self._run() + + def _run(self): + while self._active: + if self._clients >= self._max_clients: + return + + r, w, x = select.select( + [self._sock, self._s1], [], [], self._timeout) + + if self._s1 in r: + return + + if self._sock in r: + try: + conn, addr = self._sock.accept() + except BlockingIOError: + continue + except TimeoutError: + if not self._active: + return + else: + raise + else: + self._clients += 1 + conn.settimeout(self._timeout) + try: + with conn: + self._handle_client(conn) + except Exception as ex: + self._active = False + try: + raise + finally: + self._test._abort_socket_test(ex) + + def _handle_client(self, sock): + self._prog(TestSocketWrapper(sock)) + + @property + def addr(self): + return self._sock.getsockname() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py new file mode 100644 index 00000000000..92895bbb420 --- /dev/null +++ b/Lib/test/test_asyncio/test_base_events.py @@ -0,0 +1,2297 @@ +"""Tests for base_events.py""" + +import concurrent.futures +import errno +import math +import platform +import socket +import sys +import threading +import time +import unittest +from unittest import mock + +import asyncio +from asyncio import base_events +from asyncio import constants +from test.test_asyncio import utils as test_utils +from test import support +from test.support.script_helper import assert_python_ok +from test.support import os_helper +from test.support import socket_helper +import warnings + +MOCK_ANY = mock.ANY + + +class CustomError(Exception): + pass + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def mock_socket_module(): + m_socket = mock.MagicMock(spec=socket) + for name in ( + 'AF_INET', 'AF_INET6', 'AF_UNSPEC', 'IPPROTO_TCP', 'IPPROTO_UDP', + 'SOCK_STREAM', 'SOCK_DGRAM', 'SOL_SOCKET', 'SO_REUSEADDR', 'inet_pton' + ): + if hasattr(socket, name): + setattr(m_socket, name, getattr(socket, name)) + else: + delattr(m_socket, name) + + m_socket.socket = mock.MagicMock() + m_socket.socket.return_value = test_utils.mock_nonblocking_socket() + + return m_socket + + +def patch_socket(f): + return mock.patch('asyncio.base_events.socket', + new_callable=mock_socket_module)(f) + + +class BaseEventTests(test_utils.TestCase): + + def test_ipaddr_info(self): + UNSPEC = socket.AF_UNSPEC + INET = socket.AF_INET + INET6 = socket.AF_INET6 + STREAM = socket.SOCK_STREAM + DGRAM = socket.SOCK_DGRAM + TCP = socket.IPPROTO_TCP + UDP = socket.IPPROTO_UDP + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info(b'1.2.3.4', 1, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, STREAM, TCP)) + + self.assertEqual( + (INET, DGRAM, UDP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, DGRAM, UDP)) + + # Socket type STREAM implies TCP protocol. + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, STREAM, 0)) + + # Socket type DGRAM implies UDP protocol. + self.assertEqual( + (INET, DGRAM, UDP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, DGRAM, 0)) + + # No socket type. + self.assertIsNone( + base_events._ipaddr_info('1.2.3.4', 1, UNSPEC, 0, 0)) + + if socket_helper.IPV6_ENABLED: + # IPv4 address with family IPv6. + self.assertIsNone( + base_events._ipaddr_info('1.2.3.4', 1, INET6, STREAM, TCP)) + + self.assertEqual( + (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)), + base_events._ipaddr_info('::3', 1, INET6, STREAM, TCP)) + + self.assertEqual( + (INET6, STREAM, TCP, '', ('::3', 1, 0, 0)), + base_events._ipaddr_info('::3', 1, UNSPEC, STREAM, TCP)) + + # IPv6 address with family IPv4. + self.assertIsNone( + base_events._ipaddr_info('::3', 1, INET, STREAM, TCP)) + + # IPv6 address with zone index. + self.assertIsNone( + base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP)) + + def test_port_parameter_types(self): + # Test obscure kinds of arguments for "port". + INET = socket.AF_INET + STREAM = socket.SOCK_STREAM + TCP = socket.IPPROTO_TCP + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', None, INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', b'', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 0)), + base_events._ipaddr_info('1.2.3.4', '', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', '1', INET, STREAM, TCP)) + + self.assertEqual( + (INET, STREAM, TCP, '', ('1.2.3.4', 1)), + base_events._ipaddr_info('1.2.3.4', b'1', INET, STREAM, TCP)) + + @patch_socket + def test_ipaddr_info_no_inet_pton(self, m_socket): + del m_socket.inet_pton + self.assertIsNone(base_events._ipaddr_info('1.2.3.4', 1, + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP)) + + +class BaseEventLoopTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = base_events.BaseEventLoop() + self.loop._selector = mock.Mock() + self.loop._selector.select.return_value = () + self.set_event_loop(self.loop) + + def test_not_implemented(self): + m = mock.Mock() + self.assertRaises( + NotImplementedError, + self.loop._make_socket_transport, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_ssl_transport, m, m, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_datagram_transport, m, m) + self.assertRaises( + NotImplementedError, self.loop._process_events, []) + self.assertRaises( + NotImplementedError, self.loop._write_to_self) + self.assertRaises( + NotImplementedError, + self.loop._make_read_pipe_transport, m, m) + self.assertRaises( + NotImplementedError, + self.loop._make_write_pipe_transport, m, m) + gen = self.loop._make_subprocess_transport(m, m, m, m, m, m, m) + with self.assertRaises(NotImplementedError): + gen.send(None) + + def test_close(self): + self.assertFalse(self.loop.is_closed()) + self.loop.close() + self.assertTrue(self.loop.is_closed()) + + # it should be possible to call close() more than once + self.loop.close() + self.loop.close() + + # operation blocked when the loop is closed + f = self.loop.create_future() + self.assertRaises(RuntimeError, self.loop.run_forever) + self.assertRaises(RuntimeError, self.loop.run_until_complete, f) + + def test__add_callback_handle(self): + h = asyncio.Handle(lambda: False, (), self.loop, None) + + self.loop._add_callback(h) + self.assertFalse(self.loop._scheduled) + self.assertIn(h, self.loop._ready) + + def test__add_callback_cancelled_handle(self): + h = asyncio.Handle(lambda: False, (), self.loop, None) + h.cancel() + + self.loop._add_callback(h) + self.assertFalse(self.loop._scheduled) + self.assertFalse(self.loop._ready) + + def test_set_default_executor(self): + class DummyExecutor(concurrent.futures.ThreadPoolExecutor): + def submit(self, fn, *args, **kwargs): + raise NotImplementedError( + 'cannot submit into a dummy executor') + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + + executor = DummyExecutor() + self.loop.set_default_executor(executor) + self.assertIs(executor, self.loop._default_executor) + + def test_set_default_executor_error(self): + executor = mock.Mock() + + msg = 'executor must be ThreadPoolExecutor instance' + with self.assertRaisesRegex(TypeError, msg): + self.loop.set_default_executor(executor) + + self.assertIsNone(self.loop._default_executor) + + def test_shutdown_default_executor_timeout(self): + event = threading.Event() + + class DummyExecutor(concurrent.futures.ThreadPoolExecutor): + def shutdown(self, wait=True, *, cancel_futures=False): + if wait: + event.wait() + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + executor = DummyExecutor() + self.loop.set_default_executor(executor) + + try: + with self.assertWarnsRegex(RuntimeWarning, + "The executor did not finishing joining"): + self.loop.run_until_complete( + self.loop.shutdown_default_executor(timeout=0.01)) + finally: + event.set() + + def test_call_soon(self): + def cb(): + pass + + h = self.loop.call_soon(cb) + self.assertEqual(h._callback, cb) + self.assertIsInstance(h, asyncio.Handle) + self.assertIn(h, self.loop._ready) + + def test_call_soon_non_callable(self): + self.loop.set_debug(True) + with self.assertRaisesRegex(TypeError, 'a callable object'): + self.loop.call_soon(1) + + def test_call_later(self): + def cb(): + pass + + h = self.loop.call_later(10.0, cb) + self.assertIsInstance(h, asyncio.TimerHandle) + self.assertIn(h, self.loop._scheduled) + self.assertNotIn(h, self.loop._ready) + with self.assertRaises(TypeError, msg="delay must not be None"): + self.loop.call_later(None, cb) + + def test_call_later_negative_delays(self): + calls = [] + + def cb(arg): + calls.append(arg) + + self.loop._process_events = mock.Mock() + self.loop.call_later(-1, cb, 'a') + self.loop.call_later(-2, cb, 'b') + test_utils.run_briefly(self.loop) + self.assertEqual(calls, ['b', 'a']) + + def test_time_and_call_at(self): + def cb(): + self.loop.stop() + + self.loop._process_events = mock.Mock() + delay = 0.100 + + when = self.loop.time() + delay + self.loop.call_at(when, cb) + t0 = self.loop.time() + self.loop.run_forever() + dt = self.loop.time() - t0 + + # 50 ms: maximum granularity of the event loop + self.assertGreaterEqual(dt, delay - test_utils.CLOCK_RES) + with self.assertRaises(TypeError, msg="when cannot be None"): + self.loop.call_at(None, cb) + + def check_thread(self, loop, debug): + def cb(): + pass + + loop.set_debug(debug) + if debug: + msg = ("Non-thread-safe operation invoked on an event loop other " + "than the current one") + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_soon(cb) + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_later(60, cb) + with self.assertRaisesRegex(RuntimeError, msg): + loop.call_at(loop.time() + 60, cb) + else: + loop.call_soon(cb) + loop.call_later(60, cb) + loop.call_at(loop.time() + 60, cb) + + def test_check_thread(self): + def check_in_thread(loop, event, debug, create_loop, fut): + # wait until the event loop is running + event.wait() + + try: + if create_loop: + loop2 = base_events.BaseEventLoop() + try: + asyncio.set_event_loop(loop2) + self.check_thread(loop, debug) + finally: + asyncio.set_event_loop(None) + loop2.close() + else: + self.check_thread(loop, debug) + except Exception as exc: + loop.call_soon_threadsafe(fut.set_exception, exc) + else: + loop.call_soon_threadsafe(fut.set_result, None) + + def test_thread(loop, debug, create_loop=False): + event = threading.Event() + fut = loop.create_future() + loop.call_soon(event.set) + args = (loop, event, debug, create_loop, fut) + thread = threading.Thread(target=check_in_thread, args=args) + thread.start() + loop.run_until_complete(fut) + thread.join() + + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + + # raise RuntimeError if the thread has no event loop + test_thread(self.loop, True) + + # check disabled if debug mode is disabled + test_thread(self.loop, False) + + # raise RuntimeError if the event loop of the thread is not the called + # event loop + test_thread(self.loop, True, create_loop=True) + + # check disabled if debug mode is disabled + test_thread(self.loop, False, create_loop=True) + + def test__run_once(self): + h1 = asyncio.TimerHandle(time.monotonic() + 5.0, lambda: True, (), + self.loop, None) + h2 = asyncio.TimerHandle(time.monotonic() + 10.0, lambda: True, (), + self.loop, None) + + h1.cancel() + + self.loop._process_events = mock.Mock() + self.loop._scheduled.append(h1) + self.loop._scheduled.append(h2) + self.loop._run_once() + + t = self.loop._selector.select.call_args[0][0] + self.assertTrue(9.5 < t < 10.5, t) + self.assertEqual([h2], self.loop._scheduled) + self.assertTrue(self.loop._process_events.called) + + def test_set_debug(self): + self.loop.set_debug(True) + self.assertTrue(self.loop.get_debug()) + self.loop.set_debug(False) + self.assertFalse(self.loop.get_debug()) + + def test__run_once_schedule_handle(self): + handle = None + processed = False + + def cb(loop): + nonlocal processed, handle + processed = True + handle = loop.call_soon(lambda: True) + + h = asyncio.TimerHandle(time.monotonic() - 1, cb, (self.loop,), + self.loop, None) + + self.loop._process_events = mock.Mock() + self.loop._scheduled.append(h) + self.loop._run_once() + + self.assertTrue(processed) + self.assertEqual([handle], list(self.loop._ready)) + + def test__run_once_cancelled_event_cleanup(self): + self.loop._process_events = mock.Mock() + + self.assertTrue( + 0 < base_events._MIN_CANCELLED_TIMER_HANDLES_FRACTION < 1.0) + + def cb(): + pass + + # Set up one "blocking" event that will not be cancelled to + # ensure later cancelled events do not make it to the head + # of the queue and get cleaned. + not_cancelled_count = 1 + self.loop.call_later(3000, cb) + + # Add less than threshold (base_events._MIN_SCHEDULED_TIMER_HANDLES) + # cancelled handles, ensure they aren't removed + + cancelled_count = 2 + for x in range(2): + h = self.loop.call_later(3600, cb) + h.cancel() + + # Add some cancelled events that will be at head and removed + cancelled_count += 2 + for x in range(2): + h = self.loop.call_later(100, cb) + h.cancel() + + # This test is invalid if _MIN_SCHEDULED_TIMER_HANDLES is too low + self.assertLessEqual(cancelled_count + not_cancelled_count, + base_events._MIN_SCHEDULED_TIMER_HANDLES) + + self.assertEqual(self.loop._timer_cancelled_count, cancelled_count) + + self.loop._run_once() + + cancelled_count -= 2 + + self.assertEqual(self.loop._timer_cancelled_count, cancelled_count) + + self.assertEqual(len(self.loop._scheduled), + cancelled_count + not_cancelled_count) + + # Need enough events to pass _MIN_CANCELLED_TIMER_HANDLES_FRACTION + # so that deletion of cancelled events will occur on next _run_once + add_cancel_count = int(math.ceil( + base_events._MIN_SCHEDULED_TIMER_HANDLES * + base_events._MIN_CANCELLED_TIMER_HANDLES_FRACTION)) + 1 + + add_not_cancel_count = max(base_events._MIN_SCHEDULED_TIMER_HANDLES - + add_cancel_count, 0) + + # Add some events that will not be cancelled + not_cancelled_count += add_not_cancel_count + for x in range(add_not_cancel_count): + self.loop.call_later(3600, cb) + + # Add enough cancelled events + cancelled_count += add_cancel_count + for x in range(add_cancel_count): + h = self.loop.call_later(3600, cb) + h.cancel() + + # Ensure all handles are still scheduled + self.assertEqual(len(self.loop._scheduled), + cancelled_count + not_cancelled_count) + + self.loop._run_once() + + # Ensure cancelled events were removed + self.assertEqual(len(self.loop._scheduled), not_cancelled_count) + + # Ensure only uncancelled events remain scheduled + self.assertTrue(all([not x._cancelled for x in self.loop._scheduled])) + + def test_run_until_complete_type_error(self): + self.assertRaises(TypeError, + self.loop.run_until_complete, 'blah') + + def test_run_until_complete_loop(self): + task = self.loop.create_future() + other_loop = self.new_test_loop() + self.addCleanup(other_loop.close) + self.assertRaises(ValueError, + other_loop.run_until_complete, task) + + def test_run_until_complete_loop_orphan_future_close_loop(self): + class ShowStopper(SystemExit): + pass + + async def foo(delay): + await asyncio.sleep(delay) + + def throw(): + raise ShowStopper + + self.loop._process_events = mock.Mock() + self.loop.call_soon(throw) + with self.assertRaises(ShowStopper): + self.loop.run_until_complete(foo(0.1)) + + # This call fails if run_until_complete does not clean up + # done-callback for the previous future. + self.loop.run_until_complete(foo(0.2)) + + def test_subprocess_exec_invalid_args(self): + args = [sys.executable, '-c', 'pass'] + + # missing program parameter (empty args) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol) + + # expected multiple arguments, not a list + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, args) + + # program arguments must be strings, not int + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, sys.executable, 123) + + # universal_newlines, shell, bufsize must not be set + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, universal_newlines=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, shell=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_exec, + asyncio.SubprocessProtocol, *args, bufsize=4096) + + def test_subprocess_shell_invalid_args(self): + # expected a string, not an int or a list + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 123) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, [sys.executable, '-c', 'pass']) + + # universal_newlines, shell, bufsize must not be set + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', universal_newlines=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', shell=True) + self.assertRaises(TypeError, + self.loop.run_until_complete, self.loop.subprocess_shell, + asyncio.SubprocessProtocol, 'exit 0', bufsize=4096) + + def test_default_exc_handler_callback(self): + self.loop._process_events = mock.Mock() + + def zero_error(fut): + fut.set_result(True) + 1/0 + + # Test call_soon (events.Handle) + with mock.patch('asyncio.base_events.logger') as log: + fut = self.loop.create_future() + self.loop.call_soon(zero_error, fut) + fut.add_done_callback(lambda fut: self.loop.stop()) + self.loop.run_forever() + log.error.assert_called_with( + test_utils.MockPattern('Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + # Test call_later (events.TimerHandle) + with mock.patch('asyncio.base_events.logger') as log: + fut = self.loop.create_future() + self.loop.call_later(0.01, zero_error, fut) + fut.add_done_callback(lambda fut: self.loop.stop()) + self.loop.run_forever() + log.error.assert_called_with( + test_utils.MockPattern('Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + def test_default_exc_handler_coro(self): + self.loop._process_events = mock.Mock() + + async def zero_error_coro(): + await asyncio.sleep(0.01) + 1/0 + + # Test Future.__del__ + with mock.patch('asyncio.base_events.logger') as log: + fut = asyncio.ensure_future(zero_error_coro(), loop=self.loop) + fut.add_done_callback(lambda *args: self.loop.stop()) + self.loop.run_forever() + fut = None # Trigger Future.__del__ or futures._TracebackLogger + support.gc_collect() + # Future.__del__ in logs error with an actual exception context + log.error.assert_called_with( + test_utils.MockPattern('.*exception was never retrieved'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + def test_set_exc_handler_invalid(self): + with self.assertRaisesRegex(TypeError, 'A callable object or None'): + self.loop.set_exception_handler('spam') + + def test_set_exc_handler_custom(self): + def zero_error(): + 1/0 + + def run_loop(): + handle = self.loop.call_soon(zero_error) + self.loop._run_once() + return handle + + self.loop.set_debug(True) + self.loop._process_events = mock.Mock() + + self.assertIsNone(self.loop.get_exception_handler()) + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + self.assertIs(self.loop.get_exception_handler(), mock_handler) + handle = run_loop() + mock_handler.assert_called_with(self.loop, { + 'exception': MOCK_ANY, + 'message': test_utils.MockPattern( + 'Exception in callback.*zero_error'), + 'handle': handle, + 'source_traceback': handle._source_traceback, + }) + mock_handler.reset_mock() + + self.loop.set_exception_handler(None) + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern( + 'Exception in callback.*zero'), + exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY)) + + self.assertFalse(mock_handler.called) + + def test_set_exc_handler_broken(self): + def run_loop(): + def zero_error(): + 1/0 + self.loop.call_soon(zero_error) + self.loop._run_once() + + def handler(loop, context): + raise AttributeError('spam') + + self.loop._process_events = mock.Mock() + + self.loop.set_exception_handler(handler) + + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern( + 'Unhandled error in exception handler'), + exc_info=(AttributeError, MOCK_ANY, MOCK_ANY)) + + def test_default_exc_handler_broken(self): + _context = None + + class Loop(base_events.BaseEventLoop): + + _selector = mock.Mock() + _process_events = mock.Mock() + + def default_exception_handler(self, context): + nonlocal _context + _context = context + # Simulates custom buggy "default_exception_handler" + raise ValueError('spam') + + loop = Loop() + self.addCleanup(loop.close) + asyncio.set_event_loop(loop) + + def run_loop(): + def zero_error(): + 1/0 + loop.call_soon(zero_error) + loop._run_once() + + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + 'Exception in default exception handler', + exc_info=True) + + def custom_handler(loop, context): + raise ValueError('ham') + + _context = None + loop.set_exception_handler(custom_handler) + with mock.patch('asyncio.base_events.logger') as log: + run_loop() + log.error.assert_called_with( + test_utils.MockPattern('Exception in default exception.*' + 'while handling.*in custom'), + exc_info=True) + + # Check that original context was passed to default + # exception handler. + self.assertIn('context', _context) + self.assertIs(type(_context['context']['exception']), + ZeroDivisionError) + + def test_set_task_factory_invalid(self): + with self.assertRaisesRegex( + TypeError, 'task factory must be a callable or None'): + + self.loop.set_task_factory(1) + + self.assertIsNone(self.loop.get_task_factory()) + + def test_set_task_factory(self): + self.loop._process_events = mock.Mock() + + class MyTask(asyncio.Task): + pass + + async def coro(): + pass + + factory = lambda loop, coro: MyTask(coro, loop=loop) + + self.assertIsNone(self.loop.get_task_factory()) + self.loop.set_task_factory(factory) + self.assertIs(self.loop.get_task_factory(), factory) + + task = self.loop.create_task(coro()) + self.assertTrue(isinstance(task, MyTask)) + self.loop.run_until_complete(task) + + self.loop.set_task_factory(None) + self.assertIsNone(self.loop.get_task_factory()) + + task = self.loop.create_task(coro()) + self.assertTrue(isinstance(task, asyncio.Task)) + self.assertFalse(isinstance(task, MyTask)) + self.loop.run_until_complete(task) + + def test_env_var_debug(self): + code = '\n'.join(( + 'import asyncio', + 'loop = asyncio.new_event_loop()', + 'print(loop.get_debug())')) + + # Test with -E to not fail if the unit test was run with + # PYTHONASYNCIODEBUG set to a non-empty string + sts, stdout, stderr = assert_python_ok('-E', '-c', code) + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='1', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'True') + + sts, stdout, stderr = assert_python_ok('-E', '-c', code, + PYTHONASYNCIODEBUG='1') + self.assertEqual(stdout.rstrip(), b'False') + + # -X dev + sts, stdout, stderr = assert_python_ok('-E', '-X', 'dev', + '-c', code) + self.assertEqual(stdout.rstrip(), b'True') + + def test_create_task(self): + class MyTask(asyncio.Task): + pass + + async def test(): + pass + + class EventLoop(base_events.BaseEventLoop): + def create_task(self, coro): + return MyTask(coro, loop=loop) + + loop = EventLoop() + self.set_event_loop(loop) + + coro = test() + task = asyncio.ensure_future(coro, loop=loop) + self.assertIsInstance(task, MyTask) + + # make warnings quiet + task._log_destroy_pending = False + coro.close() + + def test_create_task_error_closes_coro(self): + async def test(): + pass + loop = asyncio.new_event_loop() + loop.close() + with warnings.catch_warnings(record=True) as w: + with self.assertRaises(RuntimeError): + asyncio.ensure_future(test(), loop=loop) + self.assertEqual(len(w), 0) + + + def test_create_named_task_with_default_factory(self): + async def test(): + pass + + loop = asyncio.new_event_loop() + task = loop.create_task(test(), name='test_task') + try: + self.assertEqual(task.get_name(), 'test_task') + finally: + loop.run_until_complete(task) + loop.close() + + def test_create_named_task_with_custom_factory(self): + def task_factory(loop, coro, **kwargs): + return asyncio.Task(coro, loop=loop, **kwargs) + + async def test(): + pass + + loop = asyncio.new_event_loop() + loop.set_task_factory(task_factory) + task = loop.create_task(test(), name='test_task') + try: + self.assertEqual(task.get_name(), 'test_task') + finally: + loop.run_until_complete(task) + loop.close() + + def test_run_forever_keyboard_interrupt(self): + # Python issue #22601: ensure that the temporary task created by + # run_forever() consumes the KeyboardInterrupt and so don't log + # a warning + async def raise_keyboard_interrupt(): + raise KeyboardInterrupt + + self.loop._process_events = mock.Mock() + self.loop.call_exception_handler = mock.Mock() + + try: + self.loop.run_until_complete(raise_keyboard_interrupt()) + except KeyboardInterrupt: + pass + self.loop.close() + support.gc_collect() + + self.assertFalse(self.loop.call_exception_handler.called) + + def test_run_until_complete_baseexception(self): + # Python issue #22429: run_until_complete() must not schedule a pending + # call to stop() if the future raised a BaseException + async def raise_keyboard_interrupt(): + raise KeyboardInterrupt + + self.loop._process_events = mock.Mock() + + with self.assertRaises(KeyboardInterrupt): + self.loop.run_until_complete(raise_keyboard_interrupt()) + + def func(): + self.loop.stop() + func.called = True + func.called = False + self.loop.call_soon(self.loop.call_soon, func) + self.loop.run_forever() + self.assertTrue(func.called) + + def test_single_selecter_event_callback_after_stopping(self): + # Python issue #25593: A stopped event loop may cause event callbacks + # to run more than once. + event_sentinel = object() + callcount = 0 + doer = None + + def proc_events(event_list): + nonlocal doer + if event_sentinel in event_list: + doer = self.loop.call_soon(do_event) + + def do_event(): + nonlocal callcount + callcount += 1 + self.loop.call_soon(clear_selector) + + def clear_selector(): + doer.cancel() + self.loop._selector.select.return_value = () + + self.loop._process_events = proc_events + self.loop._selector.select.return_value = (event_sentinel,) + + for i in range(1, 3): + with self.subTest('Loop %d/2' % i): + self.loop.call_soon(self.loop.stop) + self.loop.run_forever() + self.assertEqual(callcount, 1) + + def test_run_once(self): + # Simple test for test_utils.run_once(). It may seem strange + # to have a test for this (the function isn't even used!) but + # it's a de-factor standard API for library tests. This tests + # the idiom: loop.call_soon(loop.stop); loop.run_forever(). + count = 0 + + def callback(): + nonlocal count + count += 1 + + self.loop._process_events = mock.Mock() + self.loop.call_soon(callback) + test_utils.run_once(self.loop) + self.assertEqual(count, 1) + + def test_run_forever_pre_stopped(self): + # Test that the old idiom for pre-stopping the loop works. + self.loop._process_events = mock.Mock() + self.loop.stop() + self.loop.run_forever() + self.loop._selector.select.assert_called_once_with(0) + + def test_custom_run_forever_integration(self): + # Test that the run_forever_setup() and run_forever_cleanup() primitives + # can be used to implement a custom run_forever loop. + self.loop._process_events = mock.Mock() + + count = 0 + + def callback(): + nonlocal count + count += 1 + + self.loop.call_soon(callback) + + # Set up the custom event loop + self.loop._run_forever_setup() + + # Confirm the loop has been started + self.assertEqual(asyncio.get_running_loop(), self.loop) + self.assertTrue(self.loop.is_running()) + + # Our custom "event loop" just iterates 10 times before exiting. + for i in range(10): + self.loop._run_once() + + # Clean up the event loop + self.loop._run_forever_cleanup() + + # Confirm the loop has been cleaned up + with self.assertRaises(RuntimeError): + asyncio.get_running_loop() + self.assertFalse(self.loop.is_running()) + + # Confirm the loop actually did run, processing events 10 times, + # and invoking the callback once. + self.assertEqual(self.loop._process_events.call_count, 10) + self.assertEqual(count, 1) + + async def leave_unfinalized_asyncgen(self): + # Create an async generator, iterate it partially, and leave it + # to be garbage collected. + # Used in async generator finalization tests. + # Depends on implementation details of garbage collector. Changes + # in gc may break this function. + status = {'started': False, + 'stopped': False, + 'finalized': False} + + async def agen(): + status['started'] = True + try: + for item in ['ZERO', 'ONE', 'TWO', 'THREE', 'FOUR']: + yield item + finally: + status['finalized'] = True + + ag = agen() + ai = ag.__aiter__() + + async def iter_one(): + try: + item = await ai.__anext__() + except StopAsyncIteration: + return + if item == 'THREE': + status['stopped'] = True + return + asyncio.create_task(iter_one()) + + asyncio.create_task(iter_one()) + return status + + @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators + def test_asyncgen_finalization_by_gc(self): + # Async generators should be finalized when garbage collected. + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + with support.disable_gc(): + status = self.loop.run_until_complete(self.leave_unfinalized_asyncgen()) + while not status['stopped']: + test_utils.run_briefly(self.loop) + self.assertTrue(status['started']) + self.assertTrue(status['stopped']) + self.assertFalse(status['finalized']) + support.gc_collect() + test_utils.run_briefly(self.loop) + self.assertTrue(status['finalized']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - GC doesn't finalize async generators + def test_asyncgen_finalization_by_gc_in_other_thread(self): + # Python issue 34769: If garbage collector runs in another + # thread, async generators will not finalize in debug + # mode. + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + self.loop.set_debug(True) + with support.disable_gc(): + status = self.loop.run_until_complete(self.leave_unfinalized_asyncgen()) + while not status['stopped']: + test_utils.run_briefly(self.loop) + self.assertTrue(status['started']) + self.assertTrue(status['stopped']) + self.assertFalse(status['finalized']) + self.loop.run_until_complete( + self.loop.run_in_executor(None, support.gc_collect)) + test_utils.run_briefly(self.loop) + self.assertTrue(status['finalized']) + + +class MyProto(asyncio.Protocol): + done = None + + def __init__(self, create_future=False): + self.state = 'INITIAL' + self.nbytes = 0 + if create_future: + self.done = asyncio.get_running_loop().create_future() + + def _assert_state(self, *expected): + if self.state not in expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n') + + def data_received(self, data): + self._assert_state('CONNECTED') + self.nbytes += len(data) + + def eof_received(self): + self._assert_state('CONNECTED') + self.state = 'EOF' + + def connection_lost(self, exc): + self._assert_state('CONNECTED', 'EOF') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyDatagramProto(asyncio.DatagramProtocol): + done = None + + def __init__(self, create_future=False, loop=None): + self.state = 'INITIAL' + self.nbytes = 0 + if create_future: + self.done = loop.create_future() + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'INITIALIZED' + + def datagram_received(self, data, addr): + self._assert_state('INITIALIZED') + self.nbytes += len(data) + + def error_received(self, exc): + self._assert_state('INITIALIZED') + + def connection_lost(self, exc): + self._assert_state('INITIALIZED') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class BaseEventLoopWithSelectorTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.SelectorEventLoop() + self.set_event_loop(self.loop) + + @mock.patch('socket.getnameinfo') + def test_getnameinfo(self, m_gai): + m_gai.side_effect = lambda *args: 42 + r = self.loop.run_until_complete(self.loop.getnameinfo(('abc', 123))) + self.assertEqual(r, 42) + + @patch_socket + def test_create_connection_multiple_errors(self, m_socket): + + class MyProto(asyncio.Protocol): + pass + + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('107.6.106.82', 80)), + (2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + idx = -1 + errors = ['err1', 'err2'] + + def _socket(*args, **kw): + nonlocal idx, errors + idx += 1 + raise OSError(errors[idx]) + + m_socket.socket = _socket + + self.loop.getaddrinfo = getaddrinfo_task + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(coro) + + self.assertEqual(str(cm.exception), 'Multiple exceptions: err1, err2') + + idx = -1 + coro = self.loop.create_connection(MyProto, 'example.com', 80, all_errors=True) + with self.assertRaises(ExceptionGroup) as cm: + self.loop.run_until_complete(coro) + + self.assertIsInstance(cm.exception, ExceptionGroup) + for e in cm.exception.exceptions: + self.assertIsInstance(e, OSError) + + @patch_socket + def test_create_connection_timeout(self, m_socket): + # Ensure that the socket is closed on timeout + sock = mock.Mock() + m_socket.socket.return_value = sock + + def getaddrinfo(*args, **kw): + fut = self.loop.create_future() + addr = (socket.AF_INET, socket.SOCK_STREAM, 0, '', + ('127.0.0.1', 80)) + fut.set_result([addr]) + return fut + self.loop.getaddrinfo = getaddrinfo + + with mock.patch.object(self.loop, 'sock_connect', + side_effect=asyncio.TimeoutError): + coro = self.loop.create_connection(MyProto, '127.0.0.1', 80) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + @patch_socket + def test_create_connection_happy_eyeballs_empty_exceptions(self, m_socket): + # See gh-135836: Fix IndexError when Happy Eyeballs algorithm + # results in empty exceptions list + + async def getaddrinfo(*args, **kw): + return [(socket.AF_INET, socket.SOCK_STREAM, 0, '', ('127.0.0.1', 80)), + (socket.AF_INET6, socket.SOCK_STREAM, 0, '', ('::1', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + + # Mock staggered_race to return empty exceptions list + # This simulates the scenario where Happy Eyeballs algorithm + # cancels all attempts but doesn't properly collect exceptions + with mock.patch('asyncio.staggered.staggered_race') as mock_staggered: + # Return (None, []) - no winner, empty exceptions list + async def mock_race(coro_fns, delay, loop): + return None, [] + mock_staggered.side_effect = mock_race + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, happy_eyeballs_delay=0.1) + + # Should raise TimeoutError instead of IndexError + with self.assertRaisesRegex(TimeoutError, "create_connection failed"): + self.loop.run_until_complete(coro) + + def test_create_connection_host_port_sock(self): + coro = self.loop.create_connection( + MyProto, 'example.com', 80, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_wrong_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + with sock: + coro = self.loop.create_connection(MyProto, sock=sock) + with self.assertRaisesRegex(ValueError, + 'A Stream Socket was expected'): + self.loop.run_until_complete(coro) + + def test_create_server_wrong_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + with sock: + coro = self.loop.create_server(MyProto, sock=sock) + with self.assertRaisesRegex(ValueError, + 'A Stream Socket was expected'): + self.loop.run_until_complete(coro) + + def test_create_server_ssl_timeout_for_plain_socket(self): + coro = self.loop.create_server( + MyProto, 'example.com', 80, ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'no socket.SOCK_NONBLOCK (linux only)') + def test_create_server_stream_bittype(self): + sock = socket.socket( + socket.AF_INET, socket.SOCK_STREAM | socket.SOCK_NONBLOCK) + with sock: + coro = self.loop.create_server(lambda: None, sock=sock) + srv = self.loop.run_until_complete(coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'no IPv6 support') + def test_create_server_ipv6(self): + async def main(): + srv = await asyncio.start_server(lambda: None, '::1', 0) + try: + self.assertGreater(len(srv.sockets), 0) + finally: + srv.close() + await srv.wait_closed() + + try: + self.loop.run_until_complete(main()) + except OSError as ex: + if (hasattr(errno, 'EADDRNOTAVAIL') and + ex.errno == errno.EADDRNOTAVAIL): + self.skipTest('failed to bind to ::1') + else: + raise + + def test_create_datagram_endpoint_wrong_sock(self): + sock = socket.socket(socket.AF_INET) + with sock: + coro = self.loop.create_datagram_endpoint(MyProto, sock=sock) + with self.assertRaisesRegex(ValueError, + 'A datagram socket was expected'): + self.loop.run_until_complete(coro) + + def test_create_connection_no_host_port_sock(self): + coro = self.loop.create_connection(MyProto) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_no_getaddrinfo(self): + async def getaddrinfo(*args, **kw): + return [] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_connection_connect_err(self): + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + coro = self.loop.create_connection(MyProto, 'example.com', 80, all_errors=True) + with self.assertRaises(ExceptionGroup) as cm: + self.loop.run_until_complete(coro) + + self.assertIsInstance(cm.exception, ExceptionGroup) + self.assertEqual(len(cm.exception.exceptions), 1) + self.assertIsInstance(cm.exception.exceptions[0], OSError) + + @patch_socket + def test_create_connection_connect_non_os_err_close_err(self, m_socket): + # Test the case when sock_connect() raises non-OSError exception + # and sock.close() raises OSError. + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('107.6.106.82', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = CustomError + sock = mock.Mock() + m_socket.socket.return_value = sock + sock.close.side_effect = OSError + + coro = self.loop.create_connection(MyProto, 'example.com', 80) + self.assertRaises( + CustomError, self.loop.run_until_complete, coro) + + coro = self.loop.create_connection(MyProto, 'example.com', 80, all_errors=True) + self.assertRaises( + CustomError, self.loop.run_until_complete, coro) + + def test_create_connection_multiple(self): + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('0.0.0.1', 80)), + (2, 1, 6, '', ('0.0.0.2', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET) + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, all_errors=True) + with self.assertRaises(ExceptionGroup) as cm: + self.loop.run_until_complete(coro) + + self.assertIsInstance(cm.exception, ExceptionGroup) + for e in cm.exception.exceptions: + self.assertIsInstance(e, OSError) + + @patch_socket + def test_create_connection_multiple_errors_local_addr(self, m_socket): + + def bind(addr): + if addr[0] == '0.0.0.1': + err = OSError('Err') + err.strerror = 'Err' + raise err + + m_socket.socket.return_value.bind = bind + + async def getaddrinfo(*args, **kw): + return [(2, 1, 6, '', ('0.0.0.1', 80)), + (2, 1, 6, '', ('0.0.0.2', 80))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError('Err2') + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, + local_addr=(None, 8080)) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(coro) + + self.assertStartsWith(str(cm.exception), 'Multiple exceptions: ') + self.assertTrue(m_socket.socket.return_value.close.called) + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, + local_addr=(None, 8080), all_errors=True) + with self.assertRaises(ExceptionGroup) as cm: + self.loop.run_until_complete(coro) + + self.assertIsInstance(cm.exception, ExceptionGroup) + for e in cm.exception.exceptions: + self.assertIsInstance(e, OSError) + + def _test_create_connection_ip_addr(self, m_socket, allow_inet_pton): + # Test the fallback code, even if this system has inet_pton. + if not allow_inet_pton: + del m_socket.inet_pton + + m_socket.getaddrinfo = socket.getaddrinfo + sock = m_socket.socket.return_value + + self.loop._add_reader = mock.Mock() + self.loop._add_writer = mock.Mock() + + coro = self.loop.create_connection(asyncio.Protocol, '1.2.3.4', 80) + t, p = self.loop.run_until_complete(coro) + try: + sock.connect.assert_called_with(('1.2.3.4', 80)) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + if socket_helper.IPV6_ENABLED: + sock.family = socket.AF_INET6 + coro = self.loop.create_connection(asyncio.Protocol, '::1', 80) + t, p = self.loop.run_until_complete(coro) + try: + # Without inet_pton we use getaddrinfo, which transforms + # ('::1', 80) to ('::1', 80, 0, 0). The last 0s are flow info, + # scope id. + [address] = sock.connect.call_args[0] + host, port = address[:2] + self.assertRegex(host, r'::(0\.)*1') + self.assertEqual(port, 80) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET6) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'no IPv6 support') + @unittest.skipIf(sys.platform.startswith('aix'), + "bpo-25545: IPv6 scope id and getaddrinfo() behave differently on AIX") + @patch_socket + def test_create_connection_ipv6_scope(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + sock = m_socket.socket.return_value + sock.family = socket.AF_INET6 + + self.loop._add_reader = mock.Mock() + self.loop._add_writer = mock.Mock() + + coro = self.loop.create_connection(asyncio.Protocol, 'fe80::1%1', 80) + t, p = self.loop.run_until_complete(coro) + try: + sock.connect.assert_called_with(('fe80::1', 80, 0, 1)) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET6) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + @patch_socket + def test_create_connection_ip_addr(self, m_socket): + self._test_create_connection_ip_addr(m_socket, True) + + @patch_socket + def test_create_connection_no_inet_pton(self, m_socket): + self._test_create_connection_ip_addr(m_socket, False) + + @patch_socket + @unittest.skipIf( + support.is_android and platform.android_ver().api_level < 23, + "Issue gh-71123: this fails on Android before API level 23" + ) + def test_create_connection_service_name(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + sock = m_socket.socket.return_value + + self.loop._add_reader = mock.Mock() + self.loop._add_writer = mock.Mock() + + for service, port in ('http', 80), (b'http', 80): + coro = self.loop.create_connection(asyncio.Protocol, + '127.0.0.1', service) + + t, p = self.loop.run_until_complete(coro) + try: + sock.connect.assert_called_with(('127.0.0.1', port)) + _, kwargs = m_socket.socket.call_args + self.assertEqual(kwargs['family'], m_socket.AF_INET) + self.assertEqual(kwargs['type'], m_socket.SOCK_STREAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + for service in 'nonsense', b'nonsense': + coro = self.loop.create_connection(asyncio.Protocol, + '127.0.0.1', service) + + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + + def test_create_connection_no_local_addr(self): + async def getaddrinfo(host, *args, **kw): + if host == 'example.com': + return [(2, 1, 6, '', ('107.6.106.82', 80)), + (2, 1, 6, '', ('107.6.106.82', 80))] + else: + return [] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + self.loop.getaddrinfo = getaddrinfo_task + + coro = self.loop.create_connection( + MyProto, 'example.com', 80, family=socket.AF_INET, + local_addr=(None, 8080)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_connection_bluetooth(self, m_socket): + # See http://bugs.python.org/issue27136, fallback to getaddrinfo when + # we can't recognize an address is resolved, e.g. a Bluetooth address. + addr = ('00:01:02:03:04:05', 1) + + def getaddrinfo(host, port, *args, **kw): + self.assertEqual((host, port), addr) + return [(999, 1, 999, '', (addr, 1))] + + m_socket.getaddrinfo = getaddrinfo + sock = m_socket.socket() + coro = self.loop.sock_connect(sock, addr) + self.loop.run_until_complete(coro) + + def test_create_connection_ssl_server_hostname_default(self): + self.loop.getaddrinfo = mock.Mock() + + def mock_getaddrinfo(*args, **kwds): + f = self.loop.create_future() + f.set_result([(socket.AF_INET, socket.SOCK_STREAM, + socket.SOL_TCP, '', ('1.2.3.4', 80))]) + return f + + self.loop.getaddrinfo.side_effect = mock_getaddrinfo + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.return_value = self.loop.create_future() + self.loop.sock_connect.return_value.set_result(None) + self.loop._make_ssl_transport = mock.Mock() + + class _SelectorTransportMock: + _sock = None + + def get_extra_info(self, key): + return mock.Mock() + + def close(self): + self._sock.close() + + def mock_make_ssl_transport(sock, protocol, sslcontext, waiter, + **kwds): + waiter.set_result(None) + transport = _SelectorTransportMock() + transport._sock = sock + return transport + + self.loop._make_ssl_transport.side_effect = mock_make_ssl_transport + ANY = mock.ANY + handshake_timeout = object() + shutdown_timeout = object() + # First try the default server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='python.org', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + # Next try an explicit server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + server_hostname='perl.com', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='perl.com', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + # Finally try an explicit empty server_hostname. + self.loop._make_ssl_transport.reset_mock() + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + server_hostname='', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + transport, _ = self.loop.run_until_complete(coro) + transport.close() + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='', + ssl_handshake_timeout=handshake_timeout, + ssl_shutdown_timeout=shutdown_timeout) + + def test_create_connection_no_ssl_server_hostname_errors(self): + # When not using ssl, server_hostname must be None. + coro = self.loop.create_connection(MyProto, 'python.org', 80, + server_hostname='') + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + coro = self.loop.create_connection(MyProto, 'python.org', 80, + server_hostname='python.org') + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_ssl_server_hostname_errors(self): + # When using ssl, server_hostname may be None if host is non-empty. + coro = self.loop.create_connection(MyProto, '', 80, ssl=True) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + coro = self.loop.create_connection(MyProto, None, 80, ssl=True) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + sock = socket.socket() + coro = self.loop.create_connection(MyProto, None, None, + ssl=True, sock=sock) + self.addCleanup(sock.close) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + def test_create_connection_ssl_timeout_for_plain_socket(self): + coro = self.loop.create_connection( + MyProto, 'example.com', 80, ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + def test_create_server_empty_host(self): + # if host is empty string use None instead + host = object() + + async def getaddrinfo(*args, **kw): + nonlocal host + host = args[0] + return [] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + self.loop.getaddrinfo = getaddrinfo_task + fut = self.loop.create_server(MyProto, '', 0) + self.assertRaises(OSError, self.loop.run_until_complete, fut) + self.assertIsNone(host) + + def test_create_server_host_port_sock(self): + fut = self.loop.create_server( + MyProto, '0.0.0.0', 0, sock=object()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + def test_create_server_no_host_port_sock(self): + fut = self.loop.create_server(MyProto) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + def test_create_server_no_getaddrinfo(self): + getaddrinfo = self.loop.getaddrinfo = mock.Mock() + getaddrinfo.return_value = self.loop.create_future() + getaddrinfo.return_value.set_result(None) + + f = self.loop.create_server(MyProto, 'python.org', 0) + self.assertRaises(OSError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_nosoreuseport(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + del m_socket.SO_REUSEPORT + m_socket.socket.return_value = mock.Mock() + + f = self.loop.create_server( + MyProto, '0.0.0.0', 0, reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_soreuseport_only_defined(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + m_socket.socket.return_value = mock.Mock() + m_socket.SO_REUSEPORT = -1 + + f = self.loop.create_server( + MyProto, '0.0.0.0', 0, reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, f) + + @patch_socket + def test_create_server_cant_bind(self, m_socket): + + class Err(OSError): + strerror = 'error' + + m_socket.getaddrinfo.return_value = [ + (2, 1, 6, '', ('127.0.0.1', 10100))] + m_sock = m_socket.socket.return_value = mock.Mock() + m_sock.bind.side_effect = Err + + fut = self.loop.create_server(MyProto, '0.0.0.0', 0) + self.assertRaises(OSError, self.loop.run_until_complete, fut) + self.assertTrue(m_sock.close.called) + + @patch_socket + def test_create_datagram_endpoint_no_addrinfo(self, m_socket): + m_socket.getaddrinfo.return_value = [] + + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('localhost', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_datagram_endpoint_addr_error(self): + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr='localhost') + self.assertRaises( + TypeError, self.loop.run_until_complete, coro) + coro = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('localhost', 1, 2, 3)) + self.assertRaises( + TypeError, self.loop.run_until_complete, coro) + + def test_create_datagram_endpoint_connect_err(self): + self.loop.sock_connect = mock.Mock() + self.loop.sock_connect.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, remote_addr=('127.0.0.1', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + def test_create_datagram_endpoint_allow_broadcast(self): + protocol = MyDatagramProto(create_future=True, loop=self.loop) + self.loop.sock_connect = sock_connect = mock.Mock() + sock_connect.return_value = [] + + coro = self.loop.create_datagram_endpoint( + lambda: protocol, + remote_addr=('127.0.0.1', 0), + allow_broadcast=True) + + transport, _ = self.loop.run_until_complete(coro) + self.assertFalse(sock_connect.called) + + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @patch_socket + def test_create_datagram_endpoint_socket_err(self, m_socket): + m_socket.getaddrinfo = socket.getaddrinfo + m_socket.socket.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, family=socket.AF_INET) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, local_addr=('127.0.0.1', 0)) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_datagram_endpoint_no_matching_family(self): + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, + remote_addr=('127.0.0.1', 0), local_addr=('::1', 0)) + self.assertRaises( + ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_setblk_err(self, m_socket): + m_socket.socket.return_value.setblocking.side_effect = OSError + + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, family=socket.AF_INET) + self.assertRaises( + OSError, self.loop.run_until_complete, coro) + self.assertTrue( + m_socket.socket.return_value.close.called) + + def test_create_datagram_endpoint_noaddr_nofamily(self): + coro = self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol) + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_cant_bind(self, m_socket): + class Err(OSError): + pass + + m_socket.getaddrinfo = socket.getaddrinfo + m_sock = m_socket.socket.return_value = mock.Mock() + m_sock.bind.side_effect = Err + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, + local_addr=('127.0.0.1', 0), family=socket.AF_INET) + self.assertRaises(Err, self.loop.run_until_complete, fut) + self.assertTrue(m_sock.close.called) + + def test_create_datagram_endpoint_sock(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(('127.0.0.1', 0)) + fut = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + sock=sock) + transport, protocol = self.loop.run_until_complete(fut) + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_datagram_endpoint_sock_unix(self): + fut = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + family=socket.AF_UNIX) + transport, protocol = self.loop.run_until_complete(fut) + self.assertEqual(transport._sock.family, socket.AF_UNIX) + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @socket_helper.skip_unless_bind_unix_socket + def test_create_datagram_endpoint_existing_sock_unix(self): + with test_utils.unix_socket_path() as path: + sock = socket.socket(socket.AF_UNIX, type=socket.SOCK_DGRAM) + sock.bind(path) + sock.close() + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + path, family=socket.AF_UNIX) + transport, protocol = self.loop.run_until_complete(coro) + transport.close() + self.loop.run_until_complete(protocol.done) + + def test_create_datagram_endpoint_sock_sockopts(self): + class FakeSock: + type = socket.SOCK_DGRAM + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, local_addr=('127.0.0.1', 0), sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, remote_addr=('127.0.0.1', 0), sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, family=1, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, proto=1, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, flags=1, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, reuse_port=True, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + fut = self.loop.create_datagram_endpoint( + MyDatagramProto, allow_broadcast=True, sock=FakeSock()) + self.assertRaises(ValueError, self.loop.run_until_complete, fut) + + @unittest.skipIf(sys.platform == 'vxworks', + "SO_BROADCAST is enabled by default on VxWorks") + def test_create_datagram_endpoint_sockopts(self): + # Socket options should not be applied unless asked for. + # SO_REUSEPORT is not available on all platforms. + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + local_addr=('127.0.0.1', 0)) + transport, protocol = self.loop.run_until_complete(coro) + sock = transport.get_extra_info('socket') + + reuseport_supported = hasattr(socket, 'SO_REUSEPORT') + + if reuseport_supported: + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST)) + + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + local_addr=('127.0.0.1', 0), + reuse_port=reuseport_supported, + allow_broadcast=True) + transport, protocol = self.loop.run_until_complete(coro) + sock = transport.get_extra_info('socket') + + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR)) + if reuseport_supported: + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_BROADCAST)) + + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + + @patch_socket + def test_create_datagram_endpoint_nosoreuseport(self, m_socket): + del m_socket.SO_REUSEPORT + m_socket.socket.return_value = mock.Mock() + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + local_addr=('127.0.0.1', 0), + reuse_port=True) + + self.assertRaises(ValueError, self.loop.run_until_complete, coro) + + @patch_socket + def test_create_datagram_endpoint_ip_addr(self, m_socket): + def getaddrinfo(*args, **kw): + self.fail('should not have called getaddrinfo') + + m_socket.getaddrinfo = getaddrinfo + m_socket.socket.return_value.bind = bind = mock.Mock() + self.loop._add_reader = mock.Mock() + + reuseport_supported = hasattr(socket, 'SO_REUSEPORT') + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + local_addr=('1.2.3.4', 0), + reuse_port=reuseport_supported) + + t, p = self.loop.run_until_complete(coro) + try: + bind.assert_called_with(('1.2.3.4', 0)) + m_socket.socket.assert_called_with(family=m_socket.AF_INET, + proto=m_socket.IPPROTO_UDP, + type=m_socket.SOCK_DGRAM) + finally: + t.close() + test_utils.run_briefly(self.loop) # allow transport to close + + def test_accept_connection_retry(self): + sock = mock.Mock() + sock.accept.side_effect = BlockingIOError() + + self.loop._accept_connection(MyProto, sock) + self.assertFalse(sock.close.called) + + @mock.patch('asyncio.base_events.logger') + def test_accept_connection_exception(self, m_log): + sock = mock.Mock() + sock.fileno.return_value = 10 + sock.accept.side_effect = OSError(errno.EMFILE, 'Too many open files') + self.loop._remove_reader = mock.Mock() + self.loop.call_later = mock.Mock() + + self.loop._accept_connection(MyProto, sock) + self.assertTrue(m_log.error.called) + self.assertFalse(sock.close.called) + self.loop._remove_reader.assert_called_with(10) + self.loop.call_later.assert_called_with( + constants.ACCEPT_RETRY_DELAY, + # self.loop._start_serving + mock.ANY, + MyProto, sock, None, None, mock.ANY, mock.ANY, mock.ANY) + + def test_call_coroutine(self): + async def simple_coroutine(): + pass + + self.loop.set_debug(True) + coro_func = simple_coroutine + coro_obj = coro_func() + self.addCleanup(coro_obj.close) + for func in (coro_func, coro_obj): + with self.assertRaises(TypeError): + self.loop.call_soon(func) + with self.assertRaises(TypeError): + self.loop.call_soon_threadsafe(func) + with self.assertRaises(TypeError): + self.loop.call_later(60, func) + with self.assertRaises(TypeError): + self.loop.call_at(self.loop.time() + 60, func) + with self.assertRaises(TypeError): + self.loop.run_until_complete( + self.loop.run_in_executor(None, func)) + + @mock.patch('asyncio.base_events.logger') + def test_log_slow_callbacks(self, m_logger): + def stop_loop_cb(loop): + loop.stop() + + async def stop_loop_coro(loop): + loop.stop() + + asyncio.set_event_loop(self.loop) + self.loop.set_debug(True) + self.loop.slow_callback_duration = 0.0 + + # slow callback + self.loop.call_soon(stop_loop_cb, self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing " + "took .* seconds$") + + # slow task + asyncio.ensure_future(stop_loop_coro(self.loop), loop=self.loop) + self.loop.run_forever() + fmt, *args = m_logger.warning.call_args[0] + self.assertRegex(fmt % tuple(args), + "^Executing " + "took .* seconds$") + + +class RunningLoopTests(unittest.TestCase): + + def test_running_loop_within_a_loop(self): + async def runner(loop): + loop.run_forever() + + loop = asyncio.new_event_loop() + outer_loop = asyncio.new_event_loop() + try: + with self.assertRaisesRegex(RuntimeError, + 'while another loop is running'): + outer_loop.run_until_complete(runner(loop)) + finally: + loop.close() + outer_loop.close() + + +class BaseLoopSockSendfileTests(test_utils.TestCase): + + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + + class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + + def connection_made(self, transport): + self.started = True + self.transport = transport + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + self.transport = None + + async def wait_closed(self): + await self.fut + + @classmethod + def setUpClass(cls): + cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16 + with open(os_helper.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize + os_helper.unlink(os_helper.TESTFN) + super().tearDownClass() + + def setUp(self): + from asyncio.selector_events import BaseSelectorEventLoop + # BaseSelectorEventLoop() has no native implementation + self.loop = BaseSelectorEventLoop() + self.set_event_loop(self.loop) + self.file = open(os_helper.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def make_socket(self, blocking=False): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(blocking) + self.addCleanup(sock.close) + return sock + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self): + sock = self.make_socket() + proto = self.MyProto(self.loop) + server = self.run_loop(self.loop.create_server( + lambda: proto, socket_helper.HOST, 0, family=socket.AF_INET)) + addr = server.sockets[0].getsockname() + + for _ in range(10): + try: + self.run_loop(self.loop.sock_connect(sock, addr)) + except OSError: + self.run_loop(asyncio.sleep(0.5)) + continue + else: + break + else: + # One last try, so we get the exception + self.run_loop(self.loop.sock_connect(sock, addr)) + + def cleanup(): + server.close() + sock.close() + if proto.transport is not None: + proto.transport.close() + self.run_loop(proto.wait_closed()) + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test__sock_sendfile_native_failure(self): + sock, proto = self.prepare() + + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "sendfile is not available"): + self.run_loop(self.loop._sock_sendfile_native(sock, self.file, + 0, None)) + + self.assertEqual(proto.data, b'') + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_no_fallback(self): + sock, proto = self.prepare() + + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "sendfile is not available"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, + fallback=False)) + + self.assertEqual(self.file.tell(), 0) + self.assertEqual(proto.data, b'') + + def test_sock_sendfile_fallback(self): + sock, proto = self.prepare() + + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(self.file.tell(), len(self.DATA)) + self.assertEqual(proto.data, self.DATA) + + def test_sock_sendfile_fallback_offset_and_count(self): + sock, proto = self.prepare() + + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file, + 1000, 2000)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, 2000) + self.assertEqual(self.file.tell(), 3000) + self.assertEqual(proto.data, self.DATA[1000:3000]) + + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) + + def test_nonbinary_file(self): + sock = self.make_socket() + with open(os_helper.TESTFN, encoding="utf-8") as f: + with self.assertRaisesRegex(ValueError, "binary mode"): + self.run_loop(self.loop.sock_sendfile(sock, f)) + + def test_nonstream_socket(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + self.addCleanup(sock.close) + with self.assertRaisesRegex(ValueError, "only SOCK_STREAM type"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) + + def test_notint_count(self): + sock = self.make_socket() + with self.assertRaisesRegex(TypeError, + "count must be a positive integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 0, 'count')) + + def test_negative_count(self): + sock = self.make_socket() + with self.assertRaisesRegex(ValueError, + "count must be a positive integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 0, -1)) + + def test_notint_offset(self): + sock = self.make_socket() + with self.assertRaisesRegex(TypeError, + "offset must be a non-negative integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 'offset')) + + def test_negative_offset(self): + sock = self.make_socket() + with self.assertRaisesRegex(ValueError, + "offset must be a non-negative integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, -1)) + + +class TestSelectorUtils(test_utils.TestCase): + def check_set_nodelay(self, sock): + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) + self.assertFalse(opt) + + base_events._set_nodelay(sock) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) + self.assertTrue(opt) + + @unittest.skipUnless(hasattr(socket, 'TCP_NODELAY'), + 'need socket.TCP_NODELAY') + def test_set_nodelay(self): + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + self.check_set_nodelay(sock) + + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + sock.setblocking(False) + self.check_set_nodelay(sock) + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py new file mode 100644 index 00000000000..6d3edcc36f5 --- /dev/null +++ b/Lib/test/test_asyncio/test_buffered_proto.py @@ -0,0 +1,89 @@ +import asyncio +import unittest + +from test.test_asyncio import functional as func_tests + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class ReceiveStuffProto(asyncio.BufferedProtocol): + def __init__(self, cb, con_lost_fut): + self.cb = cb + self.con_lost_fut = con_lost_fut + + def get_buffer(self, sizehint): + self.buffer = bytearray(100) + return self.buffer + + def buffer_updated(self, nbytes): + self.cb(self.buffer[:nbytes]) + + def connection_lost(self, exc): + if exc is None: + self.con_lost_fut.set_result(None) + else: + self.con_lost_fut.set_exception(exc) + + +class BaseTestBufferedProtocol(func_tests.FunctionalTestCaseMixin): + + def new_loop(self): + raise NotImplementedError + + def test_buffered_proto_create_connection(self): + + NOISE = b'12345678+' * 1024 + + async def client(addr): + data = b'' + + def on_buf(buf): + nonlocal data + data += buf + if data == NOISE: + tr.write(b'1') + + conn_lost_fut = self.loop.create_future() + + tr, pr = await self.loop.create_connection( + lambda: ReceiveStuffProto(on_buf, conn_lost_fut), *addr) + + await conn_lost_fut + + async def on_server_client(reader, writer): + writer.write(NOISE) + await reader.readexactly(1) + writer.close() + await writer.wait_closed() + + srv = self.loop.run_until_complete( + asyncio.start_server( + on_server_client, '127.0.0.1', 0)) + + addr = srv.sockets[0].getsockname() + self.loop.run_until_complete( + asyncio.wait_for(client(addr), 5)) + + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + +class BufferedProtocolSelectorTests(BaseTestBufferedProtocol, + unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class BufferedProtocolProactorTests(BaseTestBufferedProtocol, + unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py new file mode 100644 index 00000000000..f85f39839cb --- /dev/null +++ b/Lib/test/test_asyncio/test_context.py @@ -0,0 +1,38 @@ +import asyncio +import decimal +import unittest + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +@unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") +class DecimalContextTest(unittest.TestCase): + + def test_asyncio_task_decimal_context(self): + async def fractions(t, precision, x, y): + with decimal.localcontext() as ctx: + ctx.prec = precision + a = decimal.Decimal(x) / decimal.Decimal(y) + await asyncio.sleep(t) + b = decimal.Decimal(x) / decimal.Decimal(y ** 2) + return a, b + + async def main(): + r1, r2 = await asyncio.gather( + fractions(0.1, 3, 1, 3), fractions(0.2, 6, 1, 3)) + + return r1, r2 + + r1, r2 = asyncio.run(main()) + + self.assertEqual(str(r1[0]), '0.333') + self.assertEqual(str(r1[1]), '0.111') + + self.assertEqual(str(r2[0]), '0.333333') + self.assertEqual(str(r2[1]), '0.111111') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py new file mode 100644 index 00000000000..0561b54a3f1 --- /dev/null +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -0,0 +1,545 @@ +"""Tests for base_events.py""" + +import asyncio +import contextvars +import unittest + +from unittest import mock +from asyncio import tasks +from test.test_asyncio import utils as test_utils +from test.support.script_helper import assert_python_ok + +MOCK_ANY = mock.ANY + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class EagerTaskFactoryLoopTests: + + Task = None + + def run_coro(self, coro): + """ + Helper method to run the `coro` coroutine in the test event loop. + It helps with making sure the event loop is running before starting + to execute `coro`. This is important for testing the eager step + functionality, since an eager step is taken only if the event loop + is already running. + """ + + async def coro_runner(): + self.assertTrue(asyncio.get_event_loop().is_running()) + return await coro + + return self.loop.run_until_complete(coro) + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.eager_task_factory = asyncio.create_eager_task_factory(self.Task) + self.loop.set_task_factory(self.eager_task_factory) + self.set_event_loop(self.loop) + + def test_eager_task_factory_set(self): + self.assertIsNotNone(self.eager_task_factory) + self.assertIs(self.loop.get_task_factory(), self.eager_task_factory) + + async def noop(): pass + + async def run(): + t = self.loop.create_task(noop()) + self.assertIsInstance(t, self.Task) + await t + + self.run_coro(run()) + + def test_await_future_during_eager_step(self): + + async def set_result(fut, val): + fut.set_result(val) + + async def run(): + fut = self.loop.create_future() + t = self.loop.create_task(set_result(fut, 'my message')) + # assert the eager step completed the task + self.assertTrue(t.done()) + return await fut + + self.assertEqual(self.run_coro(run()), 'my message') + + def test_eager_completion(self): + + async def coro(): + return 'hello' + + async def run(): + t = self.loop.create_task(coro()) + # assert the eager step completed the task + self.assertTrue(t.done()) + return await t + + self.assertEqual(self.run_coro(run()), 'hello') + + def test_block_after_eager_step(self): + + async def coro(): + await asyncio.sleep(0.1) + return 'finished after blocking' + + async def run(): + t = self.loop.create_task(coro()) + self.assertFalse(t.done()) + result = await t + self.assertTrue(t.done()) + return result + + self.assertEqual(self.run_coro(run()), 'finished after blocking') + + def test_cancellation_after_eager_completion(self): + + async def coro(): + return 'finished without blocking' + + async def run(): + t = self.loop.create_task(coro()) + t.cancel() + result = await t + # finished task can't be cancelled + self.assertFalse(t.cancelled()) + return result + + self.assertEqual(self.run_coro(run()), 'finished without blocking') + + def test_cancellation_after_eager_step_blocks(self): + + async def coro(): + await asyncio.sleep(0.1) + return 'finished after blocking' + + async def run(): + t = self.loop.create_task(coro()) + t.cancel('cancellation message') + self.assertGreater(t.cancelling(), 0) + result = await t + + with self.assertRaises(asyncio.CancelledError) as cm: + self.run_coro(run()) + + self.assertEqual('cancellation message', cm.exception.args[0]) + + def test_current_task(self): + captured_current_task = None + + async def coro(): + nonlocal captured_current_task + captured_current_task = asyncio.current_task() + # verify the task before and after blocking is identical + await asyncio.sleep(0.1) + self.assertIs(asyncio.current_task(), captured_current_task) + + async def run(): + t = self.loop.create_task(coro()) + self.assertIs(captured_current_task, t) + await t + + self.run_coro(run()) + captured_current_task = None + + def test_all_tasks_with_eager_completion(self): + captured_all_tasks = None + + async def coro(): + nonlocal captured_all_tasks + captured_all_tasks = asyncio.all_tasks() + + async def run(): + t = self.loop.create_task(coro()) + self.assertIn(t, captured_all_tasks) + self.assertNotIn(t, asyncio.all_tasks()) + + self.run_coro(run()) + + def test_all_tasks_with_blocking(self): + captured_eager_all_tasks = None + + async def coro(fut1, fut2): + nonlocal captured_eager_all_tasks + captured_eager_all_tasks = asyncio.all_tasks() + await fut1 + fut2.set_result(None) + + async def run(): + fut1 = self.loop.create_future() + fut2 = self.loop.create_future() + t = self.loop.create_task(coro(fut1, fut2)) + self.assertIn(t, captured_eager_all_tasks) + self.assertIn(t, asyncio.all_tasks()) + fut1.set_result(None) + await fut2 + self.assertNotIn(t, asyncio.all_tasks()) + + self.run_coro(run()) + + def test_context_vars(self): + cv = contextvars.ContextVar('cv', default=0) + + coro_first_step_ran = False + coro_second_step_ran = False + + async def coro(): + nonlocal coro_first_step_ran + nonlocal coro_second_step_ran + self.assertEqual(cv.get(), 1) + cv.set(2) + self.assertEqual(cv.get(), 2) + coro_first_step_ran = True + await asyncio.sleep(0.1) + self.assertEqual(cv.get(), 2) + cv.set(3) + self.assertEqual(cv.get(), 3) + coro_second_step_ran = True + + async def run(): + cv.set(1) + t = self.loop.create_task(coro()) + self.assertTrue(coro_first_step_ran) + self.assertFalse(coro_second_step_ran) + self.assertEqual(cv.get(), 1) + await t + self.assertTrue(coro_second_step_ran) + self.assertEqual(cv.get(), 1) + + self.run_coro(run()) + + def test_staggered_race_with_eager_tasks(self): + # See https://github.com/python/cpython/issues/124309 + + async def fail(): + await asyncio.sleep(0) + raise ValueError("no good") + + async def blocked(): + fut = asyncio.Future() + await fut + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: blocked(), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: fail() + ], + delay=0.25 + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIsInstance(excs[2], ValueError) + + self.run_coro(run()) + + def test_staggered_race_with_eager_tasks_no_delay(self): + # See https://github.com/python/cpython/issues/124309 + async def fail(): + raise ValueError("no good") + + async def run(): + winner, index, excs = await asyncio.staggered.staggered_race( + [ + lambda: fail(), + lambda: asyncio.sleep(1, result="sleep1"), + lambda: asyncio.sleep(0, result="sleep0"), + ], + delay=None + ) + self.assertEqual(winner, 'sleep1') + self.assertEqual(index, 1) + self.assertIsNone(excs[index]) + self.assertIsInstance(excs[0], ValueError) + self.assertEqual(len(excs), 2) + + self.run_coro(run()) + + def test_eager_start_false(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + t = asyncio.get_running_loop().create_task( + asyncfn(), eager_start=False, name="example" + ) + self.assertFalse(t.done()) + self.assertIsNone(name) + await t + self.assertEqual(name, "example") + + self.run_coro(main()) + + +class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): + Task = tasks._PyTask + + def setUp(self): + self._all_tasks = asyncio.all_tasks + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._py_all_tasks + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks + return super().tearDown() + + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + def setUp(self): + self._current_task = asyncio.current_task + self._all_tasks = asyncio.all_tasks + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._c_all_tasks + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks + return super().tearDown() + + def test_issue105987(self): + code = """if 1: + from _asyncio import _swap_current_task, _set_running_loop + + class DummyTask: + pass + + class DummyLoop: + pass + + l = DummyLoop() + _set_running_loop(l) + _swap_current_task(l, DummyTask()) + t = _swap_current_task(l, None) + """ + + _, out, err = assert_python_ok("-c", code) + self.assertFalse(err) + + def test_issue122332(self): + async def coro(): + pass + + async def run(): + task = self.loop.create_task(coro()) + await task + self.assertIsNone(task.get_coro()) + + self.run_coro(run()) + + def test_name(self): + name = None + async def coro(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + task = self.loop.create_task(coro(), name="test name") + self.assertEqual(name, "test name") + await task + + self.run_coro(coro()) + +class AsyncTaskCounter: + def __init__(self, loop, *, task_class, eager): + self.suspense_count = 0 + self.task_count = 0 + + def CountingTask(*args, eager_start=False, **kwargs): + if not eager_start: + self.task_count += 1 + kwargs["eager_start"] = eager_start + return task_class(*args, **kwargs) + + if eager: + factory = asyncio.create_eager_task_factory(CountingTask) + else: + def factory(loop, coro, **kwargs): + return CountingTask(coro, loop=loop, **kwargs) + loop.set_task_factory(factory) + + def get(self): + return self.task_count + + +async def awaitable_chain(depth): + if depth == 0: + return 0 + return 1 + await awaitable_chain(depth - 1) + + +async def recursive_taskgroups(width, depth): + if depth == 0: + return + + async with asyncio.TaskGroup() as tg: + futures = [ + tg.create_task(recursive_taskgroups(width, depth - 1)) + for _ in range(width) + ] + + +async def recursive_gather(width, depth): + if depth == 0: + return + + await asyncio.gather( + *[recursive_gather(width, depth - 1) for _ in range(width)] + ) + + +class BaseTaskCountingTests: + + Task = None + eager = None + expected_task_count = None + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.counter = AsyncTaskCounter(self.loop, task_class=self.Task, eager=self.eager) + self.set_event_loop(self.loop) + + def test_awaitables_chain(self): + observed_depth = self.loop.run_until_complete(awaitable_chain(100)) + self.assertEqual(observed_depth, 100) + self.assertEqual(self.counter.get(), 0 if self.eager else 1) + + def test_recursive_taskgroups(self): + num_tasks = self.loop.run_until_complete(recursive_taskgroups(5, 4)) + self.assertEqual(self.counter.get(), self.expected_task_count) + + def test_recursive_gather(self): + self.loop.run_until_complete(recursive_gather(5, 4)) + self.assertEqual(self.counter.get(), self.expected_task_count) + + +class BaseNonEagerTaskFactoryTests(BaseTaskCountingTests): + eager = False + expected_task_count = 781 # 1 + 5 + 5^2 + 5^3 + 5^4 + + +class BaseEagerTaskFactoryTests(BaseTaskCountingTests): + eager = True + expected_task_count = 0 + + +class NonEagerTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = asyncio.tasks._CTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + +class EagerTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = asyncio.tasks._CTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +class NonEagerPyTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = tasks._PyTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +class EagerPyTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = tasks._PyTask + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class NonEagerCTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class EagerCTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + def setUp(self): + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._current_task + return super().tearDown() + + +class DefaultTaskFactoryEagerStart(test_utils.TestCase): + def test_eager_start_true_with_default_factory(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async def main(): + t = asyncio.get_running_loop().create_task( + asyncfn(), eager_start=True, name="example" + ) + self.assertTrue(t.done()) + self.assertEqual(name, "example") + await t + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py new file mode 100644 index 00000000000..b60c7452f3f --- /dev/null +++ b/Lib/test/test_asyncio/test_events.py @@ -0,0 +1,3179 @@ +"""Tests for events.py.""" + +import concurrent.futures +import contextlib +import functools +import io +import multiprocessing +import os +import platform +import re +import signal +import socket +try: + import ssl +except ImportError: + ssl = None +import subprocess +import sys +import threading +import time +import types +import errno +import unittest +from unittest import mock +import weakref +if sys.platform not in ('win32', 'vxworks'): + import tty + +import asyncio +from asyncio import coroutines +from asyncio import events +from asyncio import selector_events +from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests +from test.test_asyncio import utils as test_utils +from test import support +from test.support import socket_helper +from test.support import threading_helper +from test.support import ALWAYS_EQ, LARGEST, SMALLEST + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def broken_unix_getsockname(): + """Return True if the platform is Mac OS 10.4 or older.""" + if sys.platform.startswith("aix"): + return True + elif sys.platform != 'darwin': + return False + version = platform.mac_ver()[0] + version = tuple(map(int, version.split('.'))) + return version < (10, 5) + + +def _test_get_event_loop_new_process__sub_proc(): + async def doit(): + return 'hello' + + with contextlib.closing(asyncio.new_event_loop()) as loop: + asyncio.set_event_loop(loop) + return loop.run_until_complete(doit()) + + +class CoroLike: + def send(self, v): + pass + + def throw(self, *exc): + pass + + def close(self): + pass + + def __await__(self): + pass + + +class MyBaseProto(asyncio.Protocol): + connected = None + done = None + + def __init__(self, loop=None): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = loop.create_future() + self.done = loop.create_future() + + def _assert_state(self, *expected): + if self.state not in expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + + def data_received(self, data): + self._assert_state('CONNECTED') + self.nbytes += len(data) + + def eof_received(self): + self._assert_state('CONNECTED') + self.state = 'EOF' + + def connection_lost(self, exc): + self._assert_state('CONNECTED', 'EOF') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyProto(MyBaseProto): + def connection_made(self, transport): + super().connection_made(transport) + transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n') + + +class MyDatagramProto(asyncio.DatagramProtocol): + done = None + + def __init__(self, loop=None): + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.done = loop.create_future() + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'INITIALIZED' + + def datagram_received(self, data, addr): + self._assert_state('INITIALIZED') + self.nbytes += len(data) + + def error_received(self, exc): + self._assert_state('INITIALIZED') + + def connection_lost(self, exc): + self._assert_state('INITIALIZED') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MyReadPipeProto(asyncio.Protocol): + done = None + + def __init__(self, loop=None): + self.state = ['INITIAL'] + self.nbytes = 0 + self.transport = None + if loop is not None: + self.done = loop.create_future() + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state(['INITIAL']) + self.state.append('CONNECTED') + + def data_received(self, data): + self._assert_state(['INITIAL', 'CONNECTED']) + self.nbytes += len(data) + + def eof_received(self): + self._assert_state(['INITIAL', 'CONNECTED']) + self.state.append('EOF') + + def connection_lost(self, exc): + if 'EOF' not in self.state: + self.state.append('EOF') # It is okay if EOF is missed. + self._assert_state(['INITIAL', 'CONNECTED', 'EOF']) + self.state.append('CLOSED') + if self.done: + self.done.set_result(None) + + +class MyWritePipeProto(asyncio.BaseProtocol): + done = None + + def __init__(self, loop=None): + self.state = 'INITIAL' + self.transport = None + if loop is not None: + self.done = loop.create_future() + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + + def connection_lost(self, exc): + self._assert_state('CONNECTED') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MySubprocessProtocol(asyncio.SubprocessProtocol): + + def __init__(self, loop): + self.state = 'INITIAL' + self.transport = None + self.connected = loop.create_future() + self.completed = loop.create_future() + self.disconnects = {fd: loop.create_future() for fd in range(3)} + self.data = {1: b'', 2: b''} + self.returncode = None + self.got_data = {1: asyncio.Event(), + 2: asyncio.Event()} + + def _assert_state(self, expected): + if self.state != expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + self.connected.set_result(None) + + def connection_lost(self, exc): + self._assert_state('CONNECTED') + self.state = 'CLOSED' + self.completed.set_result(None) + + def pipe_data_received(self, fd, data): + self._assert_state('CONNECTED') + self.data[fd] += data + self.got_data[fd].set() + + def pipe_connection_lost(self, fd, exc): + self._assert_state('CONNECTED') + if exc: + self.disconnects[fd].set_exception(exc) + else: + self.disconnects[fd].set_result(exc) + + def process_exited(self): + self._assert_state('CONNECTED') + self.returncode = self.transport.get_returncode() + + +class EventLoopTestsMixin: + + def setUp(self): + super().setUp() + self.loop = self.create_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + @unittest.expectedFailure # TODO: RUSTPYTHON; - RuntimeWarning for unawaited coroutine not triggered + def test_run_until_complete_nesting(self): + async def coro1(): + await asyncio.sleep(0) + + async def coro2(): + self.assertTrue(self.loop.is_running()) + self.loop.run_until_complete(coro1()) + + with self.assertWarnsRegex( + RuntimeWarning, + r"coroutine \S+ was never awaited" + ): + self.assertRaises( + RuntimeError, self.loop.run_until_complete, coro2()) + + # Note: because of the default Windows timing granularity of + # 15.6 msec, we use fairly long sleep times here (~100 msec). + + def test_run_until_complete(self): + delay = 0.100 + t0 = self.loop.time() + self.loop.run_until_complete(asyncio.sleep(delay)) + dt = self.loop.time() - t0 + self.assertGreaterEqual(dt, delay - test_utils.CLOCK_RES) + + def test_run_until_complete_stopped(self): + + async def cb(): + self.loop.stop() + await asyncio.sleep(0.1) + task = cb() + self.assertRaises(RuntimeError, + self.loop.run_until_complete, task) + + def test_call_later(self): + results = [] + + def callback(arg): + results.append(arg) + self.loop.stop() + + self.loop.call_later(0.1, callback, 'hello world') + self.loop.run_forever() + self.assertEqual(results, ['hello world']) + + def test_call_soon(self): + results = [] + + def callback(arg1, arg2): + results.append((arg1, arg2)) + self.loop.stop() + + self.loop.call_soon(callback, 'hello', 'world') + self.loop.run_forever() + self.assertEqual(results, [('hello', 'world')]) + + def test_call_soon_threadsafe(self): + results = [] + lock = threading.Lock() + + def callback(arg): + results.append(arg) + if len(results) >= 2: + self.loop.stop() + + def run_in_thread(): + self.loop.call_soon_threadsafe(callback, 'hello') + lock.release() + + lock.acquire() + t = threading.Thread(target=run_in_thread) + t.start() + + with lock: + self.loop.call_soon(callback, 'world') + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello', 'world']) + + def test_call_soon_threadsafe_handle_block_check_cancelled(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it should block checking for cancellation + # until it finishes + self.assertFalse(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_block_cancellation(self): + results = [] + + callback_started = threading.Event() + callback_finished = threading.Event() + def callback(arg): + callback_started.set() + results.append(arg) + time.sleep(1) + callback_finished.set() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback started so it cannot be cancelled from other thread until + # it finishes + handle.cancel() + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_same_thread(self): + results = [] + callback_started = threading.Event() + callback_finished = threading.Event() + + fut = concurrent.futures.Future() + def callback(arg): + callback_started.set() + handle = fut.result() + handle.cancel() + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + fut.set_result(handle) + self.assertIsInstance(handle, events._ThreadSafeHandle) + callback_started.wait() + # callback cancels itself from same thread so it has no effect + # it runs to completion + self.assertTrue(handle.cancelled()) + self.assertTrue(callback_finished.is_set()) + self.loop.call_soon_threadsafe(self.loop.stop) + + t = threading.Thread(target=run_in_thread) + t.start() + + self.loop.run_forever() + t.join() + self.assertEqual(results, ['hello']) + + def test_call_soon_threadsafe_handle_cancel_other_thread(self): + results = [] + ev = threading.Event() + + callback_finished = threading.Event() + def callback(arg): + results.append(arg) + callback_finished.set() + self.loop.stop() + + def run_in_thread(): + handle = self.loop.call_soon_threadsafe(callback, 'hello') + # handle can be cancelled from other thread if not started yet + self.assertIsInstance(handle, events._ThreadSafeHandle) + handle.cancel() + self.assertTrue(handle.cancelled()) + self.assertFalse(callback_finished.is_set()) + ev.set() + self.loop.call_soon_threadsafe(self.loop.stop) + + # block the main loop until the callback is added and cancelled in the + # other thread + self.loop.call_soon(ev.wait) + t = threading.Thread(target=run_in_thread) + t.start() + self.loop.run_forever() + t.join() + self.assertEqual(results, []) + self.assertFalse(callback_finished.is_set()) + + def test_call_soon_threadsafe_same_thread(self): + results = [] + + def callback(arg): + results.append(arg) + if len(results) >= 2: + self.loop.stop() + + self.loop.call_soon_threadsafe(callback, 'hello') + self.loop.call_soon(callback, 'world') + self.loop.run_forever() + self.assertEqual(results, ['hello', 'world']) + + def test_run_in_executor(self): + def run(arg): + return (arg, threading.get_ident()) + f2 = self.loop.run_in_executor(None, run, 'yo') + res, thread_id = self.loop.run_until_complete(f2) + self.assertEqual(res, 'yo') + self.assertNotEqual(thread_id, threading.get_ident()) + + def test_run_in_executor_cancel(self): + called = False + + def patched_call_soon(*args): + nonlocal called + called = True + + def run(): + time.sleep(0.05) + + f2 = self.loop.run_in_executor(None, run) + f2.cancel() + self.loop.run_until_complete( + self.loop.shutdown_default_executor()) + self.loop.close() + self.loop.call_soon = patched_call_soon + self.loop.call_soon_threadsafe = patched_call_soon + time.sleep(0.4) + self.assertFalse(called) + + def test_reader_callback(self): + r, w = socket.socketpair() + r.setblocking(False) + bytes_read = bytearray() + + def reader(): + try: + data = r.recv(1024) + except BlockingIOError: + # Spurious readiness notifications are possible + # at least on Linux -- see man select. + return + if data: + bytes_read.extend(data) + else: + self.assertTrue(self.loop.remove_reader(r.fileno())) + r.close() + + self.loop.add_reader(r.fileno(), reader) + self.loop.call_soon(w.send, b'abc') + test_utils.run_until(self.loop, lambda: len(bytes_read) >= 3) + self.loop.call_soon(w.send, b'def') + test_utils.run_until(self.loop, lambda: len(bytes_read) >= 6) + self.loop.call_soon(w.close) + self.loop.call_soon(self.loop.stop) + self.loop.run_forever() + self.assertEqual(bytes_read, b'abcdef') + + def test_writer_callback(self): + r, w = socket.socketpair() + w.setblocking(False) + + def writer(data): + w.send(data) + self.loop.stop() + + data = b'x' * 1024 + self.loop.add_writer(w.fileno(), writer, data) + self.loop.run_forever() + + self.assertTrue(self.loop.remove_writer(w.fileno())) + self.assertFalse(self.loop.remove_writer(w.fileno())) + + w.close() + read = r.recv(len(data) * 2) + r.close() + self.assertEqual(read, data) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - signal handler implementation differs + @unittest.skipUnless(hasattr(signal, 'SIGKILL'), 'No SIGKILL') + def test_add_signal_handler(self): + caught = 0 + + def my_handler(): + nonlocal caught + caught += 1 + + # Check error behavior first. + self.assertRaises( + TypeError, self.loop.add_signal_handler, 'boom', my_handler) + self.assertRaises( + TypeError, self.loop.remove_signal_handler, 'boom') + self.assertRaises( + ValueError, self.loop.add_signal_handler, signal.NSIG+1, + my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, signal.NSIG+1) + self.assertRaises( + ValueError, self.loop.add_signal_handler, 0, my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, 0) + self.assertRaises( + ValueError, self.loop.add_signal_handler, -1, my_handler) + self.assertRaises( + ValueError, self.loop.remove_signal_handler, -1) + self.assertRaises( + RuntimeError, self.loop.add_signal_handler, signal.SIGKILL, + my_handler) + # Removing SIGKILL doesn't raise, since we don't call signal(). + self.assertFalse(self.loop.remove_signal_handler(signal.SIGKILL)) + # Now set a handler and handle it. + self.loop.add_signal_handler(signal.SIGINT, my_handler) + + os.kill(os.getpid(), signal.SIGINT) + test_utils.run_until(self.loop, lambda: caught) + + # Removing it should restore the default handler. + self.assertTrue(self.loop.remove_signal_handler(signal.SIGINT)) + self.assertEqual(signal.getsignal(signal.SIGINT), + signal.default_int_handler) + # Removing again returns False. + self.assertFalse(self.loop.remove_signal_handler(signal.SIGINT)) + + @unittest.skipUnless(hasattr(signal, 'SIGALRM'), 'No SIGALRM') + @unittest.skipUnless(hasattr(signal, 'setitimer'), + 'need signal.setitimer()') + def test_signal_handling_while_selecting(self): + # Test with a signal actually arriving during a select() call. + caught = 0 + + def my_handler(): + nonlocal caught + caught += 1 + self.loop.stop() + + self.loop.add_signal_handler(signal.SIGALRM, my_handler) + + signal.setitimer(signal.ITIMER_REAL, 0.01, 0) # Send SIGALRM once. + self.loop.call_later(60, self.loop.stop) + self.loop.run_forever() + self.assertEqual(caught, 1) + + @unittest.skipUnless(hasattr(signal, 'SIGALRM'), 'No SIGALRM') + @unittest.skipUnless(hasattr(signal, 'setitimer'), + 'need signal.setitimer()') + def test_signal_handling_args(self): + some_args = (42,) + caught = 0 + + def my_handler(*args): + nonlocal caught + caught += 1 + self.assertEqual(args, some_args) + self.loop.stop() + + self.loop.add_signal_handler(signal.SIGALRM, my_handler, *some_args) + + signal.setitimer(signal.ITIMER_REAL, 0.1, 0) # Send SIGALRM once. + self.loop.call_later(60, self.loop.stop) + self.loop.run_forever() + self.assertEqual(caught, 1) + + def _basetest_create_connection(self, connection_fut, check_sockname=True): + tr, pr = self.loop.run_until_complete(connection_fut) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.assertIs(pr.transport, tr) + if check_sockname: + self.assertIsNotNone(tr.get_extra_info('sockname')) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + def test_create_connection(self): + with test_utils.run_test_server() as httpd: + conn_fut = self.loop.create_connection( + lambda: MyProto(loop=self.loop), *httpd.address) + self._basetest_create_connection(conn_fut) + + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_connection(self): + # Issue #20682: On Mac OS X Tiger, getsockname() returns a + # zero-length address for UNIX socket. + check_sockname = not broken_unix_getsockname() + + with test_utils.run_test_unix_server() as httpd: + conn_fut = self.loop.create_unix_connection( + lambda: MyProto(loop=self.loop), httpd.address) + self._basetest_create_connection(conn_fut, check_sockname) + + def check_ssl_extra_info(self, client, check_sockname=True, + peername=None, peercert={}): + if check_sockname: + self.assertIsNotNone(client.get_extra_info('sockname')) + if peername: + self.assertEqual(peername, + client.get_extra_info('peername')) + else: + self.assertIsNotNone(client.get_extra_info('peername')) + self.assertEqual(peercert, + client.get_extra_info('peercert')) + + # test SSL cipher + cipher = client.get_extra_info('cipher') + self.assertIsInstance(cipher, tuple) + self.assertEqual(len(cipher), 3, cipher) + self.assertIsInstance(cipher[0], str) + self.assertIsInstance(cipher[1], str) + self.assertIsInstance(cipher[2], int) + + # test SSL object + sslobj = client.get_extra_info('ssl_object') + self.assertIsNotNone(sslobj) + self.assertEqual(sslobj.compression(), + client.get_extra_info('compression')) + self.assertEqual(sslobj.cipher(), + client.get_extra_info('cipher')) + self.assertEqual(sslobj.getpeercert(), + client.get_extra_info('peercert')) + self.assertEqual(sslobj.compression(), + client.get_extra_info('compression')) + + def _basetest_create_ssl_connection(self, connection_fut, + check_sockname=True, + peername=None): + tr, pr = self.loop.run_until_complete(connection_fut) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.assertTrue('ssl' in tr.__class__.__name__.lower()) + self.check_ssl_extra_info(tr, check_sockname, peername) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + def _test_create_ssl_connection(self, httpd, create_connection, + check_sockname=True, peername=None): + conn_fut = create_connection(ssl=test_utils.dummy_ssl_context()) + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + + # ssl.Purpose was introduced in Python 3.4 + if hasattr(ssl, 'Purpose'): + def _dummy_ssl_create_context(purpose=ssl.Purpose.SERVER_AUTH, *, + cafile=None, capath=None, + cadata=None): + """ + A ssl.create_default_context() replacement that doesn't enable + cert validation. + """ + self.assertEqual(purpose, ssl.Purpose.SERVER_AUTH) + return test_utils.dummy_ssl_context() + + # With ssl=True, ssl.create_default_context() should be called + with mock.patch('ssl.create_default_context', + side_effect=_dummy_ssl_create_context) as m: + conn_fut = create_connection(ssl=True) + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + self.assertEqual(m.call_count, 1) + + # With the real ssl.create_default_context(), certificate + # validation will fail + with self.assertRaises(ssl.SSLError) as cm: + conn_fut = create_connection(ssl=True) + # Ignore the "SSL handshake failed" log in debug mode + with test_utils.disable_logger(): + self._basetest_create_ssl_connection(conn_fut, check_sockname, + peername) + + self.assertEqual(cm.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_ssl_connection(self): + with test_utils.run_test_server(use_ssl=True) as httpd: + create_connection = functools.partial( + self.loop.create_connection, + lambda: MyProto(loop=self.loop), + *httpd.address) + self._test_create_ssl_connection(httpd, create_connection, + peername=httpd.address) + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_ssl_unix_connection(self): + # Issue #20682: On Mac OS X Tiger, getsockname() returns a + # zero-length address for UNIX socket. + check_sockname = not broken_unix_getsockname() + + with test_utils.run_test_unix_server(use_ssl=True) as httpd: + create_connection = functools.partial( + self.loop.create_unix_connection, + lambda: MyProto(loop=self.loop), httpd.address, + server_hostname='127.0.0.1') + + self._test_create_ssl_connection(httpd, create_connection, + check_sockname, + peername=httpd.address) + + def test_create_connection_local_addr(self): + with test_utils.run_test_server() as httpd: + port = socket_helper.find_unused_port() + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + *httpd.address, local_addr=(httpd.address[0], port)) + tr, pr = self.loop.run_until_complete(f) + expected = pr.transport.get_extra_info('sockname')[1] + self.assertEqual(port, expected) + tr.close() + + @socket_helper.skip_if_tcp_blackhole + def test_create_connection_local_addr_skip_different_family(self): + # See https://github.com/python/cpython/issues/86508 + port1 = socket_helper.find_unused_port() + port2 = socket_helper.find_unused_port() + getaddrinfo_orig = self.loop.getaddrinfo + + async def getaddrinfo(host, port, *args, **kwargs): + if port == port2: + return [(socket.AF_INET6, socket.SOCK_STREAM, 0, '', ('::1', 0, 0, 0)), + (socket.AF_INET, socket.SOCK_STREAM, 0, '', ('127.0.0.1', 0))] + return await getaddrinfo_orig(host, port, *args, **kwargs) + + self.loop.getaddrinfo = getaddrinfo + + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + 'localhost', port1, local_addr=('localhost', port2)) + + with self.assertRaises(OSError): + self.loop.run_until_complete(f) + + @socket_helper.skip_if_tcp_blackhole + def test_create_connection_local_addr_nomatch_family(self): + # See https://github.com/python/cpython/issues/86508 + port1 = socket_helper.find_unused_port() + port2 = socket_helper.find_unused_port() + getaddrinfo_orig = self.loop.getaddrinfo + + async def getaddrinfo(host, port, *args, **kwargs): + if port == port2: + return [(socket.AF_INET6, socket.SOCK_STREAM, 0, '', ('::1', 0, 0, 0))] + return await getaddrinfo_orig(host, port, *args, **kwargs) + + self.loop.getaddrinfo = getaddrinfo + + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + 'localhost', port1, local_addr=('localhost', port2)) + + with self.assertRaises(OSError): + self.loop.run_until_complete(f) + + def test_create_connection_local_addr_in_use(self): + with test_utils.run_test_server() as httpd: + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), + *httpd.address, local_addr=httpd.address) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(f) + self.assertEqual(cm.exception.errno, errno.EADDRINUSE) + self.assertIn(str(httpd.address), cm.exception.strerror) + + def test_connect_accepted_socket(self, server_ssl=None, client_ssl=None): + loop = self.loop + + class MyProto(MyBaseProto): + + def connection_lost(self, exc): + super().connection_lost(exc) + loop.call_soon(loop.stop) + + def data_received(self, data): + super().data_received(data) + self.transport.write(expected_response) + + lsock = socket.create_server(('127.0.0.1', 0), backlog=1) + addr = lsock.getsockname() + + message = b'test data' + response = None + expected_response = b'roger' + + def client(): + nonlocal response + try: + csock = socket.socket() + if client_ssl is not None: + csock = client_ssl.wrap_socket(csock) + csock.connect(addr) + csock.sendall(message) + response = csock.recv(99) + csock.close() + except Exception as exc: + print( + "Failure in client thread in test_connect_accepted_socket", + exc) + + thread = threading.Thread(target=client, daemon=True) + thread.start() + + conn, _ = lsock.accept() + proto = MyProto(loop=loop) + proto.loop = loop + loop.run_until_complete( + loop.connect_accepted_socket( + (lambda: proto), conn, ssl=server_ssl)) + loop.run_forever() + proto.transport.close() + lsock.close() + + threading_helper.join_thread(thread) + self.assertFalse(thread.is_alive()) + self.assertEqual(proto.state, 'CLOSED') + self.assertEqual(proto.nbytes, len(message)) + self.assertEqual(response, expected_response) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_ssl_connect_accepted_socket(self): + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + self.test_connect_accepted_socket(server_context, client_context) + + def test_connect_accepted_socket_ssl_timeout_for_plain_socket(self): + sock = socket.socket() + self.addCleanup(sock.close) + coro = self.loop.connect_accepted_socket( + MyProto, sock, ssl_handshake_timeout=support.LOOPBACK_TIMEOUT) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + @mock.patch('asyncio.base_events.socket') + def create_server_multiple_hosts(self, family, hosts, mock_sock): + async def getaddrinfo(host, port, *args, **kw): + if family == socket.AF_INET: + return [(family, socket.SOCK_STREAM, 6, '', (host, port))] + else: + return [(family, socket.SOCK_STREAM, 6, '', (host, port, 0, 0))] + + def getaddrinfo_task(*args, **kwds): + return self.loop.create_task(getaddrinfo(*args, **kwds)) + + unique_hosts = set(hosts) + + if family == socket.AF_INET: + mock_sock.socket().getsockbyname.side_effect = [ + (host, 80) for host in unique_hosts] + else: + mock_sock.socket().getsockbyname.side_effect = [ + (host, 80, 0, 0) for host in unique_hosts] + self.loop.getaddrinfo = getaddrinfo_task + self.loop._start_serving = mock.Mock() + self.loop._stop_serving = mock.Mock() + f = self.loop.create_server(lambda: MyProto(self.loop), hosts, 80) + server = self.loop.run_until_complete(f) + self.addCleanup(server.close) + server_hosts = {sock.getsockbyname()[0] for sock in server.sockets} + self.assertEqual(server_hosts, unique_hosts) + + def test_create_server_multiple_hosts_ipv4(self): + self.create_server_multiple_hosts(socket.AF_INET, + ['1.2.3.4', '5.6.7.8', '1.2.3.4']) + + def test_create_server_multiple_hosts_ipv6(self): + self.create_server_multiple_hosts(socket.AF_INET6, + ['::1', '::2', '::1']) + + def test_create_server(self): + proto = MyProto(self.loop) + f = self.loop.create_server(lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.sendall(b'xxx') + + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('sockname')) + self.assertEqual('127.0.0.1', + proto.transport.get_extra_info('peername')[0]) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # close server + server.close() + + def test_create_server_trsock(self): + proto = MyProto(self.loop) + f = self.loop.create_server(lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertIsInstance(sock, asyncio.trsock.TransportSocket) + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + dup = sock.dup() + self.addCleanup(dup.close) + self.assertIsInstance(dup, socket.socket) + self.assertFalse(sock.get_inheritable()) + with self.assertRaises(ValueError): + sock.settimeout(1) + sock.settimeout(0) + self.assertEqual(sock.gettimeout(), 0) + with self.assertRaises(ValueError): + sock.setblocking(True) + sock.setblocking(False) + server.close() + + + @unittest.skipUnless(hasattr(socket, 'SO_REUSEPORT'), 'No SO_REUSEPORT') + def test_create_server_reuse_port(self): + proto = MyProto(self.loop) + f = self.loop.create_server( + lambda: proto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertFalse( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + server.close() + + test_utils.run_briefly(self.loop) + + proto = MyProto(self.loop) + f = self.loop.create_server( + lambda: proto, '0.0.0.0', 0, reuse_port=True) + server = self.loop.run_until_complete(f) + self.assertEqual(len(server.sockets), 1) + sock = server.sockets[0] + self.assertTrue( + sock.getsockopt( + socket.SOL_SOCKET, socket.SO_REUSEPORT)) + server.close() + + def _make_unix_server(self, factory, **kwargs): + path = test_utils.gen_unix_socket_path() + self.addCleanup(lambda: os.path.exists(path) and os.unlink(path)) + + f = self.loop.create_unix_server(factory, path, **kwargs) + server = self.loop.run_until_complete(f) + + return server, path + + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_server(self): + proto = MyProto(loop=self.loop) + server, path = self._make_unix_server(lambda: proto) + self.assertEqual(len(server.sockets), 1) + + client = socket.socket(socket.AF_UNIX) + client.connect(path) + client.sendall(b'xxx') + + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # close server + server.close() + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + def test_create_unix_server_path_socket_error(self): + proto = MyProto(loop=self.loop) + sock = socket.socket() + with sock: + f = self.loop.create_unix_server(lambda: proto, '/test', sock=sock) + with self.assertRaisesRegex(ValueError, + 'path and sock can not be specified ' + 'at the same time'): + self.loop.run_until_complete(f) + + def _create_ssl_context(self, certfile, keyfile=None): + sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.load_cert_chain(certfile, keyfile) + return sslcontext + + def _make_ssl_server(self, factory, certfile, keyfile=None): + sslcontext = self._create_ssl_context(certfile, keyfile) + + f = self.loop.create_server(factory, '127.0.0.1', 0, ssl=sslcontext) + server = self.loop.run_until_complete(f) + + sock = server.sockets[0] + host, port = sock.getsockname() + self.assertEqual(host, '127.0.0.1') + return server, host, port + + def _make_ssl_unix_server(self, factory, certfile, keyfile=None): + sslcontext = self._create_ssl_context(certfile, keyfile) + return self._make_unix_server(factory, ssl=sslcontext) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, test_utils.ONLYCERT, test_utils.ONLYKEY) + + f_c = self.loop.create_connection(MyBaseProto, host, port, + ssl=test_utils.dummy_ssl_context()) + client, pr = self.loop.run_until_complete(f_c) + + client.write(b'xxx') + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # extra info is available + self.check_ssl_extra_info(client, peername=(host, port)) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # stop serving + server.close() + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_unix_server_ssl(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, test_utils.ONLYCERT, test_utils.ONLYKEY) + + f_c = self.loop.create_unix_connection( + MyBaseProto, path, ssl=test_utils.dummy_ssl_context(), + server_hostname='') + + client, pr = self.loop.run_until_complete(f_c) + + client.write(b'xxx') + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + test_utils.run_until(self.loop, lambda: proto.nbytes > 0) + self.assertEqual(3, proto.nbytes) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + # the client socket must be closed after to avoid ECONNRESET upon + # recv()/send() on the serving socket + client.close() + + # stop serving + server.close() + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_verify_failed(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + + # no CA loaded + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + '(?i)certificate.verify.failed'): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) + + # close connection + self.assertIsNone(proto.transport) + server.close() + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_unix_server_ssl_verify_failed(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # no CA loaded + f_c = self.loop.create_unix_connection(MyProto, path, + ssl=sslcontext_client, + server_hostname='invalid') + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.SSLError, + '(?i)certificate.verify.failed'): + self.loop.run_until_complete(f_c) + + # execute the loop to log the connection error + test_utils.run_briefly(self.loop) + + # close connection + self.assertIsNone(proto.transport) + server.close() + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_match_failed(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations( + cafile=test_utils.SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # incorrect server_hostname + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with mock.patch.object(self.loop, 'call_exception_handler'): + with test_utils.disable_logger(): + with self.assertRaisesRegex(ssl.CertificateError, regex): + self.loop.run_until_complete(f_c) + + # close connection + # transport is None because TLS ALERT aborted the handshake + self.assertIsNone(proto.transport) + server.close() + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_unix_server_ssl_verified(self): + proto = MyProto(loop=self.loop) + server, path = self._make_ssl_unix_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations(cafile=test_utils.SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # Connection succeeds with correct CA and server hostname. + f_c = self.loop.create_unix_connection(MyProto, path, + ssl=sslcontext_client, + server_hostname='localhost') + client, pr = self.loop.run_until_complete(f_c) + self.loop.run_until_complete(proto.connected) + + # close connection + proto.transport.close() + client.close() + server.close() + self.loop.run_until_complete(proto.done) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - SSL peer certificate format differs + @unittest.skipIf(ssl is None, 'No ssl module') + def test_create_server_ssl_verified(self): + proto = MyProto(loop=self.loop) + server, host, port = self._make_ssl_server( + lambda: proto, test_utils.SIGNED_CERTFILE) + + sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslcontext_client.options |= ssl.OP_NO_SSLv2 + sslcontext_client.verify_mode = ssl.CERT_REQUIRED + sslcontext_client.load_verify_locations(cafile=test_utils.SIGNING_CA) + if hasattr(sslcontext_client, 'check_hostname'): + sslcontext_client.check_hostname = True + + # Connection succeeds with correct CA and server hostname. + f_c = self.loop.create_connection(MyProto, host, port, + ssl=sslcontext_client, + server_hostname='localhost') + client, pr = self.loop.run_until_complete(f_c) + self.loop.run_until_complete(proto.connected) + + # extra info is available + self.check_ssl_extra_info(client, peername=(host, port), + peercert=test_utils.PEERCERT) + + # close connection + proto.transport.close() + client.close() + server.close() + self.loop.run_until_complete(proto.done) + + def test_create_server_sock(self): + proto = self.loop.create_future() + + class TestMyProto(MyProto): + def connection_made(self, transport): + super().connection_made(transport) + proto.set_result(self) + + sock_ob = socket.create_server(('0.0.0.0', 0)) + + f = self.loop.create_server(TestMyProto, sock=sock_ob) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + self.assertEqual(sock.fileno(), sock_ob.fileno()) + + host, port = sock.getsockname() + self.assertEqual(host, '0.0.0.0') + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + client.close() + server.close() + + def test_create_server_addr_in_use(self): + sock_ob = socket.create_server(('0.0.0.0', 0)) + + f = self.loop.create_server(MyProto, sock=sock_ob) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + host, port = sock.getsockname() + + f = self.loop.create_server(MyProto, host=host, port=port) + with self.assertRaises(OSError) as cm: + self.loop.run_until_complete(f) + self.assertEqual(cm.exception.errno, errno.EADDRINUSE) + + server.close() + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_server_dual_stack(self): + f_proto = self.loop.create_future() + + class TestMyProto(MyProto): + def connection_made(self, transport): + super().connection_made(transport) + f_proto.set_result(self) + + try_count = 0 + while True: + try: + port = socket_helper.find_unused_port() + f = self.loop.create_server(TestMyProto, host=None, port=port) + server = self.loop.run_until_complete(f) + except OSError as ex: + if ex.errno == errno.EADDRINUSE: + try_count += 1 + self.assertGreaterEqual(5, try_count) + continue + else: + raise + else: + break + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + proto = self.loop.run_until_complete(f_proto) + proto.transport.close() + client.close() + + f_proto = self.loop.create_future() + client = socket.socket(socket.AF_INET6) + client.connect(('::1', port)) + client.send(b'xxx') + proto = self.loop.run_until_complete(f_proto) + proto.transport.close() + client.close() + + server.close() + + @socket_helper.skip_if_tcp_blackhole + def test_server_close(self): + f = self.loop.create_server(MyProto, '0.0.0.0', 0) + server = self.loop.run_until_complete(f) + sock = server.sockets[0] + host, port = sock.getsockname() + + client = socket.socket() + client.connect(('127.0.0.1', port)) + client.send(b'xxx') + client.close() + + server.close() + + client = socket.socket() + self.assertRaises( + ConnectionRefusedError, client.connect, ('127.0.0.1', port)) + client.close() + + def _test_create_datagram_endpoint(self, local_addr, family): + class TestMyDatagramProto(MyDatagramProto): + def __init__(inner_self): + super().__init__(loop=self.loop) + + def datagram_received(self, data, addr): + super().datagram_received(data, addr) + self.transport.sendto(b'resp:'+data, addr) + + coro = self.loop.create_datagram_endpoint( + TestMyDatagramProto, local_addr=local_addr, family=family) + s_transport, server = self.loop.run_until_complete(coro) + sockname = s_transport.get_extra_info('sockname') + host, port = socket.getnameinfo( + sockname, socket.NI_NUMERICHOST|socket.NI_NUMERICSERV) + + self.assertIsInstance(s_transport, asyncio.Transport) + self.assertIsInstance(server, TestMyDatagramProto) + self.assertEqual('INITIALIZED', server.state) + self.assertIs(server.transport, s_transport) + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + remote_addr=(host, port)) + transport, client = self.loop.run_until_complete(coro) + + self.assertIsInstance(transport, asyncio.Transport) + self.assertIsInstance(client, MyDatagramProto) + self.assertEqual('INITIALIZED', client.state) + self.assertIs(client.transport, transport) + + transport.sendto(b'xxx') + test_utils.run_until(self.loop, lambda: server.nbytes) + self.assertEqual(3, server.nbytes) + test_utils.run_until(self.loop, lambda: client.nbytes) + + # received + self.assertEqual(8, client.nbytes) + + # extra info is available + self.assertIsNotNone(transport.get_extra_info('sockname')) + + # close connection + transport.close() + self.loop.run_until_complete(client.done) + self.assertEqual('CLOSED', client.state) + server.transport.close() + + def test_create_datagram_endpoint(self): + self._test_create_datagram_endpoint(('127.0.0.1', 0), socket.AF_INET) + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 not supported or enabled') + def test_create_datagram_endpoint_ipv6(self): + self._test_create_datagram_endpoint(('::1', 0), socket.AF_INET6) + + def test_create_datagram_endpoint_sock(self): + sock = None + local_address = ('127.0.0.1', 0) + infos = self.loop.run_until_complete( + self.loop.getaddrinfo( + *local_address, type=socket.SOCK_DGRAM)) + for family, type, proto, cname, address in infos: + try: + sock = socket.socket(family=family, type=type, proto=proto) + sock.setblocking(False) + sock.bind(address) + except: + pass + else: + break + else: + self.fail('Can not create socket.') + + f = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), sock=sock) + tr, pr = self.loop.run_until_complete(f) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, MyDatagramProto) + tr.close() + self.loop.run_until_complete(pr.done) + + def test_datagram_send_to_non_listening_address(self): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages after + # sending a message to an address that wasn't listening. + loop = self.loop + + class Protocol(asyncio.DatagramProtocol): + + _received_datagram = None + + def datagram_received(self, data, addr): + self._received_datagram.set_result(data) + + async def wait_for_datagram_received(self): + self._received_datagram = loop.create_future() + result = await asyncio.wait_for(self._received_datagram, 10) + self._received_datagram = None + return result + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + transport_1, protocol_1 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_1) + ) + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + transport_2, protocol_2 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_2) + ) + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address that + # is not listening + socket_3 = create_socket() + transport_3, protocol_3 = loop.run_until_complete( + loop.create_datagram_endpoint(Protocol, sock=socket_3) + ) + addr_3 = socket_3.getsockname() + transport_3.abort() + + transport_1.sendto(b'a', addr=addr_2) + self.assertEqual(loop.run_until_complete( + protocol_2.wait_for_datagram_received() + ), b'a') + + transport_2.sendto(b'b', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'b') + + # this should send to an address that isn't listening + transport_1.sendto(b'c', addr=addr_3) + loop.run_until_complete(asyncio.sleep(0)) + + # transport 1 should still be able to receive messages after sending to + # an address that wasn't listening + transport_2.sendto(b'd', addr=addr_1) + self.assertEqual(loop.run_until_complete( + protocol_1.wait_for_datagram_received() + ), b'd') + + transport_1.close() + transport_2.close() + + def test_internal_fds(self): + loop = self.create_event_loop() + if not isinstance(loop, selector_events.BaseSelectorEventLoop): + loop.close() + self.skipTest('loop is not a BaseSelectorEventLoop') + + self.assertEqual(1, loop._internal_fds) + loop.close() + self.assertEqual(0, loop._internal_fds) + self.assertIsNone(loop._csock) + self.assertIsNone(loop._ssock) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_read_pipe(self): + proto = MyReadPipeProto(loop=self.loop) + + rpipe, wpipe = os.pipe() + pipeobj = io.open(rpipe, 'rb', 1024) + + async def connect(): + t, p = await self.loop.connect_read_pipe( + lambda: proto, pipeobj) + self.assertIs(p, proto) + self.assertIs(t, proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(0, proto.nbytes) + + self.loop.run_until_complete(connect()) + + os.write(wpipe, b'1') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 1) + self.assertEqual(1, proto.nbytes) + + os.write(wpipe, b'2345') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 5) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(5, proto.nbytes) + + os.close(wpipe) + self.loop.run_until_complete(proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_unclosed_pipe_transport(self): + # This test reproduces the issue #314 on GitHub + loop = self.create_event_loop() + read_proto = MyReadPipeProto(loop=loop) + write_proto = MyWritePipeProto(loop=loop) + + rpipe, wpipe = os.pipe() + rpipeobj = io.open(rpipe, 'rb', 1024) + wpipeobj = io.open(wpipe, 'w', 1024, encoding="utf-8") + + async def connect(): + read_transport, _ = await loop.connect_read_pipe( + lambda: read_proto, rpipeobj) + write_transport, _ = await loop.connect_write_pipe( + lambda: write_proto, wpipeobj) + return read_transport, write_transport + + # Run and close the loop without closing the transports + read_transport, write_transport = loop.run_until_complete(connect()) + loop.close() + + # These 'repr' calls used to raise an AttributeError + # See Issue #314 on GitHub + self.assertIn('open', repr(read_transport)) + self.assertIn('open', repr(write_transport)) + + # Clean up (avoid ResourceWarning) + rpipeobj.close() + wpipeobj.close() + read_transport._pipe = None + write_transport._pipe = None + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + @unittest.skipUnless(hasattr(os, 'openpty'), 'need os.openpty()') + def test_read_pty_output(self): + proto = MyReadPipeProto(loop=self.loop) + + master, slave = os.openpty() + master_read_obj = io.open(master, 'rb', 0) + + async def connect(): + t, p = await self.loop.connect_read_pipe(lambda: proto, + master_read_obj) + self.assertIs(p, proto) + self.assertIs(t, proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(0, proto.nbytes) + + self.loop.run_until_complete(connect()) + + os.write(slave, b'1') + test_utils.run_until(self.loop, lambda: proto.nbytes) + self.assertEqual(1, proto.nbytes) + + os.write(slave, b'2345') + test_utils.run_until(self.loop, lambda: proto.nbytes >= 5) + self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) + self.assertEqual(5, proto.nbytes) + + os.close(slave) + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state) + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_write_pipe(self): + rpipe, wpipe = os.pipe() + pipeobj = io.open(wpipe, 'wb', 1024) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, pipeobj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + + data = bytearray() + def reader(data): + chunk = os.read(rpipe, 1024) + data += chunk + return len(data) + + test_utils.run_until(self.loop, lambda: reader(data) >= 1) + self.assertEqual(b'1', data) + + transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5) + self.assertEqual(b'12345', data) + self.assertEqual('CONNECTED', proto.state) + + os.close(rpipe) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + def test_write_pipe_disconnect_on_close(self): + rsock, wsock = socket.socketpair() + rsock.setblocking(False) + pipeobj = io.open(wsock.detach(), 'wb', 1024) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, pipeobj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + data = self.loop.run_until_complete(self.loop.sock_recv(rsock, 1024)) + self.assertEqual(b'1', data) + + rsock.close() + + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + @unittest.skipUnless(hasattr(os, 'openpty'), 'need os.openpty()') + # select, poll and kqueue don't support character devices (PTY) on Mac OS X + # older than 10.6 (Snow Leopard) + @support.requires_mac_ver(10, 6) + def test_write_pty(self): + master, slave = os.openpty() + slave_write_obj = io.open(slave, 'wb', 0) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, slave_write_obj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertIs(transport, proto.transport) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + + data = bytearray() + def reader(data): + chunk = os.read(master, 1024) + data += chunk + return len(data) + + test_utils.run_until(self.loop, lambda: reader(data) >= 1, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(b'1', data) + + transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(b'12345', data) + self.assertEqual('CONNECTED', proto.state) + + os.close(master) + + # extra info is available + self.assertIsNotNone(proto.transport.get_extra_info('pipe')) + + # close connection + proto.transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + @unittest.skipUnless(hasattr(os, 'openpty'), 'need os.openpty()') + # select, poll and kqueue don't support character devices (PTY) on Mac OS X + # older than 10.6 (Snow Leopard) + @support.requires_mac_ver(10, 6) + def test_bidirectional_pty(self): + master, read_slave = os.openpty() + write_slave = os.dup(read_slave) + tty.setraw(read_slave) + + slave_read_obj = io.open(read_slave, 'rb', 0) + read_proto = MyReadPipeProto(loop=self.loop) + read_connect = self.loop.connect_read_pipe(lambda: read_proto, + slave_read_obj) + read_transport, p = self.loop.run_until_complete(read_connect) + self.assertIs(p, read_proto) + self.assertIs(read_transport, read_proto.transport) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(0, read_proto.nbytes) + + + slave_write_obj = io.open(write_slave, 'wb', 0) + write_proto = MyWritePipeProto(loop=self.loop) + write_connect = self.loop.connect_write_pipe(lambda: write_proto, + slave_write_obj) + write_transport, p = self.loop.run_until_complete(write_connect) + self.assertIs(p, write_proto) + self.assertIs(write_transport, write_proto.transport) + self.assertEqual('CONNECTED', write_proto.state) + + data = bytearray() + def reader(data): + chunk = os.read(master, 1024) + data += chunk + return len(data) + + write_transport.write(b'1') + test_utils.run_until(self.loop, lambda: reader(data) >= 1, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(b'1', data) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual('CONNECTED', write_proto.state) + + os.write(master, b'a') + test_utils.run_until(self.loop, lambda: read_proto.nbytes >= 1, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(1, read_proto.nbytes) + self.assertEqual('CONNECTED', write_proto.state) + + write_transport.write(b'2345') + test_utils.run_until(self.loop, lambda: reader(data) >= 5, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(b'12345', data) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual('CONNECTED', write_proto.state) + + os.write(master, b'bcde') + test_utils.run_until(self.loop, lambda: read_proto.nbytes >= 5, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(['INITIAL', 'CONNECTED'], read_proto.state) + self.assertEqual(5, read_proto.nbytes) + self.assertEqual('CONNECTED', write_proto.state) + + os.close(master) + + read_transport.close() + self.loop.run_until_complete(read_proto.done) + self.assertEqual( + ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], read_proto.state) + + write_transport.close() + self.loop.run_until_complete(write_proto.done) + self.assertEqual('CLOSED', write_proto.state) + + def test_prompt_cancellation(self): + r, w = socket.socketpair() + r.setblocking(False) + f = self.loop.create_task(self.loop.sock_recv(r, 1)) + ov = getattr(f, 'ov', None) + if ov is not None: + self.assertTrue(ov.pending) + + async def main(): + try: + self.loop.call_soon(f.cancel) + await f + except asyncio.CancelledError: + res = 'cancelled' + else: + res = None + finally: + self.loop.stop() + return res + + t = self.loop.create_task(main()) + self.loop.run_forever() + + self.assertEqual(t.result(), 'cancelled') + self.assertRaises(asyncio.CancelledError, f.result) + if ov is not None: + self.assertFalse(ov.pending) + self.loop._stop_serving(r) + + r.close() + w.close() + + def test_timeout_rounding(self): + def _run_once(): + self.loop._run_once_counter += 1 + orig_run_once() + + orig_run_once = self.loop._run_once + self.loop._run_once_counter = 0 + self.loop._run_once = _run_once + + async def wait(): + await asyncio.sleep(1e-2) + await asyncio.sleep(1e-4) + await asyncio.sleep(1e-6) + await asyncio.sleep(1e-8) + await asyncio.sleep(1e-10) + + self.loop.run_until_complete(wait()) + # The ideal number of call is 12, but on some platforms, the selector + # may sleep at little bit less than timeout depending on the resolution + # of the clock used by the kernel. Tolerate a few useless calls on + # these platforms. + self.assertLessEqual(self.loop._run_once_counter, 20, + {'clock_resolution': self.loop._clock_resolution, + 'selector': self.loop._selector.__class__.__name__}) + + def test_remove_fds_after_closing(self): + loop = self.create_event_loop() + callback = lambda: None + r, w = socket.socketpair() + self.addCleanup(r.close) + self.addCleanup(w.close) + loop.add_reader(r, callback) + loop.add_writer(w, callback) + loop.close() + self.assertFalse(loop.remove_reader(r)) + self.assertFalse(loop.remove_writer(w)) + + def test_add_fds_after_closing(self): + loop = self.create_event_loop() + callback = lambda: None + r, w = socket.socketpair() + self.addCleanup(r.close) + self.addCleanup(w.close) + loop.close() + with self.assertRaises(RuntimeError): + loop.add_reader(r, callback) + with self.assertRaises(RuntimeError): + loop.add_writer(w, callback) + + def test_close_running_event_loop(self): + async def close_loop(loop): + self.loop.close() + + coro = close_loop(self.loop) + with self.assertRaises(RuntimeError): + self.loop.run_until_complete(coro) + + def test_close(self): + self.loop.close() + + async def test(): + pass + + func = lambda: False + coro = test() + self.addCleanup(coro.close) + + # operation blocked when the loop is closed + with self.assertRaises(RuntimeError): + self.loop.run_forever() + with self.assertRaises(RuntimeError): + fut = self.loop.create_future() + self.loop.run_until_complete(fut) + with self.assertRaises(RuntimeError): + self.loop.call_soon(func) + with self.assertRaises(RuntimeError): + self.loop.call_soon_threadsafe(func) + with self.assertRaises(RuntimeError): + self.loop.call_later(1.0, func) + with self.assertRaises(RuntimeError): + self.loop.call_at(self.loop.time() + .0, func) + with self.assertRaises(RuntimeError): + self.loop.create_task(coro) + with self.assertRaises(RuntimeError): + self.loop.add_signal_handler(signal.SIGTERM, func) + + # run_in_executor test is tricky: the method is a coroutine, + # but run_until_complete cannot be called on closed loop. + # Thus iterate once explicitly. + with self.assertRaises(RuntimeError): + it = self.loop.run_in_executor(None, func).__await__() + next(it) + + +class SubprocessTestsMixin: + + def check_terminated(self, returncode): + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGTERM, returncode) + + def check_killed(self, returncode): + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + + @support.requires_subprocess() + def test_subprocess_exec(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'Python The Winner') + self.loop.run_until_complete(proto.got_data[1].wait()) + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + self.assertEqual(b'Python The Winner', proto.data[1]) + + @support.requires_subprocess() + def test_subprocess_interactive(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + self.assertEqual('CONNECTED', proto.state) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'Python ') + self.loop.run_until_complete(proto.got_data[1].wait()) + proto.got_data[1].clear() + self.assertEqual(b'Python ', proto.data[1]) + + stdin.write(b'The Winner') + self.loop.run_until_complete(proto.got_data[1].wait()) + self.assertEqual(b'Python The Winner', proto.data[1]) + + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + + @support.requires_subprocess() + def test_subprocess_shell(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'echo Python') + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.get_pipe_transport(0).close() + self.loop.run_until_complete(proto.completed) + self.assertEqual(0, proto.returncode) + self.assertTrue(all(f.done() for f in proto.disconnects.values())) + self.assertEqual(proto.data[1].rstrip(b'\r\n'), b'Python') + self.assertEqual(proto.data[2], b'') + transp.close() + + @support.requires_subprocess() + def test_subprocess_exitcode(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + transp.close() + + @support.requires_subprocess() + def test_subprocess_close_after_finish(self): + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.assertIsNone(transp.get_pipe_transport(0)) + self.assertIsNone(transp.get_pipe_transport(1)) + self.assertIsNone(transp.get_pipe_transport(2)) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + self.assertIsNone(transp.close()) + + @support.requires_subprocess() + def test_subprocess_kill(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.kill() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + transp.close() + + @support.requires_subprocess() + def test_subprocess_terminate(self): + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.terminate() + self.loop.run_until_complete(proto.completed) + self.check_terminated(proto.returncode) + transp.close() + + @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + @support.requires_subprocess() + def test_subprocess_send_signal(self): + # bpo-31034: Make sure that we get the default signal handler (killing + # the process). The parent process may have decided to ignore SIGHUP, + # and signal handlers are inherited. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + prog = os.path.join(os.path.dirname(__file__), 'echo.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + transp.send_signal(signal.SIGHUP) + self.loop.run_until_complete(proto.completed) + self.assertEqual(-signal.SIGHUP, proto.returncode) + transp.close() + finally: + signal.signal(signal.SIGHUP, old_handler) + + @support.requires_subprocess() + def test_subprocess_stderr(self): + prog = os.path.join(os.path.dirname(__file__), 'echo2.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + stdin.write(b'test') + + self.loop.run_until_complete(proto.completed) + + transp.close() + self.assertEqual(b'OUT:test', proto.data[1]) + self.assertStartsWith(proto.data[2], b'ERR:test') + self.assertEqual(0, proto.returncode) + + @support.requires_subprocess() + def test_subprocess_stderr_redirect_to_stdout(self): + prog = os.path.join(os.path.dirname(__file__), 'echo2.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog, stderr=subprocess.STDOUT) + + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + self.assertIsNotNone(transp.get_pipe_transport(1)) + self.assertIsNone(transp.get_pipe_transport(2)) + + stdin.write(b'test') + self.loop.run_until_complete(proto.completed) + self.assertStartsWith(proto.data[1], b'OUT:testERR:test') + self.assertEqual(b'', proto.data[2]) + + transp.close() + self.assertEqual(0, proto.returncode) + + @support.requires_subprocess() + def test_subprocess_close_client_stream(self): + prog = os.path.join(os.path.dirname(__file__), 'echo3.py') + + connect = self.loop.subprocess_exec( + functools.partial(MySubprocessProtocol, self.loop), + sys.executable, prog) + + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.connected) + + stdin = transp.get_pipe_transport(0) + stdout = transp.get_pipe_transport(1) + stdin.write(b'test') + self.loop.run_until_complete(proto.got_data[1].wait()) + self.assertEqual(b'OUT:test', proto.data[1]) + + stdout.close() + self.loop.run_until_complete(proto.disconnects[1]) + stdin.write(b'xxx') + self.loop.run_until_complete(proto.got_data[2].wait()) + if sys.platform != 'win32': + self.assertEqual(b'ERR:BrokenPipeError', proto.data[2]) + else: + # After closing the read-end of a pipe, writing to the + # write-end using os.write() fails with errno==EINVAL and + # GetLastError()==ERROR_INVALID_NAME on Windows!?! (Using + # WriteFile() we get ERROR_BROKEN_PIPE as expected.) + self.assertEqual(b'ERR:OSError', proto.data[2]) + with test_utils.disable_logger(): + transp.close() + self.loop.run_until_complete(proto.completed) + self.check_killed(proto.returncode) + + @support.requires_subprocess() + def test_subprocess_wait_no_same_group(self): + # start the new process in a new session + connect = self.loop.subprocess_shell( + functools.partial(MySubprocessProtocol, self.loop), + 'exit 7', stdin=None, stdout=None, stderr=None, + start_new_session=True) + transp, proto = self.loop.run_until_complete(connect) + self.assertIsInstance(proto, MySubprocessProtocol) + self.loop.run_until_complete(proto.completed) + self.assertEqual(7, proto.returncode) + transp.close() + + @support.requires_subprocess() + def test_subprocess_exec_invalid_args(self): + async def connect(**kwds): + await self.loop.subprocess_exec( + asyncio.SubprocessProtocol, + 'pwd', **kwds) + + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(universal_newlines=True)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(bufsize=4096)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(shell=True)) + + @support.requires_subprocess() + def test_subprocess_shell_invalid_args(self): + + async def connect(cmd=None, **kwds): + if not cmd: + cmd = 'pwd' + await self.loop.subprocess_shell( + asyncio.SubprocessProtocol, + cmd, **kwds) + + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(['ls', '-l'])) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(universal_newlines=True)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(bufsize=4096)) + with self.assertRaises(ValueError): + self.loop.run_until_complete(connect(shell=False)) + + +if sys.platform == 'win32': + + class SelectEventLoopTests(EventLoopTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop() + + class ProactorEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.ProactorEventLoop() + + def test_reader_callback(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") + + def test_reader_callback_cancel(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") + + def test_writer_callback(self): + raise unittest.SkipTest("IocpEventLoop does not have add_writer()") + + def test_writer_callback_cancel(self): + raise unittest.SkipTest("IocpEventLoop does not have add_writer()") + + def test_remove_fds_after_closing(self): + raise unittest.SkipTest("IocpEventLoop does not have add_reader()") +else: + import selectors + + if hasattr(selectors, 'KqueueSelector'): + class KqueueEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop( + selectors.KqueueSelector()) + + # kqueue doesn't support character devices (PTY) on Mac OS X older + # than 10.9 (Maverick) + @support.requires_mac_ver(10, 9) + # Issue #20667: KqueueEventLoopTests.test_read_pty_output() + # hangs on OpenBSD 5.5 + @unittest.skipIf(sys.platform.startswith('openbsd'), + 'test hangs on OpenBSD') + def test_read_pty_output(self): + super().test_read_pty_output() + + # kqueue doesn't support character devices (PTY) on Mac OS X older + # than 10.9 (Maverick) + @support.requires_mac_ver(10, 9) + def test_write_pty(self): + super().test_write_pty() + + if hasattr(selectors, 'EpollSelector'): + class EPollEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.EpollSelector()) + + if hasattr(selectors, 'PollSelector'): + class PollEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.PollSelector()) + + # Should always exist. + class SelectEventLoopTests(EventLoopTestsMixin, + SubprocessTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +def noop(*args, **kwargs): + pass + + +class HandleTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = mock.Mock() + self.loop.get_debug.return_value = True + + def test_handle(self): + def callback(*args): + return args + + args = () + h = asyncio.Handle(callback, args, self.loop) + self.assertIs(h._callback, callback) + self.assertIs(h._args, args) + self.assertFalse(h.cancelled()) + + h.cancel() + self.assertTrue(h.cancelled()) + + def test_callback_with_exception(self): + def callback(): + raise ValueError() + + self.loop = mock.Mock() + self.loop.call_exception_handler = mock.Mock() + + h = asyncio.Handle(callback, (), self.loop) + h._run() + + self.loop.call_exception_handler.assert_called_with({ + 'message': test_utils.MockPattern('Exception in callback.*'), + 'exception': mock.ANY, + 'handle': h, + 'source_traceback': h._source_traceback, + }) + + def test_handle_weakref(self): + wd = weakref.WeakValueDictionary() + h = asyncio.Handle(lambda: None, (), self.loop) + wd['h'] = h # Would fail without __weakref__ slot. + + def test_handle_repr(self): + self.loop.get_debug.return_value = False + + # simple function + h = asyncio.Handle(noop, (1, 2), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno)) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '') + + # decorated function + cb = types.coroutine(noop) + h = asyncio.Handle(cb, (), self.loop) + self.assertEqual(repr(h), + '' + % (filename, lineno)) + + # partial function + cb = functools.partial(noop, 1, 2) + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno)) + self.assertRegex(repr(h), regex) + + # partial method + method = HandleTests.test_handle_repr + cb = functools.partialmethod(method) + filename, lineno = test_utils.get_function_source(method) + h = asyncio.Handle(cb, (), self.loop) + + cb_regex = r'' + cb_regex = fr'functools.partialmethod\({cb_regex}\)\(\)' + regex = fr'^$' + self.assertRegex(repr(h), regex) + + def test_handle_repr_debug(self): + self.loop.get_debug.return_value = True + + # simple function + create_filename = __file__ + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(noop, (1, 2), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # cancelled handle + h.cancel() + self.assertEqual( + repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # double cancellation won't overwrite _repr + h.cancel() + self.assertEqual( + repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # partial function + cb = functools.partial(noop, 1, 2) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (3,), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + # partial function with keyword args + cb = functools.partial(noop, x=1) + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.Handle(cb, (2, 3), self.loop) + regex = (r'^$' + % (re.escape(filename), lineno, + re.escape(create_filename), create_lineno)) + self.assertRegex(repr(h), regex) + + def test_handle_source_traceback(self): + loop = asyncio.new_event_loop() + loop.set_debug(True) + self.set_event_loop(loop) + + def check_source_traceback(h): + lineno = sys._getframe(1).f_lineno - 1 + self.assertIsInstance(h._source_traceback, list) + self.assertEqual(h._source_traceback[-1][:3], + (__file__, + lineno, + 'test_handle_source_traceback')) + + # call_soon + h = loop.call_soon(noop) + check_source_traceback(h) + + # call_soon_threadsafe + h = loop.call_soon_threadsafe(noop) + check_source_traceback(h) + + # call_later + h = loop.call_later(0, noop) + check_source_traceback(h) + + # call_at + h = loop.call_later(0, noop) + check_source_traceback(h) + + def test_coroutine_like_object_debug_formatting(self): + # Test that asyncio can format coroutines that are instances of + # collections.abc.Coroutine, but lack cr_core or gi_code attributes + # (such as ones compiled with Cython). + + coro = CoroLike() + coro.__name__ = 'AAA' + self.assertTrue(asyncio.iscoroutine(coro)) + self.assertEqual(coroutines._format_coroutine(coro), 'AAA()') + + coro.__qualname__ = 'BBB' + self.assertEqual(coroutines._format_coroutine(coro), 'BBB()') + + coro.cr_running = True + self.assertEqual(coroutines._format_coroutine(coro), 'BBB() running') + + coro.__name__ = coro.__qualname__ = None + self.assertEqual(coroutines._format_coroutine(coro), + '() running') + + coro = CoroLike() + coro.__qualname__ = 'CoroLike' + # Some coroutines might not have '__name__', such as + # built-in async_gen.asend(). + self.assertEqual(coroutines._format_coroutine(coro), 'CoroLike()') + + coro = CoroLike() + coro.__qualname__ = 'AAA' + coro.cr_code = None + self.assertEqual(coroutines._format_coroutine(coro), 'AAA()') + + +class TimerTests(unittest.TestCase): + + def setUp(self): + super().setUp() + self.loop = mock.Mock() + + def test_hash(self): + when = time.monotonic() + h = asyncio.TimerHandle(when, lambda: False, (), + mock.Mock()) + self.assertEqual(hash(h), hash(when)) + + def test_when(self): + when = time.monotonic() + h = asyncio.TimerHandle(when, lambda: False, (), + mock.Mock()) + self.assertEqual(when, h.when()) + + def test_timer(self): + def callback(*args): + return args + + args = (1, 2, 3) + when = time.monotonic() + h = asyncio.TimerHandle(when, callback, args, mock.Mock()) + self.assertIs(h._callback, callback) + self.assertIs(h._args, args) + self.assertFalse(h.cancelled()) + + # cancel + h.cancel() + self.assertTrue(h.cancelled()) + self.assertIsNone(h._callback) + self.assertIsNone(h._args) + + + def test_timer_repr(self): + self.loop.get_debug.return_value = False + + # simple function + h = asyncio.TimerHandle(123, noop, (), self.loop) + src = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' % src) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '') + + def test_timer_repr_debug(self): + self.loop.get_debug.return_value = True + + # simple function + create_filename = __file__ + create_lineno = sys._getframe().f_lineno + 1 + h = asyncio.TimerHandle(123, noop, (), self.loop) + filename, lineno = test_utils.get_function_source(noop) + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + # cancelled handle + h.cancel() + self.assertEqual(repr(h), + '' + % (filename, lineno, create_filename, create_lineno)) + + + def test_timer_comparison(self): + def callback(*args): + return args + + when = time.monotonic() + + h1 = asyncio.TimerHandle(when, callback, (), self.loop) + h2 = asyncio.TimerHandle(when, callback, (), self.loop) + with self.assertRaises(AssertionError): + self.assertLess(h1, h2) + with self.assertRaises(AssertionError): + self.assertLess(h2, h1) + with self.assertRaises(AssertionError): + self.assertGreater(h1, h2) + with self.assertRaises(AssertionError): + self.assertGreater(h2, h1) + with self.assertRaises(AssertionError): + self.assertNotEqual(h1, h2) + + self.assertLessEqual(h1, h2) + self.assertLessEqual(h2, h1) + self.assertGreaterEqual(h1, h2) + self.assertGreaterEqual(h2, h1) + self.assertEqual(h1, h2) + + h2.cancel() + with self.assertRaises(AssertionError): + self.assertEqual(h1, h2) + self.assertNotEqual(h1, h2) + + h1 = asyncio.TimerHandle(when, callback, (), self.loop) + h2 = asyncio.TimerHandle(when + 10.0, callback, (), self.loop) + with self.assertRaises(AssertionError): + self.assertLess(h2, h1) + with self.assertRaises(AssertionError): + self.assertLessEqual(h2, h1) + with self.assertRaises(AssertionError): + self.assertGreater(h1, h2) + with self.assertRaises(AssertionError): + self.assertGreaterEqual(h1, h2) + with self.assertRaises(AssertionError): + self.assertEqual(h1, h2) + + self.assertLess(h1, h2) + self.assertGreater(h2, h1) + self.assertLessEqual(h1, h2) + self.assertGreaterEqual(h2, h1) + self.assertNotEqual(h1, h2) + + h3 = asyncio.Handle(callback, (), self.loop) + self.assertIs(NotImplemented, h1.__eq__(h3)) + self.assertIs(NotImplemented, h1.__ne__(h3)) + + with self.assertRaises(TypeError): + h1 < () + with self.assertRaises(TypeError): + h1 > () + with self.assertRaises(TypeError): + h1 <= () + with self.assertRaises(TypeError): + h1 >= () + with self.assertRaises(AssertionError): + self.assertEqual(h1, ()) + with self.assertRaises(AssertionError): + self.assertNotEqual(h1, ALWAYS_EQ) + with self.assertRaises(AssertionError): + self.assertGreater(h1, LARGEST) + with self.assertRaises(AssertionError): + self.assertGreaterEqual(h1, LARGEST) + with self.assertRaises(AssertionError): + self.assertLess(h1, SMALLEST) + with self.assertRaises(AssertionError): + self.assertLessEqual(h1, SMALLEST) + + self.assertNotEqual(h1, ()) + self.assertEqual(h1, ALWAYS_EQ) + self.assertLess(h1, LARGEST) + self.assertLessEqual(h1, LARGEST) + self.assertGreaterEqual(h1, SMALLEST) + self.assertGreater(h1, SMALLEST) + + +class AbstractEventLoopTests(unittest.TestCase): + + def test_not_implemented(self): + f = mock.Mock() + loop = asyncio.AbstractEventLoop() + self.assertRaises( + NotImplementedError, loop.run_forever) + self.assertRaises( + NotImplementedError, loop.run_until_complete, None) + self.assertRaises( + NotImplementedError, loop.stop) + self.assertRaises( + NotImplementedError, loop.is_running) + self.assertRaises( + NotImplementedError, loop.is_closed) + self.assertRaises( + NotImplementedError, loop.close) + self.assertRaises( + NotImplementedError, loop.create_task, None) + self.assertRaises( + NotImplementedError, loop.call_later, None, None) + self.assertRaises( + NotImplementedError, loop.call_at, f, f) + self.assertRaises( + NotImplementedError, loop.call_soon, None) + self.assertRaises( + NotImplementedError, loop.time) + self.assertRaises( + NotImplementedError, loop.call_soon_threadsafe, None) + self.assertRaises( + NotImplementedError, loop.set_default_executor, f) + self.assertRaises( + NotImplementedError, loop.add_reader, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_reader, 1) + self.assertRaises( + NotImplementedError, loop.add_writer, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_writer, 1) + self.assertRaises( + NotImplementedError, loop.add_signal_handler, 1, f) + self.assertRaises( + NotImplementedError, loop.remove_signal_handler, 1) + self.assertRaises( + NotImplementedError, loop.remove_signal_handler, 1) + self.assertRaises( + NotImplementedError, loop.set_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.default_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.call_exception_handler, f) + self.assertRaises( + NotImplementedError, loop.get_debug) + self.assertRaises( + NotImplementedError, loop.set_debug, f) + + def test_not_implemented_async(self): + + async def inner(): + f = mock.Mock() + loop = asyncio.AbstractEventLoop() + + with self.assertRaises(NotImplementedError): + await loop.run_in_executor(f, f) + with self.assertRaises(NotImplementedError): + await loop.getaddrinfo('localhost', 8080) + with self.assertRaises(NotImplementedError): + await loop.getnameinfo(('localhost', 8080)) + with self.assertRaises(NotImplementedError): + await loop.create_connection(f) + with self.assertRaises(NotImplementedError): + await loop.create_server(f) + with self.assertRaises(NotImplementedError): + await loop.create_datagram_endpoint(f) + with self.assertRaises(NotImplementedError): + await loop.sock_recv(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_recv_into(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_sendall(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_connect(f, f) + with self.assertRaises(NotImplementedError): + await loop.sock_accept(f) + with self.assertRaises(NotImplementedError): + await loop.sock_sendfile(f, f) + with self.assertRaises(NotImplementedError): + await loop.sendfile(f, f) + with self.assertRaises(NotImplementedError): + await loop.connect_read_pipe(f, mock.sentinel.pipe) + with self.assertRaises(NotImplementedError): + await loop.connect_write_pipe(f, mock.sentinel.pipe) + with self.assertRaises(NotImplementedError): + await loop.subprocess_shell(f, mock.sentinel) + with self.assertRaises(NotImplementedError): + await loop.subprocess_exec(f) + + loop = asyncio.new_event_loop() + loop.run_until_complete(inner()) + loop.close() + + +class PolicyTests(unittest.TestCase): + + def test_abstract_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.AbstractEventLoopPolicy' is deprecated"): + policy = asyncio.AbstractEventLoopPolicy() + self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy) + + def test_default_event_loop_policy_deprecation(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.DefaultEventLoopPolicy' is deprecated"): + policy = asyncio.DefaultEventLoopPolicy() + self.assertIsInstance(policy, asyncio.DefaultEventLoopPolicy) + + def test_event_loop_policy(self): + policy = asyncio.events._AbstractEventLoopPolicy() + self.assertRaises(NotImplementedError, policy.get_event_loop) + self.assertRaises(NotImplementedError, policy.set_event_loop, object()) + self.assertRaises(NotImplementedError, policy.new_event_loop) + + def test_get_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() + self.assertIsNone(policy._local._loop) + + with self.assertRaises(RuntimeError): + loop = policy.get_event_loop() + self.assertIsNone(policy._local._loop) + + def test_get_event_loop_does_not_call_set_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() + + with mock.patch.object( + policy, "set_event_loop", + wraps=policy.set_event_loop) as m_set_event_loop: + + with self.assertRaises(RuntimeError): + loop = policy.get_event_loop() + + m_set_event_loop.assert_not_called() + + def test_get_event_loop_after_set_none(self): + policy = test_utils.DefaultEventLoopPolicy() + policy.set_event_loop(None) + self.assertRaises(RuntimeError, policy.get_event_loop) + + @mock.patch('asyncio.events.threading.current_thread') + def test_get_event_loop_thread(self, m_current_thread): + + def f(): + policy = test_utils.DefaultEventLoopPolicy() + self.assertRaises(RuntimeError, policy.get_event_loop) + + th = threading.Thread(target=f) + th.start() + th.join() + + def test_new_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() + + loop = policy.new_event_loop() + self.assertIsInstance(loop, asyncio.AbstractEventLoop) + loop.close() + + def test_set_event_loop(self): + policy = test_utils.DefaultEventLoopPolicy() + old_loop = policy.new_event_loop() + policy.set_event_loop(old_loop) + + self.assertRaises(TypeError, policy.set_event_loop, object()) + + loop = policy.new_event_loop() + policy.set_event_loop(loop) + self.assertIs(loop, policy.get_event_loop()) + self.assertIsNot(old_loop, policy.get_event_loop()) + loop.close() + old_loop.close() + + def test_get_event_loop_policy(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + policy = asyncio.get_event_loop_policy() + self.assertIsInstance(policy, asyncio.events._AbstractEventLoopPolicy) + self.assertIs(policy, asyncio.get_event_loop_policy()) + + def test_set_event_loop_policy(self): + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): + self.assertRaises( + TypeError, asyncio.set_event_loop_policy, object()) + + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + old_policy = asyncio.get_event_loop_policy() + + policy = test_utils.DefaultEventLoopPolicy() + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): + asyncio.set_event_loop_policy(policy) + + with self.assertWarnsRegex( + DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): + self.assertIs(policy, asyncio.get_event_loop_policy()) + self.assertIsNot(policy, old_policy) + + +class GetEventLoopTestsMixin: + + _get_running_loop_impl = None + _set_running_loop_impl = None + get_running_loop_impl = None + get_event_loop_impl = None + + Task = None + Future = None + + def setUp(self): + self._get_running_loop_saved = events._get_running_loop + self._set_running_loop_saved = events._set_running_loop + self.get_running_loop_saved = events.get_running_loop + self.get_event_loop_saved = events.get_event_loop + self._Task_saved = asyncio.Task + self._Future_saved = asyncio.Future + + events._get_running_loop = type(self)._get_running_loop_impl + events._set_running_loop = type(self)._set_running_loop_impl + events.get_running_loop = type(self).get_running_loop_impl + events.get_event_loop = type(self).get_event_loop_impl + + asyncio._get_running_loop = type(self)._get_running_loop_impl + asyncio._set_running_loop = type(self)._set_running_loop_impl + asyncio.get_running_loop = type(self).get_running_loop_impl + asyncio.get_event_loop = type(self).get_event_loop_impl + + asyncio.Task = asyncio.tasks.Task = type(self).Task + asyncio.Future = asyncio.futures.Future = type(self).Future + super().setUp() + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def tearDown(self): + try: + super().tearDown() + finally: + self.loop.close() + asyncio.set_event_loop(None) + + events._get_running_loop = self._get_running_loop_saved + events._set_running_loop = self._set_running_loop_saved + events.get_running_loop = self.get_running_loop_saved + events.get_event_loop = self.get_event_loop_saved + + asyncio._get_running_loop = self._get_running_loop_saved + asyncio._set_running_loop = self._set_running_loop_saved + asyncio.get_running_loop = self.get_running_loop_saved + asyncio.get_event_loop = self.get_event_loop_saved + + asyncio.Task = asyncio.tasks.Task = self._Task_saved + asyncio.Future = asyncio.futures.Future = self._Future_saved + + if sys.platform != 'win32': + def test_get_event_loop_new_process(self): + # bpo-32126: The multiprocessing module used by + # ProcessPoolExecutor is not functional when the + # multiprocessing.synchronize module cannot be imported. + support.skip_if_broken_multiprocessing_synchronize() + + self.addCleanup(multiprocessing_cleanup_tests) + + async def main(): + if multiprocessing.get_start_method() == 'fork': + # Avoid 'fork' DeprecationWarning. + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None + pool = concurrent.futures.ProcessPoolExecutor( + mp_context=mp_context) + result = await self.loop.run_in_executor( + pool, _test_get_event_loop_new_process__sub_proc) + pool.shutdown() + return result + + self.assertEqual( + self.loop.run_until_complete(main()), + 'hello') + + def test_get_running_loop_already_running(self): + async def main(): + running_loop = asyncio.get_running_loop() + with contextlib.closing(asyncio.new_event_loop()) as loop: + try: + loop.run_forever() + except RuntimeError: + pass + else: + self.fail("RuntimeError not raised") + + self.assertIs(asyncio.get_running_loop(), running_loop) + + self.loop.run_until_complete(main()) + + + def test_get_event_loop_returns_running_loop(self): + class TestError(Exception): + pass + + class Policy(test_utils.DefaultEventLoopPolicy): + def get_event_loop(self): + raise TestError + + old_policy = asyncio.events._get_event_loop_policy() + try: + asyncio.events._set_event_loop_policy(Policy()) + loop = asyncio.new_event_loop() + + with self.assertRaises(TestError): + asyncio.get_event_loop() + asyncio.set_event_loop(None) + with self.assertRaises(TestError): + asyncio.get_event_loop() + + with self.assertRaisesRegex(RuntimeError, 'no running'): + asyncio.get_running_loop() + self.assertIs(asyncio._get_running_loop(), None) + + async def func(): + self.assertIs(asyncio.get_event_loop(), loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.assertIs(asyncio._get_running_loop(), loop) + + loop.run_until_complete(func()) + + asyncio.set_event_loop(loop) + with self.assertRaises(TestError): + asyncio.get_event_loop() + asyncio.set_event_loop(None) + with self.assertRaises(TestError): + asyncio.get_event_loop() + + finally: + asyncio.events._set_event_loop_policy(old_policy) + if loop is not None: + loop.close() + + with self.assertRaisesRegex(RuntimeError, 'no running'): + asyncio.get_running_loop() + + self.assertIs(asyncio._get_running_loop(), None) + + def test_get_event_loop_returns_running_loop2(self): + old_policy = asyncio.events._get_event_loop_policy() + try: + asyncio.events._set_event_loop_policy(test_utils.DefaultEventLoopPolicy()) + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() + + asyncio.set_event_loop(None) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() + + async def func(): + self.assertIs(asyncio.get_event_loop(), loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.assertIs(asyncio._get_running_loop(), loop) + + loop.run_until_complete(func()) + + asyncio.set_event_loop(loop) + self.assertIs(asyncio.get_event_loop(), loop) + + asyncio.set_event_loop(None) + with self.assertRaisesRegex(RuntimeError, 'no current'): + asyncio.get_event_loop() + + finally: + asyncio.events._set_event_loop_policy(old_policy) + if loop is not None: + loop.close() + + with self.assertRaisesRegex(RuntimeError, 'no running'): + asyncio.get_running_loop() + + self.assertIs(asyncio._get_running_loop(), None) + + +class TestPyGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): + + _get_running_loop_impl = events._py__get_running_loop + _set_running_loop_impl = events._py__set_running_loop + get_running_loop_impl = events._py_get_running_loop + get_event_loop_impl = events._py_get_event_loop + + Task = asyncio.tasks._PyTask + Future = asyncio.futures._PyFuture + +try: + import _asyncio # NoQA +except ImportError: + pass +else: + + class TestCGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): + + _get_running_loop_impl = events._c__get_running_loop + _set_running_loop_impl = events._c__set_running_loop + get_running_loop_impl = events._c_get_running_loop + get_event_loop_impl = events._c_get_event_loop + + Task = asyncio.tasks._CTask + Future = asyncio.futures._CFuture + +class TestServer(unittest.TestCase): + + def test_get_loop(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + proto = MyProto(loop) + server = loop.run_until_complete(loop.create_server(lambda: proto, '0.0.0.0', 0)) + self.assertEqual(server.get_loop(), loop) + server.close() + loop.run_until_complete(server.wait_closed()) + + +class TestAbstractServer(unittest.TestCase): + + def test_close(self): + with self.assertRaises(NotImplementedError): + events.AbstractServer().close() + + def test_wait_closed(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + with self.assertRaises(NotImplementedError): + loop.run_until_complete(events.AbstractServer().wait_closed()) + + def test_get_loop(self): + with self.assertRaises(NotImplementedError): + events.AbstractServer().get_loop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py new file mode 100644 index 00000000000..c8de0d24499 --- /dev/null +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -0,0 +1,235 @@ +import asyncio +import threading +import unittest +from threading import Thread +from unittest import TestCase +import weakref +from test import support +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +class MyException(Exception): + pass + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class TestFreeThreading: + def test_all_tasks_race(self) -> None: + async def main(): + loop = asyncio.get_running_loop() + future = loop.create_future() + + async def coro(): + await future + + tasks = set() + + async with asyncio.TaskGroup() as tg: + for _ in range(100): + tasks.add(tg.create_task(coro())) + + all_tasks = asyncio.all_tasks(loop) + self.assertEqual(len(all_tasks), 101) + + for task in all_tasks: + self.assertEqual(task.get_loop(), loop) + self.assertFalse(task.done()) + + current = asyncio.current_task() + self.assertEqual(current.get_loop(), loop) + self.assertSetEqual(all_tasks, tasks | {current}) + future.set_result(None) + + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(main()) + + threads = [] + + for _ in range(10): + thread = Thread(target=runner) + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + def test_all_tasks_different_thread(self) -> None: + loop = None + started = threading.Event() + done = threading.Event() # used for main task not finishing early + async def coro(): + await asyncio.Future() + + lock = threading.Lock() + tasks = set() + + async def main(): + nonlocal tasks, loop + loop = asyncio.get_running_loop() + started.set() + for i in range(1000): + with lock: + asyncio.create_task(coro()) + tasks = asyncio.all_tasks(loop) + done.wait() + + runner = threading.Thread(target=lambda: asyncio.run(main())) + + def check(): + started.wait() + with lock: + self.assertSetEqual(tasks & asyncio.all_tasks(loop), tasks) + + threads = [threading.Thread(target=check) for _ in range(10)] + runner.start() + + with threading_helper.start_threads(threads): + pass + + done.set() + runner.join() + + def test_task_different_thread_finalized(self) -> None: + task = None + async def func(): + nonlocal task + task = asyncio.current_task() + def runner(): + with asyncio.Runner() as runner: + loop = runner.get_loop() + loop.set_task_factory(self.factory) + runner.run(func()) + thread = Thread(target=runner) + thread.start() + thread.join() + wr = weakref.ref(task) + del thread + del task + # task finalization in different thread shouldn't crash + support.gc_collect() + self.assertIsNone(wr()) + + def test_run_coroutine_threadsafe(self) -> None: + results = [] + + def in_thread(loop: asyncio.AbstractEventLoop): + coro = asyncio.sleep(0.1, result=42) + fut = asyncio.run_coroutine_threadsafe(coro, loop) + result = fut.result() + self.assertEqual(result, 42) + results.append(result) + + async def main(): + loop = asyncio.get_running_loop() + async with asyncio.TaskGroup() as tg: + for _ in range(10): + tg.create_task(asyncio.to_thread(in_thread, loop)) + self.assertEqual(results, [42] * 10) + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + def test_run_coroutine_threadsafe_exception(self) -> None: + async def coro(): + await asyncio.sleep(0) + raise MyException("test") + + def in_thread(loop: asyncio.AbstractEventLoop): + fut = asyncio.run_coroutine_threadsafe(coro(), loop) + return fut.result() + + async def main(): + loop = asyncio.get_running_loop() + tasks = [] + for _ in range(10): + task = loop.create_task(asyncio.to_thread(in_thread, loop)) + tasks.append(task) + results = await asyncio.gather(*tasks, return_exceptions=True) + + self.assertEqual(len(results), 10) + for result in results: + self.assertIsInstance(result, MyException) + self.assertEqual(str(result), "test") + + with asyncio.Runner() as r: + loop = r.get_loop() + loop.set_task_factory(self.factory) + r.run(main()) + + +class TestPyFreeThreading(TestFreeThreading, TestCase): + + def setUp(self): + self._old_current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task + self._old_all_tasks = asyncio.all_tasks + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._py_all_tasks + self._old_Task = asyncio.Task + asyncio.Task = asyncio.tasks.Task = asyncio.tasks._PyTask + self._old_Future = asyncio.Future + asyncio.Future = asyncio.futures.Future = asyncio.futures._PyFuture + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._old_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._old_all_tasks + asyncio.Task = asyncio.tasks.Task = self._old_Task + asyncio.Future = asyncio.tasks.Future = self._old_Future + return super().tearDown() + + def factory(self, loop, coro, **kwargs): + return asyncio.tasks._PyTask(coro, loop=loop, **kwargs) + + @unittest.skip("TODO: RUSTPYTHON; hangs - Python _current_tasks dict not thread-safe") + def test_all_tasks_race(self): + return super().test_all_tasks_race() + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestCFreeThreading(TestFreeThreading, TestCase): + + def setUp(self): + self._old_current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._c_current_task + self._old_all_tasks = asyncio.all_tasks + asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._c_all_tasks + self._old_Task = asyncio.Task + asyncio.Task = asyncio.tasks.Task = asyncio.tasks._CTask + self._old_Future = asyncio.Future + asyncio.Future = asyncio.futures.Future = asyncio.futures._CFuture + return super().setUp() + + def tearDown(self): + asyncio.current_task = asyncio.tasks.current_task = self._old_current_task + asyncio.all_tasks = asyncio.tasks.all_tasks = self._old_all_tasks + asyncio.Task = asyncio.tasks.Task = self._old_Task + asyncio.Future = asyncio.futures.Future = self._old_Future + return super().tearDown() + + + def factory(self, loop, coro, **kwargs): + return asyncio.tasks._CTask(coro, loop=loop, **kwargs) + + +class TestEagerPyFreeThreading(TestPyFreeThreading): + def factory(self, loop, coro, eager_start=True, **kwargs): + return asyncio.tasks._PyTask(coro, loop=loop, **kwargs, eager_start=eager_start) + + @unittest.skip("TODO: RUSTPYTHON; hangs - Python _current_tasks dict not thread-safe") + def test_all_tasks_race(self): + return super().test_all_tasks_race() + + +@unittest.skipUnless(hasattr(asyncio.tasks, "_c_all_tasks"), "requires _asyncio") +class TestEagerCFreeThreading(TestCFreeThreading, TestCase): + def factory(self, loop, coro, eager_start=True, **kwargs): + return asyncio.tasks._CTask(coro, loop=loop, **kwargs, eager_start=eager_start) diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py new file mode 100644 index 00000000000..54bf824fef7 --- /dev/null +++ b/Lib/test/test_asyncio/test_futures.py @@ -0,0 +1,1144 @@ +"""Tests for futures.py.""" + +import concurrent.futures +import gc +import re +import sys +import threading +import traceback +import unittest +from unittest import mock +from types import GenericAlias +import asyncio +from asyncio import futures +import warnings +from test.test_asyncio import utils as test_utils +from test import support + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def _fakefunc(f): + return f + + +def first_cb(): + pass + + +def last_cb(): + pass + + +class ReachableCode(Exception): + """Exception to raise to indicate that some code was reached. + + Use this exception if using mocks is not a good alternative. + """ + + +class SimpleEvilEventLoop(asyncio.base_events.BaseEventLoop): + """Base class for UAF and other evil stuff requiring an evil event loop.""" + + def get_debug(self): # to suppress tracebacks + return False + + def __del__(self): + # Automatically close the evil event loop to avoid warnings. + if not self.is_closed() and not self.is_running(): + self.close() + + +class DuckFuture: + # Class that does not inherit from Future but aims to be duck-type + # compatible with it. + + _asyncio_future_blocking = False + __cancelled = False + __result = None + __exception = None + + def cancel(self): + if self.done(): + return False + self.__cancelled = True + return True + + def cancelled(self): + return self.__cancelled + + def done(self): + return (self.__cancelled + or self.__result is not None + or self.__exception is not None) + + def result(self): + self.assertFalse(self.cancelled()) + if self.__exception is not None: + raise self.__exception + return self.__result + + def exception(self): + self.assertFalse(self.cancelled()) + return self.__exception + + def set_result(self, result): + self.assertFalse(self.done()) + self.assertIsNotNone(result) + self.__result = result + + def set_exception(self, exception): + self.assertFalse(self.done()) + self.assertIsNotNone(exception) + self.__exception = exception + + def __iter__(self): + if not self.done(): + self._asyncio_future_blocking = True + yield self + self.assertTrue(self.done()) + return self.result() + + +class DuckTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_wrap_future(self): + f = DuckFuture() + g = asyncio.wrap_future(f) + self.assertIs(g, f) + + def test_ensure_future(self): + f = DuckFuture() + g = asyncio.ensure_future(f) + self.assertIs(g, f) + + +class BaseFutureTests: + + def _new_future(self, *args, **kwargs): + return self.cls(*args, **kwargs) + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_generic_alias(self): + future = self.cls[str] + self.assertEqual(future.__args__, (str,)) + self.assertIsInstance(future, GenericAlias) + + def test_isfuture(self): + class MyFuture: + _asyncio_future_blocking = None + + def __init__(self): + self._asyncio_future_blocking = False + + self.assertFalse(asyncio.isfuture(MyFuture)) + self.assertTrue(asyncio.isfuture(MyFuture())) + self.assertFalse(asyncio.isfuture(1)) + + # As `isinstance(Mock(), Future)` returns `False` + self.assertFalse(asyncio.isfuture(mock.Mock())) + + f = self._new_future(loop=self.loop) + self.assertTrue(asyncio.isfuture(f)) + self.assertFalse(asyncio.isfuture(type(f))) + + # As `isinstance(Mock(Future), Future)` returns `True` + self.assertTrue(asyncio.isfuture(mock.Mock(type(f)))) + + f.cancel() + + def test_initial_state(self): + f = self._new_future(loop=self.loop) + self.assertFalse(f.cancelled()) + self.assertFalse(f.done()) + f.cancel() + self.assertTrue(f.cancelled()) + + def test_constructor_without_loop(self): + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + self._new_future() + + def test_constructor_use_running_loop(self): + async def test(): + return self._new_future() + f = self.loop.run_until_complete(test()) + self.assertIs(f._loop, self.loop) + self.assertIs(f.get_loop(), self.loop) + + def test_constructor_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio.set_event_loop, None) + f = self._new_future() + self.assertIs(f._loop, self.loop) + self.assertIs(f.get_loop(), self.loop) + + def test_constructor_positional(self): + # Make sure Future doesn't accept a positional argument + self.assertRaises(TypeError, self._new_future, 42) + + def test_uninitialized(self): + # Test that C Future doesn't crash when Future.__init__() + # call was skipped. + + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, fut.result) + + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, fut.exception) + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.set_result(None) + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.set_exception(Exception) + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.cancel() + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.add_done_callback(lambda f: None) + + fut = self.cls.__new__(self.cls, loop=self.loop) + with self.assertRaises((RuntimeError, AttributeError)): + fut.remove_done_callback(lambda f: None) + + fut = self.cls.__new__(self.cls, loop=self.loop) + try: + repr(fut) + except (RuntimeError, AttributeError): + pass + + fut = self.cls.__new__(self.cls, loop=self.loop) + try: + fut.__await__() + except RuntimeError: + pass + + fut = self.cls.__new__(self.cls, loop=self.loop) + try: + iter(fut) + except RuntimeError: + pass + + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertFalse(fut.cancelled()) + self.assertFalse(fut.done()) + + def test_future_cancel_message_getter(self): + f = self._new_future(loop=self.loop) + self.assertHasAttr(f, '_cancel_message') + self.assertEqual(f._cancel_message, None) + + f.cancel('my message') + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) + self.assertEqual(f._cancel_message, 'my message') + + def test_future_cancel_message_setter(self): + f = self._new_future(loop=self.loop) + f.cancel('my message') + f._cancel_message = 'my new message' + self.assertEqual(f._cancel_message, 'my new message') + + # Also check that the value is used for cancel(). + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) + self.assertEqual(f._cancel_message, 'my new message') + + def test_cancel(self): + f = self._new_future(loop=self.loop) + self.assertTrue(f.cancel()) + self.assertTrue(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(asyncio.CancelledError, f.result) + self.assertRaises(asyncio.CancelledError, f.exception) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_result(self): + f = self._new_future(loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, f.result) + + f.set_result(42) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertEqual(f.result(), 42) + self.assertEqual(f.exception(), None) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_exception(self): + exc = RuntimeError() + f = self._new_future(loop=self.loop) + self.assertRaises(asyncio.InvalidStateError, f.exception) + + f.set_exception(exc) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(RuntimeError, f.result) + self.assertEqual(f.exception(), exc) + self.assertRaises(asyncio.InvalidStateError, f.set_result, None) + self.assertRaises(asyncio.InvalidStateError, f.set_exception, None) + self.assertFalse(f.cancel()) + + def test_stop_iteration_exception(self, stop_iteration_class=StopIteration): + exc = stop_iteration_class() + f = self._new_future(loop=self.loop) + f.set_exception(exc) + self.assertFalse(f.cancelled()) + self.assertTrue(f.done()) + self.assertRaises(RuntimeError, f.result) + exc = f.exception() + cause = exc.__cause__ + self.assertIsInstance(exc, RuntimeError) + self.assertRegex(str(exc), 'StopIteration .* cannot be raised') + self.assertIsInstance(cause, stop_iteration_class) + + def test_stop_iteration_subclass_exception(self): + class MyStopIteration(StopIteration): + pass + + self.test_stop_iteration_exception(MyStopIteration) + + def test_exception_class(self): + f = self._new_future(loop=self.loop) + f.set_exception(RuntimeError) + self.assertIsInstance(f.exception(), RuntimeError) + + def test_yield_from_twice(self): + f = self._new_future(loop=self.loop) + + def fixture(): + yield 'A' + x = yield from f + yield 'B', x + y = yield from f + yield 'C', y + + g = fixture() + self.assertEqual(next(g), 'A') # yield 'A'. + self.assertEqual(next(g), f) # First yield from f. + f.set_result(42) + self.assertEqual(next(g), ('B', 42)) # yield 'B', x. + # The second "yield from f" does not yield f. + self.assertEqual(next(g), ('C', 42)) # yield 'C', y. + + def test_future_repr(self): + self.loop.set_debug(True) + f_pending_debug = self._new_future(loop=self.loop) + frame = f_pending_debug._source_traceback[-1] + self.assertEqual( + repr(f_pending_debug), + f'<{self.cls.__name__} pending created at {frame[0]}:{frame[1]}>') + f_pending_debug.cancel() + + self.loop.set_debug(False) + f_pending = self._new_future(loop=self.loop) + self.assertEqual(repr(f_pending), f'<{self.cls.__name__} pending>') + f_pending.cancel() + + f_cancelled = self._new_future(loop=self.loop) + f_cancelled.cancel() + self.assertEqual(repr(f_cancelled), f'<{self.cls.__name__} cancelled>') + + f_result = self._new_future(loop=self.loop) + f_result.set_result(4) + self.assertEqual( + repr(f_result), f'<{self.cls.__name__} finished result=4>') + self.assertEqual(f_result.result(), 4) + + exc = RuntimeError() + f_exception = self._new_future(loop=self.loop) + f_exception.set_exception(exc) + self.assertEqual( + repr(f_exception), + f'<{self.cls.__name__} finished exception=RuntimeError()>') + self.assertIs(f_exception.exception(), exc) + + def func_repr(func): + filename, lineno = test_utils.get_function_source(func) + text = '%s() at %s:%s' % (func.__qualname__, filename, lineno) + return re.escape(text) + + f_one_callbacks = self._new_future(loop=self.loop) + f_one_callbacks.add_done_callback(_fakefunc) + fake_repr = func_repr(_fakefunc) + self.assertRegex( + repr(f_one_callbacks), + r'<' + self.cls.__name__ + r' pending cb=\[%s\]>' % fake_repr) + f_one_callbacks.cancel() + self.assertEqual(repr(f_one_callbacks), + f'<{self.cls.__name__} cancelled>') + + f_two_callbacks = self._new_future(loop=self.loop) + f_two_callbacks.add_done_callback(first_cb) + f_two_callbacks.add_done_callback(last_cb) + first_repr = func_repr(first_cb) + last_repr = func_repr(last_cb) + self.assertRegex(repr(f_two_callbacks), + r'<' + self.cls.__name__ + r' pending cb=\[%s, %s\]>' + % (first_repr, last_repr)) + + f_many_callbacks = self._new_future(loop=self.loop) + f_many_callbacks.add_done_callback(first_cb) + for i in range(8): + f_many_callbacks.add_done_callback(_fakefunc) + f_many_callbacks.add_done_callback(last_cb) + cb_regex = r'%s, <8 more>, %s' % (first_repr, last_repr) + self.assertRegex( + repr(f_many_callbacks), + r'<' + self.cls.__name__ + r' pending cb=\[%s\]>' % cb_regex) + f_many_callbacks.cancel() + self.assertEqual(repr(f_many_callbacks), + f'<{self.cls.__name__} cancelled>') + + def test_copy_state(self): + from asyncio.futures import _copy_future_state + + f = self._new_future(loop=self.loop) + f.set_result(10) + + newf = self._new_future(loop=self.loop) + _copy_future_state(f, newf) + self.assertTrue(newf.done()) + self.assertEqual(newf.result(), 10) + + f_exception = self._new_future(loop=self.loop) + f_exception.set_exception(RuntimeError()) + + newf_exception = self._new_future(loop=self.loop) + _copy_future_state(f_exception, newf_exception) + self.assertTrue(newf_exception.done()) + self.assertRaises(RuntimeError, newf_exception.result) + + f_cancelled = self._new_future(loop=self.loop) + f_cancelled.cancel() + + newf_cancelled = self._new_future(loop=self.loop) + _copy_future_state(f_cancelled, newf_cancelled) + self.assertTrue(newf_cancelled.cancelled()) + + try: + raise concurrent.futures.InvalidStateError + except BaseException as e: + f_exc = e + + f_conexc = self._new_future(loop=self.loop) + f_conexc.set_exception(f_exc) + + newf_conexc = self._new_future(loop=self.loop) + _copy_future_state(f_conexc, newf_conexc) + self.assertTrue(newf_conexc.done()) + try: + newf_conexc.result() + except BaseException as e: + newf_exc = e # assertRaises context manager drops the traceback + newf_tb = ''.join(traceback.format_tb(newf_exc.__traceback__)) + self.assertEqual(newf_tb.count('raise concurrent.futures.InvalidStateError'), 1) + + def test_iter(self): + fut = self._new_future(loop=self.loop) + + def coro(): + yield from fut + + def test(): + arg1, arg2 = coro() + + with self.assertRaisesRegex(RuntimeError, "await wasn't used"): + test() + fut.cancel() + + def test_log_traceback(self): + fut = self._new_future(loop=self.loop) + with self.assertRaisesRegex(ValueError, 'can only be set to False'): + fut._log_traceback = True + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_abandoned(self, m_log): + fut = self._new_future(loop=self.loop) + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(Exception()) + fut.cancel() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_result_unretrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_result(42) + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_result_retrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_result(42) + fut.result() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_unretrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + del fut + test_utils.run_briefly(self.loop) + support.gc_collect() + self.assertTrue(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_retrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + fut.exception() + del fut + self.assertFalse(m_log.error.called) + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_exception_result_retrieved(self, m_log): + fut = self._new_future(loop=self.loop) + fut.set_exception(RuntimeError('boom')) + self.assertRaises(RuntimeError, fut.result) + del fut + self.assertFalse(m_log.error.called) + + def test_wrap_future(self): + + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + f2 = asyncio.wrap_future(f1, loop=self.loop) + res, ident = self.loop.run_until_complete(f2) + self.assertTrue(asyncio.isfuture(f2)) + self.assertEqual(res, 'oi') + self.assertNotEqual(ident, threading.get_ident()) + ex.shutdown(wait=True) + + def test_wrap_future_future(self): + f1 = self._new_future(loop=self.loop) + f2 = asyncio.wrap_future(f1) + self.assertIs(f1, f2) + + def test_wrap_future_without_loop(self): + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.wrap_future(f1) + ex.shutdown(wait=True) + + def test_wrap_future_use_running_loop(self): + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + async def test(): + return asyncio.wrap_future(f1) + f2 = self.loop.run_until_complete(test()) + self.assertIs(self.loop, f2._loop) + ex.shutdown(wait=True) + + def test_wrap_future_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio.set_event_loop, None) + def run(arg): + return (arg, threading.get_ident()) + ex = concurrent.futures.ThreadPoolExecutor(1) + f1 = ex.submit(run, 'oi') + f2 = asyncio.wrap_future(f1) + self.assertIs(self.loop, f2._loop) + ex.shutdown(wait=True) + + def test_wrap_future_cancel(self): + f1 = concurrent.futures.Future() + f2 = asyncio.wrap_future(f1, loop=self.loop) + f2.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(f1.cancelled()) + self.assertTrue(f2.cancelled()) + + def test_wrap_future_cancel2(self): + f1 = concurrent.futures.Future() + f2 = asyncio.wrap_future(f1, loop=self.loop) + f1.set_result(42) + f2.cancel() + test_utils.run_briefly(self.loop) + self.assertFalse(f1.cancelled()) + self.assertEqual(f1.result(), 42) + self.assertTrue(f2.cancelled()) + + def test_future_source_traceback(self): + self.loop.set_debug(True) + + future = self._new_future(loop=self.loop) + lineno = sys._getframe().f_lineno - 1 + self.assertIsInstance(future._source_traceback, list) + self.assertEqual(future._source_traceback[-2][:3], + (__file__, + lineno, + 'test_future_source_traceback')) + + @mock.patch('asyncio.base_events.logger') + def check_future_exception_never_retrieved(self, debug, m_log): + self.loop.set_debug(debug) + + def memory_error(): + try: + raise MemoryError() + except BaseException as exc: + return exc + exc = memory_error() + + future = self._new_future(loop=self.loop) + future.set_exception(exc) + future = None + test_utils.run_briefly(self.loop) + support.gc_collect() + + regex = f'^{self.cls.__name__} exception was never retrieved\n' + exc_info = (type(exc), exc, exc.__traceback__) + m_log.error.assert_called_once_with(mock.ANY, exc_info=exc_info) + + message = m_log.error.call_args[0][0] + self.assertRegex(message, re.compile(regex, re.DOTALL)) + + def test_future_exception_never_retrieved(self): + self.check_future_exception_never_retrieved(False) + + def test_future_exception_never_retrieved_debug(self): + self.check_future_exception_never_retrieved(True) + + def test_set_result_unless_cancelled(self): + fut = self._new_future(loop=self.loop) + fut.cancel() + futures._set_result_unless_cancelled(fut, 2) + self.assertTrue(fut.cancelled()) + + def test_future_stop_iteration_args(self): + fut = self._new_future(loop=self.loop) + fut.set_result((1, 2)) + fi = fut.__iter__() + result = None + try: + fi.send(None) + except StopIteration as ex: + result = ex.args[0] + else: + self.fail('StopIteration was expected') + self.assertEqual(result, (1, 2)) + + def test_future_iter_throw(self): + fut = self._new_future(loop=self.loop) + fi = iter(fut) + with self.assertWarns(DeprecationWarning): + self.assertRaises(Exception, fi.throw, Exception, Exception("zebra"), None) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + self.assertRaises(TypeError, fi.throw, + Exception, Exception("elephant"), 32) + self.assertRaises(TypeError, fi.throw, + Exception("elephant"), Exception("elephant")) + # https://github.com/python/cpython/issues/101326 + self.assertRaises(ValueError, fi.throw, ValueError, None, None) + self.assertRaises(TypeError, fi.throw, list) + + def test_future_del_collect(self): + class Evil: + def __del__(self): + gc.collect() + + for i in range(100): + fut = self._new_future(loop=self.loop) + fut.set_result(Evil()) + + def test_future_cancelled_result_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.result() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + + def test_future_cancelled_exception_refcycles(self): + f = self._new_future(loop=self.loop) + f.cancel() + exc = None + try: + f.exception() + except asyncio.CancelledError as e: + exc = e + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), []) + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CFutureTests(BaseFutureTests, test_utils.TestCase): + try: + cls = futures._CFuture + except AttributeError: + cls = None + + def test_future_del_segfault(self): + fut = self._new_future(loop=self.loop) + with self.assertRaises(AttributeError): + del fut._asyncio_future_blocking + with self.assertRaises(AttributeError): + del fut._log_traceback + + def test_callbacks_copy(self): + # See https://github.com/python/cpython/issues/125789 + # In C implementation, the `_callbacks` attribute + # always returns a new list to avoid mutations of internal state + + fut = self._new_future(loop=self.loop) + f1 = lambda _: 1 + f2 = lambda _: 2 + fut.add_done_callback(f1) + fut.add_done_callback(f2) + callbacks = fut._callbacks + self.assertIsNot(callbacks, fut._callbacks) + fut.remove_done_callback(f1) + callbacks = fut._callbacks + self.assertIsNot(callbacks, fut._callbacks) + fut.remove_done_callback(f2) + self.assertIsNone(fut._callbacks) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.get_referents not implemented + def test_future_iter_get_referents_segfault(self): + return super().test_future_iter_get_referents_segfault() + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CSubFutureTests(BaseFutureTests, test_utils.TestCase): + try: + class CSubFuture(futures._CFuture): + pass + + cls = CSubFuture + except AttributeError: + cls = None + + +class PyFutureTests(BaseFutureTests, test_utils.TestCase): + cls = futures._PyFuture + + +class BaseFutureDoneCallbackTests(): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + def run_briefly(self): + test_utils.run_briefly(self.loop) + + def _make_callback(self, bag, thing): + # Create a callback function that appends thing to bag. + def bag_appender(future): + bag.append(thing) + return bag_appender + + def _new_future(self): + raise NotImplementedError + + def test_callbacks_remove_first_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb1) + f.remove_done_callback(cb1) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [17, 100]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_remove_first_and_second_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb1) + f.remove_done_callback(cb2) + f.remove_done_callback(cb1) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [100]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_remove_third_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb3) + f.remove_done_callback(cb3) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [42, 17]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_invoked_on_set_result(self): + bag = [] + f = self._new_future() + f.add_done_callback(self._make_callback(bag, 42)) + f.add_done_callback(self._make_callback(bag, 17)) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [42, 17]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_invoked_on_set_exception(self): + bag = [] + f = self._new_future() + f.add_done_callback(self._make_callback(bag, 100)) + + self.assertEqual(bag, []) + exc = RuntimeError() + f.set_exception(exc) + + self.run_briefly() + + self.assertEqual(bag, [100]) + self.assertEqual(f.exception(), exc) + + def test_remove_done_callback(self): + bag = [] + f = self._new_future() + cb1 = self._make_callback(bag, 1) + cb2 = self._make_callback(bag, 2) + cb3 = self._make_callback(bag, 3) + + # Add one cb1 and one cb2. + f.add_done_callback(cb1) + f.add_done_callback(cb2) + + # One instance of cb2 removed. Now there's only one cb1. + self.assertEqual(f.remove_done_callback(cb2), 1) + + # Never had any cb3 in there. + self.assertEqual(f.remove_done_callback(cb3), 0) + + # After this there will be 6 instances of cb1 and one of cb2. + f.add_done_callback(cb2) + for i in range(5): + f.add_done_callback(cb1) + + # Remove all instances of cb1. One cb2 remains. + self.assertEqual(f.remove_done_callback(cb1), 6) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [2]) + self.assertEqual(f.result(), 'foo') + + def test_remove_done_callbacks_list_mutation(self): + # see http://bugs.python.org/issue28963 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_remove_done_callbacks_list_clear(self): + # see https://github.com/python/cpython/issues/97592 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + class evil: + def __eq__(self, other): + fut.remove_done_callback(other) + + fut.remove_done_callback(evil()) + + def test_schedule_callbacks_list_mutation_1(self): + # see http://bugs.python.org/issue28963 for details + + def mut(f): + f.remove_done_callback(str) + + fut = self._new_future() + fut.add_done_callback(mut) + fut.add_done_callback(str) + fut.add_done_callback(str) + fut.set_result(1) + test_utils.run_briefly(self.loop) + + def test_schedule_callbacks_list_mutation_2(self): + # see http://bugs.python.org/issue30828 for details + + fut = self._new_future() + fut.add_done_callback(str) + + for _ in range(63): + fut.add_done_callback(id) + + max_extra_cbs = 100 + extra_cbs = 0 + + class evil: + def __eq__(self, other): + nonlocal extra_cbs + extra_cbs += 1 + if extra_cbs < max_extra_cbs: + fut.add_done_callback(id) + return False + + fut.remove_done_callback(evil()) + + def test_evil_call_soon_list_mutation(self): + # see: https://github.com/python/cpython/issues/125969 + called_on_fut_callback0 = False + + pad = lambda: ... + + def evil_call_soon(*args, **kwargs): + nonlocal called_on_fut_callback0 + if called_on_fut_callback0: + # Called when handling fut->fut_callbacks[0] + # and mutates the length fut->fut_callbacks. + fut.remove_done_callback(int) + fut.remove_done_callback(pad) + else: + called_on_fut_callback0 = True + + fake_event_loop = SimpleEvilEventLoop() + fake_event_loop.call_soon = evil_call_soon + + with mock.patch.object(self, 'loop', fake_event_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), fake_event_loop) + + fut.add_done_callback(str) # sets fut->fut_callback0 + fut.add_done_callback(int) # sets fut->fut_callbacks[0] + fut.add_done_callback(pad) # sets fut->fut_callbacks[1] + fut.add_done_callback(pad) # sets fut->fut_callbacks[2] + fut.set_result("boom") + + # When there are no more callbacks, the Python implementation + # returns an empty list but the C implementation returns None. + self.assertIn(fut._callbacks, (None, [])) + + def test_use_after_free_on_fut_callback_0_with_evil__eq__(self): + # Special thanks to Nico-Posada for the original PoC. + # See https://github.com/python/cpython/issues/125966. + + fut = self._new_future() + + class cb_pad: + def __eq__(self, other): + return True + + class evil(cb_pad): + def __eq__(self, other): + fut.remove_done_callback(None) + return NotImplemented + + fut.add_done_callback(cb_pad()) + fut.remove_done_callback(evil()) + + def test_use_after_free_on_fut_callback_0_with_evil__getattribute__(self): + # see: https://github.com/python/cpython/issues/125984 + + class EvilEventLoop(SimpleEvilEventLoop): + def call_soon(self, *args, **kwargs): + super().call_soon(*args, **kwargs) + raise ReachableCode + + def __getattribute__(self, name): + nonlocal fut_callback_0 + if name == 'call_soon': + fut.remove_done_callback(fut_callback_0) + del fut_callback_0 + return object.__getattribute__(self, name) + + evil_loop = EvilEventLoop() + with mock.patch.object(self, 'loop', evil_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), evil_loop) + + fut_callback_0 = lambda: ... + fut.add_done_callback(fut_callback_0) + self.assertRaises(ReachableCode, fut.set_result, "boom") + + def test_use_after_free_on_fut_context_0_with_evil__getattribute__(self): + # see: https://github.com/python/cpython/issues/125984 + + class EvilEventLoop(SimpleEvilEventLoop): + def call_soon(self, *args, **kwargs): + super().call_soon(*args, **kwargs) + raise ReachableCode + + def __getattribute__(self, name): + if name == 'call_soon': + # resets the future's event loop + fut.__init__(loop=SimpleEvilEventLoop()) + return object.__getattribute__(self, name) + + evil_loop = EvilEventLoop() + with mock.patch.object(self, 'loop', evil_loop): + fut = self._new_future() + self.assertIs(fut.get_loop(), evil_loop) + + fut_callback_0 = mock.Mock() + fut_context_0 = mock.Mock() + fut.add_done_callback(fut_callback_0, context=fut_context_0) + del fut_context_0 + del fut_callback_0 + self.assertRaises(ReachableCode, fut.set_result, "boom") + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + return futures._CFuture(loop=self.loop) + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CSubFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + class CSubFuture(futures._CFuture): + pass + return CSubFuture(loop=self.loop) + + +class PyFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + return futures._PyFuture(loop=self.loop) + + +class BaseFutureInheritanceTests: + + def _get_future_cls(self): + raise NotImplementedError + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_inherit_without_calling_super_init(self): + # See https://bugs.python.org/issue38785 for the context + cls = self._get_future_cls() + + class MyFut(cls): + def __init__(self, *args, **kwargs): + # don't call super().__init__() + pass + + fut = MyFut(loop=self.loop) + with self.assertRaisesRegex( + RuntimeError, + "Future object is not initialized." + ): + fut.get_loop() + + +class PyFutureInheritanceTests(BaseFutureInheritanceTests, + test_utils.TestCase): + def _get_future_cls(self): + return futures._PyFuture + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CFutureInheritanceTests(BaseFutureInheritanceTests, + test_utils.TestCase): + def _get_future_cls(self): + return futures._CFuture + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py new file mode 100644 index 00000000000..c7c0ebdac1b --- /dev/null +++ b/Lib/test/test_asyncio/test_futures2.py @@ -0,0 +1,95 @@ +# IsolatedAsyncioTestCase based tests +import asyncio +import contextvars +import traceback +import unittest +from asyncio import tasks + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class FutureTests: + + async def test_future_traceback(self): + + async def raise_exc(): + raise TypeError(42) + + future = self.cls(raise_exc()) + + for _ in range(5): + try: + await future + except TypeError as e: + tb = ''.join(traceback.format_tb(e.__traceback__)) + self.assertEqual(tb.count("await future"), 1) + else: + self.fail('TypeError was not raised') + + async def test_task_exc_handler_correct_context(self): + # see https://github.com/python/cpython/issues/96704 + name = contextvars.ContextVar('name', default='foo') + exc_handler_called = False + + def exc_handler(*args): + self.assertEqual(name.get(), 'bar') + nonlocal exc_handler_called + exc_handler_called = True + + async def task(): + name.set('bar') + 1/0 + + loop = asyncio.get_running_loop() + loop.set_exception_handler(exc_handler) + self.cls(task()) + await asyncio.sleep(0) + self.assertTrue(exc_handler_called) + + async def test_handle_exc_handler_correct_context(self): + # see https://github.com/python/cpython/issues/96704 + name = contextvars.ContextVar('name', default='foo') + exc_handler_called = False + + def exc_handler(*args): + self.assertEqual(name.get(), 'bar') + nonlocal exc_handler_called + exc_handler_called = True + + def callback(): + name.set('bar') + 1/0 + + loop = asyncio.get_running_loop() + loop.set_exception_handler(exc_handler) + loop.call_soon(callback) + await asyncio.sleep(0) + self.assertTrue(exc_handler_called) + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._CTask + +class PyFutureTests(FutureTests, unittest.IsolatedAsyncioTestCase): + cls = tasks._PyTask + +class FutureReprTests(unittest.IsolatedAsyncioTestCase): + + async def test_recursive_repr_for_pending_tasks(self): + # The call crashes if the guard for recursive call + # in base_futures:_future_repr_info is absent + # See Also: https://bugs.python.org/issue42183 + + async def func(): + return asyncio.all_tasks() + + # The repr() call should not raise RecursionError at first. + waiter = await asyncio.wait_for(asyncio.Task(func()),timeout=10) + self.assertIn('...', repr(waiter)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py new file mode 100644 index 00000000000..2f22fbccba4 --- /dev/null +++ b/Lib/test/test_asyncio/test_graph.py @@ -0,0 +1,445 @@ +import asyncio +import io +import unittest + + +# To prevent a warning "test altered the execution environment" +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def capture_test_stack(*, fut=None, depth=1): + + def walk(s): + ret = [ + (f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T') + if isinstance(s.future, asyncio.Task) else 'F' + ] + + ret.append( + [ + ( + f"s {entry.frame.f_code.co_name}" + if entry.frame.f_generator is None else + ( + f"a {entry.frame.f_generator.cr_code.co_name}" + if hasattr(entry.frame.f_generator, 'cr_code') else + f"ag {entry.frame.f_generator.ag_code.co_name}" + ) + ) for entry in s.call_stack + ] + ) + + ret.append( + sorted([ + walk(ab) for ab in s.awaited_by + ], key=lambda entry: entry[0]) + ) + + return ret + + buf = io.StringIO() + asyncio.print_call_graph(fut, file=buf, depth=depth+1) + + stack = asyncio.capture_call_graph(fut, depth=depth) + return walk(stack), buf.getvalue() + + +class CallStackTestBase: + + async def test_stack_tgroup(self): + + stack_for_c5 = None + + def c5(): + nonlocal stack_for_c5 + stack_for_c5 = capture_test_stack(depth=2) + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + await main() + + self.assertEqual(stack_for_c5[0], [ + # task name + 'T', + # call stack + ['s c5', 'a c4', 'a c3', 'a c2'], + # awaited by + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ], + ['T', + ['a c1'], + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ], + ['T', + ['a c1'], + [ + ['T', + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ] + ] + ]) + + self.assertIn( + ' async CallStackTestBase.test_stack_tgroup()', + stack_for_c5[1]) + + + async def test_stack_async_gen(self): + + stack_for_gen_nested_call = None + + async def gen_nested_call(): + nonlocal stack_for_gen_nested_call + stack_for_gen_nested_call = capture_test_stack() + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + await main() + + self.assertEqual(stack_for_gen_nested_call[0], [ + 'T', + [ + 's capture_test_stack', + 'a gen_nested_call', + 'ag gen', + 'a main', + 'a test_stack_async_gen' + ], + [] + ]) + + self.assertIn( + 'async generator CallStackTestBase.test_stack_async_gen..gen()', + stack_for_gen_nested_call[1]) + + async def test_stack_gather(self): + + stack_for_deep = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_deep + stack_for_deep = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + await main() + + self.assertEqual(stack_for_deep[0], [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_gather'], []] + ] + ]) + + async def test_stack_shield(self): + + stack_for_shield = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_shield + stack_for_shield = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_shield[0], [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_shield'], []] + ] + ]) + + async def test_stack_timeout(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', ['a main', 'a test_stack_timeout'], []] + ] + ]) + + async def test_stack_wait(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def c2(): + for i in range(3): + await asyncio.sleep(0) + + async def main(t1, t2): + while True: + _, pending = await asyncio.wait([t1, t2]) + if not pending: + break + + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) + try: + await main(t1, t2) + finally: + await t1 + await t2 + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', + ['a _wait', 'a wait', 'a main', 'a test_stack_wait'], + [] + ] + ] + ]) + + async def test_stack_task(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + await inner() + + async def c2(): + await asyncio.create_task(c1(), name='there there') + + async def main(): + await c2() + + await main() + + self.assertEqual(stack_for_inner[0], [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [['T', ['a c2', 'a main', 'a test_stack_task'], []]] + ]) + + async def test_stack_future(self): + + stack_for_fut = None + + async def a2(fut): + await fut + + async def a1(fut): + await a2(fut) + + async def b1(fut): + await fut + + async def main(): + nonlocal stack_for_fut + + fut = asyncio.Future() + async with asyncio.TaskGroup() as g: + g.create_task(a1(fut), name="task A") + g.create_task(b1(fut), name='task B') + + for _ in range(5): + # Do a few iterations to ensure that both a1 and b1 + # await on the future + await asyncio.sleep(0) + + stack_for_fut = capture_test_stack(fut=fut) + fut.set_result(None) + + await main() + + self.assertEqual(stack_for_fut[0], + ['F', + [], + [ + ['T', + ['a a2', 'a a1'], + [['T', ['a test_stack_future'], []]] + ], + ['T', + ['a b1'], + [['T', ['a test_stack_future'], []]] + ], + ]] + ) + + self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), + "C-accelerated asyncio call graph backend missing", +) +class TestCallStackC(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._CFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._CTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._c_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._c_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = tasks._c_current_task + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks.Task = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + asyncio.current_task = asyncio.tasks.current_task = self._current_task + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_py_future_add_to_awaited_by"), + "Pure Python asyncio call graph backend missing", +) +class TestCallStackPy(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._PyFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._PyTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._py_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._py_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + self._current_task = asyncio.current_task + asyncio.current_task = asyncio.tasks.current_task = tasks._py_current_task + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks.Task = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + asyncio.current_task = asyncio.tasks.current_task = self._current_task diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py new file mode 100644 index 00000000000..e025d2990a3 --- /dev/null +++ b/Lib/test/test_asyncio/test_locks.py @@ -0,0 +1,1825 @@ +"""Tests for locks.py""" + +import unittest +from unittest import mock +import re + +import asyncio +import collections + +STR_RGX_REPR = ( + r'^<(?P.*?) object at (?P
.*?)' + r'\[(?P' + r'(set|unset|locked|unlocked|filling|draining|resetting|broken)' + r'(, value:\d)?' + r'(, waiters:\d+)?' + r'(, waiters:\d+\/\d+)?' # barrier + r')\]>\z' +) +RGX_REPR = re.compile(STR_RGX_REPR) + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class LockTests(unittest.IsolatedAsyncioTestCase): + + async def test_repr(self): + lock = asyncio.Lock() + self.assertEndsWith(repr(lock), '[unlocked]>') + self.assertTrue(RGX_REPR.match(repr(lock))) + + await lock.acquire() + self.assertEndsWith(repr(lock), '[locked]>') + self.assertTrue(RGX_REPR.match(repr(lock))) + + async def test_lock(self): + lock = asyncio.Lock() + + with self.assertRaisesRegex( + TypeError, + "'Lock' object can't be awaited" + ): + await lock + + self.assertFalse(lock.locked()) + + async def test_lock_doesnt_accept_loop_parameter(self): + primitives_cls = [ + asyncio.Lock, + asyncio.Condition, + asyncio.Event, + asyncio.Semaphore, + asyncio.BoundedSemaphore, + ] + + loop = asyncio.get_running_loop() + + for cls in primitives_cls: + with self.assertRaisesRegex( + TypeError, + rf"{cls.__name__}\.__init__\(\) got an unexpected " + rf"keyword argument 'loop'" + ): + cls(loop=loop) + + async def test_lock_by_with_statement(self): + primitives = [ + asyncio.Lock(), + asyncio.Condition(), + asyncio.Semaphore(), + asyncio.BoundedSemaphore(), + ] + + for lock in primitives: + await asyncio.sleep(0.01) + self.assertFalse(lock.locked()) + with self.assertRaisesRegex( + TypeError, + r"'\w+' object can't be awaited" + ): + with await lock: + pass + self.assertFalse(lock.locked()) + + async def test_acquire(self): + lock = asyncio.Lock() + result = [] + + self.assertTrue(await lock.acquire()) + + async def c1(result): + if await lock.acquire(): + result.append(1) + return True + + async def c2(result): + if await lock.acquire(): + result.append(2) + return True + + async def c3(result): + if await lock.acquire(): + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + lock.release() + await asyncio.sleep(0) + self.assertEqual([1], result) + + await asyncio.sleep(0) + self.assertEqual([1], result) + + t3 = asyncio.create_task(c3(result)) + + lock.release() + await asyncio.sleep(0) + self.assertEqual([1, 2], result) + + lock.release() + await asyncio.sleep(0) + self.assertEqual([1, 2, 3], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + async def test_acquire_cancel(self): + lock = asyncio.Lock() + self.assertTrue(await lock.acquire()) + + task = asyncio.create_task(lock.acquire()) + asyncio.get_running_loop().call_soon(task.cancel) + with self.assertRaises(asyncio.CancelledError): + await task + self.assertFalse(lock._waiters) + + async def test_cancel_race(self): + # Several tasks: + # - A acquires the lock + # - B is blocked in acquire() + # - C is blocked in acquire() + # + # Now, concurrently: + # - B is cancelled + # - A releases the lock + # + # If B's waiter is marked cancelled but not yet removed from + # _waiters, A's release() call will crash when trying to set + # B's waiter; instead, it should move on to C's waiter. + + # Setup: A has the lock, b and c are waiting. + lock = asyncio.Lock() + + async def lockit(name, blocker): + await lock.acquire() + try: + if blocker is not None: + await blocker + finally: + lock.release() + + fa = asyncio.get_running_loop().create_future() + ta = asyncio.create_task(lockit('A', fa)) + await asyncio.sleep(0) + self.assertTrue(lock.locked()) + tb = asyncio.create_task(lockit('B', None)) + await asyncio.sleep(0) + self.assertEqual(len(lock._waiters), 1) + tc = asyncio.create_task(lockit('C', None)) + await asyncio.sleep(0) + self.assertEqual(len(lock._waiters), 2) + + # Create the race and check. + # Without the fix this failed at the last assert. + fa.set_result(None) + tb.cancel() + self.assertTrue(lock._waiters[0].cancelled()) + await asyncio.sleep(0) + self.assertFalse(lock.locked()) + self.assertTrue(ta.done()) + self.assertTrue(tb.cancelled()) + await tc + + async def test_cancel_release_race(self): + # Issue 32734 + # Acquire 4 locks, cancel second, release first + # and 2 locks are taken at once. + loop = asyncio.get_running_loop() + lock = asyncio.Lock() + lock_count = 0 + call_count = 0 + + async def lockit(): + nonlocal lock_count + nonlocal call_count + call_count += 1 + await lock.acquire() + lock_count += 1 + + def trigger(): + t1.cancel() + lock.release() + + await lock.acquire() + + t1 = asyncio.create_task(lockit()) + t2 = asyncio.create_task(lockit()) + t3 = asyncio.create_task(lockit()) + + # Start scheduled tasks + await asyncio.sleep(0) + + loop.call_soon(trigger) + with self.assertRaises(asyncio.CancelledError): + # Wait for cancellation + await t1 + + # Make sure only one lock was taken + self.assertEqual(lock_count, 1) + # While 3 calls were made to lockit() + self.assertEqual(call_count, 3) + self.assertTrue(t1.cancelled() and t2.done()) + + # Cleanup the task that is stuck on acquire. + t3.cancel() + await asyncio.sleep(0) + self.assertTrue(t3.cancelled()) + + async def test_finished_waiter_cancelled(self): + lock = asyncio.Lock() + + await lock.acquire() + self.assertTrue(lock.locked()) + + tb = asyncio.create_task(lock.acquire()) + await asyncio.sleep(0) + self.assertEqual(len(lock._waiters), 1) + + # Create a second waiter, wake up the first, and cancel it. + # Without the fix, the second was not woken up. + tc = asyncio.create_task(lock.acquire()) + tb.cancel() + lock.release() + await asyncio.sleep(0) + + self.assertTrue(lock.locked()) + self.assertTrue(tb.cancelled()) + + # Cleanup + await tc + + async def test_release_not_acquired(self): + lock = asyncio.Lock() + + self.assertRaises(RuntimeError, lock.release) + + async def test_release_no_waiters(self): + lock = asyncio.Lock() + await lock.acquire() + self.assertTrue(lock.locked()) + + lock.release() + self.assertFalse(lock.locked()) + + async def test_context_manager(self): + lock = asyncio.Lock() + self.assertFalse(lock.locked()) + + async with lock: + self.assertTrue(lock.locked()) + + self.assertFalse(lock.locked()) + + +class EventTests(unittest.IsolatedAsyncioTestCase): + + def test_repr(self): + ev = asyncio.Event() + self.assertEndsWith(repr(ev), '[unset]>') + match = RGX_REPR.match(repr(ev)) + self.assertEqual(match.group('extras'), 'unset') + + ev.set() + self.assertEndsWith(repr(ev), '[set]>') + self.assertTrue(RGX_REPR.match(repr(ev))) + + ev._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(ev)) + self.assertTrue(RGX_REPR.match(repr(ev))) + + async def test_wait(self): + ev = asyncio.Event() + self.assertFalse(ev.is_set()) + + result = [] + + async def c1(result): + if await ev.wait(): + result.append(1) + + async def c2(result): + if await ev.wait(): + result.append(2) + + async def c3(result): + if await ev.wait(): + result.append(3) + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + t3 = asyncio.create_task(c3(result)) + + ev.set() + await asyncio.sleep(0) + self.assertEqual([3, 1, 2], result) + + self.assertTrue(t1.done()) + self.assertIsNone(t1.result()) + self.assertTrue(t2.done()) + self.assertIsNone(t2.result()) + self.assertTrue(t3.done()) + self.assertIsNone(t3.result()) + + async def test_wait_on_set(self): + ev = asyncio.Event() + ev.set() + + res = await ev.wait() + self.assertTrue(res) + + async def test_wait_cancel(self): + ev = asyncio.Event() + + wait = asyncio.create_task(ev.wait()) + asyncio.get_running_loop().call_soon(wait.cancel) + with self.assertRaises(asyncio.CancelledError): + await wait + self.assertFalse(ev._waiters) + + async def test_clear(self): + ev = asyncio.Event() + self.assertFalse(ev.is_set()) + + ev.set() + self.assertTrue(ev.is_set()) + + ev.clear() + self.assertFalse(ev.is_set()) + + async def test_clear_with_waiters(self): + ev = asyncio.Event() + result = [] + + async def c1(result): + if await ev.wait(): + result.append(1) + return True + + t = asyncio.create_task(c1(result)) + await asyncio.sleep(0) + self.assertEqual([], result) + + ev.set() + ev.clear() + self.assertFalse(ev.is_set()) + + ev.set() + ev.set() + self.assertEqual(1, len(ev._waiters)) + + await asyncio.sleep(0) + self.assertEqual([1], result) + self.assertEqual(0, len(ev._waiters)) + + self.assertTrue(t.done()) + self.assertTrue(t.result()) + + +class ConditionTests(unittest.IsolatedAsyncioTestCase): + + async def test_wait(self): + cond = asyncio.Condition() + result = [] + + async def c1(result): + await cond.acquire() + if await cond.wait(): + result.append(1) + return True + + async def c2(result): + await cond.acquire() + if await cond.wait(): + result.append(2) + return True + + async def c3(result): + await cond.acquire() + if await cond.wait(): + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + self.assertFalse(cond.locked()) + + self.assertTrue(await cond.acquire()) + cond.notify() + await asyncio.sleep(0) + self.assertEqual([], result) + self.assertTrue(cond.locked()) + + cond.release() + await asyncio.sleep(0) + self.assertEqual([1], result) + self.assertTrue(cond.locked()) + + cond.notify(2) + await asyncio.sleep(0) + self.assertEqual([1], result) + self.assertTrue(cond.locked()) + + cond.release() + await asyncio.sleep(0) + self.assertEqual([1, 2], result) + self.assertTrue(cond.locked()) + + cond.release() + await asyncio.sleep(0) + self.assertEqual([1, 2, 3], result) + self.assertTrue(cond.locked()) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + async def test_wait_cancel(self): + cond = asyncio.Condition() + await cond.acquire() + + wait = asyncio.create_task(cond.wait()) + asyncio.get_running_loop().call_soon(wait.cancel) + with self.assertRaises(asyncio.CancelledError): + await wait + self.assertFalse(cond._waiters) + self.assertTrue(cond.locked()) + + async def test_wait_cancel_contested(self): + cond = asyncio.Condition() + + await cond.acquire() + self.assertTrue(cond.locked()) + + wait_task = asyncio.create_task(cond.wait()) + await asyncio.sleep(0) + self.assertFalse(cond.locked()) + + # Notify, but contest the lock before cancelling + await cond.acquire() + self.assertTrue(cond.locked()) + cond.notify() + asyncio.get_running_loop().call_soon(wait_task.cancel) + asyncio.get_running_loop().call_soon(cond.release) + + try: + await wait_task + except asyncio.CancelledError: + # Should not happen, since no cancellation points + pass + + self.assertTrue(cond.locked()) + + async def test_wait_cancel_after_notify(self): + # See bpo-32841 + waited = False + + cond = asyncio.Condition() + + async def wait_on_cond(): + nonlocal waited + async with cond: + waited = True # Make sure this area was reached + await cond.wait() + + waiter = asyncio.create_task(wait_on_cond()) + await asyncio.sleep(0) # Start waiting + + await cond.acquire() + cond.notify() + await asyncio.sleep(0) # Get to acquire() + waiter.cancel() + await asyncio.sleep(0) # Activate cancellation + cond.release() + await asyncio.sleep(0) # Cancellation should occur + + self.assertTrue(waiter.cancelled()) + self.assertTrue(waited) + + async def test_wait_unacquired(self): + cond = asyncio.Condition() + with self.assertRaises(RuntimeError): + await cond.wait() + + async def test_wait_for(self): + cond = asyncio.Condition() + presult = False + + def predicate(): + return presult + + result = [] + + async def c1(result): + await cond.acquire() + if await cond.wait_for(predicate): + result.append(1) + cond.release() + return True + + t = asyncio.create_task(c1(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + await cond.acquire() + cond.notify() + cond.release() + await asyncio.sleep(0) + self.assertEqual([], result) + + presult = True + await cond.acquire() + cond.notify() + cond.release() + await asyncio.sleep(0) + self.assertEqual([1], result) + + self.assertTrue(t.done()) + self.assertTrue(t.result()) + + async def test_wait_for_unacquired(self): + cond = asyncio.Condition() + + # predicate can return true immediately + res = await cond.wait_for(lambda: [1, 2, 3]) + self.assertEqual([1, 2, 3], res) + + with self.assertRaises(RuntimeError): + await cond.wait_for(lambda: False) + + async def test_notify(self): + cond = asyncio.Condition() + result = [] + + async def c1(result): + await cond.acquire() + if await cond.wait(): + result.append(1) + cond.release() + return True + + async def c2(result): + await cond.acquire() + if await cond.wait(): + result.append(2) + cond.release() + return True + + async def c3(result): + await cond.acquire() + if await cond.wait(): + result.append(3) + cond.release() + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + await cond.acquire() + cond.notify(1) + cond.release() + await asyncio.sleep(0) + self.assertEqual([1], result) + + await cond.acquire() + cond.notify(1) + cond.notify(2048) + cond.release() + await asyncio.sleep(0) + self.assertEqual([1, 2, 3], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + self.assertTrue(t3.done()) + self.assertTrue(t3.result()) + + async def test_notify_all(self): + cond = asyncio.Condition() + + result = [] + + async def c1(result): + await cond.acquire() + if await cond.wait(): + result.append(1) + cond.release() + return True + + async def c2(result): + await cond.acquire() + if await cond.wait(): + result.append(2) + cond.release() + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + + await asyncio.sleep(0) + self.assertEqual([], result) + + await cond.acquire() + cond.notify_all() + cond.release() + await asyncio.sleep(0) + self.assertEqual([1, 2], result) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + self.assertTrue(t2.done()) + self.assertTrue(t2.result()) + + def test_notify_unacquired(self): + cond = asyncio.Condition() + self.assertRaises(RuntimeError, cond.notify) + + def test_notify_all_unacquired(self): + cond = asyncio.Condition() + self.assertRaises(RuntimeError, cond.notify_all) + + async def test_repr(self): + cond = asyncio.Condition() + self.assertTrue('unlocked' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + await cond.acquire() + self.assertTrue('locked' in repr(cond)) + + cond._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + cond._waiters.append(mock.Mock()) + self.assertTrue('waiters:2' in repr(cond)) + self.assertTrue(RGX_REPR.match(repr(cond))) + + async def test_context_manager(self): + cond = asyncio.Condition() + self.assertFalse(cond.locked()) + async with cond: + self.assertTrue(cond.locked()) + self.assertFalse(cond.locked()) + + async def test_explicit_lock(self): + async def f(lock=None, cond=None): + if lock is None: + lock = asyncio.Lock() + if cond is None: + cond = asyncio.Condition(lock) + self.assertIs(cond._lock, lock) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) + async with cond: + self.assertTrue(lock.locked()) + self.assertTrue(cond.locked()) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) + async with lock: + self.assertTrue(lock.locked()) + self.assertTrue(cond.locked()) + self.assertFalse(lock.locked()) + self.assertFalse(cond.locked()) + + # All should work in the same way. + await f() + await f(asyncio.Lock()) + lock = asyncio.Lock() + await f(lock, asyncio.Condition(lock)) + + async def test_ambiguous_loops(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + async def wrong_loop_in_lock(): + with self.assertRaises(TypeError): + asyncio.Lock(loop=loop) # actively disallowed since 3.10 + lock = asyncio.Lock() + lock._loop = loop # use private API for testing + async with lock: + # acquired immediately via the fast-path + # without interaction with any event loop. + cond = asyncio.Condition(lock) + # cond.acquire() will trigger waiting on the lock + # and it will discover the event loop mismatch. + with self.assertRaisesRegex( + RuntimeError, + "is bound to a different event loop", + ): + await cond.acquire() + + async def wrong_loop_in_cond(): + # Same analogy here with the condition's loop. + lock = asyncio.Lock() + async with lock: + with self.assertRaises(TypeError): + asyncio.Condition(lock, loop=loop) + cond = asyncio.Condition(lock) + cond._loop = loop + with self.assertRaisesRegex( + RuntimeError, + "is bound to a different event loop", + ): + await cond.wait() + + await wrong_loop_in_lock() + await wrong_loop_in_cond() + + async def test_timeout_in_block(self): + condition = asyncio.Condition() + async with condition: + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(condition.wait(), timeout=0.5) + + async def test_cancelled_error_wakeup(self): + # Test that a cancelled error, received when awaiting wakeup, + # will be re-raised un-modified. + wake = False + raised = None + cond = asyncio.Condition() + + async def func(): + nonlocal raised + async with cond: + with self.assertRaises(asyncio.CancelledError) as err: + await cond.wait_for(lambda: wake) + raised = err.exception + raise raised + + task = asyncio.create_task(func()) + await asyncio.sleep(0) + # Task is waiting on the condition, cancel it there. + task.cancel(msg="foo") + with self.assertRaises(asyncio.CancelledError) as err: + await task + self.assertEqual(err.exception.args, ("foo",)) + # We should have got the _same_ exception instance as the one + # originally raised. + self.assertIs(err.exception, raised) + + async def test_cancelled_error_re_aquire(self): + # Test that a cancelled error, received when re-aquiring lock, + # will be re-raised un-modified. + wake = False + raised = None + cond = asyncio.Condition() + + async def func(): + nonlocal raised + async with cond: + with self.assertRaises(asyncio.CancelledError) as err: + await cond.wait_for(lambda: wake) + raised = err.exception + raise raised + + task = asyncio.create_task(func()) + await asyncio.sleep(0) + # Task is waiting on the condition + await cond.acquire() + wake = True + cond.notify() + await asyncio.sleep(0) + # Task is now trying to re-acquire the lock, cancel it there. + task.cancel(msg="foo") + cond.release() + with self.assertRaises(asyncio.CancelledError) as err: + await task + self.assertEqual(err.exception.args, ("foo",)) + # We should have got the _same_ exception instance as the one + # originally raised. + self.assertIs(err.exception, raised) + + async def test_cancelled_wakeup(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is awaiting initial + # wakeup on the wakeup queue. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # Cancel it while it is awaiting to be run. + # This cancellation could come from the outside + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + + async def test_cancelled_wakeup_relock(self): + # Test that a task cancelled at the "same" time as it is woken + # up as part of a Condition.notify() does not result in a lost wakeup. + # This test simulates a cancel while the target task is acquiring the lock + # again. + condition = asyncio.Condition() + state = 0 + async def consumer(): + nonlocal state + async with condition: + while True: + await condition.wait_for(lambda: state != 0) + if state < 0: + return + state -= 1 + + # create two consumers + c = [asyncio.create_task(consumer()) for _ in range(2)] + # wait for them to settle + await asyncio.sleep(0) + async with condition: + # produce one item and wake up one + state += 1 + condition.notify(1) + + # now we sleep for a bit. This allows the target task to wake up and + # settle on re-aquiring the lock + await asyncio.sleep(0) + + # Cancel it while awaiting the lock + # This cancel could come the outside. + c[0].cancel() + + # now wait for the item to be consumed + # if it doesn't means that our "notify" didn"t take hold. + # because it raced with a cancel() + try: + async with asyncio.timeout(0.01): + await condition.wait_for(lambda: state == 0) + except TimeoutError: + pass + self.assertEqual(state, 0) + + # clean up + state = -1 + condition.notify_all() + await c[1] + +class SemaphoreTests(unittest.IsolatedAsyncioTestCase): + + def test_initial_value_zero(self): + sem = asyncio.Semaphore(0) + self.assertTrue(sem.locked()) + + async def test_repr(self): + sem = asyncio.Semaphore() + self.assertEndsWith(repr(sem), '[unlocked, value:1]>') + self.assertTrue(RGX_REPR.match(repr(sem))) + + await sem.acquire() + self.assertEndsWith(repr(sem), '[locked]>') + self.assertTrue('waiters' not in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + if sem._waiters is None: + sem._waiters = collections.deque() + + sem._waiters.append(mock.Mock()) + self.assertTrue('waiters:1' in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + sem._waiters.append(mock.Mock()) + self.assertTrue('waiters:2' in repr(sem)) + self.assertTrue(RGX_REPR.match(repr(sem))) + + async def test_semaphore(self): + sem = asyncio.Semaphore() + self.assertEqual(1, sem._value) + + with self.assertRaisesRegex( + TypeError, + "'Semaphore' object can't be awaited", + ): + await sem + + self.assertFalse(sem.locked()) + self.assertEqual(1, sem._value) + + def test_semaphore_value(self): + self.assertRaises(ValueError, asyncio.Semaphore, -1) + + async def test_acquire(self): + sem = asyncio.Semaphore(3) + result = [] + + self.assertTrue(await sem.acquire()) + self.assertTrue(await sem.acquire()) + self.assertFalse(sem.locked()) + + async def c1(result): + await sem.acquire() + result.append(1) + return True + + async def c2(result): + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + result.append(3) + return True + + async def c4(result): + await sem.acquire() + result.append(4) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + self.assertEqual([1], result) + self.assertTrue(sem.locked()) + self.assertEqual(2, len(sem._waiters)) + self.assertEqual(0, sem._value) + + t4 = asyncio.create_task(c4(result)) + + sem.release() + sem.release() + self.assertEqual(0, sem._value) + + await asyncio.sleep(0) + self.assertEqual(0, sem._value) + self.assertEqual(3, len(result)) + self.assertTrue(sem.locked()) + self.assertEqual(1, len(sem._waiters)) + self.assertEqual(0, sem._value) + + self.assertTrue(t1.done()) + self.assertTrue(t1.result()) + race_tasks = [t2, t3, t4] + done_tasks = [t for t in race_tasks if t.done() and t.result()] + self.assertEqual(2, len(done_tasks)) + + # cleanup locked semaphore + sem.release() + await asyncio.gather(*race_tasks) + + async def test_acquire_cancel(self): + sem = asyncio.Semaphore() + await sem.acquire() + + acquire = asyncio.create_task(sem.acquire()) + asyncio.get_running_loop().call_soon(acquire.cancel) + with self.assertRaises(asyncio.CancelledError): + await acquire + self.assertTrue((not sem._waiters) or + all(waiter.done() for waiter in sem._waiters)) + + async def test_acquire_cancel_before_awoken(self): + sem = asyncio.Semaphore(value=0) + + t1 = asyncio.create_task(sem.acquire()) + t2 = asyncio.create_task(sem.acquire()) + t3 = asyncio.create_task(sem.acquire()) + t4 = asyncio.create_task(sem.acquire()) + + await asyncio.sleep(0) + + t1.cancel() + t2.cancel() + sem.release() + + await asyncio.sleep(0) + await asyncio.sleep(0) + num_done = sum(t.done() for t in [t3, t4]) + self.assertEqual(num_done, 1) + self.assertTrue(t3.done()) + self.assertFalse(t4.done()) + + t3.cancel() + t4.cancel() + await asyncio.sleep(0) + + async def test_acquire_hang(self): + sem = asyncio.Semaphore(value=0) + + t1 = asyncio.create_task(sem.acquire()) + t2 = asyncio.create_task(sem.acquire()) + await asyncio.sleep(0) + + t1.cancel() + sem.release() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(sem.locked()) + self.assertTrue(t2.done()) + + async def test_acquire_no_hang(self): + + sem = asyncio.Semaphore(1) + + async def c1(): + async with sem: + await asyncio.sleep(0) + t2.cancel() + + async def c2(): + async with sem: + self.assertFalse(True) + + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) + + r1, r2 = await asyncio.gather(t1, t2, return_exceptions=True) + self.assertTrue(r1 is None) + self.assertTrue(isinstance(r2, asyncio.CancelledError)) + + await asyncio.wait_for(sem.acquire(), timeout=1.0) + + def test_release_not_acquired(self): + sem = asyncio.BoundedSemaphore() + + self.assertRaises(ValueError, sem.release) + + async def test_release_no_waiters(self): + sem = asyncio.Semaphore() + await sem.acquire() + self.assertTrue(sem.locked()) + + sem.release() + self.assertFalse(sem.locked()) + + async def test_acquire_fifo_order(self): + sem = asyncio.Semaphore(1) + result = [] + + async def coro(tag): + await sem.acquire() + result.append(f'{tag}_1') + await asyncio.sleep(0.01) + sem.release() + + await sem.acquire() + result.append(f'{tag}_2') + await asyncio.sleep(0.01) + sem.release() + + async with asyncio.TaskGroup() as tg: + tg.create_task(coro('c1')) + tg.create_task(coro('c2')) + tg.create_task(coro('c3')) + + self.assertEqual( + ['c1_1', 'c2_1', 'c3_1', 'c1_2', 'c2_2', 'c3_2'], + result + ) + + async def test_acquire_fifo_order_2(self): + sem = asyncio.Semaphore(1) + result = [] + + async def c1(result): + await sem.acquire() + result.append(1) + return True + + async def c2(result): + await sem.acquire() + result.append(2) + sem.release() + await sem.acquire() + result.append(4) + return True + + async def c3(result): + await sem.acquire() + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + + sem.release() + sem.release() + + tasks = [t1, t2, t3] + await asyncio.gather(*tasks) + self.assertEqual([1, 2, 3, 4], result) + + async def test_acquire_fifo_order_3(self): + sem = asyncio.Semaphore(0) + result = [] + + async def c1(result): + await sem.acquire() + result.append(1) + return True + + async def c2(result): + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + result.append(3) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + + await asyncio.sleep(0) + + t1.cancel() + + await asyncio.sleep(0) + + sem.release() + sem.release() + + tasks = [t1, t2, t3] + await asyncio.gather(*tasks, return_exceptions=True) + self.assertEqual([2, 3], result) + + async def test_acquire_fifo_order_4(self): + # Test that a successful `acquire()` will wake up multiple Tasks + # that were waiting in the Semaphore queue due to FIFO rules. + sem = asyncio.Semaphore(0) + result = [] + count = 0 + + async def c1(result): + # First task immediately waits for semaphore. It will be awoken by c2. + self.assertEqual(sem._value, 0) + await sem.acquire() + # We should have woken up all waiting tasks now. + self.assertEqual(sem._value, 0) + # Create a fourth task. It should run after c3, not c2. + nonlocal t4 + t4 = asyncio.create_task(c4(result)) + result.append(1) + return True + + async def c2(result): + # The second task begins by releasing semaphore three times, + # for c1, c2, and c3. + sem.release() + sem.release() + sem.release() + self.assertEqual(sem._value, 2) + # It is locked, because c1 hasn't woken up yet. + self.assertTrue(sem.locked()) + await sem.acquire() + result.append(2) + return True + + async def c3(result): + await sem.acquire() + self.assertTrue(sem.locked()) + result.append(3) + return True + + async def c4(result): + result.append(4) + return True + + t1 = asyncio.create_task(c1(result)) + t2 = asyncio.create_task(c2(result)) + t3 = asyncio.create_task(c3(result)) + t4 = None + + await asyncio.sleep(0) + # Three tasks are in the queue, the first hasn't woken up yet. + self.assertEqual(sem._value, 2) + self.assertEqual(len(sem._waiters), 3) + await asyncio.sleep(0) + + tasks = [t1, t2, t3, t4] + await asyncio.gather(*tasks) + self.assertEqual([1, 2, 3, 4], result) + +class BarrierTests(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self): + await super().asyncSetUp() + self.N = 5 + + def make_tasks(self, n, coro): + tasks = [asyncio.create_task(coro()) for _ in range(n)] + return tasks + + async def gather_tasks(self, n, coro): + tasks = self.make_tasks(n, coro) + res = await asyncio.gather(*tasks) + return res, tasks + + async def test_barrier(self): + barrier = asyncio.Barrier(self.N) + self.assertIn("filling", repr(barrier)) + with self.assertRaisesRegex( + TypeError, + "'Barrier' object can't be awaited", + ): + await barrier + + self.assertIn("filling", repr(barrier)) + + async def test_repr(self): + barrier = asyncio.Barrier(self.N) + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("filling", repr(barrier)) + + waiters = [] + async def wait(barrier): + await barrier.wait() + + incr = 2 + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"waiters:{incr}/{self.N}" in repr(barrier)) + self.assertIn("filling", repr(barrier)) + + # create missing waiters + for i in range(barrier.parties - barrier.n_waiting): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("draining", repr(barrier)) + + # add a part of waiters + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + # and reset + await barrier.reset() + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("resetting", repr(barrier)) + + # add a part of waiters again + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + # and abort + await barrier.abort() + + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("broken", repr(barrier)) + self.assertTrue(barrier.broken) + + # suppress unhandled exceptions + await asyncio.gather(*waiters, return_exceptions=True) + + async def test_barrier_parties(self): + self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) + self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) + + self.assertIsInstance(asyncio.Barrier(self.N), asyncio.Barrier) + + async def test_context_manager(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + async with barrier as i: + results.append(i) + + await self.gather_tasks(self.N, coro) + + self.assertListEqual(sorted(results), list(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_one_task(self): + barrier = asyncio.Barrier(1) + + async def f(): + async with barrier as i: + return True + + ret = await f() + + self.assertTrue(ret) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_one_task_twice(self): + barrier = asyncio.Barrier(1) + + t1 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 0) + + t2 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + + self.assertEqual(t1.result(), t2.result()) + self.assertEqual(t1.done(), t2.done()) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_task_by_task(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + + t1 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 1) + self.assertIn("filling", repr(barrier)) + + t2 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 2) + self.assertIn("filling", repr(barrier)) + + t3 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + + await asyncio.wait([t1, t2, t3]) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_tasks_wait_twice(self): + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + async with barrier: + results.append(True) + + async with barrier: + results.append(False) + + await self.gather_tasks(self.N, coro) + + self.assertEqual(len(results), self.N*2) + self.assertEqual(results.count(True), self.N) + self.assertEqual(results.count(False), self.N) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_filling_tasks_check_return_value(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + + async def coro(): + async with barrier: + results1.append(True) + + async with barrier as i: + results2.append(True) + return i + + res, _ = await self.gather_tasks(self.N, coro) + + self.assertEqual(len(results1), self.N) + self.assertTrue(all(results1)) + self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) + self.assertListEqual(sorted(res), list(range(self.N))) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_draining_state(self): + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + async with barrier: + # barrier state change to filling for the last task release + results.append("draining" in repr(barrier)) + + await self.gather_tasks(self.N, coro) + + self.assertEqual(len(results), self.N) + self.assertEqual(results[-1], False) + self.assertTrue(all(results[:self.N-1])) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_blocking_tasks_while_draining(self): + rewait = 2 + barrier = asyncio.Barrier(self.N) + barrier_nowaiting = asyncio.Barrier(self.N - rewait) + results = [] + rewait_n = rewait + counter = 0 + + async def coro(): + nonlocal rewait_n + + # first time waiting + await barrier.wait() + + # after waiting once for all tasks + if rewait_n > 0: + rewait_n -= 1 + # wait again only for rewait tasks + await barrier.wait() + else: + # wait for end of draining state + await barrier_nowaiting.wait() + # wait for other waiting tasks + await barrier.wait() + + # a success means that barrier_nowaiting + # was waited for exactly N-rewait=3 times + await self.gather_tasks(self.N, coro) + + async def test_filling_tasks_cancel_one(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + await barrier.wait() + results.append(True) + + t1 = asyncio.create_task(coro()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 1) + + t2 = asyncio.create_task(coro()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 2) + + t1.cancel() + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 1) + with self.assertRaises(asyncio.CancelledError): + await t1 + self.assertTrue(t1.cancelled()) + + t3 = asyncio.create_task(coro()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 2) + + t4 = asyncio.create_task(coro()) + await asyncio.gather(t2, t3, t4) + + self.assertEqual(len(results), self.N) + self.assertTrue(all(results)) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_reset_barrier(self): + barrier = asyncio.Barrier(1) + + asyncio.create_task(barrier.reset()) + await asyncio.sleep(0) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_while_tasks_waiting(self): + barrier = asyncio.Barrier(self.N) + results = [] + + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + results.append(True) + + async def coro_reset(): + await barrier.reset() + + # N-1 tasks waiting on barrier with N parties + tasks = self.make_tasks(self.N-1, coro) + await asyncio.sleep(0) + + # reset the barrier + asyncio.create_task(coro_reset()) + await asyncio.gather(*tasks) + + self.assertEqual(len(results), self.N-1) + self.assertTrue(all(results)) + self.assertEqual(barrier.n_waiting, 0) + self.assertNotIn("resetting", repr(barrier)) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_when_tasks_half_draining(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + rest_of_tasks = self.N//2 + + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # catch here waiting tasks + results1.append(True) + else: + # here drained task outside the barrier + if rest_of_tasks == barrier._count: + # tasks outside the barrier + await barrier.reset() + + await self.gather_tasks(self.N, coro) + + self.assertEqual(results1, [True]*rest_of_tasks) + self.assertEqual(barrier.n_waiting, 0) + self.assertNotIn("resetting", repr(barrier)) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_when_tasks_half_draining_half_blocking(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + blocking_tasks = self.N//2 + count = 0 + + async def coro(): + nonlocal count + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch still waiting tasks + results1.append(True) + + # so now waiting again to reach nb_parties + await barrier.wait() + else: + count += 1 + if count > blocking_tasks: + # reset now: raise asyncio.BrokenBarrierError for waiting tasks + await barrier.reset() + + # so now waiting again to reach nb_parties + await barrier.wait() + else: + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here no catch - blocked tasks go to wait + results2.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertEqual(results1, [True]*blocking_tasks) + self.assertEqual(results2, []) + self.assertEqual(barrier.n_waiting, 0) + self.assertNotIn("resetting", repr(barrier)) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_while_tasks_waiting_and_waiting_again(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + + async def coro1(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + results1.append(True) + finally: + await barrier.wait() + results2.append(True) + + async def coro2(): + async with barrier: + results2.append(True) + + tasks = self.make_tasks(self.N-1, coro1) + + # reset barrier, N-1 waiting tasks raise an BrokenBarrierError + asyncio.create_task(barrier.reset()) + await asyncio.sleep(0) + + # complete waiting tasks in the `finally` + asyncio.create_task(coro2()) + + await asyncio.gather(*tasks) + + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), self.N-1) + self.assertTrue(all(results1)) + self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) + + self.assertEqual(barrier.n_waiting, 0) + + + async def test_reset_barrier_while_tasks_draining(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + results3 = [] + count = 0 + + async def coro(): + nonlocal count + + i = await barrier.wait() + count += 1 + if count == self.N: + # last task exited from barrier + await barrier.reset() + + # wait here to reach the `parties` + await barrier.wait() + else: + try: + # second waiting + await barrier.wait() + + # N-1 tasks here + results1.append(True) + except Exception as e: + # never goes here + results2.append(True) + + # Now, pass the barrier again + # last wait, must be completed + k = await barrier.wait() + results3.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertFalse(barrier.broken) + self.assertTrue(all(results1)) + self.assertEqual(len(results1), self.N-1) + self.assertEqual(len(results2), 0) + self.assertEqual(len(results3), self.N) + self.assertTrue(all(results3)) + + self.assertEqual(barrier.n_waiting, 0) + + async def test_abort_barrier(self): + barrier = asyncio.Barrier(1) + + asyncio.create_task(barrier.abort()) + await asyncio.sleep(0) + + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(barrier.broken) + + async def test_abort_barrier_when_tasks_half_draining_half_blocking(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + blocking_tasks = self.N//2 + count = 0 + + async def coro(): + nonlocal count + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch tasks waiting to drain + results1.append(True) + else: + count += 1 + if count > blocking_tasks: + # abort now: raise asyncio.BrokenBarrierError for all tasks + await barrier.abort() + else: + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch blocked tasks (already drained) + results2.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertTrue(barrier.broken) + self.assertEqual(results1, [True]*blocking_tasks) + self.assertEqual(results2, [True]*(self.N-blocking_tasks-1)) + self.assertEqual(barrier.n_waiting, 0) + self.assertNotIn("resetting", repr(barrier)) + + async def test_abort_barrier_when_exception(self): + # test from threading.Barrier: see `lock_tests.test_reset` + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + + async def coro(): + try: + async with barrier as i : + if i == self.N//2: + raise RuntimeError + async with barrier: + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + await barrier.abort() + + await self.gather_tasks(self.N, coro) + + self.assertTrue(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) + self.assertEqual(barrier.n_waiting, 0) + + async def test_abort_barrier_when_exception_then_resetting(self): + # test from threading.Barrier: see `lock_tests.test_abort_and_reset` + barrier1 = asyncio.Barrier(self.N) + barrier2 = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + results3 = [] + + async def coro(): + try: + i = await barrier1.wait() + if i == self.N//2: + raise RuntimeError + await barrier1.wait() + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + await barrier1.abort() + + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + i = await barrier2.wait() + if i == self.N//2: + await barrier1.reset() + await barrier2.wait() + await barrier1.wait() + results3.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertFalse(barrier1.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) + self.assertEqual(len(results3), self.N) + self.assertTrue(all(results3)) + + self.assertEqual(barrier1.n_waiting, 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py new file mode 100644 index 00000000000..a0c8434c945 --- /dev/null +++ b/Lib/test/test_asyncio/test_pep492.py @@ -0,0 +1,212 @@ +"""Tests support for new syntax introduced by PEP 492.""" + +import sys +import types +import unittest + +from unittest import mock + +import asyncio +from test.test_asyncio import utils as test_utils + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +# Test that asyncio.iscoroutine() uses collections.abc.Coroutine +class FakeCoro: + def send(self, value): + pass + + def throw(self, typ, val=None, tb=None): + pass + + def close(self): + pass + + def __await__(self): + yield + + +class BaseTest(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.BaseEventLoop() + self.loop._process_events = mock.Mock() + self.loop._selector = mock.Mock() + self.loop._selector.select.return_value = () + self.set_event_loop(self.loop) + + +class LockTests(BaseTest): + + def test_context_manager_async_with(self): + primitives = [ + asyncio.Lock(), + asyncio.Condition(), + asyncio.Semaphore(), + asyncio.BoundedSemaphore(), + ] + + async def test(lock): + await asyncio.sleep(0.01) + self.assertFalse(lock.locked()) + async with lock as _lock: + self.assertIs(_lock, None) + self.assertTrue(lock.locked()) + await asyncio.sleep(0.01) + self.assertTrue(lock.locked()) + self.assertFalse(lock.locked()) + + for primitive in primitives: + self.loop.run_until_complete(test(primitive)) + self.assertFalse(primitive.locked()) + + def test_context_manager_with_await(self): + primitives = [ + asyncio.Lock(), + asyncio.Condition(), + asyncio.Semaphore(), + asyncio.BoundedSemaphore(), + ] + + async def test(lock): + await asyncio.sleep(0.01) + self.assertFalse(lock.locked()) + with self.assertRaisesRegex( + TypeError, + "can't be awaited" + ): + with await lock: + pass + + for primitive in primitives: + self.loop.run_until_complete(test(primitive)) + self.assertFalse(primitive.locked()) + + +class StreamReaderTests(BaseTest): + + def test_readline(self): + DATA = b'line1\nline2\nline3' + + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(DATA) + stream.feed_eof() + + async def reader(): + data = [] + async for line in stream: + data.append(line) + return data + + data = self.loop.run_until_complete(reader()) + self.assertEqual(data, [b'line1\n', b'line2\n', b'line3']) + + +class CoroutineTests(BaseTest): + + def test_iscoroutine(self): + async def foo(): pass + + f = foo() + try: + self.assertTrue(asyncio.iscoroutine(f)) + finally: + f.close() # silence warning + + self.assertTrue(asyncio.iscoroutine(FakeCoro())) + + def test_iscoroutine_generator(self): + def foo(): yield + + self.assertFalse(asyncio.iscoroutine(foo())) + + def test_iscoroutinefunction(self): + async def foo(): pass + with self.assertWarns(DeprecationWarning): + self.assertTrue(asyncio.iscoroutinefunction(foo)) + + def test_async_def_coroutines(self): + async def bar(): + return 'spam' + async def foo(): + return await bar() + + # production mode + data = self.loop.run_until_complete(foo()) + self.assertEqual(data, 'spam') + + # debug mode + self.loop.set_debug(True) + data = self.loop.run_until_complete(foo()) + self.assertEqual(data, 'spam') + + def test_debug_mode_manages_coroutine_origin_tracking(self): + async def start(): + self.assertTrue(sys.get_coroutine_origin_tracking_depth() > 0) + + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0) + self.loop.set_debug(True) + self.loop.run_until_complete(start()) + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0) + + def test_types_coroutine(self): + def gen(): + yield from () + return 'spam' + + @types.coroutine + def func(): + return gen() + + async def coro(): + wrapper = func() + self.assertIsInstance(wrapper, types._GeneratorWrapper) + return await wrapper + + data = self.loop.run_until_complete(coro()) + self.assertEqual(data, 'spam') + + def test_task_print_stack(self): + T = None + + async def foo(): + f = T.get_stack(limit=1) + try: + self.assertEqual(f[0].f_code.co_name, 'foo') + finally: + f = None + + async def runner(): + nonlocal T + T = asyncio.ensure_future(foo(), loop=self.loop) + await T + + self.loop.run_until_complete(runner()) + + def test_double_await(self): + async def afunc(): + await asyncio.sleep(0.1) + + async def runner(): + coro = afunc() + t = self.loop.create_task(coro) + try: + await asyncio.sleep(0) + await coro + finally: + t.cancel() + + self.loop.set_debug(True) + with self.assertRaises( + RuntimeError, + msg='coroutine is being awaited already'): + + self.loop.run_until_complete(runner()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py new file mode 100644 index 00000000000..edfad5e11db --- /dev/null +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -0,0 +1,1094 @@ +"""Tests for proactor_events.py""" + +import io +import socket +import unittest +import sys +from unittest import mock + +import asyncio +from asyncio.proactor_events import BaseProactorEventLoop +from asyncio.proactor_events import _ProactorSocketTransport +from asyncio.proactor_events import _ProactorWritePipeTransport +from asyncio.proactor_events import _ProactorDuplexPipeTransport +from asyncio.proactor_events import _ProactorDatagramTransport +from test.support import os_helper +from test.support import socket_helper +from test.test_asyncio import utils as test_utils + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def close_transport(transport): + # Don't call transport.close() because the event loop and the IOCP proactor + # are mocked + if transport._sock is None: + return + transport._sock.close() + transport._sock = None + + +class ProactorSocketTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + self.proactor = mock.Mock() + self.loop._proactor = self.proactor + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.sock = mock.Mock(socket.socket) + self.buffer_size = 65536 + + def socket_transport(self, waiter=None): + transport = _ProactorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + fut = self.loop.create_future() + tr = self.socket_transport(waiter=fut) + test_utils.run_briefly(self.loop) + self.assertIsNone(fut.result()) + self.protocol.connection_made(tr) + self.proactor.recv_into.assert_called_with(self.sock, bytearray(self.buffer_size)) + + def test_loop_reading(self): + tr = self.socket_transport() + tr._loop_reading() + self.loop._proactor.recv_into.assert_called_with(self.sock, bytearray(self.buffer_size)) + self.assertFalse(self.protocol.data_received.called) + self.assertFalse(self.protocol.eof_received.called) + + def test_loop_reading_data(self): + buf = b'data' + res = self.loop.create_future() + res.set_result(len(buf)) + + tr = self.socket_transport() + tr._read_fut = res + tr._data[:len(buf)] = buf + tr._loop_reading(res) + called_buf = bytearray(self.buffer_size) + called_buf[:len(buf)] = buf + self.loop._proactor.recv_into.assert_called_with(self.sock, called_buf) + self.protocol.data_received.assert_called_with(buf) + # assert_called_with maps bytearray and bytes to the same thing so check manually + # regression test for https://github.com/python/cpython/issues/99941 + self.assertIsInstance(self.protocol.data_received.call_args.args[0], bytes) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_loop_reading_no_data(self): + res = self.loop.create_future() + res.set_result(0) + + tr = self.socket_transport() + self.assertRaises(AssertionError, tr._loop_reading, res) + + tr.close = mock.Mock() + tr._read_fut = res + tr._loop_reading(res) + self.assertFalse(self.loop._proactor.recv_into.called) + self.assertTrue(self.protocol.eof_received.called) + self.assertTrue(tr.close.called) + + def test_loop_reading_aborted(self): + err = self.loop._proactor.recv_into.side_effect = ConnectionAbortedError() + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._loop_reading() + tr._fatal_error.assert_called_with( + err, + 'Fatal read error on pipe transport') + + def test_loop_reading_aborted_closing(self): + self.loop._proactor.recv_into.side_effect = ConnectionAbortedError() + + tr = self.socket_transport() + tr._closing = True + tr._fatal_error = mock.Mock() + tr._loop_reading() + self.assertFalse(tr._fatal_error.called) + + def test_loop_reading_aborted_is_fatal(self): + self.loop._proactor.recv_into.side_effect = ConnectionAbortedError() + tr = self.socket_transport() + tr._closing = False + tr._fatal_error = mock.Mock() + tr._loop_reading() + self.assertTrue(tr._fatal_error.called) + + def test_loop_reading_conn_reset_lost(self): + err = self.loop._proactor.recv_into.side_effect = ConnectionResetError() + + tr = self.socket_transport() + tr._closing = False + tr._fatal_error = mock.Mock() + tr._force_close = mock.Mock() + tr._loop_reading() + self.assertFalse(tr._fatal_error.called) + tr._force_close.assert_called_with(err) + + def test_loop_reading_exception(self): + err = self.loop._proactor.recv_into.side_effect = (OSError()) + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._loop_reading() + tr._fatal_error.assert_called_with( + err, + 'Fatal read error on pipe transport') + + def test_write(self): + tr = self.socket_transport() + tr._loop_writing = mock.Mock() + tr.write(b'data') + self.assertEqual(tr._buffer, None) + tr._loop_writing.assert_called_with(data=b'data') + + def test_write_no_data(self): + tr = self.socket_transport() + tr.write(b'') + self.assertFalse(tr._buffer) + + def test_write_more(self): + tr = self.socket_transport() + tr._write_fut = mock.Mock() + tr._loop_writing = mock.Mock() + tr.write(b'data') + self.assertEqual(tr._buffer, b'data') + self.assertFalse(tr._loop_writing.called) + + def test_loop_writing(self): + tr = self.socket_transport() + tr._buffer = bytearray(b'data') + tr._loop_writing() + self.loop._proactor.send.assert_called_with(self.sock, b'data') + self.loop._proactor.send.return_value.add_done_callback.\ + assert_called_with(tr._loop_writing) + + @mock.patch('asyncio.proactor_events.logger') + def test_loop_writing_err(self, m_log): + err = self.loop._proactor.send.side_effect = OSError() + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._buffer = [b'da', b'ta'] + tr._loop_writing() + tr._fatal_error.assert_called_with( + err, + 'Fatal write error on pipe transport') + tr._conn_lost = 1 + + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + self.assertEqual(tr._buffer, None) + m_log.warning.assert_called_with('socket.send() raised exception.') + + def test_loop_writing_stop(self): + fut = self.loop.create_future() + fut.set_result(b'data') + + tr = self.socket_transport() + tr._write_fut = fut + tr._loop_writing(fut) + self.assertIsNone(tr._write_fut) + + def test_loop_writing_closing(self): + fut = self.loop.create_future() + fut.set_result(1) + + tr = self.socket_transport() + tr._write_fut = fut + tr.close() + tr._loop_writing(fut) + self.assertIsNone(tr._write_fut) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test_abort(self): + tr = self.socket_transport() + tr._force_close = mock.Mock() + tr.abort() + tr._force_close.assert_called_with(None) + + def test_close(self): + tr = self.socket_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertTrue(tr.is_closing()) + self.assertEqual(tr._conn_lost, 1) + + self.protocol.connection_lost.reset_mock() + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_close_write_fut(self): + tr = self.socket_transport() + tr._write_fut = mock.Mock() + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_close_buffer(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + tr.close() + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_close_invalid_sockobj(self): + tr = self.socket_transport() + self.sock.fileno.return_value = -1 + tr.close() + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertFalse(self.sock.shutdown.called) + + @mock.patch('asyncio.base_events.logger') + def test_fatal_error(self, m_logging): + tr = self.socket_transport() + tr._force_close = mock.Mock() + tr._fatal_error(None) + self.assertTrue(tr._force_close.called) + self.assertTrue(m_logging.error.called) + + def test_force_close(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + read_fut = tr._read_fut = mock.Mock() + write_fut = tr._write_fut = mock.Mock() + tr._force_close(None) + + read_fut.cancel.assert_called_with() + write_fut.cancel.assert_called_with() + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertEqual(None, tr._buffer) + self.assertEqual(tr._conn_lost, 1) + + def test_loop_writing_force_close(self): + exc_handler = mock.Mock() + self.loop.set_exception_handler(exc_handler) + fut = self.loop.create_future() + fut.set_result(1) + self.proactor.send.return_value = fut + + tr = self.socket_transport() + tr.write(b'data') + tr._force_close(None) + test_utils.run_briefly(self.loop) + exc_handler.assert_not_called() + + def test_force_close_idempotent(self): + tr = self.socket_transport() + tr._closing = True + tr._force_close(None) + test_utils.run_briefly(self.loop) + # See https://github.com/python/cpython/issues/89237 + # `protocol.connection_lost` should be called even if + # the transport was closed forcefully otherwise + # the resources held by protocol will never be freed + # and waiters will never be notified leading to hang. + self.assertTrue(self.protocol.connection_lost.called) + + def test_force_close_protocol_connection_lost_once(self): + tr = self.socket_transport() + self.assertFalse(self.protocol.connection_lost.called) + tr._closing = True + # Calling _force_close twice should not call + # protocol.connection_lost twice + tr._force_close(None) + tr._force_close(None) + test_utils.run_briefly(self.loop) + self.assertEqual(1, self.protocol.connection_lost.call_count) + + def test_close_protocol_connection_lost_once(self): + tr = self.socket_transport() + self.assertFalse(self.protocol.connection_lost.called) + # Calling close twice should not call + # protocol.connection_lost twice + tr.close() + tr.close() + test_utils.run_briefly(self.loop) + self.assertEqual(1, self.protocol.connection_lost.call_count) + + def test_fatal_error_2(self): + tr = self.socket_transport() + tr._buffer = [b'data'] + tr._force_close(None) + + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + self.assertEqual(None, tr._buffer) + + def test_call_connection_lost(self): + tr = self.socket_transport() + tr._call_connection_lost(None) + self.assertTrue(self.protocol.connection_lost.called) + self.assertTrue(self.sock.close.called) + + def test_write_eof(self): + tr = self.socket_transport() + self.assertTrue(tr.can_write_eof()) + tr.write_eof() + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.write_eof() + self.assertEqual(self.sock.shutdown.call_count, 1) + tr.close() + + def test_write_eof_buffer(self): + tr = self.socket_transport() + f = self.loop.create_future() + tr._loop._proactor.send.return_value = f + tr.write(b'data') + tr.write_eof() + self.assertTrue(tr._eof_written) + self.assertFalse(self.sock.shutdown.called) + tr._loop._proactor.send.assert_called_with(self.sock, b'data') + f.set_result(4) + self.loop._run_once() + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.close() + + def test_write_eof_write_pipe(self): + tr = _ProactorWritePipeTransport( + self.loop, self.sock, self.protocol) + self.assertTrue(tr.can_write_eof()) + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.loop._run_once() + self.assertTrue(self.sock.close.called) + tr.close() + + def test_write_eof_buffer_write_pipe(self): + tr = _ProactorWritePipeTransport(self.loop, self.sock, self.protocol) + f = self.loop.create_future() + tr._loop._proactor.send.return_value = f + tr.write(b'data') + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.sock.shutdown.called) + tr._loop._proactor.send.assert_called_with(self.sock, b'data') + f.set_result(4) + self.loop._run_once() + self.loop._run_once() + self.assertTrue(self.sock.close.called) + tr.close() + + def test_write_eof_duplex_pipe(self): + tr = _ProactorDuplexPipeTransport( + self.loop, self.sock, self.protocol) + self.assertFalse(tr.can_write_eof()) + with self.assertRaises(NotImplementedError): + tr.write_eof() + close_transport(tr) + + def test_pause_resume_reading(self): + tr = self.socket_transport() + index = 0 + msgs = [b'data1', b'data2', b'data3', b'data4', b'data5', b''] + reversed_msgs = list(reversed(msgs)) + + def recv_into(sock, data): + f = self.loop.create_future() + msg = reversed_msgs.pop() + + result = f.result + def monkey(): + data[:len(msg)] = msg + return result() + f.result = monkey + + f.set_result(len(msg)) + return f + + self.loop._proactor.recv_into.side_effect = recv_into + self.loop._run_once() + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + + for msg in msgs[:2]: + self.loop._run_once() + self.protocol.data_received.assert_called_with(bytearray(msg)) + + tr.pause_reading() + tr.pause_reading() + self.assertTrue(tr._paused) + self.assertFalse(tr.is_reading()) + for i in range(10): + self.loop._run_once() + self.protocol.data_received.assert_called_with(bytearray(msgs[1])) + + tr.resume_reading() + tr.resume_reading() + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + + for msg in msgs[2:4]: + self.loop._run_once() + self.protocol.data_received.assert_called_with(bytearray(msg)) + + tr.pause_reading() + tr.resume_reading() + self.loop.call_exception_handler = mock.Mock() + self.loop._run_once() + self.loop.call_exception_handler.assert_not_called() + self.protocol.data_received.assert_called_with(bytearray(msgs[4])) + tr.close() + + self.assertFalse(tr.is_reading()) + + def test_pause_reading_connection_made(self): + tr = self.socket_transport() + self.protocol.connection_made.side_effect = lambda _: tr.pause_reading() + test_utils.run_briefly(self.loop) + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + tr.resume_reading() + self.assertTrue(tr.is_reading()) + + tr.close() + self.assertFalse(tr.is_reading()) + + + def pause_writing_transport(self, high): + tr = self.socket_transport() + tr.set_write_buffer_limits(high=high) + + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertFalse(self.protocol.pause_writing.called) + self.assertFalse(self.protocol.resume_writing.called) + return tr + + def test_pause_resume_writing(self): + tr = self.pause_writing_transport(high=4) + + # write a large chunk, must pause writing + fut = self.loop.create_future() + self.loop._proactor.send.return_value = fut + tr.write(b'large data') + self.loop._run_once() + self.assertTrue(self.protocol.pause_writing.called) + + # flush the buffer + fut.set_result(None) + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertTrue(self.protocol.resume_writing.called) + + def test_pause_writing_2write(self): + tr = self.pause_writing_transport(high=4) + + # first short write, the buffer is not full (3 <= 4) + fut1 = self.loop.create_future() + self.loop._proactor.send.return_value = fut1 + tr.write(b'123') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 3) + self.assertFalse(self.protocol.pause_writing.called) + + # fill the buffer, must pause writing (6 > 4) + tr.write(b'abc') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 6) + self.assertTrue(self.protocol.pause_writing.called) + + def test_pause_writing_3write(self): + tr = self.pause_writing_transport(high=4) + + # first short write, the buffer is not full (1 <= 4) + fut = self.loop.create_future() + self.loop._proactor.send.return_value = fut + tr.write(b'1') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 1) + self.assertFalse(self.protocol.pause_writing.called) + + # second short write, the buffer is not full (3 <= 4) + tr.write(b'23') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 3) + self.assertFalse(self.protocol.pause_writing.called) + + # fill the buffer, must pause writing (6 > 4) + tr.write(b'abc') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 6) + self.assertTrue(self.protocol.pause_writing.called) + + def test_dont_pause_writing(self): + tr = self.pause_writing_transport(high=4) + + # write a large chunk which completes immediately, + # it should not pause writing + fut = self.loop.create_future() + fut.set_result(None) + self.loop._proactor.send.return_value = fut + tr.write(b'very large data') + self.loop._run_once() + self.assertEqual(tr.get_write_buffer_size(), 0) + self.assertFalse(self.protocol.pause_writing.called) + + +class ProactorDatagramTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.proactor = mock.Mock() + self.loop._proactor = self.proactor + self.protocol = test_utils.make_test_protocol(asyncio.DatagramProtocol) + self.sock = mock.Mock(spec_set=socket.socket) + self.sock.fileno.return_value = 7 + + def datagram_transport(self, address=None): + self.sock.getpeername.side_effect = None if address else OSError + transport = _ProactorDatagramTransport(self.loop, self.sock, + self.protocol, + address=address) + self.addCleanup(close_transport, transport) + return transport + + def test_sendto(self): + data = b'data' + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, data, addr=('0.0.0.0', 1234)) + self.assertFalse(transport._buffer) + self.assertEqual(0, transport._buffer_size) + + def test_sendto_bytearray(self): + data = bytearray(b'data') + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'data', addr=('0.0.0.0', 1234)) + + def test_sendto_memoryview(self): + data = memoryview(b'data') + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'data', addr=('0.0.0.0', 1234)) + + def test_sendto_no_data(self): + transport = self.datagram_transport() + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.proactor.sendto.called) + self.proactor.sendto.assert_called_with( + self.sock, b'', addr=('0.0.0.0', 1234)) + + def test_sendto_buffer(self): + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(b'data2', ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + + def test_sendto_buffer_bytearray(self): + data2 = bytearray(b'data2') + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_memoryview(self): + data2 = memoryview(b'data2') + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport._write_fut = object() + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.proactor.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + @mock.patch('asyncio.proactor_events.logger') + def test_sendto_exception(self, m_log): + data = b'data' + err = self.proactor.sendto.side_effect = RuntimeError() + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport.sendto(data, ()) + + self.assertTrue(transport._fatal_error.called) + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on datagram transport') + transport._conn_lost = 1 + + transport._address = ('123',) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + m_log.warning.assert_called_with('socket.sendto() raised exception.') + + def test_sendto_error_received(self): + data = b'data' + + self.sock.sendto.side_effect = ConnectionRefusedError + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport.sendto(data, ()) + + self.assertEqual(transport._conn_lost, 0) + self.assertFalse(transport._fatal_error.called) + + def test_sendto_error_received_connected(self): + data = b'data' + + self.proactor.send.side_effect = ConnectionRefusedError + + transport = self.datagram_transport(address=('0.0.0.0', 1)) + transport._fatal_error = mock.Mock() + transport.sendto(data) + + self.assertFalse(transport._fatal_error.called) + self.assertTrue(self.protocol.error_received.called) + + def test_sendto_str(self): + transport = self.datagram_transport() + self.assertRaises(TypeError, transport.sendto, 'str', ()) + + def test_sendto_connected_addr(self): + transport = self.datagram_transport(address=('0.0.0.0', 1)) + self.assertRaises( + ValueError, transport.sendto, b'str', ('0.0.0.0', 2)) + + def test_sendto_closing(self): + transport = self.datagram_transport(address=(1,)) + transport.close() + self.assertEqual(transport._conn_lost, 1) + transport.sendto(b'data', (1,)) + self.assertEqual(transport._conn_lost, 2) + + def test__loop_writing_closing(self): + transport = self.datagram_transport() + transport._closing = True + transport._loop_writing() + self.assertIsNone(transport._write_fut) + test_utils.run_briefly(self.loop) + self.sock.close.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + def test__loop_writing_exception(self): + err = self.proactor.sendto.side_effect = RuntimeError() + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._loop_writing() + + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on datagram transport') + + def test__loop_writing_error_received(self): + self.proactor.sendto.side_effect = ConnectionRefusedError + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._loop_writing() + + self.assertFalse(transport._fatal_error.called) + + def test__loop_writing_error_received_connection(self): + self.proactor.send.side_effect = ConnectionRefusedError + + transport = self.datagram_transport(address=('0.0.0.0', 1)) + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._loop_writing() + + self.assertFalse(transport._fatal_error.called) + self.assertTrue(self.protocol.error_received.called) + + @mock.patch('asyncio.base_events.logger.error') + def test_fatal_error_connected(self, m_exc): + transport = self.datagram_transport(address=('0.0.0.0', 1)) + err = ConnectionRefusedError() + transport._fatal_error(err) + self.assertFalse(self.protocol.error_received.called) + m_exc.assert_not_called() + + +class BaseProactorEventLoopTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + + self.sock = test_utils.mock_nonblocking_socket() + self.proactor = mock.Mock() + + self.ssock, self.csock = mock.Mock(), mock.Mock() + + with mock.patch('asyncio.proactor_events.socket.socketpair', + return_value=(self.ssock, self.csock)): + with mock.patch('signal.set_wakeup_fd'): + self.loop = BaseProactorEventLoop(self.proactor) + self.set_event_loop(self.loop) + + @mock.patch('asyncio.proactor_events.socket.socketpair') + def test_ctor(self, socketpair): + ssock, csock = socketpair.return_value = ( + mock.Mock(), mock.Mock()) + with mock.patch('signal.set_wakeup_fd'): + loop = BaseProactorEventLoop(self.proactor) + self.assertIs(loop._ssock, ssock) + self.assertIs(loop._csock, csock) + self.assertEqual(loop._internal_fds, 1) + loop.close() + + def test_close_self_pipe(self): + self.loop._close_self_pipe() + self.assertEqual(self.loop._internal_fds, 0) + self.assertTrue(self.ssock.close.called) + self.assertTrue(self.csock.close.called) + self.assertIsNone(self.loop._ssock) + self.assertIsNone(self.loop._csock) + + # Don't call close(): _close_self_pipe() cannot be called twice + self.loop._closed = True + + def test_close(self): + self.loop._close_self_pipe = mock.Mock() + self.loop.close() + self.assertTrue(self.loop._close_self_pipe.called) + self.assertTrue(self.proactor.close.called) + self.assertIsNone(self.loop._proactor) + + self.loop._close_self_pipe.reset_mock() + self.loop.close() + self.assertFalse(self.loop._close_self_pipe.called) + + def test_make_socket_transport(self): + tr = self.loop._make_socket_transport(self.sock, asyncio.Protocol()) + self.assertIsInstance(tr, _ProactorSocketTransport) + close_transport(tr) + + def test_loop_self_reading(self): + self.loop._loop_self_reading() + self.proactor.recv.assert_called_with(self.ssock, 4096) + self.proactor.recv.return_value.add_done_callback.assert_called_with( + self.loop._loop_self_reading) + + def test_loop_self_reading_fut(self): + fut = mock.Mock() + self.loop._self_reading_future = fut + self.loop._loop_self_reading(fut) + self.assertTrue(fut.result.called) + self.proactor.recv.assert_called_with(self.ssock, 4096) + self.proactor.recv.return_value.add_done_callback.assert_called_with( + self.loop._loop_self_reading) + + def test_loop_self_reading_exception(self): + self.loop.call_exception_handler = mock.Mock() + self.proactor.recv.side_effect = OSError() + self.loop._loop_self_reading() + self.assertTrue(self.loop.call_exception_handler.called) + + def test_write_to_self(self): + self.loop._write_to_self() + self.csock.send.assert_called_with(b'\0') + + def test_process_events(self): + self.loop._process_events([]) + + @mock.patch('asyncio.base_events.logger') + def test_create_server(self, m_log): + pf = mock.Mock() + call_soon = self.loop.call_soon = mock.Mock() + + self.loop._start_serving(pf, self.sock) + self.assertTrue(call_soon.called) + + # callback + loop = call_soon.call_args[0][0] + loop() + self.proactor.accept.assert_called_with(self.sock) + + # conn + fut = mock.Mock() + fut.result.return_value = (mock.Mock(), mock.Mock()) + + make_tr = self.loop._make_socket_transport = mock.Mock() + loop(fut) + self.assertTrue(fut.result.called) + self.assertTrue(make_tr.called) + + # exception + fut.result.side_effect = OSError() + loop(fut) + self.assertTrue(self.sock.close.called) + self.assertTrue(m_log.error.called) + + def test_create_server_cancel(self): + pf = mock.Mock() + call_soon = self.loop.call_soon = mock.Mock() + + self.loop._start_serving(pf, self.sock) + loop = call_soon.call_args[0][0] + + # cancelled + fut = self.loop.create_future() + fut.cancel() + loop(fut) + self.assertTrue(self.sock.close.called) + + def test_stop_serving(self): + sock1 = mock.Mock() + future1 = mock.Mock() + sock2 = mock.Mock() + future2 = mock.Mock() + self.loop._accept_futures = { + sock1.fileno(): future1, + sock2.fileno(): future2 + } + + self.loop._stop_serving(sock1) + self.assertTrue(sock1.close.called) + self.assertTrue(future1.cancel.called) + self.proactor._stop_serving.assert_called_with(sock1) + self.assertFalse(sock2.close.called) + self.assertFalse(future2.cancel.called) + + def datagram_transport(self): + self.protocol = test_utils.make_test_protocol(asyncio.DatagramProtocol) + return self.loop._make_datagram_transport(self.sock, self.protocol) + + def test_make_datagram_transport(self): + tr = self.datagram_transport() + self.assertIsInstance(tr, _ProactorDatagramTransport) + self.assertIsInstance(tr, asyncio.DatagramTransport) + close_transport(tr) + + def test_datagram_loop_writing(self): + tr = self.datagram_transport() + tr._buffer.appendleft((b'data', ('127.0.0.1', 12068))) + tr._loop_writing() + self.loop._proactor.sendto.assert_called_with(self.sock, b'data', addr=('127.0.0.1', 12068)) + self.loop._proactor.sendto.return_value.add_done_callback.\ + assert_called_with(tr._loop_writing) + + close_transport(tr) + + def test_datagram_loop_reading(self): + tr = self.datagram_transport() + tr._loop_reading() + self.loop._proactor.recvfrom.assert_called_with(self.sock, 256 * 1024) + self.assertFalse(self.protocol.datagram_received.called) + self.assertFalse(self.protocol.error_received.called) + close_transport(tr) + + def test_datagram_loop_reading_data(self): + res = self.loop.create_future() + res.set_result((b'data', ('127.0.0.1', 12068))) + + tr = self.datagram_transport() + tr._read_fut = res + tr._loop_reading(res) + self.loop._proactor.recvfrom.assert_called_with(self.sock, 256 * 1024) + self.protocol.datagram_received.assert_called_with(b'data', ('127.0.0.1', 12068)) + close_transport(tr) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_datagram_loop_reading_no_data(self): + res = self.loop.create_future() + res.set_result((b'', ('127.0.0.1', 12068))) + + tr = self.datagram_transport() + self.assertRaises(AssertionError, tr._loop_reading, res) + + tr.close = mock.Mock() + tr._read_fut = res + tr._loop_reading(res) + self.assertTrue(self.loop._proactor.recvfrom.called) + self.assertFalse(self.protocol.error_received.called) + self.assertFalse(tr.close.called) + close_transport(tr) + + def test_datagram_loop_reading_aborted(self): + err = self.loop._proactor.recvfrom.side_effect = ConnectionAbortedError() + + tr = self.datagram_transport() + tr._fatal_error = mock.Mock() + tr._protocol.error_received = mock.Mock() + tr._loop_reading() + tr._protocol.error_received.assert_called_with(err) + close_transport(tr) + + def test_datagram_loop_writing_aborted(self): + err = self.loop._proactor.sendto.side_effect = ConnectionAbortedError() + + tr = self.datagram_transport() + tr._fatal_error = mock.Mock() + tr._protocol.error_received = mock.Mock() + tr._buffer.appendleft((b'Hello', ('127.0.0.1', 12068))) + tr._loop_writing() + tr._protocol.error_received.assert_called_with(err) + close_transport(tr) + + +@unittest.skipIf(sys.platform != 'win32', + 'Proactor is supported on Windows only') +class ProactorEventLoopUnixSockSendfileTests(test_utils.TestCase): + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + + class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + + def connection_made(self, transport): + self.started = True + self.transport = transport + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + + async def wait_closed(self): + await self.fut + + @classmethod + def setUpClass(cls): + with open(os_helper.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + os_helper.unlink(os_helper.TESTFN) + super().tearDownClass() + + def setUp(self): + self.loop = asyncio.ProactorEventLoop() + self.set_event_loop(self.loop) + self.addCleanup(self.loop.close) + self.file = open(os_helper.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def make_socket(self, cleanup=True, blocking=False): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(blocking) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) + if cleanup: + self.addCleanup(sock.close) + return sock + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self): + sock = self.make_socket() + proto = self.MyProto(self.loop) + port = socket_helper.find_unused_port() + srv_sock = self.make_socket(cleanup=False) + srv_sock.bind(('127.0.0.1', port)) + server = self.run_loop(self.loop.create_server( + lambda: proto, sock=srv_sock)) + self.run_loop(self.loop.sock_connect(sock, srv_sock.getsockname())) + + def cleanup(): + if proto.transport is not None: + # can be None if the task was cancelled before + # connection_made callback + proto.transport.close() + self.run_loop(proto.wait_closed()) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test_sock_sendfile_not_a_file(self): + sock, proto = self.prepare() + f = object() + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_iobuffer(self): + sock, proto = self.prepare() + f = io.BytesIO() + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_not_regular_file(self): + sock, proto = self.prepare() + f = mock.Mock() + f.fileno.return_value = -1 + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py new file mode 100644 index 00000000000..29d3bd22705 --- /dev/null +++ b/Lib/test/test_asyncio/test_protocols.py @@ -0,0 +1,67 @@ +import unittest +from unittest import mock + +import asyncio + + +def tearDownModule(): + # not needed for the test file but added for uniformness with all other + # asyncio test files for the sake of unified cleanup + asyncio.events._set_event_loop_policy(None) + + +class ProtocolsAbsTests(unittest.TestCase): + + def test_base_protocol(self): + f = mock.Mock() + p = asyncio.BaseProtocol() + self.assertIsNone(p.connection_made(f)) + self.assertIsNone(p.connection_lost(f)) + self.assertIsNone(p.pause_writing()) + self.assertIsNone(p.resume_writing()) + self.assertNotHasAttr(p, '__dict__') + + def test_protocol(self): + f = mock.Mock() + p = asyncio.Protocol() + self.assertIsNone(p.connection_made(f)) + self.assertIsNone(p.connection_lost(f)) + self.assertIsNone(p.data_received(f)) + self.assertIsNone(p.eof_received()) + self.assertIsNone(p.pause_writing()) + self.assertIsNone(p.resume_writing()) + self.assertNotHasAttr(p, '__dict__') + + def test_buffered_protocol(self): + f = mock.Mock() + p = asyncio.BufferedProtocol() + self.assertIsNone(p.connection_made(f)) + self.assertIsNone(p.connection_lost(f)) + self.assertIsNone(p.get_buffer(100)) + self.assertIsNone(p.buffer_updated(150)) + self.assertIsNone(p.pause_writing()) + self.assertIsNone(p.resume_writing()) + self.assertNotHasAttr(p, '__dict__') + + def test_datagram_protocol(self): + f = mock.Mock() + dp = asyncio.DatagramProtocol() + self.assertIsNone(dp.connection_made(f)) + self.assertIsNone(dp.connection_lost(f)) + self.assertIsNone(dp.error_received(f)) + self.assertIsNone(dp.datagram_received(f, f)) + self.assertNotHasAttr(dp, '__dict__') + + def test_subprocess_protocol(self): + f = mock.Mock() + sp = asyncio.SubprocessProtocol() + self.assertIsNone(sp.connection_made(f)) + self.assertIsNone(sp.connection_lost(f)) + self.assertIsNone(sp.pipe_data_received(1, f)) + self.assertIsNone(sp.pipe_connection_lost(1, f)) + self.assertIsNone(sp.process_exited()) + self.assertNotHasAttr(sp, '__dict__') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py new file mode 100644 index 00000000000..54bbe79f81f --- /dev/null +++ b/Lib/test/test_asyncio/test_queues.py @@ -0,0 +1,725 @@ +"""Tests for queues.py""" + +import asyncio +import unittest +from types import GenericAlias + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class QueueBasicTests(unittest.IsolatedAsyncioTestCase): + + async def _test_repr_or_str(self, fn, expect_id): + """Test Queue's repr or str. + + fn is repr or str. expect_id is True if we expect the Queue's id to + appear in fn(Queue()). + """ + q = asyncio.Queue() + self.assertStartsWith(fn(q), ' 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task successfully finishes + await q.join() + + # Ensure get() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(get_task.done()) + with self.assertRaisesShutdown(): + await get_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + async def test_shutdown_nonempty(self): + # Test shutting down a non-empty queue + + # Setup full queue with 1 item, and join() and put() tasks + q = self.q_class(maxsize=1) + loop = asyncio.get_running_loop() + + q.put_nowait("data") + join_task = loop.create_task(q.join()) + put_task = loop.create_task(q.put("data2")) + + # Ensure put() task is not finished + await asyncio.sleep(0) + self.assertFalse(put_task.done()) + + # Perform shut-down + q.shutdown(immediate=False) # unfinished tasks: 1 -> 1 + + self.assertEqual(q.qsize(), 1) + + # Ensure put() task is finished, and raised ShutDown + await asyncio.sleep(0) + self.assertTrue(put_task.done()) + with self.assertRaisesShutdown(): + await put_task + + # Ensure get() succeeds on enqueued item + self.assertEqual(await q.get(), "data") + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task, and join() task succeeds + q.task_done() + + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate(self): + # Test immediately shutting down a queue + + # Setup queue with 1 item, and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + join_task = loop.create_task(q.join()) + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 1 -> 0 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there are no unfinished tasks + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + async def test_shutdown_immediate_with_unfinished(self): + # Test immediately shutting down a queue with unfinished tasks + + # Setup queue with 2 items (1 retrieved), and a join() task + q = self.q_class() + loop = asyncio.get_running_loop() + q.put_nowait("data") + q.put_nowait("data") + join_task = loop.create_task(q.join()) + self.assertEqual(await q.get(), "data") + + # Perform shut-down + q.shutdown(immediate=True) # unfinished tasks: 2 -> 1 + + self.assertEqual(q.qsize(), 0) + + # Ensure join() task is not finished + await asyncio.sleep(0) + self.assertFalse(join_task.done()) + + # Ensure put() and get() raise ShutDown + with self.assertRaisesShutdown(): + await q.put("data") + with self.assertRaisesShutdown(): + q.put_nowait("data") + + with self.assertRaisesShutdown(): + await q.get() + with self.assertRaisesShutdown(): + q.get_nowait() + + # Ensure there is 1 unfinished task + q.task_done() + with self.assertRaises( + ValueError, msg="Didn't appear to mark all tasks done" + ): + q.task_done() + + # Ensure join() task has successfully finished + await asyncio.sleep(0) + self.assertTrue(join_task.done()) + await join_task + + +class QueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.Queue + + +class LifoQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.LifoQueue + + +class PriorityQueueShutdownTests( + _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase +): + q_class = asyncio.PriorityQueue + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py new file mode 100644 index 00000000000..de489c2dc43 --- /dev/null +++ b/Lib/test/test_asyncio/test_runners.py @@ -0,0 +1,528 @@ +import _thread +import asyncio +import contextvars +import re +import signal +import sys +import threading +import unittest +from test.test_asyncio import utils as test_utils +from unittest import mock +from unittest.mock import patch + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +def interrupt_self(): + _thread.interrupt_main() + + +class TestPolicy(asyncio.events._AbstractEventLoopPolicy): + + def __init__(self, loop_factory): + self.loop_factory = loop_factory + self.loop = None + + def get_event_loop(self): + # shouldn't ever be called by asyncio.run() + raise RuntimeError + + def new_event_loop(self): + return self.loop_factory() + + def set_event_loop(self, loop): + if loop is not None: + # we want to check if the loop is closed + # in BaseTest.tearDown + self.loop = loop + + +class BaseTest(unittest.TestCase): + + def new_loop(self): + loop = asyncio.BaseEventLoop() + loop._process_events = mock.Mock() + # Mock waking event loop from select + loop._write_to_self = mock.Mock() + loop._write_to_self.return_value = None + loop._selector = mock.Mock() + loop._selector.select.return_value = () + loop.shutdown_ag_run = False + + async def shutdown_asyncgens(): + loop.shutdown_ag_run = True + loop.shutdown_asyncgens = shutdown_asyncgens + + return loop + + def setUp(self): + super().setUp() + + policy = TestPolicy(self.new_loop) + asyncio.events._set_event_loop_policy(policy) + + def tearDown(self): + policy = asyncio.events._get_event_loop_policy() + if policy.loop is not None: + self.assertTrue(policy.loop.is_closed()) + self.assertTrue(policy.loop.shutdown_ag_run) + + asyncio.events._set_event_loop_policy(None) + super().tearDown() + + +class RunTests(BaseTest): + + def test_asyncio_run_return(self): + async def main(): + await asyncio.sleep(0) + return 42 + + self.assertEqual(asyncio.run(main()), 42) + + def test_asyncio_run_raises(self): + async def main(): + await asyncio.sleep(0) + raise ValueError('spam') + + with self.assertRaisesRegex(ValueError, 'spam'): + asyncio.run(main()) + + def test_asyncio_run_only_coro(self): + for o in {1, lambda: None}: + with self.subTest(obj=o), \ + self.assertRaisesRegex(TypeError, + 'an awaitable is required'): + asyncio.run(o) + + def test_asyncio_run_debug(self): + async def main(expected): + loop = asyncio.get_event_loop() + self.assertIs(loop.get_debug(), expected) + + asyncio.run(main(False), debug=False) + asyncio.run(main(True), debug=True) + with mock.patch('asyncio.coroutines._is_debug_mode', lambda: True): + asyncio.run(main(True)) + asyncio.run(main(False), debug=False) + with mock.patch('asyncio.coroutines._is_debug_mode', lambda: False): + asyncio.run(main(True), debug=True) + asyncio.run(main(False)) + + def test_asyncio_run_from_running_loop(self): + async def main(): + coro = main() + try: + asyncio.run(coro) + finally: + coro.close() # Suppress ResourceWarning + + with self.assertRaisesRegex(RuntimeError, + 'cannot be called from a running'): + asyncio.run(main()) + + def test_asyncio_run_cancels_hanging_tasks(self): + lo_task = None + + async def leftover(): + await asyncio.sleep(0.1) + + async def main(): + nonlocal lo_task + lo_task = asyncio.create_task(leftover()) + return 123 + + self.assertEqual(asyncio.run(main()), 123) + self.assertTrue(lo_task.done()) + + def test_asyncio_run_reports_hanging_tasks_errors(self): + lo_task = None + call_exc_handler_mock = mock.Mock() + + async def leftover(): + try: + await asyncio.sleep(0.1) + except asyncio.CancelledError: + 1 / 0 + + async def main(): + loop = asyncio.get_running_loop() + loop.call_exception_handler = call_exc_handler_mock + + nonlocal lo_task + lo_task = asyncio.create_task(leftover()) + return 123 + + self.assertEqual(asyncio.run(main()), 123) + self.assertTrue(lo_task.done()) + + call_exc_handler_mock.assert_called_with({ + 'message': test_utils.MockPattern(r'asyncio.run.*shutdown'), + 'task': lo_task, + 'exception': test_utils.MockInstanceOf(ZeroDivisionError) + }) + + def test_asyncio_run_closes_gens_after_hanging_tasks_errors(self): + spinner = None + lazyboy = None + + class FancyExit(Exception): + pass + + async def fidget(): + while True: + yield 1 + await asyncio.sleep(1) + + async def spin(): + nonlocal spinner + spinner = fidget() + try: + async for the_meaning_of_life in spinner: # NoQA + pass + except asyncio.CancelledError: + 1 / 0 + + async def main(): + loop = asyncio.get_running_loop() + loop.call_exception_handler = mock.Mock() + + nonlocal lazyboy + lazyboy = asyncio.create_task(spin()) + raise FancyExit + + with self.assertRaises(FancyExit): + asyncio.run(main()) + + self.assertTrue(lazyboy.done()) + + self.assertIsNone(spinner.ag_frame) + self.assertFalse(spinner.ag_running) + + def test_asyncio_run_set_event_loop(self): + #See https://github.com/python/cpython/issues/93896 + + async def main(): + await asyncio.sleep(0) + return 42 + + policy = asyncio.events._get_event_loop_policy() + policy.set_event_loop = mock.Mock() + asyncio.run(main()) + self.assertTrue(policy.set_event_loop.called) + + def test_asyncio_run_without_uncancel(self): + # See https://github.com/python/cpython/issues/95097 + class Task: + def __init__(self, loop, coro, **kwargs): + self._task = asyncio.Task(coro, loop=loop, **kwargs) + + def cancel(self, *args, **kwargs): + return self._task.cancel(*args, **kwargs) + + def add_done_callback(self, *args, **kwargs): + return self._task.add_done_callback(*args, **kwargs) + + def remove_done_callback(self, *args, **kwargs): + return self._task.remove_done_callback(*args, **kwargs) + + @property + def _asyncio_future_blocking(self): + return self._task._asyncio_future_blocking + + def result(self, *args, **kwargs): + return self._task.result(*args, **kwargs) + + def done(self, *args, **kwargs): + return self._task.done(*args, **kwargs) + + def cancelled(self, *args, **kwargs): + return self._task.cancelled(*args, **kwargs) + + def exception(self, *args, **kwargs): + return self._task.exception(*args, **kwargs) + + def get_loop(self, *args, **kwargs): + return self._task.get_loop(*args, **kwargs) + + def set_name(self, *args, **kwargs): + return self._task.set_name(*args, **kwargs) + + async def main(): + interrupt_self() + await asyncio.Event().wait() + + def new_event_loop(): + loop = self.new_loop() + loop.set_task_factory(Task) + return loop + + asyncio.events._set_event_loop_policy(TestPolicy(new_event_loop)) + with self.assertRaises(asyncio.CancelledError): + asyncio.run(main()) + + def test_asyncio_run_loop_factory(self): + factory = mock.Mock() + loop = factory.return_value = self.new_loop() + + async def main(): + self.assertEqual(asyncio.get_running_loop(), loop) + + asyncio.run(main(), loop_factory=factory) + factory.assert_called_once_with() + + def test_loop_factory_default_event_loop(self): + async def main(): + if sys.platform == "win32": + self.assertIsInstance(asyncio.get_running_loop(), asyncio.ProactorEventLoop) + else: + self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) + + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + + +class RunnerTests(BaseTest): + + def test_non_debug(self): + with asyncio.Runner(debug=False) as runner: + self.assertFalse(runner.get_loop().get_debug()) + + def test_debug(self): + with asyncio.Runner(debug=True) as runner: + self.assertTrue(runner.get_loop().get_debug()) + + def test_custom_factory(self): + loop = mock.Mock() + with asyncio.Runner(loop_factory=lambda: loop) as runner: + self.assertIs(runner.get_loop(), loop) + + def test_run(self): + async def f(): + await asyncio.sleep(0) + return 'done' + + with asyncio.Runner() as runner: + self.assertEqual('done', runner.run(f())) + loop = runner.get_loop() + + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + runner.get_loop() + + self.assertTrue(loop.is_closed()) + + def test_run_non_coro(self): + with asyncio.Runner() as runner: + with self.assertRaisesRegex( + TypeError, + "an awaitable is required" + ): + runner.run(123) + + def test_run_future(self): + with asyncio.Runner() as runner: + fut = runner.get_loop().create_future() + fut.set_result('done') + self.assertEqual('done', runner.run(fut)) + + def test_run_awaitable(self): + class MyAwaitable: + def __await__(self): + return self.run().__await__() + + @staticmethod + async def run(): + return 'done' + + with asyncio.Runner() as runner: + self.assertEqual('done', runner.run(MyAwaitable())) + + def test_explicit_close(self): + runner = asyncio.Runner() + loop = runner.get_loop() + runner.close() + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + runner.get_loop() + + self.assertTrue(loop.is_closed()) + + def test_double_close(self): + runner = asyncio.Runner() + loop = runner.get_loop() + + runner.close() + self.assertTrue(loop.is_closed()) + + # the second call is no-op + runner.close() + self.assertTrue(loop.is_closed()) + + def test_second_with_block_raises(self): + ret = [] + + async def f(arg): + ret.append(arg) + + runner = asyncio.Runner() + with runner: + runner.run(f(1)) + + with self.assertRaisesRegex( + RuntimeError, + "Runner is closed" + ): + with runner: + runner.run(f(2)) + + self.assertEqual([1], ret) + + def test_run_keeps_context(self): + cvar = contextvars.ContextVar("cvar", default=-1) + + async def f(val): + old = cvar.get() + await asyncio.sleep(0) + cvar.set(val) + return old + + async def get_context(): + return contextvars.copy_context() + + with asyncio.Runner() as runner: + self.assertEqual(-1, runner.run(f(1))) + self.assertEqual(1, runner.run(f(2))) + + self.assertEqual(2, runner.run(get_context()).get(cvar)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - RuntimeWarning for unawaited coroutine not triggered + def test_recursive_run(self): + async def g(): + pass + + async def f(): + runner.run(g()) + + with asyncio.Runner() as runner: + with self.assertWarnsRegex( + RuntimeWarning, + "coroutine .+ was never awaited", + ): + with self.assertRaisesRegex( + RuntimeError, + re.escape( + "Runner.run() cannot be called from a running event loop" + ), + ): + runner.run(f()) + + def test_interrupt_call_soon(self): + # The only case when task is not suspended by waiting a future + # or another task + assert threading.current_thread() is threading.main_thread() + + async def coro(): + with self.assertRaises(asyncio.CancelledError): + while True: + await asyncio.sleep(0) + raise asyncio.CancelledError() + + with asyncio.Runner() as runner: + runner.get_loop().call_later(0.1, interrupt_self) + with self.assertRaises(KeyboardInterrupt): + runner.run(coro()) + + def test_interrupt_wait(self): + # interrupting when waiting a future cancels both future and main task + assert threading.current_thread() is threading.main_thread() + + async def coro(fut): + with self.assertRaises(asyncio.CancelledError): + await fut + raise asyncio.CancelledError() + + with asyncio.Runner() as runner: + fut = runner.get_loop().create_future() + runner.get_loop().call_later(0.1, interrupt_self) + + with self.assertRaises(KeyboardInterrupt): + runner.run(coro(fut)) + + self.assertTrue(fut.cancelled()) + + def test_interrupt_cancelled_task(self): + # interrupting cancelled main task doesn't raise KeyboardInterrupt + assert threading.current_thread() is threading.main_thread() + + async def subtask(task): + await asyncio.sleep(0) + task.cancel() + interrupt_self() + + async def coro(): + asyncio.create_task(subtask(asyncio.current_task())) + await asyncio.sleep(10) + + with asyncio.Runner() as runner: + with self.assertRaises(asyncio.CancelledError): + runner.run(coro()) + + def test_signal_install_not_supported_ok(self): + # signal.signal() can throw if the "main thread" doesn't have signals enabled + assert threading.current_thread() is threading.main_thread() + + async def coro(): + pass + + with asyncio.Runner() as runner: + with patch.object( + signal, + "signal", + side_effect=ValueError( + "signal only works in main thread of the main interpreter" + ) + ): + runner.run(coro()) + + def test_set_event_loop_called_once(self): + # See https://github.com/python/cpython/issues/95736 + async def coro(): + pass + + policy = asyncio.events._get_event_loop_policy() + policy.set_event_loop = mock.Mock() + runner = asyncio.Runner() + runner.run(coro()) + runner.run(coro()) + + self.assertEqual(1, policy.set_event_loop.call_count) + runner.close() + + def test_no_repr_is_call_on_the_task_result(self): + # See https://github.com/python/cpython/issues/112559. + class MyResult: + def __init__(self): + self.repr_count = 0 + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + async def coro(): + return MyResult() + + + with asyncio.Runner() as runner: + result = runner.run(coro()) + + self.assertEqual(0, result.repr_count) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py new file mode 100644 index 00000000000..4bb5d4fb816 --- /dev/null +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -0,0 +1,1660 @@ +"""Tests for selector_events.py""" + +import collections +import selectors +import socket +import sys +import unittest +from asyncio import selector_events +from unittest import mock + +try: + import ssl +except ImportError: + ssl = None + +import asyncio +from asyncio.selector_events import (BaseSelectorEventLoop, + _SelectorDatagramTransport, + _SelectorSocketTransport, + _SelectorTransport) +from test.test_asyncio import utils as test_utils + +MOCK_ANY = mock.ANY + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class TestBaseSelectorEventLoop(BaseSelectorEventLoop): + + def _make_self_pipe(self): + self._ssock = mock.Mock() + self._csock = mock.Mock() + self._internal_fds += 1 + + def _close_self_pipe(self): + pass + + +def list_to_buffer(l=()): + buffer = collections.deque() + buffer.extend((memoryview(i) for i in l)) + return buffer + + + +def close_transport(transport): + # Don't call transport.close() because the event loop and the selector + # are mocked + if transport._sock is None: + return + transport._sock.close() + transport._sock = None + + +class BaseSelectorEventLoopTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.selector = mock.Mock() + self.selector.select.return_value = [] + self.loop = TestBaseSelectorEventLoop(self.selector) + self.set_event_loop(self.loop) + + def test_make_socket_transport(self): + m = mock.Mock() + self.loop.add_reader = mock.Mock() + self.loop._ensure_fd_no_transport = mock.Mock() + transport = self.loop._make_socket_transport(m, asyncio.Protocol()) + self.assertIsInstance(transport, _SelectorSocketTransport) + self.assertEqual(self.loop._ensure_fd_no_transport.call_count, 1) + + # Calling repr() must not fail when the event loop is closed + self.loop.close() + repr(transport) + + close_transport(transport) + + @mock.patch('asyncio.selector_events.ssl', None) + @mock.patch('asyncio.sslproto.ssl', None) + def test_make_ssl_transport_without_ssl_error(self): + m = mock.Mock() + self.loop.add_reader = mock.Mock() + self.loop.add_writer = mock.Mock() + self.loop.remove_reader = mock.Mock() + self.loop.remove_writer = mock.Mock() + self.loop._ensure_fd_no_transport = mock.Mock() + with self.assertRaises(RuntimeError): + self.loop._make_ssl_transport(m, m, m, m) + self.assertEqual(self.loop._ensure_fd_no_transport.call_count, 1) + + def test_close(self): + class EventLoop(BaseSelectorEventLoop): + def _make_self_pipe(self): + self._ssock = mock.Mock() + self._csock = mock.Mock() + self._internal_fds += 1 + + self.loop = EventLoop(self.selector) + self.set_event_loop(self.loop) + + ssock = self.loop._ssock + ssock.fileno.return_value = 7 + csock = self.loop._csock + csock.fileno.return_value = 1 + remove_reader = self.loop._remove_reader = mock.Mock() + + self.loop._selector.close() + self.loop._selector = selector = mock.Mock() + self.assertFalse(self.loop.is_closed()) + + self.loop.close() + self.assertTrue(self.loop.is_closed()) + self.assertIsNone(self.loop._selector) + self.assertIsNone(self.loop._csock) + self.assertIsNone(self.loop._ssock) + selector.close.assert_called_with() + ssock.close.assert_called_with() + csock.close.assert_called_with() + remove_reader.assert_called_with(7) + + # it should be possible to call close() more than once + self.loop.close() + self.loop.close() + + # operation blocked when the loop is closed + f = self.loop.create_future() + self.assertRaises(RuntimeError, self.loop.run_forever) + self.assertRaises(RuntimeError, self.loop.run_until_complete, f) + fd = 0 + def callback(): + pass + self.assertRaises(RuntimeError, self.loop.add_reader, fd, callback) + self.assertRaises(RuntimeError, self.loop.add_writer, fd, callback) + + def test_close_no_selector(self): + self.loop.remove_reader = mock.Mock() + self.loop._selector.close() + self.loop._selector = None + self.loop.close() + self.assertIsNone(self.loop._selector) + + def test_read_from_self_tryagain(self): + self.loop._ssock.recv.side_effect = BlockingIOError + self.assertIsNone(self.loop._read_from_self()) + + def test_read_from_self_exception(self): + self.loop._ssock.recv.side_effect = OSError + self.assertRaises(OSError, self.loop._read_from_self) + + def test_write_to_self_tryagain(self): + self.loop._csock.send.side_effect = BlockingIOError + with test_utils.disable_logger(): + self.assertIsNone(self.loop._write_to_self()) + + def test_write_to_self_exception(self): + # _write_to_self() swallows OSError + self.loop._csock.send.side_effect = RuntimeError() + self.assertRaises(RuntimeError, self.loop._write_to_self) + + @mock.patch('socket.getaddrinfo') + def test_sock_connect_resolve_using_socket_params(self, m_gai): + addr = ('need-resolution.com', 8080) + for sock_type in [socket.SOCK_STREAM, socket.SOCK_DGRAM]: + with self.subTest(sock_type): + sock = test_utils.mock_nonblocking_socket(type=sock_type) + + m_gai.side_effect = \ + lambda *args: [(None, None, None, None, ('127.0.0.1', 0))] + + con = self.loop.create_task(self.loop.sock_connect(sock, addr)) + self.loop.run_until_complete(con) + m_gai.assert_called_with( + addr[0], addr[1], sock.family, sock.type, sock.proto, 0) + + self.loop.run_until_complete(con) + sock.connect.assert_called_with(('127.0.0.1', 0)) + + def test_add_reader(self): + self.loop._selector.get_map.return_value = {} + cb = lambda: True + self.loop.add_reader(1, cb) + + self.assertTrue(self.loop._selector.register.called) + fd, mask, (r, w) = self.loop._selector.register.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_READ, mask) + self.assertEqual(cb, r._callback) + self.assertIsNone(w) + + def test_add_reader_existing(self): + reader = mock.Mock() + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (reader, writer))} + cb = lambda: True + self.loop.add_reader(1, cb) + + self.assertTrue(reader.cancel.called) + self.assertFalse(self.loop._selector.register.called) + self.assertTrue(self.loop._selector.modify.called) + fd, mask, (r, w) = self.loop._selector.modify.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask) + self.assertEqual(cb, r._callback) + self.assertEqual(writer, w) + + def test_add_reader_existing_writer(self): + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (None, writer))} + cb = lambda: True + self.loop.add_reader(1, cb) + + self.assertFalse(self.loop._selector.register.called) + self.assertTrue(self.loop._selector.modify.called) + fd, mask, (r, w) = self.loop._selector.modify.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask) + self.assertEqual(cb, r._callback) + self.assertEqual(writer, w) + + def test_remove_reader(self): + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (None, None))} + self.assertFalse(self.loop.remove_reader(1)) + + self.assertTrue(self.loop._selector.unregister.called) + + def test_remove_reader_read_write(self): + reader = mock.Mock() + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ | selectors.EVENT_WRITE, + (reader, writer))} + self.assertTrue( + self.loop.remove_reader(1)) + + self.assertFalse(self.loop._selector.unregister.called) + self.assertEqual( + (1, selectors.EVENT_WRITE, (None, writer)), + self.loop._selector.modify.call_args[0]) + + def test_remove_reader_unknown(self): + self.loop._selector.get_map.return_value = {} + self.assertFalse( + self.loop.remove_reader(1)) + + def test_add_writer(self): + self.loop._selector.get_map.return_value = {} + cb = lambda: True + self.loop.add_writer(1, cb) + + self.assertTrue(self.loop._selector.register.called) + fd, mask, (r, w) = self.loop._selector.register.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_WRITE, mask) + self.assertIsNone(r) + self.assertEqual(cb, w._callback) + + def test_add_writer_existing(self): + reader = mock.Mock() + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (reader, writer))} + cb = lambda: True + self.loop.add_writer(1, cb) + + self.assertTrue(writer.cancel.called) + self.assertFalse(self.loop._selector.register.called) + self.assertTrue(self.loop._selector.modify.called) + fd, mask, (r, w) = self.loop._selector.modify.call_args[0] + self.assertEqual(1, fd) + self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask) + self.assertEqual(reader, r) + self.assertEqual(cb, w._callback) + + def test_remove_writer(self): + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (None, None))} + self.assertFalse(self.loop.remove_writer(1)) + + self.assertTrue(self.loop._selector.unregister.called) + + def test_remove_writer_read_write(self): + reader = mock.Mock() + writer = mock.Mock() + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ | selectors.EVENT_WRITE, + (reader, writer))} + self.assertTrue( + self.loop.remove_writer(1)) + + self.assertFalse(self.loop._selector.unregister.called) + self.assertEqual( + (1, selectors.EVENT_READ, (reader, None)), + self.loop._selector.modify.call_args[0]) + + def test_remove_writer_unknown(self): + self.loop._selector.get_map.return_value = {} + self.assertFalse( + self.loop.remove_writer(1)) + + def test_process_events_read(self): + reader = mock.Mock() + reader._cancelled = False + + self.loop._add_callback = mock.Mock() + self.loop._process_events( + [(selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (reader, None)), + selectors.EVENT_READ)]) + self.assertTrue(self.loop._add_callback.called) + self.loop._add_callback.assert_called_with(reader) + + def test_process_events_read_cancelled(self): + reader = mock.Mock() + reader.cancelled = True + + self.loop._remove_reader = mock.Mock() + self.loop._process_events( + [(selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (reader, None)), + selectors.EVENT_READ)]) + self.loop._remove_reader.assert_called_with(1) + + def test_process_events_write(self): + writer = mock.Mock() + writer._cancelled = False + + self.loop._add_callback = mock.Mock() + self.loop._process_events( + [(selectors.SelectorKey(1, 1, selectors.EVENT_WRITE, + (None, writer)), + selectors.EVENT_WRITE)]) + self.loop._add_callback.assert_called_with(writer) + + def test_process_events_write_cancelled(self): + writer = mock.Mock() + writer.cancelled = True + self.loop._remove_writer = mock.Mock() + + self.loop._process_events( + [(selectors.SelectorKey(1, 1, selectors.EVENT_WRITE, + (None, writer)), + selectors.EVENT_WRITE)]) + self.loop._remove_writer.assert_called_with(1) + + def test_accept_connection_zero_one(self): + for backlog in [0, 1]: + sock = mock.Mock() + sock.accept.return_value = (mock.Mock(), mock.Mock()) + with self.subTest(backlog): + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog + 1) + + def test_accept_connection_multiple(self): + sock = mock.Mock() + sock.accept.return_value = (mock.Mock(), mock.Mock()) + backlog = 100 + # Mock the coroutine generation for a connection to prevent + # warnings related to un-awaited coroutines. _accept_connection2 + # is an async function that is patched with AsyncMock. create_task + # creates a task out of coroutine returned by AsyncMock, so use + # asyncio.sleep(0) to ensure created tasks are complete to avoid + # task pending warnings. + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog + 1) + + def test_accept_connection_skip_connectionabortederror(self): + sock = mock.Mock() + + def mock_sock_accept(): + # mock accept(2) returning -ECONNABORTED every-other + # time that it's called. This applies most to OpenBSD + # whose sockets generate this errno more reproducibly than + # Linux and other OS. + if sock.accept.call_count % 2 == 0: + raise ConnectionAbortedError + return (mock.Mock(), mock.Mock()) + + sock.accept.side_effect = mock_sock_accept + backlog = 100 + # test that _accept_connection's loop calls sock.accept + # all 100 times, continuing past ConnectionAbortedError + # instead of unnecessarily returning early + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + # as in test_accept_connection_multiple avoid task pending + # warnings by using asyncio.sleep(0) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog + 1) + +class SelectorTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.sock = mock.Mock(socket.socket) + self.sock.fileno.return_value = 7 + + def create_transport(self): + transport = _SelectorTransport(self.loop, self.sock, self.protocol, + None) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + tr = self.create_transport() + self.assertIs(tr._loop, self.loop) + self.assertIs(tr._sock, self.sock) + self.assertIs(tr._sock_fd, 7) + + def test_abort(self): + tr = self.create_transport() + tr._force_close = mock.Mock() + + tr.abort() + tr._force_close.assert_called_with(None) + + def test_close(self): + tr = self.create_transport() + tr.close() + + self.assertTrue(tr.is_closing()) + self.assertEqual(1, self.loop.remove_reader_count[7]) + self.protocol.connection_lost(None) + self.assertEqual(tr._conn_lost, 1) + + tr.close() + self.assertEqual(tr._conn_lost, 1) + self.assertEqual(1, self.loop.remove_reader_count[7]) + + def test_close_write_buffer(self): + tr = self.create_transport() + tr._buffer.extend(b'data') + tr.close() + + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.connection_lost.called) + + def test_force_close(self): + tr = self.create_transport() + tr._buffer.extend(b'1') + self.loop._add_reader(7, mock.sentinel) + self.loop._add_writer(7, mock.sentinel) + tr._force_close(None) + + self.assertTrue(tr.is_closing()) + self.assertEqual(tr._buffer, list_to_buffer()) + self.assertFalse(self.loop.readers) + self.assertFalse(self.loop.writers) + + # second close should not remove reader + tr._force_close(None) + self.assertFalse(self.loop.readers) + self.assertEqual(1, self.loop.remove_reader_count[7]) + + @mock.patch('asyncio.log.logger.error') + def test_fatal_error(self, m_exc): + exc = OSError() + tr = self.create_transport() + tr._force_close = mock.Mock() + tr._fatal_error(exc) + + m_exc.assert_not_called() + + tr._force_close.assert_called_with(exc) + + @mock.patch('asyncio.log.logger.error') + def test_fatal_error_custom_exception(self, m_exc): + class MyError(Exception): + pass + exc = MyError() + tr = self.create_transport() + tr._force_close = mock.Mock() + tr._fatal_error(exc) + + m_exc.assert_called_with( + test_utils.MockPattern( + 'Fatal error on transport\nprotocol:.*\ntransport:.*'), + exc_info=(MyError, MOCK_ANY, MOCK_ANY)) + + tr._force_close.assert_called_with(exc) + + def test_connection_lost(self): + exc = OSError() + tr = self.create_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + tr._call_connection_lost(exc) + + self.protocol.connection_lost.assert_called_with(exc) + self.sock.close.assert_called_with() + self.assertIsNone(tr._sock) + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test__add_reader(self): + tr = self.create_transport() + tr._buffer.extend(b'1') + tr._add_reader(7, mock.sentinel) + self.assertTrue(self.loop.readers) + + tr._force_close(None) + + self.assertTrue(tr.is_closing()) + self.assertFalse(self.loop.readers) + + # can not add readers after closing + tr._add_reader(7, mock.sentinel) + self.assertFalse(self.loop.readers) + + +class SelectorSocketTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.sock = mock.Mock(socket.socket) + self.sock_fd = self.sock.fileno.return_value = 7 + + def socket_transport(self, waiter=None, sendmsg=False): + transport = _SelectorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + if sendmsg: + transport._write_ready = transport._write_sendmsg + else: + transport._write_ready = transport._write_send + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + waiter = self.loop.create_future() + tr = self.socket_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.loop.assert_reader(7, tr._read_ready) + test_utils.run_briefly(self.loop) + self.protocol.connection_made.assert_called_with(tr) + + def test_ctor_with_waiter(self): + waiter = self.loop.create_future() + self.socket_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.assertIsNone(waiter.result()) + + def test_pause_resume_reading(self): + tr = self.socket_transport() + test_utils.run_briefly(self.loop) + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + self.loop.assert_reader(7, tr._read_ready) + + tr.pause_reading() + tr.pause_reading() + self.assertTrue(tr._paused) + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + tr.resume_reading() + tr.resume_reading() + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + self.loop.assert_reader(7, tr._read_ready) + + tr.close() + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + def test_pause_reading_connection_made(self): + tr = self.socket_transport() + self.protocol.connection_made.side_effect = lambda _: tr.pause_reading() + test_utils.run_briefly(self.loop) + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + tr.resume_reading() + self.assertTrue(tr.is_reading()) + self.loop.assert_reader(7, tr._read_ready) + + tr.close() + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + + def test_read_eof_received_error(self): + transport = self.socket_transport() + transport.close = mock.Mock() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + + self.protocol.eof_received.side_effect = LookupError() + + self.sock.recv.return_value = b'' + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertTrue(transport._fatal_error.called) + + def test_data_received_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.data_received.side_effect = LookupError() + + self.sock.recv.return_value = b'data' + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.data_received.called) + + def test_read_ready(self): + transport = self.socket_transport() + + self.sock.recv.return_value = b'data' + transport._read_ready() + + self.protocol.data_received.assert_called_with(b'data') + + def test_read_ready_eof(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv.return_value = b'' + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + transport.close.assert_called_with() + + def test_read_ready_eof_keep_open(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv.return_value = b'' + self.protocol.eof_received.return_value = True + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertFalse(transport.close.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain(self, m_exc): + self.sock.recv.side_effect = BlockingIOError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain_interrupted(self, m_exc): + self.sock.recv.side_effect = InterruptedError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_conn_reset(self, m_exc): + err = self.sock.recv.side_effect = ConnectionResetError() + + transport = self.socket_transport() + transport._force_close = mock.Mock() + with test_utils.disable_logger(): + transport._read_ready() + transport._force_close.assert_called_with(err) + + @mock.patch('logging.exception') + def test_read_ready_err(self, m_exc): + err = self.sock.recv.side_effect = OSError() + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal read error on socket transport') + + def test_write(self): + data = b'data' + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport.write(data) + self.sock.send.assert_called_with(data) + + def test_write_bytearray(self): + data = bytearray(b'data') + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport.write(data) + self.sock.send.assert_called_with(data) + self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated. + + def test_write_memoryview(self): + data = memoryview(b'data') + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport.write(data) + self.sock.send.assert_called_with(data) + + def test_write_no_data(self): + transport = self.socket_transport() + transport._buffer.append(memoryview(b'data')) + transport.write(b'') + self.assertFalse(self.sock.send.called) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_buffer(self): + transport = self.socket_transport() + transport._buffer.append(b'data1') + transport.write(b'data2') + self.assertFalse(self.sock.send.called) + self.assertEqual(list_to_buffer([b'data1', b'data2']), + transport._buffer) + + def test_write_partial(self): + data = b'data' + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + def test_write_partial_bytearray(self): + data = bytearray(b'data') + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated. + + def test_write_partial_memoryview(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + def test_write_partial_none(self): + data = b'data' + self.sock.send.return_value = 0 + self.sock.fileno.return_value = 7 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_tryagain(self): + self.sock.send.side_effect = BlockingIOError + + data = b'data' + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_sendmsg_no_data(self): + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = 0 + transport = self.socket_transport(sendmsg=True) + transport._buffer.append(memoryview(b'data')) + transport.write(b'') + self.assertFalse(self.sock.sendmsg.called) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_writelines_sendmsg_full(self): + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = len(data) + + transport = self.socket_transport(sendmsg=True) + transport.writelines([data]) + self.assertTrue(self.sock.sendmsg.called) + self.assertFalse(self.loop.writers) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_writelines_sendmsg_partial(self): + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = 2 + + transport = self.socket_transport(sendmsg=True) + transport.writelines([data]) + self.assertTrue(self.sock.sendmsg.called) + self.assertTrue(self.loop.writers) + + def test_writelines_send_full(self): + data = memoryview(b'data') + self.sock.send.return_value = len(data) + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport.writelines([data]) + self.assertTrue(self.sock.send.called) + self.assertFalse(self.loop.writers) + + def test_writelines_send_partial(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport.writelines([data]) + self.assertTrue(self.sock.send.called) + self.assertTrue(self.loop.writers) + + def test_writelines_pauses_protocol(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport._high_water = 1 + transport.writelines([data]) + self.assertTrue(self.protocol.pause_writing.called) + self.assertTrue(self.sock.send.called) + self.assertTrue(self.loop.writers) + + def test_writelines_after_connection_lost(self): + # GH-136234 + transport = self.socket_transport() + self.sock.send = mock.Mock() + self.sock.send.side_effect = ConnectionResetError + transport.write(b'data1') # Will fail immediately, causing connection lost + + transport.writelines([b'data2']) + self.assertFalse(transport._buffer) + self.assertFalse(self.loop.writers) + + test_utils.run_briefly(self.loop) # Allow _call_connection_lost to run + transport.writelines([b'data2']) + self.assertFalse(transport._buffer) + self.assertFalse(self.loop.writers) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_full(self): + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + self.sock.sendmsg.return_value = len(data) + + transport = self.socket_transport(sendmsg=True) + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertFalse(self.loop.writers) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_partial(self): + + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + # Sent partial data + self.sock.sendmsg.return_value = 2 + + transport = self.socket_transport(sendmsg=True) + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertTrue(self.loop.writers) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_half_buffer(self): + data = [memoryview(b'data1'), memoryview(b'data2')] + self.sock.sendmsg = mock.Mock() + # Sent partial data + self.sock.sendmsg.return_value = 2 + + transport = self.socket_transport(sendmsg=True) + transport._buffer.extend(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertTrue(self.loop.writers) + self.assertEqual(list_to_buffer([b'ta1', b'data2']), transport._buffer) + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_OSError(self): + data = memoryview(b'data') + self.sock.sendmsg = mock.Mock() + err = self.sock.sendmsg.side_effect = OSError() + + transport = self.socket_transport(sendmsg=True) + transport._fatal_error = mock.Mock() + transport._buffer.extend(data) + # Calls _fatal_error and clears the buffer + transport._write_ready() + self.assertTrue(self.sock.sendmsg.called) + self.assertFalse(self.loop.writers) + self.assertEqual(list_to_buffer([]), transport._buffer) + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on socket transport') + + @mock.patch('asyncio.selector_events.logger') + def test_write_exception(self, m_log): + err = self.sock.send.side_effect = OSError() + + data = b'data' + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport.write(data) + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on socket transport') + transport._conn_lost = 1 + + self.sock.reset_mock() + transport.write(data) + self.assertFalse(self.sock.send.called) + self.assertEqual(transport._conn_lost, 2) + transport.write(data) + transport.write(data) + transport.write(data) + transport.write(data) + m_log.warning.assert_called_with('socket.send() raised exception.') + + def test_write_str(self): + transport = self.socket_transport() + self.assertRaises(TypeError, transport.write, 'str') + + def test_write_closing(self): + transport = self.socket_transport() + transport.close() + self.assertEqual(transport._conn_lost, 1) + transport.write(b'data') + self.assertEqual(transport._conn_lost, 2) + + def test_write_ready(self): + data = b'data' + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.send.called) + self.assertFalse(self.loop.writers) + + def test_write_ready_closing(self): + data = memoryview(b'data') + self.sock.send.return_value = len(data) + + transport = self.socket_transport() + transport._closing = True + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.assertTrue(self.sock.send.called) + self.assertFalse(self.loop.writers) + self.sock.close.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_write_ready_no_data(self): + transport = self.socket_transport() + # This is an internal error. + self.assertRaises(AssertionError, transport._write_ready) + + def test_write_ready_partial(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + + transport = self.socket_transport() + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + + def test_write_ready_partial_none(self): + data = b'data' + self.sock.send.return_value = 0 + + transport = self.socket_transport() + transport._buffer.append(data) + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(list_to_buffer([b'data']), transport._buffer) + + def test_write_ready_tryagain(self): + self.sock.send.side_effect = BlockingIOError + + transport = self.socket_transport() + buffer = list_to_buffer([b'data1', b'data2']) + transport._buffer = buffer + self.loop._add_writer(7, transport._write_ready) + transport._write_ready() + + self.loop.assert_writer(7, transport._write_ready) + self.assertEqual(buffer, transport._buffer) + + def test_write_ready_exception(self): + err = self.sock.send.side_effect = OSError() + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._buffer.extend(b'data') + transport._write_ready() + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on socket transport') + + def test_write_eof(self): + tr = self.socket_transport() + self.assertTrue(tr.can_write_eof()) + tr.write_eof() + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.write_eof() + self.assertEqual(self.sock.shutdown.call_count, 1) + tr.close() + + def test_write_eof_buffer(self): + tr = self.socket_transport() + self.sock.send.side_effect = BlockingIOError + tr.write(b'data') + tr.write_eof() + self.assertEqual(tr._buffer, list_to_buffer([b'data'])) + self.assertTrue(tr._eof) + self.assertFalse(self.sock.shutdown.called) + self.sock.send.side_effect = lambda _: 4 + tr._write_ready() + self.assertTrue(self.sock.send.called) + self.sock.shutdown.assert_called_with(socket.SHUT_WR) + tr.close() + + def test_write_eof_after_close(self): + tr = self.socket_transport() + tr.close() + self.loop.run_until_complete(asyncio.sleep(0)) + tr.write_eof() + + @mock.patch('asyncio.base_events.logger') + def test_transport_close_remove_writer(self, m_log): + remove_writer = self.loop._remove_writer = mock.Mock() + + transport = self.socket_transport() + transport.close() + remove_writer.assert_called_with(self.sock_fd) + + def test_write_buffer_after_close(self): + # gh-115514: If the transport is closed while: + # * Transport write buffer is not empty + # * Transport is paused + # * Protocol has data in its buffer, like SSLProtocol in self._outgoing + # The data is still written out. + + # Also tested with real SSL transport in + # test.test_asyncio.test_ssl.TestSSL.test_remote_shutdown_receives_trailing_data + + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + def _resume_writing(): + transport.write(b"data") + self.protocol.resume_writing.side_effect = None + + self.protocol.resume_writing.side_effect = _resume_writing + + transport = self.socket_transport() + transport._high_water = 1 + + transport.write(data) + + self.assertTrue(transport._protocol_paused) + self.assertTrue(self.sock.send.called) + self.loop.assert_writer(7, transport._write_ready) + + transport.close() + + # not called, we still have data in write buffer + self.assertFalse(self.protocol.connection_lost.called) + + self.loop.writers[7]._run() + # during this ^ run, the _resume_writing mock above was called and added more data + + self.assertEqual(transport.get_write_buffer_size(), 2) + self.loop.writers[7]._run() + + self.assertEqual(transport.get_write_buffer_size(), 0) + self.assertTrue(self.protocol.connection_lost.called) + +class SelectorSocketTransportBufferedProtocolTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + self.protocol = test_utils.make_test_protocol(asyncio.BufferedProtocol) + self.buf = bytearray(1) + self.protocol.get_buffer.side_effect = lambda hint: self.buf + + self.sock = mock.Mock(socket.socket) + self.sock_fd = self.sock.fileno.return_value = 7 + + def socket_transport(self, waiter=None): + transport = _SelectorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + waiter = self.loop.create_future() + tr = self.socket_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.loop.assert_reader(7, tr._read_ready) + test_utils.run_briefly(self.loop) + self.protocol.connection_made.assert_called_with(tr) + + def test_get_buffer_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.get_buffer.side_effect = LookupError() + + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertFalse(self.protocol.buffer_updated.called) + + def test_get_buffer_zerosized(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.get_buffer.side_effect = lambda hint: bytearray(0) + + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertFalse(self.protocol.buffer_updated.called) + + def test_proto_type_switch(self): + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + transport = self.socket_transport() + + self.sock.recv.return_value = b'data' + transport._read_ready() + + self.protocol.data_received.assert_called_with(b'data') + + # switch protocol to a BufferedProtocol + + buf_proto = test_utils.make_test_protocol(asyncio.BufferedProtocol) + buf = bytearray(4) + buf_proto.get_buffer.side_effect = lambda hint: buf + + transport.set_protocol(buf_proto) + + self.sock.recv_into.return_value = 10 + transport._read_ready() + + buf_proto.get_buffer.assert_called_with(-1) + buf_proto.buffer_updated.assert_called_with(10) + + def test_buffer_updated_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.buffer_updated.side_effect = LookupError() + + self.sock.recv_into.return_value = 10 + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertTrue(self.protocol.buffer_updated.called) + + def test_read_eof_received_error(self): + transport = self.socket_transport() + transport.close = mock.Mock() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + + self.protocol.eof_received.side_effect = LookupError() + + self.sock.recv_into.return_value = 0 + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertTrue(transport._fatal_error.called) + + def test_read_ready(self): + transport = self.socket_transport() + + self.sock.recv_into.return_value = 10 + transport._read_ready() + + self.protocol.get_buffer.assert_called_with(-1) + self.protocol.buffer_updated.assert_called_with(10) + + def test_read_ready_eof(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv_into.return_value = 0 + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + transport.close.assert_called_with() + + def test_read_ready_eof_keep_open(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv_into.return_value = 0 + self.protocol.eof_received.return_value = True + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertFalse(transport.close.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain(self, m_exc): + self.sock.recv_into.side_effect = BlockingIOError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain_interrupted(self, m_exc): + self.sock.recv_into.side_effect = InterruptedError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_conn_reset(self, m_exc): + err = self.sock.recv_into.side_effect = ConnectionResetError() + + transport = self.socket_transport() + transport._force_close = mock.Mock() + with test_utils.disable_logger(): + transport._read_ready() + transport._force_close.assert_called_with(err) + + @mock.patch('logging.exception') + def test_read_ready_err(self, m_exc): + err = self.sock.recv_into.side_effect = OSError() + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal read error on socket transport') + + +class SelectorDatagramTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.DatagramProtocol) + self.sock = mock.Mock(spec_set=socket.socket) + self.sock.fileno.return_value = 7 + + def datagram_transport(self, address=None): + self.sock.getpeername.side_effect = None if address else OSError + transport = _SelectorDatagramTransport(self.loop, self.sock, + self.protocol, + address=address) + self.addCleanup(close_transport, transport) + return transport + + def test_read_ready(self): + transport = self.datagram_transport() + + self.sock.recvfrom.return_value = (b'data', ('0.0.0.0', 1234)) + transport._read_ready() + + self.protocol.datagram_received.assert_called_with( + b'data', ('0.0.0.0', 1234)) + + def test_transport_inheritance(self): + transport = self.datagram_transport() + self.assertIsInstance(transport, asyncio.DatagramTransport) + + def test_read_ready_tryagain(self): + transport = self.datagram_transport() + + self.sock.recvfrom.side_effect = BlockingIOError + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + def test_read_ready_err(self): + transport = self.datagram_transport() + + err = self.sock.recvfrom.side_effect = RuntimeError() + transport._fatal_error = mock.Mock() + transport._read_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal read error on datagram transport') + + def test_read_ready_oserr(self): + transport = self.datagram_transport() + + err = self.sock.recvfrom.side_effect = OSError() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + self.protocol.error_received.assert_called_with(err) + + def test_sendto(self): + data = b'data' + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + + def test_sendto_bytearray(self): + data = bytearray(b'data') + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + + def test_sendto_memoryview(self): + data = memoryview(b'data') + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234))) + + def test_sendto_no_data(self): + transport = self.datagram_transport() + transport.sendto(b'', ('0.0.0.0', 1234)) + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (b'', ('0.0.0.0', 1234))) + + def test_sendto_buffer(self): + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(b'data2', ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + + def test_sendto_buffer_bytearray(self): + data2 = bytearray(b'data2') + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_memoryview(self): + data2 = memoryview(b'data2') + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'data2', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_buffer_nodata(self): + data2 = b'' + transport = self.datagram_transport() + transport._buffer.append((b'data1', ('0.0.0.0', 12345))) + transport.sendto(data2, ('0.0.0.0', 12345)) + self.assertFalse(self.sock.sendto.called) + self.assertEqual( + [(b'data1', ('0.0.0.0', 12345)), + (b'', ('0.0.0.0', 12345))], + list(transport._buffer)) + self.assertIsInstance(transport._buffer[1][0], bytes) + + def test_sendto_tryagain(self): + data = b'data' + + self.sock.sendto.side_effect = BlockingIOError + + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual( + [(b'data', ('0.0.0.0', 12345))], list(transport._buffer)) + + @mock.patch('asyncio.selector_events.logger') + def test_sendto_exception(self, m_log): + data = b'data' + err = self.sock.sendto.side_effect = RuntimeError() + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport.sendto(data, ()) + + self.assertTrue(transport._fatal_error.called) + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on datagram transport') + transport._conn_lost = 1 + + transport._address = ('123',) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + transport.sendto(data) + m_log.warning.assert_called_with('socket.send() raised exception.') + + def test_sendto_error_received(self): + data = b'data' + + self.sock.sendto.side_effect = ConnectionRefusedError + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport.sendto(data, ()) + + self.assertEqual(transport._conn_lost, 0) + self.assertFalse(transport._fatal_error.called) + + def test_sendto_error_received_connected(self): + data = b'data' + + self.sock.send.side_effect = ConnectionRefusedError + + transport = self.datagram_transport(address=('0.0.0.0', 1)) + transport._fatal_error = mock.Mock() + transport.sendto(data) + + self.assertFalse(transport._fatal_error.called) + self.assertTrue(self.protocol.error_received.called) + + def test_sendto_str(self): + transport = self.datagram_transport() + self.assertRaises(TypeError, transport.sendto, 'str', ()) + + def test_sendto_connected_addr(self): + transport = self.datagram_transport(address=('0.0.0.0', 1)) + self.assertRaises( + ValueError, transport.sendto, b'str', ('0.0.0.0', 2)) + + def test_sendto_closing(self): + transport = self.datagram_transport(address=(1,)) + transport.close() + self.assertEqual(transport._conn_lost, 1) + transport.sendto(b'data', (1,)) + self.assertEqual(transport._conn_lost, 2) + + def test_sendto_sendto_ready(self): + data = b'data' + + # First queue up the buffer by having the socket blocked + self.sock.sendto.side_effect = BlockingIOError + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + # Now let the socket send the buffer + self.sock.sendto.side_effect = None + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertFalse(self.loop.writers) + self.assertFalse(transport._buffer) + self.assertEqual(transport._buffer_size, 0) + + def test_sendto_sendto_ready_blocked(self): + data = b'data' + + # First queue up the buffer by having the socket blocked + self.sock.sendto.side_effect = BlockingIOError + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + # Now try to send the buffer, it will be added to buffer again if it fails + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertTrue(self.loop.writers) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + def test_sendto_ready(self): + data = b'data' + self.sock.sendto.return_value = len(data) + + transport = self.datagram_transport() + transport._buffer.append((data, ('0.0.0.0', 12345))) + self.loop._add_writer(7, transport._sendto_ready) + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertFalse(self.loop.writers) + + def test_sendto_ready_closing(self): + data = b'data' + self.sock.send.return_value = len(data) + + transport = self.datagram_transport() + transport._closing = True + transport._buffer.append((data, ())) + self.loop._add_writer(7, transport._sendto_ready) + transport._sendto_ready() + self.sock.sendto.assert_called_with(data, ()) + self.assertFalse(self.loop.writers) + self.sock.close.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + def test_sendto_ready_no_data(self): + transport = self.datagram_transport() + self.loop._add_writer(7, transport._sendto_ready) + transport._sendto_ready() + self.assertFalse(self.sock.sendto.called) + self.assertFalse(self.loop.writers) + + def test_sendto_ready_tryagain(self): + self.sock.sendto.side_effect = BlockingIOError + + transport = self.datagram_transport() + transport._buffer.extend([(b'data1', ()), (b'data2', ())]) + self.loop._add_writer(7, transport._sendto_ready) + transport._sendto_ready() + + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual( + [(b'data1', ()), (b'data2', ())], + list(transport._buffer)) + + def test_sendto_ready_exception(self): + err = self.sock.sendto.side_effect = RuntimeError() + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._sendto_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal write error on datagram transport') + + def test_sendto_ready_error_received(self): + self.sock.sendto.side_effect = ConnectionRefusedError + + transport = self.datagram_transport() + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._sendto_ready() + + self.assertFalse(transport._fatal_error.called) + + def test_sendto_ready_error_received_connection(self): + self.sock.send.side_effect = ConnectionRefusedError + + transport = self.datagram_transport(address=('0.0.0.0', 1)) + transport._fatal_error = mock.Mock() + transport._buffer.append((b'data', ())) + transport._sendto_ready() + + self.assertFalse(transport._fatal_error.called) + self.assertTrue(self.protocol.error_received.called) + + @mock.patch('asyncio.base_events.logger.error') + def test_fatal_error_connected(self, m_exc): + transport = self.datagram_transport(address=('0.0.0.0', 1)) + err = ConnectionRefusedError() + transport._fatal_error(err) + self.assertFalse(self.protocol.error_received.called) + m_exc.assert_not_called() + + @mock.patch('asyncio.base_events.logger.error') + def test_fatal_error_connected_custom_error(self, m_exc): + class MyException(Exception): + pass + transport = self.datagram_transport(address=('0.0.0.0', 1)) + err = MyException() + transport._fatal_error(err) + self.assertFalse(self.protocol.error_received.called) + m_exc.assert_called_with( + test_utils.MockPattern( + 'Fatal error on transport\nprotocol:.*\ntransport:.*'), + exc_info=(MyException, MOCK_ANY, MOCK_ANY)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py new file mode 100644 index 00000000000..dcd963b3355 --- /dev/null +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -0,0 +1,585 @@ +"""Tests for sendfile functionality.""" + +import asyncio +import errno +import os +import socket +import sys +import tempfile +import unittest +from asyncio import base_events +from asyncio import constants +from unittest import mock +from test import support +from test.support import os_helper +from test.support import socket_helper +from test.test_asyncio import utils as test_utils + +try: + import ssl +except ImportError: + ssl = None + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class MySendfileProto(asyncio.Protocol): + + def __init__(self, loop=None, close_after=0): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = loop.create_future() + self.done = loop.create_future() + self.data = bytearray() + self.close_after = close_after + + def _assert_state(self, *expected): + if self.state not in expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + + def eof_received(self): + self._assert_state('CONNECTED') + self.state = 'EOF' + + def connection_lost(self, exc): + self._assert_state('CONNECTED', 'EOF') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + def data_received(self, data): + self._assert_state('CONNECTED') + self.nbytes += len(data) + self.data.extend(data) + super().data_received(data) + if self.close_after and self.nbytes >= self.close_after: + self.transport.close() + + +class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + + def connection_made(self, transport): + self.started = True + self.transport = transport + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + + async def wait_closed(self): + await self.fut + + +class SendfileBase: + + # Linux >= 6.10 seems buffering up to 17 pages of data. + # So DATA should be large enough to make this test reliable even with a + # 64 KiB page configuration. + DATA = b"x" * (1024 * 17 * 64 + 1) + # Reduce socket buffer size to test on relative small data sets. + BUF_SIZE = 4 * 1024 # 4 KiB + + def create_event_loop(self): + raise NotImplementedError + + @classmethod + def setUpClass(cls): + with open(os_helper.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + os_helper.unlink(os_helper.TESTFN) + super().tearDownClass() + + def setUp(self): + self.file = open(os_helper.TESTFN, 'rb') + self.addCleanup(self.file.close) + self.loop = self.create_event_loop() + self.set_event_loop(self.loop) + super().setUp() + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + +class SockSendfileMixin(SendfileBase): + + @classmethod + def setUpClass(cls): + cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16 + super().setUpClass() + + @classmethod + def tearDownClass(cls): + constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize + super().tearDownClass() + + def make_socket(self, cleanup=True): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(False) + if cleanup: + self.addCleanup(sock.close) + return sock + + def reduce_receive_buffer_size(self, sock): + # Reduce receive socket buffer size to test on relative + # small data sets. + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.BUF_SIZE) + + def reduce_send_buffer_size(self, sock, transport=None): + # Reduce send socket buffer size to test on relative small data sets. + + # On macOS, SO_SNDBUF is reset by connect(). So this method + # should be called after the socket is connected. + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.BUF_SIZE) + + if transport is not None: + transport.set_write_buffer_limits(high=self.BUF_SIZE) + + def prepare_socksendfile(self): + proto = MyProto(self.loop) + port = socket_helper.find_unused_port() + srv_sock = self.make_socket(cleanup=False) + srv_sock.bind((socket_helper.HOST, port)) + server = self.run_loop(self.loop.create_server( + lambda: proto, sock=srv_sock)) + self.reduce_receive_buffer_size(srv_sock) + + sock = self.make_socket() + self.run_loop(self.loop.sock_connect(sock, ('127.0.0.1', port))) + self.reduce_send_buffer_size(sock) + + def cleanup(): + if proto.transport is not None: + # can be None if the task was cancelled before + # connection_made callback + proto.transport.close() + self.run_loop(proto.wait_closed()) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test_sock_sendfile_success(self): + sock, proto = self.prepare_socksendfile() + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sock_sendfile_with_offset_and_count(self): + sock, proto = self.prepare_socksendfile() + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file, + 1000, 2000)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(proto.data, self.DATA[1000:3000]) + self.assertEqual(self.file.tell(), 3000) + self.assertEqual(ret, 2000) + + def test_sock_sendfile_zero_size(self): + sock, proto = self.prepare_socksendfile() + with tempfile.TemporaryFile() as f: + ret = self.run_loop(self.loop.sock_sendfile(sock, f, + 0, None)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_mix_with_regular_send(self): + buf = b"mix_regular_send" * (4 * 1024) # 64 KiB + sock, proto = self.prepare_socksendfile() + self.run_loop(self.loop.sock_sendall(sock, buf)) + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + self.run_loop(self.loop.sock_sendall(sock, buf)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + expected = buf + self.DATA + buf + self.assertEqual(proto.data, expected) + self.assertEqual(self.file.tell(), len(self.DATA)) + + +class SendfileMixin(SendfileBase): + + # Note: sendfile via SSL transport is equal to sendfile fallback + + def prepare_sendfile(self, *, is_ssl=False, close_after=0): + port = socket_helper.find_unused_port() + srv_proto = MySendfileProto(loop=self.loop, + close_after=close_after) + if is_ssl: + if not ssl: + self.skipTest("No ssl module") + srv_ctx = test_utils.simple_server_sslcontext() + cli_ctx = test_utils.simple_client_sslcontext() + else: + srv_ctx = None + cli_ctx = None + srv_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv_sock.bind((socket_helper.HOST, port)) + server = self.run_loop(self.loop.create_server( + lambda: srv_proto, sock=srv_sock, ssl=srv_ctx)) + self.reduce_receive_buffer_size(srv_sock) + + if is_ssl: + server_hostname = socket_helper.HOST + else: + server_hostname = None + cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + cli_sock.connect((socket_helper.HOST, port)) + + cli_proto = MySendfileProto(loop=self.loop) + tr, pr = self.run_loop(self.loop.create_connection( + lambda: cli_proto, sock=cli_sock, + ssl=cli_ctx, server_hostname=server_hostname)) + self.reduce_send_buffer_size(cli_sock, transport=tr) + + def cleanup(): + srv_proto.transport.close() + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.run_loop(cli_proto.done) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + return srv_proto, cli_proto + + @unittest.skipIf(sys.platform == 'win32', "UDP sockets are not supported") + def test_sendfile_not_supported(self): + tr, pr = self.run_loop( + self.loop.create_datagram_endpoint( + asyncio.DatagramProtocol, + family=socket.AF_INET)) + try: + with self.assertRaisesRegex(RuntimeError, "not supported"): + self.run_loop( + self.loop.sendfile(tr, self.file)) + self.assertEqual(0, self.file.tell()) + finally: + # don't use self.addCleanup because it produces resource warning + tr.close() + + def test_sendfile(self): + srv_proto, cli_proto = self.prepare_sendfile() + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_force_fallback(self): + srv_proto, cli_proto = self.prepare_sendfile() + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_force_unsupported_native(self): + if sys.platform == 'win32': + if isinstance(self.loop, asyncio.ProactorEventLoop): + self.skipTest("Fails on proactor event loop") + srv_proto, cli_proto = self.prepare_sendfile() + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not supported"): + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, + fallback=False)) + + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(srv_proto.nbytes, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sendfile_ssl(self): + srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_for_closing_transp(self): + srv_proto, cli_proto = self.prepare_sendfile() + cli_proto.transport.close() + with self.assertRaisesRegex(RuntimeError, "is closing"): + self.run_loop(self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + self.assertEqual(srv_proto.nbytes, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sendfile_pre_and_post_data(self): + srv_proto, cli_proto = self.prepare_sendfile() + PREFIX = b'PREFIX__' * 1024 # 8 KiB + SUFFIX = b'--SUFFIX' * 1024 # 8 KiB + cli_proto.transport.write(PREFIX) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.write(SUFFIX) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.data, PREFIX + self.DATA + SUFFIX) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_ssl_pre_and_post_data(self): + srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True) + PREFIX = b'zxcvbnm' * 1024 + SUFFIX = b'0987654321' * 1024 + cli_proto.transport.write(PREFIX) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.write(SUFFIX) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.data, PREFIX + self.DATA + SUFFIX) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_partial(self): + srv_proto, cli_proto = self.prepare_sendfile() + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, 1000, 100)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, 100) + self.assertEqual(srv_proto.nbytes, 100) + self.assertEqual(srv_proto.data, self.DATA[1000:1100]) + self.assertEqual(self.file.tell(), 1100) + + def test_sendfile_ssl_partial(self): + srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, 1000, 100)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, 100) + self.assertEqual(srv_proto.nbytes, 100) + self.assertEqual(srv_proto.data, self.DATA[1000:1100]) + self.assertEqual(self.file.tell(), 1100) + + def test_sendfile_close_peer_after_receiving(self): + srv_proto, cli_proto = self.prepare_sendfile( + close_after=len(self.DATA)) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_ssl_close_peer_after_receiving(self): + srv_proto, cli_proto = self.prepare_sendfile( + is_ssl=True, close_after=len(self.DATA)) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + # On Solaris, lowering SO_RCVBUF on a TCP connection after it has been + # established has no effect. Due to its age, this bug affects both Oracle + # Solaris as well as all other OpenSolaris forks (unless they fixed it + # themselves). + @unittest.skipIf(sys.platform.startswith('sunos'), + "Doesn't work on Solaris") + def test_sendfile_close_peer_in_the_middle_of_receiving(self): + srv_proto, cli_proto = self.prepare_sendfile(close_after=1024) + with self.assertRaises(ConnectionError): + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + + self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), + srv_proto.nbytes) + if not (sys.platform == 'win32' + and isinstance(self.loop, asyncio.ProactorEventLoop)): + # On Windows, Proactor uses transmitFile, which does not update tell() + self.assertTrue(1024 <= self.file.tell() < len(self.DATA), + self.file.tell()) + self.assertTrue(cli_proto.transport.is_closing()) + + def test_sendfile_fallback_close_peer_in_the_middle_of_receiving(self): + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + srv_proto, cli_proto = self.prepare_sendfile(close_after=1024) + with self.assertRaises(ConnectionError): + try: + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + except OSError as e: + # macOS may raise OSError of EPROTOTYPE when writing to a + # socket that is in the process of closing down. + if e.errno == errno.EPROTOTYPE and sys.platform == "darwin": + raise ConnectionError + else: + raise + + self.run_loop(srv_proto.done) + + self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), + srv_proto.nbytes) + self.assertTrue(1024 <= self.file.tell() < len(self.DATA), + self.file.tell()) + + @unittest.skipIf(not hasattr(os, 'sendfile'), + "Don't have native sendfile support") + def test_sendfile_prevents_bare_write(self): + srv_proto, cli_proto = self.prepare_sendfile() + fut = self.loop.create_future() + + async def coro(): + fut.set_result(None) + return await self.loop.sendfile(cli_proto.transport, self.file) + + t = self.loop.create_task(coro()) + self.run_loop(fut) + with self.assertRaisesRegex(RuntimeError, + "sendfile is in progress"): + cli_proto.transport.write(b'data') + ret = self.run_loop(t) + self.assertEqual(ret, len(self.DATA)) + + def test_sendfile_no_fallback_for_fallback_transport(self): + transport = mock.Mock() + transport.is_closing.side_effect = lambda: False + transport._sendfile_compatible = constants._SendfileMode.FALLBACK + with self.assertRaisesRegex(RuntimeError, 'fallback is disabled'): + self.loop.run_until_complete( + self.loop.sendfile(transport, None, fallback=False)) + + +class SendfileTestsBase(SendfileMixin, SockSendfileMixin): + pass + + +if sys.platform == 'win32': + + class SelectEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop() + + class ProactorEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.ProactorEventLoop() + +else: + import selectors + + if hasattr(selectors, 'KqueueSelector'): + class KqueueEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop( + selectors.KqueueSelector()) + + if hasattr(selectors, 'EpollSelector'): + class EPollEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.EpollSelector()) + + if hasattr(selectors, 'PollSelector'): + class PollEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.PollSelector()) + + # Should always exist. + class SelectEventLoopTests(SendfileTestsBase, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py new file mode 100644 index 00000000000..5bd0f7e2af4 --- /dev/null +++ b/Lib/test/test_asyncio/test_server.py @@ -0,0 +1,352 @@ +import asyncio +import os +import socket +import time +import threading +import unittest + +from test.support import socket_helper +from test.test_asyncio import utils as test_utils +from test.test_asyncio import functional as func_tests + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class BaseStartServer(func_tests.FunctionalTestCaseMixin): + + def new_loop(self): + raise NotImplementedError + + def test_start_server_1(self): + HELLO_MSG = b'1' * 1024 * 5 + b'\n' + + def client(sock, addr): + for i in range(10): + time.sleep(0.2) + if srv.is_serving(): + break + else: + raise RuntimeError + + sock.settimeout(2) + sock.connect(addr) + sock.send(HELLO_MSG) + sock.recv_all(1) + sock.close() + + async def serve(reader, writer): + await reader.readline() + main_task.cancel() + writer.write(b'1') + writer.close() + await writer.wait_closed() + + async def main(srv): + async with srv: + await srv.serve_forever() + + srv = self.loop.run_until_complete(asyncio.start_server( + serve, socket_helper.HOSTv4, 0, start_serving=False)) + + self.assertFalse(srv.is_serving()) + + main_task = self.loop.create_task(main(srv)) + + addr = srv.sockets[0].getsockname() + with self.assertRaises(asyncio.CancelledError): + with self.tcp_client(lambda sock: client(sock, addr)): + self.loop.run_until_complete(main_task) + + self.assertEqual(srv.sockets, ()) + + self.assertIsNone(srv._sockets) + self.assertIsNone(srv._waiters) + self.assertFalse(srv.is_serving()) + + with self.assertRaisesRegex(RuntimeError, r'is closed'): + self.loop.run_until_complete(srv.serve_forever()) + + +class SelectorStartServerTests(BaseStartServer, unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + @socket_helper.skip_unless_bind_unix_socket + def test_start_unix_server_1(self): + HELLO_MSG = b'1' * 1024 * 5 + b'\n' + started = threading.Event() + + def client(sock, addr): + sock.settimeout(2) + started.wait(5) + sock.connect(addr) + sock.send(HELLO_MSG) + sock.recv_all(1) + sock.close() + + async def serve(reader, writer): + await reader.readline() + main_task.cancel() + writer.write(b'1') + writer.close() + await writer.wait_closed() + + async def main(srv): + async with srv: + self.assertFalse(srv.is_serving()) + await srv.start_serving() + self.assertTrue(srv.is_serving()) + started.set() + await srv.serve_forever() + + with test_utils.unix_socket_path() as addr: + srv = self.loop.run_until_complete(asyncio.start_unix_server( + serve, addr, start_serving=False)) + + main_task = self.loop.create_task(main(srv)) + + with self.assertRaises(asyncio.CancelledError): + with self.unix_client(lambda sock: client(sock, addr)): + self.loop.run_until_complete(main_task) + + self.assertEqual(srv.sockets, ()) + + self.assertIsNone(srv._sockets) + self.assertIsNone(srv._waiters) + self.assertFalse(srv.is_serving()) + + with self.assertRaisesRegex(RuntimeError, r'is closed'): + self.loop.run_until_complete(srv.serve_forever()) + + +class TestServer2(unittest.IsolatedAsyncioTestCase): + + async def test_wait_closed_basic(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + # active count = 0, not closed: should block + task1 = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task1.done()) + + # active count != 0, not closed: should block + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + task2 = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task1.done()) + self.assertFalse(task2.done()) + + srv.close() + await asyncio.sleep(0) + # active count != 0, closed: should block + task3 = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task1.done()) + self.assertFalse(task2.done()) + self.assertFalse(task3.done()) + + wr.close() + await wr.wait_closed() + # active count == 0, closed: should unblock + await task1 + await task2 + await task3 + await srv.wait_closed() # Return immediately + + async def test_wait_closed_race(self): + # Test a regression in 3.12.0, should be fixed in 3.12.1 + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + loop = asyncio.get_running_loop() + loop.call_soon(srv.close) + loop.call_soon(wr.close) + await srv.wait_closed() + + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + fut.set_result((rd, wr)) + await wr.wait_closed() + + fut = asyncio.Future() + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + (s_rd, s_wr) = await fut + + # Limit the socket buffers so we can more reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # kernel stops accepting more data in the send buffer. + # gh-122136: getsockopt() does not reliably report the buffer size + # available for message content. + # We loop until we start filling up the asyncio buffer. + # To avoid an infinite loop we cap at 10 times the expected value + c_bufsize = c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) + s_bufsize = s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) + for i in range(10): + s_wr.write(b'a' * c_bufsize) + s_wr.write(b'a' * s_bufsize) + if s_wr.transport.get_write_buffer_size() > 0: + break + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + +# Test the various corner cases of Unix server socket removal +class UnixServerCleanupTests(unittest.IsolatedAsyncioTestCase): + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_addr_cleanup(self): + # Default scenario + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr) + + srv.close() + self.assertFalse(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_sock_cleanup(self): + # Using already bound socket + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv = await asyncio.start_unix_server(serve, sock=sock) + + srv.close() + self.assertFalse(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_gone(self): + # Someone else has already cleaned up the socket + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv = await asyncio.start_unix_server(serve, sock=sock) + + os.unlink(addr) + + srv.close() + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_replaced(self): + # Someone else has replaced the socket with their own + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr) + + os.unlink(addr) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.bind(addr) + + srv.close() + self.assertTrue(os.path.exists(addr)) + + @socket_helper.skip_unless_bind_unix_socket + async def test_unix_server_cleanup_prevented(self): + # Automatic cleanup explicitly disabled + with test_utils.unix_socket_path() as addr: + async def serve(*args): + pass + + srv = await asyncio.start_unix_server(serve, addr, cleanup_socket=False) + + srv.close() + self.assertTrue(os.path.exists(addr)) + + +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class ProactorStartServerTests(BaseStartServer, unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py new file mode 100644 index 00000000000..df4ec794897 --- /dev/null +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -0,0 +1,679 @@ +import socket +import asyncio +import sys +import unittest + +from asyncio import proactor_events +from itertools import cycle, islice +from unittest.mock import Mock +from test.test_asyncio import utils as test_utils +from test import support +from test.support import socket_helper + +if socket_helper.tcp_blackhole(): + raise unittest.SkipTest('Not relevant to ProactorEventLoop') + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class MyProto(asyncio.Protocol): + connected = None + done = None + + def __init__(self, loop=None): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = loop.create_future() + self.done = loop.create_future() + + def _assert_state(self, *expected): + if self.state not in expected: + raise AssertionError(f'state: {self.state!r}, expected: {expected!r}') + + def connection_made(self, transport): + self.transport = transport + self._assert_state('INITIAL') + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n') + + def data_received(self, data): + self._assert_state('CONNECTED') + self.nbytes += len(data) + + def eof_received(self): + self._assert_state('CONNECTED') + self.state = 'EOF' + + def connection_lost(self, exc): + self._assert_state('CONNECTED', 'EOF') + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class BaseSockTestsMixin: + + def create_event_loop(self): + raise NotImplementedError + + def setUp(self): + self.loop = self.create_event_loop() + self.set_event_loop(self.loop) + super().setUp() + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + def _basetest_sock_client_ops(self, httpd, sock): + if not isinstance(self.loop, proactor_events.BaseProactorEventLoop): + # in debug mode, socket operations must fail + # if the socket is not in blocking mode + self.loop.set_debug(True) + sock.setblocking(True) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_connect(sock, httpd.address)) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_recv_into(sock, bytearray())) + with self.assertRaises(ValueError): + self.loop.run_until_complete( + self.loop.sock_accept(sock)) + + # test in non-blocking mode + sock.setblocking(False) + self.loop.run_until_complete( + self.loop.sock_connect(sock, httpd.address)) + self.loop.run_until_complete( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + data = self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + # consume data + self.loop.run_until_complete( + self.loop.sock_recv(sock, 1024)) + sock.close() + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + + def _basetest_sock_recv_into(self, httpd, sock): + # same as _basetest_sock_client_ops, but using sock_recv_into + sock.setblocking(False) + self.loop.run_until_complete( + self.loop.sock_connect(sock, httpd.address)) + self.loop.run_until_complete( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + data = bytearray(1024) + with memoryview(data) as buf: + nbytes = self.loop.run_until_complete( + self.loop.sock_recv_into(sock, buf[:1024])) + # consume data + self.loop.run_until_complete( + self.loop.sock_recv_into(sock, buf[nbytes:])) + sock.close() + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + + def test_sock_client_ops(self): + with test_utils.run_test_server() as httpd: + sock = socket.socket() + self._basetest_sock_client_ops(httpd, sock) + sock = socket.socket() + self._basetest_sock_recv_into(httpd, sock) + + async def _basetest_sock_recv_racing(self, httpd, sock): + sock.setblocking(False) + await self.loop.sock_connect(sock, httpd.address) + + task = asyncio.create_task(self.loop.sock_recv(sock, 1024)) + await asyncio.sleep(0) + task.cancel() + + asyncio.create_task( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + data = await self.loop.sock_recv(sock, 1024) + # consume data + await self.loop.sock_recv(sock, 1024) + + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + + async def _basetest_sock_recv_into_racing(self, httpd, sock): + sock.setblocking(False) + await self.loop.sock_connect(sock, httpd.address) + + data = bytearray(1024) + with memoryview(data) as buf: + task = asyncio.create_task( + self.loop.sock_recv_into(sock, buf[:1024])) + await asyncio.sleep(0) + task.cancel() + + task = asyncio.create_task( + self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n')) + nbytes = await self.loop.sock_recv_into(sock, buf[:1024]) + # consume data + await self.loop.sock_recv_into(sock, buf[nbytes:]) + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + + await task + + async def _basetest_sock_send_racing(self, listener, sock): + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + + # make connection + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) + sock.setblocking(False) + task = asyncio.create_task( + self.loop.sock_connect(sock, listener.getsockname())) + await asyncio.sleep(0) + server = listener.accept()[0] + server.setblocking(False) + + with server: + await task + + # fill the buffer until sending 5 chars would block + size = 8192 + while size >= 4: + with self.assertRaises(BlockingIOError): + while True: + sock.send(b' ' * size) + size = int(size / 2) + + # cancel a blocked sock_sendall + task = asyncio.create_task( + self.loop.sock_sendall(sock, b'hello')) + await asyncio.sleep(0) + task.cancel() + + # receive everything that is not a space + async def recv_all(): + rv = b'' + while True: + buf = await self.loop.sock_recv(server, 8192) + if not buf: + return rv + rv += buf.strip() + task = asyncio.create_task(recv_all()) + + # immediately make another sock_sendall call + await self.loop.sock_sendall(sock, b'world') + sock.shutdown(socket.SHUT_WR) + data = await task + # ProactorEventLoop could deliver hello, so endswith is necessary + self.assertEndsWith(data, b'world') + + # After the first connect attempt before the listener is ready, + # the socket needs time to "recover" to make the next connect call. + # On Linux, a second retry will do. On Windows, the waiting time is + # unpredictable; and on FreeBSD the socket may never come back + # because it's a loopback address. Here we'll just retry for a few + # times, and have to skip the test if it's not working. See also: + # https://stackoverflow.com/a/54437602/3316267 + # https://lists.freebsd.org/pipermail/freebsd-current/2005-May/049876.html + async def _basetest_sock_connect_racing(self, listener, sock): + listener.bind(('127.0.0.1', 0)) + addr = listener.getsockname() + sock.setblocking(False) + + task = asyncio.create_task(self.loop.sock_connect(sock, addr)) + await asyncio.sleep(0) + task.cancel() + + listener.listen(1) + + skip_reason = "Max retries reached" + for i in range(128): + try: + await self.loop.sock_connect(sock, addr) + except ConnectionRefusedError as e: + skip_reason = e + except OSError as e: + skip_reason = e + + # Retry only for this error: + # [WinError 10022] An invalid argument was supplied + if getattr(e, 'winerror', 0) != 10022: + break + else: + # success + return + + self.skipTest(skip_reason) + + def test_sock_client_racing(self): + with test_utils.run_test_server() as httpd: + sock = socket.socket() + with sock: + self.loop.run_until_complete(asyncio.wait_for( + self._basetest_sock_recv_racing(httpd, sock), 10)) + sock = socket.socket() + with sock: + self.loop.run_until_complete(asyncio.wait_for( + self._basetest_sock_recv_into_racing(httpd, sock), 10)) + listener = socket.socket() + sock = socket.socket() + with listener, sock: + self.loop.run_until_complete(asyncio.wait_for( + self._basetest_sock_send_racing(listener, sock), 10)) + + def test_sock_client_connect_racing(self): + listener = socket.socket() + sock = socket.socket() + with listener, sock: + self.loop.run_until_complete(asyncio.wait_for( + self._basetest_sock_connect_racing(listener, sock), 10)) + + async def _basetest_huge_content(self, address): + sock = socket.socket() + sock.setblocking(False) + DATA_SIZE = 10_000_00 + + chunk = b'0123456789' * (DATA_SIZE // 10) + + await self.loop.sock_connect(sock, address) + await self.loop.sock_sendall(sock, + (b'POST /loop HTTP/1.0\r\n' + + b'Content-Length: %d\r\n' % DATA_SIZE + + b'\r\n')) + + task = asyncio.create_task(self.loop.sock_sendall(sock, chunk)) + + data = await self.loop.sock_recv(sock, DATA_SIZE) + # HTTP headers size is less than MTU, + # they are sent by the first packet always + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + while data.find(b'\r\n\r\n') == -1: + data += await self.loop.sock_recv(sock, DATA_SIZE) + # Strip headers + headers = data[:data.index(b'\r\n\r\n') + 4] + data = data[len(headers):] + + size = DATA_SIZE + checker = cycle(b'0123456789') + + expected = bytes(islice(checker, len(data))) + self.assertEqual(data, expected) + size -= len(data) + + while True: + data = await self.loop.sock_recv(sock, DATA_SIZE) + if not data: + break + expected = bytes(islice(checker, len(data))) + self.assertEqual(data, expected) + size -= len(data) + self.assertEqual(size, 0) + + await task + sock.close() + + def test_huge_content(self): + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete( + self._basetest_huge_content(httpd.address)) + + async def _basetest_huge_content_recvinto(self, address): + sock = socket.socket() + sock.setblocking(False) + DATA_SIZE = 10_000_00 + + chunk = b'0123456789' * (DATA_SIZE // 10) + + await self.loop.sock_connect(sock, address) + await self.loop.sock_sendall(sock, + (b'POST /loop HTTP/1.0\r\n' + + b'Content-Length: %d\r\n' % DATA_SIZE + + b'\r\n')) + + task = asyncio.create_task(self.loop.sock_sendall(sock, chunk)) + + array = bytearray(DATA_SIZE) + buf = memoryview(array) + + nbytes = await self.loop.sock_recv_into(sock, buf) + data = bytes(buf[:nbytes]) + # HTTP headers size is less than MTU, + # they are sent by the first packet always + self.assertStartsWith(data, b'HTTP/1.0 200 OK') + while data.find(b'\r\n\r\n') == -1: + nbytes = await self.loop.sock_recv_into(sock, buf) + data = bytes(buf[:nbytes]) + # Strip headers + headers = data[:data.index(b'\r\n\r\n') + 4] + data = data[len(headers):] + + size = DATA_SIZE + checker = cycle(b'0123456789') + + expected = bytes(islice(checker, len(data))) + self.assertEqual(data, expected) + size -= len(data) + + while True: + nbytes = await self.loop.sock_recv_into(sock, buf) + data = buf[:nbytes] + if not data: + break + expected = bytes(islice(checker, len(data))) + self.assertEqual(data, expected) + size -= len(data) + self.assertEqual(size, 0) + + await task + sock.close() + + def test_huge_content_recvinto(self): + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete( + self._basetest_huge_content_recvinto(httpd.address)) + + async def _basetest_datagram_recvfrom(self, server_address): + # Happy path, sock.sendto() returns immediately + data = b'\x01' * 4096 + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + await self.loop.sock_sendto(sock, data, server_address) + received_data, from_addr = await self.loop.sock_recvfrom( + sock, 4096) + self.assertEqual(received_data, data) + self.assertEqual(from_addr, server_address) + + def test_recvfrom(self): + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_recvfrom(server_address)) + + async def _basetest_datagram_recvfrom_into(self, server_address): + # Happy path, sock.sendto() returns immediately + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + + buf = bytearray(4096) + data = b'\x01' * 4096 + await self.loop.sock_sendto(sock, data, server_address) + num_bytes, from_addr = await self.loop.sock_recvfrom_into( + sock, buf) + self.assertEqual(num_bytes, 4096) + self.assertEqual(buf, data) + self.assertEqual(from_addr, server_address) + + buf = bytearray(8192) + await self.loop.sock_sendto(sock, data, server_address) + num_bytes, from_addr = await self.loop.sock_recvfrom_into( + sock, buf, 4096) + self.assertEqual(num_bytes, 4096) + self.assertEqual(buf[:4096], data[:4096]) + self.assertEqual(from_addr, server_address) + + def test_recvfrom_into(self): + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_recvfrom_into(server_address)) + + async def _basetest_datagram_sendto_blocking(self, server_address): + # Sad path, sock.sendto() raises BlockingIOError + # This involves patching sock.sendto() to raise BlockingIOError but + # sendto() is not used by the proactor event loop + data = b'\x01' * 4096 + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.setblocking(False) + mock_sock = Mock(sock) + mock_sock.gettimeout = sock.gettimeout + mock_sock.sendto.configure_mock(side_effect=BlockingIOError) + mock_sock.fileno = sock.fileno + self.loop.call_soon( + lambda: setattr(mock_sock, 'sendto', sock.sendto) + ) + await self.loop.sock_sendto(mock_sock, data, server_address) + + received_data, from_addr = await self.loop.sock_recvfrom( + sock, 4096) + self.assertEqual(received_data, data) + self.assertEqual(from_addr, server_address) + + def test_sendto_blocking(self): + if sys.platform == 'win32': + if isinstance(self.loop, asyncio.ProactorEventLoop): + raise unittest.SkipTest('Not relevant to ProactorEventLoop') + + with test_utils.run_udp_echo_server() as server_address: + self.loop.run_until_complete( + self._basetest_datagram_sendto_blocking(server_address)) + + @socket_helper.skip_unless_bind_unix_socket + def test_unix_sock_client_ops(self): + with test_utils.run_test_unix_server() as httpd: + sock = socket.socket(socket.AF_UNIX) + self._basetest_sock_client_ops(httpd, sock) + sock = socket.socket(socket.AF_UNIX) + self._basetest_sock_recv_into(httpd, sock) + + def test_sock_client_fail(self): + # Make sure that we will get an unused port + address = None + try: + s = socket.socket() + s.bind(('127.0.0.1', 0)) + address = s.getsockname() + finally: + s.close() + + sock = socket.socket() + sock.setblocking(False) + with self.assertRaises(ConnectionRefusedError): + self.loop.run_until_complete( + self.loop.sock_connect(sock, address)) + sock.close() + + def test_sock_accept(self): + listener = socket.socket() + listener.setblocking(False) + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + client = socket.socket() + client.connect(listener.getsockname()) + + f = self.loop.sock_accept(listener) + conn, addr = self.loop.run_until_complete(f) + self.assertEqual(conn.gettimeout(), 0) + self.assertEqual(addr, client.getsockname()) + self.assertEqual(client.getpeername(), listener.getsockname()) + client.close() + conn.close() + listener.close() + + def test_cancel_sock_accept(self): + listener = socket.socket() + listener.setblocking(False) + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + sockaddr = listener.getsockname() + f = asyncio.wait_for(self.loop.sock_accept(listener), 0.1) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(f) + + listener.close() + client = socket.socket() + client.setblocking(False) + f = self.loop.sock_connect(client, sockaddr) + with self.assertRaises(ConnectionRefusedError): + self.loop.run_until_complete(f) + + client.close() + + def test_create_connection_sock(self): + with test_utils.run_test_server() as httpd: + sock = None + infos = self.loop.run_until_complete( + self.loop.getaddrinfo( + *httpd.address, type=socket.SOCK_STREAM)) + for family, type, proto, cname, address in infos: + try: + sock = socket.socket(family=family, type=type, proto=proto) + sock.setblocking(False) + self.loop.run_until_complete( + self.loop.sock_connect(sock, address)) + except BaseException: + pass + else: + break + else: + self.fail('Can not create socket.') + + f = self.loop.create_connection( + lambda: MyProto(loop=self.loop), sock=sock) + tr, pr = self.loop.run_until_complete(f) + self.assertIsInstance(tr, asyncio.Transport) + self.assertIsInstance(pr, asyncio.Protocol) + self.loop.run_until_complete(pr.done) + self.assertGreater(pr.nbytes, 0) + tr.close() + + +if sys.platform == 'win32': + + class SelectEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop() + + + class ProactorEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.ProactorEventLoop() + + + async def _basetest_datagram_send_to_non_listening_address(self, + recvfrom): + # see: + # https://github.com/python/cpython/issues/91227 + # https://github.com/python/cpython/issues/88906 + # https://bugs.python.org/issue47071 + # https://bugs.python.org/issue44743 + # The Proactor event loop would fail to receive datagram messages + # after sending a message to an address that wasn't listening. + + def create_socket(): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + sock.bind(('127.0.0.1', 0)) + return sock + + socket_1 = create_socket() + addr_1 = socket_1.getsockname() + + socket_2 = create_socket() + addr_2 = socket_2.getsockname() + + # creating and immediately closing this to try to get an address + # that is not listening + socket_3 = create_socket() + addr_3 = socket_3.getsockname() + socket_3.shutdown(socket.SHUT_RDWR) + socket_3.close() + + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + socket_2_recv_task = self.loop.create_task(recvfrom(socket_2)) + await asyncio.sleep(0) + + await self.loop.sock_sendto(socket_1, b'a', addr_2) + self.assertEqual(await socket_2_recv_task, b'a') + + await self.loop.sock_sendto(socket_2, b'b', addr_1) + self.assertEqual(await socket_1_recv_task, b'b') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # this should send to an address that isn't listening + await self.loop.sock_sendto(socket_1, b'c', addr_3) + self.assertEqual(await socket_1_recv_task, b'') + socket_1_recv_task = self.loop.create_task(recvfrom(socket_1)) + await asyncio.sleep(0) + + # socket 1 should still be able to receive messages after sending + # to an address that wasn't listening + socket_2.sendto(b'd', addr_1) + self.assertEqual(await socket_1_recv_task, b'd') + + socket_1.shutdown(socket.SHUT_RDWR) + socket_1.close() + socket_2.shutdown(socket.SHUT_RDWR) + socket_2.close() + + + def test_datagram_send_to_non_listening_address_recvfrom(self): + async def recvfrom(socket): + data, _ = await self.loop.sock_recvfrom(socket, 4096) + return data + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom)) + + + def test_datagram_send_to_non_listening_address_recvfrom_into(self): + async def recvfrom_into(socket): + buf = bytearray(4096) + length, _ = await self.loop.sock_recvfrom_into(socket, buf, + 4096) + return buf[:length] + + self.loop.run_until_complete( + self._basetest_datagram_send_to_non_listening_address( + recvfrom_into)) + +else: + import selectors + + if hasattr(selectors, 'KqueueSelector'): + class KqueueEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop( + selectors.KqueueSelector()) + + if hasattr(selectors, 'EpollSelector'): + class EPollEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.EpollSelector()) + + if hasattr(selectors, 'PollSelector'): + class PollEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.PollSelector()) + + # Should always exist. + class SelectEventLoopTests(BaseSockTestsMixin, + test_utils.TestCase): + + def create_event_loop(self): + return asyncio.SelectorEventLoop(selectors.SelectSelector()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py new file mode 100644 index 00000000000..ab4a2316f9c --- /dev/null +++ b/Lib/test/test_asyncio/test_ssl.py @@ -0,0 +1,1905 @@ +# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0 +# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0) +# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io + +import asyncio +import contextlib +import gc +import logging +import select +import socket +import sys +import tempfile +import threading +import time +import unittest.mock +import weakref +import unittest + +try: + import ssl +except ImportError: + ssl = None + +from test import support +from test.test_asyncio import utils as test_utils + + +MACOS = (sys.platform == 'darwin') +BUF_MULTIPLIER = 1024 if not MACOS else 64 + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class MyBaseProto(asyncio.Protocol): + connected = None + done = None + + def __init__(self, loop=None): + self.transport = None + self.state = 'INITIAL' + self.nbytes = 0 + if loop is not None: + self.connected = asyncio.Future(loop=loop) + self.done = asyncio.Future(loop=loop) + + def connection_made(self, transport): + self.transport = transport + assert self.state == 'INITIAL', self.state + self.state = 'CONNECTED' + if self.connected: + self.connected.set_result(None) + + def data_received(self, data): + assert self.state == 'CONNECTED', self.state + self.nbytes += len(data) + + def eof_received(self): + assert self.state == 'CONNECTED', self.state + self.state = 'EOF' + + def connection_lost(self, exc): + assert self.state in ('CONNECTED', 'EOF'), self.state + self.state = 'CLOSED' + if self.done: + self.done.set_result(None) + + +class MessageOutFilter(logging.Filter): + def __init__(self, msg): + self.msg = msg + + def filter(self, record): + if self.msg in record.msg: + return False + return True + + +@unittest.skipIf(ssl is None, 'No ssl module') +class TestSSL(test_utils.TestCase): + + PAYLOAD_SIZE = 1024 * 100 + TIMEOUT = support.LONG_TIMEOUT + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + self.addCleanup(self.loop.close) + + def tearDown(self): + # just in case if we have transport close callbacks + if not self.loop.is_closed(): + test_utils.run_briefly(self.loop) + + self.doCleanups() + support.gc_collect() + super().tearDown() + + def tcp_server(self, server_prog, *, + family=socket.AF_INET, + addr=None, + timeout=support.SHORT_TIMEOUT, + backlog=1, + max_clients=10): + + if addr is None: + if family == getattr(socket, "AF_UNIX", None): + with tempfile.NamedTemporaryFile() as tmp: + addr = tmp.name + else: + addr = ('127.0.0.1', 0) + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + try: + sock.bind(addr) + sock.listen(backlog) + except OSError as ex: + sock.close() + raise ex + + return TestThreadedServer( + self, sock, server_prog, timeout, max_clients) + + def tcp_client(self, client_prog, + family=socket.AF_INET, + timeout=support.SHORT_TIMEOUT): + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + return TestThreadedClient( + self, sock, client_prog, timeout) + + def unix_server(self, *args, **kwargs): + return self.tcp_server(*args, family=socket.AF_UNIX, **kwargs) + + def unix_client(self, *args, **kwargs): + return self.tcp_client(*args, family=socket.AF_UNIX, **kwargs) + + def _create_server_ssl_context(self, certfile, keyfile=None): + sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslcontext.options |= ssl.OP_NO_SSLv2 + sslcontext.load_cert_chain(certfile, keyfile) + return sslcontext + + def _create_client_ssl_context(self, *, disable_verify=True): + sslcontext = ssl.create_default_context() + sslcontext.check_hostname = False + if disable_verify: + sslcontext.verify_mode = ssl.CERT_NONE + return sslcontext + + @contextlib.contextmanager + def _silence_eof_received_warning(self): + # TODO This warning has to be fixed in asyncio. + logger = logging.getLogger('asyncio') + filter = MessageOutFilter('has no effect when using ssl') + logger.addFilter(filter) + try: + yield + finally: + logger.removeFilter(filter) + + def _abort_socket_test(self, ex): + try: + self.loop.stop() + finally: + self.fail(ex) + + def new_loop(self): + return asyncio.new_event_loop() + + def new_policy(self): + return asyncio.DefaultEventLoopPolicy() + + async def wait_closed(self, obj): + if not isinstance(obj, asyncio.StreamWriter): + return + try: + await obj.wait_closed() + except (BrokenPipeError, ConnectionError): + pass + + @support.bigmemtest(size=25, memuse=90*2**20, dry_run=False) + def test_create_server_ssl_1(self, size): + CNT = 0 # number of clients that were successful + TOTAL_CNT = size # total number of clients that test will create + TIMEOUT = support.LONG_TIMEOUT # timeout for this test + + A_DATA = b'A' * 1024 * BUF_MULTIPLIER + B_DATA = b'B' * 1024 * BUF_MULTIPLIER + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context() + + clients = [] + + async def handle_client(reader, writer): + nonlocal CNT + + data = await reader.readexactly(len(A_DATA)) + self.assertEqual(data, A_DATA) + writer.write(b'OK') + + data = await reader.readexactly(len(B_DATA)) + self.assertEqual(data, B_DATA) + writer.writelines([b'SP', bytearray(b'A'), memoryview(b'M')]) + + await writer.drain() + writer.close() + + CNT += 1 + + async def test_client(addr): + fut = asyncio.Future() + + def prog(sock): + try: + sock.starttls(client_sslctx) + sock.connect(addr) + sock.send(A_DATA) + + data = sock.recv_all(2) + self.assertEqual(data, b'OK') + + sock.send(B_DATA) + data = sock.recv_all(4) + self.assertEqual(data, b'SPAM') + + sock.close() + + except Exception as ex: + self.loop.call_soon_threadsafe(fut.set_exception, ex) + else: + self.loop.call_soon_threadsafe(fut.set_result, None) + + client = self.tcp_client(prog) + client.start() + clients.append(client) + + await fut + + async def start_server(): + extras = {} + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) + + srv = await asyncio.start_server( + handle_client, + '127.0.0.1', 0, + family=socket.AF_INET, + ssl=sslctx, + **extras) + + try: + srv_socks = srv.sockets + self.assertTrue(srv_socks) + + addr = srv_socks[0].getsockname() + + tasks = [] + for _ in range(TOTAL_CNT): + tasks.append(test_client(addr)) + + await asyncio.wait_for(asyncio.gather(*tasks), TIMEOUT) + + finally: + self.loop.call_soon(srv.close) + await srv.wait_closed() + + with self._silence_eof_received_warning(): + self.loop.run_until_complete(start_server()) + + self.assertEqual(CNT, TOTAL_CNT) + + for client in clients: + client.stop() + + def test_create_connection_ssl_1(self): + self.loop.set_exception_handler(None) + + CNT = 0 + TOTAL_CNT = 25 + + A_DATA = b'A' * 1024 * BUF_MULTIPLIER + B_DATA = b'B' * 1024 * BUF_MULTIPLIER + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context() + + def server(sock): + sock.starttls( + sslctx, + server_side=True) + + data = sock.recv_all(len(A_DATA)) + self.assertEqual(data, A_DATA) + sock.send(b'OK') + + data = sock.recv_all(len(B_DATA)) + self.assertEqual(data, B_DATA) + sock.send(b'SPAM') + + sock.close() + + async def client(addr): + extras = {} + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) + + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + **extras) + + writer.write(A_DATA) + self.assertEqual(await reader.readexactly(2), b'OK') + + writer.write(B_DATA) + self.assertEqual(await reader.readexactly(4), b'SPAM') + + nonlocal CNT + CNT += 1 + + writer.close() + await self.wait_closed(writer) + + async def client_sock(addr): + sock = socket.socket() + sock.connect(addr) + reader, writer = await asyncio.open_connection( + sock=sock, + ssl=client_sslctx, + server_hostname='') + + writer.write(A_DATA) + self.assertEqual(await reader.readexactly(2), b'OK') + + writer.write(B_DATA) + self.assertEqual(await reader.readexactly(4), b'SPAM') + + nonlocal CNT + CNT += 1 + + writer.close() + await self.wait_closed(writer) + sock.close() + + def run(coro): + nonlocal CNT + CNT = 0 + + async def _gather(*tasks): + # trampoline + return await asyncio.gather(*tasks) + + with self.tcp_server(server, + max_clients=TOTAL_CNT, + backlog=TOTAL_CNT) as srv: + tasks = [] + for _ in range(TOTAL_CNT): + tasks.append(coro(srv.addr)) + + self.loop.run_until_complete(_gather(*tasks)) + + self.assertEqual(CNT, TOTAL_CNT) + + with self._silence_eof_received_warning(): + run(client) + + with self._silence_eof_received_warning(): + run(client_sock) + + def test_create_connection_ssl_slow_handshake(self): + client_sslctx = self._create_client_ssl_context() + + # silence error logger + self.loop.set_exception_handler(lambda *args: None) + + def server(sock): + try: + sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + pass + finally: + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=1.0) + writer.close() + await self.wait_closed(writer) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaisesRegex( + ConnectionAbortedError, + r'SSL handshake.*is taking longer'): + + self.loop.run_until_complete(client(srv.addr)) + + def test_create_connection_ssl_failed_certificate(self): + # silence error logger + self.loop.set_exception_handler(lambda *args: None) + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context(disable_verify=False) + + def server(sock): + try: + sock.starttls( + sslctx, + server_side=True) + sock.connect() + except (ssl.SSLError, OSError): + pass + finally: + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=support.SHORT_TIMEOUT) + writer.close() + await self.wait_closed(writer) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(ssl.SSLCertVerificationError): + self.loop.run_until_complete(client(srv.addr)) + + def test_ssl_handshake_timeout(self): + # bpo-29970: Check that a connection is aborted if handshake is not + # completed in timeout period, instead of remaining open indefinitely + client_sslctx = test_utils.simple_client_sslcontext() + + # silence error logger + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + server_side_aborted = False + + def server(sock): + nonlocal server_side_aborted + try: + sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + server_side_aborted = True + finally: + sock.close() + + async def client(addr): + await asyncio.wait_for( + self.loop.create_connection( + asyncio.Protocol, + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=10.0), + 0.5) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(client(srv.addr)) + + self.assertTrue(server_side_aborted) + + # Python issue #23197: cancelling a handshake must not raise an + # exception or log an error, even if the handshake failed + self.assertEqual(messages, []) + + def test_ssl_handshake_connection_lost(self): + # #246: make sure that no connection_lost() is called before + # connection_made() is called first + + client_sslctx = test_utils.simple_client_sslcontext() + + # silence error logger + self.loop.set_exception_handler(lambda loop, ctx: None) + + connection_made_called = False + connection_lost_called = False + + def server(sock): + sock.recv(1024) + # break the connection during handshake + sock.close() + + class ClientProto(asyncio.Protocol): + def connection_made(self, transport): + nonlocal connection_made_called + connection_made_called = True + + def connection_lost(self, exc): + nonlocal connection_lost_called + connection_lost_called = True + + async def client(addr): + await self.loop.create_connection( + ClientProto, + *addr, + ssl=client_sslctx, + server_hostname=''), + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(ConnectionResetError): + self.loop.run_until_complete(client(srv.addr)) + + if connection_lost_called: + if connection_made_called: + self.fail("unexpected call to connection_lost()") + else: + self.fail("unexpected call to connection_lost() without" + "calling connection_made()") + elif connection_made_called: + self.fail("unexpected call to connection_made()") + + def test_ssl_connect_accepted_socket(self): + proto = ssl.PROTOCOL_TLS_SERVER + server_context = ssl.SSLContext(proto) + server_context.load_cert_chain(test_utils.ONLYCERT, test_utils.ONLYKEY) + if hasattr(server_context, 'check_hostname'): + server_context.check_hostname = False + server_context.verify_mode = ssl.CERT_NONE + + client_context = ssl.SSLContext(proto) + if hasattr(server_context, 'check_hostname'): + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + + def test_connect_accepted_socket(self, server_ssl=None, client_ssl=None): + loop = self.loop + + class MyProto(MyBaseProto): + + def connection_lost(self, exc): + super().connection_lost(exc) + loop.call_soon(loop.stop) + + def data_received(self, data): + super().data_received(data) + self.transport.write(expected_response) + + lsock = socket.socket(socket.AF_INET) + lsock.bind(('127.0.0.1', 0)) + lsock.listen(1) + addr = lsock.getsockname() + + message = b'test data' + response = None + expected_response = b'roger' + + def client(): + nonlocal response + try: + csock = socket.socket(socket.AF_INET) + if client_ssl is not None: + csock = client_ssl.wrap_socket(csock) + csock.connect(addr) + csock.sendall(message) + response = csock.recv(99) + csock.close() + except Exception as exc: + print( + "Failure in client thread in test_connect_accepted_socket", + exc) + + thread = threading.Thread(target=client, daemon=True) + thread.start() + + conn, _ = lsock.accept() + proto = MyProto(loop=loop) + proto.loop = loop + + extras = {} + if server_ssl: + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) + + f = loop.create_task( + loop.connect_accepted_socket( + (lambda: proto), conn, ssl=server_ssl, + **extras)) + loop.run_forever() + conn.close() + lsock.close() + + thread.join(1) + self.assertFalse(thread.is_alive()) + self.assertEqual(proto.state, 'CLOSED') + self.assertEqual(proto.nbytes, len(message)) + self.assertEqual(response, expected_response) + tr, _ = f.result() + + if server_ssl: + self.assertIn('SSL', tr.__class__.__name__) + + tr.close() + # let it close + self.loop.run_until_complete(asyncio.sleep(0.1)) + + def test_start_tls_client_corrupted_ssl(self): + self.loop.set_exception_handler(lambda loop, ctx: None) + + sslctx = test_utils.simple_server_sslcontext() + client_sslctx = test_utils.simple_client_sslcontext() + + def server(sock): + orig_sock = sock.dup() + try: + sock.starttls( + sslctx, + server_side=True) + sock.sendall(b'A\n') + sock.recv_all(1) + orig_sock.send(b'please corrupt the SSL connection') + except ssl.SSLError: + pass + finally: + sock.close() + orig_sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + + self.assertEqual(await reader.readline(), b'A\n') + writer.write(b'B') + with self.assertRaises(ssl.SSLError): + await reader.readline() + writer.close() + try: + await self.wait_closed(writer) + except ssl.SSLError: + pass + return 'OK' + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + res = self.loop.run_until_complete(client(srv.addr)) + + self.assertEqual(res, 'OK') + + def test_start_tls_client_reg_proto_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.starttls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.unwrap() + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data, b'O') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_create_connection_memory_leak(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_context = self._create_client_ssl_context() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + sock.starttls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.unwrap() + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + # XXX: We assume user stores the transport in protocol + proto.tr = tr + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr, + ssl=client_context) + + self.assertEqual(await on_data, b'O') + tr.write(HELLO_MSG) + await on_eof + + tr.close() + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + # No garbage is left for SSL client from loop.create_connection, even + # if user stores the SSLTransport in corresponding protocol instance + client_context = weakref.ref(client_context) + self.assertIsNone(client_context()) + + def test_start_tls_client_buf_proto_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + client_con_made_calls = 0 + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.starttls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.sendall(b'2') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.unwrap() + sock.close() + + class ClientProtoFirst(asyncio.BufferedProtocol): + def __init__(self, on_data): + self.on_data = on_data + self.buf = bytearray(1) + + def connection_made(self, tr): + nonlocal client_con_made_calls + client_con_made_calls += 1 + + def get_buffer(self, sizehint): + return self.buf + + def buffer_updated(self, nsize): + assert nsize == 1 + self.on_data.set_result(bytes(self.buf[:nsize])) + + def eof_received(self): + pass + + class ClientProtoSecond(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(self, tr): + nonlocal client_con_made_calls + client_con_made_calls += 1 + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data1 = self.loop.create_future() + on_data2 = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProtoFirst(on_data1), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data1, b'O') + new_tr.write(HELLO_MSG) + + new_tr.set_protocol(ClientProtoSecond(on_data2, on_eof)) + self.assertEqual(await on_data2, b'2') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + # connection_made() should be called only once -- when + # we establish connection for the first time. Start TLS + # doesn't call connection_made() on application protocols. + self.assertEqual(client_con_made_calls, 1) + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=self.TIMEOUT)) + + def test_start_tls_slow_client_cancel(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + client_context = test_utils.simple_client_sslcontext() + server_waits_on_handshake = self.loop.create_future() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + try: + self.loop.call_soon_threadsafe( + server_waits_on_handshake.set_result, None) + data = sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + pass + finally: + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + + await server_waits_on_handshake + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for( + self.loop.start_tls(tr, proto, client_context), + 0.5) + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + def test_start_tls_server_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def client(sock, addr): + sock.settimeout(self.TIMEOUT) + + sock.connect(addr) + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.starttls(client_context) + sock.sendall(HELLO_MSG) + + sock.unwrap() + sock.close() + + class ServerProto(asyncio.Protocol): + def __init__(self, on_con, on_eof, on_con_lost): + self.on_con = on_con + self.on_eof = on_eof + self.on_con_lost = on_con_lost + self.data = b'' + + def connection_made(self, tr): + self.on_con.set_result(tr) + + def data_received(self, data): + self.data += data + + def eof_received(self): + self.on_eof.set_result(1) + + def connection_lost(self, exc): + if exc is None: + self.on_con_lost.set_result(None) + else: + self.on_con_lost.set_exception(exc) + + async def main(proto, on_con, on_eof, on_con_lost): + tr = await on_con + tr.write(HELLO_MSG) + + self.assertEqual(proto.data, b'') + + new_tr = await self.loop.start_tls( + tr, proto, server_context, + server_side=True, + ssl_handshake_timeout=self.TIMEOUT) + + await on_eof + await on_con_lost + self.assertEqual(proto.data, HELLO_MSG) + new_tr.close() + + async def run_main(): + on_con = self.loop.create_future() + on_eof = self.loop.create_future() + on_con_lost = self.loop.create_future() + proto = ServerProto(on_con, on_eof, on_con_lost) + + server = await self.loop.create_server( + lambda: proto, '127.0.0.1', 0) + addr = server.sockets[0].getsockname() + + with self.tcp_client(lambda sock: client(sock, addr), + timeout=self.TIMEOUT): + await asyncio.wait_for( + main(proto, on_con, on_eof, on_con_lost), + timeout=self.TIMEOUT) + + server.close() + await server.wait_closed() + + self.loop.run_until_complete(run_main()) + + @support.bigmemtest(size=25, memuse=90*2**20, dry_run=False) + def test_create_server_ssl_over_ssl(self, size): + CNT = 0 # number of clients that were successful + TOTAL_CNT = size # total number of clients that test will create + TIMEOUT = support.LONG_TIMEOUT # timeout for this test + + A_DATA = b'A' * 1024 * BUF_MULTIPLIER + B_DATA = b'B' * 1024 * BUF_MULTIPLIER + + sslctx_1 = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_sslctx_1 = self._create_client_ssl_context() + sslctx_2 = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_sslctx_2 = self._create_client_ssl_context() + + clients = [] + + async def handle_client(reader, writer): + nonlocal CNT + + data = await reader.readexactly(len(A_DATA)) + self.assertEqual(data, A_DATA) + writer.write(b'OK') + + data = await reader.readexactly(len(B_DATA)) + self.assertEqual(data, B_DATA) + writer.writelines([b'SP', bytearray(b'A'), memoryview(b'M')]) + + await writer.drain() + writer.close() + + CNT += 1 + + class ServerProtocol(asyncio.StreamReaderProtocol): + def connection_made(self, transport): + super_ = super() + transport.pause_reading() + fut = self._loop.create_task(self._loop.start_tls( + transport, self, sslctx_2, server_side=True)) + + def cb(_): + try: + tr = fut.result() + except Exception as ex: + super_.connection_lost(ex) + else: + super_.connection_made(tr) + fut.add_done_callback(cb) + + def server_protocol_factory(): + reader = asyncio.StreamReader() + protocol = ServerProtocol(reader, handle_client) + return protocol + + async def test_client(addr): + fut = asyncio.Future() + + def prog(sock): + try: + sock.connect(addr) + sock.starttls(client_sslctx_1) + + # because wrap_socket() doesn't work correctly on + # SSLSocket, we have to do the 2nd level SSL manually + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + sslobj = client_sslctx_2.wrap_bio(incoming, outgoing) + + def do(func, *args): + while True: + try: + rv = func(*args) + break + except ssl.SSLWantReadError: + if outgoing.pending: + sock.send(outgoing.read()) + incoming.write(sock.recv(65536)) + if outgoing.pending: + sock.send(outgoing.read()) + return rv + + do(sslobj.do_handshake) + + do(sslobj.write, A_DATA) + data = do(sslobj.read, 2) + self.assertEqual(data, b'OK') + + do(sslobj.write, B_DATA) + data = b'' + while True: + chunk = do(sslobj.read, 4) + if not chunk: + break + data += chunk + self.assertEqual(data, b'SPAM') + + do(sslobj.unwrap) + sock.close() + + except Exception as ex: + self.loop.call_soon_threadsafe(fut.set_exception, ex) + sock.close() + else: + self.loop.call_soon_threadsafe(fut.set_result, None) + + client = self.tcp_client(prog) + client.start() + clients.append(client) + + await fut + + async def start_server(): + extras = {} + + srv = await self.loop.create_server( + server_protocol_factory, + '127.0.0.1', 0, + family=socket.AF_INET, + ssl=sslctx_1, + **extras) + + try: + srv_socks = srv.sockets + self.assertTrue(srv_socks) + + addr = srv_socks[0].getsockname() + + tasks = [] + for _ in range(TOTAL_CNT): + tasks.append(test_client(addr)) + + await asyncio.wait_for(asyncio.gather(*tasks), TIMEOUT) + + finally: + self.loop.call_soon(srv.close) + await srv.wait_closed() + + with self._silence_eof_received_warning(): + self.loop.run_until_complete(start_server()) + + self.assertEqual(CNT, TOTAL_CNT) + + for client in clients: + client.stop() + + def test_shutdown_cleanly(self): + CNT = 0 + TOTAL_CNT = 25 + + A_DATA = b'A' * 1024 * BUF_MULTIPLIER + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_sslctx = self._create_client_ssl_context() + + def server(sock): + sock.starttls( + sslctx, + server_side=True) + + data = sock.recv_all(len(A_DATA)) + self.assertEqual(data, A_DATA) + sock.send(b'OK') + + sock.unwrap() + + sock.close() + + async def client(addr): + extras = {} + extras = dict(ssl_handshake_timeout=support.SHORT_TIMEOUT) + + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + **extras) + + writer.write(A_DATA) + self.assertEqual(await reader.readexactly(2), b'OK') + + self.assertEqual(await reader.read(), b'') + + nonlocal CNT + CNT += 1 + + writer.close() + await self.wait_closed(writer) + + def run(coro): + nonlocal CNT + CNT = 0 + + async def _gather(*tasks): + return await asyncio.gather(*tasks) + + with self.tcp_server(server, + max_clients=TOTAL_CNT, + backlog=TOTAL_CNT) as srv: + tasks = [] + for _ in range(TOTAL_CNT): + tasks.append(coro(srv.addr)) + + self.loop.run_until_complete( + _gather(*tasks)) + + self.assertEqual(CNT, TOTAL_CNT) + + with self._silence_eof_received_warning(): + run(client) + + def test_flush_before_shutdown(self): + CHUNK = 1024 * 128 + SIZE = 32 + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, test_utils.ONLYKEY) + client_sslctx = self._create_client_ssl_context() + + future = None + + def server(sock): + sock.starttls(sslctx, server_side=True) + self.assertEqual(sock.recv_all(4), b'ping') + sock.send(b'pong') + time.sleep(0.5) # hopefully stuck the TCP buffer + data = sock.recv_all(CHUNK * SIZE) + self.assertEqual(len(data), CHUNK * SIZE) + sock.close() + + def run(meth): + def wrapper(sock): + try: + meth(sock) + except Exception as ex: + self.loop.call_soon_threadsafe(future.set_exception, ex) + else: + self.loop.call_soon_threadsafe(future.set_result, None) + return wrapper + + async def client(addr): + nonlocal future + future = self.loop.create_future() + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + sslprotocol = writer.transport._ssl_protocol + writer.write(b'ping') + data = await reader.readexactly(4) + self.assertEqual(data, b'pong') + + sslprotocol.pause_writing() + for _ in range(SIZE): + writer.write(b'x' * CHUNK) + + writer.close() + sslprotocol.resume_writing() + + await self.wait_closed(writer) + try: + data = await reader.read() + self.assertEqual(data, b'') + except ConnectionResetError: + pass + await future + + with self.tcp_server(run(server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + def test_remote_shutdown_receives_trailing_data(self): + CHUNK = 1024 * 128 + SIZE = 32 + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context() + future = None + + def server(sock): + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + sslobj = sslctx.wrap_bio(incoming, outgoing, server_side=True) + + while True: + try: + sslobj.do_handshake() + except ssl.SSLWantReadError: + if outgoing.pending: + sock.send(outgoing.read()) + incoming.write(sock.recv(16384)) + else: + if outgoing.pending: + sock.send(outgoing.read()) + break + + while True: + try: + data = sslobj.read(4) + except ssl.SSLWantReadError: + incoming.write(sock.recv(16384)) + else: + break + + self.assertEqual(data, b'ping') + sslobj.write(b'pong') + sock.send(outgoing.read()) + + time.sleep(0.2) # wait for the peer to fill its backlog + + # send close_notify but don't wait for response + with self.assertRaises(ssl.SSLWantReadError): + sslobj.unwrap() + sock.send(outgoing.read()) + + # should receive all data + data_len = 0 + while True: + try: + chunk = len(sslobj.read(16384)) + data_len += chunk + except ssl.SSLWantReadError: + incoming.write(sock.recv(16384)) + except ssl.SSLZeroReturnError: + break + + self.assertEqual(data_len, CHUNK * SIZE) + + # verify that close_notify is received + sslobj.unwrap() + + sock.close() + + def eof_server(sock): + sock.starttls(sslctx, server_side=True) + self.assertEqual(sock.recv_all(4), b'ping') + sock.send(b'pong') + + time.sleep(0.2) # wait for the peer to fill its backlog + + # send EOF + sock.shutdown(socket.SHUT_WR) + + # should receive all data + data = sock.recv_all(CHUNK * SIZE) + self.assertEqual(len(data), CHUNK * SIZE) + + sock.close() + + async def client(addr): + nonlocal future + future = self.loop.create_future() + + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + writer.write(b'ping') + data = await reader.readexactly(4) + self.assertEqual(data, b'pong') + + # fill write backlog in a hacky way - renegotiation won't help + for _ in range(SIZE): + writer.transport._test__append_write_backlog(b'x' * CHUNK) + + try: + data = await reader.read() + self.assertEqual(data, b'') + except (BrokenPipeError, ConnectionResetError): + pass + + await future + + writer.close() + await self.wait_closed(writer) + + def run(meth): + def wrapper(sock): + try: + meth(sock) + except Exception as ex: + self.loop.call_soon_threadsafe(future.set_exception, ex) + else: + self.loop.call_soon_threadsafe(future.set_result, None) + return wrapper + + with self.tcp_server(run(server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + with self.tcp_server(run(eof_server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + def test_remote_shutdown_receives_trailing_data_on_slow_socket(self): + # This test is the same as test_remote_shutdown_receives_trailing_data, + # except it simulates a socket that is not able to write data in time, + # thus triggering different code path in _SelectorSocketTransport. + # This triggers bug gh-115514, also tested using mocks in + # test.test_asyncio.test_selector_events.SelectorSocketTransportTests.test_write_buffer_after_close + # The slow path is triggered here by setting SO_SNDBUF, see code and comment below. + + CHUNK = 1024 * 128 + SIZE = 32 + + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + client_sslctx = self._create_client_ssl_context() + future = None + + def server(sock): + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + sslobj = sslctx.wrap_bio(incoming, outgoing, server_side=True) + + while True: + try: + sslobj.do_handshake() + except ssl.SSLWantReadError: + if outgoing.pending: + sock.send(outgoing.read()) + incoming.write(sock.recv(16384)) + else: + if outgoing.pending: + sock.send(outgoing.read()) + break + + while True: + try: + data = sslobj.read(4) + except ssl.SSLWantReadError: + incoming.write(sock.recv(16384)) + else: + break + + self.assertEqual(data, b'ping') + sslobj.write(b'pong') + sock.send(outgoing.read()) + + time.sleep(0.2) # wait for the peer to fill its backlog + + # send close_notify but don't wait for response + with self.assertRaises(ssl.SSLWantReadError): + sslobj.unwrap() + sock.send(outgoing.read()) + + # should receive all data + data_len = 0 + while True: + try: + chunk = len(sslobj.read(16384)) + data_len += chunk + except ssl.SSLWantReadError: + incoming.write(sock.recv(16384)) + except ssl.SSLZeroReturnError: + break + + self.assertEqual(data_len, CHUNK * SIZE*2) + + # verify that close_notify is received + sslobj.unwrap() + + sock.close() + + def eof_server(sock): + sock.starttls(sslctx, server_side=True) + self.assertEqual(sock.recv_all(4), b'ping') + sock.send(b'pong') + + time.sleep(0.2) # wait for the peer to fill its backlog + + # send EOF + sock.shutdown(socket.SHUT_WR) + + # should receive all data + data = sock.recv_all(CHUNK * SIZE) + self.assertEqual(len(data), CHUNK * SIZE) + + sock.close() + + async def client(addr): + nonlocal future + future = self.loop.create_future() + + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + writer.write(b'ping') + data = await reader.readexactly(4) + self.assertEqual(data, b'pong') + + # fill write backlog in a hacky way - renegotiation won't help + for _ in range(SIZE*2): + writer.transport._test__append_write_backlog(b'x' * CHUNK) + + try: + data = await reader.read() + self.assertEqual(data, b'') + except (BrokenPipeError, ConnectionResetError): + pass + + # Make sure _SelectorSocketTransport enters the delayed write + # path in its `write` method by wrapping socket in a fake class + # that acts as if there is not enough space in socket buffer. + # This triggers bug gh-115514, also tested using mocks in + # test.test_asyncio.test_selector_events.SelectorSocketTransportTests.test_write_buffer_after_close + socket_transport = writer.transport._ssl_protocol._transport + + class SocketWrapper: + def __init__(self, sock) -> None: + self.sock = sock + + def __getattr__(self, name): + return getattr(self.sock, name) + + def send(self, data): + # Fake that our write buffer is full, send only half + to_send = len(data)//2 + return self.sock.send(data[:to_send]) + + def _fake_full_write_buffer(data): + if socket_transport._read_ready_cb is None and not isinstance(socket_transport._sock, SocketWrapper): + socket_transport._sock = SocketWrapper(socket_transport._sock) + return unittest.mock.DEFAULT + + with unittest.mock.patch.object( + socket_transport, "write", + wraps=socket_transport.write, + side_effect=_fake_full_write_buffer + ): + await future + + writer.close() + await self.wait_closed(writer) + + def run(meth): + def wrapper(sock): + try: + meth(sock) + except Exception as ex: + self.loop.call_soon_threadsafe(future.set_exception, ex) + else: + self.loop.call_soon_threadsafe(future.set_result, None) + return wrapper + + with self.tcp_server(run(server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + with self.tcp_server(run(eof_server)) as srv: + self.loop.run_until_complete(client(srv.addr)) + + def test_connect_timeout_warning(self): + s = socket.socket(socket.AF_INET) + s.bind(('127.0.0.1', 0)) + addr = s.getsockname() + + async def test(): + try: + await asyncio.wait_for( + self.loop.create_connection(asyncio.Protocol, + *addr, ssl=True), + 0.1) + except (ConnectionRefusedError, asyncio.TimeoutError): + pass + else: + self.fail('TimeoutError is not raised') + + with s: + try: + with self.assertWarns(ResourceWarning) as cm: + self.loop.run_until_complete(test()) + gc.collect() + gc.collect() + gc.collect() + except AssertionError as e: + self.assertEqual(str(e), 'ResourceWarning not triggered') + else: + self.fail('Unexpected ResourceWarning: {}'.format(cm.warning)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_handshake_timeout_handler_leak(self): + s = socket.socket(socket.AF_INET) + s.bind(('127.0.0.1', 0)) + s.listen(1) + addr = s.getsockname() + + async def test(ctx): + try: + await asyncio.wait_for( + self.loop.create_connection(asyncio.Protocol, *addr, + ssl=ctx), + 0.1) + except (ConnectionRefusedError, asyncio.TimeoutError): + pass + else: + self.fail('TimeoutError is not raised') + + with s: + ctx = ssl.create_default_context() + self.loop.run_until_complete(test(ctx)) + ctx = weakref.ref(ctx) + + # SSLProtocol should be DECREF to 0 + self.assertIsNone(ctx()) + + def test_shutdown_timeout_handler_leak(self): + loop = self.loop + + def server(sock): + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + sock = sslctx.wrap_socket(sock, server_side=True) + sock.recv(32) + sock.close() + + class Protocol(asyncio.Protocol): + def __init__(self): + self.fut = asyncio.Future(loop=loop) + + def connection_lost(self, exc): + self.fut.set_result(None) + + async def client(addr, ctx): + tr, pr = await loop.create_connection(Protocol, *addr, ssl=ctx) + tr.close() + await pr.fut + + with self.tcp_server(server) as srv: + ctx = self._create_client_ssl_context() + loop.run_until_complete(client(srv.addr, ctx)) + ctx = weakref.ref(ctx) + + # asyncio has no shutdown timeout, but it ends up with a circular + # reference loop - not ideal (introduces gc glitches), but at least + # not leaking + gc.collect() + gc.collect() + gc.collect() + + # SSLProtocol should be DECREF to 0 + self.assertIsNone(ctx()) + + def test_shutdown_timeout_handler_not_set(self): + loop = self.loop + eof = asyncio.Event() + extra = None + + def server(sock): + sslctx = self._create_server_ssl_context( + test_utils.ONLYCERT, + test_utils.ONLYKEY + ) + sock = sslctx.wrap_socket(sock, server_side=True) + sock.send(b'hello') + assert sock.recv(1024) == b'world' + sock.send(b'extra bytes') + # sending EOF here + sock.shutdown(socket.SHUT_WR) + loop.call_soon_threadsafe(eof.set) + # make sure we have enough time to reproduce the issue + assert sock.recv(1024) == b'' + sock.close() + + class Protocol(asyncio.Protocol): + def __init__(self): + self.fut = asyncio.Future(loop=loop) + self.transport = None + + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + if data == b'hello': + self.transport.write(b'world') + # pause reading would make incoming data stay in the sslobj + self.transport.pause_reading() + else: + nonlocal extra + extra = data + + def connection_lost(self, exc): + if exc is None: + self.fut.set_result(None) + else: + self.fut.set_exception(exc) + + async def client(addr): + ctx = self._create_client_ssl_context() + tr, pr = await loop.create_connection(Protocol, *addr, ssl=ctx) + await eof.wait() + tr.resume_reading() + await pr.fut + tr.close() + assert extra == b'extra bytes' + + with self.tcp_server(server) as srv: + loop.run_until_complete(client(srv.addr)) + + +############################################################################### +# Socket Testing Utilities +############################################################################### + + +class TestSocketWrapper: + + def __init__(self, sock): + self.__sock = sock + + def recv_all(self, n): + buf = b'' + while len(buf) < n: + data = self.recv(n - len(buf)) + if data == b'': + raise ConnectionAbortedError + buf += data + return buf + + def starttls(self, ssl_context, *, + server_side=False, + server_hostname=None, + do_handshake_on_connect=True): + + assert isinstance(ssl_context, ssl.SSLContext) + + ssl_sock = ssl_context.wrap_socket( + self.__sock, server_side=server_side, + server_hostname=server_hostname, + do_handshake_on_connect=do_handshake_on_connect) + + if server_side: + ssl_sock.do_handshake() + + self.__sock.close() + self.__sock = ssl_sock + + def __getattr__(self, name): + return getattr(self.__sock, name) + + def __repr__(self): + return '<{} {!r}>'.format(type(self).__name__, self.__sock) + + +class SocketThread(threading.Thread): + + def stop(self): + self._active = False + self.join() + + def __enter__(self): + self.start() + return self + + def __exit__(self, *exc): + self.stop() + + +class TestThreadedClient(SocketThread): + + def __init__(self, test, sock, prog, timeout): + threading.Thread.__init__(self, None, None, 'test-client') + self.daemon = True + + self._timeout = timeout + self._sock = sock + self._active = True + self._prog = prog + self._test = test + + def run(self): + try: + self._prog(TestSocketWrapper(self._sock)) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as ex: + self._test._abort_socket_test(ex) + + +class TestThreadedServer(SocketThread): + + def __init__(self, test, sock, prog, timeout, max_clients): + threading.Thread.__init__(self, None, None, 'test-server') + self.daemon = True + + self._clients = 0 + self._finished_clients = 0 + self._max_clients = max_clients + self._timeout = timeout + self._sock = sock + self._active = True + + self._prog = prog + + self._s1, self._s2 = socket.socketpair() + self._s1.setblocking(False) + + self._test = test + + def stop(self): + try: + if self._s2 and self._s2.fileno() != -1: + try: + self._s2.send(b'stop') + except OSError: + pass + finally: + super().stop() + self._sock.close() + self._s1.close() + self._s2.close() + + def run(self): + self._sock.setblocking(False) + self._run() + + def _run(self): + while self._active: + if self._clients >= self._max_clients: + return + + r, w, x = select.select( + [self._sock, self._s1], [], [], self._timeout) + + if self._s1 in r: + return + + if self._sock in r: + try: + conn, addr = self._sock.accept() + except BlockingIOError: + continue + except socket.timeout: + if not self._active: + return + else: + raise + else: + self._clients += 1 + conn.settimeout(self._timeout) + try: + with conn: + self._handle_client(conn) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as ex: + self._active = False + try: + raise + finally: + self._test._abort_socket_test(ex) + + def _handle_client(self, sock): + self._prog(TestSocketWrapper(sock)) + + @property + def addr(self): + return self._sock.getsockname() diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py new file mode 100644 index 00000000000..7ab6e1511d7 --- /dev/null +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -0,0 +1,846 @@ +"""Tests for asyncio/sslproto.py.""" + +import logging +import socket +import unittest +import weakref +from test import support +from test.support import socket_helper +from unittest import mock +try: + import ssl +except ImportError: + ssl = None + +import asyncio +from asyncio import log +from asyncio import protocols +from asyncio import sslproto +from test.test_asyncio import utils as test_utils +from test.test_asyncio import functional as func_tests + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +@unittest.skipIf(ssl is None, 'No ssl module') +class SslProtoHandshakeTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def ssl_protocol(self, *, waiter=None, proto=None): + sslcontext = test_utils.dummy_ssl_context() + if proto is None: # app protocol + proto = asyncio.Protocol() + ssl_proto = sslproto.SSLProtocol(self.loop, proto, sslcontext, waiter, + ssl_handshake_timeout=0.1) + self.assertIs(ssl_proto._app_transport.get_protocol(), proto) + self.addCleanup(ssl_proto._app_transport.close) + return ssl_proto + + def connection_made(self, ssl_proto, *, do_handshake=None): + transport = mock.Mock() + sslobj = mock.Mock() + # emulate reading decompressed data + sslobj.read.side_effect = ssl.SSLWantReadError + sslobj.write.side_effect = ssl.SSLWantReadError + if do_handshake is not None: + sslobj.do_handshake = do_handshake + ssl_proto._sslobj = sslobj + ssl_proto.connection_made(transport) + return transport + + def test_handshake_timeout_zero(self): + sslcontext = test_utils.dummy_ssl_context() + app_proto = mock.Mock() + waiter = mock.Mock() + with self.assertRaisesRegex(ValueError, 'a positive number'): + sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, + ssl_handshake_timeout=0) + + def test_handshake_timeout_negative(self): + sslcontext = test_utils.dummy_ssl_context() + app_proto = mock.Mock() + waiter = mock.Mock() + with self.assertRaisesRegex(ValueError, 'a positive number'): + sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, + ssl_handshake_timeout=-10) + + def test_eof_received_waiter(self): + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + self.connection_made( + ssl_proto, + do_handshake=mock.Mock(side_effect=ssl.SSLWantReadError) + ) + ssl_proto.eof_received() + test_utils.run_briefly(self.loop) + self.assertIsInstance(waiter.exception(), ConnectionResetError) + + def test_fatal_error_no_name_error(self): + # From issue #363. + # _fatal_error() generates a NameError if sslproto.py + # does not import base_events. + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + # Temporarily turn off error logging so as not to spoil test output. + log_level = log.logger.getEffectiveLevel() + log.logger.setLevel(logging.FATAL) + try: + ssl_proto._fatal_error(None) + finally: + # Restore error logging. + log.logger.setLevel(log_level) + + def test_connection_lost(self): + # From issue #472. + # yield from waiter hang if lost_connection was called. + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + self.connection_made( + ssl_proto, + do_handshake=mock.Mock(side_effect=ssl.SSLWantReadError) + ) + ssl_proto.connection_lost(ConnectionAbortedError) + test_utils.run_briefly(self.loop) + self.assertIsInstance(waiter.exception(), ConnectionAbortedError) + + def test_connection_lost_when_busy(self): + # gh-118950: SSLProtocol.connection_lost not being called when OSError + # is thrown on asyncio.write. + sock = mock.Mock() + sock.fileno = mock.Mock(return_value=12345) + sock.send = mock.Mock(side_effect=BrokenPipeError) + + # construct StreamWriter chain that contains loop dependant logic this emulates + # what _make_ssl_transport() does in BaseSelectorEventLoop + reader = asyncio.StreamReader(limit=2 ** 16, loop=self.loop) + protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) + ssl_proto = self.ssl_protocol(proto=protocol) + + # emulate reading decompressed data + sslobj = mock.Mock() + sslobj.read.side_effect = ssl.SSLWantReadError + sslobj.write.side_effect = ssl.SSLWantReadError + ssl_proto._sslobj = sslobj + + # emulate outgoing data + data = b'An interesting message' + + outgoing = mock.Mock() + outgoing.read = mock.Mock(return_value=data) + outgoing.pending = len(data) + ssl_proto._outgoing = outgoing + + # use correct socket transport to initialize the SSLProtocol + self.loop._make_socket_transport(sock, ssl_proto) + + transport = ssl_proto._app_transport + writer = asyncio.StreamWriter(transport, protocol, reader, self.loop) + + async def main(): + # writes data to transport + async def write(): + writer.write(data) + await writer.drain() + + # try to write for the first time + await write() + # try to write for the second time, this raises as the connection_lost + # callback should be done with error + with self.assertRaises(ConnectionResetError): + await write() + + self.loop.run_until_complete(main()) + + def test_close_during_handshake(self): + # bpo-29743 Closing transport during handshake process leaks socket + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + + transport = self.connection_made( + ssl_proto, + do_handshake=mock.Mock(side_effect=ssl.SSLWantReadError) + ) + test_utils.run_briefly(self.loop) + + ssl_proto._app_transport.close() + self.assertTrue(transport._force_close.called) + + def test_close_during_ssl_over_ssl(self): + # gh-113214: passing exceptions from the inner wrapped SSL protocol to the + # shim transport provided by the outer SSL protocol should not raise + # attribute errors + outer = self.ssl_protocol(proto=self.ssl_protocol()) + self.connection_made(outer) + # Closing the outer app transport should not raise an exception + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + outer._app_transport.close() + self.assertEqual(messages, []) + + def test_get_extra_info_on_closed_connection(self): + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + default = object() + self.assertIs(ssl_proto._get_extra_info('socket', default), default) + self.connection_made(ssl_proto) + self.assertIsNotNone(ssl_proto._get_extra_info('socket')) + ssl_proto.connection_lost(None) + self.assertIsNone(ssl_proto._get_extra_info('socket')) + + def test_set_new_app_protocol(self): + waiter = self.loop.create_future() + ssl_proto = self.ssl_protocol(waiter=waiter) + new_app_proto = asyncio.Protocol() + ssl_proto._app_transport.set_protocol(new_app_proto) + self.assertIs(ssl_proto._app_transport.get_protocol(), new_app_proto) + self.assertIs(ssl_proto._app_protocol, new_app_proto) + + def test_data_received_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + + transp.close() + + # should not raise + self.assertIsNone(ssl_proto.buffer_updated(5)) + + def test_write_after_closing(self): + ssl_proto = self.ssl_protocol() + self.connection_made(ssl_proto) + transp = ssl_proto._app_transport + transp.close() + + # should not raise + self.assertIsNone(transp.write(b'data')) + + +############################################################################## +# Start TLS Tests +############################################################################## + + +class BaseStartTLS(func_tests.FunctionalTestCaseMixin): + + PAYLOAD_SIZE = 1024 * 100 + TIMEOUT = support.LONG_TIMEOUT + + def new_loop(self): + raise NotImplementedError + + def test_buf_feed_data(self): + + class Proto(asyncio.BufferedProtocol): + + def __init__(self, bufsize, usemv): + self.buf = bytearray(bufsize) + self.mv = memoryview(self.buf) + self.data = b'' + self.usemv = usemv + + def get_buffer(self, sizehint): + if self.usemv: + return self.mv + else: + return self.buf + + def buffer_updated(self, nsize): + if self.usemv: + self.data += self.mv[:nsize] + else: + self.data += self.buf[:nsize] + + for usemv in [False, True]: + proto = Proto(1, usemv) + protocols._feed_data_to_buffered_proto(proto, b'12345') + self.assertEqual(proto.data, b'12345') + + proto = Proto(2, usemv) + protocols._feed_data_to_buffered_proto(proto, b'12345') + self.assertEqual(proto.data, b'12345') + + proto = Proto(2, usemv) + protocols._feed_data_to_buffered_proto(proto, b'1234') + self.assertEqual(proto.data, b'1234') + + proto = Proto(4, usemv) + protocols._feed_data_to_buffered_proto(proto, b'1234') + self.assertEqual(proto.data, b'1234') + + proto = Proto(100, usemv) + protocols._feed_data_to_buffered_proto(proto, b'12345') + self.assertEqual(proto.data, b'12345') + + proto = Proto(0, usemv) + with self.assertRaisesRegex(RuntimeError, 'empty buffer'): + protocols._feed_data_to_buffered_proto(proto, b'12345') + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_start_tls_client_reg_proto_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data, b'O') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + # No garbage is left if SSL is closed uncleanly + client_context = weakref.ref(client_context) + support.gc_collect() + self.assertIsNone(client_context()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_create_connection_memory_leak(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + sock.start_tls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + # XXX: We assume user stores the transport in protocol + proto.tr = tr + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr, + ssl=client_context) + + self.assertEqual(await on_data, b'O') + tr.write(HELLO_MSG) + await on_eof + + tr.close() + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + # No garbage is left for SSL client from loop.create_connection, even + # if user stores the SSLTransport in corresponding protocol instance + client_context = weakref.ref(client_context) + support.gc_collect() + self.assertIsNone(client_context()) + + @socket_helper.skip_if_tcp_blackhole + def test_start_tls_client_buf_proto_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + client_con_made_calls = 0 + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.sendall(b'2') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + class ClientProtoFirst(asyncio.BufferedProtocol): + def __init__(self, on_data): + self.on_data = on_data + self.buf = bytearray(1) + + def connection_made(self, tr): + nonlocal client_con_made_calls + client_con_made_calls += 1 + + def get_buffer(self, sizehint): + return self.buf + + def buffer_updated(slf, nsize): + self.assertEqual(nsize, 1) + slf.on_data.set_result(bytes(slf.buf[:nsize])) + + class ClientProtoSecond(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(self, tr): + nonlocal client_con_made_calls + client_con_made_calls += 1 + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data1 = self.loop.create_future() + on_data2 = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProtoFirst(on_data1), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data1, b'O') + new_tr.write(HELLO_MSG) + + new_tr.set_protocol(ClientProtoSecond(on_data2, on_eof)) + self.assertEqual(await on_data2, b'2') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + # connection_made() should be called only once -- when + # we establish connection for the first time. Start TLS + # doesn't call connection_made() on application protocols. + self.assertEqual(client_con_made_calls, 1) + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=self.TIMEOUT)) + + def test_start_tls_slow_client_cancel(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + + client_context = test_utils.simple_client_sslcontext() + server_waits_on_handshake = self.loop.create_future() + + def serve(sock): + sock.settimeout(self.TIMEOUT) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + try: + self.loop.call_soon_threadsafe( + server_waits_on_handshake.set_result, None) + data = sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + pass + finally: + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + + await server_waits_on_handshake + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for( + self.loop.start_tls(tr, proto, client_context), + 0.5) + + with self.tcp_server(serve, timeout=self.TIMEOUT) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), + timeout=support.SHORT_TIMEOUT)) + + @socket_helper.skip_if_tcp_blackhole + def test_start_tls_server_1(self): + HELLO_MSG = b'1' * self.PAYLOAD_SIZE + ANSWER = b'answer' + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + answer = None + + def client(sock, addr): + nonlocal answer + sock.settimeout(self.TIMEOUT) + + sock.connect(addr) + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(client_context) + sock.sendall(HELLO_MSG) + answer = sock.recv_all(len(ANSWER)) + sock.close() + + class ServerProto(asyncio.Protocol): + def __init__(self, on_con, on_con_lost, on_got_hello): + self.on_con = on_con + self.on_con_lost = on_con_lost + self.on_got_hello = on_got_hello + self.data = b'' + self.transport = None + + def connection_made(self, tr): + self.transport = tr + self.on_con.set_result(tr) + + def replace_transport(self, tr): + self.transport = tr + + def data_received(self, data): + self.data += data + if len(self.data) >= len(HELLO_MSG): + self.on_got_hello.set_result(None) + + def connection_lost(self, exc): + self.transport = None + if exc is None: + self.on_con_lost.set_result(None) + else: + self.on_con_lost.set_exception(exc) + + async def main(proto, on_con, on_con_lost, on_got_hello): + tr = await on_con + tr.write(HELLO_MSG) + + self.assertEqual(proto.data, b'') + + new_tr = await self.loop.start_tls( + tr, proto, server_context, + server_side=True, + ssl_handshake_timeout=self.TIMEOUT) + proto.replace_transport(new_tr) + + await on_got_hello + new_tr.write(ANSWER) + + await on_con_lost + self.assertEqual(proto.data, HELLO_MSG) + new_tr.close() + + async def run_main(): + on_con = self.loop.create_future() + on_con_lost = self.loop.create_future() + on_got_hello = self.loop.create_future() + proto = ServerProto(on_con, on_con_lost, on_got_hello) + + server = await self.loop.create_server( + lambda: proto, '127.0.0.1', 0) + addr = server.sockets[0].getsockname() + + with self.tcp_client(lambda sock: client(sock, addr), + timeout=self.TIMEOUT): + await asyncio.wait_for( + main(proto, on_con, on_con_lost, on_got_hello), + timeout=self.TIMEOUT) + + server.close() + await server.wait_closed() + self.assertEqual(answer, ANSWER) + + self.loop.run_until_complete(run_main()) + + def test_start_tls_wrong_args(self): + async def main(): + with self.assertRaisesRegex(TypeError, 'SSLContext, got'): + await self.loop.start_tls(None, None, None) + + sslctx = test_utils.simple_server_sslcontext() + with self.assertRaisesRegex(TypeError, 'is not supported'): + await self.loop.start_tls(None, None, sslctx) + + self.loop.run_until_complete(main()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - gc.collect() doesn't release SSLContext properly + def test_handshake_timeout(self): + # bpo-29970: Check that a connection is aborted if handshake is not + # completed in timeout period, instead of remaining open indefinitely + client_sslctx = test_utils.simple_client_sslcontext() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + server_side_aborted = False + + def server(sock): + nonlocal server_side_aborted + try: + sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + server_side_aborted = True + finally: + sock.close() + + async def client(addr): + await asyncio.wait_for( + self.loop.create_connection( + asyncio.Protocol, + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=support.SHORT_TIMEOUT), + 0.5) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(client(srv.addr)) + + self.assertTrue(server_side_aborted) + + # Python issue #23197: cancelling a handshake must not raise an + # exception or log an error, even if the handshake failed + self.assertEqual(messages, []) + + # The 10s handshake timeout should be cancelled to free related + # objects without really waiting for 10s + client_sslctx = weakref.ref(client_sslctx) + support.gc_collect() + self.assertIsNone(client_sslctx()) + + def test_create_connection_ssl_slow_handshake(self): + client_sslctx = test_utils.simple_client_sslcontext() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + def server(sock): + try: + sock.recv_all(1024 * 1024) + except ConnectionAbortedError: + pass + finally: + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=1.0) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaisesRegex( + ConnectionAbortedError, + r'SSL handshake.*is taking longer'): + + self.loop.run_until_complete(client(srv.addr)) + + self.assertEqual(messages, []) + + def test_create_connection_ssl_failed_certificate(self): + self.loop.set_exception_handler(lambda loop, ctx: None) + + sslctx = test_utils.simple_server_sslcontext() + client_sslctx = test_utils.simple_client_sslcontext( + disable_verify=False) + + def server(sock): + try: + sock.start_tls( + sslctx, + server_side=True) + except ssl.SSLError: + pass + except OSError: + pass + finally: + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='', + ssl_handshake_timeout=support.LOOPBACK_TIMEOUT) + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + with self.assertRaises(ssl.SSLCertVerificationError): + self.loop.run_until_complete(client(srv.addr)) + + def test_start_tls_client_corrupted_ssl(self): + self.loop.set_exception_handler(lambda loop, ctx: None) + + sslctx = test_utils.simple_server_sslcontext() + client_sslctx = test_utils.simple_client_sslcontext() + + def server(sock): + orig_sock = sock.dup() + try: + sock.start_tls( + sslctx, + server_side=True) + sock.sendall(b'A\n') + sock.recv_all(1) + orig_sock.send(b'please corrupt the SSL connection') + except ssl.SSLError: + pass + finally: + orig_sock.close() + sock.close() + + async def client(addr): + reader, writer = await asyncio.open_connection( + *addr, + ssl=client_sslctx, + server_hostname='') + + self.assertEqual(await reader.readline(), b'A\n') + writer.write(b'B') + with self.assertRaises(ssl.SSLError): + await reader.readline() + + writer.close() + return 'OK' + + with self.tcp_server(server, + max_clients=1, + backlog=1) as srv: + + res = self.loop.run_until_complete(client(srv.addr)) + + self.assertEqual(res, 'OK') + + +@unittest.skipIf(ssl is None, 'No ssl module') +class SelectorStartTLSTests(BaseStartTLS, unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + +@unittest.skipIf(ssl is None, 'No ssl module') +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class ProactorStartTLSTests(BaseStartTLS, unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py new file mode 100644 index 00000000000..32e4817b70d --- /dev/null +++ b/Lib/test/test_asyncio/test_staggered.py @@ -0,0 +1,151 @@ +import asyncio +import unittest +from asyncio.staggered import staggered_race + +from test import support + +support.requires_working_socket(module=True) + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class StaggeredTests(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + winner, index, excs = await staggered_race( + [], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(excs, []) + + async def test_one_successful(self): + async def coro(index): + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 0') + self.assertEqual(index, 0) + self.assertEqual(excs, [None]) + + async def test_first_error_second_successful(self): + async def coro(index): + if index == 0: + raise ValueError(index) + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIs(excs[1], None) + + async def test_first_timeout_second_successful(self): + async def coro(index): + if index == 0: + await asyncio.sleep(10) # much bigger than delay + return f'Res: {index}' + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=0.1, + ) + + self.assertEqual(winner, 'Res: 1') + self.assertEqual(index, 1) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], asyncio.CancelledError) + self.assertIs(excs[1], None) + + async def test_none_successful(self): + async def coro(index): + raise ValueError(index) + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + ], + delay=None, + ) + + self.assertIs(winner, None) + self.assertIs(index, None) + self.assertEqual(len(excs), 2) + self.assertIsInstance(excs[0], ValueError) + self.assertIsInstance(excs[1], ValueError) + + + async def test_multiple_winners(self): + event = asyncio.Event() + + async def coro(index): + await event.wait() + return index + + async def do_set(): + event.set() + await asyncio.Event().wait() + + winner, index, excs = await staggered_race( + [ + lambda: coro(0), + lambda: coro(1), + do_set, + ], + delay=0.1, + ) + self.assertIs(winner, 0) + self.assertIs(index, 0) + self.assertEqual(len(excs), 3) + self.assertIsNone(excs[0], None) + self.assertIsInstance(excs[1], asyncio.CancelledError) + self.assertIsInstance(excs[2], asyncio.CancelledError) + + + async def test_cancelled(self): + log = [] + with self.assertRaises(TimeoutError): + async with asyncio.timeout(None) as cs_outer, asyncio.timeout(None) as cs_inner: + async def coro_fn(): + cs_inner.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 1") + + cs_outer.reschedule(-1) + await asyncio.sleep(0) + try: + await asyncio.sleep(0) + except asyncio.CancelledError: + log.append("cancelled 2") + try: + await staggered_race([coro_fn], delay=None) + except asyncio.CancelledError: + log.append("cancelled 3") + raise + + self.assertListEqual(log, ["cancelled 1", "cancelled 2", "cancelled 3"]) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py new file mode 100644 index 00000000000..5f0fc6a7a9d --- /dev/null +++ b/Lib/test/test_asyncio/test_streams.py @@ -0,0 +1,1221 @@ +"""Tests for streams.py.""" + +import gc +import queue +import pickle +import socket +import threading +import unittest +from unittest import mock +try: + import ssl +except ImportError: + ssl = None + +import asyncio +from test.test_asyncio import utils as test_utils +from test.support import socket_helper + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class StreamTests(test_utils.TestCase): + + DATA = b'line1\nline2\nline3\n' + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + # just in case if we have transport close callbacks + test_utils.run_briefly(self.loop) + + # set_event_loop() takes care of closing self.loop in a safe way + super().tearDown() + + def _basetest_open_connection(self, open_connection_fut): + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + reader, writer = self.loop.run_until_complete(open_connection_fut) + writer.write(b'GET / HTTP/1.0\r\n\r\n') + f = reader.readline() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + f = reader.read() + data = self.loop.run_until_complete(f) + self.assertEndsWith(data, b'\r\n\r\nTest message') + writer.close() + self.assertEqual(messages, []) + + def test_open_connection(self): + with test_utils.run_test_server() as httpd: + conn_fut = asyncio.open_connection(*httpd.address) + self._basetest_open_connection(conn_fut) + + @socket_helper.skip_unless_bind_unix_socket + def test_open_unix_connection(self): + with test_utils.run_test_unix_server() as httpd: + conn_fut = asyncio.open_unix_connection(httpd.address) + self._basetest_open_connection(conn_fut) + + def _basetest_open_connection_no_loop_ssl(self, open_connection_fut): + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + try: + reader, writer = self.loop.run_until_complete(open_connection_fut) + finally: + asyncio.set_event_loop(None) + writer.write(b'GET / HTTP/1.0\r\n\r\n') + f = reader.read() + data = self.loop.run_until_complete(f) + self.assertEndsWith(data, b'\r\n\r\nTest message') + + writer.close() + self.assertEqual(messages, []) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_open_connection_no_loop_ssl(self): + with test_utils.run_test_server(use_ssl=True) as httpd: + conn_fut = asyncio.open_connection( + *httpd.address, + ssl=test_utils.dummy_ssl_context()) + + self._basetest_open_connection_no_loop_ssl(conn_fut) + + @socket_helper.skip_unless_bind_unix_socket + @unittest.skipIf(ssl is None, 'No ssl module') + def test_open_unix_connection_no_loop_ssl(self): + with test_utils.run_test_unix_server(use_ssl=True) as httpd: + conn_fut = asyncio.open_unix_connection( + httpd.address, + ssl=test_utils.dummy_ssl_context(), + server_hostname='', + ) + + self._basetest_open_connection_no_loop_ssl(conn_fut) + + def _basetest_open_connection_error(self, open_connection_fut): + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + reader, writer = self.loop.run_until_complete(open_connection_fut) + writer._protocol.connection_lost(ZeroDivisionError()) + f = reader.read() + with self.assertRaises(ZeroDivisionError): + self.loop.run_until_complete(f) + writer.close() + test_utils.run_briefly(self.loop) + self.assertEqual(messages, []) + + def test_open_connection_error(self): + with test_utils.run_test_server() as httpd: + conn_fut = asyncio.open_connection(*httpd.address) + self._basetest_open_connection_error(conn_fut) + + @socket_helper.skip_unless_bind_unix_socket + def test_open_unix_connection_error(self): + with test_utils.run_test_unix_server() as httpd: + conn_fut = asyncio.open_unix_connection(httpd.address) + self._basetest_open_connection_error(conn_fut) + + def test_feed_empty_data(self): + stream = asyncio.StreamReader(loop=self.loop) + + stream.feed_data(b'') + self.assertEqual(b'', stream._buffer) + + def test_feed_nonempty_data(self): + stream = asyncio.StreamReader(loop=self.loop) + + stream.feed_data(self.DATA) + self.assertEqual(self.DATA, stream._buffer) + + def test_read_zero(self): + # Read zero bytes. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(self.DATA) + + data = self.loop.run_until_complete(stream.read(0)) + self.assertEqual(b'', data) + self.assertEqual(self.DATA, stream._buffer) + + def test_read(self): + # Read bytes. + stream = asyncio.StreamReader(loop=self.loop) + read_task = self.loop.create_task(stream.read(30)) + + def cb(): + stream.feed_data(self.DATA) + self.loop.call_soon(cb) + + data = self.loop.run_until_complete(read_task) + self.assertEqual(self.DATA, data) + self.assertEqual(b'', stream._buffer) + + def test_read_line_breaks(self): + # Read bytes without line breaks. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'line1') + stream.feed_data(b'line2') + + data = self.loop.run_until_complete(stream.read(5)) + + self.assertEqual(b'line1', data) + self.assertEqual(b'line2', stream._buffer) + + def test_read_eof(self): + # Read bytes, stop at eof. + stream = asyncio.StreamReader(loop=self.loop) + read_task = self.loop.create_task(stream.read(1024)) + + def cb(): + stream.feed_eof() + self.loop.call_soon(cb) + + data = self.loop.run_until_complete(read_task) + self.assertEqual(b'', data) + self.assertEqual(b'', stream._buffer) + + def test_read_until_eof(self): + # Read all bytes until eof. + stream = asyncio.StreamReader(loop=self.loop) + read_task = self.loop.create_task(stream.read(-1)) + + def cb(): + stream.feed_data(b'chunk1\n') + stream.feed_data(b'chunk2') + stream.feed_eof() + self.loop.call_soon(cb) + + data = self.loop.run_until_complete(read_task) + + self.assertEqual(b'chunk1\nchunk2', data) + self.assertEqual(b'', stream._buffer) + + def test_read_exception(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'line\n') + + data = self.loop.run_until_complete(stream.read(2)) + self.assertEqual(b'li', data) + + stream.set_exception(ValueError()) + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.read(2)) + + def test_invalid_limit(self): + with self.assertRaisesRegex(ValueError, 'imit'): + asyncio.StreamReader(limit=0, loop=self.loop) + + with self.assertRaisesRegex(ValueError, 'imit'): + asyncio.StreamReader(limit=-1, loop=self.loop) + + def test_read_limit(self): + stream = asyncio.StreamReader(limit=3, loop=self.loop) + stream.feed_data(b'chunk') + data = self.loop.run_until_complete(stream.read(5)) + self.assertEqual(b'chunk', data) + self.assertEqual(b'', stream._buffer) + + def test_readline(self): + # Read one line. 'readline' will need to wait for the data + # to come from 'cb' + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'chunk1 ') + read_task = self.loop.create_task(stream.readline()) + + def cb(): + stream.feed_data(b'chunk2 ') + stream.feed_data(b'chunk3 ') + stream.feed_data(b'\n chunk4') + self.loop.call_soon(cb) + + line = self.loop.run_until_complete(read_task) + self.assertEqual(b'chunk1 chunk2 chunk3 \n', line) + self.assertEqual(b' chunk4', stream._buffer) + + def test_readline_limit_with_existing_data(self): + # Read one line. The data is in StreamReader's buffer + # before the event loop is run. + + stream = asyncio.StreamReader(limit=3, loop=self.loop) + stream.feed_data(b'li') + stream.feed_data(b'ne1\nline2\n') + + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + # The buffer should contain the remaining data after exception + self.assertEqual(b'line2\n', stream._buffer) + + stream = asyncio.StreamReader(limit=3, loop=self.loop) + stream.feed_data(b'li') + stream.feed_data(b'ne1') + stream.feed_data(b'li') + + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + # No b'\n' at the end. The 'limit' is set to 3. So before + # waiting for the new data in buffer, 'readline' will consume + # the entire buffer, and since the length of the consumed data + # is more than 3, it will raise a ValueError. The buffer is + # expected to be empty now. + self.assertEqual(b'', stream._buffer) + + def test_at_eof(self): + stream = asyncio.StreamReader(loop=self.loop) + self.assertFalse(stream.at_eof()) + + stream.feed_data(b'some data\n') + self.assertFalse(stream.at_eof()) + + self.loop.run_until_complete(stream.readline()) + self.assertFalse(stream.at_eof()) + + stream.feed_data(b'some data\n') + stream.feed_eof() + self.loop.run_until_complete(stream.readline()) + self.assertTrue(stream.at_eof()) + + def test_readline_limit(self): + # Read one line. StreamReaders are fed with data after + # their 'readline' methods are called. + + stream = asyncio.StreamReader(limit=7, loop=self.loop) + def cb(): + stream.feed_data(b'chunk1') + stream.feed_data(b'chunk2') + stream.feed_data(b'chunk3\n') + stream.feed_eof() + self.loop.call_soon(cb) + + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + # The buffer had just one line of data, and after raising + # a ValueError it should be empty. + self.assertEqual(b'', stream._buffer) + + stream = asyncio.StreamReader(limit=7, loop=self.loop) + def cb(): + stream.feed_data(b'chunk1') + stream.feed_data(b'chunk2\n') + stream.feed_data(b'chunk3\n') + stream.feed_eof() + self.loop.call_soon(cb) + + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + self.assertEqual(b'chunk3\n', stream._buffer) + + # check strictness of the limit + stream = asyncio.StreamReader(limit=7, loop=self.loop) + stream.feed_data(b'1234567\n') + line = self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'1234567\n', line) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'12345678\n') + with self.assertRaises(ValueError) as cm: + self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'12345678') + with self.assertRaises(ValueError) as cm: + self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'', stream._buffer) + + def test_readline_nolimit_nowait(self): + # All needed data for the first 'readline' call will be + # in the buffer. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(self.DATA[:6]) + stream.feed_data(self.DATA[6:]) + + line = self.loop.run_until_complete(stream.readline()) + + self.assertEqual(b'line1\n', line) + self.assertEqual(b'line2\nline3\n', stream._buffer) + + def test_readline_eof(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'some data') + stream.feed_eof() + + line = self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'some data', line) + + def test_readline_empty_eof(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_eof() + + line = self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'', line) + + def test_readline_read_byte_count(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(self.DATA) + + self.loop.run_until_complete(stream.readline()) + + data = self.loop.run_until_complete(stream.read(7)) + + self.assertEqual(b'line2\nl', data) + self.assertEqual(b'ine3\n', stream._buffer) + + def test_readline_exception(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'line\n') + + data = self.loop.run_until_complete(stream.readline()) + self.assertEqual(b'line\n', data) + + stream.set_exception(ValueError()) + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readline()) + self.assertEqual(b'', stream._buffer) + + def test_readuntil_separator(self): + stream = asyncio.StreamReader(loop=self.loop) + with self.assertRaisesRegex(ValueError, 'Separator should be'): + self.loop.run_until_complete(stream.readuntil(separator=b'')) + with self.assertRaisesRegex(ValueError, 'Separator should be'): + self.loop.run_until_complete(stream.readuntil(separator=(b'',))) + with self.assertRaisesRegex(ValueError, 'Separator should contain'): + self.loop.run_until_complete(stream.readuntil(separator=())) + + def test_readuntil_multi_chunks(self): + stream = asyncio.StreamReader(loop=self.loop) + + stream.feed_data(b'lineAAA') + data = self.loop.run_until_complete(stream.readuntil(separator=b'AAA')) + self.assertEqual(b'lineAAA', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'lineAAA') + data = self.loop.run_until_complete(stream.readuntil(b'AAA')) + self.assertEqual(b'lineAAA', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'lineAAAxxx') + data = self.loop.run_until_complete(stream.readuntil(b'AAA')) + self.assertEqual(b'lineAAA', data) + self.assertEqual(b'xxx', stream._buffer) + + def test_readuntil_multi_chunks_1(self): + stream = asyncio.StreamReader(loop=self.loop) + + stream.feed_data(b'QWEaa') + stream.feed_data(b'XYaa') + stream.feed_data(b'a') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'QWEaaXYaaa', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'QWEaa') + stream.feed_data(b'XYa') + stream.feed_data(b'aa') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'QWEaaXYaaa', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'aaa') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'aaa', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'Xaaa') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'Xaaa', data) + self.assertEqual(b'', stream._buffer) + + stream.feed_data(b'XXX') + stream.feed_data(b'a') + stream.feed_data(b'a') + stream.feed_data(b'a') + data = self.loop.run_until_complete(stream.readuntil(b'aaa')) + self.assertEqual(b'XXXaaa', data) + self.assertEqual(b'', stream._buffer) + + def test_readuntil_eof(self): + stream = asyncio.StreamReader(loop=self.loop) + data = b'some dataAA' + stream.feed_data(data) + stream.feed_eof() + + with self.assertRaisesRegex(asyncio.IncompleteReadError, + 'undefined expected bytes') as cm: + self.loop.run_until_complete(stream.readuntil(b'AAA')) + self.assertEqual(cm.exception.partial, data) + self.assertIsNone(cm.exception.expected) + self.assertEqual(b'', stream._buffer) + + def test_readuntil_limit_found_sep(self): + stream = asyncio.StreamReader(loop=self.loop, limit=3) + stream.feed_data(b'some dataAA') + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'not found') as cm: + self.loop.run_until_complete(stream.readuntil(b'AAA')) + + self.assertEqual(b'some dataAA', stream._buffer) + + stream.feed_data(b'A') + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'is found') as cm: + self.loop.run_until_complete(stream.readuntil(b'AAA')) + + self.assertEqual(b'some dataAAA', stream._buffer) + + def test_readuntil_multi_separator(self): + stream = asyncio.StreamReader(loop=self.loop) + + # Simple case + stream.feed_data(b'line 1\nline 2\r') + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 1\n', data) + data = self.loop.run_until_complete(stream.readuntil((b'\r', b'\n'))) + self.assertEqual(b'line 2\r', data) + self.assertEqual(b'', stream._buffer) + + # First end position matches, even if that's a longer match + stream.feed_data(b'ABCDEFG') + data = self.loop.run_until_complete(stream.readuntil((b'DEF', b'BCDE'))) + self.assertEqual(b'ABCDE', data) + self.assertEqual(b'FG', stream._buffer) + + def test_readuntil_multi_separator_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=3) + stream.feed_data(b'some dataA') + + with self.assertRaisesRegex(asyncio.LimitOverrunError, + 'is found') as cm: + self.loop.run_until_complete(stream.readuntil((b'A', b'ome dataA'))) + + self.assertEqual(b'some dataA', stream._buffer) + + def test_readuntil_multi_separator_negative_offset(self): + # If the buffer is big enough for the smallest separator (but does + # not contain it) but too small for the largest, `offset` must not + # become negative. + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + + readuntil_task = self.loop.create_task(stream.readuntil((b'A', b'long sep'))) + self.loop.call_soon(stream.feed_data, b'Z') + self.loop.call_soon(stream.feed_data, b'Aaaa') + + data = self.loop.run_until_complete(readuntil_task) + self.assertEqual(b'dataZA', data) + self.assertEqual(b'aaa', stream._buffer) + + def test_readuntil_bytearray(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'some data\r\n') + data = self.loop.run_until_complete(stream.readuntil(bytearray(b'\r\n'))) + self.assertEqual(b'some data\r\n', data) + self.assertEqual(b'', stream._buffer) + + def test_readexactly_zero_or_less(self): + # Read exact number of bytes (zero or less). + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(self.DATA) + + data = self.loop.run_until_complete(stream.readexactly(0)) + self.assertEqual(b'', data) + self.assertEqual(self.DATA, stream._buffer) + + with self.assertRaisesRegex(ValueError, 'less than zero'): + self.loop.run_until_complete(stream.readexactly(-1)) + self.assertEqual(self.DATA, stream._buffer) + + def test_readexactly(self): + # Read exact number of bytes. + stream = asyncio.StreamReader(loop=self.loop) + + n = 2 * len(self.DATA) + read_task = self.loop.create_task(stream.readexactly(n)) + + def cb(): + stream.feed_data(self.DATA) + stream.feed_data(self.DATA) + stream.feed_data(self.DATA) + self.loop.call_soon(cb) + + data = self.loop.run_until_complete(read_task) + self.assertEqual(self.DATA + self.DATA, data) + self.assertEqual(self.DATA, stream._buffer) + + def test_readexactly_limit(self): + stream = asyncio.StreamReader(limit=3, loop=self.loop) + stream.feed_data(b'chunk') + data = self.loop.run_until_complete(stream.readexactly(5)) + self.assertEqual(b'chunk', data) + self.assertEqual(b'', stream._buffer) + + def test_readexactly_eof(self): + # Read exact number of bytes (eof). + stream = asyncio.StreamReader(loop=self.loop) + n = 2 * len(self.DATA) + read_task = self.loop.create_task(stream.readexactly(n)) + + def cb(): + stream.feed_data(self.DATA) + stream.feed_eof() + self.loop.call_soon(cb) + + with self.assertRaises(asyncio.IncompleteReadError) as cm: + self.loop.run_until_complete(read_task) + self.assertEqual(cm.exception.partial, self.DATA) + self.assertEqual(cm.exception.expected, n) + self.assertEqual(str(cm.exception), + '18 bytes read on a total of 36 expected bytes') + self.assertEqual(b'', stream._buffer) + + def test_readexactly_exception(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'line\n') + + data = self.loop.run_until_complete(stream.readexactly(2)) + self.assertEqual(b'li', data) + + stream.set_exception(ValueError()) + self.assertRaises( + ValueError, self.loop.run_until_complete, stream.readexactly(2)) + + def test_exception(self): + stream = asyncio.StreamReader(loop=self.loop) + self.assertIsNone(stream.exception()) + + exc = ValueError() + stream.set_exception(exc) + self.assertIs(stream.exception(), exc) + + def test_exception_waiter(self): + stream = asyncio.StreamReader(loop=self.loop) + + async def set_err(): + stream.set_exception(ValueError()) + + t1 = self.loop.create_task(stream.readline()) + t2 = self.loop.create_task(set_err()) + + self.loop.run_until_complete(asyncio.wait([t1, t2])) + + self.assertRaises(ValueError, t1.result) + + def test_exception_cancel(self): + stream = asyncio.StreamReader(loop=self.loop) + + t = self.loop.create_task(stream.readline()) + test_utils.run_briefly(self.loop) + t.cancel() + test_utils.run_briefly(self.loop) + # The following line fails if set_exception() isn't careful. + stream.set_exception(RuntimeError('message')) + test_utils.run_briefly(self.loop) + self.assertIs(stream._waiter, None) + + def test_start_server(self): + + class MyServer: + + def __init__(self, loop): + self.server = None + self.loop = loop + + async def handle_client(self, client_reader, client_writer): + data = await client_reader.readline() + client_writer.write(data) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + def start(self): + sock = socket.create_server(('127.0.0.1', 0)) + self.server = self.loop.run_until_complete( + asyncio.start_server(self.handle_client, + sock=sock)) + return sock.getsockname() + + def handle_client_callback(self, client_reader, client_writer): + self.loop.create_task(self.handle_client(client_reader, + client_writer)) + + def start_callback(self): + sock = socket.create_server(('127.0.0.1', 0)) + addr = sock.getsockname() + sock.close() + self.server = self.loop.run_until_complete( + asyncio.start_server(self.handle_client_callback, + host=addr[0], port=addr[1])) + return addr + + def stop(self): + if self.server is not None: + self.server.close() + self.loop.run_until_complete(self.server.wait_closed()) + self.server = None + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr) + # send a line + writer.write(b"hello world!\n") + # read it back + msgback = await reader.readline() + writer.close() + await writer.wait_closed() + return msgback + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + # test the server variant with a coroutine as client handler + server = MyServer(self.loop) + addr = server.start() + msg = self.loop.run_until_complete(self.loop.create_task(client(addr))) + server.stop() + self.assertEqual(msg, b"hello world!\n") + + # test the server variant with a callback as client handler + server = MyServer(self.loop) + addr = server.start_callback() + msg = self.loop.run_until_complete(self.loop.create_task(client(addr))) + server.stop() + self.assertEqual(msg, b"hello world!\n") + + self.assertEqual(messages, []) + + @socket_helper.skip_unless_bind_unix_socket + def test_start_unix_server(self): + + class MyServer: + + def __init__(self, loop, path): + self.server = None + self.loop = loop + self.path = path + + async def handle_client(self, client_reader, client_writer): + data = await client_reader.readline() + client_writer.write(data) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + def start(self): + self.server = self.loop.run_until_complete( + asyncio.start_unix_server(self.handle_client, + path=self.path)) + + def handle_client_callback(self, client_reader, client_writer): + self.loop.create_task(self.handle_client(client_reader, + client_writer)) + + def start_callback(self): + start = asyncio.start_unix_server(self.handle_client_callback, + path=self.path) + self.server = self.loop.run_until_complete(start) + + def stop(self): + if self.server is not None: + self.server.close() + self.loop.run_until_complete(self.server.wait_closed()) + self.server = None + + async def client(path): + reader, writer = await asyncio.open_unix_connection(path) + # send a line + writer.write(b"hello world!\n") + # read it back + msgback = await reader.readline() + writer.close() + await writer.wait_closed() + return msgback + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + # test the server variant with a coroutine as client handler + with test_utils.unix_socket_path() as path: + server = MyServer(self.loop, path) + server.start() + msg = self.loop.run_until_complete( + self.loop.create_task(client(path))) + server.stop() + self.assertEqual(msg, b"hello world!\n") + + # test the server variant with a callback as client handler + with test_utils.unix_socket_path() as path: + server = MyServer(self.loop, path) + server.start_callback() + msg = self.loop.run_until_complete( + self.loop.create_task(client(path))) + server.stop() + self.assertEqual(msg, b"hello world!\n") + + self.assertEqual(messages, []) + + @unittest.skipIf(ssl is None, 'No ssl module') + def test_start_tls(self): + + class MyServer: + + def __init__(self, loop): + self.server = None + self.loop = loop + + async def handle_client(self, client_reader, client_writer): + data1 = await client_reader.readline() + client_writer.write(data1) + await client_writer.drain() + assert client_writer.get_extra_info('sslcontext') is None + await client_writer.start_tls( + test_utils.simple_server_sslcontext()) + assert client_writer.get_extra_info('sslcontext') is not None + data2 = await client_reader.readline() + client_writer.write(data2) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + def start(self): + sock = socket.create_server(('127.0.0.1', 0)) + self.server = self.loop.run_until_complete( + asyncio.start_server(self.handle_client, + sock=sock)) + return sock.getsockname() + + def stop(self): + if self.server is not None: + self.server.close() + self.loop.run_until_complete(self.server.wait_closed()) + self.server = None + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr) + writer.write(b"hello world 1!\n") + await writer.drain() + msgback1 = await reader.readline() + assert writer.get_extra_info('sslcontext') is None + await writer.start_tls(test_utils.simple_client_sslcontext()) + assert writer.get_extra_info('sslcontext') is not None + writer.write(b"hello world 2!\n") + await writer.drain() + msgback2 = await reader.readline() + writer.close() + await writer.wait_closed() + return msgback1, msgback2 + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + server = MyServer(self.loop) + addr = server.start() + msg1, msg2 = self.loop.run_until_complete(client(addr)) + server.stop() + + self.assertEqual(messages, []) + self.assertEqual(msg1, b"hello world 1!\n") + self.assertEqual(msg2, b"hello world 2!\n") + + def test_streamreader_constructor_without_loop(self): + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReader() + + def test_streamreader_constructor_use_running_loop(self): + # asyncio issue #184: Ensure that StreamReaderProtocol constructor + # retrieves the current loop if the loop parameter is not set + async def test(): + return asyncio.StreamReader() + + reader = self.loop.run_until_complete(test()) + self.assertIs(reader._loop, self.loop) + + def test_streamreader_constructor_use_global_loop(self): + # asyncio issue #184: Ensure that StreamReaderProtocol constructor + # retrieves the current loop if the loop parameter is not set + # Deprecated in 3.10, undeprecated in 3.12 + self.addCleanup(asyncio.set_event_loop, None) + asyncio.set_event_loop(self.loop) + reader = asyncio.StreamReader() + self.assertIs(reader._loop, self.loop) + + + def test_streamreaderprotocol_constructor_without_loop(self): + reader = mock.Mock() + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.StreamReaderProtocol(reader) + + def test_streamreaderprotocol_constructor_use_running_loop(self): + # asyncio issue #184: Ensure that StreamReaderProtocol constructor + # retrieves the current loop if the loop parameter is not set + reader = mock.Mock() + async def test(): + return asyncio.StreamReaderProtocol(reader) + protocol = self.loop.run_until_complete(test()) + self.assertIs(protocol._loop, self.loop) + + def test_streamreaderprotocol_constructor_use_global_loop(self): + # asyncio issue #184: Ensure that StreamReaderProtocol constructor + # retrieves the current loop if the loop parameter is not set + # Deprecated in 3.10, undeprecated in 3.12 + self.addCleanup(asyncio.set_event_loop, None) + asyncio.set_event_loop(self.loop) + reader = mock.Mock() + protocol = asyncio.StreamReaderProtocol(reader) + self.assertIs(protocol._loop, self.loop) + + def test_multiple_drain(self): + # See https://github.com/python/cpython/issues/74116 + drained = 0 + + async def drainer(stream): + nonlocal drained + await stream._drain_helper() + drained += 1 + + async def main(): + loop = asyncio.get_running_loop() + stream = asyncio.streams.FlowControlMixin(loop) + stream.pause_writing() + loop.call_later(0.1, stream.resume_writing) + await asyncio.gather(*[drainer(stream) for _ in range(10)]) + self.assertEqual(drained, 10) + + self.loop.run_until_complete(main()) + + def test_drain_raises(self): + # See http://bugs.python.org/issue25441 + + # This test should not use asyncio for the mock server; the + # whole point of the test is to test for a bug in drain() + # where it never gives up the event loop but the socket is + # closed on the server side. + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + q = queue.Queue() + + def server(): + # Runs in a separate thread. + with socket.create_server(('localhost', 0)) as sock: + addr = sock.getsockname() + q.put(addr) + clt, _ = sock.accept() + clt.close() + + async def client(host, port): + reader, writer = await asyncio.open_connection(host, port) + + while True: + writer.write(b"foo\n") + await writer.drain() + + # Start the server thread and wait for it to be listening. + thread = threading.Thread(target=server) + thread.daemon = True + thread.start() + addr = q.get() + + # Should not be stuck in an infinite loop. + with self.assertRaises((ConnectionResetError, ConnectionAbortedError, + BrokenPipeError)): + self.loop.run_until_complete(client(*addr)) + + # Clean up the thread. (Only on success; on failure, it may + # be stuck in accept().) + thread.join() + self.assertEqual([], messages) + + def test___repr__(self): + stream = asyncio.StreamReader(loop=self.loop) + self.assertEqual("", repr(stream)) + + def test___repr__nondefault_limit(self): + stream = asyncio.StreamReader(loop=self.loop, limit=123) + self.assertEqual("", repr(stream)) + + def test___repr__eof(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_eof() + self.assertEqual("", repr(stream)) + + def test___repr__data(self): + stream = asyncio.StreamReader(loop=self.loop) + stream.feed_data(b'data') + self.assertEqual("", repr(stream)) + + def test___repr__exception(self): + stream = asyncio.StreamReader(loop=self.loop) + exc = RuntimeError() + stream.set_exception(exc) + self.assertEqual("", + repr(stream)) + + def test___repr__waiter(self): + stream = asyncio.StreamReader(loop=self.loop) + stream._waiter = asyncio.Future(loop=self.loop) + self.assertRegex( + repr(stream), + r">") + stream._waiter.set_result(None) + self.loop.run_until_complete(stream._waiter) + stream._waiter = None + self.assertEqual("", repr(stream)) + + def test___repr__transport(self): + stream = asyncio.StreamReader(loop=self.loop) + stream._transport = mock.Mock() + stream._transport.__repr__ = mock.Mock() + stream._transport.__repr__.return_value = "" + self.assertEqual(">", repr(stream)) + + def test_IncompleteReadError_pickleable(self): + e = asyncio.IncompleteReadError(b'abc', 10) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_protocol=proto): + e2 = pickle.loads(pickle.dumps(e, protocol=proto)) + self.assertEqual(str(e), str(e2)) + self.assertEqual(e.partial, e2.partial) + self.assertEqual(e.expected, e2.expected) + + def test_LimitOverrunError_pickleable(self): + e = asyncio.LimitOverrunError('message', 10) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_protocol=proto): + e2 = pickle.loads(pickle.dumps(e, protocol=proto)) + self.assertEqual(str(e), str(e2)) + self.assertEqual(e.consumed, e2.consumed) + + def test_wait_closed_on_close(self): + with test_utils.run_test_server() as httpd: + rd, wr = self.loop.run_until_complete( + asyncio.open_connection(*httpd.address)) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + f = rd.readline() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + f = rd.read() + data = self.loop.run_until_complete(f) + self.assertEndsWith(data, b'\r\n\r\nTest message') + self.assertFalse(wr.is_closing()) + wr.close() + self.assertTrue(wr.is_closing()) + self.loop.run_until_complete(wr.wait_closed()) + + def test_wait_closed_on_close_with_unread_data(self): + with test_utils.run_test_server() as httpd: + rd, wr = self.loop.run_until_complete( + asyncio.open_connection(*httpd.address)) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + f = rd.readline() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + wr.close() + self.loop.run_until_complete(wr.wait_closed()) + + def test_async_writer_api(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertEndsWith(data, b'\r\n\r\nTest message') + wr.close() + await wr.wait_closed() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete(inner(httpd)) + + self.assertEqual(messages, []) + + def test_async_writer_api_exception_after_close(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertEndsWith(data, b'\r\n\r\nTest message') + wr.close() + with self.assertRaises(ConnectionResetError): + wr.write(b'data') + await wr.drain() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete(inner(httpd)) + + self.assertEqual(messages, []) + + def test_eof_feed_when_closing_writer(self): + # See http://bugs.python.org/issue35065 + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + rd, wr = self.loop.run_until_complete( + asyncio.open_connection(*httpd.address)) + + wr.close() + f = wr.wait_closed() + self.loop.run_until_complete(f) + self.assertTrue(rd.at_eof()) + f = rd.read() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'') + + self.assertEqual(messages, []) + + def test_unclosed_resource_warnings(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertEndsWith(data, b'\r\n\r\nTest message') + with self.assertWarns(ResourceWarning) as cm: + del wr + gc.collect() + self.assertEqual(len(cm.warnings), 1) + self.assertStartsWith(str(cm.warnings[0].message), "unclosed " + ) + transport._returncode = None + self.assertEqual( + repr(transport), + "" + ) + transport._pid = None + transport._returncode = None + self.assertEqual( + repr(transport), + "" + ) + transport.close() + + +class SubprocessMixin: + + def test_stdin_stdout(self): + args = PROGRAM_CAT + + async def run(data): + proc = await asyncio.create_subprocess_exec( + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + # feed data + proc.stdin.write(data) + await proc.stdin.drain() + proc.stdin.close() + + # get output and exitcode + data = await proc.stdout.read() + exitcode = await proc.wait() + return (exitcode, data) + + task = run(b'some data') + task = asyncio.wait_for(task, 60.0) + exitcode, stdout = self.loop.run_until_complete(task) + self.assertEqual(exitcode, 0) + self.assertEqual(stdout, b'some data') + + def test_communicate(self): + args = PROGRAM_CAT + + async def run(data): + proc = await asyncio.create_subprocess_exec( + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate(data) + return proc.returncode, stdout + + task = run(b'some data') + task = asyncio.wait_for(task, support.LONG_TIMEOUT) + exitcode, stdout = self.loop.run_until_complete(task) + self.assertEqual(exitcode, 0) + self.assertEqual(stdout, b'some data') + + def test_communicate_none_input(self): + args = PROGRAM_CAT + + async def run(): + proc = await asyncio.create_subprocess_exec( + *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + return proc.returncode, stdout + + task = run() + task = asyncio.wait_for(task, support.LONG_TIMEOUT) + exitcode, stdout = self.loop.run_until_complete(task) + self.assertEqual(exitcode, 0) + self.assertEqual(stdout, b'') + + def test_shell(self): + proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell('exit 7') + ) + exitcode = self.loop.run_until_complete(proc.wait()) + self.assertEqual(exitcode, 7) + + def test_start_new_session(self): + # start the new process in a new session + proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell( + 'exit 8', + start_new_session=True, + ) + ) + exitcode = self.loop.run_until_complete(proc.wait()) + self.assertEqual(exitcode, 8) + + def test_kill(self): + args = PROGRAM_BLOCKED + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec(*args) + ) + proc.kill() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + + def test_kill_issue43884(self): + if sys.platform == 'win32': + blocking_shell_command = f'"{sys.executable}" -c "import time; time.sleep(2)"' + else: + blocking_shell_command = 'sleep 1; sleep 1' + creationflags = 0 + if sys.platform == 'win32': + from subprocess import CREATE_NEW_PROCESS_GROUP + # On windows create a new process group so that killing process + # kills the process and all its children. + creationflags = CREATE_NEW_PROCESS_GROUP + proc = self.loop.run_until_complete( + asyncio.create_subprocess_shell(blocking_shell_command, stdout=asyncio.subprocess.PIPE, + creationflags=creationflags) + ) + self.loop.run_until_complete(asyncio.sleep(1)) + if sys.platform == 'win32': + proc.send_signal(signal.CTRL_BREAK_EVENT) + # On windows it is an alias of terminate which sets the return code + proc.kill() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGKILL, returncode) + + def test_terminate(self): + args = PROGRAM_BLOCKED + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec(*args) + ) + proc.terminate() + returncode = self.loop.run_until_complete(proc.wait()) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + # expect 1 but sometimes get 0 + else: + self.assertEqual(-signal.SIGTERM, returncode) + + @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP") + def test_send_signal(self): + # bpo-31034: Make sure that we get the default signal handler (killing + # the process). The parent process may have decided to ignore SIGHUP, + # and signal handlers are inherited. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_DFL) + try: + code = 'import time; print("sleeping", flush=True); time.sleep(3600)' + args = [sys.executable, '-c', code] + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec( + *args, + stdout=subprocess.PIPE, + ) + ) + + async def send_signal(proc): + # basic synchronization to wait until the program is sleeping + line = await proc.stdout.readline() + self.assertEqual(line, b'sleeping\n') + + proc.send_signal(signal.SIGHUP) + returncode = await proc.wait() + return returncode + + returncode = self.loop.run_until_complete(send_signal(proc)) + self.assertEqual(-signal.SIGHUP, returncode) + finally: + signal.signal(signal.SIGHUP, old_handler) + + def test_stdin_broken_pipe(self): + # buffer large enough to feed the whole pipe buffer + large_data = b'x' * support.PIPE_MAX_SIZE + + rfd, wfd = os.pipe() + self.addCleanup(os.close, rfd) + self.addCleanup(os.close, wfd) + if support.MS_WINDOWS: + handle = msvcrt.get_osfhandle(rfd) + os.set_handle_inheritable(handle, True) + code = textwrap.dedent(f''' + import os, msvcrt + handle = {handle} + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + os.read(fd, 1) + ''') + from subprocess import STARTUPINFO + startupinfo = STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": [handle]} + kwargs = dict(startupinfo=startupinfo) + else: + code = f'import os; fd = {rfd}; os.read(fd, 1)' + kwargs = dict(pass_fds=(rfd,)) + + # the program ends before the stdin can be fed + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=subprocess.PIPE, + **kwargs + ) + ) + + async def write_stdin(proc, data): + proc.stdin.write(data) + # Only exit the child process once the write buffer is filled + os.write(wfd, b'go') + await proc.stdin.drain() + + coro = write_stdin(proc, large_data) + # drain() must raise BrokenPipeError or ConnectionResetError + with test_utils.disable_logger(): + self.assertRaises((BrokenPipeError, ConnectionResetError), + self.loop.run_until_complete, coro) + self.loop.run_until_complete(proc.wait()) + + def test_communicate_ignore_broken_pipe(self): + # buffer large enough to feed the whole pipe buffer + large_data = b'x' * support.PIPE_MAX_SIZE + + # the program ends before the stdin can be fed + proc = self.loop.run_until_complete( + asyncio.create_subprocess_exec( + sys.executable, '-c', 'pass', + stdin=subprocess.PIPE, + ) + ) + + # communicate() must ignore BrokenPipeError when feeding stdin + self.loop.set_exception_handler(lambda loop, msg: None) + self.loop.run_until_complete(proc.communicate(large_data)) + self.loop.run_until_complete(proc.wait()) + + def test_pause_reading(self): + limit = 10 + size = (limit * 2 + 1) + + async def test_pause_reading(): + code = '\n'.join(( + 'import sys', + 'sys.stdout.write("x" * %s)' % size, + 'sys.stdout.flush()', + )) + + connect_read_pipe = self.loop.connect_read_pipe + + async def connect_read_pipe_mock(*args, **kw): + transport, protocol = await connect_read_pipe(*args, **kw) + transport.pause_reading = mock.Mock() + transport.resume_reading = mock.Mock() + return (transport, protocol) + + self.loop.connect_read_pipe = connect_read_pipe_mock + + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + limit=limit, + ) + stdout_transport = proc._transport.get_pipe_transport(1) + + stdout, stderr = await proc.communicate() + + # The child process produced more than limit bytes of output, + # the stream reader transport should pause the protocol to not + # allocate too much memory. + return (stdout, stdout_transport) + + # Issue #22685: Ensure that the stream reader pauses the protocol + # when the child process produces too much data + stdout, transport = self.loop.run_until_complete(test_pause_reading()) + + self.assertEqual(stdout, b'x' * size) + self.assertTrue(transport.pause_reading.called) + self.assertTrue(transport.resume_reading.called) + + def test_stdin_not_inheritable(self): + # asyncio issue #209: stdin must not be inheritable, otherwise + # the Process.communicate() hangs + async def len_message(message): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(message) + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(len_message(b'abc')) + self.assertEqual(output.rstrip(), b'3') + self.assertEqual(exitcode, 0) + + def test_empty_input(self): + + async def empty_input(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(b'') + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_input()) + self.assertEqual(output.rstrip(), b'0') + self.assertEqual(exitcode, 0) + + def test_devnull_input(self): + + async def empty_input(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate() + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_input()) + self.assertEqual(output.rstrip(), b'0') + self.assertEqual(exitcode, 0) + + def test_devnull_output(self): + + async def empty_output(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(b"abc") + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_output()) + self.assertEqual(output, None) + self.assertEqual(exitcode, 0) + + def test_devnull_error(self): + + async def empty_error(): + code = 'import sys; data = sys.stdin.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + close_fds=False, + ) + stdout, stderr = await proc.communicate(b"abc") + exitcode = await proc.wait() + return (stderr, exitcode) + + output, exitcode = self.loop.run_until_complete(empty_error()) + self.assertEqual(output, None) + self.assertEqual(exitcode, 0) + + @unittest.skipIf(sys.platform not in ('linux', 'android'), + "Don't have /dev/stdin") + def test_devstdin_input(self): + + async def devstdin_input(message): + code = 'file = open("/dev/stdin"); data = file.read(); print(len(data))' + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + close_fds=False, + ) + stdout, stderr = await proc.communicate(message) + exitcode = await proc.wait() + return (stdout, exitcode) + + output, exitcode = self.loop.run_until_complete(devstdin_input(b'abc')) + self.assertEqual(output.rstrip(), b'3') + self.assertEqual(exitcode, 0) + + def test_cancel_process_wait(self): + # Issue #23140: cancel Process.wait() + + async def cancel_wait(): + proc = await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED) + + # Create an internal future waiting on the process exit + task = self.loop.create_task(proc.wait()) + self.loop.call_soon(task.cancel) + try: + await task + except asyncio.CancelledError: + pass + + # Cancel the future + task.cancel() + + # Kill the process and wait until it is done + proc.kill() + await proc.wait() + + self.loop.run_until_complete(cancel_wait()) + + def test_cancel_make_subprocess_transport_exec(self): + + async def cancel_make_transport(): + coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED) + task = self.loop.create_task(coro) + + self.loop.call_soon(task.cancel) + try: + await task + except asyncio.CancelledError: + pass + + # ignore the log: + # "Exception during subprocess creation, kill the subprocess" + with test_utils.disable_logger(): + self.loop.run_until_complete(cancel_make_transport()) + + def test_cancel_post_init(self): + + async def cancel_make_transport(): + coro = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + task = self.loop.create_task(coro) + + self.loop.call_soon(task.cancel) + try: + await task + except asyncio.CancelledError: + pass + + # ignore the log: + # "Exception during subprocess creation, kill the subprocess" + with test_utils.disable_logger(): + self.loop.run_until_complete(cancel_make_transport()) + test_utils.run_briefly(self.loop) + + def test_close_kill_running(self): + + async def kill_running(): + create = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + transport, protocol = await create + + kill_called = False + def kill(): + nonlocal kill_called + kill_called = True + orig_kill() + + proc = transport.get_extra_info('subprocess') + orig_kill = proc.kill + proc.kill = kill + returncode = transport.get_returncode() + transport.close() + await asyncio.wait_for(transport._wait(), 5) + return (returncode, kill_called) + + # Ignore "Close running child process: kill ..." log + with test_utils.disable_logger(): + try: + returncode, killed = self.loop.run_until_complete( + kill_running() + ) + except asyncio.TimeoutError: + self.skipTest( + "Timeout failure on waiting for subprocess stopping" + ) + self.assertIsNone(returncode) + + # transport.close() must kill the process if it is still running + self.assertTrue(killed) + test_utils.run_briefly(self.loop) + + def test_close_dont_kill_finished(self): + + async def kill_running(): + create = self.loop.subprocess_exec(asyncio.SubprocessProtocol, + *PROGRAM_BLOCKED) + transport, protocol = await create + proc = transport.get_extra_info('subprocess') + + # kill the process (but asyncio is not notified immediately) + proc.kill() + proc.wait() + + proc.kill = mock.Mock() + proc_returncode = proc.poll() + transport_returncode = transport.get_returncode() + transport.close() + return (proc_returncode, transport_returncode, proc.kill.called) + + # Ignore "Unknown child process pid ..." log of SafeChildWatcher, + # emitted because the test already consumes the exit status: + # proc.wait() + with test_utils.disable_logger(): + result = self.loop.run_until_complete(kill_running()) + test_utils.run_briefly(self.loop) + + proc_returncode, transport_return_code, killed = result + + self.assertIsNotNone(proc_returncode) + self.assertIsNone(transport_return_code) + + # transport.close() must not kill the process if it finished, even if + # the transport was not notified yet + self.assertFalse(killed) + + async def _test_popen_error(self, stdin): + if sys.platform == 'win32': + target = 'asyncio.windows_utils.Popen' + else: + target = 'subprocess.Popen' + with mock.patch(target) as popen: + exc = ZeroDivisionError + popen.side_effect = exc + + with warnings.catch_warnings(record=True) as warns: + with self.assertRaises(exc): + await asyncio.create_subprocess_exec( + sys.executable, + '-c', + 'pass', + stdin=stdin + ) + self.assertEqual(warns, []) + + def test_popen_error(self): + # Issue #24763: check that the subprocess transport is closed + # when BaseSubprocessTransport fails + self.loop.run_until_complete(self._test_popen_error(stdin=None)) + + def test_popen_error_with_stdin_pipe(self): + # Issue #35721: check that newly created socket pair is closed when + # Popen fails + self.loop.run_until_complete( + self._test_popen_error(stdin=subprocess.PIPE)) + + def test_read_stdout_after_process_exit(self): + + async def execute(): + code = '\n'.join(['import sys', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + + process = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + ) + + while True: + data = await process.stdout.read(65536) + if data: + await asyncio.sleep(0.3) + else: + break + + self.loop.run_until_complete(execute()) + + def test_create_subprocess_exec_text_mode_fails(self): + async def execute(): + with self.assertRaises(ValueError): + await subprocess.create_subprocess_exec(sys.executable, + text=True) + + with self.assertRaises(ValueError): + await subprocess.create_subprocess_exec(sys.executable, + encoding="utf-8") + + with self.assertRaises(ValueError): + await subprocess.create_subprocess_exec(sys.executable, + errors="strict") + + self.loop.run_until_complete(execute()) + + def test_create_subprocess_shell_text_mode_fails(self): + + async def execute(): + with self.assertRaises(ValueError): + await subprocess.create_subprocess_shell(sys.executable, + text=True) + + with self.assertRaises(ValueError): + await subprocess.create_subprocess_shell(sys.executable, + encoding="utf-8") + + with self.assertRaises(ValueError): + await subprocess.create_subprocess_shell(sys.executable, + errors="strict") + + self.loop.run_until_complete(execute()) + + def test_create_subprocess_exec_with_path(self): + async def execute(): + p = await subprocess.create_subprocess_exec( + os_helper.FakePath(sys.executable), '-c', 'pass') + await p.wait() + p = await subprocess.create_subprocess_exec( + sys.executable, '-c', 'pass', os_helper.FakePath('.')) + await p.wait() + + self.assertIsNone(self.loop.run_until_complete(execute())) + + async def check_stdout_output(self, coro, output): + proc = await coro + stdout, _ = await proc.communicate() + self.assertEqual(stdout, output) + self.assertEqual(proc.returncode, 0) + task = asyncio.create_task(proc.wait()) + await asyncio.sleep(0) + self.assertEqual(task.result(), proc.returncode) + + def test_create_subprocess_env_shell(self) -> None: + async def main() -> None: + executable = sys.executable + if sys.platform == "win32": + executable = f'"{executable}"' + cmd = f'''{executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' + env = os.environ.copy() + env["FOO"] = "bar" + proc = await asyncio.create_subprocess_shell( + cmd, env=env, stdout=subprocess.PIPE + ) + return proc + + self.loop.run_until_complete(self.check_stdout_output(main(), b'bar')) + + def test_create_subprocess_env_exec(self) -> None: + async def main() -> None: + cmd = [sys.executable, "-c", + "import os, sys; sys.stdout.write(os.getenv('FOO'))"] + env = os.environ.copy() + env["FOO"] = "baz" + proc = await asyncio.create_subprocess_exec( + *cmd, env=env, stdout=subprocess.PIPE + ) + return proc + + self.loop.run_until_complete(self.check_stdout_output(main(), b'baz')) + + + def test_subprocess_concurrent_wait(self) -> None: + async def main() -> None: + proc = await asyncio.create_subprocess_exec( + *PROGRAM_CAT, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + stdout, _ = await proc.communicate(b'some data') + self.assertEqual(stdout, b"some data") + self.assertEqual(proc.returncode, 0) + self.assertEqual(await asyncio.gather(*[proc.wait() for _ in range(10)]), + [proc.returncode] * 10) + + self.loop.run_until_complete(main()) + + def test_subprocess_protocol_events(self): + # gh-108973: Test that all subprocess protocol methods are called. + # The protocol methods are not called in a deterministic order. + # The order depends on the event loop and the operating system. + events = [] + fds = [1, 2] + expected = [ + ('pipe_data_received', 1, b'stdout'), + ('pipe_data_received', 2, b'stderr'), + ('pipe_connection_lost', 1), + ('pipe_connection_lost', 2), + 'process_exited', + ] + per_fd_expected = [ + 'pipe_data_received', + 'pipe_connection_lost', + ] + + class MyProtocol(asyncio.SubprocessProtocol): + def __init__(self, exit_future: asyncio.Future) -> None: + self.exit_future = exit_future + + def pipe_data_received(self, fd, data) -> None: + events.append(('pipe_data_received', fd, data)) + self.exit_maybe() + + def pipe_connection_lost(self, fd, exc) -> None: + events.append(('pipe_connection_lost', fd)) + self.exit_maybe() + + def process_exited(self) -> None: + events.append('process_exited') + self.exit_maybe() + + def exit_maybe(self): + # Only exit when we got all expected events + if len(events) >= len(expected): + self.exit_future.set_result(True) + + async def main() -> None: + loop = asyncio.get_running_loop() + exit_future = asyncio.Future() + code = 'import sys; sys.stdout.write("stdout"); sys.stderr.write("stderr")' + transport, _ = await loop.subprocess_exec(lambda: MyProtocol(exit_future), + sys.executable, '-c', code, stdin=None) + await exit_future + transport.close() + + return events + + events = self.loop.run_until_complete(main()) + + # First, make sure that we received all events + self.assertSetEqual(set(events), set(expected)) + + # Second, check order of pipe events per file descriptor + per_fd_events = {fd: [] for fd in fds} + for event in events: + if event == 'process_exited': + continue + name, fd = event[:2] + per_fd_events[fd].append(name) + + for fd in fds: + self.assertEqual(per_fd_events[fd], per_fd_expected, (fd, events)) + + def test_subprocess_communicate_stdout(self): + # See https://github.com/python/cpython/issues/100133 + async def get_command_stdout(cmd, *args): + proc = await asyncio.create_subprocess_exec( + cmd, *args, stdout=asyncio.subprocess.PIPE, + ) + stdout, _ = await proc.communicate() + return stdout.decode().strip() + + async def main(): + outputs = [f'foo{i}' for i in range(10)] + res = await asyncio.gather(*[get_command_stdout(sys.executable, '-c', + f'print({out!r})') for out in outputs]) + self.assertEqual(res, outputs) + + self.loop.run_until_complete(main()) + + @unittest.skipIf(sys.platform != 'linux', "Linux only") + def test_subprocess_send_signal_race(self): + # See https://github.com/python/cpython/issues/87744 + async def main(): + for _ in range(10): + proc = await asyncio.create_subprocess_exec('sleep', '0.1') + await asyncio.sleep(0.1) + try: + proc.send_signal(signal.SIGUSR1) + except ProcessLookupError: + pass + self.assertNotEqual(await proc.wait(), 255) + + self.loop.run_until_complete(main()) + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + asyncio.run(main()) + gc_collect() + +if sys.platform != 'win32': + # Unix + class SubprocessWatcherMixin(SubprocessMixin): + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def test_watcher_implementation(self): + loop = self.loop + watcher = loop._watcher + if unix_events.can_use_pidfd(): + self.assertIsInstance(watcher, unix_events._PidfdChildWatcher) + else: + self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher) + + + class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + def setUp(self): + self._original_can_use_pidfd = unix_events.can_use_pidfd + # Force the use of the threaded child watcher + unix_events.can_use_pidfd = mock.Mock(return_value=False) + super().setUp() + + def tearDown(self): + unix_events.can_use_pidfd = self._original_can_use_pidfd + return super().tearDown() + + @unittest.skipUnless( + unix_events.can_use_pidfd(), + "operating system does not support pidfds", + ) + class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, + test_utils.TestCase): + + pass + +else: + # Windows + class SubprocessProactorTests(SubprocessMixin, test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.ProactorEventLoop() + self.set_event_loop(self.loop) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py new file mode 100644 index 00000000000..d4b2554dda9 --- /dev/null +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -0,0 +1,1166 @@ +# Adapted with permission from the EdgeDB project; +# license: PSFL. + +import weakref +import sys +import gc +import asyncio +import contextvars +import contextlib +from asyncio import taskgroups +import unittest +import warnings + +from test.test_asyncio.utils import await_without_task + +# To prevent a warning "test altered the execution environment" +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class MyExc(Exception): + pass + + +class MyBaseExc(BaseException): + pass + + +def get_error_types(eg): + return {type(exc) for exc in eg.exceptions} + + +def no_other_refs(): + # due to gh-124392 coroutines now refer to their locals + coro = asyncio.current_task().get_coro() + frame = sys._getframe(1) + while coro.cr_frame != frame: + coro = coro.cr_await + return [coro] + + +def set_gc_state(enabled): + was_enabled = gc.isenabled() + if enabled: + gc.enable() + else: + gc.disable() + return was_enabled + + +@contextlib.contextmanager +def disable_gc(): + was_enabled = set_gc_state(enabled=False) + try: + yield + finally: + set_gc_state(enabled=was_enabled) + + +class BaseTestTaskGroup: + + async def test_taskgroup_01(self): + + async def foo1(): + await asyncio.sleep(0.1) + return 42 + + async def foo2(): + await asyncio.sleep(0.2) + return 11 + + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + t2 = g.create_task(foo2()) + + self.assertEqual(t1.result(), 42) + self.assertEqual(t2.result(), 11) + + async def test_taskgroup_02(self): + + async def foo1(): + await asyncio.sleep(0.1) + return 42 + + async def foo2(): + await asyncio.sleep(0.2) + return 11 + + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + await asyncio.sleep(0.15) + t2 = g.create_task(foo2()) + + self.assertEqual(t1.result(), 42) + self.assertEqual(t2.result(), 11) + + async def test_taskgroup_03(self): + + async def foo1(): + await asyncio.sleep(1) + return 42 + + async def foo2(): + await asyncio.sleep(0.2) + return 11 + + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + await asyncio.sleep(0.15) + # cancel t1 explicitly, i.e. everything should continue + # working as expected. + t1.cancel() + + t2 = g.create_task(foo2()) + + self.assertTrue(t1.cancelled()) + self.assertEqual(t2.result(), 11) + + async def test_taskgroup_04(self): + + NUM = 0 + t2_cancel = False + t2 = None + + async def foo1(): + await asyncio.sleep(0.1) + 1 / 0 + + async def foo2(): + nonlocal NUM, t2_cancel + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + t2_cancel = True + raise + NUM += 1 + + async def runner(): + nonlocal NUM, t2 + + async with taskgroups.TaskGroup() as g: + g.create_task(foo1()) + t2 = g.create_task(foo2()) + + NUM += 10 + + with self.assertRaises(ExceptionGroup) as cm: + await asyncio.create_task(runner()) + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + self.assertEqual(NUM, 0) + self.assertTrue(t2_cancel) + self.assertTrue(t2.cancelled()) + + async def test_cancel_children_on_child_error(self): + # When a child task raises an error, the rest of the children + # are cancelled and the errors are gathered into an EG. + + NUM = 0 + t2_cancel = False + runner_cancel = False + + async def foo1(): + await asyncio.sleep(0.1) + 1 / 0 + + async def foo2(): + nonlocal NUM, t2_cancel + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + t2_cancel = True + raise + NUM += 1 + + async def runner(): + nonlocal NUM, runner_cancel + + async with taskgroups.TaskGroup() as g: + g.create_task(foo1()) + g.create_task(foo1()) + g.create_task(foo1()) + g.create_task(foo2()) + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + runner_cancel = True + raise + + NUM += 10 + + # The 3 foo1 sub tasks can be racy when the host is busy - if the + # cancellation happens in the middle, we'll see partial sub errors here + with self.assertRaises(ExceptionGroup) as cm: + await asyncio.create_task(runner()) + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + self.assertEqual(NUM, 0) + self.assertTrue(t2_cancel) + self.assertTrue(runner_cancel) + + async def test_cancellation(self): + + NUM = 0 + + async def foo(): + nonlocal NUM + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + NUM += 1 + raise + + async def runner(): + async with taskgroups.TaskGroup() as g: + for _ in range(5): + g.create_task(foo()) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(asyncio.CancelledError) as cm: + await r + + self.assertEqual(NUM, 5) + + async def test_taskgroup_07(self): + + NUM = 0 + + async def foo(): + nonlocal NUM + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + NUM += 1 + raise + + async def runner(): + nonlocal NUM + async with taskgroups.TaskGroup() as g: + for _ in range(5): + g.create_task(foo()) + + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + NUM += 10 + raise + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(asyncio.CancelledError): + await r + + self.assertEqual(NUM, 15) + + async def test_taskgroup_08(self): + + async def foo(): + try: + await asyncio.sleep(10) + finally: + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup() as g: + for _ in range(5): + g.create_task(foo()) + + await asyncio.sleep(10) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + async def test_taskgroup_09(self): + + t1 = t2 = None + + async def foo1(): + await asyncio.sleep(1) + return 42 + + async def foo2(): + await asyncio.sleep(2) + return 11 + + async def runner(): + nonlocal t1, t2 + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + t2 = g.create_task(foo2()) + await asyncio.sleep(0.1) + 1 / 0 + + try: + await runner() + except ExceptionGroup as t: + self.assertEqual(get_error_types(t), {ZeroDivisionError}) + else: + self.fail('ExceptionGroup was not raised') + + self.assertTrue(t1.cancelled()) + self.assertTrue(t2.cancelled()) + + async def test_taskgroup_10(self): + + t1 = t2 = None + + async def foo1(): + await asyncio.sleep(1) + return 42 + + async def foo2(): + await asyncio.sleep(2) + return 11 + + async def runner(): + nonlocal t1, t2 + async with taskgroups.TaskGroup() as g: + t1 = g.create_task(foo1()) + t2 = g.create_task(foo2()) + 1 / 0 + + try: + await runner() + except ExceptionGroup as t: + self.assertEqual(get_error_types(t), {ZeroDivisionError}) + else: + self.fail('ExceptionGroup was not raised') + + self.assertTrue(t1.cancelled()) + self.assertTrue(t2.cancelled()) + + async def test_taskgroup_11(self): + + async def foo(): + try: + await asyncio.sleep(10) + finally: + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup(): + async with taskgroups.TaskGroup() as g2: + for _ in range(5): + g2.create_task(foo()) + + await asyncio.sleep(10) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + + self.assertEqual(get_error_types(cm.exception), {ExceptionGroup}) + self.assertEqual(get_error_types(cm.exception.exceptions[0]), {ZeroDivisionError}) + + async def test_taskgroup_12(self): + + async def foo(): + try: + await asyncio.sleep(10) + finally: + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(asyncio.sleep(10)) + + async with taskgroups.TaskGroup() as g2: + for _ in range(5): + g2.create_task(foo()) + + await asyncio.sleep(10) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + + self.assertEqual(get_error_types(cm.exception), {ExceptionGroup}) + self.assertEqual(get_error_types(cm.exception.exceptions[0]), {ZeroDivisionError}) + + async def test_taskgroup_13(self): + + async def crash_after(t): + await asyncio.sleep(t) + raise ValueError(t) + + async def runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(crash_after(0.1)) + + async with taskgroups.TaskGroup() as g2: + g2.create_task(crash_after(10)) + + r = asyncio.create_task(runner()) + with self.assertRaises(ExceptionGroup) as cm: + await r + + self.assertEqual(get_error_types(cm.exception), {ValueError}) + + async def test_taskgroup_14(self): + + async def crash_after(t): + await asyncio.sleep(t) + raise ValueError(t) + + async def runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(crash_after(10)) + + async with taskgroups.TaskGroup() as g2: + g2.create_task(crash_after(0.1)) + + r = asyncio.create_task(runner()) + with self.assertRaises(ExceptionGroup) as cm: + await r + + self.assertEqual(get_error_types(cm.exception), {ExceptionGroup}) + self.assertEqual(get_error_types(cm.exception.exceptions[0]), {ValueError}) + + async def test_taskgroup_15(self): + + async def crash_soon(): + await asyncio.sleep(0.3) + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(crash_soon()) + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + await asyncio.sleep(0.5) + raise + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + async def test_taskgroup_16(self): + + async def crash_soon(): + await asyncio.sleep(0.3) + 1 / 0 + + async def nested_runner(): + async with taskgroups.TaskGroup() as g1: + g1.create_task(crash_soon()) + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + await asyncio.sleep(0.5) + raise + + async def runner(): + t = asyncio.create_task(nested_runner()) + await t + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(ExceptionGroup) as cm: + await r + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + async def test_taskgroup_17(self): + NUM = 0 + + async def runner(): + nonlocal NUM + async with taskgroups.TaskGroup(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + NUM += 10 + raise + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + with self.assertRaises(asyncio.CancelledError): + await r + + self.assertEqual(NUM, 10) + + async def test_taskgroup_18(self): + NUM = 0 + + async def runner(): + nonlocal NUM + async with taskgroups.TaskGroup(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + NUM += 10 + # This isn't a good idea, but we have to support + # this weird case. + raise MyExc + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.1) + + self.assertFalse(r.done()) + r.cancel() + + try: + await r + except ExceptionGroup as t: + self.assertEqual(get_error_types(t),{MyExc}) + else: + self.fail('ExceptionGroup was not raised') + + self.assertEqual(NUM, 10) + + async def test_taskgroup_19(self): + async def crash_soon(): + await asyncio.sleep(0.1) + 1 / 0 + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise MyExc + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + r = asyncio.create_task(runner()) + try: + await r + except ExceptionGroup as t: + self.assertEqual(get_error_types(t), {MyExc, ZeroDivisionError}) + else: + self.fail('TasgGroupError was not raised') + + async def test_taskgroup_20(self): + async def crash_soon(): + await asyncio.sleep(0.1) + 1 / 0 + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise KeyboardInterrupt + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + with self.assertRaises(KeyboardInterrupt): + await runner() + + async def test_taskgroup_20a(self): + async def crash_soon(): + await asyncio.sleep(0.1) + 1 / 0 + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise MyBaseExc + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + with self.assertRaises(BaseExceptionGroup) as cm: + await runner() + + self.assertEqual( + get_error_types(cm.exception), {MyBaseExc, ZeroDivisionError} + ) + + async def _test_taskgroup_21(self): + # This test doesn't work as asyncio, currently, doesn't + # correctly propagate KeyboardInterrupt (or SystemExit) -- + # those cause the event loop itself to crash. + # (Compare to the previous (passing) test -- that one raises + # a plain exception but raises KeyboardInterrupt in nested(); + # this test does it the other way around.) + + async def crash_soon(): + await asyncio.sleep(0.1) + raise KeyboardInterrupt + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise TypeError + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + with self.assertRaises(KeyboardInterrupt): + await runner() + + async def test_taskgroup_21a(self): + + async def crash_soon(): + await asyncio.sleep(0.1) + raise MyBaseExc + + async def nested(): + try: + await asyncio.sleep(10) + finally: + raise TypeError + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(crash_soon()) + await nested() + + with self.assertRaises(BaseExceptionGroup) as cm: + await runner() + + self.assertEqual(get_error_types(cm.exception), {MyBaseExc, TypeError}) + + async def test_taskgroup_22(self): + + async def foo1(): + await asyncio.sleep(1) + return 42 + + async def foo2(): + await asyncio.sleep(2) + return 11 + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(foo1()) + g.create_task(foo2()) + + r = asyncio.create_task(runner()) + await asyncio.sleep(0.05) + r.cancel() + + with self.assertRaises(asyncio.CancelledError): + await r + + async def test_taskgroup_23(self): + + async def do_job(delay): + await asyncio.sleep(delay) + + async with taskgroups.TaskGroup() as g: + for count in range(10): + await asyncio.sleep(0.1) + g.create_task(do_job(0.3)) + if count == 5: + self.assertLess(len(g._tasks), 5) + await asyncio.sleep(1.35) + self.assertEqual(len(g._tasks), 0) + + async def test_taskgroup_24(self): + + async def root(g): + await asyncio.sleep(0.1) + g.create_task(coro1(0.1)) + g.create_task(coro1(0.2)) + + async def coro1(delay): + await asyncio.sleep(delay) + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(root(g)) + + await runner() + + async def test_taskgroup_25(self): + nhydras = 0 + + async def hydra(g): + nonlocal nhydras + nhydras += 1 + await asyncio.sleep(0.01) + g.create_task(hydra(g)) + g.create_task(hydra(g)) + + async def hercules(): + while nhydras < 10: + await asyncio.sleep(0.015) + 1 / 0 + + async def runner(): + async with taskgroups.TaskGroup() as g: + g.create_task(hydra(g)) + g.create_task(hercules()) + + with self.assertRaises(ExceptionGroup) as cm: + await runner() + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + self.assertGreaterEqual(nhydras, 10) + + async def test_taskgroup_task_name(self): + async def coro(): + await asyncio.sleep(0) + async with taskgroups.TaskGroup() as g: + t = g.create_task(coro(), name="yolo") + self.assertEqual(t.get_name(), "yolo") + + async def test_taskgroup_task_context(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async with taskgroups.TaskGroup() as g: + ctx = contextvars.copy_context() + self.assertIsNone(ctx.get(cvar)) + t1 = g.create_task(coro(1), context=ctx) + await t1 + self.assertEqual(1, ctx.get(cvar)) + t2 = g.create_task(coro(2), context=ctx) + await t2 + self.assertEqual(2, ctx.get(cvar)) + + async def test_taskgroup_no_create_task_after_failure(self): + async def coro1(): + await asyncio.sleep(0.001) + 1 / 0 + async def coro2(g): + try: + await asyncio.sleep(1) + except asyncio.CancelledError: + with self.assertRaises(RuntimeError): + g.create_task(coro1()) + + with self.assertRaises(ExceptionGroup) as cm: + async with taskgroups.TaskGroup() as g: + g.create_task(coro1()) + g.create_task(coro2(g)) + + self.assertEqual(get_error_types(cm.exception), {ZeroDivisionError}) + + async def test_taskgroup_context_manager_exit_raises(self): + # See https://github.com/python/cpython/issues/95289 + class CustomException(Exception): + pass + + async def raise_exc(): + raise CustomException + + @contextlib.asynccontextmanager + async def database(): + try: + yield + finally: + raise CustomException + + async def main(): + task = asyncio.current_task() + try: + async with taskgroups.TaskGroup() as tg: + async with database(): + tg.create_task(raise_exc()) + await asyncio.sleep(1) + except* CustomException as err: + self.assertEqual(task.cancelling(), 0) + self.assertEqual(len(err.exceptions), 2) + + else: + self.fail('CustomException not raised') + + await asyncio.create_task(main()) + + async def test_taskgroup_already_entered(self): + tg = taskgroups.TaskGroup() + async with tg: + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with tg: + pass + + async def test_taskgroup_double_enter(self): + tg = taskgroups.TaskGroup() + async with tg: + pass + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with tg: + pass + + async def test_taskgroup_finished(self): + async def create_task_after_tg_finish(): + tg = taskgroups.TaskGroup() + async with tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + + # Make sure the coroutine was closed when submitted to the inactive tg + # (if not closed, a RuntimeWarning should have been raised) + with warnings.catch_warnings(record=True) as w: + await create_task_after_tg_finish() + self.assertEqual(len(w), 0) + + async def test_taskgroup_not_entered(self): + tg = taskgroups.TaskGroup() + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + tg.create_task(coro) + + async def test_taskgroup_without_parent_task(self): + tg = taskgroups.TaskGroup() + with self.assertRaisesRegex(RuntimeError, "parent task"): + await await_without_task(tg.__aenter__()) + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + tg.create_task(coro) + + async def test_coro_closed_when_tg_closed(self): + async def run_coro_after_tg_closes(): + async with taskgroups.TaskGroup() as tg: + pass + coro = asyncio.sleep(0) + with self.assertRaisesRegex(RuntimeError, "is finished"): + tg.create_task(coro) + + await run_coro_after_tg_closes() + + async def test_cancelling_level_preserved(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(raise_after(0.0, RuntimeError)) + except* RuntimeError: + pass + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_nested_groups_both_cancelled(self): + async def raise_after(t, e): + await asyncio.sleep(t) + raise e() + + try: + async with asyncio.TaskGroup() as outer_tg: + try: + async with asyncio.TaskGroup() as inner_tg: + inner_tg.create_task(raise_after(0, RuntimeError)) + outer_tg.create_task(raise_after(0, ValueError)) + except* RuntimeError: + pass + else: + self.fail("RuntimeError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 1) + except* ValueError: + pass + else: + self.fail("ValueError not raised") + self.assertEqual(asyncio.current_task().cancelling(), 0) + + async def test_error_and_cancel(self): + event = asyncio.Event() + + async def raise_error(): + event.set() + await asyncio.sleep(0) + raise RuntimeError() + + async def inner(): + try: + async with taskgroups.TaskGroup() as tg: + tg.create_task(raise_error()) + await asyncio.sleep(1) + self.fail("Sleep in group should have been cancelled") + except* RuntimeError: + self.assertEqual(asyncio.current_task().cancelling(), 1) + self.assertEqual(asyncio.current_task().cancelling(), 1) + await asyncio.sleep(1) + self.fail("Sleep after group should have been cancelled") + + async def outer(): + t = asyncio.create_task(inner()) + await event.wait() + self.assertEqual(t.cancelling(), 0) + t.cancel() + self.assertEqual(t.cancelling(), 1) + with self.assertRaises(asyncio.CancelledError): + await t + self.assertTrue(t.cancelled()) + + await outer() + + async def test_exception_refcycles_direct(self): + """Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except ExceptionGroup as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_errors(self): + """Test that TaskGroup deletes self._errors, and __aexit__ args""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + try: + async with tg: + raise _Done + except* _Done as excs: + exc = excs.exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_parent_task(self): + """Test that TaskGroup deletes self._parent_task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + try: + async with asyncio.TaskGroup() as tg2: + tg2.create_task(coro_fn()) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + + async def test_exception_refcycles_parent_task_wr(self): + """Test that TaskGroup deletes self._parent_task and create_task() deletes task""" + tg = asyncio.TaskGroup() + exc = None + + class _Done(Exception): + pass + + async def coro_fn(): + async with tg: + raise _Done + + with disable_gc(): + try: + async with asyncio.TaskGroup() as tg2: + task_wr = weakref.ref(tg2.create_task(coro_fn())) + except* _Done as excs: + exc = excs.exceptions[0].exceptions[0] + + self.assertIsNone(task_wr()) + self.assertIsInstance(exc, _Done) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_propagate_cancellation_error(self): + """Test that TaskGroup deletes propagate_cancellation_error""" + tg = asyncio.TaskGroup() + exc = None + + try: + async with asyncio.timeout(-1): + async with tg: + await asyncio.sleep(0) + except TimeoutError as e: + exc = e.__cause__ + + self.assertIsInstance(exc, asyncio.CancelledError) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_exception_refcycles_base_error(self): + """Test that TaskGroup deletes self._base_error""" + class MyKeyboardInterrupt(KeyboardInterrupt): + pass + + tg = asyncio.TaskGroup() + exc = None + + try: + async with tg: + raise MyKeyboardInterrupt + except MyKeyboardInterrupt as e: + exc = e + + self.assertIsNotNone(exc) + self.assertListEqual(gc.get_referrers(exc), no_other_refs()) + + async def test_name(self): + name = None + + async def asyncfn(): + nonlocal name + name = asyncio.current_task().get_name() + + async with asyncio.TaskGroup() as tg: + tg.create_task(asyncfn(), name="example name") + + self.assertEqual(name, "example name") + + + async def test_cancels_task_if_created_during_creation(self): + # regression test for gh-128550 + ran = False + class MyError(Exception): + pass + + exc = None + try: + async with asyncio.TaskGroup() as tg: + async def third_task(): + raise MyError("third task failed") + + async def second_task(): + nonlocal ran + tg.create_task(third_task()) + with self.assertRaises(asyncio.CancelledError): + await asyncio.sleep(0) # eager tasks cancel here + await asyncio.sleep(0) # lazy tasks cancel here + ran = True + + tg.create_task(second_task()) + except* MyError as excs: + exc = excs.exceptions[0] + + self.assertTrue(ran) + self.assertIsInstance(exc, MyError) + + + async def test_cancellation_does_not_leak_out_of_tg(self): + class MyError(Exception): + pass + + async def throw_error(): + raise MyError + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(throw_error()) + except* MyError: + pass + else: + self.fail("should have raised one MyError in group") + + # if this test fails this current task will be cancelled + # outside the task group and inside unittest internals + # we yield to the event loop with sleep(0) so that + # cancellation happens here and error is more understandable + await asyncio.sleep(0) + + +class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes propagate_cancellation_error + async def test_exception_refcycles_propagate_cancellation_error(self): + return await super().test_exception_refcycles_propagate_cancellation_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._base_error + async def test_exception_refcycles_base_error(self): + return await super().test_exception_refcycles_base_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._errors, and __aexit__ args + async def test_exception_refcycles_errors(self): + return await super().test_exception_refcycles_errors() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task + async def test_exception_refcycles_parent_task(self): + return await super().test_exception_refcycles_parent_task() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task and create_task() deletes task + async def test_exception_refcycles_parent_task_wr(self): + return await super().test_exception_refcycles_parent_task_wr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup + async def test_exception_refcycles_direct(self): + return await super().test_exception_refcycles_direct() + +class TestEagerTaskTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase): + @staticmethod + def loop_factory(): + loop = asyncio.EventLoop() + loop.set_task_factory(asyncio.eager_task_factory) + return loop + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes propagate_cancellation_error + async def test_exception_refcycles_propagate_cancellation_error(self): + return await super().test_exception_refcycles_propagate_cancellation_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._base_error + async def test_exception_refcycles_base_error(self): + return await super().test_exception_refcycles_base_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._errors, and __aexit__ args + async def test_exception_refcycles_errors(self): + return await super().test_exception_refcycles_errors() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task + async def test_exception_refcycles_parent_task(self): + return await super().test_exception_refcycles_parent_task() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup deletes self._parent_task and create_task() deletes task + async def test_exception_refcycles_parent_task_wr(self): + return await super().test_exception_refcycles_parent_task_wr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Test that TaskGroup doesn't keep a reference to the raised ExceptionGroup + async def test_exception_refcycles_direct(self): + return await super().test_exception_refcycles_direct() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py new file mode 100644 index 00000000000..8a291f1cb7e --- /dev/null +++ b/Lib/test/test_asyncio/test_tasks.py @@ -0,0 +1,3763 @@ +"""Tests for tasks.py.""" + +import collections +import contextlib +import contextvars +import gc +import io +import random +import re +import sys +import traceback +import types +import unittest +from unittest import mock +from types import GenericAlias + +import asyncio +from asyncio import futures +from asyncio import tasks +from test.test_asyncio import utils as test_utils +from test import support +from test.support.script_helper import assert_python_ok +from test.support.warnings_helper import ignore_warnings + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +async def coroutine_function(): + pass + + +def format_coroutine(qualname, state, src, source_traceback, generator=False): + if generator: + state = '%s' % state + else: + state = '%s, defined' % state + if source_traceback is not None: + frame = source_traceback[-1] + return ('coro=<%s() %s at %s> created at %s:%s' + % (qualname, state, src, frame[0], frame[1])) + else: + return 'coro=<%s() %s at %s>' % (qualname, state, src) + + +def get_innermost_context(exc): + """ + Return information about the innermost exception context in the chain. + """ + depth = 0 + while True: + context = exc.__context__ + if context is None: + break + + exc = context + depth += 1 + + return (type(exc), exc.args, depth) + + +class Dummy: + + def __repr__(self): + return '' + + def __call__(self, *args): + pass + + +class CoroLikeObject: + def send(self, v): + raise StopIteration(42) + + def throw(self, *exc): + pass + + def close(self): + pass + + def __await__(self): + return self + + +class BaseTaskTests: + + Task = None + Future = None + all_tasks = None + + def new_task(self, loop, coro, name='TestTask', context=None, eager_start=None): + return self.__class__.Task(coro, loop=loop, name=name, context=context, eager_start=eager_start) + + def new_future(self, loop): + return self.__class__.Future(loop=loop) + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.loop.set_task_factory(self.new_task) + self.loop.create_future = lambda: self.new_future(self.loop) + + def test_generic_alias(self): + task = self.__class__.Task[str] + self.assertEqual(task.__args__, (str,)) + self.assertIsInstance(task, GenericAlias) + + def test_task_cancel_message_getter(self): + async def coro(): + pass + t = self.new_task(self.loop, coro()) + self.assertHasAttr(t, '_cancel_message') + self.assertEqual(t._cancel_message, None) + + t.cancel('my message') + self.assertEqual(t._cancel_message, 'my message') + + with self.assertRaises(asyncio.CancelledError) as cm: + self.loop.run_until_complete(t) + + self.assertEqual('my message', cm.exception.args[0]) + + def test_task_cancel_message_setter(self): + async def coro(): + pass + t = self.new_task(self.loop, coro()) + t.cancel('my message') + t._cancel_message = 'my new message' + self.assertEqual(t._cancel_message, 'my new message') + + with self.assertRaises(asyncio.CancelledError) as cm: + self.loop.run_until_complete(t) + + self.assertEqual('my new message', cm.exception.args[0]) + + def test_task_del_collect(self): + class Evil: + def __del__(self): + gc.collect() + + async def run(): + return Evil() + + self.loop.run_until_complete( + asyncio.gather(*[ + self.new_task(self.loop, run()) for _ in range(100) + ])) + + def test_other_loop_future(self): + other_loop = asyncio.new_event_loop() + fut = self.new_future(other_loop) + + async def run(fut): + await fut + + try: + with self.assertRaisesRegex(RuntimeError, + r'Task .* got Future .* attached'): + self.loop.run_until_complete(run(fut)) + finally: + other_loop.close() + + def test_task_awaits_on_itself(self): + + async def test(): + await task + + task = asyncio.ensure_future(test(), loop=self.loop) + + with self.assertRaisesRegex(RuntimeError, + 'Task cannot await on itself'): + self.loop.run_until_complete(task) + + def test_task_class(self): + async def notmuch(): + return 'ok' + t = self.new_task(self.loop, notmuch()) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + self.assertIs(t._loop, self.loop) + self.assertIs(t.get_loop(), self.loop) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + t = self.new_task(loop, notmuch()) + self.assertIs(t._loop, loop) + loop.run_until_complete(t) + loop.close() + + def test_ensure_future_coroutine(self): + async def notmuch(): + return 'ok' + t = asyncio.ensure_future(notmuch(), loop=self.loop) + self.assertIs(t._loop, self.loop) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + + a = notmuch() + self.addCleanup(a.close) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.ensure_future(a) + + async def test(): + return asyncio.ensure_future(notmuch()) + t = self.loop.run_until_complete(test()) + self.assertIs(t._loop, self.loop) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + + # Deprecated in 3.10, undeprecated in 3.12 + asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio.set_event_loop, None) + t = asyncio.ensure_future(notmuch()) + self.assertIs(t._loop, self.loop) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + + def test_ensure_future_future(self): + f_orig = self.new_future(self.loop) + f_orig.set_result('ko') + + f = asyncio.ensure_future(f_orig) + self.loop.run_until_complete(f) + self.assertTrue(f.done()) + self.assertEqual(f.result(), 'ko') + self.assertIs(f, f_orig) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + with self.assertRaises(ValueError): + f = asyncio.ensure_future(f_orig, loop=loop) + + loop.close() + + f = asyncio.ensure_future(f_orig, loop=self.loop) + self.assertIs(f, f_orig) + + def test_ensure_future_task(self): + async def notmuch(): + return 'ok' + t_orig = self.new_task(self.loop, notmuch()) + t = asyncio.ensure_future(t_orig) + self.loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'ok') + self.assertIs(t, t_orig) + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + with self.assertRaises(ValueError): + t = asyncio.ensure_future(t_orig, loop=loop) + + loop.close() + + t = asyncio.ensure_future(t_orig, loop=self.loop) + self.assertIs(t, t_orig) + + def test_ensure_future_awaitable(self): + class Aw: + def __init__(self, coro): + self.coro = coro + def __await__(self): + return self.coro.__await__() + + async def coro(): + return 'ok' + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + fut = asyncio.ensure_future(Aw(coro()), loop=loop) + loop.run_until_complete(fut) + self.assertEqual(fut.result(), 'ok') + + def test_ensure_future_task_awaitable(self): + class Aw: + def __await__(self): + return asyncio.sleep(0, result='ok').__await__() + + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + task = asyncio.ensure_future(Aw(), loop=loop) + loop.run_until_complete(task) + self.assertTrue(task.done()) + self.assertEqual(task.result(), 'ok') + self.assertIsInstance(task.get_coro(), types.CoroutineType) + loop.close() + + def test_ensure_future_neither(self): + with self.assertRaises(TypeError): + asyncio.ensure_future('ok') + + def test_ensure_future_error_msg(self): + loop = asyncio.new_event_loop() + f = self.new_future(self.loop) + with self.assertRaisesRegex(ValueError, 'The future belongs to a ' + 'different loop than the one specified as ' + 'the loop argument'): + asyncio.ensure_future(f, loop=loop) + loop.close() + + def test_get_stack(self): + T = None + + async def foo(): + await bar() + + async def bar(): + # test get_stack() + f = T.get_stack(limit=1) + try: + self.assertEqual(f[0].f_code.co_name, 'foo') + finally: + f = None + + # test print_stack() + file = io.StringIO() + T.print_stack(limit=1, file=file) + file.seek(0) + tb = file.read() + self.assertRegex(tb, r'foo\(\) running') + + async def runner(): + nonlocal T + T = asyncio.ensure_future(foo(), loop=self.loop) + await T + + self.loop.run_until_complete(runner()) + + def test_task_repr(self): + self.loop.set_debug(False) + + async def notmuch(): + return 'abc' + + # test coroutine function + self.assertEqual(notmuch.__name__, 'notmuch') + self.assertRegex(notmuch.__qualname__, + r'\w+.test_task_repr..notmuch') + self.assertEqual(notmuch.__module__, __name__) + + filename, lineno = test_utils.get_function_source(notmuch) + src = "%s:%s" % (filename, lineno) + + # test coroutine object + gen = notmuch() + coro_qualname = 'BaseTaskTests.test_task_repr..notmuch' + self.assertEqual(gen.__name__, 'notmuch') + self.assertEqual(gen.__qualname__, coro_qualname) + + # test pending Task + t = self.new_task(self.loop, gen) + t.add_done_callback(Dummy()) + + coro = format_coroutine(coro_qualname, 'running', src, + t._source_traceback, generator=True) + self.assertEqual(repr(t), + "()]>" % coro) + + # test cancelling Task + t.cancel() # Does not take immediate effect! + self.assertEqual(repr(t), + "()]>" % coro) + + # test cancelled Task + self.assertRaises(asyncio.CancelledError, + self.loop.run_until_complete, t) + coro = format_coroutine(coro_qualname, 'done', src, + t._source_traceback) + self.assertEqual(repr(t), + "" % coro) + + # test finished Task + t = self.new_task(self.loop, notmuch()) + self.loop.run_until_complete(t) + coro = format_coroutine(coro_qualname, 'done', src, + t._source_traceback) + self.assertEqual(repr(t), + "" % coro) + + def test_task_repr_autogenerated(self): + async def notmuch(): + return 123 + + t1 = self.new_task(self.loop, notmuch(), None) + t2 = self.new_task(self.loop, notmuch(), None) + self.assertNotEqual(repr(t1), repr(t2)) + + match1 = re.match(r"^' % re.escape(repr(fut))) + + fut.set_result(None) + self.loop.run_until_complete(task) + + def test_task_basics(self): + + async def outer(): + a = await inner1() + b = await inner2() + return a+b + + async def inner1(): + return 42 + + async def inner2(): + return 1000 + + t = outer() + self.assertEqual(self.loop.run_until_complete(t), 1042) + + def test_exception_chaining_after_await(self): + # Test that when awaiting on a task when an exception is already + # active, if the task raises an exception it will be chained + # with the original. + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def raise_error(): + raise ValueError + + async def run(): + try: + raise KeyError(3) + except Exception as exc: + task = self.new_task(loop, raise_error()) + try: + await task + except Exception as exc: + self.assertEqual(type(exc), ValueError) + chained = exc.__context__ + self.assertEqual((type(chained), chained.args), + (KeyError, (3,))) + + try: + task = self.new_task(loop, run()) + loop.run_until_complete(task) + finally: + loop.close() + + def test_exception_chaining_after_await_with_context_cycle(self): + # Check trying to create an exception context cycle: + # https://bugs.python.org/issue40696 + has_cycle = None + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def process_exc(exc): + raise exc + + async def run(): + nonlocal has_cycle + try: + raise KeyError('a') + except Exception as exc: + task = self.new_task(loop, process_exc(exc)) + try: + await task + except BaseException as exc: + has_cycle = (exc is exc.__context__) + # Prevent a hang if has_cycle is True. + exc.__context__ = None + + try: + task = self.new_task(loop, run()) + loop.run_until_complete(task) + finally: + loop.close() + # This also distinguishes from the initial has_cycle=None. + self.assertEqual(has_cycle, False) + + + def test_cancelling(self): + loop = asyncio.new_event_loop() + + async def task(): + await asyncio.sleep(10) + + try: + t = self.new_task(loop, task()) + self.assertFalse(t.cancelling()) + self.assertNotIn(" cancelling ", repr(t)) + self.assertTrue(t.cancel()) + self.assertTrue(t.cancelling()) + self.assertIn(" cancelling ", repr(t)) + + # Since we commented out two lines from Task.cancel(), + # this t.cancel() call now returns True. + # self.assertFalse(t.cancel()) + self.assertTrue(t.cancel()) + + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(t) + finally: + loop.close() + + def test_uncancel_basic(self): + loop = asyncio.new_event_loop() + + async def task(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + self.current_task().uncancel() + await asyncio.sleep(10) + + try: + t = self.new_task(loop, task()) + loop.run_until_complete(asyncio.sleep(0.01)) + + # Cancel first sleep + self.assertTrue(t.cancel()) + self.assertIn(" cancelling ", repr(t)) + self.assertEqual(t.cancelling(), 1) + self.assertFalse(t.cancelled()) # Task is still not complete + loop.run_until_complete(asyncio.sleep(0.01)) + + # after .uncancel() + self.assertNotIn(" cancelling ", repr(t)) + self.assertEqual(t.cancelling(), 0) + self.assertFalse(t.cancelled()) # Task is still not complete + + # Cancel second sleep + self.assertTrue(t.cancel()) + self.assertEqual(t.cancelling(), 1) + self.assertFalse(t.cancelled()) # Task is still not complete + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(t) + self.assertTrue(t.cancelled()) # Finally, task complete + self.assertTrue(t.done()) + + # uncancel is no longer effective after the task is complete + t.uncancel() + self.assertTrue(t.cancelled()) + self.assertTrue(t.done()) + finally: + loop.close() + + def test_uncancel_structured_blocks(self): + # This test recreates the following high-level structure using uncancel():: + # + # async def make_request_with_timeout(): + # try: + # async with asyncio.timeout(1): + # # Structured block affected by the timeout: + # await make_request() + # await make_another_request() + # except TimeoutError: + # pass # There was a timeout + # # Outer code not affected by the timeout: + # await unrelated_code() + + loop = asyncio.new_event_loop() + + async def make_request_with_timeout(*, sleep: float, timeout: float): + task = self.current_task() + loop = task.get_loop() + + timed_out = False + structured_block_finished = False + outer_code_reached = False + + def on_timeout(): + nonlocal timed_out + timed_out = True + task.cancel() + + timeout_handle = loop.call_later(timeout, on_timeout) + try: + try: + # Structured block affected by the timeout + await asyncio.sleep(sleep) + structured_block_finished = True + finally: + timeout_handle.cancel() + if ( + timed_out + and task.uncancel() == 0 + and type(sys.exception()) is asyncio.CancelledError + ): + # Note the five rules that are needed here to satisfy proper + # uncancellation: + # + # 1. handle uncancellation in a `finally:` block to allow for + # plain returns; + # 2. our `timed_out` flag is set, meaning that it was our event + # that triggered the need to uncancel the task, regardless of + # what exception is raised; + # 3. we can call `uncancel()` because *we* called `cancel()` + # before; + # 4. we call `uncancel()` but we only continue converting the + # CancelledError to TimeoutError if `uncancel()` caused the + # cancellation request count go down to 0. We need to look + # at the counter vs having a simple boolean flag because our + # code might have been nested (think multiple timeouts). See + # commit 7fce1063b6e5a366f8504e039a8ccdd6944625cd for + # details. + # 5. we only convert CancelledError to TimeoutError; for other + # exceptions raised due to the cancellation (like + # a ConnectionLostError from a database client), simply + # propagate them. + # + # Those checks need to take place in this exact order to make + # sure the `cancelling()` counter always stays in sync. + # + # Additionally, the original stimulus to `cancel()` the task + # needs to be unscheduled to avoid re-cancelling the task later. + # Here we do it by cancelling `timeout_handle` in the `finally:` + # block. + raise TimeoutError + except TimeoutError: + self.assertTrue(timed_out) + + # Outer code not affected by the timeout: + outer_code_reached = True + await asyncio.sleep(0) + return timed_out, structured_block_finished, outer_code_reached + + try: + # Test which timed out. + t1 = self.new_task(loop, make_request_with_timeout(sleep=10.0, timeout=0.1)) + timed_out, structured_block_finished, outer_code_reached = ( + loop.run_until_complete(t1) + ) + self.assertTrue(timed_out) + self.assertFalse(structured_block_finished) # it was cancelled + self.assertTrue(outer_code_reached) # task got uncancelled after leaving + # the structured block and continued until + # completion + self.assertEqual(t1.cancelling(), 0) # no pending cancellation of the outer task + + # Test which did not time out. + t2 = self.new_task(loop, make_request_with_timeout(sleep=0, timeout=10.0)) + timed_out, structured_block_finished, outer_code_reached = ( + loop.run_until_complete(t2) + ) + self.assertFalse(timed_out) + self.assertTrue(structured_block_finished) + self.assertTrue(outer_code_reached) + self.assertEqual(t2.cancelling(), 0) + finally: + loop.close() + + def test_uncancel_resets_must_cancel(self): + + async def coro(): + await fut + return 42 + + loop = asyncio.new_event_loop() + fut = asyncio.Future(loop=loop) + task = self.new_task(loop, coro()) + loop.run_until_complete(asyncio.sleep(0)) # Get task waiting for fut + fut.set_result(None) # Make task runnable + try: + task.cancel() # Enter cancelled state + self.assertEqual(task.cancelling(), 1) + self.assertTrue(task._must_cancel) + + task.uncancel() # Undo cancellation + self.assertEqual(task.cancelling(), 0) + self.assertFalse(task._must_cancel) + finally: + res = loop.run_until_complete(task) + self.assertEqual(res, 42) + loop.close() + + def test_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + async def task(): + await asyncio.sleep(10.0) + return 12 + + t = self.new_task(loop, task()) + loop.call_soon(t.cancel) + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t.cancel()) + + def test_cancel_with_message_then_future_result(self): + # Test Future.result() after calling cancel() with a message. + cases = [ + ((), ()), + ((None,), ()), + (('my message',), ('my message',)), + # Non-string values should roundtrip. + ((5,), (5,)), + ] + for cancel_args, expected_args in cases: + with self.subTest(cancel_args=cancel_args): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def sleep(): + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, sleep()) + await asyncio.sleep(0) + task.cancel(*cancel_args) + done, pending = await asyncio.wait([task]) + task.result() + + task = self.new_task(loop, coro()) + with self.assertRaises(asyncio.CancelledError) as cm: + loop.run_until_complete(task) + exc = cm.exception + self.assertEqual(exc.args, expected_args) + + actual = get_innermost_context(exc) + self.assertEqual(actual, + (asyncio.CancelledError, expected_args, 0)) + + def test_cancel_with_message_then_future_exception(self): + # Test Future.exception() after calling cancel() with a message. + cases = [ + ((), ()), + ((None,), ()), + (('my message',), ('my message',)), + # Non-string values should roundtrip. + ((5,), (5,)), + ] + for cancel_args, expected_args in cases: + with self.subTest(cancel_args=cancel_args): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def sleep(): + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, sleep()) + await asyncio.sleep(0) + task.cancel(*cancel_args) + done, pending = await asyncio.wait([task]) + task.exception() + + task = self.new_task(loop, coro()) + with self.assertRaises(asyncio.CancelledError) as cm: + loop.run_until_complete(task) + exc = cm.exception + self.assertEqual(exc.args, expected_args) + + actual = get_innermost_context(exc) + self.assertEqual(actual, + (asyncio.CancelledError, expected_args, 0)) + + def test_cancellation_exception_context(self): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + fut = loop.create_future() + + async def sleep(): + fut.set_result(None) + await asyncio.sleep(10) + + async def coro(): + inner_task = self.new_task(loop, sleep()) + await fut + loop.call_soon(inner_task.cancel, 'msg') + try: + await inner_task + except asyncio.CancelledError as ex: + raise ValueError("cancelled") from ex + + task = self.new_task(loop, coro()) + with self.assertRaises(ValueError) as cm: + loop.run_until_complete(task) + exc = cm.exception + self.assertEqual(exc.args, ('cancelled',)) + + actual = get_innermost_context(exc) + self.assertEqual(actual, + (asyncio.CancelledError, ('msg',), 1)) + + def test_cancel_with_message_before_starting_task(self): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def sleep(): + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, sleep()) + # We deliberately leave out the sleep here. + task.cancel('my message') + done, pending = await asyncio.wait([task]) + task.exception() + + task = self.new_task(loop, coro()) + with self.assertRaises(asyncio.CancelledError) as cm: + loop.run_until_complete(task) + exc = cm.exception + self.assertEqual(exc.args, ('my message',)) + + actual = get_innermost_context(exc) + self.assertEqual(actual, + (asyncio.CancelledError, ('my message',), 0)) + + def test_cancel_yield(self): + async def task(): + await asyncio.sleep(0) + await asyncio.sleep(0) + return 12 + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) # start coro + t.cancel() + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t.cancel()) + + def test_cancel_inner_future(self): + f = self.new_future(self.loop) + + async def task(): + await f + return 12 + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) # start task + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t) + self.assertTrue(f.cancelled()) + self.assertTrue(t.cancelled()) + + def test_cancel_both_task_and_inner_future(self): + f = self.new_future(self.loop) + + async def task(): + await f + return 12 + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) + + f.cancel() + t.cancel() + + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t) + + self.assertTrue(t.done()) + self.assertTrue(f.cancelled()) + self.assertTrue(t.cancelled()) + + def test_cancel_task_catching(self): + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) + + async def task(): + await fut1 + try: + await fut2 + except asyncio.CancelledError: + return 42 + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut1) # White-box test. + fut1.set_result(None) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut2) # White-box test. + t.cancel() + self.assertTrue(fut2.cancelled()) + res = self.loop.run_until_complete(t) + self.assertEqual(res, 42) + self.assertFalse(t.cancelled()) + + def test_cancel_task_ignoring(self): + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) + fut3 = self.new_future(self.loop) + + async def task(): + await fut1 + try: + await fut2 + except asyncio.CancelledError: + pass + res = await fut3 + return res + + t = self.new_task(self.loop, task()) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut1) # White-box test. + fut1.set_result(None) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut2) # White-box test. + t.cancel() + self.assertTrue(fut2.cancelled()) + test_utils.run_briefly(self.loop) + self.assertIs(t._fut_waiter, fut3) # White-box test. + fut3.set_result(42) + res = self.loop.run_until_complete(t) + self.assertEqual(res, 42) + self.assertFalse(fut3.cancelled()) + self.assertFalse(t.cancelled()) + + def test_cancel_current_task(self): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + # The sleep should be cancelled immediately. + await asyncio.sleep(100) + return 12 + + t = self.new_task(loop, task()) + self.assertFalse(t.cancelled()) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + + def test_cancel_at_end(self): + """coroutine end right after task is cancelled""" + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def task(): + t.cancel() + self.assertTrue(t._must_cancel) # White-box test. + return 12 + + t = self.new_task(loop, task()) + self.assertFalse(t.cancelled()) + self.assertRaises( + asyncio.CancelledError, loop.run_until_complete, t) + self.assertTrue(t.done()) + self.assertTrue(t.cancelled()) + self.assertFalse(t._must_cancel) # White-box test. + self.assertFalse(t.cancel()) + + def test_cancel_awaited_task(self): + # This tests for a relatively rare condition when + # a task cancellation is requested for a task which is not + # currently blocked, such as a task cancelling itself. + # In this situation we must ensure that whatever next future + # or task the cancelled task blocks on is cancelled correctly + # as well. See also bpo-34872. + loop = asyncio.new_event_loop() + self.addCleanup(lambda: loop.close()) + + task = nested_task = None + fut = self.new_future(loop) + + async def nested(): + await fut + + async def coro(): + nonlocal nested_task + # Create a sub-task and wait for it to run. + nested_task = self.new_task(loop, nested()) + await asyncio.sleep(0) + + # Request the current task to be cancelled. + task.cancel() + # Block on the nested task, which should be immediately + # cancelled. + await nested_task + + task = self.new_task(loop, coro()) + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(task) + + self.assertTrue(task.cancelled()) + self.assertTrue(nested_task.cancelled()) + self.assertTrue(fut.cancelled()) + + def assert_text_contains(self, text, substr): + if substr not in text: + raise RuntimeError(f'text {substr!r} not found in:\n>>>{text}<<<') + + def test_cancel_traceback_for_future_result(self): + # When calling Future.result() on a cancelled task, check that the + # line of code that was interrupted is included in the traceback. + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def nested(): + # This will get cancelled immediately. + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, nested()) + await asyncio.sleep(0) + task.cancel() + await task # search target + + task = self.new_task(loop, coro()) + try: + loop.run_until_complete(task) + except asyncio.CancelledError: + tb = traceback.format_exc() + self.assert_text_contains(tb, "await asyncio.sleep(10)") + # The intermediate await should also be included. + self.assert_text_contains(tb, "await task # search target") + else: + self.fail('CancelledError did not occur') + + def test_cancel_traceback_for_future_exception(self): + # When calling Future.exception() on a cancelled task, check that the + # line of code that was interrupted is included in the traceback. + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def nested(): + # This will get cancelled immediately. + await asyncio.sleep(10) + + async def coro(): + task = self.new_task(loop, nested()) + await asyncio.sleep(0) + task.cancel() + done, pending = await asyncio.wait([task]) + task.exception() # search target + + task = self.new_task(loop, coro()) + try: + loop.run_until_complete(task) + except asyncio.CancelledError: + tb = traceback.format_exc() + self.assert_text_contains(tb, "await asyncio.sleep(10)") + # The intermediate await should also be included. + self.assert_text_contains(tb, + "task.exception() # search target") + else: + self.fail('CancelledError did not occur') + + def test_stop_while_run_in_complete(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0.1 + self.assertAlmostEqual(0.2, when) + when = yield 0.1 + self.assertAlmostEqual(0.3, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + x = 0 + + async def task(): + nonlocal x + while x < 10: + await asyncio.sleep(0.1) + x += 1 + if x == 2: + loop.stop() + + t = self.new_task(loop, task()) + with self.assertRaises(RuntimeError) as cm: + loop.run_until_complete(t) + self.assertEqual(str(cm.exception), + 'Event loop stopped before Future completed.') + self.assertFalse(t.done()) + self.assertEqual(x, 2) + self.assertAlmostEqual(0.3, loop.time()) + + t.cancel() + self.assertRaises(asyncio.CancelledError, loop.run_until_complete, t) + + def test_log_traceback(self): + async def coro(): + pass + + task = self.new_task(self.loop, coro()) + with self.assertRaisesRegex(ValueError, 'can only be set to False'): + task._log_traceback = True + self.loop.run_until_complete(task) + + def test_wait(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + yield 0.15 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + b = self.new_task(loop, asyncio.sleep(0.15)) + + async def foo(): + done, pending = await asyncio.wait([b, a]) + self.assertEqual(done, set([a, b])) + self.assertEqual(pending, set()) + return 42 + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(res, 42) + self.assertAlmostEqual(0.15, loop.time()) + + # Doing it again should take no time and exercise a different path. + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertAlmostEqual(0.15, loop.time()) + self.assertEqual(res, 42) + + def test_wait_duplicate_coroutines(self): + + async def coro(s): + return s + c = self.loop.create_task(coro('test')) + task = self.new_task( + self.loop, + asyncio.wait([c, c, self.loop.create_task(coro('spam'))])) + + done, pending = self.loop.run_until_complete(task) + + self.assertFalse(pending) + self.assertEqual(set(f.result() for f in done), {'test', 'spam'}) + + def test_wait_errors(self): + self.assertRaises( + ValueError, self.loop.run_until_complete, + asyncio.wait(set())) + + # -1 is an invalid return_when value + sleep_coro = asyncio.sleep(10.0) + wait_coro = asyncio.wait([sleep_coro], return_when=-1) + self.assertRaises(ValueError, + self.loop.run_until_complete, wait_coro) + + sleep_coro.close() + + def test_wait_first_completed(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + when = yield 0 + self.assertAlmostEqual(0.1, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(10.0)) + b = self.new_task(loop, asyncio.sleep(0.1)) + task = self.new_task( + loop, + asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED)) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertFalse(a.done()) + self.assertTrue(b.done()) + self.assertIsNone(b.result()) + self.assertAlmostEqual(0.1, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_really_done(self): + # there is possibility that some tasks in the pending list + # became done but their callbacks haven't all been called yet + + async def coro1(): + await asyncio.sleep(0) + + async def coro2(): + await asyncio.sleep(0) + await asyncio.sleep(0) + + a = self.new_task(self.loop, coro1()) + b = self.new_task(self.loop, coro2()) + task = self.new_task( + self.loop, + asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED)) + + done, pending = self.loop.run_until_complete(task) + self.assertEqual({a, b}, done) + self.assertTrue(a.done()) + self.assertIsNone(a.result()) + self.assertTrue(b.done()) + self.assertIsNone(b.result()) + + def test_wait_first_exception(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + # first_exception, task already has exception + a = self.new_task(loop, asyncio.sleep(10.0)) + + async def exc(): + raise ZeroDivisionError('err') + + b = self.new_task(loop, exc()) + task = self.new_task( + loop, + asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION)) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertAlmostEqual(0, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_first_exception_in_wait(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + when = yield 0 + self.assertAlmostEqual(0.01, when) + yield 0.01 + + loop = self.new_test_loop(gen) + + # first_exception, exception during waiting + a = self.new_task(loop, asyncio.sleep(10.0)) + + async def exc(): + await asyncio.sleep(0.01) + raise ZeroDivisionError('err') + + b = self.new_task(loop, exc()) + task = asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION) + + done, pending = loop.run_until_complete(task) + self.assertEqual({b}, done) + self.assertEqual({a}, pending) + self.assertAlmostEqual(0.01, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_with_exception(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + yield 0.15 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + + async def sleeper(): + await asyncio.sleep(0.15) + raise ZeroDivisionError('really') + + b = self.new_task(loop, sleeper()) + + async def foo(): + done, pending = await asyncio.wait([b, a]) + self.assertEqual(len(done), 2) + self.assertEqual(pending, set()) + errors = set(f for f in done if f.exception() is not None) + self.assertEqual(len(errors), 1) + + loop.run_until_complete(self.new_task(loop, foo())) + self.assertAlmostEqual(0.15, loop.time()) + + loop.run_until_complete(self.new_task(loop, foo())) + self.assertAlmostEqual(0.15, loop.time()) + + def test_wait_with_timeout(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + when = yield 0 + self.assertAlmostEqual(0.11, when) + yield 0.11 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + b = self.new_task(loop, asyncio.sleep(0.15)) + + async def foo(): + done, pending = await asyncio.wait([b, a], timeout=0.11) + self.assertEqual(done, set([a])) + self.assertEqual(pending, set([b])) + + loop.run_until_complete(self.new_task(loop, foo())) + self.assertAlmostEqual(0.11, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_concurrent_complete(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + when = yield 0 + self.assertAlmostEqual(0.1, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + b = self.new_task(loop, asyncio.sleep(0.15)) + + done, pending = loop.run_until_complete( + asyncio.wait([b, a], timeout=0.1)) + + self.assertEqual(done, set([a])) + self.assertEqual(pending, set([b])) + self.assertAlmostEqual(0.1, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_wait_with_iterator_of_tasks(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(0.15, when) + yield 0.15 + + loop = self.new_test_loop(gen) + + a = self.new_task(loop, asyncio.sleep(0.1)) + b = self.new_task(loop, asyncio.sleep(0.15)) + + async def foo(): + done, pending = await asyncio.wait(iter([b, a])) + self.assertEqual(done, set([a, b])) + self.assertEqual(pending, set()) + return 42 + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(res, 42) + self.assertAlmostEqual(0.15, loop.time()) + + + def test_wait_generator(self): + async def func(a): + return a + + loop = self.new_test_loop() + + async def main(): + tasks = (self.new_task(loop, func(i)) for i in range(10)) + done, pending = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED) + self.assertEqual(len(done), 10) + self.assertEqual(len(pending), 0) + + loop.run_until_complete(main()) + + + def test_as_completed(self): + + def gen(): + yield 0 + yield 0 + yield 0.01 + yield 0 + + async def sleeper(dt, x): + nonlocal time_shifted + await asyncio.sleep(dt) + completed.add(x) + if not time_shifted and 'a' in completed and 'b' in completed: + time_shifted = True + loop.advance_time(0.14) + return x + + async def try_iterator(awaitables): + values = [] + for f in asyncio.as_completed(awaitables): + values.append(await f) + return values + + async def try_async_iterator(awaitables): + values = [] + async for f in asyncio.as_completed(awaitables): + values.append(await f) + return values + + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + # disable "slow callback" warning + loop.slow_callback_duration = 1.0 + + completed = set() + time_shifted = False + + a = sleeper(0.01, 'a') + b = sleeper(0.01, 'b') + c = sleeper(0.15, 'c') + + res = loop.run_until_complete(self.new_task(loop, foo([b, c, a]))) + self.assertAlmostEqual(0.15, loop.time()) + self.assertTrue('a' in res[:2]) + self.assertTrue('b' in res[:2]) + self.assertEqual(res[2], 'c') + + def test_as_completed_same_tasks_in_as_out(self): + # Ensures that asynchronously iterating as_completed's iterator + # yields awaitables are the same awaitables that were passed in when + # those awaitables are futures. + async def try_async_iterator(awaitables): + awaitables_out = set() + async for out_aw in asyncio.as_completed(awaitables): + awaitables_out.add(out_aw) + return awaitables_out + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + # Coroutines shouldn't be yielded back as finished coroutines + # can't be re-used. + awaitables_in = frozenset( + (coro(0), coro(1), coro(2), coro(3)) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in - awaitables_out != awaitables_in: + raise self.failureException('Got original coroutines ' + 'out of as_completed iterator.') + + # Tasks should be yielded back. + coro_obj_a = coro('a') + task_b = loop.create_task(coro('b')) + coro_obj_c = coro('c') + task_d = loop.create_task(coro('d')) + awaitables_in = frozenset( + (coro_obj_a, task_b, coro_obj_c, task_d) + ) + awaitables_out = loop.run_until_complete( + try_async_iterator(awaitables_in) + ) + if awaitables_in & awaitables_out != {task_b, task_d}: + raise self.failureException('Only tasks should be yielded ' + 'from as_completed iterator ' + 'as-is.') + + def test_as_completed_with_timeout(self): + + def gen(): + yield + yield 0 + yield 0 + yield 0.1 + + async def try_iterator(): + values = [] + for f in asyncio.as_completed([a, b], timeout=0.12): + if values: + loop.advance_time(0.02) + try: + v = await f + values.append((1, v)) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values + + async def try_async_iterator(): + values = [] + try: + async for f in asyncio.as_completed([a, b], timeout=0.12): + v = await f + values.append((1, v)) + loop.advance_time(0.02) + except asyncio.TimeoutError as exc: + values.append((2, exc)) + return values + + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + loop = self.new_test_loop(gen) + a = loop.create_task(asyncio.sleep(0.1, 'a')) + b = loop.create_task(asyncio.sleep(0.15, 'b')) + + res = loop.run_until_complete(self.new_task(loop, foo())) + self.assertEqual(len(res), 2, res) + self.assertEqual(res[0], (1, 'a')) + self.assertEqual(res[1][0], 2) + self.assertIsInstance(res[1][1], asyncio.TimeoutError) + self.assertAlmostEqual(0.12, loop.time()) + + # move forward to close generator + loop.advance_time(10) + loop.run_until_complete(asyncio.wait([a, b])) + + def test_as_completed_with_unused_timeout(self): + + def gen(): + yield + yield 0 + yield 0.01 + + async def try_iterator(): + for f in asyncio.as_completed([a], timeout=1): + v = await f + self.assertEqual(v, 'a') + + async def try_async_iterator(): + async for f in asyncio.as_completed([a], timeout=1): + v = await f + self.assertEqual(v, 'a') + + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + a = asyncio.sleep(0.01, 'a') + loop = self.new_test_loop(gen) + loop.run_until_complete(self.new_task(loop, foo())) + loop.close() + + def test_as_completed_resume_iterator(self): + # Test that as_completed returns an iterator that can be resumed + # the next time iteration is performed (i.e. if __iter__ is called + # again) + async def try_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + for f in iterator: + collected.append(await f) + return collected + + async def try_async_iterator(awaitables): + iterations = 0 + iterator = asyncio.as_completed(awaitables) + collected = [] + async for f in iterator: + collected.append(await f) + iterations += 1 + if iterations == 2: + break + self.assertEqual(len(collected), 2) + + # Resume same iterator: + async for f in iterator: + collected.append(await f) + return collected + + async def coro(i): + return i + + with contextlib.closing(asyncio.new_event_loop()) as loop: + for foo in try_iterator, try_async_iterator: + with self.subTest(method=foo.__name__): + results = loop.run_until_complete( + foo((coro(0), coro(1), coro(2), coro(3))) + ) + self.assertCountEqual(results, (0, 1, 2, 3)) + + def test_as_completed_reverse_wait(self): + # Tests the plain iterator style of as_completed iteration to + # ensure that the first future awaited resolves to the first + # completed awaitable from the set we passed in, even if it wasn't + # the first future generated by as_completed. + def gen(): + yield 0 + yield 0.05 + yield 0 + + loop = self.new_test_loop(gen) + + a = asyncio.sleep(0.05, 'a') + b = asyncio.sleep(0.10, 'b') + fs = {a, b} + + async def test(): + futs = list(asyncio.as_completed(fs)) + self.assertEqual(len(futs), 2) + + x = await futs[1] + self.assertEqual(x, 'a') + self.assertAlmostEqual(0.05, loop.time()) + loop.advance_time(0.05) + y = await futs[0] + self.assertEqual(y, 'b') + self.assertAlmostEqual(0.10, loop.time()) + + loop.run_until_complete(test()) + + def test_as_completed_concurrent(self): + # Ensure that more than one future or coroutine yielded from + # as_completed can be awaited concurrently. + def gen(): + when = yield + self.assertAlmostEqual(0.05, when) + when = yield 0 + self.assertAlmostEqual(0.05, when) + yield 0.05 + + async def try_iterator(fs): + return list(asyncio.as_completed(fs)) + + async def try_async_iterator(fs): + return [f async for f in asyncio.as_completed(fs)] + + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + a = asyncio.sleep(0.05, 'a') + b = asyncio.sleep(0.05, 'b') + fs = {a, b} + + async def test(): + futs = await runner(fs) + self.assertEqual(len(futs), 2) + done, pending = await asyncio.wait( + [asyncio.ensure_future(fut) for fut in futs] + ) + self.assertEqual(set(f.result() for f in done), {'a', 'b'}) + + loop = self.new_test_loop(gen) + loop.run_until_complete(test()) + + def test_as_completed_duplicate_coroutines(self): + + async def coro(s): + return s + + async def try_iterator(): + result = [] + c = coro('ham') + for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result + + async def try_async_iterator(): + result = [] + c = coro('ham') + async for f in asyncio.as_completed([c, c, coro('spam')]): + result.append(await f) + return result + + for runner in try_iterator, try_async_iterator: + with self.subTest(method=runner.__name__): + fut = self.new_task(self.loop, runner()) + self.loop.run_until_complete(fut) + result = fut.result() + self.assertEqual(set(result), {'ham', 'spam'}) + self.assertEqual(len(result), 2) + + def test_as_completed_coroutine_without_loop(self): + async def coro(): + return 42 + + a = coro() + self.addCleanup(a.close) + + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + futs = asyncio.as_completed([a]) + list(futs) + + def test_as_completed_coroutine_use_running_loop(self): + loop = self.new_test_loop() + + async def coro(): + return 42 + + async def test(): + futs = list(asyncio.as_completed([coro()])) + self.assertEqual(len(futs), 1) + self.assertEqual(await futs[0], 42) + + loop.run_until_complete(test()) + + def test_sleep(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.05, when) + when = yield 0.05 + self.assertAlmostEqual(0.1, when) + yield 0.05 + + loop = self.new_test_loop(gen) + + async def sleeper(dt, arg): + await asyncio.sleep(dt/2) + res = await asyncio.sleep(dt/2, arg) + return res + + t = self.new_task(loop, sleeper(0.1, 'yeah')) + loop.run_until_complete(t) + self.assertTrue(t.done()) + self.assertEqual(t.result(), 'yeah') + self.assertAlmostEqual(0.1, loop.time()) + + def test_sleep_when_delay_is_nan(self): + + def gen(): + yield + + loop = self.new_test_loop(gen) + + async def sleeper(): + await asyncio.sleep(float("nan")) + + t = self.new_task(loop, sleeper()) + + with self.assertRaises(ValueError): + loop.run_until_complete(t) + + def test_sleep_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + t = self.new_task(loop, asyncio.sleep(10.0, 'yeah')) + + handle = None + orig_call_later = loop.call_later + + def call_later(delay, callback, *args): + nonlocal handle + handle = orig_call_later(delay, callback, *args) + return handle + + loop.call_later = call_later + test_utils.run_briefly(loop) + + self.assertFalse(handle._cancelled) + + t.cancel() + test_utils.run_briefly(loop) + self.assertTrue(handle._cancelled) + + def test_task_cancel_sleeping_task(self): + + def gen(): + when = yield + self.assertAlmostEqual(0.1, when) + when = yield 0 + self.assertAlmostEqual(5000, when) + yield 0.1 + + loop = self.new_test_loop(gen) + + async def sleep(dt): + await asyncio.sleep(dt) + + async def doit(): + sleeper = self.new_task(loop, sleep(5000)) + loop.call_later(0.1, sleeper.cancel) + try: + await sleeper + except asyncio.CancelledError: + return 'cancelled' + else: + return 'slept in' + + doer = doit() + self.assertEqual(loop.run_until_complete(doer), 'cancelled') + self.assertAlmostEqual(0.1, loop.time()) + + def test_task_cancel_waiter_future(self): + fut = self.new_future(self.loop) + + async def coro(): + await fut + + task = self.new_task(self.loop, coro()) + test_utils.run_briefly(self.loop) + self.assertIs(task._fut_waiter, fut) + + task.cancel() + test_utils.run_briefly(self.loop) + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, task) + self.assertIsNone(task._fut_waiter) + self.assertTrue(fut.cancelled()) + + def test_task_set_methods(self): + async def notmuch(): + return 'ko' + + gen = notmuch() + task = self.new_task(self.loop, gen) + + with self.assertRaisesRegex(RuntimeError, 'not support set_result'): + task.set_result('ok') + + with self.assertRaisesRegex(RuntimeError, 'not support set_exception'): + task.set_exception(ValueError()) + + self.assertEqual( + self.loop.run_until_complete(task), + 'ko') + + def test_step_result_future(self): + # If coroutine returns future, task waits on this future. + + class Fut(asyncio.Future): + def __init__(self, *args, **kwds): + self.cb_added = False + super().__init__(*args, **kwds) + + def add_done_callback(self, *args, **kwargs): + self.cb_added = True + super().add_done_callback(*args, **kwargs) + + fut = Fut(loop=self.loop) + result = None + + async def wait_for_future(): + nonlocal result + result = await fut + + t = self.new_task(self.loop, wait_for_future()) + test_utils.run_briefly(self.loop) + self.assertTrue(fut.cb_added) + + res = object() + fut.set_result(res) + test_utils.run_briefly(self.loop) + self.assertIs(res, result) + self.assertTrue(t.done()) + self.assertIsNone(t.result()) + + def test_baseexception_during_cancel(self): + + def gen(): + when = yield + self.assertAlmostEqual(10.0, when) + yield 0 + + loop = self.new_test_loop(gen) + + async def sleeper(): + await asyncio.sleep(10) + + base_exc = SystemExit() + + async def notmutch(): + try: + await sleeper() + except asyncio.CancelledError: + raise base_exc + + task = self.new_task(loop, notmutch()) + test_utils.run_briefly(loop) + + task.cancel() + self.assertFalse(task.done()) + + self.assertRaises(SystemExit, test_utils.run_briefly, loop) + + self.assertTrue(task.done()) + self.assertFalse(task.cancelled()) + self.assertIs(task.exception(), base_exc) + + @ignore_warnings(category=DeprecationWarning) + def test_iscoroutinefunction(self): + def fn(): + pass + + self.assertFalse(asyncio.iscoroutinefunction(fn)) + + def fn1(): + yield + self.assertFalse(asyncio.iscoroutinefunction(fn1)) + + async def fn2(): + pass + self.assertTrue(asyncio.iscoroutinefunction(fn2)) + + self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) + self.assertTrue(asyncio.iscoroutinefunction(mock.AsyncMock())) + + @ignore_warnings(category=DeprecationWarning) + def test_coroutine_non_gen_function(self): + async def func(): + return 'test' + + self.assertTrue(asyncio.iscoroutinefunction(func)) + + coro = func() + self.assertTrue(asyncio.iscoroutine(coro)) + + res = self.loop.run_until_complete(coro) + self.assertEqual(res, 'test') + + def test_coroutine_non_gen_function_return_future(self): + fut = self.new_future(self.loop) + + async def func(): + return fut + + async def coro(): + fut.set_result('test') + + t1 = self.new_task(self.loop, func()) + t2 = self.new_task(self.loop, coro()) + res = self.loop.run_until_complete(t1) + self.assertEqual(res, fut) + self.assertIsNone(t2.result()) + + def test_current_task(self): + self.assertIsNone(self.current_task(loop=self.loop)) + + async def coro(loop): + self.assertIs(self.current_task(), task) + + self.assertIs(self.current_task(None), task) + self.assertIs(self.current_task(), task) + + task = self.new_task(self.loop, coro(self.loop)) + self.loop.run_until_complete(task) + self.assertIsNone(self.current_task(loop=self.loop)) + + def test_current_task_with_interleaving_tasks(self): + self.assertIsNone(self.current_task(loop=self.loop)) + + fut1 = self.new_future(self.loop) + fut2 = self.new_future(self.loop) + + async def coro1(loop): + self.assertTrue(self.current_task() is task1) + await fut1 + self.assertTrue(self.current_task() is task1) + fut2.set_result(True) + + async def coro2(loop): + self.assertTrue(self.current_task() is task2) + fut1.set_result(True) + await fut2 + self.assertTrue(self.current_task() is task2) + + task1 = self.new_task(self.loop, coro1(self.loop)) + task2 = self.new_task(self.loop, coro2(self.loop)) + + self.loop.run_until_complete(asyncio.wait((task1, task2))) + self.assertIsNone(self.current_task(loop=self.loop)) + + # Some thorough tests for cancellation propagation through + # coroutines, tasks and wait(). + + def test_yield_future_passes_cancel(self): + # Cancelling outer() cancels inner() cancels waiter. + proof = 0 + waiter = self.new_future(self.loop) + + async def inner(): + nonlocal proof + try: + await waiter + except asyncio.CancelledError: + proof += 1 + raise + else: + self.fail('got past sleep() in inner()') + + async def outer(): + nonlocal proof + try: + await inner() + except asyncio.CancelledError: + proof += 100 # Expect this path. + else: + proof += 10 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + self.loop.run_until_complete(f) + self.assertEqual(proof, 101) + self.assertTrue(waiter.cancelled()) + + def test_yield_wait_does_not_shield_cancel(self): + # Cancelling outer() makes wait() return early, leaves inner() + # running. + proof = 0 + waiter = self.new_future(self.loop) + + async def inner(): + nonlocal proof + await waiter + proof += 1 + + async def outer(): + nonlocal proof + with self.assertWarns(DeprecationWarning): + d, p = await asyncio.wait([asyncio.create_task(inner())]) + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + self.assertRaises( + asyncio.CancelledError, self.loop.run_until_complete, f) + waiter.set_result(None) + test_utils.run_briefly(self.loop) + self.assertEqual(proof, 1) + + def test_shield_result(self): + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + inner.set_result(42) + res = self.loop.run_until_complete(outer) + self.assertEqual(res, 42) + + def test_shield_exception(self): + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + exc = RuntimeError('expected') + inner.set_exception(exc) + test_utils.run_briefly(self.loop) + self.assertIs(outer.exception(), exc) + + def test_shield_cancel_inner(self): + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + inner.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(outer.cancelled()) + + def test_shield_cancel_outer(self): + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(outer.cancelled()) + self.assertEqual(0, 0 if outer._callbacks is None else len(outer._callbacks)) + + def test_shield_cancel_outer_result(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_result(1) + test_utils.run_briefly(self.loop) + mock_handler.assert_not_called() + + def test_shield_cancel_outer_exception(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + + def test_shield_duplicate_log_once(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + + def test_shield_shortcut(self): + fut = self.new_future(self.loop) + fut.set_result(42) + res = self.loop.run_until_complete(asyncio.shield(fut)) + self.assertEqual(res, 42) + + def test_shield_effect(self): + # Cancelling outer() does not affect inner(). + proof = 0 + waiter = self.new_future(self.loop) + + async def inner(): + nonlocal proof + await waiter + proof += 1 + + async def outer(): + nonlocal proof + await asyncio.shield(inner()) + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.loop) + test_utils.run_briefly(self.loop) + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) + waiter.set_result(None) + test_utils.run_briefly(self.loop) + self.assertEqual(proof, 1) + + def test_shield_gather(self): + child1 = self.new_future(self.loop) + child2 = self.new_future(self.loop) + parent = asyncio.gather(child1, child2) + outer = asyncio.shield(parent) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(outer.cancelled()) + child1.set_result(1) + child2.set_result(2) + test_utils.run_briefly(self.loop) + self.assertEqual(parent.result(), [1, 2]) + + def test_gather_shield(self): + child1 = self.new_future(self.loop) + child2 = self.new_future(self.loop) + inner1 = asyncio.shield(child1) + inner2 = asyncio.shield(child2) + parent = asyncio.gather(inner1, inner2) + test_utils.run_briefly(self.loop) + parent.cancel() + # This should cancel inner1 and inner2 but bot child1 and child2. + test_utils.run_briefly(self.loop) + self.assertIsInstance(parent.exception(), asyncio.CancelledError) + self.assertTrue(inner1.cancelled()) + self.assertTrue(inner2.cancelled()) + child1.set_result(1) + child2.set_result(2) + test_utils.run_briefly(self.loop) + + def test_shield_coroutine_without_loop(self): + async def coro(): + return 42 + + inner = coro() + self.addCleanup(inner.close) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.shield(inner) + + def test_shield_coroutine_use_running_loop(self): + async def coro(): + return 42 + + async def test(): + return asyncio.shield(coro()) + outer = self.loop.run_until_complete(test()) + self.assertEqual(outer._loop, self.loop) + res = self.loop.run_until_complete(outer) + self.assertEqual(res, 42) + + def test_shield_coroutine_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + async def coro(): + return 42 + + asyncio.set_event_loop(self.loop) + self.addCleanup(asyncio.set_event_loop, None) + outer = asyncio.shield(coro()) + self.assertEqual(outer._loop, self.loop) + res = self.loop.run_until_complete(outer) + self.assertEqual(res, 42) + + def test_as_completed_invalid_args(self): + # as_completed() expects a list of futures, not a future instance + # TypeError should be raised either on iterator construction or first + # iteration + + # Plain iterator + fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(fut) + next(iterator) + coro = coroutine_function() + with self.assertRaises(TypeError): + iterator = asyncio.as_completed(coro) + next(iterator) + coro.close() + + # Async iterator + async def try_async_iterator(aw): + async for f in asyncio.as_completed(aw): + break + + fut = self.new_future(self.loop) + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(fut)) + coro = coroutine_function() + with self.assertRaises(TypeError): + self.loop.run_until_complete(try_async_iterator(coro)) + coro.close() + + def test_wait_invalid_args(self): + fut = self.new_future(self.loop) + + # wait() expects a list of futures, not a future instance + self.assertRaises(TypeError, self.loop.run_until_complete, + asyncio.wait(fut)) + coro = coroutine_function() + self.assertRaises(TypeError, self.loop.run_until_complete, + asyncio.wait(coro)) + coro.close() + + # wait() expects at least a future + self.assertRaises(ValueError, self.loop.run_until_complete, + asyncio.wait([])) + + def test_log_destroyed_pending_task(self): + + async def kill_me(loop): + future = self.new_future(loop) + await future + # at this point, the only reference to kill_me() task is + # the Task._wakeup() method in future._callbacks + raise Exception("code never reached") + + mock_handler = mock.Mock() + self.loop.set_debug(True) + self.loop.set_exception_handler(mock_handler) + + # schedule the task + coro = kill_me(self.loop) + task = self.new_task(self.loop, coro) + + self.assertEqual(self.all_tasks(loop=self.loop), {task}) + + # execute the task so it waits for future + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(len(self.loop._ready), 0) + + coro = None + source_traceback = task._source_traceback + task = None + + # no more reference to kill_me() task: the task is destroyed by the GC + support.gc_collect() + + mock_handler.assert_called_with(self.loop, { + 'message': 'Task was destroyed but it is pending!', + 'task': mock.ANY, + 'source_traceback': source_traceback, + }) + mock_handler.reset_mock() + # task got resurrected by the exception handler + support.gc_collect() + + self.assertEqual(self.all_tasks(loop=self.loop), set()) + + def test_task_not_crash_without_finalization(self): + Task = self.__class__.Task + + class Subclass(Task): + def __del__(self): + pass + + async def corofn(): + await asyncio.sleep(0.01) + + coro = corofn() + task = Subclass(coro, loop = self.loop) + task._log_destroy_pending = False + + del task + + support.gc_collect() + + coro.close() + + @mock.patch('asyncio.base_events.logger') + def test_tb_logger_not_called_after_cancel(self, m_log): + loop = asyncio.new_event_loop() + self.set_event_loop(loop) + + async def coro(): + raise TypeError + + async def runner(): + task = self.new_task(loop, coro()) + await asyncio.sleep(0.05) + task.cancel() + task = None + + loop.run_until_complete(runner()) + self.assertFalse(m_log.error.called) + + def test_task_source_traceback(self): + self.loop.set_debug(True) + + task = self.new_task(self.loop, coroutine_function()) + lineno = sys._getframe().f_lineno - 1 + self.assertIsInstance(task._source_traceback, list) + self.assertEqual(task._source_traceback[-2][:3], + (__file__, + lineno, + 'test_task_source_traceback')) + self.loop.run_until_complete(task) + + def test_cancel_gather_1(self): + """Ensure that a gathering future refuses to be cancelled once all + children are done""" + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + fut = self.new_future(loop) + async def create(): + # The indirection fut->child_coro is needed since otherwise the + # gathering task is done at the same time as the child future + async def child_coro(): + return await fut + gather_future = asyncio.gather(child_coro()) + return asyncio.ensure_future(gather_future) + gather_task = loop.run_until_complete(create()) + + cancel_result = None + def cancelling_callback(_): + nonlocal cancel_result + cancel_result = gather_task.cancel() + fut.add_done_callback(cancelling_callback) + + fut.set_result(42) # calls the cancelling_callback after fut is done() + + # At this point the task should complete. + loop.run_until_complete(gather_task) + + # Python issue #26923: asyncio.gather drops cancellation + self.assertEqual(cancel_result, False) + self.assertFalse(gather_task.cancelled()) + self.assertEqual(gather_task.result(), [42]) + + def test_cancel_gather_2(self): + cases = [ + ((), ()), + ((None,), ()), + (('my message',), ('my message',)), + # Non-string values should roundtrip. + ((5,), (5,)), + ] + for cancel_args, expected_args in cases: + with self.subTest(cancel_args=cancel_args): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + async def test(): + time = 0 + while True: + time += 0.05 + await asyncio.gather(asyncio.sleep(0.05), + return_exceptions=True) + if time > 1: + return + + async def main(): + qwe = self.new_task(loop, test()) + await asyncio.sleep(0.2) + qwe.cancel(*cancel_args) + await qwe + + try: + loop.run_until_complete(main()) + except asyncio.CancelledError as exc: + self.assertEqual(exc.args, expected_args) + actual = get_innermost_context(exc) + self.assertEqual( + actual, + (asyncio.CancelledError, expected_args, 0), + ) + else: + self.fail( + 'gather() does not propagate CancelledError ' + 'raised by inner task to the gather() caller.' + ) + + def test_exception_traceback(self): + # See http://bugs.python.org/issue28843 + + async def foo(): + 1 / 0 + + async def main(): + task = self.new_task(self.loop, foo()) + await asyncio.sleep(0) # skip one loop iteration + self.assertIsNotNone(task.exception().__traceback__) + + self.loop.run_until_complete(main()) + + @mock.patch('asyncio.base_events.logger') + def test_error_in_call_soon(self, m_log): + def call_soon(callback, *args, **kwargs): + raise ValueError + self.loop.call_soon = call_soon + + async def coro(): + pass + + self.assertFalse(m_log.error.called) + + with self.assertRaises(ValueError): + gen = coro() + try: + self.new_task(self.loop, gen) + finally: + gen.close() + gc.collect() # For PyPy or other GCs. + + self.assertTrue(m_log.error.called) + message = m_log.error.call_args[0][0] + self.assertIn('Task was destroyed but it is pending', message) + + self.assertEqual(self.all_tasks(self.loop), set()) + + def test_create_task_with_noncoroutine(self): + with self.assertRaisesRegex(TypeError, + "a coroutine was expected, got 123"): + self.new_task(self.loop, 123) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + with self.assertRaisesRegex(TypeError, + "a coroutine was expected, got 123"): + self.new_task(self.loop, 123) + + def test_create_task_with_async_function(self): + + async def coro(): + pass + + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + def test_create_task_with_asynclike_function(self): + task = self.new_task(self.loop, CoroLikeObject()) + self.assertIsInstance(task, self.Task) + self.assertEqual(self.loop.run_until_complete(task), 42) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + task = self.new_task(self.loop, CoroLikeObject()) + self.assertIsInstance(task, self.Task) + self.assertEqual(self.loop.run_until_complete(task), 42) + + def test_bare_create_task(self): + + async def inner(): + return 1 + + async def coro(): + task = asyncio.create_task(inner()) + self.assertIsInstance(task, self.Task) + ret = await task + self.assertEqual(1, ret) + + self.loop.run_until_complete(coro()) + + def test_bare_create_named_task(self): + + async def coro_noop(): + pass + + async def coro(): + task = asyncio.create_task(coro_noop(), name='No-op') + self.assertEqual(task.get_name(), 'No-op') + await task + + self.loop.run_until_complete(coro()) + + def test_context_1(self): + cvar = contextvars.ContextVar('cvar', default='nope') + + async def sub(): + await asyncio.sleep(0.01) + self.assertEqual(cvar.get(), 'nope') + cvar.set('something else') + + async def main(): + self.assertEqual(cvar.get(), 'nope') + subtask = self.new_task(loop, sub()) + cvar.set('yes') + self.assertEqual(cvar.get(), 'yes') + await subtask + self.assertEqual(cvar.get(), 'yes') + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + loop.run_until_complete(task) + finally: + loop.close() + + def test_context_2(self): + cvar = contextvars.ContextVar('cvar', default='nope') + + async def main(): + def fut_on_done(fut): + # This change must not pollute the context + # of the "main()" task. + cvar.set('something else') + + self.assertEqual(cvar.get(), 'nope') + + for j in range(2): + fut = self.new_future(loop) + fut.add_done_callback(fut_on_done) + cvar.set(f'yes{j}') + loop.call_soon(fut.set_result, None) + await fut + self.assertEqual(cvar.get(), f'yes{j}') + + for i in range(3): + # Test that task passed its context to add_done_callback: + cvar.set(f'yes{i}-{j}') + await asyncio.sleep(0.001) + self.assertEqual(cvar.get(), f'yes{i}-{j}') + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual(cvar.get(), 'nope') + + def test_context_3(self): + # Run 100 Tasks in parallel, each modifying cvar. + + cvar = contextvars.ContextVar('cvar', default=-1) + + async def sub(num): + for i in range(10): + cvar.set(num + i) + await asyncio.sleep(random.uniform(0.001, 0.05)) + self.assertEqual(cvar.get(), num + i) + + async def main(): + tasks = [] + for i in range(100): + task = loop.create_task(sub(random.randint(0, 10))) + tasks.append(task) + + await asyncio.gather(*tasks) + + loop = asyncio.new_event_loop() + try: + loop.run_until_complete(main()) + finally: + loop.close() + + self.assertEqual(cvar.get(), -1) + + def test_context_4(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = self.new_task(loop, coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = self.new_task(loop, coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + + def test_context_5(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = asyncio.create_task(coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = asyncio.create_task(coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + + def test_context_6(self): + cvar = contextvars.ContextVar('cvar') + + async def coro(val): + await asyncio.sleep(0) + cvar.set(val) + + async def main(): + ret = [] + ctx = contextvars.copy_context() + ret.append(ctx.get(cvar)) + t1 = loop.create_task(coro(1), context=ctx) + await t1 + ret.append(ctx.get(cvar)) + t2 = loop.create_task(coro(2), context=ctx) + await t2 + ret.append(ctx.get(cvar)) + return ret + + loop = asyncio.new_event_loop() + try: + task = loop.create_task(main()) + ret = loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual([None, 1, 2], ret) + + def test_eager_start_true(self): + name = None + + async def asyncfn(): + nonlocal name + name = self.current_task().get_name() + + async def main(): + t = self.new_task(coro=asyncfn(), loop=asyncio.get_running_loop(), eager_start=True, name="example") + self.assertTrue(t.done()) + self.assertEqual(name, "example") + await t + + def test_eager_start_false(self): + name = None + + async def asyncfn(): + nonlocal name + name = self.current_task().get_name() + + async def main(): + t = self.new_task(coro=asyncfn(), loop=asyncio.get_running_loop(), eager_start=False, name="example") + self.assertFalse(t.done()) + self.assertIsNone(name) + await t + self.assertEqual(name, "example") + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + + def test_get_coro(self): + loop = asyncio.new_event_loop() + coro = coroutine_function() + try: + task = self.new_task(loop, coro) + loop.run_until_complete(task) + self.assertIs(task.get_coro(), coro) + finally: + loop.close() + + def test_get_context(self): + loop = asyncio.new_event_loop() + coro = coroutine_function() + context = contextvars.copy_context() + try: + task = self.new_task(loop, coro, context=context) + loop.run_until_complete(task) + self.assertIs(task.get_context(), context) + finally: + loop.close() + + def test_proper_refcounts(self): + # see: https://github.com/python/cpython/issues/126083 + class Break: + def __str__(self): + raise RuntimeError("break") + + obj = object() + initial_refcount = sys.getrefcount(obj) + + coro = coroutine_function() + with contextlib.closing(asyncio.EventLoop()) as loop: + task = asyncio.Task.__new__(asyncio.Task) + for _ in range(5): + with self.assertRaisesRegex(RuntimeError, 'break'): + task.__init__(coro, loop=loop, context=obj, name=Break()) + + coro.close() + task._log_destroy_pending = False + del task + + self.assertEqual(sys.getrefcount(obj), initial_refcount) + + +def add_subclass_tests(cls): + BaseTask = cls.Task + BaseFuture = cls.Future + + if BaseTask is None or BaseFuture is None: + return cls + + class CommonFuture: + def __init__(self, *args, **kwargs): + self.calls = collections.defaultdict(lambda: 0) + super().__init__(*args, **kwargs) + + def add_done_callback(self, *args, **kwargs): + self.calls['add_done_callback'] += 1 + return super().add_done_callback(*args, **kwargs) + + class Task(CommonFuture, BaseTask): + pass + + class Future(CommonFuture, BaseFuture): + pass + + def test_subclasses_ctask_cfuture(self): + fut = self.Future(loop=self.loop) + + async def func(): + self.loop.call_soon(lambda: fut.set_result('spam')) + return await fut + + task = self.Task(func(), loop=self.loop) + + result = self.loop.run_until_complete(task) + + self.assertEqual(result, 'spam') + + self.assertEqual( + dict(task.calls), + {'add_done_callback': 1}) + + self.assertEqual( + dict(fut.calls), + {'add_done_callback': 1}) + + # Add patched Task & Future back to the test case + cls.Task = Task + cls.Future = Future + + # Add an extra unit-test + cls.test_subclasses_ctask_cfuture = test_subclasses_ctask_cfuture + + # Disable the "test_task_source_traceback" test + # (the test is hardcoded for a particular call stack, which + # is slightly different for Task subclasses) + cls.test_task_source_traceback = None + + return cls + + +class SetMethodsTest: + + def test_set_result_causes_invalid_state(self): + Future = type(self).Future + self.loop.call_exception_handler = exc_handler = mock.Mock() + + async def foo(): + await asyncio.sleep(0.1) + return 10 + + coro = foo() + task = self.new_task(self.loop, coro) + Future.set_result(task, 'spam') + + self.assertEqual( + self.loop.run_until_complete(task), + 'spam') + + exc_handler.assert_called_once() + exc = exc_handler.call_args[0][0]['exception'] + with self.assertRaisesRegex(asyncio.InvalidStateError, + r'step\(\): already done'): + raise exc + + coro.close() + + def test_set_exception_causes_invalid_state(self): + class MyExc(Exception): + pass + + Future = type(self).Future + self.loop.call_exception_handler = exc_handler = mock.Mock() + + async def foo(): + await asyncio.sleep(0.1) + return 10 + + coro = foo() + task = self.new_task(self.loop, coro) + Future.set_exception(task, MyExc()) + + with self.assertRaises(MyExc): + self.loop.run_until_complete(task) + + exc_handler.assert_called_once() + exc = exc_handler.call_args[0][0]['exception'] + with self.assertRaisesRegex(asyncio.InvalidStateError, + r'step\(\): already done'): + raise exc + + coro.close() + + +@unittest.skipUnless(hasattr(futures, '_CFuture') and + hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest, + test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @support.refcount_test + def test_refleaks_in_task___init__(self): + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + async def coro(): + pass + task = self.new_task(self.loop, coro()) + self.loop.run_until_complete(task) + refs_before = gettotalrefcount() + for i in range(100): + task.__init__(coro(), loop=self.loop) + self.loop.run_until_complete(task) + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_del__log_destroy_pending_segfault(self): + async def coro(): + pass + task = self.new_task(self.loop, coro()) + self.loop.run_until_complete(task) + with self.assertRaises(AttributeError): + del task._log_destroy_pending + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + + +@unittest.skipUnless(hasattr(futures, '_CFuture') and + hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +@add_subclass_tests +class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = getattr(futures, '_CFuture', None) + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +@add_subclass_tests +class CTaskSubclass_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +@add_subclass_tests +class PyTask_CFutureSubclass_Tests(BaseTaskTests, test_utils.TestCase): + + Future = getattr(futures, '_CFuture', None) + Task = tasks._PyTask + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = futures._PyFuture + all_tasks = getattr(tasks, '_c_all_tasks', None) + current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): + + Task = tasks._PyTask + Future = getattr(futures, '_CFuture', None) + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, + test_utils.TestCase): + + Task = tasks._PyTask + Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + + +@add_subclass_tests +class PyTask_PyFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): + Task = tasks._PyTask + Future = futures._PyFuture + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Actual: not called. + def test_log_destroyed_pending_task(self): + return super().test_log_destroyed_pending_task() + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CTask_Future_Tests(test_utils.TestCase): + + def test_foobar(self): + class Fut(asyncio.Future): + @property + def get_loop(self): + raise AttributeError + + async def coro(): + await fut + return 'spam' + + self.loop = asyncio.new_event_loop() + try: + fut = Fut(loop=self.loop) + self.loop.call_later(0.1, fut.set_result, 1) + task = self.loop.create_task(coro()) + res = self.loop.run_until_complete(task) + finally: + self.loop.close() + + self.assertEqual(res, 'spam') + + +class BaseTaskIntrospectionTests: + _register_task = None + _unregister_task = None + _enter_task = None + _leave_task = None + all_tasks = None + + def test__register_task_1(self): + class TaskLike: + @property + def _loop(self): + return loop + + def done(self): + return False + + task = TaskLike() + loop = mock.Mock() + + self.assertEqual(self.all_tasks(loop), set()) + self._register_task(task) + self.assertEqual(self.all_tasks(loop), {task}) + self._unregister_task(task) + + def test__register_task_2(self): + class TaskLike: + def get_loop(self): + return loop + + def done(self): + return False + + task = TaskLike() + loop = mock.Mock() + + self.assertEqual(self.all_tasks(loop), set()) + self._register_task(task) + self.assertEqual(self.all_tasks(loop), {task}) + self._unregister_task(task) + + def test__register_task_3(self): + class TaskLike: + def get_loop(self): + return loop + + def done(self): + return True + + task = TaskLike() + loop = mock.Mock() + + self.assertEqual(self.all_tasks(loop), set()) + self._register_task(task) + self.assertEqual(self.all_tasks(loop), set()) + self._unregister_task(task) + + def test__enter_task(self): + task = mock.Mock() + loop = mock.Mock() + # _enter_task is called by Task.__step while the loop + # is running, so set the loop as the running loop + # for a more realistic test. + asyncio._set_running_loop(loop) + self.assertIsNone(self.current_task(loop)) + self._enter_task(loop, task) + self.assertIs(self.current_task(loop), task) + self._leave_task(loop, task) + asyncio._set_running_loop(None) + + def test__enter_task_failure(self): + task1 = mock.Mock() + task2 = mock.Mock() + loop = mock.Mock() + asyncio._set_running_loop(loop) + self._enter_task(loop, task1) + with self.assertRaises(RuntimeError): + self._enter_task(loop, task2) + self.assertIs(self.current_task(loop), task1) + self._leave_task(loop, task1) + asyncio._set_running_loop(None) + + def test__leave_task(self): + task = mock.Mock() + loop = mock.Mock() + asyncio._set_running_loop(loop) + self._enter_task(loop, task) + self._leave_task(loop, task) + self.assertIsNone(self.current_task(loop)) + asyncio._set_running_loop(None) + + def test__leave_task_failure1(self): + task1 = mock.Mock() + task2 = mock.Mock() + loop = mock.Mock() + # _leave_task is called by Task.__step while the loop + # is running, so set the loop as the running loop + # for a more realistic test. + asyncio._set_running_loop(loop) + self._enter_task(loop, task1) + with self.assertRaises(RuntimeError): + self._leave_task(loop, task2) + self.assertIs(self.current_task(loop), task1) + self._leave_task(loop, task1) + asyncio._set_running_loop(None) + + def test__leave_task_failure2(self): + task = mock.Mock() + loop = mock.Mock() + asyncio._set_running_loop(loop) + with self.assertRaises(RuntimeError): + self._leave_task(loop, task) + self.assertIsNone(self.current_task(loop)) + asyncio._set_running_loop(None) + + def test__unregister_task(self): + task = mock.Mock() + loop = mock.Mock() + task.get_loop = lambda: loop + self._register_task(task) + self._unregister_task(task) + self.assertEqual(self.all_tasks(loop), set()) + + def test__unregister_task_not_registered(self): + task = mock.Mock() + loop = mock.Mock() + self._unregister_task(task) + self.assertEqual(self.all_tasks(loop), set()) + + +class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): + _register_task = staticmethod(tasks._py_register_task) + _unregister_task = staticmethod(tasks._py_unregister_task) + _enter_task = staticmethod(tasks._py_enter_task) + _leave_task = staticmethod(tasks._py_leave_task) + all_tasks = staticmethod(tasks._py_all_tasks) + current_task = staticmethod(tasks._py_current_task) + + +@unittest.skipUnless(hasattr(tasks, '_c_register_task'), + 'requires the C _asyncio module') +class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): + if hasattr(tasks, '_c_register_task'): + _register_task = staticmethod(tasks._c_register_task) + _unregister_task = staticmethod(tasks._c_unregister_task) + _enter_task = staticmethod(tasks._c_enter_task) + _leave_task = staticmethod(tasks._c_leave_task) + all_tasks = staticmethod(tasks._c_all_tasks) + current_task = staticmethod(tasks._c_current_task) + else: + _register_task = _unregister_task = _enter_task = _leave_task = None + + +class BaseCurrentLoopTests: + current_task = None + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def new_task(self, coro): + raise NotImplementedError + + def test_current_task_no_running_loop(self): + self.assertIsNone(self.current_task(loop=self.loop)) + + def test_current_task_no_running_loop_implicit(self): + with self.assertRaisesRegex(RuntimeError, 'no running event loop'): + self.current_task() + + def test_current_task_with_implicit_loop(self): + async def coro(): + self.assertIs(self.current_task(loop=self.loop), task) + + self.assertIs(self.current_task(None), task) + self.assertIs(self.current_task(), task) + + task = self.new_task(coro()) + self.loop.run_until_complete(task) + self.assertIsNone(self.current_task(loop=self.loop)) + + +class PyCurrentLoopTests(BaseCurrentLoopTests, test_utils.TestCase): + current_task = staticmethod(tasks._py_current_task) + + def new_task(self, coro): + return tasks._PyTask(coro, loop=self.loop) + + +@unittest.skipUnless(hasattr(tasks, '_CTask') and + hasattr(tasks, '_c_current_task'), + 'requires the C _asyncio module') +class CCurrentLoopTests(BaseCurrentLoopTests, test_utils.TestCase): + if hasattr(tasks, '_c_current_task'): + current_task = staticmethod(tasks._c_current_task) + else: + current_task = None + + def new_task(self, coro): + return getattr(tasks, '_CTask')(coro, loop=self.loop) + + +class GenericTaskTests(test_utils.TestCase): + + def test_future_subclass(self): + self.assertIsSubclass(asyncio.Task, asyncio.Future) + + @support.cpython_only + def test_asyncio_module_compiled(self): + # Because of circular imports it's easy to make _asyncio + # module non-importable. This is a simple test that will + # fail on systems where C modules were successfully compiled + # (hence the test for _functools etc), but _asyncio somehow didn't. + try: + import _functools # noqa: F401 + import _json # noqa: F401 + import _pickle # noqa: F401 + except ImportError: + self.skipTest('C modules are not available') + else: + try: + import _asyncio # noqa: F401 + except ImportError: + self.fail('_asyncio module is missing') + + +class GatherTestsBase: + + def setUp(self): + super().setUp() + self.one_loop = self.new_test_loop() + self.other_loop = self.new_test_loop() + self.set_event_loop(self.one_loop, cleanup=False) + + def _run_loop(self, loop): + while loop._ready: + test_utils.run_briefly(loop) + + def _check_success(self, **kwargs): + a, b, c = [self.one_loop.create_future() for i in range(3)] + fut = self._gather(*self.wrap_futures(a, b, c), **kwargs) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + b.set_result(1) + a.set_result(2) + self._run_loop(self.one_loop) + self.assertEqual(cb.called, False) + self.assertFalse(fut.done()) + c.set_result(3) + self._run_loop(self.one_loop) + cb.assert_called_once_with(fut) + self.assertEqual(fut.result(), [2, 1, 3]) + + def test_success(self): + self._check_success() + self._check_success(return_exceptions=False) + + def test_result_exception_success(self): + self._check_success(return_exceptions=True) + + def test_one_exception(self): + a, b, c, d, e = [self.one_loop.create_future() for i in range(5)] + fut = self._gather(*self.wrap_futures(a, b, c, d, e)) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + exc = ZeroDivisionError() + a.set_result(1) + b.set_exception(exc) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertIs(fut.exception(), exc) + # Does nothing + c.set_result(3) + d.cancel() + e.set_exception(RuntimeError()) + e.exception() + + def test_return_exceptions(self): + a, b, c, d = [self.one_loop.create_future() for i in range(4)] + fut = self._gather(*self.wrap_futures(a, b, c, d), + return_exceptions=True) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + exc = ZeroDivisionError() + exc2 = RuntimeError() + b.set_result(1) + c.set_exception(exc) + a.set_result(3) + self._run_loop(self.one_loop) + self.assertFalse(fut.done()) + d.set_exception(exc2) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertEqual(fut.result(), [3, 1, exc, exc2]) + + def test_env_var_debug(self): + code = '\n'.join(( + 'import asyncio.coroutines', + 'print(asyncio.coroutines._is_debug_mode())')) + + # Test with -E to not fail if the unit test was run with + # PYTHONASYNCIODEBUG set to a non-empty string + sts, stdout, stderr = assert_python_ok('-E', '-c', code) + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'False') + + sts, stdout, stderr = assert_python_ok('-c', code, + PYTHONASYNCIODEBUG='1', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'True') + + sts, stdout, stderr = assert_python_ok('-E', '-c', code, + PYTHONASYNCIODEBUG='1', + PYTHONDEVMODE='') + self.assertEqual(stdout.rstrip(), b'False') + + # -X dev + sts, stdout, stderr = assert_python_ok('-E', '-X', 'dev', + '-c', code) + self.assertEqual(stdout.rstrip(), b'True') + + +class FutureGatherTests(GatherTestsBase, test_utils.TestCase): + + def wrap_futures(self, *futures): + return futures + + def _gather(self, *args, **kwargs): + return asyncio.gather(*args, **kwargs) + + def test_constructor_empty_sequence_without_loop(self): + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather() + + def test_constructor_empty_sequence_use_running_loop(self): + async def gather(): + return asyncio.gather() + fut = self.one_loop.run_until_complete(gather()) + self.assertIsInstance(fut, asyncio.Future) + self.assertIs(fut._loop, self.one_loop) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + self.assertEqual(fut.result(), []) + + def test_constructor_empty_sequence_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + asyncio.set_event_loop(self.one_loop) + self.addCleanup(asyncio.set_event_loop, None) + fut = asyncio.gather() + self.assertIsInstance(fut, asyncio.Future) + self.assertIs(fut._loop, self.one_loop) + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + self.assertEqual(fut.result(), []) + + def test_constructor_heterogenous_futures(self): + fut1 = self.one_loop.create_future() + fut2 = self.other_loop.create_future() + with self.assertRaises(ValueError): + asyncio.gather(fut1, fut2) + + def test_constructor_homogenous_futures(self): + children = [self.other_loop.create_future() for i in range(3)] + fut = asyncio.gather(*children) + self.assertIs(fut._loop, self.other_loop) + self._run_loop(self.other_loop) + self.assertFalse(fut.done()) + fut = asyncio.gather(*children) + self.assertIs(fut._loop, self.other_loop) + self._run_loop(self.other_loop) + self.assertFalse(fut.done()) + + def test_one_cancellation(self): + a, b, c, d, e = [self.one_loop.create_future() for i in range(5)] + fut = asyncio.gather(a, b, c, d, e) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + a.set_result(1) + b.cancel() + self._run_loop(self.one_loop) + self.assertTrue(fut.done()) + cb.assert_called_once_with(fut) + self.assertFalse(fut.cancelled()) + self.assertIsInstance(fut.exception(), asyncio.CancelledError) + # Does nothing + c.set_result(3) + d.cancel() + e.set_exception(RuntimeError()) + e.exception() + + def test_result_exception_one_cancellation(self): + a, b, c, d, e, f = [self.one_loop.create_future() + for i in range(6)] + fut = asyncio.gather(a, b, c, d, e, f, return_exceptions=True) + cb = test_utils.MockCallback() + fut.add_done_callback(cb) + a.set_result(1) + zde = ZeroDivisionError() + b.set_exception(zde) + c.cancel() + self._run_loop(self.one_loop) + self.assertFalse(fut.done()) + d.set_result(3) + e.cancel() + rte = RuntimeError() + f.set_exception(rte) + res = self.one_loop.run_until_complete(fut) + self.assertIsInstance(res[2], asyncio.CancelledError) + self.assertIsInstance(res[4], asyncio.CancelledError) + res[2] = res[4] = None + self.assertEqual(res, [1, zde, None, 3, None, rte]) + cb.assert_called_once_with(fut) + + +class CoroutineGatherTests(GatherTestsBase, test_utils.TestCase): + + def wrap_futures(self, *futures): + coros = [] + for fut in futures: + async def coro(fut=fut): + return await fut + coros.append(coro()) + return coros + + def _gather(self, *args, **kwargs): + async def coro(): + return asyncio.gather(*args, **kwargs) + return self.one_loop.run_until_complete(coro()) + + def test_constructor_without_loop(self): + async def coro(): + return 'abc' + gen1 = coro() + self.addCleanup(gen1.close) + gen2 = coro() + self.addCleanup(gen2.close) + with self.assertRaisesRegex(RuntimeError, 'no current event loop'): + asyncio.gather(gen1, gen2) + + def test_constructor_use_running_loop(self): + async def coro(): + return 'abc' + gen1 = coro() + gen2 = coro() + async def gather(): + return asyncio.gather(gen1, gen2) + fut = self.one_loop.run_until_complete(gather()) + self.assertIs(fut._loop, self.one_loop) + self.one_loop.run_until_complete(fut) + + def test_constructor_use_global_loop(self): + # Deprecated in 3.10, undeprecated in 3.12 + async def coro(): + return 'abc' + asyncio.set_event_loop(self.other_loop) + self.addCleanup(asyncio.set_event_loop, None) + gen1 = coro() + gen2 = coro() + fut = asyncio.gather(gen1, gen2) + self.assertIs(fut._loop, self.other_loop) + self.other_loop.run_until_complete(fut) + + def test_duplicate_coroutines(self): + async def coro(s): + return s + c = coro('abc') + fut = self._gather(c, c, coro('def'), c) + self._run_loop(self.one_loop) + self.assertEqual(fut.result(), ['abc', 'abc', 'def', 'abc']) + + def test_cancellation_broadcast(self): + # Cancelling outer() cancels all children. + proof = 0 + waiter = self.one_loop.create_future() + + async def inner(): + nonlocal proof + await waiter + proof += 1 + + child1 = asyncio.ensure_future(inner(), loop=self.one_loop) + child2 = asyncio.ensure_future(inner(), loop=self.one_loop) + gatherer = None + + async def outer(): + nonlocal proof, gatherer + gatherer = asyncio.gather(child1, child2) + await gatherer + proof += 100 + + f = asyncio.ensure_future(outer(), loop=self.one_loop) + test_utils.run_briefly(self.one_loop) + self.assertTrue(f.cancel()) + with self.assertRaises(asyncio.CancelledError): + self.one_loop.run_until_complete(f) + self.assertFalse(gatherer.cancel()) + self.assertTrue(waiter.cancelled()) + self.assertTrue(child1.cancelled()) + self.assertTrue(child2.cancelled()) + test_utils.run_briefly(self.one_loop) + self.assertEqual(proof, 0) + + def test_exception_marking(self): + # Test for the first line marked "Mark exception retrieved." + + async def inner(f): + await f + raise RuntimeError('should not be ignored') + + a = self.one_loop.create_future() + b = self.one_loop.create_future() + + async def outer(): + await asyncio.gather(inner(a), inner(b)) + + f = asyncio.ensure_future(outer(), loop=self.one_loop) + test_utils.run_briefly(self.one_loop) + a.set_result(None) + test_utils.run_briefly(self.one_loop) + b.set_result(None) + test_utils.run_briefly(self.one_loop) + self.assertIsInstance(f.exception(), RuntimeError) + + def test_issue46672(self): + with mock.patch( + 'asyncio.base_events.BaseEventLoop.call_exception_handler', + ): + async def coro(s): + return s + c = coro('abc') + + with self.assertRaises(TypeError): + self._gather(c, {}) + self._run_loop(self.one_loop) + # NameError should not happen: + self.one_loop.call_exception_handler.assert_not_called() + + +class RunCoroutineThreadsafeTests(test_utils.TestCase): + """Test case for asyncio.run_coroutine_threadsafe.""" + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) # Will cleanup properly + + async def add(self, a, b, fail=False, cancel=False): + """Wait 0.05 second and return a + b.""" + await asyncio.sleep(0.05) + if fail: + raise RuntimeError("Fail!") + if cancel: + asyncio.current_task(self.loop).cancel() + await asyncio.sleep(0) + return a + b + + def target(self, fail=False, cancel=False, timeout=None, + advance_coro=False): + """Run add coroutine in the event loop.""" + coro = self.add(1, 2, fail=fail, cancel=cancel) + future = asyncio.run_coroutine_threadsafe(coro, self.loop) + if advance_coro: + # this is for test_run_coroutine_threadsafe_task_factory_exception; + # otherwise it spills errors and breaks **other** unittests, since + # 'target' is interacting with threads. + + # With this call, `coro` will be advanced. + self.loop.call_soon_threadsafe(coro.send, None) + try: + return future.result(timeout) + finally: + future.done() or future.cancel() + + def test_run_coroutine_threadsafe(self): + """Test coroutine submission from a thread to an event loop.""" + future = self.loop.run_in_executor(None, self.target) + result = self.loop.run_until_complete(future) + self.assertEqual(result, 3) + + def test_run_coroutine_threadsafe_with_exception(self): + """Test coroutine submission from a thread to an event loop + when an exception is raised.""" + future = self.loop.run_in_executor(None, self.target, True) + with self.assertRaises(RuntimeError) as exc_context: + self.loop.run_until_complete(future) + self.assertIn("Fail!", exc_context.exception.args) + + def test_run_coroutine_threadsafe_with_timeout(self): + """Test coroutine submission from a thread to an event loop + when a timeout is raised.""" + callback = lambda: self.target(timeout=0) + future = self.loop.run_in_executor(None, callback) + with self.assertRaises(asyncio.TimeoutError): + self.loop.run_until_complete(future) + test_utils.run_briefly(self.loop) + # Check that there's no pending task (add has been cancelled) + for task in asyncio.all_tasks(self.loop): + self.assertTrue(task.done()) + + def test_run_coroutine_threadsafe_task_cancelled(self): + """Test coroutine submission from a thread to an event loop + when the task is cancelled.""" + callback = lambda: self.target(cancel=True) + future = self.loop.run_in_executor(None, callback) + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(future) + + def test_run_coroutine_threadsafe_task_factory_exception(self): + """Test coroutine submission from a thread to an event loop + when the task factory raise an exception.""" + + def task_factory(loop, coro): + raise NameError + + run = self.loop.run_in_executor( + None, lambda: self.target(advance_coro=True)) + + # Set exception handler + callback = test_utils.MockCallback() + self.loop.set_exception_handler(callback) + + # Set corrupted task factory + self.addCleanup(self.loop.set_task_factory, + self.loop.get_task_factory()) + self.loop.set_task_factory(task_factory) + + # Run event loop + with self.assertRaises(NameError) as exc_context: + self.loop.run_until_complete(run) + + # Check exceptions + self.assertEqual(len(callback.call_args_list), 1) + (loop, context), kwargs = callback.call_args + self.assertEqual(context['exception'], exc_context.exception) + + +class SleepTests(test_utils.TestCase): + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + self.loop = None + super().tearDown() + + def test_sleep_zero(self): + result = 0 + + def inc_result(num): + nonlocal result + result += num + + async def coro(): + self.loop.call_soon(inc_result, 1) + self.assertEqual(result, 0) + num = await asyncio.sleep(0, result=10) + self.assertEqual(result, 1) # inc'ed by call_soon + inc_result(num) # num should be 11 + + self.loop.run_until_complete(coro()) + self.assertEqual(result, 11) + + +class CompatibilityTests(test_utils.TestCase): + # Tests for checking a bridge between old-styled coroutines + # and async/await syntax + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + self.loop = None + super().tearDown() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py new file mode 100644 index 00000000000..8ad5f9b2c9e --- /dev/null +++ b/Lib/test/test_asyncio/test_threads.py @@ -0,0 +1,66 @@ +"""Tests for asyncio/threads.py""" + +import asyncio +import unittest + +from contextvars import ContextVar +from unittest import mock + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class ToThreadTests(unittest.IsolatedAsyncioTestCase): + async def test_to_thread(self): + result = await asyncio.to_thread(sum, [40, 2]) + self.assertEqual(result, 42) + + async def test_to_thread_exception(self): + def raise_runtime(): + raise RuntimeError("test") + + with self.assertRaisesRegex(RuntimeError, "test"): + await asyncio.to_thread(raise_runtime) + + async def test_to_thread_once(self): + func = mock.Mock() + + await asyncio.to_thread(func) + func.assert_called_once() + + async def test_to_thread_concurrent(self): + calls = [] + def func(): + calls.append(1) + + futs = [] + for _ in range(10): + fut = asyncio.to_thread(func) + futs.append(fut) + await asyncio.gather(*futs) + + self.assertEqual(sum(calls), 10) + + async def test_to_thread_args_kwargs(self): + # Unlike run_in_executor(), to_thread() should directly accept kwargs. + func = mock.Mock() + + await asyncio.to_thread(func, 'test', something=True) + + func.assert_called_once_with('test', something=True) + + async def test_to_thread_contextvars(self): + test_ctx = ContextVar('test_ctx') + + def get_ctx(): + return test_ctx.get() + + test_ctx.set('parrot') + result = await asyncio.to_thread(get_ctx) + + self.assertEqual(result, 'parrot') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py new file mode 100644 index 00000000000..f60722c48b7 --- /dev/null +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -0,0 +1,411 @@ +"""Tests for asyncio/timeouts.py""" + +import unittest +import time + +import asyncio + +from test.test_asyncio.utils import await_without_task + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + +class TimeoutTests(unittest.IsolatedAsyncioTestCase): + + async def test_timeout_basic(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01) as cm: + await asyncio.sleep(10) + self.assertTrue(cm.expired()) + + async def test_timeout_at_basic(self): + loop = asyncio.get_running_loop() + + with self.assertRaises(TimeoutError): + deadline = loop.time() + 0.01 + async with asyncio.timeout_at(deadline) as cm: + await asyncio.sleep(10) + self.assertTrue(cm.expired()) + self.assertEqual(deadline, cm.when()) + + async def test_nested_timeouts(self): + loop = asyncio.get_running_loop() + cancelled = False + with self.assertRaises(TimeoutError): + deadline = loop.time() + 0.01 + async with asyncio.timeout_at(deadline) as cm1: + # Only the topmost context manager should raise TimeoutError + try: + async with asyncio.timeout_at(deadline) as cm2: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelled = True + raise + self.assertTrue(cancelled) + self.assertTrue(cm1.expired()) + self.assertTrue(cm2.expired()) + + async def test_waiter_cancelled(self): + cancelled = False + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + cancelled = True + raise + self.assertTrue(cancelled) + + async def test_timeout_not_called(self): + loop = asyncio.get_running_loop() + async with asyncio.timeout(10) as cm: + await asyncio.sleep(0.01) + t1 = loop.time() + + self.assertFalse(cm.expired()) + self.assertGreater(cm.when(), t1) + + async def test_timeout_disabled(self): + async with asyncio.timeout(None) as cm: + await asyncio.sleep(0.01) + + self.assertFalse(cm.expired()) + self.assertIsNone(cm.when()) + + async def test_timeout_at_disabled(self): + async with asyncio.timeout_at(None) as cm: + await asyncio.sleep(0.01) + + self.assertFalse(cm.expired()) + self.assertIsNone(cm.when()) + + async def test_timeout_zero(self): + loop = asyncio.get_running_loop() + t0 = loop.time() + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0) as cm: + await asyncio.sleep(10) + t1 = loop.time() + self.assertTrue(cm.expired()) + self.assertTrue(t0 <= cm.when() <= t1) + + async def test_timeout_zero_sleep_zero(self): + loop = asyncio.get_running_loop() + t0 = loop.time() + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0) as cm: + await asyncio.sleep(0) + t1 = loop.time() + self.assertTrue(cm.expired()) + self.assertTrue(t0 <= cm.when() <= t1) + + async def test_timeout_in_the_past_sleep_zero(self): + loop = asyncio.get_running_loop() + t0 = loop.time() + with self.assertRaises(TimeoutError): + async with asyncio.timeout(-11) as cm: + await asyncio.sleep(0) + t1 = loop.time() + self.assertTrue(cm.expired()) + self.assertTrue(t0 >= cm.when() <= t1) + + async def test_foreign_exception_passed(self): + with self.assertRaises(KeyError): + async with asyncio.timeout(0.01) as cm: + raise KeyError + self.assertFalse(cm.expired()) + + async def test_timeout_exception_context(self): + with self.assertRaises(TimeoutError) as cm: + async with asyncio.timeout(0.01): + try: + 1/0 + finally: + await asyncio.sleep(1) + e = cm.exception + # Expect TimeoutError caused by CancelledError raised during handling + # of ZeroDivisionError. + e2 = e.__cause__ + self.assertIsInstance(e2, asyncio.CancelledError) + self.assertIs(e.__context__, e2) + self.assertIsNone(e2.__cause__) + self.assertIsInstance(e2.__context__, ZeroDivisionError) + + async def test_foreign_exception_on_timeout(self): + async def crash(): + try: + await asyncio.sleep(1) + finally: + 1/0 + with self.assertRaises(ZeroDivisionError) as cm: + async with asyncio.timeout(0.01): + await crash() + e = cm.exception + # Expect ZeroDivisionError raised during handling of TimeoutError + # caused by CancelledError. + self.assertIsNone(e.__cause__) + e2 = e.__context__ + self.assertIsInstance(e2, TimeoutError) + e3 = e2.__cause__ + self.assertIsInstance(e3, asyncio.CancelledError) + self.assertIs(e2.__context__, e3) + + async def test_foreign_exception_on_timeout_2(self): + with self.assertRaises(ZeroDivisionError) as cm: + async with asyncio.timeout(0.01): + try: + try: + raise ValueError + finally: + await asyncio.sleep(1) + finally: + try: + raise KeyError + finally: + 1/0 + e = cm.exception + # Expect ZeroDivisionError raised during handling of KeyError + # raised during handling of TimeoutError caused by CancelledError. + self.assertIsNone(e.__cause__) + e2 = e.__context__ + self.assertIsInstance(e2, KeyError) + self.assertIsNone(e2.__cause__) + e3 = e2.__context__ + self.assertIsInstance(e3, TimeoutError) + e4 = e3.__cause__ + self.assertIsInstance(e4, asyncio.CancelledError) + self.assertIsNone(e4.__cause__) + self.assertIsInstance(e4.__context__, ValueError) + self.assertIs(e3.__context__, e4) + + async def test_foreign_cancel_doesnt_timeout_if_not_expired(self): + with self.assertRaises(asyncio.CancelledError): + async with asyncio.timeout(10) as cm: + asyncio.current_task().cancel() + await asyncio.sleep(10) + self.assertFalse(cm.expired()) + + async def test_outer_task_is_not_cancelled(self): + async def outer() -> None: + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.001): + await asyncio.sleep(10) + + task = asyncio.create_task(outer()) + await task + self.assertFalse(task.cancelled()) + self.assertTrue(task.done()) + + async def test_nested_timeouts_concurrent(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.002): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.1): + # Pretend we crunch some numbers. + time.sleep(0.01) + await asyncio.sleep(1) + + async def test_nested_timeouts_loop_busy(self): + # After the inner timeout is an expensive operation which should + # be stopped by the outer timeout. + loop = asyncio.get_running_loop() + # Disable a message about long running task + loop.slow_callback_duration = 10 + t0 = loop.time() + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.1): # (1) + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01): # (2) + # Pretend the loop is busy for a while. + time.sleep(0.1) + await asyncio.sleep(1) + # TimeoutError was caught by (2) + await asyncio.sleep(10) # This sleep should be interrupted by (1) + t1 = loop.time() + self.assertTrue(t0 <= t1 <= t0 + 1) + + async def test_reschedule(self): + loop = asyncio.get_running_loop() + fut = loop.create_future() + deadline1 = loop.time() + 10 + deadline2 = deadline1 + 20 + + async def f(): + async with asyncio.timeout_at(deadline1) as cm: + fut.set_result(cm) + await asyncio.sleep(50) + + task = asyncio.create_task(f()) + cm = await fut + + self.assertEqual(cm.when(), deadline1) + cm.reschedule(deadline2) + self.assertEqual(cm.when(), deadline2) + cm.reschedule(None) + self.assertIsNone(cm.when()) + + task.cancel() + + with self.assertRaises(asyncio.CancelledError): + await task + self.assertFalse(cm.expired()) + + async def test_repr_active(self): + async with asyncio.timeout(10) as cm: + self.assertRegex(repr(cm), r"") + + async def test_repr_expired(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01) as cm: + await asyncio.sleep(10) + self.assertEqual(repr(cm), "") + + async def test_repr_finished(self): + async with asyncio.timeout(10) as cm: + await asyncio.sleep(0) + + self.assertEqual(repr(cm), "") + + async def test_repr_disabled(self): + async with asyncio.timeout(None) as cm: + self.assertEqual(repr(cm), r"") + + async def test_nested_timeout_in_finally(self): + with self.assertRaises(TimeoutError) as cm1: + async with asyncio.timeout(0.01): + try: + await asyncio.sleep(1) + finally: + with self.assertRaises(TimeoutError) as cm2: + async with asyncio.timeout(0.01): + await asyncio.sleep(10) + e1 = cm1.exception + # Expect TimeoutError caused by CancelledError. + e12 = e1.__cause__ + self.assertIsInstance(e12, asyncio.CancelledError) + self.assertIsNone(e12.__cause__) + self.assertIsNone(e12.__context__) + self.assertIs(e1.__context__, e12) + e2 = cm2.exception + # Expect TimeoutError caused by CancelledError raised during + # handling of other CancelledError (which is the same as in + # the above chain). + e22 = e2.__cause__ + self.assertIsInstance(e22, asyncio.CancelledError) + self.assertIsNone(e22.__cause__) + self.assertIs(e22.__context__, e12) + self.assertIs(e2.__context__, e22) + + async def test_timeout_after_cancellation(self): + try: + asyncio.current_task().cancel() + await asyncio.sleep(1) # work which will be cancelled + except asyncio.CancelledError: + pass + finally: + with self.assertRaises(TimeoutError) as cm: + async with asyncio.timeout(0.0): + await asyncio.sleep(1) # some cleanup + + async def test_cancel_in_timeout_after_cancellation(self): + try: + asyncio.current_task().cancel() + await asyncio.sleep(1) # work which will be cancelled + except asyncio.CancelledError: + pass + finally: + with self.assertRaises(asyncio.CancelledError): + async with asyncio.timeout(1.0): + asyncio.current_task().cancel() + await asyncio.sleep(2) # some cleanup + + async def test_timeout_already_entered(self): + async with asyncio.timeout(0.01) as cm: + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with cm: + pass + + async def test_timeout_double_enter(self): + async with asyncio.timeout(0.01) as cm: + pass + with self.assertRaisesRegex(RuntimeError, "has already been entered"): + async with cm: + pass + + async def test_timeout_finished(self): + async with asyncio.timeout(0.01) as cm: + pass + with self.assertRaisesRegex(RuntimeError, "finished"): + cm.reschedule(0.02) + + async def test_timeout_expired(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01) as cm: + await asyncio.sleep(1) + with self.assertRaisesRegex(RuntimeError, "expired"): + cm.reschedule(0.02) + + async def test_timeout_expiring(self): + async with asyncio.timeout(0.01) as cm: + with self.assertRaises(asyncio.CancelledError): + await asyncio.sleep(1) + with self.assertRaisesRegex(RuntimeError, "expiring"): + cm.reschedule(0.02) + + async def test_timeout_not_entered(self): + cm = asyncio.timeout(0.01) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + cm.reschedule(0.02) + + async def test_timeout_without_task(self): + cm = asyncio.timeout(0.01) + with self.assertRaisesRegex(RuntimeError, "task"): + await await_without_task(cm.__aenter__()) + with self.assertRaisesRegex(RuntimeError, "has not been entered"): + cm.reschedule(0.02) + + async def test_timeout_taskgroup(self): + async def task(): + try: + await asyncio.sleep(2) # Will be interrupted after 0.01 second + finally: + 1/0 # Crash in cleanup + + with self.assertRaises(ExceptionGroup) as cm: + async with asyncio.timeout(0.01): + async with asyncio.TaskGroup() as tg: + tg.create_task(task()) + try: + raise ValueError + finally: + await asyncio.sleep(1) + eg = cm.exception + # Expect ExceptionGroup raised during handling of TimeoutError caused + # by CancelledError raised during handling of ValueError. + self.assertIsNone(eg.__cause__) + e_1 = eg.__context__ + self.assertIsInstance(e_1, TimeoutError) + e_2 = e_1.__cause__ + self.assertIsInstance(e_2, asyncio.CancelledError) + self.assertIsNone(e_2.__cause__) + self.assertIsInstance(e_2.__context__, ValueError) + self.assertIs(e_1.__context__, e_2) + + self.assertEqual(len(eg.exceptions), 1, eg) + e1 = eg.exceptions[0] + # Expect ZeroDivisionError raised during handling of TimeoutError + # caused by CancelledError (it is a different CancelledError). + self.assertIsInstance(e1, ZeroDivisionError) + self.assertIsNone(e1.__cause__) + e2 = e1.__context__ + self.assertIsInstance(e2, TimeoutError) + e3 = e2.__cause__ + self.assertIsInstance(e3, asyncio.CancelledError) + self.assertIsNone(e3.__context__) + self.assertIsNone(e3.__cause__) + self.assertIs(e2.__context__, e3) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py new file mode 100644 index 00000000000..34e94830204 --- /dev/null +++ b/Lib/test/test_asyncio/test_tools.py @@ -0,0 +1,1706 @@ +import unittest + +from asyncio import tools + +from collections import namedtuple + +FrameInfo = namedtuple('FrameInfo', ['funcname', 'filename', 'lineno']) +CoroInfo = namedtuple('CoroInfo', ['call_stack', 'task_name']) +TaskInfo = namedtuple('TaskInfo', ['task_id', 'task_name', 'coroutine_stack', 'awaited_by']) +AwaitedInfo = namedtuple('AwaitedInfo', ['thread_id', 'awaited_by']) + + +# mock output of get_all_awaited_by function. +TEST_INPUTS_TREE = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [ + "└── (T) Task-1", + " └── main", + " └── __aexit__", + " └── _aexit", + " ├── (T) root1", + " │ └── bloch", + " │ └── blocho_caller", + " │ └── __aexit__", + " │ └── _aexit", + " │ ├── (T) child1_1", + " │ │ └── awaiter /path/to/app.py:110", + " │ │ └── awaiter2 /path/to/app.py:120", + " │ │ └── awaiter3 /path/to/app.py:130", + " │ │ └── (T) timer", + " │ └── (T) child2_1", + " │ └── awaiterB /path/to/app.py:170", + " │ └── awaiterB2 /path/to/app.py:180", + " │ └── awaiterB3 /path/to/app.py:190", + " │ └── (T) timer", + " └── (T) root2", + " └── bloch", + " └── blocho_caller", + " └── __aexit__", + " └── _aexit", + " ├── (T) child1_2", + " │ └── awaiter /path/to/app.py:110", + " │ └── awaiter2 /path/to/app.py:120", + " │ └── awaiter3 /path/to/app.py:130", + " │ └── (T) timer", + " └── (T) child2_2", + " └── awaiterB /path/to/app.py:170", + " └── awaiterB2 /path/to/app.py:180", + " └── awaiterB3 /path/to/app.py:190", + " └── (T) timer", + ] + ] + ), + ], + [ + # test case containing two roots + ( + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] + ), + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [ + "└── (T) Task-5", + " └── main2", + " ├── (T) Task-6", + " ├── (T) Task-7", + " └── (T) Task-8", + ], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] + ), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + ["└── (T) Task-5"], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], +] + +TEST_INPUTS_CYCLES_TREE = [ + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ([[4, 3, 4]]), + ], + [ + # this test case contains two cycles + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ([[4, 3, 4], [4, 6, 5, 4]]), + ], +] + +TEST_INPUTS_TABLE = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [ + 1, + "0x3", + "timer", + "", + "awaiter3 -> awaiter2 -> awaiter", + "child1_1", + "0x4", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_2", + "0x5", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_1", + "0x6", + ], + [ + 1, + "0x3", + "timer", + "", + "awaiter3 -> awaiter2 -> awaiter", + "child1_2", + "0x7", + ], + [ + 1, + "0x8", + "root1", + "", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x9", + "root2", + "", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x4", + "child1_1", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x6", + "child2_1", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x7", + "child1_2", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + [ + 1, + "0x5", + "child2_2", + "", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + ] + ), + ], + [ + # test case containing two roots + ( + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] + ), + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ), + ( + [ + [9, "0x5", "Task-5", "", "", "", "0x0"], + [9, "0x6", "Task-6", "", "main2", "Task-5", "0x5"], + [9, "0x7", "Task-7", "", "main2", "Task-5", "0x5"], + [9, "0x8", "Task-8", "", "main2", "Task-5", "0x5"], + [10, "0x1", "Task-1", "", "", "", "0x0"], + [10, "0x2", "Task-2", "", "main", "Task-1", "0x1"], + [10, "0x3", "Task-3", "", "main", "Task-1", "0x1"], + [10, "0x4", "Task-4", "", "main", "Task-1", "0x1"], + ] + ), + ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] + ), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-5", "", "", "", "0x0"], + [3, "0x4", "Task-1", "", "", "", "0x0"], + [3, "0x5", "Task-2", "", "main", "Task-1", "0x4"], + [3, "0x6", "Task-3", "", "main", "Task-1", "0x4"], + [3, "0x7", "Task-4", "", "main", "Task-1", "0x4"], + ] + ), + ], + # CASES WITH CYCLES + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [1, "0x3", "a", "", "awaiter2", "b", "0x4"], + [1, "0x3", "a", "", "main", "Task-1", "0x2"], + [1, "0x4", "b", "", "awaiter", "a", "0x3"], + ] + ), + ], + [ + # this test case contains two cycles + ( + [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=0, awaited_by=[]) + ] + ), + ( + [ + [1, "0x2", "Task-1", "", "", "", "0x0"], + [ + 1, + "0x3", + "A", + "", + "nested -> nested -> task_b", + "B", + "0x4", + ], + [ + 1, + "0x4", + "B", + "", + "nested -> nested -> task_c", + "C", + "0x5", + ], + [ + 1, + "0x4", + "B", + "", + "nested -> nested -> task_a", + "A", + "0x3", + ], + [ + 1, + "0x5", + "C", + "", + "nested -> nested", + "Task-2", + "0x6", + ], + [ + 1, + "0x6", + "Task-2", + "", + "nested -> nested -> task_b", + "B", + "0x4", + ], + ] + ), + ], +] + + +class TestAsyncioToolsTree(unittest.TestCase): + def test_asyncio_utils(self): + for input_, tree in TEST_INPUTS_TREE: + with self.subTest(input_): + result = tools.build_async_tree(input_) + self.assertEqual(result, tree) + + def test_asyncio_utils_cycles(self): + for input_, cycles in TEST_INPUTS_CYCLES_TREE: + with self.subTest(input_): + try: + tools.build_async_tree(input_) + except tools.CycleFoundException as e: + self.assertEqual(e.cycles, cycles) + + +class TestAsyncioToolsTable(unittest.TestCase): + def test_asyncio_utils(self): + for input_, table in TEST_INPUTS_TABLE: + with self.subTest(input_): + result = tools.build_task_table(input_) + self.assertEqual(result, table) + + +class TestAsyncioToolsBasic(unittest.TestCase): + def test_empty_input_tree(self): + """Test build_async_tree with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_empty_input_table(self): + """Test build_task_table with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_only_independent_tasks_tree(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected = [["└── (T) taskA"], ["└── (T) taskB"]] + result = tools.build_async_tree(input_) + self.assertEqual(sorted(result), sorted(expected)) + + def test_only_independent_tasks_table(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + self.assertEqual( + tools.build_task_table(input_), + [[1, '0xa', 'taskA', '', '', '', '0x0'], [1, '0xb', 'taskB', '', '', '', '0x0']] + ) + + def test_single_task_tree(self): + """Test build_async_tree with a single task and no awaits.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + ] + ] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_single_task_table(self): + """Test build_task_table with a single task and no awaits.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected_output = [[1, '0x2', 'Task-1', '', '', '', '0x0']] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_cycle_detection(self): + """Test build_async_tree raises CycleFoundException for cyclic input.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as context: + tools.build_async_tree(result) + self.assertEqual(context.exception.cycles, [[3, 2, 3]]) + + def test_complex_tree(self): + """Test build_async_tree with a more complex tree structure.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + " └── main", + " └── (T) Task-2", + " └── main", + " └── (T) Task-3", + ] + ] + self.assertEqual(tools.build_async_tree(result), expected_output) + + def test_complex_table(self): + """Test build_task_table with a more complex tree structure.""" + result = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] + ) + ] + expected_output = [ + [1, '0x2', 'Task-1', '', '', '', '0x0'], + [1, '0x3', 'Task-2', '', 'main', 'Task-1', '0x2'], + [1, '0x4', 'Task-3', '', 'main', 'Task-2', '0x3'] + ] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_deep_coroutine_chain(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="leaf", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("c1", "", 0), + FrameInfo("c2", "", 0), + FrameInfo("c3", "", 0), + FrameInfo("c4", "", 0), + FrameInfo("c5", "", 0) + ], + task_name=11 + ) + ] + ), + TaskInfo( + task_id=11, + task_name="root", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + expected = [ + [ + "└── (T) root", + " └── c5", + " └── c4", + " └── c3", + " └── c2", + " └── c1", + " └── (T) leaf", + ] + ] + result = tools.build_async_tree(input_) + self.assertEqual(result, expected) + + def test_multiple_cycles_same_node(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call2", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call3", "", 0)], + task_name=1 + ), + CoroInfo( + call_stack=[FrameInfo("call4", "", 0)], + task_name=2 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.build_async_tree(input_) + cycles = ctx.exception.cycles + self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) + + def test_table_output_format(self): + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("foo", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + table = tools.build_task_table(input_) + for row in table: + self.assertEqual(len(row), 7) + self.assertIsInstance(row[0], int) # thread ID + self.assertTrue( + isinstance(row[1], str) and row[1].startswith("0x") + ) # hex task ID + self.assertIsInstance(row[2], str) # task name + self.assertIsInstance(row[3], str) # coroutine stack + self.assertIsInstance(row[4], str) # coroutine chain + self.assertIsInstance(row[5], str) # awaiter name + self.assertTrue( + isinstance(row[6], str) and row[6].startswith("0x") + ) # hex awaiter ID + + +class TestAsyncioToolsEdgeCases(unittest.TestCase): + + def test_task_awaits_self(self): + """A task directly awaits itself - should raise a cycle.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Self-Awaiter", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("loopback", "", 0)], + task_name=1 + ) + ] + ) + ] + ) + ] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.build_async_tree(input_) + self.assertIn([1, 1], ctx.exception.cycles) + + def test_task_with_missing_awaiter_id(self): + """Awaiter ID not in task list - should not crash, just show 'Unknown'.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("coro", "", 0)], + task_name=999 + ) + ] + ) + ] + ) + ] + table = tools.build_task_table(input_) + self.assertEqual(len(table), 1) + self.assertEqual(table[0][5], "Unknown") + + def test_duplicate_coroutine_frames(self): + """Same coroutine frame repeated under a parent - should deduplicate.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=2 + ), + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + tree = tools.build_async_tree(input_) + # Both children should be under the same coroutine node + flat = "\n".join(tree[0]) + self.assertIn("frameA", flat) + self.assertIn("Task-2", flat) + self.assertIn("Task-1", flat) + + flat = "\n".join(tree[1]) + self.assertIn("frameA", flat) + self.assertIn("Task-3", flat) + self.assertIn("Task-1", flat) + + def test_task_with_no_name(self): + """Task with no name in id2name - should still render with fallback.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="root", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("f1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name=None, + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + # If name is None, fallback to string should not crash + tree = tools.build_async_tree(input_) + self.assertIn("(T) None", "\n".join(tree[0])) + + def test_tree_rendering_with_custom_emojis(self): + """Pass custom emojis to the tree renderer.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="MainTask", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("f1", "", 0), + FrameInfo("f2", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="SubTask", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] + tree = tools.build_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") + flat = "\n".join(tree[0]) + self.assertIn("🧵 MainTask", flat) + self.assertIn("🔁 f1", flat) + self.assertIn("🔁 f2", flat) + self.assertIn("🧵 SubTask", flat) diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py new file mode 100644 index 00000000000..dbb572e2e15 --- /dev/null +++ b/Lib/test/test_asyncio/test_transports.py @@ -0,0 +1,103 @@ +"""Tests for transports.py.""" + +import unittest +from unittest import mock + +import asyncio +from asyncio import transports + + +def tearDownModule(): + # not needed for the test file but added for uniformness with all other + # asyncio test files for the sake of unified cleanup + asyncio.events._set_event_loop_policy(None) + + +class TransportTests(unittest.TestCase): + + def test_ctor_extra_is_none(self): + transport = asyncio.Transport() + self.assertEqual(transport._extra, {}) + + def test_get_extra_info(self): + transport = asyncio.Transport({'extra': 'info'}) + self.assertEqual('info', transport.get_extra_info('extra')) + self.assertIsNone(transport.get_extra_info('unknown')) + + default = object() + self.assertIs(default, transport.get_extra_info('unknown', default)) + + def test_writelines(self): + writer = mock.Mock() + + class MyTransport(asyncio.Transport): + def write(self, data): + writer(data) + + transport = MyTransport() + + transport.writelines([b'line1', + bytearray(b'line2'), + memoryview(b'line3')]) + self.assertEqual(1, writer.call_count) + writer.assert_called_with(b'line1line2line3') + + def test_not_implemented(self): + transport = asyncio.Transport() + + self.assertRaises(NotImplementedError, + transport.set_write_buffer_limits) + self.assertRaises(NotImplementedError, transport.get_write_buffer_size) + self.assertRaises(NotImplementedError, transport.write, 'data') + self.assertRaises(NotImplementedError, transport.write_eof) + self.assertRaises(NotImplementedError, transport.can_write_eof) + self.assertRaises(NotImplementedError, transport.pause_reading) + self.assertRaises(NotImplementedError, transport.resume_reading) + self.assertRaises(NotImplementedError, transport.is_reading) + self.assertRaises(NotImplementedError, transport.close) + self.assertRaises(NotImplementedError, transport.abort) + + def test_dgram_not_implemented(self): + transport = asyncio.DatagramTransport() + + self.assertRaises(NotImplementedError, transport.sendto, 'data') + self.assertRaises(NotImplementedError, transport.abort) + + def test_subprocess_transport_not_implemented(self): + transport = asyncio.SubprocessTransport() + + self.assertRaises(NotImplementedError, transport.get_pid) + self.assertRaises(NotImplementedError, transport.get_returncode) + self.assertRaises(NotImplementedError, transport.get_pipe_transport, 1) + self.assertRaises(NotImplementedError, transport.send_signal, 1) + self.assertRaises(NotImplementedError, transport.terminate) + self.assertRaises(NotImplementedError, transport.kill) + + def test_flowcontrol_mixin_set_write_limits(self): + + class MyTransport(transports._FlowControlMixin, + transports.Transport): + + def get_write_buffer_size(self): + return 512 + + loop = mock.Mock() + transport = MyTransport(loop=loop) + transport._protocol = mock.Mock() + + self.assertFalse(transport._protocol_paused) + + with self.assertRaisesRegex(ValueError, 'high.*must be >= low'): + transport.set_write_buffer_limits(high=0, low=1) + + transport.set_write_buffer_limits(high=1024, low=128) + self.assertFalse(transport._protocol_paused) + self.assertEqual(transport.get_write_buffer_limits(), (128, 1024)) + + transport.set_write_buffer_limits(high=256, low=128) + self.assertTrue(transport._protocol_paused) + self.assertEqual(transport.get_write_buffer_limits(), (128, 256)) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py new file mode 100644 index 00000000000..520f5c733c3 --- /dev/null +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -0,0 +1,1332 @@ +"""Tests for unix_events.py.""" + +import contextlib +import errno +import io +import multiprocessing +from multiprocessing.util import _cleanup_tests as multiprocessing_cleanup_tests +import os +import signal +import socket +import stat +import sys +import time +import unittest +from unittest import mock + +from test import support +from test.support import os_helper +from test.support import socket_helper +from test.support import wait_process +from test.support import hashlib_helper + +if sys.platform == 'win32': + raise unittest.SkipTest('UNIX only') + + +import asyncio +from asyncio import unix_events +from test.test_asyncio import utils as test_utils + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +MOCK_ANY = mock.ANY + + +def EXITCODE(exitcode): + return 32768 + exitcode + + +def SIGNAL(signum): + if not 1 <= signum <= 68: + raise AssertionError(f'invalid signum {signum}') + return 32768 - signum + + +def close_pipe_transport(transport): + # Don't call transport.close() because the event loop and the selector + # are mocked + if transport._pipe is None: + return + transport._pipe.close() + transport._pipe = None + + +@unittest.skipUnless(signal, 'Signals are not supported') +class SelectorEventLoopSignalTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.SelectorEventLoop() + self.set_event_loop(self.loop) + + def test_check_signal(self): + self.assertRaises( + TypeError, self.loop._check_signal, '1') + self.assertRaises( + ValueError, self.loop._check_signal, signal.NSIG + 1) + + def test_handle_signal_no_handler(self): + self.loop._handle_signal(signal.NSIG + 1) + + def test_handle_signal_cancelled_handler(self): + h = asyncio.Handle(mock.Mock(), (), + loop=mock.Mock()) + h.cancel() + self.loop._signal_handlers[signal.NSIG + 1] = h + self.loop.remove_signal_handler = mock.Mock() + self.loop._handle_signal(signal.NSIG + 1) + self.loop.remove_signal_handler.assert_called_with(signal.NSIG + 1) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_setup_error(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + m_signal.set_wakeup_fd.side_effect = ValueError + + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_coroutine_error(self, m_signal): + m_signal.NSIG = signal.NSIG + + async def simple_coroutine(): + pass + + # callback must not be a coroutine function + coro_func = simple_coroutine + coro_obj = coro_func() + self.addCleanup(coro_obj.close) + for func in (coro_func, coro_obj): + self.assertRaisesRegex( + TypeError, 'coroutines cannot be used with add_signal_handler', + self.loop.add_signal_handler, + signal.SIGINT, func) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + cb = lambda: True + self.loop.add_signal_handler(signal.SIGHUP, cb) + h = self.loop._signal_handlers.get(signal.SIGHUP) + self.assertIsInstance(h, asyncio.Handle) + self.assertEqual(h._callback, cb) + + @mock.patch('asyncio.unix_events.signal') + def test_add_signal_handler_install_error(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + def set_wakeup_fd(fd): + if fd == -1: + raise ValueError() + m_signal.set_wakeup_fd = set_wakeup_fd + + class Err(OSError): + errno = errno.EFAULT + m_signal.signal.side_effect = Err + + self.assertRaises( + Err, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_add_signal_handler_install_error2(self, m_logging, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + + self.loop._signal_handlers[signal.SIGHUP] = lambda: True + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + self.assertFalse(m_logging.info.called) + self.assertEqual(1, m_signal.set_wakeup_fd.call_count) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_add_signal_handler_install_error3(self, m_logging, m_signal): + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + self.assertRaises( + RuntimeError, + self.loop.add_signal_handler, + signal.SIGINT, lambda: True) + self.assertFalse(m_logging.info.called) + self.assertEqual(2, m_signal.set_wakeup_fd.call_count) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + self.assertTrue( + self.loop.remove_signal_handler(signal.SIGHUP)) + self.assertTrue(m_signal.set_wakeup_fd.called) + self.assertTrue(m_signal.signal.called) + self.assertEqual( + (signal.SIGHUP, m_signal.SIG_DFL), m_signal.signal.call_args[0]) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_2(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.SIGINT = signal.SIGINT + m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGINT, lambda: True) + self.loop._signal_handlers[signal.SIGHUP] = object() + m_signal.set_wakeup_fd.reset_mock() + + self.assertTrue( + self.loop.remove_signal_handler(signal.SIGINT)) + self.assertFalse(m_signal.set_wakeup_fd.called) + self.assertTrue(m_signal.signal.called) + self.assertEqual( + (signal.SIGINT, m_signal.default_int_handler), + m_signal.signal.call_args[0]) + + @mock.patch('asyncio.unix_events.signal') + @mock.patch('asyncio.base_events.logger') + def test_remove_signal_handler_cleanup_error(self, m_logging, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + m_signal.set_wakeup_fd.side_effect = ValueError + + self.loop.remove_signal_handler(signal.SIGHUP) + self.assertTrue(m_logging.info) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_error(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + m_signal.signal.side_effect = OSError + + self.assertRaises( + OSError, self.loop.remove_signal_handler, signal.SIGHUP) + + @mock.patch('asyncio.unix_events.signal') + def test_remove_signal_handler_error2(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + class Err(OSError): + errno = errno.EINVAL + m_signal.signal.side_effect = Err + + self.assertRaises( + RuntimeError, self.loop.remove_signal_handler, signal.SIGHUP) + + @mock.patch('asyncio.unix_events.signal') + def test_close(self, m_signal): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + self.loop.add_signal_handler(signal.SIGCHLD, lambda: True) + + self.assertEqual(len(self.loop._signal_handlers), 2) + + m_signal.set_wakeup_fd.reset_mock() + + self.loop.close() + + self.assertEqual(len(self.loop._signal_handlers), 0) + m_signal.set_wakeup_fd.assert_called_once_with(-1) + + @mock.patch('asyncio.unix_events.sys') + @mock.patch('asyncio.unix_events.signal') + def test_close_on_finalizing(self, m_signal, m_sys): + m_signal.NSIG = signal.NSIG + m_signal.valid_signals = signal.valid_signals + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + self.assertEqual(len(self.loop._signal_handlers), 1) + m_sys.is_finalizing.return_value = True + m_signal.signal.reset_mock() + + with self.assertWarnsRegex(ResourceWarning, + "skipping signal handlers removal"): + self.loop.close() + + self.assertEqual(len(self.loop._signal_handlers), 0) + self.assertFalse(m_signal.signal.called) + + +@unittest.skipUnless(hasattr(socket, 'AF_UNIX'), + 'UNIX Sockets are not supported') +class SelectorEventLoopUnixSocketTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.SelectorEventLoop() + self.set_event_loop(self.loop) + + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_server_existing_path_sock(self): + with test_utils.unix_socket_path() as path: + sock = socket.socket(socket.AF_UNIX) + sock.bind(path) + sock.listen(1) + sock.close() + + coro = self.loop.create_unix_server(lambda: None, path) + srv = self.loop.run_until_complete(coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_server_pathlike(self): + with test_utils.unix_socket_path() as path: + path = os_helper.FakePath(path) + srv_coro = self.loop.create_unix_server(lambda: None, path) + srv = self.loop.run_until_complete(srv_coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + def test_create_unix_connection_pathlike(self): + with test_utils.unix_socket_path() as path: + path = os_helper.FakePath(path) + coro = self.loop.create_unix_connection(lambda: None, path) + with self.assertRaises(FileNotFoundError): + # If path-like object weren't supported, the exception would be + # different. + self.loop.run_until_complete(coro) + + def test_create_unix_server_existing_path_nonsock(self): + path = test_utils.gen_unix_socket_path() + self.addCleanup(os_helper.unlink, path) + # create the file + open(path, "wb").close() + + coro = self.loop.create_unix_server(lambda: None, path) + with self.assertRaisesRegex(OSError, + 'Address.*is already in use'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_ssl_bool(self): + coro = self.loop.create_unix_server(lambda: None, path='spam', + ssl=True) + with self.assertRaisesRegex(TypeError, + 'ssl argument must be an SSLContext'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_nopath_nosock(self): + coro = self.loop.create_unix_server(lambda: None, path=None) + with self.assertRaisesRegex(ValueError, + 'path was not specified, and no sock'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_path_inetsock(self): + sock = socket.socket() + with sock: + coro = self.loop.create_unix_server(lambda: None, path=None, + sock=sock) + with self.assertRaisesRegex(ValueError, + 'A UNIX Domain Stream.*was expected'): + self.loop.run_until_complete(coro) + + def test_create_unix_server_path_dgram(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + with sock: + coro = self.loop.create_unix_server(lambda: None, path=None, + sock=sock) + with self.assertRaisesRegex(ValueError, + 'A UNIX Domain Stream.*was expected'): + self.loop.run_until_complete(coro) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'no socket.SOCK_NONBLOCK (linux only)') + @socket_helper.skip_unless_bind_unix_socket + def test_create_unix_server_path_stream_bittype(self): + fn = test_utils.gen_unix_socket_path() + self.addCleanup(os_helper.unlink, fn) + + sock = socket.socket(socket.AF_UNIX, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) + with sock: + sock.bind(fn) + coro = self.loop.create_unix_server(lambda: None, path=None, + sock=sock) + srv = self.loop.run_until_complete(coro) + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + def test_create_unix_server_ssl_timeout_with_plain_sock(self): + coro = self.loop.create_unix_server(lambda: None, path='spam', + ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_path_inetsock(self): + sock = socket.socket() + with sock: + coro = self.loop.create_unix_connection(lambda: None, + sock=sock) + with self.assertRaisesRegex(ValueError, + 'A UNIX Domain Stream.*was expected'): + self.loop.run_until_complete(coro) + + @mock.patch('asyncio.unix_events.socket') + def test_create_unix_server_bind_error(self, m_socket): + # Ensure that the socket is closed on any bind error + sock = mock.Mock() + m_socket.socket.return_value = sock + + sock.bind.side_effect = OSError + coro = self.loop.create_unix_server(lambda: None, path="/test") + with self.assertRaises(OSError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + sock.bind.side_effect = MemoryError + coro = self.loop.create_unix_server(lambda: None, path="/test") + with self.assertRaises(MemoryError): + self.loop.run_until_complete(coro) + self.assertTrue(sock.close.called) + + def test_create_unix_connection_path_sock(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, sock=object()) + with self.assertRaisesRegex(ValueError, 'path and sock can not be'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_nopath_nosock(self): + coro = self.loop.create_unix_connection( + lambda: None, None) + with self.assertRaisesRegex(ValueError, + 'no path and sock were specified'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_nossl_serverhost(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, server_hostname='spam') + with self.assertRaisesRegex(ValueError, + 'server_hostname is only meaningful'): + self.loop.run_until_complete(coro) + + def test_create_unix_connection_ssl_noserverhost(self): + coro = self.loop.create_unix_connection( + lambda: None, os.devnull, ssl=True) + + with self.assertRaisesRegex( + ValueError, 'you have to pass server_hostname when using ssl'): + + self.loop.run_until_complete(coro) + + def test_create_unix_connection_ssl_timeout_with_plain_sock(self): + coro = self.loop.create_unix_connection(lambda: None, path='spam', + ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + +@unittest.skipUnless(hasattr(os, 'sendfile'), + 'sendfile is not supported') +class SelectorEventLoopUnixSockSendfileTests(test_utils.TestCase): + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + + class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + self._ready = loop.create_future() + + def connection_made(self, transport): + self.started = True + self.transport = transport + self._ready.set_result(None) + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + + async def wait_closed(self): + await self.fut + + @classmethod + def setUpClass(cls): + with open(os_helper.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + os_helper.unlink(os_helper.TESTFN) + super().tearDownClass() + + def setUp(self): + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + self.file = open(os_helper.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def make_socket(self, cleanup=True): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(False) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) + if cleanup: + self.addCleanup(sock.close) + return sock + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self): + sock = self.make_socket() + proto = self.MyProto(self.loop) + port = socket_helper.find_unused_port() + srv_sock = self.make_socket(cleanup=False) + srv_sock.bind((socket_helper.HOST, port)) + server = self.run_loop(self.loop.create_server( + lambda: proto, sock=srv_sock)) + self.run_loop(self.loop.sock_connect(sock, (socket_helper.HOST, port))) + self.run_loop(proto._ready) + + def cleanup(): + proto.transport.close() + self.run_loop(proto.wait_closed()) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test_sock_sendfile_not_available(self): + sock, proto = self.prepare() + with mock.patch('asyncio.unix_events.os', spec=[]): + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "os[.]sendfile[(][)] is not available"): + self.run_loop(self.loop._sock_sendfile_native(sock, self.file, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_not_a_file(self): + sock, proto = self.prepare() + f = object() + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_iobuffer(self): + sock, proto = self.prepare() + f = io.BytesIO() + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_not_regular_file(self): + sock, proto = self.prepare() + f = mock.Mock() + f.fileno.return_value = -1 + with self.assertRaisesRegex(asyncio.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_cancel1(self): + sock, proto = self.prepare() + + fut = self.loop.create_future() + fileno = self.file.fileno() + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + fut.cancel() + with contextlib.suppress(asyncio.CancelledError): + self.run_loop(fut) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + + def test_sock_sendfile_cancel2(self): + sock, proto = self.prepare() + + fut = self.loop.create_future() + fileno = self.file.fileno() + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + fut.cancel() + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), sock, fileno, + 0, None, len(self.DATA), 0) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + + def test_sock_sendfile_blocking_error(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = mock.Mock() + fut.cancelled.return_value = False + with mock.patch('os.sendfile', side_effect=BlockingIOError()): + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + key = self.loop._selector.get_key(sock) + self.assertIsNotNone(key) + fut.add_done_callback.assert_called_once_with(mock.ANY) + + def test_sock_sendfile_os_error_first_call(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + with mock.patch('os.sendfile', side_effect=OSError()): + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIsInstance(exc, asyncio.SendfileNotAvailableError) + self.assertEqual(0, self.file.tell()) + + def test_sock_sendfile_os_error_next_call(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + err = OSError() + with mock.patch('os.sendfile', side_effect=err): + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), + sock, fileno, + 1000, None, len(self.DATA), + 1000) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIs(exc, err) + self.assertEqual(1000, self.file.tell()) + + def test_sock_sendfile_exception(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + err = asyncio.SendfileNotAvailableError() + with mock.patch('os.sendfile', side_effect=err): + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), + sock, fileno, + 1000, None, len(self.DATA), + 1000) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIs(exc, err) + self.assertEqual(1000, self.file.tell()) + + +class UnixReadPipeTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.Protocol) + self.pipe = mock.Mock(spec_set=io.RawIOBase) + self.pipe.fileno.return_value = 5 + + blocking_patcher = mock.patch('os.set_blocking') + blocking_patcher.start() + self.addCleanup(blocking_patcher.stop) + + fstat_patcher = mock.patch('os.fstat') + m_fstat = fstat_patcher.start() + st = mock.Mock() + st.st_mode = stat.S_IFIFO + m_fstat.return_value = st + self.addCleanup(fstat_patcher.stop) + + def read_pipe_transport(self, waiter=None): + transport = unix_events._UnixReadPipeTransport(self.loop, self.pipe, + self.protocol, + waiter=waiter) + self.addCleanup(close_pipe_transport, transport) + return transport + + def test_ctor(self): + waiter = self.loop.create_future() + tr = self.read_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) + self.loop.assert_reader(5, tr._read_ready) + self.assertIsNone(waiter.result()) + + @mock.patch('os.read') + def test__read_ready(self, m_read): + tr = self.read_pipe_transport() + m_read.return_value = b'data' + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + self.protocol.data_received.assert_called_with(b'data') + + @mock.patch('os.read') + def test__read_ready_eof(self, m_read): + tr = self.read_pipe_transport() + m_read.return_value = b'' + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.eof_received.assert_called_with() + self.protocol.connection_lost.assert_called_with(None) + + @mock.patch('os.read') + def test__read_ready_blocked(self, m_read): + tr = self.read_pipe_transport() + m_read.side_effect = BlockingIOError + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + test_utils.run_briefly(self.loop) + self.assertFalse(self.protocol.data_received.called) + + @mock.patch('asyncio.log.logger.error') + @mock.patch('os.read') + def test__read_ready_error(self, m_read, m_logexc): + tr = self.read_pipe_transport() + err = OSError() + m_read.side_effect = err + tr._close = mock.Mock() + tr._read_ready() + + m_read.assert_called_with(5, tr.max_size) + tr._close.assert_called_with(err) + m_logexc.assert_called_with( + test_utils.MockPattern( + 'Fatal read error on pipe transport' + '\nprotocol:.*\ntransport:.*'), + exc_info=(OSError, MOCK_ANY, MOCK_ANY)) + + @mock.patch('os.read') + def test_pause_reading(self, m_read): + tr = self.read_pipe_transport() + m = mock.Mock() + self.loop.add_reader(5, m) + tr.pause_reading() + self.assertFalse(self.loop.readers) + + @mock.patch('os.read') + def test_resume_reading(self, m_read): + tr = self.read_pipe_transport() + tr.pause_reading() + tr.resume_reading() + self.loop.assert_reader(5, tr._read_ready) + + @mock.patch('os.read') + def test_close(self, m_read): + tr = self.read_pipe_transport() + tr._close = mock.Mock() + tr.close() + tr._close.assert_called_with(None) + + @mock.patch('os.read') + def test_close_already_closing(self, m_read): + tr = self.read_pipe_transport() + tr._closing = True + tr._close = mock.Mock() + tr.close() + self.assertFalse(tr._close.called) + + @mock.patch('os.read') + def test__close(self, m_read): + tr = self.read_pipe_transport() + err = object() + tr._close(err) + self.assertTrue(tr.is_closing()) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(err) + + def test__call_connection_lost(self): + tr = self.read_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = None + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test__call_connection_lost_with_err(self): + tr = self.read_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = OSError() + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test_pause_reading_on_closed_pipe(self): + tr = self.read_pipe_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.assertIsNone(tr._loop) + tr.pause_reading() + + def test_pause_reading_on_paused_pipe(self): + tr = self.read_pipe_transport() + tr.pause_reading() + # the second call should do nothing + tr.pause_reading() + + def test_resume_reading_on_closed_pipe(self): + tr = self.read_pipe_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.assertIsNone(tr._loop) + tr.resume_reading() + + def test_resume_reading_on_paused_pipe(self): + tr = self.read_pipe_transport() + # the pipe is not paused + # resuming should do nothing + tr.resume_reading() + + +class UnixWritePipeTransportTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.protocol = test_utils.make_test_protocol(asyncio.BaseProtocol) + self.pipe = mock.Mock(spec_set=io.RawIOBase) + self.pipe.fileno.return_value = 5 + + blocking_patcher = mock.patch('os.set_blocking') + blocking_patcher.start() + self.addCleanup(blocking_patcher.stop) + + fstat_patcher = mock.patch('os.fstat') + m_fstat = fstat_patcher.start() + st = mock.Mock() + st.st_mode = stat.S_IFSOCK + m_fstat.return_value = st + self.addCleanup(fstat_patcher.stop) + + def write_pipe_transport(self, waiter=None): + transport = unix_events._UnixWritePipeTransport(self.loop, self.pipe, + self.protocol, + waiter=waiter) + self.addCleanup(close_pipe_transport, transport) + return transport + + def test_ctor(self): + waiter = self.loop.create_future() + tr = self.write_pipe_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.protocol.connection_made.assert_called_with(tr) + self.loop.assert_reader(5, tr._read_ready) + self.assertEqual(None, waiter.result()) + + def test_can_write_eof(self): + tr = self.write_pipe_transport() + self.assertTrue(tr.can_write_eof()) + + @mock.patch('os.write') + def test_write(self, m_write): + tr = self.write_pipe_transport() + m_write.return_value = 4 + tr.write(b'data') + m_write.assert_called_with(5, b'data') + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + + @mock.patch('os.write') + def test_write_no_data(self, m_write): + tr = self.write_pipe_transport() + tr.write(b'') + self.assertFalse(m_write.called) + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(b''), tr._buffer) + + @mock.patch('os.write') + def test_write_partial(self, m_write): + tr = self.write_pipe_transport() + m_write.return_value = 2 + tr.write(b'data') + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'ta'), tr._buffer) + + @mock.patch('os.write') + def test_write_buffer(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'previous') + tr.write(b'data') + self.assertFalse(m_write.called) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'previousdata'), tr._buffer) + + @mock.patch('os.write') + def test_write_again(self, m_write): + tr = self.write_pipe_transport() + m_write.side_effect = BlockingIOError() + tr.write(b'data') + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('asyncio.unix_events.logger') + @mock.patch('os.write') + def test_write_err(self, m_write, m_log): + tr = self.write_pipe_transport() + err = OSError() + m_write.side_effect = err + tr._fatal_error = mock.Mock() + tr.write(b'data') + m_write.assert_called_with(5, b'data') + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + tr._fatal_error.assert_called_with( + err, + 'Fatal write error on pipe transport') + self.assertEqual(1, tr._conn_lost) + + tr.write(b'data') + self.assertEqual(2, tr._conn_lost) + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + tr.write(b'data') + # This is a bit overspecified. :-( + m_log.warning.assert_called_with( + 'pipe closed by peer or os.write(pipe, data) raised exception.') + tr.close() + + @mock.patch('os.write') + def test_write_close(self, m_write): + tr = self.write_pipe_transport() + tr._read_ready() # pipe was closed by peer + + tr.write(b'data') + self.assertEqual(tr._conn_lost, 1) + tr.write(b'data') + self.assertEqual(tr._conn_lost, 2) + + def test__read_ready(self): + tr = self.write_pipe_transport() + tr._read_ready() + self.assertFalse(self.loop.readers) + self.assertFalse(self.loop.writers) + self.assertTrue(tr.is_closing()) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + @mock.patch('os.write') + def test__write_ready(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 4 + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertEqual(bytearray(), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_partial(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 3 + tr._write_ready() + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'a'), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_again(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.side_effect = BlockingIOError() + tr._write_ready() + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('os.write') + def test__write_ready_empty(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.return_value = 0 + tr._write_ready() + m_write.assert_called_with(5, bytearray(b'data')) + self.loop.assert_writer(5, tr._write_ready) + self.assertEqual(bytearray(b'data'), tr._buffer) + + @mock.patch('asyncio.log.logger.error') + @mock.patch('os.write') + def test__write_ready_err(self, m_write, m_logexc): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._buffer = bytearray(b'data') + m_write.side_effect = err = OSError() + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertFalse(self.loop.readers) + self.assertEqual(bytearray(), tr._buffer) + self.assertTrue(tr.is_closing()) + m_logexc.assert_not_called() + self.assertEqual(1, tr._conn_lost) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(err) + + @mock.patch('os.write') + def test__write_ready_closing(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + tr._closing = True + tr._buffer = bytearray(b'data') + m_write.return_value = 4 + tr._write_ready() + self.assertFalse(self.loop.writers) + self.assertFalse(self.loop.readers) + self.assertEqual(bytearray(), tr._buffer) + self.protocol.connection_lost.assert_called_with(None) + self.pipe.close.assert_called_with() + + @mock.patch('os.write') + def test_abort(self, m_write): + tr = self.write_pipe_transport() + self.loop.add_writer(5, tr._write_ready) + self.loop.add_reader(5, tr._read_ready) + tr._buffer = [b'da', b'ta'] + tr.abort() + self.assertFalse(m_write.called) + self.assertFalse(self.loop.readers) + self.assertFalse(self.loop.writers) + self.assertEqual([], tr._buffer) + self.assertTrue(tr.is_closing()) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test__call_connection_lost(self): + tr = self.write_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = None + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test__call_connection_lost_with_err(self): + tr = self.write_pipe_transport() + self.assertIsNotNone(tr._protocol) + self.assertIsNotNone(tr._loop) + + err = OSError() + tr._call_connection_lost(err) + self.protocol.connection_lost.assert_called_with(err) + self.pipe.close.assert_called_with() + + self.assertIsNone(tr._protocol) + self.assertIsNone(tr._loop) + + def test_close(self): + tr = self.write_pipe_transport() + tr.write_eof = mock.Mock() + tr.close() + tr.write_eof.assert_called_with() + + # closing the transport twice must not fail + tr.close() + + def test_close_closing(self): + tr = self.write_pipe_transport() + tr.write_eof = mock.Mock() + tr._closing = True + tr.close() + self.assertFalse(tr.write_eof.called) + + def test_write_eof(self): + tr = self.write_pipe_transport() + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.loop.readers) + test_utils.run_briefly(self.loop) + self.protocol.connection_lost.assert_called_with(None) + + def test_write_eof_pending(self): + tr = self.write_pipe_transport() + tr._buffer = [b'data'] + tr.write_eof() + self.assertTrue(tr.is_closing()) + self.assertFalse(self.protocol.connection_lost.called) + + +class TestFunctional(unittest.TestCase): + + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + asyncio.set_event_loop(None) + + def test_add_reader_invalid_argument(self): + def assert_raises(): + return self.assertRaisesRegex(ValueError, r'Invalid file object') + + cb = lambda: None + + with assert_raises(): + self.loop.add_reader(object(), cb) + with assert_raises(): + self.loop.add_writer(object(), cb) + + with assert_raises(): + self.loop.remove_reader(object()) + with assert_raises(): + self.loop.remove_writer(object()) + + def test_add_reader_or_writer_transport_fd(self): + def assert_raises(): + return self.assertRaisesRegex( + RuntimeError, + r'File descriptor .* is used by transport') + + async def runner(): + tr, pr = await self.loop.create_connection( + lambda: asyncio.Protocol(), sock=rsock) + + try: + cb = lambda: None + + with assert_raises(): + self.loop.add_reader(rsock, cb) + with assert_raises(): + self.loop.add_reader(rsock.fileno(), cb) + + with assert_raises(): + self.loop.remove_reader(rsock) + with assert_raises(): + self.loop.remove_reader(rsock.fileno()) + + with assert_raises(): + self.loop.add_writer(rsock, cb) + with assert_raises(): + self.loop.add_writer(rsock.fileno(), cb) + + with assert_raises(): + self.loop.remove_writer(rsock) + with assert_raises(): + self.loop.remove_writer(rsock.fileno()) + + finally: + tr.close() + + rsock, wsock = socket.socketpair() + try: + self.loop.run_until_complete(runner()) + finally: + rsock.close() + wsock.close() + + +@support.requires_fork() +class TestFork(unittest.TestCase): + + def test_fork_not_share_current_task(self): + loop = object() + task = object() + asyncio._set_running_loop(loop) + self.addCleanup(asyncio._set_running_loop, None) + asyncio.tasks._enter_task(loop, task) + self.addCleanup(asyncio.tasks._leave_task, loop, task) + self.assertIs(asyncio.current_task(), task) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + asyncio._set_running_loop(loop) + current_task = asyncio.current_task() + if current_task is None: + os.write(w, b'NO TASK') + else: + os.write(w, b'TASK:' + str(id(current_task)).encode()) + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + asyncio._set_running_loop(None) + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO TASK') + wait_process(pid, exitcode=0) + + def test_fork_not_share_event_loop(self): + # The forked process should not share the event loop with the parent + loop = object() + asyncio._set_running_loop(loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.addCleanup(asyncio._set_running_loop, None) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + loop = asyncio.get_event_loop() + os.write(w, b'LOOP:' + str(id(loop)).encode()) + except RuntimeError: + os.write(w, b'NO LOOP') + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO LOOP') + wait_process(pid, exitcode=0) + + @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_fork_signal_handling(self): + self.addCleanup(multiprocessing_cleanup_tests) + + # Sending signal to the forked process should not affect the parent + # process + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + child_started = manager.Event() + child_handled = manager.Event() + parent_handled = manager.Event() + + def child_main(): + def on_sigterm(*args): + child_handled.set() + sys.exit() + + signal.signal(signal.SIGTERM, on_sigterm) + child_started.set() + while True: + time.sleep(1) + + async def main(): + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGTERM, lambda *args: parent_handled.set()) + + process = ctx.Process(target=child_main) + process.start() + child_started.wait() + os.kill(process.pid, signal.SIGTERM) + process.join(timeout=support.SHORT_TIMEOUT) + + async def func(): + await asyncio.sleep(0.1) + return 42 + + # Test parent's loop is still functional + self.assertEqual(await asyncio.create_task(func()), 42) + + asyncio.run(main()) + + child_handled.wait(timeout=support.SHORT_TIMEOUT) + self.assertFalse(parent_handled.is_set()) + self.assertTrue(child_handled.is_set()) + + @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_fork_asyncio_run(self): + self.addCleanup(multiprocessing_cleanup_tests) + + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + result = manager.Value('i', 0) + + async def child_main(): + await asyncio.sleep(0.1) + result.value = 42 + + process = ctx.Process(target=lambda: asyncio.run(child_main())) + process.start() + process.join() + + self.assertEqual(result.value, 42) + + @hashlib_helper.requires_hashdigest('md5') + @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + def test_fork_asyncio_subprocess(self): + self.addCleanup(multiprocessing_cleanup_tests) + + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + result = manager.Value('i', 1) + + async def child_main(): + proc = await asyncio.create_subprocess_exec(sys.executable, '-c', 'pass') + result.value = await proc.wait() + + process = ctx.Process(target=lambda: asyncio.run(child_main())) + process.start() + process.join() + + self.assertEqual(result.value, 0) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py new file mode 100644 index 00000000000..dedc6bf69d7 --- /dev/null +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -0,0 +1,353 @@ +import asyncio +import unittest +import time +from test import support + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +# The following value can be used as a very small timeout: +# it passes check "timeout > 0", but has almost +# no effect on the test performance +_EPSILON = 0.0001 + + +class SlowTask: + """ Task will run for this defined time, ignoring cancel requests """ + TASK_TIMEOUT = 0.2 + + def __init__(self): + self.exited = False + + async def run(self): + exitat = time.monotonic() + self.TASK_TIMEOUT + + while True: + tosleep = exitat - time.monotonic() + if tosleep <= 0: + break + + try: + await asyncio.sleep(tosleep) + except asyncio.CancelledError: + pass + + self.exited = True + + +class AsyncioWaitForTest(unittest.IsolatedAsyncioTestCase): + + async def test_asyncio_wait_for_cancelled(self): + t = SlowTask() + + waitfortask = asyncio.create_task( + asyncio.wait_for(t.run(), t.TASK_TIMEOUT * 2)) + await asyncio.sleep(0) + waitfortask.cancel() + await asyncio.wait({waitfortask}) + + self.assertTrue(t.exited) + + async def test_asyncio_wait_for_timeout(self): + t = SlowTask() + + try: + await asyncio.wait_for(t.run(), t.TASK_TIMEOUT / 2) + except asyncio.TimeoutError: + pass + + self.assertTrue(t.exited) + + async def test_wait_for_timeout_less_then_0_or_0_future_done(self): + loop = asyncio.get_running_loop() + + fut = loop.create_future() + fut.set_result('done') + + ret = await asyncio.wait_for(fut, 0) + + self.assertEqual(ret, 'done') + self.assertTrue(fut.done()) + + async def test_wait_for_timeout_less_then_0_or_0_coroutine_do_not_started(self): + foo_started = False + + async def foo(): + nonlocal foo_started + foo_started = True + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(foo(), 0) + + self.assertEqual(foo_started, False) + + async def test_wait_for_timeout_less_then_0_or_0(self): + loop = asyncio.get_running_loop() + + for timeout in [0, -1]: + with self.subTest(timeout=timeout): + foo_running = None + started = loop.create_future() + + async def foo(): + nonlocal foo_running + foo_running = True + started.set_result(None) + try: + await asyncio.sleep(10) + finally: + foo_running = False + return 'done' + + fut = asyncio.create_task(foo()) + await started + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(fut, timeout) + + self.assertTrue(fut.done()) + # it should have been cancelled due to the timeout + self.assertTrue(fut.cancelled()) + self.assertEqual(foo_running, False) + + async def test_wait_for(self): + foo_running = None + + async def foo(): + nonlocal foo_running + foo_running = True + try: + await asyncio.sleep(support.LONG_TIMEOUT) + finally: + foo_running = False + return 'done' + + fut = asyncio.create_task(foo()) + + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(fut, 0.1) + self.assertTrue(fut.done()) + # it should have been cancelled due to the timeout + self.assertTrue(fut.cancelled()) + self.assertEqual(foo_running, False) + + async def test_wait_for_blocking(self): + async def coro(): + return 'done' + + res = await asyncio.wait_for(coro(), timeout=None) + self.assertEqual(res, 'done') + + async def test_wait_for_race_condition(self): + loop = asyncio.get_running_loop() + + fut = loop.create_future() + task = asyncio.wait_for(fut, timeout=0.2) + loop.call_soon(fut.set_result, "ok") + res = await task + self.assertEqual(res, "ok") + + async def test_wait_for_cancellation_race_condition(self): + async def inner(): + with self.assertRaises(asyncio.CancelledError): + await asyncio.sleep(1) + return 1 + + result = await asyncio.wait_for(inner(), timeout=.01) + self.assertEqual(result, 1) + + async def test_wait_for_waits_for_task_cancellation(self): + task_done = False + + async def inner(): + nonlocal task_done + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + await asyncio.sleep(_EPSILON) + raise + finally: + task_done = True + + inner_task = asyncio.create_task(inner()) + + with self.assertRaises(asyncio.TimeoutError) as cm: + await asyncio.wait_for(inner_task, timeout=_EPSILON) + + self.assertTrue(task_done) + chained = cm.exception.__context__ + self.assertEqual(type(chained), asyncio.CancelledError) + + async def test_wait_for_waits_for_task_cancellation_w_timeout_0(self): + task_done = False + + async def foo(): + async def inner(): + nonlocal task_done + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + await asyncio.sleep(_EPSILON) + raise + finally: + task_done = True + + inner_task = asyncio.create_task(inner()) + await asyncio.sleep(_EPSILON) + await asyncio.wait_for(inner_task, timeout=0) + + with self.assertRaises(asyncio.TimeoutError) as cm: + await foo() + + self.assertTrue(task_done) + chained = cm.exception.__context__ + self.assertEqual(type(chained), asyncio.CancelledError) + + async def test_wait_for_reraises_exception_during_cancellation(self): + class FooException(Exception): + pass + + async def foo(): + async def inner(): + try: + await asyncio.sleep(0.2) + finally: + raise FooException + + inner_task = asyncio.create_task(inner()) + + await asyncio.wait_for(inner_task, timeout=_EPSILON) + + with self.assertRaises(FooException): + await foo() + + async def _test_cancel_wait_for(self, timeout): + loop = asyncio.get_running_loop() + + async def blocking_coroutine(): + fut = loop.create_future() + # Block: fut result is never set + await fut + + task = asyncio.create_task(blocking_coroutine()) + + wait = asyncio.create_task(asyncio.wait_for(task, timeout)) + loop.call_soon(wait.cancel) + + with self.assertRaises(asyncio.CancelledError): + await wait + + # Python issue #23219: cancelling the wait must also cancel the task + self.assertTrue(task.cancelled()) + + async def test_cancel_blocking_wait_for(self): + await self._test_cancel_wait_for(None) + + async def test_cancel_wait_for(self): + await self._test_cancel_wait_for(60.0) + + async def test_wait_for_cancel_suppressed(self): + # GH-86296: Suppressing CancelledError is discouraged + # but if a task suppresses CancelledError and returns a value, + # `wait_for` should return the value instead of raising CancelledError. + # This is the same behavior as `asyncio.timeout`. + + async def return_42(): + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + return 42 + + res = await asyncio.wait_for(return_42(), timeout=0.1) + self.assertEqual(res, 42) + + + async def test_wait_for_issue86296(self): + # GH-86296: The task should get cancelled and not run to completion. + # inner completes in one cycle of the event loop so it + # completes before the task is cancelled. + + async def inner(): + return 'done' + + inner_task = asyncio.create_task(inner()) + reached_end = False + + async def wait_for_coro(): + await asyncio.wait_for(inner_task, timeout=100) + await asyncio.sleep(1) + nonlocal reached_end + reached_end = True + + task = asyncio.create_task(wait_for_coro()) + self.assertFalse(task.done()) + # Run the task + await asyncio.sleep(0) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertTrue(inner_task.done()) + self.assertEqual(await inner_task, 'done') + self.assertFalse(reached_end) + + +class WaitForShieldTests(unittest.IsolatedAsyncioTestCase): + + async def test_zero_timeout(self): + # `asyncio.shield` creates a new task which wraps the passed in + # awaitable and shields it from cancellation so with timeout=0 + # the task returned by `asyncio.shield` aka shielded_task gets + # cancelled immediately and the task wrapped by it is scheduled + # to run. + + async def coro(): + await asyncio.sleep(0.01) + return 'done' + + task = asyncio.create_task(coro()) + with self.assertRaises(asyncio.TimeoutError): + shielded_task = asyncio.shield(task) + await asyncio.wait_for(shielded_task, timeout=0) + + # Task is running in background + self.assertFalse(task.done()) + self.assertFalse(task.cancelled()) + self.assertTrue(shielded_task.cancelled()) + + # Wait for the task to complete + await asyncio.sleep(0.1) + self.assertTrue(task.done()) + + + async def test_none_timeout(self): + # With timeout=None the timeout is disabled so it + # runs till completion. + async def coro(): + await asyncio.sleep(0.1) + return 'done' + + task = asyncio.create_task(coro()) + await asyncio.wait_for(asyncio.shield(task), timeout=None) + + self.assertTrue(task.done()) + self.assertEqual(await task, "done") + + async def test_shielded_timeout(self): + # shield prevents the task from being cancelled. + async def coro(): + await asyncio.sleep(0.1) + return 'done' + + task = asyncio.create_task(coro()) + with self.assertRaises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(task), timeout=0.01) + + self.assertFalse(task.done()) + self.assertFalse(task.cancelled()) + self.assertEqual(await task, "done") + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py new file mode 100644 index 00000000000..0af3368627a --- /dev/null +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -0,0 +1,363 @@ +import os +import signal +import socket +import sys +import time +import threading +import unittest +from unittest import mock + +if sys.platform != 'win32': + raise unittest.SkipTest('Windows only') + +import _overlapped +import _winapi + +import asyncio +from asyncio import windows_events +from test.test_asyncio import utils as test_utils + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class UpperProto(asyncio.Protocol): + def __init__(self): + self.buf = [] + + def connection_made(self, trans): + self.trans = trans + + def data_received(self, data): + self.buf.append(data) + if b'\n' in data: + self.trans.write(b''.join(self.buf).upper()) + self.trans.close() + + +class WindowsEventsTestCase(test_utils.TestCase): + def _unraisablehook(self, unraisable): + # Storing unraisable.object can resurrect an object which is being + # finalized. Storing unraisable.exc_value creates a reference cycle. + self._unraisable = unraisable + print(unraisable) + + def setUp(self): + self._prev_unraisablehook = sys.unraisablehook + self._unraisable = None + sys.unraisablehook = self._unraisablehook + + def tearDown(self): + sys.unraisablehook = self._prev_unraisablehook + self.assertIsNone(self._unraisable) + +class ProactorLoopCtrlC(WindowsEventsTestCase): + + def test_ctrl_c(self): + + def SIGINT_after_delay(): + time.sleep(0.1) + signal.raise_signal(signal.SIGINT) + + thread = threading.Thread(target=SIGINT_after_delay) + loop = asyncio.new_event_loop() + try: + # only start the loop once the event loop is running + loop.call_soon(thread.start) + loop.run_forever() + self.fail("should not fall through 'run_forever'") + except KeyboardInterrupt: + pass + finally: + self.close_loop(loop) + thread.join() + + +class ProactorMultithreading(WindowsEventsTestCase): + def test_run_from_nonmain_thread(self): + finished = False + + async def coro(): + await asyncio.sleep(0) + + def func(): + nonlocal finished + loop = asyncio.new_event_loop() + loop.run_until_complete(coro()) + # close() must not call signal.set_wakeup_fd() + loop.close() + finished = True + + thread = threading.Thread(target=func) + thread.start() + thread.join() + self.assertTrue(finished) + + +class ProactorTests(WindowsEventsTestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.ProactorEventLoop() + self.set_event_loop(self.loop) + + def test_close(self): + a, b = socket.socketpair() + trans = self.loop._make_socket_transport(a, asyncio.Protocol()) + f = asyncio.ensure_future(self.loop.sock_recv(b, 100), loop=self.loop) + trans.close() + self.loop.run_until_complete(f) + self.assertEqual(f.result(), b'') + b.close() + + def test_double_bind(self): + ADDRESS = r'\\.\pipe\test_double_bind-%s' % os.getpid() + server1 = windows_events.PipeServer(ADDRESS) + with self.assertRaises(PermissionError): + windows_events.PipeServer(ADDRESS) + server1.close() + + def test_pipe(self): + res = self.loop.run_until_complete(self._test_pipe()) + self.assertEqual(res, 'done') + + async def _test_pipe(self): + ADDRESS = r'\\.\pipe\_test_pipe-%s' % os.getpid() + + with self.assertRaises(FileNotFoundError): + await self.loop.create_pipe_connection( + asyncio.Protocol, ADDRESS) + + [server] = await self.loop.start_serving_pipe( + UpperProto, ADDRESS) + self.assertIsInstance(server, windows_events.PipeServer) + + clients = [] + for i in range(5): + stream_reader = asyncio.StreamReader(loop=self.loop) + protocol = asyncio.StreamReaderProtocol(stream_reader, + loop=self.loop) + trans, proto = await self.loop.create_pipe_connection( + lambda: protocol, ADDRESS) + self.assertIsInstance(trans, asyncio.Transport) + self.assertEqual(protocol, proto) + clients.append((stream_reader, trans)) + + for i, (r, w) in enumerate(clients): + w.write('lower-{}\n'.format(i).encode()) + + for i, (r, w) in enumerate(clients): + response = await r.readline() + self.assertEqual(response, 'LOWER-{}\n'.format(i).encode()) + w.close() + + server.close() + + with self.assertRaises(FileNotFoundError): + await self.loop.create_pipe_connection( + asyncio.Protocol, ADDRESS) + + return 'done' + + def test_connect_pipe_cancel(self): + exc = OSError() + exc.winerror = _overlapped.ERROR_PIPE_BUSY + with mock.patch.object(_overlapped, 'ConnectPipe', + side_effect=exc) as connect: + coro = self.loop._proactor.connect_pipe('pipe_address') + task = self.loop.create_task(coro) + + # check that it's possible to cancel connect_pipe() + task.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(task) + + def test_wait_for_handle(self): + event = _overlapped.CreateEvent(None, True, False, None) + self.addCleanup(_winapi.CloseHandle, event) + + # Wait for unset event with 0.5s timeout; + # result should be False at timeout + timeout = 0.5 + fut = self.loop._proactor.wait_for_handle(event, timeout) + start = self.loop.time() + done = self.loop.run_until_complete(fut) + elapsed = self.loop.time() - start + + self.assertEqual(done, False) + self.assertFalse(fut.result()) + self.assertGreaterEqual(elapsed, timeout - test_utils.CLOCK_RES) + + _overlapped.SetEvent(event) + + # Wait for set event; + # result should be True immediately + fut = self.loop._proactor.wait_for_handle(event, 10) + done = self.loop.run_until_complete(fut) + + self.assertEqual(done, True) + self.assertTrue(fut.result()) + + # asyncio issue #195: cancelling a done _WaitHandleFuture + # must not crash + fut.cancel() + + def test_wait_for_handle_cancel(self): + event = _overlapped.CreateEvent(None, True, False, None) + self.addCleanup(_winapi.CloseHandle, event) + + # Wait for unset event with a cancelled future; + # CancelledError should be raised immediately + fut = self.loop._proactor.wait_for_handle(event, 10) + fut.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(fut) + + # asyncio issue #195: cancelling a _WaitHandleFuture twice + # must not crash + fut = self.loop._proactor.wait_for_handle(event) + fut.cancel() + fut.cancel() + + def test_read_self_pipe_restart(self): + # Regression test for https://bugs.python.org/issue39010 + # Previously, restarting a proactor event loop in certain states + # would lead to spurious ConnectionResetErrors being logged. + self.loop.call_exception_handler = mock.Mock() + # Start an operation in another thread so that the self-pipe is used. + # This is theoretically timing-dependent (the task in the executor + # must complete before our start/stop cycles), but in practice it + # seems to work every time. + f = self.loop.run_in_executor(None, lambda: None) + self.loop.stop() + self.loop.run_forever() + self.loop.stop() + self.loop.run_forever() + + # Shut everything down cleanly. This is an important part of the + # test - in issue 39010, the error occurred during loop.close(), + # so we want to close the loop during the test instead of leaving + # it for tearDown. + # + # First wait for f to complete to avoid a "future's result was never + # retrieved" error. + self.loop.run_until_complete(f) + # Now shut down the loop itself (self.close_loop also shuts down the + # loop's default executor). + self.close_loop(self.loop) + self.assertFalse(self.loop.call_exception_handler.called) + + def test_address_argument_type_error(self): + # Regression test for https://github.com/python/cpython/issues/98793 + proactor = self.loop._proactor + sock = socket.socket(type=socket.SOCK_DGRAM) + bad_address = None + with self.assertRaises(TypeError): + proactor.connect(sock, bad_address) + with self.assertRaises(TypeError): + proactor.sendto(sock, b'abc', addr=bad_address) + sock.close() + + def test_client_pipe_stat(self): + res = self.loop.run_until_complete(self._test_client_pipe_stat()) + self.assertEqual(res, 'done') + + async def _test_client_pipe_stat(self): + # Regression test for https://github.com/python/cpython/issues/100573 + ADDRESS = r'\\.\pipe\test_client_pipe_stat-%s' % os.getpid() + + async def probe(): + # See https://github.com/python/cpython/pull/100959#discussion_r1068533658 + h = _overlapped.ConnectPipe(ADDRESS) + try: + _winapi.CloseHandle(_overlapped.ConnectPipe(ADDRESS)) + except OSError as e: + if e.winerror != _overlapped.ERROR_PIPE_BUSY: + raise + finally: + _winapi.CloseHandle(h) + + with self.assertRaises(FileNotFoundError): + await probe() + + [server] = await self.loop.start_serving_pipe(asyncio.Protocol, ADDRESS) + self.assertIsInstance(server, windows_events.PipeServer) + + errors = [] + self.loop.set_exception_handler(lambda _, data: errors.append(data)) + + for i in range(5): + await self.loop.create_task(probe()) + + self.assertEqual(len(errors), 0, errors) + + server.close() + + with self.assertRaises(FileNotFoundError): + await probe() + + return "done" + + def test_loop_restart(self): + # We're fishing for the "RuntimeError: <_overlapped.Overlapped object at XXX> + # still has pending operation at deallocation, the process may crash" error + stop = threading.Event() + def threadMain(): + while not stop.is_set(): + self.loop.call_soon_threadsafe(lambda: None) + time.sleep(0.01) + thr = threading.Thread(target=threadMain) + + # In 10 60-second runs of this test prior to the fix: + # time in seconds until failure: (none), 15.0, 6.4, (none), 7.6, 8.3, 1.7, 22.2, 23.5, 8.3 + # 10 seconds had a 50% failure rate but longer would be more costly + end_time = time.time() + 10 # Run for 10 seconds + self.loop.call_soon(thr.start) + while not self._unraisable: # Stop if we got an unraisable exc + self.loop.stop() + self.loop.run_forever() + if time.time() >= end_time: + break + + stop.set() + thr.join() + + +class WinPolicyTests(WindowsEventsTestCase): + + def test_selector_win_policy(self): + async def main(): + self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) + + old_policy = asyncio.events._get_event_loop_policy() + try: + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", + ): + asyncio.events._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.run(main()) + finally: + asyncio.events._set_event_loop_policy(old_policy) + + def test_proactor_win_policy(self): + async def main(): + self.assertIsInstance( + asyncio.get_running_loop(), + asyncio.ProactorEventLoop) + + old_policy = asyncio.events._get_event_loop_policy() + try: + with self.assertWarnsRegex( + DeprecationWarning, + "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", + ): + asyncio.events._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + asyncio.run(main()) + finally: + asyncio.events._set_event_loop_policy(old_policy) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py new file mode 100644 index 00000000000..97f078ff911 --- /dev/null +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -0,0 +1,133 @@ +"""Tests for window_utils""" + +import sys +import unittest +import warnings + +if sys.platform != 'win32': + raise unittest.SkipTest('Windows only') + +import _overlapped +import _winapi + +import asyncio +from asyncio import windows_utils +from test import support + + +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + +class PipeTests(unittest.TestCase): + + def test_pipe_overlapped(self): + h1, h2 = windows_utils.pipe(overlapped=(True, True)) + try: + ov1 = _overlapped.Overlapped() + self.assertFalse(ov1.pending) + self.assertEqual(ov1.error, 0) + + ov1.ReadFile(h1, 100) + self.assertTrue(ov1.pending) + self.assertEqual(ov1.error, _winapi.ERROR_IO_PENDING) + ERROR_IO_INCOMPLETE = 996 + try: + ov1.getresult() + except OSError as e: + self.assertEqual(e.winerror, ERROR_IO_INCOMPLETE) + else: + raise RuntimeError('expected ERROR_IO_INCOMPLETE') + + ov2 = _overlapped.Overlapped() + self.assertFalse(ov2.pending) + self.assertEqual(ov2.error, 0) + + ov2.WriteFile(h2, b"hello") + self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING}) + + res = _winapi.WaitForMultipleObjects([ov2.event], False, 100) + self.assertEqual(res, _winapi.WAIT_OBJECT_0) + + self.assertFalse(ov1.pending) + self.assertEqual(ov1.error, ERROR_IO_INCOMPLETE) + self.assertFalse(ov2.pending) + self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING}) + self.assertEqual(ov1.getresult(), b"hello") + finally: + _winapi.CloseHandle(h1) + _winapi.CloseHandle(h2) + + def test_pipe_handle(self): + h, _ = windows_utils.pipe(overlapped=(True, True)) + _winapi.CloseHandle(_) + p = windows_utils.PipeHandle(h) + self.assertEqual(p.fileno(), h) + self.assertEqual(p.handle, h) + + # check garbage collection of p closes handle + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", ResourceWarning) + del p + support.gc_collect() + try: + _winapi.CloseHandle(h) + except OSError as e: + self.assertEqual(e.winerror, 6) # ERROR_INVALID_HANDLE + else: + raise RuntimeError('expected ERROR_INVALID_HANDLE') + + +class PopenTests(unittest.TestCase): + + def test_popen(self): + command = r"""if 1: + import sys + s = sys.stdin.readline() + sys.stdout.write(s.upper()) + sys.stderr.write('stderr') + """ + msg = b"blah\n" + + p = windows_utils.Popen([sys.executable, '-c', command], + stdin=windows_utils.PIPE, + stdout=windows_utils.PIPE, + stderr=windows_utils.PIPE) + + for f in [p.stdin, p.stdout, p.stderr]: + self.assertIsInstance(f, windows_utils.PipeHandle) + + ovin = _overlapped.Overlapped() + ovout = _overlapped.Overlapped() + overr = _overlapped.Overlapped() + + ovin.WriteFile(p.stdin.handle, msg) + ovout.ReadFile(p.stdout.handle, 100) + overr.ReadFile(p.stderr.handle, 100) + + events = [ovin.event, ovout.event, overr.event] + # Super-long timeout for slow buildbots. + res = _winapi.WaitForMultipleObjects(events, True, + int(support.SHORT_TIMEOUT * 1000)) + self.assertEqual(res, _winapi.WAIT_OBJECT_0) + self.assertFalse(ovout.pending) + self.assertFalse(overr.pending) + self.assertFalse(ovin.pending) + + self.assertEqual(ovin.getresult(), len(msg)) + out = ovout.getresult().rstrip() + err = overr.getresult().rstrip() + + self.assertGreater(len(out), 0) + self.assertGreater(len(err), 0) + # allow for partial reads... + self.assertStartsWith(msg.upper().rstrip(), out) + self.assertStartsWith(b"stderr", err) + + # The context manager calls wait() and closes resources + with p: + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py new file mode 100644 index 00000000000..a480e16e81b --- /dev/null +++ b/Lib/test/test_asyncio/utils.py @@ -0,0 +1,609 @@ +"""Utilities shared by tests.""" + +import asyncio +import collections +import contextlib +import io +import logging +import os +import re +import selectors +import socket +import socketserver +import sys +import threading +import unittest +import weakref +from ast import literal_eval +from unittest import mock + +from http.server import HTTPServer +from wsgiref.simple_server import WSGIRequestHandler, WSGIServer + +try: + import ssl +except ImportError: # pragma: no cover + ssl = None + +from asyncio import base_events +from asyncio import events +from asyncio import format_helpers +from asyncio import tasks +from asyncio.log import logger +from test import support +from test.support import socket_helper +from test.support import threading_helper + + +# Use the maximum known clock resolution (gh-75191, gh-110088): Windows +# GetTickCount64() has a resolution of 15.6 ms. Use 50 ms to tolerate rounding +# issues. +CLOCK_RES = 0.050 + + +def data_file(*filename): + fullname = os.path.join(support.TEST_HOME_DIR, *filename) + if os.path.isfile(fullname): + return fullname + fullname = os.path.join(os.path.dirname(__file__), '..', *filename) + if os.path.isfile(fullname): + return fullname + raise FileNotFoundError(os.path.join(filename)) + + +ONLYCERT = data_file('certdata', 'ssl_cert.pem') +ONLYKEY = data_file('certdata', 'ssl_key.pem') +SIGNED_CERTFILE = data_file('certdata', 'keycert3.pem') +SIGNING_CA = data_file('certdata', 'pycacert.pem') +with open(data_file('certdata', 'keycert3.pem.reference')) as file: + PEERCERT = literal_eval(file.read()) + +def simple_server_sslcontext(): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(ONLYCERT, ONLYKEY) + server_context.check_hostname = False + server_context.verify_mode = ssl.CERT_NONE + return server_context + + +def simple_client_sslcontext(*, disable_verify=True): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + if disable_verify: + client_context.verify_mode = ssl.CERT_NONE + return client_context + + +def dummy_ssl_context(): + if ssl is None: + return None + else: + return simple_client_sslcontext(disable_verify=True) + + +def run_briefly(loop): + async def once(): + pass + gen = once() + t = loop.create_task(gen) + # Don't log a warning if the task is not done after run_until_complete(). + # It occurs if the loop is stopped or if a task raises a BaseException. + t._log_destroy_pending = False + try: + loop.run_until_complete(t) + finally: + gen.close() + + +def run_until(loop, pred, timeout=support.SHORT_TIMEOUT): + delay = 0.001 + for _ in support.busy_retry(timeout, error=False): + if pred(): + break + loop.run_until_complete(tasks.sleep(delay)) + delay = max(delay * 2, 1.0) + else: + raise TimeoutError() + + +def run_once(loop): + """Legacy API to run once through the event loop. + + This is the recommended pattern for test code. It will poll the + selector once and run all callbacks scheduled in response to I/O + events. + """ + loop.call_soon(loop.stop) + loop.run_forever() + + +class SilentWSGIRequestHandler(WSGIRequestHandler): + + def get_stderr(self): + return io.StringIO() + + def log_message(self, format, *args): + pass + + +class SilentWSGIServer(WSGIServer): + + request_timeout = support.LOOPBACK_TIMEOUT + + def get_request(self): + request, client_addr = super().get_request() + request.settimeout(self.request_timeout) + return request, client_addr + + def handle_error(self, request, client_address): + pass + + +class SSLWSGIServerMixin: + + def finish_request(self, request, client_address): + # The relative location of our test directory (which + # contains the ssl key and certificate files) differs + # between the stdlib and stand-alone asyncio. + # Prefer our own if we can find it. + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(ONLYCERT, ONLYKEY) + + ssock = context.wrap_socket(request, server_side=True) + try: + self.RequestHandlerClass(ssock, client_address, self) + ssock.close() + except OSError: + # maybe socket has been closed by peer + pass + + +class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer): + pass + + +def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls): + + def loop(environ): + size = int(environ['CONTENT_LENGTH']) + while size: + data = environ['wsgi.input'].read(min(size, 0x10000)) + yield data + size -= len(data) + + def app(environ, start_response): + status = '200 OK' + headers = [('Content-type', 'text/plain')] + start_response(status, headers) + if environ['PATH_INFO'] == '/loop': + return loop(environ) + else: + return [b'Test message'] + + # Run the test WSGI server in a separate thread in order not to + # interfere with event handling in the main thread + server_class = server_ssl_cls if use_ssl else server_cls + httpd = server_class(address, SilentWSGIRequestHandler) + httpd.set_app(app) + httpd.address = httpd.server_address + server_thread = threading.Thread( + target=lambda: httpd.serve_forever(poll_interval=0.05)) + server_thread.start() + try: + yield httpd + finally: + httpd.shutdown() + httpd.server_close() + server_thread.join() + + +if hasattr(socket, 'AF_UNIX'): + + class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer): + + def server_bind(self): + socketserver.UnixStreamServer.server_bind(self) + self.server_name = '127.0.0.1' + self.server_port = 80 + + + class UnixWSGIServer(UnixHTTPServer, WSGIServer): + + request_timeout = support.LOOPBACK_TIMEOUT + + def server_bind(self): + UnixHTTPServer.server_bind(self) + self.setup_environ() + + def get_request(self): + request, client_addr = super().get_request() + request.settimeout(self.request_timeout) + # Code in the stdlib expects that get_request + # will return a socket and a tuple (host, port). + # However, this isn't true for UNIX sockets, + # as the second return value will be a path; + # hence we return some fake data sufficient + # to get the tests going + return request, ('127.0.0.1', '') + + + class SilentUnixWSGIServer(UnixWSGIServer): + + def handle_error(self, request, client_address): + pass + + + class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer): + pass + + + def gen_unix_socket_path(): + return socket_helper.create_unix_domain_name() + + + @contextlib.contextmanager + def unix_socket_path(): + path = gen_unix_socket_path() + try: + yield path + finally: + try: + os.unlink(path) + except OSError: + pass + + + @contextlib.contextmanager + def run_test_unix_server(*, use_ssl=False): + with unix_socket_path() as path: + yield from _run_test_server(address=path, use_ssl=use_ssl, + server_cls=SilentUnixWSGIServer, + server_ssl_cls=UnixSSLWSGIServer) + + +@contextlib.contextmanager +def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False): + yield from _run_test_server(address=(host, port), use_ssl=use_ssl, + server_cls=SilentWSGIServer, + server_ssl_cls=SSLWSGIServer) + + +def echo_datagrams(sock): + while True: + data, addr = sock.recvfrom(4096) + if data == b'STOP': + sock.close() + break + else: + sock.sendto(data, addr) + + +@contextlib.contextmanager +def run_udp_echo_server(*, host='127.0.0.1', port=0): + addr_info = socket.getaddrinfo(host, port, type=socket.SOCK_DGRAM) + family, type, proto, _, sockaddr = addr_info[0] + sock = socket.socket(family, type, proto) + sock.bind((host, port)) + sockname = sock.getsockname() + thread = threading.Thread(target=lambda: echo_datagrams(sock)) + thread.start() + try: + yield sockname + finally: + # gh-122187: use a separate socket to send the stop message to avoid + # TSan reported race on the same socket. + sock2 = socket.socket(family, type, proto) + sock2.sendto(b'STOP', sockname) + sock2.close() + thread.join() + + +def make_test_protocol(base): + dct = {} + for name in dir(base): + if name.startswith('__') and name.endswith('__'): + # skip magic names + continue + dct[name] = MockCallback(return_value=None) + return type('TestProtocol', (base,) + base.__bases__, dct)() + + +class TestSelector(selectors.BaseSelector): + + def __init__(self): + self.keys = {} + + def register(self, fileobj, events, data=None): + key = selectors.SelectorKey(fileobj, 0, events, data) + self.keys[fileobj] = key + return key + + def unregister(self, fileobj): + return self.keys.pop(fileobj) + + def select(self, timeout): + return [] + + def get_map(self): + return self.keys + + +class TestLoop(base_events.BaseEventLoop): + """Loop for unittests. + + It manages self time directly. + If something scheduled to be executed later then + on next loop iteration after all ready handlers done + generator passed to __init__ is calling. + + Generator should be like this: + + def gen(): + ... + when = yield ... + ... = yield time_advance + + Value returned by yield is absolute time of next scheduled handler. + Value passed to yield is time advance to move loop's time forward. + """ + + def __init__(self, gen=None): + super().__init__() + + if gen is None: + def gen(): + yield + self._check_on_close = False + else: + self._check_on_close = True + + self._gen = gen() + next(self._gen) + self._time = 0 + self._clock_resolution = 1e-9 + self._timers = [] + self._selector = TestSelector() + + self.readers = {} + self.writers = {} + self.reset_counters() + + self._transports = weakref.WeakValueDictionary() + + def time(self): + return self._time + + def advance_time(self, advance): + """Move test time forward.""" + if advance: + self._time += advance + + def close(self): + super().close() + if self._check_on_close: + try: + self._gen.send(0) + except StopIteration: + pass + else: # pragma: no cover + raise AssertionError("Time generator is not finished") + + def _add_reader(self, fd, callback, *args): + self.readers[fd] = events.Handle(callback, args, self, None) + + def _remove_reader(self, fd): + self.remove_reader_count[fd] += 1 + if fd in self.readers: + del self.readers[fd] + return True + else: + return False + + def assert_reader(self, fd, callback, *args): + if fd not in self.readers: + raise AssertionError(f'fd {fd} is not registered') + handle = self.readers[fd] + if handle._callback != callback: + raise AssertionError( + f'unexpected callback: {handle._callback} != {callback}') + if handle._args != args: + raise AssertionError( + f'unexpected callback args: {handle._args} != {args}') + + def assert_no_reader(self, fd): + if fd in self.readers: + raise AssertionError(f'fd {fd} is registered') + + def _add_writer(self, fd, callback, *args): + self.writers[fd] = events.Handle(callback, args, self, None) + + def _remove_writer(self, fd): + self.remove_writer_count[fd] += 1 + if fd in self.writers: + del self.writers[fd] + return True + else: + return False + + def assert_writer(self, fd, callback, *args): + if fd not in self.writers: + raise AssertionError(f'fd {fd} is not registered') + handle = self.writers[fd] + if handle._callback != callback: + raise AssertionError(f'{handle._callback!r} != {callback!r}') + if handle._args != args: + raise AssertionError(f'{handle._args!r} != {args!r}') + + def _ensure_fd_no_transport(self, fd): + if not isinstance(fd, int): + try: + fd = int(fd.fileno()) + except (AttributeError, TypeError, ValueError): + # This code matches selectors._fileobj_to_fd function. + raise ValueError("Invalid file object: " + "{!r}".format(fd)) from None + try: + transport = self._transports[fd] + except KeyError: + pass + else: + raise RuntimeError( + 'File descriptor {!r} is used by transport {!r}'.format( + fd, transport)) + + def add_reader(self, fd, callback, *args): + """Add a reader callback.""" + self._ensure_fd_no_transport(fd) + return self._add_reader(fd, callback, *args) + + def remove_reader(self, fd): + """Remove a reader callback.""" + self._ensure_fd_no_transport(fd) + return self._remove_reader(fd) + + def add_writer(self, fd, callback, *args): + """Add a writer callback..""" + self._ensure_fd_no_transport(fd) + return self._add_writer(fd, callback, *args) + + def remove_writer(self, fd): + """Remove a writer callback.""" + self._ensure_fd_no_transport(fd) + return self._remove_writer(fd) + + def reset_counters(self): + self.remove_reader_count = collections.defaultdict(int) + self.remove_writer_count = collections.defaultdict(int) + + def _run_once(self): + super()._run_once() + for when in self._timers: + advance = self._gen.send(when) + self.advance_time(advance) + self._timers = [] + + def call_at(self, when, callback, *args, context=None): + self._timers.append(when) + return super().call_at(when, callback, *args, context=context) + + def _process_events(self, event_list): + return + + def _write_to_self(self): + pass + + +def MockCallback(**kwargs): + return mock.Mock(spec=['__call__'], **kwargs) + + +class MockPattern(str): + """A regex based str with a fuzzy __eq__. + + Use this helper with 'mock.assert_called_with', or anywhere + where a regex comparison between strings is needed. + + For instance: + mock_call.assert_called_with(MockPattern('spam.*ham')) + """ + def __eq__(self, other): + return bool(re.search(str(self), other, re.S)) + + +class MockInstanceOf: + def __init__(self, type): + self._type = type + + def __eq__(self, other): + return isinstance(other, self._type) + + +def get_function_source(func): + source = format_helpers._get_function_source(func) + if source is None: + raise ValueError("unable to get the source of %r" % (func,)) + return source + + +class TestCase(unittest.TestCase): + @staticmethod + def close_loop(loop): + if loop._default_executor is not None: + if not loop.is_closed(): + loop.run_until_complete(loop.shutdown_default_executor()) + else: + loop._default_executor.shutdown(wait=True) + loop.close() + + def set_event_loop(self, loop, *, cleanup=True): + if loop is None: + raise AssertionError('loop is None') + # ensure that the event loop is passed explicitly in asyncio + events.set_event_loop(None) + if cleanup: + self.addCleanup(self.close_loop, loop) + + def new_test_loop(self, gen=None): + loop = TestLoop(gen) + self.set_event_loop(loop) + return loop + + def setUp(self): + self._thread_cleanup = threading_helper.threading_setup() + + def tearDown(self): + events.set_event_loop(None) + + # Detect CPython bug #23353: ensure that yield/yield-from is not used + # in an except block of a generator + self.assertIsNone(sys.exception()) + + self.doCleanups() + threading_helper.threading_cleanup(*self._thread_cleanup) + support.reap_children() + + +@contextlib.contextmanager +def disable_logger(): + """Context manager to disable asyncio logger. + + For example, it can be used to ignore warnings in debug mode. + """ + old_level = logger.level + try: + logger.setLevel(logging.CRITICAL+1) + yield + finally: + logger.setLevel(old_level) + + +def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, + family=socket.AF_INET): + """Create a mock of a non-blocking socket.""" + sock = mock.MagicMock(socket.socket) + sock.proto = proto + sock.type = type + sock.family = family + sock.gettimeout.return_value = 0.0 + return sock + + +async def await_without_task(coro): + exc = None + def func(): + try: + for _ in coro.__await__(): + pass + except BaseException as err: + nonlocal exc + exc = err + asyncio.get_running_loop().call_soon(func) + await asyncio.sleep(0) + if exc is not None: + raise exc + + +if sys.platform == 'win32': + DefaultEventLoopPolicy = asyncio.windows_events._DefaultEventLoopPolicy +else: + DefaultEventLoopPolicy = asyncio.unix_events._DefaultEventLoopPolicy diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py deleted file mode 100644 index bd43463da31..00000000000 --- a/Lib/test/test_asyncore.py +++ /dev/null @@ -1,838 +0,0 @@ -import asyncore -import unittest -import select -import os -import socket -import sys -import time -import errno -import struct -import threading - -from test import support -from test.support import os_helper -from test.support import socket_helper -from test.support import threading_helper -from test.support import warnings_helper -from io import BytesIO - -if support.PGO: - raise unittest.SkipTest("test is not helpful for PGO") - - -TIMEOUT = 3 -HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') - -class dummysocket: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def fileno(self): - return 42 - -class dummychannel: - def __init__(self): - self.socket = dummysocket() - - def close(self): - self.socket.close() - -class exitingdummy: - def __init__(self): - pass - - def handle_read_event(self): - raise asyncore.ExitNow() - - handle_write_event = handle_read_event - handle_close = handle_read_event - handle_expt_event = handle_read_event - -class crashingdummy: - def __init__(self): - self.error_handled = False - - def handle_read_event(self): - raise Exception() - - handle_write_event = handle_read_event - handle_close = handle_read_event - handle_expt_event = handle_read_event - - def handle_error(self): - self.error_handled = True - -# used when testing senders; just collects what it gets until newline is sent -def capture_server(evt, buf, serv): - try: - serv.listen() - conn, addr = serv.accept() - except socket.timeout: - pass - else: - n = 200 - start = time.monotonic() - while n > 0 and time.monotonic() - start < 3.0: - r, w, e = select.select([conn], [], [], 0.1) - if r: - n -= 1 - data = conn.recv(10) - # keep everything except for the newline terminator - buf.write(data.replace(b'\n', b'')) - if b'\n' in data: - break - time.sleep(0.01) - - conn.close() - finally: - serv.close() - evt.set() - -def bind_af_aware(sock, addr): - """Helper function to bind a socket according to its family.""" - if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: - # Make sure the path doesn't exist. - os_helper.unlink(addr) - socket_helper.bind_unix_socket(sock, addr) - else: - sock.bind(addr) - - -class HelperFunctionTests(unittest.TestCase): - def test_readwriteexc(self): - # Check exception handling behavior of read, write and _exception - - # check that ExitNow exceptions in the object handler method - # bubbles all the way up through asyncore read/write/_exception calls - tr1 = exitingdummy() - self.assertRaises(asyncore.ExitNow, asyncore.read, tr1) - self.assertRaises(asyncore.ExitNow, asyncore.write, tr1) - self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1) - - # check that an exception other than ExitNow in the object handler - # method causes the handle_error method to get called - tr2 = crashingdummy() - asyncore.read(tr2) - self.assertEqual(tr2.error_handled, True) - - tr2 = crashingdummy() - asyncore.write(tr2) - self.assertEqual(tr2.error_handled, True) - - tr2 = crashingdummy() - asyncore._exception(tr2) - self.assertEqual(tr2.error_handled, True) - - # asyncore.readwrite uses constants in the select module that - # are not present in Windows systems (see this thread: - # http://mail.python.org/pipermail/python-list/2001-October/109973.html) - # These constants should be present as long as poll is available - - @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') - def test_readwrite(self): - # Check that correct methods are called by readwrite() - - attributes = ('read', 'expt', 'write', 'closed', 'error_handled') - - expected = ( - (select.POLLIN, 'read'), - (select.POLLPRI, 'expt'), - (select.POLLOUT, 'write'), - (select.POLLERR, 'closed'), - (select.POLLHUP, 'closed'), - (select.POLLNVAL, 'closed'), - ) - - class testobj: - def __init__(self): - self.read = False - self.write = False - self.closed = False - self.expt = False - self.error_handled = False - - def handle_read_event(self): - self.read = True - - def handle_write_event(self): - self.write = True - - def handle_close(self): - self.closed = True - - def handle_expt_event(self): - self.expt = True - - def handle_error(self): - self.error_handled = True - - for flag, expectedattr in expected: - tobj = testobj() - self.assertEqual(getattr(tobj, expectedattr), False) - asyncore.readwrite(tobj, flag) - - # Only the attribute modified by the routine we expect to be - # called should be True. - for attr in attributes: - self.assertEqual(getattr(tobj, attr), attr==expectedattr) - - # check that ExitNow exceptions in the object handler method - # bubbles all the way up through asyncore readwrite call - tr1 = exitingdummy() - self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag) - - # check that an exception other than ExitNow in the object handler - # method causes the handle_error method to get called - tr2 = crashingdummy() - self.assertEqual(tr2.error_handled, False) - asyncore.readwrite(tr2, flag) - self.assertEqual(tr2.error_handled, True) - - def test_closeall(self): - self.closeall_check(False) - - def test_closeall_default(self): - self.closeall_check(True) - - def closeall_check(self, usedefault): - # Check that close_all() closes everything in a given map - - l = [] - testmap = {} - for i in range(10): - c = dummychannel() - l.append(c) - self.assertEqual(c.socket.closed, False) - testmap[i] = c - - if usedefault: - socketmap = asyncore.socket_map - try: - asyncore.socket_map = testmap - asyncore.close_all() - finally: - testmap, asyncore.socket_map = asyncore.socket_map, socketmap - else: - asyncore.close_all(testmap) - - self.assertEqual(len(testmap), 0) - - for c in l: - self.assertEqual(c.socket.closed, True) - - def test_compact_traceback(self): - try: - raise Exception("I don't like spam!") - except: - real_t, real_v, real_tb = sys.exc_info() - r = asyncore.compact_traceback() - else: - self.fail("Expected exception") - - (f, function, line), t, v, info = r - self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py') - self.assertEqual(function, 'test_compact_traceback') - self.assertEqual(t, real_t) - self.assertEqual(v, real_v) - self.assertEqual(info, '[%s|%s|%s]' % (f, function, line)) - - -class DispatcherTests(unittest.TestCase): - def setUp(self): - pass - - def tearDown(self): - asyncore.close_all() - - def test_basic(self): - d = asyncore.dispatcher() - self.assertEqual(d.readable(), True) - self.assertEqual(d.writable(), True) - - def test_repr(self): - d = asyncore.dispatcher() - self.assertEqual(repr(d), '' % id(d)) - - def test_log(self): - d = asyncore.dispatcher() - - # capture output of dispatcher.log() (to stderr) - l1 = "Lovely spam! Wonderful spam!" - l2 = "I don't like spam!" - with support.captured_stderr() as stderr: - d.log(l1) - d.log(l2) - - lines = stderr.getvalue().splitlines() - self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2]) - - def test_log_info(self): - d = asyncore.dispatcher() - - # capture output of dispatcher.log_info() (to stdout via print) - l1 = "Have you got anything without spam?" - l2 = "Why can't she have egg bacon spam and sausage?" - l3 = "THAT'S got spam in it!" - with support.captured_stdout() as stdout: - d.log_info(l1, 'EGGS') - d.log_info(l2) - d.log_info(l3, 'SPAM') - - lines = stdout.getvalue().splitlines() - expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3] - self.assertEqual(lines, expected) - - def test_unhandled(self): - d = asyncore.dispatcher() - d.ignore_log_types = () - - # capture output of dispatcher.log_info() (to stdout via print) - with support.captured_stdout() as stdout: - d.handle_expt() - d.handle_read() - d.handle_write() - d.handle_connect() - - lines = stdout.getvalue().splitlines() - expected = ['warning: unhandled incoming priority event', - 'warning: unhandled read event', - 'warning: unhandled write event', - 'warning: unhandled connect event'] - self.assertEqual(lines, expected) - - def test_strerror(self): - # refers to bug #8573 - err = asyncore._strerror(errno.EPERM) - if hasattr(os, 'strerror'): - self.assertEqual(err, os.strerror(errno.EPERM)) - err = asyncore._strerror(-1) - self.assertTrue(err != "") - - -class dispatcherwithsend_noread(asyncore.dispatcher_with_send): - def readable(self): - return False - - def handle_connect(self): - pass - - -class DispatcherWithSendTests(unittest.TestCase): - def setUp(self): - pass - - def tearDown(self): - asyncore.close_all() - - @threading_helper.reap_threads - def test_send(self): - evt = threading.Event() - sock = socket.socket() - sock.settimeout(3) - port = socket_helper.bind_port(sock) - - cap = BytesIO() - args = (evt, cap, sock) - t = threading.Thread(target=capture_server, args=args) - t.start() - try: - # wait a little longer for the server to initialize (it sometimes - # refuses connections on slow machines without this wait) - time.sleep(0.2) - - data = b"Suppose there isn't a 16-ton weight?" - d = dispatcherwithsend_noread() - d.create_socket() - d.connect((socket_helper.HOST, port)) - - # give time for socket to connect - time.sleep(0.1) - - d.send(data) - d.send(data) - d.send(b'\n') - - n = 1000 - while d.out_buffer and n > 0: - asyncore.poll() - n -= 1 - - evt.wait() - - self.assertEqual(cap.getvalue(), data*2) - finally: - threading_helper.join_thread(t) - - -@unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), - 'asyncore.file_wrapper required') -class FileWrapperTest(unittest.TestCase): - def setUp(self): - self.d = b"It's not dead, it's sleeping!" - with open(os_helper.TESTFN, 'wb') as file: - file.write(self.d) - - def tearDown(self): - os_helper.unlink(os_helper.TESTFN) - - def test_recv(self): - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - w = asyncore.file_wrapper(fd) - os.close(fd) - - self.assertNotEqual(w.fd, fd) - self.assertNotEqual(w.fileno(), fd) - self.assertEqual(w.recv(13), b"It's not dead") - self.assertEqual(w.read(6), b", it's") - w.close() - self.assertRaises(OSError, w.read, 1) - - def test_send(self): - d1 = b"Come again?" - d2 = b"I want to buy some cheese." - fd = os.open(os_helper.TESTFN, os.O_WRONLY | os.O_APPEND) - w = asyncore.file_wrapper(fd) - os.close(fd) - - w.write(d1) - w.send(d2) - w.close() - with open(os_helper.TESTFN, 'rb') as file: - self.assertEqual(file.read(), self.d + d1 + d2) - - @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'), - 'asyncore.file_dispatcher required') - def test_dispatcher(self): - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - data = [] - class FileDispatcher(asyncore.file_dispatcher): - def handle_read(self): - data.append(self.recv(29)) - s = FileDispatcher(fd) - os.close(fd) - asyncore.loop(timeout=0.01, use_poll=True, count=2) - self.assertEqual(b"".join(data), self.d) - - def test_resource_warning(self): - # Issue #11453 - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - f = asyncore.file_wrapper(fd) - - os.close(fd) - with warnings_helper.check_warnings(('', ResourceWarning)): - f = None - support.gc_collect() - - def test_close_twice(self): - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - f = asyncore.file_wrapper(fd) - os.close(fd) - - os.close(f.fd) # file_wrapper dupped fd - with self.assertRaises(OSError): - f.close() - - self.assertEqual(f.fd, -1) - # calling close twice should not fail - f.close() - - -class BaseTestHandler(asyncore.dispatcher): - - def __init__(self, sock=None): - asyncore.dispatcher.__init__(self, sock) - self.flag = False - - def handle_accept(self): - raise Exception("handle_accept not supposed to be called") - - def handle_accepted(self): - raise Exception("handle_accepted not supposed to be called") - - def handle_connect(self): - raise Exception("handle_connect not supposed to be called") - - def handle_expt(self): - raise Exception("handle_expt not supposed to be called") - - def handle_close(self): - raise Exception("handle_close not supposed to be called") - - def handle_error(self): - raise - - -class BaseServer(asyncore.dispatcher): - """A server which listens on an address and dispatches the - connection to a handler. - """ - - def __init__(self, family, addr, handler=BaseTestHandler): - asyncore.dispatcher.__init__(self) - self.create_socket(family) - self.set_reuse_addr() - bind_af_aware(self.socket, addr) - self.listen(5) - self.handler = handler - - @property - def address(self): - return self.socket.getsockname() - - def handle_accepted(self, sock, addr): - self.handler(sock) - - def handle_error(self): - raise - - -class BaseClient(BaseTestHandler): - - def __init__(self, family, address): - BaseTestHandler.__init__(self) - self.create_socket(family) - self.connect(address) - - def handle_connect(self): - pass - - -class BaseTestAPI: - - def tearDown(self): - asyncore.close_all(ignore_all=True) - - def loop_waiting_for_flag(self, instance, timeout=5): - timeout = float(timeout) / 100 - count = 100 - while asyncore.socket_map and count > 0: - asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) - if instance.flag: - return - count -= 1 - time.sleep(timeout) - self.fail("flag not set") - - def test_handle_connect(self): - # make sure handle_connect is called on connect() - - class TestClient(BaseClient): - def handle_connect(self): - self.flag = True - - server = BaseServer(self.family, self.addr) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_accept(self): - # make sure handle_accept() is called when a client connects - - class TestListener(BaseTestHandler): - - def __init__(self, family, addr): - BaseTestHandler.__init__(self) - self.create_socket(family) - bind_af_aware(self.socket, addr) - self.listen(5) - self.address = self.socket.getsockname() - - def handle_accept(self): - self.flag = True - - server = TestListener(self.family, self.addr) - client = BaseClient(self.family, server.address) - self.loop_waiting_for_flag(server) - - def test_handle_accepted(self): - # make sure handle_accepted() is called when a client connects - - class TestListener(BaseTestHandler): - - def __init__(self, family, addr): - BaseTestHandler.__init__(self) - self.create_socket(family) - bind_af_aware(self.socket, addr) - self.listen(5) - self.address = self.socket.getsockname() - - def handle_accept(self): - asyncore.dispatcher.handle_accept(self) - - def handle_accepted(self, sock, addr): - sock.close() - self.flag = True - - server = TestListener(self.family, self.addr) - client = BaseClient(self.family, server.address) - self.loop_waiting_for_flag(server) - - - def test_handle_read(self): - # make sure handle_read is called on data received - - class TestClient(BaseClient): - def handle_read(self): - self.flag = True - - class TestHandler(BaseTestHandler): - def __init__(self, conn): - BaseTestHandler.__init__(self, conn) - self.send(b'x' * 1024) - - server = BaseServer(self.family, self.addr, TestHandler) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_write(self): - # make sure handle_write is called - - class TestClient(BaseClient): - def handle_write(self): - self.flag = True - - server = BaseServer(self.family, self.addr) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_close(self): - # make sure handle_close is called when the other end closes - # the connection - - class TestClient(BaseClient): - - def handle_read(self): - # in order to make handle_close be called we are supposed - # to make at least one recv() call - self.recv(1024) - - def handle_close(self): - self.flag = True - self.close() - - class TestHandler(BaseTestHandler): - def __init__(self, conn): - BaseTestHandler.__init__(self, conn) - self.close() - - server = BaseServer(self.family, self.addr, TestHandler) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_close_after_conn_broken(self): - # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and - # #11265). - - data = b'\0' * 128 - - class TestClient(BaseClient): - - def handle_write(self): - self.send(data) - - def handle_close(self): - self.flag = True - self.close() - - def handle_expt(self): - self.flag = True - self.close() - - class TestHandler(BaseTestHandler): - - def handle_read(self): - self.recv(len(data)) - self.close() - - def writable(self): - return False - - server = BaseServer(self.family, self.addr, TestHandler) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - @unittest.skipIf(sys.platform.startswith("sunos"), - "OOB support is broken on Solaris") - def test_handle_expt(self): - # Make sure handle_expt is called on OOB data received. - # Note: this might fail on some platforms as OOB data is - # tenuously supported and rarely used. - if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: - self.skipTest("Not applicable to AF_UNIX sockets.") - - if sys.platform == "darwin" and self.use_poll: - self.skipTest("poll may fail on macOS; see issue #28087") - - class TestClient(BaseClient): - def handle_expt(self): - self.socket.recv(1024, socket.MSG_OOB) - self.flag = True - - class TestHandler(BaseTestHandler): - def __init__(self, conn): - BaseTestHandler.__init__(self, conn) - self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB) - - server = BaseServer(self.family, self.addr, TestHandler) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_error(self): - - class TestClient(BaseClient): - def handle_write(self): - 1.0 / 0 - def handle_error(self): - self.flag = True - try: - raise - except ZeroDivisionError: - pass - else: - raise Exception("exception not raised") - - server = BaseServer(self.family, self.addr) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_connection_attributes(self): - server = BaseServer(self.family, self.addr) - client = BaseClient(self.family, server.address) - - # we start disconnected - self.assertFalse(server.connected) - self.assertTrue(server.accepting) - # this can't be taken for granted across all platforms - #self.assertFalse(client.connected) - self.assertFalse(client.accepting) - - # execute some loops so that client connects to server - asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100) - self.assertFalse(server.connected) - self.assertTrue(server.accepting) - self.assertTrue(client.connected) - self.assertFalse(client.accepting) - - # disconnect the client - client.close() - self.assertFalse(server.connected) - self.assertTrue(server.accepting) - self.assertFalse(client.connected) - self.assertFalse(client.accepting) - - # stop serving - server.close() - self.assertFalse(server.connected) - self.assertFalse(server.accepting) - - def test_create_socket(self): - s = asyncore.dispatcher() - s.create_socket(self.family) - self.assertEqual(s.socket.type, socket.SOCK_STREAM) - self.assertEqual(s.socket.family, self.family) - self.assertEqual(s.socket.gettimeout(), 0) - self.assertFalse(s.socket.get_inheritable()) - - def test_bind(self): - if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: - self.skipTest("Not applicable to AF_UNIX sockets.") - s1 = asyncore.dispatcher() - s1.create_socket(self.family) - s1.bind(self.addr) - s1.listen(5) - port = s1.socket.getsockname()[1] - - s2 = asyncore.dispatcher() - s2.create_socket(self.family) - # EADDRINUSE indicates the socket was correctly bound - self.assertRaises(OSError, s2.bind, (self.addr[0], port)) - - def test_set_reuse_addr(self): - if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: - self.skipTest("Not applicable to AF_UNIX sockets.") - - with socket.socket(self.family) as sock: - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except OSError: - unittest.skip("SO_REUSEADDR not supported on this platform") - else: - # if SO_REUSEADDR succeeded for sock we expect asyncore - # to do the same - s = asyncore.dispatcher(socket.socket(self.family)) - self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - s.socket.close() - s.create_socket(self.family) - s.set_reuse_addr() - self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - - @threading_helper.reap_threads - def test_quick_connect(self): - # see: http://bugs.python.org/issue10340 - if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): - self.skipTest("test specific to AF_INET and AF_INET6") - - server = BaseServer(self.family, self.addr) - # run the thread 500 ms: the socket should be connected in 200 ms - t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, - count=5)) - t.start() - try: - with socket.socket(self.family, socket.SOCK_STREAM) as s: - s.settimeout(.2) - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) - - try: - s.connect(server.address) - except OSError: - pass - finally: - threading_helper.join_thread(t) - -class TestAPI_UseIPv4Sockets(BaseTestAPI): - family = socket.AF_INET - addr = (socket_helper.HOST, 0) - -@unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 support required') -class TestAPI_UseIPv6Sockets(BaseTestAPI): - family = socket.AF_INET6 - addr = (socket_helper.HOSTv6, 0) - -@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required') -class TestAPI_UseUnixSockets(BaseTestAPI): - if HAS_UNIX_SOCKETS: - family = socket.AF_UNIX - addr = os_helper.TESTFN - - def tearDown(self): - os_helper.unlink(self.addr) - BaseTestAPI.tearDown(self) - -class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets, unittest.TestCase): - use_poll = False - -@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') -class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets, unittest.TestCase): - use_poll = True - -class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets, unittest.TestCase): - use_poll = False - -@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') -class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets, unittest.TestCase): - use_poll = True - -class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets, unittest.TestCase): - use_poll = False - -@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') -class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets, unittest.TestCase): - use_poll = True - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 913b7556be8..6f57ee06879 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -1,10 +1,12 @@ import atexit import os +import subprocess import textwrap import unittest +from test.support import os_helper from test import support -from test.support import script_helper - +from test.support import SuppressCrashReport, script_helper +from test.support import threading_helper class GeneralTest(unittest.TestCase): def test_general(self): @@ -46,6 +48,40 @@ def test_atexit_instances(self): self.assertEqual(res.out.decode().splitlines(), ["atexit2", "atexit1"]) self.assertFalse(res.err) + @unittest.skip("TODO: RUSTPYTHON; Flakey on CI") + @threading_helper.requires_working_threading() + @support.requires_resource("cpu") + @unittest.skipUnless(support.Py_GIL_DISABLED, "only meaningful without the GIL") + def test_atexit_thread_safety(self): + # GH-126907: atexit was not thread safe on the free-threaded build + source = """ + from threading import Thread + + def dummy(): + pass + + + def thready(): + for _ in range(100): + atexit.register(dummy) + atexit._clear() + atexit.register(dummy) + atexit.unregister(dummy) + atexit._run_exitfuncs() + + + threads = [Thread(target=thready) for _ in range(10)] + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + """ + + # atexit._clear() has some evil side effects, and we don't + # want them to affect the rest of the tests. + script_helper.assert_python_ok("-c", textwrap.dedent(source)) + @support.cpython_only class SubinterpreterTest(unittest.TestCase): @@ -100,6 +136,37 @@ def callback(): self.assertEqual(os.read(r, len(expected)), expected) os.close(r) + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + def test_atexit_with_low_memory(self): + # gh-140080: Test that setting low memory after registering an atexit + # callback doesn't cause an infinite loop during finalization. + code = textwrap.dedent(""" + import atexit + import _testcapi + + def callback(): + print("hello") + + atexit.register(callback) + # Simulate low memory condition + _testcapi.set_nomemory(0) + """) + + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script(temp_dir, 'test_atexit_script', code) + with SuppressCrashReport(): + with script_helper.spawn_python(script, + stderr=subprocess.PIPE) as proc: + proc.wait() + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + self.assertIn(proc.returncode, (0, 1)) + self.assertNotIn(b"hello", stdout) + self.assertIn(b"MemoryError", stderr) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py new file mode 100644 index 00000000000..ddd9f951143 --- /dev/null +++ b/Lib/test/test_audit.py @@ -0,0 +1,318 @@ +"""Tests for sys.audit and sys.addaudithook +""" + +import subprocess +import sys +import unittest +from test import support +from test.support import import_helper +from test.support import os_helper + + +if not hasattr(sys, "addaudithook") or not hasattr(sys, "audit"): + raise unittest.SkipTest("test only relevant when sys.audit is available") + +AUDIT_TESTS_PY = support.findfile("audit-tests.py") + + +class AuditTest(unittest.TestCase): + maxDiff = None + + @support.requires_subprocess() + def run_test_in_subprocess(self, *args): + with subprocess.Popen( + [sys.executable, "-X utf8", AUDIT_TESTS_PY, *args], + encoding="utf-8", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as p: + p.wait() + return p, p.stdout.read(), p.stderr.read() + + def do_test(self, *args): + proc, stdout, stderr = self.run_test_in_subprocess(*args) + + sys.stdout.write(stdout) + sys.stderr.write(stderr) + if proc.returncode: + self.fail(stderr) + + def run_python(self, *args, expect_stderr=False): + events = [] + proc, stdout, stderr = self.run_test_in_subprocess(*args) + if not expect_stderr or support.verbose: + sys.stderr.write(stderr) + return ( + proc.returncode, + [line.strip().partition(" ") for line in stdout.splitlines()], + stderr, + ) + + def test_basic(self): + self.do_test("test_basic") + + def test_block_add_hook(self): + self.do_test("test_block_add_hook") + + def test_block_add_hook_baseexception(self): + self.do_test("test_block_add_hook_baseexception") + + def test_marshal(self): + import_helper.import_module("marshal") + + self.do_test("test_marshal") + + def test_pickle(self): + import_helper.import_module("pickle") + + self.do_test("test_pickle") + + def test_monkeypatch(self): + self.do_test("test_monkeypatch") + + def test_open(self): + self.do_test("test_open", os_helper.TESTFN) + + def test_cantrace(self): + self.do_test("test_cantrace") + + def test_mmap(self): + self.do_test("test_mmap") + + def test_excepthook(self): + returncode, events, stderr = self.run_python("test_excepthook") + if not returncode: + self.fail(f"Expected fatal exception\n{stderr}") + + self.assertSequenceEqual( + [("sys.excepthook", " ", "RuntimeError('fatal-error')")], events + ) + + def test_unraisablehook(self): + import_helper.import_module("_testcapi") + returncode, events, stderr = self.run_python("test_unraisablehook") + if returncode: + self.fail(stderr) + + self.assertEqual(events[0][0], "sys.unraisablehook") + self.assertEqual( + events[0][2], + "RuntimeError('nonfatal-error') Exception ignored for audit hook test", + ) + + def test_winreg(self): + import_helper.import_module("winreg") + returncode, events, stderr = self.run_python("test_winreg") + if returncode: + self.fail(stderr) + + self.assertEqual(events[0][0], "winreg.OpenKey") + self.assertEqual(events[1][0], "winreg.OpenKey/result") + expected = events[1][2] + self.assertTrue(expected) + self.assertSequenceEqual(["winreg.EnumKey", " ", f"{expected} 0"], events[2]) + self.assertSequenceEqual(["winreg.EnumKey", " ", f"{expected} 10000"], events[3]) + self.assertSequenceEqual(["winreg.PyHKEY.Detach", " ", expected], events[4]) + + def test_socket(self): + import_helper.import_module("socket") + returncode, events, stderr = self.run_python("test_socket") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + self.assertEqual(events[0][0], "socket.gethostname") + self.assertEqual(events[1][0], "socket.__new__") + self.assertEqual(events[2][0], "socket.bind") + self.assertTrue(events[2][2].endswith("('127.0.0.1', 8080)")) + + def test_gc(self): + returncode, events, stderr = self.run_python("test_gc") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + self.assertEqual( + [event[0] for event in events], + ["gc.get_objects", "gc.get_referrers", "gc.get_referents"] + ) + + + @support.requires_resource('network') + def test_http(self): + import_helper.import_module("http.client") + returncode, events, stderr = self.run_python("test_http_client") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + self.assertEqual(events[0][0], "http.client.connect") + self.assertEqual(events[0][2], "www.python.org 80") + self.assertEqual(events[1][0], "http.client.send") + if events[1][2] != '[cannot send]': + self.assertIn('HTTP', events[1][2]) + + + def test_sqlite3(self): + sqlite3 = import_helper.import_module("sqlite3") + returncode, events, stderr = self.run_python("test_sqlite3") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [ev[0] for ev in events] + expected = ["sqlite3.connect", "sqlite3.connect/handle"] * 2 + + if hasattr(sqlite3.Connection, "enable_load_extension"): + expected += [ + "sqlite3.enable_load_extension", + "sqlite3.load_extension", + ] + self.assertEqual(actual, expected) + + + def test_sys_getframe(self): + returncode, events, stderr = self.run_python("test_sys_getframe") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys._getframe", "test_sys_getframe")] + + self.assertEqual(actual, expected) + + def test_sys_getframemodulename(self): + returncode, events, stderr = self.run_python("test_sys_getframemodulename") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys._getframemodulename", "0")] + + self.assertEqual(actual, expected) + + + def test_threading(self): + returncode, events, stderr = self.run_python("test_threading") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [ + ("_thread.start_new_thread", "(, (), None)"), + ("test.test_func", "()"), + ("_thread.start_joinable_thread", "(, 1, None)"), + ("test.test_func", "()"), + ] + + self.assertEqual(actual, expected) + + + def test_wmi_exec_query(self): + import_helper.import_module("_wmi") + returncode, events, stderr = self.run_python("test_wmi_exec_query") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("_wmi.exec_query", "SELECT * FROM Win32_OperatingSystem")] + + self.assertEqual(actual, expected) + + def test_syslog(self): + syslog = import_helper.import_module("syslog") + + returncode, events, stderr = self.run_python("test_syslog") + if returncode: + self.fail(stderr) + + if support.verbose: + print('Events:', *events, sep='\n ') + + self.assertSequenceEqual( + events, + [('syslog.openlog', ' ', f'python 0 {syslog.LOG_USER}'), + ('syslog.syslog', ' ', f'{syslog.LOG_INFO} test'), + ('syslog.setlogmask', ' ', f'{syslog.LOG_DEBUG}'), + ('syslog.closelog', '', ''), + ('syslog.syslog', ' ', f'{syslog.LOG_INFO} test2'), + ('syslog.openlog', ' ', f'audit-tests.py 0 {syslog.LOG_USER}'), + ('syslog.openlog', ' ', f'audit-tests.py {syslog.LOG_NDELAY} {syslog.LOG_LOCAL0}'), + ('syslog.openlog', ' ', f'None 0 {syslog.LOG_USER}'), + ('syslog.closelog', '', '')] + ) + + def test_not_in_gc(self): + returncode, _, stderr = self.run_python("test_not_in_gc") + if returncode: + self.fail(stderr) + + def test_time(self): + returncode, events, stderr = self.run_python("test_time", "print") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + + actual = [(ev[0], ev[2]) for ev in events] + expected = [("time.sleep", "0"), + ("time.sleep", "0.0625"), + ("time.sleep", "-1")] + + self.assertEqual(actual, expected) + + def test_time_fail(self): + returncode, events, stderr = self.run_python("test_time", "fail", + expect_stderr=True) + self.assertNotEqual(returncode, 0) + self.assertIn('hook failed', stderr.splitlines()[-1]) + + def test_sys_monitoring_register_callback(self): + returncode, events, stderr = self.run_python("test_sys_monitoring_register_callback") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys.monitoring.register_callback", "(None,)")] + + self.assertEqual(actual, expected) + + def test_winapi_createnamedpipe(self): + winapi = import_helper.import_module("_winapi") + + pipe_name = r"\\.\pipe\LOCAL\test_winapi_createnamed_pipe" + returncode, events, stderr = self.run_python("test_winapi_createnamedpipe", pipe_name) + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("_winapi.CreateNamedPipe", f"({pipe_name!r}, 3, 8)")] + + self.assertEqual(actual, expected) + + def test_assert_unicode(self): + # See gh-126018 + returncode, _, stderr = self.run_python("test_assert_unicode") + if returncode: + self.fail(stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index fa03fa1d61c..a6739124571 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -1,10 +1,18 @@ -import unittest import base64 import binascii import os +import unittest from array import array +from test.support import cpython_only from test.support import os_helper from test.support import script_helper +from test.support.import_helper import ensure_lazy_imports + + +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("base64", {"re", "getopt"}) class LegacyBase64TestCase(unittest.TestCase): @@ -200,18 +208,6 @@ def test_b64decode(self): self.check_other_types(base64.b64decode, b"YWJj", b"abc") self.check_decode_type_errors(base64.b64decode) - # Test with arbitrary alternative characters - tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d', - } - for (data, altchars), res in tests_altchars.items(): - data_str = data.decode('ascii') - altchars_str = altchars.decode('ascii') - - eq(base64.b64decode(data, altchars=altchars), res) - eq(base64.b64decode(data_str, altchars=altchars), res) - eq(base64.b64decode(data, altchars=altchars_str), res) - eq(base64.b64decode(data_str, altchars=altchars_str), res) - # Test standard alphabet for data, res in tests.items(): eq(base64.standard_b64decode(data), res) @@ -232,6 +228,20 @@ def test_b64decode(self): b'\xd3V\xbeo\xf7\x1d') self.check_decode_type_errors(base64.urlsafe_b64decode) + def test_b64decode_altchars(self): + # Test with arbitrary alternative characters + eq = self.assertEqual + res = b'\xd3V\xbeo\xf7\x1d' + for altchars in b'*$', b'+/', b'/+', b'+_', b'-+', b'-/', b'/_': + data = b'01a%cb%ccd' % tuple(altchars) + data_str = data.decode('ascii') + altchars_str = altchars.decode('ascii') + + eq(base64.b64decode(data, altchars=altchars), res) + eq(base64.b64decode(data_str, altchars=altchars), res) + eq(base64.b64decode(data, altchars=altchars_str), res) + eq(base64.b64decode(data_str, altchars=altchars_str), res) + def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') self.assertRaises(binascii.Error, base64.b64decode, 'abc') @@ -264,9 +274,12 @@ def test_b64decode_invalid_chars(self): base64.b64decode(bstr.decode('ascii'), validate=True) # Normal alphabet characters not discarded when alternative given - res = b'\xFB\xEF\xBE\xFF\xFF\xFF' - self.assertEqual(base64.b64decode(b'++[[//]]', b'[]'), res) - self.assertEqual(base64.urlsafe_b64decode(b'++--//__'), res) + res = b'\xfb\xef\xff' + self.assertEqual(base64.b64decode(b'++//', validate=True), res) + self.assertEqual(base64.b64decode(b'++//', '-_', validate=True), res) + self.assertEqual(base64.b64decode(b'--__', '-_', validate=True), res) + self.assertEqual(base64.urlsafe_b64decode(b'++//'), res) + self.assertEqual(base64.urlsafe_b64decode(b'--__'), res) def test_b32encode(self): eq = self.assertEqual @@ -321,23 +334,33 @@ def test_b32decode_casefold(self): self.assertRaises(binascii.Error, base64.b32decode, b'me======') self.assertRaises(binascii.Error, base64.b32decode, 'me======') + def test_b32decode_map01(self): # Mapping zero and one - eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') - eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe') - - map_tests = {(b'M1023456', b'L'): b'b\xdd\xad\xf3\xbe', - (b'M1023456', b'I'): b'b\x1d\xad\xf3\xbe', - } - for (data, map01), res in map_tests.items(): - data_str = data.decode('ascii') + eq = self.assertEqual + res_L = b'b\xdd\xad\xf3\xbe' + res_I = b'b\x1d\xad\xf3\xbe' + eq(base64.b32decode(b'MLO23456'), res_L) + eq(base64.b32decode('MLO23456'), res_L) + eq(base64.b32decode(b'MIO23456'), res_I) + eq(base64.b32decode('MIO23456'), res_I) + self.assertRaises(binascii.Error, base64.b32decode, b'M1023456') + self.assertRaises(binascii.Error, base64.b32decode, b'M1O23456') + self.assertRaises(binascii.Error, base64.b32decode, b'ML023456') + self.assertRaises(binascii.Error, base64.b32decode, b'MI023456') + + data = b'M1023456' + data_str = data.decode('ascii') + for map01, res in [(b'L', res_L), (b'I', res_I)]: map01_str = map01.decode('ascii') eq(base64.b32decode(data, map01=map01), res) eq(base64.b32decode(data_str, map01=map01), res) eq(base64.b32decode(data, map01=map01_str), res) eq(base64.b32decode(data_str, map01=map01_str), res) - self.assertRaises(binascii.Error, base64.b32decode, data) - self.assertRaises(binascii.Error, base64.b32decode, data_str) + + eq(base64.b32decode(b'M1O23456', map01=map01), res) + eq(base64.b32decode(b'M%c023456' % map01, map01=map01), res) + eq(base64.b32decode(b'M%cO23456' % map01, map01=map01), res) def test_b32decode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF'] @@ -545,6 +568,40 @@ def test_b85encode(self): self.check_other_types(base64.b85encode, b"www.python.org", b'cXxL#aCvlSZ*DGca%T') + def test_z85encode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'www.python.org': b'CxXl-AcVLsz/dgCA+t', + bytes(range(255)): b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""", + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""": + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""", + b'no padding..': b'zF{UpvpS[.zF7NO', + b'zero compression\x00\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg00000', + b'zero compression\x00\x00\x00': b'Ds.bnay/tbAb]JhB7]Mg0000', + b"""Boundary:\x00\x00\x00\x00""": b"""lt}0:wmoI7iSGcW00""", + b'Space compr: ': b'q/DePwGUG3ze:IRarR^H', + b'\xff': b'@@', + b'\xff'*2: b'%nJ', + b'\xff'*3: b'%nS9', + b'\xff'*4: b'%nSc0', + } + + for data, res in tests.items(): + eq(base64.z85encode(data), res) + + self.check_other_types(base64.z85encode, b"www.python.org", + b'CxXl-AcVLsz/dgCA+t') + def test_a85decode(self): eq = self.assertEqual @@ -586,6 +643,7 @@ def test_a85decode(self): eq(base64.a85decode(b'y+', b"www.python.org") @@ -625,6 +683,41 @@ def test_b85decode(self): self.check_other_types(base64.b85decode, b'cXxL#aCvlSZ*DGca%T', b"www.python.org") + def test_z85decode(self): + eq = self.assertEqual + + tests = { + b'': b'', + b'CxXl-AcVLsz/dgCA+t': b'www.python.org', + b"""009c61o!#m2NH?C3>iWS5d]J*6CRx17-skh9337x""" + b"""ar.{NbQB=+c[cR@eg&FcfFLssg=mfIi5%2YjuU>)kTv.7l}6Nnnj=AD""" + b"""oIFnTp/ga?r8($2sxO*itWpVyu$0IOwmYv=xLzi%y&a6dAb/]tBAI+J""" + b"""CZjQZE0{D[FpSr8GOteoH(41EJe-&}x#)cTlf[Bu8v].4}L}1:^-""" + b"""@qDP""": bytes(range(255)), + b"""vpA.SwObN*x>?B1zeKohADlbxB-}$ND3R+ylQTvjm[uizoh55PpF:[^""" + b"""q=D:$s6eQefFLssg=mfIi5@cEbqrBJdKV-ciY]OSe*aw7DWL""": + b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ""" + b"""0123456789!@#0^&*();:<>,. []{}""", + b'zF{UpvpS[.zF7NO': b'no padding..', + b'Ds.bnay/tbAb]JhB7]Mg00000': b'zero compression\x00\x00\x00\x00', + b'Ds.bnay/tbAb]JhB7]Mg0000': b'zero compression\x00\x00\x00', + b"""lt}0:wmoI7iSGcW00""": b"""Boundary:\x00\x00\x00\x00""", + b'q/DePwGUG3ze:IRarR^H': b'Space compr: ', + b'@@': b'\xff', + b'%nJ': b'\xff'*2, + b'%nS9': b'\xff'*3, + b'%nSc0': b'\xff'*4, + } + + for data, res in tests.items(): + eq(base64.z85decode(data), res) + eq(base64.z85decode(data.decode("ascii")), res) + + self.check_other_types(base64.z85decode, b'CxXl-AcVLsz/dgCA+t', + b'www.python.org') + def test_a85_padding(self): eq = self.assertEqual @@ -689,6 +782,8 @@ def test_a85decode_errors(self): self.assertRaises(ValueError, base64.a85decode, b's8W', adobe=False) self.assertRaises(ValueError, base64.a85decode, b's8W-', adobe=False) self.assertRaises(ValueError, base64.a85decode, b's8W-"', adobe=False) + self.assertRaises(ValueError, base64.a85decode, b'aaaay', + foldspaces=True) def test_b85decode_errors(self): illegal = list(range(33)) + \ @@ -704,6 +799,21 @@ def test_b85decode_errors(self): self.assertRaises(ValueError, base64.b85decode, b'|NsC') self.assertRaises(ValueError, base64.b85decode, b'|NsC1') + def test_z85decode_errors(self): + illegal = list(range(33)) + \ + list(b'"\',;_`|\\~') + \ + list(range(128, 256)) + for c in illegal: + with self.assertRaises(ValueError, msg=bytes([c])): + base64.z85decode(b'0000' + bytes([c])) + + # b'\xff\xff\xff\xff' encodes to b'%nSc0', the following will overflow: + self.assertRaises(ValueError, base64.z85decode, b'%') + self.assertRaises(ValueError, base64.z85decode, b'%n') + self.assertRaises(ValueError, base64.z85decode, b'%nS') + self.assertRaises(ValueError, base64.z85decode, b'%nSc') + self.assertRaises(ValueError, base64.z85decode, b'%nSc1') + def test_decode_nonascii_str(self): decode_funcs = (base64.b64decode, base64.standard_b64decode, @@ -711,12 +821,13 @@ def test_decode_nonascii_str(self): base64.b32decode, base64.b16decode, base64.b85decode, - base64.a85decode) + base64.a85decode, + base64.z85decode) for f in decode_funcs: self.assertRaises(ValueError, f, 'with non-ascii \xcb') def test_ErrorHeritage(self): - self.assertTrue(issubclass(binascii.Error, ValueError)) + self.assertIsSubclass(binascii.Error, ValueError) def test_RFC4648_test_cases(self): # test cases from RFC 4648 section 10 diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index 2be9405bcc2..5870dc7f9da 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -10,16 +10,12 @@ class ExceptionClassTests(unittest.TestCase): inheritance hierarchy)""" def test_builtins_new_style(self): - self.assertTrue(issubclass(Exception, object)) + self.assertIsSubclass(Exception, object) def verify_instance_interface(self, ins): for attr in ("args", "__str__", "__repr__"): - self.assertTrue(hasattr(ins, attr), - "%s missing %s attribute" % - (ins.__class__.__name__, attr)) + self.assertHasAttr(ins, attr) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_inheritance(self): # Make sure the inheritance hierarchy matches the documentation exc_set = set() @@ -67,7 +63,7 @@ def test_inheritance(self): elif last_depth > depth: while superclasses[-1][0] >= depth: superclasses.pop() - self.assertTrue(issubclass(exc, superclasses[-1][1]), + self.assertIsSubclass(exc, superclasses[-1][1], "%s is not a subclass of %s" % (exc.__name__, superclasses[-1][1].__name__)) try: # Some exceptions require arguments; just skip them @@ -81,9 +77,12 @@ def test_inheritance(self): finally: inheritance_tree.close() + # Underscore-prefixed (private) exceptions don't need to be documented + exc_set = set(e for e in exc_set if not e.startswith('_')) # RUSTPYTHON specific exc_set.discard("JitError") - + # XXX: RUSTPYTHON; IncompleteInputError will be officially introduced in Python 3.15 + exc_set.discard("IncompleteInputError") self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set) interface_tests = ("length", "args", "str", "repr") @@ -120,8 +119,6 @@ def test_interface_no_arg(self): [repr(exc), exc.__class__.__name__ + '()']) self.interface_test_driver(results) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_setstate_refcount_no_crash(self): # gh-97591: Acquire strong reference before calling tp_hash slot # in PyObject_SetAttr. @@ -137,7 +134,7 @@ class Value(str): d[HashThisKeyWillClearTheDict()] = Value() # refcount of Value() is 1 now - # Exception.__setstate__ should aquire a strong reference of key and + # Exception.__setstate__ should acquire a strong reference of key and # value in the dict. Otherwise, Value()'s refcount would go below # zero in the tp_hash call in PyObject_SetAttr(), and it would cause # crash in GC. @@ -196,8 +193,6 @@ def test_raise_string(self): # Raising a string raises TypeError. self.raise_fails("spam") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_catch_non_BaseException(self): # Trying to catch an object that does not inherit from BaseException # is not allowed. diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index a3abbbb8db2..f15dae13eb3 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -228,6 +228,10 @@ def user_exception(self, frame, exc_info): self.process_event('exception', frame) self.next_set_method() + def user_opcode(self, frame): + self.process_event('opcode', frame) + self.next_set_method() + def do_clear(self, arg): # The temporary breakpoints are deleted in user_line(). bp_list = [self.currentbp] @@ -366,7 +370,7 @@ def next_set_method(self): set_method = getattr(self, 'set_' + set_type) # The following set methods give back control to the tracer. - if set_type in ('step', 'continue', 'quit'): + if set_type in ('step', 'stepinstr', 'continue', 'quit'): set_method() return elif set_type in ('next', 'return'): @@ -586,7 +590,6 @@ def fail(self, msg=None): class StateTestCase(BaseTestCase): """Test the step, next, return, until and quit 'set_' methods.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_step(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -597,7 +600,6 @@ def test_step(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_step_next_on_last_statement(self): for set_type in ('step', 'next'): with self.subTest(set_type=set_type): @@ -612,7 +614,15 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + def test_stepinstr(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('stepinstr', ), + ('opcode', 2, 'tfunc_main'), ('next', ), + ('line', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -624,7 +634,6 @@ def test_next(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_next_over_import(self): code = """ def main(): @@ -639,7 +648,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_next_on_plain_statement(self): # Check that set_next() is equivalent to set_step() on a plain # statement. @@ -652,7 +660,6 @@ def test_next_on_plain_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_next_in_caller_frame(self): # Check that set_next() in the caller frame causes the tracer # to stop next in the caller frame. @@ -666,7 +673,6 @@ def test_next_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_return(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -679,7 +685,6 @@ def test_return(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_return_in_caller_frame(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -691,7 +696,6 @@ def test_return_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_until(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -703,7 +707,6 @@ def test_until(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_until_with_too_large_count(self): self.expect_set = [ ('line', 2, 'tfunc_main'), break_in_func('tfunc_first'), @@ -714,7 +717,6 @@ def test_until_with_too_large_count(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_until_in_caller_frame(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -726,7 +728,7 @@ def test_until_in_caller_frame(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @patch_list(sys.meta_path) def test_skip(self): # Check that tracing is skipped over the import statement in # 'tfunc_import()'. @@ -759,7 +761,6 @@ def test_skip_with_no_name_module(self): bdb = Bdb(skip=['anything*']) self.assertIs(bdb.is_skipped_module(None), False) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_down(self): # Check that set_down() raises BdbError at the newest frame. self.expect_set = [ @@ -768,7 +769,6 @@ def test_down(self): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_main) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_up(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), @@ -782,7 +782,6 @@ def test_up(self): class BreakpointTestCase(BaseTestCase): """Test the breakpoint set method.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_bp_on_non_existent_module(self): self.expect_set = [ ('line', 2, 'tfunc_import'), ('break', ('/non/existent/module.py', 1)) @@ -790,7 +789,6 @@ def test_bp_on_non_existent_module(self): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_bp_after_last_statement(self): code = """ def main(): @@ -804,7 +802,6 @@ def main(): with TracerRun(self) as tracer: self.assertRaises(BdbError, tracer.runcall, tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_temporary_bp(self): code = """ def func(): @@ -828,7 +825,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_disabled_temporary_bp(self): code = """ def func(): @@ -857,7 +853,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_bp_condition(self): code = """ def func(a): @@ -878,7 +873,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_bp_exception_on_condition_evaluation(self): code = """ def func(a): @@ -898,7 +892,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_bp_ignore_count(self): code = """ def func(): @@ -920,7 +913,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_ignore_count_on_disabled_bp(self): code = """ def func(): @@ -948,7 +940,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_clear_two_bp_on_same_line(self): code = """ def func(): @@ -974,7 +965,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_clear_at_no_bp(self): self.expect_set = [ ('line', 2, 'tfunc_import'), ('clear', (__file__, 1)) @@ -1028,7 +1018,6 @@ def test_load_bps_from_previous_Bdb_instance(self): class RunTestCase(BaseTestCase): """Test run, runeval and set_trace.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_run_step(self): # Check that the bdb 'run' method stops at the first line event. code = """ @@ -1041,7 +1030,6 @@ def test_run_step(self): with TracerRun(self) as tracer: tracer.run(compile(textwrap.dedent(code), '', 'exec')) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_runeval_step(self): # Test bdb 'runeval'. code = """ @@ -1058,13 +1046,13 @@ def main(): ('return', 1, ''), ('quit', ), ] import test_module_for_bdb + ns = {'test_module_for_bdb': test_module_for_bdb} with TracerRun(self) as tracer: - tracer.runeval('test_module_for_bdb.main()', globals(), locals()) + tracer.runeval('test_module_for_bdb.main()', ns, ns) class IssuesTestCase(BaseTestCase): """Test fixed bdb issues.""" - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_step_at_return_with_no_trace_in_caller(self): # Issue #13183. # Check that the tracer does step into the caller frame when the @@ -1095,7 +1083,6 @@ def func(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_next_until_return_in_generator(self): # Issue #16596. # Check that set_next(), set_until() and set_return() do not treat the @@ -1137,7 +1124,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_next_command_in_generator_for_loop(self): # Issue #16596. code = """ @@ -1169,7 +1155,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_next_command_in_generator_with_subiterator(self): # Issue #16596. code = """ @@ -1201,7 +1186,6 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_return_command_in_generator_with_subiterator(self): # Issue #16596. code = """ @@ -1233,6 +1217,19 @@ def main(): with TracerRun(self) as tracer: tracer.runcall(tfunc_import) + def test_next_to_botframe(self): + # gh-125422 + # Check that next command won't go to the bottom frame. + code = """ + lno = 2 + """ + self.expect_set = [ + ('line', 2, ''), ('step', ), + ('return', 2, ''), ('next', ), + ] + with TracerRun(self) as tracer: + tracer.run(compile(textwrap.dedent(code), '', 'exec')) + class TestRegressions(unittest.TestCase): def test_format_stack_entry_no_lineno(self): diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index e360ec15a8f..8f528812e35 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -638,8 +638,6 @@ def test_encode_utf7(self, size): except MemoryError: pass # acceptable on 32-bit - # TODO: RUSTPYTHON - @unittest.expectedFailure @bigmemtest(size=_4G // 4 + 5, memuse=ascii_char_size + ucs4_char_size + 4) def test_encode_utf32(self, size): try: @@ -710,8 +708,6 @@ def test_repr_large(self, size): # original (Py_UCS2) one # There's also some overallocation when resizing the ascii() result # that isn't taken into account here. - # TODO: RUSTPYTHON - @unittest.expectedFailure @bigmemtest(size=_2G // 5 + 1, memuse=ucs2_char_size + ucs4_char_size + ascii_char_size * 6) def test_unicode_repr(self, size): diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index fd52b9895c1..fa027710489 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -4,7 +4,8 @@ import binascii import array import re -from test.support import bigmemtest, _1G, _4G, warnings_helper +from test.support import bigmemtest, _1G, _4G +from test.support.hypothesis_helper import hypothesis # Note: "*_hex" functions are aliases for "(un)hexlify" @@ -27,19 +28,25 @@ class BinASCIITest(unittest.TestCase): def setUp(self): self.data = self.type2test(self.rawdata) + def assertConversion(self, original, converted, restored, **kwargs): + self.assertIsInstance(original, bytes) + self.assertIsInstance(converted, bytes) + self.assertIsInstance(restored, bytes) + if converted: + self.assertLess(max(converted), 128) + self.assertEqual(original, restored, msg=f'{self.type2test=} {kwargs=}') + def test_exceptions(self): # Check module exceptions - self.assertTrue(issubclass(binascii.Error, Exception)) - self.assertTrue(issubclass(binascii.Incomplete, Exception)) + self.assertIsSubclass(binascii.Error, Exception) + self.assertIsSubclass(binascii.Incomplete, Exception) def test_functions(self): # Check presence of all functions for name in all_functions: - self.assertTrue(hasattr(getattr(binascii, name), '__call__')) + self.assertHasAttr(getattr(binascii, name), '__call__') self.assertRaises(TypeError, getattr(binascii, name)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_returned_value(self): # Limit to the minimum of all limits (b2a_uu) MAX_ALL = 45 @@ -54,9 +61,7 @@ def test_returned_value(self): self.fail("{}/{} conversion raises {!r}".format(fb, fa, err)) self.assertEqual(res, raw, "{}/{} conversion: " "{!r} != {!r}".format(fb, fa, res, raw)) - self.assertIsInstance(res, bytes) - self.assertIsInstance(a, bytes) - self.assertLess(max(a), 128) + self.assertConversion(raw, a, res) self.assertIsInstance(binascii.crc_hqx(raw, 0), int) self.assertIsInstance(binascii.crc32(raw), int) @@ -112,6 +117,7 @@ def addnoise(line): # empty strings. TBD: shouldn't it raise an exception instead ? self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_base64_strict_mode(self): # Test base64 with strict mode on def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes): @@ -134,13 +140,21 @@ def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + def assertExcessPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) + # Test excess data exceptions assertExcessData(b'ab==a', b'i') assertExcessData(b'ab===', b'i') + assertExcessData(b'ab====', b'i') assertExcessData(b'ab==:', b'i') assertExcessData(b'abc=a', b'i\xb7') assertExcessData(b'abc=:', b'i\xb7') assertExcessData(b'ab==\n', b'i') + assertExcessData(b'abc==', b'i\xb7') + assertExcessData(b'abc===', b'i\xb7') + assertExcessData(b'abc====', b'i\xb7') + assertExcessData(b'abc=====', b'i\xb7') # Test non-base64 data exceptions assertNonBase64Data(b'\nab==', b'i') @@ -152,8 +166,16 @@ def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): assertLeadingPadding(b'=', b'') assertLeadingPadding(b'==', b'') assertLeadingPadding(b'===', b'') + assertLeadingPadding(b'====', b'') + assertLeadingPadding(b'=====', b'') assertDiscontinuousPadding(b'ab=c=', b'i\xb7') assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + assertExcessPadding(b'abcd=', b'i\xb7\x1d') + assertExcessPadding(b'abcd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd===', b'i\xb7\x1d') + assertExcessPadding(b'abcd====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=====', b'i\xb7\x1d') + def test_base64errors(self): # Test base64 with invalid padding @@ -186,8 +208,6 @@ def assertInvalidLength(data): assertInvalidLength(b'a' * (4 * 87 + 1)) assertInvalidLength(b'A\tB\nC ??DE') # only 5 valid characters - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_uu(self): MAX_UU = 45 for backtick in (True, False): @@ -225,6 +245,15 @@ def test_uu(self): with self.assertRaises(TypeError): binascii.b2a_uu(b"", True) + @hypothesis.given( + binary=hypothesis.strategies.binary(max_size=45), + backtick=hypothesis.strategies.booleans(), + ) + def test_b2a_roundtrip(self, binary, backtick): + converted = binascii.b2a_uu(self.type2test(binary), backtick=backtick) + restored = binascii.a2b_uu(self.type2test(converted)) + self.assertConversion(binary, converted, restored, backtick=backtick) + def test_crc_hqx(self): crc = binascii.crc_hqx(self.type2test(b"Test the CRC-32 of"), 0) crc = binascii.crc_hqx(self.type2test(b" this string."), crc) @@ -262,8 +291,12 @@ def test_hex(self): self.assertEqual(binascii.hexlify(self.type2test(s)), t) self.assertEqual(binascii.unhexlify(self.type2test(t)), u) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @hypothesis.given(binary=hypothesis.strategies.binary()) + def test_hex_roundtrip(self, binary): + converted = binascii.hexlify(self.type2test(binary)) + restored = binascii.unhexlify(self.type2test(converted)) + self.assertConversion(binary, converted, restored) + def test_hex_separator(self): """Test that hexlify and b2a_hex are binary versions of bytes.hex.""" # Logic of separators is tested in test_bytes.py. This checks that @@ -378,6 +411,21 @@ def test_qp(self): self.assertEqual(b2a_qp(type2test(b'a.\n')), b'a.\n') self.assertEqual(b2a_qp(type2test(b'.a')[:-1]), b'=2E') + @hypothesis.given( + binary=hypothesis.strategies.binary(), + quotetabs=hypothesis.strategies.booleans(), + istext=hypothesis.strategies.booleans(), + header=hypothesis.strategies.booleans(), + ) + def test_b2a_qp_a2b_qp_round_trip(self, binary, quotetabs, istext, header): + converted = binascii.b2a_qp( + self.type2test(binary), + quotetabs=quotetabs, istext=istext, header=header, + ) + restored = binascii.a2b_qp(self.type2test(converted), header=header) + self.assertConversion(binary, converted, restored, + quotetabs=quotetabs, istext=istext, header=header) + def test_empty_string(self): # A test for SF bug #1022953. Make sure SystemError is not raised. empty = self.type2test(b'') @@ -392,8 +440,6 @@ def test_empty_string(self): except Exception as err: self.fail("{}({!r}) raises {!r}".format(func, empty, err)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicode_b2a(self): # Unicode strings are not accepted by b2a_* functions. for func in set(all_functions) - set(a2b_functions): @@ -404,8 +450,6 @@ def test_unicode_b2a(self): # crc_hqx needs 2 arguments self.assertRaises(TypeError, binascii.crc_hqx, "test", 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicode_a2b(self): # Unicode strings are accepted by a2b_* functions. MAX_ALL = 45 @@ -437,6 +481,21 @@ def test_b2a_base64_newline(self): self.assertEqual(binascii.b2a_base64(b, newline=False), b'aGVsbG8=') + @hypothesis.given( + binary=hypothesis.strategies.binary(), + newline=hypothesis.strategies.booleans(), + ) + def test_base64_roundtrip(self, binary, newline): + converted = binascii.b2a_base64(self.type2test(binary), newline=newline) + restored = binascii.a2b_base64(self.type2test(converted)) + self.assertConversion(binary, converted, restored, newline=newline) + + def test_c_contiguity(self): + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + binascii.b2a_hex(noncontig_writable) + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): diff --git a/Lib/test/test_binop.py b/Lib/test/test_binop.py index 299af09c498..b224c3d4e60 100644 --- a/Lib/test/test_binop.py +++ b/Lib/test/test_binop.py @@ -383,7 +383,7 @@ def test_comparison_orders(self): self.assertEqual(op_sequence(le, B, C), ['C.__ge__', 'B.__le__']) self.assertEqual(op_sequence(le, C, B), ['C.__le__', 'B.__ge__']) - self.assertTrue(issubclass(V, B)) + self.assertIsSubclass(V, B) self.assertEqual(op_sequence(eq, B, V), ['B.__eq__', 'V.__eq__']) self.assertEqual(op_sequence(le, B, V), ['B.__le__', 'V.__ge__']) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index 3e83e4aceb7..dcdf7bdce03 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -7,8 +7,6 @@ class BoolTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass(self): try: class C(bool): @@ -48,8 +46,6 @@ def test_complex(self): self.assertEqual(complex(True), 1+0j) self.assertEqual(complex(True), True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_math(self): self.assertEqual(+False, 0) self.assertIsNot(+False, False) @@ -387,6 +383,10 @@ def __len__(self): __bool__ = None self.assertRaises(TypeError, bool, B()) + class C: + __len__ = None + self.assertRaises(TypeError, bool, C()) + def test_real_and_imag(self): self.assertEqual(True.real, 1) self.assertEqual(True.imag, 0) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 468c6ea9def..bc09329e6de 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -17,12 +17,14 @@ import unittest from test import support from test.support import os_helper +import inspect from itertools import permutations, product from random import randrange, sample, choice import warnings import sys, array, io, os from decimal import Decimal from fractions import Fraction +from test.support import warnings_helper try: from _testbuffer import * @@ -64,7 +66,7 @@ '?':0, 'c':0, 'b':0, 'B':0, 'h':0, 'H':0, 'i':0, 'I':0, 'l':0, 'L':0, 'n':0, 'N':0, - 'f':0, 'd':0, 'P':0 + 'e':0, 'f':0, 'd':0, 'P':0 } # NumPy does not have 'n' or 'N': @@ -89,7 +91,8 @@ 'i':(-(1<<31), 1<<31), 'I':(0, 1<<32), 'l':(-(1<<31), 1<<31), 'L':(0, 1<<32), 'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64), - 'f':(-(1<<63), 1<<63), 'd':(-(1<<1023), 1<<1023) + 'e':(-65519, 65520), 'f':(-(1<<63), 1<<63), + 'd':(-(1<<1023), 1<<1023) } def native_type_range(fmt): @@ -98,6 +101,8 @@ def native_type_range(fmt): lh = (0, 256) elif fmt == '?': lh = (0, 2) + elif fmt == 'e': + lh = (-65519, 65520) elif fmt == 'f': lh = (-(1<<63), 1<<63) elif fmt == 'd': @@ -125,7 +130,10 @@ def native_type_range(fmt): for fmt in fmtdict['@']: fmtdict['@'][fmt] = native_type_range(fmt) +# Format codes supported by the memoryview object MEMORYVIEW = NATIVE.copy() + +# Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: if not k in "bBhHiIlLfd": @@ -160,11 +168,11 @@ def randrange_fmt(mode, char, obj): if char == 'c': x = bytes([x]) if obj == 'numpy' and x == b'\x00': - # http://projects.scipy.org/numpy/ticket/1925 + # https://github.com/numpy/numpy/issues/2518 x = b'\x01' if char == '?': x = bool(x) - if char == 'f' or char == 'd': + if char in 'efd': x = struct.pack(char, x) x = struct.unpack(char, x)[0] return x @@ -959,8 +967,10 @@ def check_memoryview(m, expected_readonly=readonly): self.assertEqual(m.strides, tuple(strides)) self.assertEqual(m.suboffsets, tuple(suboffsets)) - n = 1 if ndim == 0 else len(lst) - self.assertEqual(len(m), n) + if ndim == 0: + self.assertRaises(TypeError, len, m) + else: + self.assertEqual(len(m), len(lst)) rep = result.tolist() if fmt else result.tobytes() self.assertEqual(rep, lst) @@ -1019,6 +1029,7 @@ def match(req, flag): ndim=ndim, shape=shape, strides=strides, lst=lst, sliced=sliced) + @support.requires_resource('cpu') def test_ndarray_getbuf(self): requests = ( # distinct flags @@ -1907,7 +1918,7 @@ def test_ndarray_random(self): if numpy_array: shape = t[3] if 0 in shape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 z = numpy_array_from_structure(items, fmt, t) self.verify(x, obj=None, itemsize=z.itemsize, fmt=fmt, readonly=False, @@ -1939,7 +1950,7 @@ def test_ndarray_random_invalid(self): except Exception as e: numpy_err = e.__class__ - if 0: # http://projects.scipy.org/numpy/ticket/1910 + if 0: # https://github.com/numpy/numpy/issues/2503 self.assertTrue(numpy_err) def test_ndarray_random_slice_assign(self): @@ -1985,7 +1996,7 @@ def test_ndarray_random_slice_assign(self): if numpy_array: if 0 in lshape or 0 in rshape: - continue # http://projects.scipy.org/numpy/ticket/1910 + continue # https://github.com/numpy/numpy/issues/2503 zl = numpy_array_from_structure(litems, fmt, tl) zr = numpy_array_from_structure(ritems, fmt, tr) @@ -2246,7 +2257,7 @@ def test_py_buffer_to_contiguous(self): ### ### Fortran output: ### --------------- - ### >>> fortran_buf = nd.tostring(order='F') + ### >>> fortran_buf = nd.tobytes(order='F') ### >>> fortran_buf ### b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b' ### @@ -2289,7 +2300,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='C')) + self.assertEqual(b, na.tobytes(order='C')) # 'F' request if f == 0: # 'C' to 'F' @@ -2312,7 +2323,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='F')) + self.assertEqual(b, na.tobytes(order='F')) # 'A' request if f == ND_FORTRAN: @@ -2336,7 +2347,7 @@ def test_py_buffer_to_contiguous(self): self.assertEqual(memoryview(y), memoryview(nd)) if numpy_array: - self.assertEqual(b, na.tostring(order='A')) + self.assertEqual(b, na.tobytes(order='A')) # multi-dimensional, non-contiguous input nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL) @@ -2750,6 +2761,7 @@ def iter_roundtrip(ex, m, items, fmt): m = memoryview(ex) iter_roundtrip(ex, m, items, fmt) + @support.requires_resource('cpu') def test_memoryview_cast_1D_ND(self): # Cast between C-contiguous buffers. At least one buffer must # be 1D, at least one format must be 'c', 'b' or 'B'. @@ -2867,11 +2879,11 @@ def test_memoryview_tolist(self): def test_memoryview_repr(self): m = memoryview(bytearray(9)) r = m.__repr__() - self.assertTrue(r.startswith("l:x:>l:y:}" @@ -3227,6 +3233,15 @@ class BEPoint(ctypes.BigEndianStructure): self.assertNotEqual(point, a) self.assertRaises(NotImplementedError, a.tolist) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') + def test_memoryview_compare_special_cases_deprecated_u_type_code(self): + + # Depends on issue #15625: the struct module does not understand 'u'. + a = array.array('u', 'xyz') + v = memoryview(a) + self.assertNotEqual(a, v) + self.assertNotEqual(v, a) + def test_memoryview_compare_ndim_zero(self): nd1 = ndarray(1729, shape=[], format='@L') @@ -3895,6 +3910,8 @@ def test_memoryview_check_released(self): self.assertRaises(ValueError, memoryview, m) # memoryview.cast() self.assertRaises(ValueError, m.cast, 'c') + # memoryview.__iter__() + self.assertRaises(ValueError, m.__iter__) # getbuffer() self.assertRaises(ValueError, ndarray, m) # memoryview.tolist() @@ -4422,6 +4439,14 @@ def test_issue_7385(self): x = ndarray([1,2,3], shape=[3], flags=ND_GETBUF_FAIL) self.assertRaises(BufferError, memoryview, x) + def test_bytearray_release_buffer_read_flag(self): + # See https://github.com/python/cpython/issues/126980 + obj = bytearray(b'abc') + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.READ) + with self.assertRaises(SystemError): + obj.__buffer__(inspect.BufferFlags.WRITE) + @support.cpython_only def test_pybuffer_size_from_format(self): # basic tests @@ -4429,6 +4454,383 @@ def test_pybuffer_size_from_format(self): self.assertEqual(_testcapi.PyBuffer_SizeFromFormat(format), struct.calcsize(format)) + @support.cpython_only + def test_flags_overflow(self): + # gh-126594: Check for integer overlow on large flags + try: + from _testcapi import INT_MIN, INT_MAX + except ImportError: + INT_MIN = -(2 ** 31) + INT_MAX = 2 ** 31 - 1 + + obj = b'abc' + for flags in (INT_MIN - 1, INT_MAX + 1): + with self.subTest(flags=flags): + with self.assertRaises(OverflowError): + obj.__buffer__(flags) + + +class TestPythonBufferProtocol(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_basic(self): + class MyBuffer: + def __buffer__(self, flags): + return memoryview(b"hello") + + mv = memoryview(MyBuffer()) + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(bytes(MyBuffer()), b"hello") + + def test_bad_buffer_method(self): + class MustReturnMV: + def __buffer__(self, flags): + return 42 + + self.assertRaises(TypeError, memoryview, MustReturnMV()) + + class NoBytesEither: + def __buffer__(self, flags): + return b"hello" + + self.assertRaises(TypeError, memoryview, NoBytesEither()) + + class WrongArity: + def __buffer__(self): + return memoryview(b"hello") + + self.assertRaises(TypeError, memoryview, WrongArity()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_buffer(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + return memoryview(self.ba) + + def __release_buffer__(self, buffer): + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_same_buffer_returned(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + self.created_mv = None + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + self.created_mv = memoryview(self.ba) + return self.created_mv + + def __release_buffer__(self, buffer): + assert buffer is self.created_mv + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffer_flags(self): + class PossiblyMutable: + def __init__(self, data, mutable) -> None: + self._data = bytearray(data) + self._mutable = mutable + + def __buffer__(self, flags): + if flags & inspect.BufferFlags.WRITABLE: + if not self._mutable: + raise RuntimeError("not mutable") + return memoryview(self._data) + else: + return memoryview(bytes(self._data)) + + mutable = PossiblyMutable(b"hello", True) + immutable = PossiblyMutable(b"hello", False) + with memoryview._from_flags(mutable, inspect.BufferFlags.WRITABLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(mutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"xello") + with self.assertRaises(TypeError): + mv[0] = ord(b'h') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + with self.assertRaises(RuntimeError): + memoryview._from_flags(immutable, inspect.BufferFlags.WRITABLE) + with memoryview(immutable) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_call_builtins(self): + ba = bytearray(b"hello") + mv = ba.__buffer__(0) + self.assertEqual(mv.tobytes(), b"hello") + ba.__release_buffer__(mv) + with self.assertRaises(OverflowError): + ba.__buffer__(sys.maxsize + 1) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer(self): + buf = _testcapi.testBuf() + self.assertEqual(buf.references, 0) + mv = buf.__buffer__(0) + self.assertIsInstance(mv, memoryview) + self.assertEqual(mv.tobytes(), b"test") + self.assertEqual(buf.references, 1) + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + with self.assertRaises(ValueError): + mv.tobytes() + # Calling it again doesn't cause issues + with self.assertRaises(ValueError): + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer_invalid_flags(self): + buf = _testcapi.testBuf() + self.assertRaises(SystemError, buf.__buffer__, PyBUF_READ) + self.assertRaises(SystemError, buf.__buffer__, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_invalid_flags(self): + # PyBuffer_FillInfo + source = b"abc" + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_READ) + self.assertRaises(SystemError, _testcapi.buffer_fill_info, + source, 0, PyBUF_WRITE) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_fill_buffer_readonly_and_writable(self): + source = b"abc" + with _testcapi.buffer_fill_info(source, 1, PyBUF_SIMPLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertTrue(m.readonly) + with _testcapi.buffer_fill_info(source, 0, PyBUF_WRITABLE) as m: + self.assertEqual(bytes(m), b"abc") + self.assertFalse(m.readonly) + self.assertRaises(BufferError, _testcapi.buffer_fill_info, + source, 1, PyBUF_WRITABLE) + + def test_inheritance(self): + class A(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + a = A(b"hello") + mv = memoryview(a) + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_inheritance_releasebuffer(self): + rb_call_count = 0 + class B(bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + super().__release_buffer__(view) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_inherit_but_return_something_else(self): + class A(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + + a = A(b"hello") + with memoryview(a) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + rb_call_count = 0 + rb_raised = False + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b"hello") + def __release_buffer__(self, view): + nonlocal rb_call_count + rb_call_count += 1 + try: + super().__release_buffer__(view) + except ValueError: + nonlocal rb_raised + rb_raised = True + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(rb_call_count, 0) + self.assertEqual(rb_call_count, 1) + self.assertIs(rb_raised, True) + + def test_override_only_release(self): + class C(bytearray): + def __release_buffer__(self, buffer): + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_saves_reference(self): + smuggled_buffer = None + + class C(bytearray): + def __release_buffer__(s, buffer: memoryview): + with self.assertRaises(ValueError): + memoryview(buffer) + with self.assertRaises(ValueError): + buffer.cast("b") + with self.assertRaises(ValueError): + buffer.toreadonly() + with self.assertRaises(ValueError): + buffer[:1] + with self.assertRaises(ValueError): + buffer.__buffer__(0) + nonlocal smuggled_buffer + smuggled_buffer = buffer + self.assertEqual(buffer.tobytes(), b"hello") + super().__release_buffer__(buffer) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + with self.assertRaises(ValueError): + smuggled_buffer.tobytes() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_saves_reference_no_subclassing(self): + ba = bytearray(b"hello") + + class C: + def __buffer__(self, flags): + return memoryview(ba) + + def __release_buffer__(self, buffer): + self.buffer = buffer + + c = C() + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(c.buffer.tobytes(), b"hello") + + with self.assertRaises(BufferError): + ba.clear() + c.buffer.release() + ba.clear() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_multiple_inheritance_buffer_last(self): + class A: + def __buffer__(self, flags): + return memoryview(b"hello A") + + class B(A, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello A") + + class Releaser: + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(Releaser, bytearray): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello C") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello C") + c.clear() + with self.assertRaises(ValueError): + c.buffer.tobytes() + + def test_multiple_inheritance_buffer_last_raising(self): + class A: + def __buffer__(self, flags): + raise RuntimeError("should not be called") + + def __release_buffer__(self, buffer): + raise RuntimeError("should not be called") + + class B(bytearray, A): + def __buffer__(self, flags): + return super().__buffer__(flags) + + b = B(b"hello") + with memoryview(b) as mv: + self.assertEqual(mv.tobytes(), b"hello") + + class Releaser: + buffer = None + def __release_buffer__(self, buffer): + self.buffer = buffer + + class C(bytearray, Releaser): + def __buffer__(self, flags): + return super().__buffer__(flags) + + c = C(b"hello") + with memoryview(c) as mv: + self.assertEqual(mv.tobytes(), b"hello") + c.clear() + self.assertIs(c.buffer, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release_buffer_with_exception_set(self): + class A: + def __buffer__(self, flags): + return memoryview(bytes(8)) + def __release_buffer__(self, view): + pass + + b = bytearray(8) + with memoryview(b): + # now b.extend will raise an exception due to exports + with self.assertRaises(BufferError): + b.extend(A()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_bufio.py b/Lib/test/test_bufio.py index 989d8cd349b..cb9cb4d0bc7 100644 --- a/Lib/test/test_bufio.py +++ b/Lib/test/test_bufio.py @@ -28,7 +28,7 @@ def try_one(self, s): f.write(b"\n") f.write(s) f.close() - f = open(os_helper.TESTFN, "rb") + f = self.open(os_helper.TESTFN, "rb") line = f.readline() self.assertEqual(line, s + b"\n") line = f.readline() @@ -65,9 +65,6 @@ def test_nullpat(self): class CBufferSizeTest(BufferSizeTest, unittest.TestCase): open = io.open -# TODO: RUSTPYTHON -import sys -@unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, can't cleanup temporary file on Windows") class PyBufferSizeTest(BufferSizeTest, unittest.TestCase): open = staticmethod(pyio.open) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 9e13fbb2c8e..cf0268c2ce5 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1,14 +1,15 @@ # Python test set -- built-in functions import ast -import asyncio import builtins import collections +import contextlib import decimal import fractions import gc import io import locale +import math import os import pickle import platform @@ -17,6 +18,7 @@ import sys import traceback import types +import typing import unittest import warnings from contextlib import ExitStack @@ -27,10 +29,14 @@ from types import AsyncGeneratorType, FunctionType, CellType from operator import neg from test import support -from test.support import (swap_attr, maybe_get_event_loop_policy) +from test.support import cpython_only, swap_attr +from test.support import async_yield, run_yielding_async_fn +from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin from test.support.warnings_helper import check_warnings +from test.support import requires_IEEE_754 from unittest.mock import MagicMock, patch try: import pty, signal @@ -38,6 +44,14 @@ pty = signal = None +# Detect evidence of double-rounding: sum() does not always +# get improved accuracy on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + +# used as proof of globals being used +A_GLOBAL_VALUE = 123 + class Squares: def __init__(self, max): @@ -134,7 +148,10 @@ def filter_char(arg): def map_char(arg): return chr(ord(arg)+1) -class BuiltinTest(unittest.TestCase): +def pack(*args): + return args + +class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase): # Helper to check picklability def check_iter_pickle(self, it, seq, proto): itorg = it @@ -153,8 +170,6 @@ def check_iter_pickle(self, it, seq, proto): it = pickle.loads(d) self.assertEqual(list(it), seq[1:]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_import(self): __import__('sys') __import__('time') @@ -210,6 +225,8 @@ def test_all(self): self.assertEqual(all(x > 42 for x in S), True) S = [50, 40, 60] self.assertEqual(all(x > 42 for x in S), False) + S = [50, 40, 60, TestFailingBool()] + self.assertEqual(all(x > 42 for x in S), False) def test_any(self): self.assertEqual(any([None, None, None]), False) @@ -223,11 +240,59 @@ def test_any(self): self.assertEqual(any([1, TestFailingBool()]), True) # Short-circuit S = [40, 60, 30] self.assertEqual(any(x > 42 for x in S), True) + S = [40, 60, 30, TestFailingBool()] + self.assertEqual(any(x > 42 for x in S), True) S = [10, 20, 30] self.assertEqual(any(x > 42 for x in S), False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_all_any_tuple_optimization(self): + def f_all(): + return all(x-2 for x in [1,2,3]) + + def f_any(): + return any(x-1 for x in [1,2,3]) + + def f_tuple(): + return tuple(2*x for x in [1,2,3]) + + funcs = [f_all, f_any, f_tuple] + + for f in funcs: + # check that generator code object is not duplicated + code_objs = [c for c in f.__code__.co_consts if isinstance(c, type(f.__code__))] + self.assertEqual(len(code_objs), 1) + + + # check the overriding the builtins works + + global all, any, tuple + saved = all, any, tuple + try: + all = lambda x : "all" + any = lambda x : "any" + tuple = lambda x : "tuple" + + overridden_outputs = [f() for f in funcs] + finally: + all, any, tuple = saved + + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + + # Now repeat, overriding the builtins module as well + saved = all, any, tuple + try: + builtins.all = all = lambda x : "all" + builtins.any = any = lambda x : "any" + builtins.tuple = tuple = lambda x : "tuple" + + overridden_outputs = [f() for f in funcs] + finally: + all, any, tuple = saved + builtins.all, builtins.any, builtins.tuple = saved + + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + + def test_ascii(self): self.assertEqual(ascii(''), '\'\'') self.assertEqual(ascii(0), '0') @@ -303,14 +368,13 @@ class C3(C2): pass self.assertTrue(callable(c3)) def test_chr(self): + self.assertEqual(chr(0), '\0') self.assertEqual(chr(32), ' ') self.assertEqual(chr(65), 'A') self.assertEqual(chr(97), 'a') self.assertEqual(chr(0xff), '\xff') - self.assertRaises(ValueError, chr, 1<<24) - self.assertEqual(chr(sys.maxunicode), - str('\\U0010ffff'.encode("ascii"), 'unicode-escape')) self.assertRaises(TypeError, chr) + self.assertRaises(TypeError, chr, 65.0) self.assertEqual(chr(0x0000FFFF), "\U0000FFFF") self.assertEqual(chr(0x00010000), "\U00010000") self.assertEqual(chr(0x00010001), "\U00010001") @@ -322,13 +386,15 @@ def test_chr(self): self.assertEqual(chr(0x0010FFFF), "\U0010FFFF") self.assertRaises(ValueError, chr, -1) self.assertRaises(ValueError, chr, 0x00110000) - self.assertRaises((OverflowError, ValueError), chr, 2**32) + self.assertRaises(ValueError, chr, 1<<24) + self.assertRaises(ValueError, chr, 2**32-1) + self.assertRaises(ValueError, chr, -2**32) + self.assertRaises(ValueError, chr, 2**1000) + self.assertRaises(ValueError, chr, -2**1000) def test_cmp(self): - self.assertTrue(not hasattr(builtins, "cmp")) + self.assertNotHasAttr(builtins, "cmp") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compile(self): compile('print(1)\n', '', 'exec') bom = b'\xef\xbb\xbf' @@ -340,11 +406,10 @@ def test_compile(self): self.assertRaises(TypeError, compile) self.assertRaises(ValueError, compile, 'print(42)\n', '', 'badmode') self.assertRaises(ValueError, compile, 'print(42)\n', '', 'single', 0xff) - self.assertRaises(ValueError, compile, chr(0), 'f', 'exec') self.assertRaises(TypeError, compile, 'pass', '?', 'exec', mode='eval', source='0', filename='tmp') compile('print("\xe5")\n', '', 'exec') - self.assertRaises(ValueError, compile, chr(0), 'f', 'exec') + self.assertRaises(SyntaxError, compile, chr(0), 'f', 'exec') self.assertRaises(ValueError, compile, str('a = 1'), 'f', 'bad') # test the optimize argument @@ -367,19 +432,18 @@ def f(): """doc""" (1, False, 'doc', False, False), (2, False, None, False, False)] for optval, *expected in values: + with self.subTest(optval=optval): # test both direct compilation and compilation via AST - codeobjs = [] - codeobjs.append(compile(codestr, "", "exec", optimize=optval)) - tree = ast.parse(codestr) - codeobjs.append(compile(tree, "", "exec", optimize=optval)) - for code in codeobjs: - ns = {} - exec(code, ns) - rv = ns['f']() - self.assertEqual(rv, tuple(expected)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + codeobjs = [] + codeobjs.append(compile(codestr, "", "exec", optimize=optval)) + tree = ast.parse(codestr, optimize=optval) + codeobjs.append(compile(tree, "", "exec", optimize=optval)) + for code in codeobjs: + ns = {} + exec(code, ns) + rv = ns['f']() + self.assertEqual(rv, tuple(expected)) + def test_compile_top_level_await_no_coro(self): """Make sure top level non-await codes get the correct coroutine flags""" modes = ('single', 'exec') @@ -401,14 +465,9 @@ def test_compile_top_level_await_no_coro(self): msg=f"source={source} mode={mode}") - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf( - support.is_emscripten or support.is_wasi, - "socket.accept is broken" - ) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_compile_top_level_await(self): - """Test whether code some top level await can be compiled. + """Test whether code with top level await can be compiled. Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set, and make sure the generated code object has the CO_COROUTINE flag @@ -421,12 +480,25 @@ async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + + async def sleep(delay, result=None): + assert delay == 0 + await async_yield(None) + return result + modes = ('single', 'exec') + optimizations = (-1, 0, 1, 2) code_samples = [ - '''a = await asyncio.sleep(0, result=1)''', + '''a = await sleep(0, result=1)''', '''async for i in arange(1): a = 1''', - '''async with asyncio.Lock() as l: + '''async with Lock() as l: a = 1''', '''a = [x async for x in arange(2)][1]''', '''a = 1 in {x async for x in arange(2)}''', @@ -434,45 +506,63 @@ async def arange(n): '''a = [x async for x in arange(2) async for x in arange(2)][1]''', '''a = [x async for x in (x async for x in arange(5))][1]''', '''a, = [1 for x in {x async for x in arange(1)}]''', - '''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''' + '''a = [await sleep(0, x) async for x in arange(2)][1]''', + # gh-121637: Make sure we correctly handle the case where the + # async code is optimized away + '''assert not await sleep(0); a = 1''', + '''assert [x async for x in arange(1)]; a = 1''', + '''assert {x async for x in arange(1)}; a = 1''', + '''assert {x: x async for x in arange(1)}; a = 1''', + ''' + if (a := 1) and __debug__: + async with Lock() as l: + pass + ''', + ''' + if (a := 1) and __debug__: + async for x in arange(2): + pass + ''', ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): + for mode, code_sample, optimize in product(modes, code_samples, optimizations): + with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize): source = dedent(code_sample) with self.assertRaises( SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) + compile(source, '?', mode, optimize=optimize) co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + optimize=optimize) self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, - msg=f"source={source} mode={mode}") + msg=f"source={source} mode={mode}") # test we can create and advance a function type - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - async_f = FunctionType(co, globals_) - asyncio.run(async_f()) + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(FunctionType(co, globals_)) self.assertEqual(globals_['a'], 1) # test we can await-eval, - globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange} - asyncio.run(eval(co, globals_)) + globals_ = {'Lock': Lock, 'a': 0, 'arange': arange, 'sleep': sleep} + run_yielding_async_fn(lambda: eval(co, globals_)) self.assertEqual(globals_['a'], 1) - finally: - asyncio.set_event_loop_policy(policy) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compile_top_level_await_invalid_cases(self): # helper function just to check we can run top=level async-for async def arange(n): for i in range(n): yield i + class Lock: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + pass + modes = ('single', 'exec') code_samples = [ '''def f(): await arange(10)\n''', @@ -483,30 +573,23 @@ async def arange(n): a = 1 ''', '''def f(): - async with asyncio.Lock() as l: + async with Lock() as l: a = 1 ''' ] - policy = maybe_get_event_loop_policy() - try: - for mode, code_sample in product(modes, code_samples): - source = dedent(code_sample) - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - compile(source, '?', mode) - - with self.assertRaises( - SyntaxError, msg=f"source={source} mode={mode}"): - co = compile(source, - '?', - mode, - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - finally: - asyncio.set_event_loop_policy(policy) + for mode, code_sample in product(modes, code_samples): + source = dedent(code_sample) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + compile(source, '?', mode) + with self.assertRaises( + SyntaxError, msg=f"source={source} mode={mode}"): + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compile_async_generator(self): """ With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to @@ -516,13 +599,35 @@ def test_compile_async_generator(self): code = dedent("""async def ticker(): for i in range(10): yield i - await asyncio.sleep(0)""") + await sleep(0)""") co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) glob = {} exec(co, glob) self.assertEqual(type(glob['ticker']()), AsyncGeneratorType) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <_ast.Name object at 0xb40000731e3d1360> is not an instance of + def test_compile_ast(self): + args = ("a*__debug__", "f.py", "exec") + raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] + opt1 = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + opt2 = compile(ast.parse(args[0]), *args[1:], flags = ast.PyCF_OPTIMIZED_AST).body[0] + + for tree in (raw, opt1, opt2): + self.assertIsInstance(tree.value, ast.BinOp) + self.assertIsInstance(tree.value.op, ast.Mult) + self.assertIsInstance(tree.value.left, ast.Name) + self.assertEqual(tree.value.left.id, 'a') + + raw_right = raw.value.right + self.assertIsInstance(raw_right, ast.Name) + self.assertEqual(raw_right.id, "__debug__") + + for opt in [opt1, opt2]: + opt_right = opt.value.right + self.assertIsInstance(opt_right, ast.Constant) + self.assertEqual(opt_right.value, __debug__) + def test_delattr(self): sys.spam = 1 delattr(sys, 'spam') @@ -531,8 +636,7 @@ def test_delattr(self): msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, delattr, sys, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__repr__' unexpectedly found in ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__firstlineno__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', 'bar'] def test_dir(self): # dir(wrong number of arguments) self.assertRaises(TypeError, dir, 42, 42) @@ -594,6 +698,14 @@ def __dir__(self): self.assertIsInstance(res, list) self.assertTrue(res == ["a", "b", "c"]) + # dir(obj__dir__iterable) + class Foo(object): + def __dir__(self): + return {"b", "c", "a"} + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertEqual(sorted(res), ["a", "b", "c"]) + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): @@ -610,6 +722,11 @@ def __dir__(self): # test that object has a __dir__() self.assertEqual(sorted([].__dir__()), dir([])) + def test___ne__(self): + self.assertFalse(None.__ne__(None)) + self.assertIs(None.__ne__(0), NotImplemented) + self.assertIs(None.__ne__("abc"), NotImplemented) + def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) self.assertEqual(divmod(-12, 7), (-2, 2)) @@ -627,6 +744,16 @@ def test_divmod(self): self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 1, 0, + ) + self.assertRaisesRegex( + ZeroDivisionError, + "division by zero", + divmod, 0.0, 0, + ) def test_eval(self): self.assertEqual(eval('1+1'), 2) @@ -651,6 +778,11 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_kwargs(self): + data = {"A_GLOBAL_VALUE": 456} + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", locals=data), 123) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -744,8 +876,19 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_exec_kwargs(self): + g = {} + exec('global z\nz = 1', globals=g) + if '__builtins__' in g: + del g['__builtins__'] + self.assertEqual(g, {'z': 1}) + + # if we only set locals, the global assignment will not + # reach this locals dictionary + g = {} + exec('global z\nz = 1', locals=g) + self.assertEqual(g, {}) + def test_exec_globals(self): code = compile("print('Hello World!')", "", "exec") # no builtin function @@ -755,8 +898,6 @@ def test_exec_globals(self): self.assertRaises(TypeError, exec, code, {'__builtins__': 123}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_frozen(self): class frozendict_error(Exception): pass @@ -789,8 +930,6 @@ def __setitem__(self, key, value): self.assertRaises(frozendict_error, exec, code, namespace) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_error_on_get(self): # custom `globals` or `builtins` can raise errors on item access class setonlyerror(Exception): @@ -810,8 +949,6 @@ def __getitem__(self, key): self.assertRaises(setonlyerror, exec, code, {'__builtins__': setonlydict({'superglobal': 1})}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exec_globals_dict_subclass(self): class customdict(dict): # this one should not do anything fancy pass @@ -823,6 +960,34 @@ class customdict(dict): # this one should not do anything fancy self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", exec, code, {'__builtins__': customdict()}) + def test_eval_builtins_mapping(self): + code = compile("superglobal", "test", "eval") + # works correctly + ns = {'__builtins__': types.MappingProxyType({'superglobal': 1})} + self.assertEqual(eval(code, ns), 1) + # custom builtins mapping is missing key + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", + eval, code, ns) + + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message + def test_exec_builtins_mapping_import(self): + code = compile("import foo.bar", "test", "exec") + ns = {'__builtins__': types.MappingProxyType({})} + self.assertRaisesRegex(ImportError, "__import__ not found", exec, code, ns) + ns = {'__builtins__': types.MappingProxyType({'__import__': lambda *args: args})} + exec(code, ns) + self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: AttributeError not raised by eval + def test_eval_builtins_mapping_reduce(self): + # list_iterator.__reduce__() calls _PyEval_GetBuiltin("iter") + code = compile("x.__reduce__()", "test", "eval") + ns = {'__builtins__': types.MappingProxyType({}), 'x': iter([1, 2])} + self.assertRaisesRegex(AttributeError, "iter", eval, code, ns) + ns = {'__builtins__': types.MappingProxyType({'iter': iter}), 'x': iter([1, 2])} + self.assertEqual(eval(code, ns), (iter, ([1, 2],), 0)) + def test_exec_redirected(self): savestdout = sys.stdout sys.stdout = None # Whatever that cannot flush() @@ -834,8 +999,7 @@ def test_exec_redirected(self): finally: sys.stdout = savestdout - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument closure def test_exec_closure(self): def function_without_closures(): return 3 * 5 @@ -903,8 +1067,24 @@ def four_freevars(): three_freevars.__code__, three_freevars.__globals__, closure=my_closure) + my_closure = tuple(my_closure) + + # should fail: anything passed to closure= isn't allowed + # when the source is a string + self.assertRaises(TypeError, + exec, + "pass", + closure=int) + + # should fail: correct closure= argument isn't allowed + # when the source is a string + self.assertRaises(TypeError, + exec, + "pass", + closure=my_closure) # should fail: closure tuple with one non-cell-var + my_closure = list(my_closure) my_closure[0] = int my_closure = tuple(my_closure) self.assertRaises(TypeError, @@ -945,6 +1125,20 @@ def test_filter_pickle(self): f2 = filter(filter_char, "abcdeabcde") self.check_iter_pickle(f1, list(f2), proto) + @unittest.skip("TODO: RUSTPYTHON; Segfault") + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() + @support.requires_resource('cpu') + def test_filter_dealloc(self): + # Tests recursive deallocation of nested filter objects using the + # thrashcan mechanism. See gh-102356 for more details. + max_iters = 1000000 + i = filter(bool, range(max_iters)) + for _ in range(max_iters): + i = filter(bool, i) + del i + gc.collect() + def test_getattr(self): self.assertTrue(getattr(sys, 'stdout') is sys.stdout) self.assertRaises(TypeError, getattr) @@ -996,6 +1190,16 @@ def __hash__(self): return self self.assertEqual(hash(Z(42)), hash(42)) + def test_invalid_hash_typeerror(self): + # GH-140406: The returned object from __hash__() would leak if it + # wasn't an integer. + class A: + def __hash__(self): + return 1.0 + + with self.assertRaises(TypeError): + hash(A()) + def test_hex(self): self.assertEqual(hex(16), '0x10') self.assertEqual(hex(-16), '-0x10') @@ -1157,6 +1361,124 @@ def test_map_pickle(self): m2 = map(map_char, "Is this the real life?") self.check_iter_pickle(m1, list(m2), proto) + # strict map tests based on strict zip tests + + def test_map_pickle_strict(self): + a = (1, 2, 3) + b = (4, 5, 6) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + self.check_iter_pickle(m1, t, proto) + + def test_map_pickle_strict_fail(self): + a = (1, 2, 3) + b = (4, 5, 6, 7) + t = [(1, 4), (2, 5), (3, 6)] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + m1 = map(pack, a, b, strict=True) + m2 = pickle.loads(pickle.dumps(m1, proto)) + self.assertEqual(self.iter_error(m1, ValueError), t) + self.assertEqual(self.iter_error(m2, ValueError), t) + + def test_map_strict(self): + self.assertEqual(tuple(map(pack, (1, 2, 3), 'abc', strict=True)), + ((1, 'a'), (2, 'b'), (3, 'c'))) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2, 3, 4), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), 'abc', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, (1, 2), (1, 2), 'abc', strict=True)) + + # gh-140517: Testing refleaks with mortal objects. + t1 = (None, object()) + t2 = (object(), object()) + t3 = (object(),) + + self.assertRaises(ValueError, tuple, + map(pack, t1, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, t3, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t1, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t2, t3, strict=True)) + + def test_map_strict_iterators(self): + x = iter(range(5)) + y = [0] + z = iter(range(5)) + self.assertRaises(ValueError, list, + (map(pack, x, y, z, strict=True))) + self.assertEqual(next(x), 2) + self.assertEqual(next(z), 1) + + def test_map_strict_error_handling(self): + + class Error(Exception): + pass + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise Error + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), Error) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), Error) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), Error) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), Error) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + + def test_map_strict_error_handling_stopiteration(self): + + class Iter: + def __init__(self, size): + self.size = size + def __iter__(self): + return self + def __next__(self): + self.size -= 1 + if self.size < 0: + raise StopIteration + return self.size + + l1 = self.iter_error(map(pack, "AB", Iter(1), strict=True), ValueError) + self.assertEqual(l1, [("A", 0)]) + l2 = self.iter_error(map(pack, "AB", Iter(2), "A", strict=True), ValueError) + self.assertEqual(l2, [("A", 1, "A")]) + l3 = self.iter_error(map(pack, "AB", Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l3, [("A", 1, "A"), ("B", 0, "B")]) + l4 = self.iter_error(map(pack, "AB", Iter(3), strict=True), ValueError) + self.assertEqual(l4, [("A", 2), ("B", 1)]) + l5 = self.iter_error(map(pack, Iter(1), "AB", strict=True), ValueError) + self.assertEqual(l5, [(0, "A")]) + l6 = self.iter_error(map(pack, Iter(2), "A", strict=True), ValueError) + self.assertEqual(l6, [(1, "A")]) + l7 = self.iter_error(map(pack, Iter(2), "ABC", strict=True), ValueError) + self.assertEqual(l7, [(1, "A"), (0, "B")]) + l8 = self.iter_error(map(pack, Iter(3), "AB", strict=True), ValueError) + self.assertEqual(l8, [(2, "A"), (1, "B")]) + def test_max(self): self.assertEqual(max('123123'), '3') self.assertEqual(max(1, 2, 3), 3) @@ -1174,7 +1496,11 @@ def test_max(self): max() self.assertRaises(TypeError, max, 42) - self.assertRaises(ValueError, max, ()) + with self.assertRaisesRegex( + ValueError, + r'max\(\) iterable argument is empty' + ): + max(()) class BadSeq: def __getitem__(self, index): raise ValueError @@ -1233,7 +1559,11 @@ def test_min(self): min() self.assertRaises(TypeError, min, 42) - self.assertRaises(ValueError, min, ()) + with self.assertRaisesRegex( + ValueError, + r'min\(\) iterable argument is empty' + ): + min(()) class BadSeq: def __getitem__(self, index): raise ValueError @@ -1334,18 +1664,13 @@ def test_open(self): self.assertRaises(ValueError, open, 'a\x00b') self.assertRaises(ValueError, open, b'a\x00b') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_open_default_encoding(self): - old_environ = dict(os.environ) - try: + with EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that open() uses the current locale # encoding and not the user preferred encoding - for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset('LC_ALL', 'LANG', 'LC_CTYPE') self.write_testfile() current_locale_encoding = locale.getencoding() @@ -1354,11 +1679,7 @@ def test_open_default_encoding(self): fp = open(TESTFN, 'w') with fp: self.assertEqual(fp.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') @support.requires_subprocess() def test_open_non_inheritable(self): fileobj = open(__file__, encoding="utf-8") @@ -1489,6 +1810,29 @@ def test_input(self): sys.stdout = savestdout fp.close() + def test_input_gh130163(self): + class X(io.StringIO): + def __getattribute__(self, name): + nonlocal patch + if patch: + patch = False + sys.stdout = X() + sys.stderr = X() + sys.stdin = X('input\n') + support.gc_collect() + return io.StringIO.__getattribute__(self, name) + + with (support.swap_attr(sys, 'stdout', None), + support.swap_attr(sys, 'stderr', None), + support.swap_attr(sys, 'stdin', None)): + patch = False + # the only references: + sys.stdout = X() + sys.stderr = X() + sys.stdin = X('input\n') + patch = True + input() # should not crash + # test_int(): see test_int.py for tests of built-in function int(). def test_repr(self): @@ -1504,6 +1848,11 @@ def test_repr(self): a[0] = a self.assertEqual(repr(a), '{0: {...}}') + def test_repr_blocked(self): + class C: + __repr__ = None + self.assertRaises(TypeError, repr, C()) + def test_round(self): self.assertEqual(round(0.0), 0.0) self.assertEqual(type(round(0.0)), int) @@ -1610,14 +1959,17 @@ def test_bug_27936(self): def test_setattr(self): setattr(sys, 'spam', 1) - self.assertEqual(sys.spam, 1) + try: + self.assertEqual(sys.spam, 1) + finally: + del sys.spam self.assertRaises(TypeError, setattr) self.assertRaises(TypeError, setattr, sys) self.assertRaises(TypeError, setattr, sys, 'spam') msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, setattr, sys, 1, 'spam') - # test_str(): see test_unicode.py and test_bytes.py for str() tests. + # test_str(): see test_str.py and test_bytes.py for str() tests. def test_sum(self): self.assertEqual(sum([]), 0) @@ -1647,6 +1999,8 @@ def test_sum(self): self.assertEqual(repr(sum([-0.0])), '0.0') self.assertEqual(repr(sum([-0.0], -0.0)), '-0.0') self.assertEqual(repr(sum([], -0.0)), '-0.0') + self.assertTrue(math.isinf(sum([float("inf"), float("inf")]))) + self.assertTrue(math.isinf(sum([1e308, 1e308]))) self.assertRaises(TypeError, sum) self.assertRaises(TypeError, sum, 42) @@ -1661,6 +2015,8 @@ def test_sum(self): self.assertRaises(TypeError, sum, [], '') self.assertRaises(TypeError, sum, [], b'') self.assertRaises(TypeError, sum, [], bytearray()) + self.assertRaises(OverflowError, sum, [1.0, 10**1000]) + self.assertRaises(OverflowError, sum, [1j, 10**1000]) class BadSeq: def __getitem__(self, index): @@ -1671,6 +2027,37 @@ def __getitem__(self, index): sum(([x] for x in range(10)), empty) self.assertEqual(empty, []) + xs = [complex(random.random() - .5, random.random() - .5) + for _ in range(10000)] + self.assertEqual(sum(xs), complex(sum(z.real for z in xs), + sum(z.imag for z in xs))) + + # test that sum() of complex and real numbers doesn't + # smash sign of imaginary 0 + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1, complex(1, -0.0)]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([complex(1, -0.0), 1.0]), + complex(2, -0.0)) + self.assertComplexesAreIdentical(sum([1.0, complex(1, -0.0)]), + complex(2, -0.0)) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sum accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + def test_sum_accuracy(self): + self.assertEqual(sum([0.1] * 10), 1.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100]), 2.0) + self.assertEqual(sum([1.0, 10E100, 1.0, -10E100, 2j]), 2+2j) + self.assertEqual(sum([2+1j, 10E100j, 1j, -10E100j]), 2+2j) + self.assertEqual(sum([1j, 1, 10E100j, 1j, 1.0, -10E100j]), 2+2j) + self.assertEqual(sum([2j, 1., 10E100, 1., -10E100]), 2+2j) + self.assertEqual(sum([1.0, 10**100, 1.0, -10**100]), 2.0) + self.assertEqual(sum([2j, 1.0, 10**100, 1.0, -10**100]), 2+2j) + self.assertEqual(sum([0.1j]*10 + [fractions.Fraction(1, 10)]), 0.1+1j) + def test_type(self): self.assertEqual(type(''), type('123')) self.assertNotEqual(type(''), type(())) @@ -1950,7 +2337,7 @@ def __format__(self, format_spec): # tests for object.__format__ really belong elsewhere, but # there's no good place to put them x = object().__format__('') - self.assertTrue(x.startswith(' eval() roundtrip - if stdio_encoding: - expected = terminal_input.decode(stdio_encoding, 'surrogateescape') - else: - expected = terminal_input.decode(sys.stdin.encoding) # what else? + if expected is None: + if stdio_encoding: + expected = terminal_input.decode(stdio_encoding, 'surrogateescape') + else: + expected = terminal_input.decode(sys.stdin.encoding) # what else? self.assertEqual(input_result, expected) - def test_input_tty(self): - # Test input() functionality when wired to a tty (the code path - # is different and invokes GNU readline if available). - self.check_input_tty("prompt", b"quux") - - def skip_if_readline(self): + @contextlib.contextmanager + def detach_readline(self): # bpo-13886: When the readline module is loaded, PyOS_Readline() uses # the readline implementation. In some cases, the Python readline # callback rlhandler() is called by readline with a string without - # non-ASCII characters. Skip tests on non-ASCII characters if the - # readline module is loaded, since test_builtin is not intented to test + # non-ASCII characters. + # Unlink readline temporarily from PyOS_Readline() for those tests, + # since test_builtin is not intended to test # the readline module, but the builtins module. - if 'readline' in sys.modules: - self.skipTest("the readline module is loaded") + if "readline" in sys.modules: + c = import_module("ctypes") + fp_api = "PyOS_ReadlineFunctionPointer" + prev_value = c.c_void_p.in_dll(c.pythonapi, fp_api).value + c.c_void_p.in_dll(c.pythonapi, fp_api).value = None + try: + yield + finally: + c.c_void_p.in_dll(c.pythonapi, fp_api).value = prev_value + else: + yield + + def test_input_tty(self): + # Test input() functionality when wired to a tty + self.check_input_tty("prompt", b"quux") - @unittest.skipUnless(hasattr(sys.stdin, 'detach'), 'TODO: RustPython: requires detach function in TextIOWrapper') + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_tty_non_ascii(self): - self.skip_if_readline() # Check stdin/stdout encoding is used when invoking PyOS_Readline() - self.check_input_tty("prompté", b"quux\xe9", "utf-8") + self.check_input_tty("prompté", b"quux\xc3\xa9", "utf-8") - @unittest.skipUnless(hasattr(sys.stdin, 'detach'), 'TODO: RustPython: requires detach function in TextIOWrapper') + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_tty_non_ascii_unicode_errors(self): - self.skip_if_readline() # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") + def test_input_tty_null_in_prompt(self): + self.check_input_tty("prompt\0", b"", + expected='ValueError: input: prompt string cannot contain ' + 'null characters') + + def test_input_tty_nonencodable_prompt(self): + self.check_input_tty("prompté", b"quux", "ascii", stdout_errors='strict', + expected="UnicodeEncodeError: 'ascii' codec can't encode " + "character '\\xe9' in position 6: ordinal not in " + "range(128)") + + def test_input_tty_nondecodable_input(self): + self.check_input_tty("prompt", b"quux\xe9", "ascii", stdin_errors='strict', + expected="UnicodeDecodeError: 'ascii' codec can't decode " + "byte 0xe9 in position 4: ordinal not in " + "range(128)") + + @unittest.skip("TODO: RUSTPYTHON; FAILURE, WORKER BUG") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: got 0 lines in pipe but expected 2, child output was: quux def test_input_no_stdout_fileno(self): # Issue #24402: If stdin is the original terminal but stdout.fileno() # fails, do not use the original stdout file descriptor @@ -2331,8 +2789,6 @@ def test_baddecorator(self): class ShutdownTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cleanup(self): # Issue #19255: builtins are still available at shutdown code = """if 1: @@ -2365,9 +2821,36 @@ def __del__(self): self.assertEqual(["before", "after"], out.decode().splitlines()) +@cpython_only +class ImmortalTests(unittest.TestCase): + + if sys.maxsize < (1 << 32): + IMMORTAL_REFCOUNT_MINIMUM = 1 << 30 + else: + IMMORTAL_REFCOUNT_MINIMUM = 1 << 31 + + IMMORTALS = (None, True, False, Ellipsis, NotImplemented, *range(-5, 257)) + + def assert_immortal(self, immortal): + with self.subTest(immortal): + self.assertGreater(sys.getrefcount(immortal), self.IMMORTAL_REFCOUNT_MINIMUM) + + def test_immortals(self): + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_list_repeat_respect_immortality(self): + refs = list(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_tuple_repeat_respect_immortality(self): + refs = tuple(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + class TestType(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_new_type(self): A = type('A', (), {}) self.assertEqual(A.__name__, 'A') @@ -2375,6 +2858,7 @@ def test_new_type(self): self.assertEqual(A.__module__, __name__) self.assertEqual(A.__bases__, (object,)) self.assertIs(A.__base__, object) + self.assertNotIn('__firstlineno__', A.__dict__) x = A() self.assertIs(type(x), A) self.assertIs(x.__class__, A) @@ -2404,8 +2888,6 @@ def test_type_nokwargs(self): with self.assertRaises(TypeError): type('a', (), dict={}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_type_name(self): for name in 'A', '\xc4', '\U0001f40d', 'B.A', '42', '': with self.subTest(name=name): @@ -2439,8 +2921,6 @@ def test_type_name(self): A.__name__ = b'A' self.assertEqual(A.__name__, 'C') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_type_qualname(self): A = type('A', (), {'__qualname__': 'B.C'}) self.assertEqual(A.__name__, 'A') @@ -2457,8 +2937,29 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_type_firstlineno(self): + A = type('A', (), {'__firstlineno__': 42}) + self.assertEqual(A.__name__, 'A') + self.assertEqual(A.__module__, __name__) + self.assertEqual(A.__dict__['__firstlineno__'], 42) + A.__module__ = 'testmodule' + self.assertEqual(A.__module__, 'testmodule') + self.assertNotIn('__firstlineno__', A.__dict__) + A.__firstlineno__ = 43 + self.assertEqual(A.__dict__['__firstlineno__'], 43) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'tuple' but 'str' found. + def test_type_typeparams(self): + class A[T]: + pass + T, = A.__type_params__ + self.assertIsInstance(T, typing.TypeVar) + A.__type_params__ = "whatever" + self.assertEqual(A.__type_params__, "whatever") + with self.assertRaises(TypeError): + del A.__type_params__ + self.assertEqual(A.__type_params__, "whatever") + def test_type_doc(self): for doc in 'x', '\xc4', '\U0001f40d', 'x\x00y', b'x', 42, None: A = type('A', (), {'__doc__': doc}) @@ -2472,8 +2973,6 @@ def test_type_doc(self): A.__doc__ = doc self.assertEqual(A.__doc__, doc) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_args(self): with self.assertRaises(TypeError): type() @@ -2494,8 +2993,6 @@ def test_bad_args(self): with self.assertRaises(TypeError): type('A', (int, str), {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_slots(self): with self.assertRaises(TypeError): type('A', (), {'__slots__': b'x'}) @@ -2534,7 +3031,8 @@ def test_namespace_order(self): def load_tests(loader, tests, pattern): from doctest import DocTestSuite - tests.addTest(DocTestSuite(builtins)) + if sys.float_repr_style == 'short': + tests.addTest(DocTestSuite(builtins)) return tests if __name__ == "__main__": diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index baf84642ee8..fcef9c0c972 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -10,6 +10,7 @@ import sys import copy import functools +import operator import pickle import tempfile import textwrap @@ -46,6 +47,10 @@ def __index__(self): class BaseBytesTest: + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + def test_basics(self): b = self.type2test() self.assertEqual(type(b), self.type2test) @@ -196,8 +201,7 @@ def test_constructor_value_errors(self): self.assertRaises(ValueError, self.type2test, [sys.maxsize+1]) self.assertRaises(ValueError, self.type2test, [10**100]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @bigaddrspacetest def test_constructor_overflow(self): size = MAX_Py_ssize_t @@ -209,8 +213,6 @@ def test_constructor_overflow(self): except (OverflowError, MemoryError): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_constructor_exceptions(self): # Issue #34974: bytes and bytearray constructors replace unexpected # exceptions. @@ -323,8 +325,7 @@ def test_decode(self): # Default encoding is utf-8 self.assertEqual(self.type2test(b'\xe2\x98\x83').decode(), '\u2603') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_check_encoding_errors(self): # bpo-37388: bytes(str) and bytes.encode() must check encoding # and errors arguments in dev mode @@ -739,6 +740,37 @@ def check(fmt, vals, result): check(b'%i%b %*.*b', (10, b'3', 5, 3, b'abc',), b'103 abc') check(b'%c', b'a', b'a') + class PseudoFloat: + def __init__(self, value): + self.value = float(value) + def __int__(self): + return int(self.value) + + pi = PseudoFloat(3.1415) + + exceptions_params = [ + ('%x format: an integer is required, not float', b'%x', 3.14), + ('%X format: an integer is required, not float', b'%X', 2.11), + ('%o format: an integer is required, not float', b'%o', 1.79), + ('%x format: an integer is required, not PseudoFloat', b'%x', pi), + ('%x format: an integer is required, not complex', b'%x', 3j), + ('%X format: an integer is required, not complex', b'%X', 2j), + ('%o format: an integer is required, not complex', b'%o', 1j), + ('%u format: a real number is required, not complex', b'%u', 3j), + # See https://github.com/python/cpython/issues/130928 as for why + # the exception message contains '%d' instead of '%i'. + ('%d format: a real number is required, not complex', b'%i', 2j), + ('%d format: a real number is required, not complex', b'%d', 2j), + ( + r'%c requires an integer in range\(256\) or a single byte', + b'%c', pi + ), + ] + + for msg, format_bytes, value in exceptions_params: + with self.assertRaisesRegex(TypeError, msg): + operator.mod(format_bytes, value) + def test_imod(self): b = self.type2test(b'hello, %b!') orig = b @@ -938,8 +970,7 @@ def test_integer_arguments_out_of_byte_range(self): self.assertRaises(ValueError, method, 256) self.assertRaises(ValueError, method, 9999) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_etc_raise_correct_error_messages(self): # issue 11828 b = self.type2test(b'hello') @@ -959,8 +990,7 @@ def test_find_etc_raise_correct_error_messages(self): self.assertRaisesRegex(TypeError, r'\bendswith\b', b.endswith, x, None, None, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): test.support.check_free_after_iterating(self, iter, self.type2test) test.support.check_free_after_iterating(self, reversed, self.type2test) @@ -997,13 +1027,13 @@ def test_translate(self): self.assertEqual(c, b'hllo') def test_sq_item(self): - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') obj = self.type2test((42,)) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, -2) + _testlimitedcapi.sequence_getitem(obj, -2) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, 1) - self.assertEqual(_testcapi.sequence_getitem(obj, 0), 42) + _testlimitedcapi.sequence_getitem(obj, 1) + self.assertEqual(_testlimitedcapi.sequence_getitem(obj, 0), 42) class BytesTest(BaseBytesTest, unittest.TestCase): @@ -1033,36 +1063,63 @@ def test_buffer_is_readonly(self): self.assertRaises(TypeError, f.readinto, b"") def test_custom(self): - class A: - def __bytes__(self): - return b'abc' - self.assertEqual(bytes(A()), b'abc') - class A: pass - self.assertRaises(TypeError, bytes, A()) - class A: - def __bytes__(self): - return None - self.assertRaises(TypeError, bytes, A()) - class A: + self.assertEqual(bytes(BytesSubclass(b'abc')), b'abc') + self.assertEqual(BytesSubclass(OtherBytesSubclass(b'abc')), + BytesSubclass(b'abc')) + self.assertEqual(bytes(WithBytes(b'abc')), b'abc') + self.assertEqual(BytesSubclass(WithBytes(b'abc')), BytesSubclass(b'abc')) + + class NoBytes: pass + self.assertRaises(TypeError, bytes, NoBytes()) + self.assertRaises(TypeError, bytes, WithBytes('abc')) + self.assertRaises(TypeError, bytes, WithBytes(None)) + class IndexWithBytes: def __bytes__(self): return b'a' def __index__(self): return 42 - self.assertEqual(bytes(A()), b'a') + self.assertEqual(bytes(IndexWithBytes()), b'a') # Issue #25766 - class A(str): + class StrWithBytes(str): + def __new__(cls, value): + self = str.__new__(cls, '\u20ac') + self.value = value + return self def __bytes__(self): - return b'abc' - self.assertEqual(bytes(A('\u20ac')), b'abc') - self.assertEqual(bytes(A('\u20ac'), 'iso8859-15'), b'\xa4') + return self.value + self.assertEqual(bytes(StrWithBytes(b'abc')), b'abc') + self.assertEqual(bytes(StrWithBytes(b'abc'), 'iso8859-15'), b'\xa4') + self.assertEqual(bytes(StrWithBytes(BytesSubclass(b'abc'))), b'abc') + self.assertEqual(BytesSubclass(StrWithBytes(b'abc')), BytesSubclass(b'abc')) + self.assertEqual(BytesSubclass(StrWithBytes(b'abc'), 'iso8859-15'), + BytesSubclass(b'\xa4')) + self.assertEqual(BytesSubclass(StrWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertEqual(BytesSubclass(StrWithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) # Issue #24731 - class A: + self.assertTypedEqual(bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(WithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(WithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + + class BytesWithBytes(bytes): + def __new__(cls, value): + self = bytes.__new__(cls, b'\xa4') + self.value = value + return self def __bytes__(self): - return OtherBytesSubclass(b'abc') - self.assertEqual(bytes(A()), b'abc') - self.assertIs(type(bytes(A())), OtherBytesSubclass) - self.assertEqual(BytesSubclass(A()), b'abc') - self.assertIs(type(BytesSubclass(A())), BytesSubclass) + return self.value + self.assertTypedEqual(bytes(BytesWithBytes(b'abc')), b'abc') + self.assertTypedEqual(BytesSubclass(BytesWithBytes(b'abc')), + BytesSubclass(b'abc')) + self.assertTypedEqual(bytes(BytesWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(BytesWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(BytesWithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) # Test PyBytes_FromFormat() def test_from_format(self): @@ -1231,10 +1288,21 @@ class SubBytes(bytes): self.assertNotEqual(id(s), id(1 * s)) self.assertNotEqual(id(s), id(s * 2)) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_fromhex(self): + return super().test_fromhex() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mod(self): + return super().test_mod() + class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray + # XXX: RUSTPYTHON; import_helper.import_module here cause the entire test stopping + _testlimitedcapi = None # import_helper.import_module('_testlimitedcapi') + def test_getitem_error(self): b = bytearray(b'python') msg = "bytearray indices must be integers or slices" @@ -1326,48 +1394,76 @@ def by(s): b = by("Hello, world") self.assertEqual(re.findall(br"\w+", b), [by("Hello"), by("world")]) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_setitem(self): - b = bytearray([1, 2, 3]) - b[1] = 100 - self.assertEqual(b, bytearray([1, 100, 3])) - b[-1] = 200 - self.assertEqual(b, bytearray([1, 100, 200])) - b[0] = Indexable(10) - self.assertEqual(b, bytearray([10, 100, 200])) - try: - b[3] = 0 - self.fail("Didn't raise IndexError") - except IndexError: - pass - try: - b[-10] = 0 - self.fail("Didn't raise IndexError") - except IndexError: - pass - try: - b[0] = 256 - self.fail("Didn't raise ValueError") - except ValueError: - pass - try: - b[0] = Indexable(-1) - self.fail("Didn't raise ValueError") - except ValueError: - pass - try: - b[0] = None - self.fail("Didn't raise TypeError") - except TypeError: - pass + def setitem_as_mapping(b, i, val): + b[i] = val + + def setitem_as_sequence(b, i, val): + self._testlimitedcapi.sequence_setitem(b, i, val) + + def do_tests(setitem): + b = bytearray([1, 2, 3]) + setitem(b, 1, 100) + self.assertEqual(b, bytearray([1, 100, 3])) + setitem(b, -1, 200) + self.assertEqual(b, bytearray([1, 100, 200])) + setitem(b, 0, Indexable(10)) + self.assertEqual(b, bytearray([10, 100, 200])) + try: + setitem(b, 3, 0) + self.fail("Didn't raise IndexError") + except IndexError: + pass + try: + setitem(b, -10, 0) + self.fail("Didn't raise IndexError") + except IndexError: + pass + try: + setitem(b, 0, 256) + self.fail("Didn't raise ValueError") + except ValueError: + pass + try: + setitem(b, 0, Indexable(-1)) + self.fail("Didn't raise ValueError") + except ValueError: + pass + try: + setitem(b, 0, object()) + self.fail("Didn't raise TypeError") + except TypeError: + pass + + with self.subTest("tp_as_mapping"): + do_tests(setitem_as_mapping) + + with self.subTest("tp_as_sequence"): + do_tests(setitem_as_sequence) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_delitem(self): - b = bytearray(range(10)) - del b[0] - self.assertEqual(b, bytearray(range(1, 10))) - del b[-1] - self.assertEqual(b, bytearray(range(1, 9))) - del b[4] - self.assertEqual(b, bytearray([1, 2, 3, 4, 6, 7, 8])) + def del_as_mapping(b, i): + del b[i] + + def del_as_sequence(b, i): + self._testlimitedcapi.sequence_delitem(b, i) + + def do_tests(delete): + b = bytearray(range(10)) + delete(b, 0) + self.assertEqual(b, bytearray(range(1, 10))) + delete(b, -1) + self.assertEqual(b, bytearray(range(1, 9))) + delete(b, 4) + self.assertEqual(b, bytearray([1, 2, 3, 4, 6, 7, 8])) + + with self.subTest("tp_as_mapping"): + do_tests(del_as_mapping) + + with self.subTest("tp_as_sequence"): + do_tests(del_as_sequence) def test_setslice(self): b = bytearray(range(10)) @@ -1533,6 +1629,7 @@ def g(): alloc = b.__alloc__() self.assertGreaterEqual(alloc, len(b)) # NOTE: RUSTPYTHON patched + @unittest.expectedFailure # TODO: RUSTPYTHON def test_extend(self): orig = b'hello' a = bytearray(orig) @@ -1560,6 +1657,13 @@ def test_extend(self): a = bytearray(b'') a.extend([Indexable(ord('a'))]) self.assertEqual(a, b'a') + a = bytearray(b'abc') + self.assertRaisesRegex(TypeError, # Override for string. + "expected iterable of integers; got: 'str'", + a.extend, 'def') + self.assertRaisesRegex(TypeError, # But not for others. + "can't extend bytearray with float", + a.extend, 1.0) def test_remove(self): b = bytearray(b'hello') @@ -1748,7 +1852,12 @@ def test_repeat_after_setslice(self): self.assertEqual(b1, b) self.assertEqual(b3, b'xcxcxc') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mutating_index(self): + # bytearray slice assignment can call into python code + # that reallocates the internal buffer + # See gh-91153 + class Boom: def __index__(self): b.clear() @@ -1760,10 +1869,51 @@ def __index__(self): b[0] = Boom() with self.subTest("tp_as_sequence"): - _testcapi = import_helper.import_module('_testcapi') b = bytearray(b'Now you see me...') with self.assertRaises(IndexError): - _testcapi.sequence_setitem(b, 0, Boom()) + self._testlimitedcapi.sequence_setitem(b, 0, Boom()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mutating_index_inbounds(self): + # gh-91153 continued + # Ensure buffer is not broken even if length is correct + + class MutatesOnIndex: + def __init__(self): + self.ba = bytearray(0x180) + + def __index__(self): + self.ba.clear() + self.new_ba = bytearray(0x180) # to catch out-of-bounds writes + self.ba.extend([0] * 0x180) # to check bounds checks + return 0 + + with self.subTest("skip_bounds_safety"): + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_capi"): + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self._testlimitedcapi.sequence_setitem(instance.ba, instance, ord("?")) + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_slice"): + instance = MutatesOnIndex() + instance.ba[instance:1] = [ord("?")] + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_fromhex(self): + return super().test_fromhex() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mod(self): + return super().test_mod() class AssortedBytesTest(unittest.TestCase): @@ -1771,16 +1921,45 @@ class AssortedBytesTest(unittest.TestCase): # Test various combinations of bytes and bytearray # + def test_bytes_repr(self, f=repr): + self.assertEqual(f(b''), "b''") + self.assertEqual(f(b"abc"), "b'abc'") + self.assertEqual(f(bytes([92])), r"b'\\'") + self.assertEqual(f(bytes([0, 1, 254, 255])), r"b'\x00\x01\xfe\xff'") + self.assertEqual(f(b'\a\b\t\n\v\f\r'), r"b'\x07\x08\t\n\x0b\x0c\r'") + self.assertEqual(f(b'"'), """b'"'""") # '"' + self.assertEqual(f(b"'"), '''b"'"''') # "'" + self.assertEqual(f(b"'\""), r"""b'\'"'""") # '\'"' + self.assertEqual(f(b"\"'\""), r"""b'"\'"'""") # '"\'"' + self.assertEqual(f(b"'\"'"), r"""b'\'"\''""") # '\'"\'' + self.assertEqual(f(BytesSubclass(b"abc")), "b'abc'") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytearray_repr(self, f=repr): + self.assertEqual(f(bytearray()), "bytearray(b'')") + self.assertEqual(f(bytearray(b'abc')), "bytearray(b'abc')") + self.assertEqual(f(bytearray([92])), r"bytearray(b'\\')") + self.assertEqual(f(bytearray([0, 1, 254, 255])), + r"bytearray(b'\x00\x01\xfe\xff')") + self.assertEqual(f(bytearray([7, 8, 9, 10, 11, 12, 13])), + r"bytearray(b'\x07\x08\t\n\x0b\x0c\r')") + self.assertEqual(f(bytearray(b'"')), """bytearray(b'"')""") # '"' + self.assertEqual(f(bytearray(b"'")), r'''bytearray(b"\'")''') # "\'" + self.assertEqual(f(bytearray(b"'\"")), r"""bytearray(b'\'"')""") # '\'"' + self.assertEqual(f(bytearray(b"\"'\"")), r"""bytearray(b'"\'"')""") # '"\'"' + self.assertEqual(f(bytearray(b'\'"\'')), r"""bytearray(b'\'"\'')""") # '\'"\'' + self.assertEqual(f(ByteArraySubclass(b"abc")), "ByteArraySubclass(b'abc')") + self.assertEqual(f(ByteArraySubclass.Nested(b"abc")), "Nested(b'abc')") + self.assertEqual(f(ByteArraySubclass.Ŭñıçöđë(b"abc")), "Ŭñıçöđë(b'abc')") + @check_bytes_warnings - def test_repr_str(self): - for f in str, repr: - self.assertEqual(f(bytearray()), "bytearray(b'')") - self.assertEqual(f(bytearray([0])), "bytearray(b'\\x00')") - self.assertEqual(f(bytearray([0, 1, 254, 255])), - "bytearray(b'\\x00\\x01\\xfe\\xff')") - self.assertEqual(f(b"abc"), "b'abc'") - self.assertEqual(f(b"'"), '''b"'"''') # ''' - self.assertEqual(f(b"'\""), r"""b'\'"'""") # ' + def test_bytes_str(self): + self.test_bytes_repr(str) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @check_bytes_warnings + def test_bytearray_str(self): + self.test_bytearray_repr(str) @check_bytes_warnings def test_format(self): @@ -1833,15 +2012,6 @@ def test_from_bytearray(self): b = bytearray(buf) self.assertEqual(b, bytearray(sample)) - @check_bytes_warnings - def test_to_str(self): - self.assertEqual(str(b''), "b''") - self.assertEqual(str(b'x'), "b'x'") - self.assertEqual(str(b'\x80'), "b'\\x80'") - self.assertEqual(str(bytearray(b'')), "bytearray(b'')") - self.assertEqual(str(bytearray(b'x')), "bytearray(b'x')") - self.assertEqual(str(bytearray(b'\x80')), "bytearray(b'\\x80')") - def test_literal(self): tests = [ (b"Wonderful spam", "Wonderful spam"), @@ -1992,7 +2162,6 @@ def test_join(self): s3 = s1.join([b"abcd"]) self.assertIs(type(s3), self.basetype) - @unittest.skip("TODO: RUSTPYHON, Fails on ByteArraySubclassWithSlotsTest") def test_pickle(self): a = self.type2test(b"abcd") a.x = 10 @@ -2007,7 +2176,6 @@ def test_pickle(self): self.assertEqual(type(a.z), type(b.z)) self.assertFalse(hasattr(b, 'y')) - @unittest.skip("TODO: RUSTPYHON, Fails on ByteArraySubclassWithSlotsTest") def test_copy(self): a = self.type2test(b"abcd") a.x = 10 @@ -2051,7 +2219,10 @@ def __init__(me, *args, **kwargs): class ByteArraySubclass(bytearray): - pass + class Nested(bytearray): + pass + class Ŭñıçöđë(bytearray): + pass class ByteArraySubclassWithSlots(bytearray): __slots__ = ('x', 'y', '__dict__') @@ -2062,6 +2233,12 @@ class BytesSubclass(bytes): class OtherBytesSubclass(bytes): pass +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + class ByteArraySubclassTest(SubclassTest, unittest.TestCase): basetype = bytearray type2test = ByteArraySubclass @@ -2080,6 +2257,14 @@ class ByteArraySubclassWithSlotsTest(SubclassTest, unittest.TestCase): basetype = bytearray type2test = ByteArraySubclassWithSlots + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_copy(self): + return super().test_copy() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickle(self): + return super().test_pickle() + class BytesSubclassTest(SubclassTest, unittest.TestCase): basetype = bytes type2test = BytesSubclass diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 1f0b9adc369..148d8f98c79 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -3,20 +3,20 @@ import array import unittest +import io from io import BytesIO, DEFAULT_BUFFER_SIZE import os import pickle import glob import tempfile -import pathlib import random import shutil import subprocess import threading from test.support import import_helper from test.support import threading_helper -from test.support.os_helper import unlink -import _compression +from test.support.os_helper import unlink, FakePath +from compression._common import _streams import sys @@ -126,15 +126,15 @@ def testReadMultiStream(self): def testReadMonkeyMultiStream(self): # Test BZ2File.read() on a multi-stream archive where a stream # boundary coincides with the end of the raw read buffer. - buffer_size = _compression.BUFFER_SIZE - _compression.BUFFER_SIZE = len(self.DATA) + buffer_size = _streams.BUFFER_SIZE + _streams.BUFFER_SIZE = len(self.DATA) try: self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, float()) self.assertEqual(bz2f.read(), self.TEXT * 5) finally: - _compression.BUFFER_SIZE = buffer_size + _streams.BUFFER_SIZE = buffer_size def testReadTrailingJunk(self): self.createTempFile(suffix=self.BAD_DATA) @@ -184,7 +184,7 @@ def testPeek(self): with BZ2File(self.filename) as bz2f: pdata = bz2f.peek() self.assertNotEqual(len(pdata), 0) - self.assertTrue(self.TEXT.startswith(pdata)) + self.assertStartsWith(self.TEXT, pdata) self.assertEqual(bz2f.read(), self.TEXT) def testReadInto(self): @@ -476,7 +476,6 @@ def testReadlinesNoNewline(self): self.assertEqual(xlines, [b'Test']) def testContextProtocol(self): - f = None with BZ2File(self.filename, "wb") as f: f.write(b"xxx") f = BZ2File(self.filename, "rb") @@ -537,31 +536,214 @@ def testMultiStreamOrdering(self): with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(), data1 + data2) + def testOpenFilename(self): + with BZ2File(self.filename, "wb") as f: + f.write(b'content') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, "ab") as f: + f.write(b'appendix') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithName(self): + with open(self.filename, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithoutName(self): + bio = BytesIO() + with BZ2File(bio, 'wb') as f: + f.write(b'content') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + with BZ2File(bio, 'ab') as f: + f.write(b'appendix') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + bio.seek(0) + with BZ2File(bio, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'rb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + def testOpenFileWithIntName(self): + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with open(fd, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with open(fd, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + def testOpenBytesFilename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, bytes_filename) with BZ2File(bytes_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with BZ2File(str_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, str_filename) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: != 'Z:\\TEMP\\tmphoipjcen' def testOpenPathLikeFilename(self): - filename = pathlib.Path(self.filename) + filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, self.filename) with BZ2File(filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, self.filename) def testDecompressLimited(self): """Decompressed data buffering should be limited""" bomb = bz2.compress(b'\0' * int(2e6), compresslevel=9) - self.assertLess(len(bomb), _compression.BUFFER_SIZE) + self.assertLess(len(bomb), _streams.BUFFER_SIZE) decomp = BZ2File(BytesIO(bomb)) self.assertEqual(decomp.read(1), b'\0') @@ -577,6 +759,9 @@ def testReadBytesIO(self): with BZ2File(bio) as bz2f: self.assertRaises(TypeError, bz2f.read, float()) self.assertEqual(bz2f.read(), self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'rb') self.assertFalse(bio.closed) def testPeekBytesIO(self): @@ -584,7 +769,7 @@ def testPeekBytesIO(self): with BZ2File(bio) as bz2f: pdata = bz2f.peek() self.assertNotEqual(len(pdata), 0) - self.assertTrue(self.TEXT.startswith(pdata)) + self.assertStartsWith(self.TEXT, pdata) self.assertEqual(bz2f.read(), self.TEXT) def testWriteBytesIO(self): @@ -592,6 +777,9 @@ def testWriteBytesIO(self): with BZ2File(bio, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'wb') self.assertEqual(ext_decompress(bio.getvalue()), self.TEXT) self.assertFalse(bio.closed) diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 818dc16b834..71f934756e2 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -243,8 +243,6 @@ def setUpClass(cls): if not AVAILABLE_TARGETS: raise unittest.SkipTest("No C-with-UTF-8 locale available") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_external_target_locale_configuration(self): # Explicitly setting a target locale should give the same behaviour as diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 42490c8366b..7ade4271b7a 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -3,11 +3,13 @@ from test import support from test.support.script_helper import assert_python_ok, assert_python_failure -import time -import locale -import sys +import contextlib import datetime +import io +import locale import os +import sys +import time # From https://en.wikipedia.org/wiki/Leap_year_starting_on_Saturday result_0_02_text = """\ @@ -455,6 +457,11 @@ def test_formatmonth(self): calendar.TextCalendar().formatmonth(0, 2), result_0_02_text ) + def test_formatmonth_with_invalid_month(self): + with self.assertRaises(calendar.IllegalMonthError): + calendar.TextCalendar().formatmonth(2017, 13) + with self.assertRaises(calendar.IllegalMonthError): + calendar.TextCalendar().formatmonth(2017, -1) def test_formatmonthname_with_year(self): self.assertEqual( @@ -490,6 +497,14 @@ def test_format(self): self.assertEqual(out.getvalue().strip(), "1 2 3") class CalendarTestCase(unittest.TestCase): + + def test_deprecation_warning(self): + with self.assertWarnsRegex( + DeprecationWarning, + "The 'January' attribute is deprecated, use 'JANUARY' instead" + ): + calendar.January + def test_isleap(self): # Make sure that the return is right for a few years, and # ensure that the return values are 1 or 0, not just true or @@ -541,26 +556,92 @@ def test_months(self): # verify it "acts like a sequence" in two forms of iteration self.assertEqual(value[::-1], list(reversed(value))) - def test_locale_calendars(self): + def test_locale_text_calendar(self): + try: + cal = calendar.LocaleTextCalendar(locale='') + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + except locale.Error: + # cannot set the system default locale -- skip rest of test + raise unittest.SkipTest('cannot set the system default locale') + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + cal = calendar.LocaleTextCalendar(locale=None) + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + cal = calendar.LocaleTextCalendar(locale='C') + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + def test_locale_html_calendar(self): + try: + cal = calendar.LocaleHTMLCalendar(locale='') + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + except locale.Error: + # cannot set the system default locale -- skip rest of test + raise unittest.SkipTest('cannot set the system default locale') + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + cal = calendar.LocaleHTMLCalendar(locale=None) + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + cal = calendar.LocaleHTMLCalendar(locale='C') + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + def test_locale_calendars_reset_locale_properly(self): # ensure that Locale{Text,HTML}Calendar resets the locale properly # (it is still not thread-safe though) old_october = calendar.TextCalendar().formatmonthname(2010, 10, 10) try: cal = calendar.LocaleTextCalendar(locale='') local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) local_month = cal.formatmonthname(2010, 10, 10) except locale.Error: # cannot set the system default locale -- skip rest of test raise unittest.SkipTest('cannot set the system default locale') self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) self.assertIsInstance(local_month, str) self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) self.assertGreaterEqual(len(local_month), 10) + cal = calendar.LocaleHTMLCalendar(locale='') local_weekday = cal.formatweekday(1) local_month = cal.formatmonthname(2010, 10) self.assertIsInstance(local_weekday, str) self.assertIsInstance(local_month, str) + new_october = calendar.TextCalendar().formatmonthname(2010, 10, 10) self.assertEqual(old_october, new_october) @@ -568,15 +649,34 @@ def test_locale_calendar_formatweekday(self): try: # formatweekday uses different day names based on the available width. cal = calendar.LocaleTextCalendar(locale='en_US') + # For really short widths, the abbreviated name is truncated. + self.assertEqual(cal.formatweekday(0, 1), "M") + self.assertEqual(cal.formatweekday(0, 2), "Mo") # For short widths, a centered, abbreviated name is used. + self.assertEqual(cal.formatweekday(0, 3), "Mon") self.assertEqual(cal.formatweekday(0, 5), " Mon ") - # For really short widths, even the abbreviated name is truncated. - self.assertEqual(cal.formatweekday(0, 2), "Mo") + self.assertEqual(cal.formatweekday(0, 8), " Mon ") # For long widths, the full day name is used. + self.assertEqual(cal.formatweekday(0, 9), " Monday ") self.assertEqual(cal.formatweekday(0, 10), " Monday ") except locale.Error: raise unittest.SkipTest('cannot set the en_US locale') + def test_locale_calendar_formatmonthname(self): + try: + # formatmonthname uses the same month names regardless of the width argument. + cal = calendar.LocaleTextCalendar(locale='en_US') + # For too short widths, a full name (with year) is used. + self.assertEqual(cal.formatmonthname(2022, 6, 2, withyear=False), "June") + self.assertEqual(cal.formatmonthname(2022, 6, 2, withyear=True), "June 2022") + self.assertEqual(cal.formatmonthname(2022, 6, 3, withyear=False), "June") + self.assertEqual(cal.formatmonthname(2022, 6, 3, withyear=True), "June 2022") + # For long widths, a centered name is used. + self.assertEqual(cal.formatmonthname(2022, 6, 10, withyear=False), " June ") + self.assertEqual(cal.formatmonthname(2022, 6, 15, withyear=True), " June 2022 ") + except locale.Error: + raise unittest.SkipTest('cannot set the en_US locale') + def test_locale_html_calendar_custom_css_class_month_name(self): try: cal = calendar.LocaleHTMLCalendar(locale='') @@ -832,51 +932,108 @@ def test_several_leapyears_in_range(self): def conv(s): - # XXX RUSTPYTHON TODO: TextIOWrapper newline translation - return s.encode() - # return s.replace('\n', os.linesep).encode() + return s.replace('\n', os.linesep).encode() class CommandLineTestCase(unittest.TestCase): - def run_ok(self, *args): + def setUp(self): + self.runners = [self.run_cli_ok, self.run_cmd_ok] + + @contextlib.contextmanager + def captured_stdout_with_buffer(self): + orig_stdout = sys.stdout + buffer = io.BytesIO() + sys.stdout = io.TextIOWrapper(buffer) + try: + yield sys.stdout + finally: + sys.stdout.flush() + sys.stdout.buffer.seek(0) + sys.stdout = orig_stdout + + @contextlib.contextmanager + def captured_stderr_with_buffer(self): + orig_stderr = sys.stderr + buffer = io.BytesIO() + sys.stderr = io.TextIOWrapper(buffer) + try: + yield sys.stderr + finally: + sys.stderr.flush() + sys.stderr.buffer.seek(0) + sys.stderr = orig_stderr + + def run_cli_ok(self, *args): + with self.captured_stdout_with_buffer() as stdout: + calendar.main(args) + return stdout.buffer.read() + + def run_cmd_ok(self, *args): return assert_python_ok('-m', 'calendar', *args)[1] - def assertFailure(self, *args): + def assertCLIFails(self, *args): + with self.captured_stderr_with_buffer() as stderr: + self.assertRaises(SystemExit, calendar.main, args) + stderr = stderr.buffer.read() + self.assertIn(b'usage:', stderr) + return stderr + + def assertCmdFails(self, *args): rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) self.assertIn(b'usage:', stderr) self.assertEqual(rc, 2) + return rc, stdout, stderr + def assertFailure(self, *args): + self.assertCLIFails(*args) + self.assertCmdFails(*args) + + @support.force_not_colorized def test_help(self): - stdout = self.run_ok('-h') + stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) - self.assertIn(b'calendar.py', stdout) + self.assertIn(b' -m calendar ', stdout) self.assertIn(b'--help', stdout) + # special case: stdout but sys.exit() + with self.captured_stdout_with_buffer() as output: + self.assertRaises(SystemExit, calendar.main, ['-h']) + output = output.buffer.read() + self.assertIn(b'usage:', output) + self.assertIn(b'--help', output) + def test_illegal_arguments(self): self.assertFailure('-z') self.assertFailure('spam') self.assertFailure('2004', 'spam') + self.assertFailure('2004', '1', 'spam') + self.assertFailure('2004', '1', '1') + self.assertFailure('2004', '1', '1', 'spam') self.assertFailure('-t', 'html', '2004', '1') def test_output_current_year(self): - stdout = self.run_ok() - year = datetime.datetime.now().year - self.assertIn((' %s' % year).encode(), stdout) - self.assertIn(b'January', stdout) - self.assertIn(b'Mo Tu We Th Fr Sa Su', stdout) + for run in self.runners: + output = run() + year = datetime.datetime.now().year + self.assertIn(conv(' %s' % year), output) + self.assertIn(b'January', output) + self.assertIn(b'Mo Tu We Th Fr Sa Su', output) def test_output_year(self): - stdout = self.run_ok('2004') - self.assertEqual(stdout, conv(result_2004_text)) + for run in self.runners: + output = run('2004') + self.assertEqual(output, conv(result_2004_text)) def test_output_month(self): - stdout = self.run_ok('2004', '1') - self.assertEqual(stdout, conv(result_2004_01_text)) + for run in self.runners: + output = run('2004', '1') + self.assertEqual(output, conv(result_2004_01_text)) def test_option_encoding(self): self.assertFailure('-e') self.assertFailure('--encoding') - stdout = self.run_ok('--encoding', 'utf-16-le', '2004') - self.assertEqual(stdout, result_2004_text.encode('utf-16-le')) + for run in self.runners: + output = run('--encoding', 'utf-16-le', '2004') + self.assertEqual(output, result_2004_text.encode('utf-16-le')) def test_option_locale(self): self.assertFailure('-L') @@ -894,66 +1051,75 @@ def test_option_locale(self): locale.setlocale(locale.LC_TIME, oldlocale) except (locale.Error, ValueError): self.skipTest('cannot set the system default locale') - stdout = self.run_ok('--locale', lang, '--encoding', enc, '2004') - self.assertIn('2004'.encode(enc), stdout) + for run in self.runners: + for type in ('text', 'html'): + output = run( + '--type', type, '--locale', lang, '--encoding', enc, '2004' + ) + self.assertIn('2004'.encode(enc), output) def test_option_width(self): self.assertFailure('-w') self.assertFailure('--width') self.assertFailure('-w', 'spam') - stdout = self.run_ok('--width', '3', '2004') - self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', stdout) + for run in self.runners: + output = run('--width', '3', '2004') + self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', output) def test_option_lines(self): self.assertFailure('-l') self.assertFailure('--lines') self.assertFailure('-l', 'spam') - stdout = self.run_ok('--lines', '2', '2004') - self.assertIn(conv('December\n\nMo Tu We'), stdout) + for run in self.runners: + output = run('--lines', '2', '2004') + self.assertIn(conv('December\n\nMo Tu We'), output) def test_option_spacing(self): self.assertFailure('-s') self.assertFailure('--spacing') self.assertFailure('-s', 'spam') - stdout = self.run_ok('--spacing', '8', '2004') - self.assertIn(b'Su Mo', stdout) + for run in self.runners: + output = run('--spacing', '8', '2004') + self.assertIn(b'Su Mo', output) def test_option_months(self): self.assertFailure('-m') self.assertFailure('--month') self.assertFailure('-m', 'spam') - stdout = self.run_ok('--months', '1', '2004') - self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), stdout) + for run in self.runners: + output = run('--months', '1', '2004') + self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), output) def test_option_type(self): self.assertFailure('-t') self.assertFailure('--type') self.assertFailure('-t', 'spam') - stdout = self.run_ok('--type', 'text', '2004') - self.assertEqual(stdout, conv(result_2004_text)) - stdout = self.run_ok('--type', 'html', '2004') - self.assertEqual(stdout[:6], b'Calendar for 2004', stdout) + for run in self.runners: + output = run('--type', 'text', '2004') + self.assertEqual(output, conv(result_2004_text)) + output = run('--type', 'html', '2004') + self.assertStartsWith(output, b'Calendar for 2004', output) def test_html_output_current_year(self): - stdout = self.run_ok('--type', 'html') - year = datetime.datetime.now().year - self.assertIn(('Calendar for %s' % year).encode(), - stdout) - self.assertIn(b'January', - stdout) + for run in self.runners: + output = run('--type', 'html') + year = datetime.datetime.now().year + self.assertIn(('Calendar for %s' % year).encode(), output) + self.assertIn(b'January', output) def test_html_output_year_encoding(self): - stdout = self.run_ok('-t', 'html', '--encoding', 'ascii', '2004') - self.assertEqual(stdout, - result_2004_html.format(**default_format).encode('ascii')) + for run in self.runners: + output = run('-t', 'html', '--encoding', 'ascii', '2004') + self.assertEqual(output, result_2004_html.format(**default_format).encode('ascii')) def test_html_output_year_css(self): self.assertFailure('-t', 'html', '-c') self.assertFailure('-t', 'html', '--css') - stdout = self.run_ok('-t', 'html', '--css', 'custom.css', '2004') - self.assertIn(b'', stdout) + for run in self.runners: + output = run('-t', 'html', '--css', 'custom.css', '2004') + self.assertIn(b'', output) class MiscTestCase(unittest.TestCase): @@ -961,7 +1127,7 @@ def test__all__(self): not_exported = { 'mdays', 'January', 'February', 'EPOCH', 'different_locale', 'c', 'prweek', 'week', 'format', - 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth'} + 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth', ""} support.check__all__(self, calendar, not_exported=not_exported) @@ -989,6 +1155,13 @@ def test_formatmonth(self): self.assertIn('class="text-center month"', self.cal.formatmonth(2017, 5)) + def test_formatmonth_with_invalid_month(self): + with self.assertRaises(calendar.IllegalMonthError): + self.cal.formatmonth(2017, 13) + with self.assertRaises(calendar.IllegalMonthError): + self.cal.formatmonth(2017, -1) + + def test_formatweek(self): weeks = self.cal.monthdays2calendar(2017, 5) self.assertIn('class="wed text-nowrap"', self.cal.formatweek(weeks[0])) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 8e64ffffd09..86ba0aa4b63 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,20 +1,33 @@ -import datetime import unittest -from test.support import cpython_only +from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, + set_recursion_limit, skip_on_s390x, import_helper) try: import _testcapi except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None import struct import collections import itertools import gc +import contextlib +import sys +import types + + +class BadStr(str): + def __eq__(self, other): + return True + def __hash__(self): + # Guaranteed different hash + return str.__hash__(self) ^ 3 class FunctionCalls(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_kwargs_order(self): # bpo-34320: **kwargs should preserve order of passed OrderedDict od = collections.OrderedDict([('a', 1), ('b', 2)]) @@ -28,124 +41,22 @@ def fn(**kw): self.assertIsInstance(res, dict) self.assertEqual(list(res.items()), expected) - -# The test cases here cover several paths through the function calling -# code. They depend on the METH_XXX flag that is used to define a C -# function, which can't be verified from Python. If the METH_XXX decl -# for a C function changes, these tests may not cover the right paths. - -class CFunctionCalls(unittest.TestCase): - - def test_varargs0(self): - self.assertRaises(TypeError, {}.__contains__) - - def test_varargs1(self): - {}.__contains__(0) - - def test_varargs2(self): - self.assertRaises(TypeError, {}.__contains__, 0, 1) - - def test_varargs0_ext(self): - try: - {}.__contains__(*()) - except TypeError: - pass - - def test_varargs1_ext(self): - {}.__contains__(*(0,)) - - def test_varargs2_ext(self): - try: - {}.__contains__(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_varargs1_kw(self): - self.assertRaises(TypeError, {}.__contains__, x=2) - - def test_varargs2_kw(self): - self.assertRaises(TypeError, {}.__contains__, x=2, y=2) - - def test_oldargs0_0(self): - {}.keys() - - def test_oldargs0_1(self): - self.assertRaises(TypeError, {}.keys, 0) - - def test_oldargs0_2(self): - self.assertRaises(TypeError, {}.keys, 0, 1) - - def test_oldargs0_0_ext(self): - {}.keys(*()) - - def test_oldargs0_1_ext(self): - try: - {}.keys(*(0,)) - except TypeError: + def test_frames_are_popped_after_failed_calls(self): + # GH-93252: stuff blows up if we don't pop the new frame after + # recovering from failed calls: + def f(): pass - else: - raise RuntimeError - - def test_oldargs0_2_ext(self): - try: - {}.keys(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs0_0_kw(self): - try: - {}.keys(x=2) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs0_1_kw(self): - self.assertRaises(TypeError, {}.keys, x=2) - - def test_oldargs0_2_kw(self): - self.assertRaises(TypeError, {}.keys, x=2, y=2) - - def test_oldargs1_0(self): - self.assertRaises(TypeError, [].count) - - def test_oldargs1_1(self): - [].count(1) - - def test_oldargs1_2(self): - self.assertRaises(TypeError, [].count, 1, 2) - - def test_oldargs1_0_ext(self): - try: - [].count(*()) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs1_1_ext(self): - [].count(*(1,)) - - def test_oldargs1_2_ext(self): - try: - [].count(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs1_0_kw(self): - self.assertRaises(TypeError, [].count, x=2) - - def test_oldargs1_1_kw(self): - self.assertRaises(TypeError, [].count, {}, x=2) - - def test_oldargs1_2_kw(self): - self.assertRaises(TypeError, [].count, x=2, y=2) + class C: + def m(self): + pass + callables = [f, C.m, [].__len__] + for c in callables: + for _ in range(1000): + try: + c(None) + except TypeError: + pass + # BOOM! @cpython_only @@ -160,11 +71,12 @@ def test_varargs2(self): self.assertRaisesRegex(TypeError, msg, {}.__contains__, 0, 1) def test_varargs3(self): - msg = r"^from_bytes\(\) takes exactly 2 positional arguments \(3 given\)" + msg = r"^from_bytes\(\) takes at most 2 positional arguments \(3 given\)" self.assertRaisesRegex(TypeError, msg, int.from_bytes, b'a', 'little', False) def test_varargs1min(self): - msg = r"get expected at least 1 argument, got 0" + msg = (r"get\(\) takes at least 1 argument \(0 given\)|" + r"get expected at least 1 argument, got 0") self.assertRaisesRegex(TypeError, msg, {}.get) msg = r"expected 1 argument, got 0" @@ -175,11 +87,13 @@ def test_varargs2min(self): self.assertRaisesRegex(TypeError, msg, getattr) def test_varargs1max(self): - msg = r"input expected at most 1 argument, got 2" + msg = (r"input\(\) takes at most 1 argument \(2 given\)|" + r"input expected at most 1 argument, got 2") self.assertRaisesRegex(TypeError, msg, input, 1, 2) def test_varargs2max(self): - msg = r"get expected at most 2 arguments, got 3" + msg = (r"get\(\) takes at most 2 arguments \(3 given\)|" + r"get expected at most 2 arguments, got 3") self.assertRaisesRegex(TypeError, msg, {}.get, 1, 2, 3) def test_varargs1_kw(self): @@ -195,7 +109,7 @@ def test_varargs3_kw(self): self.assertRaisesRegex(TypeError, msg, bool, x=2) def test_varargs4_kw(self): - msg = r"^index\(\) takes no keyword arguments$" + msg = r"^(list[.])?index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, [].index, x=2) def test_varargs5_kw(self): @@ -211,19 +125,19 @@ def test_varargs7_kw(self): self.assertRaisesRegex(TypeError, msg, next, x=2) def test_varargs8_kw(self): - msg = r"^pack\(\) takes no keyword arguments$" + msg = r"^_struct[.]pack\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.pack, x=2) def test_varargs9_kw(self): - msg = r"^pack_into\(\) takes no keyword arguments$" + msg = r"^_struct[.]pack_into\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.pack_into, x=2) def test_varargs10_kw(self): - msg = r"^index\(\) takes no keyword arguments$" + msg = r"^deque[.]index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, collections.deque().index, x=2) def test_varargs11_kw(self): - msg = r"^pack\(\) takes no keyword arguments$" + msg = r"^Struct[.]pack\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.Struct.pack, struct.Struct(""), x=2) def test_varargs12_kw(self): @@ -240,9 +154,9 @@ def test_varargs14_kw(self): itertools.product, 0, repeat=1, foo=2) def test_varargs15_kw(self): - msg = r"^ImportError\(\) takes at most 2 keyword arguments \(3 given\)$" + msg = r"^ImportError\(\) takes at most 3 keyword arguments \(4 given\)$" self.assertRaisesRegex(TypeError, msg, - ImportError, 0, name=1, path=2, foo=3) + ImportError, 0, name=1, path=2, name_from=3, foo=3) def test_varargs16_kw(self): msg = r"^min\(\) takes at most 2 keyword arguments \(3 given\)$" @@ -250,10 +164,22 @@ def test_varargs16_kw(self): min, 0, default=1, key=2, foo=3) def test_varargs17_kw(self): - msg = r"^print\(\) takes at most 4 keyword arguments \(5 given\)$" + msg = r"print\(\) got an unexpected keyword argument 'foo'$" self.assertRaisesRegex(TypeError, msg, print, 0, sep=1, end=2, file=3, flush=4, foo=5) + def test_varargs18_kw(self): + # _PyArg_UnpackKeywordsWithVararg() + msg = r"invalid keyword argument for print\(\)$" + with self.assertRaisesRegex(TypeError, msg): + print(0, 1, **{BadStr('foo'): ','}) + + def test_varargs19_kw(self): + # _PyArg_UnpackKeywords() + msg = r"invalid keyword argument for round\(\)$" + with self.assertRaisesRegex(TypeError, msg): + round(1.75, **{BadStr('foo'): 1}) + def test_oldargs0_1(self): msg = r"keys\(\) takes no arguments \(1 given\)" self.assertRaisesRegex(TypeError, msg, {}.keys, 0) @@ -290,6 +216,208 @@ def test_oldargs1_2_kw(self): msg = r"count\(\) takes no keyword arguments" self.assertRaisesRegex(TypeError, msg, [].count, x=2, y=2) + def test_object_not_callable(self): + msg = r"^'object' object is not callable$" + self.assertRaisesRegex(TypeError, msg, object()) + + def test_module_not_callable_no_suggestion_0(self): + msg = r"^'module' object is not callable$" + self.assertRaisesRegex(TypeError, msg, types.ModuleType("mod")) + + def test_module_not_callable_no_suggestion_1(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.mod = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_2(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + del mod.__name__ + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_3(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.__name__ = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_suggestion(self): + msg = r"^'module' object is not callable\. Did you mean: 'mod\.mod\(\.\.\.\)'\?$" + mod = types.ModuleType("mod") + mod.mod = lambda: ... + self.assertRaisesRegex(TypeError, msg, mod) + + +@unittest.skipIf(_testcapi is None, "requires _testcapi") +class TestCallingConventions(unittest.TestCase): + """Test calling using various C calling conventions (METH_*) from Python + + Subclasses test several kinds of functions (module-level, methods, + class methods static methods) using these attributes: + obj: the object that contains tested functions (as attributes) + expected_self: expected "self" argument to the C function + + The base class tests module-level functions. + """ + + def setUp(self): + self.obj = self.expected_self = _testcapi + + def test_varargs(self): + self.assertEqual( + self.obj.meth_varargs(1, 2, 3), + (self.expected_self, (1, 2, 3)), + ) + + def test_varargs_ext(self): + self.assertEqual( + self.obj.meth_varargs(*(1, 2, 3)), + (self.expected_self, (1, 2, 3)), + ) + + def test_varargs_error_kw(self): + msg = r"meth_varargs\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_varargs(k=1), + ) + + def test_varargs_keywords(self): + self.assertEqual( + self.obj.meth_varargs_keywords(1, 2, a=3, b=4), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_varargs_keywords_ext(self): + self.assertEqual( + self.obj.meth_varargs_keywords(*[1, 2], **{'a': 3, 'b': 4}), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_o(self): + self.assertEqual(self.obj.meth_o(1), (self.expected_self, 1)) + + def test_o_ext(self): + self.assertEqual(self.obj.meth_o(*[1]), (self.expected_self, 1)) + + def test_o_error_no_arg(self): + msg = r"meth_o\(\) takes exactly one argument \(0 given\)" + self.assertRaisesRegex(TypeError, msg, self.obj.meth_o) + + def test_o_error_two_args(self): + msg = r"meth_o\(\) takes exactly one argument \(2 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(1, 2), + ) + + def test_o_error_ext(self): + msg = r"meth_o\(\) takes exactly one argument \(3 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(*(1, 2, 3)), + ) + + def test_o_error_kw(self): + msg = r"meth_o\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(k=1), + ) + + def test_o_error_arg_kw(self): + msg = r"meth_o\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(k=1), + ) + + def test_noargs(self): + self.assertEqual(self.obj.meth_noargs(), self.expected_self) + + def test_noargs_ext(self): + self.assertEqual(self.obj.meth_noargs(*[]), self.expected_self) + + def test_noargs_error_arg(self): + msg = r"meth_noargs\(\) takes no arguments \(1 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(1), + ) + + def test_noargs_error_arg2(self): + msg = r"meth_noargs\(\) takes no arguments \(2 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(1, 2), + ) + + def test_noargs_error_ext(self): + msg = r"meth_noargs\(\) takes no arguments \(3 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(*(1, 2, 3)), + ) + + def test_noargs_error_kw(self): + msg = r"meth_noargs\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(k=1), + ) + + def test_fastcall(self): + self.assertEqual( + self.obj.meth_fastcall(1, 2, 3), + (self.expected_self, (1, 2, 3)), + ) + + def test_fastcall_ext(self): + self.assertEqual( + self.obj.meth_fastcall(*(1, 2, 3)), + (self.expected_self, (1, 2, 3)), + ) + + def test_fastcall_error_kw(self): + msg = r"meth_fastcall\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_fastcall(k=1), + ) + + def test_fastcall_keywords(self): + self.assertEqual( + self.obj.meth_fastcall_keywords(1, 2, a=3, b=4), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_fastcall_keywords_ext(self): + self.assertEqual( + self.obj.meth_fastcall_keywords(*(1, 2), **{'a': 3, 'b': 4}), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + +class TestCallingConventionsInstance(TestCallingConventions): + """Test calling instance methods using various calling conventions""" + + def setUp(self): + self.obj = self.expected_self = _testcapi.MethInstance() + + +class TestCallingConventionsClass(TestCallingConventions): + """Test calling class methods using various calling conventions""" + + def setUp(self): + self.obj = self.expected_self = _testcapi.MethClass + + +class TestCallingConventionsClassInstance(TestCallingConventions): + """Test calling class methods on instance""" + + def setUp(self): + self.obj = _testcapi.MethClass() + self.expected_self = _testcapi.MethClass + + +class TestCallingConventionsStatic(TestCallingConventions): + """Test calling static methods using various calling conventions""" + + def setUp(self): + self.obj = _testcapi.MethStatic() + self.expected_self = None + def pyfunc(arg1, arg2): return [arg1, arg2] @@ -317,14 +445,15 @@ def static_method(): PYTHON_INSTANCE = PythonClass() - -IGNORE_RESULT = object() +NULL_OR_EMPTY = object() -@cpython_only class FastCallTests(unittest.TestCase): + """Test calling using various callables from C + """ + # Test calls with positional arguments - CALLS_POSARGS = ( + CALLS_POSARGS = [ # (func, args: tuple, result) # Python function with 2 arguments @@ -343,31 +472,11 @@ class FastCallTests(unittest.TestCase): (PYTHON_INSTANCE.class_method, (), "classmethod"), (PYTHON_INSTANCE.static_method, (), "staticmethod"), - # C function: METH_NOARGS - (globals, (), IGNORE_RESULT), - - # C function: METH_O - (id, ("hello",), IGNORE_RESULT), - - # C function: METH_VARARGS - (dir, (1,), IGNORE_RESULT), - - # C function: METH_VARARGS | METH_KEYWORDS - (min, (5, 9), 5), - - # C function: METH_FASTCALL - (divmod, (1000, 33), (30, 10)), - - # C type static method: METH_FASTCALL | METH_CLASS - (int.from_bytes, (b'\x01\x00', 'little'), 1), - - # bpo-30524: Test that calling a C type static method with no argument - # doesn't crash (ignore the result): METH_FASTCALL | METH_CLASS - (datetime.datetime.now, (), IGNORE_RESULT), - ) + # C callables are added later + ] # Test calls with positional and keyword arguments - CALLS_KWARGS = ( + CALLS_KWARGS = [ # (func, args: tuple, kwargs: dict, result) # Python function with 2 arguments @@ -378,34 +487,57 @@ class FastCallTests(unittest.TestCase): (PYTHON_INSTANCE.method, (1,), {'arg2': 2}, [1, 2]), (PYTHON_INSTANCE.method, (), {'arg1': 1, 'arg2': 2}, [1, 2]), - # C function: METH_VARARGS | METH_KEYWORDS - (max, ([],), {'default': 9}, 9), - - # C type static method: METH_FASTCALL | METH_CLASS - (int.from_bytes, (b'\x01\x00',), {'byteorder': 'little'}, 1), - (int.from_bytes, (), {'bytes': b'\x01\x00', 'byteorder': 'little'}, 1), - ) + # C callables are added later + ] + + # Add all the calling conventions and variants of C callables + if _testcapi: + _instance = _testcapi.MethInstance() + for obj, expected_self in ( + (_testcapi, _testcapi), # module-level function + (_instance, _instance), # bound method + (_testcapi.MethClass, _testcapi.MethClass), # class method on class + (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. + (_testcapi.MethStatic, None), # static method + ): + CALLS_POSARGS.extend([ + (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), + (obj.meth_varargs_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), + (obj.meth_fastcall, (), (expected_self, ())), + (obj.meth_fastcall_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (), (expected_self, (), NULL_OR_EMPTY)), + (obj.meth_noargs, (), expected_self), + (obj.meth_o, (123, ), (expected_self, 123)), + ]) + + CALLS_KWARGS.extend([ + (obj.meth_varargs_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_varargs_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_varargs_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + ]) def check_result(self, result, expected): - if expected is IGNORE_RESULT: - return + if isinstance(expected, tuple) and expected[-1] is NULL_OR_EMPTY: + if result[-1] in ({}, None): + expected = (*expected[:-1], result[-1]) self.assertEqual(result, expected) - def test_fastcall(self): - # Test _PyObject_FastCall() - - for func, args, expected in self.CALLS_POSARGS: - with self.subTest(func=func, args=args): - result = _testcapi.pyobject_fastcall(func, args) - self.check_result(result, expected) - - if not args: - # args=NULL, nargs=0 - result = _testcapi.pyobject_fastcall(func, None) - self.check_result(result, expected) - + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall_dict(self): - # Test _PyObject_FastCallDict() + # Test PyObject_VectorcallDict() for func, args, expected in self.CALLS_POSARGS: with self.subTest(func=func, args=args): @@ -413,26 +545,19 @@ def test_vectorcall_dict(self): result = _testcapi.pyobject_fastcalldict(func, args, None) self.check_result(result, expected) - # kwargs={} - result = _testcapi.pyobject_fastcalldict(func, args, {}) - self.check_result(result, expected) - if not args: # args=NULL, nargs=0, kwargs=NULL result = _testcapi.pyobject_fastcalldict(func, None, None) self.check_result(result, expected) - # args=NULL, nargs=0, kwargs={} - result = _testcapi.pyobject_fastcalldict(func, None, {}) - self.check_result(result, expected) - for func, args, kwargs, expected in self.CALLS_KWARGS: with self.subTest(func=func, args=args, kwargs=kwargs): result = _testcapi.pyobject_fastcalldict(func, args, kwargs) self.check_result(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall(self): - # Test _PyObject_Vectorcall() + # Test PyObject_Vectorcall() for func, args, expected in self.CALLS_POSARGS: with self.subTest(func=func, args=args): @@ -460,6 +585,7 @@ def test_vectorcall(self): result = _testcapi.pyobject_vectorcall(func, args, kwnames) self.check_result(result, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'IntWithDict' found. def test_fastcall_clearing_dict(self): # Test bpo-36907: the point of the test is just checking that this # does not crash. @@ -471,7 +597,7 @@ def __index__(self): self.kwargs.clear() gc.collect() return 0 - x = IntWithDict(dont_inherit=IntWithDict()) + x = IntWithDict(optimize=IntWithDict()) # We test the argument handling of "compile" here, the compilation # itself is not relevant. When we pass flags=x below, x.__index__() is # called, which changes the keywords dict. @@ -492,10 +618,12 @@ def testfunction_kw(self, *, kw): return self +ADAPTIVE_WARMUP_DELAY = 2 + + +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_method_descriptor_flag(self): import functools cached = functools.lru_cache(1)(testfunction) @@ -510,26 +638,32 @@ def test_method_descriptor_flag(self): self.assertTrue(_testcapi.MethodDescriptorDerived.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) self.assertFalse(_testcapi.MethodDescriptorNopGet.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) - # Heap type should not inherit Py_TPFLAGS_METHOD_DESCRIPTOR + # Mutable heap types should not inherit Py_TPFLAGS_METHOD_DESCRIPTOR class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_vectorcall_flag(self): self.assertTrue(_testcapi.MethodDescriptorBase.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertTrue(_testcapi.MethodDescriptorDerived.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertFalse(_testcapi.MethodDescriptorNopGet.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertTrue(_testcapi.MethodDescriptor2.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) - # Heap type should not inherit Py_TPFLAGS_HAVE_VECTORCALL + # Mutable heap types should inherit Py_TPFLAGS_HAVE_VECTORCALL, + # but should lose it when __call__ is overridden class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): pass + self.assertTrue(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + MethodDescriptorHeap.__call__ = print + self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + + # Mutable heap types should not inherit Py_TPFLAGS_HAVE_VECTORCALL if + # they define __call__ directly + class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): + def __call__(self): + pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_vectorcall_override(self): # Check that tp_call can correctly override vectorcall. # MethodDescriptorNopGet implements tp_call but it inherits from @@ -540,14 +674,64 @@ def test_vectorcall_override(self): f = _testcapi.MethodDescriptorNopGet() self.assertIs(f(*args), args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_vectorcall_override_on_mutable_class(self): + """Setting __call__ should disable vectorcall""" + TestType = _testcapi.make_vectorcall_class() + instance = TestType() + self.assertEqual(instance(), "tp_call") + instance.set_vectorcall(TestType) + self.assertEqual(instance(), "vectorcall") # assume vectorcall is used + TestType.__call__ = lambda self: "custom" + self.assertEqual(instance(), "custom") + + def test_vectorcall_override_with_subclass(self): + """Setting __call__ on a superclass should disable vectorcall""" + SuperType = _testcapi.make_vectorcall_class() + class DerivedType(SuperType): + pass + + instance = DerivedType() + + # Derived types with its own vectorcall should be unaffected + UnaffectedType1 = _testcapi.make_vectorcall_class(DerivedType) + UnaffectedType2 = _testcapi.make_vectorcall_class(SuperType) + + # Aside: Quickly check that the C helper actually made derived types + self.assertTrue(issubclass(UnaffectedType1, DerivedType)) + self.assertTrue(issubclass(UnaffectedType2, SuperType)) + + # Initial state: tp_call + self.assertEqual(instance(), "tp_call") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + # Setting the vectorcall function + instance.set_vectorcall(SuperType) + + self.assertEqual(instance(), "vectorcall") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + # Setting __call__ should remove vectorcall from all subclasses + SuperType.__call__ = lambda self: "custom" + + self.assertEqual(instance(), "custom") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), False) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), False) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + def test_vectorcall(self): # Test a bunch of different ways to call objects: # 1. vectorcall using PyVectorcall_Call() # (only for objects that support vectorcall directly) # 2. normal call - # 3. vectorcall using _PyObject_Vectorcall() + # 3. vectorcall using PyObject_Vectorcall() # 4. call as bound method # 5. call using functools.partial @@ -618,6 +802,300 @@ def __call__(self, *args): self.assertEqual(expected, meth(*args1, **kwargs)) self.assertEqual(expected, wrapped(*args, **kwargs)) + def test_setvectorcall(self): + from _testcapi import function_setvectorcall + def f(num): return num + 1 + assert_equal = self.assertEqual + num = 10 + assert_equal(11, f(num)) + function_setvectorcall(f) + # make sure specializer is triggered by running > 50 times + for _ in range(10 * ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", f(num)) + + def test_setvectorcall_load_attr_specialization_skip(self): + from _testcapi import function_setvectorcall + + class X: + def __getattribute__(self, attr): + return attr + + assert_equal = self.assertEqual + x = X() + assert_equal("a", x.a) + function_setvectorcall(X.__getattribute__) + # make sure specialization doesn't trigger + # when vectorcall is overridden + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", x.a) + + def test_setvectorcall_load_attr_specialization_deopt(self): + from _testcapi import function_setvectorcall + + class X: + def __getattribute__(self, attr): + return attr + + def get_a(x): + return x.a + + assert_equal = self.assertEqual + x = X() + # trigger LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN specialization + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("a", get_a(x)) + function_setvectorcall(X.__getattribute__) + # make sure specialized LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN + # gets deopted due to overridden vectorcall + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", get_a(x)) + + @requires_limited_api + def test_vectorcall_limited_incoming(self): + from _testcapi import pyobject_vectorcall + obj = _testlimitedcapi.LimitedVectorCallClass() + self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called") + + @requires_limited_api + def test_vectorcall_limited_outgoing(self): + from _testlimitedcapi import call_vectorcall + + args_captured = [] + kwargs_captured = [] + + def f(*args, **kwargs): + args_captured.append(args) + kwargs_captured.append(kwargs) + return "success" + + self.assertEqual(call_vectorcall(f), "success") + self.assertEqual(args_captured, [("foo",)]) + self.assertEqual(kwargs_captured, [{"baz": "bar"}]) + + @requires_limited_api + def test_vectorcall_limited_outgoing_method(self): + from _testlimitedcapi import call_vectorcall_method + + args_captured = [] + kwargs_captured = [] + + class TestInstance: + def f(self, *args, **kwargs): + args_captured.append(args) + kwargs_captured.append(kwargs) + return "success" + + self.assertEqual(call_vectorcall_method(TestInstance()), "success") + self.assertEqual(args_captured, [("foo",)]) + self.assertEqual(kwargs_captured, [{"baz": "bar"}]) + +class A: + def method_two_args(self, x, y): + pass + + @staticmethod + def static_no_args(): + pass + + @staticmethod + def positional_only(arg, /): + pass + +@cpython_only +class TestErrorMessagesUseQualifiedName(unittest.TestCase): + + @contextlib.contextmanager + def check_raises_type_error(self, message): + with self.assertRaises(TypeError) as cm: + yield + self.assertEqual(str(cm.exception), message) + + def test_missing_arguments(self): + msg = "A.method_two_args() missing 1 required positional argument: 'y'" + with self.check_raises_type_error(msg): + A().method_two_args("x") + + def test_too_many_positional(self): + msg = "A.static_no_args() takes 0 positional arguments but 1 was given" + with self.check_raises_type_error(msg): + A.static_no_args("oops it's an arg") + + def test_positional_only_passed_as_keyword(self): + msg = "A.positional_only() got some positional-only arguments passed as keyword arguments: 'arg'" + with self.check_raises_type_error(msg): + A.positional_only(arg="x") + + def test_unexpected_keyword(self): + msg = "A.method_two_args() got an unexpected keyword argument 'bad'" + with self.check_raises_type_error(msg): + A().method_two_args(bad="x") + + def test_multiple_values(self): + msg = "A.method_two_args() got multiple values for argument 'x'" + with self.check_raises_type_error(msg): + A().method_two_args("x", "y", x="oops") + +@cpython_only +class TestErrorMessagesSuggestions(unittest.TestCase): + @contextlib.contextmanager + def check_suggestion_includes(self, message): + with self.assertRaises(TypeError) as cm: + yield + self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) + + @contextlib.contextmanager + def check_suggestion_not_present(self): + with self.assertRaises(TypeError) as cm: + yield + self.assertNotIn("Did you mean", str(cm.exception)) + + def test_unexpected_keyword_suggestion_valid_positions(self): + def foo(blech=None, /, aaa=None, *args, late1=None): + pass + + cases = [ + ("blach", None), + ("aa", "aaa"), + ("orgs", None), + ("late11", "late1"), + ] + + for keyword, suggestion in cases: + with self.subTest(keyword): + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_present() + with ctx: + foo(**{keyword:None}) + + def test_unexpected_keyword_suggestion_kinds(self): + + def substitution(noise=None, more_noise=None, a = None, blech = None): + pass + + def elimination(noise = None, more_noise = None, a = None, blch = None): + pass + + def addition(noise = None, more_noise = None, a = None, bluchin = None): + pass + + def substitution_over_elimination(blach = None, bluc = None): + pass + + def substitution_over_addition(blach = None, bluchi = None): + pass + + def elimination_over_addition(bluc = None, blucha = None): + pass + + def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): + pass + + for func, suggestion in [ + (addition, "bluchin"), + (substitution, "blech"), + (elimination, "blch"), + (addition, "bluchin"), + (substitution_over_elimination, "blach"), + (substitution_over_addition, "blach"), + (elimination_over_addition, "bluc"), + (case_change_over_substitution, "BLuch"), + ]: + with self.subTest(suggestion): + with self.check_suggestion_includes(suggestion): + func(bluch=None) + + def test_unexpected_keyword_suggestion_via_getargs(self): + with self.check_suggestion_includes("maxsplit"): + "foo".split(maxsplt=1) + + self.assertRaisesRegex( + TypeError, r"split\(\) got an unexpected keyword argument 'blech'$", + "foo".split, blech=1 + ) + with self.check_suggestion_not_present(): + "foo".split(blech=1) + with self.check_suggestion_not_present(): + "foo".split(more_noise=1, maxsplt=1) + + # Also test the vgetargskeywords path + with self.check_suggestion_includes("name"): + ImportError(namez="oops") + + self.assertRaisesRegex( + TypeError, r"ImportError\(\) got an unexpected keyword argument 'blech'$", + ImportError, blech=1 + ) + with self.check_suggestion_not_present(): + ImportError(blech=1) + with self.check_suggestion_not_present(): + ImportError(blech=1, namez="oops") + +@cpython_only +class TestRecursion(unittest.TestCase): + + @skip_on_s390x + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_super_deep(self): + + def recurse(n): + if n: + recurse(n-1) + + def py_recurse(n, m): + if n: + py_recurse(n-1, m) + else: + c_py_recurse(m-1) + + def c_recurse(n): + if n: + _testcapi.pyobject_vectorcall(c_recurse, (n-1,), ()) + + def c_py_recurse(m): + if m: + _testcapi.pyobject_vectorcall(py_recurse, (1000, m), ()) + + with set_recursion_limit(100_000): + recurse(90_000) + with self.assertRaises(RecursionError): + recurse(101_000) + c_recurse(100) + with self.assertRaises(RecursionError): + c_recurse(90_000) + c_py_recurse(90) + with self.assertRaises(RecursionError): + c_py_recurse(100_000) + + +class TestFunctionWithManyArgs(unittest.TestCase): + def test_function_with_many_args(self): + for N in (10, 500, 1000): + with self.subTest(N=N): + args = ",".join([f"a{i}" for i in range(N)]) + src = f"def f({args}) : return a{N//2}" + l = {} + exec(src, {}, l) + self.assertEqual(l['f'](*range(N)), N//2) + + +@unittest.skipIf(_testcapi is None, 'need _testcapi') +class TestCAPI(unittest.TestCase): + def test_cfunction_call(self): + def func(*args, **kwargs): + return (args, kwargs) + + # PyCFunction_Call() was removed in Python 3.13 API, but was kept in + # the stable ABI. + def PyCFunction_Call(func, *args, **kwargs): + if kwargs: + return _testcapi.pycfunction_call(func, args, kwargs) + else: + return _testcapi.pycfunction_call(func, args) + + self.assertEqual(PyCFunction_Call(func), ((), {})) + self.assertEqual(PyCFunction_Call(func, 1, 2, 3), ((1, 2, 3), {})) + self.assertEqual(PyCFunction_Call(func, "arg", num=5), (("arg",), {'num': 5})) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py deleted file mode 100644 index 43164cff314..00000000000 --- a/Lib/test/test_cgi.py +++ /dev/null @@ -1,645 +0,0 @@ -import os -import sys -import tempfile -import unittest -from collections import namedtuple -from io import StringIO, BytesIO -from test import support -from test.support import warnings_helper - -cgi = warnings_helper.import_deprecated("cgi") - - -class HackedSysModule: - # The regression test will have real values in sys.argv, which - # will completely confuse the test of the cgi module - argv = [] - stdin = sys.stdin - -cgi.sys = HackedSysModule() - -class ComparableException: - def __init__(self, err): - self.err = err - - def __str__(self): - return str(self.err) - - def __eq__(self, anExc): - if not isinstance(anExc, Exception): - return NotImplemented - return (self.err.__class__ == anExc.__class__ and - self.err.args == anExc.args) - - def __getattr__(self, attr): - return getattr(self.err, attr) - -def do_test(buf, method): - env = {} - if method == "GET": - fp = None - env['REQUEST_METHOD'] = 'GET' - env['QUERY_STRING'] = buf - elif method == "POST": - fp = BytesIO(buf.encode('latin-1')) # FieldStorage expects bytes - env['REQUEST_METHOD'] = 'POST' - env['CONTENT_TYPE'] = 'application/x-www-form-urlencoded' - env['CONTENT_LENGTH'] = str(len(buf)) - else: - raise ValueError("unknown method: %s" % method) - try: - return cgi.parse(fp, env, strict_parsing=1) - except Exception as err: - return ComparableException(err) - -parse_strict_test_cases = [ - ("", {}), - ("&", ValueError("bad query field: ''")), - ("&&", ValueError("bad query field: ''")), - # Should the next few really be valid? - ("=", {}), - ("=&=", {}), - # This rest seem to make sense - ("=a", {'': ['a']}), - ("&=a", ValueError("bad query field: ''")), - ("=a&", ValueError("bad query field: ''")), - ("=&a", ValueError("bad query field: 'a'")), - ("b=a", {'b': ['a']}), - ("b+=a", {'b ': ['a']}), - ("a=b=a", {'a': ['b=a']}), - ("a=+b=a", {'a': [' b=a']}), - ("&b=a", ValueError("bad query field: ''")), - ("b&=a", ValueError("bad query field: 'b'")), - ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), - ("a=a+b&a=b+a", {'a': ['a b', 'b a']}), - ("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), - ("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env", - {'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'], - 'cuyer': ['r'], - 'expire': ['964546263'], - 'kid': ['130003.300038'], - 'lobale': ['en-US'], - 'order_id': ['0bb2e248638833d48cb7fed300000f1b'], - 'ss': ['env'], - 'view': ['bustomer'], - }), - - ("group_id=5470&set=custom&_assigned_to=31392&_status=1&_category=100&SUBMIT=Browse", - {'SUBMIT': ['Browse'], - '_assigned_to': ['31392'], - '_category': ['100'], - '_status': ['1'], - 'group_id': ['5470'], - 'set': ['custom'], - }) - ] - -def norm(seq): - return sorted(seq, key=repr) - -def first_elts(list): - return [p[0] for p in list] - -def first_second_elts(list): - return [(p[0], p[1][0]) for p in list] - -def gen_result(data, environ): - encoding = 'latin-1' - fake_stdin = BytesIO(data.encode(encoding)) - fake_stdin.seek(0) - form = cgi.FieldStorage(fp=fake_stdin, environ=environ, encoding=encoding) - - result = {} - for k, v in dict(form).items(): - result[k] = isinstance(v, list) and form.getlist(k) or v.value - - return result - -class CgiTests(unittest.TestCase): - - def test_parse_multipart(self): - fp = BytesIO(POSTDATA.encode('latin1')) - env = {'boundary': BOUNDARY.encode('latin1'), - 'CONTENT-LENGTH': '558'} - result = cgi.parse_multipart(fp, env) - expected = {'submit': [' Add '], 'id': ['1234'], - 'file': [b'Testing 123.\n'], 'title': ['']} - self.assertEqual(result, expected) - - def test_parse_multipart_without_content_length(self): - POSTDATA = '''--JfISa01 -Content-Disposition: form-data; name="submit-name" - -just a string - ---JfISa01-- -''' - fp = BytesIO(POSTDATA.encode('latin1')) - env = {'boundary': 'JfISa01'.encode('latin1')} - result = cgi.parse_multipart(fp, env) - expected = {'submit-name': ['just a string\n']} - self.assertEqual(result, expected) - - # TODO RUSTPYTHON - see https://github.com/RustPython/RustPython/issues/935 - @unittest.expectedFailure - def test_parse_multipart_invalid_encoding(self): - BOUNDARY = "JfISa01" - POSTDATA = """--JfISa01 -Content-Disposition: form-data; name="submit-name" -Content-Length: 3 - -\u2603 ---JfISa01""" - fp = BytesIO(POSTDATA.encode('utf8')) - env = {'boundary': BOUNDARY.encode('latin1'), - 'CONTENT-LENGTH': str(len(POSTDATA.encode('utf8')))} - result = cgi.parse_multipart(fp, env, encoding="ascii", - errors="surrogateescape") - expected = {'submit-name': ["\udce2\udc98\udc83"]} - self.assertEqual(result, expected) - self.assertEqual("\u2603".encode('utf8'), - result["submit-name"][0].encode('utf8', 'surrogateescape')) - - def test_fieldstorage_properties(self): - fs = cgi.FieldStorage() - self.assertFalse(fs) - self.assertIn("FieldStorage", repr(fs)) - self.assertEqual(list(fs), list(fs.keys())) - fs.list.append(namedtuple('MockFieldStorage', 'name')('fieldvalue')) - self.assertTrue(fs) - - def test_fieldstorage_invalid(self): - self.assertRaises(TypeError, cgi.FieldStorage, "not-a-file-obj", - environ={"REQUEST_METHOD":"PUT"}) - self.assertRaises(TypeError, cgi.FieldStorage, "foo", "bar") - fs = cgi.FieldStorage(headers={'content-type':'text/plain'}) - self.assertRaises(TypeError, bool, fs) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_strict(self): - for orig, expect in parse_strict_test_cases: - # Test basic parsing - d = do_test(orig, "GET") - self.assertEqual(d, expect, "Error parsing %s method GET" % repr(orig)) - d = do_test(orig, "POST") - self.assertEqual(d, expect, "Error parsing %s method POST" % repr(orig)) - - env = {'QUERY_STRING': orig} - fs = cgi.FieldStorage(environ=env) - if isinstance(expect, dict): - # test dict interface - self.assertEqual(len(expect), len(fs)) - self.assertCountEqual(expect.keys(), fs.keys()) - ##self.assertEqual(norm(expect.values()), norm(fs.values())) - ##self.assertEqual(norm(expect.items()), norm(fs.items())) - self.assertEqual(fs.getvalue("nonexistent field", "default"), "default") - # test individual fields - for key in expect.keys(): - expect_val = expect[key] - self.assertIn(key, fs) - if len(expect_val) > 1: - self.assertEqual(fs.getvalue(key), expect_val) - else: - self.assertEqual(fs.getvalue(key), expect_val[0]) - - def test_separator(self): - parse_semicolon = [ - ("x=1;y=2.0", {'x': ['1'], 'y': ['2.0']}), - ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}), - (";", ValueError("bad query field: ''")), - (";;", ValueError("bad query field: ''")), - ("=;a", ValueError("bad query field: 'a'")), - (";b=a", ValueError("bad query field: ''")), - ("b;=a", ValueError("bad query field: 'b'")), - ("a=a+b;b=b+c", {'a': ['a b'], 'b': ['b c']}), - ("a=a+b;a=b+a", {'a': ['a b', 'b a']}), - ] - for orig, expect in parse_semicolon: - env = {'QUERY_STRING': orig} - fs = cgi.FieldStorage(separator=';', environ=env) - if isinstance(expect, dict): - for key in expect.keys(): - expect_val = expect[key] - self.assertIn(key, fs) - if len(expect_val) > 1: - self.assertEqual(fs.getvalue(key), expect_val) - else: - self.assertEqual(fs.getvalue(key), expect_val[0]) - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_log(self): - cgi.log("Testing") - - cgi.logfp = StringIO() - cgi.initlog("%s", "Testing initlog 1") - cgi.log("%s", "Testing log 2") - self.assertEqual(cgi.logfp.getvalue(), "Testing initlog 1\nTesting log 2\n") - if os.path.exists(os.devnull): - cgi.logfp = None - cgi.logfile = os.devnull - cgi.initlog("%s", "Testing log 3") - self.addCleanup(cgi.closelog) - cgi.log("Testing log 4") - - def test_fieldstorage_readline(self): - # FieldStorage uses readline, which has the capacity to read all - # contents of the input file into memory; we use readline's size argument - # to prevent that for files that do not contain any newlines in - # non-GET/HEAD requests - class TestReadlineFile: - def __init__(self, file): - self.file = file - self.numcalls = 0 - - def readline(self, size=None): - self.numcalls += 1 - if size: - return self.file.readline(size) - else: - return self.file.readline() - - def __getattr__(self, name): - file = self.__dict__['file'] - a = getattr(file, name) - if not isinstance(a, int): - setattr(self, name, a) - return a - - f = TestReadlineFile(tempfile.TemporaryFile("wb+")) - self.addCleanup(f.close) - f.write(b'x' * 256 * 1024) - f.seek(0) - env = {'REQUEST_METHOD':'PUT'} - fs = cgi.FieldStorage(fp=f, environ=env) - self.addCleanup(fs.file.close) - # if we're not chunking properly, readline is only called twice - # (by read_binary); if we are chunking properly, it will be called 5 times - # as long as the chunksize is 1 << 16. - self.assertGreater(f.numcalls, 2) - f.close() - - def test_fieldstorage_multipart(self): - #Test basic FieldStorage multipart parsing - env = { - 'REQUEST_METHOD': 'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), - 'CONTENT_LENGTH': '558'} - fp = BytesIO(POSTDATA.encode('latin-1')) - fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") - self.assertEqual(len(fs.list), 4) - expect = [{'name':'id', 'filename':None, 'value':'1234'}, - {'name':'title', 'filename':None, 'value':''}, - {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'}, - {'name':'submit', 'filename':None, 'value':' Add '}] - for x in range(len(fs.list)): - for k, exp in expect[x].items(): - got = getattr(fs.list[x], k) - self.assertEqual(got, exp) - - def test_fieldstorage_multipart_leading_whitespace(self): - env = { - 'REQUEST_METHOD': 'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), - 'CONTENT_LENGTH': '560'} - # Add some leading whitespace to our post data that will cause the - # first line to not be the innerboundary. - fp = BytesIO(b"\r\n" + POSTDATA.encode('latin-1')) - fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") - self.assertEqual(len(fs.list), 4) - expect = [{'name':'id', 'filename':None, 'value':'1234'}, - {'name':'title', 'filename':None, 'value':''}, - {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'}, - {'name':'submit', 'filename':None, 'value':' Add '}] - for x in range(len(fs.list)): - for k, exp in expect[x].items(): - got = getattr(fs.list[x], k) - self.assertEqual(got, exp) - - def test_fieldstorage_multipart_non_ascii(self): - #Test basic FieldStorage multipart parsing - env = {'REQUEST_METHOD':'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), - 'CONTENT_LENGTH':'558'} - for encoding in ['iso-8859-1','utf-8']: - fp = BytesIO(POSTDATA_NON_ASCII.encode(encoding)) - fs = cgi.FieldStorage(fp, environ=env,encoding=encoding) - self.assertEqual(len(fs.list), 1) - expect = [{'name':'id', 'filename':None, 'value':'\xe7\xf1\x80'}] - for x in range(len(fs.list)): - for k, exp in expect[x].items(): - got = getattr(fs.list[x], k) - self.assertEqual(got, exp) - - def test_fieldstorage_multipart_maxline(self): - # Issue #18167 - maxline = 1 << 16 - self.maxDiff = None - def check(content): - data = """---123 -Content-Disposition: form-data; name="upload"; filename="fake.txt" -Content-Type: text/plain - -%s ----123-- -""".replace('\n', '\r\n') % content - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', - 'REQUEST_METHOD': 'POST', - } - self.assertEqual(gen_result(data, environ), - {'upload': content.encode('latin1')}) - check('x' * (maxline - 1)) - check('x' * (maxline - 1) + '\r') - check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1)) - - def test_fieldstorage_multipart_w3c(self): - # Test basic FieldStorage multipart parsing (W3C sample) - env = { - 'REQUEST_METHOD': 'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY_W3), - 'CONTENT_LENGTH': str(len(POSTDATA_W3))} - fp = BytesIO(POSTDATA_W3.encode('latin-1')) - fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") - self.assertEqual(len(fs.list), 2) - self.assertEqual(fs.list[0].name, 'submit-name') - self.assertEqual(fs.list[0].value, 'Larry') - self.assertEqual(fs.list[1].name, 'files') - files = fs.list[1].value - self.assertEqual(len(files), 2) - expect = [{'name': None, 'filename': 'file1.txt', 'value': b'... contents of file1.txt ...'}, - {'name': None, 'filename': 'file2.gif', 'value': b'...contents of file2.gif...'}] - for x in range(len(files)): - for k, exp in expect[x].items(): - got = getattr(files[x], k) - self.assertEqual(got, exp) - - def test_fieldstorage_part_content_length(self): - BOUNDARY = "JfISa01" - POSTDATA = """--JfISa01 -Content-Disposition: form-data; name="submit-name" -Content-Length: 5 - -Larry ---JfISa01""" - env = { - 'REQUEST_METHOD': 'POST', - 'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY), - 'CONTENT_LENGTH': str(len(POSTDATA))} - fp = BytesIO(POSTDATA.encode('latin-1')) - fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1") - self.assertEqual(len(fs.list), 1) - self.assertEqual(fs.list[0].name, 'submit-name') - self.assertEqual(fs.list[0].value, 'Larry') - - def test_field_storage_multipart_no_content_length(self): - fp = BytesIO(b"""--MyBoundary -Content-Disposition: form-data; name="my-arg"; filename="foo" - -Test - ---MyBoundary-- -""") - env = { - "REQUEST_METHOD": "POST", - "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary", - "wsgi.input": fp, - } - fields = cgi.FieldStorage(fp, environ=env) - - self.assertEqual(len(fields["my-arg"].file.read()), 5) - - def test_fieldstorage_as_context_manager(self): - fp = BytesIO(b'x' * 10) - env = {'REQUEST_METHOD': 'PUT'} - with cgi.FieldStorage(fp=fp, environ=env) as fs: - content = fs.file.read() - self.assertFalse(fs.file.closed) - self.assertTrue(fs.file.closed) - self.assertEqual(content, 'x' * 10) - with self.assertRaisesRegex(ValueError, 'I/O operation on closed file'): - fs.file.read() - - _qs_result = { - 'key1': 'value1', - 'key2': ['value2x', 'value2y'], - 'key3': 'value3', - 'key4': 'value4' - } - def testQSAndUrlEncode(self): - data = "key2=value2x&key3=value3&key4=value4" - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'application/x-www-form-urlencoded', - 'QUERY_STRING': 'key1=value1&key2=value2y', - 'REQUEST_METHOD': 'POST', - } - v = gen_result(data, environ) - self.assertEqual(self._qs_result, v) - - def test_max_num_fields(self): - # For application/x-www-form-urlencoded - data = '&'.join(['a=a']*11) - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'application/x-www-form-urlencoded', - 'REQUEST_METHOD': 'POST', - } - - with self.assertRaises(ValueError): - cgi.FieldStorage( - fp=BytesIO(data.encode()), - environ=environ, - max_num_fields=10, - ) - - # For multipart/form-data - data = """---123 -Content-Disposition: form-data; name="a" - -3 ----123 -Content-Type: application/x-www-form-urlencoded - -a=4 ----123 -Content-Type: application/x-www-form-urlencoded - -a=5 ----123-- -""" - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', - 'QUERY_STRING': 'a=1&a=2', - 'REQUEST_METHOD': 'POST', - } - - # 2 GET entities - # 1 top level POST entities - # 1 entity within the second POST entity - # 1 entity within the third POST entity - with self.assertRaises(ValueError): - cgi.FieldStorage( - fp=BytesIO(data.encode()), - environ=environ, - max_num_fields=4, - ) - cgi.FieldStorage( - fp=BytesIO(data.encode()), - environ=environ, - max_num_fields=5, - ) - - def testQSAndFormData(self): - data = """---123 -Content-Disposition: form-data; name="key2" - -value2y ----123 -Content-Disposition: form-data; name="key3" - -value3 ----123 -Content-Disposition: form-data; name="key4" - -value4 ----123-- -""" - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', - 'QUERY_STRING': 'key1=value1&key2=value2x', - 'REQUEST_METHOD': 'POST', - } - v = gen_result(data, environ) - self.assertEqual(self._qs_result, v) - - def testQSAndFormDataFile(self): - data = """---123 -Content-Disposition: form-data; name="key2" - -value2y ----123 -Content-Disposition: form-data; name="key3" - -value3 ----123 -Content-Disposition: form-data; name="key4" - -value4 ----123 -Content-Disposition: form-data; name="upload"; filename="fake.txt" -Content-Type: text/plain - -this is the content of the fake file - ----123-- -""" - environ = { - 'CONTENT_LENGTH': str(len(data)), - 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', - 'QUERY_STRING': 'key1=value1&key2=value2x', - 'REQUEST_METHOD': 'POST', - } - result = self._qs_result.copy() - result.update({ - 'upload': b'this is the content of the fake file\n' - }) - v = gen_result(data, environ) - self.assertEqual(result, v) - - def test_parse_header(self): - self.assertEqual( - cgi.parse_header("text/plain"), - ("text/plain", {})) - self.assertEqual( - cgi.parse_header("text/vnd.just.made.this.up ; "), - ("text/vnd.just.made.this.up", {})) - self.assertEqual( - cgi.parse_header("text/plain;charset=us-ascii"), - ("text/plain", {"charset": "us-ascii"})) - self.assertEqual( - cgi.parse_header('text/plain ; charset="us-ascii"'), - ("text/plain", {"charset": "us-ascii"})) - self.assertEqual( - cgi.parse_header('text/plain ; charset="us-ascii"; another=opt'), - ("text/plain", {"charset": "us-ascii", "another": "opt"})) - self.assertEqual( - cgi.parse_header('attachment; filename="silly.txt"'), - ("attachment", {"filename": "silly.txt"})) - self.assertEqual( - cgi.parse_header('attachment; filename="strange;name"'), - ("attachment", {"filename": "strange;name"})) - self.assertEqual( - cgi.parse_header('attachment; filename="strange;name";size=123;'), - ("attachment", {"filename": "strange;name", "size": "123"})) - self.assertEqual( - cgi.parse_header('form-data; name="files"; filename="fo\\"o;bar"'), - ("form-data", {"name": "files", "filename": 'fo"o;bar'})) - - def test_all(self): - not_exported = { - "logfile", "logfp", "initlog", "dolog", "nolog", "closelog", "log", - "maxlen", "valid_boundary"} - support.check__all__(self, cgi, not_exported=not_exported) - - -BOUNDARY = "---------------------------721837373350705526688164684" - -POSTDATA = """-----------------------------721837373350705526688164684 -Content-Disposition: form-data; name="id" - -1234 ------------------------------721837373350705526688164684 -Content-Disposition: form-data; name="title" - - ------------------------------721837373350705526688164684 -Content-Disposition: form-data; name="file"; filename="test.txt" -Content-Type: text/plain - -Testing 123. - ------------------------------721837373350705526688164684 -Content-Disposition: form-data; name="submit" - - Add\x20 ------------------------------721837373350705526688164684-- -""" - -POSTDATA_NON_ASCII = """-----------------------------721837373350705526688164684 -Content-Disposition: form-data; name="id" - -\xe7\xf1\x80 ------------------------------721837373350705526688164684 -""" - -# http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4 -BOUNDARY_W3 = "AaB03x" -POSTDATA_W3 = """--AaB03x -Content-Disposition: form-data; name="submit-name" - -Larry ---AaB03x -Content-Disposition: form-data; name="files" -Content-Type: multipart/mixed; boundary=BbC04y - ---BbC04y -Content-Disposition: file; filename="file1.txt" -Content-Type: text/plain - -... contents of file1.txt ... ---BbC04y -Content-Disposition: file; filename="file2.gif" -Content-Type: image/gif -Content-Transfer-Encoding: binary - -...contents of file2.gif... ---BbC04y-- ---AaB03x-- -""" - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_cgitb.py b/Lib/test/test_cgitb.py deleted file mode 100644 index 501c7fcce28..00000000000 --- a/Lib/test/test_cgitb.py +++ /dev/null @@ -1,71 +0,0 @@ -from test.support.os_helper import temp_dir -from test.support.script_helper import assert_python_failure -from test.support.warnings_helper import import_deprecated -import unittest -import sys -cgitb = import_deprecated("cgitb") - -class TestCgitb(unittest.TestCase): - - def test_fonts(self): - text = "Hello Robbie!" - self.assertEqual(cgitb.small(text), "{}".format(text)) - self.assertEqual(cgitb.strong(text), "{}".format(text)) - self.assertEqual(cgitb.grey(text), - '{}'.format(text)) - - def test_blanks(self): - self.assertEqual(cgitb.small(""), "") - self.assertEqual(cgitb.strong(""), "") - self.assertEqual(cgitb.grey(""), "") - - def test_html(self): - try: - raise ValueError("Hello World") - except ValueError as err: - # If the html was templated we could do a bit more here. - # At least check that we get details on what we just raised. - html = cgitb.html(sys.exc_info()) - self.assertIn("ValueError", html) - self.assertIn(str(err), html) - - def test_text(self): - try: - raise ValueError("Hello World") - except ValueError: - text = cgitb.text(sys.exc_info()) - self.assertIn("ValueError", text) - self.assertIn("Hello World", text) - - def test_syshook_no_logdir_default_format(self): - with temp_dir() as tracedir: - rc, out, err = assert_python_failure( - '-c', - ('import cgitb; cgitb.enable(logdir=%s); ' - 'raise ValueError("Hello World")') % repr(tracedir), - PYTHONIOENCODING='utf-8') - out = out.decode() - self.assertIn("ValueError", out) - self.assertIn("Hello World", out) - self.assertIn("<module>", out) - # By default we emit HTML markup. - self.assertIn('

', out) - self.assertIn('

', out) - - def test_syshook_no_logdir_text_format(self): - # Issue 12890: we were emitting the

tag in text mode. - with temp_dir() as tracedir: - rc, out, err = assert_python_failure( - '-c', - ('import cgitb; cgitb.enable(format="text", logdir=%s); ' - 'raise ValueError("Hello World")') % repr(tracedir), - PYTHONIOENCODING='utf-8') - out = out.decode() - self.assertIn("ValueError", out) - self.assertIn("Hello World", out) - self.assertNotIn('

', out) - self.assertNotIn('

', out) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_charmapcodec.py b/Lib/test/test_charmapcodec.py index e69f1c6e4b8..0d4594d8c05 100644 --- a/Lib/test/test_charmapcodec.py +++ b/Lib/test/test_charmapcodec.py @@ -26,7 +26,6 @@ def codec_search_function(encoding): codecname = 'testcodec' class CharmapCodecTest(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_constructorx(self): self.assertEqual(str(b'abc', codecname), 'abc') self.assertEqual(str(b'xdef', codecname), 'abcdef') @@ -34,8 +33,6 @@ def test_constructorx(self): self.assertEqual(str(b'dxf', codecname), 'dabcf') self.assertEqual(str(b'dxfx', codecname), 'dabcfabc') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encodex(self): self.assertEqual('abc'.encode(codecname), b'abc') self.assertEqual('xdef'.encode(codecname), b'abcdef') @@ -43,14 +40,12 @@ def test_encodex(self): self.assertEqual('dxf'.encode(codecname), b'dabcf') self.assertEqual('dxfx'.encode(codecname), b'dabcfabc') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_constructory(self): self.assertEqual(str(b'ydef', codecname), 'def') self.assertEqual(str(b'defy', codecname), 'def') self.assertEqual(str(b'dyf', codecname), 'df') self.assertEqual(str(b'dyfy', codecname), 'df') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_maptoundefined(self): self.assertRaises(UnicodeError, str, b'abc\001', codecname) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index a6c99fbddf0..7420f289b16 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,7 +1,8 @@ "Test the functionality of Python classes implementing operators." import unittest - +from test import support +from test.support import cpython_only, import_helper, script_helper testmeths = [ @@ -134,6 +135,7 @@ def __%s__(self, *args): AllTests = type("AllTests", (object,), d) del d, statictests, method, method_template +@support.thread_unsafe("callLst is shared between threads") class ClassTests(unittest.TestCase): def setUp(self): callLst[:] = [] @@ -448,15 +450,15 @@ def __delattr__(self, *args): def testHasAttrString(self): import sys from test.support import import_helper - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') class A: def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) - self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): @@ -489,8 +491,6 @@ def index(x): for f in [float, complex, str, repr, bytes, bin, oct, hex, bool, index]: self.assertRaises(TypeError, f, BadTypeClass()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testHashStuff(self): # Test correct errors from hash() on objects with comparisons but # no __hash__ @@ -505,8 +505,60 @@ def __eq__(self, other): return 1 self.assertRaises(TypeError, hash, C2()) + def testPredefinedAttrs(self): + o = object() + + class Custom: + pass - @unittest.skip("TODO: RUSTPYTHON, segmentation fault") + c = Custom() + + methods = ( + '__class__', '__delattr__', '__dir__', '__eq__', '__format__', + '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', + '__init__', '__init_subclass__', '__le__', '__lt__', '__ne__', + '__new__', '__reduce__', '__reduce_ex__', '__repr__', + '__setattr__', '__sizeof__', '__str__', '__subclasshook__' + ) + for name in methods: + with self.subTest(name): + self.assertTrue(callable(getattr(object, name, None))) + self.assertTrue(callable(getattr(o, name, None))) + self.assertTrue(callable(getattr(Custom, name, None))) + self.assertTrue(callable(getattr(c, name, None))) + + not_defined = [ + '__abs__', '__aenter__', '__aexit__', '__aiter__', '__anext__', + '__await__', '__bool__', '__bytes__', '__ceil__', + '__complex__', '__contains__', '__del__', '__delete__', + '__delitem__', '__divmod__', '__enter__', '__exit__', + '__float__', '__floor__', '__get__', '__getattr__', '__getitem__', + '__index__', '__int__', '__invert__', '__iter__', '__len__', + '__length_hint__', '__missing__', '__neg__', '__next__', + '__objclass__', '__pos__', '__rdivmod__', '__reversed__', + '__round__', '__set__', '__setitem__', '__trunc__' + ] + augment = ( + 'add', 'and', 'floordiv', 'lshift', 'matmul', 'mod', 'mul', 'pow', + 'rshift', 'sub', 'truediv', 'xor' + ) + not_defined.extend(map("__{}__".format, augment)) + not_defined.extend(map("__r{}__".format, augment)) + not_defined.extend(map("__i{}__".format, augment)) + for name in not_defined: + with self.subTest(name): + self.assertFalse(hasattr(object, name)) + self.assertFalse(hasattr(o, name)) + self.assertFalse(hasattr(Custom, name)) + self.assertFalse(hasattr(c, name)) + + # __call__() is defined on the metaclass but not the class + self.assertFalse(hasattr(o, "__call__")) + self.assertFalse(hasattr(c, "__call__")) + + @unittest.skip("TODO: RUSTPYTHON; segmentation fault") + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def testSFBug532646(self): # Test for SF bug 532646 @@ -522,8 +574,7 @@ class A: else: self.fail("Failed to raise RecursionError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testForExceptionsRaisedInInstanceGetattr2(self): # Tests for exceptions raised in instance_getattr2(). @@ -563,7 +614,6 @@ def assertNotOrderable(self, a, b): with self.assertRaises(TypeError): a >= b - @unittest.skip("TODO: RUSTPYTHON; unstable result") def testHashComparisonOfMethods(self): # Test comparison and hash of methods class A: @@ -631,8 +681,6 @@ class A: with self.assertRaises(TypeError): type.__setattr__(A, b'x', None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testTypeAttributeAccessErrorMessages(self): class A: pass @@ -643,14 +691,21 @@ class A: with self.assertRaisesRegex(AttributeError, error_msg): del A.x - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testObjectAttributeAccessErrorMessages(self): class A: pass class B: y = 0 __slots__ = ('z',) + class C: + __slots__ = ("y",) + + def __setattr__(self, name, value) -> None: + if name == "z": + super().__setattr__("y", 1) + else: + super().__setattr__(name, value) error_msg = "'A' object has no attribute 'x'" with self.assertRaisesRegex(AttributeError, error_msg): @@ -663,8 +718,16 @@ class B: B().x with self.assertRaisesRegex(AttributeError, error_msg): del B().x - with self.assertRaisesRegex(AttributeError, error_msg): + with self.assertRaisesRegex( + AttributeError, + "'B' object has no attribute 'x' and no __dict__ for setting new attributes" + ): B().x = 0 + with self.assertRaisesRegex( + AttributeError, + "'C' object has no attribute 'x'" + ): + C().x = 0 error_msg = "'B' object attribute 'y' is read-only" with self.assertRaisesRegex(AttributeError, error_msg): @@ -678,8 +741,6 @@ class B: with self.assertRaisesRegex(AttributeError, error_msg): del B().z - # TODO: RUSTPYTHON - @unittest.expectedFailure def testConstructorErrorMessages(self): # bpo-31506: Improves the error message logic for object_new & object_init @@ -752,6 +813,251 @@ class A(0, 1, 2, 3, 4, 5, 6, 7, **d): pass class A(0, *range(1, 8), **d, foo='bar'): pass self.assertEqual(A, (tuple(range(8)), {'foo': 'bar'})) + def testClassCallRecursionLimit(self): + class C: + def __init__(self): + self.c = C() + + with self.assertRaises(RecursionError): + C() + + def add_one_level(): + #Each call to C() consumes 2 levels, so offset by 1. + C() + + with self.assertRaises(RecursionError): + add_one_level() + + def testMetaclassCallOptimization(self): + calls = 0 + + class TypeMetaclass(type): + def __call__(cls, *args, **kwargs): + nonlocal calls + calls += 1 + return type.__call__(cls, *args, **kwargs) + + class Type(metaclass=TypeMetaclass): + def __init__(self, obj): + self._obj = obj + + for i in range(100): + Type(i) + self.assertEqual(calls, 100) + + def test_specialization_class_call_doesnt_crash(self): + # gh-123185 + + class Foo: + def __init__(self, arg): + pass + + for _ in range(8): + try: + Foo() + except: + pass + + +# from _testinternalcapi import has_inline_values # XXX: RUSTPYTHON + +Py_TPFLAGS_INLINE_VALUES = (1 << 2) +Py_TPFLAGS_MANAGED_DICT = (1 << 4) + +class NoManagedDict: + __slots__ = ('a',) + + +class Plain: + pass + + +class WithAttrs: + + def __init__(self): + self.a = 1 + self.b = 2 + self.c = 3 + self.d = 4 + + +class VarSizedSubclass(tuple): + pass + + +class TestInlineValues(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name 'has_inline_values' is not defined. + def test_no_flags_for_slots_class(self): + flags = NoManagedDict.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, 0) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(NoManagedDict())) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 4 + def test_both_flags_for_regular_class(self): + for cls in (Plain, WithAttrs): + with self.subTest(cls=cls.__name__): + flags = cls.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, Py_TPFLAGS_INLINE_VALUES) + self.assertTrue(has_inline_values(cls())) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 4 + def test_managed_dict_only_for_varsized_subclass(self): + flags = VarSizedSubclass.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(VarSizedSubclass())) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_has_inline_values(self): + c = Plain() + self.assertTrue(has_inline_values(c)) + del c.__dict__ + self.assertFalse(has_inline_values(c)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_instances(self): + self.assertTrue(has_inline_values(Plain())) + self.assertTrue(has_inline_values(WithAttrs())) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_inspect_dict(self): + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__ + self.assertTrue(has_inline_values(c)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_update_dict(self): + d = { "e": 5, "f": 6 } + for cls in (Plain, WithAttrs): + c = cls() + c.__dict__.update(d) + self.assertTrue(has_inline_values(c)) + + @staticmethod + def set_100(obj): + for i in range(100): + setattr(obj, f"a{i}", i) + + def check_100(self, obj): + for i in range(100): + self.assertEqual(getattr(obj, f"a{i}"), i) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_many_attributes(self): + class C: pass + c = C() + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + c = C() + self.assertTrue(has_inline_values(c)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_many_attributes_with_dict(self): + class C: pass + c = C() + d = c.__dict__ + self.assertTrue(has_inline_values(c)) + self.set_100(c) + self.assertFalse(has_inline_values(c)) + self.check_100(c) + + def test_bug_117750(self): + "Aborted on 3.13a6" + class C: + def __init__(self): + self.__dict__.clear() + + obj = C() + self.assertEqual(obj.__dict__, {}) + obj.foo = None # Aborted here + self.assertEqual(obj.__dict__, {"foo":None}) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_store_attr_deleted_dict(self): + class Foo: + pass + + f = Foo() + del f.__dict__ + f.a = 3 + self.assertEqual(f.a, 3) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_rematerialize_object_dict(self): + # gh-121860: rematerializing an object's managed dictionary after it + # had been deleted caused a crash. + class Foo: pass + f = Foo() + f.__dict__["attr"] = 1 + del f.__dict__ + + # Using a str subclass is a way to trigger the re-materialization + class StrSubclass(str): pass + self.assertFalse(hasattr(f, StrSubclass("attr"))) + + # Changing the __class__ also triggers the re-materialization + class Bar: pass + f.__class__ = Bar + self.assertIsInstance(f, Bar) + self.assertEqual(f.__dict__, {}) + + @unittest.skip("TODO: RUSTPYTHON; unexpectedly long runtime") + def test_store_attr_type_cache(self): + """Verifies that the type cache doesn't provide a value which is + inconsistent from the dict.""" + class X: + def __del__(inner_self): + v = C.a + self.assertEqual(v, C.__dict__['a']) + + class C: + a = X() + + # prime the cache + C.a + C.a + + # destructor shouldn't be able to see inconsistent state + C.a = X() + C.a = X() + + @cpython_only + def test_detach_materialized_dict_no_memory(self): + # Skip test if _testcapi is not available: + import_helper.import_module('_testcapi') + + code = """if 1: + import test.support + import _testcapi + + class A: + def __init__(self): + self.a = 1 + self.b = 2 + a = A() + d = a.__dict__ + with test.support.catch_unraisable_exception() as ex: + _testcapi.set_nomemory(0, 1) + del a + assert ex.unraisable.exc_type is MemoryError + try: + d["a"] + except KeyError: + pass + else: + assert False, "KeyError not raised" + """ + rc, out, err = script_helper.assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertFalse(out, msg=out.decode('utf-8')) + self.assertFalse(err, msg=err.decode('utf-8')) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 51dd2ecf5fc..a96a5780b31 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -1,4 +1,5 @@ from test.support import requires_IEEE_754, cpython_only, import_helper +from test.support.testcase import ComplexesAreIdenticalMixin from test.test_math import parse_testfile, test_file import test.test_math as test_math import unittest @@ -49,7 +50,7 @@ (INF, NAN) ]] -class CMathTests(unittest.TestCase): +class CMathTests(ComplexesAreIdenticalMixin, unittest.TestCase): # list of all functions in cmath test_functions = [getattr(cmath, fname) for fname in [ 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atanh', @@ -65,39 +66,6 @@ def setUp(self): def tearDown(self): self.test_values.close() - def assertFloatIdentical(self, x, y): - """Fail unless floats x and y are identical, in the sense that: - (1) both x and y are nans, or - (2) both x and y are infinities, with the same sign, or - (3) both x and y are zeros, with the same sign, or - (4) x and y are both finite and nonzero, and x == y - - """ - msg = 'floats {!r} and {!r} are not identical' - - if math.isnan(x) or math.isnan(y): - if math.isnan(x) and math.isnan(y): - return - elif x == y: - if x != 0.0: - return - # both zero; check that signs match - elif math.copysign(1.0, x) == math.copysign(1.0, y): - return - else: - msg += ': zeros have different signs' - self.fail(msg.format(x, y)) - - def assertComplexIdentical(self, x, y): - """Fail unless complex numbers x and y have equal values and signs. - - In particular, if x and y both have real (or imaginary) part - zero, but the zeros have different signs, this test will fail. - - """ - self.assertFloatIdentical(x.real, y.real) - self.assertFloatIdentical(x.imag, y.imag) - def rAssertAlmostEqual(self, a, b, rel_err = 2e-15, abs_err = 5e-323, msg=None): """Fail if the two floating-point numbers are not almost equal. @@ -299,12 +267,6 @@ def test_cmath_matches_math(self): for v in values: z = complex_fn(v) self.rAssertAlmostEqual(float_fn(v), z.real) - # TODO: RUSTPYTHON - # This line currently fails for acos and asin. - # cmath.asin/acos(0.2) should produce a real number, - # but imaginary part is 1.1102230246251565e-16 for both. - if fn in {"asin", "acos"}: - continue self.assertEqual(0., z.imag) # test two-argument version of log with various bases @@ -314,8 +276,6 @@ def test_cmath_matches_math(self): self.rAssertAlmostEqual(math.log(v, base), z.real) self.assertEqual(0., z.imag) - # TODO: RUSTPYTHON - @unittest.expectedFailure @requires_IEEE_754 def test_specific_values(self): # Some tests need to be skipped on ancient OS X versions. @@ -563,25 +523,21 @@ def test_isinf(self): @requires_IEEE_754 def testTanhSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.tanh(z), z) + self.assertComplexesAreIdentical(cmath.tanh(z), z) # The algorithm used for atan and atanh makes use of the system # log1p function; If that system function doesn't respect the sign # of zero, then atan and atanh will also have difficulties with # the sign of complex zeros. - # TODO: RUSTPYTHON - @unittest.expectedFailure @requires_IEEE_754 def testAtanSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.atan(z), z) + self.assertComplexesAreIdentical(cmath.atan(z), z) - # TODO: RUSTPYTHON - @unittest.expectedFailure @requires_IEEE_754 def testAtanhSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.atanh(z), z) + self.assertComplexesAreIdentical(cmath.atanh(z), z) class IsCloseTests(test_math.IsCloseTests): @@ -624,8 +580,6 @@ def test_complex_near_zero(self): self.assertIsClose(0.001-0.001j, 0.001+0.001j, abs_tol=2e-03) self.assertIsNotClose(0.001-0.001j, 0.001+0.001j, abs_tol=1e-03) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_complex_special(self): self.assertIsNotClose(INF, INF*1j) self.assertIsNotClose(INF*1j, INF) diff --git a/Lib/test/test_cmd.py b/Lib/test/test_cmd.py index 319801c71f7..dbfec42fc21 100644 --- a/Lib/test/test_cmd.py +++ b/Lib/test/test_cmd.py @@ -9,7 +9,16 @@ import doctest import unittest import io +import textwrap from test import support +from test.support.import_helper import ensure_lazy_imports, import_module +from test.support.pty_helper import run_pty + +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("cmd", {"inspect", "string"}) + class samplecmdclass(cmd.Cmd): """ @@ -244,23 +253,79 @@ def test_input_reset_at_EOF(self): "(Cmd) *** Unknown syntax: EOF\n")) +class CmdPrintExceptionClass(cmd.Cmd): + """ + GH-80731 + cmd.Cmd should print the correct exception in default() + >>> mycmd = CmdPrintExceptionClass() + >>> try: + ... raise ValueError("test") + ... except ValueError: + ... mycmd.onecmd("not important") + (, ValueError('test')) + """ + + def default(self, line): + print(sys.exc_info()[:2]) + + +@support.requires_subprocess() +class CmdTestReadline(unittest.TestCase): + def setUpClass(): + # Ensure that the readline module is loaded + # If this fails, the test is skipped because SkipTest will be raised + readline = import_module('readline') + + def test_basic_completion(self): + script = textwrap.dedent(""" + import cmd + class simplecmd(cmd.Cmd): + def do_tab_completion_test(self, args): + print('tab completion success') + return True + + simplecmd().cmdloop() + """) + + # 't' and complete 'ab_completion_test' to 'tab_completion_test' + input = b"t\t\n" + + output = run_pty(script, input) + + self.assertIn(b'ab_completion_test', output) + self.assertIn(b'tab completion success', output) + + def test_bang_completion_without_do_shell(self): + script = textwrap.dedent(""" + import cmd + class simplecmd(cmd.Cmd): + def completedefault(self, text, line, begidx, endidx): + return ["hello"] + + def default(self, line): + if line.replace(" ", "") == "!hello": + print('tab completion success') + else: + print('tab completion failure') + return True + + simplecmd().cmdloop() + """) + + # '! h' or '!h' and complete 'ello' to 'hello' + for input in [b"! h\t\n", b"!h\t\n"]: + with self.subTest(input=input): + output = run_pty(script, input) + self.assertIn(b'hello', output) + self.assertIn(b'tab completion success', output) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) return tests -def test_coverage(coverdir): - trace = support.import_module('trace') - tracer=trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], - trace=0, count=1) - tracer.run('import importlib; importlib.reload(cmd); test_main()') - r=tracer.results() - print("Writing coverage results...") - r.write_results(show_missing=True, summary=True, coverdir=coverdir) if __name__ == "__main__": - if "-c" in sys.argv: - test_coverage('/tmp/cmd.cover') - elif "-i" in sys.argv: + if "-i" in sys.argv: samplecmdclass().cmdloop() else: unittest.main() diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 02f060ba2c9..eb455ebaed7 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -38,8 +38,6 @@ def verify_valid_flag(self, cmd_line): self.assertNotIn(b'Traceback', err) return out - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -82,8 +80,6 @@ def test_optimize(self): def test_site_flag(self): self.verify_valid_flag('-S') - # NOTE: RUSTPYTHON version never starts with Python - @unittest.expectedFailure def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': @@ -177,8 +173,6 @@ def test_run_module(self): # All good if module is located and run successfully assert_python_ok('-m', 'timeit', '-n', '1') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_run_module_bug1764407(self): # -m and -i need to play well together # Runs the timeit module and checks the __main__ @@ -265,7 +259,8 @@ def test_undecodable_code(self): if not stdout.startswith(pattern): raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'unexpected invalid UTF-8 code point'") + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf(sys.platform == 'win32', 'Windows has a native unicode API') def test_invalid_utf8_arg(self): @@ -278,13 +273,11 @@ def test_invalid_utf8_arg(self): code = 'import sys, os; s=os.fsencode(sys.argv[1]); print(ascii(s))' # TODO: RUSTPYTHON - @unittest.expectedFailure def run_default(arg): cmd = [sys.executable, '-c', code, arg] return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) # TODO: RUSTPYTHON - @unittest.expectedFailure def run_c_locale(arg): cmd = [sys.executable, '-c', code, arg] env = dict(os.environ) @@ -293,7 +286,6 @@ def run_c_locale(arg): text=True, env=env) # TODO: RUSTPYTHON - @unittest.expectedFailure def run_utf8_mode(arg): cmd = [sys.executable, '-X', 'utf8', '-c', code, arg] return subprocess.run(cmd, stdout=subprocess.PIPE, text=True) @@ -338,8 +330,6 @@ def test_osx_android_utf8(self): self.assertEqual(stdout, expected) self.assertEqual(p.returncode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_non_interactive_output_buffering(self): code = textwrap.dedent(""" import sys @@ -355,8 +345,6 @@ def test_non_interactive_output_buffering(self): 'False False False\n' 'False False True\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unbuffered_output(self): # Test expected operation of the '-u' switch for stream in ('stdout', 'stderr'): @@ -411,7 +399,8 @@ def test_empty_PYTHONPATH_issue16309(self): path = ":".join(sys.path) path = path.encode("ascii", "backslashreplace") sys.stdout.buffer.write(path)""" - rc1, out1, err1 = assert_python_ok('-c', code, PYTHONPATH="") + # TODO: RUSTPYTHON we must unset RUSTPYTHONPATH as well + rc1, out1, err1 = assert_python_ok('-c', code, PYTHONPATH="", RUSTPYTHONPATH="") rc2, out2, err2 = assert_python_ok('-c', code, __isolated=False) # regarding to Posix specification, outputs should be equal # for empty and unset PYTHONPATH @@ -449,8 +438,6 @@ def check_input(self, code, expected): stdout, stderr = proc.communicate() self.assertEqual(stdout.rstrip(), expected) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform.startswith('win'), "TODO: RUSTPYTHON windows has \n troubles") def test_stdin_readline(self): # Issue #11272: check that sys.stdin.readline() replaces '\r\n' by '\n' # on Windows (sys.stdin is opened in binary mode) @@ -458,16 +445,12 @@ def test_stdin_readline(self): "import sys; print(repr(sys.stdin.readline()))", b"'abc\\n'") - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform.startswith('win'), "TODO: RUSTPYTHON windows has \n troubles") def test_builtin_input(self): # Issue #11272: check that input() strips newlines ('\n' or '\r\n') self.check_input( "print(repr(input()))", b"'abc'") - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform.startswith('win'), "TODO: RUSTPYTHON windows has \n troubles") def test_output_newline(self): # Issue 13119 Newline for print() should be \r\n on Windows. code = """if 1: @@ -564,8 +547,6 @@ def test_no_stderr(self): def test_no_std_streams(self): self._test_no_stdio(['stdin', 'stdout', 'stderr']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_hash_randomization(self): # Verify that -R enables hash randomization: self.verify_valid_flag('-R') @@ -634,8 +615,6 @@ def test_unknown_options(self): self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1) self.assertEqual(b'', out) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') def test_isolatedmode(self): @@ -664,8 +643,6 @@ def test_isolatedmode(self): cwd=tmpdir) self.assertEqual(out.strip(), b"ok") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_sys_flags_set(self): # Issue 31845: a startup refactoring broke reading flags from env vars for value, expected in (("", 0), ("1", 1), ("text", 1), ("2", 2)): @@ -736,8 +713,6 @@ def run_xdev(self, *args, check_exitcode=True, xdev=True): self.assertEqual(proc.returncode, 0, proc) return proc.stdout.rstrip() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_xdev(self): # sys.flags.dev_mode code = "import sys; print(sys.flags.dev_mode)" @@ -930,8 +905,6 @@ def test_parsing_error(self): self.assertTrue(proc.stderr.startswith(err_msg), proc.stderr) self.assertNotEqual(proc.returncode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_int_max_str_digits(self): code = "import sys; print(sys.flags.int_max_str_digits, sys.get_int_max_str_digits())" @@ -986,8 +959,6 @@ def test_ignore_PYTHONPATH(self): self.run_ignoring_vars("'{}' not in sys.path".format(path), PYTHONPATH=path) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ignore_PYTHONHASHSEED(self): self.run_ignoring_vars("sys.flags.hash_randomization == 1", PYTHONHASHSEED="0") diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index e40069d7801..f977b97bcd3 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -225,8 +225,6 @@ def test_repl_stderr_flush(self): def test_repl_stderr_flush_separate_stderr(self): self.check_repl_stderr_flush(True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_script(self): with os_helper.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') @@ -249,8 +247,6 @@ def test_script_abspath(self): script_dir, None, importlib.machinery.SourceFileLoader) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_script_compiled(self): with os_helper.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') @@ -413,8 +409,6 @@ def test_issue8202(self): script_name, script_name, script_dir, 'test_pkg', importlib.machinery.SourceFileLoader) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue8202_dash_c_file_ignored(self): # Make sure a "-c" file in the current directory # does not alter the value of sys.path[0] @@ -554,8 +548,6 @@ def test_dash_m_main_traceback(self): self.assertIn(b'Exception in __main__ module', err) self.assertIn(b'Traceback', err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pep_409_verbiage(self): # Make sure PEP 409 syntax properly suppresses # the context of an exception @@ -574,6 +566,7 @@ def test_pep_409_verbiage(self): self.assertTrue(text[1].startswith(' File ')) self.assertTrue(text[3].startswith('NameError')) + @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_non_ascii(self): # Mac OS X denies the creation of a file with an invalid UTF-8 name. # Windows allows creating a name with an arbitrary bytes name, but @@ -616,8 +609,6 @@ def test_issue20500_exit_with_exception_value(self): text = stderr.decode('ascii') self.assertEqual(text.rstrip(), "some text") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntaxerror_unindented_caret_position(self): script = "1 + 1 = 2\n" with os_helper.temp_dir() as script_dir: @@ -627,8 +618,6 @@ def test_syntaxerror_unindented_caret_position(self): # Confirm that the caret is located under the '=' sign self.assertIn("\n ^^^^^\n", text) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntaxerror_indented_caret_position(self): script = textwrap.dedent("""\ if True: @@ -714,8 +703,6 @@ def test_syntaxerror_null_bytes_in_multiline_string(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 2661cbaa1f6..f2ef233a59a 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -150,7 +150,7 @@ gc_collect) from test.support.script_helper import assert_python_ok from test.support import threading_helper -from opcode import opmap +from opcode import opmap, opname COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -222,8 +222,6 @@ class List(list): obj = List([1, 2, 3]) self.assertEqual(obj[0], "Foreign getitem: 1") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_constructor(self): def func(): pass co = func.__code__ @@ -249,16 +247,12 @@ def func(): pass co.co_freevars, co.co_cellvars) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_qualname(self): self.assertEqual( CodeTest.test_qualname.__code__.co_qualname, CodeTest.test_qualname.__qualname__ ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_replace(self): def func(): x = 1 @@ -299,8 +293,6 @@ def func2(): self.assertEqual(new_code.co_varnames, code2.co_varnames) self.assertEqual(new_code.co_nlocals, code2.co_nlocals) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nlocals_mismatch(self): def func(): x = 1 @@ -349,14 +341,38 @@ def func(arg): newcode = code.replace(co_name="func") # Should not raise SystemError self.assertEqual(code, newcode) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty_linetable(self): def func(): pass new_code = code = func.__code__.replace(co_linetable=b'') self.assertEqual(list(new_code.co_lines()), []) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_co_lnotab_is_deprecated(self): # TODO: remove in 3.14 + def func(): + pass + + with self.assertWarns(DeprecationWarning): + func.__code__.co_lnotab + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_invalid_bytecode(self): + def foo(): + pass + + # assert that opcode 229 is invalid + self.assertEqual(opname[229], '<229>') + + # change first opcode to 0xeb (=229) + foo.__code__ = foo.__code__.replace( + co_code=b'\xe5' + foo.__code__.co_code[1:]) + + msg = "unknown opcode 229" + with self.assertRaisesRegex(SystemError, msg): + foo() + # TODO: RUSTPYTHON @unittest.expectedFailure # @requires_debug_ranges() @@ -409,8 +425,6 @@ def test_co_positions_artificial_instructions(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_endline_and_columntable_none_when_no_debug_ranges(self): # Make sure that if `-X no_debug_ranges` is used, there is # minimal debug info @@ -426,8 +440,6 @@ def f(): """) assert_python_ok('-X', 'no_debug_ranges', '-c', code) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_endline_and_columntable_none_when_no_debug_ranges_env(self): # Same as above but using the environment variable opt out. code = textwrap.dedent(""" @@ -444,8 +456,6 @@ def f(): # co_positions behavior when info is missing. - # TODO: RUSTPYTHON - @unittest.expectedFailure # @requires_debug_ranges() def test_co_positions_empty_linetable(self): def func(): @@ -456,8 +466,6 @@ def func(): self.assertIsNone(line) self.assertEqual(end_line, new_code.co_firstlineno + 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_code_equality(self): def f(): try: @@ -479,6 +487,32 @@ def f(): self.assertNotEqual(code_b, code_d) self.assertNotEqual(code_c, code_d) + def test_code_hash_uses_firstlineno(self): + c1 = (lambda: 1).__code__ + c2 = (lambda: 1).__code__ + self.assertNotEqual(c1, c2) + self.assertNotEqual(hash(c1), hash(c2)) + c3 = c1.replace(co_firstlineno=17) + self.assertNotEqual(c1, c3) + self.assertNotEqual(hash(c1), hash(c3)) + + def test_code_hash_uses_order(self): + # Swapping posonlyargcount and kwonlyargcount should change the hash. + c = (lambda x, y, *, z=1, w=1: 1).__code__ + self.assertEqual(c.co_argcount, 2) + self.assertEqual(c.co_posonlyargcount, 0) + self.assertEqual(c.co_kwonlyargcount, 2) + swapped = c.replace(co_posonlyargcount=2, co_kwonlyargcount=0) + self.assertNotEqual(c, swapped) + self.assertNotEqual(hash(c), hash(swapped)) + + def test_code_hash_uses_bytecode(self): + c = (lambda x, y: x + y).__code__ + d = (lambda x, y: x * y).__code__ + c1 = c.replace(co_code=d.co_code) + self.assertNotEqual(c, c1) + self.assertNotEqual(hash(c), hash(c1)) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) @@ -683,8 +717,6 @@ def check_positions(self, func): self.assertEqual(l1, l2) self.assertEqual(len(pos1), len(pos2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positions(self): self.check_positions(parse_location_table) self.check_positions(misshappen) @@ -692,14 +724,13 @@ def test_positions(self): def check_lines(self, func): co = func.__code__ - lines1 = list(dedup(l for (_, _, l) in co.co_lines())) + lines1 = [line for _, _, line in co.co_lines()] + self.assertEqual(lines1, list(dedup(lines1))) lines2 = list(lines_from_postions(positions_from_location_table(co))) for l1, l2 in zip(lines1, lines2): self.assertEqual(l1, l2) self.assertEqual(len(lines1), len(lines2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lines(self): self.check_lines(parse_location_table) self.check_lines(misshappen) @@ -714,6 +745,7 @@ def f(): pass PY_CODE_LOCATION_INFO_NO_COLUMNS = 13 f.__code__ = f.__code__.replace( + co_stacksize=1, co_firstlineno=42, co_code=bytes( [ @@ -742,15 +774,15 @@ def f(): py = ctypes.pythonapi freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) - RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex + RequestCodeExtraIndex = py.PyUnstable_Eval_RequestCodeExtraIndex RequestCodeExtraIndex.argtypes = (freefunc,) RequestCodeExtraIndex.restype = ctypes.c_ssize_t - SetExtra = py._PyCode_SetExtra + SetExtra = py.PyUnstable_Code_SetExtra SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) SetExtra.restype = ctypes.c_int - GetExtra = py._PyCode_GetExtra + GetExtra = py.PyUnstable_Code_GetExtra GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.POINTER(ctypes.c_voidp)) GetExtra.restype = ctypes.c_int diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 5ac17ef16ea..39d85d46274 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -1,20 +1,17 @@ "Test InteractiveConsole and InteractiveInterpreter from code module" import sys +import traceback import unittest from textwrap import dedent from contextlib import ExitStack from unittest import mock +from test.support import force_not_colorized_test_class from test.support import import_helper - code = import_helper.import_module('code') -class TestInteractiveConsole(unittest.TestCase): - - def setUp(self): - self.console = code.InteractiveConsole() - self.mock_sys() +class MockSys: def mock_sys(self): "Mock system environment for InteractiveConsole" @@ -32,21 +29,58 @@ def mock_sys(self): del self.sysmod.ps1 del self.sysmod.ps2 + +@force_not_colorized_test_class +class TestInteractiveConsole(unittest.TestCase, MockSys): + maxDiff = None + + def setUp(self): + self.console = code.InteractiveConsole() + self.mock_sys() + def test_ps1(self): - self.infunc.side_effect = EOFError('Finished') + self.infunc.side_effect = [ + "import code", + "code.sys.ps1", + EOFError('Finished') + ] self.console.interact() - self.assertEqual(self.sysmod.ps1, '>>> ') + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('>>> ', output) + self.assertNotHasAttr(self.sysmod, 'ps1') + + self.infunc.side_effect = [ + "import code", + "code.sys.ps1", + EOFError('Finished') + ] self.sysmod.ps1 = 'custom1> ' self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('custom1> ', output) self.assertEqual(self.sysmod.ps1, 'custom1> ') def test_ps2(self): - self.infunc.side_effect = EOFError('Finished') + self.infunc.side_effect = [ + "import code", + "code.sys.ps2", + EOFError('Finished') + ] self.console.interact() - self.assertEqual(self.sysmod.ps2, '... ') - self.sysmod.ps1 = 'custom2> ' + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('... ', output) + self.assertNotHasAttr(self.sysmod, 'ps2') + + self.infunc.side_effect = [ + "import code", + "code.sys.ps2", + EOFError('Finished') + ] + self.sysmod.ps2 = 'custom2> ' self.console.interact() - self.assertEqual(self.sysmod.ps1, 'custom2> ') + output = ''.join(''.join(call[1]) for call in self.stdout.method_calls) + self.assertIn('custom2> ', output) + self.assertEqual(self.sysmod.ps2, 'custom2> ') def test_console_stderr(self): self.infunc.side_effect = ["'antioch'", "", EOFError('Finished')] @@ -58,21 +92,154 @@ def test_console_stderr(self): raise AssertionError("no console stdout") def test_syntax_error(self): - self.infunc.side_effect = ["undefined", EOFError('Finished')] + self.infunc.side_effect = ["def f():", + " x = ?", + "", + EOFError('Finished')] self.console.interact() - for call in self.stderr.method_calls: - if 'NameError' in ''.join(call[1]): - break - else: - raise AssertionError("No syntax error from console") + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 2', + ' x = ?', + ' ^', + 'SyntaxError: invalid syntax']) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - 'IndentationError: unexpected indentation'] + def test_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[:output.index('\nnow exiting')] + self.assertEqual(output.splitlines()[1:], [ + ' File "", line 1', + ' 1', + 'IndentationError: unexpected indent']) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 1\n\nnow exiti [truncated]... doesn't start with 'UnicodeEncodeError: ' + def test_unicode_error(self): + self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[output.index('\n') + 1:] + self.assertStartsWith(output, 'UnicodeEncodeError: ') + self.assertIs(self.sysmod.last_type, UnicodeEncodeError) + self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) def test_sysexcepthook(self): - self.infunc.side_effect = ["raise ValueError('')", + self.infunc.side_effect = ["def f():", + " raise ValueError('BOOM!')", + "", + "f()", + EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + 'Traceback (most recent call last):\n', + ' File "", line 1, in \n', + ' File "", line 2, in f\n', + 'ValueError: BOOM!\n']) + + def test_sysexcepthook_syntax_error(self): + self.infunc.side_effect = ["def f():", + " x = ?", + "", EOFError('Finished')] hook = mock.Mock() self.sysmod.excepthook = hook self.console.interact() - self.assertTrue(hook.called) + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, SyntaxError) + self.assertIs(type(self.sysmod.last_value), SyntaxError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 2\n', + ' x = ?\n', + ' ^\n', + 'SyntaxError: invalid syntax\n']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + 'IndentationError: unexpected indent\n'] + def test_sysexcepthook_indentation_error(self): + self.infunc.side_effect = [" 1", EOFError('Finished')] + hook = mock.Mock() + self.sysmod.excepthook = hook + self.console.interact() + hook.assert_called() + hook.assert_called_with(self.sysmod.last_type, + self.sysmod.last_value, + self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_type, IndentationError) + self.assertIs(type(self.sysmod.last_value), IndentationError) + self.assertIsNone(self.sysmod.last_traceback) + self.assertIsNone(self.sysmod.last_value.__traceback__) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + self.assertEqual(traceback.format_exception(self.sysmod.last_exc), [ + ' File "", line 1\n', + ' 1\n', + 'IndentationError: unexpected indent\n']) + + def test_sysexcepthook_crashing_doesnt_close_repl(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + self.sysmod.excepthook = 1 + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("'int' object is not callable"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_BaseException(self): + self.infunc.side_effect = ["1/0", "a = 123", "print(a)", EOFError('Finished')] + s = "not so fast" + def raise_base(*args, **kwargs): + raise BaseException(s) + self.sysmod.excepthook = raise_base + self.console.interact() + self.assertEqual(['write', ('123', ), {}], self.stdout.method_calls[0]) + error = "".join(call.args[0] for call in self.stderr.method_calls if call[0] == 'write') + self.assertIn("Error in sys.excepthook:", error) + self.assertEqual(error.count("not so fast"), 1) + self.assertIn("Original exception was:", error) + self.assertIn("division by zero", error) + + def test_sysexcepthook_raising_SystemExit_gets_through(self): + self.infunc.side_effect = ["1/0"] + def raise_base(*args, **kwargs): + raise SystemExit + self.sysmod.excepthook = raise_base + with self.assertRaises(SystemExit): + self.console.interact() def test_banner(self): # with banner @@ -115,8 +282,7 @@ def test_exit_msg(self): expected = message + '\n' self.assertEqual(err_msg, ['write', (expected,), {}]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_cause_tb(self): self.infunc.side_effect = ["raise ValueError('') from AttributeError", EOFError('Finished')] @@ -132,9 +298,12 @@ def test_cause_tb(self): ValueError """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, ValueError) + self.assertIs(type(self.sysmod.last_value), ValueError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_context_tb(self): self.infunc.side_effect = ["try: ham\nexcept: eggs\n", EOFError('Finished')] @@ -152,6 +321,28 @@ def test_context_tb(self): NameError: name 'eggs' is not defined """) self.assertIn(expected, output) + self.assertIs(self.sysmod.last_type, NameError) + self.assertIs(type(self.sysmod.last_value), NameError) + self.assertIs(self.sysmod.last_traceback, self.sysmod.last_value.__traceback__) + self.assertIsNotNone(self.sysmod.last_traceback) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + + +class TestInteractiveConsoleLocalExit(unittest.TestCase, MockSys): + + def setUp(self): + self.console = code.InteractiveConsole(local_exit=True) + self.mock_sys() + + @unittest.skipIf(sys.flags.no_site, "exit() isn't defined unless there's a site module") + def test_exit(self): + # default exit message + self.infunc.side_effect = ["exit()"] + self.console.interact(banner='') + self.assertEqual(len(self.stderr.method_calls), 2) + err_msg = self.stderr.method_calls[1] + expected = 'now exiting InteractiveConsole...\n' + self.assertEqual(err_msg, ['write', (expected,), {}]) if __name__ == "__main__": diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 293b75a8666..763146c94fc 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -203,8 +203,6 @@ def relaxedutf8(exc): self.assertRaises(UnicodeDecodeError, sin.decode, "utf-8", "test.relaxedutf8") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_charmapencode(self): # For charmap encodings the replacement string will be # mapped through the encoding again. This means, that @@ -283,8 +281,6 @@ def handler2(exc): b"g[<252><223>]" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_longstrings(self): # test long strings to check for memory overflow problems errors = [ "strict", "ignore", "replace", "xmlcharrefreplace", @@ -329,8 +325,6 @@ def check_exceptionobjectargs(self, exctype, args, msg): exc = exctype(*args) self.assertEqual(str(exc), msg) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicodeencodeerror(self): self.check_exceptionobjectargs( UnicodeEncodeError, @@ -363,8 +357,6 @@ def test_unicodeencodeerror(self): "'ascii' codec can't encode character '\\U00010000' in position 0: ouch" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicodedecodeerror(self): self.check_exceptionobjectargs( UnicodeDecodeError, @@ -377,8 +369,6 @@ def test_unicodedecodeerror(self): "'ascii' codec can't decode bytes in position 1-2: ouch" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicodetranslateerror(self): self.check_exceptionobjectargs( UnicodeTranslateError, @@ -467,8 +457,6 @@ def test_badandgoodignoreexceptions(self): ("", 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodreplaceexceptions(self): # "replace" complains about a non-exception passed in self.assertRaises( @@ -509,8 +497,6 @@ def test_badandgoodreplaceexceptions(self): ("\ufffd", 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodxmlcharrefreplaceexceptions(self): # "xmlcharrefreplace" complains about a non-exception passed in self.assertRaises( @@ -548,8 +534,6 @@ def test_badandgoodxmlcharrefreplaceexceptions(self): ("".join("&#%d;" % c for c in cs), 1 + len(s)) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodbackslashreplaceexceptions(self): # "backslashreplace" complains about a non-exception passed in self.assertRaises( @@ -608,8 +592,6 @@ def test_badandgoodbackslashreplaceexceptions(self): (r, 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodnamereplaceexceptions(self): # "namereplace" complains about a non-exception passed in self.assertRaises( @@ -656,8 +638,6 @@ def test_badandgoodnamereplaceexceptions(self): (r, 1 + len(s)) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodsurrogateescapeexceptions(self): surrogateescape_errors = codecs.lookup_error('surrogateescape') # "surrogateescape" complains about a non-exception passed in @@ -702,8 +682,6 @@ def test_badandgoodsurrogateescapeexceptions(self): ("\udc80", 2) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badandgoodsurrogatepassexceptions(self): surrogatepass_errors = codecs.lookup_error('surrogatepass') # "surrogatepass" complains about a non-exception passed in @@ -927,8 +905,6 @@ def handle(exc): self.assertEqual(exc.object, input) self.assertEqual(exc.reason, "surrogates not allowed") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badregistercall(self): # enhance coverage of: # Modules/_codecsmodule.c::register_error() @@ -1017,8 +993,6 @@ def __getitem__(self, key): self.assertRaises(ValueError, codecs.charmap_decode, b"\xff", "strict", D()) self.assertRaises(TypeError, codecs.charmap_decode, b"\xff", "strict", {0xff: sys.maxunicode+1}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encodehelper(self): # enhance coverage of: # Objects/unicodeobject.c::unicode_encode_call_errorhandler() diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py new file mode 100644 index 00000000000..af32f624d81 --- /dev/null +++ b/Lib/test/test_codecencodings_cn.py @@ -0,0 +1,100 @@ +# +# test_codecencodings_cn.py +# Codec encoding tests for PRC encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb2312") +class Test_GB2312(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gb2312' + tstring = multibytecodec_support.load_teststring('gb2312') + codectests = ( + # invalid bytes + (b"abc\x81\x81\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x81\x81\xc1\xc4", "ignore", "abc\u804a"), + (b"\xc1\x64", "strict", None), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gbk") +class Test_GBK(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gbk' + tstring = multibytecodec_support.load_teststring('gbk') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), + (b"\x83\x34\x83\x31", "strict", None), + ("\u30fb", "strict", None), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb18030") +class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'gb18030' + tstring = multibytecodec_support.load_teststring('gb18030') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), + (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd9\ufffd9\u804a"), + ("\u30fb", "strict", b"\x819\xa79"), + (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), + (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), + (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), + # issue29990 + (b"\xff\x30\x81\x30", "strict", None), + (b"\x81\x30\xff\x30", "strict", None), + (b"abc\x81\x39\xff\x39\xc1\xc4", "replace", "abc\ufffd\x39\ufffd\x39\u804a"), + (b"abc\xab\x36\xff\x30def", "replace", 'abc\ufffd\x36\ufffd\x30def'), + (b"abc\xbf\x38\xff\x32\xc1\xc4", "ignore", "abc\x38\x32\u804a"), + ) + has_iso10646 = True + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: hz") +class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'hz' + tstring = multibytecodec_support.load_teststring('hz') + codectests = ( + # test '~\n' (3 lines) + (b'This sentence is in ASCII.\n' + b'The next sentence is in GB.~{<:Ky2;S{#,~}~\n' + b'~{NpJ)l6HK!#~}Bye.\n', + 'strict', + 'This sentence is in ASCII.\n' + 'The next sentence is in GB.' + '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' + 'Bye.\n'), + # test '~\n' (4 lines) + (b'This sentence is in ASCII.\n' + b'The next sentence is in GB.~\n' + b'~{<:Ky2;S{#,NpJ)l6HK!#~}~\n' + b'Bye.\n', + 'strict', + 'This sentence is in ASCII.\n' + 'The next sentence is in GB.' + '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' + 'Bye.\n'), + # invalid bytes + (b'ab~cd', 'replace', 'ab\uFFFDcd'), + (b'ab\xffcd', 'replace', 'ab\uFFFDcd'), + (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), + (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), + (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), + # issue 30003 + ('ab~cd', 'strict', b'ab~~cd'), # escape ~ + (b'~{Dc~~:C~}', 'strict', None), # ~~ only in ASCII mode + (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py new file mode 100644 index 00000000000..b64d19bca91 --- /dev/null +++ b/Lib/test/test_codecencodings_hk.py @@ -0,0 +1,23 @@ +# +# test_codecencodings_hk.py +# Codec encoding tests for HongKong encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5hkscs") +class Test_Big5HKSCS(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'big5hkscs' + tstring = multibytecodec_support.load_teststring('big5hkscs') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py new file mode 100644 index 00000000000..fe97aa6977e --- /dev/null +++ b/Lib/test/test_codecencodings_iso2022.py @@ -0,0 +1,92 @@ +# Codec encoding tests for ISO 2022 encodings. + +from test import multibytecodec_support +import unittest + +COMMON_CODEC_TESTS = ( + # invalid bytes + (b'ab\xFFcd', 'replace', 'ab\uFFFDcd'), + (b'ab\x1Bdef', 'replace', 'ab\x1Bdef'), + (b'ab\x1B$def', 'replace', 'ab\uFFFD'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp") +class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2") +class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'abdef'), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_3") +class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_3' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(O\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(O\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(O\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(O\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(O\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(O\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(O\x2E\x21\x1B(Bdef', 'replace', 'ab\uFFFDdef'), + ('ab\u4FF1def', 'replace', b'ab?def'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩' + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2004") +class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2004' + tstring = multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + (b'\x1B$(Q\x2E\x23\x1B(B', 'strict', '\u3402' ), + (b'\x1B$(Q\x2E\x22\x1B(B', 'strict', '\U0002000B' ), + (b'\x1B$(Q\x24\x77\x1B(B', 'strict', '\u304B\u309A'), + (b'\x1B$(P\x21\x22\x1B(B', 'strict', '\u4E02' ), + (b'\x1B$(P\x7E\x76\x1B(B', 'strict', '\U0002A6B2' ), + ('\u3402', 'strict', b'\x1B$(Q\x2E\x23\x1B(B'), + ('\U0002000B', 'strict', b'\x1B$(Q\x2E\x22\x1B(B'), + ('\u304B\u309A', 'strict', b'\x1B$(Q\x24\x77\x1B(B'), + ('\u4E02', 'strict', b'\x1B$(P\x21\x22\x1B(B'), + ('\U0002A6B2', 'strict', b'\x1B$(P\x7E\x76\x1B(B'), + (b'ab\x1B$(Q\x2E\x21\x1B(Bdef', 'replace', 'ab\u4FF1def'), + ('ab\u4FF1def', 'replace', b'ab\x1B$(Q\x2E\x21\x1B(Bdef'), + ) + xmlcharnametest = ( + '\xAB\u211C\xBB = \u2329\u1234\u232A', + b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩' + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_kr") +class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_kr' + tstring = multibytecodec_support.load_teststring('iso2022_kr') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + ) + + # iso2022_kr.txt cannot be used to test "chunk coding": the escape + # sequence is only written on the first line + @unittest.skip('iso2022_kr.txt cannot be used to test "chunk coding"') + def test_chunkcoding(self): + pass + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py new file mode 100644 index 00000000000..f78ae229f84 --- /dev/null +++ b/Lib/test/test_codecencodings_jp.py @@ -0,0 +1,133 @@ +# +# test_codecencodings_jp.py +# Codec encoding tests for Japanese encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp932") +class Test_CP932(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'cp932' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = ( + # invalid bytes + (b"abc\x81\x00\x81\x00\x82\x84", "strict", None), + (b"abc\xf8", "strict", None), + (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\x00\uff44"), + (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\x00\uff44\ufffd"), + (b"abc\x81\x00\x82\x84", "ignore", "abc\x00\uff44"), + (b"ab\xEBxy", "replace", "ab\uFFFDxy"), + (b"ab\xF0\x39xy", "replace", "ab\uFFFD9xy"), + (b"ab\xEA\xF0xy", "replace", 'ab\ufffd\ue038y'), + # sjis vs cp932 + (b"\\\x7e", "replace", "\\\x7e"), + (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\uff3c\u2225\uff0d"), + ) + +euc_commontests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u7956"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u7956\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), + (b"abc\xc8", "strict", None), + (b"abc\x8f\x83\x83", "replace", "abc\ufffd\ufffd\ufffd"), + (b"\x82\xFCxy", "replace", "\ufffd\ufffdxy"), + (b"\xc1\x64", "strict", None), + (b"\xa1\xc0", "strict", "\uff3c"), + (b"\xa1\xc0\\", "strict", "\uff3c\\"), + (b"\x8eXY", "replace", "\ufffdXY"), +) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jis_2004") +class Test_EUC_JIS_2004(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jis_2004' + tstring = multibytecodec_support.load_teststring('euc_jisx0213') + codectests = euc_commontests + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jisx0213") +class Test_EUC_JISX0213(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jisx0213' + tstring = multibytecodec_support.load_teststring('euc_jisx0213') + codectests = euc_commontests + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jp") +class Test_EUC_JP_COMPAT(multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jp' + tstring = multibytecodec_support.load_teststring('euc_jp') + codectests = euc_commontests + ( + ("\xa5", "strict", b"\x5c"), + ("\u203e", "strict", b"\x7e"), + ) + +shiftjis_commonenctests = ( + (b"abc\x80\x80\x82\x84", "strict", None), + (b"abc\xf8", "strict", None), + (b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"), +) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis") +class Test_SJIS_COMPAT(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jis' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + + (b"\\\x7e", "strict", "\\\x7e"), + (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\uff3c\u2016\u2212"), + (b"abc\x81\x39", "replace", "abc\ufffd9"), + (b"abc\xEA\xFC", "replace", "abc\ufffd\ufffd"), + (b"abc\xFF\x58", "replace", "abc\ufffdX"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis_2004") +class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jis_2004' + tstring = multibytecodec_support.load_teststring('shift_jis') + codectests = shiftjis_commonenctests + ( + (b"\\\x7e", "strict", "\xa5\u203e"), + (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\\\u2016\u2212"), + (b"abc\xEA\xFC", "strict", "abc\u64bf"), + (b"\x81\x39xy", "replace", "\ufffd9xy"), + (b"\xFF\x58xy", "replace", "\ufffdXxy"), + (b"\x80\x80\x82\x84xy", "replace", "\ufffd\ufffd\uff44xy"), + (b"\x80\x80\x82\x84\x88xy", "replace", "\ufffd\ufffd\uff44\u5864y"), + (b"\xFC\xFBxy", "replace", '\ufffd\u95b4y'), + ) + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\x85Gℜ\x85Q = ⟨ሴ⟩" + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jisx0213") +class Test_SJISX0213(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jisx0213' + tstring = multibytecodec_support.load_teststring('shift_jisx0213') + codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + + # sjis vs cp932 + (b"\\\x7e", "replace", "\xa5\u203e"), + (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\x5c\u2016\u2212"), + ) + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\x85Gℜ\x85Q = ⟨ሴ⟩" + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py new file mode 100644 index 00000000000..aee124007ae --- /dev/null +++ b/Lib/test/test_codecencodings_kr.py @@ -0,0 +1,72 @@ +# +# test_codecencodings_kr.py +# Codec encoding tests for ROK encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp949") +class Test_CP949(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'cp949' + tstring = multibytecodec_support.load_teststring('cp949') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\uc894"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_kr") +class Test_EUCKR(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'euc_kr' + tstring = multibytecodec_support.load_teststring('euc_kr') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", 'abc\ufffd\ufffd\uc894'), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), + + # composed make-up sequence errors + (b"\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "strict", "\uc4d4"), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4x", "strict", "\uc4d4x"), + (b"a\xa4\xd4\xa4\xb6\xa4", "replace", 'a\ufffd'), + (b"\xa4\xd4\xa3\xb6\xa4\xd0\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa3\xd0\xa4\xd4", "strict", None), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa3\xd4", "strict", None), + (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", '\ufffd\u6e21\ufffd\u3160\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", '\ufffd\u6e21\ub544\ufffd\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", '\ufffd\u6e21\ub544\u572d\ufffd'), + (b"\xa4\xd4\xff\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "replace", '\ufffd\ufffd\ufffd\uc4d4'), + (b"\xc1\xc4", "strict", "\uc894"), + ) + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: johab") +class Test_JOHAB(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'johab' + tstring = multibytecodec_support.load_teststring('johab') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\ucd27"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\ucd27\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\ucd27"), + (b"\xD8abc", "replace", "\uFFFDabc"), + (b"\xD8\xFFabc", "replace", "\uFFFD\uFFFDabc"), + (b"\x84bxy", "replace", "\uFFFDbxy"), + (b"\x8CBxy", "replace", "\uFFFDBxy"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py new file mode 100644 index 00000000000..ca56b23234e --- /dev/null +++ b/Lib/test/test_codecencodings_tw.py @@ -0,0 +1,23 @@ +# +# test_codecencodings_tw.py +# Codec encoding tests for ROC encodings. +# + +from test import multibytecodec_support +import unittest + +@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5") +class Test_Big5(multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'big5' + tstring = multibytecodec_support.load_teststring('big5') + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py new file mode 100644 index 00000000000..de254c6f767 --- /dev/null +++ b/Lib/test/test_codecmaps_cn.py @@ -0,0 +1,38 @@ +# +# test_codecmaps_cn.py +# Codec mapping tests for PRC encodings +# + +from test import multibytecodec_support +import unittest + +class TestGB2312Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gb2312' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-CN.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb2312 + def test_mapping_file(self): + return super().test_mapping_file() + +class TestGBKMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gbk' + mapfileurl = 'http://www.pythontest.net/unicode/CP936.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gbk + def test_mapping_file(self): + return super().test_mapping_file() + +class TestGB18030Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'gb18030' + mapfileurl = 'http://www.pythontest.net/unicode/gb-18030-2000.xml' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb18030 + def test_mapping_file(self): + return super().test_mapping_file() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py new file mode 100644 index 00000000000..f02cf486f76 --- /dev/null +++ b/Lib/test/test_codecmaps_hk.py @@ -0,0 +1,19 @@ +# +# test_codecmaps_hk.py +# Codec mapping tests for HongKong encodings +# + +from test import multibytecodec_support +import unittest + +class TestBig5HKSCSMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'big5hkscs' + mapfileurl = 'http://www.pythontest.net/unicode/BIG5HKSCS-2004.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5hkscs + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py new file mode 100644 index 00000000000..f2d52f99526 --- /dev/null +++ b/Lib/test/test_codecmaps_jp.py @@ -0,0 +1,84 @@ +# +# test_codecmaps_jp.py +# Codec mapping tests for Japanese encodings +# + +from test import multibytecodec_support +import unittest + +class TestCP932Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp932' + mapfileurl = 'http://www.pythontest.net/unicode/CP932.TXT' + supmaps = [ + (b'\x80', '\u0080'), + (b'\xa0', '\uf8f0'), + (b'\xfd', '\uf8f1'), + (b'\xfe', '\uf8f2'), + (b'\xff', '\uf8f3'), + ] + for i in range(0xa1, 0xe0): + supmaps.append((bytes([i]), chr(i+0xfec0))) + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932 + def test_mapping_file(self): + return super().test_mapping_file() + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932 + def test_mapping_supplemental(self): + return super().test_mapping_supplemental() + + +class TestEUCJPCOMPATMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_jp' + mapfilename = 'EUC-JP.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-JP.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jp + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestSJISCOMPATMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'shift_jis' + mapfilename = 'SHIFTJIS.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/SHIFTJIS.TXT' + pass_enctest = [ + (b'\x81_', '\\'), + ] + pass_dectest = [ + (b'\\', '\xa5'), + (b'~', '\u203e'), + (b'\x81_', '\\'), + ] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jis + def test_mapping_file(self): + return super().test_mapping_file() + +class TestEUCJISX0213Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_jisx0213' + mapfilename = 'EUC-JISX0213.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-JISX0213.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jisx0213 + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestSJISX0213Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'shift_jisx0213' + mapfilename = 'SHIFT_JISX0213.TXT' + mapfileurl = 'http://www.pythontest.net/unicode/SHIFT_JISX0213.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jisx0213 + def test_mapping_file(self): + return super().test_mapping_file() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py new file mode 100644 index 00000000000..b8376d36615 --- /dev/null +++ b/Lib/test/test_codecmaps_kr.py @@ -0,0 +1,49 @@ +# +# test_codecmaps_kr.py +# Codec mapping tests for ROK encodings +# + +from test import multibytecodec_support +import unittest + +class TestCP949Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp949' + mapfileurl = 'http://www.pythontest.net/unicode/CP949.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp949 + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestEUCKRMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'euc_kr' + mapfileurl = 'http://www.pythontest.net/unicode/EUC-KR.TXT' + + # A4D4 HANGUL FILLER indicates the begin of 8-bytes make-up sequence. + pass_enctest = [(b'\xa4\xd4', '\u3164')] + pass_dectest = [(b'\xa4\xd4', '\u3164')] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_kr + def test_mapping_file(self): + return super().test_mapping_file() + + +class TestJOHABMap(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'johab' + mapfileurl = 'http://www.pythontest.net/unicode/JOHAB.TXT' + # KS X 1001 standard assigned 0x5c as WON SIGN. + # But the early 90s is the only era that used johab widely, + # most software implements it as REVERSE SOLIDUS. + # So, we ignore the standard here. + pass_enctest = [(b'\\', '\u20a9')] + pass_dectest = [(b'\\', '\u20a9')] + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: johab + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py new file mode 100644 index 00000000000..4a1359ce7be --- /dev/null +++ b/Lib/test/test_codecmaps_tw.py @@ -0,0 +1,39 @@ +# +# test_codecmaps_tw.py +# Codec mapping tests for ROC encodings +# + +from test import multibytecodec_support +import unittest + +class TestBIG5Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'big5' + mapfileurl = 'http://www.pythontest.net/unicode/BIG5.TXT' + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 + def test_mapping_file(self): + return super().test_mapping_file() + +class TestCP950Map(multibytecodec_support.TestBase_Mapping, + unittest.TestCase): + encoding = 'cp950' + mapfileurl = 'http://www.pythontest.net/unicode/CP950.TXT' + pass_enctest = [ + (b'\xa2\xcc', '\u5341'), + (b'\xa2\xce', '\u5345'), + ] + codectests = ( + (b"\xFFxy", "replace", "\ufffdxy"), + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp950 + def test_errorhandle(self): + return super().test_errorhandle() + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp950 + def test_mapping_file(self): + return super().test_mapping_file() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 3a9c6d2741f..fefd062eacb 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,28 +1,34 @@ import codecs import contextlib +import copy +import importlib import io -import locale +import pickle +import os import sys import unittest import encodings from unittest import mock +import warnings from test import support from test.support import os_helper -from test.support import warnings_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None - + _testlimitedcapi = None try: - import ctypes + import _testinternalcapi except ImportError: - ctypes = None - SIZEOF_WCHAR_T = -1 -else: - SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar) + _testinternalcapi = None + + +def codecs_open_no_warn(*args, **kwargs): + """Call codecs.open(*args, **kwargs) ignoring DeprecationWarning.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return codecs.open(*args, **kwargs) def coding_checker(self, coder): def check(input, expect): @@ -31,13 +37,13 @@ def check(input, expect): # On small versions of Windows like Windows IoT or Windows Nano Server not all codepages are present def is_code_page_present(cp): - from ctypes import POINTER, WINFUNCTYPE, WinDLL + from ctypes import POINTER, WINFUNCTYPE, WinDLL, Structure from ctypes.wintypes import BOOL, BYTE, WCHAR, UINT, DWORD MAX_LEADBYTES = 12 # 5 ranges, 2 bytes ea., 0 term. MAX_DEFAULTCHAR = 2 # single or double byte MAX_PATH = 260 - class CPINFOEXW(ctypes.Structure): + class CPINFOEXW(Structure): _fields_ = [("MaxCharSize", UINT), ("DefaultChar", BYTE*MAX_DEFAULTCHAR), ("LeadByte", BYTE*MAX_LEADBYTES), @@ -384,8 +390,6 @@ def test_bug1098990_b(self): ill_formed_sequence_replace = "\ufffd" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lone_surrogates(self): self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding) self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"), @@ -461,14 +465,6 @@ class UTF32Test(ReadTest, unittest.TestCase): b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m' b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m') - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_only_one_bom(self): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream @@ -484,19 +480,15 @@ def test_only_one_bom(self): f = reader(s) self.assertEqual(f.read(), "spamspam") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(8*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -528,30 +520,22 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_32_decode(b'\x01', 'replace', True)) self.assertEqual(('', 1), codecs.utf_32_decode(b'\x01', 'ignore', True)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decoder_state(self): self.check_state_handling_decode(self.encoding, "spamspam", self.spamle) self.check_state_handling_decode(self.encoding, "spamspam", self.spambe) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -562,48 +546,11 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1098990_a(self): - super().test_bug1098990_a() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1098990_b(self): - super().test_bug1098990_b() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1175396(self): - super().test_bug1175396() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readlinequeue(self): - super().test_readlinequeue() - class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -631,19 +578,13 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x03\x02\x01\x00") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -651,48 +592,11 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1098990_a(self): - super().test_bug1098990_a() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1098990_b(self): - super().test_bug1098990_b() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1175396(self): - super().test_bug1175396() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readlinequeue(self): - super().test_readlinequeue() - class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -720,19 +624,13 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x00\x01\x02\x03") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -740,36 +638,6 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_be_decode(encoded)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1098990_a(self): - super().test_bug1098990_a() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1098990_b(self): - super().test_bug1098990_b() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1175396(self): - super().test_bug1175396() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readlinequeue(self): - super().test_readlinequeue() - class UTF16Test(ReadTest, unittest.TestCase): encoding = "utf-16" @@ -799,14 +667,12 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(b"\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -828,8 +694,6 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_16_decode(b'\x01', 'replace', True)) @@ -846,10 +710,9 @@ def test_decoder_state(self): self.check_state_handling_decode(self.encoding, "spamspam", self.spambe) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug691291(self): - # Files are always opened in binary mode, even if no binary mode was + # If encoding is not None, then + # files are always opened in binary mode, even if no binary mode was # specified. This means that no automatic conversion of '\n' is done # on reading and writing. s1 = 'Hello\r\nworld\r\n' @@ -858,17 +721,27 @@ def test_bug691291(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, 'wb') as fp: fp.write(s) - with warnings_helper.check_warnings(('', DeprecationWarning)): - reader = codecs.open(os_helper.TESTFN, 'U', encoding=self.encoding) - with reader: + with codecs_open_no_warn(os_helper.TESTFN, 'r', + encoding=self.encoding) as reader: self.assertEqual(reader.read(), s1) + def test_invalid_modes(self): + for mode in ('U', 'rU', 'r+U'): + with self.assertRaises(ValueError) as cm: + codecs_open_no_warn(os_helper.TESTFN, mode, encoding=self.encoding) + self.assertIn('invalid mode', str(cm.exception)) + + for mode in ('rt', 'wt', 'at', 'r+t'): + with self.assertRaises(ValueError) as cm: + codecs_open_no_warn(os_helper.TESTFN, mode, encoding=self.encoding) + self.assertIn("can't have text and binary mode at once", + str(cm.exception)) + + class UTF16LETest(ReadTest, unittest.TestCase): encoding = "utf-16-le" ill_formed_sequence = b"\x80\xdc" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -888,8 +761,6 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -915,8 +786,6 @@ class UTF16BETest(ReadTest, unittest.TestCase): encoding = "utf-16-be" ill_formed_sequence = b"\xdc\x80" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -936,8 +805,6 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -992,8 +859,6 @@ def test_decoder_state(self): self.check_state_handling_decode(self.encoding, u, u.encode(self.encoding)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode_error(self): for data, error_handler, expected in ( (b'[\x80\xff]', 'ignore', '[]'), @@ -1006,8 +871,6 @@ def test_decode_error(self): self.assertEqual(data.decode(self.encoding, error_handler), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for @@ -1020,8 +883,6 @@ def test_lone_surrogates(self): exc = cm.exception self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogatepass_handler(self): self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"), self.BOM + b"abc\xed\xa0\x80def") @@ -1062,13 +923,6 @@ def test_incremental_errors(self): class UTF7Test(ReadTest, unittest.TestCase): encoding = "utf-7" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ascii(self): # Set D (directly encoded characters) set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' @@ -1095,8 +949,6 @@ def test_ascii(self): b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU' b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_partial(self): self.check_partial( 'a+-b\x00c\x80d\u0100e\U00010000f', @@ -1136,8 +988,6 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): tests = [ (b'\xffb', '\ufffdb'), @@ -1168,8 +1018,6 @@ def test_errors(self): raw, 'strict', True) self.assertEqual(raw.decode('utf-7', 'replace'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nonbmp(self): self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-') self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-') @@ -1185,8 +1033,6 @@ def test_nonbmp(self): self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding), '\u20ac\u20ac\U000104A0') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lone_surrogates(self): tests = [ (b'a+2AE-b', 'a\ud801b'), @@ -1207,16 +1053,6 @@ def test_lone_surrogates(self): with self.subTest(raw=raw): self.assertEqual(raw.decode('utf-7', 'replace'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bug1175396(self): - super().test_bug1175396() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() - class UTF16ExTest(unittest.TestCase): @@ -1340,8 +1176,6 @@ def test_raw(self): if b != b'\\': self.assertEqual(decode(b + b'0'), (b + b'0', 2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_escape(self): decode = codecs.escape_decode check = coding_checker(self, decode) @@ -1362,25 +1196,42 @@ def test_escape(self): check(br"[\418]", b"[!8]") check(br"[\101]", b"[A]") check(br"[\1010]", b"[A0]") - check(br"[\501]", b"[A]") check(br"[\x41]", b"[A]") check(br"[\x410]", b"[A0]") + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_warnings(self): + decode = codecs.escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, b"\\" + b) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), b"\\" + b.upper()) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\8" is an invalid escape sequence'): check(br"\8", b"\\8") with self.assertWarns(DeprecationWarning): check(br"\9", b"\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", b"\\\xfa") + for i in range(0o400, 0o1000): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%o" is an invalid octal escape sequence' % i): + check(rb'\%o' % i, bytes([i & 0o377])) + + with self.assertWarnsRegex(DeprecationWarning, + r'"\\z" is an invalid escape sequence'): + self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\501" is an invalid octal escape sequence'): + self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): decode = codecs.escape_decode self.assertRaises(ValueError, decode, br"\x") @@ -1523,13 +1374,29 @@ def test_decode(self): def test_decode_invalid(self): testcases = [ - (b"xn--w&", "strict", UnicodeError()), + (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), + (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), + (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), + (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), + (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), + (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), + (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), + (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), (b"xn--w&", "ignore", "xn-"), ] for puny, errors, expected in testcases: with self.subTest(puny=puny, errors=errors): if isinstance(expected, Exception): - self.assertRaises(UnicodeError, puny.decode, "punycode", errors) + with self.assertRaises(UnicodeDecodeError) as cm: + puny.decode("punycode", errors) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, puny) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) else: self.assertEqual(puny.decode("punycode", errors), expected) @@ -1689,8 +1556,6 @@ def test_decode_invalid(self): class NameprepTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nameprep(self): from encodings.idna import nameprep for pos, (orig, prepped) in enumerate(nameprep_tests): @@ -1701,7 +1566,7 @@ def test_nameprep(self): orig = str(orig, "utf-8", "surrogatepass") if prepped is None: # Input contains prohibited characters - self.assertRaises(UnicodeError, nameprep, orig) + self.assertRaises(UnicodeEncodeError, nameprep, orig) else: prepped = str(prepped, "utf-8", "surrogatepass") try: @@ -1711,11 +1576,46 @@ def test_nameprep(self): class IDNACodecTest(unittest.TestCase): + + invalid_decode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFFpython.org", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFFhon.org", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF.org", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"python.\xFForg", 7, 8, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"python.o\xFFrg", 8, 9, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"python.org\xFF", 10, 11, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.org", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.org", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.org", 13, 14, "")), + ] + invalid_encode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"foo.{'\xff'*60}", 4, 64, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "あさ.\u034f", 3, 4, "")), + ] + def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") self.assertEqual(str(b"python.org.", "idna"), "python.org.") self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org") self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"XN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"xN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"Xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"bugs.xn--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + + def test_builtin_decode_invalid(self): + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + case.decode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start, msg=f'reason: {exc.reason}') + self.assertEqual(exc.end, expected.end) def test_builtin_encode(self): self.assertEqual("python.org".encode("idna"), b"python.org") @@ -1723,6 +1623,23 @@ def test_builtin_encode(self): self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org") self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") + def test_builtin_encode_invalid(self): + for case, expected in self.invalid_encode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + case.encode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + def test_builtin_decode_length_limit(self): + with self.assertRaisesRegex(UnicodeDecodeError, "way too long"): + (b"xn--016c"+b"a"*1100).decode("idna") + with self.assertRaisesRegex(UnicodeDecodeError, "too long"): + (b"xn--016c"+b"a"*70).decode("idna") + def test_stream(self): r = codecs.getreader("idna")(io.BytesIO(b"abc")) r.read(3) @@ -1758,6 +1675,39 @@ def test_incremental_decode(self): self.assertEqual(decoder.decode(b"rg."), "org.") self.assertEqual(decoder.decode(b"", True), "") + def test_incremental_decode_invalid(self): + iterdecode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFF", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"o\xFF", 1, 2, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"org\xFF", 3, 4, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.", 13, 14, "")), + ] + for case, expected in iterdecode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + list(codecs.iterdecode((bytes([c]) for c in case), "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + decoder = codecs.getincrementaldecoder("idna")() + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + decoder.decode(case) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_incremental_encode(self): self.assertEqual( b"".join(codecs.iterencode("python.org", "idna")), @@ -1786,6 +1736,23 @@ def test_incremental_encode(self): self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.") self.assertEqual(encoder.encode("", True), b"") + def test_incremental_encode_invalid(self): + iterencode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"{'\xff'*60}", 0, 60, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "\u034f", 0, 1, "")), + ] + for case, expected in iterencode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + list(codecs.iterencode(case, "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + # codecs.getincrementalencoder.encode() does not throw an error + def test_errors(self): """Only supports "strict" error handler""" "python.org".encode("idna", "strict") @@ -1812,7 +1779,6 @@ def test_decode(self): self.assertEqual(codecs.decode(b'[\xff]', 'ascii', errors='ignore'), '[]') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_encode(self): self.assertEqual(codecs.encode('\xe4\xf6\xfc', 'latin-1'), b'\xe4\xf6\xfc') @@ -1831,7 +1797,6 @@ def test_register(self): self.assertRaises(TypeError, codecs.register) self.assertRaises(TypeError, codecs.register, 42) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module '_winapi' has no attribute 'GetACP'") def test_unregister(self): name = "nonexistent_codec_name" search_function = mock.Mock() @@ -1844,42 +1809,31 @@ def test_unregister(self): self.assertRaises(LookupError, codecs.lookup, name) search_function.assert_not_called() - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_lookup(self): self.assertRaises(TypeError, codecs.lookup) self.assertRaises(LookupError, codecs.lookup, "__spam__") self.assertRaises(LookupError, codecs.lookup, " ") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getencoder(self): self.assertRaises(TypeError, codecs.getencoder) self.assertRaises(LookupError, codecs.getencoder, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getdecoder(self): self.assertRaises(TypeError, codecs.getdecoder) self.assertRaises(LookupError, codecs.getdecoder, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getreader(self): self.assertRaises(TypeError, codecs.getreader) self.assertRaises(LookupError, codecs.getreader, "__spam__") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getwriter(self): self.assertRaises(TypeError, codecs.getwriter) self.assertRaises(LookupError, codecs.getwriter, "__spam__") + @support.run_with_locale('LC_CTYPE', 'tr_TR') def test_lookup_issue1813(self): # Issue #1813: under Turkish locales, lookup of some codecs failed # because 'I' is lowercased as "ı" (dotless i) - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') c = codecs.lookup('ASCII') self.assertEqual(c.name, 'ascii') @@ -1909,9 +1863,9 @@ def test_all(self): def test_open(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) for mode in ('w', 'r', 'r+', 'w+', 'a', 'a+'): - with self.subTest(mode), \ - codecs.open(os_helper.TESTFN, mode, 'ascii') as file: - self.assertIsInstance(file, codecs.StreamReaderWriter) + with self.subTest(mode), self.assertWarns(DeprecationWarning): + with codecs.open(os_helper.TESTFN, mode, 'ascii') as file: + self.assertIsInstance(file, codecs.StreamReaderWriter) def test_undefined(self): self.assertRaises(UnicodeError, codecs.encode, 'abc', 'undefined') @@ -1924,15 +1878,85 @@ def test_undefined(self): self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_file_closes_if_lookup_error_raised(self): mock_open = mock.mock_open() with mock.patch('builtins.open', mock_open) as file: with self.assertRaises(LookupError): - codecs.open(os_helper.TESTFN, 'wt', 'invalid-encoding') + codecs_open_no_warn(os_helper.TESTFN, 'wt', 'invalid-encoding') file().close.assert_called() + def test_copy(self): + orig = codecs.lookup('utf-8') + dup = copy.copy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertTrue(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + # Test a CodecInfo with _is_text_encoding equal to false. + orig = codecs.lookup("base64") + dup = copy.copy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertFalse(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + def test_deepcopy(self): + orig = codecs.lookup('utf-8') + dup = copy.deepcopy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertTrue(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + # Test a CodecInfo with _is_text_encoding equal to false. + orig = codecs.lookup("base64") + dup = copy.deepcopy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertFalse(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + def test_pickle(self): + codec_info = codecs.lookup('utf-8') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled_codec_info = pickle.dumps(codec_info) + unpickled_codec_info = pickle.loads(pickled_codec_info) + self.assertIsNot(codec_info, unpickled_codec_info) + self.assertEqual(codec_info, unpickled_codec_info) + self.assertEqual(codec_info.name, unpickled_codec_info.name) + self.assertEqual( + codec_info.incrementalencoder, + unpickled_codec_info.incrementalencoder + ) + self.assertTrue(unpickled_codec_info._is_text_encoding) + + # Test a CodecInfo with _is_text_encoding equal to false. + codec_info = codecs.lookup('base64') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled_codec_info = pickle.dumps(codec_info) + unpickled_codec_info = pickle.loads(pickled_codec_info) + self.assertIsNot(codec_info, unpickled_codec_info) + self.assertEqual(codec_info, unpickled_codec_info) + self.assertEqual(codec_info.name, unpickled_codec_info.name) + self.assertEqual( + codec_info.incrementalencoder, + unpickled_codec_info.incrementalencoder + ) + self.assertFalse(unpickled_codec_info._is_text_encoding) + + class StreamReaderTest(unittest.TestCase): def setUp(self): @@ -1943,6 +1967,61 @@ def test_readlines(self): f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + def test_copy(self): + f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80')) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80')) + with self.assertRaisesRegex(TypeError, 'StreamReader'): + pickle.dumps(f, proto) + + +class StreamWriterTest(unittest.TestCase): + + def setUp(self): + self.writer = codecs.getwriter('utf-8') + + def test_copy(self): + f = self.writer(Queue(b'')) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = self.writer(Queue(b'')) + with self.assertRaisesRegex(TypeError, 'StreamWriter'): + pickle.dumps(f, proto) + + +class StreamReaderWriterTest(unittest.TestCase): + + def setUp(self): + self.reader = codecs.getreader('latin1') + self.writer = codecs.getwriter('utf-8') + + def test_copy(self): + f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + copy.copy(f) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + copy.deepcopy(f) + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer) + with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): + pickle.dumps(f, proto) + class EncodedFileTest(unittest.TestCase): @@ -2076,8 +2155,7 @@ def test_basic(self): class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_basics(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: @@ -2086,7 +2164,10 @@ def test_basics(self): name += "_codec" elif encoding == "latin_1": name = "latin_1" - self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-")) + # Skip the mbcs alias on Windows + if name != "mbcs": + self.assertEqual(encoding.replace("_", "-"), + name.replace("_", "-")) (b, size) = codecs.getencoder(encoding)(s) self.assertEqual(size, len(s), "encoding=%r" % encoding) @@ -2156,13 +2237,14 @@ def test_basics(self): "encoding=%r" % encoding) @support.cpython_only + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_basics_capi(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: if encoding not in broken_unicode_with_stateful: # check incremental decoder/encoder (fetched via the C API) try: - cencoder = _testcapi.codec_incrementalencoder(encoding) + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) except LookupError: # no IncrementalEncoder pass else: @@ -2171,7 +2253,7 @@ def test_basics_capi(self): for c in s: encodedresult += cencoder.encode(c) encodedresult += cencoder.encode("", True) - cdecoder = _testcapi.codec_incrementaldecoder(encoding) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) decodedresult = "" for c in encodedresult: decodedresult += cdecoder.decode(bytes([c])) @@ -2182,19 +2264,18 @@ def test_basics_capi(self): if encoding not in ("idna", "mbcs"): # check incremental decoder/encoder with errors argument try: - cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore") + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") except LookupError: # no IncrementalEncoder pass else: encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore") + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") decodedresult = "".join(cdecoder.decode(bytes([c])) for c in encodedresult) self.assertEqual(decodedresult, s, "encoding=%r" % encoding) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_seek(self): # all codecs should be able to encode these s = "%s\n%s\n" % (100*"abc123", 100*"def456") @@ -2210,8 +2291,7 @@ def test_seek(self): data = reader.read() self.assertEqual(s, data) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_bad_decode_args(self): for encoding in all_unicode_encodings: decoder = codecs.getdecoder(encoding) @@ -2219,8 +2299,7 @@ def test_bad_decode_args(self): if encoding not in ("idna", "punycode"): self.assertRaises(TypeError, decoder, 42) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_bad_encode_args(self): for encoding in all_unicode_encodings: encoder = codecs.getencoder(encoding) @@ -2232,8 +2311,7 @@ def test_encoding_map_type_initialized(self): table_type = type(cp1140.encoding_table) self.assertEqual(table_type, table_type) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5 def test_decoder_state(self): # Check that getstate() and setstate() handle the state properly u = "abc123" @@ -2244,8 +2322,6 @@ def test_decoder_state(self): class CharmapTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode_with_string_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", "abc"), @@ -2301,8 +2377,6 @@ def test_decode_with_string_map(self): ("", len(allbytes)) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode_with_int2str_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", @@ -2419,8 +2493,6 @@ def test_decode_with_int2str_map(self): b"\x00\x01\x02", "strict", {0: "A", 1: 'Bb', 2: 999999999} ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode_with_int2int_map(self): a = ord('a') b = ord('b') @@ -2513,8 +2585,7 @@ def test_streamreaderwriter(self): class TypesTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'? def test_decode_unicode(self): # Most decoders don't accept unicode input decoders = [ @@ -2560,16 +2631,6 @@ class UnicodeEscapeTest(ReadTest, unittest.TestCase): test_lone_surrogates = None - # TODO: RUSTPYTHON, TypeError: Expected type 'str', not 'bytes' - @unittest.expectedFailure - def test_incremental_surrogatepass(self): # TODO: RUSTPYTHON, remove when this passes - super().test_incremental_surrogatepass() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - def test_empty(self): self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0)) @@ -2625,20 +2686,40 @@ def test_escape_decode(self): check(br"[\x410]", "[A0]") check(br"\u20ac", "\u20ac") check(br"\U0001d120", "\U0001d120") + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_decode_warnings(self): + decode = codecs.unicode_escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtuvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, "\\" + chr(i)) if b.upper() not in b'UN': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), "\\" + chr(i-32)) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\8" is an invalid escape sequence'): check(br"\8", "\\8") with self.assertWarns(DeprecationWarning): check(br"\9", "\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", "\\\xfa") + for i in range(0o400, 0o1000): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%o" is an invalid octal escape sequence' % i): + check(rb'\%o' % i, chr(i)) + + with self.assertWarnsRegex(DeprecationWarning, + r'"\\z" is an invalid escape sequence'): + self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\501" is an invalid octal escape sequence'): + self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6)) def test_decode_errors(self): decode = codecs.unicode_escape_decode @@ -2656,8 +2737,6 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - # TODO: RUSTPYTHON, UnicodeDecodeError: ('unicodeescape', b'\\', 0, 1, '\\ at end of string') - @unittest.expectedFailure def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -2697,21 +2776,15 @@ def test_partial(self): ] ) + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range + def test_incremental_surrogatepass(self): + return super().test_incremental_surrogatepass() + class RawUnicodeEscapeTest(ReadTest, unittest.TestCase): encoding = "raw-unicode-escape" test_lone_surrogates = None - # TODO: RUSTPYTHON, AssertionError: '\\' != '' - @unittest.expectedFailure - def test_incremental_surrogatepass(self): # TODO: RUSTPYTHON, remove when this passes - super().test_incremental_surrogatepass() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - def test_empty(self): self.assertEqual(codecs.raw_unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.raw_unicode_escape_decode(b""), ("", 0)) @@ -2760,8 +2833,6 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - # TODO: RUSTPYTHON, AssertionError: '\x00\t\n\r\\' != '\x00\t\n\r' - @unittest.expectedFailure def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -2814,8 +2885,6 @@ def test_escape_encode(self): class SurrogateEscapeTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_utf8(self): # Bad byte self.assertEqual(b"foo\x80bar".decode("utf-8", "surrogateescape"), @@ -2828,8 +2897,6 @@ def test_utf8(self): self.assertEqual("\udced\udcb0\udc80".encode("utf-8", "surrogateescape"), b"\xed\xb0\x80") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ascii(self): # bad byte self.assertEqual(b"foo\x80bar".decode("ascii", "surrogateescape"), @@ -2837,8 +2904,6 @@ def test_ascii(self): self.assertEqual("foo\udc80bar".encode("ascii", "surrogateescape"), b"foo\x80bar") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_charmap(self): # bad byte: \xa5 is unmapped in iso-8859-3 self.assertEqual(b"foo\xa5bar".decode("iso-8859-3", "surrogateescape"), @@ -2846,8 +2911,6 @@ def test_charmap(self): self.assertEqual("foo\udca5bar".encode("iso-8859-3", "surrogateescape"), b"foo\xa5bar") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_latin1(self): # Issue6373 self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", "surrogateescape"), @@ -2855,8 +2918,6 @@ def test_latin1(self): class BomTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_seek0(self): data = "1234567890" tests = ("utf-16", @@ -2868,7 +2929,7 @@ def test_seek0(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) for encoding in tests: # Check if the BOM is written only once - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data) f.write(data) f.seek(0) @@ -2877,7 +2938,7 @@ def test_seek0(self): self.assertEqual(f.read(), data * 2) # Check that the BOM is written after a seek(0) - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data[0]) self.assertNotEqual(f.tell(), 0) f.seek(0) @@ -2886,7 +2947,7 @@ def test_seek0(self): self.assertEqual(f.read(), data) # (StreamWriter) Check that the BOM is written after a seek(0) - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.writer.write(data[0]) self.assertNotEqual(f.writer.tell(), 0) f.writer.seek(0) @@ -2896,7 +2957,7 @@ def test_seek0(self): # Check that the BOM is not written after a seek() at a position # different than the start - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.write(data) f.seek(f.tell()) f.write(data) @@ -2905,7 +2966,7 @@ def test_seek0(self): # (StreamWriter) Check that the BOM is not written after a seek() # at a position different than the start - with codecs.open(os_helper.TESTFN, 'w+', encoding=encoding) as f: + with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f: f.writer.write(data) f.writer.seek(f.writer.tell()) f.writer.write(data) @@ -2936,7 +2997,7 @@ def test_seek0(self): bytes_transform_encodings.append("zlib_codec") transform_aliases["zlib_codec"] = ["zip", "zlib"] try: - import bz2 + import bz2 # noqa: F401 except ImportError: pass else: @@ -2946,8 +3007,6 @@ def test_seek0(self): class TransformCodecTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basics(self): binput = bytes(range(256)) for encoding in bytes_transform_encodings: @@ -2959,8 +3018,6 @@ def test_basics(self): self.assertEqual(size, len(o)) self.assertEqual(i, binput) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_read(self): for encoding in bytes_transform_encodings: with self.subTest(encoding=encoding): @@ -2969,8 +3026,6 @@ def test_read(self): sout = reader.read() self.assertEqual(sout, b"\x80") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_readline(self): for encoding in bytes_transform_encodings: with self.subTest(encoding=encoding): @@ -2979,8 +3034,6 @@ def test_readline(self): sout = reader.readline() self.assertEqual(sout, b"\x80") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_buffer_api_usage(self): # We check all the transform codecs accept memoryview input # for encoding and decoding @@ -3043,29 +3096,21 @@ def test_binary_to_text_denylists_text_transforms(self): bad_input.decode("rot_13") self.assertIsNone(failure.exception.__cause__) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(zlib, "Requires zlib support") - def test_custom_zlib_error_is_wrapped(self): + def test_custom_zlib_error_is_noted(self): # Check zlib codec gives a good error for malformed input - msg = "^decoding with 'zlib_codec' codec failed" - with self.assertRaisesRegex(Exception, msg) as failure: + msg = "decoding with 'zlib_codec' codec failed" + with self.assertRaises(zlib.error) as failure: codecs.decode(b"hello", "zlib_codec") - self.assertIsInstance(failure.exception.__cause__, - type(failure.exception)) + self.assertEqual(msg, failure.exception.__notes__[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_custom_hex_error_is_wrapped(self): + def test_custom_hex_error_is_noted(self): # Check hex codec gives a good error for malformed input - msg = "^decoding with 'hex_codec' codec failed" - with self.assertRaisesRegex(Exception, msg) as failure: + import binascii + msg = "decoding with 'hex_codec' codec failed" + with self.assertRaises(binascii.Error) as failure: codecs.decode(b"hello", "hex_codec") - self.assertIsInstance(failure.exception.__cause__, - type(failure.exception)) - - # Unfortunately, the bz2 module throws OSError, which the codec - # machinery currently can't wrap :( + self.assertEqual(msg, failure.exception.__notes__[0]) # Ensure codec aliases from http://bugs.python.org/issue7475 work def test_aliases(self): @@ -3076,6 +3121,13 @@ def test_aliases(self): info = codecs.lookup(alias) self.assertEqual(info.name, expected_name) + def test_alias_modules_exist(self): + encodings_dir = os.path.dirname(encodings.__file__) + for value in encodings.aliases.aliases.values(): + codec_mod = f"encodings.{value}" + self.assertIsNotNone(importlib.util.find_spec(codec_mod), + f"Codec module not found: {codec_mod}") + def test_quopri_stateless(self): # Should encode with quotetabs=True encoded = codecs.encode(b"space tab\teol \n", "quopri-codec") @@ -3089,11 +3141,8 @@ def test_uu_invalid(self): self.assertRaises(ValueError, codecs.decode, b"", "uu-codec") -# The codec system tries to wrap exceptions in order to ensure the error -# mentions the operation being performed and the codec involved. We -# currently *only* want this to happen for relatively stateless -# exceptions, where the only significant information they contain is their -# type and a single str argument. +# The codec system tries to add notes to exceptions in order to ensure +# the error mentions the operation being performed and the codec involved. # Use a local codec registry to avoid appearing to leak objects when # registering multiple search functions @@ -3103,10 +3152,10 @@ def _get_test_codec(codec_name): return _TEST_CODECS.get(codec_name) -class ExceptionChainingTest(unittest.TestCase): +class ExceptionNotesTest(unittest.TestCase): def setUp(self): - self.codec_name = 'exception_chaining_test' + self.codec_name = 'exception_notes_test' codecs.register(_get_test_codec) self.addCleanup(codecs.unregister, _get_test_codec) @@ -3130,105 +3179,77 @@ def set_codec(self, encode, decode): _TEST_CODECS[self.codec_name] = codec_info @contextlib.contextmanager - def assertWrapped(self, operation, exc_type, msg): - full_msg = r"{} with {!r} codec failed \({}: {}\)".format( - operation, self.codec_name, exc_type.__name__, msg) - with self.assertRaisesRegex(exc_type, full_msg) as caught: + def assertNoted(self, operation, exc_type, msg): + full_msg = r"{} with {!r} codec failed".format( + operation, self.codec_name) + with self.assertRaises(exc_type) as caught: yield caught - self.assertIsInstance(caught.exception.__cause__, exc_type) - self.assertIsNotNone(caught.exception.__cause__.__traceback__) + self.assertIn(full_msg, caught.exception.__notes__[0]) + caught.exception.__notes__.clear() def raise_obj(self, *args, **kwds): # Helper to dynamically change the object raised by a test codec raise self.obj_to_raise - def check_wrapped(self, obj_to_raise, msg, exc_type=RuntimeError): + def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): self.obj_to_raise = obj_to_raise self.set_codec(self.raise_obj, self.raise_obj) - with self.assertWrapped("encoding", exc_type, msg): + with self.assertNoted("encoding", exc_type, msg): "str_input".encode(self.codec_name) - with self.assertWrapped("encoding", exc_type, msg): + with self.assertNoted("encoding", exc_type, msg): codecs.encode("str_input", self.codec_name) - with self.assertWrapped("decoding", exc_type, msg): + with self.assertNoted("decoding", exc_type, msg): b"bytes input".decode(self.codec_name) - with self.assertWrapped("decoding", exc_type, msg): + with self.assertNoted("decoding", exc_type, msg): codecs.decode(b"bytes input", self.codec_name) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_raise_by_type(self): - self.check_wrapped(RuntimeError, "") + self.check_note(RuntimeError, "") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_raise_by_value(self): - msg = "This should be wrapped" - self.check_wrapped(RuntimeError(msg), msg) + msg = "This should be noted" + self.check_note(RuntimeError(msg), msg) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_raise_grandchild_subclass_exact_size(self): - msg = "This should be wrapped" + msg = "This should be noted" class MyRuntimeError(RuntimeError): __slots__ = () - self.check_wrapped(MyRuntimeError(msg), msg, MyRuntimeError) + self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_raise_subclass_with_weakref_support(self): - msg = "This should be wrapped" + msg = "This should be noted" class MyRuntimeError(RuntimeError): pass - self.check_wrapped(MyRuntimeError(msg), msg, MyRuntimeError) + self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - def check_not_wrapped(self, obj_to_raise, msg): - def raise_obj(*args, **kwds): - raise obj_to_raise - self.set_codec(raise_obj, raise_obj) - with self.assertRaisesRegex(RuntimeError, msg): - "str input".encode(self.codec_name) - with self.assertRaisesRegex(RuntimeError, msg): - codecs.encode("str input", self.codec_name) - with self.assertRaisesRegex(RuntimeError, msg): - b"bytes input".decode(self.codec_name) - with self.assertRaisesRegex(RuntimeError, msg): - codecs.decode(b"bytes input", self.codec_name) - - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_init_override_is_not_wrapped(self): + def test_init_override(self): class CustomInit(RuntimeError): def __init__(self): pass - self.check_not_wrapped(CustomInit, "") + self.check_note(CustomInit, "") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_new_override_is_not_wrapped(self): + def test_new_override(self): class CustomNew(RuntimeError): def __new__(cls): return super().__new__(cls) - self.check_not_wrapped(CustomNew, "") + self.check_note(CustomNew, "") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_instance_attribute_is_not_wrapped(self): - msg = "This should NOT be wrapped" + def test_instance_attribute(self): + msg = "This should be noted" exc = RuntimeError(msg) exc.attr = 1 - self.check_not_wrapped(exc, "^{}$".format(msg)) + self.check_note(exc, "^{}$".format(msg)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_non_str_arg_is_not_wrapped(self): - self.check_not_wrapped(RuntimeError(1), "1") + def test_non_str_arg(self): + self.check_note(RuntimeError(1), "1") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_multiple_args_is_not_wrapped(self): + def test_multiple_args(self): msg_re = r"^\('a', 'b', 'c'\)$" - self.check_not_wrapped(RuntimeError('a', 'b', 'c'), msg_re) + self.check_note(RuntimeError('a', 'b', 'c'), msg_re) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") # http://bugs.python.org/issue19609 - def test_codec_lookup_failure_not_wrapped(self): + def test_codec_lookup_failure(self): msg = "^unknown encoding: {}$".format(self.codec_name) - # The initial codec lookup should not be wrapped with self.assertRaisesRegex(LookupError, msg): "str input".encode(self.codec_name) with self.assertRaisesRegex(LookupError, msg): @@ -3238,8 +3259,6 @@ def test_codec_lookup_failure_not_wrapped(self): with self.assertRaisesRegex(LookupError, msg): codecs.decode(b"bytes input", self.codec_name) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unflagged_non_text_codec_handling(self): # The stdlib non-text codecs are now marked so they're # pre-emptively skipped by the text model related methods @@ -3275,26 +3294,26 @@ def decode_to_bytes(*args, **kwds): class CodePageTest(unittest.TestCase): CP_UTF8 = 65001 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_invalid_code_page(self): self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a') self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a') self.assertRaises(OSError, codecs.code_page_encode, 123, 'a') self.assertRaises(OSError, codecs.code_page_decode, 123, b'a') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_code_page_name(self): self.assertRaisesRegex(UnicodeEncodeError, 'cp932', codecs.code_page_encode, 932, '\xff') self.assertRaisesRegex(UnicodeDecodeError, 'cp932', codecs.code_page_decode, 932, b'\x81\x00', 'strict', True) - self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8', + self.assertRaisesRegex(UnicodeDecodeError, 'cp65001', codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): - for raw, errors, expected in tests: + for raw, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: decoded = codecs.code_page_decode(cp, raw, errors, True) @@ -3311,8 +3330,21 @@ def check_decode(self, cp, tests): self.assertRaises(UnicodeDecodeError, codecs.code_page_decode, cp, raw, errors, True) + if altexpected is not None: + decoded = raw.decode(f'cp{cp}', errors) + self.assertEqual(decoded, altexpected, + '%a.decode("cp%s", %r)=%a != %a' + % (raw, cp, errors, decoded, altexpected)) + else: + self.assertRaises(UnicodeDecodeError, + raw.decode, f'cp{cp}', errors) + def check_encode(self, cp, tests): - for text, errors, expected in tests: + for text, errors, expected, *rest in tests: + if rest: + altexpected, = rest + else: + altexpected = expected if expected is not None: try: encoded = codecs.code_page_encode(cp, text, errors) @@ -3323,20 +3355,27 @@ def check_encode(self, cp, tests): '%a.encode("cp%s", %r)=%a != %a' % (text, cp, errors, encoded[0], expected)) self.assertEqual(encoded[1], len(text)) + + encoded = text.encode(f'cp{cp}', errors) + self.assertEqual(encoded, altexpected, + '%a.encode("cp%s", %r)=%a != %a' + % (text, cp, errors, encoded, altexpected)) else: self.assertRaises(UnicodeEncodeError, codecs.code_page_encode, cp, text, errors) + self.assertRaises(UnicodeEncodeError, + text.encode, f'cp{cp}', errors) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp932(self): self.check_encode(932, ( ('abc', 'strict', b'abc'), ('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'), + ('\uf8f3', 'strict', b'\xff'), # test error handlers ('\xff', 'strict', None), ('[\xff]', 'ignore', b'[]'), - ('[\xff]', 'replace', b'[y]'), + ('[\xff]', 'replace', b'[y]', b'[?]'), ('[\u20ac]', 'replace', b'[?]'), ('[\xff]', 'backslashreplace', b'[\\xff]'), ('[\xff]', 'namereplace', @@ -3350,20 +3389,18 @@ def test_cp932(self): (b'abc', 'strict', 'abc'), (b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'), # invalid bytes - (b'[\xff]', 'strict', None), - (b'[\xff]', 'ignore', '[]'), - (b'[\xff]', 'replace', '[\ufffd]'), - (b'[\xff]', 'backslashreplace', '[\\xff]'), - (b'[\xff]', 'surrogateescape', '[\udcff]'), - (b'[\xff]', 'surrogatepass', None), + (b'[\xff]', 'strict', None, '[\uf8f3]'), + (b'[\xff]', 'ignore', '[]', '[\uf8f3]'), + (b'[\xff]', 'replace', '[\ufffd]', '[\uf8f3]'), + (b'[\xff]', 'backslashreplace', '[\\xff]', '[\uf8f3]'), + (b'[\xff]', 'surrogateescape', '[\udcff]', '[\uf8f3]'), + (b'[\xff]', 'surrogatepass', None, '[\uf8f3]'), (b'\x81\x00abc', 'strict', None), (b'\x81\x00abc', 'ignore', '\x00abc'), (b'\x81\x00abc', 'replace', '\ufffd\x00abc'), (b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cp1252(self): self.check_encode(1252, ( ('abc', 'strict', b'abc'), @@ -3372,7 +3409,7 @@ def test_cp1252(self): # test error handlers ('\u0141', 'strict', None), ('\u0141', 'ignore', b''), - ('\u0141', 'replace', b'L'), + ('\u0141', 'replace', b'L', b'?'), ('\udc98', 'surrogateescape', b'\x98'), ('\udc98', 'surrogatepass', None), )) @@ -3382,8 +3419,60 @@ def test_cp1252(self): (b'\xff', 'strict', '\xff'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_cp708(self): + self.check_encode(708, ( + ('abc2%', 'strict', b'abc2%'), + ('\u060c\u0621\u064a', 'strict', b'\xac\xc1\xea'), + ('\u2562\xe7\xa0', 'strict', b'\x86\x87\xff'), + ('\x9a\x9f', 'strict', b'\x9a\x9f'), + ('\u256b', 'strict', b'\xc0'), + # test error handlers + ('[\u0662]', 'strict', None), + ('[\u0662]', 'ignore', b'[]'), + ('[\u0662]', 'replace', b'[?]'), + ('\udca0', 'surrogateescape', b'\xa0'), + ('\udca0', 'surrogatepass', None), + )) + self.check_decode(708, ( + (b'abc2%', 'strict', 'abc2%'), + (b'\xac\xc1\xea', 'strict', '\u060c\u0621\u064a'), + (b'\x86\x87\xff', 'strict', '\u2562\xe7\xa0'), + (b'\x9a\x9f', 'strict', '\x9a\x9f'), + (b'\xc0', 'strict', '\u256b'), + # test error handlers + (b'\xa0', 'strict', None), + (b'[\xa0]', 'ignore', '[]'), + (b'[\xa0]', 'replace', '[\ufffd]'), + (b'[\xa0]', 'backslashreplace', '[\\xa0]'), + (b'[\xa0]', 'surrogateescape', '[\udca0]'), + (b'[\xa0]', 'surrogatepass', None), + )) + + def test_cp20106(self): + self.check_encode(20106, ( + ('abc', 'strict', b'abc'), + ('\xa7\xc4\xdf', 'strict', b'@[~'), + # test error handlers + ('@', 'strict', None), + ('@', 'ignore', b''), + ('@', 'replace', b'?'), + ('\udcbf', 'surrogateescape', b'\xbf'), + ('\udcbf', 'surrogatepass', None), + )) + self.check_decode(20106, ( + (b'abc', 'strict', 'abc'), + (b'@[~', 'strict', '\xa7\xc4\xdf'), + (b'\xe1\xfe', 'strict', 'a\xdf'), + # test error handlers + (b'(\xbf)', 'strict', None), + (b'(\xbf)', 'ignore', '()'), + (b'(\xbf)', 'replace', '(\ufffd)'), + (b'(\xbf)', 'backslashreplace', '(\\xbf)'), + (b'(\xbf)', 'surrogateescape', '(\udcbf)'), + (b'(\xbf)', 'surrogatepass', None), + )) + + @unittest.expectedFailure # TODO: RUSTPYTHON; # TODO: RUSTPYTHON def test_cp_utf7(self): cp = 65000 self.check_encode(cp, ( @@ -3404,8 +3493,6 @@ def test_cp_utf7(self): (b'[\xff]', 'strict', '[\xff]'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multibyte_encoding(self): self.check_decode(932, ( (b'\x84\xe9\x80', 'ignore', '\u9a3e'), @@ -3420,8 +3507,6 @@ def test_multibyte_encoding(self): ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_code_page_decode_flags(self): # Issue #36312: For some code pages (e.g. UTF-7) flags for # MultiByteToWideChar() must be set to 0. @@ -3441,8 +3526,6 @@ def test_code_page_decode_flags(self): self.assertEqual(codecs.code_page_decode(42, b'abc'), ('\uf061\uf062\uf063', 3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_incremental(self): decoded = codecs.code_page_decode(932, b'\x82', 'strict', False) self.assertEqual(decoded, ('', 0)) @@ -3462,14 +3545,15 @@ def test_incremental(self): False) self.assertEqual(decoded, ('abc', 3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_mbcs_alias(self): - # Check that looking up our 'default' codepage will return - # mbcs when we don't have a more specific one available - with mock.patch('_winapi.GetACP', return_value=123): - codec = codecs.lookup('cp123') - self.assertEqual(codec.name, 'mbcs') + def test_mbcs_code_page(self): + # Check that codec for the current Windows (ANSII) code page is + # always available. + try: + from _winapi import GetACP + except ImportError: + self.skipTest('requires _winapi.GetACP') + cp = GetACP() + codecs.lookup(f'cp{cp}') @support.bigmemtest(size=2**31, memuse=7, dry_run=False) def test_large_input(self, size): @@ -3508,8 +3592,6 @@ class ASCIITest(unittest.TestCase): def test_encode(self): self.assertEqual('abc123'.encode('ascii'), b'abc123') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_error(self): for data, error_handler, expected in ( ('[\x80\xff\u20ac]', 'ignore', b'[]'), @@ -3532,8 +3614,6 @@ def test_encode_surrogateescape_error(self): def test_decode(self): self.assertEqual(b'abc'.decode('ascii'), 'abc') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode_error(self): for data, error_handler, expected in ( (b'[\x80\xff]', 'ignore', '[]'), @@ -3556,8 +3636,6 @@ def test_encode(self): with self.subTest(data=data, expected=expected): self.assertEqual(data.encode('latin1'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_errors(self): for data, error_handler, expected in ( ('[\u20ac\udc80]', 'ignore', b'[]'), @@ -3631,8 +3709,30 @@ def test_seeking_write(self): self.assertEqual(sr.readline(), b'abc\n') self.assertEqual(sr.readline(), b'789\n') + def test_copy(self): + bio = io.BytesIO() + codec = codecs.lookup('ascii') + sr = codecs.StreamRecoder(bio, codec.encode, codec.decode, + encodings.ascii.StreamReader, encodings.ascii.StreamWriter) + + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + copy.copy(sr) + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + copy.deepcopy(sr) + + def test_pickle(self): + q = Queue(b'') + codec = codecs.lookup('ascii') + sr = codecs.StreamRecoder(q, codec.encode, codec.decode, + encodings.ascii.StreamReader, encodings.ascii.StreamWriter) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + with self.assertRaisesRegex(TypeError, 'StreamRecoder'): + pickle.dumps(sr, proto) + -@unittest.skipIf(_testcapi is None, 'need _testcapi module') +@unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') class LocaleCodecTest(unittest.TestCase): """ Test indirectly _Py_DecodeUTF8Ex() and _Py_EncodeUTF8Ex(). @@ -3646,7 +3746,7 @@ class LocaleCodecTest(unittest.TestCase): SURROGATES = "\uDC80\uDCFF" def encode(self, text, errors="strict"): - return _testcapi.EncodeLocaleEx(text, 0, errors) + return _testinternalcapi.EncodeLocaleEx(text, 0, errors) def check_encode_strings(self, errors): for text in self.STRINGS: @@ -3686,7 +3786,7 @@ def test_encode_unsupported_error_handler(self): self.assertEqual(str(cm.exception), 'unsupported error handler') def decode(self, encoded, errors="strict"): - return _testcapi.DecodeLocaleEx(encoded, 0, errors) + return _testinternalcapi.DecodeLocaleEx(encoded, 0, errors) def check_decode_strings(self, errors): is_utf8 = (self.ENCODING == "utf-8") @@ -3717,7 +3817,7 @@ def check_decode_strings(self, errors): with self.assertRaises(RuntimeError) as cm: self.decode(encoded, errors) errmsg = str(cm.exception) - self.assertTrue(errmsg.startswith("decode error: "), errmsg) + self.assertStartsWith(errmsg, "decode error: ") else: decoded = self.decode(encoded, errors) self.assertEqual(decoded, expected) @@ -3773,9 +3873,10 @@ class Rot13UtilTest(unittest.TestCase): $ echo "Hello World" | python -m encodings.rot_13 """ def test_rot13_func(self): + from encodings.rot_13 import rot13 infile = io.StringIO('Gb or, be abg gb or, gung vf gur dhrfgvba') outfile = io.StringIO() - encodings.rot_13.rot13(infile, outfile) + rot13(infile, outfile) outfile.seek(0) plain_text = outfile.read() self.assertEqual( @@ -3785,8 +3886,6 @@ def test_rot13_func(self): class CodecNameNormalizationTest(unittest.TestCase): """Test codec name normalization""" - # TODO: RUSTPYTHON, AssertionError: Tuples differ: (1, 2, 3, 4) != (None, None, None, None) - @unittest.expectedFailure def test_codecs_lookup(self): FOUND = (1, 2, 3, 4) NOT_FOUND = (None, None, None, None) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 19117fa4091..bbc46021406 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -30,8 +30,7 @@ def assertInvalid(self, str, symbol='single', is_syntax=1): except OverflowError: self.assertTrue(not is_syntax) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xc99532080 file "", line 1> != at 0xc99532f80 file "", line 1> def test_valid(self): av = self.assertValid @@ -94,8 +93,7 @@ def test_valid(self): av("def f():\n pass\n#foo\n") av("@a.b.c\ndef f():\n pass\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xc99532080 file "", line 1> != None def test_incomplete(self): ai = self.assertIncomplete @@ -227,6 +225,9 @@ def test_incomplete(self): ai("(x for x in") ai("(x for x in (") + ai('a = f"""') + ai('a = \\') + def test_invalid(self): ai = self.assertInvalid ai("a b") @@ -279,13 +280,12 @@ def test_filename(self): self.assertNotEqual(compile_command("a = 1\n", "abc").co_filename, compile("a = 1\n", "def", 'single').co_filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 2 def test_warning(self): # Test that the warning is only returned once. with warnings_helper.check_warnings( ('"is" with \'str\' literal', SyntaxWarning), - ("invalid escape sequence", SyntaxWarning), + ('"\\\\e" is an invalid escape sequence', SyntaxWarning), ) as w: compile_command(r"'\e' is 0") self.assertEqual(len(w.warnings), 2) @@ -300,15 +300,13 @@ def test_warning(self): warnings.simplefilter('error', SyntaxWarning) compile_command(r"'\e'", symbol='exec') - # TODO: RUSTPYTHON - #def test_incomplete_warning(self): - # with warnings.catch_warnings(record=True) as w: - # warnings.simplefilter('always') - # self.assertIncomplete("'\\e' + (") - # self.assertEqual(w, []) + def test_incomplete_warning(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self.assertIncomplete("'\\e' + (") + self.assertEqual(w, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 1 def test_invalid_warning(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 68ca288fb1e..26d0dcb654d 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1,5 +1,6 @@ """Unit tests for collections.py.""" +import array import collections import copy import doctest @@ -25,7 +26,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString +from collections.abc import ByteString, Buffer class TestUserObjects(unittest.TestCase): @@ -52,18 +53,12 @@ def _copy_test(self, obj): self.assertEqual(obj.data, obj_copy.data) self.assertIs(obj.test, obj_copy.test) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_str_protocol(self): self._superset_test(UserString, str) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_protocol(self): self._superset_test(UserList, list) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dict_protocol(self): self._superset_test(UserDict, dict) @@ -77,6 +72,14 @@ def test_dict_copy(self): obj[123] = "abc" self._copy_test(obj) + def test_dict_missing(self): + class A(UserDict): + def __missing__(self, key): + return 456 + self.assertEqual(A()[123], 456) + # get() ignores __missing__ on dict + self.assertIs(A().get(123), None) + ################################################################################ ### ChainMap (helper class for configparser and the string module) @@ -259,8 +262,7 @@ def __contains__(self, key): d = c.new_child(b=20, c=30) self.assertEqual(d.maps, [{'b': 20, 'c': 30}, {'a': 1, 'b': 2}]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: .Subclass'> is not def test_union_operators(self): cm1 = ChainMap(dict(a=1, b=2), dict(c=3, d=4)) cm2 = ChainMap(dict(a=10, e=5), dict(b=20, d=4)) @@ -488,12 +490,8 @@ def test_instance(self): self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method - try: + with self.assertRaises(TypeError): p._replace(x=1, error=2) - except ValueError: - pass - else: - self._fail('Did not detect an incorrect fieldname') # verify that field string can have commas Point = namedtuple('Point', 'x, y') @@ -545,7 +543,9 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) - n = 5000 + @support.requires_resource('cpu') + def test_large_size(self): + n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) n = len(names) @@ -696,8 +696,6 @@ class NewPoint(tuple): self.assertEqual(np.x, 1) self.assertEqual(np.y, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_new_builtins_issue_43102(self): obj = namedtuple('C', ()) new_func = obj.__new__ @@ -739,7 +737,7 @@ def validate_abstract_methods(self, abc, *names): stubs = methodstubs.copy() del stubs[name] C = type('C', (abc,), stubs) - self.assertRaises(TypeError, C, name) + self.assertRaises(TypeError, C) def validate_isinstance(self, abc, name): stub = lambda s, *args: 0 @@ -747,11 +745,11 @@ def validate_isinstance(self, abc, name): C = type('C', (object,), {'__hash__': None}) setattr(C, name, stub) self.assertIsInstance(C(), abc) - self.assertTrue(issubclass(C, abc)) + self.assertIsSubclass(C, abc) C = type('C', (object,), {'__hash__': None}) self.assertNotIsInstance(C(), abc) - self.assertFalse(issubclass(C, abc)) + self.assertNotIsSubclass(C, abc) def validate_comparison(self, instance): ops = ['lt', 'gt', 'le', 'ge', 'ne', 'or', 'and', 'xor', 'sub'] @@ -789,8 +787,6 @@ def _test_gen(): class TestOneTrickPonyABCs(ABCTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_Awaitable(self): def gen(): yield @@ -819,12 +815,12 @@ def __await__(self): non_samples = [None, int(), gen(), object()] for x in non_samples: self.assertNotIsInstance(x, Awaitable) - self.assertFalse(issubclass(type(x), Awaitable), repr(type(x))) + self.assertNotIsSubclass(type(x), Awaitable) samples = [Bar(), MinimalCoro()] for x in samples: self.assertIsInstance(x, Awaitable) - self.assertTrue(issubclass(type(x), Awaitable)) + self.assertIsSubclass(type(x), Awaitable) c = coro() # Iterable coroutines (generators with CO_ITERABLE_COROUTINE @@ -838,13 +834,11 @@ def __await__(self): class CoroLike: pass Coroutine.register(CoroLike) - self.assertTrue(isinstance(CoroLike(), Awaitable)) - self.assertTrue(issubclass(CoroLike, Awaitable)) + self.assertIsInstance(CoroLike(), Awaitable) + self.assertIsSubclass(CoroLike, Awaitable) CoroLike = None support.gc_collect() # Kill CoroLike to clean-up ABCMeta cache - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_Coroutine(self): def gen(): yield @@ -873,12 +867,12 @@ def __await__(self): non_samples = [None, int(), gen(), object(), Bar()] for x in non_samples: self.assertNotIsInstance(x, Coroutine) - self.assertFalse(issubclass(type(x), Coroutine), repr(type(x))) + self.assertNotIsSubclass(type(x), Coroutine) samples = [MinimalCoro()] for x in samples: self.assertIsInstance(x, Awaitable) - self.assertTrue(issubclass(type(x), Awaitable)) + self.assertIsSubclass(type(x), Awaitable) c = coro() # Iterable coroutines (generators with CO_ITERABLE_COROUTINE @@ -899,8 +893,8 @@ def close(self): pass def __await__(self): pass - self.assertTrue(isinstance(CoroLike(), Coroutine)) - self.assertTrue(issubclass(CoroLike, Coroutine)) + self.assertIsInstance(CoroLike(), Coroutine) + self.assertIsSubclass(CoroLike, Coroutine) class CoroLike: def send(self, value): @@ -909,15 +903,15 @@ def close(self): pass def __await__(self): pass - self.assertFalse(isinstance(CoroLike(), Coroutine)) - self.assertFalse(issubclass(CoroLike, Coroutine)) + self.assertNotIsInstance(CoroLike(), Coroutine) + self.assertNotIsSubclass(CoroLike, Coroutine) def test_Hashable(self): # Check some non-hashables non_samples = [bytearray(), list(), set(), dict()] for x in non_samples: self.assertNotIsInstance(x, Hashable) - self.assertFalse(issubclass(type(x), Hashable), repr(type(x))) + self.assertNotIsSubclass(type(x), Hashable) # Check some hashables samples = [None, int(), float(), complex(), @@ -927,14 +921,14 @@ def test_Hashable(self): ] for x in samples: self.assertIsInstance(x, Hashable) - self.assertTrue(issubclass(type(x), Hashable), repr(type(x))) + self.assertIsSubclass(type(x), Hashable) self.assertRaises(TypeError, Hashable) # Check direct subclassing class H(Hashable): def __hash__(self): return super().__hash__() self.assertEqual(hash(H()), 0) - self.assertFalse(issubclass(int, H)) + self.assertNotIsSubclass(int, H) self.validate_abstract_methods(Hashable, '__hash__') self.validate_isinstance(Hashable, '__hash__') @@ -942,44 +936,42 @@ def test_AsyncIterable(self): class AI: def __aiter__(self): return self - self.assertTrue(isinstance(AI(), AsyncIterable)) - self.assertTrue(issubclass(AI, AsyncIterable)) + self.assertIsInstance(AI(), AsyncIterable) + self.assertIsSubclass(AI, AsyncIterable) # Check some non-iterables non_samples = [None, object, []] for x in non_samples: self.assertNotIsInstance(x, AsyncIterable) - self.assertFalse(issubclass(type(x), AsyncIterable), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncIterable) self.validate_abstract_methods(AsyncIterable, '__aiter__') self.validate_isinstance(AsyncIterable, '__aiter__') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_AsyncIterator(self): class AI: def __aiter__(self): return self async def __anext__(self): raise StopAsyncIteration - self.assertTrue(isinstance(AI(), AsyncIterator)) - self.assertTrue(issubclass(AI, AsyncIterator)) + self.assertIsInstance(AI(), AsyncIterator) + self.assertIsSubclass(AI, AsyncIterator) non_samples = [None, object, []] # Check some non-iterables for x in non_samples: self.assertNotIsInstance(x, AsyncIterator) - self.assertFalse(issubclass(type(x), AsyncIterator), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncIterator) # Similarly to regular iterators (see issue 10565) class AnextOnly: async def __anext__(self): raise StopAsyncIteration self.assertNotIsInstance(AnextOnly(), AsyncIterator) - self.validate_abstract_methods(AsyncIterator, '__anext__', '__aiter__') + self.validate_abstract_methods(AsyncIterator, '__anext__') def test_Iterable(self): # Check some non-iterables non_samples = [None, 42, 3.14, 1j] for x in non_samples: self.assertNotIsInstance(x, Iterable) - self.assertFalse(issubclass(type(x), Iterable), repr(type(x))) + self.assertNotIsSubclass(type(x), Iterable) # Check some iterables samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), @@ -989,13 +981,13 @@ def test_Iterable(self): ] for x in samples: self.assertIsInstance(x, Iterable) - self.assertTrue(issubclass(type(x), Iterable), repr(type(x))) + self.assertIsSubclass(type(x), Iterable) # Check direct subclassing class I(Iterable): def __iter__(self): return super().__iter__() self.assertEqual(list(I()), []) - self.assertFalse(issubclass(str, I)) + self.assertNotIsSubclass(str, I) self.validate_abstract_methods(Iterable, '__iter__') self.validate_isinstance(Iterable, '__iter__') # Check None blocking @@ -1003,22 +995,22 @@ class It: def __iter__(self): return iter([]) class ItBlocked(It): __iter__ = None - self.assertTrue(issubclass(It, Iterable)) - self.assertTrue(isinstance(It(), Iterable)) - self.assertFalse(issubclass(ItBlocked, Iterable)) - self.assertFalse(isinstance(ItBlocked(), Iterable)) + self.assertIsSubclass(It, Iterable) + self.assertIsInstance(It(), Iterable) + self.assertNotIsSubclass(ItBlocked, Iterable) + self.assertNotIsInstance(ItBlocked(), Iterable) def test_Reversible(self): # Check some non-reversibles non_samples = [None, 42, 3.14, 1j, set(), frozenset()] for x in non_samples: self.assertNotIsInstance(x, Reversible) - self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + self.assertNotIsSubclass(type(x), Reversible) # Check some non-reversible iterables non_reversibles = [_test_gen(), (x for x in []), iter([]), reversed([])] for x in non_reversibles: self.assertNotIsInstance(x, Reversible) - self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + self.assertNotIsSubclass(type(x), Reversible) # Check some reversible iterables samples = [bytes(), str(), tuple(), list(), OrderedDict(), OrderedDict().keys(), OrderedDict().items(), @@ -1027,11 +1019,11 @@ def test_Reversible(self): dict().keys(), dict().items(), dict().values()] for x in samples: self.assertIsInstance(x, Reversible) - self.assertTrue(issubclass(type(x), Reversible), repr(type(x))) + self.assertIsSubclass(type(x), Reversible) # Check also Mapping, MutableMapping, and Sequence - self.assertTrue(issubclass(Sequence, Reversible), repr(Sequence)) - self.assertFalse(issubclass(Mapping, Reversible), repr(Mapping)) - self.assertFalse(issubclass(MutableMapping, Reversible), repr(MutableMapping)) + self.assertIsSubclass(Sequence, Reversible) + self.assertNotIsSubclass(Mapping, Reversible) + self.assertNotIsSubclass(MutableMapping, Reversible) # Check direct subclassing class R(Reversible): def __iter__(self): @@ -1039,17 +1031,17 @@ def __iter__(self): def __reversed__(self): return iter(list()) self.assertEqual(list(reversed(R())), []) - self.assertFalse(issubclass(float, R)) + self.assertNotIsSubclass(float, R) self.validate_abstract_methods(Reversible, '__reversed__', '__iter__') # Check reversible non-iterable (which is not Reversible) class RevNoIter: def __reversed__(self): return reversed([]) class RevPlusIter(RevNoIter): def __iter__(self): return iter([]) - self.assertFalse(issubclass(RevNoIter, Reversible)) - self.assertFalse(isinstance(RevNoIter(), Reversible)) - self.assertTrue(issubclass(RevPlusIter, Reversible)) - self.assertTrue(isinstance(RevPlusIter(), Reversible)) + self.assertNotIsSubclass(RevNoIter, Reversible) + self.assertNotIsInstance(RevNoIter(), Reversible) + self.assertIsSubclass(RevPlusIter, Reversible) + self.assertIsInstance(RevPlusIter(), Reversible) # Check None blocking class Rev: def __iter__(self): return iter([]) @@ -1058,39 +1050,38 @@ class RevItBlocked(Rev): __iter__ = None class RevRevBlocked(Rev): __reversed__ = None - self.assertTrue(issubclass(Rev, Reversible)) - self.assertTrue(isinstance(Rev(), Reversible)) - self.assertFalse(issubclass(RevItBlocked, Reversible)) - self.assertFalse(isinstance(RevItBlocked(), Reversible)) - self.assertFalse(issubclass(RevRevBlocked, Reversible)) - self.assertFalse(isinstance(RevRevBlocked(), Reversible)) + self.assertIsSubclass(Rev, Reversible) + self.assertIsInstance(Rev(), Reversible) + self.assertNotIsSubclass(RevItBlocked, Reversible) + self.assertNotIsInstance(RevItBlocked(), Reversible) + self.assertNotIsSubclass(RevRevBlocked, Reversible) + self.assertNotIsInstance(RevRevBlocked(), Reversible) def test_Collection(self): # Check some non-collections non_collections = [None, 42, 3.14, 1j, lambda x: 2*x] for x in non_collections: self.assertNotIsInstance(x, Collection) - self.assertFalse(issubclass(type(x), Collection), repr(type(x))) + self.assertNotIsSubclass(type(x), Collection) # Check some non-collection iterables non_col_iterables = [_test_gen(), iter(b''), iter(bytearray()), (x for x in [])] for x in non_col_iterables: self.assertNotIsInstance(x, Collection) - self.assertFalse(issubclass(type(x), Collection), repr(type(x))) + self.assertNotIsSubclass(type(x), Collection) # Check some collections samples = [set(), frozenset(), dict(), bytes(), str(), tuple(), list(), dict().keys(), dict().items(), dict().values()] for x in samples: self.assertIsInstance(x, Collection) - self.assertTrue(issubclass(type(x), Collection), repr(type(x))) + self.assertIsSubclass(type(x), Collection) # Check also Mapping, MutableMapping, etc. - self.assertTrue(issubclass(Sequence, Collection), repr(Sequence)) - self.assertTrue(issubclass(Mapping, Collection), repr(Mapping)) - self.assertTrue(issubclass(MutableMapping, Collection), - repr(MutableMapping)) - self.assertTrue(issubclass(Set, Collection), repr(Set)) - self.assertTrue(issubclass(MutableSet, Collection), repr(MutableSet)) - self.assertTrue(issubclass(Sequence, Collection), repr(MutableSet)) + self.assertIsSubclass(Sequence, Collection) + self.assertIsSubclass(Mapping, Collection) + self.assertIsSubclass(MutableMapping, Collection) + self.assertIsSubclass(Set, Collection) + self.assertIsSubclass(MutableSet, Collection) + self.assertIsSubclass(Sequence, Collection) # Check direct subclassing class Col(Collection): def __iter__(self): @@ -1101,13 +1092,13 @@ def __contains__(self, item): return False class DerCol(Col): pass self.assertEqual(list(iter(Col())), []) - self.assertFalse(issubclass(list, Col)) - self.assertFalse(issubclass(set, Col)) - self.assertFalse(issubclass(float, Col)) + self.assertNotIsSubclass(list, Col) + self.assertNotIsSubclass(set, Col) + self.assertNotIsSubclass(float, Col) self.assertEqual(list(iter(DerCol())), []) - self.assertFalse(issubclass(list, DerCol)) - self.assertFalse(issubclass(set, DerCol)) - self.assertFalse(issubclass(float, DerCol)) + self.assertNotIsSubclass(list, DerCol) + self.assertNotIsSubclass(set, DerCol) + self.assertNotIsSubclass(float, DerCol) self.validate_abstract_methods(Collection, '__len__', '__iter__', '__contains__') # Check sized container non-iterable (which is not Collection) etc. @@ -1120,12 +1111,12 @@ def __contains__(self, item): return False class ColNoCont: def __iter__(self): return iter([]) def __len__(self): return 0 - self.assertFalse(issubclass(ColNoIter, Collection)) - self.assertFalse(isinstance(ColNoIter(), Collection)) - self.assertFalse(issubclass(ColNoSize, Collection)) - self.assertFalse(isinstance(ColNoSize(), Collection)) - self.assertFalse(issubclass(ColNoCont, Collection)) - self.assertFalse(isinstance(ColNoCont(), Collection)) + self.assertNotIsSubclass(ColNoIter, Collection) + self.assertNotIsInstance(ColNoIter(), Collection) + self.assertNotIsSubclass(ColNoSize, Collection) + self.assertNotIsInstance(ColNoSize(), Collection) + self.assertNotIsSubclass(ColNoCont, Collection) + self.assertNotIsInstance(ColNoCont(), Collection) # Check None blocking class SizeBlock: def __iter__(self): return iter([]) @@ -1135,10 +1126,10 @@ class IterBlock: def __len__(self): return 0 def __contains__(self): return True __iter__ = None - self.assertFalse(issubclass(SizeBlock, Collection)) - self.assertFalse(isinstance(SizeBlock(), Collection)) - self.assertFalse(issubclass(IterBlock, Collection)) - self.assertFalse(isinstance(IterBlock(), Collection)) + self.assertNotIsSubclass(SizeBlock, Collection) + self.assertNotIsInstance(SizeBlock(), Collection) + self.assertNotIsSubclass(IterBlock, Collection) + self.assertNotIsInstance(IterBlock(), Collection) # Check None blocking in subclass class ColImpl: def __iter__(self): @@ -1149,16 +1140,15 @@ def __contains__(self, item): return False class NonCol(ColImpl): __contains__ = None - self.assertFalse(issubclass(NonCol, Collection)) - self.assertFalse(isinstance(NonCol(), Collection)) + self.assertNotIsSubclass(NonCol, Collection) + self.assertNotIsInstance(NonCol(), Collection) + - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] for x in non_samples: self.assertNotIsInstance(x, Iterator) - self.assertFalse(issubclass(type(x), Iterator), repr(type(x))) + self.assertNotIsSubclass(type(x), Iterator) samples = [iter(bytes()), iter(str()), iter(tuple()), iter(list()), iter(dict()), iter(set()), iter(frozenset()), @@ -1169,8 +1159,8 @@ def test_Iterator(self): ] for x in samples: self.assertIsInstance(x, Iterator) - self.assertTrue(issubclass(type(x), Iterator), repr(type(x))) - self.validate_abstract_methods(Iterator, '__next__', '__iter__') + self.assertIsSubclass(type(x), Iterator) + self.validate_abstract_methods(Iterator, '__next__') # Issue 10565 class NextOnly: @@ -1202,7 +1192,7 @@ def throw(self, typ, val=None, tb=None): pass iter(()), iter([]), NonGen1(), NonGen2(), NonGen3()] for x in non_samples: self.assertNotIsInstance(x, Generator) - self.assertFalse(issubclass(type(x), Generator), repr(type(x))) + self.assertNotIsSubclass(type(x), Generator) class Gen: def __iter__(self): return self @@ -1224,7 +1214,7 @@ def gen(): for x in samples: self.assertIsInstance(x, Iterator) self.assertIsInstance(x, Generator) - self.assertTrue(issubclass(type(x), Generator), repr(type(x))) + self.assertIsSubclass(type(x), Generator) self.validate_abstract_methods(Generator, 'send', 'throw') # mixin tests @@ -1273,7 +1263,7 @@ def athrow(self, typ, val=None, tb=None): pass iter(()), iter([]), NonAGen1(), NonAGen2(), NonAGen3()] for x in non_samples: self.assertNotIsInstance(x, AsyncGenerator) - self.assertFalse(issubclass(type(x), AsyncGenerator), repr(type(x))) + self.assertNotIsSubclass(type(x), AsyncGenerator) class Gen: def __aiter__(self): return self @@ -1295,7 +1285,7 @@ async def gen(): for x in samples: self.assertIsInstance(x, AsyncIterator) self.assertIsInstance(x, AsyncGenerator) - self.assertTrue(issubclass(type(x), AsyncGenerator), repr(type(x))) + self.assertIsSubclass(type(x), AsyncGenerator) self.validate_abstract_methods(AsyncGenerator, 'asend', 'athrow') def run_async(coro): @@ -1338,14 +1328,14 @@ def test_Sized(self): ] for x in non_samples: self.assertNotIsInstance(x, Sized) - self.assertFalse(issubclass(type(x), Sized), repr(type(x))) + self.assertNotIsSubclass(type(x), Sized) samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), dict().keys(), dict().items(), dict().values(), ] for x in samples: self.assertIsInstance(x, Sized) - self.assertTrue(issubclass(type(x), Sized), repr(type(x))) + self.assertIsSubclass(type(x), Sized) self.validate_abstract_methods(Sized, '__len__') self.validate_isinstance(Sized, '__len__') @@ -1356,14 +1346,14 @@ def test_Container(self): ] for x in non_samples: self.assertNotIsInstance(x, Container) - self.assertFalse(issubclass(type(x), Container), repr(type(x))) + self.assertNotIsSubclass(type(x), Container) samples = [bytes(), str(), tuple(), list(), set(), frozenset(), dict(), dict().keys(), dict().items(), ] for x in samples: self.assertIsInstance(x, Container) - self.assertTrue(issubclass(type(x), Container), repr(type(x))) + self.assertIsSubclass(type(x), Container) self.validate_abstract_methods(Container, '__contains__') self.validate_isinstance(Container, '__contains__') @@ -1375,7 +1365,7 @@ def test_Callable(self): ] for x in non_samples: self.assertNotIsInstance(x, Callable) - self.assertFalse(issubclass(type(x), Callable), repr(type(x))) + self.assertNotIsSubclass(type(x), Callable) samples = [lambda: None, type, int, object, len, @@ -1383,7 +1373,7 @@ def test_Callable(self): ] for x in samples: self.assertIsInstance(x, Callable) - self.assertTrue(issubclass(type(x), Callable), repr(type(x))) + self.assertIsSubclass(type(x), Callable) self.validate_abstract_methods(Callable, '__call__') self.validate_isinstance(Callable, '__call__') @@ -1391,16 +1381,16 @@ def test_direct_subclassing(self): for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C(B): pass - self.assertTrue(issubclass(C, B)) - self.assertFalse(issubclass(int, C)) + self.assertIsSubclass(C, B) + self.assertNotIsSubclass(int, C) def test_registration(self): for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C: __hash__ = None # Make sure it isn't hashable by default - self.assertFalse(issubclass(C, B), B.__name__) + self.assertNotIsSubclass(C, B) B.register(C) - self.assertTrue(issubclass(C, B)) + self.assertIsSubclass(C, B) class WithSet(MutableSet): @@ -1431,7 +1421,7 @@ class TestCollectionABCs(ABCTestCase): def test_Set(self): for sample in [set, frozenset]: self.assertIsInstance(sample(), Set) - self.assertTrue(issubclass(sample, Set)) + self.assertIsSubclass(sample, Set) self.validate_abstract_methods(Set, '__contains__', '__iter__', '__len__') class MySet(Set): def __contains__(self, x): @@ -1512,9 +1502,9 @@ def __len__(self): def test_MutableSet(self): self.assertIsInstance(set(), MutableSet) - self.assertTrue(issubclass(set, MutableSet)) + self.assertIsSubclass(set, MutableSet) self.assertNotIsInstance(frozenset(), MutableSet) - self.assertFalse(issubclass(frozenset, MutableSet)) + self.assertNotIsSubclass(frozenset, MutableSet) self.validate_abstract_methods(MutableSet, '__contains__', '__iter__', '__len__', 'add', 'discard') @@ -1635,7 +1625,7 @@ def test_Set_from_iterable(self): class SetUsingInstanceFromIterable(MutableSet): def __init__(self, values, created_by): if not created_by: - raise ValueError(f'created_by must be specified') + raise ValueError('created_by must be specified') self.created_by = created_by self._values = set(values) @@ -1838,8 +1828,6 @@ def __repr__(self): self.assertTrue(f1 != l1) self.assertTrue(f1 != l2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_Set_hash_matches_frozenset(self): sets = [ {}, {1}, {None}, {-1}, {0.0}, {"abc"}, {1, 2, 3}, @@ -1852,14 +1840,11 @@ def test_Set_hash_matches_frozenset(self): fs = frozenset(s) self.assertEqual(hash(fs), Set._hash(fs), msg=s) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) - self.assertTrue(issubclass(sample, Mapping)) - self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__', - '__getitem__') + self.assertIsSubclass(sample, Mapping) + self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): def __len__(self): return 0 @@ -1870,13 +1855,11 @@ def __iter__(self): self.validate_comparison(MyMapping()) self.assertRaises(TypeError, reversed, MyMapping()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) - self.assertTrue(issubclass(sample, MutableMapping)) - self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__', + self.assertIsSubclass(sample, MutableMapping) + self.validate_abstract_methods(MutableMapping, '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') def test_MutableMapping_subclass(self): @@ -1906,19 +1889,16 @@ def test_MutableMapping_subclass(self): mymap['blue'] = 7 # Shouldn't affect 'z' self.assertEqual(z, {('orange', 3), ('red', 5)}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_Sequence(self): for sample in [tuple, list, bytes, str]: self.assertIsInstance(sample(), Sequence) - self.assertTrue(issubclass(sample, Sequence)) + self.assertIsSubclass(sample, Sequence) self.assertIsInstance(range(10), Sequence) - self.assertTrue(issubclass(range, Sequence)) + self.assertIsSubclass(range, Sequence) self.assertIsInstance(memoryview(b""), Sequence) - self.assertTrue(issubclass(memoryview, Sequence)) - self.assertTrue(issubclass(str, Sequence)) - self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', - '__getitem__') + self.assertIsSubclass(memoryview, Sequence) + self.assertIsSubclass(str, Sequence) + self.validate_abstract_methods(Sequence, '__len__', '__getitem__') def test_Sequence_mixins(self): class SequenceSubclass(Sequence): @@ -1957,26 +1937,47 @@ def assert_index_same(seq1, seq2, index_args): def test_ByteString(self): for sample in [bytes, bytearray]: - self.assertIsInstance(sample(), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(sample(), ByteString) self.assertTrue(issubclass(sample, ByteString)) for sample in [str, list, tuple]: - self.assertNotIsInstance(sample(), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(sample(), ByteString) self.assertFalse(issubclass(sample, ByteString)) - self.assertNotIsInstance(memoryview(b""), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(memoryview(b""), ByteString) self.assertFalse(issubclass(memoryview, ByteString)) + with self.assertWarns(DeprecationWarning): + self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + + with self.assertWarns(DeprecationWarning): + class X(ByteString): pass + + with self.assertWarns(DeprecationWarning): + # No metaclass conflict + class Z(ByteString, Awaitable): pass + + @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types) + def test_Buffer(self): + for sample in [bytes, bytearray, memoryview]: + self.assertIsInstance(sample(b"x"), Buffer) + self.assertIsSubclass(sample, Buffer) + for sample in [str, list, tuple]: + self.assertNotIsInstance(sample(), Buffer) + self.assertNotIsSubclass(sample, Buffer) + self.validate_abstract_methods(Buffer, '__buffer__') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) - self.assertFalse(issubclass(sample, MutableSequence)) + self.assertNotIsSubclass(sample, MutableSequence) for sample in [list, bytearray, deque]: self.assertIsInstance(sample(), MutableSequence) - self.assertTrue(issubclass(sample, MutableSequence)) - self.assertFalse(issubclass(str, MutableSequence)) - self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', - '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + self.assertIsSubclass(sample, MutableSequence) + self.assertIsSubclass(array.array, MutableSequence) + self.assertNotIsSubclass(str, MutableSequence) + self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__', + '__setitem__', '__delitem__', 'insert') def test_MutableSequence_mixins(self): # Test the mixins of MutableSequence by creating a minimal concrete @@ -2028,8 +2029,7 @@ def insert(self, index, value): self.assertEqual(len(mss), len(mss2)) self.assertEqual(list(mss), list(mss2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_illegal_patma_flags(self): with self.assertRaises(TypeError): class Both(Collection): @@ -2067,8 +2067,8 @@ def test_basics(self): self.assertEqual(c, Counter(a=3, b=2, c=1)) self.assertIsInstance(c, dict) self.assertIsInstance(c, Mapping) - self.assertTrue(issubclass(Counter, dict)) - self.assertTrue(issubclass(Counter, Mapping)) + self.assertIsSubclass(Counter, dict) + self.assertIsSubclass(Counter, Mapping) self.assertEqual(len(c), 3) self.assertEqual(sum(c.values()), 6) self.assertEqual(list(c.values()), [3, 2, 1]) @@ -2121,6 +2121,19 @@ def test_basics(self): self.assertEqual(c.setdefault('e', 5), 5) self.assertEqual(c['e'], 5) + def test_update_reentrant_add_clears_counter(self): + c = Counter() + key = object() + + class Evil(int): + def __add__(self, other): + c.clear() + return NotImplemented + + c[key] = Evil() + c.update([key]) + self.assertEqual(c[key], 1) + def test_init(self): self.assertEqual(list(Counter(self=42).items()), [('self', 42)]) self.assertEqual(list(Counter(iterable=42).items()), [('iterable', 42)]) diff --git a/Lib/test/test_colorsys.py b/Lib/test/test_colorsys.py index a24e3adcb4b..74d76294b0b 100644 --- a/Lib/test/test_colorsys.py +++ b/Lib/test/test_colorsys.py @@ -69,6 +69,16 @@ def test_hls_values(self): self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb)) self.assertTripleEqual(rgb, colorsys.hls_to_rgb(*hls)) + def test_hls_nearwhite(self): # gh-106498 + values = ( + # rgb, hls: these do not work in reverse + ((0.9999999999999999, 1, 1), (0.5, 1.0, 1.0)), + ((1, 0.9999999999999999, 0.9999999999999999), (0.0, 1.0, 1.0)), + ) + for rgb, hls in values: + self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb)) + self.assertTripleEqual((1.0, 1.0, 1.0), colorsys.hls_to_rgb(*hls)) + def test_yiq_roundtrip(self): for r in frange(0.0, 1.0, 0.2): for g in frange(0.0, 1.0, 0.2): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 51c834d7982..b469c8ab2de 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,5 +1,9 @@ +import contextlib import dis +import io +import itertools import math +import opcode import os import unittest import sys @@ -8,11 +12,18 @@ import tempfile import types import textwrap +import warnings +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + from test import support -from test.support import script_helper, requires_debug_ranges +from test.support import (script_helper, requires_debug_ranges, run_code, + requires_specialization) +from test.support.bytecode_helper import instructions_with_positions from test.support.os_helper import FakePath - class TestSpecifics(unittest.TestCase): def compile_single(self, source): @@ -108,37 +119,36 @@ def __getitem__(self, key): exec('z = a', g, d) self.assertEqual(d['z'], 12) - @unittest.skip("TODO: RUSTPYTHON; segmentation fault") + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_extended_arg(self): - # default: 1000 * 2.5 = 2500 repetitions - repeat = int(sys.getrecursionlimit() * 2.5) + repeat = 100 longexpr = 'x = x or ' + '-x' * repeat g = {} - code = ''' -def f(x): - %s - %s - %s - %s - %s - %s - %s - %s - %s - %s - # the expressions above have no effect, x == argument - while x: - x -= 1 - # EXTENDED_ARG/JUMP_ABSOLUTE here - return x -''' % ((longexpr,)*10) + code = textwrap.dedent(''' + def f(x): + %s + %s + %s + %s + %s + %s + %s + %s + %s + %s + # the expressions above have no effect, x == argument + while x: + x -= 1 + # EXTENDED_ARG/JUMP_ABSOLUTE here + return x + ''' % ((longexpr,)*10)) exec(code, g) self.assertEqual(g['f'](5), 0) def test_argument_order(self): self.assertRaises(SyntaxError, exec, 'def f(a=1, b): pass') - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: ParseFloatError { kind: Invalid }'") def test_float_literals(self): # testing bad float literals self.assertRaises(SyntaxError, eval, "2e") @@ -148,14 +158,13 @@ def test_float_literals(self): def test_indentation(self): # testing compile() of indented block w/o trailing newline" - s = """ -if 1: - if 2: - pass""" + s = textwrap.dedent(""" + if 1: + if 2: + pass + """) compile(s, "", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure # This test is probably specific to CPython and may not generalize # to other implementations. We are trying to ensure that when # the first line of code starts after 256, correct line numbers @@ -164,9 +173,8 @@ def test_leading_newlines(self): s256 = "".join(["\n"] * 256 + ["spam"]) co = compile(s256, 'fn', 'exec') self.assertEqual(co.co_firstlineno, 1) - lines = list(co.co_lines()) - self.assertEqual(lines[0][2], 0) - self.assertEqual(lines[1][2], 257) + lines = [line for _, _, line in co.co_lines()] + self.assertEqual(lines, [0, 257]) def test_literals_with_leading_zeroes(self): for arg in ["077787", "0xj", "0x.", "0e", "090000000000000", @@ -201,6 +209,7 @@ def test_literals_with_leading_zeroes(self): self.assertEqual(eval("0o777"), 511) self.assertEqual(eval("-0o0000010"), -8) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised def test_int_literals_too_long(self): n = 3000 source = f"a = 1\nb = 2\nc = {'3'*n}\nd = 4" @@ -274,6 +283,7 @@ def test_none_assignment(self): self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile def test_import(self): succeed = [ 'import sys', @@ -334,8 +344,11 @@ def test_lambda_doc(self): l = lambda: "foo" self.assertIsNone(l.__doc__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_lambda_consts(self): + l = lambda: "this is the only const" + self.assertEqual(l.__code__.co_consts, ("this is the only const",)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile def test_encoding(self): code = b'# -*- coding: badencoding -*-\npass\n' self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') @@ -440,16 +453,134 @@ class A: def f(): __mangled = 1 __not_mangled__ = 2 - import __mangled_mod - import __package__.module + import __mangled_mod # noqa: F401 + import __package__.module # noqa: F401 self.assertIn("_A__mangled", A.f.__code__.co_varnames) self.assertIn("__not_mangled__", A.f.__code__.co_varnames) self.assertIn("_A__mangled_mod", A.f.__code__.co_varnames) self.assertIn("__package__", A.f.__code__.co_varnames) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_condition_expression_with_dead_blocks_compiles(self): + # See gh-113054 + compile('if (5 if 5 else T): 0', '', 'exec') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_condition_expression_with_redundant_comparisons_compiles(self): + # See gh-113054, gh-114083 + exprs = [ + 'if 9<9<9and 9or 9:9', + 'if 9<9<9and 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9:9', + 'if 9<9<9and 9or 9or 9or 9or 9:9', + ] + for expr in exprs: + with self.subTest(expr=expr): + with self.assertWarns(SyntaxWarning): + compile(expr, '', 'exec') + + def test_dead_code_with_except_handler_compiles(self): + compile(textwrap.dedent(""" + if None: + with CM: + x = 1 + else: + x = 2 + """), '', 'exec') + + def test_try_except_in_while_with_chained_condition_compiles(self): + # see gh-124871 + compile(textwrap.dedent(""" + name_1, name_2, name_3 = 1, 2, 3 + while name_3 <= name_2 > name_1: + try: + raise + except: + pass + finally: + pass + """), '', 'exec') + + def test_compile_invalid_namedexpr(self): + # gh-109351 + m = ast.Module( + body=[ + ast.Expr( + value=ast.ListComp( + elt=ast.NamedExpr( + target=ast.Constant(value=1), + value=ast.Constant(value=3), + ), + generators=[ + ast.comprehension( + target=ast.Name(id="x", ctx=ast.Store()), + iter=ast.Name(id="y", ctx=ast.Load()), + ifs=[], + is_async=0, + ) + ], + ) + ) + ], + type_ignores=[], + ) + + with self.assertRaisesRegex(TypeError, "NamedExpr target must be a Name"): + compile(ast.fix_missing_locations(m), "", "exec") + + def test_compile_redundant_jumps_and_nops_after_moving_cold_blocks(self): + # See gh-120367 + code=textwrap.dedent(""" + try: + pass + except: + pass + else: + match name_2: + case b'': + pass + finally: + something + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + def test_compile_redundant_jump_after_convert_pseudo_ops(self): + # See gh-120367 + code=textwrap.dedent(""" + if name_2: + pass + else: + try: + pass + except: + pass + ~name_5 + """) + + tree = ast.parse(code) + + # make all instruction locations the same to create redundancies + for node in ast.walk(tree): + if hasattr(node,"lineno"): + del node.lineno + del node.end_lineno + del node.col_offset + del node.end_col_offset + + compile(ast.fix_missing_locations(tree), "", "exec") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: at 0xb77555080 file "1", line 1> != at 0xb77554f00 file "3", line 1> def test_compile_ast(self): fname = __file__ if fname.lower().endswith('pyc'): @@ -478,13 +609,33 @@ def test_compile_ast(self): self.assertRaises(TypeError, compile, co1, '', 'eval') # raise exception when node type is no start node - self.assertRaises(TypeError, compile, _ast.If(), '', 'exec') + self.assertRaises(TypeError, compile, _ast.If(test=_ast.Name(id='x', ctx=_ast.Load())), '', 'exec') # raise exception when node has invalid children ast = _ast.Module() - ast.body = [_ast.BoolOp()] + ast.body = [_ast.BoolOp(op=_ast.Or())] self.assertRaises(TypeError, compile, ast, '', 'exec') + def test_compile_invalid_typealias(self): + # gh-109341 + m = ast.Module( + body=[ + ast.TypeAlias( + name=ast.Subscript( + value=ast.Name(id="foo", ctx=ast.Load()), + slice=ast.Constant(value="x"), + ctx=ast.Store(), + ), + type_params=[], + value=ast.Name(id="Callable", ctx=ast.Load()), + ) + ], + type_ignores=[], + ) + + with self.assertRaisesRegex(TypeError, "TypeAlias with non-Name name"): + compile(ast.fix_missing_locations(m), "", "exec") + def test_dict_evaluation_order(self): i = 0 @@ -496,18 +647,32 @@ def f(): d = {f(): f(), f(): f()} self.assertEqual(d, {1: 2, 3: 4}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_compile_filename(self): for filename in 'file.py', b'file.py': code = compile('pass', filename, 'exec') self.assertEqual(code.co_filename, 'file.py') for filename in bytearray(b'file.py'), memoryview(b'file.py'): - with self.assertWarns(DeprecationWarning): - code = compile('pass', filename, 'exec') - self.assertEqual(code.co_filename, 'file.py') + with self.assertRaises(TypeError): + compile('pass', filename, 'exec') self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type bool, not EvilBool + def test_compile_filename_refleak(self): + # Regression tests for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'mode' AC conversion failed + self.assertRaises(TypeError, compile, b'', mortal_str, mode=1234) + # check error path when 'optimize' AC conversion failed + self.assertRaises(OverflowError, compile, b'', mortal_str, + 'exec', optimize=1 << 1000) + # check error path when 'dont_inherit' AC conversion failed + class EvilBool: + def __bool__(self): raise ValueError + self.assertRaises(ValueError, compile, b'', mortal_str, + 'exec', dont_inherit=EvilBool()) + @support.cpython_only def test_same_filename_used(self): s = """def f(): pass\ndef g(): pass""" @@ -533,8 +698,7 @@ def test_single_statement(self): self.compile_single("class T:\n pass") self.compile_single("c = '''\na=1\nb=2\nc=3\n'''") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised by compile_single def test_bad_single_statement(self): self.assertInvalidSingle('1\n2') self.assertInvalidSingle('def f(): pass') @@ -546,8 +710,7 @@ def test_bad_single_statement(self): self.assertInvalidSingle('x = 5 # comment\nx = 6\n') self.assertInvalidSingle("c = '''\nd=1\n'''\na = 1\n\nb = 2\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'source code cannot contain null bytes' not found in b'OSError: stream did not contain valid UTF-8\n' def test_particularly_evil_undecodable(self): # Issue 24022 src = b'0000\x00\n00000000000\n\x00\n\x9e\n' @@ -556,10 +719,9 @@ def test_particularly_evil_undecodable(self): with open(fn, "wb") as fp: fp.write(src) res = script_helper.run_python_until_end(fn)[0] - self.assertIn(b"Non-UTF-8", res.err) + self.assertIn(b"source code cannot contain null bytes", res.err) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'source code cannot contain null bytes' not found in b'OSError: stream did not contain valid UTF-8\n' def test_yet_more_evil_still_undecodable(self): # Issue #25388 src = b"#\x00\n#\xfd\n" @@ -568,30 +730,25 @@ def test_yet_more_evil_still_undecodable(self): with open(fn, "wb") as fp: fp.write(src) res = script_helper.run_python_until_end(fn)[0] - self.assertIn(b"Non-UTF-8", res.err) + self.assertIn(b"source code cannot contain null bytes", res.err) @support.cpython_only + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + # Compiler frames are small + limit = 100 + # Android test devices have less memory. + crash_depth = limit * (1000 if sys.platform == "android" else 5000) + success_depth = limit def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth compile(expect_ok, '', mode) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format( - prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - compile(broken, '', mode) + broken = prefix + repeated * crash_depth + details = f"Compiling ({prefix!r} + {repeated!r} * {crash_depth})" + with self.assertRaises(RecursionError, msg=details): + compile(broken, '', mode) check_limit("a", "()") check_limit("a", ".b") @@ -601,14 +758,13 @@ def check_limit(prefix, repeated, mode="single"): # check_limit("a", " if a else a") # check_limit("if a: pass", "\nelif a: pass", mode="exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "cannot contain null" does not match "invalid syntax (, line 1)" def test_null_terminated(self): # The source code is null-terminated internally, but bytes-like # objects are accepted, which could be not terminated. - with self.assertRaisesRegex(ValueError, "cannot contain null"): + with self.assertRaisesRegex(SyntaxError, "cannot contain null"): compile("123\x00", "", "eval") - with self.assertRaisesRegex(ValueError, "cannot contain null"): + with self.assertRaisesRegex(SyntaxError, "cannot contain null"): compile(memoryview(b"123\x00"), "", "eval") code = compile(memoryview(b"123\x00")[1:-1], "", "eval") self.assertEqual(eval(code), 23) @@ -649,7 +805,6 @@ def check_same_constant(const): self.assertEqual(repr(f1()), repr(const)) check_same_constant(None) - check_same_constant(0) check_same_constant(0.0) check_same_constant(b'abc') check_same_constant('abc') @@ -664,7 +819,7 @@ def check_same_constant(const): # Merge constants in tuple or frozenset f1, f2 = lambda: "not a name", lambda: ("not a name",) f3 = lambda x: x in {("not a name",)} - self.assertIs(f1.__code__.co_consts[1], + self.assertIs(f1.__code__.co_consts[0], f2.__code__.co_consts[1][0]) self.assertIs(next(iter(f3.__code__.co_consts[1])), f2.__code__.co_consts[1]) @@ -686,16 +841,62 @@ def test_merge_code_attrs(self): self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable) + @support.cpython_only + def test_remove_unused_consts(self): + def f(): + "docstring" + if True: + return "used" + else: + return "unused" + + self.assertEqual(f.__code__.co_consts, + (f.__doc__, "used")) + + @support.cpython_only + def test_remove_unused_consts_no_docstring(self): + # the first item (None for no docstring in this case) is + # always retained. + def f(): + if True: + return "used" + else: + return "unused" + + self.assertEqual(f.__code__.co_consts, + (True, "used")) + + @support.cpython_only + def test_remove_unused_consts_extended_args(self): + N = 1000 + code = ["def f():\n"] + code.append("\ts = ''\n") + code.append("\tfor i in range(1):\n") + for i in range(N): + code.append(f"\t\tif True: s += 't{i}'\n") + code.append(f"\t\tif False: s += 'f{i}'\n") + code.append("\treturn s\n") + + code = "".join(code) + g = {} + eval(compile(code, "file.py", "exec"), g) + exec(code, g) + f = g['f'] + expected = tuple([''] + [f't{i}' for i in range(N)]) + self.assertEqual(f.__code__.co_consts, expected) + expected = "".join(expected[1:]) + self.assertEqual(expected, f()) + # Stripping unused constants is not a strict requirement for the # Python semantics, it's a more an implementation detail. @support.cpython_only - def test_strip_unused_consts(self): + def test_strip_unused_None(self): # Python 3.10rc1 appended None to co_consts when None is not used # at all. See bpo-45056. def f1(): "docstring" return 42 - self.assertEqual(f1.__code__.co_consts, ("docstring", 42)) + self.assertEqual(f1.__code__.co_consts, (f1.__doc__,)) # This is a regression test for a CPython specific peephole optimizer # implementation bug present in a few releases. It's assertion verifies @@ -715,8 +916,90 @@ def unused_code_at_end(): 'RETURN_VALUE', list(dis.get_instructions(unused_code_at_end))[-1].opname) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only + def test_docstring(self): + src = textwrap.dedent(""" + def with_docstring(): + "docstring" + + def two_strings(): + "docstring" + "not docstring" + + def with_fstring(): + f"not docstring" + + def with_const_expression(): + "also" + " not docstring" + + def multiple_const_strings(): + "not docstring " * 3 + """) + + for opt in [0, 1, 2]: + with self.subTest(opt=opt): + code = compile(src, "", "exec", optimize=opt) + ns = {} + exec(code, ns) + + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + self.assertEqual(ns['two_strings'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + self.assertIsNone(ns['two_strings'].__doc__) + self.assertIsNone(ns['with_fstring'].__doc__) + self.assertIsNone(ns['with_const_expression'].__doc__) + self.assertIsNone(ns['multiple_const_strings'].__doc__) + + @support.cpython_only + def test_docstring_interactive_mode(self): + srcs = [ + """def with_docstring(): + "docstring" + """, + """class with_docstring: + "docstring" + """, + ] + + for opt in [0, 1, 2]: + for src in srcs: + with self.subTest(opt=opt, src=src): + code = compile(textwrap.dedent(src), "", "single", optimize=opt) + ns = {} + exec(code, ns) + if opt < 2: + self.assertEqual(ns['with_docstring'].__doc__, "docstring") + else: + self.assertIsNone(ns['with_docstring'].__doc__) + + @support.cpython_only + def test_docstring_omitted(self): + # See gh-115347 + src = textwrap.dedent(""" + def f(): + "docstring1" + def h(): + "docstring2" + return 42 + + class C: + "docstring3" + pass + + return h + """) + for opt in [-1, 0, 1, 2]: + for mode in ["exec", "single"]: + with self.subTest(opt=opt, mode=mode): + code = compile(src, "", mode, optimize=opt) + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.dis(code) + self.assertNotIn('NOP', output.getvalue()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: unable to find constant -0.0 in (0.0,) def test_dont_merge_constants(self): # Issue #25843: compile() must not merge constants which are equal # but have a different type. @@ -733,7 +1016,6 @@ def check_different_constants(const1, const2): self.assertEqual(repr(f1()), repr(const1)) self.assertEqual(repr(f2()), repr(const2)) - check_different_constants(0, 0.0) check_different_constants(+0.0, -0.0) check_different_constants((0,), (0.0,)) check_different_constants('a', b'a') @@ -761,10 +1043,13 @@ def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). compile("42", FakePath("test_compile_pathlike"), "single") + # bpo-31113: Stack overflow when compile a long sequence of + # complex statements. + @support.requires_resource('cpu') def test_stack_overflow(self): - # bpo-31113: Stack overflow when compile a long sequence of - # complex statements. - compile("if a: b\n" * 200000, "", "exec") + # Android test devices have less memory. + size = 100_000 if sys.platform == "android" else 200_000 + compile("if a: b\n" * size, "", "exec") # Multiple users rely on the fact that CPython does not generate # bytecode for dead code blocks. See bpo-37500 for more context. @@ -796,12 +1081,10 @@ def unused_block_while_else(): for func in funcs: opcodes = list(dis.get_instructions(func)) self.assertLessEqual(len(opcodes), 4) - self.assertEqual('LOAD_CONST', opcodes[-2].opname) - self.assertEqual(None, opcodes[-2].argval) self.assertEqual('RETURN_VALUE', opcodes[-1].opname) + self.assertEqual(None, opcodes[-1].argval) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 3 != 8 def test_false_while_loop(self): def break_in_while(): while False: @@ -817,10 +1100,10 @@ def continue_in_while(): for func in funcs: opcodes = list(dis.get_instructions(func)) self.assertEqual(3, len(opcodes)) - self.assertEqual('LOAD_CONST', opcodes[1].opname) + self.assertEqual('RETURN_VALUE', opcodes[-1].opname) self.assertEqual(None, opcodes[1].argval) - self.assertEqual('RETURN_VALUE', opcodes[2].opname) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_consts_in_conditionals(self): def and_true(x): return True and x @@ -876,7 +1159,33 @@ def foo(x): instructions = [opcode.opname for opcode in opcodes] self.assertNotIn('LOAD_METHOD', instructions) self.assertIn('LOAD_ATTR', instructions) - self.assertIn('PRECALL', instructions) + self.assertIn('CALL', instructions) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'LOAD_SMALL_INT' not found in ['RESUME', 'LOAD_CONST', 'RETURN_VALUE'] + def test_folding_type_param(self): + get_code_fn_cls = lambda x: x.co_consts[0].co_consts[2] + get_code_type_alias = lambda x: x.co_consts[0].co_consts[3] + snippets = [ + ("def foo[T = 40 + 5](): pass", get_code_fn_cls), + ("def foo[**P = 40 + 5](): pass", get_code_fn_cls), + ("def foo[*Ts = 40 + 5](): pass", get_code_fn_cls), + ("class foo[T = 40 + 5]: pass", get_code_fn_cls), + ("class foo[**P = 40 + 5]: pass", get_code_fn_cls), + ("class foo[*Ts = 40 + 5]: pass", get_code_fn_cls), + ("type foo[T = 40 + 5] = 1", get_code_type_alias), + ("type foo[**P = 40 + 5] = 1", get_code_type_alias), + ("type foo[*Ts = 40 + 5] = 1", get_code_type_alias), + ] + for snippet, get_code in snippets: + c = compile(snippet, "", "exec") + code = get_code(c) + opcodes = list(dis.get_instructions(code)) + instructions = [opcode.opname for opcode in opcodes] + args = [opcode.oparg for opcode in opcodes] + self.assertNotIn(40, args) + self.assertNotIn(5, args) + self.assertIn('LOAD_SMALL_INT', instructions) + self.assertIn(45, args) def test_lineno_procedure_call(self): def call(): @@ -886,6 +1195,7 @@ def call(): line1 = call.__code__.co_firstlineno + 1 assert line1 not in [line for (_, _, line) in call.__code__.co_lines()] + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_after_implicit_return(self): TRUE = True # Don't use constant True or False, as compiler will remove test @@ -929,10 +1239,12 @@ def no_code2(): for func in (no_code1, no_code2): with self.subTest(func=func): + if func is no_code1 and no_code1.__doc__ is None: + continue code = func.__code__ - lines = list(code.co_lines()) - start, end, line = lines[0] + [(start, end, line)] = code.co_lines() self.assertEqual(start, 0) + self.assertEqual(end, len(code.co_code)) self.assertEqual(line, code.co_firstlineno) def get_code_lines(self, code): @@ -944,6 +1256,7 @@ def get_code_lines(self, code): last_line = line return res + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_attribute(self): def load_attr(): return ( @@ -988,6 +1301,7 @@ def aug_store_attr(): code_lines = self.get_code_lines(func.__code__) self.assertEqual(lines, code_lines) + @unittest.expectedFailure # TODO: RUSTPYTHON; + [0] def test_line_number_genexp(self): def return_genexp(): @@ -996,9 +1310,9 @@ def return_genexp(): x in y) - genexp_lines = [0, 2, 0] + genexp_lines = [0, 4, 2, 0, 4] - genexp_code = return_genexp.__code__.co_consts[1] + genexp_code = return_genexp.__code__.co_consts[0] code_lines = self.get_code_lines(genexp_code) self.assertEqual(genexp_lines, code_lines) @@ -1012,6 +1326,73 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) + def check_line_numbers(self, code, opnames=None): + # Check that all instructions whose op matches opnames + # have a line number. opnames can be a single name, or + # a sequence of names. If it is None, match all ops. + + if isinstance(opnames, str): + opnames = (opnames, ) + for inst in dis.Bytecode(code): + if opnames and inst.opname in opnames: + self.assertIsNotNone(inst.positions.lineno) + + def test_line_number_synthetic_jump_multiple_predecessors(self): + def f(): + for x in it: + try: + if C1: + yield 2 + except OSError: + pass + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + X = 4 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_line_number_synthetic_jump_multiple_predecessors_more_nested(self): + def f(): + for x in it: + try: + X = 3 + except OSError: + try: + if C3: + if C4: + X = 4 + except OSError: + try: + if C3: + if C4: + X = 5 + except OSError: + pass + return 42 + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + + def test_lineno_of_backward_jump_conditional_in_loop(self): + # Issue gh-107901 + def f(): + for i in x: + if y: + pass + + self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') + def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles # a portion of the dictionary literal when the loop that iterates over the items @@ -1061,12 +1442,109 @@ def while_not_chained(a, b, c): for instr in dis.Bytecode(while_not_chained): self.assertNotEqual(instr.opname, "EXTENDED_ARG") + @support.cpython_only + def test_uses_slice_instructions(self): + + def check_op_count(func, op, expected): + actual = 0 + for instr in dis.Bytecode(func): + if instr.opname == op: + actual += 1 + self.assertEqual(actual, expected) + + def check_consts(func, typ, expected): + expected = set([repr(x) for x in expected]) + all_consts = set() + consts = func.__code__.co_consts + for instr in dis.Bytecode(func): + if instr.opname == "LOAD_CONST" and isinstance(consts[instr.oparg], typ): + all_consts.add(repr(consts[instr.oparg])) + self.assertEqual(all_consts, expected) + + def load(): + return x[a:b] + x [a:] + x[:b] + x[:] + + check_op_count(load, "BINARY_SLICE", 3) + check_op_count(load, "BUILD_SLICE", 0) + check_consts(load, slice, [slice(None, None, None)]) + check_op_count(load, "BINARY_OP", 4) + + def store(): + x[a:b] = y + x [a:] = y + x[:b] = y + x[:] = y + + check_op_count(store, "STORE_SLICE", 3) + check_op_count(store, "BUILD_SLICE", 0) + check_op_count(store, "STORE_SUBSCR", 1) + check_consts(store, slice, [slice(None, None, None)]) + + def long_slice(): + return x[a:b:c] + + check_op_count(long_slice, "BUILD_SLICE", 1) + check_op_count(long_slice, "BINARY_SLICE", 0) + check_consts(long_slice, slice, []) + check_op_count(long_slice, "BINARY_OP", 1) + + def aug(): + x[a:b] += y + + check_op_count(aug, "BINARY_SLICE", 1) + check_op_count(aug, "STORE_SLICE", 1) + check_op_count(aug, "BUILD_SLICE", 0) + check_op_count(aug, "BINARY_OP", 1) + check_op_count(aug, "STORE_SUBSCR", 0) + check_consts(aug, slice, []) + + def aug_const(): + x[1:2] += y + + check_op_count(aug_const, "BINARY_SLICE", 0) + check_op_count(aug_const, "STORE_SLICE", 0) + check_op_count(aug_const, "BINARY_OP", 2) + check_op_count(aug_const, "STORE_SUBSCR", 1) + check_consts(aug_const, slice, [slice(1, 2)]) + + def compound_const_slice(): + x[1:2:3, 4:5:6] = y + + check_op_count(compound_const_slice, "BINARY_SLICE", 0) + check_op_count(compound_const_slice, "BUILD_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SLICE", 0) + check_op_count(compound_const_slice, "STORE_SUBSCR", 1) + check_consts(compound_const_slice, slice, []) + check_consts(compound_const_slice, tuple, [(slice(1, 2, 3), slice(4, 5, 6))]) + + def mutable_slice(): + x[[]:] = y + + check_consts(mutable_slice, slice, {}) + + def different_but_equal(): + x[:0] = y + x[:0.0] = y + x[:False] = y + x[:None] = y + + check_consts( + different_but_equal, + slice, + [ + slice(None, 0, None), + slice(None, 0.0, None), + slice(None, False, None), + slice(None, None, None) + ] + ) + def test_compare_positions(self): - for opname, op in [ - ("COMPARE_OP", "<"), - ("COMPARE_OP", "<="), - ("COMPARE_OP", ">"), - ("COMPARE_OP", ">="), + for opname_prefix, op in [ + ("COMPARE_", "<"), + ("COMPARE_", "<="), + ("COMPARE_", ">"), + ("COMPARE_", ">="), ("CONTAINS_OP", "in"), ("CONTAINS_OP", "not in"), ("IS_OP", "is"), @@ -1081,11 +1559,313 @@ def test_compare_positions(self): actual_positions = [ instruction.positions for instruction in dis.get_instructions(code) - if instruction.opname == opname + if instruction.opname.startswith(opname_prefix) ] with self.subTest(source): self.assertEqual(actual_positions, expected_positions) + def test_if_expression_expression_empty_block(self): + # See regression in gh-99708 + exprs = [ + "assert (False if 1 else True)", + "def f():\n\tif not (False if 1 else True): raise AssertionError", + "def f():\n\tif not (False if 1 else True): return 12", + ] + for expr in exprs: + with self.subTest(expr=expr): + compile(expr, "", "exec") + + def test_multi_line_lambda_as_argument(self): + # See gh-101928 + code = textwrap.dedent(""" + def foo(param, lambda_exp): + pass + + foo(param=0, + lambda_exp=lambda: + 1) + """) + compile(code, "", "exec") + + def test_apply_static_swaps(self): + def f(x, y): + a, a = x, y + return a + self.assertEqual(f("x", "y"), "y") + + def test_apply_static_swaps_2(self): + def f(x, y, z): + a, b, a = x, y, z + return a + self.assertEqual(f("x", "y", "z"), "z") + + def test_apply_static_swaps_3(self): + def f(x, y, z): + a, a, b = x, y, z + return a + self.assertEqual(f("x", "y", "z"), "y") + + def test_variable_dependent(self): + # gh-104635: Since the value of b is dependent on the value of a + # the first STORE_FAST for a should not be skipped. (e.g POP_TOP). + # This test case is added to prevent potential regression from aggressive optimization. + def f(): + a = 42; b = a + 54; a = 54 + return a, b + self.assertEqual(f(), (54, 96)) + + def test_duplicated_small_exit_block(self): + # See gh-109627 + def f(): + while element and something: + try: + return something + except: + pass + + def test_cold_block_moved_to_end(self): + # See gh-109719 + def f(): + while name: + try: + break + except: + pass + else: + 1 if 1 else 1 + + def test_remove_empty_basic_block_with_jump_target_label(self): + # See gh-109823 + def f(x): + while x: + 0 if 1 else 0 + + def test_remove_redundant_nop_edge_case(self): + # See gh-109889 + def f(): + a if (1 if b else c) else d + + def test_global_declaration_in_except_used_in_else(self): + # See gh-111123 + code = textwrap.dedent("""\ + def f(): + try: + pass + %s Exception: + global a + else: + print(a) + """) + + g, l = {'a': 5}, {} + for kw in ("except", "except*"): + exec(code % kw, g, l); + + def test_regression_gh_120225(self): + async def name_4(): + match b'': + case True: + pass + case name_5 if f'e': + {name_3: name_4 async for name_2 in name_5} + case []: + pass + [[]] + + def test_globals_dict_subclass(self): + # gh-132386 + class WeirdDict(dict): + pass + + ns = {} + exec('def foo(): return a', WeirdDict(), ns) + + self.assertRaises(NameError, ns['foo']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + [3, 5, 3, 5] + def test_compile_warnings(self): + # Each invocation of compile() emits compiler warnings, even if they + # have the same message and line number. + source = textwrap.dedent(r""" + # tokenizer + 1or 0 # line 3 + # code generator + 1 is 1 # line 5 + """) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("default") + for i in range(2): + # Even if compile() is at the same line. + compile(source, '', 'exec') + + self.assertEqual([wm.lineno for wm in caught], [3, 5] * 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + [5, 9] + def test_compile_warning_in_finally(self): + # Ensure that warnings inside finally blocks are + # only emitted once despite the block being + # compiled twice (for normal execution and for + # exception handling). + source = textwrap.dedent(""" + try: + pass + finally: + 1 is 1 # line 5 + try: + pass + finally: # nested + 1 is 1 # line 9 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + compile(source, '', 'exec') + + self.assertEqual(sorted(wm.lineno for wm in caught), [5, 9]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + # Other code path is used for "try" with "except*". + source = textwrap.dedent(""" + try: + pass + except *Exception: + pass + finally: + 1 is 1 # line 7 + try: + pass + except *Exception: + pass + finally: # nested + 1 is 1 # line 13 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + compile(source, '', 'exec') + + self.assertEqual(sorted(wm.lineno for wm in caught), [7, 13]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.subTests('src', [ + textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + break + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + continue + """), + ]) + def test_pep_765_warnings(self, src): + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(src, '', 'exec') + with warnings.catch_warnings(): + warnings.simplefilter("error") + tree = ast.parse(src) + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(tree, '', 'exec') + + @support.subTests('src', [ + textwrap.dedent(""" + try: + pass + finally: + def f(): + return 42 + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + break + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + continue + """), + ]) + def test_pep_765_no_warnings(self, src): + with warnings.catch_warnings(): + warnings.simplefilter("error") + compile(src, '', 'exec') + + +class TestBooleanExpression(unittest.TestCase): + class Value: + def __init__(self): + self.called = 0 + + def __bool__(self): + self.called += 1 + return self.value + + class Yes(Value): + value = True + + class No(Value): + value = False + + def test_short_circuit_and(self): + v = [self.Yes(), self.No(), self.Yes()] + res = v[0] and v[1] and v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_short_circuit_or(self): + v = [self.No(), self.Yes(), self.No()] + res = v[0] or v[1] or v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_compound(self): + # See gh-124285 + v = [self.No(), self.Yes(), self.Yes(), self.Yes()] + res = v[0] and v[1] or v[2] or v[3] + self.assertIs(res, v[2]) + self.assertEqual([e.called for e in v], [1, 0, 1, 0]) + + v = [self.No(), self.No(), self.Yes(), self.Yes(), self.No()] + res = v[0] or v[1] and v[2] or v[3] or v[4] + self.assertIs(res, v[3]) + self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + + def test_exception(self): + # See gh-137288 + class Foo: + def __bool__(self): + raise NotImplementedError() + + a = Foo() + b = Foo() + + with self.assertRaises(NotImplementedError): + bool(a) + + with self.assertRaises(NotImplementedError): + c = a or b @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): @@ -1104,7 +1884,7 @@ def check_positions_against_ast(self, snippet): class SourceOffsetVisitor(ast.NodeVisitor): def generic_visit(self, node): super().generic_visit(node) - if not isinstance(node, ast.expr) and not isinstance(node, ast.stmt): + if not isinstance(node, (ast.expr, ast.stmt, ast.pattern)): return lines.add(node.lineno) end_lines.add(node.end_lineno) @@ -1133,8 +1913,8 @@ def generic_visit(self, node): def assertOpcodeSourcePositionIs(self, code, opcode, line, end_line, column, end_column, occurrence=1): - for instr, position in zip( - dis.Bytecode(code, show_caches=True), code.co_positions(), strict=True + for instr, position in instructions_with_positions( + dis.Bytecode(code), code.co_positions() ): if instr.opname == opcode: occurrence -= 1 @@ -1172,15 +1952,303 @@ def test_compiles_to_extended_op_arg(self): column=2, end_column=9, occurrence=2) def test_multiline_expression(self): - snippet = """\ -f( - 1, 2, 3, 4 -) -""" + snippet = textwrap.dedent("""\ + f( + 1, 2, 3, 4 + ) + """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', line=1, end_line=3, column=0, end_column=1) + @requires_specialization + def test_multiline_boolean_expression(self): + snippet = textwrap.dedent("""\ + if (a or + (b and not c) or + not ( + d > 0)): + x = 42 + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + # jump if a is true: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', + line=1, end_line=1, column=4, end_column=5, occurrence=1) + # jump if b is false: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', + line=2, end_line=2, column=5, end_column=6, occurrence=1) + # jump if c is false: + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', + line=2, end_line=2, column=15, end_column=16, occurrence=2) + # compare d and 0 + self.assertOpcodeSourcePositionIs(compiled_code, 'COMPARE_OP', + line=4, end_line=4, column=8, end_column=13, occurrence=1) + # jump if comparison it True + self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', + line=4, end_line=4, column=8, end_column=13, occurrence=2) + + @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") + def test_multiline_assert(self): + snippet = textwrap.dedent("""\ + assert (a > 0 and + bb > 0 and + ccc == 1000000), "error msg" + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_COMMON_CONSTANT', + line=1, end_line=3, column=0, end_column=36, occurrence=1) + # The "error msg": + self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', + line=3, end_line=3, column=25, end_column=36, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', + line=1, end_line=3, column=0, end_column=36, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS', + line=1, end_line=3, column=8, end_column=22, occurrence=1) + + def test_multiline_generator_expression(self): + snippet = textwrap.dedent("""\ + ((x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=4, end_line=4, column=7, end_column=14, occurrence=1) + + def test_multiline_async_generator_expression(self): + snippet = textwrap.dedent("""\ + ((x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)) + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + compiled_code = compiled_code.co_consts[0] + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', + line=1, end_line=2, column=1, end_column=8, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=1, end_line=6, column=0, end_column=32, occurrence=1) + + def test_multiline_list_comprehension(self): + snippet = textwrap.dedent("""\ + [(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + + def test_multiline_async_list_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + [(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)] + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_multiline_set_comprehension(self): + snippet = textwrap.dedent("""\ + {(x, + 2*x) + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=8, occurrence=1) + + def test_multiline_async_set_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + {(x, + 2*x) + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=12, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_multiline_dict_comprehension(self): + snippet = textwrap.dedent("""\ + {x: + 2*x + for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', + line=1, end_line=2, column=1, end_column=7, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=1, end_line=2, column=1, end_column=7, occurrence=1) + + def test_multiline_async_dict_comprehension(self): + snippet = textwrap.dedent("""\ + async def f(): + {x: + 2*x + async for x + in [1,2,3] if (x > 0 + and x < 100 + and x != 50)} + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + g = {} + eval(compiled_code, g) + compiled_code = g['f'].__code__ + self.assertIsInstance(compiled_code, types.CodeType) + self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', + line=2, end_line=3, column=5, end_column=11, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', + line=2, end_line=3, column=5, end_column=11, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', + line=2, end_line=7, column=4, end_column=36, occurrence=1) + + def test_matchcase_sequence(self): + snippet = textwrap.dedent("""\ + match x: + case a, b: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_SEQUENCE', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=13, occurrence=2) + + def test_matchcase_sequence_wildcard(self): + snippet = textwrap.dedent("""\ + match x: + case a, *b, c: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_EX', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=2) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=17, occurrence=3) + + def test_matchcase_mapping(self): + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b}: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_KEYS', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=26, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=26, occurrence=2) + + def test_matchcase_mapping_wildcard(self): + snippet = textwrap.dedent("""\ + match x: + case {"a" : a, "b": b, **c}: + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_KEYS', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=31, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=31, occurrence=2) + + def test_matchcase_class(self): + snippet = textwrap.dedent("""\ + match x: + case C(a, b): + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_SEQUENCE', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=16, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', + line=2, end_line=2, column=9, end_column=16, occurrence=2) + + def test_matchcase_or(self): + snippet = textwrap.dedent("""\ + match x: + case C(1) | C(2): + pass + """) + compiled_code, _ = self.check_positions_against_ast(snippet) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=9, end_column=13, occurrence=1) + self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', + line=2, end_line=2, column=16, end_column=20, occurrence=2) + def test_very_long_line_end_offset(self): # Make sure we get the correct column offset for offsets # too large to store in a byte. @@ -1195,16 +2263,16 @@ def test_complex_single_line_expression(self): snippet = "a - b @ (c * x['key'] + 23)" compiled_code, _ = self.check_positions_against_ast(snippet) - self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_SUBSCR', + self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', line=1, end_line=1, column=13, end_column=21) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=9, end_column=21, occurrence=1) + line=1, end_line=1, column=9, end_column=21, occurrence=2) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=9, end_column=26, occurrence=2) + line=1, end_line=1, column=9, end_column=26, occurrence=3) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=4, end_column=27, occurrence=3) + line=1, end_line=1, column=4, end_column=27, occurrence=4) self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', - line=1, end_line=1, column=0, end_column=27, occurrence=4) + line=1, end_line=1, column=0, end_column=27, occurrence=5) def test_multiline_assert_rewritten_as_method_call(self): # GH-94694: Don't crash if pytest rewrites a multiline assert as a @@ -1254,7 +2322,9 @@ def f(): with self.subTest(body): namespace = {} source = textwrap.dedent(source_template.format(body)) - exec(source, namespace) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', SyntaxWarning) + exec(source, namespace) code = namespace["f"].__code__ self.assertOpcodeSourcePositionIs( code, @@ -1300,7 +2370,7 @@ def test_method_call(self): source = "(\n lhs \n . \n rhs \n )()" code = compile(source, "", "exec") self.assertOpcodeSourcePositionIs( - code, "LOAD_METHOD", line=4, end_line=4, column=5, end_column=8 + code, "LOAD_ATTR", line=4, end_line=4, column=5, end_column=8 ) self.assertOpcodeSourcePositionIs( code, "CALL", line=4, end_line=5, column=5, end_column=10 @@ -1331,9 +2401,6 @@ def test_column_offset_deduplication(self): for source in [ "lambda: a", "(a for b in c)", - "[a for b in c]", - "{a for b in c}", - "{a: b for c in d}", ]: with self.subTest(source): code = compile(f"{source}, {source}", "", "eval") @@ -1346,6 +2413,141 @@ def test_column_offset_deduplication(self): list(code.co_consts[1].co_positions()), ) + def test_load_super_attr(self): + source = "class C:\n def __init__(self):\n super().__init__()" + for const in compile(source, "", "exec").co_consts[0].co_consts: + if isinstance(const, types.CodeType): + code = const + break + self.assertOpcodeSourcePositionIs( + code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 + ) + + def test_lambda_return_position(self): + snippets = [ + "f = lambda: x", + "f = lambda: 42", + "f = lambda: 1 + 2", + "f = lambda: a + b", + ] + for snippet in snippets: + with self.subTest(snippet=snippet): + lamb = run_code(snippet)["f"] + positions = lamb.__code__.co_positions() + # assert that all positions are within the lambda + for i, pos in enumerate(positions): + with self.subTest(i=i, pos=pos): + start_line, end_line, start_col, end_col = pos + if i == 0 and start_col == end_col == 0: + # ignore the RESUME in the beginning + continue + self.assertEqual(start_line, 1) + self.assertEqual(end_line, 1) + code_start = snippet.find(":") + 2 + code_end = len(snippet) + self.assertGreaterEqual(start_col, code_start) + self.assertLessEqual(end_col, code_end) + self.assertGreaterEqual(end_col, start_col) + self.assertLessEqual(end_col, code_end) + + def test_return_in_with_positions(self): + # See gh-98442 + def f(): + with xyz: + 1 + 2 + 3 + 4 + return R + + # All instructions should have locations on a single line + for instr in dis.get_instructions(f): + start_line, end_line, _, _ = instr.positions + self.assertEqual(start_line, end_line) + + # Expect four `LOAD_CONST None` instructions: + # three for the no-exception __exit__ call, and one for the return. + # They should all have the locations of the context manager ('xyz'). + + load_none = [instr for instr in dis.get_instructions(f) if + instr.opname == 'LOAD_CONST' and instr.argval is None] + return_value = [instr for instr in dis.get_instructions(f) if + instr.opname == 'RETURN_VALUE'] + + self.assertEqual(len(load_none), 4) + self.assertEqual(len(return_value), 2) + for instr in load_none + return_value: + start_line, end_line, start_col, end_col = instr.positions + self.assertEqual(start_line, f.__code__.co_firstlineno + 1) + self.assertEqual(end_line, f.__code__.co_firstlineno + 1) + self.assertEqual(start_col, 17) + self.assertEqual(end_col, 20) + + +class TestStaticAttributes(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_basic(self): + class C: + def f(self): + self.a = self.b = 42 + # read fields are not included + self.f() + self.arr[3] + + self.assertIsInstance(C.__static_attributes__, tuple) + self.assertEqual(sorted(C.__static_attributes__), ['a', 'b']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_nested_function(self): + class C: + def f(self): + self.x = 1 + self.y = 2 + self.x = 3 # check deduplication + + def g(self, obj): + self.y = 4 + self.z = 5 + + def h(self, a): + self.u = 6 + self.v = 7 + + obj.self = 8 + + self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_nested_class(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D: + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__' + def test_subclass(self): + class C: + def f(self): + self.x = 42 + self.y = 42 + + class D(C): + def g(self): + self.y = 42 + self.z = 42 + + self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) + self.assertEqual(sorted(D.__static_attributes__), ['y', 'z']) + class TestExpressionStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object @@ -1361,48 +2563,41 @@ def check_stack_size(self, code): max_size = math.ceil(math.log(len(code.co_code))) self.assertLessEqual(code.co_stacksize, max_size) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_and(self): self.check_stack_size("x and " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_or(self): self.check_stack_size("x or " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_and_or(self): self.check_stack_size("x and x or " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_chained_comparison(self): self.check_stack_size("x < " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_if_else(self): self.check_stack_size("x if x else " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_binop(self): self.check_stack_size("x + " * self.N + "x") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_list(self): self.check_stack_size("[" + "x, " * self.N + "x]") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_tuple(self): self.check_stack_size("(" + "x, " * self.N + "x)") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 101 not less than or equal to 6 def test_set(self): self.check_stack_size("{" + "x, " * self.N + "x}") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 202 not less than or equal to 7 def test_dict(self): self.check_stack_size("{" + "x:x, " * self.N + "x:x}") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 102 not less than or equal to 6 def test_func_args(self): self.check_stack_size("f(" + "x, " * self.N + ")") @@ -1410,6 +2605,7 @@ def test_func_kwargs(self): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("f(" + ", ".join(kwargs) + ")") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 102 not less than or equal to 6 def test_meth_args(self): self.check_stack_size("o.m(" + "x, " * self.N + ")") @@ -1417,8 +2613,6 @@ def test_meth_kwargs(self): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("o.m(" + ", ".join(kwargs) + ")") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_func_and(self): code = "def f(x):\n" code += " x and x\n" * self.N @@ -1448,7 +2642,9 @@ def compile_snippet(i): script = """def func():\n""" + i * snippet if async_: script = "async " + script - code = compile(script, "' + self._run_check(s, [ + ("starttag", "script", []), + ("data", content), + ("endtag", "script"), + ]) + + @support.subTests('content', [ + 'a::before { content: ""; }', + 'a::before { content: "¬-an-entity-ref;"; }', + 'a::before { content: ""; }', + 'a::before { content: "\u2603"; }', + ]) + def test_style_content(self, content): + s = f'' + self._run_check(s, [("starttag", "style", []), + ("data", content), + ("endtag", "style")]) + + @support.subTests('tag', ['title', 'textarea']) + def test_rcdata_content(self, tag): + source = f"<{tag}>{SAMPLE_RCDATA}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", SAMPLE_RCDATA), + ("endtag", tag), + ]) + source = f"<{tag}>&" + self._run_check(source, [ + ("starttag", tag, []), + ('entityref', 'amp'), + ("endtag", tag), + ]) + + @support.subTests('tag', + ['style', 'xmp', 'iframe', 'noembed', 'noframes', 'script']) + def test_rawtext_content(self, tag): + source = f"<{tag}>{SAMPLE_RAWTEXT}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", SAMPLE_RAWTEXT), + ("endtag", tag), + ]) + + def test_noscript_content(self): + source = f"" + # scripting=False -- normal mode + self._run_check(source, [ + ('starttag', 'noscript', []), + ('comment', ' not a comment '), + ('starttag', 'not', [('a', 'start tag')]), + ('unknown decl', 'CDATA[not a cdata'), + ('comment', 'not a bogus comment'), + ('endtag', 'not'), + ('data', '☃'), + ('entityref', 'amp'), + ('charref', '9786'), + ('endtag', 'noscript'), + ]) + # scripting=True -- RAWTEXT mode + self._run_check(source, [ + ("starttag", "noscript", []), + ("data", SAMPLE_RAWTEXT), + ("endtag", "noscript"), + ], collector=EventCollector(scripting=True)) + + def test_plaintext_content(self): + content = SAMPLE_RAWTEXT + '' # not closing + source = f"{content}" + self._run_check(source, [ + ("starttag", "plaintext", []), + ("data", content), + ]) + + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', + 'script/', 'script foo=bar', 'script foo=">"']) + def test_script_closing_tag(self, endtag): # see issue #13358 # make sure that HTMLParser calls handle_data only once for each CDATA. - # The normal event collector normalizes the events in get_events, - # so we override it to return the original list of events. - class Collector(EventCollector): - def get_events(self): - return self.events - content = """<!-- not a comment --> &not-an-entity-ref; <a href="" /> </p><p> <span></span></style> '</script' + '>'""" - for element in [' script', 'script ', ' script ', - '\nscript', 'script\n', '\nscript\n']: - element_lower = element.lower().strip() - s = '<script>{content}</{element}>'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)], - collector=Collector(convert_charrefs=False)) + s = f'<ScrIPt>{content}</{endtag}>' + self._run_check(s, [("starttag", "script", []), + ("data", content), + ("endtag", "script")], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tag', [ + 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', + 'textarea', 'title', 'noscript', + ]) + def test_closing_tag(self, tag): + for endtag in [tag, tag.upper(), f'{tag} ', f'{tag}\n', + f'{tag}/', f'{tag} foo=bar', f'{tag} foo=">"']: + content = "<!-- not a comment --><i>Spam</i>" + s = f'<{tag.upper()}>{content}</{endtag}>' + self._run_check(s, [ + ("starttag", tag, []), + ('data', content), + ("endtag", tag), + ], collector=EventCollectorNoNormalize(convert_charrefs=False, scripting=True)) + + @support.subTests('tag', [ + 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', + 'textarea', 'title', 'noscript', + ]) + def test_invalid_closing_tag(self, tag): + content = ( + f'< /{tag}>' + f'</ {tag}>' + f'</{tag}x>' + f'</{tag}\v>' + f'</{tag}\xa0>' + ) + source = f"<{tag}>{content}</{tag}>" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ("endtag", tag), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + + @support.subTests('tag,endtag', [ + ('title', 'tıtle'), + ('style', 'ſtyle'), + ('style', 'ſtyle'), + ('style', 'style'), + ('iframe', 'ıframe'), + ('noframes', 'noframeſ'), + ('noscript', 'noſcript'), + ('noscript', 'noscrıpt'), + ('script', 'ſcript'), + ('script', 'scrıpt'), + ]) + def test_invalid_nonascii_closing_tag(self, tag, endtag): + content = f"<br></{endtag}>" + source = f"<{tag}>{content}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + source = f"<{tag}>{content}</{tag}>" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ("endtag", tag), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('</', False), + ('</s', False), + ('</script', False), + ('</script ', True), + ('</script foo=bar', True), + ('</script foo=">', True), + ]) + def test_eof_in_script(self, tail, end): + content = "a = 123" + s = f'<ScrIPt>{content}{tail}' + self._run_check(s, [("starttag", "script", []), + ("data", content if end else content + tail)], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('</', False), + ('</t', False), + ('</title', False), + ('</title ', True), + ('</title foo=bar', True), + ('</title foo=">', True), + ]) + def test_eof_in_title(self, tail, end): + s = f'<TitLe>Egg &amp; Spam{tail}' + self._run_check(s, [("starttag", "title", []), + ("data", "Egg & Spam" + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "title", []), + ('data', 'Egg '), + ('entityref', 'amp'), + ('data', ' Spam' + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=False)) def test_comments(self): html = ("<!-- I'm a valid comment -->" '<!--me too!-->' '<!------>' + '<!----->' '<!---->' + # abrupt-closing-of-empty-comment + '<!--->' + '<!-->' '<!----I have many hyphens---->' '<!-- I have a > in the middle -->' - '<!-- and I have -- in the middle! -->') + '<!-- and I have -- in the middle! -->' + '<!--incorrectly-closed-comment--!>' + '<!----!>' + '<!----!-->' + '<!---- >-->' + '<!---!>-->' + '<!--!>-->' + # nested-comment + '<!-- <!-- nested --> -->' + '<!--<!-->' + '<!--<!--!>' + ) expected = [('comment', " I'm a valid comment "), ('comment', 'me too!'), ('comment', '--'), + ('comment', '-'), + ('comment', ''), + ('comment', ''), ('comment', ''), ('comment', '--I have many hyphens--'), ('comment', ' I have a > in the middle '), - ('comment', ' and I have -- in the middle! ')] + ('comment', ' and I have -- in the middle! '), + ('comment', 'incorrectly-closed-comment'), + ('comment', ''), + ('comment', '--!'), + ('comment', '-- >'), + ('comment', '-!>'), + ('comment', '!>'), + ('comment', ' <!-- nested '), ('data', ' -->'), + ('comment', '<!'), + ('comment', '<!'), + ] self._run_check(html, expected) def test_condcoms(self): @@ -346,18 +630,16 @@ def test_convert_charrefs(self): collector = lambda: EventCollectorCharrefs() self.assertTrue(collector().convert_charrefs) charrefs = ['&quot;', '&#34;', '&#x22;', '&quot', '&#34', '&#x22'] - # check charrefs in the middle of the text/attributes - expected = [('starttag', 'a', [('href', 'foo"zar')]), - ('data', 'a"z'), ('endtag', 'a')] + # check charrefs in the middle of the text + expected = [('starttag', 'a', []), ('data', 'a"z'), ('endtag', 'a')] for charref in charrefs: - self._run_check('<a href="foo{0}zar">a{0}z</a>'.format(charref), + self._run_check('<a>a{0}z</a>'.format(charref), expected, collector=collector()) - # check charrefs at the beginning/end of the text/attributes - expected = [('data', '"'), - ('starttag', 'a', [('x', '"'), ('y', '"X'), ('z', 'X"')]), + # check charrefs at the beginning/end of the text + expected = [('data', '"'), ('starttag', 'a', []), ('data', '"'), ('endtag', 'a'), ('data', '"')] for charref in charrefs: - self._run_check('{0}<a x="{0}" y="{0}X" z="X{0}">' + self._run_check('{0}<a>' '{0}</a>{0}'.format(charref), expected, collector=collector()) # check charrefs in <script>/<style> elements @@ -380,6 +662,35 @@ def test_convert_charrefs(self): self._run_check('no charrefs here', [('data', 'no charrefs here')], collector=collector()) + def test_convert_charrefs_in_attribute_values(self): + # default value for convert_charrefs is now True + collector = lambda: EventCollectorCharrefs() + self.assertTrue(collector().convert_charrefs) + + # always unescape terminated entity refs, numeric and hex char refs: + # - regardless whether they are at start, middle, end of attribute + # - or followed by alphanumeric, non-alphanumeric, or equals char + charrefs = ['&cent;', '&#xa2;', '&#xa2', '&#162;', '&#162'] + expected = [('starttag', 'a', + [('x', '¢'), ('x', 'z¢'), ('x', '¢z'), + ('x', 'z¢z'), ('x', '¢ z'), ('x', '¢=z')]), + ('endtag', 'a')] + for charref in charrefs: + self._run_check('<a x="{0}" x="z{0}" x="{0}z" ' + ' x="z{0}z" x="{0} z" x="{0}=z"></a>' + .format(charref), expected, collector=collector()) + + # only unescape unterminated entity matches if they are not followed by + # an alphanumeric or an equals sign + charref = '&cent' + expected = [('starttag', 'a', + [('x', '¢'), ('x', 'z¢'), ('x', '&centz'), + ('x', 'z&centz'), ('x', '¢ z'), ('x', '&cent=z')]), + ('endtag', 'a')] + self._run_check('<a x="{0}" x="z{0}" x="{0}z" ' + ' x="z{0}z" x="{0} z" x="{0}=z"></a>' + .format(charref), expected, collector=collector()) + # the remaining tests were for the "tolerant" parser (which is now # the default), and check various kind of broken markup def test_tolerant_parsing(self): @@ -391,28 +702,34 @@ def test_tolerant_parsing(self): ('data', '<'), ('starttag', 'bc<', [('a', None)]), ('endtag', 'html'), - ('data', '\n<img src="URL>'), - ('comment', '/img'), - ('endtag', 'html<')]) + ('data', '\n')]) def test_starttag_junk_chars(self): + self._run_check("<", [('data', '<')]) + self._run_check("<>", [('data', '<>')]) + self._run_check("< >", [('data', '< >')]) + self._run_check("< ", [('data', '< ')]) self._run_check("</>", []) + self._run_check("<$>", [('data', '<$>')]) self._run_check("</$>", [('comment', '$')]) self._run_check("</", [('data', '</')]) - self._run_check("</a", [('data', '</a')]) + self._run_check("</a", []) + self._run_check("</ a>", [('comment', ' a')]) + self._run_check("</ a", [('comment', ' a')]) self._run_check("<a<a>", [('starttag', 'a<a', [])]) self._run_check("</a<a>", [('endtag', 'a<a')]) - self._run_check("<!", [('data', '<!')]) - self._run_check("<a", [('data', '<a')]) - self._run_check("<a foo='bar'", [('data', "<a foo='bar'")]) - self._run_check("<a foo='bar", [('data', "<a foo='bar")]) - self._run_check("<a foo='>'", [('data', "<a foo='>'")]) - self._run_check("<a foo='>", [('data', "<a foo='>")]) + self._run_check("<!", [('comment', '')]) + self._run_check("<a", []) + self._run_check("<a foo='bar'", []) + self._run_check("<a foo='bar", []) + self._run_check("<a foo='>'", []) + self._run_check("<a foo='>", []) self._run_check("<a$>", [('starttag', 'a$', [])]) self._run_check("<a$b>", [('starttag', 'a$b', [])]) self._run_check("<a$b/>", [('startendtag', 'a$b', [])]) self._run_check("<a$b >", [('starttag', 'a$b', [])]) self._run_check("<a$b />", [('startendtag', 'a$b', [])]) + self._run_check("</a$b>", [('endtag', 'a$b')]) def test_slashes_in_starttag(self): self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])]) @@ -445,6 +762,10 @@ def test_slashes_in_starttag(self): ] self._run_check(html, expected) + def test_slashes_in_endtag(self): + self._run_check('</a/>', [('endtag', 'a')]) + self._run_check('</a foo="var"/>', [('endtag', 'a')]) + def test_declaration_junk_chars(self): self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')]) @@ -479,15 +800,11 @@ def test_invalid_end_tags(self): self._run_check(html, expected) def test_broken_invalid_end_tag(self): - # This is technically wrong (the "> shouldn't be included in the 'data') - # but is probably not worth fixing it (in addition to all the cases of - # the previous test, it would require a full attribute parsing). - # see #13993 html = '<b>This</b attr=">"> confuses the parser' expected = [('starttag', 'b', []), ('data', 'This'), ('endtag', 'b'), - ('data', '"> confuses the parser')] + ('data', ' confuses the parser')] self._run_check(html, expected) def test_correct_detection_of_start_tags(self): @@ -523,69 +840,165 @@ def test_correct_detection_of_start_tags(self): ] self._run_check(html, expected) - def test_EOF_in_charref(self): - # see #17802 - # This test checks that the UnboundLocalError reported in the issue - # is not raised, however I'm not sure the returned values are correct. - # Maybe HTMLParser should use self.unescape for these + def test_eof_in_comments(self): data = [ - ('a&', [('data', 'a&')]), - ('a&b', [('data', 'ab')]), - ('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]), - ('a&b;', [('data', 'a'), ('entityref', 'b')]), + ('<!--', [('comment', '')]), + ('<!---', [('comment', '')]), + ('<!----', [('comment', '')]), + ('<!-----', [('comment', '-')]), + ('<!------', [('comment', '--')]), + ('<!----!', [('comment', '')]), + ('<!---!', [('comment', '-!')]), + ('<!---!>', [('comment', '-!>')]), + ('<!--foo', [('comment', 'foo')]), + ('<!--foo-', [('comment', 'foo')]), + ('<!--foo--', [('comment', 'foo')]), + ('<!--foo--!', [('comment', 'foo')]), + ('<!--<!--', [('comment', '<!')]), + ('<!--<!--!', [('comment', '<!')]), ] for html, expected in data: self._run_check(html, expected) - def test_broken_comments(self): - html = ('<! not really a comment >' + def test_eof_in_declarations(self): + data = [ + ('<!', [('comment', '')]), + ('<!-', [('comment', '-')]), + ('<![', [('comment', '[')]), + ('<!DOCTYPE', [('decl', 'DOCTYPE')]), + ('<!DOCTYPE ', [('decl', 'DOCTYPE ')]), + ('<!DOCTYPE html', [('decl', 'DOCTYPE html')]), + ('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]), + ('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]), + ('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]), + ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo', + [('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]), + ] + for html, expected in data: + self._run_check(html, expected) + + @support.subTests('content', ['', 'x', 'x]', 'x]]']) + def test_eof_in_cdata(self, content): + self._run_check('<![CDATA[' + content, + [('unknown decl', 'CDATA[' + content)]) + self._run_check('<![CDATA[' + content, + [('comment', '[CDATA[' + content)], + collector=EventCollector(autocdata=True)) + self._run_check('<svg><text y="100"><![CDATA[' + content, + [('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content)]) + + def test_bogus_comments(self): + html = ('<!ELEMENT br EMPTY>' + '<! not really a comment >' '<! not a comment either -->' '<! -- close enough -->' '<!><!<-- this was an empty comment>' - '<!!! another bogus comment !!!>') + '<!!! another bogus comment !!!>' + # see #32876 + '<![with square brackets]!>' + '<![\nmultiline\nbogusness\n]!>' + '<![more brackets]-[and a hyphen]!>' + '<![cdata[should be uppercase]]>' + '<![CDATA [whitespaces are not ignored]]>' + '<![CDATA]]>' # required '[' after CDATA + ) expected = [ + ('comment', 'ELEMENT br EMPTY'), ('comment', ' not really a comment '), ('comment', ' not a comment either --'), ('comment', ' -- close enough --'), ('comment', ''), ('comment', '<-- this was an empty comment'), ('comment', '!! another bogus comment !!!'), + ('comment', '[with square brackets]!'), + ('comment', '[\nmultiline\nbogusness\n]!'), + ('comment', '[more brackets]-[and a hyphen]!'), + ('comment', '[cdata[should be uppercase]]'), + ('comment', '[CDATA [whitespaces are not ignored]]'), + ('comment', '[CDATA]]'), ] self._run_check(html, expected) def test_broken_condcoms(self): # these condcoms are missing the '--' after '<!' and before the '>' + # and they are considered bogus comments according to + # "8.2.4.42. Markup declaration open state" html = ('<![if !(IE)]>broken condcom<![endif]>' '<![if ! IE]><link href="favicon.tiff"/><![endif]>' '<![if !IE 6]><img src="firefox.png" /><![endif]>' '<![if !ie 6]><b>foo</b><![endif]>' '<![if (!IE)|(lt IE 9)]><img src="mammoth.bmp" /><![endif]>') - # According to the HTML5 specs sections "8.2.4.44 Bogus comment state" - # and "8.2.4.45 Markup declaration open state", comment tokens should - # be emitted instead of 'unknown decl', but calling unknown_decl - # provides more flexibility. - # See also Lib/_markupbase.py:parse_declaration expected = [ - ('unknown decl', 'if !(IE)'), + ('comment', '[if !(IE)]'), ('data', 'broken condcom'), - ('unknown decl', 'endif'), - ('unknown decl', 'if ! IE'), + ('comment', '[endif]'), + ('comment', '[if ! IE]'), ('startendtag', 'link', [('href', 'favicon.tiff')]), - ('unknown decl', 'endif'), - ('unknown decl', 'if !IE 6'), + ('comment', '[endif]'), + ('comment', '[if !IE 6]'), ('startendtag', 'img', [('src', 'firefox.png')]), - ('unknown decl', 'endif'), - ('unknown decl', 'if !ie 6'), + ('comment', '[endif]'), + ('comment', '[if !ie 6]'), ('starttag', 'b', []), ('data', 'foo'), ('endtag', 'b'), - ('unknown decl', 'endif'), - ('unknown decl', 'if (!IE)|(lt IE 9)'), + ('comment', '[endif]'), + ('comment', '[if (!IE)|(lt IE 9)]'), ('startendtag', 'img', [('src', 'mammoth.bmp')]), - ('unknown decl', 'endif') + ('comment', '[endif]') ] self._run_check(html, expected) + @support.subTests('content', [ + 'just some plain text', + '<!-- not a comment -->', + '&not-an-entity-ref;', + "<not a='start tag'>", + '', + '[[I have many brackets]]', + 'I have a > in the middle', + 'I have a ]] in the middle', + '] ]>', + ']] >', + ('\n' + ' if (a < b && a > b) {\n' + ' printf("[<marquee>How?</marquee>]");\n' + ' }\n'), + ]) + def test_cdata_section_content(self, content): + # See "13.2.5.42 Markup declaration open state", + # "13.2.5.69 CDATA section state", and issue bpo-32876. + html = f'<svg><text y="100"><![CDATA[{content}]]></text></svg>' + expected = [ + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content), + ('endtag', 'text'), + ('endtag', 'svg'), + ] + self._run_check(html, expected) + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + + def test_cdata_section(self): + # See "13.2.5.42 Markup declaration open state". + html = ('<![CDATA[foo<br>bar]]>' + '<svg><text y="100"><![CDATA[foo<br>bar]]></text></svg>' + '<![CDATA[foo<br>bar]]>') + expected = [ + ('comment', '[CDATA[foo<br'), + ('data', 'bar]]>'), + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[foo<br>bar'), + ('endtag', 'text'), + ('endtag', 'svg'), + ('comment', '[CDATA[foo<br'), + ('data', 'bar]]>'), + ] + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + def test_convert_charrefs_dropped_text(self): # #23144: make sure that all the events are triggered when # convert_charrefs is True, even if we don't call .close() @@ -598,6 +1011,26 @@ def test_convert_charrefs_dropped_text(self): ('endtag', 'a'), ('data', ' bar & baz')] ) + @support.requires_resource('cpu') + def test_eof_no_quadratic_complexity(self): + # Each of these examples used to take about an hour. + # Now they take a fraction of a second. + def check(source): + parser = html.parser.HTMLParser() + parser.feed(source) + parser.close() + n = 120_000 + check("<a " * n) + check("<a a=" * n) + check("</a " * 14 * n) + check("</a a=" * 11 * n) + check("<!--" * 4 * n) + check("<!" * 60 * n) + check("<?" * 19 * n) + check("</$" * 15 * n) + check("<![CDATA[" * 9 * n) + check("<!doctype" * 35 * n) + class AttributesTestCase(TestCaseBase): @@ -606,9 +1039,15 @@ def test_attr_syntax(self): ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) ] self._run_check("""<a b='v' c="v" d=v e>""", output) - self._run_check("""<a b = 'v' c = "v" d = v e>""", output) - self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output) - self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output) + self._run_check("<a foo==bar>", [('starttag', 'a', [('foo', '=bar')])]) + self._run_check("<a foo =bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\t=bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\v=bar>", [('starttag', 'a', [('foo\v', 'bar')])]) + self._run_check("<a foo\xa0=bar>", [('starttag', 'a', [('foo\xa0', 'bar')])]) + self._run_check("<a foo= bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\tbar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\vbar>", [('starttag', 'a', [('foo', '\vbar')])]) + self._run_check("<a foo=\xa0bar>", [('starttag', 'a', [('foo', '\xa0bar')])]) def test_attr_values(self): self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""", @@ -617,6 +1056,10 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""<a b='' c="">""", [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("<a b=\tx c=\ny>", + [('starttag', 'a', [('b', 'x'), ('c', 'y')])]) + self._run_check("<a b=\v c=\xa0>", + [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. self._run_check("<e a=rgb(1,2,3)>", [("starttag", "e", [("a", "rgb(1,2,3)")])]) @@ -683,13 +1126,17 @@ def test_malformed_attributes(self): ) expected = [ ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), - ('data', 'test - bad1'), ('endtag', 'a'), + ('data', 'test - bad1'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), - ('data', 'test - bad2'), ('endtag', 'a'), + ('data', 'test - bad2'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), - ('data', 'test - bad3'), ('endtag', 'a'), + ('data', 'test - bad3'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), - ('data', 'test - bad4'), ('endtag', 'a') + ('data', 'test - bad4'), + ('endtag', 'a'), ] self._run_check(html, expected) @@ -787,5 +1234,17 @@ def test_weird_chars_in_unquoted_attribute_values(self): ('starttag', 'form', [('action', 'bogus|&#()value')])]) + +class TestInheritance(unittest.TestCase): + + @patch("_markupbase.ParserBase.__init__") + @patch("_markupbase.ParserBase.reset") + def test_base_class_methods_called(self, super_reset_method, super_init_method): + with patch('_markupbase.ParserBase') as parser_base: + EventCollector() + super_init_method.assert_called_once() + super_reset_method.assert_called_once() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index ba594079cd8..04cb440cd4c 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -1,14 +1,15 @@ """Tests for http/cookiejar.py.""" import os +import stat +import sys import re -import test.support +from test import support from test.support import os_helper from test.support import warnings_helper import time import unittest import urllib.request -import pathlib from http.cookiejar import (time2isoz, http2time, iso2time, time2netscape, parse_ns_headers, join_header_words, split_header_words, Cookie, @@ -17,6 +18,7 @@ reach, is_HDN, domain_match, user_domain_match, request_path, request_port, request_host) +mswindows = (sys.platform == "win32") class DateTimeTests(unittest.TestCase): @@ -104,8 +106,7 @@ def test_http2time_formats(self): self.assertEqual(http2time(s.lower()), test_t, s.lower()) self.assertEqual(http2time(s.upper()), test_t, s.upper()) - def test_http2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Mandag 16. September 1996', @@ -120,12 +121,10 @@ def test_http2time_garbage(self): '08-01-3697739', '09 Feb 19942632 22:23:32 GMT', 'Wed, 09 Feb 1994834 22:23:32 GMT', - ]: - self.assertIsNone(http2time(test), - "http2time(%s) is not None\n" - "http2time(test) %s" % (test, http2time(test))) + ]) + def test_http2time_garbage(self, test): + self.assertIsNone(http2time(test)) - @unittest.skip("TODO: RUSTPYTHON, regressed to cubic complexity") def test_http2time_redos_regression_actually_completes(self): # LOOSE_HTTP_DATE_RE was vulnerable to malicious input which caused catastrophic backtracking (REDoS). # If we regress to cubic complexity, this test will take a very long time to succeed. @@ -149,9 +148,7 @@ def parse_date(text): self.assertEqual(parse_date("1994-02-03 19:45:29 +0530"), (1994, 2, 3, 14, 15, 29)) - def test_iso2time_formats(self): - # test iso2time for supported dates. - tests = [ + @support.subTests('s', [ '1994-02-03 00:00:00 -0000', # ISO 8601 format '1994-02-03 00:00:00 +0000', # ISO 8601 format '1994-02-03 00:00:00', # zone is optional @@ -164,16 +161,15 @@ def test_iso2time_formats(self): # A few tests with extra space at various places ' 1994-02-03 ', ' 1994-02-03T00:00:00 ', - ] - + ]) + def test_iso2time_formats(self, s): + # test iso2time for supported dates. test_t = 760233600 # assume broken POSIX counting of seconds - for s in tests: - self.assertEqual(iso2time(s), test_t, s) - self.assertEqual(iso2time(s.lower()), test_t, s.lower()) - self.assertEqual(iso2time(s.upper()), test_t, s.upper()) + self.assertEqual(iso2time(s), test_t, s) + self.assertEqual(iso2time(s.lower()), test_t, s.lower()) + self.assertEqual(iso2time(s.upper()), test_t, s.upper()) - def test_iso2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Thursday, 03-Feb-94 00:00:00 GMT', @@ -186,11 +182,10 @@ def test_iso2time_garbage(self): '01-01-1980 00:00:62', '01-01-1980T00:00:62', '19800101T250000Z', - ]: - self.assertIsNone(iso2time(test), - "iso2time(%r)" % test) + ]) + def test_iso2time_garbage(self, test): + self.assertIsNone(iso2time(test)) - @unittest.skip("TODO, RUSTPYTHON, regressed to quadratic complexity") def test_iso2time_performance_regression(self): # If ISO_DATE_RE regresses to quadratic complexity, this test will take a very long time to succeed. # If fixed, it should complete within a fraction of a second. @@ -200,24 +195,23 @@ def test_iso2time_performance_regression(self): class HeaderTests(unittest.TestCase): - def test_parse_ns_headers(self): - # quotes should be stripped - expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] - for hdr in [ + @support.subTests('hdr', [ 'foo=bar; expires=01 Jan 2040 22:23:32 GMT', 'foo=bar; expires="01 Jan 2040 22:23:32 GMT"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) - - def test_parse_ns_headers_version(self): - + ]) + def test_parse_ns_headers(self, hdr): # quotes should be stripped - expected = [[('foo', 'bar'), ('version', '1')]] - for hdr in [ + expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] + self.assertEqual(parse_ns_headers([hdr]), expected) + + @support.subTests('hdr', [ 'foo=bar; version="1"', 'foo=bar; Version="1"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) + ]) + def test_parse_ns_headers_version(self, hdr): + # quotes should be stripped + expected = [[('foo', 'bar'), ('version', '1')]] + self.assertEqual(parse_ns_headers([hdr]), expected) def test_parse_ns_headers_special_names(self): # names such as 'expires' are not special in first name=value pair @@ -227,14 +221,21 @@ def test_parse_ns_headers_special_names(self): expected = [[("expires", "01 Jan 2040 22:23:32 GMT"), ("version", "0")]] self.assertEqual(parse_ns_headers([hdr]), expected) - def test_join_header_words(self): - joined = join_header_words([[("foo", None), ("bar", "baz")]]) - self.assertEqual(joined, "foo; bar=baz") - - self.assertEqual(join_header_words([[]]), "") - - def test_split_header_words(self): - tests = [ + @support.subTests('src,expected', [ + ([[("foo", None), ("bar", "baz")]], "foo; bar=baz"), + (([]), ""), + (([[]]), ""), + (([[("a", "_")]]), "a=_"), + (([[("a", ";")]]), 'a=";"'), + ([[("n", None), ("foo", "foo;_")], [("bar", "foo_bar")]], + 'n; foo="foo;_", bar=foo_bar'), + ([[("n", "m"), ("foo", None)], [("bar", "foo_bar")]], + 'n=m; foo, bar=foo_bar'), + ]) + def test_join_header_words(self, src, expected): + self.assertEqual(join_header_words(src), expected) + + @support.subTests('arg,expect', [ ("foo", [[("foo", None)]]), ("foo=bar", [[("foo", "bar")]]), (" foo ", [[("foo", None)]]), @@ -251,24 +252,22 @@ def test_split_header_words(self): (r'foo; bar=baz, spam=, foo="\,\;\"", bar= ', [[("foo", None), ("bar", "baz")], [("spam", "")], [("foo", ',;"')], [("bar", "")]]), - ] - - for arg, expect in tests: - try: - result = split_header_words([arg]) - except: - import traceback, io - f = io.StringIO() - traceback.print_exc(None, f) - result = "(error -- traceback follows)\n\n%s" % f.getvalue() - self.assertEqual(result, expect, """ + ]) + def test_split_header_words(self, arg, expect): + try: + result = split_header_words([arg]) + except: + import traceback, io + f = io.StringIO() + traceback.print_exc(None, f) + result = "(error -- traceback follows)\n\n%s" % f.getvalue() + self.assertEqual(result, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' """ % (arg, expect, result)) - def test_roundtrip(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", "foo"), ("foo=bar", "foo=bar"), (" foo ", "foo"), @@ -277,23 +276,35 @@ def test_roundtrip(self): ("foo=bar;bar=baz", "foo=bar; bar=baz"), ('foo bar baz', "foo; bar; baz"), (r'foo="\"" bar="\\"', r'foo="\""; bar="\\"'), + ("föo=bär", 'föo="bär"'), ('foo,,,bar', 'foo, bar'), ('foo=bar,bar=baz', 'foo=bar, bar=baz'), + ("foo=\n", 'foo=""'), + ('foo="\n"', 'foo="\n"'), + ('foo=bar\n', 'foo=bar'), + ('foo="bar\n"', 'foo="bar\n"'), + ('foo=bar\nbaz', 'foo=bar; baz'), + ('foo="bar\nbaz"', 'foo="bar\nbaz"'), ('text/html; charset=iso-8859-1', - 'text/html; charset="iso-8859-1"'), + 'text/html; charset=iso-8859-1'), + + ('text/html; charset="iso-8859/1"', + 'text/html; charset="iso-8859/1"'), ('foo="bar"; port="80,81"; discard, bar=baz', 'foo=bar; port="80,81"; discard, bar=baz'), (r'Basic realm="\"foo\\\\bar\""', - r'Basic; realm="\"foo\\\\bar\""') - ] - - for arg, expect in tests: - input = split_header_words([arg]) - res = join_header_words(input) - self.assertEqual(res, expect, """ + r'Basic; realm="\"foo\\\\bar\""'), + + ('n; foo="foo;_", bar="foo,_"', + 'n; foo="foo;_", bar="foo,_"'), + ]) + def test_roundtrip(self, arg, expect): + input = split_header_words([arg]) + res = join_header_words(input) + self.assertEqual(res, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' @@ -337,9 +348,9 @@ def test_constructor_with_str(self): self.assertEqual(c.filename, filename) def test_constructor_with_path_like(self): - filename = pathlib.Path(os_helper.TESTFN) - c = LWPCookieJar(filename) - self.assertEqual(c.filename, os.fspath(filename)) + filename = os_helper.TESTFN + c = LWPCookieJar(os_helper.FakePath(filename)) + self.assertEqual(c.filename, filename) def test_constructor_with_none(self): c = LWPCookieJar(None) @@ -366,10 +377,63 @@ def test_lwp_valueless_cookie(self): c = LWPCookieJar() c.load(filename, ignore_discard=True) finally: - try: os.unlink(filename) - except OSError: pass + os_helper.unlink(filename) self.assertEqual(c._cookies["www.acme.com"]["/"]["boo"].value, None) + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_lwp_filepermissions(self): + # Cookie file should only be readable by the creator + filename = os_helper.TESTFN + c = LWPCookieJar() + interact_netscape(c, "http://www.acme.com/", 'boo') + try: + c.save(filename, ignore_discard=True) + st = os.stat(filename) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o600) + finally: + os_helper.unlink(filename) + + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_mozilla_filepermissions(self): + # Cookie file should only be readable by the creator + filename = os_helper.TESTFN + c = MozillaCookieJar() + interact_netscape(c, "http://www.acme.com/", 'boo') + try: + c.save(filename, ignore_discard=True) + st = os.stat(filename) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o600) + finally: + os_helper.unlink(filename) + + @unittest.skipIf(mswindows, "windows file permissions are incompatible with file modes") + @os_helper.skip_unless_working_chmod + def test_cookie_files_are_truncated(self): + filename = os_helper.TESTFN + for cookiejar_class in (LWPCookieJar, MozillaCookieJar): + c = cookiejar_class(filename) + + req = urllib.request.Request("http://www.acme.com/") + headers = ["Set-Cookie: pll_lang=en; Max-Age=31536000; path=/"] + res = FakeResponse(headers, "http://www.acme.com/") + c.extract_cookies(res, req) + self.assertEqual(len(c), 1) + + try: + # Save the first version with contents: + c.save() + # Now, clear cookies and re-save: + c.clear() + c.save() + # Check that file was truncated: + c.load() + finally: + os_helper.unlink(filename) + + self.assertEqual(len(c), 0) + def test_bad_magic(self): # OSErrors (eg. file doesn't exist) are allowed to propagate filename = os_helper.TESTFN @@ -393,8 +457,7 @@ def test_bad_magic(self): c = cookiejar_class() self.assertRaises(LoadError, c.load, filename) finally: - try: os.unlink(filename) - except OSError: pass + os_helper.unlink(filename) class CookieTests(unittest.TestCase): # XXX @@ -443,14 +506,7 @@ class CookieTests(unittest.TestCase): ## just the 7 special TLD's listed in their spec. And folks rely on ## that... - def test_domain_return_ok(self): - # test optimization: .domain_return_ok() should filter out most - # domains in the CookieJar before we try to access them (because that - # may require disk access -- in particular, with MSIECookieJar) - # This is only a rough check for performance reasons, so it's not too - # critical as long as it's sufficiently liberal. - pol = DefaultCookiePolicy() - for url, domain, ok in [ + @support.subTests('url,domain,ok', [ ("http://foo.bar.com/", "blah.com", False), ("http://foo.bar.com/", "rhubarb.blah.com", False), ("http://foo.bar.com/", "rhubarb.foo.bar.com", False), @@ -470,11 +526,18 @@ def test_domain_return_ok(self): ("http://foo/", ".local", True), ("http://barfoo.com", ".foo.com", False), ("http://barfoo.com", "foo.com", False), - ]: - request = urllib.request.Request(url) - r = pol.domain_return_ok(domain, request) - if ok: self.assertTrue(r) - else: self.assertFalse(r) + ]) + def test_domain_return_ok(self, url, domain, ok): + # test optimization: .domain_return_ok() should filter out most + # domains in the CookieJar before we try to access them (because that + # may require disk access -- in particular, with MSIECookieJar) + # This is only a rough check for performance reasons, so it's not too + # critical as long as it's sufficiently liberal. + pol = DefaultCookiePolicy() + request = urllib.request.Request(url) + r = pol.domain_return_ok(domain, request) + if ok: self.assertTrue(r) + else: self.assertFalse(r) def test_missing_value(self): # missing = sign in Cookie: header is regarded by Mozilla as a missing @@ -490,7 +553,7 @@ def test_missing_value(self): self.assertIsNone(cookie.value) self.assertEqual(cookie.name, '"spam"') self.assertEqual(lwp_cookie_str(cookie), ( - r'"spam"; path="/foo/"; domain="www.acme.com"; ' + r'"spam"; path="/foo/"; domain=www.acme.com; ' 'path_spec; discard; version=0')) old_str = repr(c) c.save(ignore_expires=True, ignore_discard=True) @@ -498,7 +561,7 @@ def test_missing_value(self): c = MozillaCookieJar(filename) c.revert(ignore_expires=True, ignore_discard=True) finally: - os.unlink(c.filename) + os_helper.unlink(c.filename) # cookies unchanged apart from lost info re. whether path was specified self.assertEqual( repr(c), @@ -508,10 +571,7 @@ def test_missing_value(self): self.assertEqual(interact_netscape(c, "http://www.acme.com/foo/"), '"spam"; eggs') - def test_rfc2109_handling(self): - # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, - # dependent on policy settings - for rfc2109_as_netscape, rfc2965, version in [ + @support.subTests('rfc2109_as_netscape,rfc2965,version', [ # default according to rfc2965 if not explicitly specified (None, False, 0), (None, True, 1), @@ -520,24 +580,27 @@ def test_rfc2109_handling(self): (False, True, 1), (True, False, 0), (True, True, 0), - ]: - policy = DefaultCookiePolicy( - rfc2109_as_netscape=rfc2109_as_netscape, - rfc2965=rfc2965) - c = CookieJar(policy) - interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") - try: - cookie = c._cookies["www.example.com"]["/"]["ni"] - except KeyError: - self.assertIsNone(version) # didn't expect a stored cookie - else: - self.assertEqual(cookie.version, version) - # 2965 cookies are unaffected - interact_2965(c, "http://www.example.com/", - "foo=bar; Version=1") - if rfc2965: - cookie2965 = c._cookies["www.example.com"]["/"]["foo"] - self.assertEqual(cookie2965.version, 1) + ]) + def test_rfc2109_handling(self, rfc2109_as_netscape, rfc2965, version): + # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, + # dependent on policy settings + policy = DefaultCookiePolicy( + rfc2109_as_netscape=rfc2109_as_netscape, + rfc2965=rfc2965) + c = CookieJar(policy) + interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") + try: + cookie = c._cookies["www.example.com"]["/"]["ni"] + except KeyError: + self.assertIsNone(version) # didn't expect a stored cookie + else: + self.assertEqual(cookie.version, version) + # 2965 cookies are unaffected + interact_2965(c, "http://www.example.com/", + "foo=bar; Version=1") + if rfc2965: + cookie2965 = c._cookies["www.example.com"]["/"]["foo"] + self.assertEqual(cookie2965.version, 1) def test_ns_parser(self): c = CookieJar() @@ -705,8 +768,7 @@ def test_default_path_with_query(self): # Cookie is sent back to the same URI. self.assertEqual(interact_netscape(cj, uri), value) - def test_escape_path(self): - cases = [ + @support.subTests('arg,result', [ # quoted safe ("/foo%2f/bar", "/foo%2F/bar"), ("/foo%2F/bar", "/foo%2F/bar"), @@ -726,9 +788,9 @@ def test_escape_path(self): ("/foo/bar\u00fc", "/foo/bar%C3%BC"), # UTF-8 encoded # unicode ("/foo/bar\uabcd", "/foo/bar%EA%AF%8D"), # UTF-8 encoded - ] - for arg, result in cases: - self.assertEqual(escape_path(arg), result) + ]) + def test_escape_path(self, arg, result): + self.assertEqual(escape_path(arg), result) def test_request_path(self): # with parameters @@ -922,6 +984,48 @@ def test_two_component_domain_ns(self): ## self.assertEqual(len(c), 2) self.assertEqual(len(c), 4) + def test_localhost_domain(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar; domain=localhost;") + + self.assertEqual(len(c), 1) + + def test_localhost_domain_contents(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar; domain=localhost;") + + self.assertEqual(c._cookies[".localhost"]["/"]["foo"].value, "bar") + + def test_localhost_domain_contents_2(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar;") + + self.assertEqual(c._cookies["localhost.local"]["/"]["foo"].value, "bar") + + def test_evil_nonlocal_domain(self): + c = CookieJar() + + interact_netscape(c, "http://evil.com", "foo=bar; domain=.localhost") + + self.assertEqual(len(c), 0) + + def test_evil_local_domain(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar; domain=.evil.com") + + self.assertEqual(len(c), 0) + + def test_evil_local_domain_2(self): + c = CookieJar() + + interact_netscape(c, "http://localhost", "foo=bar; domain=.someother.local") + + self.assertEqual(len(c), 0) + def test_two_component_domain_rfc2965(self): pol = DefaultCookiePolicy(rfc2965=True) c = CookieJar(pol) @@ -1253,11 +1357,11 @@ def test_Cookie_iterator(self): r'port="90,100, 80,8080"; ' r'max-age=100; Comment = "Just kidding! (\"|\\\\) "') - versions = [1, 1, 1, 0, 1] - names = ["bang", "foo", "foo", "spam", "foo"] - domains = [".sol.no", "blah.spam.org", "www.acme.com", - "www.acme.com", "www.acme.com"] - paths = ["/", "/", "/", "/blah", "/blah/"] + versions = [1, 0, 1, 1, 1] + names = ["foo", "spam", "foo", "foo", "bang"] + domains = ["blah.spam.org", "www.acme.com", "www.acme.com", + "www.acme.com", ".sol.no"] + paths = ["/", "/blah", "/blah/", "/", "/"] for i in range(4): i = 0 @@ -1422,7 +1526,7 @@ def test_netscape_example_1(self): h = req.get_header("Cookie") self.assertIn("PART_NUMBER=ROCKET_LAUNCHER_0001", h) self.assertIn("CUSTOMER=WILE_E_COYOTE", h) - self.assertTrue(h.startswith("SHIPPING=FEDEX;")) + self.assertStartsWith(h, "SHIPPING=FEDEX;") def test_netscape_example_2(self): # Second Example transaction sequence: @@ -1726,8 +1830,7 @@ def test_rejection(self): c = LWPCookieJar(policy=pol) c.load(filename, ignore_discard=True) finally: - try: os.unlink(filename) - except OSError: pass + os_helper.unlink(filename) self.assertEqual(old, repr(c)) @@ -1786,8 +1889,7 @@ def save_and_restore(cj, ignore_discard): DefaultCookiePolicy(rfc2965=True)) new_c.load(ignore_discard=ignore_discard) finally: - try: os.unlink(filename) - except OSError: pass + os_helper.unlink(filename) return new_c new_c = save_and_restore(c, True) diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 6072c7e15e9..11861b0e81c 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,10 +1,11 @@ # Simple test suite for http/cookies.py import copy -from test.support import run_unittest, run_doctest import unittest +import doctest from http import cookies import pickle +from test import support class CookieTests(unittest.TestCase): @@ -16,10 +17,10 @@ def test_basic(self): 'repr': "<SimpleCookie: chips='ahoy' vienna='finger'>", 'output': 'Set-Cookie: chips=ahoy\nSet-Cookie: vienna=finger'}, - {'data': 'keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"', - 'dict': {'keebler' : 'E=mc2; L="Loves"; fudge=\012;'}, - 'repr': '''<SimpleCookie: keebler='E=mc2; L="Loves"; fudge=\\n;'>''', - 'output': 'Set-Cookie: keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"'}, + {'data': 'keebler="E=mc2; L=\\"Loves\\"; fudge=;"', + 'dict': {'keebler' : 'E=mc2; L="Loves"; fudge=;'}, + 'repr': '''<SimpleCookie: keebler='E=mc2; L="Loves"; fudge=;'>''', + 'output': 'Set-Cookie: keebler="E=mc2; L=\\"Loves\\"; fudge=;"'}, # Check illegal cookies that have an '=' char in an unquoted value {'data': 'keebler=E=mc2', @@ -58,6 +59,90 @@ def test_basic(self): for k, v in sorted(case['dict'].items()): self.assertEqual(C[k].value, v) + def test_obsolete_rfc850_date_format(self): + # Test cases with different days and dates in obsolete RFC 850 format + test_cases = [ + # from RFC 850, change EST to GMT + # https://datatracker.ietf.org/doc/html/rfc850#section-2 + { + 'data': 'key=value; expires=Saturday, 01-Jan-83 00:00:00 GMT', + 'output': 'Saturday, 01-Jan-83 00:00:00 GMT' + }, + { + 'data': 'key=value; expires=Friday, 19-Nov-82 16:59:30 GMT', + 'output': 'Friday, 19-Nov-82 16:59:30 GMT' + }, + # from RFC 9110 + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.6.7-6 + { + 'data': 'key=value; expires=Sunday, 06-Nov-94 08:49:37 GMT', + 'output': 'Sunday, 06-Nov-94 08:49:37 GMT' + }, + # other test cases + { + 'data': 'key=value; expires=Wednesday, 09-Nov-94 08:49:37 GMT', + 'output': 'Wednesday, 09-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Friday, 11-Nov-94 08:49:37 GMT', + 'output': 'Friday, 11-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Monday, 14-Nov-94 08:49:37 GMT', + 'output': 'Monday, 14-Nov-94 08:49:37 GMT' + }, + ] + + for case in test_cases: + with self.subTest(data=case['data']): + C = cookies.SimpleCookie() + C.load(case['data']) + + # Extract the cookie name from the data string + cookie_name = case['data'].split('=')[0] + + # Check if the cookie is loaded correctly + self.assertIn(cookie_name, C) + self.assertEqual(C[cookie_name].get('expires'), case['output']) + + def test_unquote(self): + cases = [ + (r'a="b=\""', 'b="'), + (r'a="b=\\"', 'b=\\'), + (r'a="b=\="', 'b=='), + (r'a="b=\n"', 'b=n'), + (r'a="b=\042"', 'b="'), + (r'a="b=\134"', 'b=\\'), + (r'a="b=\377"', 'b=\xff'), + (r'a="b=\400"', 'b=400'), + (r'a="b=\42"', 'b=42'), + (r'a="b=\\042"', 'b=\\042'), + (r'a="b=\\134"', 'b=\\134'), + (r'a="b=\\\""', 'b=\\"'), + (r'a="b=\\\042"', 'b=\\"'), + (r'a="b=\134\""', 'b=\\"'), + (r'a="b=\134\042"', 'b=\\"'), + ] + for encoded, decoded in cases: + with self.subTest(encoded): + C = cookies.SimpleCookie() + C.load(encoded) + self.assertEqual(C['a'].value, decoded) + + @support.requires_resource('cpu') + def test_unquote_large(self): + # n = 10**6 + n = 10**4 # XXX: RUSTPYTHON; This takes more than 10 minutes to run. lower to 4 + for encoded in r'\\', r'\134': + with self.subTest(encoded): + data = 'a="b=' + encoded*n + ';"' + C = cookies.SimpleCookie() + C.load(data) + value = C['a'].value + self.assertEqual(value[:3], 'b=\\') + self.assertEqual(value[-2:], '\\;') + self.assertEqual(len(value), n + 3) + def test_load(self): C = cookies.SimpleCookie() C.load('Customer="WILE_E_COYOTE"; Version=1; Path=/acme') @@ -96,7 +181,7 @@ def test_special_attrs(self): C = cookies.SimpleCookie('Customer="WILE_E_COYOTE"') C['Customer']['expires'] = 0 # can't test exact output, it always depends on current date/time - self.assertTrue(C.output().endswith('GMT')) + self.assertEndsWith(C.output(), 'GMT') # loading 'expires' C = cookies.SimpleCookie() @@ -121,6 +206,14 @@ def test_set_secure_httponly_attrs(self): self.assertEqual(C.output(), 'Set-Cookie: Customer="WILE_E_COYOTE"; HttpOnly; Secure') + def test_set_secure_httponly_partitioned_attrs(self): + C = cookies.SimpleCookie('Customer="WILE_E_COYOTE"') + C['Customer']['secure'] = True + C['Customer']['httponly'] = True + C['Customer']['partitioned'] = True + self.assertEqual(C.output(), + 'Set-Cookie: Customer="WILE_E_COYOTE"; HttpOnly; Partitioned; Secure') + def test_samesite_attrs(self): samesite_values = ['Strict', 'Lax', 'strict', 'lax'] for val in samesite_values: @@ -479,9 +572,55 @@ def test_repr(self): r'Set-Cookie: key=coded_val; ' r'expires=\w+, \d+ \w+ \d+ \d+:\d+:\d+ \w+') -def test_main(): - run_unittest(CookieTests, MorselTests) - run_doctest(cookies) + def test_control_characters(self): + for c0 in support.control_characters_c0(): + morsel = cookies.Morsel() + + # .__setitem__() + with self.assertRaises(cookies.CookieError): + morsel[c0] = "val" + with self.assertRaises(cookies.CookieError): + morsel["path"] = c0 + + # .setdefault() + with self.assertRaises(cookies.CookieError): + morsel.setdefault("path", c0) + with self.assertRaises(cookies.CookieError): + morsel.setdefault(c0, "val") + + # .set() + with self.assertRaises(cookies.CookieError): + morsel.set(c0, "val", "coded-value") + with self.assertRaises(cookies.CookieError): + morsel.set("path", c0, "coded-value") + with self.assertRaises(cookies.CookieError): + morsel.set("path", "val", c0) + + def test_control_characters_output(self): + # Tests that even if the internals of Morsel are modified + # that a call to .output() has control character safeguards. + for c0 in support.control_characters_c0(): + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._key = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.output() + + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._coded_value = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.output() + + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(cookies)) + return tests + if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 8f095d52ac4..8ce5f853b7c 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1,3 +1,4 @@ +import enum import errno from http import client, HTTPStatus import io @@ -7,7 +8,6 @@ import re import socket import threading -import warnings import unittest from unittest import mock @@ -16,16 +16,18 @@ from test import support from test.support import os_helper from test.support import socket_helper -from test.support import warnings_helper +support.requires_working_socket(module=True) here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' -CERT_localhost = os.path.join(here, 'keycert.pem') +CERT_localhost = os.path.join(here, 'certdata', 'keycert.pem') # Self-signed cert file for 'fakehostname' -CERT_fakehostname = os.path.join(here, 'keycert2.pem') +CERT_fakehostname = os.path.join(here, 'certdata', 'keycert2.pem') # Self-signed cert file for self-signed.pythontest.net -CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') +CERT_selfsigned_pythontestdotnet = os.path.join( + here, 'certdata', 'selfsigned_pythontestdotnet.pem', +) # constants for testing chunked encoding chunked_start = ( @@ -271,7 +273,7 @@ def test_ipv6host_header(self): sock = FakeSocket('') conn.sock = sock conn.request('GET', '/foo') - self.assertTrue(sock.data.startswith(expected)) + self.assertStartsWith(sock.data, expected) expected = b'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ b'Accept-Encoding: identity\r\n\r\n' @@ -279,7 +281,23 @@ def test_ipv6host_header(self): sock = FakeSocket('') conn.sock = sock conn.request('GET', '/foo') - self.assertTrue(sock.data.startswith(expected)) + self.assertStartsWith(sock.data, expected) + + expected = b'GET /foo HTTP/1.1\r\nHost: [fe80::]\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[fe80::%2]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertStartsWith(sock.data, expected) + + expected = b'GET /foo HTTP/1.1\r\nHost: [fe80::]:81\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[fe80::%2]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertStartsWith(sock.data, expected) def test_malformed_headers_coped_with(self): # Issue 19996 @@ -317,9 +335,9 @@ def test_parse_all_octets(self): self.assertIsNotNone(resp.getheader('obs-text')) self.assertIn('obs-text', resp.msg) for folded in (resp.getheader('obs-fold'), resp.msg['obs-fold']): - self.assertTrue(folded.startswith('text')) + self.assertStartsWith(folded, 'text') self.assertIn(' folded with space', folded) - self.assertTrue(folded.endswith('folded with tab')) + self.assertEndsWith(folded, 'folded with tab') def test_invalid_headers(self): conn = client.HTTPConnection('example.com') @@ -524,6 +542,209 @@ def test_dir_with_added_behavior_on_status(self): # see issue40084 self.assertTrue({'description', 'name', 'phrase', 'value'} <= set(dir(HTTPStatus(404)))) + def test_simple_httpstatus(self): + class CheckedHTTPStatus(enum.IntEnum): + """HTTP status codes and reason phrases + + Status codes from the following RFCs are all observed: + + * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 + * RFC 6585: Additional HTTP Status Codes + * RFC 3229: Delta encoding in HTTP + * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 + * RFC 5842: Binding Extensions to WebDAV + * RFC 7238: Permanent Redirect + * RFC 2295: Transparent Content Negotiation in HTTP + * RFC 2774: An HTTP Extension Framework + * RFC 7725: An HTTP Status Code to Report Legal Obstacles + * RFC 7540: Hypertext Transfer Protocol Version 2 (HTTP/2) + * RFC 2324: Hyper Text Coffee Pot Control Protocol (HTCPCP/1.0) + * RFC 8297: An HTTP Status Code for Indicating Hints + * RFC 8470: Using Early Data in HTTP + """ + def __new__(cls, value, phrase, description=''): + obj = int.__new__(cls, value) + obj._value_ = value + + obj.phrase = phrase + obj.description = description + return obj + + @property + def is_informational(self): + return 100 <= self <= 199 + + @property + def is_success(self): + return 200 <= self <= 299 + + @property + def is_redirection(self): + return 300 <= self <= 399 + + @property + def is_client_error(self): + return 400 <= self <= 499 + + @property + def is_server_error(self): + return 500 <= self <= 599 + + # informational + CONTINUE = 100, 'Continue', 'Request received, please continue' + SWITCHING_PROTOCOLS = (101, 'Switching Protocols', + 'Switching to new protocol; obey Upgrade header') + PROCESSING = 102, 'Processing', 'Server is processing the request' + EARLY_HINTS = (103, 'Early Hints', + 'Headers sent to prepare for the response') + # success + OK = 200, 'OK', 'Request fulfilled, document follows' + CREATED = 201, 'Created', 'Document created, URL follows' + ACCEPTED = (202, 'Accepted', + 'Request accepted, processing continues off-line') + NON_AUTHORITATIVE_INFORMATION = (203, + 'Non-Authoritative Information', 'Request fulfilled from cache') + NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' + RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' + PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' + MULTI_STATUS = (207, 'Multi-Status', + 'Response contains multiple statuses in the body') + ALREADY_REPORTED = (208, 'Already Reported', + 'Operation has already been reported') + IM_USED = 226, 'IM Used', 'Request completed using instance manipulations' + # redirection + MULTIPLE_CHOICES = (300, 'Multiple Choices', + 'Object has several resources -- see URI list') + MOVED_PERMANENTLY = (301, 'Moved Permanently', + 'Object moved permanently -- see URI list') + FOUND = 302, 'Found', 'Object moved temporarily -- see URI list' + SEE_OTHER = 303, 'See Other', 'Object moved -- see Method and URL list' + NOT_MODIFIED = (304, 'Not Modified', + 'Document has not changed since given time') + USE_PROXY = (305, 'Use Proxy', + 'You must use proxy specified in Location to access this resource') + TEMPORARY_REDIRECT = (307, 'Temporary Redirect', + 'Object moved temporarily -- see URI list') + PERMANENT_REDIRECT = (308, 'Permanent Redirect', + 'Object moved permanently -- see URI list') + # client error + BAD_REQUEST = (400, 'Bad Request', + 'Bad request syntax or unsupported method') + UNAUTHORIZED = (401, 'Unauthorized', + 'No permission -- see authorization schemes') + PAYMENT_REQUIRED = (402, 'Payment Required', + 'No payment -- see charging schemes') + FORBIDDEN = (403, 'Forbidden', + 'Request forbidden -- authorization will not help') + NOT_FOUND = (404, 'Not Found', + 'Nothing matches the given URI') + METHOD_NOT_ALLOWED = (405, 'Method Not Allowed', + 'Specified method is invalid for this resource') + NOT_ACCEPTABLE = (406, 'Not Acceptable', + 'URI not available in preferred format') + PROXY_AUTHENTICATION_REQUIRED = (407, + 'Proxy Authentication Required', + 'You must authenticate with this proxy before proceeding') + REQUEST_TIMEOUT = (408, 'Request Timeout', + 'Request timed out; try again later') + CONFLICT = 409, 'Conflict', 'Request conflict' + GONE = (410, 'Gone', + 'URI no longer exists and has been permanently removed') + LENGTH_REQUIRED = (411, 'Length Required', + 'Client must specify Content-Length') + PRECONDITION_FAILED = (412, 'Precondition Failed', + 'Precondition in headers is false') + CONTENT_TOO_LARGE = (413, 'Content Too Large', + 'Content is too large') + REQUEST_ENTITY_TOO_LARGE = CONTENT_TOO_LARGE + URI_TOO_LONG = (414, 'URI Too Long', 'URI is too long') + REQUEST_URI_TOO_LONG = URI_TOO_LONG + UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type', + 'Entity body in unsupported format') + RANGE_NOT_SATISFIABLE = (416, + 'Range Not Satisfiable', + 'Cannot satisfy request range') + REQUESTED_RANGE_NOT_SATISFIABLE = RANGE_NOT_SATISFIABLE + EXPECTATION_FAILED = (417, 'Expectation Failed', + 'Expect condition could not be satisfied') + IM_A_TEAPOT = (418, 'I\'m a Teapot', + 'Server refuses to brew coffee because it is a teapot') + MISDIRECTED_REQUEST = (421, 'Misdirected Request', + 'Server is not able to produce a response') + UNPROCESSABLE_CONTENT = (422, 'Unprocessable Content', + 'Server is not able to process the contained instructions') + UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT + LOCKED = 423, 'Locked', 'Resource of a method is locked' + FAILED_DEPENDENCY = (424, 'Failed Dependency', + 'Dependent action of the request failed') + TOO_EARLY = (425, 'Too Early', + 'Server refuses to process a request that might be replayed') + UPGRADE_REQUIRED = (426, 'Upgrade Required', + 'Server refuses to perform the request using the current protocol') + PRECONDITION_REQUIRED = (428, 'Precondition Required', + 'The origin server requires the request to be conditional') + TOO_MANY_REQUESTS = (429, 'Too Many Requests', + 'The user has sent too many requests in ' + 'a given amount of time ("rate limiting")') + REQUEST_HEADER_FIELDS_TOO_LARGE = (431, + 'Request Header Fields Too Large', + 'The server is unwilling to process the request because its header ' + 'fields are too large') + UNAVAILABLE_FOR_LEGAL_REASONS = (451, + 'Unavailable For Legal Reasons', + 'The server is denying access to the ' + 'resource as a consequence of a legal demand') + # server errors + INTERNAL_SERVER_ERROR = (500, 'Internal Server Error', + 'Server got itself in trouble') + NOT_IMPLEMENTED = (501, 'Not Implemented', + 'Server does not support this operation') + BAD_GATEWAY = (502, 'Bad Gateway', + 'Invalid responses from another server/proxy') + SERVICE_UNAVAILABLE = (503, 'Service Unavailable', + 'The server cannot process the request due to a high load') + GATEWAY_TIMEOUT = (504, 'Gateway Timeout', + 'The gateway server did not receive a timely response') + HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', + 'Cannot fulfill request') + VARIANT_ALSO_NEGOTIATES = (506, 'Variant Also Negotiates', + 'Server has an internal configuration error') + INSUFFICIENT_STORAGE = (507, 'Insufficient Storage', + 'Server is not able to store the representation') + LOOP_DETECTED = (508, 'Loop Detected', + 'Server encountered an infinite loop while processing a request') + NOT_EXTENDED = (510, 'Not Extended', + 'Request does not meet the resource access policy') + NETWORK_AUTHENTICATION_REQUIRED = (511, + 'Network Authentication Required', + 'The client needs to authenticate to gain network access') + enum._test_simple_enum(CheckedHTTPStatus, HTTPStatus) + + def test_httpstatus_range(self): + """Checks that the statuses are in the 100-599 range""" + + for member in HTTPStatus.__members__.values(): + self.assertGreaterEqual(member, 100) + self.assertLessEqual(member, 599) + + def test_httpstatus_category(self): + """Checks that the statuses belong to the standard categories""" + + categories = ( + ((100, 199), "is_informational"), + ((200, 299), "is_success"), + ((300, 399), "is_redirection"), + ((400, 499), "is_client_error"), + ((500, 599), "is_server_error"), + ) + for member in HTTPStatus.__members__.values(): + for (lower, upper), category in categories: + category_indicator = getattr(member, category) + if lower <= member <= upper: + self.assertTrue(category_indicator) + else: + self.assertFalse(category_indicator) + def test_status_lines(self): # Test HTTP status lines @@ -779,8 +1000,7 @@ def test_send_file(self): sock = FakeSocket(body) conn.sock = sock conn.request('GET', '/foo', body) - self.assertTrue(sock.data.startswith(expected), '%r != %r' % - (sock.data[:len(expected)], expected)) + self.assertStartsWith(sock.data, expected) def test_send(self): expected = b'this is a test this is only a test' @@ -871,6 +1091,25 @@ def test_chunked(self): self.assertEqual(resp.read(), expected) resp.close() + # Explicit full read + for n in (-123, -1, None): + with self.subTest('full read', n=n): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertTrue(resp.chunked) + self.assertEqual(resp.read(n), expected) + resp.close() + + # Read first chunk + with self.subTest('read1(-1)'): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertTrue(resp.chunked) + self.assertEqual(resp.read1(-1), b"hello worl") + resp.close() + # Various read sizes for n in range(1, 12): sock = FakeSocket(chunked_start + last_chunk + chunked_end) @@ -1226,6 +1465,72 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_large_content_length(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + [conn, address] = serv.accept() + with conn: + while conn.recv(1024): + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" % size) + conn.sendall(b'A' * (size//3)) + conn.sendall(b'B' * (size - size//3)) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(15, 27): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertEqual(len(response.read()), size) + finally: + conn.close() + thread.join(1.0) + + def test_large_content_length_truncated(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + while True: + [conn, address] = serv.accept() + with conn: + conn.recv(1024) + if not size: + break + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"Text" % size) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(18, 65): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertRaises(client.IncompleteRead, response.read) + conn.close() + finally: + conn.close() + size = 0 + conn.request("GET", "/") + conn.close() + thread.join(1.0) + def test_putrequest_override_domain_validation(self): """ It should be possible to override the default validation @@ -1324,7 +1629,7 @@ def mypeek(n=-1): # then unbounded peek p2 = resp.peek() self.assertGreaterEqual(len(p2), len(p)) - self.assertTrue(p2.startswith(p)) + self.assertStartsWith(p2, p) next = resp.read(len(p2)) self.assertEqual(next, p2) else: @@ -1339,18 +1644,22 @@ def test_readline(self): resp = self.resp self._verify_readline(self.resp.readline, self.lines_expected) - def _verify_readline(self, readline, expected): + def test_readline_without_limit(self): + self._verify_readline(self.resp.readline, self.lines_expected, limit=-1) + + def _verify_readline(self, readline, expected, limit=5): all = [] while True: # short readlines - line = readline(5) + line = readline(limit) if line and line != b"foo": if len(line) < 5: - self.assertTrue(line.endswith(b"\n")) + self.assertEndsWith(line, b"\n") all.append(line) if not line: break self.assertEqual(b"".join(all), expected) + self.assertTrue(self.resp.isclosed()) def test_read1(self): resp = self.resp @@ -1370,6 +1679,7 @@ def test_read1_unbounded(self): break all.append(data) self.assertEqual(b"".join(all), self.lines_expected) + self.assertTrue(resp.isclosed()) def test_read1_bounded(self): resp = self.resp @@ -1381,15 +1691,22 @@ def test_read1_bounded(self): self.assertLessEqual(len(data), 10) all.append(data) self.assertEqual(b"".join(all), self.lines_expected) + self.assertTrue(resp.isclosed()) def test_read1_0(self): self.assertEqual(self.resp.read1(0), b"") + self.assertFalse(self.resp.isclosed()) def test_peek_0(self): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) +class ExtendedReadTestContentLengthKnown(ExtendedReadTest): + _header, _body = ExtendedReadTest.lines.split('\r\n\r\n', 1) + lines = _header + f'\r\nContent-Length: {len(_body)}\r\n\r\n' + _body + + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode @@ -1499,13 +1816,17 @@ def test_client_constants(self): 'GONE', 'LENGTH_REQUIRED', 'PRECONDITION_FAILED', + 'CONTENT_TOO_LARGE', 'REQUEST_ENTITY_TOO_LARGE', + 'URI_TOO_LONG', 'REQUEST_URI_TOO_LONG', 'UNSUPPORTED_MEDIA_TYPE', + 'RANGE_NOT_SATISFIABLE', 'REQUESTED_RANGE_NOT_SATISFIABLE', 'EXPECTATION_FAILED', 'IM_A_TEAPOT', 'MISDIRECTED_REQUEST', + 'UNPROCESSABLE_CONTENT', 'UNPROCESSABLE_ENTITY', 'LOCKED', 'FAILED_DEPENDENCY', @@ -1528,7 +1849,7 @@ def test_client_constants(self): ] for const in expected: with self.subTest(constant=const): - self.assertTrue(hasattr(client, const)) + self.assertHasAttr(client, const) class SourceAddressTest(TestCase): @@ -1695,8 +2016,6 @@ def test_attributes(self): h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) self.assertEqual(h.timeout, 30) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_networked(self): # Default settings: requires a valid cert from a trusted CA import ssl @@ -1734,8 +2053,6 @@ def test_networked_trusted_by_default_cert(self): h.close() self.assertIn('text/html', content_type) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_networked_good_cert(self): # We feed the server's cert as a validating cert import ssl @@ -1769,8 +2086,7 @@ def test_networked_good_cert(self): h.close() self.assertIn('nginx', server_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_networked_bad_cert(self): # We feed a "CA" cert that is unrelated to the server's cert import ssl @@ -1783,8 +2099,6 @@ def test_networked_bad_cert(self): h.request('GET', '/') self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_local_unknown_cert(self): # The custom cert isn't known to the default trust bundle import ssl @@ -1795,7 +2109,7 @@ def test_local_unknown_cert(self): self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') def test_local_good_hostname(self): - # The (valid) cert validates the HTTP hostname + # The (valid) cert validates the HTTPS hostname import ssl server = self.make_server(CERT_localhost) context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) @@ -1807,10 +2121,9 @@ def test_local_good_hostname(self): self.addCleanup(resp.close) self.assertEqual(resp.status, 404) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; Flaky on CI") def test_local_bad_hostname(self): - # The (valid) cert doesn't validate the HTTP hostname + # The (valid) cert doesn't validate the HTTPS hostname import ssl server = self.make_server(CERT_fakehostname) context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) @@ -1818,38 +2131,21 @@ def test_local_bad_hostname(self): h = client.HTTPSConnection('localhost', server.port, context=context) with self.assertRaises(ssl.CertificateError): h.request('GET', '/') - # Same with explicit check_hostname=True - with warnings_helper.check_warnings(('', DeprecationWarning)): - h = client.HTTPSConnection('localhost', server.port, - context=context, check_hostname=True) + + # Same with explicit context.check_hostname=True + context.check_hostname = True + h = client.HTTPSConnection('localhost', server.port, context=context) with self.assertRaises(ssl.CertificateError): h.request('GET', '/') - # With check_hostname=False, the mismatching is ignored - context.check_hostname = False - with warnings_helper.check_warnings(('', DeprecationWarning)): - h = client.HTTPSConnection('localhost', server.port, - context=context, check_hostname=False) - h.request('GET', '/nonexistent') - resp = h.getresponse() - resp.close() - h.close() - self.assertEqual(resp.status, 404) - # The context's check_hostname setting is used if one isn't passed to - # HTTPSConnection. + + # With context.check_hostname=False, the mismatching is ignored context.check_hostname = False h = client.HTTPSConnection('localhost', server.port, context=context) h.request('GET', '/nonexistent') resp = h.getresponse() - self.assertEqual(resp.status, 404) resp.close() h.close() - # Passing check_hostname to HTTPSConnection should override the - # context's setting. - with warnings_helper.check_warnings(('', DeprecationWarning)): - h = client.HTTPSConnection('localhost', server.port, - context=context, check_hostname=True) - with self.assertRaises(ssl.CertificateError): - h.request('GET', '/') + self.assertEqual(resp.status, 404) @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), 'http.client.HTTPSConnection not available') @@ -1871,10 +2167,11 @@ def test_host_port(self): self.assertEqual(h, c.host) self.assertEqual(p, c.port) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tls13_pha(self): import ssl - if not ssl.HAS_TLSv1_3: - self.skipTest('TLS 1.3 support required') + if not ssl.HAS_TLSv1_3 or not ssl.HAS_PHA: + self.skipTest('TLS 1.3 PHA support required') # just check status of PHA flag h = client.HTTPSConnection('localhost', 443) self.assertTrue(h._context.post_handshake_auth) @@ -1885,11 +2182,9 @@ def test_tls13_pha(self): self.assertIs(h._context, context) self.assertFalse(h._context.post_handshake_auth) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'key_file, cert_file and check_hostname are deprecated', - DeprecationWarning) - h = client.HTTPSConnection('localhost', 443, context=context, - cert_file=CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT, cert_file=CERT_localhost) + context.post_handshake_auth = True + h = client.HTTPSConnection('localhost', 443, context=context) self.assertTrue(h._context.post_handshake_auth) @@ -2027,11 +2322,12 @@ def test_getting_header_defaultint(self): class TunnelTests(TestCase): def setUp(self): response_text = ( - 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n\r\n' # Reply to CONNECT 'HTTP/1.1 200 OK\r\n' # Reply to HEAD 'Content-Length: 42\r\n\r\n' ) self.host = 'proxy.com' + self.port = client.HTTP_PORT self.conn = client.HTTPConnection(self.host) self.conn._create_connection = self._create_connection(response_text) @@ -2043,15 +2339,45 @@ def create_connection(address, timeout=None, source_address=None): return FakeSocket(response_text, host=address[0], port=address[1]) return create_connection - def test_set_tunnel_host_port_headers(self): + def test_set_tunnel_host_port_headers_add_host_missing(self): tunnel_host = 'destination.com' tunnel_port = 8888 tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'} + tunnel_headers_after = tunnel_headers.copy() + tunnel_headers_after['Host'] = '%s:%d' % (tunnel_host, tunnel_port) self.conn.set_tunnel(tunnel_host, port=tunnel_port, headers=tunnel_headers) self.conn.request('HEAD', '/', '') self.assertEqual(self.conn.sock.host, self.host) - self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertEqual(self.conn.sock.port, self.port) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers_after) + + def test_set_tunnel_host_port_headers_set_host_identical(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)', + 'Host': '%s:%d' % (tunnel_host, tunnel_port)} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers) + + def test_set_tunnel_host_port_headers_set_host_different(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)', + 'Host': '%s:%d' % ('example.com', 4200)} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) self.assertEqual(self.conn._tunnel_host, tunnel_host) self.assertEqual(self.conn._tunnel_port, tunnel_port) self.assertEqual(self.conn._tunnel_headers, tunnel_headers) @@ -2063,17 +2389,96 @@ def test_disallow_set_tunnel_after_connect(self): 'destination.com') def test_connect_with_tunnel(self): - self.conn.set_tunnel('destination.com') + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_with_default_port(self): + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii'), port=d[b'port']) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_with_nonstandard_port(self): + d = { + b'host': b'destination.com', + b'port': 8888, + } + self.conn.set_tunnel(d[b'host'].decode('ascii'), port=d[b'port']) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s:%(port)d\r\n' % d, + self.conn.sock.data) + + # This request is not RFC-valid, but it's been possible with the library + # for years, so don't break it unexpectedly... This also tests + # case-insensitivity when injecting Host: headers if they're missing. + def test_connect_with_tunnel_with_different_host_header(self): + d = { + b'host': b'destination.com', + b'tunnel_host_header': b'example.com:9876', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel( + d[b'host'].decode('ascii'), + headers={'HOST': d[b'tunnel_host_header'].decode('ascii')}) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'HOST: %(tunnel_host_header)s\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_different_host(self): + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_idna(self): + dest = '\u03b4\u03c0\u03b8.gr' + dest_port = b'%s:%d' % (dest.encode('idna'), client.HTTP_PORT) + expected = b'CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n' % ( + dest_port, dest_port) + self.conn.set_tunnel(dest) self.conn.request('HEAD', '/', '') self.assertEqual(self.conn.sock.host, self.host) self.assertEqual(self.conn.sock.port, client.HTTP_PORT) - self.assertIn(b'CONNECT destination.com', self.conn.sock.data) - # issue22095 - self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) - self.assertIn(b'Host: destination.com', self.conn.sock.data) - - # This test should be removed when CONNECT gets the HTTP/1.1 blessing - self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) + self.assertIn(expected, self.conn.sock.data) def test_tunnel_connect_single_send_connection_setup(self): """Regresstion test for https://bugs.python.org/issue43332.""" @@ -2088,17 +2493,39 @@ def test_tunnel_connect_single_send_connection_setup(self): msg=f'unexpected number of send calls: {mock_send.mock_calls}') proxy_setup_data_sent = mock_send.mock_calls[0][1][0] self.assertIn(b'CONNECT destination.com', proxy_setup_data_sent) - self.assertTrue( - proxy_setup_data_sent.endswith(b'\r\n\r\n'), + self.assertEndsWith(proxy_setup_data_sent, b'\r\n\r\n', msg=f'unexpected proxy data sent {proxy_setup_data_sent!r}') def test_connect_put_request(self): - self.conn.set_tunnel('destination.com') + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'PUT / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_put_request_ipv6(self): + self.conn.set_tunnel('[1:2:3::4]', 1234) + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data) + self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data) + + def test_connect_put_request_ipv6_port(self): + self.conn.set_tunnel('[1:2:3::4]:1234') self.conn.request('PUT', '/', '') self.assertEqual(self.conn.sock.host, self.host) self.assertEqual(self.conn.sock.port, client.HTTP_PORT) - self.assertIn(b'CONNECT destination.com', self.conn.sock.data) - self.assertIn(b'Host: destination.com', self.conn.sock.data) + self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data) + self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data) def test_tunnel_debuglog(self): expected_header = 'X-Dummy: 1' @@ -2113,6 +2540,56 @@ def test_tunnel_debuglog(self): lines = output.getvalue().splitlines() self.assertIn('header: {}'.format(expected_header), lines) + def test_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + self.conn.request('PUT', '/', '') + headers = self.conn.get_proxy_response_headers() + self.assertIn(expected_header, headers.items()) + + def test_no_proxy_response_headers(self): + expected_header = ('X-Dummy', '1') + response_text = ( + 'HTTP/1.0 200 OK\r\n' + '{0}\r\n\r\n'.format(':'.join(expected_header)) + ) + + self.conn._create_connection = self._create_connection(response_text) + + self.conn.request('PUT', '/', '') + headers = self.conn.get_proxy_response_headers() + self.assertIsNone(headers) + + def test_tunnel_leak(self): + sock = None + + def _create_connection(address, timeout=None, source_address=None): + nonlocal sock + sock = FakeSocket( + 'HTTP/1.1 404 NOT FOUND\r\n\r\n', + host=address[0], + port=address[1], + ) + return sock + + self.conn._create_connection = _create_connection + self.conn.set_tunnel('destination.com') + exc = None + try: + self.conn.request('HEAD', '/', '') + except OSError as e: + # keeping a reference to exc keeps response alive in the traceback + exc = e + self.assertIsNotNone(exc) + self.assertTrue(sock.file_closed) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 570abcec673..b7885f5d2dd 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -4,10 +4,11 @@ Josip Dzolonga, and Michael Otteneder for the 2007/08 GHOP contest. """ from collections import OrderedDict -from http.server import BaseHTTPRequestHandler, HTTPServer, \ +from http.server import BaseHTTPRequestHandler, HTTPServer, HTTPSServer, \ SimpleHTTPRequestHandler, CGIHTTPRequestHandler from http import server, HTTPStatus +import contextlib import os import socket import sys @@ -26,13 +27,20 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support -from test.support import os_helper -from test.support import threading_helper +from test.support import ( + is_apple, import_helper, os_helper, requires_subprocess, threading_helper +) +try: + import ssl +except ImportError: + ssl = None + +support.requires_working_socket(module=True) class NoLogRequestHandler: def log_message(self, *args): @@ -43,14 +51,49 @@ def read(self, n=None): return '' +class DummyRequestHandler(NoLogRequestHandler, SimpleHTTPRequestHandler): + pass + + +def create_https_server( + certfile, + keyfile=None, + password=None, + *, + address=('localhost', 0), + request_handler=DummyRequestHandler, +): + return HTTPSServer( + address, request_handler, + certfile=certfile, keyfile=keyfile, password=password + ) + + +class TestSSLDisabled(unittest.TestCase): + def test_https_server_raises_runtime_error(self): + with import_helper.isolated_modules(): + sys.modules['ssl'] = None + certfile = certdata_file("keycert.pem") + with self.assertRaises(RuntimeError): + create_https_server(certfile) + + class TestServerThread(threading.Thread): - def __init__(self, test_object, request_handler): + def __init__(self, test_object, request_handler, tls=None): threading.Thread.__init__(self) self.request_handler = request_handler self.test_object = test_object + self.tls = tls def run(self): - self.server = HTTPServer(('localhost', 0), self.request_handler) + if self.tls: + certfile, keyfile, password = self.tls + self.server = create_https_server( + certfile, keyfile, password, + request_handler=self.request_handler, + ) + else: + self.server = HTTPServer(('localhost', 0), self.request_handler) self.test_object.HOST, self.test_object.PORT = self.server.socket.getsockname() self.test_object.server_started.set() self.test_object = None @@ -65,11 +108,15 @@ def stop(self): class BaseTestCase(unittest.TestCase): + + # Optional tuple (certfile, keyfile, password) to use for HTTPS servers. + tls = None + def setUp(self): self._threads = threading_helper.threading_setup() os.environ = os_helper.EnvironmentVarGuard() self.server_started = threading.Event() - self.thread = TestServerThread(self, self.request_handler) + self.thread = TestServerThread(self, self.request_handler, self.tls) self.thread.start() self.server_started.wait() @@ -163,6 +210,27 @@ def test_version_digits(self): res = self.con.getresponse() self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + def test_version_signs_and_underscores(self): + self.con._http_vsn_str = 'HTTP/-9_9_9.+9_9_9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_major_version_number_too_long(self): + self.con._http_vsn_str = 'HTTP/909876543210.0' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_minor_version_number_too_long(self): + self.con._http_vsn_str = 'HTTP/1.909876543210' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + def test_version_none_get(self): self.con._http_vsn_str = '' self.con.putrequest('GET', '/') @@ -292,6 +360,112 @@ def test_head_via_send_error(self): self.assertEqual(b'', data) +class HTTP09ServerTestCase(BaseTestCase): + + class request_handler(NoLogRequestHandler, BaseHTTPRequestHandler): + """Request handler for HTTP/0.9 server.""" + + def do_GET(self): + self.wfile.write(f'OK: here is {self.path}\r\n'.encode()) + + def setUp(self): + super().setUp() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock = self.enterContext(self.sock) + self.sock.connect((self.HOST, self.PORT)) + + def test_simple_get(self): + self.sock.send(b'GET /index.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /index.html\r\n") + + def test_invalid_request(self): + self.sock.send(b'POST /index.html\r\n') + res = self.sock.recv(1024) + self.assertIn(b"Bad HTTP/0.9 request type ('POST')", res) + + def test_single_request(self): + self.sock.send(b'GET /foo.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /foo.html\r\n") + + # Ignore errors if the connection is already closed, + # as this is the expected behavior of HTTP/0.9. + with contextlib.suppress(OSError): + self.sock.send(b'GET /bar.html\r\n') + res = self.sock.recv(1024) + # The server should not process our request. + self.assertEqual(res, b'') + + +def certdata_file(*path): + return os.path.join(os.path.dirname(__file__), "certdata", *path) + + +@unittest.skipIf(ssl is None, "requires ssl") +class BaseHTTPSServerTestCase(BaseTestCase): + CERTFILE = certdata_file("keycert.pem") + ONLYCERT = certdata_file("ssl_cert.pem") + ONLYKEY = certdata_file("ssl_key.pem") + CERTFILE_PROTECTED = certdata_file("keycert.passwd.pem") + ONLYKEY_PROTECTED = certdata_file("ssl_key.passwd.pem") + EMPTYCERT = certdata_file("nullcert.pem") + BADCERT = certdata_file("badcert.pem") + KEY_PASSWORD = "somepass" + BADPASSWORD = "badpass" + + tls = (ONLYCERT, ONLYKEY, None) # values by default + + request_handler = DummyRequestHandler + + def test_get(self): + response = self.request('/') + self.assertEqual(response.status, HTTPStatus.OK) + + def request(self, uri, method='GET', body=None, headers={}): + context = ssl._create_unverified_context() + self.connection = http.client.HTTPSConnection( + self.HOST, self.PORT, context=context + ) + self.connection.request(method, uri, body, headers) + return self.connection.getresponse() + + def test_valid_certdata(self): + valid_certdata= [ + (self.CERTFILE, None, None), + (self.CERTFILE, self.CERTFILE, None), + (self.CERTFILE_PROTECTED, None, self.KEY_PASSWORD), + (self.ONLYCERT, self.ONLYKEY_PROTECTED, self.KEY_PASSWORD), + ] + for certfile, keyfile, password in valid_certdata: + with self.subTest( + certfile=certfile, keyfile=keyfile, password=password + ): + server = create_https_server(certfile, keyfile, password) + self.assertIsInstance(server, HTTPSServer) + server.server_close() + + def test_invalid_certdata(self): + invalid_certdata = [ + (self.BADCERT, None, None), + (self.EMPTYCERT, None, None), + (self.ONLYCERT, None, None), + (self.ONLYKEY, None, None), + (self.ONLYKEY, self.ONLYCERT, None), + (self.CERTFILE_PROTECTED, None, self.BADPASSWORD), + # TODO: test the next case and add same case to test_ssl (We + # specify a cert and a password-protected file, but no password): + # (self.CERTFILE_PROTECTED, None, None), + # see issue #132102 + ] + for certfile, keyfile, password in invalid_certdata: + with self.subTest( + certfile=certfile, keyfile=keyfile, password=password + ): + with self.assertRaises(ssl.SSLError): + create_https_server(certfile, keyfile, password) + + class RequestHandlerLoggingTestCase(BaseTestCase): class request_handler(BaseHTTPRequestHandler): protocol_version = 'HTTP/1.1' @@ -312,8 +486,7 @@ def test_get(self): self.con.request('GET', '/') self.con.getresponse() - self.assertTrue( - err.getvalue().endswith('"GET / HTTP/1.1" 200 -\n')) + self.assertEndsWith(err.getvalue(), '"GET / HTTP/1.1" 200 -\n') def test_err(self): self.con = http.client.HTTPConnection(self.HOST, self.PORT) @@ -324,8 +497,8 @@ def test_err(self): self.con.getresponse() lines = err.getvalue().split('\n') - self.assertTrue(lines[0].endswith('code 404, message File not found')) - self.assertTrue(lines[1].endswith('"ERROR / HTTP/1.1" 404 -')) + self.assertEndsWith(lines[0], 'code 404, message File not found') + self.assertEndsWith(lines[1], '"ERROR / HTTP/1.1" 404 -') class SimpleHTTPServerTestCase(BaseTestCase): @@ -333,7 +506,7 @@ class request_handler(NoLogRequestHandler, SimpleHTTPRequestHandler): pass def setUp(self): - BaseTestCase.setUp(self) + super().setUp() self.cwd = os.getcwd() basetempdir = tempfile.gettempdir() os.chdir(basetempdir) @@ -361,7 +534,7 @@ def tearDown(self): except: pass finally: - BaseTestCase.tearDown(self) + super().tearDown() def check_status_and_reason(self, response, status, data=None): def close_conn(): @@ -388,34 +561,169 @@ def close_conn(): reader.close() return body - @unittest.skipIf(sys.platform == 'darwin', - 'undecodable name cannot always be decoded on macOS') - @unittest.skipIf(sys.platform == 'win32', - 'undecodable name cannot be decoded on win32') - @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, - 'need os_helper.TESTFN_UNDECODABLE') - def test_undecodable_filename(self): + def check_list_dir_dirname(self, dirname, quotedname=None): + fullpath = os.path.join(self.tempdir, dirname) + try: + os.mkdir(os.path.join(self.tempdir, dirname)) + except (OSError, UnicodeEncodeError): + self.skipTest(f'Can not create directory {dirname!a} ' + f'on current file system') + + if quotedname is None: + quotedname = urllib.parse.quote(dirname, errors='surrogatepass') + response = self.request(self.base_url + '/' + quotedname + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + displaypath = html.escape(f'{self.base_url}/{dirname}/', quote=False) enc = sys.getfilesystemencoding() - filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' - with open(os.path.join(self.tempdir, filename), 'wb') as f: - f.write(os_helper.TESTFN_UNDECODABLE) + prefix = f'listing for {displaypath}</'.encode(enc, 'surrogateescape') + self.assertIn(prefix + b'title>', body) + self.assertIn(prefix + b'h1>', body) + + def check_list_dir_filename(self, filename): + fullpath = os.path.join(self.tempdir, filename) + content = ascii(fullpath).encode() + (os_helper.TESTFN_UNDECODABLE or b'\xff') + try: + with open(fullpath, 'wb') as f: + f.write(content) + except OSError: + self.skipTest(f'Can not create file {filename!a} ' + f'on current file system') + response = self.request(self.base_url + '/') - if sys.platform == 'darwin': - # On Mac OS the HFS+ filesystem replaces bytes that aren't valid - # UTF-8 into a percent-encoded value. - for name in os.listdir(self.tempdir): - if name != 'test': # Ignore a filename created in setUp(). - filename = name - break body = self.check_status_and_reason(response, HTTPStatus.OK) quotedname = urllib.parse.quote(filename, errors='surrogatepass') - self.assertIn(('href="%s"' % quotedname) - .encode(enc, 'surrogateescape'), body) - self.assertIn(('>%s<' % html.escape(filename, quote=False)) - .encode(enc, 'surrogateescape'), body) + enc = response.headers.get_content_charset() + self.assertIsNotNone(enc) + self.assertIn((f'href="{quotedname}"').encode('ascii'), body) + displayname = html.escape(filename, quote=False) + self.assertIn(f'>{displayname}<'.encode(enc, 'surrogateescape'), body) + response = self.request(self.base_url + '/' + quotedname) - self.check_status_and_reason(response, HTTPStatus.OK, - data=os_helper.TESTFN_UNDECODABLE) + self.check_status_and_reason(response, HTTPStatus.OK, data=content) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_dirname(self): + dirname = os_helper.TESTFN_NONASCII + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_filename(self): + filename = os_helper.TESTFN_NONASCII + '.txt' + self.check_list_dir_filename(filename) + + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') + @unittest.skipIf(sys.platform == 'win32', + 'undecodable name cannot be decoded on win32') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'need os_helper.TESTFN_UNDECODABLE') + def test_list_dir_undecodable_dirname(self): + dirname = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') + @unittest.skipIf(sys.platform == 'win32', + 'undecodable name cannot be decoded on win32') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'need os_helper.TESTFN_UNDECODABLE') + def test_list_dir_undecodable_filename(self): + filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_undecodable_dirname2(self): + dirname = '\ufffd.dir' + self.check_list_dir_dirname(dirname, quotedname='%ff.dir') + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_dirname(self): + dirname = os_helper.TESTFN_UNENCODABLE + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_filename(self): + filename = os_helper.TESTFN_UNENCODABLE + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_escape_dirname(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&amp;', '&amp', '<i>', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + dirname = name + '.dir' + self.check_list_dir_dirname(dirname, + quotedname=urllib.parse.quote(dirname, safe='&<>\'"')) + + def test_list_dir_escape_filename(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&amp;', '&amp', '<i>', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + filename = name + '.txt' + self.check_list_dir_filename(filename) + os_helper.unlink(os.path.join(self.tempdir, filename)) + + def test_list_dir_with_query_and_fragment(self): + prefix = f'listing for {self.base_url}/</'.encode('latin1') + response = self.request(self.base_url + '/#123').read() + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) + response = self.request(self.base_url + '/?x=123').read() + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) + + def test_get_dir_redirect_location_domain_injection_bug(self): + """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. + + //netloc/ in a Location header is a redirect to a new host. + https://github.com/python/cpython/issues/87389 + + This checks that a path resolving to a directory on our server cannot + resolve into a redirect to another server. + """ + os.mkdir(os.path.join(self.tempdir, 'existing_directory')) + url = f'/python.org/..%2f..%2f..%2f..%2f..%2f../%0a%0d/../{self.tempdir_name}/existing_directory' + expected_location = f'{url}/' # /python.org.../ single slash single prefix, trailing slash + # Canonicalizes to /tmp/tempdir_name/existing_directory which does + # exist and is a dir, triggering the 301 redirect logic. + response = self.request(url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + location = response.getheader('Location') + self.assertEqual(location, expected_location, msg='non-attack failed!') + + # //python.org... multi-slash prefix, no trailing slash + attack_url = f'/{url}' + response = self.request(attack_url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + location = response.getheader('Location') + self.assertNotStartsWith(location, '//') + self.assertEqual(location, expected_location, + msg='Expected Location header to start with a single / and ' + 'end with a / as this is a directory redirect.') + + # ///python.org... triple-slash prefix, no trailing slash + attack3_url = f'//{url}' + response = self.request(attack3_url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader('Location'), expected_location) + + # If the second word in the http request (Request-URI for the http + # method) is a full URI, we don't worry about it, as that'll be parsed + # and reassembled as a full URI within BaseHTTPRequestHandler.send_head + # so no errant scheme-less //netloc//evil.co/ domain mixup can happen. + attack_scheme_netloc_2slash_url = f'https://pypi.org/{url}' + expected_scheme_netloc_location = f'{attack_scheme_netloc_2slash_url}/' + response = self.request(attack_scheme_netloc_2slash_url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + location = response.getheader('Location') + # We're just ensuring that the scheme and domain make it through, if + # there are or aren't multiple slashes at the start of the path that + # follows that isn't important in this Location: header. + self.assertStartsWith(location, 'https://pypi.org/') def test_get(self): #constructs the path relative to the root directory of the HTTPServer @@ -424,10 +732,19 @@ def test_get(self): # check for trailing "/" which should return 404. See Issue17324 response = self.request(self.base_url + '/test/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2f') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2F') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request(self.base_url + '/') self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2f') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2F') + self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.base_url) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), self.base_url + "/") self.assertEqual(response.getheader("Content-Length"), "0") response = self.request(self.base_url + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) @@ -439,6 +756,9 @@ def test_get(self): self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request('/' + 'ThisDoesNotExist' + '/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + os.makedirs(os.path.join(self.tempdir, 'spam', 'index.html')) + response = self.request(self.base_url + '/spam/') + self.check_status_and_reason(response, HTTPStatus.OK) data = b"Dummy index file\r\n" with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: @@ -530,6 +850,8 @@ def test_path_without_leading_slash(self): self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), + self.tempdir_name + "/") response = self.request(self.tempdir_name + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name + '?hi=1') @@ -537,27 +859,6 @@ def test_path_without_leading_slash(self): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") - def test_html_escape_filename(self): - filename = '<test&>.txt' - fullpath = os.path.join(self.tempdir, filename) - - try: - open(fullpath, 'wb').close() - except OSError: - raise unittest.SkipTest('Can not create file %s on current file ' - 'system' % filename) - - try: - response = self.request(self.base_url + '/') - body = self.check_status_and_reason(response, HTTPStatus.OK) - enc = response.headers.get_content_charset() - finally: - os.unlink(fullpath) # avoid affecting test_undecodable_filename - - self.assertIsNotNone(enc) - html_text = '>%s<' % html.escape(filename, quote=False) - self.assertIn(html_text.encode(enc), body) - cgi_file1 = """\ #!%s @@ -569,14 +870,19 @@ def test_html_escape_filename(self): cgi_file2 = """\ #!%s -import cgi +import os +import sys +import urllib.parse print("Content-type: text/html") print() -form = cgi.FieldStorage() -print("%%s, %%s, %%s" %% (form.getfirst("spam"), form.getfirst("eggs"), - form.getfirst("bacon"))) +content_length = int(os.environ["CONTENT_LENGTH"]) +query_string = sys.stdin.buffer.read(content_length) +params = {key.decode("utf-8"): val.decode("utf-8") + for key, val in urllib.parse.parse_qsl(query_string)} + +print("%%s, %%s, %%s" %% (params["spam"], params["eggs"], params["bacon"])) """ cgi_file4 = """\ @@ -607,19 +913,40 @@ def test_html_escape_filename(self): print("</pre>") """ -@unittest.skipIf(not hasattr(os, '_exit'), - "TODO: RUSTPYTHON, run_cgi in http/server.py gets stuck as os._exit(127) doesn't currently kill forked processes") +cgi_file7 = """\ +#!%s +import os +import sys + +print("Content-type: text/plain") +print() + +content_length = int(os.environ["CONTENT_LENGTH"]) +body = sys.stdin.buffer.read(content_length) + +print(f"{content_length} {len(body)}") +""" + + @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") +@requires_subprocess() class CGIHTTPServerTestCase(BaseTestCase): class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): - pass + _test_case_self = None # populated by each setUp() method call. + + def __init__(self, *args, **kwargs): + with self._test_case_self.assertWarnsRegex( + DeprecationWarning, + r'http\.server\.CGIHTTPRequestHandler'): + # This context also happens to catch and silence the + # threading DeprecationWarning from os.fork(). + super().__init__(*args, **kwargs) linesep = os.linesep.encode('ascii') - # TODO: RUSTPYTHON - linesep = b'\n' def setUp(self): + self.request_handler._test_case_self = self # practical, but yuck. BaseTestCase.setUp(self) self.cwd = os.getcwd() self.parent_dir = tempfile.mkdtemp() @@ -639,12 +966,13 @@ def setUp(self): self.file3_path = None self.file4_path = None self.file5_path = None + self.file6_path = None + self.file7_path = None # The shebang line should be pure ASCII: use symlink if possible. # See issue #7668. self._pythonexe_symlink = None - # TODO: RUSTPYTHON; dl_nt not supported yet - if os_helper.can_symlink() and sys.platform != 'win32': + if os_helper.can_symlink(): self.pythonexe = os.path.join(self.parent_dir, 'python') self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__() else: @@ -694,9 +1022,15 @@ def setUp(self): file6.write(cgi_file6 % self.pythonexe) os.chmod(self.file6_path, 0o777) + self.file7_path = os.path.join(self.cgi_dir, 'file7.py') + with open(self.file7_path, 'w', encoding='utf-8') as file7: + file7.write(cgi_file7 % self.pythonexe) + os.chmod(self.file7_path, 0o777) + os.chdir(self.parent_dir) def tearDown(self): + self.request_handler._test_case_self = None try: os.chdir(self.cwd) if self._pythonexe_symlink: @@ -715,11 +1049,16 @@ def tearDown(self): os.remove(self.file5_path) if self.file6_path: os.remove(self.file6_path) + if self.file7_path: + os.remove(self.file7_path) os.rmdir(self.cgi_child_dir) os.rmdir(self.cgi_dir) os.rmdir(self.cgi_dir_in_sub_dir) os.rmdir(self.sub_dir_2) os.rmdir(self.sub_dir_1) + # The 'gmon.out' file can be written in the current working + # directory if C-level code profiling with gprof is enabled. + os_helper.unlink(os.path.join(self.parent_dir, 'gmon.out')) os.rmdir(self.parent_dir) finally: BaseTestCase.tearDown(self) @@ -766,8 +1105,6 @@ def test_url_collapse_path(self): msg='path = %r\nGot: %r\nWanted: %r' % (path, actual, expected)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_headers_and_content(self): res = self.request('/cgi-bin/file1.py') self.assertEqual( @@ -778,8 +1115,6 @@ def test_issue19435(self): res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh') self.assertEqual(res.status, HTTPStatus.NOT_FOUND) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_post(self): params = urllib.parse.urlencode( {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) @@ -788,13 +1123,28 @@ def test_post(self): self.assertEqual(res.read(), b'1, python, 123456' + self.linesep) + def test_large_content_length(self): + for w in range(15, 25): + size = 1 << w + body = b'X' * size + headers = {'Content-Length' : str(size)} + res = self.request('/cgi-bin/file7.py', 'POST', body, headers) + self.assertEqual(res.read(), b'%d %d' % (size, size) + self.linesep) + + @unittest.skip("TODO: RUSTPYTHON; flaky test") + def test_large_content_length_truncated(self): + with support.swap_attr(self.request_handler, 'timeout', 0.001): + for w in range(18, 65): + size = 1 << w + headers = {'Content-Length' : str(size)} + res = self.request('/cgi-bin/file1.py', 'POST', b'x', headers) + self.assertEqual(res.read(), b'Hello World' + self.linesep) + def test_invaliduri(self): res = self.request('/cgi-bin/invalid') res.read() self.assertEqual(res.status, HTTPStatus.NOT_FOUND) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_authorization(self): headers = {b'Authorization' : b'Basic ' + base64.b64encode(b'username:pass')} @@ -803,8 +1153,6 @@ def test_authorization(self): (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_no_leading_slash(self): # http://bugs.python.org/issue2254 res = self.request('cgi-bin/file1.py') @@ -812,8 +1160,6 @@ def test_no_leading_slash(self): (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_os_environ_is_not_altered(self): signature = "Test CGI Server" os.environ['SERVER_SOFTWARE'] = signature @@ -823,32 +1169,24 @@ def test_os_environ_is_not_altered(self): (res.read(), res.getheader('Content-type'), res.status)) self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_urlquote_decoding_in_cgi_check(self): res = self.request('/cgi-bin%2ffile1.py') self.assertEqual( (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_nested_cgi_path_issue21323(self): res = self.request('/cgi-bin/child-dir/file3.py') self.assertEqual( (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_query_with_multiple_question_mark(self): res = self.request('/cgi-bin/file4.py?a=b?c=d') self.assertEqual( (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_query_with_continuous_slashes(self): res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//') self.assertEqual( @@ -856,8 +1194,6 @@ def test_query_with_continuous_slashes(self): 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_cgi_path_in_sub_directories(self): try: CGIHTTPRequestHandler.cgi_directories.append('/sub/dir/cgi-bin') @@ -868,8 +1204,6 @@ def test_cgi_path_in_sub_directories(self): finally: CGIHTTPRequestHandler.cgi_directories.remove('/sub/dir/cgi-bin') - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON; works only on windows") def test_accept(self): browser_accept = \ 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' @@ -961,6 +1295,27 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + log_message(self.handler, '"GET /foo\\b"ar\007 HTTP/1.0"') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + self.assertIn(r'"GET /foo\\b"ar\x07 HTTP/1.0"', lines[4]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) @@ -997,7 +1352,7 @@ def test_extra_space(self): b'Host: dummy\r\n' b'\r\n' ) - self.assertTrue(result[0].startswith(b'HTTP/1.1 400 ')) + self.assertStartsWith(result[0], b'HTTP/1.1 400 ') self.verify_expected_headers(result[1:result.index(b'\r\n')]) self.assertFalse(self.handler.get_called) @@ -1111,7 +1466,7 @@ def test_request_length(self): # Issue #10714: huge request lines are discarded, to avoid Denial # of Service attacks. result = self.send_typical_request(b'GET ' + b'x' * 65537) - self.assertEqual(result[0], b'HTTP/1.1 414 Request-URI Too Long\r\n') + self.assertEqual(result[0], b'HTTP/1.1 414 URI Too Long\r\n') self.assertFalse(self.handler.get_called) self.assertIsInstance(self.handler.requestline, str) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py new file mode 100644 index 00000000000..9155a43a06e --- /dev/null +++ b/Lib/test/test_imaplib.py @@ -0,0 +1,1128 @@ +from test import support +from test.support import socket_helper + +from contextlib import contextmanager +import imaplib +import os.path +import socketserver +import time +import calendar +import threading +import re +import socket + +from test.support import verbose, run_with_tz, run_with_locale, cpython_only +from test.support import hashlib_helper +from test.support import threading_helper +import unittest +from unittest import mock +from datetime import datetime, timezone, timedelta +try: + import ssl +except ImportError: + ssl = None + +support.requires_working_socket(module=True) + +CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "certdata", "keycert3.pem") +CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "certdata", "pycacert.pem") + + +class TestImaplib(unittest.TestCase): + + def test_Internaldate2tuple(self): + t0 = calendar.timegm((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + tt = imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "01-Jan-2000 00:00:00 +0000")') + self.assertEqual(time.mktime(tt), t0) + tt = imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "01-Jan-2000 11:30:00 +1130")') + self.assertEqual(time.mktime(tt), t0) + tt = imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "31-Dec-1999 12:30:00 -1130")') + self.assertEqual(time.mktime(tt), t0) + + @run_with_tz('MST+07MDT,M4.1.0,M10.5.0') + def test_Internaldate2tuple_issue10941(self): + self.assertNotEqual(imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "02-Apr-2000 02:30:00 +0000")'), + imaplib.Internaldate2tuple( + b'25 (INTERNALDATE "02-Apr-2000 03:30:00 +0000")')) + + def timevalues(self): + return [2000000000, 2000000000.0, time.localtime(2000000000), + (2033, 5, 18, 5, 33, 20, -1, -1, -1), + (2033, 5, 18, 5, 33, 20, -1, -1, 1), + datetime.fromtimestamp(2000000000, + timezone(timedelta(0, 2 * 60 * 60))), + '"18-May-2033 05:33:20 +0200"'] + + @run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '') + # DST rules included to work around quirk where the Gnu C library may not + # otherwise restore the previous time zone + @run_with_tz('STD-1DST,M3.2.0,M11.1.0') + def test_Time2Internaldate(self): + expected = '"18-May-2033 05:33:20 +0200"' + + for t in self.timevalues(): + internal = imaplib.Time2Internaldate(t) + self.assertEqual(internal, expected) + + def test_that_Time2Internaldate_returns_a_result(self): + # Without tzset, we can check only that it successfully + # produces a result, not the correctness of the result itself, + # since the result depends on the timezone the machine is in. + for t in self.timevalues(): + imaplib.Time2Internaldate(t) + + @socket_helper.skip_if_tcp_blackhole + def test_imap4_host_default_value(self): + # Check whether the IMAP4_PORT is truly unavailable. + with socket.socket() as s: + try: + s.connect(('', imaplib.IMAP4_PORT)) + self.skipTest( + "Cannot run the test with local IMAP server running.") + except socket.error: + pass + + # This is the exception that should be raised. + expected_errnos = socket_helper.get_socket_conn_refused_errs() + with self.assertRaises(OSError) as cm: + imaplib.IMAP4() + self.assertIn(cm.exception.errno, expected_errnos) + + +if ssl: + class SecureTCPServer(socketserver.TCPServer): + + def get_request(self): + newsocket, fromaddr = self.socket.accept() + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(CERTFILE) + connstream = context.wrap_socket(newsocket, server_side=True) + return connstream, fromaddr + + IMAP4_SSL = imaplib.IMAP4_SSL + +else: + + class SecureTCPServer: + pass + + IMAP4_SSL = None + + +class SimpleIMAPHandler(socketserver.StreamRequestHandler): + timeout = support.LOOPBACK_TIMEOUT + continuation = None + capabilities = '' + + def setup(self): + super().setup() + self.server.is_selected = False + self.server.logged = None + + def _send(self, message): + if verbose: + print("SENT: %r" % message.strip()) + self.wfile.write(message) + + def _send_line(self, message): + self._send(message + b'\r\n') + + def _send_textline(self, message): + self._send_line(message.encode('ASCII')) + + def _send_tagged(self, tag, code, message): + self._send_textline(' '.join((tag, code, message))) + + def handle(self): + # Send a welcome message. + self._send_textline('* OK IMAP4rev1') + while 1: + # Gather up input until we receive a line terminator or we timeout. + # Accumulate read(1) because it's simpler to handle the differences + # between naked sockets and SSL sockets. + line = b'' + while 1: + try: + part = self.rfile.read(1) + if part == b'': + # Naked sockets return empty strings.. + return + line += part + except OSError: + # ..but SSLSockets raise exceptions. + return + if line.endswith(b'\r\n'): + break + + if verbose: + print('GOT: %r' % line.strip()) + if self.continuation: + try: + self.continuation.send(line) + except StopIteration: + self.continuation = None + continue + splitline = line.decode('ASCII').split() + tag = splitline[0] + cmd = splitline[1] + args = splitline[2:] + + if hasattr(self, 'cmd_' + cmd): + continuation = getattr(self, 'cmd_' + cmd)(tag, args) + if continuation: + self.continuation = continuation + next(continuation) + else: + self._send_tagged(tag, 'BAD', cmd + ' unknown') + + def cmd_CAPABILITY(self, tag, args): + caps = ('IMAP4rev1 ' + self.capabilities + if self.capabilities + else 'IMAP4rev1') + self._send_textline('* CAPABILITY ' + caps) + self._send_tagged(tag, 'OK', 'CAPABILITY completed') + + def cmd_LOGOUT(self, tag, args): + self.server.logged = None + self._send_textline('* BYE IMAP4ref1 Server logging out') + self._send_tagged(tag, 'OK', 'LOGOUT completed') + + def cmd_LOGIN(self, tag, args): + self.server.logged = args[0] + self._send_tagged(tag, 'OK', 'LOGIN completed') + + def cmd_SELECT(self, tag, args): + self.server.is_selected = True + self._send_line(b'* 2 EXISTS') + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_UNSELECT(self, tag, args): + if self.server.is_selected: + self.server.is_selected = False + self._send_tagged(tag, 'OK', 'Returned to authenticated state. (Success)') + else: + self._send_tagged(tag, 'BAD', 'No mailbox selected') + + +class IdleCmdDenyHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + self._send_tagged(tag, 'NO', 'IDLE is not allowed at this time') + + +class IdleCmdHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + # pre-idle-continuation response + self._send_line(b'* 0 EXISTS') + self._send_textline('+ idling') + # simple response + self._send_line(b'* 2 EXISTS') + # complex response: fragmented data due to literal string + self._send_line(b'* 1 FETCH (BODY[HEADER.FIELDS (DATE)] {41}') + self._send(b'Date: Fri, 06 Dec 2024 06:00:00 +0000\r\n\r\n') + self._send_line(b')') + # simple response following a fragmented one + self._send_line(b'* 3 EXISTS') + # response arriving later + time.sleep(1) + self._send_line(b'* 1 RECENT') + r = yield + if r == b'DONE\r\n': + self._send_line(b'* 9 RECENT') + self._send_tagged(tag, 'OK', 'Idle completed') + else: + self._send_tagged(tag, 'BAD', 'Expected DONE') + + +class IdleCmdDelayedPacketHandler(SimpleIMAPHandler): + capabilities = 'IDLE' + def cmd_IDLE(self, tag, args): + self._send_textline('+ idling') + # response line spanning multiple packets, the last one delayed + self._send(b'* 1 EX') + time.sleep(0.2) + self._send(b'IS') + time.sleep(1) + self._send(b'TS\r\n') + r = yield + if r == b'DONE\r\n': + self._send_tagged(tag, 'OK', 'Idle completed') + else: + self._send_tagged(tag, 'BAD', 'Expected DONE') + + +class AuthHandler_CRAM_MD5(SimpleIMAPHandler): + capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' + 'VzdG9uLm1jaS5uZXQ=') + r = yield + if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' + b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): + self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') + else: + self._send_tagged(tag, 'NO', 'No access') + + +class NewIMAPTestsMixin: + client = None + + def _setup(self, imap_handler, connect=True): + """ + Sets up imap_handler for tests. imap_handler should inherit from either: + - SimpleIMAPHandler - for testing IMAP commands, + - socketserver.StreamRequestHandler - if raw access to stream is needed. + Returns (client, server). + """ + class TestTCPServer(self.server_class): + def handle_error(self, request, client_address): + """ + End request and raise the error if one occurs. + """ + self.close_request(request) + self.server_close() + raise + + self.addCleanup(self._cleanup) + self.server = self.server_class((socket_helper.HOST, 0), imap_handler) + self.thread = threading.Thread( + name=self._testMethodName+'-server', + target=self.server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval': 0.01}) + self.thread.daemon = True # In case this function raises. + self.thread.start() + + if connect: + self.client = self.imap_class(*self.server.server_address) + + return self.client, self.server + + def _cleanup(self): + """ + Cleans up the test server. This method should not be called manually, + it is added to the cleanup queue in the _setup method already. + """ + # if logout was called already we'd raise an exception trying to + # shutdown the client once again + if self.client is not None and self.client.state != 'LOGOUT': + self.client.shutdown() + # cleanup the server + self.server.shutdown() + self.server.server_close() + threading_helper.join_thread(self.thread) + # Explicitly clear the attribute to prevent dangling thread + self.thread = None + + def test_EOF_without_complete_welcome_message(self): + # http://bugs.python.org/issue5949 + class EOFHandler(socketserver.StreamRequestHandler): + def handle(self): + self.wfile.write(b'* OK') + _, server = self._setup(EOFHandler, connect=False) + self.assertRaises(imaplib.IMAP4.abort, self.imap_class, + *server.server_address) + + def test_line_termination(self): + class BadNewlineHandler(SimpleIMAPHandler): + def cmd_CAPABILITY(self, tag, args): + self._send(b'* CAPABILITY IMAP4rev1 AUTH\n') + self._send_tagged(tag, 'OK', 'CAPABILITY completed') + _, server = self._setup(BadNewlineHandler, connect=False) + self.assertRaises(imaplib.IMAP4.abort, self.imap_class, + *server.server_address) + + def test_enable_raises_error_if_not_AUTH(self): + class EnableHandler(SimpleIMAPHandler): + capabilities = 'AUTH ENABLE UTF8=ACCEPT' + client, _ = self._setup(EnableHandler) + self.assertFalse(client.utf8_enabled) + with self.assertRaisesRegex(imaplib.IMAP4.error, 'ENABLE.*NONAUTH'): + client.enable('foo') + self.assertFalse(client.utf8_enabled) + + def test_enable_raises_error_if_no_capability(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'does not support ENABLE'): + client.enable('foo') + + def test_enable_UTF8_raises_error_if_not_supported(self): + client, _ = self._setup(SimpleIMAPHandler) + typ, data = client.login('user', 'pass') + self.assertEqual(typ, 'OK') + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'does not support ENABLE'): + client.enable('UTF8=ACCEPT') + + def test_enable_UTF8_True_append(self): + class UTF8AppendServer(SimpleIMAPHandler): + capabilities = 'ENABLE UTF8=ACCEPT' + def cmd_ENABLE(self, tag, args): + self._send_tagged(tag, 'OK', 'ENABLE successful') + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + def cmd_APPEND(self, tag, args): + self._send_textline('+') + self.server.response = args + literal = yield + self.server.response.append(literal) + literal = yield + self.server.response.append(literal) + self._send_tagged(tag, 'OK', 'okay') + client, server = self._setup(UTF8AppendServer) + self.assertEqual(client._encoding, 'ascii') + code, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') # b64 encoded 'fake' + code, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(code, 'OK') + self.assertEqual(client._encoding, 'utf-8') + msg_string = 'Subject: üñí©öðé' + typ, data = client.append( + None, None, None, (msg_string + '\n').encode('utf-8')) + self.assertEqual(typ, 'OK') + self.assertEqual(server.response, + ['INBOX', 'UTF8', + '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'), + b')\r\n' ]) + + def test_search_disallows_charset_in_utf8_mode(self): + class UTF8Server(SimpleIMAPHandler): + capabilities = 'AUTH ENABLE UTF8=ACCEPT' + def cmd_ENABLE(self, tag, args): + self._send_tagged(tag, 'OK', 'ENABLE successful') + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + client, _ = self._setup(UTF8Server) + typ, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(typ, 'OK') + typ, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(typ, 'OK') + self.assertTrue(client.utf8_enabled) + with self.assertRaisesRegex(imaplib.IMAP4.error, 'charset.*UTF8'): + client.search('foo', 'bar') + + def test_bad_auth_name(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_tagged(tag, 'NO', + 'unrecognized authentication type {}'.format(args[0])) + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'unrecognized authentication type METHOD'): + client.authenticate('METHOD', lambda: 1) + + def test_invalid_authentication(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.response = yield + self._send_tagged(tag, 'NO', '[AUTHENTICATIONFAILED] invalid') + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'\[AUTHENTICATIONFAILED\] invalid'): + client.authenticate('MYAUTH', lambda x: b'fake') + + def test_valid_authentication_bytes(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + client, server = self._setup(MyServer) + code, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') # b64 encoded 'fake' + + def test_valid_authentication_plain_text(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + client, server = self._setup(MyServer) + code, _ = client.authenticate('MYAUTH', lambda x: 'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') # b64 encoded 'fake' + + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def test_login_cram_md5_bytes(self): + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) + ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf") + self.assertEqual(ret, "OK") + + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def test_login_cram_md5_plain_text(self): + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) + ret, _ = client.login_cram_md5("tim", "tanstaaftanstaaf") + self.assertEqual(ret, "OK") + + def test_login_cram_md5_blocked(self): + def side_effect(*a, **kw): + raise ValueError + + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) + msg = re.escape("CRAM-MD5 authentication is not supported") + with ( + mock.patch("hmac.HMAC", side_effect=side_effect), + self.assertRaisesRegex(imaplib.IMAP4.error, msg) + ): + client.login_cram_md5("tim", b"tanstaaftanstaaf") + + def test_aborted_authentication(self): + class MyServer(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.response = yield + if self.response == b'*\r\n': + self._send_tagged( + tag, + 'NO', + '[AUTHENTICATIONFAILED] aborted') + else: + self._send_tagged(tag, 'OK', 'MYAUTH successful') + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'\[AUTHENTICATIONFAILED\] aborted'): + client.authenticate('MYAUTH', lambda x: None) + + @mock.patch('imaplib._MAXLINE', 10) + def test_linetoolong(self): + class TooLongHandler(SimpleIMAPHandler): + def handle(self): + # send response line longer than the limit set in the next line + self.wfile.write(b'* OK ' + 11 * b'x' + b'\r\n') + _, server = self._setup(TooLongHandler, connect=False) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'got more than 10 bytes'): + self.imap_class(*server.server_address) + + def test_simple_with_statement(self): + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address): + pass + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'socket' object has no attribute 'timeout'. Did you mean: 'gettimeout'? + def test_imaplib_timeout_test(self): + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address, timeout=None) as client: + self.assertEqual(client.sock.timeout, None) + with self.imap_class(*server.server_address, timeout=support.LOOPBACK_TIMEOUT) as client: + self.assertEqual(client.sock.timeout, support.LOOPBACK_TIMEOUT) + with self.assertRaises(ValueError): + self.imap_class(*server.server_address, timeout=0) + + def test_imaplib_timeout_functionality_test(self): + class TimeoutHandler(SimpleIMAPHandler): + def handle(self): + time.sleep(1) + SimpleIMAPHandler.handle(self) + + _, server = self._setup(TimeoutHandler) + addr = server.server_address[1] + with self.assertRaises(TimeoutError): + client = self.imap_class("localhost", addr, timeout=0.001) + + def test_with_statement(self): + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + self.assertIsNone(server.logged) + + def test_with_statement_logout(self): + # It is legal to log out explicitly inside the with block + _, server = self._setup(SimpleIMAPHandler, connect=False) + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + imap.logout() + self.assertIsNone(server.logged) + self.assertIsNone(server.logged) + + # command tests + + def test_idle_capability(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertRaisesRegex(imaplib.IMAP4.error, + 'does not support IMAP4 IDLE'): + with client.idle(): + pass + + def test_idle_denied(self): + client, _ = self._setup(IdleCmdDenyHandler) + client.login('user', 'pass') + with self.assertRaises(imaplib.IMAP4.error): + with client.idle() as idler: + pass + + def test_idle_iter(self): + client, _ = self._setup(IdleCmdHandler) + client.login('user', 'pass') + with client.idle() as idler: + # iteration should include response between 'IDLE' & '+ idling' + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'0'])) + # iteration should produce responses + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'2'])) + # fragmented response (with literal string) should arrive whole + expected_fetch_data = [ + (b'1 (BODY[HEADER.FIELDS (DATE)] {41}', + b'Date: Fri, 06 Dec 2024 06:00:00 +0000\r\n\r\n'), + b')'] + typ, data = next(idler) + self.assertEqual(typ, 'FETCH') + self.assertEqual(data, expected_fetch_data) + # response after a fragmented one should arrive separately + response = next(idler) + self.assertEqual(response, ('EXISTS', [b'3'])) + # iteration should have consumed untagged responses + _, data = client.response('EXISTS') + self.assertEqual(data, [None]) + # responses not iterated should be available after idle + _, data = client.response('RECENT') + self.assertEqual(data[0], b'1') + # responses received after 'DONE' should be available after idle + self.assertEqual(data[1], b'9') + + def test_idle_burst(self): + client, _ = self._setup(IdleCmdHandler) + client.login('user', 'pass') + # burst() should yield immediately available responses + with client.idle() as idler: + batch = list(idler.burst()) + self.assertEqual(len(batch), 4) + # burst() should not have consumed later responses + _, data = client.response('RECENT') + self.assertEqual(data, [b'1', b'9']) + + def test_idle_delayed_packet(self): + client, _ = self._setup(IdleCmdDelayedPacketHandler) + client.login('user', 'pass') + # If our readline() implementation fails to preserve line fragments + # when idle timeouts trigger, a response spanning delayed packets + # can be corrupted, leaving the protocol stream in a bad state. + try: + with client.idle(0.5) as idler: + self.assertRaises(StopIteration, next, idler) + except client.abort as err: + self.fail('multi-packet response was corrupted by idle timeout') + + def test_login(self): + client, _ = self._setup(SimpleIMAPHandler) + typ, data = client.login('user', 'pass') + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'LOGIN completed') + self.assertEqual(client.state, 'AUTH') + + def test_logout(self): + client, _ = self._setup(SimpleIMAPHandler) + typ, data = client.login('user', 'pass') + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'LOGIN completed') + typ, data = client.logout() + self.assertEqual(typ, 'BYE', (typ, data)) + self.assertEqual(data[0], b'IMAP4ref1 Server logging out', (typ, data)) + self.assertEqual(client.state, 'LOGOUT') + + def test_lsub(self): + class LsubCmd(SimpleIMAPHandler): + def cmd_LSUB(self, tag, args): + self._send_textline('* LSUB () "." directoryA') + return self._send_tagged(tag, 'OK', 'LSUB completed') + client, _ = self._setup(LsubCmd) + client.login('user', 'pass') + typ, data = client.lsub() + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'() "." directoryA') + + def test_unselect(self): + client, _ = self._setup(SimpleIMAPHandler) + client.login('user', 'pass') + typ, data = client.select() + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'2') + + typ, data = client.unselect() + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'Returned to authenticated state. (Success)') + self.assertEqual(client.state, 'AUTH') + + # property tests + + def test_file_property_should_not_be_accessed(self): + client, _ = self._setup(SimpleIMAPHandler) + # the 'file' property replaced a private attribute that is now unsafe + with self.assertWarns(RuntimeWarning): + client.file + + +class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): + imap_class = imaplib.IMAP4 + server_class = socketserver.TCPServer + + +@unittest.skipUnless(ssl, "SSL not available") +class NewIMAPSSLTests(NewIMAPTestsMixin, unittest.TestCase): + imap_class = IMAP4_SSL + server_class = SecureTCPServer + + def test_ssl_raises(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ssl_context.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ssl_context.check_hostname, True) + ssl_context.load_verify_locations(CAFILE) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.CertificateError, regex): + _, server = self._setup(SimpleIMAPHandler, connect=False) + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) + client.shutdown() + + def test_ssl_verified(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(CAFILE) + + _, server = self._setup(SimpleIMAPHandler, connect=False) + client = self.imap_class("localhost", server.server_address[1], + ssl_context=ssl_context) + client.shutdown() + +class ThreadedNetworkedTests(unittest.TestCase): + server_class = socketserver.TCPServer + imap_class = imaplib.IMAP4 + + def make_server(self, addr, hdlr): + + class MyServer(self.server_class): + def handle_error(self, request, client_address): + self.close_request(request) + self.server_close() + raise + + if verbose: + print("creating server") + server = MyServer(addr, hdlr) + self.assertEqual(server.server_address, server.socket.getsockname()) + + if verbose: + print("server created") + print("ADDR =", addr) + print("CLASS =", self.server_class) + print("HDLR =", server.RequestHandlerClass) + + t = threading.Thread( + name='%s serving' % self.server_class, + target=server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval': 0.01}) + t.daemon = True # In case this function raises. + t.start() + if verbose: + print("server running") + return server, t + + def reap_server(self, server, thread): + if verbose: + print("waiting for server") + server.shutdown() + server.server_close() + thread.join() + if verbose: + print("done") + + @contextmanager + def reaped_server(self, hdlr): + server, thread = self.make_server((socket_helper.HOST, 0), hdlr) + try: + yield server + finally: + self.reap_server(server, thread) + + @contextmanager + def reaped_pair(self, hdlr): + with self.reaped_server(hdlr) as server: + client = self.imap_class(*server.server_address) + try: + yield server, client + finally: + client.logout() + + @threading_helper.reap_threads + def test_connect(self): + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class(*server.server_address) + client.shutdown() + + @threading_helper.reap_threads + def test_bracket_flags(self): + + # This violates RFC 3501, which disallows ']' characters in tag names, + # but imaplib has allowed producing such tags forever, other programs + # also produce them (eg: OtherInbox's Organizer app as of 20140716), + # and Gmail, for example, accepts them and produces them. So we + # support them. See issue #21815. + + class BracketFlagHandler(SimpleIMAPHandler): + + def handle(self): + self.flags = ['Answered', 'Flagged', 'Deleted', 'Seen', 'Draft'] + super().handle() + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + def cmd_SELECT(self, tag, args): + flag_msg = ' \\'.join(self.flags) + self._send_line(('* FLAGS (%s)' % flag_msg).encode('ascii')) + self._send_line(b'* 2 EXISTS') + self._send_line(b'* 0 RECENT') + msg = ('* OK [PERMANENTFLAGS %s \\*)] Flags permitted.' + % flag_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_STORE(self, tag, args): + new_flags = args[2].strip('(').strip(')').split() + self.flags.extend(new_flags) + flags_msg = '(FLAGS (%s))' % ' \\'.join(self.flags) + msg = '* %s FETCH %s' % (args[0], flags_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', 'STORE completed.') + + with self.reaped_pair(BracketFlagHandler) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') + client.select('test') + typ, [data] = client.store(b'1', "+FLAGS", "[test]") + self.assertIn(b'[test]', data) + client.select('test') + typ, [data] = client.response('PERMANENTFLAGS') + self.assertIn(b'[test]', data) + + @threading_helper.reap_threads + def test_issue5949(self): + + class EOFHandler(socketserver.StreamRequestHandler): + def handle(self): + # EOF without sending a complete welcome message. + self.wfile.write(b'* OK') + + with self.reaped_server(EOFHandler) as server: + self.assertRaises(imaplib.IMAP4.abort, + self.imap_class, *server.server_address) + + @threading_helper.reap_threads + def test_line_termination(self): + + class BadNewlineHandler(SimpleIMAPHandler): + + def cmd_CAPABILITY(self, tag, args): + self._send(b'* CAPABILITY IMAP4rev1 AUTH\n') + self._send_tagged(tag, 'OK', 'CAPABILITY completed') + + with self.reaped_server(BadNewlineHandler) as server: + self.assertRaises(imaplib.IMAP4.abort, + self.imap_class, *server.server_address) + + class UTF8Server(SimpleIMAPHandler): + capabilities = 'AUTH ENABLE UTF8=ACCEPT' + + def cmd_ENABLE(self, tag, args): + self._send_tagged(tag, 'OK', 'ENABLE successful') + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + @threading_helper.reap_threads + def test_enable_raises_error_if_not_AUTH(self): + with self.reaped_pair(self.UTF8Server) as (server, client): + self.assertFalse(client.utf8_enabled) + self.assertRaises(imaplib.IMAP4.error, client.enable, 'foo') + self.assertFalse(client.utf8_enabled) + + # XXX Also need a test that enable after SELECT raises an error. + + @threading_helper.reap_threads + def test_enable_raises_error_if_no_capability(self): + class NoEnableServer(self.UTF8Server): + capabilities = 'AUTH' + with self.reaped_pair(NoEnableServer) as (server, client): + self.assertRaises(imaplib.IMAP4.error, client.enable, 'foo') + + @threading_helper.reap_threads + def test_enable_UTF8_raises_error_if_not_supported(self): + class NonUTF8Server(SimpleIMAPHandler): + pass + with self.assertRaises(imaplib.IMAP4.error): + with self.reaped_pair(NonUTF8Server) as (server, client): + typ, data = client.login('user', 'pass') + self.assertEqual(typ, 'OK') + client.enable('UTF8=ACCEPT') + + @threading_helper.reap_threads + def test_enable_UTF8_True_append(self): + + class UTF8AppendServer(self.UTF8Server): + def cmd_APPEND(self, tag, args): + self._send_textline('+') + self.server.response = args + literal = yield + self.server.response.append(literal) + literal = yield + self.server.response.append(literal) + self._send_tagged(tag, 'OK', 'okay') + + with self.reaped_pair(UTF8AppendServer) as (server, client): + self.assertEqual(client._encoding, 'ascii') + code, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, + b'ZmFrZQ==\r\n') # b64 encoded 'fake' + code, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(code, 'OK') + self.assertEqual(client._encoding, 'utf-8') + msg_string = 'Subject: üñí©öðé' + typ, data = client.append( + None, None, None, (msg_string + '\n').encode('utf-8')) + self.assertEqual(typ, 'OK') + self.assertEqual(server.response, + ['INBOX', 'UTF8', + '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'), + b')\r\n' ]) + + # XXX also need a test that makes sure that the Literal and Untagged_status + # regexes uses unicode in UTF8 mode instead of the default ASCII. + + @threading_helper.reap_threads + def test_search_disallows_charset_in_utf8_mode(self): + with self.reaped_pair(self.UTF8Server) as (server, client): + typ, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(typ, 'OK') + typ, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(typ, 'OK') + self.assertTrue(client.utf8_enabled) + self.assertRaises(imaplib.IMAP4.error, client.search, 'foo', 'bar') + + @threading_helper.reap_threads + def test_bad_auth_name(self): + + class MyServer(SimpleIMAPHandler): + + def cmd_AUTHENTICATE(self, tag, args): + self._send_tagged(tag, 'NO', 'unrecognized authentication ' + 'type {}'.format(args[0])) + + with self.reaped_pair(MyServer) as (server, client): + with self.assertRaises(imaplib.IMAP4.error): + client.authenticate('METHOD', lambda: 1) + + @threading_helper.reap_threads + def test_invalid_authentication(self): + + class MyServer(SimpleIMAPHandler): + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.response = yield + self._send_tagged(tag, 'NO', '[AUTHENTICATIONFAILED] invalid') + + with self.reaped_pair(MyServer) as (server, client): + with self.assertRaises(imaplib.IMAP4.error): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + + @threading_helper.reap_threads + def test_valid_authentication(self): + + class MyServer(SimpleIMAPHandler): + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + with self.reaped_pair(MyServer) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, + b'ZmFrZQ==\r\n') # b64 encoded 'fake' + + with self.reaped_pair(MyServer) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: 'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, + b'ZmFrZQ==\r\n') # b64 encoded 'fake' + + @threading_helper.reap_threads + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def test_login_cram_md5(self): + + class AuthHandler(SimpleIMAPHandler): + + capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' + 'VzdG9uLm1jaS5uZXQ=') + r = yield + if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' + b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): + self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') + else: + self._send_tagged(tag, 'NO', 'No access') + + with self.reaped_pair(AuthHandler) as (server, client): + self.assertTrue('AUTH=CRAM-MD5' in client.capabilities) + ret, data = client.login_cram_md5("tim", "tanstaaftanstaaf") + self.assertEqual(ret, "OK") + + with self.reaped_pair(AuthHandler) as (server, client): + self.assertTrue('AUTH=CRAM-MD5' in client.capabilities) + ret, data = client.login_cram_md5("tim", b"tanstaaftanstaaf") + self.assertEqual(ret, "OK") + + + @threading_helper.reap_threads + def test_aborted_authentication(self): + + class MyServer(SimpleIMAPHandler): + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.response = yield + + if self.response == b'*\r\n': + self._send_tagged(tag, 'NO', '[AUTHENTICATIONFAILED] aborted') + else: + self._send_tagged(tag, 'OK', 'MYAUTH successful') + + with self.reaped_pair(MyServer) as (server, client): + with self.assertRaises(imaplib.IMAP4.error): + code, data = client.authenticate('MYAUTH', lambda x: None) + + + def test_linetoolong(self): + class TooLongHandler(SimpleIMAPHandler): + def handle(self): + # Send a very long response line + self.wfile.write(b'* OK ' + imaplib._MAXLINE * b'x' + b'\r\n') + + with self.reaped_server(TooLongHandler) as server: + self.assertRaises(imaplib.IMAP4.error, + self.imap_class, *server.server_address) + + def test_truncated_large_literal(self): + size = 0 + class BadHandler(SimpleIMAPHandler): + def handle(self): + self._send_textline('* OK {%d}' % size) + self._send_textline('IMAP4rev1') + + for exponent in range(15, 64): + size = 1 << exponent + with self.subTest(f"size=2e{size}"): + with self.reaped_server(BadHandler) as server: + with self.assertRaises(imaplib.IMAP4.abort): + self.imap_class(*server.server_address) + + @threading_helper.reap_threads + def test_simple_with_statement(self): + # simplest call + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address): + pass + + @threading_helper.reap_threads + def test_with_statement(self): + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + self.assertIsNone(server.logged) + + @threading_helper.reap_threads + def test_with_statement_logout(self): + # what happens if already logout in the block? + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address) as imap: + imap.login('user', 'pass') + self.assertEqual(server.logged, 'user') + imap.logout() + self.assertIsNone(server.logged) + self.assertIsNone(server.logged) + + @threading_helper.reap_threads + @cpython_only + @unittest.skipUnless(__debug__, "Won't work if __debug__ is False") + def test_dump_ur(self): + # See: http://bugs.python.org/issue26543 + untagged_resp_dict = {'READ-WRITE': [b'']} + + with self.reaped_server(SimpleIMAPHandler) as server: + with self.imap_class(*server.server_address) as imap: + with mock.patch.object(imap, '_mesg') as mock_mesg: + imap._dump_ur(untagged_resp_dict) + mock_mesg.assert_called_with( + "untagged responses dump:READ-WRITE: [b'']" + ) + + +@unittest.skipUnless(ssl, "SSL not available") +class ThreadedNetworkedTestsSSL(ThreadedNetworkedTests): + server_class = SecureTCPServer + imap_class = IMAP4_SSL + + @threading_helper.reap_threads + def test_ssl_verified(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(CAFILE) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.CertificateError, regex): + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class(*server.server_address, + ssl_context=ssl_context) + client.shutdown() + + with self.reaped_server(SimpleIMAPHandler) as server: + client = self.imap_class("localhost", server.server_address[1], + ssl_context=ssl_context) + client.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py deleted file mode 100644 index 1ccd072b3ff..00000000000 --- a/Lib/test/test_imp.py +++ /dev/null @@ -1,499 +0,0 @@ -import gc -import importlib -import importlib.util -import os -import os.path -import py_compile -import sys -from test import support -from test.support import import_helper -from test.support import os_helper -from test.support import script_helper -from test.support import warnings_helper -import unittest -import warnings -imp = warnings_helper.import_deprecated('imp') -import _imp - - -OS_PATH_NAME = os.path.__name__ - - -def requires_load_dynamic(meth): - """Decorator to skip a test if not running under CPython or lacking - imp.load_dynamic().""" - meth = support.cpython_only(meth) - return unittest.skipIf(getattr(imp, 'load_dynamic', None) is None, - 'imp.load_dynamic() required')(meth) - - -class LockTests(unittest.TestCase): - - """Very basic test of import lock functions.""" - - def verify_lock_state(self, expected): - self.assertEqual(imp.lock_held(), expected, - "expected imp.lock_held() to be %r" % expected) - def testLock(self): - LOOPS = 50 - - # The import lock may already be held, e.g. if the test suite is run - # via "import test.autotest". - lock_held_at_start = imp.lock_held() - self.verify_lock_state(lock_held_at_start) - - for i in range(LOOPS): - imp.acquire_lock() - self.verify_lock_state(True) - - for i in range(LOOPS): - imp.release_lock() - - # The original state should be restored now. - self.verify_lock_state(lock_held_at_start) - - if not lock_held_at_start: - try: - imp.release_lock() - except RuntimeError: - pass - else: - self.fail("release_lock() without lock should raise " - "RuntimeError") - -class ImportTests(unittest.TestCase): - def setUp(self): - mod = importlib.import_module('test.encoded_modules') - self.test_strings = mod.test_strings - self.test_path = mod.__path__ - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_import_encoded_module(self): - for modname, encoding, teststr in self.test_strings: - mod = importlib.import_module('test.encoded_modules.' - 'module_' + modname) - self.assertEqual(teststr, mod.test) - - def test_find_module_encoding(self): - for mod, encoding, _ in self.test_strings: - with imp.find_module('module_' + mod, self.test_path)[0] as fd: - self.assertEqual(fd.encoding, encoding) - - path = [os.path.dirname(__file__)] - with self.assertRaises(SyntaxError): - imp.find_module('badsyntax_pep3120', path) - - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_issue1267(self): - for mod, encoding, _ in self.test_strings: - fp, filename, info = imp.find_module('module_' + mod, - self.test_path) - with fp: - self.assertNotEqual(fp, None) - self.assertEqual(fp.encoding, encoding) - self.assertEqual(fp.tell(), 0) - self.assertEqual(fp.readline(), '# test %s encoding\n' - % encoding) - - fp, filename, info = imp.find_module("tokenize") - with fp: - self.assertNotEqual(fp, None) - self.assertEqual(fp.encoding, "utf-8") - self.assertEqual(fp.tell(), 0) - self.assertEqual(fp.readline(), - '"""Tokenization help for Python programs.\n') - - def test_issue3594(self): - temp_mod_name = 'test_imp_helper' - sys.path.insert(0, '.') - try: - with open(temp_mod_name + '.py', 'w', encoding="latin-1") as file: - file.write("# coding: cp1252\nu = 'test.test_imp'\n") - file, filename, info = imp.find_module(temp_mod_name) - file.close() - self.assertEqual(file.encoding, 'cp1252') - finally: - del sys.path[0] - os_helper.unlink(temp_mod_name + '.py') - os_helper.unlink(temp_mod_name + '.pyc') - - def test_issue5604(self): - # Test cannot cover imp.load_compiled function. - # Martin von Loewis note what shared library cannot have non-ascii - # character because init_xxx function cannot be compiled - # and issue never happens for dynamic modules. - # But sources modified to follow generic way for processing paths. - - # the return encoding could be uppercase or None - fs_encoding = sys.getfilesystemencoding() - - # covers utf-8 and Windows ANSI code pages - # one non-space symbol from every page - # (http://en.wikipedia.org/wiki/Code_page) - known_locales = { - 'utf-8' : b'\xc3\xa4', - 'cp1250' : b'\x8C', - 'cp1251' : b'\xc0', - 'cp1252' : b'\xc0', - 'cp1253' : b'\xc1', - 'cp1254' : b'\xc0', - 'cp1255' : b'\xe0', - 'cp1256' : b'\xe0', - 'cp1257' : b'\xc0', - 'cp1258' : b'\xc0', - } - - if sys.platform == 'darwin': - self.assertEqual(fs_encoding, 'utf-8') - # Mac OS X uses the Normal Form D decomposition - # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html - special_char = b'a\xcc\x88' - else: - special_char = known_locales.get(fs_encoding) - - if not special_char: - self.skipTest("can't run this test with %s as filesystem encoding" - % fs_encoding) - decoded_char = special_char.decode(fs_encoding) - temp_mod_name = 'test_imp_helper_' + decoded_char - test_package_name = 'test_imp_helper_package_' + decoded_char - init_file_name = os.path.join(test_package_name, '__init__.py') - try: - # if the curdir is not in sys.path the test fails when run with - # ./python ./Lib/test/regrtest.py test_imp - sys.path.insert(0, os.curdir) - with open(temp_mod_name + '.py', 'w', encoding="utf-8") as file: - file.write('a = 1\n') - file, filename, info = imp.find_module(temp_mod_name) - with file: - self.assertIsNotNone(file) - self.assertTrue(filename[:-3].endswith(temp_mod_name)) - self.assertEqual(info[0], '.py') - self.assertEqual(info[1], 'r') - self.assertEqual(info[2], imp.PY_SOURCE) - - mod = imp.load_module(temp_mod_name, file, filename, info) - self.assertEqual(mod.a, 1) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - mod = imp.load_source(temp_mod_name, temp_mod_name + '.py') - self.assertEqual(mod.a, 1) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - if not sys.dont_write_bytecode: - mod = imp.load_compiled( - temp_mod_name, - imp.cache_from_source(temp_mod_name + '.py')) - self.assertEqual(mod.a, 1) - - if not os.path.exists(test_package_name): - os.mkdir(test_package_name) - with open(init_file_name, 'w', encoding="utf-8") as file: - file.write('b = 2\n') - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - package = imp.load_package(test_package_name, test_package_name) - self.assertEqual(package.b, 2) - finally: - del sys.path[0] - for ext in ('.py', '.pyc'): - os_helper.unlink(temp_mod_name + ext) - os_helper.unlink(init_file_name + ext) - os_helper.rmtree(test_package_name) - os_helper.rmtree('__pycache__') - - def test_issue9319(self): - path = os.path.dirname(__file__) - self.assertRaises(SyntaxError, - imp.find_module, "badsyntax_pep3120", [path]) - - def test_load_from_source(self): - # Verify that the imp module can correctly load and find .py files - # XXX (ncoghlan): It would be nice to use import_helper.CleanImport - # here, but that breaks because the os module registers some - # handlers in copy_reg on import. Since CleanImport doesn't - # revert that registration, the module is left in a broken - # state after reversion. Reinitialising the module contents - # and just reverting os.environ to its previous state is an OK - # workaround - with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME): - import os - orig_path = os.path - orig_getenv = os.getenv - with os_helper.EnvironmentVarGuard(): - x = imp.find_module("os") - self.addCleanup(x[0].close) - new_os = imp.load_module("os", *x) - self.assertIs(os, new_os) - self.assertIs(orig_path, new_os.path) - self.assertIsNot(orig_getenv, new_os.getenv) - - @requires_load_dynamic - def test_issue15828_load_extensions(self): - # Issue 15828 picked up that the adapter between the old imp API - # and importlib couldn't handle C extensions - example = "_heapq" - x = imp.find_module(example) - file_ = x[0] - if file_ is not None: - self.addCleanup(file_.close) - mod = imp.load_module(example, *x) - self.assertEqual(mod.__name__, example) - - @requires_load_dynamic - def test_issue16421_multiple_modules_in_one_dll(self): - # Issue 16421: loading several modules from the same compiled file fails - m = '_testimportmultiple' - fileobj, pathname, description = imp.find_module(m) - fileobj.close() - mod0 = imp.load_dynamic(m, pathname) - mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname) - mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname) - self.assertEqual(mod0.__name__, m) - self.assertEqual(mod1.__name__, '_testimportmultiple_foo') - self.assertEqual(mod2.__name__, '_testimportmultiple_bar') - with self.assertRaises(ImportError): - imp.load_dynamic('nonexistent', pathname) - - @requires_load_dynamic - def test_load_dynamic_ImportError_path(self): - # Issue #1559549 added `name` and `path` attributes to ImportError - # in order to provide better detail. Issue #10854 implemented those - # attributes on import failures of extensions on Windows. - path = 'bogus file path' - name = 'extension' - with self.assertRaises(ImportError) as err: - imp.load_dynamic(name, path) - self.assertIn(path, err.exception.path) - self.assertEqual(name, err.exception.name) - - @requires_load_dynamic - def test_load_module_extension_file_is_None(self): - # When loading an extension module and the file is None, open one - # on the behalf of imp.load_dynamic(). - # Issue #15902 - name = '_testimportmultiple' - found = imp.find_module(name) - if found[0] is not None: - found[0].close() - if found[2][2] != imp.C_EXTENSION: - self.skipTest("found module doesn't appear to be a C extension") - imp.load_module(name, None, *found[1:]) - - @requires_load_dynamic - def test_issue24748_load_module_skips_sys_modules_check(self): - name = 'test.imp_dummy' - try: - del sys.modules[name] - except KeyError: - pass - try: - module = importlib.import_module(name) - spec = importlib.util.find_spec('_testmultiphase') - module = imp.load_dynamic(name, spec.origin) - self.assertEqual(module.__name__, name) - self.assertEqual(module.__spec__.name, name) - self.assertEqual(module.__spec__.origin, spec.origin) - self.assertRaises(AttributeError, getattr, module, 'dummy_name') - self.assertEqual(module.int_const, 1969) - self.assertIs(sys.modules[name], module) - finally: - try: - del sys.modules[name] - except KeyError: - pass - - @unittest.skipIf(sys.dont_write_bytecode, - "test meaningful only when writing bytecode") - def test_bug7732(self): - with os_helper.temp_cwd(): - source = os_helper.TESTFN + '.py' - os.mkdir(source) - self.assertRaisesRegex(ImportError, '^No module', - imp.find_module, os_helper.TESTFN, ["."]) - - def test_multiple_calls_to_get_data(self): - # Issue #18755: make sure multiple calls to get_data() can succeed. - loader = imp._LoadSourceCompatibility('imp', imp.__file__, - open(imp.__file__, encoding="utf-8")) - loader.get_data(imp.__file__) # File should be closed - loader.get_data(imp.__file__) # Will need to create a newly opened file - - def test_load_source(self): - # Create a temporary module since load_source(name) modifies - # sys.modules[name] attributes like __loader___ - modname = f"tmp{__name__}" - mod = type(sys.modules[__name__])(modname) - with support.swap_item(sys.modules, modname, mod): - with self.assertRaisesRegex(ValueError, 'embedded null'): - imp.load_source(modname, __file__ + "\0") - - @support.cpython_only - def test_issue31315(self): - # There shouldn't be an assertion failure in imp.create_dynamic(), - # when spec.name is not a string. - create_dynamic = support.get_attribute(imp, 'create_dynamic') - class BadSpec: - name = None - origin = 'foo' - with self.assertRaises(TypeError): - create_dynamic(BadSpec()) - - def test_issue_35321(self): - # Both _frozen_importlib and _frozen_importlib_external - # should have a spec origin of "frozen" and - # no need to clean up imports in this case. - - import _frozen_importlib_external - self.assertEqual(_frozen_importlib_external.__spec__.origin, "frozen") - - import _frozen_importlib - self.assertEqual(_frozen_importlib.__spec__.origin, "frozen") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_source_hash(self): - self.assertEqual(_imp.source_hash(42, b'hi'), b'\xfb\xd9G\x05\xaf$\x9b~') - self.assertEqual(_imp.source_hash(43, b'hi'), b'\xd0/\x87C\xccC\xff\xe2') - - def test_pyc_invalidation_mode_from_cmdline(self): - cases = [ - ([], "default"), - (["--check-hash-based-pycs", "default"], "default"), - (["--check-hash-based-pycs", "always"], "always"), - (["--check-hash-based-pycs", "never"], "never"), - ] - for interp_args, expected in cases: - args = interp_args + [ - "-c", - "import _imp; print(_imp.check_hash_based_pycs)", - ] - res = script_helper.assert_python_ok(*args) - self.assertEqual(res.out.strip().decode('utf-8'), expected) - - def test_find_and_load_checked_pyc(self): - # issue 34056 - with os_helper.temp_cwd(): - with open('mymod.py', 'wb') as fp: - fp.write(b'x = 42\n') - py_compile.compile( - 'mymod.py', - doraise=True, - invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, - ) - file, path, description = imp.find_module('mymod', path=['.']) - mod = imp.load_module('mymod', file, path, description) - self.assertEqual(mod.x, 42) - - - @support.cpython_only - def test_create_builtin_subinterp(self): - # gh-99578: create_builtin() behavior changes after the creation of the - # first sub-interpreter. Test both code paths, before and after the - # creation of a sub-interpreter. Previously, create_builtin() had - # a reference leak after the creation of the first sub-interpreter. - - import builtins - create_builtin = support.get_attribute(_imp, "create_builtin") - class Spec: - name = "builtins" - spec = Spec() - - def check_get_builtins(): - refcnt = sys.getrefcount(builtins) - mod = _imp.create_builtin(spec) - self.assertIs(mod, builtins) - self.assertEqual(sys.getrefcount(builtins), refcnt + 1) - # Check that a GC collection doesn't crash - gc.collect() - - check_get_builtins() - - ret = support.run_in_subinterp("import builtins") - self.assertEqual(ret, 0) - - check_get_builtins() - - -class ReloadTests(unittest.TestCase): - - """Very basic tests to make sure that imp.reload() operates just like - reload().""" - - def test_source(self): - # XXX (ncoghlan): It would be nice to use test.import_helper.CleanImport - # here, but that breaks because the os module registers some - # handlers in copy_reg on import. Since CleanImport doesn't - # revert that registration, the module is left in a broken - # state after reversion. Reinitialising the module contents - # and just reverting os.environ to its previous state is an OK - # workaround - with os_helper.EnvironmentVarGuard(): - import os - imp.reload(os) - - def test_extension(self): - with import_helper.CleanImport('time'): - import time - imp.reload(time) - - def test_builtin(self): - with import_helper.CleanImport('marshal'): - import marshal - imp.reload(marshal) - - def test_with_deleted_parent(self): - # see #18681 - from html import parser - html = sys.modules.pop('html') - def cleanup(): - sys.modules['html'] = html - self.addCleanup(cleanup) - with self.assertRaisesRegex(ImportError, 'html'): - imp.reload(parser) - - -class PEP3147Tests(unittest.TestCase): - """Tests of PEP 3147.""" - - tag = imp.get_tag() - - @unittest.skipUnless(sys.implementation.cache_tag is not None, - 'requires sys.implementation.cache_tag not be None') - def test_cache_from_source(self): - # Given the path to a .py file, return the path to its PEP 3147 - # defined .pyc file (i.e. under __pycache__). - path = os.path.join('foo', 'bar', 'baz', 'qux.py') - expect = os.path.join('foo', 'bar', 'baz', '__pycache__', - 'qux.{}.pyc'.format(self.tag)) - self.assertEqual(imp.cache_from_source(path, True), expect) - - @unittest.skipUnless(sys.implementation.cache_tag is not None, - 'requires sys.implementation.cache_tag to not be ' - 'None') - def test_source_from_cache(self): - # Given the path to a PEP 3147 defined .pyc file, return the path to - # its source. This tests the good path. - path = os.path.join('foo', 'bar', 'baz', '__pycache__', - 'qux.{}.pyc'.format(self.tag)) - expect = os.path.join('foo', 'bar', 'baz', 'qux.py') - self.assertEqual(imp.source_from_cache(path), expect) - - -class NullImporterTests(unittest.TestCase): - @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, - "Need an undecodeable filename") - def test_unencodeable(self): - name = os_helper.TESTFN_UNENCODABLE - os.mkdir(name) - try: - self.assertRaises(ImportError, imp.NullImporter, name) - finally: - os.rmdir(name) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index c2f181cc86a..6920cf45533 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1,9 +1,16 @@ import builtins -import contextlib import errno import glob +import json import importlib.util from importlib._bootstrap_external import _get_sourcefile +from importlib.machinery import ( + AppleFrameworkLoader, + BuiltinImporter, + ExtensionFileLoader, + FrozenImporter, + SourceFileLoader, +) import marshal import os import py_compile @@ -15,27 +22,125 @@ import textwrap import threading import time +import types import unittest from unittest import mock +import _imp from test.support import os_helper from test.support import ( - STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only, is_emscripten, - is_wasi) + STDLIB_DIR, + swap_attr, + swap_item, + cpython_only, + is_apple_mobile, + is_emscripten, + is_wasm32, + run_in_subinterp, + run_in_subinterp_with_config, + Py_TRACE_REFS, + requires_gil_enabled, + Py_GIL_DISABLED, + no_rerun, + force_not_colorized_test_class, +) from test.support.import_helper import ( - forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) + forget, make_legacy_pyc, unlink, unload, ready_to_import, + DirsOnSysPath, CleanImport, import_module) from test.support.os_helper import ( - TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE, temp_dir) + TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE) from test.support import script_helper from test.support import threading_helper from test.test_importlib.util import uncache from types import ModuleType +try: + import _testsinglephase +except ImportError: + _testsinglephase = None +try: + import _testmultiphase +except ImportError: + _testmultiphase = None +try: + import _interpreters +except ModuleNotFoundError: + _interpreters = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None skip_if_dont_write_bytecode = unittest.skipIf( sys.dont_write_bytecode, "test meaningful only when writing bytecode") + +def _require_loader(module, loader, skip): + if isinstance(module, str): + module = __import__(module) + + MODULE_KINDS = { + BuiltinImporter: 'built-in', + ExtensionFileLoader: 'extension', + AppleFrameworkLoader: 'framework extension', + FrozenImporter: 'frozen', + SourceFileLoader: 'pure Python', + } + + expected = loader + assert isinstance(expected, type), expected + expected = MODULE_KINDS[expected] + + actual = module.__spec__.loader + if not isinstance(actual, type): + actual = type(actual) + actual = MODULE_KINDS[actual] + + if actual != expected: + err = f'expected module to be {expected}, got {module.__spec__}' + if skip: + raise unittest.SkipTest(err) + raise Exception(err) + return module + +def require_builtin(module, *, skip=False): + module = _require_loader(module, BuiltinImporter, skip) + assert module.__spec__.origin == 'built-in', module.__spec__ + +def require_extension(module, *, skip=False): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + _require_loader(module, AppleFrameworkLoader, skip) + else: + _require_loader(module, ExtensionFileLoader, skip) + +def require_frozen(module, *, skip=True): + module = _require_loader(module, FrozenImporter, skip) + assert module.__spec__.origin == 'frozen', module.__spec__ + +def require_pure_python(module, *, skip=False): + _require_loader(module, SourceFileLoader, skip) + +def create_extension_loader(modname, filename): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + return AppleFrameworkLoader(modname, filename) + else: + return ExtensionFileLoader(modname, filename) + +def import_extension_from_file(modname, filename, *, put_in_sys_modules=True): + loader = create_extension_loader(modname, filename) + spec = importlib.util.spec_from_loader(modname, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + if put_in_sys_modules: + sys.modules[modname] = module + return module + + def remove_files(name): for f in (name + ".py", name + ".pyc", @@ -45,27 +150,202 @@ def remove_files(name): rmtree('__pycache__') -@contextlib.contextmanager -def _ready_to_import(name=None, source=""): - # sets up a temporary directory and removes it - # creates the module file - # temporarily clears the module from sys.modules (if any) - # reverts or removes the module when cleaning up - name = name or "spam" - with temp_dir() as tempdir: - path = script_helper.make_script(tempdir, name, source) - old_module = sys.modules.pop(name, None) +if _testsinglephase is not None: + def restore__testsinglephase(*, _orig=_testsinglephase): + # We started with the module imported and want to restore + # it to its nominal state. + sys.modules.pop('_testsinglephase', None) + _orig._clear_globals() + origin = _orig.__spec__.origin + _testinternalcapi.clear_extension('_testsinglephase', origin) + import _testsinglephase + + +def requires_singlephase_init(meth): + """Decorator to skip if single-phase init modules are not supported.""" + if not isinstance(meth, type): + def meth(self, _meth=meth): + try: + return _meth(self) + finally: + restore__testsinglephase() + meth = cpython_only(meth) + msg = "gh-117694: free-threaded build does not currently support single-phase init modules in sub-interpreters" + meth = requires_gil_enabled(msg)(meth) + return unittest.skipIf(_testsinglephase is None, + 'test requires _testsinglephase module')(meth) + + +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(_interpreters is None, + 'subinterpreters required')(meth) + + +class ModuleSnapshot(types.SimpleNamespace): + """A representation of a module for testing. + + Fields: + + * id - the module's object ID + * module - the actual module or an adequate substitute + * __file__ + * __spec__ + * name + * origin + * ns - a copy (dict) of the module's __dict__ (or None) + * ns_id - the object ID of the module's __dict__ + * cached - the sys.modules[mod.__spec__.name] entry (or None) + * cached_id - the object ID of the sys.modules entry (or None) + + In cases where the value is not available (e.g. due to serialization), + the value will be None. + """ + _fields = tuple('id module ns ns_id cached cached_id'.split()) + + @classmethod + def from_module(cls, mod): + name = mod.__spec__.name + cached = sys.modules.get(name) + return cls( + id=id(mod), + module=mod, + ns=types.SimpleNamespace(**mod.__dict__), + ns_id=id(mod.__dict__), + cached=cached, + cached_id=id(cached), + ) + + SCRIPT = textwrap.dedent(''' + {imports} + + name = {name!r} + + {prescript} + + mod = {name} + + {body} + + {postscript} + ''') + IMPORTS = textwrap.dedent(''' + import sys + ''').strip() + SCRIPT_BODY = textwrap.dedent(''' + # Capture the snapshot data. + cached = sys.modules.get(name) + snapshot = dict( + id=id(mod), + module=dict( + __file__=mod.__file__, + __spec__=dict( + name=mod.__spec__.name, + origin=mod.__spec__.origin, + ), + ), + ns=None, + ns_id=id(mod.__dict__), + cached=None, + cached_id=id(cached) if cached else None, + ) + ''').strip() + CLEANUP_SCRIPT = textwrap.dedent(''' + # Clean up the module. + sys.modules.pop(name, None) + ''').strip() + + @classmethod + def build_script(cls, name, *, + prescript=None, + import_first=False, + postscript=None, + postcleanup=False, + ): + if postcleanup is True: + postcleanup = cls.CLEANUP_SCRIPT + elif isinstance(postcleanup, str): + postcleanup = textwrap.dedent(postcleanup).strip() + postcleanup = cls.CLEANUP_SCRIPT + os.linesep + postcleanup + else: + postcleanup = '' + prescript = textwrap.dedent(prescript).strip() if prescript else '' + postscript = textwrap.dedent(postscript).strip() if postscript else '' + + if postcleanup: + if postscript: + postscript = postscript + os.linesep * 2 + postcleanup + else: + postscript = postcleanup + + if import_first: + prescript += textwrap.dedent(f''' + + # Now import the module. + assert name not in sys.modules + import {name}''') + + return cls.SCRIPT.format( + imports=cls.IMPORTS.strip(), + name=name, + prescript=prescript.strip(), + body=cls.SCRIPT_BODY.strip(), + postscript=postscript, + ) + + @classmethod + def parse(cls, text): + raw = json.loads(text) + mod = raw['module'] + mod['__spec__'] = types.SimpleNamespace(**mod['__spec__']) + raw['module'] = types.SimpleNamespace(**mod) + return cls(**raw) + + @classmethod + def from_subinterp(cls, name, interpid=None, *, pipe=None, **script_kwds): + if pipe is not None: + return cls._from_subinterp(name, interpid, pipe, script_kwds) + pipe = os.pipe() try: - sys.path.insert(0, tempdir) - yield name, path - sys.path.remove(tempdir) + return cls._from_subinterp(name, interpid, pipe, script_kwds) finally: - if old_module is not None: - sys.modules[name] = old_module - elif name in sys.modules: - del sys.modules[name] + r, w = pipe + os.close(r) + os.close(w) + + @classmethod + def _from_subinterp(cls, name, interpid, pipe, script_kwargs): + r, w = pipe + + # Build the script. + postscript = textwrap.dedent(f''' + # Send the result over the pipe. + import json + import os + os.write({w}, json.dumps(snapshot).encode()) + + ''') + _postscript = script_kwargs.get('postscript') + if _postscript: + _postscript = textwrap.dedent(_postscript).lstrip() + postscript += _postscript + script_kwargs['postscript'] = postscript.strip() + script = cls.build_script(name, **script_kwargs) + + # Run the script. + if interpid is None: + ret = run_in_subinterp(script) + if ret != 0: + raise AssertionError(f'{ret} != 0') + else: + _interpreters.run_string(interpid, script) + # Parse the results. + text = os.read(r, 1000) + return cls.parse(text.decode()) + +@force_not_colorized_test_class class ImportTests(unittest.TestCase): def setUp(self): @@ -87,8 +367,6 @@ def test_from_import_missing_attr_raises_ImportError(self): with self.assertRaises(ImportError): from importlib import something_that_should_not_exist_anywhere - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_missing_attr_has_name_and_path(self): with CleanImport('os'): import os @@ -100,15 +378,19 @@ def test_from_import_missing_attr_has_name_and_path(self): @cpython_only def test_from_import_missing_attr_has_name_and_so_path(self): - import _testcapi + _testcapi = import_module("_testcapi") with self.assertRaises(ImportError) as cm: from _testcapi import i_dont_exist self.assertEqual(cm.exception.name, '_testcapi') if hasattr(_testcapi, "__file__"): - self.assertEqual(cm.exception.path, _testcapi.__file__) + # The path on the exception is strictly the spec origin, not the + # module's __file__. For most cases, these are the same; but on + # iOS, the Framework relocation process results in the exception + # being raised from the spec location. + self.assertEqual(cm.exception.path, _testcapi.__spec__.origin) self.assertRegex( str(cm.exception), - r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)" + r"cannot import name 'i_dont_exist' from '_testcapi' \(.*(\.(so|pyd))?\)" ) else: self.assertEqual( @@ -123,19 +405,15 @@ def test_from_import_missing_attr_has_name(self): self.assertEqual(cm.exception.name, '_warning') self.assertIsNone(cm.exception.path) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_missing_attr_path_is_canonical(self): with self.assertRaises(ImportError) as cm: from os.path import i_dont_exist self.assertIn(cm.exception.name, {'posixpath', 'ntpath'}) self.assertIsNotNone(cm.exception) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_star_invalid_type(self): import re - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("__all__ = [b'invalid_type']") globals = {} @@ -144,7 +422,7 @@ def test_from_import_star_invalid_type(self): ): exec(f"from {name} import *", globals) self.assertNotIn(b"invalid_type", globals) - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("globals()[b'invalid_type'] = object()") globals = {} @@ -161,18 +439,18 @@ def test_case_sensitivity(self): import RAnDoM def test_double_const(self): - # Another brief digression to test the accuracy of manifest float - # constants. - from test import double_const # don't blink -- that *was* the test + # Importing double_const checks that float constants + # serialized by marshal as PYC files don't lose precision + # (SF bug 422177). + from test.test_import.data import double_const + unload('test.test_import.data.double_const') + from test.test_import.data import double_const # noqa: F811 def test_import(self): def test_with_extension(ext): # The extension is normally ".py", perhaps ".pyw". source = TESTFN + ext - if is_jython: - pyc = TESTFN + "$py.class" - else: - pyc = TESTFN + ".pyc" + pyc = TESTFN + ".pyc" with open(source, "w", encoding='utf-8') as f: print("# This tests Python's ability to import a", @@ -274,7 +552,7 @@ def test_import_name_binding(self): import test as x import test.support self.assertIs(x, test, x.__name__) - self.assertTrue(hasattr(test.support, "__file__")) + self.assertHasAttr(test.support, "__file__") # import x.y.z as w binds z as w import test.support as y @@ -295,7 +573,7 @@ def test_issue31286(self): # import in a 'for' loop resulted in segmentation fault for i in range(2): - import test.support.script_helper as x + import test.support.script_helper as x # noqa: F811 def test_failing_reload(self): # A failing reload should leave the module object in sys.modules. @@ -345,7 +623,7 @@ def test_file_to_source(self): sys.path.insert(0, os.curdir) try: mod = __import__(TESTFN) - self.assertTrue(mod.__file__.endswith('.py')) + self.assertEndsWith(mod.__file__, '.py') os.remove(source) del sys.modules[TESTFN] make_legacy_pyc(source) @@ -427,8 +705,6 @@ def test_from_import_message_for_existing_module(self): with self.assertRaisesRegex(ImportError, "^cannot import name 'bogus'"): from re import bogus - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_from_import_AttributeError(self): # Issue #24492: trying to import an attribute that raises an # AttributeError should lead to an ImportError. @@ -492,7 +768,7 @@ def run(): finally: del sys.path[0] - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; no C extension support @unittest.skipUnless(sys.platform == "win32", "Windows-specific") def test_dll_dependency_import(self): from _winapi import GetModuleFileName @@ -538,6 +814,445 @@ def test_dll_dependency_import(self): env=env, cwd=os.path.dirname(pyexe)) + def test_issue105979(self): + # this used to crash + with self.assertRaises(ImportError) as cm: + _imp.get_frozen_object("x", b"6\'\xd5Cu\x12") + self.assertIn("Frozen object named 'x' is invalid", + str(cm.exception)) + + def test_frozen_module_from_import_error(self): + with self.assertRaises(ImportError) as cm: + from os import this_will_never_exist + self.assertIn( + f"cannot import name 'this_will_never_exist' from 'os' ({os.__file__})", + str(cm.exception), + ) + with self.assertRaises(ImportError) as cm: + from sys import this_will_never_exist + self.assertIn( + "cannot import name 'this_will_never_exist' from 'sys' (unknown location)", + str(cm.exception), + ) + + scripts = [ + """ +import os +os.__spec__.has_location = False +os.__file__ = [] +from os import this_will_never_exist +""", + """ +import os +os.__spec__.has_location = False +del os.__file__ +from os import this_will_never_exist +""", + """ +import os +os.__spec__.origin = [] +os.__file__ = [] +from os import this_will_never_exist +""" + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"cannot import name 'this_will_never_exist' " + b"from 'os' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + + def test_non_module_from_import_error(self): + prefix = """ +import sys +class NotAModule: ... +nm = NotAModule() +nm.symbol = 123 +sys.modules["not_a_module"] = nm +from not_a_module import symbol +""" + scripts = [ + prefix + "from not_a_module import missing_symbol", + prefix + "nm.__spec__ = []\nfrom not_a_module import missing_symbol", + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"ImportError: cannot import name 'missing_symbol' from " + b"'<unknown module name>' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + + def test_script_shadowing_stdlib(self): + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # and there's no error at all when using -P + popen = script_helper.spawn_python('-P', 'fractions.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') + + tmp_child = os.path.join(tmp, "child") + os.mkdir(tmp_child) + + # test the logic with different cwd + popen = script_helper.spawn_python(os.path.join(tmp, "fractions.py"), cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error + + popen = script_helper.spawn_python('-c', 'import fractions', cwd=tmp_child) + stdout, stderr = popen.communicate() + self.assertEqual(stdout, b'') # no error + + def test_package_shadowing_stdlib_module(self): + script_errors = [ + ( + "fractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + os.mkdir(os.path.join(tmp, "fractions")) + with open( + os.path.join(tmp, "fractions", "__init__.py"), "w", encoding='utf-8' + ) as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write("import fractions; fractions.shadowing_module\n") + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*[\\/]fractions[\\/]+__init__.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "main.py"), cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'main', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # and there's no shadowing at all when using -P + popen = script_helper.spawn_python('-P', 'main.py', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, b"module 'fractions' has no attribute 'shadowing_module'") + + def test_script_shadowing_third_party(self): + script_errors = [ + ( + "import numpy\nnumpy.array", + rb"AttributeError: module 'numpy' has no attribute 'array'" + ), + ( + "from numpy import array", + rb"ImportError: cannot import name 'array' from 'numpy'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write(script) + + expected_error = error + ( + rb" \(consider renaming '.*numpy.py' if it has the " + rb"same name as a library you intended to import\)\s+\z" + ) + + popen = script_helper.spawn_python(os.path.join(tmp, "numpy.py")) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-m', 'numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + popen = script_helper.spawn_python('-c', 'import numpy', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_script_maybe_not_shadowing_third_party(self): + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "numpy.py"), "w", encoding='utf-8') as f: + f.write("this_script_does_not_attempt_to_import_numpy = True") + + expected_error = ( + rb"AttributeError: module 'numpy' has no attribute 'attr'\s+\z" + ) + popen = script_helper.spawn_python('-c', 'import numpy; numpy.attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + expected_error = ( + rb"ImportError: cannot import name 'attr' from 'numpy' \(.*\)\s+\z" + ) + popen = script_helper.spawn_python('-c', 'from numpy import attr', cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + def test_script_shadowing_stdlib_edge_cases(self): + with os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + + # Unhashable str subclass + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +class substr(str): + __hash__ = None +fractions.__name__ = substr('fractions') +try: + fractions.Fraction +except TypeError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertIn(b"unhashable type: 'substr'", stdout.rstrip()) + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +class substr(str): + __hash__ = None +fractions.__name__ = substr('fractions') +try: + from fractions import Fraction +except TypeError as e: + print(str(e)) +""") + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertIn(b"unhashable type: 'substr'", stdout.rstrip()) + + # Various issues with sys module + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module + +import sys +sys.stdlib_module_names = None +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +del sys.stdlib_module_names +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +sys.path = [0] +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 3) + for line in lines: + self.assertEqual(line, b"module 'fractions' has no attribute 'Fraction'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module + +import sys +sys.stdlib_module_names = None +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +del sys.stdlib_module_names +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +sys.path = [0] +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 3) + for line in lines: + self.assertRegex(line, rb"cannot import name 'Fraction' from 'fractions' \(.*\)") + + # Various issues with origin + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +del fractions.__spec__.origin +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) + +fractions.__spec__.origin = [] +try: + fractions.Fraction +except AttributeError as e: + print(str(e)) +""") + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 2) + for line in lines: + self.assertEqual(line, b"module 'fractions' has no attribute 'Fraction'") + + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write(""" +import fractions +fractions.shadowing_module +del fractions.__spec__.origin +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) + +fractions.__spec__.origin = [] +try: + from fractions import Fraction +except ImportError as e: + print(str(e)) +""") + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + lines = stdout.splitlines() + self.assertEqual(len(lines), 2) + for line in lines: + self.assertRegex(line, rb"cannot import name 'Fraction' from 'fractions' \(.*\)") + + @unittest.skipIf(sys.platform == 'win32', 'Cannot delete cwd on Windows') + @unittest.skipIf(sys.platform == 'sunos5', 'Cannot delete cwd on Solaris/Illumos') + @unittest.skipIf(sys.platform.startswith('aix'), 'Cannot delete cwd on AIX') + def test_script_shadowing_stdlib_cwd_failure(self): + with os_helper.temp_dir() as tmp: + subtmp = os.path.join(tmp, "subtmp") + os.mkdir(subtmp) + with open(os.path.join(subtmp, "main.py"), "w", encoding='utf-8') as f: + f.write(f""" +import sys +assert sys.path[0] == '' + +import os +import shutil +shutil.rmtree(os.getcwd()) + +os.does_not_exist +""") + # Use -c to ensure sys.path[0] is "" + popen = script_helper.spawn_python("-c", "import main", cwd=subtmp) + stdout, stderr = popen.communicate() + expected_error = rb"AttributeError: module 'os' has no attribute 'does_not_exist'" + self.assertRegex(stdout, expected_error) + + def test_script_shadowing_stdlib_sys_path_modification(self): + script_errors = [ + ( + "import fractions\nfractions.Fraction", + rb"AttributeError: module 'fractions' has no attribute 'Fraction'" + ), + ( + "from fractions import Fraction", + rb"ImportError: cannot import name 'Fraction' from 'fractions'" + ) + ] + for script, error in script_errors: + with self.subTest(script=script), os_helper.temp_dir() as tmp: + with open(os.path.join(tmp, "fractions.py"), "w", encoding='utf-8') as f: + f.write("shadowing_module = True") + with open(os.path.join(tmp, "main.py"), "w", encoding='utf-8') as f: + f.write('import sys; sys.path.insert(0, "this_folder_does_not_exist")\n') + f.write(script) + expected_error = error + ( + rb" \(consider renaming '.*fractions.py' since it has the " + rb"same name as the standard library module named 'fractions' " + rb"and prevents importing that standard library module\)" + ) + + popen = script_helper.spawn_python("main.py", cwd=tmp) + stdout, stderr = popen.communicate() + self.assertRegex(stdout, expected_error) + + # TODO: RUSTPYTHON: _imp.create_dynamic is for C extensions, not applicable + @unittest.skip("TODO: RustPython _imp.create_dynamic not implemented") + def test_create_dynamic_null(self): + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec: + name = "a\x00b" + origin = "abc" + _imp.create_dynamic(Spec()) + + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec2: + name = "abc" + origin = "a\x00b" + _imp.create_dynamic(Spec2()) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -546,12 +1261,12 @@ class FilePermissionTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Emscripten's/WASI's umask is a stub." ) def test_creation_mode(self): mask = 0o022 - with temp_umask(mask), _ready_to_import() as (name, path): + with temp_umask(mask), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) module = __import__(name) if not os.path.exists(cached_path): @@ -570,7 +1285,7 @@ def test_creation_mode(self): def test_cached_mode_issue_2051(self): # permissions of .pyc should match those of .py, regardless of mask mode = 0o600 - with temp_umask(0o022), _ready_to_import() as (name, path): + with temp_umask(0o022), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) os.chmod(path, mode) __import__(name) @@ -586,7 +1301,7 @@ def test_cached_mode_issue_2051(self): @os_helper.skip_unless_working_chmod def test_cached_readonly(self): mode = 0o400 - with temp_umask(0o022), _ready_to_import() as (name, path): + with temp_umask(0o022), ready_to_import() as (name, path): cached_path = importlib.util.cache_from_source(path) os.chmod(path, mode) __import__(name) @@ -601,7 +1316,7 @@ def test_cached_readonly(self): def test_pyc_always_writable(self): # Initially read-only .pyc files on Windows used to cause problems # with later updates, see issue #6074 for details - with _ready_to_import() as (name, path): + with ready_to_import() as (name, path): # Write a Python file, make it read-only and import it with open(path, 'w', encoding='utf-8') as f: f.write("x = 'original'\n") @@ -639,7 +1354,7 @@ class PycRewritingTests(unittest.TestCase): import sys code_filename = sys._getframe().f_code.co_filename module_filename = __file__ -constant = 1 +constant = 1000 def func(): pass func_filename = func.__code__.co_filename @@ -683,8 +1398,6 @@ def test_basics(self): self.assertEqual(mod.code_filename, self.file_name) self.assertEqual(mod.func_filename, self.file_name) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_incorrect_code_name(self): py_compile.compile(self.file_name, dfile="another_module.py") mod = self.import_module() @@ -710,7 +1423,7 @@ def test_foreign_code(self): code = marshal.load(f) constants = list(code.co_consts) foreign_code = importlib.import_module.__code__ - pos = constants.index(1) + pos = constants.index(1000) constants[pos] = foreign_code code = code.replace(co_consts=tuple(constants)) with open(self.compiled_name, "wb") as f: @@ -770,7 +1483,7 @@ def test_UNC_path(self): self.fail("could not import 'test_unc_path' from %r: %r" % (unc, e)) self.assertEqual(mod.testdata, 'test_unc_path') - self.assertTrue(mod.__file__.startswith(unc), mod.__file__) + self.assertStartsWith(mod.__file__, unc) unload("test_unc_path") @@ -783,7 +1496,7 @@ def tearDown(self): def test_relimport_star(self): # This will import * from .test_import. from .. import relimport - self.assertTrue(hasattr(relimport, "RelativeImportTests")) + self.assertHasAttr(relimport, "RelativeImportTests") def test_issue3221(self): # Note for mergers: the 'absolute' tests from the 2.x branch @@ -842,10 +1555,36 @@ def test_import_from_unloaded_package(self): import package2.submodule1 package2.submodule1.submodule2 + def test_rebinding(self): + # The same data is also used for testing pkgutil.resolve_name() + # in test_pkgutil and mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + + def test_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + from package4 import submodule + self.assertEqual(submodule.attr, 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + from package4 import submodule + self.assertEqual(submodule.attr, 'origin') + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + class OverridingImportBuiltinTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_override_builtin(self): # Test that overriding builtins.__import__ can bypass sys.modules. import os @@ -1087,7 +1826,7 @@ def test_frozen_importlib_is_bootstrap(self): self.assertIs(mod, _bootstrap) self.assertEqual(mod.__name__, 'importlib._bootstrap') self.assertEqual(mod.__package__, 'importlib') - self.assertTrue(mod.__file__.endswith('_bootstrap.py'), mod.__file__) + self.assertEndsWith(mod.__file__, '_bootstrap.py') def test_frozen_importlib_external_is_bootstrap_external(self): from importlib import _bootstrap_external @@ -1095,7 +1834,7 @@ def test_frozen_importlib_external_is_bootstrap_external(self): self.assertIs(mod, _bootstrap_external) self.assertEqual(mod.__name__, 'importlib._bootstrap_external') self.assertEqual(mod.__package__, 'importlib') - self.assertTrue(mod.__file__.endswith('_bootstrap_external.py'), mod.__file__) + self.assertEndsWith(mod.__file__, '_bootstrap_external.py') def test_there_can_be_only_one(self): # Issue #15386 revealed a tricky loophole in the bootstrapping @@ -1305,6 +2044,7 @@ def exec_module(*args): else: importlib.SourceLoader.exec_module = old_exec_module + @unittest.expectedFailure # TODO: RUSTPYTHON; subprocess fails on Windows @unittest.skipUnless(TESTFN_UNENCODABLE, 'need TESTFN_UNENCODABLE') def test_unencodable_filename(self): # Issue #11619: The Python parser and the import machinery must not @@ -1355,8 +2095,6 @@ def test_rebinding(self): from test.test_import.data.circular_imports.subpkg import util self.assertIs(util.util, rebinding.util) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_binding(self): try: import test.test_import.data.circular_imports.binding @@ -1367,8 +2105,6 @@ def test_crossreference1(self): import test.test_import.data.circular_imports.use import test.test_import.data.circular_imports.source - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_crossreference2(self): with self.assertRaises(AttributeError) as cm: import test.test_import.data.circular_imports.source @@ -1378,8 +2114,6 @@ def test_crossreference2(self): self.assertIn('partially initialized module', errmsg) self.assertIn('circular import', errmsg) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_circular_from_import(self): with self.assertRaises(ImportError) as cm: import test.test_import.data.circular_imports.from_cycle1 @@ -1390,8 +2124,14 @@ def test_circular_from_import(self): str(cm.exception), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_circular_import(self): + with self.assertRaisesRegex( + AttributeError, + r"partially initialized module 'test.test_import.data.circular_imports.import_cycle' " + r"from '.*' has no attribute 'some_attribute' \(most likely due to a circular import\)" + ): + import test.test_import.data.circular_imports.import_cycle + def test_absolute_circular_submodule(self): with self.assertRaises(AttributeError) as cm: import test.test_import.data.circular_imports.subpkg2.parent @@ -1402,8 +2142,37 @@ def test_absolute_circular_submodule(self): str(cm.exception), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @requires_singlephase_init + @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + def test_singlephase_circular(self): + """Regression test for gh-123950 + + Import a single-phase-init module that imports itself + from the PyInit_* function (before it's added to sys.modules). + Manages its own cache (which is `static`, and so incompatible + with multiple interpreters or interpreter reset). + """ + name = '_testsinglephase_circular' + helper_name = 'test.test_import.data.circular_imports.singlephase' + with uncache(name, helper_name): + filename = _testsinglephase.__file__ + # We don't put the module in sys.modules: that the *inner* + # import should do that. + mod = import_extension_from_file(name, filename, + put_in_sys_modules=False) + + self.assertEqual(mod.helper_mod_name, helper_name) + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + + self.assertIn(name, sys.modules) + self.assertIn(helper_name, sys.modules) + self.assertNotIn(name, sys.modules) + self.assertNotIn(helper_name, sys.modules) + self.assertIs(mod.clear_static_var(), mod) + _testinternalcapi.clear_extension('_testsinglephase_circular', + mod.__spec__.origin) + def test_unwritable_module(self): self.addCleanup(unload, "test.test_import.data.unwritable") self.addCleanup(unload, "test.test_import.data.unwritable.x") @@ -1418,6 +2187,1198 @@ def test_unwritable_module(self): unwritable.x = 42 +class SubinterpImportTests(unittest.TestCase): + + RUN_KWARGS = dict( + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + # Isolation-related config values aren't included here. + ) + ISOLATED = dict( + use_main_obmalloc=False, + gil=2, + ) + NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()} + NOT_ISOLATED['gil'] = 1 + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def pipe(self): + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + if hasattr(os, 'set_blocking'): + os.set_blocking(r, False) + return (r, w) + + def import_script(self, name, fd, filename=None, check_override=None): + override_text = '' + if check_override is not None: + override_text = f''' + import _imp + _imp._override_multi_interp_extensions_check({check_override}) + ''' + if filename: + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + + return textwrap.dedent(f''' + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import {loader} + import os, sys + {override_text} + loader = {loader}({name!r}, {filename!r}) + spec = spec_from_loader({name!r}, loader) + try: + module = module_from_spec(spec) + loader.exec_module(module) + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + else: + return textwrap.dedent(f''' + import os, sys + {override_text} + try: + import {name} + except ImportError as exc: + text = 'ImportError: ' + str(exc) + else: + text = 'okay' + os.write({fd}, text.encode('utf-8')) + ''') + + def run_here(self, name, filename=None, *, + check_singlephase_setting=False, + check_singlephase_override=None, + isolated=False, + ): + """ + Try importing the named module in a subinterpreter. + + The subinterpreter will be in the current process. + The module will have already been imported in the main interpreter. + Thus, for extension/builtin modules, the module definition will + have been loaded already and cached globally. + + "check_singlephase_setting" determines whether or not + the interpreter will be configured to check for modules + that are not compatible with use in multiple interpreters. + + This should always return "okay" for all modules if the + setting is False (with no override). + """ + __import__(name) + + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=check_singlephase_setting, + ) + + r, w = self.pipe() + script = self.import_script(name, w, filename, + check_singlephase_override) + + ret = run_in_subinterp_with_config(script, **kwargs) + self.assertEqual(ret, 0) + return os.read(r, 100) + + def check_compatible_here(self, name, filename=None, *, + strict=False, + isolated=False, + ): + # Verify that the named module may be imported in a subinterpreter. + # (See run_here() for more info.) + out = self.run_here(name, filename, + check_singlephase_setting=strict, + isolated=isolated, + ) + self.assertEqual(out, b'okay') + + def check_incompatible_here(self, name, filename=None, *, isolated=False): + # Differences from check_compatible_here(): + # * verify that import fails + # * "strict" is always True + out = self.run_here(name, filename, + check_singlephase_setting=True, + isolated=isolated, + ) + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + def check_compatible_fresh(self, name, *, strict=False, isolated=False): + # Differences from check_compatible_here(): + # * subinterpreter in a new process + # * module has never been imported before in that process + # * this tests importing the module for the first time + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=strict, + ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testinternalcapi, sys + assert ( + {name!r} in sys.builtin_module_names or + {name!r} not in sys.modules + ), repr({name!r}) + config = type(sys.implementation)(**{kwargs}) + ret = _testinternalcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + config, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual(out, b'okay') + + def check_incompatible_fresh(self, name, *, isolated=False): + # Differences from check_compatible_fresh(): + # * verify that import fails + # * "strict" is always True + kwargs = dict( + **self.RUN_KWARGS, + **(self.ISOLATED if isolated else self.NOT_ISOLATED), + check_multi_interp_extensions=True, + ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) + _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' + import _testinternalcapi, sys + assert {name!r} not in sys.modules, {name!r} + config = type(sys.implementation)(**{kwargs}) + ret = _testinternalcapi.run_in_subinterp_with_config( + {self.import_script(name, "sys.stdout.fileno()")!r}, + config, + ) + assert ret == 0, ret + ''')) + self.assertEqual(err, b'') + self.assertEqual( + out.decode('utf-8'), + f'ImportError: module {name} does not support loading in subinterpreters', + ) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_builtin_compat(self): + # For now we avoid using sys or builtins + # since they still don't implement multi-phase init. + module = '_imp' + require_builtin(module) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + + @cpython_only + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_frozen_compat(self): + module = '_frozen_importlib' + require_frozen(module, skip=True) + if __import__(module).__spec__.origin != 'frozen': + raise unittest.SkipTest(f'{module} is unexpectedly not frozen') + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + + @requires_singlephase_init + def test_single_init_extension_compat(self): + module = '_testsinglephase' + require_extension(module) + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_incompatible_here(module) + with self.subTest(f'{module}: strict, fresh'): + self.check_incompatible_fresh(module) + with self.subTest(f'{module}: isolated, fresh'): + self.check_incompatible_fresh(module, isolated=True) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_compat(self): + # Module with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + module = '_testmultiphase' + require_extension(module) + + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_non_isolated_compat(self): + # Module with Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + # and Py_MOD_GIL_NOT_USED + modname = '_test_non_isolated' + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename) + + require_extension(module) + with self.subTest(f'{modname}: isolated'): + self.check_incompatible_here(modname, filename, isolated=True) + with self.subTest(f'{modname}: not isolated'): + self.check_incompatible_here(modname, filename, isolated=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{modname}: not strict'): + self.check_compatible_here(modname, filename, strict=False) + + @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + def test_multi_init_extension_per_interpreter_gil_compat(self): + + # _test_shared_gil_only: + # Explicit Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED (default) + # and Py_MOD_GIL_NOT_USED + # _test_no_multiple_interpreter_slot: + # No Py_mod_multiple_interpreters slot + # and Py_MOD_GIL_NOT_USED + for modname in ('_test_shared_gil_only', + '_test_no_multiple_interpreter_slot'): + with self.subTest(modname=modname): + + filename = _testmultiphase.__file__ + module = import_extension_from_file(modname, filename) + + require_extension(module) + with self.subTest(f'{modname}: isolated, strict'): + self.check_incompatible_here(modname, filename, + isolated=True) + with self.subTest(f'{modname}: not isolated, strict'): + self.check_compatible_here(modname, filename, + strict=True, isolated=False) + if not Py_GIL_DISABLED: + with self.subTest(f'{modname}: not isolated, not strict'): + self.check_compatible_here( + modname, filename, strict=False, isolated=False) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_python_compat(self): + module = 'threading' + require_pure_python(module) + if not Py_GIL_DISABLED: + with self.subTest(f'{module}: not strict'): + self.check_compatible_here(module, strict=False) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True) + + @requires_singlephase_init + def test_singlephase_check_with_setting_and_override(self): + module = '_testsinglephase' + require_extension(module) + + def check_compatible(setting, override): + out = self.run_here( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertEqual(out, b'okay') + + def check_incompatible(setting, override): + out = self.run_here( + module, + check_singlephase_setting=setting, + check_singlephase_override=override, + ) + self.assertNotEqual(out, b'okay') + + with self.subTest('config: check enabled; override: enabled'): + check_incompatible(True, 1) + with self.subTest('config: check enabled; override: use config'): + check_incompatible(True, 0) + with self.subTest('config: check enabled; override: disabled'): + check_compatible(True, -1) + + with self.subTest('config: check disabled; override: enabled'): + check_incompatible(False, 1) + with self.subTest('config: check disabled; override: use config'): + check_compatible(False, 0) + with self.subTest('config: check disabled; override: disabled'): + check_compatible(False, -1) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_isolated_config(self): + module = 'threading' + require_pure_python(module) + with self.subTest(f'{module}: strict, not fresh'): + self.check_compatible_here(module, strict=True, isolated=True) + with self.subTest(f'{module}: strict, fresh'): + self.check_compatible_fresh(module, strict=True, isolated=True) + + @requires_subinterpreters + @requires_singlephase_init + def test_disallowed_reimport(self): + # See https://github.com/python/cpython/issues/104621. + script = textwrap.dedent(''' + import _testsinglephase + print(_testsinglephase) + ''') + interpid = _interpreters.create() + self.addCleanup(lambda: _interpreters.destroy(interpid)) + + excsnap = _interpreters.run_string(interpid, script) + self.assertIsNot(excsnap, None) + + excsnap = _interpreters.run_string(interpid, script) + self.assertIsNot(excsnap, None) + + +class TestSinglePhaseSnapshot(ModuleSnapshot): + """A representation of a single-phase init module for testing. + + Fields from ModuleSnapshot: + + * id - id(mod) + * module - mod or a SimpleNamespace with __file__ & __spec__ + * ns - a shallow copy of mod.__dict__ + * ns_id - id(mod.__dict__) + * cached - sys.modules[name] (or None if not there or not snapshotable) + * cached_id - id(sys.modules[name]) (or None if not there) + + Extra fields: + + * summed - the result of calling "mod.sum(1, 2)" + * lookedup - the result of calling "mod.look_up_self()" + * lookedup_id - the object ID of self.lookedup + * state_initialized - the result of calling "mod.state_initialized()" + * init_count - (optional) the result of calling "mod.initialized_count()" + + Overridden methods from ModuleSnapshot: + + * from_module() + * parse() + + Other methods from ModuleSnapshot: + + * build_script() + * from_subinterp() + + ---- + + There are 5 modules in Modules/_testsinglephase.c: + + * _testsinglephase + * has global state + * extra loads skip the init function, copy def.m_base.m_copy + * counts calls to init function + * _testsinglephase_basic_wrapper + * _testsinglephase by another name (and separate init function symbol) + * _testsinglephase_basic_copy + * same as _testsinglephase but with own def (and init func) + * _testsinglephase_with_reinit + * has no global or module state + * mod.state_initialized returns None + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + * _testsinglephase_with_state + * has module state + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + + (See Modules/_testsinglephase.c for more info.) + + For all those modules, the snapshot after the initial load (not in + the global extensions cache) would look like the following: + + * initial load + * id: ID of nww module object + * ns: exactly what the module init put there + * ns_id: ID of new module's __dict__ + * cached_id: same as self.id + * summed: 3 (never changes) + * lookedup_id: same as self.id + * state_initialized: a timestamp between the time of the load + and the time of the snapshot + * init_count: 1 (None for _testsinglephase_with_reinit) + + For the other scenarios it varies. + + For the _testsinglephase, _testsinglephase_basic_wrapper, and + _testsinglephase_basic_copy modules, the snapshot should look + like the following: + + * reloaded + * id: no change + * ns: matches what the module init function put there, + including the IDs of all contained objects, + plus any extra attributes added before the reload + * ns_id: no change + * cached_id: no change + * lookedup_id: no change + * state_initialized: no change + * init_count: no change + * already loaded + * (same as initial load except for ns and state_initialized) + * ns: matches the initial load, incl. IDs of contained objects + * state_initialized: no change from initial load + + For _testsinglephase_with_reinit: + + * reloaded: same as initial load (old module & ns is discarded) + * already loaded: same as initial load (old module & ns is discarded) + + For _testsinglephase_with_state: + + * reloaded + * (same as initial load (old module & ns is discarded), + except init_count) + * init_count: increase by 1 + * already loaded: same as reloaded + """ + + @classmethod + def from_module(cls, mod): + self = super().from_module(mod) + self.summed = mod.sum(1, 2) + self.lookedup = mod.look_up_self() + self.lookedup_id = id(self.lookedup) + self.state_initialized = mod.state_initialized() + if hasattr(mod, 'initialized_count'): + self.init_count = mod.initialized_count() + return self + + SCRIPT_BODY = ModuleSnapshot.SCRIPT_BODY + textwrap.dedent(''' + snapshot['module'].update(dict( + int_const=mod.int_const, + str_const=mod.str_const, + _module_initialized=mod._module_initialized, + )) + snapshot.update(dict( + summed=mod.sum(1, 2), + lookedup_id=id(mod.look_up_self()), + state_initialized=mod.state_initialized(), + init_count=mod.initialized_count(), + has_spam=hasattr(mod, 'spam'), + spam=getattr(mod, 'spam', None), + )) + ''').rstrip() + + @classmethod + def parse(cls, text): + self = super().parse(text) + if not self.has_spam: + del self.spam + del self.has_spam + return self + + +@requires_singlephase_init +class SinglephaseInitTests(unittest.TestCase): + + NAME = '_testsinglephase' + + @classmethod + def setUpClass(cls): + spec = importlib.util.find_spec(cls.NAME) + cls.LOADER = type(spec.loader) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader, and we need to differentiate between the + # spec.origin and the original file location. + if is_apple_mobile: + assert cls.LOADER is AppleFrameworkLoader + + cls.ORIGIN = spec.origin + with open(spec.origin + ".origin", "r") as f: + cls.FILE = os.path.join( + os.path.dirname(sys.executable), + f.read().strip() + ) + else: + assert cls.LOADER is ExtensionFileLoader + + cls.ORIGIN = spec.origin + cls.FILE = spec.origin + + # Start fresh. + cls.clean_up() + + def tearDown(self): + # Clean up the module. + self.clean_up() + + @classmethod + def clean_up(cls): + name = cls.NAME + if name in sys.modules: + if hasattr(sys.modules[name], '_clear_globals'): + assert sys.modules[name].__file__ == cls.FILE, \ + f"{sys.modules[name].__file__} != {cls.FILE}" + + sys.modules[name]._clear_globals() + del sys.modules[name] + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, cls.ORIGIN) + + ######################### + # helpers + + def add_module_cleanup(self, name): + def clean_up(): + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, self.ORIGIN) + self.addCleanup(clean_up) + + def _load_dynamic(self, name, path): + """ + Load an extension module. + """ + # This is essentially copied from the old imp module. + from importlib._bootstrap import _load + loader = self.LOADER(name, path) + + # Issue bpo-24748: Skip the sys.modules check in _load_module_shim; + # always load new extension. + spec = importlib.util.spec_from_file_location(name, path, + loader=loader) + return _load(spec) + + def load(self, name): + try: + already_loaded = self.already_loaded + except AttributeError: + already_loaded = self.already_loaded = {} + assert name not in already_loaded + mod = self._load_dynamic(name, self.ORIGIN) + self.assertNotIn(mod, already_loaded.values()) + already_loaded[name] = mod + return types.SimpleNamespace( + name=name, + module=mod, + snapshot=TestSinglePhaseSnapshot.from_module(mod), + ) + + def re_load(self, name, mod): + assert sys.modules[name] is mod + assert mod.__dict__ == mod.__dict__ + reloaded = self._load_dynamic(name, self.ORIGIN) + return types.SimpleNamespace( + name=name, + module=reloaded, + snapshot=TestSinglePhaseSnapshot.from_module(reloaded), + ) + + # subinterpreters + + def add_subinterpreter(self): + interpid = _interpreters.create('legacy') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + _interpreters.exec(interpid, textwrap.dedent(''' + import sys + import _testinternalcapi + ''')) + def clean_up(): + _interpreters.exec(interpid, textwrap.dedent(f''' + name = {self.NAME!r} + if name in sys.modules: + sys.modules.pop(name)._clear_globals() + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) + ''')) + _interpreters.destroy(interpid) + self.addCleanup(clean_up) + return interpid + + def import_in_subinterp(self, interpid=None, *, + postscript=None, + postcleanup=False, + ): + name = self.NAME + + if postcleanup: + import_ = 'import _testinternalcapi' if interpid is None else '' + postcleanup = f''' + {import_} + mod._clear_globals() + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) + ''' + + try: + pipe = self._pipe + except AttributeError: + r, w = pipe = self._pipe = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + + snapshot = TestSinglePhaseSnapshot.from_subinterp( + name, + interpid, + pipe=pipe, + import_first=True, + postscript=postscript, + postcleanup=postcleanup, + ) + + return types.SimpleNamespace( + name=name, + module=None, + snapshot=snapshot, + ) + + # checks + + def check_common(self, loaded): + isolated = False + + mod = loaded.module + if not mod: + # It came from a subinterpreter. + isolated = True + mod = loaded.snapshot.module + # mod.__name__ might not match, but the spec will. + self.assertEqual(mod.__spec__.name, loaded.name) + self.assertEqual(mod.__file__, self.FILE) + self.assertEqual(mod.__spec__.origin, self.ORIGIN) + if not isolated: + self.assertIsSubclass(mod.error, Exception) + self.assertEqual(mod.int_const, 1969) + self.assertEqual(mod.str_const, 'something different') + self.assertIsInstance(mod._module_initialized, float) + self.assertGreater(mod._module_initialized, 0) + + snap = loaded.snapshot + self.assertEqual(snap.summed, 3) + if snap.state_initialized is not None: + self.assertIsInstance(snap.state_initialized, float) + self.assertGreater(snap.state_initialized, 0) + if isolated: + # The "looked up" module is interpreter-specific + # (interp->imports.modules_by_index was set for the module). + self.assertEqual(snap.lookedup_id, snap.id) + self.assertEqual(snap.cached_id, snap.id) + with self.assertRaises(AttributeError): + snap.spam + else: + self.assertIs(snap.lookedup, mod) + self.assertIs(snap.cached, mod) + + def check_direct(self, loaded): + # The module has its own PyModuleDef, with a matching name. + self.assertEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_indirect(self, loaded, orig): + # The module re-uses another's PyModuleDef, with a different name. + assert orig is not loaded.module + assert orig.__name__ != loaded.name + self.assertNotEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_basic(self, loaded, expected_init_count): + # m_size == -1 + # The module loads fresh the first time and copies m_copy after. + snap = loaded.snapshot + self.assertIsNot(snap.state_initialized, None) + self.assertIsInstance(snap.init_count, int) + self.assertGreater(snap.init_count, 0) + self.assertEqual(snap.init_count, expected_init_count) + + def check_with_reinit(self, loaded): + # m_size >= 0 + # The module loads fresh every time. + pass + + def check_fresh(self, loaded): + """ + The module had not been loaded before (at least since fully reset). + """ + snap = loaded.snapshot + # The module's init func was run. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # _PyRuntime.imports.extensions was set. + self.assertEqual(snap.init_count, 1) + # The global state was initialized. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + + def check_semi_fresh(self, loaded, base, prev): + """ + The module had been loaded before and then reset + (but the module global state wasn't). + """ + snap = loaded.snapshot + # The module's init func was run again. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertNotEqual(snap.id, prev.snapshot.id) + self.assertEqual(snap.init_count, prev.snapshot.init_count + 1) + # The global state was updated. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertNotEqual(snap.state_initialized, + base.snapshot.state_initialized) + self.assertNotEqual(snap.state_initialized, + prev.snapshot.state_initialized) + + def check_copied(self, loaded, base): + """ + The module had been loaded before and never reset. + """ + snap = loaded.snapshot + # The module's init func was not run again. + # The interpreter copied m_copy, as set by the other interpreter, + # with objects owned by the other interpreter. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertEqual(snap.init_count, base.snapshot.init_count) + # The global state was not updated since the init func did not run. + # The module attrs were not directly initialized from that state. + # The state and module attrs still match the previous loading. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertEqual(snap.state_initialized, + base.snapshot.state_initialized) + + ######################### + # the tests + + def test_cleared_globals(self): + loaded = self.load(self.NAME) + _testsinglephase = loaded.module + init_before = _testsinglephase.state_initialized() + + _testsinglephase._clear_globals() + init_after = _testsinglephase.state_initialized() + init_count = _testsinglephase.initialized_count() + + self.assertGreater(init_before, 0) + self.assertEqual(init_after, 0) + self.assertEqual(init_count, -1) + + def test_variants(self): + # Exercise the most meaningful variants described in Python/import.c. + self.maxDiff = None + + # Check the "basic" module. + + name = self.NAME + expected_init_count = 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + basic = loaded.module + + # Check its indirect variants. + + name = f'{self.NAME}_basic_wrapper' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_indirect(loaded, basic) + self.check_basic(loaded, expected_init_count) + + # Currently PyState_AddModule() always replaces the cached module. + self.assertIs(basic.look_up_self(), loaded.module) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # The cached module shouldn't change after this point. + basic_lookedup = loaded.module + + # Check its direct variant. + + name = f'{self.NAME}_basic_copy' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the non-basic variant that has no state. + + name = f'{self.NAME}_with_reinit' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.assertIs(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the basic variant that has state. + + name = f'{self.NAME}_with_state' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + self.addCleanup(loaded.module._clear_module_state) + + self.check_common(loaded) + self.assertIsNot(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + def test_basic_reloaded(self): + # m_copy is copied into the existing module object. + # Global state is not changed. + self.maxDiff = None + + for name in [ + self.NAME, # the "basic" module + f'{self.NAME}_basic_wrapper', # the indirect variant + f'{self.NAME}_basic_copy', # the direct variant + ]: + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + reloaded = self.re_load(name, loaded.module) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIs(reloaded.module, loaded.module) + self.assertIs(reloaded.module.__dict__, loaded.module.__dict__) + # It only happens to be the same but that's good enough here. + # We really just want to verify that the re-loaded attrs + # didn't change. + self.assertIs(reloaded.snapshot.lookedup, + loaded.snapshot.lookedup) + self.assertEqual(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + self.assertEqual(reloaded.snapshot.init_count, + loaded.snapshot.init_count) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + def test_with_reinit_reloaded(self): + # The module's m_init func is run again. + self.maxDiff = None + + # Keep a reference around. + basic = self.load(self.NAME) + + for name, has_state in [ + (f'{self.NAME}_with_reinit', False), # m_size == 0 + (f'{self.NAME}_with_state', True), # m_size > 0 + ]: + self.add_module_cleanup(name) + with self.subTest(name=name, has_state=has_state): + loaded = self.load(name) + if has_state: + self.addCleanup(loaded.module._clear_module_state) + + reloaded = self.re_load(name, loaded.module) + if has_state: + self.addCleanup(reloaded.module._clear_module_state) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIsNot(reloaded.module, loaded.module) + self.assertNotEqual(reloaded.module.__dict__, + loaded.module.__dict__) + self.assertIs(reloaded.snapshot.lookedup, reloaded.module) + if loaded.snapshot.state_initialized is None: + self.assertIs(reloaded.snapshot.state_initialized, None) + else: + self.assertGreater(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_check_state_first(self): + for variant in ['', '_with_reinit', '_with_state']: + name = f'{self.NAME}{variant}_check_cache_first' + with self.subTest(name): + mod = self._load_dynamic(name, self.ORIGIN) + self.assertEqual(mod.__name__, name) + sys.modules.pop(name, None) + _testinternalcapi.clear_extension(name, self.ORIGIN) + + # Currently, for every single-phrase init module loaded + # in multiple interpreters, those interpreters share a + # PyModuleDef for that object, which can be a problem. + # Also, we test with a single-phase module that has global state, + # which is shared by all interpreters. + + @no_rerun(reason="module state is not cleared (see gh-140657)") + @requires_subinterpreters + def test_basic_multiple_interpreters_main_no_reset(self): + # without resetting; already loaded in main interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + main_loaded = self.load(self.NAME) + _testsinglephase = main_loaded.module + # Attrs set after loading are not in m_copy. + _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' + + self.check_common(main_loaded) + self.check_fresh(main_loaded) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # At this point: + # * alive in 1 interpreter (main) + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy was copied from the main interpreter (was NULL) + # * module's global state was initialized + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp() + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 1 interpreter (main) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy is NULL (cleared when the interpreter was destroyed) + # (was from main interpreter) + # * module's global state was updated, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 2 interpreters (main, interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 + # * module's global state was updated, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 3 interpreters (main, interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was updated, not reset + + @no_rerun(reason="rerun not possible; module state is never cleared (see gh-102251)") + @requires_subinterpreters + def test_basic_multiple_interpreters_deleted_no_reset(self): + # without resetting; already loaded in a deleted interpreter + + if Py_TRACE_REFS: + # It's a Py_TRACE_REFS build. + # This test breaks interpreter isolation a little, + # which causes problems on Py_TRACE_REF builds. + raise unittest.SkipTest('crashes on Py_TRACE_REFS builds') + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # First, load in the main interpreter but then completely clear it. + loaded_main = self.load(self.NAME) + loaded_main.module._clear_globals() + _testinternalcapi.clear_extension(self.NAME, self.ORIGIN) + + # At this point: + # * alive in 0 interpreters + # * module def loaded already + # * module def was in _PyRuntime.imports.extensions, but cleared + # * mod init func ran for the first time (since reset, at least) + # * m_copy was set, but cleared (was NULL) + # * module's global state was initialized but cleared + + # Start with an interpreter that gets destroyed right away. + base = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''') + self.check_common(base) + self.check_fresh(base) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset) + # * m_copy is still set (owned by main interpreter) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded_interp1 = self.import_in_subinterp(interpid1) + self.check_common(loaded_interp1) + self.check_copied(loaded_interp1, base) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched + + # Use a subinterpreter while the previous one is still alive. + loaded_interp2 = self.import_in_subinterp(interpid2) + self.check_common(loaded_interp2) + self.check_copied(loaded_interp2, loaded_interp1) + + # At this point: + # * alive in 2 interpreters (interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched + + @requires_subinterpreters + def test_basic_multiple_interpreters_reset_each(self): + # resetting between each interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''', + postcleanup=True, + ) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy is NULL (cleared when the interpreter was destroyed) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 (was NULL) + # * module's global state was initialized, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 2 interpreters (interp2, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was initialized, not reset + + +@cpython_only +class TestMagicNumber(unittest.TestCase): + def test_magic_number_endianness(self): + magic_number_bytes = _imp.pyc_magic_number_token.to_bytes(4, 'little') + self.assertEqual(magic_number_bytes[2:], b'\r\n') + # Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. + magic_number = int.from_bytes(magic_number_bytes[:2], 'little') + start = 2900 + sys.version_info.minor * 50 + self.assertIn(magic_number, range(start, start + 50)) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_import/data/circular_imports/import_cycle.py b/Lib/test/test_import/data/circular_imports/import_cycle.py new file mode 100644 index 00000000000..cd9507b5f69 --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/import_cycle.py @@ -0,0 +1,3 @@ +import test.test_import.data.circular_imports.import_cycle as m + +m.some_attribute diff --git a/Lib/test/test_import/data/circular_imports/singlephase.py b/Lib/test/test_import/data/circular_imports/singlephase.py new file mode 100644 index 00000000000..05618bc72f9 --- /dev/null +++ b/Lib/test/test_import/data/circular_imports/singlephase.py @@ -0,0 +1,13 @@ +"""Circular import involving a single-phase-init extension. + +This module is imported from the _testsinglephase_circular module from +_testsinglephase, and imports that module again. +""" + +import importlib +import _testsinglephase +from test.test_import import import_extension_from_file + +name = '_testsinglephase_circular' +filename = _testsinglephase.__file__ +mod = import_extension_from_file(name, filename) diff --git a/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py b/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py +++ b/Lib/test/test_import/data/circular_imports/subpkg2/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_import/data/double_const.py b/Lib/test/test_import/data/double_const.py new file mode 100644 index 00000000000..67852aaf982 --- /dev/null +++ b/Lib/test/test_import/data/double_const.py @@ -0,0 +1,30 @@ +from test.support import TestFailed + +# A test for SF bug 422177: manifest float constants varied way too much in +# precision depending on whether Python was loading a module for the first +# time, or reloading it from a precompiled .pyc. The "expected" failure +# mode is that when test_import imports this after all .pyc files have been +# erased, it passes, but when test_import imports this from +# double_const.pyc, it fails. This indicates a woeful loss of precision in +# the marshal format for doubles. It's also possible that repr() doesn't +# produce enough digits to get reasonable precision for this box. + +PI = 3.14159265358979324 +TWOPI = 6.28318530717958648 + +PI_str = "3.14159265358979324" +TWOPI_str = "6.28318530717958648" + +# Verify that the double x is within a few bits of eval(x_str). +def check_ok(x, x_str): + assert x > 0.0 + x2 = eval(x_str) + assert x2 > 0.0 + diff = abs(x - x2) + # If diff is no larger than 3 ULP (wrt x2), then diff/8 is no larger + # than 0.375 ULP, so adding diff/8 to x2 should have no effect. + if x2 + (diff / 8.) != x2: + raise TestFailed("Manifest const %s lost too much precision " % x_str) + +check_ok(PI, PI_str) +check_ok(TWOPI, TWOPI_str) diff --git a/Lib/test/test_import/data/package/submodule.py b/Lib/test/test_import/data/package/submodule.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_import/data/package/submodule.py +++ b/Lib/test/test_import/data/package/submodule.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_import/data/package2/submodule2.py b/Lib/test/test_import/data/package2/submodule2.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_import/data/package2/submodule2.py +++ b/Lib/test/test_import/data/package2/submodule2.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_import/data/package3/__init__.py b/Lib/test/test_import/data/package3/__init__.py new file mode 100644 index 00000000000..7033c22a719 --- /dev/null +++ b/Lib/test/test_import/data/package3/__init__.py @@ -0,0 +1,2 @@ +"""Rebinding the package attribute after importing the module.""" +from .submodule import submodule diff --git a/Lib/test/test_import/data/package3/submodule.py b/Lib/test/test_import/data/package3/submodule.py new file mode 100644 index 00000000000..cd7b30db15e --- /dev/null +++ b/Lib/test/test_import/data/package3/submodule.py @@ -0,0 +1,7 @@ +attr = 'submodule' +class A: + attr = 'submodule' +class submodule: + attr = 'rebound' + class B: + attr = 'rebound' diff --git a/Lib/test/test_import/data/package4/__init__.py b/Lib/test/test_import/data/package4/__init__.py new file mode 100644 index 00000000000..d8af60ab38a --- /dev/null +++ b/Lib/test/test_import/data/package4/__init__.py @@ -0,0 +1,5 @@ +"""Binding the package attribute without importing the module.""" +class submodule: + attr = 'origin' + class B: + attr = 'origin' diff --git a/Lib/test/test_import/data/package4/submodule.py b/Lib/test/test_import/data/package4/submodule.py new file mode 100644 index 00000000000..c861417aece --- /dev/null +++ b/Lib/test/test_import/data/package4/submodule.py @@ -0,0 +1,3 @@ +attr = 'submodule' +class A: + attr = 'submodule' diff --git a/Lib/test/test_import/data/unwritable/x.py b/Lib/test/test_import/data/unwritable/x.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_import/data/unwritable/x.py +++ b/Lib/test/test_import/data/unwritable/x.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py index 111c4af1ea7..1fb1d2f9efa 100644 --- a/Lib/test/test_importlib/builtin/test_finder.py +++ b/Lib/test/test_importlib/builtin/test_finder.py @@ -4,7 +4,6 @@ import sys import unittest -import warnings @unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') diff --git a/Lib/test/test_importlib/extension/_test_nonmodule_cases.py b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py new file mode 100644 index 00000000000..8ffd18d221d --- /dev/null +++ b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py @@ -0,0 +1,44 @@ +import types +import unittest +from test.test_importlib import util + +machinery = util.import_importlib('importlib.machinery') + +from test.test_importlib.extension.test_loader import MultiPhaseExtensionModuleTests + + +class NonModuleExtensionTests: + setUp = MultiPhaseExtensionModuleTests.setUp + load_module_by_name = MultiPhaseExtensionModuleTests.load_module_by_name + + def _test_nonmodule(self): + # Test returning a non-module object from create works. + name = self.name + '_nonmodule' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + + # issue 27782 + def test_nonmodule_with_methods(self): + # Test creating a non-module object with methods defined. + name = self.name + '_nonmodule_with_methods' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + self.assertEqual(mod.bar(10, 1), 9) + + def test_null_slots(self): + # Test that NULL slots aren't a problem. + name = self.name + '_null_slots' + module = self.load_module_by_name(name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, name) + + +(Frozen_NonModuleExtensionTests, + Source_NonModuleExtensionTests + ) = util.test_both(NonModuleExtensionTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py index 0bb74fff5fc..5183719162e 100644 --- a/Lib/test/test_importlib/extension/test_case_sensitivity.py +++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py @@ -1,4 +1,3 @@ -from importlib import _bootstrap_external from test.support import os_helper import unittest import sys @@ -8,7 +7,8 @@ machinery = util.import_importlib('importlib.machinery') -@unittest.skipIf(util.EXTENSIONS.filename is None, f'{util.EXTENSIONS.name} not available') +@unittest.skipIf(util.EXTENSIONS is None or util.EXTENSIONS.filename is None, + 'dynamic loading not supported or test module not available') @util.case_insensitive_tests class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase): diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 35ff9fbef58..cdc8884d668 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -11,7 +12,7 @@ class FinderTests(abc.FinderTests): """Test the finder for extension modules.""" def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") if util.EXTENSIONS.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -19,14 +20,30 @@ def setUp(self): ) def find_spec(self, fullname): - importer = self.machinery.FileFinder(util.EXTENSIONS.path, - (self.machinery.ExtensionFileLoader, - self.machinery.EXTENSION_SUFFIXES)) + if is_apple_mobile: + # Apple mobile platforms require a specialist loader that uses + # .fwork files as placeholders for the true `.so` files. + loaders = [ + ( + self.machinery.AppleFrameworkLoader, + [ + ext.replace(".so", ".fwork") + for ext in self.machinery.EXTENSION_SUFFIXES + ] + ) + ] + else: + loaders = [ + ( + self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES + ) + ] + + importer = self.machinery.FileFinder(util.EXTENSIONS.path, *loaders) return importer.find_spec(fullname) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_module(self): self.assertTrue(self.find_spec(util.EXTENSIONS.name)) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index d06558f2ade..0dd21e079eb 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -1,4 +1,4 @@ -from warnings import catch_warnings +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -10,7 +10,8 @@ import warnings import importlib.util import importlib -from test.support.script_helper import assert_python_failure +from test import support +from test.support import MISSING_C_DOCSTRINGS, script_helper class LoaderTests: @@ -18,14 +19,21 @@ class LoaderTests: """Test ExtensionFileLoader.""" def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") if util.EXTENSIONS.name in sys.builtin_module_names: raise unittest.SkipTest( f"{util.EXTENSIONS.name} is a builtin module" ) - self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + + self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) def load_module(self, fullname): with warnings.catch_warnings(): @@ -33,13 +41,11 @@ def load_module(self, fullname): return self.loader.load_module(fullname) def test_equality(self): - other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertEqual(self.loader, other) def test_inequality(self): - other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) def test_load_module_API(self): @@ -51,8 +57,6 @@ def test_load_module_API(self): with self.assertRaises(ImportError): self.load_module('XXX') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_module(self): with util.uncache(util.EXTENSIONS.name): module = self.load_module(util.EXTENSIONS.name) @@ -61,8 +65,7 @@ def test_module(self): ('__package__', '')]: self.assertEqual(getattr(module, attr), value) self.assertIn(util.EXTENSIONS.name, sys.modules) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -85,13 +88,11 @@ def test_module_reuse(self): module2 = self.load_module(util.EXTENSIONS.name) self.assertIs(module1, module2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_is_package(self): self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: path = os.path.join('some', 'path', 'pkg', '__init__' + suffix) - loader = self.machinery.ExtensionFileLoader('pkg', path) + loader = self.LoaderClass('pkg', path) self.assertTrue(loader.is_package('pkg')) @@ -104,8 +105,16 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules without multi-phase initialization. def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testsinglephase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -114,8 +123,8 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): with warnings.catch_warnings(): @@ -125,7 +134,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -142,8 +151,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -181,13 +189,20 @@ def test_unloadable_nonascii(self): ) = util.test_both(SinglePhaseExtensionModuleTests, machinery=machinery) -# @unittest.skip("TODO: RUSTPYTHON, AssertionError") class MultiPhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules with multi-phase initialization (PEP 489). def setUp(self): - if not self.machinery.EXTENSION_SUFFIXES: + if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testmultiphase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -196,8 +211,7 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): # Load the module from the test extension. @@ -208,7 +222,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -234,8 +248,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) def test_functionality(self): # Test basic functionality of stuff defined in an extension module. @@ -313,29 +326,6 @@ def test_unloadable_nonascii(self): self.load_module_by_name(name) self.assertEqual(cm.exception.name, name) - def test_nonmodule(self): - # Test returning a non-module object from create works. - name = self.name + '_nonmodule' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - - # issue 27782 - def test_nonmodule_with_methods(self): - # Test creating a non-module object with methods defined. - name = self.name + '_nonmodule_with_methods' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - self.assertEqual(mod.bar(10, 1), 9) - - def test_null_slots(self): - # Test that NULL slots aren't a problem. - name = self.name + '_null_slots' - module = self.load_module_by_name(name) - self.assertIsInstance(module, types.ModuleType) - self.assertEqual(module.__name__, name) - def test_bad_modules(self): # Test SystemError is raised for misbehaving extensions. for name_base in [ @@ -380,7 +370,8 @@ def test_nonascii(self): with self.subTest(name): module = self.load_module_by_name(name) self.assertEqual(module.__name__, name) - self.assertEqual(module.__doc__, "Module named in %s" % lang) + if not MISSING_C_DOCSTRINGS: + self.assertEqual(module.__doc__, "Module named in %s" % lang) (Frozen_MultiPhaseExtensionModuleTests, @@ -388,5 +379,14 @@ def test_nonascii(self): ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery) +class NonModuleExtensionTests(unittest.TestCase): + def test_nonmodule_cases(self): + # The test cases in this file cause the GIL to be enabled permanently + # in free-threaded builds, so they are run in a subprocess to isolate + # this effect. + script = support.findfile("test_importlib/extension/_test_nonmodule_cases.py") + script_helper.run_test_script(script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index ec9644dc520..941dcd5432c 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -5,6 +5,8 @@ import unittest +@unittest.skipIf(util.EXTENSIONS is None or util.EXTENSIONS.filename is None, + 'dynamic loading not supported or test module not available') class PathHookTests: """Test the path hook for extension modules.""" @@ -19,7 +21,7 @@ def hook(self, entry): def test_success(self): # Path hook should handle a directory where a known extension module # exists. - self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_spec')) + self.assertHasAttr(self.hook(util.EXTENSIONS.path), 'find_spec') (Frozen_PathHooksTests, diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index 5bb075f3770..971cc28b6d3 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -2,11 +2,8 @@ machinery = util.import_importlib('importlib.machinery') -import _imp -import marshal import os.path import unittest -import warnings from test.support import import_helper, REPO_ROOT, STDLIB_DIR @@ -70,8 +67,6 @@ def check_search_locations(self, spec): expected = [os.path.dirname(filename)] self.assertListEqual(spec.submodule_search_locations, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_module(self): modules = [ '__hello__', @@ -114,8 +109,6 @@ def test_module(self): self.check_basic(spec, name) self.check_loader_state(spec, origname, filename) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_package(self): packages = [ '__phello__', @@ -170,8 +163,6 @@ def test_failure(self): spec = self.find('<not real>') self.assertIsNone(spec) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_not_using_frozen(self): finder = self.machinery.FrozenImporter with import_helper.frozen_modules(enabled=False): diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index 4f1af454b52..0824af53e05 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -3,10 +3,9 @@ machinery = util.import_importlib('importlib.machinery') from test.support import captured_stdout, import_helper, STDLIB_DIR -import _imp import contextlib -import marshal import os.path +import sys import types import unittest import warnings @@ -63,7 +62,7 @@ def exec_module(self, name, origname=None): module.main() self.assertTrue(module.initialized) - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') self.assertEqual(module.__spec__.origin, 'frozen') return module, stdout.getvalue() @@ -74,9 +73,10 @@ def test_module(self): for attr, value in check.items(): self.assertEqual(getattr(module, attr), value) self.assertEqual(output, 'Hello world!\n') - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') self.assertEqual(module.__spec__.loader_state.origname, name) + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON") def test_package(self): name = '__phello__' module, output = self.exec_module(name) @@ -138,7 +138,7 @@ def test_get_code(self): exec(code, mod.__dict__) with captured_stdout() as stdout: mod.main() - self.assertTrue(hasattr(mod, 'initialized')) + self.assertHasAttr(mod, 'initialized') self.assertEqual(stdout.getvalue(), 'Hello world!\n') def test_get_source(self): @@ -147,6 +147,7 @@ def test_get_source(self): result = self.machinery.FrozenImporter.get_source('__hello__') self.assertIsNone(result) + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON") def test_is_package(self): # Should be able to tell what is a package. test_for = (('__hello__', False), ('__phello__', True), diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index a14163919af..858b37effc6 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -1,8 +1,5 @@ from importlib import machinery -import sys -import types import unittest -import warnings from test.test_importlib import util diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 431faea5b4e..7130c99a6fc 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -56,8 +56,6 @@ def test_using___name__(self): '__path__': []}) self.assertEqual(module.__name__, 'pkg') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warn_when_using___name__(self): with self.assertWarns(ImportWarning): self.import_module({'__name__': 'pkg.fake', '__path__': []}) @@ -75,8 +73,6 @@ def test_spec_fallback(self): module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) self.assertEqual(module.__name__, 'pkg') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warn_when_package_and_spec_disagree(self): # Raise a DeprecationWarning if __package__ != __spec__.parent. with self.assertWarns(DeprecationWarning): diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index aedf0fd4f9d..718e7d041b0 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -78,7 +78,7 @@ def test_using_cache_for_assigning_to_attribute(self): with self.create_mock('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg.module') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(id(module.module), id(sys.modules['pkg.module'])) @@ -88,7 +88,7 @@ def test_using_cache_for_fromlist(self): with self.create_mock('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(id(module.module), id(sys.modules['pkg.module'])) diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py index 4b4b9bc3f5e..feccc7be09a 100644 --- a/Lib/test/test_importlib/import_/test_fromlist.py +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -63,7 +63,7 @@ def test_nonexistent_object(self): with util.import_state(meta_path=[importer]): module = self.__import__('module', fromlist=['non_existent']) self.assertEqual(module.__name__, 'module') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_module_from_package(self): # [module] @@ -71,7 +71,7 @@ def test_module_from_package(self): with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.__name__, 'pkg.module') def test_nonexistent_from_package(self): @@ -79,7 +79,7 @@ def test_nonexistent_from_package(self): with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['non_existent']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_module_from_package_triggers_ModuleNotFoundError(self): # If a submodule causes an ModuleNotFoundError because it tries @@ -107,7 +107,7 @@ def basic_star_test(self, fromlist=['*']): mock['pkg'].__all__ = ['module'] module = self.__import__('pkg', fromlist=fromlist) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.__name__, 'pkg.module') def test_using_star(self): @@ -125,8 +125,8 @@ def test_star_with_others(self): mock['pkg'].__all__ = ['module1'] module = self.__import__('pkg', fromlist=['module2', '*']) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module1')) - self.assertTrue(hasattr(module, 'module2')) + self.assertHasAttr(module, 'module1') + self.assertHasAttr(module, 'module2') self.assertEqual(module.module1.__name__, 'pkg.module1') self.assertEqual(module.module2.__name__, 'pkg.module2') @@ -136,7 +136,7 @@ def test_nonexistent_in_all(self): importer['pkg'].__all__ = ['non_existent'] module = self.__import__('pkg', fromlist=['*']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_star_in_all(self): with util.mock_spec('pkg.__init__') as importer: @@ -144,7 +144,7 @@ def test_star_in_all(self): importer['pkg'].__all__ = ['*'] module = self.__import__('pkg', fromlist=['*']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, '*')) + self.assertNotHasAttr(module, '*') def test_invalid_type(self): with util.mock_spec('pkg.__init__') as importer: diff --git a/Lib/test/test_importlib/import_/test_helpers.py b/Lib/test/test_importlib/import_/test_helpers.py index 28cdc0e526e..550f88d1d7a 100644 --- a/Lib/test/test_importlib/import_/test_helpers.py +++ b/Lib/test/test_importlib/import_/test_helpers.py @@ -126,8 +126,6 @@ def test_gh86298_loader_is_none_and_spec_loader_is_none(self): ValueError, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_no_spec(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -137,8 +135,6 @@ def test_gh86298_no_spec(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_spec_is_none(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -148,8 +144,6 @@ def test_gh86298_spec_is_none(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_no_spec_loader(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -159,8 +153,6 @@ def test_gh86298_no_spec_loader(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_loader_and_spec_loader_disagree(self): bar = ModuleType('bar') bar.__loader__ = object() diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index 26e7b070b95..4c00f60681a 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -30,8 +30,6 @@ def test_continuing(self): with util.import_state(meta_path=[first, second]): self.assertIs(self.__import__(mod_name), second.modules[mod_name]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty(self): # Raise an ImportWarning if sys.meta_path is empty. module_name = 'nothing' @@ -45,7 +43,7 @@ def test_empty(self): self.assertIsNone(importlib._bootstrap._find_spec('nothing', None)) self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, ImportWarning)) + self.assertIsSubclass(w[-1].category, ImportWarning) (Frozen_CallingOrder, diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py index eb0831f7d6d..0c29d608326 100644 --- a/Lib/test/test_importlib/import_/test_packages.py +++ b/Lib/test/test_importlib/import_/test_packages.py @@ -1,7 +1,6 @@ from test.test_importlib import util import sys import unittest -from test import support from test.support import import_helper diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index 9cf3a77cb84..79e0bdca94c 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -1,3 +1,4 @@ +from test.support import os_helper from test.test_importlib import util importlib = util.import_importlib('importlib') @@ -68,8 +69,6 @@ def test_path_hooks(self): self.assertIn(path, sys.path_importer_cache) self.assertIs(sys.path_importer_cache[path], importer) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty_path_hooks(self): # Test that if sys.path_hooks is empty a warning is raised, # sys.path_importer_cache gets None set, and PathFinder returns None. @@ -82,7 +81,7 @@ def test_empty_path_hooks(self): self.assertIsNone(self.find('os')) self.assertIsNone(sys.path_importer_cache[path_entry]) self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, ImportWarning)) + self.assertIsSubclass(w[-1].category, ImportWarning) def test_path_importer_cache_empty_string(self): # The empty string should create a finder using the cwd. @@ -155,6 +154,32 @@ def test_deleted_cwd(self): # Do not want FileNotFoundError raised. self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + @os_helper.skip_unless_working_chmod + def test_permission_error_cwd(self): + # gh-115911: Test that an unreadable CWD does not break imports, in + # particular during early stages of interpreter startup. + + def noop_hook(*args): + raise ImportError + + with ( + os_helper.temp_dir() as new_dir, + os_helper.save_mode(new_dir), + os_helper.change_cwd(new_dir), + util.import_state(path=[''], path_hooks=[noop_hook]), + ): + # chmod() is done here (inside the 'with' block) because the order + # of teardown operations cannot be the reverse of setup order. See + # https://github.com/python/cpython/pull/116131#discussion_r1739649390 + try: + os.chmod(new_dir, 0o000) + except OSError: + self.skipTest("platform does not allow " + "changing mode of the cwd") + + # Do not want PermissionError raised. + self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + def test_invalidate_caches_finders(self): # Finders with an invalidate_caches() method have it called. class FakeFinder: diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 99c24f1fd94..1549cbe96ce 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -81,7 +81,7 @@ def callback(global_): self.__import__('pkg') # For __import__(). module = self.__import__('', global_, fromlist=['mod2'], level=1) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'mod2')) + self.assertHasAttr(module, 'mod2') self.assertEqual(module.mod2.attr, 'pkg.mod2') self.relative_import_test(create, globals_, callback) @@ -107,7 +107,7 @@ def callback(global_): module = self.__import__('', global_, fromlist=['module'], level=1) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.attr, 'pkg.module') self.relative_import_test(create, globals_, callback) @@ -131,7 +131,7 @@ def callback(global_): module = self.__import__('', global_, fromlist=['subpkg2'], level=2) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'subpkg2')) + self.assertHasAttr(module, 'subpkg2') self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__') self.relative_import_test(create, globals_, callback) @@ -223,6 +223,21 @@ def test_relative_import_no_package_exists_absolute(self): self.__import__('sys', {'__package__': '', '__spec__': None}, level=1) + def test_malicious_relative_import(self): + # https://github.com/python/cpython/issues/134100 + # Test to make sure UAF bug with error msg doesn't come back to life + import sys + loooong = "".ljust(0x23000, "b") + name = f"a.{loooong}.c" + + with util.uncache(name): + sys.modules[name] = {} + with self.assertRaisesRegex( + KeyError, + r"'a\.b+' not in sys\.modules as expected" + ): + __import__(f"{loooong}.c", {"__package__": "a"}, level=1) + (Frozen_RelativeImports, Source_RelativeImports diff --git a/Lib/test/test_importlib/metadata/__init__.py b/Lib/test/test_importlib/metadata/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/Lib/test/test_importlib/metadata/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/metadata/_context.py b/Lib/test/test_importlib/metadata/_context.py new file mode 100644 index 00000000000..8a53eb55d15 --- /dev/null +++ b/Lib/test/test_importlib/metadata/_context.py @@ -0,0 +1,13 @@ +import contextlib + + +# from jaraco.context 4.3 +class suppress(contextlib.suppress, contextlib.ContextDecorator): + """ + A version of contextlib.suppress with decorator support. + + >>> @suppress(KeyError) + ... def key_error(): + ... {}[''] + >>> key_error() + """ diff --git a/Lib/test/test_importlib/metadata/_path.py b/Lib/test/test_importlib/metadata/_path.py new file mode 100644 index 00000000000..b3cfb9cd549 --- /dev/null +++ b/Lib/test/test_importlib/metadata/_path.py @@ -0,0 +1,115 @@ +# from jaraco.path 3.7 + +import functools +import pathlib +from typing import Dict, Protocol, Union +from typing import runtime_checkable + + +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] # type: ignore + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover + + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore +): + """ + Build a set of files/directories, as described by the spec. + + Each key represents a pathname, and the value represents + the content. Content may be a nested directory. + + >>> spec = { + ... 'README.txt': "A README file", + ... "foo": { + ... "__init__.py": "", + ... "bar": { + ... "__init__.py": "", + ... }, + ... "baz.py": "# Some code", + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), + ... } + >>> target = getfixture('tmp_path') + >>> build(spec, target) + >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') + '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' + """ + for name, contents in spec.items(): + create(contents, _ensure_tree_maker(prefix) / name) + + +@functools.singledispatch +def create(content: Union[str, bytes, FilesSpec], path): + path.mkdir(exist_ok=True) + build(content, prefix=path) # type: ignore + + +@create.register +def _(content: bytes, path): + path.write_bytes(content) + + +@create.register +def _(content: str, path): + path.write_text(content, encoding='utf-8') + + +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + +class Recording: + """ + A TreeMaker object that records everything that would be written. + + >>> r = Recording() + >>> build({'foo': {'foo1.txt': 'yes'}, 'bar.txt': 'abc'}, r) + >>> r.record + ['foo/foo1.txt', 'bar.txt'] + """ + + def __init__(self, loc=pathlib.PurePosixPath(), record=None): + self.loc = loc + self.record = record if record is not None else [] + + def __truediv__(self, other): + return Recording(self.loc / other, self.record) + + def write_text(self, content, **kwargs): + self.record.append(str(self.loc)) + + write_bytes = write_text + + def mkdir(self, **kwargs): + return + + def symlink_to(self, target): + pass diff --git a/Lib/test/test_importlib/metadata/data/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl new file mode 100644 index 00000000000..641ab07f7aa Binary files /dev/null and b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl differ diff --git a/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg new file mode 100644 index 00000000000..cdb298a19b0 Binary files /dev/null and b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg differ diff --git a/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl new file mode 100644 index 00000000000..5ca93657f81 Binary files /dev/null and b/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl differ diff --git a/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py new file mode 100644 index 00000000000..ba73b743394 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py @@ -0,0 +1,2 @@ +def main(): + return 'example' diff --git a/Lib/test/test_importlib/metadata/data/sources/example/setup.py b/Lib/test/test_importlib/metadata/data/sources/example/setup.py new file mode 100644 index 00000000000..479488a0348 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name='example', + version='21.12', + license='Apache Software License', + packages=['example'], + entry_points={ + 'console_scripts': ['example = example:main', 'Example=example:main'], + }, +) diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py new file mode 100644 index 00000000000..de645c2e8bc --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py @@ -0,0 +1,2 @@ +def main(): + return "example" diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml new file mode 100644 index 00000000000..011f4751fb9 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'trampolim' +requires = ['trampolim'] + +[project] +name = 'example2' +version = '1.0.0' + +[project.scripts] +example = 'example2:main' diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py new file mode 100644 index 00000000000..826b1b3259b --- /dev/null +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -0,0 +1,395 @@ +import sys +import copy +import json +import shutil +import pathlib +import textwrap +import functools +import contextlib + +from test.support import import_helper +from test.support import os_helper +from test.support import requires_zlib + +from . import _path +from ._path import FilesSpec + + +try: + from importlib import resources # type: ignore + + getattr(resources, 'files') + getattr(resources, 'as_file') +except (ImportError, AttributeError): + import importlib_resources as resources # type: ignore + + +@contextlib.contextmanager +def tmp_path(): + """ + Like os_helper.temp_dir, but yields a pathlib.Path. + """ + with os_helper.temp_dir() as path: + yield pathlib.Path(path) + + +@contextlib.contextmanager +def install_finder(finder): + sys.meta_path.append(finder) + try: + yield + finally: + sys.meta_path.remove(finder) + + +class Fixtures: + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + +class SiteDir(Fixtures): + def setUp(self): + super().setUp() + self.site_dir = self.fixtures.enter_context(tmp_path()) + + +class OnSysPath(Fixtures): + @staticmethod + @contextlib.contextmanager + def add_sys_path(dir): + sys.path[:0] = [str(dir)] + try: + yield + finally: + sys.path.remove(str(dir)) + + def setUp(self): + super().setUp() + self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + self.fixtures.enter_context(import_helper.isolated_modules()) + + +class SiteBuilder(SiteDir): + def setUp(self): + super().setUp() + for cls in self.__class__.mro(): + with contextlib.suppress(AttributeError): + build_files(cls.files, prefix=self.site_dir) + + +class DistInfoPkg(OnSysPath, SiteBuilder): + files: FilesSpec = { + "distinfo_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Author: Steven Ma + Version: 1.0.0 + Requires-Dist: wheel >= 1.0 + Requires-Dist: pytest; extra == 'test' + Keywords: sample package + + Once upon a time + There was a distinfo pkg + """, + "RECORD": "mod.py,sha256=abc,20\n", + "entry_points.txt": """ + [entries] + main = mod:main + ns:sub = mod:main + """, + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def make_uppercase(self): + """ + Rewrite metadata with everything uppercase. + """ + shutil.rmtree(self.site_dir / "distinfo_pkg-1.0.0.dist-info") + files = copy.deepcopy(DistInfoPkg.files) + info = files["distinfo_pkg-1.0.0.dist-info"] + info["METADATA"] = info["METADATA"].upper() + build_files(files, self.site_dir) + + +class DistInfoPkgEditable(DistInfoPkg): + """ + Package with a PEP 660 direct_url.json. + """ + + some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' + files: FilesSpec = { + 'distinfo_pkg-1.0.0.dist-info': { + 'direct_url.json': json.dumps({ + "archive_info": { + "hash": f"sha256={some_hash}", + "hashes": {"sha256": f"{some_hash}"}, + }, + "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", + }) + }, + } + + +class DistInfoPkgWithDot(OnSysPath, SiteBuilder): + files: FilesSpec = { + "pkg_dot-1.0.0.dist-info": { + "METADATA": """ + Name: pkg.dot + Version: 1.0.0 + """, + }, + } + + +class DistInfoPkgWithDotLegacy(OnSysPath, SiteBuilder): + files: FilesSpec = { + "pkg.dot-1.0.0.dist-info": { + "METADATA": """ + Name: pkg.dot + Version: 1.0.0 + """, + }, + "pkg.lot.egg-info": { + "METADATA": """ + Name: pkg.lot + Version: 1.0.0 + """, + }, + } + + +class DistInfoPkgOffPath(SiteBuilder): + files = DistInfoPkg.files + + +class EggInfoPkg(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egginfo_pkg.egg-info": { + "PKG-INFO": """ + Name: egginfo-pkg + Author: Steven Ma + License: Unknown + Version: 1.0.0 + Classifier: Intended Audience :: Developers + Classifier: Topic :: Software Development :: Libraries + Keywords: sample package + Description: Once upon a time + There was an egginfo package + """, + "SOURCES.txt": """ + mod.py + egginfo_pkg.egg-info/top_level.txt + """, + "entry_points.txt": """ + [entries] + main = mod:main + """, + "requires.txt": """ + wheel >= 1.0; python_version >= "2.7" + [test] + pytest + """, + "top_level.txt": "mod\n", + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module.json + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../../../etc/jupyter/jupyter_notebook_config.d/relative.json + /etc/jupyter/jupyter_notebook_config.d/absolute.json + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_no_modules_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_no_modules-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + setup.py + egg_with_no_modules_pkg.egg-info/PKG-INFO + egg_with_no_modules_pkg.egg-info/SOURCES.txt + egg_with_no_modules_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + PKG-INFO + SOURCES.txt + top_level.txt + """, + # top_level.txt correctly reflects that no modules are installed + "top_level.txt": b"\n", + }, + } + + +class EggInfoPkgSourcesFallback(OnSysPath, SiteBuilder): + files: FilesSpec = { + "sources_fallback_pkg.egg-info": { + "PKG-INFO": "Name: sources_fallback-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + sources_fallback.py + setup.py + sources_fallback_pkg.egg-info/PKG-INFO + sources_fallback_pkg.egg-info/SOURCES.txt + """, + # missing installed-files.txt (i.e. not installed by pip) and + # missing top_level.txt (to trigger fallback to SOURCES.txt) + }, + "sources_fallback.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoFile(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egginfo_file.egg-info": """ + Metadata-Version: 1.0 + Name: egginfo_file + Version: 0.1 + Summary: An example package + Home-page: www.example.com + Author: Eric Haffa-Vee + Author-email: eric@example.coms + License: UNKNOWN + Description: UNKNOWN + Platform: UNKNOWN + """, + } + + +# dedent all text strings before writing +orig = _path.create.registry[str] +_path.create.register(str, lambda content, path: orig(DALS(content), path)) + + +build_files = _path.build + + +def build_record(file_defs): + return ''.join(f'{name},,\n' for name in record_names(file_defs)) + + +def record_names(file_defs): + recording = _path.Recording() + _path.build(file_defs, recording) + return recording.record + + +class FileBuilder: + def unicode_filename(self): + return os_helper.FS_NONASCII or self.skip( + "File system does not support non-ascii." + ) + + +def DALS(str): + "Dedent and left-strip" + return textwrap.dedent(str).lstrip() + + +@requires_zlib() +class ZipFixtures: + root = 'test.test_importlib.metadata.data' + + def _fixture_on_path(self, filename): + pkg_file = resources.files(self.root).joinpath(filename) + file = self.resources.enter_context(resources.as_file(pkg_file)) + assert file.name.startswith('example'), file.name + sys.path.insert(0, str(file)) + self.resources.callback(sys.path.pop, 0) + + def setUp(self): + # Add self.zip_name to the front of sys.path. + self.resources = contextlib.ExitStack() + self.addCleanup(self.resources.close) + + +def parameterize(*args_set): + """Run test method with a series of parameters.""" + + def wrapper(func): + @functools.wraps(func) + def _inner(self): + for args in args_set: + with self.subTest(**args): + func(self, **args) + + return _inner + + return wrapper diff --git a/Lib/test/test_importlib/metadata/stubs.py b/Lib/test/test_importlib/metadata/stubs.py new file mode 100644 index 00000000000..e5b011c399f --- /dev/null +++ b/Lib/test/test_importlib/metadata/stubs.py @@ -0,0 +1,10 @@ +import unittest + + +class fake_filesystem_unittest: + """ + Stubbed version of the pyfakefs module + """ + class TestCase(unittest.TestCase): + def setUpPyfakefs(self): + self.skipTest("pyfakefs not available") diff --git a/Lib/test/test_importlib/metadata/test_api.py b/Lib/test/test_importlib/metadata/test_api.py new file mode 100644 index 00000000000..2256e0c502e --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -0,0 +1,323 @@ +import re +import textwrap +import unittest +import warnings +import importlib +import contextlib + +from . import fixtures +from importlib.metadata import ( + Distribution, + PackageNotFoundError, + distribution, + entry_points, + files, + metadata, + requires, + version, +) + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default', category=DeprecationWarning) + yield ctx + + +class APITests( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgPipInstalledExternalDataFiles, + fixtures.EggInfoPkgSourcesFallback, + fixtures.DistInfoPkg, + fixtures.DistInfoPkgWithDot, + fixtures.EggInfoFile, + unittest.TestCase, +): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + pkg_version = version('egginfo-pkg') + assert isinstance(pkg_version, str) + assert re.match(self.version_pattern, pkg_version) + + def test_retrieves_version_of_distinfo_pkg(self): + pkg_version = version('distinfo-pkg') + assert isinstance(pkg_version, str) + assert re.match(self.version_pattern, pkg_version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + distribution('does-not-exist') + + def test_name_normalization(self): + names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.dot' + + def test_prefix_not_matched(self): + prefixes = 'p', 'pkg', 'pkg.' + for prefix in prefixes: + with self.subTest(prefix): + with self.assertRaises(PackageNotFoundError): + distribution(prefix) + + def test_for_top_level(self): + tests = [ + ('egginfo-pkg', 'mod'), + ('egg_with_no_modules-pkg', ''), + ] + for pkg_name, expect_content in tests: + with self.subTest(pkg_name): + self.assertEqual( + distribution(pkg_name).read_text('top_level.txt').strip(), + expect_content, + ) + + def test_read_text(self): + tests = [ + ('egginfo-pkg', 'mod\n'), + ('egg_with_no_modules-pkg', '\n'), + ] + for pkg_name, expect_content in tests: + with self.subTest(pkg_name): + top_level = [ + path for path in files(pkg_name) if path.name == 'top_level.txt' + ][0] + self.assertEqual(top_level.read_text(), expect_content) + + def test_entry_points(self): + eps = entry_points() + assert 'entries' in eps.groups + entries = eps.select(group='entries') + assert 'main' in entries.names + ep = entries['main'] + self.assertEqual(ep.value, 'mod:main') + self.assertEqual(ep.extras, []) + + def test_entry_points_distribution(self): + entries = entry_points(group='entries') + for entry in ("main", "ns:sub"): + ep = entries[entry] + self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) + self.assertEqual(ep.dist.version, "1.0.0") + + def test_entry_points_unique_packages_normalized(self): + """ + Entry points should only be exposed for the first package + on sys.path with a given name (even when normalized). + """ + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + alt_pkg = { + "DistInfo_pkg-1.1.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Version: 1.1.0 + """, + "entry_points.txt": """ + [entries] + main = mod:altmain + """, + }, + } + fixtures.build_files(alt_pkg, alt_site_dir) + entries = entry_points(group='entries') + assert not any( + ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' + for ep in entries + ) + # ns:sub doesn't exist in alt_pkg + assert 'ns:sub' not in entries.names + + def test_entry_points_missing_name(self): + with self.assertRaises(KeyError): + entry_points(group='entries')['missing'] + + def test_entry_points_missing_group(self): + assert entry_points(group='missing') == () + + def test_entry_points_allows_no_attributes(self): + ep = entry_points().select(group='entries', name='main') + with self.assertRaises(AttributeError): + ep.foo = 4 + + def test_metadata_for_this_package(self): + md = metadata('egginfo-pkg') + assert md['author'] == 'Steven Ma' + assert md['LICENSE'] == 'Unknown' + assert md['Name'] == 'egginfo-pkg' + classifiers = md.get_all('Classifier') + assert 'Topic :: Software Development :: Libraries' in classifiers + + def test_missing_key_legacy(self): + """ + Requesting a missing key will still return None, but warn. + """ + md = metadata('distinfo-pkg') + with suppress_known_deprecation(): + assert md['does-not-exist'] is None + + def test_get_key(self): + """ + Getting a key gets the key. + """ + md = metadata('egginfo-pkg') + assert md.get('Name') == 'egginfo-pkg' + + def test_get_missing_key(self): + """ + Requesting a missing key will return None. + """ + md = metadata('distinfo-pkg') + assert md.get('does-not-exist') is None + + @staticmethod + def _test_files(files): + root = files[0].root + for file in files: + assert file.root == root + assert not file.hash or file.hash.value + assert not file.hash or file.hash.mode == 'sha256' + assert not file.size or file.size >= 0 + assert file.locate().exists() + assert isinstance(file.read_binary(), bytes) + if file.name.endswith('.py'): + file.read_text() + + def test_file_hash_repr(self): + util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0] + self.assertRegex(repr(util.hash), '<FileHash mode: sha256 value: .*>') + + def test_files_dist_info(self): + self._test_files(files('distinfo-pkg')) + + def test_files_egg_info(self): + self._test_files(files('egginfo-pkg')) + self._test_files(files('egg_with_module-pkg')) + self._test_files(files('egg_with_no_modules-pkg')) + self._test_files(files('sources_fallback-pkg')) + + def test_version_egg_info_file(self): + self.assertEqual(version('egginfo-file'), '0.1') + + def test_requires_egg_info_file(self): + requirements = requires('egginfo-file') + self.assertIsNone(requirements) + + def test_requires_egg_info(self): + deps = requires('egginfo-pkg') + assert len(deps) == 2 + assert any(dep == 'wheel >= 1.0; python_version >= "2.7"' for dep in deps) + + def test_requires_egg_info_empty(self): + fixtures.build_files( + { + 'requires.txt': '', + }, + self.site_dir.joinpath('egginfo_pkg.egg-info'), + ) + deps = requires('egginfo-pkg') + assert deps == [] + + def test_requires_dist_info(self): + deps = requires('distinfo-pkg') + assert len(deps) == 2 + assert all(deps) + assert 'wheel >= 1.0' in deps + assert "pytest; extra == 'test'" in deps + + def test_more_complex_deps_requires_text(self): + requires = textwrap.dedent( + """ + dep1 + dep2 + + [:python_version < "3"] + dep3 + + [extra1] + dep4 + dep6@ git+https://example.com/python/dep.git@v1.0.0 + + [extra2:python_version < "3"] + dep5 + """ + ) + deps = sorted(Distribution._deps_from_requires_text(requires)) + expected = [ + 'dep1', + 'dep2', + 'dep3; python_version < "3"', + 'dep4; extra == "extra1"', + 'dep5; (python_version < "3") and extra == "extra2"', + 'dep6@ git+https://example.com/python/dep.git@v1.0.0 ; extra == "extra1"', + ] + # It's important that the environment marker expression be + # wrapped in parentheses to avoid the following 'and' binding more + # tightly than some other part of the environment expression. + + assert deps == expected + + def test_as_json(self): + md = metadata('distinfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['requires_dist']) == 2 + + def test_as_json_egg_info(self): + md = metadata('egginfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['classifier']) == 2 + + def test_as_json_odd_case(self): + self.make_uppercase() + md = metadata('distinfo-pkg').json + assert 'name' in md + assert len(md['requires_dist']) == 2 + assert md['keywords'] == ['SAMPLE', 'PACKAGE'] + + +class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): + def test_name_normalization(self): + names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.dot' + + def test_name_normalization_versionless_egg_info(self): + names = 'pkg.lot', 'pkg_lot', 'pkg-lot', 'pkg..lot', 'Pkg.Lot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.lot' + + +class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase): + def test_find_distributions_specified_path(self): + dists = Distribution.discover(path=[str(self.site_dir)]) + assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) + + def test_distribution_at_pathlib(self): + """Demonstrate how to load metadata direct from a directory.""" + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(dist_info_path) + assert dist.version == '1.0.0' + + def test_distribution_at_str(self): + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(str(dist_info_path)) + assert dist.version == '1.0.0' + + +class InvalidateCache(unittest.TestCase): + def test_invalidate_cache(self): + # No externally observable behavior, but ensures test coverage... + importlib.invalidate_caches() diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py new file mode 100644 index 00000000000..e4218076f8c --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -0,0 +1,468 @@ +import re +import pickle +import unittest +import warnings +import importlib +import importlib.metadata +import contextlib +from test.support import os_helper + +try: + import pyfakefs.fake_filesystem_unittest as ffs +except ImportError: + from .stubs import fake_filesystem_unittest as ffs + +from . import fixtures +from ._context import suppress +from ._path import Symlink +from importlib.metadata import ( + Distribution, + EntryPoint, + PackageNotFoundError, + _unique, + distributions, + entry_points, + metadata, + packages_distributions, + version, +) + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default', category=DeprecationWarning) + yield ctx + + +class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + dist = Distribution.from_name('distinfo-pkg') + assert isinstance(dist.version, str) + assert re.match(self.version_pattern, dist.version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + Distribution.from_name('does-not-exist') + + def test_package_not_found_mentions_metadata(self): + """ + When a package is not found, that could indicate that the + package is not installed or that it is installed without + metadata. Ensure the exception mentions metadata to help + guide users toward the cause. See #124. + """ + with self.assertRaises(PackageNotFoundError) as ctx: + Distribution.from_name('does-not-exist') + + assert "metadata" in str(ctx.exception) + + # expected to fail until ABC is enforced + @suppress(AssertionError) + @suppress_known_deprecation() + def test_abc_enforced(self): + with self.assertRaises(TypeError): + type('DistributionSubclass', (Distribution,), {})() + + @fixtures.parameterize( + dict(name=None), + dict(name=''), + ) + def test_invalid_inputs_to_from_name(self, name): + with self.assertRaises(Exception): + Distribution.from_name(name) + + +class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): + def test_import_nonexistent_module(self): + # Ensure that the MetadataPathFinder does not crash an import of a + # non-existent module. + with self.assertRaises(ImportError): + importlib.import_module('does_not_exist') + + def test_resolve(self): + ep = entry_points(group='entries')['main'] + self.assertEqual(ep.load().__name__, "main") + + def test_entrypoint_with_colon_in_name(self): + ep = entry_points(group='entries')['ns:sub'] + self.assertEqual(ep.value, 'mod:main') + + def test_resolve_without_attr(self): + ep = EntryPoint( + name='ep', + value='importlib.metadata', + group='grp', + ) + assert ep.load() is importlib.metadata + + +class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def make_pkg(name): + """ + Create minimal metadata for a dist-info package with + the indicated name on the file system. + """ + return { + f'{name}.dist-info': { + 'METADATA': 'VERSION: 1.0\n', + }, + } + + def test_dashes_in_dist_name_found_as_underscores(self): + """ + For a package with a dash in the name, the dist-info metadata + uses underscores in the name. Ensure the metadata loads. + """ + fixtures.build_files(self.make_pkg('my_pkg'), self.site_dir) + assert version('my-pkg') == '1.0' + + def test_dist_name_found_as_any_case(self): + """ + Ensure the metadata loads when queried with any case. + """ + pkg_name = 'CherryPy' + fixtures.build_files(self.make_pkg(pkg_name), self.site_dir) + assert version(pkg_name) == '1.0' + assert version(pkg_name.lower()) == '1.0' + assert version(pkg_name.upper()) == '1.0' + + def test_unique_distributions(self): + """ + Two distributions varying only by non-normalized name on + the file system should resolve as the same. + """ + fixtures.build_files(self.make_pkg('abc'), self.site_dir) + before = list(_unique(distributions())) + + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) + after = list(_unique(distributions())) + + assert len(after) == len(before) + + +class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_non_ascii_description(site_dir): + """ + Create minimal metadata for a package with non-ASCII in + the description. + """ + contents = { + 'portend.dist-info': { + 'METADATA': 'Description: pôrˈtend', + }, + } + fixtures.build_files(contents, site_dir) + return 'portend' + + @staticmethod + def pkg_with_non_ascii_description_egg_info(site_dir): + """ + Create minimal metadata for an egg-info package with + non-ASCII in the description. + """ + contents = { + 'portend.dist-info': { + 'METADATA': """ + Name: portend + + pôrˈtend""", + }, + } + fixtures.build_files(contents, site_dir) + return 'portend' + + def test_metadata_loads(self): + pkg_name = self.pkg_with_non_ascii_description(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + def test_metadata_loads_egg_info(self): + pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + +class DiscoveryTests( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgSourcesFallback, + fixtures.DistInfoPkg, + unittest.TestCase, +): + def test_package_discovery(self): + dists = list(distributions()) + assert all(isinstance(dist, Distribution) for dist in dists) + assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'egg_with_module-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'egg_with_no_modules-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'sources_fallback-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) + + def test_invalid_usage(self): + with self.assertRaises(ValueError): + list(distributions(context='something', name='else')) + + def test_interleaved_discovery(self): + """ + Ensure interleaved searches are safe. + + When the search is cached, it is possible for searches to be + interleaved, so make sure those use-cases are safe. + + Ref #293 + """ + dists = distributions() + next(dists) + version('egginfo-pkg') + next(dists) + + +class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + def test_egg_info(self): + # make an `EGG-INFO` directory that's unrelated + self.site_dir.joinpath('EGG-INFO').mkdir() + # used to crash with `IsADirectoryError` + with self.assertRaises(PackageNotFoundError): + version('unknown-package') + + def test_egg(self): + egg = self.site_dir.joinpath('foo-3.6.egg') + egg.mkdir() + with self.add_sys_path(egg): + with self.assertRaises(PackageNotFoundError): + version('foo') + + +class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): + site_dir = '/does-not-exist' + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + importlib.metadata.distributions() + + +class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): + site_dir = '/access-denied' + + def setUp(self): + super().setUp() + self.setUpPyfakefs() + self.fs.create_dir(self.site_dir, perm_bits=000) + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + list(importlib.metadata.distributions()) + + +class TestEntryPoints(unittest.TestCase): + def __init__(self, *args): + super().__init__(*args) + self.ep = importlib.metadata.EntryPoint( + name='name', value='value', group='group' + ) + + def test_entry_point_pickleable(self): + revived = pickle.loads(pickle.dumps(self.ep)) + assert revived == self.ep + + def test_positional_args(self): + """ + Capture legacy (namedtuple) construction, discouraged. + """ + EntryPoint('name', 'value', 'group') + + def test_immutable(self): + """EntryPoints should be immutable""" + with self.assertRaises(AttributeError): + self.ep.name = 'badactor' + + def test_repr(self): + assert 'EntryPoint' in repr(self.ep) + assert 'name=' in repr(self.ep) + assert "'name'" in repr(self.ep) + + def test_hashable(self): + """EntryPoints should be hashable""" + hash(self.ep) + + def test_module(self): + assert self.ep.module == 'value' + + def test_attr(self): + assert self.ep.attr is None + + def test_sortable(self): + """ + EntryPoint objects are sortable, but result is undefined. + """ + sorted([ + EntryPoint(name='b', value='val', group='group'), + EntryPoint(name='a', value='val', group='group'), + ]) + + +class FileSystem( + fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, unittest.TestCase +): + def test_unicode_dir_on_sys_path(self): + """ + Ensure a Unicode subdirectory of a directory on sys.path + does not crash. + """ + fixtures.build_files( + {self.unicode_filename(): {}}, + prefix=self.site_dir, + ) + list(distributions()) + + +class PackagesDistributionsPrebuiltTest(fixtures.ZipFixtures, unittest.TestCase): + def test_packages_distributions_example(self): + self._fixture_on_path('example-21.12-py3-none-any.whl') + assert packages_distributions()['example'] == ['example'] + + def test_packages_distributions_example2(self): + """ + Test packages_distributions on a wheel built + by trampolim. + """ + self._fixture_on_path('example2-1.0.0-py3-none-any.whl') + assert packages_distributions()['example2'] == ['example2'] + + +class PackagesDistributionsTest( + fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase +): + def test_packages_distributions_neither_toplevel_nor_files(self): + """ + Test a package built without 'top-level.txt' or a file list. + """ + fixtures.build_files( + { + 'trim_example-1.0.0.dist-info': { + 'METADATA': """ + Name: trim_example + Version: 1.0.0 + """, + } + }, + prefix=self.site_dir, + ) + packages_distributions() + + def test_packages_distributions_all_module_types(self): + """ + Test top-level modules detected on a package without 'top-level.txt'. + """ + suffixes = importlib.machinery.all_suffixes() + metadata = dict( + METADATA=""" + Name: all_distributions + Version: 1.0.0 + """, + ) + files = { + 'all_distributions-1.0.0.dist-info': metadata, + } + for i, suffix in enumerate(suffixes): + files.update({ + f'importable-name {i}{suffix}': '', + f'in_namespace_{i}': { + f'mod{suffix}': '', + }, + f'in_package_{i}': { + '__init__.py': '', + f'mod{suffix}': '', + }, + }) + metadata.update(RECORD=fixtures.build_record(files)) + fixtures.build_files(files, prefix=self.site_dir) + + distributions = packages_distributions() + + for i in range(len(suffixes)): + assert distributions[f'importable-name {i}'] == ['all_distributions'] + assert distributions[f'in_namespace_{i}'] == ['all_distributions'] + assert distributions[f'in_package_{i}'] == ['all_distributions'] + + assert not any(name.endswith('.dist-info') for name in distributions) + + @os_helper.skip_unless_symlink + def test_packages_distributions_symlinked_top_level(self) -> None: + """ + Distribution is resolvable from a simple top-level symlink in RECORD. + See #452. + """ + + files: fixtures.FilesSpec = { + "symlinked_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: symlinked-pkg + Version: 1.0.0 + """, + "RECORD": "symlinked,,\n", + }, + ".symlink.target": {}, + "symlinked": Symlink(".symlink.target"), + } + + fixtures.build_files(files, self.site_dir) + assert packages_distributions()['symlinked'] == ['symlinked-pkg'] + + +class PackagesDistributionsEggTest( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgSourcesFallback, + unittest.TestCase, +): + def test_packages_distributions_on_eggs(self): + """ + Test old-style egg packages with a variation of 'top_level.txt', + 'SOURCES.txt', and 'installed-files.txt', available. + """ + distributions = packages_distributions() + + def import_names_from_package(package_name): + return { + import_name + for import_name, package_names in distributions.items() + if package_name in package_names + } + + # egginfo-pkg declares one import ('mod') via top_level.txt + assert import_names_from_package('egginfo-pkg') == {'mod'} + + # egg_with_module-pkg has one import ('egg_with_module') inferred from + # installed-files.txt (top_level.txt is missing) + assert import_names_from_package('egg_with_module-pkg') == {'egg_with_module'} + + # egg_with_no_modules-pkg should not be associated with any import names + # (top_level.txt is empty, and installed-files.txt has no .py files) + assert import_names_from_package('egg_with_no_modules-pkg') == set() + + # sources_fallback-pkg has one import ('sources_fallback') inferred from + # SOURCES.txt (top_level.txt and installed-files.txt is missing) + assert import_names_from_package('sources_fallback-pkg') == {'sources_fallback'} + + +class EditableDistributionTest(fixtures.DistInfoPkgEditable, unittest.TestCase): + def test_origin(self): + dist = Distribution.from_name('distinfo-pkg') + assert dist.origin.url.endswith('.whl') + assert dist.origin.archive_info.hashes.sha256 diff --git a/Lib/test/test_importlib/metadata/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py new file mode 100644 index 00000000000..276f6288c91 --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_zip.py @@ -0,0 +1,62 @@ +import sys +import unittest + +from . import fixtures +from importlib.metadata import ( + PackageNotFoundError, + distribution, + distributions, + entry_points, + files, + version, +) + + +class TestZip(fixtures.ZipFixtures, unittest.TestCase): + def setUp(self): + super().setUp() + self._fixture_on_path('example-21.12-py3-none-any.whl') + + def test_zip_version(self): + self.assertEqual(version('example'), '21.12') + + def test_zip_version_does_not_match(self): + with self.assertRaises(PackageNotFoundError): + version('definitely-not-installed') + + def test_zip_entry_points(self): + scripts = entry_points(group='console_scripts') + entry_point = scripts['example'] + self.assertEqual(entry_point.value, 'example:main') + entry_point = scripts['Example'] + self.assertEqual(entry_point.value, 'example:main') + + def test_missing_metadata(self): + self.assertIsNone(distribution('example').read_text('does not exist')) + + def test_case_insensitive(self): + self.assertEqual(version('Example'), '21.12') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.whl/' in path, path + + def test_one_distribution(self): + dists = list(distributions(path=sys.path[:1])) + assert len(dists) == 1 + + +class TestEgg(TestZip): + def setUp(self): + super().setUp() + self._fixture_on_path('example-21.12-py3.6.egg') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.egg/' in path, path + + def test_normalized_name(self): + dist = distribution('example') + assert dist._normalized_name == 'example' diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py +++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/resources/__init__.py b/Lib/test/test_importlib/resources/__init__.py index e69de29bb2d..8b137891791 100644 --- a/Lib/test/test_importlib/resources/__init__.py +++ b/Lib/test/test_importlib/resources/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index 1f97c961469..b144628cb73 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -2,15 +2,44 @@ import functools from typing import Dict, Union +from typing import runtime_checkable +from typing import Protocol #### -# from jaraco.path 3.4.1 +# from jaraco.path 3.7.1 -FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover -def build(spec: FilesSpec, prefix=pathlib.Path()): + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value] + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment] +): """ Build a set of files/directories, as described by the spec. @@ -25,21 +54,25 @@ def build(spec: FilesSpec, prefix=pathlib.Path()): ... "__init__.py": "", ... }, ... "baz.py": "# Some code", - ... } + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), ... } >>> target = getfixture('tmp_path') >>> build(spec, target) >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): - create(contents, pathlib.Path(prefix) / name) + create(contents, _ensure_tree_maker(prefix) / name) @functools.singledispatch def create(content: Union[str, bytes, FilesSpec], path): path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore + build(content, prefix=path) # type: ignore[arg-type] @create.register @@ -52,5 +85,10 @@ def _(content: str, path): path.write_text(content, encoding='utf-8') +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + # end from jaraco.path #### diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 1a13f043a86..4e4e0e9c337 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -19,25 +18,21 @@ def test_contents(self): assert self.expected <= contents -class ContentsDiskTests(ContentsTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ContentsDiskTests(ContentsTests, util.DiskSetup, unittest.TestCase): + pass class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): pass -class ContentsNamespaceTests(ContentsTests, unittest.TestCase): +class ContentsNamespaceTests(ContentsTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + expected = { # no __init__ because of namespace design - # no subdirectory as incidental difference in fixture 'binary.file', + 'subdirectory', 'utf-16.file', 'utf-8.file', } - - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 73127209a27..640f90fc0dd 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -5,6 +5,7 @@ from test.support import os_helper from importlib import resources +from importlib.resources import abc from importlib.resources.abc import TraversableResources, ResourceReader from . import util @@ -39,8 +40,9 @@ def setUp(self): self.addCleanup(self.fixtures.close) def test_custom_loader(self): - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + temp_dir = pathlib.Path(self.fixtures.enter_context(os_helper.temp_dir())) loader = SimpleLoader(MagicResources(temp_dir)) pkg = util.create_package_from_loader(loader) files = resources.files(pkg) - assert files is temp_dir + assert isinstance(files, abc.Traversable) + assert list(files.iterdir()) == [] diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 1450cfb3109..3ce44999f98 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,4 +1,5 @@ -import typing +import pathlib +import py_compile import textwrap import unittest import warnings @@ -7,11 +8,8 @@ from importlib import resources from importlib.resources.abc import Traversable -from . import data01 from . import util -from . import _path -from test.support import os_helper -from test.support import import_helper +from test.support import os_helper, import_helper @contextlib.contextmanager @@ -32,13 +30,14 @@ def test_read_text(self): actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' - @unittest.skipUnless( - hasattr(typing, 'runtime_checkable'), - "Only suitable when typing supports runtime_checkable", - ) def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_joinpath_with_multiple_args(self): + files = resources.files(self.data) + binfile = files.joinpath('subdirectory', 'binary.file') + self.assertTrue(binfile.is_file()) + def test_old_parameter(self): """ Files used to take a 'package' parameter. Make sure anyone @@ -48,66 +47,145 @@ def test_old_parameter(self): resources.files(package=self.data) -class OpenDiskTests(FilesTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): + pass class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass -class OpenNamespaceTests(FilesTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 +class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_non_paths_in_dunder_path(self): + """ + Non-path items in a namespace package's ``__path__`` are ignored. + + As reported in python/importlib_resources#311, some tools + like Setuptools, when creating editable packages, will inject + non-paths into a namespace package's ``__path__``, a + sentinel like + ``__editable__.sample_namespace-1.0.finder.__path_hook__`` + to cause the ``PathEntryFinder`` to be called when searching + for packages. In that case, resources should still be loadable. + """ + import namespacedata01 + + namespacedata01.__path__.append( + '__editable__.sample_namespace-1.0.finder.__path_hook__' + ) + + resources.files(namespacedata01) + + +class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' - self.data = namespacedata01 +class DirectSpec: + """ + Override behavior of ModuleSetup to write a full spec directly. + """ -class SiteDir: - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) - self.fixtures.enter_context(import_helper.CleanImport()) + MODULE = 'unused' + def load_fixture(self, name): + self.tree_on_path(self.spec) + + +class ModulesFiles: + spec = { + 'mod.py': '', + 'res.txt': 'resources are the best', + } -class ModulesFilesTests(SiteDir, unittest.TestCase): def test_module_resources(self): """ A module can have resources found adjacent to the module. """ - spec = { - 'mod.py': '', - 'res.txt': 'resources are the best', - } - _path.build(spec, self.site_dir) - import mod + import mod # type: ignore[import-not-found] actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') - assert actual == spec['res.txt'] + assert actual == self.spec['res.txt'] + + +class ModuleFilesDiskTests(DirectSpec, util.DiskSetup, ModulesFiles, unittest.TestCase): + pass -class ImplicitContextFilesTests(SiteDir, unittest.TestCase): - def test_implicit_files(self): +class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.TestCase): + pass + + +class ImplicitContextFiles: + set_val = textwrap.dedent( + f""" + import {resources.__name__} as res + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') + """ + ) + spec = { + 'somepkg': { + '__init__.py': set_val, + 'submod.py': set_val, + 'res.txt': 'resources are the best', + }, + 'frozenpkg': { + '__init__.py': set_val.replace(resources.__name__, 'c_resources'), + 'res.txt': 'resources are the best', + }, + } + + def test_implicit_files_package(self): """ Without any parameter, files() will infer the location as the caller. """ - spec = { - 'somepkg': { - '__init__.py': textwrap.dedent( - """ - import importlib.resources as res - val = res.files().joinpath('res.txt').read_text(encoding='utf-8') - """ - ), - 'res.txt': 'resources are the best', - }, - } - _path.build(spec, self.site_dir) assert importlib.import_module('somepkg').val == 'resources are the best' + def test_implicit_files_submodule(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + assert importlib.import_module('somepkg.submod').val == 'resources are the best' + + def _compile_importlib(self): + """ + Make a compiled-only copy of the importlib resources package. + + Currently only code is copied, as importlib resources doesn't itself + have any resources. + """ + bin_site = self.fixtures.enter_context(os_helper.temp_dir()) + c_resources = pathlib.Path(bin_site, 'c_resources') + sources = pathlib.Path(resources.__file__).parent + + for source_path in sources.glob('**/*.py'): + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + py_compile.compile(source_path, c_path) + self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) + + def test_implicit_files_with_compiled_importlib(self): + """ + Caller detection works for compiled-only resources module. + + python/cpython#123085 + """ + self._compile_importlib() + assert importlib.import_module('frozenpkg').val == 'resources are the best' + + +class ImplicitContextFilesDiskTests( + DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase +): + pass + + +class ImplicitContextFilesZipTests( + DirectSpec, util.ZipSetup, ImplicitContextFiles, unittest.TestCase +): + pass + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py new file mode 100644 index 00000000000..e8d25fa4d9f --- /dev/null +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -0,0 +1,249 @@ +import unittest +import os +import importlib + +from test.support import warnings_helper + +from importlib import resources + +from . import util + +# Since the functional API forwards to Traversable, we only test +# filesystem resources here -- not zip files, namespace packages etc. +# We do test for two kinds of Anchor, though. + + +class StringAnchorMixin: + anchor01 = 'data01' + anchor02 = 'data02' + + +class ModuleAnchorMixin: + @property + def anchor01(self): + return importlib.import_module('data01') + + @property + def anchor02(self): + return importlib.import_module('data02') + + +class FunctionalAPIBase(util.DiskSetup): + def setUp(self): + super().setUp() + self.load_fixture('data02') + + def _gen_resourcetxt_path_parts(self): + """Yield various names of a text file in anchor02, each in a subTest""" + for path_parts in ( + ('subdirectory', 'subsubdir', 'resource.txt'), + ('subdirectory/subsubdir/resource.txt',), + ('subdirectory/subsubdir', 'resource.txt'), + ): + with self.subTest(path_parts=path_parts): + yield path_parts + + def test_read_text(self): + self.assertEqual( + resources.read_text(self.anchor01, 'utf-8.file'), + 'Hello, UTF-8 world!\n', + ) + self.assertEqual( + resources.read_text( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + encoding='utf-8', + ), + 'a resource', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ), + 'a resource', + ) + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.read_text(self.anchor01) + with self.assertRaises(OSError): + resources.read_text(self.anchor01, 'no-such-file') + with self.assertRaises(UnicodeDecodeError): + resources.read_text(self.anchor01, 'utf-16.file') + self.assertEqual( + resources.read_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ), + '\x00\x01\x02\x03', + ) + self.assertEndsWith( # ignore the BOM + resources.read_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_read_binary(self): + self.assertEqual( + resources.read_binary(self.anchor01, 'utf-8.file'), + b'Hello, UTF-8 world!\n', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_binary(self.anchor02, *path_parts), + b'a resource', + ) + + def test_open_text(self): + with resources.open_text(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ) as f: + self.assertEqual(f.read(), 'a resource') + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.open_text(self.anchor01) + with self.assertRaises(OSError): + resources.open_text(self.anchor01, 'no-such-file') + with resources.open_text(self.anchor01, 'utf-16.file') as f: + with self.assertRaises(UnicodeDecodeError): + f.read() + with resources.open_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ) as f: + self.assertEqual(f.read(), '\x00\x01\x02\x03') + with resources.open_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ) as f: + self.assertEndsWith( # ignore the BOM + f.read(), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_open_binary(self): + with resources.open_binary(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_binary( + self.anchor02, + *path_parts, + ) as f: + self.assertEqual(f.read(), b'a resource') + + def test_path(self): + with resources.path(self.anchor01, 'utf-8.file') as path: + with open(str(path), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + with resources.path(self.anchor01) as path: + with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + + def test_is_resource(self): + is_resource = resources.is_resource + self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) + self.assertFalse(is_resource(self.anchor01, 'no_such_file')) + self.assertFalse(is_resource(self.anchor01)) + self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertTrue(is_resource(self.anchor02, *path_parts)) + + def test_contents(self): + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01) + self.assertGreaterEqual( + set(c), + {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, + ) + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, 'utf-8.file')) + + for path_parts in self._gen_resourcetxt_path_parts(): + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, *path_parts)) + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01, 'subdirectory') + self.assertGreaterEqual( + set(c), + {'binary.file'}, + ) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_common_errors(self): + for func in ( + resources.read_text, + resources.read_binary, + resources.open_text, + resources.open_binary, + resources.path, + resources.is_resource, + resources.contents, + ): + with self.subTest(func=func): + # Rejecting None anchor + with self.assertRaises(TypeError): + func(None) + # Rejecting invalid anchor type + with self.assertRaises((TypeError, AttributeError)): + func(1234) + # Unknown module + with self.assertRaises(ModuleNotFoundError): + func('$missing module$') + + def test_text_errors(self): + for func in ( + resources.read_text, + resources.open_text, + ): + with self.subTest(func=func): + # Multiple path arguments need explicit encoding argument. + with self.assertRaises(TypeError): + func( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + ) + + +class FunctionalAPITest_StringAnchor( + StringAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_ModuleAnchor( + ModuleAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 86becb4bfaa..8c00378ad3c 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -24,7 +23,7 @@ def test_open_binary(self): target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, b'\x00\x01\x02\x03') + self.assertEqual(result, bytes(range(4))) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' @@ -65,21 +64,21 @@ def test_open_text_FileNotFoundError(self): target.open(encoding='utf-8') -class OpenDiskTests(OpenTests, unittest.TestCase): - def setUp(self): - self.data = data01 - +class OpenDiskTests(OpenTests, util.DiskSetup, unittest.TestCase): + pass -class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class OpenDiskNamespaceTests(OpenTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 34a6bdd2d58..903911f57b3 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,8 +1,8 @@ import io +import pathlib import unittest from importlib import resources -from . import data01 from . import util @@ -15,23 +15,16 @@ def execute(self, package, path): class PathTests: def test_reading(self): """ - Path should be readable. - - Test also implicitly verifies the returned object is a pathlib.Path - instance. + Path should be readable and a pathlib.Path instance. """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: - self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) - # pathlib.Path.read_text() was introduced in Python 3.5. - with path.open('r', encoding='utf-8') as file: - text = file.read() - self.assertEqual('Hello, UTF-8 world!\n', text) - + self.assertIsInstance(path, pathlib.Path) + self.assertEndsWith(path.name, "utf-8.file") + self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) -class PathDiskTests(PathTests, unittest.TestCase): - data = data01 +class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): def test_natural_path(self): # Guarantee the internal implementation detail that # file-system-backed resources do not get the tempdir diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 088982681e8..59c237d9641 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -1,7 +1,7 @@ import unittest from importlib import import_module, resources -from . import data01 + from . import util @@ -18,7 +18,7 @@ def execute(self, package, path): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4))) def test_read_text_default_encoding(self): result = ( @@ -51,30 +51,42 @@ def test_read_text_with_errors(self): ) -class ReadDiskTests(ReadTests, unittest.TestCase): - data = data01 +class ReadDiskTests(ReadTests, util.DiskSetup, unittest.TestCase): + pass class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .read_bytes() + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) + +class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' -class ReadNamespaceTests(ReadTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - self.data = namespacedata01 +class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_read_submodule_resource(self): + submodule = import_module('namespacedata01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(12, 16))) + + def test_read_submodule_resource_by_name(self): + result = ( + resources.files('namespacedata01.subdirectory') + .joinpath('binary.file') + .read_bytes() + ) + self.assertEqual(result, bytes(range(12, 16))) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 8670f72a334..ed5693ab416 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -1,17 +1,21 @@ import os.path -import sys import pathlib import unittest from importlib import import_module from importlib.readers import MultiplexedPath, NamespaceReader +from . import util -class MultiplexedPathTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) + +class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def setUp(self): + super().setUp() + self.folder = pathlib.Path(self.data.__path__[0]) + self.data01 = pathlib.Path(self.load_fixture('data01').__file__).parent + self.data02 = pathlib.Path(self.load_fixture('data02').__file__).parent def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +23,7 @@ def test_init_no_paths(self): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -27,12 +31,13 @@ def test_iterdir(self): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} + ) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) contents = { - path.name for path in MultiplexedPath(self.folder, data01).iterdir() + path.name for path in MultiplexedPath(self.folder, self.data01).iterdir() } for remove in ('__pycache__', '__init__.pyc'): try: @@ -60,17 +65,16 @@ def test_open_file(self): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - path = MultiplexedPath(self.folder, data01) + prefix = str(self.folder.parent) + path = MultiplexedPath(self.folder, self.data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), ) - self.assertEqual( - str(path.joinpath('subdirectory'))[len(prefix) + 1 :], - os.path.join('data01', 'subdirectory'), - ) + sub = path.joinpath('subdirectory') + assert isinstance(sub, MultiplexedPath) + assert 'namespacedata01' in str(sub) + assert 'data01' in str(sub) self.assertEqual( str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), @@ -82,10 +86,8 @@ def test_join_path_compound(self): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - data02 = os.path.join(prefix, 'data02') - path = MultiplexedPath(data01, data02) + prefix = str(self.data02.parent) + path = MultiplexedPath(self.data01, self.data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], @@ -105,16 +107,8 @@ def test_name(self): ) -class NamespaceReaderTest(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' def test_init_error(self): with self.assertRaises(ValueError): @@ -124,7 +118,7 @@ def test_resource_path(self): namespacedata01 = import_module('namespacedata01') reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + root = self.data.__path__[0] self.assertEqual( reader.resource_path('binary.file'), os.path.join(root, 'binary.file') ) @@ -133,9 +127,8 @@ def test_resource_path(self): ) def test_files(self): - namespacedata01 = import_module('namespacedata01') - reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + reader = NamespaceReader(self.data.__spec__.submodule_search_locations) + root = self.data.__path__[0] self.assertIsInstance(reader.files(), MultiplexedPath) self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index 6f75cf57f03..83fc2ef4ed0 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,15 +1,8 @@ -import contextlib -import sys +import os import unittest -import uuid -import pathlib -from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper, os_helper -from test.support.os_helper import unlink class ResourceTests: @@ -29,9 +22,8 @@ def test_is_dir(self): self.assertTrue(target.is_dir()) -class ResourceDiskTests(ResourceTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): + pass class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): @@ -42,33 +34,39 @@ def names(traversable): return {item.name for item in traversable.iterdir()} -class ResourceLoaderTests(unittest.TestCase): +class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): def test_resource_contents(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) def test_is_file(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('B').is_file()) def test_is_dir(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('D').is_dir()) def test_resource_missing(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertFalse(resources.files(package).joinpath('Z').is_file()) -class ResourceCornerCaseTests(unittest.TestCase): +class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): def test_package_has_no_reader_fallback(self): """ Test odd ball packages which: @@ -77,7 +75,7 @@ def test_package_has_no_reader_fallback(self): # 3. Are not in a zip file """ module = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) # Give the module a dummy loader. module.__loader__ = object() @@ -88,43 +86,39 @@ def test_package_has_no_reader_fallback(self): self.assertFalse(resources.files(module).joinpath('A').is_file()) -class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore - +class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) assert not data.parent.exists() -class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore +class ResourceFromZipsTest02(util.ZipSetup, unittest.TestCase): + MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,93 +126,48 @@ def test_unrelated_contents(self): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetup, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -237,7 +186,9 @@ def test_submodule_contents(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -245,7 +196,41 @@ def test_submodule_contents_by_name(self): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) + + def test_submodule_sub_contents(self): + contents = names(resources.files(import_module('namespacedata01.subdirectory'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + def test_submodule_sub_contents_by_name(self): + contents = names(resources.files('namespacedata01.subdirectory')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + +class ResourceFromNamespaceDiskTests( + util.DiskSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' + + +class ResourceFromNamespaceZipTests( + util.ZipSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index dbe6ee81476..e2d995f5963 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -4,11 +4,12 @@ import sys import types import pathlib +import contextlib -from . import data01 -from . import zipdata01 from importlib.resources.abc import ResourceReader -from test.support import import_helper +from test.support import import_helper, os_helper +from . import zip as zip_ +from . import _path from importlib.machinery import ModuleSpec @@ -67,7 +68,7 @@ def create_package(file=None, path=None, is_package=True, contents=()): ) -class CommonTests(metaclass=abc.ABCMeta): +class CommonTestsBase(metaclass=abc.ABCMeta): """ Tests shared by test_open, test_path, and test_read. """ @@ -83,34 +84,34 @@ def test_package_name(self): """ Passing in the package name should succeed. """ - self.execute(data01.__name__, 'utf-8.file') + self.execute(self.data.__name__, 'utf-8.file') def test_package_object(self): """ Passing in the package itself should succeed. """ - self.execute(data01, 'utf-8.file') + self.execute(self.data, 'utf-8.file') def test_string_path(self): """ Passing in a string for the path should succeed. """ path = 'utf-8.file' - self.execute(data01, path) + self.execute(self.data, path) def test_pathlib_path(self): """ Passing in a pathlib.PurePath object for the path should succeed. """ path = pathlib.PurePath('utf-8.file') - self.execute(data01, path) + self.execute(self.data, path) def test_importing_module_as_side_effect(self): """ The anchor package can already be imported. """ - del sys.modules[data01.__name__] - self.execute(data01.__name__, 'utf-8.file') + del sys.modules[self.data.__name__] + self.execute(self.data.__name__, 'utf-8.file') def test_missing_path(self): """ @@ -140,40 +141,66 @@ def test_useless_loader(self): self.execute(package, 'utf-8.file') -class ZipSetupBase: - ZIP_MODULE = None - - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass - +fixtures = dict( + data01={ + '__init__.py': '', + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + '__init__.py': '', + 'binary.file': bytes(range(4, 8)), + }, + }, + data02={ + '__init__.py': '', + 'one': {'__init__.py': '', 'resource1.txt': 'one resource'}, + 'two': {'__init__.py': '', 'resource2.txt': 'two resource'}, + 'subdirectory': {'subsubdir': {'resource.txt': 'a resource'}}, + }, + namespacedata01={ + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + 'binary.file': bytes(range(12, 16)), + }, + }, +) + + +class ModuleSetup: def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + self.fixtures.enter_context(import_helper.isolated_modules()) + self.data = self.load_fixture(self.MODULE) + + def load_fixture(self, module): + self.tree_on_path({module: fixtures[module]}) + return importlib.import_module(module) + + +class ZipSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(spec, modules))) + ) + + +class DiskSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + _path.build(spec, pathlib.Path(temp_dir)) + self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) -class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore +class CommonTests(DiskSetup, CommonTestsBase): + pass diff --git a/Lib/test/test_importlib/resources/zip.py b/Lib/test/test_importlib/resources/zip.py new file mode 100644 index 00000000000..fc453f02060 --- /dev/null +++ b/Lib/test/test_importlib/resources/zip.py @@ -0,0 +1,24 @@ +""" +Generate zip test data files. +""" + +import zipfile + + +def make_zip_file(tree, dst): + """ + Zip the files in tree into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for name, contents in walk(tree): + zf.writestr(name, contents) + zipfile._path.CompleteDirs.inject(zf) + return dst + + +def walk(tree, prefix=''): + for name, contents in tree.items(): + if isinstance(contents, dict): + yield from walk(contents, prefix=f'{prefix}{name}/') + else: + yield f'{prefix}{name}', contents diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index 6a06313319d..e52829e6280 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -9,7 +9,6 @@ import os from test.support import os_helper import unittest -import warnings @util.case_insensitive_tests diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index 9c85bd234f3..f35adec1a8e 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -236,7 +236,6 @@ def test_unloadable(self): warnings.simplefilter('ignore', DeprecationWarning) loader.load_module('bad name') - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -293,7 +292,6 @@ def test_overridden_checked_hash_based_pyc(self): loader.exec_module(mod) self.assertEqual(mod.state, 'old') - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -324,7 +322,6 @@ def test_unchecked_hash_based_pyc(self): data[8:16], ) - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_overridden_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping, \ @@ -672,30 +669,28 @@ def test_read_only_bytecode(self): os.chmod(bytecode_path, stat.S_IWUSR) -# TODO: RUSTPYTHON -# class SourceLoaderBadBytecodeTestPEP451( -# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): -# pass +class SourceLoaderBadBytecodeTestPEP451( + SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): + pass -# (Frozen_SourceBadBytecodePEP451, -# Source_SourceBadBytecodePEP451 -# ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +(Frozen_SourceBadBytecodePEP451, + Source_SourceBadBytecodePEP451 + ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) -# TODO: RUSTPYTHON -# class SourceLoaderBadBytecodeTestPEP302( -# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): -# pass +class SourceLoaderBadBytecodeTestPEP302( + SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): + pass -# (Frozen_SourceBadBytecodePEP302, -# Source_SourceBadBytecodePEP302 -# ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +(Frozen_SourceBadBytecodePEP302, + Source_SourceBadBytecodePEP302 + ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) class SourcelessLoaderBadBytecodeTest: @@ -772,29 +767,28 @@ def test_non_code_marshal(self): self._test_non_code_marshal(del_source=True) -# TODO: RUSTPYTHON -# class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, -# BadBytecodeTestPEP451): -# pass +class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, + BadBytecodeTestPEP451): + pass -# (Frozen_SourcelessBadBytecodePEP451, -# Source_SourcelessBadBytecodePEP451 -# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +(Frozen_SourcelessBadBytecodePEP451, + Source_SourcelessBadBytecodePEP451 + ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) -# class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, -# BadBytecodeTestPEP302): -# pass +class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, + BadBytecodeTestPEP302): + pass -# (Frozen_SourcelessBadBytecodePEP302, -# Source_SourcelessBadBytecodePEP302 -# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +(Frozen_SourcelessBadBytecodePEP302, + Source_SourcelessBadBytecodePEP302 + ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 17d09d4ceed..4de736a6bf3 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -10,7 +10,6 @@ import tempfile from test.support.import_helper import make_legacy_pyc import unittest -import warnings class FinderTests(abc.FinderTests): @@ -74,7 +73,7 @@ def run_test(self, test, create=None, *, compile_=None, unlink=None): if error.errno != errno.ENOENT: raise loader = self.import_(mapping['.root'], test) - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') return loader def test_module(self): @@ -101,7 +100,7 @@ def test_module_in_package(self): with util.create_modules('pkg.__init__', 'pkg.sub') as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') # [sub package] def test_package_in_package(self): @@ -109,7 +108,7 @@ def test_package_in_package(self): with context as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') # [package over modules] def test_package_over_module(self): @@ -130,7 +129,7 @@ def test_empty_string_for_dir(self): file.write("# test file for importlib") try: loader = self._find(finder, 'mod', loader_only=True) - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') finally: os.unlink('mod.py') @@ -168,7 +167,6 @@ def test_no_read_directory(self): found = self._find(finder, 'doesnotexist') self.assertEqual(found, self.NOT_FOUND) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_ignore_file(self): # If a directory got changed to a file from underneath us, then don't # worry about looking for submodules. diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index f274330e0b3..6e1c23e6a98 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -15,12 +15,12 @@ def path_hook(self): def test_success(self): with util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(self.path_hook()(mapping['.root']), - 'find_spec')) + self.assertHasAttr(self.path_hook()(mapping['.root']), + 'find_spec') def test_empty_string(self): # The empty string represents the cwd. - self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) + self.assertHasAttr(self.path_hook()(''), 'find_spec') (Frozen_PathHookTest, diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index 4f206accf97..c09c9aa12b8 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -61,16 +61,12 @@ def test_non_obvious_encoding(self): def test_default_encoding(self): self.run_test(self.source_line.encode('utf-8')) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 17 - @unittest.expectedFailure # [encoding first line] def test_encoding_on_first_line(self): encoding = 'Latin-1' source = self.create_source(encoding) self.run_test(source) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 34 - @unittest.expectedFailure # [encoding second line] def test_encoding_on_second_line(self): source = b"#/usr/bin/python\n" + self.create_source('Latin-1') @@ -85,8 +81,6 @@ def test_bom_and_utf_8(self): source = codecs.BOM_UTF8 + self.create_source('utf-8') self.run_test(source) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 20 - @unittest.expectedFailure # [BOM conflict] def test_bom_conflict(self): source = codecs.BOM_UTF8 + self.create_source('latin-1') diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index a231ae1d5f8..dd943210ffc 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -43,14 +43,12 @@ def setUp(self): def test_subclasses(self): # Test that the expected subclasses inherit. for subclass in self.subclasses: - self.assertTrue(issubclass(subclass, self.__test), - "{0} is not a subclass of {1}".format(subclass, self.__test)) + self.assertIsSubclass(subclass, self.__test) def test_superclasses(self): # Test that the class inherits from the expected superclasses. for superclass in self.superclasses: - self.assertTrue(issubclass(self.__test, superclass), - "{0} is not a superclass of {1}".format(superclass, self.__test)) + self.assertIsSubclass(self.__test, superclass) class MetaPathFinder(InheritanceTests): @@ -416,14 +414,14 @@ def test_source_to_code_source(self): # Since compile() can handle strings, so should source_to_code(). source = 'attr = 42' module = self.source_to_module(source) - self.assertTrue(hasattr(module, 'attr')) + self.assertHasAttr(module, 'attr') self.assertEqual(module.attr, 42) def test_source_to_code_bytes(self): # Since compile() can handle bytes, so should source_to_code(). source = b'attr = 42' module = self.source_to_module(source) - self.assertTrue(hasattr(module, 'attr')) + self.assertHasAttr(module, 'attr') self.assertEqual(module.attr, 42) def test_source_to_code_path(self): @@ -701,8 +699,6 @@ def verify_code(self, code_object): class SourceOnlyLoaderTests(SourceLoaderTestHarness): """Test importlib.abc.SourceLoader for source-only loading.""" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_source(self): # Verify the source code is returned as a string. # If an OSError is raised by get_data then raise ImportError. @@ -759,10 +755,8 @@ def test_package_settings(self): warnings.simplefilter('ignore', DeprecationWarning) module = self.loader.load_module(self.name) self.verify_module(module) - self.assertFalse(hasattr(module, '__path__')) + self.assertNotHasAttr(module, '__path__') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_source_encoding(self): # Source is considered encoded in UTF-8 by default unless otherwise # specified by an encoding line. @@ -799,6 +793,9 @@ def verify_code(self, code_object, *, bytecode_written=False): data.extend(self.init._pack_uint32(0)) data.extend(self.init._pack_uint32(self.loader.source_mtime)) data.extend(self.init._pack_uint32(self.loader.source_size)) + # Make sure there's > 1 reference to code_object so that the + # marshaled representation below matches the cached representation + l = [code_object] data.extend(marshal.dumps(code_object)) self.assertEqual(self.loader.written[self.cached], bytes(data)) @@ -882,8 +879,6 @@ class SourceLoaderGetSourceTests: """Tests for importlib.abc.SourceLoader.get_source().""" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_default_encoding(self): # Should have no problems with UTF-8 text. name = 'mod' @@ -893,8 +888,6 @@ def test_default_encoding(self): returned_source = mock.get_source(name) self.assertEqual(returned_source, source) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decoded_source(self): # Decoding should work. name = 'mod' @@ -905,8 +898,6 @@ def test_decoded_source(self): returned_source = mock.get_source(name) self.assertEqual(returned_source, source) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_universal_newlines(self): # PEP 302 says universal newlines should be used. name = 'mod' @@ -923,5 +914,30 @@ def test_universal_newlines(self): SourceOnlyLoaderMock=SPLIT_SOL) +class SourceLoaderDeprecationWarningsTests(unittest.TestCase): + """Tests SourceLoader deprecation warnings.""" + + def test_deprecated_path_mtime(self): + from importlib.abc import SourceLoader + class DummySourceLoader(SourceLoader): + def get_data(self, path): + return b'' + + def get_filename(self, fullname): + return 'foo.py' + + def path_stats(self, path): + return {'mtime': 1} + + loader = DummySourceLoader() + + with self.assertWarnsRegex( + DeprecationWarning, + r"SourceLoader\.path_mtime is deprecated in favour of " + r"SourceLoader\.path_stats\(\)\." + ): + loader.path_mtime('foo.py') + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index ecf2c47c462..1bc531a2fe3 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -6,11 +6,12 @@ import os.path import sys +from test import support from test.support import import_helper from test.support import os_helper +import traceback import types import unittest -import warnings class ImportModuleTests: @@ -354,6 +355,20 @@ def test_module_missing_spec(self): with self.assertRaises(ModuleNotFoundError): self.init.reload(module) + def test_reload_traceback_with_non_str(self): + # gh-125519 + with support.captured_stdout() as stdout: + try: + self.init.reload("typing") + except TypeError as exc: + traceback.print_exception(exc, file=stdout) + else: + self.fail("Expected TypeError to be raised") + printed_traceback = stdout.getvalue() + self.assertIn("TypeError", printed_traceback) + self.assertNotIn("AttributeError", printed_traceback) + self.assertNotIn("module.__spec__.name", printed_traceback) + (Frozen_ReloadTests, Source_ReloadTests @@ -415,8 +430,7 @@ def test_everyone_has___loader__(self): for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): with self.subTest(name=name): - self.assertTrue(hasattr(module, '__loader__'), - '{!r} lacks a __loader__ attribute'.format(name)) + self.assertHasAttr(module, '__loader__') if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__loader__, None) elif self.machinery.FrozenImporter.find_spec(name): @@ -426,7 +440,7 @@ def test_everyone_has___spec__(self): for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): with self.subTest(name=name): - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__spec__, None) elif self.machinery.FrozenImporter.find_spec(name): @@ -438,5 +452,57 @@ def test_everyone_has___spec__(self): ) = test_util.test_both(StartupTests, machinery=machinery) +class TestModuleAll(unittest.TestCase): + def test_machinery(self): + extra = ( + # from importlib._bootstrap and importlib._bootstrap_external + 'AppleFrameworkLoader', + 'BYTECODE_SUFFIXES', + 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', + 'FileFinder', + 'FrozenImporter', + 'ModuleSpec', + 'NamespaceLoader', + 'OPTIMIZED_BYTECODE_SUFFIXES', + 'PathFinder', + 'SOURCE_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'WindowsRegistryFinder', + ) + support.check__all__(self, machinery['Source'], extra=extra) + + def test_util(self): + extra = ( + # from importlib.abc, importlib._bootstrap + # and importlib._bootstrap_external + 'Loader', + 'MAGIC_NUMBER', + 'cache_from_source', + 'decode_source', + 'module_from_spec', + 'source_from_cache', + 'spec_from_file_location', + 'spec_from_loader', + ) + support.check__all__(self, util['Source'], extra=extra) + + +class TestDeprecations(unittest.TestCase): + def test_machinery_deprecated_attributes(self): + from importlib import machinery + attributes = ( + 'DEBUG_BYTECODE_SUFFIXES', + 'OPTIMIZED_BYTECODE_SUFFIXES', + ) + for attr in attributes: + with self.subTest(attr=attr): + with self.assertWarns(DeprecationWarning): + getattr(machinery, attr) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index 7539a8c4352..e48fad8898f 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -2,9 +2,12 @@ from importlib import abc from importlib import util import sys +import time +import threading import types import unittest +from test.support import threading_helper from test.test_importlib import util as test_util @@ -40,6 +43,7 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): module_name = 'lazy_loader_test' mutated_name = 'changed' loaded = None + load_count = 0 source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name) def find_spec(self, name, path, target=None): @@ -48,8 +52,10 @@ def find_spec(self, name, path, target=None): return util.spec_from_loader(name, util.LazyLoader(self)) def exec_module(self, module): + time.sleep(0.01) # Simulate a slow load. exec(self.source_code, module.__dict__) self.loaded = module + self.load_count += 1 class LazyLoaderTests(unittest.TestCase): @@ -59,8 +65,9 @@ def test_init(self): # Classes that don't define exec_module() trigger TypeError. util.LazyLoader(object) - def new_module(self, source_code=None): - loader = TestingImporter() + def new_module(self, source_code=None, loader=None): + if loader is None: + loader = TestingImporter() if source_code is not None: loader.source_code = source_code spec = util.spec_from_loader(TestingImporter.module_name, @@ -75,8 +82,6 @@ def new_module(self, source_code=None): self.assertIsNone(loader.loaded) return module - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_e2e(self): # End-to-end test to verify the load is in fact lazy. importer = TestingImporter() @@ -90,24 +95,18 @@ def test_e2e(self): self.assertIsNotNone(importer.loaded) self.assertEqual(module, importer.loaded) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attr_unchanged(self): # An attribute only mutated as a side-effect of import should not be # changed needlessly. module = self.new_module() self.assertEqual(TestingImporter.mutated_name, module.__name__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_new_attr(self): # A new attribute should persist. module = self.new_module() module.new_attr = 42 self.assertEqual(42, module.new_attr) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mutated_preexisting_attr(self): # Changing an attribute that already existed on the module -- # e.g. __name__ -- should persist. @@ -115,8 +114,6 @@ def test_mutated_preexisting_attr(self): module.__name__ = 'bogus' self.assertEqual('bogus', module.__name__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mutated_attr(self): # Changing an attribute that comes into existence after an import # should persist. @@ -124,23 +121,17 @@ def test_mutated_attr(self): module.attr = 6 self.assertEqual(6, module.attr) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_delete_eventual_attr(self): # Deleting an attribute should stay deleted. module = self.new_module() del module.attr - self.assertFalse(hasattr(module, 'attr')) + self.assertNotHasAttr(module, 'attr') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_delete_preexisting_attr(self): module = self.new_module() del module.__name__ - self.assertFalse(hasattr(module, '__name__')) + self.assertNotHasAttr(module, '__name__') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_module_substitution_error(self): with test_util.uncache(TestingImporter.module_name): fresh_module = types.ModuleType(TestingImporter.module_name) @@ -149,8 +140,6 @@ def test_module_substitution_error(self): with self.assertRaisesRegex(ValueError, "substituted"): module.__name__ - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_module_already_in_sys(self): with test_util.uncache(TestingImporter.module_name): module = self.new_module() @@ -158,6 +147,83 @@ def test_module_already_in_sys(self): # Force the load; just care that no exception is raised. module.__name__ + @threading_helper.requires_working_threading() + def test_module_load_race(self): + with test_util.uncache(TestingImporter.module_name): + loader = TestingImporter() + module = self.new_module(loader=loader) + self.assertEqual(loader.load_count, 0) + + class RaisingThread(threading.Thread): + exc = None + def run(self): + try: + super().run() + except Exception as exc: + self.exc = exc + + def access_module(): + return module.attr + + threads = [] + for _ in range(2): + threads.append(thread := RaisingThread(target=access_module)) + thread.start() + + # Races could cause errors + for thread in threads: + thread.join() + self.assertIsNone(thread.exc) + + # Or multiple load attempts + self.assertEqual(loader.load_count, 1) + + def test_lazy_self_referential_modules(self): + # Directory modules with submodules that reference the parent can attempt to access + # the parent module during a load. Verify that this common pattern works with lazy loading. + # json is a good example in the stdlib. + json_modules = [name for name in sys.modules if name.startswith('json')] + with test_util.uncache(*json_modules): + # Standard lazy loading, unwrapped + spec = util.find_spec('json') + loader = util.LazyLoader(spec.loader) + spec.loader = loader + module = util.module_from_spec(spec) + sys.modules['json'] = module + loader.exec_module(module) + + # Trigger load with attribute lookup, ensure expected behavior + test_load = module.loads('{}') + self.assertEqual(test_load, {}) + + def test_lazy_module_type_override(self): + # Verify that lazy loading works with a module that modifies + # its __class__ to be a custom type. + + # Example module from PEP 726 + module = self.new_module(source_code="""\ +import sys +from types import ModuleType + +CONSTANT = 3.14 + +class ImmutableModule(ModuleType): + def __setattr__(self, name, value): + raise AttributeError('Read-only attribute!') + + def __delattr__(self, name): + raise AttributeError('Read-only attribute!') + +sys.modules[__name__].__class__ = ImmutableModule +""") + sys.modules[TestingImporter.module_name] = module + self.assertIsInstance(module, util._LazyModule) + self.assertEqual(module.CONSTANT, 3.14) + with self.assertRaises(AttributeError): + module.CONSTANT = 2.71 + with self.assertRaises(AttributeError): + del module.CONSTANT + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index 17cce741cce..b1f5f9d6c8b 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -29,9 +29,12 @@ class ModuleLockAsRLockTests: test_timeout = None # _release_save() unsupported test_release_save_unacquired = None + # _recursion_count() unsupported + test_recursion_count = None # lock status in repr unsupported test_repr = None test_locked_repr = None + test_repr_count = None def tearDown(self): for splitinit in init.values(): @@ -47,7 +50,6 @@ def tearDown(self): LockType=LOCK_TYPES) -@unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON") class DeadlockAvoidanceTests: def setUp(self): @@ -92,11 +94,12 @@ def f(): b.release() if ra: a.release() - lock_tests.Bunch(f, NTHREADS).wait_for_finished() + with lock_tests.Bunch(f, NTHREADS): + pass self.assertEqual(len(results), NTHREADS) return results - @unittest.skip("TODO: RUSTPYTHON, sometimes hangs") + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_deadlock(self): results = self.run_deadlock_avoidance_test(True) # At least one of the threads detected a potential deadlock on its @@ -106,7 +109,7 @@ def test_deadlock(self): self.assertGreaterEqual(nb_deadlocks, 1) self.assertEqual(results.count((True, True)), len(results) - nb_deadlocks) - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip("TODO: RUSTPYTHON; flaky test") def test_no_deadlock(self): results = self.run_deadlock_avoidance_test(False) self.assertEqual(results.count((True, False)), 0) @@ -145,10 +148,14 @@ def test_all_locks(self): self.assertEqual(0, len(self.bootstrap._module_locks), self.bootstrap._module_locks) -# TODO: RUSTPYTHON -# (Frozen_LifetimeTests, -# Source_LifetimeTests -# ) = test_util.test_both(LifetimeTests, init=init) + +(Frozen_LifetimeTests, + Source_LifetimeTests + ) = test_util.test_both(LifetimeTests, init=init) + +# TODO: RUSTPYTHON; dead weakref module locks not cleaned up in frozen bootstrap +Frozen_LifetimeTests.test_all_locks = unittest.skip("TODO: RUSTPYTHON")( + Frozen_LifetimeTests.test_all_locks) def setUpModule(): diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py index 55c9f8007e5..33c6e85ee94 100644 --- a/Lib/test/test_importlib/test_metadata_api.py +++ b/Lib/test/test_importlib/test_metadata_api.py @@ -139,8 +139,6 @@ def test_entry_points_missing_name(self): def test_entry_points_missing_group(self): assert entry_points(group='missing') == () - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_entry_points_allows_no_attributes(self): ep = entry_points().select(group='entries', name='main') with self.assertRaises(AttributeError): diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 65428c3d3ea..6ca0978f9bc 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -6,7 +6,6 @@ import sys import tempfile import unittest -import warnings from test.test_importlib import util @@ -81,7 +80,7 @@ def test_cant_import_other(self): def test_simple_repr(self): import foo.one - assert repr(foo).startswith("<module 'foo' (namespace) from [") + self.assertStartsWith(repr(foo), "<module 'foo' (namespace) from [") class DynamicPathNamespacePackage(NamespacePackageTest): @@ -287,25 +286,24 @@ def test_project3_succeeds(self): class ZipWithMissingDirectory(NamespacePackageTest): paths = ['missing_directory.zip'] + # missing_directory.zip contains: + # Length Date Time Name + # --------- ---------- ----- ---- + # 29 2012-05-03 18:13 foo/one.py + # 0 2012-05-03 20:57 bar/ + # 38 2012-05-03 20:57 bar/two.py + # --------- ------- + # 67 3 files - @unittest.expectedFailure def test_missing_directory(self): - # This will fail because missing_directory.zip contains: - # Length Date Time Name - # --------- ---------- ----- ---- - # 29 2012-05-03 18:13 foo/one.py - # 0 2012-05-03 20:57 bar/ - # 38 2012-05-03 20:57 bar/two.py - # --------- ------- - # 67 3 files - - # Because there is no 'foo/', the zipimporter currently doesn't - # know that foo is a namespace package - import foo.one + self.assertEqual(foo.one.attr, 'portion1 foo one') + + def test_missing_directory2(self): + import foo + self.assertNotHasAttr(foo, 'one') def test_present_directory(self): - # This succeeds because there is a "bar/" in the zip file import bar.two self.assertEqual(bar.two.attr, 'missing_directory foo two') diff --git a/Lib/test/test_importlib/test_pkg_import.py b/Lib/test/test_importlib/test_pkg_import.py index 66f5f8bc253..5ffae6222ba 100644 --- a/Lib/test/test_importlib/test_pkg_import.py +++ b/Lib/test/test_importlib/test_pkg_import.py @@ -55,7 +55,7 @@ def test_package_import__semantics(self): except SyntaxError: pass else: raise RuntimeError('Failed to induce SyntaxError') # self.fail()? self.assertNotIn(self.module_name, sys.modules) - self.assertFalse(hasattr(sys.modules[self.package_name], 'foo')) + self.assertNotHasAttr(sys.modules[self.package_name], 'foo') # ...make up a variable name that isn't bound in __builtins__ var = 'a' diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py index 921b6bbece0..aebeabaf83f 100644 --- a/Lib/test/test_importlib/test_spec.py +++ b/Lib/test/test_importlib/test_spec.py @@ -237,7 +237,7 @@ def test_exec(self): self.spec.loader = NewLoader() module = self.util.module_from_spec(self.spec) sys.modules[self.name] = module - self.assertFalse(hasattr(module, 'eggs')) + self.assertNotHasAttr(module, 'eggs') self.bootstrap._exec(self.spec, module) self.assertEqual(module.eggs, 1) @@ -348,9 +348,9 @@ def test_reload_init_module_attrs(self): self.assertIs(loaded.__loader__, self.spec.loader) self.assertEqual(loaded.__package__, self.spec.parent) self.assertIs(loaded.__spec__, self.spec) - self.assertFalse(hasattr(loaded, '__path__')) - self.assertFalse(hasattr(loaded, '__file__')) - self.assertFalse(hasattr(loaded, '__cached__')) + self.assertNotHasAttr(loaded, '__path__') + self.assertNotHasAttr(loaded, '__file__') + self.assertNotHasAttr(loaded, '__cached__') (Frozen_ModuleSpecMethodsTests, @@ -502,7 +502,8 @@ def test_spec_from_loader_is_package_true_with_fileloader(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -601,7 +602,8 @@ def test_spec_from_file_location_smsl_empty(self): self.assertEqual(spec.loader, self.fileloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -626,7 +628,8 @@ def test_spec_from_file_location_smsl_default(self): self.assertEqual(spec.loader, self.pkgloader) self.assertEqual(spec.origin, self.path) self.assertIs(spec.loader_state, None) - self.assertEqual(spec.submodule_search_locations, [os.getcwd()]) + location = cwd if (cwd := os.getcwd()) != '/' else '' + self.assertEqual(spec.submodule_search_locations, [location]) self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) @@ -686,10 +689,9 @@ def test_spec_from_file_location_relative_path(self): self.assertEqual(spec.cached, self.cached) self.assertTrue(spec.has_location) -# TODO: RUSTPYTHON -# (Frozen_FactoryTests, -# Source_FactoryTests -# ) = test_util.test_both(FactoryTests, util=util, machinery=machinery) +(Frozen_FactoryTests, + Source_FactoryTests + ) = test_util.test_both(FactoryTests, util=util, machinery=machinery) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 148b2e4370b..c8d156f7f7c 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -6,7 +6,6 @@ # randrange, and then Python hangs. import _imp as imp -import _multiprocessing # TODO: RUSTPYTHON import os import importlib import sys @@ -14,7 +13,7 @@ import shutil import threading import unittest -from unittest import mock +from test import support from test.support import verbose from test.support.import_helper import forget, mock_register_at_fork from test.support.os_helper import (TESTFN, unlink, rmtree) @@ -136,14 +135,13 @@ def check_parallel_module_init(self, mock_os): if verbose: print("OK.") - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_parallel_module_init(self): + @unittest.skip("TODO: RUSTPYTHON; flaky") + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_module_init(self, size): self.check_parallel_module_init() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_parallel_meta_path(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_meta_path(self, size): finder = Finder() sys.meta_path.insert(0, finder) try: @@ -153,9 +151,9 @@ def test_parallel_meta_path(self): finally: sys.meta_path.remove(finder) - # TODO: RUSTPYTHON; maybe hang? - @unittest.expectedFailure - def test_parallel_path_hooks(self): + @unittest.expectedFailure # TODO: RUSTPYTHON; maybe hang? + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_path_hooks(self, size): # Here the Finder instance is only used to check concurrent calls # to path_hook(). finder = Finder() @@ -249,16 +247,17 @@ def target(): __import__(TESTFN) del sys.modules[TESTFN] - @unittest.skip("TODO: RUSTPYTHON; hang") - def test_concurrent_futures_circular_import(self): + @unittest.skip("TODO: RUSTPYTHON; hang; Suspected cause of crashes in Windows CI - PermissionError: [WinError 32] Permission denied: \"C:\\Users\\RUNNER~1\\AppData\\Local\\Temp\\test_python_0cdrhhs_\\test_python_6340æ\"") + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_concurrent_futures_circular_import(self, size): # Regression test for bpo-43515 fn = os.path.join(os.path.dirname(__file__), 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) - @unittest.skipUnless(hasattr(_multiprocessing, "SemLock"), "TODO: RUSTPYTHON, pool_in_threads.py needs _multiprocessing.SemLock") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_multiprocessing_pool_circular_import(self): + @unittest.skip("TODO: RUSTPYTHON; hang") + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_multiprocessing_pool_circular_import(self, size): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), 'partial', 'pool_in_threads.py') @@ -271,7 +270,7 @@ def setUpModule(): try: old_switchinterval = sys.getswitchinterval() unittest.addModuleCleanup(sys.setswitchinterval, old_switchinterval) - sys.setswitchinterval(1e-5) + support.setswitchinterval(1e-5) except AttributeError: pass diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index dc27e4aa991..8c14b96271a 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -6,12 +6,13 @@ importlib_util = util.import_importlib('importlib.util') import importlib.util +from importlib import _bootstrap_external import os import pathlib -import re import string import sys from test import support +from test.support import os_helper import textwrap import types import unittest @@ -27,7 +28,7 @@ except ImportError: _testmultiphase = None try: - import _xxsubinterpreters as _interpreters + import _interpreters except ModuleNotFoundError: _interpreters = None @@ -36,22 +37,16 @@ class DecodeSourceBytesTests: source = "string ='ü'" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ut8_default(self): source_bytes = self.source.encode('utf-8') self.assertEqual(self.util.decode_source(source_bytes), self.source) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_specified_encoding(self): source = '# coding=latin-1\n' + self.source source_bytes = source.encode('latin-1') assert source_bytes != source.encode('utf-8') self.assertEqual(self.util.decode_source(source_bytes), source) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_universal_newlines(self): source = '\r\n'.join([self.source, self.source]) source_bytes = source.encode('utf-8') @@ -325,7 +320,7 @@ def test_length(self): def test_incorporates_rn(self): # The magic number uses \r\n to come out wrong when splitting on lines. - self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n')) + self.assertEndsWith(self.util.MAGIC_NUMBER, b'\r\n') (Frozen_MagicNumberTests, @@ -333,15 +328,6 @@ def test_incorporates_rn(self): ) = util.test_both(MagicNumberTests, util=importlib_util) -# TODO: RUSTPYTHON -@unittest.expectedFailure -def test_incorporates_rn_MONKEYPATCH(self): - self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n')) - -# TODO: RUSTPYTHON -Frozen_MagicNumberTests.test_incorporates_rn = test_incorporates_rn_MONKEYPATCH - - class PEP3147Tests: """Tests of PEP 3147-related functions: cache_from_source and source_from_cache.""" @@ -373,8 +359,6 @@ def test_cache_from_source_no_dot(self): self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cache_from_source_debug_override(self): # Given the path to a .py file, return the path to its PEP 3147/PEP 488 # defined .pyc file (i.e. under __pycache__). @@ -594,7 +578,19 @@ def test_cache_from_source_respects_pycache_prefix_relative(self): with util.temporary_pycache_prefix(pycache_prefix): self.assertEqual( self.util.cache_from_source(path, optimization=''), - expect) + os.path.normpath(expect)) + + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_cache_from_source_in_root_with_pycache_prefix(self): + # Regression test for gh-82916 + pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode') + path = 'qux.py' + expect = os.path.join(os.path.sep, 'tmp', 'bytecode', + f'qux.{self.tag}.pyc') + with util.temporary_pycache_prefix(pycache_prefix): + with os_helper.change_cwd('/'): + self.assertEqual(self.util.cache_from_source(path), expect) @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag to not be None') @@ -651,7 +647,7 @@ def test_magic_number(self): # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3531 + EXPECTED_MAGIC_NUMBER = 3627 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( @@ -672,27 +668,36 @@ def test_magic_number(self): @unittest.skipIf(_interpreters is None, 'subinterpreters required') class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): - ERROR = re.compile("^<class 'ImportError'>: module (.*) does not support loading in subinterpreters") - def run_with_own_gil(self, script): - interpid = _interpreters.create(isolated=True) - try: - _interpreters.run_string(interpid, script) - except _interpreters.RunFailedError as exc: - if m := self.ERROR.match(str(exc)): - modname, = m.groups() - raise ImportError(modname) + interpid = _interpreters.create('isolated') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) + if excsnap is not None: + if excsnap.type.__name__ == 'ImportError': + raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create(isolated=False) - try: - _interpreters.run_string(interpid, script) - except _interpreters.RunFailedError as exc: - if m := self.ERROR.match(str(exc)): - modname, = m.groups() - raise ImportError(modname) + interpid = _interpreters.create('legacy') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) + if excsnap is not None: + if excsnap.type.__name__ == 'ImportError': + raise ImportError(excsnap.msg) @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + # gh-117649: single-phase init modules are not currently supported in + # subinterpreters in the free-threaded build + @support.expected_failure_if_gil_disabled() def test_single_phase_init_module(self): script = textwrap.dedent(''' from importlib.util import _incompatible_extension_module_restrictions @@ -717,14 +722,22 @@ def test_single_phase_init_module(self): self.run_with_own_gil(script) @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + @support.requires_gil_enabled("gh-117649: not supported in free-threaded build") def test_incomplete_multi_phase_init_module(self): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + prescript = textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} name = '_test_shared_gil_only' filename = {_testmultiphase.__file__!r} - loader = ExtensionFileLoader(name, filename) + loader = {loader}(name, filename) spec = spec_from_loader(name, loader) ''') @@ -775,5 +788,74 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) +class PatchAtomicWrites: + def __init__(self, truncate_at_length, never_complete=False): + self.truncate_at_length = truncate_at_length + self.never_complete = never_complete + self.seen_write = False + self._children = [] + + def __enter__(self): + import _pyio + + oldwrite = os.write + + # Emulate an os.write that only writes partial data. + def write(fd, data): + if self.seen_write and self.never_complete: + return None + self.seen_write = True + return oldwrite(fd, data[:self.truncate_at_length]) + + # Need to patch _io to be _pyio, so that io.FileIO is affected by the + # os.write patch. + self.children = [ + support.swap_attr(_bootstrap_external, '_io', _pyio), + support.swap_attr(os, 'write', write) + ] + for child in self.children: + child.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for child in self.children: + child.__exit__(exc_type, exc_val, exc_tb) + + +class MiscTests(unittest.TestCase): + + def test_atomic_write_retries_incomplete_writes(self): + truncate_at_length = 100 + length = truncate_at_length * 2 + + with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm: + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * length + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length) + os.unlink(support.os_helper.TESTFN) + + def test_atomic_write_errors_if_unable_to_complete(self): + truncate_at_length = 100 + + with ( + PatchAtomicWrites( + truncate_at_length=truncate_at_length, never_complete=True, + ) as cm, + self.assertRaises(OSError) + ): + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * (truncate_at_length * 2) + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + with self.assertRaises(OSError): + os.stat(support.os_helper.TESTFN) # Check that the file did not get written. + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index f8a9ead9ac8..0ae911bc43d 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -5,7 +5,7 @@ import re import sys import unittest -import warnings +from test import support from test.support import import_helper from contextlib import contextmanager from test.test_importlib.util import temp_module @@ -91,31 +91,61 @@ class WindowsRegistryFinderTests: test_module = "spamham{}".format(os.getpid()) def test_find_spec_missing(self): - spec = self.machinery.WindowsRegistryFinder.find_spec('spam') + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec('spam') self.assertIsNone(spec) def test_module_found(self): with setup_module(self.machinery, self.test_module): - spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNotNone(spec) def test_module_not_found(self): with setup_module(self.machinery, self.test_module, path="."): - spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNone(spec) + def test_raises_deprecation_warning(self): + # WindowsRegistryFinder is not meant to be instantiated, so the + # deprecation warning is raised in the 'find_spec' method instead. + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + self.machinery.WindowsRegistryFinder.find_spec('spam') + (Frozen_WindowsRegistryFinderTests, Source_WindowsRegistryFinderTests ) = test_util.test_both(WindowsRegistryFinderTests, machinery=machinery) @unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows') class WindowsExtensionSuffixTests: - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; no C extension (.pyd) support def test_tagged_suffix(self): suffixes = self.machinery.EXTENSION_SUFFIXES - expected_tag = ".cp{0.major}{0.minor}-{1}.pyd".format(sys.version_info, - re.sub('[^a-zA-Z0-9]', '_', get_platform())) + abi_flags = "t" if support.Py_GIL_DISABLED else "" + ver = sys.version_info + platform = re.sub('[^a-zA-Z0-9]', '_', get_platform()) + expected_tag = f".cp{ver.major}{ver.minor}{abi_flags}-{platform}.pyd" try: untagged_i = suffixes.index(".pyd") except ValueError: diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c25be096e52..85e7ffcb608 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -6,13 +6,20 @@ import marshal import os import os.path +from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper import unittest import sys import tempfile import types +try: + _testsinglephase = import_helper.import_module("_testsinglephase") +except unittest.SkipTest: + _testsinglephase = None # TODO: RUSTPYTHON + BUILTINS = types.SimpleNamespace() BUILTINS.good_name = None @@ -22,25 +29,39 @@ if 'importlib' not in sys.builtin_module_names: BUILTINS.bad_name = 'importlib' -EXTENSIONS = types.SimpleNamespace() -EXTENSIONS.path = None -EXTENSIONS.ext = None -EXTENSIONS.filename = None -EXTENSIONS.file_path = None -EXTENSIONS.name = '_testsinglephase' - -def _extension_details(): - global EXTENSIONS - for path in sys.path: - for ext in machinery.EXTENSION_SUFFIXES: - filename = EXTENSIONS.name + ext - file_path = os.path.join(path, filename) - if os.path.exists(file_path): - EXTENSIONS.path = path - EXTENSIONS.ext = ext - EXTENSIONS.filename = filename - EXTENSIONS.file_path = file_path - return +if support.is_wasi: + # dlopen() is a shim for WASI as of WASI SDK which fails by default. + # We don't provide an implementation, so tests will fail. + # But we also don't want to turn off dynamic loading for those that provide + # a working implementation. + def _extension_details(): + global EXTENSIONS + EXTENSIONS = None +else: + EXTENSIONS = types.SimpleNamespace() + EXTENSIONS.path = None + EXTENSIONS.ext = None + EXTENSIONS.filename = None + EXTENSIONS.file_path = None + EXTENSIONS.name = '_testsinglephase' + + def _extension_details(): + global EXTENSIONS + for path in sys.path: + for ext in machinery.EXTENSION_SUFFIXES: + # Apple mobile platforms mechanically load .so files, + # but the findable files are labelled .fwork + if is_apple_mobile: + ext = ext.replace(".so", ".fwork") + + filename = EXTENSIONS.name + ext + file_path = os.path.join(path, filename) + if os.path.exists(file_path): + EXTENSIONS.path = path + EXTENSIONS.ext = ext + EXTENSIONS.filename = filename + EXTENSIONS.file_path = file_path + return _extension_details() diff --git a/Lib/test/test_inspect/__init__.py b/Lib/test/test_inspect/__init__.py new file mode 100644 index 00000000000..f2a39a3fe29 --- /dev/null +++ b/Lib/test/test_inspect/__init__.py @@ -0,0 +1,6 @@ +import os +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_inspect/inspect_deferred_annotations.py b/Lib/test/test_inspect/inspect_deferred_annotations.py new file mode 100644 index 00000000000..bb59ef1035b --- /dev/null +++ b/Lib/test/test_inspect/inspect_deferred_annotations.py @@ -0,0 +1,2 @@ +def f(x: undefined): + pass diff --git a/Lib/test/test_inspect/inspect_fodder.py b/Lib/test/test_inspect/inspect_fodder.py new file mode 100644 index 00000000000..febd54c86fe --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder.py @@ -0,0 +1,120 @@ +# line 1 +'A module docstring.' + +import inspect +# line 5 + +# line 7 +def spam(a, /, b, c, d=3, e=4, f=5, *g, **h): + eggs(b + d, c + f) + +# line 11 +def eggs(x, y): + "A docstring." + global fr, st + fr = inspect.currentframe() + st = inspect.stack() + p = x + q = y / 0 + +# line 20 +class StupidGit: + """A longer, + + indented + + docstring.""" +# line 27 + + def abuse(self, a, b, c): + """Another + +\tdocstring + + containing + +\ttabs +\t + """ + self.argue(a, b, c) +# line 40 + def argue(self, a, b, c): + try: + spam(a, b, c) + except BaseException as e: + self.ex = e + self.tr = inspect.trace() + + @property + def contradiction(self): + 'The automatic gainsaying.' + pass + +# line 53 +class MalodorousPervert(StupidGit): + def abuse(self, a, b, c): + pass + + @property + def contradiction(self): + pass + +Tit = MalodorousPervert + +class ParrotDroppings: + pass + +class FesteringGob(MalodorousPervert, ParrotDroppings): + def abuse(self, a, b, c): + pass + + def _getter(self): + pass + contradiction = property(_getter) + +async def lobbest(grenade): + pass + +currentframe = inspect.currentframe() +try: + raise Exception() +except BaseException as e: + tb = e.__traceback__ + +class Callable: + def __call__(self, *args): + return args + + def as_method_of(self, obj): + from types import MethodType + return MethodType(self, obj) + +custom_method = Callable().as_method_of(42) +del Callable + +# line 95 +class WhichComments: + # line 97 + # before f + def f(self): + # line 100 + # start f + return 1 + # line 103 + # end f + # line 105 + # after f + + # before asyncf - line 108 + async def asyncf(self): + # start asyncf + return 2 + # end asyncf + # after asyncf - line 113 + # end of WhichComments - line 114 + # after WhichComments - line 115 + +# Test that getsource works on a line that includes +# a closing parenthesis with the opening paren being in another line +( +); after_closing = lambda: 1 diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py new file mode 100644 index 00000000000..157e12167b5 --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -0,0 +1,403 @@ +# line 1 +def wrap(foo=None): + def wrapper(func): + return func + return wrapper + +# line 7 +def replace(func): + def insteadfunc(): + print('hello') + return insteadfunc + +# line 13 +@wrap() +@wrap(wrap) +def wrapped(): + pass + +# line 19 +@replace +def gone(): + pass + +# line 24 +oll = lambda m: m + +# line 27 +tll = lambda g: g and \ +g and \ +g + +# line 32 +tlli = lambda d: d and \ + d + +# line 36 +def onelinefunc(): pass + +# line 39 +def manyargs(arg1, arg2, +arg3, arg4): pass + +# line 43 +def twolinefunc(m): return m and \ +m + +# line 47 +a = [None, + lambda x: x, + None] + +# line 52 +def setfunc(func): + globals()["anonymous"] = func +setfunc(lambda x, y: x*y) + +# line 57 +def with_comment(): # hello + world + +# line 61 +multiline_sig = [ + lambda x, \ + y: x+y, + None, + ] + +# line 68 +def func69(): + class cls70: + def func71(): + pass + return cls70 +extra74 = 74 + +# line 76 +def func77(): pass +(extra78, stuff78) = 'xy' +extra79 = 'stop' + +# line 81 +class cls82: + def func83(): pass +(extra84, stuff84) = 'xy' +extra85 = 'stop' + +# line 87 +def func88(): + # comment + return 90 + +# line 92 +def f(): + class X: + def g(): + "doc" + return 42 + return X +method_in_dynamic_class = f().g + +#line 101 +def keyworded(*arg1, arg2=1): + pass + +#line 105 +def annotated(arg1: list): + pass + +#line 109 +def keyword_only_arg(*, arg): + pass + +@wrap(lambda: None) +def func114(): + return 115 + +class ClassWithMethod: + def method(self): + pass + +from functools import wraps + +def decorator(func): + @wraps(func) + def fake(): + return 42 + return fake + +#line 129 +@decorator +def real(): + return 20 + +#line 134 +class cls135: + def func136(): + def func137(): + never_reached1 + never_reached2 + +# line 141 +class cls142: + a = """ +class cls149: + ... +""" + +# line 148 +class cls149: + + def func151(self): + pass + +''' +class cls160: + pass +''' + +# line 159 +class cls160: + + def func162(self): + pass + +# line 165 +class cls166: + a = ''' + class cls175: + ... + ''' + +# line 172 +class cls173: + + class cls175: + pass + +# line 178 +class cls179: + pass + +# line 182 +class cls183: + + class cls185: + + def func186(self): + pass + +def class_decorator(cls): + return cls + +# line 193 +@class_decorator +@class_decorator +class cls196: + + @class_decorator + @class_decorator + class cls200: + pass + +class cls203: + class cls204: + class cls205: + pass + class cls207: + class cls205: + pass + +# line 211 +def func212(): + class cls213: + pass + return cls213 + +# line 217 +class cls213: + def func219(self): + class cls220: + pass + return cls220 + +# line 224 +async def func225(): + class cls226: + pass + return cls226 + +# line 230 +class cls226: + async def func232(self): + class cls233: + pass + return cls233 + +if True: + class cls238: + class cls239: + '''if clause cls239''' +else: + class cls238: + class cls239: + '''else clause 239''' + pass + +#line 247 +def positional_only_arg(a, /): + pass + +#line 251 +def all_markers(a, b, /, c, d, *, e, f): + pass + +# line 255 +def all_markers_with_args_and_kwargs(a, b, /, c, d, *args, e, f, **kwargs): + pass + +#line 259 +def all_markers_with_defaults(a, b=1, /, c=2, d=3, *, e=4, f=5): + pass + +# line 263 +def deco_factory(**kwargs): + def deco(f): + @wraps(f) + def wrapper(*a, **kwd): + kwd.update(kwargs) + return f(*a, **kwd) + return wrapper + return deco + +@deco_factory(foo=(1 + 2), bar=lambda: 1) +def complex_decorated(foo=0, bar=lambda: 0): + return foo + bar() + +# line 276 +parenthesized_lambda = ( + lambda: ()) +parenthesized_lambda2 = [ + lambda: ()][0] +parenthesized_lambda3 = {0: + lambda: ()}[0] + +# line 285 +post_line_parenthesized_lambda1 = (lambda: () +) + +# line 289 +nested_lambda = ( + lambda right: [].map( + lambda length: ())) + +# line 294 +if True: + class cls296: + def f(): + pass +else: + class cls296: + def g(): + pass + +# line 304 +if False: + class cls310: + def f(): + pass +else: + class cls310: + def g(): + pass + +# line 314 +class ClassWithCodeObject: + import sys + code = sys._getframe(0).f_code + +import enum + +# line 321 +class enum322(enum.Enum): + A = 'a' + +# line 325 +class enum326(enum.IntEnum): + A = 1 + +# line 329 +class flag330(enum.Flag): + A = 1 + +# line 333 +class flag334(enum.IntFlag): + A = 1 + +# line 337 +simple_enum338 = enum.Enum('simple_enum338', 'A') +simple_enum339 = enum.IntEnum('simple_enum339', 'A') +simple_flag340 = enum.Flag('simple_flag340', 'A') +simple_flag341 = enum.IntFlag('simple_flag341', 'A') + +import typing + +# line 345 +class nt346(typing.NamedTuple): + x: int + y: int + +# line 350 +nt351 = typing.NamedTuple('nt351', (('x', int), ('y', int))) + +# line 353 +class td354(typing.TypedDict): + x: int + y: int + +# line 358 +td359 = typing.TypedDict('td359', (('x', int), ('y', int))) + +import dataclasses + +# line 363 +@dataclasses.dataclass +class dc364: + x: int + y: int + +# line 369 +dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) +dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) + +import inspect +import itertools + +# line 376 +ge377 = ( + inspect.currentframe() + for i in itertools.count() +) + +# line 382 +def func383(): + # line 384 + ge385 = ( + inspect.currentframe() + for i in itertools.count() + ) + return ge385 + +# line 391 +@decorator +# comment +def func394(): + return 395 + +# line 397 +@decorator + +def func400(): + return 401 + +pass # end of file diff --git a/Lib/test/test_inspect/inspect_stock_annotations.py b/Lib/test/test_inspect/inspect_stock_annotations.py new file mode 100644 index 00000000000..d115a25b650 --- /dev/null +++ b/Lib/test/test_inspect/inspect_stock_annotations.py @@ -0,0 +1,28 @@ +a:int=3 +b:str="foo" + +class MyClass: + a:int=4 + b:str="bar" + def __init__(self, a, b): + self.a = a + self.b = b + def __eq__(self, other): + return isinstance(other, MyClass) and self.a == other.a and self.b == other.b + +def function(a:int, b:str) -> MyClass: + return MyClass(a, b) + + +def function2(a:int, b:"str", c:MyClass) -> MyClass: + pass + + +def function3(a:"int", b:"str", c:"MyClass"): + pass + + +class UnannotatedClass: + pass + +def unannotated_function(a, b, c): pass diff --git a/Lib/test/test_inspect/inspect_stringized_annotations.py b/Lib/test/test_inspect/inspect_stringized_annotations.py new file mode 100644 index 00000000000..a56fb050ead --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +a:int=3 +b:str="foo" + +class MyClass: + a:int=4 + b:str="bar" + def __init__(self, a, b): + self.a = a + self.b = b + def __eq__(self, other): + return isinstance(other, MyClass) and self.a == other.a and self.b == other.b + +def function(a:int, b:str) -> MyClass: + return MyClass(a, b) + + +def function2(a:int, b:"str", c:MyClass) -> MyClass: + pass + + +def function3(a:"int", b:"str", c:"MyClass"): + pass + + +class UnannotatedClass: + pass + +def unannotated_function(a, b, c): pass + +class MyClassWithLocalAnnotations: + mytype = int + x: mytype diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_2.py b/Lib/test/test_inspect/inspect_stringized_annotations_2.py new file mode 100644 index 00000000000..87206d5a646 --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations_2.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +def foo(a, b, c): pass diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py new file mode 100644 index 00000000000..39bfe2edb03 --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py @@ -0,0 +1,87 @@ +from __future__ import annotations +from typing import Callable, Unpack + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: Unpack[Ts], z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +# Eggs is `int` in globals, a TypeVar in type_params, and `str` in locals: +class E[Eggs]: + Eggs = str + x: Eggs + + + +def nested(): + from types import SimpleNamespace + from inspect import get_annotations + + Eggs = bytes + Spam = memoryview + + + class F[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + # Eggs is `int` in globals, `bytes` in the function scope, + # a TypeVar in the type_params, and `str` in locals: + class G[Eggs]: + Eggs = str + x: Eggs + + + return SimpleNamespace( + F=F, + F_annotations=get_annotations(F, eval_str=True), + F_meth_annotations=get_annotations(F.generic_method, eval_str=True), + G_annotations=get_annotations(G, eval_str=True), + generic_func=generic_function, + generic_func_annotations=get_annotations(generic_function, eval_str=True) + ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py new file mode 100644 index 00000000000..c09115bde85 --- /dev/null +++ b/Lib/test/test_inspect/test_inspect.py @@ -0,0 +1,6597 @@ +from annotationlib import Format, ForwardRef +import asyncio +import builtins +import collections +import copy +import datetime +import functools +import gc +import importlib +import inspect +import io +import linecache +import os +import dis +from os.path import normcase +# import _pickle # TODO: RUSTPYTHON +import pickle +import shutil +import stat +import sys +import subprocess +import time +import types +import tempfile +import textwrap +import unicodedata +import unittest +import unittest.mock +import warnings +import weakref + + +try: + from concurrent.futures import ThreadPoolExecutor +except ImportError: + ThreadPoolExecutor = None + +from test.support import cpython_only, import_helper +from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ +from test.support import run_no_yield_async_fn, EqualToForwardRef +from test.support.import_helper import DirsOnSysPath, ready_to_import +from test.support.os_helper import TESTFN, temp_cwd +from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python +from test.support import has_subprocess_support +from test import support + +from test.test_inspect import inspect_fodder as mod +from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_deferred_annotations + + +# Functions tested in this suite: +# ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, +# isbuiltin, isroutine, isgenerator, ispackage, isgeneratorfunction, getmembers, +# getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, +# getclasstree, getargvalues, formatargvalues, currentframe, +# stack, trace, ismethoddescriptor, isdatadescriptor, ismethodwrapper + +# NOTE: There are some additional tests relating to interaction with +# zipimport in the test_zipimport_support test module. + +modfile = mod.__file__ +if modfile.endswith(('c', 'o')): + modfile = modfile[:-1] + +# Normalize file names: on Windows, the case of file names of compiled +# modules depends on the path used to start the python executable. +modfile = normcase(modfile) + +def revise(filename, *args): + return (normcase(filename),) + args + +git = mod.StupidGit() + + +def signatures_with_lexicographic_keyword_only_parameters(): + """ + Yields a whole bunch of functions with only keyword-only parameters, + where those parameters are always in lexicographically sorted order. + """ + parameters = ['a', 'bar', 'c', 'delta', 'ephraim', 'magical', 'yoyo', 'z'] + for i in range(1, 2**len(parameters)): + p = [] + bit = 1 + for j in range(len(parameters)): + if i & (bit << j): + p.append(parameters[j]) + fn_text = "def foo(*, " + ", ".join(p) + "): pass" + symbols = {} + exec(fn_text, symbols, symbols) + yield symbols['foo'] + + +def unsorted_keyword_only_parameters_fn(*, throw, out, the, baby, with_, + the_, bathwater): + pass + +unsorted_keyword_only_parameters = 'throw out the baby with_ the_ bathwater'.split() + +class IsTestBase(unittest.TestCase): + predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, + inspect.isframe, inspect.isfunction, inspect.ismethod, + inspect.ismodule, inspect.istraceback, inspect.ispackage, + inspect.isgenerator, inspect.isgeneratorfunction, + inspect.iscoroutine, inspect.iscoroutinefunction, + inspect.isasyncgen, inspect.isasyncgenfunction, + inspect.ismethodwrapper]) + + def istest(self, predicate, exp): + obj = eval(exp) + self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp)) + + for other in self.predicates - set([predicate]): + if (predicate == inspect.isgeneratorfunction or \ + predicate == inspect.isasyncgenfunction or \ + predicate == inspect.iscoroutinefunction) and \ + other == inspect.isfunction: + continue + if predicate == inspect.ispackage and other == inspect.ismodule: + self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp)) + else: + self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; First has 0, Second has 1: 'iskeyword' + def test__all__(self): + support.check__all__(self, inspect, not_exported=("modulesbyfile",), extra=("get_annotations",)) + +def generator_function_example(self): + for i in range(2): + yield i + +async def async_generator_function_example(self): + async for i in range(2): + yield i + +async def coroutine_function_example(self): + return 'spam' + +@types.coroutine +def gen_coroutine_function_example(self): + yield + return 'spam' + +def meth_noargs(): pass +def meth_o(object, /): pass +def meth_self_noargs(self, /): pass +def meth_self_o(self, object, /): pass +def meth_type_noargs(type, /): pass +def meth_type_o(type, object, /): pass + +# Decorator decorator that returns a simple wrapped function +def identity_wrapper(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + return func(*args, **kwargs) + return wrapped + +# Original signature of the simple wrapped function returned by +# identity_wrapper(). +varargs_signature = ( + (('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ..., +) + +# Decorator decorator that returns a simple descriptor +class custom_descriptor: + def __init__(self, func): + self.func = func + + def __get__(self, instance, owner): + return self.func.__get__(instance, owner) + + +class TestPredicates(IsTestBase): + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'defaultdict' has no attribute 'default_factory' + def test_excluding_predicates(self): + global tb + self.istest(inspect.isbuiltin, 'sys.exit') + self.istest(inspect.isbuiltin, '[].append') + self.istest(inspect.iscode, 'mod.spam.__code__') + try: + 1/0 + except Exception as e: + tb = e.__traceback__ + self.istest(inspect.isframe, 'tb.tb_frame') + self.istest(inspect.istraceback, 'tb') + if hasattr(types, 'GetSetDescriptorType'): + self.istest(inspect.isgetsetdescriptor, + 'type(tb.tb_frame).f_locals') + else: + self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals)) + finally: + # Clear traceback and all the frames and local variables hanging to it. + tb = None + self.istest(inspect.isfunction, 'mod.spam') + self.istest(inspect.isfunction, 'mod.StupidGit.abuse') + self.istest(inspect.ismethod, 'git.argue') + self.istest(inspect.ismethod, 'mod.custom_method') + self.istest(inspect.ismodule, 'mod') + self.istest(inspect.ismethoddescriptor, 'int.__add__') + self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory') + self.istest(inspect.isgenerator, '(x for x in range(2))') + self.istest(inspect.isgeneratorfunction, 'generator_function_example') + self.istest(inspect.isasyncgen, + 'async_generator_function_example(1)') + self.istest(inspect.isasyncgenfunction, + 'async_generator_function_example') + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.istest(inspect.iscoroutine, 'coroutine_function_example(1)') + self.istest(inspect.iscoroutinefunction, 'coroutine_function_example') + + if hasattr(types, 'MemberDescriptorType'): + self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days') + else: + self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days)) + self.istest(inspect.ismethodwrapper, "object().__str__") + self.istest(inspect.ismethodwrapper, "object().__eq__") + self.istest(inspect.ismethodwrapper, "object().__repr__") + self.assertFalse(inspect.ismethodwrapper(type)) + self.assertFalse(inspect.ismethodwrapper(int)) + self.assertFalse(inspect.ismethodwrapper(type("AnyClass", (), {}))) + + def test_ispackage(self): + self.istest(inspect.ispackage, 'unittest') + self.istest(inspect.ispackage, 'importlib') + self.assertFalse(inspect.ispackage(inspect)) + self.assertFalse(inspect.ispackage(mod)) + self.assertFalse(inspect.ispackage(':)')) + + class FakePackage: + __path__ = None + + self.assertFalse(inspect.ispackage(FakePackage())) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true + def test_iscoroutine(self): + async_gen_coro = async_generator_function_example(1) + gen_coro = gen_coroutine_function_example(1) + coro = coroutine_function_example(1) + + class PMClass: + async_generator_partialmethod_example = functools.partialmethod( + async_generator_function_example) + coroutine_partialmethod_example = functools.partialmethod( + coroutine_function_example) + gen_coroutine_partialmethod_example = functools.partialmethod( + gen_coroutine_function_example) + + # partialmethods on the class, bound to an instance + pm_instance = PMClass() + async_gen_coro_pmi = pm_instance.async_generator_partialmethod_example + gen_coro_pmi = pm_instance.gen_coroutine_partialmethod_example + coro_pmi = pm_instance.coroutine_partialmethod_example + + # partialmethods on the class, unbound but accessed via the class + async_gen_coro_pmc = PMClass.async_generator_partialmethod_example + gen_coro_pmc = PMClass.gen_coroutine_partialmethod_example + coro_pmc = PMClass.coroutine_partialmethod_example + + self.assertFalse( + inspect.iscoroutinefunction(gen_coroutine_function_example)) + self.assertFalse( + inspect.iscoroutinefunction( + functools.partial(functools.partial( + gen_coroutine_function_example)))) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmi)) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmc)) + self.assertFalse(inspect.iscoroutinefunction(inspect)) + self.assertFalse(inspect.iscoroutine(gen_coro)) + + self.assertTrue( + inspect.isgeneratorfunction(gen_coroutine_function_example)) + self.assertTrue( + inspect.isgeneratorfunction( + functools.partial(functools.partial( + gen_coroutine_function_example)))) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmi)) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmc)) + self.assertTrue(inspect.isgenerator(gen_coro)) + + async def _fn3(): + pass + + @inspect.markcoroutinefunction + def fn3(): + return _fn3() + + self.assertTrue(inspect.iscoroutinefunction(fn3)) + self.assertTrue( + inspect.iscoroutinefunction( + inspect.markcoroutinefunction(lambda: _fn3()) + ) + ) + + class Cl: + async def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl)) + # instances with async def __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl())) + # unless explicitly marked. + self.assertTrue(inspect.iscoroutinefunction( + inspect.markcoroutinefunction(Cl()) + )) + + class Cl2: + @inspect.markcoroutinefunction + def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl2)) + # instances with marked __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl2())) + # unless explicitly marked. + self.assertTrue(inspect.iscoroutinefunction( + inspect.markcoroutinefunction(Cl2()) + )) + + class Cl3: + @inspect.markcoroutinefunction + @classmethod + def do_something_classy(cls): + pass + + @inspect.markcoroutinefunction + @staticmethod + def do_something_static(): + pass + + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_classy)) + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_static)) + + self.assertFalse( + inspect.iscoroutinefunction(unittest.mock.Mock())) + self.assertTrue( + inspect.iscoroutinefunction(unittest.mock.AsyncMock())) + self.assertTrue( + inspect.iscoroutinefunction(coroutine_function_example)) + self.assertTrue( + inspect.iscoroutinefunction( + functools.partial(functools.partial( + coroutine_function_example)))) + self.assertTrue(inspect.iscoroutinefunction(coro_pmi)) + self.assertTrue(inspect.iscoroutinefunction(coro_pmc)) + self.assertTrue(inspect.iscoroutine(coro)) + + self.assertFalse( + inspect.isgeneratorfunction(unittest.mock.Mock())) + self.assertFalse( + inspect.isgeneratorfunction(unittest.mock.AsyncMock())) + self.assertFalse( + inspect.isgeneratorfunction(coroutine_function_example)) + self.assertFalse( + inspect.isgeneratorfunction( + functools.partial(functools.partial( + coroutine_function_example)))) + self.assertFalse(inspect.isgeneratorfunction(coro_pmi)) + self.assertFalse(inspect.isgeneratorfunction(coro_pmc)) + self.assertFalse(inspect.isgenerator(coro)) + + self.assertFalse( + inspect.isasyncgenfunction(unittest.mock.Mock())) + self.assertFalse( + inspect.isasyncgenfunction(unittest.mock.AsyncMock())) + self.assertFalse( + inspect.isasyncgenfunction(coroutine_function_example)) + self.assertTrue( + inspect.isasyncgenfunction(async_generator_function_example)) + self.assertTrue( + inspect.isasyncgenfunction( + functools.partial(functools.partial( + async_generator_function_example)))) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmi)) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmc)) + self.assertTrue(inspect.isasyncgen(async_gen_coro)) + + coro.close(); gen_coro.close(); # silence warnings + + def test_isawaitable(self): + def gen(): yield + self.assertFalse(inspect.isawaitable(gen())) + + coro = coroutine_function_example(1) + gen_coro = gen_coroutine_function_example(1) + + self.assertTrue(inspect.isawaitable(coro)) + self.assertTrue(inspect.isawaitable(gen_coro)) + + class Future: + def __await__(): + pass + self.assertTrue(inspect.isawaitable(Future())) + self.assertFalse(inspect.isawaitable(Future)) + + class NotFuture: pass + not_fut = NotFuture() + not_fut.__await__ = lambda: None + self.assertFalse(inspect.isawaitable(not_fut)) + + coro.close(); gen_coro.close() # silence warnings + + def test_isroutine(self): + # method + self.assertTrue(inspect.isroutine(git.argue)) + self.assertTrue(inspect.isroutine(mod.custom_method)) + self.assertTrue(inspect.isroutine([].count)) + # function + self.assertTrue(inspect.isroutine(mod.spam)) + self.assertTrue(inspect.isroutine(mod.StupidGit.abuse)) + # slot-wrapper + self.assertTrue(inspect.isroutine(object.__init__)) + self.assertTrue(inspect.isroutine(object.__str__)) + self.assertTrue(inspect.isroutine(object.__lt__)) + self.assertTrue(inspect.isroutine(int.__lt__)) + # method-wrapper + self.assertTrue(inspect.isroutine(object().__init__)) + self.assertTrue(inspect.isroutine(object().__str__)) + self.assertTrue(inspect.isroutine(object().__lt__)) + self.assertTrue(inspect.isroutine((42).__lt__)) + # method-descriptor + self.assertTrue(inspect.isroutine(str.join)) + self.assertTrue(inspect.isroutine(list.append)) + self.assertTrue(inspect.isroutine(''.join)) + self.assertTrue(inspect.isroutine([].append)) + # object + self.assertFalse(inspect.isroutine(object)) + self.assertFalse(inspect.isroutine(object())) + self.assertFalse(inspect.isroutine(str())) + # module + self.assertFalse(inspect.isroutine(mod)) + # type + self.assertFalse(inspect.isroutine(type)) + self.assertFalse(inspect.isroutine(int)) + self.assertFalse(inspect.isroutine(type('some_class', (), {}))) + # partial + self.assertTrue(inspect.isroutine(functools.partial(mod.spam))) + + def test_isroutine_singledispatch(self): + self.assertTrue(inspect.isroutine(functools.singledispatch(mod.spam))) + + class A: + @functools.singledispatchmethod + def method(self, arg): + pass + @functools.singledispatchmethod + @classmethod + def class_method(cls, arg): + pass + @functools.singledispatchmethod + @staticmethod + def static_method(arg): + pass + + self.assertTrue(inspect.isroutine(A.method)) + self.assertTrue(inspect.isroutine(A().method)) + self.assertTrue(inspect.isroutine(A.static_method)) + self.assertTrue(inspect.isroutine(A.class_method)) + + def test_isclass(self): + self.istest(inspect.isclass, 'mod.StupidGit') + self.assertTrue(inspect.isclass(list)) + + class CustomGetattr(object): + def __getattr__(self, attr): + return None + self.assertFalse(inspect.isclass(CustomGetattr())) + + def test_get_slot_members(self): + class C(object): + __slots__ = ("a", "b") + x = C() + x.a = 42 + members = dict(inspect.getmembers(x)) + self.assertIn('a', members) + self.assertNotIn('b', members) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true + def test_isabstract(self): + from abc import ABCMeta, abstractmethod + + class AbstractClassExample(metaclass=ABCMeta): + + @abstractmethod + def foo(self): + pass + + class ClassExample(AbstractClassExample): + def foo(self): + pass + + a = ClassExample() + + # Test general behaviour. + self.assertTrue(inspect.isabstract(AbstractClassExample)) + self.assertFalse(inspect.isabstract(ClassExample)) + self.assertFalse(inspect.isabstract(a)) + self.assertFalse(inspect.isabstract(int)) + self.assertFalse(inspect.isabstract(5)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + [True, False] + def test_isabstract_during_init_subclass(self): + from abc import ABCMeta, abstractmethod + isabstract_checks = [] + class AbstractChecker(metaclass=ABCMeta): + def __init_subclass__(cls): + isabstract_checks.append(inspect.isabstract(cls)) + class AbstractClassExample(AbstractChecker): + @abstractmethod + def foo(self): + pass + class ClassExample(AbstractClassExample): + def foo(self): + pass + self.assertEqual(isabstract_checks, [True, False]) + + isabstract_checks.clear() + class AbstractChild(AbstractClassExample): + pass + class AbstractGrandchild(AbstractChild): + pass + class ConcreteGrandchild(ClassExample): + pass + self.assertEqual(isabstract_checks, [True, True, False]) + + +class TestInterpreterStack(IsTestBase): + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + + git.abuse(7, 8, 9) + + def test_abuse_done(self): + self.istest(inspect.istraceback, 'git.ex.__traceback__') + self.istest(inspect.isframe, 'mod.fr') + + def test_stack(self): + self.assertTrue(len(mod.st) >= 5) + frame1, frame2, frame3, frame4, *_ = mod.st + frameinfo = revise(*frame1[1:]) + self.assertEqual(frameinfo, + (modfile, 16, 'eggs', [' st = inspect.stack()\n'], 0)) + self.assertEqual(frame1.positions, dis.Positions(16, 16, 9, 24)) + frameinfo = revise(*frame2[1:]) + self.assertEqual(frameinfo, + (modfile, 9, 'spam', [' eggs(b + d, c + f)\n'], 0)) + self.assertEqual(frame2.positions, dis.Positions(9, 9, 4, 22)) + frameinfo = revise(*frame3[1:]) + self.assertEqual(frameinfo, + (modfile, 43, 'argue', [' spam(a, b, c)\n'], 0)) + self.assertEqual(frame3.positions, dis.Positions(43, 43, 12, 25)) + frameinfo = revise(*frame4[1:]) + self.assertEqual(frameinfo, + (modfile, 39, 'abuse', [' self.argue(a, b, c)\n'], 0)) + self.assertEqual(frame4.positions, dis.Positions(39, 39, 8, 27)) + # Test named tuple fields + record = mod.st[0] + self.assertIs(record.frame, mod.fr) + self.assertEqual(record.lineno, 16) + self.assertEqual(record.filename, mod.__file__) + self.assertEqual(record.function, 'eggs') + self.assertIn('inspect.stack()', record.code_context[0]) + self.assertEqual(record.index, 0) + + def test_trace(self): + self.assertEqual(len(git.tr), 3) + frame1, frame2, frame3, = git.tr + self.assertEqual(revise(*frame1[1:]), + (modfile, 43, 'argue', [' spam(a, b, c)\n'], 0)) + self.assertEqual(frame1.positions, dis.Positions(43, 43, 12, 25)) + self.assertEqual(revise(*frame2[1:]), + (modfile, 9, 'spam', [' eggs(b + d, c + f)\n'], 0)) + self.assertEqual(frame2.positions, dis.Positions(9, 9, 4, 22)) + self.assertEqual(revise(*frame3[1:]), + (modfile, 18, 'eggs', [' q = y / 0\n'], 0)) + self.assertEqual(frame3.positions, dis.Positions(18, 18, 8, 13)) + + def test_frame(self): + args, varargs, varkw, locals = inspect.getargvalues(mod.fr) + self.assertEqual(args, ['x', 'y']) + self.assertEqual(varargs, None) + self.assertEqual(varkw, None) + self.assertEqual(locals, {'x': 11, 'p': 11, 'y': 14}) + self.assertEqual(inspect.formatargvalues(args, varargs, varkw, locals), + '(x=11, y=14)') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'f_code' + def test_previous_frame(self): + args, varargs, varkw, locals = inspect.getargvalues(mod.fr.f_back) + self.assertEqual(args, ['a', 'b', 'c', 'd', 'e', 'f']) + self.assertEqual(varargs, 'g') + self.assertEqual(varkw, 'h') + self.assertEqual(inspect.formatargvalues(args, varargs, varkw, locals), + '(a=7, b=8, c=9, d=3, e=4, f=5, *g=(), **h={})') + +class GetSourceBase(unittest.TestCase): + # Subclasses must override. + fodderModule = None + + def setUp(self): + with open(inspect.getsourcefile(self.fodderModule), encoding="utf-8") as fp: + self.source = fp.read() + + def sourcerange(self, top, bottom): + lines = self.source.split("\n") + return "\n".join(lines[top-1:bottom]) + ("\n" if bottom else "") + + def assertSourceEqual(self, obj, top, bottom): + self.assertEqual(inspect.getsource(obj), + self.sourcerange(top, bottom)) + +class SlotUser: + 'Docstrings for __slots__' + __slots__ = {'power': 'measured in kilowatts', + 'distance': 'measured in kilometers'} + +class TestRetrievingSourceCode(GetSourceBase): + fodderModule = mod + + def test_getclasses(self): + classes = inspect.getmembers(mod, inspect.isclass) + self.assertEqual(classes, + [('FesteringGob', mod.FesteringGob), + ('MalodorousPervert', mod.MalodorousPervert), + ('ParrotDroppings', mod.ParrotDroppings), + ('StupidGit', mod.StupidGit), + ('Tit', mod.MalodorousPervert), + ('WhichComments', mod.WhichComments), + ]) + tree = inspect.getclasstree([cls[1] for cls in classes]) + self.assertEqual(tree, + [(object, ()), + [(mod.ParrotDroppings, (object,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ], + (mod.StupidGit, (object,)), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ], + (mod.WhichComments, (object,),) + ] + ]) + tree = inspect.getclasstree([cls[1] for cls in classes], True) + self.assertEqual(tree, + [(object, ()), + [(mod.ParrotDroppings, (object,)), + (mod.StupidGit, (object,)), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ], + (mod.WhichComments, (object,),) + ] + ]) + + def test_getfunctions(self): + functions = inspect.getmembers(mod, inspect.isfunction) + self.assertEqual(functions, [('after_closing', mod.after_closing), + ('eggs', mod.eggs), + ('lobbest', mod.lobbest), + ('spam', mod.spam)]) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc(self): + self.assertEqual(inspect.getdoc(mod), 'A module docstring.') + self.assertEqual(inspect.getdoc(mod.StupidGit), + 'A longer,\n\nindented\n\ndocstring.') + self.assertEqual(inspect.getdoc(git.abuse), + 'Another\n\ndocstring\n\ncontaining\n\ntabs') + self.assertEqual(inspect.getdoc(SlotUser.power), + 'measured in kilowatts') + self.assertEqual(inspect.getdoc(SlotUser.distance), + 'measured in kilometers') + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc_inherited(self): + self.assertEqual(inspect.getdoc(mod.FesteringGob), + 'A longer,\n\nindented\n\ndocstring.') + self.assertEqual(inspect.getdoc(mod.FesteringGob.abuse), + 'Another\n\ndocstring\n\ncontaining\n\ntabs') + self.assertEqual(inspect.getdoc(mod.FesteringGob().abuse), + 'Another\n\ndocstring\n\ncontaining\n\ntabs') + self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), + 'The automatic gainsaying.') + + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") + def test_finddoc(self): + finddoc = inspect._finddoc + self.assertEqual(finddoc(int), int.__doc__) + self.assertEqual(finddoc(int.to_bytes), int.to_bytes.__doc__) + self.assertEqual(finddoc(int().to_bytes), int.to_bytes.__doc__) + self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) + self.assertEqual(finddoc(int.real), int.real.__doc__) + + cleandoc_testdata = [ + # first line should have different margin + (' An\n indented\n docstring.', 'An\nindented\n docstring.'), + # trailing whitespace are not removed. + (' An \n \n indented \n docstring. ', + 'An \n \nindented \n docstring. '), + # NUL is not termination. + ('doc\0string\n\n second\0line\n third\0line\0', + 'doc\0string\n\nsecond\0line\nthird\0line\0'), + # first line is lstrip()-ped. other lines are kept when no margin.[w: + (' ', ''), + # compiler.cleandoc() doesn't strip leading/trailing newlines + # to keep maximum backward compatibility. + # inspect.cleandoc() removes them. + ('\n\n\n first paragraph\n\n second paragraph\n\n', + '\n\n\nfirst paragraph\n\n second paragraph\n\n'), + (' \n \n \n ', '\n \n \n '), + ] + + def test_cleandoc(self): + func = inspect.cleandoc + for i, (input, expected) in enumerate(self.cleandoc_testdata): + # only inspect.cleandoc() strip \n + expected = expected.strip('\n') + with self.subTest(i=i): + self.assertEqual(func(input), expected) + + @cpython_only + def test_c_cleandoc(self): + try: + import _testinternalcapi + except ImportError: + return unittest.skip("requires _testinternalcapi") + func = _testinternalcapi.compiler_cleandoc + for i, (input, expected) in enumerate(self.cleandoc_testdata): + with self.subTest(i=i): + self.assertEqual(func(input), expected) + + def test_getcomments(self): + self.assertEqual(inspect.getcomments(mod), '# line 1\n') + self.assertEqual(inspect.getcomments(mod.StupidGit), '# line 20\n') + self.assertEqual(inspect.getcomments(mod2.cls160), '# line 159\n') + # If the object source file is not available, return None. + co = compile('x=1', '_non_existing_filename.py', 'exec') + self.assertIsNone(inspect.getcomments(co)) + # If the object has been defined in C, return None. + self.assertIsNone(inspect.getcomments(list)) + + def test_getmodule(self): + # Check actual module + self.assertEqual(inspect.getmodule(mod), mod) + # Check class (uses __module__ attribute) + self.assertEqual(inspect.getmodule(mod.StupidGit), mod) + # Check a method (no __module__ attribute, falls back to filename) + self.assertEqual(inspect.getmodule(mod.StupidGit.abuse), mod) + # Do it again (check the caching isn't broken) + self.assertEqual(inspect.getmodule(mod.StupidGit.abuse), mod) + # Check a builtin + self.assertEqual(inspect.getmodule(str), sys.modules["builtins"]) + # Check filename override + self.assertEqual(inspect.getmodule(None, modfile), mod) + + def test_getmodule_file_not_found(self): + # See bpo-45406 + def _getabsfile(obj, _filename): + raise FileNotFoundError('bad file') + with unittest.mock.patch('inspect.getabsfile', _getabsfile): + f = inspect.currentframe() + self.assertIsNone(inspect.getmodule(f)) + inspect.getouterframes(f) # smoke test + + def test_getframeinfo_get_first_line(self): + frame_info = inspect.getframeinfo(self.fodderModule.fr, 50) + self.assertEqual(frame_info.code_context[0], "# line 1\n") + self.assertEqual(frame_info.code_context[1], "'A module docstring.'\n") + + def test_getsource(self): + self.assertSourceEqual(git.abuse, 29, 39) + self.assertSourceEqual(mod.StupidGit, 21, 51) + self.assertSourceEqual(mod.lobbest, 75, 76) + self.assertSourceEqual(mod.after_closing, 120, 120) + + def test_getsourcefile(self): + self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile) + self.assertEqual(normcase(inspect.getsourcefile(git.abuse)), modfile) + fn = "_non_existing_filename_used_for_sourcefile_test.py" + co = compile("x=1", fn, "exec") + self.assertEqual(inspect.getsourcefile(co), None) + linecache.cache[co.co_filename] = (1, None, "None", co.co_filename) + try: + self.assertEqual(normcase(inspect.getsourcefile(co)), fn) + finally: + del linecache.cache[co.co_filename] + + def test_getsource_empty_file(self): + with temp_cwd() as cwd: + with open('empty_file.py', 'w'): + pass + sys.path.insert(0, cwd) + try: + import empty_file + self.assertEqual(inspect.getsource(empty_file), '\n') + self.assertEqual(inspect.getsourcelines(empty_file), (['\n'], 0)) + finally: + sys.path.remove(cwd) + + def test_getfile(self): + self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__) + + def test_getfile_builtin_module(self): + with self.assertRaises(TypeError) as e: + inspect.getfile(sys) + self.assertStartsWith(str(e.exception), '<module') + + def test_getfile_builtin_class(self): + with self.assertRaises(TypeError) as e: + inspect.getfile(int) + self.assertStartsWith(str(e.exception), '<class') + + def test_getfile_builtin_function_or_method(self): + with self.assertRaises(TypeError) as e_abs: + inspect.getfile(abs) + self.assertIn('expected, got', str(e_abs.exception)) + with self.assertRaises(TypeError) as e_append: + inspect.getfile(list.append) + self.assertIn('expected, got', str(e_append.exception)) + + def test_getfile_class_without_module(self): + class CM(type): + @property + def __module__(cls): + raise AttributeError + class C(metaclass=CM): + pass + with self.assertRaises(TypeError): + inspect.getfile(C) + + def test_getfile_broken_repr(self): + class ErrorRepr: + def __repr__(self): + raise Exception('xyz') + er = ErrorRepr() + with self.assertRaises(TypeError): + inspect.getfile(er) + + def test_getmodule_recursion(self): + from types import ModuleType + name = '__inspect_dummy' + m = sys.modules[name] = ModuleType(name) + m.__file__ = "<string>" # hopefully not a real filename... + m.__loader__ = "dummy" # pretend the filename is understood by a loader + exec("def x(): pass", m.__dict__) + self.assertEqual(inspect.getsourcefile(m.x.__code__), '<string>') + del sys.modules[name] + inspect.getmodule(compile('a=10','','single')) + + def test_proceed_with_fake_filename(self): + '''doctest monkeypatches linecache to enable inspection''' + fn, source = '<test>', 'def x(): pass\n' + getlines = linecache.getlines + def monkey(filename, module_globals=None): + if filename == fn: + return source.splitlines(keepends=True) + else: + return getlines(filename, module_globals) + linecache.getlines = monkey + try: + ns = {} + exec(compile(source, fn, 'single'), ns) + inspect.getsource(ns["x"]) + finally: + linecache.getlines = getlines + + def test_getsource_on_code_object(self): + self.assertSourceEqual(mod.eggs.__code__, 12, 18) + + def test_getsource_on_generated_class(self): + A = type('A', (unittest.TestCase,), {}) + self.assertEqual(inspect.getsourcefile(A), __file__) + self.assertEqual(inspect.getfile(A), __file__) + self.assertIs(inspect.getmodule(A), sys.modules[__name__]) + self.assertRaises(OSError, inspect.getsource, A) + self.assertRaises(OSError, inspect.getsourcelines, A) + self.assertIsNone(inspect.getcomments(A)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by getsource + def test_getsource_on_class_without_firstlineno(self): + __firstlineno__ = 1 + class C: + nonlocal __firstlineno__ + self.assertRaises(OSError, inspect.getsource, C) + +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + +class TestGetsourceInteractive(unittest.TestCase): + @support.force_not_colorized + def test_getclasses_interactive(self): + # bpo-44648: simulate a REPL session; + # there is no `__file__` in the __main__ module + code = "import sys, inspect; \ + assert not hasattr(sys.modules['__main__'], '__file__'); \ + A = type('A', (), {}); \ + inspect.getsource(A)" + _, _, stderr = assert_python_failure("-c", code, __isolated=True) + self.assertIn(b'OSError: source code not available', stderr) + +class TestGettingSourceOfToplevelFrames(GetSourceBase): + fodderModule = mod + + def test_range_toplevel_frame(self): + self.maxDiff = None + self.assertSourceEqual(mod.currentframe, 1, None) + + def test_range_traceback_toplevel_frame(self): + self.assertSourceEqual(mod.tb, 1, None) + +class TestDecorators(GetSourceBase): + fodderModule = mod2 + + @unittest.expectedFailure # TODO: RUSTPYTHON; pass + def test_wrapped_decorator(self): + self.assertSourceEqual(mod2.wrapped, 14, 17) + + def test_replacing_decorator(self): + self.assertSourceEqual(mod2.gone, 9, 10) + + def test_getsource_unwrap(self): + self.assertSourceEqual(mod2.real, 130, 132) + + def test_decorator_with_lambda(self): + self.assertSourceEqual(mod2.func114, 113, 115) + +class TestOneliners(GetSourceBase): + fodderModule = mod2 + def test_oneline_lambda(self): + # Test inspect.getsource with a one-line lambda function. + self.assertSourceEqual(mod2.oll, 25, 25) + + def test_threeline_lambda(self): + # Test inspect.getsource with a three-line lambda function, + # where the second and third lines are _not_ indented. + self.assertSourceEqual(mod2.tll, 28, 30) + + def test_twoline_indented_lambda(self): + # Test inspect.getsource with a two-line lambda function, + # where the second line _is_ indented. + self.assertSourceEqual(mod2.tlli, 33, 34) + + def test_parenthesized_multiline_lambda(self): + # Test inspect.getsource with a parenthesized multi-line lambda + # function. + self.assertSourceEqual(mod2.parenthesized_lambda, 279, 279) + self.assertSourceEqual(mod2.parenthesized_lambda2, 281, 281) + self.assertSourceEqual(mod2.parenthesized_lambda3, 283, 283) + + def test_post_line_parenthesized_lambda(self): + # Test inspect.getsource with a parenthesized multi-line lambda + # function. + self.assertSourceEqual(mod2.post_line_parenthesized_lambda1, 286, 287) + + def test_nested_lambda(self): + # Test inspect.getsource with a nested lambda function. + self.assertSourceEqual(mod2.nested_lambda, 291, 292) + + def test_onelinefunc(self): + # Test inspect.getsource with a regular one-line function. + self.assertSourceEqual(mod2.onelinefunc, 37, 37) + + def test_manyargs(self): + # Test inspect.getsource with a regular function where + # the arguments are on two lines and _not_ indented and + # the body on the second line with the last arguments. + self.assertSourceEqual(mod2.manyargs, 40, 41) + + def test_twolinefunc(self): + # Test inspect.getsource with a regular function where + # the body is on two lines, following the argument list and + # continued on the next line by a \\. + self.assertSourceEqual(mod2.twolinefunc, 44, 45) + + def test_lambda_in_list(self): + # Test inspect.getsource with a one-line lambda function + # defined in a list, indented. + self.assertSourceEqual(mod2.a[1], 49, 49) + + def test_anonymous(self): + # Test inspect.getsource with a lambda function defined + # as argument to another function. + self.assertSourceEqual(mod2.anonymous, 55, 55) + + def test_enum(self): + self.assertSourceEqual(mod2.enum322, 322, 323) + self.assertSourceEqual(mod2.enum326, 326, 327) + self.assertSourceEqual(mod2.flag330, 330, 331) + self.assertSourceEqual(mod2.flag334, 334, 335) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum338) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum339) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag340) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag341) + + def test_namedtuple(self): + self.assertSourceEqual(mod2.nt346, 346, 348) + self.assertRaises(OSError, inspect.getsource, mod2.nt351) + + def test_typeddict(self): + self.assertSourceEqual(mod2.td354, 354, 356) + self.assertRaises(OSError, inspect.getsource, mod2.td359) + + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + +class TestBlockComments(GetSourceBase): + fodderModule = mod + + def test_toplevel_class(self): + self.assertSourceEqual(mod.WhichComments, 96, 114) + + def test_class_method(self): + self.assertSourceEqual(mod.WhichComments.f, 99, 104) + + def test_class_async_method(self): + self.assertSourceEqual(mod.WhichComments.asyncf, 109, 112) + +class TestBuggyCases(GetSourceBase): + fodderModule = mod2 + + def test_with_comment(self): + self.assertSourceEqual(mod2.with_comment, 58, 59) + + def test_multiline_sig(self): + self.assertSourceEqual(mod2.multiline_sig[0], 63, 64) + + def test_nested_class(self): + self.assertSourceEqual(mod2.func69().func71, 71, 72) + + def test_one_liner_followed_by_non_name(self): + self.assertSourceEqual(mod2.func77, 77, 77) + + def test_one_liner_dedent_non_name(self): + self.assertSourceEqual(mod2.cls82.func83, 83, 83) + + def test_with_comment_instead_of_docstring(self): + self.assertSourceEqual(mod2.func88, 88, 90) + + def test_method_in_dynamic_class(self): + self.assertSourceEqual(mod2.method_in_dynamic_class, 95, 97) + + # This should not skip for CPython, but might on a repackaged python where + # unicodedata is not an external module, or on pypy. + @unittest.skipIf(not hasattr(unicodedata, '__file__') or + unicodedata.__file__.endswith('.py'), + "unicodedata is not an external binary module") + def test_findsource_binary(self): + self.assertRaises(OSError, inspect.getsource, unicodedata) + self.assertRaises(OSError, inspect.findsource, unicodedata) + + def test_findsource_code_in_linecache(self): + lines = ["x=1"] + co = compile(lines[0], "_dynamically_created_file", "exec") + self.assertRaises(OSError, inspect.findsource, co) + self.assertRaises(OSError, inspect.getsource, co) + linecache.cache[co.co_filename] = (1, None, lines, co.co_filename) + try: + self.assertEqual(inspect.findsource(co), (lines,0)) + self.assertEqual(inspect.getsource(co), lines[0]) + finally: + del linecache.cache[co.co_filename] + + def test_findsource_without_filename(self): + for fname in ['', '<string>']: + co = compile('x=1', fname, "exec") + self.assertRaises(IOError, inspect.findsource, co) + self.assertRaises(IOError, inspect.getsource, co) + + def test_findsource_on_func_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "def f(): pass" + co = compile(src, mod.__file__, "exec") + g, l = {}, {} + eval(co, g, l) + func = l['f'] + self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(func) + + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + + def test_getsource_on_method(self): + self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) + + def test_getsource_on_class_code_object(self): + self.assertSourceEqual(mod2.ClassWithCodeObject.code, 315, 317) + + def test_nested_func(self): + self.assertSourceEqual(mod2.cls135.func136, 136, 139) + + def test_class_definition_in_multiline_string_definition(self): + self.assertSourceEqual(mod2.cls149, 149, 152) + + def test_class_definition_in_multiline_comment(self): + self.assertSourceEqual(mod2.cls160, 160, 163) + + def test_nested_class_definition_indented_string(self): + self.assertSourceEqual(mod2.cls173.cls175, 175, 176) + + def test_nested_class_definition(self): + self.assertSourceEqual(mod2.cls183, 183, 188) + self.assertSourceEqual(mod2.cls183.cls185, 185, 188) + + @unittest.expectedFailure # TODO: RUSTPYTHON; pass + def test_class_decorator(self): + self.assertSourceEqual(mod2.cls196, 194, 201) + self.assertSourceEqual(mod2.cls196.cls200, 198, 201) + + @support.requires_docstrings + def test_class_inside_conditional(self): + # We skip this test when docstrings are not present, + # because docstrings are one of the main factors of + # finding the correct class in the source code. + self.assertSourceEqual(mod2.cls238.cls239, 239, 240) + + def test_multiple_children_classes(self): + self.assertSourceEqual(mod2.cls203, 203, 209) + self.assertSourceEqual(mod2.cls203.cls204, 204, 206) + self.assertSourceEqual(mod2.cls203.cls204.cls205, 205, 206) + self.assertSourceEqual(mod2.cls203.cls207, 207, 209) + self.assertSourceEqual(mod2.cls203.cls207.cls205, 208, 209) + + def test_nested_class_definition_inside_function(self): + self.assertSourceEqual(mod2.func212(), 213, 214) + self.assertSourceEqual(mod2.cls213, 218, 222) + self.assertSourceEqual(mod2.cls213().func219(), 220, 221) + + def test_class_with_method_from_other_module(self): + with tempfile.TemporaryDirectory() as tempdir: + with open(os.path.join(tempdir, 'inspect_actual%spy' % os.extsep), + 'w', encoding='utf-8') as f: + f.write(textwrap.dedent(""" + import inspect_other + class A: + def f(self): + pass + class A: + def f(self): + pass # correct one + A.f = inspect_other.A.f + """)) + + with open(os.path.join(tempdir, 'inspect_other%spy' % os.extsep), + 'w', encoding='utf-8') as f: + f.write(textwrap.dedent(""" + class A: + def f(self): + pass + """)) + + with DirsOnSysPath(tempdir): + import inspect_actual + self.assertIn("correct", inspect.getsource(inspect_actual.A)) + # Remove the module from sys.modules to force it to be reloaded. + # This is necessary when the test is run multiple times. + sys.modules.pop("inspect_actual") + + def test_nested_class_definition_inside_async_function(self): + run = run_no_yield_async_fn + + self.assertSourceEqual(run(mod2.func225), 226, 227) + self.assertSourceEqual(mod2.cls226, 231, 235) + self.assertSourceEqual(mod2.cls226.func232, 232, 235) + self.assertSourceEqual(run(mod2.cls226().func232), 233, 234) + + def test_class_definition_same_name_diff_methods(self): + self.assertSourceEqual(mod2.cls296, 296, 298) + self.assertSourceEqual(mod2.cls310, 310, 312) + + def test_generator_expression(self): + self.assertSourceEqual(next(mod2.ge377), 377, 380) + self.assertSourceEqual(next(mod2.func383()), 385, 388) + + def test_comment_or_empty_line_after_decorator(self): + self.assertSourceEqual(mod2.func394, 392, 395) + self.assertSourceEqual(mod2.func400, 398, 401) + + +class TestNoEOL(GetSourceBase): + def setUp(self): + self.tempdir = TESTFN + '_dir' + os.mkdir(self.tempdir) + with open(os.path.join(self.tempdir, 'inspect_fodder3%spy' % os.extsep), + 'w', encoding='utf-8') as f: + f.write("class X:\n pass # No EOL") + with DirsOnSysPath(self.tempdir): + import inspect_fodder3 as mod3 + self.fodderModule = mod3 + super().setUp() + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_class(self): + self.assertSourceEqual(self.fodderModule.X, 1, 2) + + +class TestComplexDecorator(GetSourceBase): + fodderModule = mod2 + + @unittest.expectedFailure # TODO: RUSTPYTHON; return foo + bar() + def test_parens_in_decorator(self): + self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) + +class _BrokenDataDescriptor(object): + """ + A broken data descriptor. See bug #1785. + """ + def __get__(*args): + raise AttributeError("broken data descriptor") + + def __set__(*args): + raise RuntimeError + + def __getattr__(*args): + raise AttributeError("broken data descriptor") + + +class _BrokenMethodDescriptor(object): + """ + A broken method descriptor. See bug #1785. + """ + def __get__(*args): + raise AttributeError("broken method descriptor") + + def __getattr__(*args): + raise AttributeError("broken method descriptor") + + +# Helper for testing classify_class_attrs. +def attrs_wo_objs(cls): + return [t[:3] for t in inspect.classify_class_attrs(cls)] + + +class TestClassesAndFunctions(unittest.TestCase): + def test_newstyle_mro(self): + # The same w/ new-class MRO. + class A(object): pass + class B(A): pass + class C(A): pass + class D(B, C): pass + + expected = (D, B, C, A, object) + got = inspect.getmro(D) + self.assertEqual(expected, got) + + def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, + varkw_e=None, defaults_e=None, + posonlyargs_e=[], kwonlyargs_e=[], + kwonlydefaults_e=None, + ann_e={}): + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ + inspect.getfullargspec(routine) + self.assertEqual(args, args_e) + self.assertEqual(varargs, varargs_e) + self.assertEqual(varkw, varkw_e) + self.assertEqual(defaults, defaults_e) + self.assertEqual(kwonlyargs, kwonlyargs_e) + self.assertEqual(kwonlydefaults, kwonlydefaults_e) + self.assertEqual(ann, ann_e) + + def test_getfullargspec(self): + self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', + kwonlyargs_e=['arg2'], + kwonlydefaults_e={'arg2':1}) + + self.assertFullArgSpecEquals(mod2.annotated, ['arg1'], + ann_e={'arg1' : list}) + self.assertFullArgSpecEquals(mod2.keyword_only_arg, [], + kwonlyargs_e=['arg']) + + self.assertFullArgSpecEquals(mod2.all_markers, ['a', 'b', 'c', 'd'], + kwonlyargs_e=['e', 'f']) + + self.assertFullArgSpecEquals(mod2.all_markers_with_args_and_kwargs, + ['a', 'b', 'c', 'd'], + varargs_e='args', + varkw_e='kwargs', + kwonlyargs_e=['e', 'f']) + + self.assertFullArgSpecEquals(mod2.all_markers_with_defaults, ['a', 'b', 'c', 'd'], + defaults_e=(1,2,3), + kwonlyargs_e=['e', 'f'], + kwonlydefaults_e={'e': 4, 'f': 5}) + + def test_argspec_api_ignores_wrapped(self): + # Issue 20684: low level introspection API must ignore __wrapped__ + @functools.wraps(mod.spam) + def ham(x, y): + pass + # Basic check + self.assertFullArgSpecEquals(ham, ['x', 'y']) + self.assertFullArgSpecEquals(functools.partial(ham), + ['x', 'y']) + + def test_getfullargspec_signature_attr(self): + def test(): + pass + spam_param = inspect.Parameter('spam', inspect.Parameter.POSITIONAL_ONLY) + test.__signature__ = inspect.Signature(parameters=(spam_param,)) + + self.assertFullArgSpecEquals(test, ['spam']) + + def test_getfullargspec_signature_annos(self): + def test(a:'spam') -> 'ham': pass + spec = inspect.getfullargspec(test) + self.assertEqual(test.__annotations__, spec.annotations) + + def test(): pass + spec = inspect.getfullargspec(test) + self.assertEqual(test.__annotations__, spec.annotations) + + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name '_pickle' is not defined. Did you mean: 'pickle'? Or did you forget to import '_pickle'? + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_getfullargspec_builtin_methods(self): + self.assertFullArgSpecEquals(_pickle.Pickler.dump, ['self', 'obj']) + + self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump, ['self', 'obj']) + + self.assertFullArgSpecEquals( + os.stat, + args_e=['path'], + kwonlyargs_e=['dir_fd', 'follow_symlinks'], + kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True}) + + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_getfullargspec_builtin_func(self): + _testcapi = import_helper.import_module("_testcapi") + builtin = _testcapi.docstring_with_signature_with_defaults + spec = inspect.getfullargspec(builtin) + self.assertEqual(spec.defaults[0], 'avocado') + + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_getfullargspec_builtin_func_no_signature(self): + _testcapi = import_helper.import_module("_testcapi") + builtin = _testcapi.docstring_no_signature + with self.assertRaises(TypeError): + inspect.getfullargspec(builtin) + + cls = _testcapi.DocStringNoSignatureTest + obj = _testcapi.DocStringNoSignatureTest() + tests = [ + (_testcapi.docstring_no_signature_noargs, meth_noargs), + (_testcapi.docstring_no_signature_o, meth_o), + (cls.meth_noargs, meth_self_noargs), + (cls.meth_o, meth_self_o), + (obj.meth_noargs, meth_self_noargs), + (obj.meth_o, meth_self_o), + (cls.meth_noargs_class, meth_type_noargs), + (cls.meth_o_class, meth_type_o), + (cls.meth_noargs_static, meth_noargs), + (cls.meth_o_static, meth_o), + (cls.meth_noargs_coexist, meth_self_noargs), + (cls.meth_o_coexist, meth_self_o), + + (time.time, meth_noargs), + (str.lower, meth_self_noargs), + (''.lower, meth_self_noargs), + (set.add, meth_self_o), + (set().add, meth_self_o), + (set.__contains__, meth_self_o), + (set().__contains__, meth_self_o), + (datetime.datetime.__dict__['utcnow'], meth_type_noargs), + (datetime.datetime.utcnow, meth_type_noargs), + (dict.__dict__['__class_getitem__'], meth_type_o), + (dict.__class_getitem__, meth_type_o), + ] + try: + import _stat # noqa: F401 + except ImportError: + # if the _stat extension is not available, stat.S_IMODE() is + # implemented in Python, not in C + pass + else: + tests.append((stat.S_IMODE, meth_o)) + for builtin, template in tests: + with self.subTest(builtin): + self.assertEqual(inspect.getfullargspec(builtin), + inspect.getfullargspec(template)) + + def test_getfullargspec_definition_order_preserved_on_kwonly(self): + for fn in signatures_with_lexicographic_keyword_only_parameters(): + signature = inspect.getfullargspec(fn) + l = list(signature.kwonlyargs) + sorted_l = sorted(l) + self.assertTrue(l) + self.assertEqual(l, sorted_l) + signature = inspect.getfullargspec(unsorted_keyword_only_parameters_fn) + l = list(signature.kwonlyargs) + self.assertEqual(l, unsorted_keyword_only_parameters) + + def test_classify_newstyle(self): + class A(object): + + def s(): pass + s = staticmethod(s) + + def c(cls): pass + c = classmethod(c) + + def getp(self): pass + p = property(getp) + + def m(self): pass + + def m1(self): pass + + datablob = '1' + + dd = _BrokenDataDescriptor() + md = _BrokenMethodDescriptor() + + attrs = attrs_wo_objs(A) + + self.assertIn(('__new__', 'static method', object), attrs, + 'missing __new__') + self.assertIn(('__init__', 'method', object), attrs, 'missing __init__') + + self.assertIn(('s', 'static method', A), attrs, 'missing static method') + self.assertIn(('c', 'class method', A), attrs, 'missing class method') + self.assertIn(('p', 'property', A), attrs, 'missing property') + self.assertIn(('m', 'method', A), attrs, + 'missing plain method: %r' % attrs) + self.assertIn(('m1', 'method', A), attrs, 'missing plain method') + self.assertIn(('datablob', 'data', A), attrs, 'missing data') + self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') + self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') + + class B(A): + + def m(self): pass + + attrs = attrs_wo_objs(B) + self.assertIn(('s', 'static method', A), attrs, 'missing static method') + self.assertIn(('c', 'class method', A), attrs, 'missing class method') + self.assertIn(('p', 'property', A), attrs, 'missing property') + self.assertIn(('m', 'method', B), attrs, 'missing plain method') + self.assertIn(('m1', 'method', A), attrs, 'missing plain method') + self.assertIn(('datablob', 'data', A), attrs, 'missing data') + self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') + self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') + + + class C(A): + + def m(self): pass + def c(self): pass + + attrs = attrs_wo_objs(C) + self.assertIn(('s', 'static method', A), attrs, 'missing static method') + self.assertIn(('c', 'method', C), attrs, 'missing plain method') + self.assertIn(('p', 'property', A), attrs, 'missing property') + self.assertIn(('m', 'method', C), attrs, 'missing plain method') + self.assertIn(('m1', 'method', A), attrs, 'missing plain method') + self.assertIn(('datablob', 'data', A), attrs, 'missing data') + self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') + self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') + + class D(B, C): + + def m1(self): pass + + attrs = attrs_wo_objs(D) + self.assertIn(('s', 'static method', A), attrs, 'missing static method') + self.assertIn(('c', 'method', C), attrs, 'missing plain method') + self.assertIn(('p', 'property', A), attrs, 'missing property') + self.assertIn(('m', 'method', B), attrs, 'missing plain method') + self.assertIn(('m1', 'method', D), attrs, 'missing plain method') + self.assertIn(('datablob', 'data', A), attrs, 'missing data') + self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') + self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ('to_bytes', 'method', <class 'int'>) not found in [('__abs__', 'method', <class 'bool'>), ('__add__', 'method', <class 'bool'>), ('__and__', 'method', <class 'bool'>), ('__bool__', 'method', <class 'bool'>), ('__ceil__', 'class method', <class 'int'>), ('__class__', 'data', <class 'object'>), ('__delattr__', 'class method', <class 'object'>), ('__dir__', 'class method', <class 'object'>), ('__divmod__', 'method', <class 'bool'>), ('__doc__', 'data', <class 'bool'>), ('__eq__', 'class method', <class 'int'>), ('__float__', 'method', <class 'bool'>), ('__floor__', 'class method', <class 'int'>), ('__floordiv__', 'method', <class 'bool'>), ('__format__', 'class method', <class 'bool'>), ('__ge__', 'class method', <class 'int'>), ('__getattribute__', 'class method', <class 'object'>), ('__getnewargs__', 'class method', <class 'int'>), ('__getstate__', 'class method', <class 'object'>), ('__gt__', 'class method', <class 'int'>), ('__hash__', 'method', <class 'int'>), ('__index__', 'method', <class 'bool'>), ('__init__', 'method', <class 'object'>), ('__init_subclass__', 'class method', <class 'object'>), ('__int__', 'method', <class 'bool'>), ('__invert__', 'method', <class 'bool'>), ('__le__', 'class method', <class 'int'>), ('__lshift__', 'method', <class 'bool'>), ('__lt__', 'class method', <class 'int'>), ('__mod__', 'method', <class 'bool'>), ('__mul__', 'method', <class 'bool'>), ('__ne__', 'class method', <class 'int'>), ('__neg__', 'method', <class 'bool'>), ('__new__', 'static method', <class 'bool'>), ('__or__', 'method', <class 'bool'>), ('__pos__', 'method', <class 'bool'>), ('__pow__', 'method', <class 'bool'>), ('__radd__', 'method', <class 'bool'>), ('__rand__', 'method', <class 'bool'>), ('__rdivmod__', 'method', <class 'bool'>), ('__reduce__', 'class method', <class 'object'>), ('__reduce_ex__', 'class method', <class 'object'>), ('__repr__', 'method', <class 'bool'>), ('__rfloordiv__', 'method', <class 'bool'>), ('__rlshift__', 'method', <class 'bool'>), ('__rmod__', 'method', <class 'bool'>), ('__rmul__', 'method', <class 'bool'>), ('__ror__', 'method', <class 'bool'>), ('__round__', 'class method', <class 'int'>), ('__rpow__', 'method', <class 'bool'>), ('__rrshift__', 'method', <class 'bool'>), ('__rshift__', 'method', <class 'bool'>), ('__rsub__', 'method', <class 'bool'>), ('__rtruediv__', 'method', <class 'bool'>), ('__rxor__', 'method', <class 'bool'>), ('__setattr__', 'class method', <class 'object'>), ('__sizeof__', 'class method', <class 'int'>), ('__str__', 'method', <class 'object'>), ('__sub__', 'method', <class 'bool'>), ('__subclasshook__', 'class method', <class 'object'>), ('__truediv__', 'method', <class 'bool'>), ('__trunc__', 'class method', <class 'int'>), ('__xor__', 'method', <class 'bool'>), ('as_integer_ratio', 'class method', <class 'int'>), ('bit_count', 'class method', <class 'int'>), ('bit_length', 'class method', <class 'int'>), ('conjugate', 'class method', <class 'int'>), ('denominator', 'data', <class 'int'>), ('from_bytes', 'class method', <class 'int'>), ('imag', 'data', <class 'int'>), ('is_integer', 'class method', <class 'int'>), ('numerator', 'data', <class 'int'>), ('real', 'data', <class 'int'>), ('to_bytes', 'class method', <class 'int'>)] : missing plain method + def test_classify_builtin_types(self): + # Simple sanity check that all built-in types can have their + # attributes classified. + for name in dir(__builtins__): + builtin = getattr(__builtins__, name) + if isinstance(builtin, type): + inspect.classify_class_attrs(builtin) + + attrs = attrs_wo_objs(bool) + self.assertIn(('__new__', 'static method', bool), attrs, + 'missing __new__') + self.assertIn(('from_bytes', 'class method', int), attrs, + 'missing class method') + self.assertIn(('to_bytes', 'method', int), attrs, + 'missing plain method') + self.assertIn(('__add__', 'method', int), attrs, + 'missing plain method') + self.assertIn(('__and__', 'method', bool), attrs, + 'missing plain method') + + def test_classify_DynamicClassAttribute(self): + class Meta(type): + def __getattr__(self, name): + if name == 'ham': + return 'spam' + return super().__getattr__(name) + class VA(metaclass=Meta): + @types.DynamicClassAttribute + def ham(self): + return 'eggs' + should_find_dca = inspect.Attribute('ham', 'data', VA, VA.__dict__['ham']) + self.assertIn(should_find_dca, inspect.classify_class_attrs(VA)) + should_find_ga = inspect.Attribute('ham', 'data', Meta, 'spam') + self.assertIn(should_find_ga, inspect.classify_class_attrs(VA)) + + def test_classify_overrides_bool(self): + class NoBool(object): + def __eq__(self, other): + return NoBool() + + def __bool__(self): + raise NotImplementedError( + "This object does not specify a boolean value") + + class HasNB(object): + dd = NoBool() + + should_find_attr = inspect.Attribute('dd', 'data', HasNB, HasNB.dd) + self.assertIn(should_find_attr, inspect.classify_class_attrs(HasNB)) + + def test_classify_metaclass_class_attribute(self): + class Meta(type): + fish = 'slap' + def __dir__(self): + return ['__class__', '__module__', '__name__', 'fish'] + class Class(metaclass=Meta): + pass + should_find = inspect.Attribute('fish', 'data', Meta, 'slap') + self.assertIn(should_find, inspect.classify_class_attrs(Class)) + + def test_classify_VirtualAttribute(self): + class Meta(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'BOOM'] + def __getattr__(self, name): + if name =='BOOM': + return 42 + return super().__getattr(name) + class Class(metaclass=Meta): + pass + should_find = inspect.Attribute('BOOM', 'data', Meta, 42) + self.assertIn(should_find, inspect.classify_class_attrs(Class)) + + def test_classify_VirtualAttribute_multi_classes(self): + class Meta1(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'one'] + def __getattr__(self, name): + if name =='one': + return 1 + return super().__getattr__(name) + class Meta2(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'two'] + def __getattr__(self, name): + if name =='two': + return 2 + return super().__getattr__(name) + class Meta3(Meta1, Meta2): + def __dir__(cls): + return list(sorted(set(['__class__', '__module__', '__name__', 'three'] + + Meta1.__dir__(cls) + Meta2.__dir__(cls)))) + def __getattr__(self, name): + if name =='three': + return 3 + return super().__getattr__(name) + class Class1(metaclass=Meta1): + pass + class Class2(Class1, metaclass=Meta3): + pass + + should_find1 = inspect.Attribute('one', 'data', Meta1, 1) + should_find2 = inspect.Attribute('two', 'data', Meta2, 2) + should_find3 = inspect.Attribute('three', 'data', Meta3, 3) + cca = inspect.classify_class_attrs(Class2) + for sf in (should_find1, should_find2, should_find3): + self.assertIn(sf, cca) + + def test_classify_class_attrs_with_buggy_dir(self): + class M(type): + def __dir__(cls): + return ['__class__', '__name__', 'missing'] + class C(metaclass=M): + pass + attrs = [a[0] for a in inspect.classify_class_attrs(C)] + self.assertNotIn('missing', attrs) + + def test_getmembers_descriptors(self): + class A(object): + dd = _BrokenDataDescriptor() + md = _BrokenMethodDescriptor() + + def pred_wrapper(pred): + # A quick'n'dirty way to discard standard attributes of new-style + # classes. + class Empty(object): + pass + def wrapped(x): + if '__name__' in dir(x) and hasattr(Empty, x.__name__): + return False + return pred(x) + return wrapped + + ismethoddescriptor = pred_wrapper(inspect.ismethoddescriptor) + isdatadescriptor = pred_wrapper(inspect.isdatadescriptor) + + self.assertEqual(inspect.getmembers(A, ismethoddescriptor), + [('md', A.__dict__['md'])]) + self.assertEqual(inspect.getmembers(A, isdatadescriptor), + [('dd', A.__dict__['dd'])]) + + class B(A): + pass + + self.assertEqual(inspect.getmembers(B, ismethoddescriptor), + [('md', A.__dict__['md'])]) + self.assertEqual(inspect.getmembers(B, isdatadescriptor), + [('dd', A.__dict__['dd'])]) + + def test_getmembers_method(self): + class B: + def f(self): + pass + + self.assertIn(('f', B.f), inspect.getmembers(B)) + self.assertNotIn(('f', B.f), inspect.getmembers(B, inspect.ismethod)) + b = B() + self.assertIn(('f', b.f), inspect.getmembers(b)) + self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod)) + + def test_getmembers_custom_dir(self): + class CorrectDir: + def __init__(self, attr): + self.attr = attr + def method(self): + return self.attr + 1 + def __dir__(self): + return ['attr', 'method'] + + cd = CorrectDir(5) + self.assertEqual(inspect.getmembers(cd), [ + ('attr', 5), + ('method', cd.method), + ]) + self.assertEqual(inspect.getmembers(cd, inspect.ismethod), [ + ('method', cd.method), + ]) + + def test_getmembers_custom_broken_dir(self): + # inspect.getmembers calls `dir()` on the passed object inside. + # if `__dir__` mentions some non-existent attribute, + # we still need to return others correctly. + class BrokenDir: + existing = 1 + def method(self): + return self.existing + 1 + def __dir__(self): + return ['method', 'missing', 'existing'] + + bd = BrokenDir() + self.assertEqual(inspect.getmembers(bd), [ + ('existing', 1), + ('method', bd.method), + ]) + self.assertEqual(inspect.getmembers(bd, inspect.ismethod), [ + ('method', bd.method), + ]) + + def test_getmembers_custom_duplicated_dir(self): + # Duplicates in `__dir__` must not fail and return just one result. + class DuplicatedDir: + attr = 1 + def __dir__(self): + return ['attr', 'attr'] + + dd = DuplicatedDir() + self.assertEqual(inspect.getmembers(dd), [ + ('attr', 1), + ]) + + def test_getmembers_VirtualAttribute(self): + class M(type): + def __getattr__(cls, name): + if name == 'eggs': + return 'scrambled' + return super().__getattr__(name) + class A(metaclass=M): + @types.DynamicClassAttribute + def eggs(self): + return 'spam' + class B: + def __getattr__(self, attribute): + return None + self.assertIn(('eggs', 'scrambled'), inspect.getmembers(A)) + self.assertIn(('eggs', 'spam'), inspect.getmembers(A())) + b = B() + self.assertIn(('__getattr__', b.__getattr__), inspect.getmembers(b)) + + def test_getmembers_static(self): + class A: + @property + def name(self): + raise NotImplementedError + @types.DynamicClassAttribute + def eggs(self): + raise NotImplementedError + + a = A() + instance_members = inspect.getmembers_static(a) + class_members = inspect.getmembers_static(A) + self.assertIn(('name', inspect.getattr_static(a, 'name')), instance_members) + self.assertIn(('eggs', inspect.getattr_static(a, 'eggs')), instance_members) + self.assertIn(('name', inspect.getattr_static(A, 'name')), class_members) + self.assertIn(('eggs', inspect.getattr_static(A, 'eggs')), class_members) + + def test_getmembers_with_buggy_dir(self): + class M(type): + def __dir__(cls): + return ['__class__', '__name__', 'missing'] + class C(metaclass=M): + pass + attrs = [a[0] for a in inspect.getmembers(C)] + self.assertNotIn('missing', attrs) + + +class TestFormatAnnotation(unittest.TestCase): + def test_typing_replacement(self): + from test.typinganndata.ann_module9 import A, ann, ann1 + self.assertEqual(inspect.formatannotation(ann), 'List[str] | int') + self.assertEqual(inspect.formatannotation(ann1), 'List[testModule.typing.A] | int') + + self.assertEqual(inspect.formatannotation(A, 'testModule.typing'), 'A') + self.assertEqual(inspect.formatannotation(A, 'other'), 'testModule.typing.A') + self.assertEqual( + inspect.formatannotation(ann1, 'testModule.typing'), + 'List[testModule.typing.A] | int', + ) + + def test_forwardref(self): + fwdref = ForwardRef('fwdref') + self.assertEqual(inspect.formatannotation(fwdref), 'fwdref') + + def test_formatannotationrelativeto(self): + from test.typinganndata.ann_module9 import A, ann1 + + # Builtin types: + self.assertEqual( + inspect.formatannotationrelativeto(object)(type), + 'type', + ) + + # Custom types: + self.assertEqual( + inspect.formatannotationrelativeto(None)(A), + 'testModule.typing.A', + ) + + class B: ... + B.__module__ = 'testModule.typing' + + self.assertEqual( + inspect.formatannotationrelativeto(B)(A), + 'A', + ) + + self.assertEqual( + inspect.formatannotationrelativeto(object)(A), + 'testModule.typing.A', + ) + + # Not an instance of "type": + self.assertEqual( + inspect.formatannotationrelativeto(A)(ann1), + 'List[testModule.typing.A] | int', + ) + + +class TestIsMethodDescriptor(unittest.TestCase): + + def test_custom_descriptors(self): + class MethodDescriptor: + def __get__(self, *_): pass + class MethodDescriptorSub(MethodDescriptor): + pass + class DataDescriptorWithNoGet: + def __set__(self, *_): pass + class DataDescriptorWithGetSet: + def __get__(self, *_): pass + def __set__(self, *_): pass + class DataDescriptorWithGetDelete: + def __get__(self, *_): pass + def __delete__(self, *_): pass + class DataDescriptorSub(DataDescriptorWithNoGet, + DataDescriptorWithGetDelete): + pass + + # Custom method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptor()), + '__get__ and no __set__/__delete__ => method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptorSub()), + '__get__ (inherited) and no __set__/__delete__' + ' => method descriptor') + + # Custom data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithNoGet()), + '__set__ (and no __get__) => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetSet()), + '__get__ and __set__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetDelete()), + '__get__ and __delete__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorSub()), + '__get__, __set__ and __delete__ => not a method descriptor') + + # Classes of descriptors (are *not* descriptors themselves): + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptor)) + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptorSub)) + self.assertFalse(inspect.ismethoddescriptor(DataDescriptorSub)) + + def test_builtin_descriptors(self): + builtin_slot_wrapper = int.__add__ # This one is mentioned in docs. + class Owner: + def instance_method(self): pass + @classmethod + def class_method(cls): pass + @staticmethod + def static_method(): pass + @property + def a_property(self): pass + class Slotermeyer: + __slots__ = 'a_slot', + def function(): + pass + a_lambda = lambda: None + + # Example builtin method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(builtin_slot_wrapper), + 'a builtin slot wrapper is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['class_method']), + 'a classmethod object is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['static_method']), + 'a staticmethod object is a method descriptor') + + # Example builtin data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(Owner.__dict__['a_property']), + 'a property is not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(Slotermeyer.__dict__['a_slot']), + 'a slot is not a method descriptor') + + # `types.MethodType`/`types.FunctionType` instances (they *are* + # method descriptors, but `ismethoddescriptor()` explicitly + # excludes them): + self.assertFalse(inspect.ismethoddescriptor(Owner().instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().static_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.static_method)) + self.assertFalse(inspect.ismethoddescriptor(function)) + self.assertFalse(inspect.ismethoddescriptor(a_lambda)) + self.assertTrue(inspect.ismethoddescriptor(functools.partial(function))) + + def test_descriptor_being_a_class(self): + class MethodDescriptorMeta(type): + def __get__(self, *_): pass + class ClassBeingMethodDescriptor(metaclass=MethodDescriptorMeta): + pass + # `ClassBeingMethodDescriptor` itself *is* a method descriptor, + # but it is *also* a class, and `ismethoddescriptor()` explicitly + # excludes classes. + self.assertFalse( + inspect.ismethoddescriptor(ClassBeingMethodDescriptor), + 'classes (instances of type) are explicitly excluded') + + def test_non_descriptors(self): + class Test: + pass + self.assertFalse(inspect.ismethoddescriptor(Test())) + self.assertFalse(inspect.ismethoddescriptor(Test)) + self.assertFalse(inspect.ismethoddescriptor([42])) + self.assertFalse(inspect.ismethoddescriptor(42)) + + +class TestIsDataDescriptor(unittest.TestCase): + + def test_custom_descriptors(self): + class NonDataDescriptor: + def __get__(self, value, type=None): pass + class DataDescriptor0: + def __set__(self, name, value): pass + class DataDescriptor1: + def __delete__(self, name): pass + class DataDescriptor2: + __set__ = None + self.assertFalse(inspect.isdatadescriptor(NonDataDescriptor()), + 'class with only __get__ not a data descriptor') + self.assertTrue(inspect.isdatadescriptor(DataDescriptor0()), + 'class with __set__ is a data descriptor') + self.assertTrue(inspect.isdatadescriptor(DataDescriptor1()), + 'class with __delete__ is a data descriptor') + self.assertTrue(inspect.isdatadescriptor(DataDescriptor2()), + 'class with __set__ = None is a data descriptor') + + def test_slot(self): + class Slotted: + __slots__ = 'foo', + self.assertTrue(inspect.isdatadescriptor(Slotted.foo), + 'a slot is a data descriptor') + + def test_property(self): + class Propertied: + @property + def a_property(self): + pass + self.assertTrue(inspect.isdatadescriptor(Propertied.a_property), + 'a property is a data descriptor') + + def test_functions(self): + class Test(object): + def instance_method(self): pass + @classmethod + def class_method(cls): pass + @staticmethod + def static_method(): pass + def function(): + pass + a_lambda = lambda: None + self.assertFalse(inspect.isdatadescriptor(Test().instance_method), + 'a instance method is not a data descriptor') + self.assertFalse(inspect.isdatadescriptor(Test().class_method), + 'a class method is not a data descriptor') + self.assertFalse(inspect.isdatadescriptor(Test().static_method), + 'a static method is not a data descriptor') + self.assertFalse(inspect.isdatadescriptor(function), + 'a function is not a data descriptor') + self.assertFalse(inspect.isdatadescriptor(a_lambda), + 'a lambda is not a data descriptor') + + +_global_ref = object() +class TestGetClosureVars(unittest.TestCase): + + def test_name_resolution(self): + # Basic test of the 4 different resolution mechanisms + def f(nonlocal_ref): + def g(local_ref): + print(local_ref, nonlocal_ref, _global_ref, unbound_ref) + return g + _arg = object() + nonlocal_vars = {"nonlocal_ref": _arg} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"unbound_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(f(_arg)), expected) + + def test_generator_closure(self): + def f(nonlocal_ref): + def g(local_ref): + print(local_ref, nonlocal_ref, _global_ref, unbound_ref) + yield + return g + _arg = object() + nonlocal_vars = {"nonlocal_ref": _arg} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"unbound_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(f(_arg)), expected) + + def test_method_closure(self): + class C: + def f(self, nonlocal_ref): + def g(local_ref): + print(local_ref, nonlocal_ref, _global_ref, unbound_ref) + return g + _arg = object() + nonlocal_vars = {"nonlocal_ref": _arg} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"unbound_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(C().f(_arg)), expected) + + def test_attribute_same_name_as_global_var(self): + class C: + _global_ref = object() + def f(): + print(C._global_ref, _global_ref) + nonlocal_vars = {"C": C} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"_global_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(f), expected) + + def test_nonlocal_vars(self): + # More complex tests of nonlocal resolution + def _nonlocal_vars(f): + return inspect.getclosurevars(f).nonlocals + + def make_adder(x): + def add(y): + return x + y + return add + + def curry(func, arg1): + return lambda arg2: func(arg1, arg2) + + def less_than(a, b): + return a < b + + # The infamous Y combinator. + def Y(le): + def g(f): + return le(lambda x: f(f)(x)) + Y.g_ref = g + return g(g) + + def check_y_combinator(func): + self.assertEqual(_nonlocal_vars(func), {'f': Y.g_ref}) + + inc = make_adder(1) + add_two = make_adder(2) + greater_than_five = curry(less_than, 5) + + self.assertEqual(_nonlocal_vars(inc), {'x': 1}) + self.assertEqual(_nonlocal_vars(add_two), {'x': 2}) + self.assertEqual(_nonlocal_vars(greater_than_five), + {'arg1': 5, 'func': less_than}) + self.assertEqual(_nonlocal_vars((lambda x: lambda y: x + y)(3)), + {'x': 3}) + Y(check_y_combinator) + + def test_getclosurevars_empty(self): + def foo(): pass + _empty = inspect.ClosureVars({}, {}, {}, set()) + self.assertEqual(inspect.getclosurevars(lambda: True), _empty) + self.assertEqual(inspect.getclosurevars(foo), _empty) + + def test_getclosurevars_error(self): + class T: pass + self.assertRaises(TypeError, inspect.getclosurevars, 1) + self.assertRaises(TypeError, inspect.getclosurevars, list) + self.assertRaises(TypeError, inspect.getclosurevars, {}) + + def _private_globals(self): + code = """def f(): print(path)""" + ns = {} + exec(code, ns) + return ns["f"], ns + + def test_builtins_fallback(self): + f, ns = self._private_globals() + ns.pop("__builtins__", None) + expected = inspect.ClosureVars({}, {}, {"print":print}, {"path"}) + self.assertEqual(inspect.getclosurevars(f), expected) + + def test_builtins_as_dict(self): + f, ns = self._private_globals() + ns["__builtins__"] = {"path":1} + expected = inspect.ClosureVars({}, {}, {"path":1}, {"print"}) + self.assertEqual(inspect.getclosurevars(f), expected) + + def test_builtins_as_module(self): + f, ns = self._private_globals() + ns["__builtins__"] = os + expected = inspect.ClosureVars({}, {}, {"path":os.path}, {"print"}) + self.assertEqual(inspect.getclosurevars(f), expected) + + +class TestGetcallargsFunctions(unittest.TestCase): + + def assertEqualCallArgs(self, func, call_params_string, locs=None): + locs = dict(locs or {}, func=func) + r1 = eval('func(%s)' % call_params_string, None, locs) + r2 = eval('inspect.getcallargs(func, %s)' % call_params_string, None, + locs) + self.assertEqual(r1, r2) + + def assertEqualException(self, func, call_param_string, locs=None): + locs = dict(locs or {}, func=func) + try: + eval('func(%s)' % call_param_string, None, locs) + except Exception as e: + ex1 = e + else: + self.fail('Exception not raised') + try: + eval('inspect.getcallargs(func, %s)' % call_param_string, None, + locs) + except Exception as e: + ex2 = e + else: + self.fail('Exception not raised') + self.assertIs(type(ex1), type(ex2)) + self.assertEqual(str(ex1), str(ex2)) + del ex1, ex2 + + def makeCallable(self, signature): + """Create a function that returns its locals()""" + code = "lambda %s: locals()" + return eval(code % signature) + + def test_plain(self): + f = self.makeCallable('a, b=1') + self.assertEqualCallArgs(f, '2') + self.assertEqualCallArgs(f, '2, 3') + self.assertEqualCallArgs(f, 'a=2') + self.assertEqualCallArgs(f, 'b=3, a=2') + self.assertEqualCallArgs(f, '2, b=3') + # expand *iterable / **mapping + self.assertEqualCallArgs(f, '*(2,)') + self.assertEqualCallArgs(f, '*[2]') + self.assertEqualCallArgs(f, '*(2, 3)') + self.assertEqualCallArgs(f, '*[2, 3]') + self.assertEqualCallArgs(f, '**{"a":2}') + self.assertEqualCallArgs(f, 'b=3, **{"a":2}') + self.assertEqualCallArgs(f, '2, **{"b":3}') + self.assertEqualCallArgs(f, '**{"b":3, "a":2}') + # expand UserList / UserDict + self.assertEqualCallArgs(f, '*collections.UserList([2])') + self.assertEqualCallArgs(f, '*collections.UserList([2, 3])') + self.assertEqualCallArgs(f, '**collections.UserDict(a=2)') + self.assertEqualCallArgs(f, '2, **collections.UserDict(b=3)') + self.assertEqualCallArgs(f, 'b=2, **collections.UserDict(a=3)') + + def test_varargs(self): + f = self.makeCallable('a, b=1, *c') + self.assertEqualCallArgs(f, '2') + self.assertEqualCallArgs(f, '2, 3') + self.assertEqualCallArgs(f, '2, 3, 4') + self.assertEqualCallArgs(f, '*(2,3,4)') + self.assertEqualCallArgs(f, '2, *[3,4]') + self.assertEqualCallArgs(f, '2, 3, *collections.UserList([4])') + + def test_varkw(self): + f = self.makeCallable('a, b=1, **c') + self.assertEqualCallArgs(f, 'a=2') + self.assertEqualCallArgs(f, '2, b=3, c=4') + self.assertEqualCallArgs(f, 'b=3, a=2, c=4') + self.assertEqualCallArgs(f, 'c=4, **{"a":2, "b":3}') + self.assertEqualCallArgs(f, '2, c=4, **{"b":3}') + self.assertEqualCallArgs(f, 'b=2, **{"a":3, "c":4}') + self.assertEqualCallArgs(f, '**collections.UserDict(a=2, b=3, c=4)') + self.assertEqualCallArgs(f, '2, c=4, **collections.UserDict(b=3)') + self.assertEqualCallArgs(f, 'b=2, **collections.UserDict(a=3, c=4)') + + def test_varkw_only(self): + # issue11256: + f = self.makeCallable('**c') + self.assertEqualCallArgs(f, '') + self.assertEqualCallArgs(f, 'a=1') + self.assertEqualCallArgs(f, 'a=1, b=2') + self.assertEqualCallArgs(f, 'c=3, **{"a": 1, "b": 2}') + self.assertEqualCallArgs(f, '**collections.UserDict(a=1, b=2)') + self.assertEqualCallArgs(f, 'c=3, **collections.UserDict(a=1, b=2)') + + def test_keyword_only(self): + f = self.makeCallable('a=3, *, c, d=2') + self.assertEqualCallArgs(f, 'c=3') + self.assertEqualCallArgs(f, 'c=3, a=3') + self.assertEqualCallArgs(f, 'a=2, c=4') + self.assertEqualCallArgs(f, '4, c=4') + self.assertEqualException(f, '') + self.assertEqualException(f, '3') + self.assertEqualException(f, 'a=3') + self.assertEqualException(f, 'd=4') + + f = self.makeCallable('*, c, d=2') + self.assertEqualCallArgs(f, 'c=3') + self.assertEqualCallArgs(f, 'c=3, d=4') + self.assertEqualCallArgs(f, 'd=4, c=3') + + def test_multiple_features(self): + f = self.makeCallable('a, b=2, *f, **g') + self.assertEqualCallArgs(f, '2, 3, 7') + self.assertEqualCallArgs(f, '2, 3, x=8') + self.assertEqualCallArgs(f, '2, 3, x=8, *[(4,[5,6]), 7]') + self.assertEqualCallArgs(f, '2, x=8, *[3, (4,[5,6]), 7], y=9') + self.assertEqualCallArgs(f, 'x=8, *[2, 3, (4,[5,6])], y=9') + self.assertEqualCallArgs(f, 'x=8, *collections.UserList(' + '[2, 3, (4,[5,6])]), **{"y":9, "z":10}') + self.assertEqualCallArgs(f, '2, x=8, *collections.UserList([3, ' + '(4,[5,6])]), **collections.UserDict(' + 'y=9, z=10)') + + f = self.makeCallable('a, b=2, *f, x, y=99, **g') + self.assertEqualCallArgs(f, '2, 3, x=8') + self.assertEqualCallArgs(f, '2, 3, x=8, *[(4,[5,6]), 7]') + self.assertEqualCallArgs(f, '2, x=8, *[3, (4,[5,6]), 7], y=9, z=10') + self.assertEqualCallArgs(f, 'x=8, *[2, 3, (4,[5,6])], y=9, z=10') + self.assertEqualCallArgs(f, 'x=8, *collections.UserList(' + '[2, 3, (4,[5,6])]), q=0, **{"y":9, "z":10}') + self.assertEqualCallArgs(f, '2, x=8, *collections.UserList([3, ' + '(4,[5,6])]), q=0, **collections.UserDict(' + 'y=9, z=10)') + + @unittest.expectedFailure # TODO: RUSTPYTHON; + <lambda>() got an unexpected keyword argument 'x' + def test_errors(self): + f0 = self.makeCallable('') + f1 = self.makeCallable('a, b') + f2 = self.makeCallable('a, b=1') + # f0 takes no arguments + self.assertEqualException(f0, '1') + self.assertEqualException(f0, 'x=1') + self.assertEqualException(f0, '1,x=1') + # f1 takes exactly 2 arguments + self.assertEqualException(f1, '') + self.assertEqualException(f1, '1') + self.assertEqualException(f1, 'a=2') + self.assertEqualException(f1, 'b=3') + # f2 takes at least 1 argument + self.assertEqualException(f2, '') + self.assertEqualException(f2, 'b=3') + for f in f1, f2: + # f1/f2 takes exactly/at most 2 arguments + self.assertEqualException(f, '2, 3, 4') + self.assertEqualException(f, '1, 2, 3, a=1') + self.assertEqualException(f, '2, 3, 4, c=5') + self.assertEqualException(f, '2, 3, 4, a=1, c=5') + # f got an unexpected keyword argument + self.assertEqualException(f, 'c=2') + self.assertEqualException(f, '2, c=3') + self.assertEqualException(f, '2, 3, c=4') + self.assertEqualException(f, '2, c=4, b=3') + self.assertEqualException(f, '**{u"\u03c0\u03b9": 4}') + # f got multiple values for keyword argument + self.assertEqualException(f, '1, a=2') + self.assertEqualException(f, '1, **{"a":2}') + self.assertEqualException(f, '1, 2, b=3') + self.assertEqualException(f, '1, c=3, a=2') + # issue11256: + f3 = self.makeCallable('**c') + self.assertEqualException(f3, '1, 2') + self.assertEqualException(f3, '1, 2, a=1, b=2') + f4 = self.makeCallable('*, a, b=0') + self.assertEqualException(f4, '1, 2') + self.assertEqualException(f4, '1, 2, a=1, b=2') + self.assertEqualException(f4, 'a=1, a=3') + self.assertEqualException(f4, 'a=1, c=3') + self.assertEqualException(f4, 'a=1, a=3, b=4') + self.assertEqualException(f4, 'a=1, b=2, a=3, b=4') + self.assertEqualException(f4, 'a=1, a=2, a=3, b=4') + + # issue #20816: getcallargs() fails to iterate over non-existent + # kwonlydefaults and raises a wrong TypeError + def f5(*, a): pass + with self.assertRaisesRegex(TypeError, + 'missing 1 required keyword-only'): + inspect.getcallargs(f5) + + + # issue20817: + def f6(a, b, c): + pass + with self.assertRaisesRegex(TypeError, "'a', 'b' and 'c'"): + inspect.getcallargs(f6) + + # bpo-33197 + with self.assertRaisesRegex(ValueError, + 'variadic keyword parameters cannot' + ' have default values'): + inspect.Parameter("foo", kind=inspect.Parameter.VAR_KEYWORD, + default=42) + with self.assertRaisesRegex(ValueError, + "value 5 is not a valid Parameter.kind"): + inspect.Parameter("bar", kind=5, default=42) + + with self.assertRaisesRegex(TypeError, + 'name must be a str, not a int'): + inspect.Parameter(123, kind=4) + +class TestGetcallargsMethods(TestGetcallargsFunctions): + + def setUp(self): + class Foo(object): + pass + self.cls = Foo + self.inst = Foo() + + def makeCallable(self, signature): + assert 'self' not in signature + mk = super(TestGetcallargsMethods, self).makeCallable + self.cls.method = mk('self, ' + signature) + return self.inst.method + +class TestGetcallargsUnboundMethods(TestGetcallargsMethods): + + def makeCallable(self, signature): + super(TestGetcallargsUnboundMethods, self).makeCallable(signature) + return self.cls.method + + def assertEqualCallArgs(self, func, call_params_string, locs=None): + return super(TestGetcallargsUnboundMethods, self).assertEqualCallArgs( + *self._getAssertEqualParams(func, call_params_string, locs)) + + def assertEqualException(self, func, call_params_string, locs=None): + return super(TestGetcallargsUnboundMethods, self).assertEqualException( + *self._getAssertEqualParams(func, call_params_string, locs)) + + def _getAssertEqualParams(self, func, call_params_string, locs=None): + assert 'inst' not in call_params_string + locs = dict(locs or {}, inst=self.inst) + return (func, 'inst,' + call_params_string, locs) + + +class TestGetattrStatic(unittest.TestCase): + + def test_basic(self): + class Thing(object): + x = object() + + thing = Thing() + self.assertEqual(inspect.getattr_static(thing, 'x'), Thing.x) + self.assertEqual(inspect.getattr_static(thing, 'x', None), Thing.x) + with self.assertRaises(AttributeError): + inspect.getattr_static(thing, 'y') + + self.assertEqual(inspect.getattr_static(thing, 'y', 3), 3) + + def test_inherited(self): + class Thing(object): + x = object() + class OtherThing(Thing): + pass + + something = OtherThing() + self.assertEqual(inspect.getattr_static(something, 'x'), Thing.x) + + def test_instance_attr(self): + class Thing(object): + x = 2 + def __init__(self, x): + self.x = x + thing = Thing(3) + self.assertEqual(inspect.getattr_static(thing, 'x'), 3) + del thing.x + self.assertEqual(inspect.getattr_static(thing, 'x'), 2) + + def test_property(self): + class Thing(object): + @property + def x(self): + raise AttributeError("I'm pretending not to exist") + thing = Thing() + self.assertEqual(inspect.getattr_static(thing, 'x'), Thing.x) + + def test_descriptor_raises_AttributeError(self): + class descriptor(object): + def __get__(*_): + raise AttributeError("I'm pretending not to exist") + desc = descriptor() + class Thing(object): + x = desc + thing = Thing() + self.assertEqual(inspect.getattr_static(thing, 'x'), desc) + + def test_classAttribute(self): + class Thing(object): + x = object() + + self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.x) + + def test_classVirtualAttribute(self): + class Thing(object): + @types.DynamicClassAttribute + def x(self): + return self._x + _x = object() + + self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.__dict__['x']) + + def test_inherited_classattribute(self): + class Thing(object): + x = object() + class OtherThing(Thing): + pass + + self.assertEqual(inspect.getattr_static(OtherThing, 'x'), Thing.x) + + def test_slots(self): + class Thing(object): + y = 'bar' + __slots__ = ['x'] + def __init__(self): + self.x = 'foo' + thing = Thing() + self.assertEqual(inspect.getattr_static(thing, 'x'), Thing.x) + self.assertEqual(inspect.getattr_static(thing, 'y'), 'bar') + + del thing.x + self.assertEqual(inspect.getattr_static(thing, 'x'), Thing.x) + + def test_metaclass(self): + class meta(type): + attr = 'foo' + class Thing(object, metaclass=meta): + pass + self.assertEqual(inspect.getattr_static(Thing, 'attr'), 'foo') + + class sub(meta): + pass + class OtherThing(object, metaclass=sub): + x = 3 + self.assertEqual(inspect.getattr_static(OtherThing, 'attr'), 'foo') + + class OtherOtherThing(OtherThing): + pass + # this test is odd, but it was added as it exposed a bug + self.assertEqual(inspect.getattr_static(OtherOtherThing, 'x'), 3) + + def test_no_dict_no_slots(self): + self.assertEqual(inspect.getattr_static(1, 'foo', None), None) + self.assertNotEqual(inspect.getattr_static('foo', 'lower'), None) + + def test_no_dict_no_slots_instance_member(self): + # returns descriptor + with open(__file__, encoding='utf-8') as handle: + self.assertEqual(inspect.getattr_static(handle, 'name'), type(handle).name) + + def test_inherited_slots(self): + # returns descriptor + class Thing(object): + __slots__ = ['x'] + def __init__(self): + self.x = 'foo' + + class OtherThing(Thing): + pass + # it would be nice if this worked... + # we get the descriptor instead of the instance attribute + self.assertEqual(inspect.getattr_static(OtherThing(), 'x'), Thing.x) + + def test_descriptor(self): + class descriptor(object): + def __get__(self, instance, owner): + return 3 + class Foo(object): + d = descriptor() + + foo = Foo() + + # for a non data descriptor we return the instance attribute + foo.__dict__['d'] = 1 + self.assertEqual(inspect.getattr_static(foo, 'd'), 1) + + # if the descriptor is a data-descriptor we should return the + # descriptor + descriptor.__set__ = lambda s, i, v: None + self.assertEqual(inspect.getattr_static(foo, 'd'), Foo.__dict__['d']) + + del descriptor.__set__ + descriptor.__delete__ = lambda s, i, o: None + self.assertEqual(inspect.getattr_static(foo, 'd'), Foo.__dict__['d']) + + def test_metaclass_with_descriptor(self): + class descriptor(object): + def __get__(self, instance, owner): + return 3 + class meta(type): + d = descriptor() + class Thing(object, metaclass=meta): + pass + self.assertEqual(inspect.getattr_static(Thing, 'd'), meta.__dict__['d']) + + + def test_class_as_property(self): + class Base(object): + foo = 3 + + class Something(Base): + executed = False + @property + def __class__(self): + self.executed = True + return object + + instance = Something() + self.assertEqual(inspect.getattr_static(instance, 'foo'), 3) + self.assertFalse(instance.executed) + self.assertEqual(inspect.getattr_static(Something, 'foo'), 3) + + def test_mro_as_property(self): + class Meta(type): + @property + def __mro__(self): + return (object,) + + class Base(object): + foo = 3 + + class Something(Base, metaclass=Meta): + pass + + self.assertEqual(inspect.getattr_static(Something(), 'foo'), 3) + self.assertEqual(inspect.getattr_static(Something, 'foo'), 3) + + def test_dict_as_property(self): + test = self + test.called = False + + class Foo(dict): + a = 3 + @property + def __dict__(self): + test.called = True + return {} + + foo = Foo() + foo.a = 4 + self.assertEqual(inspect.getattr_static(foo, 'a'), 3) + self.assertFalse(test.called) + + class Bar(Foo): pass + + bar = Bar() + bar.a = 5 + self.assertEqual(inspect.getattr_static(bar, 'a'), 3) + self.assertFalse(test.called) + + def test_mutated_mro(self): + test = self + test.called = False + + class Foo(dict): + a = 3 + @property + def __dict__(self): + test.called = True + return {} + + class Bar(dict): + a = 4 + + class Baz(Bar): pass + + baz = Baz() + self.assertEqual(inspect.getattr_static(baz, 'a'), 4) + Baz.__bases__ = (Foo,) + self.assertEqual(inspect.getattr_static(baz, 'a'), 3) + self.assertFalse(test.called) + + def test_custom_object_dict(self): + test = self + test.called = False + + class Custom(dict): + def get(self, key, default=None): + test.called = True + super().get(key, default) + + class Foo(object): + a = 3 + foo = Foo() + foo.__dict__ = Custom() + self.assertEqual(inspect.getattr_static(foo, 'a'), 3) + self.assertFalse(test.called) + + def test_metaclass_dict_as_property(self): + class Meta(type): + @property + def __dict__(self): + self.executed = True + + class Thing(metaclass=Meta): + executed = False + + def __init__(self): + self.spam = 42 + + instance = Thing() + self.assertEqual(inspect.getattr_static(instance, "spam"), 42) + self.assertFalse(Thing.executed) + + def test_module(self): + sentinel = object() + self.assertIsNot(inspect.getattr_static(sys, "version", sentinel), + sentinel) + + def test_metaclass_with_metaclass_with_dict_as_property(self): + class MetaMeta(type): + @property + def __dict__(self): + self.executed = True + return dict(spam=42) + + class Meta(type, metaclass=MetaMeta): + executed = False + + class Thing(metaclass=Meta): + pass + + with self.assertRaises(AttributeError): + inspect.getattr_static(Thing, "spam") + self.assertFalse(Thing.executed) + + def test_custom___getattr__(self): + test = self + test.called = False + + class Foo: + def __getattr__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'whatever') + + self.assertFalse(test.called) + + def test_custom___getattribute__(self): + test = self + test.called = False + + class Foo: + def __getattribute__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'really_could_be_anything') + + self.assertFalse(test.called) + + def test_cache_does_not_cause_classes_to_persist(self): + # regression test for gh-118013: + # check that the internal _shadowed_dict cache does not cause + # dynamically created classes to have extended lifetimes even + # when no other strong references to those classes remain. + # Since these classes can themselves hold strong references to + # other objects, this can cause unexpected memory consumption. + class Foo: pass + Foo.instance = Foo() + weakref_to_class = weakref.ref(Foo) + inspect.getattr_static(Foo.instance, 'whatever', 'irrelevant') + del Foo + gc.collect() + self.assertIsNone(weakref_to_class()) + + +class TestGetGeneratorState(unittest.TestCase): + + def setUp(self): + def number_generator(): + for number in range(5): + yield number + self.generator = number_generator() + + def _generatorstate(self): + return inspect.getgeneratorstate(self.generator) + + def test_created(self): + self.assertEqual(self._generatorstate(), inspect.GEN_CREATED) + + def test_suspended(self): + next(self.generator) + self.assertEqual(self._generatorstate(), inspect.GEN_SUSPENDED) + + def test_closed_after_exhaustion(self): + for i in self.generator: + pass + self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + + def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + self.generator.throw(RuntimeError) + self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + + def test_closed_after_close(self): + self.generator.close() + self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + + def test_running(self): + # As mentioned on issue #10220, checking for the RUNNING state only + # makes sense inside the generator itself. + # The following generator checks for this by using the closure's + # reference to self and the generator state checking helper method + def running_check_generator(): + for number in range(5): + self.assertEqual(self._generatorstate(), inspect.GEN_RUNNING) + yield number + self.assertEqual(self._generatorstate(), inspect.GEN_RUNNING) + self.generator = running_check_generator() + # Running up to the first yield + next(self.generator) + # Running after the first yield + next(self.generator) + + def test_types_coroutine_wrapper_state(self): + def gen(): + yield 1 + yield 2 + + @types.coroutine + def wrapped_generator_coro(): + # return a generator iterator so types.coroutine + # wraps it into types._GeneratorWrapper. + return gen() + + g = wrapped_generator_coro() + self.addCleanup(g.close) + self.assertIs(type(g), types._GeneratorWrapper) + + # _GeneratorWrapper must provide gi_suspended/cr_suspended + # so inspect.get*state() doesn't raise AttributeError. + self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_CREATED) + self.assertEqual(inspect.getcoroutinestate(g), inspect.CORO_CREATED) + + next(g) + self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_SUSPENDED) + self.assertEqual(inspect.getcoroutinestate(g), inspect.CORO_SUSPENDED) + + def test_easy_debugging(self): + # repr() and str() of a generator state should contain the state name + names = 'GEN_CREATED GEN_RUNNING GEN_SUSPENDED GEN_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + def test_getgeneratorlocals(self): + def each(lst, a=None): + b=(1, 2, 3) + for v in lst: + if v == 3: + c = 12 + yield v + + numbers = each([1, 2, 3]) + self.assertEqual(inspect.getgeneratorlocals(numbers), + {'a': None, 'lst': [1, 2, 3]}) + next(numbers) + self.assertEqual(inspect.getgeneratorlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 1, + 'b': (1, 2, 3)}) + next(numbers) + self.assertEqual(inspect.getgeneratorlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 2, + 'b': (1, 2, 3)}) + next(numbers) + self.assertEqual(inspect.getgeneratorlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 3, + 'b': (1, 2, 3), 'c': 12}) + try: + next(numbers) + except StopIteration: + pass + self.assertEqual(inspect.getgeneratorlocals(numbers), {}) + + def test_getgeneratorlocals_empty(self): + def yield_one(): + yield 1 + one = yield_one() + self.assertEqual(inspect.getgeneratorlocals(one), {}) + try: + next(one) + except StopIteration: + pass + self.assertEqual(inspect.getgeneratorlocals(one), {}) + + def test_getgeneratorlocals_error(self): + self.assertRaises(TypeError, inspect.getgeneratorlocals, 1) + self.assertRaises(TypeError, inspect.getgeneratorlocals, lambda x: True) + self.assertRaises(TypeError, inspect.getgeneratorlocals, set) + self.assertRaises(TypeError, inspect.getgeneratorlocals, (2,3)) + + +class TestGetCoroutineState(unittest.TestCase): + + def setUp(self): + @types.coroutine + def number_coroutine(): + for number in range(5): + yield number + async def coroutine(): + await number_coroutine() + self.coroutine = coroutine() + + def tearDown(self): + self.coroutine.close() + + def _coroutinestate(self): + return inspect.getcoroutinestate(self.coroutine) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' + def test_created(self): + self.assertEqual(self._coroutinestate(), inspect.CORO_CREATED) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' + def test_suspended(self): + self.coroutine.send(None) + self.assertEqual(self._coroutinestate(), inspect.CORO_SUSPENDED) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' + def test_closed_after_exhaustion(self): + while True: + try: + self.coroutine.send(None) + except StopIteration: + break + + self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' + def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + self.coroutine.throw(RuntimeError) + self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'coroutine' object has no attribute 'cr_suspended' + def test_closed_after_close(self): + self.coroutine.close() + self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + + def test_easy_debugging(self): + # repr() and str() of a coroutine state should contain the state name + names = 'CORO_CREATED CORO_RUNNING CORO_SUSPENDED CORO_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + def test_getcoroutinelocals(self): + @types.coroutine + def gencoro(): + yield + + gencoro = gencoro() + async def func(a=None): + b = 'spam' + await gencoro + + coro = func() + self.assertEqual(inspect.getcoroutinelocals(coro), + {'a': None, 'gencoro': gencoro}) + coro.send(None) + self.assertEqual(inspect.getcoroutinelocals(coro), + {'a': None, 'gencoro': gencoro, 'b': 'spam'}) + + +@support.requires_working_socket() +class TestGetAsyncGenState(unittest.IsolatedAsyncioTestCase): + + def setUp(self): + async def number_asyncgen(): + for number in range(5): + yield number + self.asyncgen = number_asyncgen() + + async def asyncTearDown(self): + await self.asyncgen.aclose() + + @classmethod + def tearDownClass(cls): + asyncio.events._set_event_loop_policy(None) + + def _asyncgenstate(self): + return inspect.getasyncgenstate(self.asyncgen) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' + def test_created(self): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CREATED) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' + async def test_suspended(self): + value = await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + self.assertEqual(value, 0) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' + async def test_closed_after_exhaustion(self): + countdown = 7 + with self.assertRaises(StopAsyncIteration): + while countdown := countdown - 1: + await anext(self.asyncgen) + self.assertEqual(countdown, 1) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' + async def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + await self.asyncgen.athrow(RuntimeError) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'async_generator' object has no attribute 'ag_suspended' + async def test_running(self): + async def running_check_asyncgen(): + for number in range(5): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + yield number + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + self.asyncgen = running_check_asyncgen() + # Running up to the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + # Running after the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + + def test_easy_debugging(self): + # repr() and str() of a asyncgen state should contain the state name + names = 'AGEN_CREATED AGEN_RUNNING AGEN_SUSPENDED AGEN_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + async def test_getasyncgenlocals(self): + async def each(lst, a=None): + b=(1, 2, 3) + for v in lst: + if v == 3: + c = 12 + yield v + + numbers = each([1, 2, 3]) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3]}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 1, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 2, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 3, + 'b': (1, 2, 3), 'c': 12}) + with self.assertRaises(StopAsyncIteration): + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), {}) + + async def test_getasyncgenlocals_empty(self): + async def yield_one(): + yield 1 + one = yield_one() + self.assertEqual(inspect.getasyncgenlocals(one), {}) + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + with self.assertRaises(StopAsyncIteration): + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + + def test_getasyncgenlocals_error(self): + self.assertRaises(TypeError, inspect.getasyncgenlocals, 1) + self.assertRaises(TypeError, inspect.getasyncgenlocals, lambda x: True) + self.assertRaises(TypeError, inspect.getasyncgenlocals, set) + self.assertRaises(TypeError, inspect.getasyncgenlocals, (2,3)) + + +class MySignature(inspect.Signature): + # Top-level to make it picklable; + # used in test_signature_object_pickle + pass + +class MyParameter(inspect.Parameter): + # Top-level to make it picklable; + # used in test_signature_object_pickle + pass + + + +class TestSignatureObject(unittest.TestCase): + @staticmethod + def signature(func, **kw): + sig = inspect.signature(func, **kw) + return (tuple((param.name, + (... if param.default is param.empty else param.default), + (... if param.annotation is param.empty + else param.annotation), + str(param.kind).lower()) + for param in sig.parameters.values()), + (... if sig.return_annotation is sig.empty + else sig.return_annotation)) + + def test_signature_object(self): + S = inspect.Signature + P = inspect.Parameter + + self.assertEqual(str(S()), '()') + self.assertEqual(repr(S().parameters), 'mappingproxy(OrderedDict())') + + def test(po, /, pk, pkd=100, *args, ko, kod=10, **kwargs): + pass + + sig = inspect.signature(test) + self.assertStartsWith(repr(sig), '<Signature') + self.assertTrue('(po, /, pk' in repr(sig)) + + # We need two functions, because it is impossible to represent + # all param kinds in a single one. + def test2(pod=42, /): + pass + + sig2 = inspect.signature(test2) + self.assertStartsWith(repr(sig2), '<Signature') + self.assertTrue('(pod=42, /)' in repr(sig2)) + + po = sig.parameters['po'] + pod = sig2.parameters['pod'] + pk = sig.parameters['pk'] + pkd = sig.parameters['pkd'] + args = sig.parameters['args'] + ko = sig.parameters['ko'] + kod = sig.parameters['kod'] + kwargs = sig.parameters['kwargs'] + + S((po, pk, args, ko, kwargs)) + S((po, pk, ko, kod)) + S((po, pod, ko)) + S((po, pod, kod)) + S((pod, ko, kod)) + S((pod, kod)) + S((pod, args, kod, kwargs)) + # keyword-only parameters without default values + # can follow keyword-only parameters with default values: + S((kod, ko)) + S((kod, ko, kwargs)) + S((args, kod, ko)) + + with self.assertRaisesRegex(ValueError, 'wrong parameter order'): + S((pk, po, args, ko, kwargs)) + + with self.assertRaisesRegex(ValueError, 'wrong parameter order'): + S((po, args, pk, ko, kwargs)) + + with self.assertRaisesRegex(ValueError, 'wrong parameter order'): + S((args, po, pk, ko, kwargs)) + + with self.assertRaisesRegex(ValueError, 'wrong parameter order'): + S((po, pk, args, kwargs, ko)) + + kwargs2 = kwargs.replace(name='args') + with self.assertRaisesRegex(ValueError, 'duplicate parameter name'): + S((po, pk, args, kwargs2, ko)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((pod, po)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((pod, pk)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((po, pod, pk)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((po, pkd, pk)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((pkd, pk)) + + second_args = args.replace(name="second_args") + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, second_args)) + + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, ko, second_args)) + + second_kwargs = kwargs.replace(name="second_kwargs") + with self.assertRaisesRegex(ValueError, 'more than one variadic keyword parameter'): + S((kwargs, second_kwargs)) + + def test_signature_object_pickle(self): + def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass + foo_partial = functools.partial(foo, a=1) + + sig = inspect.signature(foo_partial) + + for ver in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_ver=ver, subclass=False): + sig_pickled = pickle.loads(pickle.dumps(sig, ver)) + self.assertEqual(sig, sig_pickled) + + # Test that basic sub-classing works + sig = inspect.signature(foo) + myparam = MyParameter(name='z', kind=inspect.Parameter.POSITIONAL_ONLY) + myparams = collections.OrderedDict(sig.parameters, a=myparam) + mysig = MySignature().replace(parameters=myparams.values(), + return_annotation=sig.return_annotation) + self.assertTrue(isinstance(mysig, MySignature)) + self.assertTrue(isinstance(mysig.parameters['z'], MyParameter)) + + for ver in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_ver=ver, subclass=True): + sig_pickled = pickle.loads(pickle.dumps(mysig, ver)) + self.assertEqual(mysig, sig_pickled) + self.assertTrue(isinstance(sig_pickled, MySignature)) + self.assertTrue(isinstance(sig_pickled.parameters['z'], + MyParameter)) + + def test_signature_immutability(self): + def test(a): + pass + sig = inspect.signature(test) + + with self.assertRaises(AttributeError): + sig.foo = 'bar' + + with self.assertRaises(TypeError): + sig.parameters['a'] = None + + def test_signature_on_noarg(self): + def test(): + pass + self.assertEqual(self.signature(test), ((), ...)) + + def test_signature_on_wargs(self): + def test(a, b:'foo') -> 123: + pass + self.assertEqual(self.signature(test), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., 'foo', "positional_or_keyword")), + 123)) + + def test_signature_on_wkwonly(self): + def test(*, a:float, b:str) -> int: + pass + self.assertEqual(self.signature(test), + ((('a', ..., float, "keyword_only"), + ('b', ..., str, "keyword_only")), + int)) + + def test_signature_on_complex_args(self): + def test(a, b:'foo'=10, *args:'bar', spam:'baz', ham=123, **kwargs:int): + pass + self.assertEqual(self.signature(test), + ((('a', ..., ..., "positional_or_keyword"), + ('b', 10, 'foo', "positional_or_keyword"), + ('args', ..., 'bar', "var_positional"), + ('spam', ..., 'baz', "keyword_only"), + ('ham', 123, ..., "keyword_only"), + ('kwargs', ..., int, "var_keyword")), + ...)) + + def test_signature_without_self(self): + def test_args_only(*args): # NOQA + pass + + def test_args_kwargs_only(*args, **kwargs): # NOQA + pass + + class A: + @classmethod + def test_classmethod(*args): # NOQA + pass + + @staticmethod + def test_staticmethod(*args): # NOQA + pass + + f1 = functools.partialmethod((test_classmethod), 1) + f2 = functools.partialmethod((test_args_only), 1) + f3 = functools.partialmethod((test_staticmethod), 1) + f4 = functools.partialmethod((test_args_kwargs_only),1) + + self.assertEqual(self.signature(test_args_only), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(test_args_kwargs_only), + ((('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), ...)) + self.assertEqual(self.signature(A.f1), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f2), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f3), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f4), + ((('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), ...)) + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_on_builtins(self): + _testcapi = import_helper.import_module("_testcapi") + + def test_unbound_method(o): + """Use this to test unbound methods (things that should have a self)""" + signature = inspect.signature(o) + self.assertTrue(isinstance(signature, inspect.Signature)) + self.assertEqual(list(signature.parameters.values())[0].name, 'self') + return signature + + def test_callable(o): + """Use this to test bound methods or normal callables (things that don't expect self)""" + signature = inspect.signature(o) + self.assertTrue(isinstance(signature, inspect.Signature)) + if signature.parameters: + self.assertNotEqual(list(signature.parameters.values())[0].name, 'self') + return signature + + signature = test_callable(_testcapi.docstring_with_signature_with_defaults) + def p(name): return signature.parameters[name].default + self.assertEqual(p('s'), 'avocado') + self.assertEqual(p('b'), b'bytes') + self.assertEqual(p('d'), 3.14) + self.assertEqual(p('i'), 35) + self.assertEqual(p('n'), None) + self.assertEqual(p('t'), True) + self.assertEqual(p('f'), False) + self.assertEqual(p('local'), 3) + self.assertEqual(p('sys'), sys.maxsize) + self.assertEqual(p('exp'), sys.maxsize - 1) + + test_callable(object) + + # normal method + # (PyMethodDescr_Type, "method_descriptor") + test_unbound_method(_pickle.Pickler.dump) + d = _pickle.Pickler(io.StringIO()) + test_callable(d.dump) + + # static method + test_callable(bytes.maketrans) + test_callable(b'abc'.maketrans) + + # class method + test_callable(dict.fromkeys) + test_callable({}.fromkeys) + + # wrapper around slot (PyWrapperDescr_Type, "wrapper_descriptor") + test_unbound_method(type.__call__) + test_unbound_method(int.__add__) + test_callable((3).__add__) + + # _PyMethodWrapper_Type + # support for 'method-wrapper' + test_callable(min.__call__) + + # This doesn't work now. + # (We don't have a valid signature for "type" in 3.4) + class ThisWorksNow: + __call__ = type + # TODO: Support type. + self.assertEqual(ThisWorksNow()(1), int) + self.assertEqual(ThisWorksNow()('A', (), {}).__name__, 'A') + with self.assertRaisesRegex(ValueError, "no signature found"): + test_callable(ThisWorksNow()) + + # Regression test for issue #20786 + test_unbound_method(dict.__delitem__) + test_unbound_method(property.__delete__) + + # Regression test for issue #20586 + test_callable(_testcapi.docstring_with_signature_but_no_doc) + + # Regression test for gh-104955 + method = bytearray.__release_buffer__ + sig = test_unbound_method(method) + self.assertEqual(list(sig.parameters), ['self', 'buffer']) + + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_on_decorated_builtins(self): + _testcapi = import_helper.import_module("_testcapi") + func = _testcapi.docstring_with_signature_with_defaults + + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs) -> int: + return func(*args, **kwargs) + return wrapper + + decorated_func = decorator(func) + + self.assertEqual(inspect.signature(func), + inspect.signature(decorated_func)) + + def wrapper_like(*args, **kwargs) -> int: pass + self.assertEqual(inspect.signature(decorated_func, + follow_wrapped=False), + inspect.signature(wrapper_like)) + + @cpython_only + def test_signature_on_builtins_no_signature(self): + _testcapi = import_helper.import_module("_testcapi") + with self.assertRaisesRegex(ValueError, + 'no signature found for builtin'): + inspect.signature(_testcapi.docstring_no_signature) + + with self.assertRaisesRegex(ValueError, + 'no signature found for builtin'): + inspect.signature(str) + + cls = _testcapi.DocStringNoSignatureTest + obj = _testcapi.DocStringNoSignatureTest() + tests = [ + (_testcapi.docstring_no_signature_noargs, meth_noargs), + (_testcapi.docstring_no_signature_o, meth_o), + (cls.meth_noargs, meth_self_noargs), + (cls.meth_o, meth_self_o), + (obj.meth_noargs, meth_noargs), + (obj.meth_o, meth_o), + (cls.meth_noargs_class, meth_noargs), + (cls.meth_o_class, meth_o), + (cls.meth_noargs_static, meth_noargs), + (cls.meth_o_static, meth_o), + (cls.meth_noargs_coexist, meth_self_noargs), + (cls.meth_o_coexist, meth_self_o), + + (time.time, meth_noargs), + (str.lower, meth_self_noargs), + (''.lower, meth_noargs), + (set.add, meth_self_o), + (set().add, meth_o), + (set.__contains__, meth_self_o), + (set().__contains__, meth_o), + (datetime.datetime.__dict__['utcnow'], meth_type_noargs), + (datetime.datetime.utcnow, meth_noargs), + (dict.__dict__['__class_getitem__'], meth_type_o), + (dict.__class_getitem__, meth_o), + ] + try: + import _stat # noqa: F401 + except ImportError: + # if the _stat extension is not available, stat.S_IMODE() is + # implemented in Python, not in C + pass + else: + tests.append((stat.S_IMODE, meth_o)) + for builtin, template in tests: + with self.subTest(builtin): + self.assertEqual(inspect.signature(builtin), + inspect.signature(template)) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_parsing_with_defaults(self): + _testcapi = import_helper.import_module("_testcapi") + meth = _testcapi.DocStringUnrepresentableSignatureTest.with_default + self.assertEqual(str(inspect.signature(meth)), '(self, /, x=1)') + + def test_signature_on_non_function(self): + with self.assertRaisesRegex(TypeError, 'is not a callable object'): + inspect.signature(42) + + def test_signature_from_functionlike_object(self): + def func(a,b, *args, kwonly=True, kwonlyreq, **kwargs): + pass + + class funclike: + # Has to be callable, and have correct + # __code__, __annotations__, __defaults__, __name__, + # and __kwdefaults__ attributes + + def __init__(self, func): + self.__name__ = func.__name__ + self.__code__ = func.__code__ + self.__annotations__ = func.__annotations__ + self.__defaults__ = func.__defaults__ + self.__kwdefaults__ = func.__kwdefaults__ + self.func = func + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + sig_func = inspect.Signature.from_callable(func) + + sig_funclike = inspect.Signature.from_callable(funclike(func)) + self.assertEqual(sig_funclike, sig_func) + + sig_funclike = inspect.signature(funclike(func)) + self.assertEqual(sig_funclike, sig_func) + + # If object is not a duck type of function, then + # signature will try to get a signature for its '__call__' + # method + fl = funclike(func) + del fl.__defaults__ + self.assertEqual(self.signature(fl), + ((('args', ..., ..., "var_positional"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # Test with cython-like builtins: + _orig_isdesc = inspect.ismethoddescriptor + def _isdesc(obj): + if hasattr(obj, '_builtinmock'): + return True + return _orig_isdesc(obj) + + with unittest.mock.patch('inspect.ismethoddescriptor', _isdesc): + builtin_func = funclike(func) + # Make sure that our mock setup is working + self.assertFalse(inspect.ismethoddescriptor(builtin_func)) + builtin_func._builtinmock = True + self.assertTrue(inspect.ismethoddescriptor(builtin_func)) + self.assertEqual(inspect.signature(builtin_func), sig_func) + + def test_signature_functionlike_class(self): + # We only want to duck type function-like objects, + # not classes. + + def func(a,b, *args, kwonly=True, kwonlyreq, **kwargs): + pass + + class funclike: + def __init__(self, marker): + pass + + __name__ = func.__name__ + __code__ = func.__code__ + __annotations__ = func.__annotations__ + __defaults__ = func.__defaults__ + __kwdefaults__ = func.__kwdefaults__ + + self.assertEqual(str(inspect.signature(funclike)), '(marker)') + + def test_signature_on_method(self): + class Test: + def __init__(*args): + pass + def m1(self, arg1, arg2=1) -> int: + pass + def m2(*args): + pass + def __call__(*, a): + pass + + self.assertEqual(self.signature(Test().m1), + ((('arg1', ..., ..., "positional_or_keyword"), + ('arg2', 1, ..., "positional_or_keyword")), + int)) + + self.assertEqual(self.signature(Test().m2), + ((('args', ..., ..., "var_positional"),), + ...)) + + self.assertEqual(self.signature(Test), + ((('args', ..., ..., "var_positional"),), + ...)) + + with self.assertRaisesRegex(ValueError, 'invalid method signature'): + self.signature(Test()) + + def test_signature_wrapped_bound_method(self): + # Issue 24298 + class Test: + def m1(self, arg1, arg2=1) -> int: + pass + @functools.wraps(Test().m1) + def m1d(*args, **kwargs): + pass + self.assertEqual(self.signature(m1d), + ((('arg1', ..., ..., "positional_or_keyword"), + ('arg2', 1, ..., "positional_or_keyword")), + int)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: no signature found for builtin type <class 'classmethod'> + def test_signature_on_classmethod(self): + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(classmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + + class Test: + @classmethod + def foo(cls, arg1, *, arg2=1): + pass + + meth = Test().foo + self.assertEqual(self.signature(meth), + ((('arg1', ..., ..., "positional_or_keyword"), + ('arg2', 1, ..., "keyword_only")), + ...)) + + meth = Test.foo + self.assertEqual(self.signature(meth), + ((('arg1', ..., ..., "positional_or_keyword"), + ('arg2', 1, ..., "keyword_only")), + ...)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: no signature found for builtin type <class 'staticmethod'> + def test_signature_on_staticmethod(self): + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(staticmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + + class Test: + @staticmethod + def foo(cls, *, arg): + pass + + meth = Test().foo + self.assertEqual(self.signature(meth), + ((('cls', ..., ..., "positional_or_keyword"), + ('arg', ..., ..., "keyword_only")), + ...)) + + meth = Test.foo + self.assertEqual(self.signature(meth), + ((('cls', ..., ..., "positional_or_keyword"), + ('arg', ..., ..., "keyword_only")), + ...)) + + def test_signature_on_partial(self): + from functools import partial, Placeholder + + def test(): + pass + + self.assertEqual(self.signature(partial(test)), ((), ...)) + + with self.assertRaisesRegex(ValueError, "has incorrect arguments"): + inspect.signature(partial(test, 1)) + + with self.assertRaisesRegex(ValueError, "has incorrect arguments"): + inspect.signature(partial(test, a=1)) + + def test(a, b, *, c, d): + pass + + self.assertEqual(self.signature(partial(test)), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, 1)), + ((('b', ..., ..., "positional_or_keyword"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, 1, c=2)), + ((('b', ..., ..., "positional_or_keyword"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, b=1, c=2)), + ((('a', ..., ..., "positional_or_keyword"), + ('b', 1, ..., "keyword_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, 0, b=1, c=2)), + ((('b', 1, ..., "keyword_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, a=1)), + ((('a', 1, ..., "keyword_only"), + ('b', ..., ..., "keyword_only"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + # With Placeholder + self.assertEqual(self.signature(partial(test, Placeholder, 1)), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)), + ((('a', ..., ..., "positional_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + # Ensure unittest.mock.ANY & similar do not get picked up as a Placeholder + self.assertEqual(self.signature(partial(test, unittest.mock.ANY, 1, c=2)), + ((('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + def test(a, *args, b, **kwargs): + pass + + self.assertEqual(self.signature(partial(test, 1)), + ((('args', ..., ..., "var_positional"), + ('b', ..., ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, a=1)), + ((('a', 1, ..., "keyword_only"), + ('b', ..., ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, 1, 2, 3)), + ((('args', ..., ..., "var_positional"), + ('b', ..., ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, 1, 2, 3, test=True)), + ((('args', ..., ..., "var_positional"), + ('b', ..., ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, 1, 2, 3, test=1, b=0)), + ((('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, b=0)), + ((('a', ..., ..., "positional_or_keyword"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, b=0, test=1)), + ((('a', ..., ..., "positional_or_keyword"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # With Placeholder + p = partial(test, Placeholder, Placeholder, 1, b=0, test=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + def test(a, b, c:int) -> 42: + pass + + sig = test.__signature__ = inspect.signature(test) + + self.assertEqual(self.signature(partial(partial(test, 1))), + ((('b', ..., ..., "positional_or_keyword"), + ('c', ..., int, "positional_or_keyword")), + 42)) + + self.assertEqual(self.signature(partial(partial(test, 1), 2)), + ((('c', ..., int, "positional_or_keyword"),), + 42)) + + def foo(a): + return a + _foo = partial(partial(foo, a=10), a=20) + self.assertEqual(self.signature(_foo), + ((('a', 20, ..., "keyword_only"),), + ...)) + # check that we don't have any side-effects in signature(), + # and the partial object is still functioning + self.assertEqual(_foo(), 20) + + def foo(a, b, c): + return a, b, c + _foo = partial(partial(foo, 1, b=20), b=30) + + self.assertEqual(self.signature(_foo), + ((('b', 30, ..., "keyword_only"), + ('c', ..., ..., "keyword_only")), + ...)) + self.assertEqual(_foo(c=10), (1, 30, 10)) + + def foo(a, b, c, *, d): + return a, b, c, d + _foo = partial(partial(foo, d=20, c=20), b=10, d=30) + self.assertEqual(self.signature(_foo), + ((('a', ..., ..., "positional_or_keyword"), + ('b', 10, ..., "keyword_only"), + ('c', 20, ..., "keyword_only"), + ('d', 30, ..., "keyword_only"), + ), + ...)) + ba = inspect.signature(_foo).bind(a=200, b=11) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (200, 11, 20, 30)) + + def foo(a=1, b=2, c=3): + return a, b, c + _foo = partial(foo, c=13) # (a=1, b=2, *, c=13) + + ba = inspect.signature(_foo).bind(a=11) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 2, 13)) + + ba = inspect.signature(_foo).bind(11, 12) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 12, 13)) + + ba = inspect.signature(_foo).bind(11, b=12) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 12, 13)) + + ba = inspect.signature(_foo).bind(b=12) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (1, 12, 13)) + + _foo = partial(_foo, b=10, c=20) + ba = inspect.signature(_foo).bind(12) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (12, 10, 20)) + + + def foo(a, b, /, c, d, **kwargs): + pass + sig = inspect.signature(foo) + self.assertEqual(str(sig), '(a, b, /, c, d, **kwargs)') + + self.assertEqual(self.signature(partial(foo, 1)), + ((('b', ..., ..., 'positional_only'), + ('c', ..., ..., 'positional_or_keyword'), + ('d', ..., ..., 'positional_or_keyword'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + self.assertEqual(self.signature(partial(foo, 1, 2)), + ((('c', ..., ..., 'positional_or_keyword'), + ('d', ..., ..., 'positional_or_keyword'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + self.assertEqual(self.signature(partial(foo, 1, 2, 3)), + ((('d', ..., ..., 'positional_or_keyword'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + self.assertEqual(self.signature(partial(foo, 1, 2, c=3)), + ((('c', 3, ..., 'keyword_only'), + ('d', ..., ..., 'keyword_only'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + self.assertEqual(self.signature(partial(foo, 1, c=3)), + ((('b', ..., ..., 'positional_only'), + ('c', 3, ..., 'keyword_only'), + ('d', ..., ..., 'keyword_only'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + # Positional only With Placeholder + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # Optionals Positional With Placeholder + def foo(a=0, b=1, /, c=2, d=3): + pass + + # Positional + p = partial(foo, Placeholder, 1, c=0, d=1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', 0, ..., "keyword_only"), + ('d', 1, ..., "keyword_only")), + ...)) + + # Positional or Keyword - transformed to positional + p = partial(foo, Placeholder, 1, Placeholder, 1) + self.assertEqual(self.signature(p), + ((('a', ..., ..., "positional_only"), + ('c', ..., ..., "positional_only")), + ...)) + + def test_signature_on_partialmethod(self): + from functools import partialmethod + + class Spam: + def test(): + pass + ham = partialmethod(test) + + with self.assertRaisesRegex(ValueError, "has incorrect arguments"): + inspect.signature(Spam.ham) + + class Spam: + def test(it, a, b, *, c) -> 'spam': + pass + ham = partialmethod(test, c=1) + bar = partialmethod(test, functools.Placeholder, 1, c=1) + + self.assertEqual(self.signature(Spam.ham, eval_str=False), + ((('it', ..., ..., 'positional_or_keyword'), + ('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + self.assertEqual(self.signature(Spam().ham, eval_str=False), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + # With Placeholder + self.assertEqual(self.signature(Spam.bar, eval_str=False), + ((('it', ..., ..., 'positional_only'), + ('a', ..., ..., 'positional_only'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + self.assertEqual(self.signature(Spam().bar, eval_str=False), + ((('a', ..., ..., 'positional_only'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + class Spam: + def test(self: 'anno', x): + pass + + g = partialmethod(test, 1) + + self.assertEqual(self.signature(Spam.g, eval_str=False), + ((('self', ..., 'anno', 'positional_or_keyword'),), + ...)) + + def test_signature_on_fake_partialmethod(self): + def foo(a): pass + foo.__partialmethod__ = 'spam' + self.assertEqual(str(inspect.signature(foo)), '(a)') + + def test_signature_on_decorated(self): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs) -> int: + return func(*args, **kwargs) + return wrapper + + class Foo: + @decorator + def bar(self, a, b): + pass + + bar = decorator(Foo().bar) + + self.assertEqual(self.signature(Foo.bar), + ((('self', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(Foo().bar), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(Foo.bar, follow_wrapped=False), + ((('args', ..., ..., "var_positional"), + ('kwargs', ..., ..., "var_keyword")), + ...)) # functools.wraps will copy __annotations__ + # from "func" to "wrapper", hence no + # return_annotation + + self.assertEqual(self.signature(bar), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + # Test that we handle method wrappers correctly + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs) -> int: + return func(42, *args, **kwargs) + sig = inspect.signature(func) + new_params = tuple(sig.parameters.values())[1:] + wrapper.__signature__ = sig.replace(parameters=new_params) + return wrapper + + class Foo: + @decorator + def __call__(self, a, b): + pass + + self.assertEqual(self.signature(Foo.__call__), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(Foo().__call__), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + # Test we handle __signature__ partway down the wrapper stack + def wrapped_foo_call(): + pass + wrapped_foo_call.__wrapped__ = Foo.__call__ + + self.assertEqual(self.signature(wrapped_foo_call), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_signature_on_class(self): + class C: + def __init__(self, a): + pass + + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + class CM(type): + def __call__(cls, a): + pass + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class CM(type): + @classmethod + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class CM(type): + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) + class C(metaclass=CM): + def __init__(self, c): + pass + + self.assertEqual(C(1), (2, C, 1)) + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class CM(type): + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class CM(type): + __call__ = ':'.join + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(['a', 'bc']), 'a:bc') + # BUG: Returns '<Signature (b)>' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class CM(type): + __call__ = (2).__pow__ + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(3), 8) + self.assertEqual(C(3, 7), 1) + if not support.MISSING_C_DOCSTRINGS: + # BUG: Returns '<Signature (b)>' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature((0).__pow__)) + + class CM(type): + def __new__(mcls, name, bases, dct, *, foo=1): + return super().__new__(mcls, name, bases, dct) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + self.assertEqual(self.signature(CM), + ((('name', ..., ..., "positional_or_keyword"), + ('bases', ..., ..., "positional_or_keyword"), + ('dct', ..., ..., "positional_or_keyword"), + ('foo', 1, ..., "keyword_only")), + ...)) + + class CMM(type): + def __new__(mcls, name, bases, dct, *, foo=1): + return super().__new__(mcls, name, bases, dct) + def __call__(cls, nm, bs, dt): + return type(nm, bs, dt) + class CM(type, metaclass=CMM): + def __new__(mcls, name, bases, dct, *, bar=2): + return super().__new__(mcls, name, bases, dct) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(CMM), + ((('name', ..., ..., "positional_or_keyword"), + ('bases', ..., ..., "positional_or_keyword"), + ('dct', ..., ..., "positional_or_keyword"), + ('foo', 1, ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(CM), + ((('nm', ..., ..., "positional_or_keyword"), + ('bs', ..., ..., "positional_or_keyword"), + ('dt', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + class CM(type): + def __init__(cls, name, bases, dct, *, bar=2): + return super().__init__(name, bases, dct) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(CM), + ((('name', ..., ..., "positional_or_keyword"), + ('bases', ..., ..., "positional_or_keyword"), + ('dct', ..., ..., "positional_or_keyword"), + ('bar', 2, ..., "keyword_only")), + ...)) + + def test_signature_on_class_with_wrapped_metaclass_call(self): + class CM(type): + @identity_wrapper + def __call__(cls, a): + pass + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class CM(type): + @classmethod + @identity_wrapper + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + @identity_wrapper + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class CM(type): + @custom_descriptor + @identity_wrapper + def __call__(self, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__call__), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + self.assertEqual(self.signature(C.__call__, follow_wrapped=False), + varargs_signature) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_signature_on_class_with_wrapped_init(self): + class C: + @identity_wrapper + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + @identity_wrapper + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + @identity_wrapper + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, a): + pass + + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(identity_wrapper(lambda x, a, b: None), 2) + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + @identity_wrapper + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class C: + @custom_descriptor + @identity_wrapper + def __init__(self, a): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__init__), + ((('self', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + if support.MISSING_C_DOCSTRINGS: + self.assertRaisesRegex( + ValueError, "no signature found", + self.signature, C.__new__, follow_wrapped=False, + ) + else: + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_signature_on_class_with_wrapped_new(self): + with self.subTest('FunctionType'): + class C: + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + @identity_wrapper + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(identity_wrapper(lambda x, cls, a: (x, a)), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(identity_wrapper(lambda cls, x, a: (x, a)), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class C: + @custom_descriptor + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__new__), + ((('cls', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) + + def test_signature_on_class_with_init(self): + class C: + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + pass + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(lambda x, a, b: None, 2) + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_signature_on_class_with_new(self): + with self.subTest('FunctionType'): + class C: + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(lambda x, cls, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(lambda cls, x, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __new__ = str.__subclasscheck__ + + self.assertEqual(C(), False) + # TODO: Support BuiltinMethodType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + with self.subTest('MethodWrapperType'): + class C: + __new__ = type.__or__.__get__(int, type) + + self.assertEqual(C(), C | int) + # TODO: Support MethodWrapperType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + # TODO: Test ClassMethodDescriptorType + + with self.subTest('MethodDescriptorType'): + class C: + __new__ = type.__dict__['__subclasscheck__'] + + self.assertEqual(C(C), True) + self.assertEqual(self.signature(C), self.signature(C.__subclasscheck__)) + + with self.subTest('WrapperDescriptorType'): + class C: + __new__ = type.__or__ + + self.assertEqual(C(int), C | int) + # TODO: Support WrapperDescriptorType + # self.assertEqual(self.signature(C), self.signature(C.__or__)) + self.assertRaises(ValueError, self.signature, C) + + def test_signature_on_subclass(self): + class A: + def __new__(cls, a=1, *args, **kwargs): + return object.__new__(cls) + class B(A): + def __init__(self, b): + pass + class C(A): + def __new__(cls, a=1, b=2, *args, **kwargs): + return object.__new__(cls) + class D(A): + pass + + self.assertEqual(self.signature(B), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C), + ((('a', 1, ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword'), + ('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + self.assertEqual(self.signature(D), + ((('a', 1, ..., 'positional_or_keyword'), + ('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + def test_signature_on_generic_subclass(self): + from typing import Generic, TypeVar + + T = TypeVar('T') + + class A(Generic[T]): + def __init__(self, *, a: int) -> None: + pass + + self.assertEqual(self.signature(A), + ((('a', ..., int, 'keyword_only'),), + None)) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_on_class_without_init(self): + # Test classes without user-defined __init__ or __new__ + class C: pass + self.assertEqual(str(inspect.signature(C)), '()') + class D(C): pass + self.assertEqual(str(inspect.signature(D)), '()') + + # Test meta-classes without user-defined __init__ or __new__ + class C(type): pass + class D(C): pass + self.assertEqual(C('A', (), {}).__name__, 'A') + # TODO: Support type. + with self.assertRaisesRegex(ValueError, "callable.*is not supported"): + self.assertEqual(inspect.signature(C), None) + self.assertEqual(D('A', (), {}).__name__, 'A') + with self.assertRaisesRegex(ValueError, "callable.*is not supported"): + self.assertEqual(inspect.signature(D), None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name '_pickle' is not defined. Did you mean: 'pickle'? Or did you forget to import '_pickle'? + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_on_builtin_class(self): + expected = ('(file, protocol=None, fix_imports=True, ' + 'buffer_callback=None)') + self.assertEqual(str(inspect.signature(_pickle.Pickler)), expected) + + class P(_pickle.Pickler): pass + class EmptyTrait: pass + class P2(EmptyTrait, P): pass + self.assertEqual(str(inspect.signature(P)), expected) + self.assertEqual(str(inspect.signature(P2)), expected) + + class P3(P2): + def __init__(self, spam): + pass + self.assertEqual(str(inspect.signature(P3)), '(spam)') + + class MetaP(type): + def __call__(cls, foo, bar): + pass + class P4(P2, metaclass=MetaP): + pass + self.assertEqual(str(inspect.signature(P4)), '(foo, bar)') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_signature_on_callable_objects(self): + class Foo: + def __call__(self, a): + pass + + self.assertEqual(self.signature(Foo()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + class Spam: + pass + with self.assertRaisesRegex(TypeError, "is not a callable object"): + inspect.signature(Spam()) + + class Bar(Spam, Foo): + pass + + self.assertEqual(self.signature(Bar()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __call__(cls, a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __call__(a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class C: + __call__ = A().call + + self.assertEqual(C()(1), 1) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __call__ = functools.partial(lambda x, a, b: (x, a, b), 2) + + c = C() + self.assertEqual(c(1), (2, c, 1)) + self.assertEqual(self.signature(C()), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + + self.assertEqual(C()(1), (2, 1)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __call__ = ':'.join + + self.assertEqual(C()(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class C: + __call__ = (2).__pow__ + + self.assertEqual(C()(3), 8) + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('ClassMethodDescriptorType'): + class C(dict): + __call__ = dict.__dict__['fromkeys'] + + res = C()([1, 2], 3) + self.assertEqual(res, {1: 3, 2: 3}) + self.assertEqual(type(res), C) + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(C()), self.signature(dict.fromkeys)) + + with self.subTest('MethodDescriptorType'): + class C(str): + __call__ = str.join + + self.assertEqual(C(':')(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('WrapperDescriptorType'): + class C(int): + __call__ = int.__pow__ + + self.assertEqual(C(2)(3), 8) + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('MemberDescriptorType'): + class C: + __slots__ = '__call__' + c = C() + c.__call__ = lambda a: a + self.assertEqual(c(1), 1) + self.assertEqual(self.signature(c), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_callable_objects_with_text_signature_attr(self): + class C: + __text_signature__ = '(a, /, b, c=True)' + def __call__(self, *args, **kwargs): + pass + + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(C), ((), ...)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_only"), + ('b', ..., ..., "positional_or_keyword"), + ('c', True, ..., "positional_or_keyword"), + ), + ...)) + + c = C() + c.__text_signature__ = '(x, y)' + self.assertEqual(self.signature(c), + ((('x', ..., ..., "positional_or_keyword"), + ('y', ..., ..., "positional_or_keyword"), + ), + ...)) + + def test_signature_on_wrapper(self): + class Wrapper: + def __call__(self, b): + pass + wrapper = Wrapper() + wrapper.__wrapped__ = lambda a: None + self.assertEqual(self.signature(wrapper), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + # wrapper loop: + wrapper = Wrapper() + wrapper.__wrapped__ = wrapper + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + self.signature(wrapper) + + def test_signature_on_lambdas(self): + self.assertEqual(self.signature((lambda a=10: a)), + ((('a', 10, ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_mocks(self): + # https://github.com/python/cpython/issues/96127 + for mock in ( + unittest.mock.Mock(), + unittest.mock.AsyncMock(), + unittest.mock.MagicMock(), + ): + with self.subTest(mock=mock): + self.assertEqual(str(inspect.signature(mock)), '(*args, **kwargs)') + + def test_signature_on_noncallable_mocks(self): + for mock in ( + unittest.mock.NonCallableMock(), + unittest.mock.NonCallableMagicMock(), + ): + with self.subTest(mock=mock): + with self.assertRaises(TypeError): + inspect.signature(mock) + + def test_signature_equality(self): + def foo(a, *, b:int) -> float: pass + self.assertFalse(inspect.signature(foo) == 42) + self.assertTrue(inspect.signature(foo) != 42) + self.assertTrue(inspect.signature(foo) == ALWAYS_EQ) + self.assertFalse(inspect.signature(foo) != ALWAYS_EQ) + + def bar(a, *, b:int) -> float: pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, *, b:int) -> int: pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, *, b:int): pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, *, b:int=42) -> float: pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, *, c) -> float: pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, b:int) -> float: pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + def spam(b:int, a) -> float: pass + self.assertFalse(inspect.signature(spam) == inspect.signature(bar)) + self.assertTrue(inspect.signature(spam) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(spam)), hash(inspect.signature(bar))) + + def foo(*, a, b, c): pass + def bar(*, c, b, a): pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def foo(*, a=1, b, c): pass + def bar(*, c, b, a=1): pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def foo(pos, *, a=1, b, c): pass + def bar(pos, *, c, b, a=1): pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def foo(pos, *, a, b, c): pass + def bar(pos, *, c, b, a=1): pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def foo(pos, *args, a=42, b, c, **kwargs:int): pass + def bar(pos, *args, c, b, a=42, **kwargs:int): pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def test_signature_hashable(self): + S = inspect.Signature + P = inspect.Parameter + + def foo(a): pass + foo_sig = inspect.signature(foo) + + manual_sig = S(parameters=[P('a', P.POSITIONAL_OR_KEYWORD)]) + + self.assertEqual(hash(foo_sig), hash(manual_sig)) + self.assertNotEqual(hash(foo_sig), + hash(manual_sig.replace(return_annotation='spam'))) + + def bar(a) -> 1: pass + self.assertNotEqual(hash(foo_sig), hash(inspect.signature(bar))) + + def foo(a={}): pass + with self.assertRaisesRegex(TypeError, 'unhashable type'): + hash(inspect.signature(foo)) + + def foo(a) -> {}: pass + with self.assertRaisesRegex(TypeError, 'unhashable type'): + hash(inspect.signature(foo)) + + def test_signature_str(self): + def foo(a:int=1, *, b, c=None, **kwargs) -> 42: + pass + self.assertEqual(str(inspect.signature(foo)), + '(a: int = 1, *, b, c=None, **kwargs) -> 42') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + def foo(a:int=1, *args, b, c=None, **kwargs) -> 42: + pass + self.assertEqual(str(inspect.signature(foo)), + '(a: int = 1, *args, b, c=None, **kwargs) -> 42') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + def foo(): + pass + self.assertEqual(str(inspect.signature(foo)), '()') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + def foo(a: list[str]) -> tuple[str, float]: + pass + self.assertEqual(str(inspect.signature(foo)), + '(a: list[str]) -> tuple[str, float]') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + from typing import Tuple + def foo(a: list[str]) -> Tuple[str, float]: + pass + self.assertEqual(str(inspect.signature(foo)), + '(a: list[str]) -> Tuple[str, float]') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + def foo(x: undef): + pass + sig = inspect.signature(foo, annotation_format=Format.FORWARDREF) + self.assertEqual(str(sig), '(x: undef)') + + def test_signature_str_positional_only(self): + P = inspect.Parameter + S = inspect.Signature + + def test(a_po, /, *, b, **kwargs): + return a_po, kwargs + + self.assertEqual(str(inspect.signature(test)), + '(a_po, /, *, b, **kwargs)') + self.assertEqual(str(inspect.signature(test)), + inspect.signature(test).format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY)]) + self.assertEqual(str(test), '(foo, /)') + self.assertEqual(str(test), test.format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY), + P('bar', P.VAR_KEYWORD)]) + self.assertEqual(str(test), '(foo, /, **bar)') + self.assertEqual(str(test), test.format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY), + P('bar', P.VAR_POSITIONAL)]) + self.assertEqual(str(test), '(foo, /, *bar)') + self.assertEqual(str(test), test.format()) + + def test_signature_format(self): + from typing import Annotated, Literal + + def func(x: Annotated[int, 'meta'], y: Literal['a', 'b'], z: 'LiteralString'): + pass + + expected_singleline = "(x: Annotated[int, 'meta'], y: Literal['a', 'b'], z: 'LiteralString')" + expected_multiline = """( + x: Annotated[int, 'meta'], + y: Literal['a', 'b'], + z: 'LiteralString' +)""" + self.assertEqual( + inspect.signature(func).format(), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=None), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=len(expected_singleline)), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=len(expected_singleline) - 1), + expected_multiline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=0), + expected_multiline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=-1), + expected_multiline, + ) + + def test_signature_format_all_arg_types(self): + from typing import Annotated, Literal + + def func( + x: Annotated[int, 'meta'], + /, + y: Literal['a', 'b'], + *, + z: 'LiteralString', + **kwargs: object, + ) -> None: + pass + + expected_multiline = """( + x: Annotated[int, 'meta'], + /, + y: Literal['a', 'b'], + *, + z: 'LiteralString', + **kwargs: object +) -> None""" + self.assertEqual( + inspect.signature(func).format(max_width=-1), + expected_multiline, + ) + + def test_signature_format_unquote(self): + def func(x: 'int') -> 'str': ... + + self.assertEqual( + inspect.signature(func).format(), + "(x: 'int') -> 'str'" + ) + self.assertEqual( + inspect.signature(func).format(quote_annotation_strings=False), + "(x: int) -> str" + ) + + def test_signature_replace_parameters(self): + def test(a, b) -> 42: + pass + + sig = inspect.signature(test) + parameters = sig.parameters + sig = sig.replace(parameters=list(parameters.values())[1:]) + self.assertEqual(list(sig.parameters), ['b']) + self.assertEqual(sig.parameters['b'], parameters['b']) + self.assertEqual(sig.return_annotation, 42) + sig = sig.replace(parameters=()) + self.assertEqual(dict(sig.parameters), {}) + + sig = inspect.signature(test) + parameters = sig.parameters + sig = copy.replace(sig, parameters=list(parameters.values())[1:]) + self.assertEqual(list(sig.parameters), ['b']) + self.assertEqual(sig.parameters['b'], parameters['b']) + self.assertEqual(sig.return_annotation, 42) + sig = copy.replace(sig, parameters=()) + self.assertEqual(dict(sig.parameters), {}) + + def test_signature_replace_anno(self): + def test() -> 42: + pass + + sig = inspect.signature(test) + sig = sig.replace(return_annotation=None) + self.assertIs(sig.return_annotation, None) + sig = sig.replace(return_annotation=sig.empty) + self.assertIs(sig.return_annotation, sig.empty) + sig = sig.replace(return_annotation=42) + self.assertEqual(sig.return_annotation, 42) + self.assertEqual(sig, inspect.signature(test)) + + sig = inspect.signature(test) + sig = copy.replace(sig, return_annotation=None) + self.assertIs(sig.return_annotation, None) + sig = copy.replace(sig, return_annotation=sig.empty) + self.assertIs(sig.return_annotation, sig.empty) + sig = copy.replace(sig, return_annotation=42) + self.assertEqual(sig.return_annotation, 42) + self.assertEqual(sig, inspect.signature(test)) + + def test_signature_replaced(self): + def test(): + pass + + spam_param = inspect.Parameter('spam', inspect.Parameter.POSITIONAL_ONLY) + sig = test.__signature__ = inspect.Signature(parameters=(spam_param,)) + self.assertEqual(sig, inspect.signature(test)) + + def test_signature_on_mangled_parameters(self): + class Spam: + def foo(self, __p1:1=2, *, __p2:2=3): + pass + class Ham(Spam): + pass + + self.assertEqual(self.signature(Spam.foo), + ((('self', ..., ..., "positional_or_keyword"), + ('_Spam__p1', 2, 1, "positional_or_keyword"), + ('_Spam__p2', 3, 2, "keyword_only")), + ...)) + + self.assertEqual(self.signature(Spam.foo), + self.signature(Ham.foo)) + + def test_signature_from_callable_python_obj(self): + class MySignature(inspect.Signature): pass + def foo(a, *, b:1): pass + foo_sig = MySignature.from_callable(foo) + self.assertIsInstance(foo_sig, MySignature) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_from_callable_class(self): + # A regression test for a class inheriting its signature from `object`. + class MySignature(inspect.Signature): pass + class foo: pass + foo_sig = MySignature.from_callable(foo) + self.assertIsInstance(foo_sig, MySignature) + + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name '_pickle' is not defined. Did you mean: 'pickle'? Or did you forget to import '_pickle'? + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_from_callable_builtin_obj(self): + class MySignature(inspect.Signature): pass + sig = MySignature.from_callable(_pickle.Pickler) + self.assertIsInstance(sig, MySignature) + + def test_signature_definition_order_preserved_on_kwonly(self): + for fn in signatures_with_lexicographic_keyword_only_parameters(): + signature = inspect.signature(fn) + l = list(signature.parameters) + sorted_l = sorted(l) + self.assertTrue(l) + self.assertEqual(l, sorted_l) + signature = inspect.signature(unsorted_keyword_only_parameters_fn) + l = list(signature.parameters) + self.assertEqual(l, unsorted_keyword_only_parameters) + + def test_signater_parameters_is_ordered(self): + p1 = inspect.signature(lambda x, y: None).parameters + p2 = inspect.signature(lambda y, x: None).parameters + self.assertNotEqual(p1, p2) + + def test_signature_annotations_with_local_namespaces(self): + class Foo: ... + def func(foo: Foo) -> int: pass + def func2(foo: Foo, bar: 'Bar') -> int: pass + + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func = signature_func): + sig1 = signature_func(func) + self.assertEqual(sig1.return_annotation, int) + self.assertEqual(sig1.parameters['foo'].annotation, Foo) + + sig2 = signature_func(func, locals=locals()) + self.assertEqual(sig2.return_annotation, int) + self.assertEqual(sig2.parameters['foo'].annotation, Foo) + + sig3 = signature_func(func2, globals={'Bar': int}, locals=locals()) + self.assertEqual(sig3.return_annotation, int) + self.assertEqual(sig3.parameters['foo'].annotation, Foo) + self.assertEqual(sig3.parameters['bar'].annotation, 'Bar') + + def test_signature_eval_str(self): + isa = inspect_stringized_annotations + sig = inspect.Signature + par = inspect.Parameter + PORK = inspect.Parameter.POSITIONAL_OR_KEYWORD + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func = signature_func): + self.assertEqual( + signature_func(isa.MyClass), + sig( + parameters=( + par('a', PORK), + par('b', PORK), + ))) + self.assertEqual( + signature_func(isa.function), + sig( + return_annotation='MyClass', + parameters=( + par('a', PORK, annotation='int'), + par('b', PORK, annotation='str'), + ))) + self.assertEqual( + signature_func(isa.function2), + sig( + return_annotation='MyClass', + parameters=( + par('a', PORK, annotation='int'), + par('b', PORK, annotation="'str'"), + par('c', PORK, annotation="MyClass"), + ))) + self.assertEqual( + signature_func(isa.function3), + sig( + parameters=( + par('a', PORK, annotation="'int'"), + par('b', PORK, annotation="'str'"), + par('c', PORK, annotation="'MyClass'"), + ))) + + if not MISSING_C_DOCSTRINGS: + self.assertEqual(signature_func(isa.UnannotatedClass), sig()) + self.assertEqual(signature_func(isa.unannotated_function), + sig( + parameters=( + par('a', PORK), + par('b', PORK), + par('c', PORK), + ))) + + self.assertEqual( + signature_func(isa.MyClass, eval_str=True), + sig( + parameters=( + par('a', PORK), + par('b', PORK), + ))) + self.assertEqual( + signature_func(isa.function, eval_str=True), + sig( + return_annotation=isa.MyClass, + parameters=( + par('a', PORK, annotation=int), + par('b', PORK, annotation=str), + ))) + self.assertEqual( + signature_func(isa.function2, eval_str=True), + sig( + return_annotation=isa.MyClass, + parameters=( + par('a', PORK, annotation=int), + par('b', PORK, annotation='str'), + par('c', PORK, annotation=isa.MyClass), + ))) + self.assertEqual( + signature_func(isa.function3, eval_str=True), + sig( + parameters=( + par('a', PORK, annotation='int'), + par('b', PORK, annotation='str'), + par('c', PORK, annotation='MyClass'), + ))) + + globalns = {'int': float, 'str': complex} + localns = {'str': tuple, 'MyClass': dict} + with self.assertRaises(NameError): + signature_func(isa.function, eval_str=True, globals=globalns) + + self.assertEqual( + signature_func(isa.function, eval_str=True, locals=localns), + sig( + return_annotation=dict, + parameters=( + par('a', PORK, annotation=int), + par('b', PORK, annotation=tuple), + ))) + + self.assertEqual( + signature_func(isa.function, eval_str=True, globals=globalns, locals=localns), + sig( + return_annotation=dict, + parameters=( + par('a', PORK, annotation=float), + par('b', PORK, annotation=tuple), + ))) + + def test_signature_annotation_format(self): + ida = inspect_deferred_annotations + sig = inspect.Signature + par = inspect.Parameter + PORK = inspect.Parameter.POSITIONAL_OR_KEYWORD + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func=signature_func): + self.assertEqual( + signature_func(ida.f, annotation_format=Format.STRING), + sig([par("x", PORK, annotation="undefined")]) + ) + s1 = signature_func(ida.f, annotation_format=Format.FORWARDREF) + s2 = sig([par("x", PORK, annotation=EqualToForwardRef("undefined", owner=ida.f))]) + #breakpoint() + self.assertEqual( + signature_func(ida.f, annotation_format=Format.FORWARDREF), + sig([par("x", PORK, annotation=EqualToForwardRef("undefined", owner=ida.f))]) + ) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f, annotation_format=Format.VALUE) + with self.assertRaisesRegex(NameError, "undefined"): + signature_func(ida.f) + + def test_signature_deferred_annotations(self): + def f(x: undef): + pass + + class C: + x: undef + + def __init__(self, x: undef): + self.x = x + + sig = inspect.signature(f, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + sig = inspect.signature(C, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + + class CallableWrapper: + def __init__(self, func): + self.func = func + self.__annotate__ = func.__annotate__ + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + @property + def __annotations__(self): + return self.__annotate__(Format.VALUE) + + cw = CallableWrapper(f) + sig = inspect.signature(cw, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['args', 'kwargs']) + + def test_signature_none_annotation(self): + class funclike: + # Has to be callable, and have correct + # __code__, __annotations__, __defaults__, __name__, + # and __kwdefaults__ attributes + + def __init__(self, func): + self.__name__ = func.__name__ + self.__code__ = func.__code__ + self.__annotations__ = func.__annotations__ + self.__defaults__ = func.__defaults__ + self.__kwdefaults__ = func.__kwdefaults__ + self.func = func + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + def foo(): pass + foo = funclike(foo) + foo.__annotations__ = None + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func = signature_func): + self.assertEqual(signature_func(foo), inspect.Signature()) + self.assertEqual(inspect.get_annotations(foo), {}) + + def test_signature_on_derived_classes(self): + # gh-105080: Make sure that signatures are consistent on derived classes + + class B: + def __new__(self, *args, **kwargs): + return super().__new__(self) + def __init__(self, value): + self.value = value + + class D1(B): + def __init__(self, value): + super().__init__(value) + + class D2(D1): + pass + + self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + + +class TestParameterObject(unittest.TestCase): + def test_signature_parameter_kinds(self): + P = inspect.Parameter + self.assertTrue(P.POSITIONAL_ONLY < P.POSITIONAL_OR_KEYWORD < \ + P.VAR_POSITIONAL < P.KEYWORD_ONLY < P.VAR_KEYWORD) + + self.assertEqual(str(P.POSITIONAL_ONLY), 'POSITIONAL_ONLY') + self.assertTrue('POSITIONAL_ONLY' in repr(P.POSITIONAL_ONLY)) + + def test_signature_parameter_object(self): + p = inspect.Parameter('foo', default=10, + kind=inspect.Parameter.POSITIONAL_ONLY) + self.assertEqual(p.name, 'foo') + self.assertEqual(p.default, 10) + self.assertIs(p.annotation, p.empty) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + with self.assertRaisesRegex(ValueError, "value '123' is " + "not a valid Parameter.kind"): + inspect.Parameter('foo', default=10, kind='123') + + with self.assertRaisesRegex(ValueError, 'not a valid parameter name'): + inspect.Parameter('1', kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, 'not a valid parameter name'): + inspect.Parameter('from', kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(TypeError, 'name must be a str'): + inspect.Parameter(None, kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, + 'is not a valid parameter name'): + inspect.Parameter('$', kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, + 'is not a valid parameter name'): + inspect.Parameter('.a', kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, 'cannot have default values'): + inspect.Parameter('a', default=42, + kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, 'cannot have default values'): + inspect.Parameter('a', default=42, + kind=inspect.Parameter.VAR_POSITIONAL) + + p = inspect.Parameter('a', default=42, + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD) + with self.assertRaisesRegex(ValueError, 'cannot have default values'): + p.replace(kind=inspect.Parameter.VAR_POSITIONAL) + + self.assertStartsWith(repr(p), '<Parameter') + self.assertTrue('"a=42"' in repr(p)) + + def test_signature_parameter_hashable(self): + P = inspect.Parameter + foo = P('foo', kind=P.POSITIONAL_ONLY) + self.assertEqual(hash(foo), hash(P('foo', kind=P.POSITIONAL_ONLY))) + self.assertNotEqual(hash(foo), hash(P('foo', kind=P.POSITIONAL_ONLY, + default=42))) + self.assertNotEqual(hash(foo), + hash(foo.replace(kind=P.VAR_POSITIONAL))) + + def test_signature_parameter_equality(self): + P = inspect.Parameter + p = P('foo', default=42, kind=inspect.Parameter.KEYWORD_ONLY) + + self.assertTrue(p == p) + self.assertFalse(p != p) + self.assertFalse(p == 42) + self.assertTrue(p != 42) + self.assertTrue(p == ALWAYS_EQ) + self.assertFalse(p != ALWAYS_EQ) + + self.assertTrue(p == P('foo', default=42, + kind=inspect.Parameter.KEYWORD_ONLY)) + self.assertFalse(p != P('foo', default=42, + kind=inspect.Parameter.KEYWORD_ONLY)) + + def test_signature_parameter_replace(self): + p = inspect.Parameter('foo', default=42, + kind=inspect.Parameter.KEYWORD_ONLY) + + self.assertIsNot(p.replace(), p) + self.assertEqual(p.replace(), p) + self.assertIsNot(copy.replace(p), p) + self.assertEqual(copy.replace(p), p) + + p2 = p.replace(annotation=1) + self.assertEqual(p2.annotation, 1) + p2 = p2.replace(annotation=p2.empty) + self.assertEqual(p2, p) + p3 = copy.replace(p, annotation=1) + self.assertEqual(p3.annotation, 1) + p3 = copy.replace(p3, annotation=p3.empty) + self.assertEqual(p3, p) + + p2 = p2.replace(name='bar') + self.assertEqual(p2.name, 'bar') + self.assertNotEqual(p2, p) + p3 = copy.replace(p3, name='bar') + self.assertEqual(p3.name, 'bar') + self.assertNotEqual(p3, p) + + with self.assertRaisesRegex(ValueError, + 'name is a required attribute'): + p2 = p2.replace(name=p2.empty) + with self.assertRaisesRegex(ValueError, + 'name is a required attribute'): + p3 = copy.replace(p3, name=p3.empty) + + p2 = p2.replace(name='foo', default=None) + self.assertIs(p2.default, None) + self.assertNotEqual(p2, p) + p3 = copy.replace(p3, name='foo', default=None) + self.assertIs(p3.default, None) + self.assertNotEqual(p3, p) + + p2 = p2.replace(name='foo', default=p2.empty) + self.assertIs(p2.default, p2.empty) + p3 = copy.replace(p3, name='foo', default=p3.empty) + self.assertIs(p3.default, p3.empty) + + p2 = p2.replace(default=42, kind=p2.POSITIONAL_OR_KEYWORD) + self.assertEqual(p2.kind, p2.POSITIONAL_OR_KEYWORD) + self.assertNotEqual(p2, p) + p3 = copy.replace(p3, default=42, kind=p3.POSITIONAL_OR_KEYWORD) + self.assertEqual(p3.kind, p3.POSITIONAL_OR_KEYWORD) + self.assertNotEqual(p3, p) + + with self.assertRaisesRegex(ValueError, + "value <class 'inspect._empty'> " + "is not a valid Parameter.kind"): + p2 = p2.replace(kind=p2.empty) + with self.assertRaisesRegex(ValueError, + "value <class 'inspect._empty'> " + "is not a valid Parameter.kind"): + p3 = copy.replace(p3, kind=p3.empty) + + p2 = p2.replace(kind=p2.KEYWORD_ONLY) + self.assertEqual(p2, p) + p3 = copy.replace(p3, kind=p3.KEYWORD_ONLY) + self.assertEqual(p3, p) + + def test_signature_parameter_positional_only(self): + with self.assertRaisesRegex(TypeError, 'name must be a str'): + inspect.Parameter(None, kind=inspect.Parameter.POSITIONAL_ONLY) + + @cpython_only + def test_signature_parameter_implicit(self): + with self.assertRaisesRegex(ValueError, + 'implicit arguments must be passed as ' + 'positional or keyword arguments, ' + 'not positional-only'): + inspect.Parameter('.0', kind=inspect.Parameter.POSITIONAL_ONLY) + + param = inspect.Parameter( + '.0', kind=inspect.Parameter.POSITIONAL_OR_KEYWORD) + self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_ONLY) + self.assertEqual(param.name, 'implicit0') + + def test_signature_parameter_immutability(self): + p = inspect.Parameter('spam', kind=inspect.Parameter.KEYWORD_ONLY) + + with self.assertRaises(AttributeError): + p.foo = 'bar' + + with self.assertRaises(AttributeError): + p.kind = 123 + + +class TestSignatureBind(unittest.TestCase): + @staticmethod + def call(func, *args, **kwargs): + sig = inspect.signature(func) + ba = sig.bind(*args, **kwargs) + # Prevent unexpected success of assertRaises(TypeError, ...) + try: + return func(*ba.args, **ba.kwargs) + except TypeError as e: + raise AssertionError from e + + def test_signature_bind_empty(self): + def test(): + return 42 + + self.assertEqual(self.call(test), 42) + with self.assertRaisesRegex(TypeError, 'too many positional arguments'): + self.call(test, 1) + with self.assertRaisesRegex(TypeError, 'too many positional arguments'): + self.call(test, 1, spam=10) + with self.assertRaisesRegex( + TypeError, "got an unexpected keyword argument 'spam'"): + + self.call(test, spam=1) + + def test_signature_bind_var(self): + def test(*args, **kwargs): + return args, kwargs + + self.assertEqual(self.call(test), ((), {})) + self.assertEqual(self.call(test, 1), ((1,), {})) + self.assertEqual(self.call(test, 1, 2), ((1, 2), {})) + self.assertEqual(self.call(test, foo='bar'), ((), {'foo': 'bar'})) + self.assertEqual(self.call(test, 1, foo='bar'), ((1,), {'foo': 'bar'})) + self.assertEqual(self.call(test, args=10), ((), {'args': 10})) + self.assertEqual(self.call(test, 1, 2, foo='bar'), + ((1, 2), {'foo': 'bar'})) + + def test_signature_bind_just_args(self): + def test(a, b, c): + return a, b, c + + self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3)) + + with self.assertRaisesRegex(TypeError, 'too many positional arguments'): + self.call(test, 1, 2, 3, 4) + + with self.assertRaisesRegex(TypeError, + "missing a required argument: 'b'"): + self.call(test, 1) + + with self.assertRaisesRegex(TypeError, + "missing a required argument: 'a'"): + self.call(test) + + def test(a, b, c=10): + return a, b, c + self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3)) + self.assertEqual(self.call(test, 1, 2), (1, 2, 10)) + + def test(a=1, b=2, c=3): + return a, b, c + self.assertEqual(self.call(test, a=10, c=13), (10, 2, 13)) + self.assertEqual(self.call(test, a=10), (10, 2, 3)) + self.assertEqual(self.call(test, b=10), (1, 10, 3)) + + def test_signature_bind_varargs_order(self): + def test(*args): + return args + + self.assertEqual(self.call(test), ()) + self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3)) + + def test_signature_bind_args_and_varargs(self): + def test(a, b, c=3, *args): + return a, b, c, args + + self.assertEqual(self.call(test, 1, 2, 3, 4, 5), (1, 2, 3, (4, 5))) + self.assertEqual(self.call(test, 1, 2), (1, 2, 3, ())) + self.assertEqual(self.call(test, b=1, a=2), (2, 1, 3, ())) + self.assertEqual(self.call(test, 1, b=2), (1, 2, 3, ())) + + with self.assertRaisesRegex(TypeError, + "multiple values for argument 'c'"): + self.call(test, 1, 2, 3, c=4) + + def test_signature_bind_just_kwargs(self): + def test(**kwargs): + return kwargs + + self.assertEqual(self.call(test), {}) + self.assertEqual(self.call(test, foo='bar', spam='ham'), + {'foo': 'bar', 'spam': 'ham'}) + + def test_signature_bind_args_and_kwargs(self): + def test(a, b, c=3, **kwargs): + return a, b, c, kwargs + + self.assertEqual(self.call(test, 1, 2), (1, 2, 3, {})) + self.assertEqual(self.call(test, 1, 2, foo='bar', spam='ham'), + (1, 2, 3, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, b=2, a=1, foo='bar', spam='ham'), + (1, 2, 3, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, a=1, b=2, foo='bar', spam='ham'), + (1, 2, 3, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, 1, b=2, foo='bar', spam='ham'), + (1, 2, 3, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, 1, b=2, c=4, foo='bar', spam='ham'), + (1, 2, 4, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, 1, 2, 4, foo='bar'), + (1, 2, 4, {'foo': 'bar'})) + self.assertEqual(self.call(test, c=5, a=4, b=3), + (4, 3, 5, {})) + + def test_signature_bind_kwonly(self): + def test(*, foo): + return foo + with self.assertRaisesRegex(TypeError, + 'too many positional arguments'): + self.call(test, 1) + self.assertEqual(self.call(test, foo=1), 1) + + def test(a, *, foo=1, bar): + return foo + with self.assertRaisesRegex(TypeError, + "missing a required argument: 'bar'"): + self.call(test, 1) + + def test(foo, *, bar): + return foo, bar + self.assertEqual(self.call(test, 1, bar=2), (1, 2)) + self.assertEqual(self.call(test, bar=2, foo=1), (1, 2)) + + with self.assertRaisesRegex( + TypeError, "got an unexpected keyword argument 'spam'"): + + self.call(test, bar=2, foo=1, spam=10) + + with self.assertRaisesRegex(TypeError, + 'too many positional arguments'): + self.call(test, 1, 2) + + with self.assertRaisesRegex(TypeError, + 'too many positional arguments'): + self.call(test, 1, 2, bar=2) + + with self.assertRaisesRegex( + TypeError, "got an unexpected keyword argument 'spam'"): + + self.call(test, 1, bar=2, spam='ham') + + with self.assertRaisesRegex(TypeError, + "missing a required keyword-only " + "argument: 'bar'"): + self.call(test, 1) + + def test(foo, *, bar, **bin): + return foo, bar, bin + self.assertEqual(self.call(test, 1, bar=2), (1, 2, {})) + self.assertEqual(self.call(test, foo=1, bar=2), (1, 2, {})) + self.assertEqual(self.call(test, 1, bar=2, spam='ham'), + (1, 2, {'spam': 'ham'})) + self.assertEqual(self.call(test, spam='ham', foo=1, bar=2), + (1, 2, {'spam': 'ham'})) + with self.assertRaisesRegex(TypeError, + "missing a required argument: 'foo'"): + self.call(test, spam='ham', bar=2) + self.assertEqual(self.call(test, 1, bar=2, bin=1, spam=10), + (1, 2, {'bin': 1, 'spam': 10})) + + def test_signature_bind_arguments(self): + def test(a, *args, b, z=100, **kwargs): + pass + sig = inspect.signature(test) + ba = sig.bind(10, 20, b=30, c=40, args=50, kwargs=60) + # we won't have 'z' argument in the bound arguments object, as we didn't + # pass it to the 'bind' + self.assertEqual(tuple(ba.arguments.items()), + (('a', 10), ('args', (20,)), ('b', 30), + ('kwargs', {'c': 40, 'args': 50, 'kwargs': 60}))) + self.assertEqual(ba.kwargs, + {'b': 30, 'c': 40, 'args': 50, 'kwargs': 60}) + self.assertEqual(ba.args, (10, 20)) + + def test_signature_bind_positional_only(self): + def test(a_po, b_po, c_po=3, /, foo=42, *, bar=50, **kwargs): + return a_po, b_po, c_po, foo, bar, kwargs + + self.assertEqual(self.call(test, 1, 2, 4, 5, bar=6), + (1, 2, 4, 5, 6, {})) + + self.assertEqual(self.call(test, 1, 2), + (1, 2, 3, 42, 50, {})) + + self.assertEqual(self.call(test, 1, 2, foo=4, bar=5), + (1, 2, 3, 4, 5, {})) + + self.assertEqual(self.call(test, 1, 2, foo=4, bar=5, c_po=10), + (1, 2, 3, 4, 5, {'c_po': 10})) + + self.assertEqual(self.call(test, 1, 2, 30, c_po=31, foo=4, bar=5), + (1, 2, 30, 4, 5, {'c_po': 31})) + + self.assertEqual(self.call(test, 1, 2, 30, foo=4, bar=5, c_po=31), + (1, 2, 30, 4, 5, {'c_po': 31})) + + self.assertEqual(self.call(test, 1, 2, c_po=4), + (1, 2, 3, 42, 50, {'c_po': 4})) + + with self.assertRaisesRegex(TypeError, "missing a required positional-only argument: 'a_po'"): + self.call(test, a_po=1, b_po=2) + + def without_var_kwargs(c_po=3, d_po=4, /): + return c_po, d_po + + with self.assertRaisesRegex( + TypeError, + "positional-only arguments passed as keyword arguments: 'c_po, d_po'", + ): + self.call(without_var_kwargs, c_po=33, d_po=44) + + def test_signature_bind_with_self_arg(self): + # Issue #17071: one of the parameters is named "self + def test(a, self, b): + pass + sig = inspect.signature(test) + ba = sig.bind(1, 2, 3) + self.assertEqual(ba.args, (1, 2, 3)) + ba = sig.bind(1, self=2, b=3) + self.assertEqual(ba.args, (1, 2, 3)) + + def test_signature_bind_vararg_name(self): + def test(a, *args): + return a, args + sig = inspect.signature(test) + + with self.assertRaisesRegex( + TypeError, "got an unexpected keyword argument 'args'"): + + sig.bind(a=0, args=1) + + def test(*args, **kwargs): + return args, kwargs + self.assertEqual(self.call(test, args=1), ((), {'args': 1})) + + sig = inspect.signature(test) + ba = sig.bind(args=1) + self.assertEqual(ba.arguments, {'kwargs': {'args': 1}}) + + @cpython_only + def test_signature_bind_implicit_arg(self): + # Issue #19611: getcallargs should work with comprehensions + def make_set(): + return set(z * z for z in range(5)) + gencomp_code = make_set.__code__.co_consts[0] + gencomp_func = types.FunctionType(gencomp_code, {}) + + iterator = iter(range(5)) + self.assertEqual(set(self.call(gencomp_func, iterator)), {0, 1, 4, 9, 16}) + + def test_signature_bind_posonly_kwargs(self): + def foo(bar, /, **kwargs): + return bar, kwargs.get(bar) + + sig = inspect.signature(foo) + result = sig.bind("pos-only", bar="keyword") + + self.assertEqual(result.kwargs, {"bar": "keyword"}) + self.assertIn(("bar", "pos-only"), result.arguments.items()) + + +class TestBoundArguments(unittest.TestCase): + def test_signature_bound_arguments_unhashable(self): + def foo(a): pass + ba = inspect.signature(foo).bind(1) + + with self.assertRaisesRegex(TypeError, 'unhashable type'): + hash(ba) + + def test_signature_bound_arguments_equality(self): + def foo(a): pass + ba = inspect.signature(foo).bind(1) + self.assertTrue(ba == ba) + self.assertFalse(ba != ba) + self.assertTrue(ba == ALWAYS_EQ) + self.assertFalse(ba != ALWAYS_EQ) + + ba2 = inspect.signature(foo).bind(1) + self.assertTrue(ba == ba2) + self.assertFalse(ba != ba2) + + ba3 = inspect.signature(foo).bind(2) + self.assertFalse(ba == ba3) + self.assertTrue(ba != ba3) + ba3.arguments['a'] = 1 + self.assertTrue(ba == ba3) + self.assertFalse(ba != ba3) + + def bar(b): pass + ba4 = inspect.signature(bar).bind(1) + self.assertFalse(ba == ba4) + self.assertTrue(ba != ba4) + + def foo(*, a, b): pass + sig = inspect.signature(foo) + ba1 = sig.bind(a=1, b=2) + ba2 = sig.bind(b=2, a=1) + self.assertTrue(ba1 == ba2) + self.assertFalse(ba1 != ba2) + + def test_signature_bound_arguments_pickle(self): + def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass + sig = inspect.signature(foo) + ba = sig.bind(20, 30, z={}) + + for ver in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_ver=ver): + ba_pickled = pickle.loads(pickle.dumps(ba, ver)) + self.assertEqual(ba, ba_pickled) + + def test_signature_bound_arguments_repr(self): + def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass + sig = inspect.signature(foo) + ba = sig.bind(20, 30, z={}) + self.assertRegex(repr(ba), r'<BoundArguments \(a=20,.*\}\}\)>') + + def test_signature_bound_arguments_apply_defaults(self): + def foo(a, b=1, *args, c:1={}, **kw): pass + sig = inspect.signature(foo) + + ba = sig.bind(20) + ba.apply_defaults() + self.assertEqual( + list(ba.arguments.items()), + [('a', 20), ('b', 1), ('args', ()), ('c', {}), ('kw', {})]) + + # Make sure that we preserve the order: + # i.e. 'c' should be *before* 'kw'. + ba = sig.bind(10, 20, 30, d=1) + ba.apply_defaults() + self.assertEqual( + list(ba.arguments.items()), + [('a', 10), ('b', 20), ('args', (30,)), ('c', {}), ('kw', {'d':1})]) + + # Make sure that BoundArguments produced by bind_partial() + # are supported. + def foo(a, b): pass + sig = inspect.signature(foo) + ba = sig.bind_partial(20) + ba.apply_defaults() + self.assertEqual( + list(ba.arguments.items()), + [('a', 20)]) + + # Test no args + def foo(): pass + sig = inspect.signature(foo) + ba = sig.bind() + ba.apply_defaults() + self.assertEqual(list(ba.arguments.items()), []) + + # Make sure a no-args binding still acquires proper defaults. + def foo(a='spam'): pass + sig = inspect.signature(foo) + ba = sig.bind() + ba.apply_defaults() + self.assertEqual(list(ba.arguments.items()), [('a', 'spam')]) + + def test_signature_bound_arguments_arguments_type(self): + def foo(a): pass + ba = inspect.signature(foo).bind(1) + self.assertIs(type(ba.arguments), dict) + +class TestSignaturePrivateHelpers(unittest.TestCase): + def _strip_non_python_syntax(self, input, + clean_signature, self_parameter): + computed_clean_signature, \ + computed_self_parameter = \ + inspect._signature_strip_non_python_syntax(input) + self.assertEqual(computed_clean_signature, clean_signature) + self.assertEqual(computed_self_parameter, self_parameter) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + (module, /, path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True) + def test_signature_strip_non_python_syntax(self): + self._strip_non_python_syntax( + "($module, /, path, mode, *, dir_fd=None, " + + "effective_ids=False,\n follow_symlinks=True)", + "(module, /, path, mode, *, dir_fd=None, " + + "effective_ids=False, follow_symlinks=True)", + 0) + + self._strip_non_python_syntax( + "($module, word, salt, /)", + "(module, word, salt, /)", + 0) + + self._strip_non_python_syntax( + "(x, y=None, z=None, /)", + "(x, y=None, z=None, /)", + None) + + self._strip_non_python_syntax( + "(x, y=None, z=None)", + "(x, y=None, z=None)", + None) + + self._strip_non_python_syntax( + "(x,\n y=None,\n z = None )", + "(x, y=None, z=None)", + None) + + self._strip_non_python_syntax( + "", + "", + None) + + self._strip_non_python_syntax( + None, + None, + None) + +class TestSignatureDefinitions(unittest.TestCase): + # This test case provides a home for checking that particular APIs + # have signatures available for introspection + + @staticmethod + def is_public(name): + return not name.startswith('_') or name.startswith('__') and name.endswith('__') + + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def _test_module_has_signatures(self, module, + no_signature=(), unsupported_signature=(), + methods_no_signature={}, methods_unsupported_signature={}, + good_exceptions=()): + # This checks all builtin callables in CPython have signatures + # A few have signatures Signature can't yet handle, so we skip those + # since they will have to wait until PEP 457 adds the required + # introspection support to the inspect module + # Some others also haven't been converted yet for various other + # reasons, so we also skip those for the time being, but design + # the test to fail in order to indicate when it needs to be + # updated. + no_signature = no_signature or set() + # Check the signatures we expect to be there + ns = vars(module) + try: + names = set(module.__all__) + except AttributeError: + names = set(name for name in ns if self.is_public(name)) + for name, obj in sorted(ns.items()): + if name not in names: + continue + if not callable(obj): + continue + if (isinstance(obj, type) and + issubclass(obj, BaseException) and + name not in good_exceptions): + no_signature.add(name) + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', builtin=name): + self.assertIsNotNone(inspect.signature(obj)) + if isinstance(obj, type): + with self.subTest(type=name): + self._test_builtin_methods_have_signatures(obj, + methods_no_signature.get(name, ()), + methods_unsupported_signature.get(name, ())) + # Check callables that haven't been converted don't claim a signature + # This ensures this test will start failing as more signatures are + # added, so the affected items can be moved into the scope of the + # regression test above + for name in no_signature: + with self.subTest('none', builtin=name): + obj = ns[name] + self.assertIsNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + for name in unsupported_signature: + with self.subTest('unsupported', builtin=name): + obj = ns[name] + self.assertIsNotNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + + def _test_builtin_methods_have_signatures(self, cls, no_signature, unsupported_signature): + ns = vars(cls) + for name in ns: + obj = getattr(cls, name, None) + if not callable(obj) or isinstance(obj, type): + continue + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', method=name): + self.assertIsNotNone(inspect.signature(obj)) + for name in no_signature: + with self.subTest('none', method=name): + self.assertIsNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + for name in unsupported_signature: + with self.subTest('unsupported', method=name): + self.assertIsNotNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + + def test_builtins_have_signatures(self): + no_signature = {'type', 'super', 'bytearray', 'bytes', 'dict', 'int', 'str'} + # These need PEP 457 groups + needs_groups = {"range", "slice", "dir", "getattr", + "next", "iter", "vars"} + no_signature |= needs_groups + # These have unrepresentable parameter default values of NULL + unsupported_signature = {"anext"} + # These need *args support in Argument Clinic + needs_varargs = {"min", "max", "__build_class__"} + no_signature |= needs_varargs + + methods_no_signature = { + 'dict': {'update'}, + 'object': {'__class__'}, + } + methods_unsupported_signature = { + 'bytearray': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'bytes': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'dict': {'pop'}, + 'memoryview': {'cast', 'hex'}, + 'str': {'count', 'endswith', 'find', 'index', 'maketrans', 'rfind', 'rindex', 'startswith'}, + } + self._test_module_has_signatures(builtins, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_types_module_has_signatures(self): + unsupported_signature = {'CellType'} + methods_no_signature = { + 'AsyncGeneratorType': {'athrow'}, + 'CoroutineType': {'throw'}, + 'GeneratorType': {'throw'}, + } + self._test_module_has_signatures(types, + unsupported_signature=unsupported_signature, + methods_no_signature=methods_no_signature) + + def test_sys_module_has_signatures(self): + no_signature = {'getsizeof', 'set_asyncgen_hooks'} + no_signature |= {name for name in ['getobjects'] + if hasattr(sys, name)} + self._test_module_has_signatures(sys, no_signature) + + def test_abc_module_has_signatures(self): + import abc + self._test_module_has_signatures(abc) + + def test_atexit_module_has_signatures(self): + import atexit + self._test_module_has_signatures(atexit) + + def test_codecs_module_has_signatures(self): + import codecs + methods_no_signature = {'StreamReader': {'charbuffertype'}} + self._test_module_has_signatures(codecs, + methods_no_signature=methods_no_signature) + + def test_collections_module_has_signatures(self): + no_signature = {'OrderedDict', 'defaultdict'} + unsupported_signature = {'deque'} + methods_no_signature = { + 'OrderedDict': {'update'}, + } + methods_unsupported_signature = { + 'deque': {'index'}, + 'OrderedDict': {'pop'}, + 'UserString': {'maketrans'}, + } + self._test_module_has_signatures(collections, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_collections_abc_module_has_signatures(self): + import collections.abc + self._test_module_has_signatures(collections.abc) + + def test_errno_module_has_signatures(self): + import errno + self._test_module_has_signatures(errno) + + def test_faulthandler_module_has_signatures(self): + import faulthandler + unsupported_signature = {'dump_traceback', 'dump_traceback_later', 'enable', 'dump_c_stack'} + unsupported_signature |= {name for name in ['register'] + if hasattr(faulthandler, name)} + self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) + + def test_functools_module_has_signatures(self): + unsupported_signature = {"reduce"} + self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + + def test_gc_module_has_signatures(self): + import gc + no_signature = {'set_threshold'} + self._test_module_has_signatures(gc, no_signature) + + def test_io_module_has_signatures(self): + methods_no_signature = { + 'BufferedRWPair': {'read', 'peek', 'read1', 'readinto', 'readinto1', 'write'}, + } + self._test_module_has_signatures(io, + methods_no_signature=methods_no_signature) + + def test_itertools_module_has_signatures(self): + import itertools + no_signature = {'islice', 'repeat'} + self._test_module_has_signatures(itertools, no_signature) + + def test_locale_module_has_signatures(self): + import locale + self._test_module_has_signatures(locale) + + def test_marshal_module_has_signatures(self): + import marshal + self._test_module_has_signatures(marshal) + + def test_operator_module_has_signatures(self): + import operator + self._test_module_has_signatures(operator) + + def test_os_module_has_signatures(self): + unsupported_signature = {'chmod', 'utime'} + unsupported_signature |= {name for name in + ['get_terminal_size', 'link', 'posix_spawn', 'posix_spawnp', + 'register_at_fork', 'startfile'] + if hasattr(os, name)} + self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) + + def test_pwd_module_has_signatures(self): + pwd = import_helper.import_module('pwd') + self._test_module_has_signatures(pwd) + + def test_re_module_has_signatures(self): + import re + methods_no_signature = {'Match': {'group'}} + self._test_module_has_signatures(re, + methods_no_signature=methods_no_signature, + good_exceptions={'error', 'PatternError'}) + + def test_signal_module_has_signatures(self): + import signal + self._test_module_has_signatures(signal) + + def test_stat_module_has_signatures(self): + import stat + self._test_module_has_signatures(stat) + + def test_string_module_has_signatures(self): + import string + self._test_module_has_signatures(string) + + def test_symtable_module_has_signatures(self): + import symtable + self._test_module_has_signatures(symtable) + + def test_sysconfig_module_has_signatures(self): + import sysconfig + self._test_module_has_signatures(sysconfig) + + def test_threading_module_has_signatures(self): + import threading + self._test_module_has_signatures(threading) + self.assertIsNotNone(inspect.signature(threading.__excepthook__)) + + def test_thread_module_has_signatures(self): + import _thread + no_signature = {'RLock'} + self._test_module_has_signatures(_thread, no_signature) + + def test_time_module_has_signatures(self): + no_signature = { + 'asctime', 'ctime', 'get_clock_info', 'gmtime', 'localtime', + 'strftime', 'strptime' + } + no_signature |= {name for name in + ['clock_getres', 'clock_settime', 'clock_settime_ns', + 'pthread_getcpuclockid'] + if hasattr(time, name)} + self._test_module_has_signatures(time, no_signature) + + def test_tokenize_module_has_signatures(self): + import tokenize + self._test_module_has_signatures(tokenize) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ModuleNotFoundError: No module named 'tracemalloc' + def test_tracemalloc_module_has_signatures(self): + import tracemalloc + self._test_module_has_signatures(tracemalloc) + + def test_typing_module_has_signatures(self): + import typing + no_signature = {'ParamSpec', 'ParamSpecArgs', 'ParamSpecKwargs', + 'Text', 'TypeAliasType', 'TypeVar', 'TypeVarTuple'} + methods_no_signature = { + 'Generic': {'__class_getitem__', '__init_subclass__'}, + } + methods_unsupported_signature = { + 'Text': {'count', 'find', 'index', 'rfind', 'rindex', 'startswith', 'endswith', 'maketrans'}, + } + self._test_module_has_signatures(typing, no_signature, + methods_no_signature=methods_no_signature, + methods_unsupported_signature=methods_unsupported_signature) + + def test_warnings_module_has_signatures(self): + unsupported_signature = {'warn', 'warn_explicit'} + self._test_module_has_signatures(warnings, unsupported_signature=unsupported_signature) + + def test_weakref_module_has_signatures(self): + import weakref + no_signature = {'ReferenceType', 'ref'} + self._test_module_has_signatures(weakref, no_signature) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: <function TestSignatureDefinitions.test_python_function_override_signature.<locals>.func at 0xa4c07a580> builtin has invalid signature + def test_python_function_override_signature(self): + def func(*args, **kwargs): + pass + func.__text_signature__ = '($self, a, b=1, *args, c, d=2, **kwargs)' + sig = inspect.signature(func) + self.assertIsNotNone(sig) + self.assertEqual(str(sig), '(self, /, a, b=1, *args, c, d=2, **kwargs)') + + func.__text_signature__ = '($self, a, b=1, /, *args, c, d=2, **kwargs)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + + func.__text_signature__ = '(self, a=1+2, b=4-3, c=1 | 3 | 16)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=3, b=1, c=19)') + + func.__text_signature__ = '(self, a=1,\nb=2,\n\n\n c=3)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=1, b=2, c=3)') + + func.__text_signature__ = '(self, x=does_not_exist)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, x=sys, y=inspect)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, 123)' + with self.assertRaises(ValueError): + inspect.signature(func) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != '(raw, buffer_size=DEFAULT_BUFFER_SIZE)' + @support.requires_docstrings + def test_base_class_have_text_signature(self): + # see issue 43118 + from test.typinganndata.ann_module7 import BufferedReader + class MyBufferedReader(BufferedReader): + """buffer reader class.""" + + text_signature = BufferedReader.__text_signature__ + self.assertEqual(text_signature, '(raw, buffer_size=DEFAULT_BUFFER_SIZE)') + sig = inspect.signature(MyBufferedReader) + self.assertEqual(str(sig), '(raw, buffer_size=8192)') + + +class NTimesUnwrappable: + def __init__(self, n): + self.n = n + self._next = None + + @property + def __wrapped__(self): + if self.n <= 0: + raise Exception("Unwrapped too many times") + if self._next is None: + self._next = NTimesUnwrappable(self.n - 1) + return self._next + +class TestUnwrap(unittest.TestCase): + + def test_unwrap_one(self): + def func(a, b): + return a + b + wrapper = functools.lru_cache(maxsize=20)(func) + self.assertIs(inspect.unwrap(wrapper), func) + + def test_unwrap_several(self): + def func(a, b): + return a + b + wrapper = func + for __ in range(10): + @functools.wraps(wrapper) + def wrapper(): + pass + self.assertIsNot(wrapper.__wrapped__, func) + self.assertIs(inspect.unwrap(wrapper), func) + + def test_stop(self): + def func1(a, b): + return a + b + @functools.wraps(func1) + def func2(): + pass + @functools.wraps(func2) + def wrapper(): + pass + func2.stop_here = 1 + unwrapped = inspect.unwrap(wrapper, + stop=(lambda f: hasattr(f, "stop_here"))) + self.assertIs(unwrapped, func2) + + def test_cycle(self): + def func1(): pass + func1.__wrapped__ = func1 + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func1) + + def func2(): pass + func2.__wrapped__ = func1 + func1.__wrapped__ = func2 + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func1) + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func2) + + def test_unhashable(self): + def func(): pass + func.__wrapped__ = None + class C: + __hash__ = None + __wrapped__ = func + self.assertIsNone(inspect.unwrap(C())) + + def test_recursion_limit(self): + obj = NTimesUnwrappable(sys.getrecursionlimit() + 1) + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(obj) + + def test_wrapped_descriptor(self): + self.assertIs(inspect.unwrap(NTimesUnwrappable), NTimesUnwrappable) + self.assertIs(inspect.unwrap(staticmethod), staticmethod) + self.assertIs(inspect.unwrap(classmethod), classmethod) + self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) + self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) + + +class TestMain(unittest.TestCase): + def test_only_source(self): + module = importlib.import_module('unittest') + rc, out, err = assert_python_ok('-m', 'inspect', + 'unittest') + lines = out.decode().splitlines() + # ignore the final newline + self.assertEqual(lines[:-1], inspect.getsource(module).splitlines()) + self.assertEqual(err, b'') + + def test_custom_getattr(self): + def foo(): + pass + foo.__signature__ = 42 + with self.assertRaises(TypeError): + inspect.signature(foo) + + @unittest.skipIf(ThreadPoolExecutor is None, + 'threads required to test __qualname__ for source files') + def test_qualname_source(self): + rc, out, err = assert_python_ok('-m', 'inspect', + 'concurrent.futures:ThreadPoolExecutor') + lines = out.decode().splitlines() + # ignore the final newline + self.assertEqual(lines[:-1], + inspect.getsource(ThreadPoolExecutor).splitlines()) + self.assertEqual(err, b'') + + def test_builtins(self): + _, out, err = assert_python_failure('-m', 'inspect', + 'sys') + lines = err.decode().splitlines() + self.assertEqual(lines, ["Can't get info for builtin modules."]) + + def test_details(self): + module = importlib.import_module('unittest') + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', + 'unittest', '--details') + output = out.decode() + # Just a quick sanity check on the output + self.assertIn(module.__spec__.name, output) + self.assertIn(module.__name__, output) + self.assertIn(module.__spec__.origin, output) + self.assertIn(module.__file__, output) + self.assertIn(module.__spec__.cached, output) + self.assertIn(module.__cached__, output) + self.assertEqual(err, b'') + + +class TestReload(unittest.TestCase): + + src_before = textwrap.dedent("""\ +def foo(): + print("Bla") + """) + + src_after = textwrap.dedent("""\ +def foo(): + print("Oh no!") + """) + + def assertInspectEqual(self, path, source): + inspected_src = inspect.getsource(source) + with open(path, encoding='utf-8') as src: + self.assertEqual( + src.read().splitlines(True), + inspected_src.splitlines(True) + ) + + def test_getsource_reload(self): + # see issue 1218234 + with ready_to_import('reload_bug', self.src_before) as (name, path): + module = importlib.import_module(name) + self.assertInspectEqual(path, module) + with open(path, 'w', encoding='utf-8') as src: + src.write(self.src_after) + self.assertInspectEqual(path, module) + + +class TestRepl(unittest.TestCase): + + def spawn_repl(self, *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): + """Run the Python REPL with the given arguments. + + kw is extra keyword args to pass to subprocess.Popen. Returns a Popen + object. + """ + + # To run the REPL without using a terminal, spawn python with the command + # line option '-i' and the process name set to '<stdin>'. + # The directory of argv[0] must match the directory of the Python + # executable for the Popen() call to python to succeed as the directory + # path may be used by Py_GetPath() to build the default module search + # path. + stdin_fname = os.path.join(os.path.dirname(sys.executable), "<stdin>") + cmd_line = [stdin_fname, '-E', '-i'] + cmd_line.extend(args) + + # Set TERM=vt100, for the rationale see the comments in spawn_python() of + # test.support.script_helper. + env = kw.setdefault('env', dict(os.environ)) + env['TERM'] = 'vt100' + return subprocess.Popen(cmd_line, + executable=sys.executable, + text=True, + stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + **kw) + + def run_on_interactive_mode(self, source): + """Spawn a new Python interpreter, pass the given + input source code from the stdin and return the + result back. If the interpreter exits non-zero, it + raises a ValueError.""" + + process = self.spawn_repl() + process.stdin.write(source) + output = kill_python(process) + + if process.returncode != 0: + raise ValueError("Process didn't exit properly.") + return output + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'The source is: <<<def f():\n print(0)\n return 1 + 2\n>>>' not found in 'Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n File "/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/inspect.py", line 1161, in getsource\n lines, lnum = getsourcelines(object)\n ~~~~~~~~~~~~~~^^^^^^^^\n File "/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/inspect.py", line 1143, in getsourcelines\n lines, lnum = findsource(object)\n ~~~~~~~~~~^^^^^^^^\n File "/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/inspect.py", line 978, in findsource\n raise OSError(\'could not get source code\')\nOSError: could not get source code\n' + @unittest.skipIf(not has_subprocess_support, "test requires subprocess") + def test_getsource(self): + output = self.run_on_interactive_mode(textwrap.dedent("""\ + def f(): + print(0) + return 1 + 2 + + import inspect + print(f"The source is: <<<{inspect.getsource(f)}>>>") + """)) + + expected = "The source is: <<<def f():\n print(0)\n return 1 + 2\n>>>" + self.assertIn(expected, output) + + + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 7ac83288f4c..f7b26e37e3a 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -1,9 +1,26 @@ import sys import unittest +# TODO: RUSTPYTHON +# This is one of the tests that we run on wasi. `unittest.mock` requires `_socket` +# which we don't have on wasi (yet). Also, every test here the needs `unittest.mock` +# is cpython specifc, so this import is redundent anyway. +# from unittest import mock from test import support -from test.test_grammar import (VALID_UNDERSCORE_LITERALS, - INVALID_UNDERSCORE_LITERALS) +from test.support.numbers import ( + VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS, +) + +try: + import _pylong +except ImportError: + _pylong = None + +try: + import _decimal +except ImportError: + _decimal = None L = [ ('0', 0), @@ -83,6 +100,7 @@ def test_basic(self): self.assertRaises(TypeError, int, 1, 12) + self.assertRaises(TypeError, int, "10", 2, 1) self.assertEqual(int('0o123', 0), 83) self.assertEqual(int('0x123', 16), 291) @@ -148,6 +166,8 @@ def test_basic(self): self.assertEqual(int(' 0O123 ', 0), 83) self.assertEqual(int(' 0X123 ', 0), 291) self.assertEqual(int(' 0B100 ', 0), 4) + with self.assertRaises(ValueError): + int('010', 0) # without base still base 10 self.assertEqual(int('0123'), 123) @@ -214,6 +234,25 @@ def test_basic(self): self.assertEqual(int('2br45qc', 35), 4294967297) self.assertEqual(int('1z141z5', 36), 4294967297) + def test_invalid_signs(self): + with self.assertRaises(ValueError): + int('+') + with self.assertRaises(ValueError): + int('-') + with self.assertRaises(ValueError): + int('- 1') + with self.assertRaises(ValueError): + int('+ 1') + with self.assertRaises(ValueError): + int(' + 1 ') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unicode(self): + self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890) + self.assertEqual(int("१२३४५६७८९०1234567890", 0), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠', 0), 1234567890) + def test_underscores(self): for lit in VALID_UNDERSCORE_LITERALS: if any(ch in lit for ch in '.eEjJ'): @@ -233,7 +272,7 @@ def test_underscores(self): self.assertRaises(ValueError, int, "1__00") self.assertRaises(ValueError, int, "100_") - # @support.cpython_only + @support.cpython_only def test_small_ints(self): # Bug #3236: Return small longs from PyLong_FromString self.assertIs(int('10'), 10) @@ -340,6 +379,7 @@ def test_int_memoryview(self): def test_string_float(self): self.assertRaises(ValueError, int, '1.2') + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_intconversion(self): # Test __int__() class ClassicMissingMethods: @@ -369,62 +409,8 @@ def __trunc__(self): class JustTrunc(base): def __trunc__(self): return 42 - self.assertEqual(int(JustTrunc()), 42) - - class ExceptionalTrunc(base): - def __trunc__(self): - 1 / 0 - with self.assertRaises(ZeroDivisionError): - int(ExceptionalTrunc()) - - for trunc_result_base in (object, Classic): - class Index(trunc_result_base): - def __index__(self): - return 42 - - class TruncReturnsNonInt(base): - def __trunc__(self): - return Index() - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class Intable(trunc_result_base): - def __int__(self): - return 42 - - class TruncReturnsNonIndex(base): - def __trunc__(self): - return Intable() - self.assertEqual(int(TruncReturnsNonInt()), 42) - - class NonIntegral(trunc_result_base): - def __trunc__(self): - # Check that we avoid infinite recursion. - return NonIntegral() - - class TruncReturnsNonIntegral(base): - def __trunc__(self): - return NonIntegral() - try: - int(TruncReturnsNonIntegral()) - except TypeError as e: - self.assertEqual(str(e), - "__trunc__ returned non-Integral" - " (type NonIntegral)") - else: - self.fail("Failed to raise TypeError with %s" % - ((base, trunc_result_base),)) - - # Regression test for bugs.python.org/issue16060. - class BadInt(trunc_result_base): - def __int__(self): - return 42.0 - - class TruncReturnsBadInt(base): - def __trunc__(self): - return BadInt() - - with self.assertRaises(TypeError): - int(TruncReturnsBadInt()) + with self.assertRaises(TypeError): + int(JustTrunc()) def test_int_subclass_with_index(self): class MyIndex(int): @@ -475,18 +461,6 @@ class BadInt2(int): def __int__(self): return True - class TruncReturnsBadIndex: - def __trunc__(self): - return BadIndex() - - class TruncReturnsBadInt: - def __trunc__(self): - return BadInt() - - class TruncReturnsIntSubclass: - def __trunc__(self): - return True - bad_int = BadIndex() with self.assertWarns(DeprecationWarning): n = int(bad_int) @@ -510,23 +484,6 @@ def __trunc__(self): self.assertEqual(n, 1) self.assertIs(type(n), int) - bad_int = TruncReturnsBadIndex() - with self.assertWarns(DeprecationWarning): - n = int(bad_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - - bad_int = TruncReturnsBadInt() - self.assertRaises(TypeError, int, bad_int) - - good_int = TruncReturnsIntSubclass() - n = int(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), int) - n = IntSubclass(good_int) - self.assertEqual(n, 1) - self.assertIs(type(n), IntSubclass) - def test_error_message(self): def check(s, base=None): with self.assertRaises(ValueError, @@ -567,6 +524,425 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + @support.cpython_only + def test_round_with_none_arg_direct_call(self): + for val in [(1).__round__(None), + round(1), + round(1, None)]: + self.assertEqual(val, 1) + self.assertIs(type(val), int) + +class IntStrDigitLimitsTests(unittest.TestCase): + + int_class = int # Override this in subclasses to reuse the suite. + + def setUp(self): + super().setUp() + self._previous_limit = sys.get_int_max_str_digits() + sys.set_int_max_str_digits(2048) + + def tearDown(self): + sys.set_int_max_str_digits(self._previous_limit) + super().tearDown() + + def test_disabled_limit(self): + self.assertGreater(sys.get_int_max_str_digits(), 0) + self.assertLess(sys.get_int_max_str_digits(), 20_000) + with support.adjust_int_max_str_digits(0): + self.assertEqual(sys.get_int_max_str_digits(), 0) + i = self.int_class('1' * 20_000) + str(i) + self.assertGreater(sys.get_int_max_str_digits(), 0) + + def test_max_str_digits_edge_cases(self): + """Ignore the +/- sign and space padding.""" + int_class = self.int_class + maxdigits = sys.get_int_max_str_digits() + + int_class('1' * maxdigits) + int_class(' ' + '1' * maxdigits) + int_class('1' * maxdigits + ' ') + int_class('+' + '1' * maxdigits) + int_class('-' + '1' * maxdigits) + self.assertEqual(len(str(10 ** (maxdigits - 1))), maxdigits) + + def check(self, i, base=None): + with self.assertRaises(ValueError): + if base is None: + self.int_class(i) + else: + self.int_class(i, base) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_max_str_digits(self): + maxdigits = sys.get_int_max_str_digits() + + self.check('1' * (maxdigits + 1)) + self.check(' ' + '1' * (maxdigits + 1)) + self.check('1' * (maxdigits + 1) + ' ') + self.check('+' + '1' * (maxdigits + 1)) + self.check('-' + '1' * (maxdigits + 1)) + self.check('1' * (maxdigits + 1)) + + i = 10 ** maxdigits + with self.assertRaises(ValueError): + str(i) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_denial_of_service_prevented_int_to_str(self): + """Regression test: ensure we fail before performing O(N**2) work.""" + maxdigits = sys.get_int_max_str_digits() + assert maxdigits < 50_000, maxdigits # A test prerequisite. + + huge_int = int(f'0x{"c"*65_000}', base=16) # 78268 decimal digits. + digits = 78_268 + with ( + support.adjust_int_max_str_digits(digits), + support.Stopwatch() as sw_convert): + huge_decimal = str(huge_int) + self.assertEqual(len(huge_decimal), digits) + # Ensuring that we chose a slow enough conversion to measure. + # It takes 0.1 seconds on a Zen based cloud VM in an opt build. + # Some OSes have a low res 1/64s timer, skip if hard to measure. + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: + raise unittest.SkipTest('"slow" conversion took only ' + f'{sw_convert.seconds} seconds.') + + # We test with the limit almost at the size needed to check performance. + # The performant limit check is slightly fuzzy, give it a some room. + with support.adjust_int_max_str_digits(int(.995 * digits)): + with ( + self.assertRaises(ValueError) as err, + support.Stopwatch() as sw_fail_huge): + str(huge_int) + self.assertIn('conversion', str(err.exception)) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) + + # Now we test that a conversion that would take 30x as long also fails + # in a similarly fast fashion. + extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits. + with ( + self.assertRaises(ValueError) as err, + support.Stopwatch() as sw_fail_extra_huge): + # If not limited, 8 seconds said Zen based cloud VM. + str(extra_huge_int) + self.assertIn('conversion', str(err.exception)) + self.assertLess(sw_fail_extra_huge.seconds, sw_convert.seconds/2) + + @unittest.skip("TODO: RUSTPYTHON; flaky test") + def test_denial_of_service_prevented_str_to_int(self): + """Regression test: ensure we fail before performing O(N**2) work.""" + maxdigits = sys.get_int_max_str_digits() + assert maxdigits < 100_000, maxdigits # A test prerequisite. + + digits = 133700 + huge = '8'*digits + with ( + support.adjust_int_max_str_digits(digits), + support.Stopwatch() as sw_convert): + int(huge) + # Ensuring that we chose a slow enough conversion to measure. + # It takes 0.1 seconds on a Zen based cloud VM in an opt build. + # Some OSes have a low res 1/64s timer, skip if hard to measure. + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: + raise unittest.SkipTest('"slow" conversion took only ' + f'{sw_convert.seconds} seconds.') + + with support.adjust_int_max_str_digits(digits - 1): + with ( + self.assertRaises(ValueError) as err, + support.Stopwatch() as sw_fail_huge): + int(huge) + self.assertIn('conversion', str(err.exception)) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) + + # Now we test that a conversion that would take 30x as long also fails + # in a similarly fast fashion. + extra_huge = '7'*1_200_000 + with ( + self.assertRaises(ValueError) as err, + support.Stopwatch() as sw_fail_extra_huge): + # If not limited, 8 seconds in the Zen based cloud VM. + int(extra_huge) + self.assertIn('conversion', str(err.exception)) + self.assertLessEqual(sw_fail_extra_huge.seconds, sw_convert.seconds/2) + + def test_power_of_two_bases_unlimited(self): + """The limit does not apply to power of 2 bases.""" + maxdigits = sys.get_int_max_str_digits() + + for base in (2, 4, 8, 16, 32): + with self.subTest(base=base): + self.int_class('1' * (maxdigits + 1), base) + assert maxdigits < 100_000 + self.int_class('1' * 100_000, base) + + def test_underscores_ignored(self): + maxdigits = sys.get_int_max_str_digits() + + triples = maxdigits // 3 + s = '111' * triples + s_ = '1_11' * triples + self.int_class(s) # succeeds + self.int_class(s_) # succeeds + self.check(f'{s}111') + self.check(f'{s_}_111') + + def test_sign_not_counted(self): + int_class = self.int_class + max_digits = sys.get_int_max_str_digits() + s = '5' * max_digits + i = int_class(s) + pos_i = int_class(f'+{s}') + assert i == pos_i + neg_i = int_class(f'-{s}') + assert -pos_i == neg_i + str(pos_i) + str(neg_i) + + def _other_base_helper(self, base): + int_class = self.int_class + max_digits = sys.get_int_max_str_digits() + s = '2' * max_digits + i = int_class(s, base) + if base > 10: + with self.assertRaises(ValueError): + str(i) + elif base < 10: + str(i) + with self.assertRaises(ValueError) as err: + int_class(f'{s}1', base) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_int_from_other_bases(self): + base = 3 + with self.subTest(base=base): + self._other_base_helper(base) + base = 36 + with self.subTest(base=base): + self._other_base_helper(base) + + def test_int_max_str_digits_is_per_interpreter(self): + # Changing the limit in one interpreter does not change others. + code = """if 1: + # Subinterpreters maintain and enforce their own limit + import sys + sys.set_int_max_str_digits(2323) + try: + int('3'*3333) + except ValueError: + pass + else: + raise AssertionError('Expected a int max str digits ValueError.') + """ + with support.adjust_int_max_str_digits(4000): + before_value = sys.get_int_max_str_digits() + self.assertEqual(support.run_in_subinterp(code), 0, + 'subinterp code failure, check stderr.') + after_value = sys.get_int_max_str_digits() + self.assertEqual(before_value, after_value) + + +class IntSubclassStrDigitLimitsTests(IntStrDigitLimitsTests): + int_class = IntSubclass + + +class PyLongModuleTests(unittest.TestCase): + # Tests of the functions in _pylong.py. Those get used when the + # number of digits in the input values are large enough. + + def setUp(self): + super().setUp() + self._previous_limit = sys.get_int_max_str_digits() + sys.set_int_max_str_digits(0) + + def tearDown(self): + sys.set_int_max_str_digits(self._previous_limit) + super().tearDown() + + def _test_pylong_int_to_decimal(self, n, suffix): + s = str(n) + self.assertEqual(s[-10:], suffix) + s2 = str(-n) + self.assertEqual(s2, '-' + s) + s3 = '%d' % n + self.assertEqual(s3, s) + s4 = b'%d' % n + self.assertEqual(s4, s.encode('ascii')) + + def test_pylong_int_to_decimal(self): + self._test_pylong_int_to_decimal((1 << 100_000), '9883109376') + self._test_pylong_int_to_decimal((1 << 100_000) - 1, '9883109375') + self._test_pylong_int_to_decimal(10**30_000, '0000000000') + self._test_pylong_int_to_decimal(10**30_000 - 1, '9999999999') + self._test_pylong_int_to_decimal(3**60_000, '9313200001') + + @support.requires_resource('cpu') + def test_pylong_int_to_decimal_2(self): + self._test_pylong_int_to_decimal(2**1_000_000, '2747109376') + self._test_pylong_int_to_decimal(10**300_000, '0000000000') + self._test_pylong_int_to_decimal(3**600_000, '3132000001') + + def test_pylong_int_divmod(self): + n = (1 << 100_000) + a, b = divmod(n*3 + 1, n) + assert a == 3 and b == 1 + + @support.cpython_only # tests implementation details of CPython. + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_int_divmod_crash(self): + # Regression test for https://github.com/python/cpython/issues/142554. + bad_int_divmod = lambda a, b: (1,) + # 'k' chosen such that divmod(2**(2*k), 2**k) uses _pylong.int_divmod() + k = 10_000 + a, b = (1 << (2 * k)), (1 << k) + with mock.patch.object(_pylong, "int_divmod", wraps=bad_int_divmod): + msg = r"tuple of length 2 is required from int_divmod\(\)" + self.assertRaisesRegex(ValueError, msg, divmod, a, b) + + def test_pylong_str_to_int(self): + v1 = 1 << 100_000 + s = str(v1) + v2 = int(s) + assert v1 == v2 + v3 = int(' -' + s) + assert -v1 == v3 + v4 = int(' +' + s + ' ') + assert v1 == v4 + with self.assertRaises(ValueError) as err: + int(s + 'z') + with self.assertRaises(ValueError) as err: + int(s + '_') + with self.assertRaises(ValueError) as err: + int('_' + s) + + @support.cpython_only # tests implementation details of CPython. + @unittest.skipUnless(_pylong, "_pylong module required") + #@mock.patch.object(_pylong, "int_to_decimal_string") # NOTE(RUSTPYTHON): See comment at top of file + def test_pylong_misbehavior_error_path_to_str( + self, mock_int_to_str): + with support.adjust_int_max_str_digits(20_000): + big_value = int('7'*19_999) + mock_int_to_str.return_value = None # not a str + with self.assertRaises(TypeError) as ctx: + str(big_value) + self.assertIn('_pylong.int_to_decimal_string did not', + str(ctx.exception)) + mock_int_to_str.side_effect = RuntimeError("testABC") + with self.assertRaises(RuntimeError): + str(big_value) + + @support.cpython_only # tests implementation details of CPython. + @unittest.skipUnless(_pylong, "_pylong module required") + #@mock.patch.object(_pylong, "int_from_string") # NOTE(RUSTPYTHON): See comment at top of file + def test_pylong_misbehavior_error_path_from_str( + self, mock_int_from_str): + big_value = '7'*19_999 + with support.adjust_int_max_str_digits(20_000): + mock_int_from_str.return_value = b'not an int' + with self.assertRaises(TypeError) as ctx: + int(big_value) + self.assertIn('_pylong.int_from_string did not', + str(ctx.exception)) + + mock_int_from_str.side_effect = RuntimeError("test123") + with self.assertRaises(RuntimeError): + int(big_value) + + def test_pylong_roundtrip(self): + from random import randrange, getrandbits + bits = 5000 + while bits <= 1_000_000: + bits += randrange(-100, 101) # break bitlength patterns + hibit = 1 << (bits - 1) + n = hibit | getrandbits(bits - 1) + assert n.bit_length() == bits + sn = str(n) + self.assertNotStartsWith(sn, '0') + self.assertEqual(n, int(sn)) + bits <<= 1 + + @support.requires_resource('cpu') + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_pylong_roundtrip_huge(self): + # k blocks of 1234567890 + k = 1_000_000 # so 10 million digits in all + tentoten = 10**10 + n = 1234567890 * ((tentoten**k - 1) // (tentoten - 1)) + sn = "1234567890" * k + self.assertEqual(n, int(sn)) + self.assertEqual(sn, str(n)) + + @support.requires_resource('cpu') + @unittest.skipUnless(_pylong, "_pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_failsafe(self): + # While I believe the number of GUARD digits in this function is + # always enough so that no more than one correction step is ever + # needed, the code has a "failsafe" path that takes over if I'm + # wrong about that. We have no input that reaches that block. + # Here we test a contrived input that _does_ reach that block, + # provided the number of guard digits is reduced to 1. + sn = "9" * 2000156 + n = 10**len(sn) - 1 + orig_spread = _pylong._spread.copy() + _pylong._spread.clear() + try: + self.assertEqual(n, _pylong._dec_str_to_int_inner(sn, GUARD=1)) + self.assertIn(999, _pylong._spread) + finally: + _pylong._spread.clear() + _pylong._spread.update(orig_spread) + + @unittest.skipUnless(_pylong, "pylong module required") + @unittest.skipUnless(_decimal, "C _decimal module required") + def test_whitebox_dec_str_to_int_inner_monster(self): + # I don't think anyone has enough RAM to build a string long enough + # for this function to complain. So lie about the string length. + + class LyingStr(str): + def __len__(self): + return int((1 << 47) / _pylong._LOG_10_BASE_256) + + liar = LyingStr("42") + # We have to pass the liar directly to the complaining function. If we + # just try `int(liar)`, earlier layers will replace it with plain old + # "43". + # Embedding `len(liar)` into the f-string failed on the WASI testbot + # (don't know what that is): + # OverflowError: cannot fit 'int' into an index-sized integer + # So a random stab at worming around that. + self.assertRaisesRegex(ValueError, + f"^cannot convert string of len {liar.__len__()} to int$", + _pylong._dec_str_to_int_inner, + liar) + + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_compute_powers(self): + # Basic sanity tests. See end of _pylong.py for manual heavy tests. + def consumer(w, base, limit, need_hi): + seen = set() + need = set() + def inner(w): + if w <= limit or w in seen: + return + seen.add(w) + lo = w >> 1 + hi = w - lo + need.add(hi if need_hi else lo) + inner(lo) + inner(hi) + inner(w) + d = _pylong.compute_powers(w, base, limit, need_hi=need_hi) + self.assertEqual(d.keys(), need) + for k, v in d.items(): + self.assertEqual(v, base ** k) + + for base in 2, 5: + for need_hi in False, True: + for limit in 1, 11: + for w in range(250, 550): + consumer(w, base, limit, need_hi) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 2e09bd2e92b..497c4e6e224 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -28,7 +28,6 @@ import random import signal import sys -import sysconfig import textwrap import threading import time @@ -40,10 +39,9 @@ from test import support from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import import_helper -from test.support import os_helper -from test.support import threading_helper -from test.support import warnings_helper +from test.support import ( + import_helper, is_apple, os_helper, threading_helper, warnings_helper, +) from test.support.os_helper import FakePath import codecs @@ -66,27 +64,20 @@ def byteslike(*pos, **kw): class EmptyStruct(ctypes.Structure): pass -_cflags = sysconfig.get_config_var('CFLAGS') or '' -_config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' -MEMORY_SANITIZER = ( - '-fsanitize=memory' in _cflags or - '--with-memory-sanitizer' in _config_args -) - -ADDRESS_SANITIZER = ( - '-fsanitize=address' in _cflags -) - -# Does io.IOBase finalizer log the exception if the close() method fails? -# The exception is ignored silently by default in release build. -IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode) - def _default_chunk_size(): """Get the default TextIOWrapper chunk size""" with open(__file__, "r", encoding="latin-1") as f: return f._CHUNK_SIZE +requires_alarm = unittest.skipUnless( + hasattr(signal, "alarm"), "test requires signal.alarm()" +) + + +class BadIndex: + def __index__(self): + 1/0 class MockRawIOWithoutRead: """A RawIO implementation without read(), so as to exercise the default @@ -266,6 +257,27 @@ class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): UnsupportedOperation = pyio.UnsupportedOperation +class MockCharPseudoDevFileIO(MockFileIO): + # GH-95782 + # ftruncate() does not work on these special files (and CPython then raises + # appropriate exceptions), so truncate() does not have to be accounted for + # here. + def __init__(self, data): + super().__init__(data) + + def seek(self, *args): + return 0 + + def tell(self, *args): + return 0 + +class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): + pass + +class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): + pass + + class MockNonBlockWriterIO: def __init__(self): @@ -415,8 +427,8 @@ def test_invalid_operations(self): self.assertRaises(exc, fp.read) self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "wb") as fp: - self.assertRaises(exc, fp.read) - self.assertRaises(exc, fp.readline) + self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: self.assertRaises(exc, fp.read) self.assertRaises(exc, fp.readline) @@ -433,6 +445,26 @@ def test_invalid_operations(self): self.assertRaises(exc, fp.seek, 1, self.SEEK_CUR) self.assertRaises(exc, fp.seek, -1, self.SEEK_END) + @support.cpython_only + def test_startup_optimization(self): + # gh-132952: Test that `io` is not imported at startup and that the + # __module__ of UnsupportedOperation is set to "io". + assert_python_ok("-S", "-c", textwrap.dedent( + """ + import sys + assert "io" not in sys.modules + try: + sys.stdin.truncate() + except Exception as e: + typ = type(e) + assert typ.__module__ == "io", (typ, typ.__module__) + assert typ.__name__ == "UnsupportedOperation", (typ, typ.__name__) + else: + raise AssertionError("Expected UnsupportedOperation") + """ + )) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_optional_abilities(self): # Test for OSError when optional APIs are not supported # The purpose of this test is to try fileno(), reading, writing and @@ -485,57 +517,65 @@ class UnseekableWriter(self.MockUnseekableIO): (text_reader, "r"), (text_writer, "w"), (self.BytesIO, "rws"), (self.StringIO, "rws"), ) - for [test, abilities] in tests: - with self.subTest(test), test() as obj: - readable = "r" in abilities - self.assertEqual(obj.readable(), readable) - writable = "w" in abilities - self.assertEqual(obj.writable(), writable) - - if isinstance(obj, self.TextIOBase): - data = "3" - elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)): - data = b"3" - else: - self.fail("Unknown base class") - if "f" in abilities: - obj.fileno() - else: - self.assertRaises(OSError, obj.fileno) + def do_test(test, obj, abilities): + readable = "r" in abilities + self.assertEqual(obj.readable(), readable) + writable = "w" in abilities + self.assertEqual(obj.writable(), writable) - if readable: - obj.read(1) - obj.read() - else: - self.assertRaises(OSError, obj.read, 1) - self.assertRaises(OSError, obj.read) + if isinstance(obj, self.TextIOBase): + data = "3" + elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)): + data = b"3" + else: + self.fail("Unknown base class") - if writable: - obj.write(data) - else: - self.assertRaises(OSError, obj.write, data) - - if sys.platform.startswith("win") and test in ( - pipe_reader, pipe_writer): - # Pipes seem to appear as seekable on Windows - continue - seekable = "s" in abilities - self.assertEqual(obj.seekable(), seekable) - - if seekable: - obj.tell() - obj.seek(0) - else: - self.assertRaises(OSError, obj.tell) - self.assertRaises(OSError, obj.seek, 0) + if "f" in abilities: + obj.fileno() + else: + self.assertRaises(OSError, obj.fileno) + + if readable: + obj.read(1) + obj.read() + else: + self.assertRaises(OSError, obj.read, 1) + self.assertRaises(OSError, obj.read) + + if writable: + obj.write(data) + else: + self.assertRaises(OSError, obj.write, data) + + if sys.platform.startswith("win") and test in ( + pipe_reader, pipe_writer): + # Pipes seem to appear as seekable on Windows + return + seekable = "s" in abilities + self.assertEqual(obj.seekable(), seekable) + + if seekable: + obj.tell() + obj.seek(0) + else: + self.assertRaises(OSError, obj.tell) + self.assertRaises(OSError, obj.seek, 0) + + if writable and seekable: + obj.truncate() + obj.truncate(0) + else: + self.assertRaises(OSError, obj.truncate) + self.assertRaises(OSError, obj.truncate, 0) + + for [test, abilities] in tests: + with self.subTest(test): + if test == pipe_writer and not threading_helper.can_start_thread: + self.skipTest("Need threads") + with test() as obj: + do_test(test, obj, abilities) - if writable and seekable: - obj.truncate() - obj.truncate(0) - else: - self.assertRaises(OSError, obj.truncate) - self.assertRaises(OSError, obj.truncate, 0) def test_open_handles_NUL_chars(self): fn_with_NUL = 'foo\0bar' @@ -609,10 +649,10 @@ def test_raw_bytes_io(self): self.read_ops(f, True) def test_large_file_ops(self): - # On Windows and Mac OSX this test consumes large resources; It takes - # a long time to build the >2 GiB file and takes >2 GiB of disk space - # therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: support.requires( 'largefile', 'test requires %s bytes and a long time to run' % self.LARGE) @@ -623,11 +663,9 @@ def test_large_file_ops(self): def test_with_open(self): for bufsize in (0, 100): - f = None with self.open(os_helper.TESTFN, "wb", bufsize) as f: f.write(b"xxx") self.assertEqual(f.closed, True) - f = None try: with self.open(os_helper.TESTFN, "wb", bufsize) as f: 1/0 @@ -766,8 +804,8 @@ def test_closefd_attr(self): file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) self.assertEqual(file.buffer.raw.closefd, False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): # FileIO objects are collected, and collecting them flushes # all data to disk. @@ -857,6 +895,22 @@ def test_RawIOBase_read(self): self.assertEqual(rawio.read(2), None) self.assertEqual(rawio.read(2), b"") + def test_RawIOBase_read_bounds_checking(self): + # Make sure a `.readinto` call which returns a value outside + # (0, len(buffer)) raises. + class Misbehaved(self.RawIOBase): + def __init__(self, readinto_return) -> None: + self._readinto_return = readinto_return + def readinto(self, b): + return self._readinto_return + + with self.assertRaises(ValueError) as cm: + Misbehaved(2).read(1) + self.assertEqual(str(cm.exception), "readinto returned 2 outside buffer size 1") + for bad_size in (2147483647, sys.maxsize, -1, -1000): + with self.assertRaises(ValueError): + Misbehaved(bad_size).read() + def test_types_have_dict(self): test = ( self.IOBase(), @@ -866,7 +920,7 @@ def test_types_have_dict(self): self.BytesIO() ) for obj in test: - self.assertTrue(hasattr(obj, "__dict__")) + self.assertHasAttr(obj, "__dict__") def test_opener(self): with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: @@ -882,7 +936,7 @@ def test_bad_opener_negative_1(self): def badopener(fname, flags): return -1 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -1') def test_bad_opener_other_negative(self): @@ -890,9 +944,17 @@ def test_bad_opener_other_negative(self): def badopener(fname, flags): return -2 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -2') + def test_opener_invalid_fd(self): + # Check that OSError is raised with error code EBADF if the + # opener returns an invalid file descriptor (see gh-82212). + fd = os_helper.make_bad_fd() + with self.assertRaises(OSError) as cm: + self.open('foo', opener=lambda name, flags: fd) + self.assertEqual(cm.exception.errno, errno.EBADF) + def test_fileio_closefd(self): # Issue #4841 with self.open(__file__, 'rb') as f1, \ @@ -1018,11 +1080,41 @@ def flush(self): # Silence destructor error R.flush = lambda self: None + @threading_helper.requires_working_threading() + def test_write_readline_races(self): + # gh-134908: Concurrent iteration over a file caused races + thread_count = 2 + write_count = 100 + read_count = 100 + + def writer(file, barrier): + barrier.wait() + for _ in range(write_count): + file.write("x") + + def reader(file, barrier): + barrier.wait() + for _ in range(read_count): + for line in file: + self.assertEqual(line, "") + + with self.open(os_helper.TESTFN, "w+") as f: + barrier = threading.Barrier(thread_count + 1) + reader = threading.Thread(target=reader, args=(f, barrier)) + writers = [threading.Thread(target=writer, args=(f, barrier)) + for _ in range(thread_count)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(writers + [reader]): + pass + self.assertIsNone(cm.exc_type) + + self.assertEqual(os.stat(os_helper.TESTFN).st_size, + write_count * thread_count) + class CIOTest(IOTest): - # TODO: RUSTPYTHON, cyclic gc - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_IOBase_finalize(self): # Issue #12149: segmentation fault on _PyIOBase_finalize when both a # class which inherits IOBase and an object of this class are caught @@ -1040,11 +1132,110 @@ def close(self): del obj support.gc_collect() self.assertIsNone(wr(), wr) - - # TODO: RUSTPYTHON, AssertionError: filter ('', ResourceWarning) did not catch any warning - @unittest.expectedFailure - def test_destructor(self): - super().test_destructor(self) + +@support.cpython_only +class TestIOCTypes(unittest.TestCase): + def setUp(self): + _io = import_helper.import_module("_io") + self.types = [ + _io.BufferedRWPair, + _io.BufferedRandom, + _io.BufferedReader, + _io.BufferedWriter, + _io.BytesIO, + _io.FileIO, + _io.IncrementalNewlineDecoder, + _io.StringIO, + _io.TextIOWrapper, + _io._BufferedIOBase, + _io._BytesIOBuffer, + _io._IOBase, + _io._RawIOBase, + _io._TextIOBase, + ] + if sys.platform == "win32": + self.types.append(_io._WindowsConsoleIO) + self._io = _io + + def test_immutable_types(self): + for tp in self.types: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = "bar" + + def test_class_hierarchy(self): + def check_subs(types, base): + for tp in types: + with self.subTest(tp=tp, base=base): + self.assertIsSubclass(tp, base) + + def recursive_check(d): + for k, v in d.items(): + if isinstance(v, dict): + recursive_check(v) + elif isinstance(v, set): + check_subs(v, k) + else: + self.fail("corrupt test dataset") + + _io = self._io + hierarchy = { + _io._IOBase: { + _io._BufferedIOBase: { + _io.BufferedRWPair, + _io.BufferedRandom, + _io.BufferedReader, + _io.BufferedWriter, + _io.BytesIO, + }, + _io._RawIOBase: { + _io.FileIO, + }, + _io._TextIOBase: { + _io.StringIO, + _io.TextIOWrapper, + }, + }, + } + if sys.platform == "win32": + hierarchy[_io._IOBase][_io._RawIOBase].add(_io._WindowsConsoleIO) + + recursive_check(hierarchy) + + def test_subclassing(self): + _io = self._io + dataset = {k: True for k in self.types} + dataset[_io._BytesIOBuffer] = False + + for tp, is_basetype in dataset.items(): + with self.subTest(tp=tp, is_basetype=is_basetype): + name = f"{tp.__name__}_subclass" + bases = (tp,) + if is_basetype: + _ = type(name, bases, {}) + else: + msg = "not an acceptable base type" + with self.assertRaisesRegex(TypeError, msg): + _ = type(name, bases, {}) + + def test_disallow_instantiation(self): + _io = self._io + support.check_disallow_instantiation(self, _io._BytesIOBuffer) + + def test_stringio_setstate(self): + # gh-127182: Calling __setstate__() with invalid arguments must not crash + obj = self._io.StringIO() + with self.assertRaisesRegex( + TypeError, + 'initial_value must be str or None, not int', + ): + obj.__setstate__((1, '', 0, {})) + + obj.__setstate__((None, '', 0, {})) # should not crash + self.assertEqual(obj.getvalue(), '') + + obj.__setstate__(('', '', 0, {})) + self.assertEqual(obj.getvalue(), '') class PyIOTest(IOTest): pass @@ -1056,7 +1247,7 @@ class APIMismatchTest(unittest.TestCase): def test_RawIOBase_io_in_pyio_match(self): """Test that pyio RawIOBase class has all c RawIOBase methods""" mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__',)) + ignore=('__weakref__', '__static_attributes__')) self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') def test_RawIOBase_pyio_in_io_match(self): @@ -1133,10 +1324,7 @@ def test_error_through_destructor(self): with self.assertRaises(AttributeError): self.tp(rawio).xyzzy - if not IOBASE_EMITS_UNRAISABLE: - self.assertIsNone(cm.unraisable) - elif cm.unraisable is not None: - self.assertEqual(cm.unraisable.exc_type, OSError) + self.assertEqual(cm.unraisable.exc_type, OSError) def test_repr(self): raw = self.MockRawIO() @@ -1152,11 +1340,9 @@ def test_recursive_repr(self): # Issue #25455 raw = self.MockRawIO() b = self.tp(raw) - with support.swap_attr(raw, 'name', b): - try: + with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(b) # Should not crash - except RuntimeError: - pass def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush @@ -1237,6 +1423,28 @@ def test_readonly_attributes(self): with self.assertRaises(AttributeError): buf.raw = x + def test_pickling_subclass(self): + global MyBufferedIO + class MyBufferedIO(self.tp): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.raw.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + buf = MyBufferedIO(raw, tag='ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(buf, proto) + newbuf = pickle.loads(pickled) + self.assertEqual(newbuf.raw.getvalue(), b'data') + self.assertEqual(newbuf.tag, 'ham') + del MyBufferedIO + class SizeofTest: @@ -1463,6 +1671,7 @@ def test_read_all(self): self.assertEqual(b"abcdefg", bufio.read()) + @threading_helper.requires_working_threading() @support.requires_resource('cpu') def test_threads(self): try: @@ -1542,11 +1751,25 @@ def test_no_extraneous_read(self): def test_read_on_closed(self): # Issue #23796 - b = io.BufferedReader(io.BytesIO(b"12")) + b = self.BufferedReader(self.BytesIO(b"12")) b.read(1) b.close() - self.assertRaises(ValueError, b.peek) - self.assertRaises(ValueError, b.read1, 1) + with self.subTest('peek'): + self.assertRaises(ValueError, b.peek) + with self.subTest('read1'): + self.assertRaises(ValueError, b.read1, 1) + with self.subTest('read'): + self.assertRaises(ValueError, b.read) + with self.subTest('readinto'): + self.assertRaises(ValueError, b.readinto, bytearray()) + with self.subTest('readinto1'): + self.assertRaises(ValueError, b.readinto1, bytearray()) + with self.subTest('flush'): + self.assertRaises(ValueError, b.flush) + with self.subTest('truncate'): + self.assertRaises(ValueError, b.truncate) + with self.subTest('seek'): + self.assertRaises(ValueError, b.seek, 0) def test_truncate_on_read_only(self): rawio = self.MockFileIO(b"abc") @@ -1555,23 +1778,34 @@ def test_truncate_on_read_only(self): self.assertRaises(self.UnsupportedOperation, bufio.truncate) self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) + def test_tell_character_device_file(self): + # GH-95782 + # For the (former) bug in BufferedIO to manifest, the wrapped IO obj + # must be able to produce at least 2 bytes. + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.read(1), b"1") + self.assertEqual(buf.tell(), 0) + + def test_seek_character_device_file(self): + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.seek(1, io.SEEK_SET), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.read(1), b"1") + + # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, + # which means that the next seek() could return a negative offset if it + # does not sanity-check: + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedReaderTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO([b"abc"]) bufio = self.tp(rawio) @@ -1589,8 +1823,8 @@ def test_misbehaved_io_read(self): # checking this is not so easy. self.assertRaises(OSError, bufio.read, 10) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): # C BufferedReader objects are collected. # The Python version has __del__, so it ends into gc.garbage instead @@ -1607,36 +1841,24 @@ def test_garbage_collection(self): def test_args_error(self): # Issue #17275 with self.assertRaisesRegex(TypeError, "BufferedReader"): - self.tp(io.BytesIO(), 1024, 1024, 1024) + self.tp(self.BytesIO(), 1024, 1024, 1024) def test_bad_readinto_value(self): - rawio = io.BufferedReader(io.BytesIO(b"12")) + rawio = self.tp(self.BytesIO(b"12")) rawio.readinto = lambda buf: -1 bufio = self.tp(rawio) with self.assertRaises(OSError) as cm: bufio.readline() self.assertIsNone(cm.exception.__cause__) - # TODO: RUSTPYTHON, TypeError: 'bytes' object cannot be interpreted as an integer") - @unittest.expectedFailure def test_bad_readinto_type(self): - rawio = io.BufferedReader(io.BytesIO(b"12")) + rawio = self.tp(self.BytesIO(b"12")) rawio.readinto = lambda buf: b'' bufio = self.tp(rawio) with self.assertRaises(OSError) as cm: bufio.readline() self.assertIsInstance(cm.exception.__cause__, TypeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_flush_error_on_close(self): - super().test_flush_error_on_close() - - # TODO: RUSTPYTHON, AssertionError: UnsupportedOperation not raised by truncate - @unittest.expectedFailure - def test_truncate_on_read_only(self): # TODO: RUSTPYTHON, remove when this passes - super().test_truncate_on_read_only() # TODO: RUSTPYTHON, remove when this passes - class PyBufferedReaderTest(BufferedReaderTest): tp = pyio.BufferedReader @@ -1700,7 +1922,7 @@ def test_write_overflow(self): flushed = b"".join(writer._write_stack) # At least (total - 8) bytes were implicitly flushed, perhaps more # depending on the implementation. - self.assertTrue(flushed.startswith(contents[:-8]), flushed) + self.assertStartsWith(flushed, contents[:-8]) def check_writes(self, intermediate_func): # Lots of writes, test the flushed output is as expected. @@ -1745,8 +1967,6 @@ def _seekrel(bufio): def test_writes_and_truncates(self): self.check_writes(lambda bufio: bufio.truncate(bufio.tell())) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_non_blocking(self): raw = self.MockNonBlockWriterIO() bufio = self.tp(raw, 8) @@ -1772,10 +1992,10 @@ def test_write_non_blocking(self): self.assertEqual(bufio.write(b"ABCDEFGHI"), 9) s = raw.pop_written() # Previously buffered bytes were flushed - self.assertTrue(s.startswith(b"01234567A"), s) + self.assertStartsWith(s, b"01234567A") def test_write_and_rewind(self): - raw = io.BytesIO() + raw = self.BytesIO() bufio = self.tp(raw, 4) self.assertEqual(bufio.write(b"abcdef"), 6) self.assertEqual(bufio.tell(), 6) @@ -1854,6 +2074,7 @@ def test_truncate_after_write(self): f.truncate() self.assertEqual(f.tell(), buffer_size + 2) + @threading_helper.requires_working_threading() @support.requires_resource('cpu') def test_threads(self): try: @@ -1925,6 +2146,7 @@ def bad_write(b): self.assertRaises(OSError, b.close) # exception not swallowed self.assertTrue(b.closed) + @threading_helper.requires_working_threading() def test_slow_close_from_thread(self): # Issue #31976 rawio = self.SlowFlushRawIO() @@ -1941,19 +2163,6 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedWriterTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO() bufio = self.tp(rawio) @@ -1964,8 +2173,8 @@ def test_initialization(self): self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) self.assertRaises(ValueError, bufio.write, b"def") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): # C BufferedWriter objects are collected, and collecting them flushes # all data to disk. @@ -1986,12 +2195,7 @@ def test_garbage_collection(self): def test_args_error(self): # Issue #17275 with self.assertRaisesRegex(TypeError, "BufferedWriter"): - self.tp(io.BytesIO(), 1024, 1024, 1024) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_flush_error_on_close(self): - super().test_flush_error_on_close() + self.tp(self.BytesIO(), 1024, 1024, 1024) class PyBufferedWriterTest(BufferedWriterTest): @@ -2086,7 +2290,7 @@ def test_write(self): def test_peek(self): pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - self.assertTrue(pair.peek(3).startswith(b"abc")) + self.assertStartsWith(pair.peek(3), b"abc") self.assertEqual(pair.read(3), b"abc") def test_readable(self): @@ -2425,6 +2629,28 @@ def test_interleaved_read_write(self): f.flush() self.assertEqual(raw.getvalue(), b'a2c') + def test_read1_after_write(self): + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(1), b'b') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(), b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + # XXX: read(100) returns different numbers of bytes + # in Python and C implementations. + self.assertEqual(f.read1(100)[:3], b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + def test_interleaved_readline_write(self): with self.BytesIO(b'ab\ncdef\ng\n') as raw: with self.tp(raw) as f: @@ -2448,21 +2674,8 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedRandomTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): CBufferedReaderTest.test_garbage_collection(self) CBufferedWriterTest.test_garbage_collection(self) @@ -2470,12 +2683,7 @@ def test_garbage_collection(self): def test_args_error(self): # Issue #17275 with self.assertRaisesRegex(TypeError, "BufferedRandom"): - self.tp(io.BytesIO(), 1024, 1024, 1024) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_flush_error_on_close(self): - super().test_flush_error_on_close() + self.tp(self.BytesIO(), 1024, 1024, 1024) class PyBufferedRandomTest(BufferedRandomTest): @@ -2647,8 +2855,29 @@ def test_constructor(self): self.assertEqual(t.encoding, "utf-8") self.assertEqual(t.line_buffering, True) self.assertEqual("\xe9\n", t.readline()) - self.assertRaises(TypeError, t.__init__, b, encoding="utf-8", newline=42) - self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') + invalid_type = TypeError if self.is_C else ValueError + with self.assertRaises(invalid_type): + t.__init__(b, encoding=42) + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding='utf-8\0') + with self.assertRaises(invalid_type): + t.__init__(b, encoding="utf-8", errors=42) + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding="utf-8", errors='\udcfe') + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", errors='replace\0') + with self.assertRaises(TypeError): + t.__init__(b, encoding="utf-8", newline=42) + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\n\0') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='xyzzy') def test_uninitialized(self): t = self.TextIOWrapper.__new__(self.TextIOWrapper) @@ -2714,11 +2943,16 @@ def test_recursive_repr(self): # Issue #25455 raw = self.BytesIO() t = self.TextIOWrapper(raw, encoding="utf-8") - with support.swap_attr(raw, 'name', t): - try: + with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(t) # Should not crash - except RuntimeError: - pass + + def test_subclass_repr(self): + class TestSubclass(self.TextIOWrapper): + pass + + f = TestSubclass(self.StringIO()) + self.assertIn(TestSubclass.__name__, repr(f)) def test_line_buffering(self): r = self.BytesIO() @@ -2760,35 +2994,18 @@ def test_reconfigure_line_buffering(self): @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_default_encoding(self): - old_environ = dict(os.environ) - try: + with os_helper.EnvironmentVarGuard() as env: # try to get a user preferred encoding different than the current # locale encoding to check that TextIOWrapper() uses the current # locale encoding and not the user preferred encoding - for key in ('LC_ALL', 'LANG', 'LC_CTYPE'): - if key in os.environ: - del os.environ[key] + env.unset('LC_ALL', 'LANG', 'LC_CTYPE') - current_locale_encoding = locale.getpreferredencoding(False) + current_locale_encoding = locale.getencoding() b = self.BytesIO() with warnings.catch_warnings(): warnings.simplefilter("ignore", EncodingWarning) - t = self.TextIOWrapper(b) + t = self.TextIOWrapper(b) self.assertEqual(t.encoding, current_locale_encoding) - finally: - os.environ.clear() - os.environ.update(old_environ) - - @support.cpython_only - @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") - def test_device_encoding(self): - # Issue 15989 - import _testcapi - b = self.BytesIO() - b.fileno = lambda: _testcapi.INT_MAX + 1 - self.assertRaises(OverflowError, self.TextIOWrapper, b, encoding="locale") - b.fileno = lambda: _testcapi.UINT_MAX + 1 - self.assertRaises(OverflowError, self.TextIOWrapper, b, encoding="locale") def test_encoding(self): # Check the encoding attribute is always set, and valid @@ -2797,7 +3014,7 @@ def test_encoding(self): self.assertEqual(t.encoding, "utf-8") with warnings.catch_warnings(): warnings.simplefilter("ignore", EncodingWarning) - t = self.TextIOWrapper(b) + t = self.TextIOWrapper(b) self.assertIsNotNone(t.encoding) codecs.lookup(t.encoding) @@ -2964,10 +3181,7 @@ def test_error_through_destructor(self): with self.assertRaises(AttributeError): self.TextIOWrapper(rawio, encoding="utf-8").xyzzy - if not IOBASE_EMITS_UNRAISABLE: - self.assertIsNone(cm.unraisable) - elif cm.unraisable is not None: - self.assertEqual(cm.unraisable.exc_type, OSError) + self.assertEqual(cm.unraisable.exc_type, OSError) # Systematic tests of the text I/O API @@ -3117,8 +3331,7 @@ def test_seek_and_tell_with_data(data, min_pos=0): finally: StatefulIncrementalDecoder.codecEnabled = 0 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jp def test_multibyte_seek_and_tell(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jp") f.write("AB\n\u3046\u3048\n") @@ -3134,8 +3347,24 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") @@ -3149,8 +3378,6 @@ def test_seek_with_encoder_state(self): self.assertEqual(f.readline(), "\u00e6\u0300\u0300") f.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encoded_writes(self): data = "1234567890" tests = ("utf-16", @@ -3289,8 +3516,6 @@ def test_issue2282(self): self.assertEqual(buffer.seekable(), txt.seekable()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_append_bom(self): # The BOM is not written again when appending to a non-empty file filename = os_helper.TESTFN @@ -3306,8 +3531,6 @@ def test_append_bom(self): with self.open(filename, 'rb') as f: self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_seek_bom(self): # Same test, but when seeking manually filename = os_helper.TESTFN @@ -3323,8 +3546,6 @@ def test_seek_bom(self): with self.open(filename, 'rb') as f: self.assertEqual(f.read(), 'bbbzzz'.encode(charset)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_seek_append_bom(self): # Same test, but first seek to the start and then to the end filename = os_helper.TESTFN @@ -3345,6 +3566,7 @@ def test_errors_property(self): self.assertEqual(f.errors, "replace") @support.no_tracing + @threading_helper.requires_working_threading() def test_threads_write(self): # Issue6750: concurrent writes could duplicate data event = threading.Event() @@ -3529,7 +3751,7 @@ def test_illegal_encoder(self): # encode() is invalid shouldn't cause an assertion failure. rot13 = codecs.lookup("rot13") with support.swap_attr(rot13, '_is_text_encoding', True): - t = io.TextIOWrapper(io.BytesIO(b'foo'), encoding="rot13") + t = self.TextIOWrapper(self.BytesIO(b'foo'), encoding="rot13") self.assertRaises(TypeError, t.write, 'bar') def test_illegal_decoder(self): @@ -3584,17 +3806,13 @@ def _check_create_at_shutdown(self, **kwargs): codecs.lookup('utf-8') class C: - def __init__(self): - self.buf = io.BytesIO() def __del__(self): - io.TextIOWrapper(self.buf, **{kwargs}) + io.TextIOWrapper(io.BytesIO(), **{kwargs}) print("ok") c = C() """.format(iomod=iomod, kwargs=kwargs) return assert_python_ok("-c", code) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_create_at_shutdown_without_encoding(self): rc, out, err = self._check_create_at_shutdown() if err: @@ -3604,8 +3822,6 @@ def test_create_at_shutdown_without_encoding(self): else: self.assertEqual("ok", out.decode().strip()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_create_at_shutdown_with_encoding(self): rc, out, err = self._check_create_at_shutdown(encoding='utf-8', errors='strict') @@ -3638,6 +3854,10 @@ def seekable(self): return True F.tell = lambda x: 0 t = self.TextIOWrapper(F(), encoding='utf-8') + def test_reconfigure_locale(self): + wrapper = self.TextIOWrapper(self.BytesIO(b"test")) + wrapper.reconfigure(encoding="locale") + def test_reconfigure_encoding_read(self): # latin1 -> utf8 # (latin1 can decode utf-8 encoded string) @@ -3719,6 +3939,59 @@ def test_reconfigure_defaults(self): self.assertEqual(txt.detach().getvalue(), b'LF\nCRLF\r\n') + def test_reconfigure_errors(self): + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\r') + with self.assertRaises(TypeError): # there was a crash + txt.reconfigure(encoding=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(encoding='\udcfe') + with self.assertRaises(LookupError): + txt.reconfigure(encoding='locale\0') + # TODO: txt.reconfigure(encoding='utf-8\0') + # TODO: txt.reconfigure(encoding='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(errors=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(errors='\udcfe') + # TODO: txt.reconfigure(errors='ignore\0') + # TODO: txt.reconfigure(errors='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(newline=42) + with self.assertRaises(ValueError): + txt.reconfigure(newline='\udcfe') + with self.assertRaises(ValueError): + txt.reconfigure(newline='xyz') + if not self.is_C: + # TODO: Should fail in C too. + with self.assertRaises(ValueError): + txt.reconfigure(newline='\n\0') + if self.is_C: + # TODO: Use __bool__(), not __index__(). + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(line_buffering=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(line_buffering=2**1000) + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(write_through=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(write_through=2**1000) + with self.assertRaises(ZeroDivisionError): # there was a crash + txt.reconfigure(line_buffering=BadIndex(), + write_through=BadIndex()) + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + self.assertIs(txt.line_buffering, False) + self.assertIs(txt.write_through, False) + + txt.reconfigure(encoding='latin1', errors='ignore', newline='\r\n', + line_buffering=True, write_through=True) + self.assertEqual(txt.encoding, 'latin1') + self.assertEqual(txt.errors, 'ignore') + self.assertIs(txt.line_buffering, True) + self.assertIs(txt.write_through, True) + def test_reconfigure_newline(self): raw = self.BytesIO(b'CR\rEOF') txt = self.TextIOWrapper(raw, 'ascii', newline='\n') @@ -3766,6 +4039,52 @@ def test_issue25862(self): t.write('x') t.tell() + def test_issue35928(self): + p = self.BufferedRWPair(self.BytesIO(b'foo\nbar\n'), self.BytesIO()) + f = self.TextIOWrapper(p) + res = f.readline() + self.assertEqual(res, 'foo\n') + f.write(res) + self.assertEqual(res + f.readline(), 'foo\nbar\n') + + def test_pickling_subclass(self): + global MyTextIO + class MyTextIO(self.TextIOWrapper): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.buffer.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + txt = MyTextIO(raw, 'ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(txt, proto) + newtxt = pickle.loads(pickled) + self.assertEqual(newtxt.buffer.getvalue(), b'data') + self.assertEqual(newtxt.tag, 'ham') + del MyTextIO + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_read_non_blocking(self): + import os + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.io.open(r, 'rt') as textfile: + r = None + # Nothing has been written so a non-blocking read raises a BlockingIOError exception. + with self.assertRaises(BlockingIOError): + textfile.read() + finally: + if r is not None: + os.close(r) + os.close(w) + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews @@ -3790,128 +4109,11 @@ class CTextIOWrapperTest(TextIOWrapperTest): io = io shutdown_error = "LookupError: unknown encoding: ascii" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_constructor(self): - super().test_constructor() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_detach(self): - super().test_detach() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_encoding_read(self): - super().test_reconfigure_encoding_read() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_line_buffering(self): - super().test_reconfigure_line_buffering() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_basic_io(self): - super().test_basic_io() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_telling(self): - super().test_telling() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_uninitialized(self): - super().test_uninitialized() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_non_text_encoding_codecs_are_rejected(self): - super().test_non_text_encoding_codecs_are_rejected() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_repr(self): - super().test_repr() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines(self): - super().test_newlines() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines_input(self): - super().test_newlines_input() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_read_one_by_one(self): - super().test_read_one_by_one() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_read_by_chunk(self): - super().test_read_by_chunk() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_1(self): - super().test_issue1395_1() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_2(self): - super().test_issue1395_2() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_3(self): - super().test_issue1395_3() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_4(self): - super().test_issue1395_4() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue1395_5(self): - super().test_issue1395_5() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write_through(self): - super().test_reconfigure_write_through() - - # TODO: RUSTPYTHON + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") @unittest.expectedFailure - def test_reconfigure_write_fromascii(self): - super().test_reconfigure_write_fromascii() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write(self): - super().test_reconfigure_write() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write_non_seekable(self): - super().test_reconfigure_write_non_seekable() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_defaults(self): - super().test_reconfigure_defaults() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_newline(self): - super().test_reconfigure_newline() + def test_read_non_blocking(self): + return super().test_read_non_blocking() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_initialization(self): r = self.BytesIO(b"\xc3\xa9\n\n") b = self.BufferedReader(r, 1000) @@ -3922,14 +4124,14 @@ def test_initialization(self): t = self.TextIOWrapper.__new__(self.TextIOWrapper) self.assertRaises(Exception, repr, t) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") def test_garbage_collection(self): # C TextIOWrapper objects are collected, and collecting them flushes # all data to disk. # The Python version has __del__, so it ends in gc.garbage instead. with warnings_helper.check_warnings(('', ResourceWarning)): - rawio = io.FileIO(os_helper.TESTFN, "wb") + rawio = self.FileIO(os_helper.TESTFN, "wb") b = self.BufferedWriter(rawio) t = self.TextIOWrapper(b, encoding="ascii") t.write("456def") @@ -3987,15 +4189,70 @@ def write(self, data): t.write("x"*chunk_size) self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + + def test_issue142594(self): + wrapper = None + detached = False + class ReentrantRawIO(self.RawIOBase): + @property + def closed(self): + nonlocal detached + if wrapper is not None and not detached: + detached = True + wrapper.detach() + return False + + raw = ReentrantRawIO() + wrapper = self.TextIOWrapper(raw) + wrapper.close() # should not crash + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jis_2004 + def test_seek_with_encoder_state(self): + return super().test_seek_with_encoder_state() + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'NoneType' + def test_read_non_blocking(self): + return super().test_read_non_blocking() + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio shutdown_error = "LookupError: unknown encoding: ascii" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines(self): - super().test_newlines() + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.set_blocking not available on Windows") + def test_read_non_blocking(self): + return super().test_read_non_blocking() + + @unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jis_2004 + def test_seek_with_encoder_state(self): + return super().test_seek_with_encoder_state() + + if sys.platform == "win32": + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @unittest.expectedFailure + def test_read_non_blocking(self): + return super().test_read_non_blocking() class IncrementalNewlineDecoderTest(unittest.TestCase): @@ -4075,8 +4332,6 @@ def _decode_bytewise(s): self.assertEqual(decoder.decode(input), "abc") self.assertEqual(decoder.newlines, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_newline_decoder(self): encodings = ( # None meaning the IncrementalNewlineDecoder takes unicode input @@ -4094,8 +4349,6 @@ def test_newline_decoder(self): self.check_newline_decoding_utf8(decoder) self.assertRaises(TypeError, decoder.setstate, 42) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_newline_bytes(self): # Issue 5433: Excessive optimization in IncrementalNewlineDecoder def _check(dec): @@ -4109,8 +4362,6 @@ def _check(dec): dec = self.IncrementalNewlineDecoder(None, translate=True) _check(dec) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_translate(self): # issue 35062 for translate in (-2, -1, 1, 2): @@ -4122,7 +4373,15 @@ def test_translate(self): self.assertEqual(decoder.decode(b"\r\r\n"), "\r\r\n") class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): - pass + @support.cpython_only + def test_uninitialized(self): + uninitialized = self.IncrementalNewlineDecoder.__new__( + self.IncrementalNewlineDecoder) + self.assertRaises(ValueError, uninitialized.decode, b'bar') + self.assertRaises(ValueError, uninitialized.getstate) + self.assertRaises(ValueError, uninitialized.setstate, (b'foo', 0)) + self.assertRaises(ValueError, uninitialized.reset) + class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): pass @@ -4132,38 +4391,24 @@ class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): class MiscIOTest(unittest.TestCase): + # for test__all__, actual values are set in subclasses + name_of_module = None + extra_exported = () + not_exported = () + def tearDown(self): os_helper.unlink(os_helper.TESTFN) def test___all__(self): - for name in self.io.__all__: - obj = getattr(self.io, name, None) - self.assertIsNotNone(obj, name) - if name in ("open", "open_code"): - continue - elif "error" in name.lower() or name == "UnsupportedOperation": - self.assertTrue(issubclass(obj, Exception), name) - elif not name.startswith("SEEK_"): - self.assertTrue(issubclass(obj, self.IOBase)) + support.check__all__(self, self.io, self.name_of_module, + extra=self.extra_exported, + not_exported=self.not_exported) def test_attributes(self): f = self.open(os_helper.TESTFN, "wb", buffering=0) self.assertEqual(f.mode, "wb") f.close() - # XXX RUSTPYTHON: universal mode is deprecated anyway, so I - # feel fine about skipping it - - # with warnings_helper.check_warnings(('', DeprecationWarning)): - # f = self.open(os_helper.TESTFN, "U", encoding="utf-8") - # self.assertEqual(f.name, os_helper.TESTFN) - # self.assertEqual(f.buffer.name, os_helper.TESTFN) - # self.assertEqual(f.buffer.raw.name, os_helper.TESTFN) - # self.assertEqual(f.mode, "U") - # self.assertEqual(f.buffer.mode, "rb") - # self.assertEqual(f.buffer.raw.mode, "rb") - # f.close() - f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") self.assertEqual(f.mode, "w+") self.assertEqual(f.buffer.mode, "rb+") # Does it really matter? @@ -4177,6 +4422,14 @@ def test_attributes(self): f.close() g.close() + def test_removed_u_mode(self): + # bpo-37330: The "U" mode has been removed in Python 3.11 + for mode in ("U", "rU", "r+U"): + with self.assertRaises(ValueError) as cm: + self.open(os_helper.TESTFN, mode) + self.assertIn('invalid mode', str(cm.exception)) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_open_pipe_with_append(self): # bpo-27805: Ignore ESPIPE from lseek() in open(). r, w = os.pipe() @@ -4237,8 +4490,7 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.writelines, []) self.assertRaises(ValueError, next, f) - # TODO: RUSTPYTHON, cyclic gc - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_blockingioerror(self): # Various BlockingIOError issues class C(str): @@ -4286,15 +4538,13 @@ def test_abc_inheritance_official(self): self._check_abc_inheritance(io) def _check_warn_on_dealloc(self, *args, **kwargs): - f = open(*args, **kwargs) + f = self.open(*args, **kwargs) r = repr(f) with self.assertWarns(ResourceWarning) as cm: f = None support.gc_collect() self.assertIn(r, str(cm.warning.args[0])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warn_on_dealloc(self): self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0) self._check_warn_on_dealloc(os_helper.TESTFN, "wb") @@ -4317,10 +4567,9 @@ def cleanup_fds(): r, w = os.pipe() fds += r, w with warnings_helper.check_no_resource_warning(self): - open(r, *args, closefd=False, **kwargs) + self.open(r, *args, closefd=False, **kwargs) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_warn_on_dealloc_fd(self): self._check_warn_on_dealloc_fd("rb", buffering=0) self._check_warn_on_dealloc_fd("rb") @@ -4329,6 +4578,7 @@ def test_warn_on_dealloc_fd(self): def test_pickling(self): # Pickling file objects is forbidden + msg = "cannot pickle" for kwargs in [ {"mode": "w"}, {"mode": "wb"}, @@ -4343,21 +4593,22 @@ def test_pickling(self): if "b" not in kwargs["mode"]: kwargs["encoding"] = "utf-8" for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - with self.open(os_helper.TESTFN, **kwargs) as f: - self.assertRaises(TypeError, pickle.dumps, f, protocol) + with self.subTest(protocol=protocol, kwargs=kwargs): + with self.open(os_helper.TESTFN, **kwargs) as f: + with self.assertRaisesRegex(TypeError, msg): + pickle.dumps(f, protocol) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") def test_nonblock_pipe_write_bigbuf(self): self._test_nonblock_pipe_write(16*1024) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") def test_nonblock_pipe_write_smallbuf(self): self._test_nonblock_pipe_write(1024) @unittest.skipUnless(hasattr(os, 'set_blocking'), 'os.set_blocking() required for this test') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def _test_nonblock_pipe_write(self, bufsize): sent = [] received = [] @@ -4470,8 +4721,6 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) - # TODO: RUSTPYTHON, AssertionError: 0 != 2 - @unittest.expectedFailure def test_check_encoding_warning(self): # PEP 597: Raise warning when encoding is not specified # and sys.flags.warn_default_encoding is set. @@ -4490,22 +4739,25 @@ def test_check_encoding_warning(self): proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) warnings = proc.err.splitlines() self.assertEqual(len(warnings), 2) - self.assertTrue( - warnings[0].startswith(b"<string>:5: EncodingWarning: ")) - self.assertTrue( - warnings[1].startswith(b"<string>:8: EncodingWarning: ")) + self.assertStartsWith(warnings[0], b"<string>:5: EncodingWarning: ") + self.assertStartsWith(warnings[1], b"<string>:8: EncodingWarning: ") - @support.cpython_only - # Depending if OpenWrapper was already created or not, the warning is - # emitted or not. For example, the attribute is already created when this - # test is run multiple times. - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_openwrapper(self): - self.assertIs(self.io.OpenWrapper, self.io.open) + def test_text_encoding(self): + # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" + # based on sys.flags.utf8_mode + code = "import io; print(io.text_encoding(None))" + + proc = assert_python_ok('-X', 'utf8=0', '-c', code) + self.assertEqual(b"locale", proc.out.strip()) + + proc = assert_python_ok('-X', 'utf8=1', '-c', code) + self.assertEqual(b"utf-8", proc.out.strip()) class CMiscIOTest(MiscIOTest): io = io + name_of_module = "io", "_io" + extra_exported = "BlockingIOError", def test_readinto_buffer_overflow(self): # Issue #18025 @@ -4557,20 +4809,22 @@ def run(): else: self.assertFalse(err.strip('.!')) + @threading_helper.requires_working_threading() + @support.requires_resource('walltime') def test_daemon_threads_shutdown_stdout_deadlock(self): self.check_daemon_threads_shutdown_deadlock('stdout') + @threading_helper.requires_working_threading() + @support.requires_resource('walltime') def test_daemon_threads_shutdown_stderr_deadlock(self): self.check_daemon_threads_shutdown_deadlock('stderr') - # TODO: RUSTPYTHON, AssertionError: 22 != 10 : _PythonRunResult(rc=22, out=b'', err=b'') - @unittest.expectedFailure - def test_check_encoding_errors(self): # TODO: RUSTPYTHON, remove when this passes - super().test_check_encoding_errors() # TODO: RUSTPYTHON, remove when this passes - class PyMiscIOTest(MiscIOTest): io = pyio + name_of_module = "_pyio", "io" + extra_exported = "BlockingIOError", "open_code", + not_exported = "valid_seek_flags", @unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.') @@ -4662,12 +4916,18 @@ def _read(): if e.errno != errno.EBADF: raise + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_interrupted_write_unbuffered(self): self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0) + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_interrupted_write_buffered(self): self.check_interrupted_write(b"xy", b"xy", mode="wb") + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_interrupted_write_text(self): self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") @@ -4693,15 +4953,17 @@ def on_alarm(*args): os.read(r, len(data) * 100) exc = cm.exception if isinstance(exc, RuntimeError): - self.assertTrue(str(exc).startswith("reentrant call"), str(exc)) + self.assertStartsWith(str(exc), "reentrant call") finally: signal.alarm(0) wio.close() os.close(r) + @requires_alarm def test_reentrant_write_buffered(self): self.check_reentrant_write(b"xy", mode="wb") + @requires_alarm def test_reentrant_write_text(self): self.check_reentrant_write("xy", mode="w", encoding="ascii") @@ -4729,14 +4991,14 @@ def alarm_handler(sig, frame): os.close(w) os.close(r) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @requires_alarm + @support.requires_resource('walltime') def test_interrupted_read_retry_buffered(self): self.check_interrupted_read_retry(lambda x: x.decode('latin1'), mode="rb") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @requires_alarm + @support.requires_resource('walltime') def test_interrupted_read_retry_text(self): self.check_interrupted_read_retry(lambda x: x, mode="r", encoding="latin1") @@ -4809,11 +5071,15 @@ def alarm2(sig, frame): if e.errno != errno.EBADF: raise - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'already borrowed: BorrowMutError'") + @unittest.skip("TODO: RUSTPYTHON; thread 'main' (103833) panicked at crates/vm/src/stdlib/signal.rs:233:43: RefCell already borrowed") + @requires_alarm + @support.requires_resource('walltime') def test_interrupted_write_retry_buffered(self): self.check_interrupted_write_retry(b"x", mode="wb") - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'already borrowed: BorrowMutError'") + @unittest.skip("TODO: RUSTPYTHON; thread 'main' (103833) panicked at crates/vm/src/stdlib/signal.rs:233:43: RefCell already borrowed") + @requires_alarm + @support.requires_resource('walltime') def test_interrupted_write_retry_text(self): self.check_interrupted_write_retry("x", mode="w", encoding="latin1") @@ -4821,6 +5087,10 @@ def test_interrupted_write_retry_text(self): class CSignalsTest(SignalsTest): io = io + @unittest.skip("TODO: RUSTPYTHON; thread 'main' (103833) panicked at crates/vm/src/stdlib/signal.rs:233:43: RefCell already borrowed") + def test_interrupted_read_retry_buffered(self): + return super().test_interrupted_read_retry_buffered() + class PySignalsTest(SignalsTest): io = pyio @@ -4830,7 +5100,25 @@ class PySignalsTest(SignalsTest): test_reentrant_write_text = None -def load_tests(*args): +class ProtocolsTest(unittest.TestCase): + class MyReader: + def read(self, sz=-1): + return b"" + + class MyWriter: + def write(self, b: bytes): + pass + + def test_reader_subclass(self): + self.assertIsSubclass(self.MyReader, io.Reader) + self.assertNotIsSubclass(str, io.Reader) + + def test_writer_subclass(self): + self.assertIsSubclass(self.MyWriter, io.Writer) + self.assertNotIsSubclass(str, io.Writer) + + +def load_tests(loader, tests, pattern): tests = (CIOTest, PyIOTest, APIMismatchTest, CBufferedReaderTest, PyBufferedReaderTest, CBufferedWriterTest, PyBufferedWriterTest, @@ -4840,32 +5128,34 @@ def load_tests(*args): CIncrementalNewlineDecoderTest, PyIncrementalNewlineDecoderTest, CTextIOWrapperTest, PyTextIOWrapperTest, CMiscIOTest, PyMiscIOTest, - CSignalsTest, PySignalsTest, + CSignalsTest, PySignalsTest, TestIOCTypes, + ProtocolsTest, ) # Put the namespaces of the IO module we are testing and some useful mock # classes in the __dict__ of each test. mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO, MockNonBlockWriterIO, MockUnseekableIO, MockRawIOWithoutRead, - SlowFlushRawIO) - all_members = io.__all__# + ["IncrementalNewlineDecoder"] XXX RUSTPYTHON + SlowFlushRawIO, MockCharPseudoDevFileIO) + all_members = io.__all__ c_io_ns = {name : getattr(io, name) for name in all_members} py_io_ns = {name : getattr(pyio, name) for name in all_members} globs = globals() c_io_ns.update((x.__name__, globs["C" + x.__name__]) for x in mocks) py_io_ns.update((x.__name__, globs["Py" + x.__name__]) for x in mocks) - # TODO: RUSTPYTHON (need to update io.py, see bpo-43680) - # Avoid turning open into a bound method. - py_io_ns["open"] = pyio.OpenWrapper for test in tests: if test.__name__.startswith("C"): for name, obj in c_io_ns.items(): setattr(test, name, obj) + test.is_C = True elif test.__name__.startswith("Py"): for name, obj in py_io_ns.items(): setattr(test, name, obj) + test.is_C = False - suite = unittest.TestSuite([unittest.makeSuite(test) for test in tests]) + suite = loader.suiteClass() + for test in tests: + suite.addTest(loader.loadTestsFromTestCase(test)) return suite if __name__ == "__main__": diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index a5388b2e5de..8af91e857d8 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -4,6 +4,7 @@ """Unittest for ipaddress module.""" +import copy import unittest import re import contextlib @@ -11,6 +12,7 @@ import pickle import ipaddress import weakref +from collections.abc import Iterator from test.support import LARGEST, SMALLEST @@ -302,6 +304,14 @@ def test_pickle(self): def test_weakref(self): weakref.ref(self.factory('192.0.2.1')) + def test_ipv6_mapped(self): + self.assertEqual(ipaddress.IPv4Address('192.168.1.1').ipv6_mapped, + ipaddress.IPv6Address('::ffff:192.168.1.1')) + self.assertEqual(ipaddress.IPv4Address('192.168.1.1').ipv6_mapped, + ipaddress.IPv6Address('::ffff:c0a8:101')) + self.assertEqual(ipaddress.IPv4Address('192.168.1.1').ipv6_mapped.ipv4_mapped, + ipaddress.IPv4Address('192.168.1.1')) + class AddressTestCase_v6(BaseTestCase, CommonTestMixin_v6): factory = ipaddress.IPv6Address @@ -388,6 +398,19 @@ def assertBadSplit(addr): # A trailing IPv4 address is two parts assertBadSplit("10:9:8:7:6:5:4:3:42.42.42.42%scope") + def test_bad_address_split_v6_too_long(self): + def assertBadSplit(addr): + msg = r"At most 45 characters expected in '%s" + with self.assertAddressError(msg, re.escape(addr[:45])): + ipaddress.IPv6Address(addr) + + # Long IPv6 address + long_addr = ("0:" * 10000) + "0" + assertBadSplit(long_addr) + assertBadSplit(long_addr + "%zoneid") + assertBadSplit(long_addr + ":255.255.255.255") + assertBadSplit(long_addr + ":ffff:255.255.255.255") + def test_bad_address_split_v6_too_many_parts(self): def assertBadSplit(addr): msg = "Exactly 8 parts expected without '::' in %r" @@ -542,11 +565,17 @@ def assertBadPart(addr, part): def test_pickle(self): self.pickle_test('2001:db8::') + self.pickle_test('2001:db8::%scope') def test_weakref(self): weakref.ref(self.factory('2001:db8::')) weakref.ref(self.factory('2001:db8::%scope')) + def test_copy(self): + addr = self.factory('2001:db8::%scope') + self.assertEqual(addr, copy.copy(addr)) + self.assertEqual(addr, copy.deepcopy(addr)) + class NetmaskTestMixin_v4(CommonTestMixin_v4): """Input validation on interfaces and networks is very similar""" @@ -879,8 +908,8 @@ class ComparisonTests(unittest.TestCase): v6net = ipaddress.IPv6Network(1) v6intf = ipaddress.IPv6Interface(1) v6addr_scoped = ipaddress.IPv6Address('::1%scope') - v6net_scoped= ipaddress.IPv6Network('::1%scope') - v6intf_scoped= ipaddress.IPv6Interface('::1%scope') + v6net_scoped = ipaddress.IPv6Network('::1%scope') + v6intf_scoped = ipaddress.IPv6Interface('::1%scope') v4_addresses = [v4addr, v4intf] v4_objects = v4_addresses + [v4net] @@ -1068,6 +1097,7 @@ def setUp(self): self.ipv6_scoped_interface = ipaddress.IPv6Interface( '2001:658:22a:cafe:200:0:0:1%scope/64') self.ipv6_scoped_network = ipaddress.IPv6Network('2001:658:22a:cafe::%scope/64') + self.ipv6_with_ipv4_part = ipaddress.IPv6Interface('::1.2.3.4') def testRepr(self): self.assertEqual("IPv4Interface('1.2.3.4/32')", @@ -1321,6 +1351,17 @@ def testGetIp(self): self.assertEqual(str(self.ipv6_scoped_interface.ip), '2001:658:22a:cafe:200::1') + def testIPv6IPv4MappedStringRepresentation(self): + long_prefix = '0000:0000:0000:0000:0000:ffff:' + short_prefix = '::ffff:' + ipv4 = '1.2.3.4' + ipv6_ipv4_str = short_prefix + ipv4 + ipv6_ipv4_addr = ipaddress.IPv6Address(ipv6_ipv4_str) + ipv6_ipv4_iface = ipaddress.IPv6Interface(ipv6_ipv4_str) + self.assertEqual(str(ipv6_ipv4_addr), ipv6_ipv4_str) + self.assertEqual(ipv6_ipv4_addr.exploded, long_prefix + ipv4) + self.assertEqual(str(ipv6_ipv4_iface.ip), ipv6_ipv4_str) + def testGetScopeId(self): self.assertEqual(self.ipv6_address.scope_id, None) @@ -1432,18 +1473,27 @@ def testGetSupernet4(self): self.ipv6_scoped_network.supernet(new_prefix=62)) def testHosts(self): + hosts = self.ipv4_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), next(hosts)) hosts = list(self.ipv4_network.hosts()) self.assertEqual(254, len(hosts)) self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = ipv6_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), next(hosts)) hosts = list(ipv6_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) ipv6_scoped_network = ipaddress.IPv6Network('2001:658:22a:cafe::%scope/120') + hosts = ipv6_scoped_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual((ipaddress.IPv6Address('2001:658:22a:cafe::1')), next(hosts)) hosts = list(ipv6_scoped_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) @@ -1454,6 +1504,12 @@ def testHosts(self): ipaddress.IPv4Address('2.0.0.1')] str_args = '2.0.0.0/31' tpl_args = ('2.0.0.0', 31) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1463,6 +1519,12 @@ def testHosts(self): addrs = [ipaddress.IPv4Address('1.2.3.4')] str_args = '1.2.3.4/32' tpl_args = ('1.2.3.4', 32) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1472,6 +1534,12 @@ def testHosts(self): ipaddress.IPv6Address('2001:658:22a:cafe::1')] str_args = '2001:658:22a:cafe::/127' tpl_args = ('2001:658:22a:cafe::', 127) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1480,6 +1548,12 @@ def testHosts(self): addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::1'), ] str_args = '2001:658:22a:cafe::1/128' tpl_args = ('2001:658:22a:cafe::1', 128) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1687,6 +1761,8 @@ def testEqual(self): self.assertTrue(self.ipv6_scoped_interface == ipaddress.IPv6Interface('2001:658:22a:cafe:200::1%scope/64')) + self.assertTrue(self.ipv6_with_ipv4_part == + ipaddress.IPv6Interface('0000:0000:0000:0000:0000:0000:0102:0304')) self.assertFalse(self.ipv6_scoped_interface == ipaddress.IPv6Interface('2001:658:22a:cafe:200::1%scope/63')) self.assertFalse(self.ipv6_scoped_interface == @@ -2149,6 +2225,11 @@ def testIPv6AddressTooLarge(self): self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1'), ipaddress.ip_address('FFFF::c000:201')) + self.assertEqual(ipaddress.ip_address('0000:0000:0000:0000:0000:FFFF:192.168.255.255'), + ipaddress.ip_address('::ffff:c0a8:ffff')) + self.assertEqual(ipaddress.ip_address('FFFF:0000:0000:0000:0000:0000:192.168.255.255'), + ipaddress.ip_address('ffff::c0a8:ffff')) + self.assertEqual(ipaddress.ip_address('::FFFF:192.0.2.1%scope'), ipaddress.ip_address('::FFFF:c000:201%scope')) self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1%scope'), @@ -2161,13 +2242,24 @@ def testIPv6AddressTooLarge(self): ipaddress.ip_address('::FFFF:c000:201%scope')) self.assertNotEqual(ipaddress.ip_address('FFFF::192.0.2.1'), ipaddress.ip_address('FFFF::c000:201%scope')) + self.assertEqual(ipaddress.ip_address('0000:0000:0000:0000:0000:FFFF:192.168.255.255%scope'), + ipaddress.ip_address('::ffff:c0a8:ffff%scope')) + self.assertEqual(ipaddress.ip_address('FFFF:0000:0000:0000:0000:0000:192.168.255.255%scope'), + ipaddress.ip_address('ffff::c0a8:ffff%scope')) def testIPVersion(self): + self.assertEqual(ipaddress.IPv4Address.version, 4) + self.assertEqual(ipaddress.IPv6Address.version, 6) + self.assertEqual(self.ipv4_address.version, 4) self.assertEqual(self.ipv6_address.version, 6) self.assertEqual(self.ipv6_scoped_address.version, 6) + self.assertEqual(self.ipv6_with_ipv4_part.version, 6) def testMaxPrefixLength(self): + self.assertEqual(ipaddress.IPv4Address.max_prefixlen, 32) + self.assertEqual(ipaddress.IPv6Address.max_prefixlen, 128) + self.assertEqual(self.ipv4_interface.max_prefixlen, 32) self.assertEqual(self.ipv6_interface.max_prefixlen, 128) self.assertEqual(self.ipv6_scoped_interface.max_prefixlen, 128) @@ -2262,6 +2354,10 @@ def testReservedIpv4(self): self.assertEqual(True, ipaddress.ip_address( '172.31.255.255').is_private) self.assertEqual(False, ipaddress.ip_address('172.32.0.0').is_private) + self.assertFalse(ipaddress.ip_address('192.0.0.0').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.9').is_global) + self.assertTrue(ipaddress.ip_address('192.0.0.10').is_global) + self.assertFalse(ipaddress.ip_address('192.0.0.255').is_global) self.assertEqual(True, ipaddress.ip_address('169.254.100.200').is_link_local) @@ -2287,6 +2383,7 @@ def testPrivateNetworks(self): self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) + self.assertEqual(False, ipaddress.ip_network("192.0.0.9/32").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) @@ -2303,8 +2400,8 @@ def testPrivateNetworks(self): self.assertEqual(True, ipaddress.ip_network("::/128").is_private) self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) - self.assertEqual(True, ipaddress.ip_network("2001::/23").is_private) self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) + self.assertEqual(False, ipaddress.ip_network("2001:3::/48").is_private) self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) @@ -2383,6 +2480,22 @@ def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_address('0::0').is_unspecified) self.assertEqual(False, ipaddress.ip_address('::1').is_unspecified) + self.assertFalse(ipaddress.ip_address('64:ff9b:1::').is_global) + self.assertFalse(ipaddress.ip_address('2001::').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::1').is_global) + self.assertTrue(ipaddress.ip_address('2001:1::2').is_global) + self.assertFalse(ipaddress.ip_address('2001:2::').is_global) + self.assertTrue(ipaddress.ip_address('2001:3::').is_global) + self.assertFalse(ipaddress.ip_address('2001:4::').is_global) + self.assertTrue(ipaddress.ip_address('2001:4:112::').is_global) + self.assertFalse(ipaddress.ip_address('2001:10::').is_global) + self.assertTrue(ipaddress.ip_address('2001:20::').is_global) + self.assertTrue(ipaddress.ip_address('2001:30::').is_global) + self.assertFalse(ipaddress.ip_address('2001:40::').is_global) + self.assertFalse(ipaddress.ip_address('2002::').is_global) + # gh-124217: conform with RFC 9637 + self.assertFalse(ipaddress.ip_address('3fff::').is_global) + # some generic IETF reserved addresses self.assertEqual(True, ipaddress.ip_address('100::').is_reserved) self.assertEqual(True, ipaddress.ip_network('4000::1/128').is_reserved) @@ -2395,12 +2508,52 @@ def testIpv4Mapped(self): self.assertEqual(ipaddress.ip_address('::ffff:c0a8:101').ipv4_mapped, ipaddress.ip_address('192.168.1.1')) + def testIpv4MappedProperties(self): + # Test that an IPv4 mapped IPv6 address has + # the same properties as an IPv4 address. + for addr4 in ( + "178.62.3.251", # global + "169.254.169.254", # link local + "127.0.0.1", # loopback + "224.0.0.1", # multicast + "192.168.0.1", # private + "0.0.0.0", # unspecified + "100.64.0.1", # public and not global + ): + with self.subTest(addr4): + ipv4 = ipaddress.IPv4Address(addr4) + ipv6 = ipaddress.IPv6Address(f"::ffff:{addr4}") + + self.assertEqual(ipv4.is_global, ipv6.is_global) + self.assertEqual(ipv4.is_private, ipv6.is_private) + self.assertEqual(ipv4.is_reserved, ipv6.is_reserved) + self.assertEqual(ipv4.is_multicast, ipv6.is_multicast) + self.assertEqual(ipv4.is_unspecified, ipv6.is_unspecified) + self.assertEqual(ipv4.is_link_local, ipv6.is_link_local) + self.assertEqual(ipv4.is_loopback, ipv6.is_loopback) + def testIpv4MappedPrivateCheck(self): self.assertEqual( True, ipaddress.ip_address('::ffff:192.168.1.1').is_private) self.assertEqual( False, ipaddress.ip_address('::ffff:172.32.0.0').is_private) + def testIpv4MappedLoopbackCheck(self): + # test networks + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.100.200.254/128').is_loopback) + self.assertEqual(True, ipaddress.ip_network( + '::ffff:127.42.0.0/112').is_loopback) + self.assertEqual(False, ipaddress.ip_network( + '::ffff:128.0.0.0').is_loopback) + # test addresses + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.100.200.254').is_loopback) + self.assertEqual(True, ipaddress.ip_address( + '::ffff:127.42.0.0').is_loopback) + self.assertEqual(False, ipaddress.ip_address( + '::ffff:128.0.0.0').is_loopback) + def testAddrExclude(self): addr1 = ipaddress.ip_network('10.1.1.0/24') addr2 = ipaddress.ip_network('10.1.1.0/26') @@ -2502,6 +2655,10 @@ def testCompressIPv6Address(self): '::7:6:5:4:3:2:0': '0:7:6:5:4:3:2:0/128', '7:6:5:4:3:2:1::': '7:6:5:4:3:2:1:0/128', '0:6:5:4:3:2:1::': '0:6:5:4:3:2:1:0/128', + '0000:0000:0000:0000:0000:0000:255.255.255.255': '::ffff:ffff/128', + '0000:0000:0000:0000:0000:ffff:255.255.255.255': '::ffff:255.255.255.255/128', + 'ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255': + 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128', } for uncompressed, compressed in list(test_addresses.items()): self.assertEqual(compressed, str(ipaddress.IPv6Interface( @@ -2524,12 +2681,42 @@ def testExplodeShortHandIpStr(self): self.assertEqual('192.168.178.1', addr4.exploded) def testReversePointer(self): - addr1 = ipaddress.IPv4Address('127.0.0.1') - addr2 = ipaddress.IPv6Address('2001:db8::1') - self.assertEqual('1.0.0.127.in-addr.arpa', addr1.reverse_pointer) - self.assertEqual('1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.' + - 'b.d.0.1.0.0.2.ip6.arpa', - addr2.reverse_pointer) + for addr_v4, expected in [ + ('127.0.0.1', '1.0.0.127.in-addr.arpa'), + # test vector: https://www.rfc-editor.org/rfc/rfc1035, §3.5 + ('10.2.0.52', '52.0.2.10.in-addr.arpa'), + ]: + with self.subTest('ipv4_reverse_pointer', addr=addr_v4): + addr = ipaddress.IPv4Address(addr_v4) + self.assertEqual(addr.reverse_pointer, expected) + + for addr_v6, expected in [ + ( + '2001:db8::1', ( + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.' + '0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.' + 'ip6.arpa' + ) + ), + ( + '::FFFF:192.168.1.35', ( + '3.2.1.0.8.a.0.c.f.f.f.f.0.0.0.0.' + '0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.' + 'ip6.arpa' + ) + ), + # test vector: https://www.rfc-editor.org/rfc/rfc3596, §2.5 + ( + '4321:0:1:2:3:4:567:89ab', ( + 'b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.' + '2.0.0.0.1.0.0.0.0.0.0.0.1.2.3.4.' + 'ip6.arpa' + ) + ) + ]: + with self.subTest('ipv6_reverse_pointer', addr=addr_v6): + addr = ipaddress.IPv6Address(addr_v6) + self.assertEqual(addr.reverse_pointer, expected) def testIntRepresentation(self): self.assertEqual(16909060, int(self.ipv4_address)) @@ -2635,6 +2822,34 @@ def testV6HashIsNotConstant(self): ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2") self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__()) + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV4HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.255/32").__hash__(), + ipaddress.IPv4Network("192.168.1.0/24").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("172.24.255.0/24").__hash__(), + ipaddress.IPv4Network("172.24.0.0/16").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.87/32").__hash__(), + ipaddress.IPv4Network("192.168.1.86/31").__hash__() + ) + + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV6HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv6Network("fe80::/64").__hash__(), + ipaddress.IPv6Network("fe80::ffff:ffff:ffff:0/112").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("10.0.0.0/8").__hash__(), + ipaddress.IPv6Network( + "ffff:ffff:ffff:ffff:ffff:ffff:aff:0/112" + ).__hash__() + ) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 9d37cff9903..95a119ba683 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -3,12 +3,11 @@ # testing of error conditions uncovered when using extension types. import unittest -import sys import typing from test import support - + class TestIsInstanceExceptions(unittest.TestCase): # Test to make sure that an AttributeError when accessing the instance's # class's bases is masked. This was actually a bug in Python 2.2 and @@ -97,7 +96,7 @@ def getclass(self): class D: pass self.assertRaises(RuntimeError, isinstance, c, D) - + # These tests are similar to above, but tickle certain code paths in # issubclass() instead of isinstance() -- really PyObject_IsSubclass() # vs. PyObject_IsInstance(). @@ -147,7 +146,7 @@ def getbases(self): self.assertRaises(TypeError, issubclass, B, C()) - + # meta classes for creating abstract classes and instances class AbstractClass(object): def __init__(self, bases): @@ -179,7 +178,7 @@ class Super: class Child(Super): pass - + class TestIsInstanceIsSubclass(unittest.TestCase): # Tests to ensure that isinstance and issubclass work on abstract # classes and instances. Before the 2.2 release, TypeErrors were @@ -225,7 +224,7 @@ def test_isinstance_with_or_union(self): with self.assertRaises(TypeError): isinstance(2, list[int] | int) with self.assertRaises(TypeError): - isinstance(2, int | str | list[int] | float) + isinstance(2, float | str | list[int] | int) @@ -311,7 +310,7 @@ class X: @property def __bases__(self): return self.__bases__ - with support.infinite_recursion(): + with support.infinite_recursion(25): self.assertRaises(RecursionError, issubclass, X(), int) self.assertRaises(RecursionError, issubclass, int, X()) self.assertRaises(RecursionError, isinstance, 1, X()) @@ -345,7 +344,7 @@ class B: pass A.__getattr__ = B.__getattr__ = X.__getattr__ return (A(), B()) - with support.infinite_recursion(): + with support.infinite_recursion(25): self.assertRaises(RecursionError, issubclass, X(), int) @@ -353,10 +352,10 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - for cnt in range(sys.getrecursionlimit()+5): + for cnt in range(support.exceeds_recursion_limit()): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) - + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 42b94a55c1d..9c26eb08583 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -4,11 +4,14 @@ import unittest from test.support import cpython_only from test.support.os_helper import TESTFN, unlink -# XXX: RUSTPYTHON -# from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ +from test.support import BrokenIter import pickle import collections.abc +import functools +import contextlib +import builtins +import traceback # Test result of triple loop (too big to inline) TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2), @@ -83,6 +86,22 @@ class BadIterableClass: def __iter__(self): raise ZeroDivisionError +class CallableIterClass: + def __init__(self): + self.i = 0 + def __call__(self): + i = self.i + self.i = i + 1 + if i > 100: + raise IndexError # Emergency stop + return i + +class EmptyIterClass: + def __len__(self): + return 0 + def __getitem__(self, i): + raise StopIteration + # Main test suite class TestCase(unittest.TestCase): @@ -230,6 +249,78 @@ def test_mutating_seq_class_exhausted_iter(self): self.assertEqual(list(empit), [5, 6]) self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6]) + def test_reduce_mutating_builtins_iter(self): + # This is a reproducer of issue #101765 + # where iter `__reduce__` calls could lead to a segfault or SystemError + # depending on the order of C argument evaluation, which is undefined + + # Backup builtins + builtins_dict = builtins.__dict__ + orig = {"iter": iter, "reversed": reversed} + + def run(builtin_name, item, sentinel=None): + it = iter(item) if sentinel is None else iter(item, sentinel) + + class CustomStr: + def __init__(self, name, iterator): + self.name = name + self.iterator = iterator + def __hash__(self): + return hash(self.name) + def __eq__(self, other): + # Here we exhaust our iterator, possibly changing + # its `it_seq` pointer to NULL + # The `__reduce__` call should correctly get + # the pointers after this call + list(self.iterator) + return other == self.name + + # del is required here + # to not prematurely call __eq__ from + # the hash collision with the old key + del builtins_dict[builtin_name] + builtins_dict[CustomStr(builtin_name, it)] = orig[builtin_name] + + return it.__reduce__() + + types = [ + (EmptyIterClass(),), + (bytes(8),), + (bytearray(8),), + ((1, 2, 3),), + (lambda: 0, 0), + (tuple[int],) # GenericAlias + ] + + try: + run_iter = functools.partial(run, "iter") + # The returned value of `__reduce__` should not only be valid + # but also *empty*, as `it` was exhausted during `__eq__` + # i.e "xyz" returns (iter, ("",)) + self.assertEqual(run_iter("xyz"), (orig["iter"], ("",))) + self.assertEqual(run_iter([1, 2, 3]), (orig["iter"], ([],))) + + # _PyEval_GetBuiltin is also called for `reversed` in a branch of + # listiter_reduce_general + self.assertEqual( + run("reversed", orig["reversed"](list(range(8)))), + (reversed, ([],)) + ) + + for case in types: + self.assertEqual(run_iter(*case), (orig["iter"], ((),))) + finally: + # Restore original builtins + for key, func in orig.items(): + # need to suppress KeyErrors in case + # a failed test deletes the key without setting anything + with contextlib.suppress(KeyError): + # del is required here + # to not invoke our custom __eq__ from + # the hash collision with the old key + del builtins_dict[key] + builtins_dict[key] = func + # Test a new_style class with __iter__ but no next() method def test_new_style_iter_class(self): class IterClass(object): @@ -239,16 +330,7 @@ def __iter__(self): # Test two-argument iter() with callable instance def test_iter_callable(self): - class C: - def __init__(self): - self.i = 0 - def __call__(self): - i = self.i - self.i = i + 1 - if i > 100: - raise IndexError # Emergency stop - return i - self.check_iterator(iter(C(), 10), list(range(10)), pickle=False) + self.check_iterator(iter(CallableIterClass(), 10), list(range(10)), pickle=True) # Test two-argument iter() with function def test_iter_function(self): @@ -268,6 +350,31 @@ def spam(state=[0]): return i self.check_iterator(iter(spam, 20), list(range(10)), pickle=False) + def test_iter_function_concealing_reentrant_exhaustion(self): + # gh-101892: Test two-argument iter() with a function that + # exhausts its associated iterator but forgets to either return + # a sentinel value or raise StopIteration. + HAS_MORE = 1 + NO_MORE = 2 + + def exhaust(iterator): + """Exhaust an iterator without raising StopIteration.""" + list(iterator) + + def spam(): + # Touching the iterator with exhaust() below will call + # spam() once again so protect against recursion. + if spam.is_recursive_call: + return NO_MORE + spam.is_recursive_call = True + exhaust(spam.iterator) + return HAS_MORE + + spam.is_recursive_call = False + spam.iterator = iter(spam, NO_MORE) + with self.assertRaises(StopIteration): + next(spam.iterator) + # Test exception propagation through function iterator def test_exception_function(self): def spam(state=[0]): @@ -1030,8 +1137,7 @@ def test_iter_neg_setstate(self): self.assertEqual(next(it), 0) self.assertEqual(next(it), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_free_after_iterating(self): check_free_after_iterating(self, iter, SequenceClass, (0,)) @@ -1040,6 +1146,46 @@ def test_error_iter(self): self.assertRaises(TypeError, iter, typ()) self.assertRaises(ZeroDivisionError, iter, BadIterableClass()) + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should be the iterator expression + + def init_raises(): + try: + for x in BrokenIter(init_raises=True): + pass + except Exception as e: + return e + + def next_raises(): + try: + for x in BrokenIter(next_raises=True): + pass + except Exception as e: + return e + + def iter_raises(): + try: + for x in BrokenIter(iter_raises=True): + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index cf1107c45af..44addde1948 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1,7 +1,8 @@ import doctest import unittest +import itertools from test import support -from test.support import threading_helper +from test.support import threading_helper, script_helper from itertools import * import weakref from decimal import Decimal @@ -124,8 +125,6 @@ def expand(it, i=0): c = expand(compare[took:]) self.assertEqual(a, c); - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]) @@ -152,15 +151,54 @@ def test_accumulate(self): [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) with self.assertRaises(TypeError): list(accumulate(s, chr)) # unary-operation - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, accumulate(range(10))) # test pickling - self.pickletest(proto, accumulate(range(10), initial=7)) self.assertEqual(list(accumulate([10, 5, 1], initial=None)), [10, 15, 16]) self.assertEqual(list(accumulate([10, 5, 1], initial=100)), [100, 110, 115, 116]) self.assertEqual(list(accumulate([], initial=100)), [100]) with self.assertRaises(TypeError): list(accumulate([10, 20], 100)) + def test_batched(self): + self.assertEqual(list(batched('ABCDEFG', 3)), + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)]) + self.assertEqual(list(batched('ABCDEFG', 2)), + [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)]) + self.assertEqual(list(batched('ABCDEFG', 1)), + [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)]) + self.assertEqual(list(batched('ABCDEF', 2, strict=True)), + [('A', 'B'), ('C', 'D'), ('E', 'F')]) + + with self.assertRaises(ValueError): # Incomplete batch when strict + list(batched('ABCDEFG', 3, strict=True)) + with self.assertRaises(TypeError): # Too few arguments + list(batched('ABCDEFG')) + with self.assertRaises(TypeError): + list(batched('ABCDEFG', 3, None)) # Too many arguments + with self.assertRaises(TypeError): + list(batched(None, 3)) # Non-iterable input + with self.assertRaises(TypeError): + list(batched('ABCDEFG', 'hello')) # n is a string + with self.assertRaises(ValueError): + list(batched('ABCDEFG', 0)) # n is zero + with self.assertRaises(ValueError): + list(batched('ABCDEFG', -1)) # n is negative + + data = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + for n in range(1, 6): + for i in range(len(data)): + s = data[:i] + batches = list(batched(s, n)) + with self.subTest(s=s, n=n, batches=batches): + # Order is preserved and no data is lost + self.assertEqual(''.join(chain(*batches)), s) + # Each batch is an exact tuple + self.assertTrue(all(type(batch) is tuple for batch in batches)) + # All but the last batch is of size n + if batches: + last_batch = batches.pop() + self.assertTrue(all(len(batch) == n for batch in batches)) + self.assertTrue(len(last_batch) <= n) + batches.append(last_batch) + def test_chain(self): def chain2(*iterables): @@ -182,59 +220,14 @@ def test_chain_from_iterable(self): self.assertEqual(list(chain.from_iterable([''])), []) self.assertEqual(take(4, chain.from_iterable(['abc', 'def'])), list('abcd')) self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) + self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) - def test_chain_reducible(self): - for oper in [copy.deepcopy] + picklecopiers: - it = chain('abc', 'def') - self.assertEqual(list(oper(it)), list('abcdef')) - self.assertEqual(next(it), 'a') - self.assertEqual(list(oper(it)), list('bcdef')) - - self.assertEqual(list(oper(chain(''))), []) - self.assertEqual(take(4, oper(chain('abc', 'def'))), list('abcd')) - self.assertRaises(TypeError, list, oper(chain(2, 3))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) - - def test_chain_setstate(self): - self.assertRaises(TypeError, chain().__setstate__, ()) - self.assertRaises(TypeError, chain().__setstate__, []) - self.assertRaises(TypeError, chain().__setstate__, 0) - self.assertRaises(TypeError, chain().__setstate__, ([],)) - self.assertRaises(TypeError, chain().__setstate__, (iter([]), [])) - it = chain() - it.__setstate__((iter(['abc', 'def']),)) - self.assertEqual(list(it), ['a', 'b', 'c', 'd', 'e', 'f']) - it = chain() - it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) - self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_combinations(self): self.assertRaises(TypeError, combinations, 'abc') # missing r argument self.assertRaises(TypeError, combinations, 'abc', 2, 1) # too many arguments self.assertRaises(TypeError, combinations, None) # pool is not iterable self.assertRaises(ValueError, combinations, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(combinations('abc', 32))), []) # r > n - - self.assertEqual(list(op(combinations('ABCD', 2))), - [('A','B'), ('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - testIntermediate = combinations('ABCD', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','C'), ('A','D'), ('B','C'), ('B','D'), ('C','D')]) - - self.assertEqual(list(op(combinations(range(4), 3))), - [(0,1,2), (0,1,3), (0,2,3), (1,2,3)]) - testIntermediate = combinations(range(4), 3) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [(0,1,3), (0,2,3), (1,2,3)]) - - def combinations1(iterable, r): 'Pure python version shown in the docs' pool = tuple(iterable) @@ -288,9 +281,6 @@ def combinations3(iterable, r): self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, combinations(values, r)) # test pickling - @support.bigaddrspacetest def test_combinations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -302,8 +292,6 @@ def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_combinations_with_replacement(self): cwr = combinations_with_replacement self.assertRaises(TypeError, cwr, 'abc') # missing r argument @@ -311,15 +299,6 @@ def test_combinations_with_replacement(self): self.assertRaises(TypeError, cwr, None) # pool is not iterable self.assertRaises(ValueError, cwr, 'abc', -2) # r is negative - for op in [lambda a:a] + picklecopiers: - self.assertEqual(list(op(cwr('ABC', 2))), - [('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - testIntermediate = cwr('ABC', 2) - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), - [('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')]) - - def cwr1(iterable, r): 'Pure python version shown in the docs' # number items returned: (n+r-1)! / r! / (n-1)! when n>0 @@ -377,23 +356,18 @@ def numcombs(n, r): self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cwr(values,r)) # test pickling - @support.bigaddrspacetest def test_combinations_with_replacement_overflow(self): with self.assertRaises((OverflowError, MemoryError)): combinations_with_replacement("AA", 2**30) - # Test implementation detail: tuple re-use + # Test implementation detail: tuple re-use @support.impl_detail("tuple reuse is specific to CPython") def test_combinations_with_replacement_tuple_reuse(self): cwr = combinations_with_replacement self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_permutations(self): self.assertRaises(TypeError, permutations) # too few arguments self.assertRaises(TypeError, permutations, 'abc', 2, 1) # too many arguments @@ -454,9 +428,6 @@ def permutations2(iterable, r=None): self.assertEqual(result, list(permutations(values, None))) # test r as None self.assertEqual(result, list(permutations(values))) # test default r - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, permutations(values, r)) # test pickling - @support.bigaddrspacetest def test_permutations_overflow(self): with self.assertRaises((OverflowError, MemoryError)): @@ -516,24 +487,6 @@ def test_compress(self): self.assertRaises(TypeError, compress, range(6)) # too few args self.assertRaises(TypeError, compress, range(6), None) # too many args - # check copy, deepcopy, pickle - for op in [lambda a:copy.copy(a), lambda a:copy.deepcopy(a)] + picklecopiers: - for data, selectors, result1, result2 in [ - ('ABCDEF', [1,0,1,0,1,1], 'ACEF', 'CEF'), - ('ABCDEF', [0,0,0,0,0,0], '', ''), - ('ABCDEF', [1,1,1,1,1,1], 'ABCDEF', 'BCDEF'), - ('ABCDEF', [1,0,1], 'AC', 'C'), - ('ABC', [0,1,1,1,1,1], 'BC', 'C'), - ]: - - self.assertEqual(list(op(compress(data=data, selectors=selectors))), list(result1)) - self.assertEqual(list(op(compress(data, selectors))), list(result1)) - testIntermediate = compress(data, selectors) - if result1: - next(testIntermediate) - self.assertEqual(list(op(testIntermediate)), list(result2)) - - def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) self.assertEqual(lzip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)]) @@ -542,6 +495,8 @@ def test_count(self): self.assertEqual(take(2, zip('abc',count(-3))), [('a', -3), ('b', -2)]) self.assertRaises(TypeError, count, 2, 3, 4) self.assertRaises(TypeError, count, 'a') + self.assertEqual(take(3, count(maxsize)), + [maxsize, maxsize + 1, maxsize + 2]) self.assertEqual(take(10, count(maxsize-5)), list(range(maxsize-5, maxsize+5))) self.assertEqual(take(10, count(-maxsize-5)), @@ -564,6 +519,15 @@ def test_count(self): self.assertEqual(next(c), -8) self.assertEqual(repr(count(10.25)), 'count(10.25)') self.assertEqual(repr(count(10.0)), 'count(10.0)') + + self.assertEqual(repr(count(maxsize)), f'count({maxsize})') + c = count(maxsize - 1) + self.assertEqual(repr(c), f'count({maxsize - 1})') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize})') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 1})') + self.assertEqual(type(next(count(10.0))), float) for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): # Test repr @@ -571,20 +535,11 @@ def test_count(self): r2 = 'count(%r)'.__mod__(i) self.assertEqual(r1, r2) - # check copy, deepcopy, pickle - for value in -3, 3, maxsize-5, maxsize+5: - c = count(value) - self.assertEqual(next(copy.copy(c)), value) - self.assertEqual(next(copy.deepcopy(c)), value) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(value)) - #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_count_with_stride(self): + @unittest.expectedFailure # TODO: RUSTPYTHON; 'count(10.5)' != 'count(10.5, 1.0)' + def test_count_with_step(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), [('a', 2), ('b', 5), ('c', 8)]) @@ -598,6 +553,12 @@ def test_count_with_stride(self): self.assertEqual(take(20, count(-maxsize-15, 3)), take(20, range(-maxsize-15,-maxsize+100, 3))) self.assertEqual(take(3, count(10, maxsize+5)), list(range(10, 10+3*(maxsize+5), maxsize+5))) + self.assertEqual(take(3, count(maxsize, 2)), + [maxsize, maxsize + 2, maxsize + 4]) + self.assertEqual(take(3, count(maxsize, maxsize)), + [maxsize, 2 * maxsize, 3 * maxsize]) + self.assertEqual(take(3, count(-maxsize, maxsize)), + [-maxsize, 0, maxsize]) self.assertEqual(take(3, count(2, 1.25)), [2, 3.25, 4.5]) self.assertEqual(take(3, count(2, 3.25-4j)), [2, 5.25-4j, 8.5-8j]) self.assertEqual(take(3, count(Decimal('1.1'), Decimal('.1'))), @@ -627,17 +588,42 @@ def test_count_with_stride(self): c = count(10, 1.0) self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) - for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): - for j in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 1, 10, sys.maxsize-5, sys.maxsize+5): - # Test repr - r1 = repr(count(i, j)) - if j == 1: - r2 = ('count(%r)' % i) - else: - r2 = ('count(%r, %r)' % (i, j)) - self.assertEqual(r1, r2) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, count(i, j)) + + c = count(maxsize -2, 2) + self.assertEqual(repr(c), f'count({maxsize - 2}, 2)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, 2)') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 2}, 2)') + + c = count(maxsize + 1, -1) + self.assertEqual(repr(c), f'count({maxsize + 1}, -1)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, -1)') + next(c) + self.assertEqual(repr(c), f'count({maxsize - 1}, -1)') + + @threading_helper.requires_working_threading() + def test_count_threading(self, step=1): + # this test verifies multithreading consistency, which is + # mostly for testing builds without GIL, but nice to test anyway + count_to = 10_000 + num_threads = 10 + c = count(step=step) + def counting_thread(): + for i in range(count_to): + next(c) + threads = [] + for i in range(num_threads): + thread = threading.Thread(target=counting_thread) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + self.assertEqual(next(c), count_to * num_threads * step) + + def test_count_with_step_threading(self): + self.test_count_threading(step=5) def test_cycle(self): self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) @@ -646,117 +632,6 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_cycle_copy_pickle(self): - # check copy, deepcopy, pickle - c = cycle('abc') - self.assertEqual(next(c), 'a') - #simple copy currently not supported, because __reduce__ returns - #an internal iterator - #self.assertEqual(take(10, copy.copy(c)), list('bcabcabcab')) - self.assertEqual(take(10, copy.deepcopy(c)), list('bcabcabcab')) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('bcabcabcab')) - next(c) - self.assertEqual(take(10, pickle.loads(pickle.dumps(c, proto))), - list('cabcabcabc')) - next(c) - next(c) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, cycle('abc')) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - # test with partial consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(2)] # consume 2 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - # test with completely consumed input iterable - it = iter('abcde') - c = cycle(it) - _ = [next(c) for i in range(7)] # consume 7 of 5 inputs - p = pickle.dumps(c, proto) - d = pickle.loads(p) # rebuild the cycle object - self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_cycle_unpickle_compat(self): - testcases = [ - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', - - b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', - b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', - b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', - ] - assert len(testcases) == 20 - for t in testcases: - it = pickle.loads(t) - self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_cycle_setstate(self): - # Verify both modes for restoring state - - # Mode 0 is efficient. It uses an incompletely consumed input - # iterator to build a cycle object and then passes in state with - # a list of previously consumed values. There is no data - # overlap between the two. - c = cycle('defg') - c.__setstate__((list('abc'), 0)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # Mode 1 is inefficient. It starts with a cycle object built - # from an iterator over the remaining elements in a partial - # cycle and then passes in state with all of the previously - # seen values (this overlaps values included in the iterator). - c = cycle('defg') - c.__setstate__((list('abcdefg'), 1)) - self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) - - # The first argument to setstate needs to be a tuple - with self.assertRaises(TypeError): - cycle('defg').__setstate__([list('abcdefg'), 0]) - - # The first argument in the setstate tuple must be a list - with self.assertRaises(TypeError): - c = cycle('defg') - c.__setstate__((tuple('defg'), 0)) - take(20, c) - - # The second argument in the setstate tuple must be an int - with self.assertRaises(TypeError): - cycle('defg').__setstate__((list('abcdefg'), 'x')) - - self.assertRaises(TypeError, cycle('').__setstate__, ()) - self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) @@ -775,15 +650,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check normal pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for elem in g: - self.assertEqual(k, elem[0]) - dup.append(elem) - self.assertEqual(s, dup) - # Check nested case dup = [] for k, g in groupby(s, testR): @@ -794,18 +660,6 @@ def test_groupby(self): dup.append(elem) self.assertEqual(s, dup) - # Check nested and pickled - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - dup = [] - for k, g in pickle.loads(pickle.dumps(groupby(s, testR), proto)): - for ik, ig in pickle.loads(pickle.dumps(groupby(g, testR2), proto)): - for elem in ig: - self.assertEqual(k, elem[0]) - self.assertEqual(ik, elem[2]) - dup.append(elem) - self.assertEqual(s, dup) - - # Check case where inner iterator is not used keys = [k for k, g in groupby(s, testR)] expectedkeys = set([r[0] for r in s]) @@ -825,13 +679,6 @@ def test_groupby(self): list(it) # exhaust the groupby iterator self.assertEqual(list(g3), []) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = groupby(s, testR) - _, g = next(it) - next(it) - next(it) - self.assertEqual(list(pickle.loads(pickle.dumps(g, proto))), []) - # Exercise pipes and filters style s = 'abracadabra' # sort s | uniq @@ -924,8 +771,6 @@ def test_filterfalse(self): self.assertRaises(TypeError, filterfalse, lambda x:x, range(6), 7) self.assertRaises(TypeError, filterfalse, isEven, 3) self.assertRaises(TypeError, next, filterfalse(range(6), range(6))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, filterfalse(isEven, range(6))) def test_zip(self): # XXX This is rather silly now that builtin zip() calls zip()... @@ -950,26 +795,6 @@ def test_zip_tuple_reuse(self): ids = list(map(id, list(zip('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - # check copy, deepcopy, pickle - ans = [(x,y) for x, y in copy.copy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - ans = [(x,y) for x, y in copy.deepcopy(zip('abc',count()))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(zip('abc',count()), proto))] - self.assertEqual(ans, [('a', 0), ('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - testIntermediate = zip('abc',count()) - next(testIntermediate) - ans = [(x,y) for x, y in pickle.loads(pickle.dumps(testIntermediate, proto))] - self.assertEqual(ans, [('b', 1), ('c', 2)]) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip('abc', count())) - def test_ziplongest(self): for args in [ ['abc', range(6)], @@ -1019,13 +844,6 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - def test_zip_longest_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, zip_longest("abc", "def")) - self.pickletest(proto, zip_longest("abc", "defgh")) - self.pickletest(proto, zip_longest("abc", "defgh", fillvalue=1)) - self.pickletest(proto, zip_longest("", "defgh")) - def test_zip_longest_bad_iterable(self): exception = TypeError() @@ -1097,6 +915,78 @@ def test_pairwise(self): with self.assertRaises(TypeError): pairwise(None) # non-iterable argument + def test_pairwise_reenter(self): + def check(reenter_at, expected): + class I: + count = 0 + def __iter__(self): + return self + def __next__(self): + self.count +=1 + if self.count in reenter_at: + return next(it) + return [self.count] # new object + + it = pairwise(I()) + for item in expected: + self.assertEqual(next(it), item) + + check({1}, [ + (([2], [3]), [4]), + ([4], [5]), + ]) + check({2}, [ + ([1], ([1], [3])), + (([1], [3]), [4]), + ([4], [5]), + ]) + check({3}, [ + ([1], [2]), + ([2], ([2], [4])), + (([2], [4]), [5]), + ([5], [6]), + ]) + check({1, 2}, [ + ((([3], [4]), [5]), [6]), + ([6], [7]), + ]) + check({1, 3}, [ + (([2], ([2], [4])), [5]), + ([5], [6]), + ]) + check({1, 4}, [ + (([2], [3]), (([2], [3]), [5])), + ((([2], [3]), [5]), [6]), + ([6], [7]), + ]) + check({2, 3}, [ + ([1], ([1], ([1], [4]))), + (([1], ([1], [4])), [5]), + ([5], [6]), + ]) + + def test_pairwise_reenter2(self): + def check(maxcount, expected): + class I: + count = 0 + def __iter__(self): + return self + def __next__(self): + if self.count >= maxcount: + raise StopIteration + self.count +=1 + if self.count == 1: + return next(it, None) + return [self.count] # new object + + it = pairwise(I()) + self.assertEqual(list(it), expected) + + check(1, []) + check(2, []) + check(3, []) + check(4, [(([2], [3]), [4])]) + def test_product(self): for args, result in [ ([], [()]), # zero iterables @@ -1135,12 +1025,16 @@ def product1(*args, **kwds): else: return - def product2(*args, **kwds): + def product2(*iterables, repeat=1): 'Pure python version used in docs' - pools = list(map(tuple, args)) * kwds.get('repeat', 1) + if repeat < 0: + raise ValueError('repeat argument cannot be negative') + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -1165,34 +1059,6 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) - - def test_product_pickling(self): - # check copy, deepcopy, pickle - for args, result in [ - ([], [()]), # zero iterables - (['ab'], [('a',), ('b',)]), # one iterable - ([range(2), range(3)], [(0,0), (0,1), (0,2), (1,0), (1,1), (1,2)]), # two iterables - ([range(0), range(2), range(3)], []), # first iterable with zero length - ([range(2), range(0), range(3)], []), # middle iterable with zero length - ([range(2), range(3), range(0)], []), # last iterable with zero length - ]: - self.assertEqual(list(copy.copy(product(*args))), result) - self.assertEqual(list(copy.deepcopy(product(*args))), result) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, product(*args)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_product_issue_25021(self): - # test that indices are properly clamped to the length of the tuples - p = product((1, 2),(3,)) - p.__setstate__((0, 0x1000)) # will access tuple element 1 if not clamped - self.assertEqual(next(p), (2, 3)) - # test that empty tuple in the list will result in an immediate StopIteration - p = product((1, 2), (), (3,)) - p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped - self.assertRaises(StopIteration, next, p) - def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) self.assertEqual(lzip(range(3),repeat('a')), @@ -1211,22 +1077,13 @@ def test_repeat(self): list(r) self.assertEqual(repr(r), 'repeat((1+0j), 0)') - # check copy, deepcopy, pickle - c = repeat(object='a', times=10) - self.assertEqual(next(c), 'a') - self.assertEqual(take(2, copy.copy(c)), list('a' * 2)) - self.assertEqual(take(2, copy.deepcopy(c)), list('a' * 2)) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, repeat(object='a', times=10)) - def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', -1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', -2)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), [0**1, 1**2, 2**3]) @@ -1244,19 +1101,6 @@ def test_map(self): self.assertRaises(ValueError, next, map(errfunc, [4], [5])) self.assertRaises(TypeError, next, map(onearg, [4], [5])) - # check copy, deepcopy, pickle - ans = [('a',0),('b',1),('c',2)] - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.copy(c)), ans) - - c = map(tupleize, 'abc', count()) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = map(tupleize, 'abc', count()) - self.pickletest(proto, c) - def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), [0**1, 1**2, 2**3]) @@ -1271,21 +1115,7 @@ def test_starmap(self): self.assertRaises(ValueError, next, starmap(errfunc, [(4,5)])) self.assertRaises(TypeError, next, starmap(onearg, [(4,5)])) - # check copy, deepcopy, pickle - ans = [0**1, 1**2, 2**3] - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.copy(c)), ans) - - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.assertEqual(list(copy.deepcopy(c)), ans) - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - c = starmap(operator.pow, zip(range(3), range(1,7))) - self.pickletest(proto, c) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_islice(self): for args in [ # islice(args) should agree with range(args) (10, 20, 3), @@ -1342,21 +1172,6 @@ def test_islice(self): self.assertEqual(list(islice(c, 1, 3, 50)), [1]) self.assertEqual(next(c), 3) - # check copy, deepcopy, pickle - for args in [ # islice(args) should agree with range(args) - (10, 20, 3), - (10, 3, 20), - (10, 20), - (10, 3), - (20,) - ]: - self.assertEqual(list(copy.copy(islice(range(100), *args))), - list(range(*args))) - self.assertEqual(list(copy.deepcopy(islice(range(100), *args))), - list(range(*args))) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, islice(range(100), *args)) - # Issue #21321: check source iterator is not referenced # from islice() after the latter has been exhausted it = (x for x in (1, 2)) @@ -1393,15 +1208,7 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(takewhile(underten, data))), [1, 3, 5]) - self.assertEqual(list(copy.deepcopy(takewhile(underten, data))), - [1, 3, 5]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, takewhile(underten, data)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) @@ -1412,15 +1219,7 @@ def test_dropwhile(self): self.assertRaises(TypeError, next, dropwhile(10, [(4,5)])) self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)])) - # check copy, deepcopy, pickle - self.assertEqual(list(copy.copy(dropwhile(underten, data))), [20, 2, 4, 6, 8]) - self.assertEqual(list(copy.deepcopy(dropwhile(underten, data))), - [20, 2, 4, 6, 8]) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, dropwhile(underten, data)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tee(self): n = 200 @@ -1487,10 +1286,11 @@ def test_tee(self): self.assertEqual(len(result), n) self.assertEqual([list(x) for x in result], [list('abc')]*n) - # tee pass-through to copyable iterator + # tee objects are independent (see bug gh-123884) a, b = tee('abc') c, d = tee(a) - self.assertTrue(a is c) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) # test tee_new t1, t2 = tee('abc') @@ -1500,7 +1300,7 @@ def test_tee(self): t3 = tnew(t1) self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) - # test that tee objects are weak referencable + # test that tee objects are weak referenceable a, b = tee(range(10)) p = weakref.proxy(a) self.assertEqual(getattr(p, '__class__'), type(b)) @@ -1535,40 +1335,13 @@ def test_tee(self): self.assertEqual(list(a), long_ans[100:]) self.assertEqual(list(b), long_ans[60:]) - # check deepcopy - a, b = tee('abc') - self.assertEqual(list(copy.deepcopy(a)), ans) - self.assertEqual(list(copy.deepcopy(b)), ans) - self.assertEqual(list(a), ans) - self.assertEqual(list(b), ans) - a, b = tee(range(10000)) - self.assertEqual(list(copy.deepcopy(a)), long_ans) - self.assertEqual(list(copy.deepcopy(b)), long_ans) - self.assertEqual(list(a), long_ans) - self.assertEqual(list(b), long_ans) - - # check partially consumed deepcopy - a, b = tee('abc') - take(2, a) - take(1, b) - self.assertEqual(list(copy.deepcopy(a)), ans[2:]) - self.assertEqual(list(copy.deepcopy(b)), ans[1:]) - self.assertEqual(list(a), ans[2:]) - self.assertEqual(list(b), ans[1:]) - a, b = tee(range(10000)) - take(100, a) - take(60, b) - self.assertEqual(list(copy.deepcopy(a)), long_ans[100:]) - self.assertEqual(list(copy.deepcopy(b)), long_ans[60:]) - self.assertEqual(list(a), long_ans[100:]) - self.assertEqual(list(b), long_ans[60:]) - - # check pickle - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.pickletest(proto, iter(tee('abc'))) - a, b = tee('abc') - self.pickletest(proto, a, compare=ans) - self.pickletest(proto, b, compare=ans) + def test_tee_dealloc_segfault(self): + # gh-115874: segfaults when accessing module state in tp_dealloc. + script = ( + "import typing, copyreg, itertools; " + "copyreg.buggy_tee = itertools.tee(())" + ) + script_helper.assert_python_ok("-c", script) # Issue 13454: Crash when deleting backward iterator from tee() def test_tee_del_backward(self): @@ -1580,8 +1353,6 @@ def test_tee_del_backward(self): del forward, backward raise - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tee_reenter(self): class I: first = True @@ -1597,7 +1368,6 @@ def __next__(self): with self.assertRaisesRegex(RuntimeError, "tee"): next(a) - @unittest.skip("TODO: RUSTPYTHON, hangs") @threading_helper.requires_working_threading() def test_tee_concurrent(self): start = threading.Event() @@ -1687,39 +1457,51 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + + @support.cpython_only + def test_immutable_types(self): + from itertools import _grouper, _tee, _tee_dataobject + dataset = ( + accumulate, + batched, + chain, + combinations, + combinations_with_replacement, + compress, + count, + cycle, + dropwhile, + filterfalse, + groupby, + _grouper, + islice, + pairwise, + permutations, + product, + repeat, + starmap, + takewhile, + _tee, + _tee_dataobject, + zip_longest, + ) + for tp in dataset: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foobar = 1 + class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) - def test_accumulate_reducible(self): - # check copy, deepcopy, pickle - data = [1, 2, 3, 4, 5] - accumulated = [1, 3, 6, 10, 15] - - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = accumulate(data) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[:]) - self.assertEqual(next(it), 1) - self.assertEqual(list(pickle.loads(pickle.dumps(it, proto))), accumulated[1:]) - it = accumulate(data) - self.assertEqual(next(it), 1) - self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) - self.assertEqual(list(copy.copy(it)), accumulated[1:]) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_accumulate_reducible_none(self): - # Issue #25718: total is None - it = accumulate([None, None, None], operator.is_) - self.assertEqual(next(it), None) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it_copy = pickle.loads(pickle.dumps(it, proto)) - self.assertEqual(list(it_copy), [True, False]) - self.assertEqual(list(copy.deepcopy(it)), [True, False]) - self.assertEqual(list(copy.copy(it)), [True, False]) - def test_chain(self): self.assertEqual(''.join(chain('ABC', 'DEF')), 'ABCDEF') @@ -1802,27 +1584,194 @@ def test_takewhile(self): class TestPurePythonRoughEquivalents(unittest.TestCase): + def test_batched_recipe(self): + def batched_recipe(iterable, n): + "Batch data into tuples of length n. The last batch may be shorter." + # batched('ABCDEFG', 3) --> ABC DEF G + if n < 1: + raise ValueError('n must be at least one') + it = iter(iterable) + while batch := tuple(islice(it, n)): + yield batch + + for iterable, n in product( + ['', 'a', 'ab', 'abc', 'abcd', 'abcde', 'abcdef', 'abcdefg', None], + [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, None]): + with self.subTest(iterable=iterable, n=n): + try: + e1, r1 = None, list(batched(iterable, n)) + except Exception as e: + e1, r1 = type(e), None + try: + e2, r2 = None, list(batched_recipe(iterable, n)) + except Exception as e: + e2, r2 = type(e), None + self.assertEqual(r1, r2) + self.assertEqual(e1, e2) + + + def test_groupby_recipe(self): + + # Begin groupby() recipe ####################################### + + def groupby(iterable, key=None): + # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B + # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D + + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: + return + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass + + # End groupby() recipe ######################################### + + # Check whether it accepts arguments correctly + self.assertEqual([], list(groupby([]))) + self.assertEqual([], list(groupby([], key=id))) + self.assertRaises(TypeError, list, groupby('abc', [])) + if False: + # Test not applicable to the recipe + self.assertRaises(TypeError, list, groupby('abc', None)) + self.assertRaises(TypeError, groupby, 'abc', lambda x:x, 10) + + # Check normal input + s = [(0, 10, 20), (0, 11,21), (0,12,21), (1,13,21), (1,14,22), + (2,15,22), (3,16,23), (3,17,23)] + dup = [] + for k, g in groupby(s, lambda r:r[0]): + for elem in g: + self.assertEqual(k, elem[0]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check nested case + dup = [] + for k, g in groupby(s, testR): + for ik, ig in groupby(g, testR2): + for elem in ig: + self.assertEqual(k, elem[0]) + self.assertEqual(ik, elem[2]) + dup.append(elem) + self.assertEqual(s, dup) + + # Check case where inner iterator is not used + keys = [k for k, g in groupby(s, testR)] + expectedkeys = set([r[0] for r in s]) + self.assertEqual(set(keys), expectedkeys) + self.assertEqual(len(keys), len(expectedkeys)) + + # Check case where inner iterator is used after advancing the groupby + # iterator + s = list(zip('AABBBAAAA', range(9))) + it = groupby(s, testR) + _, g1 = next(it) + _, g2 = next(it) + _, g3 = next(it) + self.assertEqual(list(g1), []) + self.assertEqual(list(g2), []) + self.assertEqual(next(g3), ('A', 5)) + list(it) # exhaust the groupby iterator + self.assertEqual(list(g3), []) + + # Exercise pipes and filters style + s = 'abracadabra' + # sort s | uniq + r = [k for k, g in groupby(sorted(s))] + self.assertEqual(r, ['a', 'b', 'c', 'd', 'r']) + # sort s | uniq -d + r = [k for k, g in groupby(sorted(s)) if list(islice(g,1,2))] + self.assertEqual(r, ['a', 'b', 'r']) + # sort s | uniq -c + r = [(len(list(g)), k) for k, g in groupby(sorted(s))] + self.assertEqual(r, [(5, 'a'), (2, 'b'), (1, 'c'), (1, 'd'), (2, 'r')]) + # sort s | uniq -c | sort -rn | head -3 + r = sorted([(len(list(g)) , k) for k, g in groupby(sorted(s))], reverse=True)[:3] + self.assertEqual(r, [(5, 'a'), (2, 'r'), (2, 'b')]) + + # iter.__next__ failure + class ExpectedError(Exception): + pass + def delayed_raise(n=0): + for i in range(n): + yield 'yo' + raise ExpectedError + def gulp(iterable, keyp=None, func=list): + return [func(g) for k, g in groupby(iterable, keyp)] + + # iter.__next__ failure on outer object + self.assertRaises(ExpectedError, gulp, delayed_raise(0)) + # iter.__next__ failure on inner object + self.assertRaises(ExpectedError, gulp, delayed_raise(1)) + + # __eq__ failure + class DummyCmp: + def __eq__(self, dst): + raise ExpectedError + s = [DummyCmp(), DummyCmp(), None] + + # __eq__ failure on outer object + self.assertRaises(ExpectedError, gulp, s, func=id) + # __eq__ failure on inner object + self.assertRaises(ExpectedError, gulp, s) + + # keyfunc failure + def keyfunc(obj): + if keyfunc.skip > 0: + keyfunc.skip -= 1 + return obj + else: + raise ExpectedError + + # keyfunc failure on outer object + keyfunc.skip = 0 + self.assertRaises(ExpectedError, gulp, [None], keyfunc) + keyfunc.skip = 1 + self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + + @staticmethod def islice(iterable, *args): + # islice('ABCDEFG', 2) → A B + # islice('ABCDEFG', 2, 4) → C D + # islice('ABCDEFG', 2, None) → C D E F G + # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + + indices = count() if stop is None else range(max(start, stop)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step def test_islice_recipe(self): self.assertEqual(list(self.islice('ABCDEFG', 2)), list('AB')) @@ -1842,6 +1791,172 @@ def test_islice_recipe(self): self.assertEqual(next(c), 3) + def test_tee_recipe(self): + + # Begin tee() recipe ########################################### + + def tee(iterable, n=2): + if n < 0: + raise ValueError + if n == 0: + return () + iterator = _tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_tee(iterator)) + return tuple(result) + + class _tee: + + def __init__(self, iterable): + it = iter(iterable) + if isinstance(it, _tee): + self.iterator = it.iterator + self.link = it.link + else: + self.iterator = it + self.link = [None, None] + + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value + + # End tee() recipe ############################################# + + n = 200 + + a, b = tee([]) # test empty iterator + self.assertEqual(list(a), []) + self.assertEqual(list(b), []) + + a, b = tee(irange(n)) # test 100% interleaved + self.assertEqual(lzip(a,b), lzip(range(n), range(n))) + + a, b = tee(irange(n)) # test 0% interleaved + self.assertEqual(list(a), list(range(n))) + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of leading iterator + for i in range(100): + self.assertEqual(next(a), i) + del a + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of trailing iterator + for i in range(100): + self.assertEqual(next(a), i) + del b + self.assertEqual(list(a), list(range(100, n))) + + for j in range(5): # test randomly interleaved + order = [0]*n + [1]*n + random.shuffle(order) + lists = ([], []) + its = tee(irange(n)) + for i in order: + value = next(its[i]) + lists[i].append(value) + self.assertEqual(lists[0], list(range(n))) + self.assertEqual(lists[1], list(range(n))) + + # test argument format checking + self.assertRaises(TypeError, tee) + self.assertRaises(TypeError, tee, 3) + self.assertRaises(TypeError, tee, [1,2], 'x') + self.assertRaises(TypeError, tee, [1,2], 3, 'x') + + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) + + # test long-lagged and multi-way split + a, b, c = tee(range(2000), 3) + for i in range(100): + self.assertEqual(next(a), i) + self.assertEqual(list(b), list(range(2000))) + self.assertEqual([next(c), next(c)], list(range(2))) + self.assertEqual(list(a), list(range(100,2000))) + self.assertEqual(list(c), list(range(2,2000))) + + # test invalid values of n + self.assertRaises(TypeError, tee, 'abc', 'invalid') + self.assertRaises(ValueError, tee, [], -1) + + for n in range(5): + result = tee('abc', n) + self.assertEqual(type(result), tuple) + self.assertEqual(len(result), n) + self.assertEqual([list(x) for x in result], [list('abc')]*n) + + # tee objects are independent (see bug gh-123884) + a, b = tee('abc') + c, d = tee(a) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) + + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + + # test that tee objects are weak referencable + a, b = tee(range(10)) + p = weakref.proxy(a) + self.assertEqual(getattr(p, '__class__'), type(b)) + del a + gc.collect() # For PyPy or other GCs. + self.assertRaises(ReferenceError, getattr, p, '__class__') + + ans = list('abc') + long_ans = list(range(10000)) + + # Tests not applicable to the tee() recipe + if False: + # check copy + a, b = tee('abc') + self.assertEqual(list(copy.copy(a)), ans) + self.assertEqual(list(copy.copy(b)), ans) + a, b = tee(list(range(10000))) + self.assertEqual(list(copy.copy(a)), long_ans) + self.assertEqual(list(copy.copy(b)), long_ans) + + # check partially consumed copy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.copy(a)), ans[2:]) + self.assertEqual(list(copy.copy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.copy(a)), long_ans[100:]) + self.assertEqual(list(copy.copy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # Issue 13454: Crash when deleting backward iterator from tee() + forward, backward = tee(repeat(None, 2000)) # 20000000 + try: + any(forward) # exhaust the iterator + del backward + except: + del forward, backward + raise + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): @@ -1853,6 +1968,10 @@ def test_accumulate(self): a = [] self.makecycle(accumulate([1,2,a,3]), a) + def test_batched(self): + a = [] + self.makecycle(batched([1,2,a,3], 2), a) + def test_chain(self): a = [] self.makecycle(chain(a), a) @@ -2010,6 +2129,20 @@ def __iter__(self): def __next__(self): 3 // 0 +class E2: + 'Test propagation of exceptions after two iterations' + def __init__(self, seqn): + self.seqn = seqn + self.i = 0 + def __iter__(self): + return self + def __next__(self): + if self.i == 2: + raise ZeroDivisionError + v = self.seqn[self.i] + self.i += 1 + return v + class S: 'Test immediate stop' def __init__(self, seqn): @@ -2037,6 +2170,19 @@ def test_accumulate(self): self.assertRaises(TypeError, accumulate, N(s)) self.assertRaises(ZeroDivisionError, list, accumulate(E(s))) + def test_batched(self): + s = 'abcde' + r = [('a', 'b'), ('c', 'd'), ('e',)] + n = 2 + for g in (G, I, Ig, L, R): + with self.subTest(g=g): + self.assertEqual(list(batched(g(s), n)), r) + self.assertEqual(list(batched(S(s), 2)), []) + self.assertRaises(TypeError, batched, X(s), 2) + self.assertRaises(TypeError, batched, N(s), 2) + self.assertRaises(ZeroDivisionError, list, batched(E(s), 2)) + self.assertRaises(ZeroDivisionError, list, batched(E2(s), 4)) + def test_chain(self): for s in ("123", "", range(1000), ('do', 1.2), range(2000,2200,5)): for g in (G, I, Ig, S, L, R): @@ -2263,6 +2409,7 @@ def gen2(x): self.assertEqual(hist, [0,1]) @support.skip_if_pgo_task + @support.requires_resource('cpu') def test_long_chain_of_empty_iterables(self): # Make sure itertools.chain doesn't run into recursion limits when # dealing with long chains of empty iterables. Even with a high @@ -2296,9 +2443,7 @@ def __eq__(self, other): class SubclassWithKwargsTest(unittest.TestCase): - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords_in_subclass(self): # count is not subclassable... testcases = [ @@ -2326,10 +2471,10 @@ class subclass(cls): subclass(*args, newarg=3) for cls, args, result in testcases: - # Constructors of repeat, zip, compress accept keyword arguments. + # Constructors of repeat, zip, map, compress accept keyword arguments. # Their subclasses need overriding __new__ to support new # keyword arguments. - if cls in [repeat, zip, compress]: + if cls in [repeat, zip, map, compress]: continue with self.subTest(cls): class subclass_with_init(cls): @@ -2393,7 +2538,7 @@ def test_permutations_sizeof(self): def load_tests(loader, tests, pattern): - tests.addTest(doctest.DocTestSuite()) + tests.addTest(doctest.DocTestSuite(itertools)) return tests diff --git a/Lib/test/test_json/__init__.py b/Lib/test/test_json/__init__.py index b919af2328f..41c06beaa38 100644 --- a/Lib/test/test_json/__init__.py +++ b/Lib/test/test_json/__init__.py @@ -41,8 +41,7 @@ def test_pyjson(self): 'json.encoder') class TestCTest(CTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cjson(self): self.assertEqual(self.json.scanner.make_scanner.__module__, '_json') self.assertEqual(self.json.decoder.scanstring.__module__, '_json') diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index e48e36c54b9..7b3b30ce449 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -2,21 +2,49 @@ from io import StringIO from collections import OrderedDict from test.test_json import PyTest, CTest +from test import support -import unittest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) + self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): rval = self.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) + self.assertIsInstance(rval, float) self.assertEqual(rval, 1.0) + @unittest.skip("TODO: RUSTPYTHON; called `Result::unwrap()` on an `Err` value: ParseFloatError { kind: Invalid }") + def test_nonascii_digits_rejected(self): + # JSON specifies only ascii digits, see gh-125687 + for num in ["1\uff10", "0.\uff10", "0e\uff10"]: + with self.assertRaises(self.JSONDecodeError): + self.loads(num) + + def test_bytes(self): + self.assertEqual(self.loads(b"1"), 1) + + def test_parse_constant(self): + for constant, expected in [ + ("Infinity", "INFINITY"), + ("-Infinity", "-INFINITY"), + ("NaN", "NAN"), + ]: + self.assertEqual( + self.loads(constant, parse_constant=str.upper), expected + ) + + def test_constant_invalid_case(self): + for constant in [ + "nan", "NAN", "naN", "infinity", "INFINITY", "inFiniTy" + ]: + with self.assertRaises(self.JSONDecodeError): + self.loads(constant) + def test_empty_objects(self): self.assertEqual(self.loads('{}'), {}) self.assertEqual(self.loads('[]'), []) @@ -89,7 +117,8 @@ def test_string_with_utf8_bom(self): self.json.load(StringIO(bom_json)) self.assertIn('BOM', str(cm.exception)) # make sure that the BOM is not detected in the middle of a string - bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8')) + bom = ''.encode('utf-8-sig').decode('utf-8') + bom_in_str = f'"{bom}"' self.assertEqual(self.loads(bom_in_str), '\ufeff') self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff') @@ -97,10 +126,16 @@ def test_negative_index(self): d = self.json.JSONDecoder() self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000) + def test_limit_int(self): + maxdigits = 5000 + with support.adjust_int_max_str_digits(maxdigits): + self.loads('1' * maxdigits) + with self.assertRaises(ValueError): + self.loads('1' * (maxdigits + 1)) + + class TestPyDecode(TestDecode, PyTest): pass -# TODO: RUSTPYTHON -class TestCDecode(TestDecode, CTest): # pass - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_keys_reuse(self): - return super().test_keys_reuse() +class TestCDecode(TestDecode, CTest): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_limit_int(self): + return super().test_limit_int() diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 9b8325e9c38..4d569dadfa4 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -1,5 +1,8 @@ +import collections from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class TestDefault: def test_default(self): @@ -7,6 +10,35 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bad_default(self): + def default(obj): + if obj is NotImplemented: + raise ValueError + if obj is ...: + return NotImplemented + if obj is type: + return collections + return [...] + + with self.assertRaises(ValueError) as cm: + self.dumps(type, default=default) + self.assertEqual(cm.exception.__notes__, + ['when serializing ellipsis object', + 'when serializing list item 0', + 'when serializing module object', + 'when serializing type object']) + + def test_ordereddict(self): + od = collections.OrderedDict(a=1, b=2, c=3, d=4) + od.move_to_end('b') + self.assertEqual( + self.dumps(od), + '{"a": 1, "c": 3, "d": 4, "b": 2}') + self.assertEqual( + self.dumps(od, sort_keys=True), + '{"a": 1, "b": 2, "c": 3, "d": 4}') + class TestPyDefault(TestDefault, PyTest): pass class TestCDefault(TestDefault, CTest): pass diff --git a/Lib/test/test_json/test_dump.py b/Lib/test/test_json/test_dump.py index 13b40020781..39470754003 100644 --- a/Lib/test/test_json/test_dump.py +++ b/Lib/test/test_json/test_dump.py @@ -22,6 +22,14 @@ def test_dump_skipkeys(self): self.assertIn('valid_key', o) self.assertNotIn(b'invalid_key', o) + def test_dump_skipkeys_indent_empty(self): + v = {b'invalid_key': False} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{}') + + def test_skipkeys_indent(self): + v = {b'invalid_key': False, 'valid_key': True} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}') + def test_encode_truefalse(self): self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 4bbc6c71489..c90d3e968e5 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -8,13 +8,12 @@ ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ('controls', '"controls"'), ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), + ('\x00\x1f\x7f', '"\\u0000\\u001f\\u007f"'), ('{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), (' s p a c e d ', '" s p a c e d "'), ('\U0001d120', '"\\ud834\\udd20"'), ('\u03b1\u03a9', '"\\u03b1\\u03a9"'), ("`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] class TestEncodeBasestringAscii: @@ -23,8 +22,7 @@ def test_encode_basestring_ascii(self): for input_string, expect in CASES: result = self.json.encoder.encode_basestring_ascii(input_string) self.assertEqual(result, expect, - '{0!r} != {1!r} for {2}({3!r})'.format( - result, expect, fname, input_string)) + f'{result!r} != {expect!r} for {fname}({input_string!r})') def test_ordered_dict(self): # See issue 6105 diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index dcadc1c51c2..4adfcb17c4d 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -1,75 +1,76 @@ -import unittest from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + # 2007-10-05 JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json + # https://json.org/JSON_checker/test/fail1.json '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json + # https://json.org/JSON_checker/test/fail2.json '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json + # https://json.org/JSON_checker/test/fail3.json '{unquoted_key: "keys must be quoted"}', - # http://json.org/JSON_checker/test/fail4.json + # https://json.org/JSON_checker/test/fail4.json '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json + # https://json.org/JSON_checker/test/fail5.json '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json + # https://json.org/JSON_checker/test/fail6.json '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json + # https://json.org/JSON_checker/test/fail7.json '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json + # https://json.org/JSON_checker/test/fail8.json '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json + # https://json.org/JSON_checker/test/fail9.json '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json + # https://json.org/JSON_checker/test/fail10.json '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json + # https://json.org/JSON_checker/test/fail11.json '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json + # https://json.org/JSON_checker/test/fail12.json '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json + # https://json.org/JSON_checker/test/fail13.json '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json + # https://json.org/JSON_checker/test/fail14.json '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json + # https://json.org/JSON_checker/test/fail15.json '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json + # https://json.org/JSON_checker/test/fail16.json '[\\naked]', - # http://json.org/JSON_checker/test/fail17.json + # https://json.org/JSON_checker/test/fail17.json '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json + # https://json.org/JSON_checker/test/fail18.json '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json + # https://json.org/JSON_checker/test/fail19.json '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json + # https://json.org/JSON_checker/test/fail20.json '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json + # https://json.org/JSON_checker/test/fail21.json '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json + # https://json.org/JSON_checker/test/fail22.json '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json + # https://json.org/JSON_checker/test/fail23.json '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json + # https://json.org/JSON_checker/test/fail24.json "['single quote']", - # http://json.org/JSON_checker/test/fail25.json + # https://json.org/JSON_checker/test/fail25.json '["\ttab\tcharacter\tin\tstring\t"]', - # http://json.org/JSON_checker/test/fail26.json + # https://json.org/JSON_checker/test/fail26.json '["tab\\ character\\ in\\ string\\ "]', - # http://json.org/JSON_checker/test/fail27.json + # https://json.org/JSON_checker/test/fail27.json '["line\nbreak"]', - # http://json.org/JSON_checker/test/fail28.json + # https://json.org/JSON_checker/test/fail28.json '["line\\\nbreak"]', - # http://json.org/JSON_checker/test/fail29.json + # https://json.org/JSON_checker/test/fail29.json '[0e]', - # http://json.org/JSON_checker/test/fail30.json + # https://json.org/JSON_checker/test/fail30.json '[0e+]', - # http://json.org/JSON_checker/test/fail31.json + # https://json.org/JSON_checker/test/fail31.json '[0e+-1]', - # http://json.org/JSON_checker/test/fail32.json + # https://json.org/JSON_checker/test/fail32.json '{"Comma instead if closing brace": true,', - # http://json.org/JSON_checker/test/fail33.json + # https://json.org/JSON_checker/test/fail33.json '["mismatch"}', - # http://code.google.com/p/simplejson/issues/detail?id=3 + # https://code.google.com/archive/p/simplejson/issues/3 '["A\u001FZ control characters in string"]', ] @@ -90,7 +91,7 @@ def test_failures(self): except self.JSONDecodeError: pass else: - self.fail("Expected failure for fail{0}.json: {1!r}".format(idx, doc)) + self.fail(f"Expected failure for fail{idx}.json: {doc!r}") def test_non_string_keys_dict(self): data = {'a' : 1, (1, 2) : 2} @@ -101,8 +102,27 @@ def test_non_string_keys_dict(self): def test_not_serializable(self): import sys with self.assertRaisesRegex(TypeError, - 'Object of type module is not JSON serializable'): + 'Object of type module is not JSON serializable') as cm: self.dumps(sys) + self.assertNotHasAttr(cm.exception, '__notes__') + + with self.assertRaises(TypeError) as cm: + self.dumps([1, [2, 3, sys]]) + self.assertEqual(cm.exception.__notes__, + ['when serializing list item 2', + 'when serializing list item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps((1, (2, 3, sys))) + self.assertEqual(cm.exception.__notes__, + ['when serializing tuple item 2', + 'when serializing tuple item 1']) + + with self.assertRaises(TypeError) as cm: + self.dumps({'a': {'b': sys}}) + self.assertEqual(cm.exception.__notes__, + ["when serializing dict item 'b'", + "when serializing dict item 'a'"]) def test_truncated_input(self): test_cases = [ @@ -144,11 +164,11 @@ def test_unexpected_data(self): ('{"spam":[}', 'Expecting value', 9), ('[42:', "Expecting ',' delimiter", 3), ('[42 "spam"', "Expecting ',' delimiter", 4), - ('[42,]', 'Expecting value', 4), + ('[42,]', "Illegal trailing comma before end of array", 3), ('{"spam":[42}', "Expecting ',' delimiter", 11), ('["]', 'Unterminated string starting at', 1), ('["spam":', "Expecting ',' delimiter", 7), - ('["spam",]', 'Expecting value', 8), + ('["spam",]', "Illegal trailing comma before end of array", 7), ('{:', 'Expecting property name enclosed in double quotes', 1), ('{,', 'Expecting property name enclosed in double quotes', 1), ('{42', 'Expecting property name enclosed in double quotes', 1), @@ -160,7 +180,9 @@ def test_unexpected_data(self): ('[{"spam":]', 'Expecting value', 9), ('{"spam":42 "ham"', "Expecting ',' delimiter", 11), ('[{"spam":42]', "Expecting ',' delimiter", 11), - ('{"spam":42,}', 'Expecting property name enclosed in double quotes', 11), + ('{"spam":42,}', "Illegal trailing comma before end of object", 10), + ('{"spam":42 , }', "Illegal trailing comma before end of object", 11), + ('[123 , ]', "Illegal trailing comma before end of array", 6), ] for data, msg, idx in test_cases: with self.assertRaises(self.JSONDecodeError) as cm: @@ -217,10 +239,7 @@ def test_linecol(self): (line, col, idx)) class TestPyFail(TestFail, PyTest): pass -class TestCFail(TestFail, CTest): pass -# TODO: RUSTPYTHON class TestCFail(TestFail, CTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_failures(self): - super().test_failures() + return super().test_failures() diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index d0c7214334d..61540a3a02c 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -26,7 +26,8 @@ def test_allow_nan(self): res = self.loads(out) self.assertEqual(len(res), 1) self.assertNotEqual(res[0], res[0]) - self.assertRaises(ValueError, self.dumps, [val], allow_nan=False) + msg = f'Out of range float values are not JSON compliant: {val}' + self.assertRaisesRegex(ValueError, msg, self.dumps, [val], allow_nan=False) class TestPyFloat(TestFloat, PyTest): pass diff --git a/Lib/test/test_json/test_pass1.py b/Lib/test/test_json/test_pass1.py index 15e64b0aeae..26bf3cdbd77 100644 --- a/Lib/test/test_json/test_pass1.py +++ b/Lib/test/test_json/test_pass1.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass1.json +# from https://json.org/JSON_checker/test/pass1.json JSON = r''' [ "JSON Test Pattern pass1", diff --git a/Lib/test/test_json/test_pass2.py b/Lib/test/test_json/test_pass2.py index 35075249e3b..9340de665aa 100644 --- a/Lib/test/test_json/test_pass2.py +++ b/Lib/test/test_json/test_pass2.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass2.json +# from https://json.org/JSON_checker/test/pass2.json JSON = r''' [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] ''' diff --git a/Lib/test/test_json/test_pass3.py b/Lib/test/test_json/test_pass3.py index cd0cf170d27..0adccc1c2a5 100644 --- a/Lib/test/test_json/test_pass3.py +++ b/Lib/test/test_json/test_pass3.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass3.json +# from https://json.org/JSON_checker/test/pass3.json JSON = r''' { "JSON Test Pattern pass3": { diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 9919d7fbe54..2fa8d3f528c 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -1,6 +1,8 @@ from test import support from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class JSONTestObject: pass @@ -12,8 +14,8 @@ def test_listrecursion(self): x.append(x) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]) else: self.fail("didn't raise ValueError on list recursion") x = [] @@ -21,8 +23,8 @@ def test_listrecursion(self): x.append(y) try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing list item 0"]*2) else: self.fail("didn't raise ValueError on alternating list recursion") y = [] @@ -35,8 +37,8 @@ def test_dictrecursion(self): x["test"] = x try: self.dumps(x) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, ["when serializing dict item 'test'"]) else: self.fail("didn't raise ValueError on dict recursion") x = {} @@ -60,37 +62,51 @@ def default(self, o): enc.recurse = True try: enc.encode(JSONTestObject) - except ValueError: - pass + except ValueError as exc: + self.assertEqual(exc.__notes__, + ["when serializing list item 0", + "when serializing type object"]) else: self.fail("didn't raise ValueError on default recursion") + @unittest.skip("TODO: RUSTPYTHON; crashes") + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_highly_nested_objects_decoding(self): + very_deep = 500_000 # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 with self.assertRaises(RecursionError): with support.infinite_recursion(): - self.loads('{"a":' * 100000 + '1' + '}' * 100000) + self.loads('{"a":' * very_deep + '1' + '}' * very_deep) with self.assertRaises(RecursionError): with support.infinite_recursion(): - self.loads('{"a":' * 100000 + '[1]' + '}' * 100000) + self.loads('{"a":' * very_deep + '[1]' + '}' * very_deep) with self.assertRaises(RecursionError): with support.infinite_recursion(): - self.loads('[' * 100000 + '1' + ']' * 100000) + self.loads('[' * very_deep + '1' + ']' * very_deep) + @support.skip_if_unlimited_stack_size + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() + @support.requires_resource('cpu') def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} - for x in range(100000): + for x in range(500_000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(l) with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(d) + @support.skip_if_unlimited_stack_size + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): @@ -99,7 +115,7 @@ def default(self, o): return [o] with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index 140a7c12a2d..e77ec152280 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -1,7 +1,8 @@ -import unittest import sys from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class TestScanstring: def test_scanstring(self): @@ -86,8 +87,6 @@ def test_scanstring(self): scanstring('["Bad value", truth]', 2, True), ('Bad value', 12)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogates(self): scanstring = self.json.decoder.scanstring def assertScan(given, expect): @@ -119,6 +118,11 @@ def test_bad_escapes(self): '"\\u012z"', '"\\u0x12"', '"\\u0X12"', + '"\\u{0}"'.format("\uff10" * 4), + '"\\u 123"', + '"\\u-123"', + '"\\u+123"', + '"\\u1_23"', '"\\ud834\\"', '"\\ud834\\u"', '"\\ud834\\ud"', @@ -130,23 +134,21 @@ def test_bad_escapes(self): '"\\ud834\\udd2z"', '"\\ud834\\u0x20"', '"\\ud834\\u0X20"', + '"\\ud834\\u{0}"'.format("\uff10" * 4), + '"\\ud834\\u 123"', + '"\\ud834\\u-123"', + '"\\ud834\\u+123"', + '"\\ud834\\u1_23"', ] for s in bad_escapes: with self.assertRaises(self.JSONDecodeError, msg=s): scanstring(s, 1, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_overflow(self): with self.assertRaises(OverflowError): - self.json.decoder.scanstring(b"xxx", sys.maxsize+1) + self.json.decoder.scanstring("xxx", sys.maxsize+1) class TestPyScanstring(TestScanstring, PyTest): pass -# TODO: RUSTPYTHON -class TestPyScanstring(TestScanstring, PyTest): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bad_escapes(self): - super().test_bad_escapes() class TestCScanstring(TestScanstring, CTest): pass diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 66ce1d1b338..370a2539d10 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -1,6 +1,7 @@ from test.test_json import CTest -import unittest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class BadBool: def __bool__(self): @@ -39,8 +40,7 @@ def test_make_encoder(self): b"\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) - # TODO: RUSTPYTHON, TypeError: 'NoneType' object is not callable - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'NoneType' object is not callable def test_bad_str_encoder(self): # Issue #31505: There shouldn't be an assertion failure in case # c_make_encoder() receives a bad encoder() argument. @@ -62,8 +62,7 @@ def bad_encoder2(*args): with self.assertRaises(ZeroDivisionError): enc('spam', 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_markers_argument_to_encoder(self): # https://bugs.python.org/issue45269 with self.assertRaisesRegex( @@ -73,8 +72,7 @@ def test_bad_markers_argument_to_encoder(self): self.json.encoder.c_make_encoder(1, None, None, None, ': ', ', ', False, False, False) - # TODO: RUSTPYTHON, translate the encoder to Rust - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ZeroDivisionError not raised by test def test_bad_bool_args(self): def test(name): self.json.encoder.JSONEncoder(**{name: BadBool()}).encode({'a': 1}) @@ -87,3 +85,35 @@ def test(name): def test_unsortable_keys(self): with self.assertRaises(TypeError): self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'}) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'NoneType' object is not callable + def test_current_indent_level(self): + enc = self.json.encoder.c_make_encoder( + markers=None, + default=str, + encoder=self.json.encoder.c_encode_basestring, + indent='\t', + key_separator=': ', + item_separator=', ', + sort_keys=False, + skipkeys=False, + allow_nan=False) + expected = ( + '[\n' + '\t"spam", \n' + '\t{\n' + '\t\t"ham": "eggs"\n' + '\t}\n' + ']') + self.assertEqual(enc(['spam', {'ham': 'eggs'}], 0)[0], expected) + self.assertEqual(enc(['spam', {'ham': 'eggs'}], -3)[0], expected) + expected2 = ( + '[\n' + '\t\t\t\t"spam", \n' + '\t\t\t\t{\n' + '\t\t\t\t\t"ham": "eggs"\n' + '\t\t\t\t}\n' + '\t\t\t]') + self.assertEqual(enc(['spam', {'ham': 'eggs'}], 3)[0], expected2) + self.assertRaises(TypeError, enc, ['spam', {'ham': 'eggs'}], 3.0) + self.assertRaises(TypeError, enc, ['spam', {'ham': 'eggs'}]) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 1d7fca6efb1..7b5d217a215 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -6,11 +6,15 @@ import subprocess from test import support -from test.support import os_helper +from test.support import force_colorized, force_not_colorized, os_helper from test.support.script_helper import assert_python_ok +from _colorize import get_theme -class TestTool(unittest.TestCase): + +@support.requires_subprocess() +@support.skip_if_pgo_task +class TestMain(unittest.TestCase): data = """ [["blorpie"],[ "whoops" ] , [ @@ -18,6 +22,7 @@ class TestTool(unittest.TestCase): "i-vhbjkhnth", {"nifty":87}, {"morefield" :\tfalse,"field" :"yes"} ] """ + module = 'json' expect_without_sort_keys = textwrap.dedent("""\ [ @@ -85,8 +90,9 @@ class TestTool(unittest.TestCase): } """) + @force_not_colorized def test_stdin_stdout(self): - args = sys.executable, '-m', 'json.tool' + args = sys.executable, '-m', self.module process = subprocess.run(args, input=self.data, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, self.expect) self.assertEqual(process.stderr, '') @@ -100,7 +106,8 @@ def _create_infile(self, data=None): def test_infile_stdout(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', infile) + rc, out, err = assert_python_ok('-m', self.module, infile, + PYTHON_COLORS='0') self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) self.assertEqual(err, b'') @@ -114,7 +121,8 @@ def test_non_ascii_infile(self): ''').encode() infile = self._create_infile(data) - rc, out, err = assert_python_ok('-m', 'json.tool', infile) + rc, out, err = assert_python_ok('-m', self.module, infile, + PYTHON_COLORS='0') self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), expect.splitlines()) @@ -123,7 +131,8 @@ def test_non_ascii_infile(self): def test_infile_outfile(self): infile = self._create_infile() outfile = os_helper.TESTFN + '.out' - rc, out, err = assert_python_ok('-m', 'json.tool', infile, outfile) + rc, out, err = assert_python_ok('-m', self.module, infile, outfile, + PYTHON_COLORS='0') self.addCleanup(os.remove, outfile) with open(outfile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) @@ -133,33 +142,38 @@ def test_infile_outfile(self): def test_writing_in_place(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', infile, infile) + rc, out, err = assert_python_ok('-m', self.module, infile, infile, + PYTHON_COLORS='0') with open(infile, "r", encoding="utf-8") as fp: self.assertEqual(fp.read(), self.expect) self.assertEqual(rc, 0) self.assertEqual(out, b'') self.assertEqual(err, b'') + @force_not_colorized def test_jsonlines(self): - args = sys.executable, '-m', 'json.tool', '--json-lines' + args = sys.executable, '-m', self.module, '--json-lines' process = subprocess.run(args, input=self.jsonlines_raw, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, self.jsonlines_expect) self.assertEqual(process.stderr, '') def test_help_flag(self): - rc, out, err = assert_python_ok('-m', 'json.tool', '-h') + rc, out, err = assert_python_ok('-m', self.module, '-h', + PYTHON_COLORS='0') self.assertEqual(rc, 0) - self.assertTrue(out.startswith(b'usage: ')) + self.assertStartsWith(out, b'usage: ') self.assertEqual(err, b'') def test_sort_keys_flag(self): infile = self._create_infile() - rc, out, err = assert_python_ok('-m', 'json.tool', '--sort-keys', infile) + rc, out, err = assert_python_ok('-m', self.module, '--sort-keys', infile, + PYTHON_COLORS='0') self.assertEqual(rc, 0) self.assertEqual(out.splitlines(), self.expect_without_sort_keys.encode().splitlines()) self.assertEqual(err, b'') + @force_not_colorized def test_indent(self): input_ = '[1, 2]' expect = textwrap.dedent('''\ @@ -168,31 +182,34 @@ def test_indent(self): 2 ] ''') - args = sys.executable, '-m', 'json.tool', '--indent', '2' + args = sys.executable, '-m', self.module, '--indent', '2' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @force_not_colorized def test_no_indent(self): input_ = '[1,\n2]' expect = '[1, 2]\n' - args = sys.executable, '-m', 'json.tool', '--no-indent' + args = sys.executable, '-m', self.module, '--no-indent' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @force_not_colorized def test_tab(self): input_ = '[1, 2]' expect = '[\n\t1,\n\t2\n]\n' - args = sys.executable, '-m', 'json.tool', '--tab' + args = sys.executable, '-m', self.module, '--tab' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') + @force_not_colorized def test_compact(self): input_ = '[ 1 ,\n 2]' expect = '[1,2]\n' - args = sys.executable, '-m', 'json.tool', '--compact' + args = sys.executable, '-m', self.module, '--compact' process = subprocess.run(args, input=input_, capture_output=True, text=True, check=True) self.assertEqual(process.stdout, expect) self.assertEqual(process.stderr, '') @@ -201,7 +218,8 @@ def test_no_ensure_ascii_flag(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' self.addCleanup(os.remove, outfile) - assert_python_ok('-m', 'json.tool', '--no-ensure-ascii', infile, outfile) + assert_python_ok('-m', self.module, '--no-ensure-ascii', infile, + outfile, PYTHON_COLORS='0') with open(outfile, "rb") as f: lines = f.read().splitlines() # asserting utf-8 encoded output file @@ -212,20 +230,100 @@ def test_ensure_ascii_default(self): infile = self._create_infile('{"key":"💩"}') outfile = os_helper.TESTFN + '.out' self.addCleanup(os.remove, outfile) - assert_python_ok('-m', 'json.tool', infile, outfile) + assert_python_ok('-m', self.module, infile, outfile, PYTHON_COLORS='0') with open(outfile, "rb") as f: lines = f.read().splitlines() # asserting an ascii encoded output file expected = [b'{', rb' "key": "\ud83d\udca9"', b"}"] self.assertEqual(lines, expected) + @force_not_colorized @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows") def test_broken_pipe_error(self): - cmd = [sys.executable, '-m', 'json.tool'] + cmd = [sys.executable, '-m', self.module] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE) - # bpo-39828: Closing before json.tool attempts to write into stdout. + # bpo-39828: Closing before json attempts to write into stdout. proc.stdout.close() proc.communicate(b'"{}"') self.assertEqual(proc.returncode, errno.EPIPE) + + @force_colorized + def test_colors(self): + infile = os_helper.TESTFN + self.addCleanup(os.remove, infile) + + t = get_theme().syntax + ob = "{" + cb = "}" + + cases = ( + ('{}', '{}'), + ('[]', '[]'), + ('null', f'{t.keyword}null{t.reset}'), + ('true', f'{t.keyword}true{t.reset}'), + ('false', f'{t.keyword}false{t.reset}'), + ('NaN', f'{t.number}NaN{t.reset}'), + ('Infinity', f'{t.number}Infinity{t.reset}'), + ('-Infinity', f'{t.number}-Infinity{t.reset}'), + ('"foo"', f'{t.string}"foo"{t.reset}'), + (r'" \"foo\" "', f'{t.string}" \\"foo\\" "{t.reset}'), + ('"α"', f'{t.string}"\\u03b1"{t.reset}'), + ('123', f'{t.number}123{t.reset}'), + ('-1.25e+23', f'{t.number}-1.25e+23{t.reset}'), + (r'{"\\": ""}', + f'''\ +{ob} + {t.definition}"\\\\"{t.reset}: {t.string}""{t.reset} +{cb}'''), + (r'{"\\\\": ""}', + f'''\ +{ob} + {t.definition}"\\\\\\\\"{t.reset}: {t.string}""{t.reset} +{cb}'''), + ('''\ +{ + "foo": "bar", + "baz": 1234, + "qux": [true, false, null], + "xyz": [NaN, -Infinity, Infinity] +}''', + f'''\ +{ob} + {t.definition}"foo"{t.reset}: {t.string}"bar"{t.reset}, + {t.definition}"baz"{t.reset}: {t.number}1234{t.reset}, + {t.definition}"qux"{t.reset}: [ + {t.keyword}true{t.reset}, + {t.keyword}false{t.reset}, + {t.keyword}null{t.reset} + ], + {t.definition}"xyz"{t.reset}: [ + {t.number}NaN{t.reset}, + {t.number}-Infinity{t.reset}, + {t.number}Infinity{t.reset} + ] +{cb}'''), + ) + + for input_, expected in cases: + with self.subTest(input=input_): + with open(infile, "w", encoding="utf-8") as fp: + fp.write(input_) + _, stdout_b, _ = assert_python_ok( + '-m', self.module, infile, FORCE_COLOR='1', __isolated='1' + ) + stdout = stdout_b.decode() + stdout = stdout.replace('\r\n', '\n') # normalize line endings + stdout = stdout.strip() + self.assertEqual(stdout, expected) + + +@support.requires_subprocess() +@support.skip_if_pgo_task +class TestTool(TestMain): + module = 'json.tool' + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index e5882a1ef4d..ab1be6ea6e8 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -1,8 +1,9 @@ -import unittest import codecs from collections import OrderedDict from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class TestUnicode: # test_encoding1 and test_encoding2 from 2.x are irrelevant (only str @@ -21,12 +22,40 @@ def test_encoding4(self): def test_encoding5(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps(u, ensure_ascii=False) - self.assertEqual(j, '"{0}"'.format(u)) + self.assertEqual(j, f'"{u}"') def test_encoding6(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps([u], ensure_ascii=False) - self.assertEqual(j, '["{0}"]'.format(u)) + self.assertEqual(j, f'["{u}"]') + + def test_encoding7(self): + u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' + j = self.dumps(u + "\n", ensure_ascii=False) + self.assertEqual(j, f'"{u}\\n"') + + def test_ascii_non_printable_encode(self): + u = '\b\t\n\f\r\x00\x1f\x7f' + self.assertEqual(self.dumps(u), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\\u007f"') + self.assertEqual(self.dumps(u, ensure_ascii=False), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\x7f"') + + def test_ascii_non_printable_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), + '\b\t\n\f\r') + s = ''.join(map(chr, range(32))) + for c in s: + self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') + self.assertEqual(self.loads(f'"{s}"', strict=False), s) + self.assertEqual(self.loads('"\x7f"'), '\x7f') + + def test_escaped_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), '\b\t\n\f\r') + self.assertEqual(self.loads('"\\"\\\\\\/"'), '"\\/') + for c in set(map(chr, range(0x100))) - set('"\\/bfnrt'): + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"') + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"', strict=False) def test_big_unicode_encode(self): u = '\U0001d120' @@ -35,15 +64,27 @@ def test_big_unicode_encode(self): def test_big_unicode_decode(self): u = 'z\U0001d120x' - self.assertEqual(self.loads('"' + u + '"'), u) + self.assertEqual(self.loads(f'"{u}"'), u) self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = chr(i) - s = '"\\u{0:04x}"'.format(i) + s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) + def test_single_surrogate_encode(self): + self.assertEqual(self.dumps('\uD83D'), '"\\ud83d"') + self.assertEqual(self.dumps('\uD83D', ensure_ascii=False), '"\ud83d"') + self.assertEqual(self.dumps('\uDC0D'), '"\\udc0d"') + self.assertEqual(self.dumps('\uDC0D', ensure_ascii=False), '"\udc0d"') + + def test_single_surrogate_decode(self): + self.assertEqual(self.loads('"\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\udc0d"'), '\udc0d') + self.assertEqual(self.loads('"\\udc0d"'), '\udc0d') + def test_unicode_preservation(self): self.assertEqual(type(self.loads('""')), str) self.assertEqual(type(self.loads('"a"')), str) @@ -53,8 +94,6 @@ def test_bytes_encode(self): self.assertRaises(TypeError, self.dumps, b"hi") self.assertRaises(TypeError, self.dumps, [b"hi"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bytes_decode(self): for encoding, bom in [ ('utf-8', codecs.BOM_UTF8), @@ -98,4 +137,15 @@ def test_object_pairs_hook_with_unicode(self): class TestPyUnicode(TestUnicode, PyTest): pass -class TestCUnicode(TestUnicode, CTest): pass +class TestCUnicode(TestUnicode, CTest): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_ascii_non_printable_encode(self): + return super().test_ascii_non_printable_encode() + + @unittest.skip("TODO: RUSTPYTHON; panics with 'str has surrogates'") + def test_single_surrogate_decode(self): + return super().test_single_surrogate_decode() + + @unittest.skip("TODO: RUSTPYTHON; panics with 'str has surrogates'") + def test_single_surrogate_encode(self): + return super().test_single_surrogate_encode() diff --git a/Lib/test/test_keyword.py b/Lib/test/test_keyword.py index 3e2a8b3fb7f..858e5de3b92 100644 --- a/Lib/test/test_keyword.py +++ b/Lib/test/test_keyword.py @@ -20,18 +20,37 @@ def test_changing_the_kwlist_does_not_affect_iskeyword(self): keyword.kwlist = ['its', 'all', 'eggs', 'beans', 'and', 'a', 'slice'] self.assertFalse(keyword.iskeyword('eggs')) + def test_changing_the_softkwlist_does_not_affect_issoftkeyword(self): + oldlist = keyword.softkwlist + self.addCleanup(setattr, keyword, "softkwlist", oldlist) + keyword.softkwlist = ["foo", "bar", "spam", "egs", "case"] + self.assertFalse(keyword.issoftkeyword("spam")) + def test_all_keywords_fail_to_be_used_as_names(self): for key in keyword.kwlist: with self.assertRaises(SyntaxError): exec(f"{key} = 42") + def test_all_soft_keywords_can_be_used_as_names(self): + for key in keyword.softkwlist: + exec(f"{key} = 42") + def test_async_and_await_are_keywords(self): self.assertIn("async", keyword.kwlist) self.assertIn("await", keyword.kwlist) + def test_soft_keywords(self): + self.assertIn("type", keyword.softkwlist) + self.assertIn("match", keyword.softkwlist) + self.assertIn("case", keyword.softkwlist) + self.assertIn("_", keyword.softkwlist) + def test_keywords_are_sorted(self): self.assertListEqual(sorted(keyword.kwlist), keyword.kwlist) + def test_softkeywords_are_sorted(self): + self.assertListEqual(sorted(keyword.softkwlist), keyword.softkwlist) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index 46dce636df5..918f953cae5 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -40,8 +40,6 @@ def shouldRaiseSyntaxError(s): compile(s, "<test>", "single") self.assertRaises(SyntaxError, shouldRaiseSyntaxError, codestr) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testSyntaxErrorForFunctionDefinition(self): self.assertRaisesSyntaxError("def f(p, *):\n pass\n") self.assertRaisesSyntaxError("def f(p1, *, p1=100):\n pass\n") @@ -60,8 +58,6 @@ def testSyntaxForManyArguments(self): fundef = "def f(*, %s):\n pass\n" % ', '.join('i%d' % i for i in range(300)) compile(fundef, "<test>", "single") - # TODO: RUSTPYTHON - @unittest.expectedFailure def testTooManyPositionalErrorMessage(self): def f(a, b=None, *, c=None): pass @@ -160,8 +156,6 @@ def test_issue13343(self): # used to fail with a SystemError. lambda *, k1=unittest: None - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mangling(self): class X: def f(self, *, __a=42): @@ -176,7 +170,7 @@ def f(v=a, x=b, *, y=c, z=d): pass self.assertEqual(str(err.exception), "name 'b' is not defined") with self.assertRaises(NameError) as err: - f = lambda v=a, x=b, *, y=c, z=d: None + g = lambda v=a, x=b, *, y=c, z=d: None self.assertEqual(str(err.exception), "name 'b' is not defined") diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index 998fd9d4649..e94edcbc107 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -5,6 +5,7 @@ import os import select import socket +from test import support import time import unittest @@ -256,6 +257,23 @@ def test_fd_non_inheritable(self): self.addCleanup(kqueue.close) self.assertEqual(os.get_inheritable(kqueue.fileno()), False) + @support.requires_fork() + def test_fork(self): + # gh-110395: kqueue objects must be closed after fork + kqueue = select.kqueue() + if (pid := os.fork()) == 0: + try: + self.assertTrue(kqueue.closed) + with self.assertRaisesRegex(ValueError, "closed kqueue"): + kqueue.fileno() + except: + os._exit(1) + finally: + os._exit(0) + else: + support.wait_process(pid, exitcode=0) + self.assertFalse(kqueue.closed) # child done, we're still open. + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py new file mode 100644 index 00000000000..caa1603c78e --- /dev/null +++ b/Lib/test/test_launcher.py @@ -0,0 +1,796 @@ +import contextlib +import itertools +import os +import re +import shutil +import subprocess +import sys +import sysconfig +import tempfile +import unittest +from pathlib import Path +from test import support + +if sys.platform != "win32": + raise unittest.SkipTest("test only applies to Windows") + +# Get winreg after the platform check +import winreg + + +PY_EXE = "py.exe" +DEBUG_BUILD = False +if sys.executable.casefold().endswith("_d.exe".casefold()): + PY_EXE = "py_d.exe" + DEBUG_BUILD = True + +# Registry data to create. On removal, everything beneath top-level names will +# be deleted. +TEST_DATA = { + "PythonTestSuite": { + "DisplayName": "Python Test Suite", + "SupportUrl": "https://www.python.org/", + "3.100": { + "DisplayName": "X.Y version", + "InstallPath": { + None: sys.prefix, + "ExecutablePath": "X.Y.exe", + } + }, + "3.100-32": { + "DisplayName": "X.Y-32 version", + "InstallPath": { + None: sys.prefix, + "ExecutablePath": "X.Y-32.exe", + } + }, + "3.100-arm64": { + "DisplayName": "X.Y-arm64 version", + "InstallPath": { + None: sys.prefix, + "ExecutablePath": "X.Y-arm64.exe", + "ExecutableArguments": "-X fake_arg_for_test", + } + }, + "ignored": { + "DisplayName": "Ignored because no ExecutablePath", + "InstallPath": { + None: sys.prefix, + } + }, + }, + "PythonTestSuite1": { + "DisplayName": "Python Test Suite Single", + "3.100": { + "DisplayName": "Single Interpreter", + "InstallPath": { + None: sys.prefix, + "ExecutablePath": sys.executable, + } + } + }, +} + + +TEST_PY_ENV = dict( + PY_PYTHON="PythonTestSuite/3.100", + PY_PYTHON2="PythonTestSuite/3.100-32", + PY_PYTHON3="PythonTestSuite/3.100-arm64", +) + + +TEST_PY_DEFAULTS = "\n".join([ + "[defaults]", + *[f"{k[3:].lower()}={v}" for k, v in TEST_PY_ENV.items()], +]) + + +TEST_PY_COMMANDS = "\n".join([ + "[commands]", + "test-command=TEST_EXE.exe", +]) + + +def quote(s): + s = str(s) + return f'"{s}"' if " " in s else s + + +def create_registry_data(root, data): + def _create_registry_data(root, key, value): + if isinstance(value, dict): + # For a dict, we recursively create keys + with winreg.CreateKeyEx(root, key) as hkey: + for k, v in value.items(): + _create_registry_data(hkey, k, v) + elif isinstance(value, str): + # For strings, we set values. 'key' may be None in this case + winreg.SetValueEx(root, key, None, winreg.REG_SZ, value) + else: + raise TypeError("don't know how to create data for '{}'".format(value)) + + for k, v in data.items(): + _create_registry_data(root, k, v) + + +def enum_keys(root): + for i in itertools.count(): + try: + yield winreg.EnumKey(root, i) + except OSError as ex: + if ex.winerror == 259: + break + raise + + +def delete_registry_data(root, keys): + ACCESS = winreg.KEY_WRITE | winreg.KEY_ENUMERATE_SUB_KEYS + for key in list(keys): + with winreg.OpenKey(root, key, access=ACCESS) as hkey: + delete_registry_data(hkey, enum_keys(hkey)) + winreg.DeleteKey(root, key) + + +def is_installed(tag): + key = rf"Software\Python\PythonCore\{tag}\InstallPath" + for root, flag in [ + (winreg.HKEY_CURRENT_USER, 0), + (winreg.HKEY_LOCAL_MACHINE, winreg.KEY_WOW64_64KEY), + (winreg.HKEY_LOCAL_MACHINE, winreg.KEY_WOW64_32KEY), + ]: + try: + winreg.CloseKey(winreg.OpenKey(root, key, access=winreg.KEY_READ | flag)) + return True + except OSError: + pass + return False + + +class PreservePyIni: + def __init__(self, path, content): + self.path = Path(path) + self.content = content + self._preserved = None + + def __enter__(self): + try: + self._preserved = self.path.read_bytes() + except FileNotFoundError: + self._preserved = None + self.path.write_text(self.content, encoding="utf-16") + + def __exit__(self, *exc_info): + if self._preserved is None: + self.path.unlink() + else: + self.path.write_bytes(self._preserved) + + +class RunPyMixin: + py_exe = None + + @classmethod + def find_py(cls): + py_exe = None + if sysconfig.is_python_build(): + py_exe = Path(sys.executable).parent / PY_EXE + else: + for p in os.getenv("PATH").split(";"): + if p: + py_exe = Path(p) / PY_EXE + if py_exe.is_file(): + break + else: + py_exe = None + + # Test launch and check version, to exclude installs of older + # releases when running outside of a source tree + if py_exe: + try: + with subprocess.Popen( + [py_exe, "-h"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="ascii", + errors="ignore", + ) as p: + p.stdin.close() + version = next(p.stdout, "\n").splitlines()[0].rpartition(" ")[2] + p.stdout.read() + p.wait(10) + if not sys.version.startswith(version): + py_exe = None + except OSError: + py_exe = None + + if not py_exe: + raise unittest.SkipTest( + "cannot locate '{}' for test".format(PY_EXE) + ) + return py_exe + + def get_py_exe(self): + if not self.py_exe: + self.py_exe = self.find_py() + return self.py_exe + + def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=None): + if not self.py_exe: + self.py_exe = self.find_py() + + ignore = {"VIRTUAL_ENV", "PY_PYTHON", "PY_PYTHON2", "PY_PYTHON3"} + env = { + **{k.upper(): v for k, v in os.environ.items() if k.upper() not in ignore}, + "PYLAUNCHER_DEBUG": "1", + "PYLAUNCHER_DRYRUN": "1", + "PYLAUNCHER_LIMIT_TO_COMPANY": "", + **{k.upper(): v for k, v in (env or {}).items()}, + } + if not argv: + argv = [self.py_exe, *args] + with subprocess.Popen( + argv, + env=env, + executable=self.py_exe, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as p: + p.stdin.close() + p.wait(10) + out = p.stdout.read().decode("utf-8", "replace") + err = p.stderr.read().decode("ascii", "replace").replace("\uFFFD", "?") + if p.returncode != expect_returncode and support.verbose and not allow_fail: + print("++ COMMAND ++") + print([self.py_exe, *args]) + print("++ STDOUT ++") + print(out) + print("++ STDERR ++") + print(err) + if allow_fail and p.returncode != expect_returncode: + raise subprocess.CalledProcessError(p.returncode, [self.py_exe, *args], out, err) + else: + self.assertEqual(expect_returncode, p.returncode) + data = { + s.partition(":")[0]: s.partition(":")[2].lstrip() + for s in err.splitlines() + if not s.startswith("#") and ":" in s + } + data["stdout"] = out + data["stderr"] = err + return data + + def py_ini(self, content): + local_appdata = os.environ.get("LOCALAPPDATA") + if not local_appdata: + raise unittest.SkipTest("LOCALAPPDATA environment variable is " + "missing or empty") + return PreservePyIni(Path(local_appdata) / "py.ini", content) + + @contextlib.contextmanager + def script(self, content, encoding="utf-8"): + file = Path(tempfile.mktemp(dir=os.getcwd()) + ".py") + if isinstance(content, bytes): + file.write_bytes(content) + else: + file.write_text(content, encoding=encoding) + try: + yield file + finally: + file.unlink() + + @contextlib.contextmanager + def fake_venv(self): + venv = Path.cwd() / "Scripts" + venv.mkdir(exist_ok=True, parents=True) + venv_exe = (venv / ("python_d.exe" if DEBUG_BUILD else "python.exe")) + venv_exe.touch() + try: + yield venv_exe, {"VIRTUAL_ENV": str(venv.parent)} + finally: + shutil.rmtree(venv) + + +class TestLauncher(unittest.TestCase, RunPyMixin): + @classmethod + def setUpClass(cls): + with winreg.CreateKey(winreg.HKEY_CURRENT_USER, rf"Software\Python") as key: + create_registry_data(key, TEST_DATA) + + if support.verbose: + p = subprocess.check_output("reg query HKCU\\Software\\Python /s") + #print(p.decode('mbcs')) + + + @classmethod + def tearDownClass(cls): + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, rf"Software\Python", access=winreg.KEY_WRITE | winreg.KEY_ENUMERATE_SUB_KEYS) as key: + delete_registry_data(key, TEST_DATA) + + + def test_version(self): + data = self.run_py(["-0"]) + self.assertEqual(self.py_exe, Path(data["argv0"])) + self.assertEqual(sys.version.partition(" ")[0], data["version"]) + + def test_help_option(self): + data = self.run_py(["-h"]) + self.assertEqual("True", data["SearchInfo.help"]) + + def test_list_option(self): + for opt, v1, v2 in [ + ("-0", "True", "False"), + ("-0p", "False", "True"), + ("--list", "True", "False"), + ("--list-paths", "False", "True"), + ]: + with self.subTest(opt): + data = self.run_py([opt]) + self.assertEqual(v1, data["SearchInfo.list"]) + self.assertEqual(v2, data["SearchInfo.listPaths"]) + + def test_list(self): + data = self.run_py(["--list"]) + found = {} + expect = {} + for line in data["stdout"].splitlines(): + m = re.match(r"\s*(.+?)\s+?(\*\s+)?(.+)$", line) + if m: + found[m.group(1)] = m.group(3) + for company in TEST_DATA: + company_data = TEST_DATA[company] + tags = [t for t in company_data if isinstance(company_data[t], dict)] + for tag in tags: + arg = f"-V:{company}/{tag}" + expect[arg] = company_data[tag]["DisplayName"] + expect.pop(f"-V:{company}/ignored", None) + + actual = {k: v for k, v in found.items() if k in expect} + try: + self.assertDictEqual(expect, actual) + except: + if support.verbose: + print("*** STDOUT ***") + print(data["stdout"]) + raise + + def test_list_paths(self): + data = self.run_py(["--list-paths"]) + found = {} + expect = {} + for line in data["stdout"].splitlines(): + m = re.match(r"\s*(.+?)\s+?(\*\s+)?(.+)$", line) + if m: + found[m.group(1)] = m.group(3) + for company in TEST_DATA: + company_data = TEST_DATA[company] + tags = [t for t in company_data if isinstance(company_data[t], dict)] + for tag in tags: + arg = f"-V:{company}/{tag}" + install = company_data[tag]["InstallPath"] + try: + expect[arg] = install["ExecutablePath"] + try: + expect[arg] += " " + install["ExecutableArguments"] + except KeyError: + pass + except KeyError: + expect[arg] = str(Path(install[None]) / Path(sys.executable).name) + + expect.pop(f"-V:{company}/ignored", None) + + actual = {k: v for k, v in found.items() if k in expect} + try: + self.assertDictEqual(expect, actual) + except: + if support.verbose: + print("*** STDOUT ***") + print(data["stdout"]) + raise + + def test_filter_to_company(self): + company = "PythonTestSuite" + data = self.run_py([f"-V:{company}/"]) + self.assertEqual("X.Y.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100", data["env.tag"]) + + def test_filter_to_company_with_default(self): + company = "PythonTestSuite" + data = self.run_py([f"-V:{company}/"], env=dict(PY_PYTHON="3.0")) + self.assertEqual("X.Y.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100", data["env.tag"]) + + def test_filter_to_tag(self): + company = "PythonTestSuite" + data = self.run_py(["-V:3.100"]) + self.assertEqual("X.Y.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100", data["env.tag"]) + + data = self.run_py(["-V:3.100-32"]) + self.assertEqual("X.Y-32.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100-32", data["env.tag"]) + + data = self.run_py(["-V:3.100-arm64"]) + self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100-arm64", data["env.tag"]) + + def test_filter_to_company_and_tag(self): + company = "PythonTestSuite" + data = self.run_py([f"-V:{company}/3.1"], expect_returncode=103) + + data = self.run_py([f"-V:{company}/3.100"]) + self.assertEqual("X.Y.exe", data["LaunchCommand"]) + self.assertEqual(company, data["env.company"]) + self.assertEqual("3.100", data["env.tag"]) + + def test_filter_with_single_install(self): + company = "PythonTestSuite1" + data = self.run_py( + ["-V:Nonexistent"], + env={"PYLAUNCHER_LIMIT_TO_COMPANY": company}, + expect_returncode=103, + ) + + def test_search_major_3(self): + try: + data = self.run_py(["-3"], allow_fail=True) + except subprocess.CalledProcessError: + raise unittest.SkipTest("requires at least one Python 3.x install") + self.assertEqual("PythonCore", data["env.company"]) + self.assertStartsWith(data["env.tag"], "3.") + + def test_search_major_3_32(self): + try: + data = self.run_py(["-3-32"], allow_fail=True) + except subprocess.CalledProcessError: + if not any(is_installed(f"3.{i}-32") for i in range(5, 11)): + raise unittest.SkipTest("requires at least one 32-bit Python 3.x install") + raise + self.assertEqual("PythonCore", data["env.company"]) + self.assertStartsWith(data["env.tag"], "3.") + self.assertEndsWith(data["env.tag"], "-32") + + def test_search_major_2(self): + try: + data = self.run_py(["-2"], allow_fail=True) + except subprocess.CalledProcessError: + if not is_installed("2.7"): + raise unittest.SkipTest("requires at least one Python 2.x install") + self.assertEqual("PythonCore", data["env.company"]) + self.assertStartsWith(data["env.tag"], "2.") + + def test_py_default(self): + with self.py_ini(TEST_PY_DEFAULTS): + data = self.run_py(["-arg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual("X.Y.exe -arg", data["stdout"].strip()) + + def test_py2_default(self): + with self.py_ini(TEST_PY_DEFAULTS): + data = self.run_py(["-2", "-arg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-32", data["SearchInfo.tag"]) + self.assertEqual("X.Y-32.exe -arg", data["stdout"].strip()) + + def test_py3_default(self): + with self.py_ini(TEST_PY_DEFAULTS): + data = self.run_py(["-3", "-arg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) + self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test -arg", data["stdout"].strip()) + + def test_py_default_env(self): + data = self.run_py(["-arg"], env=TEST_PY_ENV) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual("X.Y.exe -arg", data["stdout"].strip()) + + def test_py2_default_env(self): + data = self.run_py(["-2", "-arg"], env=TEST_PY_ENV) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-32", data["SearchInfo.tag"]) + self.assertEqual("X.Y-32.exe -arg", data["stdout"].strip()) + + def test_py3_default_env(self): + data = self.run_py(["-3", "-arg"], env=TEST_PY_ENV) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) + self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test -arg", data["stdout"].strip()) + + def test_py_default_short_argv0(self): + with self.py_ini(TEST_PY_DEFAULTS): + for argv0 in ['"py.exe"', 'py.exe', '"py"', 'py']: + with self.subTest(argv0): + data = self.run_py(["--version"], argv=f'{argv0} --version') + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual("X.Y.exe --version", data["stdout"].strip()) + + def test_py_default_in_list(self): + data = self.run_py(["-0"], env=TEST_PY_ENV) + default = None + for line in data["stdout"].splitlines(): + m = re.match(r"\s*-V:(.+?)\s+?\*\s+(.+)$", line) + if m: + default = m.group(1) + break + self.assertEqual("PythonTestSuite/3.100", default) + + def test_virtualenv_in_list(self): + with self.fake_venv() as (venv_exe, env): + data = self.run_py(["-0p"], env=env) + for line in data["stdout"].splitlines(): + m = re.match(r"\s*\*\s+(.+)$", line) + if m: + self.assertEqual(str(venv_exe), m.group(1)) + break + else: + if support.verbose: + print(data["stdout"]) + print(data["stderr"]) + self.fail("did not find active venv path") + + data = self.run_py(["-0"], env=env) + for line in data["stdout"].splitlines(): + m = re.match(r"\s*\*\s+(.+)$", line) + if m: + self.assertEqual("Active venv", m.group(1)) + break + else: + self.fail("did not find active venv entry") + + def test_virtualenv_with_env(self): + with self.fake_venv() as (venv_exe, env): + data1 = self.run_py([], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) + data2 = self.run_py(["-V:PythonTestSuite/3"], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) + # Compare stdout, because stderr goes via ascii + self.assertEqual(data1["stdout"].strip(), quote(venv_exe)) + self.assertEqual(data1["SearchInfo.lowPriorityTag"], "True") + # Ensure passing the argument doesn't trigger the same behaviour + self.assertNotEqual(data2["stdout"].strip(), quote(venv_exe)) + self.assertNotEqual(data2["SearchInfo.lowPriorityTag"], "True") + + def test_py_shebang(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) + + def test_python_shebang(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! python -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) + + def test_py2_shebang(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python2 -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-32", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py3_shebang(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python3 -prearg") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py_shebang_nl(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python -prearg\n") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py2_shebang_nl(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python2 -prearg\n") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-32", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py3_shebang_nl(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python3 -prearg\n") as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_py_shebang_short_argv0(self): + with self.py_ini(TEST_PY_DEFAULTS): + with self.script("#! /usr/bin/python -prearg") as script: + # Override argv to only pass "py.exe" as the command + data = self.run_py([script, "-postarg"], argv=f'"py.exe" "{script}" -postarg') + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f'X.Y.exe -prearg "{script}" -postarg', data["stdout"].strip()) + + def test_py_shebang_valid_bom(self): + with self.py_ini(TEST_PY_DEFAULTS): + content = "#! /usr/bin/python -prearg".encode("utf-8") + with self.script(b"\xEF\xBB\xBF" + content) as script: + data = self.run_py([script, "-postarg"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) + + def test_py_shebang_invalid_bom(self): + with self.py_ini(TEST_PY_DEFAULTS): + content = "#! /usr/bin/python3 -prearg".encode("utf-8") + with self.script(b"\xEF\xAA\xBF" + content) as script: + data = self.run_py([script, "-postarg"]) + self.assertIn("Invalid BOM", data["stderr"]) + self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) + self.assertEqual("3.100", data["SearchInfo.tag"]) + self.assertEqual(f"X.Y.exe {quote(script)} -postarg", data["stdout"].strip()) + + def test_py_handle_64_in_ini(self): + with self.py_ini("\n".join(["[defaults]", "python=3.999-64"])): + # Expect this to fail, but should get oldStyleTag flipped on + data = self.run_py([], allow_fail=True, expect_returncode=103) + self.assertEqual("3.999-64", data["SearchInfo.tag"]) + self.assertEqual("True", data["SearchInfo.oldStyleTag"]) + + def test_search_path(self): + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) + with self.py_ini(TEST_PY_DEFAULTS): + with self.script(f"#! /usr/bin/env {exe.stem} -prearg") as script: + data = self.run_py( + [script, "-postarg"], + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, + ) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_search_path_exe(self): + # Leave the .exe on the name to ensure we don't add it a second time + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) + with self.py_ini(TEST_PY_DEFAULTS): + with self.script(f"#! /usr/bin/env {exe.name} -prearg") as script: + data = self.run_py( + [script, "-postarg"], + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, + ) + self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", + data["stdout"].strip()) + + def test_recursive_search_path(self): + stem = self.get_py_exe().stem + with self.py_ini(TEST_PY_DEFAULTS): + with self.script(f"#! /usr/bin/env {stem}") as script: + data = self.run_py( + [script], + env={"PATH": f"{self.get_py_exe().parent};{os.getenv('PATH')}"}, + ) + # The recursive search is ignored and we get normal "py" behavior + self.assertEqual(f"X.Y.exe {quote(script)}", data["stdout"].strip()) + + def test_install(self): + data = self.run_py(["-V:3.10"], env={"PYLAUNCHER_ALWAYS_INSTALL": "1"}, expect_returncode=111) + cmd = data["stdout"].strip() + # If winget is runnable, we should find it. Otherwise, we'll be trying + # to open the Store. + try: + subprocess.check_call(["winget.exe", "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except FileNotFoundError: + self.assertIn("ms-windows-store://", cmd) + else: + self.assertIn("winget.exe", cmd) + # Both command lines include the store ID + self.assertIn("9PJPW5LDXLZ5", cmd) + + def test_literal_shebang_absolute(self): + with self.script("#! C:/some_random_app -witharg") as script: + data = self.run_py([script]) + self.assertEqual( + f"C:\\some_random_app -witharg {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_relative(self): + with self.script("#! ..\\some_random_app -witharg") as script: + data = self.run_py([script]) + self.assertEqual( + f"{quote(script.parent.parent / 'some_random_app')} -witharg {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_quoted(self): + with self.script('#! "some random app" -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", + data["stdout"].strip(), + ) + + with self.script('#! some" random "app -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_quoted_escape(self): + with self.script('#! some\\" random "app -witharg') as script: + data = self.run_py([script]) + self.assertEqual( + f"{quote(script.parent / 'some/ random app')} -witharg {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_command(self): + with self.py_ini(TEST_PY_COMMANDS): + with self.script('#! test-command arg1') as script: + data = self.run_py([script]) + self.assertEqual( + f"TEST_EXE.exe arg1 {quote(script)}", + data["stdout"].strip(), + ) + + def test_literal_shebang_invalid_template(self): + with self.script('#! /usr/bin/not-python arg1') as script: + data = self.run_py([script]) + expect = script.parent / "/usr/bin/not-python" + self.assertEqual( + f"{quote(expect)} arg1 {quote(script)}", + data["stdout"].strip(), + ) + + def test_shebang_command_in_venv(self): + stem = "python-that-is-not-on-path" + + # First ensure that our test name doesn't exist, and the launcher does + # not match any installed env + with self.script(f'#! /usr/bin/env {stem} arg1') as script: + data = self.run_py([script], expect_returncode=103) + + with self.fake_venv() as (venv_exe, env): + # Put a "normal" Python on PATH as a distraction. + # The active VIRTUAL_ENV should be preferred when the name isn't an + # exact match. + exe = Path(Path(venv_exe).name).absolute() + exe.touch() + self.addCleanup(exe.unlink) + env["PATH"] = f"{exe.parent};{os.environ['PATH']}" + + with self.script(f'#! /usr/bin/env {stem} arg1') as script: + data = self.run_py([script], env=env) + self.assertEqual(data["stdout"].strip(), f"{quote(venv_exe)} arg1 {quote(script)}") + + with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: + data = self.run_py([script], env=env) + self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") + + def test_shebang_executable_extension(self): + with self.script('#! /usr/bin/env python3.99') as script: + data = self.run_py([script], expect_returncode=103) + expect = "# Search PATH for python3.99.exe" + actual = [line.strip() for line in data["stderr"].splitlines() + if line.startswith("# Search PATH")] + self.assertEqual([expect], actual) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 706daf65be9..02f65338428 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -4,9 +4,13 @@ import unittest import os.path import tempfile +import threading import tokenize +from importlib.machinery import ModuleSpec from test import support from test.support import os_helper +from test.support import threading_helper +from test.support.script_helper import assert_python_ok FILENAME = linecache.__file__ @@ -82,6 +86,10 @@ def test_getlines(self): class EmptyFile(GetLineTestsGoodData, unittest.TestCase): file_list = [] + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, ['\n']) + class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): file_list = ['\n'] @@ -97,6 +105,16 @@ class BadUnicode_WithDeclaration(GetLineTestsBadData, unittest.TestCase): file_byte_string = b'# coding=utf-8\n\x80abc' +class FakeLoader: + def get_source(self, fullname): + return f'source for {fullname}' + + +class NoSourceLoader: + def get_source(self, fullname): + return None + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -187,8 +205,6 @@ def test_lazycache_no_globals(self): self.assertEqual(False, linecache.lazycache(FILENAME, None)) self.assertEqual(lines, linecache.getlines(FILENAME)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazycache_smoke(self): lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) linecache.clearcache() @@ -199,8 +215,6 @@ def test_lazycache_smoke(self): # globals: this would error if the lazy value wasn't resolved. self.assertEqual(lines, linecache.getlines(NONEXISTENT_FILENAME)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazycache_provide_after_failed_lookup(self): linecache.clearcache() lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) @@ -219,8 +233,6 @@ def test_lazycache_bad_filename(self): self.assertEqual(False, linecache.lazycache('', globals())) self.assertEqual(False, linecache.lazycache('<foo>', globals())) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazycache_already_cached(self): linecache.clearcache() lines = linecache.getlines(NONEXISTENT_FILENAME, globals()) @@ -244,6 +256,83 @@ def raise_memoryerror(*args, **kwargs): self.assertEqual(lines3, []) self.assertEqual(linecache.getlines(FILENAME), lines) + def test_loader(self): + filename = 'scheme://path' + + for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': loader} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + for spec in (None, object(), ModuleSpec('', FakeLoader())): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + linecache.clearcache() + spec = ModuleSpec('x.y.z', FakeLoader()) + module_globals = {'__name__': 'a.b.c', '__loader__': spec.loader, + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for x.y.z\n']) + + def test_frozen(self): + filename = '<frozen fakemodule>' + module_globals = {'__file__': FILENAME} + empty = linecache.getlines(filename) + self.assertEqual(empty, []) + lines = linecache.getlines(filename, module_globals) + self.assertGreater(len(lines), 0) + lines_cached = linecache.getlines(filename) + self.assertEqual(lines, lines_cached) + linecache.clearcache() + empty = linecache.getlines(filename) + self.assertEqual(empty, []) + + def test_invalid_names(self): + for name, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + # A filename with surrogate codes. A UnicodeEncodeError is raised + # by os.stat() upon querying, which is a subclass of ValueError. + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + # For POSIX platforms, an OSError will be raised but for Windows + # platforms, a ValueError is raised due to the path_t converter. + # See: https://github.com/python/cpython/issues/122170 + ('a' * 1_000_000, 'very long filename'), + ]: + with self.subTest(f'updatecache: {desc}'): + linecache.clearcache() + lines = linecache.updatecache(name) + self.assertListEqual(lines, []) + self.assertNotIn(name, linecache.cache) + + # hack into the cache (it shouldn't be allowed + # but we never know what people do...) + for key, fullname in [(name, 'ok'), ('key', name), (name, name)]: + with self.subTest(f'checkcache: {desc}', + key=key, fullname=fullname): + linecache.clearcache() + linecache.cache[key] = (0, 1234, [], fullname) + linecache.checkcache(key) + self.assertNotIn(key, linecache.cache) + + # just to be sure that we did not mess with cache + linecache.clearcache() + + def test_linecache_python_string(self): + cmdline = "import linecache;assert len(linecache.cache) == 0" + retcode, stdout, stderr = assert_python_ok('-c', cmdline) + self.assertEqual(retcode, 0) + self.assertEqual(stdout, b'') + self.assertEqual(stderr, b'') class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): @@ -287,5 +376,40 @@ def test_checkcache_with_no_parameter(self): self.assertIn(self.unchanged_file, linecache.cache) +class MultiThreadingTest(unittest.TestCase): + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_read_write_safety(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + filenames = [] + for i in range(10): + name = os.path.join(tmpdirname, f"test_{i}.py") + with open(name, "w") as h: + h.write("import time\n") + h.write("import system\n") + filenames.append(name) + + def linecache_get_line(b): + b.wait() + for _ in range(100): + for name in filenames: + linecache.getline(name, 1) + + def check(funcs): + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + check([linecache_get_line] * 20) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 575a77db0b1..e320a2008a8 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -1,6 +1,10 @@ +import signal import sys -from test import list_tests +import textwrap +from test import list_tests, support from test.support import cpython_only +from test.support.import_helper import import_module +from test.support.script_helper import assert_python_failure, assert_python_ok import pickle import unittest @@ -22,6 +26,8 @@ def test_basic(self): if sys.maxsize == 0x7fffffff: # This test can currently only work on 32-bit machines. + # XXX If/when PySequence_Length() returns a ssize_t, it should be + # XXX re-enabled. # Verify clearing of bug #556025. # This assumes that the max data size (sys.maxint) == max # address size this also assumes that the address size is at @@ -44,8 +50,7 @@ def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): list(sequence=[]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords_in_subclass(self): class subclass(list): pass @@ -96,8 +101,12 @@ def imul(a, b): a *= b self.assertRaises((MemoryError, OverflowError), mul, lst, n) self.assertRaises((MemoryError, OverflowError), imul, lst, n) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_empty_slice(self): + x = [] + x[:] = x + self.assertEqual(x, []) + + @unittest.skip("TODO: RUSTPYTHON; crash") def test_list_resize_overflow(self): # gh-97616: test new_allocated * sizeof(PyObject*) overflow # check in list_resize() @@ -105,12 +114,25 @@ def test_list_resize_overflow(self): del lst[1:] self.assertEqual(len(lst), 1) - size = ((2 ** (tuple.__itemsize__ * 8) - 1) // 2) + size = sys.maxsize with self.assertRaises((MemoryError, OverflowError)): lst * size with self.assertRaises((MemoryError, OverflowError)): lst *= size + def test_repr_mutate(self): + class Obj: + @staticmethod + def __repr__(): + try: + mylist.pop() + except IndexError: + pass + return 'obj' + + mylist = [Obj() for _ in range(5)] + self.assertEqual(repr(mylist), '[obj, obj, obj]') + def test_repr_large(self): # Check the repr of large list objects def check(n): @@ -232,6 +254,32 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) + @unittest.skip("TODO: RUSTPYTHON; hang") + def test_lt_operator_modifying_operand(self): + # See gh-120298 + class evil: + def __lt__(self, other): + other.clear() + return NotImplemented + + a = [[evil()]] + with self.assertRaises(TypeError): + a[0] < a + + def test_list_index_modifing_operand(self): + # See gh-120384 + class evil: + def __init__(self, lst): + self.lst = lst + def __iter__(self): + yield from self.lst + self.lst.clear() + + lst = list(range(5)) + operand = evil(lst) + with self.assertRaises(ValueError): + lst[::-1] = operand + @cpython_only def test_preallocation(self): iterable = [0] * 10 @@ -272,6 +320,54 @@ def __eq__(self, other): lst = [X(), X()] X() in lst + def test_tier2_invalidates_iterator(self): + # GH-121012 + for _ in range(100): + a = [1, 2, 3] + it = iter(a) + for _ in it: + pass + a.append(4) + self.assertEqual(list(it), []) + + @support.cpython_only + def test_no_memory(self): + # gh-118331: Make sure we don't crash if list allocation fails + import_module("_testcapi") + code = textwrap.dedent(""" + import _testcapi, sys + # Prime the freelist + l = [None] + del l + _testcapi.set_nomemory(0) + l = [None] + """) + rc, _, _ = assert_python_failure("-c", code) + if support.MS_WINDOWS: + # STATUS_ACCESS_VIOLATION + self.assertNotEqual(rc, 0xC0000005) + else: + self.assertNotEqual(rc, -int(signal.SIGSEGV)) + + def test_deopt_from_append_list(self): + # gh-132011: it used to crash, because + # of `CALL_LIST_APPEND` specialization failure. + code = textwrap.dedent(""" + l = [] + def lappend(l, x, y): + l.append((x, y)) + for x in range(3): + lappend(l, None, None) + try: + lappend(list, None, None) + except TypeError: + pass + else: + raise AssertionError + """) + + rc, _, _ = assert_python_ok("-c", code) + self.assertEqual(rc, 0) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 91bf2547edc..964383966c2 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -1,6 +1,11 @@ import doctest +import textwrap +import traceback +import types import unittest +from test.support import BrokenIter + doctests = """ ########### Tests borrowed from or inspired by test_genexps.py ############ @@ -87,65 +92,687 @@ >>> [None for i in range(10)] [None, None, None, None, None, None, None, None, None, None] -########### Tests for various scoping corner cases ############ - -Return lambdas that use the iteration variable as a default argument - - >>> items = [(lambda i=i: i) for i in range(5)] - >>> [x() for x in items] - [0, 1, 2, 3, 4] - -Same again, only this time as a closure variable - - >>> items = [(lambda: i) for i in range(5)] - >>> [x() for x in items] - [4, 4, 4, 4, 4] - -Another way to test that the iteration variable is local to the list comp - - >>> items = [(lambda: i) for i in range(5)] - >>> i = 20 - >>> [x() for x in items] - [4, 4, 4, 4, 4] - -And confirm that a closure can jump over the list comp scope - - >>> items = [(lambda: y) for i in range(5)] - >>> y = 2 - >>> [x() for x in items] - [2, 2, 2, 2, 2] - -We also repeat each of the above scoping tests inside a function - - >>> def test_func(): - ... items = [(lambda i=i: i) for i in range(5)] - ... return [x() for x in items] - >>> test_func() - [0, 1, 2, 3, 4] - - >>> def test_func(): - ... items = [(lambda: i) for i in range(5)] - ... return [x() for x in items] - >>> test_func() - [4, 4, 4, 4, 4] - - >>> def test_func(): - ... items = [(lambda: i) for i in range(5)] - ... i = 20 - ... return [x() for x in items] - >>> test_func() - [4, 4, 4, 4, 4] - - >>> def test_func(): - ... items = [(lambda: y) for i in range(5)] - ... y = 2 - ... return [x() for x in items] - >>> test_func() - [2, 2, 2, 2, 2] - """ +class ListComprehensionTest(unittest.TestCase): + def _check_in_scopes(self, code, outputs=None, ns=None, scopes=None, raises=(), + exec_func=exec): + code = textwrap.dedent(code) + scopes = scopes or ["module", "class", "function"] + for scope in scopes: + with self.subTest(scope=scope): + if scope == "class": + newcode = textwrap.dedent(""" + class _C: + {code} + """).format(code=textwrap.indent(code, " ")) + def get_output(moddict, name): + return getattr(moddict["_C"], name) + elif scope == "function": + newcode = textwrap.dedent(""" + def _f(): + {code} + return locals() + _out = _f() + """).format(code=textwrap.indent(code, " ")) + def get_output(moddict, name): + return moddict["_out"][name] + else: + newcode = code + def get_output(moddict, name): + return moddict[name] + newns = ns.copy() if ns else {} + try: + exec_func(newcode, newns) + except raises as e: + # We care about e.g. NameError vs UnboundLocalError + self.assertIs(type(e), raises) + else: + for k, v in (outputs or {}).items(): + self.assertEqual(get_output(newns, k), v, k) + + def test_lambdas_with_iteration_var_as_default(self): + code = """ + items = [(lambda i=i: i) for i in range(5)] + y = [x() for x in items] + """ + outputs = {"y": [0, 1, 2, 3, 4]} + self._check_in_scopes(code, outputs) + + def test_lambdas_with_free_var(self): + code = """ + items = [(lambda: i) for i in range(5)] + y = [x() for x in items] + """ + outputs = {"y": [4, 4, 4, 4, 4]} + self._check_in_scopes(code, outputs) + + def test_class_scope_free_var_with_class_cell(self): + class C: + def method(self): + super() + return __class__ + items = [(lambda: i) for i in range(5)] + y = [x() for x in items] + + self.assertEqual(C.y, [4, 4, 4, 4, 4]) + self.assertIs(C().method(), C) + + def test_references_super(self): + code = """ + res = [super for x in [1]] + """ + self._check_in_scopes(code, outputs={"res": [super]}) + + def test_references___class__(self): + code = """ + res = [__class__ for x in [1]] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___class___defined(self): + code = """ + __class__ = 2 + res = [__class__ for x in [1]] + """ + self._check_in_scopes( + code, outputs={"res": [2]}, scopes=["module", "function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_references___class___enclosing(self): + code = """ + __class__ = 2 + class C: + res = [__class__ for x in [1]] + res = C.res + """ + self._check_in_scopes(code, raises=NameError) + + def test_super_and_class_cell_in_sibling_comps(self): + code = """ + [super for _ in [1]] + [__class__ for _ in [1]] + """ + self._check_in_scopes(code, raises=NameError) + + def test_inner_cell_shadows_outer(self): + code = """ + items = [(lambda: i) for i in range(5)] + i = 20 + y = [x() for x in items] + """ + outputs = {"y": [4, 4, 4, 4, 4], "i": 20} + self._check_in_scopes(code, outputs) + + def test_inner_cell_shadows_outer_no_store(self): + code = """ + def f(x): + return [lambda: x for x in range(x)], x + fns, x = f(2) + y = [fn() for fn in fns] + """ + outputs = {"y": [1, 1], "x": 2} + self._check_in_scopes(code, outputs) + + def test_closure_can_jump_over_comp_scope(self): + code = """ + items = [(lambda: y) for i in range(5)] + y = 2 + z = [x() for x in items] + """ + outputs = {"z": [2, 2, 2, 2, 2]} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_cell_inner_free_outer(self): + code = """ + def f(): + return [lambda: x for x in (x, [1])[1]] + x = ... + y = [fn() for fn in f()] + """ + outputs = {"y": [1]} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_free_inner_cell_outer(self): + code = """ + g = 2 + def f(): + return g + y = [g for x in [1]] + """ + outputs = {"y": [2]} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + self._check_in_scopes(code, scopes=["class"], raises=NameError) + + def test_inner_cell_shadows_outer_redefined(self): + code = """ + y = 10 + items = [(lambda: y) for y in range(5)] + x = y + y = 20 + out = [z() for z in items] + """ + outputs = {"x": 10, "out": [4, 4, 4, 4, 4]} + self._check_in_scopes(code, outputs) + + def test_shadows_outer_cell(self): + code = """ + def inner(): + return g + [g for g in range(5)] + x = inner() + """ + outputs = {"x": -1} + self._check_in_scopes(code, outputs, ns={"g": -1}) + + def test_explicit_global(self): + code = """ + global g + x = g + g = 2 + items = [g for g in [1]] + y = g + """ + outputs = {"x": 1, "y": 2, "items": [1]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + + def test_explicit_global_2(self): + code = """ + global g + x = g + g = 2 + items = [g for x in [1]] + y = g + """ + outputs = {"x": 1, "y": 2, "items": [2]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + + def test_explicit_global_3(self): + code = """ + global g + fns = [lambda: g for g in [2]] + items = [fn() for fn in fns] + """ + outputs = {"items": [2]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + + def test_assignment_expression(self): + code = """ + x = -1 + items = [(x:=y) for y in range(3)] + """ + outputs = {"x": 2} + # assignment expression in comprehension is disallowed in class scope + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_free_var_in_comp_child(self): + code = """ + lst = range(3) + funcs = [lambda: x for x in lst] + inc = [x + 1 for x in lst] + [x for x in inc] + x = funcs[0]() + """ + outputs = {"x": 2} + self._check_in_scopes(code, outputs) + + def test_shadow_with_free_and_local(self): + code = """ + lst = range(3) + x = -1 + funcs = [lambda: x for x in lst] + items = [x + 1 for x in lst] + """ + outputs = {"x": -1} + self._check_in_scopes(code, outputs) + + def test_shadow_comp_iterable_name(self): + code = """ + x = [1] + y = [x for x in x] + """ + outputs = {"x": [1]} + self._check_in_scopes(code, outputs) + + def test_nested_free(self): + code = """ + x = 1 + def g(): + [x for x in range(3)] + return x + g() + """ + outputs = {"x": 1} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_introspecting_frame_locals(self): + code = """ + import sys + [i for i in range(2)] + i = 20 + sys._getframe().f_locals + """ + outputs = {"i": 20} + self._check_in_scopes(code, outputs) + + def test_nested(self): + code = """ + l = [2, 3] + y = [[x ** 2 for x in range(x)] for x in l] + """ + outputs = {"y": [[0, 1], [0, 1, 4]]} + self._check_in_scopes(code, outputs) + + def test_nested_2(self): + code = """ + l = [1, 2, 3] + x = 3 + y = [x for [x ** x for x in range(x)][x - 1] in l] + """ + outputs = {"y": [3, 3, 3]} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + self._check_in_scopes(code, scopes=["class"], raises=NameError) + + def test_nested_3(self): + code = """ + l = [(1, 2), (3, 4), (5, 6)] + y = [x for (x, [x ** x for x in range(x)][x - 1]) in l] + """ + outputs = {"y": [1, 3, 5]} + self._check_in_scopes(code, outputs) + + def test_nested_4(self): + code = """ + items = [([lambda: x for x in range(2)], lambda: x) for x in range(3)] + out = [([fn() for fn in fns], fn()) for fns, fn in items] + """ + outputs = {"out": [([1, 1], 2), ([1, 1], 2), ([1, 1], 2)]} + self._check_in_scopes(code, outputs) + + def test_nameerror(self): + code = """ + [x for x in [1]] + x + """ + + self._check_in_scopes(code, raises=NameError) + + def test_dunder_name(self): + code = """ + y = [__x for __x in [1]] + """ + outputs = {"y": [1]} + self._check_in_scopes(code, outputs) + + def test_unbound_local_after_comprehension(self): + def f(): + if False: + x = 0 + [x for x in [1]] + return x + + with self.assertRaises(UnboundLocalError): + f() + + def test_unbound_local_inside_comprehension(self): + def f(): + l = [None] + return [1 for (l[0], l) in [[1, 2]]] + + with self.assertRaises(UnboundLocalError): + f() + + def test_global_outside_cellvar_inside_plus_freevar(self): + code = """ + a = 1 + def f(): + func, = [(lambda: b) for b in [a]] + return b, func() + x = f() + """ + self._check_in_scopes( + code, {"x": (2, 1)}, ns={"b": 2}, scopes=["function", "module"]) + # inside a class, the `a = 1` assignment is not visible + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_cell_in_nested_comprehension(self): + code = """ + a = 1 + def f(): + (func, inner_b), = [[lambda: b for b in c] + [b] for c in [[a]]] + return b, inner_b, func() + x = f() + """ + self._check_in_scopes( + code, {"x": (2, 2, 1)}, ns={"b": 2}, scopes=["function", "module"]) + # inside a class, the `a = 1` assignment is not visible + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_name_error_in_class_scope(self): + code = """ + y = 1 + [x + y for x in range(2)] + """ + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_global_in_class_scope(self): + code = """ + y = 2 + vals = [(x, y) for x in range(2)] + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["class"]) + + def test_in_class_scope_inside_function_1(self): + code = """ + class C: + y = 2 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, ns={"y": 1}, scopes=["function"]) + + def test_in_class_scope_inside_function_2(self): + code = """ + y = 1 + class C: + y = 2 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, scopes=["function"]) + + def test_in_class_scope_with_global(self): + code = """ + y = 1 + class C: + global y + y = 2 + # Ensure the listcomp uses the global, not the value in the + # class namespace + locals()['y'] = 3 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 2), (1, 2)]} + self._check_in_scopes(code, outputs, scopes=["module", "class"]) + outputs = {"vals": [(0, 1), (1, 1)]} + self._check_in_scopes(code, outputs, scopes=["function"]) + + def test_in_class_scope_with_nonlocal(self): + code = """ + y = 1 + class C: + nonlocal y + y = 2 + # Ensure the listcomp uses the global, not the value in the + # class namespace + locals()['y'] = 3 + vals = [(x, y) for x in range(2)] + vals = C.vals + """ + outputs = {"vals": [(0, 2), (1, 2)]} + self._check_in_scopes(code, outputs, scopes=["function"]) + + def test_nested_has_free_var(self): + code = """ + items = [a for a in [1] if [a for _ in [0]]] + """ + outputs = {"items": [1]} + self._check_in_scopes(code, outputs, scopes=["class"]) + + def test_nested_free_var_not_bound_in_outer_comp(self): + code = """ + z = 1 + items = [a for a in [1] if [x for x in [1] if z]] + """ + self._check_in_scopes(code, {"items": [1]}, scopes=["module", "function"]) + self._check_in_scopes(code, {"items": []}, ns={"z": 0}, scopes=["class"]) + + def test_nested_free_var_in_iter(self): + code = """ + items = [_C for _C in [1] for [0, 1][[x for x in [1] if _C][0]] in [2]] + """ + self._check_in_scopes(code, {"items": [1]}) + + def test_nested_free_var_in_expr(self): + code = """ + items = [(_C, [x for x in [1] if _C]) for _C in [0, 1]] + """ + self._check_in_scopes(code, {"items": [(0, []), (1, [1])]}) + + def test_nested_listcomp_in_lambda(self): + code = """ + f = [(z, lambda y: [(x, y, z) for x in [3]]) for z in [1]] + (z, func), = f + out = func(2) + """ + self._check_in_scopes(code, {"z": 1, "out": [(3, 2, 1)]}) + + def test_lambda_in_iter(self): + code = """ + (func, c), = [(a, b) for b in [1] for a in [lambda : a]] + d = func() + assert d is func + # must use "a" in this scope + e = a if False else None + """ + self._check_in_scopes(code, {"c": 1, "e": None}) + + def test_assign_to_comp_iter_var_in_outer_function(self): + code = """ + a = [1 for a in [0]] + """ + self._check_in_scopes(code, {"a": [1]}, scopes=["function"]) + + def test_no_leakage_to_locals(self): + code = """ + def b(): + [a for b in [1] for _ in []] + return b, locals() + r, s = b() + x = r is b + y = list(s.keys()) + """ + self._check_in_scopes(code, {"x": True, "y": []}, scopes=["module"]) + self._check_in_scopes(code, {"x": True, "y": ["b"]}, scopes=["function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_iter_var_available_in_locals(self): + code = """ + l = [1, 2] + y = 0 + items = [locals()["x"] for x in l] + items2 = [vars()["x"] for x in l] + items3 = [("x" in dir()) for x in l] + items4 = [eval("x") for x in l] + # x is available, and does not overwrite y + [exec("y = x") for x in l] + """ + self._check_in_scopes( + code, + { + "items": [1, 2], + "items2": [1, 2], + "items3": [True, True], + "items4": [1, 2], + "y": 0 + } + ) + + def test_comp_in_try_except(self): + template = """ + value = ["ab"] + result = snapshot = None + try: + result = [{func}(value) for value in value] + except ValueError: + snapshot = value + raise + """ + # No exception. + code = template.format(func='len') + self._check_in_scopes(code, {"value": ["ab"], "result": [2], "snapshot": None}) + # Handles exception. + code = template.format(func='int') + self._check_in_scopes(code, {"value": ["ab"], "result": None, "snapshot": ["ab"]}, + raises=ValueError) + + def test_comp_in_try_finally(self): + template = """ + value = ["ab"] + result = snapshot = None + try: + result = [{func}(value) for value in value] + finally: + snapshot = value + """ + # No exception. + code = template.format(func='len') + self._check_in_scopes(code, {"value": ["ab"], "result": [2], "snapshot": ["ab"]}) + # Handles exception. + code = template.format(func='int') + self._check_in_scopes(code, {"value": ["ab"], "result": None, "snapshot": ["ab"]}, + raises=ValueError) + + def test_exception_in_post_comp_call(self): + code = """ + value = [1, None] + try: + [v for v in value].sort() + except TypeError: + pass + """ + self._check_in_scopes(code, {"value": [1, None]}) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_frame_locals(self): + code = """ + val = "a" in [sys._getframe().f_locals for a in [0]][0] + """ + import sys + self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) + + code = """ + val = [sys._getframe().f_locals["a"] for a in [0]][0] + """ + self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) + + def _recursive_replace(self, maybe_code): + if not isinstance(maybe_code, types.CodeType): + return maybe_code + return maybe_code.replace(co_consts=tuple( + self._recursive_replace(c) for c in maybe_code.co_consts + )) + + def _replacing_exec(self, code_string, ns): + co = compile(code_string, "<string>", "exec") + co = self._recursive_replace(co) + exec(co, ns) + + def test_code_replace(self): + code = """ + x = 3 + [x for x in (1, 2)] + dir() + y = [x] + """ + self._check_in_scopes(code, {"y": [3], "x": 3}) + self._check_in_scopes(code, {"y": [3], "x": 3}, exec_func=self._replacing_exec) + + def test_code_replace_extended_arg(self): + num_names = 300 + assignments = "; ".join(f"x{i} = {i}" for i in range(num_names)) + name_list = ", ".join(f"x{i}" for i in range(num_names)) + expected = { + "y": list(range(num_names)), + **{f"x{i}": i for i in range(num_names)} + } + code = f""" + {assignments} + [({name_list}) for {name_list} in (range(300),)] + dir() + y = [{name_list}] + """ + self._check_in_scopes(code, expected) + self._check_in_scopes(code, expected, exec_func=self._replacing_exec) + + def test_multiple_comprehension_name_reuse(self): + code = """ + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"y": [3]}, ns={"x": 3}) + + code = """ + x = 2 + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"x": 2, "y": [3]}, ns={"x": 3}, scopes=["class"]) + self._check_in_scopes(code, {"x": 2, "y": [2]}, ns={"x": 3}, scopes=["function", "module"]) + + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should be the iterator expression + + def init_raises(): + try: + [x for x in BrokenIter(init_raises=True)] + except Exception as e: + return e + + def next_raises(): + try: + [x for x in BrokenIter(next_raises=True)] + except Exception as e: + return e + + def iter_raises(): + try: + [x for x in BrokenIter(iter_raises=True)] + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + def test_only_calls_dunder_iter_once(self): + + class Iterator: + + def __init__(self): + self.val = 0 + + def __next__(self): + if self.val == 2: + raise StopIteration + self.val += 1 + return self.val + + # No __iter__ method + + class C: + + def __iter__(self): + return Iterator() + + self.assertEqual([1, 2], [i for i in C()]) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index b0d79985597..8e49aa8954e 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,11 +1,19 @@ from decimal import Decimal -from test.support import verbose, is_android, is_emscripten, is_wasi +from test.support import cpython_only, verbose, is_android, linked_to_musl, os_helper from test.support.warnings_helper import check_warnings +from test.support.import_helper import ensure_lazy_imports, import_fresh_module +from unittest import mock import unittest import locale +import os import sys import codecs +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("locale", {"re", "warnings"}) + class BaseLocalizedTest(unittest.TestCase): # @@ -342,26 +350,23 @@ def setUp(self): enc = codecs.lookup(locale.getencoding() or 'ascii').name if enc not in ('utf-8', 'iso8859-1', 'cp1252'): raise unittest.SkipTest('encoding not suitable') - if enc != 'iso8859-1' and (sys.platform == 'darwin' or is_android or - sys.platform.startswith('freebsd')): + if enc != 'iso8859-1' and is_android: raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs') BaseLocalizedTest.setUp(self) @unittest.skipIf(sys.platform.startswith('aix'), 'bpo-29972: broken test on AIX') - @unittest.skipIf( - is_emscripten or is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) + @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") def test_strcoll_with_diacritic(self): self.assertLess(locale.strcoll('à', 'b'), 0) @unittest.skipIf(sys.platform.startswith('aix'), 'bpo-29972: broken test on AIX') - @unittest.skipIf( - is_emscripten or is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) + @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") def test_strxfrm_with_diacritic(self): self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) @@ -382,6 +387,10 @@ def test_c(self): self.check('c', 'C') self.check('posix', 'C') + def test_c_utf8(self): + self.check('c.utf8', 'C.UTF-8') + self.check('C.UTF-8', 'C.UTF-8') + def test_english(self): self.check('en', 'en_US.ISO8859-1') self.check('EN', 'en_US.ISO8859-1') @@ -477,13 +486,60 @@ def test_japanese(self): self.check('jp_jp', 'ja_JP.eucJP') +class TestRealLocales(unittest.TestCase): + def setUp(self): + oldlocale = locale.setlocale(locale.LC_CTYPE) + self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) + + def test_getsetlocale_issue1813(self): + # Issue #1813: setting and getting the locale under a Turkish locale + try: + locale.setlocale(locale.LC_CTYPE, 'tr_TR') + except locale.Error: + # Unsupported locale on this system + self.skipTest('test needs Turkish locale') + loc = locale.getlocale(locale.LC_CTYPE) + if verbose: + print('testing with %a' % (loc,), end=' ', flush=True) + try: + locale.setlocale(locale.LC_CTYPE, loc) + except locale.Error as exc: + # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) + # and the tr_TR locale on Windows. getlocale() builds a locale + # which is not recognize by setlocale(). + self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") + self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) + + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + def test_setlocale_long_encoding(self): + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'English.%016d' % 1252) + locale.setlocale(locale.LC_CTYPE, 'English.%015d' % 1252) + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.1252', loc) + loc2 = loc.replace('.1252', '.%016d' % 1252, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + loc2 = loc.replace('.1252', '.%015d' % 1252, 1) + locale.setlocale(locale.LC_ALL, loc2) + + # gh-137273: Debug assertion failure on Windows for long encoding. + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'en_US.' + 'x'*16) + locale.setlocale(locale.LC_CTYPE, 'en_US.UTF-8') + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.UTF-8', loc) + loc2 = loc.replace('.UTF-8', '.' + 'x'*16, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + + class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is # valid. Furthermore LC_CTYPE=UTF is used by the UTF-8 locale coercing # during interpreter startup (on macOS). import _locale - import os self.assertEqual(locale._parse_localename('UTF-8'), (None, 'UTF-8')) @@ -493,25 +549,14 @@ def test_defaults_UTF8(self): else: orig_getlocale = None - orig_env = {} try: - for key in ('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE'): - if key in os.environ: - orig_env[key] = os.environ[key] - del os.environ[key] - - os.environ['LC_CTYPE'] = 'UTF-8' - - with check_warnings(('', DeprecationWarning)): - self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) + with os_helper.EnvironmentVarGuard() as env: + env.unset('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE') + env.set('LC_CTYPE', 'UTF-8') + with check_warnings(('', DeprecationWarning)): + self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) finally: - for k in orig_env: - os.environ[k] = orig_env[k] - - if 'LC_CTYPE' not in orig_env: - del os.environ['LC_CTYPE'] - if orig_getlocale is not None: _locale._getdefaultlocale = orig_getlocale @@ -523,6 +568,15 @@ def test_getencoding(self): # make sure it is valid codecs.lookup(enc) + def test_getencoding_fallback(self): + # When _locale.getencoding() is missing, locale.getencoding() uses + # the Python filesystem + encoding = 'FALLBACK_ENCODING' + with mock.patch.object(sys, 'getfilesystemencoding', + return_value=encoding): + locale_fallback = import_fresh_module('locale', blocked=['_locale']) + self.assertEqual(locale_fallback.getencoding(), encoding) + def test_getpreferredencoding(self): # Invoke getpreferredencoding to make sure it does not cause exceptions. enc = locale.getpreferredencoding() @@ -546,27 +600,6 @@ def test_setlocale_category(self): # crasher from bug #7419 self.assertRaises(locale.Error, locale.setlocale, 12345) - def test_getsetlocale_issue1813(self): - # Issue #1813: setting and getting the locale under a Turkish locale - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') - loc = locale.getlocale(locale.LC_CTYPE) - if verbose: - print('testing with %a' % (loc,), end=' ', flush=True) - try: - locale.setlocale(locale.LC_CTYPE, loc) - except locale.Error as exc: - # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) - # and the tr_TR locale on Windows. getlocale() builds a locale - # which is not recognize by setlocale(). - self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") - self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) - def test_invalid_locale_format_in_localetuple(self): with self.assertRaises(TypeError): locale.setlocale(locale.LC_ALL, b'fi_FI') diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py new file mode 100644 index 00000000000..ef52f147764 --- /dev/null +++ b/Lib/test/test_logging.py @@ -0,0 +1,7164 @@ +# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved. +# +# Permission to use, copy, modify, and distribute this software and its +# documentation for any purpose and without fee is hereby granted, +# provided that the above copyright notice appear in all copies and that +# both that copyright notice and this permission notice appear in +# supporting documentation, and that the name of Vinay Sajip +# not be used in advertising or publicity pertaining to distribution +# of the software without specific, written prior permission. +# VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING +# ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +# VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR +# ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER +# IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""Test harness for the logging module. Run all tests. + +Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved. +""" +import logging +import logging.handlers +import logging.config + +import codecs +import configparser +import copy +import datetime +import pathlib +import pickle +import io +import itertools +import gc +import json +import os +import queue +import random +import re +import shutil +import socket +import struct +import sys +import tempfile +from test.support.script_helper import assert_python_ok, assert_python_failure +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import socket_helper +from test.support import threading_helper +from test.support import warnings_helper +from test.support import asyncore +from test.support import smtpd +from test.support.logging_helper import TestHandler +import textwrap +import threading +import asyncio +import time +import unittest +import warnings +import weakref + +from http.server import HTTPServer, BaseHTTPRequestHandler +from unittest.mock import patch +from urllib.parse import urlparse, parse_qs +from socketserver import (ThreadingUDPServer, DatagramRequestHandler, + ThreadingTCPServer, StreamRequestHandler) + +try: + import win32evtlog, win32evtlogutil, pywintypes +except ImportError: + win32evtlog = win32evtlogutil = pywintypes = None + +try: + import zlib +except ImportError: + pass + + +# gh-89363: Skip fork() test if Python is built with Address Sanitizer (ASAN) +# to work around a libasan race condition, dead lock in pthread_create(). +skip_if_asan_fork = unittest.skipIf( + support.HAVE_ASAN_FORK_BUG, + "libasan has a pthread_create() dead lock related to thread+fork") +skip_if_tsan_fork = unittest.skipIf( + support.check_sanitizer(thread=True), + "TSAN doesn't support threads after fork") + + +class BaseTest(unittest.TestCase): + + """Base class for logging tests.""" + + log_format = "%(name)s -> %(levelname)s: %(message)s" + expected_log_pat = r"^([\w.]+) -> (\w+): (\d+)$" + message_num = 0 + + def setUp(self): + """Setup the default logging stream to an internal StringIO instance, + so that we can examine log output as we want.""" + self._threading_key = threading_helper.threading_setup() + + logger_dict = logging.getLogger().manager.loggerDict + with logging._lock: + self.saved_handlers = logging._handlers.copy() + self.saved_handler_list = logging._handlerList[:] + self.saved_loggers = saved_loggers = logger_dict.copy() + self.saved_name_to_level = logging._nameToLevel.copy() + self.saved_level_to_name = logging._levelToName.copy() + self.logger_states = logger_states = {} + for name in saved_loggers: + logger_states[name] = getattr(saved_loggers[name], + 'disabled', None) + + # Set two unused loggers + self.logger1 = logging.getLogger("\xab\xd7\xbb") + self.logger2 = logging.getLogger("\u013f\u00d6\u0047") + + self.root_logger = logging.getLogger("") + self.original_logging_level = self.root_logger.getEffectiveLevel() + + self.stream = io.StringIO() + self.root_logger.setLevel(logging.DEBUG) + self.root_hdlr = logging.StreamHandler(self.stream) + self.root_formatter = logging.Formatter(self.log_format) + self.root_hdlr.setFormatter(self.root_formatter) + if self.logger1.hasHandlers(): + hlist = self.logger1.handlers + self.root_logger.handlers + raise AssertionError('Unexpected handlers: %s' % hlist) + if self.logger2.hasHandlers(): + hlist = self.logger2.handlers + self.root_logger.handlers + raise AssertionError('Unexpected handlers: %s' % hlist) + self.root_logger.addHandler(self.root_hdlr) + self.assertTrue(self.logger1.hasHandlers()) + self.assertTrue(self.logger2.hasHandlers()) + + def tearDown(self): + """Remove our logging stream, and restore the original logging + level.""" + self.stream.close() + self.root_logger.removeHandler(self.root_hdlr) + while self.root_logger.handlers: + h = self.root_logger.handlers[0] + self.root_logger.removeHandler(h) + h.close() + self.root_logger.setLevel(self.original_logging_level) + with logging._lock: + logging._levelToName.clear() + logging._levelToName.update(self.saved_level_to_name) + logging._nameToLevel.clear() + logging._nameToLevel.update(self.saved_name_to_level) + logging._handlers.clear() + logging._handlers.update(self.saved_handlers) + logging._handlerList[:] = self.saved_handler_list + manager = logging.getLogger().manager + manager.disable = 0 + loggerDict = manager.loggerDict + loggerDict.clear() + loggerDict.update(self.saved_loggers) + logger_states = self.logger_states + for name in self.logger_states: + if logger_states[name] is not None: + self.saved_loggers[name].disabled = logger_states[name] + + self.doCleanups() + threading_helper.threading_cleanup(*self._threading_key) + + def assert_log_lines(self, expected_values, stream=None, pat=None): + """Match the collected log lines against the regular expression + self.expected_log_pat, and compare the extracted group values to + the expected_values list of tuples.""" + stream = stream or self.stream + pat = re.compile(pat or self.expected_log_pat) + actual_lines = stream.getvalue().splitlines() + self.assertEqual(len(actual_lines), len(expected_values)) + for actual, expected in zip(actual_lines, expected_values): + match = pat.search(actual) + if not match: + self.fail("Log line does not match expected pattern:\n" + + actual) + self.assertEqual(tuple(match.groups()), expected) + s = stream.read() + if s: + self.fail("Remaining output at end of log stream:\n" + s) + + def next_message(self): + """Generate a message consisting solely of an auto-incrementing + integer.""" + self.message_num += 1 + return "%d" % self.message_num + + +class BuiltinLevelsTest(BaseTest): + """Test builtin levels and their inheritance.""" + + def test_flat(self): + # Logging levels in a flat logger namespace. + m = self.next_message + + ERR = logging.getLogger("ERR") + ERR.setLevel(logging.ERROR) + INF = logging.LoggerAdapter(logging.getLogger("INF"), {}) + INF.setLevel(logging.INFO) + DEB = logging.getLogger("DEB") + DEB.setLevel(logging.DEBUG) + + # These should log. + ERR.log(logging.CRITICAL, m()) + ERR.error(m()) + + INF.log(logging.CRITICAL, m()) + INF.error(m()) + INF.warning(m()) + INF.info(m()) + + DEB.log(logging.CRITICAL, m()) + DEB.error(m()) + DEB.warning(m()) + DEB.info(m()) + DEB.debug(m()) + + # These should not log. + ERR.warning(m()) + ERR.info(m()) + ERR.debug(m()) + + INF.debug(m()) + + self.assert_log_lines([ + ('ERR', 'CRITICAL', '1'), + ('ERR', 'ERROR', '2'), + ('INF', 'CRITICAL', '3'), + ('INF', 'ERROR', '4'), + ('INF', 'WARNING', '5'), + ('INF', 'INFO', '6'), + ('DEB', 'CRITICAL', '7'), + ('DEB', 'ERROR', '8'), + ('DEB', 'WARNING', '9'), + ('DEB', 'INFO', '10'), + ('DEB', 'DEBUG', '11'), + ]) + + def test_nested_explicit(self): + # Logging levels in a nested namespace, all explicitly set. + m = self.next_message + + INF = logging.getLogger("INF") + INF.setLevel(logging.INFO) + INF_ERR = logging.getLogger("INF.ERR") + INF_ERR.setLevel(logging.ERROR) + + # These should log. + INF_ERR.log(logging.CRITICAL, m()) + INF_ERR.error(m()) + + # These should not log. + INF_ERR.warning(m()) + INF_ERR.info(m()) + INF_ERR.debug(m()) + + self.assert_log_lines([ + ('INF.ERR', 'CRITICAL', '1'), + ('INF.ERR', 'ERROR', '2'), + ]) + + def test_nested_inherited(self): + # Logging levels in a nested namespace, inherited from parent loggers. + m = self.next_message + + INF = logging.getLogger("INF") + INF.setLevel(logging.INFO) + INF_ERR = logging.getLogger("INF.ERR") + INF_ERR.setLevel(logging.ERROR) + INF_UNDEF = logging.getLogger("INF.UNDEF") + INF_ERR_UNDEF = logging.getLogger("INF.ERR.UNDEF") + UNDEF = logging.getLogger("UNDEF") + + # These should log. + INF_UNDEF.log(logging.CRITICAL, m()) + INF_UNDEF.error(m()) + INF_UNDEF.warning(m()) + INF_UNDEF.info(m()) + INF_ERR_UNDEF.log(logging.CRITICAL, m()) + INF_ERR_UNDEF.error(m()) + + # These should not log. + INF_UNDEF.debug(m()) + INF_ERR_UNDEF.warning(m()) + INF_ERR_UNDEF.info(m()) + INF_ERR_UNDEF.debug(m()) + + self.assert_log_lines([ + ('INF.UNDEF', 'CRITICAL', '1'), + ('INF.UNDEF', 'ERROR', '2'), + ('INF.UNDEF', 'WARNING', '3'), + ('INF.UNDEF', 'INFO', '4'), + ('INF.ERR.UNDEF', 'CRITICAL', '5'), + ('INF.ERR.UNDEF', 'ERROR', '6'), + ]) + + def test_nested_with_virtual_parent(self): + # Logging levels when some parent does not exist yet. + m = self.next_message + + INF = logging.getLogger("INF") + GRANDCHILD = logging.getLogger("INF.BADPARENT.UNDEF") + CHILD = logging.getLogger("INF.BADPARENT") + INF.setLevel(logging.INFO) + + # These should log. + GRANDCHILD.log(logging.FATAL, m()) + GRANDCHILD.info(m()) + CHILD.log(logging.FATAL, m()) + CHILD.info(m()) + + # These should not log. + GRANDCHILD.debug(m()) + CHILD.debug(m()) + + self.assert_log_lines([ + ('INF.BADPARENT.UNDEF', 'CRITICAL', '1'), + ('INF.BADPARENT.UNDEF', 'INFO', '2'), + ('INF.BADPARENT', 'CRITICAL', '3'), + ('INF.BADPARENT', 'INFO', '4'), + ]) + + def test_regression_22386(self): + """See issue #22386 for more information.""" + self.assertEqual(logging.getLevelName('INFO'), logging.INFO) + self.assertEqual(logging.getLevelName(logging.INFO), 'INFO') + + def test_issue27935(self): + fatal = logging.getLevelName('FATAL') + self.assertEqual(fatal, logging.FATAL) + + def test_regression_29220(self): + """See issue #29220 for more information.""" + logging.addLevelName(logging.INFO, '') + self.addCleanup(logging.addLevelName, logging.INFO, 'INFO') + self.assertEqual(logging.getLevelName(logging.INFO), '') + self.assertEqual(logging.getLevelName(logging.NOTSET), 'NOTSET') + self.assertEqual(logging.getLevelName('NOTSET'), logging.NOTSET) + +class BasicFilterTest(BaseTest): + + """Test the bundled Filter class.""" + + def test_filter(self): + # Only messages satisfying the specified criteria pass through the + # filter. + filter_ = logging.Filter("spam.eggs") + handler = self.root_logger.handlers[0] + try: + handler.addFilter(filter_) + spam = logging.getLogger("spam") + spam_eggs = logging.getLogger("spam.eggs") + spam_eggs_fish = logging.getLogger("spam.eggs.fish") + spam_bakedbeans = logging.getLogger("spam.bakedbeans") + + spam.info(self.next_message()) + spam_eggs.info(self.next_message()) # Good. + spam_eggs_fish.info(self.next_message()) # Good. + spam_bakedbeans.info(self.next_message()) + + self.assert_log_lines([ + ('spam.eggs', 'INFO', '2'), + ('spam.eggs.fish', 'INFO', '3'), + ]) + finally: + handler.removeFilter(filter_) + + def test_callable_filter(self): + # Only messages satisfying the specified criteria pass through the + # filter. + + def filterfunc(record): + parts = record.name.split('.') + prefix = '.'.join(parts[:2]) + return prefix == 'spam.eggs' + + handler = self.root_logger.handlers[0] + try: + handler.addFilter(filterfunc) + spam = logging.getLogger("spam") + spam_eggs = logging.getLogger("spam.eggs") + spam_eggs_fish = logging.getLogger("spam.eggs.fish") + spam_bakedbeans = logging.getLogger("spam.bakedbeans") + + spam.info(self.next_message()) + spam_eggs.info(self.next_message()) # Good. + spam_eggs_fish.info(self.next_message()) # Good. + spam_bakedbeans.info(self.next_message()) + + self.assert_log_lines([ + ('spam.eggs', 'INFO', '2'), + ('spam.eggs.fish', 'INFO', '3'), + ]) + finally: + handler.removeFilter(filterfunc) + + def test_empty_filter(self): + f = logging.Filter() + r = logging.makeLogRecord({'name': 'spam.eggs'}) + self.assertTrue(f.filter(r)) + +# +# First, we define our levels. There can be as many as you want - the only +# limitations are that they should be integers, the lowest should be > 0 and +# larger values mean less information being logged. If you need specific +# level values which do not fit into these limitations, you can use a +# mapping dictionary to convert between your application levels and the +# logging system. +# +SILENT = 120 +TACITURN = 119 +TERSE = 118 +EFFUSIVE = 117 +SOCIABLE = 116 +VERBOSE = 115 +TALKATIVE = 114 +GARRULOUS = 113 +CHATTERBOX = 112 +BORING = 111 + +LEVEL_RANGE = range(BORING, SILENT + 1) + +# +# Next, we define names for our levels. You don't need to do this - in which +# case the system will use "Level n" to denote the text for the level. +# +my_logging_levels = { + SILENT : 'Silent', + TACITURN : 'Taciturn', + TERSE : 'Terse', + EFFUSIVE : 'Effusive', + SOCIABLE : 'Sociable', + VERBOSE : 'Verbose', + TALKATIVE : 'Talkative', + GARRULOUS : 'Garrulous', + CHATTERBOX : 'Chatterbox', + BORING : 'Boring', +} + +class GarrulousFilter(logging.Filter): + + """A filter which blocks garrulous messages.""" + + def filter(self, record): + return record.levelno != GARRULOUS + +class VerySpecificFilter(logging.Filter): + + """A filter which blocks sociable and taciturn messages.""" + + def filter(self, record): + return record.levelno not in [SOCIABLE, TACITURN] + + +class CustomLevelsAndFiltersTest(BaseTest): + + """Test various filtering possibilities with custom logging levels.""" + + # Skip the logger name group. + expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$" + + def setUp(self): + BaseTest.setUp(self) + for k, v in my_logging_levels.items(): + logging.addLevelName(k, v) + + def log_at_all_levels(self, logger): + for lvl in LEVEL_RANGE: + logger.log(lvl, self.next_message()) + + def test_handler_filter_replaces_record(self): + def replace_message(record: logging.LogRecord): + record = copy.copy(record) + record.msg = "new message!" + return record + + # Set up a logging hierarchy such that "child" and it's handler + # (and thus `replace_message()`) always get called before + # propagating up to "parent". + # Then we can confirm that `replace_message()` was able to + # replace the log record without having a side effect on + # other loggers or handlers. + parent = logging.getLogger("parent") + child = logging.getLogger("parent.child") + stream_1 = io.StringIO() + stream_2 = io.StringIO() + handler_1 = logging.StreamHandler(stream_1) + handler_2 = logging.StreamHandler(stream_2) + handler_2.addFilter(replace_message) + parent.addHandler(handler_1) + child.addHandler(handler_2) + + child.info("original message") + handler_1.flush() + handler_2.flush() + self.assertEqual(stream_1.getvalue(), "original message\n") + self.assertEqual(stream_2.getvalue(), "new message!\n") + + def test_logging_filter_replaces_record(self): + records = set() + + class RecordingFilter(logging.Filter): + def filter(self, record: logging.LogRecord): + records.add(id(record)) + return copy.copy(record) + + logger = logging.getLogger("logger") + logger.setLevel(logging.INFO) + logger.addFilter(RecordingFilter()) + logger.addFilter(RecordingFilter()) + + logger.info("msg") + + self.assertEqual(2, len(records)) + + def test_logger_filter(self): + # Filter at logger level. + self.root_logger.setLevel(VERBOSE) + # Levels >= 'Verbose' are good. + self.log_at_all_levels(self.root_logger) + self.assert_log_lines([ + ('Verbose', '5'), + ('Sociable', '6'), + ('Effusive', '7'), + ('Terse', '8'), + ('Taciturn', '9'), + ('Silent', '10'), + ]) + + def test_handler_filter(self): + # Filter at handler level. + self.root_logger.handlers[0].setLevel(SOCIABLE) + try: + # Levels >= 'Sociable' are good. + self.log_at_all_levels(self.root_logger) + self.assert_log_lines([ + ('Sociable', '6'), + ('Effusive', '7'), + ('Terse', '8'), + ('Taciturn', '9'), + ('Silent', '10'), + ]) + finally: + self.root_logger.handlers[0].setLevel(logging.NOTSET) + + def test_specific_filters(self): + # Set a specific filter object on the handler, and then add another + # filter object on the logger itself. + handler = self.root_logger.handlers[0] + specific_filter = None + garr = GarrulousFilter() + handler.addFilter(garr) + try: + self.log_at_all_levels(self.root_logger) + first_lines = [ + # Notice how 'Garrulous' is missing + ('Boring', '1'), + ('Chatterbox', '2'), + ('Talkative', '4'), + ('Verbose', '5'), + ('Sociable', '6'), + ('Effusive', '7'), + ('Terse', '8'), + ('Taciturn', '9'), + ('Silent', '10'), + ] + self.assert_log_lines(first_lines) + + specific_filter = VerySpecificFilter() + self.root_logger.addFilter(specific_filter) + self.log_at_all_levels(self.root_logger) + self.assert_log_lines(first_lines + [ + # Not only 'Garrulous' is still missing, but also 'Sociable' + # and 'Taciturn' + ('Boring', '11'), + ('Chatterbox', '12'), + ('Talkative', '14'), + ('Verbose', '15'), + ('Effusive', '17'), + ('Terse', '18'), + ('Silent', '20'), + ]) + finally: + if specific_filter: + self.root_logger.removeFilter(specific_filter) + handler.removeFilter(garr) + + +def make_temp_file(*args, **kwargs): + fd, fn = tempfile.mkstemp(*args, **kwargs) + os.close(fd) + return fn + + +class HandlerTest(BaseTest): + def test_name(self): + h = logging.Handler() + h.name = 'generic' + self.assertEqual(h.name, 'generic') + h.name = 'anothergeneric' + self.assertEqual(h.name, 'anothergeneric') + self.assertRaises(NotImplementedError, h.emit, None) + + def test_builtin_handlers(self): + # We can't actually *use* too many handlers in the tests, + # but we can try instantiating them with various options + if sys.platform in ('linux', 'android', 'darwin'): + for existing in (True, False): + fn = make_temp_file() + if not existing: + os.unlink(fn) + h = logging.handlers.WatchedFileHandler(fn, encoding='utf-8', delay=True) + if existing: + dev, ino = h.dev, h.ino + self.assertEqual(dev, -1) + self.assertEqual(ino, -1) + r = logging.makeLogRecord({'msg': 'Test'}) + h.handle(r) + # Now remove the file. + os.unlink(fn) + self.assertFalse(os.path.exists(fn)) + # The next call should recreate the file. + h.handle(r) + self.assertTrue(os.path.exists(fn)) + else: + self.assertEqual(h.dev, -1) + self.assertEqual(h.ino, -1) + h.close() + if existing: + os.unlink(fn) + if sys.platform == 'darwin': + sockname = '/var/run/syslog' + else: + sockname = '/dev/log' + try: + h = logging.handlers.SysLogHandler(sockname) + self.assertEqual(h.facility, h.LOG_USER) + self.assertTrue(h.unixsocket) + h.close() + except OSError: # syslogd might not be available + pass + for method in ('GET', 'POST', 'PUT'): + if method == 'PUT': + self.assertRaises(ValueError, logging.handlers.HTTPHandler, + 'localhost', '/log', method) + else: + h = logging.handlers.HTTPHandler('localhost', '/log', method) + h.close() + h = logging.handlers.BufferingHandler(0) + r = logging.makeLogRecord({}) + self.assertTrue(h.shouldFlush(r)) + h.close() + h = logging.handlers.BufferingHandler(1) + self.assertFalse(h.shouldFlush(r)) + h.close() + + def test_pathlike_objects(self): + """ + Test that path-like objects are accepted as filename arguments to handlers. + + See Issue #27493. + """ + fn = make_temp_file() + os.unlink(fn) + pfn = os_helper.FakePath(fn) + cases = ( + (logging.FileHandler, (pfn, 'w')), + (logging.handlers.RotatingFileHandler, (pfn, 'a')), + (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), + ) + if sys.platform in ('linux', 'android', 'darwin'): + cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) + for cls, args in cases: + h = cls(*args, encoding="utf-8") + self.assertTrue(os.path.exists(fn)) + h.close() + os.unlink(fn) + + @unittest.skipIf(os.name == 'nt', 'WatchedFileHandler not appropriate for Windows.') + @unittest.skipIf( + support.is_emscripten, "Emscripten cannot fstat unlinked files." + ) + @threading_helper.requires_working_threading() + @support.requires_resource('walltime') + def test_race(self): + # Issue #14632 refers. + def remove_loop(fname, tries): + for _ in range(tries): + try: + os.unlink(fname) + self.deletion_time = time.time() + except OSError: + pass + time.sleep(0.004 * random.randint(0, 4)) + + del_count = 500 + log_count = 500 + + self.handle_time = None + self.deletion_time = None + + for delay in (False, True): + fn = make_temp_file('.log', 'test_logging-3-') + remover = threading.Thread(target=remove_loop, args=(fn, del_count)) + remover.daemon = True + remover.start() + h = logging.handlers.WatchedFileHandler(fn, encoding='utf-8', delay=delay) + f = logging.Formatter('%(asctime)s: %(levelname)s: %(message)s') + h.setFormatter(f) + try: + for _ in range(log_count): + time.sleep(0.005) + r = logging.makeLogRecord({'msg': 'testing' }) + try: + self.handle_time = time.time() + h.handle(r) + except Exception: + print('Deleted at %s, ' + 'opened at %s' % (self.deletion_time, + self.handle_time)) + raise + finally: + remover.join() + h.close() + if os.path.exists(fn): + os.unlink(fn) + + # The implementation relies on os.register_at_fork existing, but we test + # based on os.fork existing because that is what users and this test use. + # This helps ensure that when fork exists (the important concept) that the + # register_at_fork mechanism is also present and used. + @support.requires_fork() + @threading_helper.requires_working_threading() + @skip_if_asan_fork + @skip_if_tsan_fork + @unittest.skip("TODO: RUSTPYTHON; Flaky") + def test_post_fork_child_no_deadlock(self): + """Ensure child logging locks are not held; bpo-6721 & bpo-36533.""" + class _OurHandler(logging.Handler): + def __init__(self): + super().__init__() + self.sub_handler = logging.StreamHandler( + stream=open('/dev/null', 'wt', encoding='utf-8')) + + def emit(self, record): + with self.sub_handler.lock: + self.sub_handler.emit(record) + + self.assertEqual(len(logging._handlers), 0) + refed_h = _OurHandler() + self.addCleanup(refed_h.sub_handler.stream.close) + refed_h.name = 'because we need at least one for this test' + self.assertGreater(len(logging._handlers), 0) + self.assertGreater(len(logging._at_fork_reinit_lock_weakset), 1) + test_logger = logging.getLogger('test_post_fork_child_no_deadlock') + test_logger.addHandler(refed_h) + test_logger.setLevel(logging.DEBUG) + + locks_held__ready_to_fork = threading.Event() + fork_happened__release_locks_and_end_thread = threading.Event() + + def lock_holder_thread_fn(): + with logging._lock, refed_h.lock: + # Tell the main thread to do the fork. + locks_held__ready_to_fork.set() + + # If the deadlock bug exists, the fork will happen + # without dealing with the locks we hold, deadlocking + # the child. + + # Wait for a successful fork or an unreasonable amount of + # time before releasing our locks. To avoid a timing based + # test we'd need communication from os.fork() as to when it + # has actually happened. Given this is a regression test + # for a fixed issue, potentially less reliably detecting + # regression via timing is acceptable for simplicity. + # The test will always take at least this long. :( + fork_happened__release_locks_and_end_thread.wait(0.5) + + lock_holder_thread = threading.Thread( + target=lock_holder_thread_fn, + name='test_post_fork_child_no_deadlock lock holder') + lock_holder_thread.start() + + locks_held__ready_to_fork.wait() + pid = os.fork() + if pid == 0: + # Child process + try: + test_logger.info(r'Child process did not deadlock. \o/') + finally: + os._exit(0) + else: + # Parent process + test_logger.info(r'Parent process returned from fork. \o/') + fork_happened__release_locks_and_end_thread.set() + lock_holder_thread.join() + + support.wait_process(pid, exitcode=0) + + +class BadStream(object): + def write(self, data): + raise RuntimeError('deliberate mistake') + +class TestStreamHandler(logging.StreamHandler): + def handleError(self, record): + self.error_record = record + +class StreamWithIntName(object): + level = logging.NOTSET + name = 2 + +class StreamHandlerTest(BaseTest): + def test_error_handling(self): + h = TestStreamHandler(BadStream()) + r = logging.makeLogRecord({}) + old_raise = logging.raiseExceptions + + try: + h.handle(r) + self.assertIs(h.error_record, r) + + h = logging.StreamHandler(BadStream()) + with support.captured_stderr() as stderr: + h.handle(r) + msg = '\nRuntimeError: deliberate mistake\n' + self.assertIn(msg, stderr.getvalue()) + + logging.raiseExceptions = False + with support.captured_stderr() as stderr: + h.handle(r) + self.assertEqual('', stderr.getvalue()) + finally: + logging.raiseExceptions = old_raise + + def test_stream_setting(self): + """ + Test setting the handler's stream + """ + h = logging.StreamHandler() + stream = io.StringIO() + old = h.setStream(stream) + self.assertIs(old, sys.stderr) + actual = h.setStream(old) + self.assertIs(actual, stream) + # test that setting to existing value returns None + actual = h.setStream(old) + self.assertIsNone(actual) + + def test_can_represent_stream_with_int_name(self): + h = logging.StreamHandler(StreamWithIntName()) + self.assertEqual(repr(h), '<StreamHandler 2 (NOTSET)>') + +# -- The following section could be moved into a server_helper.py module +# -- if it proves to be of wider utility than just test_logging + +class TestSMTPServer(smtpd.SMTPServer): + """ + This class implements a test SMTP server. + + :param addr: A (host, port) tuple which the server listens on. + You can specify a port value of zero: the server's + *port* attribute will hold the actual port number + used, which can be used in client connections. + :param handler: A callable which will be called to process + incoming messages. The handler will be passed + the client address tuple, who the message is from, + a list of recipients and the message data. + :param poll_interval: The interval, in seconds, used in the underlying + :func:`select` or :func:`poll` call by + :func:`asyncore.loop`. + :param sockmap: A dictionary which will be used to hold + :class:`asyncore.dispatcher` instances used by + :func:`asyncore.loop`. This avoids changing the + :mod:`asyncore` module's global state. + """ + + def __init__(self, addr, handler, poll_interval, sockmap): + smtpd.SMTPServer.__init__(self, addr, None, map=sockmap, + decode_data=True) + self.port = self.socket.getsockname()[1] + self._handler = handler + self._thread = None + self._quit = False + self.poll_interval = poll_interval + + def process_message(self, peer, mailfrom, rcpttos, data): + """ + Delegates to the handler passed in to the server's constructor. + + Typically, this will be a test case method. + :param peer: The client (host, port) tuple. + :param mailfrom: The address of the sender. + :param rcpttos: The addresses of the recipients. + :param data: The message. + """ + self._handler(peer, mailfrom, rcpttos, data) + + def start(self): + """ + Start the server running on a separate daemon thread. + """ + self._thread = t = threading.Thread(target=self.serve_forever, + args=(self.poll_interval,)) + t.daemon = True + t.start() + + def serve_forever(self, poll_interval): + """ + Run the :mod:`asyncore` loop until normal termination + conditions arise. + :param poll_interval: The interval, in seconds, used in the underlying + :func:`select` or :func:`poll` call by + :func:`asyncore.loop`. + """ + while not self._quit: + asyncore.loop(poll_interval, map=self._map, count=1) + + def stop(self): + """ + Stop the thread by closing the server instance. + Wait for the server thread to terminate. + """ + self._quit = True + threading_helper.join_thread(self._thread) + self._thread = None + self.close() + asyncore.close_all(map=self._map, ignore_all=True) + + +class ControlMixin(object): + """ + This mixin is used to start a server on a separate thread, and + shut it down programmatically. Request handling is simplified - instead + of needing to derive a suitable RequestHandler subclass, you just + provide a callable which will be passed each received request to be + processed. + + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. This handler is called on the + server thread, effectively meaning that requests are + processed serially. While not quite web scale ;-), + this should be fine for testing applications. + :param poll_interval: The polling interval in seconds. + """ + def __init__(self, handler, poll_interval): + self._thread = None + self.poll_interval = poll_interval + self._handler = handler + self.ready = threading.Event() + + def start(self): + """ + Create a daemon thread to run the server, and start it. + """ + self._thread = t = threading.Thread(target=self.serve_forever, + args=(self.poll_interval,)) + t.daemon = True + t.start() + + def serve_forever(self, poll_interval): + """ + Run the server. Set the ready flag before entering the + service loop. + """ + self.ready.set() + super(ControlMixin, self).serve_forever(poll_interval) + + def stop(self): + """ + Tell the server thread to stop, and wait for it to do so. + """ + self.shutdown() + if self._thread is not None: + threading_helper.join_thread(self._thread) + self._thread = None + self.server_close() + self.ready.clear() + +class TestHTTPServer(ControlMixin, HTTPServer): + """ + An HTTP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. + :param poll_interval: The polling interval in seconds. + :param log: Pass ``True`` to enable log messages. + """ + def __init__(self, addr, handler, poll_interval=0.5, + log=False, sslctx=None): + class DelegatingHTTPRequestHandler(BaseHTTPRequestHandler): + def __getattr__(self, name, default=None): + if name.startswith('do_'): + return self.process_request + raise AttributeError(name) + + def process_request(self): + self.server._handler(self) + + def log_message(self, format, *args): + if log: + super(DelegatingHTTPRequestHandler, + self).log_message(format, *args) + HTTPServer.__init__(self, addr, DelegatingHTTPRequestHandler) + ControlMixin.__init__(self, handler, poll_interval) + self.sslctx = sslctx + + def get_request(self): + try: + sock, addr = self.socket.accept() + if self.sslctx: + sock = self.sslctx.wrap_socket(sock, server_side=True) + except OSError as e: + # socket errors are silenced by the caller, print them here + sys.stderr.write("Got an error:\n%s\n" % e) + raise + return sock, addr + +class TestTCPServer(ControlMixin, ThreadingTCPServer): + """ + A TCP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a single + parameter - the request - in order to process the request. + :param poll_interval: The polling interval in seconds. + :bind_and_activate: If True (the default), binds the server and starts it + listening. If False, you need to call + :meth:`server_bind` and :meth:`server_activate` at + some later time before calling :meth:`start`, so that + the server will set up the socket and listen on it. + """ + + allow_reuse_address = True + + def __init__(self, addr, handler, poll_interval=0.5, + bind_and_activate=True): + class DelegatingTCPRequestHandler(StreamRequestHandler): + + def handle(self): + self.server._handler(self) + ThreadingTCPServer.__init__(self, addr, DelegatingTCPRequestHandler, + bind_and_activate) + ControlMixin.__init__(self, handler, poll_interval) + + def server_bind(self): + super(TestTCPServer, self).server_bind() + self.port = self.socket.getsockname()[1] + +class TestUDPServer(ControlMixin, ThreadingUDPServer): + """ + A UDP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. + :param poll_interval: The polling interval for shutdown requests, + in seconds. + :bind_and_activate: If True (the default), binds the server and + starts it listening. If False, you need to + call :meth:`server_bind` and + :meth:`server_activate` at some later time + before calling :meth:`start`, so that the server will + set up the socket and listen on it. + """ + def __init__(self, addr, handler, poll_interval=0.5, + bind_and_activate=True): + class DelegatingUDPRequestHandler(DatagramRequestHandler): + + def handle(self): + self.server._handler(self) + + def finish(self): + data = self.wfile.getvalue() + if data: + try: + super(DelegatingUDPRequestHandler, self).finish() + except OSError: + if not self.server._closed: + raise + + ThreadingUDPServer.__init__(self, addr, + DelegatingUDPRequestHandler, + bind_and_activate) + ControlMixin.__init__(self, handler, poll_interval) + self._closed = False + + def server_bind(self): + super(TestUDPServer, self).server_bind() + self.port = self.socket.getsockname()[1] + + def server_close(self): + super(TestUDPServer, self).server_close() + self._closed = True + +if hasattr(socket, "AF_UNIX"): + class TestUnixStreamServer(TestTCPServer): + address_family = socket.AF_UNIX + + class TestUnixDatagramServer(TestUDPServer): + address_family = socket.AF_UNIX + +# - end of server_helper section + +@support.requires_working_socket() +@threading_helper.requires_working_threading() +class SMTPHandlerTest(BaseTest): + # bpo-14314, bpo-19665, bpo-34092: don't wait forever + TIMEOUT = support.LONG_TIMEOUT + + def test_basic(self): + sockmap = {} + server = TestSMTPServer((socket_helper.HOST, 0), self.process_message, 0.001, + sockmap) + server.start() + addr = (socket_helper.HOST, server.port) + h = logging.handlers.SMTPHandler(addr, 'me', 'you', 'Log', + timeout=self.TIMEOUT) + self.assertEqual(h.toaddrs, ['you']) + self.messages = [] + r = logging.makeLogRecord({'msg': 'Hello \u2713'}) + self.handled = threading.Event() + h.handle(r) + self.handled.wait(self.TIMEOUT) + server.stop() + self.assertTrue(self.handled.is_set()) + self.assertEqual(len(self.messages), 1) + peer, mailfrom, rcpttos, data = self.messages[0] + self.assertEqual(mailfrom, 'me') + self.assertEqual(rcpttos, ['you']) + self.assertIn('\nSubject: Log\n', data) + self.assertTrue(data.endswith('\n\nHello \u2713')) + h.close() + + def process_message(self, *args): + self.messages.append(args) + self.handled.set() + +class MemoryHandlerTest(BaseTest): + + """Tests for the MemoryHandler.""" + + # Do not bother with a logger name group. + expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$" + + def setUp(self): + BaseTest.setUp(self) + self.mem_hdlr = logging.handlers.MemoryHandler(10, logging.WARNING, + self.root_hdlr) + self.mem_logger = logging.getLogger('mem') + self.mem_logger.propagate = 0 + self.mem_logger.addHandler(self.mem_hdlr) + + def tearDown(self): + self.mem_hdlr.close() + BaseTest.tearDown(self) + + def test_flush(self): + # The memory handler flushes to its target handler based on specific + # criteria (message count and message level). + self.mem_logger.debug(self.next_message()) + self.assert_log_lines([]) + self.mem_logger.info(self.next_message()) + self.assert_log_lines([]) + # This will flush because the level is >= logging.WARNING + self.mem_logger.warning(self.next_message()) + lines = [ + ('DEBUG', '1'), + ('INFO', '2'), + ('WARNING', '3'), + ] + self.assert_log_lines(lines) + for n in (4, 14): + for i in range(9): + self.mem_logger.debug(self.next_message()) + self.assert_log_lines(lines) + # This will flush because it's the 10th message since the last + # flush. + self.mem_logger.debug(self.next_message()) + lines = lines + [('DEBUG', str(i)) for i in range(n, n + 10)] + self.assert_log_lines(lines) + + self.mem_logger.debug(self.next_message()) + self.assert_log_lines(lines) + + def test_flush_on_close(self): + """ + Test that the flush-on-close configuration works as expected. + """ + self.mem_logger.debug(self.next_message()) + self.assert_log_lines([]) + self.mem_logger.info(self.next_message()) + self.assert_log_lines([]) + self.mem_logger.removeHandler(self.mem_hdlr) + # Default behaviour is to flush on close. Check that it happens. + self.mem_hdlr.close() + lines = [ + ('DEBUG', '1'), + ('INFO', '2'), + ] + self.assert_log_lines(lines) + # Now configure for flushing not to be done on close. + self.mem_hdlr = logging.handlers.MemoryHandler(10, logging.WARNING, + self.root_hdlr, + False) + self.mem_logger.addHandler(self.mem_hdlr) + self.mem_logger.debug(self.next_message()) + self.assert_log_lines(lines) # no change + self.mem_logger.info(self.next_message()) + self.assert_log_lines(lines) # no change + self.mem_logger.removeHandler(self.mem_hdlr) + self.mem_hdlr.close() + # assert that no new lines have been added + self.assert_log_lines(lines) # no change + + def test_shutdown_flush_on_close(self): + """ + Test that the flush-on-close configuration is respected by the + shutdown method. + """ + self.mem_logger.debug(self.next_message()) + self.assert_log_lines([]) + self.mem_logger.info(self.next_message()) + self.assert_log_lines([]) + # Default behaviour is to flush on close. Check that it happens. + logging.shutdown(handlerList=[logging.weakref.ref(self.mem_hdlr)]) + lines = [ + ('DEBUG', '1'), + ('INFO', '2'), + ] + self.assert_log_lines(lines) + # Now configure for flushing not to be done on close. + self.mem_hdlr = logging.handlers.MemoryHandler(10, logging.WARNING, + self.root_hdlr, + False) + self.mem_logger.addHandler(self.mem_hdlr) + self.mem_logger.debug(self.next_message()) + self.assert_log_lines(lines) # no change + self.mem_logger.info(self.next_message()) + self.assert_log_lines(lines) # no change + # assert that no new lines have been added after shutdown + logging.shutdown(handlerList=[logging.weakref.ref(self.mem_hdlr)]) + self.assert_log_lines(lines) # no change + + @threading_helper.requires_working_threading() + def test_race_between_set_target_and_flush(self): + class MockRaceConditionHandler: + def __init__(self, mem_hdlr): + self.mem_hdlr = mem_hdlr + self.threads = [] + + def removeTarget(self): + self.mem_hdlr.setTarget(None) + + def handle(self, msg): + thread = threading.Thread(target=self.removeTarget) + self.threads.append(thread) + thread.start() + + target = MockRaceConditionHandler(self.mem_hdlr) + try: + self.mem_hdlr.setTarget(target) + + for _ in range(10): + time.sleep(0.005) + self.mem_logger.info("not flushed") + self.mem_logger.warning("flushed") + finally: + for thread in target.threads: + threading_helper.join_thread(thread) + + +class ExceptionFormatter(logging.Formatter): + """A special exception formatter.""" + def formatException(self, ei): + return "Got a [%s]" % ei[0].__name__ + +def closeFileHandler(h, fn): + h.close() + os.remove(fn) + +class ConfigFileTest(BaseTest): + + """Reading logging config from a .ini-style config file.""" + + check_no_resource_warning = warnings_helper.check_no_resource_warning + expected_log_pat = r"^(\w+) \+\+ (\w+)$" + + # config0 is a standard configuration. + config0 = """ + [loggers] + keys=root + + [handlers] + keys=hand1 + + [formatters] + keys=form1 + + [logger_root] + level=WARNING + handlers=hand1 + + [handler_hand1] + class=StreamHandler + level=NOTSET + formatter=form1 + args=(sys.stdout,) + + [formatter_form1] + format=%(levelname)s ++ %(message)s + datefmt= + """ + + # config1 adds a little to the standard configuration. + config1 = """ + [loggers] + keys=root,parser + + [handlers] + keys=hand1 + + [formatters] + keys=form1 + + [logger_root] + level=WARNING + handlers= + + [logger_parser] + level=DEBUG + handlers=hand1 + propagate=1 + qualname=compiler.parser + + [handler_hand1] + class=StreamHandler + level=NOTSET + formatter=form1 + args=(sys.stdout,) + + [formatter_form1] + format=%(levelname)s ++ %(message)s + datefmt= + """ + + # config1a moves the handler to the root. + config1a = """ + [loggers] + keys=root,parser + + [handlers] + keys=hand1 + + [formatters] + keys=form1 + + [logger_root] + level=WARNING + handlers=hand1 + + [logger_parser] + level=DEBUG + handlers= + propagate=1 + qualname=compiler.parser + + [handler_hand1] + class=StreamHandler + level=NOTSET + formatter=form1 + args=(sys.stdout,) + + [formatter_form1] + format=%(levelname)s ++ %(message)s + datefmt= + """ + + # config2 has a subtle configuration error that should be reported + config2 = config1.replace("sys.stdout", "sys.stbout") + + # config3 has a less subtle configuration error + config3 = config1.replace("formatter=form1", "formatter=misspelled_name") + + # config4 specifies a custom formatter class to be loaded + config4 = """ + [loggers] + keys=root + + [handlers] + keys=hand1 + + [formatters] + keys=form1 + + [logger_root] + level=NOTSET + handlers=hand1 + + [handler_hand1] + class=StreamHandler + level=NOTSET + formatter=form1 + args=(sys.stdout,) + + [formatter_form1] + class=""" + __name__ + """.ExceptionFormatter + format=%(levelname)s:%(name)s:%(message)s + datefmt= + """ + + # config5 specifies a custom handler class to be loaded + config5 = config1.replace('class=StreamHandler', 'class=logging.StreamHandler') + + # config6 uses ', ' delimiters in the handlers and formatters sections + config6 = """ + [loggers] + keys=root,parser + + [handlers] + keys=hand1, hand2 + + [formatters] + keys=form1, form2 + + [logger_root] + level=WARNING + handlers= + + [logger_parser] + level=DEBUG + handlers=hand1 + propagate=1 + qualname=compiler.parser + + [handler_hand1] + class=StreamHandler + level=NOTSET + formatter=form1 + args=(sys.stdout,) + + [handler_hand2] + class=StreamHandler + level=NOTSET + formatter=form1 + args=(sys.stderr,) + + [formatter_form1] + format=%(levelname)s ++ %(message)s + datefmt= + + [formatter_form2] + format=%(message)s + datefmt= + """ + + # config7 adds a compiler logger, and uses kwargs instead of args. + config7 = """ + [loggers] + keys=root,parser,compiler + + [handlers] + keys=hand1 + + [formatters] + keys=form1 + + [logger_root] + level=WARNING + handlers=hand1 + + [logger_compiler] + level=DEBUG + handlers= + propagate=1 + qualname=compiler + + [logger_parser] + level=DEBUG + handlers= + propagate=1 + qualname=compiler.parser + + [handler_hand1] + class=StreamHandler + level=NOTSET + formatter=form1 + kwargs={'stream': sys.stdout,} + + [formatter_form1] + format=%(levelname)s ++ %(message)s + datefmt= + """ + + # config 8, check for resource warning + config8 = r""" + [loggers] + keys=root + + [handlers] + keys=file + + [formatters] + keys= + + [logger_root] + level=DEBUG + handlers=file + + [handler_file] + class=FileHandler + level=DEBUG + args=("{tempfile}",) + kwargs={{"encoding": "utf-8"}} + """ + + + config9 = """ + [loggers] + keys=root + + [handlers] + keys=hand1 + + [formatters] + keys=form1 + + [logger_root] + level=WARNING + handlers=hand1 + + [handler_hand1] + class=StreamHandler + level=NOTSET + formatter=form1 + args=(sys.stdout,) + + [formatter_form1] + format=%(message)s ++ %(customfield)s + defaults={"customfield": "defaultvalue"} + """ + + disable_test = """ + [loggers] + keys=root + + [handlers] + keys=screen + + [formatters] + keys= + + [logger_root] + level=DEBUG + handlers=screen + + [handler_screen] + level=DEBUG + class=StreamHandler + args=(sys.stdout,) + formatter= + """ + + def apply_config(self, conf, **kwargs): + file = io.StringIO(textwrap.dedent(conf)) + logging.config.fileConfig(file, encoding="utf-8", **kwargs) + + def test_config0_ok(self): + # A simple config file which overrides the default settings. + with support.captured_stdout() as output: + self.apply_config(self.config0) + logger = logging.getLogger() + # Won't output anything + logger.info(self.next_message()) + # Outputs a message + logger.error(self.next_message()) + self.assert_log_lines([ + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + def test_config0_using_cp_ok(self): + # A simple config file which overrides the default settings. + with support.captured_stdout() as output: + file = io.StringIO(textwrap.dedent(self.config0)) + cp = configparser.ConfigParser() + cp.read_file(file) + logging.config.fileConfig(cp) + logger = logging.getLogger() + # Won't output anything + logger.info(self.next_message()) + # Outputs a message + logger.error(self.next_message()) + self.assert_log_lines([ + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + def test_config1_ok(self, config=config1): + # A config file defining a sub-parser as well. + with support.captured_stdout() as output: + self.apply_config(config) + logger = logging.getLogger("compiler.parser") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + def test_config2_failure(self): + # A simple config file which overrides the default settings. + self.assertRaises(Exception, self.apply_config, self.config2) + + def test_config3_failure(self): + # A simple config file which overrides the default settings. + self.assertRaises(Exception, self.apply_config, self.config3) + + def test_config4_ok(self): + # A config file specifying a custom formatter class. + with support.captured_stdout() as output: + self.apply_config(self.config4) + logger = logging.getLogger() + try: + raise RuntimeError() + except RuntimeError: + logging.exception("just testing") + sys.stdout.seek(0) + self.assertEqual(output.getvalue(), + "ERROR:root:just testing\nGot a [RuntimeError]\n") + # Original logger output is empty + self.assert_log_lines([]) + + def test_config5_ok(self): + self.test_config1_ok(config=self.config5) + + def test_config6_ok(self): + self.test_config1_ok(config=self.config6) + + def test_config7_ok(self): + with support.captured_stdout() as output: + self.apply_config(self.config1a) + logger = logging.getLogger("compiler.parser") + # See issue #11424. compiler-hyphenated sorts + # between compiler and compiler.xyz and this + # was preventing compiler.xyz from being included + # in the child loggers of compiler because of an + # overzealous loop termination condition. + hyphenated = logging.getLogger('compiler-hyphenated') + # All will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + hyphenated.critical(self.next_message()) + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ('CRITICAL', '3'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + with support.captured_stdout() as output: + self.apply_config(self.config7) + logger = logging.getLogger("compiler.parser") + self.assertFalse(logger.disabled) + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + logger = logging.getLogger("compiler.lexer") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + # Will not appear + hyphenated.critical(self.next_message()) + self.assert_log_lines([ + ('INFO', '4'), + ('ERROR', '5'), + ('INFO', '6'), + ('ERROR', '7'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + def test_config8_ok(self): + + with self.check_no_resource_warning(): + fn = make_temp_file(".log", "test_logging-X-") + + # Replace single backslash with double backslash in windows + # to avoid unicode error during string formatting + if os.name == "nt": + fn = fn.replace("\\", "\\\\") + + config8 = self.config8.format(tempfile=fn) + + self.apply_config(config8) + self.apply_config(config8) + + handler = logging.root.handlers[0] + self.addCleanup(closeFileHandler, handler, fn) + + def test_config9_ok(self): + self.apply_config(self.config9) + formatter = logging.root.handlers[0].formatter + result = formatter.format(logging.makeLogRecord({'msg': 'test'})) + self.assertEqual(result, 'test ++ defaultvalue') + result = formatter.format(logging.makeLogRecord( + {'msg': 'test', 'customfield': "customvalue"})) + self.assertEqual(result, 'test ++ customvalue') + + + def test_logger_disabling(self): + self.apply_config(self.disable_test) + logger = logging.getLogger('some_pristine_logger') + self.assertFalse(logger.disabled) + self.apply_config(self.disable_test) + self.assertTrue(logger.disabled) + self.apply_config(self.disable_test, disable_existing_loggers=False) + self.assertFalse(logger.disabled) + + def test_config_set_handler_names(self): + test_config = """ + [loggers] + keys=root + + [handlers] + keys=hand1 + + [formatters] + keys=form1 + + [logger_root] + handlers=hand1 + + [handler_hand1] + class=StreamHandler + formatter=form1 + + [formatter_form1] + format=%(levelname)s ++ %(message)s + """ + self.apply_config(test_config) + self.assertEqual(logging.getLogger().handlers[0].name, 'hand1') + + def test_exception_if_confg_file_is_invalid(self): + test_config = """ + [loggers] + keys=root + + [handlers] + keys=hand1 + + [formatters] + keys=form1 + + [logger_root] + handlers=hand1 + + [handler_hand1] + class=StreamHandler + formatter=form1 + + [formatter_form1] + format=%(levelname)s ++ %(message)s + + prince + """ + + file = io.StringIO(textwrap.dedent(test_config)) + self.assertRaises(RuntimeError, logging.config.fileConfig, file) + + def test_exception_if_confg_file_is_empty(self): + fd, fn = tempfile.mkstemp(prefix='test_empty_', suffix='.ini') + os.close(fd) + self.assertRaises(RuntimeError, logging.config.fileConfig, fn) + os.remove(fn) + + def test_exception_if_config_file_does_not_exist(self): + self.assertRaises(FileNotFoundError, logging.config.fileConfig, 'filenotfound') + + def test_defaults_do_no_interpolation(self): + """bpo-33802 defaults should not get interpolated""" + ini = textwrap.dedent(""" + [formatters] + keys=default + + [formatter_default] + + [handlers] + keys=console + + [handler_console] + class=logging.StreamHandler + args=tuple() + + [loggers] + keys=root + + [logger_root] + formatter=default + handlers=console + """).strip() + fd, fn = tempfile.mkstemp(prefix='test_logging_', suffix='.ini') + try: + os.write(fd, ini.encode('ascii')) + os.close(fd) + logging.config.fileConfig( + fn, + encoding="utf-8", + defaults=dict( + version=1, + disable_existing_loggers=False, + formatters={ + "generic": { + "format": "%(asctime)s [%(process)d] [%(levelname)s] %(message)s", + "datefmt": "[%Y-%m-%d %H:%M:%S %z]", + "class": "logging.Formatter" + }, + }, + ) + ) + finally: + os.unlink(fn) + + +@support.requires_working_socket() +@threading_helper.requires_working_threading() +class SocketHandlerTest(BaseTest): + + """Test for SocketHandler objects.""" + + server_class = TestTCPServer + address = ('localhost', 0) + + def setUp(self): + """Set up a TCP server to receive log messages, and a SocketHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + # Issue #29177: deal with errors that happen during setup + self.server = self.sock_hdlr = self.server_exception = None + try: + self.server = server = self.server_class(self.address, + self.handle_socket, 0.01) + server.start() + # Uncomment next line to test error recovery in setUp() + # raise OSError('dummy error raised') + except OSError as e: + self.server_exception = e + return + server.ready.wait() + hcls = logging.handlers.SocketHandler + if isinstance(server.server_address, tuple): + self.sock_hdlr = hcls('localhost', server.port) + else: + self.sock_hdlr = hcls(server.server_address, None) + self.log_output = '' + self.root_logger.removeHandler(self.root_logger.handlers[0]) + self.root_logger.addHandler(self.sock_hdlr) + self.handled = threading.Semaphore(0) + + def tearDown(self): + """Shutdown the TCP server.""" + try: + if self.sock_hdlr: + self.root_logger.removeHandler(self.sock_hdlr) + self.sock_hdlr.close() + if self.server: + self.server.stop() + finally: + BaseTest.tearDown(self) + + def handle_socket(self, request): + conn = request.connection + while True: + chunk = conn.recv(4) + if len(chunk) < 4: + break + slen = struct.unpack(">L", chunk)[0] + chunk = conn.recv(slen) + while len(chunk) < slen: + chunk = chunk + conn.recv(slen - len(chunk)) + obj = pickle.loads(chunk) + record = logging.makeLogRecord(obj) + self.log_output += record.msg + '\n' + self.handled.release() + + def test_output(self): + # The log message sent to the SocketHandler is properly received. + if self.server_exception: + self.skipTest(self.server_exception) + logger = logging.getLogger("tcp") + logger.error("spam") + self.handled.acquire() + logger.debug("eggs") + self.handled.acquire() + self.assertEqual(self.log_output, "spam\neggs\n") + + def test_noserver(self): + if self.server_exception: + self.skipTest(self.server_exception) + # Avoid timing-related failures due to SocketHandler's own hard-wired + # one-second timeout on socket.create_connection() (issue #16264). + self.sock_hdlr.retryStart = 2.5 + # Kill the server + self.server.stop() + # The logging call should try to connect, which should fail + try: + raise RuntimeError('Deliberate mistake') + except RuntimeError: + self.root_logger.exception('Never sent') + self.root_logger.error('Never sent, either') + now = time.time() + self.assertGreater(self.sock_hdlr.retryTime, now) + time.sleep(self.sock_hdlr.retryTime - now + 0.001) + self.root_logger.error('Nor this') + + +@unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") +class UnixSocketHandlerTest(SocketHandlerTest): + + """Test for SocketHandler with unix sockets.""" + + if hasattr(socket, "AF_UNIX"): + server_class = TestUnixStreamServer + + def setUp(self): + # override the definition in the base class + self.address = socket_helper.create_unix_domain_name() + self.addCleanup(os_helper.unlink, self.address) + SocketHandlerTest.setUp(self) + +@support.requires_working_socket() +@threading_helper.requires_working_threading() +class DatagramHandlerTest(BaseTest): + + """Test for DatagramHandler.""" + + server_class = TestUDPServer + address = ('localhost', 0) + + def setUp(self): + """Set up a UDP server to receive log messages, and a DatagramHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + # Issue #29177: deal with errors that happen during setup + self.server = self.sock_hdlr = self.server_exception = None + try: + self.server = server = self.server_class(self.address, + self.handle_datagram, 0.01) + server.start() + # Uncomment next line to test error recovery in setUp() + # raise OSError('dummy error raised') + except OSError as e: + self.server_exception = e + return + server.ready.wait() + hcls = logging.handlers.DatagramHandler + if isinstance(server.server_address, tuple): + self.sock_hdlr = hcls('localhost', server.port) + else: + self.sock_hdlr = hcls(server.server_address, None) + self.log_output = '' + self.root_logger.removeHandler(self.root_logger.handlers[0]) + self.root_logger.addHandler(self.sock_hdlr) + self.handled = threading.Event() + + def tearDown(self): + """Shutdown the UDP server.""" + try: + if self.server: + self.server.stop() + if self.sock_hdlr: + self.root_logger.removeHandler(self.sock_hdlr) + self.sock_hdlr.close() + finally: + BaseTest.tearDown(self) + + def handle_datagram(self, request): + slen = struct.pack('>L', 0) # length of prefix + packet = request.packet[len(slen):] + obj = pickle.loads(packet) + record = logging.makeLogRecord(obj) + self.log_output += record.msg + '\n' + self.handled.set() + + def test_output(self): + # The log message sent to the DatagramHandler is properly received. + if self.server_exception: + self.skipTest(self.server_exception) + logger = logging.getLogger("udp") + logger.error("spam") + self.handled.wait() + self.handled.clear() + logger.error("eggs") + self.handled.wait() + self.assertEqual(self.log_output, "spam\neggs\n") + +@unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") +class UnixDatagramHandlerTest(DatagramHandlerTest): + + """Test for DatagramHandler using Unix sockets.""" + + if hasattr(socket, "AF_UNIX"): + server_class = TestUnixDatagramServer + + def setUp(self): + # override the definition in the base class + self.address = socket_helper.create_unix_domain_name() + self.addCleanup(os_helper.unlink, self.address) + DatagramHandlerTest.setUp(self) + +@support.requires_working_socket() +@threading_helper.requires_working_threading() +class SysLogHandlerTest(BaseTest): + + """Test for SysLogHandler using UDP.""" + + server_class = TestUDPServer + address = ('localhost', 0) + + def setUp(self): + """Set up a UDP server to receive log messages, and a SysLogHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + # Issue #29177: deal with errors that happen during setup + self.server = self.sl_hdlr = self.server_exception = None + try: + self.server = server = self.server_class(self.address, + self.handle_datagram, 0.01) + server.start() + # Uncomment next line to test error recovery in setUp() + # raise OSError('dummy error raised') + except OSError as e: + self.server_exception = e + return + server.ready.wait() + hcls = logging.handlers.SysLogHandler + if isinstance(server.server_address, tuple): + self.sl_hdlr = hcls((server.server_address[0], server.port)) + else: + self.sl_hdlr = hcls(server.server_address) + self.log_output = b'' + self.root_logger.removeHandler(self.root_logger.handlers[0]) + self.root_logger.addHandler(self.sl_hdlr) + self.handled = threading.Event() + + def tearDown(self): + """Shutdown the server.""" + try: + if self.server: + self.server.stop() + if self.sl_hdlr: + self.root_logger.removeHandler(self.sl_hdlr) + self.sl_hdlr.close() + finally: + BaseTest.tearDown(self) + + def handle_datagram(self, request): + self.log_output = request.packet + self.handled.set() + + def test_output(self): + if self.server_exception: + self.skipTest(self.server_exception) + # The log message sent to the SysLogHandler is properly received. + logger = logging.getLogger("slh") + logger.error("sp\xe4m") + self.handled.wait(support.LONG_TIMEOUT) + self.assertEqual(self.log_output, b'<11>sp\xc3\xa4m\x00') + self.handled.clear() + self.sl_hdlr.append_nul = False + logger.error("sp\xe4m") + self.handled.wait(support.LONG_TIMEOUT) + self.assertEqual(self.log_output, b'<11>sp\xc3\xa4m') + self.handled.clear() + self.sl_hdlr.ident = "h\xe4m-" + logger.error("sp\xe4m") + self.handled.wait(support.LONG_TIMEOUT) + self.assertEqual(self.log_output, b'<11>h\xc3\xa4m-sp\xc3\xa4m') + + def test_udp_reconnection(self): + logger = logging.getLogger("slh") + self.sl_hdlr.close() + self.handled.clear() + logger.error("sp\xe4m") + self.handled.wait(support.LONG_TIMEOUT) + self.assertEqual(self.log_output, b'<11>sp\xc3\xa4m\x00') + +@unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") +class UnixSysLogHandlerTest(SysLogHandlerTest): + + """Test for SysLogHandler with Unix sockets.""" + + if hasattr(socket, "AF_UNIX"): + server_class = TestUnixDatagramServer + + def setUp(self): + # override the definition in the base class + self.address = socket_helper.create_unix_domain_name() + self.addCleanup(os_helper.unlink, self.address) + SysLogHandlerTest.setUp(self) + +@unittest.skipUnless(socket_helper.IPV6_ENABLED, + 'IPv6 support required for this test.') +class IPv6SysLogHandlerTest(SysLogHandlerTest): + + """Test for SysLogHandler with IPv6 host.""" + + server_class = TestUDPServer + address = ('::1', 0) + + def setUp(self): + self.server_class.address_family = socket.AF_INET6 + super(IPv6SysLogHandlerTest, self).setUp() + + def tearDown(self): + self.server_class.address_family = socket.AF_INET + super(IPv6SysLogHandlerTest, self).tearDown() + +@support.requires_working_socket() +@threading_helper.requires_working_threading() +class HTTPHandlerTest(BaseTest): + """Test for HTTPHandler.""" + + def setUp(self): + """Set up an HTTP server to receive log messages, and a HTTPHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + self.handled = threading.Event() + + def handle_request(self, request): + self.command = request.command + self.log_data = urlparse(request.path) + if self.command == 'POST': + try: + rlen = int(request.headers['Content-Length']) + self.post_data = request.rfile.read(rlen) + except: + self.post_data = None + request.send_response(200) + request.end_headers() + self.handled.set() + + def test_output(self): + # The log message sent to the HTTPHandler is properly received. + logger = logging.getLogger("http") + root_logger = self.root_logger + root_logger.removeHandler(self.root_logger.handlers[0]) + for secure in (False, True): + addr = ('localhost', 0) + if secure: + try: + import ssl + except ImportError: + sslctx = None + else: + here = os.path.dirname(__file__) + localhost_cert = os.path.join(here, "certdata", "keycert.pem") + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.load_cert_chain(localhost_cert) + + context = ssl.create_default_context(cafile=localhost_cert) + else: + sslctx = None + context = None + self.server = server = TestHTTPServer(addr, self.handle_request, + 0.01, sslctx=sslctx) + server.start() + server.ready.wait() + host = 'localhost:%d' % server.server_port + secure_client = secure and sslctx + self.h_hdlr = logging.handlers.HTTPHandler(host, '/frob', + secure=secure_client, + context=context, + credentials=('foo', 'bar')) + self.log_data = None + root_logger.addHandler(self.h_hdlr) + + for method in ('GET', 'POST'): + self.h_hdlr.method = method + self.handled.clear() + msg = "sp\xe4m" + logger.error(msg) + handled = self.handled.wait(support.SHORT_TIMEOUT) + self.assertTrue(handled, "HTTP request timed out") + self.assertEqual(self.log_data.path, '/frob') + self.assertEqual(self.command, method) + if method == 'GET': + d = parse_qs(self.log_data.query) + else: + d = parse_qs(self.post_data.decode('utf-8')) + self.assertEqual(d['name'], ['http']) + self.assertEqual(d['funcName'], ['test_output']) + self.assertEqual(d['msg'], [msg]) + + self.server.stop() + self.root_logger.removeHandler(self.h_hdlr) + self.h_hdlr.close() + +class MemoryTest(BaseTest): + + """Test memory persistence of logger objects.""" + + def setUp(self): + """Create a dict to remember potentially destroyed objects.""" + BaseTest.setUp(self) + self._survivors = {} + + def _watch_for_survival(self, *args): + """Watch the given objects for survival, by creating weakrefs to + them.""" + for obj in args: + key = id(obj), repr(obj) + self._survivors[key] = weakref.ref(obj) + + def _assertTruesurvival(self): + """Assert that all objects watched for survival have survived.""" + # Trigger cycle breaking. + gc.collect() + dead = [] + for (id_, repr_), ref in self._survivors.items(): + if ref() is None: + dead.append(repr_) + if dead: + self.fail("%d objects should have survived " + "but have been destroyed: %s" % (len(dead), ", ".join(dead))) + + def test_persistent_loggers(self): + # Logger objects are persistent and retain their configuration, even + # if visible references are destroyed. + self.root_logger.setLevel(logging.INFO) + foo = logging.getLogger("foo") + self._watch_for_survival(foo) + foo.setLevel(logging.DEBUG) + self.root_logger.debug(self.next_message()) + foo.debug(self.next_message()) + self.assert_log_lines([ + ('foo', 'DEBUG', '2'), + ]) + del foo + # foo has survived. + self._assertTruesurvival() + # foo has retained its settings. + bar = logging.getLogger("foo") + bar.debug(self.next_message()) + self.assert_log_lines([ + ('foo', 'DEBUG', '2'), + ('foo', 'DEBUG', '3'), + ]) + + +class EncodingTest(BaseTest): + def test_encoding_plain_file(self): + # In Python 2.x, a plain file object is treated as having no encoding. + log = logging.getLogger("test") + fn = make_temp_file(".log", "test_logging-1-") + # the non-ascii data we write to the log. + data = "foo\x80" + try: + handler = logging.FileHandler(fn, encoding="utf-8") + log.addHandler(handler) + try: + # write non-ascii data to the log. + log.warning(data) + finally: + log.removeHandler(handler) + handler.close() + # check we wrote exactly those bytes, ignoring trailing \n etc + f = open(fn, encoding="utf-8") + try: + self.assertEqual(f.read().rstrip(), data) + finally: + f.close() + finally: + if os.path.isfile(fn): + os.remove(fn) + + def test_encoding_cyrillic_unicode(self): + log = logging.getLogger("test") + # Get a message in Unicode: Do svidanya in Cyrillic (meaning goodbye) + message = '\u0434\u043e \u0441\u0432\u0438\u0434\u0430\u043d\u0438\u044f' + # Ensure it's written in a Cyrillic encoding + writer_class = codecs.getwriter('cp1251') + writer_class.encoding = 'cp1251' + stream = io.BytesIO() + writer = writer_class(stream, 'strict') + handler = logging.StreamHandler(writer) + log.addHandler(handler) + try: + log.warning(message) + finally: + log.removeHandler(handler) + handler.close() + # check we wrote exactly those bytes, ignoring trailing \n etc + s = stream.getvalue() + # Compare against what the data should be when encoded in CP-1251 + self.assertEqual(s, b'\xe4\xee \xf1\xe2\xe8\xe4\xe0\xed\xe8\xff\n') + + +class WarningsTest(BaseTest): + + def test_warnings(self): + with warnings.catch_warnings(): + logging.captureWarnings(True) + self.addCleanup(logging.captureWarnings, False) + warnings.filterwarnings("always", category=UserWarning) + stream = io.StringIO() + h = logging.StreamHandler(stream) + logger = logging.getLogger("py.warnings") + logger.addHandler(h) + warnings.warn("I'm warning you...") + logger.removeHandler(h) + s = stream.getvalue() + h.close() + self.assertGreater(s.find("UserWarning: I'm warning you...\n"), 0) + + # See if an explicit file uses the original implementation + a_file = io.StringIO() + warnings.showwarning("Explicit", UserWarning, "dummy.py", 42, + a_file, "Dummy line") + s = a_file.getvalue() + a_file.close() + self.assertEqual(s, + "dummy.py:42: UserWarning: Explicit\n Dummy line\n") + + def test_warnings_no_handlers(self): + with warnings.catch_warnings(): + logging.captureWarnings(True) + self.addCleanup(logging.captureWarnings, False) + + # confirm our assumption: no loggers are set + logger = logging.getLogger("py.warnings") + self.assertEqual(logger.handlers, []) + + warnings.showwarning("Explicit", UserWarning, "dummy.py", 42) + self.assertEqual(len(logger.handlers), 1) + self.assertIsInstance(logger.handlers[0], logging.NullHandler) + + +def formatFunc(format, datefmt=None): + return logging.Formatter(format, datefmt) + +class myCustomFormatter: + def __init__(self, fmt, datefmt=None): + pass + +def handlerFunc(): + return logging.StreamHandler() + +class CustomHandler(logging.StreamHandler): + pass + +class CustomListener(logging.handlers.QueueListener): + pass + +class CustomQueue(queue.Queue): + pass + +class CustomQueueProtocol: + def __init__(self, maxsize=0): + self.queue = queue.Queue(maxsize) + + def __getattr__(self, attribute): + queue = object.__getattribute__(self, 'queue') + return getattr(queue, attribute) + +class CustomQueueFakeProtocol(CustomQueueProtocol): + # An object implementing the minimial Queue API for + # the logging module but with incorrect signatures. + # + # The object will be considered a valid queue class since we + # do not check the signatures (only callability of methods) + # but will NOT be usable in production since a TypeError will + # be raised due to the extra argument in 'put_nowait'. + def put_nowait(self): + pass + +class CustomQueueWrongProtocol(CustomQueueProtocol): + put_nowait = None + +class MinimalQueueProtocol: + def put_nowait(self, x): pass + def get(self): pass + +def queueMaker(): + return queue.Queue() + +def listenerMaker(arg1, arg2, respect_handler_level=False): + def func(queue, *handlers, **kwargs): + kwargs.setdefault('respect_handler_level', respect_handler_level) + return CustomListener(queue, *handlers, **kwargs) + return func + +class ConfigDictTest(BaseTest): + + """Reading logging config from a dictionary.""" + + check_no_resource_warning = warnings_helper.check_no_resource_warning + expected_log_pat = r"^(\w+) \+\+ (\w+)$" + + # config0 is a standard configuration. + config0 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + + # config1 adds a little to the standard configuration. + config1 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # config1a moves the handler to the root. Used with config8a + config1a = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + + # config2 has a subtle configuration error that should be reported + config2 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdbout', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # As config1 but with a misspelt level on a handler + config2a = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NTOSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + + # As config1 but with a misspelt level on a logger + config2b = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WRANING', + }, + } + + # config3 has a less subtle configuration error + config3 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'misspelled_name', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # config4 specifies a custom formatter class to be loaded + config4 = { + 'version': 1, + 'formatters': { + 'form1' : { + '()' : __name__ + '.ExceptionFormatter', + 'format' : '%(levelname)s:%(name)s:%(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'root' : { + 'level' : 'NOTSET', + 'handlers' : ['hand1'], + }, + } + + # As config4 but using an actual callable rather than a string + config4a = { + 'version': 1, + 'formatters': { + 'form1' : { + '()' : ExceptionFormatter, + 'format' : '%(levelname)s:%(name)s:%(message)s', + }, + 'form2' : { + '()' : __name__ + '.formatFunc', + 'format' : '%(levelname)s:%(name)s:%(message)s', + }, + 'form3' : { + '()' : formatFunc, + 'format' : '%(levelname)s:%(name)s:%(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + 'hand2' : { + '()' : handlerFunc, + }, + }, + 'root' : { + 'level' : 'NOTSET', + 'handlers' : ['hand1'], + }, + } + + # config5 specifies a custom handler class to be loaded + config5 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : __name__ + '.CustomHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # config6 specifies a custom handler class to be loaded + # but has bad arguments + config6 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : __name__ + '.CustomHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + '9' : 'invalid parameter name', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # config 7 does not define compiler.parser but defines compiler.lexer + # so compiler.parser should be disabled after applying it + config7 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler.lexer' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # config8 defines both compiler and compiler.lexer + # so compiler.parser should not be disabled (since + # compiler is defined) + config8 = { + 'version': 1, + 'disable_existing_loggers' : False, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + 'compiler.lexer' : { + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # config8a disables existing loggers + config8a = { + 'version': 1, + 'disable_existing_loggers' : True, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + 'compiler.lexer' : { + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + config9 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'WARNING', + 'stream' : 'ext://sys.stdout', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'NOTSET', + }, + } + + config9a = { + 'version': 1, + 'incremental' : True, + 'handlers' : { + 'hand1' : { + 'level' : 'WARNING', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'INFO', + }, + }, + } + + config9b = { + 'version': 1, + 'incremental' : True, + 'handlers' : { + 'hand1' : { + 'level' : 'INFO', + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'INFO', + }, + }, + } + + # As config1 but with a filter added + config10 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'filters' : { + 'filt1' : { + 'name' : 'compiler.parser', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + 'filters' : ['filt1'], + }, + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'filters' : ['filt1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + + # As config1 but using cfg:// references + config11 = { + 'version': 1, + 'true_formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handler_configs': { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'formatters' : 'cfg://true_formatters', + 'handlers' : { + 'hand1' : 'cfg://handler_configs[hand1]', + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # As config11 but missing the version key + config12 = { + 'true_formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handler_configs': { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'formatters' : 'cfg://true_formatters', + 'handlers' : { + 'hand1' : 'cfg://handler_configs[hand1]', + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # As config11 but using an unsupported version + config13 = { + 'version': 2, + 'true_formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handler_configs': { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'formatters' : 'cfg://true_formatters', + 'handlers' : { + 'hand1' : 'cfg://handler_configs[hand1]', + }, + 'loggers' : { + 'compiler.parser' : { + 'level' : 'DEBUG', + 'handlers' : ['hand1'], + }, + }, + 'root' : { + 'level' : 'WARNING', + }, + } + + # As config0, but with properties + config14 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(levelname)s ++ %(message)s', + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + '.': { + 'foo': 'bar', + 'terminator': '!\n', + } + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + + # config0 but with default values for formatter. Skipped 15, it is defined + # in the test code. + config16 = { + 'version': 1, + 'formatters': { + 'form1' : { + 'format' : '%(message)s ++ %(customfield)s', + 'defaults': {"customfield": "defaultvalue"} + }, + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'form1', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + + class CustomFormatter(logging.Formatter): + custom_property = "." + + def format(self, record): + return super().format(record) + + config17 = { + 'version': 1, + 'formatters': { + "custom": { + "()": CustomFormatter, + "style": "{", + "datefmt": "%Y-%m-%d %H:%M:%S", + "format": "{message}", # <-- to force an exception when configuring + ".": { + "custom_property": "value" + } + } + }, + 'handlers' : { + 'hand1' : { + 'class' : 'logging.StreamHandler', + 'formatter' : 'custom', + 'level' : 'NOTSET', + 'stream' : 'ext://sys.stdout', + }, + }, + 'root' : { + 'level' : 'WARNING', + 'handlers' : ['hand1'], + }, + } + + config18 = { + "version": 1, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + }, + "buffering": { + "class": "logging.handlers.MemoryHandler", + "capacity": 5, + "target": "console", + "level": "DEBUG", + "flushLevel": "ERROR" + } + }, + "loggers": { + "mymodule": { + "level": "DEBUG", + "handlers": ["buffering"], + "propagate": "true" + } + } + } + + bad_format = { + "version": 1, + "formatters": { + "mySimpleFormatter": { + "format": "%(asctime)s (%(name)s) %(levelname)s: %(message)s", + "style": "$" + } + }, + "handlers": { + "fileGlobal": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "mySimpleFormatter" + }, + "bufferGlobal": { + "class": "logging.handlers.MemoryHandler", + "capacity": 5, + "formatter": "mySimpleFormatter", + "target": "fileGlobal", + "level": "DEBUG" + } + }, + "loggers": { + "mymodule": { + "level": "DEBUG", + "handlers": ["bufferGlobal"], + "propagate": "true" + } + } + } + + # Configuration with custom logging.Formatter subclass as '()' key and 'validate' set to False + custom_formatter_class_validate = { + 'version': 1, + 'formatters': { + 'form1': { + '()': __name__ + '.ExceptionFormatter', + 'format': '%(levelname)s:%(name)s:%(message)s', + 'validate': False, + }, + }, + 'handlers' : { + 'hand1' : { + 'class': 'logging.StreamHandler', + 'formatter': 'form1', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdout', + }, + }, + "loggers": { + "my_test_logger_custom_formatter": { + "level": "DEBUG", + "handlers": ["hand1"], + "propagate": "true" + } + } + } + + # Configuration with custom logging.Formatter subclass as 'class' key and 'validate' set to False + custom_formatter_class_validate2 = { + 'version': 1, + 'formatters': { + 'form1': { + 'class': __name__ + '.ExceptionFormatter', + 'format': '%(levelname)s:%(name)s:%(message)s', + 'validate': False, + }, + }, + 'handlers' : { + 'hand1' : { + 'class': 'logging.StreamHandler', + 'formatter': 'form1', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdout', + }, + }, + "loggers": { + "my_test_logger_custom_formatter": { + "level": "DEBUG", + "handlers": ["hand1"], + "propagate": "true" + } + } + } + + # Configuration with custom class that is not inherited from logging.Formatter + custom_formatter_class_validate3 = { + 'version': 1, + 'formatters': { + 'form1': { + 'class': __name__ + '.myCustomFormatter', + 'format': '%(levelname)s:%(name)s:%(message)s', + 'validate': False, + }, + }, + 'handlers' : { + 'hand1' : { + 'class': 'logging.StreamHandler', + 'formatter': 'form1', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdout', + }, + }, + "loggers": { + "my_test_logger_custom_formatter": { + "level": "DEBUG", + "handlers": ["hand1"], + "propagate": "true" + } + } + } + + # Configuration with custom function, 'validate' set to False and no defaults + custom_formatter_with_function = { + 'version': 1, + 'formatters': { + 'form1': { + '()': formatFunc, + 'format': '%(levelname)s:%(name)s:%(message)s', + 'validate': False, + }, + }, + 'handlers' : { + 'hand1' : { + 'class': 'logging.StreamHandler', + 'formatter': 'form1', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdout', + }, + }, + "loggers": { + "my_test_logger_custom_formatter": { + "level": "DEBUG", + "handlers": ["hand1"], + "propagate": "true" + } + } + } + + # Configuration with custom function, and defaults + custom_formatter_with_defaults = { + 'version': 1, + 'formatters': { + 'form1': { + '()': formatFunc, + 'format': '%(levelname)s:%(name)s:%(message)s:%(customfield)s', + 'defaults': {"customfield": "myvalue"} + }, + }, + 'handlers' : { + 'hand1' : { + 'class': 'logging.StreamHandler', + 'formatter': 'form1', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdout', + }, + }, + "loggers": { + "my_test_logger_custom_formatter": { + "level": "DEBUG", + "handlers": ["hand1"], + "propagate": "true" + } + } + } + + config_queue_handler = { + 'version': 1, + 'handlers' : { + 'h1' : { + 'class': 'logging.FileHandler', + }, + # key is before depended on handlers to test that deferred config works + 'ah' : { + 'class': 'logging.handlers.QueueHandler', + 'handlers': ['h1'] + }, + }, + "root": { + "level": "DEBUG", + "handlers": ["ah"] + } + } + + def apply_config(self, conf): + logging.config.dictConfig(conf) + + def check_handler(self, name, cls): + h = logging.getHandlerByName(name) + self.assertIsInstance(h, cls) + + def test_config0_ok(self): + # A simple config which overrides the default settings. + with support.captured_stdout() as output: + self.apply_config(self.config0) + self.check_handler('hand1', logging.StreamHandler) + logger = logging.getLogger() + # Won't output anything + logger.info(self.next_message()) + # Outputs a message + logger.error(self.next_message()) + self.assert_log_lines([ + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + def test_config1_ok(self, config=config1): + # A config defining a sub-parser as well. + with support.captured_stdout() as output: + self.apply_config(config) + logger = logging.getLogger("compiler.parser") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + def test_config2_failure(self): + # A simple config which overrides the default settings. + self.assertRaises(Exception, self.apply_config, self.config2) + + def test_config2a_failure(self): + # A simple config which overrides the default settings. + self.assertRaises(Exception, self.apply_config, self.config2a) + + def test_config2b_failure(self): + # A simple config which overrides the default settings. + self.assertRaises(Exception, self.apply_config, self.config2b) + + def test_config3_failure(self): + # A simple config which overrides the default settings. + self.assertRaises(Exception, self.apply_config, self.config3) + + def test_config4_ok(self): + # A config specifying a custom formatter class. + with support.captured_stdout() as output: + self.apply_config(self.config4) + self.check_handler('hand1', logging.StreamHandler) + #logger = logging.getLogger() + try: + raise RuntimeError() + except RuntimeError: + logging.exception("just testing") + sys.stdout.seek(0) + self.assertEqual(output.getvalue(), + "ERROR:root:just testing\nGot a [RuntimeError]\n") + # Original logger output is empty + self.assert_log_lines([]) + + def test_config4a_ok(self): + # A config specifying a custom formatter class. + with support.captured_stdout() as output: + self.apply_config(self.config4a) + #logger = logging.getLogger() + try: + raise RuntimeError() + except RuntimeError: + logging.exception("just testing") + sys.stdout.seek(0) + self.assertEqual(output.getvalue(), + "ERROR:root:just testing\nGot a [RuntimeError]\n") + # Original logger output is empty + self.assert_log_lines([]) + + def test_config5_ok(self): + self.test_config1_ok(config=self.config5) + self.check_handler('hand1', CustomHandler) + + def test_config6_failure(self): + self.assertRaises(Exception, self.apply_config, self.config6) + + def test_config7_ok(self): + with support.captured_stdout() as output: + self.apply_config(self.config1) + logger = logging.getLogger("compiler.parser") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + with support.captured_stdout() as output: + self.apply_config(self.config7) + self.check_handler('hand1', logging.StreamHandler) + logger = logging.getLogger("compiler.parser") + self.assertTrue(logger.disabled) + logger = logging.getLogger("compiler.lexer") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '3'), + ('ERROR', '4'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + # Same as test_config_7_ok but don't disable old loggers. + def test_config_8_ok(self): + with support.captured_stdout() as output: + self.apply_config(self.config1) + logger = logging.getLogger("compiler.parser") + # All will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + with support.captured_stdout() as output: + self.apply_config(self.config8) + self.check_handler('hand1', logging.StreamHandler) + logger = logging.getLogger("compiler.parser") + self.assertFalse(logger.disabled) + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + logger = logging.getLogger("compiler.lexer") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '3'), + ('ERROR', '4'), + ('INFO', '5'), + ('ERROR', '6'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + def test_config_8a_ok(self): + with support.captured_stdout() as output: + self.apply_config(self.config1a) + self.check_handler('hand1', logging.StreamHandler) + logger = logging.getLogger("compiler.parser") + # See issue #11424. compiler-hyphenated sorts + # between compiler and compiler.xyz and this + # was preventing compiler.xyz from being included + # in the child loggers of compiler because of an + # overzealous loop termination condition. + hyphenated = logging.getLogger('compiler-hyphenated') + # All will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + hyphenated.critical(self.next_message()) + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ('CRITICAL', '3'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + with support.captured_stdout() as output: + self.apply_config(self.config8a) + self.check_handler('hand1', logging.StreamHandler) + logger = logging.getLogger("compiler.parser") + self.assertFalse(logger.disabled) + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + logger = logging.getLogger("compiler.lexer") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + # Will not appear + hyphenated.critical(self.next_message()) + self.assert_log_lines([ + ('INFO', '4'), + ('ERROR', '5'), + ('INFO', '6'), + ('ERROR', '7'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + def test_config_9_ok(self): + with support.captured_stdout() as output: + self.apply_config(self.config9) + self.check_handler('hand1', logging.StreamHandler) + logger = logging.getLogger("compiler.parser") + # Nothing will be output since both handler and logger are set to WARNING + logger.info(self.next_message()) + self.assert_log_lines([], stream=output) + self.apply_config(self.config9a) + # Nothing will be output since handler is still set to WARNING + logger.info(self.next_message()) + self.assert_log_lines([], stream=output) + self.apply_config(self.config9b) + # Message should now be output + logger.info(self.next_message()) + self.assert_log_lines([ + ('INFO', '3'), + ], stream=output) + + def test_config_10_ok(self): + with support.captured_stdout() as output: + self.apply_config(self.config10) + self.check_handler('hand1', logging.StreamHandler) + logger = logging.getLogger("compiler.parser") + logger.warning(self.next_message()) + logger = logging.getLogger('compiler') + # Not output, because filtered + logger.warning(self.next_message()) + logger = logging.getLogger('compiler.lexer') + # Not output, because filtered + logger.warning(self.next_message()) + logger = logging.getLogger("compiler.parser.codegen") + # Output, as not filtered + logger.error(self.next_message()) + self.assert_log_lines([ + ('WARNING', '1'), + ('ERROR', '4'), + ], stream=output) + + def test_config11_ok(self): + self.test_config1_ok(self.config11) + + def test_config12_failure(self): + self.assertRaises(Exception, self.apply_config, self.config12) + + def test_config13_failure(self): + self.assertRaises(Exception, self.apply_config, self.config13) + + def test_config14_ok(self): + with support.captured_stdout() as output: + self.apply_config(self.config14) + h = logging._handlers['hand1'] + self.assertEqual(h.foo, 'bar') + self.assertEqual(h.terminator, '!\n') + logging.warning('Exclamation') + self.assertTrue(output.getvalue().endswith('Exclamation!\n')) + + def test_config15_ok(self): + + with self.check_no_resource_warning(): + fn = make_temp_file(".log", "test_logging-X-") + + config = { + "version": 1, + "handlers": { + "file": { + "class": "logging.FileHandler", + "filename": fn, + "encoding": "utf-8", + } + }, + "root": { + "handlers": ["file"] + } + } + + self.apply_config(config) + self.apply_config(config) + + handler = logging.root.handlers[0] + self.addCleanup(closeFileHandler, handler, fn) + + def test_config16_ok(self): + self.apply_config(self.config16) + h = logging._handlers['hand1'] + + # Custom value + result = h.formatter.format(logging.makeLogRecord( + {'msg': 'Hello', 'customfield': 'customvalue'})) + self.assertEqual(result, 'Hello ++ customvalue') + + # Default value + result = h.formatter.format(logging.makeLogRecord( + {'msg': 'Hello'})) + self.assertEqual(result, 'Hello ++ defaultvalue') + + def test_config17_ok(self): + self.apply_config(self.config17) + h = logging._handlers['hand1'] + self.assertEqual(h.formatter.custom_property, 'value') + + def test_config18_ok(self): + self.apply_config(self.config18) + handler = logging.getLogger('mymodule').handlers[0] + self.assertEqual(handler.flushLevel, logging.ERROR) + + def setup_via_listener(self, text, verify=None): + text = text.encode("utf-8") + # Ask for a randomly assigned port (by using port 0) + t = logging.config.listen(0, verify) + t.start() + t.ready.wait() + # Now get the port allocated + port = t.port + t.ready.clear() + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(2.0) + sock.connect(('localhost', port)) + + slen = struct.pack('>L', len(text)) + s = slen + text + sentsofar = 0 + left = len(s) + while left > 0: + sent = sock.send(s[sentsofar:]) + sentsofar += sent + left -= sent + sock.close() + finally: + t.ready.wait(2.0) + logging.config.stopListening() + threading_helper.join_thread(t) + + @support.requires_working_socket() + def test_listen_config_10_ok(self): + with support.captured_stdout() as output: + self.setup_via_listener(json.dumps(self.config10)) + self.check_handler('hand1', logging.StreamHandler) + logger = logging.getLogger("compiler.parser") + logger.warning(self.next_message()) + logger = logging.getLogger('compiler') + # Not output, because filtered + logger.warning(self.next_message()) + logger = logging.getLogger('compiler.lexer') + # Not output, because filtered + logger.warning(self.next_message()) + logger = logging.getLogger("compiler.parser.codegen") + # Output, as not filtered + logger.error(self.next_message()) + self.assert_log_lines([ + ('WARNING', '1'), + ('ERROR', '4'), + ], stream=output) + + @support.requires_working_socket() + def test_listen_config_1_ok(self): + with support.captured_stdout() as output: + self.setup_via_listener(textwrap.dedent(ConfigFileTest.config1)) + logger = logging.getLogger("compiler.parser") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], stream=output) + # Original logger output is empty. + self.assert_log_lines([]) + + @support.requires_working_socket() + def test_listen_verify(self): + + def verify_fail(stuff): + return None + + def verify_reverse(stuff): + return stuff[::-1] + + logger = logging.getLogger("compiler.parser") + to_send = textwrap.dedent(ConfigFileTest.config1) + # First, specify a verification function that will fail. + # We expect to see no output, since our configuration + # never took effect. + with support.captured_stdout() as output: + self.setup_via_listener(to_send, verify_fail) + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([], stream=output) + # Original logger output has the stuff we logged. + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + + # Now, perform no verification. Our configuration + # should take effect. + + with support.captured_stdout() as output: + self.setup_via_listener(to_send) # no verify callable specified + logger = logging.getLogger("compiler.parser") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '3'), + ('ERROR', '4'), + ], stream=output) + # Original logger output still has the stuff we logged before. + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + + # Now, perform verification which transforms the bytes. + + with support.captured_stdout() as output: + self.setup_via_listener(to_send[::-1], verify_reverse) + logger = logging.getLogger("compiler.parser") + # Both will output a message + logger.info(self.next_message()) + logger.error(self.next_message()) + self.assert_log_lines([ + ('INFO', '5'), + ('ERROR', '6'), + ], stream=output) + # Original logger output still has the stuff we logged before. + self.assert_log_lines([ + ('INFO', '1'), + ('ERROR', '2'), + ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + + def test_bad_format(self): + self.assertRaises(ValueError, self.apply_config, self.bad_format) + + def test_bad_format_with_dollar_style(self): + config = copy.deepcopy(self.bad_format) + config['formatters']['mySimpleFormatter']['format'] = "${asctime} (${name}) ${levelname}: ${message}" + + self.apply_config(config) + handler = logging.getLogger('mymodule').handlers[0] + self.assertIsInstance(handler.target, logging.Handler) + self.assertIsInstance(handler.formatter._style, + logging.StringTemplateStyle) + self.assertEqual(sorted(logging.getHandlerNames()), + ['bufferGlobal', 'fileGlobal']) + + def test_custom_formatter_class_with_validate(self): + self.apply_config(self.custom_formatter_class_validate) + handler = logging.getLogger("my_test_logger_custom_formatter").handlers[0] + self.assertIsInstance(handler.formatter, ExceptionFormatter) + + def test_custom_formatter_class_with_validate2(self): + self.apply_config(self.custom_formatter_class_validate2) + handler = logging.getLogger("my_test_logger_custom_formatter").handlers[0] + self.assertIsInstance(handler.formatter, ExceptionFormatter) + + def test_custom_formatter_class_with_validate2_with_wrong_fmt(self): + config = self.custom_formatter_class_validate.copy() + config['formatters']['form1']['style'] = "$" + + # Exception should not be raised as we have configured 'validate' to False + self.apply_config(config) + handler = logging.getLogger("my_test_logger_custom_formatter").handlers[0] + self.assertIsInstance(handler.formatter, ExceptionFormatter) + + def test_custom_formatter_class_with_validate3(self): + self.assertRaises(ValueError, self.apply_config, self.custom_formatter_class_validate3) + + def test_custom_formatter_function_with_validate(self): + self.assertRaises(ValueError, self.apply_config, self.custom_formatter_with_function) + + def test_custom_formatter_function_with_defaults(self): + self.assertRaises(ValueError, self.apply_config, self.custom_formatter_with_defaults) + + def test_baseconfig(self): + d = { + 'atuple': (1, 2, 3), + 'alist': ['a', 'b', 'c'], + 'adict': { + 'd': 'e', 'f': 3 , + 'alpha numeric 1 with spaces' : 5, + 'alpha numeric 1 %( - © ©ß¯' : 9, + 'alpha numeric ] 1 with spaces' : 15, + 'alpha ]] numeric 1 %( - © ©ß¯]' : 19, + ' alpha [ numeric 1 %( - © ©ß¯] ' : 11, + ' alpha ' : 32, + '' : 10, + 'nest4' : { + 'd': 'e', 'f': 3 , + 'alpha numeric 1 with spaces' : 5, + 'alpha numeric 1 %( - © ©ß¯' : 9, + '' : 10, + 'somelist' : ('g', ('h', 'i'), 'j'), + 'somedict' : { + 'a' : 1, + 'a with 1 and space' : 3, + 'a with ( and space' : 4, + } + } + }, + 'nest1': ('g', ('h', 'i'), 'j'), + 'nest2': ['k', ['l', 'm'], 'n'], + 'nest3': ['o', 'cfg://alist', 'p'], + } + bc = logging.config.BaseConfigurator(d) + self.assertEqual(bc.convert('cfg://atuple[1]'), 2) + self.assertEqual(bc.convert('cfg://alist[1]'), 'b') + self.assertEqual(bc.convert('cfg://nest1[1][0]'), 'h') + self.assertEqual(bc.convert('cfg://nest2[1][1]'), 'm') + self.assertEqual(bc.convert('cfg://adict.d'), 'e') + self.assertEqual(bc.convert('cfg://adict[f]'), 3) + self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 with spaces]'), 5) + self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[]'), 10) + self.assertEqual(bc.convert('cfg://adict.nest4.d'), 'e') + self.assertEqual(bc.convert('cfg://adict.nest4[d]'), 'e') + self.assertEqual(bc.convert('cfg://adict[nest4].d'), 'e') + self.assertEqual(bc.convert('cfg://adict[nest4][f]'), 3) + self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 with spaces]'), 5) + self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[nest4][]'), 10) + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][0]'), 'g') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][1][0]'), 'h') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][1][1]'), 'i') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][2]'), 'j') + self.assertEqual(bc.convert('cfg://adict[nest4].somedict.a'), 1) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a]'), 1) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a with 1 and space]'), 3) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a with ( and space]'), 4) + self.assertEqual(bc.convert('cfg://adict.nest4.somelist[1][1]'), 'i') + self.assertEqual(bc.convert('cfg://adict.nest4.somelist[2]'), 'j') + self.assertEqual(bc.convert('cfg://adict.nest4.somedict.a'), 1) + self.assertEqual(bc.convert('cfg://adict.nest4.somedict[a]'), 1) + v = bc.convert('cfg://nest3') + self.assertEqual(v.pop(1), ['a', 'b', 'c']) + self.assertRaises(KeyError, bc.convert, 'cfg://nosuch') + self.assertRaises(ValueError, bc.convert, 'cfg://!') + self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]') + self.assertRaises(KeyError, bc.convert, 'cfg://adict[alpha numeric ] 1 with spaces]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ alpha ]] numeric 1 %( - © ©ß¯] ]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ alpha [ numeric 1 %( - © ©ß¯] ]') + + def test_namedtuple(self): + # see bpo-39142 + from collections import namedtuple + + class MyHandler(logging.StreamHandler): + def __init__(self, resource, *args, **kwargs): + super().__init__(*args, **kwargs) + self.resource: namedtuple = resource + + def emit(self, record): + record.msg += f' {self.resource.type}' + return super().emit(record) + + Resource = namedtuple('Resource', ['type', 'labels']) + resource = Resource(type='my_type', labels=['a']) + + config = { + 'version': 1, + 'handlers': { + 'myhandler': { + '()': MyHandler, + 'resource': resource + } + }, + 'root': {'level': 'INFO', 'handlers': ['myhandler']}, + } + with support.captured_stderr() as stderr: + self.apply_config(config) + logging.info('some log') + self.assertEqual(stderr.getvalue(), 'some log my_type\n') + + def test_config_callable_filter_works(self): + def filter_(_): + return 1 + self.apply_config({ + "version": 1, "root": {"level": "DEBUG", "filters": [filter_]} + }) + assert logging.getLogger().filters[0] is filter_ + logging.getLogger().filters = [] + + def test_config_filter_works(self): + filter_ = logging.Filter("spam.eggs") + self.apply_config({ + "version": 1, "root": {"level": "DEBUG", "filters": [filter_]} + }) + assert logging.getLogger().filters[0] is filter_ + logging.getLogger().filters = [] + + def test_config_filter_method_works(self): + class FakeFilter: + def filter(self, _): + return 1 + filter_ = FakeFilter() + self.apply_config({ + "version": 1, "root": {"level": "DEBUG", "filters": [filter_]} + }) + assert logging.getLogger().filters[0] is filter_ + logging.getLogger().filters = [] + + def test_invalid_type_raises(self): + class NotAFilter: pass + for filter_ in [None, 1, NotAFilter()]: + self.assertRaises( + ValueError, + self.apply_config, + {"version": 1, "root": {"level": "DEBUG", "filters": [filter_]}} + ) + + def do_queuehandler_configuration(self, qspec, lspec): + cd = copy.deepcopy(self.config_queue_handler) + fn = make_temp_file('.log', 'test_logging-cqh-') + cd['handlers']['h1']['filename'] = fn + if qspec is not None: + cd['handlers']['ah']['queue'] = qspec + if lspec is not None: + cd['handlers']['ah']['listener'] = lspec + qh = None + try: + self.apply_config(cd) + qh = logging.getHandlerByName('ah') + self.assertEqual(sorted(logging.getHandlerNames()), ['ah', 'h1']) + self.assertIsNotNone(qh.listener) + qh.listener.start() + logging.debug('foo') + logging.info('bar') + logging.warning('baz') + + # Need to let the listener thread finish its work + while support.sleeping_retry(support.LONG_TIMEOUT, + "queue not empty"): + if qh.listener.queue.empty(): + break + + # wait until the handler completed its last task + qh.listener.queue.join() + + with open(fn, encoding='utf-8') as f: + data = f.read().splitlines() + self.assertEqual(data, ['foo', 'bar', 'baz']) + finally: + if qh: + qh.listener.stop() + h = logging.getHandlerByName('h1') + if h: + self.addCleanup(closeFileHandler, h, fn) + else: + self.addCleanup(os.remove, fn) + + @threading_helper.requires_working_threading() + @support.requires_subprocess() + def test_config_queue_handler(self): + qs = [CustomQueue(), CustomQueueProtocol()] + dqs = [{'()': f'{__name__}.{cls}', 'maxsize': 10} + for cls in ['CustomQueue', 'CustomQueueProtocol']] + dl = { + '()': __name__ + '.listenerMaker', + 'arg1': None, + 'arg2': None, + 'respect_handler_level': True + } + qvalues = (None, __name__ + '.queueMaker', __name__ + '.CustomQueue', *dqs, *qs) + lvalues = (None, __name__ + '.CustomListener', dl, CustomListener) + for qspec, lspec in itertools.product(qvalues, lvalues): + self.do_queuehandler_configuration(qspec, lspec) + + # Some failure cases + qvalues = (None, 4, int, '', 'foo') + lvalues = (None, 4, int, '', 'bar') + for qspec, lspec in itertools.product(qvalues, lvalues): + if lspec is None and qspec is None: + continue + with self.assertRaises(ValueError) as ctx: + self.do_queuehandler_configuration(qspec, lspec) + msg = str(ctx.exception) + self.assertEqual(msg, "Unable to configure handler 'ah'") + + def _apply_simple_queue_listener_configuration(self, qspec): + self.apply_config({ + "version": 1, + "handlers": { + "queue_listener": { + "class": "logging.handlers.QueueHandler", + "queue": qspec, + }, + }, + }) + + @threading_helper.requires_working_threading() + @support.requires_subprocess() + @patch("multiprocessing.Manager") + def test_config_queue_handler_does_not_create_multiprocessing_manager(self, manager): + # gh-120868, gh-121723, gh-124653 + + for qspec in [ + {"()": "queue.Queue", "maxsize": -1}, + queue.Queue(), + # queue.SimpleQueue does not inherit from queue.Queue + queue.SimpleQueue(), + # CustomQueueFakeProtocol passes the checks but will not be usable + # since the signatures are incompatible. Checking the Queue API + # without testing the type of the actual queue is a trade-off + # between usability and the work we need to do in order to safely + # check that the queue object correctly implements the API. + CustomQueueFakeProtocol(), + MinimalQueueProtocol(), + ]: + with self.subTest(qspec=qspec): + self._apply_simple_queue_listener_configuration(qspec) + manager.assert_not_called() + + @patch("multiprocessing.Manager") + def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_manager(self, manager): + # gh-120868, gh-121723 + + for qspec in [object(), CustomQueueWrongProtocol()]: + with self.subTest(qspec=qspec), self.assertRaises(ValueError): + self._apply_simple_queue_listener_configuration(qspec) + manager.assert_not_called() + + @skip_if_tsan_fork + @support.requires_subprocess() + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" + " assertions in multiprocessing") + def test_config_reject_simple_queue_handler_multiprocessing_context(self): + # multiprocessing.SimpleQueue does not implement 'put_nowait' + # and thus cannot be used as a queue-like object (gh-124653) + + import multiprocessing + + if support.MS_WINDOWS: + start_methods = ['spawn'] + else: + start_methods = ['spawn', 'fork', 'forkserver'] + + for start_method in start_methods: + with self.subTest(start_method=start_method): + ctx = multiprocessing.get_context(start_method) + qspec = ctx.SimpleQueue() + with self.assertRaises(ValueError): + self._apply_simple_queue_listener_configuration(qspec) + + @skip_if_tsan_fork + @support.requires_subprocess() + @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" + " assertions in multiprocessing") + def test_config_queue_handler_multiprocessing_context(self): + # regression test for gh-121723 + if support.MS_WINDOWS: + start_methods = ['spawn'] + else: + start_methods = ['spawn', 'fork', 'forkserver'] + for start_method in start_methods: + with self.subTest(start_method=start_method): + ctx = multiprocessing.get_context(start_method) + with ctx.Manager() as manager: + q = manager.Queue() + records = [] + # use 1 process and 1 task per child to put 1 record + with ctx.Pool(1, initializer=self._mpinit_issue121723, + initargs=(q, "text"), maxtasksperchild=1): + records.append(q.get(timeout=60)) + self.assertTrue(q.empty()) + self.assertEqual(len(records), 1) + + @staticmethod + def _mpinit_issue121723(qspec, message_to_log): + # static method for pickling support + logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': True, + 'handlers': { + 'log_to_parent': { + 'class': 'logging.handlers.QueueHandler', + 'queue': qspec + } + }, + 'root': {'handlers': ['log_to_parent'], 'level': 'DEBUG'} + }) + # log a message (this creates a record put in the queue) + logging.getLogger().info(message_to_log) + + @unittest.skip('TODO: RUSTPYTHON, flaky EOFError') + # TODO: RUSTPYTHON - SemLock not implemented on Windows + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @skip_if_tsan_fork + @support.requires_subprocess() + def test_multiprocessing_queues(self): + # See gh-119819 + + cd = copy.deepcopy(self.config_queue_handler) + from multiprocessing import Queue as MQ, Manager as MM + q1 = MQ() # this can't be pickled + q2 = MM().Queue() # a proxy queue for use when pickling is needed + q3 = MM().JoinableQueue() # a joinable proxy queue + for qspec in (q1, q2, q3): + fn = make_temp_file('.log', 'test_logging-cmpqh-') + cd['handlers']['h1']['filename'] = fn + cd['handlers']['ah']['queue'] = qspec + qh = None + try: + self.apply_config(cd) + qh = logging.getHandlerByName('ah') + self.assertEqual(sorted(logging.getHandlerNames()), ['ah', 'h1']) + self.assertIsNotNone(qh.listener) + self.assertIs(qh.queue, qspec) + self.assertIs(qh.listener.queue, qspec) + finally: + h = logging.getHandlerByName('h1') + if h: + self.addCleanup(closeFileHandler, h, fn) + else: + self.addCleanup(os.remove, fn) + + def test_90195(self): + # See gh-90195 + config = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + 'a': { + 'level': 'DEBUG', + 'handlers': ['console'] + } + } + } + logger = logging.getLogger('a') + self.assertFalse(logger.disabled) + self.apply_config(config) + self.assertFalse(logger.disabled) + # Should disable all loggers ... + self.apply_config({'version': 1}) + self.assertTrue(logger.disabled) + del config['disable_existing_loggers'] + self.apply_config(config) + # Logger should be enabled, since explicitly mentioned + self.assertFalse(logger.disabled) + + def test_111615(self): + # See gh-111615 + import_helper.import_module('_multiprocessing') # see gh-113692 + mp = import_helper.import_module('multiprocessing') + + config = { + 'version': 1, + 'handlers': { + 'sink': { + 'class': 'logging.handlers.QueueHandler', + 'queue': mp.get_context('spawn').Queue(), + }, + }, + 'root': { + 'handlers': ['sink'], + 'level': 'DEBUG', + }, + } + logging.config.dictConfig(config) + + # gh-118868: check if kwargs are passed to logging QueueHandler + def test_kwargs_passing(self): + class CustomQueueHandler(logging.handlers.QueueHandler): + def __init__(self, *args, **kwargs): + super().__init__(queue.Queue()) + self.custom_kwargs = kwargs + + custom_kwargs = {'foo': 'bar'} + + config = { + 'version': 1, + 'handlers': { + 'custom': { + 'class': CustomQueueHandler, + **custom_kwargs + }, + }, + 'root': { + 'level': 'DEBUG', + 'handlers': ['custom'] + } + } + + logging.config.dictConfig(config) + + handler = logging.getHandlerByName('custom') + self.assertEqual(handler.custom_kwargs, custom_kwargs) + + +class ManagerTest(BaseTest): + def test_manager_loggerclass(self): + logged = [] + + class MyLogger(logging.Logger): + def _log(self, level, msg, args, exc_info=None, extra=None): + logged.append(msg) + + man = logging.Manager(None) + self.assertRaises(TypeError, man.setLoggerClass, int) + man.setLoggerClass(MyLogger) + logger = man.getLogger('test') + logger.warning('should appear in logged') + logging.warning('should not appear in logged') + + self.assertEqual(logged, ['should appear in logged']) + + def test_set_log_record_factory(self): + man = logging.Manager(None) + expected = object() + man.setLogRecordFactory(expected) + self.assertEqual(man.logRecordFactory, expected) + +class ChildLoggerTest(BaseTest): + def test_child_loggers(self): + r = logging.getLogger() + l1 = logging.getLogger('abc') + l2 = logging.getLogger('def.ghi') + c1 = r.getChild('xyz') + c2 = r.getChild('uvw.xyz') + self.assertIs(c1, logging.getLogger('xyz')) + self.assertIs(c2, logging.getLogger('uvw.xyz')) + c1 = l1.getChild('def') + c2 = c1.getChild('ghi') + c3 = l1.getChild('def.ghi') + self.assertIs(c1, logging.getLogger('abc.def')) + self.assertIs(c2, logging.getLogger('abc.def.ghi')) + self.assertIs(c2, c3) + + def test_get_children(self): + r = logging.getLogger() + l1 = logging.getLogger('foo') + l2 = logging.getLogger('foo.bar') + l3 = logging.getLogger('foo.bar.baz.bozz') + l4 = logging.getLogger('bar') + kids = r.getChildren() + expected = {l1, l4} + self.assertEqual(expected, kids & expected) # might be other kids for root + self.assertNotIn(l2, expected) + kids = l1.getChildren() + self.assertEqual({l2}, kids) + kids = l2.getChildren() + self.assertEqual(set(), kids) + +class DerivedLogRecord(logging.LogRecord): + pass + +class LogRecordFactoryTest(BaseTest): + + def setUp(self): + class CheckingFilter(logging.Filter): + def __init__(self, cls): + self.cls = cls + + def filter(self, record): + t = type(record) + if t is not self.cls: + msg = 'Unexpected LogRecord type %s, expected %s' % (t, + self.cls) + raise TypeError(msg) + return True + + BaseTest.setUp(self) + self.filter = CheckingFilter(DerivedLogRecord) + self.root_logger.addFilter(self.filter) + self.orig_factory = logging.getLogRecordFactory() + + def tearDown(self): + self.root_logger.removeFilter(self.filter) + BaseTest.tearDown(self) + logging.setLogRecordFactory(self.orig_factory) + + def test_logrecord_class(self): + self.assertRaises(TypeError, self.root_logger.warning, + self.next_message()) + logging.setLogRecordFactory(DerivedLogRecord) + self.root_logger.error(self.next_message()) + self.assert_log_lines([ + ('root', 'ERROR', '2'), + ]) + + +@threading_helper.requires_working_threading() +class QueueHandlerTest(BaseTest): + # Do not bother with a logger name group. + expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$" + + def setUp(self): + BaseTest.setUp(self) + self.queue = queue.Queue(-1) + self.que_hdlr = logging.handlers.QueueHandler(self.queue) + self.name = 'que' + self.que_logger = logging.getLogger('que') + self.que_logger.propagate = False + self.que_logger.setLevel(logging.WARNING) + self.que_logger.addHandler(self.que_hdlr) + + def tearDown(self): + self.que_hdlr.close() + BaseTest.tearDown(self) + + def test_queue_handler(self): + self.que_logger.debug(self.next_message()) + self.assertRaises(queue.Empty, self.queue.get_nowait) + self.que_logger.info(self.next_message()) + self.assertRaises(queue.Empty, self.queue.get_nowait) + msg = self.next_message() + self.que_logger.warning(msg) + data = self.queue.get_nowait() + self.assertTrue(isinstance(data, logging.LogRecord)) + self.assertEqual(data.name, self.que_logger.name) + self.assertEqual((data.msg, data.args), (msg, None)) + + def test_formatting(self): + msg = self.next_message() + levelname = logging.getLevelName(logging.WARNING) + log_format_str = '{name} -> {levelname}: {message}' + formatted_msg = log_format_str.format(name=self.name, + levelname=levelname, message=msg) + formatter = logging.Formatter(self.log_format) + self.que_hdlr.setFormatter(formatter) + self.que_logger.warning(msg) + log_record = self.queue.get_nowait() + self.assertEqual(formatted_msg, log_record.msg) + self.assertEqual(formatted_msg, log_record.message) + + def test_queue_listener(self): + handler = TestHandler(support.Matcher()) + listener = logging.handlers.QueueListener(self.queue, handler) + listener.start() + try: + self.que_logger.warning(self.next_message()) + self.que_logger.error(self.next_message()) + self.que_logger.critical(self.next_message()) + finally: + listener.stop() + listener.stop() # gh-114706 - ensure no crash if called again + self.assertTrue(handler.matches(levelno=logging.WARNING, message='1')) + self.assertTrue(handler.matches(levelno=logging.ERROR, message='2')) + self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3')) + handler.close() + + # Now test with respect_handler_level set + + handler = TestHandler(support.Matcher()) + handler.setLevel(logging.CRITICAL) + listener = logging.handlers.QueueListener(self.queue, handler, + respect_handler_level=True) + listener.start() + try: + self.que_logger.warning(self.next_message()) + self.que_logger.error(self.next_message()) + self.que_logger.critical(self.next_message()) + finally: + listener.stop() + self.assertFalse(handler.matches(levelno=logging.WARNING, message='4')) + self.assertFalse(handler.matches(levelno=logging.ERROR, message='5')) + self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='6')) + handler.close() + + # doesn't hurt to call stop() more than once. + listener.stop() + self.assertIsNone(listener._thread) + + def test_queue_listener_multi_start(self): + handler = TestHandler(support.Matcher()) + listener = logging.handlers.QueueListener(self.queue, handler) + listener.start() + self.assertRaises(RuntimeError, listener.start) + listener.stop() + self.assertIsNone(listener._thread) + + def test_queue_listener_with_StreamHandler(self): + # Test that traceback and stack-info only appends once (bpo-34334, bpo-46755). + listener = logging.handlers.QueueListener(self.queue, self.root_hdlr) + listener.start() + try: + 1 / 0 + except ZeroDivisionError as e: + exc = e + self.que_logger.exception(self.next_message(), exc_info=exc) + self.que_logger.error(self.next_message(), stack_info=True) + listener.stop() + self.assertEqual(self.stream.getvalue().strip().count('Traceback'), 1) + self.assertEqual(self.stream.getvalue().strip().count('Stack'), 1) + + def test_queue_listener_with_multiple_handlers(self): + # Test that queue handler format doesn't affect other handler formats (bpo-35726). + self.que_hdlr.setFormatter(self.root_formatter) + self.que_logger.addHandler(self.root_hdlr) + + listener = logging.handlers.QueueListener(self.queue, self.que_hdlr) + listener.start() + self.que_logger.error("error") + listener.stop() + self.assertEqual(self.stream.getvalue().strip(), "que -> ERROR: error") + +if hasattr(logging.handlers, 'QueueListener'): + import multiprocessing + from unittest.mock import patch + + @skip_if_tsan_fork + @threading_helper.requires_working_threading() + class QueueListenerTest(BaseTest): + """ + Tests based on patch submitted for issue #27930. Ensure that + QueueListener handles all log messages. + """ + + repeat = 20 + + @staticmethod + def setup_and_log(log_queue, ident): + """ + Creates a logger with a QueueHandler that logs to a queue read by a + QueueListener. Starts the listener, logs five messages, and stops + the listener. + """ + logger = logging.getLogger('test_logger_with_id_%s' % ident) + logger.setLevel(logging.DEBUG) + handler = logging.handlers.QueueHandler(log_queue) + logger.addHandler(handler) + listener = logging.handlers.QueueListener(log_queue) + listener.start() + + logger.info('one') + logger.info('two') + logger.info('three') + logger.info('four') + logger.info('five') + + listener.stop() + logger.removeHandler(handler) + handler.close() + + @patch.object(logging.handlers.QueueListener, 'handle') + def test_handle_called_with_queue_queue(self, mock_handle): + for i in range(self.repeat): + log_queue = queue.Queue() + self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) + self.assertEqual(mock_handle.call_count, 5 * self.repeat, + 'correct number of handled log messages') + + @patch.object(logging.handlers.QueueListener, 'handle') + def test_handle_called_with_mp_queue(self, mock_handle): + # bpo-28668: The multiprocessing (mp) module is not functional + # when the mp.synchronize module cannot be imported. + support.skip_if_broken_multiprocessing_synchronize() + for i in range(self.repeat): + log_queue = multiprocessing.Queue() + self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) + log_queue.close() + log_queue.join_thread() + self.assertEqual(mock_handle.call_count, 5 * self.repeat, + 'correct number of handled log messages') + + @staticmethod + def get_all_from_queue(log_queue): + try: + while True: + yield log_queue.get_nowait() + except queue.Empty: + return [] + + def test_no_messages_in_queue_after_stop(self): + """ + Five messages are logged then the QueueListener is stopped. This + test then gets everything off the queue. Failure of this test + indicates that messages were not registered on the queue until + _after_ the QueueListener stopped. + """ + # bpo-28668: The multiprocessing (mp) module is not functional + # when the mp.synchronize module cannot be imported. + support.skip_if_broken_multiprocessing_synchronize() + for i in range(self.repeat): + queue = multiprocessing.Queue() + self.setup_and_log(queue, '%s_%s' %(self.id(), i)) + # time.sleep(1) + items = list(self.get_all_from_queue(queue)) + queue.close() + queue.join_thread() + + expected = [[], [logging.handlers.QueueListener._sentinel]] + self.assertIn(items, expected, + 'Found unexpected messages in queue: %s' % ( + [m.msg if isinstance(m, logging.LogRecord) + else m for m in items])) + + def test_calls_task_done_after_stop(self): + # Issue 36813: Make sure queue.join does not deadlock. + log_queue = queue.Queue() + listener = logging.handlers.QueueListener(log_queue) + listener.start() + listener.stop() + with self.assertRaises(ValueError): + # Make sure all tasks are done and .join won't block. + log_queue.task_done() + + +ZERO = datetime.timedelta(0) + +class UTC(datetime.tzinfo): + def utcoffset(self, dt): + return ZERO + + dst = utcoffset + + def tzname(self, dt): + return 'UTC' + +utc = UTC() + +class AssertErrorMessage: + + def assert_error_message(self, exception, message, *args, **kwargs): + try: + self.assertRaises((), *args, **kwargs) + except exception as e: + self.assertEqual(message, str(e)) + +class FormatterTest(unittest.TestCase, AssertErrorMessage): + def setUp(self): + self.common = { + 'name': 'formatter.test', + 'level': logging.DEBUG, + 'pathname': os.path.join('path', 'to', 'dummy.ext'), + 'lineno': 42, + 'exc_info': None, + 'func': None, + 'msg': 'Message with %d %s', + 'args': (2, 'placeholders'), + } + self.variants = { + 'custom': { + 'custom': 1234 + } + } + + def get_record(self, name=None): + result = dict(self.common) + if name is not None: + result.update(self.variants[name]) + return logging.makeLogRecord(result) + + def test_percent(self): + # Test %-formatting + r = self.get_record() + f = logging.Formatter('${%(message)s}') + self.assertEqual(f.format(r), '${Message with 2 placeholders}') + f = logging.Formatter('%(random)s') + self.assertRaises(ValueError, f.format, r) + self.assertFalse(f.usesTime()) + f = logging.Formatter('%(asctime)s') + self.assertTrue(f.usesTime()) + f = logging.Formatter('%(asctime)-15s') + self.assertTrue(f.usesTime()) + f = logging.Formatter('%(asctime)#15s') + self.assertTrue(f.usesTime()) + + def test_braces(self): + # Test {}-formatting + r = self.get_record() + f = logging.Formatter('$%{message}%$', style='{') + self.assertEqual(f.format(r), '$%Message with 2 placeholders%$') + f = logging.Formatter('{random}', style='{') + self.assertRaises(ValueError, f.format, r) + f = logging.Formatter("{message}", style='{') + self.assertFalse(f.usesTime()) + f = logging.Formatter('{asctime}', style='{') + self.assertTrue(f.usesTime()) + f = logging.Formatter('{asctime!s:15}', style='{') + self.assertTrue(f.usesTime()) + f = logging.Formatter('{asctime:15}', style='{') + self.assertTrue(f.usesTime()) + + def test_dollars(self): + # Test $-formatting + r = self.get_record() + f = logging.Formatter('${message}', style='$') + self.assertEqual(f.format(r), 'Message with 2 placeholders') + f = logging.Formatter('$message', style='$') + self.assertEqual(f.format(r), 'Message with 2 placeholders') + f = logging.Formatter('$$%${message}%$$', style='$') + self.assertEqual(f.format(r), '$%Message with 2 placeholders%$') + f = logging.Formatter('${random}', style='$') + self.assertRaises(ValueError, f.format, r) + self.assertFalse(f.usesTime()) + f = logging.Formatter('${asctime}', style='$') + self.assertTrue(f.usesTime()) + f = logging.Formatter('$asctime', style='$') + self.assertTrue(f.usesTime()) + f = logging.Formatter('${message}', style='$') + self.assertFalse(f.usesTime()) + f = logging.Formatter('${asctime}--', style='$') + self.assertTrue(f.usesTime()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Unexpected error parsing format string + def test_format_validate(self): + # Check correct formatting + # Percentage style + f = logging.Formatter("%(levelname)-15s - %(message) 5s - %(process)03d - %(module) - %(asctime)*.3s") + self.assertEqual(f._fmt, "%(levelname)-15s - %(message) 5s - %(process)03d - %(module) - %(asctime)*.3s") + f = logging.Formatter("%(asctime)*s - %(asctime)*.3s - %(process)-34.33o") + self.assertEqual(f._fmt, "%(asctime)*s - %(asctime)*.3s - %(process)-34.33o") + f = logging.Formatter("%(process)#+027.23X") + self.assertEqual(f._fmt, "%(process)#+027.23X") + f = logging.Formatter("%(foo)#.*g") + self.assertEqual(f._fmt, "%(foo)#.*g") + + # StrFormat Style + f = logging.Formatter("$%{message}%$ - {asctime!a:15} - {customfield['key']}", style="{") + self.assertEqual(f._fmt, "$%{message}%$ - {asctime!a:15} - {customfield['key']}") + f = logging.Formatter("{process:.2f} - {custom.f:.4f}", style="{") + self.assertEqual(f._fmt, "{process:.2f} - {custom.f:.4f}") + f = logging.Formatter("{customfield!s:#<30}", style="{") + self.assertEqual(f._fmt, "{customfield!s:#<30}") + f = logging.Formatter("{message!r}", style="{") + self.assertEqual(f._fmt, "{message!r}") + f = logging.Formatter("{message!s}", style="{") + self.assertEqual(f._fmt, "{message!s}") + f = logging.Formatter("{message!a}", style="{") + self.assertEqual(f._fmt, "{message!a}") + f = logging.Formatter("{process!r:4.2}", style="{") + self.assertEqual(f._fmt, "{process!r:4.2}") + f = logging.Formatter("{process!s:<#30,.12f}- {custom:=+#30,.1d} - {module:^30}", style="{") + self.assertEqual(f._fmt, "{process!s:<#30,.12f}- {custom:=+#30,.1d} - {module:^30}") + f = logging.Formatter("{process!s:{w},.{p}}", style="{") + self.assertEqual(f._fmt, "{process!s:{w},.{p}}") + f = logging.Formatter("{foo:12.{p}}", style="{") + self.assertEqual(f._fmt, "{foo:12.{p}}") + f = logging.Formatter("{foo:{w}.6}", style="{") + self.assertEqual(f._fmt, "{foo:{w}.6}") + f = logging.Formatter("{foo[0].bar[1].baz}", style="{") + self.assertEqual(f._fmt, "{foo[0].bar[1].baz}") + f = logging.Formatter("{foo[k1].bar[k2].baz}", style="{") + self.assertEqual(f._fmt, "{foo[k1].bar[k2].baz}") + f = logging.Formatter("{12[k1].bar[k2].baz}", style="{") + self.assertEqual(f._fmt, "{12[k1].bar[k2].baz}") + + # Dollar style + f = logging.Formatter("${asctime} - $message", style="$") + self.assertEqual(f._fmt, "${asctime} - $message") + f = logging.Formatter("$bar $$", style="$") + self.assertEqual(f._fmt, "$bar $$") + f = logging.Formatter("$bar $$$$", style="$") + self.assertEqual(f._fmt, "$bar $$$$") # this would print two $($$) + + # Testing when ValueError being raised from incorrect format + # Percentage Style + self.assertRaises(ValueError, logging.Formatter, "%(asctime)Z") + self.assertRaises(ValueError, logging.Formatter, "%(asctime)b") + self.assertRaises(ValueError, logging.Formatter, "%(asctime)*") + self.assertRaises(ValueError, logging.Formatter, "%(asctime)*3s") + self.assertRaises(ValueError, logging.Formatter, "%(asctime)_") + self.assertRaises(ValueError, logging.Formatter, '{asctime}') + self.assertRaises(ValueError, logging.Formatter, '${message}') + self.assertRaises(ValueError, logging.Formatter, '%(foo)#12.3*f') # with both * and decimal number as precision + self.assertRaises(ValueError, logging.Formatter, '%(foo)0*.8*f') + + # StrFormat Style + # Testing failure for '-' in field name + self.assert_error_message( + ValueError, + "invalid format: invalid field name/expression: 'name-thing'", + logging.Formatter, "{name-thing}", style="{" + ) + # Testing failure for style mismatch + self.assert_error_message( + ValueError, + "invalid format: no fields", + logging.Formatter, '%(asctime)s', style='{' + ) + # Testing failure for invalid conversion + self.assert_error_message( + ValueError, + "invalid conversion: 'Z'" + ) + self.assertRaises(ValueError, logging.Formatter, '{asctime!s:#30,15f}', style='{') + self.assert_error_message( + ValueError, + "invalid format: expected ':' after conversion specifier", + logging.Formatter, '{asctime!aa:15}', style='{' + ) + # Testing failure for invalid spec + self.assert_error_message( + ValueError, + "invalid format: bad specifier: '.2ff'", + logging.Formatter, '{process:.2ff}', style='{' + ) + self.assertRaises(ValueError, logging.Formatter, '{process:.2Z}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{process!s:<##30,12g}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{process!s:<#30#,12g}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{process!s:{{w}},{{p}}}', style='{') + # Testing failure for mismatch braces + self.assert_error_message( + ValueError, + "invalid format: expected '}' before end of string", + logging.Formatter, '{process', style='{' + ) + self.assert_error_message( + ValueError, + "invalid format: Single '}' encountered in format string", + logging.Formatter, 'process}', style='{' + ) + self.assertRaises(ValueError, logging.Formatter, '{{foo!r:4.2}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{{foo!r:4.2}}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{foo/bar}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{foo:{{w}}.{{p}}}}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{foo!X:{{w}}.{{p}}}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{foo!a:random}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{foo!a:ran{dom}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{foo!a:ran{d}om}', style='{') + self.assertRaises(ValueError, logging.Formatter, '{foo.!a:d}', style='{') + + # Dollar style + # Testing failure for mismatch bare $ + self.assert_error_message( + ValueError, + "invalid format: bare \'$\' not allowed", + logging.Formatter, '$bar $$$', style='$' + ) + self.assert_error_message( + ValueError, + "invalid format: bare \'$\' not allowed", + logging.Formatter, 'bar $', style='$' + ) + self.assert_error_message( + ValueError, + "invalid format: bare \'$\' not allowed", + logging.Formatter, 'foo $.', style='$' + ) + # Testing failure for mismatch style + self.assert_error_message( + ValueError, + "invalid format: no fields", + logging.Formatter, '{asctime}', style='$' + ) + self.assertRaises(ValueError, logging.Formatter, '%(asctime)s', style='$') + + # Testing failure for incorrect fields + self.assert_error_message( + ValueError, + "invalid format: no fields", + logging.Formatter, 'foo', style='$' + ) + self.assertRaises(ValueError, logging.Formatter, '${asctime', style='$') + + def test_defaults_parameter(self): + fmts = ['%(custom)s %(message)s', '{custom} {message}', '$custom $message'] + styles = ['%', '{', '$'] + for fmt, style in zip(fmts, styles): + f = logging.Formatter(fmt, style=style, defaults={'custom': 'Default'}) + r = self.get_record() + self.assertEqual(f.format(r), 'Default Message with 2 placeholders') + r = self.get_record("custom") + self.assertEqual(f.format(r), '1234 Message with 2 placeholders') + + # Without default + f = logging.Formatter(fmt, style=style) + r = self.get_record() + self.assertRaises(ValueError, f.format, r) + + # Non-existing default is ignored + f = logging.Formatter(fmt, style=style, defaults={'Non-existing': 'Default'}) + r = self.get_record("custom") + self.assertEqual(f.format(r), '1234 Message with 2 placeholders') + + def test_invalid_style(self): + self.assertRaises(ValueError, logging.Formatter, None, None, 'x') + + def test_time(self): + r = self.get_record() + dt = datetime.datetime(1993, 4, 21, 8, 3, 0, 0, utc) + # We use None to indicate we want the local timezone + # We're essentially converting a UTC time to local time + r.created = time.mktime(dt.astimezone(None).timetuple()) + r.msecs = 123 + f = logging.Formatter('%(asctime)s %(message)s') + f.converter = time.gmtime + self.assertEqual(f.formatTime(r), '1993-04-21 08:03:00,123') + self.assertEqual(f.formatTime(r, '%Y:%d'), '1993:21') + f.format(r) + self.assertEqual(r.asctime, '1993-04-21 08:03:00,123') + + def test_default_msec_format_none(self): + class NoMsecFormatter(logging.Formatter): + default_msec_format = None + default_time_format = '%d/%m/%Y %H:%M:%S' + + r = self.get_record() + dt = datetime.datetime(1993, 4, 21, 8, 3, 0, 123, utc) + r.created = time.mktime(dt.astimezone(None).timetuple()) + f = NoMsecFormatter() + f.converter = time.gmtime + self.assertEqual(f.formatTime(r), '21/04/1993 08:03:00') + + def test_issue_89047(self): + f = logging.Formatter(fmt='{asctime}.{msecs:03.0f} {message}', style='{', datefmt="%Y-%m-%d %H:%M:%S") + for i in range(2500): + time.sleep(0.0004) + r = logging.makeLogRecord({'msg': 'Message %d' % (i + 1)}) + s = f.format(r) + self.assertNotIn('.1000', s) + + def test_msecs_has_no_floating_point_precision_loss(self): + # See issue gh-102402 + tests = ( + # time_ns is approx. 2023-03-04 04:25:20 UTC + # (time_ns, expected_msecs_value) + (1_677_902_297_100_000_000, 100.0), # exactly 100ms + (1_677_903_920_999_998_503, 999.0), # check truncating doesn't round + (1_677_903_920_000_998_503, 0.0), # check truncating doesn't round + (1_677_903_920_999_999_900, 0.0), # check rounding up + ) + for ns, want in tests: + with patch('time.time_ns') as patched_ns: + patched_ns.return_value = ns + record = logging.makeLogRecord({'msg': 'test'}) + with self.subTest(ns): + self.assertEqual(record.msecs, want) + self.assertEqual(record.created, ns / 1e9) + self.assertAlmostEqual(record.created - int(record.created), + record.msecs / 1e3, + delta=1e-3) + + def test_relativeCreated_has_higher_precision(self): + # See issue gh-102402. + # Run the code in the subprocess, because the time module should + # be patched before the first import of the logging package. + # Temporary unloading and re-importing the logging package has + # side effects (including registering the atexit callback and + # references leak). + start_ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC + offsets_ns = (200, 500, 12_354, 99_999, 1_677_903_456_999_123_456) + code = textwrap.dedent(f""" + start_ns = {start_ns!r} + offsets_ns = {offsets_ns!r} + start_monotonic_ns = start_ns - 1 + + import time + # Only time.time_ns needs to be patched for the current + # implementation, but patch also other functions to make + # the test less implementation depending. + old_time_ns = time.time_ns + old_time = time.time + old_monotonic_ns = time.monotonic_ns + old_monotonic = time.monotonic + time_ns_result = start_ns + time.time_ns = lambda: time_ns_result + time.time = lambda: time.time_ns()/1e9 + time.monotonic_ns = lambda: time_ns_result - start_monotonic_ns + time.monotonic = lambda: time.monotonic_ns()/1e9 + try: + import logging + + for offset_ns in offsets_ns: + # mock for log record creation + time_ns_result = start_ns + offset_ns + record = logging.makeLogRecord({{'msg': 'test'}}) + print(record.created, record.relativeCreated) + finally: + time.time_ns = old_time_ns + time.time = old_time + time.monotonic_ns = old_monotonic_ns + time.monotonic = old_monotonic + """) + rc, out, err = assert_python_ok("-c", code) + out = out.decode() + for offset_ns, line in zip(offsets_ns, out.splitlines(), strict=True): + with self.subTest(offset_ns=offset_ns): + created, relativeCreated = map(float, line.split()) + self.assertAlmostEqual(created, (start_ns + offset_ns) / 1e9, places=6) + # After PR gh-102412, precision (places) increases from 3 to 7 + self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) + + +class TestBufferingFormatter(logging.BufferingFormatter): + def formatHeader(self, records): + return '[(%d)' % len(records) + + def formatFooter(self, records): + return '(%d)]' % len(records) + +class BufferingFormatterTest(unittest.TestCase): + def setUp(self): + self.records = [ + logging.makeLogRecord({'msg': 'one'}), + logging.makeLogRecord({'msg': 'two'}), + ] + + def test_default(self): + f = logging.BufferingFormatter() + self.assertEqual('', f.format([])) + self.assertEqual('onetwo', f.format(self.records)) + + def test_custom(self): + f = TestBufferingFormatter() + self.assertEqual('[(2)onetwo(2)]', f.format(self.records)) + lf = logging.Formatter('<%(message)s>') + f = TestBufferingFormatter(lf) + self.assertEqual('[(2)<one><two>(2)]', f.format(self.records)) + +class ExceptionTest(BaseTest): + def test_formatting(self): + r = self.root_logger + h = RecordingHandler() + r.addHandler(h) + try: + raise RuntimeError('deliberate mistake') + except: + logging.exception('failed', stack_info=True) + r.removeHandler(h) + h.close() + r = h.records[0] + self.assertTrue(r.exc_text.startswith('Traceback (most recent ' + 'call last):\n')) + self.assertTrue(r.exc_text.endswith('\nRuntimeError: ' + 'deliberate mistake')) + self.assertTrue(r.stack_info.startswith('Stack (most recent ' + 'call last):\n')) + self.assertTrue(r.stack_info.endswith('logging.exception(\'failed\', ' + 'stack_info=True)')) + + +class LastResortTest(BaseTest): + def test_last_resort(self): + # Test the last resort handler + root = self.root_logger + root.removeHandler(self.root_hdlr) + old_lastresort = logging.lastResort + old_raise_exceptions = logging.raiseExceptions + + try: + with support.captured_stderr() as stderr: + root.debug('This should not appear') + self.assertEqual(stderr.getvalue(), '') + root.warning('Final chance!') + self.assertEqual(stderr.getvalue(), 'Final chance!\n') + + # No handlers and no last resort, so 'No handlers' message + logging.lastResort = None + with support.captured_stderr() as stderr: + root.warning('Final chance!') + msg = 'No handlers could be found for logger "root"\n' + self.assertEqual(stderr.getvalue(), msg) + + # 'No handlers' message only printed once + with support.captured_stderr() as stderr: + root.warning('Final chance!') + self.assertEqual(stderr.getvalue(), '') + + # If raiseExceptions is False, no message is printed + root.manager.emittedNoHandlerWarning = False + logging.raiseExceptions = False + with support.captured_stderr() as stderr: + root.warning('Final chance!') + self.assertEqual(stderr.getvalue(), '') + finally: + root.addHandler(self.root_hdlr) + logging.lastResort = old_lastresort + logging.raiseExceptions = old_raise_exceptions + + +class FakeHandler: + + def __init__(self, identifier, called): + for method in ('acquire', 'flush', 'close', 'release'): + setattr(self, method, self.record_call(identifier, method, called)) + + def record_call(self, identifier, method_name, called): + def inner(): + called.append('{} - {}'.format(identifier, method_name)) + return inner + + +class RecordingHandler(logging.NullHandler): + + def __init__(self, *args, **kwargs): + super(RecordingHandler, self).__init__(*args, **kwargs) + self.records = [] + + def handle(self, record): + """Keep track of all the emitted records.""" + self.records.append(record) + + +class ShutdownTest(BaseTest): + + """Test suite for the shutdown method.""" + + def setUp(self): + super(ShutdownTest, self).setUp() + self.called = [] + + raise_exceptions = logging.raiseExceptions + self.addCleanup(setattr, logging, 'raiseExceptions', raise_exceptions) + + def raise_error(self, error): + def inner(): + raise error() + return inner + + def test_no_failure(self): + # create some fake handlers + handler0 = FakeHandler(0, self.called) + handler1 = FakeHandler(1, self.called) + handler2 = FakeHandler(2, self.called) + + # create live weakref to those handlers + handlers = map(logging.weakref.ref, [handler0, handler1, handler2]) + + logging.shutdown(handlerList=list(handlers)) + + expected = ['2 - acquire', '2 - flush', '2 - close', '2 - release', + '1 - acquire', '1 - flush', '1 - close', '1 - release', + '0 - acquire', '0 - flush', '0 - close', '0 - release'] + self.assertEqual(expected, self.called) + + def _test_with_failure_in_method(self, method, error): + handler = FakeHandler(0, self.called) + setattr(handler, method, self.raise_error(error)) + handlers = [logging.weakref.ref(handler)] + + logging.shutdown(handlerList=list(handlers)) + + self.assertEqual('0 - release', self.called[-1]) + + def test_with_ioerror_in_acquire(self): + self._test_with_failure_in_method('acquire', OSError) + + def test_with_ioerror_in_flush(self): + self._test_with_failure_in_method('flush', OSError) + + def test_with_ioerror_in_close(self): + self._test_with_failure_in_method('close', OSError) + + def test_with_valueerror_in_acquire(self): + self._test_with_failure_in_method('acquire', ValueError) + + def test_with_valueerror_in_flush(self): + self._test_with_failure_in_method('flush', ValueError) + + def test_with_valueerror_in_close(self): + self._test_with_failure_in_method('close', ValueError) + + def test_with_other_error_in_acquire_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('acquire', IndexError) + + def test_with_other_error_in_flush_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('flush', IndexError) + + def test_with_other_error_in_close_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('close', IndexError) + + def test_with_other_error_in_acquire_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'acquire', IndexError) + + def test_with_other_error_in_flush_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'flush', IndexError) + + def test_with_other_error_in_close_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'close', IndexError) + + +class ModuleLevelMiscTest(BaseTest): + + """Test suite for some module level methods.""" + + def test_disable(self): + old_disable = logging.root.manager.disable + # confirm our assumptions are correct + self.assertEqual(old_disable, 0) + self.addCleanup(logging.disable, old_disable) + + logging.disable(83) + self.assertEqual(logging.root.manager.disable, 83) + + self.assertRaises(ValueError, logging.disable, "doesnotexists") + + class _NotAnIntOrString: + pass + + self.assertRaises(TypeError, logging.disable, _NotAnIntOrString()) + + logging.disable("WARN") + + # test the default value introduced in 3.7 + # (Issue #28524) + logging.disable() + self.assertEqual(logging.root.manager.disable, logging.CRITICAL) + + def _test_log(self, method, level=None): + called = [] + support.patch(self, logging, 'basicConfig', + lambda *a, **kw: called.append((a, kw))) + + recording = RecordingHandler() + logging.root.addHandler(recording) + + log_method = getattr(logging, method) + if level is not None: + log_method(level, "test me: %r", recording) + else: + log_method("test me: %r", recording) + + self.assertEqual(len(recording.records), 1) + record = recording.records[0] + self.assertEqual(record.getMessage(), "test me: %r" % recording) + + expected_level = level if level is not None else getattr(logging, method.upper()) + self.assertEqual(record.levelno, expected_level) + + # basicConfig was not called! + self.assertEqual(called, []) + + def test_log(self): + self._test_log('log', logging.ERROR) + + def test_debug(self): + self._test_log('debug') + + def test_info(self): + self._test_log('info') + + def test_warning(self): + self._test_log('warning') + + def test_error(self): + self._test_log('error') + + def test_critical(self): + self._test_log('critical') + + def test_set_logger_class(self): + self.assertRaises(TypeError, logging.setLoggerClass, object) + + class MyLogger(logging.Logger): + pass + + logging.setLoggerClass(MyLogger) + self.assertEqual(logging.getLoggerClass(), MyLogger) + + logging.setLoggerClass(logging.Logger) + self.assertEqual(logging.getLoggerClass(), logging.Logger) + + def test_subclass_logger_cache(self): + # bpo-37258 + message = [] + + class MyLogger(logging.getLoggerClass()): + def __init__(self, name='MyLogger', level=logging.NOTSET): + super().__init__(name, level) + message.append('initialized') + + logging.setLoggerClass(MyLogger) + logger = logging.getLogger('just_some_logger') + self.assertEqual(message, ['initialized']) + stream = io.StringIO() + h = logging.StreamHandler(stream) + logger.addHandler(h) + try: + logger.setLevel(logging.DEBUG) + logger.debug("hello") + self.assertEqual(stream.getvalue().strip(), "hello") + + stream.truncate(0) + stream.seek(0) + + logger.setLevel(logging.INFO) + logger.debug("hello") + self.assertEqual(stream.getvalue(), "") + finally: + logger.removeHandler(h) + h.close() + logging.setLoggerClass(logging.Logger) + + def test_logging_at_shutdown(self): + # bpo-20037: Doing text I/O late at interpreter shutdown must not crash + code = textwrap.dedent(""" + import logging + + class A: + def __del__(self): + try: + raise ValueError("some error") + except Exception: + logging.exception("exception in __del__") + + a = A() + """) + rc, out, err = assert_python_ok("-c", code) + err = err.decode() + self.assertIn("exception in __del__", err) + self.assertIn("ValueError: some error", err) + + def test_logging_at_shutdown_open(self): + # bpo-26789: FileHandler keeps a reference to the builtin open() + # function to be able to open or reopen the file during Python + # finalization. + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + code = textwrap.dedent(f""" + import builtins + import logging + + class A: + def __del__(self): + logging.error("log in __del__") + + # basicConfig() opens the file, but logging.shutdown() closes + # it at Python exit. When A.__del__() is called, + # FileHandler._open() must be called again to re-open the file. + logging.basicConfig(filename={filename!r}, encoding="utf-8") + + a = A() + + # Simulate the Python finalization which removes the builtin + # open() function. + del builtins.open + """) + assert_python_ok("-c", code) + + with open(filename, encoding="utf-8") as fp: + self.assertEqual(fp.read().rstrip(), "ERROR:root:log in __del__") + + def test_recursion_error(self): + # Issue 36272 + code = textwrap.dedent(""" + import logging + + def rec(): + logging.error("foo") + rec() + + rec() + """) + rc, out, err = assert_python_failure("-c", code) + err = err.decode() + self.assertNotIn("Cannot recover from stack overflow.", err) + self.assertEqual(rc, 1) + + def test_get_level_names_mapping(self): + mapping = logging.getLevelNamesMapping() + self.assertEqual(logging._nameToLevel, mapping) # value is equivalent + self.assertIsNot(logging._nameToLevel, mapping) # but not the internal data + new_mapping = logging.getLevelNamesMapping() # another call -> another copy + self.assertIsNot(mapping, new_mapping) # verify not the same object as before + self.assertEqual(mapping, new_mapping) # but equivalent in value + + +class LogRecordTest(BaseTest): + def test_str_rep(self): + r = logging.makeLogRecord({}) + s = str(r) + self.assertTrue(s.startswith('<LogRecord: ')) + self.assertTrue(s.endswith('>')) + + def test_dict_arg(self): + h = RecordingHandler() + r = logging.getLogger() + r.addHandler(h) + d = {'less' : 'more' } + logging.warning('less is %(less)s', d) + self.assertIs(h.records[0].args, d) + self.assertEqual(h.records[0].message, 'less is more') + r.removeHandler(h) + h.close() + + @staticmethod # pickled as target of child process in the following test + def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): + prev_logMultiprocessing = logging.logMultiprocessing + logging.logMultiprocessing = logMultiprocessing + try: + import multiprocessing as mp + name = mp.current_process().name + + r1 = logging.makeLogRecord({'msg': f'msg1_{key}'}) + + # https://bugs.python.org/issue45128 + with support.swap_item(sys.modules, 'multiprocessing', None): + r2 = logging.makeLogRecord({'msg': f'msg2_{key}'}) + + results = {'processName' : name, + 'r1.processName': r1.processName, + 'r2.processName': r2.processName, + } + finally: + logging.logMultiprocessing = prev_logMultiprocessing + if conn: + conn.send(results) + else: + return results + + @skip_if_tsan_fork + def test_multiprocessing(self): + support.skip_if_broken_multiprocessing_synchronize() + multiprocessing_imported = 'multiprocessing' in sys.modules + try: + # logMultiprocessing is True by default + self.assertEqual(logging.logMultiprocessing, True) + + LOG_MULTI_PROCESSING = True + # When logMultiprocessing == True: + # In the main process processName = 'MainProcess' + r = logging.makeLogRecord({}) + self.assertEqual(r.processName, 'MainProcess') + + results = self._extract_logrecord_process_name(1, LOG_MULTI_PROCESSING) + self.assertEqual('MainProcess', results['processName']) + self.assertEqual('MainProcess', results['r1.processName']) + self.assertEqual('MainProcess', results['r2.processName']) + + # In other processes, processName is correct when multiprocessing in imported, + # but it is (incorrectly) defaulted to 'MainProcess' otherwise (bpo-38762). + import multiprocessing + parent_conn, child_conn = multiprocessing.Pipe() + p = multiprocessing.Process( + target=self._extract_logrecord_process_name, + args=(2, LOG_MULTI_PROCESSING, child_conn,) + ) + p.start() + results = parent_conn.recv() + self.assertNotEqual('MainProcess', results['processName']) + self.assertEqual(results['processName'], results['r1.processName']) + self.assertEqual('MainProcess', results['r2.processName']) + p.join() + + finally: + if multiprocessing_imported: + import multiprocessing + + def test_optional(self): + NONE = self.assertIsNone + NOT_NONE = self.assertIsNotNone + + r = logging.makeLogRecord({}) + NOT_NONE(r.thread) + NOT_NONE(r.threadName) + NOT_NONE(r.process) + NOT_NONE(r.processName) + NONE(r.taskName) + log_threads = logging.logThreads + log_processes = logging.logProcesses + log_multiprocessing = logging.logMultiprocessing + log_asyncio_tasks = logging.logAsyncioTasks + try: + logging.logThreads = False + logging.logProcesses = False + logging.logMultiprocessing = False + logging.logAsyncioTasks = False + r = logging.makeLogRecord({}) + + NONE(r.thread) + NONE(r.threadName) + NONE(r.process) + NONE(r.processName) + NONE(r.taskName) + finally: + logging.logThreads = log_threads + logging.logProcesses = log_processes + logging.logMultiprocessing = log_multiprocessing + logging.logAsyncioTasks = log_asyncio_tasks + + async def _make_record_async(self, assertion): + r = logging.makeLogRecord({}) + assertion(r.taskName) + + @support.requires_working_socket() + def test_taskName_with_asyncio_imported(self): + try: + make_record = self._make_record_async + with asyncio.Runner() as runner: + logging.logAsyncioTasks = True + runner.run(make_record(self.assertIsNotNone)) + logging.logAsyncioTasks = False + runner.run(make_record(self.assertIsNone)) + finally: + asyncio.set_event_loop_policy(None) + + @support.requires_working_socket() + def test_taskName_without_asyncio_imported(self): + try: + make_record = self._make_record_async + with asyncio.Runner() as runner, support.swap_item(sys.modules, 'asyncio', None): + logging.logAsyncioTasks = True + runner.run(make_record(self.assertIsNone)) + logging.logAsyncioTasks = False + runner.run(make_record(self.assertIsNone)) + finally: + asyncio.set_event_loop_policy(None) + + +class BasicConfigTest(unittest.TestCase): + + """Test suite for logging.basicConfig.""" + + def setUp(self): + super(BasicConfigTest, self).setUp() + self.handlers = logging.root.handlers + self.saved_handlers = logging._handlers.copy() + self.saved_handler_list = logging._handlerList[:] + self.original_logging_level = logging.root.level + self.addCleanup(self.cleanup) + logging.root.handlers = [] + + def tearDown(self): + for h in logging.root.handlers[:]: + logging.root.removeHandler(h) + h.close() + super(BasicConfigTest, self).tearDown() + + def cleanup(self): + setattr(logging.root, 'handlers', self.handlers) + logging._handlers.clear() + logging._handlers.update(self.saved_handlers) + logging._handlerList[:] = self.saved_handler_list + logging.root.setLevel(self.original_logging_level) + + def test_no_kwargs(self): + logging.basicConfig() + + # handler defaults to a StreamHandler to sys.stderr + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.StreamHandler) + self.assertEqual(handler.stream, sys.stderr) + + formatter = handler.formatter + # format defaults to logging.BASIC_FORMAT + self.assertEqual(formatter._style._fmt, logging.BASIC_FORMAT) + # datefmt defaults to None + self.assertIsNone(formatter.datefmt) + # style defaults to % + self.assertIsInstance(formatter._style, logging.PercentStyle) + + # level is not explicitly set + self.assertEqual(logging.root.level, self.original_logging_level) + + def test_strformatstyle(self): + with support.captured_stdout() as output: + logging.basicConfig(stream=sys.stdout, style="{") + logging.error("Log an error") + sys.stdout.seek(0) + self.assertEqual(output.getvalue().strip(), + "ERROR:root:Log an error") + + def test_stringtemplatestyle(self): + with support.captured_stdout() as output: + logging.basicConfig(stream=sys.stdout, style="$") + logging.error("Log an error") + sys.stdout.seek(0) + self.assertEqual(output.getvalue().strip(), + "ERROR:root:Log an error") + + def test_filename(self): + + def cleanup(h1, h2, fn): + h1.close() + h2.close() + os.remove(fn) + + logging.basicConfig(filename='test.log', encoding='utf-8') + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.FileHandler) + + expected = logging.FileHandler('test.log', 'a', encoding='utf-8') + self.assertEqual(handler.stream.mode, expected.stream.mode) + self.assertEqual(handler.stream.name, expected.stream.name) + self.addCleanup(cleanup, handler, expected, 'test.log') + + def test_filemode(self): + + def cleanup(h1, h2, fn): + h1.close() + h2.close() + os.remove(fn) + + logging.basicConfig(filename='test.log', filemode='wb') + + handler = logging.root.handlers[0] + expected = logging.FileHandler('test.log', 'wb') + self.assertEqual(handler.stream.mode, expected.stream.mode) + self.addCleanup(cleanup, handler, expected, 'test.log') + + def test_stream(self): + stream = io.StringIO() + self.addCleanup(stream.close) + logging.basicConfig(stream=stream) + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.StreamHandler) + self.assertEqual(handler.stream, stream) + + def test_format(self): + logging.basicConfig(format='%(asctime)s - %(message)s') + + formatter = logging.root.handlers[0].formatter + self.assertEqual(formatter._style._fmt, '%(asctime)s - %(message)s') + + def test_datefmt(self): + logging.basicConfig(datefmt='bar') + + formatter = logging.root.handlers[0].formatter + self.assertEqual(formatter.datefmt, 'bar') + + def test_style(self): + logging.basicConfig(style='$') + + formatter = logging.root.handlers[0].formatter + self.assertIsInstance(formatter._style, logging.StringTemplateStyle) + + def test_level(self): + old_level = logging.root.level + self.addCleanup(logging.root.setLevel, old_level) + + logging.basicConfig(level=57) + self.assertEqual(logging.root.level, 57) + # Test that second call has no effect + logging.basicConfig(level=58) + self.assertEqual(logging.root.level, 57) + + def test_incompatible(self): + assertRaises = self.assertRaises + handlers = [logging.StreamHandler()] + stream = sys.stderr + assertRaises(ValueError, logging.basicConfig, filename='test.log', + stream=stream) + assertRaises(ValueError, logging.basicConfig, filename='test.log', + handlers=handlers) + assertRaises(ValueError, logging.basicConfig, stream=stream, + handlers=handlers) + # Issue 23207: test for invalid kwargs + assertRaises(ValueError, logging.basicConfig, loglevel=logging.INFO) + # Should pop both filename and filemode even if filename is None + logging.basicConfig(filename=None, filemode='a') + + def test_handlers(self): + handlers = [ + logging.StreamHandler(), + logging.StreamHandler(sys.stdout), + logging.StreamHandler(), + ] + f = logging.Formatter() + handlers[2].setFormatter(f) + logging.basicConfig(handlers=handlers) + self.assertIs(handlers[0], logging.root.handlers[0]) + self.assertIs(handlers[1], logging.root.handlers[1]) + self.assertIs(handlers[2], logging.root.handlers[2]) + self.assertIsNotNone(handlers[0].formatter) + self.assertIsNotNone(handlers[1].formatter) + self.assertIs(handlers[2].formatter, f) + self.assertIs(handlers[0].formatter, handlers[1].formatter) + + def test_force(self): + old_string_io = io.StringIO() + new_string_io = io.StringIO() + old_handlers = [logging.StreamHandler(old_string_io)] + new_handlers = [logging.StreamHandler(new_string_io)] + logging.basicConfig(level=logging.WARNING, handlers=old_handlers) + logging.warning('warn') + logging.info('info') + logging.debug('debug') + self.assertEqual(len(logging.root.handlers), 1) + logging.basicConfig(level=logging.INFO, handlers=new_handlers, + force=True) + logging.warning('warn') + logging.info('info') + logging.debug('debug') + self.assertEqual(len(logging.root.handlers), 1) + self.assertEqual(old_string_io.getvalue().strip(), + 'WARNING:root:warn') + self.assertEqual(new_string_io.getvalue().strip(), + 'WARNING:root:warn\nINFO:root:info') + + def test_encoding(self): + try: + encoding = 'utf-8' + logging.basicConfig(filename='test.log', encoding=encoding, + errors='strict', + format='%(message)s', level=logging.DEBUG) + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.FileHandler) + self.assertEqual(handler.encoding, encoding) + logging.debug('The Øresund Bridge joins Copenhagen to Malmö') + finally: + handler.close() + with open('test.log', encoding='utf-8') as f: + data = f.read().strip() + os.remove('test.log') + self.assertEqual(data, + 'The Øresund Bridge joins Copenhagen to Malmö') + + def test_encoding_errors(self): + try: + encoding = 'ascii' + logging.basicConfig(filename='test.log', encoding=encoding, + errors='ignore', + format='%(message)s', level=logging.DEBUG) + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.FileHandler) + self.assertEqual(handler.encoding, encoding) + logging.debug('The Øresund Bridge joins Copenhagen to Malmö') + finally: + handler.close() + with open('test.log', encoding='utf-8') as f: + data = f.read().strip() + os.remove('test.log') + self.assertEqual(data, 'The resund Bridge joins Copenhagen to Malm') + + def test_encoding_errors_default(self): + try: + encoding = 'ascii' + logging.basicConfig(filename='test.log', encoding=encoding, + format='%(message)s', level=logging.DEBUG) + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.FileHandler) + self.assertEqual(handler.encoding, encoding) + self.assertEqual(handler.errors, 'backslashreplace') + logging.debug('😂: ☃️: The Øresund Bridge joins Copenhagen to Malmö') + finally: + handler.close() + with open('test.log', encoding='utf-8') as f: + data = f.read().strip() + os.remove('test.log') + self.assertEqual(data, r'\U0001f602: \u2603\ufe0f: The \xd8resund ' + r'Bridge joins Copenhagen to Malm\xf6') + + def test_encoding_errors_none(self): + # Specifying None should behave as 'strict' + try: + encoding = 'ascii' + logging.basicConfig(filename='test.log', encoding=encoding, + errors=None, + format='%(message)s', level=logging.DEBUG) + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.FileHandler) + self.assertEqual(handler.encoding, encoding) + self.assertIsNone(handler.errors) + + message = [] + + def dummy_handle_error(record): + message.append(str(sys.exception())) + + handler.handleError = dummy_handle_error + logging.debug('The Øresund Bridge joins Copenhagen to Malmö') + self.assertTrue(message) + self.assertIn("'ascii' codec can't encode " + "character '\\xd8' in position 4:", message[0]) + finally: + handler.close() + with open('test.log', encoding='utf-8') as f: + data = f.read().strip() + os.remove('test.log') + # didn't write anything due to the encoding error + self.assertEqual(data, r'') + + @support.requires_working_socket() + def test_log_taskName(self): + async def log_record(): + logging.warning('hello world') + + handler = None + log_filename = make_temp_file('.log', 'test-logging-taskname-') + self.addCleanup(os.remove, log_filename) + try: + encoding = 'utf-8' + logging.basicConfig(filename=log_filename, errors='strict', + encoding=encoding, level=logging.WARNING, + format='%(taskName)s - %(message)s') + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.FileHandler) + + with asyncio.Runner(debug=True) as runner: + logging.logAsyncioTasks = True + runner.run(log_record()) + with open(log_filename, encoding='utf-8') as f: + data = f.read().strip() + self.assertRegex(data, r'Task-\d+ - hello world') + finally: + asyncio.set_event_loop_policy(None) + if handler: + handler.close() + + + def _test_log(self, method, level=None): + # logging.root has no handlers so basicConfig should be called + called = [] + + old_basic_config = logging.basicConfig + def my_basic_config(*a, **kw): + old_basic_config() + old_level = logging.root.level + logging.root.setLevel(100) # avoid having messages in stderr + self.addCleanup(logging.root.setLevel, old_level) + called.append((a, kw)) + + support.patch(self, logging, 'basicConfig', my_basic_config) + + log_method = getattr(logging, method) + if level is not None: + log_method(level, "test me") + else: + log_method("test me") + + # basicConfig was called with no arguments + self.assertEqual(called, [((), {})]) + + def test_log(self): + self._test_log('log', logging.WARNING) + + def test_debug(self): + self._test_log('debug') + + def test_info(self): + self._test_log('info') + + def test_warning(self): + self._test_log('warning') + + def test_error(self): + self._test_log('error') + + def test_critical(self): + self._test_log('critical') + + +class LoggerAdapterTest(unittest.TestCase): + def setUp(self): + super(LoggerAdapterTest, self).setUp() + old_handler_list = logging._handlerList[:] + + self.recording = RecordingHandler() + self.logger = logging.root + self.logger.addHandler(self.recording) + self.addCleanup(self.logger.removeHandler, self.recording) + self.addCleanup(self.recording.close) + + def cleanup(): + logging._handlerList[:] = old_handler_list + + self.addCleanup(cleanup) + self.addCleanup(logging.shutdown) + self.adapter = logging.LoggerAdapter(logger=self.logger, extra=None) + + def test_exception(self): + msg = 'testing exception: %r' + exc = None + try: + 1 / 0 + except ZeroDivisionError as e: + exc = e + self.adapter.exception(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.ERROR) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.exc_info, + (exc.__class__, exc, exc.__traceback__)) + + def test_exception_excinfo(self): + try: + 1 / 0 + except ZeroDivisionError as e: + exc = e + + self.adapter.exception('exc_info test', exc_info=exc) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.exc_info, + (exc.__class__, exc, exc.__traceback__)) + + def test_critical(self): + msg = 'critical test! %r' + self.adapter.critical(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.CRITICAL) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.funcName, 'test_critical') + + def test_is_enabled_for(self): + old_disable = self.adapter.logger.manager.disable + self.adapter.logger.manager.disable = 33 + self.addCleanup(setattr, self.adapter.logger.manager, 'disable', + old_disable) + self.assertFalse(self.adapter.isEnabledFor(32)) + + def test_has_handlers(self): + self.assertTrue(self.adapter.hasHandlers()) + + for handler in self.logger.handlers: + self.logger.removeHandler(handler) + + self.assertFalse(self.logger.hasHandlers()) + self.assertFalse(self.adapter.hasHandlers()) + + def test_nested(self): + msg = 'Adapters can be nested, yo.' + adapter = PrefixAdapter(logger=self.logger, extra=None) + adapter_adapter = PrefixAdapter(logger=adapter, extra=None) + adapter_adapter.prefix = 'AdapterAdapter' + self.assertEqual(repr(adapter), repr(adapter_adapter)) + adapter_adapter.log(logging.CRITICAL, msg, self.recording) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.CRITICAL) + self.assertEqual(record.msg, f"Adapter AdapterAdapter {msg}") + self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.funcName, 'test_nested') + orig_manager = adapter_adapter.manager + self.assertIs(adapter.manager, orig_manager) + self.assertIs(self.logger.manager, orig_manager) + temp_manager = object() + try: + adapter_adapter.manager = temp_manager + self.assertIs(adapter_adapter.manager, temp_manager) + self.assertIs(adapter.manager, temp_manager) + self.assertIs(self.logger.manager, temp_manager) + finally: + adapter_adapter.manager = orig_manager + self.assertIs(adapter_adapter.manager, orig_manager) + self.assertIs(adapter.manager, orig_manager) + self.assertIs(self.logger.manager, orig_manager) + + def test_styled_adapter(self): + # Test an example from the Cookbook. + records = self.recording.records + adapter = StyleAdapter(self.logger) + adapter.warning('Hello, {}!', 'world') + self.assertEqual(str(records[-1].msg), 'Hello, world!') + self.assertEqual(records[-1].funcName, 'test_styled_adapter') + adapter.log(logging.WARNING, 'Goodbye {}.', 'world') + self.assertEqual(str(records[-1].msg), 'Goodbye world.') + self.assertEqual(records[-1].funcName, 'test_styled_adapter') + + def test_nested_styled_adapter(self): + records = self.recording.records + adapter = PrefixAdapter(self.logger) + adapter.prefix = '{}' + adapter2 = StyleAdapter(adapter) + adapter2.warning('Hello, {}!', 'world') + self.assertEqual(str(records[-1].msg), '{} Hello, world!') + self.assertEqual(records[-1].funcName, 'test_nested_styled_adapter') + adapter2.log(logging.WARNING, 'Goodbye {}.', 'world') + self.assertEqual(str(records[-1].msg), '{} Goodbye world.') + self.assertEqual(records[-1].funcName, 'test_nested_styled_adapter') + + def test_find_caller_with_stacklevel(self): + the_level = 1 + trigger = self.adapter.warning + + def innermost(): + trigger('test', stacklevel=the_level) + + def inner(): + innermost() + + def outer(): + inner() + + records = self.recording.records + outer() + self.assertEqual(records[-1].funcName, 'innermost') + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'inner') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'outer') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'test_find_caller_with_stacklevel') + self.assertGreater(records[-1].lineno, lineno) + + def test_extra_in_records(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}) + + self.adapter.critical('foo should be here') + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertEqual(record.foo, '1') + + def test_extra_not_merged_by_default(self): + self.adapter.critical('foo should NOT be here', extra={'foo': 'nope'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertFalse(hasattr(record, 'foo')) + + def test_extra_merged(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}, + merge_extra=True) + + self.adapter.critical('foo and bar should be here', extra={'bar': '2'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertTrue(hasattr(record, 'bar')) + self.assertEqual(record.foo, '1') + self.assertEqual(record.bar, '2') + + def test_extra_merged_log_call_has_precedence(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}, + merge_extra=True) + + self.adapter.critical('foo shall be min', extra={'foo': '2'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertEqual(record.foo, '2') + + +class PrefixAdapter(logging.LoggerAdapter): + prefix = 'Adapter' + + def process(self, msg, kwargs): + return f"{self.prefix} {msg}", kwargs + + +class Message: + def __init__(self, fmt, args): + self.fmt = fmt + self.args = args + + def __str__(self): + return self.fmt.format(*self.args) + + +class StyleAdapter(logging.LoggerAdapter): + def log(self, level, msg, /, *args, stacklevel=1, **kwargs): + if self.isEnabledFor(level): + msg, kwargs = self.process(msg, kwargs) + self.logger.log(level, Message(msg, args), **kwargs, + stacklevel=stacklevel+1) + + +class LoggerTest(BaseTest, AssertErrorMessage): + + def setUp(self): + super(LoggerTest, self).setUp() + self.recording = RecordingHandler() + self.logger = logging.Logger(name='blah') + self.logger.addHandler(self.recording) + self.addCleanup(self.logger.removeHandler, self.recording) + self.addCleanup(self.recording.close) + self.addCleanup(logging.shutdown) + + def test_set_invalid_level(self): + self.assert_error_message( + TypeError, 'Level not an integer or a valid string: None', + self.logger.setLevel, None) + self.assert_error_message( + TypeError, 'Level not an integer or a valid string: (0, 0)', + self.logger.setLevel, (0, 0)) + + def test_exception(self): + msg = 'testing exception: %r' + exc = None + try: + 1 / 0 + except ZeroDivisionError as e: + exc = e + self.logger.exception(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.ERROR) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.exc_info, + (exc.__class__, exc, exc.__traceback__)) + + def test_log_invalid_level_with_raise(self): + with support.swap_attr(logging, 'raiseExceptions', True): + self.assertRaises(TypeError, self.logger.log, '10', 'test message') + + def test_log_invalid_level_no_raise(self): + with support.swap_attr(logging, 'raiseExceptions', False): + self.logger.log('10', 'test message') # no exception happens + + def test_find_caller_with_stack_info(self): + called = [] + support.patch(self, logging.traceback, 'print_stack', + lambda f, file: called.append(file.getvalue())) + + self.logger.findCaller(stack_info=True) + + self.assertEqual(len(called), 1) + self.assertEqual('Stack (most recent call last):\n', called[0]) + + def test_find_caller_with_stacklevel(self): + the_level = 1 + trigger = self.logger.warning + + def innermost(): + trigger('test', stacklevel=the_level) + + def inner(): + innermost() + + def outer(): + inner() + + records = self.recording.records + outer() + self.assertEqual(records[-1].funcName, 'innermost') + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'inner') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'outer') + self.assertGreater(records[-1].lineno, lineno) + lineno = records[-1].lineno + root_logger = logging.getLogger() + root_logger.addHandler(self.recording) + trigger = logging.warning + outer() + self.assertEqual(records[-1].funcName, 'outer') + root_logger.removeHandler(self.recording) + trigger = self.logger.warning + the_level += 1 + outer() + self.assertEqual(records[-1].funcName, 'test_find_caller_with_stacklevel') + self.assertGreater(records[-1].lineno, lineno) + + def test_make_record_with_extra_overwrite(self): + name = 'my record' + level = 13 + fn = lno = msg = args = exc_info = func = sinfo = None + rv = logging._logRecordFactory(name, level, fn, lno, msg, args, + exc_info, func, sinfo) + + for key in ('message', 'asctime') + tuple(rv.__dict__.keys()): + extra = {key: 'some value'} + self.assertRaises(KeyError, self.logger.makeRecord, name, level, + fn, lno, msg, args, exc_info, + extra=extra, sinfo=sinfo) + + def test_make_record_with_extra_no_overwrite(self): + name = 'my record' + level = 13 + fn = lno = msg = args = exc_info = func = sinfo = None + extra = {'valid_key': 'some value'} + result = self.logger.makeRecord(name, level, fn, lno, msg, args, + exc_info, extra=extra, sinfo=sinfo) + self.assertIn('valid_key', result.__dict__) + + def test_has_handlers(self): + self.assertTrue(self.logger.hasHandlers()) + + for handler in self.logger.handlers: + self.logger.removeHandler(handler) + self.assertFalse(self.logger.hasHandlers()) + + def test_has_handlers_no_propagate(self): + child_logger = logging.getLogger('blah.child') + child_logger.propagate = False + self.assertFalse(child_logger.hasHandlers()) + + def test_is_enabled_for(self): + old_disable = self.logger.manager.disable + self.logger.manager.disable = 23 + self.addCleanup(setattr, self.logger.manager, 'disable', old_disable) + self.assertFalse(self.logger.isEnabledFor(22)) + + def test_is_enabled_for_disabled_logger(self): + old_disabled = self.logger.disabled + old_disable = self.logger.manager.disable + + self.logger.disabled = True + self.logger.manager.disable = 21 + + self.addCleanup(setattr, self.logger, 'disabled', old_disabled) + self.addCleanup(setattr, self.logger.manager, 'disable', old_disable) + + self.assertFalse(self.logger.isEnabledFor(22)) + + def test_root_logger_aliases(self): + root = logging.getLogger() + self.assertIs(root, logging.root) + self.assertIs(root, logging.getLogger(None)) + self.assertIs(root, logging.getLogger('')) + self.assertIs(root, logging.getLogger('root')) + self.assertIs(root, logging.getLogger('foo').root) + self.assertIs(root, logging.getLogger('foo.bar').root) + self.assertIs(root, logging.getLogger('foo').parent) + + self.assertIsNot(root, logging.getLogger('\0')) + self.assertIsNot(root, logging.getLogger('foo.bar').parent) + + def test_invalid_names(self): + self.assertRaises(TypeError, logging.getLogger, any) + self.assertRaises(TypeError, logging.getLogger, b'foo') + + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for name in ('', 'root', 'foo', 'foo.bar', 'baz.bar'): + logger = logging.getLogger(name) + s = pickle.dumps(logger, proto) + unpickled = pickle.loads(s) + self.assertIs(unpickled, logger) + + def test_caching(self): + root = self.root_logger + logger1 = logging.getLogger("abc") + logger2 = logging.getLogger("abc.def") + + # Set root logger level and ensure cache is empty + root.setLevel(logging.ERROR) + self.assertEqual(logger2.getEffectiveLevel(), logging.ERROR) + self.assertEqual(logger2._cache, {}) + + # Ensure cache is populated and calls are consistent + self.assertTrue(logger2.isEnabledFor(logging.ERROR)) + self.assertFalse(logger2.isEnabledFor(logging.DEBUG)) + self.assertEqual(logger2._cache, {logging.ERROR: True, logging.DEBUG: False}) + self.assertEqual(root._cache, {}) + self.assertTrue(logger2.isEnabledFor(logging.ERROR)) + + # Ensure root cache gets populated + self.assertEqual(root._cache, {}) + self.assertTrue(root.isEnabledFor(logging.ERROR)) + self.assertEqual(root._cache, {logging.ERROR: True}) + + # Set parent logger level and ensure caches are emptied + logger1.setLevel(logging.CRITICAL) + self.assertEqual(logger2.getEffectiveLevel(), logging.CRITICAL) + self.assertEqual(logger2._cache, {}) + + # Ensure logger2 uses parent logger's effective level + self.assertFalse(logger2.isEnabledFor(logging.ERROR)) + + # Set level to NOTSET and ensure caches are empty + logger2.setLevel(logging.NOTSET) + self.assertEqual(logger2.getEffectiveLevel(), logging.CRITICAL) + self.assertEqual(logger2._cache, {}) + self.assertEqual(logger1._cache, {}) + self.assertEqual(root._cache, {}) + + # Verify logger2 follows parent and not root + self.assertFalse(logger2.isEnabledFor(logging.ERROR)) + self.assertTrue(logger2.isEnabledFor(logging.CRITICAL)) + self.assertFalse(logger1.isEnabledFor(logging.ERROR)) + self.assertTrue(logger1.isEnabledFor(logging.CRITICAL)) + self.assertTrue(root.isEnabledFor(logging.ERROR)) + + # Disable logging in manager and ensure caches are clear + logging.disable() + self.assertEqual(logger2.getEffectiveLevel(), logging.CRITICAL) + self.assertEqual(logger2._cache, {}) + self.assertEqual(logger1._cache, {}) + self.assertEqual(root._cache, {}) + + # Ensure no loggers are enabled + self.assertFalse(logger1.isEnabledFor(logging.CRITICAL)) + self.assertFalse(logger2.isEnabledFor(logging.CRITICAL)) + self.assertFalse(root.isEnabledFor(logging.CRITICAL)) + + +class BaseFileTest(BaseTest): + "Base class for handler tests that write log files" + + def setUp(self): + BaseTest.setUp(self) + self.fn = make_temp_file(".log", "test_logging-2-") + self.rmfiles = [] + + def tearDown(self): + for fn in self.rmfiles: + os.unlink(fn) + if os.path.exists(self.fn): + os.unlink(self.fn) + BaseTest.tearDown(self) + + def assertLogFile(self, filename): + "Assert a log file is there and register it for deletion" + self.assertTrue(os.path.exists(filename), + msg="Log file %r does not exist" % filename) + self.rmfiles.append(filename) + + def next_rec(self): + return logging.LogRecord('n', logging.DEBUG, 'p', 1, + self.next_message(), None, None, None) + +class FileHandlerTest(BaseFileTest): + def test_delay(self): + os.unlink(self.fn) + fh = logging.FileHandler(self.fn, encoding='utf-8', delay=True) + self.assertIsNone(fh.stream) + self.assertFalse(os.path.exists(self.fn)) + fh.handle(logging.makeLogRecord({})) + self.assertIsNotNone(fh.stream) + self.assertTrue(os.path.exists(self.fn)) + fh.close() + + def test_emit_after_closing_in_write_mode(self): + # Issue #42378 + os.unlink(self.fn) + fh = logging.FileHandler(self.fn, encoding='utf-8', mode='w') + fh.setFormatter(logging.Formatter('%(message)s')) + fh.emit(self.next_rec()) # '1' + fh.close() + fh.emit(self.next_rec()) # '2' + with open(self.fn) as fp: + self.assertEqual(fp.read().strip(), '1') + +class RotatingFileHandlerTest(BaseFileTest): + def test_should_not_rollover(self): + # If file is empty rollover never occurs + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=1) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + # If maxBytes is zero rollover never occurs + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=0) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + with open(self.fn, 'wb') as f: + f.write(b'\n') + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", maxBytes=0) + self.assertFalse(rh.shouldRollover(None)) + rh.close() + + @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") + def test_should_not_rollover_non_file(self): + # bpo-45401 - test with special file + # We set maxBytes to 1 so that rollover would normally happen, except + # for the check for regular files + rh = logging.handlers.RotatingFileHandler( + os.devnull, encoding="utf-8", maxBytes=1) + self.assertFalse(rh.shouldRollover(self.next_rec())) + rh.close() + + def test_should_rollover(self): + with open(self.fn, 'wb') as f: + f.write(b'\n') + rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8", maxBytes=2) + self.assertTrue(rh.shouldRollover(self.next_rec())) + rh.close() + + def test_file_created(self): + # checks that the file is created and assumes it was created + # by us + os.unlink(self.fn) + rh = logging.handlers.RotatingFileHandler(self.fn, encoding="utf-8") + rh.emit(self.next_rec()) + self.assertLogFile(self.fn) + rh.close() + + def test_max_bytes(self, delay=False): + kwargs = {'delay': delay} if delay else {} + os.unlink(self.fn) + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", backupCount=2, maxBytes=100, **kwargs) + self.assertIs(os.path.exists(self.fn), not delay) + small = logging.makeLogRecord({'msg': 'a'}) + large = logging.makeLogRecord({'msg': 'b'*100}) + self.assertFalse(rh.shouldRollover(small)) + self.assertFalse(rh.shouldRollover(large)) + rh.emit(small) + self.assertLogFile(self.fn) + self.assertFalse(os.path.exists(self.fn + ".1")) + self.assertFalse(rh.shouldRollover(small)) + self.assertTrue(rh.shouldRollover(large)) + rh.emit(large) + self.assertTrue(os.path.exists(self.fn)) + self.assertLogFile(self.fn + ".1") + self.assertFalse(os.path.exists(self.fn + ".2")) + self.assertTrue(rh.shouldRollover(small)) + self.assertTrue(rh.shouldRollover(large)) + rh.close() + + def test_max_bytes_delay(self): + self.test_max_bytes(delay=True) + + def test_rollover_filenames(self): + def namer(name): + return name + ".test" + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", backupCount=2, maxBytes=1) + rh.namer = namer + rh.emit(self.next_rec()) + self.assertLogFile(self.fn) + self.assertFalse(os.path.exists(namer(self.fn + ".1"))) + rh.emit(self.next_rec()) + self.assertLogFile(namer(self.fn + ".1")) + self.assertFalse(os.path.exists(namer(self.fn + ".2"))) + rh.emit(self.next_rec()) + self.assertLogFile(namer(self.fn + ".2")) + self.assertFalse(os.path.exists(namer(self.fn + ".3"))) + rh.emit(self.next_rec()) + self.assertFalse(os.path.exists(namer(self.fn + ".3"))) + rh.close() + + def test_namer_rotator_inheritance(self): + class HandlerWithNamerAndRotator(logging.handlers.RotatingFileHandler): + def namer(self, name): + return name + ".test" + + def rotator(self, source, dest): + if os.path.exists(source): + os.replace(source, dest + ".rotated") + + rh = HandlerWithNamerAndRotator( + self.fn, encoding="utf-8", backupCount=2, maxBytes=1) + self.assertEqual(rh.namer(self.fn), self.fn + ".test") + rh.emit(self.next_rec()) + self.assertLogFile(self.fn) + rh.emit(self.next_rec()) + self.assertLogFile(rh.namer(self.fn + ".1") + ".rotated") + self.assertFalse(os.path.exists(rh.namer(self.fn + ".1"))) + rh.close() + + @support.requires_zlib() + def test_rotator(self): + def namer(name): + return name + ".gz" + + def rotator(source, dest): + with open(source, "rb") as sf: + data = sf.read() + compressed = zlib.compress(data, 9) + with open(dest, "wb") as df: + df.write(compressed) + os.remove(source) + + rh = logging.handlers.RotatingFileHandler( + self.fn, encoding="utf-8", backupCount=2, maxBytes=1) + rh.rotator = rotator + rh.namer = namer + m1 = self.next_rec() + rh.emit(m1) + self.assertLogFile(self.fn) + m2 = self.next_rec() + rh.emit(m2) + fn = namer(self.fn + ".1") + self.assertLogFile(fn) + newline = os.linesep + with open(fn, "rb") as f: + compressed = f.read() + data = zlib.decompress(compressed) + self.assertEqual(data.decode("ascii"), m1.msg + newline) + rh.emit(self.next_rec()) + fn = namer(self.fn + ".2") + self.assertLogFile(fn) + with open(fn, "rb") as f: + compressed = f.read() + data = zlib.decompress(compressed) + self.assertEqual(data.decode("ascii"), m1.msg + newline) + rh.emit(self.next_rec()) + fn = namer(self.fn + ".2") + with open(fn, "rb") as f: + compressed = f.read() + data = zlib.decompress(compressed) + self.assertEqual(data.decode("ascii"), m2.msg + newline) + self.assertFalse(os.path.exists(namer(self.fn + ".3"))) + rh.close() + +class TimedRotatingFileHandlerTest(BaseFileTest): + @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") + def test_should_not_rollover(self): + # See bpo-45401. Should only ever rollover regular files + fh = logging.handlers.TimedRotatingFileHandler( + os.devnull, 'S', encoding="utf-8", backupCount=1) + time.sleep(1.1) # a little over a second ... + r = logging.makeLogRecord({'msg': 'testing - device file'}) + self.assertFalse(fh.shouldRollover(r)) + fh.close() + + # other test methods added below + def test_rollover(self): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, 'S', encoding="utf-8", backupCount=1) + fmt = logging.Formatter('%(asctime)s %(message)s') + fh.setFormatter(fmt) + r1 = logging.makeLogRecord({'msg': 'testing - initial'}) + fh.emit(r1) + self.assertLogFile(self.fn) + time.sleep(1.1) # a little over a second ... + r2 = logging.makeLogRecord({'msg': 'testing - after delay'}) + fh.emit(r2) + fh.close() + # At this point, we should have a recent rotated file which we + # can test for the existence of. However, in practice, on some + # machines which run really slowly, we don't know how far back + # in time to go to look for the log file. So, we go back a fair + # bit, and stop as soon as we see a rotated file. In theory this + # could of course still fail, but the chances are lower. + found = False + now = datetime.datetime.now() + GO_BACK = 5 * 60 # seconds + for secs in range(GO_BACK): + prev = now - datetime.timedelta(seconds=secs) + fn = self.fn + prev.strftime(".%Y-%m-%d_%H-%M-%S") + found = os.path.exists(fn) + if found: + self.rmfiles.append(fn) + break + msg = 'No rotated files found, went back %d seconds' % GO_BACK + if not found: + # print additional diagnostics + dn, fn = os.path.split(self.fn) + files = [f for f in os.listdir(dn) if f.startswith(fn)] + print('Test time: %s' % now.strftime("%Y-%m-%d %H-%M-%S"), file=sys.stderr) + print('The only matching files are: %s' % files, file=sys.stderr) + for f in files: + print('Contents of %s:' % f) + path = os.path.join(dn, f) + with open(path, 'r') as tf: + print(tf.read()) + self.assertTrue(found, msg=msg) + + def test_rollover_at_midnight(self, weekly=False): + os_helper.unlink(self.fn) + now = datetime.datetime.now() + atTime = now.time() + if not 0.1 < atTime.microsecond/1e6 < 0.9: + # The test requires all records to be emitted within + # the range of the same whole second. + time.sleep((0.1 - atTime.microsecond/1e6) % 1.0) + now = datetime.datetime.now() + atTime = now.time() + atTime = atTime.replace(microsecond=0) + fmt = logging.Formatter('%(asctime)s %(message)s') + when = f'W{now.weekday()}' if weekly else 'MIDNIGHT' + for i in range(3): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) + fh.emit(r2) + fh.close() + self.assertLogFile(self.fn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + os.utime(self.fn, (now.timestamp() - 1,)*2) + for i in range(2): + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) + fh.emit(r2) + fh.close() + rolloverDate = now - datetime.timedelta(days=7 if weekly else 1) + otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' + self.assertLogFile(otherfn) + with open(self.fn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing2 {i}', line) + with open(otherfn, encoding="utf-8") as f: + for i, line in enumerate(f): + self.assertIn(f'testing1 {i}', line) + + def test_rollover_at_weekday(self): + self.test_rollover_at_midnight(weekly=True) + + def test_invalid(self): + assertRaises = self.assertRaises + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'X', encoding="utf-8", delay=True) + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'W', encoding="utf-8", delay=True) + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'W7', encoding="utf-8", delay=True) + + # TODO: Test for utc=False. + def test_compute_rollover_daily_attime(self): + currentTime = 0 + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', + utc=True, atTime=None) + try: + actual = rh.computeRollover(currentTime) + self.assertEqual(actual, currentTime + 24 * 60 * 60) + + actual = rh.computeRollover(currentTime + 24 * 60 * 60 - 1) + self.assertEqual(actual, currentTime + 24 * 60 * 60) + + actual = rh.computeRollover(currentTime + 24 * 60 * 60) + self.assertEqual(actual, currentTime + 48 * 60 * 60) + + actual = rh.computeRollover(currentTime + 25 * 60 * 60) + self.assertEqual(actual, currentTime + 48 * 60 * 60) + finally: + rh.close() + + atTime = datetime.time(12, 0, 0) + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', + utc=True, atTime=atTime) + try: + actual = rh.computeRollover(currentTime) + self.assertEqual(actual, currentTime + 12 * 60 * 60) + + actual = rh.computeRollover(currentTime + 12 * 60 * 60 - 1) + self.assertEqual(actual, currentTime + 12 * 60 * 60) + + actual = rh.computeRollover(currentTime + 12 * 60 * 60) + self.assertEqual(actual, currentTime + 36 * 60 * 60) + + actual = rh.computeRollover(currentTime + 13 * 60 * 60) + self.assertEqual(actual, currentTime + 36 * 60 * 60) + finally: + rh.close() + + # TODO: Test for utc=False. + def test_compute_rollover_weekly_attime(self): + currentTime = int(time.time()) + today = currentTime - currentTime % 86400 + + atTime = datetime.time(12, 0, 0) + + wday = time.gmtime(today).tm_wday + for day in range(7): + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W%d' % day, interval=1, backupCount=0, + utc=True, atTime=atTime) + try: + if wday > day: + # The rollover day has already passed this week, so we + # go over into next week + expected = (7 - wday + day) + else: + expected = (day - wday) + # At this point expected is in days from now, convert to seconds + expected *= 24 * 60 * 60 + # Add in the rollover time + expected += 12 * 60 * 60 + # Add in adjustment for today + expected += today + + actual = rh.computeRollover(today) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + + actual = rh.computeRollover(today + 12 * 60 * 60 - 1) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + + if day == wday: + # goes into following week + expected += 7 * 24 * 60 * 60 + actual = rh.computeRollover(today + 12 * 60 * 60) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + + actual = rh.computeRollover(today + 13 * 60 * 60) + if actual != expected: + print('failed in timezone: %d' % time.timezone) + print('local vars: %s' % locals()) + self.assertEqual(actual, expected) + finally: + rh.close() + + def test_compute_files_to_delete(self): + # See bpo-46063 for background + wd = tempfile.mkdtemp(prefix='test_logging_') + self.addCleanup(shutil.rmtree, wd) + times = [] + dt = datetime.datetime.now() + for i in range(10): + times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) + dt += datetime.timedelta(seconds=5) + prefixes = ('a.b', 'a.b.c', 'd.e', 'd.e.f', 'g') + files = [] + rotators = [] + for prefix in prefixes: + p = os.path.join(wd, '%s.log' % prefix) + rotator = logging.handlers.TimedRotatingFileHandler(p, when='s', + interval=5, + backupCount=7, + delay=True) + rotators.append(rotator) + if prefix.startswith('a.b'): + for t in times: + files.append('%s.log.%s' % (prefix, t)) + elif prefix.startswith('d.e'): + def namer(filename): + dirname, basename = os.path.split(filename) + basename = basename.replace('.log', '') + '.log' + return os.path.join(dirname, basename) + rotator.namer = namer + for t in times: + files.append('%s.%s.log' % (prefix, t)) + elif prefix == 'g': + def namer(filename): + dirname, basename = os.path.split(filename) + basename = 'g' + basename[6:] + '.oldlog' + return os.path.join(dirname, basename) + rotator.namer = namer + for t in times: + files.append('g%s.oldlog' % t) + # Create empty files + for fn in files: + p = os.path.join(wd, fn) + with open(p, 'wb') as f: + pass + # Now the checks that only the correct files are offered up for deletion + for i, prefix in enumerate(prefixes): + rotator = rotators[i] + candidates = rotator.getFilesToDelete() + self.assertEqual(len(candidates), 3, candidates) + if prefix.startswith('a.b'): + p = '%s.log.' % prefix + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.startswith(p)) + elif prefix.startswith('d.e'): + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.endswith('.log'), fn) + self.assertTrue(fn.startswith(prefix + '.') and + fn[len(prefix) + 2].isdigit()) + elif prefix == 'g': + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.endswith('.oldlog')) + self.assertTrue(fn.startswith('g') and fn[1].isdigit()) + + def test_compute_files_to_delete_same_filename_different_extensions(self): + # See GH-93205 for background + wd = pathlib.Path(tempfile.mkdtemp(prefix='test_logging_')) + self.addCleanup(shutil.rmtree, wd) + times = [] + dt = datetime.datetime.now() + n_files = 10 + for _ in range(n_files): + times.append(dt.strftime('%Y-%m-%d_%H-%M-%S')) + dt += datetime.timedelta(seconds=5) + prefixes = ('a.log', 'a.log.b') + files = [] + rotators = [] + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = logging.handlers.TimedRotatingFileHandler(wd / prefix, when='s', + interval=5, + backupCount=backupCount, + delay=True) + rotators.append(rotator) + for t in times: + files.append('%s.%s' % (prefix, t)) + for t in times: + files.append('a.log.%s.c' % t) + # Create empty files + for f in files: + (wd / f).touch() + # Now the checks that only the correct files are offered up for deletion + for i, prefix in enumerate(prefixes): + backupCount = i+1 + rotator = rotators[i] + candidates = rotator.getFilesToDelete() + self.assertEqual(len(candidates), n_files - backupCount, candidates) + matcher = re.compile(r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\Z") + for c in candidates: + d, fn = os.path.split(c) + self.assertTrue(fn.startswith(prefix+'.')) + suffix = fn[(len(prefix)+1):] + self.assertRegex(suffix, matcher) + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False) + + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 12, 0, 0)) + + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 5, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 11, 12, 0)) + + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 4, 12, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(2, 0, 0)) + + test(DT(2012, 3, 10, 1, 59, 59), DT(2012, 3, 10, 2, 0)) + # 2:00:00 is the same as 3:00:00 at 2012-3-11. + test(DT(2012, 3, 10, 2, 0), DT(2012, 3, 11, 3, 0)) + test(DT(2012, 3, 10, 3, 0), DT(2012, 3, 11, 3, 0)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 0)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 2, 0)) + test(DT(2012, 3, 11, 4, 0), DT(2012, 3, 12, 2, 0)) + + test(DT(2012, 11, 3, 1, 59, 59), DT(2012, 11, 3, 2, 0)) + test(DT(2012, 11, 3, 2, 0), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 3, 3, 0), DT(2012, 11, 4, 2, 0)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 5, 2, 0)) + test(DT(2012, 11, 4, 3, 0), DT(2012, 11, 5, 2, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(2, 30, 0)) + + test(DT(2012, 3, 10, 2, 29, 59), DT(2012, 3, 10, 2, 30)) + # No time 2:30:00 at 2012-3-11. + test(DT(2012, 3, 10, 2, 30), DT(2012, 3, 11, 3, 30)) + test(DT(2012, 3, 10, 3, 0), DT(2012, 3, 11, 3, 30)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 2, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 12, 2, 30)) + + test(DT(2012, 11, 3, 2, 29, 59), DT(2012, 11, 3, 2, 30)) + test(DT(2012, 11, 3, 2, 30), DT(2012, 11, 4, 2, 30)) + test(DT(2012, 11, 3, 3, 0), DT(2012, 11, 4, 2, 30)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, + atTime=datetime.time(1, 30, 0)) + + test(DT(2012, 3, 11, 1, 29, 59), DT(2012, 3, 11, 1, 30)) + test(DT(2012, 3, 11, 1, 30), DT(2012, 3, 12, 1, 30)) + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 12, 1, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 12, 1, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 12, 1, 30)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 29, 59), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 30), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 5, 1, 30)) + # It is weird, but the rollover date jumps back from 2012-11-5 + # to 2012-11-4. + test(DT(2012, 11, 4, 1, 0, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 29, 59, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 30, fold=1), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 5, 1, 30)) + test(DT(2012, 11, 4, 2, 30), DT(2012, 11, 5, 1, 30)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False) + + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 12, 0, 0)) + + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 5, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 18, 0, 0)) + + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 11, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 11, 12, 0)) + + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 4, 12, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(2, 0, 0)) + + test(DT(2012, 3, 4, 1, 59, 59), DT(2012, 3, 4, 2, 0)) + # 2:00:00 is the same as 3:00:00 at 2012-3-11. + test(DT(2012, 3, 4, 2, 0), DT(2012, 3, 11, 3, 0)) + test(DT(2012, 3, 4, 3, 0), DT(2012, 3, 11, 3, 0)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 0)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 2, 0)) + test(DT(2012, 3, 11, 4, 0), DT(2012, 3, 18, 2, 0)) + + test(DT(2012, 10, 28, 1, 59, 59), DT(2012, 10, 28, 2, 0)) + test(DT(2012, 10, 28, 2, 0), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 10, 28, 3, 0), DT(2012, 11, 4, 2, 0)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 4, 2, 0)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 11, 2, 0)) + test(DT(2012, 11, 4, 3, 0), DT(2012, 11, 11, 2, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(2, 30, 0)) + + test(DT(2012, 3, 4, 2, 29, 59), DT(2012, 3, 4, 2, 30)) + # No time 2:30:00 at 2012-3-11. + test(DT(2012, 3, 4, 2, 30), DT(2012, 3, 11, 3, 30)) + test(DT(2012, 3, 4, 3, 0), DT(2012, 3, 11, 3, 30)) + + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 11, 3, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 2, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 18, 2, 30)) + + test(DT(2012, 10, 28, 2, 29, 59), DT(2012, 10, 28, 2, 30)) + test(DT(2012, 10, 28, 2, 30), DT(2012, 11, 4, 2, 30)) + test(DT(2012, 10, 28, 3, 0), DT(2012, 11, 4, 2, 30)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, + atTime=datetime.time(1, 30, 0)) + + test(DT(2012, 3, 11, 1, 29, 59), DT(2012, 3, 11, 1, 30)) + test(DT(2012, 3, 11, 1, 30), DT(2012, 3, 18, 1, 30)) + test(DT(2012, 3, 11, 1, 59, 59), DT(2012, 3, 18, 1, 30)) + # No time between 2:00:00 and 3:00:00 at 2012-3-11. + test(DT(2012, 3, 11, 3, 0), DT(2012, 3, 18, 1, 30)) + test(DT(2012, 3, 11, 3, 30), DT(2012, 3, 18, 1, 30)) + + # 1:00:00-2:00:00 is repeated twice at 2012-11-4. + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 29, 59), DT(2012, 11, 4, 1, 30)) + test(DT(2012, 11, 4, 1, 30), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59), DT(2012, 11, 11, 1, 30)) + # It is weird, but the rollover date jumps back from 2012-11-11 + # to 2012-11-4. + test(DT(2012, 11, 4, 1, 0, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 29, 59, fold=1), DT(2012, 11, 4, 1, 30, fold=1)) + test(DT(2012, 11, 4, 1, 30, fold=1), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 1, 59, 59, fold=1), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 2, 0), DT(2012, 11, 11, 1, 30)) + test(DT(2012, 11, 4, 2, 30), DT(2012, 11, 11, 1, 30)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_MIDNIGHT_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3) + + test(DT(2012, 3, 8, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 3, 9, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 9, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 13, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 3, 14, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 3, 14, 0, 0)) + + test(DT(2012, 11, 1, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 11, 2, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 2, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 6, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 7, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 7, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='MIDNIGHT', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 3, 8, 11, 59, 59), DT(2012, 3, 10, 12, 0)) + test(DT(2012, 3, 8, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 8, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 10, 11, 59, 59), DT(2012, 3, 12, 12, 0)) + test(DT(2012, 3, 10, 12, 0), DT(2012, 3, 13, 12, 0)) + test(DT(2012, 3, 10, 13, 0), DT(2012, 3, 13, 12, 0)) + + test(DT(2012, 11, 1, 11, 59, 59), DT(2012, 11, 3, 12, 0)) + test(DT(2012, 11, 1, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 1, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 11, 3, 11, 59, 59), DT(2012, 11, 5, 12, 0)) + test(DT(2012, 11, 3, 12, 0), DT(2012, 11, 6, 12, 0)) + test(DT(2012, 11, 3, 13, 0), DT(2012, 11, 6, 12, 0)) + + fh.close() + + # Run with US-style DST rules: DST begins 2 a.m. on second Sunday in + # March (M3.2.0) and ends 2 a.m. on first Sunday in November (M11.1.0). + @support.run_with_tz('EST+05EDT,M3.2.0,M11.1.0') + def test_compute_rollover_W6_local_interval(self): + # DST begins at 2012-3-11T02:00:00 and ends at 2012-11-4T02:00:00. + DT = datetime.datetime + def test(current, expected): + actual = fh.computeRollover(current.timestamp()) + diff = actual - expected.timestamp() + if diff: + self.assertEqual(diff, 0, datetime.timedelta(seconds=diff)) + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3) + + test(DT(2012, 2, 19, 23, 59, 59), DT(2012, 3, 5, 0, 0)) + test(DT(2012, 2, 20, 0, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 2, 20, 1, 0), DT(2012, 3, 12, 0, 0)) + test(DT(2012, 3, 4, 23, 59, 59), DT(2012, 3, 19, 0, 0)) + test(DT(2012, 3, 5, 0, 0), DT(2012, 3, 26, 0, 0)) + test(DT(2012, 3, 5, 1, 0), DT(2012, 3, 26, 0, 0)) + + test(DT(2012, 10, 14, 23, 59, 59), DT(2012, 10, 29, 0, 0)) + test(DT(2012, 10, 15, 0, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 15, 1, 0), DT(2012, 11, 5, 0, 0)) + test(DT(2012, 10, 28, 23, 59, 59), DT(2012, 11, 12, 0, 0)) + test(DT(2012, 10, 29, 0, 0), DT(2012, 11, 19, 0, 0)) + test(DT(2012, 10, 29, 1, 0), DT(2012, 11, 19, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(0, 0, 0)) + + test(DT(2012, 2, 25, 23, 59, 59), DT(2012, 3, 11, 0, 0)) + test(DT(2012, 2, 26, 0, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 2, 26, 1, 0), DT(2012, 3, 18, 0, 0)) + test(DT(2012, 3, 10, 23, 59, 59), DT(2012, 3, 25, 0, 0)) + test(DT(2012, 3, 11, 0, 0), DT(2012, 4, 1, 0, 0)) + test(DT(2012, 3, 11, 1, 0), DT(2012, 4, 1, 0, 0)) + + test(DT(2012, 10, 20, 23, 59, 59), DT(2012, 11, 4, 0, 0)) + test(DT(2012, 10, 21, 0, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 10, 21, 1, 0), DT(2012, 11, 11, 0, 0)) + test(DT(2012, 11, 3, 23, 59, 59), DT(2012, 11, 18, 0, 0)) + test(DT(2012, 11, 4, 0, 0), DT(2012, 11, 25, 0, 0)) + test(DT(2012, 11, 4, 1, 0), DT(2012, 11, 25, 0, 0)) + + fh.close() + + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when='W6', utc=False, interval=3, + atTime=datetime.time(12, 0, 0)) + + test(DT(2012, 2, 18, 11, 59, 59), DT(2012, 3, 4, 12, 0)) + test(DT(2012, 2, 19, 12, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 2, 19, 13, 0), DT(2012, 3, 11, 12, 0)) + test(DT(2012, 3, 4, 11, 59, 59), DT(2012, 3, 18, 12, 0)) + test(DT(2012, 3, 4, 12, 0), DT(2012, 3, 25, 12, 0)) + test(DT(2012, 3, 4, 13, 0), DT(2012, 3, 25, 12, 0)) + + test(DT(2012, 10, 14, 11, 59, 59), DT(2012, 10, 28, 12, 0)) + test(DT(2012, 10, 14, 12, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 14, 13, 0), DT(2012, 11, 4, 12, 0)) + test(DT(2012, 10, 28, 11, 59, 59), DT(2012, 11, 11, 12, 0)) + test(DT(2012, 10, 28, 12, 0), DT(2012, 11, 18, 12, 0)) + test(DT(2012, 10, 28, 13, 0), DT(2012, 11, 18, 12, 0)) + + fh.close() + + +def secs(**kw): + return datetime.timedelta(**kw) // datetime.timedelta(seconds=1) + +for when, exp in (('S', 1), + ('M', 60), + ('H', 60 * 60), + ('D', 60 * 60 * 24), + ('MIDNIGHT', 60 * 60 * 24), + # current time (epoch start) is a Thursday, W0 means Monday + ('W0', secs(days=4, hours=24)), + ): + for interval in 1, 3: + def test_compute_rollover(self, when=when, interval=interval, exp=exp): + rh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, interval=interval, backupCount=0, utc=True) + currentTime = 0.0 + actual = rh.computeRollover(currentTime) + if when.startswith('W'): + exp += secs(days=7*(interval-1)) + else: + exp *= interval + if exp != actual: + # Failures occur on some systems for MIDNIGHT and W0. + # Print detailed calculation for MIDNIGHT so we can try to see + # what's going on + if when == 'MIDNIGHT': + try: + if rh.utc: + t = time.gmtime(currentTime) + else: + t = time.localtime(currentTime) + currentHour = t[3] + currentMinute = t[4] + currentSecond = t[5] + # r is the number of seconds left between now and midnight + r = logging.handlers._MIDNIGHT - ((currentHour * 60 + + currentMinute) * 60 + + currentSecond) + result = currentTime + r + print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) + print('currentHour: %s' % currentHour, file=sys.stderr) + print('currentMinute: %s' % currentMinute, file=sys.stderr) + print('currentSecond: %s' % currentSecond, file=sys.stderr) + print('r: %s' % r, file=sys.stderr) + print('result: %s' % result, file=sys.stderr) + except Exception as e: + print('exception in diagnostic code: %s' % e, file=sys.stderr) + self.assertEqual(exp, actual) + rh.close() + name = "test_compute_rollover_%s" % when + if interval > 1: + name += "_interval" + test_compute_rollover.__name__ = name + setattr(TimedRotatingFileHandlerTest, name, test_compute_rollover) + + +@unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') +class NTEventLogHandlerTest(BaseTest): + def test_basic(self): + logtype = 'Application' + elh = win32evtlog.OpenEventLog(None, logtype) + num_recs = win32evtlog.GetNumberOfEventLogRecords(elh) + + try: + h = logging.handlers.NTEventLogHandler('test_logging') + except pywintypes.error as e: + if e.winerror == 5: # access denied + raise unittest.SkipTest('Insufficient privileges to run test') + raise + + r = logging.makeLogRecord({'msg': 'Test Log Message'}) + h.handle(r) + h.close() + # Now see if the event is recorded + self.assertLess(num_recs, win32evtlog.GetNumberOfEventLogRecords(elh)) + flags = win32evtlog.EVENTLOG_BACKWARDS_READ | \ + win32evtlog.EVENTLOG_SEQUENTIAL_READ + found = False + GO_BACK = 100 + events = win32evtlog.ReadEventLog(elh, flags, GO_BACK) + for e in events: + if e.SourceName != 'test_logging': + continue + msg = win32evtlogutil.SafeFormatMessage(e, logtype) + if msg != 'Test Log Message\r\n': + continue + found = True + break + msg = 'Record not found in event log, went back %d records' % GO_BACK + self.assertTrue(found, msg=msg) + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + not_exported = { + 'logThreads', 'logMultiprocessing', 'logProcesses', 'currentframe', + 'PercentStyle', 'StrFormatStyle', 'StringTemplateStyle', + 'Filterer', 'PlaceHolder', 'Manager', 'RootLogger', 'root', + 'threading', 'logAsyncioTasks'} + support.check__all__(self, logging, not_exported=not_exported) + + +# Set the locale to the platform-dependent default. I have no idea +# why the test does this, but in any case we save the current locale +# first and restore it at the end. +def setUpModule(): + unittest.enterModuleContext(support.run_with_locale('LC_ALL', '')) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index a25eff5b066..a879c6f5df8 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -378,8 +378,6 @@ def test_long(self): self.assertRaises(ValueError, int, '\u3053\u3093\u306b\u3061\u306f') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_conversion(self): class JustLong: @@ -388,15 +386,6 @@ def __long__(self): return 42 self.assertRaises(TypeError, int, JustLong()) - class LongTrunc: - # __long__ should be ignored in 3.x - def __long__(self): - return 42 - def __trunc__(self): - return 1729 - with self.assertWarns(DeprecationWarning): - self.assertEqual(int(LongTrunc()), 1729) - def check_float_conversion(self, n): # Check that int -> float conversion behaviour matches # that of the pure Python version above. @@ -414,8 +403,7 @@ def check_float_conversion(self, n): "Got {}, expected {}.".format(n, actual, expected)) self.assertEqual(actual, expected, msg) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_IEEE_754 def test_float_conversion(self): @@ -486,6 +474,12 @@ def test_float_conversion(self): self.check_float_conversion(value) self.check_float_conversion(-value) + @support.requires_IEEE_754 + @support.bigmemtest(2**32, memuse=0.2) + def test_float_conversion_huge_integer(self, size): + v = 1 << size + self.assertRaises(OverflowError, float, v) + def test_float_overflow(self): for x in -2.0, -1.0, 0.0, 1.0, 2.0: self.assertEqual(float(int(x)), x) @@ -627,8 +621,56 @@ def __lt__(self, other): eq(x > y, Rcmp > 0) eq(x >= y, Rcmp >= 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_IEEE_754 + @support.bigmemtest(2**32, memuse=0.2) + def test_mixed_compares_huge_integer(self, size): + v = 1 << size + f = sys.float_info.max + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, True) + self.assertIs(f <= v, True) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + f = float('inf') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, True) + self.assertIs(f >= v, True) + f = float('nan') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + + del v + v = (-1) << size + f = -sys.float_info.max + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, True) + self.assertIs(f >= v, True) + f = float('-inf') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, True) + self.assertIs(f <= v, True) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + f = float('nan') + self.assertIs(f == v, False) + self.assertIs(f != v, True) + self.assertIs(f < v, False) + self.assertIs(f <= v, False) + self.assertIs(f > v, False) + self.assertIs(f >= v, False) + def test__format__(self): self.assertEqual(format(123456789, 'd'), '123456789') self.assertEqual(format(123456789, 'd'), '123456789') @@ -828,8 +870,7 @@ def check_truediv(self, a, b, skip_small=True): self.assertEqual(expected, got, "Incorrectly rounded division {}/{}: " "expected {}, got {}".format(a, b, expected, got)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_IEEE_754 def test_correctly_rounded_true_division(self): # more stringent tests than those above, checking that the @@ -950,9 +991,12 @@ def test_huge_lshift_of_zero(self): self.assertEqual(0 << (sys.maxsize + 1), 0) @support.cpython_only - @support.bigmemtest(sys.maxsize + 1000, memuse=2/15 * 2, dry_run=False) + @support.bigmemtest(2**32, memuse=0.2) def test_huge_lshift(self, size): - self.assertEqual(1 << (sys.maxsize + 1000), 1 << 1000 << sys.maxsize) + v = 5 << size + self.assertEqual(v.bit_length(), size + 3) + self.assertEqual(v.bit_count(), 2) + self.assertEqual(v >> size, 5) def test_huge_rshift(self): huge_shift = 1 << 1000 @@ -964,11 +1008,13 @@ def test_huge_rshift(self): self.assertEqual(-2**128 >> huge_shift, -1) @support.cpython_only - @support.bigmemtest(sys.maxsize + 500, memuse=2/15, dry_run=False) + @support.bigmemtest(2**32, memuse=0.2) def test_huge_rshift_of_huge(self, size): - huge = ((1 << 500) + 11) << sys.maxsize - self.assertEqual(huge >> (sys.maxsize + 1), (1 << 499) + 5) - self.assertEqual(huge >> (sys.maxsize + 1000), 0) + huge = ((1 << 500) + 11) << size + self.assertEqual(huge.bit_length(), size + 501) + self.assertEqual(huge.bit_count(), 4) + self.assertEqual(huge >> (size + 1), (1 << 499) + 5) + self.assertEqual(huge >> (size + 1000), 0) def test_small_rshift(self): self.assertEqual(42 >> 1, 21) @@ -1141,8 +1187,6 @@ def test_bit_count(self): self.assertEqual((a ^ 63).bit_count(), 7) self.assertEqual(((a - 1) ^ 510).bit_count(), exp - 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_round(self): # check round-half-even algorithm. For round to nearest ten; # rounding map is invariant under adding multiples of 20 @@ -1332,17 +1376,22 @@ def equivalent_python(n, length, byteorder, signed=False): check(tests4, 'little', signed=False) self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=False) - self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=True) self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=False) - self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (128).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (128).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (-129).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (-129).to_bytes, 1, 'little', signed=True) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False) self.assertEqual((0).to_bytes(0, 'big'), b'') + self.assertEqual((0).to_bytes(0, 'big', signed=True), b'') self.assertEqual((1).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x01') self.assertEqual((0).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x00') self.assertEqual((-1).to_bytes(5, 'big', signed=True), b'\xff\xff\xff\xff\xff') self.assertRaises(OverflowError, (1).to_bytes, 0, 'big') + self.assertRaises(OverflowError, (-1).to_bytes, 0, 'big', signed=True) + self.assertRaises(OverflowError, (-1).to_bytes, 0, 'little', signed=True) # gh-98783 class SubStr(str): @@ -1350,8 +1399,7 @@ class SubStr(str): self.assertEqual((0).to_bytes(1, SubStr('big')), b'\x00') self.assertEqual((0).to_bytes(0, SubStr('little')), b'') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_from_bytes(self): def check(tests, byteorder, signed=False): def equivalent_python(byte_array, byteorder, signed=False): @@ -1430,7 +1478,6 @@ def equivalent_python(byte_array, byteorder, signed=False): b'\x00': 0, b'\x00\x00': 0, b'\x01': 1, - b'\x00\x01': 256, b'\xff': -1, b'\xff\xff': -1, b'\x81': -127, @@ -1565,6 +1612,11 @@ def test_from_bytes_small(self): b = i.to_bytes(2, signed=True) self.assertIs(int.from_bytes(b, signed=True), i) + def test_is_integer(self): + self.assertTrue((-1).is_integer()) + self.assertTrue((0).is_integer()) + self.assertTrue((1).is_integer()) + def test_access_to_nonexistent_digit_0(self): # http://bugs.python.org/issue14630: A bug in _PyLong_Copy meant that # ob_digit[0] was being incorrectly accessed for instances of a @@ -1608,5 +1660,63 @@ def test_square(self): self.assertEqual(n**2, (1 << (2 * bitlen)) - (1 << (bitlen + 1)) + 1) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test___sizeof__(self): + self.assertEqual(int.__itemsize__, sys.int_info.sizeof_digit) + + # Pairs (test_value, number of allocated digits) + test_values = [ + # We always allocate space for at least one digit, even for + # a value of zero; sys.getsizeof should reflect that. + (0, 1), + (1, 1), + (-1, 1), + (BASE-1, 1), + (1-BASE, 1), + (BASE, 2), + (-BASE, 2), + (BASE*BASE - 1, 2), + (BASE*BASE, 3), + ] + + for value, ndigits in test_values: + with self.subTest(value): + self.assertEqual( + value.__sizeof__(), + int.__basicsize__ + int.__itemsize__ * ndigits + ) + + # Same test for a subclass of int. + class MyInt(int): + pass + + self.assertEqual(MyInt.__itemsize__, sys.int_info.sizeof_digit) + + for value, ndigits in test_values: + with self.subTest(value): + self.assertEqual( + MyInt(value).__sizeof__(), + MyInt.__basicsize__ + MyInt.__itemsize__ * ndigits + ) + + # GH-117195 -- This shouldn't crash + object.__sizeof__(1) + + def test_hash(self): + # gh-136599 + self.assertEqual(hash(-1), -2) + self.assertEqual(hash(0), 0) + self.assertEqual(hash(10), 10) + + self.assertEqual(hash(sys.hash_info.modulus - 2), sys.hash_info.modulus - 2) + self.assertEqual(hash(sys.hash_info.modulus - 1), sys.hash_info.modulus - 1) + self.assertEqual(hash(sys.hash_info.modulus), 0) + self.assertEqual(hash(sys.hash_info.modulus + 1), 1) + + self.assertEqual(hash(-sys.hash_info.modulus - 2), -2) + self.assertEqual(hash(-sys.hash_info.modulus - 1), -2) + self.assertEqual(hash(-sys.hash_info.modulus), 0) + self.assertEqual(hash(-sys.hash_info.modulus + 1), -sys.hash_info.modulus + 1) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py new file mode 100644 index 00000000000..a433f0c6193 --- /dev/null +++ b/Lib/test/test_lzma.py @@ -0,0 +1,2110 @@ +import array +from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE +import os +import pickle +import random +import sys +from test import support +import unittest +from compression._common import _streams + +from test.support import _4G, bigmemtest +from test.support.import_helper import import_module +from test.support.os_helper import ( + TESTFN, unlink, FakePath +) + +lzma = import_module("lzma") +from lzma import LZMACompressor, LZMADecompressor, LZMAError, LZMAFile + + +class CompressorDecompressorTestCase(unittest.TestCase): + + # Test error cases. + + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format + def test_simple_bad_args(self): + self.assertRaises(TypeError, LZMACompressor, []) + self.assertRaises(TypeError, LZMACompressor, format=3.45) + self.assertRaises(TypeError, LZMACompressor, check="") + self.assertRaises(TypeError, LZMACompressor, preset="asdf") + self.assertRaises(TypeError, LZMACompressor, filters=3) + # Can't specify FORMAT_AUTO when compressing. + self.assertRaises(ValueError, LZMACompressor, format=lzma.FORMAT_AUTO) + # Can't specify a preset and a custom filter chain at the same time. + with self.assertRaises(ValueError): + LZMACompressor(preset=7, filters=[{"id": lzma.FILTER_LZMA2}]) + + self.assertRaises(TypeError, LZMADecompressor, ()) + self.assertRaises(TypeError, LZMADecompressor, memlimit=b"qw") + with self.assertRaises(TypeError): + LZMADecompressor(lzma.FORMAT_RAW, filters="zzz") + # Cannot specify a memory limit with FILTER_RAW. + with self.assertRaises(ValueError): + LZMADecompressor(lzma.FORMAT_RAW, memlimit=0x1000000) + # Can only specify a custom filter chain with FILTER_RAW. + self.assertRaises(ValueError, LZMADecompressor, filters=FILTERS_RAW_1) + with self.assertRaises(ValueError): + LZMADecompressor(format=lzma.FORMAT_XZ, filters=FILTERS_RAW_1) + with self.assertRaises(ValueError): + LZMADecompressor(format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1) + + lzc = LZMACompressor() + self.assertRaises(TypeError, lzc.compress) + self.assertRaises(TypeError, lzc.compress, b"foo", b"bar") + self.assertRaises(TypeError, lzc.flush, b"blah") + empty = lzc.flush() + self.assertRaises(ValueError, lzc.compress, b"quux") + self.assertRaises(ValueError, lzc.flush) + + lzd = LZMADecompressor() + self.assertRaises(TypeError, lzd.decompress) + self.assertRaises(TypeError, lzd.decompress, b"foo", b"bar") + lzd.decompress(empty) + self.assertRaises(EOFError, lzd.decompress, b"quux") + + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Failed to initialize encoder + def test_bad_filter_spec(self): + self.assertRaises(TypeError, LZMACompressor, filters=[b"wobsite"]) + self.assertRaises(ValueError, LZMACompressor, filters=[{"xyzzy": 3}]) + self.assertRaises(ValueError, LZMACompressor, filters=[{"id": 98765}]) + with self.assertRaises(ValueError): + LZMACompressor(filters=[{"id": lzma.FILTER_LZMA2, "foo": 0}]) + with self.assertRaises(ValueError): + LZMACompressor(filters=[{"id": lzma.FILTER_DELTA, "foo": 0}]) + with self.assertRaises(ValueError): + LZMACompressor(filters=[{"id": lzma.FILTER_X86, "foo": 0}]) + + def test_decompressor_after_eof(self): + lzd = LZMADecompressor() + lzd.decompress(COMPRESSED_XZ) + self.assertRaises(EOFError, lzd.decompress, b"nyan") + + def test_decompressor_memlimit(self): + lzd = LZMADecompressor(memlimit=1024) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + + lzd = LZMADecompressor(lzma.FORMAT_XZ, memlimit=1024) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + + lzd = LZMADecompressor(lzma.FORMAT_ALONE, memlimit=1024) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_ALONE) + + # Test LZMADecompressor on known-good input data. + + def _test_decompressor(self, lzd, data, check, unused_data=b""): + self.assertFalse(lzd.eof) + out = lzd.decompress(data) + self.assertEqual(out, INPUT) + self.assertEqual(lzd.check, check) + self.assertTrue(lzd.eof) + self.assertEqual(lzd.unused_data, unused_data) + + def test_decompressor_auto(self): + lzd = LZMADecompressor() + self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) + + lzd = LZMADecompressor() + self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) + + def test_decompressor_xz(self): + lzd = LZMADecompressor(lzma.FORMAT_XZ) + self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) + + def test_decompressor_alone(self): + lzd = LZMADecompressor(lzma.FORMAT_ALONE) + self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) + + def test_decompressor_raw_1(self): + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE) + + def test_decompressor_raw_2(self): + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2) + self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE) + + def test_decompressor_raw_3(self): + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3) + self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE) + + def test_decompressor_raw_4(self): + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE) + + def test_decompressor_chunks(self): + lzd = LZMADecompressor() + out = [] + for i in range(0, len(COMPRESSED_XZ), 10): + self.assertFalse(lzd.eof) + out.append(lzd.decompress(COMPRESSED_XZ[i:i+10])) + out = b"".join(out) + self.assertEqual(out, INPUT) + self.assertEqual(lzd.check, lzma.CHECK_CRC64) + self.assertTrue(lzd.eof) + self.assertEqual(lzd.unused_data, b"") + + @unittest.expectedFailure # TODO: RUSTPYTHON; EOFError: End of stream already reached + def test_decompressor_chunks_empty(self): + lzd = LZMADecompressor() + out = [] + for i in range(0, len(COMPRESSED_XZ), 10): + self.assertFalse(lzd.eof) + out.append(lzd.decompress(b'')) + out.append(lzd.decompress(b'')) + out.append(lzd.decompress(b'')) + out.append(lzd.decompress(COMPRESSED_XZ[i:i+10])) + out = b"".join(out) + self.assertEqual(out, INPUT) + self.assertEqual(lzd.check, lzma.CHECK_CRC64) + self.assertTrue(lzd.eof) + self.assertEqual(lzd.unused_data, b"") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check' + def test_decompressor_chunks_maxsize(self): + lzd = LZMADecompressor() + max_length = 100 + out = [] + + # Feed first half the input + len_ = len(COMPRESSED_XZ) // 2 + out.append(lzd.decompress(COMPRESSED_XZ[:len_], + max_length=max_length)) + self.assertFalse(lzd.needs_input) + self.assertEqual(len(out[-1]), max_length) + + # Retrieve more data without providing more input + out.append(lzd.decompress(b'', max_length=max_length)) + self.assertFalse(lzd.needs_input) + self.assertEqual(len(out[-1]), max_length) + + # Retrieve more data while providing more input + out.append(lzd.decompress(COMPRESSED_XZ[len_:], + max_length=max_length)) + self.assertLessEqual(len(out[-1]), max_length) + + # Retrieve remaining uncompressed data + while not lzd.eof: + out.append(lzd.decompress(b'', max_length=max_length)) + self.assertLessEqual(len(out[-1]), max_length) + + out = b"".join(out) + self.assertEqual(out, INPUT) + self.assertEqual(lzd.check, lzma.CHECK_CRC64) + self.assertEqual(lzd.unused_data, b"") + + def test_decompressor_inputbuf_1(self): + # Test reusing input buffer after moving existing + # contents to beginning + lzd = LZMADecompressor() + out = [] + + # Create input buffer and fill it + self.assertEqual(lzd.decompress(COMPRESSED_XZ[:100], + max_length=0), b'') + + # Retrieve some results, freeing capacity at beginning + # of input buffer + out.append(lzd.decompress(b'', 2)) + + # Add more data that fits into input buffer after + # moving existing data to beginning + out.append(lzd.decompress(COMPRESSED_XZ[100:105], 15)) + + # Decompress rest of data + out.append(lzd.decompress(COMPRESSED_XZ[105:])) + self.assertEqual(b''.join(out), INPUT) + + def test_decompressor_inputbuf_2(self): + # Test reusing input buffer by appending data at the + # end right away + lzd = LZMADecompressor() + out = [] + + # Create input buffer and empty it + self.assertEqual(lzd.decompress(COMPRESSED_XZ[:200], + max_length=0), b'') + out.append(lzd.decompress(b'')) + + # Fill buffer with new data + out.append(lzd.decompress(COMPRESSED_XZ[200:280], 2)) + + # Append some more data, not enough to require resize + out.append(lzd.decompress(COMPRESSED_XZ[280:300], 2)) + + # Decompress rest of data + out.append(lzd.decompress(COMPRESSED_XZ[300:])) + self.assertEqual(b''.join(out), INPUT) + + def test_decompressor_inputbuf_3(self): + # Test reusing input buffer after extending it + + lzd = LZMADecompressor() + out = [] + + # Create almost full input buffer + out.append(lzd.decompress(COMPRESSED_XZ[:200], 5)) + + # Add even more data to it, requiring resize + out.append(lzd.decompress(COMPRESSED_XZ[200:300], 5)) + + # Decompress rest of data + out.append(lzd.decompress(COMPRESSED_XZ[300:])) + self.assertEqual(b''.join(out), INPUT) + + def test_decompressor_unused_data(self): + lzd = LZMADecompressor() + extra = b"fooblibar" + self._test_decompressor(lzd, COMPRESSED_XZ + extra, lzma.CHECK_CRC64, + unused_data=extra) + + def test_decompressor_bad_input(self): + lzd = LZMADecompressor() + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) + + lzd = LZMADecompressor(lzma.FORMAT_XZ) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_ALONE) + + lzd = LZMADecompressor(lzma.FORMAT_ALONE) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + + def test_decompressor_bug_28275(self): + # Test coverage for Issue 28275 + lzd = LZMADecompressor() + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) + # Previously, a second call could crash due to internal inconsistency + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) + + # Test that LZMACompressor->LZMADecompressor preserves the input data. + + def test_roundtrip_xz(self): + lzc = LZMACompressor() + cdata = lzc.compress(INPUT) + lzc.flush() + lzd = LZMADecompressor() + self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) + + def test_roundtrip_alone(self): + lzc = LZMACompressor(lzma.FORMAT_ALONE) + cdata = lzc.compress(INPUT) + lzc.flush() + lzd = LZMADecompressor() + self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + + def test_roundtrip_raw(self): + lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + cdata = lzc.compress(INPUT) + lzc.flush() + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + + def test_roundtrip_raw_empty(self): + lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + cdata = lzc.compress(INPUT) + cdata += lzc.compress(b'') + cdata += lzc.compress(b'') + cdata += lzc.compress(b'') + cdata += lzc.flush() + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + + def test_roundtrip_chunks(self): + lzc = LZMACompressor() + cdata = [] + for i in range(0, len(INPUT), 10): + cdata.append(lzc.compress(INPUT[i:i+10])) + cdata.append(lzc.flush()) + cdata = b"".join(cdata) + lzd = LZMADecompressor() + self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) + + def test_roundtrip_empty_chunks(self): + lzc = LZMACompressor() + cdata = [] + for i in range(0, len(INPUT), 10): + cdata.append(lzc.compress(INPUT[i:i+10])) + cdata.append(lzc.compress(b'')) + cdata.append(lzc.compress(b'')) + cdata.append(lzc.compress(b'')) + cdata.append(lzc.flush()) + cdata = b"".join(cdata) + lzd = LZMADecompressor() + self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) + + # LZMADecompressor intentionally does not handle concatenated streams. + + def test_decompressor_multistream(self): + lzd = LZMADecompressor() + self._test_decompressor(lzd, COMPRESSED_XZ + COMPRESSED_ALONE, + lzma.CHECK_CRC64, unused_data=COMPRESSED_ALONE) + + # Test with inputs larger than 4GiB. + + @support.skip_if_pgo_task + @bigmemtest(size=_4G + 100, memuse=2) + def test_compressor_bigmem(self, size): + lzc = LZMACompressor() + cdata = lzc.compress(b"x" * size) + lzc.flush() + ddata = lzma.decompress(cdata) + try: + self.assertEqual(len(ddata), size) + self.assertEqual(len(ddata.strip(b"x")), 0) + finally: + ddata = None + + @support.skip_if_pgo_task + @bigmemtest(size=_4G + 100, memuse=3) + def test_decompressor_bigmem(self, size): + lzd = LZMADecompressor() + blocksize = min(10 * 1024 * 1024, size) + block = random.randbytes(blocksize) + try: + input = block * ((size-1) // blocksize + 1) + cdata = lzma.compress(input) + ddata = lzd.decompress(cdata) + self.assertEqual(ddata, input) + finally: + input = cdata = ddata = None + + # Pickling raises an exception; there's no way to serialize an lzma_stream. + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(LZMACompressor(), proto) + with self.assertRaises(TypeError): + pickle.dumps(LZMADecompressor(), proto) + + @support.refcount_test + def test_refleaks_in_decompressor___init__(self): + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + lzd = LZMADecompressor() + refs_before = gettotalrefcount() + for i in range(100): + lzd.__init__() + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_uninitialized_LZMADecompressor_crash(self): + self.assertEqual(LZMADecompressor.__new__(LZMADecompressor). + decompress(bytes()), b'') + + +class CompressDecompressFunctionTestCase(unittest.TestCase): + + # Test error cases: + + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Failed to initialize encoder + def test_bad_args(self): + self.assertRaises(TypeError, lzma.compress) + self.assertRaises(TypeError, lzma.compress, []) + self.assertRaises(TypeError, lzma.compress, b"", format="xz") + self.assertRaises(TypeError, lzma.compress, b"", check="none") + self.assertRaises(TypeError, lzma.compress, b"", preset="blah") + self.assertRaises(TypeError, lzma.compress, b"", filters=1024) + # Can't specify a preset and a custom filter chain at the same time. + with self.assertRaises(ValueError): + lzma.compress(b"", preset=3, filters=[{"id": lzma.FILTER_LZMA2}]) + + self.assertRaises(TypeError, lzma.decompress) + self.assertRaises(TypeError, lzma.decompress, []) + self.assertRaises(TypeError, lzma.decompress, b"", format="lzma") + self.assertRaises(TypeError, lzma.decompress, b"", memlimit=7.3e9) + with self.assertRaises(TypeError): + lzma.decompress(b"", format=lzma.FORMAT_RAW, filters={}) + # Cannot specify a memory limit with FILTER_RAW. + with self.assertRaises(ValueError): + lzma.decompress(b"", format=lzma.FORMAT_RAW, memlimit=0x1000000) + # Can only specify a custom filter chain with FILTER_RAW. + with self.assertRaises(ValueError): + lzma.decompress(b"", filters=FILTERS_RAW_1) + with self.assertRaises(ValueError): + lzma.decompress(b"", format=lzma.FORMAT_XZ, filters=FILTERS_RAW_1) + with self.assertRaises(ValueError): + lzma.decompress( + b"", format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1) + + def test_decompress_memlimit(self): + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_XZ, memlimit=1024) + with self.assertRaises(LZMAError): + lzma.decompress( + COMPRESSED_XZ, format=lzma.FORMAT_XZ, memlimit=1024) + with self.assertRaises(LZMAError): + lzma.decompress( + COMPRESSED_ALONE, format=lzma.FORMAT_ALONE, memlimit=1024) + + # Test LZMADecompressor on known-good input data. + + def test_decompress_good_input(self): + ddata = lzma.decompress(COMPRESSED_XZ) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress(COMPRESSED_ALONE) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress(COMPRESSED_XZ, lzma.FORMAT_XZ) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress(COMPRESSED_ALONE, lzma.FORMAT_ALONE) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress( + COMPRESSED_RAW_1, lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress( + COMPRESSED_RAW_2, lzma.FORMAT_RAW, filters=FILTERS_RAW_2) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress( + COMPRESSED_RAW_3, lzma.FORMAT_RAW, filters=FILTERS_RAW_3) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress( + COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self.assertEqual(ddata, INPUT) + + def test_decompress_incomplete_input(self): + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ[:128]) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_ALONE[:128]) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_1[:128], + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_2[:128], + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_3[:128], + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_4[:128], + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + + def test_decompress_bad_input(self): + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_BOGUS) + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_RAW_1) + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_ALONE, format=lzma.FORMAT_XZ) + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_XZ, format=lzma.FORMAT_ALONE) + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_XZ, format=lzma.FORMAT_RAW, + filters=FILTERS_RAW_1) + + # Test that compress()->decompress() preserves the input data. + + def test_roundtrip(self): + cdata = lzma.compress(INPUT) + ddata = lzma.decompress(cdata) + self.assertEqual(ddata, INPUT) + + cdata = lzma.compress(INPUT, lzma.FORMAT_XZ) + ddata = lzma.decompress(cdata) + self.assertEqual(ddata, INPUT) + + cdata = lzma.compress(INPUT, lzma.FORMAT_ALONE) + ddata = lzma.decompress(cdata) + self.assertEqual(ddata, INPUT) + + cdata = lzma.compress(INPUT, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + ddata = lzma.decompress(cdata, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self.assertEqual(ddata, INPUT) + + # Unlike LZMADecompressor, decompress() *does* handle concatenated streams. + + def test_decompress_multistream(self): + ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_ALONE) + self.assertEqual(ddata, INPUT * 2) + + # Test robust handling of non-LZMA data following the compressed stream(s). + + def test_decompress_trailing_junk(self): + ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_BOGUS) + self.assertEqual(ddata, INPUT) + + def test_decompress_multistream_trailing_junk(self): + ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS) + self.assertEqual(ddata, INPUT * 3) + + +class TempFile: + """Context manager - creates a file, and deletes it on __exit__.""" + + def __init__(self, filename, data=b""): + self.filename = filename + self.data = data + + def __enter__(self): + with open(self.filename, "wb") as f: + f.write(self.data) + + def __exit__(self, *args): + unlink(self.filename) + + +class FileTestCase(unittest.TestCase): + + def test_init(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "rb") + with LZMAFile(BytesIO(), "w") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(BytesIO(), "x") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(BytesIO(), "a") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <FakePath '@test_23396_tmp챈'> != '@test_23396_tmp챈' + def test_init_with_PathLike_filename(self): + filename = FakePath(TESTFN) + with TempFile(filename, COMPRESSED_XZ): + with LZMAFile(filename) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) + with LZMAFile(filename, "a") as f: + f.write(INPUT) + self.assertEqual(f.name, TESTFN) + with LZMAFile(filename) as f: + self.assertEqual(f.read(), INPUT * 2) + self.assertEqual(f.name, TESTFN) + + def test_init_with_filename(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with LZMAFile(TESTFN) as f: + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'rb') + with LZMAFile(TESTFN, "w") as f: + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') + with LZMAFile(TESTFN, "a") as f: + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') + + def test_init_mode(self): + with TempFile(TESTFN): + with LZMAFile(TESTFN, "r") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "rb") + with LZMAFile(TESTFN, "rb") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "rb") + with LZMAFile(TESTFN, "w") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(TESTFN, "wb") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(TESTFN, "a") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(TESTFN, "ab") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + + def test_init_with_x_mode(self): + self.addCleanup(unlink, TESTFN) + for mode in ("x", "xb"): + unlink(TESTFN) + with LZMAFile(TESTFN, mode) as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(FileExistsError): + LZMAFile(TESTFN, mode) + + def test_init_bad_mode(self): + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), (3, "x")) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "xt") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "x+") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "rx") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "wx") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "rt") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "r+") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "wt") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "w+") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "rw") + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Invalid check value + def test_init_bad_check(self): + with self.assertRaises(TypeError): + LZMAFile(BytesIO(), "w", check=b"asd") + # CHECK_UNKNOWN and anything above CHECK_ID_MAX should be invalid. + with self.assertRaises(LZMAError): + LZMAFile(BytesIO(), "w", check=lzma.CHECK_UNKNOWN) + with self.assertRaises(LZMAError): + LZMAFile(BytesIO(), "w", check=lzma.CHECK_ID_MAX + 3) + # Cannot specify a check with mode="r". + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_NONE) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_CRC32) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_CRC64) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_SHA256) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_UNKNOWN) + + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u32 + def test_init_bad_preset(self): + with self.assertRaises(TypeError): + LZMAFile(BytesIO(), "w", preset=4.39) + with self.assertRaises(LZMAError): + LZMAFile(BytesIO(), "w", preset=10) + with self.assertRaises(LZMAError): + LZMAFile(BytesIO(), "w", preset=23) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", preset=-1) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", preset=-7) + with self.assertRaises(OverflowError): + LZMAFile(BytesIO(), "w", preset=2**1000) + with self.assertRaises(TypeError): + LZMAFile(BytesIO(), "w", preset="foo") + # Cannot specify a preset with mode="r". + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), preset=3) + + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Failed to initialize encoder + def test_init_bad_filter_spec(self): + with self.assertRaises(TypeError): + LZMAFile(BytesIO(), "w", filters=[b"wobsite"]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", filters=[{"xyzzy": 3}]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", filters=[{"id": 98765}]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", + filters=[{"id": lzma.FILTER_LZMA2, "foo": 0}]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", + filters=[{"id": lzma.FILTER_DELTA, "foo": 0}]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", + filters=[{"id": lzma.FILTER_X86, "foo": 0}]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; lzma.LZMAError: Invalid format + def test_init_with_preset_and_filters(self): + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", format=lzma.FORMAT_RAW, + preset=6, filters=FILTERS_RAW_1) + + def test_close(self): + with BytesIO(COMPRESSED_XZ) as src: + f = LZMAFile(src) + f.close() + # LZMAFile.close() should not close the underlying file object. + self.assertFalse(src.closed) + # Try closing an already-closed LZMAFile. + f.close() + self.assertFalse(src.closed) + + # Test with a real file on disk, opened directly by LZMAFile. + with TempFile(TESTFN, COMPRESSED_XZ): + f = LZMAFile(TESTFN) + fp = f._fp + f.close() + # Here, LZMAFile.close() *should* close the underlying file object. + self.assertTrue(fp.closed) + # Try closing an already-closed LZMAFile. + f.close() + + def test_closed(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertFalse(f.closed) + f.read() + self.assertFalse(f.closed) + finally: + f.close() + self.assertTrue(f.closed) + + f = LZMAFile(BytesIO(), "w") + try: + self.assertFalse(f.closed) + finally: + f.close() + self.assertTrue(f.closed) + + def test_fileno(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertRaises(UnsupportedOperation, f.fileno) + finally: + f.close() + self.assertRaises(ValueError, f.fileno) + with TempFile(TESTFN, COMPRESSED_XZ): + f = LZMAFile(TESTFN) + try: + self.assertEqual(f.fileno(), f._fp.fileno()) + self.assertIsInstance(f.fileno(), int) + finally: + f.close() + self.assertRaises(ValueError, f.fileno) + + def test_seekable(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertTrue(f.seekable()) + f.read() + self.assertTrue(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + f = LZMAFile(BytesIO(), "w") + try: + self.assertFalse(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + src = BytesIO(COMPRESSED_XZ) + src.seekable = lambda: False + f = LZMAFile(src) + try: + self.assertFalse(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + def test_readable(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertTrue(f.readable()) + f.read() + self.assertTrue(f.readable()) + finally: + f.close() + self.assertRaises(ValueError, f.readable) + + f = LZMAFile(BytesIO(), "w") + try: + self.assertFalse(f.readable()) + finally: + f.close() + self.assertRaises(ValueError, f.readable) + + def test_writable(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertFalse(f.writable()) + f.read() + self.assertFalse(f.writable()) + finally: + f.close() + self.assertRaises(ValueError, f.writable) + + f = LZMAFile(BytesIO(), "w") + try: + self.assertTrue(f.writable()) + finally: + f.close() + self.assertRaises(ValueError, f.writable) + + def test_read(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f: + self.assertEqual(f.read(), INPUT) + with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_RAW_1), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_1) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_RAW_2), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_RAW_3), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_RAW_4), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + + def test_read_0(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertEqual(f.read(0), b"") + with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f: + self.assertEqual(f.read(0), b"") + with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f: + self.assertEqual(f.read(0), b"") + with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f: + self.assertEqual(f.read(0), b"") + + def test_read_10(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + chunks = [] + while result := f.read(10): + self.assertLessEqual(len(result), 10) + chunks.append(result) + self.assertEqual(b"".join(chunks), INPUT) + + def test_read_multistream(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: + self.assertEqual(f.read(), INPUT * 5) + with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_ALONE)) as f: + self.assertEqual(f.read(), INPUT * 2) + with LZMAFile(BytesIO(COMPRESSED_RAW_3 * 4), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3) as f: + self.assertEqual(f.read(), INPUT * 4) + + def test_read_multistream_buffer_size_aligned(self): + # Test the case where a stream boundary coincides with the end + # of the raw read buffer. + saved_buffer_size = _streams.BUFFER_SIZE + _streams.BUFFER_SIZE = len(COMPRESSED_XZ) + try: + with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: + self.assertEqual(f.read(), INPUT * 5) + finally: + _streams.BUFFER_SIZE = saved_buffer_size + + def test_read_trailing_junk(self): + with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_BOGUS)) as f: + self.assertEqual(f.read(), INPUT) + + def test_read_multistream_trailing_junk(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 5 + COMPRESSED_BOGUS)) as f: + self.assertEqual(f.read(), INPUT * 5) + + def test_read_from_file(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with LZMAFile(TESTFN) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, TESTFN) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_file_with_bytes_filename(self): + bytes_filename = os.fsencode(TESTFN) + with TempFile(TESTFN, COMPRESSED_XZ): + with LZMAFile(bytes_filename) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, bytes_filename) + + def test_read_from_fileobj(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with open(TESTFN, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_fileobj_with_int_name(self): + with TempFile(TESTFN, COMPRESSED_XZ): + fd = os.open(TESTFN, os.O_RDONLY) + with open(fd, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_incomplete(self): + with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f: + self.assertRaises(EOFError, f.read) + + def test_read_truncated(self): + # Drop stream footer: CRC (4 bytes), index size (4 bytes), + # flags (2 bytes) and magic number (2 bytes). + truncated = COMPRESSED_XZ[:-12] + with LZMAFile(BytesIO(truncated)) as f: + self.assertRaises(EOFError, f.read) + with LZMAFile(BytesIO(truncated)) as f: + self.assertEqual(f.read(len(INPUT)), INPUT) + self.assertRaises(EOFError, f.read, 1) + # Incomplete 12-byte header. + for i in range(12): + with LZMAFile(BytesIO(truncated[:i])) as f: + self.assertRaises(EOFError, f.read, 1) + + def test_read_bad_args(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + f.close() + self.assertRaises(ValueError, f.read) + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(ValueError, f.read) + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertRaises(TypeError, f.read, float()) + + def test_read_bad_data(self): + with LZMAFile(BytesIO(COMPRESSED_BOGUS)) as f: + self.assertRaises(LZMAError, f.read) + + def test_read1(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + blocks = [] + while result := f.read1(): + blocks.append(result) + self.assertEqual(b"".join(blocks), INPUT) + self.assertEqual(f.read1(), b"") + + def test_read1_0(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertEqual(f.read1(0), b"") + + def test_read1_10(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + blocks = [] + while result := f.read1(10): + blocks.append(result) + self.assertEqual(b"".join(blocks), INPUT) + self.assertEqual(f.read1(), b"") + + def test_read1_multistream(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: + blocks = [] + while result := f.read1(): + blocks.append(result) + self.assertEqual(b"".join(blocks), INPUT * 5) + self.assertEqual(f.read1(), b"") + + def test_read1_bad_args(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + f.close() + self.assertRaises(ValueError, f.read1) + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(ValueError, f.read1) + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertRaises(TypeError, f.read1, None) + + def test_peek(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + result = f.peek() + self.assertGreater(len(result), 0) + self.assertStartsWith(INPUT, result) + self.assertEqual(f.read(), INPUT) + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + result = f.peek(10) + self.assertGreater(len(result), 0) + self.assertStartsWith(INPUT, result) + self.assertEqual(f.read(), INPUT) + + def test_peek_bad_args(self): + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(ValueError, f.peek) + + def test_iterator(self): + with BytesIO(INPUT) as f: + lines = f.readlines() + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertListEqual(list(iter(f)), lines) + with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f: + self.assertListEqual(list(iter(f)), lines) + with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f: + self.assertListEqual(list(iter(f)), lines) + with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f: + self.assertListEqual(list(iter(f)), lines) + with LZMAFile(BytesIO(COMPRESSED_RAW_2), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2) as f: + self.assertListEqual(list(iter(f)), lines) + + def test_readline(self): + with BytesIO(INPUT) as f: + lines = f.readlines() + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + for line in lines: + self.assertEqual(f.readline(), line) + + def test_readlines(self): + with BytesIO(INPUT) as f: + lines = f.readlines() + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertListEqual(f.readlines(), lines) + + def test_decompress_limited(self): + """Decompressed data buffering should be limited""" + bomb = lzma.compress(b'\0' * int(2e6), preset=6) + self.assertLess(len(bomb), _streams.BUFFER_SIZE) + + decomp = LZMAFile(BytesIO(bomb)) + self.assertEqual(decomp.read(1), b'\0') + max_decomp = 1 + DEFAULT_BUFFER_SIZE + self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, + "Excessive amount of data was decompressed") + + def test_write(self): + with BytesIO() as dst: + with LZMAFile(dst, "w") as f: + f.write(INPUT) + with self.assertRaises(AttributeError): + f.name + expected = lzma.compress(INPUT) + self.assertEqual(dst.getvalue(), expected) + with BytesIO() as dst: + with LZMAFile(dst, "w", format=lzma.FORMAT_XZ) as f: + f.write(INPUT) + expected = lzma.compress(INPUT, format=lzma.FORMAT_XZ) + self.assertEqual(dst.getvalue(), expected) + with BytesIO() as dst: + with LZMAFile(dst, "w", format=lzma.FORMAT_ALONE) as f: + f.write(INPUT) + expected = lzma.compress(INPUT, format=lzma.FORMAT_ALONE) + self.assertEqual(dst.getvalue(), expected) + with BytesIO() as dst: + with LZMAFile(dst, "w", format=lzma.FORMAT_RAW, + filters=FILTERS_RAW_2) as f: + f.write(INPUT) + expected = lzma.compress(INPUT, format=lzma.FORMAT_RAW, + filters=FILTERS_RAW_2) + self.assertEqual(dst.getvalue(), expected) + + def test_write_10(self): + with BytesIO() as dst: + with LZMAFile(dst, "w") as f: + for start in range(0, len(INPUT), 10): + f.write(INPUT[start:start+10]) + expected = lzma.compress(INPUT) + self.assertEqual(dst.getvalue(), expected) + + def test_write_append(self): + part1 = INPUT[:1024] + part2 = INPUT[1024:1536] + part3 = INPUT[1536:] + expected = b"".join(lzma.compress(x) for x in (part1, part2, part3)) + with BytesIO() as dst: + with LZMAFile(dst, "w") as f: + f.write(part1) + self.assertEqual(f.mode, 'wb') + with LZMAFile(dst, "a") as f: + f.write(part2) + self.assertEqual(f.mode, 'wb') + with LZMAFile(dst, "a") as f: + f.write(part3) + self.assertEqual(f.mode, 'wb') + self.assertEqual(dst.getvalue(), expected) + + def test_write_to_file(self): + try: + with LZMAFile(TESTFN, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, TESTFN) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_file_with_bytes_filename(self): + bytes_filename = os.fsencode(TESTFN) + try: + with LZMAFile(bytes_filename, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, bytes_filename) + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj(self): + try: + with open(TESTFN, "wb") as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj_with_int_name(self): + try: + fd = os.open(TESTFN, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_append_to_file(self): + part1 = INPUT[:1024] + part2 = INPUT[1024:1536] + part3 = INPUT[1536:] + expected = b"".join(lzma.compress(x) for x in (part1, part2, part3)) + try: + with LZMAFile(TESTFN, "w") as f: + f.write(part1) + self.assertEqual(f.mode, 'wb') + with LZMAFile(TESTFN, "a") as f: + f.write(part2) + self.assertEqual(f.mode, 'wb') + with LZMAFile(TESTFN, "a") as f: + f.write(part3) + self.assertEqual(f.mode, 'wb') + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_bad_args(self): + f = LZMAFile(BytesIO(), "w") + f.close() + self.assertRaises(ValueError, f.write, b"foo") + with LZMAFile(BytesIO(COMPRESSED_XZ), "r") as f: + self.assertRaises(ValueError, f.write, b"bar") + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(TypeError, f.write, None) + self.assertRaises(TypeError, f.write, "text") + self.assertRaises(TypeError, f.write, 789) + + def test_writelines(self): + with BytesIO(INPUT) as f: + lines = f.readlines() + with BytesIO() as dst: + with LZMAFile(dst, "w") as f: + f.writelines(lines) + expected = lzma.compress(INPUT) + self.assertEqual(dst.getvalue(), expected) + + def test_seek_forward(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(555) + self.assertEqual(f.read(), INPUT[555:]) + + def test_seek_forward_across_streams(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 2)) as f: + f.seek(len(INPUT) + 123) + self.assertEqual(f.read(), INPUT[123:]) + + def test_seek_forward_relative_to_current(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.read(100) + f.seek(1236, 1) + self.assertEqual(f.read(), INPUT[1336:]) + + def test_seek_forward_relative_to_end(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(-555, 2) + self.assertEqual(f.read(), INPUT[-555:]) + + def test_seek_backward(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.read(1001) + f.seek(211) + self.assertEqual(f.read(), INPUT[211:]) + + def test_seek_backward_across_streams(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 2)) as f: + f.read(len(INPUT) + 333) + f.seek(737) + self.assertEqual(f.read(), INPUT[737:] + INPUT) + + def test_seek_backward_relative_to_end(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(-150, 2) + self.assertEqual(f.read(), INPUT[-150:]) + + def test_seek_past_end(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(len(INPUT) + 9001) + self.assertEqual(f.tell(), len(INPUT)) + self.assertEqual(f.read(), b"") + + def test_seek_past_start(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(-88) + self.assertEqual(f.tell(), 0) + self.assertEqual(f.read(), INPUT) + + def test_seek_bad_args(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + f.close() + self.assertRaises(ValueError, f.seek, 0) + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(ValueError, f.seek, 0) + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertRaises(ValueError, f.seek, 0, 3) + # io.BufferedReader raises TypeError instead of ValueError + self.assertRaises((TypeError, ValueError), f.seek, 9, ()) + self.assertRaises(TypeError, f.seek, None) + self.assertRaises(TypeError, f.seek, b"derp") + + def test_tell(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + pos = 0 + while True: + self.assertEqual(f.tell(), pos) + result = f.read(183) + if not result: + break + pos += len(result) + self.assertEqual(f.tell(), len(INPUT)) + with LZMAFile(BytesIO(), "w") as f: + for pos in range(0, len(INPUT), 144): + self.assertEqual(f.tell(), pos) + f.write(INPUT[pos:pos+144]) + self.assertEqual(f.tell(), len(INPUT)) + + def test_tell_bad_args(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + f.close() + self.assertRaises(ValueError, f.tell) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: True is not false + def test_issue21872(self): + # sometimes decompress data incompletely + + # --------------------- + # when max_length == -1 + # --------------------- + d1 = LZMADecompressor() + entire = d1.decompress(ISSUE_21872_DAT, max_length=-1) + self.assertEqual(len(entire), 13160) + self.assertTrue(d1.eof) + + # --------------------- + # when max_length > 0 + # --------------------- + d2 = LZMADecompressor() + + # When this value of max_length is used, the input and output + # buffers are exhausted at the same time, and lzs's internal + # state still have 11 bytes can be output. + out1 = d2.decompress(ISSUE_21872_DAT, max_length=13149) + self.assertFalse(d2.needs_input) # ensure needs_input mechanism works + self.assertFalse(d2.eof) + + # simulate needs_input mechanism + # output internal state's 11 bytes + out2 = d2.decompress(b'') + self.assertEqual(len(out2), 11) + self.assertTrue(d2.eof) + self.assertEqual(out1 + out2, entire) + + def test_issue44439(self): + q = array.array('Q', [1, 2, 3, 4, 5]) + LENGTH = len(q) * q.itemsize + + with LZMAFile(BytesIO(), 'w') as f: + self.assertEqual(f.write(q), LENGTH) + self.assertEqual(f.tell(), LENGTH) + + +class OpenTestCase(unittest.TestCase): + + def test_binary_modes(self): + with lzma.open(BytesIO(COMPRESSED_XZ), "rb") as f: + self.assertEqual(f.read(), INPUT) + with BytesIO() as bio: + with lzma.open(bio, "wb") as f: + f.write(INPUT) + file_data = lzma.decompress(bio.getvalue()) + self.assertEqual(file_data, INPUT) + with lzma.open(bio, "ab") as f: + f.write(INPUT) + file_data = lzma.decompress(bio.getvalue()) + self.assertEqual(file_data, INPUT * 2) + + def test_text_modes(self): + uncompressed = INPUT.decode("ascii") + uncompressed_raw = uncompressed.replace("\n", os.linesep) + with lzma.open(BytesIO(COMPRESSED_XZ), "rt", encoding="ascii") as f: + self.assertEqual(f.read(), uncompressed) + with BytesIO() as bio: + with lzma.open(bio, "wt", encoding="ascii") as f: + f.write(uncompressed) + file_data = lzma.decompress(bio.getvalue()).decode("ascii") + self.assertEqual(file_data, uncompressed_raw) + with lzma.open(bio, "at", encoding="ascii") as f: + f.write(uncompressed) + file_data = lzma.decompress(bio.getvalue()).decode("ascii") + self.assertEqual(file_data, uncompressed_raw * 2) + + def test_filename(self): + with TempFile(TESTFN): + with lzma.open(TESTFN, "wb") as f: + f.write(INPUT) + with open(TESTFN, "rb") as f: + file_data = lzma.decompress(f.read()) + self.assertEqual(file_data, INPUT) + with lzma.open(TESTFN, "rb") as f: + self.assertEqual(f.read(), INPUT) + with lzma.open(TESTFN, "ab") as f: + f.write(INPUT) + with lzma.open(TESTFN, "rb") as f: + self.assertEqual(f.read(), INPUT * 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <FakePath '@test_23396_tmp챈'> != '@test_23396_tmp챈' + def test_with_pathlike_filename(self): + filename = FakePath(TESTFN) + with TempFile(filename): + with lzma.open(filename, "wb") as f: + f.write(INPUT) + self.assertEqual(f.name, TESTFN) + with open(filename, "rb") as f: + file_data = lzma.decompress(f.read()) + self.assertEqual(file_data, INPUT) + with lzma.open(filename, "rb") as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) + + def test_bad_params(self): + # Test invalid parameter combinations. + with self.assertRaises(ValueError): + lzma.open(TESTFN, "") + with self.assertRaises(ValueError): + lzma.open(TESTFN, "rbt") + with self.assertRaises(ValueError): + lzma.open(TESTFN, "rb", encoding="utf-8") + with self.assertRaises(ValueError): + lzma.open(TESTFN, "rb", errors="ignore") + with self.assertRaises(ValueError): + lzma.open(TESTFN, "rb", newline="\n") + + def test_format_and_filters(self): + # Test non-default format and filter chain. + options = {"format": lzma.FORMAT_RAW, "filters": FILTERS_RAW_1} + with lzma.open(BytesIO(COMPRESSED_RAW_1), "rb", **options) as f: + self.assertEqual(f.read(), INPUT) + with BytesIO() as bio: + with lzma.open(bio, "wb", **options) as f: + f.write(INPUT) + file_data = lzma.decompress(bio.getvalue(), **options) + self.assertEqual(file_data, INPUT) + + def test_encoding(self): + # Test non-default encoding. + uncompressed = INPUT.decode("ascii") + uncompressed_raw = uncompressed.replace("\n", os.linesep) + with BytesIO() as bio: + with lzma.open(bio, "wt", encoding="utf-16-le") as f: + f.write(uncompressed) + file_data = lzma.decompress(bio.getvalue()).decode("utf-16-le") + self.assertEqual(file_data, uncompressed_raw) + bio.seek(0) + with lzma.open(bio, "rt", encoding="utf-16-le") as f: + self.assertEqual(f.read(), uncompressed) + + def test_encoding_error_handler(self): + # Test with non-default encoding error handler. + with BytesIO(lzma.compress(b"foo\xffbar")) as bio: + with lzma.open(bio, "rt", encoding="ascii", errors="ignore") as f: + self.assertEqual(f.read(), "foobar") + + def test_newline(self): + # Test with explicit newline (universal newline mode disabled). + text = INPUT.decode("ascii") + with BytesIO() as bio: + with lzma.open(bio, "wt", encoding="ascii", newline="\n") as f: + f.write(text) + bio.seek(0) + with lzma.open(bio, "rt", encoding="ascii", newline="\r") as f: + self.assertEqual(f.readlines(), [text]) + + def test_x_mode(self): + self.addCleanup(unlink, TESTFN) + for mode in ("x", "xb", "xt"): + unlink(TESTFN) + encoding = "ascii" if "t" in mode else None + with lzma.open(TESTFN, mode, encoding=encoding): + pass + with self.assertRaises(FileExistsError): + with lzma.open(TESTFN, mode): + pass + + +class MiscellaneousTestCase(unittest.TestCase): + + def test_is_check_supported(self): + # CHECK_NONE and CHECK_CRC32 should always be supported, + # regardless of the options liblzma was compiled with. + self.assertTrue(lzma.is_check_supported(lzma.CHECK_NONE)) + self.assertTrue(lzma.is_check_supported(lzma.CHECK_CRC32)) + + # The .xz format spec cannot store check IDs above this value. + self.assertFalse(lzma.is_check_supported(lzma.CHECK_ID_MAX + 1)) + + # This value should not be a valid check ID. + self.assertFalse(lzma.is_check_supported(lzma.CHECK_UNKNOWN)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 0 arguments, got 1 + def test__encode_filter_properties(self): + with self.assertRaises(TypeError): + lzma._encode_filter_properties(b"not a dict") + with self.assertRaises(ValueError): + lzma._encode_filter_properties({"id": 0x100}) + with self.assertRaises(ValueError): + lzma._encode_filter_properties({"id": lzma.FILTER_LZMA2, "junk": 12}) + with self.assertRaises(lzma.LZMAError): + lzma._encode_filter_properties({"id": lzma.FILTER_DELTA, + "dist": 9001}) + + # Test with parameters used by zipfile module. + props = lzma._encode_filter_properties({ + "id": lzma.FILTER_LZMA1, + "pb": 2, + "lp": 0, + "lc": 3, + "dict_size": 8 << 20, + }) + self.assertEqual(props, b"]\x00\x00\x80\x00") + + def test__decode_filter_properties(self): + with self.assertRaises(TypeError): + lzma._decode_filter_properties(lzma.FILTER_X86, {"should be": bytes}) + with self.assertRaises(lzma.LZMAError): + lzma._decode_filter_properties(lzma.FILTER_DELTA, b"too long") + + # Test with parameters used by zipfile module. + filterspec = lzma._decode_filter_properties( + lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") + self.assertEqual(filterspec["id"], lzma.FILTER_LZMA1) + self.assertEqual(filterspec["pb"], 2) + self.assertEqual(filterspec["lp"], 0) + self.assertEqual(filterspec["lc"], 3) + self.assertEqual(filterspec["dict_size"], 8 << 20) + + # see gh-104282 + filters = [lzma.FILTER_X86, lzma.FILTER_POWERPC, + lzma.FILTER_IA64, lzma.FILTER_ARM, + lzma.FILTER_ARMTHUMB, lzma.FILTER_SPARC] + for f in filters: + filterspec = lzma._decode_filter_properties(f, b"") + self.assertEqual(filterspec, {"id": f}) + + def test_filter_properties_roundtrip(self): + spec1 = lzma._decode_filter_properties( + lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") + reencoded = lzma._encode_filter_properties(spec1) + spec2 = lzma._decode_filter_properties(lzma.FILTER_LZMA1, reencoded) + self.assertEqual(spec1, spec2) + + +# Test data: + +INPUT = b""" +LAERTES + + O, fear me not. + I stay too long: but here my father comes. + + Enter POLONIUS + + A double blessing is a double grace, + Occasion smiles upon a second leave. + +LORD POLONIUS + + Yet here, Laertes! aboard, aboard, for shame! + The wind sits in the shoulder of your sail, + And you are stay'd for. There; my blessing with thee! + And these few precepts in thy memory + See thou character. Give thy thoughts no tongue, + Nor any unproportioned thought his act. + Be thou familiar, but by no means vulgar. + Those friends thou hast, and their adoption tried, + Grapple them to thy soul with hoops of steel; + But do not dull thy palm with entertainment + Of each new-hatch'd, unfledged comrade. Beware + Of entrance to a quarrel, but being in, + Bear't that the opposed may beware of thee. + Give every man thy ear, but few thy voice; + Take each man's censure, but reserve thy judgment. + Costly thy habit as thy purse can buy, + But not express'd in fancy; rich, not gaudy; + For the apparel oft proclaims the man, + And they in France of the best rank and station + Are of a most select and generous chief in that. + Neither a borrower nor a lender be; + For loan oft loses both itself and friend, + And borrowing dulls the edge of husbandry. + This above all: to thine ownself be true, + And it must follow, as the night the day, + Thou canst not then be false to any man. + Farewell: my blessing season this in thee! + +LAERTES + + Most humbly do I take my leave, my lord. + +LORD POLONIUS + + The time invites you; go; your servants tend. + +LAERTES + + Farewell, Ophelia; and remember well + What I have said to you. + +OPHELIA + + 'Tis in my memory lock'd, + And you yourself shall keep the key of it. + +LAERTES + + Farewell. +""" + +COMPRESSED_BOGUS = b"this is not a valid lzma stream" + +COMPRESSED_XZ = ( + b"\xfd7zXZ\x00\x00\x04\xe6\xd6\xb4F\x02\x00!\x01\x16\x00\x00\x00t/\xe5\xa3" + b"\xe0\x07\x80\x03\xdf]\x00\x05\x14\x07bX\x19\xcd\xddn\x98\x15\xe4\xb4\x9d" + b"o\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8\xe2\xfc\xe7\xd9\xfe6\xb8(" + b"\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02\x17/\xa6=\xf0\xa2\xdf/M\x89" + b"\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ\x15\x80\x8c\xf8\x8do\xfa\x12" + b"\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t\xca6 BF$\xe5Q\xa4\x98\xee\xde" + b"l\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81\xe4N\xc8\x86\x153\xf5x2\xa2O" + b"\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z\xc4\xcdS\xb6t<\x16\xf2\x9cI#" + b"\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0\xaa\x96-Pe\xade:\x04\t\x1b\xf7" + b"\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b" + b"\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa" + b"\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8\x84b\xf6\xc3\xd4c-H\x93oJl\xd0iQ\xe4k" + b"\x84\x0b\xc1\xb7\xbc\xb1\x17\x88\xb1\xca?@\xf6\x07\xea\xe6x\xf1H12P\x0f" + b"\x8a\xc9\xeauw\xe3\xbe\xaai\xa9W\xd0\x80\xcd#cb5\x99\xd8]\xa9d\x0c\xbd" + b"\xa2\xdcWl\xedUG\xbf\x89yF\xf77\x81v\xbd5\x98\xbeh8\x18W\x08\xf0\x1b\x99" + b"5:\x1a?rD\x96\xa1\x04\x0f\xae\xba\x85\xeb\x9d5@\xf5\x83\xd37\x83\x8ac" + b"\x06\xd4\x97i\xcdt\x16S\x82k\xf6K\x01vy\x88\x91\x9b6T\xdae\r\xfd]:k\xbal" + b"\xa9\xbba\xc34\xf9r\xeb}r\xdb\xc7\xdb*\x8f\x03z\xdc8h\xcc\xc9\xd3\xbcl" + b"\xa5-\xcb\xeaK\xa2\xc5\x15\xc0\xe3\xc1\x86Z\xfb\xebL\xe13\xcf\x9c\xe3" + b"\x1d\xc9\xed\xc2\x06\xcc\xce!\x92\xe5\xfe\x9c^\xa59w \x9bP\xa3PK\x08d" + b"\xf9\xe2Z}\xa7\xbf\xed\xeb%$\x0c\x82\xb8/\xb0\x01\xa9&,\xf7qh{Q\x96)\xf2" + b"q\x96\xc3\x80\xb4\x12\xb0\xba\xe6o\xf4!\xb4[\xd4\x8aw\x10\xf7t\x0c\xb3" + b"\xd9\xd5\xc3`^\x81\x11??\\\xa4\x99\x85R\xd4\x8e\x83\xc9\x1eX\xbfa\xf1" + b"\xac\xb0\xea\xea\xd7\xd0\xab\x18\xe2\xf2\xed\xe1\xb7\xc9\x18\xcbS\xe4>" + b"\xc9\x95H\xe8\xcb\t\r%\xeb\xc7$.o\xf1\xf3R\x17\x1db\xbb\xd8U\xa5^\xccS" + b"\x16\x01\x87\xf3/\x93\xd1\xf0v\xc0r\xd7\xcc\xa2Gkz\xca\x80\x0e\xfd\xd0" + b"\x8b\xbb\xd2Ix\xb3\x1ey\xca-0\xe3z^\xd6\xd6\x8f_\xf1\x9dP\x9fi\xa7\xd1" + b"\xe8\x90\x84\xdc\xbf\xcdky\x8e\xdc\x81\x7f\xa3\xb2+\xbf\x04\xef\xd8\\" + b"\xc4\xdf\xe1\xb0\x01\xe9\x93\xe3Y\xf1\x1dY\xe8h\x81\xcf\xf1w\xcc\xb4\xef" + b" \x8b|\x04\xea\x83ej\xbe\x1f\xd4z\x9c`\xd3\x1a\x92A\x06\xe5\x8f\xa9\x13" + b"\t\x9e=\xfa\x1c\xe5_\x9f%v\x1bo\x11ZO\xd8\xf4\t\xddM\x16-\x04\xfc\x18<\"" + b"CM\xddg~b\xf6\xef\x8e\x0c\xd0\xde|\xa0'\x8a\x0c\xd6x\xae!J\xa6F\x88\x15u" + b"\x008\x17\xbc7y\xb3\xd8u\xac_\x85\x8d\xe7\xc1@\x9c\xecqc\xa3#\xad\xf1" + b"\x935\xb5)_\r\xec3]\x0fo]5\xd0my\x07\x9b\xee\x81\xb5\x0f\xcfK+\x00\xc0" + b"\xe4b\x10\xe4\x0c\x1a \x9b\xe0\x97t\xf6\xa1\x9e\x850\xba\x0c\x9a\x8d\xc8" + b"\x8f\x07\xd7\xae\xc8\xf9+i\xdc\xb9k\xb0>f\x19\xb8\r\xa8\xf8\x1f$\xa5{p" + b"\xc6\x880\xce\xdb\xcf\xca_\x86\xac\x88h6\x8bZ%'\xd0\n\xbf\x0f\x9c\"\xba" + b"\xe5\x86\x9f\x0f7X=mNX[\xcc\x19FU\xc9\x860\xbc\x90a+* \xae_$\x03\x1e\xd3" + b"\xcd_\xa0\x9c\xde\xaf46q\xa5\xc9\x92\xd7\xca\xe3`\x9d\x85}\xb4\xff\xb3" + b"\x83\xfb\xb6\xca\xae`\x0bw\x7f\xfc\xd8\xacVe\x19\xc8\x17\x0bZ\xad\x88" + b"\xeb#\x97\x03\x13\xb1d\x0f{\x0c\x04w\x07\r\x97\xbd\xd6\xc1\xc3B:\x95\x08" + b"^\x10V\xaeaH\x02\xd9\xe3\n\\\x01X\xf6\x9c\x8a\x06u#%\xbe*\xa1\x18v\x85" + b"\xec!\t4\x00\x00\x00\x00Vj?uLU\xf3\xa6\x00\x01\xfb\x07\x81\x0f\x00\x00tw" + b"\x99P\xb1\xc4g\xfb\x02\x00\x00\x00\x00\x04YZ" +) + +COMPRESSED_ALONE = ( + b"]\x00\x00\x80\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x05\x14\x07bX\x19" + b"\xcd\xddn\x98\x15\xe4\xb4\x9do\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8" + b"\xe2\xfc\xe7\xd9\xfe6\xb8(\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02" + b"\x17/\xa6=\xf0\xa2\xdf/M\x89\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ" + b"\x15\x80\x8c\xf8\x8do\xfa\x12\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t" + b"\xca6 BF$\xe5Q\xa4\x98\xee\xdel\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81" + b"\xe4N\xc8\x86\x153\xf5x2\xa2O\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z" + b"\xc4\xcdS\xb6t<\x16\xf2\x9cI#\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0" + b"\xaa\x96-Pe\xade:\x04\t\x1b\xf7\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9" + b"\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7" + b"\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8" + b"\x84b\xf8\x1epl\xeajr\xd1=\t\x03\xdd\x13\x1b3!E\xf9vV\xdaF\xf3\xd7\xb4" + b"\x0c\xa9P~\xec\xdeE\xe37\xf6\x1d\xc6\xbb\xddc%\xb6\x0fI\x07\xf0;\xaf\xe7" + b"\xa0\x8b\xa7Z\x99(\xe9\xe2\xf0o\x18>`\xe1\xaa\xa8\xd9\xa1\xb2}\xe7\x8d" + b"\x834T\xb6\xef\xc1\xde\xe3\x98\xbcD\x03MA@\xd8\xed\xdc\xc8\x93\x03\x1a" + b"\x93\x0b\x7f\x94\x12\x0b\x02Sa\x18\xc9\xc5\x9bTJE}\xf6\xc8g\x17#ZV\x01" + b"\xc9\x9dc\x83\x0e>0\x16\x90S\xb8/\x03y_\x18\xfa(\xd7\x0br\xa2\xb0\xba?" + b"\x8c\xe6\x83@\x84\xdf\x02:\xc5z\x9e\xa6\x84\xc9\xf5BeyX\x83\x1a\xf1 :\t" + b"\xf7\x19\xfexD\\&G\xf3\x85Y\xa2J\xf9\x0bv{\x89\xf6\xe7)A\xaf\x04o\x00" + b"\x075\xd3\xe0\x7f\x97\x98F\x0f?v\x93\xedVtTf\xb5\x97\x83\xed\x19\xd7\x1a" + b"'k\xd7\xd9\xc5\\Y\xd1\xdc\x07\x15|w\xbc\xacd\x87\x08d\xec\xa7\xf6\x82" + b"\xfc\xb3\x93\xeb\xb9 \x8d\xbc ,\xb3X\xb0\xd2s\xd7\xd1\xffv\x05\xdf}\xa2" + b"\x96\xfb%\n\xdf\xa2\x7f\x08.\xa16\n\xe0\x19\x93\x7fh\n\x1c\x8c\x0f \x11" + b"\xc6Bl\x95\x19U}\xe4s\xb5\x10H\xea\x86pB\xe88\x95\xbe\x8cZ\xdb\xe4\x94A" + b"\x92\xb9;z\xaa\xa7{\x1c5!\xc0\xaf\xc1A\xf9\xda\xf0$\xb0\x02qg\xc8\xc7/|" + b"\xafr\x99^\x91\x88\xbf\x03\xd9=\xd7n\xda6{>8\n\xc7:\xa9'\xba.\x0b\xe2" + b"\xb5\x1d\x0e\n\x9a\x8e\x06\x8f:\xdd\x82'[\xc3\"wD$\xa7w\xecq\x8c,1\x93" + b"\xd0,\xae2w\x93\x12$Jd\x19mg\x02\x93\x9cA\x95\x9d&\xca8i\x9c\xb0;\xe7NQ" + b"\x1frh\x8beL;\xb0m\xee\x07Q\x9b\xc6\xd8\x03\xb5\xdeN\xd4\xfe\x98\xd0\xdc" + b"\x1a[\x04\xde\x1a\xf6\x91j\xf8EOli\x8eB^\x1d\x82\x07\xb2\xb5R]\xb7\xd7" + b"\xe9\xa6\xc3.\xfb\xf0-\xb4e\x9b\xde\x03\x88\xc6\xc1iN\x0e\x84wbQ\xdf~" + b"\xe9\xa4\x884\x96kM\xbc)T\xf3\x89\x97\x0f\x143\xe7)\xa0\xb3B\x00\xa8\xaf" + b"\x82^\xcb\xc7..\xdb\xc7\t\x9dH\xee5\xe9#\xe6NV\x94\xcb$Kk\xe3\x7f\r\xe3t" + b"\x12\xcf'\xefR\x8b\xf42\xcf-LH\xac\xe5\x1f0~?SO\xeb\xc1E\x1a\x1c]\xf2" + b"\xc4<\x11\x02\x10Z0a*?\xe4r\xff\xfb\xff\xf6\x14nG\xead^\xd6\xef8\xb6uEI" + b"\x99\nV\xe2\xb3\x95\x8e\x83\xf6i!\xb5&1F\xb1DP\xf4 SO3D!w\x99_G\x7f+\x90" + b".\xab\xbb]\x91>\xc9#h;\x0f5J\x91K\xf4^-[\x9e\x8a\\\x94\xca\xaf\xf6\x19" + b"\xd4\xa1\x9b\xc4\xb8p\xa1\xae\x15\xe9r\x84\xe0\xcar.l []\x8b\xaf+0\xf2g" + b"\x01aKY\xdfI\xcf,\n\xe8\xf0\xe7V\x80_#\xb2\xf2\xa9\x06\x8c>w\xe2W,\xf4" + b"\x8c\r\xf963\xf5J\xcc2\x05=kT\xeaUti\xe5_\xce\x1b\xfa\x8dl\x02h\xef\xa8" + b"\xfbf\x7f\xff\xf0\x19\xeax" +) + +FILTERS_RAW_1 = [{"id": lzma.FILTER_LZMA2, "preset": 3}] +COMPRESSED_RAW_1 = ( + b"\xe0\x07\x80\x03\xfd]\x00\x05\x14\x07bX\x19\xcd\xddn\x96cyq\xa1\xdd\xee" + b"\xf8\xfam\xe3'\x88\xd3\xff\xe4\x9e \xceQ\x91\xa4\x14I\xf6\xb9\x9dVL8\x15" + b"_\x0e\x12\xc3\xeb\xbc\xa5\xcd\nW\x1d$=R;\x1d\xf8k8\t\xb1{\xd4\xc5+\x9d" + b"\x87c\xe5\xef\x98\xb4\xd7S3\xcd\xcc\xd2\xed\xa4\x0em\xe5\xf4\xdd\xd0b" + b"\xbe4*\xaa\x0b\xc5\x08\x10\x85+\x81.\x17\xaf9\xc9b\xeaZrA\xe20\x7fs\"r" + b"\xdaG\x81\xde\x90cu\xa5\xdb\xa9.A\x08l\xb0<\xf6\x03\xddOi\xd0\xc5\xb4" + b"\xec\xecg4t6\"\xa6\xb8o\xb5?\x18^}\xb6}\x03[:\xeb\x03\xa9\n[\x89l\x19g" + b"\x16\xc82\xed\x0b\xfb\x86n\xa2\x857@\x93\xcd6T\xc3u\xb0\t\xf9\x1b\x918" + b"\xfc[\x1b\x1e4\xb3\x14\x06PCV\xa8\"\xf5\x81x~\xe9\xb5N\x9cK\x9f\xc6\xc3%" + b"\xc8k:{6\xe7\xf7\xbd\x05\x02\xb4\xc4\xc3\xd3\xfd\xc3\xa8\\\xfc@\xb1F_" + b"\xc8\x90\xd9sU\x98\xad8\x05\x07\xde7J\x8bM\xd0\xb3;X\xec\x87\xef\xae\xb3" + b"eO,\xb1z,d\x11y\xeejlB\x02\x1d\xf28\x1f#\x896\xce\x0b\xf0\xf5\xa9PK\x0f" + b"\xb3\x13P\xd8\x88\xd2\xa1\x08\x04C?\xdb\x94_\x9a\"\xe9\xe3e\x1d\xde\x9b" + b"\xa1\xe8>H\x98\x10;\xc5\x03#\xb5\x9d4\x01\xe7\xc5\xba%v\xa49\x97A\xe0\"" + b"\x8c\xc22\xe3i\xc1\x9d\xab3\xdf\xbe\xfdDm7\x1b\x9d\xab\xb5\x15o:J\x92" + b"\xdb\x816\x17\xc2O\x99\x1b\x0e\x8d\xf3\tQ\xed\x8e\x95S/\x16M\xb2S\x04" + b"\x0f\xc3J\xc6\xc7\xe4\xcb\xc5\xf4\xe7d\x14\xe4=^B\xfb\xd3E\xd3\x1e\xcd" + b"\x91\xa5\xd0G\x8f.\xf6\xf9\x0bb&\xd9\x9f\xc2\xfdj\xa2\x9e\xc4\\\x0e\x1dC" + b"v\xe8\xd2\x8a?^H\xec\xae\xeb>\xfe\xb8\xab\xd4IqY\x8c\xd4K7\x11\xf4D\xd0W" + b"\xa5\xbe\xeaO\xbf\xd0\x04\xfdl\x10\xae5\xd4U\x19\x06\xf9{\xaa\xe0\x81" + b"\x0f\xcf\xa3k{\x95\xbd\x19\xa2\xf8\xe4\xa3\x08O*\xf1\xf1B-\xc7(\x0eR\xfd" + b"@E\x9f\xd3\x1e:\xfdV\xb7\x04Y\x94\xeb]\x83\xc4\xa5\xd7\xc0gX\x98\xcf\x0f" + b"\xcd3\x00]n\x17\xec\xbd\xa3Y\x86\xc5\xf3u\xf6*\xbdT\xedA$A\xd9A\xe7\x98" + b"\xef\x14\x02\x9a\xfdiw\xec\xa0\x87\x11\xd9%\xc5\xeb\x8a=\xae\xc0\xc4\xc6" + b"D\x80\x8f\xa8\xd1\xbbq\xb2\xc0\xa0\xf5Cqp\xeeL\xe3\xe5\xdc \x84\"\xe9" + b"\x80t\x83\x05\xba\xf1\xc5~\x93\xc9\xf0\x01c\xceix\x9d\xed\xc5)l\x16)\xd1" + b"\x03@l\x04\x7f\x87\xa5yn\x1b\x01D\xaa:\xd2\x96\xb4\xb3?\xb0\xf9\xce\x07" + b"\xeb\x81\x00\xe4\xc3\xf5%_\xae\xd4\xf9\xeb\xe2\rh\xb2#\xd67Q\x16D\x82hn" + b"\xd1\xa3_?q\xf0\xe2\xac\xf317\x9e\xd0_\x83|\xf1\xca\xb7\x95S\xabW\x12" + b"\xff\xddt\xf69L\x01\xf2|\xdaW\xda\xees\x98L\x18\xb8_\xe8$\x82\xea\xd6" + b"\xd1F\xd4\x0b\xcdk\x01vf\x88h\xc3\xae\xb91\xc7Q\x9f\xa5G\xd9\xcc\x1f\xe3" + b"5\xb1\xdcy\x7fI\x8bcw\x8e\x10rIp\x02:\x19p_\xc8v\xcea\"\xc1\xd9\x91\x03" + b"\xbfe\xbe\xa6\xb3\xa8\x14\x18\xc3\xabH*m}\xc2\xc1\x9a}>l%\xce\x84\x99" + b"\xb3d\xaf\xd3\x82\x15\xdf\xc1\xfc5fOg\x9b\xfc\x8e^&\t@\xce\x9f\x06J\xb8" + b"\xb5\x86\x1d\xda{\x9f\xae\xb0\xff\x02\x81r\x92z\x8cM\xb7ho\xc9^\x9c\xb6" + b"\x9c\xae\xd1\xc9\xf4\xdfU7\xd6\\!\xea\x0b\x94k\xb9Ud~\x98\xe7\x86\x8az" + b"\x10;\xe3\x1d\xe5PG\xf8\xa4\x12\x05w\x98^\xc4\xb1\xbb\xfb\xcf\xe0\x7f" + b"\x033Sf\x0c \xb1\xf6@\x94\xe5\xa3\xb2\xa7\x10\x9a\xc0\x14\xc3s\xb5xRD" + b"\xf4`W\xd9\xe5\xd3\xcf\x91\rTZ-X\xbe\xbf\xb5\xe2\xee|\x1a\xbf\xfb\x08" + b"\x91\xe1\xfc\x9a\x18\xa3\x8b\xd6^\x89\xf5[\xef\x87\xd1\x06\x1c7\xd6\xa2" + b"\t\tQ5/@S\xc05\xd2VhAK\x03VC\r\x9b\x93\xd6M\xf1xO\xaaO\xed\xb9<\x0c\xdae" + b"*\xd0\x07Hk6\x9fG+\xa1)\xcd\x9cl\x87\xdb\xe1\xe7\xefK}\x875\xab\xa0\x19u" + b"\xf6*F\xb32\x00\x00\x00" +) + +FILTERS_RAW_2 = [{"id": lzma.FILTER_DELTA, "dist": 2}, + {"id": lzma.FILTER_LZMA2, + "preset": lzma.PRESET_DEFAULT | lzma.PRESET_EXTREME}] +COMPRESSED_RAW_2 = ( + b"\xe0\x07\x80\x05\x91]\x00\x05\x14\x06-\xd4\xa8d?\xef\xbe\xafH\xee\x042" + b"\xcb.\xb5g\x8f\xfb\x14\xab\xa5\x9f\x025z\xa4\xdd\xd8\t[}W\xf8\x0c\x1dmH" + b"\xfa\x05\xfcg\xba\xe5\x01Q\x0b\x83R\xb6A\x885\xc0\xba\xee\n\x1cv~\xde:o" + b"\x06:J\xa7\x11Cc\xea\xf7\xe5*o\xf7\x83\\l\xbdE\x19\x1f\r\xa8\x10\xb42" + b"\x0caU{\xd7\xb8w\xdc\xbe\x1b\xfc8\xb4\xcc\xd38\\\xf6\x13\xf6\xe7\x98\xfa" + b"\xc7[\x17_9\x86%\xa8\xf8\xaa\xb8\x8dfs#\x1e=\xed<\x92\x10\\t\xff\x86\xfb" + b"=\x9e7\x18\x1dft\\\xb5\x01\x95Q\xc5\x19\xb38\xe0\xd4\xaa\x07\xc3\x7f\xd8" + b"\xa2\x00>-\xd3\x8e\xa1#\xfa\x83ArAm\xdbJ~\x93\xa3B\x82\xe0\xc7\xcc(\x08`" + b"WK\xad\x1b\x94kaj\x04 \xde\xfc\xe1\xed\xb0\x82\x91\xefS\x84%\x86\xfbi" + b"\x99X\xf1B\xe7\x90;E\xfde\x98\xda\xca\xd6T\xb4bg\xa4\n\x9aj\xd1\x83\x9e]" + b"\"\x7fM\xb5\x0fr\xd2\\\xa5j~P\x10GH\xbfN*Z\x10.\x81\tpE\x8a\x08\xbe1\xbd" + b"\xcd\xa9\xe1\x8d\x1f\x04\xf9\x0eH\xb9\xae\xd6\xc3\xc1\xa5\xa9\x95P\xdc~" + b"\xff\x01\x930\xa9\x04\xf6\x03\xfe\xb5JK\xc3]\xdd9\xb1\xd3\xd7F\xf5\xd1" + b"\x1e\xa0\x1c_\xed[\x0c\xae\xd4\x8b\x946\xeb\xbf\xbb\xe3$kS{\xb5\x80,f:Sj" + b"\x0f\x08z\x1c\xf5\xe8\xe6\xae\x98\xb0Q~r\x0f\xb0\x05?\xb6\x90\x19\x02&" + b"\xcb\x80\t\xc4\xea\x9c|x\xce\x10\x9c\xc5|\xcbdhh+\x0c'\xc5\x81\xc33\xb5" + b"\x14q\xd6\xc5\xe3`Z#\xdc\x8a\xab\xdd\xea\x08\xc2I\xe7\x02l{\xec\x196\x06" + b"\x91\x8d\xdc\xd5\xb3x\xe1hz%\xd1\xf8\xa5\xdd\x98!\x8c\x1c\xc1\x17RUa\xbb" + b"\x95\x0f\xe4X\xea1\x0c\xf1=R\xbe\xc60\xe3\xa4\x9a\x90bd\x97$]B\x01\xdd" + b"\x1f\xe3h2c\x1e\xa0L`4\xc6x\xa3Z\x8a\r\x14]T^\xd8\x89\x1b\x92\r;\xedY" + b"\x0c\xef\x8d9z\xf3o\xb6)f\xa9]$n\rp\x93\xd0\x10\xa4\x08\xb8\xb2\x8b\xb6" + b"\x8f\x80\xae;\xdcQ\xf1\xfa\x9a\x06\x8e\xa5\x0e\x8cK\x9c @\xaa:UcX\n!\xc6" + b"\x02\x12\xcb\x1b\"=\x16.\x1f\x176\xf2g=\xe1Wn\xe9\xe1\xd4\xf1O\xad\x15" + b"\x86\xe9\xa3T\xaf\xa9\xd7D\xb5\xd1W3pnt\x11\xc7VOj\xb7M\xc4i\xa1\xf1$3" + b"\xbb\xdc\x8af\xb0\xc5Y\r\xd1\xfb\xf2\xe7K\xe6\xc5hwO\xfe\x8c2^&\x07\xd5" + b"\x1fV\x19\xfd\r\x14\xd2i=yZ\xe6o\xaf\xc6\xb6\x92\x9d\xc4\r\xb3\xafw\xac%" + b"\xcfc\x1a\xf1`]\xf2\x1a\x9e\x808\xedm\xedQ\xb2\xfe\xe4h`[q\xae\xe0\x0f" + b"\xba0g\xb6\"N\xc3\xfb\xcfR\x11\xc5\x18)(\xc40\\\xa3\x02\xd9G!\xce\x1b" + b"\xc1\x96x\xb5\xc8z\x1f\x01\xb4\xaf\xde\xc2\xcd\x07\xe7H\xb3y\xa8M\n\\A\t" + b"ar\xddM\x8b\x9a\xea\x84\x9b!\xf1\x8d\xb1\xf1~\x1e\r\xa5H\xba\xf1\x84o" + b"\xda\x87\x01h\xe9\xa2\xbe\xbeqN\x9d\x84\x0b!WG\xda\xa1\xa5A\xb7\xc7`j" + b"\x15\xf2\xe9\xdd?\x015B\xd2~E\x06\x11\xe0\x91!\x05^\x80\xdd\xa8y\x15}" + b"\xa1)\xb1)\x81\x18\xf4\xf4\xf8\xc0\xefD\xe3\xdb2f\x1e\x12\xabu\xc9\x97" + b"\xcd\x1e\xa7\x0c\x02x4_6\x03\xc4$t\xf39\x94\x1d=\xcb\xbfv\\\xf5\xa3\x1d" + b"\x9d8jk\x95\x13)ff\xf9n\xc4\xa9\xe3\x01\xb8\xda\xfb\xab\xdfM\x99\xfb\x05" + b"\xe0\xe9\xb0I\xf4E\xab\xe2\x15\xa3\x035\xe7\xdeT\xee\x82p\xb4\x88\xd3" + b"\x893\x9c/\xc0\xd6\x8fou;\xf6\x95PR\xa9\xb2\xc1\xefFj\xe2\xa7$\xf7h\xf1" + b"\xdfK(\xc9c\xba7\xe8\xe3)\xdd\xb2,\x83\xfb\x84\x18.y\x18Qi\x88\xf8`h-" + b"\xef\xd5\xed\x8c\t\xd8\xc3^\x0f\x00\xb7\xd0[!\xafM\x9b\xd7.\x07\xd8\xfb" + b"\xd9\xe2-S+\xaa8,\xa0\x03\x1b \xea\xa8\x00\xc3\xab~\xd0$e\xa5\x7f\xf7" + b"\x95P]\x12\x19i\xd9\x7fo\x0c\xd8g^\rE\xa5\x80\x18\xc5\x01\x80\xaek`\xff~" + b"\xb6y\xe7+\xe5\x11^D\xa7\x85\x18\"!\xd6\xd2\xa7\xf4\x1eT\xdb\x02\xe15" + b"\x02Y\xbc\x174Z\xe7\x9cH\x1c\xbf\x0f\xc6\xe9f]\xcf\x8cx\xbc\xe5\x15\x94" + b"\xfc3\xbc\xa7TUH\xf1\x84\x1b\xf7\xa9y\xc07\x84\xf8X\xd8\xef\xfc \x1c\xd8" + b"( /\xf2\xb7\xec\xc1\\\x8c\xf6\x95\xa1\x03J\x83vP8\xe1\xe3\xbb~\xc24kA" + b"\x98y\xa1\xf2P\xe9\x9d\xc9J\xf8N\x99\xb4\xceaO\xde\x16\x1e\xc2\x19\xa7" + b"\x03\xd2\xe0\x8f:\x15\xf3\x84\x9e\xee\xe6e\xb8\x02q\xc7AC\x1emw\xfd\t" + b"\x9a\x1eu\xc1\xa9\xcaCwUP\x00\xa5\xf78L4w!\x91L2 \x87\xd0\xf2\x06\x81j" + b"\x80;\x03V\x06\x87\x92\xcb\x90lv@E\x8d\x8d\xa5\xa6\xe7Z[\xdf\xd6E\x03`>" + b"\x8f\xde\xa1bZ\x84\xd0\xa9`\x05\x0e{\x80;\xe3\xbef\x8d\x1d\xebk1.\xe3" + b"\xe9N\x15\xf7\xd4(\xfa\xbb\x15\xbdu\xf7\x7f\x86\xae!\x03L\x1d\xb5\xc1" + b"\xb9\x11\xdb\xd0\x93\xe4\x02\xe1\xd2\xcbBjc_\xe8}d\xdb\xc3\xa0Y\xbe\xc9/" + b"\x95\x01\xa3,\xe6bl@\x01\xdbp\xc2\xce\x14\x168\xc2q\xe3uH\x89X\xa4\xa9" + b"\x19\x1d\xc1}\x7fOX\x19\x9f\xdd\xbe\x85\x83\xff\x96\x1ee\x82O`CF=K\xeb$I" + b"\x17_\xefX\x8bJ'v\xde\x1f+\xd9.v\xf8Tv\x17\xf2\x9f5\x19\xe1\xb9\x91\xa8S" + b"\x86\xbd\x1a\"(\xa5x\x8dC\x03X\x81\x91\xa8\x11\xc4pS\x13\xbc\xf2'J\xae!" + b"\xef\xef\x84G\t\x8d\xc4\x10\x132\x00oS\x9e\xe0\xe4d\x8f\xb8y\xac\xa6\x9f" + b",\xb8f\x87\r\xdf\x9eE\x0f\xe1\xd0\\L\x00\xb2\xe1h\x84\xef}\x98\xa8\x11" + b"\xccW#\\\x83\x7fo\xbbz\x8f\x00" +) + +FILTERS_RAW_3 = [{"id": lzma.FILTER_IA64, "start_offset": 0x100}, + {"id": lzma.FILTER_LZMA2}] +COMPRESSED_RAW_3 = ( + b"\xe0\x07\x80\x03\xdf]\x00\x05\x14\x07bX\x19\xcd\xddn\x98\x15\xe4\xb4\x9d" + b"o\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8\xe2\xfc\xe7\xd9\xfe6\xb8(" + b"\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02\x17/\xa6=\xf0\xa2\xdf/M\x89" + b"\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ\x15\x80\x8c\xf8\x8do\xfa\x12" + b"\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t\xca6 BF$\xe5Q\xa4\x98\xee\xde" + b"l\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81\xe4N\xc8\x86\x153\xf5x2\xa2O" + b"\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z\xc4\xcdS\xb6t<\x16\xf2\x9cI#" + b"\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0\xaa\x96-Pe\xade:\x04\t\x1b\xf7" + b"\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b" + b"\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa" + b"\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8\x84b\xf6\xc3\xd4c-H\x93oJl\xd0iQ\xe4k" + b"\x84\x0b\xc1\xb7\xbc\xb1\x17\x88\xb1\xca?@\xf6\x07\xea\xe6x\xf1H12P\x0f" + b"\x8a\xc9\xeauw\xe3\xbe\xaai\xa9W\xd0\x80\xcd#cb5\x99\xd8]\xa9d\x0c\xbd" + b"\xa2\xdcWl\xedUG\xbf\x89yF\xf77\x81v\xbd5\x98\xbeh8\x18W\x08\xf0\x1b\x99" + b"5:\x1a?rD\x96\xa1\x04\x0f\xae\xba\x85\xeb\x9d5@\xf5\x83\xd37\x83\x8ac" + b"\x06\xd4\x97i\xcdt\x16S\x82k\xf6K\x01vy\x88\x91\x9b6T\xdae\r\xfd]:k\xbal" + b"\xa9\xbba\xc34\xf9r\xeb}r\xdb\xc7\xdb*\x8f\x03z\xdc8h\xcc\xc9\xd3\xbcl" + b"\xa5-\xcb\xeaK\xa2\xc5\x15\xc0\xe3\xc1\x86Z\xfb\xebL\xe13\xcf\x9c\xe3" + b"\x1d\xc9\xed\xc2\x06\xcc\xce!\x92\xe5\xfe\x9c^\xa59w \x9bP\xa3PK\x08d" + b"\xf9\xe2Z}\xa7\xbf\xed\xeb%$\x0c\x82\xb8/\xb0\x01\xa9&,\xf7qh{Q\x96)\xf2" + b"q\x96\xc3\x80\xb4\x12\xb0\xba\xe6o\xf4!\xb4[\xd4\x8aw\x10\xf7t\x0c\xb3" + b"\xd9\xd5\xc3`^\x81\x11??\\\xa4\x99\x85R\xd4\x8e\x83\xc9\x1eX\xbfa\xf1" + b"\xac\xb0\xea\xea\xd7\xd0\xab\x18\xe2\xf2\xed\xe1\xb7\xc9\x18\xcbS\xe4>" + b"\xc9\x95H\xe8\xcb\t\r%\xeb\xc7$.o\xf1\xf3R\x17\x1db\xbb\xd8U\xa5^\xccS" + b"\x16\x01\x87\xf3/\x93\xd1\xf0v\xc0r\xd7\xcc\xa2Gkz\xca\x80\x0e\xfd\xd0" + b"\x8b\xbb\xd2Ix\xb3\x1ey\xca-0\xe3z^\xd6\xd6\x8f_\xf1\x9dP\x9fi\xa7\xd1" + b"\xe8\x90\x84\xdc\xbf\xcdky\x8e\xdc\x81\x7f\xa3\xb2+\xbf\x04\xef\xd8\\" + b"\xc4\xdf\xe1\xb0\x01\xe9\x93\xe3Y\xf1\x1dY\xe8h\x81\xcf\xf1w\xcc\xb4\xef" + b" \x8b|\x04\xea\x83ej\xbe\x1f\xd4z\x9c`\xd3\x1a\x92A\x06\xe5\x8f\xa9\x13" + b"\t\x9e=\xfa\x1c\xe5_\x9f%v\x1bo\x11ZO\xd8\xf4\t\xddM\x16-\x04\xfc\x18<\"" + b"CM\xddg~b\xf6\xef\x8e\x0c\xd0\xde|\xa0'\x8a\x0c\xd6x\xae!J\xa6F\x88\x15u" + b"\x008\x17\xbc7y\xb3\xd8u\xac_\x85\x8d\xe7\xc1@\x9c\xecqc\xa3#\xad\xf1" + b"\x935\xb5)_\r\xec3]\x0fo]5\xd0my\x07\x9b\xee\x81\xb5\x0f\xcfK+\x00\xc0" + b"\xe4b\x10\xe4\x0c\x1a \x9b\xe0\x97t\xf6\xa1\x9e\x850\xba\x0c\x9a\x8d\xc8" + b"\x8f\x07\xd7\xae\xc8\xf9+i\xdc\xb9k\xb0>f\x19\xb8\r\xa8\xf8\x1f$\xa5{p" + b"\xc6\x880\xce\xdb\xcf\xca_\x86\xac\x88h6\x8bZ%'\xd0\n\xbf\x0f\x9c\"\xba" + b"\xe5\x86\x9f\x0f7X=mNX[\xcc\x19FU\xc9\x860\xbc\x90a+* \xae_$\x03\x1e\xd3" + b"\xcd_\xa0\x9c\xde\xaf46q\xa5\xc9\x92\xd7\xca\xe3`\x9d\x85}\xb4\xff\xb3" + b"\x83\xfb\xb6\xca\xae`\x0bw\x7f\xfc\xd8\xacVe\x19\xc8\x17\x0bZ\xad\x88" + b"\xeb#\x97\x03\x13\xb1d\x0f{\x0c\x04w\x07\r\x97\xbd\xd6\xc1\xc3B:\x95\x08" + b"^\x10V\xaeaH\x02\xd9\xe3\n\\\x01X\xf6\x9c\x8a\x06u#%\xbe*\xa1\x18v\x85" + b"\xec!\t4\x00\x00\x00" +) + +FILTERS_RAW_4 = [{"id": lzma.FILTER_DELTA, "dist": 4}, + {"id": lzma.FILTER_X86, "start_offset": 0x40}, + {"id": lzma.FILTER_LZMA2, "preset": 4, "lc": 2}] +COMPRESSED_RAW_4 = ( + b"\xe0\x07\x80\x06\x0e\\\x00\x05\x14\x07bW\xaah\xdd\x10\xdc'\xd6\x90,\xc6v" + b"Jq \x14l\xb7\x83xB\x0b\x97f=&fx\xba\n>Tn\xbf\x8f\xfb\x1dF\xca\xc3v_\xca?" + b"\xfbV<\x92#\xd4w\xa6\x8a\xeb\xf6\x03\xc0\x01\x94\xd8\x9e\x13\x12\x98\xd1" + b"*\xfa]c\xe8\x1e~\xaf\xb5]Eg\xfb\x9e\x01\"8\xb2\x90\x06=~\xe9\x91W\xcd" + b"\xecD\x12\xc7\xfa\xe1\x91\x06\xc7\x99\xb9\xe3\x901\x87\x19u\x0f\x869\xff" + b"\xc1\xb0hw|\xb0\xdcl\xcck\xb16o7\x85\xee{Y_b\xbf\xbc$\xf3=\x8d\x8bw\xe5Z" + b"\x08@\xc4kmE\xad\xfb\xf6*\xd8\xad\xa1\xfb\xc5{\xdej,)\x1emB\x1f<\xaeca" + b"\x80(\xee\x07 \xdf\xe9\xf8\xeb\x0e-\x97\x86\x90c\xf9\xea'B\xf7`\xd7\xb0" + b"\x92\xbd\xa0\x82]\xbd\x0e\x0eB\x19\xdc\x96\xc6\x19\xd86D\xf0\xd5\x831" + b"\x03\xb7\x1c\xf7&5\x1a\x8f PZ&j\xf8\x98\x1bo\xcc\x86\x9bS\xd3\xa5\xcdu" + b"\xf9$\xcc\x97o\xe5V~\xfb\x97\xb5\x0b\x17\x9c\xfdxW\x10\xfep4\x80\xdaHDY" + b"\xfa)\xfet\xb5\"\xd4\xd3F\x81\xf4\x13\x1f\xec\xdf\xa5\x13\xfc\"\x91x\xb7" + b"\x99\xce\xc8\x92\n\xeb[\x10l*Y\xd8\xb1@\x06\xc8o\x8d7r\xebu\xfd5\x0e\x7f" + b"\xf1$U{\t}\x1fQ\xcfxN\x9d\x9fXX\xe9`\x83\xc1\x06\xf4\x87v-f\x11\xdb/\\" + b"\x06\xff\xd7)B\xf3g\x06\x88#2\x1eB244\x7f4q\t\xc893?mPX\x95\xa6a\xfb)d" + b"\x9b\xfc\x98\x9aj\x04\xae\x9b\x9d\x19w\xba\xf92\xfaA\x11\\\x17\x97C3\xa4" + b"\xbc!\x88\xcdo[\xec:\x030\x91.\x85\xe0@\\4\x16\x12\x9d\xcaJv\x97\xb04" + b"\xack\xcbkf\xa3ss\xfc\x16^\x8ce\x85a\xa5=&\xecr\xb3p\xd1E\xd5\x80y\xc7" + b"\xda\xf6\xfek\xbcT\xbfH\xee\x15o\xc5\x8c\x830\xec\x1d\x01\xae\x0c-e\\" + b"\x91\x90\x94\xb2\xf8\x88\x91\xe8\x0b\xae\xa7>\x98\xf6\x9ck\xd2\xc6\x08" + b"\xe6\xab\t\x98\xf2!\xa0\x8c^\xacqA\x99<\x1cEG\x97\xc8\xf1\xb6\xb9\x82" + b"\x8d\xf7\x08s\x98a\xff\xe3\xcc\x92\x0e\xd2\xb6U\xd7\xd9\x86\x7fa\xe5\x1c" + b"\x8dTG@\t\x1e\x0e7*\xfc\xde\xbc]6N\xf7\xf1\x84\x9e\x9f\xcf\xe9\x1e\xb5'" + b"\xf4<\xdf\x99sq\xd0\x9d\xbd\x99\x0b\xb4%p4\xbf{\xbb\x8a\xd2\x0b\xbc=M" + b"\x94H:\xf5\xa8\xd6\xa4\xc90\xc2D\xb9\xd3\xa8\xb0S\x87 `\xa2\xeb\xf3W\xce" + b" 7\xf9N#\r\xe6\xbe\t\x9d\xe7\x811\xf9\x10\xc1\xc2\x14\xf6\xfc\xcba\xb7" + b"\xb1\x7f\x95l\xe4\tjA\xec:\x10\xe5\xfe\xc2\\=D\xe2\x0c\x0b3]\xf7\xc1\xf7" + b"\xbceZ\xb1A\xea\x16\xe5\xfddgFQ\xed\xaf\x04\xa3\xd3\xf8\xa2q\x19B\xd4r" + b"\xc5\x0c\x9a\x14\x94\xea\x91\xc4o\xe4\xbb\xb4\x99\xf4@\xd1\xe6\x0c\xe3" + b"\xc6d\xa0Q\n\xf2/\xd8\xb8S5\x8a\x18:\xb5g\xac\x95D\xce\x17\x07\xd4z\xda" + b"\x90\xe65\x07\x19H!\t\xfdu\x16\x8e\x0eR\x19\xf4\x8cl\x0c\xf9Q\xf1\x80" + b"\xe3\xbf\xd7O\xf8\x8c\x18\x0b\x9c\xf1\x1fb\xe1\tR\xb2\xf1\xe1A\xea \xcf-" + b"IGE\xf1\x14\x98$\x83\x15\xc9\xd8j\xbf\x19\x0f\xd5\xd1\xaa\xb3\xf3\xa5I2s" + b"\x8d\x145\xca\xd5\xd93\x9c\xb8D0\xe6\xaa%\xd0\xc0P}JO^h\x8e\x08\xadlV." + b"\x18\x88\x13\x05o\xb0\x07\xeaw\xe0\xb6\xa4\xd5*\xe4r\xef\x07G+\xc1\xbei[" + b"w\xe8\xab@_\xef\x15y\xe5\x12\xc9W\x1b.\xad\x85-\xc2\xf7\xe3mU6g\x8eSA" + b"\x01(\xd3\xdb\x16\x13=\xde\x92\xf9,D\xb8\x8a\xb2\xb4\xc9\xc3\xefnE\xe8\\" + b"\xa6\xe2Y\xd2\xcf\xcb\x8c\xb6\xd5\xe9\x1d\x1e\x9a\x8b~\xe2\xa6\rE\x84uV" + b"\xed\xc6\x99\xddm<\x10[\x0fu\x1f\xc1\x1d1\n\xcfw\xb2%!\xf0[\xce\x87\x83B" + b"\x08\xaa,\x08%d\xcef\x94\"\xd9g.\xc83\xcbXY+4\xec\x85qA\n\x1d=9\xf0*\xb1" + b"\x1f/\xf3s\xd61b\x7f@\xfb\x9d\xe3FQ\\\xbd\x82\x1e\x00\xf3\xce\xd3\xe1" + b"\xca,E\xfd7[\xab\xb6\xb7\xac!mA}\xbd\x9d3R5\x9cF\xabH\xeb\x92)cc\x13\xd0" + b"\xbd\xee\xe9n{\x1dIJB\xa5\xeb\x11\xe8`w&`\x8b}@Oxe\t\x8a\x07\x02\x95\xf2" + b"\xed\xda|\xb1e\xbe\xaa\xbbg\x19@\xe1Y\x878\x84\x0f\x8c\xe3\xc98\xf2\x9e" + b"\xd5N\xb5J\xef\xab!\xe2\x8dq\xe1\xe5q\xc5\xee\x11W\xb7\xe4k*\x027\xa0" + b"\xa3J\xf4\xd8m\xd0q\x94\xcb\x07\n:\xb6`.\xe4\x9c\x15+\xc0)\xde\x80X\xd4" + b"\xcfQm\x01\xc2cP\x1cA\x85'\xc9\xac\x8b\xe6\xb2)\xe6\x84t\x1c\x92\xe4Z" + b"\x1cR\xb0\x9e\x96\xd1\xfb\x1c\xa6\x8b\xcb`\x10\x12]\xf2gR\x9bFT\xe0\xc8H" + b"S\xfb\xac<\x04\xc7\xc1\xe8\xedP\xf4\x16\xdb\xc0\xd7e\xc2\x17J^\x1f\xab" + b"\xff[\x08\x19\xb4\xf5\xfb\x19\xb4\x04\xe5c~']\xcb\xc2A\xec\x90\xd0\xed" + b"\x06,\xc5K{\x86\x03\xb1\xcdMx\xdeQ\x8c3\xf9\x8a\xea=\x89\xaba\xd2\xc89a" + b"\xd72\xf0\xc3\x19\x8a\xdfs\xd4\xfd\xbb\x81b\xeaE\"\xd8\xf4d\x0cD\xf7IJ!" + b"\xe5d\xbbG\xe9\xcam\xaa\x0f_r\x95\x91NBq\xcaP\xce\xa7\xa9\xb5\x10\x94eP!" + b"|\x856\xcd\xbfIir\xb8e\x9bjP\x97q\xabwS7\x1a\x0ehM\xe7\xca\x86?\xdeP}y~" + b"\x0f\x95I\xfc\x13\xe1<Q\x1b\x868\x1d\x11\xdf\x94\xf4\x82>r\xa9k\x88\xcb" + b"\xfd\xc3v\xe2\xb9\x8a\x02\x8eq\x92I\xf8\xf6\xf1\x03s\x9b\xb8\xe3\"\xe3" + b"\xa9\xa5>D\xb8\x96;\xe7\x92\xd133\xe8\xdd'e\xc9.\xdc;\x17\x1f\xf5H\x13q" + b"\xa4W\x0c\xdb~\x98\x01\xeb\xdf\xe32\x13\x0f\xddx\n6\xa0\t\x10\xb6\xbb" + b"\xb0\xc3\x18\xb6;\x9fj[\xd9\xd5\xc9\x06\x8a\x87\xcd\xe5\xee\xfc\x9c-%@" + b"\xee\xe0\xeb\xd2\xe3\xe8\xfb\xc0\x122\\\xc7\xaf\xc2\xa1Oth\xb3\x8f\x82" + b"\xb3\x18\xa8\x07\xd5\xee_\xbe\xe0\x1cA\x1e_\r\x9a\xb0\x17W&\xa2D\x91\x94" + b"\x1a\xb2\xef\xf2\xdc\x85;X\xb0,\xeb>-7S\xe5\xca\x07)\x1fp\x7f\xcaQBL\xca" + b"\xf3\xb9d\xfc\xb5su\xb0\xc8\x95\x90\xeb*)\xa0v\xe4\x9a{FW\xf4l\xde\xcdj" + b"\x00" +) + +ISSUE_21872_DAT = ( + b']\x00\x00@\x00h3\x00\x00\x00\x00\x00\x00\x00\x00`D\x0c\x99\xc8' + b'\xd1\xbbZ^\xc43+\x83\xcd\xf1\xc6g\xec-\x061F\xb1\xbb\xc7\x17%-\xea' + b'\xfap\xfb\x8fs\x128\xb2,\x88\xe4\xc0\x12|*x\xd0\xa2\xc4b\x1b!\x02c' + b'\xab\xd9\x87U\xb8n \xfaVJ\x9a"\xb78\xff%_\x17`?@*\xc2\x82' + b"\xf2^\x1b\xb8\x04&\xc0\xbb\x03g\x9d\xca\xe9\xa4\xc9\xaf'\xe5\x8e}" + b'F\xdd\x11\xf3\x86\xbe\x1fN\x95\\\xef\xa2Mz-\xcb\x9a\xe3O@' + b"\x19\x07\xf6\xee\x9e\x9ag\xc6\xa5w\rnG'\x99\xfd\xfeGI\xb0" + b'\xbb\xf9\xc2\xe1\xff\xc5r\xcf\x85y[\x01\xa1\xbd\xcc/\xa3\x1b\x83\xaa' + b'\xc6\xf9\x99\x0c\xb6_\xc9MQ+x\xa2F\xda]\xdd\xe8\xfb\x1a&' + b',\xc4\x19\x1df\x81\x1e\x90\xf3\xb8Hgr\x85v\xbe\xa3qx\x01Y\xb5\x9fF' + b"\x13\x18\x01\xe69\x9b\xc8'\x1e\x9d\xd6\xe4F\x84\xac\xf8d<\x11\xd5" + b'\\\x0b\xeb\x0e\x82\xab\xb1\xe6\x1fka\xe1i\xc4 C\xb1"4)\xd6\xa7`\x02' + b'\xec\x11\x8c\xf0\x14\xb0\x1d\x1c\xecy\xf0\xb7|\x11j\x85X\xb2!\x1c' + b'\xac\xb5N\xc7\x85j\x9ev\xf5\xe6\x0b\xc1]c\xc15\x16\x9f\xd5\x99' + b"\xfei^\xd2G\x9b\xbdl\xab:\xbe,\xa9'4\x82\xe5\xee\xb3\xc1" + b'$\x93\x95\xa8Y\x16\xf5\xbf\xacw\x91\x04\x1d\x18\x06\xe9\xc5\xfdk\x06' + b'\xe8\xfck\xc5\x86>\x8b~\xa4\xcb\xf1\xb3\x04\xf1\x04G5\xe2\xcc]' + b'\x16\xbf\x140d\x18\xe2\xedw#(3\xca\xa1\x80bX\x7f\xb3\x84' + b'\x9d\xdb\xe7\x08\x97\xcd\x16\xb9\xf1\xd5r+m\x1e\xcb3q\xc5\x9e\x92' + b"\x7f\x8e*\xc7\xde\xe9\xe26\xcds\xb1\x10-\xf6r\x02?\x9d\xddCgJN'" + b'\x11M\xfa\nQ\n\xe6`m\xb8N\xbbq\x8el\x0b\x02\xc7:q\x04G\xa1T' + b'\xf1\xfe!0\x85~\xe5\x884\xe9\x89\xfb\x13J8\x15\xe42\xb6\xad' + b'\x877A\x9a\xa6\xbft]\xd0\xe35M\xb0\x0cK\xc8\xf6\x88\xae\xed\xa9,j7' + b'\x81\x13\xa0(\xcb\xe1\xe9l2\x7f\xcd\xda\x95(\xa70B\xbd\xf4\xe3' + b'hp\x94\xbdJ\xd7\t\xc7g\xffo?\x89?\xf8}\x7f\xbc\x1c\x87' + b'\x14\xc0\xcf\x8cV:\x9a\x0e\xd0\xb2\x1ck\xffk\xb9\xe0=\xc7\x8d/' + b'\xb8\xff\x7f\x1d\x87`\x19.\x98X*~\xa7j\xb9\x0b"\xf4\xe4;V`\xb9\xd7' + b'\x03\x1e\xd0t0\xd3\xde\x1fd\xb9\xe2)\x16\x81}\xb1\\b\x7fJ' + b'\x92\xf4\xff\n+V!\xe9\xde\x98\xa0\x8fK\xdf7\xb9\xc0\x12\x1f\xe2' + b'\xe9\xb0`\xae\x14\r\xa7\xc4\x81~\xd8\x8d\xc5\x06\xd8m\xb0Y\x8a)' + b'\x06/\xbb\xf9\xac\xeaP\xe0\x91\x05m[\xe5z\xe6Z\xf3\x9f\xc7\xd0' + b'\xd3\x8b\xf3\x8a\x1b\xfa\xe4Pf\xbc0\x17\x10\xa9\xd0\x95J{\xb3\xc3' + b'\xfdW\x9bop\x0f\xbe\xaee\xa3]\x93\x9c\xda\xb75<\xf6g!\xcc\xb1\xfc\\' + b'7\x152Mc\x17\x84\x9d\xcd35\r0\xacL-\xf3\xfb\xcb\x96\x1e\xe9U\x7f' + b'\xd7\xca\xb0\xcc\x89\x0c*\xce\x14\xd1P\xf1\x03\xb6.~9o?\xe8' + b'\r\x86\xe0\x92\x87}\xa3\x84\x03P\xe0\xc2\x7f\n;m\x9d\x9e\xb4|' + b'\x8c\x18\xc0#0\xfe3\x07<\xda\xd8\xcf^\xd4Hi\xd6\xb3\x0bT' + b'\x1dF\x88\x85q}\x02\xc6&\xc4\xae\xce\x9cU\xfa\x0f\xcc\xb6\x1f\x11' + b'drw\x9eN\x19\xbd\xffz\x0f\xf0\x04s\xadR\xc1\xc0\xbfl\xf1\xba\xf95^' + b'e\xb1\xfbVY\xd9\x9f\x1c\xbf*\xc4\xa86\x08+\xd6\x88[\xc4_rc\xf0f' + b'\xb8\xd4\xec\x1dx\x19|\xbf\xa7\xe0\x82\x0b\x8c~\x10L/\x90\xd6\xfb' + b'\x81\xdb\x98\xcc\x02\x14\xa5C\xb2\xa7i\xfd\xcd\x1fO\xf7\xe9\x89t\xf0' + b'\x17\xa5\x1c\xad\xfe<Q`%\x075k\n7\x9eI\x82<#)&\x04\xc2\xf0C\xd4`!' + b'\xcb\xa9\xf9\xb3F\x86\xb5\xc3M\xbeu\x12\xb2\xca\x95e\x10\x0b\xb1\xcc' + b'\x01b\x9bXa\x1b[B\x8c\x07\x11Of;\xeaC\xebr\x8eb\xd9\x9c\xe4i]<z\x9a' + b'\x03T\x8b9pF\x10\x8c\x84\xc7\x0e\xeaPw\xe5\xa0\x94\x1f\x84\xdd' + b'a\xe8\x85\xc2\x00\xebq\xe7&Wo5q8\xc2t\x98\xab\xb7\x7f\xe64-H' + b'\t\xb4d\xbe\x06\xe3Q\x8b\xa9J\xb0\x00\xd7s.\x85"\xc0p\x05' + b'\x1c\x06N\x87\xa5\xf8\xc3g\x1b}\x0f\x0f\xc3|\x90\xea\xefd3X' + b'[\xab\x04E\xf2\xf2\xc9\x08\x8a\xa8+W\xa2v\xec\x15G\x08/I<L\\1' + b'\xff\x15O\xaa\x89{\xd1mW\x13\xbd~\xe1\x90^\xc4@\r\xed\xb5D@\xb4\x08' + b'A\x90\xe69;\xc7BO\xdb\xda\xebu\x9e\xa9tN\xae\x8aJ5\xcd\x11\x1d\xea' + b"\xe5\xa7\x04\xe6\x82Z\xc7O\xe46[7\xdco*[\xbe\x0b\xc9\xb7a\xab'\xf6" + b"\xd1u\xdb\xd9q\xf5+y\x1b\x00\xb4\xf3a\xae\xf1M\xc4\xbc\xd00'\x06pQ" + b'\x8dH\xaa\xaa\xc4\xd2K\x9b\xc0\xe9\xec=n\xa9\x1a\x8a\xc2\xe8\x18\xbc' + b'\x93\xb8F\xa1\x8fOY\xe7\xda\xcf0\t\xff|\xd9\xe5\xcf\xe7\xf6\xbe' + b'\xf8\x04\x17\xf2\xe5P\xa7y~\xce\x11h0\x81\x80d[\x00_v\xbbc\xdbI' + b'3\xbc`W\xc0yrkB\xf5\x9f\xe9i\xc5\x8a^\x8d\xd4\x81\xd9\x05\xc1\xfc>' + b'"\xd1v`\x82\xd5$\x89\xcf^\xd52.\xafd\xe8d@\xaa\xd5Y|\x90\x84' + b'j\xdb}\x84riV\x8e\xf0X4rB\xf2NPS[\x8e\x88\xd4\x0fI\xb8' + b'\xdd\xcb\x1d\xf2(\xdf;9\x9e|\xef^0;.*[\x9fl\x7f\xa2_X\xaff!\xbb\x03' + b'\xff\x19\x8f\x88\xb5\xb6\x884\xa3\x05\xde3D{\xe3\xcb\xce\xe4t]' + b'\x875\xe3Uf\xae\xea\x88\x1c\x03b\n\xb1,Q\xec\xcf\x08\t\xde@\x83\xaa<' + b',-\xe4\xee\x9b\x843\xe5\x007\tK\xac\x057\xd6*X\xa3\xc6~\xba\xe6O' + b'\x81kz"\xbe\xe43sL\xf1\xfa;\xf4^\x1e\xb4\x80\xe2\xbd\xaa\x17Z\xe1f' + b'\xda\xa6\xb9\x07:]}\x9fa\x0b?\xba\xe7\xf15\x04M\xe3\n}M\xa4\xcb\r' + b'2\x8a\x88\xa9\xa7\x92\x93\x84\x81Yo\x00\xcc\xc4\xab\x9aT\x96\x0b\xbe' + b'U\xac\x1d\x8d\x1b\x98"\xf8\x8f\xf1u\xc1n\xcc\xfcA\xcc\x90\xb7i' + b'\x83\x9c\x9c~\x1d4\xa2\xf0*J\xe7t\x12\xb4\xe3\xa0u\xd7\x95Z' + b'\xf7\xafG\x96~ST,\xa7\rC\x06\xf4\xf0\xeb`2\x9e>Q\x0e\xf6\xf5\xc5' + b'\x9b\xb5\xaf\xbe\xa3\x8f\xc0\xa3hu\x14\x12 \x97\x99\x04b\x8e\xc7\x1b' + b'VKc\xc1\xf3 \xde\x85-:\xdc\x1f\xac\xce*\x06\xb3\x80;`' + b'\xdb\xdd\x97\xfdg\xbf\xe7\xa8S\x08}\xf55e7\xb8/\xf0!\xc8' + b"Y\xa8\x9a\x07'\xe2\xde\r\x02\xe1\xb2\x0c\xf4C\xcd\xf9\xcb(\xe8\x90" + b'\xd3bTD\x15_\xf6\xc3\xfb\xb3E\xfc\xd6\x98{\xc6\\fz\x81\xa99\x85\xcb' + b'\xa5\xb1\x1d\x94bqW\x1a!;z~\x18\x88\xe8i\xdb\x1b\x8d\x8d' + b'\x06\xaa\x0e\x99s+5k\x00\xe4\xffh\xfe\xdbt\xa6\x1bU\xde\xa3' + b'\xef\xcb\x86\x9e\x81\x16j\n\x9d\xbc\xbbC\x80?\x010\xc7Jj;' + b'\xc4\xe5\x86\xd5\x0e0d#\xc6;\xb8\xd1\xc7c\xb5&8?\xd9J\xe5\xden\xb9' + b'\xe9cb4\xbb\xe6\x14\xe0\xe7l\x1b\x85\x94\x1fh\xf1n\xdeZ\xbe' + b'\x88\xff\xc2e\xca\xdc,B-\x8ac\xc9\xdf\xf5|&\xe4LL\xf0\x1f\xaa8\xbd' + b'\xc26\x94bVi\xd3\x0c\x1c\xb6\xbb\x99F\x8f\x0e\xcc\x8e4\xc6/^W\xf5?' + b'\xdc\x84(\x14dO\x9aD6\x0f4\xa3,\x0c\x0bS\x9fJ\xe1\xacc^\x8a0\t\x80D[' + b'\xb8\xe6\x86\xb0\xe8\xd4\xf9\x1en\xf1\xf5^\xeb\xb8\xb8\xf8' + b')\xa8\xbf\xaa\x84\x86\xb1a \x95\x16\x08\x1c\xbb@\xbd+\r/\xfb' + b'\x92\xfbh\xf1\x8d3\xf9\x92\xde`\xf1\x86\x03\xaa+\xd9\xd9\xc6P\xaf' + b'\xe3-\xea\xa5\x0fB\xca\xde\xd5n^\xe3/\xbf\xa6w\xc8\x0e<M' + b'\xc2\x1e!\xd4\xc6E\xf2\xad\x0c\xbc\x1d\x88Y\x03\x98<\x92\xd9\xa6B' + b'\xc7\x83\xb5"\x97D|&\xc4\xd4\xfad\x0e\xde\x06\xa3\xc2\x9c`\xf2' + b'7\x03\x1a\xed\xd80\x10\xe9\x0co\x10\xcf\x18\x16\xa7\x1c' + b"\xe5\x96\xa4\xd9\xe1\xa5v;]\xb7\xa9\xdc'hA\xe3\x9c&\x98\x0b9\xdf~@" + b'\xf8\xact\x87<\xf94\x0c\x9d\x93\xb0)\xe1\xa2\x0f\x1e=:&\xd56\xa5A+' + b'\xab\xc4\x00\x8d\x81\x93\xd4\xd8<\x82k\\d\xd8v\xab\xbd^l5C?\xd4\xa0' + b'M\x12C\xc8\x80\r\xc83\xe8\xc0\xf5\xdf\xca\x05\xf4BPjy\xbe\x91\x9bzE' + b"\xd8[\x93oT\r\x13\x16'\x1a\xbd*H\xd6\xfe\r\xf3\x91M\x8b\xee\x8f7f" + b"\x0b;\xaa\x85\xf2\xdd'\x0fwM \xbd\x13\xb9\xe5\xb8\xb7 D+P\x1c\xe4g" + b'n\xd2\xf1kc\x15\xaf\xc6\x90V\x03\xc2UovfZ\xcc\xd23^\xb3\xe7\xbf' + b'\xacv\x1d\x82\xedx\xa3J\xa9\xb7\xcf\x0c\xe6j\x96n*o\x18>' + b'\xc6\xfd\x97_+D{\x03\x15\xe8s\xb1\xc8HAG\xcf\xf4\x1a\xdd' + b'\xad\x11\xbf\x157q+\xdeW\x89g"X\x82\xfd~\xf7\xab4\xf6`\xab\xf1q' + b')\x82\x10K\xe9sV\xfe\xe45\xafs*\x14\xa7;\xac{\x06\x9d<@\x93G' + b'j\x1d\xefL\xe9\xd8\x92\x19&\xa1\x16\x19\x04\tu5\x01]\xf6\xf4' + b'\xcd\\\xd8A|I\xd4\xeb\x05\x88C\xc6e\xacQ\xe9*\x97~\x9au\xf8Xy' + b"\x17P\x10\x9f\n\x8c\xe2fZEu>\x9b\x1e\x91\x0b'`\xbd\xc0\xa8\x86c\x1d" + b'Z\xe2\xdc8j\x95\xffU\x90\x1e\xf4o\xbc\xe5\xe3e>\xd2R\xc0b#\xbc\x15' + b'H-\xb9!\xde\x9d\x90k\xdew\x9b{\x99\xde\xf7/K)A\xfd\xf5\xe6:\xda' + b'UM\xcc\xbb\xa2\x0b\x9a\x93\xf5{9\xc0 \xd2((6i\xc0\xbbu\xd8\x9e\x8d' + b'\xf8\x04q\x10\xd4\x14\x9e7-\xb9B\xea\x01Q8\xc8v\x9a\x12A\x88Cd\x92' + b"\x1c\x8c!\xf4\x94\x87'\xe3\xcd\xae\xf7\xd8\x93\xfa\xde\xa8b\x9e\xee2" + b'K\xdb\x00l\x9d\t\xb1|D\x05U\xbb\xf4>\xf1w\x887\xd1}W\x9d|g|1\xb0\x13' + b"\xa3 \xe5\xbfm@\xc06+\xb7\t\xcf\x15D\x9a \x1fM\x1f\xd2\xb5'\xa9\xbb" + b'~Co\x82\xfa\xc2\t\xe6f\xfc\xbeI\xae1\x8e\xbe\xb8\xcf\x86\x17' + b'\x9f\xe2`\xbd\xaf\xba\xb9\xbc\x1b\xa3\xcd\x82\x8fwc\xefd\xa9\xd5\x14' + b'\xe2C\xafUE\xb6\x11MJH\xd0=\x05\xd4*I\xff"\r\x1b^\xcaS6=\xec@\xd5' + b'\x11,\xe0\x87Gr\xaa[\xb8\xbc>n\xbd\x81\x0c\x07<\xe9\x92(' + b'\xb2\xff\xac}\xe7\xb6\x15\x90\x9f~4\x9a\xe6\xd6\xd8s\xed\x99tf' + b'\xa0f\xf8\xf1\x87\t\x96/)\x85\xb6\n\xd7\xb2w\x0b\xbc\xba\x99\xee' + b'Q\xeen\x1d\xad\x03\xc3s\xd1\xfd\xa2\xc6\xb7\x9a\x9c(G<6\xad[~H ' + b'\x16\x89\x89\xd0\xc3\xd2\xca~\xac\xea\xa5\xed\xe5\xfb\r:' + b'\x8e\xa6\xf1e\xbb\xba\xbd\xe0(\xa3\x89_\x01(\xb5c\xcc\x9f\x1fg' + b'v\xfd\x17\xb3\x08S=S\xee\xfc\x85>\x91\x8d\x8d\nYR\xb3G\xd1A\xa2\xb1' + b'\xec\xb0\x01\xd2\xcd\xf9\xfe\x82\x06O\xb3\xecd\xa9c\xe0\x8eP\x90\xce' + b'\xe0\xcd\xd8\xd8\xdc\x9f\xaa\x01"[Q~\xe4\x88\xa1#\xc1\x12C\xcf' + b'\xbe\x80\x11H\xbf\x86\xd8\xbem\xcfWFQ(X\x01DK\xdfB\xaa\x10.-' + b'\xd5\x9e|\x86\x15\x86N]\xc7Z\x17\xcd=\xd7)M\xde\x15\xa4LTi\xa0\x15' + b'\xd1\xe7\xbdN\xa4?\xd1\xe7\x02\xfe4\xe4O\x89\x98&\x96\x0f\x02\x9c' + b'\x9e\x19\xaa\x13u7\xbd0\xdc\xd8\x93\xf4BNE\x1d\x93\x82\x81\x16' + b'\xe5y\xcf\x98D\xca\x9a\xe2\xfd\xcdL\xcc\xd1\xfc_\x0b\x1c\xa0]\xdc' + b'\xa91 \xc9c\xd8\xbf\x97\xcfp\xe6\x19-\xad\xff\xcc\xd1N(\xe8' + b'\xeb#\x182\x96I\xf7l\xf3r\x00' +) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py new file mode 100644 index 00000000000..0169948e453 --- /dev/null +++ b/Lib/test/test_mailbox.py @@ -0,0 +1,2493 @@ +import os +import sys +import time +import socket +import email +import email.message +import re +import io +import tempfile +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import refleak_helper +from test.support import socket_helper +import unittest +import textwrap +import mailbox +import glob + + +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + + +class TestBase: + + all_mailbox_types = (mailbox.Message, mailbox.MaildirMessage, + mailbox.mboxMessage, mailbox.MHMessage, + mailbox.BabylMessage, mailbox.MMDFMessage) + + def _check_sample(self, msg): + # Inspect a mailbox.Message representation of the sample message + self.assertIsInstance(msg, email.message.Message) + self.assertIsInstance(msg, mailbox.Message) + for key, value in _sample_headers: + self.assertIn(value, msg.get_all(key)) + self.assertTrue(msg.is_multipart()) + self.assertEqual(len(msg.get_payload()), len(_sample_payloads)) + for i, payload in enumerate(_sample_payloads): + part = msg.get_payload(i) + self.assertIsInstance(part, email.message.Message) + self.assertNotIsInstance(part, mailbox.Message) + self.assertEqual(part.get_payload(), payload) + + def _delete_recursively(self, target): + # Delete a file or delete a directory recursively + if os.path.isdir(target): + os_helper.rmtree(target) + elif os.path.exists(target): + os_helper.unlink(target) + + +class TestMailbox(TestBase): + + maxDiff = None + + _factory = None # Overridden by subclasses to reuse tests + _template = 'From: foo\n\n%s\n' + + def setUp(self): + self._path = os_helper.TESTFN + self._delete_recursively(self._path) + self._box = self._factory(self._path) + + def tearDown(self): + self._box.close() + self._delete_recursively(self._path) + + def test_add(self): + # Add copies of a sample message + keys = [] + keys.append(self._box.add(self._template % 0)) + self.assertEqual(len(self._box), 1) + keys.append(self._box.add(mailbox.Message(_sample_message))) + self.assertEqual(len(self._box), 2) + keys.append(self._box.add(email.message_from_string(_sample_message))) + self.assertEqual(len(self._box), 3) + keys.append(self._box.add(io.BytesIO(_bytes_sample_message))) + self.assertEqual(len(self._box), 4) + keys.append(self._box.add(_sample_message)) + self.assertEqual(len(self._box), 5) + keys.append(self._box.add(_bytes_sample_message)) + self.assertEqual(len(self._box), 6) + with self.assertWarns(DeprecationWarning): + keys.append(self._box.add( + io.TextIOWrapper(io.BytesIO(_bytes_sample_message), encoding="utf-8"))) + self.assertEqual(len(self._box), 7) + self.assertEqual(self._box.get_string(keys[0]), self._template % 0) + for i in (1, 2, 3, 4, 5, 6): + self._check_sample(self._box[keys[i]]) + + _nonascii_msg = textwrap.dedent("""\ + From: foo + Subject: Falinaptár házhozszállítással. Már rendeltél? + + 0 + """) + + def test_add_invalid_8bit_bytes_header(self): + key = self._box.add(self._nonascii_msg.encode('latin-1')) + self.assertEqual(len(self._box), 1) + self.assertEqual(self._box.get_bytes(key), + self._nonascii_msg.encode('latin-1')) + + def test_invalid_nonascii_header_as_string(self): + subj = self._nonascii_msg.splitlines()[1] + key = self._box.add(subj.encode('latin-1')) + self.assertEqual(self._box.get_string(key), + 'Subject: =?unknown-8bit?b?RmFsaW5hcHThciBo4Xpob3pzeuFsbO104XNz' + 'YWwuIE3hciByZW5kZWx06Ww/?=\n\n') + + def test_add_nonascii_string_header_raises(self): + with self.assertRaisesRegex(ValueError, "ASCII-only"): + self._box.add(self._nonascii_msg) + self._box.flush() + self.assertEqual(len(self._box), 0) + self.assertMailboxEmpty() + + def test_add_that_raises_leaves_mailbox_empty(self): + class CustomError(Exception): ... + exc_msg = "a fake error" + + def raiser(*args, **kw): + raise CustomError(exc_msg) + support.patch(self, email.generator.BytesGenerator, 'flatten', raiser) + with self.assertRaisesRegex(CustomError, exc_msg): + self._box.add(email.message_from_string("From: Alphöso")) + self.assertEqual(len(self._box), 0) + self._box.close() + self.assertMailboxEmpty() + + _non_latin_bin_msg = textwrap.dedent("""\ + From: foo@bar.com + To: báz + Subject: Maintenant je vous présente mon collègue, le pouf célèbre + \tJean de Baddie + Mime-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + Да, они летят. + """).encode('utf-8') + + def test_add_8bit_body(self): + key = self._box.add(self._non_latin_bin_msg) + self.assertEqual(self._box.get_bytes(key), + self._non_latin_bin_msg) + with self._box.get_file(key) as f: + self.assertEqual(f.read(), + self._non_latin_bin_msg.replace(b'\n', + os.linesep.encode())) + self.assertEqual(self._box[key].get_payload(), + "Да, они летят.\n") + + def test_add_binary_file(self): + with tempfile.TemporaryFile('wb+') as f: + f.write(_bytes_sample_message) + f.seek(0) + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + _bytes_sample_message.split(b'\n')) + + def test_add_binary_nonascii_file(self): + with tempfile.TemporaryFile('wb+') as f: + f.write(self._non_latin_bin_msg) + f.seek(0) + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + self._non_latin_bin_msg.split(b'\n')) + + def test_add_text_file_warns(self): + with tempfile.TemporaryFile('w+', encoding='utf-8') as f: + f.write(_sample_message) + f.seek(0) + with self.assertWarns(DeprecationWarning): + key = self._box.add(f) + self.assertEqual(self._box.get_bytes(key).split(b'\n'), + _bytes_sample_message.split(b'\n')) + + def test_add_StringIO_warns(self): + with self.assertWarns(DeprecationWarning): + key = self._box.add(io.StringIO(self._template % "0")) + self.assertEqual(self._box.get_string(key), self._template % "0") + + def test_add_nonascii_StringIO_raises(self): + with self.assertWarns(DeprecationWarning): + with self.assertRaisesRegex(ValueError, "ASCII-only"): + self._box.add(io.StringIO(self._nonascii_msg)) + self.assertEqual(len(self._box), 0) + self._box.close() + self.assertMailboxEmpty() + + def test_remove(self): + # Remove messages using remove() + self._test_remove_or_delitem(self._box.remove) + + def test_delitem(self): + # Remove messages using __delitem__() + self._test_remove_or_delitem(self._box.__delitem__) + + def _test_remove_or_delitem(self, method): + # (Used by test_remove() and test_delitem().) + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(len(self._box), 2) + method(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + self.assertRaises(KeyError, lambda: method(key0)) + self.assertEqual(self._box.get_string(key1), self._template % 1) + key2 = self._box.add(self._template % 2) + self.assertEqual(len(self._box), 2) + method(key2) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key2]) + self.assertRaises(KeyError, lambda: method(key2)) + self.assertEqual(self._box.get_string(key1), self._template % 1) + method(key1) + self.assertEqual(len(self._box), 0) + self.assertRaises(KeyError, lambda: self._box[key1]) + self.assertRaises(KeyError, lambda: method(key1)) + + def test_discard(self, repetitions=10): + # Discard messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(len(self._box), 2) + self._box.discard(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + self._box.discard(key0) + self.assertEqual(len(self._box), 1) + self.assertRaises(KeyError, lambda: self._box[key0]) + + def test_get(self): + # Retrieve messages using get() + key0 = self._box.add(self._template % 0) + msg = self._box.get(key0) + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '0\n') + self.assertIsNone(self._box.get('foo')) + self.assertIs(self._box.get('foo', False), False) + self._box.close() + self._box = self._factory(self._path) + key1 = self._box.add(self._template % 1) + msg = self._box.get(key1) + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '1\n') + + def test_getitem(self): + # Retrieve message using __getitem__() + key0 = self._box.add(self._template % 0) + msg = self._box[key0] + self.assertEqual(msg['from'], 'foo') + self.assertEqual(msg.get_payload(), '0\n') + self.assertRaises(KeyError, lambda: self._box['foo']) + self._box.discard(key0) + self.assertRaises(KeyError, lambda: self._box[key0]) + + def test_get_message(self): + # Get Message representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + msg0 = self._box.get_message(key0) + self.assertIsInstance(msg0, mailbox.Message) + self.assertEqual(msg0['from'], 'foo') + self.assertEqual(msg0.get_payload(), '0\n') + self._check_sample(self._box.get_message(key1)) + + def test_get_bytes(self): + # Get bytes representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + self.assertEqual(self._box.get_bytes(key0), + (self._template % 0).encode('ascii')) + self.assertEqual(self._box.get_bytes(key1), _bytes_sample_message) + + def test_get_string(self): + # Get string representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + self.assertEqual(self._box.get_string(key0), self._template % 0) + self.assertEqual(self._box.get_string(key1).split('\n'), + _sample_message.split('\n')) + + def test_get_file(self): + # Get file representations of messages + key0 = self._box.add(self._template % 0) + key1 = self._box.add(_sample_message) + with self._box.get_file(key0) as file: + data0 = file.read() + with self._box.get_file(key1) as file: + data1 = file.read() + self.assertEqual(data0.decode('ascii').replace(os.linesep, '\n'), + self._template % 0) + self.assertEqual(data1.decode('ascii').replace(os.linesep, '\n'), + _sample_message) + + def test_get_file_can_be_closed_twice(self): + # Issue 11700 + key = self._box.add(_sample_message) + f = self._box.get_file(key) + f.close() + f.close() + + def test_iterkeys(self): + # Get keys using iterkeys() + self._check_iteration(self._box.iterkeys, do_keys=True, do_values=False) + + def test_keys(self): + # Get keys using keys() + self._check_iteration(self._box.keys, do_keys=True, do_values=False) + + def test_itervalues(self): + # Get values using itervalues() + self._check_iteration(self._box.itervalues, do_keys=False, + do_values=True) + + def test_iter(self): + # Get values using __iter__() + self._check_iteration(self._box.__iter__, do_keys=False, + do_values=True) + + def test_values(self): + # Get values using values() + self._check_iteration(self._box.values, do_keys=False, do_values=True) + + def test_iteritems(self): + # Get keys and values using iteritems() + self._check_iteration(self._box.iteritems, do_keys=True, + do_values=True) + + def test_items(self): + # Get keys and values using items() + self._check_iteration(self._box.items, do_keys=True, do_values=True) + + def _check_iteration(self, method, do_keys, do_values, repetitions=10): + for value in method(): + self.fail("Not empty") + keys, values = [], [] + for i in range(repetitions): + keys.append(self._box.add(self._template % i)) + values.append(self._template % i) + if do_keys and not do_values: + returned_keys = list(method()) + elif do_values and not do_keys: + returned_values = list(method()) + else: + returned_keys, returned_values = [], [] + for key, value in method(): + returned_keys.append(key) + returned_values.append(value) + if do_keys: + self.assertEqual(len(keys), len(returned_keys)) + self.assertEqual(set(keys), set(returned_keys)) + if do_values: + count = 0 + for value in returned_values: + self.assertEqual(value['from'], 'foo') + self.assertLess(int(value.get_payload()), repetitions) + count += 1 + self.assertEqual(len(values), count) + + def test_contains(self): + # Check existence of keys using __contains__() + self.assertNotIn('foo', self._box) + key0 = self._box.add(self._template % 0) + self.assertIn(key0, self._box) + self.assertNotIn('foo', self._box) + key1 = self._box.add(self._template % 1) + self.assertIn(key1, self._box) + self.assertIn(key0, self._box) + self.assertNotIn('foo', self._box) + self._box.remove(key0) + self.assertNotIn(key0, self._box) + self.assertIn(key1, self._box) + self.assertNotIn('foo', self._box) + self._box.remove(key1) + self.assertNotIn(key1, self._box) + self.assertNotIn(key0, self._box) + self.assertNotIn('foo', self._box) + + def test_len(self, repetitions=10): + # Get message count + keys = [] + for i in range(repetitions): + self.assertEqual(len(self._box), i) + keys.append(self._box.add(self._template % i)) + self.assertEqual(len(self._box), i + 1) + for i in range(repetitions): + self.assertEqual(len(self._box), repetitions - i) + self._box.remove(keys[i]) + self.assertEqual(len(self._box), repetitions - i - 1) + + def test_set_item(self): + # Modify messages using __setitem__() + key0 = self._box.add(self._template % 'original 0') + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + key1 = self._box.add(self._template % 'original 1') + self.assertEqual(self._box.get_string(key1), + self._template % 'original 1') + self._box[key0] = self._template % 'changed 0' + self.assertEqual(self._box.get_string(key0), + self._template % 'changed 0') + self._box[key1] = self._template % 'changed 1' + self.assertEqual(self._box.get_string(key1), + self._template % 'changed 1') + self._box[key0] = _sample_message + self._check_sample(self._box[key0]) + self._box[key1] = self._box[key0] + self._check_sample(self._box[key1]) + self._box[key0] = self._template % 'original 0' + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + self._check_sample(self._box[key1]) + self.assertRaises(KeyError, + lambda: self._box.__setitem__('foo', 'bar')) + self.assertRaises(KeyError, lambda: self._box['foo']) + self.assertEqual(len(self._box), 2) + + def test_clear(self, iterations=10): + # Remove all messages using clear() + keys = [] + for i in range(iterations): + self._box.add(self._template % i) + for i, key in enumerate(keys): + self.assertEqual(self._box.get_string(key), self._template % i) + self._box.clear() + self.assertEqual(len(self._box), 0) + for i, key in enumerate(keys): + self.assertRaises(KeyError, lambda: self._box.get_string(key)) + + def test_pop(self): + # Get and remove a message using pop() + key0 = self._box.add(self._template % 0) + self.assertIn(key0, self._box) + key1 = self._box.add(self._template % 1) + self.assertIn(key1, self._box) + self.assertEqual(self._box.pop(key0).get_payload(), '0\n') + self.assertNotIn(key0, self._box) + self.assertIn(key1, self._box) + key2 = self._box.add(self._template % 2) + self.assertIn(key2, self._box) + self.assertEqual(self._box.pop(key2).get_payload(), '2\n') + self.assertNotIn(key2, self._box) + self.assertIn(key1, self._box) + self.assertEqual(self._box.pop(key1).get_payload(), '1\n') + self.assertNotIn(key1, self._box) + self.assertEqual(len(self._box), 0) + + def test_popitem(self, iterations=10): + # Get and remove an arbitrary (key, message) using popitem() + keys = [] + for i in range(10): + keys.append(self._box.add(self._template % i)) + seen = [] + for i in range(10): + key, msg = self._box.popitem() + self.assertIn(key, keys) + self.assertNotIn(key, seen) + seen.append(key) + self.assertEqual(int(msg.get_payload()), keys.index(key)) + self.assertEqual(len(self._box), 0) + for key in keys: + self.assertRaises(KeyError, lambda: self._box[key]) + + def test_update(self): + # Modify multiple messages using update() + key0 = self._box.add(self._template % 'original 0') + key1 = self._box.add(self._template % 'original 1') + key2 = self._box.add(self._template % 'original 2') + self._box.update({key0: self._template % 'changed 0', + key2: _sample_message}) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % 'changed 0') + self.assertEqual(self._box.get_string(key1), + self._template % 'original 1') + self._check_sample(self._box[key2]) + self._box.update([(key2, self._template % 'changed 2'), + (key1, self._template % 'changed 1'), + (key0, self._template % 'original 0')]) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % 'original 0') + self.assertEqual(self._box.get_string(key1), + self._template % 'changed 1') + self.assertEqual(self._box.get_string(key2), + self._template % 'changed 2') + self.assertRaises(KeyError, + lambda: self._box.update({'foo': 'bar', + key0: self._template % "changed 0"})) + self.assertEqual(len(self._box), 3) + self.assertEqual(self._box.get_string(key0), + self._template % "changed 0") + self.assertEqual(self._box.get_string(key1), + self._template % "changed 1") + self.assertEqual(self._box.get_string(key2), + self._template % "changed 2") + + def test_flush(self): + # Write changes to disk + self._test_flush_or_close(self._box.flush, True) + + def test_popitem_and_flush_twice(self): + # See #15036. + self._box.add(self._template % 0) + self._box.add(self._template % 1) + self._box.flush() + + self._box.popitem() + self._box.flush() + self._box.popitem() + self._box.flush() + + def test_lock_unlock(self): + # Lock and unlock the mailbox + self.assertFalse(os.path.exists(self._get_lock_path())) + self._box.lock() + self.assertTrue(os.path.exists(self._get_lock_path())) + self._box.unlock() + self.assertFalse(os.path.exists(self._get_lock_path())) + + def test_close(self): + # Close mailbox and flush changes to disk + self._test_flush_or_close(self._box.close, False) + + def _test_flush_or_close(self, method, should_call_close): + contents = [self._template % i for i in range(3)] + self._box.add(contents[0]) + self._box.add(contents[1]) + self._box.add(contents[2]) + oldbox = self._box + method() + if should_call_close: + self._box.close() + self._box = self._factory(self._path) + keys = self._box.keys() + self.assertEqual(len(keys), 3) + for key in keys: + self.assertIn(self._box.get_string(key), contents) + oldbox.close() + + def test_dump_message(self): + # Write message representations to disk + for input in (email.message_from_string(_sample_message), + _sample_message, io.BytesIO(_bytes_sample_message)): + output = io.BytesIO() + self._box._dump_message(input, output) + self.assertEqual(output.getvalue(), + _bytes_sample_message.replace(b'\n', os.linesep.encode())) + output = io.BytesIO() + self.assertRaises(TypeError, + lambda: self._box._dump_message(None, output)) + + def _get_lock_path(self): + # Return the path of the dot lock file. May be overridden. + return self._path + '.lock' + + +class TestMailboxSuperclass(TestBase, unittest.TestCase): + + def test_notimplemented(self): + # Test that all Mailbox methods raise NotImplementedException. + box = mailbox.Mailbox('path') + self.assertRaises(NotImplementedError, lambda: box.add('')) + self.assertRaises(NotImplementedError, lambda: box.remove('')) + self.assertRaises(NotImplementedError, lambda: box.__delitem__('')) + self.assertRaises(NotImplementedError, lambda: box.discard('')) + self.assertRaises(NotImplementedError, lambda: box.__setitem__('', '')) + self.assertRaises(NotImplementedError, lambda: box.iterkeys()) + self.assertRaises(NotImplementedError, lambda: box.keys()) + self.assertRaises(NotImplementedError, lambda: box.itervalues().__next__()) + self.assertRaises(NotImplementedError, lambda: box.__iter__().__next__()) + self.assertRaises(NotImplementedError, lambda: box.values()) + self.assertRaises(NotImplementedError, lambda: box.iteritems().__next__()) + self.assertRaises(NotImplementedError, lambda: box.items()) + self.assertRaises(NotImplementedError, lambda: box.get('')) + self.assertRaises(NotImplementedError, lambda: box.__getitem__('')) + self.assertRaises(NotImplementedError, lambda: box.get_message('')) + self.assertRaises(NotImplementedError, lambda: box.get_string('')) + self.assertRaises(NotImplementedError, lambda: box.get_bytes('')) + self.assertRaises(NotImplementedError, lambda: box.get_file('')) + self.assertRaises(NotImplementedError, lambda: '' in box) + self.assertRaises(NotImplementedError, lambda: box.__contains__('')) + self.assertRaises(NotImplementedError, lambda: box.__len__()) + self.assertRaises(NotImplementedError, lambda: box.clear()) + self.assertRaises(NotImplementedError, lambda: box.pop('')) + self.assertRaises(NotImplementedError, lambda: box.popitem()) + self.assertRaises(NotImplementedError, lambda: box.update((('', ''),))) + self.assertRaises(NotImplementedError, lambda: box.flush()) + self.assertRaises(NotImplementedError, lambda: box.lock()) + self.assertRaises(NotImplementedError, lambda: box.unlock()) + self.assertRaises(NotImplementedError, lambda: box.close()) + + +class TestMaildir(TestMailbox, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.Maildir(path, factory) + + def setUp(self): + TestMailbox.setUp(self) + if (os.name == 'nt') or (sys.platform == 'cygwin'): + self._box.colon = '!' + + def assertMailboxEmpty(self): + self.assertEqual(os.listdir(os.path.join(self._path, 'tmp')), []) + + def test_add_MM(self): + # Add a MaildirMessage instance + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_info('foo') + key = self._box.add(msg) + self.assertTrue(os.path.exists(os.path.join(self._path, 'cur', '%s%sfoo' % + (key, self._box.colon)))) + + def test_get_MM(self): + # Get a MaildirMessage instance + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_flags('RF') + key = self._box.add(msg) + msg_returned = self._box.get_message(key) + self.assertIsInstance(msg_returned, mailbox.MaildirMessage) + self.assertEqual(msg_returned.get_subdir(), 'cur') + self.assertEqual(msg_returned.get_flags(), 'FR') + + def test_set_MM(self): + # Set with a MaildirMessage instance + msg0 = mailbox.MaildirMessage(self._template % 0) + msg0.set_flags('TP') + key = self._box.add(msg0) + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), 'PT') + msg1 = mailbox.MaildirMessage(self._template % 1) + self._box[key] = msg1 + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), '') + self.assertEqual(msg_returned.get_payload(), '1\n') + msg2 = mailbox.MaildirMessage(self._template % 2) + msg2.set_info('2,S') + self._box[key] = msg2 + self._box[key] = self._template % 3 + msg_returned = self._box.get_message(key) + self.assertEqual(msg_returned.get_subdir(), 'new') + self.assertEqual(msg_returned.get_flags(), 'S') + self.assertEqual(msg_returned.get_payload(), '3\n') + + def test_consistent_factory(self): + # Add a message. + msg = mailbox.MaildirMessage(self._template % 0) + msg.set_subdir('cur') + msg.set_flags('RF') + key = self._box.add(msg) + + # Create new mailbox with + class FakeMessage(mailbox.MaildirMessage): + pass + box = mailbox.Maildir(self._path, factory=FakeMessage) + box.colon = self._box.colon + msg2 = box.get_message(key) + self.assertIsInstance(msg2, FakeMessage) + + def test_initialize_new(self): + # Initialize a non-existent mailbox + self.tearDown() + self._box = mailbox.Maildir(self._path) + self._check_basics() + self._delete_recursively(self._path) + self._box = self._factory(self._path, factory=None) + self._check_basics() + + def test_initialize_existing(self): + # Initialize an existing mailbox + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + self._box = mailbox.Maildir(self._path) + self._check_basics() + + def test_filename_leading_dot(self): + self.tearDown() + for subdir in '', 'tmp', 'new', 'cur': + os.mkdir(os.path.normpath(os.path.join(self._path, subdir))) + for subdir in 'tmp', 'new', 'cur': + fname = os.path.join(self._path, subdir, '.foo' + subdir) + with open(fname, 'wb') as f: + f.write(b"@") + self._box = mailbox.Maildir(self._path) + self.assertNotIn('.footmp', self._box) + self.assertNotIn('.foonew', self._box) + self.assertNotIn('.foocur', self._box) + self.assertEqual(list(self._box.iterkeys()), []) + + def _check_basics(self, factory=None): + # (Used by test_open_new() and test_open_existing().) + self.assertEqual(self._box._path, os.path.abspath(self._path)) + self.assertEqual(self._box._factory, factory) + for subdir in '', 'tmp', 'new', 'cur': + path = os.path.join(self._path, subdir) + self.assertTrue(os.path.isdir(path), f"Not a directory: {path!r}") + + def test_list_folders(self): + # List folders + self._box.add_folder('one') + self._box.add_folder('two') + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 3) + self.assertEqual(set(self._box.list_folders()), + set(('one', 'two', 'three'))) + + def test_get_folder(self): + # Open folders + self._box.add_folder('foo.bar') + folder0 = self._box.get_folder('foo.bar') + folder0.add(self._template % 'bar') + self.assertTrue(os.path.isdir(os.path.join(self._path, '.foo.bar'))) + folder1 = self._box.get_folder('foo.bar') + self.assertEqual(folder1.get_string(folder1.keys()[0]), + self._template % 'bar') + + def test_add_and_remove_folders(self): + # Delete folders + self._box.add_folder('one') + self._box.add_folder('two') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('one', 'two'))) + self._box.remove_folder('one') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('two', 'three'))) + self._box.remove_folder('three') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.remove_folder('two') + self.assertEqual(len(self._box.list_folders()), 0) + self.assertEqual(self._box.list_folders(), []) + + def test_clean(self): + # Remove old files from 'tmp' + foo_path = os.path.join(self._path, 'tmp', 'foo') + bar_path = os.path.join(self._path, 'tmp', 'bar') + with open(foo_path, 'w', encoding='utf-8') as f: + f.write("@") + with open(bar_path, 'w', encoding='utf-8') as f: + f.write("@") + self._box.clean() + self.assertTrue(os.path.exists(foo_path)) + self.assertTrue(os.path.exists(bar_path)) + foo_stat = os.stat(foo_path) + os.utime(foo_path, (time.time() - 129600 - 2, + foo_stat.st_mtime)) + self._box.clean() + self.assertFalse(os.path.exists(foo_path)) + self.assertTrue(os.path.exists(bar_path)) + + def test_create_tmp(self, repetitions=10): + # Create files in tmp directory + hostname = socket.gethostname() + if '/' in hostname: + hostname = hostname.replace('/', r'\057') + if ':' in hostname: + hostname = hostname.replace(':', r'\072') + pid = os.getpid() + pattern = re.compile(r"(?P<time>\d+)\.M(?P<M>\d{1,6})P(?P<P>\d+)" + r"Q(?P<Q>\d+)\.(?P<host>[^:/]*)") + previous_groups = None + for x in range(repetitions): + tmp_file = self._box._create_tmp() + head, tail = os.path.split(tmp_file.name) + self.assertEqual(head, os.path.abspath(os.path.join(self._path, + "tmp")), + "File in wrong location: '%s'" % head) + match = pattern.match(tail) + self.assertIsNotNone(match, "Invalid file name: '%s'" % tail) + groups = match.groups() + if previous_groups is not None: + self.assertGreaterEqual(int(groups[0]), int(previous_groups[0]), + "Non-monotonic seconds: '%s' before '%s'" % + (previous_groups[0], groups[0])) + if int(groups[0]) == int(previous_groups[0]): + self.assertGreaterEqual(int(groups[1]), int(previous_groups[1]), + "Non-monotonic milliseconds: '%s' before '%s'" % + (previous_groups[1], groups[1])) + self.assertEqual(int(groups[2]), pid, + "Process ID mismatch: '%s' should be '%s'" % + (groups[2], pid)) + self.assertEqual(int(groups[3]), int(previous_groups[3]) + 1, + "Non-sequential counter: '%s' before '%s'" % + (previous_groups[3], groups[3])) + self.assertEqual(groups[4], hostname, + "Host name mismatch: '%s' should be '%s'" % + (groups[4], hostname)) + previous_groups = groups + tmp_file.write(_bytes_sample_message) + tmp_file.seek(0) + self.assertEqual(tmp_file.read(), _bytes_sample_message) + tmp_file.close() + file_count = len(os.listdir(os.path.join(self._path, "tmp"))) + self.assertEqual(file_count, repetitions, + "Wrong file count: '%s' should be '%s'" % + (file_count, repetitions)) + + def test_refresh(self): + # Update the table of contents + self.assertEqual(self._box._toc, {}) + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + self.assertEqual(self._box._toc, {}) + self._box._refresh() + self.assertEqual(self._box._toc, {key0: os.path.join('new', key0), + key1: os.path.join('new', key1)}) + key2 = self._box.add(self._template % 2) + self.assertEqual(self._box._toc, {key0: os.path.join('new', key0), + key1: os.path.join('new', key1)}) + self._box._refresh() + self.assertEqual(self._box._toc, {key0: os.path.join('new', key0), + key1: os.path.join('new', key1), + key2: os.path.join('new', key2)}) + + def test_refresh_after_safety_period(self): + # Issue #13254: Call _refresh after the "file system safety + # period" of 2 seconds has passed; _toc should still be + # updated because this is the first call to _refresh. + key0 = self._box.add(self._template % 0) + key1 = self._box.add(self._template % 1) + + self._box = self._factory(self._path) + self.assertEqual(self._box._toc, {}) + + # Emulate sleeping. Instead of sleeping for 2 seconds, use the + # skew factor to make _refresh think that the filesystem + # safety period has passed and re-reading the _toc is only + # required if mtimes differ. + self._box._skewfactor = -3 + + self._box._refresh() + self.assertEqual(sorted(self._box._toc.keys()), sorted([key0, key1])) + + def test_lookup(self): + # Look up message subpaths in the TOC + self.assertRaises(KeyError, lambda: self._box._lookup('foo')) + key0 = self._box.add(self._template % 0) + self.assertEqual(self._box._lookup(key0), os.path.join('new', key0)) + os.remove(os.path.join(self._path, 'new', key0)) + self.assertEqual(self._box._toc, {key0: os.path.join('new', key0)}) + # Be sure that the TOC is read back from disk (see issue #6896 + # about bad mtime behaviour on some systems). + self._box.flush() + self.assertRaises(KeyError, lambda: self._box._lookup(key0)) + self.assertEqual(self._box._toc, {}) + + def test_lock_unlock(self): + # Lock and unlock the mailbox. For Maildir, this does nothing. + self._box.lock() + self._box.unlock() + + def test_get_info(self): + # Test getting message info from Maildir, not the message. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_info(key), '') + msg.set_info('OurTestInfo') + self._box[key] = msg + self.assertEqual(self._box.get_info(key), 'OurTestInfo') + + def test_set_info(self): + # Test setting message info from Maildir, not the message. + # This should immediately rename the message file. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + def check_info(oldinfo, newinfo): + oldfilename = os.path.join(self._box._path, self._box._lookup(key)) + newsubpath = self._box._lookup(key).split(self._box.colon)[0] + if newinfo: + newsubpath += self._box.colon + newinfo + newfilename = os.path.join(self._box._path, newsubpath) + # assert initial conditions + self.assertEqual(self._box.get_info(key), oldinfo) + if not oldinfo: + self.assertNotIn(self._box._lookup(key), self._box.colon) + self.assertTrue(os.path.exists(oldfilename)) + if oldinfo != newinfo: + self.assertFalse(os.path.exists(newfilename)) + # do the rename + self._box.set_info(key, newinfo) + # assert post conditions + if not newinfo: + self.assertNotIn(self._box._lookup(key), self._box.colon) + if oldinfo != newinfo: + self.assertFalse(os.path.exists(oldfilename)) + self.assertTrue(os.path.exists(newfilename)) + self.assertEqual(self._box.get_info(key), newinfo) + # none -> has info + check_info('', 'info1') + # has info -> same info + check_info('info1', 'info1') + # has info -> different info + check_info('info1', 'info2') + # has info -> none + check_info('info2', '') + # none -> none + check_info('', '') + + def test_get_flags(self): + # Test getting message flags from Maildir, not the message. + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + msg.set_flags('T') + self._box[key] = msg + self.assertEqual(self._box.get_flags(key), 'T') + + def test_set_flags(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + self._box.set_flags(key, 'S') + self.assertEqual(self._box.get_flags(key), 'S') + + def test_add_flag(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self.assertEqual(self._box.get_flags(key), '') + self._box.add_flag(key, 'B') + self.assertEqual(self._box.get_flags(key), 'B') + self._box.add_flag(key, 'B') + self.assertEqual(self._box.get_flags(key), 'B') + self._box.add_flag(key, 'AC') + self.assertEqual(self._box.get_flags(key), 'ABC') + + def test_remove_flag(self): + msg = mailbox.MaildirMessage(self._template % 0) + key = self._box.add(msg) + self._box.set_flags(key, 'abc') + self.assertEqual(self._box.get_flags(key), 'abc') + self._box.remove_flag(key, 'b') + self.assertEqual(self._box.get_flags(key), 'ac') + self._box.remove_flag(key, 'b') + self.assertEqual(self._box.get_flags(key), 'ac') + self._box.remove_flag(key, 'ac') + self.assertEqual(self._box.get_flags(key), '') + + def test_folder (self): + # Test for bug #1569790: verify that folders returned by .get_folder() + # use the same factory function. + def dummy_factory (s): + return None + box = self._factory(self._path, factory=dummy_factory) + folder = box.add_folder('folder1') + self.assertIs(folder._factory, dummy_factory) + + folder1_alias = box.get_folder('folder1') + self.assertIs(folder1_alias._factory, dummy_factory) + + def test_directory_in_folder (self): + # Test that mailboxes still work if there's a stray extra directory + # in a folder. + for i in range(10): + self._box.add(mailbox.Message(_sample_message)) + + # Create a stray directory + os.mkdir(os.path.join(self._path, 'cur', 'stray-dir')) + + # Check that looping still works with the directory present. + for msg in self._box: + pass + + @unittest.skipUnless(hasattr(os, 'umask'), 'test needs os.umask()') + def test_file_permissions(self): + # Verify that message files are created without execute permissions + msg = mailbox.MaildirMessage(self._template % 0) + orig_umask = os.umask(0) + try: + key = self._box.add(msg) + finally: + os.umask(orig_umask) + path = os.path.join(self._path, self._box._lookup(key)) + mode = os.stat(path).st_mode + self.assertFalse(mode & 0o111) + + @unittest.skipUnless(hasattr(os, 'umask'), 'test needs os.umask()') + def test_folder_file_perms(self): + # From bug #3228, we want to verify that the file created inside a Maildir + # subfolder isn't marked as executable. + orig_umask = os.umask(0) + try: + subfolder = self._box.add_folder('subfolder') + finally: + os.umask(orig_umask) + + path = os.path.join(subfolder._path, 'maildirfolder') + st = os.stat(path) + perms = st.st_mode + self.assertFalse((perms & 0o111)) # Execute bits should all be off. + + def test_reread(self): + # Do an initial unconditional refresh + self._box._refresh() + + # Put the last modified times more than two seconds into the past + # (because mtime may have a two second granularity) + for subdir in ('cur', 'new'): + os.utime(os.path.join(self._box._path, subdir), + (time.time()-5,)*2) + + # Because mtime has a two second granularity in worst case (FAT), a + # refresh is done unconditionally if called for within + # two-second-plus-a-bit of the last one, just in case the mbox has + # changed; so now we have to wait for that interval to expire. + # + # Because this is a test, emulate sleeping. Instead of + # sleeping for 2 seconds, use the skew factor to make _refresh + # think that 2 seconds have passed and re-reading the _toc is + # only required if mtimes differ. + self._box._skewfactor = -3 + + # Re-reading causes the ._toc attribute to be assigned a new dictionary + # object, so we'll check that the ._toc attribute isn't a different + # object. + orig_toc = self._box._toc + def refreshed(): + return self._box._toc is not orig_toc + + self._box._refresh() + self.assertFalse(refreshed()) + + # Now, write something into cur and remove it. This changes + # the mtime and should cause a re-read. Note that "sleep + # emulation" is still in effect, as skewfactor is -3. + filename = os.path.join(self._path, 'cur', 'stray-file') + os_helper.create_empty_file(filename) + os.unlink(filename) + self._box._refresh() + self.assertTrue(refreshed()) + + +class _TestSingleFile(TestMailbox): + '''Common tests for single-file mailboxes''' + + def test_add_doesnt_rewrite(self): + # When only adding messages, flush() should not rewrite the + # mailbox file. See issue #9559. + + # Inode number changes if the contents are written to another + # file which is then renamed over the original file. So we + # must check that the inode number doesn't change. + inode_before = os.stat(self._path).st_ino + + self._box.add(self._template % 0) + self._box.flush() + + inode_after = os.stat(self._path).st_ino + self.assertEqual(inode_before, inode_after) + + # Make sure the message was really added + self._box.close() + self._box = self._factory(self._path) + self.assertEqual(len(self._box), 1) + + def test_permissions_after_flush(self): + # See issue #5346 + + # Make the mailbox world writable. It's unlikely that the new + # mailbox file would have these permissions after flush(), + # because umask usually prevents it. + mode = os.stat(self._path).st_mode | 0o666 + os.chmod(self._path, mode) + + self._box.add(self._template % 0) + i = self._box.add(self._template % 1) + # Need to remove one message to make flush() create a new file + self._box.remove(i) + self._box.flush() + + self.assertEqual(os.stat(self._path).st_mode, mode) + + @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') + def test_ownership_after_flush(self): + # See issue gh-117467 + + pwd = import_helper.import_module('pwd') + grp = import_helper.import_module('grp') + st = os.stat(self._path) + + for e in pwd.getpwall(): + if e.pw_uid != st.st_uid: + other_uid = e.pw_uid + break + else: + self.skipTest("test needs more than one user") + + for e in grp.getgrall(): + if e.gr_gid != st.st_gid: + other_gid = e.gr_gid + break + else: + self.skipTest("test needs more than one group") + + try: + os.chown(self._path, other_uid, other_gid) + except OSError: + self.skipTest('test needs root privilege') + # Change permissions as in test_permissions_after_flush. + mode = st.st_mode | 0o666 + os.chmod(self._path, mode) + + self._box.add(self._template % 0) + i = self._box.add(self._template % 1) + # Need to remove one message to make flush() create a new file + self._box.remove(i) + self._box.flush() + + st = os.stat(self._path) + self.assertEqual(st.st_uid, other_uid) + self.assertEqual(st.st_gid, other_gid) + self.assertEqual(st.st_mode, mode) + + +class _TestMboxMMDF(_TestSingleFile): + + def tearDown(self): + super().tearDown() + self._box.close() + self._delete_recursively(self._path) + for lock_remnant in glob.glob(glob.escape(self._path) + '.*'): + os_helper.unlink(lock_remnant) + + def assertMailboxEmpty(self): + with open(self._path, 'rb') as f: + self.assertEqual(f.readlines(), []) + + def test_get_bytes_from(self): + # Get bytes representations of messages with _unixfrom. + unixfrom = 'From foo@bar blah\n' + key0 = self._box.add(unixfrom + self._template % 0) + key1 = self._box.add(unixfrom + _sample_message) + self.assertEqual(self._box.get_bytes(key0, from_=False), + (self._template % 0).encode('ascii')) + self.assertEqual(self._box.get_bytes(key1, from_=False), + _bytes_sample_message) + self.assertEqual(self._box.get_bytes(key0, from_=True), + (unixfrom + self._template % 0).encode('ascii')) + self.assertEqual(self._box.get_bytes(key1, from_=True), + unixfrom.encode('ascii') + _bytes_sample_message) + + def test_get_string_from(self): + # Get string representations of messages with _unixfrom. + unixfrom = 'From foo@bar blah\n' + key0 = self._box.add(unixfrom + self._template % 0) + key1 = self._box.add(unixfrom + _sample_message) + self.assertEqual(self._box.get_string(key0, from_=False), + self._template % 0) + self.assertEqual(self._box.get_string(key1, from_=False).split('\n'), + _sample_message.split('\n')) + self.assertEqual(self._box.get_string(key0, from_=True), + unixfrom + self._template % 0) + self.assertEqual(self._box.get_string(key1, from_=True).split('\n'), + (unixfrom + _sample_message).split('\n')) + + def test_add_from_string(self): + # Add a string starting with 'From ' to the mailbox + key = self._box.add('From foo@bar blah\nFrom: foo\n\n0\n') + self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') + self.assertEqual(self._box[key].get_payload(), '0\n') + + def test_add_from_bytes(self): + # Add a byte string starting with 'From ' to the mailbox + key = self._box.add(b'From foo@bar blah\nFrom: foo\n\n0\n') + self.assertEqual(self._box[key].get_from(), 'foo@bar blah') + self.assertEqual(self._box[key].get_unixfrom(), 'From foo@bar blah') + self.assertEqual(self._box[key].get_payload(), '0\n') + + def test_add_mbox_or_mmdf_message(self): + # Add an mboxMessage or MMDFMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg = class_('From foo@bar blah\nFrom: foo\n\n0\n') + key = self._box.add(msg) + + def test_open_close_open(self): + # Open and inspect previously-created mailbox + values = [self._template % i for i in range(3)] + for value in values: + self._box.add(value) + self._box.close() + mtime = os.path.getmtime(self._path) + self._box = self._factory(self._path) + self.assertEqual(len(self._box), 3) + for key in self._box.iterkeys(): + self.assertIn(self._box.get_string(key), values) + self._box.close() + self.assertEqual(mtime, os.path.getmtime(self._path)) + + def test_add_and_close(self): + # Verifying that closing a mailbox doesn't change added items + self._box.add(_sample_message) + for i in range(3): + self._box.add(self._template % i) + self._box.add(_sample_message) + self._box._file.flush() + self._box._file.seek(0) + contents = self._box._file.read() + self._box.close() + with open(self._path, 'rb') as f: + self.assertEqual(contents, f.read()) + self._box = self._factory(self._path) + + @support.requires_fork() + @unittest.skipUnless(hasattr(socket, 'socketpair'), "Test needs socketpair().") + def test_lock_conflict(self): + # Fork off a child process that will lock the mailbox temporarily, + # unlock it and exit. + c, p = socket.socketpair() + self.addCleanup(c.close) + self.addCleanup(p.close) + + pid = os.fork() + if pid == 0: + # child + try: + # lock the mailbox, and signal the parent it can proceed + self._box.lock() + c.send(b'c') + + # wait until the parent is done, and unlock the mailbox + c.recv(1) + self._box.unlock() + finally: + os._exit(0) + + # In the parent, wait until the child signals it locked the mailbox. + p.recv(1) + try: + self.assertRaises(mailbox.ExternalClashError, + self._box.lock) + finally: + # Signal the child it can now release the lock and exit. + p.send(b'p') + # Wait for child to exit. Locking should now succeed. + support.wait_process(pid, exitcode=0) + + self._box.lock() + self._box.unlock() + + def test_relock(self): + # Test case for bug #1575506: the mailbox class was locking the + # wrong file object in its flush() method. + msg = "Subject: sub\n\nbody\n" + key1 = self._box.add(msg) + self._box.flush() + self._box.close() + + self._box = self._factory(self._path) + self._box.lock() + key2 = self._box.add(msg) + self._box.flush() + self.assertTrue(self._box._locked) + self._box.close() + + +class TestMbox(_TestMboxMMDF, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.mbox(path, factory) + + @unittest.skipUnless(hasattr(os, 'umask'), 'test needs os.umask()') + def test_file_perms(self): + # From bug #3228, we want to verify that the mailbox file isn't executable, + # even if the umask is set to something that would leave executable bits set. + # We only run this test on platforms that support umask. + try: + old_umask = os.umask(0o077) + self._box.close() + os.unlink(self._path) + self._box = mailbox.mbox(self._path, create=True) + self._box.add('') + self._box.close() + finally: + os.umask(old_umask) + + st = os.stat(self._path) + perms = st.st_mode + self.assertFalse((perms & 0o111)) # Execute bits should all be off. + + def test_terminating_newline(self): + message = email.message.Message() + message['From'] = 'john@example.com' + message.set_payload('No newline at the end') + i = self._box.add(message) + + # A newline should have been appended to the payload + message = self._box.get(i) + self.assertEqual(message.get_payload(), 'No newline at the end\n') + + def test_message_separator(self): + # Check there's always a single blank line after each message + self._box.add('From: foo\n\n0') # No newline at the end + with open(self._path, encoding='utf-8') as f: + data = f.read() + self.assertEndsWith(data, '0\n\n') + + self._box.add('From: foo\n\n0\n') # Newline at the end + with open(self._path, encoding='utf-8') as f: + data = f.read() + self.assertEndsWith(data, '0\n\n') + + +class TestMMDF(_TestMboxMMDF, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.MMDF(path, factory) + + +class TestMH(TestMailbox, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.MH(path, factory) + + def assertMailboxEmpty(self): + self.assertEqual(os.listdir(self._path), ['.mh_sequences']) + + def test_list_folders(self): + # List folders + self._box.add_folder('one') + self._box.add_folder('two') + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 3) + self.assertEqual(set(self._box.list_folders()), + set(('one', 'two', 'three'))) + + def test_get_folder(self): + # Open folders + def dummy_factory (s): + return None + self._box = self._factory(self._path, dummy_factory) + + new_folder = self._box.add_folder('foo.bar') + folder0 = self._box.get_folder('foo.bar') + folder0.add(self._template % 'bar') + self.assertTrue(os.path.isdir(os.path.join(self._path, 'foo.bar'))) + folder1 = self._box.get_folder('foo.bar') + self.assertEqual(folder1.get_string(folder1.keys()[0]), + self._template % 'bar') + + # Test for bug #1569790: verify that folders returned by .get_folder() + # use the same factory function. + self.assertIs(new_folder._factory, self._box._factory) + self.assertIs(folder0._factory, self._box._factory) + + def test_add_and_remove_folders(self): + # Delete folders + self._box.add_folder('one') + self._box.add_folder('two') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('one', 'two'))) + self._box.remove_folder('one') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.add_folder('three') + self.assertEqual(len(self._box.list_folders()), 2) + self.assertEqual(set(self._box.list_folders()), set(('two', 'three'))) + self._box.remove_folder('three') + self.assertEqual(len(self._box.list_folders()), 1) + self.assertEqual(set(self._box.list_folders()), set(('two',))) + self._box.remove_folder('two') + self.assertEqual(len(self._box.list_folders()), 0) + self.assertEqual(self._box.list_folders(), []) + + def test_sequences(self): + # Get and set sequences + self.assertEqual(self._box.get_sequences(), {}) + msg0 = mailbox.MHMessage(self._template % 0) + msg0.add_sequence('foo') + key0 = self._box.add(msg0) + self.assertEqual(self._box.get_sequences(), {'foo':[key0]}) + msg1 = mailbox.MHMessage(self._template % 1) + msg1.set_sequences(['bar', 'replied', 'foo']) + key1 = self._box.add(msg1) + self.assertEqual(self._box.get_sequences(), + {'foo':[key0, key1], 'bar':[key1], 'replied':[key1]}) + msg0.set_sequences(['flagged']) + self._box[key0] = msg0 + self.assertEqual(self._box.get_sequences(), + {'foo':[key1], 'bar':[key1], 'replied':[key1], + 'flagged':[key0]}) + self._box.remove(key1) + self.assertEqual(self._box.get_sequences(), {'flagged':[key0]}) + + self._box.set_sequences({'foo':[key0]}) + self.assertEqual(self._box.get_sequences(), {'foo':[key0]}) + + def test_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + self.assertEqual(box.get_sequences(), {}) + self.assertEqual(os.listdir(path), []) + box.set_sequences({}) + self.assertEqual(os.listdir(path), ['.mh_sequences']) + + def test_lock_unlock_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + box.lock() + box.unlock() + self.assertEqual(os.listdir(path), ['.mh_sequences']) + + def test_issue2625(self): + msg0 = mailbox.MHMessage(self._template % 0) + msg0.add_sequence('foo') + key0 = self._box.add(msg0) + refmsg0 = self._box.get_message(key0) + + def test_issue7627(self): + msg0 = mailbox.MHMessage(self._template % 0) + key0 = self._box.add(msg0) + self._box.lock() + self._box.remove(key0) + self._box.unlock() + + def test_pack(self): + # Pack the contents of the mailbox + msg0 = mailbox.MHMessage(self._template % 0) + msg1 = mailbox.MHMessage(self._template % 1) + msg2 = mailbox.MHMessage(self._template % 2) + msg3 = mailbox.MHMessage(self._template % 3) + msg0.set_sequences(['foo', 'unseen']) + msg1.set_sequences(['foo']) + msg2.set_sequences(['foo', 'flagged']) + msg3.set_sequences(['foo', 'bar', 'replied']) + key0 = self._box.add(msg0) + key1 = self._box.add(msg1) + key2 = self._box.add(msg2) + key3 = self._box.add(msg3) + self.assertEqual(self._box.get_sequences(), + {'foo':[key0,key1,key2,key3], 'unseen':[key0], + 'flagged':[key2], 'bar':[key3], 'replied':[key3]}) + self._box.remove(key2) + self.assertEqual(self._box.get_sequences(), + {'foo':[key0,key1,key3], 'unseen':[key0], 'bar':[key3], + 'replied':[key3]}) + self._box.pack() + self.assertEqual(self._box.keys(), [1, 2, 3]) + key0 = key0 + key1 = key0 + 1 + key2 = key1 + 1 + self.assertEqual(self._box.get_sequences(), + {'foo':[1, 2, 3], 'unseen':[1], 'bar':[3], 'replied':[3]}) + + # Test case for packing while holding the mailbox locked. + key0 = self._box.add(msg1) + key1 = self._box.add(msg1) + key2 = self._box.add(msg1) + key3 = self._box.add(msg1) + + self._box.remove(key0) + self._box.remove(key2) + self._box.lock() + self._box.pack() + self._box.unlock() + self.assertEqual(self._box.get_sequences(), + {'foo':[1, 2, 3, 4, 5], + 'unseen':[1], 'bar':[3], 'replied':[3]}) + + def _get_lock_path(self): + return os.path.join(self._path, '.mh_sequences.lock') + + +class TestBabyl(_TestSingleFile, unittest.TestCase): + + _factory = lambda self, path, factory=None: mailbox.Babyl(path, factory) + + def assertMailboxEmpty(self): + with open(self._path, 'rb') as f: + self.assertEqual(f.readlines(), []) + + def tearDown(self): + super().tearDown() + self._box.close() + self._delete_recursively(self._path) + for lock_remnant in glob.glob(glob.escape(self._path) + '.*'): + os_helper.unlink(lock_remnant) + + def test_labels(self): + # Get labels from the mailbox + self.assertEqual(self._box.get_labels(), []) + msg0 = mailbox.BabylMessage(self._template % 0) + msg0.add_label('foo') + key0 = self._box.add(msg0) + self.assertEqual(self._box.get_labels(), ['foo']) + msg1 = mailbox.BabylMessage(self._template % 1) + msg1.set_labels(['bar', 'answered', 'foo']) + key1 = self._box.add(msg1) + self.assertEqual(set(self._box.get_labels()), set(['foo', 'bar'])) + msg0.set_labels(['blah', 'filed']) + self._box[key0] = msg0 + self.assertEqual(set(self._box.get_labels()), + set(['foo', 'bar', 'blah'])) + self._box.remove(key1) + self.assertEqual(set(self._box.get_labels()), set(['blah'])) + + +class FakeFileLikeObject: + + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + +class FakeMailBox(mailbox.Mailbox): + + def __init__(self): + mailbox.Mailbox.__init__(self, '', lambda file: None) + self.files = [FakeFileLikeObject() for i in range(10)] + + def get_file(self, key): + return self.files[key] + + +class TestFakeMailBox(unittest.TestCase): + + def test_closing_fd(self): + box = FakeMailBox() + for i in range(10): + self.assertFalse(box.files[i].closed) + for i in range(10): + box[i] + for i in range(10): + self.assertTrue(box.files[i].closed) + + +class TestMessage(TestBase, unittest.TestCase): + + _factory = mailbox.Message # Overridden by subclasses to reuse tests + + def setUp(self): + self._path = os_helper.TESTFN + + def tearDown(self): + self._delete_recursively(self._path) + + def test_initialize_with_eMM(self): + # Initialize based on email.message.Message instance + eMM = email.message_from_string(_sample_message) + msg = self._factory(eMM) + self._post_initialize_hook(msg) + self._check_sample(msg) + + def test_initialize_with_string(self): + # Initialize based on string + msg = self._factory(_sample_message) + self._post_initialize_hook(msg) + self._check_sample(msg) + + def test_initialize_with_file(self): + # Initialize based on contents of file + with open(self._path, 'w+', encoding='utf-8') as f: + f.write(_sample_message) + f.seek(0) + msg = self._factory(f) + self._post_initialize_hook(msg) + self._check_sample(msg) + + def test_initialize_with_binary_file(self): + # Initialize based on contents of binary file + with open(self._path, 'wb+') as f: + f.write(_bytes_sample_message) + f.seek(0) + msg = self._factory(f) + self._post_initialize_hook(msg) + self._check_sample(msg) + + def test_initialize_with_nothing(self): + # Initialize without arguments + msg = self._factory() + self._post_initialize_hook(msg) + self.assertIsInstance(msg, email.message.Message) + self.assertIsInstance(msg, mailbox.Message) + self.assertIsInstance(msg, self._factory) + self.assertEqual(msg.keys(), []) + self.assertFalse(msg.is_multipart()) + self.assertIsNone(msg.get_payload()) + + def test_initialize_incorrectly(self): + # Initialize with invalid argument + self.assertRaises(TypeError, lambda: self._factory(object())) + + def test_all_eMM_attributes_exist(self): + # Issue 12537 + eMM = email.message_from_string(_sample_message) + msg = self._factory(_sample_message) + for attr in eMM.__dict__: + self.assertIn(attr, msg.__dict__, + '{} attribute does not exist'.format(attr)) + + def test_become_message(self): + # Take on the state of another message + eMM = email.message_from_string(_sample_message) + msg = self._factory() + msg._become_message(eMM) + self._check_sample(msg) + + def test_explain_to(self): + # Copy self's format-specific data to other message formats. + # This test is superficial; better ones are in TestMessageConversion. + msg = self._factory() + for class_ in self.all_mailbox_types: + other_msg = class_() + msg._explain_to(other_msg) + other_msg = email.message.Message() + self.assertRaises(TypeError, lambda: msg._explain_to(other_msg)) + + def _post_initialize_hook(self, msg): + # Overridden by subclasses to check extra things after initialization + pass + + +class TestMaildirMessage(TestMessage, unittest.TestCase): + + _factory = mailbox.MaildirMessage + + def _post_initialize_hook(self, msg): + self.assertEqual(msg._subdir, 'new') + self.assertEqual(msg._info, '') + + def test_subdir(self): + # Use get_subdir() and set_subdir() + msg = mailbox.MaildirMessage(_sample_message) + self.assertEqual(msg.get_subdir(), 'new') + msg.set_subdir('cur') + self.assertEqual(msg.get_subdir(), 'cur') + msg.set_subdir('new') + self.assertEqual(msg.get_subdir(), 'new') + self.assertRaises(ValueError, lambda: msg.set_subdir('tmp')) + self.assertEqual(msg.get_subdir(), 'new') + msg.set_subdir('new') + self.assertEqual(msg.get_subdir(), 'new') + self._check_sample(msg) + + def test_flags(self): + # Use get_flags(), set_flags(), add_flag(), remove_flag() + msg = mailbox.MaildirMessage(_sample_message) + self.assertEqual(msg.get_flags(), '') + self.assertEqual(msg.get_subdir(), 'new') + msg.set_flags('F') + self.assertEqual(msg.get_subdir(), 'new') + self.assertEqual(msg.get_flags(), 'F') + msg.set_flags('SDTP') + self.assertEqual(msg.get_flags(), 'DPST') + msg.add_flag('FT') + self.assertEqual(msg.get_flags(), 'DFPST') + msg.remove_flag('TDRP') + self.assertEqual(msg.get_flags(), 'FS') + self.assertEqual(msg.get_subdir(), 'new') + self._check_sample(msg) + + def test_date(self): + # Use get_date() and set_date() + msg = mailbox.MaildirMessage(_sample_message) + self.assertLess(abs(msg.get_date() - time.time()), 60) + msg.set_date(0.0) + self.assertEqual(msg.get_date(), 0.0) + + def test_info(self): + # Use get_info() and set_info() + msg = mailbox.MaildirMessage(_sample_message) + self.assertEqual(msg.get_info(), '') + msg.set_info('1,foo=bar') + self.assertEqual(msg.get_info(), '1,foo=bar') + self.assertRaises(TypeError, lambda: msg.set_info(None)) + self._check_sample(msg) + + def test_info_and_flags(self): + # Test interaction of info and flag methods + msg = mailbox.MaildirMessage(_sample_message) + self.assertEqual(msg.get_info(), '') + msg.set_flags('SF') + self.assertEqual(msg.get_flags(), 'FS') + self.assertEqual(msg.get_info(), '2,FS') + msg.set_info('1,') + self.assertEqual(msg.get_flags(), '') + self.assertEqual(msg.get_info(), '1,') + msg.remove_flag('RPT') + self.assertEqual(msg.get_flags(), '') + self.assertEqual(msg.get_info(), '1,') + msg.add_flag('D') + self.assertEqual(msg.get_flags(), 'D') + self.assertEqual(msg.get_info(), '2,D') + self._check_sample(msg) + + +class _TestMboxMMDFMessage: + + _factory = mailbox._mboxMMDFMessage + + def _post_initialize_hook(self, msg): + self._check_from(msg) + + def test_initialize_with_unixfrom(self): + # Initialize with a message that already has a _unixfrom attribute + msg = mailbox.Message(_sample_message) + msg.set_unixfrom('From foo@bar blah') + msg = mailbox.mboxMessage(msg) + self.assertEqual(msg.get_from(), 'foo@bar blah') + self.assertEqual(msg.get_unixfrom(), 'From foo@bar blah') + + def test_from(self): + # Get and set "From " line + msg = mailbox.mboxMessage(_sample_message) + self._check_from(msg) + self.assertIsNone(msg.get_unixfrom()) + msg.set_from('foo bar') + self.assertEqual(msg.get_from(), 'foo bar') + self.assertIsNone(msg.get_unixfrom()) + msg.set_from('foo@bar', True) + self._check_from(msg, 'foo@bar') + self.assertIsNone(msg.get_unixfrom()) + msg.set_from('blah@temp', time.localtime()) + self._check_from(msg, 'blah@temp') + self.assertIsNone(msg.get_unixfrom()) + + def test_flags(self): + # Use get_flags(), set_flags(), add_flag(), remove_flag() + msg = mailbox.mboxMessage(_sample_message) + self.assertEqual(msg.get_flags(), '') + msg.set_flags('F') + self.assertEqual(msg.get_flags(), 'F') + msg.set_flags('XODR') + self.assertEqual(msg.get_flags(), 'RODX') + msg.add_flag('FA') + self.assertEqual(msg.get_flags(), 'RODFAX') + msg.remove_flag('FDXA') + self.assertEqual(msg.get_flags(), 'RO') + self._check_sample(msg) + + def _check_from(self, msg, sender=None): + # Check contents of "From " line + if sender is None: + sender = "MAILER-DAEMON" + self.assertIsNotNone(re.match( + sender + r" \w{3} \w{3} [\d ]\d [\d ]\d:\d{2}:\d{2} \d{4}", + msg.get_from())) + + +class TestMboxMessage(_TestMboxMMDFMessage, TestMessage): + + _factory = mailbox.mboxMessage + + +class TestMHMessage(TestMessage, unittest.TestCase): + + _factory = mailbox.MHMessage + + def _post_initialize_hook(self, msg): + self.assertEqual(msg._sequences, []) + + def test_sequences(self): + # Get, set, join, and leave sequences + msg = mailbox.MHMessage(_sample_message) + self.assertEqual(msg.get_sequences(), []) + msg.set_sequences(['foobar']) + self.assertEqual(msg.get_sequences(), ['foobar']) + msg.set_sequences([]) + self.assertEqual(msg.get_sequences(), []) + msg.add_sequence('unseen') + self.assertEqual(msg.get_sequences(), ['unseen']) + msg.add_sequence('flagged') + self.assertEqual(msg.get_sequences(), ['unseen', 'flagged']) + msg.add_sequence('flagged') + self.assertEqual(msg.get_sequences(), ['unseen', 'flagged']) + msg.remove_sequence('unseen') + self.assertEqual(msg.get_sequences(), ['flagged']) + msg.add_sequence('foobar') + self.assertEqual(msg.get_sequences(), ['flagged', 'foobar']) + msg.remove_sequence('replied') + self.assertEqual(msg.get_sequences(), ['flagged', 'foobar']) + msg.set_sequences(['foobar', 'replied']) + self.assertEqual(msg.get_sequences(), ['foobar', 'replied']) + + +class TestBabylMessage(TestMessage, unittest.TestCase): + + _factory = mailbox.BabylMessage + + def _post_initialize_hook(self, msg): + self.assertEqual(msg._labels, []) + + def test_labels(self): + # Get, set, join, and leave labels + msg = mailbox.BabylMessage(_sample_message) + self.assertEqual(msg.get_labels(), []) + msg.set_labels(['foobar']) + self.assertEqual(msg.get_labels(), ['foobar']) + msg.set_labels([]) + self.assertEqual(msg.get_labels(), []) + msg.add_label('filed') + self.assertEqual(msg.get_labels(), ['filed']) + msg.add_label('resent') + self.assertEqual(msg.get_labels(), ['filed', 'resent']) + msg.add_label('resent') + self.assertEqual(msg.get_labels(), ['filed', 'resent']) + msg.remove_label('filed') + self.assertEqual(msg.get_labels(), ['resent']) + msg.add_label('foobar') + self.assertEqual(msg.get_labels(), ['resent', 'foobar']) + msg.remove_label('unseen') + self.assertEqual(msg.get_labels(), ['resent', 'foobar']) + msg.set_labels(['foobar', 'answered']) + self.assertEqual(msg.get_labels(), ['foobar', 'answered']) + + def test_visible(self): + # Get, set, and update visible headers + msg = mailbox.BabylMessage(_sample_message) + visible = msg.get_visible() + self.assertEqual(visible.keys(), []) + self.assertIsNone(visible.get_payload()) + visible['User-Agent'] = 'FooBar 1.0' + visible['X-Whatever'] = 'Blah' + self.assertEqual(msg.get_visible().keys(), []) + msg.set_visible(visible) + visible = msg.get_visible() + self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever']) + self.assertEqual(visible['User-Agent'], 'FooBar 1.0') + self.assertEqual(visible['X-Whatever'], 'Blah') + self.assertIsNone(visible.get_payload()) + msg.update_visible() + self.assertEqual(visible.keys(), ['User-Agent', 'X-Whatever']) + self.assertIsNone(visible.get_payload()) + visible = msg.get_visible() + self.assertEqual(visible.keys(), ['User-Agent', 'Date', 'From', 'To', + 'Subject']) + for header in ('User-Agent', 'Date', 'From', 'To', 'Subject'): + self.assertEqual(visible[header], msg[header]) + + +class TestMMDFMessage(_TestMboxMMDFMessage, TestMessage): + + _factory = mailbox.MMDFMessage + + +class TestMessageConversion(TestBase, unittest.TestCase): + + def test_plain_to_x(self): + # Convert Message to all formats + for class_ in self.all_mailbox_types: + msg_plain = mailbox.Message(_sample_message) + msg = class_(msg_plain) + self._check_sample(msg) + + def test_x_to_plain(self): + # Convert all formats to Message + for class_ in self.all_mailbox_types: + msg = class_(_sample_message) + msg_plain = mailbox.Message(msg) + self._check_sample(msg_plain) + + def test_x_from_bytes(self): + # Convert all formats to Message + for class_ in self.all_mailbox_types: + msg = class_(_bytes_sample_message) + self._check_sample(msg) + + def test_x_to_invalid(self): + # Convert all formats to an invalid format + for class_ in self.all_mailbox_types: + self.assertRaises(TypeError, lambda: class_(False)) + + def test_type_specific_attributes_removed_on_conversion(self): + reference = {class_: class_(_sample_message).__dict__ + for class_ in self.all_mailbox_types} + for class1 in self.all_mailbox_types: + for class2 in self.all_mailbox_types: + if class1 is class2: + continue + source = class1(_sample_message) + target = class2(source) + type_specific = [a for a in reference[class1] + if a not in reference[class2]] + for attr in type_specific: + self.assertNotIn(attr, target.__dict__, + "while converting {} to {}".format(class1, class2)) + + def test_maildir_to_maildir(self): + # Convert MaildirMessage to MaildirMessage + msg_maildir = mailbox.MaildirMessage(_sample_message) + msg_maildir.set_flags('DFPRST') + msg_maildir.set_subdir('cur') + date = msg_maildir.get_date() + msg = mailbox.MaildirMessage(msg_maildir) + self._check_sample(msg) + self.assertEqual(msg.get_flags(), 'DFPRST') + self.assertEqual(msg.get_subdir(), 'cur') + self.assertEqual(msg.get_date(), date) + + def test_maildir_to_mboxmmdf(self): + # Convert MaildirMessage to mboxmessage and MMDFMessage + pairs = (('D', ''), ('F', 'F'), ('P', ''), ('R', 'A'), ('S', 'R'), + ('T', 'D'), ('DFPRST', 'RDFA')) + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg_maildir = mailbox.MaildirMessage(_sample_message) + msg_maildir.set_date(0.0) + for setting, result in pairs: + msg_maildir.set_flags(setting) + msg = class_(msg_maildir) + self.assertEqual(msg.get_flags(), result) + self.assertEqual(msg.get_from(), 'MAILER-DAEMON %s' % + time.asctime(time.gmtime(0.0))) + self.assertIsNone(msg.get_unixfrom()) + msg_maildir.set_subdir('cur') + self.assertEqual(class_(msg_maildir).get_flags(), 'RODFA') + + def test_maildir_to_mh(self): + # Convert MaildirMessage to MHMessage + msg_maildir = mailbox.MaildirMessage(_sample_message) + pairs = (('D', ['unseen']), ('F', ['unseen', 'flagged']), + ('P', ['unseen']), ('R', ['unseen', 'replied']), ('S', []), + ('T', ['unseen']), ('DFPRST', ['replied', 'flagged'])) + for setting, result in pairs: + msg_maildir.set_flags(setting) + self.assertEqual(mailbox.MHMessage(msg_maildir).get_sequences(), + result) + + def test_maildir_to_babyl(self): + # Convert MaildirMessage to Babyl + msg_maildir = mailbox.MaildirMessage(_sample_message) + pairs = (('D', ['unseen']), ('F', ['unseen']), + ('P', ['unseen', 'forwarded']), ('R', ['unseen', 'answered']), + ('S', []), ('T', ['unseen', 'deleted']), + ('DFPRST', ['deleted', 'answered', 'forwarded'])) + for setting, result in pairs: + msg_maildir.set_flags(setting) + self.assertEqual(mailbox.BabylMessage(msg_maildir).get_labels(), + result) + + def test_mboxmmdf_to_maildir(self): + # Convert mboxMessage and MMDFMessage to MaildirMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg_mboxMMDF = class_(_sample_message) + msg_mboxMMDF.set_from('foo@bar', time.gmtime(0.0)) + pairs = (('R', 'S'), ('O', ''), ('D', 'T'), ('F', 'F'), ('A', 'R'), + ('RODFA', 'FRST')) + for setting, result in pairs: + msg_mboxMMDF.set_flags(setting) + msg = mailbox.MaildirMessage(msg_mboxMMDF) + self.assertEqual(msg.get_flags(), result) + self.assertEqual(msg.get_date(), 0.0) + msg_mboxMMDF.set_flags('O') + self.assertEqual(mailbox.MaildirMessage(msg_mboxMMDF).get_subdir(), + 'cur') + + def test_mboxmmdf_to_mboxmmdf(self): + # Convert mboxMessage and MMDFMessage to mboxMessage and MMDFMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg_mboxMMDF = class_(_sample_message) + msg_mboxMMDF.set_flags('RODFA') + msg_mboxMMDF.set_from('foo@bar') + self.assertIsNone(msg_mboxMMDF.get_unixfrom()) + for class2_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg2 = class2_(msg_mboxMMDF) + self.assertEqual(msg2.get_flags(), 'RODFA') + self.assertEqual(msg2.get_from(), 'foo@bar') + self.assertIsNone(msg2.get_unixfrom()) + + def test_mboxmmdf_to_mh(self): + # Convert mboxMessage and MMDFMessage to MHMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg_mboxMMDF = class_(_sample_message) + pairs = (('R', []), ('O', ['unseen']), ('D', ['unseen']), + ('F', ['unseen', 'flagged']), + ('A', ['unseen', 'replied']), + ('RODFA', ['replied', 'flagged'])) + for setting, result in pairs: + msg_mboxMMDF.set_flags(setting) + self.assertEqual(mailbox.MHMessage(msg_mboxMMDF).get_sequences(), + result) + + def test_mboxmmdf_to_babyl(self): + # Convert mboxMessage and MMDFMessage to BabylMessage + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg = class_(_sample_message) + pairs = (('R', []), ('O', ['unseen']), + ('D', ['unseen', 'deleted']), ('F', ['unseen']), + ('A', ['unseen', 'answered']), + ('RODFA', ['deleted', 'answered'])) + for setting, result in pairs: + msg.set_flags(setting) + self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result) + + def test_mh_to_maildir(self): + # Convert MHMessage to MaildirMessage + pairs = (('unseen', ''), ('replied', 'RS'), ('flagged', 'FS')) + for setting, result in pairs: + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence(setting) + self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result) + self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur') + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence('unseen') + msg.add_sequence('replied') + msg.add_sequence('flagged') + self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'FR') + self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur') + + def test_mh_to_mboxmmdf(self): + # Convert MHMessage to mboxMessage and MMDFMessage + pairs = (('unseen', 'O'), ('replied', 'ROA'), ('flagged', 'ROF')) + for setting, result in pairs: + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence(setting) + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + self.assertEqual(class_(msg).get_flags(), result) + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence('unseen') + msg.add_sequence('replied') + msg.add_sequence('flagged') + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + self.assertEqual(class_(msg).get_flags(), 'OFA') + + def test_mh_to_mh(self): + # Convert MHMessage to MHMessage + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence('unseen') + msg.add_sequence('replied') + msg.add_sequence('flagged') + self.assertEqual(mailbox.MHMessage(msg).get_sequences(), + ['unseen', 'replied', 'flagged']) + + def test_mh_to_babyl(self): + # Convert MHMessage to BabylMessage + pairs = (('unseen', ['unseen']), ('replied', ['answered']), + ('flagged', [])) + for setting, result in pairs: + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence(setting) + self.assertEqual(mailbox.BabylMessage(msg).get_labels(), result) + msg = mailbox.MHMessage(_sample_message) + msg.add_sequence('unseen') + msg.add_sequence('replied') + msg.add_sequence('flagged') + self.assertEqual(mailbox.BabylMessage(msg).get_labels(), + ['unseen', 'answered']) + + def test_babyl_to_maildir(self): + # Convert BabylMessage to MaildirMessage + pairs = (('unseen', ''), ('deleted', 'ST'), ('filed', 'S'), + ('answered', 'RS'), ('forwarded', 'PS'), ('edited', 'S'), + ('resent', 'PS')) + for setting, result in pairs: + msg = mailbox.BabylMessage(_sample_message) + msg.add_label(setting) + self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), result) + self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur') + msg = mailbox.BabylMessage(_sample_message) + for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded', + 'edited', 'resent'): + msg.add_label(label) + self.assertEqual(mailbox.MaildirMessage(msg).get_flags(), 'PRT') + self.assertEqual(mailbox.MaildirMessage(msg).get_subdir(), 'cur') + + def test_babyl_to_mboxmmdf(self): + # Convert BabylMessage to mboxMessage and MMDFMessage + pairs = (('unseen', 'O'), ('deleted', 'ROD'), ('filed', 'RO'), + ('answered', 'ROA'), ('forwarded', 'RO'), ('edited', 'RO'), + ('resent', 'RO')) + for setting, result in pairs: + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + msg = mailbox.BabylMessage(_sample_message) + msg.add_label(setting) + self.assertEqual(class_(msg).get_flags(), result) + msg = mailbox.BabylMessage(_sample_message) + for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded', + 'edited', 'resent'): + msg.add_label(label) + for class_ in (mailbox.mboxMessage, mailbox.MMDFMessage): + self.assertEqual(class_(msg).get_flags(), 'ODA') + + def test_babyl_to_mh(self): + # Convert BabylMessage to MHMessage + pairs = (('unseen', ['unseen']), ('deleted', []), ('filed', []), + ('answered', ['replied']), ('forwarded', []), ('edited', []), + ('resent', [])) + for setting, result in pairs: + msg = mailbox.BabylMessage(_sample_message) + msg.add_label(setting) + self.assertEqual(mailbox.MHMessage(msg).get_sequences(), result) + msg = mailbox.BabylMessage(_sample_message) + for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded', + 'edited', 'resent'): + msg.add_label(label) + self.assertEqual(mailbox.MHMessage(msg).get_sequences(), + ['unseen', 'replied']) + + def test_babyl_to_babyl(self): + # Convert BabylMessage to BabylMessage + msg = mailbox.BabylMessage(_sample_message) + msg.update_visible() + for label in ('unseen', 'deleted', 'filed', 'answered', 'forwarded', + 'edited', 'resent'): + msg.add_label(label) + msg2 = mailbox.BabylMessage(msg) + self.assertEqual(msg2.get_labels(), ['unseen', 'deleted', 'filed', + 'answered', 'forwarded', 'edited', + 'resent']) + self.assertEqual(msg.get_visible().keys(), msg2.get_visible().keys()) + for key in msg.get_visible().keys(): + self.assertEqual(msg.get_visible()[key], msg2.get_visible()[key]) + + +class TestProxyFileBase(TestBase): + + def _test_read(self, proxy): + # Read by byte + proxy.seek(0) + self.assertEqual(proxy.read(), b'bar') + proxy.seek(1) + self.assertEqual(proxy.read(), b'ar') + proxy.seek(0) + self.assertEqual(proxy.read(2), b'ba') + proxy.seek(1) + self.assertEqual(proxy.read(-1), b'ar') + proxy.seek(2) + self.assertEqual(proxy.read(1000), b'r') + + def _test_readline(self, proxy): + # Read by line + linesep = os.linesep.encode() + proxy.seek(0) + self.assertEqual(proxy.readline(), b'foo' + linesep) + self.assertEqual(proxy.readline(), b'bar' + linesep) + self.assertEqual(proxy.readline(), b'fred' + linesep) + self.assertEqual(proxy.readline(), b'bob') + proxy.seek(2) + self.assertEqual(proxy.readline(), b'o' + linesep) + proxy.seek(6 + 2 * len(os.linesep)) + self.assertEqual(proxy.readline(), b'fred' + linesep) + proxy.seek(6 + 2 * len(os.linesep)) + self.assertEqual(proxy.readline(2), b'fr') + self.assertEqual(proxy.readline(-10), b'ed' + linesep) + + def _test_readlines(self, proxy): + # Read multiple lines + linesep = os.linesep.encode() + proxy.seek(0) + self.assertEqual(proxy.readlines(), [b'foo' + linesep, + b'bar' + linesep, + b'fred' + linesep, b'bob']) + proxy.seek(0) + self.assertEqual(proxy.readlines(2), [b'foo' + linesep]) + proxy.seek(3 + len(linesep)) + self.assertEqual(proxy.readlines(4 + len(linesep)), + [b'bar' + linesep, b'fred' + linesep]) + proxy.seek(3) + self.assertEqual(proxy.readlines(1000), [linesep, b'bar' + linesep, + b'fred' + linesep, b'bob']) + + def _test_iteration(self, proxy): + # Iterate by line + linesep = os.linesep.encode() + proxy.seek(0) + iterator = iter(proxy) + self.assertEqual(next(iterator), b'foo' + linesep) + self.assertEqual(next(iterator), b'bar' + linesep) + self.assertEqual(next(iterator), b'fred' + linesep) + self.assertEqual(next(iterator), b'bob') + self.assertRaises(StopIteration, next, iterator) + + def _test_seek_and_tell(self, proxy): + # Seek and use tell to check position + linesep = os.linesep.encode() + proxy.seek(3) + self.assertEqual(proxy.tell(), 3) + self.assertEqual(proxy.read(len(linesep)), linesep) + proxy.seek(2, 1) + self.assertEqual(proxy.read(1 + len(linesep)), b'r' + linesep) + proxy.seek(-3 - len(linesep), 2) + self.assertEqual(proxy.read(3), b'bar') + proxy.seek(2, 0) + self.assertEqual(proxy.read(), b'o' + linesep + b'bar' + linesep) + proxy.seek(100) + self.assertFalse(proxy.read()) + + def _test_close(self, proxy): + # Close a file + self.assertFalse(proxy.closed) + proxy.close() + self.assertTrue(proxy.closed) + # Issue 11700 subsequent closes should be a no-op. + proxy.close() + self.assertTrue(proxy.closed) + + +class TestProxyFile(TestProxyFileBase, unittest.TestCase): + + def setUp(self): + self._path = os_helper.TESTFN + self._file = open(self._path, 'wb+') + + def tearDown(self): + self._file.close() + self._delete_recursively(self._path) + + def test_initialize(self): + # Initialize and check position + self._file.write(b'foo') + pos = self._file.tell() + proxy0 = mailbox._ProxyFile(self._file) + self.assertEqual(proxy0.tell(), pos) + self.assertEqual(self._file.tell(), pos) + proxy1 = mailbox._ProxyFile(self._file, 0) + self.assertEqual(proxy1.tell(), 0) + self.assertEqual(self._file.tell(), pos) + + def test_read(self): + self._file.write(b'bar') + self._test_read(mailbox._ProxyFile(self._file)) + + def test_readline(self): + self._file.write(bytes('foo%sbar%sfred%sbob' % (os.linesep, os.linesep, + os.linesep), 'ascii')) + self._test_readline(mailbox._ProxyFile(self._file)) + + def test_readlines(self): + self._file.write(bytes('foo%sbar%sfred%sbob' % (os.linesep, os.linesep, + os.linesep), 'ascii')) + self._test_readlines(mailbox._ProxyFile(self._file)) + + def test_iteration(self): + self._file.write(bytes('foo%sbar%sfred%sbob' % (os.linesep, os.linesep, + os.linesep), 'ascii')) + self._test_iteration(mailbox._ProxyFile(self._file)) + + def test_seek_and_tell(self): + self._file.write(bytes('foo%sbar%s' % (os.linesep, os.linesep), 'ascii')) + self._test_seek_and_tell(mailbox._ProxyFile(self._file)) + + def test_close(self): + self._file.write(bytes('foo%sbar%s' % (os.linesep, os.linesep), 'ascii')) + self._test_close(mailbox._ProxyFile(self._file)) + + +class TestPartialFile(TestProxyFileBase, unittest.TestCase): + + def setUp(self): + self._path = os_helper.TESTFN + self._file = open(self._path, 'wb+') + + def tearDown(self): + self._file.close() + self._delete_recursively(self._path) + + def test_initialize(self): + # Initialize and check position + self._file.write(bytes('foo' + os.linesep + 'bar', 'ascii')) + pos = self._file.tell() + proxy = mailbox._PartialFile(self._file, 2, 5) + self.assertEqual(proxy.tell(), 0) + self.assertEqual(self._file.tell(), pos) + + def test_read(self): + self._file.write(bytes('***bar***', 'ascii')) + self._test_read(mailbox._PartialFile(self._file, 3, 6)) + + def test_readline(self): + self._file.write(bytes('!!!!!foo%sbar%sfred%sbob!!!!!' % + (os.linesep, os.linesep, os.linesep), 'ascii')) + self._test_readline(mailbox._PartialFile(self._file, 5, + 18 + 3 * len(os.linesep))) + + def test_readlines(self): + self._file.write(bytes('foo%sbar%sfred%sbob?????' % + (os.linesep, os.linesep, os.linesep), 'ascii')) + self._test_readlines(mailbox._PartialFile(self._file, 0, + 13 + 3 * len(os.linesep))) + + def test_iteration(self): + self._file.write(bytes('____foo%sbar%sfred%sbob####' % + (os.linesep, os.linesep, os.linesep), 'ascii')) + self._test_iteration(mailbox._PartialFile(self._file, 4, + 17 + 3 * len(os.linesep))) + + def test_seek_and_tell(self): + self._file.write(bytes('(((foo%sbar%s$$$' % (os.linesep, os.linesep), 'ascii')) + self._test_seek_and_tell(mailbox._PartialFile(self._file, 3, + 9 + 2 * len(os.linesep))) + + def test_close(self): + self._file.write(bytes('&foo%sbar%s^' % (os.linesep, os.linesep), 'ascii')) + self._test_close(mailbox._PartialFile(self._file, 1, + 6 + 3 * len(os.linesep))) + + +## Start: tests from the original module (for backward compatibility). + +FROM_ = "From some.body@dummy.domain Sat Jul 24 13:43:35 2004\n" +DUMMY_MESSAGE = """\ +From: some.body@dummy.domain +To: me@my.domain +Subject: Simple Test + +This is a dummy message. +""" + +class MaildirTestCase(unittest.TestCase): + + def setUp(self): + # create a new maildir mailbox to work with: + self._dir = os_helper.TESTFN + if os.path.isdir(self._dir): + os_helper.rmtree(self._dir) + elif os.path.isfile(self._dir): + os_helper.unlink(self._dir) + os.mkdir(self._dir) + os.mkdir(os.path.join(self._dir, "cur")) + os.mkdir(os.path.join(self._dir, "tmp")) + os.mkdir(os.path.join(self._dir, "new")) + self._counter = 1 + self._msgfiles = [] + + def tearDown(self): + list(map(os.unlink, self._msgfiles)) + os_helper.rmdir(os.path.join(self._dir, "cur")) + os_helper.rmdir(os.path.join(self._dir, "tmp")) + os_helper.rmdir(os.path.join(self._dir, "new")) + os_helper.rmdir(self._dir) + + def createMessage(self, dir, mbox=False): + t = int(time.time() % 1000000) + pid = self._counter + self._counter += 1 + filename = ".".join((str(t), str(pid), "myhostname", "mydomain")) + tmpname = os.path.join(self._dir, "tmp", filename) + newname = os.path.join(self._dir, dir, filename) + with open(tmpname, "w", encoding="utf-8") as fp: + self._msgfiles.append(tmpname) + if mbox: + fp.write(FROM_) + fp.write(DUMMY_MESSAGE) + try: + os.link(tmpname, newname) + except (AttributeError, PermissionError): + with open(newname, "w") as fp: + fp.write(DUMMY_MESSAGE) + self._msgfiles.append(newname) + return tmpname + + def test_empty_maildir(self): + """Test an empty maildir mailbox""" + # Test for regression on bug #117490: + # Make sure the boxes attribute actually gets set. + self.mbox = mailbox.Maildir(os_helper.TESTFN) + #self.assertHasAttr(self.mbox, "boxes") + #self.assertEqual(len(self.mbox.boxes), 0) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + + def test_nonempty_maildir_cur(self): + self.createMessage("cur") + self.mbox = mailbox.Maildir(os_helper.TESTFN) + #self.assertEqual(len(self.mbox.boxes), 1) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + + def test_nonempty_maildir_new(self): + self.createMessage("new") + self.mbox = mailbox.Maildir(os_helper.TESTFN) + #self.assertEqual(len(self.mbox.boxes), 1) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + + def test_nonempty_maildir_both(self): + self.createMessage("cur") + self.createMessage("new") + self.mbox = mailbox.Maildir(os_helper.TESTFN) + #self.assertEqual(len(self.mbox.boxes), 2) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNotNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + self.assertIsNone(self.mbox.next()) + +## End: tests from the original module (for backward compatibility). + + +_sample_message = """\ +Return-Path: <gkj@gregorykjohnson.com> +X-Original-To: gkj+person@localhost +Delivered-To: gkj+person@localhost +Received: from localhost (localhost [127.0.0.1]) + by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17 + for <gkj+person@localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT) +Delivered-To: gkj@sundance.gregorykjohnson.com +Received: from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.5) + for gkj+person@localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT) +Received: from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228]) + by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746 + for <gkj@gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT) +Received: by andy.gregorykjohnson.com (Postfix, from userid 1000) + id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT) +Date: Wed, 13 Jul 2005 17:23:11 -0400 +From: "Gregory K. Johnson" <gkj@gregorykjohnson.com> +To: gkj@gregorykjohnson.com +Subject: Sample message +Message-ID: <20050713212311.GC4701@andy.gregorykjohnson.com> +Mime-Version: 1.0 +Content-Type: multipart/mixed; boundary="NMuMz9nt05w80d4+" +Content-Disposition: inline +User-Agent: Mutt/1.5.9i + + +--NMuMz9nt05w80d4+ +Content-Type: text/plain; charset=us-ascii +Content-Disposition: inline + +This is a sample message. + +-- +Gregory K. Johnson + +--NMuMz9nt05w80d4+ +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="text.gz" +Content-Transfer-Encoding: base64 + +H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs +3FYlAAAA + +--NMuMz9nt05w80d4+-- +""" + +_bytes_sample_message = _sample_message.encode('ascii') + +_sample_headers = [ + ("Return-Path", "<gkj@gregorykjohnson.com>"), + ("X-Original-To", "gkj+person@localhost"), + ("Delivered-To", "gkj+person@localhost"), + ("Received", """from localhost (localhost [127.0.0.1]) + by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17 + for <gkj+person@localhost>; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)"""), + ("Delivered-To", "gkj@sundance.gregorykjohnson.com"), + ("Received", """from localhost [127.0.0.1] + by localhost with POP3 (fetchmail-6.2.5) + for gkj+person@localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)"""), + ("Received", """from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228]) + by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746 + for <gkj@gregorykjohnson.com>; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)"""), + ("Received", """by andy.gregorykjohnson.com (Postfix, from userid 1000) + id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)"""), + ("Date", "Wed, 13 Jul 2005 17:23:11 -0400"), + ("From", """"Gregory K. Johnson" <gkj@gregorykjohnson.com>"""), + ("To", "gkj@gregorykjohnson.com"), + ("Subject", "Sample message"), + ("Mime-Version", "1.0"), + ("Content-Type", """multipart/mixed; boundary="NMuMz9nt05w80d4+\""""), + ("Content-Disposition", "inline"), + ("User-Agent", "Mutt/1.5.9i"), +] + +_sample_payloads = ("""This is a sample message. + +-- +Gregory K. Johnson +""", +"""H4sICM2D1UIAA3RleHQAC8nILFYAokSFktSKEoW0zJxUPa7wzJIMhZLyfIWczLzUYj0uAHTs +3FYlAAAA +""") + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, mailbox, + not_exported={"linesep", "fcntl"}) + + +def tearDownModule(): + support.reap_children() + # reap_children may have re-populated caches: + if refleak_helper.hunting_for_refleaks(): + sys._clear_internal_caches() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 2161e06b2f2..7afcb5e6f50 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -1,5 +1,6 @@ from test import support -from test.support import os_helper +from test.support import is_apple_mobile, os_helper, requires_debug_ranges, is_emscripten +from test.support.script_helper import assert_python_ok import array import io import marshal @@ -7,6 +8,7 @@ import unittest import os import types +import textwrap try: import _testcapi @@ -26,6 +28,13 @@ def helper(self, sample, *extra): finally: os_helper.unlink(os_helper.TESTFN) +def omit_last_byte(data): + """return data[:-1]""" + # This file's code is used in CompatibilityTestCase, + # but slices need marshal version 5. + # Avoid the slice literal. + return data[slice(0, -1)] + class IntTestCase(unittest.TestCase, HelperMixin): def test_ints(self): # Test a range of Python ints larger than the machine word size. @@ -34,8 +43,13 @@ def test_ints(self): for expected in (-n, n): self.helper(expected) n = n >> 1 + n = 1 << 100 + while n: + for expected in (-n, -n+1, n-1, n): + self.helper(expected) + n = n >> 1 - @unittest.skip("TODO: RUSTPYTHON; hang") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_int64(self): # Simulate int marshaling with TYPE_INT64. maxint64 = (1 << 63) - 1 @@ -109,8 +123,7 @@ def test_exceptions(self): self.assertEqual(StopIteration, new) class CodeTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_code(self): co = ExceptionTestCase.test_exceptions.__code__ new = marshal.loads(marshal.dumps(co)) @@ -118,8 +131,8 @@ def test_code(self): def test_many_codeobjects(self): # Issue2957: bad recursion count on code objects - count = 5000 # more than MAX_MARSHAL_STACK_DEPTH - codes = (ExceptionTestCase.test_exceptions.__code__,) * count + # more than MAX_MARSHAL_STACK_DEPTH + codes = (ExceptionTestCase.test_exceptions.__code__,) * 10_000 marshal.loads(marshal.dumps(codes)) def test_different_filenames(self): @@ -129,6 +142,57 @@ def test_different_filenames(self): self.assertEqual(co1.co_filename, "f1") self.assertEqual(co2.co_filename, "f2") + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument allow_code + def test_no_allow_code(self): + data = {'a': [({0},)]} + dump = marshal.dumps(data, allow_code=False) + self.assertEqual(marshal.loads(dump, allow_code=False), data) + + f = io.BytesIO() + marshal.dump(data, f, allow_code=False) + f.seek(0) + self.assertEqual(marshal.load(f, allow_code=False), data) + + co = ExceptionTestCase.test_exceptions.__code__ + data = {'a': [({co, 0},)]} + dump = marshal.dumps(data, allow_code=True) + self.assertEqual(marshal.loads(dump, allow_code=True), data) + with self.assertRaises(ValueError): + marshal.dumps(data, allow_code=False) + with self.assertRaises(ValueError): + marshal.loads(dump, allow_code=False) + + marshal.dump(data, io.BytesIO(), allow_code=True) + self.assertEqual(marshal.load(io.BytesIO(dump), allow_code=True), data) + with self.assertRaises(ValueError): + marshal.dump(data, io.BytesIO(), allow_code=False) + with self.assertRaises(ValueError): + marshal.load(io.BytesIO(dump), allow_code=False) + + @requires_debug_ranges() + def test_minimal_linetable_with_no_debug_ranges(self): + # Make sure when demarshalling objects with `-X no_debug_ranges` + # that the columns are None. + co = ExceptionTestCase.test_exceptions.__code__ + code = textwrap.dedent(""" + import sys + import marshal + with open(sys.argv[1], 'rb') as f: + co = marshal.load(f) + positions = list(co.co_positions()) + assert positions[0][2] is None + assert positions[0][3] is None + """) + + try: + with open(os_helper.TESTFN, 'wb') as f: + marshal.dump(co, f) + + assert_python_ok('-X', 'no_debug_ranges', + '-c', code, os_helper.TESTFN) + finally: + os_helper.unlink(os_helper.TESTFN) + @support.cpython_only def test_same_filename_used(self): s = """def f(): pass\ndef g(): pass""" @@ -164,22 +228,21 @@ def test_sets(self): class BufferTestCase(unittest.TestCase, HelperMixin): + def test_bytearray(self): b = bytearray(b"abc") self.helper(b) new = marshal.loads(marshal.dumps(b)) self.assertEqual(type(new), bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_memoryview(self): b = memoryview(b"abc") self.helper(b) new = marshal.loads(marshal.dumps(b)) self.assertEqual(type(new), bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_array(self): a = array.array('B', b"abc") new = marshal.loads(marshal.dumps(a)) @@ -194,7 +257,8 @@ def test_bug_5888452(self): def test_patch_873224(self): self.assertRaises(Exception, marshal.loads, b'0') self.assertRaises(Exception, marshal.loads, b'f') - self.assertRaises(Exception, marshal.loads, marshal.dumps(2**65)[:-1]) + self.assertRaises(Exception, marshal.loads, + omit_last_byte(marshal.dumps(2**65))) def test_version_argument(self): # Python 2.4.0 crashes for any call to marshal.dumps(x, y) @@ -211,8 +275,7 @@ def test_fuzz(self): except Exception: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_loads_recursion(self): def run_tests(N, check): # (((...None...),),) @@ -232,16 +295,19 @@ def check(s): self.assertRaises(ValueError, marshal.loads, s) run_tests(2**20, check) - @unittest.skip("TODO: RUSTPYTHON; segfault") + @unittest.skipIf(support.is_android, "TODO: RUSTPYTHON; segfault") + @unittest.expectedFailure # TODO: RUSTPYTHON; segfault def test_recursion_limit(self): # Create a deeply nested structure. head = last = [] # The max stack depth should match the value in Python/marshal.c. # BUG: https://bugs.python.org/issue33720 # Windows always limits the maximum depth on release and debug builds - #if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'): + #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 + elif sys.platform == 'wasi' or is_emscripten or is_apple_mobile: + MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 for i in range(MAX_MARSHAL_STACK_DEPTH - 2): @@ -259,8 +325,7 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_exact_type_match(self): # Former bug: # >>> class Int(int): pass @@ -284,8 +349,7 @@ def test_invalid_longs(self): invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00' self.assertRaises(ValueError, marshal.loads, invalid_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiple_dumps_and_loads(self): # Issue 12291: marshal.load() should be callable multiple times # with interleaved data written by non-marshal code @@ -315,8 +379,7 @@ def test_loads_reject_unicode_strings(self): unicode_string = 'T' self.assertRaises(TypeError, marshal.loads, unicode_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_reader(self): class BadReader(io.BytesIO): def readinto(self, buf): @@ -333,6 +396,56 @@ def test_eof(self): for i in range(len(data)): self.assertRaises(EOFError, marshal.loads, data[0: i]) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_deterministic_sets(self): + # bpo-37596: To support reproducible builds, sets and frozensets need to + # have their elements serialized in a consistent order (even when they + # have been scrambled by hash randomization): + for kind in ("set", "frozenset"): + for elements in ( + "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'", + # Also test for bad interactions with backreferencing: + "('Spam', 0), ('Spam', 1), ('Spam', 2), ('Spam', 3), ('Spam', 4), ('Spam', 5)", + ): + s = f"{kind}([{elements}])" + with self.subTest(s): + # First, make sure that our test case still has different + # orders under hash seeds 0 and 1. If this check fails, we + # need to update this test with different elements. Skip + # this part if we are configured to use any other hash + # algorithm (for example, using Py_HASH_EXTERNAL): + if sys.hash_info.algorithm in {"fnv", "siphash24"}: + args = ["-c", f"print({s})"] + _, repr_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0") + _, repr_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1") + self.assertNotEqual(repr_0, repr_1) + # Then, perform the actual test: + args = ["-c", f"import marshal; print(marshal.dumps({s}))"] + _, dump_0, _ = assert_python_ok(*args, PYTHONHASHSEED="0") + _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1") + self.assertEqual(dump_0, dump_1) + + @unittest.skip("TODO: RUSTPYTHON; unexpected payload for constant python value") + def test_unmarshallable(self): + # Check no crash after encountering unmarshallable objects. + # See https://github.com/python/cpython/issues/106287. + fset = frozenset([int]) + code = compile("a = 1", "<string>", "exec") + code = code.replace(co_consts=(1, fset, None)) + cases = (('tuple', (fset,)), + ('list', [fset]), + ('set', fset), + ('dict key', {fset: 'x'}), + ('dict value', {'x': fset}), + ('dict key & value', {fset: fset}), + ('slice', slice(fset, fset)), + ('code', code)) + for name, arg in cases: + with self.subTest(name, arg=arg): + with self.assertRaisesRegex(ValueError, "unmarshallable object"): + marshal.dumps((arg, memoryview(b''))) + + LARGE_SIZE = 2**31 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 @@ -420,76 +533,66 @@ def helper3(self, rsample, recursive=False, simple=False): else: self.assertGreaterEqual(len(s2), len(s3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testInt(self): intobj = 123321 self.helper(intobj) self.helper3(intobj, simple=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testFloat(self): floatobj = 1.2345 self.helper(floatobj) self.helper3(floatobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testStr(self): strobj = "abcde"*3 self.helper(strobj) self.helper3(strobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testBytes(self): bytesobj = b"abcde"*3 self.helper(bytesobj) self.helper3(bytesobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testList(self): for obj in self.keys: listobj = [obj, obj] self.helper(listobj) self.helper3(listobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testTuple(self): for obj in self.keys: tupleobj = (obj, obj) self.helper(tupleobj) self.helper3(tupleobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testSet(self): for obj in self.keys: setobj = {(obj, 1), (obj, 2)} self.helper(setobj) self.helper3(setobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testFrozenSet(self): for obj in self.keys: frozensetobj = frozenset({(obj, 1), (obj, 2)}) self.helper(frozensetobj) self.helper3(frozensetobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testDict(self): for obj in self.keys: dictobj = {"hello": obj, "goodbye": obj, obj: "hello"} self.helper(dictobj) self.helper3(dictobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testModule(self): with open(__file__, "rb") as f: code = f.read() @@ -533,8 +636,7 @@ class InterningTestCase(unittest.TestCase, HelperMixin): strobj = "this is an interned string" strobj = sys.intern(strobj) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testIntern(self): s = marshal.loads(marshal.dumps(self.strobj)) self.assertEqual(s, self.strobj) @@ -549,6 +651,20 @@ def testNoIntern(self): s2 = sys.intern(s) self.assertNotEqual(id(s2), id(s)) +class SliceTestCase(unittest.TestCase, HelperMixin): + @unittest.expectedFailure # TODO: RUSTPYTHON; NotImplementedError: TODO: not implemented yet or marshal unsupported type + def test_slice(self): + for obj in ( + slice(None), slice(1), slice(1, 2), slice(1, 2, 3), + slice({'set'}, ('tuple', {'with': 'dict'}, ), self.helper.__code__) + ): + with self.subTest(obj=str(obj)): + self.helper(obj) + + for version in range(4): + with self.assertRaises(ValueError): + marshal.dumps(obj, version) + @support.cpython_only @unittest.skipUnless(_testcapi, 'requires _testcapi') class CAPI_TestCase(unittest.TestCase, HelperMixin): @@ -609,7 +725,7 @@ def test_read_last_object_from_file(self): self.assertEqual(r, obj) with open(os_helper.TESTFN, 'wb') as f: - f.write(data[:1]) + f.write(omit_last_byte(data)) with self.assertRaises(EOFError): _testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN) os_helper.unlink(os_helper.TESTFN) @@ -626,7 +742,7 @@ def test_read_object_from_file(self): self.assertEqual(p, len(data)) with open(os_helper.TESTFN, 'wb') as f: - f.write(data[:1]) + f.write(omit_last_byte(data)) with self.assertRaises(EOFError): _testcapi.pymarshal_read_object_from_file(os_helper.TESTFN) os_helper.unlink(os_helper.TESTFN) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 2f641806526..d14336f8bac 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -4,6 +4,7 @@ from test.support import verbose, requires_IEEE_754 from test import support import unittest +import fractions import itertools import decimal import math @@ -32,8 +33,8 @@ else: file = __file__ test_dir = os.path.dirname(file) or os.curdir -math_testcases = os.path.join(test_dir, 'math_testcases.txt') -test_file = os.path.join(test_dir, 'cmath_testcases.txt') +math_testcases = os.path.join(test_dir, 'mathdata', 'math_testcases.txt') +test_file = os.path.join(test_dir, 'mathdata', 'cmath_testcases.txt') def to_ulps(x): @@ -186,6 +187,9 @@ def result_check(expected, got, ulp_tol=5, abs_tol=0.0): # Check exactly equal (applies also to strings representing exceptions) if got == expected: + if not got and not expected: + if math.copysign(1, got) != math.copysign(1, expected): + return f"expected {expected}, got {got} (zero has wrong sign)" return None failure = "not equal" @@ -234,6 +238,10 @@ def __init__(self, value): def __index__(self): return self.value +class BadDescr: + def __get__(self, obj, objtype=None): + raise ValueError + class MathTests(unittest.TestCase): def ftest(self, name, got, expected, ulp_tol=5, abs_tol=0.0): @@ -323,6 +331,7 @@ def testAtan2(self): self.ftest('atan2(0, 1)', math.atan2(0, 1), 0) self.ftest('atan2(1, 1)', math.atan2(1, 1), math.pi/4) self.ftest('atan2(1, 0)', math.atan2(1, 0), math.pi/2) + self.ftest('atan2(1, -1)', math.atan2(1, -1), 3*math.pi/4) # math.atan2(0, x) self.ftest('atan2(0., -inf)', math.atan2(0., NINF), math.pi) @@ -416,16 +425,22 @@ def __ceil__(self): return 42 class TestNoCeil: pass + class TestBadCeil: + __ceil__ = BadDescr() self.assertEqual(math.ceil(TestCeil()), 42) self.assertEqual(math.ceil(FloatCeil()), 42) self.assertEqual(math.ceil(FloatLike(42.5)), 43) self.assertRaises(TypeError, math.ceil, TestNoCeil()) + self.assertRaises(ValueError, math.ceil, TestBadCeil()) t = TestNoCeil() t.__ceil__ = lambda *args: args self.assertRaises(TypeError, math.ceil, t) self.assertRaises(TypeError, math.ceil, t, 0) + self.assertEqual(math.ceil(FloatLike(+1.0)), +1.0) + self.assertEqual(math.ceil(FloatLike(-1.0)), -1.0) + @requires_IEEE_754 def testCopysign(self): self.assertEqual(math.copysign(1, 42), 1.0) @@ -558,6 +573,8 @@ def testFloor(self): #self.assertEqual(math.ceil(NINF), NINF) #self.assertTrue(math.isnan(math.floor(NAN))) + class TestFloorIsNone(float): + __floor__ = None class TestFloor: def __floor__(self): return 42 @@ -566,16 +583,23 @@ def __floor__(self): return 42 class TestNoFloor: pass + class TestBadFloor: + __floor__ = BadDescr() self.assertEqual(math.floor(TestFloor()), 42) self.assertEqual(math.floor(FloatFloor()), 42) self.assertEqual(math.floor(FloatLike(41.9)), 41) self.assertRaises(TypeError, math.floor, TestNoFloor()) + self.assertRaises(ValueError, math.floor, TestBadFloor()) + self.assertRaises(TypeError, math.floor, TestFloorIsNone(3.5)) t = TestNoFloor() t.__floor__ = lambda *args: args self.assertRaises(TypeError, math.floor, t) self.assertRaises(TypeError, math.floor, t, 0) + self.assertEqual(math.floor(FloatLike(+1.0)), +1.0) + self.assertEqual(math.floor(FloatLike(-1.0)), -1.0) + def testFmod(self): self.assertRaises(TypeError, math.fmod) self.ftest('fmod(10, 1)', math.fmod(10, 1), 0.0) @@ -597,6 +621,7 @@ def testFmod(self): self.assertEqual(math.fmod(-3.0, NINF), -3.0) self.assertEqual(math.fmod(0.0, 3.0), 0.0) self.assertEqual(math.fmod(0.0, NINF), 0.0) + self.assertRaises(ValueError, math.fmod, INF, INF) def testFrexp(self): self.assertRaises(TypeError, math.frexp) @@ -638,7 +663,7 @@ def testFsum(self): def msum(iterable): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function - at http://code.activestate.com/recipes/393090/ + at https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/ """ tmant, texp = 0, 0 @@ -666,6 +691,7 @@ def msum(iterable): ([], 0.0), ([0.0], 0.0), ([1e100, 1.0, -1e100, 1e-100, 1e50, -1.0, -1e50], 1e-100), + ([1e100, 1.0, -1e100, 1e-100, 1e50, -1, -1e50], 1e-100), ([2.0**53, -0.5, -2.0**-54], 2.0**53-1.0), ([2.0**53, 1.0, 2.0**-100], 2.0**53+2.0), ([2.0**53+10.0, 1.0, 2.0**-100], 2.0**53+12.0), @@ -713,6 +739,22 @@ def msum(iterable): s = msum(vals) self.assertEqual(msum(vals), math.fsum(vals)) + self.assertEqual(math.fsum([1.0, math.inf]), math.inf) + self.assertTrue(math.isnan(math.fsum([math.nan, 1.0]))) + self.assertEqual(math.fsum([1e100, FloatLike(1.0), -1e100, 1e-100, + 1e50, FloatLike(-1.0), -1e50]), 1e-100) + self.assertRaises(OverflowError, math.fsum, [1e+308, 1e+308]) + self.assertRaises(ValueError, math.fsum, [math.inf, -math.inf]) + self.assertRaises(TypeError, math.fsum, ['spam']) + self.assertRaises(TypeError, math.fsum, 1) + self.assertRaises(OverflowError, math.fsum, [10**1000]) + + def bad_iter(): + yield 1.0 + raise ZeroDivisionError + + self.assertRaises(ZeroDivisionError, math.fsum, bad_iter()) + def testGcd(self): gcd = math.gcd self.assertEqual(gcd(0, 0), 0) @@ -773,9 +815,13 @@ def testHypot(self): # Test allowable types (those with __float__) self.assertEqual(hypot(12.0, 5.0), 13.0) self.assertEqual(hypot(12, 5), 13) + self.assertEqual(hypot(0.75, -1), 1.25) + self.assertEqual(hypot(-1, 0.75), 1.25) + self.assertEqual(hypot(0.75, FloatLike(-1.)), 1.25) + self.assertEqual(hypot(FloatLike(-1.), 0.75), 1.25) self.assertEqual(hypot(Decimal(12), Decimal(5)), 13) self.assertEqual(hypot(Fraction(12, 32), Fraction(5, 32)), Fraction(13, 32)) - self.assertEqual(hypot(bool(1), bool(0), bool(1), bool(1)), math.sqrt(3)) + self.assertEqual(hypot(True, False, True, True, True), 2.0) # Test corner cases self.assertEqual(hypot(0.0, 0.0), 0.0) # Max input is zero @@ -830,6 +876,8 @@ def testHypot(self): scale = FLOAT_MIN / 2.0 ** exp self.assertEqual(math.hypot(4*scale, 3*scale), 5*scale) + self.assertRaises(TypeError, math.hypot, *([1.0]*18), 'spam') + @requires_IEEE_754 @unittest.skipIf(HAVE_DOUBLE_ROUNDING, "hypot() loses accuracy on machines with double rounding") @@ -922,12 +970,16 @@ def testDist(self): # Test allowable types (those with __float__) self.assertEqual(dist((14.0, 1.0), (2.0, -4.0)), 13.0) self.assertEqual(dist((14, 1), (2, -4)), 13) + self.assertEqual(dist((FloatLike(14.), 1), (2, -4)), 13) + self.assertEqual(dist((11, 1), (FloatLike(-1.), -4)), 13) + self.assertEqual(dist((14, FloatLike(-1.)), (2, -6)), 13) + self.assertEqual(dist((14, -1), (2, -6)), 13) self.assertEqual(dist((D(14), D(1)), (D(2), D(-4))), D(13)) self.assertEqual(dist((F(14, 32), F(1, 32)), (F(2, 32), F(-4, 32))), F(13, 32)) - self.assertEqual(dist((True, True, False, True, False), - (True, False, True, True, False)), - sqrt(2.0)) + self.assertEqual(dist((True, True, False, False, True, True), + (True, False, True, False, False, False)), + 2.0) # Test corner cases self.assertEqual(dist((13.25, 12.5, -3.25), @@ -965,6 +1017,8 @@ class T(tuple): dist((1, 2, 3, 4), (5, 6, 7)) with self.assertRaises(ValueError): # Check dimension agree dist((1, 2, 3), (4, 5, 6, 7)) + with self.assertRaises(TypeError): + dist((1,)*17 + ("spam",), (1,)*18) with self.assertRaises(TypeError): # Rejects invalid types dist("abc", "xyz") int_too_big_for_float = 10 ** (sys.float_info.max_10_exp + 5) @@ -972,6 +1026,16 @@ class T(tuple): dist((1, int_too_big_for_float), (2, 3)) with self.assertRaises((ValueError, OverflowError)): dist((2, 3), (1, int_too_big_for_float)) + with self.assertRaises(TypeError): + dist((1,), 2) + with self.assertRaises(TypeError): + dist([1], 2) + + class BadFloat: + __float__ = BadDescr() + + with self.assertRaises(ValueError): + dist([1], [BadFloat()]) # Verify that the one dimensional case is equivalent to abs() for i in range(20): @@ -1064,6 +1128,15 @@ def __index__(self): with self.assertRaises(TypeError): math.isqrt(value) + @support.bigmemtest(2**32, memuse=0.85) + def test_isqrt_huge(self, size): + if size & 1: + size += 1 + v = 1 << size + w = math.isqrt(v) + self.assertEqual(w.bit_length(), size // 2 + 1) + self.assertEqual(w.bit_count(), 1) + def test_lcm(self): lcm = math.lcm self.assertEqual(lcm(0, 0), 0) @@ -1110,6 +1183,7 @@ def test_lcm(self): def testLdexp(self): self.assertRaises(TypeError, math.ldexp) + self.assertRaises(TypeError, math.ldexp, 2.0, 1.1) self.ftest('ldexp(0,1)', math.ldexp(0,1), 0) self.ftest('ldexp(1,1)', math.ldexp(1,1), 2) self.ftest('ldexp(1,-1)', math.ldexp(1,-1), 0.5) @@ -1140,8 +1214,15 @@ def testLdexp(self): self.assertEqual(math.ldexp(NINF, n), NINF) self.assertTrue(math.isnan(math.ldexp(NAN, n))) + @requires_IEEE_754 + def testLdexp_denormal(self): + # Denormal output incorrectly rounded (truncated) + # on some Windows. + self.assertEqual(math.ldexp(6993274598585239, -1126), 1e-323) + def testLog(self): self.assertRaises(TypeError, math.log) + self.assertRaises(TypeError, math.log, 1, 2, 3) self.ftest('log(1/e)', math.log(1/math.e), -1) self.ftest('log(1)', math.log(1), 0) self.ftest('log(e)', math.log(math.e), 1) @@ -1152,6 +1233,7 @@ def testLog(self): 2302.5850929940457) self.assertRaises(ValueError, math.log, -1.5) self.assertRaises(ValueError, math.log, -10**1000) + self.assertRaises(ValueError, math.log, 10, -10) self.assertRaises(ValueError, math.log, NINF) self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) @@ -1202,6 +1284,284 @@ def testLog10(self): self.assertEqual(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) + @support.bigmemtest(2**32, memuse=0.2) + def test_log_huge_integer(self, size): + v = 1 << size + self.assertAlmostEqual(math.log2(v), size) + self.assertAlmostEqual(math.log(v), size * 0.6931471805599453) + self.assertAlmostEqual(math.log10(v), size * 0.3010299956639812) + + def testSumProd(self): + sumprod = math.sumprod + Decimal = decimal.Decimal + Fraction = fractions.Fraction + + # Core functionality + self.assertEqual(sumprod(iter([10, 20, 30]), (1, 2, 3)), 140) + self.assertEqual(sumprod([1.5, 2.5], [3.5, 4.5]), 16.5) + self.assertEqual(sumprod([], []), 0) + self.assertEqual(sumprod([-1], [1.]), -1) + self.assertEqual(sumprod([1.], [-1]), -1) + + # Type preservation and coercion + for v in [ + (10, 20, 30), + (1.5, -2.5), + (Fraction(3, 5), Fraction(4, 5)), + (Decimal(3.5), Decimal(4.5)), + (2.5, 10), # float/int + (2.5, Fraction(3, 5)), # float/fraction + (25, Fraction(3, 5)), # int/fraction + (25, Decimal(4.5)), # int/decimal + ]: + for p, q in [(v, v), (v, v[::-1])]: + with self.subTest(p=p, q=q): + expected = sum(p_i * q_i for p_i, q_i in zip(p, q, strict=True)) + actual = sumprod(p, q) + self.assertEqual(expected, actual) + self.assertEqual(type(expected), type(actual)) + + # Bad arguments + self.assertRaises(TypeError, sumprod) # No args + self.assertRaises(TypeError, sumprod, []) # One arg + self.assertRaises(TypeError, sumprod, [], [], []) # Three args + self.assertRaises(TypeError, sumprod, None, [10]) # Non-iterable + self.assertRaises(TypeError, sumprod, [10], None) # Non-iterable + self.assertRaises(TypeError, sumprod, ['x'], [1.0]) + + # Uneven lengths + self.assertRaises(ValueError, sumprod, [10, 20], [30]) + self.assertRaises(ValueError, sumprod, [10], [20, 30]) + + # Overflows + self.assertEqual(sumprod([10**20], [1]), 10**20) + self.assertEqual(sumprod([1], [10**20]), 10**20) + self.assertEqual(sumprod([10**10], [10**10]), 10**20) + self.assertEqual(sumprod([10**7]*10**5, [10**7]*10**5), 10**19) + self.assertRaises(OverflowError, sumprod, [10**1000], [1.0]) + self.assertRaises(OverflowError, sumprod, [1.0], [10**1000]) + + # Error in iterator + def raise_after(n): + for i in range(n): + yield i + raise RuntimeError + with self.assertRaises(RuntimeError): + sumprod(range(10), raise_after(5)) + with self.assertRaises(RuntimeError): + sumprod(raise_after(5), range(10)) + + from test.test_iter import BasicIterClass + + self.assertEqual(sumprod(BasicIterClass(1), [1]), 0) + self.assertEqual(sumprod([1], BasicIterClass(1)), 0) + + # Error in multiplication + class BadMultiply: + def __mul__(self, other): + raise RuntimeError + def __rmul__(self, other): + raise RuntimeError + with self.assertRaises(RuntimeError): + sumprod([10, BadMultiply(), 30], [1, 2, 3]) + with self.assertRaises(RuntimeError): + sumprod([1, 2, 3], [10, BadMultiply(), 30]) + + # Error in addition + with self.assertRaises(TypeError): + sumprod(['abc', 3], [5, 10]) + with self.assertRaises(TypeError): + sumprod([5, 10], ['abc', 3]) + + # Special values should give the same as the pure python recipe + self.assertEqual(sumprod([10.1, math.inf], [20.2, 30.3]), math.inf) + self.assertEqual(sumprod([10.1, math.inf], [math.inf, 30.3]), math.inf) + self.assertEqual(sumprod([10.1, math.inf], [math.inf, math.inf]), math.inf) + self.assertEqual(sumprod([10.1, -math.inf], [20.2, 30.3]), -math.inf) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [-math.inf, math.inf]))) + self.assertTrue(math.isnan(sumprod([10.1, math.nan], [20.2, 30.3]))) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [math.nan, 30.3]))) + self.assertTrue(math.isnan(sumprod([10.1, math.inf], [20.3, math.nan]))) + + # Error cases that arose during development + args = ((-5, -5, 10), (1.5, 4611686018427387904, 2305843009213693952)) + self.assertEqual(sumprod(*args), 0.0) + + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sumprod() accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + def test_sumprod_accuracy(self): + sumprod = math.sumprod + self.assertEqual(sumprod([0.1] * 10, [1]*10), 1.0) + self.assertEqual(sumprod([0.1] * 20, [True, False] * 10), 1.0) + self.assertEqual(sumprod([True, False] * 10, [0.1] * 20), 1.0) + self.assertEqual(sumprod([1.0, 10E100, 1.0, -10E100], [1.0]*4), 2.0) + + @support.requires_resource('cpu') + def test_sumprod_stress(self): + sumprod = math.sumprod + product = itertools.product + Decimal = decimal.Decimal + Fraction = fractions.Fraction + + class Int(int): + def __add__(self, other): + return Int(int(self) + int(other)) + def __mul__(self, other): + return Int(int(self) * int(other)) + __radd__ = __add__ + __rmul__ = __mul__ + def __repr__(self): + return f'Int({int(self)})' + + class Flt(float): + def __add__(self, other): + return Int(int(self) + int(other)) + def __mul__(self, other): + return Int(int(self) * int(other)) + __radd__ = __add__ + __rmul__ = __mul__ + def __repr__(self): + return f'Flt({int(self)})' + + def baseline_sumprod(p, q): + """This defines the target behavior including exceptions and special values. + However, it is subject to rounding errors, so float inputs should be exactly + representable with only a few bits. + """ + total = 0 + for p_i, q_i in zip(p, q, strict=True): + total += p_i * q_i + return total + + def run(func, *args): + "Make comparing functions easier. Returns error status, type, and result." + try: + result = func(*args) + except (AssertionError, NameError): + raise + except Exception as e: + return type(e), None, 'None' + return None, type(result), repr(result) + + pools = [ + (-5, 10, -2**20, 2**31, 2**40, 2**61, 2**62, 2**80, 1.5, Int(7)), + (5.25, -3.5, 4.75, 11.25, 400.5, 0.046875, 0.25, -1.0, -0.078125), + (-19.0*2**500, 11*2**1000, -3*2**1500, 17*2*333, + 5.25, -3.25, -3.0*2**(-333), 3, 2**513), + (3.75, 2.5, -1.5, float('inf'), -float('inf'), float('NaN'), 14, + 9, 3+4j, Flt(13), 0.0), + (13.25, -4.25, Decimal('10.5'), Decimal('-2.25'), Fraction(13, 8), + Fraction(-11, 16), 4.75 + 0.125j, 97, -41, Int(3)), + (Decimal('6.125'), Decimal('12.375'), Decimal('-2.75'), Decimal(0), + Decimal('Inf'), -Decimal('Inf'), Decimal('NaN'), 12, 13.5), + (-2.0 ** -1000, 11*2**1000, 3, 7, -37*2**32, -2*2**-537, -2*2**-538, + 2*2**-513), + (-7 * 2.0 ** -510, 5 * 2.0 ** -520, 17, -19.0, -6.25), + (11.25, -3.75, -0.625, 23.375, True, False, 7, Int(5)), + ] + + for pool in pools: + for size in range(4): + for args1 in product(pool, repeat=size): + for args2 in product(pool, repeat=size): + args = (args1, args2) + self.assertEqual( + run(baseline_sumprod, *args), + run(sumprod, *args), + args, + ) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "sumprod() accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Other implementations may choose a different algorithm + @support.requires_resource('cpu') + def test_sumprod_extended_precision_accuracy(self): + import operator + from fractions import Fraction + from itertools import starmap + from collections import namedtuple + from math import log2, exp2, fabs + from random import choices, uniform, shuffle + from statistics import median + + DotExample = namedtuple('DotExample', ('x', 'y', 'target_sumprod', 'condition')) + + def DotExact(x, y): + vec1 = map(Fraction, x) + vec2 = map(Fraction, y) + return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) + + def Condition(x, y): + return 2.0 * DotExact(map(abs, x), map(abs, y)) / abs(DotExact(x, y)) + + def linspace(lo, hi, n): + width = (hi - lo) / (n - 1) + return [lo + width * i for i in range(n)] + + def GenDot(n, c): + """ Algorithm 6.1 (GenDot) works as follows. The condition number (5.7) of + the dot product xT y is proportional to the degree of cancellation. In + order to achieve a prescribed cancellation, we generate the first half of + the vectors x and y randomly within a large exponent range. This range is + chosen according to the anticipated condition number. The second half of x + and y is then constructed choosing xi randomly with decreasing exponent, + and calculating yi such that some cancellation occurs. Finally, we permute + the vectors x, y randomly and calculate the achieved condition number. + """ + + assert n >= 6 + n2 = n // 2 + x = [0.0] * n + y = [0.0] * n + b = log2(c) + + # First half with exponents from 0 to |_b/2_| and random ints in between + e = choices(range(int(b/2)), k=n2) + e[0] = int(b / 2) + 1 + e[-1] = 0.0 + + x[:n2] = [uniform(-1.0, 1.0) * exp2(p) for p in e] + y[:n2] = [uniform(-1.0, 1.0) * exp2(p) for p in e] + + # Second half + e = list(map(round, linspace(b/2, 0.0 , n-n2))) + for i in range(n2, n): + x[i] = uniform(-1.0, 1.0) * exp2(e[i - n2]) + y[i] = (uniform(-1.0, 1.0) * exp2(e[i - n2]) - DotExact(x, y)) / x[i] + + # Shuffle + pairs = list(zip(x, y)) + shuffle(pairs) + x, y = zip(*pairs) + + return DotExample(x, y, DotExact(x, y), Condition(x, y)) + + def RelativeError(res, ex): + x, y, target_sumprod, condition = ex + n = DotExact(list(x) + [-res], list(y) + [1]) + return fabs(n / target_sumprod) + + def Trial(dotfunc, c, n): + ex = GenDot(10, c) + res = dotfunc(ex.x, ex.y) + return RelativeError(res, ex) + + times = 1000 # Number of trials + n = 20 # Length of vectors + c = 1e30 # Target condition number + + # If the following test fails, it means that the C math library + # implementation of fma() is not compliant with the C99 standard + # and is inaccurate. To solve this problem, make a new build + # with the symbol UNRELIABLE_FMA defined. That will enable a + # slower but accurate code path that avoids the fma() call. + relative_err = median(Trial(math.sumprod, c, n) for i in range(times)) + self.assertLess(relative_err, 1e-16) + def testModf(self): self.assertRaises(TypeError, math.modf) @@ -1235,6 +1595,7 @@ def testPow(self): self.assertTrue(math.isnan(math.pow(2, NAN))) self.assertTrue(math.isnan(math.pow(0, NAN))) self.assertEqual(math.pow(1, NAN), 1) + self.assertRaises(OverflowError, math.pow, 1e+100, 1e+100) # pow(0., x) self.assertEqual(math.pow(0., INF), 0.) @@ -1550,7 +1911,7 @@ def testTan(self): try: self.assertTrue(math.isnan(math.tan(INF))) self.assertTrue(math.isnan(math.tan(NINF))) - except: + except ValueError: self.assertRaises(ValueError, math.tan, INF) self.assertRaises(ValueError, math.tan, NINF) self.assertTrue(math.isnan(math.tan(NAN))) @@ -1591,6 +1952,8 @@ def __trunc__(self): return 23 class TestNoTrunc: pass + class TestBadTrunc: + __trunc__ = BadDescr() self.assertEqual(math.trunc(TestTrunc()), 23) self.assertEqual(math.trunc(FloatTrunc()), 23) @@ -1599,6 +1962,7 @@ class TestNoTrunc: self.assertRaises(TypeError, math.trunc, 1, 2) self.assertRaises(TypeError, math.trunc, FloatLike(23.5)) self.assertRaises(TypeError, math.trunc, TestNoTrunc()) + self.assertRaises(ValueError, math.trunc, TestBadTrunc()) def testIsfinite(self): self.assertTrue(math.isfinite(0.0)) @@ -1626,11 +1990,11 @@ def testIsinf(self): self.assertFalse(math.isinf(0.)) self.assertFalse(math.isinf(1.)) - @requires_IEEE_754 def test_nan_constant(self): + # `math.nan` must be a quiet NaN with positive sign bit self.assertTrue(math.isnan(math.nan)) + self.assertEqual(math.copysign(1., math.nan), 1.) - @requires_IEEE_754 def test_inf_constant(self): self.assertTrue(math.isinf(math.inf)) self.assertGreater(math.inf, 0.0) @@ -1674,8 +2038,6 @@ def test_exceptions(self): else: self.fail("sqrt(-1) didn't raise ValueError") - # TODO: RUSTPYTHON - @unittest.expectedFailure @requires_IEEE_754 def test_testfile(self): # Some tests need to be skipped on ancient OS X versions. @@ -1719,6 +2081,13 @@ def test_testfile(self): except OverflowError: result = 'OverflowError' + # C99+ says for math.h's sqrt: If the argument is +∞ or ±0, it is + # returned, unmodified. On another hand, for csqrt: If z is ±0+0i, + # the result is +0+0i. Lets correct zero sign of er to follow + # first convention. + if id in ['sqrt0002', 'sqrt0003', 'sqrt1001', 'sqrt1023']: + er = math.copysign(er, ar) + # Default tolerances ulp_tol, abs_tol = 5, 0.0 @@ -1733,7 +2102,6 @@ def test_testfile(self): self.fail('Failures in test_testfile:\n ' + '\n '.join(failures)) - @unittest.skip("TODO: RUSTPYTHON, Currently hangs. Function never finishes.") @requires_IEEE_754 def test_mtestfile(self): fail_fmt = "{}: {}({!r}): {}" @@ -1802,6 +2170,8 @@ def test_mtestfile(self): '\n '.join(failures)) def test_prod(self): + from fractions import Fraction as F + prod = math.prod self.assertEqual(prod([]), 1) self.assertEqual(prod([], start=5), 5) @@ -1813,6 +2183,14 @@ def test_prod(self): self.assertEqual(prod([1.0, 2.0, 3.0, 4.0, 5.0]), 120.0) self.assertEqual(prod([1, 2, 3, 4.0, 5.0]), 120.0) self.assertEqual(prod([1.0, 2.0, 3.0, 4, 5]), 120.0) + self.assertEqual(prod([1., F(3, 2)]), 1.5) + + # Error in multiplication + class BadMultiply: + def __rmul__(self, other): + raise RuntimeError + with self.assertRaises(RuntimeError): + prod([10., BadMultiply()]) # Test overflow in fast-path for integers self.assertEqual(prod([1, 1, 2**32, 1, 1]), 2**32) @@ -2044,11 +2422,20 @@ def test_nextafter(self): float.fromhex('0x1.fffffffffffffp-1')) self.assertEqual(math.nextafter(1.0, INF), float.fromhex('0x1.0000000000001p+0')) + self.assertEqual(math.nextafter(1.0, -INF, steps=1), + float.fromhex('0x1.fffffffffffffp-1')) + self.assertEqual(math.nextafter(1.0, INF, steps=1), + float.fromhex('0x1.0000000000001p+0')) + self.assertEqual(math.nextafter(1.0, -INF, steps=3), + float.fromhex('0x1.ffffffffffffdp-1')) + self.assertEqual(math.nextafter(1.0, INF, steps=3), + float.fromhex('0x1.0000000000003p+0')) # x == y: y is returned - self.assertEqual(math.nextafter(2.0, 2.0), 2.0) - self.assertEqualSign(math.nextafter(-0.0, +0.0), +0.0) - self.assertEqualSign(math.nextafter(+0.0, -0.0), -0.0) + for steps in range(1, 5): + self.assertEqual(math.nextafter(2.0, 2.0, steps=steps), 2.0) + self.assertEqualSign(math.nextafter(-0.0, +0.0, steps=steps), +0.0) + self.assertEqualSign(math.nextafter(+0.0, -0.0, steps=steps), -0.0) # around 0.0 smallest_subnormal = sys.float_info.min * sys.float_info.epsilon @@ -2073,6 +2460,11 @@ def test_nextafter(self): self.assertIsNaN(math.nextafter(1.0, NAN)) self.assertIsNaN(math.nextafter(NAN, NAN)) + self.assertEqual(1.0, math.nextafter(1.0, INF, steps=0)) + with self.assertRaises(ValueError): + math.nextafter(1.0, INF, steps=-1) + + @requires_IEEE_754 def test_ulp(self): self.assertEqual(math.ulp(1.0), sys.float_info.epsilon) @@ -2112,6 +2504,54 @@ def __float__(self): # argument to a float. self.assertFalse(getattr(y, "converted", False)) + def test_input_exceptions(self): + self.assertRaises(TypeError, math.exp, "spam") + self.assertRaises(TypeError, math.erf, "spam") + self.assertRaises(TypeError, math.atan2, "spam", 1.0) + self.assertRaises(TypeError, math.atan2, 1.0, "spam") + self.assertRaises(TypeError, math.atan2, 1.0) + self.assertRaises(TypeError, math.atan2, 1.0, 2.0, 3.0) + + def test_exception_messages(self): + x = -1.1 + with self.assertRaisesRegex(ValueError, + f"expected a nonnegative input, got {x}"): + math.sqrt(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(123, x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x, 123) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log2(x) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log10(x) + x = decimal.Decimal('-1.1') + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {x}"): + math.log(x) + x = fractions.Fraction(1, 10**400) + with self.assertRaisesRegex(ValueError, + f"expected a positive input, got {float(x)}"): + math.log(x) + x = -123 + with self.assertRaisesRegex(ValueError, + "expected a positive input$"): + math.log(x) + with self.assertRaisesRegex(ValueError, + f"expected a noninteger or positive integer, got {x}"): + math.gamma(x) + x = 1.0 + with self.assertRaisesRegex(ValueError, + f"expected a number between -1 and 1, got {x}"): + math.atanh(x) + # Custom assertions. def assertIsNaN(self, value): @@ -2250,9 +2690,260 @@ def test_fractions(self): self.assertAllNotClose(fraction_examples, rel_tol=1e-9) +class FMATests(unittest.TestCase): + """ Tests for math.fma. """ + + def test_fma_nan_results(self): + # Selected representative values. + values = [ + -math.inf, -1e300, -2.3, -1e-300, -0.0, + 0.0, 1e-300, 2.3, 1e300, math.inf, math.nan + ] + + # If any input is a NaN, the result should be a NaN, too. + for a, b in itertools.product(values, repeat=2): + with self.subTest(a=a, b=b): + self.assertIsNaN(math.fma(math.nan, a, b)) + self.assertIsNaN(math.fma(a, math.nan, b)) + self.assertIsNaN(math.fma(a, b, math.nan)) + + def test_fma_infinities(self): + # Cases involving infinite inputs or results. + positives = [1e-300, 2.3, 1e300, math.inf] + finites = [-1e300, -2.3, -1e-300, -0.0, 0.0, 1e-300, 2.3, 1e300] + non_nans = [-math.inf, -2.3, -0.0, 0.0, 2.3, math.inf] + + # ValueError due to inf * 0 computation. + for c in non_nans: + for infinity in [math.inf, -math.inf]: + for zero in [0.0, -0.0]: + with self.subTest(c=c, infinity=infinity, zero=zero): + with self.assertRaises(ValueError): + math.fma(infinity, zero, c) + with self.assertRaises(ValueError): + math.fma(zero, infinity, c) + + # ValueError when a*b and c both infinite of opposite signs. + for b in positives: + with self.subTest(b=b): + with self.assertRaises(ValueError): + math.fma(math.inf, b, -math.inf) + with self.assertRaises(ValueError): + math.fma(math.inf, -b, math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, -b, -math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, b, math.inf) + with self.assertRaises(ValueError): + math.fma(b, math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(-b, math.inf, math.inf) + with self.assertRaises(ValueError): + math.fma(-b, -math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(b, -math.inf, math.inf) + + # Infinite result when a*b and c both infinite of the same sign. + for b in positives: + with self.subTest(b=b): + self.assertEqual(math.fma(math.inf, b, math.inf), math.inf) + self.assertEqual(math.fma(math.inf, -b, -math.inf), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, math.inf), math.inf) + self.assertEqual(math.fma(-math.inf, b, -math.inf), -math.inf) + self.assertEqual(math.fma(b, math.inf, math.inf), math.inf) + self.assertEqual(math.fma(-b, math.inf, -math.inf), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, math.inf), math.inf) + self.assertEqual(math.fma(b, -math.inf, -math.inf), -math.inf) + + # Infinite result when a*b finite, c infinite. + for a, b in itertools.product(finites, finites): + with self.subTest(b=b): + self.assertEqual(math.fma(a, b, math.inf), math.inf) + self.assertEqual(math.fma(a, b, -math.inf), -math.inf) + + # Infinite result when a*b infinite, c finite. + for b, c in itertools.product(positives, finites): + with self.subTest(b=b, c=c): + self.assertEqual(math.fma(math.inf, b, c), math.inf) + self.assertEqual(math.fma(-math.inf, b, c), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, c), math.inf) + self.assertEqual(math.fma(math.inf, -b, c), -math.inf) + + self.assertEqual(math.fma(b, math.inf, c), math.inf) + self.assertEqual(math.fma(b, -math.inf, c), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, c), math.inf) + self.assertEqual(math.fma(-b, math.inf, c), -math.inf) + + # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 + # properly: it doesn't use the right sign when the result is zero. + @unittest.skipIf( + sys.platform.startswith(("freebsd", "wasi", "netbsd", "emscripten")) + or (sys.platform == "android" and platform.machine() == "x86_64") + or support.linked_to_musl(), # gh-131032 + f"this platform doesn't implement IEE 754-2008 properly") + # gh-131032: musl is fixed but the fix is not yet released; when the fixed + # version is known change this to: + # or support.linked_to_musl() < (1, <m>, <p>) + def test_fma_zero_result(self): + nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] + + # Zero results from exact zero inputs. + for b in nonnegative_finites: + with self.subTest(b=b): + self.assertIsPositiveZero(math.fma(0.0, b, 0.0)) + self.assertIsPositiveZero(math.fma(0.0, b, -0.0)) + self.assertIsNegativeZero(math.fma(0.0, -b, -0.0)) + self.assertIsPositiveZero(math.fma(0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, -0.0)) + self.assertIsNegativeZero(math.fma(-0.0, b, -0.0)) + self.assertIsPositiveZero(math.fma(-0.0, b, 0.0)) + + self.assertIsPositiveZero(math.fma(b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(b, 0.0, -0.0)) + self.assertIsNegativeZero(math.fma(-b, 0.0, -0.0)) + self.assertIsPositiveZero(math.fma(-b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, -0.0)) + self.assertIsNegativeZero(math.fma(b, -0.0, -0.0)) + self.assertIsPositiveZero(math.fma(b, -0.0, 0.0)) + + # Exact zero result from nonzero inputs. + self.assertIsPositiveZero(math.fma(2.0, 2.0, -4.0)) + self.assertIsPositiveZero(math.fma(2.0, -2.0, 4.0)) + self.assertIsPositiveZero(math.fma(-2.0, -2.0, -4.0)) + self.assertIsPositiveZero(math.fma(-2.0, 2.0, 4.0)) + + # Underflow to zero. + tiny = 1e-300 + self.assertIsPositiveZero(math.fma(tiny, tiny, 0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, 0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, 0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, 0.0)) + self.assertIsPositiveZero(math.fma(tiny, tiny, -0.0)) + self.assertIsNegativeZero(math.fma(tiny, -tiny, -0.0)) + self.assertIsPositiveZero(math.fma(-tiny, -tiny, -0.0)) + self.assertIsNegativeZero(math.fma(-tiny, tiny, -0.0)) + + # Corner case where rounding the multiplication would + # give the wrong result. + x = float.fromhex('0x1p-500') + y = float.fromhex('0x1p-550') + z = float.fromhex('0x1p-1000') + self.assertIsNegativeZero(math.fma(x-y, x+y, -z)) + self.assertIsPositiveZero(math.fma(y-x, x+y, z)) + self.assertIsNegativeZero(math.fma(y-x, -(x+y), -z)) + self.assertIsPositiveZero(math.fma(x-y, -(x+y), z)) + + def test_fma_overflow(self): + a = b = float.fromhex('0x1p512') + c = float.fromhex('0x1p1023') + # Overflow from multiplication. + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + self.assertEqual(math.fma(a, b/2.0, 0.0), c) + # Overflow from the addition. + with self.assertRaises(OverflowError): + math.fma(a, b/2.0, c) + # No overflow, even though a*b overflows a float. + self.assertEqual(math.fma(a, b, -c), c) + + # Extreme case: a * b is exactly at the overflow boundary, so the + # tiniest offset makes a difference between overflow and a finite + # result. + a = float.fromhex('0x1.ffffffc000000p+511') + b = float.fromhex('0x1.0000002000000p+512') + c = float.fromhex('0x0.0000000000001p-1022') + with self.assertRaises(OverflowError): + math.fma(a, b, 0.0) + with self.assertRaises(OverflowError): + math.fma(a, b, c) + self.assertEqual(math.fma(a, b, -c), + float.fromhex('0x1.fffffffffffffp+1023')) + + # Another extreme case: here a*b is about as large as possible subject + # to math.fma(a, b, c) being finite. + a = float.fromhex('0x1.ae565943785f9p+512') + b = float.fromhex('0x1.3094665de9db8p+512') + c = float.fromhex('0x1.fffffffffffffp+1023') + self.assertEqual(math.fma(a, b, -c), c) + + def test_fma_single_round(self): + a = float.fromhex('0x1p-50') + self.assertEqual(math.fma(a - 1.0, a + 1.0, 1.0), a*a) + + def test_random(self): + # A collection of randomly generated inputs for which the naive FMA + # (with two rounds) gives a different result from a singly-rounded FMA. + + # tuples (a, b, c, expected) + test_values = [ + ('0x1.694adde428b44p-1', '0x1.371b0d64caed7p-1', + '0x1.f347e7b8deab8p-4', '0x1.19f10da56c8adp-1'), + ('0x1.605401ccc6ad6p-2', '0x1.ce3a40bf56640p-2', + '0x1.96e3bf7bf2e20p-2', '0x1.1af6d8aa83101p-1'), + ('0x1.e5abd653a67d4p-2', '0x1.a2e400209b3e6p-1', + '0x1.a90051422ce13p-1', '0x1.37d68cc8c0fbbp+0'), + ('0x1.f94e8efd54700p-2', '0x1.123065c812cebp-1', + '0x1.458f86fb6ccd0p-1', '0x1.ccdcee26a3ff3p-1'), + ('0x1.bd926f1eedc96p-1', '0x1.eee9ca68c5740p-1', + '0x1.960c703eb3298p-2', '0x1.3cdcfb4fdb007p+0'), + ('0x1.27348350fbccdp-1', '0x1.3b073914a53f1p-1', + '0x1.e300da5c2b4cbp-1', '0x1.4c51e9a3c4e29p+0'), + ('0x1.2774f00b3497bp-1', '0x1.7038ec336bff0p-2', + '0x1.2f6f2ccc3576bp-1', '0x1.99ad9f9c2688bp-1'), + ('0x1.51d5a99300e5cp-1', '0x1.5cd74abd445a1p-1', + '0x1.8880ab0bbe530p-1', '0x1.3756f96b91129p+0'), + ('0x1.73cb965b821b8p-2', '0x1.218fd3d8d5371p-1', + '0x1.d1ea966a1f758p-2', '0x1.5217b8fd90119p-1'), + ('0x1.4aa98e890b046p-1', '0x1.954d85dff1041p-1', + '0x1.122b59317ebdfp-1', '0x1.0bf644b340cc5p+0'), + ('0x1.e28f29e44750fp-1', '0x1.4bcc4fdcd18fep-1', + '0x1.fd47f81298259p-1', '0x1.9b000afbc9995p+0'), + ('0x1.d2e850717fe78p-3', '0x1.1dd7531c303afp-1', + '0x1.e0869746a2fc2p-2', '0x1.316df6eb26439p-1'), + ('0x1.cf89c75ee6fbap-2', '0x1.b23decdc66825p-1', + '0x1.3d1fe76ac6168p-1', '0x1.00d8ea4c12abbp+0'), + ('0x1.3265ae6f05572p-2', '0x1.16d7ec285f7a2p-1', + '0x1.0b8405b3827fbp-1', '0x1.5ef33c118a001p-1'), + ('0x1.c4d1bf55ec1a5p-1', '0x1.bc59618459e12p-2', + '0x1.ce5b73dc1773dp-1', '0x1.496cf6164f99bp+0'), + ('0x1.d350026ac3946p-1', '0x1.9a234e149a68cp-2', + '0x1.f5467b1911fd6p-2', '0x1.b5cee3225caa5p-1'), + ] + for a_hex, b_hex, c_hex, expected_hex in test_values: + with self.subTest(a_hex=a_hex, b_hex=b_hex, c_hex=c_hex, + expected_hex=expected_hex): + a = float.fromhex(a_hex) + b = float.fromhex(b_hex) + c = float.fromhex(c_hex) + expected = float.fromhex(expected_hex) + self.assertEqual(math.fma(a, b, c), expected) + self.assertEqual(math.fma(b, a, c), expected) + + # Custom assertions. + def assertIsNaN(self, value): + self.assertTrue( + math.isnan(value), + msg="Expected a NaN, got {!r}".format(value) + ) + + def assertIsPositiveZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) > 0, + msg="Expected a positive zero, got {!r}".format(value) + ) + + def assertIsNegativeZero(self, value): + self.assertTrue( + value == 0 and math.copysign(1, value) < 0, + msg="Expected a negative zero, got {!r}".format(value) + ) + + def load_tests(loader, tests, pattern): from doctest import DocFileSuite - # tests.addTest(DocFileSuite("ieee754.txt")) + tests.addTest(DocFileSuite(os.path.join("mathdata", "ieee754.txt"))) return tests if __name__ == '__main__': diff --git a/Lib/test/test_math_property.py b/Lib/test/test_math_property.py new file mode 100644 index 00000000000..7d51aa17b4c --- /dev/null +++ b/Lib/test/test_math_property.py @@ -0,0 +1,41 @@ +import functools +import unittest +from math import isnan, nextafter +from test.support import requires_IEEE_754 +from test.support.hypothesis_helper import hypothesis + +floats = hypothesis.strategies.floats +integers = hypothesis.strategies.integers + + +def assert_equal_float(x, y): + assert isnan(x) and isnan(y) or x == y + + +def via_reduce(x, y, steps): + return functools.reduce(nextafter, [y] * steps, x) + + +class NextafterTests(unittest.TestCase): + @requires_IEEE_754 + @hypothesis.given( + x=floats(), + y=floats(), + steps=integers(min_value=0, max_value=2**16)) + def test_count(self, x, y, steps): + assert_equal_float(via_reduce(x, y, steps), + nextafter(x, y, steps=steps)) + + @requires_IEEE_754 + @hypothesis.given( + x=floats(), + y=floats(), + a=integers(min_value=0), + b=integers(min_value=0)) + def test_addition_commutes(self, x, y, a, b): + first = nextafter(x, y, steps=a) + second = nextafter(first, y, steps=b) + combined = nextafter(x, y, steps=a+b) + hypothesis.note(f"{first} -> {second} == {combined}") + + assert_equal_float(second, combined) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 5b695f167ad..7b321600e88 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -6,10 +6,12 @@ import unittest from test import support +import gc import io import _pyio as pyio import pickle import sys +import weakref class IntLike: def __init__(self, num): @@ -52,6 +54,12 @@ def testSeek(self): self.assertEqual(buf[3:], bytesIo.read()) self.assertRaises(TypeError, bytesIo.seek, 0.0) + self.assertEqual(sys.maxsize, bytesIo.seek(sys.maxsize)) + self.assertEqual(self.EOF, bytesIo.read(4)) + + self.assertEqual(sys.maxsize - 2, bytesIo.seek(sys.maxsize - 2)) + self.assertEqual(self.EOF, bytesIo.read(4)) + def testTell(self): buf = self.buftype("1234567890") bytesIo = self.ioclass(buf) @@ -263,8 +271,8 @@ def test_iterator(self): memio = self.ioclass(buf * 10) self.assertEqual(iter(memio), memio) - self.assertTrue(hasattr(memio, '__iter__')) - self.assertTrue(hasattr(memio, '__next__')) + self.assertHasAttr(memio, '__iter__') + self.assertHasAttr(memio, '__next__') i = 0 for line in memio: self.assertEqual(line, buf) @@ -463,6 +471,39 @@ def test_getbuffer(self): memio.close() self.assertRaises(ValueError, memio.getbuffer) + def test_getbuffer_empty(self): + memio = self.ioclass() + buf = memio.getbuffer() + self.assertEqual(bytes(buf), b"") + # Trying to change the size of the BytesIO while a buffer is exported + # raises a BufferError. + self.assertRaises(BufferError, memio.write, b'x') + buf2 = memio.getbuffer() + self.assertRaises(BufferError, memio.write, b'x') + buf.release() + self.assertRaises(BufferError, memio.write, b'x') + buf2.release() + memio.write(b'x') + + def test_getbuffer_gc_collect(self): + memio = self.ioclass(b"1234567890") + buf = memio.getbuffer() + memiowr = weakref.ref(memio) + bufwr = weakref.ref(buf) + # Create a reference loop. + a = [buf] + a.append(a) + # The Python implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + del memio + del buf + del a + # The C implementation emits an unraisable exception. + with support.catch_unraisable_exception(): + gc.collect() + self.assertIsNone(memiowr()) + self.assertIsNone(bufwr()) + def test_read1(self): buf = self.buftype("1234567890") self.assertEqual(self.ioclass(buf).read1(), buf) @@ -517,6 +558,14 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_issue141311(self): + memio = self.ioclass() + # Seek allows PY_SSIZE_T_MAX, read should handle that. + # Past end of buffer read should always return 0 (EOF). + self.assertEqual(sys.maxsize, memio.seek(sys.maxsize)) + buf = bytearray(2) + self.assertEqual(0, memio.readinto(buf)) + def test_unicode(self): memio = self.ioclass() @@ -538,6 +587,75 @@ def test_issue5449(self): self.ioclass(initial_bytes=buf) self.assertRaises(TypeError, self.ioclass, buf, foo=None) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_write_concurrent_close(self): + class B: + def __buffer__(self, flags): + memio.close() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(ValueError, memio.write, B()) + + # Prevent crashes when memio.write() or memio.writelines() + # concurrently mutates (e.g., closes or exports) 'memio'. + # See: https://github.com/python/cpython/issues/143378. + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_writelines_concurrent_close(self): + class B: + def __buffer__(self, flags): + memio.close() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(ValueError, memio.writelines, [B()]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_write_concurrent_export(self): + class B: + buf = None + def __buffer__(self, flags): + self.buf = memio.getbuffer() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(BufferError, memio.write, B()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_writelines_concurrent_export(self): + class B: + buf = None + def __buffer__(self, flags): + self.buf = memio.getbuffer() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(BufferError, memio.writelines, [B()]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' + def test_write_mutating_buffer(self): + # Test that buffer is exported only once during write(). + # See: https://github.com/python/cpython/issues/143602. + class B: + count = 0 + def __buffer__(self, flags): + self.count += 1 + if self.count == 1: + return memoryview(b"AAA") + else: + return memoryview(b"BBBBBBBBB") + + memio = self.ioclass(b'0123456789') + memio.seek(2) + b = B() + n = memio.write(b) + + self.assertEqual(b.count, 1) + self.assertEqual(n, 3) + self.assertEqual(memio.getvalue(), b"01AAA56789") + self.assertEqual(memio.tell(), 5) + class TextIOTestMixin: @@ -724,67 +842,6 @@ class CBytesIOTest(PyBytesIOTest): ioclass = io.BytesIO UnsupportedOperation = io.UnsupportedOperation - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytes_array(self): - super().test_bytes_array() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_flags(self): - super().test_flags() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_getbuffer(self): - super().test_getbuffer() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_init(self): - super().test_init() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue5449(self): - super().test_issue5449() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickling(self): - super().test_pickling() - - def test_read(self): - super().test_read() - - def test_readline(self): - super().test_readline() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_relative_seek(self): - super().test_relative_seek() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_seek(self): - super().test_seek() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_subclassing(self): - super().test_subclassing() - - def test_truncate(self): - super().test_truncate() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_write(self): - super().test_write() - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getstate(self): memio = self.ioclass() state = memio.__getstate__() @@ -796,8 +853,7 @@ def test_getstate(self): memio.close() self.assertRaises(ValueError, memio.__getstate__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'bytes' but 'bytearray' found. def test_setstate(self): # This checks whether __setstate__ does proper input validation. memio = self.ioclass() @@ -829,7 +885,7 @@ def test_sizeof(self): def _test_cow_mutation(self, mutation): # Common code for all BytesIO copy-on-write mutation tests. - imm = b' ' * 1024 + imm = (' ' * 1024).encode("ascii") old_rc = sys.getrefcount(imm) memio = self.ioclass(imm) self.assertEqual(sys.getrefcount(imm), old_rc + 1) @@ -870,88 +926,25 @@ def test_cow_mutable(self): memio = self.ioclass(ba) self.assertEqual(sys.getrefcount(ba), old_rc) -class CStringIOTest(PyStringIOTest): - ioclass = io.StringIO - UnsupportedOperation = io.UnsupportedOperation - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_detach(self): - super().test_detach() - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by writable def test_flags(self): - super().test_flags() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_init(self): - super().test_init() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue5265(self): - super().test_issue5265() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_cr(self): - super().test_newline_cr() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_crlf(self): - super().test_newline_crlf() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_empty(self): - super().test_newline_empty() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_none(self): - super().test_newline_none() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines_property(self): - super().test_newlines_property() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickling(self): - super().test_pickling() - - def test_read(self): - super().test_read() - - def test_readline(self): - super().test_readline() + return super().test_flags() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_relative_seek(self): - super().test_relative_seek() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by write + def test_write(self): + return super().test_write() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u64 def test_seek(self): - super().test_seek() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_textio_properties(self): - super().test_textio_properties() + return super().test_seek() - def test_truncate(self): - super().test_truncate() +class CStringIOTest(PyStringIOTest): + ioclass = io.StringIO + UnsupportedOperation = io.UnsupportedOperation # XXX: For the Python version of io.StringIO, this is highly # dependent on the encoding used for the underlying buffer. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 8 != 2 def test_widechar(self): buf = self.buftype("\U0002030a\U00020347") memio = self.ioclass(buf) @@ -964,8 +957,6 @@ def test_widechar(self): self.assertEqual(memio.tell(), len(buf) * 2) self.assertEqual(memio.getvalue(), buf + buf) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getstate(self): memio = self.ioclass() state = memio.__getstate__() @@ -978,8 +969,7 @@ def test_getstate(self): memio.close() self.assertRaises(ValueError, memio.__getstate__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by __setstate__ def test_setstate(self): # This checks whether __setstate__ does proper input validation. memio = self.ioclass() @@ -997,59 +987,49 @@ def test_setstate(self): memio.close() self.assertRaises(ValueError, memio.__setstate__, ("closed", "", 0, None)) + @unittest.expectedFailure # TODO: RUSTPYTHON; + + def test_issue5265(self): + return super().test_issue5265() -class CStringIOPickleTest(PyStringIOPickleTest): - UnsupportedOperation = io.UnsupportedOperation + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++ + def test_newline_empty(self): + return super().test_newline_empty() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_issue5265(self): - super().test_issue5265() + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^ + def test_newline_none(self): + return super().test_newline_none() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_cr(self): - super().test_newline_cr() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by seek + def test_relative_seek(self): + return super().test_relative_seek() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_crlf(self): - super().test_newline_crlf() + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by writable + def test_flags(self): + return super().test_flags() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_default(self): - super().test_newline_default() + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'detach' + def test_detach(self): + return super().test_detach() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_empty(self): - super().test_newline_empty() + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? + def test_newlines_property(self): + return super().test_newlines_property() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_lf(self): - super().test_newline_lf() + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u64 + def test_seek(self): + return super().test_seek() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newline_none(self): - super().test_newline_none() + @unittest.expectedFailure # TODO: RUSTPYTHON; d + def test_newline_cr(self): + return super().test_newline_cr() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines_property(self): - super().test_newlines_property() + @unittest.expectedFailure # TODO: RUSTPYTHON; d + def test_newline_crlf(self): + return super().test_newline_crlf() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_relative_seek(self): - super().test_relative_seek() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_textio_properties(self): - super().test_textio_properties() +class CStringIOPickleTest(PyStringIOPickleTest): + UnsupportedOperation = io.UnsupportedOperation class ioclass(io.StringIO): def __new__(cls, *args, **kwargs): @@ -1057,6 +1037,34 @@ def __new__(cls, *args, **kwargs): def __init__(self, *args, **kwargs): pass + @unittest.expectedFailure # TODO: RUSTPYTHON; + + def test_issue5265(self): + return super().test_issue5265() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++ + def test_newline_empty(self): + return super().test_newline_empty() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^ + def test_newline_none(self): + return super().test_newline_none() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by seek + def test_relative_seek(self): + return super().test_relative_seek() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? + def test_newlines_property(self): + return super().test_newlines_property() + + @unittest.expectedFailure # TODO: RUSTPYTHON; d + def test_newline_cr(self): + return super().test_newline_cr() + + @unittest.expectedFailure # TODO: RUSTPYTHON; d + def test_newline_crlf(self): + return super().test_newline_crlf() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 5ab9441da4f..891c4d76745 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -13,8 +13,15 @@ import io import copy import pickle +import struct -from test.support import import_helper +from itertools import product +from test import support +from test.support import import_helper, threading_helper + + +class MyObject: + pass class AbstractMemoryTests: @@ -53,12 +60,53 @@ def test_getitem(self): for tp in self._types: self.check_getitem_with_type(tp) + def test_index(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) # may be a sub-view + l = m.tolist() + k = 2 * len(self._source) + + for chi in self._source: + if chi in l: + self.assertEqual(m.index(chi), l.index(chi)) + else: + self.assertRaises(ValueError, m.index, chi) + + for start, stop in product(range(-k, k), range(-k, k)): + index = -1 + try: + index = l.index(chi, start, stop) + except ValueError: + pass + + if index == -1: + self.assertRaises(ValueError, m.index, chi, start, stop) + else: + self.assertEqual(m.index(chi, start, stop), index) + def test_iter(self): for tp in self._types: b = tp(self._source) m = self._view(b) self.assertEqual(list(m), [m[i] for i in range(len(m))]) + def test_count(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) + l = m.tolist() + for ch in list(m): + self.assertEqual(m.count(ch), l.count(ch)) + + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # may be sliced + l = m.tolist() + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), l.count(ord('a'))) + self.assertEqual(m.count(ord('b')), l.count(ord('b'))) + self.assertEqual(m.count(ord('c')), l.count(ord('c'))) + def test_setitem_readonly(self): if not self.ro_type: self.skipTest("no read-only type to test") @@ -191,16 +239,12 @@ def check_attributes_with_type(self, tp): self.assertEqual(m.suboffsets, ()) return m - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attributes_readonly(self): if not self.ro_type: self.skipTest("no read-only type to test") m = self.check_attributes_with_type(self.ro_type) self.assertEqual(m.readonly, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attributes_writable(self): if not self.rw_type: self.skipTest("no writable type to test") @@ -231,8 +275,6 @@ def __init__(self, base): self.m = memoryview(base) class MySource(tp): pass - class MyObject: - pass # Create a reference cycle through a memoryview object. # This exercises mbuf_clear(). @@ -345,6 +387,21 @@ def test_hash_writable(self): m = self._view(b) self.assertRaises(ValueError, hash, m) + @unittest.expectedFailure # TODO: RUSTPYTHON; re-entrant buffer release not detected + def test_hash_use_after_free(self): + # Prevent crash in memoryview(v).__hash__ with re-entrant v.__hash__. + # Regression test for https://github.com/python/cpython/issues/142664. + class E(array.array): + def __hash__(self): + mv.release() + self.clear() + return 123 + + v = E('B', b'A' * 4096) + mv = memoryview(v).toreadonly() # must be read-only for hash() + self.assertRaises(BufferError, hash, mv) + self.assertRaises(BufferError, mv.__hash__) + def test_weakref(self): # Check memoryviews are weakrefable for tp in self._types: @@ -400,6 +457,21 @@ def test_issue22668(self): self.assertEqual(c.format, "H") self.assertEqual(d.format, "H") + @unittest.expectedFailure # TODO: RUSTPYTHON; re-entrant buffer release not detected + def test_hex_use_after_free(self): + # Prevent UAF in memoryview.hex(sep) with re-entrant sep.__len__. + # Regression test for https://github.com/python/cpython/issues/143195. + ba = bytearray(b'A' * 1024) + mv = memoryview(ba) + + class S(bytes): + def __len__(self): + mv.release() + ba.clear() + return 1 + + self.assertRaises(BufferError, mv.hex, S(b':')) + # Variations on source objects for the buffer: bytes-like objects, then arrays # with itemsize > 1. @@ -439,6 +511,18 @@ def _view(self, obj): def _check_contents(self, tp, obj, contents): self.assertEqual(obj, tp(contents)) + def test_count(self): + super().test_count() + for tp in self._types: + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # should not be sliced + self.assertEqual(len(b), len(m)) + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), 5) + self.assertEqual(m.count(ord('b')), 0) + self.assertEqual(m.count(ord('c')), 3) + + class BaseMemorySliceTests: source_bytes = b"XabcdefY" @@ -472,11 +556,6 @@ def _check_contents(self, tp, obj, contents): class BytesMemoryviewTest(unittest.TestCase, BaseMemoryviewTests, BaseBytesMemoryTests): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gc(self): - super().test_gc() - def test_constructor(self): for tp in self._types: ob = tp(self._source) @@ -487,6 +566,10 @@ def test_constructor(self): self.assertRaises(TypeError, memoryview, argument=ob) self.assertRaises(TypeError, memoryview, ob, argument=True) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : <test.test_memoryview.MyObject object at 0x84ecb1920> + def test_gc(self): + return super().test_gc() + class ArrayMemoryviewTest(unittest.TestCase, BaseMemoryviewTests, BaseArrayMemoryTests): @@ -501,24 +584,24 @@ def test_array_assign(self): class BytesMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseBytesMemoryTests): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gc(self): - super().test_gc() pass + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : <test.test_memoryview.MyObject object at 0x84ecb13e0> + def test_gc(self): + return super().test_gc() + class ArrayMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseArrayMemoryTests): pass class BytesMemorySliceSliceTest(unittest.TestCase, BaseMemorySliceSliceTests, BaseBytesMemoryTests): - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_gc(self): - super().test_gc() pass + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : <test.test_memoryview.MyObject object at 0x84ddca1c0> + def test_gc(self): + return super().test_gc() + class ArrayMemorySliceSliceTest(unittest.TestCase, BaseMemorySliceSliceTests, BaseArrayMemoryTests): pass @@ -544,6 +627,14 @@ def test_ctypes_cast(self): m[2:] = memoryview(p6).cast(format)[2:] self.assertEqual(d.value, 0.6) + def test_half_float(self): + half_data = struct.pack('eee', 0.0, -1.5, 1.5) + float_data = struct.pack('fff', 0.0, -1.5, 1.5) + half_view = memoryview(half_data).cast('e') + float_view = memoryview(float_data).cast('f') + self.assertEqual(half_view.nbytes * 2, float_view.nbytes) + self.assertListEqual(half_view.tolist(), float_view.tolist()) + def test_memoryview_hex(self): # Issue #9951: memoryview.hex() segfaults with non-contiguous buffers. x = b'0' * 200000 @@ -551,6 +642,26 @@ def test_memoryview_hex(self): m2 = m1[::-1] self.assertEqual(m2.hex(), '30' * 200000) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument sep + def test_memoryview_hex_separator(self): + x = bytes(range(97, 102)) + m1 = memoryview(x) + m2 = m1[::-1] + self.assertEqual(m2.hex(':'), '65:64:63:62:61') + self.assertEqual(m2.hex(':', 2), '65:6463:6261') + self.assertEqual(m2.hex(':', -2), '6564:6362:61') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=2), '65:6463:6261') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=-2), '6564:6362:61') + for bytes_per_sep in 5, -5, 2**31-1, -(2**31-1): + with self.subTest(bytes_per_sep=bytes_per_sep): + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + with self.subTest(bytes_per_sep=bytes_per_sep): + try: + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + except OverflowError: + pass + def test_copy(self): m = memoryview(b'abc') with self.assertRaises(TypeError): @@ -562,8 +673,7 @@ def test_pickle(self): with self.assertRaises(TypeError): pickle.dumps(m, proto) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised def test_use_released_memory(self): # gh-92888: Previously it was possible to use a memoryview even after # backing buffer is freed in certain cases. This tests that those @@ -666,5 +776,56 @@ def __bool__(self): m[0] = MyBool() self.assertEqual(ba[:8], b'\0'*8) + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'memoryview' object has no attribute '__buffer__' + def test_buffer_reference_loop(self): + m = memoryview(b'abc').__buffer__(0) + o = MyObject() + o.m = m + o.o = o + wr = weakref.ref(o) + del m, o + gc.collect() + self.assertIsNone(wr()) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'pickle' has no attribute 'PickleBuffer' + def test_picklebuffer_reference_loop(self): + pb = pickle.PickleBuffer(memoryview(b'abc')) + o = MyObject() + o.pb = pb + o.o = o + wr = weakref.ref(o) + del pb, o + gc.collect() + self.assertIsNone(wr()) + + +@threading_helper.requires_working_threading() +@support.requires_resource("cpu") +class RacingTest(unittest.TestCase): + def test_racing_getbuf_and_releasebuf(self): + """Repeatly access the memoryview for racing.""" + try: + from multiprocessing.managers import SharedMemoryManager + except ImportError: + self.skipTest("Test requires multiprocessing") + from threading import Thread, Event + + start = Event() + with SharedMemoryManager() as smm: + obj = smm.ShareableList(range(100)) + def test(): + # Issue gh-127085, the `ShareableList.count` is just a + # convenient way to mess the `exports` counter of `memoryview`, + # this issue has no direct relation with `ShareableList`. + start.wait(support.SHORT_TIMEOUT) + for i in range(10): + obj.count(1) + threads = [Thread(target=test) for _ in range(10)] + with threading_helper.start_threads(threads): + start.set() + + del obj + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py new file mode 100644 index 00000000000..1707df9075a --- /dev/null +++ b/Lib/test/test_metaclass.py @@ -0,0 +1,321 @@ +import doctest +import unittest + + +doctests = """ + +Basic class construction. + + >>> class C: + ... def meth(self): print("Hello") + ... + >>> C.__class__ is type + True + >>> a = C() + >>> a.__class__ is C + True + >>> a.meth() + Hello + >>> + +Use *args notation for the bases. + + >>> class A: pass + >>> class B: pass + >>> bases = (A, B) + >>> class C(*bases): pass + >>> C.__bases__ == bases + True + >>> + +Use a trivial metaclass. + + >>> class M(type): + ... pass + ... + >>> class C(metaclass=M): + ... def meth(self): print("Hello") + ... + >>> C.__class__ is M + True + >>> a = C() + >>> a.__class__ is C + True + >>> a.meth() + Hello + >>> + +Use **kwds notation for the metaclass keyword. + + >>> kwds = {'metaclass': M} + >>> class C(**kwds): pass + ... + >>> C.__class__ is M + True + >>> a = C() + >>> a.__class__ is C + True + >>> + +Use a metaclass with a __prepare__ static method. + + >>> class M(type): + ... @staticmethod + ... def __prepare__(*args, **kwds): + ... print("Prepare called:", args, kwds) + ... return dict() + ... def __new__(cls, name, bases, namespace, **kwds): + ... print("New called:", kwds) + ... return type.__new__(cls, name, bases, namespace) + ... def __init__(cls, *args, **kwds): + ... pass + ... + >>> class C(metaclass=M): + ... def meth(self): print("Hello") + ... + Prepare called: ('C', ()) {} + New called: {} + >>> + +Also pass another keyword. + + >>> class C(object, metaclass=M, other="haha"): + ... pass + ... + Prepare called: ('C', (<class 'object'>,)) {'other': 'haha'} + New called: {'other': 'haha'} + >>> C.__class__ is M + True + >>> C.__bases__ == (object,) + True + >>> a = C() + >>> a.__class__ is C + True + >>> + +Check that build_class doesn't mutate the kwds dict. + + >>> kwds = {'metaclass': type} + >>> class C(**kwds): pass + ... + >>> kwds == {'metaclass': type} + True + >>> + +Use various combinations of explicit keywords and **kwds. + + >>> bases = (object,) + >>> kwds = {'metaclass': M, 'other': 'haha'} + >>> class C(*bases, **kwds): pass + ... + Prepare called: ('C', (<class 'object'>,)) {'other': 'haha'} + New called: {'other': 'haha'} + >>> C.__class__ is M + True + >>> C.__bases__ == (object,) + True + >>> class B: pass + >>> kwds = {'other': 'haha'} + >>> class C(B, metaclass=M, *bases, **kwds): pass + ... + Prepare called: ('C', (<class 'test.test_metaclass.B'>, <class 'object'>)) {'other': 'haha'} + New called: {'other': 'haha'} + >>> C.__class__ is M + True + >>> C.__bases__ == (B, object) + True + >>> + +Check for duplicate keywords. + + # TODO: RUSTPYTHON + >>> class C(metaclass=type, metaclass=type): pass # doctest: +SKIP + ... + Traceback (most recent call last): + [...] + SyntaxError: keyword argument repeated: metaclass + >>> + +Another way. + + >>> kwds = {'metaclass': type} + + # TODO: RUSTPYTHON + >>> class C(metaclass=type, **kwds): pass # doctest: +SKIP + ... + Traceback (most recent call last): + [...] + TypeError: __build_class__() got multiple values for keyword argument 'metaclass' + >>> + +Use a __prepare__ method that returns an instrumented dict. + + >>> class LoggingDict(dict): + ... def __setitem__(self, key, value): + ... print("d[%r] = %r" % (key, value)) + ... dict.__setitem__(self, key, value) + ... + >>> class Meta(type): + ... @staticmethod + ... def __prepare__(name, bases): + ... return LoggingDict() + ... + + # TODO: RUSTPYTHON + >>> class C(metaclass=Meta): # doctest: +SKIP + ... foo = 2+2 + ... foo = 42 + ... bar = 123 + ... + d['__module__'] = 'test.test_metaclass' + d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 + d['foo'] = 4 + d['foo'] = 42 + d['bar'] = 123 + d['__static_attributes__'] = () + >>> + +Use a metaclass that doesn't derive from type. + + >>> def meta(name, bases, namespace, **kwds): + ... print("meta:", name, bases) + ... print("ns:", sorted(namespace.items())) + ... print("kw:", sorted(kwds.items())) + ... return namespace + ... + + # TODO: RUSTPYTHON + >>> class C(metaclass=meta): # doctest: +SKIP + ... a = 42 + ... b = 24 + ... + meta: C () + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] + kw: [] + + # TODO: RUSTPYTHON + >>> type(C) is dict # doctest: +SKIP + True + + # TODO: RUSTPYTHON + >>> print(sorted(C.items())) # doctest: +SKIP + [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] + >>> + +And again, with a __prepare__ attribute. + + >>> def prepare(name, bases, **kwds): + ... print("prepare:", name, bases, sorted(kwds.items())) + ... return LoggingDict() + ... + >>> meta.__prepare__ = prepare + + # TODO: RUSTPYTHON + >>> class C(metaclass=meta, other="booh"): # doctest: +SKIP + ... a = 1 + ... a = 2 + ... b = 3 + ... + prepare: C () [('other', 'booh')] + d['__module__'] = 'test.test_metaclass' + d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 + d['a'] = 1 + d['a'] = 2 + d['b'] = 3 + d['__static_attributes__'] = () + meta: C () + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] + kw: [('other', 'booh')] + >>> + +The default metaclass must define a __prepare__() method. + + >>> type.__prepare__() + {} + >>> + +Make sure it works with subclassing. + + >>> class M(type): + ... @classmethod + ... def __prepare__(cls, *args, **kwds): + ... d = super().__prepare__(*args, **kwds) + ... d["hello"] = 42 + ... return d + ... + >>> class C(metaclass=M): + ... print(hello) + ... + 42 + >>> print(C.hello) + 42 + >>> + +Test failures in looking up the __prepare__ method work. + >>> class ObscureException(Exception): + ... pass + >>> class FailDescr: + ... def __get__(self, instance, owner): + ... raise ObscureException + >>> class Meta(type): + ... __prepare__ = FailDescr() + >>> class X(metaclass=Meta): + ... pass + Traceback (most recent call last): + [...] + test.test_metaclass.ObscureException + +Test setting attributes with a non-base type in mro() (gh-127773). + + >>> class Base: + ... value = 1 + ... + >>> class Meta(type): + ... def mro(cls): + ... return (cls, Base, object) + ... + >>> class WeirdClass(metaclass=Meta): + ... pass + ... + >>> Base.value + 1 + + # TODO: RUSTPYTHON; AttributeError: type object 'WeirdClass' has no attribute 'value' + >>> WeirdClass.value # doctest: +SKIP + 1 + >>> Base.value = 2 + >>> Base.value + 2 + + # TODO: RUSTPYTHON; AttributeError: type object 'WeirdClass' has no attribute 'value' + >>> WeirdClass.value # doctest: +SKIP + 2 + >>> Base.value = 3 + >>> Base.value + 3 + + # TODO: RUSTPYTHON; AttributeError: type object 'WeirdClass' has no attribute 'value' + >>> WeirdClass.value # doctest: +SKIP + 3 + +""" + +import sys + +# Trace function introduces __locals__ which causes various tests to fail. +if hasattr(sys, 'gettrace') and sys.gettrace(): + __test__ = {} +else: + __test__ = {'doctests' : doctests} + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests + + +if __name__ == "__main__": + # set __name__ to match doctest expectations + __name__ = "test.test_metaclass" + unittest.main() diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 23092ffd0f3..c1806b1c133 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -1,12 +1,18 @@ import io -import locale import mimetypes -import pathlib +import os +import shlex import sys -import unittest - -from test import support +import unittest.mock from platform import win32_edition +from test import support +from test.support import cpython_only, force_not_colorized, os_helper +from test.support.import_helper import ensure_lazy_imports + +try: + import _winapi +except ImportError: + _winapi = None def setUpModule(): @@ -28,15 +34,30 @@ class MimeTypesTestCase(unittest.TestCase): def setUp(self): self.db = mimetypes.MimeTypes() + def test_case_sensitivity(self): + eq = self.assertEqual + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) + def test_default_data(self): eq = self.assertEqual - eq(self.db.guess_type("foo.html"), ("text/html", None)) - eq(self.db.guess_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual @@ -50,12 +71,10 @@ def test_file_parsing(self): eq = self.assertEqual sio = io.StringIO("x-application/x-unittest pyunit\n") self.db.readfp(sio) - eq(self.db.guess_type("foo.pyunit"), + eq(self.db.guess_file_type("foo.pyunit"), ("x-application/x-unittest", None)) eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_read_mime_types(self): eq = self.assertEqual @@ -64,32 +83,40 @@ def test_read_mime_types(self): with os_helper.temp_dir() as directory: data = "x-application/x-unittest pyunit\n" - file = pathlib.Path(directory, "sample.mimetype") - file.write_text(data) + file = os.path.join(directory, "sample.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) mime_dict = mimetypes.read_mime_types(file) eq(mime_dict[".pyunit"], "x-application/x-unittest") + data = "x-application/x-unittest2 pyunit2\n" + file = os.path.join(directory, "sample2.mimetype") + with open(file, 'w', encoding="utf-8") as f: + f.write(data) + mime_dict = mimetypes.read_mime_types(os_helper.FakePath(file)) + eq(mime_dict[".pyunit2"], "x-application/x-unittest2") + # bpo-41048: read_mime_types should read the rule file with 'utf-8' encoding. # Not with locale encoding. _bootlocale has been imported because io.open(...) # uses it. - with os_helper.temp_dir() as directory: - data = "application/no-mans-land Fran\u00E7ais" - file = pathlib.Path(directory, "sample.mimetype") - file.write_text(data, encoding='utf-8') - import _bootlocale - with support.swap_attr(_bootlocale, 'getpreferredencoding', lambda do_setlocale=True: 'ASCII'): - mime_dict = mimetypes.read_mime_types(file) - eq(mime_dict[".Français"], "application/no-mans-land") + data = "application/no-mans-land Fran\u00E7ais" + filename = "filename" + fp = io.StringIO(data) + with unittest.mock.patch.object(mimetypes, 'open', + return_value=fp) as mock_open: + mime_dict = mimetypes.read_mime_types(filename) + mock_open.assert_called_with(filename, encoding='utf-8') + eq(mime_dict[".Français"], "application/no-mans-land") def test_non_standard_types(self): eq = self.assertEqual # First try strict - eq(self.db.guess_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict - eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') @@ -99,37 +126,77 @@ def test_filename_with_url_delimiters(self): # compared to when interpreted as filename because of the semicolon. eq = self.assertEqual gzip_expected = ('application/x-tar', 'gzip') - eq(self.db.guess_type(";1.tar.gz"), gzip_expected) - eq(self.db.guess_type("?1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1.tar.gz"), gzip_expected) - eq(self.db.guess_type("#1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";1#.tar.gz"), gzip_expected) - eq(self.db.guess_type(";&1=123;?.tar.gz"), gzip_expected) - eq(self.db.guess_type("?k1=v1&k2=v2.tar.gz"), gzip_expected) + for name in ( + ';1.tar.gz', + '?1.tar.gz', + '#1.tar.gz', + '#1#.tar.gz', + ';1#.tar.gz', + ';&1=123;?.tar.gz', + '?k1=v1&k2=v2.tar.gz', + ): + for prefix in ('', '/', '\\', + 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', + '//share/server/', '\\\\share\\server\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) + eq(self.db.guess_type(path), gzip_expected) + expected = (None, None) if os.name == 'nt' else gzip_expected + for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) + eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') + result = self.db.guess_type('http://host.html') + msg = 'URL only has a host name, not a file' + self.assertSequenceEqual(result, (None, None), msg) + result = self.db.guess_type('http://example.com/host.html') + msg = 'Should be text/html' + self.assertSequenceEqual(result, ('text/html', None), msg) + result = self.db.guess_type('http://example.com/host.html#x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + result = self.db.guess_type('http://example.com/host.html?q=x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + def test_guess_all_types(self): - eq = self.assertEqual - unless = self.assertTrue # First try strict. Use a set here for testing the results because if # test_urllib2 is run before test_mimetypes, global state is modified # such that the 'all' set will have more items in it. - all = set(self.db.guess_all_extensions('text/plain', strict=True)) - unless(all >= set(['.bat', '.c', '.h', '.ksh', '.pl', '.txt'])) + all = self.db.guess_all_extensions('text/plain', strict=True) + self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) + self.assertEqual(len(set(all)), len(all)) # no duplicates # And now non-strict all = self.db.guess_all_extensions('image/jpg', strict=False) - all.sort() - eq(all, ['.jpg']) + self.assertEqual(all, ['.jpg']) # And now for no hits all = self.db.guess_all_extensions('image/jpg', strict=True) - eq(all, []) + self.assertEqual(all, []) + # And now for type existing in both strict and non-strict mappings. + self.db.add_type('test-type', '.strict-ext') + self.db.add_type('test-type', '.non-strict-ext', strict=False) + all = self.db.guess_all_extensions('test-type', strict=False) + self.assertEqual(all, ['.strict-ext', '.non-strict-ext']) + all = self.db.guess_all_extensions('test-type') + self.assertEqual(all, ['.strict-ext']) + # Test that changing the result list does not affect the global state + all.append('.no-such-ext') + all = self.db.guess_all_extensions('test-type') + self.assertNotIn('.no-such-ext', all) def test_encoding(self): - getpreferredencoding = locale.getpreferredencoding - self.addCleanup(setattr, locale, 'getpreferredencoding', - getpreferredencoding) - locale.getpreferredencoding = lambda: 'ascii' - filename = support.findfile("mime.types") mimes = mimetypes.MimeTypes([filename]) exts = mimes.guess_all_extensions('application/vnd.geocube+xml', @@ -146,29 +213,110 @@ def test_init_reinitializes(self): # Poison should be gone. self.assertEqual(mimetypes.guess_extension('foo/bar'), None) + @unittest.skipIf(sys.platform.startswith("win"), "Non-Windows only") + def test_guess_known_extensions(self): + # Issue 37529 + # The test fails on Windows because Windows adds mime types from the Registry + # and that creates some duplicates. + from mimetypes import types_map + for v in types_map.values(): + self.assertIsNotNone(mimetypes.guess_extension(v)) + def test_preferred_extension(self): def check_extensions(): - self.assertEqual(mimetypes.guess_extension('application/octet-stream'), '.bin') - self.assertEqual(mimetypes.guess_extension('application/postscript'), '.ps') - self.assertEqual(mimetypes.guess_extension('application/vnd.apple.mpegurl'), '.m3u') - self.assertEqual(mimetypes.guess_extension('application/vnd.ms-excel'), '.xls') - self.assertEqual(mimetypes.guess_extension('application/vnd.ms-powerpoint'), '.ppt') - self.assertEqual(mimetypes.guess_extension('application/x-texinfo'), '.texi') - self.assertEqual(mimetypes.guess_extension('application/x-troff'), '.roff') - self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') - self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') - self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') - self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') - self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') - self.assertEqual(mimetypes.guess_extension('text/html'), '.html') - self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') - self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') - self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') + for mime_type, ext in ( + ("application/epub+zip", ".epub"), + ("application/octet-stream", ".bin"), + ("application/gzip", ".gz"), + ("application/ogg", ".ogx"), + ("application/postscript", ".ps"), + ("application/vnd.apple.mpegurl", ".m3u"), + ("application/vnd.ms-excel", ".xls"), + ("application/vnd.ms-fontobject", ".eot"), + ("application/vnd.ms-powerpoint", ".ppt"), + ("application/vnd.oasis.opendocument.graphics", ".odg"), + ("application/vnd.oasis.opendocument.presentation", ".odp"), + ("application/vnd.oasis.opendocument.spreadsheet", ".ods"), + ("application/vnd.oasis.opendocument.text", ".odt"), + ("application/vnd.openxmlformats-officedocument.presentationml.presentation", ".pptx"), + ("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ".xlsx"), + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".docx"), + ("application/vnd.rar", ".rar"), + ("application/x-7z-compressed", ".7z"), + ("application/x-debian-package", ".deb"), + ("application/x-httpd-php", ".php"), + ("application/x-rpm", ".rpm"), + ("application/x-texinfo", ".texi"), + ("application/x-troff", ".roff"), + ("application/xml", ".xsl"), + ("application/yaml", ".yaml"), + ("audio/flac", ".flac"), + ("audio/matroska", ".mka"), + ("audio/mp4", ".m4a"), + ("audio/mpeg", ".mp3"), + ("audio/ogg", ".ogg"), + ("audio/vnd.wave", ".wav"), + ("audio/webm", ".weba"), + ("font/otf", ".otf"), + ("font/ttf", ".ttf"), + ("font/woff", ".woff"), + ("font/woff2", ".woff2"), + ("image/avif", ".avif"), + ("image/emf", ".emf"), + ("image/fits", ".fits"), + ("image/g3fax", ".g3"), + ("image/jp2", ".jp2"), + ("image/jpeg", ".jpg"), + ("image/jpm", ".jpm"), + ("image/t38", ".t38"), + ("image/tiff", ".tiff"), + ("image/tiff-fx", ".tfx"), + ("image/webp", ".webp"), + ("image/wmf", ".wmf"), + ("message/rfc822", ".eml"), + ("model/gltf+json", ".gltf"), + ("model/gltf-binary", ".glb"), + ("model/stl", ".stl"), + ("text/html", ".html"), + ("text/plain", ".txt"), + ("text/rtf", ".rtf"), + ("text/x-rst", ".rst"), + ("video/matroska", ".mkv"), + ("video/matroska-3d", ".mk3d"), + ("video/mpeg", ".mpeg"), + ("video/ogg", ".ogv"), + ("video/quicktime", ".mov"), + ("video/vnd.avi", ".avi"), + ("video/x-m4v", ".m4v"), + ("video/x-ms-wmv", ".wmv"), + ): + with self.subTest(mime_type=mime_type, ext=ext): + self.assertEqual(mimetypes.guess_extension(mime_type), ext) check_extensions() mimetypes.init() check_extensions() + def test_guess_file_type(self): + def check_file_type(): + for mime_type, ext in ( + ("application/yaml", ".yaml"), + ("application/yaml", ".yml"), + ("audio/mpeg", ".mp2"), + ("audio/mpeg", ".mp3"), + ("video/mpeg", ".m1v"), + ("video/mpeg", ".mpe"), + ("video/mpeg", ".mpeg"), + ("video/mpeg", ".mpg"), + ): + with self.subTest(mime_type=mime_type, ext=ext): + result, _ = mimetypes.guess_file_type(f"filename{ext}") + self.assertEqual(result, mime_type) + + check_file_type() + mimetypes.init() + check_file_type() + def test_init_stability(self): mimetypes.init() @@ -189,27 +337,59 @@ def test_init_stability(self): def test_path_like_ob(self): filename = "LICENSE.txt" - filepath = pathlib.Path(filename) - filepath_with_abs_dir = pathlib.Path('/dir/'+filename) - filepath_relative = pathlib.Path('../dir/'+filename) - path_dir = pathlib.Path('./') + filepath = os_helper.FakePath(filename) + filepath_with_abs_dir = os_helper.FakePath('/dir/'+filename) + filepath_relative = os_helper.FakePath('../dir/'+filename) + path_dir = os_helper.FakePath('./') - expected = self.db.guess_type(filename) + expected = self.db.guess_file_type(filename) + self.assertEqual(self.db.guess_file_type(filepath), expected) self.assertEqual(self.db.guess_type(filepath), expected) + self.assertEqual(self.db.guess_file_type( + filepath_with_abs_dir), expected) self.assertEqual(self.db.guess_type( filepath_with_abs_dir), expected) + self.assertEqual(self.db.guess_file_type(filepath_relative), expected) self.assertEqual(self.db.guess_type(filepath_relative), expected) + + self.assertEqual(self.db.guess_file_type(path_dir), (None, None)) self.assertEqual(self.db.guess_type(path_dir), (None, None)) + def test_bytes_path(self): + self.assertEqual(self.db.guess_file_type(b'foo.html'), + self.db.guess_file_type('foo.html')) + self.assertEqual(self.db.guess_file_type(b'foo.tar.gz'), + self.db.guess_file_type('foo.tar.gz')) + self.assertEqual(self.db.guess_file_type(b'foo.tgz'), + self.db.guess_file_type('foo.tgz')) + def test_keywords_args_api(self): + self.assertEqual(self.db.guess_file_type( + path="foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_type( - url="foo.html", strict=True), ("text/html", None)) + url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( type='image/jpg', strict=True), []) self.assertEqual(self.db.guess_extension( type='image/jpg', strict=False), '.jpg') + def test_added_types_are_used(self): + mimetypes.add_type('testing/default-type', '') + mime_type, _ = mimetypes.guess_type('') + self.assertEqual(mime_type, 'testing/default-type') + + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, None) + + mimetypes.add_type('testing/type', '.myext') + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, 'testing/type') + + def test_add_type_with_undotted_extension_deprecated(self): + with self.assertWarns(DeprecationWarning): + mimetypes.add_type("testing/type", "undotted") + @unittest.skipUnless(sys.platform.startswith("win"), "Windows only") class Win32MimeTypesTestCase(unittest.TestCase): @@ -236,58 +416,94 @@ def test_registry_parsing(self): eq(self.db.guess_type("image.jpg"), ("image/jpeg", None)) eq(self.db.guess_type("image.png"), ("image/png", None)) + @unittest.skipIf(not hasattr(_winapi, "_mimetypes_read_windows_registry"), + "read_windows_registry accelerator unavailable") + def test_registry_accelerator(self): + from_accel = {} + from_reg = {} + _winapi._mimetypes_read_windows_registry( + lambda v, k: from_accel.setdefault(k, set()).add(v) + ) + mimetypes.MimeTypes._read_windows_registry( + lambda v, k: from_reg.setdefault(k, set()).add(v) + ) + self.assertEqual(list(from_reg), list(from_accel)) + for k in from_reg: + self.assertEqual(from_reg[k], from_accel[k]) + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, mimetypes) - -class MimetypesCliTestCase(unittest.TestCase): - - def mimetypes_cmd(self, *args, **kwargs): - support.patch(self, sys, "argv", [sys.executable, *args]) - with support.captured_stdout() as output: - mimetypes._main() - return output.getvalue().strip() - - def test_help_option(self): - support.patch(self, sys, "argv", [sys.executable, "-h"]) - with support.captured_stdout() as output: - with self.assertRaises(SystemExit) as cm: - mimetypes._main() - - self.assertIn("Usage: mimetypes.py", output.getvalue()) - self.assertEqual(cm.exception.code, 0) - - def test_invalid_option(self): - support.patch(self, sys, "argv", [sys.executable, "--invalid"]) - with support.captured_stdout() as output: - with self.assertRaises(SystemExit) as cm: - mimetypes._main() - - self.assertIn("Usage: mimetypes.py", output.getvalue()) - self.assertEqual(cm.exception.code, 1) - - def test_guess_extension(self): - eq = self.assertEqual - - extension = self.mimetypes_cmd("-l", "-e", "image/jpg") - eq(extension, ".jpg") - - extension = self.mimetypes_cmd("-e", "image/jpg") - eq(extension, "I don't know anything about type image/jpg") - - extension = self.mimetypes_cmd("-e", "image/jpeg") - eq(extension, ".jpg") - - def test_guess_type(self): - eq = self.assertEqual - - type_info = self.mimetypes_cmd("-l", "foo.pic") - eq(type_info, "type: image/pict encoding: None") - - type_info = self.mimetypes_cmd("foo.pic") - eq(type_info, "I don't know anything about type foo.pic") + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("mimetypes", {"os", "posixpath", "urllib.parse", "argparse"}) + + +class CommandLineTest(unittest.TestCase): + @force_not_colorized + def test_parse_args(self): + args, help_text = mimetypes._parse_args("-h") + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = mimetypes._parse_args("--invalid") + self.assertTrue(help_text.startswith("usage: ")) + + args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpg")) + self.assertTrue(args.extension) + self.assertTrue(args.lenient) + self.assertEqual(args.type, ["image/jpg"]) + + args, _ = mimetypes._parse_args(shlex.split("-e image/jpg")) + self.assertTrue(args.extension) + self.assertFalse(args.lenient) + self.assertEqual(args.type, ["image/jpg"]) + + args, _ = mimetypes._parse_args(shlex.split("-l foo.webp")) + self.assertFalse(args.extension) + self.assertTrue(args.lenient) + self.assertEqual(args.type, ["foo.webp"]) + + args, _ = mimetypes._parse_args(shlex.split("foo.pic")) + self.assertFalse(args.extension) + self.assertFalse(args.lenient) + self.assertEqual(args.type, ["foo.pic"]) + + def test_multiple_inputs(self): + result = "\n".join(mimetypes._main(shlex.split("foo.pdf foo.png"))) + self.assertEqual( + result, + "type: application/pdf encoding: None\n" + "type: image/png encoding: None" + ) + + def test_multiple_inputs_error(self): + result = "\n".join(mimetypes._main(shlex.split("foo.pdf foo.bar_ext"))) + self.assertEqual( + result, + "type: application/pdf encoding: None\n" + "error: media type unknown for foo.bar_ext" + ) + + + def test_invocation(self): + for command, expected in [ + ("-l -e image/jpg", ".jpg"), + ("-e image/jpeg", ".jpg"), + ("-l foo.webp", "type: image/webp encoding: None"), + ]: + result = "\n".join(mimetypes._main(shlex.split(command))) + self.assertEqual(result, expected) + + def test_invocation_error(self): + for command, expected in [ + ("-e image/jpg", "error: unknown type image/jpg"), + ("foo.bar_ext", "error: media type unknown for foo.bar_ext"), + ]: + with self.subTest(command=command): + result = "\n".join(mimetypes._main(shlex.split(command))) + self.assertEqual(result, expected) if __name__ == "__main__": diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py new file mode 100644 index 00000000000..26fd366355f --- /dev/null +++ b/Lib/test/test_minidom.py @@ -0,0 +1,1813 @@ +# test for xml.dom.minidom + +import copy +import pickle +import io +from test import support +import unittest + +import xml.dom.minidom + +from xml.dom.minidom import parse, Attr, Node, Document, Element, parseString +from xml.dom.minidom import getDOMImplementation +from xml.parsers.expat import ExpatError + + +tstfile = support.findfile("test.xml", subdir="xmltestdata") +sample = ("<?xml version='1.0' encoding='us-ascii'?>\n" + "<!DOCTYPE doc PUBLIC 'http://xml.python.org/public'" + " 'http://xml.python.org/system' [\n" + " <!ELEMENT e EMPTY>\n" + " <!ENTITY ent SYSTEM 'http://xml.python.org/entity'>\n" + "]><doc attr='value'> text\n" + "<?pi sample?> <!-- comment --> <e/> </doc>") + +# The tests of DocumentType importing use these helpers to construct +# the documents to work with, since not all DOM builders actually +# create the DocumentType nodes. +def create_doc_without_doctype(doctype=None): + return getDOMImplementation().createDocument(None, "doc", doctype) + +def create_nonempty_doctype(): + doctype = getDOMImplementation().createDocumentType("doc", None, None) + doctype.entities._seq = [] + doctype.notations._seq = [] + notation = xml.dom.minidom.Notation("my-notation", None, + "http://xml.python.org/notations/my") + doctype.notations._seq.append(notation) + entity = xml.dom.minidom.Entity("my-entity", None, + "http://xml.python.org/entities/my", + "my-notation") + entity.version = "1.0" + entity.encoding = "utf-8" + entity.actualEncoding = "us-ascii" + doctype.entities._seq.append(entity) + return doctype + +def create_doc_with_doctype(): + doctype = create_nonempty_doctype() + doc = create_doc_without_doctype(doctype) + doctype.entities.item(0).ownerDocument = doc + doctype.notations.item(0).ownerDocument = doc + return doc + +class MinidomTest(unittest.TestCase): + def confirm(self, test, testname = "Test"): + self.assertTrue(test, testname) + + def checkWholeText(self, node, s): + t = node.wholeText + self.assertEqual(t, s, "looking for %r, found %r" % (s, t)) + + def testDocumentAsyncAttr(self): + doc = Document() + self.assertFalse(doc.async_) + self.assertFalse(Document.async_) + + def testParseFromBinaryFile(self): + with open(tstfile, 'rb') as file: + dom = parse(file) + dom.unlink() + self.assertIsInstance(dom, Document) + + def testParseFromTextFile(self): + with open(tstfile, 'r', encoding='iso-8859-1') as file: + dom = parse(file) + dom.unlink() + self.assertIsInstance(dom, Document) + + def testAttrModeSetsParamsAsAttrs(self): + attr = Attr("qName", "namespaceURI", "localName", "prefix") + self.assertEqual(attr.name, "qName") + self.assertEqual(attr.namespaceURI, "namespaceURI") + self.assertEqual(attr.prefix, "prefix") + self.assertEqual(attr.localName, "localName") + + def testAttrModeSetsNonOptionalAttrs(self): + attr = Attr("qName", "namespaceURI", None, "prefix") + self.assertEqual(attr.name, "qName") + self.assertEqual(attr.namespaceURI, "namespaceURI") + self.assertEqual(attr.prefix, "prefix") + self.assertEqual(attr.localName, attr.name) + + def testGetElementsByTagName(self): + dom = parse(tstfile) + self.assertEqual(dom.getElementsByTagName("LI"), + dom.documentElement.getElementsByTagName("LI")) + dom.unlink() + + def testInsertBefore(self): + dom = parseString("<doc><foo/></doc>") + root = dom.documentElement + elem = root.childNodes[0] + nelem = dom.createElement("element") + root.insertBefore(nelem, elem) + self.assertEqual(len(root.childNodes), 2) + self.assertEqual(root.childNodes.length, 2) + self.assertIs(root.childNodes[0], nelem) + self.assertIs(root.childNodes.item(0), nelem) + self.assertIs(root.childNodes[1], elem) + self.assertIs(root.childNodes.item(1), elem) + self.assertIs(root.firstChild, nelem) + self.assertIs(root.lastChild, elem) + self.assertEqual(root.toxml(), "<doc><element/><foo/></doc>") + nelem = dom.createElement("element") + root.insertBefore(nelem, None) + self.assertEqual(len(root.childNodes), 3) + self.assertEqual(root.childNodes.length, 3) + self.assertIs(root.childNodes[1], elem) + self.assertIs(root.childNodes.item(1), elem) + self.assertIs(root.childNodes[2], nelem) + self.assertIs(root.childNodes.item(2), nelem) + self.assertIs(root.lastChild, nelem) + self.assertIs(nelem.previousSibling, elem) + self.assertEqual(root.toxml(), "<doc><element/><foo/><element/></doc>") + nelem2 = dom.createElement("bar") + root.insertBefore(nelem2, nelem) + self.assertEqual(len(root.childNodes), 4) + self.assertEqual(root.childNodes.length, 4) + self.assertIs(root.childNodes[2], nelem2) + self.assertIs(root.childNodes.item(2), nelem2) + self.assertIs(root.childNodes[3], nelem) + self.assertIs(root.childNodes.item(3), nelem) + self.assertIs(nelem2.nextSibling, nelem) + self.assertIs(nelem.previousSibling, nelem2) + self.assertEqual(root.toxml(), + "<doc><element/><foo/><bar/><element/></doc>") + dom.unlink() + + def _create_fragment_test_nodes(self): + dom = parseString("<doc/>") + orig = dom.createTextNode("original") + c1 = dom.createTextNode("foo") + c2 = dom.createTextNode("bar") + c3 = dom.createTextNode("bat") + dom.documentElement.appendChild(orig) + frag = dom.createDocumentFragment() + frag.appendChild(c1) + frag.appendChild(c2) + frag.appendChild(c3) + return dom, orig, c1, c2, c3, frag + + def testInsertBeforeFragment(self): + dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() + dom.documentElement.insertBefore(frag, None) + self.assertTupleEqual(tuple(dom.documentElement.childNodes), + (orig, c1, c2, c3), + "insertBefore(<fragment>, None)") + frag.unlink() + dom.unlink() + + dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() + dom.documentElement.insertBefore(frag, orig) + self.assertTupleEqual(tuple(dom.documentElement.childNodes), + (c1, c2, c3, orig), + "insertBefore(<fragment>, orig)") + frag.unlink() + dom.unlink() + + def testAppendChild(self): + dom = parse(tstfile) + dom.documentElement.appendChild(dom.createComment("Hello")) + self.assertEqual(dom.documentElement.childNodes[-1].nodeName, "#comment") + self.assertEqual(dom.documentElement.childNodes[-1].data, "Hello") + dom.unlink() + + @support.requires_resource('cpu') + def testAppendChildNoQuadraticComplexity(self): + impl = getDOMImplementation() + + def work(n): + doc = impl.createDocument(None, "some_tag", None) + element = doc.documentElement + total_calls = 0 + + # Count attribute accesses as a proxy for work done + def getattribute_counter(self, attr): + nonlocal total_calls + total_calls += 1 + return object.__getattribute__(self, attr) + + with support.swap_attr(Element, "__getattribute__", getattribute_counter): + for _ in range(n): + child = doc.createElement("child") + element.appendChild(child) + element = child + return total_calls + + # Doubling N should not ~quadruple the work. + w1 = work(1024) + w2 = work(2048) + w3 = work(4096) + + self.assertGreater(w1, 0) + r1 = w2 / w1 + r2 = w3 / w2 + self.assertLess( + max(r1, r2), 3.2, + msg=f"Possible quadratic behavior: work={w1,w2,w3} ratios={r1,r2}" + ) + + def testSetAttributeNodeWithoutOwnerDocument(self): + # regression test for gh-142754 + elem = Element("test") + attr = Attr("id") + attr.value = "test-id" + elem.setAttributeNode(attr) + self.assertEqual(elem.getAttribute("id"), "test-id") + + def testAppendChildFragment(self): + dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() + dom.documentElement.appendChild(frag) + self.assertTupleEqual(tuple(dom.documentElement.childNodes), + (orig, c1, c2, c3), + "appendChild(<fragment>)") + frag.unlink() + dom.unlink() + + def testReplaceChildFragment(self): + dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() + dom.documentElement.replaceChild(frag, orig) + orig.unlink() + self.assertTupleEqual(tuple(dom.documentElement.childNodes), (c1, c2, c3), + "replaceChild(<fragment>)") + frag.unlink() + dom.unlink() + + def testLegalChildren(self): + dom = Document() + elem = dom.createElement('element') + text = dom.createTextNode('text') + self.assertRaises(xml.dom.HierarchyRequestErr, dom.appendChild, text) + + dom.appendChild(elem) + self.assertRaises(xml.dom.HierarchyRequestErr, dom.insertBefore, text, + elem) + self.assertRaises(xml.dom.HierarchyRequestErr, dom.replaceChild, text, + elem) + + nodemap = elem.attributes + self.assertRaises(xml.dom.HierarchyRequestErr, nodemap.setNamedItem, + text) + self.assertRaises(xml.dom.HierarchyRequestErr, nodemap.setNamedItemNS, + text) + + elem.appendChild(text) + dom.unlink() + + def testNamedNodeMapSetItem(self): + dom = Document() + elem = dom.createElement('element') + attrs = elem.attributes + attrs["foo"] = "bar" + a = attrs.item(0) + self.assertIs(a.ownerDocument, dom, + "NamedNodeMap.__setitem__() sets ownerDocument") + self.assertIs(a.ownerElement, elem, + "NamedNodeMap.__setitem__() sets ownerElement") + self.assertEqual(a.value, "bar", + "NamedNodeMap.__setitem__() sets value") + self.assertEqual(a.nodeValue, "bar", + "NamedNodeMap.__setitem__() sets nodeValue") + elem.unlink() + dom.unlink() + + def testNonZero(self): + dom = parse(tstfile) + self.assertTrue(dom) # should not be zero + dom.appendChild(dom.createComment("foo")) + self.assertFalse(dom.childNodes[-1].childNodes) + dom.unlink() + + def testUnlink(self): + dom = parse(tstfile) + self.assertTrue(dom.childNodes) + dom.unlink() + self.assertFalse(dom.childNodes) + + def testContext(self): + with parse(tstfile) as dom: + self.assertTrue(dom.childNodes) + self.assertFalse(dom.childNodes) + + def testElement(self): + dom = Document() + dom.appendChild(dom.createElement("abc")) + self.assertTrue(dom.documentElement) + dom.unlink() + + def testAAA(self): + dom = parseString("<abc/>") + el = dom.documentElement + el.setAttribute("spam", "jam2") + self.assertEqual(el.toxml(), '<abc spam="jam2"/>', "testAAA") + a = el.getAttributeNode("spam") + self.assertIs(a.ownerDocument, dom, + "setAttribute() sets ownerDocument") + self.assertIs(a.ownerElement, dom.documentElement, + "setAttribute() sets ownerElement") + dom.unlink() + + def testAAB(self): + dom = parseString("<abc/>") + el = dom.documentElement + el.setAttribute("spam", "jam") + el.setAttribute("spam", "jam2") + self.assertEqual(el.toxml(), '<abc spam="jam2"/>', "testAAB") + dom.unlink() + + def testAddAttr(self): + dom = Document() + child = dom.appendChild(dom.createElement("abc")) + + child.setAttribute("def", "ghi") + self.assertEqual(child.getAttribute("def"), "ghi") + self.assertEqual(child.attributes["def"].value, "ghi") + + child.setAttribute("jkl", "mno") + self.assertEqual(child.getAttribute("jkl"), "mno") + self.assertEqual(child.attributes["jkl"].value, "mno") + + self.assertEqual(len(child.attributes), 2) + + child.setAttribute("def", "newval") + self.assertEqual(child.getAttribute("def"), "newval") + self.assertEqual(child.attributes["def"].value, "newval") + + self.assertEqual(len(child.attributes), 2) + dom.unlink() + + def testDeleteAttr(self): + dom = Document() + child = dom.appendChild(dom.createElement("abc")) + + self.assertEqual(len(child.attributes), 0) + child.setAttribute("def", "ghi") + self.assertEqual(len(child.attributes), 1) + del child.attributes["def"] + self.assertEqual(len(child.attributes), 0) + dom.unlink() + + def testRemoveAttr(self): + dom = Document() + child = dom.appendChild(dom.createElement("abc")) + + child.setAttribute("def", "ghi") + self.assertEqual(len(child.attributes), 1) + self.assertRaises(xml.dom.NotFoundErr, child.removeAttribute, "foo") + child.removeAttribute("def") + self.assertEqual(len(child.attributes), 0) + dom.unlink() + + def testRemoveAttrNS(self): + dom = Document() + child = dom.appendChild( + dom.createElementNS("http://www.python.org", "python:abc")) + child.setAttributeNS("http://www.w3.org", "xmlns:python", + "http://www.python.org") + child.setAttributeNS("http://www.python.org", "python:abcattr", "foo") + self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNS, + "foo", "http://www.python.org") + self.assertEqual(len(child.attributes), 2) + child.removeAttributeNS("http://www.python.org", "abcattr") + self.assertEqual(len(child.attributes), 1) + dom.unlink() + + def testRemoveAttributeNode(self): + dom = Document() + child = dom.appendChild(dom.createElement("foo")) + child.setAttribute("spam", "jam") + self.assertEqual(len(child.attributes), 1) + node = child.getAttributeNode("spam") + self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNode, + None) + self.assertIs(node, child.removeAttributeNode(node)) + self.assertEqual(len(child.attributes), 0) + self.assertIsNone(child.getAttributeNode("spam")) + dom2 = Document() + child2 = dom2.appendChild(dom2.createElement("foo")) + node2 = child2.getAttributeNode("spam") + self.assertRaises(xml.dom.NotFoundErr, child2.removeAttributeNode, + node2) + dom.unlink() + + def testHasAttribute(self): + dom = Document() + child = dom.appendChild(dom.createElement("foo")) + child.setAttribute("spam", "jam") + self.assertTrue(child.hasAttribute("spam")) + + def testChangeAttr(self): + dom = parseString("<abc/>") + el = dom.documentElement + el.setAttribute("spam", "jam") + self.assertEqual(len(el.attributes), 1) + el.setAttribute("spam", "bam") + # Set this attribute to be an ID and make sure that doesn't change + # when changing the value: + el.setIdAttribute("spam") + self.assertEqual(len(el.attributes), 1) + self.assertEqual(el.attributes["spam"].value, "bam") + self.assertEqual(el.attributes["spam"].nodeValue, "bam") + self.assertEqual(el.getAttribute("spam"), "bam") + self.assertTrue(el.getAttributeNode("spam").isId) + el.attributes["spam"] = "ham" + self.assertEqual(len(el.attributes), 1) + self.assertEqual(el.attributes["spam"].value, "ham") + self.assertEqual(el.attributes["spam"].nodeValue, "ham") + self.assertEqual(el.getAttribute("spam"), "ham") + self.assertTrue(el.attributes["spam"].isId) + el.setAttribute("spam2", "bam") + self.assertEqual(len(el.attributes), 2) + self.assertEqual(el.attributes["spam"].value, "ham") + self.assertEqual(el.attributes["spam"].nodeValue, "ham") + self.assertEqual(el.getAttribute("spam"), "ham") + self.assertEqual(el.attributes["spam2"].value, "bam") + self.assertEqual(el.attributes["spam2"].nodeValue, "bam") + self.assertEqual(el.getAttribute("spam2"), "bam") + el.attributes["spam2"] = "bam2" + + self.assertEqual(len(el.attributes), 2) + self.assertEqual(el.attributes["spam"].value, "ham") + self.assertEqual(el.attributes["spam"].nodeValue, "ham") + self.assertEqual(el.getAttribute("spam"), "ham") + self.assertEqual(el.attributes["spam2"].value, "bam2") + self.assertEqual(el.attributes["spam2"].nodeValue, "bam2") + self.assertEqual(el.getAttribute("spam2"), "bam2") + dom.unlink() + + def testGetAttrList(self): + dom = parseString("<abc/>") + self.addCleanup(dom.unlink) + el = dom.documentElement + el.setAttribute("spam", "jam") + self.assertEqual(len(el.attributes.items()), 1) + el.setAttribute("foo", "bar") + items = el.attributes.items() + self.assertEqual(len(items), 2) + self.assertIn(('spam', 'jam'), items) + self.assertIn(('foo', 'bar'), items) + + def testGetAttrValues(self): + dom = parseString("<abc/>") + self.addCleanup(dom.unlink) + el = dom.documentElement + el.setAttribute("spam", "jam") + values = [x.value for x in el.attributes.values()] + self.assertIn("jam", values) + el.setAttribute("foo", "bar") + values = [x.value for x in el.attributes.values()] + self.assertIn("bar", values) + self.assertIn("jam", values) + + def testGetAttribute(self): + dom = Document() + child = dom.appendChild( + dom.createElementNS("http://www.python.org", "python:abc")) + self.assertEqual(child.getAttribute('missing'), '') + + def testGetAttributeNS(self): + dom = Document() + child = dom.appendChild( + dom.createElementNS("http://www.python.org", "python:abc")) + child.setAttributeNS("http://www.w3.org", "xmlns:python", + "http://www.python.org") + self.assertEqual(child.getAttributeNS("http://www.w3.org", "python"), + 'http://www.python.org') + self.assertEqual(child.getAttributeNS("http://www.w3.org", "other"), + '') + child2 = child.appendChild(dom.createElement('abc')) + self.assertEqual(child2.getAttributeNS("http://www.python.org", "missing"), + '') + + def testGetAttributeNode(self): pass + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testGetElementsByTagNameNS(self): + d="""<foo xmlns:minidom='http://pyxml.sf.net/minidom'> + <minidom:myelem/> + </foo>""" + dom = parseString(d) + elems = dom.getElementsByTagNameNS("http://pyxml.sf.net/minidom", + "myelem") + self.assertEqual(len(elems), 1) + self.assertEqual(elems[0].namespaceURI, "http://pyxml.sf.net/minidom") + self.assertEqual(elems[0].localName, "myelem") + self.assertEqual(elems[0].prefix, "minidom") + self.assertEqual(elems[0].tagName, "minidom:myelem") + self.assertEqual(elems[0].nodeName, "minidom:myelem") + dom.unlink() + + def get_empty_nodelist_from_elements_by_tagName_ns_helper(self, doc, nsuri, + lname): + nodelist = doc.getElementsByTagNameNS(nsuri, lname) + self.assertEqual(len(nodelist), 0) + + def testGetEmptyNodeListFromElementsByTagNameNS(self): + doc = parseString('<doc/>') + self.get_empty_nodelist_from_elements_by_tagName_ns_helper( + doc, 'http://xml.python.org/namespaces/a', 'localname') + self.get_empty_nodelist_from_elements_by_tagName_ns_helper( + doc, '*', 'splat') + self.get_empty_nodelist_from_elements_by_tagName_ns_helper( + doc, 'http://xml.python.org/namespaces/a', '*') + + doc = parseString('<doc xmlns="http://xml.python.org/splat"><e/></doc>') + self.get_empty_nodelist_from_elements_by_tagName_ns_helper( + doc, "http://xml.python.org/splat", "not-there") + self.get_empty_nodelist_from_elements_by_tagName_ns_helper( + doc, "*", "not-there") + self.get_empty_nodelist_from_elements_by_tagName_ns_helper( + doc, "http://somewhere.else.net/not-there", "e") + + def testElementReprAndStr(self): + dom = Document() + el = dom.appendChild(dom.createElement("abc")) + string1 = repr(el) + string2 = str(el) + self.assertEqual(string1, string2) + dom.unlink() + + def testElementReprAndStrUnicode(self): + dom = Document() + el = dom.appendChild(dom.createElement("abc")) + string1 = repr(el) + string2 = str(el) + self.assertEqual(string1, string2) + dom.unlink() + + def testElementReprAndStrUnicodeNS(self): + dom = Document() + el = dom.appendChild( + dom.createElementNS("http://www.slashdot.org", "slash:abc")) + string1 = repr(el) + string2 = str(el) + self.assertEqual(string1, string2) + self.assertIn("slash:abc", string1) + dom.unlink() + + def testAttributeRepr(self): + dom = Document() + el = dom.appendChild(dom.createElement("abc")) + node = el.setAttribute("abc", "def") + self.assertEqual(str(node), repr(node)) + dom.unlink() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testWriteXML(self): + str = '<?xml version="1.0" ?><a b="c"/>' + dom = parseString(str) + domstr = dom.toxml() + dom.unlink() + self.assertEqual(str, domstr) + + def test_toxml_quote_text(self): + dom = Document() + elem = dom.appendChild(dom.createElement('elem')) + elem.appendChild(dom.createTextNode('&<>"')) + cr = elem.appendChild(dom.createElement('cr')) + cr.appendChild(dom.createTextNode('\r')) + crlf = elem.appendChild(dom.createElement('crlf')) + crlf.appendChild(dom.createTextNode('\r\n')) + lflf = elem.appendChild(dom.createElement('lflf')) + lflf.appendChild(dom.createTextNode('\n\n')) + ws = elem.appendChild(dom.createElement('ws')) + ws.appendChild(dom.createTextNode('\t\n\r ')) + domstr = dom.toxml() + dom.unlink() + self.assertEqual(domstr, '<?xml version="1.0" ?>' + '<elem>&amp;&lt;&gt;"' + '<cr>\r</cr>' + '<crlf>\r\n</crlf>' + '<lflf>\n\n</lflf>' + '<ws>\t\n\r </ws></elem>') + + def test_toxml_quote_attrib(self): + dom = Document() + elem = dom.appendChild(dom.createElement('elem')) + elem.setAttribute("a", '&<>"') + elem.setAttribute("cr", "\r") + elem.setAttribute("lf", "\n") + elem.setAttribute("crlf", "\r\n") + elem.setAttribute("lflf", "\n\n") + elem.setAttribute("ws", "\t\n\r ") + domstr = dom.toxml() + dom.unlink() + self.assertEqual(domstr, '<?xml version="1.0" ?>' + '<elem a="&amp;&lt;&gt;&quot;" ' + 'cr="&#13;" ' + 'lf="&#10;" ' + 'crlf="&#13;&#10;" ' + 'lflf="&#10;&#10;" ' + 'ws="&#9;&#10;&#13; "/>') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testAltNewline(self): + str = '<?xml version="1.0" ?>\n<a b="c"/>\n' + dom = parseString(str) + domstr = dom.toprettyxml(newl="\r\n") + dom.unlink() + self.assertEqual(domstr, str.replace("\n", "\r\n")) + + def test_toprettyxml_with_text_nodes(self): + # see issue #4147, text nodes are not indented + decl = '<?xml version="1.0" ?>\n' + self.assertEqual(parseString('<B>A</B>').toprettyxml(), + decl + '<B>A</B>\n') + self.assertEqual(parseString('<C>A<B>A</B></C>').toprettyxml(), + decl + '<C>\n\tA\n\t<B>A</B>\n</C>\n') + self.assertEqual(parseString('<C><B>A</B>A</C>').toprettyxml(), + decl + '<C>\n\t<B>A</B>\n\tA\n</C>\n') + self.assertEqual(parseString('<C><B>A</B><B>A</B></C>').toprettyxml(), + decl + '<C>\n\t<B>A</B>\n\t<B>A</B>\n</C>\n') + self.assertEqual(parseString('<C><B>A</B>A<B>A</B></C>').toprettyxml(), + decl + '<C>\n\t<B>A</B>\n\tA\n\t<B>A</B>\n</C>\n') + + def test_toprettyxml_with_adjacent_text_nodes(self): + # see issue #4147, adjacent text nodes are indented normally + dom = Document() + elem = dom.createElement('elem') + elem.appendChild(dom.createTextNode('TEXT')) + elem.appendChild(dom.createTextNode('TEXT')) + dom.appendChild(elem) + decl = '<?xml version="1.0" ?>\n' + self.assertEqual(dom.toprettyxml(), + decl + '<elem>\n\tTEXT\n\tTEXT\n</elem>\n') + + def test_toprettyxml_preserves_content_of_text_node(self): + # see issue #4147 + for str in ('<B>A</B>', '<A><B>C</B></A>'): + dom = parseString(str) + dom2 = parseString(dom.toprettyxml()) + self.assertEqual( + dom.getElementsByTagName('B')[0].childNodes[0].toxml(), + dom2.getElementsByTagName('B')[0].childNodes[0].toxml()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testProcessingInstruction(self): + dom = parseString('<e><?mypi \t\n data \t\n ?></e>') + pi = dom.documentElement.firstChild + self.assertEqual(pi.target, "mypi") + self.assertEqual(pi.data, "data \t\n ") + self.assertEqual(pi.nodeName, "mypi") + self.assertEqual(pi.nodeType, Node.PROCESSING_INSTRUCTION_NODE) + self.assertIsNone(pi.attributes) + self.assertFalse(pi.hasChildNodes()) + self.assertEqual(len(pi.childNodes), 0) + self.assertIsNone(pi.firstChild) + self.assertIsNone(pi.lastChild) + self.assertIsNone(pi.localName) + self.assertEqual(pi.namespaceURI, xml.dom.EMPTY_NAMESPACE) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testProcessingInstructionRepr(self): + dom = parseString('<e><?mypi \t\n data \t\n ?></e>') + pi = dom.documentElement.firstChild + self.assertEqual(str(pi.nodeType), repr(pi.nodeType)) + + def testTextRepr(self): + dom = Document() + self.addCleanup(dom.unlink) + elem = dom.createElement("elem") + elem.appendChild(dom.createTextNode("foo")) + el = elem.firstChild + self.assertEqual(str(el), repr(el)) + self.assertEqual('<DOM Text node "\'foo\'">', str(el)) + + def testWriteText(self): pass + + def testDocumentElement(self): pass + + def testTooManyDocumentElements(self): + doc = parseString("<doc/>") + elem = doc.createElement("extra") + # Should raise an exception when adding an extra document element. + self.assertRaises(xml.dom.HierarchyRequestErr, doc.appendChild, elem) + elem.unlink() + doc.unlink() + + def testCreateElementNS(self): pass + + def testCreateAttributeNS(self): pass + + def testParse(self): pass + + def testParseString(self): pass + + def testComment(self): pass + + def testAttrListItem(self): pass + + def testAttrListItems(self): pass + + def testAttrListItemNS(self): pass + + def testAttrListKeys(self): pass + + def testAttrListKeysNS(self): pass + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testRemoveNamedItem(self): + doc = parseString("<doc a=''/>") + e = doc.documentElement + attrs = e.attributes + a1 = e.getAttributeNode("a") + a2 = attrs.removeNamedItem("a") + self.assertTrue(a1.isSameNode(a2)) + self.assertRaises(xml.dom.NotFoundErr, attrs.removeNamedItem, "a") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testRemoveNamedItemNS(self): + doc = parseString("<doc xmlns:a='http://xml.python.org/' a:b=''/>") + e = doc.documentElement + attrs = e.attributes + a1 = e.getAttributeNodeNS("http://xml.python.org/", "b") + a2 = attrs.removeNamedItemNS("http://xml.python.org/", "b") + self.assertTrue(a1.isSameNode(a2)) + self.assertRaises(xml.dom.NotFoundErr, attrs.removeNamedItemNS, + "http://xml.python.org/", "b") + + def testAttrListValues(self): pass + + def testAttrListLength(self): pass + + def testAttrList__getitem__(self): pass + + def testAttrList__setitem__(self): pass + + def testSetAttrValueandNodeValue(self): pass + + def testParseElement(self): pass + + def testParseAttributes(self): pass + + def testParseElementNamespaces(self): pass + + def testParseAttributeNamespaces(self): pass + + def testParseProcessingInstructions(self): pass + + def testChildNodes(self): pass + + def testFirstChild(self): pass + + def testHasChildNodes(self): + dom = parseString("<doc><foo/></doc>") + doc = dom.documentElement + self.assertTrue(doc.hasChildNodes()) + dom2 = parseString("<doc/>") + doc2 = dom2.documentElement + self.assertFalse(doc2.hasChildNodes()) + + def _testCloneElementCopiesAttributes(self, e1, e2, test): + attrs1 = e1.attributes + attrs2 = e2.attributes + keys1 = list(attrs1.keys()) + keys2 = list(attrs2.keys()) + keys1.sort() + keys2.sort() + self.assertEqual(keys1, keys2) + for i in range(len(keys1)): + a1 = attrs1.item(i) + a2 = attrs2.item(i) + self.assertIsNot(a1, a2) + self.assertEqual(a1.value, a2.value) + self.assertEqual(a1.nodeValue, a2.nodeValue) + self.assertEqual(a1.namespaceURI,a2.namespaceURI) + self.assertEqual(a1.localName, a2.localName) + self.assertIs(a2.ownerElement, e2) + + def _setupCloneElement(self, deep): + dom = parseString("<doc attr='value'><foo/></doc>") + root = dom.documentElement + clone = root.cloneNode(deep) + self._testCloneElementCopiesAttributes( + root, clone, "testCloneElement" + (deep and "Deep" or "Shallow")) + # mutilate the original so shared data is detected + root.tagName = root.nodeName = "MODIFIED" + root.setAttribute("attr", "NEW VALUE") + root.setAttribute("added", "VALUE") + return dom, clone + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testCloneElementShallow(self): + dom, clone = self._setupCloneElement(0) + self.assertEqual(len(clone.childNodes), 0) + self.assertEqual(clone.childNodes.length, 0) + self.assertIsNone(clone.parentNode) + self.assertEqual(clone.toxml(), '<doc attr="value"/>') + + dom.unlink() + + def testCloneElementDeep(self): + dom, clone = self._setupCloneElement(1) + self.assertEqual(len(clone.childNodes), 1) + self.assertEqual(clone.childNodes.length, 1) + self.assertIsNone(clone.parentNode) + self.assertTrue(clone.toxml(), '<doc attr="value"><foo/></doc>') + dom.unlink() + + def testCloneDocumentShallow(self): + doc = parseString("<?xml version='1.0'?>\n" + "<!-- comment -->" + "<!DOCTYPE doc [\n" + "<!NOTATION notation SYSTEM 'http://xml.python.org/'>\n" + "]>\n" + "<doc attr='value'/>") + doc2 = doc.cloneNode(0) + self.assertIsNone(doc2, + "testCloneDocumentShallow:" + " shallow cloning of documents makes no sense!") + + def testCloneDocumentDeep(self): + doc = parseString("<?xml version='1.0'?>\n" + "<!-- comment -->" + "<!DOCTYPE doc [\n" + "<!NOTATION notation SYSTEM 'http://xml.python.org/'>\n" + "]>\n" + "<doc attr='value'/>") + doc2 = doc.cloneNode(1) + self.assertFalse((doc.isSameNode(doc2) or doc2.isSameNode(doc)), + "testCloneDocumentDeep: document objects not distinct") + self.assertEqual(len(doc.childNodes), len(doc2.childNodes), + "testCloneDocumentDeep: wrong number of Document children") + self.assertEqual(doc2.documentElement.nodeType, Node.ELEMENT_NODE, + "testCloneDocumentDeep: documentElement not an ELEMENT_NODE") + self.assertTrue(doc2.documentElement.ownerDocument.isSameNode(doc2), + "testCloneDocumentDeep: documentElement owner is not new document") + self.assertFalse(doc.documentElement.isSameNode(doc2.documentElement), + "testCloneDocumentDeep: documentElement should not be shared") + if doc.doctype is not None: + # check the doctype iff the original DOM maintained it + self.assertEqual(doc2.doctype.nodeType, Node.DOCUMENT_TYPE_NODE, + "testCloneDocumentDeep: doctype not a DOCUMENT_TYPE_NODE") + self.assertTrue(doc2.doctype.ownerDocument.isSameNode(doc2)) + self.assertFalse(doc.doctype.isSameNode(doc2.doctype)) + + def testCloneDocumentTypeDeepOk(self): + doctype = create_nonempty_doctype() + clone = doctype.cloneNode(1) + self.confirm(clone is not None + and clone.nodeName == doctype.nodeName + and clone.name == doctype.name + and clone.publicId == doctype.publicId + and clone.systemId == doctype.systemId + and len(clone.entities) == len(doctype.entities) + and clone.entities.item(len(clone.entities)) is None + and len(clone.notations) == len(doctype.notations) + and clone.notations.item(len(clone.notations)) is None + and len(clone.childNodes) == 0) + for i in range(len(doctype.entities)): + se = doctype.entities.item(i) + ce = clone.entities.item(i) + self.confirm((not se.isSameNode(ce)) + and (not ce.isSameNode(se)) + and ce.nodeName == se.nodeName + and ce.notationName == se.notationName + and ce.publicId == se.publicId + and ce.systemId == se.systemId + and ce.encoding == se.encoding + and ce.actualEncoding == se.actualEncoding + and ce.version == se.version) + for i in range(len(doctype.notations)): + sn = doctype.notations.item(i) + cn = clone.notations.item(i) + self.confirm((not sn.isSameNode(cn)) + and (not cn.isSameNode(sn)) + and cn.nodeName == sn.nodeName + and cn.publicId == sn.publicId + and cn.systemId == sn.systemId) + + def testCloneDocumentTypeDeepNotOk(self): + doc = create_doc_with_doctype() + clone = doc.doctype.cloneNode(1) + self.assertIsNone(clone) + + def testCloneDocumentTypeShallowOk(self): + doctype = create_nonempty_doctype() + clone = doctype.cloneNode(0) + self.confirm(clone is not None + and clone.nodeName == doctype.nodeName + and clone.name == doctype.name + and clone.publicId == doctype.publicId + and clone.systemId == doctype.systemId + and len(clone.entities) == 0 + and clone.entities.item(0) is None + and len(clone.notations) == 0 + and clone.notations.item(0) is None + and len(clone.childNodes) == 0) + + def testCloneDocumentTypeShallowNotOk(self): + doc = create_doc_with_doctype() + clone = doc.doctype.cloneNode(0) + self.assertIsNone(clone) + + def check_import_document(self, deep, testName): + doc1 = parseString("<doc/>") + doc2 = parseString("<doc/>") + self.assertRaises(xml.dom.NotSupportedErr, doc1.importNode, doc2, deep) + + def testImportDocumentShallow(self): + self.check_import_document(0, "testImportDocumentShallow") + + def testImportDocumentDeep(self): + self.check_import_document(1, "testImportDocumentDeep") + + def testImportDocumentTypeShallow(self): + src = create_doc_with_doctype() + target = create_doc_without_doctype() + self.assertRaises(xml.dom.NotSupportedErr, target.importNode, + src.doctype, 0) + + def testImportDocumentTypeDeep(self): + src = create_doc_with_doctype() + target = create_doc_without_doctype() + self.assertRaises(xml.dom.NotSupportedErr, target.importNode, + src.doctype, 1) + + # Testing attribute clones uses a helper, and should always be deep, + # even if the argument to cloneNode is false. + def check_clone_attribute(self, deep, testName): + doc = parseString("<doc attr='value'/>") + attr = doc.documentElement.getAttributeNode("attr") + self.assertIsNotNone(attr) + clone = attr.cloneNode(deep) + self.assertFalse(clone.isSameNode(attr)) + self.assertFalse(attr.isSameNode(clone)) + self.assertIsNone(clone.ownerElement, + testName + ": ownerElement should be None") + self.confirm(clone.ownerDocument.isSameNode(attr.ownerDocument), + testName + ": ownerDocument does not match") + self.confirm(clone.specified, + testName + ": cloned attribute must have specified == True") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testCloneAttributeShallow(self): + self.check_clone_attribute(0, "testCloneAttributeShallow") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testCloneAttributeDeep(self): + self.check_clone_attribute(1, "testCloneAttributeDeep") + + def check_clone_pi(self, deep, testName): + doc = parseString("<?target data?><doc/>") + pi = doc.firstChild + self.assertEqual(pi.nodeType, Node.PROCESSING_INSTRUCTION_NODE) + clone = pi.cloneNode(deep) + self.confirm(clone.target == pi.target + and clone.data == pi.data) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testClonePIShallow(self): + self.check_clone_pi(0, "testClonePIShallow") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testClonePIDeep(self): + self.check_clone_pi(1, "testClonePIDeep") + + def check_clone_node_entity(self, clone_document): + # bpo-35052: Test user data handler in cloneNode() on a document with + # an entity + document = xml.dom.minidom.parseString(""" + <?xml version="1.0" ?> + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" + "http://www.w3.org/TR/html4/strict.dtd" + [ <!ENTITY smile "☺"> ] + > + <doc>Don't let entities make you frown &smile;</doc> + """.strip()) + + class Handler: + def handle(self, operation, key, data, src, dst): + self.operation = operation + self.key = key + self.data = data + self.src = src + self.dst = dst + + handler = Handler() + doctype = document.doctype + entity = doctype.entities['smile'] + entity.setUserData("key", "data", handler) + + if clone_document: + # clone Document + clone = document.cloneNode(deep=True) + + self.assertEqual(clone.documentElement.firstChild.wholeText, + "Don't let entities make you frown ☺") + operation = xml.dom.UserDataHandler.NODE_IMPORTED + dst = clone.doctype.entities['smile'] + else: + # clone DocumentType + with support.swap_attr(doctype, 'ownerDocument', None): + clone = doctype.cloneNode(deep=True) + + operation = xml.dom.UserDataHandler.NODE_CLONED + dst = clone.entities['smile'] + + self.assertEqual(handler.operation, operation) + self.assertEqual(handler.key, "key") + self.assertEqual(handler.data, "data") + self.assertIs(handler.src, entity) + self.assertIs(handler.dst, dst) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testCloneNodeEntity(self): + self.check_clone_node_entity(False) + self.check_clone_node_entity(True) + + def testNormalize(self): + doc = parseString("<doc/>") + root = doc.documentElement + root.appendChild(doc.createTextNode("first")) + root.appendChild(doc.createTextNode("second")) + self.confirm(len(root.childNodes) == 2 + and root.childNodes.length == 2, + "testNormalize -- preparation") + doc.normalize() + self.confirm(len(root.childNodes) == 1 + and root.childNodes.length == 1 + and root.firstChild is root.lastChild + and root.firstChild.data == "firstsecond" + , "testNormalize -- result") + doc.unlink() + + doc = parseString("<doc/>") + root = doc.documentElement + root.appendChild(doc.createTextNode("")) + doc.normalize() + self.confirm(len(root.childNodes) == 0 + and root.childNodes.length == 0, + "testNormalize -- single empty node removed") + doc.unlink() + + def testNormalizeCombineAndNextSibling(self): + doc = parseString("<doc/>") + root = doc.documentElement + root.appendChild(doc.createTextNode("first")) + root.appendChild(doc.createTextNode("second")) + root.appendChild(doc.createElement("i")) + self.confirm(len(root.childNodes) == 3 + and root.childNodes.length == 3, + "testNormalizeCombineAndNextSibling -- preparation") + doc.normalize() + self.confirm(len(root.childNodes) == 2 + and root.childNodes.length == 2 + and root.firstChild.data == "firstsecond" + and root.firstChild is not root.lastChild + and root.firstChild.nextSibling is root.lastChild + and root.firstChild.previousSibling is None + and root.lastChild.previousSibling is root.firstChild + and root.lastChild.nextSibling is None + , "testNormalizeCombinedAndNextSibling -- result") + doc.unlink() + + def testNormalizeDeleteWithPrevSibling(self): + doc = parseString("<doc/>") + root = doc.documentElement + root.appendChild(doc.createTextNode("first")) + root.appendChild(doc.createTextNode("")) + self.confirm(len(root.childNodes) == 2 + and root.childNodes.length == 2, + "testNormalizeDeleteWithPrevSibling -- preparation") + doc.normalize() + self.confirm(len(root.childNodes) == 1 + and root.childNodes.length == 1 + and root.firstChild.data == "first" + and root.firstChild is root.lastChild + and root.firstChild.nextSibling is None + and root.firstChild.previousSibling is None + , "testNormalizeDeleteWithPrevSibling -- result") + doc.unlink() + + def testNormalizeDeleteWithNextSibling(self): + doc = parseString("<doc/>") + root = doc.documentElement + root.appendChild(doc.createTextNode("")) + root.appendChild(doc.createTextNode("second")) + self.confirm(len(root.childNodes) == 2 + and root.childNodes.length == 2, + "testNormalizeDeleteWithNextSibling -- preparation") + doc.normalize() + self.confirm(len(root.childNodes) == 1 + and root.childNodes.length == 1 + and root.firstChild.data == "second" + and root.firstChild is root.lastChild + and root.firstChild.nextSibling is None + and root.firstChild.previousSibling is None + , "testNormalizeDeleteWithNextSibling -- result") + doc.unlink() + + def testNormalizeDeleteWithTwoNonTextSiblings(self): + doc = parseString("<doc/>") + root = doc.documentElement + root.appendChild(doc.createElement("i")) + root.appendChild(doc.createTextNode("")) + root.appendChild(doc.createElement("i")) + self.confirm(len(root.childNodes) == 3 + and root.childNodes.length == 3, + "testNormalizeDeleteWithTwoSiblings -- preparation") + doc.normalize() + self.confirm(len(root.childNodes) == 2 + and root.childNodes.length == 2 + and root.firstChild is not root.lastChild + and root.firstChild.nextSibling is root.lastChild + and root.firstChild.previousSibling is None + and root.lastChild.previousSibling is root.firstChild + and root.lastChild.nextSibling is None + , "testNormalizeDeleteWithTwoSiblings -- result") + doc.unlink() + + def testNormalizeDeleteAndCombine(self): + doc = parseString("<doc/>") + root = doc.documentElement + root.appendChild(doc.createTextNode("")) + root.appendChild(doc.createTextNode("second")) + root.appendChild(doc.createTextNode("")) + root.appendChild(doc.createTextNode("fourth")) + root.appendChild(doc.createTextNode("")) + self.confirm(len(root.childNodes) == 5 + and root.childNodes.length == 5, + "testNormalizeDeleteAndCombine -- preparation") + doc.normalize() + self.confirm(len(root.childNodes) == 1 + and root.childNodes.length == 1 + and root.firstChild is root.lastChild + and root.firstChild.data == "secondfourth" + and root.firstChild.previousSibling is None + and root.firstChild.nextSibling is None + , "testNormalizeDeleteAndCombine -- result") + doc.unlink() + + def testNormalizeRecursion(self): + doc = parseString("<doc>" + "<o>" + "<i/>" + "t" + # + #x + "</o>" + "<o>" + "<o>" + "t2" + #x2 + "</o>" + "t3" + #x3 + "</o>" + # + "</doc>") + root = doc.documentElement + root.childNodes[0].appendChild(doc.createTextNode("")) + root.childNodes[0].appendChild(doc.createTextNode("x")) + root.childNodes[1].childNodes[0].appendChild(doc.createTextNode("x2")) + root.childNodes[1].appendChild(doc.createTextNode("x3")) + root.appendChild(doc.createTextNode("")) + self.confirm(len(root.childNodes) == 3 + and root.childNodes.length == 3 + and len(root.childNodes[0].childNodes) == 4 + and root.childNodes[0].childNodes.length == 4 + and len(root.childNodes[1].childNodes) == 3 + and root.childNodes[1].childNodes.length == 3 + and len(root.childNodes[1].childNodes[0].childNodes) == 2 + and root.childNodes[1].childNodes[0].childNodes.length == 2 + , "testNormalize2 -- preparation") + doc.normalize() + self.confirm(len(root.childNodes) == 2 + and root.childNodes.length == 2 + and len(root.childNodes[0].childNodes) == 2 + and root.childNodes[0].childNodes.length == 2 + and len(root.childNodes[1].childNodes) == 2 + and root.childNodes[1].childNodes.length == 2 + and len(root.childNodes[1].childNodes[0].childNodes) == 1 + and root.childNodes[1].childNodes[0].childNodes.length == 1 + , "testNormalize2 -- childNodes lengths") + self.confirm(root.childNodes[0].childNodes[1].data == "tx" + and root.childNodes[1].childNodes[0].childNodes[0].data == "t2x2" + and root.childNodes[1].childNodes[1].data == "t3x3" + , "testNormalize2 -- joined text fields") + self.confirm(root.childNodes[0].childNodes[1].nextSibling is None + and root.childNodes[0].childNodes[1].previousSibling + is root.childNodes[0].childNodes[0] + and root.childNodes[0].childNodes[0].previousSibling is None + and root.childNodes[0].childNodes[0].nextSibling + is root.childNodes[0].childNodes[1] + and root.childNodes[1].childNodes[1].nextSibling is None + and root.childNodes[1].childNodes[1].previousSibling + is root.childNodes[1].childNodes[0] + and root.childNodes[1].childNodes[0].previousSibling is None + and root.childNodes[1].childNodes[0].nextSibling + is root.childNodes[1].childNodes[1] + , "testNormalize2 -- sibling pointers") + doc.unlink() + + + def testBug0777884(self): + doc = parseString("<o>text</o>") + text = doc.documentElement.childNodes[0] + self.assertEqual(text.nodeType, Node.TEXT_NODE) + # Should run quietly, doing nothing. + text.normalize() + doc.unlink() + + def testBug1433694(self): + doc = parseString("<o><i/>t</o>") + node = doc.documentElement + node.childNodes[1].nodeValue = "" + node.normalize() + self.assertIsNone(node.childNodes[-1].nextSibling, + "Final child's .nextSibling should be None") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testSiblings(self): + doc = parseString("<doc><?pi?>text?<elm/></doc>") + root = doc.documentElement + (pi, text, elm) = root.childNodes + + self.confirm(pi.nextSibling is text and + pi.previousSibling is None and + text.nextSibling is elm and + text.previousSibling is pi and + elm.nextSibling is None and + elm.previousSibling is text, "testSiblings") + + doc.unlink() + + def testParents(self): + doc = parseString( + "<doc><elm1><elm2/><elm2><elm3/></elm2></elm1></doc>") + root = doc.documentElement + elm1 = root.childNodes[0] + (elm2a, elm2b) = elm1.childNodes + elm3 = elm2b.childNodes[0] + + self.confirm(root.parentNode is doc and + elm1.parentNode is root and + elm2a.parentNode is elm1 and + elm2b.parentNode is elm1 and + elm3.parentNode is elm2b, "testParents") + doc.unlink() + + def testNodeListItem(self): + doc = parseString("<doc><e/><e/></doc>") + children = doc.childNodes + docelem = children[0] + self.confirm(children[0] is children.item(0) + and children.item(1) is None + and docelem.childNodes.item(0) is docelem.childNodes[0] + and docelem.childNodes.item(1) is docelem.childNodes[1] + and docelem.childNodes.item(0).childNodes.item(0) is None, + "test NodeList.item()") + doc.unlink() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testEncodings(self): + doc = parseString('<foo>&#x20ac;</foo>') + self.assertEqual(doc.toxml(), + '<?xml version="1.0" ?><foo>\u20ac</foo>') + self.assertEqual(doc.toxml('utf-8'), + b'<?xml version="1.0" encoding="utf-8"?><foo>\xe2\x82\xac</foo>') + self.assertEqual(doc.toxml('iso-8859-15'), + b'<?xml version="1.0" encoding="iso-8859-15"?><foo>\xa4</foo>') + self.assertEqual(doc.toxml('us-ascii'), + b'<?xml version="1.0" encoding="us-ascii"?><foo>&#8364;</foo>') + self.assertEqual(doc.toxml('utf-16'), + '<?xml version="1.0" encoding="utf-16"?>' + '<foo>\u20ac</foo>'.encode('utf-16')) + + # Verify that character decoding errors raise exceptions instead + # of crashing + with self.assertRaises((UnicodeDecodeError, ExpatError)): + parseString( + b'<fran\xe7ais>Comment \xe7a va ? Tr\xe8s bien ?</fran\xe7ais>' + ) + + doc.unlink() + + def testStandalone(self): + doc = parseString('<foo>&#x20ac;</foo>') + self.assertEqual(doc.toxml(), + '<?xml version="1.0" ?><foo>\u20ac</foo>') + self.assertEqual(doc.toxml(standalone=None), + '<?xml version="1.0" ?><foo>\u20ac</foo>') + self.assertEqual(doc.toxml(standalone=True), + '<?xml version="1.0" standalone="yes"?><foo>\u20ac</foo>') + self.assertEqual(doc.toxml(standalone=False), + '<?xml version="1.0" standalone="no"?><foo>\u20ac</foo>') + self.assertEqual(doc.toxml('utf-8', True), + b'<?xml version="1.0" encoding="utf-8" standalone="yes"?>' + b'<foo>\xe2\x82\xac</foo>') + + doc.unlink() + + class UserDataHandler: + called = 0 + def handle(self, operation, key, data, src, dst): + dst.setUserData(key, data + 1, self) + src.setUserData(key, None, None) + self.called = 1 + + def testUserData(self): + dom = Document() + n = dom.createElement('e') + self.assertIsNone(n.getUserData("foo")) + n.setUserData("foo", None, None) + self.assertIsNone(n.getUserData("foo")) + n.setUserData("foo", 12, 12) + n.setUserData("bar", 13, 13) + self.assertEqual(n.getUserData("foo"), 12) + self.assertEqual(n.getUserData("bar"), 13) + n.setUserData("foo", None, None) + self.assertIsNone(n.getUserData("foo")) + self.assertEqual(n.getUserData("bar"), 13) + + handler = self.UserDataHandler() + n.setUserData("bar", 12, handler) + c = n.cloneNode(1) + self.confirm(handler.called + and n.getUserData("bar") is None + and c.getUserData("bar") == 13) + n.unlink() + c.unlink() + dom.unlink() + + def checkRenameNodeSharedConstraints(self, doc, node): + # Make sure illegal NS usage is detected: + self.assertRaises(xml.dom.NamespaceErr, doc.renameNode, node, + "http://xml.python.org/ns", "xmlns:foo") + doc2 = parseString("<doc/>") + self.assertRaises(xml.dom.WrongDocumentErr, doc2.renameNode, node, + xml.dom.EMPTY_NAMESPACE, "foo") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testRenameAttribute(self): + doc = parseString("<doc a='v'/>") + elem = doc.documentElement + attrmap = elem.attributes + attr = elem.attributes['a'] + + # Simple renaming + attr = doc.renameNode(attr, xml.dom.EMPTY_NAMESPACE, "b") + self.confirm(attr.name == "b" + and attr.nodeName == "b" + and attr.localName is None + and attr.namespaceURI == xml.dom.EMPTY_NAMESPACE + and attr.prefix is None + and attr.value == "v" + and elem.getAttributeNode("a") is None + and elem.getAttributeNode("b").isSameNode(attr) + and attrmap["b"].isSameNode(attr) + and attr.ownerDocument.isSameNode(doc) + and attr.ownerElement.isSameNode(elem)) + + # Rename to have a namespace, no prefix + attr = doc.renameNode(attr, "http://xml.python.org/ns", "c") + self.confirm(attr.name == "c" + and attr.nodeName == "c" + and attr.localName == "c" + and attr.namespaceURI == "http://xml.python.org/ns" + and attr.prefix is None + and attr.value == "v" + and elem.getAttributeNode("a") is None + and elem.getAttributeNode("b") is None + and elem.getAttributeNode("c").isSameNode(attr) + and elem.getAttributeNodeNS( + "http://xml.python.org/ns", "c").isSameNode(attr) + and attrmap["c"].isSameNode(attr) + and attrmap[("http://xml.python.org/ns", "c")].isSameNode(attr)) + + # Rename to have a namespace, with prefix + attr = doc.renameNode(attr, "http://xml.python.org/ns2", "p:d") + self.confirm(attr.name == "p:d" + and attr.nodeName == "p:d" + and attr.localName == "d" + and attr.namespaceURI == "http://xml.python.org/ns2" + and attr.prefix == "p" + and attr.value == "v" + and elem.getAttributeNode("a") is None + and elem.getAttributeNode("b") is None + and elem.getAttributeNode("c") is None + and elem.getAttributeNodeNS( + "http://xml.python.org/ns", "c") is None + and elem.getAttributeNode("p:d").isSameNode(attr) + and elem.getAttributeNodeNS( + "http://xml.python.org/ns2", "d").isSameNode(attr) + and attrmap["p:d"].isSameNode(attr) + and attrmap[("http://xml.python.org/ns2", "d")].isSameNode(attr)) + + # Rename back to a simple non-NS node + attr = doc.renameNode(attr, xml.dom.EMPTY_NAMESPACE, "e") + self.confirm(attr.name == "e" + and attr.nodeName == "e" + and attr.localName is None + and attr.namespaceURI == xml.dom.EMPTY_NAMESPACE + and attr.prefix is None + and attr.value == "v" + and elem.getAttributeNode("a") is None + and elem.getAttributeNode("b") is None + and elem.getAttributeNode("c") is None + and elem.getAttributeNode("p:d") is None + and elem.getAttributeNodeNS( + "http://xml.python.org/ns", "c") is None + and elem.getAttributeNode("e").isSameNode(attr) + and attrmap["e"].isSameNode(attr)) + + self.assertRaises(xml.dom.NamespaceErr, doc.renameNode, attr, + "http://xml.python.org/ns", "xmlns") + self.checkRenameNodeSharedConstraints(doc, attr) + doc.unlink() + + def testRenameElement(self): + doc = parseString("<doc/>") + elem = doc.documentElement + + # Simple renaming + elem = doc.renameNode(elem, xml.dom.EMPTY_NAMESPACE, "a") + self.confirm(elem.tagName == "a" + and elem.nodeName == "a" + and elem.localName is None + and elem.namespaceURI == xml.dom.EMPTY_NAMESPACE + and elem.prefix is None + and elem.ownerDocument.isSameNode(doc)) + + # Rename to have a namespace, no prefix + elem = doc.renameNode(elem, "http://xml.python.org/ns", "b") + self.confirm(elem.tagName == "b" + and elem.nodeName == "b" + and elem.localName == "b" + and elem.namespaceURI == "http://xml.python.org/ns" + and elem.prefix is None + and elem.ownerDocument.isSameNode(doc)) + + # Rename to have a namespace, with prefix + elem = doc.renameNode(elem, "http://xml.python.org/ns2", "p:c") + self.confirm(elem.tagName == "p:c" + and elem.nodeName == "p:c" + and elem.localName == "c" + and elem.namespaceURI == "http://xml.python.org/ns2" + and elem.prefix == "p" + and elem.ownerDocument.isSameNode(doc)) + + # Rename back to a simple non-NS node + elem = doc.renameNode(elem, xml.dom.EMPTY_NAMESPACE, "d") + self.confirm(elem.tagName == "d" + and elem.nodeName == "d" + and elem.localName is None + and elem.namespaceURI == xml.dom.EMPTY_NAMESPACE + and elem.prefix is None + and elem.ownerDocument.isSameNode(doc)) + + self.checkRenameNodeSharedConstraints(doc, elem) + doc.unlink() + + def testRenameOther(self): + # We have to create a comment node explicitly since not all DOM + # builders used with minidom add comments to the DOM. + doc = xml.dom.minidom.getDOMImplementation().createDocument( + xml.dom.EMPTY_NAMESPACE, "e", None) + node = doc.createComment("comment") + self.assertRaises(xml.dom.NotSupportedErr, doc.renameNode, node, + xml.dom.EMPTY_NAMESPACE, "foo") + doc.unlink() + + def testWholeText(self): + doc = parseString("<doc>a</doc>") + elem = doc.documentElement + text = elem.childNodes[0] + self.assertEqual(text.nodeType, Node.TEXT_NODE) + + self.checkWholeText(text, "a") + elem.appendChild(doc.createTextNode("b")) + self.checkWholeText(text, "ab") + elem.insertBefore(doc.createCDATASection("c"), text) + self.checkWholeText(text, "cab") + + # make sure we don't cross other nodes + splitter = doc.createComment("comment") + elem.appendChild(splitter) + text2 = doc.createTextNode("d") + elem.appendChild(text2) + self.checkWholeText(text, "cab") + self.checkWholeText(text2, "d") + + x = doc.createElement("x") + elem.replaceChild(x, splitter) + splitter = x + self.checkWholeText(text, "cab") + self.checkWholeText(text2, "d") + + x = doc.createProcessingInstruction("y", "z") + elem.replaceChild(x, splitter) + splitter = x + self.checkWholeText(text, "cab") + self.checkWholeText(text2, "d") + + elem.removeChild(splitter) + self.checkWholeText(text, "cabd") + self.checkWholeText(text2, "cabd") + + def testPatch1094164(self): + doc = parseString("<doc><e/></doc>") + elem = doc.documentElement + e = elem.firstChild + self.assertIs(e.parentNode, elem, "Before replaceChild()") + # Check that replacing a child with itself leaves the tree unchanged + elem.replaceChild(e, e) + self.assertIs(e.parentNode, elem, "After replaceChild()") + + def testReplaceWholeText(self): + def setup(): + doc = parseString("<doc>a<e/>d</doc>") + elem = doc.documentElement + text1 = elem.firstChild + text2 = elem.lastChild + splitter = text1.nextSibling + elem.insertBefore(doc.createTextNode("b"), splitter) + elem.insertBefore(doc.createCDATASection("c"), text1) + return doc, elem, text1, splitter, text2 + + doc, elem, text1, splitter, text2 = setup() + text = text1.replaceWholeText("new content") + self.checkWholeText(text, "new content") + self.checkWholeText(text2, "d") + self.assertEqual(len(elem.childNodes), 3) + + doc, elem, text1, splitter, text2 = setup() + text = text2.replaceWholeText("new content") + self.checkWholeText(text, "new content") + self.checkWholeText(text1, "cab") + self.assertEqual(len(elem.childNodes), 5) + + doc, elem, text1, splitter, text2 = setup() + text = text1.replaceWholeText("") + self.checkWholeText(text2, "d") + self.confirm(text is None + and len(elem.childNodes) == 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testSchemaType(self): + doc = parseString( + "<!DOCTYPE doc [\n" + " <!ENTITY e1 SYSTEM 'http://xml.python.org/e1'>\n" + " <!ENTITY e2 SYSTEM 'http://xml.python.org/e2'>\n" + " <!ATTLIST doc id ID #IMPLIED \n" + " ref IDREF #IMPLIED \n" + " refs IDREFS #IMPLIED \n" + " enum (a|b) #IMPLIED \n" + " ent ENTITY #IMPLIED \n" + " ents ENTITIES #IMPLIED \n" + " nm NMTOKEN #IMPLIED \n" + " nms NMTOKENS #IMPLIED \n" + " text CDATA #IMPLIED \n" + " >\n" + "]><doc id='name' notid='name' text='splat!' enum='b'" + " ref='name' refs='name name' ent='e1' ents='e1 e2'" + " nm='123' nms='123 abc' />") + elem = doc.documentElement + # We don't want to rely on any specific loader at this point, so + # just make sure we can get to all the names, and that the + # DTD-based namespace is right. The names can vary by loader + # since each supports a different level of DTD information. + t = elem.schemaType + self.confirm(t.name is None + and t.namespace == xml.dom.EMPTY_NAMESPACE) + names = "id notid text enum ref refs ent ents nm nms".split() + for name in names: + a = elem.getAttributeNode(name) + t = a.schemaType + self.confirm(hasattr(t, "name") + and t.namespace == xml.dom.EMPTY_NAMESPACE) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testSetIdAttribute(self): + doc = parseString("<doc a1='v' a2='w'/>") + e = doc.documentElement + a1 = e.getAttributeNode("a1") + a2 = e.getAttributeNode("a2") + self.confirm(doc.getElementById("v") is None + and not a1.isId + and not a2.isId) + e.setIdAttribute("a1") + self.confirm(e.isSameNode(doc.getElementById("v")) + and a1.isId + and not a2.isId) + e.setIdAttribute("a2") + self.confirm(e.isSameNode(doc.getElementById("v")) + and e.isSameNode(doc.getElementById("w")) + and a1.isId + and a2.isId) + # replace the a1 node; the new node should *not* be an ID + a3 = doc.createAttribute("a1") + a3.value = "v" + e.setAttributeNode(a3) + self.confirm(doc.getElementById("v") is None + and e.isSameNode(doc.getElementById("w")) + and not a1.isId + and a2.isId + and not a3.isId) + # renaming an attribute should not affect its ID-ness: + doc.renameNode(a2, xml.dom.EMPTY_NAMESPACE, "an") + self.confirm(e.isSameNode(doc.getElementById("w")) + and a2.isId) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testSetIdAttributeNS(self): + NS1 = "http://xml.python.org/ns1" + NS2 = "http://xml.python.org/ns2" + doc = parseString("<doc" + " xmlns:ns1='" + NS1 + "'" + " xmlns:ns2='" + NS2 + "'" + " ns1:a1='v' ns2:a2='w'/>") + e = doc.documentElement + a1 = e.getAttributeNodeNS(NS1, "a1") + a2 = e.getAttributeNodeNS(NS2, "a2") + self.confirm(doc.getElementById("v") is None + and not a1.isId + and not a2.isId) + e.setIdAttributeNS(NS1, "a1") + self.confirm(e.isSameNode(doc.getElementById("v")) + and a1.isId + and not a2.isId) + e.setIdAttributeNS(NS2, "a2") + self.confirm(e.isSameNode(doc.getElementById("v")) + and e.isSameNode(doc.getElementById("w")) + and a1.isId + and a2.isId) + # replace the a1 node; the new node should *not* be an ID + a3 = doc.createAttributeNS(NS1, "a1") + a3.value = "v" + e.setAttributeNode(a3) + self.assertTrue(e.isSameNode(doc.getElementById("w"))) + self.assertFalse(a1.isId) + self.assertTrue(a2.isId) + self.assertFalse(a3.isId) + self.assertIsNone(doc.getElementById("v")) + # renaming an attribute should not affect its ID-ness: + doc.renameNode(a2, xml.dom.EMPTY_NAMESPACE, "an") + self.confirm(e.isSameNode(doc.getElementById("w")) + and a2.isId) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testSetIdAttributeNode(self): + NS1 = "http://xml.python.org/ns1" + NS2 = "http://xml.python.org/ns2" + doc = parseString("<doc" + " xmlns:ns1='" + NS1 + "'" + " xmlns:ns2='" + NS2 + "'" + " ns1:a1='v' ns2:a2='w'/>") + e = doc.documentElement + a1 = e.getAttributeNodeNS(NS1, "a1") + a2 = e.getAttributeNodeNS(NS2, "a2") + self.confirm(doc.getElementById("v") is None + and not a1.isId + and not a2.isId) + e.setIdAttributeNode(a1) + self.confirm(e.isSameNode(doc.getElementById("v")) + and a1.isId + and not a2.isId) + e.setIdAttributeNode(a2) + self.confirm(e.isSameNode(doc.getElementById("v")) + and e.isSameNode(doc.getElementById("w")) + and a1.isId + and a2.isId) + # replace the a1 node; the new node should *not* be an ID + a3 = doc.createAttributeNS(NS1, "a1") + a3.value = "v" + e.setAttributeNode(a3) + self.assertTrue(e.isSameNode(doc.getElementById("w"))) + self.assertFalse(a1.isId) + self.assertTrue(a2.isId) + self.assertFalse(a3.isId) + self.assertIsNone(doc.getElementById("v")) + # renaming an attribute should not affect its ID-ness: + doc.renameNode(a2, xml.dom.EMPTY_NAMESPACE, "an") + self.confirm(e.isSameNode(doc.getElementById("w")) + and a2.isId) + + def assert_recursive_equal(self, doc, doc2): + stack = [(doc, doc2)] + while stack: + n1, n2 = stack.pop() + self.assertEqual(n1.nodeType, n2.nodeType) + self.assertEqual(len(n1.childNodes), len(n2.childNodes)) + self.assertEqual(n1.nodeName, n2.nodeName) + self.assertFalse(n1.isSameNode(n2)) + self.assertFalse(n2.isSameNode(n1)) + if n1.nodeType == Node.DOCUMENT_TYPE_NODE: + len(n1.entities) + len(n2.entities) + len(n1.notations) + len(n2.notations) + self.assertEqual(len(n1.entities), len(n2.entities)) + self.assertEqual(len(n1.notations), len(n2.notations)) + for i in range(len(n1.notations)): + # XXX this loop body doesn't seem to be executed? + no1 = n1.notations.item(i) + no2 = n1.notations.item(i) + self.assertEqual(no1.name, no2.name) + self.assertEqual(no1.publicId, no2.publicId) + self.assertEqual(no1.systemId, no2.systemId) + stack.append((no1, no2)) + for i in range(len(n1.entities)): + e1 = n1.entities.item(i) + e2 = n2.entities.item(i) + self.assertEqual(e1.notationName, e2.notationName) + self.assertEqual(e1.publicId, e2.publicId) + self.assertEqual(e1.systemId, e2.systemId) + stack.append((e1, e2)) + if n1.nodeType != Node.DOCUMENT_NODE: + self.assertTrue(n1.ownerDocument.isSameNode(doc)) + self.assertTrue(n2.ownerDocument.isSameNode(doc2)) + for i in range(len(n1.childNodes)): + stack.append((n1.childNodes[i], n2.childNodes[i])) + + def testPickledDocument(self): + doc = parseString(sample) + for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(doc, proto) + doc2 = pickle.loads(s) + self.assert_recursive_equal(doc, doc2) + + def testDeepcopiedDocument(self): + doc = parseString(sample) + doc2 = copy.deepcopy(doc) + self.assert_recursive_equal(doc, doc2) + + def testSerializeCommentNodeWithDoubleHyphen(self): + doc = create_doc_without_doctype() + doc.appendChild(doc.createComment("foo--bar")) + self.assertRaises(ValueError, doc.toxml) + + + def testEmptyXMLNSValue(self): + doc = parseString("<element xmlns=''>\n" + "<foo/>\n</element>") + doc2 = parseString(doc.toxml()) + self.assertEqual(doc2.namespaceURI, xml.dom.EMPTY_NAMESPACE) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def testExceptionOnSpacesInXMLNSValue(self): + with self.assertRaises((ValueError, ExpatError)): + parseString( + '<element xmlns:abc="http:abc.com/de f g/hi/j k">' + + '<abc:foo /></element>' + ) + + def testDocRemoveChild(self): + doc = parse(tstfile) + title_tag = doc.documentElement.getElementsByTagName("TITLE")[0] + self.assertRaises( xml.dom.NotFoundErr, doc.removeChild, title_tag) + num_children_before = len(doc.childNodes) + doc.removeChild(doc.childNodes[0]) + num_children_after = len(doc.childNodes) + self.assertEqual(num_children_after, num_children_before - 1) + + def testProcessingInstructionNameError(self): + # wrong variable in .nodeValue property will + # lead to "NameError: name 'data' is not defined" + doc = parse(tstfile) + pi = doc.createProcessingInstruction("y", "z") + pi.nodeValue = "crash" + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_minidom_attribute_order(self): + xml_str = '<?xml version="1.0" ?><curriculum status="public" company="example"/>' + doc = parseString(xml_str) + output = io.StringIO() + doc.writexml(output) + self.assertEqual(output.getvalue(), xml_str) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_toxml_with_attributes_ordered(self): + xml_str = '<?xml version="1.0" ?><curriculum status="public" company="example"/>' + doc = parseString(xml_str) + self.assertEqual(doc.toxml(), xml_str) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_toprettyxml_with_attributes_ordered(self): + xml_str = '<?xml version="1.0" ?><curriculum status="public" company="example"/>' + doc = parseString(xml_str) + self.assertEqual(doc.toprettyxml(), + '<?xml version="1.0" ?>\n' + '<curriculum status="public" company="example"/>\n') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_toprettyxml_with_cdata(self): + xml_str = '<?xml version="1.0" ?><root><node><![CDATA[</data>]]></node></root>' + doc = parseString(xml_str) + self.assertEqual(doc.toprettyxml(), + '<?xml version="1.0" ?>\n' + '<root>\n' + '\t<node><![CDATA[</data>]]></node>\n' + '</root>\n') + + def test_cdata_parsing(self): + xml_str = '<?xml version="1.0" ?><root><node><![CDATA[</data>]]></node></root>' + dom1 = parseString(xml_str) + self.checkWholeText(dom1.getElementsByTagName('node')[0].firstChild, '</data>') + dom2 = parseString(dom1.toprettyxml()) + self.checkWholeText(dom2.getElementsByTagName('node')[0].firstChild, '</data>') + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index ed96be53cca..448ea2325fe 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -867,6 +867,7 @@ def test_resize_fails_if_mapping_held_elsewhere(self): finally: f.close() + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_resize_succeeds_with_error_for_second_named_mapping(self): """If a more than one mapping exists of the same name, none of them can diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index d8a0ba0803d..59c74fd0d41 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -293,8 +293,6 @@ class M(ModuleType): melon = Descr() self.assertRaises(RuntimeError, getattr, M("mymod"), "melon") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazy_create_annotations(self): # module objects lazy create their __annotations__ dict on demand. # the annotations dict is stored in module.__dict__. @@ -334,7 +332,11 @@ def test_annotations_getset_raises(self): del foo.__annotations__ def test_annotations_are_created_correctly(self): - ann_module4 = import_helper.import_fresh_module('test.ann_module4') + ann_module4 = import_helper.import_fresh_module( + 'test.typinganndata.ann_module4', + ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py new file mode 100644 index 00000000000..51f7fd257e0 --- /dev/null +++ b/Lib/test/test_modulefinder.py @@ -0,0 +1,440 @@ +import os +import errno +import importlib.machinery +import py_compile +import shutil +import unittest +import tempfile + +from test import support + +import modulefinder + +# Each test description is a list of 5 items: +# +# 1. a module name that will be imported by modulefinder +# 2. a list of module names that modulefinder is required to find +# 3. a list of module names that modulefinder should complain +# about because they are not found +# 4. a list of module names that modulefinder should complain +# about because they MAY be not found +# 5. a string specifying packages to create; the format is obvious imo. +# +# Each package will be created in test_dir, and test_dir will be +# removed after the tests again. +# Modulefinder searches in a path that contains test_dir, plus +# the standard Lib directory. + +maybe_test = [ + "a.module", + ["a", "a.module", "sys", + "b"], + ["c"], ["b.something"], + """\ +a/__init__.py +a/module.py + from b import something + from c import something +b/__init__.py + from sys import * +""", +] + +maybe_test_new = [ + "a.module", + ["a", "a.module", "sys", + "b", "__future__"], + ["c"], ["b.something"], + """\ +a/__init__.py +a/module.py + from b import something + from c import something +b/__init__.py + from __future__ import absolute_import + from sys import * +"""] + +package_test = [ + "a.module", + ["a", "a.b", "a.c", "a.module", "mymodule", "sys"], + ["blahblah", "c"], [], + """\ +mymodule.py +a/__init__.py + import blahblah + from a import b + import c +a/module.py + import sys + from a import b as x + from a.c import sillyname +a/b.py +a/c.py + from a.module import x + import mymodule as sillyname + from sys import version_info +"""] + +absolute_import_test = [ + "a.module", + ["a", "a.module", + "b", "b.x", "b.y", "b.z", + "__future__", "sys", "gc"], + ["blahblah", "z"], [], + """\ +mymodule.py +a/__init__.py +a/module.py + from __future__ import absolute_import + import sys # sys + import blahblah # fails + import gc # gc + import b.x # b.x + from b import y # b.y + from b.z import * # b.z.* +a/gc.py +a/sys.py + import mymodule +a/b/__init__.py +a/b/x.py +a/b/y.py +a/b/z.py +b/__init__.py + import z +b/unused.py +b/x.py +b/y.py +b/z.py +"""] + +relative_import_test = [ + "a.module", + ["__future__", + "a", "a.module", + "a.b", "a.b.y", "a.b.z", + "a.b.c", "a.b.c.moduleC", + "a.b.c.d", "a.b.c.e", + "a.b.x", + "gc"], + [], [], + """\ +mymodule.py +a/__init__.py + from .b import y, z # a.b.y, a.b.z +a/module.py + from __future__ import absolute_import # __future__ + import gc # gc +a/gc.py +a/sys.py +a/b/__init__.py + from ..b import x # a.b.x + #from a.b.c import moduleC + from .c import moduleC # a.b.moduleC +a/b/x.py +a/b/y.py +a/b/z.py +a/b/g.py +a/b/c/__init__.py + from ..c import e # a.b.c.e +a/b/c/moduleC.py + from ..c import d # a.b.c.d +a/b/c/d.py +a/b/c/e.py +a/b/c/x.py +"""] + +relative_import_test_2 = [ + "a.module", + ["a", "a.module", + "a.sys", + "a.b", "a.b.y", "a.b.z", + "a.b.c", "a.b.c.d", + "a.b.c.e", + "a.b.c.moduleC", + "a.b.c.f", + "a.b.x", + "a.another"], + [], [], + """\ +mymodule.py +a/__init__.py + from . import sys # a.sys +a/another.py +a/module.py + from .b import y, z # a.b.y, a.b.z +a/gc.py +a/sys.py +a/b/__init__.py + from .c import moduleC # a.b.c.moduleC + from .c import d # a.b.c.d +a/b/x.py +a/b/y.py +a/b/z.py +a/b/c/__init__.py + from . import e # a.b.c.e +a/b/c/moduleC.py + # + from . import f # a.b.c.f + from .. import x # a.b.x + from ... import another # a.another +a/b/c/d.py +a/b/c/e.py +a/b/c/f.py +"""] + +relative_import_test_3 = [ + "a.module", + ["a", "a.module"], + ["a.bar"], + [], + """\ +a/__init__.py + def foo(): pass +a/module.py + from . import foo + from . import bar +"""] + +relative_import_test_4 = [ + "a.module", + ["a", "a.module"], + [], + [], + """\ +a/__init__.py + def foo(): pass +a/module.py + from . import * +"""] + +bytecode_test = [ + "a", + ["a"], + [], + [], + "" +] + +syntax_error_test = [ + "a.module", + ["a", "a.module", "b"], + ["b.module"], [], + """\ +a/__init__.py +a/module.py + import b.module +b/__init__.py +b/module.py + ? # SyntaxError: invalid syntax +"""] + + +same_name_as_bad_test = [ + "a.module", + ["a", "a.module", "b", "b.c"], + ["c"], [], + """\ +a/__init__.py +a/module.py + import c + from b import c +b/__init__.py +b/c.py +"""] + +coding_default_utf8_test = [ + "a_utf8", + ["a_utf8", "b_utf8"], + [], [], + """\ +a_utf8.py + # use the default of utf8 + print('Unicode test A code point 2090 \u2090 that is not valid in cp1252') + import b_utf8 +b_utf8.py + # use the default of utf8 + print('Unicode test B code point 2090 \u2090 that is not valid in cp1252') +"""] + +coding_explicit_utf8_test = [ + "a_utf8", + ["a_utf8", "b_utf8"], + [], [], + """\ +a_utf8.py + # coding=utf8 + print('Unicode test A code point 2090 \u2090 that is not valid in cp1252') + import b_utf8 +b_utf8.py + # use the default of utf8 + print('Unicode test B code point 2090 \u2090 that is not valid in cp1252') +"""] + +coding_explicit_cp1252_test = [ + "a_cp1252", + ["a_cp1252", "b_utf8"], + [], [], + b"""\ +a_cp1252.py + # coding=cp1252 + # 0xe2 is not allowed in utf8 + print('CP1252 test P\xe2t\xe9') + import b_utf8 +""" + """\ +b_utf8.py + # use the default of utf8 + print('Unicode test A code point 2090 \u2090 that is not valid in cp1252') +""".encode('utf-8')] + +def open_file(path): + dirname = os.path.dirname(path) + try: + os.makedirs(dirname) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return open(path, 'wb') + + +def create_package(test_dir, source): + ofi = None + try: + for line in source.splitlines(): + if type(line) != bytes: + line = line.encode('utf-8') + if line.startswith(b' ') or line.startswith(b'\t'): + ofi.write(line.strip() + b'\n') + else: + if ofi: + ofi.close() + if type(line) == bytes: + line = line.decode('utf-8') + ofi = open_file(os.path.join(test_dir, line.strip())) + finally: + if ofi: + ofi.close() + +class ModuleFinderTest(unittest.TestCase): + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.test_path = [self.test_dir, os.path.dirname(tempfile.__file__)] + + def tearDown(self): + shutil.rmtree(self.test_dir) + + def _do_test(self, info, report=False, debug=0, replace_paths=[], modulefinder_class=modulefinder.ModuleFinder): + import_this, modules, missing, maybe_missing, source = info + create_package(self.test_dir, source) + mf = modulefinder_class(path=self.test_path, debug=debug, + replace_paths=replace_paths) + mf.import_hook(import_this) + if report: + mf.report() +## # This wouldn't work in general when executed several times: +## opath = sys.path[:] +## sys.path = self.test_path +## try: +## __import__(import_this) +## except: +## import traceback; traceback.print_exc() +## sys.path = opath +## return + modules = sorted(set(modules)) + found = sorted(mf.modules) + # check if we found what we expected, not more, not less + self.assertEqual(found, modules) + + # check for missing and maybe missing modules + bad, maybe = mf.any_missing_maybe() + self.assertEqual(bad, missing) + self.assertEqual(maybe, maybe_missing) + + def test_package(self): + self._do_test(package_test) + + def test_maybe(self): + self._do_test(maybe_test) + + def test_maybe_new(self): + self._do_test(maybe_test_new) + + def test_absolute_imports(self): + self._do_test(absolute_import_test) + + def test_relative_imports(self): + self._do_test(relative_import_test) + + def test_relative_imports_2(self): + self._do_test(relative_import_test_2) + + def test_relative_imports_3(self): + self._do_test(relative_import_test_3) + + def test_relative_imports_4(self): + self._do_test(relative_import_test_4) + + def test_syntax_error(self): + self._do_test(syntax_error_test) + + def test_same_name_as_bad(self): + self._do_test(same_name_as_bad_test) + + def test_bytecode(self): + base_path = os.path.join(self.test_dir, 'a') + source_path = base_path + importlib.machinery.SOURCE_SUFFIXES[0] + bytecode_path = base_path + importlib.machinery.BYTECODE_SUFFIXES[0] + with open_file(source_path) as file: + file.write('testing_modulefinder = True\n'.encode('utf-8')) + py_compile.compile(source_path, cfile=bytecode_path) + os.remove(source_path) + self._do_test(bytecode_test) + + # TODO: RUSTPYTHON; panics at code.rs with 'called Option::unwrap() on a None value' + @unittest.skip("TODO: RUSTPYTHON; panics in co_filename replacement") + def test_replace_paths(self): + old_path = os.path.join(self.test_dir, 'a', 'module.py') + new_path = os.path.join(self.test_dir, 'a', 'spam.py') + with support.captured_stdout() as output: + self._do_test(maybe_test, debug=2, + replace_paths=[(old_path, new_path)]) + output = output.getvalue() + expected = "co_filename %r changed to %r" % (old_path, new_path) + self.assertIn(expected, output) + + def test_extended_opargs(self): + extended_opargs_test = [ + "a", + ["a", "b"], + [], [], + """\ +a.py + %r + import b +b.py +""" % list(range(2**16))] # 2**16 constants + self._do_test(extended_opargs_test) + + def test_coding_default_utf8(self): + self._do_test(coding_default_utf8_test) + + def test_coding_explicit_utf8(self): + self._do_test(coding_explicit_utf8_test) + + def test_coding_explicit_cp1252(self): + self._do_test(coding_explicit_cp1252_test) + + def test_load_module_api(self): + class CheckLoadModuleApi(modulefinder.ModuleFinder): + def __init__(self, *args, **kwds): + super().__init__(*args, **kwds) + + def load_module(self, fqname, fp, pathname, file_info): + # confirm that the fileinfo is a tuple of 3 elements + suffix, mode, type = file_info + return super().load_module(fqname, fp, pathname, file_info) + + self._do_test(absolute_import_test, modulefinder_class=CheckLoadModuleApi) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py new file mode 100644 index 00000000000..ed28ae07f86 --- /dev/null +++ b/Lib/test/test_monitoring.py @@ -0,0 +1,2466 @@ +"""Test suite for the sys.monitoring.""" + +import collections +import dis +import functools +import inspect +import math +import operator +import sys +import textwrap +import types +import unittest + +import test.support +from test.support import requires_specialization_ft, script_helper + +try: + import _testcapi +except ImportError: + _testcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + +PAIR = (0,1) + +def f1(): + pass + +def f2(): + len([]) + sys.getsizeof(0) + +def floop(): + for item in PAIR: + pass + +def gen(): + yield + yield + +def g1(): + for _ in gen(): + pass + +TEST_TOOL = 2 +TEST_TOOL2 = 3 +TEST_TOOL3 = 4 + +def nth_line(func, offset): + return func.__code__.co_firstlineno + offset + +class MonitoringBasicTest(unittest.TestCase): + + def tearDown(self): + sys.monitoring.free_tool_id(TEST_TOOL) + + def test_has_objects(self): + m = sys.monitoring + m.events + m.use_tool_id + m.clear_tool_id + m.free_tool_id + m.get_tool + m.get_events + m.set_events + m.get_local_events + m.set_local_events + m.register_callback + m.restart_events + m.DISABLE + m.MISSING + m.events.NO_EVENTS + + def test_tool(self): + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), "MonitoringTest.Tool") + sys.monitoring.set_events(TEST_TOOL, 15) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 15) + sys.monitoring.set_events(TEST_TOOL, 0) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RETURN) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.C_RAISE) + sys.monitoring.free_tool_id(TEST_TOOL) + self.assertEqual(sys.monitoring.get_tool(TEST_TOOL), None) + with self.assertRaises(ValueError): + sys.monitoring.set_events(TEST_TOOL, sys.monitoring.events.CALL) + + def test_clear(self): + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, lambda *args: events.append(args)) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + def f(): + a = 1 + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + + f() + sys.monitoring.clear_tool_id(TEST_TOOL) + f() + + # the first f() should trigger a PY_START and a LINE event + # the second f() after clear_tool_id should not trigger any event + # the callback function should be cleared as well + self.assertEqual(len(events), 2) + callback = sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertIs(callback, None) + + sys.monitoring.free_tool_id(TEST_TOOL) + + events = [] + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + sys.monitoring.register_callback(TEST_TOOL, E.LINE, lambda *args: events.append(args)) + sys.monitoring.set_local_events(TEST_TOOL, f.__code__, E.LINE) + f() + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.use_tool_id(TEST_TOOL, "MonitoringTest.Tool") + f() + # the first f() should trigger a LINE event, and even if we use the + # tool id immediately after freeing it, the second f() should not + # trigger any event + self.assertEqual(len(events), 1) + sys.monitoring.free_tool_id(TEST_TOOL) + + +class MonitoringTestBase: + + def setUp(self): + # Check that a previous test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL2), None) + self.assertIs(sys.monitoring.get_tool(TEST_TOOL3), None) + sys.monitoring.use_tool_id(TEST_TOOL, "test " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL2, "test2 " + self.__class__.__name__) + sys.monitoring.use_tool_id(TEST_TOOL3, "test3 " + self.__class__.__name__) + + def tearDown(self): + # Check that test hasn't left monitoring on. + for tool in range(6): + self.assertEqual(sys.monitoring.get_events(tool), 0) + sys.monitoring.free_tool_id(TEST_TOOL) + sys.monitoring.free_tool_id(TEST_TOOL2) + sys.monitoring.free_tool_id(TEST_TOOL3) + + +class MonitoringCountTest(MonitoringTestBase, unittest.TestCase): + + def check_event_count(self, func, event, expected): + + class Counter: + def __init__(self): + self.count = 0 + def __call__(self, *args): + self.count += 1 + + counter = Counter() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + self.assertEqual(counter.count, 0) + counter.count = 0 + func() + self.assertEqual(counter.count, expected) + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + counter.count = 0 + func() + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_start_count(self): + self.check_event_count(f1, E.PY_START, 1) + + def test_resume_count(self): + self.check_event_count(g1, E.PY_RESUME, 2) + + def test_return_count(self): + self.check_event_count(f1, E.PY_RETURN, 1) + + def test_call_count(self): + self.check_event_count(f2, E.CALL, 3) + + def test_c_return_count(self): + self.check_event_count(f2, E.C_RETURN, 2) + + +E = sys.monitoring.events + +INSTRUMENTED_EVENTS = [ + (E.PY_START, "start"), + (E.PY_RESUME, "resume"), + (E.PY_RETURN, "return"), + (E.PY_YIELD, "yield"), + (E.JUMP, "jump"), + (E.BRANCH, "branch"), +] + +EXCEPT_EVENTS = [ + (E.RAISE, "raise"), + (E.PY_UNWIND, "unwind"), + (E.EXCEPTION_HANDLED, "exception_handled"), +] + +SIMPLE_EVENTS = INSTRUMENTED_EVENTS + EXCEPT_EVENTS + [ + (E.C_RAISE, "c_raise"), + (E.C_RETURN, "c_return"), +] + + +SIMPLE_EVENT_SET = functools.reduce(operator.or_, [ev for (ev, _) in SIMPLE_EVENTS], 0) | E.CALL + + +def just_pass(): + pass + +just_pass.events = [ + "py_call", + "start", + "return", +] + +def just_raise(): + raise Exception + +just_raise.events = [ + 'py_call', + "start", + "raise", + "unwind", +] + +def just_call(): + len([]) + +just_call.events = [ + 'py_call', + "start", + "c_call", + "c_return", + "return", +] + +def caught(): + try: + 1/0 + except Exception: + pass + +caught.events = [ + 'py_call', + "start", + "raise", + "exception_handled", + "branch", + "return", +] + +def nested_call(): + just_pass() + +nested_call.events = [ + "py_call", + "start", + "py_call", + "start", + "return", + "return", +] + +PY_CALLABLES = (types.FunctionType, types.MethodType) + +class MonitoringEventsBase(MonitoringTestBase): + + def gather_events(self, func): + events = [] + for event, event_name in SIMPLE_EVENTS: + def record(*args, event_name=event_name): + events.append(event_name) + sys.monitoring.register_callback(TEST_TOOL, event, record) + def record_call(code, offset, obj, arg): + if isinstance(obj, PY_CALLABLES): + events.append("py_call") + else: + events.append("c_call") + sys.monitoring.register_callback(TEST_TOOL, E.CALL, record_call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + events = [] + try: + func() + except: + pass + sys.monitoring.set_events(TEST_TOOL, 0) + #Remove the final event, the call to `sys.monitoring.set_events` + events = events[:-1] + return events + + def check_events(self, func, expected=None): + events = self.gather_events(func) + if expected is None: + expected = func.events + self.assertEqual(events, expected) + +class MonitoringEventsTest(MonitoringEventsBase, unittest.TestCase): + + def test_just_pass(self): + self.check_events(just_pass) + + def test_just_raise(self): + try: + self.check_events(just_raise) + except Exception: + pass + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + + def test_just_call(self): + self.check_events(just_call) + + def test_caught(self): + self.check_events(caught) + + def test_nested_call(self): + self.check_events(nested_call) + +UP_EVENTS = (E.C_RETURN, E.C_RAISE, E.PY_RETURN, E.PY_UNWIND, E.PY_YIELD) +DOWN_EVENTS = (E.PY_START, E.PY_RESUME) + +from test.profilee import testfunc + +class SimulateProfileTest(MonitoringEventsBase, unittest.TestCase): + + def test_balanced(self): + events = self.gather_events(testfunc) + c = collections.Counter(events) + self.assertEqual(c["c_call"], c["c_return"]) + self.assertEqual(c["start"], c["return"] + c["unwind"]) + self.assertEqual(c["raise"], c["exception_handled"] + c["unwind"]) + + def test_frame_stack(self): + self.maxDiff = None + stack = [] + errors = [] + seen = set() + def up(*args): + frame = sys._getframe(1) + if not stack: + errors.append("empty") + else: + expected = stack.pop() + if frame != expected: + errors.append(f" Popping {frame} expected {expected}") + def down(*args): + frame = sys._getframe(1) + stack.append(frame) + seen.add(frame.f_code) + def call(code, offset, callable, arg): + if not isinstance(callable, PY_CALLABLES): + stack.append(sys._getframe(1)) + for event in UP_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, up) + for event in DOWN_EVENTS: + sys.monitoring.register_callback(TEST_TOOL, event, down) + sys.monitoring.register_callback(TEST_TOOL, E.CALL, call) + sys.monitoring.set_events(TEST_TOOL, SIMPLE_EVENT_SET) + testfunc() + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(errors, []) + self.assertEqual(stack, [sys._getframe()]) + self.assertEqual(len(seen), 9) + + +class CounterWithDisable: + + def __init__(self): + self.disable = False + self.count = 0 + + def __call__(self, *args): + self.count += 1 + if self.disable: + return sys.monitoring.DISABLE + + +class RecorderWithDisable: + + def __init__(self, events): + self.disable = False + self.events = events + + def __call__(self, code, event): + self.events.append(event) + if self.disable: + return sys.monitoring.DISABLE + + +class MontoringDisableAndRestartTest(MonitoringTestBase, unittest.TestCase): + + def test_disable(self): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(counter.count, 0) + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.disable = True + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() + + def test_restart(self): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + counter.disable = True + f1() + counter.count = 0 + f1() + self.assertEqual(counter.count, 0) + sys.monitoring.restart_events() + counter.count = 0 + f1() + self.assertEqual(counter.count, 1) + sys.monitoring.set_events(TEST_TOOL, 0) + finally: + sys.monitoring.restart_events() + + +class MultipleMonitorsTest(MonitoringTestBase, unittest.TestCase): + + def test_two_same(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_three_same(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + counter3 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, counter3) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + sys.monitoring.set_events(TEST_TOOL3, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL3), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2) | (1 << TEST_TOOL3)}) + counter1.count = 0 + counter2.count = 0 + counter3.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + count3 = counter3.count + self.assertEqual((count1, count2, count3), (1, 1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.set_events(TEST_TOOL3, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL3, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_two_different(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_RETURN) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_RETURN) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': 1 << TEST_TOOL, 'PY_RETURN': 1 << TEST_TOOL2}) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_RETURN, None) + self.assertEqual(sys.monitoring._all_events(), {}) + + def test_two_with_disable(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + counter1 = CounterWithDisable() + counter2 = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, counter1) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, counter2) + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + self.assertEqual(sys.monitoring._all_events(), {'PY_START': (1 << TEST_TOOL) | (1 << TEST_TOOL2)}) + counter1.count = 0 + counter2.count = 0 + counter1.disable = True + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (1, 1)) + counter1.count = 0 + counter2.count = 0 + f1() + count1 = counter1.count + count2 = counter2.count + self.assertEqual((count1, count2), (0, 1)) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.PY_START, None) + sys.monitoring.register_callback(TEST_TOOL2, E.PY_START, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_with_instruction_event(self): + """Test that the second tool can set events with instruction events set by the first tool.""" + def f(): + pass + code = f.__code__ + + try: + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.set_local_events(TEST_TOOL, code, E.INSTRUCTION | E.LINE) + sys.monitoring.set_local_events(TEST_TOOL2, code, E.LINE) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + self.assertEqual(sys.monitoring._all_events(), {}) + + +class LineMonitoringTest(MonitoringTestBase, unittest.TestCase): + + def test_lines_single(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + start = nth_line(LineMonitoringTest.test_lines_single, 0) + self.assertEqual(events, [start+7, nth_line(f1, 1), start+8]) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_lines_loop(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + floop() + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + start = nth_line(LineMonitoringTest.test_lines_loop, 0) + floop_1 = nth_line(floop, 1) + floop_2 = nth_line(floop, 2) + self.assertEqual( + events, + [start+7, floop_1, floop_2, floop_1, floop_2, floop_1, start+8] + ) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def test_lines_two(self): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + events2 = [] + recorder2 = RecorderWithDisable(events2) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, recorder2) + sys.monitoring.set_events(TEST_TOOL, E.LINE); sys.monitoring.set_events(TEST_TOOL2, E.LINE) + f1() + sys.monitoring.set_events(TEST_TOOL, 0); sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + start = nth_line(LineMonitoringTest.test_lines_two, 0) + expected = [start+10, nth_line(f1, 1), start+11] + self.assertEqual(events, expected) + self.assertEqual(events2, expected) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.set_events(TEST_TOOL2, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.register_callback(TEST_TOOL2, E.LINE, None) + self.assertEqual(sys.monitoring._all_events(), {}) + sys.monitoring.restart_events() + + def check_lines(self, func, expected, tool=TEST_TOOL): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = RecorderWithDisable(events) + sys.monitoring.register_callback(tool, E.LINE, recorder) + sys.monitoring.set_events(tool, E.LINE) + func() + sys.monitoring.set_events(tool, 0) + sys.monitoring.register_callback(tool, E.LINE, None) + lines = [ line - func.__code__.co_firstlineno for line in events[1:-1] ] + self.assertEqual(lines, expected) + finally: + sys.monitoring.set_events(tool, 0) + + + def test_linear(self): + + def func(): + line = 1 + line = 2 + line = 3 + line = 4 + line = 5 + + self.check_lines(func, [1,2,3,4,5]) + + def test_branch(self): + def func(): + if "true".startswith("t"): + line = 2 + line = 3 + else: + line = 5 + line = 6 + + self.check_lines(func, [1,2,3,6]) + + def test_try_except(self): + + def func1(): + try: + line = 2 + line = 3 + except: + line = 5 + line = 6 + + self.check_lines(func1, [1,2,3,6]) + + def func2(): + try: + line = 2 + raise 3 + except: + line = 5 + line = 6 + + self.check_lines(func2, [1,2,3,4,5,6]) + + def test_generator_with_line(self): + + def f(): + def a(): + yield + def b(): + yield from a() + next(b()) + + self.check_lines(f, [1,3,5,4,2,4]) + +class TestDisable(MonitoringTestBase, unittest.TestCase): + + def gen(self, cond): + for i in range(10): + if cond: + yield 1 + else: + yield 2 + + def raise_handle_reraise(self): + try: + 1/0 + except: + raise + + def test_disable_legal_events(self): + for event, name in INSTRUMENTED_EVENTS: + try: + counter = CounterWithDisable() + counter.disable = True + sys.monitoring.register_callback(TEST_TOOL, event, counter) + sys.monitoring.set_events(TEST_TOOL, event) + for _ in self.gen(1): + pass + self.assertLess(counter.count, 4) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, event, None) + + + def test_disable_illegal_events(self): + for event, name in EXCEPT_EVENTS: + try: + counter = CounterWithDisable() + counter.disable = True + sys.monitoring.register_callback(TEST_TOOL, event, counter) + sys.monitoring.set_events(TEST_TOOL, event) + with self.assertRaises(ValueError): + self.raise_handle_reraise() + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, event, None) + + +class ExceptionRecorder: + + event_type = E.RAISE + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, exc): + self.events.append(("raise", type(exc))) + +class CheckEvents(MonitoringTestBase, unittest.TestCase): + + def get_events(self, func, tool, recorders): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_events(tool, all_events) + func() + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + return event_list + finally: + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + def check_events(self, func, expected, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + events = self.get_events(func, tool, recorders) + self.assertEqual(events, expected) + + def check_balanced(self, func, recorders): + events = self.get_events(func, TEST_TOOL, recorders) + self.assertEqual(len(events)%2, 0) + for r, h in zip(events[::2],events[1::2]): + r0 = r[0] + self.assertIn(r0, ("raise", "reraise")) + h0 = h[0] + self.assertIn(h0, ("handled", "unwind")) + self.assertEqual(r[1], h[1]) + + +class StopiterationRecorder(ExceptionRecorder): + + event_type = E.STOP_ITERATION + +class ReraiseRecorder(ExceptionRecorder): + + event_type = E.RERAISE + + def __call__(self, code, offset, exc): + self.events.append(("reraise", type(exc))) + +class UnwindRecorder(ExceptionRecorder): + + event_type = E.PY_UNWIND + + def __call__(self, code, offset, exc): + self.events.append(("unwind", type(exc), code.co_name)) + +class ExceptionHandledRecorder(ExceptionRecorder): + + event_type = E.EXCEPTION_HANDLED + + def __call__(self, code, offset, exc): + self.events.append(("handled", type(exc))) + +class ThrowRecorder(ExceptionRecorder): + + event_type = E.PY_THROW + + def __call__(self, code, offset, exc): + self.events.append(("throw", type(exc))) + +class CallRecorder: + + event_type = E.CALL + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append(("call", func.__name__, arg)) + +class ReturnRecorder: + + event_type = E.PY_RETURN + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, val): + self.events.append(("return", code.co_name, val)) + + +class ExceptionMonitoringTest(CheckEvents): + + exception_recorders = ( + ExceptionRecorder, + ReraiseRecorder, + ExceptionHandledRecorder, + UnwindRecorder + ) + + def test_simple_try_except(self): + + def func1(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func1, [("raise", KeyError)]) + + @unittest.skipUnless(_testinternalcapi, "requires _testinternalcapi") + def test_implicit_stop_iteration(self): + """Generators are documented as raising a StopIteration + when they terminate. + However, we don't do that if we can avoid it, for speed. + sys.monitoring handles that by injecting a STOP_ITERATION + event when we would otherwise have skip the RAISE event. + This test checks that both paths record an equivalent event. + """ + + def gen(): + yield 1 + return 2 + + def implicit_stop_iteration(iterator=None): + if iterator is None: + iterator = gen() + for _ in iterator: + pass + + recorders=(ExceptionRecorder, StopiterationRecorder,) + expected = [("raise", StopIteration)] + + # Make sure that the loop is unspecialized, and that it will not + # re-specialize immediately, so that we can we can test the + # unspecialized version of the loop first. + # Note: this assumes that we don't specialize loops over sets. + implicit_stop_iteration(set(range(_testinternalcapi.SPECIALIZATION_THRESHOLD))) + + # This will record a RAISE event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) + + # Now specialize, so that we see a STOP_ITERATION event. + for _ in range(_testinternalcapi.SPECIALIZATION_COOLDOWN): + implicit_stop_iteration() + + # This will record a STOP_ITERATION event for the StopIteration. + self.check_events(implicit_stop_iteration, expected, recorders=recorders) + + initial = [ + ("raise", ZeroDivisionError), + ("handled", ZeroDivisionError) + ] + + reraise = [ + ("reraise", ZeroDivisionError), + ("handled", ZeroDivisionError) + ] + + def test_explicit_reraise(self): + + def func(): + try: + try: + 1/0 + except: + raise + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_explicit_reraise_named(self): + + def func(): + try: + try: + 1/0 + except Exception as ex: + raise + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_implicit_reraise(self): + + def func(): + try: + try: + 1/0 + except ValueError: + pass + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + + def test_implicit_reraise_named(self): + + def func(): + try: + try: + 1/0 + except ValueError as ex: + pass + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_try_finally(self): + + def func(): + try: + try: + 1/0 + finally: + pass + except: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_async_for(self): + + def func(): + + async def async_generator(): + for i in range(1): + raise ZeroDivisionError + yield i + + async def async_loop(): + try: + async for item in async_generator(): + pass + except Exception: + pass + + try: + async_loop().send(None) + except StopIteration: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + def test_throw(self): + + def gen(): + yield 1 + yield 2 + + def func(): + try: + g = gen() + next(g) + g.throw(IndexError) + except IndexError: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + events = self.get_events( + func, + TEST_TOOL, + self.exception_recorders + (ThrowRecorder,) + ) + self.assertEqual(events[0], ("throw", IndexError)) + + @unittest.skipUnless(_testinternalcapi, "requires _testinternalcapi") + @requires_specialization_ft + def test_no_unwind_for_shim_frame(self): + class ValueErrorRaiser: + def __init__(self): + raise ValueError() + + def f(): + try: + return ValueErrorRaiser() + except ValueError: + pass + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f() + recorders = ( + ReturnRecorder, + UnwindRecorder + ) + events = self.get_events(f, TEST_TOOL, recorders) + adaptive_insts = dis.get_instructions(f, adaptive=True) + self.assertIn( + "CALL_ALLOC_AND_ENTER_INIT", + [i.opname for i in adaptive_insts] + ) + #There should be only one unwind event + expected = [ + ('unwind', ValueError, '__init__'), + ('return', 'f', None), + ] + + self.assertEqual(events, expected) + + # gh-140373 + def test_gen_unwind(self): + def gen(): + yield 1 + + def f(): + g = gen() + next(g) + g.close() + + recorders = ( + UnwindRecorder, + ) + events = self.get_events(f, TEST_TOOL, recorders) + expected = [ + ("unwind", GeneratorExit, "gen"), + ] + self.assertEqual(events, expected) + +class LineRecorder: + + event_type = E.LINE + + + def __init__(self, events): + self.events = events + + def __call__(self, code, line): + self.events.append(("line", code.co_name, line - code.co_firstlineno)) + +class CEventRecorder: + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, func, arg): + self.events.append((self.event_name, func.__name__, arg)) + +class CReturnRecorder(CEventRecorder): + + event_type = E.C_RETURN + event_name = "C return" + +class CRaiseRecorder(CEventRecorder): + + event_type = E.C_RAISE + event_name = "C raise" + +MANY_RECORDERS = ExceptionRecorder, CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder + +class TestManyEvents(CheckEvents): + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = MANY_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('call', 'func1', sys.monitoring.MISSING), + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3), + ('line', 'get_events', 11), + ('call', 'set_events', 2)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = MANY_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('call', 'func2', sys.monitoring.MISSING), + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3), + ('line', 'get_events', 11), + ('call', 'set_events', 2)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = MANY_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('call', 'func3', sys.monitoring.MISSING), + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('raise', KeyError), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6), + ('line', 'get_events', 11), + ('call', 'set_events', 2)]) + +class InstructionRecorder: + + event_type = E.INSTRUCTION + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset): + # Filter out instructions in check_events to lower noise + if code.co_name != "get_events": + self.events.append(("instruction", code.co_name, offset)) + + +LINE_AND_INSTRUCTION_RECORDERS = InstructionRecorder, LineRecorder + +class TestLineAndInstructionEvents(CheckEvents): + maxDiff = None + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('instruction', 'func1', 16), + ('line', 'get_events', 11)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func2', 1), + ('instruction', 'func2', 2), + ('instruction', 'func2', 4), + ('line', 'func2', 2), + ('instruction', 'func2', 6), + ('instruction', 'func2', 8), + ('instruction', 'func2', 28), + ('instruction', 'func2', 30), + ('instruction', 'func2', 38), + ('line', 'func2', 3), + ('instruction', 'func2', 40), + ('instruction', 'func2', 42), + ('instruction', 'func2', 44), + ('instruction', 'func2', 46), + ('line', 'get_events', 11)]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - instruction offsets differ from CPython + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func3', 1), + ('instruction', 'func3', 2), + ('line', 'func3', 2), + ('instruction', 'func3', 4), + ('instruction', 'func3', 6), + ('line', 'func3', 3), + ('instruction', 'func3', 8), + ('instruction', 'func3', 18), + ('instruction', 'func3', 20), + ('line', 'func3', 4), + ('instruction', 'func3', 22), + ('line', 'func3', 5), + ('instruction', 'func3', 24), + ('instruction', 'func3', 26), + ('instruction', 'func3', 28), + ('line', 'func3', 6), + ('instruction', 'func3', 30), + ('instruction', 'func3', 32), + ('instruction', 'func3', 34), + ('instruction', 'func3', 36), + ('line', 'get_events', 11)]) + + def test_with_restart(self): + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('instruction', 'func1', 16), + ('line', 'get_events', 11)]) + + sys.monitoring.restart_events() + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('instruction', 'func1', 16), + ('line', 'get_events', 11)]) + + def test_turn_off_only_instruction(self): + """ + LINE events should be recorded after INSTRUCTION event is turned off + """ + events = [] + def line(*args): + events.append("line") + sys.monitoring.set_events(TEST_TOOL, 0) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, line) + sys.monitoring.register_callback(TEST_TOOL, E.INSTRUCTION, lambda *args: None) + sys.monitoring.set_events(TEST_TOOL, E.LINE | E.INSTRUCTION) + sys.monitoring.set_events(TEST_TOOL, E.LINE) + events = [] + a = 0 + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertGreater(len(events), 0) + +class TestInstallIncrementally(MonitoringTestBase, unittest.TestCase): + + def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + all_events |= recorder.event_type + sys.monitoring.set_events(tool, all_events) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, recorder(event_list)) + func() + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + for line in must_include: + self.assertIn(line, event_list) + finally: + sys.monitoring.set_events(tool, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + @staticmethod + def func1(): + line1 = 1 + + MUST_INCLUDE_LI = [ + ('instruction', 'func1', 2), + ('line', 'func1', 2), + ('instruction', 'func1', 4), + ('instruction', 'func1', 6)] + + def test_line_then_instruction(self): + recorders = [ LineRecorder, InstructionRecorder ] + self.check_events(self.func1, + recorders = recorders, must_include = self.MUST_INCLUDE_LI) + + def test_instruction_then_line(self): + recorders = [ InstructionRecorder, LineRecorder ] + self.check_events(self.func1, + recorders = recorders, must_include = self.MUST_INCLUDE_LI) + + @staticmethod + def func2(): + len(()) + + MUST_INCLUDE_CI = [ + ('instruction', 'func2', 2), + ('call', 'func2', sys.monitoring.MISSING), + ('call', 'len', ()), + ('instruction', 'func2', 12), + ('instruction', 'func2', 14)] + + + + def test_call_then_instruction(self): + recorders = [ CallRecorder, InstructionRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + + def test_instruction_then_call(self): + recorders = [ InstructionRecorder, CallRecorder ] + self.check_events(self.func2, + recorders = recorders, must_include = self.MUST_INCLUDE_CI) + +LOCAL_RECORDERS = CallRecorder, LineRecorder, CReturnRecorder, CRaiseRecorder + +class TestLocalEvents(MonitoringTestBase, unittest.TestCase): + + def check_events(self, func, expected, tool=TEST_TOOL, recorders=()): + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(tool, func.__code__, all_events) + func() + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + self.assertEqual(event_list, expected) + finally: + sys.monitoring.set_local_events(tool, func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + + def test_simple(self): + + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LOCAL_RECORDERS, expected = [ + ('line', 'func1', 1), + ('line', 'func1', 2), + ('line', 'func1', 3)]) + + def test_c_call(self): + + def func2(): + line1 = 1 + [].append(2) + line3 = 3 + + self.check_events(func2, recorders = LOCAL_RECORDERS, expected = [ + ('line', 'func2', 1), + ('line', 'func2', 2), + ('call', 'append', [2]), + ('C return', 'append', [2]), + ('line', 'func2', 3)]) + + def test_try_except(self): + + def func3(): + try: + line = 2 + raise KeyError + except: + line = 5 + line = 6 + + self.check_events(func3, recorders = LOCAL_RECORDERS, expected = [ + ('line', 'func3', 1), + ('line', 'func3', 2), + ('line', 'func3', 3), + ('line', 'func3', 4), + ('line', 'func3', 5), + ('line', 'func3', 6)]) + + def test_set_non_local_event(self): + with self.assertRaises(ValueError): + sys.monitoring.set_local_events(TEST_TOOL, just_call.__code__, E.RAISE) + +def line_from_offset(code, offset): + for start, end, line in code.co_lines(): + if start <= offset < end: + if line is None: + return f"[offset={offset}]" + return line - code.co_firstlineno + return -1 + +class JumpRecorder: + + event_type = E.JUMP + name = "jump" + + def __init__(self, events): + self.events = events + + def __call__(self, code, from_, to): + from_line = line_from_offset(code, from_) + to_line = line_from_offset(code, to) + self.events.append((self.name, code.co_name, from_line, to_line)) + + +class BranchRecorder(JumpRecorder): + + event_type = E.BRANCH + name = "branch" + +class BranchRightRecorder(JumpRecorder): + + event_type = E.BRANCH_RIGHT + name = "branch right" + +class BranchLeftRecorder(JumpRecorder): + + event_type = E.BRANCH_LEFT + name = "branch left" + +class JumpOffsetRecorder: + + event_type = E.JUMP + name = "jump" + + def __init__(self, events, offsets=False): + self.events = events + + def __call__(self, code, from_, to): + self.events.append((self.name, code.co_name, from_, to)) + +class BranchLeftOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH_LEFT + name = "branch left" + +class BranchRightOffsetRecorder(JumpOffsetRecorder): + + event_type = E.BRANCH_RIGHT + name = "branch right" + + +JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder +JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder +FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder + +BRANCHES_RECORDERS = BranchLeftRecorder, BranchRightRecorder +BRANCH_OFFSET_RECORDERS = BranchLeftOffsetRecorder, BranchRightOffsetRecorder + +class TestBranchAndJumpEvents(CheckEvents): + maxDiff = None + + @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython + def test_loop(self): + + def func(): + x = 1 + for a in range(2): + if a: + x = 4 + else: + x = 6 + 7 + + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + + self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [ + ('branch', 'func', 2, 2), + ('branch', 'func', 3, 6), + ('jump', 'func', 6, 2), + ('branch', 'func', 2, 2), + ('branch', 'func', 3, 4), + ('jump', 'func', 4, 2), + ('branch', 'func', 2, 7)]) + + self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func', 1), + ('line', 'func', 2), + ('branch', 'func', 2, 2), + ('line', 'func', 3), + ('branch', 'func', 3, 6), + ('line', 'func', 6), + ('jump', 'func', 6, 2), + ('line', 'func', 2), + ('branch', 'func', 2, 2), + ('line', 'func', 3), + ('branch', 'func', 3, 4), + ('line', 'func', 4), + ('jump', 'func', 4, 2), + ('line', 'func', 2), + ('branch', 'func', 2, 7), + ('line', 'func', 7), + ('line', 'get_events', 11)]) + + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'func', 2, 2), + ('branch right', 'func', 3, 6), + ('branch left', 'func', 2, 2), + ('branch left', 'func', 3, 4), + ('branch right', 'func', 2, 7)]) + + self.check_events(whilefunc, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch left', 'whilefunc', 1, 2), + ('branch right', 'whilefunc', 1, 3)]) + + self.check_events(func, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + ('branch left', 'func', 28, 32), + ('branch right', 'func', 44, 58), + ('branch left', 'func', 28, 32), + ('branch left', 'func', 44, 50), + ('branch right', 'func', 28, 70)]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython + def test_except_star(self): + + class Foo: + def meth(self): + pass + + def func(): + try: + try: + raise KeyError + except* Exception as e: + f = Foo(); f.meth() + except KeyError: + pass + + + self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func', 1), + ('line', 'func', 2), + ('line', 'func', 3), + ('line', 'func', 4), + ('branch', 'func', 4, 4), + ('line', 'func', 5), + ('line', 'meth', 1), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), + ('line', 'get_events', 11)]) + + self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ + ('line', 'get_events', 10), + ('line', 'func', 1), + ('line', 'func', 2), + ('line', 'func', 3), + ('raise', KeyError), + ('line', 'func', 4), + ('branch', 'func', 4, 4), + ('line', 'func', 5), + ('line', 'meth', 1), + ('return', 'meth', None), + ('jump', 'func', 5, '[offset=120]'), + ('branch', 'func', '[offset=124]', '[offset=130]'), + ('return', 'func', None), + ('line', 'get_events', 11)]) + + def test_while_offset_consistency(self): + + def foo(n=0): + while n<4: + pass + n += 1 + return None + + in_loop = ('branch left', 'foo', 10, 16) + exit_loop = ('branch right', 'foo', 10, 40) + self.check_events(foo, recorders = BRANCH_OFFSET_RECORDERS, expected = [ + in_loop, + in_loop, + in_loop, + in_loop, + exit_loop]) + + def test_async_for(self): + + def func(): + async def gen(): + yield 2 + yield 3 + + async def foo(): + async for y in gen(): + 2 + pass # line 3 + + try: + foo().send(None) + except StopIteration: + pass + + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'foo', 1, 1), + ('branch left', 'foo', 1, 1), + ('branch right', 'foo', 1, 3), + ('branch left', 'func', 12, 12)]) + + + @unittest.expectedFailure # TODO: RUSTPYTHON; - bytecode layout differs from CPython + def test_match(self): + + def func(v=1): + x = 0 + for v in range(4): + match v: + case 1: + x += 1 + case 2: + x += 2 + case _: + x += 3 + return x + + self.check_events(func, recorders = BRANCHES_RECORDERS, expected = [ + ('branch left', 'func', 2, 2), + ('branch right', 'func', 4, 6), + ('branch right', 'func', 6, 8), + ('branch left', 'func', 2, 2), + ('branch left', 'func', 4, 5), + ('branch left', 'func', 2, 2), + ('branch right', 'func', 4, 6), + ('branch left', 'func', 6, 7), + ('branch left', 'func', 2, 2), + ('branch right', 'func', 4, 6), + ('branch right', 'func', 6, 8), + ('branch right', 'func', 2, 10)]) + + def test_callback_set_frame_lineno(self): + def func(s: str) -> int: + if s.startswith("t"): + return 1 + else: + return 0 + + def callback(code, from_, to): + # try set frame.f_lineno + frame = inspect.currentframe() + while frame and frame.f_code is not code: + frame = frame.f_back + + self.assertIsNotNone(frame) + frame.f_lineno = frame.f_lineno + 1 # run next instruction + + sys.monitoring.set_local_events(TEST_TOOL, func.__code__, E.BRANCH_LEFT) + sys.monitoring.register_callback(TEST_TOOL, E.BRANCH_LEFT, callback) + + self.assertEqual(func("true"), 1) + + +class TestBranchConsistency(MonitoringTestBase, unittest.TestCase): + + def check_branches(self, run_func, test_func=None, tool=TEST_TOOL, recorders=BRANCH_OFFSET_RECORDERS): + if test_func is None: + test_func = run_func + try: + self.assertEqual(sys.monitoring._all_events(), {}) + event_list = [] + all_events = 0 + for recorder in recorders: + ev = recorder.event_type + sys.monitoring.register_callback(tool, ev, recorder(event_list)) + all_events |= ev + sys.monitoring.set_local_events(tool, test_func.__code__, all_events) + run_func() + sys.monitoring.set_local_events(tool, test_func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + lefts = set() + rights = set() + for (src, left, right) in test_func.__code__.co_branches(): + lefts.add((src, left)) + rights.add((src, right)) + for event in event_list: + way, _, src, dest = event + if "left" in way: + self.assertIn((src, dest), lefts) + else: + self.assertIn("right", way) + self.assertIn((src, dest), rights) + finally: + sys.monitoring.set_local_events(tool, test_func.__code__, 0) + for recorder in recorders: + sys.monitoring.register_callback(tool, recorder.event_type, None) + + def test_simple(self): + + def func(): + x = 1 + for a in range(2): + if a: + x = 4 + else: + x = 6 + 7 + + self.check_branches(func) + + def whilefunc(n=0): + while n < 3: + n += 1 # line 2 + 3 + + self.check_branches(whilefunc) + + def test_except_star(self): + + class Foo: + def meth(self): + pass + + def func(): + try: + try: + raise KeyError + except* Exception as e: + f = Foo(); f.meth() + except KeyError: + pass + + + self.check_branches(func) + + def test4(self): + + def foo(n=0): + while n<4: + pass + n += 1 + return None + + self.check_branches(foo) + + def test_async_for(self): + + async def gen(): + yield 2 + yield 3 + + async def foo(): + async for y in gen(): + 2 + pass # line 3 + + def func(): + try: + foo().send(None) + except StopIteration: + pass + + self.check_branches(func, foo) + + +class TestLoadSuperAttr(CheckEvents): + RECORDERS = CallRecorder, LineRecorder, CRaiseRecorder, CReturnRecorder + + def _exec(self, co): + d = {} + exec(co, d, d) + return d + + def _exec_super(self, codestr, optimized=False): + # The compiler checks for statically visible shadowing of the name + # `super`, and declines to emit `LOAD_SUPER_ATTR` if shadowing is found. + # So inserting `super = super` prevents the compiler from emitting + # `LOAD_SUPER_ATTR`, and allows us to test that monitoring events for + # `LOAD_SUPER_ATTR` are equivalent to those we'd get from the + # un-optimized `LOAD_GLOBAL super; CALL; LOAD_ATTR` form. + assignment = "x = 1" if optimized else "super = super" + codestr = f"{assignment}\n{textwrap.dedent(codestr)}" + co = compile(codestr, "<string>", "exec") + # validate that we really do have a LOAD_SUPER_ATTR, only when optimized + self.assertEqual(self._has_load_super_attr(co), optimized) + return self._exec(co) + + def _has_load_super_attr(self, co): + has = any(instr.opname == "LOAD_SUPER_ATTR" for instr in dis.get_instructions(co)) + if not has: + has = any( + isinstance(c, types.CodeType) and self._has_load_super_attr(c) + for c in co.co_consts + ) + return has + + def _super_method_call(self, optimized=False): + codestr = """ + class A: + def method(self, x): + return x + + class B(A): + def method(self, x): + return super( + ).method( + x + ) + + b = B() + def f(): + return b.method(1) + """ + d = self._exec_super(codestr, optimized) + expected = [ + ('line', 'get_events', 10), + ('call', 'f', sys.monitoring.MISSING), + ('line', 'f', 1), + ('call', 'method', d["b"]), + ('line', 'method', 1), + ('call', 'super', sys.monitoring.MISSING), + ('C return', 'super', sys.monitoring.MISSING), + ('line', 'method', 2), + ('line', 'method', 3), + ('line', 'method', 2), + ('call', 'method', d["b"]), + ('line', 'method', 1), + ('line', 'method', 1), + ('line', 'get_events', 11), + ('call', 'set_events', 2), + ] + return d["f"], expected + + @unittest.expectedFailure # TODO: RUSTPYTHON; line number differences in multi-line super() calls + def test_method_call(self): + nonopt_func, nonopt_expected = self._super_method_call(optimized=False) + opt_func, opt_expected = self._super_method_call(optimized=True) + + self.check_events(nonopt_func, recorders=self.RECORDERS, expected=nonopt_expected) + self.check_events(opt_func, recorders=self.RECORDERS, expected=opt_expected) + + def _super_method_call_error(self, optimized=False): + codestr = """ + class A: + def method(self, x): + return x + + class B(A): + def method(self, x): + return super( + x, + self, + ).method( + x + ) + + b = B() + def f(): + try: + return b.method(1) + except TypeError: + pass + else: + assert False, "should have raised TypeError" + """ + d = self._exec_super(codestr, optimized) + expected = [ + ('line', 'get_events', 10), + ('call', 'f', sys.monitoring.MISSING), + ('line', 'f', 1), + ('line', 'f', 2), + ('call', 'method', d["b"]), + ('line', 'method', 1), + ('line', 'method', 2), + ('line', 'method', 3), + ('line', 'method', 1), + ('call', 'super', 1), + ('C raise', 'super', 1), + ('line', 'f', 3), + ('line', 'f', 4), + ('line', 'get_events', 11), + ('call', 'set_events', 2), + ] + return d["f"], expected + + @unittest.expectedFailure # TODO: RUSTPYTHON; line number differences in multi-line super() calls + def test_method_call_error(self): + nonopt_func, nonopt_expected = self._super_method_call_error(optimized=False) + opt_func, opt_expected = self._super_method_call_error(optimized=True) + + self.check_events(nonopt_func, recorders=self.RECORDERS, expected=nonopt_expected) + self.check_events(opt_func, recorders=self.RECORDERS, expected=opt_expected) + + def _super_attr(self, optimized=False): + codestr = """ + class A: + x = 1 + + class B(A): + def method(self): + return super( + ).x + + b = B() + def f(): + return b.method() + """ + d = self._exec_super(codestr, optimized) + expected = [ + ('line', 'get_events', 10), + ('call', 'f', sys.monitoring.MISSING), + ('line', 'f', 1), + ('call', 'method', d["b"]), + ('line', 'method', 1), + ('call', 'super', sys.monitoring.MISSING), + ('C return', 'super', sys.monitoring.MISSING), + ('line', 'method', 2), + ('line', 'method', 1), + ('line', 'get_events', 11), + ('call', 'set_events', 2) + ] + return d["f"], expected + + @unittest.expectedFailure # TODO: RUSTPYTHON; line number differences in multi-line super() calls + def test_attr(self): + nonopt_func, nonopt_expected = self._super_attr(optimized=False) + opt_func, opt_expected = self._super_attr(optimized=True) + + self.check_events(nonopt_func, recorders=self.RECORDERS, expected=nonopt_expected) + self.check_events(opt_func, recorders=self.RECORDERS, expected=opt_expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON; line number differences in multi-line super() calls + def test_vs_other_type_call(self): + code_template = textwrap.dedent(""" + class C: + def method(self): + return {cls}().__repr__{call} + c = C() + def f(): + return c.method() + """) + + def get_expected(name, call_method, ns): + repr_arg = 0 if name == "int" else sys.monitoring.MISSING + return [ + ('line', 'get_events', 10), + ('call', 'f', sys.monitoring.MISSING), + ('line', 'f', 1), + ('call', 'method', ns["c"]), + ('line', 'method', 1), + ('call', name, sys.monitoring.MISSING), + ('C return', name, sys.monitoring.MISSING), + *( + [ + ('call', '__repr__', repr_arg), + ('C return', '__repr__', repr_arg), + ] if call_method else [] + ), + ('line', 'get_events', 11), + ('call', 'set_events', 2), + ] + + for call_method in [True, False]: + with self.subTest(call_method=call_method): + call_str = "()" if call_method else "" + code_super = code_template.format(cls="super", call=call_str) + code_int = code_template.format(cls="int", call=call_str) + co_super = compile(code_super, '<string>', 'exec') + self.assertTrue(self._has_load_super_attr(co_super)) + ns_super = self._exec(co_super) + ns_int = self._exec(code_int) + + self.check_events( + ns_super["f"], + recorders=self.RECORDERS, + expected=get_expected("super", call_method, ns_super) + ) + self.check_events( + ns_int["f"], + recorders=self.RECORDERS, + expected=get_expected("int", call_method, ns_int) + ) + + +class TestSetGetEvents(MonitoringTestBase, unittest.TestCase): + + def test_global(self): + sys.monitoring.set_events(TEST_TOOL, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), E.PY_START) + sys.monitoring.set_events(TEST_TOOL2, E.PY_START) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), E.PY_START) + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL), 0) + sys.monitoring.set_events(TEST_TOOL2,0) + self.assertEqual(sys.monitoring.get_events(TEST_TOOL2), 0) + + def test_local(self): + code = f1.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + sys.monitoring.set_local_events(TEST_TOOL, code, E.BRANCH) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.BRANCH_LEFT | E.BRANCH_RIGHT) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + sys.monitoring.set_local_events(TEST_TOOL2, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), E.PY_START) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + sys.monitoring.set_local_events(TEST_TOOL2, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL2, code), 0) + +class TestUninitialized(unittest.TestCase, MonitoringTestBase): + + @staticmethod + def f(): + pass + + def test_get_local_events_uninitialized(self): + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, self.f.__code__), 0) + +class TestRegressions(MonitoringTestBase, unittest.TestCase): + + def test_105162(self): + caught = None + + def inner(): + nonlocal caught + try: + yield + except Exception: + caught = "inner" + yield + + def outer(): + nonlocal caught + try: + yield from inner() + except Exception: + caught = "outer" + yield + + def run(): + gen = outer() + gen.send(None) + gen.throw(Exception) + run() + self.assertEqual(caught, "inner") + caught = None + try: + sys.monitoring.set_events(TEST_TOOL, E.PY_RESUME) + run() + self.assertEqual(caught, "inner") + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + @unittest.skipUnless(_testinternalcapi, "requires _testinternalcapi") + def test_108390(self): + + class Foo: + def __init__(self, set_event): + if set_event: + sys.monitoring.set_events(TEST_TOOL, E.PY_RESUME) + + def make_foo_optimized_then_set_event(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 1): + Foo(i == _testinternalcapi.SPECIALIZATION_THRESHOLD) + + try: + make_foo_optimized_then_set_event() + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_gh108976(self): + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.LINE, lambda *args: sys.monitoring.set_events(0, 0)) + sys.monitoring.register_callback(0, E.INSTRUCTION, lambda *args: 0) + sys.monitoring.set_events(0, E.LINE | E.INSTRUCTION) + sys.monitoring.set_events(0, 0) + + def test_call_function_ex(self): + def f(a=1, b=2): + return a + b + args = (1, 2) + empty_args = [] + + call_data = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, lambda code, offset, callable, arg0: call_data.append((callable, arg0))) + sys.monitoring.set_events(0, E.CALL) + f(*args) + f(*empty_args) + sys.monitoring.set_events(0, 0) + self.assertEqual(call_data[0], (f, 1)) + self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + + def test_instruction_explicit_callback(self): + # gh-122247 + # Calling the instruction event callback explicitly should not + # crash CPython + def callback(code, instruction_offset): + pass + + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.register_callback(0, sys.monitoring.events.INSTRUCTION, callback) + sys.monitoring.set_events(0, sys.monitoring.events.INSTRUCTION) + callback(None, 0) # call the *same* handler while it is registered + sys.monitoring.restart_events() + sys.monitoring.set_events(0, 0) + + +class TestOptimizer(MonitoringTestBase, unittest.TestCase): + + def test_for_loop(self): + def test_func(x): + i = 0 + while i < x: + i += 1 + + code = test_func.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + test_func(1000) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + +class TestTier2Optimizer(CheckEvents): + + @unittest.skipUnless(_testinternalcapi, "requires _testinternalcapi") + def test_monitoring_already_opimized_loop(self): + def test_func(recorder): + set_events = sys.monitoring.set_events + line = E.LINE + i = 0 + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 51): + # Turn on events without branching once i reaches _testinternalcapi.SPECIALIZATION_THRESHOLD. + set_events(TEST_TOOL, line * int(i >= _testinternalcapi.SPECIALIZATION_THRESHOLD)) + pass + pass + pass + + self.assertEqual(sys.monitoring._all_events(), {}) + events = [] + recorder = LineRecorder(events) + sys.monitoring.register_callback(TEST_TOOL, E.LINE, recorder) + try: + test_func(recorder) + finally: + sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) + sys.monitoring.set_events(TEST_TOOL, 0) + self.assertGreater(len(events), 250) + +class TestMonitoringAtShutdown(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON; - requires subprocess support + def test_monitoring_live_at_shutdown(self): + # gh-115832: An object destructor running during the final GC of + # interpreter shutdown triggered an infinite loop in the + # instrumentation code. + script = test.support.findfile("_test_monitoring_shutdown.py") + script_helper.run_test_script(script) + + +@unittest.skipUnless(_testcapi, "requires _testcapi") +class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase): + + class Scope: + def __init__(self, *args): + self.args = args + + def __enter__(self): + _testcapi.monitoring_enter_scope(*self.args) + + def __exit__(self, *args): + _testcapi.monitoring_exit_scope() + + def setUp(self): + super(TestCApiEventGeneration, self).setUp() + + capi = _testcapi + + self.codelike = capi.CodeLike(2) + + self.cases = [ + # (Event, function, *args) + ( 1, E.PY_START, capi.fire_event_py_start), + ( 1, E.PY_RESUME, capi.fire_event_py_resume), + ( 1, E.PY_YIELD, capi.fire_event_py_yield, 10), + ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), + ( 2, E.CALL, capi.fire_event_call, callable, 40), + ( 1, E.JUMP, capi.fire_event_jump, 60), + ( 1, E.BRANCH_RIGHT, capi.fire_event_branch_right, 70), + ( 1, E.BRANCH_LEFT, capi.fire_event_branch_left, 80), + ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), + ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), + ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), + ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, 7), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, StopIteration(8)), + ] + + self.EXPECT_RAISED_EXCEPTION = [E.PY_THROW, E.RAISE, E.EXCEPTION_HANDLED, E.PY_UNWIND] + + + def check_event_count(self, event, func, args, expected, callback_raises=None): + class Counter: + def __init__(self, callback_raises): + self.callback_raises = callback_raises + self.count = 0 + + def __call__(self, *args): + self.count += 1 + if self.callback_raises: + exc = self.callback_raises + self.callback_raises = None + raise exc + + try: + counter = Counter(callback_raises) + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + try: + func(*args) + except ValueError as e: + self.assertIsInstance(expected, ValueError) + self.assertEqual(str(e), str(expected)) + return + else: + self.assertEqual(counter.count, expected) + + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_fire_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + self.check_event_count(event, function, args_, expected) + + def test_missing_exception(self): + for _, event, function, *args in self.cases: + if event not in self.EXPECT_RAISED_EXCEPTION: + continue + assert args and isinstance(args[-1], BaseException) + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,) + evt = int(math.log2(event)) + expected = ValueError(f"Firing event {evt} with no exception set") + self.check_event_count(event, function, args_, expected) + + def test_fire_event_failing_callback(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + exc = OSError(42) + with self.assertRaises(type(exc)): + self.check_event_count(event, function, args_, expected, + callback_raises=exc) + + + CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE, + E.EXCEPTION_HANDLED, E.PY_UNWIND } + + def check_disable(self, event, func, args, expected): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.disable = True + if event in self.CANNOT_DISABLE: + # use try-except rather then assertRaises to avoid + # events from framework code + try: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + except ValueError: + pass + else: + self.Error("Expected a ValueError") + else: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected - 1) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_disable_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(2) + with self.subTest(function.__name__): + args_ = (self.codelike, 0) + tuple(args) + self.check_disable(event, function, args_, expected) + + def test_enter_scope_two_events(self): + try: + yield_counter = CounterWithDisable() + unwind_counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_YIELD, yield_counter) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, unwind_counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_YIELD | E.PY_UNWIND) + + yield_value = int(math.log2(E.PY_YIELD)) + unwind_value = int(math.log2(E.PY_UNWIND)) + cl = _testcapi.CodeLike(2) + common_args = (cl, 0) + with self.Scope(cl, yield_value, unwind_value): + yield_counter.count = 0 + unwind_counter.count = 0 + + _testcapi.fire_event_py_unwind(*common_args, ValueError(42)) + assert(yield_counter.count == 0) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 1) + assert(unwind_counter.count == 1) + + yield_counter.disable = True + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + finally: + sys.monitoring.set_events(TEST_TOOL, 0) diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py new file mode 100644 index 00000000000..1c6905bd1ee --- /dev/null +++ b/Lib/test/test_msvcrt.py @@ -0,0 +1,120 @@ +import os +import subprocess +import sys +import unittest +from textwrap import dedent + +from test.support import os_helper, requires_resource +from test.support.os_helper import TESTFN, TESTFN_ASCII + +if sys.platform != "win32": + raise unittest.SkipTest("windows related tests") + +import _winapi +import msvcrt + + +class TestFileOperations(unittest.TestCase): + def test_locking(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + self.assertRaises(OSError, msvcrt.locking, f.fileno(), msvcrt.LK_NBLCK, 1) + + def test_unlockfile(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1) + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + + def test_setmode(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.setmode(f.fileno(), os.O_BINARY) + msvcrt.setmode(f.fileno(), os.O_TEXT) + + def test_open_osfhandle(self): + h = _winapi.CreateFile(TESTFN_ASCII, _winapi.GENERIC_WRITE, 0, 0, 1, 128, 0) + self.addCleanup(os_helper.unlink, TESTFN_ASCII) + + try: + fd = msvcrt.open_osfhandle(h, os.O_RDONLY) + h = None + os.close(fd) + finally: + if h: + _winapi.CloseHandle(h) + + def test_get_osfhandle(self): + with open(TESTFN, "w") as f: + self.addCleanup(os_helper.unlink, TESTFN) + + msvcrt.get_osfhandle(f.fileno()) + + +c = '\u5b57' # unicode CJK char (meaning 'character') for 'wide-char' tests +c_encoded = b'\x57\x5b' # utf-16-le (which windows internally used) encoded char for this CJK char + + +class TestConsoleIO(unittest.TestCase): + # CREATE_NEW_CONSOLE creates a "popup" window. + @requires_resource('gui') + def run_in_separated_process(self, code): + # Run test in a separated process to avoid stdin conflicts. + # See: gh-110147 + cmd = [sys.executable, '-c', code] + subprocess.run(cmd, check=True, capture_output=True, + creationflags=subprocess.CREATE_NEW_CONSOLE) + + def test_kbhit(self): + code = dedent(''' + import msvcrt + assert msvcrt.kbhit() == 0 + ''') + self.run_in_separated_process(code) + + def test_getch(self): + msvcrt.ungetch(b'c') + self.assertEqual(msvcrt.getch(), b'c') + + def check_getwch(self, funcname): + code = dedent(f''' + import msvcrt + from _testconsole import write_input + with open("CONIN$", "rb", buffering=0) as stdin: + write_input(stdin, {ascii(c_encoded)}) + assert msvcrt.{funcname}() == "{c}" + ''') + self.run_in_separated_process(code) + + def test_getwch(self): + self.check_getwch('getwch') + + def test_getche(self): + msvcrt.ungetch(b'c') + self.assertEqual(msvcrt.getche(), b'c') + + def test_getwche(self): + self.check_getwch('getwche') + + def test_putch(self): + msvcrt.putch(b'c') + + def test_putwch(self): + msvcrt.putwch(c) + + +class TestOther(unittest.TestCase): + def test_heap_min(self): + try: + msvcrt.heapmin() + except OSError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/__init__.py b/Lib/test/test_multiprocessing_fork/__init__.py new file mode 100644 index 00000000000..b35e82879d7 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/__init__.py @@ -0,0 +1,19 @@ +import os.path +import sys +import unittest +from test import support + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +if sys.platform == "win32": + raise unittest.SkipTest("fork is not available on Windows") + +if sys.platform == 'darwin': + raise unittest.SkipTest("test may crash on macOS (bpo-33725)") + +if support.check_sanitizer(thread=True): + raise unittest.SkipTest("TSAN doesn't support threads after fork") + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_fork/test_manager.py b/Lib/test/test_multiprocessing_fork/test_manager.py new file mode 100644 index 00000000000..9efbb83bbb7 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_misc.py b/Lib/test/test_multiprocessing_fork/test_misc.py new file mode 100644 index 00000000000..891a494020c --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_processes.py b/Lib/test/test_multiprocessing_fork/test_processes.py new file mode 100644 index 00000000000..987d5d7e3c6 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_processes.py @@ -0,0 +1,8 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="processes") + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_fork/test_threads.py b/Lib/test/test_multiprocessing_fork/test_threads.py new file mode 100644 index 00000000000..1670e34cb17 --- /dev/null +++ b/Lib/test/test_multiprocessing_fork/test_threads.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'fork', only_type="threads") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/__init__.py b/Lib/test/test_multiprocessing_forkserver/__init__.py new file mode 100644 index 00000000000..d91715a344d --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/__init__.py @@ -0,0 +1,13 @@ +import os.path +import sys +import unittest +from test import support + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +if sys.platform == "win32": + raise unittest.SkipTest("forkserver is not available on Windows") + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_forkserver/test_manager.py b/Lib/test/test_multiprocessing_forkserver/test_manager.py new file mode 100644 index 00000000000..14f8f10dfb4 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_misc.py b/Lib/test/test_multiprocessing_forkserver/test_misc.py new file mode 100644 index 00000000000..9cae1b50f71 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_processes.py b/Lib/test/test_multiprocessing_forkserver/test_processes.py new file mode 100644 index 00000000000..6f6b8f56837 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_processes.py @@ -0,0 +1,21 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="processes") + +import os, sys # TODO: RUSTPYTHON +class WithProcessesTestCondition(WithProcessesTestCondition): # TODO: RUSTPYTHON + @unittest.skip('TODO: RUSTPYTHON flaky timeout') + def test_notify(self): super().test_notify() + @unittest.skip('TODO: RUSTPYTHON flaky timeout') + def test_notify_n(self): super().test_notify_n() + +class WithProcessesTestLock(WithProcessesTestLock): # TODO: RUSTPYTHON + @unittest.skipIf( # TODO: RUSTPYTHON + sys.platform == 'linux', # TODO: RUSTPYTHON + 'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError' + ) # TODO: RUSTPYTHON + def test_repr_rlock(self): super().test_repr_rlock() # TODO: RUSTPYTHON + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_forkserver/test_threads.py b/Lib/test/test_multiprocessing_forkserver/test_threads.py new file mode 100644 index 00000000000..719c752aa05 --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_threads.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'forkserver', only_type="threads") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_main_handling.py b/Lib/test/test_multiprocessing_main_handling.py new file mode 100644 index 00000000000..6b30a893167 --- /dev/null +++ b/Lib/test/test_multiprocessing_main_handling.py @@ -0,0 +1,300 @@ +# tests __main__ module handling in multiprocessing +from test import support +from test.support import import_helper +# Skip tests if _multiprocessing wasn't built. +import_helper.import_module('_multiprocessing') + +import importlib +import importlib.machinery +import unittest +import sys +import os +import os.path +import py_compile + +from test.support import os_helper +from test.support.script_helper import ( + make_pkg, make_script, make_zip_pkg, make_zip_script, + assert_python_ok) + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +# Look up which start methods are available to test +import multiprocessing +AVAILABLE_START_METHODS = set(multiprocessing.get_all_start_methods()) + +# Issue #22332: Skip tests if sem_open implementation is broken. +support.skip_if_broken_multiprocessing_synchronize() + +verbose = support.verbose + +test_source = """\ +# multiprocessing includes all sorts of shenanigans to make __main__ +# attributes accessible in the subprocess in a pickle compatible way. + +# We run the "doesn't work in the interactive interpreter" example from +# the docs to make sure it *does* work from an executed __main__, +# regardless of the invocation mechanism + +import sys +import time +from multiprocessing import Pool, set_start_method +from test import support + +# We use this __main__ defined function in the map call below in order to +# check that multiprocessing in correctly running the unguarded +# code in child processes and then making it available as __main__ +def f(x): + return x*x + +# Check explicit relative imports +if "check_sibling" in __file__: + # We're inside a package and not in a __main__.py file + # so make sure explicit relative imports work correctly + from . import sibling + +if __name__ == '__main__': + start_method = sys.argv[1] + set_start_method(start_method) + results = [] + with Pool(5) as pool: + pool.map_async(f, [1, 2, 3], callback=results.extend) + + # up to 1 min to report the results + for _ in support.sleeping_retry(support.LONG_TIMEOUT, + "Timed out waiting for results"): + if results: + break + + results.sort() + print(start_method, "->", results) + + pool.join() +""" + +test_source_main_skipped_in_children = """\ +# __main__.py files have an implied "if __name__ == '__main__'" so +# multiprocessing should always skip running them in child processes + +# This means we can't use __main__ defined functions in child processes, +# so we just use "int" as a passthrough operation below + +if __name__ != "__main__": + raise RuntimeError("Should only be called as __main__!") + +import sys +import time +from multiprocessing import Pool, set_start_method +from test import support + +start_method = sys.argv[1] +set_start_method(start_method) +results = [] +with Pool(5) as pool: + pool.map_async(int, [1, 4, 9], callback=results.extend) + # up to 1 min to report the results + for _ in support.sleeping_retry(support.LONG_TIMEOUT, + "Timed out waiting for results"): + if results: + break + +results.sort() +print(start_method, "->", results) + +pool.join() +""" + +# These helpers were copied from test_cmd_line_script & tweaked a bit... + +def _make_test_script(script_dir, script_basename, + source=test_source, omit_suffix=False): + to_return = make_script(script_dir, script_basename, + source, omit_suffix) + # Hack to check explicit relative imports + if script_basename == "check_sibling": + make_script(script_dir, "sibling", "") + importlib.invalidate_caches() + return to_return + +def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, + source=test_source, depth=1): + to_return = make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, + source, depth) + importlib.invalidate_caches() + return to_return + +# There's no easy way to pass the script directory in to get +# -m to work (avoiding that is the whole point of making +# directories and zipfiles executable!) +# So we fake it for testing purposes with a custom launch script +launch_source = """\ +import sys, os.path, runpy +sys.path.insert(0, %s) +runpy._run_module_as_main(%r) +""" + +def _make_launch_script(script_dir, script_basename, module_name, path=None): + if path is None: + path = "os.path.dirname(__file__)" + else: + path = repr(path) + source = launch_source % (path, module_name) + to_return = make_script(script_dir, script_basename, source) + importlib.invalidate_caches() + return to_return + +class MultiProcessingCmdLineMixin(): + maxDiff = None # Show full tracebacks on subprocess failure + + def setUp(self): + if self.start_method not in AVAILABLE_START_METHODS: + self.skipTest("%r start method not available" % self.start_method) + + def _check_output(self, script_name, exit_code, out, err): + if verbose > 1: + print("Output from test script %r:" % script_name) + print(repr(out)) + self.assertEqual(exit_code, 0) + self.assertEqual(err.decode('utf-8'), '') + expected_results = "%s -> [1, 4, 9]" % self.start_method + self.assertEqual(out.decode('utf-8').strip(), expected_results) + + def _check_script(self, script_name, *cmd_line_switches): + if not __debug__: + cmd_line_switches += ('-' + 'O' * sys.flags.optimize,) + run_args = cmd_line_switches + (script_name, self.start_method) + rc, out, err = assert_python_ok(*run_args, __isolated=False) + self._check_output(script_name, rc, out, err) + + def test_basic_script(self): + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, 'script') + self._check_script(script_name) + + def test_basic_script_no_suffix(self): + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, 'script', + omit_suffix=True) + self._check_script(script_name) + + def test_ipython_workaround(self): + # Some versions of the IPython launch script are missing the + # __name__ = "__main__" guard, and multiprocessing has long had + # a workaround for that case + # See https://github.com/ipython/ipython/issues/4698 + source = test_source_main_skipped_in_children + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, 'ipython', + source=source) + self._check_script(script_name) + script_no_suffix = _make_test_script(script_dir, 'ipython', + source=source, + omit_suffix=True) + self._check_script(script_no_suffix) + + def test_script_compiled(self): + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, 'script') + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + self._check_script(pyc_file) + + def test_directory(self): + source = self.main_in_children_source + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', + source=source) + self._check_script(script_dir) + + def test_directory_compiled(self): + source = self.main_in_children_source + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', + source=source) + py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + self._check_script(script_dir) + + def test_zipfile(self): + source = self.main_in_children_source + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', + source=source) + zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name) + self._check_script(zip_name) + + def test_zipfile_compiled(self): + source = self.main_in_children_source + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, '__main__', + source=source) + compiled_name = py_compile.compile(script_name, doraise=True) + zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name) + self._check_script(zip_name) + + def test_module_in_package(self): + with os_helper.temp_dir() as script_dir: + pkg_dir = os.path.join(script_dir, 'test_pkg') + make_pkg(pkg_dir) + script_name = _make_test_script(pkg_dir, 'check_sibling') + launch_name = _make_launch_script(script_dir, 'launch', + 'test_pkg.check_sibling') + self._check_script(launch_name) + + def test_module_in_package_in_zipfile(self): + with os_helper.temp_dir() as script_dir: + zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') + launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name) + self._check_script(launch_name) + + def test_module_in_subpackage_in_zipfile(self): + with os_helper.temp_dir() as script_dir: + zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) + launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name) + self._check_script(launch_name) + + def test_package(self): + source = self.main_in_children_source + with os_helper.temp_dir() as script_dir: + pkg_dir = os.path.join(script_dir, 'test_pkg') + make_pkg(pkg_dir) + script_name = _make_test_script(pkg_dir, '__main__', + source=source) + launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') + self._check_script(launch_name) + + def test_package_compiled(self): + source = self.main_in_children_source + with os_helper.temp_dir() as script_dir: + pkg_dir = os.path.join(script_dir, 'test_pkg') + make_pkg(pkg_dir) + script_name = _make_test_script(pkg_dir, '__main__', + source=source) + compiled_name = py_compile.compile(script_name, doraise=True) + os.remove(script_name) + pyc_file = import_helper.make_legacy_pyc(script_name) + launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') + self._check_script(launch_name) + +# Test all supported start methods (setupClass skips as appropriate) + +class SpawnCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase): + start_method = 'spawn' + main_in_children_source = test_source_main_skipped_in_children + +class ForkCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase): + start_method = 'fork' + main_in_children_source = test_source + +class ForkServerCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase): + start_method = 'forkserver' + main_in_children_source = test_source_main_skipped_in_children + +def tearDownModule(): + support.reap_children() + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/__init__.py b/Lib/test/test_multiprocessing_spawn/__init__.py new file mode 100644 index 00000000000..3fd0f9b3906 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/__init__.py @@ -0,0 +1,9 @@ +import os.path +import unittest +from test import support + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_multiprocessing_spawn/test_manager.py b/Lib/test/test_multiprocessing_spawn/test_manager.py new file mode 100644 index 00000000000..b40bea0bf61 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_manager.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="manager") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_misc.py b/Lib/test/test_multiprocessing_spawn/test_misc.py new file mode 100644 index 00000000000..32f37c5cc81 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_misc.py @@ -0,0 +1,7 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', exclude_types=True) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_processes.py b/Lib/test/test_multiprocessing_spawn/test_processes.py new file mode 100644 index 00000000000..21fd6abd655 --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_processes.py @@ -0,0 +1,19 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="processes") + +import os, sys # TODO: RUSTPYTHON +class WithProcessesTestCondition(WithProcessesTestCondition): # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == 'darwin', 'TODO: RUSTPYTHON flaky timeout') + def test_notify(self): super().test_notify() + +class WithProcessesTestLock(WithProcessesTestLock): # TODO: RUSTPYTHON + @unittest.skipIf( # TODO: RUSTPYTHON + sys.platform == 'linux', # TODO: RUSTPYTHON + 'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError' + ) # TODO: RUSTPYTHON + def test_repr_rlock(self): super().test_repr_rlock() # TODO: RUSTPYTHON + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_multiprocessing_spawn/test_threads.py b/Lib/test/test_multiprocessing_spawn/test_threads.py new file mode 100644 index 00000000000..54c52c4188b --- /dev/null +++ b/Lib/test/test_multiprocessing_spawn/test_threads.py @@ -0,0 +1,12 @@ +import unittest +from test._test_multiprocessing import install_tests_in_module_dict + +install_tests_in_module_dict(globals(), 'spawn', only_type="threads") + +import os, sys # TODO: RUSTPYTHON +class WithThreadsTestPool(WithThreadsTestPool): # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; flaky environment pollution when running rustpython -m test --fail-env-changed due to unknown reason") + def test_terminate(self): super().test_terminate() # TODO: RUSTPYTHON + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 797b0f512f3..fea86fe4308 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -4,32 +4,40 @@ class NamedExpressionInvalidTest(unittest.TestCase): + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_01(self): code = """x := 0""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_02(self): code = """x = y := 0""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_03(self): code = """y := f(x)""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_04(self): code = """y0 = y1 := f(x)""" with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure # wrong error message + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_06(self): code = """((a, b) := (1, 2))""" @@ -60,6 +68,8 @@ def test_named_expression_invalid_10(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_11(self): code = """spam(a=1, b := 2)""" @@ -67,6 +77,8 @@ def test_named_expression_invalid_11(self): "positional argument follows keyword argument"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_12(self): code = """spam(a=1, (b := 2))""" @@ -74,6 +86,8 @@ def test_named_expression_invalid_12(self): "positional argument follows keyword argument"): exec(code, {}, {}) + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_13(self): code = """spam(a=1, (b := 2))""" @@ -87,8 +101,8 @@ def test_named_expression_invalid_14(self): with self.assertRaisesRegex(SyntaxError, "invalid syntax"): exec(code, {}, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure # wrong error message + # TODO: RUSTPYTHON: wrong error message + @unittest.expectedFailure def test_named_expression_invalid_15(self): code = """(lambda: x := 1)""" @@ -139,8 +153,6 @@ def test_named_expression_invalid_rebinding_list_comprehension_iteration_variabl with self.assertRaisesRegex(SyntaxError, msg): exec(code, {}, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure # wrong error message def test_named_expression_invalid_rebinding_list_comprehension_inner_loop(self): cases = [ ("Inner reuse", 'j', "[i for i in range(5) if (j := 0) for j in range(5)]"), @@ -197,8 +209,6 @@ def test_named_expression_invalid_rebinding_set_comprehension_iteration_variable with self.assertRaisesRegex(SyntaxError, msg): exec(code, {}, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_named_expression_invalid_rebinding_set_comprehension_inner_loop(self): cases = [ ("Inner reuse", 'j', "{i for i in range(5) if (j := 0) for j in range(5)}"), diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 573d636de95..9d720f62710 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,10 +1,6 @@ import netrc, os, unittest, sys, textwrap -from test.support import os_helper, run_unittest - -try: - import pwd -except ImportError: - pwd = None +from test import support +from test.support import os_helper temp_filename = os_helper.TESTFN @@ -269,9 +265,14 @@ def test_comment_at_end_of_machine_line_pass_has_hash(self): machine bar.domain.com login foo password pass """, '#pass') + @unittest.skipUnless(support.is_wasi, 'WASI only test') + def test_security_on_WASI(self): + self.assertFalse(netrc._can_security_check()) + self.assertEqual(netrc._getpwuid(0), 'uid 0') + self.assertEqual(netrc._getpwuid(123456), 'uid 123456') @unittest.skipUnless(os.name == 'posix', 'POSIX only test') - @unittest.skipIf(pwd is None, 'security check requires pwd module') + @unittest.skipUnless(hasattr(os, 'getuid'), "os.getuid is required") @os_helper.skip_unless_working_chmod def test_security(self): # This test is incomplete since we are normally not run as root and @@ -308,8 +309,6 @@ def test_security(self): self.assertEqual(nrc.hosts['foo.domain.com'], ('anonymous', '', 'pass')) -def test_main(): - run_unittest(NetrcTestCase) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index f75fce6e06a..9270f325706 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1,11 +1,14 @@ +import inspect import ntpath import os import string +import subprocess import sys import unittest import warnings +from test import support from test.support import os_helper -from test.support import TestFailed, is_emscripten +from ntpath import ALLOW_MISSING from test.support.os_helper import FakePath from test import test_genericpath from tempfile import TemporaryFile @@ -55,7 +58,7 @@ def tester(fn, wantResult): fn = fn.replace("\\", "\\\\") gotResult = eval(fn) if wantResult != gotResult and _norm(wantResult) != _norm(gotResult): - raise TestFailed("%s should return: %s but returned: %s" \ + raise support.TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), str(gotResult))) # then with bytes @@ -71,10 +74,14 @@ def tester(fn, wantResult): warnings.simplefilter("ignore", DeprecationWarning) gotResult = eval(fn) if _norm(wantResult) != _norm(gotResult): - raise TestFailed("%s should return: %s but returned: %s" \ + raise support.TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), repr(gotResult))) +def _parameterize(*parameters): + return support.subTests('kwargs', parameters, _do_cleanups=True) + + class NtpathTestCase(unittest.TestCase): def assertPathEqual(self, path1, path2): if path1 == path2 or _norm(path1) == _norm(path2): @@ -99,58 +106,150 @@ def test_splitext(self): tester('ntpath.splitext("c:a/b\\c.d")', ('c:a/b\\c', '.d')) def test_splitdrive(self): - tester('ntpath.splitdrive("c:\\foo\\bar")', - ('c:', '\\foo\\bar')) - tester('ntpath.splitdrive("c:/foo/bar")', - ('c:', '/foo/bar')) + tester("ntpath.splitdrive('')", ('', '')) + tester("ntpath.splitdrive('foo')", ('', 'foo')) + tester("ntpath.splitdrive('foo\\bar')", ('', 'foo\\bar')) + tester("ntpath.splitdrive('foo/bar')", ('', 'foo/bar')) + tester("ntpath.splitdrive('\\')", ('', '\\')) + tester("ntpath.splitdrive('/')", ('', '/')) + tester("ntpath.splitdrive('\\foo\\bar')", ('', '\\foo\\bar')) + tester("ntpath.splitdrive('/foo/bar')", ('', '/foo/bar')) + tester('ntpath.splitdrive("c:foo\\bar")', ('c:', 'foo\\bar')) + tester('ntpath.splitdrive("c:foo/bar")', ('c:', 'foo/bar')) + tester('ntpath.splitdrive("c:\\foo\\bar")', ('c:', '\\foo\\bar')) + tester('ntpath.splitdrive("c:/foo/bar")', ('c:', '/foo/bar')) + tester("ntpath.splitdrive('\\\\')", ('\\\\', '')) + tester("ntpath.splitdrive('//')", ('//', '')) tester('ntpath.splitdrive("\\\\conky\\mountpoint\\foo\\bar")', ('\\\\conky\\mountpoint', '\\foo\\bar')) tester('ntpath.splitdrive("//conky/mountpoint/foo/bar")', ('//conky/mountpoint', '/foo/bar')) - tester('ntpath.splitdrive("\\\\\\conky\\mountpoint\\foo\\bar")', - ('\\\\\\conky', '\\mountpoint\\foo\\bar')) - tester('ntpath.splitdrive("///conky/mountpoint/foo/bar")', - ('///conky', '/mountpoint/foo/bar')) - tester('ntpath.splitdrive("\\\\conky\\\\mountpoint\\foo\\bar")', - ('\\\\conky\\', '\\mountpoint\\foo\\bar')) - tester('ntpath.splitdrive("//conky//mountpoint/foo/bar")', - ('//conky/', '/mountpoint/foo/bar')) + tester('ntpath.splitdrive("\\\\?\\UNC\\server\\share\\dir")', + ("\\\\?\\UNC\\server\\share", "\\dir")) + tester('ntpath.splitdrive("//?/UNC/server/share/dir")', + ("//?/UNC/server/share", "/dir")) + + def test_splitdrive_invalid_paths(self): + splitdrive = ntpath.splitdrive + self.assertEqual(splitdrive('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\di\x00r')) + self.assertEqual(splitdrive(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\di\x00r')) + self.assertEqual(splitdrive("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitdrive(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\\xfd')) + + def test_splitroot(self): + tester("ntpath.splitroot('')", ('', '', '')) + tester("ntpath.splitroot('foo')", ('', '', 'foo')) + tester("ntpath.splitroot('foo\\bar')", ('', '', 'foo\\bar')) + tester("ntpath.splitroot('foo/bar')", ('', '', 'foo/bar')) + tester("ntpath.splitroot('\\')", ('', '\\', '')) + tester("ntpath.splitroot('/')", ('', '/', '')) + tester("ntpath.splitroot('\\foo\\bar')", ('', '\\', 'foo\\bar')) + tester("ntpath.splitroot('/foo/bar')", ('', '/', 'foo/bar')) + tester('ntpath.splitroot("c:foo\\bar")', ('c:', '', 'foo\\bar')) + tester('ntpath.splitroot("c:foo/bar")', ('c:', '', 'foo/bar')) + tester('ntpath.splitroot("c:\\foo\\bar")', ('c:', '\\', 'foo\\bar')) + tester('ntpath.splitroot("c:/foo/bar")', ('c:', '/', 'foo/bar')) + + # Redundant slashes are not included in the root. + tester("ntpath.splitroot('c:\\\\a')", ('c:', '\\', '\\a')) + tester("ntpath.splitroot('c:\\\\\\a/b')", ('c:', '\\', '\\\\a/b')) + + # Mixed path separators. + tester("ntpath.splitroot('c:/\\')", ('c:', '/', '\\')) + tester("ntpath.splitroot('c:\\/')", ('c:', '\\', '/')) + tester("ntpath.splitroot('/\\a/b\\/\\')", ('/\\a/b', '\\', '/\\')) + tester("ntpath.splitroot('\\/a\\b/\\/')", ('\\/a\\b', '/', '\\/')) + + # UNC paths. + tester("ntpath.splitroot('\\\\')", ('\\\\', '', '')) + tester("ntpath.splitroot('//')", ('//', '', '')) + tester('ntpath.splitroot("\\\\conky\\mountpoint\\foo\\bar")', + ('\\\\conky\\mountpoint', '\\', 'foo\\bar')) + tester('ntpath.splitroot("//conky/mountpoint/foo/bar")', + ('//conky/mountpoint', '/', 'foo/bar')) + tester('ntpath.splitroot("\\\\\\conky\\mountpoint\\foo\\bar")', + ('\\\\\\conky', '\\', 'mountpoint\\foo\\bar')) + tester('ntpath.splitroot("///conky/mountpoint/foo/bar")', + ('///conky', '/', 'mountpoint/foo/bar')) + tester('ntpath.splitroot("\\\\conky\\\\mountpoint\\foo\\bar")', + ('\\\\conky\\', '\\', 'mountpoint\\foo\\bar')) + tester('ntpath.splitroot("//conky//mountpoint/foo/bar")', + ('//conky/', '/', 'mountpoint/foo/bar')) + # Issue #19911: UNC part containing U+0130 - self.assertEqual(ntpath.splitdrive('//conky/MOUNTPOİNT/foo/bar'), - ('//conky/MOUNTPOİNT', '/foo/bar')) + self.assertEqual(ntpath.splitroot('//conky/MOUNTPOİNT/foo/bar'), + ('//conky/MOUNTPOİNT', '/', 'foo/bar')) # gh-81790: support device namespace, including UNC drives. - tester('ntpath.splitdrive("//?/c:")', ("//?/c:", "")) - tester('ntpath.splitdrive("//?/c:/")', ("//?/c:", "/")) - tester('ntpath.splitdrive("//?/c:/dir")', ("//?/c:", "/dir")) - tester('ntpath.splitdrive("//?/UNC")', ("//?/UNC", "")) - tester('ntpath.splitdrive("//?/UNC/")', ("//?/UNC/", "")) - tester('ntpath.splitdrive("//?/UNC/server/")', ("//?/UNC/server/", "")) - tester('ntpath.splitdrive("//?/UNC/server/share")', ("//?/UNC/server/share", "")) - tester('ntpath.splitdrive("//?/UNC/server/share/dir")', ("//?/UNC/server/share", "/dir")) - tester('ntpath.splitdrive("//?/VOLUME{00000000-0000-0000-0000-000000000000}/spam")', - ('//?/VOLUME{00000000-0000-0000-0000-000000000000}', '/spam')) - tester('ntpath.splitdrive("//?/BootPartition/")', ("//?/BootPartition", "/")) - - tester('ntpath.splitdrive("\\\\?\\c:")', ("\\\\?\\c:", "")) - tester('ntpath.splitdrive("\\\\?\\c:\\")', ("\\\\?\\c:", "\\")) - tester('ntpath.splitdrive("\\\\?\\c:\\dir")', ("\\\\?\\c:", "\\dir")) - tester('ntpath.splitdrive("\\\\?\\UNC")', ("\\\\?\\UNC", "")) - tester('ntpath.splitdrive("\\\\?\\UNC\\")', ("\\\\?\\UNC\\", "")) - tester('ntpath.splitdrive("\\\\?\\UNC\\server\\")', ("\\\\?\\UNC\\server\\", "")) - tester('ntpath.splitdrive("\\\\?\\UNC\\server\\share")', ("\\\\?\\UNC\\server\\share", "")) - tester('ntpath.splitdrive("\\\\?\\UNC\\server\\share\\dir")', - ("\\\\?\\UNC\\server\\share", "\\dir")) - tester('ntpath.splitdrive("\\\\?\\VOLUME{00000000-0000-0000-0000-000000000000}\\spam")', - ('\\\\?\\VOLUME{00000000-0000-0000-0000-000000000000}', '\\spam')) - tester('ntpath.splitdrive("\\\\?\\BootPartition\\")', ("\\\\?\\BootPartition", "\\")) + tester('ntpath.splitroot("//?/c:")', ("//?/c:", "", "")) + tester('ntpath.splitroot("//./c:")', ("//./c:", "", "")) + tester('ntpath.splitroot("//?/c:/")', ("//?/c:", "/", "")) + tester('ntpath.splitroot("//?/c:/dir")', ("//?/c:", "/", "dir")) + tester('ntpath.splitroot("//?/UNC")', ("//?/UNC", "", "")) + tester('ntpath.splitroot("//?/UNC/")', ("//?/UNC/", "", "")) + tester('ntpath.splitroot("//?/UNC/server/")', ("//?/UNC/server/", "", "")) + tester('ntpath.splitroot("//?/UNC/server/share")', ("//?/UNC/server/share", "", "")) + tester('ntpath.splitroot("//?/UNC/server/share/dir")', ("//?/UNC/server/share", "/", "dir")) + tester('ntpath.splitroot("//?/VOLUME{00000000-0000-0000-0000-000000000000}/spam")', + ('//?/VOLUME{00000000-0000-0000-0000-000000000000}', '/', 'spam')) + tester('ntpath.splitroot("//?/BootPartition/")', ("//?/BootPartition", "/", "")) + tester('ntpath.splitroot("//./BootPartition/")', ("//./BootPartition", "/", "")) + tester('ntpath.splitroot("//./PhysicalDrive0")', ("//./PhysicalDrive0", "", "")) + tester('ntpath.splitroot("//./nul")', ("//./nul", "", "")) + + tester('ntpath.splitroot("\\\\?\\c:")', ("\\\\?\\c:", "", "")) + tester('ntpath.splitroot("\\\\.\\c:")', ("\\\\.\\c:", "", "")) + tester('ntpath.splitroot("\\\\?\\c:\\")', ("\\\\?\\c:", "\\", "")) + tester('ntpath.splitroot("\\\\?\\c:\\dir")', ("\\\\?\\c:", "\\", "dir")) + tester('ntpath.splitroot("\\\\?\\UNC")', ("\\\\?\\UNC", "", "")) + tester('ntpath.splitroot("\\\\?\\UNC\\")', ("\\\\?\\UNC\\", "", "")) + tester('ntpath.splitroot("\\\\?\\UNC\\server\\")', ("\\\\?\\UNC\\server\\", "", "")) + tester('ntpath.splitroot("\\\\?\\UNC\\server\\share")', + ("\\\\?\\UNC\\server\\share", "", "")) + tester('ntpath.splitroot("\\\\?\\UNC\\server\\share\\dir")', + ("\\\\?\\UNC\\server\\share", "\\", "dir")) + tester('ntpath.splitroot("\\\\?\\VOLUME{00000000-0000-0000-0000-000000000000}\\spam")', + ('\\\\?\\VOLUME{00000000-0000-0000-0000-000000000000}', '\\', 'spam')) + tester('ntpath.splitroot("\\\\?\\BootPartition\\")', ("\\\\?\\BootPartition", "\\", "")) + tester('ntpath.splitroot("\\\\.\\BootPartition\\")', ("\\\\.\\BootPartition", "\\", "")) + tester('ntpath.splitroot("\\\\.\\PhysicalDrive0")', ("\\\\.\\PhysicalDrive0", "", "")) + tester('ntpath.splitroot("\\\\.\\nul")', ("\\\\.\\nul", "", "")) # gh-96290: support partial/invalid UNC drives - tester('ntpath.splitdrive("//")', ("//", "")) # empty server & missing share - tester('ntpath.splitdrive("///")', ("///", "")) # empty server & empty share - tester('ntpath.splitdrive("///y")', ("///y", "")) # empty server & non-empty share - tester('ntpath.splitdrive("//x")', ("//x", "")) # non-empty server & missing share - tester('ntpath.splitdrive("//x/")', ("//x/", "")) # non-empty server & empty share + tester('ntpath.splitroot("//")', ("//", "", "")) # empty server & missing share + tester('ntpath.splitroot("///")', ("///", "", "")) # empty server & empty share + tester('ntpath.splitroot("///y")', ("///y", "", "")) # empty server & non-empty share + tester('ntpath.splitroot("//x")', ("//x", "", "")) # non-empty server & missing share + tester('ntpath.splitroot("//x/")', ("//x/", "", "")) # non-empty server & empty share + + # gh-101363: match GetFullPathNameW() drive letter parsing behaviour + tester('ntpath.splitroot(" :/foo")', (" :", "/", "foo")) + tester('ntpath.splitroot("/:/foo")', ("", "/", ":/foo")) + + def test_splitroot_invalid_paths(self): + splitroot = ntpath.splitroot + self.assertEqual(splitroot('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\', 'di\x00r')) + self.assertEqual(splitroot(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\', b'di\x00r')) + self.assertEqual(splitroot("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\', '\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitroot(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\', b'\xfd')) def test_split(self): tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar')) @@ -164,11 +263,34 @@ def test_split(self): tester('ntpath.split("c:/")', ('c:/', '')) tester('ntpath.split("//conky/mountpoint/")', ('//conky/mountpoint/', '')) + def test_split_invalid_paths(self): + split = ntpath.split + self.assertEqual(split('c:\\fo\x00o\\ba\x00r'), + ('c:\\fo\x00o', 'ba\x00r')) + self.assertEqual(split(b'c:\\fo\x00o\\ba\x00r'), + (b'c:\\fo\x00o', b'ba\x00r')) + self.assertEqual(split('c:\\\udfff\\\udffe'), + ('c:\\\udfff', '\udffe')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, split, b'c:\\\xff\\bar') + self.assertRaises(UnicodeDecodeError, split, b'c:\\foo\\\xff') + else: + self.assertEqual(split(b'c:\\\xff\\\xfe'), + (b'c:\\\xff', b'\xfe')) + def test_isabs(self): + tester('ntpath.isabs("foo\\bar")', 0) + tester('ntpath.isabs("foo/bar")', 0) tester('ntpath.isabs("c:\\")', 1) + tester('ntpath.isabs("c:\\foo\\bar")', 1) + tester('ntpath.isabs("c:/foo/bar")', 1) tester('ntpath.isabs("\\\\conky\\mountpoint\\")', 1) - tester('ntpath.isabs("\\foo")', 1) - tester('ntpath.isabs("\\foo\\bar")', 1) + + # gh-44626: paths with only a drive or root are not absolute. + tester('ntpath.isabs("\\foo\\bar")', 0) + tester('ntpath.isabs("/foo/bar")', 0) + tester('ntpath.isabs("c:foo\\bar")', 0) + tester('ntpath.isabs("c:foo/bar")', 0) # gh-96290: normal UNC paths and device paths without trailing backslashes tester('ntpath.isabs("\\\\conky\\mountpoint")', 1) @@ -194,6 +316,7 @@ def test_join(self): tester('ntpath.join("a", "b", "c")', 'a\\b\\c') tester('ntpath.join("a\\", "b", "c")', 'a\\b\\c') tester('ntpath.join("a", "b\\", "c")', 'a\\b\\c') + tester('ntpath.join("a", "b", "c\\")', 'a\\b\\c\\') tester('ntpath.join("a", "b", "\\c")', '\\c') tester('ntpath.join("d:\\", "\\pleep")', 'd:\\pleep') tester('ntpath.join("d:\\", "a", "b")', 'd:\\a\\b') @@ -247,6 +370,45 @@ def test_join(self): tester("ntpath.join('//computer/share', 'a', 'b')", '//computer/share\\a\\b') tester("ntpath.join('//computer/share', 'a/b')", '//computer/share\\a/b') + tester("ntpath.join('\\\\', 'computer')", '\\\\computer') + tester("ntpath.join('\\\\computer\\', 'share')", '\\\\computer\\share') + tester("ntpath.join('\\\\computer\\share\\', 'a')", '\\\\computer\\share\\a') + tester("ntpath.join('\\\\computer\\share\\a\\', 'b')", '\\\\computer\\share\\a\\b') + # Second part is anchored, so that the first part is ignored. + tester("ntpath.join('a', 'Z:b', 'c')", 'Z:b\\c') + tester("ntpath.join('a', 'Z:\\b', 'c')", 'Z:\\b\\c') + tester("ntpath.join('a', '\\\\b\\c', 'd')", '\\\\b\\c\\d') + # Second part has a root but not drive. + tester("ntpath.join('a', '\\b', 'c')", '\\b\\c') + tester("ntpath.join('Z:/a', '/b', 'c')", 'Z:\\b\\c') + tester("ntpath.join('//?/Z:/a', '/b', 'c')", '\\\\?\\Z:\\b\\c') + tester("ntpath.join('D:a', './c:b')", 'D:a\\.\\c:b') + tester("ntpath.join('D:/a', './c:b')", 'D:\\a\\.\\c:b') + + def test_normcase(self): + normcase = ntpath.normcase + self.assertEqual(normcase(''), '') + self.assertEqual(normcase(b''), b'') + self.assertEqual(normcase('ABC'), 'abc') + self.assertEqual(normcase(b'ABC'), b'abc') + self.assertEqual(normcase('\xc4\u0141\u03a8'), '\xe4\u0142\u03c8') + expected = '\u03c9\u2126' if sys.platform == 'win32' else '\u03c9\u03c9' + self.assertEqual(normcase('\u03a9\u2126'), expected) + if sys.platform == 'win32' or sys.getfilesystemencoding() == 'utf-8': + self.assertEqual(normcase('\xc4\u0141\u03a8'.encode()), + '\xe4\u0142\u03c8'.encode()) + self.assertEqual(normcase('\u03a9\u2126'.encode()), + expected.encode()) + + def test_normcase_invalid_paths(self): + normcase = ntpath.normcase + self.assertEqual(normcase('abc\x00def'), 'abc\x00def') + self.assertEqual(normcase(b'abc\x00def'), b'abc\x00def') + self.assertEqual(normcase('\udfff'), '\udfff') + if sys.platform == 'win32': + path = b'ABC' + bytes(range(128, 256)) + self.assertEqual(normcase(path), path.lower()) + def test_normpath(self): tester("ntpath.normpath('A//////././//.//B')", r'A\B') tester("ntpath.normpath('A/./B')", r'A\B') @@ -261,13 +423,18 @@ def test_normpath(self): tester("ntpath.normpath('..')", r'..') tester("ntpath.normpath('.')", r'.') + tester("ntpath.normpath('c:.')", 'c:') tester("ntpath.normpath('')", r'.') tester("ntpath.normpath('/')", '\\') tester("ntpath.normpath('c:/')", 'c:\\') tester("ntpath.normpath('/../.././..')", '\\') tester("ntpath.normpath('c:/../../..')", 'c:\\') + tester("ntpath.normpath('/./a/b')", r'\a\b') + tester("ntpath.normpath('c:/./a/b')", r'c:\a\b') tester("ntpath.normpath('../.././..')", r'..\..\..') tester("ntpath.normpath('K:../.././..')", r'K:..\..\..') + tester("ntpath.normpath('./a/b')", r'a\b') + tester("ntpath.normpath('c:./a/b')", r'c:a\b') tester("ntpath.normpath('C:////a/b')", r'C:\a\b') tester("ntpath.normpath('//machine/share//a/b')", r'\\machine\share\a\b') @@ -288,6 +455,22 @@ def test_normpath(self): tester("ntpath.normpath('\\\\foo\\')", '\\\\foo\\') tester("ntpath.normpath('\\\\foo')", '\\\\foo') tester("ntpath.normpath('\\\\')", '\\\\') + tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') + + def test_normpath_invalid_paths(self): + normpath = ntpath.normpath + self.assertEqual(normpath('fo\x00o'), 'fo\x00o') + self.assertEqual(normpath(b'fo\x00o'), b'fo\x00o') + self.assertEqual(normpath('fo\x00o\\..\\bar'), 'bar') + self.assertEqual(normpath(b'fo\x00o\\..\\bar'), b'bar') + self.assertEqual(normpath('\udfff'), '\udfff') + self.assertEqual(normpath('\udfff\\..\\foo'), 'foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, normpath, b'\xff') + self.assertRaises(UnicodeDecodeError, normpath, b'\xff\\..\\foo') + else: + self.assertEqual(normpath(b'\xff'), b'\xff') + self.assertEqual(normpath(b'\xff\\..\\foo'), b'foo') def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) @@ -297,6 +480,27 @@ def test_realpath_curdir(self): tester("ntpath.realpath('.\\.')", expected) tester("ntpath.realpath('\\'.join(['.'] * 100))", expected) + def test_realpath_curdir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=True)", expected) + tester("ntpath.realpath('./.', strict=True)", expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=True)", expected) + tester("ntpath.realpath('.\\.', strict=True)", expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=True)", expected) + + def test_realpath_curdir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('./.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('.\\.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + def test_realpath_pardir(self): expected = ntpath.normpath(os.getcwd()) tester("ntpath.realpath('..')", ntpath.dirname(expected)) @@ -309,28 +513,59 @@ def test_realpath_pardir(self): tester("ntpath.realpath('\\'.join(['..'] * 50))", ntpath.splitdrive(expected)[0] + '\\') + def test_realpath_pardir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=True)", ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + + def test_realpath_pardir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=ALLOW_MISSING)", + ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_basic(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ABSTFN + "1") - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) - self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")), + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) + self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1"), **kwargs), os.fsencode(ABSTFN)) # gh-88013: call ntpath.realpath with binary drive name may raise a # TypeError. The drive should not exist to reproduce the bug. - for c in string.ascii_uppercase: - d = f"{c}:\\" - if not ntpath.exists(d): - break + drives = {f"{c}:\\" for c in string.ascii_uppercase} - set(os.listdrives()) + d = drives.pop().encode() + self.assertEqual(ntpath.realpath(d, strict=False), d) + + # gh-106242: Embedded nulls and non-strict fallback to abspath + if kwargs: + with self.assertRaises(OSError): + ntpath.realpath(os_helper.TESTFN + "\0spam", + **kwargs) else: - raise OSError("No free drive letters available") - self.assertEqual(ntpath.realpath(d), d) + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", **kwargs)) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -343,19 +578,77 @@ def test_realpath_strict(self): self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_invalid_paths(self): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFN + '\x00' + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(realpath(path, strict=False), path) + # gh-106242: Embedded nulls should raise OSError (not ValueError) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFNb + b'\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFN + '\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFNb + b'\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFN + '\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwd()) + self.assertEqual(realpath(path, strict=True), os.getcwd()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwd()) + path = ABSTFNb + b'\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwdb()) + self.assertEqual(realpath(path, strict=True), os.getcwdb()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwdb()) + path = ABSTFN + '\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFN + '\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFN + '\\nonexistent') + path = ABSTFNb + b'\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFNb + b'\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFNb + b'\\nonexistent') + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_invalid_unicode_paths(self, kwargs): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFNb + b'\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\\nonexistent\\\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\\nonexistent\\\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_relative(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ntpath.relpath(ABSTFN + "1")) - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) - # TODO: RUSTPYTHON - @unittest.expectedFailure @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_broken_symlinks(self): @@ -506,7 +799,62 @@ def test_realpath_symlink_loops_strict(self): @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_symlink_prefix(self): + def test_realpath_symlink_loops_raise(self): + # Symlink loops raise OSError in ALLOW_MISSING mode + ABSTFN = ntpath.abspath(os_helper.TESTFN) + self.addCleanup(os_helper.unlink, ABSTFN) + self.addCleanup(os_helper.unlink, ABSTFN + "1") + self.addCleanup(os_helper.unlink, ABSTFN + "2") + self.addCleanup(os_helper.unlink, ABSTFN + "y") + self.addCleanup(os_helper.unlink, ABSTFN + "c") + self.addCleanup(os_helper.unlink, ABSTFN + "a") + self.addCleanup(os_helper.unlink, ABSTFN + "x") + + os.symlink(ABSTFN, ABSTFN) + self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=ALLOW_MISSING) + + os.symlink(ABSTFN + "1", ABSTFN + "2") + os.symlink(ABSTFN + "2", ABSTFN + "1") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x", + strict=ALLOW_MISSING) + + # Windows eliminates '..' components before resolving links; + # realpath is not expected to raise if this removes the loop. + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."), + ntpath.dirname(ABSTFN)) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"), + ntpath.dirname(ABSTFN) + "\\x") + + os.symlink(ABSTFN + "x", ABSTFN + "y") + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\" + + ntpath.basename(ABSTFN) + "y"), + ABSTFN + "x") + self.assertRaises( + OSError, ntpath.realpath, + ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1", + strict=ALLOW_MISSING) + + os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a", + strict=ALLOW_MISSING) + + os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN)) + + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c", + strict=ALLOW_MISSING) + + # Test using relative path as well. + self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN), + strict=ALLOW_MISSING) + + @os_helper.skip_unless_symlink + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_prefix(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) self.addCleanup(os_helper.unlink, ABSTFN + "3") self.addCleanup(os_helper.unlink, "\\\\?\\" + ABSTFN + "3.") @@ -521,9 +869,9 @@ def test_realpath_symlink_prefix(self): f.write(b'1') os.symlink("\\\\?\\" + ABSTFN + "3.", ABSTFN + "3.link") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3link", **kwargs), ABSTFN + "3") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") # Resolved paths should be usable to open target files @@ -533,14 +881,17 @@ def test_realpath_symlink_prefix(self): self.assertEqual(f.read(), b'1') # When the prefix is included, it is not stripped - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link", **kwargs), "\\\\?\\" + ABSTFN + "3") - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_nul(self): tester("ntpath.realpath('NUL')", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=False)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=True)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=ALLOW_MISSING)", r'\\.\NUL') @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @unittest.skipUnless(HAVE_GETSHORTPATHNAME, 'need _getshortpathname') @@ -564,14 +915,66 @@ def test_realpath_cwd(self): self.assertPathEqual(test_file_long, ntpath.realpath(test_file_short)) - with os_helper.change_cwd(test_dir_long): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_long.lower()): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_short): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) + for kwargs in {}, {'strict': True}, {'strict': ALLOW_MISSING}: + with self.subTest(**kwargs): + with os_helper.change_cwd(test_dir_long): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_long.lower()): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_short): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_permission(self): + # Test whether python can resolve the real filename of a + # shortened file name even if it does not have permission to access it. + ABSTFN = ntpath.realpath(os_helper.TESTFN) + + os_helper.unlink(ABSTFN) + os_helper.rmtree(ABSTFN) + os.mkdir(ABSTFN) + self.addCleanup(os_helper.rmtree, ABSTFN) + + test_file = ntpath.join(ABSTFN, "LongFileName123.txt") + test_file_short = ntpath.join(ABSTFN, "LONGFI~1.TXT") + + with open(test_file, "wb") as f: + f.write(b"content") + # Automatic generation of short names may be disabled on + # NTFS volumes for the sake of performance. + # They're not supported at all on ReFS and exFAT. + p = subprocess.run( + # Try to set the short name manually. + ['fsutil.exe', 'file', 'setShortName', test_file, 'LONGFI~1.TXT'], + creationflags=subprocess.DETACHED_PROCESS + ) + + if p.returncode: + raise unittest.SkipTest('failed to set short name') + + try: + self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) + except AssertionError: + raise unittest.SkipTest('the filesystem seems to lack support for short filenames') + + # Deny the right to [S]YNCHRONIZE on the file to + # force nt._getfinalpathname to fail with ERROR_ACCESS_DENIED. + p = subprocess.run( + ['icacls.exe', test_file, '/deny', '*S-1-5-32-545:(S)'], + creationflags=subprocess.DETACHED_PROCESS + ) + + if p.returncode: + raise unittest.SkipTest('failed to deny access to the test file') + + self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ValueError: illegal environment variable name") def test_expandvars(self): with os_helper.EnvironmentVarGuard() as env: env.clear() @@ -598,7 +1001,6 @@ def test_expandvars(self): tester('ntpath.expandvars("\'%foo%\'%bar")', "\'%foo%\'%bar") tester('ntpath.expandvars("bar\'%foo%")', "bar\'%foo%") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ValueError: illegal environment variable name") @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') def test_expandvars_nonascii(self): def check(value, expected): @@ -619,7 +1021,19 @@ def check(value, expected): check('%spam%bar', '%sbar' % nonascii) check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = ntpath.expandvars + with os_helper.EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('%A%'*n), 'B'*n) + self.assertEqual(expandvars('%A%A'*n), 'BA'*n) + self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%') + self.assertEqual(expandvars("%%"*n), "%"*n) + self.assertEqual(expandvars("$$"*n), "$"*n) + def test_expanduser(self): tester('ntpath.expanduser("test")', 'test') @@ -680,6 +1094,7 @@ def test_abspath(self): tester('ntpath.abspath("C:\\spam. . .")', "C:\\spam") tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") + self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") @@ -713,6 +1128,26 @@ def test_abspath(self): drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") + def test_abspath_invalid_paths(self): + abspath = ntpath.abspath + if sys.platform == 'win32': + self.assertEqual(abspath("C:\x00"), ntpath.join(abspath("C:"), "\x00")) + self.assertEqual(abspath(b"C:\x00"), ntpath.join(abspath(b"C:"), b"\x00")) + self.assertEqual(abspath("\x00:spam"), "\x00:\\spam") + self.assertEqual(abspath(b"\x00:spam"), b"\x00:\\spam") + self.assertEqual(abspath('c:\\fo\x00o'), 'c:\\fo\x00o') + self.assertEqual(abspath(b'c:\\fo\x00o'), b'c:\\fo\x00o') + self.assertEqual(abspath('c:\\fo\x00o\\..\\bar'), 'c:\\bar') + self.assertEqual(abspath(b'c:\\fo\x00o\\..\\bar'), b'c:\\bar') + self.assertEqual(abspath('c:\\\udfff'), 'c:\\\udfff') + self.assertEqual(abspath('c:\\\udfff\\..\\foo'), 'c:\\foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff') + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff\\..\\foo') + else: + self.assertEqual(abspath(b'c:\\\xff'), b'c:\\\xff') + self.assertEqual(abspath(b'c:\\\xff\\..\\foo'), b'c:\\foo') + def test_relpath(self): tester('ntpath.relpath("a")', 'a') tester('ntpath.relpath(ntpath.abspath("a"))', 'a') @@ -741,43 +1176,47 @@ def test_commonpath(self): def check(paths, expected): tester(('ntpath.commonpath(%r)' % paths).replace('\\\\', '\\'), expected) - def check_error(exc, paths): - self.assertRaises(exc, ntpath.commonpath, paths) - self.assertRaises(exc, ntpath.commonpath, - [os.fsencode(p) for p in paths]) - + def check_error(paths, expected): + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths[::-1]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths[::-1]]) + + self.assertRaises(TypeError, ntpath.commonpath, None) self.assertRaises(ValueError, ntpath.commonpath, []) - check_error(ValueError, ['C:\\Program Files', 'Program Files']) - check_error(ValueError, ['C:\\Program Files', 'C:Program Files']) - check_error(ValueError, ['\\Program Files', 'Program Files']) - check_error(ValueError, ['Program Files', 'C:\\Program Files']) - check(['C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files'], - 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files\\'], - 'C:\\Program Files') - check(['C:\\\\Program Files', 'C:\\Program Files\\\\'], - 'C:\\Program Files') - check(['C:\\.\\Program Files', 'C:\\Program Files\\.'], - 'C:\\Program Files') - check(['C:\\', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Projects'], 'C:\\') - check(['C:\\Program Files\\', 'C:\\Projects'], 'C:\\') - - check(['C:\\Program Files\\Foo', 'C:/Program Files/Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'c:/program files/bar'], - 'C:\\Program Files') - check(['c:/program files/bar', 'C:\\Program Files\\Foo'], - 'c:\\program files') - - check_error(ValueError, ['C:\\Program Files', 'D:\\Program Files']) + self.assertRaises(ValueError, ntpath.commonpath, iter([])) + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'C:Foo'], "Can't mix absolute and relative paths") + check_error(['C:\\Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'Foo'], "Paths don't have the same drive") + check_error(['C:Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'Foo'], "Paths don't have the same drive") + check_error(['\\Foo', 'Foo'], "Can't mix rooted and not-rooted paths") + + check(['C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo\\'], 'C:\\Foo') + check(['C:\\\\Foo', 'C:\\Foo\\\\'], 'C:\\Foo') + check(['C:\\.\\Foo', 'C:\\Foo\\.'], 'C:\\Foo') + check(['C:\\', 'C:\\baz'], 'C:\\') + check(['C:\\Bar', 'C:\\baz'], 'C:\\') + check(['C:\\Foo', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Bar', 'C:\\Baz'], 'C:\\') + check(['C:\\Bar\\', 'C:\\Baz'], 'C:\\') + + check(['C:\\Foo\\Bar', 'C:/Foo/Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'c:/foo/baz'], 'C:\\Foo') + check(['c:/foo/bar', 'C:\\Foo\\Baz'], 'c:\\foo') + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'D:\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'D:Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'D:Foo'], "Paths don't have the same drive") check(['spam'], 'spam') check(['spam', 'spam'], 'spam') @@ -791,23 +1230,17 @@ def check_error(exc, paths): check([''], '') check(['', 'spam\\alot'], '') - check_error(ValueError, ['', '\\spam\\alot']) - - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['Program Files', b'C:\\Program Files\\Foo']) - - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - @unittest.skipIf(is_emscripten, "Emscripten cannot fstat unnamed files.") + + # gh-117381: Logical error messages + check_error(['', '\\spam\\alot'], "Can't mix rooted and not-rooted paths") + + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['Foo', b'C:\\Foo\\Baz']) + def test_sameopenfile(self): with TemporaryFile() as tf1, TemporaryFile() as tf2: # Make sure the same file is really the same @@ -857,6 +1290,74 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) + def test_ismount_invalid_paths(self): + ismount = ntpath.ismount + self.assertFalse(ismount("c:\\\udfff")) + if sys.platform == 'win32': + self.assertRaises(ValueError, ismount, "c:\\\x00") + self.assertRaises(ValueError, ismount, b"c:\\\x00") + self.assertRaises(UnicodeDecodeError, ismount, b"c:\\\xff") + else: + self.assertFalse(ismount("c:\\\x00")) + self.assertFalse(ismount(b"c:\\\x00")) + self.assertFalse(ismount(b"c:\\\xff")) + + def test_isreserved(self): + self.assertFalse(ntpath.isreserved('')) + self.assertFalse(ntpath.isreserved('.')) + self.assertFalse(ntpath.isreserved('..')) + self.assertFalse(ntpath.isreserved('/')) + self.assertFalse(ntpath.isreserved('/foo/bar')) + # A name that ends with a space or dot is reserved. + self.assertTrue(ntpath.isreserved('foo.')) + self.assertTrue(ntpath.isreserved('foo ')) + # ASCII control characters are reserved. + self.assertTrue(ntpath.isreserved('\foo')) + # Wildcard characters, colon, and pipe are reserved. + self.assertTrue(ntpath.isreserved('foo*bar')) + self.assertTrue(ntpath.isreserved('foo?bar')) + self.assertTrue(ntpath.isreserved('foo"bar')) + self.assertTrue(ntpath.isreserved('foo<bar')) + self.assertTrue(ntpath.isreserved('foo>bar')) + self.assertTrue(ntpath.isreserved('foo:bar')) + self.assertTrue(ntpath.isreserved('foo|bar')) + # Case-insensitive DOS-device names are reserved. + self.assertTrue(ntpath.isreserved('nul')) + self.assertTrue(ntpath.isreserved('aux')) + self.assertTrue(ntpath.isreserved('prn')) + self.assertTrue(ntpath.isreserved('con')) + self.assertTrue(ntpath.isreserved('conin$')) + self.assertTrue(ntpath.isreserved('conout$')) + # COM/LPT + 1-9 or + superscript 1-3 are reserved. + self.assertTrue(ntpath.isreserved('COM1')) + self.assertTrue(ntpath.isreserved('LPT9')) + self.assertTrue(ntpath.isreserved('com\xb9')) + self.assertTrue(ntpath.isreserved('com\xb2')) + self.assertTrue(ntpath.isreserved('lpt\xb3')) + # DOS-device name matching ignores characters after a dot or + # a colon and also ignores trailing spaces. + self.assertTrue(ntpath.isreserved('NUL.txt')) + self.assertTrue(ntpath.isreserved('PRN ')) + self.assertTrue(ntpath.isreserved('AUX .txt')) + self.assertTrue(ntpath.isreserved('COM1:bar')) + self.assertTrue(ntpath.isreserved('LPT9 :bar')) + # DOS-device names are only matched at the beginning + # of a path component. + self.assertFalse(ntpath.isreserved('bar.com9')) + self.assertFalse(ntpath.isreserved('bar.lpt9')) + # The entire path is checked, except for the drive. + self.assertTrue(ntpath.isreserved('c:/bar/baz/NUL')) + self.assertTrue(ntpath.isreserved('c:/NUL/bar/baz')) + self.assertFalse(ntpath.isreserved('//./NUL')) + # Bytes are supported. + self.assertFalse(ntpath.isreserved(b'')) + self.assertFalse(ntpath.isreserved(b'.')) + self.assertFalse(ntpath.isreserved(b'..')) + self.assertFalse(ntpath.isreserved(b'/')) + self.assertFalse(ntpath.isreserved(b'/foo/bar')) + self.assertTrue(ntpath.isreserved(b'foo.')) + self.assertTrue(ntpath.isreserved(b'nul')) + def assertEqualCI(self, s1, s2): """Assert that two strings are equal ignoring case differences.""" self.assertEqual(s1.lower(), s2.lower()) @@ -891,48 +1392,112 @@ def test_nt_helpers(self): self.assertIsInstance(b_final_path, bytes) self.assertGreater(len(b_final_path), 0) + @unittest.skipIf(sys.platform != 'win32', "Can only test junctions with creation on win32.") + def test_isjunction(self): + with os_helper.temp_dir() as d: + with os_helper.change_cwd(d): + os.mkdir('tmpdir') + + import _winapi + try: + _winapi.CreateJunction('tmpdir', 'testjunc') + except OSError: + raise unittest.SkipTest('creating the test junction failed') + + self.assertTrue(ntpath.isjunction('testjunc')) + self.assertFalse(ntpath.isjunction('tmpdir')) + self.assertPathEqual(ntpath.realpath('testjunc'), ntpath.realpath('tmpdir')) + + def test_isfile_invalid_paths(self): + isfile = ntpath.isfile + self.assertIs(isfile('/tmp\udfffabcds'), False) + self.assertIs(isfile(b'/tmp\xffabcds'), False) + self.assertIs(isfile('/tmp\x00abcds'), False) + self.assertIs(isfile(b'/tmp\x00abcds'), False) + + @unittest.skipIf(sys.platform != 'win32', "drive letters are a windows concept") + def test_isfile_driveletter(self): + drive = os.environ.get('SystemDrive') + if drive is None or len(drive) != 2 or drive[1] != ':': + raise unittest.SkipTest('SystemDrive is not defined or malformed') + self.assertFalse(os.path.isfile('\\\\.\\' + drive)) + + @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()") + def test_isfile_anonymous_pipe(self): + pr, pw = os.pipe() + try: + self.assertFalse(ntpath.isfile(pr)) + finally: + os.close(pr) + os.close(pw) + + @unittest.skipIf(sys.platform != 'win32', "windows only") + def test_isfile_named_pipe(self): + import _winapi + named_pipe = f'//./PIPE/python_isfile_test_{os.getpid()}' + h = _winapi.CreateNamedPipe(named_pipe, + _winapi.PIPE_ACCESS_INBOUND, + 0, 1, 0, 0, 0, 0) + try: + self.assertFalse(ntpath.isfile(named_pipe)) + finally: + _winapi.CloseHandle(h) + + @unittest.skipIf(sys.platform != 'win32', "windows only") + def test_con_device(self): + self.assertFalse(os.path.isfile(r"\\.\CON")) + self.assertFalse(os.path.isdir(r"\\.\CON")) + self.assertFalse(os.path.islink(r"\\.\CON")) + self.assertTrue(os.path.exists(r"\\.\CON")) + + @unittest.skipIf(sys.platform != 'win32', "Fast paths are only for win32") + @support.cpython_only + def test_fast_paths_in_use(self): + # There are fast paths of these functions implemented in posixmodule.c. + # Confirm that they are being used, and not the Python fallbacks in + # genericpath.py. + self.assertTrue(os.path.splitroot is nt._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is nt._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) + self.assertTrue(os.path.isdir is nt._path_isdir) + self.assertFalse(inspect.isfunction(os.path.isdir)) + self.assertTrue(os.path.isfile is nt._path_isfile) + self.assertFalse(inspect.isfunction(os.path.isfile)) + self.assertTrue(os.path.islink is nt._path_islink) + self.assertFalse(inspect.isfunction(os.path.islink)) + self.assertTrue(os.path.isjunction is nt._path_isjunction) + self.assertFalse(inspect.isfunction(os.path.isjunction)) + self.assertTrue(os.path.exists is nt._path_exists) + self.assertFalse(inspect.isfunction(os.path.exists)) + self.assertTrue(os.path.lexists is nt._path_lexists) + self.assertFalse(inspect.isfunction(os.path.lexists)) + + @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") + def test_isdevdrive(self): + # Result may be True or False, but shouldn't raise + self.assertIn(ntpath.isdevdrive(os_helper.TESTFN), (True, False)) + # ntpath.isdevdrive can handle relative paths + self.assertIn(ntpath.isdevdrive("."), (True, False)) + self.assertIn(ntpath.isdevdrive(b"."), (True, False)) + # Volume syntax is supported + self.assertIn(ntpath.isdevdrive(os.listvolumes()[0]), (True, False)) + # Invalid volume returns False from os.path method + self.assertFalse(ntpath.isdevdrive(r"\\?\Volume{00000000-0000-0000-0000-000000000000}\\")) + # Invalid volume raises from underlying helper + with self.assertRaises(OSError): + nt._path_isdevdrive(r"\\?\Volume{00000000-0000-0000-0000-000000000000}\\") + + @unittest.skipIf(os.name == 'nt', "isdevdrive fallback only used off Win32") + def test_isdevdrive_fallback(self): + # Fallback always returns False + self.assertFalse(ntpath.isdevdrive(os_helper.TESTFN)) + + class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = ntpath attributes = ['relpath'] - # TODO: RUSTPYTHON - if sys.platform == "linux": - @unittest.expectedFailure - def test_nonascii_abspath(self): - super().test_nonascii_abspath() - - # TODO: RUSTPYTHON - if sys.platform == "win32": - # TODO: RUSTPYTHON, ValueError: illegal environment variable name - @unittest.expectedFailure - def test_expandvars(self): # TODO: RUSTPYTHON; remove when done - super().test_expandvars() - - # TODO: RUSTPYTHON, ValueError: illegal environment variable name - @unittest.expectedFailure - def test_expandvars_nonascii(self): # TODO: RUSTPYTHON; remove when done - super().test_expandvars_nonascii() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_samefile(self): # TODO: RUSTPYTHON; remove when done - super().test_samefile() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_samefile_on_link(self): # TODO: RUSTPYTHON; remove when done - super().test_samefile_on_link() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_samestat(self): # TODO: RUSTPYTHON; remove when done - super().test_samestat() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_samestat_on_link(self): # TODO: RUSTPYTHON; remove when done - super().test_samestat_on_link() - class PathLikeTests(NtpathTestCase): @@ -948,11 +1513,8 @@ def setUp(self): def _check_function(self, func): self.assertPathEqual(func(self.file_path), func(self.file_name)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: 'ωω' != 'ωΩ'") def test_path_normcase(self): self._check_function(self.path.normcase) - if sys.platform == 'win32': - self.assertEqual(ntpath.normcase('\u03a9\u2126'), 'ωΩ') def test_path_isabs(self): self._check_function(self.path.isabs) @@ -970,6 +1532,9 @@ def test_path_splitext(self): def test_path_splitdrive(self): self._check_function(self.path.splitdrive) + def test_path_splitroot(self): + self._check_function(self.path.splitroot) + def test_path_basename(self): self._check_function(self.path.basename) diff --git a/Lib/test/test_nturl2path.py b/Lib/test/test_nturl2path.py new file mode 100644 index 00000000000..a6a3422a0f7 --- /dev/null +++ b/Lib/test/test_nturl2path.py @@ -0,0 +1,107 @@ +import unittest + +from test.support import warnings_helper + + +nturl2path = warnings_helper.import_deprecated("nturl2path") + + +class NTURL2PathTest(unittest.TestCase): + """Test pathname2url() and url2pathname()""" + + def test_basic(self): + # Make sure simple tests pass + expected_path = r"parts\of\a\path" + expected_url = "parts/of/a/path" + result = nturl2path.pathname2url(expected_path) + self.assertEqual(expected_url, result, + "pathname2url() failed; %s != %s" % + (result, expected_url)) + result = nturl2path.url2pathname(expected_url) + self.assertEqual(expected_path, result, + "url2pathame() failed; %s != %s" % + (result, expected_path)) + + def test_pathname2url(self): + # Test special prefixes are correctly handled in pathname2url() + fn = nturl2path.pathname2url + self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir') + self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir') + self.assertEqual(fn("C:"), '///C:') + self.assertEqual(fn("C:\\"), '///C:/') + self.assertEqual(fn('c:\\a\\b.c'), '///c:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c\\'), '///C:/a/b.c/') + self.assertEqual(fn('C:\\a\\\\b.c'), '///C:/a//b.c') + self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c') + self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9') + self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo") + # NTFS alternate data streams + self.assertEqual(fn('C:\\foo:bar'), '///C:/foo%3Abar') + self.assertEqual(fn('foo:bar'), 'foo%3Abar') + # No drive letter + self.assertEqual(fn("\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/') + self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') + self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c') + self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9') + # Alternate path separator + self.assertEqual(fn('C:/a/b.c'), '///C:/a/b.c') + self.assertEqual(fn('//some/share/a/b.c'), '//some/share/a/b.c') + self.assertEqual(fn('//?/C:/dir'), '///C:/dir') + self.assertEqual(fn('//?/unc/server/share/dir'), '//server/share/dir') + # Round-tripping + urls = ['///C:', + '///folder/test/', + '///C:/foo/bar/spam.foo'] + for url in urls: + self.assertEqual(fn(nturl2path.url2pathname(url)), url) + + def test_url2pathname(self): + fn = nturl2path.url2pathname + self.assertEqual(fn('/'), '\\') + self.assertEqual(fn('/C:/'), 'C:\\') + self.assertEqual(fn("///C|"), 'C:') + self.assertEqual(fn("///C:"), 'C:') + self.assertEqual(fn('///C:/'), 'C:\\') + self.assertEqual(fn('/C|//'), 'C:\\\\') + self.assertEqual(fn('///C|/path'), 'C:\\path') + # No DOS drive + self.assertEqual(fn("///C/test/"), '\\C\\test\\') + self.assertEqual(fn("////C/test/"), '\\\\C\\test\\') + # DOS drive paths + self.assertEqual(fn('c:/path/to/file'), 'c:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file/'), 'C:\\path\\to\\file\\') + self.assertEqual(fn('C:/path/to//file'), 'C:\\path\\to\\\\file') + self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn("///C|/foo/bar/spam.foo"), 'C:\\foo\\bar\\spam.foo') + # Colons in URI + self.assertEqual(fn('///\u00e8|/'), '\u00e8:\\') + self.assertEqual(fn('//host/share/spam.txt:eggs'), '\\\\host\\share\\spam.txt:eggs') + self.assertEqual(fn('///c:/spam.txt:eggs'), 'c:\\spam.txt:eggs') + # UNC paths + self.assertEqual(fn('//server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('/////server/path/to/file'), '\\\\server\\path\\to\\file') + # Localhost paths + self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/path/to/file'), '\\path\\to\\file') + self.assertEqual(fn('//localhost//server/path/to/file'), '\\\\server\\path\\to\\file') + # Percent-encoded forward slashes are preserved for backwards compatibility + self.assertEqual(fn('C:/foo%2fbar'), 'C:\\foo/bar') + self.assertEqual(fn('//server/share/foo%2fbar'), '\\\\server\\share\\foo/bar') + # Round-tripping + paths = ['C:', + r'\C\test\\', + r'C:\foo\bar\spam.foo'] + for path in paths: + self.assertEqual(fn(nturl2path.pathname2url(path)), path) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_numeric_tower.py b/Lib/test/test_numeric_tower.py index 9cd85e13634..337682d6bac 100644 --- a/Lib/test/test_numeric_tower.py +++ b/Lib/test/test_numeric_tower.py @@ -145,7 +145,7 @@ def test_fractions(self): # The numbers ABC doesn't enforce that the "true" division # of integers produces a float. This tests that the # Rational.__float__() method has required type conversions. - x = F(DummyIntegral(1), DummyIntegral(2), _normalize=False) + x = F._from_coprime_ints(DummyIntegral(1), DummyIntegral(2)) self.assertRaises(TypeError, lambda: x.numerator/x.denominator) self.assertEqual(float(x), 0.5) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 978cc930535..aeb4de6caa5 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1,4 +1,61 @@ +import copy +import pickle +import dis +import threading +import types import unittest +from test.support import threading_helper, check_impl_detail, requires_specialization +from test.support.import_helper import import_module + +# Skip this module on other interpreters, it is cpython specific: +if check_impl_detail(cpython=False): + raise unittest.SkipTest('implementation detail specific to cpython') + +_testinternalcapi = import_module("_testinternalcapi") + + +def disabling_optimizer(func): + def wrapper(*args, **kwargs): + if not hasattr(_testinternalcapi, "get_optimizer"): + return func(*args, **kwargs) + old_opt = _testinternalcapi.get_optimizer() + _testinternalcapi.set_optimizer(None) + try: + return func(*args, **kwargs) + finally: + _testinternalcapi.set_optimizer(old_opt) + + return wrapper + + +class TestBase(unittest.TestCase): + def assert_specialized(self, f, opname): + instructions = dis.get_instructions(f, adaptive=True) + opnames = {instruction.opname for instruction in instructions} + self.assertIn(opname, opnames) + + +class TestLoadSuperAttrCache(unittest.TestCase): + def test_descriptor_not_double_executed_on_spec_fail(self): + calls = [] + class Descriptor: + def __get__(self, instance, owner): + calls.append((instance, owner)) + return lambda: 1 + + class C: + d = Descriptor() + + class D(C): + def f(self): + return super().d() + + d = D() + + self.assertEqual(d.f(), 1) # warmup + calls.clear() + self.assertEqual(d.f(), 1) # try to specialize + self.assertEqual(calls, [(d, D)]) class TestLoadAttrCache(unittest.TestCase): @@ -152,8 +209,6 @@ def f(): for _ in range(1025): self.assertTrue(f()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_metaclass_swap(self): class OldMetaclass(type): @property @@ -179,8 +234,6 @@ def f(): for _ in range(1025): self.assertFalse(f()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_load_shadowing_slot_should_raise_type_error(self): class Class: __slots__ = ("slot",) @@ -199,8 +252,6 @@ def f(o): with self.assertRaises(TypeError): f(o) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_store_shadowing_slot_should_raise_type_error(self): class Class: __slots__ = ("slot",) @@ -218,7 +269,6 @@ def f(o): with self.assertRaises(TypeError): f(o) - @unittest.skip("TODO: RUSTPYTHON") def test_load_borrowed_slot_should_not_crash(self): class Class: __slots__ = ("slot",) @@ -235,7 +285,6 @@ def f(o): with self.assertRaises(TypeError): f(o) - @unittest.skip("TODO: RUSTPYTHON") def test_store_borrowed_slot_should_not_crash(self): class Class: __slots__ = ("slot",) @@ -411,8 +460,6 @@ def f(): for _ in range(1025): self.assertTrue(f()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_metaclass_swap(self): class OldMetaclass(type): @property @@ -439,6 +486,693 @@ def f(): self.assertFalse(f()) +class TestCallCache(TestBase): + def test_too_many_defaults_0(self): + def f(): + pass + + f.__defaults__ = (None,) + for _ in range(1025): + f() + + def test_too_many_defaults_1(self): + def f(x): + pass + + f.__defaults__ = (None, None) + for _ in range(1025): + f(None) + f() + + def test_too_many_defaults_2(self): + def f(x, y): + pass + + f.__defaults__ = (None, None, None) + for _ in range(1025): + f(None, None) + f(None) + f() + + @disabling_optimizer + @requires_specialization + def test_assign_init_code(self): + class MyClass: + def __init__(self): + pass + + def instantiate(): + return MyClass() + + # Trigger specialization + for _ in range(1025): + instantiate() + self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") + + def count_args(self, *args): + self.num_args = len(args) + + # Set MyClass.__init__.__code__ to a code object that is incompatible + # (uses varargs) with the current specialization + MyClass.__init__.__code__ = count_args.__code__ + instantiate() + + +@threading_helper.requires_working_threading() +@requires_specialization +class TestRacesDoNotCrash(TestBase): + # Careful with these. Bigger numbers have a higher chance of catching bugs, + # but you can also burn through a *ton* of type/dict/function versions: + ITEMS = 1000 + LOOPS = 4 + WARMUPS = 2 + WRITERS = 2 + + @disabling_optimizer + def assert_races_do_not_crash( + self, opname, get_items, read, write, *, check_items=False + ): + # This might need a few dozen loops in some cases: + for _ in range(self.LOOPS): + items = get_items() + # Reset: + if check_items: + for item in items: + item.__code__ = item.__code__.replace() + else: + read.__code__ = read.__code__.replace() + # Specialize: + for _ in range(self.WARMUPS): + read(items) + if check_items: + for item in items: + self.assert_specialized(item, opname) + else: + self.assert_specialized(read, opname) + # Create writers: + writers = [] + for _ in range(self.WRITERS): + writer = threading.Thread(target=write, args=[items]) + writers.append(writer) + # Run: + for writer in writers: + writer.start() + read(items) # BOOM! + for writer in writers: + writer.join() + + def test_binary_subscr_getitem(self): + def get_items(): + class C: + __getitem__ = lambda self, item: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item[None] + except TypeError: + pass + + def write(items): + for item in items: + try: + del item.__getitem__ + except AttributeError: + pass + type(item).__getitem__ = lambda self, item: None + + opname = "BINARY_SUBSCR_GETITEM" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_binary_subscr_list_int(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = [None] + items.append(item) + return items + + def read(items): + for item in items: + try: + item[0] + except IndexError: + pass + + def write(items): + for item in items: + item.clear() + item.append(None) + + opname = "BINARY_SUBSCR_LIST_INT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_for_iter_gen(self): + def get_items(): + def g(): + yield + yield + + items = [] + for _ in range(self.ITEMS): + item = g() + items.append(item) + return items + + def read(items): + for item in items: + try: + for _ in item: + break + except ValueError: + pass + + def write(items): + for item in items: + try: + for _ in item: + break + except ValueError: + pass + + opname = "FOR_ITER_GEN" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_for_iter_list(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = [None] + items.append(item) + return items + + def read(items): + for item in items: + for item in item: + break + + def write(items): + for item in items: + item.clear() + item.append(None) + + opname = "FOR_ITER_LIST" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_class(self): + def get_items(): + class C: + a = object() + + items = [] + for _ in range(self.ITEMS): + item = C + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.a + except AttributeError: + pass + item.a = object() + + opname = "LOAD_ATTR_CLASS" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_getattribute_overridden(self): + def get_items(): + class C: + __getattribute__ = lambda self, name: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.__getattribute__ + except AttributeError: + pass + type(item).__getattribute__ = lambda self, name: None + + opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_instance_value(self): + def get_items(): + class C: + pass + + items = [] + for _ in range(self.ITEMS): + item = C() + item.a = None + items.append(item) + return items + + def read(items): + for item in items: + item.a + + def write(items): + for item in items: + item.__dict__[None] = None + + opname = "LOAD_ATTR_INSTANCE_VALUE" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_method_lazy_dict(self): + def get_items(): + class C(Exception): + m = lambda self: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.m() + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.m + except AttributeError: + pass + type(item).m = lambda self: None + + opname = "LOAD_ATTR_METHOD_LAZY_DICT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_method_no_dict(self): + def get_items(): + class C: + __slots__ = () + m = lambda self: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.m() + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.m + except AttributeError: + pass + type(item).m = lambda self: None + + opname = "LOAD_ATTR_METHOD_NO_DICT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_method_with_values(self): + def get_items(): + class C: + m = lambda self: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.m() + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.m + except AttributeError: + pass + type(item).m = lambda self: None + + opname = "LOAD_ATTR_METHOD_WITH_VALUES" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_module(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = types.ModuleType("<item>") + items.append(item) + return items + + def read(items): + for item in items: + try: + item.__name__ + except AttributeError: + pass + + def write(items): + for item in items: + d = item.__dict__.copy() + item.__dict__.clear() + item.__dict__.update(d) + + opname = "LOAD_ATTR_MODULE" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_property(self): + def get_items(): + class C: + a = property(lambda self: None) + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del type(item).a + except AttributeError: + pass + type(item).a = property(lambda self: None) + + opname = "LOAD_ATTR_PROPERTY" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_with_hint(self): + def get_items(): + class C: + pass + + items = [] + for _ in range(self.ITEMS): + item = C() + item.a = None + # Resize into a combined unicode dict: + for i in range(29): + setattr(item, f"_{i}", None) + items.append(item) + return items + + def read(items): + for item in items: + item.a + + def write(items): + for item in items: + item.__dict__[None] = None + + opname = "LOAD_ATTR_WITH_HINT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_global_module(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = eval("lambda: x", {"x": None}) + items.append(item) + return items + + def read(items): + for item in items: + item() + + def write(items): + for item in items: + item.__globals__[None] = None + + opname = "LOAD_GLOBAL_MODULE" + self.assert_races_do_not_crash( + opname, get_items, read, write, check_items=True + ) + + def test_store_attr_instance_value(self): + def get_items(): + class C: + pass + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + item.a = None + + def write(items): + for item in items: + item.__dict__[None] = None + + opname = "STORE_ATTR_INSTANCE_VALUE" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_store_attr_with_hint(self): + def get_items(): + class C: + pass + + items = [] + for _ in range(self.ITEMS): + item = C() + # Resize into a combined unicode dict: + for i in range(29): + setattr(item, f"_{i}", None) + items.append(item) + return items + + def read(items): + for item in items: + item.a = None + + def write(items): + for item in items: + item.__dict__[None] = None + + opname = "STORE_ATTR_WITH_HINT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_store_subscr_list_int(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = [None] + items.append(item) + return items + + def read(items): + for item in items: + try: + item[0] = None + except IndexError: + pass + + def write(items): + for item in items: + item.clear() + item.append(None) + + opname = "STORE_SUBSCR_LIST_INT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_unpack_sequence_list(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = [None] + items.append(item) + return items + + def read(items): + for item in items: + try: + [_] = item + except ValueError: + pass + + def write(items): + for item in items: + item.clear() + item.append(None) + + opname = "UNPACK_SEQUENCE_LIST" + self.assert_races_do_not_crash(opname, get_items, read, write) + +class C: + pass + +@requires_specialization +class TestInstanceDict(unittest.TestCase): + + def setUp(self): + c = C() + c.a, c.b, c.c = 0,0,0 + + def test_values_on_instance(self): + c = C() + c.a = 1 + C().b = 2 + c.c = 3 + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, '<NULL>', 3) + ) + + def test_dict_materialization(self): + c = C() + c.a = 1 + c.b = 2 + c.__dict__ + self.assertEqual(c.__dict__, {"a":1, "b": 2}) + + def test_dict_dematerialization(self): + c = C() + c.a = 1 + c.b = 2 + c.__dict__ + for _ in range(100): + c.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '<NULL>') + ) + + def test_dict_dematerialization_multiple_refs(self): + c = C() + c.a = 1 + c.b = 2 + d = c.__dict__ + for _ in range(100): + c.a + self.assertIs(c.__dict__, d) + + def test_dict_dematerialization_copy(self): + c = C() + c.a = 1 + c.b = 2 + c2 = copy.copy(c) + for _ in range(100): + c.a + c2.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '<NULL>') + ) + self.assertEqual( + _testinternalcapi.get_object_dict_values(c2), + (1, 2, '<NULL>') + ) + c3 = copy.deepcopy(c) + for _ in range(100): + c.a + c3.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '<NULL>') + ) + #NOTE -- c3.__dict__ does not de-materialize + + def test_dict_dematerialization_pickle(self): + c = C() + c.a = 1 + c.b = 2 + c2 = pickle.loads(pickle.dumps(c)) + for _ in range(100): + c.a + c2.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '<NULL>') + ) + self.assertEqual( + _testinternalcapi.get_object_dict_values(c2), + (1, 2, '<NULL>') + ) + + def test_dict_dematerialization_subclass(self): + class D(dict): pass + c = C() + c.a = 1 + c.b = 2 + c.__dict__ = D(c.__dict__) + for _ in range(100): + c.a + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + self.assertEqual( + c.__dict__, + {'a':1, 'b':2} + ) + + def test_store_attr_with_hint(self): + # gh-133441: Regression test for STORE_ATTR_WITH_HINT bytecode + class Node: + def __init__(self): + self.parents = {} + + def __setstate__(self, data_dict): + self.__dict__ = data_dict + self.parents = {} + + class Dict(dict): + pass + + obj = Node() + obj.__setstate__({'parents': {}}) + obj.__setstate__({'parents': {}}) + obj.__setstate__(Dict({'parents': {}})) + + if __name__ == "__main__": - import unittest unittest.main() diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b..f7cc8331b8d 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -39,16 +39,19 @@ class C: pass def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) - self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + self.assertEqual(ns['__annotations__'], {1: 2}) def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): - del globals()['__annotations__'] + globals().pop('__annotations__', None) class C: - del __annotations__ - with self.assertRaises(NameError): - x: int + try: + del __annotations__ + except NameError: + pass + x: int + self.assertEqual(C.__annotations__, {"x": int}) def test_raise_class_exceptions(self): diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index b7e38c23349..1f89986c777 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -1,6 +1,9 @@ import unittest +import inspect import pickle import sys +from decimal import Decimal +from fractions import Fraction from test import support from test.support import import_helper @@ -208,6 +211,9 @@ def test_indexOf(self): nan = float("nan") self.assertEqual(operator.indexOf([nan, nan, 21], nan), 0) self.assertEqual(operator.indexOf([{}, 1, {}, 2], {}), 0) + it = iter('leave the iterator at exactly the position after the match') + self.assertEqual(operator.indexOf(it, 'a'), 2) + self.assertEqual(next(it), 'v') def test_invert(self): operator = self.module @@ -341,6 +347,26 @@ def test_is_not(self): self.assertFalse(operator.is_not(a, b)) self.assertTrue(operator.is_not(a,c)) + def test_is_none(self): + operator = self.module + a = 'xyzpdq' + b = '' + c = None + self.assertRaises(TypeError, operator.is_none) + self.assertFalse(operator.is_none(a)) + self.assertFalse(operator.is_none(b)) + self.assertTrue(operator.is_none(c)) + + def test_is_not_none(self): + operator = self.module + a = 'xyzpdq' + b = '' + c = None + self.assertRaises(TypeError, operator.is_not_none) + self.assertTrue(operator.is_not_none(a)) + self.assertTrue(operator.is_not_none(b)) + self.assertFalse(operator.is_not_none(c)) + def test_attrgetter(self): operator = self.module class A: @@ -456,6 +482,8 @@ def bar(self, f=42): return f def baz(*args, **kwds): return kwds['name'], kwds['self'] + def return_arguments(self, *args, **kwds): + return args, kwds a = A() f = operator.methodcaller('foo') self.assertRaises(IndexError, f, a) @@ -472,6 +500,17 @@ def baz(*args, **kwds): f = operator.methodcaller('baz', name='spam', self='eggs') self.assertEqual(f(a), ('spam', 'eggs')) + many_positional_arguments = tuple(range(10)) + many_kw_arguments = dict(zip('abcdefghij', range(10))) + f = operator.methodcaller('return_arguments', *many_positional_arguments) + self.assertEqual(f(a), (many_positional_arguments, {})) + + f = operator.methodcaller('return_arguments', **many_kw_arguments) + self.assertEqual(f(a), ((), many_kw_arguments)) + + f = operator.methodcaller('return_arguments', *many_positional_arguments, **many_kw_arguments) + self.assertEqual(f(a), (many_positional_arguments, many_kw_arguments)) + def test_inplace(self): operator = self.module class C(object): @@ -505,6 +544,44 @@ def __getitem__(self, other): return 5 # so that C is a sequence self.assertEqual(operator.ixor (c, 5), "ixor") self.assertEqual(operator.iconcat (c, c), "iadd") + def test_iconcat_without_getitem(self): + operator = self.module + + msg = "'int' object can't be concatenated" + with self.assertRaisesRegex(TypeError, msg): + operator.iconcat(1, 0.5) + + def test_index(self): + operator = self.module + class X: + def __index__(self): + return 1 + + self.assertEqual(operator.index(X()), 1) + self.assertEqual(operator.index(0), 0) + self.assertEqual(operator.index(1), 1) + self.assertEqual(operator.index(2), 2) + with self.assertRaises((AttributeError, TypeError)): + operator.index(1.5) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Fraction(3, 7)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Decimal(1)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(None) + + def test_not_(self): + operator = self.module + class C: + def __bool__(self): + raise SyntaxError + self.assertRaises(TypeError, operator.not_) + self.assertRaises(SyntaxError, operator.not_, C()) + self.assertFalse(operator.not_(5)) + self.assertFalse(operator.not_([0])) + self.assertTrue(operator.not_(0)) + self.assertTrue(operator.not_([])) + def test_length_hint(self): operator = self.module class X(object): @@ -530,6 +607,13 @@ def __length_hint__(self): with self.assertRaises(LookupError): operator.length_hint(X(LookupError)) + class Y: pass + + msg = "'str' object cannot be interpreted as an integer" + with self.assertRaisesRegex(TypeError, msg): + operator.length_hint(X(2), "abc") + self.assertEqual(operator.length_hint(Y(), 10), 10) + def test_call(self): operator = self.module @@ -552,6 +636,31 @@ def test_dunder_is_original(self): if dunder: self.assertIs(dunder, orig) + @support.requires_docstrings + def test_attrgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.attrgetter) + self.assertEqual(str(sig), '(attr, /, *attrs)') + sig = inspect.signature(operator.attrgetter('x', 'z', 'y')) + self.assertEqual(str(sig), '(obj, /)') + + @support.requires_docstrings + def test_itemgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.itemgetter) + self.assertEqual(str(sig), '(item, /, *items)') + sig = inspect.signature(operator.itemgetter(2, 3, 5)) + self.assertEqual(str(sig), '(obj, /)') + + @support.requires_docstrings + def test_methodcaller_signature(self): + operator = self.module + sig = inspect.signature(operator.methodcaller) + self.assertEqual(str(sig), '(name, /, *args, **kwargs)') + sig = inspect.signature(operator.methodcaller('foo', 2, y=3)) + self.assertEqual(str(sig), '(obj, /)') + + class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): module = py_operator @@ -560,6 +669,7 @@ class COperatorTestCase(OperatorTestCase, unittest.TestCase): module = c_operator +@support.thread_unsafe("swaps global operator module") class OperatorPickleTestCase: def copy(self, obj, proto): with support.swap_item(sys.modules, 'operator', self.module): diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 28b27446238..e476e472780 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -14,8 +14,9 @@ from io import StringIO from test import support -from test.support import os_helper - +from test.support import cpython_only, os_helper +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots +from test.support.import_helper import ensure_lazy_imports import optparse from optparse import make_option, Option, \ @@ -614,9 +615,9 @@ def test_float_default(self): self.parser.add_option( "-p", "--prob", help="blow up with probability PROB [default: %default]") - self.parser.set_defaults(prob=0.43) + self.parser.set_defaults(prob=0.25) expected_help = self.help_prefix + \ - " -p PROB, --prob=PROB blow up with probability PROB [default: 0.43]\n" + " -p PROB, --prob=PROB blow up with probability PROB [default: 0.25]\n" self.assertHelp(self.parser, expected_help) def test_alt_expand(self): @@ -1655,6 +1656,19 @@ def test__all__(self): not_exported = {'check_builtin', 'AmbiguousOptionError', 'NO_DEFAULT'} support.check__all__(self, optparse, not_exported=not_exported) + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("optparse", {"textwrap"}) + + +class TestTranslations(TestTranslationsBase): + def test_translations(self): + self.assertMsgidsEqual(optparse) + if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(optparse) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index cfb87d78292..378f6c5ab59 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -2,7 +2,9 @@ import contextlib import copy import gc +import operator import pickle +import re from random import randrange, shuffle import struct import sys @@ -122,6 +124,17 @@ def items(self): self.OrderedDict(Spam()) self.assertEqual(calls, ['keys']) + def test_overridden_init(self): + # Sync-up pure Python OD class with C class where + # a consistent internal state is created in __new__ + # rather than __init__. + OrderedDict = self.OrderedDict + class ODNI(OrderedDict): + def __init__(*args, **kwargs): + pass + od = ODNI() + od['a'] = 1 # This used to fail because __init__ was bypassed + def test_fromkeys(self): OrderedDict = self.OrderedDict od = OrderedDict.fromkeys('abc') @@ -134,7 +147,7 @@ def test_fromkeys(self): def test_abc(self): OrderedDict = self.OrderedDict self.assertIsInstance(OrderedDict(), MutableMapping) - self.assertTrue(issubclass(OrderedDict, MutableMapping)) + self.assertIsSubclass(OrderedDict, MutableMapping) def test_clear(self): OrderedDict = self.OrderedDict @@ -281,8 +294,6 @@ def test_equality(self): # different length implied inequality self.assertNotEqual(od1, OrderedDict(pairs[:-1])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copying(self): OrderedDict = self.OrderedDict # Check that ordered dicts are copyable, deepcopyable, picklable, @@ -303,14 +314,14 @@ def check(dup): check(dup) self.assertIs(dup.x, od.x) self.assertIs(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') dup = copy.deepcopy(od) check(dup) self.assertEqual(dup.x, od.x) self.assertIsNot(dup.x, od.x) self.assertEqual(dup.z, od.z) self.assertIsNot(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') # pickle directly pulls the module, so we have to fake it with replaced_module('collections', self.module): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -319,15 +330,13 @@ def check(dup): check(dup) self.assertEqual(dup.x, od.x) self.assertEqual(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') check(eval(repr(od))) update_test = OrderedDict() update_test.update(od) check(update_test) check(OrderedDict(od)) - @unittest.expectedFailure - # TODO: RUSTPYTHON def test_yaml_linkage(self): OrderedDict = self.OrderedDict # Verify that __reduce__ is setup in a way that supports PyYAML's dump() feature. @@ -338,8 +347,6 @@ def test_yaml_linkage(self): # '!!python/object/apply:__main__.OrderedDict\n- - [a, 1]\n - [b, 2]\n' self.assertTrue(all(type(pair)==list for pair in od.__reduce__()[1])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_reduce_not_too_fat(self): OrderedDict = self.OrderedDict # do not save instance dictionary if not needed @@ -351,8 +358,6 @@ def test_reduce_not_too_fat(self): self.assertEqual(od.__dict__['x'], 10) self.assertEqual(od.__reduce__()[2], {'x': 10}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickle_recursive(self): OrderedDict = self.OrderedDict od = OrderedDict() @@ -370,7 +375,7 @@ def test_repr(self): OrderedDict = self.OrderedDict od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]) self.assertEqual(repr(od), - "OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])") + "OrderedDict({'c': 1, 'b': 2, 'a': 3, 'd': 4, 'e': 5, 'f': 6})") self.assertEqual(eval(repr(od)), od) self.assertEqual(repr(OrderedDict()), "OrderedDict()") @@ -380,7 +385,7 @@ def test_repr_recursive(self): od = OrderedDict.fromkeys('abc') od['x'] = od self.assertEqual(repr(od), - "OrderedDict([('a', None), ('b', None), ('c', None), ('x', ...)])") + "OrderedDict({'a': None, 'b': None, 'c': None, 'x': ...})") def test_repr_recursive_values(self): OrderedDict = self.OrderedDict @@ -664,8 +669,6 @@ def test_dict_update(self): dict.update(od, [('spam', 1)]) self.assertNotIn('NULL', repr(od)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_reference_loop(self): # Issue 25935 OrderedDict = self.OrderedDict @@ -677,8 +680,7 @@ class A: gc.collect() self.assertIsNone(r()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.OrderedDict) support.check_free_after_iterating(self, lambda d: iter(d.keys()), self.OrderedDict) @@ -740,11 +742,44 @@ def test_ordered_dict_items_result_gc(self): # when it's mutated and returned from __next__: self.assertTrue(gc.is_tracked(next(it))) + +class _TriggerSideEffectOnEqual: + count = 0 # number of calls to __eq__ + trigger = 1 # count value when to trigger side effect + + def __eq__(self, other): + if self.__class__.count == self.__class__.trigger: + self.side_effect() + self.__class__.count += 1 + return True + + def __hash__(self): + # all instances represent the same key + return -1 + + def side_effect(self): + raise NotImplementedError + class PurePythonOrderedDictTests(OrderedDictTests, unittest.TestCase): module = py_coll OrderedDict = py_coll.OrderedDict + def test_issue119004_attribute_error(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + # This causes an AttributeError due to the linked list being changed + msg = re.escape("'NoneType' object has no attribute 'key'") + self.assertRaisesRegex(AttributeError, msg, operator.eq, dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + class CPythonBuiltinDictTests(unittest.TestCase): """Builtin dict preserves insertion order. @@ -765,8 +800,90 @@ class CPythonBuiltinDictTests(unittest.TestCase): del method +class CPythonOrderedDictSideEffects: + + def check_runtime_error_issue119004(self, dict1, dict2): + msg = re.escape("OrderedDict mutated during iteration") + self.assertRaisesRegex(RuntimeError, msg, operator.eq, dict1, dict2) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised by eq + def test_issue119004_change_size_by_clear(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + dict1.clear() + + dict1 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, {}) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'key' + def test_issue119004_change_size_by_delete_key(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised by eq + def test_issue119004_change_linked_list_by_clear(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + dict1.clear() + dict1['a'] = dict1['b'] = 'c' + + dict1 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys(('a', 'b'), 'c')) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'NoneType' object has no attribute 'key' + def test_issue119004_change_linked_list_by_delete_key(self): + class Key(_TriggerSideEffectOnEqual): + def side_effect(self): + del dict1[TODEL] + dict1['a'] = 'c' + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.check_runtime_error_issue119004(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, {0: None, 'a': 'c', 4.2: None}) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 + def test_issue119004_change_size_by_delete_key_in_dict_eq(self): + class Key(_TriggerSideEffectOnEqual): + trigger = 0 + def side_effect(self): + del dict1[TODEL] + + TODEL = Key() + dict1 = self.OrderedDict(dict.fromkeys((0, TODEL, 4.2))) + dict2 = self.OrderedDict(dict.fromkeys((0, Key(), 4.2))) + self.assertEqual(Key.count, 0) + # the side effect is in dict.__eq__ and modifies the length + self.assertNotEqual(dict1, dict2) + self.assertEqual(Key.count, 2) + self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) + self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + + @unittest.skipUnless(c_coll, 'requires the C version of the collections module') -class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase): +class CPythonOrderedDictTests(OrderedDictTests, + CPythonOrderedDictSideEffects, + unittest.TestCase): module = c_coll OrderedDict = c_coll.OrderedDict @@ -805,8 +922,7 @@ def test_sizeof_exact(self): check(iter(od.items()), itersize) check(iter(od.values()), itersize) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised def test_key_change_during_iteration(self): OrderedDict = self.OrderedDict @@ -824,8 +940,7 @@ def test_key_change_during_iteration(self): del od['c'] self.assertEqual(list(od), list('bdeaf')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_iterators_pickling(self): OrderedDict = self.OrderedDict pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] @@ -877,8 +992,7 @@ class CPythonOrderedDictSubclassTests(CPythonOrderedDictTests): class OrderedDict(c_coll.OrderedDict): pass -# TODO: RUSTPYTHON -@unittest.expectedFailure + class PurePythonOrderedDictWithSlotsCopyingTests(unittest.TestCase): module = py_coll @@ -886,8 +1000,7 @@ class OrderedDict(py_coll.OrderedDict): __slots__ = ('x', 'y') test_copying = OrderedDictTests.test_copying -# TODO: RUSTPYTHON -@unittest.expectedFailure + @unittest.skipUnless(c_coll, 'requires the C version of the collections module') class CPythonOrderedDictWithSlotsCopyingTests(unittest.TestCase): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index a7f8cfe900a..00bd75bab51 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2,6 +2,7 @@ # does add tests for a few functions which have been determined to be more # portable than they had been thought to be. +import asyncio import codecs import contextlib import decimal @@ -10,14 +11,10 @@ import fractions import itertools import locale -# XXX: RUSTPYTHON -try: - import mmap -except ImportError: - pass import os import pickle import select +import selectors import shutil import signal import socket @@ -28,7 +25,6 @@ import sysconfig import tempfile import textwrap -import threading import time import types import unittest @@ -38,15 +34,10 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper -from test.support import threading_helper +from test.support import infinite_recursion from test.support import warnings_helper from platform import win32_is_iot -with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - import asynchat - import asyncore - try: import resource except ImportError: @@ -65,14 +56,16 @@ except (ImportError, AttributeError): all_users = [] try: + import _testcapi from _testcapi import INT_MAX, PY_SSIZE_T_MAX except ImportError: + _testcapi = None INT_MAX = PY_SSIZE_T_MAX = sys.maxsize try: - import _testcapi + import mmap except ImportError: - _testcapi = None + mmap = None from test.support.script_helper import assert_python_ok from test.support import unix_shell @@ -110,6 +103,10 @@ def create_file(filename, content=b'content'): 'on AIX, splice() only accepts sockets') +def tearDownModule(): + asyncio.events._set_event_loop_policy(None) + + class MiscTests(unittest.TestCase): def test_getcwd(self): cwd = os.getcwd() @@ -190,7 +187,9 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") + @unittest.skipIf( + support.is_wasi, "WASI does not support dup." + ) def test_closerange(self): first = os.open(os_helper.TESTFN, os.O_CREAT|os.O_RDWR) # We must allocate two consecutive file descriptors, otherwise @@ -230,6 +229,94 @@ def test_read(self): self.assertEqual(type(s), bytes) self.assertEqual(s, b"spam") + def test_readinto(self): + with open(os_helper.TESTFN, "w+b") as fobj: + fobj.write(b"spam") + fobj.flush() + fd = fobj.fileno() + os.lseek(fd, 0, 0) + # Oversized so readinto without hitting end. + buffer = bytearray(7) + s = os.readinto(fd, buffer) + self.assertEqual(type(s), int) + self.assertEqual(s, 4) + # Should overwrite the first 4 bytes of the buffer. + self.assertEqual(buffer[:4], b"spam") + + # Readinto at EOF should return 0 and not touch buffer. + buffer[:] = b"notspam" + s = os.readinto(fd, buffer) + self.assertEqual(type(s), int) + self.assertEqual(s, 0) + self.assertEqual(bytes(buffer), b"notspam") + s = os.readinto(fd, buffer) + self.assertEqual(s, 0) + self.assertEqual(bytes(buffer), b"notspam") + + # Readinto a 0 length bytearray when at EOF should return 0 + self.assertEqual(os.readinto(fd, bytearray()), 0) + + # Readinto a 0 length bytearray with data available should return 0. + os.lseek(fd, 0, 0) + self.assertEqual(os.readinto(fd, bytearray()), 0) + + @unittest.skipUnless(hasattr(os, 'get_blocking'), + 'needs os.get_blocking() and os.set_blocking()') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @unittest.skipIf(support.is_emscripten, "set_blocking does not work correctly") + def test_readinto_non_blocking(self): + # Verify behavior of a readinto which would block on a non-blocking fd. + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.assertRaises(BlockingIOError): + os.readinto(r, bytearray(5)) + + # Pass some data through + os.write(w, b"spam") + self.assertEqual(os.readinto(r, bytearray(4)), 4) + + # Still don't block or return 0. + with self.assertRaises(BlockingIOError): + os.readinto(r, bytearray(5)) + + # At EOF should return size 0 + os.close(w) + w = None + self.assertEqual(os.readinto(r, bytearray(5)), 0) + self.assertEqual(os.readinto(r, bytearray(5)), 0) # Still EOF + + finally: + os.close(r) + if w is not None: + os.close(w) + + def test_readinto_badarg(self): + with open(os_helper.TESTFN, "w+b") as fobj: + fobj.write(b"spam") + fobj.flush() + fd = fobj.fileno() + os.lseek(fd, 0, 0) + + for bad_arg in ("test", bytes(), 14): + with self.subTest(f"bad buffer {type(bad_arg)}"): + with self.assertRaises(TypeError): + os.readinto(fd, bad_arg) + + with self.subTest("doesn't work on file objects"): + with self.assertRaises(TypeError): + os.readinto(fobj, bytearray(5)) + + # takes two args + with self.assertRaises(TypeError): + os.readinto(fd) + + # No data should have been read with the bad arguments. + buffer = bytearray(4) + s = os.readinto(fd, buffer) + self.assertEqual(s, 4) + self.assertEqual(buffer, b"spam") + @support.cpython_only # Skip the test on 32-bit platforms: the number of bytes must fit in a # Py_ssize_t type @@ -249,6 +336,29 @@ def test_large_read(self, size): # operating system is free to return less bytes than requested. self.assertEqual(data, b'test') + + @support.cpython_only + # Skip the test on 32-bit platforms: the number of bytes must fit in a + # Py_ssize_t type + @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, + "needs INT_MAX < PY_SSIZE_T_MAX") + @support.bigmemtest(size=INT_MAX + 10, memuse=1, dry_run=False) + def test_large_readinto(self, size): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + create_file(os_helper.TESTFN, b'test') + + # Issue #21932: For readinto the buffer contains the length rather than + # a length being passed explicitly to read, should still get capped to a + # valid size / not raise an OverflowError for sizes larger than INT_MAX. + buffer = bytearray(INT_MAX + 10) + with open(os_helper.TESTFN, "rb") as fp: + length = os.readinto(fp.fileno(), buffer) + + # The test does not try to read more than 2 GiB at once because the + # operating system is free to return less bytes than requested. + self.assertEqual(length, 4) + self.assertEqual(buffer[:4], b'test') + def test_write(self): # os.write() accepts bytes- and buffer-like objects but not strings fd = os.open(os_helper.TESTFN, os.O_CREAT | os.O_WRONLY) @@ -557,6 +667,15 @@ def trunc(x): return x nanosecondy = getattr(result, name + "_ns") // 10000 self.assertAlmostEqual(floaty, nanosecondy, delta=2) + # Ensure both birthtime and birthtime_ns roughly agree, if present + try: + floaty = int(result.st_birthtime * 100000) + nanosecondy = result.st_birthtime_ns // 10000 + except AttributeError: + pass + else: + self.assertAlmostEqual(floaty, nanosecondy, delta=2) + try: result[200] self.fail("No exception raised") @@ -608,12 +727,13 @@ def test_stat_attributes_bytes(self): def test_stat_result_pickle(self): result = os.stat(self.fname) for proto in range(pickle.HIGHEST_PROTOCOL + 1): - p = pickle.dumps(result, proto) - self.assertIn(b'stat_result', p) - if proto < 4: - self.assertIn(b'cos\nstat_result\n', p) - unpickled = pickle.loads(p) - self.assertEqual(result, unpickled) + with self.subTest(f'protocol {proto}'): + p = pickle.dumps(result, proto) + self.assertIn(b'stat_result', p) + if proto < 4: + self.assertIn(b'cos\nstat_result\n', p) + unpickled = pickle.loads(p) + self.assertEqual(result, unpickled) @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') def test_statvfs_attributes(self): @@ -697,11 +817,10 @@ def test_15261(self): self.assertEqual(ctx.exception.errno, errno.EBADF) def check_file_attributes(self, result): - self.assertTrue(hasattr(result, 'st_file_attributes')) + self.assertHasAttr(result, 'st_file_attributes') self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat return value doesnt have st_file_attributes attribute") @unittest.skipUnless(sys.platform == "win32", "st_file_attributes is Win32 specific") def test_file_attributes(self): @@ -723,14 +842,13 @@ def test_file_attributes(self): result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, stat.FILE_ATTRIBUTE_DIRECTORY) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_access_denied(self): # Default to FindFirstFile WIN32_FIND_DATA when access is # denied. See issue 28075. # os.environ['TEMP'] should be located on a volume that # supports file ACLs. - fname = os.path.join(os.environ['TEMP'], self.fname) + fname = os.path.join(os.environ['TEMP'], self.fname + "_access") self.addCleanup(os_helper.unlink, fname) create_file(fname, b'ABC') # Deny the right to [S]YNCHRONIZE on the file to @@ -744,8 +862,8 @@ def test_access_denied(self): ) result = os.stat(fname) self.assertNotEqual(result.st_size, 0) + self.assertTrue(os.path.isfile(fname)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 1] Incorrect function.)") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_stat_block_device(self): # bpo-38030: os.stat fails for block devices @@ -794,16 +912,29 @@ def _test_utime(self, set_time, filename=None): set_time(filename, (atime_ns, mtime_ns)) st = os.stat(filename) - if support_subsecond: - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) - self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + if support.is_emscripten: + # Emscripten timestamps are roundtripped through a 53 bit integer of + # nanoseconds. If we want to represent ~50 years which is an 11 + # digits number of seconds: + # 2*log10(60) + log10(24) + log10(365) + log10(60) + log10(50) + # is about 11. Because 53 * log10(2) is about 16, we only have 5 + # digits worth of sub-second precision. + # Some day it would be good to fix this upstream. + delta=1e-5 + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) + self.assertAlmostEqual(st.st_mtime_ns, mtime_ns, delta=1e9 * 1e-5) else: - self.assertEqual(st.st_atime, atime_ns * 1e-9) - self.assertEqual(st.st_mtime, mtime_ns * 1e-9) - self.assertEqual(st.st_atime_ns, atime_ns) - self.assertEqual(st.st_mtime_ns, mtime_ns) + if support_subsecond: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + else: + self.assertEqual(st.st_atime, atime_ns * 1e-9) + self.assertEqual(st.st_mtime, mtime_ns * 1e-9) + self.assertEqual(st.st_atime_ns, atime_ns) + self.assertEqual(st.st_mtime_ns, mtime_ns) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") def test_utime(self): def set_time(filename, ns): # test the ns keyword parameter @@ -817,10 +948,8 @@ def ns_to_sec(ns): # issue, os.utime() rounds towards minus infinity. return (ns * 1e-9) + 0.5e-9 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_utime_by_indexed(self): - # pass times as floating point seconds as the second indexed parameter + # pass times as floating-point seconds as the second indexed parameter def set_time(filename, ns): atime_ns, mtime_ns = ns atime = self.ns_to_sec(atime_ns) @@ -830,8 +959,6 @@ def set_time(filename, ns): os.utime(filename, (atime, mtime)) self._test_utime(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_utime_by_times(self): def set_time(filename, ns): atime_ns, mtime_ns = ns @@ -871,7 +998,6 @@ def set_time(filename, ns): os.utime(name, dir_fd=dirfd, ns=ns) self._test_utime(set_time) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") def test_utime_directory(self): def set_time(filename, ns): # test calling os.utime() on a directory @@ -900,20 +1026,25 @@ def _test_utime_current(self, set_time): self.assertAlmostEqual(st.st_mtime, current, delta=delta, msg=msg) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005)") def test_utime_current(self): def set_time(filename): # Set to the current time in the new way os.utime(self.fname) self._test_utime_current(set_time) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052)") def test_utime_current_old(self): def set_time(filename): # Set to the current time in the old explicit way. os.utime(self.fname, None) self._test_utime_current(set_time) + def test_utime_nonexistent(self): + now = time.time() + filename = 'nonexistent' + with self.assertRaises(FileNotFoundError) as cm: + os.utime(filename, (now, now)) + self.assertEqual(cm.exception.filename, filename) + def get_file_system(self, path): if sys.platform == 'win32': root = os.path.splitdrive(os.path.abspath(path))[0] + '\\' @@ -927,7 +1058,6 @@ def get_file_system(self, path): return buf.value # return None if the filesystem is unknown - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_large_time(self): # Many filesystems are limited to the year 2038. At least, the test # pass with NTFS filesystem. @@ -938,7 +1068,6 @@ def test_large_time(self): os.utime(self.fname, (large, large)) self.assertEqual(os.stat(self.fname).st_mtime, large) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: NotImplementedError not raised)") def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive with self.assertRaises(ValueError): @@ -1015,6 +1144,7 @@ def _empty_mapping(self): @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), 'requires a shell') @unittest.skipUnless(hasattr(os, 'popen'), "needs os.popen()") + @support.requires_subprocess() def test_update2(self): os.environ.clear() os.environ.update(HELLO="World") @@ -1025,6 +1155,7 @@ def test_update2(self): @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), 'requires a shell') @unittest.skipUnless(hasattr(os, 'popen'), "needs os.popen()") + @support.requires_subprocess() def test_os_popen_iter(self): with os.popen("%s -c 'echo \"line1\nline2\nline3\"'" % unix_shell) as popen: @@ -1049,9 +1180,11 @@ def test_items(self): def test___repr__(self): """Check that the repr() of os.environ looks like environ({...}).""" env = os.environ - self.assertEqual(repr(env), 'environ({{{}}})'.format(', '.join( - '{!r}: {!r}'.format(key, value) - for key, value in env.items()))) + formatted_items = ", ".join( + f"{key!r}: {value!r}" + for key, value in env.items() + ) + self.assertEqual(repr(env), f"environ({{{formatted_items}}})") def test_get_exec_path(self): defpath_list = os.defpath.split(os.pathsep) @@ -1096,9 +1229,6 @@ def test_get_exec_path(self): @unittest.skipUnless(os.supports_bytes_environ, "os.environb required for this test.") - # TODO: RUSTPYTHON (UnicodeDecodeError: can't decode bytes for utf-8) - # Need to fix 'surrogateescape' - @unittest.expectedFailure def test_environb(self): # os.environ -> os.environb value = 'euro\u20ac' @@ -1120,6 +1250,7 @@ def test_environb(self): value_str = value.decode(sys.getfilesystemencoding(), 'surrogateescape') self.assertEqual(os.environ['bytes'], value_str) + @support.requires_subprocess() def test_putenv_unsetenv(self): name = "PYTHONTESTVAR" value = "testvalue" @@ -1138,15 +1269,17 @@ def test_putenv_unsetenv(self): stdout=subprocess.PIPE, text=True) self.assertEqual(proc.stdout.rstrip(), repr(None)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: ValueError not raised by putenv)") # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). @support.requires_mac_ver(10, 6) def test_putenv_unsetenv_error(self): # Empty variable name is invalid. # "=" and null character are not allowed in a variable name. - for name in ('', '=name', 'na=me', 'name=', 'name\0', 'na\0me'): + for name in ('', '=name', 'na=me', 'name='): self.assertRaises((OSError, ValueError), os.putenv, name, "value") self.assertRaises((OSError, ValueError), os.unsetenv, name) + for name in ('name\0', 'na\0me'): + self.assertRaises(ValueError, os.putenv, name, "value") + self.assertRaises(ValueError, os.unsetenv, name) if sys.platform == "win32": # On Windows, an environment variable string ("name=value" string) @@ -1197,6 +1330,8 @@ def test_iter_error_when_changing_os_environ_values(self): def _test_underlying_process_env(self, var, expected): if not (unix_shell and os.path.exists(unix_shell)): return + elif not support.has_subprocess_support: + return with os.popen(f"{unix_shell} -c 'echo ${var}'") as popen: value = popen.read().strip() @@ -1284,9 +1419,56 @@ def test_ror_operator(self): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) + def test_reload_environ(self): + # Test os.reload_environ() + has_environb = hasattr(os, 'environb') + + # Test with putenv() which doesn't update os.environ + os.environ['test_env'] = 'python_value' + os.putenv("test_env", "new_value") + self.assertEqual(os.environ['test_env'], 'python_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'python_value') + + os.reload_environ() + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + # Test with unsetenv() which doesn't update os.environ + os.unsetenv('test_env') + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + os.reload_environ() + self.assertNotIn('test_env', os.environ) + if has_environb: + self.assertNotIn(b'test_env', os.environb) + + if has_environb: + # test reload_environ() on os.environb with putenv() + os.environb[b'test_env'] = b'python_value2' + os.putenv("test_env", "new_value2") + self.assertEqual(os.environb[b'test_env'], b'python_value2') + self.assertEqual(os.environ['test_env'], 'python_value2') + + os.reload_environ() + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + # test reload_environ() on os.environb with unsetenv() + os.unsetenv('test_env') + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + os.reload_environ() + self.assertNotIn(b'test_env', os.environb) + self.assertNotIn('test_env', os.environ) class WalkTests(unittest.TestCase): """Tests for os.walk().""" + is_fwalk = False # Wrapper to hide minor differences between os.walk and os.fwalk # to tests both functions with the same code base @@ -1321,14 +1503,14 @@ def setUp(self): self.sub11_path = join(self.sub1_path, "SUB11") sub2_path = join(self.walk_path, "SUB2") sub21_path = join(sub2_path, "SUB21") - tmp1_path = join(self.walk_path, "tmp1") + self.tmp1_path = join(self.walk_path, "tmp1") tmp2_path = join(self.sub1_path, "tmp2") tmp3_path = join(sub2_path, "tmp3") tmp5_path = join(sub21_path, "tmp3") self.link_path = join(sub2_path, "link") t2_path = join(os_helper.TESTFN, "TEST2") tmp4_path = join(os_helper.TESTFN, "TEST2", "tmp4") - broken_link_path = join(sub2_path, "broken_link") + self.broken_link_path = join(sub2_path, "broken_link") broken_link2_path = join(sub2_path, "broken_link2") broken_link3_path = join(sub2_path, "broken_link3") @@ -1338,13 +1520,13 @@ def setUp(self): os.makedirs(sub21_path) os.makedirs(t2_path) - for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: + for path in self.tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: with open(path, "x", encoding='utf-8') as f: f.write("I'm " + path + " and proud of it. Blame test_os.\n") if os_helper.can_symlink(): os.symlink(os.path.abspath(t2_path), self.link_path) - os.symlink('broken', broken_link_path, True) + os.symlink('broken', self.broken_link_path, True) os.symlink(join('tmp3', 'broken'), broken_link2_path, True) os.symlink(join('SUB21', 'tmp5'), broken_link3_path, True) self.sub2_tree = (sub2_path, ["SUB21", "link"], @@ -1438,6 +1620,11 @@ def test_walk_symlink(self): else: self.fail("Didn't follow symlink with followlinks=True") + walk_it = self.walk(self.broken_link_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1459,6 +1646,73 @@ def test_walk_bad_dir(self): finally: os.rename(path1new, path1) + def test_walk_bad_dir2(self): + walk_it = self.walk('nonexisting') + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk('nonexisting', follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(FileNotFoundError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(self.tmp1_path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe(self): + path = os_helper.TESTFN + '-pipe' + os.mkfifo(path) + self.addCleanup(os.unlink, path) + + walk_it = self.walk(path) + self.assertRaises(StopIteration, next, walk_it) + + walk_it = self.walk(path, follow_symlinks=True) + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_walk_named_pipe2(self): + path = os_helper.TESTFN + '-dir' + os.mkdir(path) + self.addCleanup(shutil.rmtree, path) + os.mkfifo(os.path.join(path, 'mypipe')) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + next(walk_it) + self.assertRaises(StopIteration, next, walk_it) + self.assertEqual(errors, []) + + errors = [] + walk_it = self.walk(path, onerror=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(root, path) + self.assertEqual(dirs, []) + self.assertEqual(files, ['mypipe']) + dirs.extend(files) + files.clear() + if self.is_fwalk: + self.assertRaises(NotADirectoryError, next, walk_it) + self.assertRaises(StopIteration, next, walk_it) + if self.is_fwalk: + self.assertEqual(errors, []) + else: + self.assertEqual(len(errors), 1, errors) + self.assertIsInstance(errors[0], NotADirectoryError) + def test_walk_many_open_files(self): depth = 30 base = os.path.join(os_helper.TESTFN, 'deep') @@ -1480,10 +1734,51 @@ def test_walk_many_open_files(self): self.assertEqual(next(it), expected) p = os.path.join(p, 'd') + def test_walk_above_recursion_limit(self): + depth = 50 + os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) + with infinite_recursion(depth - 5): + all = list(self.walk(self.walk_path)) + + sub2_path = self.sub2_tree[0] + for root, dirs, files in all: + if root == sub2_path: + dirs.sort() + files.sort() + + d_entries = [] + d_path = self.walk_path + for _ in range(depth): + d_path = os.path.join(d_path, "d") + d_entries.append((d_path, ["d"], [])) + d_entries[-1][1].clear() + + # Sub-sequences where the order is known + sections = { + "SUB1": [ + (self.sub1_path, ["SUB11"], ["tmp2"]), + (self.sub11_path, [], []), + ], + "SUB2": [self.sub2_tree], + "d": d_entries, + } + + # The ordering of sub-dirs is arbitrary but determines the order in + # which sub-sequences appear + dirs = all[0][1] + expected = [(self.walk_path, dirs, ["tmp1"])] + for d in dirs: + expected.extend(sections[d]) + + self.assertEqual(len(all), depth + 4) + self.assertEqual(sorted(dirs), ["SUB1", "SUB2", "d"]) + self.assertEqual(all, expected) + @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class FwalkTests(WalkTests): """Tests for os.fwalk().""" + is_fwalk = True def walk(self, top, **kwargs): for root, dirs, files, root_fd in self.fwalk(top, **kwargs): @@ -1536,6 +1831,9 @@ def test_yields_correct_dir_fd(self): # check that listdir() returns consistent information self.assertEqual(set(os.listdir(rootfd)), set(dirs) | set(files)) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) def test_fd_leak(self): # Since we're opening a lot of FDs, we must be careful to avoid leaks: # we both check that calling fwalk() a large number of times doesn't @@ -1549,6 +1847,24 @@ def test_fd_leak(self): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) + def test_fd_finalization(self): + # Check that close()ing the fwalk() generator closes FDs + def getfd(): + fd = os.dup(1) + os.close(fd) + return fd + for topdown in (False, True): + old_fd = getfd() + it = self.fwalk(os_helper.TESTFN, topdown=topdown) + self.assertEqual(getfd(), old_fd) + next(it) + self.assertGreater(getfd(), old_fd) + it.close() + self.assertEqual(getfd(), old_fd) + # fwalk() keeps file descriptors open test_walk_many_open_files = None @@ -1566,6 +1882,7 @@ def walk(self, top, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) + @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class BytesFwalkTests(FwalkTests): """Tests for os.walk() with bytes.""" @@ -1578,26 +1895,6 @@ def fwalk(self, top='.', *args, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure - def test_compare_to_walk(self): - super().test_compare_to_walk() - - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure - def test_dir_fd(self): - super().test_dir_fd() - - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure - def test_yields_correct_dir_fd(self): - super().test_yields_correct_dir_fd() - - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure - def test_walk_bottom_up(self): - super().test_walk_bottom_up() - class MakedirTests(unittest.TestCase): def setUp(self): @@ -1618,7 +1915,13 @@ def test_makedir(self): 'dir5', 'dir6') os.makedirs(path) + @unittest.skipIf( + support.is_wasi, + "WASI's umask is a stub." + ) def test_mode(self): + # Note: in some cases, the umask might already be 2 in which case this + # will pass even if os.umask is actually broken. with os_helper.temp_umask(0o002): base = os_helper.TESTFN parent = os.path.join(base, 'dir1') @@ -1630,7 +1933,10 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") + @unittest.skipIf( + support.is_wasi, + "WASI's umask is a stub." + ) def test_exist_ok_existing_directory(self): path = os.path.join(os_helper.TESTFN, 'dir1') mode = 0o777 @@ -1645,7 +1951,10 @@ def test_exist_ok_existing_directory(self): # Issue #25583: A drive root could raise PermissionError on Windows os.makedirs(os.path.abspath('/'), exist_ok=True) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") + @unittest.skipIf( + support.is_wasi, + "WASI's umask is a stub." + ) def test_exist_ok_s_isgid_directory(self): path = os.path.join(os_helper.TESTFN, 'dir1') S_ISGID = stat.S_ISGID @@ -1683,6 +1992,19 @@ def test_exist_ok_existing_regular_file(self): self.assertRaises(OSError, os.makedirs, path, exist_ok=True) os.remove(path) + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + base = os_helper.TESTFN + path = os.path.abspath(os.path.join(os_helper.TESTFN, 'dir')) + os.mkdir(path, mode=0o700) + out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem") + os.rmdir(path) + out = out.strip().rsplit(" ", 1)[1] + self.assertEqual( + out, + '"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', + ) + def tearDown(self): path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', 'dir6') @@ -1695,7 +2017,7 @@ def tearDown(self): os.removedirs(path) -@unittest.skipUnless(hasattr(os, 'chown'), "Test needs chown") +@unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") class ChownFileTests(unittest.TestCase): @classmethod @@ -1796,6 +2118,7 @@ def test_remove_nothing(self): self.assertTrue(os.path.exists(os_helper.TESTFN)) +@unittest.skipIf(support.is_wasi, "WASI has no /dev/null") class DevNullTests(unittest.TestCase): def test_devnull(self): with open(os.devnull, 'wb', 0) as f: @@ -1830,7 +2153,6 @@ def get_urandom_subprocess(self, count): self.assertEqual(len(stdout), count) return stdout - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named 'os'") def test_urandom_subprocess(self): data1 = self.get_urandom_subprocess(16) data2 = self.get_urandom_subprocess(16) @@ -1861,7 +2183,7 @@ def test_getrandom0(self): self.assertEqual(empty, b'') def test_getrandom_random(self): - self.assertTrue(hasattr(os, 'GRND_RANDOM')) + self.assertHasAttr(os, 'GRND_RANDOM') # Don't test os.getrandom(1, os.GRND_RANDOM) to not consume the rare # resource /dev/random @@ -1915,7 +2237,6 @@ def test_urandom_failure(self): """ assert_python_ok('-c', code) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON on Windows (ModuleNotFoundError: No module named 'os')") def test_urandom_fd_closed(self): # Issue #21207: urandom() should reopen its fd to /dev/urandom if # closed. @@ -1930,7 +2251,6 @@ def test_urandom_fd_closed(self): """ rc, out, err = assert_python_ok('-Sc', code) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named 'os'") def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. @@ -1991,8 +2311,7 @@ def mock_execve(name, *args): try: orig_execv = os.execv - # NOTE: RUSTPYTHON os.execve not implemented yet for all platforms - orig_execve = getattr(os, "execve", None) + orig_execve = os.execve orig_defpath = os.defpath os.execv = mock_execv os.execve = mock_execve @@ -2001,10 +2320,7 @@ def mock_execve(name, *args): yield calls finally: os.execv = orig_execv - if orig_execve: - os.execve = orig_execve - else: - del os.execve + os.execve = orig_execve os.defpath = orig_defpath @unittest.skipUnless(hasattr(os, 'execv'), @@ -2022,7 +2338,6 @@ def test_execv_with_bad_arglist(self): self.assertRaises(ValueError, os.execv, 'notepad', ('',)) self.assertRaises(ValueError, os.execv, 'notepad', ['']) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) @@ -2082,7 +2397,6 @@ def test_internal_execvpe_str(self): if os.name != "nt": self._test_internal_execvpe(bytes) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") def test_execve_invalid_env(self): args = [sys.executable, '-c', 'pass'] @@ -2104,7 +2418,6 @@ def test_execve_invalid_env(self): with self.assertRaises(ValueError): os.execve(args[0], args, newenv) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") @unittest.skipUnless(sys.platform == "win32", "Win32-specific test") def test_execve_with_empty_path(self): # bpo-32890: Check GetLastError() misuse @@ -2152,24 +2465,26 @@ def test_chmod(self): self.assertRaises(OSError, os.chmod, os_helper.TESTFN, 0) +@unittest.skipIf(support.is_wasi, "Cannot create invalid FD on WASI.") class TestInvalidFD(unittest.TestCase): - singles = ["fchdir", "dup", "fdatasync", "fstat", - "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] + singles = ["fchdir", "dup", "fstat", "fstatvfs", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir"} + # systemd-nspawn --suppress-sync=true does not verify fd passed + # fdatasync() and fsync(), and always returns success + if not support.in_systemd_nspawn_sync_suppressed(): + singles += ["fdatasync", "fsync"] + singles_fildes |= {"fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): def helper(self): if hasattr(os, f): self.check(getattr(os, f)) + if f in self.singles_fildes: + self.check_bool(getattr(os, f)) return helper for f in singles: - # TODO: RUSTPYTHON: 'fstat' and 'fsync' currently fail on windows, so we've added the if - # statement here to wrap them. When completed remove the if clause and just leave a call to: - # locals()["test_"+f] = get_single(f) - if f in ("fstat", "fsync"): - locals()["test_"+f] = unittest.expectedFailureIfWindows("TODO: RUSTPYTHON fstat test (OSError: [Errno 18] There are no more files.")(get_single(f)) - else: - locals()["test_"+f] = get_single(f) + locals()["test_"+f] = get_single(f) def check(self, f, *args, **kwargs): try: @@ -2180,9 +2495,16 @@ def check(self, f, *args, **kwargs): self.fail("%r didn't raise an OSError with a bad file descriptor" % f) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: <function fdopen at 0x1caa19ae6c0> didn't raise an OSError with a bad file descriptor)") + def check_bool(self, f, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + for fd in False, True: + with self.assertRaises(RuntimeWarning): + f(fd, *args, **kwargs) + def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") + self.check_bool(os.fdopen, encoding="utf-8") @unittest.skipUnless(hasattr(os, 'isatty'), 'test needs os.isatty()') def test_isatty(self): @@ -2208,6 +2530,22 @@ def test_closerange(self): def test_dup2(self): self.check(os.dup2, 20) + @unittest.skipUnless(hasattr(os, 'dup2'), 'test needs os.dup2()') + def test_dup2_negative_fd(self): + valid_fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, valid_fd) + fds = [ + valid_fd, + -1, + -2**31, + ] + for fd, fd2 in itertools.product(fds, repeat=2): + if fd != fd2: + with self.subTest(fd=fd, fd2=fd2): + with self.assertRaises(OSError) as ctx: + os.dup2(fd, fd2) + self.assertEqual(ctx.exception.errno, errno.EBADF) + @unittest.skipUnless(hasattr(os, 'fchmod'), 'test needs os.fchmod()') def test_fchmod(self): self.check(os.fchmod, 0) @@ -2218,25 +2556,37 @@ def test_fchown(self): @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') def test_fpathconf(self): + self.assertIn("PC_NAME_MAX", os.pathconf_names) + self.check_bool(os.pathconf, "PC_NAME_MAX") + self.check_bool(os.fpathconf, "PC_NAME_MAX") + + @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') + @unittest.skipIf( + support.linked_to_musl(), + 'musl pathconf ignores the file descriptor and returns a constant', + ) + def test_fpathconf_bad_fd(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: <builtin_function_or_method object at 0x1f330cd8e60> didn't raise an OSError with a bad file descriptor)") @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) + self.check_bool(os.truncate, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): self.check(os.lseek, 0, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'read'), 'test needs os.read()') def test_read(self): self.check(os.read, 1) + @unittest.skipUnless(hasattr(os, 'readinto'), 'test needs os.readinto()') + def test_readinto(self): + self.check(os.readinto, bytearray(5)) + @unittest.skipUnless(hasattr(os, 'readv'), 'test needs os.readv()') def test_readv(self): buf = bytearray(10) @@ -2246,7 +2596,6 @@ def test_readv(self): def test_tcsetpgrpt(self): self.check(os.tcsetpgrp, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'write'), 'test needs os.write()') def test_write(self): self.check(os.write, b" ") @@ -2255,7 +2604,7 @@ def test_write(self): def test_writev(self): self.check(os.writev, [b'abc']) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @support.requires_subprocess() def test_inheritable(self): self.check(os.get_inheritable) self.check(os.set_inheritable, True) @@ -2267,6 +2616,10 @@ def test_blocking(self): self.check(os.set_blocking, True) + + + +@unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') class LinkTests(unittest.TestCase): def setUp(self): self.file1 = os_helper.TESTFN @@ -2348,6 +2701,7 @@ def test_setreuid(self): self.assertRaises(OverflowError, os.setreuid, 0, self.UID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()') + @support.requires_subprocess() def test_setreuid_neg1(self): # Needs to accept -1. We run this in a subprocess to avoid # altering the test runner's process state (issue8045). @@ -2356,6 +2710,7 @@ def test_setreuid_neg1(self): 'import os,sys;os.setreuid(-1,-1);sys.exit(0)']) @unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()') + @support.requires_subprocess() def test_setregid(self): if os.getuid() != 0 and not HAVE_WHEEL_GROUP: self.assertRaises(OSError, os.setregid, 0, 0) @@ -2365,6 +2720,7 @@ def test_setregid(self): self.assertRaises(OverflowError, os.setregid, 0, self.GID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()') + @support.requires_subprocess() def test_setregid_neg1(self): # Needs to accept -1. We run this in a subprocess to avoid # altering the test runner's process state (issue8045). @@ -2421,8 +2777,10 @@ def test_listdir(self): # test listdir without arguments current_directory = os.getcwd() try: - os.chdir(os.sep) - self.assertEqual(set(os.listdir()), set(os.listdir(os.sep))) + # The root directory is not readable on Android, so use a directory + # we created ourselves. + os.chdir(self.dir) + self.assertEqual(set(os.listdir()), expected) finally: os.chdir(current_directory) @@ -2500,46 +2858,50 @@ def _kill(self, sig): os.kill(proc.pid, sig) self.assertEqual(proc.wait(), sig) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_kill_sigterm(self): # SIGTERM doesn't mean anything special, but make sure it works self._kill(signal.SIGTERM) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_kill_int(self): # os.kill on Windows can take an int which gets set as the exit code self._kill(100) + @unittest.skipIf(mmap is None, "requires mmap") def _kill_with_event(self, event, name): tagname = "test_os_%s" % uuid.uuid1() m = mmap.mmap(-1, 1, tagname) m[0] = 0 + # Run a script which has console control handling enabled. - proc = subprocess.Popen([sys.executable, - os.path.join(os.path.dirname(__file__), - "win_console_handler.py"), tagname], - creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) - # Let the interpreter startup before we send signals. See #3137. - count, max = 0, 100 - while count < max and proc.poll() is None: - if m[0] == 1: - break - time.sleep(0.1) - count += 1 - else: - # Forcefully kill the process if we weren't able to signal it. - os.kill(proc.pid, signal.SIGINT) - self.fail("Subprocess didn't finish initialization") - os.kill(proc.pid, event) - # proc.send_signal(event) could also be done here. - # Allow time for the signal to be passed and the process to exit. - time.sleep(0.5) - if not proc.poll(): - # Forcefully kill the process if we weren't able to signal it. - os.kill(proc.pid, signal.SIGINT) - self.fail("subprocess did not stop on {}".format(name)) + script = os.path.join(os.path.dirname(__file__), + "win_console_handler.py") + cmd = [sys.executable, script, tagname] + proc = subprocess.Popen(cmd, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + + with proc: + # Let the interpreter startup before we send signals. See #3137. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if proc.poll() is None: + break + else: + # Forcefully kill the process if we weren't able to signal it. + proc.kill() + self.fail("Subprocess didn't finish initialization") + + os.kill(proc.pid, event) + + try: + # proc.send_signal(event) could also be done here. + # Allow time for the signal to be passed and the process to exit. + proc.wait(timeout=support.SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + # Forcefully kill the process if we weren't able to signal it. + proc.kill() + self.fail("subprocess did not stop on {}".format(name)) @unittest.skip("subprocesses aren't inheriting Ctrl+C property") + @support.requires_subprocess() def test_CTRL_C_EVENT(self): from ctypes import wintypes import ctypes @@ -2558,8 +2920,7 @@ def test_CTRL_C_EVENT(self): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_subprocess() def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") @@ -2612,6 +2973,54 @@ def test_listdir_extended_path(self): [os.fsencode(path) for path in self.created_paths]) +@unittest.skipUnless(os.name == "nt", "NT specific tests") +class Win32ListdriveTests(unittest.TestCase): + """Test listdrive, listmounts and listvolume on Windows.""" + + def setUp(self): + # Get drives and volumes from fsutil + out = subprocess.check_output( + ["fsutil.exe", "volume", "list"], + cwd=os.path.join(os.getenv("SystemRoot", "\\Windows"), "System32"), + encoding="mbcs", + errors="ignore", + ) + lines = out.splitlines() + self.known_volumes = {l for l in lines if l.startswith('\\\\?\\')} + self.known_drives = {l for l in lines if l[1:] == ':\\'} + self.known_mounts = {l for l in lines if l[1:3] == ':\\'} + + def test_listdrives(self): + drives = os.listdrives() + self.assertIsInstance(drives, list) + self.assertSetEqual( + self.known_drives, + self.known_drives & set(drives), + ) + + def test_listvolumes(self): + volumes = os.listvolumes() + self.assertIsInstance(volumes, list) + self.assertSetEqual( + self.known_volumes, + self.known_volumes & set(volumes), + ) + + def test_listmounts(self): + for volume in os.listvolumes(): + try: + mounts = os.listmounts(volume) + except OSError as ex: + if support.verbose: + print("Skipping", volume, "because of", ex) + else: + self.assertIsInstance(mounts, list) + self.assertSetEqual( + set(mounts), + self.known_mounts & set(mounts), + ) + + @unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') class ReadlinkTests(unittest.TestCase): filelink = 'readlinktest' @@ -2845,6 +3254,7 @@ def test_appexeclink(self): self.assertEqual(st, os.stat(alias)) self.assertFalse(stat.S_ISLNK(st.st_mode)) self.assertEqual(st.st_reparse_tag, stat.IO_REPARSE_TAG_APPEXECLINK) + self.assertTrue(os.path.isfile(alias)) # testing the first one we see is sufficient break else: @@ -2863,7 +3273,6 @@ def tearDown(self): if os.path.lexists(self.junction): os.unlink(self.junction) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_create_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.lexists(self.junction)) @@ -2877,7 +3286,6 @@ def test_create_junction(self): self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), os.path.normcase(os.readlink(self.junction))) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_unlink_removes_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.exists(self.junction)) @@ -2891,7 +3299,8 @@ class Win32NtTests(unittest.TestCase): def test_getfinalpathname_handles(self): nt = import_helper.import_module('nt') ctypes = import_helper.import_module('ctypes') - import ctypes.wintypes + # Ruff false positive -- it thinks we're redefining `ctypes` here + import ctypes.wintypes # noqa: F811 kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE @@ -2936,7 +3345,7 @@ def test_getfinalpathname_handles(self): self.assertEqual(0, handle_delta) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") + @support.requires_subprocess() def test_stat_unlink_race(self): # bpo-46785: the implementation of os.stat() falls back to reading # the parent directory if CreateFileW() fails with a permission @@ -2978,6 +3387,64 @@ def test_stat_unlink_race(self): except subprocess.TimeoutExpired: proc.terminate() + @support.requires_subprocess() + def test_stat_inaccessible_file(self): + filename = os_helper.TESTFN + ICACLS = os.path.expandvars(r"%SystemRoot%\System32\icacls.exe") + + with open(filename, "wb") as f: + f.write(b'Test data') + + stat1 = os.stat(filename) + + try: + # Remove all permissions from the file + subprocess.check_output([ICACLS, filename, "/inheritance:r"], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as ex: + if support.verbose: + print(ICACLS, filename, "/inheritance:r", "failed.") + print(ex.stdout.decode("oem", "replace").rstrip()) + try: + os.unlink(filename) + except OSError: + pass + self.skipTest("Unable to create inaccessible file") + + def cleanup(): + # Give delete permission to the owner (us) + subprocess.check_output([ICACLS, filename, "/grant", "*WD:(D)"], + stderr=subprocess.STDOUT) + os.unlink(filename) + + self.addCleanup(cleanup) + + if support.verbose: + print("File:", filename) + print("stat with access:", stat1) + + # First test - we shouldn't raise here, because we still have access to + # the directory and can extract enough information from its metadata. + stat2 = os.stat(filename) + + if support.verbose: + print(" without access:", stat2) + + # We may not get st_dev/st_ino, so ensure those are 0 or match + self.assertIn(stat2.st_dev, (0, stat1.st_dev)) + self.assertIn(stat2.st_ino, (0, stat1.st_ino)) + + # st_mode and st_size should match (for a normal file, at least) + self.assertEqual(stat1.st_mode, stat2.st_mode) + self.assertEqual(stat1.st_size, stat2.st_size) + + # st_ctime and st_mtime should be the same + self.assertEqual(stat1.st_ctime, stat2.st_ctime) + self.assertEqual(stat1.st_mtime, stat2.st_mtime) + + # st_atime should be the same or later + self.assertGreaterEqual(stat1.st_atime, stat2.st_atime) + @os_helper.skip_unless_symlink class NonLocalSymlinkTests(unittest.TestCase): @@ -3042,14 +3509,17 @@ def test_device_encoding(self): self.assertTrue(codecs.lookup(encoding)) +@support.requires_subprocess() class PidTests(unittest.TestCase): @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid") def test_getppid(self): - p = subprocess.Popen([sys.executable, '-c', + p = subprocess.Popen([sys._base_executable, '-c', 'import os; print(os.getppid())'], - stdout=subprocess.PIPE) - stdout, _ = p.communicate() + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, error = p.communicate() # We are the parent of our subprocess + self.assertEqual(error, b'') self.assertEqual(int(stdout), os.getpid()) def check_waitpid(self, code, exitcode, callback=None): @@ -3081,7 +3551,6 @@ def test_waitstatus_to_exitcode(self): with self.assertRaises(TypeError): os.waitstatus_to_exitcode(0.0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.spawnv not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitpid_windows(self): # bpo-40138: test os.waitpid() and os.waitstatus_to_exitcode() @@ -3090,7 +3559,6 @@ def test_waitpid_windows(self): code = f'import _winapi; _winapi.ExitProcess({STATUS_CONTROL_C_EXIT})' self.check_waitpid(code, exitcode=STATUS_CONTROL_C_EXIT) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OverflowError: Python int too large to convert to Rust i32)") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitstatus_to_exitcode_windows(self): max_exitcode = 2 ** 32 - 1 @@ -3103,7 +3571,7 @@ def test_waitstatus_to_exitcode_windows(self): os.waitstatus_to_exitcode((max_exitcode + 1) << 8) with self.assertRaises(OverflowError): os.waitstatus_to_exitcode(-1) - + # Skip the test on Windows @unittest.skipUnless(hasattr(signal, 'SIGKILL'), 'need signal.SIGKILL') def test_waitstatus_to_exitcode_kill(self): @@ -3116,7 +3584,16 @@ def kill_process(pid): self.check_waitpid(code, exitcode=-signum, callback=kill_process) +@support.requires_subprocess() class SpawnTests(unittest.TestCase): + @staticmethod + def quote_args(args): + # On Windows, os.spawn* simply joins arguments with spaces: + # arguments need to be quoted + if os.name != 'nt': + return args + return [f'"{arg}"' if " " in arg.strip() else arg for arg in args] + def create_args(self, *, with_env=False, use_bytes=False): self.exitcode = 17 @@ -3137,117 +3614,118 @@ def create_args(self, *, with_env=False, use_bytes=False): with open(filename, "w", encoding="utf-8") as fp: fp.write(code) - args = [sys.executable, filename] + program = sys.executable + args = self.quote_args([program, filename]) if use_bytes: + program = os.fsencode(program) args = [os.fsencode(a) for a in args] self.env = {os.fsencode(k): os.fsencode(v) for k, v in self.env.items()} - return args - + return program, args + @requires_os_func('spawnl') def test_spawnl(self): - args = self.create_args() - exitcode = os.spawnl(os.P_WAIT, args[0], *args) + program, args = self.create_args() + exitcode = os.spawnl(os.P_WAIT, program, *args) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnle') def test_spawnle(self): - args = self.create_args(with_env=True) - exitcode = os.spawnle(os.P_WAIT, args[0], *args, self.env) + program, args = self.create_args(with_env=True) + exitcode = os.spawnle(os.P_WAIT, program, *args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnlp') def test_spawnlp(self): - args = self.create_args() - exitcode = os.spawnlp(os.P_WAIT, args[0], *args) + program, args = self.create_args() + exitcode = os.spawnlp(os.P_WAIT, program, *args) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnlpe') def test_spawnlpe(self): - args = self.create_args(with_env=True) - exitcode = os.spawnlpe(os.P_WAIT, args[0], *args, self.env) + program, args = self.create_args(with_env=True) + exitcode = os.spawnlpe(os.P_WAIT, program, *args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnv') def test_spawnv(self): - args = self.create_args() - exitcode = os.spawnv(os.P_WAIT, args[0], args) + program, args = self.create_args() + exitcode = os.spawnv(os.P_WAIT, program, args) self.assertEqual(exitcode, self.exitcode) # Test for PyUnicode_FSConverter() - exitcode = os.spawnv(os.P_WAIT, FakePath(args[0]), args) + exitcode = os.spawnv(os.P_WAIT, FakePath(program), args) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnve') def test_spawnve(self): - args = self.create_args(with_env=True) - exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) + program, args = self.create_args(with_env=True) + exitcode = os.spawnve(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnvp') def test_spawnvp(self): - args = self.create_args() - exitcode = os.spawnvp(os.P_WAIT, args[0], args) + program, args = self.create_args() + exitcode = os.spawnvp(os.P_WAIT, program, args) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnvpe') def test_spawnvpe(self): - args = self.create_args(with_env=True) - exitcode = os.spawnvpe(os.P_WAIT, args[0], args, self.env) + program, args = self.create_args(with_env=True) + exitcode = os.spawnvpe(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnv') def test_nowait(self): - args = self.create_args() - pid = os.spawnv(os.P_NOWAIT, args[0], args) + program, args = self.create_args() + pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) - # TODO: RUSTPYTHON fix spawnv bytes - @unittest.expectedFailure @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) - args = self.create_args(with_env=True, use_bytes=True) - exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) + program, args = self.create_args(with_env=True, use_bytes=True) + exitcode = os.spawnve(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnl') def test_spawnl_noargs(self): - args = self.create_args() - self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, args[0]) - self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, args[0], '') + program, __ = self.create_args() + self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, program) + self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, program, '') @requires_os_func('spawnle') def test_spawnle_noargs(self): - args = self.create_args() - self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, args[0], {}) - self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, args[0], '', {}) + program, __ = self.create_args() + self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, program, {}) + self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, program, '', {}) @requires_os_func('spawnv') def test_spawnv_noargs(self): - args = self.create_args() - self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ()) - self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], []) - self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ('',)) - self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ['']) + program, __ = self.create_args() + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ()) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, []) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ('',)) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ['']) @requires_os_func('spawnve') def test_spawnve_noargs(self): - args = self.create_args() - self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], (), {}) - self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [], {}) - self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], ('',), {}) - self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [''], {}) + program, __ = self.create_args() + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, (), {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, [], {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, ('',), {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, [''], {}) def _test_invalid_env(self, spawn): - args = [sys.executable, '-c', 'pass'] + program = sys.executable + args = self.quote_args([program, '-c', 'pass']) # null character in the environment variable name newenv = os.environ.copy() newenv["FRUIT\0VEGETABLE"] = "cabbage" try: - exitcode = spawn(os.P_WAIT, args[0], args, newenv) + exitcode = spawn(os.P_WAIT, program, args, newenv) except ValueError: pass else: @@ -3257,7 +3735,7 @@ def _test_invalid_env(self, spawn): newenv = os.environ.copy() newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" try: - exitcode = spawn(os.P_WAIT, args[0], args, newenv) + exitcode = spawn(os.P_WAIT, program, args, newenv) except ValueError: pass else: @@ -3267,7 +3745,7 @@ def _test_invalid_env(self, spawn): newenv = os.environ.copy() newenv["FRUIT=ORANGE"] = "lemon" try: - exitcode = spawn(os.P_WAIT, args[0], args, newenv) + exitcode = spawn(os.P_WAIT, program, args, newenv) except ValueError: pass else: @@ -3280,10 +3758,11 @@ def _test_invalid_env(self, spawn): fp.write('import sys, os\n' 'if os.getenv("FRUIT") != "orange=lemon":\n' ' raise AssertionError') - args = [sys.executable, filename] + + args = self.quote_args([program, filename]) newenv = os.environ.copy() newenv["FRUIT"] = "orange=lemon" - exitcode = spawn(os.P_WAIT, args[0], args, newenv) + exitcode = spawn(os.P_WAIT, program, args, newenv) self.assertEqual(exitcode, 0) @requires_os_func('spawnve') @@ -3311,117 +3790,30 @@ class ProgramPriorityTests(unittest.TestCase): """Tests for os.getpriority() and os.setpriority().""" def test_set_get_priority(self): - base = os.getpriority(os.PRIO_PROCESS, os.getpid()) - os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1) - try: - new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) - if base >= 19 and new_prio <= 19: - raise unittest.SkipTest("unable to reliably test setpriority " - "at current nice level of %s" % base) - else: - self.assertEqual(new_prio, base + 1) - finally: - try: - os.setpriority(os.PRIO_PROCESS, os.getpid(), base) - except OSError as err: - if err.errno != errno.EACCES: - raise - - -class SendfileTestServer(asyncore.dispatcher, threading.Thread): - - class Handler(asynchat.async_chat): - - def __init__(self, conn): - asynchat.async_chat.__init__(self, conn) - self.in_buffer = [] - self.accumulate = True - self.closed = False - self.push(b"220 ready\r\n") - - def handle_read(self): - data = self.recv(4096) - if self.accumulate: - self.in_buffer.append(data) - - def get_data(self): - return b''.join(self.in_buffer) - - def handle_close(self): - self.close() - self.closed = True - - def handle_error(self): - raise + code = f"""if 1: + import os + os.setpriority(os.PRIO_PROCESS, os.getpid(), {base} + 1) + print(os.getpriority(os.PRIO_PROCESS, os.getpid())) + """ - def __init__(self, address): - threading.Thread.__init__(self) - asyncore.dispatcher.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.bind(address) - self.listen(5) - self.host, self.port = self.socket.getsockname()[:2] - self.handler_instance = None - self._active = False - self._active_lock = threading.Lock() - - # --- public API - - @property - def running(self): - return self._active - - def start(self): - assert not self.running - self.__flag = threading.Event() - threading.Thread.start(self) - self.__flag.wait() - - def stop(self): - assert self.running - self._active = False - self.join() - - def wait(self): - # wait for handler connection to be closed, then stop the server - while not getattr(self.handler_instance, "closed", False): - time.sleep(0.001) - self.stop() - - # --- internals - - def run(self): - self._active = True - self.__flag.set() - while self._active and asyncore.socket_map: - self._active_lock.acquire() - asyncore.loop(timeout=0.001, count=1) - self._active_lock.release() - asyncore.close_all() - - def handle_accept(self): - conn, addr = self.accept() - self.handler_instance = self.Handler(conn) - - def handle_connect(self): - self.close() - handle_read = handle_connect - - def writable(self): - return 0 - - def handle_error(self): - raise + # Subprocess inherits the current process' priority. + _, out, _ = assert_python_ok("-c", code) + new_prio = int(out) + # nice value cap is 19 for linux and 20 for FreeBSD + if base >= 19 and new_prio <= base: + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) + else: + self.assertEqual(new_prio, base + 1) @unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") -class TestSendfile(unittest.TestCase): +class TestSendfile(unittest.IsolatedAsyncioTestCase): DATA = b"12345abcde" * 16 * 1024 # 160 KiB - SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ - not sys.platform.startswith("solaris") and \ - not sys.platform.startswith("sunos") + SUPPORT_HEADERS_TRAILERS = ( + not sys.platform.startswith(("linux", "android", "solaris", "sunos"))) requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') requires_32b = unittest.skipUnless(sys.maxsize < 2**32, @@ -3429,40 +3821,52 @@ class TestSendfile(unittest.TestCase): @classmethod def setUpClass(cls): - cls.key = threading_helper.threading_setup() create_file(os_helper.TESTFN, cls.DATA) @classmethod def tearDownClass(cls): - threading_helper.threading_cleanup(*cls.key) os_helper.unlink(os_helper.TESTFN) - def setUp(self): - self.server = SendfileTestServer((socket_helper.HOST, 0)) - self.server.start() + @staticmethod + async def chunks(reader): + while not reader.at_eof(): + yield await reader.read() + + async def handle_new_client(self, reader, writer): + self.server_buffer = b''.join([x async for x in self.chunks(reader)]) + writer.close() + self.server.close() # The test server processes a single client only + + async def asyncSetUp(self): + self.server_buffer = b'' + self.server = await asyncio.start_server(self.handle_new_client, + socket_helper.HOSTv4) + server_name = self.server.sockets[0].getsockname() self.client = socket.socket() - self.client.connect((self.server.host, self.server.port)) - self.client.settimeout(1) - # synchronize by waiting for "220 ready" response - self.client.recv(1024) + self.client.setblocking(False) + await asyncio.get_running_loop().sock_connect(self.client, server_name) self.sockno = self.client.fileno() self.file = open(os_helper.TESTFN, 'rb') self.fileno = self.file.fileno() - def tearDown(self): + async def asyncTearDown(self): self.file.close() self.client.close() - if self.server.running: - self.server.stop() - self.server = None + await self.server.wait_closed() - def sendfile_wrapper(self, *args, **kwargs): + # Use the test subject instead of asyncio.loop.sendfile + @staticmethod + async def async_sendfile(*args, **kwargs): + return await asyncio.to_thread(os.sendfile, *args, **kwargs) + + @staticmethod + async def sendfile_wrapper(*args, **kwargs): """A higher level wrapper representing how an application is supposed to use sendfile(). """ while True: try: - return os.sendfile(*args, **kwargs) + return await TestSendfile.async_sendfile(*args, **kwargs) except OSError as err: if err.errno == errno.ECONNRESET: # disconnected @@ -3473,13 +3877,14 @@ def sendfile_wrapper(self, *args, **kwargs): else: raise - def test_send_whole_file(self): + async def test_send_whole_file(self): # normal send total_sent = 0 offset = 0 nbytes = 4096 while total_sent < len(self.DATA): - sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) + sent = await self.sendfile_wrapper(self.sockno, self.fileno, + offset, nbytes) if sent == 0: break offset += sent @@ -3490,19 +3895,19 @@ def test_send_whole_file(self): self.assertEqual(total_sent, len(self.DATA)) self.client.shutdown(socket.SHUT_RDWR) self.client.close() - self.server.wait() - data = self.server.handler_instance.get_data() - self.assertEqual(len(data), len(self.DATA)) - self.assertEqual(data, self.DATA) + await self.server.wait_closed() + self.assertEqual(len(self.server_buffer), len(self.DATA)) + self.assertEqual(self.server_buffer, self.DATA) - def test_send_at_certain_offset(self): + async def test_send_at_certain_offset(self): # start sending a file at a certain offset total_sent = 0 offset = len(self.DATA) // 2 must_send = len(self.DATA) - offset nbytes = 4096 while total_sent < must_send: - sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) + sent = await self.sendfile_wrapper(self.sockno, self.fileno, + offset, nbytes) if sent == 0: break offset += sent @@ -3511,18 +3916,18 @@ def test_send_at_certain_offset(self): self.client.shutdown(socket.SHUT_RDWR) self.client.close() - self.server.wait() - data = self.server.handler_instance.get_data() + await self.server.wait_closed() expected = self.DATA[len(self.DATA) // 2:] self.assertEqual(total_sent, len(expected)) - self.assertEqual(len(data), len(expected)) - self.assertEqual(data, expected) + self.assertEqual(len(self.server_buffer), len(expected)) + self.assertEqual(self.server_buffer, expected) - def test_offset_overflow(self): + async def test_offset_overflow(self): # specify an offset > file size offset = len(self.DATA) + 4096 try: - sent = os.sendfile(self.sockno, self.fileno, offset, 4096) + sent = await self.async_sendfile(self.sockno, self.fileno, + offset, 4096) except OSError as e: # Solaris can raise EINVAL if offset >= file length, ignore. if e.errno != errno.EINVAL: @@ -3531,39 +3936,38 @@ def test_offset_overflow(self): self.assertEqual(sent, 0) self.client.shutdown(socket.SHUT_RDWR) self.client.close() - self.server.wait() - data = self.server.handler_instance.get_data() - self.assertEqual(data, b'') + await self.server.wait_closed() + self.assertEqual(self.server_buffer, b'') - def test_invalid_offset(self): + async def test_invalid_offset(self): with self.assertRaises(OSError) as cm: - os.sendfile(self.sockno, self.fileno, -1, 4096) + await self.async_sendfile(self.sockno, self.fileno, -1, 4096) self.assertEqual(cm.exception.errno, errno.EINVAL) - def test_keywords(self): + async def test_keywords(self): # Keyword arguments should be supported - os.sendfile(out_fd=self.sockno, in_fd=self.fileno, - offset=0, count=4096) + await self.async_sendfile(out_fd=self.sockno, in_fd=self.fileno, + offset=0, count=4096) if self.SUPPORT_HEADERS_TRAILERS: - os.sendfile(out_fd=self.sockno, in_fd=self.fileno, - offset=0, count=4096, - headers=(), trailers=(), flags=0) + await self.async_sendfile(out_fd=self.sockno, in_fd=self.fileno, + offset=0, count=4096, + headers=(), trailers=(), flags=0) # --- headers / trailers tests @requires_headers_trailers - def test_headers(self): + async def test_headers(self): total_sent = 0 expected_data = b"x" * 512 + b"y" * 256 + self.DATA[:-1] - sent = os.sendfile(self.sockno, self.fileno, 0, 4096, - headers=[b"x" * 512, b"y" * 256]) + sent = await self.async_sendfile(self.sockno, self.fileno, 0, 4096, + headers=[b"x" * 512, b"y" * 256]) self.assertLessEqual(sent, 512 + 256 + 4096) total_sent += sent offset = 4096 while total_sent < len(expected_data): nbytes = min(len(expected_data) - total_sent, 4096) - sent = self.sendfile_wrapper(self.sockno, self.fileno, - offset, nbytes) + sent = await self.sendfile_wrapper(self.sockno, self.fileno, + offset, nbytes) if sent == 0: break self.assertLessEqual(sent, nbytes) @@ -3572,12 +3976,11 @@ def test_headers(self): self.assertEqual(total_sent, len(expected_data)) self.client.close() - self.server.wait() - data = self.server.handler_instance.get_data() - self.assertEqual(hash(data), hash(expected_data)) + await self.server.wait_closed() + self.assertEqual(hash(self.server_buffer), hash(expected_data)) @requires_headers_trailers - def test_trailers(self): + async def test_trailers(self): TESTFN2 = os_helper.TESTFN + "2" file_data = b"abcdef" @@ -3585,38 +3988,37 @@ def test_trailers(self): create_file(TESTFN2, file_data) with open(TESTFN2, 'rb') as f: - os.sendfile(self.sockno, f.fileno(), 0, 5, - trailers=[b"123456", b"789"]) + await self.async_sendfile(self.sockno, f.fileno(), 0, 5, + trailers=[b"123456", b"789"]) self.client.close() - self.server.wait() - data = self.server.handler_instance.get_data() - self.assertEqual(data, b"abcde123456789") + await self.server.wait_closed() + self.assertEqual(self.server_buffer, b"abcde123456789") @requires_headers_trailers @requires_32b - def test_headers_overflow_32bits(self): + async def test_headers_overflow_32bits(self): self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: - os.sendfile(self.sockno, self.fileno, 0, 0, - headers=[b"x" * 2**16] * 2**15) + await self.async_sendfile(self.sockno, self.fileno, 0, 0, + headers=[b"x" * 2**16] * 2**15) self.assertEqual(cm.exception.errno, errno.EINVAL) @requires_headers_trailers @requires_32b - def test_trailers_overflow_32bits(self): + async def test_trailers_overflow_32bits(self): self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: - os.sendfile(self.sockno, self.fileno, 0, 0, - trailers=[b"x" * 2**16] * 2**15) + await self.async_sendfile(self.sockno, self.fileno, 0, 0, + trailers=[b"x" * 2**16] * 2**15) self.assertEqual(cm.exception.errno, errno.EINVAL) @requires_headers_trailers @unittest.skipUnless(hasattr(os, 'SF_NODISKIO'), 'test needs os.SF_NODISKIO') - def test_flags(self): + async def test_flags(self): try: - os.sendfile(self.sockno, self.fileno, 0, 4096, - flags=os.SF_NODISKIO) + await self.async_sendfile(self.sockno, self.fileno, 0, 4096, + flags=os.SF_NODISKIO) except OSError as err: if err.errno not in (errno.EBUSY, errno.EAGAIN): raise @@ -3684,10 +4086,10 @@ def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwa xattr.remove("user.test") self.assertEqual(set(listxattr(fn)), xattr) self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo") - setxattr(fn, s("user.test"), b"a"*1024, **kwargs) - self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*1024) + setxattr(fn, s("user.test"), b"a"*256, **kwargs) + self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*256) removexattr(fn, s("user.test"), **kwargs) - many = sorted("user.test{}".format(i) for i in range(100)) + many = sorted("user.test{}".format(i) for i in range(32)) for thing in many: setxattr(fn, thing, b"x", **kwargs) self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) @@ -3734,7 +4136,12 @@ def test_does_not_crash(self): try: size = os.get_terminal_size() except OSError as e: - if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): + known_errnos = [errno.EINVAL, errno.ENOTTY] + if sys.platform == "android": + # The Android testbed redirects the native stdout to a pipe, + # which returns a different error code. + known_errnos.append(errno.EACCES) + if sys.platform == "win32" or e.errno in known_errnos: # Under win32 a generic OSError can be thrown if the # handle cannot be retrieved self.skipTest("failed to query terminal size") @@ -3743,6 +4150,7 @@ def test_does_not_crash(self): self.assertGreaterEqual(size.columns, 0) self.assertGreaterEqual(size.lines, 0) + @support.requires_subprocess() def test_stty_match(self): """Check if stty returns the same results @@ -3771,6 +4179,19 @@ def test_stty_match(self): raise self.assertEqual(expected, actual) + @unittest.skipUnless(sys.platform == 'win32', 'Windows specific test') + def test_windows_fd(self): + """Check if get_terminal_size() returns a meaningful value in Windows""" + try: + conout = open('conout$', 'w') + except OSError: + self.skipTest('failed to open conout$') + with conout: + size = os.get_terminal_size(conout.fileno()) + + self.assertGreaterEqual(size.columns, 0) + self.assertGreaterEqual(size.lines, 0) + @unittest.skipUnless(hasattr(os, 'memfd_create'), 'requires os.memfd_create') @support.requires_linux_version(3, 17) @@ -3871,6 +4292,333 @@ def test_eventfd_select(self): self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) os.eventfd_read(fd) +@unittest.skipUnless(hasattr(os, 'timerfd_create'), 'requires os.timerfd_create') +@unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") +@support.requires_linux_version(2, 6, 30) +class TimerfdTests(unittest.TestCase): + # gh-126112: Use 10 ms to tolerate slow buildbots + CLOCK_RES_PLACES = 2 # 10 ms + CLOCK_RES = 10 ** -CLOCK_RES_PLACES + CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) + + def timerfd_create(self, *args, **kwargs): + fd = os.timerfd_create(*args, **kwargs) + self.assertGreaterEqual(fd, 0) + self.assertFalse(os.get_inheritable(fd)) + self.addCleanup(os.close, fd) + return fd + + def read_count_signaled(self, fd): + # read 8 bytes + data = os.read(fd, 8) + return int.from_bytes(data, byteorder=sys.byteorder) + + def test_timerfd_initval(self): + fd = self.timerfd_create(time.CLOCK_REALTIME) + + initial_expiration = 0.25 + interval = 0.125 + + # 1st call + next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + self.assertAlmostEqual(interval2, 0.0, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, 0.0, places=self.CLOCK_RES_PLACES) + + # 2nd call + next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) + + # timerfd_gettime + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) + + def test_timerfd_non_blocking(self): + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + # 0.1 second later + initial_expiration = 0.1 + os.timerfd_settime(fd, initial=initial_expiration, interval=0) + + # read() raises OSError with errno is EAGAIN for non-blocking timer. + with self.assertRaises(OSError) as ctx: + self.read_count_signaled(fd) + self.assertEqual(ctx.exception.errno, errno.EAGAIN) + + # Wait more than 0.1 seconds + time.sleep(initial_expiration + 0.1) + + # confirm if timerfd is readable and read() returns 1 as bytes. + self.assertEqual(self.read_count_signaled(fd), 1) + + @unittest.skipIf(sys.platform.startswith('netbsd'), + "gh-131263: Skip on NetBSD due to system freeze " + "with negative timer values") + def test_timerfd_negative(self): + one_sec_in_nsec = 10**9 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + test_flags = [0, os.TFD_TIMER_ABSTIME] + if hasattr(os, 'TFD_TIMER_CANCEL_ON_SET'): + test_flags.append(os.TFD_TIMER_ABSTIME | os.TFD_TIMER_CANCEL_ON_SET) + + # Any of 'initial' and 'interval' is negative value. + for initial, interval in ( (-1, 0), (1, -1), (-1, -1), (-0.1, 0), (1, -0.1), (-0.1, -0.1)): + for flags in test_flags: + with self.subTest(flags=flags, initial=initial, interval=interval): + with self.assertRaises(OSError) as context: + os.timerfd_settime(fd, flags=flags, initial=initial, interval=interval) + self.assertEqual(context.exception.errno, errno.EINVAL) + + with self.assertRaises(OSError) as context: + initial_ns = int( one_sec_in_nsec * initial ) + interval_ns = int( one_sec_in_nsec * interval ) + os.timerfd_settime_ns(fd, flags=flags, initial=initial_ns, interval=interval_ns) + self.assertEqual(context.exception.errno, errno.EINVAL) + + def test_timerfd_interval(self): + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1 second + initial_expiration = 1 + # 0.5 second + interval = 0.5 + + os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + + # timerfd_gettime + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) + + count = 3 + t = time.perf_counter() + for _ in range(count): + self.assertEqual(self.read_count_signaled(fd), 1) + t = time.perf_counter() - t + + total_time = initial_expiration + interval * (count - 1) + self.assertGreater(t, total_time - self.CLOCK_RES) + + # wait 3.5 time of interval + time.sleep( (count+0.5) * interval) + self.assertEqual(self.read_count_signaled(fd), count) + + def test_timerfd_TFD_TIMER_ABSTIME(self): + fd = self.timerfd_create(time.CLOCK_REALTIME) + + now = time.clock_gettime(time.CLOCK_REALTIME) + + # 1 second later from now. + offset = 1 + initial_expiration = now + offset + # not interval timer + interval = 0 + + os.timerfd_settime(fd, flags=os.TFD_TIMER_ABSTIME, initial=initial_expiration, interval=interval) + + # timerfd_gettime + # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, offset, places=self.CLOCK_RES_PLACES) + + t = time.perf_counter() + count_signaled = self.read_count_signaled(fd) + t = time.perf_counter() - t + self.assertEqual(count_signaled, 1) + + self.assertGreater(t, offset - self.CLOCK_RES) + + def test_timerfd_select(self): + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + rfd, wfd, xfd = select.select([fd], [fd], [fd], 0) + self.assertEqual((rfd, wfd, xfd), ([], [], [])) + + # 0.25 second + initial_expiration = 0.25 + # every 0.125 second + interval = 0.125 + + os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + + count = 3 + t = time.perf_counter() + for _ in range(count): + rfd, wfd, xfd = select.select([fd], [fd], [fd], initial_expiration + interval) + self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) + self.assertEqual(self.read_count_signaled(fd), 1) + t = time.perf_counter() - t + + total_time = initial_expiration + interval * (count - 1) + self.assertGreater(t, total_time - self.CLOCK_RES) + + def check_timerfd_poll(self, nanoseconds): + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + selector = selectors.DefaultSelector() + selector.register(fd, selectors.EVENT_READ) + self.addCleanup(selector.close) + + sec_to_nsec = 10 ** 9 + # 0.25 second + initial_expiration_ns = sec_to_nsec // 4 + # every 0.125 second + interval_ns = sec_to_nsec // 8 + + if nanoseconds: + os.timerfd_settime_ns(fd, + initial=initial_expiration_ns, + interval=interval_ns) + else: + os.timerfd_settime(fd, + initial=initial_expiration_ns / sec_to_nsec, + interval=interval_ns / sec_to_nsec) + + count = 3 + if nanoseconds: + t = time.perf_counter_ns() + else: + t = time.perf_counter() + for i in range(count): + timeout_margin_ns = interval_ns + if i == 0: + timeout_ns = initial_expiration_ns + interval_ns + timeout_margin_ns + else: + timeout_ns = interval_ns + timeout_margin_ns + + ready = selector.select(timeout_ns / sec_to_nsec) + self.assertEqual(len(ready), 1, ready) + event = ready[0][1] + self.assertEqual(event, selectors.EVENT_READ) + + self.assertEqual(self.read_count_signaled(fd), 1) + + total_time = initial_expiration_ns + interval_ns * (count - 1) + if nanoseconds: + dt = time.perf_counter_ns() - t + self.assertGreater(dt, total_time - self.CLOCK_RES_NS) + else: + dt = time.perf_counter() - t + self.assertGreater(dt, total_time / sec_to_nsec - self.CLOCK_RES) + selector.unregister(fd) + + def test_timerfd_poll(self): + self.check_timerfd_poll(False) + + def test_timerfd_ns_poll(self): + self.check_timerfd_poll(True) + + def test_timerfd_ns_initval(self): + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1st call + initial_expiration_ns = 0 + interval_ns = one_sec_in_nsec // 1000 + next_expiration_ns, interval_ns2 = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + self.assertEqual(interval_ns2, 0) + self.assertEqual(next_expiration_ns, 0) + + # 2nd call + next_expiration_ns, interval_ns2 = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + self.assertEqual(interval_ns2, interval_ns) + self.assertEqual(next_expiration_ns, initial_expiration_ns) + + # timerfd_gettime + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertEqual(interval_ns2, interval_ns) + self.assertLessEqual(next_expiration_ns, initial_expiration_ns) + + self.assertAlmostEqual(next_expiration_ns, initial_expiration_ns, delta=limit_error) + + def test_timerfd_ns_interval(self): + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1 second + initial_expiration_ns = one_sec_in_nsec + # every 0.5 second + interval_ns = one_sec_in_nsec // 2 + + os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + + # timerfd_gettime + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertEqual(interval_ns2, interval_ns) + self.assertLessEqual(next_expiration_ns, initial_expiration_ns) + + count = 3 + t = time.perf_counter_ns() + for _ in range(count): + self.assertEqual(self.read_count_signaled(fd), 1) + t = time.perf_counter_ns() - t + + total_time_ns = initial_expiration_ns + interval_ns * (count - 1) + self.assertGreater(t, total_time_ns - self.CLOCK_RES_NS) + + # wait 3.5 time of interval + time.sleep( (count+0.5) * interval_ns / one_sec_in_nsec) + self.assertEqual(self.read_count_signaled(fd), count) + + + def test_timerfd_ns_TFD_TIMER_ABSTIME(self): + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + now_ns = time.clock_gettime_ns(time.CLOCK_REALTIME) + + # 1 second later from now. + offset_ns = one_sec_in_nsec + initial_expiration_ns = now_ns + offset_ns + # not interval timer + interval_ns = 0 + + os.timerfd_settime_ns(fd, flags=os.TFD_TIMER_ABSTIME, initial=initial_expiration_ns, interval=interval_ns) + + # timerfd_gettime + # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertLess(abs(interval_ns2 - interval_ns), limit_error) + self.assertLess(abs(next_expiration_ns - offset_ns), limit_error) + + t = time.perf_counter_ns() + count_signaled = self.read_count_signaled(fd) + t = time.perf_counter_ns() - t + self.assertEqual(count_signaled, 1) + + self.assertGreater(t, offset_ns - self.CLOCK_RES_NS) + + def test_timerfd_ns_select(self): + one_sec_in_nsec = 10**9 + + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + rfd, wfd, xfd = select.select([fd], [fd], [fd], 0) + self.assertEqual((rfd, wfd, xfd), ([], [], [])) + + # 0.25 second + initial_expiration_ns = one_sec_in_nsec // 4 + # every 0.125 second + interval_ns = one_sec_in_nsec // 8 + + os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + + count = 3 + t = time.perf_counter_ns() + for _ in range(count): + rfd, wfd, xfd = select.select([fd], [fd], [fd], (initial_expiration_ns + interval_ns) / 1e9 ) + self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) + self.assertEqual(self.read_count_signaled(fd), 1) + t = time.perf_counter_ns() - t + + total_time_ns = initial_expiration_ns + interval_ns * (count - 1) + self.assertGreater(t, total_time_ns - self.CLOCK_RES_NS) class OSErrorTests(unittest.TestCase): def setUp(self): @@ -3890,37 +4638,23 @@ class Str(str): else: encoded = os.fsencode(os_helper.TESTFN) self.bytes_filenames.append(encoded) - self.bytes_filenames.append(bytearray(encoded)) - self.bytes_filenames.append(memoryview(encoded)) self.filenames = self.bytes_filenames + self.unicode_filenames - # TODO: RUSTPYTHON (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : <built-in function chdir>) - @unittest.expectedFailure def test_oserror_filename(self): funcs = [ (self.filenames, os.chdir,), - (self.filenames, os.chmod, 0o777), (self.filenames, os.lstat,), (self.filenames, os.open, os.O_RDONLY), (self.filenames, os.rmdir,), (self.filenames, os.stat,), (self.filenames, os.unlink,), + (self.filenames, os.listdir,), + (self.filenames, os.rename, "dst"), + (self.filenames, os.replace, "dst"), ] - if sys.platform == "win32": - funcs.extend(( - (self.bytes_filenames, os.rename, b"dst"), - (self.bytes_filenames, os.replace, b"dst"), - (self.unicode_filenames, os.rename, "dst"), - (self.unicode_filenames, os.replace, "dst"), - (self.unicode_filenames, os.listdir, ), - )) - else: - funcs.extend(( - (self.filenames, os.listdir,), - (self.filenames, os.rename, "dst"), - (self.filenames, os.replace, "dst"), - )) + if os_helper.can_chmod(): + funcs.append((self.filenames, os.chmod, 0o777)) if hasattr(os, "chown"): funcs.append((self.filenames, os.chown, 0, 0)) if hasattr(os, "lchown"): @@ -3934,11 +4668,7 @@ def test_oserror_filename(self): if hasattr(os, "chroot"): funcs.append((self.filenames, os.chroot,)) if hasattr(os, "link"): - if sys.platform == "win32": - funcs.append((self.bytes_filenames, os.link, b"dst")) - funcs.append((self.unicode_filenames, os.link, "dst")) - else: - funcs.append((self.filenames, os.link, "dst")) + funcs.append((self.filenames, os.link, "dst")) if hasattr(os, "listxattr"): funcs.extend(( (self.filenames, os.listxattr,), @@ -3951,34 +4681,58 @@ def test_oserror_filename(self): if hasattr(os, "readlink"): funcs.append((self.filenames, os.readlink,)) - for filenames, func, *func_args in funcs: for name in filenames: try: - if isinstance(name, (str, bytes)): - func(name, *func_args) - else: - with self.assertWarnsRegex(DeprecationWarning, 'should be'): - func(name, *func_args) + func(name, *func_args) except OSError as err: self.assertIs(err.filename, name, str(func)) except UnicodeDecodeError: pass else: - self.fail("No exception thrown by {}".format(func)) + self.fail(f"No exception thrown by {func}") class CPUCountTests(unittest.TestCase): + def check_cpu_count(self, cpus): + if cpus is None: + self.skipTest("Could not determine the number of CPUs") + + self.assertIsInstance(cpus, int) + self.assertGreater(cpus, 0) + def test_cpu_count(self): cpus = os.cpu_count() - if cpus is not None: - self.assertIsInstance(cpus, int) - self.assertGreater(cpus, 0) - else: + self.check_cpu_count(cpus) + + def test_process_cpu_count(self): + cpus = os.process_cpu_count() + self.assertLessEqual(cpus, os.cpu_count()) + self.check_cpu_count(cpus) + + @unittest.skipUnless(hasattr(os, 'sched_setaffinity'), + "don't have sched affinity support") + def test_process_cpu_count_affinity(self): + affinity1 = os.process_cpu_count() + if affinity1 is None: self.skipTest("Could not determine the number of CPUs") + # Disable one CPU + mask = os.sched_getaffinity(0) + if len(mask) <= 1: + self.skipTest(f"sched_getaffinity() returns less than " + f"2 CPUs: {sorted(mask)}") + self.addCleanup(os.sched_setaffinity, 0, list(mask)) + mask.pop() + os.sched_setaffinity(0, mask) + + # test process_cpu_count() + affinity2 = os.process_cpu_count() + self.assertEqual(affinity2, affinity1 - 1) + +# FD inheritance check is only useful for systems with process support. +@support.requires_subprocess() class FDInheritanceTests(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4023,7 +4777,6 @@ def test_get_set_inheritable_o_path(self): os.set_inheritable(fd, False) self.assertEqual(os.get_inheritable(fd), False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable_badf(self): fd = os_helper.make_bad_fd() @@ -4039,7 +4792,6 @@ def test_get_set_inheritable_badf(self): os.set_inheritable(fd, False) self.assertEqual(ctx.exception.errno, errno.EBADF) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4053,7 +4805,6 @@ def test_pipe(self): self.assertEqual(os.get_inheritable(rfd), False) self.assertEqual(os.get_inheritable(wfd), False) - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup(self): fd1 = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd1) @@ -4062,13 +4813,11 @@ def test_dup(self): self.addCleanup(os.close, fd2) self.assertEqual(os.get_inheritable(fd2), False) - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup_standard_stream(self): fd = os.dup(1) self.addCleanup(os.close, fd) self.assertGreater(fd, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.dup not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_dup_nul(self): # os.dup() was creating inheritable fds for character files. @@ -4095,13 +4844,105 @@ def test_dup2(self): self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) self.assertFalse(os.get_inheritable(fd3)) - @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class PseudoterminalTests(unittest.TestCase): + def open_pty(self): + """Open a pty fd-pair, and schedule cleanup for it""" + main_fd, second_fd = os.openpty() + self.addCleanup(os.close, main_fd) + self.addCleanup(os.close, second_fd) + return main_fd, second_fd + def test_openpty(self): - master_fd, slave_fd = os.openpty() - self.addCleanup(os.close, master_fd) - self.addCleanup(os.close, slave_fd) - self.assertEqual(os.get_inheritable(master_fd), False) - self.assertEqual(os.get_inheritable(slave_fd), False) + main_fd, second_fd = self.open_pty() + self.assertEqual(os.get_inheritable(main_fd), False) + self.assertEqual(os.get_inheritable(second_fd), False) + + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_open_via_ptsname(self): + main_fd, second_fd = self.open_pty() + second_path = os.ptsname(main_fd) + reopened_second_fd = os.open(second_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, reopened_second_fd) + os.write(reopened_second_fd, b'foo') + self.assertEqual(os.read(main_fd, 3), b'foo') + + @unittest.skipUnless(hasattr(os, 'posix_openpt'), "need os.posix_openpt()") + @unittest.skipUnless(hasattr(os, 'grantpt'), "need os.grantpt()") + @unittest.skipUnless(hasattr(os, 'unlockpt'), "need os.unlockpt()") + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_posix_pty_functions(self): + mother_fd = os.posix_openpt(os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, mother_fd) + os.grantpt(mother_fd) + os.unlockpt(mother_fd) + son_path = os.ptsname(mother_fd) + son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, son_fd) + self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) + + @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") + @support.requires_subprocess() + def test_pipe_spawnl(self): + # gh-77046: On Windows, os.pipe() file descriptors must be created with + # _O_NOINHERIT to make them non-inheritable. UCRT has no public API to + # get (_osfile(fd) & _O_NOINHERIT), so use a functional test. + # + # Make sure that fd is not inherited by a child process created by + # os.spawnl(): get_osfhandle() and dup() must fail with EBADF. + + fd, fd2 = os.pipe() + self.addCleanup(os.close, fd) + self.addCleanup(os.close, fd2) + + code = textwrap.dedent(f""" + import errno + import os + import test.support + try: + import msvcrt + except ImportError: + msvcrt = None + + fd = {fd} + + with test.support.SuppressCrashReport(): + if msvcrt is not None: + try: + handle = msvcrt.get_osfhandle(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # get_osfhandle(fd) failed with EBADF as expected + else: + raise Exception("get_osfhandle() must fail") + + try: + fd3 = os.dup(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # os.dup(fd) failed with EBADF as expected + else: + os.close(fd3) + raise Exception("dup must fail") + """) + + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(filename, "w") as fp: + print(code, file=fp, end="") + + executable = sys.executable + cmd = [executable, filename] + if os.name == "nt" and " " in cmd[0]: + cmd[0] = f'"{cmd[0]}"' + exitcode = os.spawnl(os.P_WAIT, executable, *cmd) + self.assertEqual(exitcode, 0) class PathTConverterTests(unittest.TestCase): @@ -4113,11 +4954,9 @@ class PathTConverterTests(unittest.TestCase): ('access', False, (os.F_OK,), None), ('chflags', False, (0,), None), ('lchflags', False, (0,), None), - ('open', False, (0,), getattr(os, 'close', None)), + ('open', False, (os.O_RDONLY,), getattr(os, 'close', None)), ] - # TODO: RUSTPYTHON (AssertionError: TypeError not raised) - @unittest.expectedFailure def test_path_t_converter(self): str_filename = os_helper.TESTFN if os.name == 'nt': @@ -4174,6 +5013,8 @@ def test_path_t_converter_and_custom_class(self): @unittest.skipUnless(hasattr(os, 'get_blocking'), 'needs os.get_blocking() and os.set_blocking()') +@unittest.skipIf(support.is_emscripten, "Cannot unset blocking flag") +@unittest.skipIf(sys.platform == 'win32', 'Windows only supports blocking on pipes') class BlockingTests(unittest.TestCase): def test_blocking(self): fd = os.open(__file__, os.O_RDONLY) @@ -4200,13 +5041,9 @@ def setUp(self): self.addCleanup(os_helper.rmtree, self.path) os.mkdir(self.path) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by DirEntry) - @unittest.expectedFailure def test_uninstantiable(self): self.assertRaises(TypeError, os.DirEntry) - # TODO: RUSTPYTHON (pickle.PicklingError: Can't pickle <class '_os.DirEntry'>: it's not found as _os.DirEntry) - @unittest.expectedFailure def test_unpickable(self): filename = create_file(os.path.join(self.path, "file.txt"), b'python') entry = [entry for entry in os.scandir(self.path)].pop() @@ -4242,7 +5079,8 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): for attr in dir(stat1): if not attr.startswith("st_"): continue - if attr in ("st_dev", "st_ino", "st_nlink"): + if attr in ("st_dev", "st_ino", "st_nlink", "st_ctime", + "st_ctime_ns"): continue self.assertEqual(getattr(stat1, attr), getattr(stat2, attr), @@ -4250,8 +5088,6 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): else: self.assertEqual(stat1, stat2) - # TODO: RUSTPPYTHON (AssertionError: TypeError not raised by ScandirIter) - @unittest.expectedFailure def test_uninstantiable(self): scandir_iter = os.scandir(self.path) self.assertRaises(TypeError, type(scandir_iter)) @@ -4285,6 +5121,8 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): self.assertEqual(entry.is_file(follow_symlinks=False), stat.S_ISREG(entry_lstat.st_mode)) + self.assertEqual(entry.is_junction(), os.path.isjunction(entry.path)) + self.assert_stat_equal(entry.stat(), entry_stat, os.name == 'nt' and not is_symlink) @@ -4292,9 +5130,9 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): entry_lstat, os.name == 'nt') - @unittest.skipIf(sys.platform == "linux", "TODO: RUSTPYTHON, flaky test") + @unittest.skipIf(sys.platform == "linux", "TODO: RUSTPYTHON; flaky test") def test_attributes(self): - link = hasattr(os, 'link') + link = os_helper.can_hardlink() symlink = os_helper.can_symlink() dirname = os.path.join(self.path, "dir") @@ -4334,6 +5172,21 @@ def test_attributes(self): entry = entries['symlink_file.txt'] self.check_entry(entry, 'symlink_file.txt', False, True, True) + @unittest.skipIf(sys.platform != 'win32', "Can only test junctions with creation on win32.") + def test_attributes_junctions(self): + dirname = os.path.join(self.path, "tgtdir") + os.mkdir(dirname) + + import _winapi + try: + _winapi.CreateJunction(dirname, os.path.join(self.path, "srcjunc")) + except OSError: + raise unittest.SkipTest('creating the test junction failed') + + entries = self.get_entries(['srcjunc', 'tgtdir']) + self.assertEqual(entries['srcjunc'].is_junction(), True) + self.assertEqual(entries['tgtdir'].is_junction(), False) + def get_entry(self, name): path = self.bytes_path if isinstance(name, bytes) else self.path entries = list(os.scandir(path)) @@ -4377,7 +5230,6 @@ def test_fspath_protocol_bytes(self): self.assertEqual(fspath, os.path.join(os.fsencode(self.path),bytes_filename)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_dir() is False") def test_removed_dir(self): path = os.path.join(self.path, 'dir') @@ -4400,7 +5252,6 @@ def test_removed_dir(self): self.assertRaises(FileNotFoundError, entry.stat) self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_file() is False") def test_removed_file(self): entry = self.create_file_entry() os.unlink(entry.path) @@ -4453,26 +5304,14 @@ def test_bytes(self): self.assertEqual(entry.path, os.fsencode(os.path.join(self.path, 'file.txt'))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bytes_like(self): self.create_file("file.txt") for cls in bytearray, memoryview: path_bytes = cls(os.fsencode(self.path)) - with self.assertWarns(DeprecationWarning): - entries = list(os.scandir(path_bytes)) - self.assertEqual(len(entries), 1, entries) - entry = entries[0] - - self.assertEqual(entry.name, b'file.txt') - self.assertEqual(entry.path, - os.fsencode(os.path.join(self.path, 'file.txt'))) - self.assertIs(type(entry.name), bytes) - self.assertIs(type(entry.path), bytes) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + with self.assertRaises(TypeError): + os.scandir(path_bytes) + @unittest.skipUnless(os.listdir in os.supports_fd, 'fd support for listdir required for this test.') def test_fd(self): @@ -4499,7 +5338,7 @@ def test_fd(self): st = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) self.assertEqual(entry.stat(follow_symlinks=False), st) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: FileNotFoundError not raised by scandir)") + @unittest.skipIf(support.is_wasi, "WASI maps '' to cwd") def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') @@ -4554,8 +5393,6 @@ def test_context_manager_exception(self): with self.check_no_resource_warning(): del iterator - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_resource_warning(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -4595,8 +5432,8 @@ def test_fsencode_fsdecode(self): def test_pathlike(self): self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) - self.assertTrue(issubclass(FakePath, os.PathLike)) - self.assertTrue(isinstance(FakePath('x'), os.PathLike)) + self.assertIsSubclass(FakePath, os.PathLike) + self.assertIsInstance(FakePath('x'), os.PathLike) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -4622,12 +5459,57 @@ def test_pathlike_subclasshook(self): # true on abstract implementation. class A(os.PathLike): pass - self.assertFalse(issubclass(FakePath, A)) - self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertNotIsSubclass(FakePath, A) + self.assertIsSubclass(FakePath, os.PathLike) def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) + def test_pathlike_subclass_slots(self): + class A(os.PathLike): + __slots__ = () + def __fspath__(self): + return '' + self.assertNotHasAttr(A(), '__dict__') + + def test_fspath_set_to_None(self): + class Foo: + __fspath__ = None + + class Bar: + def __fspath__(self): + return 'bar' + + class Baz(Bar): + __fspath__ = None + + good_error_msg = ( + r"expected str, bytes or os.PathLike object, not {}".format + ) + + with self.assertRaisesRegex(TypeError, good_error_msg("Foo")): + self.fspath(Foo()) + + self.assertEqual(self.fspath(Bar()), 'bar') + + with self.assertRaisesRegex(TypeError, good_error_msg("Baz")): + self.fspath(Baz()) + + with self.assertRaisesRegex(TypeError, good_error_msg("Foo")): + open(Foo()) + + with self.assertRaisesRegex(TypeError, good_error_msg("Baz")): + open(Baz()) + + other_good_error_msg = ( + r"should be string, bytes or os.PathLike, not {}".format + ) + + with self.assertRaisesRegex(TypeError, other_good_error_msg("Foo")): + os.rename(Foo(), "foooo") + + with self.assertRaisesRegex(TypeError, other_good_error_msg("Baz")): + os.rename(Baz(), "bazzz") class TimesTests(unittest.TestCase): def test_times(self): @@ -4645,7 +5527,7 @@ def test_times(self): self.assertEqual(times.elapsed, 0) -@requires_os_func('fork') +@support.requires_fork() class ForkTests(unittest.TestCase): def test_fork(self): # bpo-42540: ensure os.fork() with non-default memory allocator does @@ -4658,11 +5540,14 @@ def test_fork(self): support.wait_process(pid, exitcode=0) """ assert_python_ok("-c", code) - assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + if support.Py_GIL_DISABLED: + assert_python_ok("-c", code, PYTHONMALLOC="mimalloc_debug") + else: + assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") - @unittest.skipIf(_testcapi is None, 'TODO: RUSTPYTHON; needs _testcapi') - @unittest.skipUnless(sys.platform in ("linux", "darwin"), + @unittest.skipUnless(sys.platform in ("linux", "android", "darwin"), "Only Linux and macOS detect this today.") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_fork_warns_when_non_python_thread_exists(self): code = """if 1: import os, threading, warnings @@ -4689,6 +5574,23 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") + def test_fork_at_finalization(self): + code = """if 1: + import atexit + import os + + class AtFinalization: + def __del__(self): + print("OK") + pid = os.fork() + if pid != 0: + print("shouldn't be printed") + at_finalization = AtFinalization() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(b"OK\n", out) + self.assertIn(b"can't fork at interpreter shutdown", err) + # Only test if the C version is provided, otherwise TestPEP519 already tested # the pure Python implementation. diff --git a/Lib/test/test_osx_env.py b/Lib/test/test_osx_env.py new file mode 100644 index 00000000000..80198edcb80 --- /dev/null +++ b/Lib/test/test_osx_env.py @@ -0,0 +1,34 @@ +""" +Test suite for OS X interpreter environment variables. +""" + +from test.support.os_helper import EnvironmentVarGuard +import subprocess +import sys +import sysconfig +import unittest + +@unittest.skipUnless(sys.platform == 'darwin' and + sysconfig.get_config_var('WITH_NEXT_FRAMEWORK'), + 'unnecessary on this platform') +class OSXEnvironmentVariableTestCase(unittest.TestCase): + def _check_sys(self, ev, cond, sv, val = sys.executable + 'dummy'): + with EnvironmentVarGuard() as evg: + subpc = [str(sys.executable), '-c', + 'import sys; sys.exit(2 if "%s" %s %s else 3)' % (val, cond, sv)] + # ensure environment variable does not exist + evg.unset(ev) + # test that test on sys.xxx normally fails + rc = subprocess.call(subpc) + self.assertEqual(rc, 3, "expected %s not %s %s" % (ev, cond, sv)) + # set environ variable + evg.set(ev, val) + # test that sys.xxx has been influenced by the environ value + rc = subprocess.call(subpc) + self.assertEqual(rc, 2, "expected %s %s %s" % (ev, cond, sv)) + + def test_pythonexecutable_sets_sys_executable(self): + self._check_sys('PYTHONEXECUTABLE', '==', 'sys.executable') + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py deleted file mode 100644 index 5c4480cf31f..00000000000 --- a/Lib/test/test_pathlib.py +++ /dev/null @@ -1,2742 +0,0 @@ -import collections.abc -import io -import os -import sys -import errno -import pathlib -import pickle -import socket -import stat -import tempfile -import unittest -from unittest import mock - -from test.support import import_helper -from test.support import os_helper -from test.support.os_helper import TESTFN, FakePath - -try: - import grp, pwd -except ImportError: - grp = pwd = None - - -class _BaseFlavourTest(object): - - def _check_parse_parts(self, arg, expected): - f = self.flavour.parse_parts - sep = self.flavour.sep - altsep = self.flavour.altsep - actual = f([x.replace('/', sep) for x in arg]) - self.assertEqual(actual, expected) - if altsep: - actual = f([x.replace('/', altsep) for x in arg]) - self.assertEqual(actual, expected) - - def test_parse_parts_common(self): - check = self._check_parse_parts - sep = self.flavour.sep - # Unanchored parts. - check([], ('', '', [])) - check(['a'], ('', '', ['a'])) - check(['a/'], ('', '', ['a'])) - check(['a', 'b'], ('', '', ['a', 'b'])) - # Expansion. - check(['a/b'], ('', '', ['a', 'b'])) - check(['a/b/'], ('', '', ['a', 'b'])) - check(['a', 'b/c', 'd'], ('', '', ['a', 'b', 'c', 'd'])) - # Collapsing and stripping excess slashes. - check(['a', 'b//c', 'd'], ('', '', ['a', 'b', 'c', 'd'])) - check(['a', 'b/c/', 'd'], ('', '', ['a', 'b', 'c', 'd'])) - # Eliminating standalone dots. - check(['.'], ('', '', [])) - check(['.', '.', 'b'], ('', '', ['b'])) - check(['a', '.', 'b'], ('', '', ['a', 'b'])) - check(['a', '.', '.'], ('', '', ['a'])) - # The first part is anchored. - check(['/a/b'], ('', sep, [sep, 'a', 'b'])) - check(['/a', 'b'], ('', sep, [sep, 'a', 'b'])) - check(['/a/', 'b'], ('', sep, [sep, 'a', 'b'])) - # Ignoring parts before an anchored part. - check(['a', '/b', 'c'], ('', sep, [sep, 'b', 'c'])) - check(['a', '/b', '/c'], ('', sep, [sep, 'c'])) - - -class PosixFlavourTest(_BaseFlavourTest, unittest.TestCase): - flavour = pathlib._posix_flavour - - def test_parse_parts(self): - check = self._check_parse_parts - # Collapsing of excess leading slashes, except for the double-slash - # special case. - check(['//a', 'b'], ('', '//', ['//', 'a', 'b'])) - check(['///a', 'b'], ('', '/', ['/', 'a', 'b'])) - check(['////a', 'b'], ('', '/', ['/', 'a', 'b'])) - # Paths which look like NT paths aren't treated specially. - check(['c:a'], ('', '', ['c:a'])) - check(['c:\\a'], ('', '', ['c:\\a'])) - check(['\\a'], ('', '', ['\\a'])) - - def test_splitroot(self): - f = self.flavour.splitroot - self.assertEqual(f(''), ('', '', '')) - self.assertEqual(f('a'), ('', '', 'a')) - self.assertEqual(f('a/b'), ('', '', 'a/b')) - self.assertEqual(f('a/b/'), ('', '', 'a/b/')) - self.assertEqual(f('/a'), ('', '/', 'a')) - self.assertEqual(f('/a/b'), ('', '/', 'a/b')) - self.assertEqual(f('/a/b/'), ('', '/', 'a/b/')) - # The root is collapsed when there are redundant slashes - # except when there are exactly two leading slashes, which - # is a special case in POSIX. - self.assertEqual(f('//a'), ('', '//', 'a')) - self.assertEqual(f('///a'), ('', '/', 'a')) - self.assertEqual(f('///a/b'), ('', '/', 'a/b')) - # Paths which look like NT paths aren't treated specially. - self.assertEqual(f('c:/a/b'), ('', '', 'c:/a/b')) - self.assertEqual(f('\\/a/b'), ('', '', '\\/a/b')) - self.assertEqual(f('\\a\\b'), ('', '', '\\a\\b')) - - -class NTFlavourTest(_BaseFlavourTest, unittest.TestCase): - flavour = pathlib._windows_flavour - - def test_parse_parts(self): - check = self._check_parse_parts - # First part is anchored. - check(['c:'], ('c:', '', ['c:'])) - check(['c:/'], ('c:', '\\', ['c:\\'])) - check(['/'], ('', '\\', ['\\'])) - check(['c:a'], ('c:', '', ['c:', 'a'])) - check(['c:/a'], ('c:', '\\', ['c:\\', 'a'])) - check(['/a'], ('', '\\', ['\\', 'a'])) - # UNC paths. - check(['//a/b'], ('\\\\a\\b', '\\', ['\\\\a\\b\\'])) - check(['//a/b/'], ('\\\\a\\b', '\\', ['\\\\a\\b\\'])) - check(['//a/b/c'], ('\\\\a\\b', '\\', ['\\\\a\\b\\', 'c'])) - # Second part is anchored, so that the first part is ignored. - check(['a', 'Z:b', 'c'], ('Z:', '', ['Z:', 'b', 'c'])) - check(['a', 'Z:/b', 'c'], ('Z:', '\\', ['Z:\\', 'b', 'c'])) - # UNC paths. - check(['a', '//b/c', 'd'], ('\\\\b\\c', '\\', ['\\\\b\\c\\', 'd'])) - # Collapsing and stripping excess slashes. - check(['a', 'Z://b//c/', 'd/'], ('Z:', '\\', ['Z:\\', 'b', 'c', 'd'])) - # UNC paths. - check(['a', '//b/c//', 'd'], ('\\\\b\\c', '\\', ['\\\\b\\c\\', 'd'])) - # Extended paths. - check(['//?/c:/'], ('\\\\?\\c:', '\\', ['\\\\?\\c:\\'])) - check(['//?/c:/a'], ('\\\\?\\c:', '\\', ['\\\\?\\c:\\', 'a'])) - check(['//?/c:/a', '/b'], ('\\\\?\\c:', '\\', ['\\\\?\\c:\\', 'b'])) - # Extended UNC paths (format is "\\?\UNC\server\share"). - check(['//?/UNC/b/c'], ('\\\\?\\UNC\\b\\c', '\\', ['\\\\?\\UNC\\b\\c\\'])) - check(['//?/UNC/b/c/d'], ('\\\\?\\UNC\\b\\c', '\\', ['\\\\?\\UNC\\b\\c\\', 'd'])) - # Second part has a root but not drive. - check(['a', '/b', 'c'], ('', '\\', ['\\', 'b', 'c'])) - check(['Z:/a', '/b', 'c'], ('Z:', '\\', ['Z:\\', 'b', 'c'])) - check(['//?/Z:/a', '/b', 'c'], ('\\\\?\\Z:', '\\', ['\\\\?\\Z:\\', 'b', 'c'])) - - def test_splitroot(self): - f = self.flavour.splitroot - self.assertEqual(f(''), ('', '', '')) - self.assertEqual(f('a'), ('', '', 'a')) - self.assertEqual(f('a\\b'), ('', '', 'a\\b')) - self.assertEqual(f('\\a'), ('', '\\', 'a')) - self.assertEqual(f('\\a\\b'), ('', '\\', 'a\\b')) - self.assertEqual(f('c:a\\b'), ('c:', '', 'a\\b')) - self.assertEqual(f('c:\\a\\b'), ('c:', '\\', 'a\\b')) - # Redundant slashes in the root are collapsed. - self.assertEqual(f('\\\\a'), ('', '\\', 'a')) - self.assertEqual(f('\\\\\\a/b'), ('', '\\', 'a/b')) - self.assertEqual(f('c:\\\\a'), ('c:', '\\', 'a')) - self.assertEqual(f('c:\\\\\\a/b'), ('c:', '\\', 'a/b')) - # Valid UNC paths. - self.assertEqual(f('\\\\a\\b'), ('\\\\a\\b', '\\', '')) - self.assertEqual(f('\\\\a\\b\\'), ('\\\\a\\b', '\\', '')) - self.assertEqual(f('\\\\a\\b\\c\\d'), ('\\\\a\\b', '\\', 'c\\d')) - # These are non-UNC paths (according to ntpath.py and test_ntpath). - # However, command.com says such paths are invalid, so it's - # difficult to know what the right semantics are. - self.assertEqual(f('\\\\\\a\\b'), ('', '\\', 'a\\b')) - self.assertEqual(f('\\\\a'), ('', '\\', 'a')) - - -# -# Tests for the pure classes. -# - -class _BasePurePathTest(object): - - # Keys are canonical paths, values are list of tuples of arguments - # supposed to produce equal paths. - equivalences = { - 'a/b': [ - ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), - ('a/b/',), ('a//b',), ('a//b//',), - # Empty components get removed. - ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), - ], - '/b/c/d': [ - ('a', '/b/c', 'd'), ('a', '///b//c', 'd/'), - ('/a', '/b/c', 'd'), - # Empty components get removed. - ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), - ], - } - - def setUp(self): - p = self.cls('a') - self.flavour = p._flavour - self.sep = self.flavour.sep - self.altsep = self.flavour.altsep - - def test_constructor_common(self): - P = self.cls - p = P('a') - self.assertIsInstance(p, P) - P('a', 'b', 'c') - P('/a', 'b', 'c') - P('a/b/c') - P('/a/b/c') - P(FakePath("a/b/c")) - self.assertEqual(P(P('a')), P('a')) - self.assertEqual(P(P('a'), 'b'), P('a/b')) - self.assertEqual(P(P('a'), P('b')), P('a/b')) - self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) - - def _check_str_subclass(self, *args): - # Issue #21127: it should be possible to construct a PurePath object - # from a str subclass instance, and it then gets converted to - # a pure str object. - class StrSubclass(str): - pass - P = self.cls - p = P(*(StrSubclass(x) for x in args)) - self.assertEqual(p, P(*args)) - for part in p.parts: - self.assertIs(type(part), str) - - def test_str_subclass_common(self): - self._check_str_subclass('') - self._check_str_subclass('.') - self._check_str_subclass('a') - self._check_str_subclass('a/b.txt') - self._check_str_subclass('/a/b.txt') - - def test_join_common(self): - P = self.cls - p = P('a/b') - pp = p.joinpath('c') - self.assertEqual(pp, P('a/b/c')) - self.assertIs(type(pp), type(p)) - pp = p.joinpath('c', 'd') - self.assertEqual(pp, P('a/b/c/d')) - pp = p.joinpath(P('c')) - self.assertEqual(pp, P('a/b/c')) - pp = p.joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div_common(self): - # Basically the same as joinpath(). - P = self.cls - p = P('a/b') - pp = p / 'c' - self.assertEqual(pp, P('a/b/c')) - self.assertIs(type(pp), type(p)) - pp = p / 'c/d' - self.assertEqual(pp, P('a/b/c/d')) - pp = p / 'c' / 'd' - self.assertEqual(pp, P('a/b/c/d')) - pp = 'c' / p / 'd' - self.assertEqual(pp, P('c/a/b/d')) - pp = p / P('c') - self.assertEqual(pp, P('a/b/c')) - pp = p/ '/c' - self.assertEqual(pp, P('/c')) - - def _check_str(self, expected, args): - p = self.cls(*args) - self.assertEqual(str(p), expected.replace('/', self.sep)) - - def test_str_common(self): - # Canonicalized paths roundtrip. - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self._check_str(pathstr, (pathstr,)) - # Special case for the empty path. - self._check_str('.', ('',)) - # Other tests for str() are in test_equivalences(). - - def test_as_posix_common(self): - P = self.cls - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - self.assertEqual(P(pathstr).as_posix(), pathstr) - # Other tests for as_posix() are in test_equivalences(). - - def test_as_bytes_common(self): - sep = os.fsencode(self.sep) - P = self.cls - self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') - - def test_as_uri_common(self): - P = self.cls - with self.assertRaises(ValueError): - P('a').as_uri() - with self.assertRaises(ValueError): - P().as_uri() - - def test_repr_common(self): - for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): - p = self.cls(pathstr) - clsname = p.__class__.__name__ - r = repr(p) - # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) - inner = r[len(clsname) + 1 : -1] - self.assertEqual(eval(inner), p.as_posix()) - # The repr() roundtrips. - q = eval(r, pathlib.__dict__) - self.assertIs(q.__class__, p.__class__) - self.assertEqual(q, p) - self.assertEqual(repr(q), r) - - def test_eq_common(self): - P = self.cls - self.assertEqual(P('a/b'), P('a/b')) - self.assertEqual(P('a/b'), P('a', 'b')) - self.assertNotEqual(P('a/b'), P('a')) - self.assertNotEqual(P('a/b'), P('/a/b')) - self.assertNotEqual(P('a/b'), P()) - self.assertNotEqual(P('/a/b'), P('/')) - self.assertNotEqual(P(), P('/')) - self.assertNotEqual(P(), "") - self.assertNotEqual(P(), {}) - self.assertNotEqual(P(), int) - - def test_match_common(self): - P = self.cls - self.assertRaises(ValueError, P('a').match, '') - self.assertRaises(ValueError, P('a').match, '.') - # Simple relative pattern. - self.assertTrue(P('b.py').match('b.py')) - self.assertTrue(P('a/b.py').match('b.py')) - self.assertTrue(P('/a/b.py').match('b.py')) - self.assertFalse(P('a.py').match('b.py')) - self.assertFalse(P('b/py').match('b.py')) - self.assertFalse(P('/a.py').match('b.py')) - self.assertFalse(P('b.py/c').match('b.py')) - # Wildcard relative pattern. - self.assertTrue(P('b.py').match('*.py')) - self.assertTrue(P('a/b.py').match('*.py')) - self.assertTrue(P('/a/b.py').match('*.py')) - self.assertFalse(P('b.pyc').match('*.py')) - self.assertFalse(P('b./py').match('*.py')) - self.assertFalse(P('b.py/c').match('*.py')) - # Multi-part relative pattern. - self.assertTrue(P('ab/c.py').match('a*/*.py')) - self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) - self.assertFalse(P('a.py').match('a*/*.py')) - self.assertFalse(P('/dab/c.py').match('a*/*.py')) - self.assertFalse(P('ab/c.py/d').match('a*/*.py')) - # Absolute pattern. - self.assertTrue(P('/b.py').match('/*.py')) - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('a/b.py').match('/*.py')) - self.assertFalse(P('/a/b.py').match('/*.py')) - # Multi-part absolute pattern. - self.assertTrue(P('/a/b.py').match('/a/*.py')) - self.assertFalse(P('/ab.py').match('/a/*.py')) - self.assertFalse(P('/a/b/c.py').match('/a/*.py')) - # Multi-part glob-style pattern. - self.assertFalse(P('/a/b/c.py').match('/**/*.py')) - self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) - - def test_ordering_common(self): - # Ordering is tuple-alike. - def assertLess(a, b): - self.assertLess(a, b) - self.assertGreater(b, a) - P = self.cls - a = P('a') - b = P('a/b') - c = P('abc') - d = P('b') - assertLess(a, b) - assertLess(a, c) - assertLess(a, d) - assertLess(b, c) - assertLess(c, d) - P = self.cls - a = P('/a') - b = P('/a/b') - c = P('/abc') - d = P('/b') - assertLess(a, b) - assertLess(a, c) - assertLess(a, d) - assertLess(b, c) - assertLess(c, d) - with self.assertRaises(TypeError): - P() < {} - - def test_parts_common(self): - # `parts` returns a tuple. - sep = self.sep - P = self.cls - p = P('a/b') - parts = p.parts - self.assertEqual(parts, ('a', 'b')) - # The object gets reused. - self.assertIs(parts, p.parts) - # When the path is absolute, the anchor is a separate part. - p = P('/a/b') - parts = p.parts - self.assertEqual(parts, (sep, 'a', 'b')) - - def test_fspath_common(self): - P = self.cls - p = P('a/b') - self._check_str(p.__fspath__(), ('a/b',)) - self._check_str(os.fspath(p), ('a/b',)) - - def test_equivalences(self): - for k, tuples in self.equivalences.items(): - canon = k.replace('/', self.sep) - posix = k.replace(self.sep, '/') - if canon != posix: - tuples = tuples + [ - tuple(part.replace('/', self.sep) for part in t) - for t in tuples - ] - tuples.append((posix, )) - pcanon = self.cls(canon) - for t in tuples: - p = self.cls(*t) - self.assertEqual(p, pcanon, "failed with args {}".format(t)) - self.assertEqual(hash(p), hash(pcanon)) - self.assertEqual(str(p), canon) - self.assertEqual(p.as_posix(), posix) - - def test_parent_common(self): - # Relative - P = self.cls - p = P('a/b/c') - self.assertEqual(p.parent, P('a/b')) - self.assertEqual(p.parent.parent, P('a')) - self.assertEqual(p.parent.parent.parent, P()) - self.assertEqual(p.parent.parent.parent.parent, P()) - # Anchored - p = P('/a/b/c') - self.assertEqual(p.parent, P('/a/b')) - self.assertEqual(p.parent.parent, P('/a')) - self.assertEqual(p.parent.parent.parent, P('/')) - self.assertEqual(p.parent.parent.parent.parent, P('/')) - - def test_parents_common(self): - # Relative - P = self.cls - p = P('a/b/c') - par = p.parents - self.assertEqual(len(par), 3) - self.assertEqual(par[0], P('a/b')) - self.assertEqual(par[1], P('a')) - self.assertEqual(par[2], P('.')) - self.assertEqual(par[-1], P('.')) - self.assertEqual(par[-2], P('a')) - self.assertEqual(par[-3], P('a/b')) - self.assertEqual(par[0:1], (P('a/b'),)) - self.assertEqual(par[:2], (P('a/b'), P('a'))) - self.assertEqual(par[:-1], (P('a/b'), P('a'))) - self.assertEqual(par[1:], (P('a'), P('.'))) - self.assertEqual(par[::2], (P('a/b'), P('.'))) - self.assertEqual(par[::-1], (P('.'), P('a'), P('a/b'))) - self.assertEqual(list(par), [P('a/b'), P('a'), P('.')]) - with self.assertRaises(IndexError): - par[-4] - with self.assertRaises(IndexError): - par[3] - with self.assertRaises(TypeError): - par[0] = p - # Anchored - p = P('/a/b/c') - par = p.parents - self.assertEqual(len(par), 3) - self.assertEqual(par[0], P('/a/b')) - self.assertEqual(par[1], P('/a')) - self.assertEqual(par[2], P('/')) - self.assertEqual(par[-1], P('/')) - self.assertEqual(par[-2], P('/a')) - self.assertEqual(par[-3], P('/a/b')) - self.assertEqual(par[0:1], (P('/a/b'),)) - self.assertEqual(par[:2], (P('/a/b'), P('/a'))) - self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) - self.assertEqual(par[1:], (P('/a'), P('/'))) - self.assertEqual(par[::2], (P('/a/b'), P('/'))) - self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) - self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) - with self.assertRaises(IndexError): - par[-4] - with self.assertRaises(IndexError): - par[3] - - def test_drive_common(self): - P = self.cls - self.assertEqual(P('a/b').drive, '') - self.assertEqual(P('/a/b').drive, '') - self.assertEqual(P('').drive, '') - - def test_root_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').root, '') - self.assertEqual(P('a/b').root, '') - self.assertEqual(P('/').root, sep) - self.assertEqual(P('/a/b').root, sep) - - def test_anchor_common(self): - P = self.cls - sep = self.sep - self.assertEqual(P('').anchor, '') - self.assertEqual(P('a/b').anchor, '') - self.assertEqual(P('/').anchor, sep) - self.assertEqual(P('/a/b').anchor, sep) - - def test_name_common(self): - P = self.cls - self.assertEqual(P('').name, '') - self.assertEqual(P('.').name, '') - self.assertEqual(P('/').name, '') - self.assertEqual(P('a/b').name, 'b') - self.assertEqual(P('/a/b').name, 'b') - self.assertEqual(P('/a/b/.').name, 'b') - self.assertEqual(P('a/b.py').name, 'b.py') - self.assertEqual(P('/a/b.py').name, 'b.py') - - def test_suffix_common(self): - P = self.cls - self.assertEqual(P('').suffix, '') - self.assertEqual(P('.').suffix, '') - self.assertEqual(P('..').suffix, '') - self.assertEqual(P('/').suffix, '') - self.assertEqual(P('a/b').suffix, '') - self.assertEqual(P('/a/b').suffix, '') - self.assertEqual(P('/a/b/.').suffix, '') - self.assertEqual(P('a/b.py').suffix, '.py') - self.assertEqual(P('/a/b.py').suffix, '.py') - self.assertEqual(P('a/.hgrc').suffix, '') - self.assertEqual(P('/a/.hgrc').suffix, '') - self.assertEqual(P('a/.hg.rc').suffix, '.rc') - self.assertEqual(P('/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') - - def test_suffixes_common(self): - P = self.cls - self.assertEqual(P('').suffixes, []) - self.assertEqual(P('.').suffixes, []) - self.assertEqual(P('/').suffixes, []) - self.assertEqual(P('a/b').suffixes, []) - self.assertEqual(P('/a/b').suffixes, []) - self.assertEqual(P('/a/b/.').suffixes, []) - self.assertEqual(P('a/b.py').suffixes, ['.py']) - self.assertEqual(P('/a/b.py').suffixes, ['.py']) - self.assertEqual(P('a/.hgrc').suffixes, []) - self.assertEqual(P('/a/.hgrc').suffixes, []) - self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem_common(self): - P = self.cls - self.assertEqual(P('').stem, '') - self.assertEqual(P('.').stem, '') - self.assertEqual(P('..').stem, '..') - self.assertEqual(P('/').stem, '') - self.assertEqual(P('a/b').stem, 'b') - self.assertEqual(P('a/b.py').stem, 'b') - self.assertEqual(P('a/.hgrc').stem, '.hgrc') - self.assertEqual(P('a/.hg.rc').stem, '.hg') - self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name_common(self): - P = self.cls - self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) - self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) - self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) - self.assertRaises(ValueError, P('').with_name, 'd.xml') - self.assertRaises(ValueError, P('.').with_name, 'd.xml') - self.assertRaises(ValueError, P('/').with_name, 'd.xml') - self.assertRaises(ValueError, P('a/b').with_name, '') - self.assertRaises(ValueError, P('a/b').with_name, '/c') - self.assertRaises(ValueError, P('a/b').with_name, 'c/') - self.assertRaises(ValueError, P('a/b').with_name, 'c/d') - - def test_with_stem_common(self): - P = self.cls - self.assertEqual(P('a/b').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) - self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) - self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) - self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) - self.assertRaises(ValueError, P('').with_stem, 'd') - self.assertRaises(ValueError, P('.').with_stem, 'd') - self.assertRaises(ValueError, P('/').with_stem, 'd') - self.assertRaises(ValueError, P('a/b').with_stem, '') - self.assertRaises(ValueError, P('a/b').with_stem, '/c') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/') - self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') - - def test_with_suffix_common(self): - P = self.cls - self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) - self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) - self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) - # Stripping suffix. - self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) - self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') - self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') - self.assertRaises(ValueError, P('a/b').with_suffix, './.d') - self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') - self.assertRaises(ValueError, P('a/b').with_suffix, - (self.flavour.sep, 'd')) - - def test_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.relative_to) - self.assertRaises(TypeError, p.relative_to, b'a') - self.assertEqual(p.relative_to(P()), P('a/b')) - self.assertEqual(p.relative_to(''), P('a/b')) - self.assertEqual(p.relative_to(P('a')), P('b')) - self.assertEqual(p.relative_to('a'), P('b')) - self.assertEqual(p.relative_to('a/'), P('b')) - self.assertEqual(p.relative_to(P('a/b')), P()) - self.assertEqual(p.relative_to('a/b'), P()) - # With several args. - self.assertEqual(p.relative_to('a', 'b'), P()) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('c')) - self.assertRaises(ValueError, p.relative_to, P('a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('a/c')) - self.assertRaises(ValueError, p.relative_to, P('/a')) - p = P('/a/b') - self.assertEqual(p.relative_to(P('/')), P('a/b')) - self.assertEqual(p.relative_to('/'), P('a/b')) - self.assertEqual(p.relative_to(P('/a')), P('b')) - self.assertEqual(p.relative_to('/a'), P('b')) - self.assertEqual(p.relative_to('/a/'), P('b')) - self.assertEqual(p.relative_to(P('/a/b')), P()) - self.assertEqual(p.relative_to('/a/b'), P()) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) - self.assertRaises(ValueError, p.relative_to, P('/a/c')) - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('a')) - - def test_is_relative_to_common(self): - P = self.cls - p = P('a/b') - self.assertRaises(TypeError, p.is_relative_to) - self.assertRaises(TypeError, p.is_relative_to, b'a') - self.assertTrue(p.is_relative_to(P())) - self.assertTrue(p.is_relative_to('')) - self.assertTrue(p.is_relative_to(P('a'))) - self.assertTrue(p.is_relative_to('a/')) - self.assertTrue(p.is_relative_to(P('a/b'))) - self.assertTrue(p.is_relative_to('a/b')) - # With several args. - self.assertTrue(p.is_relative_to('a', 'b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('c'))) - self.assertFalse(p.is_relative_to(P('a/b/c'))) - self.assertFalse(p.is_relative_to(P('a/c'))) - self.assertFalse(p.is_relative_to(P('/a'))) - p = P('/a/b') - self.assertTrue(p.is_relative_to(P('/'))) - self.assertTrue(p.is_relative_to('/')) - self.assertTrue(p.is_relative_to(P('/a'))) - self.assertTrue(p.is_relative_to('/a')) - self.assertTrue(p.is_relative_to('/a/')) - self.assertTrue(p.is_relative_to(P('/a/b'))) - self.assertTrue(p.is_relative_to('/a/b')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/c'))) - self.assertFalse(p.is_relative_to(P('/a/b/c'))) - self.assertFalse(p.is_relative_to(P('/a/c'))) - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('a'))) - - def test_pickling_common(self): - P = self.cls - p = P('/a/b') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertIs(pp.__class__, p.__class__) - self.assertEqual(pp, p) - self.assertEqual(hash(pp), hash(p)) - self.assertEqual(str(pp), str(p)) - - -class PurePosixPathTest(_BasePurePathTest, unittest.TestCase): - cls = pathlib.PurePosixPath - - def test_root(self): - P = self.cls - self.assertEqual(P('/a/b').root, '/') - self.assertEqual(P('///a/b').root, '/') - # POSIX special case for two leading slashes. - self.assertEqual(P('//a/b').root, '//') - - def test_eq(self): - P = self.cls - self.assertNotEqual(P('a/b'), P('A/b')) - self.assertEqual(P('/a'), P('///a')) - self.assertNotEqual(P('/a'), P('//a')) - - def test_as_uri(self): - P = self.cls - self.assertEqual(P('/').as_uri(), 'file:///') - self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c') - self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c') - - def test_as_uri_non_ascii(self): - from urllib.parse import quote_from_bytes - P = self.cls - try: - os.fsencode('\xe9') - except UnicodeEncodeError: - self.skipTest("\\xe9 cannot be encoded to the filesystem encoding") - self.assertEqual(P('/a/b\xe9').as_uri(), - 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) - - def test_match(self): - P = self.cls - self.assertFalse(P('A.py').match('a.PY')) - - def test_is_absolute(self): - P = self.cls - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertTrue(P('/').is_absolute()) - self.assertTrue(P('/a').is_absolute()) - self.assertTrue(P('/a/b/').is_absolute()) - self.assertTrue(P('//a').is_absolute()) - self.assertTrue(P('//a/b').is_absolute()) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved()) - - def test_join(self): - P = self.cls - p = P('//a') - pp = p.joinpath('b') - self.assertEqual(pp, P('//a/b')) - pp = P('/a').joinpath('//c') - self.assertEqual(pp, P('//c')) - pp = P('//a').joinpath('/c') - self.assertEqual(pp, P('/c')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('//a') - pp = p / 'b' - self.assertEqual(pp, P('//a/b')) - pp = P('/a') / '//c' - self.assertEqual(pp, P('//c')) - pp = P('//a') / '/c' - self.assertEqual(pp, P('/c')) - - -class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): - cls = pathlib.PureWindowsPath - - equivalences = _BasePurePathTest.equivalences.copy() - equivalences.update({ - 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('/', 'c:', 'a') ], - 'c:/a': [ - ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), - ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), - ], - '//a/b/': [ ('//a/b',) ], - '//a/b/c': [ - ('//a/b', 'c'), ('//a/b/', 'c'), - ], - }) - - def test_str(self): - p = self.cls('a/b/c') - self.assertEqual(str(p), 'a\\b\\c') - p = self.cls('c:/a/b/c') - self.assertEqual(str(p), 'c:\\a\\b\\c') - p = self.cls('//a/b') - self.assertEqual(str(p), '\\\\a\\b\\') - p = self.cls('//a/b/c') - self.assertEqual(str(p), '\\\\a\\b\\c') - p = self.cls('//a/b/c/d') - self.assertEqual(str(p), '\\\\a\\b\\c\\d') - - def test_str_subclass(self): - self._check_str_subclass('c:') - self._check_str_subclass('c:a') - self._check_str_subclass('c:a\\b.txt') - self._check_str_subclass('c:\\') - self._check_str_subclass('c:\\a') - self._check_str_subclass('c:\\a\\b.txt') - self._check_str_subclass('\\\\some\\share') - self._check_str_subclass('\\\\some\\share\\a') - self._check_str_subclass('\\\\some\\share\\a\\b.txt') - - def test_eq(self): - P = self.cls - self.assertEqual(P('c:a/b'), P('c:a/b')) - self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) - self.assertNotEqual(P('c:a/b'), P('d:a/b')) - self.assertNotEqual(P('c:a/b'), P('c:/a/b')) - self.assertNotEqual(P('/a/b'), P('c:/a/b')) - # Case-insensitivity. - self.assertEqual(P('a/B'), P('A/b')) - self.assertEqual(P('C:a/B'), P('c:A/b')) - self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) - - def test_as_uri(self): - P = self.cls - with self.assertRaises(ValueError): - P('/a/b').as_uri() - with self.assertRaises(ValueError): - P('c:a/b').as_uri() - self.assertEqual(P('c:/').as_uri(), 'file:///c:/') - self.assertEqual(P('c:/a/b.c').as_uri(), 'file:///c:/a/b.c') - self.assertEqual(P('c:/a/b%#c').as_uri(), 'file:///c:/a/b%25%23c') - self.assertEqual(P('c:/a/b\xe9').as_uri(), 'file:///c:/a/b%C3%A9') - self.assertEqual(P('//some/share/').as_uri(), 'file://some/share/') - self.assertEqual(P('//some/share/a/b.c').as_uri(), - 'file://some/share/a/b.c') - self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(), - 'file://some/share/a/b%25%23c%C3%A9') - - def test_match_common(self): - P = self.cls - # Absolute patterns. - self.assertTrue(P('c:/b.py').match('/*.py')) - self.assertTrue(P('c:/b.py').match('c:*.py')) - self.assertTrue(P('c:/b.py').match('c:/*.py')) - self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive - self.assertFalse(P('b.py').match('/*.py')) - self.assertFalse(P('b.py').match('c:*.py')) - self.assertFalse(P('b.py').match('c:/*.py')) - self.assertFalse(P('c:b.py').match('/*.py')) - self.assertFalse(P('c:b.py').match('c:/*.py')) - self.assertFalse(P('/b.py').match('c:*.py')) - self.assertFalse(P('/b.py').match('c:/*.py')) - # UNC patterns. - self.assertTrue(P('//some/share/a.py').match('/*.py')) - self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) - self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) - # Case-insensitivity. - self.assertTrue(P('B.py').match('b.PY')) - self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) - self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) - - def test_ordering_common(self): - # Case-insensitivity. - def assertOrderedEqual(a, b): - self.assertLessEqual(a, b) - self.assertGreaterEqual(b, a) - P = self.cls - p = P('c:A/b') - q = P('C:a/B') - assertOrderedEqual(p, q) - self.assertFalse(p < q) - self.assertFalse(p > q) - p = P('//some/Share/A/b') - q = P('//Some/SHARE/a/B') - assertOrderedEqual(p, q) - self.assertFalse(p < q) - self.assertFalse(p > q) - - def test_parts(self): - P = self.cls - p = P('c:a/b') - parts = p.parts - self.assertEqual(parts, ('c:', 'a', 'b')) - p = P('c:/a/b') - parts = p.parts - self.assertEqual(parts, ('c:\\', 'a', 'b')) - p = P('//a/b/c/d') - parts = p.parts - self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) - - def test_parent(self): - # Anchored - P = self.cls - p = P('z:a/b/c') - self.assertEqual(p.parent, P('z:a/b')) - self.assertEqual(p.parent.parent, P('z:a')) - self.assertEqual(p.parent.parent.parent, P('z:')) - self.assertEqual(p.parent.parent.parent.parent, P('z:')) - p = P('z:/a/b/c') - self.assertEqual(p.parent, P('z:/a/b')) - self.assertEqual(p.parent.parent, P('z:/a')) - self.assertEqual(p.parent.parent.parent, P('z:/')) - self.assertEqual(p.parent.parent.parent.parent, P('z:/')) - p = P('//a/b/c/d') - self.assertEqual(p.parent, P('//a/b/c')) - self.assertEqual(p.parent.parent, P('//a/b')) - self.assertEqual(p.parent.parent.parent, P('//a/b')) - - def test_parents(self): - # Anchored - P = self.cls - p = P('z:a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:a')) - self.assertEqual(par[1], P('z:')) - self.assertEqual(par[0:1], (P('z:a'),)) - self.assertEqual(par[:-1], (P('z:a'),)) - self.assertEqual(par[:2], (P('z:a'), P('z:'))) - self.assertEqual(par[1:], (P('z:'),)) - self.assertEqual(par[::2], (P('z:a'),)) - self.assertEqual(par[::-1], (P('z:'), P('z:a'))) - self.assertEqual(list(par), [P('z:a'), P('z:')]) - with self.assertRaises(IndexError): - par[2] - p = P('z:/a/b/') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('z:/a')) - self.assertEqual(par[1], P('z:/')) - self.assertEqual(par[0:1], (P('z:/a'),)) - self.assertEqual(par[0:-1], (P('z:/a'),)) - self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) - self.assertEqual(par[1:], (P('z:/'),)) - self.assertEqual(par[::2], (P('z:/a'),)) - self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) - self.assertEqual(list(par), [P('z:/a'), P('z:/')]) - with self.assertRaises(IndexError): - par[2] - p = P('//a/b/c/d') - par = p.parents - self.assertEqual(len(par), 2) - self.assertEqual(par[0], P('//a/b/c')) - self.assertEqual(par[1], P('//a/b')) - self.assertEqual(par[0:1], (P('//a/b/c'),)) - self.assertEqual(par[0:-1], (P('//a/b/c'),)) - self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b'))) - self.assertEqual(par[1:], (P('//a/b'),)) - self.assertEqual(par[::2], (P('//a/b/c'),)) - self.assertEqual(par[::-1], (P('//a/b'), P('//a/b/c'))) - self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')]) - with self.assertRaises(IndexError): - par[2] - - def test_drive(self): - P = self.cls - self.assertEqual(P('c:').drive, 'c:') - self.assertEqual(P('c:a/b').drive, 'c:') - self.assertEqual(P('c:/').drive, 'c:') - self.assertEqual(P('c:/a/b/').drive, 'c:') - self.assertEqual(P('//a/b').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/').drive, '\\\\a\\b') - self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') - - def test_root(self): - P = self.cls - self.assertEqual(P('c:').root, '') - self.assertEqual(P('c:a/b').root, '') - self.assertEqual(P('c:/').root, '\\') - self.assertEqual(P('c:/a/b/').root, '\\') - self.assertEqual(P('//a/b').root, '\\') - self.assertEqual(P('//a/b/').root, '\\') - self.assertEqual(P('//a/b/c/d').root, '\\') - - def test_anchor(self): - P = self.cls - self.assertEqual(P('c:').anchor, 'c:') - self.assertEqual(P('c:a/b').anchor, 'c:') - self.assertEqual(P('c:/').anchor, 'c:\\') - self.assertEqual(P('c:/a/b/').anchor, 'c:\\') - self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\') - self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\') - - def test_name(self): - P = self.cls - self.assertEqual(P('c:').name, '') - self.assertEqual(P('c:/').name, '') - self.assertEqual(P('c:a/b').name, 'b') - self.assertEqual(P('c:/a/b').name, 'b') - self.assertEqual(P('c:a/b.py').name, 'b.py') - self.assertEqual(P('c:/a/b.py').name, 'b.py') - self.assertEqual(P('//My.py/Share.php').name, '') - self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') - - def test_suffix(self): - P = self.cls - self.assertEqual(P('c:').suffix, '') - self.assertEqual(P('c:/').suffix, '') - self.assertEqual(P('c:a/b').suffix, '') - self.assertEqual(P('c:/a/b').suffix, '') - self.assertEqual(P('c:a/b.py').suffix, '.py') - self.assertEqual(P('c:/a/b.py').suffix, '.py') - self.assertEqual(P('c:a/.hgrc').suffix, '') - self.assertEqual(P('c:/a/.hgrc').suffix, '') - self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') - self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('//My.py/Share.php').suffix, '') - self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') - - def test_suffixes(self): - P = self.cls - self.assertEqual(P('c:').suffixes, []) - self.assertEqual(P('c:/').suffixes, []) - self.assertEqual(P('c:a/b').suffixes, []) - self.assertEqual(P('c:/a/b').suffixes, []) - self.assertEqual(P('c:a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) - self.assertEqual(P('c:a/.hgrc').suffixes, []) - self.assertEqual(P('c:/a/.hgrc').suffixes, []) - self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) - self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('//My.py/Share.php').suffixes, []) - self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) - - def test_stem(self): - P = self.cls - self.assertEqual(P('c:').stem, '') - self.assertEqual(P('c:.').stem, '') - self.assertEqual(P('c:..').stem, '..') - self.assertEqual(P('c:/').stem, '') - self.assertEqual(P('c:a/b').stem, 'b') - self.assertEqual(P('c:a/b.py').stem, 'b') - self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') - self.assertEqual(P('c:a/.hg.rc').stem, '.hg') - self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') - - def test_with_name(self): - P = self.cls - self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml')) - self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml')) - self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml')) - self.assertRaises(ValueError, P('c:').with_name, 'd.xml') - self.assertRaises(ValueError, P('c:/').with_name, 'd.xml') - self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml') - self.assertRaises(ValueError, P('c:a/b').with_name, 'd:') - self.assertRaises(ValueError, P('c:a/b').with_name, 'd:e') - self.assertRaises(ValueError, P('c:a/b').with_name, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_name, '//My/Share') - - def test_with_stem(self): - P = self.cls - self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) - self.assertRaises(ValueError, P('c:').with_stem, 'd') - self.assertRaises(ValueError, P('c:/').with_stem, 'd') - self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') - self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:') - self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:e') - self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') - self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') - - def test_with_suffix(self): - P = self.cls - self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) - self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) - self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) - # Path doesn't have a "filename" component. - self.assertRaises(ValueError, P('').with_suffix, '.gz') - self.assertRaises(ValueError, P('.').with_suffix, '.gz') - self.assertRaises(ValueError, P('/').with_suffix, '.gz') - self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') - # Invalid suffix. - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') - self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') - - def test_relative_to(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) - self.assertEqual(p.relative_to('c:foO'), P('Bar')) - self.assertEqual(p.relative_to('c:foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:foO/baR')), P()) - self.assertEqual(p.relative_to('c:foO/baR'), P()) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P()) - self.assertRaises(ValueError, p.relative_to, '') - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('Foo')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) - p = P('C:/Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('/Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('/Foo/Bar')) - self.assertEqual(str(p.relative_to(P('c:'))), '\\Foo\\Bar') - self.assertEqual(str(p.relative_to('c:')), '\\Foo\\Bar') - self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) - self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) - self.assertEqual(p.relative_to('c:/foO'), P('Bar')) - self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) - self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) - self.assertEqual(p.relative_to('c:/foO/baR'), P()) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) - self.assertRaises(ValueError, p.relative_to, P('C:Foo')) - self.assertRaises(ValueError, p.relative_to, P('d:')) - self.assertRaises(ValueError, p.relative_to, P('d:/')) - self.assertRaises(ValueError, p.relative_to, P('/')) - self.assertRaises(ValueError, p.relative_to, P('/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) - self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) - self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) - # Unrelated paths. - self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) - self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) - - def test_is_relative_to(self): - P = self.cls - p = P('C:Foo/Bar') - self.assertTrue(p.is_relative_to(P('c:'))) - self.assertTrue(p.is_relative_to('c:')) - self.assertTrue(p.is_relative_to(P('c:foO'))) - self.assertTrue(p.is_relative_to('c:foO')) - self.assertTrue(p.is_relative_to('c:foO/')) - self.assertTrue(p.is_relative_to(P('c:foO/baR'))) - self.assertTrue(p.is_relative_to('c:foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P())) - self.assertFalse(p.is_relative_to('')) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('Foo'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('C:/Foo'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) - p = P('C:/Foo/Bar') - self.assertTrue(p.is_relative_to('c:')) - self.assertTrue(p.is_relative_to(P('c:/'))) - self.assertTrue(p.is_relative_to(P('c:/foO'))) - self.assertTrue(p.is_relative_to('c:/foO/')) - self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) - self.assertTrue(p.is_relative_to('c:/foO/baR')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('C:/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) - self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) - self.assertFalse(p.is_relative_to(P('C:Foo'))) - self.assertFalse(p.is_relative_to(P('d:'))) - self.assertFalse(p.is_relative_to(P('d:/'))) - self.assertFalse(p.is_relative_to(P('/'))) - self.assertFalse(p.is_relative_to(P('/Foo'))) - self.assertFalse(p.is_relative_to(P('//C/Foo'))) - # UNC paths. - p = P('//Server/Share/Foo/Bar') - self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) - self.assertTrue(p.is_relative_to('//sErver/sHare')) - self.assertTrue(p.is_relative_to('//sErver/sHare/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) - self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) - self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) - # Unrelated paths. - self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) - self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) - - def test_is_absolute(self): - P = self.cls - # Under NT, only paths with both a drive and a root are absolute. - self.assertFalse(P().is_absolute()) - self.assertFalse(P('a').is_absolute()) - self.assertFalse(P('a/b/').is_absolute()) - self.assertFalse(P('/').is_absolute()) - self.assertFalse(P('/a').is_absolute()) - self.assertFalse(P('/a/b/').is_absolute()) - self.assertFalse(P('c:').is_absolute()) - self.assertFalse(P('c:a').is_absolute()) - self.assertFalse(P('c:a/b/').is_absolute()) - self.assertTrue(P('c:/').is_absolute()) - self.assertTrue(P('c:/a').is_absolute()) - self.assertTrue(P('c:/a/b/').is_absolute()) - # UNC paths are absolute by definition. - self.assertTrue(P('//a/b').is_absolute()) - self.assertTrue(P('//a/b/').is_absolute()) - self.assertTrue(P('//a/b/c').is_absolute()) - self.assertTrue(P('//a/b/c/d').is_absolute()) - - def test_join(self): - P = self.cls - p = P('C:/a/b') - pp = p.joinpath('x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('/x/y') - self.assertEqual(pp, P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - pp = p.joinpath('D:x/y') - self.assertEqual(pp, P('D:x/y')) - pp = p.joinpath('D:/x/y') - self.assertEqual(pp, P('D:/x/y')) - pp = p.joinpath('//host/share/x/y') - self.assertEqual(pp, P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - pp = p.joinpath('c:x/y') - self.assertEqual(pp, P('C:/a/b/x/y')) - pp = p.joinpath('c:/x/y') - self.assertEqual(pp, P('C:/x/y')) - - def test_div(self): - # Basically the same as joinpath(). - P = self.cls - p = P('C:/a/b') - self.assertEqual(p / 'x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y')) - self.assertEqual(p / '/x/y', P('C:/x/y')) - self.assertEqual(p / '/x' / 'y', P('C:/x/y')) - # Joining with a different drive => the first path is ignored, even - # if the second path is relative. - self.assertEqual(p / 'D:x/y', P('D:x/y')) - self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) - self.assertEqual(p / 'D:/x/y', P('D:/x/y')) - self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) - self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) - self.assertEqual(p / 'c:/x/y', P('C:/x/y')) - - def test_is_reserved(self): - P = self.cls - self.assertIs(False, P('').is_reserved()) - self.assertIs(False, P('/').is_reserved()) - self.assertIs(False, P('/foo/bar').is_reserved()) - # UNC paths are never reserved. - self.assertIs(False, P('//my/share/nul/con/aux').is_reserved()) - # Case-insensitive DOS-device names are reserved. - self.assertIs(True, P('nul').is_reserved()) - self.assertIs(True, P('aux').is_reserved()) - self.assertIs(True, P('prn').is_reserved()) - self.assertIs(True, P('con').is_reserved()) - self.assertIs(True, P('conin$').is_reserved()) - self.assertIs(True, P('conout$').is_reserved()) - # COM/LPT + 1-9 or + superscript 1-3 are reserved. - self.assertIs(True, P('COM1').is_reserved()) - self.assertIs(True, P('LPT9').is_reserved()) - self.assertIs(True, P('com\xb9').is_reserved()) - self.assertIs(True, P('com\xb2').is_reserved()) - self.assertIs(True, P('lpt\xb3').is_reserved()) - # DOS-device name mataching ignores characters after a dot or - # a colon and also ignores trailing spaces. - self.assertIs(True, P('NUL.txt').is_reserved()) - self.assertIs(True, P('PRN ').is_reserved()) - self.assertIs(True, P('AUX .txt').is_reserved()) - self.assertIs(True, P('COM1:bar').is_reserved()) - self.assertIs(True, P('LPT9 :bar').is_reserved()) - # DOS-device names are only matched at the beginning - # of a path component. - self.assertIs(False, P('bar.com9').is_reserved()) - self.assertIs(False, P('bar.lpt9').is_reserved()) - # Only the last path component matters. - self.assertIs(True, P('c:/baz/con/NUL').is_reserved()) - self.assertIs(False, P('c:/NUL/con/baz').is_reserved()) - -class PurePathTest(_BasePurePathTest, unittest.TestCase): - cls = pathlib.PurePath - - def test_concrete_class(self): - p = self.cls('a') - self.assertIs(type(p), - pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath) - - def test_different_flavours_unequal(self): - p = pathlib.PurePosixPath('a') - q = pathlib.PureWindowsPath('a') - self.assertNotEqual(p, q) - - def test_different_flavours_unordered(self): - p = pathlib.PurePosixPath('a') - q = pathlib.PureWindowsPath('a') - with self.assertRaises(TypeError): - p < q - with self.assertRaises(TypeError): - p <= q - with self.assertRaises(TypeError): - p > q - with self.assertRaises(TypeError): - p >= q - - -# -# Tests for the concrete classes. -# - -# Make sure any symbolic links in the base test path are resolved. -BASE = os.path.realpath(TESTFN) -join = lambda *x: os.path.join(BASE, *x) -rel_join = lambda *x: os.path.join(TESTFN, *x) - -only_nt = unittest.skipIf(os.name != 'nt', - 'test requires a Windows-compatible system') -only_posix = unittest.skipIf(os.name == 'nt', - 'test requires a POSIX-compatible system') - -@only_posix -class PosixPathAsPureTest(PurePosixPathTest): - cls = pathlib.PosixPath - -@only_nt -class WindowsPathAsPureTest(PureWindowsPathTest): - cls = pathlib.WindowsPath - - def test_owner(self): - P = self.cls - with self.assertRaises(NotImplementedError): - P('c:/').owner() - - def test_group(self): - P = self.cls - with self.assertRaises(NotImplementedError): - P('c:/').group() - - -class _BasePathTest(object): - """Tests for the FS-accessing functionalities of the Path classes.""" - - # (BASE) - # | - # |-- brokenLink -> non-existing - # |-- dirA - # | `-- linkC -> ../dirB - # |-- dirB - # | |-- fileB - # | `-- linkD -> ../dirB - # |-- dirC - # | |-- dirD - # | | `-- fileD - # | `-- fileC - # |-- dirE # No permissions - # |-- fileA - # |-- linkA -> fileA - # |-- linkB -> dirB - # `-- brokenLinkLoop -> brokenLinkLoop - # - - def setUp(self): - def cleanup(): - os.chmod(join('dirE'), 0o777) - os_helper.rmtree(BASE) - self.addCleanup(cleanup) - os.mkdir(BASE) - os.mkdir(join('dirA')) - os.mkdir(join('dirB')) - os.mkdir(join('dirC')) - os.mkdir(join('dirC', 'dirD')) - os.mkdir(join('dirE')) - with open(join('fileA'), 'wb') as f: - f.write(b"this is file A\n") - with open(join('dirB', 'fileB'), 'wb') as f: - f.write(b"this is file B\n") - with open(join('dirC', 'fileC'), 'wb') as f: - f.write(b"this is file C\n") - with open(join('dirC', 'dirD', 'fileD'), 'wb') as f: - f.write(b"this is file D\n") - os.chmod(join('dirE'), 0) - if os_helper.can_symlink(): - # Relative symlinks. - os.symlink('fileA', join('linkA')) - os.symlink('non-existing', join('brokenLink')) - self.dirlink('dirB', join('linkB')) - self.dirlink(os.path.join('..', 'dirB'), join('dirA', 'linkC')) - # This one goes upwards, creating a loop. - self.dirlink(os.path.join('..', 'dirB'), join('dirB', 'linkD')) - # Broken symlink (pointing to itself). - os.symlink('brokenLinkLoop', join('brokenLinkLoop')) - - if os.name == 'nt': - # Workaround for http://bugs.python.org/issue13772. - def dirlink(self, src, dest): - os.symlink(src, dest, target_is_directory=True) - else: - def dirlink(self, src, dest): - os.symlink(src, dest) - - def assertSame(self, path_a, path_b): - self.assertTrue(os.path.samefile(str(path_a), str(path_b)), - "%r and %r don't point to the same file" % - (path_a, path_b)) - - def assertFileNotFound(self, func, *args, **kwargs): - with self.assertRaises(FileNotFoundError) as cm: - func(*args, **kwargs) - self.assertEqual(cm.exception.errno, errno.ENOENT) - - def assertEqualNormCase(self, path_a, path_b): - self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) - - def _test_cwd(self, p): - q = self.cls(os.getcwd()) - self.assertEqual(p, q) - self.assertEqualNormCase(str(p), str(q)) - self.assertIs(type(p), type(q)) - self.assertTrue(p.is_absolute()) - - def test_cwd(self): - p = self.cls.cwd() - self._test_cwd(p) - - def _test_home(self, p): - q = self.cls(os.path.expanduser('~')) - self.assertEqual(p, q) - self.assertEqualNormCase(str(p), str(q)) - self.assertIs(type(p), type(q)) - self.assertTrue(p.is_absolute()) - - def test_home(self): - with os_helper.EnvironmentVarGuard() as env: - self._test_home(self.cls.home()) - - env.clear() - env['USERPROFILE'] = os.path.join(BASE, 'userprofile') - self._test_home(self.cls.home()) - - # bpo-38883: ignore `HOME` when set on windows - env['HOME'] = os.path.join(BASE, 'home') - self._test_home(self.cls.home()) - - def test_samefile(self): - fileA_path = os.path.join(BASE, 'fileA') - fileB_path = os.path.join(BASE, 'dirB', 'fileB') - p = self.cls(fileA_path) - pp = self.cls(fileA_path) - q = self.cls(fileB_path) - self.assertTrue(p.samefile(fileA_path)) - self.assertTrue(p.samefile(pp)) - self.assertFalse(p.samefile(fileB_path)) - self.assertFalse(p.samefile(q)) - # Test the non-existent file case - non_existent = os.path.join(BASE, 'foo') - r = self.cls(non_existent) - self.assertRaises(FileNotFoundError, p.samefile, r) - self.assertRaises(FileNotFoundError, p.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, p) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, r) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - - def test_empty_path(self): - # The empty path points to '.' - p = self.cls('') - self.assertEqual(p.stat(), os.stat('.')) - - def test_expanduser_common(self): - P = self.cls - p = P('~') - self.assertEqual(p.expanduser(), P(os.path.expanduser('~'))) - p = P('foo') - self.assertEqual(p.expanduser(), p) - p = P('/~') - self.assertEqual(p.expanduser(), p) - p = P('../~') - self.assertEqual(p.expanduser(), p) - p = P(P('').absolute().anchor) / '~' - self.assertEqual(p.expanduser(), p) - - def test_exists(self): - P = self.cls - p = P(BASE) - self.assertIs(True, p.exists()) - self.assertIs(True, (p / 'dirA').exists()) - self.assertIs(True, (p / 'fileA').exists()) - self.assertIs(False, (p / 'fileA' / 'bah').exists()) - if os_helper.can_symlink(): - self.assertIs(True, (p / 'linkA').exists()) - self.assertIs(True, (p / 'linkB').exists()) - self.assertIs(True, (p / 'linkB' / 'fileB').exists()) - self.assertIs(False, (p / 'linkA' / 'bah').exists()) - self.assertIs(False, (p / 'foo').exists()) - self.assertIs(False, P('/xyzzy').exists()) - self.assertIs(False, P(BASE + '\udfff').exists()) - self.assertIs(False, P(BASE + '\x00').exists()) - - def test_open_common(self): - p = self.cls(BASE) - with (p / 'fileA').open('r') as f: - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.read(), "this is file A\n") - with (p / 'fileA').open('rb') as f: - self.assertIsInstance(f, io.BufferedIOBase) - self.assertEqual(f.read().strip(), b"this is file A") - with (p / 'fileA').open('rb', buffering=0) as f: - self.assertIsInstance(f, io.RawIOBase) - self.assertEqual(f.read().strip(), b"this is file A") - - def test_read_write_bytes(self): - p = self.cls(BASE) - (p / 'fileA').write_bytes(b'abcdefg') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - # Check that trying to write str does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_bytes, 'somestr') - self.assertEqual((p / 'fileA').read_bytes(), b'abcdefg') - - def test_read_write_text(self): - p = self.cls(BASE) - (p / 'fileA').write_text('äbcdefg', encoding='latin-1') - self.assertEqual((p / 'fileA').read_text( - encoding='utf-8', errors='ignore'), 'bcdefg') - # Check that trying to write bytes does not truncate the file. - self.assertRaises(TypeError, (p / 'fileA').write_text, b'somebytes') - self.assertEqual((p / 'fileA').read_text(encoding='latin-1'), 'äbcdefg') - - def test_write_text_with_newlines(self): - p = self.cls(BASE) - # Check that `\n` character change nothing - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\nfghlk\n\rmnopq') - # Check that `\r` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\rfghlk\r\rmnopq') - # Check that `\r\n` character replaces `\n` - (p / 'fileA').write_text('abcde\r\nfghlk\n\rmnopq', newline='\r\n') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde\r\r\nfghlk\r\n\rmnopq') - # Check that no argument passed will change `\n` to `os.linesep` - os_linesep_byte = bytes(os.linesep, encoding='ascii') - (p / 'fileA').write_text('abcde\nfghlk\n\rmnopq') - self.assertEqual((p / 'fileA').read_bytes(), - b'abcde' + os_linesep_byte + b'fghlk' + os_linesep_byte + b'\rmnopq') - - def test_iterdir(self): - P = self.cls - p = P(BASE) - it = p.iterdir() - paths = set(it) - expected = ['dirA', 'dirB', 'dirC', 'dirE', 'fileA'] - if os_helper.can_symlink(): - expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] - self.assertEqual(paths, { P(BASE, q) for q in expected }) - - @os_helper.skip_unless_symlink - def test_iterdir_symlink(self): - # __iter__ on a symlink to a directory. - P = self.cls - p = P(BASE, 'linkB') - paths = set(p.iterdir()) - expected = { P(BASE, 'linkB', q) for q in ['fileB', 'linkD'] } - self.assertEqual(paths, expected) - - def test_iterdir_nodir(self): - # __iter__ on something that is not a directory. - p = self.cls(BASE, 'fileA') - with self.assertRaises(OSError) as cm: - next(p.iterdir()) - # ENOENT or EINVAL under Windows, ENOTDIR otherwise - # (see issue #12802). - self.assertIn(cm.exception.errno, (errno.ENOTDIR, - errno.ENOENT, errno.EINVAL)) - - def test_glob_common(self): - def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - it = p.glob("fileA") - self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.glob("fileB"), []) - _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"]) - if not os_helper.can_symlink(): - _check(p.glob("*A"), ['dirA', 'fileA']) - else: - _check(p.glob("*A"), ['dirA', 'fileA', 'linkA']) - if not os_helper.can_symlink(): - _check(p.glob("*B/*"), ['dirB/fileB']) - else: - _check(p.glob("*B/*"), ['dirB/fileB', 'dirB/linkD', - 'linkB/fileB', 'linkB/linkD']) - if not os_helper.can_symlink(): - _check(p.glob("*/fileB"), ['dirB/fileB']) - else: - _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB']) - - def test_rglob_common(self): - def _check(glob, expected): - self.assertEqual(set(glob), { P(BASE, q) for q in expected }) - P = self.cls - p = P(BASE) - it = p.rglob("fileA") - self.assertIsInstance(it, collections.abc.Iterator) - _check(it, ["fileA"]) - _check(p.rglob("fileB"), ["dirB/fileB"]) - _check(p.rglob("*/fileA"), []) - if not os_helper.can_symlink(): - _check(p.rglob("*/fileB"), ["dirB/fileB"]) - else: - _check(p.rglob("*/fileB"), ["dirB/fileB", "dirB/linkD/fileB", - "linkB/fileB", "dirA/linkC/fileB"]) - _check(p.rglob("file*"), ["fileA", "dirB/fileB", - "dirC/fileC", "dirC/dirD/fileD"]) - p = P(BASE, "dirC") - _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) - - @os_helper.skip_unless_symlink - def test_rglob_symlink_loop(self): - # Don't get fooled by symlink loops (Issue #26012). - P = self.cls - p = P(BASE) - given = set(p.rglob('*')) - expect = {'brokenLink', - 'dirA', 'dirA/linkC', - 'dirB', 'dirB/fileB', 'dirB/linkD', - 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', 'dirC/fileC', - 'dirE', - 'fileA', - 'linkA', - 'linkB', - 'brokenLinkLoop', - } - self.assertEqual(given, {p / x for x in expect}) - - def test_glob_many_open_files(self): - depth = 30 - P = self.cls - base = P(BASE) / 'deep' - p = P(base, *(['d']*depth)) - p.mkdir(parents=True) - pattern = '/'.join(['*'] * depth) - iters = [base.glob(pattern) for j in range(100)] - for it in iters: - self.assertEqual(next(it), p) - iters = [base.rglob('d') for j in range(100)] - p = base - for i in range(depth): - p = p / 'd' - for it in iters: - self.assertEqual(next(it), p) - - def test_glob_dotdot(self): - # ".." is not special in globs. - P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) - self.assertEqual(set(p.glob("../xyzzy")), set()) - - @os_helper.skip_unless_symlink - def test_glob_permissions(self): - # See bpo-38894 - P = self.cls - base = P(BASE) / 'permissions' - base.mkdir() - - file1 = base / "file1" - file1.touch() - file2 = base / "file2" - file2.touch() - - subdir = base / "subdir" - - file3 = base / "file3" - file3.symlink_to(subdir / "other") - - # Patching is needed to avoid relying on the filesystem - # to return the order of the files as the error will not - # happen if the symlink is the last item. - - with mock.patch("os.scandir") as scandir: - scandir.return_value = sorted(os.scandir(base)) - self.assertEqual(len(set(base.glob("*"))), 3) - - subdir.mkdir() - - with mock.patch("os.scandir") as scandir: - scandir.return_value = sorted(os.scandir(base)) - self.assertEqual(len(set(base.glob("*"))), 4) - - subdir.chmod(000) - - with mock.patch("os.scandir") as scandir: - scandir.return_value = sorted(os.scandir(base)) - self.assertEqual(len(set(base.glob("*"))), 4) - - def _check_resolve(self, p, expected, strict=True): - q = p.resolve(strict) - self.assertEqual(q, expected) - - # This can be used to check both relative and absolute resolutions. - _check_resolve_relative = _check_resolve_absolute = _check_resolve - - @os_helper.skip_unless_symlink - def test_resolve_common(self): - P = self.cls - p = P(BASE, 'foo') - with self.assertRaises(OSError) as cm: - p.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ENOENT) - # Non-strict - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) - p = P(BASE, 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo', 'in', 'spam')) - p = P(BASE, '..', 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo', 'in', 'spam'))) - # These are all relative symlinks. - p = P(BASE, 'dirB', 'fileB') - self._check_resolve_relative(p, p) - p = P(BASE, 'linkA') - self._check_resolve_relative(p, P(BASE, 'fileA')) - p = P(BASE, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - p = P(BASE, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB')) - # Non-strict - p = P(BASE, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB', 'foo', 'in', - 'spam'), False) - p = P(BASE, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if os.name == 'nt': - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(BASE, 'dirA', 'foo', 'in', - 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) - # Now create absolute symlinks. - d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', - dir=os.getcwd())) - self.addCleanup(os_helper.rmtree, d) - os.symlink(os.path.join(d), join('dirA', 'linkX')) - os.symlink(join('dirB'), os.path.join(d, 'linkY')) - p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) - # Non-strict - p = P(BASE, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(BASE, 'dirB', 'foo', 'in', 'spam'), - False) - p = P(BASE, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if os.name == 'nt': - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(BASE, 'foo', 'in', 'spam'), False) - - @os_helper.skip_unless_symlink - def test_resolve_dot(self): - # See https://bitbucket.org/pitrou/pathlib/issue/9/pathresolve-fails-on-complex-symlinks - p = self.cls(BASE) - self.dirlink('.', join('0')) - self.dirlink(os.path.join('0', '0'), join('1')) - self.dirlink(os.path.join('1', '1'), join('2')) - q = p / '2' - self.assertEqual(q.resolve(strict=True), p) - r = q / '3' / '4' - self.assertRaises(FileNotFoundError, r.resolve, strict=True) - # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3' / '4') - - def test_resolve_nonexist_relative_issue38671(self): - p = self.cls('non', 'exist') - - old_cwd = os.getcwd() - os.chdir(BASE) - try: - self.assertEqual(p.resolve(), self.cls(BASE, p)) - finally: - os.chdir(old_cwd) - - def test_with(self): - p = self.cls(BASE) - it = p.iterdir() - it2 = p.iterdir() - next(it2) - with p: - pass - # Using a path as a context manager is a no-op, thus the following - # operations should still succeed after the context manage exits. - next(it) - next(it2) - p.exists() - p.resolve() - p.absolute() - with p: - pass - - def test_chmod(self): - p = self.cls(BASE) / 'fileA' - mode = p.stat().st_mode - # Clear writable bit. - new_mode = mode & ~0o222 - p.chmod(new_mode) - self.assertEqual(p.stat().st_mode, new_mode) - # Set writable bit. - new_mode = mode | 0o222 - p.chmod(new_mode) - self.assertEqual(p.stat().st_mode, new_mode) - - # On Windows, os.chmod does not follow symlinks (issue #15411) - @only_posix - def test_chmod_follow_symlinks_true(self): - p = self.cls(BASE) / 'linkA' - q = p.resolve() - mode = q.stat().st_mode - # Clear writable bit. - new_mode = mode & ~0o222 - p.chmod(new_mode, follow_symlinks=True) - self.assertEqual(q.stat().st_mode, new_mode) - # Set writable bit - new_mode = mode | 0o222 - p.chmod(new_mode, follow_symlinks=True) - self.assertEqual(q.stat().st_mode, new_mode) - - # XXX also need a test for lchmod. - - def test_stat(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(p.stat(), st) - # Change file mode by flipping write bit. - p.chmod(st.st_mode ^ 0o222) - self.addCleanup(p.chmod, st.st_mode) - self.assertNotEqual(p.stat(), st) - - @os_helper.skip_unless_symlink - def test_stat_no_follow_symlinks(self): - p = self.cls(BASE) / 'linkA' - st = p.stat() - self.assertNotEqual(st, p.stat(follow_symlinks=False)) - - def test_stat_no_follow_symlinks_nosymlink(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(st, p.stat(follow_symlinks=False)) - - @os_helper.skip_unless_symlink - def test_lstat(self): - p = self.cls(BASE)/ 'linkA' - st = p.stat() - self.assertNotEqual(st, p.lstat()) - - def test_lstat_nosymlink(self): - p = self.cls(BASE) / 'fileA' - st = p.stat() - self.assertEqual(st, p.lstat()) - - @unittest.skipUnless(pwd, "the pwd module is needed for this test") - def test_owner(self): - p = self.cls(BASE) / 'fileA' - uid = p.stat().st_uid - try: - name = pwd.getpwuid(uid).pw_name - except KeyError: - self.skipTest( - "user %d doesn't have an entry in the system database" % uid) - self.assertEqual(name, p.owner()) - - @unittest.skipUnless(grp, "the grp module is needed for this test") - def test_group(self): - p = self.cls(BASE) / 'fileA' - gid = p.stat().st_gid - try: - name = grp.getgrgid(gid).gr_name - except KeyError: - self.skipTest( - "group %d doesn't have an entry in the system database" % gid) - self.assertEqual(name, p.group()) - - def test_unlink(self): - p = self.cls(BASE) / 'fileA' - p.unlink() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - - def test_unlink_missing_ok(self): - p = self.cls(BASE) / 'fileAAA' - self.assertFileNotFound(p.unlink) - p.unlink(missing_ok=True) - - def test_rmdir(self): - p = self.cls(BASE) / 'dirA' - for q in p.iterdir(): - q.unlink() - p.rmdir() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - - @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") - def test_link_to(self): - P = self.cls(BASE) - p = P / 'fileA' - size = p.stat().st_size - # linking to another path. - q = P / 'dirA' / 'fileAA' - try: - with self.assertWarns(DeprecationWarning): - p.link_to(q) - except PermissionError as e: - self.skipTest('os.link(): %s' % e) - self.assertEqual(q.stat().st_size, size) - self.assertEqual(os.path.samefile(p, q), True) - self.assertTrue(p.stat) - # Linking to a str of a relative path. - r = rel_join('fileAAA') - with self.assertWarns(DeprecationWarning): - q.link_to(r) - self.assertEqual(os.stat(r).st_size, size) - self.assertTrue(q.stat) - - @unittest.skipUnless(hasattr(os, "link"), "os.link() is not present") - def test_hardlink_to(self): - P = self.cls(BASE) - target = P / 'fileA' - size = target.stat().st_size - # linking to another path. - link = P / 'dirA' / 'fileAA' - link.hardlink_to(target) - self.assertEqual(link.stat().st_size, size) - self.assertTrue(os.path.samefile(target, link)) - self.assertTrue(target.exists()) - # Linking to a str of a relative path. - link2 = P / 'dirA' / 'fileAAA' - target2 = rel_join('fileA') - link2.hardlink_to(target2) - self.assertEqual(os.stat(target2).st_size, size) - self.assertTrue(link2.exists()) - - @unittest.skipIf(hasattr(os, "link"), "os.link() is present") - def test_link_to_not_implemented(self): - P = self.cls(BASE) - p = P / 'fileA' - # linking to another path. - q = P / 'dirA' / 'fileAA' - with self.assertRaises(NotImplementedError): - p.link_to(q) - - def test_rename(self): - P = self.cls(BASE) - p = P / 'fileA' - size = p.stat().st_size - # Renaming to another path. - q = P / 'dirA' / 'fileAA' - renamed_p = p.rename(q) - self.assertEqual(renamed_p, q) - self.assertEqual(q.stat().st_size, size) - self.assertFileNotFound(p.stat) - # Renaming to a str of a relative path. - r = rel_join('fileAAA') - renamed_q = q.rename(r) - self.assertEqual(renamed_q, self.cls(r)) - self.assertEqual(os.stat(r).st_size, size) - self.assertFileNotFound(q.stat) - - def test_replace(self): - P = self.cls(BASE) - p = P / 'fileA' - size = p.stat().st_size - # Replacing a non-existing path. - q = P / 'dirA' / 'fileAA' - replaced_p = p.replace(q) - self.assertEqual(replaced_p, q) - self.assertEqual(q.stat().st_size, size) - self.assertFileNotFound(p.stat) - # Replacing another (existing) path. - r = rel_join('dirB', 'fileB') - replaced_q = q.replace(r) - self.assertEqual(replaced_q, self.cls(r)) - self.assertEqual(os.stat(r).st_size, size) - self.assertFileNotFound(q.stat) - - @os_helper.skip_unless_symlink - def test_readlink(self): - P = self.cls(BASE) - self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) - self.assertEqual((P / 'brokenLink').readlink(), - self.cls('non-existing')) - self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) - with self.assertRaises(OSError): - (P / 'fileA').readlink() - - def test_touch_common(self): - P = self.cls(BASE) - p = P / 'newfileA' - self.assertFalse(p.exists()) - p.touch() - self.assertTrue(p.exists()) - st = p.stat() - old_mtime = st.st_mtime - old_mtime_ns = st.st_mtime_ns - # Rewind the mtime sufficiently far in the past to work around - # filesystem-specific timestamp granularity. - os.utime(str(p), (old_mtime - 10, old_mtime - 10)) - # The file mtime should be refreshed by calling touch() again. - p.touch() - st = p.stat() - self.assertGreaterEqual(st.st_mtime_ns, old_mtime_ns) - self.assertGreaterEqual(st.st_mtime, old_mtime) - # Now with exist_ok=False. - p = P / 'newfileB' - self.assertFalse(p.exists()) - p.touch(mode=0o700, exist_ok=False) - self.assertTrue(p.exists()) - self.assertRaises(OSError, p.touch, exist_ok=False) - - def test_touch_nochange(self): - P = self.cls(BASE) - p = P / 'fileA' - p.touch() - with p.open('rb') as f: - self.assertEqual(f.read().strip(), b"this is file A") - - def test_mkdir(self): - P = self.cls(BASE) - p = P / 'newdirA' - self.assertFalse(p.exists()) - p.mkdir() - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(OSError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_parents(self): - # Creating a chain of directories. - p = self.cls(BASE, 'newdirB', 'newdirC') - self.assertFalse(p.exists()) - with self.assertRaises(OSError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.ENOENT) - p.mkdir(parents=True) - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(OSError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - # Test `mode` arg. - mode = stat.S_IMODE(p.stat().st_mode) # Default mode. - p = self.cls(BASE, 'newdirD', 'newdirE') - p.mkdir(0o555, parents=True) - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - if os.name != 'nt': - # The directory's permissions follow the mode argument. - self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o7555 & mode) - # The parent's permissions follow the default process settings. - self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode) - - def test_mkdir_exist_ok(self): - p = self.cls(BASE, 'dirB') - st_ctime_first = p.stat().st_ctime - self.assertTrue(p.exists()) - self.assertTrue(p.is_dir()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - p.mkdir(exist_ok=True) - self.assertTrue(p.exists()) - self.assertEqual(p.stat().st_ctime, st_ctime_first) - - def test_mkdir_exist_ok_with_parent(self): - p = self.cls(BASE, 'dirC') - self.assertTrue(p.exists()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - p = p / 'newdirC' - p.mkdir(parents=True) - st_ctime_first = p.stat().st_ctime - self.assertTrue(p.exists()) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - p.mkdir(parents=True, exist_ok=True) - self.assertTrue(p.exists()) - self.assertEqual(p.stat().st_ctime, st_ctime_first) - - def test_mkdir_exist_ok_root(self): - # Issue #25803: A drive root could raise PermissionError on Windows. - self.cls('/').resolve().mkdir(exist_ok=True) - self.cls('/').resolve().mkdir(parents=True, exist_ok=True) - - @only_nt # XXX: not sure how to test this on POSIX. - def test_mkdir_with_unknown_drive(self): - for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': - p = self.cls(d + ':\\') - if not p.is_dir(): - break - else: - self.skipTest("cannot find a drive that doesn't exist") - with self.assertRaises(OSError): - (p / 'child' / 'path').mkdir(parents=True) - - def test_mkdir_with_child_file(self): - p = self.cls(BASE, 'dirB', 'fileB') - self.assertTrue(p.exists()) - # An exception is raised when the last path component is an existing - # regular file, regardless of whether exist_ok is true or not. - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(parents=True, exist_ok=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_no_parents_file(self): - p = self.cls(BASE, 'fileA') - self.assertTrue(p.exists()) - # An exception is raised when the last path component is an existing - # regular file, regardless of whether exist_ok is true or not. - with self.assertRaises(FileExistsError) as cm: - p.mkdir() - self.assertEqual(cm.exception.errno, errno.EEXIST) - with self.assertRaises(FileExistsError) as cm: - p.mkdir(exist_ok=True) - self.assertEqual(cm.exception.errno, errno.EEXIST) - - def test_mkdir_concurrent_parent_creation(self): - for pattern_num in range(32): - p = self.cls(BASE, 'dirCPC%d' % pattern_num) - self.assertFalse(p.exists()) - - def my_mkdir(path, mode=0o777): - path = str(path) - # Emulate another process that would create the directory - # just before we try to create it ourselves. We do it - # in all possible pattern combinations, assuming that this - # function is called at most 5 times (dirCPC/dir1/dir2, - # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). - if pattern.pop(): - os.mkdir(path, mode) # From another process. - concurrently_created.add(path) - os.mkdir(path, mode) # Our real call. - - pattern = [bool(pattern_num & (1 << n)) for n in range(5)] - concurrently_created = set() - p12 = p / 'dir1' / 'dir2' - try: - with mock.patch("pathlib._normal_accessor.mkdir", my_mkdir): - p12.mkdir(parents=True, exist_ok=False) - except FileExistsError: - self.assertIn(str(p12), concurrently_created) - else: - self.assertNotIn(str(p12), concurrently_created) - self.assertTrue(p.exists()) - - @os_helper.skip_unless_symlink - def test_symlink_to(self): - P = self.cls(BASE) - target = P / 'fileA' - # Symlinking a path target. - link = P / 'dirA' / 'linkAA' - link.symlink_to(target) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - # Symlinking a str target. - link = P / 'dirA' / 'linkAAA' - link.symlink_to(str(target)) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - self.assertFalse(link.is_dir()) - # Symlinking to a directory. - target = P / 'dirB' - link = P / 'dirA' / 'linkAAAA' - link.symlink_to(target, target_is_directory=True) - self.assertEqual(link.stat(), target.stat()) - self.assertNotEqual(link.lstat(), target.stat()) - self.assertTrue(link.is_dir()) - self.assertTrue(list(link.iterdir())) - - def test_is_dir(self): - P = self.cls(BASE) - self.assertTrue((P / 'dirA').is_dir()) - self.assertFalse((P / 'fileA').is_dir()) - self.assertFalse((P / 'non-existing').is_dir()) - self.assertFalse((P / 'fileA' / 'bah').is_dir()) - if os_helper.can_symlink(): - self.assertFalse((P / 'linkA').is_dir()) - self.assertTrue((P / 'linkB').is_dir()) - self.assertFalse((P/ 'brokenLink').is_dir(), False) - self.assertIs((P / 'dirA\udfff').is_dir(), False) - self.assertIs((P / 'dirA\x00').is_dir(), False) - - def test_is_file(self): - P = self.cls(BASE) - self.assertTrue((P / 'fileA').is_file()) - self.assertFalse((P / 'dirA').is_file()) - self.assertFalse((P / 'non-existing').is_file()) - self.assertFalse((P / 'fileA' / 'bah').is_file()) - if os_helper.can_symlink(): - self.assertTrue((P / 'linkA').is_file()) - self.assertFalse((P / 'linkB').is_file()) - self.assertFalse((P/ 'brokenLink').is_file()) - self.assertIs((P / 'fileA\udfff').is_file(), False) - self.assertIs((P / 'fileA\x00').is_file(), False) - - @only_posix - def test_is_mount(self): - P = self.cls(BASE) - R = self.cls('/') # TODO: Work out Windows. - self.assertFalse((P / 'fileA').is_mount()) - self.assertFalse((P / 'dirA').is_mount()) - self.assertFalse((P / 'non-existing').is_mount()) - self.assertFalse((P / 'fileA' / 'bah').is_mount()) - self.assertTrue(R.is_mount()) - if os_helper.can_symlink(): - self.assertFalse((P / 'linkA').is_mount()) - self.assertIs(self.cls('/\udfff').is_mount(), False) - self.assertIs(self.cls('/\x00').is_mount(), False) - - def test_is_symlink(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_symlink()) - self.assertFalse((P / 'dirA').is_symlink()) - self.assertFalse((P / 'non-existing').is_symlink()) - self.assertFalse((P / 'fileA' / 'bah').is_symlink()) - if os_helper.can_symlink(): - self.assertTrue((P / 'linkA').is_symlink()) - self.assertTrue((P / 'linkB').is_symlink()) - self.assertTrue((P/ 'brokenLink').is_symlink()) - self.assertIs((P / 'fileA\udfff').is_file(), False) - self.assertIs((P / 'fileA\x00').is_file(), False) - if os_helper.can_symlink(): - self.assertIs((P / 'linkA\udfff').is_file(), False) - self.assertIs((P / 'linkA\x00').is_file(), False) - - def test_is_fifo_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_fifo()) - self.assertFalse((P / 'dirA').is_fifo()) - self.assertFalse((P / 'non-existing').is_fifo()) - self.assertFalse((P / 'fileA' / 'bah').is_fifo()) - self.assertIs((P / 'fileA\udfff').is_fifo(), False) - self.assertIs((P / 'fileA\x00').is_fifo(), False) - - @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") - @unittest.skipIf(sys.platform == "vxworks", - "fifo requires special path on VxWorks") - def test_is_fifo_true(self): - P = self.cls(BASE, 'myfifo') - try: - os.mkfifo(str(P)) - except PermissionError as e: - self.skipTest('os.mkfifo(): %s' % e) - self.assertTrue(P.is_fifo()) - self.assertFalse(P.is_socket()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'myfifo\udfff').is_fifo(), False) - self.assertIs(self.cls(BASE, 'myfifo\x00').is_fifo(), False) - - def test_is_socket_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_socket()) - self.assertFalse((P / 'dirA').is_socket()) - self.assertFalse((P / 'non-existing').is_socket()) - self.assertFalse((P / 'fileA' / 'bah').is_socket()) - self.assertIs((P / 'fileA\udfff').is_socket(), False) - self.assertIs((P / 'fileA\x00').is_socket(), False) - - @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") - def test_is_socket_true(self): - P = self.cls(BASE, 'mysock') - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.addCleanup(sock.close) - try: - sock.bind(str(P)) - except OSError as e: - if (isinstance(e, PermissionError) or - "AF_UNIX path too long" in str(e)): - self.skipTest("cannot bind Unix socket: " + str(e)) - self.assertTrue(P.is_socket()) - self.assertFalse(P.is_fifo()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls(BASE, 'mysock\udfff').is_socket(), False) - self.assertIs(self.cls(BASE, 'mysock\x00').is_socket(), False) - - def test_is_block_device_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_block_device()) - self.assertFalse((P / 'dirA').is_block_device()) - self.assertFalse((P / 'non-existing').is_block_device()) - self.assertFalse((P / 'fileA' / 'bah').is_block_device()) - self.assertIs((P / 'fileA\udfff').is_block_device(), False) - self.assertIs((P / 'fileA\x00').is_block_device(), False) - - def test_is_char_device_false(self): - P = self.cls(BASE) - self.assertFalse((P / 'fileA').is_char_device()) - self.assertFalse((P / 'dirA').is_char_device()) - self.assertFalse((P / 'non-existing').is_char_device()) - self.assertFalse((P / 'fileA' / 'bah').is_char_device()) - self.assertIs((P / 'fileA\udfff').is_char_device(), False) - self.assertIs((P / 'fileA\x00').is_char_device(), False) - - def test_is_char_device_true(self): - # Under Unix, /dev/null should generally be a char device. - P = self.cls('/dev/null') - if not P.exists(): - self.skipTest("/dev/null required") - self.assertTrue(P.is_char_device()) - self.assertFalse(P.is_block_device()) - self.assertFalse(P.is_file()) - self.assertIs(self.cls('/dev/null\udfff').is_char_device(), False) - self.assertIs(self.cls('/dev/null\x00').is_char_device(), False) - - def test_pickling_common(self): - p = self.cls(BASE, 'fileA') - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - dumped = pickle.dumps(p, proto) - pp = pickle.loads(dumped) - self.assertEqual(pp.stat(), p.stat()) - - def test_parts_interning(self): - P = self.cls - p = P('/usr/bin/foo') - q = P('/usr/local/bin') - # 'usr' - self.assertIs(p.parts[1], q.parts[1]) - # 'bin' - self.assertIs(p.parts[2], q.parts[3]) - - def _check_complex_symlinks(self, link0_target): - # Test solving a non-looping chain of symlinks (issue #19887). - P = self.cls(BASE) - self.dirlink(os.path.join('link0', 'link0'), join('link1')) - self.dirlink(os.path.join('link1', 'link1'), join('link2')) - self.dirlink(os.path.join('link2', 'link2'), join('link3')) - self.dirlink(link0_target, join('link0')) - - # Resolve absolute paths. - p = (P / 'link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = (P / 'link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - - # Resolve relative paths. - old_path = os.getcwd() - os.chdir(BASE) - try: - p = self.cls('link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - p = self.cls('link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), BASE) - finally: - os.chdir(old_path) - - @os_helper.skip_unless_symlink - def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(BASE) - - @os_helper.skip_unless_symlink - def test_complex_symlinks_relative(self): - self._check_complex_symlinks('.') - - @os_helper.skip_unless_symlink - def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(os.path.join('dirA', '..')) - - -class PathTest(_BasePathTest, unittest.TestCase): - cls = pathlib.Path - - def test_class_getitem(self): - self.assertIs(self.cls[str], self.cls) - - def test_concrete_class(self): - p = self.cls('a') - self.assertIs(type(p), - pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath) - - def test_unsupported_flavour(self): - if os.name == 'nt': - self.assertRaises(NotImplementedError, pathlib.PosixPath) - else: - self.assertRaises(NotImplementedError, pathlib.WindowsPath) - - def test_glob_empty_pattern(self): - p = self.cls() - with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): - list(p.glob('')) - - -@only_posix -class PosixPathTest(_BasePathTest, unittest.TestCase): - cls = pathlib.PosixPath - - def _check_symlink_loop(self, *args, strict=True): - path = self.cls(*args) - with self.assertRaises(RuntimeError): - print(path.resolve(strict)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_open_mode(self): - old_mask = os.umask(0) - self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) - with (p / 'new_file').open('wb'): - pass - st = os.stat(join('new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) - with (p / 'other_new_file').open('wb'): - pass - st = os.stat(join('other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) - - def test_resolve_root(self): - current_directory = os.getcwd() - try: - os.chdir('/') - p = self.cls('spam') - self.assertEqual(str(p.resolve()), '/spam') - finally: - os.chdir(current_directory) - - def test_touch_mode(self): - old_mask = os.umask(0) - self.addCleanup(os.umask, old_mask) - p = self.cls(BASE) - (p / 'new_file').touch() - st = os.stat(join('new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o666) - os.umask(0o022) - (p / 'other_new_file').touch() - st = os.stat(join('other_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o644) - (p / 'masked_new_file').touch(mode=0o750) - st = os.stat(join('masked_new_file')) - self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) - - @os_helper.skip_unless_symlink - def test_resolve_loop(self): - # Loops with relative symlinks. - os.symlink('linkX/inside', join('linkX')) - self._check_symlink_loop(BASE, 'linkX') - os.symlink('linkY', join('linkY')) - self._check_symlink_loop(BASE, 'linkY') - os.symlink('linkZ/../linkZ', join('linkZ')) - self._check_symlink_loop(BASE, 'linkZ') - # Non-strict - self._check_symlink_loop(BASE, 'linkZ', 'foo', strict=False) - # Loops with absolute symlinks. - os.symlink(join('linkU/inside'), join('linkU')) - self._check_symlink_loop(BASE, 'linkU') - os.symlink(join('linkV'), join('linkV')) - self._check_symlink_loop(BASE, 'linkV') - os.symlink(join('linkW/../linkW'), join('linkW')) - self._check_symlink_loop(BASE, 'linkW') - # Non-strict - self._check_symlink_loop(BASE, 'linkW', 'foo', strict=False) - - def test_glob(self): - P = self.cls - p = P(BASE) - given = set(p.glob("FILEa")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.glob("FILEa*")), set()) - - def test_rglob(self): - P = self.cls - p = P(BASE, "dirC") - given = set(p.rglob("FILEd")) - expect = set() if not os_helper.fs_is_case_insensitive(BASE) else given - self.assertEqual(given, expect) - self.assertEqual(set(p.rglob("FILEd*")), set()) - - @unittest.skipUnless(hasattr(pwd, 'getpwall'), - 'pwd module does not expose getpwall()') - @unittest.skipIf(sys.platform == "vxworks", - "no home directory on VxWorks") - def test_expanduser(self): - P = self.cls - import_helper.import_module('pwd') - import pwd - pwdent = pwd.getpwuid(os.getuid()) - username = pwdent.pw_name - userhome = pwdent.pw_dir.rstrip('/') or '/' - # Find arbitrary different user (if exists). - for pwdent in pwd.getpwall(): - othername = pwdent.pw_name - otherhome = pwdent.pw_dir.rstrip('/') - if othername != username and otherhome: - break - else: - othername = username - otherhome = userhome - - fakename = 'fakeuser' - # This user can theoretically exist on a test runner. Create unique name: - try: - while pwd.getpwnam(fakename): - fakename += '1' - except KeyError: - pass # Non-existent name found - - p1 = P('~/Documents') - p2 = P(f'~{username}/Documents') - p3 = P(f'~{othername}/Documents') - p4 = P(f'../~{username}/Documents') - p5 = P(f'/~{username}/Documents') - p6 = P('') - p7 = P(f'~{fakename}/Documents') - - with os_helper.EnvironmentVarGuard() as env: - env.pop('HOME', None) - - self.assertEqual(p1.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - self.assertRaises(RuntimeError, p7.expanduser) - - env['HOME'] = '/tmp' - self.assertEqual(p1.expanduser(), P('/tmp/Documents')) - self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') - self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - self.assertRaises(RuntimeError, p7.expanduser) - - @unittest.skipIf(sys.platform != "darwin", - "Bad file descriptor in /dev/fd affects only macOS") - def test_handling_bad_descriptor(self): - try: - file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] - if not file_descriptors: - self.skipTest("no file descriptors - issue was not reproduced") - # Checking all file descriptors because there is no guarantee - # which one will fail. - for f in file_descriptors: - f.exists() - f.is_dir() - f.is_file() - f.is_symlink() - f.is_block_device() - f.is_char_device() - f.is_fifo() - f.is_socket() - except OSError as e: - if e.errno == errno.EBADF: - self.fail("Bad file descriptor not handled.") - raise - - -@only_nt -class WindowsPathTest(_BasePathTest, unittest.TestCase): - cls = pathlib.WindowsPath - - def test_glob(self): - P = self.cls - p = P(BASE) - self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") }) - self.assertEqual(set(p.glob("F*a")), { P(BASE, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\FILEa"}) - self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) - - def test_rglob(self): - P = self.cls - p = P(BASE, "dirC") - self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\FILEd"}) - - def test_expanduser(self): - P = self.cls - with os_helper.EnvironmentVarGuard() as env: - env.pop('HOME', None) - env.pop('USERPROFILE', None) - env.pop('HOMEPATH', None) - env.pop('HOMEDRIVE', None) - env['USERNAME'] = 'alice' - - # test that the path returns unchanged - p1 = P('~/My Documents') - p2 = P('~alice/My Documents') - p3 = P('~bob/My Documents') - p4 = P('/~/My Documents') - p5 = P('d:~/My Documents') - p6 = P('') - self.assertRaises(RuntimeError, p1.expanduser) - self.assertRaises(RuntimeError, p2.expanduser) - self.assertRaises(RuntimeError, p3.expanduser) - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - - def check(): - env.pop('USERNAME', None) - self.assertEqual(p1.expanduser(), - P('C:/Users/alice/My Documents')) - self.assertRaises(RuntimeError, p2.expanduser) - env['USERNAME'] = 'alice' - self.assertEqual(p2.expanduser(), - P('C:/Users/alice/My Documents')) - self.assertEqual(p3.expanduser(), - P('C:/Users/bob/My Documents')) - self.assertEqual(p4.expanduser(), p4) - self.assertEqual(p5.expanduser(), p5) - self.assertEqual(p6.expanduser(), p6) - - env['HOMEPATH'] = 'C:\\Users\\alice' - check() - - env['HOMEDRIVE'] = 'C:\\' - env['HOMEPATH'] = 'Users\\alice' - check() - - env.pop('HOMEDRIVE', None) - env.pop('HOMEPATH', None) - env['USERPROFILE'] = 'C:\\Users\\alice' - check() - - # bpo-38883: ignore `HOME` when set on windows - env['HOME'] = 'C:\\Users\\eve' - check() - - -class CompatiblePathTest(unittest.TestCase): - """ - Test that a type can be made compatible with PurePath - derivatives by implementing division operator overloads. - """ - - class CompatPath: - """ - Minimum viable class to test PurePath compatibility. - Simply uses the division operator to join a given - string and the string value of another object with - a forward slash. - """ - def __init__(self, string): - self.string = string - - def __truediv__(self, other): - return type(self)(f"{self.string}/{other}") - - def __rtruediv__(self, other): - return type(self)(f"{other}/{self.string}") - - def test_truediv(self): - result = pathlib.PurePath("test") / self.CompatPath("right") - self.assertIsInstance(result, self.CompatPath) - self.assertEqual(result.string, "test/right") - - with self.assertRaises(TypeError): - # Verify improper operations still raise a TypeError - pathlib.PurePath("test") / 10 - - def test_rtruediv(self): - result = self.CompatPath("left") / pathlib.PurePath("test") - self.assertIsInstance(result, self.CompatPath) - self.assertEqual(result.string, "left/test") - - with self.assertRaises(TypeError): - # Verify improper operations still raise a TypeError - 10 / pathlib.PurePath("test") - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_pathlib/__init__.py b/Lib/test/test_pathlib/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_pathlib/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pathlib/support/__init__.py b/Lib/test/test_pathlib/support/__init__.py new file mode 100644 index 00000000000..dcaef654d77 --- /dev/null +++ b/Lib/test/test_pathlib/support/__init__.py @@ -0,0 +1,2 @@ +# Set to 'True' if the tests are run against the pathlib-abc PyPI package. +is_pypi = False diff --git a/Lib/test/test_pathlib/support/lexical_path.py b/Lib/test/test_pathlib/support/lexical_path.py new file mode 100644 index 00000000000..f29a521af9b --- /dev/null +++ b/Lib/test/test_pathlib/support/lexical_path.py @@ -0,0 +1,51 @@ +""" +Simple implementation of JoinablePath, for use in pathlib tests. +""" + +import ntpath +import os.path +import posixpath + +from . import is_pypi + +if is_pypi: + from pathlib_abc import _JoinablePath +else: + from pathlib.types import _JoinablePath + + +class LexicalPath(_JoinablePath): + __slots__ = ('_segments',) + parser = os.path + + def __init__(self, *pathsegments): + self._segments = pathsegments + + def __hash__(self): + return hash(str(self)) + + def __eq__(self, other): + if not isinstance(other, LexicalPath): + return NotImplemented + return str(self) == str(other) + + def __str__(self): + if not self._segments: + return '' + return self.parser.join(*self._segments) + + def __repr__(self): + return f'{type(self).__name__}({str(self)!r})' + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments) + + +class LexicalPosixPath(LexicalPath): + __slots__ = () + parser = posixpath + + +class LexicalWindowsPath(LexicalPath): + __slots__ = () + parser = ntpath diff --git a/Lib/test/test_pathlib/support/local_path.py b/Lib/test/test_pathlib/support/local_path.py new file mode 100644 index 00000000000..d481fd45ead --- /dev/null +++ b/Lib/test/test_pathlib/support/local_path.py @@ -0,0 +1,177 @@ +""" +Implementations of ReadablePath and WritablePath for local paths, for use in +pathlib tests. + +LocalPathGround is also defined here. It helps establish the "ground truth" +about local paths in tests. +""" + +import os + +from . import is_pypi +from .lexical_path import LexicalPath + +if is_pypi: + from shutil import rmtree + from pathlib_abc import PathInfo, _ReadablePath, _WritablePath + can_symlink = True + testfn = "TESTFN" +else: + from pathlib.types import PathInfo, _ReadablePath, _WritablePath + from test.support import os_helper + can_symlink = os_helper.can_symlink() + testfn = os_helper.TESTFN + rmtree = os_helper.rmtree + + +class LocalPathGround: + can_symlink = can_symlink + + def __init__(self, path_cls): + self.path_cls = path_cls + + def setup(self, local_suffix=""): + root = self.path_cls(testfn + local_suffix) + os.mkdir(root) + return root + + def teardown(self, root): + rmtree(root) + + def create_file(self, p, data=b''): + with open(p, 'wb') as f: + f.write(data) + + def create_dir(self, p): + os.mkdir(p) + + def create_symlink(self, p, target): + os.symlink(target, p) + + def create_hierarchy(self, p): + os.mkdir(os.path.join(p, 'dirA')) + os.mkdir(os.path.join(p, 'dirB')) + os.mkdir(os.path.join(p, 'dirC')) + os.mkdir(os.path.join(p, 'dirC', 'dirD')) + with open(os.path.join(p, 'fileA'), 'wb') as f: + f.write(b"this is file A\n") + with open(os.path.join(p, 'dirB', 'fileB'), 'wb') as f: + f.write(b"this is file B\n") + with open(os.path.join(p, 'dirC', 'fileC'), 'wb') as f: + f.write(b"this is file C\n") + with open(os.path.join(p, 'dirC', 'novel.txt'), 'wb') as f: + f.write(b"this is a novel\n") + with open(os.path.join(p, 'dirC', 'dirD', 'fileD'), 'wb') as f: + f.write(b"this is file D\n") + if self.can_symlink: + # Relative symlinks. + os.symlink('fileA', os.path.join(p, 'linkA')) + os.symlink('non-existing', os.path.join(p, 'brokenLink')) + os.symlink('dirB', + os.path.join(p, 'linkB'), + target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), + os.path.join(p, 'dirA', 'linkC'), + target_is_directory=True) + # Broken symlink (pointing to itself). + os.symlink('brokenLinkLoop', os.path.join(p, 'brokenLinkLoop')) + + isdir = staticmethod(os.path.isdir) + isfile = staticmethod(os.path.isfile) + islink = staticmethod(os.path.islink) + readlink = staticmethod(os.readlink) + + def readtext(self, p): + with open(p, 'r', encoding='utf-8') as f: + return f.read() + + def readbytes(self, p): + with open(p, 'rb') as f: + return f.read() + + +class LocalPathInfo(PathInfo): + """ + Simple implementation of PathInfo for a local path + """ + __slots__ = ('_path', '_exists', '_is_dir', '_is_file', '_is_symlink') + + def __init__(self, path): + self._path = str(path) + self._exists = None + self._is_dir = None + self._is_file = None + self._is_symlink = None + + def exists(self, *, follow_symlinks=True): + """Whether this path exists.""" + if not follow_symlinks and self.is_symlink(): + return True + if self._exists is None: + self._exists = os.path.exists(self._path) + return self._exists + + def is_dir(self, *, follow_symlinks=True): + """Whether this path is a directory.""" + if not follow_symlinks and self.is_symlink(): + return False + if self._is_dir is None: + self._is_dir = os.path.isdir(self._path) + return self._is_dir + + def is_file(self, *, follow_symlinks=True): + """Whether this path is a regular file.""" + if not follow_symlinks and self.is_symlink(): + return False + if self._is_file is None: + self._is_file = os.path.isfile(self._path) + return self._is_file + + def is_symlink(self): + """Whether this path is a symbolic link.""" + if self._is_symlink is None: + self._is_symlink = os.path.islink(self._path) + return self._is_symlink + + +class ReadableLocalPath(_ReadablePath, LexicalPath): + """ + Simple implementation of a ReadablePath class for local filesystem paths. + """ + __slots__ = ('info',) + + def __init__(self, *pathsegments): + super().__init__(*pathsegments) + self.info = LocalPathInfo(self) + + def __fspath__(self): + return str(self) + + def __open_rb__(self, buffering=-1): + return open(self, 'rb') + + def iterdir(self): + return (self / name for name in os.listdir(self)) + + def readlink(self): + return self.with_segments(os.readlink(self)) + + +class WritableLocalPath(_WritablePath, LexicalPath): + """ + Simple implementation of a WritablePath class for local filesystem paths. + """ + + __slots__ = () + + def __fspath__(self): + return str(self) + + def __open_wb__(self, buffering=-1): + return open(self, 'wb') + + def mkdir(self, mode=0o777): + os.mkdir(self, mode) + + def symlink_to(self, target, target_is_directory=False): + os.symlink(target, self, target_is_directory) diff --git a/Lib/test/test_pathlib/support/zip_path.py b/Lib/test/test_pathlib/support/zip_path.py new file mode 100644 index 00000000000..2905260c9df --- /dev/null +++ b/Lib/test/test_pathlib/support/zip_path.py @@ -0,0 +1,336 @@ +""" +Implementations of ReadablePath and WritablePath for zip file members, for use +in pathlib tests. + +ZipPathGround is also defined here. It helps establish the "ground truth" +about zip file members in tests. +""" + +import errno +import io +import posixpath +import stat +import zipfile +from stat import S_IFMT, S_ISDIR, S_ISREG, S_ISLNK + +from . import is_pypi + +if is_pypi: + from pathlib_abc import PathInfo, _ReadablePath, _WritablePath +else: + from pathlib.types import PathInfo, _ReadablePath, _WritablePath + + +class ZipPathGround: + can_symlink = True + + def __init__(self, path_cls): + self.path_cls = path_cls + + def setup(self, local_suffix=""): + return self.path_cls(zip_file=zipfile.ZipFile(io.BytesIO(), "w")) + + def teardown(self, root): + root.zip_file.close() + + def create_file(self, path, data=b''): + path.zip_file.writestr(str(path), data) + + def create_dir(self, path): + zip_info = zipfile.ZipInfo(str(path) + '/') + zip_info.external_attr |= stat.S_IFDIR << 16 + zip_info.external_attr |= stat.FILE_ATTRIBUTE_DIRECTORY + path.zip_file.writestr(zip_info, '') + + def create_symlink(self, path, target): + zip_info = zipfile.ZipInfo(str(path)) + zip_info.external_attr = stat.S_IFLNK << 16 + path.zip_file.writestr(zip_info, target.encode()) + + def create_hierarchy(self, p): + # Add regular files + self.create_file(p.joinpath('fileA'), b'this is file A\n') + self.create_file(p.joinpath('dirB/fileB'), b'this is file B\n') + self.create_file(p.joinpath('dirC/fileC'), b'this is file C\n') + self.create_file(p.joinpath('dirC/dirD/fileD'), b'this is file D\n') + self.create_file(p.joinpath('dirC/novel.txt'), b'this is a novel\n') + # Add symlinks + self.create_symlink(p.joinpath('linkA'), 'fileA') + self.create_symlink(p.joinpath('linkB'), 'dirB') + self.create_symlink(p.joinpath('dirA/linkC'), '../dirB') + self.create_symlink(p.joinpath('brokenLink'), 'non-existing') + self.create_symlink(p.joinpath('brokenLinkLoop'), 'brokenLinkLoop') + + def readtext(self, p): + with p.zip_file.open(str(p), 'r') as f: + f = io.TextIOWrapper(f, encoding='utf-8') + return f.read() + + def readbytes(self, p): + with p.zip_file.open(str(p), 'r') as f: + return f.read() + + readlink = readtext + + def isdir(self, p): + path_str = str(p) + "/" + return path_str in p.zip_file.NameToInfo + + def isfile(self, p): + info = p.zip_file.NameToInfo.get(str(p)) + if info is None: + return False + return not stat.S_ISLNK(info.external_attr >> 16) + + def islink(self, p): + info = p.zip_file.NameToInfo.get(str(p)) + if info is None: + return False + return stat.S_ISLNK(info.external_attr >> 16) + + +class MissingZipPathInfo(PathInfo): + """ + PathInfo implementation that is used when a zip file member is missing. + """ + __slots__ = () + + def exists(self, follow_symlinks=True): + return False + + def is_dir(self, follow_symlinks=True): + return False + + def is_file(self, follow_symlinks=True): + return False + + def is_symlink(self): + return False + + def resolve(self): + return self + + +missing_zip_path_info = MissingZipPathInfo() + + +class ZipPathInfo(PathInfo): + """ + PathInfo implementation for an existing zip file member. + """ + __slots__ = ('zip_file', 'zip_info', 'parent', 'children') + + def __init__(self, zip_file, parent=None): + self.zip_file = zip_file + self.zip_info = None + self.parent = parent or self + self.children = {} + + def exists(self, follow_symlinks=True): + if follow_symlinks and self.is_symlink(): + return self.resolve().exists() + return True + + def is_dir(self, follow_symlinks=True): + if follow_symlinks and self.is_symlink(): + return self.resolve().is_dir() + elif self.zip_info is None: + return True + elif fmt := S_IFMT(self.zip_info.external_attr >> 16): + return S_ISDIR(fmt) + else: + return self.zip_info.filename.endswith('/') + + def is_file(self, follow_symlinks=True): + if follow_symlinks and self.is_symlink(): + return self.resolve().is_file() + elif self.zip_info is None: + return False + elif fmt := S_IFMT(self.zip_info.external_attr >> 16): + return S_ISREG(fmt) + else: + return not self.zip_info.filename.endswith('/') + + def is_symlink(self): + if self.zip_info is None: + return False + elif fmt := S_IFMT(self.zip_info.external_attr >> 16): + return S_ISLNK(fmt) + else: + return False + + def resolve(self, path=None, create=False, follow_symlinks=True): + """ + Traverse zip hierarchy (parents, children and symlinks) starting + from this PathInfo. This is called from three places: + + - When a zip file member is added to ZipFile.filelist, this method + populates the ZipPathInfo tree (using create=True). + - When ReadableZipPath.info is accessed, this method is finds a + ZipPathInfo entry for the path without resolving any final symlink + (using follow_symlinks=False) + - When ZipPathInfo methods are called with follow_symlinks=True, this + method resolves any symlink in the final path position. + """ + link_count = 0 + stack = path.split('/')[::-1] if path else [] + info = self + while True: + if info.is_symlink() and (follow_symlinks or stack): + link_count += 1 + if link_count >= 40: + return missing_zip_path_info # Symlink loop! + path = info.zip_file.read(info.zip_info).decode() + stack += path.split('/')[::-1] if path else [] + info = info.parent + + if stack: + name = stack.pop() + else: + return info + + if name == '..': + info = info.parent + elif name and name != '.': + if name not in info.children: + if create: + info.children[name] = ZipPathInfo(info.zip_file, info) + else: + return missing_zip_path_info # No such child! + info = info.children[name] + + +class ZipFileList: + """ + `list`-like object that we inject as `ZipFile.filelist`. We maintain a + tree of `ZipPathInfo` objects representing the zip file members. + """ + + __slots__ = ('tree', '_items') + + def __init__(self, zip_file): + self.tree = ZipPathInfo(zip_file) + self._items = [] + for item in zip_file.filelist: + self.append(item) + + def __len__(self): + return len(self._items) + + def __iter__(self): + return iter(self._items) + + def append(self, item): + self._items.append(item) + self.tree.resolve(item.filename, create=True).zip_info = item + + +class ReadableZipPath(_ReadablePath): + """ + Simple implementation of a ReadablePath class for .zip files. + """ + + __slots__ = ('_segments', 'zip_file') + parser = posixpath + + def __init__(self, *pathsegments, zip_file): + self._segments = pathsegments + self.zip_file = zip_file + if not isinstance(zip_file.filelist, ZipFileList): + zip_file.filelist = ZipFileList(zip_file) + + def __hash__(self): + return hash((str(self), self.zip_file)) + + def __eq__(self, other): + if not isinstance(other, ReadableZipPath): + return NotImplemented + return str(self) == str(other) and self.zip_file is other.zip_file + + def __str__(self): + if not self._segments: + return '' + return self.parser.join(*self._segments) + + def __repr__(self): + return f'{type(self).__name__}({str(self)!r}, zip_file={self.zip_file!r})' + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, zip_file=self.zip_file) + + @property + def info(self): + tree = self.zip_file.filelist.tree + return tree.resolve(str(self), follow_symlinks=False) + + def __open_rb__(self, buffering=-1): + info = self.info.resolve() + if not info.exists(): + raise FileNotFoundError(errno.ENOENT, "File not found", self) + elif info.is_dir(): + raise IsADirectoryError(errno.EISDIR, "Is a directory", self) + return self.zip_file.open(info.zip_info, 'r') + + def iterdir(self): + info = self.info.resolve() + if not info.exists(): + raise FileNotFoundError(errno.ENOENT, "File not found", self) + elif not info.is_dir(): + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", self) + return (self / name for name in info.children) + + def readlink(self): + info = self.info + if not info.exists(): + raise FileNotFoundError(errno.ENOENT, "File not found", self) + elif not info.is_symlink(): + raise OSError(errno.EINVAL, "Not a symlink", self) + return self.with_segments(self.zip_file.read(info.zip_info).decode()) + + +class WritableZipPath(_WritablePath): + """ + Simple implementation of a WritablePath class for .zip files. + """ + + __slots__ = ('_segments', 'zip_file') + parser = posixpath + + def __init__(self, *pathsegments, zip_file): + self._segments = pathsegments + self.zip_file = zip_file + + def __hash__(self): + return hash((str(self), self.zip_file)) + + def __eq__(self, other): + if not isinstance(other, WritableZipPath): + return NotImplemented + return str(self) == str(other) and self.zip_file is other.zip_file + + def __str__(self): + if not self._segments: + return '' + return self.parser.join(*self._segments) + + def __repr__(self): + return f'{type(self).__name__}({str(self)!r}, zip_file={self.zip_file!r})' + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, zip_file=self.zip_file) + + def __open_wb__(self, buffering=-1): + return self.zip_file.open(str(self), 'w') + + def mkdir(self, mode=0o777): + zinfo = zipfile.ZipInfo(str(self) + '/') + zinfo.external_attr |= stat.S_IFDIR << 16 + zinfo.external_attr |= stat.FILE_ATTRIBUTE_DIRECTORY + self.zip_file.writestr(zinfo, '') + + def symlink_to(self, target, target_is_directory=False): + zinfo = zipfile.ZipInfo(str(self)) + zinfo.external_attr = stat.S_IFLNK << 16 + if target_is_directory: + zinfo.external_attr |= 0x10 + self.zip_file.writestr(zinfo, str(target)) diff --git a/Lib/test/test_pathlib/test_copy.py b/Lib/test/test_pathlib/test_copy.py new file mode 100644 index 00000000000..5f4cf82a031 --- /dev/null +++ b/Lib/test/test_pathlib/test_copy.py @@ -0,0 +1,174 @@ +""" +Tests for copying from pathlib.types._ReadablePath to _WritablePath. +""" + +import contextlib +import unittest + +from .support import is_pypi +from .support.local_path import LocalPathGround +from .support.zip_path import ZipPathGround, ReadableZipPath, WritableZipPath + + +class CopyTestBase: + def setUp(self): + self.source_root = self.source_ground.setup() + self.source_ground.create_hierarchy(self.source_root) + self.target_root = self.target_ground.setup(local_suffix="_target") + + def tearDown(self): + self.source_ground.teardown(self.source_root) + self.target_ground.teardown(self.target_root) + + def test_copy_file(self): + source = self.source_root / 'fileA' + target = self.target_root / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isfile(target)) + self.assertEqual(self.source_ground.readbytes(source), + self.target_ground.readbytes(result)) + + def test_copy_file_empty(self): + source = self.source_root / 'empty' + target = self.target_root / 'copyA' + self.source_ground.create_file(source, b'') + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isfile(target)) + self.assertEqual(self.target_ground.readbytes(result), b'') + + def test_copy_file_to_existing_file(self): + source = self.source_root / 'fileA' + target = self.target_root / 'copyA' + self.target_ground.create_file(target, b'this is a copy\n') + with contextlib.ExitStack() as stack: + if isinstance(target, WritableZipPath): + stack.enter_context(self.assertWarns(UserWarning)) + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isfile(target)) + self.assertEqual(self.source_ground.readbytes(source), + self.target_ground.readbytes(result)) + + def test_copy_file_to_directory(self): + if isinstance(self.target_root, WritableZipPath): + self.skipTest('needs local target') + source = self.source_root / 'fileA' + target = self.target_root / 'copyA' + self.target_ground.create_dir(target) + self.assertRaises(OSError, source.copy, target) + + def test_copy_file_to_itself(self): + source = self.source_root / 'fileA' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + def test_copy_dir(self): + source = self.source_root / 'dirC' + target = self.target_root / 'copyC' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isdir(target)) + self.assertTrue(self.target_ground.isfile(target / 'fileC')) + self.assertEqual(self.target_ground.readtext(target / 'fileC'), 'this is file C\n') + self.assertTrue(self.target_ground.isdir(target / 'dirD')) + self.assertTrue(self.target_ground.isfile(target / 'dirD' / 'fileD')) + self.assertEqual(self.target_ground.readtext(target / 'dirD' / 'fileD'), 'this is file D\n') + + def test_copy_dir_follow_symlinks_true(self): + if not self.source_ground.can_symlink: + self.skipTest('needs symlink support on source') + source = self.source_root / 'dirC' + target = self.target_root / 'copyC' + self.source_ground.create_symlink(source / 'linkC', 'fileC') + self.source_ground.create_symlink(source / 'linkD', 'dirD') + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isdir(target)) + self.assertFalse(self.target_ground.islink(target / 'linkC')) + self.assertTrue(self.target_ground.isfile(target / 'linkC')) + self.assertEqual(self.target_ground.readtext(target / 'linkC'), 'this is file C\n') + self.assertFalse(self.target_ground.islink(target / 'linkD')) + self.assertTrue(self.target_ground.isdir(target / 'linkD')) + self.assertTrue(self.target_ground.isfile(target / 'linkD' / 'fileD')) + self.assertEqual(self.target_ground.readtext(target / 'linkD' / 'fileD'), 'this is file D\n') + + def test_copy_dir_follow_symlinks_false(self): + if not self.source_ground.can_symlink: + self.skipTest('needs symlink support on source') + if not self.target_ground.can_symlink: + self.skipTest('needs symlink support on target') + source = self.source_root / 'dirC' + target = self.target_root / 'copyC' + self.source_ground.create_symlink(source / 'linkC', 'fileC') + self.source_ground.create_symlink(source / 'linkD', 'dirD') + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(self.target_ground.isdir(target)) + self.assertTrue(self.target_ground.islink(target / 'linkC')) + self.assertEqual(self.target_ground.readlink(target / 'linkC'), 'fileC') + self.assertTrue(self.target_ground.islink(target / 'linkD')) + self.assertEqual(self.target_ground.readlink(target / 'linkD'), 'dirD') + + def test_copy_dir_to_existing_directory(self): + if isinstance(self.target_root, WritableZipPath): + self.skipTest('needs local target') + source = self.source_root / 'dirC' + target = self.target_root / 'copyC' + self.target_ground.create_dir(target) + self.assertRaises(FileExistsError, source.copy, target) + + def test_copy_dir_to_itself(self): + source = self.source_root / 'dirC' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + def test_copy_dir_into_itself(self): + source = self.source_root / 'dirC' + target = self.source_root / 'dirC' / 'dirD' / 'copyC' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + def test_copy_into(self): + source = self.source_root / 'fileA' + target_dir = self.target_root / 'dirA' + self.target_ground.create_dir(target_dir) + result = source.copy_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertTrue(self.target_ground.isfile(result)) + self.assertEqual(self.source_ground.readbytes(source), + self.target_ground.readbytes(result)) + + def test_copy_into_empty_name(self): + source = self.source_root.with_segments() + target_dir = self.target_root / 'dirA' + self.target_ground.create_dir(target_dir) + self.assertRaises(ValueError, source.copy_into, target_dir) + + +class ZipToZipPathCopyTest(CopyTestBase, unittest.TestCase): + source_ground = ZipPathGround(ReadableZipPath) + target_ground = ZipPathGround(WritableZipPath) + + +if not is_pypi: + from pathlib import Path + + class ZipToLocalPathCopyTest(CopyTestBase, unittest.TestCase): + source_ground = ZipPathGround(ReadableZipPath) + target_ground = LocalPathGround(Path) + + + class LocalToZipPathCopyTest(CopyTestBase, unittest.TestCase): + source_ground = LocalPathGround(Path) + target_ground = ZipPathGround(WritableZipPath) + + + class LocalToLocalPathCopyTest(CopyTestBase, unittest.TestCase): + source_ground = LocalPathGround(Path) + target_ground = LocalPathGround(Path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_join.py b/Lib/test/test_pathlib/test_join.py new file mode 100644 index 00000000000..6b51a09e5ac --- /dev/null +++ b/Lib/test/test_pathlib/test_join.py @@ -0,0 +1,395 @@ +""" +Tests for pathlib.types._JoinablePath +""" + +import unittest +import threading +from test.support import threading_helper + +from .support import is_pypi +from .support.lexical_path import LexicalPath + +if is_pypi: + from pathlib_abc import _PathParser, _JoinablePath +else: + from pathlib.types import _PathParser, _JoinablePath + + +class JoinTestBase: + def test_is_joinable(self): + p = self.cls() + self.assertIsInstance(p, _JoinablePath) + + def test_parser(self): + self.assertIsInstance(self.cls.parser, _PathParser) + + def test_constructor(self): + P = self.cls + p = P('a') + self.assertIsInstance(p, P) + P() + P('a', 'b', 'c') + P('/a', 'b', 'c') + P('a/b/c') + P('/a/b/c') + + def test_with_segments(self): + class P(self.cls): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + p = P('foo', 'bar', session_id=42) + self.assertEqual(42, (p / 'foo').session_id) + self.assertEqual(42, ('foo' / p).session_id) + self.assertEqual(42, p.joinpath('foo').session_id) + self.assertEqual(42, p.with_name('foo').session_id) + self.assertEqual(42, p.with_stem('foo').session_id) + self.assertEqual(42, p.with_suffix('.foo').session_id) + self.assertEqual(42, p.with_segments('foo').session_id) + self.assertEqual(42, p.parent.session_id) + for parent in p.parents: + self.assertEqual(42, parent.session_id) + + def test_join(self): + P = self.cls + sep = self.cls.parser.sep + p = P(f'a{sep}b') + pp = p.joinpath('c') + self.assertEqual(pp, P(f'a{sep}b{sep}c')) + self.assertIs(type(pp), type(p)) + pp = p.joinpath('c', 'd') + self.assertEqual(pp, P(f'a{sep}b{sep}c{sep}d')) + pp = p.joinpath(f'{sep}c') + self.assertEqual(pp, P(f'{sep}c')) + + def test_div(self): + # Basically the same as joinpath(). + P = self.cls + sep = self.cls.parser.sep + p = P(f'a{sep}b') + pp = p / 'c' + self.assertEqual(pp, P(f'a{sep}b{sep}c')) + self.assertIs(type(pp), type(p)) + pp = p / f'c{sep}d' + self.assertEqual(pp, P(f'a{sep}b{sep}c{sep}d')) + pp = p / 'c' / 'd' + self.assertEqual(pp, P(f'a{sep}b{sep}c{sep}d')) + pp = 'c' / p / 'd' + self.assertEqual(pp, P(f'c{sep}a{sep}b{sep}d')) + pp = p/ f'{sep}c' + self.assertEqual(pp, P(f'{sep}c')) + + def test_full_match(self): + P = self.cls + # Simple relative pattern. + self.assertTrue(P('b.py').full_match('b.py')) + self.assertFalse(P('a/b.py').full_match('b.py')) + self.assertFalse(P('/a/b.py').full_match('b.py')) + self.assertFalse(P('a.py').full_match('b.py')) + self.assertFalse(P('b/py').full_match('b.py')) + self.assertFalse(P('/a.py').full_match('b.py')) + self.assertFalse(P('b.py/c').full_match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').full_match('*.py')) + self.assertFalse(P('a/b.py').full_match('*.py')) + self.assertFalse(P('/a/b.py').full_match('*.py')) + self.assertFalse(P('b.pyc').full_match('*.py')) + self.assertFalse(P('b./py').full_match('*.py')) + self.assertFalse(P('b.py/c').full_match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('/d/ab/c.py').full_match('a*/*.py')) + self.assertFalse(P('a.py').full_match('a*/*.py')) + self.assertFalse(P('/dab/c.py').full_match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').full_match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').full_match('/*.py')) + self.assertFalse(P('b.py').full_match('/*.py')) + self.assertFalse(P('a/b.py').full_match('/*.py')) + self.assertFalse(P('/a/b.py').full_match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').full_match('/a/*.py')) + self.assertFalse(P('/ab.py').full_match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').full_match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertTrue(P('a').full_match('**')) + self.assertTrue(P('c.py').full_match('**')) + self.assertTrue(P('a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('**')) + self.assertTrue(P('/a/b/c.py').full_match('/**')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**')) + self.assertTrue(P('/a/b/c.py').full_match('**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/a/b/**/*.py')) + self.assertTrue(P('/a/b/c.py').full_match('/**/**/**/**/*.py')) + self.assertFalse(P('c.py').full_match('**/a.py')) + self.assertFalse(P('c.py').full_match('c/**')) + self.assertFalse(P('a/b/c.py').full_match('**/a')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c.')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('**/a/b/c./**')) + self.assertFalse(P('a/b/c.py').full_match('/a/b/c.py/**')) + self.assertFalse(P('a/b/c.py').full_match('/**/a/b/c.py')) + # Matching against empty path + self.assertFalse(P('').full_match('*')) + self.assertTrue(P('').full_match('**')) + self.assertFalse(P('').full_match('**/*')) + # Matching with empty pattern + self.assertTrue(P('').full_match('')) + self.assertTrue(P('.').full_match('.')) + self.assertFalse(P('/').full_match('')) + self.assertFalse(P('/').full_match('.')) + self.assertFalse(P('foo').full_match('')) + self.assertFalse(P('foo').full_match('.')) + + def test_parts(self): + # `parts` returns a tuple. + sep = self.cls.parser.sep + P = self.cls + p = P(f'a{sep}b') + parts = p.parts + self.assertEqual(parts, ('a', 'b')) + # When the path is absolute, the anchor is a separate part. + p = P(f'{sep}a{sep}b') + parts = p.parts + self.assertEqual(parts, (sep, 'a', 'b')) + + @threading_helper.requires_working_threading() + def test_parts_multithreaded(self): + P = self.cls + + NUM_THREADS = 10 + NUM_ITERS = 10 + + for _ in range(NUM_ITERS): + b = threading.Barrier(NUM_THREADS) + path = P('a') / 'b' / 'c' / 'd' / 'e' + expected = ('a', 'b', 'c', 'd', 'e') + + def check_parts(): + b.wait() + self.assertEqual(path.parts, expected) + + threads = [threading.Thread(target=check_parts) for _ in range(NUM_THREADS)] + with threading_helper.start_threads(threads): + pass + + def test_parent(self): + # Relative + P = self.cls + p = P('a/b/c') + self.assertEqual(p.parent, P('a/b')) + self.assertEqual(p.parent.parent, P('a')) + self.assertEqual(p.parent.parent.parent, P('')) + self.assertEqual(p.parent.parent.parent.parent, P('')) + # Anchored + p = P('/a/b/c') + self.assertEqual(p.parent, P('/a/b')) + self.assertEqual(p.parent.parent, P('/a')) + self.assertEqual(p.parent.parent.parent, P('/')) + self.assertEqual(p.parent.parent.parent.parent, P('/')) + + def test_parents(self): + # Relative + P = self.cls + p = P('a/b/c') + par = p.parents + self.assertEqual(len(par), 3) + self.assertEqual(par[0], P('a/b')) + self.assertEqual(par[1], P('a')) + self.assertEqual(par[2], P('')) + self.assertEqual(par[-1], P('')) + self.assertEqual(par[-2], P('a')) + self.assertEqual(par[-3], P('a/b')) + self.assertEqual(par[0:1], (P('a/b'),)) + self.assertEqual(par[:2], (P('a/b'), P('a'))) + self.assertEqual(par[:-1], (P('a/b'), P('a'))) + self.assertEqual(par[1:], (P('a'), P(''))) + self.assertEqual(par[::2], (P('a/b'), P(''))) + self.assertEqual(par[::-1], (P(''), P('a'), P('a/b'))) + self.assertEqual(list(par), [P('a/b'), P('a'), P('')]) + with self.assertRaises(IndexError): + par[-4] + with self.assertRaises(IndexError): + par[3] + with self.assertRaises(TypeError): + par[0] = p + # Anchored + p = P('/a/b/c') + par = p.parents + self.assertEqual(len(par), 3) + self.assertEqual(par[0], P('/a/b')) + self.assertEqual(par[1], P('/a')) + self.assertEqual(par[2], P('/')) + self.assertEqual(par[-1], P('/')) + self.assertEqual(par[-2], P('/a')) + self.assertEqual(par[-3], P('/a/b')) + self.assertEqual(par[0:1], (P('/a/b'),)) + self.assertEqual(par[:2], (P('/a/b'), P('/a'))) + self.assertEqual(par[:-1], (P('/a/b'), P('/a'))) + self.assertEqual(par[1:], (P('/a'), P('/'))) + self.assertEqual(par[::2], (P('/a/b'), P('/'))) + self.assertEqual(par[::-1], (P('/'), P('/a'), P('/a/b'))) + self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')]) + with self.assertRaises(IndexError): + par[-4] + with self.assertRaises(IndexError): + par[3] + + def test_anchor(self): + P = self.cls + sep = self.cls.parser.sep + self.assertEqual(P('').anchor, '') + self.assertEqual(P(f'a{sep}b').anchor, '') + self.assertEqual(P(sep).anchor, sep) + self.assertEqual(P(f'{sep}a{sep}b').anchor, sep) + + def test_name(self): + P = self.cls + self.assertEqual(P('').name, '') + self.assertEqual(P('/').name, '') + self.assertEqual(P('a/b').name, 'b') + self.assertEqual(P('/a/b').name, 'b') + self.assertEqual(P('a/b.py').name, 'b.py') + self.assertEqual(P('/a/b.py').name, 'b.py') + + def test_suffix(self): + P = self.cls + self.assertEqual(P('').suffix, '') + self.assertEqual(P('.').suffix, '') + self.assertEqual(P('..').suffix, '') + self.assertEqual(P('/').suffix, '') + self.assertEqual(P('a/b').suffix, '') + self.assertEqual(P('/a/b').suffix, '') + self.assertEqual(P('/a/b/.').suffix, '') + self.assertEqual(P('a/b.py').suffix, '.py') + self.assertEqual(P('/a/b.py').suffix, '.py') + self.assertEqual(P('a/.hgrc').suffix, '') + self.assertEqual(P('/a/.hgrc').suffix, '') + self.assertEqual(P('a/.hg.rc').suffix, '.rc') + self.assertEqual(P('/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('a/trailing.dot.').suffix, '.') + self.assertEqual(P('/a/trailing.dot.').suffix, '.') + self.assertEqual(P('a/..d.o.t..').suffix, '.') + self.assertEqual(P('a/inn.er..dots').suffix, '.dots') + self.assertEqual(P('photo').suffix, '') + self.assertEqual(P('photo.jpg').suffix, '.jpg') + + def test_suffixes(self): + P = self.cls + self.assertEqual(P('').suffixes, []) + self.assertEqual(P('.').suffixes, []) + self.assertEqual(P('/').suffixes, []) + self.assertEqual(P('a/b').suffixes, []) + self.assertEqual(P('/a/b').suffixes, []) + self.assertEqual(P('/a/b/.').suffixes, []) + self.assertEqual(P('a/b.py').suffixes, ['.py']) + self.assertEqual(P('/a/b.py').suffixes, ['.py']) + self.assertEqual(P('a/.hgrc').suffixes, []) + self.assertEqual(P('/a/.hgrc').suffixes, []) + self.assertEqual(P('a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('/a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('a/..d.o.t..').suffixes, ['.o', '.t', '.', '.']) + self.assertEqual(P('a/inn.er..dots').suffixes, ['.er', '.', '.dots']) + self.assertEqual(P('photo').suffixes, []) + self.assertEqual(P('photo.jpg').suffixes, ['.jpg']) + + def test_stem(self): + P = self.cls + self.assertEqual(P('..').stem, '..') + self.assertEqual(P('').stem, '') + self.assertEqual(P('/').stem, '') + self.assertEqual(P('a/b').stem, 'b') + self.assertEqual(P('a/b.py').stem, 'b') + self.assertEqual(P('a/.hgrc').stem, '.hgrc') + self.assertEqual(P('a/.hg.rc').stem, '.hg') + self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('a/trailing.dot.').stem, 'trailing.dot') + self.assertEqual(P('a/..d.o.t..').stem, '..d.o.t.') + self.assertEqual(P('a/inn.er..dots').stem, 'inn.er.') + self.assertEqual(P('photo').stem, 'photo') + self.assertEqual(P('photo.jpg').stem, 'photo') + + def test_with_name(self): + P = self.cls + self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml')) + self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml')) + self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml')) + self.assertRaises(ValueError, P('a/b').with_name, '/c') + self.assertRaises(ValueError, P('a/b').with_name, 'c/') + self.assertRaises(ValueError, P('a/b').with_name, 'c/d') + + def test_with_stem(self): + P = self.cls + self.assertEqual(P('a/b').with_stem('d'), P('a/d')) + self.assertEqual(P('/a/b').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) + self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) + self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d.')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d.')) + self.assertRaises(ValueError, P('foo.gz').with_stem, '') + self.assertRaises(ValueError, P('/a/b/foo.gz').with_stem, '') + self.assertRaises(ValueError, P('a/b').with_stem, '/c') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/') + self.assertRaises(ValueError, P('a/b').with_stem, 'c/d') + + def test_with_suffix(self): + P = self.cls + self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz')) + self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz')) + self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz')) + # Stripping suffix. + self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) + self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Single dot + self.assertEqual(P('a/b').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b').with_suffix('.'), P('/a/b.')) + self.assertEqual(P('a/b.py').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b.py').with_suffix('.'), P('/a/b.')) + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('a/b').with_suffix, '/') + self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') + self.assertRaises(ValueError, P('a/b').with_suffix, './.d') + self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.') + self.assertRaises(TypeError, P('a/b').with_suffix, None) + + +class LexicalPathJoinTest(JoinTestBase, unittest.TestCase): + cls = LexicalPath + + +if not is_pypi: + from pathlib import PurePath, Path + + class PurePathJoinTest(JoinTestBase, unittest.TestCase): + cls = PurePath + + class PathJoinTest(JoinTestBase, unittest.TestCase): + cls = Path + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_join_posix.py b/Lib/test/test_pathlib/test_join_posix.py new file mode 100644 index 00000000000..d24fb1087c9 --- /dev/null +++ b/Lib/test/test_pathlib/test_join_posix.py @@ -0,0 +1,51 @@ +""" +Tests for Posix-flavoured pathlib.types._JoinablePath +""" + +import os +import unittest + +from .support import is_pypi +from .support.lexical_path import LexicalPosixPath + + +class JoinTestBase: + def test_join(self): + P = self.cls + p = P('//a') + pp = p.joinpath('b') + self.assertEqual(pp, P('//a/b')) + pp = P('/a').joinpath('//c') + self.assertEqual(pp, P('//c')) + pp = P('//a').joinpath('/c') + self.assertEqual(pp, P('/c')) + + def test_div(self): + # Basically the same as joinpath(). + P = self.cls + p = P('//a') + pp = p / 'b' + self.assertEqual(pp, P('//a/b')) + pp = P('/a') / '//c' + self.assertEqual(pp, P('//c')) + pp = P('//a') / '/c' + self.assertEqual(pp, P('/c')) + + +class LexicalPosixPathJoinTest(JoinTestBase, unittest.TestCase): + cls = LexicalPosixPath + + +if not is_pypi: + from pathlib import PurePosixPath, PosixPath + + class PurePosixPathJoinTest(JoinTestBase, unittest.TestCase): + cls = PurePosixPath + + if os.name != 'nt': + class PosixPathJoinTest(JoinTestBase, unittest.TestCase): + cls = PosixPath + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_join_windows.py b/Lib/test/test_pathlib/test_join_windows.py new file mode 100644 index 00000000000..2cc634f25ef --- /dev/null +++ b/Lib/test/test_pathlib/test_join_windows.py @@ -0,0 +1,290 @@ +""" +Tests for Windows-flavoured pathlib.types._JoinablePath +""" + +import os +import unittest + +from .support import is_pypi +from .support.lexical_path import LexicalWindowsPath + + +class JoinTestBase: + def test_join(self): + P = self.cls + p = P('C:/a/b') + pp = p.joinpath('x/y') + self.assertEqual(pp, P(r'C:/a/b\x/y')) + pp = p.joinpath('/x/y') + self.assertEqual(pp, P('C:/x/y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + pp = p.joinpath('D:x/y') + self.assertEqual(pp, P('D:x/y')) + pp = p.joinpath('D:/x/y') + self.assertEqual(pp, P('D:/x/y')) + pp = p.joinpath('//host/share/x/y') + self.assertEqual(pp, P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + pp = p.joinpath('c:x/y') + self.assertEqual(pp, P(r'c:/a/b\x/y')) + pp = p.joinpath('c:/x/y') + self.assertEqual(pp, P('c:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath('./d:s') + self.assertEqual(pp, P(r'C:/a/b\./d:s')) + pp = p.joinpath('./dd:s') + self.assertEqual(pp, P(r'C:/a/b\./dd:s')) + pp = p.joinpath('E:d:s') + self.assertEqual(pp, P('E:d:s')) + # Joining onto a UNC path with no root + pp = P('//server').joinpath('share') + self.assertEqual(pp, P(r'//server\share')) + pp = P('//./BootPartition').joinpath('Windows') + self.assertEqual(pp, P(r'//./BootPartition\Windows')) + + def test_div(self): + # Basically the same as joinpath(). + P = self.cls + p = P('C:/a/b') + self.assertEqual(p / 'x/y', P(r'C:/a/b\x/y')) + self.assertEqual(p / 'x' / 'y', P(r'C:/a/b\x\y')) + self.assertEqual(p / '/x/y', P('C:/x/y')) + self.assertEqual(p / '/x' / 'y', P(r'C:/x\y')) + # Joining with a different drive => the first path is ignored, even + # if the second path is relative. + self.assertEqual(p / 'D:x/y', P('D:x/y')) + self.assertEqual(p / 'D:' / 'x/y', P('D:x/y')) + self.assertEqual(p / 'D:/x/y', P('D:/x/y')) + self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y')) + self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + self.assertEqual(p / 'c:x/y', P(r'c:/a/b\x/y')) + self.assertEqual(p / 'c:/x/y', P('c:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / './d:s', P(r'C:/a/b\./d:s')) + self.assertEqual(p / './dd:s', P(r'C:/a/b\./dd:s')) + self.assertEqual(p / 'E:d:s', P('E:d:s')) + + def test_str(self): + p = self.cls(r'a\b\c') + self.assertEqual(str(p), 'a\\b\\c') + p = self.cls(r'c:\a\b\c') + self.assertEqual(str(p), 'c:\\a\\b\\c') + p = self.cls('\\\\a\\b\\') + self.assertEqual(str(p), '\\\\a\\b\\') + p = self.cls(r'\\a\b\c') + self.assertEqual(str(p), '\\\\a\\b\\c') + p = self.cls(r'\\a\b\c\d') + self.assertEqual(str(p), '\\\\a\\b\\c\\d') + + def test_parts(self): + P = self.cls + p = P(r'c:a\b') + parts = p.parts + self.assertEqual(parts, ('c:', 'a', 'b')) + p = P(r'c:\a\b') + parts = p.parts + self.assertEqual(parts, ('c:\\', 'a', 'b')) + p = P(r'\\a\b\c\d') + parts = p.parts + self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd')) + + def test_parent(self): + # Anchored + P = self.cls + p = P('z:a/b/c') + self.assertEqual(p.parent, P('z:a/b')) + self.assertEqual(p.parent.parent, P('z:a')) + self.assertEqual(p.parent.parent.parent, P('z:')) + self.assertEqual(p.parent.parent.parent.parent, P('z:')) + p = P('z:/a/b/c') + self.assertEqual(p.parent, P('z:/a/b')) + self.assertEqual(p.parent.parent, P('z:/a')) + self.assertEqual(p.parent.parent.parent, P('z:/')) + self.assertEqual(p.parent.parent.parent.parent, P('z:/')) + p = P('//a/b/c/d') + self.assertEqual(p.parent, P('//a/b/c')) + self.assertEqual(p.parent.parent, P('//a/b/')) + self.assertEqual(p.parent.parent.parent, P('//a/b/')) + + def test_parents(self): + # Anchored + P = self.cls + p = P('z:a/b') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:a')) + self.assertEqual(par[1], P('z:')) + self.assertEqual(par[0:1], (P('z:a'),)) + self.assertEqual(par[:-1], (P('z:a'),)) + self.assertEqual(par[:2], (P('z:a'), P('z:'))) + self.assertEqual(par[1:], (P('z:'),)) + self.assertEqual(par[::2], (P('z:a'),)) + self.assertEqual(par[::-1], (P('z:'), P('z:a'))) + self.assertEqual(list(par), [P('z:a'), P('z:')]) + with self.assertRaises(IndexError): + par[2] + p = P('z:/a/b') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('z:/a')) + self.assertEqual(par[1], P('z:/')) + self.assertEqual(par[0:1], (P('z:/a'),)) + self.assertEqual(par[0:-1], (P('z:/a'),)) + self.assertEqual(par[:2], (P('z:/a'), P('z:/'))) + self.assertEqual(par[1:], (P('z:/'),)) + self.assertEqual(par[::2], (P('z:/a'),)) + self.assertEqual(par[::-1], (P('z:/'), P('z:/a'),)) + self.assertEqual(list(par), [P('z:/a'), P('z:/')]) + with self.assertRaises(IndexError): + par[2] + p = P('//a/b/c/d') + par = p.parents + self.assertEqual(len(par), 2) + self.assertEqual(par[0], P('//a/b/c')) + self.assertEqual(par[1], P('//a/b/')) + self.assertEqual(par[0:1], (P('//a/b/c'),)) + self.assertEqual(par[0:-1], (P('//a/b/c'),)) + self.assertEqual(par[:2], (P('//a/b/c'), P('//a/b/'))) + self.assertEqual(par[1:], (P('//a/b/'),)) + self.assertEqual(par[::2], (P('//a/b/c'),)) + self.assertEqual(par[::-1], (P('//a/b/'), P('//a/b/c'))) + self.assertEqual(list(par), [P('//a/b/c'), P('//a/b/')]) + with self.assertRaises(IndexError): + par[2] + + def test_anchor(self): + P = self.cls + self.assertEqual(P('c:').anchor, 'c:') + self.assertEqual(P('c:a/b').anchor, 'c:') + self.assertEqual(P('c:\\').anchor, 'c:\\') + self.assertEqual(P('c:\\a\\b\\').anchor, 'c:\\') + self.assertEqual(P('\\\\a\\b\\').anchor, '\\\\a\\b\\') + self.assertEqual(P('\\\\a\\b\\c\\d').anchor, '\\\\a\\b\\') + + def test_name(self): + P = self.cls + self.assertEqual(P('c:').name, '') + self.assertEqual(P('c:/').name, '') + self.assertEqual(P('c:a/b').name, 'b') + self.assertEqual(P('c:/a/b').name, 'b') + self.assertEqual(P('c:a/b.py').name, 'b.py') + self.assertEqual(P('c:/a/b.py').name, 'b.py') + self.assertEqual(P('//My.py/Share.php').name, '') + self.assertEqual(P('//My.py/Share.php/a/b').name, 'b') + + def test_stem(self): + P = self.cls + self.assertEqual(P('c:').stem, '') + self.assertEqual(P('c:..').stem, '..') + self.assertEqual(P('c:/').stem, '') + self.assertEqual(P('c:a/b').stem, 'b') + self.assertEqual(P('c:a/b.py').stem, 'b') + self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') + self.assertEqual(P('c:a/.hg.rc').stem, '.hg') + self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') + self.assertEqual(P('c:a/trailing.dot.').stem, 'trailing.dot') + + def test_suffix(self): + P = self.cls + self.assertEqual(P('c:').suffix, '') + self.assertEqual(P('c:/').suffix, '') + self.assertEqual(P('c:a/b').suffix, '') + self.assertEqual(P('c:/a/b').suffix, '') + self.assertEqual(P('c:a/b.py').suffix, '.py') + self.assertEqual(P('c:/a/b.py').suffix, '.py') + self.assertEqual(P('c:a/.hgrc').suffix, '') + self.assertEqual(P('c:/a/.hgrc').suffix, '') + self.assertEqual(P('c:a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') + self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') + self.assertEqual(P('c:a/trailing.dot.').suffix, '.') + self.assertEqual(P('c:/a/trailing.dot.').suffix, '.') + self.assertEqual(P('//My.py/Share.php').suffix, '') + self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') + + def test_suffixes(self): + P = self.cls + self.assertEqual(P('c:').suffixes, []) + self.assertEqual(P('c:/').suffixes, []) + self.assertEqual(P('c:a/b').suffixes, []) + self.assertEqual(P('c:/a/b').suffixes, []) + self.assertEqual(P('c:a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:/a/b.py').suffixes, ['.py']) + self.assertEqual(P('c:a/.hgrc').suffixes, []) + self.assertEqual(P('c:/a/.hgrc').suffixes, []) + self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc']) + self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) + self.assertEqual(P('//My.py/Share.php').suffixes, []) + self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) + self.assertEqual(P('c:a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('c:/a/trailing.dot.').suffixes, ['.dot', '.']) + + def test_with_name(self): + P = self.cls + self.assertEqual(P(r'c:a\b').with_name('d.xml'), P(r'c:a\d.xml')) + self.assertEqual(P(r'c:\a\b').with_name('d.xml'), P(r'c:\a\d.xml')) + self.assertEqual(P(r'c:a\Dot ending.').with_name('d.xml'), P(r'c:a\d.xml')) + self.assertEqual(P(r'c:\a\Dot ending.').with_name('d.xml'), P(r'c:\a\d.xml')) + self.assertRaises(ValueError, P(r'c:a\b').with_name, r'd:\e') + self.assertRaises(ValueError, P(r'c:a\b').with_name, r'\\My\Share') + + def test_with_stem(self): + P = self.cls + self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) + self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d.')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d.')) + self.assertRaises(ValueError, P('c:a/b').with_stem, 'd:/e') + self.assertRaises(ValueError, P('c:a/b').with_stem, '//My/Share') + + def test_with_suffix(self): + P = self.cls + self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz')) + self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz')) + self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz')) + # Path doesn't have a "filename" component. + self.assertRaises(ValueError, P('').with_suffix, '.gz') + self.assertRaises(ValueError, P('/').with_suffix, '.gz') + self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz') + # Invalid suffix. + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d') + self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d') + self.assertRaises(TypeError, P('c:a/b').with_suffix, None) + + +class LexicalWindowsPathJoinTest(JoinTestBase, unittest.TestCase): + cls = LexicalWindowsPath + + +if not is_pypi: + from pathlib import PureWindowsPath, WindowsPath + + class PureWindowsPathJoinTest(JoinTestBase, unittest.TestCase): + cls = PureWindowsPath + + if os.name == 'nt': + class WindowsPathJoinTest(JoinTestBase, unittest.TestCase): + cls = WindowsPath + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py new file mode 100644 index 00000000000..a1ea69a6b90 --- /dev/null +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -0,0 +1,3687 @@ +import collections +import contextlib +import io +import os +import sys +import errno +import ntpath +import pathlib +import pickle +import posixpath +import socket +import stat +import tempfile +import unittest +from unittest import mock +from urllib.request import pathname2url + +from test.support import import_helper +from test.support import cpython_only +from test.support import is_emscripten, is_wasi, is_wasm32 +from test.support import infinite_recursion +from test.support import os_helper +from test.support.os_helper import TESTFN, FS_NONASCII, FakePath +try: + import fcntl +except ImportError: + fcntl = None +try: + import grp, pwd +except ImportError: + grp = pwd = None +try: + import posix +except ImportError: + posix = None + + +root_in_posix = False +if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + + +def patch_replace(old_test): + def new_replace(self, target): + raise OSError(errno.EXDEV, "Cross-device link", self, target) + + def new_test(self): + old_replace = self.cls.replace + self.cls.replace = new_replace + try: + old_test(self) + finally: + self.cls.replace = old_replace + return new_test + + +_tests_needing_posix = set() +_tests_needing_windows = set() +_tests_needing_symlinks = set() + +def needs_posix(fn): + """Decorator that marks a test as requiring a POSIX-flavoured path class.""" + _tests_needing_posix.add(fn.__name__) + return fn + +def needs_windows(fn): + """Decorator that marks a test as requiring a Windows-flavoured path class.""" + _tests_needing_windows.add(fn.__name__) + return fn + +def needs_symlinks(fn): + """Decorator that marks a test as requiring a path class that supports symlinks.""" + _tests_needing_symlinks.add(fn.__name__) + return fn + + + +class UnsupportedOperationTest(unittest.TestCase): + def test_is_notimplemented(self): + self.assertIsSubclass(pathlib.UnsupportedOperation, NotImplementedError) + self.assertIsInstance(pathlib.UnsupportedOperation(), NotImplementedError) + + +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports("pathlib", {"shutil"}) + + +# +# Tests for the pure classes. +# + +class PurePathTest(unittest.TestCase): + cls = pathlib.PurePath + + # Make sure any symbolic links in the base test path are resolved. + base = os.path.realpath(TESTFN) + + # Keys are canonical paths, values are list of tuples of arguments + # supposed to produce equal paths. + equivalences = { + 'a/b': [ + ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'), + ('a/b/',), ('a//b',), ('a//b//',), + # Empty components get removed. + ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''), + ], + '/b/c/d': [ + ('a', '/b/c', 'd'), ('/a', '/b/c', 'd'), + # Empty components get removed. + ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'), + ], + } + + def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_posix and self.cls.parser is not posixpath: + self.skipTest('requires POSIX-flavoured path class') + if name in _tests_needing_windows and self.cls.parser is posixpath: + self.skipTest('requires Windows-flavoured path class') + p = self.cls('a') + self.parser = p.parser + self.sep = self.parser.sep + self.altsep = self.parser.altsep + + def _check_str_subclass(self, *args): + # Issue #21127: it should be possible to construct a PurePath object + # from a str subclass instance, and it then gets converted to + # a pure str object. + class StrSubclass(str): + pass + P = self.cls + p = P(*(StrSubclass(x) for x in args)) + self.assertEqual(p, P(*args)) + for part in p.parts: + self.assertIs(type(part), str) + + def test_str_subclass_common(self): + self._check_str_subclass('') + self._check_str_subclass('.') + self._check_str_subclass('a') + self._check_str_subclass('a/b.txt') + self._check_str_subclass('/a/b.txt') + + @needs_windows + def test_str_subclass_windows(self): + self._check_str_subclass('.\\a:b') + self._check_str_subclass('c:') + self._check_str_subclass('c:a') + self._check_str_subclass('c:a\\b.txt') + self._check_str_subclass('c:\\') + self._check_str_subclass('c:\\a') + self._check_str_subclass('c:\\a\\b.txt') + self._check_str_subclass('\\\\some\\share') + self._check_str_subclass('\\\\some\\share\\a') + self._check_str_subclass('\\\\some\\share\\a\\b.txt') + + def _check_str(self, expected, args): + p = self.cls(*args) + self.assertEqual(str(p), expected.replace('/', self.sep)) + + def test_str_common(self): + # Canonicalized paths roundtrip. + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self._check_str(pathstr, (pathstr,)) + # Other tests for str() are in test_equivalences(). + + @needs_windows + def test_str_windows(self): + p = self.cls('a/b/c') + self.assertEqual(str(p), 'a\\b\\c') + p = self.cls('c:/a/b/c') + self.assertEqual(str(p), 'c:\\a\\b\\c') + p = self.cls('//a/b') + self.assertEqual(str(p), '\\\\a\\b\\') + p = self.cls('//a/b/c') + self.assertEqual(str(p), '\\\\a\\b\\c') + p = self.cls('//a/b/c/d') + self.assertEqual(str(p), '\\\\a\\b\\c\\d') + + def test_concrete_class(self): + if self.cls is pathlib.PurePath: + expected = pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_concrete_parser(self): + if self.cls is pathlib.PurePosixPath: + expected = posixpath + elif self.cls is pathlib.PureWindowsPath: + expected = ntpath + else: + expected = os.path + p = self.cls('a') + self.assertIs(p.parser, expected) + + def test_different_parsers_unequal(self): + p = self.cls('a') + if p.parser is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + self.assertNotEqual(p, q) + + def test_different_parsers_unordered(self): + p = self.cls('a') + if p.parser is posixpath: + q = pathlib.PureWindowsPath('a') + else: + q = pathlib.PurePosixPath('a') + with self.assertRaises(TypeError): + p < q + with self.assertRaises(TypeError): + p <= q + with self.assertRaises(TypeError): + p > q + with self.assertRaises(TypeError): + p >= q + + def test_constructor_nested(self): + P = self.cls + P(FakePath("a/b/c")) + self.assertEqual(P(P('a')), P('a')) + self.assertEqual(P(P('a'), 'b'), P('a/b')) + self.assertEqual(P(P('a'), P('b')), P('a/b')) + self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + self.assertEqual(P(P('./a:b')), P('./a:b')) + + @needs_windows + def test_constructor_nested_foreign_flavour(self): + # See GH-125069. + p1 = pathlib.PurePosixPath('b/c:\\d') + p2 = pathlib.PurePosixPath('b/', 'c:\\d') + self.assertEqual(p1, p2) + self.assertEqual(self.cls(p1), self.cls('b/c:/d')) + self.assertEqual(self.cls(p2), self.cls('b/c:/d')) + + def _check_parse_path(self, raw_path, *expected): + sep = self.parser.sep + actual = self.cls._parse_path(raw_path.replace('/', sep)) + self.assertEqual(actual, expected) + if altsep := self.parser.altsep: + actual = self.cls._parse_path(raw_path.replace('/', altsep)) + self.assertEqual(actual, expected) + + def test_parse_path_common(self): + check = self._check_parse_path + sep = self.parser.sep + check('', '', '', []) + check('a', '', '', ['a']) + check('a/', '', '', ['a']) + check('a/b', '', '', ['a', 'b']) + check('a/b/', '', '', ['a', 'b']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b//c/d', '', '', ['a', 'b', 'c', 'd']) + check('a/b/c/d', '', '', ['a', 'b', 'c', 'd']) + check('.', '', '', []) + check('././b', '', '', ['b']) + check('a/./b', '', '', ['a', 'b']) + check('a/./.', '', '', ['a']) + check('/a/b', '', sep, ['a', 'b']) + + def test_empty_path(self): + # The empty path points to '.' + p = self.cls('') + self.assertEqual(str(p), '.') + # Special case for the empty path. + self._check_str('.', ('',)) + + def test_join_nested(self): + P = self.cls + p = P('a/b').joinpath(P('c')) + self.assertEqual(p, P('a/b/c')) + + def test_div_nested(self): + P = self.cls + p = P('a/b') / P('c') + self.assertEqual(p, P('a/b/c')) + + def test_pickling_common(self): + P = self.cls + for pathstr in ('a', 'a/', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c', 'a/b/c/'): + with self.subTest(pathstr=pathstr): + p = P(pathstr) + for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): + dumped = pickle.dumps(p, proto) + pp = pickle.loads(dumped) + self.assertIs(pp.__class__, p.__class__) + self.assertEqual(pp, p) + self.assertEqual(hash(pp), hash(p)) + self.assertEqual(str(pp), str(p)) + + def test_unpicking_3_13(self): + data = (b"\x80\x04\x95'\x00\x00\x00\x00\x00\x00\x00\x8c\x0e" + b"pathlib._local\x94\x8c\rPurePosixPath\x94\x93\x94)R\x94.") + p = pickle.loads(data) + self.assertIsInstance(p, pathlib.PurePosixPath) + + def test_repr_common(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + clsname = p.__class__.__name__ + r = repr(p) + # The repr() is in the form ClassName("forward-slashes path"). + self.assertStartsWith(r, clsname + '(') + self.assertEndsWith(r, ')') + inner = r[len(clsname) + 1 : -1] + self.assertEqual(eval(inner), p.as_posix()) + + def test_fspath_common(self): + P = self.cls + p = P('a/b') + self._check_str(p.__fspath__(), ('a/b',)) + self._check_str(os.fspath(p), ('a/b',)) + + def test_bytes(self): + P = self.cls + with self.assertRaises(TypeError): + P(b'a') + with self.assertRaises(TypeError): + P(b'a', 'b') + with self.assertRaises(TypeError): + P('a', b'b') + with self.assertRaises(TypeError): + P('a').joinpath(b'b') + with self.assertRaises(TypeError): + P('a') / b'b' + with self.assertRaises(TypeError): + b'a' / P('b') + with self.assertRaises(TypeError): + P('a').match(b'b') + with self.assertRaises(TypeError): + P('a').relative_to(b'b') + with self.assertRaises(TypeError): + P('a').with_name(b'b') + with self.assertRaises(TypeError): + P('a').with_stem(b'b') + with self.assertRaises(TypeError): + P('a').with_suffix(b'b') + + def test_bytes_exc_message(self): + P = self.cls + message = (r"argument should be a str or an os\.PathLike object " + r"where __fspath__ returns a str, not 'bytes'") + with self.assertRaisesRegex(TypeError, message): + P(b'a') + with self.assertRaisesRegex(TypeError, message): + P(b'a', 'b') + with self.assertRaisesRegex(TypeError, message): + P('a', b'b') + + def test_as_bytes_common(self): + sep = os.fsencode(self.sep) + P = self.cls + self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b') + + def test_as_posix_common(self): + P = self.cls + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + self.assertEqual(P(pathstr).as_posix(), pathstr) + # Other tests for as_posix() are in test_equivalences(). + + def test_eq_common(self): + P = self.cls + self.assertEqual(P('a/b'), P('a/b')) + self.assertEqual(P('a/b'), P('a', 'b')) + self.assertNotEqual(P('a/b'), P('a')) + self.assertNotEqual(P('a/b'), P('/a/b')) + self.assertNotEqual(P('a/b'), P()) + self.assertNotEqual(P('/a/b'), P('/')) + self.assertNotEqual(P(), P('/')) + self.assertNotEqual(P(), "") + self.assertNotEqual(P(), {}) + self.assertNotEqual(P(), int) + + def test_equivalences(self, equivalences=None): + if equivalences is None: + equivalences = self.equivalences + for k, tuples in equivalences.items(): + canon = k.replace('/', self.sep) + posix = k.replace(self.sep, '/') + if canon != posix: + tuples = tuples + [ + tuple(part.replace('/', self.sep) for part in t) + for t in tuples + ] + tuples.append((posix, )) + pcanon = self.cls(canon) + for t in tuples: + p = self.cls(*t) + self.assertEqual(p, pcanon, "failed with args {}".format(t)) + self.assertEqual(hash(p), hash(pcanon)) + self.assertEqual(str(p), canon) + self.assertEqual(p.as_posix(), posix) + + def test_ordering_common(self): + # Ordering is tuple-alike. + def assertLess(a, b): + self.assertLess(a, b) + self.assertGreater(b, a) + P = self.cls + a = P('a') + b = P('a/b') + c = P('abc') + d = P('b') + assertLess(a, b) + assertLess(a, c) + assertLess(a, d) + assertLess(b, c) + assertLess(c, d) + P = self.cls + a = P('/a') + b = P('/a/b') + c = P('/abc') + d = P('/b') + assertLess(a, b) + assertLess(a, c) + assertLess(a, d) + assertLess(b, c) + assertLess(c, d) + with self.assertRaises(TypeError): + P() < {} + + def make_uri(self, path): + if isinstance(path, pathlib.Path): + return path.as_uri() + with self.assertWarns(DeprecationWarning): + return path.as_uri() + + def test_as_uri_common(self): + P = self.cls + with self.assertRaises(ValueError): + self.make_uri(P('a')) + with self.assertRaises(ValueError): + self.make_uri(P()) + + def test_repr_roundtrips(self): + for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): + with self.subTest(pathstr=pathstr): + p = self.cls(pathstr) + r = repr(p) + # The repr() roundtrips. + q = eval(r, pathlib.__dict__) + self.assertIs(q.__class__, p.__class__) + self.assertEqual(q, p) + self.assertEqual(repr(q), r) + + def test_drive_common(self): + P = self.cls + self.assertEqual(P('a/b').drive, '') + self.assertEqual(P('/a/b').drive, '') + self.assertEqual(P('').drive, '') + + @needs_windows + def test_drive_windows(self): + P = self.cls + self.assertEqual(P('c:').drive, 'c:') + self.assertEqual(P('c:a/b').drive, 'c:') + self.assertEqual(P('c:/').drive, 'c:') + self.assertEqual(P('c:/a/b/').drive, 'c:') + self.assertEqual(P('//a/b').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/').drive, '\\\\a\\b') + self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') + + + def test_root_common(self): + P = self.cls + sep = self.sep + self.assertEqual(P('').root, '') + self.assertEqual(P('a/b').root, '') + self.assertEqual(P('/').root, sep) + self.assertEqual(P('/a/b').root, sep) + + @needs_posix + def test_root_posix(self): + P = self.cls + self.assertEqual(P('/a/b').root, '/') + # POSIX special case for two leading slashes. + self.assertEqual(P('//a/b').root, '//') + + @needs_windows + def test_root_windows(self): + P = self.cls + self.assertEqual(P('c:').root, '') + self.assertEqual(P('c:a/b').root, '') + self.assertEqual(P('c:/').root, '\\') + self.assertEqual(P('c:/a/b/').root, '\\') + self.assertEqual(P('//a/b').root, '\\') + self.assertEqual(P('//a/b/').root, '\\') + self.assertEqual(P('//a/b/c/d').root, '\\') + + def test_name_empty(self): + P = self.cls + self.assertEqual(P('').name, '') + self.assertEqual(P('.').name, '') + self.assertEqual(P('/a/b/.').name, 'b') + + def test_stem_empty(self): + P = self.cls + self.assertEqual(P('').stem, '') + self.assertEqual(P('.').stem, '') + + @needs_windows + def test_with_name_windows(self): + P = self.cls + self.assertRaises(ValueError, P(r'c:').with_name, 'd.xml') + self.assertRaises(ValueError, P(r'c:\\').with_name, 'd.xml') + self.assertRaises(ValueError, P(r'\\My\Share').with_name, 'd.xml') + # NTFS alternate data streams + self.assertEqual(str(P('a').with_name('d:')), '.\\d:') + self.assertEqual(str(P('a').with_name('d:e')), '.\\d:e') + self.assertEqual(P(r'c:a\b').with_name('d:'), P(r'c:a\d:')) + self.assertEqual(P(r'c:a\b').with_name('d:e'), P(r'c:a\d:e')) + + def test_with_name_empty(self): + P = self.cls + self.assertRaises(ValueError, P('').with_name, 'd.xml') + self.assertRaises(ValueError, P('.').with_name, 'd.xml') + self.assertRaises(ValueError, P('/').with_name, 'd.xml') + self.assertRaises(ValueError, P('a/b').with_name, '') + self.assertRaises(ValueError, P('a/b').with_name, '.') + + @needs_windows + def test_with_stem_windows(self): + P = self.cls + self.assertRaises(ValueError, P('c:').with_stem, 'd') + self.assertRaises(ValueError, P('c:/').with_stem, 'd') + self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') + # NTFS alternate data streams + self.assertEqual(str(P('a').with_stem('d:')), '.\\d:') + self.assertEqual(str(P('a').with_stem('d:e')), '.\\d:e') + self.assertEqual(P('c:a/b').with_stem('d:'), P('c:a/d:')) + self.assertEqual(P('c:a/b').with_stem('d:e'), P('c:a/d:e')) + + def test_with_stem_empty(self): + P = self.cls + self.assertRaises(ValueError, P('').with_stem, 'd') + self.assertRaises(ValueError, P('.').with_stem, 'd') + self.assertRaises(ValueError, P('/').with_stem, 'd') + self.assertRaises(ValueError, P('a/b').with_stem, '') + self.assertRaises(ValueError, P('a/b').with_stem, '.') + + def test_is_reserved_deprecated(self): + P = self.cls + p = P('a/b') + with self.assertWarns(DeprecationWarning): + p.is_reserved() + + def test_full_match_case_sensitive(self): + P = self.cls + self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').full_match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').full_match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').full_match('/A/*/*.Py', case_sensitive=False)) + + def test_match_empty(self): + P = self.cls + self.assertRaises(ValueError, P('a').match, '') + self.assertRaises(ValueError, P('a').match, '.') + + def test_match_common(self): + P = self.cls + # Simple relative pattern. + self.assertTrue(P('b.py').match('b.py')) + self.assertTrue(P('a/b.py').match('b.py')) + self.assertTrue(P('/a/b.py').match('b.py')) + self.assertFalse(P('a.py').match('b.py')) + self.assertFalse(P('b/py').match('b.py')) + self.assertFalse(P('/a.py').match('b.py')) + self.assertFalse(P('b.py/c').match('b.py')) + # Wildcard relative pattern. + self.assertTrue(P('b.py').match('*.py')) + self.assertTrue(P('a/b.py').match('*.py')) + self.assertTrue(P('/a/b.py').match('*.py')) + self.assertFalse(P('b.pyc').match('*.py')) + self.assertFalse(P('b./py').match('*.py')) + self.assertFalse(P('b.py/c').match('*.py')) + # Multi-part relative pattern. + self.assertTrue(P('ab/c.py').match('a*/*.py')) + self.assertTrue(P('/d/ab/c.py').match('a*/*.py')) + self.assertFalse(P('a.py').match('a*/*.py')) + self.assertFalse(P('/dab/c.py').match('a*/*.py')) + self.assertFalse(P('ab/c.py/d').match('a*/*.py')) + # Absolute pattern. + self.assertTrue(P('/b.py').match('/*.py')) + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('a/b.py').match('/*.py')) + self.assertFalse(P('/a/b.py').match('/*.py')) + # Multi-part absolute pattern. + self.assertTrue(P('/a/b.py').match('/a/*.py')) + self.assertFalse(P('/ab.py').match('/a/*.py')) + self.assertFalse(P('/a/b/c.py').match('/a/*.py')) + # Multi-part glob-style pattern. + self.assertFalse(P('/a/b/c.py').match('/**/*.py')) + self.assertTrue(P('/a/b/c.py').match('/a/**/*.py')) + # Case-sensitive flag + self.assertFalse(P('A.py').match('a.PY', case_sensitive=True)) + self.assertTrue(P('A.py').match('a.PY', case_sensitive=False)) + self.assertFalse(P('c:/a/B.Py').match('C:/A/*.pY', case_sensitive=True)) + self.assertTrue(P('/a/b/c.py').match('/A/*/*.Py', case_sensitive=False)) + # Matching against empty path + self.assertFalse(P('').match('*')) + self.assertFalse(P('').match('**')) + self.assertFalse(P('').match('**/*')) + + @needs_posix + def test_match_posix(self): + P = self.cls + self.assertFalse(P('A.py').match('a.PY')) + + @needs_windows + def test_match_windows(self): + P = self.cls + # Absolute patterns. + self.assertTrue(P('c:/b.py').match('*:/*.py')) + self.assertTrue(P('c:/b.py').match('c:/*.py')) + self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive + self.assertFalse(P('b.py').match('/*.py')) + self.assertFalse(P('b.py').match('c:*.py')) + self.assertFalse(P('b.py').match('c:/*.py')) + self.assertFalse(P('c:b.py').match('/*.py')) + self.assertFalse(P('c:b.py').match('c:/*.py')) + self.assertFalse(P('/b.py').match('c:*.py')) + self.assertFalse(P('/b.py').match('c:/*.py')) + # UNC patterns. + self.assertTrue(P('//some/share/a.py').match('//*/*/*.py')) + self.assertTrue(P('//some/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//other/share/a.py').match('//some/share/*.py')) + self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py')) + # Case-insensitivity. + self.assertTrue(P('B.py').match('b.PY')) + self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY')) + self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY')) + # Path anchor doesn't match pattern anchor + self.assertFalse(P('c:/b.py').match('/*.py')) # 'c:/' vs '/' + self.assertFalse(P('c:/b.py').match('c:*.py')) # 'c:/' vs 'c:' + self.assertFalse(P('//some/share/a.py').match('/*.py')) # '//some/share/' vs '/' + + @needs_posix + def test_parse_path_posix(self): + check = self._check_parse_path + # Collapsing of excess leading slashes, except for the double-slash + # special case. + check('//a/b', '', '//', ['a', 'b']) + check('///a/b', '', '/', ['a', 'b']) + check('////a/b', '', '/', ['a', 'b']) + # Paths which look like NT paths aren't treated specially. + check('c:a', '', '', ['c:a',]) + check('c:\\a', '', '', ['c:\\a',]) + check('\\a', '', '', ['\\a',]) + + @needs_posix + def test_eq_posix(self): + P = self.cls + self.assertNotEqual(P('a/b'), P('A/b')) + self.assertEqual(P('/a'), P('///a')) + self.assertNotEqual(P('/a'), P('//a')) + + @needs_posix + def test_as_uri_posix(self): + P = self.cls + self.assertEqual(self.make_uri(P('/')), 'file:///') + self.assertEqual(self.make_uri(P('/a/b.c')), 'file:///a/b.c') + self.assertEqual(self.make_uri(P('/a/b%#c')), 'file:///a/b%25%23c') + + @needs_posix + def test_as_uri_non_ascii(self): + from urllib.parse import quote_from_bytes + P = self.cls + try: + os.fsencode('\xe9') + except UnicodeEncodeError: + self.skipTest("\\xe9 cannot be encoded to the filesystem encoding") + self.assertEqual(self.make_uri(P('/a/b\xe9')), + 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9'))) + + @needs_posix + def test_parse_windows_path(self): + P = self.cls + p = P('c:', 'a', 'b') + pp = P(pathlib.PureWindowsPath('c:\\a\\b')) + self.assertEqual(p, pp) + + windows_equivalences = { + './a:b': [ ('./a:b',) ], + 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], + 'c:/a': [ + ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), + ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), + ], + '//a/b/': [ ('//a/b',) ], + '//a/b/c': [ + ('//a/b', 'c'), ('//a/b/', 'c'), + ], + } + + @needs_windows + def test_equivalences_windows(self): + self.test_equivalences(self.windows_equivalences) + + @needs_windows + def test_parse_path_windows(self): + check = self._check_parse_path + # First part is anchored. + check('c:', 'c:', '', []) + check('c:/', 'c:', '\\', []) + check('/', '', '\\', []) + check('c:a', 'c:', '', ['a']) + check('c:/a', 'c:', '\\', ['a']) + check('/a', '', '\\', ['a']) + # UNC paths. + check('//', '\\\\', '', []) + check('//a', '\\\\a', '', []) + check('//a/', '\\\\a\\', '', []) + check('//a/b', '\\\\a\\b', '\\', []) + check('//a/b/', '\\\\a\\b', '\\', []) + check('//a/b/c', '\\\\a\\b', '\\', ['c']) + # Collapsing and stripping excess slashes. + check('Z://b//c/d/', 'Z:', '\\', ['b', 'c', 'd']) + # UNC paths. + check('//b/c//d', '\\\\b\\c', '\\', ['d']) + # Extended paths. + check('//./c:', '\\\\.\\c:', '', []) + check('//?/c:/', '\\\\?\\c:', '\\', []) + check('//?/c:/a', '\\\\?\\c:', '\\', ['a']) + # Extended UNC paths (format is "\\?\UNC\server\share"). + check('//?', '\\\\?', '', []) + check('//?/', '\\\\?\\', '', []) + check('//?/UNC', '\\\\?\\UNC', '', []) + check('//?/UNC/', '\\\\?\\UNC\\', '', []) + check('//?/UNC/b', '\\\\?\\UNC\\b', '', []) + check('//?/UNC/b/', '\\\\?\\UNC\\b\\', '', []) + check('//?/UNC/b/c', '\\\\?\\UNC\\b\\c', '\\', []) + check('//?/UNC/b/c/', '\\\\?\\UNC\\b\\c', '\\', []) + check('//?/UNC/b/c/d', '\\\\?\\UNC\\b\\c', '\\', ['d']) + # UNC device paths + check('//./BootPartition/', '\\\\.\\BootPartition', '\\', []) + check('//?/BootPartition/', '\\\\?\\BootPartition', '\\', []) + check('//./PhysicalDrive0', '\\\\.\\PhysicalDrive0', '', []) + check('//?/Volume{}/', '\\\\?\\Volume{}', '\\', []) + check('//./nul', '\\\\.\\nul', '', []) + # Paths to files with NTFS alternate data streams + check('./c:s', '', '', ['c:s']) + check('cc:s', '', '', ['cc:s']) + check('C:c:s', 'C:', '', ['c:s']) + check('C:/c:s', 'C:', '\\', ['c:s']) + check('D:a/c:b', 'D:', '', ['a', 'c:b']) + check('D:/a/c:b', 'D:', '\\', ['a', 'c:b']) + + @needs_windows + def test_eq_windows(self): + P = self.cls + self.assertEqual(P('c:a/b'), P('c:a/b')) + self.assertEqual(P('c:a/b'), P('c:', 'a', 'b')) + self.assertNotEqual(P('c:a/b'), P('d:a/b')) + self.assertNotEqual(P('c:a/b'), P('c:/a/b')) + self.assertNotEqual(P('/a/b'), P('c:/a/b')) + # Case-insensitivity. + self.assertEqual(P('a/B'), P('A/b')) + self.assertEqual(P('C:a/B'), P('c:A/b')) + self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b')) + self.assertEqual(P('\u0130'), P('i\u0307')) + + @needs_windows + def test_as_uri_windows(self): + P = self.cls + with self.assertRaises(ValueError): + self.make_uri(P('/a/b')) + with self.assertRaises(ValueError): + self.make_uri(P('c:a/b')) + self.assertEqual(self.make_uri(P('c:/')), 'file:///c:/') + self.assertEqual(self.make_uri(P('c:/a/b.c')), 'file:///c:/a/b.c') + self.assertEqual(self.make_uri(P('c:/a/b%#c')), 'file:///c:/a/b%25%23c') + self.assertEqual(self.make_uri(P('//some/share/')), 'file://some/share/') + self.assertEqual(self.make_uri(P('//some/share/a/b.c')), + 'file://some/share/a/b.c') + + from urllib.parse import quote_from_bytes + QUOTED_FS_NONASCII = quote_from_bytes(os.fsencode(FS_NONASCII)) + self.assertEqual(self.make_uri(P('c:/a/b' + FS_NONASCII)), + 'file:///c:/a/b' + QUOTED_FS_NONASCII) + self.assertEqual(self.make_uri(P('//some/share/a/b%#c' + FS_NONASCII)), + 'file://some/share/a/b%25%23c' + QUOTED_FS_NONASCII) + + @needs_windows + def test_ordering_windows(self): + # Case-insensitivity. + def assertOrderedEqual(a, b): + self.assertLessEqual(a, b) + self.assertGreaterEqual(b, a) + P = self.cls + p = P('c:A/b') + q = P('C:a/B') + assertOrderedEqual(p, q) + self.assertFalse(p < q) + self.assertFalse(p > q) + p = P('//some/Share/A/b') + q = P('//Some/SHARE/a/B') + assertOrderedEqual(p, q) + self.assertFalse(p < q) + self.assertFalse(p > q) + + @needs_posix + def test_is_absolute_posix(self): + P = self.cls + self.assertFalse(P('').is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertTrue(P('/').is_absolute()) + self.assertTrue(P('/a').is_absolute()) + self.assertTrue(P('/a/b/').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + + @needs_windows + def test_is_absolute_windows(self): + P = self.cls + # Under NT, only paths with both a drive and a root are absolute. + self.assertFalse(P().is_absolute()) + self.assertFalse(P('a').is_absolute()) + self.assertFalse(P('a/b/').is_absolute()) + self.assertFalse(P('/').is_absolute()) + self.assertFalse(P('/a').is_absolute()) + self.assertFalse(P('/a/b/').is_absolute()) + self.assertFalse(P('c:').is_absolute()) + self.assertFalse(P('c:a').is_absolute()) + self.assertFalse(P('c:a/b/').is_absolute()) + self.assertTrue(P('c:/').is_absolute()) + self.assertTrue(P('c:/a').is_absolute()) + self.assertTrue(P('c:/a/b/').is_absolute()) + # UNC paths are absolute by definition. + self.assertTrue(P('//').is_absolute()) + self.assertTrue(P('//a').is_absolute()) + self.assertTrue(P('//a/b').is_absolute()) + self.assertTrue(P('//a/b/').is_absolute()) + self.assertTrue(P('//a/b/c').is_absolute()) + self.assertTrue(P('//a/b/c/d').is_absolute()) + self.assertTrue(P('//?/UNC/').is_absolute()) + self.assertTrue(P('//?/UNC/spam').is_absolute()) + + def test_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.relative_to) + self.assertRaises(TypeError, p.relative_to, b'a') + self.assertEqual(p.relative_to(P('')), P('a/b')) + self.assertEqual(p.relative_to(''), P('a/b')) + self.assertEqual(p.relative_to(P('a')), P('b')) + self.assertEqual(p.relative_to('a'), P('b')) + self.assertEqual(p.relative_to('a/'), P('b')) + self.assertEqual(p.relative_to(P('a/b')), P('')) + self.assertEqual(p.relative_to('a/b'), P('')) + self.assertEqual(p.relative_to(P(''), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('c')) + self.assertRaises(ValueError, p.relative_to, P('a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('a/c')) + self.assertRaises(ValueError, p.relative_to, P('/a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + p = P('/a/b') + self.assertEqual(p.relative_to(P('/')), P('a/b')) + self.assertEqual(p.relative_to('/'), P('a/b')) + self.assertEqual(p.relative_to(P('/a')), P('b')) + self.assertEqual(p.relative_to('/a'), P('b')) + self.assertEqual(p.relative_to('/a/'), P('b')) + self.assertEqual(p.relative_to(P('/a/b')), P('')) + self.assertEqual(p.relative_to('/a/b'), P('')) + self.assertEqual(p.relative_to(P('/'), walk_up=True), P('a/b')) + self.assertEqual(p.relative_to('/', walk_up=True), P('a/b')) + self.assertEqual(p.relative_to(P('/a'), walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a', walk_up=True), P('b')) + self.assertEqual(p.relative_to('/a/', walk_up=True), P('b')) + self.assertEqual(p.relative_to(P('/a/b'), walk_up=True), P('')) + self.assertEqual(p.relative_to('/a/b', walk_up=True), P('')) + self.assertEqual(p.relative_to(P('/a/c'), walk_up=True), P('../b')) + self.assertEqual(p.relative_to('/a/c', walk_up=True), P('../b')) + self.assertEqual(p.relative_to(P('/a/b/c'), walk_up=True), P('..')) + self.assertEqual(p.relative_to('/a/b/c', walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('/c'), walk_up=True), P('../a/b')) + self.assertEqual(p.relative_to('/c', walk_up=True), P('../a/b')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/b/c')) + self.assertRaises(ValueError, p.relative_to, P('/a/c')) + self.assertRaises(ValueError, p.relative_to, P('')) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('a')) + self.assertRaises(ValueError, p.relative_to, P("../a")) + self.assertRaises(ValueError, p.relative_to, P("a/..")) + self.assertRaises(ValueError, p.relative_to, P("/a/..")) + self.assertRaises(ValueError, p.relative_to, P(''), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('a'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("../a"), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("a/.."), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P("/a/.."), walk_up=True) + + @needs_windows + def test_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO')), P('Bar')) + self.assertEqual(p.relative_to('c:foO'), P('Bar')) + self.assertEqual(p.relative_to('c:foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR')), P()) + self.assertEqual(p.relative_to('c:foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to(P('C:Foo/Bar/Baz'), walk_up=True), P('..')) + self.assertEqual(p.relative_to(P('C:Foo/Baz'), walk_up=True), P('../Bar')) + self.assertEqual(p.relative_to(P('C:Baz/Bar'), walk_up=True), P('../../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P()) + self.assertRaises(ValueError, p.relative_to, '') + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P(), walk_up=True) + self.assertRaises(ValueError, p.relative_to, '', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) + p = P('C:/Foo/Bar') + self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) + self.assertEqual(p.relative_to('c:/foO'), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) + self.assertEqual(p.relative_to('c:/foO/baR'), P()) + self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('c:/foO/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('c:/foO/baR'), walk_up=True), P()) + self.assertEqual(p.relative_to('c:/foO/baR', walk_up=True), P()) + self.assertEqual(p.relative_to('C:/Baz', walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) + self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, 'c:') + self.assertRaises(ValueError, p.relative_to, P('c:')) + self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) + self.assertRaises(ValueError, p.relative_to, P('C:Foo')) + self.assertRaises(ValueError, p.relative_to, P('d:')) + self.assertRaises(ValueError, p.relative_to, P('d:/')) + self.assertRaises(ValueError, p.relative_to, P('/')) + self.assertRaises(ValueError, p.relative_to, P('/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) + self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//C/Foo'), walk_up=True) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare'), walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/', walk_up=True), P('Foo/Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo'), walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/', walk_up=True), P('Bar')) + self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar'), walk_up=True), P()) + self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar', walk_up=True), P()) + self.assertEqual(p.relative_to(P('//sErver/sHare/bar'), walk_up=True), P('../Foo/Bar')) + self.assertEqual(p.relative_to('//sErver/sHare/bar', walk_up=True), P('../Foo/Bar')) + # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo')) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo')) + self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'), walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'), walk_up=True) + + def test_is_relative_to_common(self): + P = self.cls + p = P('a/b') + self.assertRaises(TypeError, p.is_relative_to) + self.assertRaises(TypeError, p.is_relative_to, b'a') + self.assertTrue(p.is_relative_to(P(''))) + self.assertTrue(p.is_relative_to('')) + self.assertTrue(p.is_relative_to(P('a'))) + self.assertTrue(p.is_relative_to('a/')) + self.assertTrue(p.is_relative_to(P('a/b'))) + self.assertTrue(p.is_relative_to('a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('c'))) + self.assertFalse(p.is_relative_to(P('a/b/c'))) + self.assertFalse(p.is_relative_to(P('a/c'))) + self.assertFalse(p.is_relative_to(P('/a'))) + p = P('/a/b') + self.assertTrue(p.is_relative_to(P('/'))) + self.assertTrue(p.is_relative_to('/')) + self.assertTrue(p.is_relative_to(P('/a'))) + self.assertTrue(p.is_relative_to('/a')) + self.assertTrue(p.is_relative_to('/a/')) + self.assertTrue(p.is_relative_to(P('/a/b'))) + self.assertTrue(p.is_relative_to('/a/b')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/c'))) + self.assertFalse(p.is_relative_to(P('/a/b/c'))) + self.assertFalse(p.is_relative_to(P('/a/c'))) + self.assertFalse(p.is_relative_to(P(''))) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('a'))) + + @needs_windows + def test_is_relative_to_windows(self): + P = self.cls + p = P('C:Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:'))) + self.assertTrue(p.is_relative_to('c:')) + self.assertTrue(p.is_relative_to(P('c:foO'))) + self.assertTrue(p.is_relative_to('c:foO')) + self.assertTrue(p.is_relative_to('c:foO/')) + self.assertTrue(p.is_relative_to(P('c:foO/baR'))) + self.assertTrue(p.is_relative_to('c:foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P())) + self.assertFalse(p.is_relative_to('')) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('Foo'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('C:/Foo'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) + p = P('C:/Foo/Bar') + self.assertTrue(p.is_relative_to(P('c:/'))) + self.assertTrue(p.is_relative_to(P('c:/foO'))) + self.assertTrue(p.is_relative_to('c:/foO/')) + self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) + self.assertTrue(p.is_relative_to('c:/foO/baR')) + # Unrelated paths. + self.assertFalse(p.is_relative_to('c:')) + self.assertFalse(p.is_relative_to(P('C:/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) + self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) + self.assertFalse(p.is_relative_to(P('C:Foo'))) + self.assertFalse(p.is_relative_to(P('d:'))) + self.assertFalse(p.is_relative_to(P('d:/'))) + self.assertFalse(p.is_relative_to(P('/'))) + self.assertFalse(p.is_relative_to(P('/Foo'))) + self.assertFalse(p.is_relative_to(P('//C/Foo'))) + # UNC paths. + p = P('//Server/Share/Foo/Bar') + self.assertTrue(p.is_relative_to(P('//sErver/sHare'))) + self.assertTrue(p.is_relative_to('//sErver/sHare')) + self.assertTrue(p.is_relative_to('//sErver/sHare/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo')) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/')) + self.assertTrue(p.is_relative_to(P('//sErver/sHare/Foo/Bar'))) + self.assertTrue(p.is_relative_to('//sErver/sHare/Foo/Bar')) + # Unrelated paths. + self.assertFalse(p.is_relative_to(P('/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('c:/Server/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//z/Share/Foo'))) + self.assertFalse(p.is_relative_to(P('//Server/z/Foo'))) + + +class PurePosixPathTest(PurePathTest): + cls = pathlib.PurePosixPath + + +class PureWindowsPathTest(PurePathTest): + cls = pathlib.PureWindowsPath + + +class PurePathSubclassTest(PurePathTest): + class cls(pathlib.PurePath): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + +# +# Tests for the concrete classes. +# + +class PathTest(PurePathTest): + """Tests for the FS-accessing functionalities of the Path classes.""" + cls = pathlib.Path + can_symlink = os_helper.can_symlink() + + def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') + super().setUp() + os.mkdir(self.base) + os.mkdir(os.path.join(self.base, 'dirA')) + os.mkdir(os.path.join(self.base, 'dirB')) + os.mkdir(os.path.join(self.base, 'dirC')) + os.mkdir(os.path.join(self.base, 'dirC', 'dirD')) + os.mkdir(os.path.join(self.base, 'dirE')) + with open(os.path.join(self.base, 'fileA'), 'wb') as f: + f.write(b"this is file A\n") + with open(os.path.join(self.base, 'dirB', 'fileB'), 'wb') as f: + f.write(b"this is file B\n") + with open(os.path.join(self.base, 'dirC', 'fileC'), 'wb') as f: + f.write(b"this is file C\n") + with open(os.path.join(self.base, 'dirC', 'novel.txt'), 'wb') as f: + f.write(b"this is a novel\n") + with open(os.path.join(self.base, 'dirC', 'dirD', 'fileD'), 'wb') as f: + f.write(b"this is file D\n") + os.chmod(os.path.join(self.base, 'dirE'), 0) + if self.can_symlink: + # Relative symlinks. + os.symlink('fileA', os.path.join(self.base, 'linkA')) + os.symlink('non-existing', os.path.join(self.base, 'brokenLink')) + os.symlink('dirB', + os.path.join(self.base, 'linkB'), + target_is_directory=True) + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirA', 'linkC'), + target_is_directory=True) + # This one goes upwards, creating a loop. + os.symlink(os.path.join('..', 'dirB'), + os.path.join(self.base, 'dirB', 'linkD'), + target_is_directory=True) + # Broken symlink (pointing to itself). + os.symlink('brokenLinkLoop', os.path.join(self.base, 'brokenLinkLoop')) + + def tearDown(self): + os.chmod(os.path.join(self.base, 'dirE'), 0o777) + os_helper.rmtree(self.base) + + def assertFileNotFound(self, func, *args, **kwargs): + with self.assertRaises(FileNotFoundError) as cm: + func(*args, **kwargs) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def assertEqualNormCase(self, path_a, path_b): + normcase = self.parser.normcase + self.assertEqual(normcase(path_a), normcase(path_b)) + + def tempdir(self): + d = os_helper._longpath(tempfile.mkdtemp(suffix='-dirD', + dir=os.getcwd())) + self.addCleanup(os_helper.rmtree, d) + return d + + def test_matches_writablepath_docstrings(self): + path_names = {name for name in dir(pathlib.types._WritablePath) if name[0] != '_'} + for attr_name in path_names: + if attr_name == 'parser': + # On Windows, Path.parser is ntpath, but WritablePath.parser is + # posixpath, and so their docstrings differ. + continue + our_attr = getattr(self.cls, attr_name) + path_attr = getattr(pathlib.types._WritablePath, attr_name) + self.assertEqual(our_attr.__doc__, path_attr.__doc__) + + def test_concrete_class(self): + if self.cls is pathlib.Path: + expected = pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath + else: + expected = self.cls + p = self.cls('a') + self.assertIs(type(p), expected) + + def test_unsupported_parser(self): + if self.cls.parser is os.path: + self.skipTest("path parser is supported") + else: + self.assertRaises(pathlib.UnsupportedOperation, self.cls) + + def _test_cwd(self, p): + q = self.cls(os.getcwd()) + self.assertEqual(p, q) + self.assertEqualNormCase(str(p), str(q)) + self.assertIs(type(p), type(q)) + self.assertTrue(p.is_absolute()) + + def test_cwd(self): + p = self.cls.cwd() + self._test_cwd(p) + + def test_absolute_common(self): + P = self.cls + + with mock.patch("os.getcwd") as getcwd: + getcwd.return_value = self.base + + # Simple relative paths. + self.assertEqual(str(P().absolute()), self.base) + self.assertEqual(str(P('.').absolute()), self.base) + self.assertEqual(str(P('a').absolute()), os.path.join(self.base, 'a')) + self.assertEqual(str(P('a', 'b', 'c').absolute()), os.path.join(self.base, 'a', 'b', 'c')) + + # Symlinks should not be resolved. + self.assertEqual(str(P('linkB', 'fileB').absolute()), os.path.join(self.base, 'linkB', 'fileB')) + self.assertEqual(str(P('brokenLink').absolute()), os.path.join(self.base, 'brokenLink')) + self.assertEqual(str(P('brokenLinkLoop').absolute()), os.path.join(self.base, 'brokenLinkLoop')) + + # '..' entries should be preserved and not normalised. + self.assertEqual(str(P('..').absolute()), os.path.join(self.base, '..')) + self.assertEqual(str(P('a', '..').absolute()), os.path.join(self.base, 'a', '..')) + self.assertEqual(str(P('..', 'b').absolute()), os.path.join(self.base, '..', 'b')) + + def _test_home(self, p): + q = self.cls(os.path.expanduser('~')) + self.assertEqual(p, q) + self.assertEqualNormCase(str(p), str(q)) + self.assertIs(type(p), type(q)) + self.assertTrue(p.is_absolute()) + + @unittest.skipIf( + pwd is None, reason="Test requires pwd module to get homedir." + ) + def test_home(self): + with os_helper.EnvironmentVarGuard() as env: + self._test_home(self.cls.home()) + + env.clear() + env['USERPROFILE'] = os.path.join(self.base, 'userprofile') + self._test_home(self.cls.home()) + + # bpo-38883: ignore `HOME` when set on windows + env['HOME'] = os.path.join(self.base, 'home') + self._test_home(self.cls.home()) + + @unittest.skipIf(is_wasi, "WASI has no user accounts.") + def test_expanduser_common(self): + P = self.cls + p = P('~') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'))) + p = P('foo') + self.assertEqual(p.expanduser(), p) + p = P('/~') + self.assertEqual(p.expanduser(), p) + p = P('../~') + self.assertEqual(p.expanduser(), p) + p = P(P('').absolute().anchor) / '~' + self.assertEqual(p.expanduser(), p) + p = P('~/a:b') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) + + def test_with_segments(self): + class P(self.cls): + def __init__(self, *pathsegments, session_id): + super().__init__(*pathsegments) + self.session_id = session_id + + def with_segments(self, *pathsegments): + return type(self)(*pathsegments, session_id=self.session_id) + p = P(self.base, session_id=42) + self.assertEqual(42, p.absolute().session_id) + self.assertEqual(42, p.resolve().session_id) + if not is_wasi: # WASI has no user accounts. + self.assertEqual(42, p.with_segments('~').expanduser().session_id) + self.assertEqual(42, (p / 'fileA').rename(p / 'fileB').session_id) + self.assertEqual(42, (p / 'fileB').replace(p / 'fileA').session_id) + if self.can_symlink: + self.assertEqual(42, (p / 'linkA').readlink().session_id) + for path in p.iterdir(): + self.assertEqual(42, path.session_id) + for path in p.glob('*'): + self.assertEqual(42, path.session_id) + for path in p.rglob('*'): + self.assertEqual(42, path.session_id) + for dirpath, dirnames, filenames in p.walk(): + self.assertEqual(42, dirpath.session_id) + + def test_open_common(self): + p = self.cls(self.base) + with (p / 'fileA').open('r') as f: + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.read(), "this is file A\n") + with (p / 'fileA').open('rb') as f: + self.assertIsInstance(f, io.BufferedIOBase) + self.assertEqual(f.read().strip(), b"this is file A") + + def test_open_unbuffered(self): + p = self.cls(self.base) + with (p / 'fileA').open('rb', buffering=0) as f: + self.assertIsInstance(f, io.RawIOBase) + self.assertEqual(f.read().strip(), b"this is file A") + + def test_copy_file_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'fileA' + if hasattr(os, 'chmod'): + os.chmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source, stat.UF_NODUMP) + source_st = source.stat() + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + target_st = target.stat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @needs_symlinks + def test_copy_file_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @needs_symlinks + def test_copy_file_to_existing_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @os_helper.skip_unless_xattr + def test_copy_file_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'fileA' + os.setxattr(source, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + self.assertEqual(os.getxattr(target, b'user.foo'), b'42') + + @needs_symlinks + def test_copy_symlink_follow_symlinks_true(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.copy, source) + + @needs_symlinks + def test_copy_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_directory_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_into_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'linkB' / 'copyB' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + self.assertFalse(target.exists()) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC') + target.symlink_to(base / 'fileA') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC' / 'dirD') + target.symlink_to(base / 'dirC') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'source' + target = base / 'target' + + source.mkdir() + source.joinpath('link').symlink_to('nonexistent') + + self.assertRaises(FileNotFoundError, source.copy, target) + + target2 = base / 'target2' + result = source.copy(target2, follow_symlinks=False) + self.assertEqual(result, target2) + self.assertTrue(target2.joinpath('link').is_symlink()) + self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) + + @needs_symlinks + def test_copy_link_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'linkA' + if hasattr(os, 'lchmod'): + os.lchmod(source, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(source, stat.UF_NODUMP) + source_st = source.lstat() + target = base / 'copyA' + source.copy(target, follow_symlinks=False, preserve_metadata=True) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + target_st = target.lstat() + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + def test_copy_error_handling(self): + def make_raiser(err): + def raiser(*args, **kwargs): + raise OSError(err, os.strerror(err)) + return raiser + + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'copyA' + + # Raise non-fatal OSError from all available fast copy functions. + with contextlib.ExitStack() as ctx: + if fcntl and hasattr(fcntl, 'FICLONE'): + ctx.enter_context(mock.patch('fcntl.ioctl', make_raiser(errno.EXDEV))) + if posix and hasattr(posix, '_fcopyfile'): + ctx.enter_context(mock.patch('posix._fcopyfile', make_raiser(errno.ENOTSUP))) + if hasattr(os, 'copy_file_range'): + ctx.enter_context(mock.patch('os.copy_file_range', make_raiser(errno.EXDEV))) + if hasattr(os, 'sendfile'): + ctx.enter_context(mock.patch('os.sendfile', make_raiser(errno.ENOTSOCK))) + + source.copy(target) + self.assertTrue(target.exists()) + self.assertEqual(source.read_text(), target.read_text()) + + # Raise fatal OSError from first available fast copy function. + if fcntl and hasattr(fcntl, 'FICLONE'): + patchpoint = 'fcntl.ioctl' + elif posix and hasattr(posix, '_fcopyfile'): + patchpoint = 'posix._fcopyfile' + elif hasattr(os, 'copy_file_range'): + patchpoint = 'os.copy_file_range' + elif hasattr(os, 'sendfile'): + patchpoint = 'os.sendfile' + else: + return + with mock.patch(patchpoint, make_raiser(errno.ENOENT)): + self.assertRaises(FileNotFoundError, source.copy, target) + + @unittest.skipIf(sys.platform == "win32" or sys.platform == "wasi", "directories are always readable on Windows and WASI") + @unittest.skipIf(root_in_posix, "test fails with root privilege") + def test_copy_dir_no_read_permission(self): + base = self.cls(self.base) + source = base / 'dirE' + target = base / 'copyE' + self.assertRaises(PermissionError, source.copy, target) + self.assertFalse(target.exists()) + + def test_copy_dir_preserve_metadata(self): + base = self.cls(self.base) + source = base / 'dirC' + if hasattr(os, 'chmod'): + os.chmod(source / 'dirD', stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'chflags') and hasattr(stat, 'UF_NODUMP'): + os.chflags(source / 'fileC', stat.UF_NODUMP) + target = base / 'copyA' + + subpaths = ['.', 'fileC', 'dirD', 'dirD/fileD'] + source_sts = [source.joinpath(subpath).stat() for subpath in subpaths] + source.copy(target, preserve_metadata=True) + target_sts = [target.joinpath(subpath).stat() for subpath in subpaths] + + for source_st, target_st in zip(source_sts, target_sts): + self.assertLessEqual(source_st.st_atime, target_st.st_atime) + self.assertLessEqual(source_st.st_mtime, target_st.st_mtime) + self.assertEqual(source_st.st_mode, target_st.st_mode) + if hasattr(source_st, 'st_flags'): + self.assertEqual(source_st.st_flags, target_st.st_flags) + + @os_helper.skip_unless_xattr + def test_copy_dir_preserve_metadata_xattrs(self): + base = self.cls(self.base) + source = base / 'dirC' + source_file = source.joinpath('dirD', 'fileD') + os.setxattr(source_file, b'user.foo', b'42') + target = base / 'copyA' + source.copy(target, preserve_metadata=True) + target_file = target.joinpath('dirD', 'fileD') + self.assertEqual(os.getxattr(target_file, b'user.foo'), b'42') + + @needs_symlinks + def test_move_file_symlink(self): + base = self.cls(self.base) + source = base / 'linkA' + source_readlink = source.readlink() + target = base / 'linkA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_file_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dir_symlink(self): + base = self.cls(self.base) + source = base / 'linkB' + source_readlink = source.readlink() + target = base / 'linkB_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_dir_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'brokenLink' + source_readlink = source.readlink() + target = base / 'brokenLink_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + def test_move_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'fileA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + + @patch_replace + def test_move_file_other_fs(self): + self.test_move_file() + + def test_move_file_to_file(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target = base / 'dirB' / 'fileB' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.exists()) + self.assertEqual(source_text, target.read_text()) + + @patch_replace + def test_move_file_to_file_other_fs(self): + self.test_move_file_to_file() + + def test_move_file_to_dir(self): + base = self.cls(self.base) + source = base / 'fileA' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + + @patch_replace + def test_move_file_to_dir_other_fs(self): + self.test_move_file_to_dir() + + def test_move_file_to_itself(self): + base = self.cls(self.base) + source = base / 'fileA' + self.assertRaises(OSError, source.move, source) + + def test_move_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_dir()) + self.assertTrue(target.joinpath('dirD').is_dir()) + self.assertTrue(target.joinpath('dirD', 'fileD').is_file()) + self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), + "this is file D\n") + self.assertTrue(target.joinpath('fileC').is_file()) + self.assertTrue(target.joinpath('fileC').read_text(), + "this is file C\n") + + @patch_replace + def test_move_dir_other_fs(self): + self.test_move_dir() + + def test_move_dir_to_dir(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirB' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertTrue(target.exists()) + + @patch_replace + def test_move_dir_to_dir_other_fs(self): + self.test_move_dir_to_dir() + + def test_move_dir_to_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + self.assertRaises(OSError, source.move, source) + self.assertTrue(source.exists()) + + def test_move_dir_into_itself(self): + base = self.cls(self.base) + source = base / 'dirC' + target = base / 'dirC' / 'bar' + self.assertRaises(OSError, source.move, target) + self.assertTrue(source.exists()) + self.assertFalse(target.exists()) + + @patch_replace + def test_move_dir_into_itself_other_fs(self): + self.test_move_dir_into_itself() + + @patch_replace + @needs_symlinks + def test_move_file_symlink_other_fs(self): + self.test_move_file_symlink() + + @patch_replace + @needs_symlinks + def test_move_file_symlink_to_itself_other_fs(self): + self.test_move_file_symlink_to_itself() + + @patch_replace + @needs_symlinks + def test_move_dir_symlink_other_fs(self): + self.test_move_dir_symlink() + + @patch_replace + @needs_symlinks + def test_move_dir_symlink_to_itself_other_fs(self): + self.test_move_dir_symlink_to_itself() + + @patch_replace + @needs_symlinks + def test_move_dangling_symlink_other_fs(self): + self.test_move_dangling_symlink() + + def test_move_into(self): + base = self.cls(self.base) + source = base / 'fileA' + source_text = source.read_text() + target_dir = base / 'dirA' + result = source.move_into(target_dir) + self.assertEqual(result, target_dir / 'fileA') + self.assertFalse(source.exists()) + self.assertTrue(result.exists()) + self.assertEqual(source_text, result.read_text()) + + @patch_replace + def test_move_into_other_os(self): + self.test_move_into() + + def test_move_into_empty_name(self): + source = self.cls('') + target_dir = self.base + self.assertRaises(ValueError, source.move_into, target_dir) + + @patch_replace + def test_move_into_empty_name_other_os(self): + self.test_move_into_empty_name() + + @needs_symlinks + def test_complex_symlinks_absolute(self): + self._check_complex_symlinks(self.base) + + @needs_symlinks + def test_complex_symlinks_relative(self): + self._check_complex_symlinks('.') + + @needs_symlinks + def test_complex_symlinks_relative_dot_dot(self): + self._check_complex_symlinks(self.parser.join('dirA', '..')) + + def _check_complex_symlinks(self, link0_target): + # Test solving a non-looping chain of symlinks (issue #19887). + parser = self.parser + P = self.cls(self.base) + P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) + + # Resolve absolute paths. + p = (P / 'link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + + # Resolve relative paths. + old_path = os.getcwd() + os.chdir(self.base) + try: + p = self.cls('link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = self.cls('link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + finally: + os.chdir(old_path) + + def _check_resolve(self, p, expected, strict=True): + q = p.resolve(strict) + self.assertEqual(q, expected) + + # This can be used to check both relative and absolute resolutions. + _check_resolve_relative = _check_resolve_absolute = _check_resolve + + @needs_symlinks + def test_resolve_common(self): + P = self.cls + p = P(self.base, 'foo') + with self.assertRaises(OSError) as cm: + p.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ENOENT) + # Non-strict + parser = self.parser + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(self.base, 'foo')) + p = P(self.base, 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(self.base, 'foo', 'in', 'spam')) + p = P(self.base, '..', 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) + # These are all relative symlinks. + p = P(self.base, 'dirB', 'fileB') + self._check_resolve_relative(p, p) + p = P(self.base, 'linkA') + self._check_resolve_relative(p, P(self.base, 'fileA')) + p = P(self.base, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + p = P(self.base, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) + p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', + 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + # Now create absolute symlinks. + d = self.tempdir() + P(self.base, 'dirA', 'linkX').symlink_to(d) + P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) + p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), + False) + p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + + @needs_symlinks + def test_resolve_dot(self): + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ + parser = self.parser + p = self.cls(self.base) + p.joinpath('0').symlink_to('.', target_is_directory=True) + p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) + q = p / '2' + self.assertEqual(q.resolve(strict=True), p) + r = q / '3' / '4' + self.assertRaises(FileNotFoundError, r.resolve, strict=True) + # Non-strict + self.assertEqual(r.resolve(strict=False), p / '3' / '4') + + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + @needs_posix + @needs_symlinks + def test_resolve_loop(self): + # Loops with relative symlinks. + self.cls(self.base, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(self.base, 'linkX') + self.cls(self.base, 'linkY').symlink_to('linkY') + self._check_symlink_loop(self.base, 'linkY') + self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(self.base, 'linkZ') + # Non-strict + p = self.cls(self.base, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) + self._check_symlink_loop(self.base, 'linkU') + self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) + self._check_symlink_loop(self.base, 'linkV') + self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) + self._check_symlink_loop(self.base, 'linkW') + # Non-strict + q = self.cls(self.base, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + + def test_resolve_nonexist_relative_issue38671(self): + p = self.cls('non', 'exist') + + old_cwd = os.getcwd() + os.chdir(self.base) + try: + self.assertEqual(p.resolve(), self.cls(self.base, p)) + finally: + os.chdir(old_cwd) + + @needs_symlinks + def test_readlink(self): + P = self.cls(self.base) + self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) + self.assertEqual((P / 'brokenLink').readlink(), + self.cls('non-existing')) + self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) + with self.assertRaises(OSError): + (P / 'fileA').readlink() + + @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") + def test_readlink_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.readlink(p) + + @os_helper.skip_unless_working_chmod + def test_chmod(self): + p = self.cls(self.base) / 'fileA' + mode = p.stat().st_mode + # Clear writable bit. + new_mode = mode & ~0o222 + p.chmod(new_mode) + self.assertEqual(p.stat().st_mode, new_mode) + # Set writable bit. + new_mode = mode | 0o222 + p.chmod(new_mode) + self.assertEqual(p.stat().st_mode, new_mode) + + # On Windows, os.chmod does not follow symlinks (issue #15411) + @needs_posix + @os_helper.skip_unless_working_chmod + def test_chmod_follow_symlinks_true(self): + p = self.cls(self.base) / 'linkA' + q = p.resolve() + mode = q.stat().st_mode + # Clear writable bit. + new_mode = mode & ~0o222 + p.chmod(new_mode, follow_symlinks=True) + self.assertEqual(q.stat().st_mode, new_mode) + # Set writable bit + new_mode = mode | 0o222 + p.chmod(new_mode, follow_symlinks=True) + self.assertEqual(q.stat().st_mode, new_mode) + + # XXX also need a test for lchmod. + + def _get_pw_name_or_skip_test(self, uid): + try: + return pwd.getpwuid(uid).pw_name + except KeyError: + self.skipTest( + "user %d doesn't have an entry in the system database" % uid) + + @unittest.skipUnless(pwd, "the pwd module is needed for this test") + def test_owner(self): + p = self.cls(self.base) / 'fileA' + expected_uid = p.stat().st_uid + expected_name = self._get_pw_name_or_skip_test(expected_uid) + + self.assertEqual(expected_name, p.owner()) + + @unittest.skipUnless(pwd, "the pwd module is needed for this test") + @unittest.skipUnless(root_in_posix, "test needs root privilege") + def test_owner_no_follow_symlinks(self): + all_users = [u.pw_uid for u in pwd.getpwall()] + if len(all_users) < 2: + self.skipTest("test needs more than one user") + + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' + + uid_1, uid_2 = all_users[:2] + os.chown(target, uid_1, -1) + os.chown(link, uid_2, -1, follow_symlinks=False) + + expected_uid = link.stat(follow_symlinks=False).st_uid + expected_name = self._get_pw_name_or_skip_test(expected_uid) + + self.assertEqual(expected_uid, uid_2) + self.assertEqual(expected_name, link.owner(follow_symlinks=False)) + + def _get_gr_name_or_skip_test(self, gid): + try: + return grp.getgrgid(gid).gr_name + except KeyError: + self.skipTest( + "group %d doesn't have an entry in the system database" % gid) + + @unittest.skipUnless(grp, "the grp module is needed for this test") + def test_group(self): + p = self.cls(self.base) / 'fileA' + expected_gid = p.stat().st_gid + expected_name = self._get_gr_name_or_skip_test(expected_gid) + + self.assertEqual(expected_name, p.group()) + + @unittest.skipUnless(grp, "the grp module is needed for this test") + @unittest.skipUnless(root_in_posix, "test needs root privilege") + def test_group_no_follow_symlinks(self): + all_groups = [g.gr_gid for g in grp.getgrall()] + if len(all_groups) < 2: + self.skipTest("test needs more than one group") + + target = self.cls(self.base) / 'fileA' + link = self.cls(self.base) / 'linkA' + + gid_1, gid_2 = all_groups[:2] + os.chown(target, -1, gid_1) + os.chown(link, -1, gid_2, follow_symlinks=False) + + expected_gid = link.stat(follow_symlinks=False).st_gid + expected_name = self._get_gr_name_or_skip_test(expected_gid) + + self.assertEqual(expected_gid, gid_2) + self.assertEqual(expected_name, link.group(follow_symlinks=False)) + + def test_unlink(self): + p = self.cls(self.base) / 'fileA' + p.unlink() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_unlink_missing_ok(self): + p = self.cls(self.base) / 'fileAAA' + self.assertFileNotFound(p.unlink) + p.unlink(missing_ok=True) + + def test_rmdir(self): + p = self.cls(self.base) / 'dirA' + for q in p.iterdir(): + q.unlink() + p.rmdir() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_delete_file(self): + p = self.cls(self.base) / 'fileA' + p._delete() + self.assertFalse(p.exists()) + self.assertFileNotFound(p._delete) + + def test_delete_dir(self): + base = self.cls(self.base) + base.joinpath('dirA')._delete() + self.assertFalse(base.joinpath('dirA').exists()) + self.assertFalse(base.joinpath('dirA', 'linkC').exists( + follow_symlinks=False)) + base.joinpath('dirB')._delete() + self.assertFalse(base.joinpath('dirB').exists()) + self.assertFalse(base.joinpath('dirB', 'fileB').exists()) + self.assertFalse(base.joinpath('dirB', 'linkD').exists( + follow_symlinks=False)) + base.joinpath('dirC')._delete() + self.assertFalse(base.joinpath('dirC').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD').exists()) + self.assertFalse(base.joinpath('dirC', 'dirD', 'fileD').exists()) + self.assertFalse(base.joinpath('dirC', 'fileC').exists()) + self.assertFalse(base.joinpath('dirC', 'novel.txt').exists()) + + def test_delete_missing(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + # filename is guaranteed not to exist + filename = tmp / 'foo' + self.assertRaises(FileNotFoundError, filename._delete) + + @needs_symlinks + def test_delete_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir_ = tmp / 'dir' + dir_.mkdir() + link = tmp / 'link' + link.symlink_to(dir_) + link._delete() + self.assertTrue(dir_.exists()) + self.assertFalse(link.exists(follow_symlinks=False)) + + @needs_symlinks + def test_delete_inner_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + link1.symlink_to(dir2) + link2 = dir1 / 'link2' + link2.symlink_to(dir3) + link3 = dir1 / 'link3' + link3.symlink_to(file1) + # make sure symlinks are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_delete_unwritable(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + child_file_path = tmp / 'a' + child_dir_path = tmp / 'b' + child_file_path.write_text("") + child_dir_path.mkdir() + old_dir_mode = tmp.stat().st_mode + old_child_file_mode = child_file_path.stat().st_mode + old_child_dir_mode = child_dir_path.stat().st_mode + # Make unwritable. + new_mode = stat.S_IREAD | stat.S_IEXEC + try: + child_file_path.chmod(new_mode) + child_dir_path.chmod(new_mode) + tmp.chmod(new_mode) + + self.assertRaises(PermissionError, tmp._delete) + finally: + tmp.chmod(old_dir_mode) + child_file_path.chmod(old_child_file_mode) + child_dir_path.chmod(old_child_dir_mode) + + @needs_windows + def test_delete_inner_junction(self): + import _winapi + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + _winapi.CreateJunction(str(dir2), str(link1)) + link2 = dir1 / 'link2' + _winapi.CreateJunction(str(dir3), str(link2)) + link3 = dir1 / 'link3' + _winapi.CreateJunction(str(file1), str(link3)) + # make sure junctions are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + + @needs_windows + def test_delete_outer_junction(self): + import _winapi + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + src = tmp / 'cheese' + dst = tmp / 'shop' + src.mkdir() + spam = src / 'spam' + spam.write_text('') + _winapi.CreateJunction(str(src), str(dst)) + dst._delete() + self.assertFalse(dst.exists()) + self.assertTrue(spam.exists()) + self.assertTrue(src.exists()) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_delete_on_named_pipe(self): + p = self.cls(self.base, 'pipe') + os.mkfifo(p) + p._delete() + self.assertFalse(p.exists()) + + p = self.cls(self.base, 'dir') + p.mkdir() + os.mkfifo(p / 'mypipe') + p._delete() + self.assertFalse(p.exists()) + + def test_delete_does_not_choke_on_failing_lstat(self): + try: + orig_lstat = os.lstat + tmp = self.cls(self.base, 'delete') + + def raiser(fn, *args, **kwargs): + if fn != tmp: + raise OSError() + else: + return orig_lstat(fn) + + os.lstat = raiser + + tmp.mkdir() + foo = tmp / 'foo' + foo.write_text('') + tmp._delete() + finally: + os.lstat = orig_lstat + + @os_helper.skip_unless_hardlink + def test_hardlink_to(self): + P = self.cls(self.base) + target = P / 'fileA' + size = target.stat().st_size + # linking to another path. + link = P / 'dirA' / 'fileAA' + link.hardlink_to(target) + self.assertEqual(link.stat().st_size, size) + self.assertTrue(os.path.samefile(target, link)) + self.assertTrue(target.exists()) + # Linking to a str of a relative path. + link2 = P / 'dirA' / 'fileAAA' + target2 = self.parser.join(TESTFN, 'fileA') + link2.hardlink_to(target2) + self.assertEqual(os.stat(target2).st_size, size) + self.assertTrue(link2.exists()) + + @unittest.skipIf(hasattr(os, "link"), "os.link() is present") + def test_hardlink_to_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + # linking to another path. + q = P / 'dirA' / 'fileAA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.hardlink_to(p) + + def test_rename(self): + P = self.cls(self.base) + p = P / 'fileA' + size = p.stat().st_size + # Renaming to another path. + q = P / 'dirA' / 'fileAA' + renamed_p = p.rename(q) + self.assertEqual(renamed_p, q) + self.assertEqual(q.stat().st_size, size) + self.assertFileNotFound(p.stat) + # Renaming to a str of a relative path. + r = self.parser.join(TESTFN, 'fileAAA') + renamed_q = q.rename(r) + self.assertEqual(renamed_q, self.cls(r)) + self.assertEqual(os.stat(r).st_size, size) + self.assertFileNotFound(q.stat) + + def test_replace(self): + P = self.cls(self.base) + p = P / 'fileA' + size = p.stat().st_size + # Replacing a non-existing path. + q = P / 'dirA' / 'fileAA' + replaced_p = p.replace(q) + self.assertEqual(replaced_p, q) + self.assertEqual(q.stat().st_size, size) + self.assertFileNotFound(p.stat) + # Replacing another (existing) path. + r = self.parser.join(TESTFN, 'dirB', 'fileB') + replaced_q = q.replace(r) + self.assertEqual(replaced_q, self.cls(r)) + self.assertEqual(os.stat(r).st_size, size) + self.assertFileNotFound(q.stat) + + def test_touch_common(self): + P = self.cls(self.base) + p = P / 'newfileA' + self.assertFalse(p.exists()) + p.touch() + self.assertTrue(p.exists()) + st = p.stat() + old_mtime = st.st_mtime + old_mtime_ns = st.st_mtime_ns + # Rewind the mtime sufficiently far in the past to work around + # filesystem-specific timestamp granularity. + os.utime(str(p), (old_mtime - 10, old_mtime - 10)) + # The file mtime should be refreshed by calling touch() again. + p.touch() + st = p.stat() + self.assertGreaterEqual(st.st_mtime_ns, old_mtime_ns) + self.assertGreaterEqual(st.st_mtime, old_mtime) + # Now with exist_ok=False. + p = P / 'newfileB' + self.assertFalse(p.exists()) + p.touch(mode=0o700, exist_ok=False) + self.assertTrue(p.exists()) + self.assertRaises(OSError, p.touch, exist_ok=False) + + def test_touch_nochange(self): + P = self.cls(self.base) + p = P / 'fileA' + p.touch() + with p.open('rb') as f: + self.assertEqual(f.read().strip(), b"this is file A") + + def test_mkdir(self): + P = self.cls(self.base) + p = P / 'newdirA' + self.assertFalse(p.exists()) + p.mkdir() + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(OSError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_parents(self): + # Creating a chain of directories. + p = self.cls(self.base, 'newdirB', 'newdirC') + self.assertFalse(p.exists()) + with self.assertRaises(OSError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.ENOENT) + p.mkdir(parents=True) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(OSError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + # Test `mode` arg. + mode = stat.S_IMODE(p.stat().st_mode) # Default mode. + p = self.cls(self.base, 'newdirD', 'newdirE') + p.mkdir(0o555, parents=True) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + if os.name != 'nt': + # The directory's permissions follow the mode argument. + self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o7555 & mode) + # The parent's permissions follow the default process settings. + self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode) + + def test_mkdir_exist_ok(self): + p = self.cls(self.base, 'dirB') + st_ctime_first = p.stat().st_ctime + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + p.mkdir(exist_ok=True) + self.assertTrue(p.exists()) + self.assertEqual(p.stat().st_ctime, st_ctime_first) + + def test_mkdir_exist_ok_with_parent(self): + p = self.cls(self.base, 'dirC') + self.assertTrue(p.exists()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + p = p / 'newdirC' + p.mkdir(parents=True) + st_ctime_first = p.stat().st_ctime + self.assertTrue(p.exists()) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + p.mkdir(parents=True, exist_ok=True) + self.assertTrue(p.exists()) + self.assertEqual(p.stat().st_ctime, st_ctime_first) + + def test_mkdir_exist_ok_root(self): + # Issue #25803: A drive root could raise PermissionError on Windows. + self.cls('/').resolve().mkdir(exist_ok=True) + self.cls('/').resolve().mkdir(parents=True, exist_ok=True) + + @needs_windows # XXX: not sure how to test this on POSIX. + def test_mkdir_with_unknown_drive(self): + for d in 'ZYXWVUTSRQPONMLKJIHGFEDCBA': + p = self.cls(d + ':\\') + if not p.is_dir(): + break + else: + self.skipTest("cannot find a drive that doesn't exist") + with self.assertRaises(OSError): + (p / 'child' / 'path').mkdir(parents=True) + + def test_mkdir_with_child_file(self): + p = self.cls(self.base, 'dirB', 'fileB') + self.assertTrue(p.exists()) + # An exception is raised when the last path component is an existing + # regular file, regardless of whether exist_ok is true or not. + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(parents=True, exist_ok=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_no_parents_file(self): + p = self.cls(self.base, 'fileA') + self.assertTrue(p.exists()) + # An exception is raised when the last path component is an existing + # regular file, regardless of whether exist_ok is true or not. + with self.assertRaises(FileExistsError) as cm: + p.mkdir() + self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(FileExistsError) as cm: + p.mkdir(exist_ok=True) + self.assertEqual(cm.exception.errno, errno.EEXIST) + + def test_mkdir_concurrent_parent_creation(self): + for pattern_num in range(32): + p = self.cls(self.base, 'dirCPC%d' % pattern_num) + self.assertFalse(p.exists()) + + real_mkdir = os.mkdir + def my_mkdir(path, mode=0o777): + path = str(path) + # Emulate another process that would create the directory + # just before we try to create it ourselves. We do it + # in all possible pattern combinations, assuming that this + # function is called at most 5 times (dirCPC/dir1/dir2, + # dirCPC/dir1, dirCPC, dirCPC/dir1, dirCPC/dir1/dir2). + if pattern.pop(): + real_mkdir(path, mode) # From another process. + concurrently_created.add(path) + real_mkdir(path, mode) # Our real call. + + pattern = [bool(pattern_num & (1 << n)) for n in range(5)] + concurrently_created = set() + p12 = p / 'dir1' / 'dir2' + try: + with mock.patch("os.mkdir", my_mkdir): + p12.mkdir(parents=True, exist_ok=False) + except FileExistsError: + self.assertIn(str(p12), concurrently_created) + else: + self.assertNotIn(str(p12), concurrently_created) + self.assertTrue(p.exists()) + + @needs_symlinks + def test_symlink_to(self): + P = self.cls(self.base) + target = P / 'fileA' + # Symlinking a path target. + link = P / 'dirA' / 'linkAA' + link.symlink_to(target) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + # Symlinking a str target. + link = P / 'dirA' / 'linkAAA' + link.symlink_to(str(target)) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + self.assertFalse(link.is_dir()) + # Symlinking to a directory. + target = P / 'dirB' + link = P / 'dirA' / 'linkAAAA' + link.symlink_to(target, target_is_directory=True) + self.assertEqual(link.stat(), target.stat()) + self.assertNotEqual(link.lstat(), target.stat()) + self.assertTrue(link.is_dir()) + self.assertTrue(list(link.iterdir())) + + @unittest.skipIf(hasattr(os, "symlink"), "os.symlink() is present") + def test_symlink_to_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + # linking to another path. + q = P / 'dirA' / 'fileAA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.symlink_to(p) + + def test_info_exists_caching(self): + p = self.cls(self.base) + q = p / 'myfile' + self.assertFalse(q.info.exists()) + self.assertFalse(q.info.exists(follow_symlinks=False)) + q.write_text('hullo') + self.assertFalse(q.info.exists()) + self.assertFalse(q.info.exists(follow_symlinks=False)) + + def test_info_is_dir_caching(self): + p = self.cls(self.base) + q = p / 'mydir' + self.assertFalse(q.info.is_dir()) + self.assertFalse(q.info.is_dir(follow_symlinks=False)) + q.mkdir() + self.assertFalse(q.info.is_dir()) + self.assertFalse(q.info.is_dir(follow_symlinks=False)) + + def test_info_is_file_caching(self): + p = self.cls(self.base) + q = p / 'myfile' + self.assertFalse(q.info.is_file()) + self.assertFalse(q.info.is_file(follow_symlinks=False)) + q.write_text('hullo') + self.assertFalse(q.info.is_file()) + self.assertFalse(q.info.is_file(follow_symlinks=False)) + + @needs_symlinks + def test_info_is_symlink_caching(self): + p = self.cls(self.base) + q = p / 'mylink' + self.assertFalse(q.info.is_symlink()) + q.symlink_to('blah') + self.assertFalse(q.info.is_symlink()) + + q = p / 'mylink' # same path, new instance. + self.assertTrue(q.info.is_symlink()) + q.unlink() + self.assertTrue(q.info.is_symlink()) + + def test_stat(self): + statA = self.cls(self.base).joinpath('fileA').stat() + statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() + statC = self.cls(self.base).joinpath('dirC').stat() + # st_mode: files are the same, directory differs. + self.assertIsInstance(statA.st_mode, int) + self.assertEqual(statA.st_mode, statB.st_mode) + self.assertNotEqual(statA.st_mode, statC.st_mode) + self.assertNotEqual(statB.st_mode, statC.st_mode) + # st_ino: all different, + self.assertIsInstance(statA.st_ino, int) + self.assertNotEqual(statA.st_ino, statB.st_ino) + self.assertNotEqual(statA.st_ino, statC.st_ino) + self.assertNotEqual(statB.st_ino, statC.st_ino) + # st_dev: all the same. + self.assertIsInstance(statA.st_dev, int) + self.assertEqual(statA.st_dev, statB.st_dev) + self.assertEqual(statA.st_dev, statC.st_dev) + # other attributes not used by pathlib. + + def test_stat_no_follow_symlinks_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.stat(follow_symlinks=False)) + + @needs_symlinks + def test_stat_no_follow_symlinks(self): + p = self.cls(self.base) / 'linkA' + st = p.stat() + self.assertNotEqual(st, p.stat(follow_symlinks=False)) + + @needs_symlinks + def test_lstat(self): + p = self.cls(self.base)/ 'linkA' + st = p.stat() + self.assertNotEqual(st, p.lstat()) + + def test_lstat_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.lstat()) + + def test_exists(self): + P = self.cls + p = P(self.base) + self.assertIs(True, p.exists()) + self.assertIs(True, (p / 'dirA').exists()) + self.assertIs(True, (p / 'fileA').exists()) + self.assertIs(False, (p / 'fileA' / 'bah').exists()) + if self.can_symlink: + self.assertIs(True, (p / 'linkA').exists()) + self.assertIs(True, (p / 'linkB').exists()) + self.assertIs(True, (p / 'linkB' / 'fileB').exists()) + self.assertIs(False, (p / 'linkA' / 'bah').exists()) + self.assertIs(False, (p / 'brokenLink').exists()) + self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) + self.assertIs(False, (p / 'foo').exists()) + self.assertIs(False, P('/xyzzy').exists()) + self.assertIs(False, P(self.base + '\udfff').exists()) + self.assertIs(False, P(self.base + '\x00').exists()) + + def test_is_dir(self): + P = self.cls(self.base) + self.assertTrue((P / 'dirA').is_dir()) + self.assertFalse((P / 'fileA').is_dir()) + self.assertFalse((P / 'non-existing').is_dir()) + self.assertFalse((P / 'fileA' / 'bah').is_dir()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir()) + self.assertTrue((P / 'linkB').is_dir()) + self.assertFalse((P/ 'brokenLink').is_dir()) + self.assertFalse((P / 'dirA\udfff').is_dir()) + self.assertFalse((P / 'dirA\x00').is_dir()) + + def test_is_dir_no_follow_symlinks(self): + P = self.cls(self.base) + self.assertTrue((P / 'dirA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_dir(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_dir(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\udfff').is_dir(follow_symlinks=False)) + self.assertFalse((P / 'dirA\x00').is_dir(follow_symlinks=False)) + + def test_is_file(self): + P = self.cls(self.base) + self.assertTrue((P / 'fileA').is_file()) + self.assertFalse((P / 'dirA').is_file()) + self.assertFalse((P / 'non-existing').is_file()) + self.assertFalse((P / 'fileA' / 'bah').is_file()) + if self.can_symlink: + self.assertTrue((P / 'linkA').is_file()) + self.assertFalse((P / 'linkB').is_file()) + self.assertFalse((P/ 'brokenLink').is_file()) + self.assertFalse((P / 'fileA\udfff').is_file()) + self.assertFalse((P / 'fileA\x00').is_file()) + + def test_is_file_no_follow_symlinks(self): + P = self.cls(self.base) + self.assertTrue((P / 'fileA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'dirA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'non-existing').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA' / 'bah').is_file(follow_symlinks=False)) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_file(follow_symlinks=False)) + self.assertFalse((P / 'linkB').is_file(follow_symlinks=False)) + self.assertFalse((P/ 'brokenLink').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) + self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) + + def test_is_symlink(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_symlink()) + self.assertFalse((P / 'dirA').is_symlink()) + self.assertFalse((P / 'non-existing').is_symlink()) + self.assertFalse((P / 'fileA' / 'bah').is_symlink()) + if self.can_symlink: + self.assertTrue((P / 'linkA').is_symlink()) + self.assertTrue((P / 'linkB').is_symlink()) + self.assertTrue((P/ 'brokenLink').is_symlink()) + self.assertIs((P / 'fileA\udfff').is_file(), False) + self.assertIs((P / 'fileA\x00').is_file(), False) + if self.can_symlink: + self.assertIs((P / 'linkA\udfff').is_file(), False) + self.assertIs((P / 'linkA\x00').is_file(), False) + + def test_is_junction_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_junction()) + self.assertFalse((P / 'dirA').is_junction()) + self.assertFalse((P / 'non-existing').is_junction()) + self.assertFalse((P / 'fileA' / 'bah').is_junction()) + self.assertFalse((P / 'fileA\udfff').is_junction()) + self.assertFalse((P / 'fileA\x00').is_junction()) + + def test_is_junction_true(self): + P = self.cls(self.base) + + with mock.patch.object(P.parser, 'isjunction'): + self.assertEqual(P.is_junction(), P.parser.isjunction.return_value) + P.parser.isjunction.assert_called_once_with(P) + + def test_is_fifo_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_fifo()) + self.assertFalse((P / 'dirA').is_fifo()) + self.assertFalse((P / 'non-existing').is_fifo()) + self.assertFalse((P / 'fileA' / 'bah').is_fifo()) + self.assertIs((P / 'fileA\udfff').is_fifo(), False) + self.assertIs((P / 'fileA\x00').is_fifo(), False) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_is_fifo_true(self): + P = self.cls(self.base, 'myfifo') + try: + os.mkfifo(str(P)) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) + self.assertTrue(P.is_fifo()) + self.assertFalse(P.is_socket()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls(self.base, 'myfifo\udfff').is_fifo(), False) + self.assertIs(self.cls(self.base, 'myfifo\x00').is_fifo(), False) + + def test_is_socket_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_socket()) + self.assertFalse((P / 'dirA').is_socket()) + self.assertFalse((P / 'non-existing').is_socket()) + self.assertFalse((P / 'fileA' / 'bah').is_socket()) + self.assertIs((P / 'fileA\udfff').is_socket(), False) + self.assertIs((P / 'fileA\x00').is_socket(), False) + + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") + @unittest.skipIf( + is_emscripten, "Unix sockets are not implemented on Emscripten." + ) + @unittest.skipIf( + is_wasi, "Cannot create socket on WASI." + ) + def test_is_socket_true(self): + P = self.cls(self.base, 'mysock') + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.addCleanup(sock.close) + try: + sock.bind(str(P)) + except OSError as e: + if (isinstance(e, PermissionError) or + "AF_UNIX path too long" in str(e)): + self.skipTest("cannot bind Unix socket: " + str(e)) + self.assertTrue(P.is_socket()) + self.assertFalse(P.is_fifo()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls(self.base, 'mysock\udfff').is_socket(), False) + self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) + + def test_is_block_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_block_device()) + self.assertFalse((P / 'dirA').is_block_device()) + self.assertFalse((P / 'non-existing').is_block_device()) + self.assertFalse((P / 'fileA' / 'bah').is_block_device()) + self.assertIs((P / 'fileA\udfff').is_block_device(), False) + self.assertIs((P / 'fileA\x00').is_block_device(), False) + + def test_is_char_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_char_device()) + self.assertFalse((P / 'dirA').is_char_device()) + self.assertFalse((P / 'non-existing').is_char_device()) + self.assertFalse((P / 'fileA' / 'bah').is_char_device()) + self.assertIs((P / 'fileA\udfff').is_char_device(), False) + self.assertIs((P / 'fileA\x00').is_char_device(), False) + + def test_is_char_device_true(self): + # os.devnull should generally be a char device. + P = self.cls(os.devnull) + if not P.exists(): + self.skipTest("null device required") + self.assertTrue(P.is_char_device()) + self.assertFalse(P.is_block_device()) + self.assertFalse(P.is_file()) + self.assertIs(self.cls(f'{os.devnull}\udfff').is_char_device(), False) + self.assertIs(self.cls(f'{os.devnull}\x00').is_char_device(), False) + + def test_is_mount(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_mount()) + self.assertFalse((P / 'dirA').is_mount()) + self.assertFalse((P / 'non-existing').is_mount()) + self.assertFalse((P / 'fileA' / 'bah').is_mount()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_mount()) + if os.name == 'nt': + R = self.cls('c:\\') + else: + R = self.cls('/') + self.assertTrue(R.is_mount()) + self.assertFalse((R / '\udfff').is_mount()) + + def test_samefile(self): + parser = self.parser + fileA_path = parser.join(self.base, 'fileA') + fileB_path = parser.join(self.base, 'dirB', 'fileB') + p = self.cls(fileA_path) + pp = self.cls(fileA_path) + q = self.cls(fileB_path) + self.assertTrue(p.samefile(fileA_path)) + self.assertTrue(p.samefile(pp)) + self.assertFalse(p.samefile(fileB_path)) + self.assertFalse(p.samefile(q)) + # Test the non-existent file case + non_existent = parser.join(self.base, 'foo') + r = self.cls(non_existent) + self.assertRaises(FileNotFoundError, p.samefile, r) + self.assertRaises(FileNotFoundError, p.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, p) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, r) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + + def test_passing_kwargs_errors(self): + with self.assertRaises(TypeError): + self.cls(foo="bar") + + @needs_symlinks + def test_iterdir_symlink(self): + # __iter__ on a symlink to a directory. + P = self.cls + p = P(self.base, 'linkB') + paths = set(p.iterdir()) + expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } + self.assertEqual(paths, expected) + + @needs_posix + def test_glob_posix(self): + P = self.cls + p = P(self.base) + q = p / "FILEa" + given = set(p.glob("FILEa")) + expect = {q} if q.info.exists() else set() + self.assertEqual(given, expect) + self.assertEqual(set(p.glob("FILEa*")), set()) + + @needs_windows + def test_glob_windows(self): + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("FILEa")), { P(self.base, "fileA") }) + self.assertEqual(set(p.glob("*a\\")), { P(self.base, "dirA/") }) + self.assertEqual(set(p.glob("F*a")), { P(self.base, "fileA") }) + + def test_glob_empty_pattern(self): + p = self.cls('') + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('.')) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('./')) + + def test_glob_many_open_files(self): + depth = 30 + P = self.cls + p = base = P(self.base) / 'deep' + p.mkdir() + for _ in range(depth): + p /= 'd' + p.mkdir() + pattern = '/'.join(['*'] * depth) + iters = [base.glob(pattern) for j in range(100)] + for it in iters: + self.assertEqual(next(it), p) + iters = [base.rglob('d') for j in range(100)] + p = base + for i in range(depth): + p = p / 'd' + for it in iters: + self.assertEqual(next(it), p) + + def test_glob_above_recursion_limit(self): + recursion_limit = 50 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with infinite_recursion(recursion_limit): + list(base.glob('**/')) + + def test_glob_pathlike(self): + P = self.cls + p = P(self.base) + pattern = "dir*/file*" + expect = {p / "dirB/fileB", p / "dirC/fileC"} + self.assertEqual(expect, set(p.glob(P(pattern)))) + self.assertEqual(expect, set(p.glob(FakePath(pattern)))) + + def test_glob_case_sensitive(self): + P = self.cls + def _check(path, pattern, case_sensitive, expected): + actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} + expected = {str(P(self.base, q)) for q in expected} + self.assertEqual(actual, expected) + path = P(self.base) + _check(path, "DIRB/FILE*", True, []) + _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) + _check(path, "dirb/file*", True, []) + _check(path, "dirb/file*", False, ["dirB/fileB"]) + + @needs_symlinks + def test_glob_dot(self): + P = self.cls + with os_helper.change_cwd(P(self.base, "dirC")): + self.assertEqual( + set(P('.').glob('*')), {P("fileC"), P("novel.txt"), P("dirD")}) + self.assertEqual( + set(P('.').glob('**')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD"), P(".")}) + self.assertEqual( + set(P('.').glob('**/*')), {P("fileC"), P("novel.txt"), P("dirD"), P("dirD/fileD")}) + self.assertEqual( + set(P('.').glob('**/*/*')), {P("dirD/fileD")}) + + # See https://github.com/WebAssembly/wasi-filesystem/issues/26 + @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") + def test_glob_dotdot(self): + # ".." is not special in globs. + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) + self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) + self.assertEqual(set(p.glob("../xyzzy")), set()) + if self.cls.parser is posixpath: + self.assertEqual(set(p.glob("xyzzy/..")), set()) + else: + # ".." segments are normalized first on Windows, so this path is stat()able. + self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) + if sys.platform == "emscripten": + # Emscripten will return ELOOP if there are 49 or more ..'s. + # Can remove when https://github.com/emscripten-core/emscripten/pull/24591 is merged. + NDOTDOTS = 48 + else: + NDOTDOTS = 50 + self.assertEqual(set(p.glob("/".join([".."] * NDOTDOTS))), { P(self.base, *[".."] * NDOTDOTS)}) + + def test_glob_inaccessible(self): + P = self.cls + p = P(self.base, "mydir1", "mydir2") + p.mkdir(parents=True) + p.parent.chmod(0) + self.assertEqual(set(p.glob('*')), set()) + + def test_rglob_pathlike(self): + P = self.cls + p = P(self.base, "dirC") + pattern = "**/file*" + expect = {p / "fileC", p / "dirD/fileD"} + self.assertEqual(expect, set(p.rglob(P(pattern)))) + self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) + + @needs_symlinks + def test_glob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) + _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirB/linkD/..", "dirA/linkC/linkD/..", + "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", + "dirB/linkD/..", "dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + @needs_symlinks + def test_rglob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", + "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) + _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) + + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + def test_rglob_recurse_symlinks_false(self): + def _check(path, glob, expected): + actual = set(path.rglob(glob, recurse_symlinks=False)) + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + it = p.rglob("fileA") + self.assertIsInstance(it, collections.abc.Iterator) + _check(p, "fileA", ["fileA"]) + _check(p, "fileB", ["dirB/fileB"]) + _check(p, "**/fileB", ["dirB/fileB"]) + _check(p, "*/fileA", []) + + if self.can_symlink: + _check(p, "*/fileB", ["dirB/fileB", "dirB/linkD/fileB", + "linkB/fileB", "dirA/linkC/fileB"]) + _check(p, "*/", [ + "dirA/", "dirA/linkC/", "dirB/", "dirB/linkD/", "dirC/", + "dirC/dirD/", "dirE/", "linkB/"]) + else: + _check(p, "*/fileB", ["dirB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirC/dirD/", "dirE/"]) + + _check(p, "file*", ["fileA", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "", ["", "dirA/", "dirB/", "dirC/", "dirE/", "dirC/dirD/"]) + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "**/file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "dir*/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/**/", ["dirC/dirD/"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + _check(p, "**", ["dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) + _check(p, "**/", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + @needs_posix + def test_rglob_posix(self): + P = self.cls + p = P(self.base, "dirC") + q = p / "dirD" / "FILEd" + given = set(p.rglob("FILEd")) + expect = {q} if q.exists() else set() + self.assertEqual(given, expect) + self.assertEqual(set(p.rglob("FILEd*")), set()) + + @needs_windows + def test_rglob_windows(self): + P = self.cls + p = P(self.base, "dirC") + self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) + self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) + + @needs_symlinks + def test_rglob_symlink_loop(self): + # Don't get fooled by symlink loops (Issue #26012). + P = self.cls + p = P(self.base) + given = set(p.rglob('*', recurse_symlinks=False)) + expect = {'brokenLink', + 'dirA', 'dirA/linkC', + 'dirB', 'dirB/fileB', 'dirB/linkD', + 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', + 'dirC/fileC', 'dirC/novel.txt', + 'dirE', + 'fileA', + 'linkA', + 'linkB', + 'brokenLinkLoop', + } + self.assertEqual(given, {p / x for x in expect}) + + @needs_symlinks + def test_glob_permissions(self): + # See bpo-38894 + P = self.cls + base = P(self.base) / 'permissions' + base.mkdir() + + for i in range(100): + link = base / f"link{i}" + if i % 2: + link.symlink_to(P(self.base, "dirE", "nonexistent")) + else: + link.symlink_to(P(self.base, "dirC"), target_is_directory=True) + + self.assertEqual(len(set(base.glob("*"))), 100) + self.assertEqual(len(set(base.glob("*/"))), 50) + self.assertEqual(len(set(base.glob("*/fileC"))), 50) + self.assertEqual(len(set(base.glob("*/file*"))), 50) + + @needs_symlinks + def test_glob_long_symlink(self): + # See gh-87695 + base = self.cls(self.base) / 'long_symlink' + base.mkdir() + bad_link = base / 'bad_link' + bad_link.symlink_to("bad" * 200) + self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + + @needs_posix + def test_absolute_posix(self): + P = self.cls + self.assertEqual(str(P('/').absolute()), '/') + self.assertEqual(str(P('/a').absolute()), '/a') + self.assertEqual(str(P('/a/b').absolute()), '/a/b') + + # '//'-prefixed absolute path (supported by POSIX). + self.assertEqual(str(P('//').absolute()), '//') + self.assertEqual(str(P('//a').absolute()), '//a') + self.assertEqual(str(P('//a/b').absolute()), '//a/b') + + @unittest.skipIf( + is_wasm32, + "umask is not implemented on Emscripten/WASI." + ) + @needs_posix + def test_open_mode(self): + # Unmask all permissions except world-write, which may + # not be supported on some filesystems (see GH-85633.) + old_mask = os.umask(0o002) + self.addCleanup(os.umask, old_mask) + p = self.cls(self.base) + with (p / 'new_file').open('wb'): + pass + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o664) + os.umask(0o026) + with (p / 'other_new_file').open('wb'): + pass + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o640) + + @needs_posix + def test_resolve_root(self): + current_directory = os.getcwd() + try: + os.chdir('/') + p = self.cls('spam') + self.assertEqual(str(p.resolve()), '/spam') + finally: + os.chdir(current_directory) + + @unittest.skipIf( + is_wasm32, + "umask is not implemented on Emscripten/WASI." + ) + @needs_posix + def test_touch_mode(self): + # Unmask all permissions except world-write, which may + # not be supported on some filesystems (see GH-85633.) + old_mask = os.umask(0o002) + self.addCleanup(os.umask, old_mask) + p = self.cls(self.base) + (p / 'new_file').touch() + st = os.stat(self.parser.join(self.base, 'new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o664) + os.umask(0o026) + (p / 'other_new_file').touch() + st = os.stat(self.parser.join(self.base, 'other_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o640) + (p / 'masked_new_file').touch(mode=0o750) + st = os.stat(self.parser.join(self.base, 'masked_new_file')) + self.assertEqual(stat.S_IMODE(st.st_mode), 0o750) + + @unittest.skipUnless(hasattr(pwd, 'getpwall'), + 'pwd module does not expose getpwall()') + @unittest.skipIf(sys.platform == "vxworks", + "no home directory on VxWorks") + @needs_posix + def test_expanduser_posix(self): + P = self.cls + import_helper.import_module('pwd') + import pwd + pwdent = pwd.getpwuid(os.getuid()) + username = pwdent.pw_name + userhome = pwdent.pw_dir.rstrip('/') or '/' + # Find arbitrary different user (if exists). + for pwdent in pwd.getpwall(): + othername = pwdent.pw_name + otherhome = pwdent.pw_dir.rstrip('/') + if othername != username and otherhome: + break + else: + othername = username + otherhome = userhome + + fakename = 'fakeuser' + # This user can theoretically exist on a test runner. Create unique name: + try: + while pwd.getpwnam(fakename): + fakename += '1' + except KeyError: + pass # Non-existent name found + + p1 = P('~/Documents') + p2 = P(f'~{username}/Documents') + p3 = P(f'~{othername}/Documents') + p4 = P(f'../~{username}/Documents') + p5 = P(f'/~{username}/Documents') + p6 = P('') + p7 = P(f'~{fakename}/Documents') + + with os_helper.EnvironmentVarGuard() as env: + env.unset('HOME') + + self.assertEqual(p1.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + self.assertRaises(RuntimeError, p7.expanduser) + + env['HOME'] = '/tmp' + self.assertEqual(p1.expanduser(), P('/tmp/Documents')) + self.assertEqual(p2.expanduser(), P(userhome) / 'Documents') + self.assertEqual(p3.expanduser(), P(otherhome) / 'Documents') + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + self.assertRaises(RuntimeError, p7.expanduser) + + @unittest.skipIf(sys.platform != "darwin", + "Bad file descriptor in /dev/fd affects only macOS") + @needs_posix + def test_handling_bad_descriptor(self): + try: + file_descriptors = list(pathlib.Path('/dev/fd').rglob("*"))[3:] + if not file_descriptors: + self.skipTest("no file descriptors - issue was not reproduced") + # Checking all file descriptors because there is no guarantee + # which one will fail. + for f in file_descriptors: + f.exists() + f.is_dir() + f.is_file() + f.is_symlink() + f.is_block_device() + f.is_char_device() + f.is_fifo() + f.is_socket() + except OSError as e: + if e.errno == errno.EBADF: + self.fail("Bad file descriptor not handled.") + raise + + @needs_posix + def test_from_uri_posix(self): + P = self.cls + self.assertEqual(P.from_uri('file:/foo/bar'), P('/foo/bar')) + self.assertRaises(ValueError, P.from_uri, 'file://foo/bar') + self.assertEqual(P.from_uri('file:///foo/bar'), P('/foo/bar')) + self.assertEqual(P.from_uri('file:////foo/bar'), P('//foo/bar')) + self.assertEqual(P.from_uri('file://localhost/foo/bar'), P('/foo/bar')) + if not is_wasi: + self.assertEqual(P.from_uri(f'file://{socket.gethostname()}/foo/bar'), + P('/foo/bar')) + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, '/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + @needs_posix + def test_from_uri_pathname2url_posix(self): + P = self.cls + self.assertEqual(P.from_uri(pathname2url('/foo/bar', add_scheme=True)), P('/foo/bar')) + self.assertEqual(P.from_uri(pathname2url('//foo/bar', add_scheme=True)), P('//foo/bar')) + + @needs_windows + def test_absolute_windows(self): + P = self.cls + + # Simple absolute paths. + self.assertEqual(str(P('c:\\').absolute()), 'c:\\') + self.assertEqual(str(P('c:\\a').absolute()), 'c:\\a') + self.assertEqual(str(P('c:\\a\\b').absolute()), 'c:\\a\\b') + + # UNC absolute paths. + share = '\\\\server\\share\\' + self.assertEqual(str(P(share).absolute()), share) + self.assertEqual(str(P(share + 'a').absolute()), share + 'a') + self.assertEqual(str(P(share + 'a\\b').absolute()), share + 'a\\b') + + # UNC relative paths. + with mock.patch("os.getcwd") as getcwd: + getcwd.return_value = share + + self.assertEqual(str(P().absolute()), share) + self.assertEqual(str(P('.').absolute()), share) + self.assertEqual(str(P('a').absolute()), os.path.join(share, 'a')) + self.assertEqual(str(P('a', 'b', 'c').absolute()), + os.path.join(share, 'a', 'b', 'c')) + + drive = os.path.splitdrive(self.base)[0] + with os_helper.change_cwd(self.base): + # Relative path with root + self.assertEqual(str(P('\\').absolute()), drive + '\\') + self.assertEqual(str(P('\\foo').absolute()), drive + '\\foo') + + # Relative path on current drive + self.assertEqual(str(P(drive).absolute()), self.base) + self.assertEqual(str(P(drive + 'foo').absolute()), os.path.join(self.base, 'foo')) + + with os_helper.subst_drive(self.base) as other_drive: + # Set the working directory on the substitute drive + saved_cwd = os.getcwd() + other_cwd = f'{other_drive}\\dirA' + os.chdir(other_cwd) + os.chdir(saved_cwd) + + # Relative path on another drive + self.assertEqual(str(P(other_drive).absolute()), other_cwd) + self.assertEqual(str(P(other_drive + 'foo').absolute()), other_cwd + '\\foo') + + @needs_windows + def test_expanduser_windows(self): + P = self.cls + with os_helper.EnvironmentVarGuard() as env: + env.unset('HOME', 'USERPROFILE', 'HOMEPATH', 'HOMEDRIVE') + env['USERNAME'] = 'alice' + + # test that the path returns unchanged + p1 = P('~/My Documents') + p2 = P('~alice/My Documents') + p3 = P('~bob/My Documents') + p4 = P('/~/My Documents') + p5 = P('d:~/My Documents') + p6 = P('') + self.assertRaises(RuntimeError, p1.expanduser) + self.assertRaises(RuntimeError, p2.expanduser) + self.assertRaises(RuntimeError, p3.expanduser) + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + + def check(): + env.pop('USERNAME', None) + self.assertEqual(p1.expanduser(), + P('C:/Users/alice/My Documents')) + self.assertRaises(RuntimeError, p2.expanduser) + env['USERNAME'] = 'alice' + self.assertEqual(p2.expanduser(), + P('C:/Users/alice/My Documents')) + self.assertEqual(p3.expanduser(), + P('C:/Users/bob/My Documents')) + self.assertEqual(p4.expanduser(), p4) + self.assertEqual(p5.expanduser(), p5) + self.assertEqual(p6.expanduser(), p6) + + env['HOMEPATH'] = 'C:\\Users\\alice' + check() + + env['HOMEDRIVE'] = 'C:\\' + env['HOMEPATH'] = 'Users\\alice' + check() + + env.unset('HOMEDRIVE', 'HOMEPATH') + env['USERPROFILE'] = 'C:\\Users\\alice' + check() + + # bpo-38883: ignore `HOME` when set on windows + env['HOME'] = 'C:\\Users\\eve' + check() + + @needs_windows + def test_from_uri_windows(self): + P = self.cls + # DOS drive paths + self.assertEqual(P.from_uri('file:c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:/c|/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:///c|/path/to/file'), P('c:/path/to/file')) + # UNC paths + self.assertEqual(P.from_uri('file://server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file:////server/path/to/file'), P('//server/path/to/file')) + self.assertEqual(P.from_uri('file://///server/path/to/file'), P('//server/path/to/file')) + # Localhost paths + self.assertEqual(P.from_uri('file://localhost/c:/path/to/file'), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file://localhost/c|/path/to/file'), P('c:/path/to/file')) + # Invalid paths + self.assertRaises(ValueError, P.from_uri, 'foo/bar') + self.assertRaises(ValueError, P.from_uri, 'c:/foo/bar') + self.assertRaises(ValueError, P.from_uri, '//foo/bar') + self.assertRaises(ValueError, P.from_uri, 'file:foo/bar') + self.assertRaises(ValueError, P.from_uri, 'http://foo/bar') + + @needs_windows + def test_from_uri_pathname2url_windows(self): + P = self.cls + self.assertEqual(P.from_uri('file:' + pathname2url(r'c:\path\to\file')), P('c:/path/to/file')) + self.assertEqual(P.from_uri('file:' + pathname2url(r'\\server\path\to\file')), P('//server/path/to/file')) + + @needs_windows + def test_owner_windows(self): + P = self.cls + with self.assertRaises(pathlib.UnsupportedOperation): + P('c:/').owner() + + @needs_windows + def test_group_windows(self): + P = self.cls + with self.assertRaises(pathlib.UnsupportedOperation): + P('c:/').group() + + +class PathWalkTest(unittest.TestCase): + cls = pathlib.Path + base = PathTest.base + can_symlink = PathTest.can_symlink + + def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') + self.walk_path = self.cls(self.base, "TEST1") + self.sub1_path = self.walk_path / "SUB1" + self.sub11_path = self.sub1_path / "SUB11" + self.sub2_path = self.walk_path / "SUB2" + self.link_path = self.sub2_path / "link" + self.sub2_tree = (self.sub2_path, [], ["tmp3"]) + + # Build: + # TESTFN/ + # TEST1/ a file kid and two directory kids + # tmp1 + # SUB1/ a file kid and a directory kid + # tmp2 + # SUB11/ no kids + # SUB2/ a file kid and a dirsymlink kid + # tmp3 + # link/ a symlink to TEST2 + # broken_link + # broken_link2 + # TEST2/ + # tmp4 a lone file + t2_path = self.cls(self.base, "TEST2") + os.makedirs(self.sub11_path) + os.makedirs(self.sub2_path) + os.makedirs(t2_path) + + tmp1_path = self.walk_path / "tmp1" + tmp2_path = self.sub1_path / "tmp2" + tmp3_path = self.sub2_path / "tmp3" + tmp4_path = self.cls(self.base, "TEST2", "tmp4") + for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: + with open(path, "w", encoding='utf-8') as f: + f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n") + + if self.can_symlink: + broken_link_path = self.sub2_path / "broken_link" + broken_link2_path = self.sub2_path / "broken_link2" + os.symlink(t2_path, self.link_path, target_is_directory=True) + os.symlink('broken', broken_link_path) + os.symlink(os.path.join('tmp3', 'broken'), broken_link2_path) + self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"]) + sub21_path= self.sub2_path / "SUB21" + tmp5_path = sub21_path / "tmp3" + broken_link3_path = self.sub2_path / "broken_link3" + + os.makedirs(sub21_path) + tmp5_path.write_text("I am tmp5, blame test_pathlib.") + if self.can_symlink: + os.symlink(tmp5_path, broken_link3_path) + self.sub2_tree[2].append('broken_link3') + self.sub2_tree[2].sort() + os.chmod(sub21_path, 0) + try: + os.listdir(sub21_path) + except PermissionError: + self.sub2_tree[1].append('SUB21') + else: + os.chmod(sub21_path, stat.S_IRWXU) + os.unlink(tmp5_path) + os.rmdir(sub21_path) + + def tearDown(self): + if 'SUB21' in self.sub2_tree[1]: + os.chmod(self.sub2_path / "SUB21", stat.S_IRWXU) + os_helper.rmtree(self.base) + + def test_walk_bad_dir(self): + errors = [] + walk_it = self.walk_path.walk(on_error=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(errors, []) + dir1 = 'SUB1' + path1 = root / dir1 + path1new = (root / dir1).with_suffix(".new") + path1.rename(path1new) + try: + roots = [r for r, _, _ in walk_it] + self.assertTrue(errors) + self.assertNotIn(path1, roots) + self.assertNotIn(path1new, roots) + for dir2 in dirs: + if dir2 != dir1: + self.assertIn(root / dir2, roots) + finally: + path1new.rename(path1) + + def test_walk_many_open_files(self): + depth = 30 + base = self.cls(self.base, 'deep') + path = self.cls(base, *(['d']*depth)) + path.mkdir(parents=True) + + iters = [base.walk(top_down=False) for _ in range(100)] + for i in range(depth + 1): + expected = (path, ['d'] if i else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path.parent + + iters = [base.walk(top_down=True) for _ in range(100)] + path = base + for i in range(depth + 1): + expected = (path, ['d'] if i < depth else [], []) + for it in iters: + self.assertEqual(next(it), expected) + path = path / 'd' + + def test_walk_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = self.cls(self.base, 'deep') + path = base.joinpath(*(['d'] * directory_depth)) + path.mkdir(parents=True) + + with infinite_recursion(recursion_limit): + list(base.walk()) + list(base.walk(top_down=False)) + + @needs_symlinks + def test_walk_follow_symlinks(self): + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.link_path: + self.assertEqual(dirs, []) + self.assertEqual(files, ["tmp4"]) + break + else: + self.fail("Didn't follow symlink with follow_symlinks=True") + + @needs_symlinks + def test_walk_symlink_location(self): + # Tests whether symlinks end up in filenames or dirnames depending + # on the `follow_symlinks` argument. + walk_it = self.walk_path.walk(follow_symlinks=False) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", files) + break + else: + self.fail("symlink not found") + + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", dirs) + break + else: + self.fail("symlink not found") + + +@unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system') +class PosixPathTest(PathTest, PurePosixPathTest): + cls = pathlib.PosixPath + + +@unittest.skipIf(os.name != 'nt', 'test requires a Windows-compatible system') +class WindowsPathTest(PathTest, PureWindowsPathTest): + cls = pathlib.WindowsPath + + +class PathSubclassTest(PathTest): + class cls(pathlib.Path): + pass + + # repr() roundtripping is not supported in custom subclass. + test_repr_roundtrips = None + + +class CompatiblePathTest(unittest.TestCase): + """ + Test that a type can be made compatible with PurePath + derivatives by implementing division operator overloads. + """ + + class CompatPath: + """ + Minimum viable class to test PurePath compatibility. + Simply uses the division operator to join a given + string and the string value of another object with + a forward slash. + """ + def __init__(self, string): + self.string = string + + def __truediv__(self, other): + return type(self)(f"{self.string}/{other}") + + def __rtruediv__(self, other): + return type(self)(f"{other}/{self.string}") + + def test_truediv(self): + result = pathlib.PurePath("test") / self.CompatPath("right") + self.assertIsInstance(result, self.CompatPath) + self.assertEqual(result.string, "test/right") + + with self.assertRaises(TypeError): + # Verify improper operations still raise a TypeError + pathlib.PurePath("test") / 10 + + def test_rtruediv(self): + result = self.CompatPath("left") / pathlib.PurePath("test") + self.assertIsInstance(result, self.CompatPath) + self.assertEqual(result.string, "left/test") + + with self.assertRaises(TypeError): + # Verify improper operations still raise a TypeError + 10 / pathlib.PurePath("test") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_read.py b/Lib/test/test_pathlib/test_read.py new file mode 100644 index 00000000000..482203c290a --- /dev/null +++ b/Lib/test/test_pathlib/test_read.py @@ -0,0 +1,343 @@ +""" +Tests for pathlib.types._ReadablePath +""" + +import collections.abc +import io +import sys +import unittest + +from .support import is_pypi +from .support.local_path import ReadableLocalPath, LocalPathGround +from .support.zip_path import ReadableZipPath, ZipPathGround + +if is_pypi: + from pathlib_abc import PathInfo, _ReadablePath + from pathlib_abc._os import magic_open +else: + from pathlib.types import PathInfo, _ReadablePath + from pathlib._os import magic_open + + +class ReadTestBase: + def setUp(self): + self.root = self.ground.setup() + self.ground.create_hierarchy(self.root) + + def tearDown(self): + self.ground.teardown(self.root) + + def test_is_readable(self): + self.assertIsInstance(self.root, _ReadablePath) + + def test_open_r(self): + p = self.root / 'fileA' + with magic_open(p, 'r', encoding='utf-8') as f: + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.read(), 'this is file A\n') + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + def test_open_r_encoding_warning(self): + p = self.root / 'fileA' + with self.assertWarns(EncodingWarning) as wc: + with magic_open(p, 'r'): + pass + self.assertEqual(wc.filename, __file__) + + def test_open_rb(self): + p = self.root / 'fileA' + with magic_open(p, 'rb') as f: + self.assertEqual(f.read(), b'this is file A\n') + self.assertRaises(ValueError, magic_open, p, 'rb', encoding='utf8') + self.assertRaises(ValueError, magic_open, p, 'rb', errors='strict') + self.assertRaises(ValueError, magic_open, p, 'rb', newline='') + + def test_read_bytes(self): + p = self.root / 'fileA' + self.assertEqual(p.read_bytes(), b'this is file A\n') + + def test_read_text(self): + p = self.root / 'fileA' + self.assertEqual(p.read_text(encoding='utf-8'), 'this is file A\n') + q = self.root / 'abc' + self.ground.create_file(q, b'\xe4bcdefg') + self.assertEqual(q.read_text(encoding='latin-1'), 'äbcdefg') + self.assertEqual(q.read_text(encoding='utf-8', errors='ignore'), 'bcdefg') + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + def test_read_text_encoding_warning(self): + p = self.root / 'fileA' + with self.assertWarns(EncodingWarning) as wc: + p.read_text() + self.assertEqual(wc.filename, __file__) + + def test_read_text_with_newlines(self): + p = self.root / 'abc' + self.ground.create_file(p, b'abcde\r\nfghlk\n\rmnopq') + # Check that `\n` character change nothing + self.assertEqual(p.read_text(encoding='utf-8', newline='\n'), 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r` character replaces `\n` + self.assertEqual(p.read_text(encoding='utf-8', newline='\r'), 'abcde\r\nfghlk\n\rmnopq') + # Check that `\r\n` character replaces `\n` + self.assertEqual(p.read_text(encoding='utf-8', newline='\r\n'), 'abcde\r\nfghlk\n\rmnopq') + + def test_iterdir(self): + expected = ['dirA', 'dirB', 'dirC', 'fileA'] + if self.ground.can_symlink: + expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] + expected = {self.root.joinpath(name) for name in expected} + actual = set(self.root.iterdir()) + self.assertEqual(actual, expected) + + def test_iterdir_nodir(self): + p = self.root / 'fileA' + self.assertRaises(OSError, p.iterdir) + + def test_iterdir_info(self): + for child in self.root.iterdir(): + self.assertIsInstance(child.info, PathInfo) + self.assertTrue(child.info.exists(follow_symlinks=False)) + + def test_glob(self): + if not self.ground.can_symlink: + self.skipTest("requires symlinks") + + p = self.root + sep = self.root.parser.sep + altsep = self.root.parser.altsep + def check(pattern, expected): + if altsep: + expected = {name.replace(altsep, sep) for name in expected} + expected = {p.joinpath(name) for name in expected} + actual = set(p.glob(pattern, recurse_symlinks=True)) + self.assertEqual(actual, expected) + + it = p.glob("fileA") + self.assertIsInstance(it, collections.abc.Iterator) + self.assertEqual(list(it), [p.joinpath("fileA")]) + check("*A", ["dirA", "fileA", "linkA"]) + check("*A", ['dirA', 'fileA', 'linkA']) + check("*B/*", ["dirB/fileB", "linkB/fileB"]) + check("*B/*", ['dirB/fileB', 'linkB/fileB']) + check("brokenLink", ['brokenLink']) + check("brokenLinkLoop", ['brokenLinkLoop']) + check("**/", ["", "dirA/", "dirA/linkC/", "dirB/", "dirC/", "dirC/dirD/", "linkB/"]) + check("**/*/", ["dirA/", "dirA/linkC/", "dirB/", "dirC/", "dirC/dirD/", "linkB/"]) + check("*/", ["dirA/", "dirB/", "dirC/", "linkB/"]) + check("*/dirD/**/", ["dirC/dirD/"]) + check("*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + check("dir*/**", ["dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirB/", "dirB/fileB", "dirC/", + "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt"]) + check("dir*/**/", ["dirA/", "dirA/linkC/", "dirB/", "dirC/", "dirC/dirD/"]) + check("dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", "dirC/..", "dirC/dirD/.."]) + check("dir*/*/**", ["dirA/linkC/", "dirA/linkC/fileB", "dirC/dirD/", "dirC/dirD/fileD"]) + check("dir*/*/**/", ["dirA/linkC/", "dirC/dirD/"]) + check("dir*/*/**/..", ["dirA/linkC/..", "dirC/dirD/.."]) + check("dir*/*/..", ["dirC/dirD/..", "dirA/linkC/.."]) + check("dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + check("dir*/**/fileC", ["dirC/fileC"]) + check("dir*/file*", ["dirB/fileB", "dirC/fileC"]) + check("**/*/fileA", []) + check("fileB", []) + check("**/*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + check("**/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB"]) + check("*/fileB", ["dirB/fileB", "linkB/fileB"]) + check("*/fileB", ['dirB/fileB', 'linkB/fileB']) + check("**/file*", + ["fileA", "dirA/linkC/fileB", "dirB/fileB", "dirC/fileC", "dirC/dirD/fileD", + "linkB/fileB"]) + with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): + list(p.glob('')) + + def test_walk_top_down(self): + it = self.root.walk() + + path, dirnames, filenames = next(it) + dirnames.sort() + filenames.sort() + self.assertEqual(path, self.root) + self.assertEqual(dirnames, ['dirA', 'dirB', 'dirC']) + self.assertEqual(filenames, ['brokenLink', 'brokenLinkLoop', 'fileA', 'linkA', 'linkB'] + if self.ground.can_symlink else ['fileA']) + + path, dirnames, filenames = next(it) + self.assertEqual(path, self.root / 'dirA') + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['linkC'] if self.ground.can_symlink else []) + + path, dirnames, filenames = next(it) + self.assertEqual(path, self.root / 'dirB') + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['fileB']) + + path, dirnames, filenames = next(it) + filenames.sort() + self.assertEqual(path, self.root / 'dirC') + self.assertEqual(dirnames, ['dirD']) + self.assertEqual(filenames, ['fileC', 'novel.txt']) + + path, dirnames, filenames = next(it) + self.assertEqual(path, self.root / 'dirC' / 'dirD') + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['fileD']) + + self.assertRaises(StopIteration, next, it) + + def test_walk_prune(self): + expected = {self.root, self.root / 'dirA', self.root / 'dirC', self.root / 'dirC' / 'dirD'} + actual = set() + for path, dirnames, filenames in self.root.walk(): + actual.add(path) + if path == self.root: + dirnames.remove('dirB') + self.assertEqual(actual, expected) + + def test_walk_bottom_up(self): + seen_root = seen_dira = seen_dirb = seen_dirc = seen_dird = False + for path, dirnames, filenames in self.root.walk(top_down=False): + if path == self.root: + self.assertFalse(seen_root) + self.assertTrue(seen_dira) + self.assertTrue(seen_dirb) + self.assertTrue(seen_dirc) + self.assertEqual(sorted(dirnames), ['dirA', 'dirB', 'dirC']) + self.assertEqual(sorted(filenames), + ['brokenLink', 'brokenLinkLoop', 'fileA', 'linkA', 'linkB'] + if self.ground.can_symlink else ['fileA']) + seen_root = True + elif path == self.root / 'dirA': + self.assertFalse(seen_root) + self.assertFalse(seen_dira) + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['linkC'] if self.ground.can_symlink else []) + seen_dira = True + elif path == self.root / 'dirB': + self.assertFalse(seen_root) + self.assertFalse(seen_dirb) + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['fileB']) + seen_dirb = True + elif path == self.root / 'dirC': + self.assertFalse(seen_root) + self.assertFalse(seen_dirc) + self.assertTrue(seen_dird) + self.assertEqual(dirnames, ['dirD']) + self.assertEqual(sorted(filenames), ['fileC', 'novel.txt']) + seen_dirc = True + elif path == self.root / 'dirC' / 'dirD': + self.assertFalse(seen_root) + self.assertFalse(seen_dirc) + self.assertFalse(seen_dird) + self.assertEqual(dirnames, []) + self.assertEqual(filenames, ['fileD']) + seen_dird = True + else: + raise AssertionError(f"Unexpected path: {path}") + self.assertTrue(seen_root) + + def test_info_exists(self): + p = self.root + self.assertTrue(p.info.exists()) + self.assertTrue((p / 'dirA').info.exists()) + self.assertTrue((p / 'dirA').info.exists(follow_symlinks=False)) + self.assertTrue((p / 'fileA').info.exists()) + self.assertTrue((p / 'fileA').info.exists(follow_symlinks=False)) + self.assertFalse((p / 'non-existing').info.exists()) + self.assertFalse((p / 'non-existing').info.exists(follow_symlinks=False)) + if self.ground.can_symlink: + self.assertTrue((p / 'linkA').info.exists()) + self.assertTrue((p / 'linkA').info.exists(follow_symlinks=False)) + self.assertTrue((p / 'linkB').info.exists()) + self.assertTrue((p / 'linkB').info.exists(follow_symlinks=True)) + self.assertFalse((p / 'brokenLink').info.exists()) + self.assertTrue((p / 'brokenLink').info.exists(follow_symlinks=False)) + self.assertFalse((p / 'brokenLinkLoop').info.exists()) + self.assertTrue((p / 'brokenLinkLoop').info.exists(follow_symlinks=False)) + self.assertFalse((p / 'fileA\udfff').info.exists()) + self.assertFalse((p / 'fileA\udfff').info.exists(follow_symlinks=False)) + self.assertFalse((p / 'fileA\x00').info.exists()) + self.assertFalse((p / 'fileA\x00').info.exists(follow_symlinks=False)) + + def test_info_is_dir(self): + p = self.root + self.assertTrue((p / 'dirA').info.is_dir()) + self.assertTrue((p / 'dirA').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'fileA').info.is_dir()) + self.assertFalse((p / 'fileA').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'non-existing').info.is_dir()) + self.assertFalse((p / 'non-existing').info.is_dir(follow_symlinks=False)) + if self.ground.can_symlink: + self.assertFalse((p / 'linkA').info.is_dir()) + self.assertFalse((p / 'linkA').info.is_dir(follow_symlinks=False)) + self.assertTrue((p / 'linkB').info.is_dir()) + self.assertFalse((p / 'linkB').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'brokenLink').info.is_dir()) + self.assertFalse((p / 'brokenLink').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'brokenLinkLoop').info.is_dir()) + self.assertFalse((p / 'brokenLinkLoop').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'dirA\udfff').info.is_dir()) + self.assertFalse((p / 'dirA\udfff').info.is_dir(follow_symlinks=False)) + self.assertFalse((p / 'dirA\x00').info.is_dir()) + self.assertFalse((p / 'dirA\x00').info.is_dir(follow_symlinks=False)) + + def test_info_is_file(self): + p = self.root + self.assertTrue((p / 'fileA').info.is_file()) + self.assertTrue((p / 'fileA').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'dirA').info.is_file()) + self.assertFalse((p / 'dirA').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'non-existing').info.is_file()) + self.assertFalse((p / 'non-existing').info.is_file(follow_symlinks=False)) + if self.ground.can_symlink: + self.assertTrue((p / 'linkA').info.is_file()) + self.assertFalse((p / 'linkA').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'linkB').info.is_file()) + self.assertFalse((p / 'linkB').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'brokenLink').info.is_file()) + self.assertFalse((p / 'brokenLink').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'brokenLinkLoop').info.is_file()) + self.assertFalse((p / 'brokenLinkLoop').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'fileA\udfff').info.is_file()) + self.assertFalse((p / 'fileA\udfff').info.is_file(follow_symlinks=False)) + self.assertFalse((p / 'fileA\x00').info.is_file()) + self.assertFalse((p / 'fileA\x00').info.is_file(follow_symlinks=False)) + + def test_info_is_symlink(self): + p = self.root + self.assertFalse((p / 'fileA').info.is_symlink()) + self.assertFalse((p / 'dirA').info.is_symlink()) + self.assertFalse((p / 'non-existing').info.is_symlink()) + if self.ground.can_symlink: + self.assertTrue((p / 'linkA').info.is_symlink()) + self.assertTrue((p / 'linkB').info.is_symlink()) + self.assertTrue((p / 'brokenLink').info.is_symlink()) + self.assertFalse((p / 'linkA\udfff').info.is_symlink()) + self.assertFalse((p / 'linkA\x00').info.is_symlink()) + self.assertTrue((p / 'brokenLinkLoop').info.is_symlink()) + self.assertFalse((p / 'fileA\udfff').info.is_symlink()) + self.assertFalse((p / 'fileA\x00').info.is_symlink()) + + +class ZipPathReadTest(ReadTestBase, unittest.TestCase): + ground = ZipPathGround(ReadableZipPath) + + +class LocalPathReadTest(ReadTestBase, unittest.TestCase): + ground = LocalPathGround(ReadableLocalPath) + + +if not is_pypi: + from pathlib import Path + + class PathReadTest(ReadTestBase, unittest.TestCase): + ground = LocalPathGround(Path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pathlib/test_write.py b/Lib/test/test_pathlib/test_write.py new file mode 100644 index 00000000000..15054e804ec --- /dev/null +++ b/Lib/test/test_pathlib/test_write.py @@ -0,0 +1,143 @@ +""" +Tests for pathlib.types._WritablePath +""" + +import io +import os +import sys +import unittest + +from .support import is_pypi +from .support.local_path import WritableLocalPath, LocalPathGround +from .support.zip_path import WritableZipPath, ZipPathGround + +if is_pypi: + from pathlib_abc import _WritablePath + from pathlib_abc._os import magic_open +else: + from pathlib.types import _WritablePath + from pathlib._os import magic_open + + +class WriteTestBase: + def setUp(self): + self.root = self.ground.setup() + + def tearDown(self): + self.ground.teardown(self.root) + + def test_is_writable(self): + self.assertIsInstance(self.root, _WritablePath) + + def test_open_w(self): + p = self.root / 'fileA' + with magic_open(p, 'w', encoding='utf-8') as f: + self.assertIsInstance(f, io.TextIOBase) + f.write('this is file A\n') + self.assertEqual(self.ground.readtext(p), 'this is file A\n') + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + def test_open_w_encoding_warning(self): + p = self.root / 'fileA' + with self.assertWarns(EncodingWarning) as wc: + with magic_open(p, 'w'): + pass + self.assertEqual(wc.filename, __file__) + + def test_open_wb(self): + p = self.root / 'fileA' + with magic_open(p, 'wb') as f: + #self.assertIsInstance(f, io.BufferedWriter) + f.write(b'this is file A\n') + self.assertEqual(self.ground.readbytes(p), b'this is file A\n') + self.assertRaises(ValueError, magic_open, p, 'wb', encoding='utf8') + self.assertRaises(ValueError, magic_open, p, 'wb', errors='strict') + self.assertRaises(ValueError, magic_open, p, 'wb', newline='') + + def test_write_bytes(self): + p = self.root / 'fileA' + data = b'abcdefg' + self.assertEqual(len(data), p.write_bytes(data)) + self.assertEqual(self.ground.readbytes(p), data) + # Check that trying to write str does not truncate the file. + self.assertRaises(TypeError, p.write_bytes, 'somestr') + self.assertEqual(self.ground.readbytes(p), data) + + def test_write_text(self): + p = self.root / 'fileA' + data = 'äbcdefg' + self.assertEqual(len(data), p.write_text(data, encoding='latin-1')) + self.assertEqual(self.ground.readbytes(p), b'\xe4bcdefg') + # Check that trying to write bytes does not truncate the file. + self.assertRaises(TypeError, p.write_text, b'somebytes', encoding='utf-8') + self.assertEqual(self.ground.readbytes(p), b'\xe4bcdefg') + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + def test_write_text_encoding_warning(self): + p = self.root / 'fileA' + with self.assertWarns(EncodingWarning) as wc: + p.write_text('abcdefg') + self.assertEqual(wc.filename, __file__) + + def test_write_text_with_newlines(self): + # Check that `\n` character change nothing + p = self.root / 'fileA' + p.write_text('abcde\r\nfghlk\n\rmnopq', encoding='utf-8', newline='\n') + self.assertEqual(self.ground.readbytes(p), b'abcde\r\nfghlk\n\rmnopq') + + # Check that `\r` character replaces `\n` + p = self.root / 'fileB' + p.write_text('abcde\r\nfghlk\n\rmnopq', encoding='utf-8', newline='\r') + self.assertEqual(self.ground.readbytes(p), b'abcde\r\rfghlk\r\rmnopq') + + # Check that `\r\n` character replaces `\n` + p = self.root / 'fileC' + p.write_text('abcde\r\nfghlk\n\rmnopq', encoding='utf-8', newline='\r\n') + self.assertEqual(self.ground.readbytes(p), b'abcde\r\r\nfghlk\r\n\rmnopq') + + # Check that no argument passed will change `\n` to `os.linesep` + os_linesep_byte = bytes(os.linesep, encoding='ascii') + p = self.root / 'fileD' + p.write_text('abcde\nfghlk\n\rmnopq', encoding='utf-8') + self.assertEqual(self.ground.readbytes(p), + b'abcde' + os_linesep_byte + + b'fghlk' + os_linesep_byte + b'\rmnopq') + + def test_mkdir(self): + p = self.root / 'newdirA' + self.assertFalse(self.ground.isdir(p)) + p.mkdir() + self.assertTrue(self.ground.isdir(p)) + + def test_symlink_to(self): + if not self.ground.can_symlink: + self.skipTest('needs symlinks') + link = self.root.joinpath('linkA') + link.symlink_to('fileA') + self.assertTrue(self.ground.islink(link)) + self.assertEqual(self.ground.readlink(link), 'fileA') + + +class ZipPathWriteTest(WriteTestBase, unittest.TestCase): + ground = ZipPathGround(WritableZipPath) + + +class LocalPathWriteTest(WriteTestBase, unittest.TestCase): + ground = LocalPathGround(WritableLocalPath) + + +if not is_pypi: + from pathlib import Path + + class PathWriteTest(WriteTestBase, unittest.TestCase): + ground = LocalPathGround(Path) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py new file mode 100644 index 00000000000..6ca1fa0ba40 --- /dev/null +++ b/Lib/test/test_patma.py @@ -0,0 +1,3571 @@ +import array +import collections +import dataclasses +import dis +import enum +import inspect +import sys +import unittest + + +@dataclasses.dataclass +class Point: + x: int + y: int + + +class TestCompiler(unittest.TestCase): + + def test_refleaks(self): + # Hunting for leaks using -R doesn't catch leaks in the compiler itself, + # just the code under test. This test ensures that if there are leaks in + # the pattern compiler, those runs will fail: + with open(__file__) as file: + compile(file.read(), __file__, "exec") + + +class TestInheritance(unittest.TestCase): + + @staticmethod + def check_sequence_then_mapping(x): + match x: + case [*_]: + return "seq" + case {}: + return "map" + + @staticmethod + def check_mapping_then_sequence(x): + match x: + case {}: + return "map" + case [*_]: + return "seq" + + def test_multiple_inheritance_mapping(self): + class C: + pass + class M1(collections.UserDict, collections.abc.Sequence): + pass + class M2(C, collections.UserDict, collections.abc.Sequence): + pass + class M3(collections.UserDict, C, list): + pass + class M4(dict, collections.abc.Sequence, C): + pass + self.assertEqual(self.check_sequence_then_mapping(M1()), "map") + self.assertEqual(self.check_sequence_then_mapping(M2()), "map") + self.assertEqual(self.check_sequence_then_mapping(M3()), "map") + self.assertEqual(self.check_sequence_then_mapping(M4()), "map") + self.assertEqual(self.check_mapping_then_sequence(M1()), "map") + self.assertEqual(self.check_mapping_then_sequence(M2()), "map") + self.assertEqual(self.check_mapping_then_sequence(M3()), "map") + self.assertEqual(self.check_mapping_then_sequence(M4()), "map") + + def test_multiple_inheritance_sequence(self): + class C: + pass + class S1(collections.UserList, collections.abc.Mapping): + pass + class S2(C, collections.UserList, collections.abc.Mapping): + pass + class S3(list, C, collections.abc.Mapping): + pass + class S4(collections.UserList, dict, C): + pass + self.assertEqual(self.check_sequence_then_mapping(S1()), "seq") + self.assertEqual(self.check_sequence_then_mapping(S2()), "seq") + self.assertEqual(self.check_sequence_then_mapping(S3()), "seq") + self.assertEqual(self.check_sequence_then_mapping(S4()), "seq") + self.assertEqual(self.check_mapping_then_sequence(S1()), "seq") + self.assertEqual(self.check_mapping_then_sequence(S2()), "seq") + self.assertEqual(self.check_mapping_then_sequence(S3()), "seq") + self.assertEqual(self.check_mapping_then_sequence(S4()), "seq") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_late_registration_mapping(self): + class Parent: + pass + class ChildPre(Parent): + pass + class GrandchildPre(ChildPre): + pass + collections.abc.Mapping.register(Parent) + class ChildPost(Parent): + pass + class GrandchildPost(ChildPost): + pass + self.assertEqual(self.check_sequence_then_mapping(Parent()), "map") + self.assertEqual(self.check_sequence_then_mapping(ChildPre()), "map") + self.assertEqual(self.check_sequence_then_mapping(GrandchildPre()), "map") + self.assertEqual(self.check_sequence_then_mapping(ChildPost()), "map") + self.assertEqual(self.check_sequence_then_mapping(GrandchildPost()), "map") + self.assertEqual(self.check_mapping_then_sequence(Parent()), "map") + self.assertEqual(self.check_mapping_then_sequence(ChildPre()), "map") + self.assertEqual(self.check_mapping_then_sequence(GrandchildPre()), "map") + self.assertEqual(self.check_mapping_then_sequence(ChildPost()), "map") + self.assertEqual(self.check_mapping_then_sequence(GrandchildPost()), "map") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_late_registration_sequence(self): + class Parent: + pass + class ChildPre(Parent): + pass + class GrandchildPre(ChildPre): + pass + collections.abc.Sequence.register(Parent) + class ChildPost(Parent): + pass + class GrandchildPost(ChildPost): + pass + self.assertEqual(self.check_sequence_then_mapping(Parent()), "seq") + self.assertEqual(self.check_sequence_then_mapping(ChildPre()), "seq") + self.assertEqual(self.check_sequence_then_mapping(GrandchildPre()), "seq") + self.assertEqual(self.check_sequence_then_mapping(ChildPost()), "seq") + self.assertEqual(self.check_sequence_then_mapping(GrandchildPost()), "seq") + self.assertEqual(self.check_mapping_then_sequence(Parent()), "seq") + self.assertEqual(self.check_mapping_then_sequence(ChildPre()), "seq") + self.assertEqual(self.check_mapping_then_sequence(GrandchildPre()), "seq") + self.assertEqual(self.check_mapping_then_sequence(ChildPost()), "seq") + self.assertEqual(self.check_mapping_then_sequence(GrandchildPost()), "seq") + + +class TestPatma(unittest.TestCase): + + def test_patma_000(self): + match 0: + case 0: + x = True + self.assertIs(x, True) + + def test_patma_001(self): + match 0: + case 0 if False: + x = False + case 0 if True: + x = True + self.assertIs(x, True) + + def test_patma_002(self): + match 0: + case 0: + x = True + case 0: + x = False + self.assertIs(x, True) + + def test_patma_003(self): + x = False + match 0: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, True) + + def test_patma_004(self): + x = False + match 1: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, True) + + def test_patma_005(self): + x = False + match 2: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, True) + + def test_patma_006(self): + x = False + match 3: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, True) + + def test_patma_007(self): + x = False + match 4: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, False) + + def test_patma_008(self): + x = 0 + class A: + y = 1 + match x: + case A.y as z: + pass + self.assertEqual(x, 0) + self.assertEqual(A.y, 1) + + def test_patma_009(self): + class A: + B = 0 + match 0: + case x if x: + z = 0 + case _ as y if y == x and y: + z = 1 + case A.B: + z = 2 + self.assertEqual(A.B, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 2) + + def test_patma_010(self): + match (): + case []: + x = 0 + self.assertEqual(x, 0) + + def test_patma_011(self): + match (0, 1, 2): + case [*x]: + y = 0 + self.assertEqual(x, [0, 1, 2]) + self.assertEqual(y, 0) + + def test_patma_012(self): + match (0, 1, 2): + case [0, *x]: + y = 0 + self.assertEqual(x, [1, 2]) + self.assertEqual(y, 0) + + def test_patma_013(self): + match (0, 1, 2): + case [0, 1, *x,]: + y = 0 + self.assertEqual(x, [2]) + self.assertEqual(y, 0) + + def test_patma_014(self): + match (0, 1, 2): + case [0, 1, 2, *x]: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_015(self): + match (0, 1, 2): + case [*x, 2,]: + y = 0 + self.assertEqual(x, [0, 1]) + self.assertEqual(y, 0) + + def test_patma_016(self): + match (0, 1, 2): + case [*x, 1, 2]: + y = 0 + self.assertEqual(x, [0]) + self.assertEqual(y, 0) + + def test_patma_017(self): + match (0, 1, 2): + case [*x, 0, 1, 2,]: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_018(self): + match (0, 1, 2): + case [0, *x, 2]: + y = 0 + self.assertEqual(x, [1]) + self.assertEqual(y, 0) + + def test_patma_019(self): + match (0, 1, 2): + case [0, 1, *x, 2,]: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_020(self): + match (0, 1, 2): + case [0, *x, 1, 2]: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_021(self): + match (0, 1, 2): + case [*x,]: + y = 0 + self.assertEqual(x, [0, 1, 2]) + self.assertEqual(y, 0) + + def test_patma_022(self): + x = {} + match x: + case {}: + y = 0 + self.assertEqual(x, {}) + self.assertEqual(y, 0) + + def test_patma_023(self): + x = {0: 0} + match x: + case {}: + y = 0 + self.assertEqual(x, {0: 0}) + self.assertEqual(y, 0) + + def test_patma_024(self): + x = {} + y = None + match x: + case {0: 0}: + y = 0 + self.assertEqual(x, {}) + self.assertIs(y, None) + + def test_patma_025(self): + x = {0: 0} + match x: + case {0: (0 | 1 | 2 as z)}: + y = 0 + self.assertEqual(x, {0: 0}) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_026(self): + x = {0: 1} + match x: + case {0: (0 | 1 | 2 as z)}: + y = 0 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 0) + self.assertEqual(z, 1) + + def test_patma_027(self): + x = {0: 2} + match x: + case {0: (0 | 1 | 2 as z)}: + y = 0 + self.assertEqual(x, {0: 2}) + self.assertEqual(y, 0) + self.assertEqual(z, 2) + + def test_patma_028(self): + x = {0: 3} + y = None + match x: + case {0: (0 | 1 | 2 as z)}: + y = 0 + self.assertEqual(x, {0: 3}) + self.assertIs(y, None) + + def test_patma_029(self): + x = {} + y = None + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {}) + self.assertIs(y, None) + + def test_patma_030(self): + x = {False: (True, 2.0, {})} + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {False: (True, 2.0, {})}) + self.assertEqual(y, 0) + + def test_patma_031(self): + x = {False: (True, 2.0, {}), 1: [[]], 2: 0} + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {False: (True, 2.0, {}), 1: [[]], 2: 0}) + self.assertEqual(y, 0) + + def test_patma_032(self): + x = {False: (True, 2.0, {}), 1: [[]], 2: 0} + match x: + case {0: [1, 2]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {False: (True, 2.0, {}), 1: [[]], 2: 0}) + self.assertEqual(y, 1) + + def test_patma_033(self): + x = [] + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, []) + self.assertEqual(y, 2) + + def test_patma_034(self): + x = {0: 0} + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: ([1, 2, {}] | False)} | {1: [[]]} | {0: [1, 2, {}]} | [] | "X" | {}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {0: 0}) + self.assertEqual(y, 1) + + def test_patma_035(self): + x = {0: 0} + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}] | True} | {1: [[]]} | {0: [1, 2, {}]} | [] | "X" | {}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {0: 0}) + self.assertEqual(y, 1) + + def test_patma_036(self): + x = 0 + match x: + case 0 | 1 | 2: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_037(self): + x = 1 + match x: + case 0 | 1 | 2: + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + + def test_patma_038(self): + x = 2 + match x: + case 0 | 1 | 2: + y = 0 + self.assertEqual(x, 2) + self.assertEqual(y, 0) + + def test_patma_039(self): + x = 3 + y = None + match x: + case 0 | 1 | 2: + y = 0 + self.assertEqual(x, 3) + self.assertIs(y, None) + + def test_patma_040(self): + x = 0 + match x: + case (0 as z) | (1 as z) | (2 as z) if z == x % 2: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_041(self): + x = 1 + match x: + case (0 as z) | (1 as z) | (2 as z) if z == x % 2: + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + self.assertEqual(z, 1) + + def test_patma_042(self): + x = 2 + y = None + match x: + case (0 as z) | (1 as z) | (2 as z) if z == x % 2: + y = 0 + self.assertEqual(x, 2) + self.assertIs(y, None) + self.assertEqual(z, 2) + + def test_patma_043(self): + x = 3 + y = None + match x: + case (0 as z) | (1 as z) | (2 as z) if z == x % 2: + y = 0 + self.assertEqual(x, 3) + self.assertIs(y, None) + + def test_patma_044(self): + x = () + match x: + case []: + y = 0 + self.assertEqual(x, ()) + self.assertEqual(y, 0) + + def test_patma_045(self): + x = () + match x: + case (): + y = 0 + self.assertEqual(x, ()) + self.assertEqual(y, 0) + + def test_patma_046(self): + x = (0,) + match x: + case [0]: + y = 0 + self.assertEqual(x, (0,)) + self.assertEqual(y, 0) + + def test_patma_047(self): + x = ((),) + match x: + case [[]]: + y = 0 + self.assertEqual(x, ((),)) + self.assertEqual(y, 0) + + def test_patma_048(self): + x = [0, 1] + match x: + case [0, 1] | [1, 0]: + y = 0 + self.assertEqual(x, [0, 1]) + self.assertEqual(y, 0) + + def test_patma_049(self): + x = [1, 0] + match x: + case [0, 1] | [1, 0]: + y = 0 + self.assertEqual(x, [1, 0]) + self.assertEqual(y, 0) + + def test_patma_050(self): + x = [0, 0] + y = None + match x: + case [0, 1] | [1, 0]: + y = 0 + self.assertEqual(x, [0, 0]) + self.assertIs(y, None) + + def test_patma_051(self): + w = None + x = [1, 0] + match x: + case [(0 as w)]: + y = 0 + case [z] | [1, (0 | 1 as z)] | [z]: + y = 1 + self.assertIs(w, None) + self.assertEqual(x, [1, 0]) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_052(self): + x = [1, 0] + match x: + case [0]: + y = 0 + case [1, 0] if (x := x[:0]): + y = 1 + case [1, 0]: + y = 2 + self.assertEqual(x, []) + self.assertEqual(y, 2) + + def test_patma_053(self): + x = {0} + y = None + match x: + case [0]: + y = 0 + self.assertEqual(x, {0}) + self.assertIs(y, None) + + def test_patma_054(self): + x = set() + y = None + match x: + case []: + y = 0 + self.assertEqual(x, set()) + self.assertIs(y, None) + + def test_patma_055(self): + x = iter([1, 2, 3]) + y = None + match x: + case []: + y = 0 + self.assertEqual([*x], [1, 2, 3]) + self.assertIs(y, None) + + def test_patma_056(self): + x = {} + y = None + match x: + case []: + y = 0 + self.assertEqual(x, {}) + self.assertIs(y, None) + + def test_patma_057(self): + x = {0: False, 1: True} + y = None + match x: + case [0, 1]: + y = 0 + self.assertEqual(x, {0: False, 1: True}) + self.assertIs(y, None) + + def test_patma_058(self): + x = 0 + match x: + case 0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_059(self): + x = 0 + y = None + match x: + case False: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, None) + + def test_patma_060(self): + x = 0 + y = None + match x: + case 1: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_061(self): + x = 0 + y = None + match x: + case None: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_062(self): + x = 0 + match x: + case 0: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_063(self): + x = 0 + y = None + match x: + case 1: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_064(self): + x = "x" + match x: + case "x": + y = 0 + case "y": + y = 1 + self.assertEqual(x, "x") + self.assertEqual(y, 0) + + def test_patma_065(self): + x = "x" + match x: + case "y": + y = 0 + case "x": + y = 1 + self.assertEqual(x, "x") + self.assertEqual(y, 1) + + def test_patma_066(self): + x = "x" + match x: + case "": + y = 0 + case "x": + y = 1 + self.assertEqual(x, "x") + self.assertEqual(y, 1) + + def test_patma_067(self): + x = b"x" + match x: + case b"y": + y = 0 + case b"x": + y = 1 + self.assertEqual(x, b"x") + self.assertEqual(y, 1) + + def test_patma_068(self): + x = 0 + match x: + case 0 if False: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_069(self): + x = 0 + y = None + match x: + case 0 if 0: + y = 0 + case 0 if 0: + y = 1 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_070(self): + x = 0 + match x: + case 0 if True: + y = 0 + case 0 if True: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_071(self): + x = 0 + match x: + case 0 if 1: + y = 0 + case 0 if 1: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_072(self): + x = 0 + match x: + case 0 if True: + y = 0 + case 0 if True: + y = 1 + y = 2 + self.assertEqual(x, 0) + self.assertEqual(y, 2) + + def test_patma_073(self): + x = 0 + match x: + case 0 if 0: + y = 0 + case 0 if 1: + y = 1 + y = 2 + self.assertEqual(x, 0) + self.assertEqual(y, 2) + + def test_patma_074(self): + x = 0 + y = None + match x: + case 0 if not (x := 1): + y = 0 + case 1: + y = 1 + self.assertEqual(x, 1) + self.assertIs(y, None) + + def test_patma_075(self): + x = "x" + match x: + case ["x"]: + y = 0 + case "x": + y = 1 + self.assertEqual(x, "x") + self.assertEqual(y, 1) + + def test_patma_076(self): + x = b"x" + match x: + case [b"x"]: + y = 0 + case ["x"]: + y = 1 + case [120]: + y = 2 + case b"x": + y = 4 + self.assertEqual(x, b"x") + self.assertEqual(y, 4) + + def test_patma_077(self): + x = bytearray(b"x") + y = None + match x: + case [120]: + y = 0 + case 120: + y = 1 + self.assertEqual(x, b"x") + self.assertIs(y, None) + + def test_patma_078(self): + x = "" + match x: + case []: + y = 0 + case [""]: + y = 1 + case "": + y = 2 + self.assertEqual(x, "") + self.assertEqual(y, 2) + + def test_patma_079(self): + x = "xxx" + match x: + case ["x", "x", "x"]: + y = 0 + case ["xxx"]: + y = 1 + case "xxx": + y = 2 + self.assertEqual(x, "xxx") + self.assertEqual(y, 2) + + def test_patma_080(self): + x = b"xxx" + match x: + case [120, 120, 120]: + y = 0 + case [b"xxx"]: + y = 1 + case b"xxx": + y = 2 + self.assertEqual(x, b"xxx") + self.assertEqual(y, 2) + + def test_patma_081(self): + x = 0 + match x: + case 0 if not (x := 1): + y = 0 + case (0 as z): + y = 1 + self.assertEqual(x, 1) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_082(self): + x = 0 + match x: + case (1 as z) if not (x := 1): + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_083(self): + x = 0 + match x: + case (0 as z): + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_084(self): + x = 0 + y = None + match x: + case (1 as z): + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_085(self): + x = 0 + y = None + match x: + case (0 as z) if (w := 0): + y = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertIs(y, None) + self.assertEqual(z, 0) + + def test_patma_086(self): + x = 0 + match x: + case ((0 as w) as z): + y = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_087(self): + x = 0 + match x: + case (0 | 1) | 2: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_088(self): + x = 1 + match x: + case (0 | 1) | 2: + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + + def test_patma_089(self): + x = 2 + match x: + case (0 | 1) | 2: + y = 0 + self.assertEqual(x, 2) + self.assertEqual(y, 0) + + def test_patma_090(self): + x = 3 + y = None + match x: + case (0 | 1) | 2: + y = 0 + self.assertEqual(x, 3) + self.assertIs(y, None) + + def test_patma_091(self): + x = 0 + match x: + case 0 | (1 | 2): + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_092(self): + x = 1 + match x: + case 0 | (1 | 2): + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + + def test_patma_093(self): + x = 2 + match x: + case 0 | (1 | 2): + y = 0 + self.assertEqual(x, 2) + self.assertEqual(y, 0) + + def test_patma_094(self): + x = 3 + y = None + match x: + case 0 | (1 | 2): + y = 0 + self.assertEqual(x, 3) + self.assertIs(y, None) + + def test_patma_095(self): + x = 0 + match x: + case -0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_096(self): + x = 0 + match x: + case -0.0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_097(self): + x = 0 + match x: + case -0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_098(self): + x = 0 + match x: + case -0.0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_099(self): + x = -1 + match x: + case -1: + y = 0 + self.assertEqual(x, -1) + self.assertEqual(y, 0) + + def test_patma_100(self): + x = -1.5 + match x: + case -1.5: + y = 0 + self.assertEqual(x, -1.5) + self.assertEqual(y, 0) + + def test_patma_101(self): + x = -1j + match x: + case -1j: + y = 0 + self.assertEqual(x, -1j) + self.assertEqual(y, 0) + + def test_patma_102(self): + x = -1.5j + match x: + case -1.5j: + y = 0 + self.assertEqual(x, -1.5j) + self.assertEqual(y, 0) + + def test_patma_103(self): + x = 0 + match x: + case 0 + 0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_104(self): + x = 0 + match x: + case 0 - 0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_105(self): + x = 0 + match x: + case -0 + 0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_106(self): + x = 0 + match x: + case -0 - 0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_107(self): + x = 0.25 + 1.75j + match x: + case 0.25 + 1.75j: + y = 0 + self.assertEqual(x, 0.25 + 1.75j) + self.assertEqual(y, 0) + + def test_patma_108(self): + x = 0.25 - 1.75j + match x: + case 0.25 - 1.75j: + y = 0 + self.assertEqual(x, 0.25 - 1.75j) + self.assertEqual(y, 0) + + def test_patma_109(self): + x = -0.25 + 1.75j + match x: + case -0.25 + 1.75j: + y = 0 + self.assertEqual(x, -0.25 + 1.75j) + self.assertEqual(y, 0) + + def test_patma_110(self): + x = -0.25 - 1.75j + match x: + case -0.25 - 1.75j: + y = 0 + self.assertEqual(x, -0.25 - 1.75j) + self.assertEqual(y, 0) + + def test_patma_111(self): + class A: + B = 0 + x = 0 + match x: + case A.B: + y = 0 + self.assertEqual(A.B, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_112(self): + class A: + class B: + C = 0 + x = 0 + match x: + case A.B.C: + y = 0 + self.assertEqual(A.B.C, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_113(self): + class A: + class B: + C = 0 + D = 1 + x = 1 + match x: + case A.B.C: + y = 0 + case A.B.D: + y = 1 + self.assertEqual(A.B.C, 0) + self.assertEqual(A.B.D, 1) + self.assertEqual(x, 1) + self.assertEqual(y, 1) + + def test_patma_114(self): + class A: + class B: + class C: + D = 0 + x = 0 + match x: + case A.B.C.D: + y = 0 + self.assertEqual(A.B.C.D, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_115(self): + class A: + class B: + class C: + D = 0 + E = 1 + x = 1 + match x: + case A.B.C.D: + y = 0 + case A.B.C.E: + y = 1 + self.assertEqual(A.B.C.D, 0) + self.assertEqual(A.B.C.E, 1) + self.assertEqual(x, 1) + self.assertEqual(y, 1) + + def test_patma_116(self): + match = case = 0 + match match: + case case: + x = 0 + self.assertEqual(match, 0) + self.assertEqual(case, 0) + self.assertEqual(x, 0) + + def test_patma_117(self): + match = case = 0 + match case: + case match: + x = 0 + self.assertEqual(match, 0) + self.assertEqual(case, 0) + self.assertEqual(x, 0) + + def test_patma_118(self): + x = [] + match x: + case [*_, _]: + y = 0 + case []: + y = 1 + self.assertEqual(x, []) + self.assertEqual(y, 1) + + def test_patma_119(self): + x = collections.defaultdict(int) + match x: + case {0: 0}: + y = 0 + case {}: + y = 1 + self.assertEqual(x, {}) + self.assertEqual(y, 1) + + def test_patma_120(self): + x = collections.defaultdict(int) + match x: + case {0: 0}: + y = 0 + case {**z}: + y = 1 + self.assertEqual(x, {}) + self.assertEqual(y, 1) + self.assertEqual(z, {}) + + def test_patma_121(self): + match (): + case (): + x = 0 + self.assertEqual(x, 0) + + def test_patma_122(self): + match (0, 1, 2): + case (*x,): + y = 0 + self.assertEqual(x, [0, 1, 2]) + self.assertEqual(y, 0) + + def test_patma_123(self): + match (0, 1, 2): + case 0, *x: + y = 0 + self.assertEqual(x, [1, 2]) + self.assertEqual(y, 0) + + def test_patma_124(self): + match (0, 1, 2): + case (0, 1, *x,): + y = 0 + self.assertEqual(x, [2]) + self.assertEqual(y, 0) + + def test_patma_125(self): + match (0, 1, 2): + case 0, 1, 2, *x: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_126(self): + match (0, 1, 2): + case *x, 2,: + y = 0 + self.assertEqual(x, [0, 1]) + self.assertEqual(y, 0) + + def test_patma_127(self): + match (0, 1, 2): + case (*x, 1, 2): + y = 0 + self.assertEqual(x, [0]) + self.assertEqual(y, 0) + + def test_patma_128(self): + match (0, 1, 2): + case *x, 0, 1, 2,: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_129(self): + match (0, 1, 2): + case (0, *x, 2): + y = 0 + self.assertEqual(x, [1]) + self.assertEqual(y, 0) + + def test_patma_130(self): + match (0, 1, 2): + case 0, 1, *x, 2,: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_131(self): + match (0, 1, 2): + case (0, *x, 1, 2): + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_132(self): + match (0, 1, 2): + case *x,: + y = 0 + self.assertEqual(x, [0, 1, 2]) + self.assertEqual(y, 0) + + def test_patma_133(self): + x = collections.defaultdict(int, {0: 1}) + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 1 + case {}: + y = 2 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 2) + + def test_patma_134(self): + x = collections.defaultdict(int, {0: 1}) + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 1 + case {**z}: + y = 2 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 2) + self.assertEqual(z, {0: 1}) + + def test_patma_135(self): + x = collections.defaultdict(int, {0: 1}) + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 1 + case {0: _, **z}: + y = 2 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 2) + self.assertEqual(z, {}) + + def test_patma_136(self): + x = {0: 1} + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 0 + case {}: + y = 1 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 1) + + def test_patma_137(self): + x = {0: 1} + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 0 + case {**z}: + y = 1 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 1) + self.assertEqual(z, {0: 1}) + + def test_patma_138(self): + x = {0: 1} + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 0 + case {0: _, **z}: + y = 1 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 1) + self.assertEqual(z, {}) + + def test_patma_139(self): + x = False + match x: + case bool(z): + y = 0 + self.assertIs(x, False) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_140(self): + x = True + match x: + case bool(z): + y = 0 + self.assertIs(x, True) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_141(self): + x = bytearray() + match x: + case bytearray(z): + y = 0 + self.assertEqual(x, bytearray()) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_142(self): + x = b"" + match x: + case bytes(z): + y = 0 + self.assertEqual(x, b"") + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_143(self): + x = {} + match x: + case dict(z): + y = 0 + self.assertEqual(x, {}) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_144(self): + x = 0.0 + match x: + case float(z): + y = 0 + self.assertEqual(x, 0.0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_145(self): + x = frozenset() + match x: + case frozenset(z): + y = 0 + self.assertEqual(x, frozenset()) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_146(self): + x = 0 + match x: + case int(z): + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_147(self): + x = [] + match x: + case list(z): + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_148(self): + x = set() + match x: + case set(z): + y = 0 + self.assertEqual(x, set()) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_149(self): + x = "" + match x: + case str(z): + y = 0 + self.assertEqual(x, "") + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_150(self): + x = () + match x: + case tuple(z): + y = 0 + self.assertEqual(x, ()) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_151(self): + x = 0 + match x,: + case y,: + z = 0 + self.assertEqual(x, 0) + self.assertIs(y, x) + self.assertIs(z, 0) + + def test_patma_152(self): + w = 0 + x = 0 + match w, x: + case y, z: + v = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertIs(y, w) + self.assertIs(z, x) + self.assertEqual(v, 0) + + def test_patma_153(self): + x = 0 + match w := x,: + case y as v,: + z = 0 + self.assertEqual(x, 0) + self.assertIs(y, x) + self.assertEqual(z, 0) + self.assertIs(w, x) + self.assertIs(v, y) + + def test_patma_154(self): + x = 0 + y = None + match x: + case 0 if x: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_155(self): + x = 0 + y = None + match x: + case 1e1000: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_156(self): + x = 0 + match x: + case z: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_157(self): + x = 0 + y = None + match x: + case _ if x: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_158(self): + x = 0 + match x: + case -1e1000: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_159(self): + x = 0 + match x: + case 0 if not x: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_160(self): + x = 0 + z = None + match x: + case 0: + y = 0 + case z if x: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, None) + + def test_patma_161(self): + x = 0 + match x: + case 0: + y = 0 + case _: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_162(self): + x = 0 + match x: + case 1 if x: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_163(self): + x = 0 + y = None + match x: + case 1: + y = 0 + case 1 if not x: + y = 1 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_164(self): + x = 0 + match x: + case 1: + y = 0 + case z: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertIs(z, x) + + def test_patma_165(self): + x = 0 + match x: + case 1 if x: + y = 0 + case _: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_166(self): + x = 0 + match x: + case z if not z: + y = 0 + case 0 if x: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_167(self): + x = 0 + match x: + case z if not z: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_168(self): + x = 0 + match x: + case z if not x: + y = 0 + case z: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_169(self): + x = 0 + match x: + case z if not z: + y = 0 + case _ if x: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_170(self): + x = 0 + match x: + case _ if not x: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_171(self): + x = 0 + y = None + match x: + case _ if x: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_172(self): + x = 0 + z = None + match x: + case _ if not x: + y = 0 + case z if not x: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, None) + + def test_patma_173(self): + x = 0 + match x: + case _ if not x: + y = 0 + case _: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_174(self): + def http_error(status): + match status: + case 400: + return "Bad request" + case 401: + return "Unauthorized" + case 403: + return "Forbidden" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + case _: + return "Something else" + self.assertEqual(http_error(400), "Bad request") + self.assertEqual(http_error(401), "Unauthorized") + self.assertEqual(http_error(403), "Forbidden") + self.assertEqual(http_error(404), "Not found") + self.assertEqual(http_error(418), "I'm a teapot") + self.assertEqual(http_error(123), "Something else") + self.assertEqual(http_error("400"), "Something else") + self.assertEqual(http_error(401 | 403 | 404), "Something else") # 407 + + def test_patma_175(self): + def http_error(status): + match status: + case 400: + return "Bad request" + case 401 | 403 | 404: + return "Not allowed" + case 418: + return "I'm a teapot" + self.assertEqual(http_error(400), "Bad request") + self.assertEqual(http_error(401), "Not allowed") + self.assertEqual(http_error(403), "Not allowed") + self.assertEqual(http_error(404), "Not allowed") + self.assertEqual(http_error(418), "I'm a teapot") + self.assertIs(http_error(123), None) + self.assertIs(http_error("400"), None) + self.assertIs(http_error(401 | 403 | 404), None) # 407 + + def test_patma_176(self): + def whereis(point): + match point: + case (0, 0): + return "Origin" + case (0, y): + return f"Y={y}" + case (x, 0): + return f"X={x}" + case (x, y): + return f"X={x}, Y={y}" + case _: + return "Not a point" + self.assertEqual(whereis((0, 0)), "Origin") + self.assertEqual(whereis((0, -1.0)), "Y=-1.0") + self.assertEqual(whereis(("X", 0)), "X=X") + self.assertEqual(whereis((None, 1j)), "X=None, Y=1j") + self.assertEqual(whereis(42), "Not a point") + + def test_patma_177(self): + def whereis(point): + match point: + case Point(0, 0): + return "Origin" + case Point(0, y): + return f"Y={y}" + case Point(x, 0): + return f"X={x}" + case Point(): + return "Somewhere else" + case _: + return "Not a point" + self.assertEqual(whereis(Point(1, 0)), "X=1") + self.assertEqual(whereis(Point(0, 0)), "Origin") + self.assertEqual(whereis(10), "Not a point") + self.assertEqual(whereis(Point(False, False)), "Origin") + self.assertEqual(whereis(Point(0, -1.0)), "Y=-1.0") + self.assertEqual(whereis(Point("X", 0)), "X=X") + self.assertEqual(whereis(Point(None, 1j)), "Somewhere else") + self.assertEqual(whereis(Point), "Not a point") + self.assertEqual(whereis(42), "Not a point") + + def test_patma_178(self): + def whereis(point): + match point: + case Point(1, var): + return var + self.assertEqual(whereis(Point(1, 0)), 0) + self.assertIs(whereis(Point(0, 0)), None) + + def test_patma_179(self): + def whereis(point): + match point: + case Point(1, y=var): + return var + self.assertEqual(whereis(Point(1, 0)), 0) + self.assertIs(whereis(Point(0, 0)), None) + + def test_patma_180(self): + def whereis(point): + match point: + case Point(x=1, y=var): + return var + self.assertEqual(whereis(Point(1, 0)), 0) + self.assertIs(whereis(Point(0, 0)), None) + + def test_patma_181(self): + def whereis(point): + match point: + case Point(y=var, x=1): + return var + self.assertEqual(whereis(Point(1, 0)), 0) + self.assertIs(whereis(Point(0, 0)), None) + + def test_patma_182(self): + def whereis(points): + match points: + case []: + return "No points" + case [Point(0, 0)]: + return "The origin" + case [Point(x, y)]: + return f"Single point {x}, {y}" + case [Point(0, y1), Point(0, y2)]: + return f"Two on the Y axis at {y1}, {y2}" + case _: + return "Something else" + self.assertEqual(whereis([]), "No points") + self.assertEqual(whereis([Point(0, 0)]), "The origin") + self.assertEqual(whereis([Point(0, 1)]), "Single point 0, 1") + self.assertEqual(whereis([Point(0, 0), Point(0, 0)]), "Two on the Y axis at 0, 0") + self.assertEqual(whereis([Point(0, 1), Point(0, 1)]), "Two on the Y axis at 1, 1") + self.assertEqual(whereis([Point(0, 0), Point(1, 0)]), "Something else") + self.assertEqual(whereis([Point(0, 0), Point(0, 0), Point(0, 0)]), "Something else") + self.assertEqual(whereis([Point(0, 1), Point(0, 1), Point(0, 1)]), "Something else") + + def test_patma_183(self): + def whereis(point): + match point: + case Point(x, y) if x == y: + return f"Y=X at {x}" + case Point(x, y): + return "Not on the diagonal" + self.assertEqual(whereis(Point(0, 0)), "Y=X at 0") + self.assertEqual(whereis(Point(0, False)), "Y=X at 0") + self.assertEqual(whereis(Point(False, 0)), "Y=X at False") + self.assertEqual(whereis(Point(-1 - 1j, -1 - 1j)), "Y=X at (-1-1j)") + self.assertEqual(whereis(Point("X", "X")), "Y=X at X") + self.assertEqual(whereis(Point("X", "x")), "Not on the diagonal") + + def test_patma_184(self): + class Seq(collections.abc.Sequence): + __getitem__ = None + def __len__(self): + return 0 + match Seq(): + case []: + y = 0 + self.assertEqual(y, 0) + + def test_patma_185(self): + class Seq(collections.abc.Sequence): + __getitem__ = None + def __len__(self): + return 42 + match Seq(): + case [*_]: + y = 0 + self.assertEqual(y, 0) + + def test_patma_186(self): + class Seq(collections.abc.Sequence): + def __getitem__(self, i): + return i + def __len__(self): + return 42 + match Seq(): + case [x, *_, y]: + z = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 41) + self.assertEqual(z, 0) + + def test_patma_187(self): + w = range(10) + match w: + case [x, y, *rest]: + z = 0 + self.assertEqual(w, range(10)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + self.assertEqual(rest, list(range(2, 10))) + + def test_patma_188(self): + w = range(100) + match w: + case (x, y, *rest): + z = 0 + self.assertEqual(w, range(100)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + self.assertEqual(rest, list(range(2, 100))) + + def test_patma_189(self): + w = range(1000) + match w: + case x, y, *rest: + z = 0 + self.assertEqual(w, range(1000)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + self.assertEqual(rest, list(range(2, 1000))) + + def test_patma_190(self): + w = range(1 << 10) + match w: + case [x, y, *_]: + z = 0 + self.assertEqual(w, range(1 << 10)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_191(self): + w = range(1 << 20) + match w: + case (x, y, *_): + z = 0 + self.assertEqual(w, range(1 << 20)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_192(self): + w = range(1 << 30) + match w: + case x, y, *_: + z = 0 + self.assertEqual(w, range(1 << 30)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_193(self): + x = {"bandwidth": 0, "latency": 1} + match x: + case {"bandwidth": b, "latency": l}: + y = 0 + self.assertEqual(x, {"bandwidth": 0, "latency": 1}) + self.assertIs(b, x["bandwidth"]) + self.assertIs(l, x["latency"]) + self.assertEqual(y, 0) + + def test_patma_194(self): + x = {"bandwidth": 0, "latency": 1, "key": "value"} + match x: + case {"latency": l, "bandwidth": b}: + y = 0 + self.assertEqual(x, {"bandwidth": 0, "latency": 1, "key": "value"}) + self.assertIs(l, x["latency"]) + self.assertIs(b, x["bandwidth"]) + self.assertEqual(y, 0) + + def test_patma_195(self): + x = {"bandwidth": 0, "latency": 1, "key": "value"} + match x: + case {"bandwidth": b, "latency": l, **rest}: + y = 0 + self.assertEqual(x, {"bandwidth": 0, "latency": 1, "key": "value"}) + self.assertIs(b, x["bandwidth"]) + self.assertIs(l, x["latency"]) + self.assertEqual(rest, {"key": "value"}) + self.assertEqual(y, 0) + + def test_patma_196(self): + x = {"bandwidth": 0, "latency": 1} + match x: + case {"latency": l, "bandwidth": b, **rest}: + y = 0 + self.assertEqual(x, {"bandwidth": 0, "latency": 1}) + self.assertIs(l, x["latency"]) + self.assertIs(b, x["bandwidth"]) + self.assertEqual(rest, {}) + self.assertEqual(y, 0) + + def test_patma_197(self): + w = [Point(-1, 0), Point(1, 2)] + match w: + case (Point(x1, y1), Point(x2, y2) as p2): + z = 0 + self.assertEqual(w, [Point(-1, 0), Point(1, 2)]) + self.assertIs(x1, w[0].x) + self.assertIs(y1, w[0].y) + self.assertIs(p2, w[1]) + self.assertIs(x2, w[1].x) + self.assertIs(y2, w[1].y) + self.assertIs(z, 0) + + def test_patma_198(self): + class Color(enum.Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + def f(color): + match color: + case Color.RED: + return "I see red!" + case Color.GREEN: + return "Grass is green" + case Color.BLUE: + return "I'm feeling the blues :(" + self.assertEqual(f(Color.RED), "I see red!") + self.assertEqual(f(Color.GREEN), "Grass is green") + self.assertEqual(f(Color.BLUE), "I'm feeling the blues :(") + self.assertIs(f(Color), None) + self.assertIs(f(0), None) + self.assertIs(f(1), None) + self.assertIs(f(2), None) + self.assertIs(f(3), None) + self.assertIs(f(False), None) + self.assertIs(f(True), None) + self.assertIs(f(2+0j), None) + self.assertIs(f(3.0), None) + + def test_patma_199(self): + class Color(int, enum.Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + def f(color): + match color: + case Color.RED: + return "I see red!" + case Color.GREEN: + return "Grass is green" + case Color.BLUE: + return "I'm feeling the blues :(" + self.assertEqual(f(Color.RED), "I see red!") + self.assertEqual(f(Color.GREEN), "Grass is green") + self.assertEqual(f(Color.BLUE), "I'm feeling the blues :(") + self.assertIs(f(Color), None) + self.assertEqual(f(0), "I see red!") + self.assertEqual(f(1), "Grass is green") + self.assertEqual(f(2), "I'm feeling the blues :(") + self.assertIs(f(3), None) + self.assertEqual(f(False), "I see red!") + self.assertEqual(f(True), "Grass is green") + self.assertEqual(f(2+0j), "I'm feeling the blues :(") + self.assertIs(f(3.0), None) + + def test_patma_200(self): + class Class: + __match_args__ = ("a", "b") + c = Class() + c.a = 0 + c.b = 1 + match c: + case Class(x, y): + z = 0 + self.assertIs(x, c.a) + self.assertIs(y, c.b) + self.assertEqual(z, 0) + + def test_patma_201(self): + class Class: + __match_args__ = ("a", "b") + c = Class() + c.a = 0 + c.b = 1 + match c: + case Class(x, b=y): + z = 0 + self.assertIs(x, c.a) + self.assertIs(y, c.b) + self.assertEqual(z, 0) + + def test_patma_202(self): + class Parent: + __match_args__ = "a", "b" + class Child(Parent): + __match_args__ = ("c", "d") + c = Child() + c.a = 0 + c.b = 1 + match c: + case Parent(x, y): + z = 0 + self.assertIs(x, c.a) + self.assertIs(y, c.b) + self.assertEqual(z, 0) + + def test_patma_203(self): + class Parent: + __match_args__ = ("a", "b") + class Child(Parent): + __match_args__ = "c", "d" + c = Child() + c.a = 0 + c.b = 1 + match c: + case Parent(x, b=y): + z = 0 + self.assertIs(x, c.a) + self.assertIs(y, c.b) + self.assertEqual(z, 0) + + def test_patma_204(self): + def f(w): + match w: + case 42: + out = locals() + del out["w"] + return out + self.assertEqual(f(42), {}) + self.assertIs(f(0), None) + self.assertEqual(f(42.0), {}) + self.assertIs(f("42"), None) + + def test_patma_205(self): + def f(w): + match w: + case 42.0: + out = locals() + del out["w"] + return out + self.assertEqual(f(42.0), {}) + self.assertEqual(f(42), {}) + self.assertIs(f(0.0), None) + self.assertIs(f(0), None) + + def test_patma_206(self): + def f(w): + match w: + case 1 | 2 | 3: + out = locals() + del out["w"] + return out + self.assertEqual(f(1), {}) + self.assertEqual(f(2), {}) + self.assertEqual(f(3), {}) + self.assertEqual(f(3.0), {}) + self.assertIs(f(0), None) + self.assertIs(f(4), None) + self.assertIs(f("1"), None) + + def test_patma_207(self): + def f(w): + match w: + case [1, 2] | [3, 4]: + out = locals() + del out["w"] + return out + self.assertEqual(f([1, 2]), {}) + self.assertEqual(f([3, 4]), {}) + self.assertIs(f(42), None) + self.assertIs(f([2, 3]), None) + self.assertIs(f([1, 2, 3]), None) + self.assertEqual(f([1, 2.0]), {}) + + def test_patma_208(self): + def f(w): + match w: + case x: + out = locals() + del out["w"] + return out + self.assertEqual(f(42), {"x": 42}) + self.assertEqual(f((1, 2)), {"x": (1, 2)}) + self.assertEqual(f(None), {"x": None}) + + def test_patma_209(self): + def f(w): + match w: + case _: + out = locals() + del out["w"] + return out + self.assertEqual(f(42), {}) + self.assertEqual(f(None), {}) + self.assertEqual(f((1, 2)), {}) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_patma_210(self): + def f(w): + match w: + case (x, y, z): + out = locals() + del out["w"] + return out + self.assertEqual(f((1, 2, 3)), {"x": 1, "y": 2, "z": 3}) + self.assertIs(f((1, 2)), None) + self.assertIs(f((1, 2, 3, 4)), None) + self.assertIs(f(123), None) + self.assertIs(f("abc"), None) + self.assertIs(f(b"abc"), None) + self.assertEqual(f(array.array("b", b"abc")), {'x': 97, 'y': 98, 'z': 99}) + self.assertEqual(f(memoryview(b"abc")), {"x": 97, "y": 98, "z": 99}) + self.assertIs(f(bytearray(b"abc")), None) + + def test_patma_211(self): + def f(w): + match w: + case {"x": x, "y": "y", "z": z}: + out = locals() + del out["w"] + return out + self.assertEqual(f({"x": "x", "y": "y", "z": "z"}), {"x": "x", "z": "z"}) + self.assertEqual(f({"x": "x", "y": "y", "z": "z", "a": "a"}), {"x": "x", "z": "z"}) + self.assertIs(f(({"x": "x", "y": "yy", "z": "z", "a": "a"})), None) + self.assertIs(f(({"x": "x", "y": "y"})), None) + + def test_patma_212(self): + def f(w): + match w: + case Point(int(xx), y="hello"): + out = locals() + del out["w"] + return out + self.assertEqual(f(Point(42, "hello")), {"xx": 42}) + + def test_patma_213(self): + def f(w): + match w: + case (p, q) as x: + out = locals() + del out["w"] + return out + self.assertEqual(f((1, 2)), {"p": 1, "q": 2, "x": (1, 2)}) + self.assertEqual(f([1, 2]), {"p": 1, "q": 2, "x": [1, 2]}) + self.assertIs(f(12), None) + self.assertIs(f((1, 2, 3)), None) + + def test_patma_214(self): + def f(): + match 42: + case 42: + return locals() + self.assertEqual(set(f()), set()) + + def test_patma_215(self): + def f(): + match 1: + case 1 | 2 | 3: + return locals() + self.assertEqual(set(f()), set()) + + def test_patma_216(self): + def f(): + match ...: + case _: + return locals() + self.assertEqual(set(f()), set()) + + def test_patma_217(self): + def f(): + match ...: + case abc: + return locals() + self.assertEqual(set(f()), {"abc"}) + + def test_patma_218(self): + def f(): + match ..., ...: + case a, b: + return locals() + self.assertEqual(set(f()), {"a", "b"}) + + def test_patma_219(self): + def f(): + match {"k": ..., "l": ...}: + case {"k": a, "l": b}: + return locals() + self.assertEqual(set(f()), {"a", "b"}) + + def test_patma_220(self): + def f(): + match Point(..., ...): + case Point(x, y=y): + return locals() + self.assertEqual(set(f()), {"x", "y"}) + + def test_patma_221(self): + def f(): + match ...: + case b as a: + return locals() + self.assertEqual(set(f()), {"a", "b"}) + + def test_patma_222(self): + def f(x): + match x: + case _: + return 0 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 0) + self.assertEqual(f(2), 0) + self.assertEqual(f(3), 0) + + def test_patma_223(self): + def f(x): + match x: + case 0: + return 0 + self.assertEqual(f(0), 0) + self.assertIs(f(1), None) + self.assertIs(f(2), None) + self.assertIs(f(3), None) + + def test_patma_224(self): + def f(x): + match x: + case 0: + return 0 + case _: + return 1 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 1) + self.assertEqual(f(2), 1) + self.assertEqual(f(3), 1) + + def test_patma_225(self): + def f(x): + match x: + case 0: + return 0 + case 1: + return 1 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 1) + self.assertIs(f(2), None) + self.assertIs(f(3), None) + + def test_patma_226(self): + def f(x): + match x: + case 0: + return 0 + case 1: + return 1 + case _: + return 2 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 1) + self.assertEqual(f(2), 2) + self.assertEqual(f(3), 2) + + def test_patma_227(self): + def f(x): + match x: + case 0: + return 0 + case 1: + return 1 + case 2: + return 2 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 1) + self.assertEqual(f(2), 2) + self.assertIs(f(3), None) + + def test_patma_228(self): + match(): + case(): + x = 0 + self.assertEqual(x, 0) + + def test_patma_229(self): + x = 0 + match(x): + case(x): + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_230(self): + x = 0 + match x: + case False: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_231(self): + x = 1 + match x: + case True: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 1) + self.assertEqual(y, 1) + + def test_patma_232(self): + class Eq: + def __eq__(self, other): + return True + x = eq = Eq() + # None + y = None + match x: + case None: + y = 0 + self.assertIs(x, eq) + self.assertEqual(y, None) + # True + y = None + match x: + case True: + y = 0 + self.assertIs(x, eq) + self.assertEqual(y, None) + # False + y = None + match x: + case False: + y = 0 + self.assertIs(x, eq) + self.assertEqual(y, None) + + def test_patma_233(self): + x = False + match x: + case False: + y = 0 + self.assertIs(x, False) + self.assertEqual(y, 0) + + def test_patma_234(self): + x = True + match x: + case True: + y = 0 + self.assertIs(x, True) + self.assertEqual(y, 0) + + def test_patma_235(self): + x = None + match x: + case None: + y = 0 + self.assertIs(x, None) + self.assertEqual(y, 0) + + def test_patma_236(self): + x = 0 + match x: + case (0 as w) as z: + y = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_237(self): + x = 0 + match x: + case (0 as w) as z: + y = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_238(self): + x = ((0, 1), (2, 3)) + match x: + case ((a as b, c as d) as e) as w, ((f as g, h) as i) as z: + y = 0 + self.assertEqual(a, 0) + self.assertEqual(b, 0) + self.assertEqual(c, 1) + self.assertEqual(d, 1) + self.assertEqual(e, (0, 1)) + self.assertEqual(f, 2) + self.assertEqual(g, 2) + self.assertEqual(h, 3) + self.assertEqual(i, (2, 3)) + self.assertEqual(w, (0, 1)) + self.assertEqual(x, ((0, 1), (2, 3))) + self.assertEqual(y, 0) + self.assertEqual(z, (2, 3)) + + def test_patma_239(self): + x = collections.UserDict({0: 1, 2: 3}) + match x: + case {2: 3}: + y = 0 + self.assertEqual(x, {0: 1, 2: 3}) + self.assertEqual(y, 0) + + def test_patma_240(self): + x = collections.UserDict({0: 1, 2: 3}) + match x: + case {2: 3, **z}: + y = 0 + self.assertEqual(x, {0: 1, 2: 3}) + self.assertEqual(y, 0) + self.assertEqual(z, {0: 1}) + + # TODO: RUSTPYTHON + # def test_patma_241(self): + # x = [[{0: 0}]] + # match x: + # case list([({-0-0j: int(real=0+0j, imag=0-0j) | (1) as z},)]): + # y = 0 + # self.assertEqual(x, [[{0: 0}]]) + # self.assertEqual(y, 0) + # self.assertEqual(z, 0) + + def test_patma_242(self): + x = range(3) + match x: + case [y, *_, z]: + w = 0 + self.assertEqual(w, 0) + self.assertEqual(x, range(3)) + self.assertEqual(y, 0) + self.assertEqual(z, 2) + + def test_patma_243(self): + x = range(3) + match x: + case [_, *_, y]: + z = 0 + self.assertEqual(x, range(3)) + self.assertEqual(y, 2) + self.assertEqual(z, 0) + + def test_patma_244(self): + x = range(3) + match x: + case [*_, y]: + z = 0 + self.assertEqual(x, range(3)) + self.assertEqual(y, 2) + self.assertEqual(z, 0) + + def test_patma_245(self): + x = {"y": 1} + match x: + case {"y": (0 as y) | (1 as y)}: + z = 0 + self.assertEqual(x, {"y": 1}) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_246(self): + def f(x): + match x: + case ((a, b, c, d, e, f, g, h, i, 9) | + (h, g, i, a, b, d, e, c, f, 10) | + (g, b, a, c, d, -5, e, h, i, f) | + (-1, d, f, b, g, e, i, a, h, c)): + w = 0 + out = locals() + del out["x"] + return out + alts = [ + dict(a=0, b=1, c=2, d=3, e=4, f=5, g=6, h=7, i=8, w=0), + dict(h=1, g=2, i=3, a=4, b=5, d=6, e=7, c=8, f=9, w=0), + dict(g=0, b=-1, a=-2, c=-3, d=-4, e=-6, h=-7, i=-8, f=-9, w=0), + dict(d=-2, f=-3, b=-4, g=-5, e=-6, i=-7, a=-8, h=-9, c=-10, w=0), + dict(), + ] + self.assertEqual(f(range(10)), alts[0]) + self.assertEqual(f(range(1, 11)), alts[1]) + self.assertEqual(f(range(0, -10, -1)), alts[2]) + self.assertEqual(f(range(-1, -11, -1)), alts[3]) + self.assertEqual(f(range(10, 20)), alts[4]) + + def test_patma_247(self): + def f(x): + match x: + case [y, (a, b, c, d, e, f, g, h, i, 9) | + (h, g, i, a, b, d, e, c, f, 10) | + (g, b, a, c, d, -5, e, h, i, f) | + (-1, d, f, b, g, e, i, a, h, c), z]: + w = 0 + out = locals() + del out["x"] + return out + alts = [ + dict(a=0, b=1, c=2, d=3, e=4, f=5, g=6, h=7, i=8, w=0, y=False, z=True), + dict(h=1, g=2, i=3, a=4, b=5, d=6, e=7, c=8, f=9, w=0, y=False, z=True), + dict(g=0, b=-1, a=-2, c=-3, d=-4, e=-6, h=-7, i=-8, f=-9, w=0, y=False, z=True), + dict(d=-2, f=-3, b=-4, g=-5, e=-6, i=-7, a=-8, h=-9, c=-10, w=0, y=False, z=True), + dict(), + ] + self.assertEqual(f((False, range(10), True)), alts[0]) + self.assertEqual(f((False, range(1, 11), True)), alts[1]) + self.assertEqual(f((False, range(0, -10, -1), True)), alts[2]) + self.assertEqual(f((False, range(-1, -11, -1), True)), alts[3]) + self.assertEqual(f((False, range(10, 20), True)), alts[4]) + + def test_patma_248(self): + class C(dict): + @staticmethod + def get(key, default=None): + return 'bar' + + x = C({'foo': 'bar'}) + match x: + case {'foo': bar}: + y = bar + + self.assertEqual(y, 'bar') + + def test_patma_249(self): + class C: + __attr = "eggs" # mangled to _C__attr + _Outer__attr = "bacon" + class Outer: + def f(self, x): + match x: + # looks up __attr, not _C__attr or _Outer__attr + case C(__attr=y): + return y + c = C() + setattr(c, "__attr", "spam") # setattr is needed because we're in a class scope + self.assertEqual(Outer().f(c), "spam") + + def test_patma_250(self): + def f(x): + match x: + case {"foo": y} if y >= 0: + return True + case {"foo": y} if y < 0: + return False + + self.assertIs(f({"foo": 1}), True) + self.assertIs(f({"foo": -1}), False) + + def test_patma_251(self): + def f(v, x): + match v: + case x.attr if x.attr >= 0: + return True + case x.attr if x.attr < 0: + return False + case _: + return None + + class X: + def __init__(self, attr): + self.attr = attr + + self.assertIs(f(1, X(1)), True) + self.assertIs(f(-1, X(-1)), False) + self.assertIs(f(1, X(-1)), None) + + def test_patma_252(self): + # Side effects must be possible in guards: + effects = [] + def lt(x, y): + effects.append((x, y)) + return x < y + + res = None + match {"foo": 1}: + case {"foo": x} if lt(x, 0): + res = 0 + case {"foo": x} if lt(x, 1): + res = 1 + case {"foo": x} if lt(x, 2): + res = 2 + + self.assertEqual(res, 2) + self.assertEqual(effects, [(1, 0), (1, 1), (1, 2)]) + + def test_patma_253(self): + def f(v): + match v: + case [x] | x: + return x + + self.assertEqual(f(1), 1) + self.assertEqual(f([1]), 1) + + def test_patma_254(self): + def f(v): + match v: + case {"x": x} | x: + return x + + self.assertEqual(f(1), 1) + self.assertEqual(f({"x": 1}), 1) + + def test_patma_255(self): + x = [] + match x: + case [] as z if z.append(None): + y = 0 + case [None]: + y = 1 + self.assertEqual(x, [None]) + self.assertEqual(y, 1) + self.assertIs(z, x) + + def test_patma_runtime_checkable_protocol(self): + # Runtime-checkable protocol + from typing import Protocol, runtime_checkable + + @runtime_checkable + class P(Protocol): + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + + def test_patma_generic_protocol(self): + # Runtime-checkable generic protocol + from typing import Generic, TypeVar, Protocol, runtime_checkable + + T = TypeVar('T') # not using PEP695 to be able to backport changes + + @runtime_checkable + class P(Protocol[T]): + a: T + b: T + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class G(Generic[T]): + def __init__(self, x: T, y: T): + self.x = x + self.y = y + + for cls in (A, G): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P(): + w = 1 + self.assertEqual(w, 0) + + def test_patma_protocol_with_match_args(self): + # Runtime-checkable protocol with `__match_args__` + from typing import Protocol, runtime_checkable + + # Used to fail before + # https://github.com/python/cpython/issues/110682 + @runtime_checkable + class P(Protocol): + __match_args__ = ('x', 'y') + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + j = 0 + match inst: + case P(x=1, y=2): + j = 1 + self.assertEqual(j, 1) + + g = 0 + match inst: + case P(x, y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + g = 1 + self.assertEqual(g, 1) + + h = 0 + match inst: + case P(1, 2): + h = 1 + self.assertEqual(h, 1) + + +class TestSyntaxErrors(unittest.TestCase): + + def assert_syntax_error(self, code: str): + with self.assertRaises(SyntaxError): + compile(inspect.cleandoc(code), "<test>", "exec") + + def test_alternative_patterns_bind_different_names_0(self): + self.assert_syntax_error(""" + match ...: + case "a" | a: + pass + """) + + def test_alternative_patterns_bind_different_names_1(self): + self.assert_syntax_error(""" + match ...: + case [a, [b] | [c] | [d]]: + pass + """) + + + def test_attribute_name_repeated_in_class_pattern(self): + self.assert_syntax_error(""" + match ...: + case Class(a=_, a=_): + pass + """) + + def test_imaginary_number_required_in_complex_literal_0(self): + self.assert_syntax_error(""" + match ...: + case 0+0: + pass + """) + + def test_imaginary_number_required_in_complex_literal_1(self): + self.assert_syntax_error(""" + match ...: + case {0+0: _}: + pass + """) + + def test_invalid_syntax_0(self): + self.assert_syntax_error(""" + match ...: + case {**rest, "key": value}: + pass + """) + + def test_invalid_syntax_1(self): + self.assert_syntax_error(""" + match ...: + case {"first": first, **rest, "last": last}: + pass + """) + + def test_invalid_syntax_2(self): + self.assert_syntax_error(""" + match ...: + case {**_}: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_invalid_syntax_3(self): + self.assert_syntax_error(""" + match ...: + case 42 as _: + pass + """) + + def test_len1_tuple_sequence_pattern_comma(self): + # correct syntax would be `case(*x,):` + self.assert_syntax_error(""" + match ...: + case (*x): + pass + """) + + def test_mapping_pattern_keys_may_only_match_literals_and_attribute_lookups(self): + self.assert_syntax_error(""" + match ...: + case {f"": _}: + pass + """) + + def test_multiple_assignments_to_name_in_pattern_0(self): + self.assert_syntax_error(""" + match ...: + case a, a: + pass + """) + + def test_multiple_assignments_to_name_in_pattern_1(self): + self.assert_syntax_error(""" + match ...: + case {"k": a, "l": a}: + pass + """) + + def test_multiple_assignments_to_name_in_pattern_2(self): + self.assert_syntax_error(""" + match ...: + case MyClass(x, x): + pass + """) + + def test_multiple_assignments_to_name_in_pattern_3(self): + self.assert_syntax_error(""" + match ...: + case MyClass(x=x, y=x): + pass + """) + + def test_multiple_assignments_to_name_in_pattern_4(self): + self.assert_syntax_error(""" + match ...: + case MyClass(x, y=x): + pass + """) + + def test_multiple_assignments_to_name_in_pattern_5(self): + self.assert_syntax_error(""" + match ...: + case a as a: + pass + """) + + def test_multiple_starred_names_in_sequence_pattern_0(self): + self.assert_syntax_error(""" + match ...: + case *a, b, *c, d, *e: + pass + """) + + def test_multiple_starred_names_in_sequence_pattern_1(self): + self.assert_syntax_error(""" + match ...: + case a, *b, c, *d, e: + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_0(self): + self.assert_syntax_error(""" + match ...: + case a | "a": + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_1(self): + self.assert_syntax_error(""" + match 42: + case x: + pass + case y: + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_2(self): + self.assert_syntax_error(""" + match ...: + case x | [_ as x] if x: + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_3(self): + self.assert_syntax_error(""" + match ...: + case x: + pass + case [x] if x: + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_4(self): + self.assert_syntax_error(""" + match ...: + case x: + pass + case _: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_patterns_may_only_match_literals_and_attribute_lookups_0(self): + self.assert_syntax_error(""" + match ...: + case f"": + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_patterns_may_only_match_literals_and_attribute_lookups_1(self): + self.assert_syntax_error(""" + match ...: + case f"{x}": + pass + """) + + def test_real_number_required_in_complex_literal_0(self): + self.assert_syntax_error(""" + match ...: + case 0j+0: + pass + """) + + def test_real_number_required_in_complex_literal_1(self): + self.assert_syntax_error(""" + match ...: + case 0j+0j: + pass + """) + + def test_real_number_required_in_complex_literal_2(self): + self.assert_syntax_error(""" + match ...: + case {0j+0: _}: + pass + """) + + def test_real_number_required_in_complex_literal_3(self): + self.assert_syntax_error(""" + match ...: + case {0j+0j: _}: + pass + """) + + def test_real_number_multiple_ops(self): + self.assert_syntax_error(""" + match ...: + case 0 + 0j + 0: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_real_number_wrong_ops(self): + for op in ["*", "/", "@", "**", "%", "//"]: + with self.subTest(op=op): + self.assert_syntax_error(f""" + match ...: + case 0 {op} 0j: + pass + """) + self.assert_syntax_error(f""" + match ...: + case 0j {op} 0: + pass + """) + self.assert_syntax_error(f""" + match ...: + case -0j {op} 0: + pass + """) + self.assert_syntax_error(f""" + match ...: + case 0j {op} -0: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_0(self): + self.assert_syntax_error(""" + match ...: + case _ | _: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_1(self): + self.assert_syntax_error(""" + match ...: + case (_ as x) | [x]: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_2(self): + self.assert_syntax_error(""" + match ...: + case _ | _ if condition(): + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_3(self): + self.assert_syntax_error(""" + match ...: + case _: + pass + case None: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_4(self): + self.assert_syntax_error(""" + match ...: + case (None | _) | _: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_5(self): + self.assert_syntax_error(""" + match ...: + case _ | (True | False): + pass + """) + + def test_mapping_pattern_duplicate_key(self): + self.assert_syntax_error(""" + match ...: + case {"a": _, "a": _}: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mapping_pattern_duplicate_key_edge_case0(self): + self.assert_syntax_error(""" + match ...: + case {0: _, False: _}: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mapping_pattern_duplicate_key_edge_case1(self): + self.assert_syntax_error(""" + match ...: + case {0: _, 0.0: _}: + pass + """) + + def test_mapping_pattern_duplicate_key_edge_case2(self): + self.assert_syntax_error(""" + match ...: + case {0: _, -0: _}: + pass + """) + + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mapping_pattern_duplicate_key_edge_case3(self): + self.assert_syntax_error(""" + match ...: + case {0: _, 0j: _}: + pass + """) + +class TestTypeErrors(unittest.TestCase): + + def test_accepts_positional_subpatterns_0(self): + class Class: + __match_args__ = () + x = Class() + y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y): + z = 0 + self.assertIs(y, None) + self.assertIs(z, None) + + def test_accepts_positional_subpatterns_1(self): + x = range(10) + y = None + with self.assertRaises(TypeError): + match x: + case range(10): + y = 0 + self.assertEqual(x, range(10)) + self.assertIs(y, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_got_multiple_subpatterns_for_attribute_0(self): + class Class: + __match_args__ = ("a", "a") + a = None + x = Class() + w = y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y, z): + w = 0 + self.assertIs(w, None) + self.assertIs(y, None) + self.assertIs(z, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_got_multiple_subpatterns_for_attribute_1(self): + class Class: + __match_args__ = ("a",) + a = None + x = Class() + w = y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y, a=z): + w = 0 + self.assertIs(w, None) + self.assertIs(y, None) + self.assertIs(z, None) + + def test_match_args_elements_must_be_strings(self): + class Class: + __match_args__ = (None,) + x = Class() + y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y): + z = 0 + self.assertIs(y, None) + self.assertIs(z, None) + + def test_match_args_must_be_a_tuple_0(self): + class Class: + __match_args__ = None + x = Class() + y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y): + z = 0 + self.assertIs(y, None) + self.assertIs(z, None) + + def test_match_args_must_be_a_tuple_1(self): + class Class: + __match_args__ = "XYZ" + x = Class() + y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y): + z = 0 + self.assertIs(y, None) + self.assertIs(z, None) + + def test_match_args_must_be_a_tuple_2(self): + class Class: + __match_args__ = ["spam", "eggs"] + spam = 0 + eggs = 1 + x = Class() + w = y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y, z): + w = 0 + self.assertIs(w, None) + self.assertIs(y, None) + self.assertIs(z, None) + + def test_class_pattern_not_type(self): + w = None + with self.assertRaises(TypeError): + match 1: + case max(0, 1): + w = 0 + self.assertIsNone(w) + + def test_regular_protocol(self): + from typing import Protocol + class P(Protocol): ... + msg = ( + 'Instance and class checks can only be used ' + 'with @runtime_checkable protocols' + ) + w = None + with self.assertRaisesRegex(TypeError, msg): + match 1: + case P(): + w = 0 + self.assertIsNone(w) + + def test_positional_patterns_with_regular_protocol(self): + from typing import Protocol + class P(Protocol): + x: int # no `__match_args__` + y: int + class A: + x = 1 + y = 2 + w = None + with self.assertRaises(TypeError): + match A(): + case P(x, y): + w = 0 + self.assertIsNone(w) + + +class TestValueErrors(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mapping_pattern_checks_duplicate_key_1(self): + class Keys: + KEY = "a" + x = {"a": 0, "b": 1} + w = y = z = None + with self.assertRaises(ValueError): + match x: + case {Keys.KEY: y, "a": z}: + w = 0 + self.assertIs(w, None) + self.assertIs(y, None) + self.assertIs(z, None) + +class TestSourceLocations(unittest.TestCase): + def test_jump_threading(self): + # See gh-123048 + def f(): + x = 0 + v = 1 + match v: + case 1: + if x < 0: + x = 1 + case 2: + if x < 0: + x = 1 + x += 1 + + for inst in dis.get_instructions(f): + if inst.opcode in dis.hasjump: + self.assertIsNotNone(inst.positions.lineno, "jump without location") + +class TestTracing(unittest.TestCase): + + @staticmethod + def _trace(func, *args, **kwargs): + actual_linenos = [] + + def trace(frame, event, arg): + if event == "line" and frame.f_code.co_name == func.__name__: + assert arg is None + relative_lineno = frame.f_lineno - func.__code__.co_firstlineno + actual_linenos.append(relative_lineno) + return trace + + old_trace = sys.gettrace() + sys.settrace(trace) + try: + func(*args, **kwargs) + finally: + sys.settrace(old_trace) + return actual_linenos + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_default_wildcard(self): + def f(command): # 0 + match command.split(): # 1 + case ["go", direction] if direction in "nesw": # 2 + return f"go {direction}" # 3 + case ["go", _]: # 4 + return "no go" # 5 + case _: # 6 + return "default" # 7 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 4, 5]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 4, 6, 7]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_default_capture(self): + def f(command): # 0 + match command.split(): # 1 + case ["go", direction] if direction in "nesw": # 2 + return f"go {direction}" # 3 + case ["go", _]: # 4 + return "no go" # 5 + case x: # 6 + return x # 7 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 4, 5]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 4, 6, 7]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_no_default(self): + def f(command): # 0 + match command.split(): # 1 + case ["go", direction] if direction in "nesw": # 2 + return f"go {direction}" # 3 + case ["go", _]: # 4 + return "no go" # 5 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 4, 5]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 4]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_only_default_wildcard(self): + def f(command): # 0 + match command.split(): # 1 + case _: # 2 + return "default" # 3 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 3]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_only_default_capture(self): + def f(command): # 0 + match command.split(): # 1 + case x: # 2 + return x # 3 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 3]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unreachable_code(self): + def f(command): # 0 + match command: # 1 + case 1: # 2 + if False: # 3 + return 1 # 4 + case _: # 5 + if False: # 6 + return 0 # 7 + + self.assertListEqual(self._trace(f, 1), [1, 2, 3]) + self.assertListEqual(self._trace(f, 0), [1, 2, 5, 6]) + + def test_parser_deeply_nested_patterns(self): + # Deeply nested patterns can cause exponential backtracking when parsing. + # See gh-93671 for more information. + + levels = 100 + + patterns = [ + "A" + "(" * levels + ")" * levels, + "{1:" * levels + "1" + "}" * levels, + "[" * levels + "1" + "]" * levels, + ] + + for pattern in patterns: + with self.subTest(pattern): + code = inspect.cleandoc(""" + match None: + case {}: + pass + """.format(pattern)) + compile(code, "<string>", "exec") + + +if __name__ == "__main__": + """ + # From inside environment using this Python, with pyperf installed: + sudo $(which pyperf) system tune && \ + $(which python) -m test.test_patma --rigorous; \ + sudo $(which pyperf) system reset + """ + import pyperf + + + class PerfPatma(TestPatma): + + def assertEqual(*_, **__): + pass + + def assertIs(*_, **__): + pass + + def assertRaises(*_, **__): + assert False, "this test should be a method of a different class!" + + def run_perf(self, count): + tests = [] + for attr in vars(TestPatma): + if attr.startswith("test_"): + tests.append(getattr(self, attr)) + tests *= count + start = pyperf.perf_counter() + for test in tests: + test() + return pyperf.perf_counter() - start + + + runner = pyperf.Runner() + runner.bench_time_func("patma", PerfPatma().run_perf) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py new file mode 100644 index 00000000000..d9cec936e47 --- /dev/null +++ b/Lib/test/test_peepholer.py @@ -0,0 +1,2759 @@ +import dis +import gc +from itertools import combinations, product +import opcode +import sys +import textwrap +import unittest +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + +from test import support +from test.support.bytecode_helper import ( + BytecodeTestCase, CfgOptimizationTestCase, CompilationStepTestCase) + + +def compile_pattern_with_fast_locals(pattern): + source = textwrap.dedent( + f""" + def f(x): + match x: + case {pattern}: + pass + """ + ) + namespace = {} + exec(source, namespace) + return namespace["f"].__code__ + + +def count_instr_recursively(f, opname): + count = 0 + for instr in dis.get_instructions(f): + if instr.opname == opname: + count += 1 + if hasattr(f, '__code__'): + f = f.__code__ + for c in f.co_consts: + if hasattr(c, 'co_code'): + count += count_instr_recursively(c, opname) + return count + + +def get_binop_argval(arg): + for i, nb_op in enumerate(opcode._nb_ops): + if arg == nb_op[0]: + return i + assert False, f"{arg} is not a valid BINARY_OP argument." + + +class TestTranforms(BytecodeTestCase): + + def check_jump_targets(self, code): + instructions = list(dis.get_instructions(code)) + targets = {instr.offset: instr for instr in instructions} + for instr in instructions: + if 'JUMP_' not in instr.opname: + continue + tgt = targets[instr.argval] + # jump to unconditional jump + if tgt.opname in ('JUMP_BACKWARD', 'JUMP_FORWARD'): + self.fail(f'{instr.opname} at {instr.offset} ' + f'jumps to {tgt.opname} at {tgt.offset}') + # unconditional jump to RETURN_VALUE + if (instr.opname in ('JUMP_BACKWARD', 'JUMP_FORWARD') and + tgt.opname == 'RETURN_VALUE'): + self.fail(f'{instr.opname} at {instr.offset} ' + f'jumps to {tgt.opname} at {tgt.offset}') + + def check_lnotab(self, code): + "Check that the lnotab byte offsets are sensible." + code = dis._get_code_object(code) + lnotab = list(dis.findlinestarts(code)) + # Don't bother checking if the line info is sensible, because + # most of the line info we can get at comes from lnotab. + min_bytecode = min(t[0] for t in lnotab) + max_bytecode = max(t[0] for t in lnotab) + self.assertGreaterEqual(min_bytecode, 0) + self.assertLess(max_bytecode, len(code.co_code)) + # This could conceivably test more (and probably should, as there + # aren't very many tests of lnotab), if peepholer wasn't scheduled + # to be replaced anyway. + + def test_unot(self): + # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' + def unot(x): + if not x == 2: + del x + self.assertNotInBytecode(unot, 'UNARY_NOT') + self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE') + self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') + self.check_lnotab(unot) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_elim_inversion_of_is_or_in(self): + for line, cmp_op, invert in ( + ('not a is b', 'IS_OP', 1,), + ('not a is not b', 'IS_OP', 0,), + ('not a in b', 'CONTAINS_OP', 1,), + ('not a not in b', 'CONTAINS_OP', 0,), + ): + with self.subTest(line=line): + code = compile(line, '', 'single') + self.assertInBytecode(code, cmp_op, invert) + self.check_lnotab(code) + + def test_global_as_constant(self): + # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False + def f(): + x = None + x = None + return x + def g(): + x = True + return x + def h(): + x = False + return x + + for func, elem in ((f, None), (g, True), (h, False)): + with self.subTest(func=func): + self.assertNotInBytecode(func, 'LOAD_GLOBAL') + self.assertInBytecode(func, 'LOAD_CONST', elem) + self.check_lnotab(func) + + def f(): + 'Adding a docstring made this test fail in Py2.5.0' + return None + + self.assertNotInBytecode(f, 'LOAD_GLOBAL') + self.assertInBytecode(f, 'LOAD_CONST', None) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; RETURN_VALUE + def test_while_one(self): + # Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx + def f(): + while 1: + pass + return list + for elem in ('LOAD_CONST', 'POP_JUMP_IF_FALSE'): + self.assertNotInBytecode(f, elem) + for elem in ('JUMP_BACKWARD',): + self.assertInBytecode(f, elem) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pack_unpack(self): + for line, elem in ( + ('a, = a,', 'LOAD_CONST',), + ('a, b = a, b', 'SWAP',), + ('a, b, c = a, b, c', 'SWAP',), + ): + with self.subTest(line=line): + code = compile(line,'','single') + self.assertInBytecode(code, elem) + self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.assertNotInBytecode(code, 'UNPACK_SEQUENCE') + self.check_lnotab(code) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 + def test_constant_folding_tuples_of_constants(self): + for line, elem in ( + ('a = 1,2,3', (1, 2, 3)), + ('("a","b","c")', ('a', 'b', 'c')), + ('a,b,c,d = 1,2,3,4', (1, 2, 3, 4)), + ('(None, 1, None)', (None, 1, None)), + ('((1, 2), 3, 4)', ((1, 2), 3, 4)), + ): + with self.subTest(line=line): + code = compile(line,'','single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.assertNotInBytecode(code, 'BUILD_TUPLE') + self.check_lnotab(code) + + # Long tuples should be folded too. + code = compile(repr(tuple(range(10000))),'','single') + self.assertNotInBytecode(code, 'BUILD_TUPLE') + # One LOAD_CONST for the tuple, one for the None return value + load_consts = [instr for instr in dis.get_instructions(code) + if instr.opname == 'LOAD_CONST'] + self.assertEqual(len(load_consts), 2) + self.check_lnotab(code) + + # Bug 1053819: Tuple of constants misidentified when presented with: + # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . + # The following would segfault upon compilation + def crater(): + (~[ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + ],) + self.check_lnotab(crater) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_lists_of_constants(self): + for line, elem in ( + # in/not in constants with BUILD_LIST should be folded to a tuple: + ('a in [1,2,3]', (1, 2, 3)), + ('a not in ["a","b","c"]', ('a', 'b', 'c')), + ('a in [None, 1, None]', (None, 1, None)), + ('a not in [(1, 2), 3, 4]', ((1, 2), 3, 4)), + ): + with self.subTest(line=line): + code = compile(line, '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.assertNotInBytecode(code, 'BUILD_LIST') + self.check_lnotab(code) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_sets_of_constants(self): + for line, elem in ( + # in/not in constants with BUILD_SET should be folded to a frozenset: + ('a in {1,2,3}', frozenset({1, 2, 3})), + ('a not in {"a","b","c"}', frozenset({'a', 'c', 'b'})), + ('a in {None, 1, None}', frozenset({1, None})), + ('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})), + ('a in {1, 2, 3, 3, 2, 1}', frozenset({1, 2, 3})), + ): + with self.subTest(line=line): + code = compile(line, '', 'single') + self.assertNotInBytecode(code, 'BUILD_SET') + self.assertInBytecode(code, 'LOAD_CONST', elem) + self.check_lnotab(code) + + # Ensure that the resulting code actually works: + def f(a): + return a in {1, 2, 3} + + def g(a): + return a not in {1, 2, 3} + + self.assertTrue(f(3)) + self.assertTrue(not f(4)) + self.check_lnotab(f) + + self.assertTrue(not g(3)) + self.assertTrue(g(4)) + self.check_lnotab(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constant_folding_small_int(self): + tests = [ + ('(0, )[0]', 0), + ('(1 + 2, )[0]', 3), + ('(2 + 2 * 2, )[0]', 6), + ('(1, (1 + 2 + 3, ))[1][0]', 6), + ('1 + 2', 3), + ('2 + 2 * 2 // 2 - 2', 2), + ('(255, )[0]', 255), + ('(256, )[0]', None), + ('(1000, )[0]', None), + ('(1 - 2, )[0]', None), + ('255 + 0', 255), + ('255 + 1', None), + ('-1', None), + ('--1', 1), + ('--255', 255), + ('--256', None), + ('~1', None), + ('~~1', 1), + ('~~255', 255), + ('~~256', None), + ('++255', 255), + ('++256', None), + ] + for expr, oparg in tests: + with self.subTest(expr=expr, oparg=oparg): + code = compile(expr, '', 'single') + if oparg is not None: + self.assertInBytecode(code, 'LOAD_SMALL_INT', oparg) + else: + self.assertNotInBytecode(code, 'LOAD_SMALL_INT') + self.check_lnotab(code) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'UNARY_NEGATIVE' starts with 'UNARY_' + def test_constant_folding_unaryop(self): + intrinsic_positive = 5 + tests = [ + ('-0', 'UNARY_NEGATIVE', None, True, 'LOAD_SMALL_INT', 0), + ('-0.0', 'UNARY_NEGATIVE', None, True, 'LOAD_CONST', -0.0), + ('-(1.0-1.0)', 'UNARY_NEGATIVE', None, True, 'LOAD_CONST', -0.0), + ('-0.5', 'UNARY_NEGATIVE', None, True, 'LOAD_CONST', -0.5), + ('---1', 'UNARY_NEGATIVE', None, True, 'LOAD_CONST', -1), + ('---""', 'UNARY_NEGATIVE', None, False, None, None), + ('~~~1', 'UNARY_INVERT', None, True, 'LOAD_CONST', -2), + ('~~~""', 'UNARY_INVERT', None, False, None, None), + ('not not True', 'UNARY_NOT', None, True, 'LOAD_CONST', True), + ('not not x', 'UNARY_NOT', None, True, 'LOAD_NAME', 'x'), # this should be optimized regardless of constant or not + ('+++1', 'CALL_INTRINSIC_1', intrinsic_positive, True, 'LOAD_SMALL_INT', 1), + ('---x', 'UNARY_NEGATIVE', None, False, None, None), + ('~~~x', 'UNARY_INVERT', None, False, None, None), + ('+++x', 'CALL_INTRINSIC_1', intrinsic_positive, False, None, None), + ('~True', 'UNARY_INVERT', None, False, None, None), + ] + + for ( + expr, + original_opcode, + original_argval, + is_optimized, + optimized_opcode, + optimized_argval, + ) in tests: + with self.subTest(expr=expr, is_optimized=is_optimized): + code = compile(expr, "", "single") + if is_optimized: + self.assertNotInBytecode(code, original_opcode, argval=original_argval) + self.assertInBytecode(code, optimized_opcode, argval=optimized_argval) + else: + self.assertInBytecode(code, original_opcode, argval=original_argval) + self.check_lnotab(code) + + # Check that -0.0 works after marshaling + def negzero(): + return -(1.0-1.0) + + for instr in dis.get_instructions(negzero): + self.assertNotStartsWith(instr.opname, 'UNARY_') + self.check_lnotab(negzero) + + @unittest.expectedFailure # TODO: RUSTPYTHON; BINARY_OP 26 ([]) + def test_constant_folding_binop(self): + tests = [ + ('1 + 2', 'NB_ADD', True, 'LOAD_SMALL_INT', 3), + ('1 + 2 + 3', 'NB_ADD', True, 'LOAD_SMALL_INT', 6), + ('1 + ""', 'NB_ADD', False, None, None), + ('1 - 2', 'NB_SUBTRACT', True, 'LOAD_CONST', -1), + ('1 - 2 - 3', 'NB_SUBTRACT', True, 'LOAD_CONST', -4), + ('1 - ""', 'NB_SUBTRACT', False, None, None), + ('2 * 2', 'NB_MULTIPLY', True, 'LOAD_SMALL_INT', 4), + ('2 * 2 * 2', 'NB_MULTIPLY', True, 'LOAD_SMALL_INT', 8), + ('2 / 2', 'NB_TRUE_DIVIDE', True, 'LOAD_CONST', 1.0), + ('2 / 2 / 2', 'NB_TRUE_DIVIDE', True, 'LOAD_CONST', 0.5), + ('2 / ""', 'NB_TRUE_DIVIDE', False, None, None), + ('2 // 2', 'NB_FLOOR_DIVIDE', True, 'LOAD_SMALL_INT', 1), + ('2 // 2 // 2', 'NB_FLOOR_DIVIDE', True, 'LOAD_SMALL_INT', 0), + ('2 // ""', 'NB_FLOOR_DIVIDE', False, None, None), + ('2 % 2', 'NB_REMAINDER', True, 'LOAD_SMALL_INT', 0), + ('2 % 2 % 2', 'NB_REMAINDER', True, 'LOAD_SMALL_INT', 0), + ('2 % ()', 'NB_REMAINDER', False, None, None), + ('2 ** 2', 'NB_POWER', True, 'LOAD_SMALL_INT', 4), + ('2 ** 2 ** 2', 'NB_POWER', True, 'LOAD_SMALL_INT', 16), + ('2 ** ""', 'NB_POWER', False, None, None), + ('2 << 2', 'NB_LSHIFT', True, 'LOAD_SMALL_INT', 8), + ('2 << 2 << 2', 'NB_LSHIFT', True, 'LOAD_SMALL_INT', 32), + ('2 << ""', 'NB_LSHIFT', False, None, None), + ('2 >> 2', 'NB_RSHIFT', True, 'LOAD_SMALL_INT', 0), + ('2 >> 2 >> 2', 'NB_RSHIFT', True, 'LOAD_SMALL_INT', 0), + ('2 >> ""', 'NB_RSHIFT', False, None, None), + ('2 | 2', 'NB_OR', True, 'LOAD_SMALL_INT', 2), + ('2 | 2 | 2', 'NB_OR', True, 'LOAD_SMALL_INT', 2), + ('2 | ""', 'NB_OR', False, None, None), + ('2 & 2', 'NB_AND', True, 'LOAD_SMALL_INT', 2), + ('2 & 2 & 2', 'NB_AND', True, 'LOAD_SMALL_INT', 2), + ('2 & ""', 'NB_AND', False, None, None), + ('2 ^ 2', 'NB_XOR', True, 'LOAD_SMALL_INT', 0), + ('2 ^ 2 ^ 2', 'NB_XOR', True, 'LOAD_SMALL_INT', 2), + ('2 ^ ""', 'NB_XOR', False, None, None), + ('(1, )[0]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 1), + ('(1, )[-1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 1), + ('(1 + 2, )[0]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 3), + ('(1, (1, 2))[1][1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 2), + ('(1, 2)[2-1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 2), + ('(1, (1, 2))[1][2-1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 2), + ('(1, (1, 2))[1:6][0][2-1]', 'NB_SUBSCR', True, 'LOAD_SMALL_INT', 2), + ('"a"[0]', 'NB_SUBSCR', True, 'LOAD_CONST', 'a'), + ('("a" + "b")[1]', 'NB_SUBSCR', True, 'LOAD_CONST', 'b'), + ('("a" + "b", )[0][1]', 'NB_SUBSCR', True, 'LOAD_CONST', 'b'), + ('("a" * 10)[9]', 'NB_SUBSCR', True, 'LOAD_CONST', 'a'), + ('(1, )[1]', 'NB_SUBSCR', False, None, None), + ('(1, )[-2]', 'NB_SUBSCR', False, None, None), + ('"a"[1]', 'NB_SUBSCR', False, None, None), + ('"a"[-2]', 'NB_SUBSCR', False, None, None), + ('("a" + "b")[2]', 'NB_SUBSCR', False, None, None), + ('("a" + "b", )[0][2]', 'NB_SUBSCR', False, None, None), + ('("a" + "b", )[1][0]', 'NB_SUBSCR', False, None, None), + ('("a" * 10)[10]', 'NB_SUBSCR', False, None, None), + ('(1, (1, 2))[2:6][0][2-1]', 'NB_SUBSCR', False, None, None), + ] + + for ( + expr, + nb_op, + is_optimized, + optimized_opcode, + optimized_argval + ) in tests: + with self.subTest(expr=expr, is_optimized=is_optimized): + code = compile(expr, '', 'single') + nb_op_val = get_binop_argval(nb_op) + if is_optimized: + self.assertNotInBytecode(code, 'BINARY_OP', argval=nb_op_val) + self.assertInBytecode(code, optimized_opcode, argval=optimized_argval) + else: + self.assertInBytecode(code, 'BINARY_OP', argval=nb_op_val) + self.check_lnotab(code) + + # Verify that large sequences do not result from folding + code = compile('"x"*10000', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 10000) + self.assertNotIn("x"*10000, code.co_consts) + self.check_lnotab(code) + code = compile('1<<1000', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 1000) + self.assertNotIn(1<<1000, code.co_consts) + self.check_lnotab(code) + code = compile('2**1000', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 1000) + self.assertNotIn(2**1000, code.co_consts) + self.check_lnotab(code) + + # Test binary subscript on unicode + # valid code get optimized + code = compile('"foo"[0]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', 'f') + self.assertNotInBytecode(code, 'BINARY_OP') + self.check_lnotab(code) + code = compile('"\u0061\uffff"[1]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', '\uffff') + self.assertNotInBytecode(code,'BINARY_OP') + self.check_lnotab(code) + + # With PEP 393, non-BMP char get optimized + code = compile('"\U00012345"[0]', '', 'single') + self.assertInBytecode(code, 'LOAD_CONST', '\U00012345') + self.assertNotInBytecode(code, 'BINARY_OP') + self.check_lnotab(code) + + # invalid code doesn't get optimized + # out of range + code = compile('"fuu"[10]', '', 'single') + self.assertInBytecode(code, 'BINARY_OP') + self.check_lnotab(code) + + + def test_constant_folding_remove_nop_location(self): + sources = [ + """ + (- + - + - + 1) + """, + + """ + (1 + + + 2 + + + 3) + """, + + """ + (1, + 2, + 3)[0] + """, + + """ + [1, + 2, + 3] + """, + + """ + {1, + 2, + 3} + """, + + """ + 1 in [ + 1, + 2, + 3 + ] + """, + + """ + 1 in { + 1, + 2, + 3 + } + """, + + """ + for _ in [1, + 2, + 3]: + pass + """, + + """ + for _ in [1, + 2, + x]: + pass + """, + + """ + for _ in {1, + 2, + 3}: + pass + """ + ] + + for source in sources: + code = compile(textwrap.dedent(source), '', 'single') + self.assertNotInBytecode(code, 'NOP') + + def test_elim_extra_return(self): + # RETURN LOAD_CONST None RETURN --> RETURN + def f(x): + return x + self.assertNotInBytecode(f, 'LOAD_CONST', None) + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertEqual(len(returns), 1) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; KeyError: 20 + def test_elim_jump_to_return(self): + # JUMP_FORWARD to RETURN --> RETURN + def f(cond, true_value, false_value): + # Intentionally use two-line expression to test issue37213. + return (true_value if cond + else false_value) + self.check_jump_targets(f) + self.assertNotInBytecode(f, 'JUMP_FORWARD') + self.assertNotInBytecode(f, 'JUMP_BACKWARD') + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertEqual(len(returns), 2) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; absolute jump encoding + def test_elim_jump_to_uncond_jump(self): + # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump + def f(): + if a: + # Intentionally use two-line expression to test issue37213. + if (c + or d): + foo() + else: + baz() + self.check_jump_targets(f) + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; KeyError: 38 + def test_elim_jump_to_uncond_jump2(self): + # POP_JUMP_IF_FALSE to JUMP_BACKWARD --> POP_JUMP_IF_FALSE to non-jump + def f(): + while a: + # Intentionally use two-line expression to test issue37213. + if (c + or d): + a = foo() + self.check_jump_targets(f) + self.check_lnotab(f) + + def test_elim_jump_to_uncond_jump3(self): + # Intentionally use two-line expressions to test issue37213. + # POP_JUMP_IF_FALSE to POP_JUMP_IF_FALSE --> POP_JUMP_IF_FALSE to non-jump + def f(a, b, c): + return ((a and b) + and c) + self.check_jump_targets(f) + self.check_lnotab(f) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 2) + # POP_JUMP_IF_TRUE to POP_JUMP_IF_TRUE --> POP_JUMP_IF_TRUE to non-jump + def f(a, b, c): + return ((a or b) + or c) + self.check_jump_targets(f) + self.check_lnotab(f) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 2) + # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump + def f(a, b, c): + return ((a and b) + or c) + self.check_jump_targets(f) + self.check_lnotab(f) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 1) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 1) + # POP_JUMP_IF_TRUE to POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE to non-jump + def f(a, b, c): + return ((a or b) + and c) + self.check_jump_targets(f) + self.check_lnotab(f) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 1) + self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON; KeyError: 6 + def test_elim_jump_to_uncond_jump4(self): + def f(): + for i in range(5): + if i > 3: + print(i) + self.check_jump_targets(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; 611 JUMP_BACKWARD 16 + def test_elim_jump_after_return1(self): + # Eliminate dead code: jumps immediately after returns can't be reached + def f(cond1, cond2): + if cond1: return 1 + if cond2: return 2 + while 1: + return 3 + while 1: + if cond1: return 4 + return 5 + return 6 + self.assertNotInBytecode(f, 'JUMP_FORWARD') + self.assertNotInBytecode(f, 'JUMP_BACKWARD') + returns = [instr for instr in dis.get_instructions(f) + if instr.opname == 'RETURN_VALUE'] + self.assertLessEqual(len(returns), 6) + self.check_lnotab(f) + + def test_make_function_doesnt_bail(self): + def f(): + def g()->1+1: + pass + return g + self.assertNotInBytecode(f, 'BINARY_OP') + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; no BUILD_LIST to BUILD_TUPLE optimization + def test_in_literal_list(self): + def containtest(): + return x in [a, b] + self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) + self.check_lnotab(containtest) + + @unittest.expectedFailure # TODO: RUSTPYTHON; no BUILD_LIST to BUILD_TUPLE optimization + def test_iterate_literal_list(self): + def forloop(): + for x in [a, b]: + pass + self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) + self.check_lnotab(forloop) + + def test_condition_with_binop_with_bools(self): + def f(): + if True or False: + return 1 + return 0 + self.assertEqual(f(), 1) + self.check_lnotab(f) + + def test_if_with_if_expression(self): + # Check bpo-37289 + def f(x): + if (True if x else False): + return True + return False + self.assertTrue(f(True)) + self.check_lnotab(f) + + def test_trailing_nops(self): + # Check the lnotab of a function that even after trivial + # optimization has trailing nops, which the lnotab adjustment has to + # handle properly (bpo-38115). + def f(x): + while 1: + return 3 + while 1: + return 5 + return 6 + self.check_lnotab(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 2 != 1 + def test_assignment_idiom_in_comprehensions(self): + def listcomp(): + return [y for x in a for y in [f(x)]] + self.assertEqual(count_instr_recursively(listcomp, 'FOR_ITER'), 1) + def setcomp(): + return {y for x in a for y in [f(x)]} + self.assertEqual(count_instr_recursively(setcomp, 'FOR_ITER'), 1) + def dictcomp(): + return {y: y for x in a for y in [f(x)]} + self.assertEqual(count_instr_recursively(dictcomp, 'FOR_ITER'), 1) + def genexpr(): + return (y for x in a for y in [f(x)]) + self.assertEqual(count_instr_recursively(genexpr, 'FOR_ITER'), 1) + + @support.requires_resource('cpu') + def test_format_combinations(self): + flags = '-+ #0' + testcases = [ + *product(('', '1234', 'абвг'), 'sra'), + *product((1234, -1234), 'duioxX'), + *product((1234.5678901, -1234.5678901), 'duifegFEG'), + *product((float('inf'), -float('inf')), 'fegFEG'), + ] + width_precs = [ + *product(('', '1', '30'), ('', '.', '.0', '.2')), + ('', '.40'), + ('30', '.40'), + ] + for value, suffix in testcases: + for width, prec in width_precs: + for r in range(len(flags) + 1): + for spec in combinations(flags, r): + fmt = '%' + ''.join(spec) + width + prec + suffix + with self.subTest(fmt=fmt, value=value): + s1 = fmt % value + s2 = eval(f'{fmt!r} % (x,)', {'x': value}) + self.assertEqual(s2, s1, f'{fmt = }') + + def test_format_misc(self): + def format(fmt, *values): + vars = [f'x{i+1}' for i in range(len(values))] + if len(vars) == 1: + args = '(' + vars[0] + ',)' + else: + args = '(' + ', '.join(vars) + ')' + return eval(f'{fmt!r} % {args}', dict(zip(vars, values))) + + self.assertEqual(format('string'), 'string') + self.assertEqual(format('x = %s!', 1234), 'x = 1234!') + self.assertEqual(format('x = %d!', 1234), 'x = 1234!') + self.assertEqual(format('x = %x!', 1234), 'x = 4d2!') + self.assertEqual(format('x = %f!', 1234), 'x = 1234.000000!') + self.assertEqual(format('x = %s!', 1234.0000625), 'x = 1234.0000625!') + self.assertEqual(format('x = %f!', 1234.0000625), 'x = 1234.000063!') + self.assertEqual(format('x = %d!', 1234.0000625), 'x = 1234!') + self.assertEqual(format('x = %s%% %%%%', 1234), 'x = 1234% %%') + self.assertEqual(format('x = %s!', '%% %s'), 'x = %% %s!') + self.assertEqual(format('x = %s, y = %d', 12, 34), 'x = 12, y = 34') + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: unsupported format character 'z' (0x7a) at index 3 + def test_format_errors(self): + with self.assertRaisesRegex(TypeError, + 'not enough arguments for format string'): + eval("'%s' % ()") + with self.assertRaisesRegex(TypeError, + 'not all arguments converted during string formatting'): + eval("'%s' % (x, y)", {'x': 1, 'y': 2}) + with self.assertRaisesRegex(ValueError, 'incomplete format'): + eval("'%s%' % (x,)", {'x': 1234}) + with self.assertRaisesRegex(ValueError, 'incomplete format'): + eval("'%s%%%' % (x,)", {'x': 1234}) + with self.assertRaisesRegex(TypeError, + 'not enough arguments for format string'): + eval("'%s%z' % (x,)", {'x': 1234}) + with self.assertRaisesRegex(ValueError, 'unsupported format character'): + eval("'%s%z' % (x, 5)", {'x': 1234}) + with self.assertRaisesRegex(TypeError, 'a real number is required, not str'): + eval("'%d' % (x,)", {'x': '1234'}) + with self.assertRaisesRegex(TypeError, 'an integer is required, not float'): + eval("'%x' % (x,)", {'x': 1234.56}) + with self.assertRaisesRegex(TypeError, 'an integer is required, not str'): + eval("'%x' % (x,)", {'x': '1234'}) + with self.assertRaisesRegex(TypeError, 'must be real number, not str'): + eval("'%f' % (x,)", {'x': '1234'}) + with self.assertRaisesRegex(TypeError, + 'not enough arguments for format string'): + eval("'%s, %s' % (x, *y)", {'x': 1, 'y': []}) + with self.assertRaisesRegex(TypeError, + 'not all arguments converted during string formatting'): + eval("'%s, %s' % (x, *y)", {'x': 1, 'y': [2, 3]}) + + def test_static_swaps_unpack_two(self): + def f(a, b): + a, b = a, b + b, a = a, b + self.assertNotInBytecode(f, "SWAP") + + def test_static_swaps_unpack_three(self): + def f(a, b, c): + a, b, c = a, b, c + a, c, b = a, b, c + b, a, c = a, b, c + b, c, a = a, b, c + c, a, b = a, b, c + c, b, a = a, b, c + self.assertNotInBytecode(f, "SWAP") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_static_swaps_match_mapping(self): + for a, b, c in product("_a", "_b", "_c"): + pattern = f"{{'a': {a}, 'b': {b}, 'c': {c}}}" + with self.subTest(pattern): + code = compile_pattern_with_fast_locals(pattern) + self.assertNotInBytecode(code, "SWAP") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_static_swaps_match_class(self): + forms = [ + "C({}, {}, {})", + "C({}, {}, c={})", + "C({}, b={}, c={})", + "C(a={}, b={}, c={})" + ] + for a, b, c in product("_a", "_b", "_c"): + for form in forms: + pattern = form.format(a, b, c) + with self.subTest(pattern): + code = compile_pattern_with_fast_locals(pattern) + self.assertNotInBytecode(code, "SWAP") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_static_swaps_match_sequence(self): + swaps = {"*_, b, c", "a, *_, c", "a, b, *_"} + forms = ["{}, {}, {}", "{}, {}, *{}", "{}, *{}, {}", "*{}, {}, {}"] + for a, b, c in product("_a", "_b", "_c"): + for form in forms: + pattern = form.format(a, b, c) + with self.subTest(pattern): + code = compile_pattern_with_fast_locals(pattern) + if pattern in swaps: + # If this fails... great! Remove this pattern from swaps + # to prevent regressing on any improvement: + self.assertInBytecode(code, "SWAP") + else: + self.assertNotInBytecode(code, "SWAP") + + +class TestBuglets(unittest.TestCase): + + def test_bug_11510(self): + # folded constant set optimization was commingled with the tuple + # unpacking optimization which would fail if the set had duplicate + # elements so that the set length was unexpected + def f(): + x, y = {1, 1} + return x, y + with self.assertRaises(ValueError): + f() + + def test_bpo_42057(self): + for i in range(10): + try: + raise Exception + except Exception or Exception: + pass + + def test_bpo_45773_pop_jump_if_true(self): + compile("while True or spam: pass", "<test>", "exec") + + def test_bpo_45773_pop_jump_if_false(self): + compile("while True or not spam: pass", "<test>", "exec") + + +class TestMarkingVariablesAsUnKnown(BytecodeTestCase): + + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; BINARY_OP 0 (+) + def test_load_fast_known_simple(self): + def f(): + x = 1 + y = x + x + self.assertInBytecode(f, 'LOAD_FAST_BORROW_LOAD_FAST_BORROW') + + @unittest.expectedFailure # TODO: RUSTPYTHON; RETURN_VALUE + def test_load_fast_unknown_simple(self): + def f(): + if condition(): + x = 1 + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + @unittest.expectedFailure # TODO: RUSTPYTHON; RETURN_VALUE + def test_load_fast_unknown_because_del(self): + def f(): + x = 1 + del x + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + def test_load_fast_known_because_parameter(self): + def f1(x): + print(x) + self.assertInBytecode(f1, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f1, 'LOAD_FAST_CHECK') + + def f2(*, x): + print(x) + self.assertInBytecode(f2, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f2, 'LOAD_FAST_CHECK') + + def f3(*args): + print(args) + self.assertInBytecode(f3, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f3, 'LOAD_FAST_CHECK') + + def f4(**kwargs): + print(kwargs) + self.assertInBytecode(f4, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f4, 'LOAD_FAST_CHECK') + + def f5(x=0): + print(x) + self.assertInBytecode(f5, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f5, 'LOAD_FAST_CHECK') + + @unittest.expectedFailure # TODO: RUSTPYTHON; RETURN_VALUE + def test_load_fast_known_because_already_loaded(self): + def f(): + if condition(): + x = 1 + print(x) + print(x) + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertInBytecode(f, 'LOAD_FAST_BORROW') + + def test_load_fast_known_multiple_branches(self): + def f(): + if condition(): + x = 1 + else: + x = 2 + print(x) + self.assertInBytecode(f, 'LOAD_FAST_BORROW') + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK') + + @unittest.expectedFailure # TODO: RUSTPYTHON; L5 to L6 -> L6 [1] lasti + def test_load_fast_unknown_after_error(self): + def f(): + try: + res = 1 / 0 + except ZeroDivisionError: + pass + return res + # LOAD_FAST (known) still occurs in the no-exception branch. + # Assert that it doesn't occur in the LOAD_FAST_CHECK branch. + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + + @unittest.expectedFailure # TODO: RUSTPYTHON; L5 to L6 -> L6 [1] lasti + def test_load_fast_unknown_after_error_2(self): + def f(): + try: + 1 / 0 + except ZeroDivisionError: + print(a, b, c, d, e, f, g) + a = b = c = d = e = f = g = 1 + self.assertInBytecode(f, 'LOAD_FAST_CHECK') + self.assertNotInBytecode(f, 'LOAD_FAST') + + @unittest.expectedFailure # TODO: RUSTPYTHON; RETURN_VALUE + def test_load_fast_too_many_locals(self): + # When there get to be too many locals to analyze completely, + # later locals are all converted to LOAD_FAST_CHECK, except + # when a store or prior load occurred in the same basicblock. + def f(): + a00 = a01 = a02 = a03 = a04 = a05 = a06 = a07 = a08 = a09 = 1 + a10 = a11 = a12 = a13 = a14 = a15 = a16 = a17 = a18 = a19 = 1 + a20 = a21 = a22 = a23 = a24 = a25 = a26 = a27 = a28 = a29 = 1 + a30 = a31 = a32 = a33 = a34 = a35 = a36 = a37 = a38 = a39 = 1 + a40 = a41 = a42 = a43 = a44 = a45 = a46 = a47 = a48 = a49 = 1 + a50 = a51 = a52 = a53 = a54 = a55 = a56 = a57 = a58 = a59 = 1 + a60 = a61 = a62 = a63 = a64 = a65 = a66 = a67 = a68 = a69 = 1 + a70 = a71 = a72 = a73 = a74 = a75 = a76 = a77 = a78 = a79 = 1 + del a72, a73 + print(a73) + print(a70, a71, a72, a73) + while True: + print(a00, a01, a62, a63) + print(a64, a65, a78, a79) + + self.assertInBytecode(f, 'LOAD_FAST_BORROW_LOAD_FAST_BORROW', ("a00", "a01")) + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', "a00") + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', "a01") + for i in 62, 63: + # First 64 locals: analyze completely + self.assertInBytecode(f, 'LOAD_FAST_BORROW', f"a{i:02}") + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") + for i in 64, 65, 78, 79: + # Locals >=64 not in the same basicblock + self.assertInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") + self.assertNotInBytecode(f, 'LOAD_FAST', f"a{i:02}") + for i in 70, 71: + # Locals >=64 in the same basicblock + self.assertInBytecode(f, 'LOAD_FAST_BORROW', f"a{i:02}") + self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") + # del statements should invalidate within basicblocks. + self.assertInBytecode(f, 'LOAD_FAST_CHECK', "a72") + self.assertNotInBytecode(f, 'LOAD_FAST', "a72") + # previous checked loads within a basicblock enable unchecked loads + self.assertInBytecode(f, 'LOAD_FAST_CHECK', "a73") + self.assertInBytecode(f, 'LOAD_FAST_BORROW', "a73") + + def test_setting_lineno_no_undefined(self): + code = textwrap.dedent("""\ + def f(): + x = y = 2 + if not x: + return 4 + for i in range(55): + x + 6 + L = 7 + L = 8 + L = 9 + L = 10 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + co_code = f.__code__.co_code + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 9: + frame.f_lineno = 3 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + result = f() + self.assertIsNone(result) + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + self.assertEqual(f.__code__.co_code, co_code) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_setting_lineno_one_undefined(self): + code = textwrap.dedent("""\ + def f(): + x = y = 2 + if not x: + return 4 + for i in range(55): + x + 6 + del x + L = 8 + L = 9 + L = 10 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + co_code = f.__code__.co_code + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 9: + frame.f_lineno = 3 + sys.settrace(None) + return None + return trace + e = r"assigning None to 1 unbound local" + with self.assertWarnsRegex(RuntimeWarning, e): + sys.settrace(trace) + result = f() + self.assertEqual(result, 4) + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + self.assertEqual(f.__code__.co_code, co_code) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered + def test_setting_lineno_two_undefined(self): + code = textwrap.dedent("""\ + def f(): + x = y = 2 + if not x: + return 4 + for i in range(55): + x + 6 + del x, y + L = 8 + L = 9 + L = 10 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + co_code = f.__code__.co_code + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 9: + frame.f_lineno = 3 + sys.settrace(None) + return None + return trace + e = r"assigning None to 2 unbound locals" + with self.assertWarnsRegex(RuntimeWarning, e): + sys.settrace(trace) + result = f() + self.assertEqual(result, 4) + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + self.assertEqual(f.__code__.co_code, co_code) + + def make_function_with_no_checks(self): + code = textwrap.dedent("""\ + def f(): + x = 2 + L = 3 + L = 4 + L = 5 + if not L: + x + 7 + y = 2 + """) + ns = {} + exec(code, ns) + f = ns['f'] + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + return f + + def test_modifying_local_does_not_add_check(self): + f = self.make_function_with_no_checks() + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 4: + frame.f_locals["x"] = 42 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + f() + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + + def test_initializing_local_does_not_add_check(self): + f = self.make_function_with_no_checks() + def trace(frame, event, arg): + if event == 'line' and frame.f_lineno == 4: + frame.f_locals["y"] = 42 + sys.settrace(None) + return None + return trace + sys.settrace(trace) + f() + self.assertInBytecode(f, "LOAD_FAST_BORROW") + self.assertNotInBytecode(f, "LOAD_FAST_CHECK") + + +class DirectCfgOptimizerTests(CfgOptimizationTestCase): + + def cfg_optimization_test(self, insts, expected_insts, + consts=None, expected_consts=None, + nlocals=0): + + self.check_instructions(insts) + self.check_instructions(expected_insts) + + if expected_consts is None: + expected_consts = consts + seq = self.seq_from_insts(insts) + opt_insts, opt_consts = self.get_optimized(seq, consts, nlocals) + expected_insts = self.seq_from_insts(expected_insts).get_instructions() + self.assertInstructionsMatch(opt_insts, expected_insts) + self.assertEqual(opt_consts, expected_consts) + + def test_conditional_jump_forward_non_const_condition(self): + insts = [ + ('LOAD_NAME', 1, 11), + ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), + ('LOAD_CONST', 2, 13), + ('RETURN_VALUE', None, 13), + lbl, + ('LOAD_CONST', 3, 14), + ('RETURN_VALUE', None, 14), + ] + expected_insts = [ + ('LOAD_NAME', 1, 11), + ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), + ('LOAD_SMALL_INT', 2, 13), + ('RETURN_VALUE', None, 13), + lbl, + ('LOAD_SMALL_INT', 3, 14), + ('RETURN_VALUE', None, 14), + ] + self.cfg_optimization_test(insts, + expected_insts, + consts=[0, 1, 2, 3, 4], + expected_consts=[0]) + + def test_list_exceeding_stack_use_guideline(self): + def f(): + return [ + 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, + 35, 36, 37, 38, 39 + ] + self.assertEqual(f(), list(range(40))) + + def test_set_exceeding_stack_use_guideline(self): + def f(): + return { + 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, + 35, 36, 37, 38, 39 + } + self.assertEqual(f(), frozenset(range(40))) + + def test_nested_const_foldings(self): + # (1, (--2 + ++2 * 2 // 2 - 2, )[0], ~~3, not not True) ==> (1, 2, 3, True) + intrinsic_positive = 5 + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('UNARY_NEGATIVE', None, 0), + ('NOP', None, 0), + ('UNARY_NEGATIVE', None, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('CALL_INTRINSIC_1', intrinsic_positive, 0), + ('NOP', None, 0), + ('CALL_INTRINSIC_1', intrinsic_positive, 0), + ('BINARY_OP', get_binop_argval('NB_MULTIPLY')), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('BINARY_OP', get_binop_argval('NB_FLOOR_DIVIDE')), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', get_binop_argval('NB_ADD')), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('BINARY_OP', get_binop_argval('NB_SUBTRACT')), + ('NOP', None, 0), + ('BUILD_TUPLE', 1, 0), + ('LOAD_SMALL_INT', 0, 0), + ('BINARY_OP', get_binop_argval('NB_SUBSCR'), 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('UNARY_INVERT', None, 0), + ('NOP', None, 0), + ('UNARY_INVERT', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('UNARY_NOT', None, 0), + ('NOP', None, 0), + ('UNARY_NOT', None, 0), + ('NOP', None, 0), + ('BUILD_TUPLE', 4, 0), + ('NOP', None, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[-2, (1, 2, 3, True)]) + + def test_build_empty_tuple(self): + before = [ + ('BUILD_TUPLE', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[()]) + + def test_fold_tuple_of_constants(self): + before = [ + ('NOP', None, 0), + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('BUILD_TUPLE', 3, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)]) + + # not all consts + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_TUPLE', 3, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(same, same, consts=[]) + + def test_fold_constant_intrinsic_list_to_tuple(self): + INTRINSIC_LIST_TO_TUPLE = 6 + + # long tuple + consts = 1000 + before = ( + [('BUILD_LIST', 0, 0)] + + [('LOAD_CONST', 0, 0), ('LIST_APPEND', 1, 0)] * consts + + [('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), ('RETURN_VALUE', None, 0)] + ) + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0) + ] + result_const = tuple(["test"] * consts) + self.cfg_optimization_test(before, after, consts=["test"], expected_consts=["test", result_const]) + + # empty list + before = [ + ('BUILD_LIST', 0, 0), + ('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[()]) + + # multiple BUILD_LIST 0: ([], 1, [], 2) + same = [ + ('BUILD_LIST', 0, 0), + ('BUILD_LIST', 0, 0), + ('LIST_APPEND', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LIST_APPEND', 1, 0), + ('BUILD_LIST', 0, 0), + ('LIST_APPEND', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('LIST_APPEND', 1, 0), + ('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(same, same, consts=[]) + + # nested folding: (1, 1+1, 3) + before = [ + ('BUILD_LIST', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LIST_APPEND', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', get_binop_argval('NB_ADD'), 0), + ('LIST_APPEND', 1, 0), + ('LOAD_SMALL_INT', 3, 0), + ('LIST_APPEND', 1, 0), + ('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)]) + + # NOP's in between: (1, 2, 3) + before = [ + ('BUILD_LIST', 0, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LIST_APPEND', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LIST_APPEND', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('LIST_APPEND', 1, 0), + ('NOP', None, 0), + ('CALL_INTRINSIC_1', INTRINSIC_LIST_TO_TUPLE, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)]) + + def test_optimize_if_const_list(self): + before = [ + ('NOP', None, 0), + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('BUILD_LIST', 3, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('BUILD_LIST', 0, 0), + ('LOAD_CONST', 0, 0), + ('LIST_EXTEND', 1, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[(1, 2, 3)]) + + # need minimum 3 consts to optimize + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_LIST', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[]) + + # not all consts + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 3, 0), + ('BUILD_LIST', 3, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[]) + + def test_optimize_if_const_set(self): + before = [ + ('NOP', None, 0), + ('LOAD_SMALL_INT', 1, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 2, 0), + ('NOP', None, 0), + ('NOP', None, 0), + ('LOAD_SMALL_INT', 3, 0), + ('NOP', None, 0), + ('BUILD_SET', 3, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('BUILD_SET', 0, 0), + ('LOAD_CONST', 0, 0), + ('SET_UPDATE', 1, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[frozenset({1, 2, 3})]) + + # need minimum 3 consts to optimize + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_SET', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[]) + + # not all consts + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 3, 0), + ('BUILD_SET', 3, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[]) + + def test_optimize_literal_list_for_iter(self): + # for _ in [1, 2]: pass ==> for _ in (1, 2): pass + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_LIST', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 1, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None, (1, 2)]) + + # for _ in [1, x]: pass ==> for _ in (1, x): pass + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('BUILD_LIST', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('BUILD_TUPLE', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None]) + + def test_optimize_literal_set_for_iter(self): + # for _ in {1, 2}: pass ==> for _ in (1, 2): pass + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_SET', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 1, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None, frozenset({1, 2})]) + + # non constant literal set is not changed + # for _ in {1, x}: pass ==> for _ in {1, x}: pass + same = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 0, 0), + ('BUILD_SET', 2, 0), + ('GET_ITER', None, 0), + start := self.Label(), + ('FOR_ITER', end := self.Label(), 0), + ('STORE_FAST', 0, 0), + ('JUMP', start, 0), + end, + ('END_FOR', None, 0), + ('POP_ITER', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[None], expected_consts=[None]) + + def test_optimize_literal_list_contains(self): + # x in [1, 2] ==> x in (1, 2) + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_LIST', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_CONST', 1, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None, (1, 2)]) + + # x in [1, y] ==> x in (1, y) + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 1, 0), + ('BUILD_LIST', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 1, 0), + ('BUILD_TUPLE', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None]) + + def test_optimize_literal_set_contains(self): + # x in {1, 2} ==> x in (1, 2) + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BUILD_SET', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_CONST', 1, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[None], expected_consts=[None, frozenset({1, 2})]) + + # non constant literal set is not changed + # x in {1, y} ==> x in {1, y} + same = [ + ('LOAD_NAME', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_NAME', 1, 0), + ('BUILD_SET', 2, 0), + ('CONTAINS_OP', 0, 0), + ('POP_TOP', None, 0), + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(same, same, consts=[None], expected_consts=[None]) + + def test_optimize_unary_not(self): + # test folding + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[True, False]) + + # test cancel out + before = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test folding & cancel out + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[True]) + + # test folding & eliminate to bool + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[True, False]) + + # test cancel out & eliminate to bool (to bool stays as we are not iterating to a fixed point) + before = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + is_ = in_ = 0 + isnot = notin = 1 + + # test is/isnot + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', isnot, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test is/isnot cancel out + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test is/isnot eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', isnot, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test is/isnot cancel out & eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('IS_OP', is_, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test in/notin + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', notin, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test in/notin cancel out + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test is/isnot & eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', notin, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test in/notin cancel out & eliminate to bool + before = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('UNARY_NOT', None, 0), + ('UNARY_NOT', None, 0), + ('TO_BOOL', None, 0), + ('RETURN_VALUE', None, 0), + ] + after = [ + ('LOAD_NAME', 0, 0), + ('LOAD_NAME', 1, 0), + ('CONTAINS_OP', in_, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + def test_optimize_if_const_unaryop(self): + # test unary negative + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('UNARY_NEGATIVE', None, 0), + ('UNARY_NEGATIVE', None, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[-2]) + + # test unary invert + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('UNARY_INVERT', None, 0), + ('UNARY_INVERT', None, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[-3]) + + # test unary positive + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('CALL_INTRINSIC_1', 5, 0), + ('CALL_INTRINSIC_1', 5, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 2, 0), + ('RETURN_VALUE', None, 0), + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + def test_optimize_if_const_binop(self): + add = get_binop_argval('NB_ADD') + sub = get_binop_argval('NB_SUBTRACT') + mul = get_binop_argval('NB_MULTIPLY') + div = get_binop_argval('NB_TRUE_DIVIDE') + floor = get_binop_argval('NB_FLOOR_DIVIDE') + rem = get_binop_argval('NB_REMAINDER') + pow = get_binop_argval('NB_POWER') + lshift = get_binop_argval('NB_LSHIFT') + rshift = get_binop_argval('NB_RSHIFT') + or_ = get_binop_argval('NB_OR') + and_ = get_binop_argval('NB_AND') + xor = get_binop_argval('NB_XOR') + subscr = get_binop_argval('NB_SUBSCR') + + # test add + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', add, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', add, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 6, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test sub + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', sub, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', sub, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[-2]) + + # test mul + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', mul, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', mul, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 8, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test div + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', div, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', div, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_CONST', 1, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[1.0, 0.5]) + + # test floor + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', floor, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', floor, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test rem + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', rem, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', rem, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 0, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test pow + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', pow, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', pow, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 16, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test lshift + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', lshift, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', lshift, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 4, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test rshift + before = [ + ('LOAD_SMALL_INT', 4, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', rshift, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', rshift, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 1, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test or + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', or_, 0), + ('LOAD_SMALL_INT', 4, 0), + ('BINARY_OP', or_, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 7, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test and + before = [ + ('LOAD_SMALL_INT', 1, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', and_, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', and_, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 1, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test xor + before = [ + ('LOAD_SMALL_INT', 2, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', xor, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', xor, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 2, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) + + # test subscr + before = [ + ('LOAD_CONST', 0, 0), + ('LOAD_SMALL_INT', 1, 0), + ('BINARY_OP', subscr, 0), + ('LOAD_SMALL_INT', 2, 0), + ('BINARY_OP', subscr, 0), + ('RETURN_VALUE', None, 0) + ] + after = [ + ('LOAD_SMALL_INT', 3, 0), + ('RETURN_VALUE', None, 0) + ] + self.cfg_optimization_test(before, after, consts=[(1, (1, 2, 3))], expected_consts=[(1, (1, 2, 3))]) + + + def test_conditional_jump_forward_const_condition(self): + # The unreachable branch of the jump is removed, the jump + # becomes redundant and is replaced by a NOP (for the lineno) + + insts = [ + ('LOAD_CONST', 3, 11), + ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), + ('LOAD_CONST', 2, 13), + lbl, + ('LOAD_CONST', 3, 14), + ('RETURN_VALUE', None, 14), + ] + expected_insts = [ + ('NOP', None, 11), + ('NOP', None, 12), + ('LOAD_SMALL_INT', 3, 14), + ('RETURN_VALUE', None, 14), + ] + self.cfg_optimization_test(insts, + expected_insts, + consts=[0, 1, 2, 3, 4], + expected_consts=[0]) + + def test_conditional_jump_backward_non_const_condition(self): + insts = [ + lbl1 := self.Label(), + ('LOAD_NAME', 1, 11), + ('POP_JUMP_IF_TRUE', lbl1, 12), + ('LOAD_NAME', 2, 13), + ('RETURN_VALUE', None, 13), + ] + expected = [ + lbl := self.Label(), + ('LOAD_NAME', 1, 11), + ('POP_JUMP_IF_TRUE', lbl, 12), + ('LOAD_NAME', 2, 13), + ('RETURN_VALUE', None, 13), + ] + self.cfg_optimization_test(insts, expected, consts=list(range(5))) + + def test_conditional_jump_backward_const_condition(self): + # The unreachable branch of the jump is removed + insts = [ + lbl1 := self.Label(), + ('LOAD_CONST', 3, 11), + ('POP_JUMP_IF_TRUE', lbl1, 12), + ('LOAD_CONST', 2, 13), + ('RETURN_VALUE', None, 13), + ] + expected_insts = [ + lbl := self.Label(), + ('NOP', None, 11), + ('JUMP', lbl, 12), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + def test_except_handler_label(self): + insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('POP_BLOCK', None, -1), + ('LOAD_CONST', 1, 11), + ('RETURN_VALUE', None, 11), + handler, + ('LOAD_CONST', 2, 12), + ('RETURN_VALUE', None, 12), + ] + expected_insts = [ + ('SETUP_FINALLY', handler := self.Label(), 10), + ('LOAD_SMALL_INT', 1, 11), + ('RETURN_VALUE', None, 11), + handler, + ('LOAD_SMALL_INT', 2, 12), + ('RETURN_VALUE', None, 12), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + def test_no_unsafe_static_swap(self): + # We can't change order of two stores to the same location + insts = [ + ('LOAD_CONST', 0, 1), + ('LOAD_CONST', 1, 2), + ('LOAD_CONST', 2, 3), + ('SWAP', 3, 4), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 4), + ('POP_TOP', None, 4), + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) + ] + expected_insts = [ + ('LOAD_SMALL_INT', 0, 1), + ('LOAD_SMALL_INT', 1, 2), + ('NOP', None, 3), + ('STORE_FAST', 1, 4), + ('POP_TOP', None, 4), + ('LOAD_SMALL_INT', 0, 5), + ('RETURN_VALUE', None, 5) + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) + + def test_dead_store_elimination_in_same_lineno(self): + insts = [ + ('LOAD_CONST', 0, 1), + ('LOAD_CONST', 1, 2), + ('LOAD_CONST', 2, 3), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 4), + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) + ] + expected_insts = [ + ('LOAD_SMALL_INT', 0, 1), + ('LOAD_SMALL_INT', 1, 2), + ('NOP', None, 3), + ('POP_TOP', None, 4), + ('STORE_FAST', 1, 4), + ('LOAD_SMALL_INT', 0, 5), + ('RETURN_VALUE', None, 5) + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) + + def test_no_dead_store_elimination_in_different_lineno(self): + insts = [ + ('LOAD_CONST', 0, 1), + ('LOAD_CONST', 1, 2), + ('LOAD_CONST', 2, 3), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 5), + ('STORE_FAST', 1, 6), + ('LOAD_CONST', 0, 5), + ('RETURN_VALUE', None, 5) + ] + expected_insts = [ + ('LOAD_SMALL_INT', 0, 1), + ('LOAD_SMALL_INT', 1, 2), + ('LOAD_SMALL_INT', 2, 3), + ('STORE_FAST', 1, 4), + ('STORE_FAST', 1, 5), + ('STORE_FAST', 1, 6), + ('LOAD_SMALL_INT', 0, 5), + ('RETURN_VALUE', None, 5) + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) + + def test_unconditional_jump_threading(self): + + def get_insts(lno1, lno2, op1, op2): + return [ + lbl2 := self.Label(), + ('LOAD_NAME', 0, 10), + ('POP_TOP', None, 10), + (op1, lbl1 := self.Label(), lno1), + ('LOAD_NAME', 1, 20), + lbl1, + (op2, lbl2, lno2), + ] + + for op1 in ('JUMP', 'JUMP_NO_INTERRUPT'): + for op2 in ('JUMP', 'JUMP_NO_INTERRUPT'): + # different lines + lno1, lno2 = (4, 5) + with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): + insts = get_insts(lno1, lno2, op1, op2) + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' + expected_insts = [ + ('LOAD_NAME', 0, 10), + ('POP_TOP', None, 10), + ('NOP', None, 4), + (op, 0, 5), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + # Threading + for lno1, lno2 in [(-1, -1), (-1, 5), (6, -1), (7, 7)]: + with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): + insts = get_insts(lno1, lno2, op1, op2) + lno = lno1 if lno1 != -1 else lno2 + if lno == -1: + lno = 10 # Propagated from the line before + + op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' + expected_insts = [ + ('LOAD_NAME', 0, 10), + ('POP_TOP', None, 10), + (op, 0, lno), + ] + self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) + + def test_list_to_tuple_get_iter(self): + # for _ in (*foo, *bar) -> for _ in [*foo, *bar] + INTRINSIC_LIST_TO_TUPLE = 6 + insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST", 1, 4), + ("LIST_EXTEND", 1, 5), + ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + expected_insts = [ + ("BUILD_LIST", 0, 1), + ("LOAD_FAST_BORROW", 0, 2), + ("LIST_EXTEND", 1, 3), + ("LOAD_FAST_BORROW", 1, 4), + ("LIST_EXTEND", 1, 5), + ("NOP", None, 6), # ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6), + ("GET_ITER", None, 7), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 8), + ("STORE_FAST", 2, 9), + ("JUMP", top, 10), + end, + ("END_FOR", None, 11), + ("POP_TOP", None, 12), + ("LOAD_CONST", 0, 13), + ("RETURN_VALUE", None, 14), + ] + self.cfg_optimization_test(insts, expected_insts, consts=[None]) + + def test_list_to_tuple_get_iter_is_safe(self): + a, b = [], [] + for item in (*(items := [0, 1, 2, 3]),): + a.append(item) + b.append(items.pop()) + self.assertEqual(a, [0, 1, 2, 3]) + self.assertEqual(b, [3, 2, 1, 0]) + self.assertEqual(items, []) + + +class OptimizeLoadFastTestCase(DirectCfgOptimizerTests): + def make_bb(self, insts): + last_loc = insts[-1][2] + maxconst = 0 + for op, arg, _ in insts: + if op == "LOAD_CONST": + maxconst = max(maxconst, arg) + consts = [None for _ in range(maxconst + 1)] + return insts + [ + ("LOAD_CONST", 0, last_loc + 1), + ("RETURN_VALUE", None, last_loc + 2), + ], consts + + def check(self, insts, expected_insts, consts=None): + insts_bb, insts_consts = self.make_bb(insts) + expected_insts_bb, exp_consts = self.make_bb(expected_insts) + self.cfg_optimization_test(insts_bb, expected_insts_bb, + consts=insts_consts, expected_consts=exp_consts) + + def test_optimized(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("BINARY_OP", 2, 3), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("BINARY_OP", 2, 3), + ] + self.check(insts, expected) + + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_CONST", 1, 2), + ("SWAP", 2, 3), + ("POP_TOP", None, 4), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_CONST", 1, 2), + ("SWAP", 2, 3), + ("POP_TOP", None, 4), + ] + self.check(insts, expected) + + def test_unoptimized_if_unconsumed(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, expected) + + insts = [ + ("LOAD_FAST", 0, 1), + ("COPY", 1, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("NOP", None, 2), + ("NOP", None, 3), + ] + self.check(insts, expected) + + def test_unoptimized_if_support_killed(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_CONST", 0, 2), + ("STORE_FAST", 0, 3), + ("POP_TOP", None, 4), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_CONST", 0, 2), + ("LOAD_CONST", 0, 3), + ("STORE_FAST_STORE_FAST", ((0 << 4) | 1), 4), + ("POP_TOP", None, 5), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("DELETE_FAST", 0, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, insts) + + def test_unoptimized_if_aliased(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("STORE_FAST", 1, 2), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_CONST", 0, 3), + ("STORE_FAST_STORE_FAST", ((0 << 4) | 1), 4), + ] + self.check(insts, insts) + + def test_consume_no_inputs(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_LEN", None, 2), + ("STORE_FAST", 1 , 3), + ("STORE_FAST", 2, 4), + ] + self.check(insts, insts) + + def test_consume_some_inputs_no_outputs(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_LEN", None, 2), + ("LIST_APPEND", 0, 3), + ] + self.check(insts, insts) + + def test_check_exc_match(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("CHECK_EXC_MATCH", None, 3) + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("CHECK_EXC_MATCH", None, 3) + ] + self.check(insts, expected) + + def test_for_iter(self): + insts = [ + ("LOAD_FAST", 0, 1), + top := self.Label(), + ("FOR_ITER", end := self.Label(), 2), + ("STORE_FAST", 2, 3), + ("JUMP", top, 4), + end, + ("END_FOR", None, 5), + ("POP_TOP", None, 6), + ("LOAD_CONST", 0, 7), + ("RETURN_VALUE", None, 8), + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + def test_load_attr(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_ATTR", 0, 2), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_ATTR", 0, 2), + ] + self.check(insts, expected) + + # Method call, leaves self on stack unconsumed + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_ATTR", 1, 2), + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("LOAD_ATTR", 1, 2), + ] + self.check(insts, expected) + + def test_super_attr(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("LOAD_FAST", 2, 3), + ("LOAD_SUPER_ATTR", 0, 4), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("LOAD_FAST_BORROW", 2, 3), + ("LOAD_SUPER_ATTR", 0, 4), + ] + self.check(insts, expected) + + # Method call, leaves self on stack unconsumed + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("LOAD_FAST", 2, 3), + ("LOAD_SUPER_ATTR", 1, 4), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("LOAD_FAST", 2, 3), + ("LOAD_SUPER_ATTR", 1, 4), + ] + self.check(insts, expected) + + def test_send(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST", 1, 2), + ("SEND", end := self.Label(), 3), + ("LOAD_CONST", 0, 4), + ("RETURN_VALUE", None, 5), + end, + ("LOAD_CONST", 0, 6), + ("RETURN_VALUE", None, 7) + ] + expected = [ + ("LOAD_FAST", 0, 1), + ("LOAD_FAST_BORROW", 1, 2), + ("SEND", end := self.Label(), 3), + ("LOAD_CONST", 0, 4), + ("RETURN_VALUE", None, 5), + end, + ("LOAD_CONST", 0, 6), + ("RETURN_VALUE", None, 7) + ] + self.cfg_optimization_test(insts, expected, consts=[None]) + + def test_format_simple(self): + # FORMAT_SIMPLE will leave its operand on the stack if it's a unicode + # object. We treat it conservatively and assume that it always leaves + # its operand on the stack. + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, expected) + + def test_set_function_attribute(self): + # SET_FUNCTION_ATTRIBUTE leaves the function on the stack + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("STORE_FAST", 1, 4), + ("LOAD_CONST", 0, 5), + ("RETURN_VALUE", None, 6) + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + expected = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST_BORROW", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + self.cfg_optimization_test(insts, expected, consts=[None]) + + def test_get_yield_from_iter(self): + # GET_YIELD_FROM_ITER may leave its operand on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_YIELD_FROM_ITER", None, 2), + ("LOAD_CONST", 0, 3), + send := self.Label(), + ("SEND", end := self.Label(), 5), + ("YIELD_VALUE", 1, 6), + ("RESUME", 2, 7), + ("JUMP", send, 8), + end, + ("END_SEND", None, 9), + ("LOAD_CONST", 0, 10), + ("RETURN_VALUE", None, 11), + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + def test_push_exc_info(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("PUSH_EXC_INFO", None, 2), + ] + self.check(insts, insts) + + def test_load_special(self): + # LOAD_SPECIAL may leave self on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_SPECIAL", 0, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + + def test_del_in_finally(self): + # This loads `obj` onto the stack, executes `del obj`, then returns the + # `obj` from the stack. See gh-133371 for more details. + def create_obj(): + obj = [42] + try: + return obj + finally: + del obj + + obj = create_obj() + # The crash in the linked issue happens while running GC during + # interpreter finalization, so run it here manually. + gc.collect() + self.assertEqual(obj, [42]) + + def test_format_simple_unicode(self): + # Repro from gh-134889 + def f(): + var = f"{1}" + var = f"{var}" + return var + self.assertEqual(f(), "1") + + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pep646_syntax.py b/Lib/test/test_pep646_syntax.py new file mode 100644 index 00000000000..d9a0aa9a90e --- /dev/null +++ b/Lib/test/test_pep646_syntax.py @@ -0,0 +1,338 @@ +import doctest +import unittest + +doctests = """ + +Setup + + >>> class AClass: + ... def __init__(self): + ... self._setitem_name = None + ... self._setitem_val = None + ... self._delitem_name = None + ... def __setitem__(self, name, val): + ... self._delitem_name = None + ... self._setitem_name = name + ... self._setitem_val = val + ... def __repr__(self): + ... if self._setitem_name is not None: + ... return f"A[{self._setitem_name}]={self._setitem_val}" + ... elif self._delitem_name is not None: + ... return f"delA[{self._delitem_name}]" + ... def __getitem__(self, name): + ... return ParameterisedA(name) + ... def __delitem__(self, name): + ... self._setitem_name = None + ... self._delitem_name = name + ... + >>> class ParameterisedA: + ... def __init__(self, name): + ... self._name = name + ... def __repr__(self): + ... return f"A[{self._name}]" + ... def __iter__(self): + ... for p in self._name: + ... yield p + >>> class B: + ... def __iter__(self): + ... yield StarredB() + ... def __repr__(self): + ... return "B" + >>> class StarredB: + ... def __repr__(self): + ... return "StarredB" + >>> A = AClass() + >>> b = B() + +Slices that are supposed to work, starring our custom B class + + >>> A[*b] + A[(StarredB,)] + >>> A[*b] = 1; A + A[(StarredB,)]=1 + >>> del A[*b]; A + delA[(StarredB,)] + + >>> A[*b, *b] + A[(StarredB, StarredB)] + >>> A[*b, *b] = 1; A + A[(StarredB, StarredB)]=1 + >>> del A[*b, *b]; A + delA[(StarredB, StarredB)] + + >>> A[b, *b] + A[(B, StarredB)] + >>> A[b, *b] = 1; A + A[(B, StarredB)]=1 + >>> del A[b, *b]; A + delA[(B, StarredB)] + + >>> A[*b, b] + A[(StarredB, B)] + >>> A[*b, b] = 1; A + A[(StarredB, B)]=1 + >>> del A[*b, b]; A + delA[(StarredB, B)] + + >>> A[b, b, *b] + A[(B, B, StarredB)] + >>> A[b, b, *b] = 1; A + A[(B, B, StarredB)]=1 + >>> del A[b, b, *b]; A + delA[(B, B, StarredB)] + + >>> A[*b, b, b] + A[(StarredB, B, B)] + >>> A[*b, b, b] = 1; A + A[(StarredB, B, B)]=1 + >>> del A[*b, b, b]; A + delA[(StarredB, B, B)] + + >>> A[b, *b, b] + A[(B, StarredB, B)] + >>> A[b, *b, b] = 1; A + A[(B, StarredB, B)]=1 + >>> del A[b, *b, b]; A + delA[(B, StarredB, B)] + + >>> A[b, b, *b, b] + A[(B, B, StarredB, B)] + >>> A[b, b, *b, b] = 1; A + A[(B, B, StarredB, B)]=1 + >>> del A[b, b, *b, b]; A + delA[(B, B, StarredB, B)] + + >>> A[b, *b, b, b] + A[(B, StarredB, B, B)] + >>> A[b, *b, b, b] = 1; A + A[(B, StarredB, B, B)]=1 + >>> del A[b, *b, b, b]; A + delA[(B, StarredB, B, B)] + + >>> A[A[b, *b, b]] + A[A[(B, StarredB, B)]] + >>> A[A[b, *b, b]] = 1; A + A[A[(B, StarredB, B)]]=1 + >>> del A[A[b, *b, b]]; A + delA[A[(B, StarredB, B)]] + + >>> A[*A[b, *b, b]] + A[(B, StarredB, B)] + >>> A[*A[b, *b, b]] = 1; A + A[(B, StarredB, B)]=1 + >>> del A[*A[b, *b, b]]; A + delA[(B, StarredB, B)] + + >>> A[b, ...] + A[(B, Ellipsis)] + >>> A[b, ...] = 1; A + A[(B, Ellipsis)]=1 + >>> del A[b, ...]; A + delA[(B, Ellipsis)] + + >>> A[*A[b, ...]] + A[(B, Ellipsis)] + >>> A[*A[b, ...]] = 1; A + A[(B, Ellipsis)]=1 + >>> del A[*A[b, ...]]; A + delA[(B, Ellipsis)] + +Slices that are supposed to work, starring a list + + >>> l = [1, 2, 3] + + >>> A[*l] + A[(1, 2, 3)] + >>> A[*l] = 1; A + A[(1, 2, 3)]=1 + >>> del A[*l]; A + delA[(1, 2, 3)] + + >>> A[*l, 4] + A[(1, 2, 3, 4)] + >>> A[*l, 4] = 1; A + A[(1, 2, 3, 4)]=1 + >>> del A[*l, 4]; A + delA[(1, 2, 3, 4)] + + >>> A[0, *l] + A[(0, 1, 2, 3)] + >>> A[0, *l] = 1; A + A[(0, 1, 2, 3)]=1 + >>> del A[0, *l]; A + delA[(0, 1, 2, 3)] + + >>> A[1:2, *l] + A[(slice(1, 2, None), 1, 2, 3)] + >>> A[1:2, *l] = 1; A + A[(slice(1, 2, None), 1, 2, 3)]=1 + >>> del A[1:2, *l]; A + delA[(slice(1, 2, None), 1, 2, 3)] + + >>> repr(A[1:2, *l]) == repr(A[1:2, 1, 2, 3]) + True + +Slices that are supposed to work, starring a tuple + + >>> t = (1, 2, 3) + + >>> A[*t] + A[(1, 2, 3)] + >>> A[*t] = 1; A + A[(1, 2, 3)]=1 + >>> del A[*t]; A + delA[(1, 2, 3)] + + >>> A[*t, 4] + A[(1, 2, 3, 4)] + >>> A[*t, 4] = 1; A + A[(1, 2, 3, 4)]=1 + >>> del A[*t, 4]; A + delA[(1, 2, 3, 4)] + + >>> A[0, *t] + A[(0, 1, 2, 3)] + >>> A[0, *t] = 1; A + A[(0, 1, 2, 3)]=1 + >>> del A[0, *t]; A + delA[(0, 1, 2, 3)] + + >>> A[1:2, *t] + A[(slice(1, 2, None), 1, 2, 3)] + >>> A[1:2, *t] = 1; A + A[(slice(1, 2, None), 1, 2, 3)]=1 + >>> del A[1:2, *t]; A + delA[(slice(1, 2, None), 1, 2, 3)] + + >>> repr(A[1:2, *t]) == repr(A[1:2, 1, 2, 3]) + True + +Starring an expression (rather than a name) in a slice + + >>> def returns_list(): + ... return [1, 2, 3] + + >>> A[returns_list()] + A[[1, 2, 3]] + >>> A[returns_list()] = 1; A + A[[1, 2, 3]]=1 + >>> del A[returns_list()]; A + delA[[1, 2, 3]] + + >>> A[returns_list(), 4] + A[([1, 2, 3], 4)] + >>> A[returns_list(), 4] = 1; A + A[([1, 2, 3], 4)]=1 + >>> del A[returns_list(), 4]; A + delA[([1, 2, 3], 4)] + + >>> A[*returns_list()] + A[(1, 2, 3)] + >>> A[*returns_list()] = 1; A + A[(1, 2, 3)]=1 + >>> del A[*returns_list()]; A + delA[(1, 2, 3)] + + >>> A[*returns_list(), 4] + A[(1, 2, 3, 4)] + >>> A[*returns_list(), 4] = 1; A + A[(1, 2, 3, 4)]=1 + >>> del A[*returns_list(), 4]; A + delA[(1, 2, 3, 4)] + + >>> A[0, *returns_list()] + A[(0, 1, 2, 3)] + >>> A[0, *returns_list()] = 1; A + A[(0, 1, 2, 3)]=1 + >>> del A[0, *returns_list()]; A + delA[(0, 1, 2, 3)] + + >>> A[*returns_list(), *returns_list()] + A[(1, 2, 3, 1, 2, 3)] + >>> A[*returns_list(), *returns_list()] = 1; A + A[(1, 2, 3, 1, 2, 3)]=1 + >>> del A[*returns_list(), *returns_list()]; A + delA[(1, 2, 3, 1, 2, 3)] + +Using both a starred object and a start:stop in a slice +(See also tests in test_syntax confirming that starring *inside* a start:stop +is *not* valid syntax.) + + >>> A[1:2, *b] + A[(slice(1, 2, None), StarredB)] + >>> A[*b, 1:2] + A[(StarredB, slice(1, 2, None))] + >>> A[1:2, *b, 1:2] + A[(slice(1, 2, None), StarredB, slice(1, 2, None))] + >>> A[*b, 1:2, *b] + A[(StarredB, slice(1, 2, None), StarredB)] + + >>> A[1:, *b] + A[(slice(1, None, None), StarredB)] + >>> A[*b, 1:] + A[(StarredB, slice(1, None, None))] + >>> A[1:, *b, 1:] + A[(slice(1, None, None), StarredB, slice(1, None, None))] + >>> A[*b, 1:, *b] + A[(StarredB, slice(1, None, None), StarredB)] + + >>> A[:1, *b] + A[(slice(None, 1, None), StarredB)] + >>> A[*b, :1] + A[(StarredB, slice(None, 1, None))] + >>> A[:1, *b, :1] + A[(slice(None, 1, None), StarredB, slice(None, 1, None))] + >>> A[*b, :1, *b] + A[(StarredB, slice(None, 1, None), StarredB)] + + >>> A[:, *b] + A[(slice(None, None, None), StarredB)] + >>> A[*b, :] + A[(StarredB, slice(None, None, None))] + >>> A[:, *b, :] + A[(slice(None, None, None), StarredB, slice(None, None, None))] + >>> A[*b, :, *b] + A[(StarredB, slice(None, None, None), StarredB)] + +*args annotated as starred expression + + >>> def f1(*args: *b): pass + >>> f1.__annotations__ + {'args': StarredB} + + >>> def f2(*args: *b, arg1): pass + >>> f2.__annotations__ + {'args': StarredB} + + >>> def f3(*args: *b, arg1: int): pass + >>> f3.__annotations__ # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + {'args': StarredB, 'arg1': <class 'int'>} + + >>> def f4(*args: *b, arg1: int = 2): pass + >>> f4.__annotations__ # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + {'args': StarredB, 'arg1': <class 'int'>} + + >>> def f5(*args: *b = (1,)): pass # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: invalid syntax +""" + +__test__ = {'doctests' : doctests} + +EXPECTED_FAILURE = doctest.register_optionflag('EXPECTED_FAILURE') # TODO: RUSTPYTHON +class CustomOutputChecker(doctest.OutputChecker): # TODO: RUSTPYTHON + def check_output(self, want, got, optionflags): # TODO: RUSTPYTHON + if optionflags & EXPECTED_FAILURE: # TODO: RUSTPYTHON + if want == got: # TODO: RUSTPYTHON + return False # TODO: RUSTPYTHON + return True # TODO: RUSTPYTHON + return super().check_output(want, got, optionflags) # TODO: RUSTPYTHON + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(checker=CustomOutputChecker())) # TODO: RUSTPYTHON + return tests + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 9450d8c08ed..c9d4a348448 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -1,21 +1,26 @@ from _compat_pickle import (IMPORT_MAPPING, REVERSE_IMPORT_MAPPING, NAME_MAPPING, REVERSE_NAME_MAPPING) import builtins -import pickle -import io import collections +import contextlib +import io +import pickle import struct import sys +import tempfile import warnings import weakref +from textwrap import dedent import doctest import unittest from test import support -from test.support import import_helper +from test.support import cpython_only, import_helper, os_helper +from test.support.import_helper import ensure_lazy_imports from test.pickletester import AbstractHookTests from test.pickletester import AbstractUnpickleTests +from test.pickletester import AbstractPicklingErrorTests from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests from test.pickletester import AbstractPersistentPicklerTests @@ -32,6 +37,12 @@ has_c_implementation = False +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("pickle", {"re"}) + + class PyPickleTests(AbstractPickleModuleTests, unittest.TestCase): dump = staticmethod(pickle._dump) dumps = staticmethod(pickle._dumps) @@ -40,15 +51,13 @@ class PyPickleTests(AbstractPickleModuleTests, unittest.TestCase): Pickler = pickle._Pickler Unpickler = pickle._Unpickler - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_dump_load_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_dump_load_oob_buffers() # TODO: RUSTPYTHON, remove when this passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dump_load_oob_buffers(self): + return super().test_dump_load_oob_buffers() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_dumps_loads_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_dumps_loads_oob_buffers() # TODO: RUSTPYTHON, remove when this passes + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dumps_loads_oob_buffers(self): + return super().test_dumps_loads_oob_buffers() class PyUnpicklerTests(AbstractUnpickleTests, unittest.TestCase): @@ -59,72 +68,41 @@ class PyUnpicklerTests(AbstractUnpickleTests, unittest.TestCase): AttributeError, ValueError, struct.error, IndexError, ImportError) - # TODO: RUSTPYTHON, AssertionError: ValueError not raised - @unittest.expectedFailure - def test_badly_escaped_string(self): # TODO: RUSTPYTHON, remove when this passes - super().test_badly_escaped_string() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError - @unittest.expectedFailure - def test_correctly_quoted_string(self): # TODO: RUSTPYTHON, remove when this passes - super().test_correctly_quoted_string() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError - @unittest.expectedFailure - def test_load_python2_str_as_bytes(self): # TODO: RUSTPYTHON, remove when this passes - super().test_load_python2_str_as_bytes() # TODO: RUSTPYTHON, remove when this passes - def loads(self, buf, **kwds): f = io.BytesIO(buf) u = self.unpickler(f, **kwds) return u.load() +class PyPicklingErrorTests(AbstractPicklingErrorTests, unittest.TestCase): + + pickler = pickle._Pickler + + def dumps(self, arg, proto=None, **kwargs): + f = io.BytesIO() + p = self.pickler(f, proto, **kwargs) + p.dump(arg) + f.seek(0) + return bytes(f.read()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffer_callback_error(self): + return super().test_buffer_callback_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_non_continuous_buffer(self): + return super().test_non_continuous_buffer() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_picklebuffer_error(self): + return super().test_picklebuffer_error() + + class PyPicklerTests(AbstractPickleTests, unittest.TestCase): pickler = pickle._Pickler unpickler = pickle._Unpickler - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffer_callback_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffer_callback_error() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffers_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffers_error() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO: RUSTPYTHON, remove when this passes - super().test_complex_newobj_ex() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, TypeError: cannot pickle 'method' object - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_in_band_buffers() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers_writable_to_readonly() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray' - @unittest.expectedFailure - def test_optional_frames(self): # TODO: RUSTPYTHON, remove when this passes - super().test_optional_frames() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_picklebuffer_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_picklebuffer_error() # TODO: RUSTPYTHON, remove when this passes - def dumps(self, arg, proto=None, **kwargs): f = io.BytesIO() p = self.pickler(f, proto, **kwargs) @@ -137,6 +115,34 @@ def loads(self, buf, **kwds): u = self.unpickler(f, **kwds) return u.load() + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffers_error(self): + return super().test_buffers_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytearray_memoization(self): + return super().test_bytearray_memoization() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytes_memoization(self): + return super().test_bytes_memoization() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_c_methods(self): + return super().test_c_methods() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_in_band_buffers(self): + return super().test_in_band_buffers() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers(self): + return super().test_oob_buffers() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers_writable_to_readonly(self): + return super().test_oob_buffers_writable_to_readonly() + class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, BigmemPickleTests, unittest.TestCase): @@ -146,61 +152,6 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, AttributeError, ValueError, struct.error, IndexError, ImportError) - # TODO: RUSTPYTHON, AssertionError: ValueError not raised - @unittest.expectedFailure - def test_badly_escaped_string(self): # TODO: RUSTPYTHON, remove when this passes - super().test_badly_escaped_string() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffer_callback_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffer_callback_error() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffers_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffers_error() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO: RUSTPYTHON, remove when this passes - super().test_complex_newobj_ex() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError - @unittest.expectedFailure - def test_correctly_quoted_string(self): # TODO: RUSTPYTHON, remove when this passes - super().test_correctly_quoted_string() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, TypeError: cannot pickle 'method' object - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_in_band_buffers() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError - @unittest.expectedFailure - def test_load_python2_str_as_bytes(self): # TODO: RUSTPYTHON, remove when this passes - super().test_load_python2_str_as_bytes() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers_writable_to_readonly() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray' - @unittest.expectedFailure - def test_optional_frames(self): # TODO: RUSTPYTHON, remove when this passes - super().test_optional_frames() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_picklebuffer_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_picklebuffer_error() # TODO: RUSTPYTHON, remove when this passes - def dumps(self, arg, protocol=None, **kwargs): return pickle.dumps(arg, protocol, **kwargs) @@ -208,6 +159,36 @@ def loads(self, buf, **kwds): return pickle.loads(buf, **kwds) test_framed_write_sizes_with_delayed_writer = None + test_find_class = None + test_custom_find_class = None + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffers_error(self): + return super().test_buffers_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytearray_memoization(self): + return super().test_bytearray_memoization() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytes_memoization(self): + return super().test_bytes_memoization() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_c_methods(self): + return super().test_c_methods() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_in_band_buffers(self): + return super().test_in_band_buffers() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers(self): + return super().test_oob_buffers() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers_writable_to_readonly(self): + return super().test_oob_buffers_writable_to_readonly() class PersistentPicklerUnpicklerMixin(object): @@ -242,6 +223,7 @@ class PyIdPersPicklerTests(AbstractIdentityPersistentPicklerTests, pickler = pickle._Pickler unpickler = pickle._Unpickler + persistent_load_error = pickle.UnpicklingError @support.cpython_only def test_pickler_reference_cycle(self): @@ -296,7 +278,6 @@ class DispatchTable: support.gc_collect() self.assertIsNone(table_ref()) - @support.cpython_only def test_unpickler_reference_cycle(self): def check(Unpickler): @@ -326,6 +307,112 @@ def persistent_load(pid): return pid check(PersUnpickler) + def test_pickler_super(self): + class PersPickler(self.pickler): + def persistent_id(subself, obj): + called.append(obj) + self.assertIsNone(super().persistent_id(obj)) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = PersPickler(f, proto) + called = [] + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + + def test_unpickler_super(self): + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + called.append(pid) + with self.assertRaises(self.persistent_load_error): + super().persistent_load(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + + def test_pickler_instance_attribute(self): + def persistent_id(obj): + called.append(obj) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = self.pickler(f, proto) + called = [] + old_persistent_id = pickler.persistent_id + pickler.persistent_id = persistent_id + self.assertEqual(pickler.persistent_id, persistent_id) + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + del pickler.persistent_id + self.assertEqual(pickler.persistent_id, old_persistent_id) + + def test_unpickler_instance_attribute(self): + def persistent_load(pid): + called.append(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = self.unpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + old_persistent_load = unpickler.persistent_load + unpickler.persistent_load = persistent_load + self.assertEqual(unpickler.persistent_load, persistent_load) + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + del unpickler.persistent_load + self.assertEqual(unpickler.persistent_load, old_persistent_load) + + def test_pickler_super_instance_attribute(self): + class PersPickler(self.pickler): + def persistent_id(subself, obj): + raise AssertionError('should never be called') + def _persistent_id(subself, obj): + called.append(obj) + self.assertIsNone(super().persistent_id(obj)) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = PersPickler(f, proto) + called = [] + old_persistent_id = pickler.persistent_id + pickler.persistent_id = pickler._persistent_id + self.assertEqual(pickler.persistent_id, pickler._persistent_id) + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + del pickler.persistent_id + self.assertEqual(pickler.persistent_id, old_persistent_id) + + def test_unpickler_super_instance_attribute(self): + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + raise AssertionError('should never be called') + def _persistent_load(subself, pid): + called.append(pid) + with self.assertRaises(self.persistent_load_error): + super().persistent_load(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + old_persistent_load = unpickler.persistent_load + unpickler.persistent_load = unpickler._persistent_load + self.assertEqual(unpickler.persistent_load, unpickler._persistent_load) + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + del unpickler.persistent_load + self.assertEqual(unpickler.persistent_load, old_persistent_load) + class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): @@ -365,6 +452,9 @@ class CUnpicklerTests(PyUnpicklerTests): bad_stack_errors = (pickle.UnpicklingError,) truncated_errors = (pickle.UnpicklingError,) + class CPicklingErrorTests(PyPicklingErrorTests): + pickler = _pickle.Pickler + class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler @@ -376,6 +466,7 @@ class CPersPicklerTests(PyPersPicklerTests): class CIdPersPicklerTests(PyIdPersPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler + persistent_load_error = _pickle.UnpicklingError class CDumpPickle_LoadPickle(PyPicklerTests): pickler = _pickle.Pickler @@ -398,6 +489,46 @@ def test_issue18339(self): unpickler.memo = {-1: None} unpickler.memo = {1: None} + def test_concurrent_pickler_dump(self): + f = io.BytesIO() + pickler = self.pickler_class(f) + class X: + def __reduce__(slf): + self.assertRaises(RuntimeError, pickler.dump, 42) + return list, () + pickler.dump(X()) # should not crash + self.assertEqual(pickle.loads(f.getvalue()), []) + + def test_concurrent_pickler_dump_and_init(self): + f = io.BytesIO() + pickler = self.pickler_class(f) + class X: + def __reduce__(slf): + self.assertRaises(RuntimeError, pickler.__init__, f) + return list, () + pickler.dump([X()]) # should not fail + self.assertEqual(pickle.loads(f.getvalue()), [[]]) + + def test_concurrent_unpickler_load(self): + global reducer + def reducer(): + self.assertRaises(RuntimeError, unpickler.load) + return 42 + f = io.BytesIO(b'(c%b\nreducer\n(tRl.' % (__name__.encode(),)) + unpickler = self.unpickler_class(f) + unpickled = unpickler.load() # should not fail + self.assertEqual(unpickled, [42]) + + def test_concurrent_unpickler_load_and_init(self): + global reducer + def reducer(): + self.assertRaises(RuntimeError, unpickler.__init__, f) + return 42 + f = io.BytesIO(b'(c%b\nreducer\n(tRl.' % (__name__.encode(),)) + unpickler = self.unpickler_class(f) + unpickled = unpickler.load() # should not crash + self.assertEqual(unpickled, [42]) + class CDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): pickler_class = pickle.Pickler def get_dispatch_table(self): @@ -413,12 +544,40 @@ class CustomCPicklerClass(_pickle.Pickler, AbstractCustomPicklerClass): pass pickler_class = CustomCPicklerClass + @support.cpython_only + class HeapTypesTests(unittest.TestCase): + def setUp(self): + pickler = _pickle.Pickler(io.BytesIO()) + unpickler = _pickle.Unpickler(io.BytesIO()) + + self._types = ( + _pickle.Pickler, + _pickle.Unpickler, + type(pickler.memo), + type(unpickler.memo), + + # We cannot test the _pickle.Pdata; + # there's no way to get to it. + ) + + def test_have_gc(self): + import gc + for tp in self._types: + with self.subTest(tp=tp): + self.assertTrue(gc.is_tracked(tp)) + + def test_immutable(self): + for tp in self._types: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = "bar" + @support.cpython_only class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('7P2n3i2n3i2P') + basesize = support.calcobjsize('7P2n3i2n4i2P') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -435,7 +594,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i') + basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n3i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. @@ -471,7 +630,9 @@ def recurse(deep): check_unpickler(recurse(1), 32, 20) check_unpickler(recurse(20), 32, 20) check_unpickler(recurse(50), 64, 60) - check_unpickler(recurse(100), 128, 140) + if not (support.is_wasi and support.Py_DEBUG): + # stack depth too shallow in pydebug WASI. + check_unpickler(recurse(100), 128, 140) u = unpickler(io.BytesIO(pickle.dumps('a', 0)), encoding='ASCII', errors='strict') @@ -566,10 +727,10 @@ def test_name_mapping(self): with self.subTest(((module3, name3), (module2, name2))): if (module2, name2) == ('exceptions', 'OSError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, OSError)) + self.assertIsSubclass(attr, OSError) elif (module2, name2) == ('exceptions', 'ImportError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, ImportError)) + self.assertIsSubclass(attr, ImportError) else: module, name = mapping(module2, name2) if module3[:1] != '_': @@ -614,6 +775,7 @@ def test_reverse_name_mapping(self): module, name = mapping(module, name) self.assertEqual((module, name), (module3, name3)) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_exceptions(self): self.assertEqual(mapping('exceptions', 'StandardError'), ('builtins', 'Exception')) @@ -631,10 +793,12 @@ def test_exceptions(self): if exc in (BlockingIOError, ResourceWarning, StopAsyncIteration, + PythonFinalizationError, RecursionError, EncodingWarning, - #ExceptionGroup, # TODO: RUSTPYTHON - BaseExceptionGroup): + BaseExceptionGroup, + ExceptionGroup, + _IncompleteInputError): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), @@ -653,6 +817,8 @@ def test_exceptions(self): def test_multiprocessing_exceptions(self): module = import_helper.import_module('multiprocessing.context') for name, exc in get_exceptions(module): + if issubclass(exc, Warning): + continue with self.subTest(name): self.assertEqual(reverse_mapping('multiprocessing.context', name), ('multiprocessing', name)) @@ -660,8 +826,61 @@ def test_multiprocessing_exceptions(self): ('multiprocessing.context', name)) +class CommandLineTest(unittest.TestCase): + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + """Dedent *string* and strip it from its surrounding whitespaces. + + This method is used by the other utility functions so that any + string to write or to match against can be freely indented. + """ + return dedent(string).strip() + + def set_pickle_data(self, data): + with open(self.filename, 'wb') as f: + pickle.dump(data, f) + + def invoke_pickle(self, *flags): + output = io.StringIO() + with contextlib.redirect_stdout(output): + pickle._main(args=[*flags, self.filename]) + return self.text_normalize(output.getvalue()) + + def test_invocation(self): + # test 'python -m pickle pickle_file' + data = { + 'a': [1, 2.0, 3+4j], + 'b': ('character string', b'byte string'), + 'c': 'string' + } + expect = ''' + {'a': [1, 2.0, (3+4j)], + 'b': ('character string', b'byte string'), + 'c': 'string'} + ''' + self.set_pickle_data(data) + + with self.subTest(data=data): + res = self.invoke_pickle() + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + @support.force_not_colorized + def test_unknown_flag(self): + stderr = io.StringIO() + with self.assertRaises(SystemExit): + # check that the parser help is shown + with contextlib.redirect_stderr(stderr): + _ = self.invoke_pickle('--unknown') + self.assertStartsWith(stderr.getvalue(), 'usage: ') + + def load_tests(loader, tests, pattern): - tests.addTest(doctest.DocTestSuite()) + tests.addTest(doctest.DocTestSuite(pickle)) return tests diff --git a/Lib/test/test_picklebuffer.py b/Lib/test/test_picklebuffer.py new file mode 100644 index 00000000000..f63be69cfc8 --- /dev/null +++ b/Lib/test/test_picklebuffer.py @@ -0,0 +1,168 @@ +"""Unit tests for the PickleBuffer object. + +Pickling tests themselves are in pickletester.py. +""" + +import gc +# TODO: RUSTPYTHON; Implment PickleBuffer +try: + from pickle import PickleBuffer +except ImportError: + PickleBuffer = None +import weakref +import unittest + +from test.support import import_helper + + +class B(bytes): + pass + + +class PickleBufferTest(unittest.TestCase): + + def check_memoryview(self, pb, equiv): + with memoryview(pb) as m: + with memoryview(equiv) as expected: + self.assertEqual(m.nbytes, expected.nbytes) + self.assertEqual(m.readonly, expected.readonly) + self.assertEqual(m.itemsize, expected.itemsize) + self.assertEqual(m.shape, expected.shape) + self.assertEqual(m.strides, expected.strides) + self.assertEqual(m.c_contiguous, expected.c_contiguous) + self.assertEqual(m.f_contiguous, expected.f_contiguous) + self.assertEqual(m.format, expected.format) + self.assertEqual(m.tobytes(), expected.tobytes()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constructor_failure(self): + with self.assertRaises(TypeError): + PickleBuffer() + with self.assertRaises(TypeError): + PickleBuffer("foo") + # Released memoryview fails taking a buffer + m = memoryview(b"foo") + m.release() + with self.assertRaises(ValueError): + PickleBuffer(m) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_basics(self): + pb = PickleBuffer(b"foo") + self.assertEqual(b"foo", bytes(pb)) + with memoryview(pb) as m: + self.assertTrue(m.readonly) + + pb = PickleBuffer(bytearray(b"foo")) + self.assertEqual(b"foo", bytes(pb)) + with memoryview(pb) as m: + self.assertFalse(m.readonly) + m[0] = 48 + self.assertEqual(b"0oo", bytes(pb)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_release(self): + pb = PickleBuffer(b"foo") + pb.release() + with self.assertRaises(ValueError) as raises: + memoryview(pb) + self.assertIn("operation forbidden on released PickleBuffer object", + str(raises.exception)) + # Idempotency + pb.release() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_cycle(self): + b = B(b"foo") + pb = PickleBuffer(b) + b.cycle = pb + wpb = weakref.ref(pb) + del b, pb + gc.collect() + self.assertIsNone(wpb()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_ndarray_2d(self): + # C-contiguous + ndarray = import_helper.import_module("_testbuffer").ndarray + arr = ndarray(list(range(12)), shape=(4, 3), format='<i') + self.assertTrue(arr.c_contiguous) + self.assertFalse(arr.f_contiguous) + pb = PickleBuffer(arr) + self.check_memoryview(pb, arr) + # Non-contiguous + arr = arr[::2] + self.assertFalse(arr.c_contiguous) + self.assertFalse(arr.f_contiguous) + pb = PickleBuffer(arr) + self.check_memoryview(pb, arr) + # F-contiguous + arr = ndarray(list(range(12)), shape=(3, 4), strides=(4, 12), format='<i') + self.assertTrue(arr.f_contiguous) + self.assertFalse(arr.c_contiguous) + pb = PickleBuffer(arr) + self.check_memoryview(pb, arr) + + # Tests for PickleBuffer.raw() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def check_raw(self, obj, equiv): + pb = PickleBuffer(obj) + with pb.raw() as m: + self.assertIsInstance(m, memoryview) + self.check_memoryview(m, equiv) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_raw(self): + for obj in (b"foo", bytearray(b"foo")): + with self.subTest(obj=obj): + self.check_raw(obj, obj) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_raw_ndarray(self): + # 1-D, contiguous + ndarray = import_helper.import_module("_testbuffer").ndarray + arr = ndarray(list(range(3)), shape=(3,), format='<h') + equiv = b"\x00\x00\x01\x00\x02\x00" + self.check_raw(arr, equiv) + # 2-D, C-contiguous + arr = ndarray(list(range(6)), shape=(2, 3), format='<h') + equiv = b"\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x05\x00" + self.check_raw(arr, equiv) + # 2-D, F-contiguous + arr = ndarray(list(range(6)), shape=(2, 3), strides=(2, 4), + format='<h') + # Note this is different from arr.tobytes() + equiv = b"\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x05\x00" + self.check_raw(arr, equiv) + # 0-D + arr = ndarray(456, shape=(), format='<i') + equiv = b'\xc8\x01\x00\x00' + self.check_raw(arr, equiv) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def check_raw_non_contiguous(self, obj): + pb = PickleBuffer(obj) + with self.assertRaisesRegex(BufferError, "non-contiguous"): + pb.raw() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_raw_non_contiguous(self): + # 1-D + ndarray = import_helper.import_module("_testbuffer").ndarray + arr = ndarray(list(range(6)), shape=(6,), format='<i')[::2] + self.check_raw_non_contiguous(arr) + # 2-D + arr = ndarray(list(range(12)), shape=(4, 3), format='<i')[::2] + self.check_raw_non_contiguous(arr) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_raw_released(self): + pb = PickleBuffer(b"foo") + pb.release() + with self.assertRaises(ValueError) as raises: + pb.raw() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 7dde4b17ed2..1a6256e097a 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -1,3 +1,4 @@ +import io import pickle import pickletools from test import support @@ -7,52 +8,6 @@ class OptimizedPickleTests(AbstractPickleTests, unittest.TestCase): - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffer_callback_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffer_callback_error() - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffers_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffers_error() - - def test_compat_pickle(self): # TODO: RUSTPYTHON, remove when this passes - super().test_compat_pickle() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO: RUSTPYTHON, remove when this passes - super().test_complex_newobj_ex() - - # TODO: RUSTPYTHON, TypeError: cannot pickle 'method' object - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_in_band_buffers() - - def test_notimplemented(self): # TODO: RUSTPYTHON, remove when this passes - super().test_notimplemented() - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers() - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers_writable_to_readonly() - - # TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray' - @unittest.expectedFailure - def test_optional_frames(self): # TODO: RUSTPYTHON, remove when this passes - super().test_optional_frames() - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_picklebuffer_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_picklebuffer_error() - def dumps(self, arg, proto=None, **kwargs): return pickletools.optimize(pickle.dumps(arg, proto, **kwargs)) @@ -65,8 +20,6 @@ def loads(self, buf, **kwds): # Test relies on writing by chunks into a file object. test_framed_write_sizes_with_delayed_writer = None - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_optimize_long_binget(self): data = [str(i) for i in range(257)] data.append(data[-1]) @@ -109,6 +62,452 @@ def test_optimize_binput_and_memoize(self): self.assertIs(unpickled2[1], unpickled2[2]) self.assertNotIn(pickle.BINPUT, pickled2) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_buffers_error(self): + return super().test_buffers_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytearray_memoization(self): + return super().test_bytearray_memoization() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bytes_memoization(self): + return super().test_bytes_memoization() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_c_methods(self): + return super().test_c_methods() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_in_band_buffers(self): + return super().test_in_band_buffers() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers(self): + return super().test_oob_buffers() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_oob_buffers_writable_to_readonly(self): + return super().test_oob_buffers_writable_to_readonly() + + +class SimpleReader: + def __init__(self, data): + self.data = data + self.pos = 0 + + def read(self, n): + data = self.data[self.pos: self.pos + n] + self.pos += n + return data + + def readline(self): + nl = self.data.find(b'\n', self.pos) + 1 + if not nl: + nl = len(self.data) + data = self.data[self.pos: nl] + self.pos = nl + return data + + +class GenopsTests(unittest.TestCase): + def test_genops(self): + it = pickletools.genops(b'(I123\nK\x12J\x12\x34\x56\x78t.') + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 0), + ('INT', 123, 1), + ('BININT1', 0x12, 6), + ('BININT', 0x78563412, 8), + ('TUPLE', None, 13), + ('STOP', None, 14), + ]) + + def test_from_file(self): + f = io.BytesIO(b'prefix(I123\nK\x12J\x12\x34\x56\x78t.suffix') + self.assertEqual(f.read(6), b'prefix') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 6), + ('INT', 123, 7), + ('BININT1', 0x12, 12), + ('BININT', 0x78563412, 14), + ('TUPLE', None, 19), + ('STOP', None, 20), + ]) + self.assertEqual(f.read(), b'suffix') + + def test_without_pos(self): + f = SimpleReader(b'(I123\nK\x12J\x12\x34\x56\x78t.') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, None), + ('INT', 123, None), + ('BININT1', 0x12, None), + ('BININT', 0x78563412, None), + ('TUPLE', None, None), + ('STOP', None, None), + ]) + + def test_no_stop(self): + it = pickletools.genops(b'N') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + 'pickle exhausted before seeing STOP'): + next(it) + + def test_truncated_data(self): + it = pickletools.genops(b'I123') + with self.assertRaisesRegex(ValueError, + 'no newline found when trying to read stringnl'): + next(it) + it = pickletools.genops(b'J\x12\x34') + with self.assertRaisesRegex(ValueError, + 'not enough data in stream to read int4'): + next(it) + + def test_unknown_opcode(self): + it = pickletools.genops(b'N\xff') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position 1, opcode b'\\xff' unknown"): + next(it) + + def test_unknown_opcode_without_pos(self): + f = SimpleReader(b'N\xff') + it = pickletools.genops(f) + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position <unknown>, opcode b'\\xff' unknown"): + next(it) + + +class DisTests(unittest.TestCase): + maxDiff = None + + def check_dis(self, data, expected, **kwargs): + out = io.StringIO() + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def check_dis_error(self, data, expected, expected_error, **kwargs): + out = io.StringIO() + with self.assertRaisesRegex(ValueError, expected_error): + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def test_mark(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_indentlevel(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''', indentlevel=2) + + def test_mark_without_pos(self): + self.check_dis(SimpleReader(b'(N(tl.'), '''\ +( MARK +N NONE +( MARK +t TUPLE (MARK at unknown opcode offset) +l LIST (MARK at unknown opcode offset) +. STOP +highest protocol among opcodes = 0 +''') + + def test_no_mark(self): + self.check_dis_error(b'Nt.', '''\ + 0: N NONE + 1: t TUPLE +''', 'no MARK exists on stack') + + def test_put(self): + self.check_dis(b'Np0\nq\x01r\x02\x00\x00\x00\x94.', '''\ + 0: N NONE + 1: p PUT 0 + 4: q BINPUT 1 + 6: r LONG_BINPUT 2 + 11: \\x94 MEMOIZE (as 3) + 12: . STOP +highest protocol among opcodes = 4 +''') + + def test_put_redefined(self): + self.check_dis(b'Np1\np1\nq\x01r\x01\x00\x00\x00\x94.', '''\ + 0: N NONE + 1: p PUT 1 + 4: p PUT 1 + 7: q BINPUT 1 + 9: r LONG_BINPUT 1 + 14: \\x94 MEMOIZE (as 1) + 15: . STOP +highest protocol among opcodes = 4 +''') + + def test_put_empty_stack(self): + self.check_dis_error(b'p0\n', '''\ + 0: p PUT 0 +''', "stack is empty -- can't store into memo") + + def test_put_markobject(self): + self.check_dis_error(b'(p0\n', '''\ + 0: ( MARK + 1: p PUT 0 +''', "can't store markobject in the memo") + + def test_get(self): + self.check_dis(b'(Np1\ng1\nh\x01j\x01\x00\x00\x00t.', '''\ + 0: ( MARK + 1: N NONE + 2: p PUT 1 + 5: g GET 1 + 8: h BINGET 1 + 10: j LONG_BINGET 1 + 15: t TUPLE (MARK at 0) + 16: . STOP +highest protocol among opcodes = 1 +''') + + def test_get_without_put(self): + self.check_dis_error(b'g1\n.', '''\ + 0: g GET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'h\x01.', '''\ + 0: h BINGET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'j\x01\x00\x00\x00.', '''\ + 0: j LONG_BINGET 1 +''', 'memo key 1 has never been stored into') + + def test_memo(self): + memo = {} + self.check_dis(b'Np1\n.', '''\ + 0: N NONE + 1: p PUT 1 + 4: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + self.check_dis(b'g1\n.', '''\ + 0: g GET 1 + 3: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + + def test_mark_pop(self): + self.check_dis(b'(N00N.', '''\ + 0: ( MARK + 1: N NONE + 2: 0 POP + 3: 0 POP (MARK at 0) + 4: N NONE + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_too_small_stack(self): + self.check_dis_error(b'a', '''\ + 0: a APPEND +''', 'tries to pop 2 items from stack with only 0 items') + self.check_dis_error(b']a', '''\ + 0: ] EMPTY_LIST + 1: a APPEND +''', 'tries to pop 2 items from stack with only 1 items') + + def test_no_stop(self): + self.check_dis_error(b'N', '''\ + 0: N NONE +''', 'pickle exhausted before seeing STOP') + + def test_truncated_data(self): + self.check_dis_error(b'NI123', '''\ + 0: N NONE +''', 'no newline found when trying to read stringnl') + self.check_dis_error(b'NJ\x12\x34', '''\ + 0: N NONE +''', 'not enough data in stream to read int4') + + def test_unknown_opcode(self): + self.check_dis_error(b'N\xff', '''\ + 0: N NONE +''', r"at position 1, opcode b'\\xff' unknown") + + def test_stop_not_empty_stack(self): + self.check_dis_error(b']N.', '''\ + 0: ] EMPTY_LIST + 1: N NONE + 2: . STOP +highest protocol among opcodes = 1 +''', r'stack not empty after STOP: \[list\]') + + def test_annotate(self): + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=1) + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + self.check_dis(b'(((((((ttttttt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: ( MARK Push markobject onto the stack. + 2: ( MARK Push markobject onto the stack. + 3: ( MARK Push markobject onto the stack. + 4: ( MARK Push markobject onto the stack. + 5: ( MARK Push markobject onto the stack. + 6: ( MARK Push markobject onto the stack. + 7: t TUPLE (MARK at 6) Build a tuple out of the topmost stack slice, after markobject. + 8: t TUPLE (MARK at 5) Build a tuple out of the topmost stack slice, after markobject. + 9: t TUPLE (MARK at 4) Build a tuple out of the topmost stack slice, after markobject. + 10: t TUPLE (MARK at 3) Build a tuple out of the topmost stack slice, after markobject. + 11: t TUPLE (MARK at 2) Build a tuple out of the topmost stack slice, after markobject. + 12: t TUPLE (MARK at 1) Build a tuple out of the topmost stack slice, after markobject. + 13: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 14: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + + def test_string(self): + self.check_dis(b"S'abc'\n.", '''\ + 0: S STRING 'abc' + 7: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b'S"abc"\n.', '''\ + 0: S STRING 'abc' + 7: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b"S'\xc3\xb5'\n.", '''\ + 0: S STRING '\\xc3\\xb5' + 6: . STOP +highest protocol among opcodes = 0 +''') + + def test_string_without_quotes(self): + self.check_dis_error(b"Sabc'\n.", '', + 'no string quotes around b"abc\'"') + self.check_dis_error(b'Sabc"\n.', '', + "no string quotes around b'abc\"'") + self.check_dis_error(b"S'abc\n.", '', + '''string quote b"'" not found at both ends of b"'abc"''') + self.check_dis_error(b'S"abc\n.', '', + r"""string quote b'"' not found at both ends of b'"abc'""") + self.check_dis_error(b"S'abc\"\n.", '', + r"""string quote b"'" not found at both ends of b'\\'abc"'""") + self.check_dis_error(b"S\"abc'\n.", '', + r"""string quote b'"' not found at both ends of b'"abc\\''""") + + def test_binstring(self): + self.check_dis(b"T\x03\x00\x00\x00abc.", '''\ + 0: T BINSTRING 'abc' + 8: . STOP +highest protocol among opcodes = 1 +''') + self.check_dis(b"T\x02\x00\x00\x00\xc3\xb5.", '''\ + 0: T BINSTRING '\\xc3\\xb5' + 7: . STOP +highest protocol among opcodes = 1 +''') + + def test_short_binstring(self): + self.check_dis(b"U\x03abc.", '''\ + 0: U SHORT_BINSTRING 'abc' + 5: . STOP +highest protocol among opcodes = 1 +''') + self.check_dis(b"U\x02\xc3\xb5.", '''\ + 0: U SHORT_BINSTRING '\\xc3\\xb5' + 4: . STOP +highest protocol among opcodes = 1 +''') + + def test_global(self): + self.check_dis(b"cmodule\nname\n.", '''\ + 0: c GLOBAL 'module name' + 13: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b"cm\xc3\xb6dule\nn\xc3\xa4me\n.", '''\ + 0: c GLOBAL 'm\xf6dule n\xe4me' + 15: . STOP +highest protocol among opcodes = 0 +''') + + def test_inst(self): + self.check_dis(b"(imodule\nname\n.", '''\ + 0: ( MARK + 1: i INST 'module name' (MARK at 0) + 14: . STOP +highest protocol among opcodes = 0 +''') + + def test_persid(self): + self.check_dis(b"Pabc\n.", '''\ + 0: P PERSID 'abc' + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_constants(self): + self.check_dis(b"(NI00\nI01\n\x89\x88t.", '''\ + 0: ( MARK + 1: N NONE + 2: I INT False + 6: I INT True + 10: \\x89 NEWFALSE + 11: \\x88 NEWTRUE + 12: t TUPLE (MARK at 0) + 13: . STOP +highest protocol among opcodes = 2 +''') + + def test_integers(self): + self.check_dis(b"(I0\nI1\nI10\nI011\nL12\nL13L\nL014\nL015L\nt.", '''\ + 0: ( MARK + 1: I INT 0 + 4: I INT 1 + 7: I INT 10 + 11: I INT 11 + 16: L LONG 12 + 20: L LONG 13 + 25: L LONG 14 + 30: L LONG 15 + 36: t TUPLE (MARK at 0) + 37: . STOP +highest protocol among opcodes = 0 +''') + + def test_nondecimal_integers(self): + self.check_dis_error(b'I0b10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0o10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0x10\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0b10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0o10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0x10L\n.', '', 'invalid literal for int') + class MiscTestCase(unittest.TestCase): def test__all__(self): @@ -144,8 +543,8 @@ def test__all__(self): def load_tests(loader, tests, pattern): - # TODO: RUSTPYTHON - # tests.addTest(doctest.DocTestSuite(pickletools)) + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON; Remove this + tests.addTest(doctest.DocTestSuite(pickletools, checker=DocTestChecker())) # XXX: RUSTPYTHON return tests diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 3a10ec8fc33..f5444409593 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -1,4 +1,5 @@ -from test.support.import_helper import unload, CleanImport +from pathlib import Path +from test.support.import_helper import unload from test.support.warnings_helper import check_warnings import unittest import sys @@ -11,6 +12,10 @@ import shutil import zipfile +from test.support.import_helper import DirsOnSysPath +from test.support.os_helper import FakePath +from test.test_importlib.util import uncache + # Note: pkgutil.walk_packages is currently tested in test_runpy. This is # a hack to get a major issue resolved for 3.3b2. Longer term, it should # be moved back here, perhaps by factoring out the helper code for @@ -91,6 +96,45 @@ def test_getdata_zipfile(self): del sys.modules[pkg] + def test_issue44061_iter_modules(self): + #see: issue44061 + zip = 'test_getdata_zipfile.zip' + pkg = 'test_getdata_zipfile' + + # Include a LF and a CRLF, to test that binary data is read back + RESOURCE_DATA = b'Hello, world!\nSecond line\r\nThird line' + + # Make a package with some resources + zip_file = os.path.join(self.dirname, zip) + z = zipfile.ZipFile(zip_file, 'w') + + # Empty init.py + z.writestr(pkg + '/__init__.py', "") + # Resource files, res.txt + z.writestr(pkg + '/res.txt', RESOURCE_DATA) + z.close() + + # Check we can read the resources + sys.path.insert(0, zip_file) + try: + res = pkgutil.get_data(pkg, 'res.txt') + self.assertEqual(res, RESOURCE_DATA) + + # make sure iter_modules accepts Path objects + names = [] + for moduleinfo in pkgutil.iter_modules([FakePath(zip_file)]): + self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo) + names.append(moduleinfo.name) + self.assertEqual(names, [pkg]) + finally: + del sys.path[0] + sys.modules.pop(pkg, None) + + # assert path must be None or list of paths + expected_msg = "path must be None or list of paths to look for modules in" + with self.assertRaisesRegex(ValueError, expected_msg): + list(pkgutil.iter_modules("invalid_path")) + def test_unreadable_dir_on_syspath(self): # issue7367 - walk_packages failed if unreadable dir on sys.path package_name = "unreadable_package" @@ -187,8 +231,7 @@ def test_walk_packages_raises_on_string_or_bytes_input(self): with self.assertRaises((TypeError, ValueError)): list(pkgutil.walk_packages(bytes_input)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_name_resolution(self): import logging import logging.handlers @@ -280,6 +323,38 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_import_rebinding(self): + # The same data is also used for testing import in test_import and + # mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + class PkgutilPEP302Tests(unittest.TestCase): @@ -391,7 +466,7 @@ def test_iter_importers(self): importers = list(iter_importers(fullname)) expected_importer = get_importer(pathitem) for finder in importers: - spec = pkgutil._get_spec(finder, fullname) + spec = finder.find_spec(fullname) loader = spec.loader try: loader = loader.loader @@ -403,7 +478,7 @@ def test_iter_importers(self): self.assertEqual(finder, expected_importer) self.assertIsInstance(loader, importlib.machinery.SourceFileLoader) - self.assertIsNone(pkgutil._get_spec(finder, pkgname)) + self.assertIsNone(finder.find_spec(pkgname)) with self.assertRaises(ImportError): list(iter_importers('invalid.module')) @@ -448,7 +523,43 @@ def test_mixed_namespace(self): del sys.modules['foo.bar'] del sys.modules['foo.baz'] - # XXX: test .pkg files + + def test_extend_path_argument_types(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + # If the input path is not a list it is returned unchanged + self.assertEqual('notalist', pkgutil.extend_path('notalist', 'foo')) + self.assertEqual(('not', 'a', 'list'), pkgutil.extend_path(('not', 'a', 'list'), 'foo')) + self.assertEqual(123, pkgutil.extend_path(123, 'foo')) + self.assertEqual(None, pkgutil.extend_path(None, 'foo')) + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] + + + def test_extend_path_pkg_files(self): + pkgname = 'foo' + dirname_0 = self.create_init(pkgname) + + with open(os.path.join(dirname_0, 'bar.pkg'), 'w') as pkg_file: + pkg_file.write('\n'.join([ + 'baz', + '/foo/bar/baz', + '', + '#comment' + ])) + + extended_paths = pkgutil.extend_path(sys.path, 'bar') + + self.assertEqual(extended_paths[:-2], sys.path) + self.assertEqual(extended_paths[-2], 'baz') + self.assertEqual(extended_paths[-1], '/foo/bar/baz') + + # Cleanup + shutil.rmtree(dirname_0) + del sys.path[0] class NestedNamespacePackageTest(unittest.TestCase): @@ -497,84 +608,18 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - def check_deprecated(self): - return check_warnings( - ("This emulation is deprecated and slated for removal in " - "Python 3.12; use 'importlib' instead", - DeprecationWarning)) - - def test_importer_deprecated(self): - with self.check_deprecated(): - pkgutil.ImpImporter("") - - def test_loader_deprecated(self): - with self.check_deprecated(): - pkgutil.ImpLoader("", "", "", "") - - def test_get_loader_avoids_emulation(self): - with check_warnings() as w: - self.assertIsNotNone(pkgutil.get_loader("sys")) - self.assertIsNotNone(pkgutil.get_loader("os")) - self.assertIsNotNone(pkgutil.get_loader("test.support")) - self.assertEqual(len(w.warnings), 0) - - @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') - def test_get_loader_handles_missing_loader_attribute(self): - global __loader__ - this_loader = __loader__ - del __loader__ - try: - with check_warnings() as w: - self.assertIsNotNone(pkgutil.get_loader(__name__)) - self.assertEqual(len(w.warnings), 0) - finally: - __loader__ = this_loader - - def test_get_loader_handles_missing_spec_attribute(self): - name = 'spam' - mod = type(sys)(name) - del mod.__spec__ - with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - self.assertIsNone(loader) - - def test_get_loader_handles_spec_attribute_none(self): - name = 'spam' - mod = type(sys)(name) - mod.__spec__ = None - with CleanImport(name): - sys.modules[name] = mod - loader = pkgutil.get_loader(name) - self.assertIsNone(loader) - - def test_get_loader_None_in_sys_modules(self): - name = 'totally bogus' - sys.modules[name] = None - try: - loader = pkgutil.get_loader(name) - finally: - del sys.modules[name] - self.assertIsNone(loader) - - def test_find_loader_missing_module(self): - name = 'totally bogus' - loader = pkgutil.find_loader(name) - self.assertIsNone(loader) - - def test_find_loader_avoids_emulation(self): - with check_warnings() as w: - self.assertIsNotNone(pkgutil.find_loader("sys")) - self.assertIsNotNone(pkgutil.find_loader("os")) - self.assertIsNotNone(pkgutil.find_loader("test.support")) - self.assertEqual(len(w.warnings), 0) - def test_get_importer_avoids_emulation(self): # We use an illegal path so *none* of the path hooks should fire with check_warnings() as w: self.assertIsNone(pkgutil.get_importer("*??")) self.assertEqual(len(w.warnings), 0) + def test_issue44061(self): + try: + pkgutil.get_importer(Path("/home")) + except AttributeError: + self.fail("Unexpected AttributeError when calling get_importer") + def test_iter_importers_avoids_emulation(self): with check_warnings() as w: for importer in pkgutil.iter_importers(): pass diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index c9f27575b51..ed277276b51 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -10,6 +10,14 @@ from test import support from test.support import os_helper +try: + # Some of the iOS tests need ctypes to operate. + # Confirm that the ctypes module is available + # is available. + import _ctypes +except ImportError: + _ctypes = None + FEDORA_OS_RELEASE = """\ NAME=Fedora VERSION="32 (Thirty Two)" @@ -123,10 +131,6 @@ def test_sys_version(self): for input, output in ( ('2.4.3 (#1, Jun 21 2006, 13:54:21) \n[GCC 3.3.4 (pre 3.3.5 20040809)]', ('CPython', '2.4.3', '', '', '1', 'Jun 21 2006 13:54:21', 'GCC 3.3.4 (pre 3.3.5 20040809)')), - ('IronPython 1.0.60816 on .NET 2.0.50727.42', - ('IronPython', '1.0.60816', '', '', '', '', '.NET 2.0.50727.42')), - ('IronPython 1.0 (1.0.61005.1977) on .NET 2.0.50727.42', - ('IronPython', '1.0.0', '', '', '', '', '.NET 2.0.50727.42')), ('2.4.3 (truncation, date, t) \n[GCC]', ('CPython', '2.4.3', '', '', 'truncation', 'date t', 'GCC')), ('2.4.3 (truncation, date, ) \n[GCC]', @@ -161,20 +165,11 @@ def test_sys_version(self): ('r261:67515', 'Dec 6 2008 15:26:00'), 'GCC 4.0.1 (Apple Computer, Inc. build 5370)'), - ("IronPython 2.0 (2.0.0.0) on .NET 2.0.50727.3053", None, "cli") - : - ("IronPython", "2.0.0", "", "", ("", ""), - ".NET 2.0.50727.3053"), - - ("2.6.1 (IronPython 2.6.1 (2.6.10920.0) on .NET 2.0.50727.1433)", None, "cli") + ("3.10.8 (tags/v3.10.8:aaaf517424, Feb 14 2023, 16:28:12) [GCC 9.4.0]", + None, "linux") : - ("IronPython", "2.6.1", "", "", ("", ""), - ".NET 2.0.50727.1433"), - - ("2.7.4 (IronPython 2.7.4 (2.7.0.40) on Mono 4.0.30319.1 (32-bit))", None, "cli") - : - ("IronPython", "2.7.4", "", "", ("", ""), - "Mono 4.0.30319.1 (32-bit)"), + ('CPython', '3.10.8', '', '', + ('tags/v3.10.8:aaaf517424', 'Feb 14 2023 16:28:12'), 'GCC 9.4.0'), ("2.5 (trunk:6107, Mar 26 2009, 13:02:18) \n[Java HotSpot(TM) Client VM (\"Apple Computer, Inc.\")]", ('Jython', 'trunk', '6107'), "java1.5.0_16") @@ -205,6 +200,9 @@ def test_sys_version(self): self.assertEqual(platform.python_build(), info[4]) self.assertEqual(platform.python_compiler(), info[5]) + with self.assertRaises(ValueError): + platform._sys_version('2. 4.3 (truncation) \n[GCC]') + def test_system_alias(self): res = platform.system_alias( platform.system(), @@ -229,6 +227,38 @@ def test_uname(self): self.assertEqual(res[-1], res.processor) self.assertEqual(len(res), 6) + if os.name == "posix": + uname = os.uname() + self.assertEqual(res.node, uname.nodename) + self.assertEqual(res.version, uname.version) + self.assertEqual(res.machine, uname.machine) + + if sys.platform == "android": + self.assertEqual(res.system, "Android") + self.assertEqual(res.release, platform.android_ver().release) + elif sys.platform == "ios": + # Platform module needs ctypes for full operation. If ctypes + # isn't available, there's no ObjC module, and dummy values are + # returned. + if _ctypes: + self.assertIn(res.system, {"iOS", "iPadOS"}) + self.assertEqual(res.release, platform.ios_ver().release) + else: + self.assertEqual(res.system, "") + self.assertEqual(res.release, "") + else: + self.assertEqual(res.system, uname.sysname) + self.assertEqual(res.release, uname.release) + + + @unittest.skipUnless(sys.platform.startswith('win'), "windows only test") + def test_uname_win32_without_wmi(self): + def raises_oserror(*a): + raise OSError() + + with support.swap_attr(platform, '_wmi_query', raises_oserror): + self.test_uname() + def test_uname_cast_to_tuple(self): res = platform.uname() expected = ( @@ -297,28 +327,66 @@ def test_uname_win32_ARCHITEW6432(self): # on 64 bit Windows: if PROCESSOR_ARCHITEW6432 exists we should be # using it, per # http://blogs.msdn.com/david.wang/archive/2006/03/26/HOWTO-Detect-Process-Bitness.aspx - try: + + # We also need to suppress WMI checks, as those are reliable and + # overrule the environment variables + def raises_oserror(*a): + raise OSError() + + with support.swap_attr(platform, '_wmi_query', raises_oserror): with os_helper.EnvironmentVarGuard() as environ: - if 'PROCESSOR_ARCHITEW6432' in environ: + try: del environ['PROCESSOR_ARCHITEW6432'] - environ['PROCESSOR_ARCHITECTURE'] = 'foo' - platform._uname_cache = None - system, node, release, version, machine, processor = platform.uname() - self.assertEqual(machine, 'foo') - environ['PROCESSOR_ARCHITEW6432'] = 'bar' - platform._uname_cache = None - system, node, release, version, machine, processor = platform.uname() - self.assertEqual(machine, 'bar') - finally: - platform._uname_cache = None + environ['PROCESSOR_ARCHITECTURE'] = 'foo' + platform._uname_cache = None + system, node, release, version, machine, processor = platform.uname() + self.assertEqual(machine, 'foo') + environ['PROCESSOR_ARCHITEW6432'] = 'bar' + platform._uname_cache = None + system, node, release, version, machine, processor = platform.uname() + self.assertEqual(machine, 'bar') + finally: + platform._uname_cache = None def test_java_ver(self): - res = platform.java_ver() - if sys.platform == 'java': - self.assertTrue(all(res)) + import re + msg = re.escape( + "'java_ver' is deprecated and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, msg): + res = platform.java_ver() + self.assertEqual(len(res), 4) + @unittest.skipUnless(support.MS_WINDOWS, 'This test only makes sense on Windows') def test_win32_ver(self): - res = platform.win32_ver() + release1, version1, csd1, ptype1 = 'a', 'b', 'c', 'd' + res = platform.win32_ver(release1, version1, csd1, ptype1) + self.assertEqual(len(res), 4) + release, version, csd, ptype = res + if release: + # Currently, release names always come from internal dicts, + # but this could change over time. For now, we just check that + # release is something different from what we have passed. + self.assertNotEqual(release, release1) + if version: + # It is rather hard to test explicit version without + # going deep into the details. + self.assertIn('.', version) + for v in version.split('.'): + int(v) # should not fail + if csd: + self.assertTrue(csd.startswith('SP'), msg=csd) + if ptype: + if os.cpu_count() > 1: + self.assertIn('Multiprocessor', ptype) + else: + self.assertIn('Uniprocessor', ptype) + + @unittest.skipIf(support.MS_WINDOWS, 'This test only makes sense on non Windows') + def test_win32_ver_on_non_windows(self): + release, version, csd, ptype = 'a', '1.0', 'c', 'd' + res = platform.win32_ver(release, version, csd, ptype) + self.assertSequenceEqual(res, (release, version, csd, ptype), seq_type=tuple) def test_mac_ver(self): res = platform.mac_ver() @@ -372,6 +440,56 @@ def test_mac_ver_with_fork(self): # parent support.wait_process(pid, exitcode=0) + def test_ios_ver(self): + result = platform.ios_ver() + + # ios_ver is only fully available on iOS where ctypes is available. + if sys.platform == "ios" and _ctypes: + system, release, model, is_simulator = result + # Result is a namedtuple + self.assertEqual(result.system, system) + self.assertEqual(result.release, release) + self.assertEqual(result.model, model) + self.assertEqual(result.is_simulator, is_simulator) + + # We can't assert specific values without reproducing the logic of + # ios_ver(), so we check that the values are broadly what we expect. + + # System is either iOS or iPadOS, depending on the test device + self.assertIn(system, {"iOS", "iPadOS"}) + + # Release is a numeric version specifier with at least 2 parts + parts = release.split(".") + self.assertGreaterEqual(len(parts), 2) + self.assertTrue(all(part.isdigit() for part in parts)) + + # If this is a simulator, we get a high level device descriptor + # with no identifying model number. If this is a physical device, + # we get a model descriptor like "iPhone13,1" + if is_simulator: + self.assertIn(model, {"iPhone", "iPad"}) + else: + self.assertTrue( + (model.startswith("iPhone") or model.startswith("iPad")) + and "," in model + ) + + self.assertEqual(type(is_simulator), bool) + else: + # On non-iOS platforms, calling ios_ver doesn't fail; you get + # default values + self.assertEqual(result.system, "") + self.assertEqual(result.release, "") + self.assertEqual(result.model, "") + self.assertFalse(result.is_simulator) + + # Check the fallback values can be overridden by arguments + override = platform.ios_ver("Foo", "Bar", "Whiz", True) + self.assertEqual(override.system, "Foo") + self.assertEqual(override.release, "Bar") + self.assertEqual(override.model, "Whiz") + self.assertTrue(override.is_simulator) + @unittest.skipIf(support.is_emscripten, "Does not apply to Emscripten") def test_libc_ver(self): # check that libc_ver(executable) doesn't raise an exception @@ -421,6 +539,43 @@ def test_libc_ver(self): self.assertEqual(platform.libc_ver(filename, chunksize=chunksize), ('glibc', '1.23.4')) + def test_android_ver(self): + res = platform.android_ver() + self.assertIsInstance(res, tuple) + self.assertEqual(res, (res.release, res.api_level, res.manufacturer, + res.model, res.device, res.is_emulator)) + + if sys.platform == "android": + for name in ["release", "manufacturer", "model", "device"]: + with self.subTest(name): + value = getattr(res, name) + self.assertIsInstance(value, str) + self.assertNotEqual(value, "") + + self.assertIsInstance(res.api_level, int) + self.assertGreaterEqual(res.api_level, sys.getandroidapilevel()) + + self.assertIsInstance(res.is_emulator, bool) + + # When not running on Android, it should return the default values. + else: + self.assertEqual(res.release, "") + self.assertEqual(res.api_level, 0) + self.assertEqual(res.manufacturer, "") + self.assertEqual(res.model, "") + self.assertEqual(res.device, "") + self.assertEqual(res.is_emulator, False) + + # Default values may also be overridden using parameters. + res = platform.android_ver( + "alpha", 1, "bravo", "charlie", "delta", True) + self.assertEqual(res.release, "alpha") + self.assertEqual(res.api_level, 1) + self.assertEqual(res.manufacturer, "bravo") + self.assertEqual(res.model, "charlie") + self.assertEqual(res.device, "delta") + self.assertEqual(res.is_emulator, True) + @support.cpython_only def test__comparable_version(self): from platform import _comparable_version as V @@ -467,7 +622,8 @@ def test_macos(self): 'root:xnu-4570.71.2~1/RELEASE_X86_64'), 'x86_64', 'i386') arch = ('64bit', '') - with mock.patch.object(platform, 'uname', return_value=uname), \ + with mock.patch.object(sys, "platform", "darwin"), \ + mock.patch.object(platform, 'uname', return_value=uname), \ mock.patch.object(platform, 'architecture', return_value=arch): for mac_ver, expected_terse, expected in [ # darwin: mac_ver() returns empty strings diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index c6d4cfe5c6b..389da145e6d 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -6,10 +6,15 @@ import unittest import plistlib import os +import sys +import json import datetime import codecs +import subprocess import binascii import collections +import time +import zoneinfo from test import support from test.support import os_helper from io import BytesIO @@ -505,6 +510,19 @@ def test_bytes(self): data2 = plistlib.dumps(pl2) self.assertEqual(data, data2) + def test_loads_str_with_xml_fmt(self): + pl = self._create() + b = plistlib.dumps(pl) + s = b.decode() + self.assertIsInstance(s, str) + pl2 = plistlib.loads(s) + self.assertEqual(pl, pl2) + + def test_loads_str_with_binary_fmt(self): + msg = "value must be bytes-like object when fmt is FMT_BINARY" + with self.assertRaisesRegex(TypeError, msg): + plistlib.loads('test', fmt=plistlib.FMT_BINARY) + def test_indentation_array(self): data = [[[[[[[[{'test': b'aaaaaa'}]]]]]]]] self.assertEqual(plistlib.loads(plistlib.dumps(data)), data) @@ -734,8 +752,6 @@ def test_non_bmp_characters(self): data = plistlib.dumps(pl, fmt=fmt) self.assertEqual(plistlib.loads(data), pl) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lone_surrogates(self): for fmt in ALL_FORMATS: with self.subTest(fmt=fmt): @@ -754,8 +770,7 @@ def test_nondictroot(self): self.assertEqual(test1, result1) self.assertEqual(test2, result2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalidarray(self): for i in ["<key>key inside an array</key>", "<key>key inside an array2</key><real>3</real>", @@ -763,8 +778,7 @@ def test_invalidarray(self): self.assertRaises(ValueError, plistlib.loads, ("<plist><array>%s</array></plist>"%i).encode()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invaliddict(self): for i in ["<key><true/>k</key><string>compound key</string>", "<key>single key</key>", @@ -776,14 +790,12 @@ def test_invaliddict(self): self.assertRaises(ValueError, plistlib.loads, ("<plist><array><dict>%s</dict></array></plist>"%i).encode()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalidinteger(self): self.assertRaises(ValueError, plistlib.loads, b"<plist><integer>not integer</integer></plist>") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalidreal(self): self.assertRaises(ValueError, plistlib.loads, b"<plist><integer>not real</integer></plist>") @@ -840,18 +852,64 @@ def test_modified_uid_huge(self): with self.assertRaises(OverflowError): plistlib.dumps(huge_uid, fmt=plistlib.FMT_BINARY) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xml_plist_with_entity_decl(self): with self.assertRaisesRegex(plistlib.InvalidFileException, "XML entity declarations are not supported"): plistlib.loads(XML_PLIST_WITH_ENTITY, fmt=plistlib.FMT_XML) + def test_load_aware_datetime(self): + dt = plistlib.loads(b"<plist><date>2023-12-10T08:03:30Z</date></plist>", + aware_datetime=True) + self.assertEqual(dt.tzinfo, datetime.UTC) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + def test_dump_utc_aware_datetime(self): + dt = datetime.datetime(2345, 6, 7, 8, 9, 10, tzinfo=datetime.UTC) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + loaded_dt = plistlib.loads(s, fmt=fmt, aware_datetime=True) + self.assertEqual(loaded_dt.tzinfo, datetime.UTC) + self.assertEqual(loaded_dt, dt) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + s = plistlib.dumps(dt, fmt=plistlib.FMT_XML, aware_datetime=False) + self.assertIn(b"2345-06-07T08:00:00Z", s) + + def test_dump_naive_datetime_with_aware_datetime_option(self): + # Save a naive datetime with aware_datetime set to true. This will lead + # to having different time as compared to the current machine's + # timezone, which is UTC. + dt = datetime.datetime(2003, 6, 7, 8, tzinfo=None) + for fmt in ALL_FORMATS: + s = plistlib.dumps(dt, fmt=fmt, aware_datetime=True) + parsed = plistlib.loads(s, aware_datetime=False) + expected = dt.astimezone(datetime.UTC).replace(tzinfo=None) + self.assertEqual(parsed, expected) + class TestBinaryPlistlib(unittest.TestCase): - @staticmethod - def decode(*objects, offset_size=1, ref_size=1): + def build(self, *objects, offset_size=1, ref_size=1): data = [b'bplist00'] offset = 8 offsets = [] @@ -863,7 +921,11 @@ def decode(*objects, offset_size=1, ref_size=1): len(objects), 0, offset) data.extend(offsets) data.append(tail) - return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY) + return b''.join(data) + + def decode(self, *objects, offset_size=1, ref_size=1): + data = self.build(*objects, offset_size=offset_size, ref_size=ref_size) + return plistlib.loads(data, fmt=plistlib.FMT_BINARY) def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers @@ -917,12 +979,12 @@ def test_cycles(self): self.assertIs(b['x'], b) def test_deep_nesting(self): - for N in [300, 100000]: + for N in [50, 300, 100_000]: chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] try: result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4) except RecursionError: - pass + self.assertGreater(N, sys.getrecursionlimit()) else: for i in range(N): self.assertIsInstance(result, list) @@ -934,7 +996,7 @@ def test_large_timestamp(self): # Issue #26709: 32-bit timestamp out of range for ts in -2**31-1, 2**31: with self.subTest(ts=ts): - d = (datetime.datetime.utcfromtimestamp(0) + + d = (datetime.datetime(1970, 1, 1, 0, 0) + datetime.timedelta(seconds=ts)) data = plistlib.dumps(d, fmt=plistlib.FMT_BINARY) self.assertEqual(plistlib.loads(data), d) @@ -971,6 +1033,60 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_truncated_large_data(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + def check(data): + with open(os_helper.TESTFN, 'wb') as f: + f.write(data) + # buffered file + with open(os_helper.TESTFN, 'rb') as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + # unbuffered file + with open(os_helper.TESTFN, 'rb', buffering=0) as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + for w in range(20, 64): + s = 1 << w + # data + check(self.build(b'\x4f\x13' + s.to_bytes(8, 'big'))) + # ascii string + check(self.build(b'\x5f\x13' + s.to_bytes(8, 'big'))) + # unicode string + check(self.build(b'\x6f\x13' + s.to_bytes(8, 'big'))) + # array + check(self.build(b'\xaf\x13' + s.to_bytes(8, 'big'))) + # dict + check(self.build(b'\xdf\x13' + s.to_bytes(8, 'big'))) + # number of objects + check(b'bplist00' + struct.pack('>6xBBQQQ', 1, 1, s, 0, 8)) + + def test_load_aware_datetime(self): + data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' + b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x11') + self.assertEqual(plistlib.loads(data, aware_datetime=True), + datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC)) + + @unittest.skipUnless("America/Los_Angeles" in zoneinfo.available_timezones(), + "Can't find timezone datebase") + def test_dump_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, + tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles")) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + + # TODO: RUSTPYTHON + # The error message is different + # In CPython, there is a separate .c file for datetime, which raises a different error message + @unittest.expectedFailure + def test_dump_utc_aware_datetime_without_aware_datetime_option(self): + dt = datetime.datetime(2345, 6, 7, 8, tzinfo=datetime.UTC) + msg = "can't subtract offset-naive and offset-aware datetimes" + with self.assertRaisesRegex(TypeError, msg): + plistlib.dumps(dt, fmt=plistlib.FMT_BINARY, aware_datetime=False) + class TestKeyedArchive(unittest.TestCase): def test_keyed_archive_data(self): @@ -1009,6 +1125,78 @@ def test__all__(self): not_exported = {"PlistFormat", "PLISTHEADER"} support.check__all__(self, plistlib, not_exported=not_exported) +@unittest.skipUnless(sys.platform == "darwin", "plutil utility is for Mac os") +class TestPlutil(unittest.TestCase): + file_name = "plutil_test.plist" + properties = { + "fname" : "H", + "lname":"A", + "marks" : {"a":100, "b":0x10} + } + exptected_properties = { + "fname" : "H", + "lname": "A", + "marks" : {"a":100, "b":16} + } + pl = { + "HexType" : 0x0100000c, + "IntType" : 0o123 + } + + @classmethod + def setUpClass(cls) -> None: + ## Generate plist file with plistlib and parse with plutil + with open(cls.file_name,'wb') as f: + plistlib.dump(cls.properties, f, fmt=plistlib.FMT_BINARY) + + @classmethod + def tearDownClass(cls) -> None: + os.remove(cls.file_name) + + def get_lint_status(self): + return subprocess.run(['plutil', "-lint", self.file_name], capture_output=True, text=True).stdout + + def convert_to_json(self): + """Convert binary file to json using plutil + """ + subprocess.run(['plutil', "-convert", 'json', self.file_name]) + + def convert_to_bin(self): + """Convert file to binary using plutil + """ + subprocess.run(['plutil', "-convert", 'binary1', self.file_name]) + + def write_pl(self): + """Write Hex properties to file using writePlist + """ + with open(self.file_name, 'wb') as f: + plistlib.dump(self.pl, f, fmt=plistlib.FMT_BINARY) + + def test_lint_status(self): + # check lint status of file using plutil + self.assertEqual(f"{self.file_name}: OK\n", self.get_lint_status()) + + def check_content(self): + # check file content with plutil converting binary to json + self.convert_to_json() + with open(self.file_name) as f: + ff = json.loads(f.read()) + self.assertEqual(ff, self.exptected_properties) + + def check_plistlib_parse(self): + # Generate plist files with plutil and parse with plistlib + self.convert_to_bin() + with open(self.file_name, 'rb') as f: + self.assertEqual(plistlib.load(f), self.exptected_properties) + + def test_octal_and_hex(self): + self.write_pl() + self.convert_to_json() + with open(self.file_name, 'r') as f: + p = json.loads(f.read()) + self.assertEqual(p.get("HexType"), 16777228) + self.assertEqual(p.get("IntType"), 83) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 88eed5303f4..a9bfb755c3f 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -152,8 +152,6 @@ def test_poll2(self): else: self.fail('Unexpected return value from select.poll: %s' % fdlist) - # TODO: RUSTPYTHON int overflow - @unittest.expectedFailure def test_poll3(self): # test int overflow pollster = select.poll() @@ -211,8 +209,6 @@ def test_threaded_poll(self): os.write(w, b'spam') t.join() - # TODO: RUSTPYTHON add support for negative timeout - @unittest.expectedFailure @unittest.skipUnless(threading, 'Threading required for this test.') @threading_helper.reap_threads def test_poll_blocks_with_negative_ms(self): diff --git a/Lib/test/test_popen.py b/Lib/test/test_popen.py index e3030ee02e2..34cda35b17b 100644 --- a/Lib/test/test_popen.py +++ b/Lib/test/test_popen.py @@ -54,19 +54,24 @@ def test_return_code(self): else: self.assertEqual(os.waitstatus_to_exitcode(status), 42) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_contextmanager(self): with os.popen("echo hello") as f: self.assertEqual(f.read(), "hello\n") + self.assertFalse(f.closed) + self.assertTrue(f.closed) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_iterating(self): with os.popen("echo hello") as f: self.assertEqual(list(f), ["hello\n"]) + self.assertFalse(f.closed) + self.assertTrue(f.closed) def test_keywords(self): - with os.popen(cmd="exit 0", mode="w", buffering=-1): - pass + with os.popen(cmd="echo hello", mode="r", buffering=-1) as f: + self.assertEqual(f.read(), "hello\n") + self.assertFalse(f.closed) + self.assertTrue(f.closed) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py new file mode 100644 index 00000000000..ef2da97f867 --- /dev/null +++ b/Lib/test/test_poplib.py @@ -0,0 +1,571 @@ +"""Test script for poplib module.""" + +# Modified by Giampaolo Rodola' to give poplib.POP3 and poplib.POP3_SSL +# a real test suite + +import poplib +import socket +import os +import errno +import threading + +import unittest +from unittest import TestCase, skipUnless +from test import support as test_support +from test.support import hashlib_helper +from test.support import socket_helper +from test.support import threading_helper +from test.support import asynchat +from test.support import asyncore + + +test_support.requires_working_socket(module=True) + +HOST = socket_helper.HOST +PORT = 0 + +SUPPORTS_SSL = False +if hasattr(poplib, 'POP3_SSL'): + import ssl + + SUPPORTS_SSL = True + CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "certdata", "keycert3.pem") + CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "certdata", "pycacert.pem") + +requires_ssl = skipUnless(SUPPORTS_SSL, 'SSL not supported') + +# the dummy data returned by server when LIST and RETR commands are issued +LIST_RESP = b'1 1\r\n2 2\r\n3 3\r\n4 4\r\n5 5\r\n.\r\n' +RETR_RESP = b"""From: postmaster@python.org\ +\r\nContent-Type: text/plain\r\n\ +MIME-Version: 1.0\r\n\ +Subject: Dummy\r\n\ +\r\n\ +line1\r\n\ +line2\r\n\ +line3\r\n\ +.\r\n""" + + +class DummyPOP3Handler(asynchat.async_chat): + + CAPAS = {'UIDL': [], 'IMPLEMENTATION': ['python-testlib-pop-server']} + enable_UTF8 = False + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + self.set_terminator(b"\r\n") + self.in_buffer = [] + self.push('+OK dummy pop3 server ready. <timestamp>') + self.tls_active = False + self.tls_starting = False + + def collect_incoming_data(self, data): + self.in_buffer.append(data) + + def found_terminator(self): + line = b''.join(self.in_buffer) + line = str(line, 'ISO-8859-1') + self.in_buffer = [] + cmd = line.split(' ')[0].lower() + space = line.find(' ') + if space != -1: + arg = line[space + 1:] + else: + arg = "" + if hasattr(self, 'cmd_' + cmd): + method = getattr(self, 'cmd_' + cmd) + method(arg) + else: + self.push('-ERR unrecognized POP3 command "%s".' %cmd) + + def handle_error(self): + raise + + def push(self, data): + asynchat.async_chat.push(self, data.encode("ISO-8859-1") + b'\r\n') + + def cmd_echo(self, arg): + # sends back the received string (used by the test suite) + self.push(arg) + + def cmd_user(self, arg): + if arg != "guido": + self.push("-ERR no such user") + self.push('+OK password required') + + def cmd_pass(self, arg): + if arg != "python": + self.push("-ERR wrong password") + self.push('+OK 10 messages') + + def cmd_stat(self, arg): + self.push('+OK 10 100') + + def cmd_list(self, arg): + if arg: + self.push('+OK %s %s' % (arg, arg)) + else: + self.push('+OK') + asynchat.async_chat.push(self, LIST_RESP) + + cmd_uidl = cmd_list + + def cmd_retr(self, arg): + self.push('+OK %s bytes' %len(RETR_RESP)) + asynchat.async_chat.push(self, RETR_RESP) + + cmd_top = cmd_retr + + def cmd_dele(self, arg): + self.push('+OK message marked for deletion.') + + def cmd_noop(self, arg): + self.push('+OK done nothing.') + + def cmd_rpop(self, arg): + self.push('+OK done nothing.') + + def cmd_apop(self, arg): + self.push('+OK done nothing.') + + def cmd_quit(self, arg): + self.push('+OK closing.') + self.close_when_done() + + def _get_capas(self): + _capas = dict(self.CAPAS) + if not self.tls_active and SUPPORTS_SSL: + _capas['STLS'] = [] + return _capas + + def cmd_capa(self, arg): + self.push('+OK Capability list follows') + if self._get_capas(): + for cap, params in self._get_capas().items(): + _ln = [cap] + if params: + _ln.extend(params) + self.push(' '.join(_ln)) + self.push('.') + + def cmd_utf8(self, arg): + self.push('+OK I know RFC6856' + if self.enable_UTF8 + else '-ERR What is UTF8?!') + + if SUPPORTS_SSL: + + def cmd_stls(self, arg): + if self.tls_active is False: + self.push('+OK Begin TLS negotiation') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(CERTFILE) + tls_sock = context.wrap_socket(self.socket, + server_side=True, + do_handshake_on_connect=False, + suppress_ragged_eofs=False) + self.del_channel() + self.set_socket(tls_sock) + self.tls_active = True + self.tls_starting = True + self.in_buffer = [] + self._do_tls_handshake() + else: + self.push('-ERR Command not permitted when TLS active') + + def _do_tls_handshake(self): + try: + self.socket.do_handshake() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + elif err.args[0] == ssl.SSL_ERROR_EOF: + return self.handle_close() + # TODO: SSLError does not expose alert information + elif ("SSLV3_ALERT_BAD_CERTIFICATE" in err.args[1] or + "SSLV3_ALERT_CERTIFICATE_UNKNOWN" in err.args[1]): + return self.handle_close() + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self.tls_active = True + self.tls_starting = False + + def handle_read(self): + if self.tls_starting: + self._do_tls_handshake() + else: + try: + asynchat.async_chat.handle_read(self) + except ssl.SSLEOFError: + self.handle_close() + +class DummyPOP3Server(asyncore.dispatcher, threading.Thread): + + handler = DummyPOP3Handler + + def __init__(self, address, af=socket.AF_INET): + threading.Thread.__init__(self) + asyncore.dispatcher.__init__(self) + self.daemon = True + self.create_socket(af, socket.SOCK_STREAM) + self.bind(address) + self.listen(5) + self.active = False + self.active_lock = threading.Lock() + self.host, self.port = self.socket.getsockname()[:2] + self.handler_instance = None + + def start(self): + assert not self.active + self.__flag = threading.Event() + threading.Thread.start(self) + self.__flag.wait() + + def run(self): + self.active = True + self.__flag.set() + try: + while self.active and asyncore.socket_map: + with self.active_lock: + asyncore.loop(timeout=0.1, count=1) + finally: + asyncore.close_all(ignore_all=True) + + def stop(self): + assert self.active + self.active = False + self.join() + + def handle_accepted(self, conn, addr): + self.handler_instance = self.handler(conn) + + def handle_connect(self): + self.close() + handle_read = handle_connect + + def writable(self): + return 0 + + def handle_error(self): + raise + + +class TestPOP3Class(TestCase): + def assertOK(self, resp): + self.assertStartsWith(resp, b"+OK") + + def setUp(self): + self.server = DummyPOP3Server((HOST, PORT)) + self.server.start() + self.client = poplib.POP3(self.server.host, self.server.port, + timeout=test_support.LOOPBACK_TIMEOUT) + + def tearDown(self): + self.client.close() + self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None + + def test_getwelcome(self): + self.assertEqual(self.client.getwelcome(), + b'+OK dummy pop3 server ready. <timestamp>') + + def test_exceptions(self): + self.assertRaises(poplib.error_proto, self.client._shortcmd, 'echo -err') + + def test_user(self): + self.assertOK(self.client.user('guido')) + self.assertRaises(poplib.error_proto, self.client.user, 'invalid') + + def test_pass_(self): + self.assertOK(self.client.pass_('python')) + self.assertRaises(poplib.error_proto, self.client.user, 'invalid') + + def test_stat(self): + self.assertEqual(self.client.stat(), (10, 100)) + + original_shortcmd = self.client._shortcmd + def mock_shortcmd_invalid_format(cmd): + if cmd == 'STAT': + return b'+OK' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_invalid_format + with self.assertRaises(poplib.error_proto): + self.client.stat() + + def mock_shortcmd_invalid_data(cmd): + if cmd == 'STAT': + return b'+OK abc def' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_invalid_data + with self.assertRaises(poplib.error_proto): + self.client.stat() + + def mock_shortcmd_extra_fields(cmd): + if cmd == 'STAT': + return b'+OK 1 2 3 4 5' + return original_shortcmd(cmd) + + self.client._shortcmd = mock_shortcmd_extra_fields + + result = self.client.stat() + self.assertEqual(result, (1, 2)) + + self.client._shortcmd = original_shortcmd + + def test_list(self): + self.assertEqual(self.client.list()[1:], + ([b'1 1', b'2 2', b'3 3', b'4 4', b'5 5'], + 25)) + self.assertEndsWith(self.client.list('1'), b"OK 1 1") + + def test_retr(self): + expected = (b'+OK 116 bytes', + [b'From: postmaster@python.org', b'Content-Type: text/plain', + b'MIME-Version: 1.0', b'Subject: Dummy', + b'', b'line1', b'line2', b'line3'], + 113) + foo = self.client.retr('foo') + self.assertEqual(foo, expected) + + def test_too_long_lines(self): + self.assertRaises(poplib.error_proto, self.client._shortcmd, + 'echo +%s' % ((poplib._MAXLINE + 10) * 'a')) + + def test_dele(self): + self.assertOK(self.client.dele('foo')) + + def test_noop(self): + self.assertOK(self.client.noop()) + + def test_rpop(self): + self.assertOK(self.client.rpop('foo')) + + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def test_apop_normal(self): + self.assertOK(self.client.apop('foo', 'dummypassword')) + + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def test_apop_REDOS(self): + # Replace welcome with very long evil welcome. + # NB The upper bound on welcome length is currently 2048. + # At this length, evil input makes each apop call take + # on the order of milliseconds instead of microseconds. + evil_welcome = b'+OK' + (b'<' * 1000000) + with test_support.swap_attr(self.client, 'welcome', evil_welcome): + # The evil welcome is invalid, so apop should throw. + self.assertRaises(poplib.error_proto, self.client.apop, 'a', 'kb') + + def test_top(self): + expected = (b'+OK 116 bytes', + [b'From: postmaster@python.org', b'Content-Type: text/plain', + b'MIME-Version: 1.0', b'Subject: Dummy', b'', + b'line1', b'line2', b'line3'], + 113) + self.assertEqual(self.client.top(1, 1), expected) + + def test_uidl(self): + self.client.uidl() + self.client.uidl('foo') + + def test_utf8_raises_if_unsupported(self): + self.server.handler.enable_UTF8 = False + self.assertRaises(poplib.error_proto, self.client.utf8) + + def test_utf8(self): + self.server.handler.enable_UTF8 = True + expected = b'+OK I know RFC6856' + result = self.client.utf8() + self.assertEqual(result, expected) + + def test_capa(self): + capa = self.client.capa() + self.assertTrue('IMPLEMENTATION' in capa.keys()) + + def test_quit(self): + resp = self.client.quit() + self.assertTrue(resp) + self.assertIsNone(self.client.sock) + self.assertIsNone(self.client.file) + + @requires_ssl + def test_stls_capa(self): + capa = self.client.capa() + self.assertTrue('STLS' in capa.keys()) + + @requires_ssl + def test_stls(self): + expected = b'+OK Begin TLS negotiation' + resp = self.client.stls() + self.assertEqual(resp, expected) + + @requires_ssl + def test_stls_context(self): + expected = b'+OK Begin TLS negotiation' + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(CAFILE) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.check_hostname, True) + with self.assertRaises(ssl.CertificateError): + resp = self.client.stls(context=ctx) + self.client = poplib.POP3("localhost", self.server.port, + timeout=test_support.LOOPBACK_TIMEOUT) + resp = self.client.stls(context=ctx) + self.assertEqual(resp, expected) + + +if SUPPORTS_SSL: + from test.test_ftplib import SSLConnection + + class DummyPOP3_SSLHandler(SSLConnection, DummyPOP3Handler): + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + self.secure_connection() + self.set_terminator(b"\r\n") + self.in_buffer = [] + self.push('+OK dummy pop3 server ready. <timestamp>') + self.tls_active = True + self.tls_starting = False + + +@requires_ssl +class TestPOP3_SSLClass(TestPOP3Class): + # repeat previous tests by using poplib.POP3_SSL + + def setUp(self): + self.server = DummyPOP3Server((HOST, PORT)) + self.server.handler = DummyPOP3_SSLHandler + self.server.start() + self.client = poplib.POP3_SSL(self.server.host, self.server.port) + + def test__all__(self): + self.assertIn('POP3_SSL', poplib.__all__) + + def test_context(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + self.client.quit() + self.client = poplib.POP3_SSL(self.server.host, self.server.port, + context=ctx) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + self.assertIs(self.client.sock.context, ctx) + self.assertStartsWith(self.client.noop(), b'+OK') + + def test_stls(self): + self.assertRaises(poplib.error_proto, self.client.stls) + + test_stls_context = test_stls + + def test_stls_capa(self): + capa = self.client.capa() + self.assertFalse('STLS' in capa.keys()) + + +@requires_ssl +class TestPOP3_TLSClass(TestPOP3Class): + # repeat previous tests by using poplib.POP3.stls() + + def setUp(self): + self.server = DummyPOP3Server((HOST, PORT)) + self.server.start() + self.client = poplib.POP3(self.server.host, self.server.port, + timeout=test_support.LOOPBACK_TIMEOUT) + self.client.stls() + + def tearDown(self): + if self.client.file is not None and self.client.sock is not None: + try: + self.client.quit() + except poplib.error_proto: + # happens in the test_too_long_lines case; the overlong + # response will be treated as response to QUIT and raise + # this exception + self.client.close() + self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None + + def test_stls(self): + self.assertRaises(poplib.error_proto, self.client.stls) + + test_stls_context = test_stls + + def test_stls_capa(self): + capa = self.client.capa() + self.assertFalse(b'STLS' in capa.keys()) + + +class TestTimeouts(TestCase): + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(60) # Safety net. Look issue 11812 + self.port = socket_helper.bind_port(self.sock) + self.thread = threading.Thread(target=self.server, args=(self.evt, self.sock)) + self.thread.daemon = True + self.thread.start() + self.evt.wait() + + def tearDown(self): + self.thread.join() + # Explicitly clear the attribute to prevent dangling thread + self.thread = None + + def server(self, evt, serv): + serv.listen() + evt.set() + try: + conn, addr = serv.accept() + conn.send(b"+ Hola mundo\n") + conn.close() + except TimeoutError: + pass + finally: + serv.close() + + def testTimeoutDefault(self): + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(test_support.LOOPBACK_TIMEOUT) + try: + pop = poplib.POP3(HOST, self.port) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(pop.sock.gettimeout(), test_support.LOOPBACK_TIMEOUT) + pop.close() + + def testTimeoutNone(self): + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + pop = poplib.POP3(HOST, self.port, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(pop.sock.gettimeout()) + pop.close() + + def testTimeoutValue(self): + pop = poplib.POP3(HOST, self.port, timeout=test_support.LOOPBACK_TIMEOUT) + self.assertEqual(pop.sock.gettimeout(), test_support.LOOPBACK_TIMEOUT) + pop.close() + with self.assertRaises(ValueError): + poplib.POP3(HOST, self.port, timeout=0) + + +def setUpModule(): + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index d9d8b030699..e002babab44 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -2,6 +2,7 @@ import dis import pickle +import types import unittest from test.support import check_syntax_error @@ -22,11 +23,13 @@ def assertRaisesSyntaxError(self, codestr, regex="invalid syntax"): with self.assertRaisesRegex(SyntaxError, regex): compile(codestr + "\n", "<test>", "single") + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_invalid_syntax_errors(self): - check_syntax_error(self, "def f(a, b = 5, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "non-default argument follows default argument") - check_syntax_error(self, "def f(a = 5, b, /): pass", "non-default argument follows default argument") + check_syntax_error(self, "def f(a, b = 5, /, c): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "def f(a = 5, b, /, c): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "def f(a = 5, b=1, /, c, *, d=2): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "def f(a = 5, b, /): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "def f(a, /, b = 5, c): pass", "parameter without a default follows parameter with a default") check_syntax_error(self, "def f(*args, /): pass") check_syntax_error(self, "def f(*args, a, /): pass") check_syntax_error(self, "def f(**kwargs, /): pass") @@ -43,11 +46,13 @@ def test_invalid_syntax_errors(self): check_syntax_error(self, "def f(a, /, c, /, d, *, e): pass") check_syntax_error(self, "def f(a, *, c, /, d, e): pass") + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_invalid_syntax_errors_async(self): - check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "non-default argument follows default argument") - check_syntax_error(self, "async def f(a = 5, b, /): pass", "non-default argument follows default argument") + check_syntax_error(self, "async def f(a, b = 5, /, c): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "async def f(a = 5, b, /, c): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "async def f(a = 5, b=1, /, c, d=2): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "async def f(a = 5, b, /): pass", "parameter without a default follows parameter with a default") + check_syntax_error(self, "async def f(a, /, b = 5, c): pass", "parameter without a default follows parameter with a default") check_syntax_error(self, "async def f(*args, /): pass") check_syntax_error(self, "async def f(*args, a, /): pass") check_syntax_error(self, "async def f(**kwargs, /): pass") @@ -148,8 +153,6 @@ def f(a, b, /, c): with self.assertRaisesRegex(TypeError, r"f\(\) takes 3 positional arguments but 4 were given"): f(1, 2, 3, 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positional_only_and_optional_arg_invalid_calls(self): def f(a, b, /, c=3): pass @@ -161,8 +164,6 @@ def f(a, b, /, c=3): with self.assertRaisesRegex(TypeError, r"f\(\) takes from 2 to 3 positional arguments but 4 were given"): f(1, 2, 3, 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positional_only_and_kwonlyargs_invalid_calls(self): def f(a, b, /, c, *, d, e): pass @@ -194,8 +195,6 @@ def f(a, b, /): with self.assertRaisesRegex(TypeError, r"f\(\) takes 2 positional arguments but 3 were given"): f(1, 2, 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positional_only_with_optional_invalid_calls(self): def f(a, b=2, /): pass @@ -236,10 +235,13 @@ def test_lambdas(self): x = lambda a, b, /, : a + b self.assertEqual(x(1, 2), 3) + @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_invalid_syntax_lambda(self): - check_syntax_error(self, "lambda a, b = 5, /, c: None", "non-default argument follows default argument") - check_syntax_error(self, "lambda a = 5, b, /, c: None", "non-default argument follows default argument") - check_syntax_error(self, "lambda a = 5, b, /: None", "non-default argument follows default argument") + check_syntax_error(self, "lambda a, b = 5, /, c: None", "parameter without a default follows parameter with a default") + check_syntax_error(self, "lambda a = 5, b, /, c: None", "parameter without a default follows parameter with a default") + check_syntax_error(self, "lambda a = 5, b=1, /, c, *, d=2: None", "parameter without a default follows parameter with a default") + check_syntax_error(self, "lambda a = 5, b, /: None", "parameter without a default follows parameter with a default") + check_syntax_error(self, "lambda a, /, b = 5, c: None", "parameter without a default follows parameter with a default") check_syntax_error(self, "lambda *args, /: None") check_syntax_error(self, "lambda *args, a, /: None") check_syntax_error(self, "lambda **kwargs, /: None") @@ -336,8 +338,6 @@ def f(something,/,**kwargs): self.assertEqual(f(42), (42, {})) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mangling(self): class X: def f(self, __a=42, /): @@ -437,8 +437,7 @@ def method(self, /): self.assertEqual(C().method(), sentinel) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_annotations_constant_fold(self): def g(): def f(x: not (int is int), /): ... @@ -446,7 +445,9 @@ def f(x: not (int is int), /): ... # without constant folding we end up with # COMPARE_OP(is), IS_OP (0) # with constant folding we should expect a IS_OP (1) - codes = [(i.opname, i.argval) for i in dis.get_instructions(g)] + code_obj = next(const for const in g.__code__.co_consts + if isinstance(const, types.CodeType) and const.co_name == "__annotate__") + codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)] self.assertNotIn(('UNARY_NOT', None), codes) self.assertIn(('IS_OP', 1), codes) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index f8e1e7bc207..eb0e4d25dc1 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1,20 +1,19 @@ "Test posix functions" from test import support -from test.support import import_helper +from test.support import is_apple from test.support import os_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok -# Skip these tests if there is no posix module. -posix = import_helper.import_module('posix') - +import copy import errno import sys import signal import time import os import platform +import pickle import stat import tempfile import unittest @@ -22,6 +21,11 @@ import textwrap from contextlib import contextmanager +try: + import posix +except ImportError: + import nt as posix + try: import pwd except ImportError: @@ -231,6 +235,9 @@ def test_register_at_fork(self): with self.assertRaises(TypeError, msg="Invalid arg was allowed"): # Ensure a combination of valid and invalid is an error. os.register_at_fork(before=None, after_in_parent=lambda: 3) + with self.assertRaises(TypeError, msg="At least one argument is required"): + # when no arg is passed + os.register_at_fork() with self.assertRaises(TypeError, msg="Invalid arg was allowed"): # Ensure a combination of valid and invalid is an error. os.register_at_fork(before=lambda: None, after_in_child='') @@ -406,8 +413,10 @@ def test_posix_fallocate(self): # issue33655: Also ignore EINVAL on *BSD since ZFS is also # often used there. if inst.errno == errno.EINVAL and sys.platform.startswith( - ('sunos', 'freebsd', 'netbsd', 'openbsd', 'gnukfreebsd')): + ('sunos', 'freebsd', 'openbsd', 'gnukfreebsd')): raise unittest.SkipTest("test may fail on ZFS filesystems") + elif inst.errno == errno.EOPNOTSUPP and sys.platform.startswith("netbsd"): + raise unittest.SkipTest("test may fail on FFS filesystems") else: raise finally: @@ -560,8 +569,37 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') def test_confstr(self): - self.assertRaises(ValueError, posix.confstr, "CS_garbage") - self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True) + with self.assertRaisesRegex( + ValueError, "unrecognized configuration name" + ): + posix.confstr("CS_garbage") + + with self.assertRaisesRegex( + TypeError, "configuration names must be strings or integers" + ): + posix.confstr(1.23) + + path = posix.confstr("CS_PATH") + self.assertGreater(len(path), 0) + self.assertEqual(posix.confstr(posix.confstr_names["CS_PATH"]), path) + + @unittest.skipUnless(hasattr(posix, 'sysconf'), + 'test needs posix.sysconf()') + def test_sysconf(self): + with self.assertRaisesRegex( + ValueError, "unrecognized configuration name" + ): + posix.sysconf("SC_garbage") + + with self.assertRaisesRegex( + TypeError, "configuration names must be strings or integers" + ): + posix.sysconf(1.23) + + arg_max = posix.sysconf("SC_ARG_MAX") + self.assertGreater(arg_max, 0) + self.assertEqual( + posix.sysconf(posix.sysconf_names["SC_ARG_MAX"]), arg_max) @unittest.skipUnless(hasattr(posix, 'dup2'), 'test needs posix.dup2()') @@ -630,13 +668,24 @@ def test_fstat(self): finally: fp.close() - # TODO: RUSTPYTHON: AssertionError: DeprecationWarning not triggered by stat - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(hasattr(posix, 'stat'), + 'test needs posix.stat()') + @unittest.skipUnless(os.stat in os.supports_follow_symlinks, + 'test needs follow_symlinks support in os.stat()') + def test_stat_fd_zero_follow_symlinks(self): + with self.assertRaisesRegex(ValueError, + 'cannot use fd and follow_symlinks together'): + posix.stat(0, follow_symlinks=False) + with self.assertRaisesRegex(ValueError, + 'cannot use fd and follow_symlinks together'): + posix.stat(1, follow_symlinks=False) + def test_stat(self): self.assertTrue(posix.stat(os_helper.TESTFN)) self.assertTrue(posix.stat(os.fsencode(os_helper.TESTFN))) - self.assertWarnsRegex(DeprecationWarning, + self.assertRaisesRegex(TypeError, 'should be string, bytes, os.PathLike or integer, not', posix.stat, bytearray(os.fsencode(os_helper.TESTFN))) self.assertRaisesRegex(TypeError, @@ -700,7 +749,8 @@ def test_makedev(self): self.assertEqual(posix.major(dev), major) self.assertRaises(TypeError, posix.major, float(dev)) self.assertRaises(TypeError, posix.major) - self.assertRaises((ValueError, OverflowError), posix.major, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.major, x) minor = posix.minor(dev) self.assertIsInstance(minor, int) @@ -708,13 +758,23 @@ def test_makedev(self): self.assertEqual(posix.minor(dev), minor) self.assertRaises(TypeError, posix.minor, float(dev)) self.assertRaises(TypeError, posix.minor) - self.assertRaises((ValueError, OverflowError), posix.minor, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.minor, x) self.assertEqual(posix.makedev(major, minor), dev) self.assertRaises(TypeError, posix.makedev, float(major), minor) self.assertRaises(TypeError, posix.makedev, major, float(minor)) self.assertRaises(TypeError, posix.makedev, major) self.assertRaises(TypeError, posix.makedev) + for x in -2, 2**32, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) + self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) + + if sys.platform == 'linux' and not support.linked_to_musl(): + NODEV = -1 + self.assertEqual(posix.major(NODEV), NODEV) + self.assertEqual(posix.minor(NODEV), NODEV) + self.assertEqual(posix.makedev(NODEV, NODEV), NODEV) def _test_all_chown_common(self, chown_func, first_param, stat_func): """Common code for chown, fchown and lchown tests.""" @@ -778,9 +838,10 @@ def check_stat(uid, gid): check_stat(uid, gid) self.assertRaises(OSError, chown_func, first_param, 0, -1) check_stat(uid, gid) - if 0 not in os.getgroups(): - self.assertRaises(OSError, chown_func, first_param, -1, 0) - check_stat(uid, gid) + if hasattr(os, 'getgroups'): + if 0 not in os.getgroups(): + self.assertRaises(OSError, chown_func, first_param, -1, 0) + check_stat(uid, gid) # test illegal types for t in str, float: self.assertRaises(TypeError, chown_func, first_param, t(uid), gid) @@ -788,7 +849,7 @@ def check_stat(uid, gid): self.assertRaises(TypeError, chown_func, first_param, uid, t(gid)) check_stat(uid, gid) - @os_helper.skip_unless_working_chmod + @unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") @unittest.skipIf(support.is_emscripten, "getgid() is a stub") def test_chown(self): # raise an OSError if the file does not exist @@ -841,15 +902,10 @@ def test_listdir_bytes(self): # the returned strings are of type bytes. self.assertIn(os.fsencode(os_helper.TESTFN), posix.listdir(b'.')) - # TODO: RUSTPYTHON: AssertionError: DeprecationWarning not triggered - @unittest.expectedFailure def test_listdir_bytes_like(self): for cls in bytearray, memoryview: - with self.assertWarns(DeprecationWarning): - names = posix.listdir(cls(b'.')) - self.assertIn(os.fsencode(os_helper.TESTFN), names) - for name in names: - self.assertIs(type(name), bytes) + with self.assertRaises(TypeError): + posix.listdir(cls(b'.')) @unittest.skipUnless(posix.listdir in os.supports_fd, "test needs fd support for posix.listdir()") @@ -937,9 +993,134 @@ def test_utime(self): posix.utime(os_helper.TESTFN, (int(now), int(now))) posix.utime(os_helper.TESTFN, (now, now)) + def check_chmod(self, chmod_func, target, **kwargs): + closefd = not isinstance(target, int) + mode = os.stat(target).st_mode + try: + new_mode = mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(target, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + if stat.S_ISREG(mode): + try: + with open(target, 'wb+', closefd=closefd): + pass + except PermissionError: + pass + new_mode = mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(target, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + if stat.S_ISREG(mode): + with open(target, 'wb+', closefd=closefd): + pass + finally: + chmod_func(target, mode) + + @os_helper.skip_unless_working_chmod + def test_chmod_file(self): + self.check_chmod(posix.chmod, os_helper.TESTFN) + + def tempdir(self): + target = os_helper.TESTFN + 'd' + posix.mkdir(target) + self.addCleanup(posix.rmdir, target) + return target + + @os_helper.skip_unless_working_chmod + def test_chmod_dir(self): + target = self.tempdir() + self.check_chmod(posix.chmod, target) + + @os_helper.skip_unless_working_chmod + def test_fchmod_file(self): + with open(os_helper.TESTFN, 'wb+') as f: + self.check_chmod(posix.fchmod, f.fileno()) + self.check_chmod(posix.chmod, f.fileno()) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + def test_lchmod_file(self): + self.check_chmod(posix.lchmod, os_helper.TESTFN) + self.check_chmod(posix.chmod, os_helper.TESTFN, follow_symlinks=False) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + def test_lchmod_dir(self): + target = self.tempdir() + self.check_chmod(posix.lchmod, target) + self.check_chmod(posix.chmod, target, follow_symlinks=False) + + def check_chmod_link(self, chmod_func, target, link, **kwargs): + target_mode = os.stat(target).st_mode + link_mode = os.lstat(link).st_mode + try: + new_mode = target_mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + self.assertEqual(os.lstat(link).st_mode, link_mode) + new_mode = target_mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, new_mode) + self.assertEqual(os.lstat(link).st_mode, link_mode) + finally: + posix.chmod(target, target_mode) + + def check_lchmod_link(self, chmod_func, target, link, **kwargs): + target_mode = os.stat(target).st_mode + link_mode = os.lstat(link).st_mode + new_mode = link_mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, target_mode) + self.assertEqual(os.lstat(link).st_mode, new_mode) + new_mode = link_mode | (stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) + chmod_func(link, new_mode, **kwargs) + self.assertEqual(os.stat(target).st_mode, target_mode) + self.assertEqual(os.lstat(link).st_mode, new_mode) + + @os_helper.skip_unless_symlink + def test_chmod_file_symlink(self): + target = os_helper.TESTFN + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + if os.name == 'nt': + self.check_lchmod_link(posix.chmod, target, link) + else: + self.check_chmod_link(posix.chmod, target, link) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + + @os_helper.skip_unless_symlink + def test_chmod_dir_symlink(self): + target = self.tempdir() + link = os_helper.TESTFN + '-link' + os.symlink(target, link, target_is_directory=True) + self.addCleanup(posix.unlink, link) + if os.name == 'nt': + self.check_lchmod_link(posix.chmod, target, link) + else: + self.check_chmod_link(posix.chmod, target, link) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + @os_helper.skip_unless_symlink + def test_lchmod_file_symlink(self): + target = os_helper.TESTFN + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + self.check_lchmod_link(posix.chmod, target, link, follow_symlinks=False) + self.check_lchmod_link(posix.lchmod, target, link) + + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') + @os_helper.skip_unless_symlink + def test_lchmod_dir_symlink(self): + target = self.tempdir() + link = os_helper.TESTFN + '-link' + os.symlink(target, link) + self.addCleanup(posix.unlink, link) + self.check_lchmod_link(posix.chmod, target, link, follow_symlinks=False) + self.check_lchmod_link(posix.lchmod, target, link) + def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs): st = os.stat(target_file) - self.assertTrue(hasattr(st, 'st_flags')) + self.assertHasAttr(st, 'st_flags') # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE. flags = st.st_flags | stat.UF_IMMUTABLE @@ -975,7 +1156,7 @@ def test_lchflags_regular_file(self): def test_lchflags_symlink(self): testfn_st = os.stat(os_helper.TESTFN) - self.assertTrue(hasattr(testfn_st, 'st_flags')) + self.assertHasAttr(testfn_st, 'st_flags') self.addCleanup(os_helper.unlink, _DUMMY_SYMLINK) os.symlink(os_helper.TESTFN, _DUMMY_SYMLINK) @@ -1016,16 +1197,17 @@ def test_environ(self): def test_putenv(self): with self.assertRaises(ValueError): os.putenv('FRUIT\0VEGETABLE', 'cabbage') - with self.assertRaises(ValueError): - os.putenv(b'FRUIT\0VEGETABLE', b'cabbage') with self.assertRaises(ValueError): os.putenv('FRUIT', 'orange\0VEGETABLE=cabbage') - with self.assertRaises(ValueError): - os.putenv(b'FRUIT', b'orange\0VEGETABLE=cabbage') with self.assertRaises(ValueError): os.putenv('FRUIT=ORANGE', 'lemon') - with self.assertRaises(ValueError): - os.putenv(b'FRUIT=ORANGE', b'lemon') + if os.name == 'posix': + with self.assertRaises(ValueError): + os.putenv(b'FRUIT\0VEGETABLE', b'cabbage') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT', b'orange\0VEGETABLE=cabbage') + with self.assertRaises(ValueError): + os.putenv(b'FRUIT=ORANGE', b'lemon') @unittest.skipUnless(hasattr(posix, 'getcwd'), 'test needs posix.getcwd()') def test_getcwd_long_pathnames(self): @@ -1132,8 +1314,8 @@ def test_sched_priority(self): self.assertIsInstance(lo, int) self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) - # OSX evidently just returns 15 without checking the argument. - if sys.platform != "darwin": + # Apple platforms return 15 without checking the argument. + if not is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) @@ -1145,9 +1327,10 @@ def test_get_and_set_scheduler_and_param(self): self.assertIn(mine, possible_schedulers) try: parent = posix.sched_getscheduler(os.getppid()) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + # POSIX specifies EPERM, but Android returns EACCES. Both errno + # values are mapped to PermissionError. + pass else: self.assertIn(parent, possible_schedulers) self.assertRaises(OSError, posix.sched_getscheduler, -1) @@ -1162,9 +1345,8 @@ def test_get_and_set_scheduler_and_param(self): try: posix.sched_setscheduler(0, mine, param) posix.sched_setparam(0, param) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + pass self.assertRaises(OSError, posix.sched_setparam, -1, param) self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param) @@ -1178,6 +1360,33 @@ def test_get_and_set_scheduler_and_param(self): param = posix.sched_param(sched_priority=-large) self.assertRaises(OverflowError, posix.sched_setparam, 0, param) + @requires_sched + def test_sched_param(self): + param = posix.sched_param(1) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + newparam = pickle.loads(pickle.dumps(param, proto)) + self.assertEqual(newparam, param) + newparam = copy.copy(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.deepcopy(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.replace(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.replace(param, sched_priority=0) + self.assertNotEqual(newparam, param) + self.assertEqual(newparam.sched_priority, 0) + + @requires_sched + def test_bug_140634(self): + sched_priority = float('inf') # any new reference + param = posix.sched_param(sched_priority) + param.__reduce__() + del sched_priority, param # should not crash + support.gc_collect() # just to be sure + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: @@ -1209,12 +1418,22 @@ def test_sched_getaffinity(self): @requires_sched_affinity def test_sched_setaffinity(self): mask = posix.sched_getaffinity(0) + self.addCleanup(posix.sched_setaffinity, 0, list(mask)) + if len(mask) > 1: # Empty masks are forbidden mask.pop() posix.sched_setaffinity(0, mask) self.assertEqual(posix.sched_getaffinity(0), mask) - self.assertRaises(OSError, posix.sched_setaffinity, 0, []) + + try: + posix.sched_setaffinity(0, []) + # gh-117061: On RHEL9, sched_setaffinity(0, []) does not fail + except OSError: + # sched_setaffinity() manual page documents EINVAL error + # when the mask is empty. + pass + self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10]) self.assertRaises(ValueError, posix.sched_setaffinity, 0, map(int, "0X")) self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128]) @@ -1223,6 +1442,7 @@ def test_sched_setaffinity(self): self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) @unittest.skipIf(support.is_wasi, "No dynamic linking on WASI") + @unittest.skipUnless(os.name == 'posix', "POSIX-only test") def test_rtld_constants(self): # check presence of major RTLD_* constants posix.RTLD_LAZY @@ -1322,6 +1542,51 @@ def test_pidfd_open(self): self.assertEqual(cm.exception.errno, errno.EINVAL) os.close(os.pidfd_open(os.getpid(), 0)) + @os_helper.skip_unless_hardlink + @os_helper.skip_unless_symlink + def test_link_follow_symlinks(self): + default_follow = sys.platform.startswith( + ('darwin', 'freebsd', 'netbsd', 'openbsd', 'dragonfly', 'sunos5')) + default_no_follow = sys.platform.startswith(('win32', 'linux')) + orig = os_helper.TESTFN + symlink = orig + 'symlink' + posix.symlink(orig, symlink) + self.addCleanup(os_helper.unlink, symlink) + + with self.subTest('no follow_symlinks'): + # no follow_symlinks -> platform depending + link = orig + 'link' + posix.link(symlink, link) + self.addCleanup(os_helper.unlink, link) + if os.link in os.supports_follow_symlinks or default_follow: + self.assertEqual(posix.lstat(link), posix.lstat(orig)) + elif default_no_follow: + self.assertEqual(posix.lstat(link), posix.lstat(symlink)) + + with self.subTest('follow_symlinks=False'): + # follow_symlinks=False -> duplicate the symlink itself + link = orig + 'link_nofollow' + try: + posix.link(symlink, link, follow_symlinks=False) + except NotImplementedError: + if os.link in os.supports_follow_symlinks or default_no_follow: + raise + else: + self.addCleanup(os_helper.unlink, link) + self.assertEqual(posix.lstat(link), posix.lstat(symlink)) + + with self.subTest('follow_symlinks=True'): + # follow_symlinks=True -> duplicate the target file + link = orig + 'link_following' + try: + posix.link(symlink, link, follow_symlinks=True) + except NotImplementedError: + if os.link in os.supports_follow_symlinks or default_follow: + raise + else: + self.addCleanup(os_helper.unlink, link) + self.assertEqual(posix.lstat(link), posix.lstat(orig)) + # tests for the posix *at functions follow class TestPosixDirFd(unittest.TestCase): @@ -1387,6 +1652,13 @@ def test_stat_dir_fd(self): self.assertRaises(OverflowError, posix.stat, name, dir_fd=10**20) + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + with self.assertRaises(OSError): + posix.stat('nonexisting', dir_fd=fd) + self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): @@ -1602,8 +1874,6 @@ def test_no_such_executable(self): self.assertEqual(pid2, pid) self.assertNotEqual(status, 0) - # TODO: RUSTPYTHON: TypeError: '_Environ' object is not a mapping - @unittest.expectedFailure def test_specify_environment(self): envfile = os_helper.TESTFN self.addCleanup(os_helper.unlink, envfile) @@ -1637,8 +1907,6 @@ def test_empty_file_actions(self): ) support.wait_process(pid, exitcode=0) - # TODO: RUSTPYTHON: TypeError: Unexpected keyword argument resetids - @unittest.expectedFailure def test_resetids_explicit_default(self): pid = self.spawn_func( sys.executable, @@ -1648,8 +1916,6 @@ def test_resetids_explicit_default(self): ) support.wait_process(pid, exitcode=0) - # TODO: RUSTPYTHON: TypeError: Unexpected keyword argument resetids - @unittest.expectedFailure def test_resetids(self): pid = self.spawn_func( sys.executable, @@ -1659,14 +1925,6 @@ def test_resetids(self): ) support.wait_process(pid, exitcode=0) - def test_resetids_wrong_type(self): - with self.assertRaises(TypeError): - self.spawn_func(sys.executable, - [sys.executable, "-c", "pass"], - os.environ, resetids=None) - - # TODO: RUSTPYTHON: TypeError: Unexpected keyword argument setpgroup - @unittest.expectedFailure def test_setpgroup(self): pid = self.spawn_func( sys.executable, @@ -1697,8 +1955,6 @@ def test_setsigmask(self): ) support.wait_process(pid, exitcode=0) - # TODO: RUSTPYTHON: TypeError: Unexpected keyword argument setsigmask - @unittest.expectedFailure def test_setsigmask_wrong_type(self): with self.assertRaises(TypeError): self.spawn_func(sys.executable, @@ -1714,8 +1970,6 @@ def test_setsigmask_wrong_type(self): os.environ, setsigmask=[signal.NSIG, signal.NSIG+1]) - # TODO: RUSTPYTHON: TypeError: Unexpected keyword argument setsid - @unittest.expectedFailure def test_setsid(self): rfd, wfd = os.pipe() self.addCleanup(os.close, rfd) @@ -1780,8 +2034,7 @@ def test_setsigdef_wrong_type(self): [sys.executable, "-c", "pass"], os.environ, setsigdef=[signal.NSIG, signal.NSIG+1]) - # TODO: RUSTPYTHON: TypeError: Unexpected keyword argument scheduler - @unittest.expectedFailure + @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented") @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") @@ -1802,11 +2055,16 @@ def test_setscheduler_only_param(self): ) support.wait_process(pid, exitcode=0) - # TODO: RUSTPYTHON: TypeError: Unexpected keyword argument scheduler - @unittest.expectedFailure + @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented") @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") + @unittest.skipIf(platform.libc_ver()[0] == 'glibc' and + os.sched_getscheduler(0) in [ + os.SCHED_BATCH, + os.SCHED_IDLE, + os.SCHED_DEADLINE], + "Skip test due to glibc posix_spawn policy") def test_setscheduler_with_policy(self): policy = os.sched_getscheduler(0) priority = os.sched_get_priority_min(policy) @@ -1885,8 +2143,7 @@ def test_open_file(self): with open(outfile, encoding="utf-8") as f: self.assertEqual(f.read(), 'hello') - # TODO: RUSTPYTHON: FileNotFoundError: [Errno 2] No such file or directory (os error 2): '@test_55144_tmp' -> 'None' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; the rust runtime reopens closed stdio fds at startup, so this test fails, even though POSIX_SPAWN_CLOSE does actually have an effect def test_close_file(self): closefile = os_helper.TESTFN self.addCleanup(os_helper.unlink, closefile) @@ -1926,11 +2183,13 @@ def test_dup2(self): @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +@support.requires_subprocess() class TestPosixSpawn(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawn', None) @unittest.skipUnless(hasattr(os, 'posix_spawnp'), "test needs os.posix_spawnp") +@support.requires_subprocess() class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawnp', None) @@ -1989,12 +2248,12 @@ def _verify_available(self, name): def test_pwritev(self): self._verify_available("HAVE_PWRITEV") if self.mac_ver >= (10, 16): - self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") - self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + self.assertHasAttr(os, "pwritev") + self.assertHasAttr(os, "preadv") else: - self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") - self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + self.assertNotHasAttr(os, "pwritev") + self.assertNotHasAttr(os, "preadv") def test_stat(self): self._verify_available("HAVE_FSTATAT") @@ -2007,6 +2266,13 @@ def test_stat(self): with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): os.stat("file", dir_fd=0) + def test_ptsname_r(self): + self._verify_available("HAVE_PTSNAME_R") + if self.mac_ver >= (10, 13, 4): + self.assertIn("HAVE_PTSNAME_R", posix._have_functions) + else: + self.assertNotIn("HAVE_PTSNAME_R", posix._have_functions) + def test_access(self): self._verify_available("HAVE_FACCESSAT") if self.mac_ver >= (10, 10): @@ -2216,6 +2482,53 @@ def test_utime(self): os.utime("path", dir_fd=0) +class NamespacesTests(unittest.TestCase): + """Tests for os.unshare() and os.setns().""" + + @unittest.skipUnless(hasattr(os, 'unshare'), 'needs os.unshare()') + @unittest.skipUnless(hasattr(os, 'setns'), 'needs os.setns()') + @unittest.skipUnless(os.path.exists('/proc/self/ns/uts'), 'need /proc/self/ns/uts') + @support.requires_linux_version(3, 0, 0) + def test_unshare_setns(self): + code = """if 1: + import errno + import os + import sys + fd = os.open('/proc/self/ns/uts', os.O_RDONLY) + try: + original = os.readlink('/proc/self/ns/uts') + try: + os.unshare(os.CLONE_NEWUTS) + except OSError as e: + if e.errno == errno.ENOSPC: + # skip test if limit is exceeded + sys.exit() + raise + new = os.readlink('/proc/self/ns/uts') + if original == new: + raise Exception('os.unshare failed') + os.setns(fd, os.CLONE_NEWUTS) + restored = os.readlink('/proc/self/ns/uts') + if original != restored: + raise Exception('os.setns failed') + except PermissionError: + # The calling process did not have the required privileges + # for this operation + pass + except OSError as e: + # Skip the test on these errors: + # - ENOSYS: syscall not available + # - EINVAL: kernel was not configured with the CONFIG_UTS_NS option + # - ENOMEM: not enough memory + if e.errno not in (errno.ENOSYS, errno.EINVAL, errno.ENOMEM): + raise + finally: + os.close(fd) + """ + + assert_python_ok("-c", code) + + def tearDownModule(): support.reap_children() diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index ab36d20540d..21f06712548 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,12 +1,16 @@ +import inspect import os import posixpath +import random import sys import unittest -from posixpath import realpath, abspath, dirname, basename +from functools import partial +from posixpath import realpath, abspath, dirname, basename, ALLOW_MISSING +from test import support from test import test_genericpath from test.support import import_helper from test.support import os_helper -from test.support.os_helper import FakePath +from test.support.os_helper import FakePath, TESTFN from unittest import mock try: @@ -18,7 +22,7 @@ # An absolute path to a temporary filename for testing. We can't rely on TESTFN # being an absolute path, so we need this. -ABSTFN = abspath(os_helper.TESTFN) +ABSTFN = abspath(TESTFN) def skip_if_ABSTFN_contains_backslash(test): """ @@ -30,35 +34,40 @@ def skip_if_ABSTFN_contains_backslash(test): msg = "ABSTFN is not a posix path - tests fail" return [test, unittest.skip(msg)(test)][found_backslash] -def safe_rmdir(dirname): - try: - os.rmdir(dirname) - except OSError: - pass + +def _parameterize(*parameters): + return support.subTests('kwargs', parameters) + class PosixPathTest(unittest.TestCase): def setUp(self): - self.tearDown() - - def tearDown(self): for suffix in ["", "1", "2"]: - os_helper.unlink(os_helper.TESTFN + suffix) - safe_rmdir(os_helper.TESTFN + suffix) + self.assertFalse(posixpath.lexists(ABSTFN + suffix)) def test_join(self): - self.assertEqual(posixpath.join("/foo", "bar", "/bar", "baz"), - "/bar/baz") - self.assertEqual(posixpath.join("/foo", "bar", "baz"), "/foo/bar/baz") - self.assertEqual(posixpath.join("/foo/", "bar/", "baz/"), - "/foo/bar/baz/") - - self.assertEqual(posixpath.join(b"/foo", b"bar", b"/bar", b"baz"), - b"/bar/baz") - self.assertEqual(posixpath.join(b"/foo", b"bar", b"baz"), - b"/foo/bar/baz") - self.assertEqual(posixpath.join(b"/foo/", b"bar/", b"baz/"), - b"/foo/bar/baz/") + fn = posixpath.join + self.assertEqual(fn("/foo", "bar", "/bar", "baz"), "/bar/baz") + self.assertEqual(fn("/foo", "bar", "baz"), "/foo/bar/baz") + self.assertEqual(fn("/foo/", "bar/", "baz/"), "/foo/bar/baz/") + + self.assertEqual(fn(b"/foo", b"bar", b"/bar", b"baz"), b"/bar/baz") + self.assertEqual(fn(b"/foo", b"bar", b"baz"), b"/foo/bar/baz") + self.assertEqual(fn(b"/foo/", b"bar/", b"baz/"), b"/foo/bar/baz/") + + self.assertEqual(fn("a", ""), "a/") + self.assertEqual(fn("a", "", ""), "a/") + self.assertEqual(fn("a", "b"), "a/b") + self.assertEqual(fn("a", "b/"), "a/b/") + self.assertEqual(fn("a/", "b"), "a/b") + self.assertEqual(fn("a/", "b/"), "a/b/") + self.assertEqual(fn("a", "b/c", "d"), "a/b/c/d") + self.assertEqual(fn("a", "b//c", "d"), "a/b//c/d") + self.assertEqual(fn("a", "b/c/", "d"), "a/b/c/d") + self.assertEqual(fn("/a", "b"), "/a/b") + self.assertEqual(fn("/a/", "b"), "/a/b") + self.assertEqual(fn("a", "/b", "c"), "/b/c") + self.assertEqual(fn("a", "/b", "/c"), "/c") def test_split(self): self.assertEqual(posixpath.split("/foo/bar"), ("/foo", "bar")) @@ -115,6 +124,32 @@ def test_splitext(self): self.splitextTest("........", "........", "") self.splitextTest("", "", "") + def test_splitroot(self): + f = posixpath.splitroot + self.assertEqual(f(''), ('', '', '')) + self.assertEqual(f('a'), ('', '', 'a')) + self.assertEqual(f('a/b'), ('', '', 'a/b')) + self.assertEqual(f('a/b/'), ('', '', 'a/b/')) + self.assertEqual(f('/a'), ('', '/', 'a')) + self.assertEqual(f('/a/b'), ('', '/', 'a/b')) + self.assertEqual(f('/a/b/'), ('', '/', 'a/b/')) + # The root is collapsed when there are redundant slashes + # except when there are exactly two leading slashes, which + # is a special case in POSIX. + self.assertEqual(f('//a'), ('', '//', 'a')) + self.assertEqual(f('///a'), ('', '/', '//a')) + self.assertEqual(f('///a/b'), ('', '/', '//a/b')) + # Paths which look like NT paths aren't treated specially. + self.assertEqual(f('c:/a/b'), ('', '', 'c:/a/b')) + self.assertEqual(f('\\/a/b'), ('', '', '\\/a/b')) + self.assertEqual(f('\\a\\b'), ('', '', '\\a\\b')) + # Byte paths are supported + self.assertEqual(f(b''), (b'', b'', b'')) + self.assertEqual(f(b'a'), (b'', b'', b'a')) + self.assertEqual(f(b'/a'), (b'', b'/', b'a')) + self.assertEqual(f(b'//a'), (b'', b'//', b'a')) + self.assertEqual(f(b'///a'), (b'', b'/', b'//a')) + def test_isabs(self): self.assertIs(posixpath.isabs(""), False) self.assertIs(posixpath.isabs("/"), True) @@ -154,31 +189,35 @@ def test_dirname(self): self.assertEqual(posixpath.dirname(b"////foo"), b"////") self.assertEqual(posixpath.dirname(b"//foo//bar"), b"//foo") - @unittest.expectedFailureIf(os.name == "nt", "TODO: RUSTPYTHON") def test_islink(self): - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), False) - with open(os_helper.TESTFN + "1", "wb") as f: + self.addCleanup(os_helper.unlink, TESTFN + "1") + with open(TESTFN + "1", "wb") as f: f.write(b"foo") - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) if os_helper.can_symlink(): - os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN + "2") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - os.remove(os_helper.TESTFN + "1") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - self.assertIs(posixpath.exists(os_helper.TESTFN + "2"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), True) - - self.assertIs(posixpath.islink(os_helper.TESTFN + "\udfff"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\xff"), False) - self.assertIs(posixpath.islink(os_helper.TESTFN + "\x00"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\x00"), False) + self.addCleanup(os_helper.unlink, TESTFN + "2") + os.symlink(TESTFN + "1", TESTFN + "2") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + os.remove(TESTFN + "1") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + self.assertIs(posixpath.exists(TESTFN + "2"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), True) + + def test_islink_invalid_paths(self): + self.assertIs(posixpath.islink(TESTFN + "\udfff"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\xff"), False) + self.assertIs(posixpath.islink(TESTFN + "\x00"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\x00"), False) def test_ismount(self): self.assertIs(posixpath.ismount("/"), True) self.assertIs(posixpath.ismount(b"/"), True) + self.assertIs(posixpath.ismount(FakePath("/")), True) + self.assertIs(posixpath.ismount(FakePath(b"/")), True) def test_ismount_non_existent(self): # Non-existent mountpoint. @@ -187,23 +226,22 @@ def test_ismount_non_existent(self): os.mkdir(ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) + def test_ismount_invalid_paths(self): self.assertIs(posixpath.ismount('/\udfff'), False) self.assertIs(posixpath.ismount(b'/\xff'), False) self.assertIs(posixpath.ismount('/\x00'), False) self.assertIs(posixpath.ismount(b'/\x00'), False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - @unittest.skipUnless(os_helper.can_symlink(), - "Test requires symlink support") + @os_helper.skip_unless_symlink def test_ismount_symlinks(self): # Symlinks are never mountpoints. try: os.symlink("/", ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - os.unlink(ABSTFN) + os_helper.unlink(ABSTFN) @unittest.skipIf(posix is None, "Test requires posix module") def test_ismount_different_device(self): @@ -245,6 +283,19 @@ def fake_lstat(path): finally: os.lstat = save_lstat + def test_isjunction(self): + self.assertFalse(posixpath.isjunction(ABSTFN)) + + @unittest.skipIf(sys.platform == 'win32', "Fast paths are not for win32") + @support.cpython_only + def test_fast_paths_in_use(self): + # There are fast paths of these functions implemented in posixmodule.c. + # Confirm that they are being used, and not the Python fallbacks + self.assertTrue(os.path.splitroot is posix._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is posix._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) + def test_expanduser(self): self.assertEqual(posixpath.expanduser("foo"), "foo") self.assertEqual(posixpath.expanduser(b"foo"), b"foo") @@ -306,59 +357,130 @@ def test_expanduser_pwd(self): for path in ('~', '~/.local', '~vstinner/'): self.assertEqual(posixpath.expanduser(path), path) + @unittest.skipIf(sys.platform == "vxworks", + "no home directory on VxWorks") + def test_expanduser_pwd2(self): + pwd = import_helper.import_module('pwd') + getpwall = support.get_attribute(pwd, 'getpwall') + names = [entry.pw_name for entry in getpwall()] + maxusers = 1000 if support.is_resource_enabled('cpu') else 100 + if len(names) > maxusers: + # Select random names, half of them with non-ASCII name, + # if available. + random.shuffle(names) + names.sort(key=lambda name: name.isascii()) + del names[maxusers//2:-maxusers//2] + for name in names: + # gh-121200: pw_dir can be different between getpwall() and + # getpwnam(), so use getpwnam() pw_dir as expanduser() does. + entry = pwd.getpwnam(name) + home = entry.pw_dir + home = home.rstrip('/') or '/' + + with self.subTest(name=name, pw_dir=entry.pw_dir): + self.assertEqual(posixpath.expanduser('~' + name), home) + self.assertEqual(posixpath.expanduser(os.fsencode('~' + name)), + os.fsencode(home)) + + NORMPATH_CASES = [ + ("", "."), + ("/", "/"), + ("/.", "/"), + ("/./", "/"), + ("/.//.", "/"), + ("/./foo/bar", "/foo/bar"), + ("/foo", "/foo"), + ("/foo/bar", "/foo/bar"), + ("//", "//"), + ("///", "/"), + ("///foo/.//bar//", "/foo/bar"), + ("///foo/.//bar//.//..//.//baz///", "/foo/baz"), + ("///..//./foo/.//bar", "/foo/bar"), + (".", "."), + (".//.", "."), + ("./foo/bar", "foo/bar"), + ("..", ".."), + ("../", ".."), + ("../foo", "../foo"), + ("../../foo", "../../foo"), + ("../foo/../bar", "../bar"), + ("../../foo/../bar/./baz/boom/..", "../../bar/baz"), + ("/..", "/"), + ("/..", "/"), + ("/../", "/"), + ("/..//", "/"), + ("//.", "//"), + ("//..", "//"), + ("//...", "//..."), + ("//../foo", "//foo"), + ("//../../foo", "//foo"), + ("/../foo", "/foo"), + ("/../../foo", "/foo"), + ("/../foo/../", "/"), + ("/../foo/../bar", "/bar"), + ("/../../foo/../bar/./baz/boom/..", "/bar/baz"), + ("/../../foo/../bar/./baz/boom/.", "/bar/baz/boom"), + ("foo/../bar/baz", "bar/baz"), + ("foo/../../bar/baz", "../bar/baz"), + ("foo/../../../bar/baz", "../../bar/baz"), + ("foo///../bar/.././../baz/boom", "../baz/boom"), + ("foo/bar/../..///../../baz/boom", "../../baz/boom"), + ("/foo/..", "/"), + ("/foo/../..", "/"), + ("//foo/..", "//"), + ("//foo/../..", "//"), + ("///foo/..", "/"), + ("///foo/../..", "/"), + ("////foo/..", "/"), + ("/////foo/..", "/"), + ] + def test_normpath(self): - self.assertEqual(posixpath.normpath(""), ".") - self.assertEqual(posixpath.normpath("/"), "/") - self.assertEqual(posixpath.normpath("//"), "//") - self.assertEqual(posixpath.normpath("///"), "/") - self.assertEqual(posixpath.normpath("///foo/.//bar//"), "/foo/bar") - self.assertEqual(posixpath.normpath("///foo/.//bar//.//..//.//baz"), - "/foo/baz") - self.assertEqual(posixpath.normpath("///..//./foo/.//bar"), "/foo/bar") - - self.assertEqual(posixpath.normpath(b""), b".") - self.assertEqual(posixpath.normpath(b"/"), b"/") - self.assertEqual(posixpath.normpath(b"//"), b"//") - self.assertEqual(posixpath.normpath(b"///"), b"/") - self.assertEqual(posixpath.normpath(b"///foo/.//bar//"), b"/foo/bar") - self.assertEqual(posixpath.normpath(b"///foo/.//bar//.//..//.//baz"), - b"/foo/baz") - self.assertEqual(posixpath.normpath(b"///..//./foo/.//bar"), - b"/foo/bar") + for path, expected in self.NORMPATH_CASES: + with self.subTest(path): + result = posixpath.normpath(path) + self.assertEqual(result, expected) + + path = path.encode('utf-8') + expected = expected.encode('utf-8') + with self.subTest(path, type=bytes): + result = posixpath.normpath(path) + self.assertEqual(result, expected) @skip_if_ABSTFN_contains_backslash - def test_realpath_curdir(self): - self.assertEqual(realpath('.'), os.getcwd()) - self.assertEqual(realpath('./.'), os.getcwd()) - self.assertEqual(realpath('/'.join(['.'] * 100)), os.getcwd()) + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_curdir(self, kwargs): + self.assertEqual(realpath('.', **kwargs), os.getcwd()) + self.assertEqual(realpath('./.', **kwargs), os.getcwd()) + self.assertEqual(realpath('/'.join(['.'] * 100), **kwargs), os.getcwd()) - self.assertEqual(realpath(b'.'), os.getcwdb()) - self.assertEqual(realpath(b'./.'), os.getcwdb()) - self.assertEqual(realpath(b'/'.join([b'.'] * 100)), os.getcwdb()) + self.assertEqual(realpath(b'.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'./.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'/'.join([b'.'] * 100), **kwargs), os.getcwdb()) @skip_if_ABSTFN_contains_backslash - def test_realpath_pardir(self): - self.assertEqual(realpath('..'), dirname(os.getcwd())) - self.assertEqual(realpath('../..'), dirname(dirname(os.getcwd()))) - self.assertEqual(realpath('/'.join(['..'] * 100)), '/') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_pardir(self, kwargs): + self.assertEqual(realpath('..', **kwargs), dirname(os.getcwd())) + self.assertEqual(realpath('../..', **kwargs), dirname(dirname(os.getcwd()))) + self.assertEqual(realpath('/'.join(['..'] * 100), **kwargs), '/') - self.assertEqual(realpath(b'..'), dirname(os.getcwdb())) - self.assertEqual(realpath(b'../..'), dirname(dirname(os.getcwdb()))) - self.assertEqual(realpath(b'/'.join([b'..'] * 100)), b'/') + self.assertEqual(realpath(b'..', **kwargs), dirname(os.getcwdb())) + self.assertEqual(realpath(b'../..', **kwargs), dirname(dirname(os.getcwdb()))) + self.assertEqual(realpath(b'/'.join([b'..'] * 100), **kwargs), b'/') - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_basic(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): # Basic operation. try: os.symlink(ABSTFN+"1", ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_strict(self): # Bug #43757: raise FileNotFoundError in strict mode if we encounter @@ -370,18 +492,123 @@ def test_realpath_strict(self): finally: os_helper.unlink(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + def test_realpath_invalid_paths(self): + path = '/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = '/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/') + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/nonexistent') + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/nonexistent') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + path = b'/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + path = b'/nonexistent/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_relative(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): try: os.symlink(posixpath.relpath(ABSTFN+"1"), ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_missing_pardir(self, kwargs): + try: + os.symlink(TESTFN + "1", TESTFN) + self.assertEqual( + realpath("nonexistent/../" + TESTFN, **kwargs), ABSTFN + "1") + finally: + os_helper.unlink(TESTFN) + + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops(self): # Bug #930024, return the path unchanged if we get into an infinite @@ -400,7 +627,7 @@ def test_realpath_symlink_loops(self): self.assertEqual(realpath(ABSTFN+"1/../x"), dirname(ABSTFN) + "/x") os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "y"), - ABSTFN + "y") + ABSTFN + "x") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "1"), ABSTFN + "1") @@ -422,40 +649,40 @@ def test_realpath_symlink_loops(self): os_helper.unlink(ABSTFN+"c") os_helper.unlink(ABSTFN+"a") - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_symlink_loops_strict(self): + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_loops_strict(self, kwargs): # Bug #43757, raise OSError if we get into an infinite symlink loop in - # strict mode. + # the strict modes. try: os.symlink(ABSTFN, ABSTFN) - self.assertRaises(OSError, realpath, ABSTFN, strict=True) + self.assertRaises(OSError, realpath, ABSTFN, **kwargs) os.symlink(ABSTFN+"1", ABSTFN+"2") os.symlink(ABSTFN+"2", ABSTFN+"1") - self.assertRaises(OSError, realpath, ABSTFN+"1", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"2", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"2", **kwargs) - self.assertRaises(OSError, realpath, ABSTFN+"1/x", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/..", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/../x", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1/x", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/..", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/../x", **kwargs) os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "y", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "y", **kwargs) self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "1", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "1", **kwargs) os.symlink(basename(ABSTFN) + "a/b", ABSTFN+"a") - self.assertRaises(OSError, realpath, ABSTFN+"a", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"a", **kwargs) os.symlink("../" + basename(dirname(ABSTFN)) + "/" + basename(ABSTFN) + "c", ABSTFN+"c") - self.assertRaises(OSError, realpath, ABSTFN+"c", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"c", **kwargs) # Test using relative path as well. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertRaises(OSError, realpath, basename(ABSTFN), strict=True) + self.assertRaises(OSError, realpath, basename(ABSTFN), **kwargs) finally: os_helper.unlink(ABSTFN) os_helper.unlink(ABSTFN+"1") @@ -464,32 +691,32 @@ def test_realpath_symlink_loops_strict(self): os_helper.unlink(ABSTFN+"c") os_helper.unlink(ABSTFN+"a") - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_repeated_indirect_symlinks(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_repeated_indirect_symlinks(self, kwargs): # Issue #6975. try: os.mkdir(ABSTFN) os.symlink('../' + basename(ABSTFN), ABSTFN + '/self') os.symlink('self/self/self', ABSTFN + '/link') - self.assertEqual(realpath(ABSTFN + '/link'), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/link', **kwargs), ABSTFN) finally: os_helper.unlink(ABSTFN + '/self') os_helper.unlink(ABSTFN + '/link') - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_deep_recursion(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_deep_recursion(self, kwargs): depth = 10 try: os.mkdir(ABSTFN) for i in range(depth): os.symlink('/'.join(['%d' % i] * 10), ABSTFN + '/%d' % (i + 1)) os.symlink('.', ABSTFN + '/0') - self.assertEqual(realpath(ABSTFN + '/%d' % depth), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/%d' % depth, **kwargs), ABSTFN) # Test using relative path as well. with os_helper.change_cwd(ABSTFN): @@ -497,12 +724,12 @@ def test_realpath_deep_recursion(self): finally: for i in range(depth + 1): os_helper.unlink(ABSTFN + '/%d' % i) - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_parents(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_parents(self, kwargs): # We also need to resolve any symlinks in the parents of a relative # path passed to realpath. E.g.: current working directory is # /usr/doc with 'doc' being a symlink to /usr/share/doc. We call @@ -513,16 +740,17 @@ def test_realpath_resolve_parents(self): os.symlink(ABSTFN + "/y", ABSTFN + "/k") with os_helper.change_cwd(ABSTFN + "/k"): - self.assertEqual(realpath("a"), ABSTFN + "/y/a") + self.assertEqual(realpath("a", **kwargs), + ABSTFN + "/y/a") finally: os_helper.unlink(ABSTFN + "/k") - safe_rmdir(ABSTFN + "/y") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/y") + os_helper.rmdir(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_before_normalizing(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_before_normalizing(self, kwargs): # Bug #990669: Symbolic links should be resolved before we # normalize the path. E.g.: if we have directories 'a', 'k' and 'y' # in the following hierarchy: @@ -537,21 +765,21 @@ def test_realpath_resolve_before_normalizing(self): os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y") # Absolute path. - self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k") + self.assertEqual(realpath(ABSTFN + "/link-y/..", **kwargs), ABSTFN + "/k") # Relative path. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."), + self.assertEqual(realpath(basename(ABSTFN) + "/link-y/..", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "/link-y") - safe_rmdir(ABSTFN + "/k/y") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k/y") + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) - @unittest.skipUnless(hasattr(os, "symlink"), - "Missing symlink implementation") + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_first(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_first(self, kwargs): # Bug #1213894: The first component of the path, if not absolute, # must be resolved too. @@ -561,17 +789,192 @@ def test_realpath_resolve_first(self): os.symlink(ABSTFN, ABSTFN + "link") with os_helper.change_cwd(dirname(ABSTFN)): base = basename(ABSTFN) - self.assertEqual(realpath(base + "link"), ABSTFN) - self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k") + self.assertEqual(realpath(base + "link", **kwargs), ABSTFN) + self.assertEqual(realpath(base + "link/k", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "link") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + def test_realpath_unreadable_symlink(self): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + self.assertEqual(realpath(ABSTFN), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') + self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') + self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_unreadable_symlink_strict(self, kwargs): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + with self.assertRaises(PermissionError): + realpath(ABSTFN, **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo', **kwargs), + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/../foo', **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo/..', **kwargs) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os.unlink(ABSTFN) + + @skip_if_ABSTFN_contains_backslash + @os_helper.skip_unless_symlink + def test_realpath_unreadable_directory(self): + try: + os.mkdir(ABSTFN) + os.mkdir(ABSTFN + '/k') + os.chmod(ABSTFN, 0o000) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + + try: + os.stat(ABSTFN) + except PermissionError: + pass + else: + self.skipTest('Cannot block permissions') + + self.assertEqual(realpath(ABSTFN + '/k', strict=False), + ABSTFN + '/k') + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + '/missing', strict=False), + ABSTFN + '/missing') + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=ALLOW_MISSING) + finally: + os.chmod(ABSTFN, 0o755) + os_helper.rmdir(ABSTFN + '/k') + os_helper.rmdir(ABSTFN) + + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_file(self): + try: + with open(ABSTFN, 'w') as f: + f.write('test_posixpath wuz ere') + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) + finally: + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_symlink_to_file(self): + try: + with open(ABSTFN + "1", 'w') as f: + f.write('test_posixpath wuz ere') + os.symlink(ABSTFN + "1", ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN + "1") + + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "1/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) + finally: + os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_symlink_to_symlinks_to_file(self): + try: + with open(ABSTFN + "2", 'w') as f: + f.write('test_posixpath wuz ere') + os.symlink(ABSTFN + "2", ABSTFN + "1") + os.symlink(ABSTFN + "1", ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "2/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) + finally: + os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") + os_helper.unlink(ABSTFN + "2") def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: curdir = os.path.split(os.getcwd())[-1] + self.assertRaises(TypeError, posixpath.relpath, None) self.assertRaises(ValueError, posixpath.relpath, "") self.assertEqual(posixpath.relpath("a"), "a") self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") @@ -634,7 +1037,9 @@ def check_error(exc, paths): self.assertRaises(exc, posixpath.commonpath, [os.fsencode(p) for p in paths]) + self.assertRaises(TypeError, posixpath.commonpath, None) self.assertRaises(ValueError, posixpath.commonpath, []) + self.assertRaises(ValueError, posixpath.commonpath, iter([])) check_error(ValueError, ['/usr', 'usr']) check_error(ValueError, ['usr', '/usr']) @@ -679,62 +1084,18 @@ def check_error(exc, paths): ['usr/lib/', b'/usr/lib/python3']) -@unittest.skip("TODO: RUSTPYTHON, flaky tests") class PosixCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = posixpath attributes = ['relpath', 'samefile', 'sameopenfile', 'samestat'] - # TODO: RUSTPYTHON - if os.name == "posix" and os.getenv("CI"): - @unittest.expectedFailure - def test_exists(self): - super().test_exists() - - # TODO: RUSTPYTHON - import sys - @unittest.skipIf(sys.platform.startswith("linux") and os.getenv("CI"), "TODO: RUSTPYTHON, flaky test") - def test_filetime(self): - super().test_filetime() - - # TODO: RUSTPYTHON - if sys.platform.startswith("linux"): - @unittest.expectedFailure - def test_nonascii_abspath(self): - super().test_nonascii_abspath() - - # TODO: RUSTPYTHON - if os.name == "nt": - @unittest.expectedFailure - def test_samefile(self): - super().test_samefile() - - # TODO: RUSTPYTHON - if os.name == "nt": - @unittest.expectedFailure - def test_samefile_on_link(self): - super().test_samefile_on_link() - - # TODO: RUSTPYTHON - if os.name == "nt": - @unittest.expectedFailure - def test_samestat(self): - super().test_samestat() - - # TODO: RUSTPYTHON - if os.name == "nt": - @unittest.expectedFailure - def test_samestat_on_link(self): - super().test_samestat_on_link() - - -@unittest.skipIf(os.getenv("CI"), "TODO: RUSTPYTHON, FileExistsError: (17, 'File exists (os error 17)')") + class PathLikeTests(unittest.TestCase): path = posixpath def setUp(self): - self.file_name = os_helper.TESTFN - self.file_path = FakePath(os_helper.TESTFN) + self.file_name = TESTFN + self.file_path = FakePath(TESTFN) self.addCleanup(os_helper.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -761,6 +1122,9 @@ def test_path_splitext(self): def test_path_splitdrive(self): self.assertPathEqual(self.path.splitdrive) + def test_path_splitroot(self): + self.assertPathEqual(self.path.splitroot) + def test_path_basename(self): self.assertPathEqual(self.path.basename) @@ -788,9 +1152,12 @@ def test_path_normpath(self): def test_path_abspath(self): self.assertPathEqual(self.path.abspath) - def test_path_realpath(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_path_realpath(self, kwargs): self.assertPathEqual(self.path.realpath) + self.assertPathEqual(partial(self.path.realpath, **kwargs)) + def test_path_relpath(self): self.assertPathEqual(self.path.relpath) diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py index 5cea9ceb20f..eeb482ec4b2 100644 --- a/Lib/test/test_pow.py +++ b/Lib/test/test_pow.py @@ -19,12 +19,11 @@ def powtest(self, type): self.assertEqual(pow(2, i), pow2) if i != 30 : pow2 = pow2*2 - for othertype in (int,): - for i in list(range(-10, 0)) + list(range(1, 10)): - ii = type(i) - for j in range(1, 11): - jj = -othertype(j) - pow(ii, jj) + for i in list(range(-10, 0)) + list(range(1, 10)): + ii = type(i) + inv = pow(ii, -1) # inverse of ii + for jj in range(-10, 0): + self.assertAlmostEqual(pow(ii, jj), pow(inv, -jj)) for othertype in int, float: for i in range(1, 100): diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index c7b98939434..403d2e90084 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -7,11 +7,13 @@ import itertools import pprint import random -import test.support -import test.test_set +import re import types import unittest +from test.support import cpython_only +from test.support.import_helper import ensure_lazy_imports + # list, tuple and dict subclasses that do or don't overwrite __repr__ class list2(list): pass @@ -130,6 +132,10 @@ def setUp(self): self.b = list(range(200)) self.a[-12] = self.b + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("pprint", {"dataclasses", "re"}) + def test_init(self): pp = pprint.PrettyPrinter() pp = pprint.PrettyPrinter(indent=4, width=40, depth=5, @@ -203,7 +209,7 @@ def test_knotted(self): def test_unreadable(self): # Not recursive but not readable anyway pp = pprint.PrettyPrinter() - for unreadable in type(3), pprint, pprint.isrecursive: + for unreadable in object(), int, pprint, pprint.isrecursive: # module-level convenience functions self.assertFalse(pprint.isrecursive(unreadable), "expected not isrecursive for %r" % (unreadable,)) @@ -374,7 +380,7 @@ def __new__(cls, celsius_degrees): return super().__new__(Temperature, celsius_degrees) def __repr__(self): kelvin_degrees = self + 273.15 - return f"{kelvin_degrees}°K" + return f"{kelvin_degrees:.2f}°K" self.assertEqual(pprint.pformat(Temperature(1000)), '1273.15°K') def test_sorted_dict(self): @@ -535,7 +541,10 @@ def test_dataclass_with_repr(self): def test_dataclass_no_repr(self): dc = dataclass3() formatted = pprint.pformat(dc, width=10) - self.assertRegex(formatted, r"<test.test_pprint.dataclass3 object at \w+>") + self.assertRegex( + formatted, + fr"<{re.escape(__name__)}.dataclass3 object at \w+>", + ) def test_recursive_dataclass(self): dc = dataclass4(None) @@ -619,9 +628,6 @@ def test_set_reprs(self): self.assertEqual(pprint.pformat(frozenset3(range(7)), width=20), 'frozenset3({0, 1, 2, 3, 4, 5, 6})') - @unittest.expectedFailure - #See http://bugs.python.org/issue13907 - @test.support.cpython_only def test_set_of_sets_reprs(self): # This test creates a complex arrangement of frozensets and # compares the pretty-printed repr against a string hard-coded in @@ -632,204 +638,106 @@ def test_set_of_sets_reprs(self): # partial ordering (subset relationships), the output of the # list.sort() method is undefined for lists of sets." # - # In a nutshell, the test assumes frozenset({0}) will always - # sort before frozenset({1}), but: - # # >>> frozenset({0}) < frozenset({1}) # False # >>> frozenset({1}) < frozenset({0}) # False # - # Consequently, this test is fragile and - # implementation-dependent. Small changes to Python's sort - # algorithm cause the test to fail when it should pass. - # XXX Or changes to the dictionary implementation... - - cube_repr_tgt = """\ -{frozenset(): frozenset({frozenset({2}), frozenset({0}), frozenset({1})}), - frozenset({0}): frozenset({frozenset(), - frozenset({0, 2}), - frozenset({0, 1})}), - frozenset({1}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 1})}), - frozenset({2}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 2})}), - frozenset({1, 2}): frozenset({frozenset({2}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 2}): frozenset({frozenset({2}), - frozenset({0}), - frozenset({0, 1, 2})}), - frozenset({0, 1}): frozenset({frozenset({0}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 1, 2}): frozenset({frozenset({1, 2}), - frozenset({0, 2}), - frozenset({0, 1})})}""" - cube = test.test_set.cube(3) - self.assertEqual(pprint.pformat(cube), cube_repr_tgt) - cubo_repr_tgt = """\ -{frozenset({frozenset({0, 2}), frozenset({0})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1}), frozenset({1})}): frozenset({frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})})}), - frozenset({frozenset({1, 2}), frozenset({1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({1, 2}), frozenset({2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({0})}): frozenset({frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset()}): frozenset({frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({0}), frozenset({0, 1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({1, 2}), frozenset({0, 1, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})})})}""" - - cubo = test.test_set.linegraph(cube) - self.assertEqual(pprint.pformat(cubo), cubo_repr_tgt) + # In this test we list all possible invariants of the result + # for unordered frozensets. + # + # This test has a long history, see: + # - https://github.com/python/cpython/commit/969fe57baa0eb80332990f9cda936a33e13fabef + # - https://github.com/python/cpython/issues/58115 + # - https://github.com/python/cpython/issues/111147 + + import textwrap + + # Single-line, always ordered: + fs0 = frozenset() + fs1 = frozenset(('abc', 'xyz')) + data = frozenset((fs0, fs1)) + self.assertEqual(pprint.pformat(data), + 'frozenset({%r, %r})' % (fs0, fs1)) + self.assertEqual(pprint.pformat(data), repr(data)) + + fs2 = frozenset(('one', 'two')) + data = {fs2: frozenset((fs0, fs1))} + self.assertEqual(pprint.pformat(data), + "{%r: frozenset({%r, %r})}" % (fs2, fs0, fs1)) + self.assertEqual(pprint.pformat(data), repr(data)) + + # Single-line, unordered: + fs1 = frozenset(("xyz", "qwerty")) + fs2 = frozenset(("abcd", "spam")) + fs = frozenset((fs1, fs2)) + self.assertEqual(pprint.pformat(fs), repr(fs)) + + # Multiline, unordered: + def check(res, invariants): + self.assertIn(res, [textwrap.dedent(i).strip() for i in invariants]) + + # Inner-most frozensets are singleline, result is multiline, unordered: + fs1 = frozenset(('regular string', 'other string')) + fs2 = frozenset(('third string', 'one more string')) + check( + pprint.pformat(frozenset((fs1, fs2))), + [ + """ + frozenset({%r, + %r}) + """ % (fs1, fs2), + """ + frozenset({%r, + %r}) + """ % (fs2, fs1), + ], + ) + + # Everything is multiline, unordered: + check( + pprint.pformat( + frozenset(( + frozenset(( + "xyz very-very long string", + "qwerty is also absurdly long", + )), + frozenset(( + "abcd is even longer that before", + "spam is not so long", + )), + )), + ), + [ + """ + frozenset({frozenset({'abcd is even longer that before', + 'spam is not so long'}), + frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'})}) + """, + + """ + frozenset({frozenset({'abcd is even longer that before', + 'spam is not so long'}), + frozenset({'xyz very-very long string', + 'qwerty is also absurdly long'})}) + """, + + """ + frozenset({frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'}), + frozenset({'abcd is even longer that before', + 'spam is not so long'})}) + """, + + """ + frozenset({frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'}), + frozenset({'spam is not so long', + 'abcd is even longer that before'})}) + """, + ], + ) def test_depth(self): nested_tuple = (1, (2, (3, (4, (5, 6))))) diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 8445a501cf9..7c71c5eefe2 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -129,14 +129,24 @@ def flush(self): raise RuntimeError self.assertRaises(RuntimeError, print, 1, file=noflush(), flush=True) + def test_gh130163(self): + class X: + def __str__(self): + sys.stdout = StringIO() + support.gc_collect() + return 'foo' + + with support.swap_attr(sys, 'stdout', None): + sys.stdout = StringIO() # the only reference + print(X()) # should not crash + class TestPy2MigrationHint(unittest.TestCase): """Test that correct hint is produced analogous to Python3 syntax, if print statement is executed as in Python 2. """ - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_normal_string(self): python2_print_str = 'print "Hello World"' with self.assertRaises(SyntaxError) as context: @@ -145,8 +155,7 @@ def test_normal_string(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_soft_space(self): python2_print_str = 'print "Hello World",' with self.assertRaises(SyntaxError) as context: @@ -155,8 +164,7 @@ def test_string_with_soft_space(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_excessive_whitespace(self): python2_print_str = 'print "Hello World", ' with self.assertRaises(SyntaxError) as context: @@ -165,8 +173,7 @@ def test_string_with_excessive_whitespace(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_leading_whitespace(self): python2_print_str = '''if 1: print "Hello World" @@ -180,9 +187,7 @@ def test_string_with_leading_whitespace(self): # bpo-32685: Suggestions for print statement should be proper when # it is in the same line as the header of a compound statement # and/or followed by a semicolon - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_semicolon(self): python2_print_str = 'print p;' with self.assertRaises(SyntaxError) as context: @@ -191,8 +196,7 @@ def test_string_with_semicolon(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_in_loop_on_same_line(self): python2_print_str = 'for i in s: print i' with self.assertRaises(SyntaxError) as context: @@ -201,40 +205,6 @@ def test_string_in_loop_on_same_line(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_stream_redirection_hint_for_py2_migration(self): - # Test correct hint produced for Py2 redirection syntax - with self.assertRaises(TypeError) as context: - print >> sys.stderr, "message" - self.assertIn('Did you mean "print(<message>, ' - 'file=<output_stream>)"?', str(context.exception)) - - # Test correct hint is produced in the case where RHS implements - # __rrshift__ but returns NotImplemented - with self.assertRaises(TypeError) as context: - print >> 42 - self.assertIn('Did you mean "print(<message>, ' - 'file=<output_stream>)"?', str(context.exception)) - - # Test stream redirection hint is specific to print - with self.assertRaises(TypeError) as context: - max >> sys.stderr - self.assertNotIn('Did you mean ', str(context.exception)) - - # Test stream redirection hint is specific to rshift - with self.assertRaises(TypeError) as context: - print << sys.stderr - self.assertNotIn('Did you mean', str(context.exception)) - - # Ensure right operand implementing rrshift still works - class OverrideRRShift: - def __rrshift__(self, lhs): - return 42 # Force result independent of LHS - - self.assertEqual(print >> OverrideRRShift(), 42) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index 5312925d937..26aefdbf042 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -87,8 +87,8 @@ def test_property_decorator_baseclass(self): self.assertEqual(base.spam, 10) self.assertEqual(base._spam, 10) delattr(base, "spam") - self.assertTrue(not hasattr(base, "spam")) - self.assertTrue(not hasattr(base, "_spam")) + self.assertNotHasAttr(base, "spam") + self.assertNotHasAttr(base, "_spam") base.spam = 20 self.assertEqual(base.spam, 20) self.assertEqual(base._spam, 20) @@ -100,32 +100,24 @@ def test_property_decorator_subclass(self): self.assertRaises(PropertySet, setattr, sub, "spam", None) self.assertRaises(PropertyDel, delattr, sub, "spam") - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_decorator_subclass_doc(self): sub = SubClass() self.assertEqual(sub.__class__.spam.__doc__, "SubClass.getter") - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_decorator_baseclass_doc(self): base = BaseClass() self.assertEqual(base.__class__.spam.__doc__, "BaseClass.getter") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_property_decorator_doc(self): base = PropertyDocBase() sub = PropertyDocSub() self.assertEqual(base.__class__.spam.__doc__, "spam spam spam") self.assertEqual(sub.__class__.spam.__doc__, "spam spam spam") - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_getter_doc_override(self): @@ -136,8 +128,6 @@ def test_property_getter_doc_override(self): self.assertEqual(newgetter.spam, 8) self.assertEqual(newgetter.__class__.spam.__doc__, "new docstring") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_property___isabstractmethod__descriptor(self): for val in (True, False, [], [1], '', '1'): class C(object): @@ -169,8 +159,6 @@ def test_property_builtin_doc_writable(self): p.__doc__ = 'extended' self.assertEqual(p.__doc__, 'extended') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_decorator_doc_writable(self): @@ -195,26 +183,76 @@ def test_refleaks_in___init__(self): fake_prop.__init__('fget', 'fset', 'fdel', 'doc') self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -O2 and above") - def test_class_property(self): - class A: - @classmethod - @property - def __doc__(cls): - return 'A doc for %r' % cls.__name__ - self.assertEqual(A.__doc__, "A doc for 'A'") + @support.refcount_test + def test_gh_115618(self): + # Py_XDECREF() was improperly called for None argument + # in property methods. + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + prop = property() + refs_before = gettotalrefcount() + for i in range(100): + prop = prop.getter(None) + self.assertIsNone(prop.fget) + for i in range(100): + prop = prop.setter(None) + self.assertIsNone(prop.fset) + for i in range(100): + prop = prop.deleter(None) + self.assertIsNone(prop.fdel) + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_property_name(self): + def getter(self): + return 42 + + def setter(self, value): + pass - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -O2 and above") - def test_class_property_override(self): class A: - """First""" - @classmethod @property - def __doc__(cls): - return 'Second' - self.assertEqual(A.__doc__, 'Second') + def foo(self): + return 1 + + @foo.setter + def oof(self, value): + pass + + bar = property(getter) + baz = property(None, setter) + + self.assertEqual(A.foo.__name__, 'foo') + self.assertEqual(A.oof.__name__, 'oof') + self.assertEqual(A.bar.__name__, 'bar') + self.assertEqual(A.baz.__name__, 'baz') + + A.quux = property(getter) + self.assertEqual(A.quux.__name__, 'getter') + A.quux.__name__ = 'myquux' + self.assertEqual(A.quux.__name__, 'myquux') + self.assertEqual(A.bar.__name__, 'bar') # not affected + A.quux.__name__ = None + self.assertIsNone(A.quux.__name__) + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(None, setter).__name__ + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(1).__name__ + + class Err: + def __getattr__(self, attr): + raise RuntimeError('fail') + + p = property(Err()) + with self.assertRaisesRegex(RuntimeError, 'fail'): + p.__name__ + + p.__name__ = 'not_fail' + self.assertEqual(p.__name__, 'not_fail') def test_property_set_name_incorrect_args(self): p = property() @@ -248,28 +286,111 @@ class A: class PropertySub(property): """This is a subclass of property""" +class PropertySubWoDoc(property): + pass + class PropertySubSlots(property): """This is a subclass of property that defines __slots__""" __slots__ = () class PropertySubclassTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_docstrings def test_slots_docstring_copy_exception(self): - try: + # A special case error that we preserve despite the GH-98963 behavior + # that would otherwise silently ignore this error. + # This came from commit b18500d39d791c879e9904ebac293402b4a7cd34 + # as part of https://bugs.python.org/issue5890 which allowed docs to + # be set via property subclasses in the first place. + with self.assertRaises(AttributeError): class Foo(object): @PropertySubSlots def spam(self): """Trying to copy this docstring will raise an exception""" return 1 - except AttributeError: + + def test_property_with_slots_no_docstring(self): + # https://github.com/python/cpython/issues/98963#issuecomment-1574413319 + class slotted_prop(property): + __slots__ = ("foo",) + + p = slotted_prop() # no AttributeError + self.assertIsNone(getattr(p, "__doc__", None)) + + def undocumented_getter(): + return 4 + + p = slotted_prop(undocumented_getter) # New in 3.12: no AttributeError + self.assertIsNone(getattr(p, "__doc__", None)) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_with_slots_docstring_silently_dropped(self): + # https://github.com/python/cpython/issues/98963#issuecomment-1574413319 + class slotted_prop(property): + __slots__ = ("foo",) + + p = slotted_prop(doc="what's up") # no AttributeError + self.assertIsNone(p.__doc__) + + def documented_getter(): + """getter doc.""" + return 4 + + # Historical behavior: A docstring from a getter always raises. + # (matches test_slots_docstring_copy_exception above). + with self.assertRaises(AttributeError): + p = slotted_prop(documented_getter) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_with_slots_and_doc_slot_docstring_present(self): + # https://github.com/python/cpython/issues/98963#issuecomment-1574413319 + class slotted_prop(property): + __slots__ = ("foo", "__doc__") + + p = slotted_prop(doc="what's up") + self.assertEqual("what's up", p.__doc__) # new in 3.12: This gets set. + + def documented_getter(): + """what's up getter doc?""" + return 4 + + p = slotted_prop(documented_getter) + self.assertEqual("what's up getter doc?", p.__doc__) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_issue41287(self): + + self.assertEqual(PropertySub.__doc__, "This is a subclass of property", + "Docstring of `property` subclass is ignored") + + doc = PropertySub(None, None, None, "issue 41287 is fixed").__doc__ + self.assertEqual(doc, "issue 41287 is fixed", + "Subclasses of `property` ignores `doc` constructor argument") + + def getter(x): + """Getter docstring""" + + def getter_wo_doc(x): pass - else: - raise Exception("AttributeError not raised") - # TODO: RUSTPYTHON - @unittest.expectedFailure + for ps in property, PropertySub, PropertySubWoDoc: + doc = ps(getter, None, None, "issue 41287 is fixed").__doc__ + self.assertEqual(doc, "issue 41287 is fixed", + "Getter overrides explicit property docstring (%s)" % ps.__name__) + + doc = ps(getter, None, None, None).__doc__ + self.assertEqual(doc, "Getter docstring", "Getter docstring is not picked-up (%s)" % ps.__name__) + + doc = ps(getter_wo_doc, None, None, "issue 41287 is fixed").__doc__ + self.assertEqual(doc, "issue 41287 is fixed", + "Getter overrides explicit property docstring (%s)" % ps.__name__) + + doc = ps(getter_wo_doc, None, None, None).__doc__ + self.assertIsNone(doc, "Property class doc appears in instance __doc__ (%s)" % ps.__name__) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_docstring_copy(self): @@ -282,8 +403,100 @@ def spam(self): Foo.spam.__doc__, "spam wrapped in property subclass") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_docstring_copy2(self): + """ + Property tries to provide the best docstring it finds for its instances. + If a user-provided docstring is available, it is preserved on copies. + If no docstring is available during property creation, the property + will utilize the docstring from the getter if available. + """ + def getter1(self): + return 1 + def getter2(self): + """doc 2""" + return 2 + def getter3(self): + """doc 3""" + return 3 + + # Case-1: user-provided doc is preserved in copies + # of property with undocumented getter + p = property(getter1, None, None, "doc-A") + + p2 = p.getter(getter2) + self.assertEqual(p.__doc__, "doc-A") + self.assertEqual(p2.__doc__, "doc-A") + + # Case-2: user-provided doc is preserved in copies + # of property with documented getter + p = property(getter2, None, None, "doc-A") + + p2 = p.getter(getter3) + self.assertEqual(p.__doc__, "doc-A") + self.assertEqual(p2.__doc__, "doc-A") + + # Case-3: with no user-provided doc new getter doc + # takes precedence + p = property(getter2, None, None, None) + + p2 = p.getter(getter3) + self.assertEqual(p.__doc__, "doc 2") + self.assertEqual(p2.__doc__, "doc 3") + + # Case-4: A user-provided doc is assigned after property construction + # with documented getter. The doc IS NOT preserved. + # It's an odd behaviour, but it's a strange enough + # use case with no easy solution. + p = property(getter2, None, None, None) + p.__doc__ = "user" + p2 = p.getter(getter3) + self.assertEqual(p.__doc__, "user") + self.assertEqual(p2.__doc__, "doc 3") + + # Case-5: A user-provided doc is assigned after property construction + # with UNdocumented getter. The doc IS preserved. + p = property(getter1, None, None, None) + p.__doc__ = "user" + p2 = p.getter(getter2) + self.assertEqual(p.__doc__, "user") + self.assertEqual(p2.__doc__, "user") + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_prefer_explicit_doc(self): + # Issue 25757: subclasses of property lose docstring + self.assertEqual(property(doc="explicit doc").__doc__, "explicit doc") + self.assertEqual(PropertySub(doc="explicit doc").__doc__, "explicit doc") + + class Foo: + spam = PropertySub(doc="spam explicit doc") + + @spam.getter + def spam(self): + """ignored as doc already set""" + return 1 + + def _stuff_getter(self): + """ignored as doc set directly""" + stuff = PropertySub(doc="stuff doc argument", fget=_stuff_getter) + + #self.assertEqual(Foo.spam.__doc__, "spam explicit doc") + self.assertEqual(Foo.stuff.__doc__, "stuff doc argument") + + def test_property_no_doc_on_getter(self): + # If a property's getter has no __doc__ then the property's doc should + # be None; test that this is consistent with subclasses as well; see + # GH-2487 + class NoDoc: + @property + def __doc__(self): + raise AttributeError + + self.assertEqual(property(NoDoc()).__doc__, None) + self.assertEqual(PropertySub(NoDoc()).__doc__, None) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_setter_copies_getter_docstring(self): @@ -317,8 +530,6 @@ def spam(self, value): FooSub.spam.__doc__, "spam wrapped in property subclass") - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_new_getter_new_docstring(self): @@ -358,20 +569,14 @@ def _format_exc_msg(self, msg): def setUpClass(cls): cls.obj = cls.cls() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_property(self): with self.assertRaisesRegex(AttributeError, self._format_exc_msg("has no getter")): self.obj.foo - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_set_property(self): with self.assertRaisesRegex(AttributeError, self._format_exc_msg("has no setter")): self.obj.foo = None - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_del_property(self): with self.assertRaisesRegex(AttributeError, self._format_exc_msg("has no deleter")): del self.obj.foo @@ -384,7 +589,6 @@ class cls: foo = property() - class PropertyUnreachableAttributeNoName(_PropertyUnreachableAttribute, unittest.TestCase): msg_format = r"^property of 'PropertyUnreachableAttributeNoName\.cls' object {}$" diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index b1c1f6abffb..6d3c47195c6 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,10 +1,16 @@ -from test import support -from test.support import verbose, reap_children +import unittest +from test.support import ( + is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose +) from test.support.import_helper import import_module +from test.support.os_helper import TESTFN, unlink # Skip these tests if termios is not available import_module('termios') +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("pty is not available on this platform") + import errno import os import pty @@ -14,21 +20,12 @@ import signal import socket import io # readline -import unittest - -import struct -import fcntl import warnings TEST_STRING_1 = b"I wish to buy a fish license.\n" TEST_STRING_2 = b"For my pet fish, Eric.\n" -try: - _TIOCGWINSZ = tty.TIOCGWINSZ - _TIOCSWINSZ = tty.TIOCSWINSZ - _HAVE_WINSZ = True -except AttributeError: - _HAVE_WINSZ = False +_HAVE_WINSZ = hasattr(tty, "TIOCGWINSZ") and hasattr(tty, "TIOCSWINSZ") if verbose: def debug(msg): @@ -82,90 +79,78 @@ def expectedFailureIfStdinIsTTY(fun): pass return fun -def _get_term_winsz(fd): - s = struct.pack("HHHH", 0, 0, 0, 0) - return fcntl.ioctl(fd, _TIOCGWINSZ, s) -def _set_term_winsz(fd, winsz): - fcntl.ioctl(fd, _TIOCSWINSZ, winsz) +def write_all(fd, data): + written = os.write(fd, data) + if written != len(data): + # gh-73256, gh-110673: It should never happen, but check just in case + raise Exception(f"short write: os.write({fd}, {len(data)} bytes) " + f"wrote {written} bytes") # Marginal testing of pty suite. Cannot do extensive 'do or fail' testing # because pty code is not too portable. class PtyTest(unittest.TestCase): def setUp(self): - old_alarm = signal.signal(signal.SIGALRM, self.handle_sig) - self.addCleanup(signal.signal, signal.SIGALRM, old_alarm) - old_sighup = signal.signal(signal.SIGHUP, self.handle_sighup) self.addCleanup(signal.signal, signal.SIGHUP, old_sighup) - # isatty() and close() can hang on some platforms. Set an alarm - # before running the test to make sure we don't hang forever. - self.addCleanup(signal.alarm, 0) - signal.alarm(10) - - # Save original stdin window size - self.stdin_rows = None - self.stdin_cols = None + # Save original stdin window size. + self.stdin_dim = None if _HAVE_WINSZ: try: - stdin_dim = os.get_terminal_size(pty.STDIN_FILENO) - self.stdin_rows = stdin_dim.lines - self.stdin_cols = stdin_dim.columns - old_stdin_winsz = struct.pack("HHHH", self.stdin_rows, - self.stdin_cols, 0, 0) - self.addCleanup(_set_term_winsz, pty.STDIN_FILENO, old_stdin_winsz) - except OSError: + self.stdin_dim = tty.tcgetwinsize(pty.STDIN_FILENO) + self.addCleanup(tty.tcsetwinsize, pty.STDIN_FILENO, + self.stdin_dim) + except tty.error: pass - def handle_sig(self, sig, frame): - self.fail("isatty hung") - @staticmethod def handle_sighup(signum, frame): pass @expectedFailureIfStdinIsTTY + @unittest.skip('TODO: RUSTPYTHON; "Not runnable. tty.tcgetwinsize" is required to setUp') def test_openpty(self): try: mode = tty.tcgetattr(pty.STDIN_FILENO) except tty.error: - # not a tty or bad/closed fd + # Not a tty or bad/closed fd. debug("tty.tcgetattr(pty.STDIN_FILENO) failed") mode = None - new_stdin_winsz = None - if self.stdin_rows is not None and self.stdin_cols is not None: + new_dim = None + if self.stdin_dim: try: # Modify pty.STDIN_FILENO window size; we need to # check if pty.openpty() is able to set pty slave # window size accordingly. - debug("Setting pty.STDIN_FILENO window size") - debug(f"original size: (rows={self.stdin_rows}, cols={self.stdin_cols})") - target_stdin_rows = self.stdin_rows + 1 - target_stdin_cols = self.stdin_cols + 1 - debug(f"target size: (rows={target_stdin_rows}, cols={target_stdin_cols})") - target_stdin_winsz = struct.pack("HHHH", target_stdin_rows, - target_stdin_cols, 0, 0) - _set_term_winsz(pty.STDIN_FILENO, target_stdin_winsz) + debug("Setting pty.STDIN_FILENO window size.") + debug(f"original size: (row, col) = {self.stdin_dim}") + target_dim = (self.stdin_dim[0] + 1, self.stdin_dim[1] + 1) + debug(f"target size: (row, col) = {target_dim}") + tty.tcsetwinsize(pty.STDIN_FILENO, target_dim) # Were we able to set the window size # of pty.STDIN_FILENO successfully? - new_stdin_winsz = _get_term_winsz(pty.STDIN_FILENO) - self.assertEqual(new_stdin_winsz, target_stdin_winsz, + new_dim = tty.tcgetwinsize(pty.STDIN_FILENO) + self.assertEqual(new_dim, target_dim, "pty.STDIN_FILENO window size unchanged") - except OSError: - warnings.warn("Failed to set pty.STDIN_FILENO window size") + except OSError as e: + logging.getLogger(__name__).warning( + "Failed to set pty.STDIN_FILENO window size.", exc_info=e, + ) pass try: debug("Calling pty.openpty()") try: - master_fd, slave_fd = pty.openpty(mode, new_stdin_winsz) + master_fd, slave_fd, slave_name = pty.openpty(mode, new_dim, + True) except TypeError: master_fd, slave_fd = pty.openpty() - debug(f"Got master_fd '{master_fd}', slave_fd '{slave_fd}'") + slave_name = None + debug(f"Got {master_fd=}, {slave_fd=}, {slave_name=}") except OSError: # " An optional feature could not be imported " ... ? raise unittest.SkipTest("Pseudo-terminals (seemingly) not functional.") @@ -181,8 +166,8 @@ def test_openpty(self): if mode: self.assertEqual(tty.tcgetattr(slave_fd), mode, "openpty() failed to set slave termios") - if new_stdin_winsz: - self.assertEqual(_get_term_winsz(slave_fd), new_stdin_winsz, + if new_dim: + self.assertEqual(tty.tcgetwinsize(slave_fd), new_dim, "openpty() failed to set slave window size") # Ensure the fd is non-blocking in case there's nothing to read. @@ -200,18 +185,18 @@ def test_openpty(self): os.set_blocking(master_fd, blocking) debug("Writing to slave_fd") - os.write(slave_fd, TEST_STRING_1) + write_all(slave_fd, TEST_STRING_1) s1 = _readline(master_fd) self.assertEqual(b'I wish to buy a fish license.\n', normalize_output(s1)) debug("Writing chunked output") - os.write(slave_fd, TEST_STRING_2[:5]) - os.write(slave_fd, TEST_STRING_2[5:]) + write_all(slave_fd, TEST_STRING_2[:5]) + write_all(slave_fd, TEST_STRING_2[5:]) s2 = _readline(master_fd) self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) - @support.requires_fork() + @unittest.skip('TODO: RUSTPYTHON; "Not runnable. tty.tcgetwinsize" is required to setUp') def test_fork(self): debug("calling pty.fork()") pid, master_fd = pty.fork() @@ -294,6 +279,7 @@ def test_fork(self): ##else: ## raise TestFailed("Read from master_fd did not raise exception") + @unittest.skip('TODO: RUSTPYTHON; AttributeError: module "tty" has no attribute "tcgetwinsize"') def test_master_read(self): # XXX(nnorwitz): this test leaks fds when there is an error. debug("Calling pty.openpty()") @@ -313,8 +299,28 @@ def test_master_read(self): self.assertEqual(data, b"") + @unittest.skip('TODO: RUSTPYTHON; AttributeError: module "tty" has no attribute "tcgetwinsize"') def test_spawn_doesnt_hang(self): - pty.spawn([sys.executable, '-c', 'print("hi there")']) + self.addCleanup(unlink, TESTFN) + with open(TESTFN, 'wb') as f: + STDOUT_FILENO = 1 + dup_stdout = os.dup(STDOUT_FILENO) + os.dup2(f.fileno(), STDOUT_FILENO) + buf = b'' + def master_read(fd): + nonlocal buf + data = os.read(fd, 1024) + buf += data + return data + try: + pty.spawn([sys.executable, '-c', 'print("hi there")'], + master_read) + finally: + os.dup2(dup_stdout, STDOUT_FILENO) + os.close(dup_stdout) + self.assertEqual(buf, b'hi there\r\n') + with open(TESTFN, 'rb') as f: + self.assertEqual(f.read(), b'hi there\r\n') class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" @@ -332,8 +338,8 @@ def setUp(self): self.orig_pty_waitpid = pty.waitpid self.fds = [] # A list of file descriptors to close. self.files = [] - self.select_rfds_lengths = [] - self.select_rfds_results = [] + self.select_input = [] + self.select_output = [] self.tcsetattr_mode_setting = None def tearDown(self): @@ -368,11 +374,10 @@ def _socketpair(self): self.files.extend(socketpair) return socketpair - def _mock_select(self, rfds, wfds, xfds, timeout=0): + def _mock_select(self, rfds, wfds, xfds): # This will raise IndexError when no more expected calls exist. - # This ignores the timeout - self.assertEqual(self.select_rfds_lengths.pop(0), len(rfds)) - return self.select_rfds_results.pop(0), [], [] + self.assertEqual((rfds, wfds, xfds), self.select_input.pop(0)) + return self.select_output.pop(0) def _make_mock_fork(self, pid): def mock_fork(): @@ -392,14 +397,16 @@ def test__copy_to_each(self): masters = [s.fileno() for s in socketpair] # Feed data. Smaller than PIPEBUF. These writes will not block. - os.write(masters[1], b'from master') - os.write(write_to_stdin_fd, b'from stdin') + write_all(masters[1], b'from master') + write_all(write_to_stdin_fd, b'from stdin') - # Expect two select calls, the last one will cause IndexError + # Expect three select calls, the last one will cause IndexError pty.select = self._mock_select - self.select_rfds_lengths.append(2) - self.select_rfds_results.append([mock_stdin_fd, masters[0]]) - self.select_rfds_lengths.append(2) + self.select_input.append(([mock_stdin_fd, masters[0]], [], [])) + self.select_output.append(([mock_stdin_fd, masters[0]], [], [])) + self.select_input.append(([mock_stdin_fd, masters[0]], [mock_stdout_fd, masters[0]], [])) + self.select_output.append(([], [mock_stdout_fd, masters[0]], [])) + self.select_input.append(([mock_stdin_fd, masters[0]], [], [])) with self.assertRaises(IndexError): pty._copy(masters[0]) @@ -410,28 +417,6 @@ def test__copy_to_each(self): self.assertEqual(os.read(read_from_stdout_fd, 20), b'from master') self.assertEqual(os.read(masters[1], 20), b'from stdin') - def test__copy_eof_on_all(self): - """Test the empty read EOF case on both master_fd and stdin.""" - read_from_stdout_fd, mock_stdout_fd = self._pipe() - pty.STDOUT_FILENO = mock_stdout_fd - mock_stdin_fd, write_to_stdin_fd = self._pipe() - pty.STDIN_FILENO = mock_stdin_fd - socketpair = self._socketpair() - masters = [s.fileno() for s in socketpair] - - socketpair[1].close() - os.close(write_to_stdin_fd) - - pty.select = self._mock_select - self.select_rfds_lengths.append(2) - self.select_rfds_results.append([mock_stdin_fd, masters[0]]) - # We expect that both fds were removed from the fds list as they - # both encountered an EOF before the second select call. - self.select_rfds_lengths.append(0) - - # We expect the function to return without error. - self.assertEqual(pty._copy(masters[0]), None) - def test__restore_tty_mode_normal_return(self): """Test that spawn resets the tty mode no when _copy returns normally.""" diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py index 1308c73be6e..f91fa1f8a0f 100644 --- a/Lib/test/test_pulldom.py +++ b/Lib/test/test_pulldom.py @@ -23,8 +23,8 @@ class PullDOMTestCase(unittest.TestCase): - # TODO: RUSTPYTHON FileNotFoundError: [Errno 2] No such file or directory (os error 2): 'xmltestdata/test.xml' -> 'None' - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON; FileNotFoundError: [Errno 2] No such file or directory (os error 2): 'xmltestdata/test.xml' -> 'None' def test_parse(self): """Minimal test of DOMEventStream.parse()""" @@ -41,15 +41,14 @@ def test_parse(self): with open(tstfile, "rb") as fin: list(pulldom.parse(fin)) - # TODO: RUSTPYTHON implement DOM semantic - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; implement DOM semantic def test_parse_semantics(self): """Test DOMEventStream parsing semantics.""" items = pulldom.parseString(SMALL_SAMPLE) evt, node = next(items) # Just check the node is a Document: - self.assertTrue(hasattr(node, "createElement")) + self.assertHasAttr(node, "createElement") self.assertEqual(pulldom.START_DOCUMENT, evt) evt, node = next(items) self.assertEqual(pulldom.START_ELEMENT, evt) @@ -105,8 +104,7 @@ def test_parse_semantics(self): #evt, node = next(items) #self.assertEqual(pulldom.END_DOCUMENT, evt) - # TODO: RUSTPYTHON pulldom.parseString(SMALL_SAMPLE) return iterator with tuple with 2 elements - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; pulldom.parseString(SMALL_SAMPLE) return iterator with tuple with 2 elements def test_expandItem(self): """Ensure expandItem works as expected.""" items = pulldom.parseString(SMALL_SAMPLE) @@ -197,7 +195,7 @@ def _test_thorough(self, pd, before_root=True): evt, node = next(pd) self.assertEqual(pulldom.START_DOCUMENT, evt) # Just check the node is a Document: - self.assertTrue(hasattr(node, "createElement")) + self.assertHasAttr(node, "createElement") if before_root: evt, node = next(pd) @@ -303,8 +301,7 @@ class SAX2DOMTestCase(unittest.TestCase): def confirm(self, test, testname="Test"): self.assertTrue(test, testname) - # TODO: RUSTPYTHON read from stream io - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; read from stream io def test_basic(self): """Ensure SAX2DOM can parse from a stream.""" with io.StringIO(SMALL_SAMPLE) as fin: diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 750afc1de71..54786505d00 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -78,7 +78,6 @@ def test_absolute_path(self): self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_do_not_overwrite_symlinks(self): # In the face of a cfile argument being a symlink, bail out. # Issue #17222 @@ -110,18 +109,17 @@ def test_cwd(self): self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) - import platform - @unittest.expectedFailureIf(sys.platform == "darwin" and int(platform.release().split(".")[0]) < 20, "TODO: RUSTPYTHON") + @unittest.expectedFailureIf(sys.platform == "darwin" and int(__import__("platform").release().split(".")[0]) < 20, "TODO: RUSTPYTHON") def test_relative_path(self): py_compile.compile(os.path.relpath(self.source_path), os.path.relpath(self.pyc_path)) self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) - @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, - 'non-root user required') + @os_helper.skip_if_dac_override @unittest.skipIf(os.name == 'nt', 'cannot control directory permissions on Windows') + @os_helper.skip_unless_working_chmod def test_exceptions_propagate(self): # Make sure that exceptions raised thanks to issues with writing # bytecode. @@ -134,10 +132,11 @@ def test_exceptions_propagate(self): finally: os.chmod(self.directory, mode.st_mode) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_coding(self): - bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py') + bad_coding = os.path.join(os.path.dirname(__file__), + 'tokenizedata', + 'bad_coding2.py') with support.captured_stderr(): self.assertIsNone(py_compile.compile(bad_coding, doraise=False)) self.assertFalse(os.path.exists( @@ -199,6 +198,18 @@ def test_invalidation_mode(self): fp.read(), 'test', {}) self.assertEqual(flags, 0b1) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_quiet(self): + bad_coding = os.path.join(os.path.dirname(__file__), + 'tokenizedata', + 'bad_coding2.py') + with support.captured_stderr() as stderr: + self.assertIsNone(py_compile.compile(bad_coding, doraise=False, quiet=2)) + self.assertIsNone(py_compile.compile(bad_coding, doraise=True, quiet=2)) + self.assertEqual(stderr.getvalue(), '') + with self.assertRaises(py_compile.PyCompileError): + py_compile.compile(bad_coding, doraise=True, quiet=1) + class PyCompileTestsWithSourceEpoch(PyCompileTestsBase, unittest.TestCase, @@ -219,27 +230,31 @@ class PyCompileCLITestCase(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.cache_path = importlib.util.cache_from_source(self.source_path) + self.cache_path = importlib.util.cache_from_source(self.source_path, + optimization='' if __debug__ else 1) with open(self.source_path, 'w') as file: file.write('x = 123\n') def tearDown(self): os_helper.rmtree(self.directory) + @support.requires_subprocess() def pycompilecmd(self, *args, **kwargs): # assert_python_* helpers don't return proc object. We'll just use # subprocess.run() instead of spawn_python() and its friends to test # stdin support of the CLI. + opts = '-m' if __debug__ else '-Om' if args and args[0] == '-' and 'input' in kwargs: - return subprocess.run([sys.executable, '-m', 'py_compile', '-'], + return subprocess.run([sys.executable, opts, 'py_compile', '-'], input=kwargs['input'].encode(), capture_output=True) - return script_helper.assert_python_ok('-m', 'py_compile', *args, **kwargs) + return script_helper.assert_python_ok(opts, 'py_compile', *args, **kwargs) def pycompilecmd_failure(self, *args): return script_helper.assert_python_failure('-m', 'py_compile', *args) def test_stdin(self): + self.assertFalse(os.path.exists(self.cache_path)) result = self.pycompilecmd('-', input=self.source_path) self.assertEqual(result.returncode, 0) self.assertEqual(result.stdout, b'') @@ -254,20 +269,23 @@ def test_with_files(self): self.assertTrue(os.path.exists(self.cache_path)) def test_bad_syntax(self): - bad_syntax = os.path.join(os.path.dirname(__file__), 'badsyntax_3131.py') + bad_syntax = os.path.join(os.path.dirname(__file__), + 'tokenizedata', + 'badsyntax_3131.py') rc, stdout, stderr = self.pycompilecmd_failure(bad_syntax) self.assertEqual(rc, 1) self.assertEqual(stdout, b'') self.assertIn(b'SyntaxError', stderr) def test_bad_syntax_with_quiet(self): - bad_syntax = os.path.join(os.path.dirname(__file__), 'badsyntax_3131.py') + bad_syntax = os.path.join(os.path.dirname(__file__), + 'tokenizedata', + 'badsyntax_3131.py') rc, stdout, stderr = self.pycompilecmd_failure('-q', bad_syntax) self.assertEqual(rc, 1) self.assertEqual(stdout, b'') self.assertEqual(stderr, b'') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_file_not_exists(self): should_not_exists = os.path.join(os.path.dirname(__file__), 'should_not_exists.py') rc, stdout, stderr = self.pycompilecmd_failure(self.source_path, should_not_exists) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py new file mode 100644 index 00000000000..9e7a67ebee5 --- /dev/null +++ b/Lib/test/test_pyclbr.py @@ -0,0 +1,295 @@ +''' + Test cases for pyclbr.py + Nick Mathewson +''' + +import importlib.machinery +import sys +from contextlib import contextmanager +from textwrap import dedent +from types import FunctionType, MethodType, BuiltinFunctionType +import pyclbr +from unittest import TestCase, main as unittest_main +from test.test_importlib import util as test_importlib_util +import warnings + +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + + +StaticMethodType = type(staticmethod(lambda: None)) +ClassMethodType = type(classmethod(lambda c: None)) + +# Here we test the python class browser code. +# +# The main function in this suite, 'testModule', compares the output +# of pyclbr with the introspected members of a module. Because pyclbr +# is imperfect (as designed), testModule is called with a set of +# members to ignore. + + +@contextmanager +def temporary_main_spec(): + """ + A context manager that temporarily sets the `__spec__` attribute + of the `__main__` module if it's missing. + """ + main_mod = sys.modules.get("__main__") + if main_mod is None: + yield # Do nothing if __main__ is not present + return + + original_spec = getattr(main_mod, "__spec__", None) + if original_spec is None: + main_mod.__spec__ = importlib.machinery.ModuleSpec( + name="__main__", loader=None, origin="built-in" + ) + try: + yield + finally: + main_mod.__spec__ = original_spec + + +class PyclbrTest(TestCase): + + def assertListEq(self, l1, l2, ignore): + ''' succeed iff {l1} - {ignore} == {l2} - {ignore} ''' + missing = (set(l1) ^ set(l2)) - set(ignore) + if missing: + print("l1=%r\nl2=%r\nignore=%r" % (l1, l2, ignore), file=sys.stderr) + self.fail("%r missing" % missing.pop()) + + def assertHaskey(self, obj, key, ignore): + ''' succeed iff key in obj or key in ignore. ''' + if key in ignore: return + if key not in obj: + print("***",key, file=sys.stderr) + self.assertIn(key, obj) + + def assertEqualsOrIgnored(self, a, b, ignore): + ''' succeed iff a == b or a in ignore or b in ignore ''' + if a not in ignore and b not in ignore: + self.assertEqual(a, b) + + def checkModule(self, moduleName, module=None, ignore=()): + ''' succeed iff pyclbr.readmodule_ex(modulename) corresponds + to the actual module object, module. Any identifiers in + ignore are ignored. If no module is provided, the appropriate + module is loaded with __import__.''' + + ignore = set(ignore) | set(['object']) + + if module is None: + # Import it. + # ('<silly>' is to work around an API silliness in __import__) + module = __import__(moduleName, globals(), {}, ['<silly>']) + + dict = pyclbr.readmodule_ex(moduleName) + + def ismethod(oclass, obj, name): + classdict = oclass.__dict__ + if isinstance(obj, MethodType): + # could be a classmethod + if (not isinstance(classdict[name], ClassMethodType) or + obj.__self__ is not oclass): + return False + elif not isinstance(obj, FunctionType): + return False + + objname = obj.__name__ + if objname.startswith("__") and not objname.endswith("__"): + if stripped_typename := oclass.__name__.lstrip('_'): + objname = f"_{stripped_typename}{objname}" + return objname == name + + # Make sure the toplevel functions and classes are the same. + for name, value in dict.items(): + if name in ignore: + continue + self.assertHasAttr(module, name) + py_item = getattr(module, name) + if isinstance(value, pyclbr.Function): + self.assertIsInstance(py_item, (FunctionType, BuiltinFunctionType)) + if py_item.__module__ != moduleName: + continue # skip functions that came from somewhere else + self.assertEqual(py_item.__module__, value.module) + else: + self.assertIsInstance(py_item, type) + if py_item.__module__ != moduleName: + continue # skip classes that came from somewhere else + + real_bases = [base.__name__ for base in py_item.__bases__] + pyclbr_bases = [ getattr(base, 'name', base) + for base in value.super ] + + try: + self.assertListEq(real_bases, pyclbr_bases, ignore) + except: + print("class=%s" % py_item, file=sys.stderr) + raise + + actualMethods = [] + for m in py_item.__dict__.keys(): + if m == "__annotate__": + continue + if ismethod(py_item, getattr(py_item, m), m): + actualMethods.append(m) + + if stripped_typename := name.lstrip('_'): + foundMethods = [] + for m in value.methods.keys(): + if m.startswith('__') and not m.endswith('__'): + foundMethods.append(f"_{stripped_typename}{m}") + else: + foundMethods.append(m) + else: + foundMethods = list(value.methods.keys()) + + try: + self.assertListEq(foundMethods, actualMethods, ignore) + self.assertEqual(py_item.__module__, value.module) + + self.assertEqualsOrIgnored(py_item.__name__, value.name, + ignore) + # can't check file or lineno + except: + print("class=%s" % py_item, file=sys.stderr) + raise + + # Now check for missing stuff. + def defined_in(item, module): + if isinstance(item, type): + return item.__module__ == module.__name__ + if isinstance(item, FunctionType): + return item.__globals__ is module.__dict__ + return False + for name in dir(module): + item = getattr(module, name) + if isinstance(item, (type, FunctionType)): + if defined_in(item, module): + self.assertHaskey(dict, name, ignore) + + def test_easy(self): + self.checkModule('pyclbr') + # XXX: Metaclasses are not supported + # self.checkModule('ast') + with temporary_main_spec(): + self.checkModule('doctest', ignore=("TestResults", "_SpoofOut", + "DocTestCase", '_DocTestSuite')) + self.checkModule('difflib', ignore=("Match",)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_cases(self): + # see test.pyclbr_input for the rationale behind the ignored symbols + self.checkModule('test.pyclbr_input', ignore=['om', 'f']) + + def test_nested(self): + mb = pyclbr + # Set arguments for descriptor creation and _creat_tree call. + m, p, f, t, i = 'test', '', 'test.py', {}, None + source = dedent("""\ + def f0(): + def f1(a,b,c): + def f2(a=1, b=2, c=3): pass + return f1(a,b,d) + class c1: pass + class C0: + "Test class." + def F1(): + "Method." + return 'return' + class C1(): + class C2: + "Class nested within nested class." + def F3(): return 1+1 + + """) + actual = mb._create_tree(m, p, f, source, t, i) + + # Create descriptors, linked together, and expected dict. + f0 = mb.Function(m, 'f0', f, 1, end_lineno=5) + f1 = mb._nest_function(f0, 'f1', 2, 4) + f2 = mb._nest_function(f1, 'f2', 3, 3) + c1 = mb._nest_class(f0, 'c1', 5, 5) + C0 = mb.Class(m, 'C0', None, f, 6, end_lineno=14) + F1 = mb._nest_function(C0, 'F1', 8, 10) + C1 = mb._nest_class(C0, 'C1', 11, 14) + C2 = mb._nest_class(C1, 'C2', 12, 14) + F3 = mb._nest_function(C2, 'F3', 14, 14) + expected = {'f0':f0, 'C0':C0} + + def compare(parent1, children1, parent2, children2): + """Return equality of tree pairs. + + Each parent,children pair define a tree. The parents are + assumed equal. Comparing the children dictionaries as such + does not work due to comparison by identity and double + linkage. We separate comparing string and number attributes + from comparing the children of input children. + """ + self.assertEqual(children1.keys(), children2.keys()) + for ob in children1.values(): + self.assertIs(ob.parent, parent1) + for ob in children2.values(): + self.assertIs(ob.parent, parent2) + for key in children1.keys(): + o1, o2 = children1[key], children2[key] + t1 = type(o1), o1.name, o1.file, o1.module, o1.lineno, o1.end_lineno + t2 = type(o2), o2.name, o2.file, o2.module, o2.lineno, o2.end_lineno + self.assertEqual(t1, t2) + if type(o1) is mb.Class: + self.assertEqual(o1.methods, o2.methods) + # Skip superclasses for now as not part of example + compare(o1, o1.children, o2, o2.children) + + compare(None, actual, None, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_others(self): + cm = self.checkModule + + # These were once some of the longest modules. + cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator + cm('pickle', ignore=('partial', 'PickleBuffer')) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + cm('sre_parse', ignore=('dump', 'groups', 'pos')) # from sre_constants import *; property + with temporary_main_spec(): + cm( + 'pdb', + # pyclbr does not handle elegantly `typing` or properties + ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget', 'curframe_locals', + '_InteractState', 'rlcompleter'), + ) + cm('pydoc', ignore=('input', 'output',)) # properties + + # Tests for modules inside packages + cm('email.parser') + cm('test.test_pyclbr') + + +class ReadmoduleTests(TestCase): + + def setUp(self): + self._modules = pyclbr._modules.copy() + + def tearDown(self): + pyclbr._modules = self._modules + + + def test_dotted_name_not_a_package(self): + # test ImportError is raised when the first part of a dotted name is + # not a package. + # + # Issue #14798. + self.assertRaises(ImportError, pyclbr.readmodule_ex, 'asyncio.foo') + + def test_module_has_no_spec(self): + module_name = "doesnotexist" + assert module_name not in pyclbr._modules + with test_importlib_util.uncache(module_name): + with self.assertRaises(ModuleNotFoundError): + pyclbr.readmodule_ex(module_name) + + +if __name__ == "__main__": + unittest_main() diff --git a/Lib/test/test_pydoc/__init__.py b/Lib/test/test_pydoc/__init__.py new file mode 100644 index 00000000000..f2a39a3fe29 --- /dev/null +++ b/Lib/test/test_pydoc/__init__.py @@ -0,0 +1,6 @@ +import os +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pydoc/module_none.py b/Lib/test/test_pydoc/module_none.py new file mode 100644 index 00000000000..ebb50fc86e2 --- /dev/null +++ b/Lib/test/test_pydoc/module_none.py @@ -0,0 +1,8 @@ +def func(): + pass +func.__module__ = None + +class A: + def method(self): + pass + method.__module__ = None diff --git a/Lib/test/test_pydoc/pydoc_mod.py b/Lib/test/test_pydoc/pydoc_mod.py new file mode 100644 index 00000000000..80c287fb10c --- /dev/null +++ b/Lib/test/test_pydoc/pydoc_mod.py @@ -0,0 +1,51 @@ +"""This is a test module for test_pydoc""" + +from __future__ import print_function + +import types +import typing + +__author__ = "Benjamin Peterson" +__credits__ = "Nobody" +__version__ = "1.2.3.4" +__xyz__ = "X, Y and Z" + +class A: + """Hello and goodbye""" + def __init__(): + """Wow, I have no function!""" + pass + +class B(object): + NO_MEANING: str = "eggs" + pass + +class C(object): + def say_no(self): + return "no" + def get_answer(self): + """ Return say_no() """ + return self.say_no() + def is_it_true(self): + """ Return self.get_answer() """ + return self.get_answer() + def __class_getitem__(self, item): + return types.GenericAlias(self, item) + +def doc_func(): + """ + This function solves all of the world's problems: + hunger + lack of Python + war + """ + +def nodoc_func(): + pass + + +list_alias1 = typing.List[int] +list_alias2 = list[int] +c_alias = C[int] +type_union1 = typing.Union[int, str] +type_union2 = int | str diff --git a/Lib/test/test_pydoc/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py new file mode 100644 index 00000000000..412aa3743e4 --- /dev/null +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -0,0 +1,191 @@ +"""Something just to look at via pydoc.""" + +import types + +def global_func(x, y): + """Module global function""" + +def global_func2(x, y): + """Module global function 2""" + +class A: + "A class." + + def A_method(self): + "Method defined in A." + def AB_method(self): + "Method defined in A and B." + def AC_method(self): + "Method defined in A and C." + def AD_method(self): + "Method defined in A and D." + def ABC_method(self): + "Method defined in A, B and C." + def ABD_method(self): + "Method defined in A, B and D." + def ACD_method(self): + "Method defined in A, C and D." + def ABCD_method(self): + "Method defined in A, B, C and D." + + def A_classmethod(cls, x): + "A class method defined in A." + A_classmethod = classmethod(A_classmethod) + + def A_staticmethod(x, y): + "A static method defined in A." + A_staticmethod = staticmethod(A_staticmethod) + + def _getx(self): + "A property getter function." + def _setx(self, value): + "A property setter function." + def _delx(self): + "A property deleter function." + A_property = property(fdel=_delx, fget=_getx, fset=_setx, + doc="A sample property defined in A.") + + A_int_alias = int + +class B(A): + "A class, derived from A." + + def AB_method(self): + "Method defined in A and B." + def ABC_method(self): + "Method defined in A, B and C." + def ABD_method(self): + "Method defined in A, B and D." + def ABCD_method(self): + "Method defined in A, B, C and D." + def B_method(self): + "Method defined in B." + def BC_method(self): + "Method defined in B and C." + def BD_method(self): + "Method defined in B and D." + def BCD_method(self): + "Method defined in B, C and D." + + @classmethod + def B_classmethod(cls, x): + "A class method defined in B." + + global_func = global_func # same name + global_func_alias = global_func + global_func2_alias = global_func2 + B_classmethod_alias = B_classmethod + A_classmethod_ref = A.A_classmethod + A_staticmethod = A.A_staticmethod # same name + A_staticmethod_alias = A.A_staticmethod + A_method_ref = A().A_method + A_method_alias = A.A_method + B_method_alias = B_method + count = list.count # same name + list_count = list.count + __repr__ = object.__repr__ # same name + object_repr = object.__repr__ + get = {}.get # same name + dict_get = {}.get + from math import sin + + +B.B_classmethod_ref = B.B_classmethod + + +class C(A): + "A class, derived from A." + + def AC_method(self): + "Method defined in A and C." + def ABC_method(self): + "Method defined in A, B and C." + def ACD_method(self): + "Method defined in A, C and D." + def ABCD_method(self): + "Method defined in A, B, C and D." + def BC_method(self): + "Method defined in B and C." + def BCD_method(self): + "Method defined in B, C and D." + def C_method(self): + "Method defined in C." + def CD_method(self): + "Method defined in C and D." + +class D(B, C): + """A class, derived from B and C. + """ + + def AD_method(self): + "Method defined in A and D." + def ABD_method(self): + "Method defined in A, B and D." + def ACD_method(self): + "Method defined in A, C and D." + def ABCD_method(self): + "Method defined in A, B, C and D." + def BD_method(self): + "Method defined in B and D." + def BCD_method(self): + "Method defined in B, C and D." + def CD_method(self): + "Method defined in C and D." + def D_method(self): + "Method defined in D." + +class FunkyProperties(object): + """From SF bug 472347, by Roeland Rengelink. + + Property getters etc may not be vanilla functions or methods, + and this used to make GUI pydoc blow up. + """ + + def __init__(self): + self.desc = {'x':0} + + class get_desc: + def __init__(self, attr): + self.attr = attr + def __call__(self, inst): + print('Get called', self, inst) + return inst.desc[self.attr] + class set_desc: + def __init__(self, attr): + self.attr = attr + def __call__(self, inst, val): + print('Set called', self, inst, val) + inst.desc[self.attr] = val + class del_desc: + def __init__(self, attr): + self.attr = attr + def __call__(self, inst): + print('Del called', self, inst) + del inst.desc[self.attr] + + x = property(get_desc('x'), set_desc('x'), del_desc('x'), 'prop x') + + +submodule = types.ModuleType(__name__ + '.submodule', + """A submodule, which should appear in its parent's summary""") + +global_func_alias = global_func +A_classmethod = A.A_classmethod # same name +A_classmethod2 = A.A_classmethod +A_classmethod3 = B.A_classmethod +A_staticmethod = A.A_staticmethod # same name +A_staticmethod_alias = A.A_staticmethod +A_staticmethod_ref = A().A_staticmethod +A_staticmethod_ref2 = B().A_staticmethod +A_method = A().A_method # same name +A_method2 = A().A_method +A_method3 = B().A_method +B_method = B.B_method # same name +B_method2 = B.B_method +count = list.count # same name +list_count = list.count +__repr__ = object.__repr__ # same name +object_repr = object.__repr__ +get = {}.get # same name +dict_get = {}.get +from math import sin # noqa: F401 diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py new file mode 100644 index 00000000000..066b9b4dbe1 --- /dev/null +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -0,0 +1,2458 @@ +import datetime +import os +import sys +import contextlib +import importlib.util +import inspect +import io +import pydoc +import py_compile +import keyword +try: # TODO: RUSTPYTHON; Implement `_pickle` + import _pickle +except ImportError: + _pickle = None +import pkgutil +import re +import tempfile +import test.support +import time +import types +import typing +import unittest +import unittest.mock +import urllib.parse +import xml.etree +import xml.etree.ElementTree +import textwrap +from io import StringIO +from collections import namedtuple +from urllib.request import urlopen, urlcleanup +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support.script_helper import (assert_python_ok, + assert_python_failure, spawn_python) +from test.support import threading_helper +from test.support import (reap_children, captured_stdout, + captured_stderr, is_wasm32, + requires_docstrings, MISSING_C_DOCSTRINGS) +from test.support.os_helper import (TESTFN, rmtree, unlink) +from test.test_pydoc import pydoc_mod +from test.test_pydoc import pydocfodder + + +class nonascii: + 'Це не латиниця' + pass + +if test.support.HAVE_DOCSTRINGS: + expected_data_docstrings = ( + 'dictionary for instance variables', + 'list of weak references to the object', + ) * 2 +else: + expected_data_docstrings = ('', '', '', '') + +expected_text_pattern = """ +NAME + test.test_pydoc.pydoc_mod - This is a test module for test_pydoc +%s +CLASSES + builtins.object + A + B + C + + class A(builtins.object) + | Hello and goodbye + | + | Methods defined here: + | + | __init__() + | Wow, I have no function! + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s + + class B(builtins.object) + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s + | + | ---------------------------------------------------------------------- + | Data and other attributes defined here: + | + | NO_MEANING = 'eggs' + + class C(builtins.object) + | Methods defined here: + | + | get_answer(self) + | Return say_no() + | + | is_it_true(self) + | Return self.get_answer() + | + | say_no(self) + | + | ---------------------------------------------------------------------- + | Class methods defined here: + | + | __class_getitem__(item) + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__ + | dictionary for instance variables + | + | __weakref__ + | list of weak references to the object + +FUNCTIONS + doc_func() + This function solves all of the world's problems: + hunger + lack of Python + war + + nodoc_func() + +DATA + __xyz__ = 'X, Y and Z' + c_alias = test.test_pydoc.pydoc_mod.C[int] + list_alias1 = typing.List[int] + list_alias2 = list[int] + type_union1 = int | str + type_union2 = int | str + +VERSION + 1.2.3.4 + +AUTHOR + Benjamin Peterson + +CREDITS + Nobody + +FILE + %s +""".strip() + +expected_text_data_docstrings = tuple('\n | ' + s if s else '' + for s in expected_data_docstrings) + +html2text_of_expected = """ +test.test_pydoc.pydoc_mod (version 1.2.3.4) +This is a test module for test_pydoc + +Modules + types + typing + +Classes + builtins.object + A + B + C + +class A(builtins.object) + Hello and goodbye + + Methods defined here: + __init__() + Wow, I have no function! + ---------------------------------------------------------------------- + Data descriptors defined here: + __dict__ + dictionary for instance variables + __weakref__ + list of weak references to the object + +class B(builtins.object) + Data descriptors defined here: + __dict__ + dictionary for instance variables + __weakref__ + list of weak references to the object + ---------------------------------------------------------------------- + Data and other attributes defined here: + NO_MEANING = 'eggs' + + +class C(builtins.object) + Methods defined here: + get_answer(self) + Return say_no() + is_it_true(self) + Return self.get_answer() + say_no(self) + ---------------------------------------------------------------------- + Class methods defined here: + __class_getitem__(item) + ---------------------------------------------------------------------- + Data descriptors defined here: + __dict__ + dictionary for instance variables + __weakref__ + list of weak references to the object + +Functions + doc_func() + This function solves all of the world's problems: + hunger + lack of Python + war + nodoc_func() + +Data + __xyz__ = 'X, Y and Z' + c_alias = test.test_pydoc.pydoc_mod.C[int] + list_alias1 = typing.List[int] + list_alias2 = list[int] + type_union1 = int | str + type_union2 = int | str + +Author + Benjamin Peterson + +Credits + Nobody +""" + +expected_html_data_docstrings = tuple(s.replace(' ', '&nbsp;') + for s in expected_data_docstrings) + +# output pattern for missing module +missing_pattern = '''\ +No Python documentation found for %r. +Use help() to get the interactive help utility. +Use help(str) for help on the str class.'''.replace('\n', os.linesep) + +# output pattern for module with bad imports +badimport_pattern = "problem in %s - ModuleNotFoundError: No module named %r" + +expected_dynamicattribute_pattern = """ +Help on class DA in module %s: + +class DA(builtins.object) + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s + | + | ham + | + | ---------------------------------------------------------------------- + | Data and other attributes inherited from Meta: + | + | ham = 'spam' +""".strip() + +expected_virtualattribute_pattern1 = """ +Help on class Class in module %s: + +class Class(builtins.object) + | Data and other attributes inherited from Meta: + | + | LIFE = 42 +""".strip() + +expected_virtualattribute_pattern2 = """ +Help on class Class1 in module %s: + +class Class1(builtins.object) + | Data and other attributes inherited from Meta1: + | + | one = 1 +""".strip() + +expected_virtualattribute_pattern3 = """ +Help on class Class2 in module %s: + +class Class2(Class1) + | Method resolution order: + | Class2 + | Class1 + | builtins.object + | + | Data and other attributes inherited from Meta1: + | + | one = 1 + | + | ---------------------------------------------------------------------- + | Data and other attributes inherited from Meta3: + | + | three = 3 + | + | ---------------------------------------------------------------------- + | Data and other attributes inherited from Meta2: + | + | two = 2 +""".strip() + +expected_missingattribute_pattern = """ +Help on class C in module %s: + +class C(builtins.object) + | Data and other attributes defined here: + | + | here = 'present!' +""".strip() + +def run_pydoc(module_name, *args, **env): + """ + Runs pydoc on the specified module. Returns the stripped + output of pydoc. + """ + args = args + (module_name,) + # do not write bytecode files to avoid caching errors + rc, out, err = assert_python_ok('-B', pydoc.__file__, *args, **env) + return out.strip() + +def run_pydoc_fail(module_name, *args, **env): + """ + Runs pydoc on the specified module expecting a failure. + """ + args = args + (module_name,) + rc, out, err = assert_python_failure('-B', pydoc.__file__, *args, **env) + return out.strip() + +def get_pydoc_html(module): + "Returns pydoc generated output as html" + doc = pydoc.HTMLDoc() + output = doc.docmodule(module) + loc = doc.getdocloc(pydoc_mod) or "" + if loc: + loc = "<br><a href=\"" + loc + "\">Module Docs</a>" + return output.strip(), loc + +def clean_text(doc): + # clean up the extra text formatting that pydoc performs + return re.sub('\b.', '', doc) + +def get_pydoc_link(module): + "Returns a documentation web link of a module" + abspath = os.path.abspath + dirname = os.path.dirname + basedir = dirname(dirname(dirname(abspath(__file__)))) + doc = pydoc.TextDoc() + loc = doc.getdocloc(module, basedir=basedir) + return loc + +def get_pydoc_text(module): + "Returns pydoc generated output as text" + doc = pydoc.TextDoc() + loc = doc.getdocloc(pydoc_mod) or "" + if loc: + loc = "\nMODULE DOCS\n " + loc + "\n" + + output = doc.docmodule(module) + output = clean_text(output) + return output.strip(), loc + +def get_html_title(text): + # Bit of hack, but good enough for test purposes + header, _, _ = text.partition("</head>") + _, _, title = header.partition("<title>") + title, _, _ = title.partition("</title>") + return title + + +def html2text(html): + """A quick and dirty implementation of html2text. + + Tailored for pydoc tests only. + """ + html = html.replace("<dd>", "\n") + html = html.replace("<hr>", "-"*70) + html = re.sub("<.*?>", "", html) + html = pydoc.replace(html, "&nbsp;", " ", "&gt;", ">", "&lt;", "<") + return html + + +class PydocBaseTest(unittest.TestCase): + def tearDown(self): + # Self-testing. Mocking only works if sys.modules['pydoc'] and pydoc + # are the same. But some pydoc functions reload the module and change + # sys.modules, so check that it was restored. + self.assertIs(sys.modules['pydoc'], pydoc) + + def _restricted_walk_packages(self, walk_packages, path=None): + """ + A version of pkgutil.walk_packages() that will restrict itself to + a given path. + """ + default_path = path or [os.path.dirname(__file__)] + def wrapper(path=None, prefix='', onerror=None): + return walk_packages(path or default_path, prefix, onerror) + return wrapper + + @contextlib.contextmanager + def restrict_walk_packages(self, path=None): + walk_packages = pkgutil.walk_packages + pkgutil.walk_packages = self._restricted_walk_packages(walk_packages, + path) + try: + yield + finally: + pkgutil.walk_packages = walk_packages + + def call_url_handler(self, url, expected_title): + text = pydoc._url_handler(url, "text/html") + result = get_html_title(text) + # Check the title to ensure an unexpected error page was not returned + self.assertEqual(result, expected_title, text) + return text + + +class PydocDocTest(unittest.TestCase): + maxDiff = None + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_html_doc(self): + result, doc_loc = get_pydoc_html(pydoc_mod) + text_result = html2text(result) + text_lines = [line.strip() for line in text_result.splitlines()] + text_lines = [line for line in text_lines if line] + del text_lines[1] + expected_lines = html2text_of_expected.splitlines() + expected_lines = [line.strip() for line in expected_lines if line] + self.assertEqual(text_lines, expected_lines) + mod_file = inspect.getabsfile(pydoc_mod) + mod_url = urllib.parse.quote(mod_file) + self.assertIn(mod_url, result) + self.assertIn(mod_file, result) + self.assertIn(doc_loc, result) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_text_doc(self): + result, doc_loc = get_pydoc_text(pydoc_mod) + expected_text = expected_text_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual(expected_text, result) + + def test_text_enum_member_with_value_zero(self): + # Test issue #20654 to ensure enum member with value 0 can be + # displayed. It used to throw KeyError: 'zero'. + import enum + class BinaryInteger(enum.IntEnum): + zero = 0 + one = 1 + doc = pydoc.render_doc(BinaryInteger) + self.assertIn('BinaryInteger.zero', doc) + + def test_slotted_dataclass_with_field_docs(self): + import dataclasses + @dataclasses.dataclass(slots=True) + class My: + x: int = dataclasses.field(doc='Docstring for x') + doc = pydoc.render_doc(My) + self.assertIn('Docstring for x', doc) + + def test_mixed_case_module_names_are_lower_cased(self): + # issue16484 + doc_link = get_pydoc_link(xml.etree.ElementTree) + self.assertIn('xml.etree.elementtree', doc_link) + + def test_issue8225(self): + # Test issue8225 to ensure no doc link appears for xml.etree + result, doc_loc = get_pydoc_text(xml.etree) + self.assertEqual(doc_loc, "", "MODULE DOCS incorrectly includes a link") + + def test_online_docs_link(self): + import encodings.idna + import importlib._bootstrap + + module_docs = { + 'encodings': 'codecs#module-encodings', + 'encodings.idna': 'codecs#module-encodings.idna', + } + + with unittest.mock.patch('pydoc_data.module_docs.module_docs', module_docs): + doc = pydoc.TextDoc() + + basedir = os.path.dirname(encodings.__file__) + doc_link = doc.getdocloc(encodings, basedir=basedir) + self.assertIsNotNone(doc_link) + self.assertIn('codecs#module-encodings', doc_link) + self.assertNotIn('encodings.html', doc_link) + + doc_link = doc.getdocloc(encodings.idna, basedir=basedir) + self.assertIsNotNone(doc_link) + self.assertIn('codecs#module-encodings.idna', doc_link) + self.assertNotIn('encodings.idna.html', doc_link) + + doc_link = doc.getdocloc(importlib._bootstrap, basedir=basedir) + self.assertIsNone(doc_link) + + def test_getpager_with_stdin_none(self): + previous_stdin = sys.stdin + try: + sys.stdin = None + pydoc.getpager() # Shouldn't fail. + finally: + sys.stdin = previous_stdin + + def test_non_str_name(self): + # issue14638 + # Treat illegal (non-str) name like no name + + class A: + __name__ = 42 + class B: + pass + adoc = pydoc.render_doc(A()) + bdoc = pydoc.render_doc(B()) + self.assertEqual(adoc.replace("A", "B"), bdoc) + + def test_not_here(self): + missing_module = "test.i_am_not_here" + result = str(run_pydoc_fail(missing_module), 'ascii') + expected = missing_pattern % missing_module + self.assertEqual(expected, result, + "documentation for missing module found") + + @requires_docstrings + def test_not_ascii(self): + result = run_pydoc('test.test_pydoc.test_pydoc.nonascii', PYTHONIOENCODING='ascii') + encoded = nonascii.__doc__.encode('ascii', 'backslashreplace') + self.assertIn(encoded, result) + + def test_input_strip(self): + missing_module = " test.i_am_not_here " + result = str(run_pydoc_fail(missing_module), 'ascii') + expected = missing_pattern % missing_module.strip() + self.assertEqual(expected, result) + + def test_stripid(self): + # test with strings, other implementations might have different repr() + stripid = pydoc.stripid + # strip the id + self.assertEqual(stripid('<function stripid at 0x88dcee4>'), + '<function stripid>') + self.assertEqual(stripid('<function stripid at 0x01F65390>'), + '<function stripid>') + # nothing to strip, return the same text + self.assertEqual(stripid('42'), '42') + self.assertEqual(stripid("<type 'exceptions.Exception'>"), + "<type 'exceptions.Exception'>") + + @unittest.skip("TODO: RUSTPYTHON; Panic") + def test_builtin_with_more_than_four_children(self): + """Tests help on builtin object which have more than four child classes. + + When running help() on a builtin class which has child classes, it + should contain a "Built-in subclasses" section and only 4 classes + should be displayed with a hint on how many more subclasses are present. + For example: + + >>> help(object) + Help on class object in module builtins: + + class object + | The most base type + | + | Built-in subclasses: + | async_generator + | BaseException + | builtin_function_or_method + | bytearray + | ... and 82 other subclasses + """ + doc = pydoc.TextDoc() + try: + # Make sure HeapType, which has no __module__ attribute, is one + # of the known subclasses of object. (doc.docclass() used to + # fail if HeapType was imported before running this test, like + # when running tests sequentially.) + from _testcapi import HeapType + except ImportError: + pass + text = doc.docclass(object) + snip = (" | Built-in subclasses:\n" + " | async_generator\n" + " | BaseException\n" + " | builtin_function_or_method\n" + " | bytearray\n" + " | ... and \\d+ other subclasses") + self.assertRegex(text, snip) + + def test_builtin_with_child(self): + """Tests help on builtin object which have only child classes. + + When running help() on a builtin class which has child classes, it + should contain a "Built-in subclasses" section. For example: + + >>> help(ArithmeticError) + Help on class ArithmeticError in module builtins: + + class ArithmeticError(Exception) + | Base class for arithmetic errors. + | + ... + | + | Built-in subclasses: + | FloatingPointError + | OverflowError + | ZeroDivisionError + """ + doc = pydoc.TextDoc() + text = doc.docclass(ArithmeticError) + snip = (" | Built-in subclasses:\n" + " | FloatingPointError\n" + " | OverflowError\n" + " | ZeroDivisionError") + self.assertIn(snip, text) + + def test_builtin_with_grandchild(self): + """Tests help on builtin classes which have grandchild classes. + + When running help() on a builtin class which has child classes, it + should contain a "Built-in subclasses" section. However, if it also has + grandchildren, these should not show up on the subclasses section. + For example: + + >>> help(Exception) + Help on class Exception in module builtins: + + class Exception(BaseException) + | Common base class for all non-exit exceptions. + | + ... + | + | Built-in subclasses: + | ArithmeticError + | AssertionError + | AttributeError + ... + """ + doc = pydoc.TextDoc() + text = doc.docclass(Exception) + snip = (" | Built-in subclasses:\n" + " | ArithmeticError\n" + " | AssertionError\n" + " | AttributeError") + self.assertIn(snip, text) + # Testing that the grandchild ZeroDivisionError does not show up + self.assertNotIn('ZeroDivisionError', text) + + def test_builtin_no_child(self): + """Tests help on builtin object which have no child classes. + + When running help() on a builtin class which has no child classes, it + should not contain any "Built-in subclasses" section. For example: + + >>> help(ZeroDivisionError) + + Help on class ZeroDivisionError in module builtins: + + class ZeroDivisionError(ArithmeticError) + | Second argument to a division or modulo operation was zero. + | + | Method resolution order: + | ZeroDivisionError + | ArithmeticError + | Exception + | BaseException + | object + | + | Methods defined here: + ... + """ + doc = pydoc.TextDoc() + text = doc.docclass(ZeroDivisionError) + # Testing that the subclasses section does not appear + self.assertNotIn('Built-in subclasses', text) + + def test_builtin_on_metaclasses(self): + """Tests help on metaclasses. + + When running help() on a metaclasses such as type, it + should not contain any "Built-in subclasses" section. + """ + doc = pydoc.TextDoc() + text = doc.docclass(type) + # Testing that the subclasses section does not appear + self.assertNotIn('Built-in subclasses', text) + + def test_fail_help_cli(self): + elines = (missing_pattern % 'abd').splitlines() + with spawn_python("-c" "help()") as proc: + out, _ = proc.communicate(b"abd") + olines = out.decode().splitlines()[-9:-6] + olines[0] = olines[0].removeprefix('help> ') + self.assertEqual(elines, olines) + + def test_fail_help_output_redirect(self): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.help("abd") + expected = missing_pattern % "abd" + self.assertEqual(expected, buf.getvalue().strip().replace('\n', os.linesep)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @unittest.mock.patch('pydoc.pager') + @requires_docstrings + def test_help_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + self.maxDiff = None + + unused, doc_loc = get_pydoc_text(pydoc_mod) + module = "test.test_pydoc.pydoc_mod" + help_header = """ + Help on module test.test_pydoc.pydoc_mod in test.test_pydoc: + + """.lstrip() + help_header = textwrap.dedent(help_header) + expected_help_pattern = help_header + expected_text_pattern + + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(module) + result = buf.getvalue().strip() + expected_text = expected_help_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertEqual(expected_text, result) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_help_output_redirect_various_requests(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + + def run_pydoc_for_request(request, expected_text_part): + """Helper function to run pydoc with its output redirected""" + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.help(request) + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue(), msg=f'failed on request "{request}"') + self.assertEqual('', err.getvalue(), msg=f'failed on request "{request}"') + self.assertIn(expected_text_part, result, msg=f'failed on request "{request}"') + pager_mock.assert_not_called() + + self.maxDiff = None + + # test for "keywords" + run_pydoc_for_request('keywords', 'Here is a list of the Python keywords.') + # test for "symbols" + run_pydoc_for_request('symbols', 'Here is a list of the punctuation symbols') + # test for "topics" + run_pydoc_for_request('topics', 'Here is a list of available topics.') + # test for "modules" skipped, see test_modules() + # test for symbol "%" + run_pydoc_for_request('%', 'The power operator') + # test for special True, False, None keywords + run_pydoc_for_request('True', 'class bool(int)') + run_pydoc_for_request('False', 'class bool(int)') + run_pydoc_for_request('None', 'class NoneType(object)') + # test for keyword "assert" + run_pydoc_for_request('assert', 'The "assert" statement') + # test for topic "TYPES" + run_pydoc_for_request('TYPES', 'The standard type hierarchy') + # test for "pydoc.Helper.help" + run_pydoc_for_request('pydoc.Helper.help', 'Help on function help in pydoc.Helper:') + # test for pydoc.Helper.help + run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') + # test for pydoc.Helper() instance skipped because it is always meant to be interactive + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_help_output_pager(self): + def run_pydoc_pager(request, what, expected_first_line): + with (captured_stdout() as output, + captured_stderr() as err, + unittest.mock.patch('pydoc.pager') as pager_mock, + self.subTest(repr(request))): + helper = pydoc.Helper() + helper.help(request) + self.assertEqual('', err.getvalue()) + self.assertEqual('\n', output.getvalue()) + pager_mock.assert_called_once() + result = clean_text(pager_mock.call_args.args[0]) + self.assertEqual(result.splitlines()[0], expected_first_line) + self.assertEqual(pager_mock.call_args.args[1], f'Help on {what}') + + run_pydoc_pager('%', 'EXPRESSIONS', 'Operator precedence') + run_pydoc_pager('True', 'bool object', 'Help on bool object:') + run_pydoc_pager(True, 'bool object', 'Help on bool object:') + run_pydoc_pager('assert', 'assert', 'The "assert" statement') + run_pydoc_pager('TYPES', 'TYPES', 'The standard type hierarchy') + run_pydoc_pager('pydoc.Helper.help', 'pydoc.Helper.help', + 'Help on function help in pydoc.Helper:') + run_pydoc_pager(pydoc.Helper.help, 'Helper.help', + 'Help on function help in module pydoc:') + run_pydoc_pager('str', 'str', 'Help on class str in module builtins:') + run_pydoc_pager(str, 'str', 'Help on class str in module builtins:') + run_pydoc_pager('str.upper', 'str.upper', + 'Help on method descriptor upper in str:') + run_pydoc_pager(str.upper, 'str.upper', + 'Help on method descriptor upper:') + run_pydoc_pager(''.upper, 'str.upper', + 'Help on built-in function upper:') + run_pydoc_pager(str.__add__, + 'str.__add__', 'Help on method descriptor __add__:') + run_pydoc_pager(''.__add__, + 'str.__add__', 'Help on method wrapper __add__:') + run_pydoc_pager(int.numerator, 'int.numerator', + 'Help on getset descriptor builtins.int.numerator:') + run_pydoc_pager(list[int], 'list', + 'Help on GenericAlias in module builtins:') + run_pydoc_pager('sys', 'sys', 'Help on built-in module sys:') + run_pydoc_pager(sys, 'sys', 'Help on built-in module sys:') + + def test_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('with') + helptext = showtopic_io.getvalue() + self.assertIn('The "with" statement', helptext) + + def test_fail_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('abd') + expected = "no documentation found for 'abd'" + self.assertEqual(expected, showtopic_io.getvalue().strip()) + + @unittest.mock.patch('pydoc.pager') + def test_fail_showtopic_output_redirect(self, pager_mock): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic("abd") + expected = "no documentation found for 'abd'" + self.assertEqual(expected, buf.getvalue().strip()) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_showtopic_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.showtopic should be redirected + self.maxDiff = None + + with captured_stdout() as output, captured_stderr() as err: + buf = StringIO() + helper = pydoc.Helper(output=buf) + helper.showtopic('with') + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertIn('The "with" statement', result) + + pager_mock.assert_not_called() + + def test_lambda_with_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"return": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a, b, c) -> int", helptext) + + def test_lambda_without_return_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int)", helptext) + + def test_lambda_with_return_and_params_annotation(self): + func = lambda a, b, c: 1 + func.__annotations__ = {"a": int, "b": int, "c": int, "return": int} + with captured_stdout() as help_io: + pydoc.help(func) + helptext = help_io.getvalue() + self.assertIn("lambda (a: int, b: int, c: int) -> int", helptext) + + def test_namedtuple_fields(self): + Person = namedtuple('Person', ['nickname', 'firstname']) + with captured_stdout() as help_io: + pydoc.help(Person) + helptext = help_io.getvalue() + self.assertIn("nickname", helptext) + self.assertIn("firstname", helptext) + self.assertIn("Alias for field number 0", helptext) + self.assertIn("Alias for field number 1", helptext) + + def test_namedtuple_public_underscore(self): + NT = namedtuple('NT', ['abc', 'def'], rename=True) + with captured_stdout() as help_io: + pydoc.help(NT) + helptext = help_io.getvalue() + self.assertIn('_1', helptext) + self.assertIn('_replace', helptext) + self.assertIn('_asdict', helptext) + + def test_synopsis(self): + self.addCleanup(unlink, TESTFN) + for encoding in ('ISO-8859-1', 'UTF-8'): + with open(TESTFN, 'w', encoding=encoding) as script: + if encoding != 'UTF-8': + print('#coding: {}'.format(encoding), file=script) + print('"""line 1: h\xe9', file=script) + print('line 2: hi"""', file=script) + synopsis = pydoc.synopsis(TESTFN, {}) + self.assertEqual(synopsis, 'line 1: h\xe9') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_source_synopsis(self): + def check(source, expected, encoding=None): + if isinstance(source, str): + source_file = StringIO(source) + else: + source_file = io.TextIOWrapper(io.BytesIO(source), encoding=encoding) + with source_file: + result = pydoc.source_synopsis(source_file) + self.assertEqual(result, expected) + + check('"""Single line docstring."""', + 'Single line docstring.') + check('"""First line of docstring.\nSecond line.\nThird line."""', + 'First line of docstring.') + check('"""First line of docstring.\\nSecond line.\\nThird line."""', + 'First line of docstring.') + check('""" Whitespace around docstring. """', + 'Whitespace around docstring.') + check('import sys\n"""No docstring"""', + None) + check(' \n"""Docstring after empty line."""', + 'Docstring after empty line.') + check('# Comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check(' # Indented comment\n"""Docstring after comment."""', + 'Docstring after comment.') + check('""""""', # Empty docstring + '') + check('', # Empty file + None) + check('"""Embedded\0null byte"""', + None) + check('"""Embedded null byte"""\0', + None) + check('"""Café and résumé."""', + 'Café and résumé.') + check("'''Triple single quotes'''", + 'Triple single quotes') + check('"Single double quotes"', + 'Single double quotes') + check("'Single single quotes'", + 'Single single quotes') + check('"""split\\\nline"""', + 'splitline') + check('"""Unrecognized escape \\sequence"""', + 'Unrecognized escape \\sequence') + check('"""Invalid escape seq\\uence"""', + None) + check('r"""Raw \\stri\\ng"""', + 'Raw \\stri\\ng') + check('b"""Bytes literal"""', + None) + check('f"""f-string"""', + None) + check('"""Concatenated""" \\\n"string" \'literals\'', + 'Concatenatedstringliterals') + check('"""String""" + """expression"""', + None) + check('("""In parentheses""")', + 'In parentheses') + check('("""Multiple lines """\n"""in parentheses""")', + 'Multiple lines in parentheses') + check('()', # tuple + None) + check(b'# coding: iso-8859-15\n"""\xa4uro sign"""', + '€uro sign', encoding='iso-8859-15') + check(b'"""\xa4"""', # Decoding error + None, encoding='utf-8') + + with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as temp_file: + temp_file.write('"""Real file test."""\n') + temp_file.flush() + temp_file.seek(0) + result = pydoc.source_synopsis(temp_file) + self.assertEqual(result, "Real file test.") + + @requires_docstrings + def test_synopsis_sourceless(self): + os = import_helper.import_fresh_module('os') + expected = os.__doc__.splitlines()[0] + filename = os.__spec__.cached + synopsis = pydoc.synopsis(filename) + + self.assertEqual(synopsis, expected) + + def test_synopsis_sourceless_empty_doc(self): + with os_helper.temp_cwd() as test_dir: + init_path = os.path.join(test_dir, 'foomod42.py') + cached_path = importlib.util.cache_from_source(init_path) + with open(init_path, 'w') as fobj: + fobj.write("foo = 1") + py_compile.compile(init_path) + synopsis = pydoc.synopsis(init_path, {}) + self.assertIsNone(synopsis) + synopsis_cached = pydoc.synopsis(cached_path, {}) + self.assertIsNone(synopsis_cached) + + def test_splitdoc_with_description(self): + example_string = "I Am A Doc\n\n\nHere is my description" + self.assertEqual(pydoc.splitdoc(example_string), + ('I Am A Doc', '\nHere is my description')) + + def test_is_package_when_not_package(self): + with os_helper.temp_cwd() as test_dir: + with self.assertWarns(DeprecationWarning) as cm: + self.assertFalse(pydoc.ispackage(test_dir)) + self.assertEqual(cm.filename, __file__) + + def test_is_package_when_is_package(self): + with os_helper.temp_cwd() as test_dir: + init_path = os.path.join(test_dir, '__init__.py') + open(init_path, 'w').close() + with self.assertWarns(DeprecationWarning) as cm: + self.assertTrue(pydoc.ispackage(test_dir)) + os.remove(init_path) + self.assertEqual(cm.filename, __file__) + + def test_allmethods(self): + # issue 17476: allmethods was no longer returning unbound methods. + # This test is a bit fragile in the face of changes to object and type, + # but I can't think of a better way to do it without duplicating the + # logic of the function under test. + + class TestClass(object): + def method_returning_true(self): + return True + + # What we expect to get back: everything on object... + expected = dict(vars(object)) + # ...plus our unbound method... + expected['method_returning_true'] = TestClass.method_returning_true + # ...but not the non-methods on object. + del expected['__doc__'] + del expected['__class__'] + # inspect resolves descriptors on type into methods, but vars doesn't, + # so we need to update __subclasshook__ and __init_subclass__. + expected['__subclasshook__'] = TestClass.__subclasshook__ + expected['__init_subclass__'] = TestClass.__init_subclass__ + + methods = pydoc.allmethods(TestClass) + self.assertDictEqual(methods, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_method_aliases(self): + class A: + def tkraise(self, aboveThis=None): + """Raise this widget in the stacking order.""" + lift = tkraise + def a_size(self): + """Return size""" + class B(A): + def itemconfigure(self, tagOrId, cnf=None, **kw): + """Configure resources of an item TAGORID.""" + itemconfig = itemconfigure + b_size = A.a_size + + doc = pydoc.render_doc(B) + doc = clean_text(doc) + self.assertEqual(doc, '''\ +Python Library Documentation: class B in module %s + +class B(A) + | Method resolution order: + | B + | A + | builtins.object + | + | Methods defined here: + | + | b_size = a_size(self) + | + | itemconfig = itemconfigure(self, tagOrId, cnf=None, **kw) + | + | itemconfigure(self, tagOrId, cnf=None, **kw) + | Configure resources of an item TAGORID. + | + | ---------------------------------------------------------------------- + | Methods inherited from A: + | + | a_size(self) + | Return size + | + | lift = tkraise(self, aboveThis=None) + | + | tkraise(self, aboveThis=None) + | Raise this widget in the stacking order. + | + | ---------------------------------------------------------------------- + | Data descriptors inherited from A: + | + | __dict__ + | dictionary for instance variables + | + | __weakref__ + | list of weak references to the object +''' % __name__) + + doc = pydoc.render_doc(B, renderer=pydoc.HTMLDoc()) + expected_text = f""" +Python Library Documentation + +class B in module {__name__} +class B(A) + Method resolution order: + B + A + builtins.object + + Methods defined here: + b_size = a_size(self) + itemconfig = itemconfigure(self, tagOrId, cnf=None, **kw) + itemconfigure(self, tagOrId, cnf=None, **kw) + Configure resources of an item TAGORID. + + Methods inherited from A: + a_size(self) + Return size + lift = tkraise(self, aboveThis=None) + tkraise(self, aboveThis=None) + Raise this widget in the stacking order. + + Data descriptors inherited from A: + __dict__ + dictionary for instance variables + __weakref__ + list of weak references to the object +""" + as_text = html2text(doc) + expected_lines = [line.strip() for line in expected_text.split("\n") if line] + for expected_line in expected_lines: + self.assertIn(expected_line, as_text) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_long_signatures(self): + from collections.abc import Callable + from typing import Literal, Annotated + + class A: + def __init__(self, + arg1: Callable[[int, int, int], str], + arg2: Literal['some value', 'other value'], + arg3: Annotated[int, 'some docs about this type'], + ) -> None: + ... + + doc = pydoc.render_doc(A) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: class A in module %s + +class A(builtins.object) + | A( + | arg1: Callable[[int, int, int], str], + | arg2: Literal['some value', 'other value'], + | arg3: Annotated[int, 'some docs about this type'] + | ) -> None + | + | Methods defined here: + | + | __init__( + | self, + | arg1: Callable[[int, int, int], str], + | arg2: Literal['some value', 'other value'], + | arg3: Annotated[int, 'some docs about this type'] + | ) -> None + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__%s + | + | __weakref__%s +''' % (__name__, + '' if MISSING_C_DOCSTRINGS else '\n | dictionary for instance variables', + '' if MISSING_C_DOCSTRINGS else '\n | list of weak references to the object', + )) + + def func( + arg1: Callable[[Annotated[int, 'Some doc']], str], + arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8], + ) -> Annotated[int, 'Some other']: + ... + + doc = pydoc.render_doc(func) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function func in module %s + +func( + arg1: Callable[[Annotated[int, 'Some doc']], str], + arg2: Literal[1, 2, 3, 4, 5, 6, 7, 8] +) -> Annotated[int, 'Some other'] +''' % __name__) + + def function_with_really_long_name_so_annotations_can_be_rather_small( + arg1: int, + arg2: str, + ): + ... + + doc = pydoc.render_doc(function_with_really_long_name_so_annotations_can_be_rather_small) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function function_with_really_long_name_so_annotations_can_be_rather_small in module %s + +function_with_really_long_name_so_annotations_can_be_rather_small( + arg1: int, + arg2: str +) +''' % __name__) + + does_not_have_name = lambda \ + very_long_parameter_name_that_should_not_fit_into_a_single_line, \ + second_very_long_parameter_name: ... + + doc = pydoc.render_doc(does_not_have_name) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: function <lambda> in module %s + +<lambda> lambda very_long_parameter_name_that_should_not_fit_into_a_single_line, second_very_long_parameter_name +''' % __name__) + + def test__future__imports(self): + # __future__ features are excluded from module help, + # except when it's the __future__ module itself + import __future__ + future_text, _ = get_pydoc_text(__future__) + future_html, _ = get_pydoc_html(__future__) + pydoc_mod_text, _ = get_pydoc_text(pydoc_mod) + pydoc_mod_html, _ = get_pydoc_html(pydoc_mod) + + for feature in __future__.all_feature_names: + txt = f"{feature} = _Feature" + html = f"<strong>{feature}</strong> = _Feature" + self.assertIn(txt, future_text) + self.assertIn(html, future_html) + self.assertNotIn(txt, pydoc_mod_text) + self.assertNotIn(html, pydoc_mod_html) + + +class PydocImportTest(PydocBaseTest): + + def setUp(self): + self.test_dir = os.mkdir(TESTFN) + self.addCleanup(rmtree, TESTFN) + importlib.invalidate_caches() + + def test_badimport(self): + # This tests the fix for issue 5230, where if pydoc found the module + # but the module had an internal import error pydoc would report no doc + # found. + modname = 'testmod_xyzzy' + testpairs = ( + ('i_am_not_here', 'i_am_not_here'), + ('test.i_am_not_here_either', 'test.i_am_not_here_either'), + ('test.i_am_not_here.neither_am_i', 'test.i_am_not_here'), + ('i_am_not_here.{}'.format(modname), 'i_am_not_here'), + ('test.{}'.format(modname), 'test.{}'.format(modname)), + ) + + sourcefn = os.path.join(TESTFN, modname) + os.extsep + "py" + for importstring, expectedinmsg in testpairs: + with open(sourcefn, 'w') as f: + f.write("import {}\n".format(importstring)) + result = run_pydoc_fail(modname, PYTHONPATH=TESTFN).decode("ascii") + expected = badimport_pattern % (modname, expectedinmsg) + self.assertEqual(expected, result) + + def test_apropos_with_bad_package(self): + # Issue 7425 - pydoc -k failed when bad package on path + pkgdir = os.path.join(TESTFN, "syntaxerr") + os.mkdir(pkgdir) + badsyntax = os.path.join(pkgdir, "__init__") + os.extsep + "py" + with open(badsyntax, 'w') as f: + f.write("invalid python syntax = $1\n") + with self.restrict_walk_packages(path=[TESTFN]): + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('xyzzy') + # No result, no error + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), '') + # The package name is still matched + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('syntaxerr') + self.assertEqual(out.getvalue().strip(), 'syntaxerr') + self.assertEqual(err.getvalue(), '') + + def test_apropos_with_unreadable_dir(self): + # Issue 7367 - pydoc -k failed when unreadable dir on path + self.unreadable_dir = os.path.join(TESTFN, "unreadable") + os.mkdir(self.unreadable_dir, 0) + self.addCleanup(os.rmdir, self.unreadable_dir) + # Note, on Windows the directory appears to be still + # readable so this is not really testing the issue there + with self.restrict_walk_packages(path=[TESTFN]): + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('SOMEKEY') + # No result, no error + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), '') + + def test_apropos_empty_doc(self): + pkgdir = os.path.join(TESTFN, 'walkpkg') + os.mkdir(pkgdir) + self.addCleanup(rmtree, pkgdir) + init_path = os.path.join(pkgdir, '__init__.py') + with open(init_path, 'w') as fobj: + fobj.write("foo = 1") + with self.restrict_walk_packages(path=[TESTFN]), captured_stdout() as stdout: + pydoc.apropos('') + self.assertIn('walkpkg', stdout.getvalue()) + + def test_url_search_package_error(self): + # URL handler search should cope with packages that raise exceptions + pkgdir = os.path.join(TESTFN, "test_error_package") + os.mkdir(pkgdir) + init = os.path.join(pkgdir, "__init__.py") + with open(init, "wt", encoding="ascii") as f: + f.write("""raise ValueError("ouch")\n""") + with self.restrict_walk_packages(path=[TESTFN]): + # Package has to be importable for the error to have any effect + saved_paths = tuple(sys.path) + sys.path.insert(0, TESTFN) + try: + with self.assertRaisesRegex(ValueError, "ouch"): + # Sanity check + import test_error_package # noqa: F401 + + text = self.call_url_handler("search?key=test_error_package", + "Pydoc: Search Results") + found = ('<a href="test_error_package.html">' + 'test_error_package</a>') + self.assertIn(found, text) + finally: + sys.path[:] = saved_paths + + @unittest.skip('causes undesirable side-effects (#20128)') + def test_modules(self): + # See Helper.listmodules(). + num_header_lines = 2 + num_module_lines_min = 5 # Playing it safe. + num_footer_lines = 3 + expected = num_header_lines + num_module_lines_min + num_footer_lines + + output = StringIO() + helper = pydoc.Helper(output=output) + helper('modules') + result = output.getvalue().strip() + num_lines = len(result.splitlines()) + + self.assertGreaterEqual(num_lines, expected) + + @unittest.skip('causes undesirable side-effects (#20128)') + def test_modules_search(self): + # See Helper.listmodules(). + expected = 'pydoc - ' + + output = StringIO() + helper = pydoc.Helper(output=output) + with captured_stdout() as help_io: + helper('modules pydoc') + result = help_io.getvalue() + + self.assertIn(expected, result) + + @unittest.skip('some buildbots are not cooperating (#20128)') + def test_modules_search_builtin(self): + expected = 'gc - ' + + output = StringIO() + helper = pydoc.Helper(output=output) + with captured_stdout() as help_io: + helper('modules garbage') + result = help_io.getvalue() + + self.assertStartsWith(result, expected) + + def test_importfile(self): + try: + loaded_pydoc = pydoc.importfile(pydoc.__file__) + + self.assertIsNot(loaded_pydoc, pydoc) + self.assertEqual(loaded_pydoc.__name__, 'pydoc') + self.assertEqual(loaded_pydoc.__file__, pydoc.__file__) + self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__) + finally: + sys.modules['pydoc'] = pydoc + + +class Rect: + @property + def area(self): + '''Area of the rect''' + return self.w * self.h + + +class Square(Rect): + area = property(lambda self: self.side**2) + + +class TestDescriptions(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + def test_module(self): + # Check that pydocfodder module can be described + doc = pydoc.render_doc(pydocfodder) + self.assertIn("pydocfodder", doc) + + def test_class(self): + class C: "New-style class" + c = C() + + self.assertEqual(pydoc.describe(C), 'class C') + self.assertEqual(pydoc.describe(c), 'C') + expected = 'C in module %s object' % __name__ + self.assertIn(expected, pydoc.render_doc(c)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_generic_alias(self): + self.assertEqual(pydoc.describe(typing.List[int]), '_GenericAlias') + doc = pydoc.render_doc(typing.List[int], renderer=pydoc.plaintext) + self.assertIn('_GenericAlias in module typing', doc) + self.assertIn('List = class list(object)', doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) + + self.assertEqual(pydoc.describe(list[int]), 'GenericAlias') + doc = pydoc.render_doc(list[int], renderer=pydoc.plaintext) + self.assertIn('GenericAlias in module builtins', doc) + self.assertIn('\nclass list(object)', doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(list.__doc__.strip().splitlines()[0], doc) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_union_type(self): + self.assertEqual(pydoc.describe(typing.Union[int, str]), 'Union') + doc = pydoc.render_doc(typing.Union[int, str], renderer=pydoc.plaintext) + self.assertIn('Union in module typing', doc) + self.assertIn('class Union(builtins.object)', doc) + if typing.Union.__doc__: + self.assertIn(typing.Union.__doc__.strip().splitlines()[0], doc) + + self.assertEqual(pydoc.describe(int | str), 'Union') + doc = pydoc.render_doc(int | str, renderer=pydoc.plaintext) + self.assertIn('Union in module typing', doc) + self.assertIn('class Union(builtins.object)', doc) + if not MISSING_C_DOCSTRINGS: + self.assertIn(types.UnionType.__doc__.strip().splitlines()[0], doc) + + def test_special_form(self): + self.assertEqual(pydoc.describe(typing.NoReturn), '_SpecialForm') + doc = pydoc.render_doc(typing.NoReturn, renderer=pydoc.plaintext) + self.assertIn('_SpecialForm in module typing', doc) + if typing.NoReturn.__doc__: + self.assertIn('NoReturn = typing.NoReturn', doc) + self.assertIn(typing.NoReturn.__doc__.strip().splitlines()[0], doc) + else: + self.assertIn('NoReturn = class _SpecialForm(_Final)', doc) + + def test_typing_pydoc(self): + def foo(data: typing.List[typing.Any], + x: int) -> typing.Iterator[typing.Tuple[int, typing.Any]]: + ... + T = typing.TypeVar('T') + class C(typing.Generic[T], typing.Mapping[int, str]): ... + self.assertEqual(pydoc.render_doc(foo).splitlines()[-1], + 'f\x08fo\x08oo\x08o(data: typing.List[typing.Any], x: int)' + ' -> typing.Iterator[typing.Tuple[int, typing.Any]]') + self.assertEqual(pydoc.render_doc(C).splitlines()[2], + 'class C\x08C(collections.abc.Mapping, typing.Generic)') + + def test_builtin(self): + for name in ('str', 'str.translate', 'builtins.str', + 'builtins.str.translate'): + # test low-level function + self.assertIsNotNone(pydoc.locate(name)) + # test high-level function + try: + pydoc.render_doc(name) + except ImportError: + self.fail('finding the doc of {!r} failed'.format(name)) + + for name in ('notbuiltins', 'strrr', 'strr.translate', + 'str.trrrranslate', 'builtins.strrr', + 'builtins.str.trrranslate'): + self.assertIsNone(pydoc.locate(name)) + self.assertRaises(ImportError, pydoc.render_doc, name) + + @staticmethod + def _get_summary_line(o): + text = pydoc.plain(pydoc.render_doc(o)) + lines = text.split('\n') + assert len(lines) >= 2 + return lines[2] + + @staticmethod + def _get_summary_lines(o): + text = pydoc.plain(pydoc.render_doc(o)) + lines = text.split('\n') + return '\n'.join(lines[2:]) + + # these should include "self" + def test_unbound_python_method(self): + self.assertEqual(self._get_summary_line(textwrap.TextWrapper.wrap), + "wrap(self, text)") + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_unbound_builtin_method(self): + self.assertEqual(self._get_summary_line(_pickle.Pickler.dump), + "dump(self, obj, /) unbound _pickle.Pickler method") + + # these no longer include "self" + def test_bound_python_method(self): + t = textwrap.TextWrapper() + self.assertEqual(self._get_summary_line(t.wrap), + "wrap(text) method of textwrap.TextWrapper instance") + def test_field_order_for_named_tuples(self): + Person = namedtuple('Person', ['nickname', 'firstname', 'agegroup']) + s = pydoc.render_doc(Person) + self.assertLess(s.index('nickname'), s.index('firstname')) + self.assertLess(s.index('firstname'), s.index('agegroup')) + + class NonIterableFields: + _fields = None + + class NonHashableFields: + _fields = [[]] + + # Make sure these doesn't fail + pydoc.render_doc(NonIterableFields) + pydoc.render_doc(NonHashableFields) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_bound_builtin_method(self): + s = StringIO() + p = _pickle.Pickler(s) + self.assertEqual(self._get_summary_line(p.dump), + "dump(obj, /) method of _pickle.Pickler instance") + + # this should *never* include self! + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_module_level_callable(self): + self.assertEqual(self._get_summary_line(os.stat), + "stat(path, *, dir_fd=None, follow_symlinks=True)") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_module_level_callable_noargs(self): + self.assertEqual(self._get_summary_line(time.time), + "time()") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_module_level_callable_o(self): + try: + import _stat + except ImportError: + # stat.S_IMODE() and _stat.S_IMODE() have a different signature + self.skipTest('_stat extension is missing') + + self.assertEqual(self._get_summary_line(_stat.S_IMODE), + "S_IMODE(object, /)") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_method_noargs(self): + self.assertEqual(self._get_summary_line(str.lower), + "lower(self, /) unbound builtins.str method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_method_noargs(self): + self.assertEqual(self._get_summary_line(''.lower), + "lower() method of builtins.str instance") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_method_o(self): + self.assertEqual(self._get_summary_line(set.add), + "add(self, object, /) unbound builtins.set method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_method_o(self): + self.assertEqual(self._get_summary_line(set().add), + "add(object, /) method of builtins.set instance") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_method_coexist_o(self): + self.assertEqual(self._get_summary_line(set.__contains__), + "__contains__(self, object, /) unbound builtins.set method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_method_coexist_o(self): + self.assertEqual(self._get_summary_line(set().__contains__), + "__contains__(object, /) method of builtins.set instance") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_classmethod_noargs(self): + self.assertEqual(self._get_summary_line(datetime.datetime.__dict__['utcnow']), + "utcnow(type, /) unbound datetime.datetime method") + + def test_bound_builtin_classmethod_noargs(self): + self.assertEqual(self._get_summary_line(datetime.datetime.utcnow), + "utcnow() class method of datetime.datetime") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unbound_builtin_classmethod_o(self): + self.assertEqual(self._get_summary_line(dict.__dict__['__class_getitem__']), + "__class_getitem__(type, object, /) unbound builtins.dict method") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bound_builtin_classmethod_o(self): + self.assertEqual(self._get_summary_line(dict.__class_getitem__), + "__class_getitem__(object, /) class method of builtins.dict") + + @support.cpython_only + @requires_docstrings + def test_module_level_callable_unrepresentable_default(self): + _testcapi = import_helper.import_module("_testcapi") + builtin = _testcapi.func_with_unrepresentable_signature + self.assertEqual(self._get_summary_line(builtin), + "func_with_unrepresentable_signature(a, b=<x>)") + + @support.cpython_only + @requires_docstrings + def test_builtin_staticmethod_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(str.maketrans), + "maketrans(x, y=<unrepresentable>, z=<unrepresentable>, /)") + _testcapi = import_helper.import_module("_testcapi") + cls = _testcapi.DocStringUnrepresentableSignatureTest + self.assertEqual(self._get_summary_line(cls.staticmeth), + "staticmeth(a, b=<x>)") + + @support.cpython_only + @requires_docstrings + def test_unbound_builtin_method_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(dict.pop), + "pop(self, key, default=<unrepresentable>, /) " + "unbound builtins.dict method") + _testcapi = import_helper.import_module("_testcapi") + cls = _testcapi.DocStringUnrepresentableSignatureTest + self.assertEqual(self._get_summary_line(cls.meth), + "meth(self, /, a, b=<x>) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") + + @support.cpython_only + @requires_docstrings + def test_bound_builtin_method_unrepresentable_default(self): + self.assertEqual(self._get_summary_line({}.pop), + "pop(key, default=<unrepresentable>, /) " + "method of builtins.dict instance") + _testcapi = import_helper.import_module("_testcapi") + obj = _testcapi.DocStringUnrepresentableSignatureTest() + self.assertEqual(self._get_summary_line(obj.meth), + "meth(a, b=<x>) " + "method of _testcapi.DocStringUnrepresentableSignatureTest instance") + + @support.cpython_only + @requires_docstrings + def test_unbound_builtin_classmethod_unrepresentable_default(self): + _testcapi = import_helper.import_module("_testcapi") + cls = _testcapi.DocStringUnrepresentableSignatureTest + descr = cls.__dict__['classmeth'] + self.assertEqual(self._get_summary_line(descr), + "classmeth(type, /, a, b=<x>) unbound " + "_testcapi.DocStringUnrepresentableSignatureTest method") + + @support.cpython_only + @requires_docstrings + def test_bound_builtin_classmethod_unrepresentable_default(self): + _testcapi = import_helper.import_module("_testcapi") + cls = _testcapi.DocStringUnrepresentableSignatureTest + self.assertEqual(self._get_summary_line(cls.classmeth), + "classmeth(a, b=<x>) class method of " + "_testcapi.DocStringUnrepresentableSignatureTest") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_overridden_text_signature(self): + class C: + def meth(*args, **kwargs): + pass + @classmethod + def cmeth(*args, **kwargs): + pass + @staticmethod + def smeth(*args, **kwargs): + pass + for text_signature, unbound, bound in [ + ("($slf)", "(slf, /)", "()"), + ("($slf, /)", "(slf, /)", "()"), + ("($slf, /, arg)", "(slf, /, arg)", "(arg)"), + ("($slf, /, arg=<x>)", "(slf, /, arg=<x>)", "(arg=<x>)"), + ("($slf, arg, /)", "(slf, arg, /)", "(arg, /)"), + ("($slf, arg=<x>, /)", "(slf, arg=<x>, /)", "(arg=<x>, /)"), + ("(/, slf, arg)", "(/, slf, arg)", "(/, slf, arg)"), + ("(/, slf, arg=<x>)", "(/, slf, arg=<x>)", "(/, slf, arg=<x>)"), + ("(slf, /, arg)", "(slf, /, arg)", "(arg)"), + ("(slf, /, arg=<x>)", "(slf, /, arg=<x>)", "(arg=<x>)"), + ("(slf, arg, /)", "(slf, arg, /)", "(arg, /)"), + ("(slf, arg=<x>, /)", "(slf, arg=<x>, /)", "(arg=<x>, /)"), + ]: + with self.subTest(text_signature): + C.meth.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.meth), + "meth" + unbound) + self.assertEqual(self._get_summary_line(C().meth), + "meth" + bound + " method of test.test_pydoc.test_pydoc.C instance") + C.cmeth.__func__.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.cmeth), + "cmeth" + bound + " class method of test.test_pydoc.test_pydoc.C") + C.smeth.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.smeth), + "smeth" + unbound) + + @requires_docstrings + def test_staticmethod(self): + class X: + @staticmethod + def sm(x, y): + '''A static method''' + ... + self.assertEqual(self._get_summary_lines(X.__dict__['sm']), + 'sm(x, y)\n' + ' A static method\n') + self.assertEqual(self._get_summary_lines(X.sm), """\ +sm(x, y) + A static method +""") + self.assertIn(""" + | Static methods defined here: + | + | sm(x, y) + | A static method +""", pydoc.plain(pydoc.render_doc(X))) + + @requires_docstrings + def test_classmethod(self): + class X: + @classmethod + def cm(cls, x): + '''A class method''' + ... + self.assertEqual(self._get_summary_lines(X.__dict__['cm']), + 'cm(...)\n' + ' A class method\n') + self.assertEqual(self._get_summary_lines(X.cm), """\ +cm(x) class method of test.test_pydoc.test_pydoc.X + A class method +""") + self.assertIn(""" + | Class methods defined here: + | + | cm(x) + | A class method +""", pydoc.plain(pydoc.render_doc(X))) + + @requires_docstrings + def test_getset_descriptor(self): + # Currently these attributes are implemented as getset descriptors + # in CPython. + self.assertEqual(self._get_summary_line(int.numerator), "numerator") + self.assertEqual(self._get_summary_line(float.real), "real") + self.assertEqual(self._get_summary_line(Exception.args), "args") + self.assertEqual(self._get_summary_line(memoryview.obj), "obj") + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_member_descriptor(self): + # Currently these attributes are implemented as member descriptors + # in CPython. + self.assertEqual(self._get_summary_line(complex.real), "real") + self.assertEqual(self._get_summary_line(range.start), "start") + self.assertEqual(self._get_summary_line(slice.start), "start") + self.assertEqual(self._get_summary_line(property.fget), "fget") + self.assertEqual(self._get_summary_line(StopIteration.value), "value") + + @requires_docstrings + def test_slot_descriptor(self): + class Point: + __slots__ = 'x', 'y' + self.assertEqual(self._get_summary_line(Point.x), "x") + + @requires_docstrings + def test_dict_attr_descriptor(self): + class NS: + pass + self.assertEqual(self._get_summary_line(NS.__dict__['__dict__']), + "__dict__") + + @requires_docstrings + def test_structseq_member_descriptor(self): + self.assertEqual(self._get_summary_line(type(sys.hash_info).width), + "width") + self.assertEqual(self._get_summary_line(type(sys.flags).debug), + "debug") + self.assertEqual(self._get_summary_line(type(sys.version_info).major), + "major") + self.assertEqual(self._get_summary_line(type(sys.float_info).max), + "max") + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_docstrings + def test_namedtuple_field_descriptor(self): + Box = namedtuple('Box', ('width', 'height')) + self.assertEqual(self._get_summary_lines(Box.width), """\ + Alias for field number 0 +""") + + @requires_docstrings + def test_property(self): + self.assertEqual(self._get_summary_lines(Rect.area), """\ +area + Area of the rect +""") + # inherits the docstring from Rect.area + self.assertEqual(self._get_summary_lines(Square.area), """\ +area + Area of the rect +""") + self.assertIn(""" + | area + | Area of the rect +""", pydoc.plain(pydoc.render_doc(Rect))) + + @requires_docstrings + def test_custom_non_data_descriptor(self): + class Descr: + def __get__(self, obj, cls): + if obj is None: + return self + return 42 + class X: + attr = Descr() + + self.assertEqual(self._get_summary_lines(X.attr), f"""\ +<{__name__}.TestDescriptions.test_custom_non_data_descriptor.<locals>.Descr object>""") + + X.attr.__doc__ = 'Custom descriptor' + self.assertEqual(self._get_summary_lines(X.attr), f"""\ +<{__name__}.TestDescriptions.test_custom_non_data_descriptor.<locals>.Descr object> + Custom descriptor +""") + + X.attr.__name__ = 'foo' + self.assertEqual(self._get_summary_lines(X.attr), """\ +foo(...) + Custom descriptor +""") + + @requires_docstrings + def test_custom_data_descriptor(self): + class Descr: + def __get__(self, obj, cls): + if obj is None: + return self + return 42 + def __set__(self, obj, cls): + 1/0 + class X: + attr = Descr() + + self.assertEqual(self._get_summary_lines(X.attr), "") + + X.attr.__doc__ = 'Custom descriptor' + self.assertEqual(self._get_summary_lines(X.attr), """\ + Custom descriptor +""") + + X.attr.__name__ = 'foo' + self.assertEqual(self._get_summary_lines(X.attr), """\ +foo + Custom descriptor +""") + + def test_async_annotation(self): + async def coro_function(ign) -> int: + return 1 + + text = pydoc.plain(pydoc.plaintext.document(coro_function)) + self.assertIn('async coro_function', text) + + html = pydoc.HTMLDoc().document(coro_function) + self.assertIn( + 'async <a name="-coro_function"><strong>coro_function', + html) + + def test_async_generator_annotation(self): + async def an_async_generator(): + yield 1 + + text = pydoc.plain(pydoc.plaintext.document(an_async_generator)) + self.assertIn('async an_async_generator', text) + + html = pydoc.HTMLDoc().document(an_async_generator) + self.assertIn( + 'async <a name="-an_async_generator"><strong>an_async_generator', + html) + + @requires_docstrings + def test_html_for_https_links(self): + def a_fn_with_https_link(): + """a link https://localhost/""" + pass + + html = pydoc.HTMLDoc().document(a_fn_with_https_link) + self.assertIn( + '<a href="https://localhost/">https://localhost/</a>', + html + ) + + def test_module_none(self): + # Issue #128772 + from test.test_pydoc import module_none + pydoc.render_doc(module_none) + + +class PydocFodderTest(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + def getsection(self, text, beginline, endline): + lines = text.splitlines() + beginindex, endindex = 0, None + if beginline is not None: + beginindex = lines.index(beginline) + if endline is not None: + endindex = lines.index(endline, beginindex) + return lines[beginindex:endindex] + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_text_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.TextDoc() + result = doc.docclass(cls) + result = clean_text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f' | Methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_method_alias = A_method(self)', lines) + self.assertIn(' | B_method_alias = B_method(self)', lines) + self.assertIn(' | A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn(' | A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn(' | global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn(' | global_func_alias = global_func(x, y)', lines) + self.assertIn(' | global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' | count(self, value, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, value, /)', lines) + self.assertIn(' | __repr__(self, /) from builtins.object', lines) + self.assertIn(' | object_repr = __repr__(self, /)', lines) + else: + self.assertIn(' | count(self, object, /) from builtins.list', lines) + self.assertIn(' | list_count = count(self, object, /)', lines) + self.assertIn(' | __repr__(...) from builtins.object', lines) + self.assertIn(' | object_repr = __repr__(...)', lines) + + lines = self.getsection(result, f' | Static methods {where}:', ' | ' + '-'*70) + self.assertIn(' | A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn(' | B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn(' | A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' | get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | sin(x, /)', lines) + else: + self.assertIn(' | get(...) method of builtins.dict instance', lines) + self.assertIn(' | dict_get = get(...) method of builtins.dict instance', lines) + self.assertIn(' | sin(object, /)', lines) + + lines = self.getsection(result, f' | Class methods {where}:', ' | ' + '-'*70) + self.assertIn(' | B_classmethod(x)', lines) + self.assertIn(' | B_classmethod_alias = B_classmethod(x)', lines) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_html_doc_routines_in_class(self, cls=pydocfodder.B): + doc = pydoc.HTMLDoc() + result = doc.docclass(cls) + result = html2text(result) + where = 'defined here' if cls is pydocfodder.B else 'inherited from B' + lines = self.getsection(result, f'Methods {where}:', '-'*70) + self.assertIn('A_method_alias = A_method(self)', lines) + self.assertIn('B_method_alias = B_method(self)', lines) + self.assertIn('A_staticmethod(x, y) from test.test_pydoc.pydocfodder.A', lines) + self.assertIn('A_staticmethod_alias = A_staticmethod(x, y)', lines) + self.assertIn('global_func(x, y) from test.test_pydoc.pydocfodder', lines) + self.assertIn('global_func_alias = global_func(x, y)', lines) + self.assertIn('global_func2_alias = global_func2(x, y) from test.test_pydoc.pydocfodder', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn('count(self, value, /) from builtins.list', lines) + self.assertIn('list_count = count(self, value, /)', lines) + self.assertIn('__repr__(self, /) from builtins.object', lines) + self.assertIn('object_repr = __repr__(self, /)', lines) + else: + self.assertIn('count(self, object, /) from builtins.list', lines) + self.assertIn('list_count = count(self, object, /)', lines) + self.assertIn('__repr__(...) from builtins.object', lines) + self.assertIn('object_repr = __repr__(...)', lines) + + lines = self.getsection(result, f'Static methods {where}:', '-'*70) + self.assertIn('A_classmethod_ref = A_classmethod(x) class method of test.test_pydoc.pydocfodder.A', lines) + note = '' if cls is pydocfodder.B else ' class method of test.test_pydoc.pydocfodder.B' + self.assertIn('B_classmethod_ref = B_classmethod(x)' + note, lines) + self.assertIn('A_method_ref = A_method() method of test.test_pydoc.pydocfodder.A instance', lines) + + lines = self.getsection(result, f'Class methods {where}:', '-'*70) + self.assertIn('B_classmethod(x)', lines) + self.assertIn('B_classmethod_alias = B_classmethod(x)', lines) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_text_doc_inherited_routines_in_class(self): + self.test_text_doc_routines_in_class(pydocfodder.D) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_html_doc_inherited_routines_in_class(self): + self.test_html_doc_routines_in_class(pydocfodder.D) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_text_doc_routines_in_module(self): + doc = pydoc.TextDoc() + result = doc.docmodule(pydocfodder) + result = clean_text(result) + lines = self.getsection(result, 'FUNCTIONS', 'FILE') + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn(' A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + else: + self.assertIn(' get(...) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(...) method of builtins.dict instance', lines) + + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + else: + self.assertIn(' count(self, object, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, object, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(...) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(...) unbound builtins.object method', lines) + + # builtin functions + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' sin(x, /)', lines) + else: + self.assertIn(' sin(object, /)', lines) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_html_doc_routines_in_module(self): + doc = pydoc.HTMLDoc() + result = doc.docmodule(pydocfodder) + result = html2text(result) + lines = self.getsection(result, ' Functions', None) + # function alias + self.assertIn(' global_func_alias = global_func(x, y)', lines) + self.assertIn(' A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_alias = A_staticmethod(x, y)', lines) + # bound class methods + self.assertIn('A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod2 = A_classmethod(x) class method of A', lines) + self.assertIn(' A_classmethod3 = A_classmethod(x) class method of B', lines) + # bound methods + self.assertIn(' A_method() method of A instance', lines) + self.assertIn(' A_method2 = A_method() method of A instance', lines) + self.assertIn(' A_method3 = A_method() method of B instance', lines) + self.assertIn(' A_staticmethod_ref = A_staticmethod(x, y)', lines) + self.assertIn(' A_staticmethod_ref2 = A_staticmethod(y) method of B instance', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + else: + self.assertIn(' get(...) method of builtins.dict instance', lines) + self.assertIn(' dict_get = get(...) method of builtins.dict instance', lines) + # unbound methods + self.assertIn(' B_method(self)', lines) + self.assertIn(' B_method2 = B_method(self)', lines) + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, value, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(self, /) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(self, /) unbound builtins.object method', lines) + else: + self.assertIn(' count(self, object, /) unbound builtins.list method', lines) + self.assertIn(' list_count = count(self, object, /) unbound builtins.list method', lines) + self.assertIn(' __repr__(...) unbound builtins.object method', lines) + self.assertIn(' object_repr = __repr__(...) unbound builtins.object method', lines) + + # builtin functions + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' sin(x, /)', lines) + else: + self.assertIn(' sin(object, /)', lines) + + +@unittest.skipIf( + is_wasm32, + "Socket server not available on Emscripten/WASI." +) +class PydocServerTest(unittest.TestCase): + """Tests for pydoc._start_server""" + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + def test_server(self): + # Minimal test that starts the server, checks that it works, then stops + # it and checks its cleanup. + def my_url_handler(url, content_type): + text = 'the URL sent was: (%s, %s)' % (url, content_type) + return text + + serverthread = pydoc._start_server( + my_url_handler, + hostname='localhost', + port=0, + ) + self.assertEqual(serverthread.error, None) + self.assertTrue(serverthread.serving) + self.addCleanup( + lambda: serverthread.stop() if serverthread.serving else None + ) + self.assertIn('localhost', serverthread.url) + + self.addCleanup(urlcleanup) + self.assertEqual( + b'the URL sent was: (/test, text/html)', + urlopen(urllib.parse.urljoin(serverthread.url, '/test')).read(), + ) + self.assertEqual( + b'the URL sent was: (/test.css, text/css)', + urlopen(urllib.parse.urljoin(serverthread.url, '/test.css')).read(), + ) + + serverthread.stop() + self.assertFalse(serverthread.serving) + self.assertIsNone(serverthread.docserver) + self.assertIsNone(serverthread.url) + + +class PydocUrlHandlerTest(PydocBaseTest): + """Tests for pydoc._url_handler""" + + @unittest.skip("TODO: RUSTPYTHON; Panic") + def test_content_type_err(self): + f = pydoc._url_handler + self.assertRaises(TypeError, f, 'A', '') + self.assertRaises(TypeError, f, 'B', 'foobar') + + def test_url_requests(self): + # Test for the correct title in the html pages returned. + # This tests the different parts of the URL handler without + # getting too picky about the exact html. + requests = [ + ("", "Pydoc: Index of Modules"), + ("get?key=", "Pydoc: Index of Modules"), + ("index", "Pydoc: Index of Modules"), + ("topics", "Pydoc: Topics"), + ("keywords", "Pydoc: Keywords"), + ("pydoc", "Pydoc: module pydoc"), + ("get?key=pydoc", "Pydoc: module pydoc"), + ("search?key=pydoc", "Pydoc: Search Results"), + ("topic?key=def", "Pydoc: KEYWORD def"), + ("topic?key=STRINGS", "Pydoc: TOPIC STRINGS"), + ("foobar", "Pydoc: Error - foobar"), + ] + + self.assertIs(sys.modules['pydoc'], pydoc) + try: + with self.restrict_walk_packages(): + for url, title in requests: + self.call_url_handler(url, title) + finally: + # Some requests reload the module and change sys.modules. + sys.modules['pydoc'] = pydoc + + +class TestHelper(unittest.TestCase): + def mock_interactive_session(self, inputs): + """ + Given a list of inputs, run an interactive help session. Returns a string + of what would be shown on screen. + """ + input_iter = iter(inputs) + + def mock_getline(prompt): + output.write(prompt) + next_input = next(input_iter) + output.write(next_input + os.linesep) + return next_input + + with captured_stdout() as output: + helper = pydoc.Helper(output=output) + with unittest.mock.patch.object(helper, "getline", mock_getline): + helper.interact() + + # handle different line endings across platforms consistently + return output.getvalue().strip().splitlines(keepends=False) + + def test_keywords(self): + self.assertEqual(sorted(pydoc.Helper.keywords), + sorted(keyword.kwlist)) + + def test_interact_empty_line_continues(self): + # gh-138568: test pressing Enter without input should continue in help session + self.assertEqual( + self.mock_interactive_session(["", " ", "quit"]), + ["help> ", "help> ", "help> quit"], + ) + + def test_interact_quit_commands_exit(self): + quit_commands = ["quit", "q", "exit"] + for quit_cmd in quit_commands: + with self.subTest(quit_command=quit_cmd): + self.assertEqual( + self.mock_interactive_session([quit_cmd]), + [f"help> {quit_cmd}"], + ) + + +class PydocWithMetaClasses(unittest.TestCase): + def tearDown(self): + self.assertIs(sys.modules['pydoc'], pydoc) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_DynamicClassAttribute(self): + class Meta(type): + def __getattr__(self, name): + if name == 'ham': + return 'spam' + return super().__getattr__(name) + class DA(metaclass=Meta): + @types.DynamicClassAttribute + def ham(self): + return 'eggs' + expected_text_data_docstrings = tuple('\n | ' + s if s else '' + for s in expected_data_docstrings) + output = StringIO() + helper = pydoc.Helper(output=output) + helper(DA) + expected_text = expected_dynamicattribute_pattern % ( + (__name__,) + expected_text_data_docstrings[:2]) + result = output.getvalue().strip() + self.assertEqual(expected_text, result) + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_virtualClassAttributeWithOneMeta(self): + class Meta(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'LIFE'] + def __getattr__(self, name): + if name =='LIFE': + return 42 + return super().__getattr(name) + class Class(metaclass=Meta): + pass + output = StringIO() + helper = pydoc.Helper(output=output) + helper(Class) + expected_text = expected_virtualattribute_pattern1 % __name__ + result = output.getvalue().strip() + self.assertEqual(expected_text, result) + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_virtualClassAttributeWithTwoMeta(self): + class Meta1(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'one'] + def __getattr__(self, name): + if name =='one': + return 1 + return super().__getattr__(name) + class Meta2(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'two'] + def __getattr__(self, name): + if name =='two': + return 2 + return super().__getattr__(name) + class Meta3(Meta1, Meta2): + def __dir__(cls): + return list(sorted(set( + ['__class__', '__module__', '__name__', 'three'] + + Meta1.__dir__(cls) + Meta2.__dir__(cls)))) + def __getattr__(self, name): + if name =='three': + return 3 + return super().__getattr__(name) + class Class1(metaclass=Meta1): + pass + class Class2(Class1, metaclass=Meta3): + pass + output = StringIO() + helper = pydoc.Helper(output=output) + helper(Class1) + expected_text1 = expected_virtualattribute_pattern2 % __name__ + result1 = output.getvalue().strip() + self.assertEqual(expected_text1, result1) + output = StringIO() + helper = pydoc.Helper(output=output) + helper(Class2) + expected_text2 = expected_virtualattribute_pattern3 % __name__ + result2 = output.getvalue().strip() + self.assertEqual(expected_text2, result2) + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + def test_buggy_dir(self): + class M(type): + def __dir__(cls): + return ['__class__', '__name__', 'missing', 'here'] + class C(metaclass=M): + here = 'present!' + output = StringIO() + helper = pydoc.Helper(output=output) + helper(C) + expected_text = expected_missingattribute_pattern % __name__ + result = output.getvalue().strip() + self.assertEqual(expected_text, result) + + def test_resolve_false(self): + # Issue #23008: pydoc enum.{,Int}Enum failed + # because bool(enum.Enum) is False. + with captured_stdout() as help_io: + pydoc.help('enum.Enum') + helptext = help_io.getvalue() + self.assertIn('class Enum', helptext) + + +class TestInternalUtilities(unittest.TestCase): + + def setUp(self): + tmpdir = tempfile.TemporaryDirectory() + self.argv0dir = tmpdir.name + self.argv0 = os.path.join(tmpdir.name, "nonexistent") + self.addCleanup(tmpdir.cleanup) + self.abs_curdir = abs_curdir = os.getcwd() + self.curdir_spellings = ["", os.curdir, abs_curdir] + + def _get_revised_path(self, given_path, argv0=None): + # Checking that pydoc.cli() actually calls pydoc._get_revised_path() + # is handled via code review (at least for now). + if argv0 is None: + argv0 = self.argv0 + return pydoc._get_revised_path(given_path, argv0) + + def _get_starting_path(self): + # Get a copy of sys.path without the current directory. + clean_path = sys.path.copy() + for spelling in self.curdir_spellings: + for __ in range(clean_path.count(spelling)): + clean_path.remove(spelling) + return clean_path + + def test_sys_path_adjustment_adds_missing_curdir(self): + clean_path = self._get_starting_path() + expected_path = [self.abs_curdir] + clean_path + self.assertEqual(self._get_revised_path(clean_path), expected_path) + + def test_sys_path_adjustment_removes_argv0_dir(self): + clean_path = self._get_starting_path() + expected_path = [self.abs_curdir] + clean_path + leading_argv0dir = [self.argv0dir] + clean_path + self.assertEqual(self._get_revised_path(leading_argv0dir), expected_path) + trailing_argv0dir = clean_path + [self.argv0dir] + self.assertEqual(self._get_revised_path(trailing_argv0dir), expected_path) + + def test_sys_path_adjustment_protects_pydoc_dir(self): + def _get_revised_path(given_path): + return self._get_revised_path(given_path, argv0=pydoc.__file__) + clean_path = self._get_starting_path() + leading_argv0dir = [self.argv0dir] + clean_path + expected_path = [self.abs_curdir] + leading_argv0dir + self.assertEqual(_get_revised_path(leading_argv0dir), expected_path) + trailing_argv0dir = clean_path + [self.argv0dir] + expected_path = [self.abs_curdir] + trailing_argv0dir + self.assertEqual(_get_revised_path(trailing_argv0dir), expected_path) + + def test_sys_path_adjustment_when_curdir_already_included(self): + clean_path = self._get_starting_path() + for spelling in self.curdir_spellings: + with self.subTest(curdir_spelling=spelling): + # If curdir is already present, no alterations are made at all + leading_curdir = [spelling] + clean_path + self.assertIsNone(self._get_revised_path(leading_curdir)) + trailing_curdir = clean_path + [spelling] + self.assertIsNone(self._get_revised_path(trailing_curdir)) + leading_argv0dir = [self.argv0dir] + leading_curdir + self.assertIsNone(self._get_revised_path(leading_argv0dir)) + trailing_argv0dir = trailing_curdir + [self.argv0dir] + self.assertIsNone(self._get_revised_path(trailing_argv0dir)) + + +def setUpModule(): + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) + unittest.addModuleCleanup(reap_children) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 33e1ffb836c..e28c15de62d 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -1,13 +1,18 @@ # XXX TypeErrors on calling handlers, or on bad return values from a # handler, are obscure and unhelpful. -from io import BytesIO +import abc +import functools import os -import platform +import re import sys import sysconfig +import textwrap import unittest import traceback +from io import BytesIO +from test import support +from test.support import import_helper, os_helper from xml.parsers import expat from xml.parsers.expat import errors @@ -19,32 +24,24 @@ class SetAttributeTest(unittest.TestCase): def setUp(self): self.parser = expat.ParserCreate(namespace_separator='!') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_buffer_text(self): self.assertIs(self.parser.buffer_text, False) for x in 0, 1, 2, 0: self.parser.buffer_text = x self.assertIs(self.parser.buffer_text, bool(x)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_namespace_prefixes(self): self.assertIs(self.parser.namespace_prefixes, False) for x in 0, 1, 2, 0: self.parser.namespace_prefixes = x self.assertIs(self.parser.namespace_prefixes, bool(x)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ordered_attributes(self): self.assertIs(self.parser.ordered_attributes, False) for x in 0, 1, 2, 0: self.parser.ordered_attributes = x self.assertIs(self.parser.ordered_attributes, bool(x)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_specified_attributes(self): self.assertIs(self.parser.specified_attributes, False) for x in 0, 1, 2, 0: @@ -234,8 +231,6 @@ def _verify_parse_output(self, operations): for operation, expected_operation in zip(operations, expected_operations): self.assertEqual(operation, expected_operation) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parse_bytes(self): out = self.Outputter() parser = expat.ParserCreate(namespace_separator='!') @@ -248,8 +243,6 @@ def test_parse_bytes(self): # Issue #6697. self.assertRaises(AttributeError, getattr, parser, '\uD800') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parse_str(self): out = self.Outputter() parser = expat.ParserCreate(namespace_separator='!') @@ -260,8 +253,6 @@ def test_parse_str(self): operations = out.out self._verify_parse_output(operations) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parse_file(self): # Try parsing a file out = self.Outputter() @@ -274,8 +265,7 @@ def test_parse_file(self): operations = out.out self._verify_parse_output(operations) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_again(self): parser = expat.ParserCreate() file = BytesIO(data) @@ -289,8 +279,6 @@ def test_parse_again(self): expat.errors.XML_ERROR_FINISHED) class NamespaceSeparatorTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_legal(self): # Tests that make sure we get errors when the namespace_separator value # is illegal, and that we don't for good values: @@ -298,15 +286,12 @@ def test_legal(self): expat.ParserCreate(namespace_separator=None) expat.ParserCreate(namespace_separator=' ') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_illegal(self): - try: + with self.assertRaisesRegex(TypeError, + r"ParserCreate\(\) argument (2|'namespace_separator') " + r"must be str or None, not int"): expat.ParserCreate(namespace_separator=42) - self.fail() - except TypeError as e: - self.assertEqual(str(e), - "ParserCreate() argument 'namespace_separator' must be str or None, not int") try: expat.ParserCreate(namespace_separator='too long') @@ -328,9 +313,7 @@ def test_zero_length(self): class InterningTest(unittest.TestCase): - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test(self): # Test the interning machinery. p = expat.ParserCreate() @@ -346,8 +329,7 @@ def collector(name, *args): # L should have the same string repeated over and over. self.assertTrue(tag is entry) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue9402(self): # create an ExternalEntityParserCreate with buffer text class ExternalOutputter: @@ -405,8 +387,7 @@ def test_default_to_disabled(self): parser = expat.ParserCreate() self.assertFalse(parser.buffer_text) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_buffering_enabled(self): # Make sure buffering is turned on self.assertTrue(self.parser.buffer_text) @@ -414,8 +395,7 @@ def test_buffering_enabled(self): self.assertEqual(self.stuff, ['123'], "buffered text not properly collapsed") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test1(self): # XXX This test exposes more detail of Expat's text chunking than we # XXX like, but it tests what we need to concisely. @@ -425,23 +405,18 @@ def test1(self): ["<a>", "1", "<b>", "2", "\n", "3", "<c>", "4\n5"], "buffering control not reacting as expected") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test2(self): self.parser.Parse(b"<a>1<b/>&lt;2&gt;<c/>&#32;\n&#x20;3</a>", True) self.assertEqual(self.stuff, ["1<2> \n 3"], "buffered text not properly collapsed") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test3(self): self.setHandlers(["StartElementHandler"]) self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True) self.assertEqual(self.stuff, ["<a>", "1", "<b>", "2", "<c>", "3"], "buffered text not properly split") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test4(self): self.setHandlers(["StartElementHandler", "EndElementHandler"]) self.parser.CharacterDataHandler = None @@ -449,16 +424,12 @@ def test4(self): self.assertEqual(self.stuff, ["<a>", "<b>", "</b>", "<c>", "</c>", "</a>"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test5(self): self.setHandlers(["StartElementHandler", "EndElementHandler"]) self.parser.Parse(b"<a>1<b></b>2<c/>3</a>", True) self.assertEqual(self.stuff, ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", "</a>"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test6(self): self.setHandlers(["CommentHandler", "EndElementHandler", "StartElementHandler"]) @@ -467,8 +438,7 @@ def test6(self): ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "345", "</a>"], "buffered text not properly split") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test7(self): self.setHandlers(["CommentHandler", "EndElementHandler", "StartElementHandler"]) @@ -482,35 +452,59 @@ def test7(self): # Test handling of exception from callback: class HandlerExceptionTest(unittest.TestCase): def StartElementHandler(self, name, attrs): - raise RuntimeError(name) + raise RuntimeError(f'StartElementHandler: <{name}>') def check_traceback_entry(self, entry, filename, funcname): - self.assertEqual(os.path.basename(entry[0]), filename) - self.assertEqual(entry[2], funcname) + self.assertEqual(os.path.basename(entry.filename), filename) + self.assertEqual(entry.name, funcname) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_exception(self): + # gh-66652: test _PyTraceback_Add() used by pyexpat.c to inject frames + + # Change the current directory to the Python source code directory + # if it is available. + src_dir = sysconfig.get_config_var('abs_builddir') + if src_dir: + have_source = os.path.isdir(src_dir) + else: + have_source = False + if have_source: + with os_helper.change_cwd(src_dir): + self._test_exception(have_source) + else: + self._test_exception(have_source) + + def _test_exception(self, have_source): + # Use path relative to the current directory which should be the Python + # source code directory (if it is available). + PYEXPAT_C = os.path.join('Modules', 'pyexpat.c') + parser = expat.ParserCreate() parser.StartElementHandler = self.StartElementHandler try: parser.Parse(b"<a><b><c/></b></a>", True) - self.fail() - except RuntimeError as e: - self.assertEqual(e.args[0], 'a', - "Expected RuntimeError for element 'a', but" + \ - " found %r" % e.args[0]) - # Check that the traceback contains the relevant line in pyexpat.c - entries = traceback.extract_tb(e.__traceback__) - self.assertEqual(len(entries), 3) - self.check_traceback_entry(entries[0], - "test_pyexpat.py", "test_exception") - self.check_traceback_entry(entries[1], - "pyexpat.c", "StartElement") - self.check_traceback_entry(entries[2], - "test_pyexpat.py", "StartElementHandler") - if sysconfig.is_python_build() and not (sys.platform == 'win32' and platform.machine() == 'ARM'): - self.assertIn('call_with_frame("StartElement"', entries[1][3]) + + self.fail("the parser did not raise RuntimeError") + except RuntimeError as exc: + self.assertEqual(exc.args[0], 'StartElementHandler: <a>', exc) + entries = traceback.extract_tb(exc.__traceback__) + + self.assertEqual(len(entries), 3, entries) + self.check_traceback_entry(entries[0], + "test_pyexpat.py", "_test_exception") + self.check_traceback_entry(entries[1], + os.path.basename(PYEXPAT_C), + "StartElement") + self.check_traceback_entry(entries[2], + "test_pyexpat.py", "StartElementHandler") + + # Check that the traceback contains the relevant line in + # Modules/pyexpat.c. Skip the test if Modules/pyexpat.c is not + # available. + if have_source and os.path.exists(PYEXPAT_C): + self.assertIn('call_with_frame("StartElement"', + entries[1].line) # Test Current* members: @@ -533,8 +527,6 @@ def check_pos(self, event): 'Expected position %s, got position %s' %(pos, expected)) self.upto += 1 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test(self): self.parser = expat.ParserCreate() self.parser.StartElementHandler = self.StartElementHandler @@ -548,9 +540,9 @@ def test(self): class sf1296433Test(unittest.TestCase): - + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'str' but 'bytes' found. def test_parse_only_xml_data(self): - # http://python.org/sf/1296433 + # https://bugs.python.org/issue1296433 # xml = "<?xml version='1.0' encoding='iso8859'?><s>%s</s>" % ('a' * 1025) # this one doesn't crash @@ -565,25 +557,22 @@ def handler(text): parser = expat.ParserCreate() parser.CharacterDataHandler = handler - self.assertRaises(Exception, parser.Parse, xml.encode('iso8859')) + self.assertRaises(SpecificException, parser.Parse, xml.encode('iso8859')) class ChardataBufferTest(unittest.TestCase): """ test setting of chardata buffer size """ - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_1025_bytes(self): self.assertEqual(self.small_buffer_test(1025), 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_1000_bytes(self): self.assertEqual(self.small_buffer_test(1000), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_wrong_size(self): parser = expat.ParserCreate() parser.buffer_text = 1 @@ -596,8 +585,7 @@ def test_wrong_size(self): with self.assertRaises(TypeError): parser.buffer_size = 512.0 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unchanged_size(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><s>" + b'a' * 512 xml2 = b'a'*512 + b'</s>' @@ -620,8 +608,8 @@ def test_unchanged_size(self): parser.Parse(xml2) self.assertEqual(self.n, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_disabling_buffer(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>" + b'a' * 512 xml2 = b'b' * 1024 @@ -666,8 +654,7 @@ def small_buffer_test(self, buffer_len): parser.Parse(xml) return self.n - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_change_size_1(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><a><s>" + b'a' * 1024 xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>' @@ -684,8 +671,7 @@ def test_change_size_1(self): parser.Parse(xml2, True) self.assertEqual(self.n, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_change_size_2(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>a<s>" + b'a' * 1023 xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>' @@ -702,10 +688,26 @@ def test_change_size_2(self): parser.Parse(xml2, True) self.assertEqual(self.n, 4) -class MalformedInputTest(unittest.TestCase): +class ElementDeclHandlerTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by Parse + def test_trigger_leak(self): + # Unfixed, this test would leak the memory of the so-called + # "content model" in function ``my_ElementDeclHandler`` of pyexpat. + # See https://github.com/python/cpython/issues/140593. + data = textwrap.dedent('''\ + <!DOCTYPE quotations SYSTEM "quotations.dtd" [ + <!ELEMENT root ANY> + ]> + <root/> + ''').encode('UTF-8') + + parser = expat.ParserCreate() + parser.NotStandaloneHandler = lambda: 1.234 # arbitrary float + parser.ElementDeclHandler = lambda _1, _2: None + self.assertRaises(TypeError, parser.Parse, data, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure +class MalformedInputTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON def test1(self): xml = b"\0\r\n" parser = expat.ParserCreate() @@ -715,8 +717,7 @@ def test1(self): except expat.ExpatError as e: self.assertEqual(str(e), 'unclosed token: line 2, column 0') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test2(self): # \xc2\x85 is UTF-8 encoded U+0085 (NEXT LINE) xml = b"<?xml version\xc2\x85='1.0'?>\r\n" @@ -726,16 +727,13 @@ def test2(self): parser.Parse(xml, True) class ErrorMessageTest(unittest.TestCase): - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_codes(self): # verify mapping of errors.codes and errors.messages self.assertEqual(errors.XML_ERROR_SYNTAX, errors.messages[errors.codes[errors.XML_ERROR_SYNTAX]]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expaterror(self): xml = b'<' parser = expat.ParserCreate() @@ -751,9 +749,7 @@ class ForeignDTDTests(unittest.TestCase): """ Tests for the UseForeignDTD method of expat parser objects. """ - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_use_foreign_dtd(self): """ If UseForeignDTD is passed True and a document without an external @@ -782,8 +778,7 @@ def resolve_entity(context, base, system_id, public_id): parser.Parse(b"<?xml version='1.0'?><element/>") self.assertEqual(handler_call_args, [(None, None)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ignore_use_foreign_dtd(self): """ If UseForeignDTD is passed True and a document with an external @@ -804,5 +799,295 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ParentParserLifetimeTest(unittest.TestCase): + """ + Subparsers make use of their parent XML_Parser inside of Expat. + As a result, parent parsers need to outlive subparsers. + + See https://github.com/python/cpython/issues/139400. + """ + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'ExternalEntityParserCreate' + def test_parent_parser_outlives_its_subparsers__single(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'ExternalEntityParserCreate' + def test_parent_parser_outlives_its_subparsers__multiple(self): + parser = expat.ParserCreate() + subparser_one = parser.ExternalEntityParserCreate(None) + subparser_two = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'ExternalEntityParserCreate' + def test_parent_parser_outlives_its_subparsers__chain(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + subsubparser = subparser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parsers + # while they are still being referenced by a related subparser. + del parser + del subparser + + +class ReparseDeferralTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'GetReparseDeferralEnabled' + def test_getter_setter_round_trip(self): + parser = expat.ParserCreate() + enabled = (expat.version_info >= (2, 6, 0)) + + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + parser.SetReparseDeferralEnabled(False) + self.assertIs(parser.GetReparseDeferralEnabled(), False) + parser.SetReparseDeferralEnabled(True) + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'GetReparseDeferralEnabled' + def test_reparse_deferral_enabled(self): + if expat.version_info < (2, 6, 0): + self.skipTest(f'Expat {expat.version_info} does not ' + 'support reparse deferral') + + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + self.assertTrue(parser.GetReparseDeferralEnabled()) + + for chunk in (b'<doc', b'/>'): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: no. + self.assertEqual(started, []) + + parser.Parse(b'', True) + + self.assertEqual(started, ['doc']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetReparseDeferralEnabled' + def test_reparse_deferral_disabled(self): + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + if expat.version_info >= (2, 6, 0): + parser.SetReparseDeferralEnabled(False) + self.assertFalse(parser.GetReparseDeferralEnabled()) + + for chunk in (b'<doc', b'/>'): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: yes. + self.assertEqual(started, ['doc']) + + +class AttackProtectionTestBase(abc.ABC): + """ + Base class for testing protections against XML payloads with + disproportionate amplification. + + The protections being tested should detect and prevent attacks + that leverage disproportionate amplification from small inputs. + """ + + @staticmethod + def exponential_expansion_payload(*, nrows, ncols, text='.'): + """Create a billion laughs attack payload. + + Be careful: the number of total items is pow(n, k), thereby + requiring at least pow(ncols, nrows) * sizeof(text) memory! + """ + template = textwrap.dedent(f"""\ + <?xml version="1.0"?> + <!DOCTYPE doc [ + <!ENTITY row0 "{text}"> + <!ELEMENT doc (#PCDATA)> + {{body}} + ]> + <doc>&row{nrows};</doc> + """).rstrip() + + body = '\n'.join( + f'<!ENTITY row{i + 1} "{f"&row{i};" * ncols}">' + for i in range(nrows) + ) + body = textwrap.indent(body, ' ' * 4) + return template.format(body=body) + + def test_payload_generation(self): + # self-test for exponential_expansion_payload() + payload = self.exponential_expansion_payload(nrows=2, ncols=3) + self.assertEqual(payload, textwrap.dedent("""\ + <?xml version="1.0"?> + <!DOCTYPE doc [ + <!ENTITY row0 "."> + <!ELEMENT doc (#PCDATA)> + <!ENTITY row1 "&row0;&row0;&row0;"> + <!ENTITY row2 "&row1;&row1;&row1;"> + ]> + <doc>&row2;</doc> + """).rstrip()) + + def assert_root_parser_failure(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) is invalid for a sub-parser.""" + msg = "parser must be a root parser" + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + @abc.abstractmethod + def assert_rejected(self, func, /, *args, **kwargs): + """Assert that func(*args, **kwargs) triggers the attack protection. + + Note: this method must ensure that the attack protection being tested + is the one that is actually triggered at runtime, e.g., by matching + the exact error message. + """ + + @abc.abstractmethod + def set_activation_threshold(self, parser, threshold): + """Set the activation threshold for the tested protection.""" + + @abc.abstractmethod + def set_maximum_amplification(self, parser, max_factor): + """Set the maximum amplification factor for the tested protection.""" + + @abc.abstractmethod + def test_set_activation_threshold__threshold_reached(self): + """Test when the activation threshold is exceeded.""" + + @abc.abstractmethod + def test_set_activation_threshold__threshold_not_reached(self): + """Test when the activation threshold is not exceeded.""" + + def test_set_activation_threshold__invalid_threshold_type(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_activation_threshold, parser) + + self.assertRaises(TypeError, setter, 1.0) + self.assertRaises(TypeError, setter, -1.5) + self.assertRaises(ValueError, setter, -5) + + def test_set_activation_threshold__invalid_threshold_range(self): + _testcapi = import_helper.import_module("_testcapi") + parser = expat.ParserCreate() + setter = functools.partial(self.set_activation_threshold, parser) + + self.assertRaises(OverflowError, setter, _testcapi.ULLONG_MAX + 1) + + def test_set_activation_threshold__fail_for_subparser(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + setter = functools.partial(self.set_activation_threshold, subparser) + self.assert_root_parser_failure(setter, 12345) + + @abc.abstractmethod + def test_set_maximum_amplification__amplification_exceeded(self): + """Test when the amplification factor is exceeded.""" + + @abc.abstractmethod + def test_set_maximum_amplification__amplification_not_exceeded(self): + """Test when the amplification factor is not exceeded.""" + + def test_set_maximum_amplification__infinity(self): + inf = float('inf') # an 'inf' threshold is allowed by Expat + parser = expat.ParserCreate() + self.assertIsNone(self.set_maximum_amplification(parser, inf)) + + def test_set_maximum_amplification__invalid_max_factor_type(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_maximum_amplification, parser) + + self.assertRaises(TypeError, setter, None) + self.assertRaises(TypeError, setter, 'abc') + + def test_set_maximum_amplification__invalid_max_factor_range(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_maximum_amplification, parser) + + msg = re.escape("'max_factor' must be at least 1.0") + self.assertRaisesRegex(expat.ExpatError, msg, setter, float('nan')) + self.assertRaisesRegex(expat.ExpatError, msg, setter, 0.99) + + def test_set_maximum_amplification__fail_for_subparser(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + setter = functools.partial(self.set_maximum_amplification, subparser) + self.assert_root_parser_failure(setter, 123.45) + + +@unittest.skipIf(expat.version_info < (2, 7, 2), "requires Expat >= 2.7.2") +class MemoryProtectionTest(AttackProtectionTestBase, unittest.TestCase): + + # NOTE: with the default Expat configuration, the billion laughs protection + # may hit before the allocation limiter if exponential_expansion_payload() + # is not carefully parametrized. As such, the payloads should be chosen so + # that either the allocation limiter is hit before other protections are + # triggered or no protection at all is triggered. + + def assert_rejected(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) hits the allocation limit.""" + msg = r"out of memory: line \d+, column \d+" + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + def set_activation_threshold(self, parser, threshold): + return parser.SetAllocTrackerActivationThreshold(threshold) + + def set_maximum_amplification(self, parser, max_factor): + return parser.SetAllocTrackerMaximumAmplification(max_factor) + + def test_set_activation_threshold__threshold_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be always reached. + self.set_activation_threshold(parser, 3) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_activation_threshold__threshold_not_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be never reached. + self.set_activation_threshold(parser, pow(10, 5)) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assertIsNotNone(parser.Parse(payload, True)) + + def test_set_maximum_amplification__amplification_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to always be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + # Craft a payload for which the peak amplification factor is > 1.0. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_maximum_amplification__amplification_not_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to never be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1e4)) + # Craft a payload for which the peak amplification factor is < 1e4. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assertIsNotNone(parser.Parse(payload, True)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index cfa6003a867..c855fb8fe2b 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -6,10 +6,12 @@ import time import unittest import weakref -from test.support import gc_collect +from test.support import gc_collect, bigmemtest from test.support import import_helper from test.support import threading_helper +# queue module depends on threading primitives +threading_helper.requires_working_threading(module=True) py_queue = import_helper.import_fresh_module('queue', blocked=['_queue']) c_queue = import_helper.import_fresh_module('queue', fresh=['_queue']) @@ -239,6 +241,418 @@ def test_shrinking_queue(self): with self.assertRaises(self.queue.Full): q.put_nowait(4) + def test_shutdown_empty(self): + q = self.type2test() + q.shutdown() + with self.assertRaises(self.queue.ShutDown): + q.put("data") + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_nonempty(self): + q = self.type2test() + q.put("data") + q.shutdown() + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_immediate(self): + q = self.type2test() + q.put("data") + q.shutdown(immediate=True) + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_allowed_transitions(self): + # allowed transitions would be from alive via shutdown to immediate + q = self.type2test() + self.assertFalse(q.is_shutdown) + + q.shutdown() + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=True) + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=False) + + def _shutdown_all_methods_in_one_thread(self, immediate): + q = self.type2test(2) + q.put("L") + q.put_nowait("O") + q.shutdown(immediate) + + with self.assertRaises(self.queue.ShutDown): + q.put("E") + with self.assertRaises(self.queue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + with self.assertRaises(ValueError): + q.task_done() + q.join() + else: + self.assertIn(q.get(), "LO") + q.task_done() + self.assertIn(q.get(), "LO") + q.task_done() + q.join() + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(self.queue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() # p.get(False) + with self.assertRaises(self.queue.ShutDown): + q.get(True, 1.0) + + def test_shutdown_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(False) + + def test_shutdown_immediate_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(True) + + def _write_msg_thread(self, q, n, results, + i_when_exec_shutdown, event_shutdown, + barrier_start): + # All `write_msg_threads` + # put several items into the queue. + for i in range(0, i_when_exec_shutdown//2): + q.put((i, 'LOYD')) + # Wait for the barrier to be complete. + barrier_start.wait() + + for i in range(i_when_exec_shutdown//2, n): + try: + q.put((i, "YDLO")) + except self.queue.ShutDown: + results.append(False) + break + + # Trigger queue shutdown. + if i == i_when_exec_shutdown: + # Only one thread should call shutdown(). + if not event_shutdown.is_set(): + event_shutdown.set() + results.append(True) + + def _read_msg_thread(self, q, results, barrier_start): + # Get at least one item. + q.get(True) + q.task_done() + # Wait for the barrier to be complete. + barrier_start.wait() + while True: + try: + q.get(False) + q.task_done() + except self.queue.ShutDown: + results.append(True) + break + except self.queue.Empty: + pass + + def _shutdown_thread(self, q, results, event_end, immediate): + event_end.wait() + q.shutdown(immediate) + results.append(q.qsize() == 0) + + def _join_thread(self, q, barrier_start): + # Wait for the barrier to be complete. + barrier_start.wait() + q.join() + + def _shutdown_all_methods_in_many_threads(self, immediate): + # Run a 'multi-producers/consumers queue' use case, + # with enough items into the queue. + # When shutdown, all running threads will be joined. + q = self.type2test() + ps = [] + res_puts = [] + res_gets = [] + res_shutdown = [] + write_threads = 4 + read_threads = 6 + join_threads = 2 + nb_msgs = 1024*64 + nb_msgs_w = nb_msgs // write_threads + when_exec_shutdown = nb_msgs_w // 2 + # Use of a Barrier to ensure that + # - all write threads put all their items into the queue, + # - all read thread get at least one item from the queue, + # and keep on running until shutdown. + # The join thread is started only when shutdown is immediate. + nparties = write_threads + read_threads + if immediate: + nparties += join_threads + barrier_start = threading.Barrier(nparties) + ev_exec_shutdown = threading.Event() + lprocs = [ + (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, + when_exec_shutdown, ev_exec_shutdown, + barrier_start)), + (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), + (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), + ] + if immediate: + lprocs.append((self._join_thread, join_threads, (q, barrier_start))) + # start all threads. + for func, n, args in lprocs: + for i in range(n): + ps.append(threading.Thread(target=func, args=args)) + ps[-1].start() + for thread in ps: + thread.join() + + self.assertTrue(True in res_puts) + self.assertEqual(res_gets.count(True), read_threads) + if immediate: + self.assertListEqual(res_shutdown, [True]) + self.assertTrue(q.empty()) + + def test_shutdown_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(False) + + def test_shutdown_immediate_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(True) + + def _get(self, q, go, results, shutdown=False): + go.wait() + try: + msg = q.get() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _get_shutdown(self, q, go, results): + return self._get(q, go, results, True) + + def _get_task_done(self, q, go, results): + go.wait() + try: + msg = q.get() + q.task_done() + results.append(True) + return msg + except self.queue.ShutDown: + results.append(False) + return False + + def _put(self, q, msg, go, results, shutdown=False): + go.wait() + try: + q.put(msg) + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _put_shutdown(self, q, msg, go, results): + return self._put(q, msg, go, results, True) + + def _join(self, q, results, shutdown=False): + try: + q.join() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _join_shutdown(self, q, results): + return self._join(q, results, True) + + def _shutdown_get(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue full + + if immediate: + thrds = ( + (self._get_shutdown, (q, go, results)), + (self._get_shutdown, (q, go, results)), + ) + else: + thrds = ( + # on shutdown(immediate=False) + # one of these threads should raise Shutdown + (self._get, (q, go, results)), + (self._get, (q, go, results)), + (self._get, (q, go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + if immediate: + self.assertListEqual(results, [True, True]) + else: + self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) + + def test_shutdown_get(self): + return self._shutdown_get(False) + + def test_shutdown_immediate_get(self): + return self._shutdown_get(True) + + def _shutdown_put(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._put_shutdown, (q, "W", go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown() + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_put(self): + return self._shutdown_put(False) + + def test_shutdown_immediate_put(self): + return self._shutdown_put(True) + + def _shutdown_join(self, immediate): + q = self.type2test() + results = [] + q.put("Y") + go = threading.Event() + nb = q.qsize() + + thrds = ( + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + if not immediate: + res = [] + for i in range(nb): + threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_join(self): + return self._shutdown_join(True) + + def test_shutdown_join(self): + return self._shutdown_join(False) + + def _shutdown_put_join(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + # queue not fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self.assertEqual(q.unfinished_tasks, 1) + + q.shutdown(immediate) + go.set() + + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + else: + result = q.get() + self.assertEqual(result, "Y") + q.task_done() + + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_put_join(self): + return self._shutdown_put_join(True) + + def test_shutdown_put_join(self): + return self._shutdown_put_join(False) + + def test_shutdown_get_task_done_join(self): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + self.assertEqual(q.unfinished_tasks, q.qsize()) + + thrds = ( + (self._get_task_done, (q, go, results)), + (self._get_task_done, (q, go, results)), + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + go.set() + q.shutdown(False) + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_pending_get(self): + def get(): + try: + results.append(q.get()) + except Exception as e: + results.append(e) + + q = self.type2test() + results = [] + get_thread = threading.Thread(target=get) + get_thread.start() + q.shutdown(immediate=False) + get_thread.join(timeout=10.0) + self.assertFalse(get_thread.is_alive()) + self.assertEqual(len(results), 1) + self.assertIsInstance(results[0], self.queue.ShutDown) + + class QueueTest(BaseQueueTestMixin): def setUp(self): @@ -289,6 +703,7 @@ class CPriorityQueueTest(PriorityQueueTest, unittest.TestCase): # A Queue subclass that can provoke failure at a moment's notice :) class FailingQueueException(Exception): pass + class FailingQueueTest(BlockingTestMixin): def setUp(self): @@ -548,33 +963,33 @@ def test_order(self): # One producer, one consumer => results appended in well-defined order self.assertEqual(results, inputs) - def test_many_threads(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads(self, size): # Test multiple concurrent put() and get() - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, self.feed, self.consume) + results = self.run_threads(size, q, inputs, self.feed, self.consume) # Multiple consumers without synchronization append the # results in random order self.assertEqual(sorted(results), inputs) - def test_many_threads_nonblock(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_nonblock(self, size): # Test multiple concurrent put() and get(block=False) - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_nonblock) self.assertEqual(sorted(results), inputs) - def test_many_threads_timeout(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_timeout(self, size): # Test multiple concurrent put() and get(timeout=...) - N = 50 q = self.q inputs = list(range(1000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_timeout) self.assertEqual(sorted(results), inputs) diff --git a/Lib/test/test_quopri.py b/Lib/test/test_quopri.py index 715544c8a96..152d1858dcd 100644 --- a/Lib/test/test_quopri.py +++ b/Lib/test/test_quopri.py @@ -3,6 +3,7 @@ import sys, io, subprocess import quopri +from test import support ENCSAMPLE = b"""\ @@ -180,6 +181,7 @@ def test_decode_header(self): for p, e in self.HSTRINGS: self.assertEqual(quopri.decodestring(e, header=True), p) + @support.requires_subprocess() def test_scriptencode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen([sys.executable, "-mquopri"], @@ -196,6 +198,7 @@ def test_scriptencode(self): self.assertEqual(cout[i], e[i]) self.assertEqual(cout, e) + @support.requires_subprocess() def test_scriptdecode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen([sys.executable, "-mquopri", "-d"], diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 94f42c84f1a..645ef291a58 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -48,7 +48,7 @@ def test_except_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: try: raise KeyError("caught") except KeyError: @@ -60,7 +60,7 @@ def test_finally_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: try: raise KeyError("caught") finally: @@ -73,7 +73,7 @@ def nested_reraise(): def reraise(): try: raise TypeError("foo") - except: + except TypeError: nested_reraise() self.assertRaises(TypeError, reraise) @@ -81,7 +81,7 @@ def test_raise_from_None(self): try: try: raise TypeError("foo") - except: + except TypeError: raise ValueError() from None except ValueError as e: self.assertIsInstance(e.__context__, TypeError) @@ -91,7 +91,7 @@ def test_with_reraise1(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: with Context(): pass raise @@ -101,7 +101,7 @@ def test_with_reraise2(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: with Context(): raise KeyError("caught") raise @@ -111,7 +111,7 @@ def test_yield_reraise(self): def reraise(): try: raise TypeError("foo") - except: + except TypeError: yield 1 raise g = reraise() @@ -185,6 +185,16 @@ def test_class_cause(self): else: self.fail("No exception raised") + def test_class_cause_nonexception_result(self): + # See https://github.com/python/cpython/issues/140530. + class ConstructMortal(BaseException): + def __new__(*args, **kwargs): + return ["mortal value"] + + msg = ".*should have returned an instance of BaseException.*" + with self.assertRaisesRegex(TypeError, msg): + raise IndexError from ConstructMortal + def test_instance_cause(self): cause = KeyError() try: @@ -233,8 +243,6 @@ class TestTracebackType(unittest.TestCase): def raiser(self): raise ValueError - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attrs(self): try: self.raiser() @@ -270,8 +278,6 @@ def test_attrs(self): tb.tb_next = new_tb self.assertIs(tb.tb_next, new_tb) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_constructor(self): other_tb = get_tb() frame = sys._getframe() @@ -304,7 +310,7 @@ def test_instance_context_instance_raise(self): try: try: raise context - except: + except IndexError: raise OSError() except OSError as e: self.assertIs(e.__context__, context) @@ -316,7 +322,7 @@ def test_class_context_instance_raise(self): try: try: raise context - except: + except IndexError: raise OSError() except OSError as e: self.assertIsNot(e.__context__, context) @@ -329,7 +335,7 @@ def test_class_context_class_raise(self): try: try: raise context - except: + except IndexError: raise OSError except OSError as e: self.assertIsNot(e.__context__, context) @@ -341,7 +347,7 @@ def test_c_exception_context(self): try: try: 1/0 - except: + except ZeroDivisionError: raise OSError except OSError as e: self.assertIsInstance(e.__context__, ZeroDivisionError) @@ -352,7 +358,7 @@ def test_c_exception_raise(self): try: try: 1/0 - except: + except ZeroDivisionError: xyzzy except NameError as e: self.assertIsInstance(e.__context__, ZeroDivisionError) @@ -449,7 +455,7 @@ def f(): try: try: raise ValueError - except: + except ValueError: del g raise KeyError except Exception as e: @@ -465,7 +471,7 @@ class C: def __del__(self): try: 1/0 - except: + except ZeroDivisionError: raise def f(): diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 4bf00d5dbb3..0217ebd132b 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -4,6 +4,7 @@ import os import time import pickle +import shlex import warnings import test.support @@ -13,6 +14,15 @@ from fractions import Fraction from collections import abc, Counter + +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + class TestBasicOps: # Superclass with tests common to all generators. # Subclasses must arrange for self.gen to retrieve the Random instance @@ -22,8 +32,6 @@ def randomlist(self, n): """Helper function to make a list of random numbers""" return [self.gen.random() for i in range(n)] - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_autoseed(self): self.gen.seed() state1 = self.gen.getstate() @@ -32,8 +40,6 @@ def test_autoseed(self): state2 = self.gen.getstate() self.assertNotEqual(state1, state2) - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_saverestore(self): N = 1000 self.gen.seed() @@ -60,7 +66,6 @@ def __hash__(self): self.assertRaises(TypeError, self.gen.seed, 1, 2, 3, 4) self.assertRaises(TypeError, type(self.gen), []) - @unittest.skip("TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray'") def test_seed_no_mutate_bug_44018(self): a = bytearray(b'1234') self.gen.seed(a) @@ -146,6 +151,7 @@ def test_sample(self): # Exception raised if size of sample exceeds that of population self.assertRaises(ValueError, self.gen.sample, population, N+1) self.assertRaises(ValueError, self.gen.sample, [], -1) + self.assertRaises(TypeError, self.gen.sample, population, 1.0) def test_sample_distribution(self): # For the entire allowable range of 0 <= k <= N, validate that @@ -229,8 +235,6 @@ def test_sample_with_counts(self): sample(['red', 'green', 'blue'], counts=10, k=10) # counts not iterable with self.assertRaises(ValueError): sample(['red', 'green', 'blue'], counts=[-3, -7, -8], k=2) # counts are negative - with self.assertRaises(ValueError): - sample(['red', 'green', 'blue'], counts=[0, 0, 0], k=2) # counts are zero with self.assertRaises(ValueError): sample(['red', 'green'], counts=[10, 10], k=21) # population too small with self.assertRaises(ValueError): @@ -238,6 +242,20 @@ def test_sample_with_counts(self): with self.assertRaises(ValueError): sample(['red', 'green', 'blue'], counts=[1, 2, 3, 4], k=2) # too many counts + # Cases with zero counts match equivalents without counts (see gh-130285) + self.assertEqual( + sample('abc', k=0, counts=[0, 0, 0]), + sample([], k=0), + ) + self.assertEqual( + sample([], 0, counts=[]), + sample([], 0), + ) + with self.assertRaises(ValueError): + sample([], 1, counts=[]) + with self.assertRaises(ValueError): + sample('x', 1, counts=[0]) + def test_choices(self): choices = self.gen.choices data = ['red', 'green', 'blue', 'yellow'] @@ -251,6 +269,7 @@ def test_choices(self): choices(data, range(4), k=5), choices(k=5, population=data, weights=range(4)), choices(k=5, population=data, cum_weights=range(4)), + choices(data, k=MyIndex(5)), ]: self.assertEqual(len(sample), 5) self.assertEqual(type(sample), list) @@ -361,33 +380,164 @@ def test_gauss(self): self.assertEqual(x1, x2) self.assertEqual(y1, y2) + @support.requires_IEEE_754 + def test_53_bits_per_float(self): + span = 2 ** 53 + cum = 0 + for i in range(100): + cum |= int(self.gen.random() * span) + self.assertEqual(cum, span-1) + def test_getrandbits(self): + getrandbits = self.gen.getrandbits # Verify ranges for k in range(1, 1000): - self.assertTrue(0 <= self.gen.getrandbits(k) < 2**k) - self.assertEqual(self.gen.getrandbits(0), 0) + self.assertTrue(0 <= getrandbits(k) < 2**k) + self.assertEqual(getrandbits(0), 0) # Verify all bits active - getbits = self.gen.getrandbits for span in [1, 2, 3, 4, 31, 32, 32, 52, 53, 54, 119, 127, 128, 129]: all_bits = 2**span-1 cum = 0 cpl_cum = 0 for i in range(100): - v = getbits(span) + v = getrandbits(span) cum |= v cpl_cum |= all_bits ^ v self.assertEqual(cum, all_bits) self.assertEqual(cpl_cum, all_bits) # Verify argument checking - self.assertRaises(TypeError, self.gen.getrandbits) - self.assertRaises(TypeError, self.gen.getrandbits, 1, 2) - self.assertRaises(ValueError, self.gen.getrandbits, -1) - self.assertRaises(TypeError, self.gen.getrandbits, 10.1) + self.assertRaises(TypeError, getrandbits) + self.assertRaises(TypeError, getrandbits, 1, 2) + self.assertRaises(ValueError, getrandbits, -1) + self.assertRaises(OverflowError, getrandbits, 1<<1000) + self.assertRaises(ValueError, getrandbits, -1<<1000) + self.assertRaises(TypeError, getrandbits, 10.1) + + def test_bigrand(self): + # The randrange routine should build-up the required number of bits + # in stages so that all bit positions are active. + span = 2 ** 500 + cum = 0 + for i in range(100): + r = self.gen.randrange(span) + self.assertTrue(0 <= r < span) + cum |= r + self.assertEqual(cum, span-1) + + def test_bigrand_ranges(self): + for i in [40,80, 160, 200, 211, 250, 375, 512, 550]: + start = self.gen.randrange(2 ** (i-2)) + stop = self.gen.randrange(2 ** i) + if stop <= start: + continue + self.assertTrue(start <= self.gen.randrange(start, stop) < stop) + + def test_rangelimits(self): + for start, stop in [(-2,0), (-(2**60)-2,-(2**60)), (2**60,2**60+2)]: + self.assertEqual(set(range(start,stop)), + set([self.gen.randrange(start,stop) for i in range(100)])) + + def test_randrange_nonunit_step(self): + rint = self.gen.randrange(0, 10, 2) + self.assertIn(rint, (0, 2, 4, 6, 8)) + rint = self.gen.randrange(0, 2, 2) + self.assertEqual(rint, 0) + + def test_randrange_errors(self): + raises_value_error = partial(self.assertRaises, ValueError, self.gen.randrange) + raises_type_error = partial(self.assertRaises, TypeError, self.gen.randrange) + + # Empty range + raises_value_error(3, 3) + raises_value_error(-721) + raises_value_error(0, 100, -12) + + # Zero step + raises_value_error(0, 42, 0) + raises_type_error(0, 42, 0.0) + raises_type_error(0, 0, 0.0) + + # Non-integer stop + raises_type_error(3.14159) + raises_type_error(3.0) + raises_type_error(Fraction(3, 1)) + raises_type_error('3') + raises_type_error(0, 2.71827) + raises_type_error(0, 2.0) + raises_type_error(0, Fraction(2, 1)) + raises_type_error(0, '2') + raises_type_error(0, 2.71827, 2) + + # Non-integer start + raises_type_error(2.71827, 5) + raises_type_error(2.0, 5) + raises_type_error(Fraction(2, 1), 5) + raises_type_error('2', 5) + raises_type_error(2.71827, 5, 2) + + # Non-integer step + raises_type_error(0, 42, 3.14159) + raises_type_error(0, 42, 3.0) + raises_type_error(0, 42, Fraction(3, 1)) + raises_type_error(0, 42, '3') + raises_type_error(0, 42, 1.0) + raises_type_error(0, 0, 1.0) + + def test_randrange_step(self): + # bpo-42772: When stop is None, the step argument was being ignored. + randrange = self.gen.randrange + with self.assertRaises(TypeError): + randrange(1000, step=100) + with self.assertRaises(TypeError): + randrange(1000, None, step=100) + with self.assertRaises(TypeError): + randrange(1000, step=MyIndex(1)) + with self.assertRaises(TypeError): + randrange(1000, None, step=MyIndex(1)) + + def test_randbelow_logic(self, _log=log, int=int): + # check bitcount transition points: 2**i and 2**(i+1)-1 + # show that: k = int(1.001 + _log(n, 2)) + # is equal to or one greater than the number of bits in n + for i in range(1, 1000): + n = 1 << i # check an exact power of two + numbits = i+1 + k = int(1.00001 + _log(n, 2)) + self.assertEqual(k, numbits) + self.assertEqual(n, 2**(k-1)) + + n += n - 1 # check 1 below the next power of two + k = int(1.00001 + _log(n, 2)) + self.assertIn(k, [numbits, numbits+1]) + self.assertTrue(2**k > n > 2**(k-2)) + + n -= n >> 15 # check a little farther below the next power of two + k = int(1.00001 + _log(n, 2)) + self.assertEqual(k, numbits) # note the stronger assertion + self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion + + def test_randrange_index(self): + randrange = self.gen.randrange + self.assertIn(randrange(MyIndex(5)), range(5)) + self.assertIn(randrange(MyIndex(2), MyIndex(7)), range(2, 7)) + self.assertIn(randrange(MyIndex(5), MyIndex(15), MyIndex(2)), range(5, 15, 2)) + + def test_randint(self): + randint = self.gen.randint + self.assertIn(randint(2, 5), (2, 3, 4, 5)) + self.assertEqual(randint(2, 2), 2) + self.assertIn(randint(MyIndex(2), MyIndex(5)), (2, 3, 4, 5)) + self.assertEqual(randint(MyIndex(2), MyIndex(2)), 2) + + self.assertRaises(ValueError, randint, 5, 2) + self.assertRaises(TypeError, randint) + self.assertRaises(TypeError, randint, 2) + self.assertRaises(TypeError, randint, 2, 5, 1) + self.assertRaises(TypeError, randint, 2.0, 5) + self.assertRaises(TypeError, randint, 2, 5.0) - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): state = pickle.dumps(self.gen, proto) @@ -396,8 +546,6 @@ def test_pickling(self): restoredseq = [newgen.random() for i in range(10)] self.assertEqual(origseq, restoredseq) - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_bug_1727780(self): # verify that version-2-pickles can be loaded # fine, whether they are created on 32-bit or 64-bit @@ -418,6 +566,14 @@ def test_bug_9025(self): k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) self.assertTrue(0.30 < k/n < .37, (k/n)) + def test_randrange_bug_1590891(self): + start = 1000000000000 + stop = -100000000000000000000 + step = -200 + x = self.gen.randrange(start, stop, step) + self.assertTrue(stop < x <= start) + self.assertEqual((x+stop)%step, 0) + def test_randbytes(self): # Verify ranges for n in range(1, 10): @@ -431,6 +587,8 @@ def test_randbytes(self): self.assertRaises(TypeError, self.gen.randbytes) self.assertRaises(TypeError, self.gen.randbytes, 1, 2) self.assertRaises(ValueError, self.gen.randbytes, -1) + self.assertRaises(OverflowError, self.gen.randbytes, 1<<1000) + self.assertRaises((ValueError, OverflowError), self.gen.randbytes, -1<<1000) self.assertRaises(TypeError, self.gen.randbytes, 1.0) def test_mu_sigma_default_args(self): @@ -470,119 +628,6 @@ def test_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.assertRaises(NotImplementedError, pickle.dumps, self.gen, proto) - def test_53_bits_per_float(self): - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) - - def test_bigrand(self): - # The randrange routine should build-up the required number of bits - # in stages so that all bit positions are active. - span = 2 ** 500 - cum = 0 - for i in range(100): - r = self.gen.randrange(span) - self.assertTrue(0 <= r < span) - cum |= r - self.assertEqual(cum, span-1) - - def test_bigrand_ranges(self): - for i in [40,80, 160, 200, 211, 250, 375, 512, 550]: - start = self.gen.randrange(2 ** (i-2)) - stop = self.gen.randrange(2 ** i) - if stop <= start: - continue - self.assertTrue(start <= self.gen.randrange(start, stop) < stop) - - def test_rangelimits(self): - for start, stop in [(-2,0), (-(2**60)-2,-(2**60)), (2**60,2**60+2)]: - self.assertEqual(set(range(start,stop)), - set([self.gen.randrange(start,stop) for i in range(100)])) - - def test_randrange_nonunit_step(self): - rint = self.gen.randrange(0, 10, 2) - self.assertIn(rint, (0, 2, 4, 6, 8)) - rint = self.gen.randrange(0, 2, 2) - self.assertEqual(rint, 0) - - def test_randrange_errors(self): - raises = partial(self.assertRaises, ValueError, self.gen.randrange) - # Empty range - raises(3, 3) - raises(-721) - raises(0, 100, -12) - # Non-integer start/stop - self.assertWarns(DeprecationWarning, raises, 3.14159) - self.assertWarns(DeprecationWarning, self.gen.randrange, 3.0) - self.assertWarns(DeprecationWarning, self.gen.randrange, Fraction(3, 1)) - self.assertWarns(DeprecationWarning, raises, '3') - self.assertWarns(DeprecationWarning, raises, 0, 2.71828) - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 2.0) - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, Fraction(2, 1)) - self.assertWarns(DeprecationWarning, raises, 0, '2') - # Zero and non-integer step - raises(0, 42, 0) - self.assertWarns(DeprecationWarning, raises, 0, 42, 0.0) - self.assertWarns(DeprecationWarning, raises, 0, 0, 0.0) - self.assertWarns(DeprecationWarning, raises, 0, 42, 3.14159) - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, 3.0) - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, Fraction(3, 1)) - self.assertWarns(DeprecationWarning, raises, 0, 42, '3') - self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, 1.0) - self.assertWarns(DeprecationWarning, raises, 0, 0, 1.0) - - def test_randrange_argument_handling(self): - randrange = self.gen.randrange - with self.assertWarns(DeprecationWarning): - randrange(10.0, 20, 2) - with self.assertWarns(DeprecationWarning): - randrange(10, 20.0, 2) - with self.assertWarns(DeprecationWarning): - randrange(10, 20, 1.0) - with self.assertWarns(DeprecationWarning): - randrange(10, 20, 2.0) - with self.assertWarns(DeprecationWarning): - with self.assertRaises(ValueError): - randrange(10.5) - with self.assertWarns(DeprecationWarning): - with self.assertRaises(ValueError): - randrange(10, 20.5) - with self.assertWarns(DeprecationWarning): - with self.assertRaises(ValueError): - randrange(10, 20, 1.5) - - def test_randrange_step(self): - # bpo-42772: When stop is None, the step argument was being ignored. - randrange = self.gen.randrange - with self.assertRaises(TypeError): - randrange(1000, step=100) - with self.assertRaises(TypeError): - randrange(1000, None, step=100) - - def test_randbelow_logic(self, _log=log, int=int): - # check bitcount transition points: 2**i and 2**(i+1)-1 - # show that: k = int(1.001 + _log(n, 2)) - # is equal to or one greater than the number of bits in n - for i in range(1, 1000): - n = 1 << i # check an exact power of two - numbits = i+1 - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) - self.assertEqual(n, 2**(k-1)) - - n += n - 1 # check 1 below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertIn(k, [numbits, numbits+1]) - self.assertTrue(2**k > n > 2**(k-2)) - - n -= n >> 15 # check a little farther below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) # note the stronger assertion - self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion - class TestRawMersenneTwister(unittest.TestCase): @test.support.cpython_only @@ -606,11 +651,6 @@ def test_bug_42008(self): class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): gen = random.Random() - # TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray' - @unittest.expectedFailure - def test_seed_no_mutate_bug_44018(self): # TODO: RUSTPYTHON, remove when this passes - super().test_seed_no_mutate_bug_44018() # TODO: RUSTPYTHON, remove when this passes - def test_guaranteed_stable(self): # These sequences are guaranteed to stay the same across versions of python self.gen.seed(3456147, version=1) @@ -681,8 +721,6 @@ def test_bug_31482(self): def test_setstate_first_arg(self): self.assertRaises(ValueError, self.gen.setstate, (1, None, None)) - # TODO: RUSTPYTHON AttributeError: 'super' object has no attribute 'getstate' - @unittest.expectedFailure def test_setstate_middle_arg(self): start_state = self.gen.getstate() # Wrong type, s/b tuple @@ -775,38 +813,6 @@ def test_long_seed(self): seed = (1 << (10000 * 8)) - 1 # about 10K bytes self.gen.seed(seed) - def test_53_bits_per_float(self): - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) - - def test_bigrand(self): - # The randrange routine should build-up the required number of bits - # in stages so that all bit positions are active. - span = 2 ** 500 - cum = 0 - for i in range(100): - r = self.gen.randrange(span) - self.assertTrue(0 <= r < span) - cum |= r - self.assertEqual(cum, span-1) - - def test_bigrand_ranges(self): - for i in [40,80, 160, 200, 211, 250, 375, 512, 550]: - start = self.gen.randrange(2 ** (i-2)) - stop = self.gen.randrange(2 ** i) - if stop <= start: - continue - self.assertTrue(start <= self.gen.randrange(start, stop) < stop) - - def test_rangelimits(self): - for start, stop in [(-2,0), (-(2**60)-2,-(2**60)), (2**60,2**60+2)]: - self.assertEqual(set(range(start,stop)), - set([self.gen.randrange(start,stop) for i in range(100)])) - def test_getrandbits(self): super().test_getrandbits() @@ -814,6 +820,25 @@ def test_getrandbits(self): self.gen.seed(1234567) self.assertEqual(self.gen.getrandbits(100), 97904845777343510404718956115) + self.gen.seed(1234567) + self.assertEqual(self.gen.getrandbits(MyIndex(100)), + 97904845777343510404718956115) + + def test_getrandbits_2G_bits(self): + size = 2**31 + self.gen.seed(1234567) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) + self.assertEqual(x & (2**100-1), 890186470919986886340158459475) + self.assertEqual(x >> (size-100), 1226514312032729439655761284440) + + @support.bigmemtest(size=2**32, memuse=1/8+2/15, dry_run=False) + def test_getrandbits_4G_bits(self, size): + self.gen.seed(1234568) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) + self.assertEqual(x & (2**100-1), 287241425661104632871036099814) + self.assertEqual(x >> (size-100), 739728759900339699429794460738) def test_randrange_uses_getrandbits(self): # Verify use of getrandbits by randrange @@ -825,27 +850,6 @@ def test_randrange_uses_getrandbits(self): self.assertEqual(self.gen.randrange(2**99), 97904845777343510404718956115) - def test_randbelow_logic(self, _log=log, int=int): - # check bitcount transition points: 2**i and 2**(i+1)-1 - # show that: k = int(1.001 + _log(n, 2)) - # is equal to or one greater than the number of bits in n - for i in range(1, 1000): - n = 1 << i # check an exact power of two - numbits = i+1 - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) - self.assertEqual(n, 2**(k-1)) - - n += n - 1 # check 1 below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertIn(k, [numbits, numbits+1]) - self.assertTrue(2**k > n > 2**(k-2)) - - n -= n >> 15 # check a little farther below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) # note the stronger assertion - self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion - def test_randbelow_without_getrandbits(self): # Random._randbelow() can only use random() when the built-in one # has been overridden but no new getrandbits() method was supplied. @@ -880,14 +884,6 @@ def test_randbelow_without_getrandbits(self): self.gen._randbelow_without_getrandbits(n, maxsize=maxsize) self.assertEqual(random_mock.call_count, 2) - def test_randrange_bug_1590891(self): - start = 1000000000000 - stop = -100000000000000000000 - step = -200 - x = self.gen.randrange(start, stop, step) - self.assertTrue(stop < x <= start) - self.assertEqual((x+stop)%step, 0) - def test_choices_algorithms(self): # The various ways of specifying weights should produce the same results choices = self.gen.choices @@ -971,6 +967,14 @@ def test_randbytes_getrandbits(self): self.assertEqual(self.gen.randbytes(n), gen2.getrandbits(n * 8).to_bytes(n, 'little')) + @support.bigmemtest(size=2**29, memuse=1+16/15, dry_run=False) + def test_randbytes_256M(self, size): + self.gen.seed(2849427419) + x = self.gen.randbytes(size) + self.assertEqual(len(x), size) + self.assertEqual(x[:12].hex(), 'f6fd9ae63855ab91ea238b4f') + self.assertEqual(x[-12:].hex(), '0e7af69a84ee99bf4a11becc') + def test_sample_counts_equivalence(self): # Test the documented strong equivalence to a sample with repeated elements. # We run this test on random.Random() which makes deterministic selections @@ -1018,8 +1022,6 @@ def gamma(z, sqrt2pi=(2.0*pi)**0.5): ]) class TestDistributions(unittest.TestCase): - # TODO: RUSTPYTHON ValueError: math domain error - @unittest.expectedFailure def test_zeroinputs(self): # Verify that distributions can handle a series of zero inputs' g = random.Random() @@ -1027,6 +1029,7 @@ def test_zeroinputs(self): g.random = x[:].pop; g.uniform(1,10) g.random = x[:].pop; g.paretovariate(1.0) g.random = x[:].pop; g.expovariate(1.0) + g.random = x[:].pop; g.expovariate() g.random = x[:].pop; g.weibullvariate(1.0, 1.0) g.random = x[:].pop; g.vonmisesvariate(1.0, 1.0) g.random = x[:].pop; g.normalvariate(0.0, 1.0) @@ -1084,6 +1087,9 @@ def test_constant(self): (g.lognormvariate, (0.0, 0.0), 1.0), (g.lognormvariate, (-float('inf'), 0.0), 0.0), (g.normalvariate, (10.0, 0.0), 10.0), + (g.binomialvariate, (0, 0.5), 0), + (g.binomialvariate, (10, 0.0), 0), + (g.binomialvariate, (10, 1.0), 10), (g.paretovariate, (float('inf'),), 1.0), (g.weibullvariate, (10.0, float('inf')), 10.0), (g.weibullvariate, (0.0, 10.0), 0.0), @@ -1091,6 +1097,63 @@ def test_constant(self): for i in range(N): self.assertEqual(variate(*args), expected) + def test_binomialvariate(self): + B = random.binomialvariate + + # Cover all the code paths + with self.assertRaises(ValueError): + B(n=-1) # Negative n + with self.assertRaises(ValueError): + B(n=1, p=-0.5) # Negative p + with self.assertRaises(ValueError): + B(n=1, p=1.5) # p > 1.0 + self.assertEqual(B(0, 0.5), 0) # n == 0 + self.assertEqual(B(10, 0.0), 0) # p == 0.0 + self.assertEqual(B(10, 1.0), 10) # p == 1.0 + self.assertTrue(B(1, 0.3) in {0, 1}) # n == 1 fast path + self.assertTrue(B(1, 0.9) in {0, 1}) # n == 1 fast path + self.assertTrue(B(1, 0.0) in {0}) # n == 1 fast path + self.assertTrue(B(1, 1.0) in {1}) # n == 1 fast path + + # BG method very small p + self.assertEqual(B(5, 1e-18), 0) + + # BG method p <= 0.5 and n*p=1.25 + self.assertTrue(B(5, 0.25) in set(range(6))) + + # BG method p >= 0.5 and n*(1-p)=1.25 + self.assertTrue(B(5, 0.75) in set(range(6))) + + # BTRS method p <= 0.5 and n*p=25 + self.assertTrue(B(100, 0.25) in set(range(101))) + + # BTRS method p > 0.5 and n*(1-p)=25 + self.assertTrue(B(100, 0.75) in set(range(101))) + + # Statistical tests chosen such that they are + # exceedingly unlikely to ever fail for correct code. + + # BG code path + # Expected dist: [31641, 42188, 21094, 4688, 391] + c = Counter(B(4, 0.25) for i in range(100_000)) + self.assertTrue(29_641 <= c[0] <= 33_641, c) + self.assertTrue(40_188 <= c[1] <= 44_188) + self.assertTrue(19_094 <= c[2] <= 23_094) + self.assertTrue(2_688 <= c[3] <= 6_688) + self.assertEqual(set(c), {0, 1, 2, 3, 4}) + + # BTRS code path + # Sum of c[20], c[21], c[22], c[23], c[24] expected to be 36,214 + c = Counter(B(100, 0.25) for i in range(100_000)) + self.assertTrue(34_214 <= c[20]+c[21]+c[22]+c[23]+c[24] <= 38_214) + self.assertTrue(set(c) <= set(range(101))) + self.assertEqual(c.total(), 100_000) + + # Demonstrate the BTRS works for huge values of n + self.assertTrue(19_000_000 <= B(100_000_000, 0.2) <= 21_000_000) + self.assertTrue(89_000_000 <= B(100_000_000, 0.9) <= 91_000_000) + + def test_von_mises_range(self): # Issue 17149: von mises variates were not consistently in the # range [0, 2*PI]. @@ -1233,8 +1296,6 @@ def test_betavariate_return_zero(self, gammavariate_mock): class TestRandomSubclassing(unittest.TestCase): - # TODO: RUSTPYTHON Unexpected keyword argument newarg - @unittest.expectedFailure def test_random_subclass_with_kwargs(self): # SF bug #1486663 -- this used to erroneously raise a TypeError class Subclass(random.Random): @@ -1362,5 +1423,48 @@ def test_after_fork(self): support.wait_process(pid, exitcode=0) +class CommandLineTest(unittest.TestCase): + @support.force_not_colorized + def test_parse_args(self): + args, help_text = random._parse_args(shlex.split("--choice a b c")) + self.assertEqual(args.choice, ["a", "b", "c"]) + self.assertStartsWith(help_text, "usage: ") + + args, help_text = random._parse_args(shlex.split("--integer 5")) + self.assertEqual(args.integer, 5) + self.assertStartsWith(help_text, "usage: ") + + args, help_text = random._parse_args(shlex.split("--float 2.5")) + self.assertEqual(args.float, 2.5) + self.assertStartsWith(help_text, "usage: ") + + args, help_text = random._parse_args(shlex.split("a b c")) + self.assertEqual(args.input, ["a", "b", "c"]) + self.assertStartsWith(help_text, "usage: ") + + args, help_text = random._parse_args(shlex.split("5")) + self.assertEqual(args.input, ["5"]) + self.assertStartsWith(help_text, "usage: ") + + args, help_text = random._parse_args(shlex.split("2.5")) + self.assertEqual(args.input, ["2.5"]) + self.assertStartsWith(help_text, "usage: ") + + def test_main(self): + for command, expected in [ + ("--choice a b c", "b"), + ('"a b c"', "b"), + ("a b c", "b"), + ("--choice 'a a' 'b b' 'c c'", "b b"), + ("'a a' 'b b' 'c c'", "b b"), + ("--integer 5", 4), + ("5", 4), + ("--float 2.5", 2.1110546288126204), + ("2.5", 2.1110546288126204), + ]: + random.seed(0) + self.assertEqual(random.main(shlex.split(command)), expected) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 9b30b4137ca..df132aa787d 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,15 +1,24 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, - check_disallow_instantiation) + check_disallow_instantiation, linked_to_musl, + warnings_helper, SHORT_TIMEOUT, Stopwatch, requires_resource) import locale import re -import sre_compile import string +import sys import unittest import warnings from re import Scanner from weakref import proxy +# some platforms lack working multiprocessing +try: + import _multiprocessing # noqa: F401 +except ImportError: + multiprocessing = None +else: + import multiprocessing + # Misc tests from Tim Peters' re.doc # WARNING: Don't change details in these tests if you don't know @@ -37,7 +46,7 @@ def recurse(actual, expect): recurse(actual, expect) def checkPatternError(self, pattern, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(pattern) with self.subTest(pattern=pattern): err = cm.exception @@ -46,7 +55,7 @@ def checkPatternError(self, pattern, errmsg, pos=None): self.assertEqual(err.pos, pos) def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.sub(pattern, repl, string) with self.subTest(pattern=pattern, repl=repl): err = cm.exception @@ -54,8 +63,10 @@ def checkTemplateError(self, pattern, repl, string, errmsg, pos=None): if pos is not None: self.assertEqual(err.pos, pos) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_error_is_PatternError_alias(self): + assert re.error is re.PatternError + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keep_buffer(self): # See bug 14212 b = bytearray(b'x') @@ -85,6 +96,23 @@ def test_search_star_plus(self): self.assertEqual(re.match('x*', 'xxxa').span(), (0, 3)) self.assertIsNone(re.match('a+', 'xxx')) + def test_branching(self): + """Test Branching + Test expressions using the OR ('|') operator.""" + self.assertEqual(re.match('(ab|ba)', 'ab').span(), (0, 2)) + self.assertEqual(re.match('(ab|ba)', 'ba').span(), (0, 2)) + self.assertEqual(re.match('(abc|bac|ca|cb)', 'abc').span(), + (0, 3)) + self.assertEqual(re.match('(abc|bac|ca|cb)', 'bac').span(), + (0, 3)) + self.assertEqual(re.match('(abc|bac|ca|cb)', 'ca').span(), + (0, 2)) + self.assertEqual(re.match('(abc|bac|ca|cb)', 'cb').span(), + (0, 2)) + self.assertEqual(re.match('((a)|(b)|(c))', 'a').span(), (0, 1)) + self.assertEqual(re.match('((a)|(b)|(c))', 'b').span(), (0, 1)) + self.assertEqual(re.match('((a)|(b)|(c))', 'c').span(), (0, 1)) + def bump_num(self, matchobj): int_value = int(matchobj.group(0)) return str(int_value + 1) @@ -102,8 +130,10 @@ def test_basic_re_sub(self): self.assertEqual(re.sub("(?i)b+", "x", "bbbb BBBB"), 'x x') self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y'), '9.3 -3 24x100y') - self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), - '9.3 -3 23x99y') + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), + '9.3 -3 23x99y') + self.assertEqual(w.filename, __file__) self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', count=3), '9.3 -3 23x99y') @@ -119,6 +149,7 @@ def test_basic_re_sub(self): self.assertEqual(re.sub('(?P<a>x)', r'\g<a>\g<1>', 'xx'), 'xxxx') self.assertEqual(re.sub('(?P<unk>x)', r'\g<unk>\g<unk>', 'xx'), 'xxxx') self.assertEqual(re.sub('(?P<unk>x)', r'\g<1>\g<1>', 'xx'), 'xxxx') + self.assertEqual(re.sub('()x', r'\g<0>\g<0>', 'xx'), 'xxxx') self.assertEqual(re.sub('a', r'\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b') self.assertEqual(re.sub('a', '\t\n\v\r\f\a\b', 'a'), '\t\n\v\r\f\a\b') @@ -126,7 +157,7 @@ def test_basic_re_sub(self): (chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)+chr(8))) for c in 'cdehijklmopqsuwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ': with self.subTest(c): - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): self.assertEqual(re.sub('a', '\\' + c, 'a'), '\\' + c) self.assertEqual(re.sub(r'^\s*', 'X', 'test'), 'Xtest') @@ -209,9 +240,42 @@ def test_sub_template_numeric_escape(self): def test_qualified_re_sub(self): self.assertEqual(re.sub('a', 'b', 'aaaaa'), 'bbbbb') - self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + self.assertEqual(w.filename, __file__) self.assertEqual(re.sub('a', 'b', 'aaaaa', count=1), 'baaaa') + with self.assertRaisesRegex(TypeError, + r"sub\(\) got multiple values for argument 'count'"): + re.sub('a', 'b', 'aaaaa', 1, count=1) + with self.assertRaisesRegex(TypeError, + r"sub\(\) got multiple values for argument 'flags'"): + re.sub('a', 'b', 'aaaaa', 1, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"sub\(\) takes from 3 to 5 positional arguments but 6 " + r"were given"): + re.sub('a', 'b', 'aaaaa', 1, 0, 0) + + def test_misuse_flags(self): + with self.assertWarns(DeprecationWarning) as w: + result = re.sub('a', 'b', 'aaaaa', re.I) + self.assertEqual(result, re.sub('a', 'b', 'aaaaa', count=int(re.I))) + self.assertEqual(str(w.warning), + "'count' is passed as positional argument") + self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + result = re.subn("b*", "x", "xyz", re.I) + self.assertEqual(result, re.subn("b*", "x", "xyz", count=int(re.I))) + self.assertEqual(str(w.warning), + "'count' is passed as positional argument") + self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + result = re.split(":", ":a:b::c", re.I) + self.assertEqual(result, re.split(":", ":a:b::c", maxsplit=int(re.I))) + self.assertEqual(str(w.warning), + "'maxsplit' is passed as positional argument") + self.assertEqual(w.filename, __file__) + def test_bug_114660(self): self.assertEqual(re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there'), 'hello there') @@ -258,6 +322,12 @@ def test_symbolic_groups_errors(self): self.checkPatternError('(?P<©>x)', "bad character in group name '©'", 4) self.checkPatternError('(?P=©)', "bad character in group name '©'", 4) self.checkPatternError('(?(©)y)', "bad character in group name '©'", 3) + self.checkPatternError(b'(?P<\xc2\xb5>x)', + r"bad character in group name '\xc2\xb5'", 4) + self.checkPatternError(b'(?P=\xc2\xb5)', + r"bad character in group name '\xc2\xb5'", 4) + self.checkPatternError(b'(?(\xc2\xb5)y)', + r"bad character in group name '\xc2\xb5'", 3) def test_symbolic_refs(self): self.assertEqual(re.sub('(?P<a>x)|(?P<b>y)', r'\g<b>', 'xx'), '') @@ -290,21 +360,44 @@ def test_symbolic_refs_errors(self): re.sub('(?P<a>x)', r'\g<ab>', 'xx') self.checkTemplateError('(?P<a>x)', r'\g<-1>', 'xx', "bad character in group name '-1'", 3) + self.checkTemplateError('(?P<a>x)', r'\g<+1>', 'xx', + "bad character in group name '+1'", 3) + self.checkTemplateError('()'*10, r'\g<1_0>', 'xx', + "bad character in group name '1_0'", 3) + self.checkTemplateError('(?P<a>x)', r'\g< 1 >', 'xx', + "bad character in group name ' 1 '", 3) self.checkTemplateError('(?P<a>x)', r'\g<©>', 'xx', "bad character in group name '©'", 3) + self.checkTemplateError(b'(?P<a>x)', b'\\g<\xc2\xb5>', b'xx', + r"bad character in group name '\xc2\xb5'", 3) self.checkTemplateError('(?P<a>x)', r'\g<㊀>', 'xx', "bad character in group name '㊀'", 3) self.checkTemplateError('(?P<a>x)', r'\g<¹>', 'xx', "bad character in group name '¹'", 3) + self.checkTemplateError('(?P<a>x)', r'\g<१>', 'xx', + "bad character in group name '१'", 3) def test_re_subn(self): self.assertEqual(re.subn("(?i)b+", "x", "bbbb BBBB"), ('x x', 2)) self.assertEqual(re.subn("b+", "x", "bbbb BBBB"), ('x BBBB', 1)) self.assertEqual(re.subn("b+", "x", "xyz"), ('xyz', 0)) self.assertEqual(re.subn("b*", "x", "xyz"), ('xxxyxzx', 4)) - self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + self.assertEqual(w.filename, __file__) self.assertEqual(re.subn("b*", "x", "xyz", count=2), ('xxxyz', 2)) + with self.assertRaisesRegex(TypeError, + r"subn\(\) got multiple values for argument 'count'"): + re.subn('a', 'b', 'aaaaa', 1, count=1) + with self.assertRaisesRegex(TypeError, + r"subn\(\) got multiple values for argument 'flags'"): + re.subn('a', 'b', 'aaaaa', 1, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"subn\(\) takes from 3 to 5 positional arguments but 6 " + r"were given"): + re.subn('a', 'b', 'aaaaa', 1, 0, 0) + def test_re_split(self): for string in ":a:b::c", S(":a:b::c"): self.assertTypedEqual(re.split(":", string), @@ -359,7 +452,9 @@ def test_re_split(self): self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) def test_qualified_re_split(self): - self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + self.assertEqual(w.filename, __file__) self.assertEqual(re.split(":", ":a:b::c", maxsplit=2), ['', 'a', 'b::c']) self.assertEqual(re.split(':', 'a:b:c:d', maxsplit=2), ['a', 'b', 'c:d']) self.assertEqual(re.split("(:)", ":a:b::c", maxsplit=2), @@ -369,6 +464,17 @@ def test_qualified_re_split(self): self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), ['', ':', '', '', 'a:b::c']) + with self.assertRaisesRegex(TypeError, + r"split\(\) got multiple values for argument 'maxsplit'"): + re.split(":", ":a:b::c", 2, maxsplit=2) + with self.assertRaisesRegex(TypeError, + r"split\(\) got multiple values for argument 'flags'"): + re.split(":", ":a:b::c", 2, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"split\(\) takes from 2 to 4 positional arguments but 5 " + r"were given"): + re.split(":", ":a:b::c", 2, 0, 0) + def test_re_findall(self): self.assertEqual(re.findall(":+", "abc"), []) for string in "a:b::c:::d", S("a:b::c:::d"): @@ -514,6 +620,7 @@ def test_re_fullmatch(self): self.assertEqual(re.fullmatch(r"a.*?b", "axxb").span(), (0, 4)) self.assertIsNone(re.fullmatch(r"a+", "ab")) self.assertIsNone(re.fullmatch(r"abc$", "abc\n")) + self.assertIsNone(re.fullmatch(r"abc\z", "abc\n")) self.assertIsNone(re.fullmatch(r"abc\Z", "abc\n")) self.assertIsNone(re.fullmatch(r"(?m)abc$", "abc\n")) self.assertEqual(re.fullmatch(r"ab(?=c)cd", "abcd").span(), (0, 4)) @@ -557,16 +664,22 @@ def test_re_groupref_exists(self): pat = '(?:%s)(?(200)z)' % pat self.assertEqual(re.match(pat, 'xc8yz').span(), (0, 5)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_re_groupref_exists_errors(self): self.checkPatternError(r'(?P<a>)(?(0)a|b)', 'bad group number', 10) self.checkPatternError(r'()(?(-1)a|b)', "bad character in group name '-1'", 5) + self.checkPatternError(r'()(?(+1)a|b)', + "bad character in group name '+1'", 5) + self.checkPatternError(r'()'*10 + r'(?(1_0)a|b)', + "bad character in group name '1_0'", 23) + self.checkPatternError(r'()(?( 1 )a|b)', + "bad character in group name ' 1 '", 5) self.checkPatternError(r'()(?(㊀)a|b)', "bad character in group name '㊀'", 5) self.checkPatternError(r'()(?(¹)a|b)', "bad character in group name '¹'", 5) + self.checkPatternError(r'()(?(१)a|b)', + "bad character in group name '१'", 5) self.checkPatternError(r'()(?(1', "missing ), unterminated name", 5) self.checkPatternError(r'()(?(1)a', @@ -582,8 +695,13 @@ def test_re_groupref_exists_errors(self): self.checkPatternError(r'()(?(2)a)', "invalid group reference 2", 5) + def test_re_groupref_exists_validation_bug(self): + for i in range(256): + with self.subTest(code=i): + re.compile(r'()(?(1)\x%02x?)' % i) + def test_re_groupref_overflow(self): - from sre_constants import MAXGROUPS + from re._constants import MAXGROUPS self.checkTemplateError('()', r'\g<%s>' % MAXGROUPS, 'xx', 'invalid group reference %d' % MAXGROUPS, 3) self.checkPatternError(r'(?P<a>)(?(%d))' % MAXGROUPS, @@ -608,6 +726,7 @@ def test_groupdict(self): 'first second').groupdict(), {'first':'first', 'second':'second'}) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expand(self): self.assertEqual(re.match("(?P<first>first) (?P<second>second)", "first second") @@ -654,8 +773,7 @@ def test_repeat_minmax(self): self.checkPatternError(r'x{2,1}', 'min repeat greater than max repeat', 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_getattr(self): self.assertEqual(re.compile("(?i)(a)(b)").pattern, "(?i)(a)(b)") self.assertEqual(re.compile("(?i)(a)(b)").flags, re.I | re.U) @@ -688,6 +806,8 @@ def test_special_escapes(self): self.assertEqual(re.search(r"\B(b.)\B", "abc bcd bc abxd", re.ASCII).group(1), "bx") self.assertEqual(re.search(r"^abc$", "\nabc\n", re.M).group(0), "abc") + self.assertEqual(re.search(r"^\Aabc\z$", "abc", re.M).group(0), "abc") + self.assertIsNone(re.search(r"^\Aabc\z$", "\nabc\n", re.M)) self.assertEqual(re.search(r"^\Aabc\Z$", "abc", re.M).group(0), "abc") self.assertIsNone(re.search(r"^\Aabc\Z$", "\nabc\n", re.M)) self.assertEqual(re.search(br"\b(b.)\b", @@ -699,6 +819,8 @@ def test_special_escapes(self): self.assertEqual(re.search(br"\B(b.)\B", b"abc bcd bc abxd", re.LOCALE).group(1), b"bx") self.assertEqual(re.search(br"^abc$", b"\nabc\n", re.M).group(0), b"abc") + self.assertEqual(re.search(br"^\Aabc\z$", b"abc", re.M).group(0), b"abc") + self.assertIsNone(re.search(br"^\Aabc\z$", b"\nabc\n", re.M)) self.assertEqual(re.search(br"^\Aabc\Z$", b"abc", re.M).group(0), b"abc") self.assertIsNone(re.search(br"^\Aabc\Z$", b"\nabc\n", re.M)) self.assertEqual(re.search(r"\d\D\w\W\s\S", @@ -722,15 +844,13 @@ def test_other_escapes(self): self.assertEqual(re.match(r"[\^a]+", 'a^').group(), 'a^') self.assertIsNone(re.match(r"[\^a]+", 'b')) re.purge() # for warnings - for c in 'ceghijklmopqyzCEFGHIJKLMNOPQRTVXY': + for c in 'ceghijklmopqyCEFGHIJKLMNOPQRTVXY': with self.subTest(c): - self.assertRaises(re.error, re.compile, '\\%c' % c) + self.assertRaises(re.PatternError, re.compile, '\\%c' % c) for c in 'ceghijklmopqyzABCEFGHIJKLMNOPQRTVXYZ': with self.subTest(c): - self.assertRaises(re.error, re.compile, '[\\%c]' % c) + self.assertRaises(re.PatternError, re.compile, '[\\%c]' % c) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_named_unicode_escapes(self): # test individual Unicode named escapes self.assertTrue(re.match(r'\N{LESS-THAN SIGN}', '<')) @@ -771,34 +891,137 @@ def test_named_unicode_escapes(self): self.checkPatternError(br'\N{LESS-THAN SIGN}', r'bad escape \N', 0) self.checkPatternError(br'[\N{LESS-THAN SIGN}]', r'bad escape \N', 1) - def test_string_boundaries(self): + # TODO: RUSTPYTHON; re.search(r"\B", "") now returns a match in CPython 3.14 + @unittest.expectedFailure + def test_word_boundaries(self): # See http://bugs.python.org/issue10713 - self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), - "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc", re.ASCII).group(1), "abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc").group(1), b"abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc", re.LOCALE).group(1), b"abc") + self.assertEqual(re.search(r"\b(ьюя)\b", "ьюя").group(1), "ьюя") + self.assertIsNone(re.search(r"\b(ьюя)\b", "ьюя", re.ASCII)) + # There's a word boundary between a word and a non-word. + self.assertTrue(re.match(r".\b", "a=")) + self.assertTrue(re.match(r".\b", "a=", re.ASCII)) + self.assertTrue(re.match(br".\b", b"a=")) + self.assertTrue(re.match(br".\b", b"a=", re.LOCALE)) + self.assertTrue(re.match(r".\b", "я=")) + self.assertIsNone(re.match(r".\b", "я=", re.ASCII)) + # There's a word boundary between a non-word and a word. + self.assertTrue(re.match(r".\b", "=a")) + self.assertTrue(re.match(r".\b", "=a", re.ASCII)) + self.assertTrue(re.match(br".\b", b"=a")) + self.assertTrue(re.match(br".\b", b"=a", re.LOCALE)) + self.assertTrue(re.match(r".\b", "=я")) + self.assertIsNone(re.match(r".\b", "=я", re.ASCII)) + # There is no word boundary inside a word. + self.assertIsNone(re.match(r".\b", "ab")) + self.assertIsNone(re.match(r".\b", "ab", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"ab")) + self.assertIsNone(re.match(br".\b", b"ab", re.LOCALE)) + self.assertIsNone(re.match(r".\b", "юя")) + self.assertIsNone(re.match(r".\b", "юя", re.ASCII)) + # There is no word boundary between a non-word characters. + self.assertIsNone(re.match(r".\b", "=-")) + self.assertIsNone(re.match(r".\b", "=-", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"=-")) + self.assertIsNone(re.match(br".\b", b"=-", re.LOCALE)) + # There is no non-boundary match between a word and a non-word. + self.assertIsNone(re.match(r".\B", "a=")) + self.assertIsNone(re.match(r".\B", "a=", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"a=")) + self.assertIsNone(re.match(br".\B", b"a=", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "я=")) + self.assertTrue(re.match(r".\B", "я=", re.ASCII)) + # There is no non-boundary match between a non-word and a word. + self.assertIsNone(re.match(r".\B", "=a")) + self.assertIsNone(re.match(r".\B", "=a", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"=a")) + self.assertIsNone(re.match(br".\B", b"=a", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "=я")) + self.assertTrue(re.match(r".\B", "=я", re.ASCII)) + # There's a non-boundary match inside a word. + self.assertTrue(re.match(r".\B", "ab")) + self.assertTrue(re.match(r".\B", "ab", re.ASCII)) + self.assertTrue(re.match(br".\B", b"ab")) + self.assertTrue(re.match(br".\B", b"ab", re.LOCALE)) + self.assertTrue(re.match(r".\B", "юя")) + self.assertTrue(re.match(r".\B", "юя", re.ASCII)) + # There's a non-boundary match between a non-word characters. + self.assertTrue(re.match(r".\B", "=-")) + self.assertTrue(re.match(r".\B", "=-", re.ASCII)) + self.assertTrue(re.match(br".\B", b"=-")) + self.assertTrue(re.match(br".\B", b"=-", re.LOCALE)) # There's a word boundary at the start of a string. self.assertTrue(re.match(r"\b", "abc")) + self.assertTrue(re.match(r"\b", "abc", re.ASCII)) + self.assertTrue(re.match(br"\b", b"abc")) + self.assertTrue(re.match(br"\b", b"abc", re.LOCALE)) + self.assertTrue(re.match(r"\b", "ьюя")) + self.assertIsNone(re.match(r"\b", "ьюя", re.ASCII)) + # There's a word boundary at the end of a string. + self.assertTrue(re.fullmatch(r".+\b", "abc")) + self.assertTrue(re.fullmatch(r".+\b", "abc", re.ASCII)) + self.assertTrue(re.fullmatch(br".+\b", b"abc")) + self.assertTrue(re.fullmatch(br".+\b", b"abc", re.LOCALE)) + self.assertTrue(re.fullmatch(r".+\b", "ьюя")) + self.assertIsNone(re.search(r"\b", "ьюя", re.ASCII)) # A non-empty string includes a non-boundary zero-length match. - self.assertTrue(re.search(r"\B", "abc")) + self.assertEqual(re.search(r"\B", "abc").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "abc", re.ASCII).span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc").span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc", re.LOCALE).span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя", re.ASCII).span(), (0, 0)) # There is no non-boundary match at the start of a string. - self.assertFalse(re.match(r"\B", "abc")) - # However, an empty string contains no word boundaries, and also no - # non-boundaries. - self.assertIsNone(re.search(r"\B", "")) - # This one is questionable and different from the perlre behaviour, - # but describes current behavior. + self.assertIsNone(re.match(r"\B", "abc")) + self.assertIsNone(re.match(r"\B", "abc", re.ASCII)) + self.assertIsNone(re.match(br"\B", b"abc")) + self.assertIsNone(re.match(br"\B", b"abc", re.LOCALE)) + self.assertIsNone(re.match(r"\B", "ьюя")) + self.assertTrue(re.match(r"\B", "ьюя", re.ASCII)) + # There is no non-boundary match at the end of a string. + self.assertIsNone(re.fullmatch(r".+\B", "abc")) + self.assertIsNone(re.fullmatch(r".+\B", "abc", re.ASCII)) + self.assertIsNone(re.fullmatch(br".+\B", b"abc")) + self.assertIsNone(re.fullmatch(br".+\B", b"abc", re.LOCALE)) + self.assertIsNone(re.fullmatch(r".+\B", "ьюя")) + self.assertTrue(re.fullmatch(r".+\B", "ьюя", re.ASCII)) + # However, an empty string contains no word boundaries. self.assertIsNone(re.search(r"\b", "")) + self.assertIsNone(re.search(r"\b", "", re.ASCII)) + self.assertIsNone(re.search(br"\b", b"")) + self.assertIsNone(re.search(br"\b", b"", re.LOCALE)) + self.assertTrue(re.search(r"\B", "")) + self.assertTrue(re.search(r"\B", "", re.ASCII)) + self.assertTrue(re.search(br"\B", b"")) + self.assertTrue(re.search(br"\B", b"", re.LOCALE)) # A single word-character string has two boundaries, but no # non-boundary gaps. self.assertEqual(len(re.findall(r"\b", "a")), 2) + self.assertEqual(len(re.findall(r"\b", "a", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\b", b"a")), 2) + self.assertEqual(len(re.findall(br"\b", b"a", re.LOCALE)), 2) self.assertEqual(len(re.findall(r"\B", "a")), 0) + self.assertEqual(len(re.findall(r"\B", "a", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\B", b"a")), 0) + self.assertEqual(len(re.findall(br"\B", b"a", re.LOCALE)), 0) # If there are no words, there are no boundaries self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) # Can match around the whitespace. self.assertEqual(len(re.findall(r"\B", " ")), 2) + self.assertEqual(len(re.findall(r"\B", " ", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\B", b" ")), 2) + self.assertEqual(len(re.findall(br"\B", b" ", re.LOCALE)), 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bigcharset(self): self.assertEqual(re.match("([\u2222\u2223])", "\u2222").group(1), "\u2222") @@ -862,17 +1085,15 @@ def test_lookbehind(self): self.assertIsNone(re.match(r'(?:(a)|(x))b(?<=(?(1)c|x))c', 'abc')) self.assertTrue(re.match(r'(?:(a)|(x))b(?<=(?(1)b|x))c', 'abc')) # Group used before defined. - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?(2)b|x))(c)') self.assertIsNone(re.match(r'(a)b(?<=(?(1)c|x))(c)', 'abc')) self.assertTrue(re.match(r'(a)b(?<=(?(1)b|x))(c)', 'abc')) # Group defined in the same lookbehind pattern - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)\2)(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(?P<a>.)(?P=a))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') - self.assertRaises(re.error, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)\2)(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(?P<a>.)(?P=a))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(a)(?(2)b|x))(c)') + self.assertRaises(re.PatternError, re.compile, r'(a)b(?<=(.)(?<=\2))(c)') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ignore_case(self): self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC") self.assertEqual(re.match(b"abc", b"ABC", re.I).group(0), b"ABC") @@ -913,8 +1134,6 @@ def test_ignore_case(self): self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I)) self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ignore_case_set(self): self.assertTrue(re.match(r'[19A]', 'A', re.I)) self.assertTrue(re.match(r'[19a]', 'a', re.I)) @@ -924,6 +1143,39 @@ def test_ignore_case_set(self): self.assertTrue(re.match(br'[19a]', b'a', re.I)) self.assertTrue(re.match(br'[19a]', b'A', re.I)) self.assertTrue(re.match(br'[19A]', b'a', re.I)) + self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I)) + self.assertTrue(re.match(r'[19\xc7]', '\xe7', re.I)) + self.assertTrue(re.match(r'[19\xe7]', '\xc7', re.I)) + self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I)) + self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I)) + self.assertTrue(re.match(r'[19\u0400]', '\u0450', re.I)) + self.assertTrue(re.match(r'[19\u0450]', '\u0400', re.I)) + self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010428', re.I)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I)) + + self.assertTrue(re.match(br'[19A]', b'A', re.I)) + self.assertTrue(re.match(br'[19a]', b'a', re.I)) + self.assertTrue(re.match(br'[19a]', b'A', re.I)) + self.assertTrue(re.match(br'[19A]', b'a', re.I)) + self.assertTrue(re.match(r'[19A]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[19a]', 'a', re.I|re.A)) + self.assertTrue(re.match(r'[19a]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[19A]', 'a', re.I|re.A)) + self.assertTrue(re.match(r'[19\xc7]', '\xc7', re.I|re.A)) + self.assertIsNone(re.match(r'[19\xc7]', '\xe7', re.I|re.A)) + self.assertIsNone(re.match(r'[19\xe7]', '\xc7', re.I|re.A)) + self.assertTrue(re.match(r'[19\xe7]', '\xe7', re.I|re.A)) + self.assertTrue(re.match(r'[19\u0400]', '\u0400', re.I|re.A)) + self.assertIsNone(re.match(r'[19\u0400]', '\u0450', re.I|re.A)) + self.assertIsNone(re.match(r'[19\u0450]', '\u0400', re.I|re.A)) + self.assertTrue(re.match(r'[19\u0450]', '\u0450', re.I|re.A)) + self.assertTrue(re.match(r'[19\U00010400]', '\U00010400', re.I|re.A)) + self.assertIsNone(re.match(r'[19\U00010400]', '\U00010428', re.I|re.A)) + self.assertIsNone(re.match(r'[19\U00010428]', '\U00010400', re.I|re.A)) + self.assertTrue(re.match(r'[19\U00010428]', '\U00010428', re.I|re.A)) # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' @@ -953,8 +1205,6 @@ def test_ignore_case_set(self): self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I)) self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ignore_case_range(self): # Issues #3511, #17381. self.assertTrue(re.match(r'[9-a]', '_', re.I)) @@ -962,8 +1212,10 @@ def test_ignore_case_range(self): self.assertTrue(re.match(br'[9-a]', b'_', re.I)) self.assertIsNone(re.match(br'[9-A]', b'_', re.I)) self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xe7', re.I)) self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I)) self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xc7', re.I)) self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I)) self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I)) self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0400', re.I)) @@ -974,6 +1226,26 @@ def test_ignore_case_range(self): self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I)) self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I)) + self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I|re.A)) + self.assertIsNone(re.match(r'[\xc0-\xde]', '\xe7', re.I|re.A)) + self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7', re.I|re.A)) + self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xc7', re.I|re.A)) + self.assertTrue(re.match(r'[\u0430-\u045f]', '\u0450', re.I|re.A)) + self.assertIsNone(re.match(r'[\u0430-\u045f]', '\u0400', re.I|re.A)) + self.assertIsNone(re.match(r'[\u0400-\u042f]', '\u0450', re.I|re.A)) + self.assertTrue(re.match(r'[\u0400-\u042f]', '\u0400', re.I|re.A)) + self.assertTrue(re.match(r'[\U00010428-\U0001044f]', '\U00010428', re.I|re.A)) + self.assertIsNone(re.match(r'[\U00010428-\U0001044f]', '\U00010400', re.I|re.A)) + self.assertIsNone(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I|re.A)) + self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I|re.A)) + + self.assertTrue(re.match(r'[N-\x7f]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\x7f]', 'Z', re.I|re.A)) + self.assertTrue(re.match(r'[N-\uffff]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\uffff]', 'Z', re.I|re.A)) + self.assertTrue(re.match(r'[N-\U00010000]', 'A', re.I|re.A)) + self.assertTrue(re.match(r'[n-\U00010000]', 'Z', re.I|re.A)) + # Two different characters have the same lowercase. assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K' self.assertTrue(re.match(r'[J-M]', '\u212a', re.I)) @@ -1005,80 +1277,82 @@ def test_ignore_case_range(self): def test_category(self): self.assertEqual(re.match(r"(\s)", " ").group(1), " ") - @cpython_only - def test_case_helpers(self): - import _sre - for i in range(128): - c = chr(i) - lo = ord(c.lower()) - self.assertEqual(_sre.ascii_tolower(i), lo) - self.assertEqual(_sre.unicode_tolower(i), lo) - iscased = c in string.ascii_letters - self.assertEqual(_sre.ascii_iscased(i), iscased) - self.assertEqual(_sre.unicode_iscased(i), iscased) - - for i in list(range(128, 0x1000)) + [0x10400, 0x10428]: - c = chr(i) - self.assertEqual(_sre.ascii_tolower(i), i) - if i != 0x0130: - self.assertEqual(_sre.unicode_tolower(i), ord(c.lower())) - iscased = c != c.lower() or c != c.upper() - self.assertFalse(_sre.ascii_iscased(i)) - self.assertEqual(_sre.unicode_iscased(i), - c != c.lower() or c != c.upper()) - - self.assertEqual(_sre.ascii_tolower(0x0130), 0x0130) - self.assertEqual(_sre.unicode_tolower(0x0130), ord('i')) - self.assertFalse(_sre.ascii_iscased(0x0130)) - self.assertTrue(_sre.unicode_iscased(0x0130)) - def test_not_literal(self): self.assertEqual(re.search(r"\s([^a])", " b").group(1), "b") self.assertEqual(re.search(r"\s([^a]*)", " bb").group(1), "bb") def test_possible_set_operations(self): s = bytes(range(128)).decode() - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[0-9--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('-./0123456789')) + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: + self.assertEqual(re.findall(r'[0-9--2]', s), list('-./0123456789')) + self.assertEqual(w.filename, __file__) + self.assertEqual(re.findall(r'[--1]', s), list('-./01')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: p = re.compile(r'[%--1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-1")) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set difference ') as w: p = re.compile(r'[%--]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list("%&'()*+,-")) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[0-9&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: + self.assertEqual(re.findall(r'[0-8&&1]', s), list('&012345678')) + self.assertEqual(w.filename, __file__) + + with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: p = re.compile(r'[\d&&1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('&0123456789')) + self.assertEqual(re.findall(r'[&&1]', s), list('&1')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[0-9||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: p = re.compile(r'[\d||a]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789a|')) + self.assertEqual(re.findall(r'[||1]', s), list('1|')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[0-9~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) - with self.assertWarns(FutureWarning): + + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[\d~~1]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) + self.assertEqual(re.findall(r'[~~1]', s), list('1~')) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[0-9]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789[]')) + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: + self.assertEqual(re.findall(r'[[0-8]|]', s), list('012345678[]')) + self.assertEqual(w.filename, __file__) - with self.assertWarns(FutureWarning): + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: p = re.compile(r'[[:digit:]|]') + self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list(':[]dgit')) def test_search_coverage(self): @@ -1151,10 +1425,9 @@ def test_pickling(self): newpat = pickle.loads(pickled) self.assertEqual(newpat, oldpat) # current pickle expects the _compile() reconstructor in re module - from re import _compile + from re import _compile # noqa: F401 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_copying(self): import copy p = re.compile(r'(?P<int>\d+)(?:\.(?P<frac>\d*))?') @@ -1245,8 +1518,8 @@ def test_sre_byte_literals(self): self.assertTrue(re.match((r"\x%02x" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"\x%02x0" % i).encode(), bytes([i])+b"0")) self.assertTrue(re.match((r"\x%02xz" % i).encode(), bytes([i])+b"z")) - self.assertRaises(re.error, re.compile, br"\u1234") - self.assertRaises(re.error, re.compile, br"\U00012345") + self.assertRaises(re.PatternError, re.compile, br"\u1234") + self.assertRaises(re.PatternError, re.compile, br"\U00012345") self.assertTrue(re.match(br"\0", b"\000")) self.assertTrue(re.match(br"\08", b"\0008")) self.assertTrue(re.match(br"\01", b"\001")) @@ -1268,8 +1541,8 @@ def test_sre_byte_class_literals(self): self.assertTrue(re.match((r"[\x%02x]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02x0]" % i).encode(), bytes([i]))) self.assertTrue(re.match((r"[\x%02xz]" % i).encode(), bytes([i]))) - self.assertRaises(re.error, re.compile, br"[\u1234]") - self.assertRaises(re.error, re.compile, br"[\U00012345]") + self.assertRaises(re.PatternError, re.compile, br"[\u1234]") + self.assertRaises(re.PatternError, re.compile, br"[\U00012345]") self.checkPatternError(br"[\567]", r'octal escape value \567 outside of ' r'range 0-0o377', 1) @@ -1332,11 +1605,13 @@ def test_nothing_to_repeat(self): 'nothing to repeat', 3) def test_multiple_repeat(self): - for outer_reps in '*', '+', '{1,2}': - for outer_mod in '', '?': + for outer_reps in '*', '+', '?', '{1,2}': + for outer_mod in '', '?', '+': outer_op = outer_reps + outer_mod for inner_reps in '*', '+', '?', '{1,2}': - for inner_mod in '', '?': + for inner_mod in '', '?', '+': + if inner_mod + outer_reps in ('?', '+'): + continue inner_op = inner_reps + inner_mod self.checkPatternError(r'x%s%s' % (inner_op, outer_op), 'multiple repeat', 1 + len(inner_op)) @@ -1460,8 +1735,7 @@ def test_bug_817234(self): self.assertEqual(next(iter).span(), (4, 4)) self.assertRaises(StopIteration, next, iter) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_6561(self): # '\d' should match characters in Unicode category 'Nd' # (Number, Decimal Digit), but not those in 'Nl' (Number, @@ -1483,16 +1757,16 @@ def test_bug_6561(self): for x in not_decimal_digits: self.assertIsNone(re.match(r'^\d$', x)) + @unittest.expectedFailure # TODO: RUSTPYTHON; a = array.array(typecode)\n ValueError: bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d) + @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') def test_empty_array(self): # SF buf 1647541 import array - for typecode in 'bBuhHiIlLfd': + for typecode in 'bBhuwHiIlLfd': a = array.array(typecode) self.assertIsNone(re.compile(b"bla").match(a)) self.assertEqual(re.compile(b"").match(a).groups(), ()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_inline_flags(self): # Bug #1700 upper_char = '\u1ea0' # Latin Capital Letter A with Dot Below @@ -1536,70 +1810,27 @@ def test_inline_flags(self): self.assertTrue(re.match('(?x) (?i) ' + upper_char, lower_char)) self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char, re.X)) - p = upper_char + '(?i)' - with self.assertWarns(DeprecationWarning) as warns: - self.assertTrue(re.match(p, lower_char)) - self.assertEqual( - str(warns.warnings[0].message), - 'Flags not at the start of the expression %r' - ' but at position 1' % p - ) - self.assertEqual(warns.warnings[0].filename, __file__) - - p = upper_char + '(?i)%s' % ('.?' * 100) - with self.assertWarns(DeprecationWarning) as warns: - self.assertTrue(re.match(p, lower_char)) - self.assertEqual( - str(warns.warnings[0].message), - 'Flags not at the start of the expression %r (truncated)' - ' but at position 1' % p[:20] - ) - self.assertEqual(warns.warnings[0].filename, __file__) + msg = "global flags not at the start of the expression" + self.checkPatternError(upper_char + '(?i)', msg, 1) # bpo-30605: Compiling a bytes instance regex was throwing a BytesWarning with warnings.catch_warnings(): warnings.simplefilter('error', BytesWarning) - p = b'A(?i)' - with self.assertWarns(DeprecationWarning) as warns: - self.assertTrue(re.match(p, b'a')) - self.assertEqual( - str(warns.warnings[0].message), - 'Flags not at the start of the expression %r' - ' but at position 1' % p - ) - self.assertEqual(warns.warnings[0].filename, __file__) - - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match('(?s).(?i)' + upper_char, '\n' + lower_char)) - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match('(?i) ' + upper_char + ' (?x)', lower_char)) - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match(' (?x) (?i) ' + upper_char, lower_char)) - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match('^(?i)' + upper_char, lower_char)) - with self.assertWarns(DeprecationWarning): - self.assertTrue(re.match('$|(?i)' + upper_char, lower_char)) - with self.assertWarns(DeprecationWarning) as warns: - self.assertTrue(re.match('(?:(?i)' + upper_char + ')', lower_char)) - self.assertRegex(str(warns.warnings[0].message), - 'Flags not at the start') - self.assertEqual(warns.warnings[0].filename, __file__) - with self.assertWarns(DeprecationWarning) as warns: - self.assertTrue(re.fullmatch('(^)?(?(1)(?i)' + upper_char + ')', - lower_char)) - self.assertRegex(str(warns.warnings[0].message), - 'Flags not at the start') - self.assertEqual(warns.warnings[0].filename, __file__) - with self.assertWarns(DeprecationWarning) as warns: - self.assertTrue(re.fullmatch('($)?(?(1)|(?i)' + upper_char + ')', - lower_char)) - self.assertRegex(str(warns.warnings[0].message), - 'Flags not at the start') - self.assertEqual(warns.warnings[0].filename, __file__) + self.checkPatternError(b'A(?i)', msg, 1) + + self.checkPatternError('(?s).(?i)' + upper_char, msg, 5) + self.checkPatternError('(?i) ' + upper_char + ' (?x)', msg, 7) + self.checkPatternError(' (?x) (?i) ' + upper_char, msg, 1) + self.checkPatternError('^(?i)' + upper_char, msg, 1) + self.checkPatternError('$|(?i)' + upper_char, msg, 2) + self.checkPatternError('(?:(?i)' + upper_char + ')', msg, 3) + self.checkPatternError('(^)?(?(1)(?i)' + upper_char + ')', msg, 9) + self.checkPatternError('($)?(?(1)|(?i)' + upper_char + ')', msg, 10) def test_dollar_matches_twice(self): - "$ matches the end of string, and just before the terminating \n" + r"""Test that $ does not include \n + $ matches the end of string, and just before the terminating \n""" pattern = re.compile('$') self.assertEqual(pattern.sub('#', 'a\nb\n'), 'a\nb#\n#') self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a\nb\nc#') @@ -1646,11 +1877,11 @@ def test_ascii_and_unicode_flag(self): self.assertIsNone(pat.match(b'\xe0')) # Incompatibilities self.assertRaises(ValueError, re.compile, br'\w', re.UNICODE) - self.assertRaises(re.error, re.compile, br'(?u)\w') + self.assertRaises(re.PatternError, re.compile, br'(?u)\w') self.assertRaises(ValueError, re.compile, r'\w', re.UNICODE | re.ASCII) self.assertRaises(ValueError, re.compile, r'(?u)\w', re.ASCII) self.assertRaises(ValueError, re.compile, r'(?a)\w', re.UNICODE) - self.assertRaises(re.error, re.compile, r'(?au)\w') + self.assertRaises(re.PatternError, re.compile, r'(?au)\w') def test_locale_flag(self): enc = locale.getpreferredencoding() @@ -1691,11 +1922,11 @@ def test_locale_flag(self): self.assertIsNone(pat.match(bletter)) # Incompatibilities self.assertRaises(ValueError, re.compile, '', re.LOCALE) - self.assertRaises(re.error, re.compile, '(?L)') + self.assertRaises(re.PatternError, re.compile, '(?L)') self.assertRaises(ValueError, re.compile, b'', re.LOCALE | re.ASCII) self.assertRaises(ValueError, re.compile, b'(?L)', re.ASCII) self.assertRaises(ValueError, re.compile, b'(?a)', re.LOCALE) - self.assertRaises(re.error, re.compile, b'(?aL)') + self.assertRaises(re.PatternError, re.compile, b'(?aL)') def test_scoped_flags(self): self.assertTrue(re.match(r'(?i:a)b', 'Ab')) @@ -1775,24 +2006,6 @@ def test_bug_6509(self): pat = re.compile(b'..') self.assertEqual(pat.sub(lambda m: b'bytes', b'a5'), b'bytes') - # RUSTPYTHON: here in rustpython, we borrow the string only at the - # time of matching, so we will not check the string type when creating - # SRE_Scanner, expect this, other tests has passed - @cpython_only - def test_dealloc(self): - # issue 3299: check for segfault in debug build - import _sre - # the overflow limit is different on wide and narrow builds and it - # depends on the definition of SRE_CODE (see sre.h). - # 2**128 should be big enough to overflow on both. For smaller values - # a RuntimeError is raised instead of OverflowError. - long_overflow = 2**128 - self.assertRaises(TypeError, re.finditer, "a", {}) - with self.assertRaises(OverflowError): - _sre.compile("abc", 0, [long_overflow], 0, {}, ()) - with self.assertRaises(TypeError): - _sre.compile({}, 0, [], 0, [], []) - def test_search_dot_unicode(self): self.assertTrue(re.search("123.*-", '123abc-')) self.assertTrue(re.search("123.*-", '123\xe9-')) @@ -1850,20 +2063,28 @@ def test_repeat_minmax_overflow(self): self.assertRaises(OverflowError, re.compile, r".{%d,}?" % 2**128) self.assertRaises(OverflowError, re.compile, r".{%d,%d}" % (2**129, 2**128)) - @cpython_only - def test_repeat_minmax_overflow_maxrepeat(self): - try: - from _sre import MAXREPEAT - except ImportError: - self.skipTest('requires _sre.MAXREPEAT constant') - string = "x" * 100000 - self.assertIsNone(re.match(r".{%d}" % (MAXREPEAT - 1), string)) - self.assertEqual(re.match(r".{,%d}" % (MAXREPEAT - 1), string).span(), - (0, 100000)) - self.assertIsNone(re.match(r".{%d,}?" % (MAXREPEAT - 1), string)) - self.assertRaises(OverflowError, re.compile, r".{%d}" % MAXREPEAT) - self.assertRaises(OverflowError, re.compile, r".{,%d}" % MAXREPEAT) - self.assertRaises(OverflowError, re.compile, r".{%d,}?" % MAXREPEAT) + def test_look_behind_overflow(self): + string = "x" * 2_500_000 + p1 = r"(?<=((.{%d}){%d}){%d})" + p2 = r"(?<!((.{%d}){%d}){%d})" + # Test that the templates are valid and look-behind with width 2**21 + # (larger than sys.maxunicode) are supported. + self.assertEqual(re.search(p1 % (2**7, 2**7, 2**7), string).span(), + (2**21, 2**21)) + self.assertEqual(re.search(p2 % (2**7, 2**7, 2**7), string).span(), + (0, 0)) + # Test that 2**22 is accepted as a repetition number and look-behind + # width. + re.compile(p1 % (2**22, 1, 1)) + re.compile(p1 % (1, 2**22, 1)) + re.compile(p1 % (1, 1, 2**22)) + re.compile(p2 % (2**22, 1, 1)) + re.compile(p2 % (1, 2**22, 1)) + re.compile(p2 % (1, 1, 2**22)) + # But 2**66 is too large for look-behind width. + errmsg = "looks too much behind" + self.assertRaisesRegex(re.error, errmsg, re.compile, p1 % (2**22, 2**22, 2**22)) + self.assertRaisesRegex(re.error, errmsg, re.compile, p2 % (2**22, 2**22, 2**22)) def test_backref_group_name_in_exception(self): # Issue 17341: Poor error message when compiling invalid regex @@ -1885,8 +2106,6 @@ def test_issue17998(self): self.assertEqual(re.compile(pattern, re.S).findall(b'xyz'), [b'xyz'], msg=pattern) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_match_repr(self): for string in '[abracadabra]', S('[abracadabra]'): m = re.search(r'(.+)(.*?)\1', string) @@ -1933,9 +2152,6 @@ def test_zerowidth(self): self.assertEqual([m.span() for m in re.finditer(r"\b|\w+", "a::bc")], [(0, 0), (0, 1), (1, 1), (3, 3), (3, 5), (5, 5)]) - # TODO: RUSTPYTHON - # @unittest.expectedFailure - @unittest.skip("") def test_bug_2537(self): # issue 2537: empty submatches for outer_op in ('{0,}', '*', '+', '{1,187}'): @@ -1946,60 +2162,6 @@ def test_bug_2537(self): self.assertEqual(m.group(1), "") self.assertEqual(m.group(2), "y") - @cpython_only - def test_debug_flag(self): - pat = r'(\.)(?:[ch]|py)(?(1)$|: )' - with captured_stdout() as out: - re.compile(pat, re.DEBUG) - self.maxDiff = None - dump = '''\ -SUBPATTERN 1 0 0 - LITERAL 46 -BRANCH - IN - LITERAL 99 - LITERAL 104 -OR - LITERAL 112 - LITERAL 121 -GROUPREF_EXISTS 1 - AT AT_END -ELSE - LITERAL 58 - LITERAL 32 - - 0. INFO 8 0b1 2 5 (to 9) - prefix_skip 0 - prefix [0x2e] ('.') - overlap [0] - 9: MARK 0 -11. LITERAL 0x2e ('.') -13. MARK 1 -15. BRANCH 10 (to 26) -17. IN 6 (to 24) -19. LITERAL 0x63 ('c') -21. LITERAL 0x68 ('h') -23. FAILURE -24: JUMP 9 (to 34) -26: branch 7 (to 33) -27. LITERAL 0x70 ('p') -29. LITERAL 0x79 ('y') -31. JUMP 2 (to 34) -33: FAILURE -34: GROUPREF_EXISTS 0 6 (to 41) -37. AT END -39. JUMP 5 (to 45) -41: LITERAL 0x3a (':') -43. LITERAL 0x20 (' ') -45: SUCCESS -''' - self.assertEqual(out.getvalue(), dump) - # Debug output is output again even a second time (bypassing - # the cache -- issue #20426). - with captured_stdout() as out: - re.compile(pat, re.DEBUG) - self.assertEqual(out.getvalue(), dump) - def test_keyword_parameters(self): # Issue #20283: Accepting the string keyword parameter. pat = re.compile(r'(ab)') @@ -2023,6 +2185,10 @@ def test_bug_20998(self): # with ignore case. self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3)) + @unittest.expectedFailure # TODO: RUSTPYTHON; self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))\n AssertionError: None is not true + @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("sunos"), + "test doesn't work on Solaris, gh-91214") def test_locale_caching(self): # Issue #22410 oldlocale = locale.setlocale(locale.LC_CTYPE) @@ -2059,6 +2225,9 @@ def check_en_US_utf8(self): self.assertIsNone(re.match(b'(?Li)\xc5', b'\xe5')) self.assertIsNone(re.match(b'(?Li)\xe5', b'\xc5')) + @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("sunos"), + "test doesn't work on Solaris, gh-91214") def test_locale_compiled(self): oldlocale = locale.setlocale(locale.LC_CTYPE) self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) @@ -2092,7 +2261,7 @@ def test_locale_compiled(self): self.assertIsNone(p4.match(b'\xc5\xc5')) def test_error(self): - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile('(\u20ac))') err = cm.exception self.assertIsInstance(err.pattern, str) @@ -2104,14 +2273,14 @@ def test_error(self): self.assertIn(' at position 3', str(err)) self.assertNotIn(' at position 3', err.msg) # Bytes pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(b'(\xa4))') err = cm.exception self.assertIsInstance(err.pattern, bytes) self.assertEqual(err.pattern, b'(\xa4))') self.assertEqual(err.pos, 3) # Multiline pattern - with self.assertRaises(re.error) as cm: + with self.assertRaises(re.PatternError) as cm: re.compile(""" ( abc @@ -2236,6 +2405,429 @@ def test_bug_34294(self): {'tag': 'foo', 'text': None}, {'tag': 'foo', 'text': None}]) + def test_MARK_PUSH_macro_bug(self): + # issue35859, MARK_PUSH() macro didn't protect MARK-0 if it + # was the only available mark. + self.assertEqual(re.match(r'(ab|a)*?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(ab|a)+?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(ab|a){0,2}?b', 'ab').groups(), ('a',)) + self.assertEqual(re.match(r'(.b|a)*?b', 'ab').groups(), ('a',)) + + def test_MIN_UNTIL_mark_bug(self): + # Fixed in issue35859, reported in issue9134. + # JUMP_MIN_UNTIL_2 should MARK_PUSH() if in a repeat + s = 'axxzbcz' + p = r'(?:(?:a|bc)*?(xx)??z)*' + self.assertEqual(re.match(p, s).groups(), ('xx',)) + + # test-case provided by issue9134 + s = 'xtcxyzxc' + p = r'((x|yz)+?(t)??c)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 8)) + self.assertEqual(m.span(2), (6, 7)) + self.assertEqual(m.groups(), ('xyzxc', 'x', 't')) + + def test_REPEAT_ONE_mark_bug(self): + # issue35859 + # JUMP_REPEAT_ONE_1 should MARK_PUSH() if in a repeat + s = 'aabaab' + p = r'(?:[^b]*a(?=(b)|(a))ab)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 6)) + self.assertEqual(m.span(2), (4, 5)) + self.assertEqual(m.groups(), (None, 'a')) + + # JUMP_REPEAT_ONE_2 should MARK_PUSH() if in a repeat + s = 'abab' + p = r'(?:[^b]*(?=(b)|(a))ab)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.span(2), (2, 3)) + self.assertEqual(m.groups(), (None, 'a')) + + self.assertEqual(re.match(r'(ab?)*?b', 'ab').groups(), ('a',)) + + def test_MIN_REPEAT_ONE_mark_bug(self): + # issue35859 + # JUMP_MIN_REPEAT_ONE should MARK_PUSH() if in a repeat + s = 'abab' + p = r'(?:.*?(?=(a)|(b))b)*' + m = re.match(p, s) + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.span(2), (3, 4)) + self.assertEqual(m.groups(), (None, 'b')) + + s = 'axxzaz' + p = r'(?:a*?(xx)??z)*' + self.assertEqual(re.match(p, s).groups(), ('xx',)) + + def test_ASSERT_NOT_mark_bug(self): + # Fixed in issue35859, reported in issue725149. + # JUMP_ASSERT_NOT should LASTMARK_SAVE() + self.assertEqual(re.match(r'(?!(..)c)', 'ab').groups(), (None,)) + + # JUMP_ASSERT_NOT should MARK_PUSH() if in a repeat + m = re.match(r'((?!(ab)c)(.))*', 'abab') + self.assertEqual(m.span(), (0, 4)) + self.assertEqual(m.span(1), (3, 4)) + self.assertEqual(m.span(3), (3, 4)) + self.assertEqual(m.groups(), ('b', None, 'b')) + + def test_bug_40736(self): + with self.assertRaisesRegex(TypeError, "got 'int'"): + re.search("x*", 5) + with self.assertRaisesRegex(TypeError, "got 'type'"): + re.search("x*", type) + + # gh-117594: The test is not slow by itself, but it relies on + # the absolute computation time and can fail on very slow computers. + @unittest.skip('TODO: RUSTPYTHON; flaky, improve perf') + @requires_resource('cpu') + def test_search_anchor_at_beginning(self): + s = 'x'*10**7 + with Stopwatch() as stopwatch: + for p in r'\Ay', r'^y': + self.assertIsNone(re.search(p, s)) + self.assertEqual(re.split(p, s), [s]) + self.assertEqual(re.findall(p, s), []) + self.assertEqual(list(re.finditer(p, s)), []) + self.assertEqual(re.sub(p, '', s), s) + # Without optimization it takes 1 second on my computer. + # With optimization -- 0.0003 seconds. + self.assertLess(stopwatch.seconds, 0.1) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_possessive_quantifiers(self): + """Test Possessive Quantifiers + Test quantifiers of the form @+ for some repetition operator @, + e.g. x{3,5}+ meaning match from 3 to 5 greadily and proceed + without creating a stack frame for rolling the stack back and + trying 1 or more fewer matches.""" + self.assertIsNone(re.match('e*+e', 'eeee')) + self.assertEqual(re.match('e++a', 'eeea').group(0), 'eeea') + self.assertEqual(re.match('e?+a', 'ea').group(0), 'ea') + self.assertEqual(re.match('e{2,4}+a', 'eeea').group(0), 'eeea') + self.assertIsNone(re.match('(.)++.', 'ee')) + self.assertEqual(re.match('(ae)*+a', 'aea').groups(), ('ae',)) + self.assertEqual(re.match('([ae][ae])?+a', 'aea').groups(), + ('ae',)) + self.assertEqual(re.match('(e?){2,4}+a', 'eeea').groups(), + ('',)) + self.assertEqual(re.match('()*+a', 'a').groups(), ('',)) + self.assertEqual(re.search('x*+', 'axx').span(), (0, 0)) + self.assertEqual(re.search('x++', 'axx').span(), (1, 3)) + self.assertEqual(re.match('a*+', 'xxx').span(), (0, 0)) + self.assertEqual(re.match('x*+', 'xxxa').span(), (0, 3)) + self.assertIsNone(re.match('a++', 'xxx')) + self.assertIsNone(re.match(r"^(\w){1}+$", "abc")) + self.assertIsNone(re.match(r"^(\w){1,2}+$", "abc")) + + self.assertEqual(re.match(r"^(\w){3}+$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,3}+$", "abc").group(1), "c") + self.assertEqual(re.match(r"^(\w){1,4}+$", "abc").group(1), "c") + + self.assertIsNone(re.match("^x{1}+$", "xxx")) + self.assertIsNone(re.match("^x{1,2}+$", "xxx")) + + self.assertTrue(re.match("^x{3}+$", "xxx")) + self.assertTrue(re.match("^x{1,3}+$", "xxx")) + self.assertTrue(re.match("^x{1,4}+$", "xxx")) + + self.assertIsNone(re.match("^x{}+$", "xxx")) + self.assertTrue(re.match("^x{}+$", "x{}")) + + def test_fullmatch_possessive_quantifiers(self): + self.assertTrue(re.fullmatch(r'a++', 'a')) + self.assertTrue(re.fullmatch(r'a*+', 'a')) + self.assertTrue(re.fullmatch(r'a?+', 'a')) + self.assertTrue(re.fullmatch(r'a{1,3}+', 'a')) + self.assertIsNone(re.fullmatch(r'a++', 'ab')) + self.assertIsNone(re.fullmatch(r'a*+', 'ab')) + self.assertIsNone(re.fullmatch(r'a?+', 'ab')) + self.assertIsNone(re.fullmatch(r'a{1,3}+', 'ab')) + self.assertTrue(re.fullmatch(r'a++b', 'ab')) + self.assertTrue(re.fullmatch(r'a*+b', 'ab')) + self.assertTrue(re.fullmatch(r'a?+b', 'ab')) + self.assertTrue(re.fullmatch(r'a{1,3}+b', 'ab')) + + self.assertTrue(re.fullmatch(r'(?:ab)++', 'ab')) + self.assertTrue(re.fullmatch(r'(?:ab)*+', 'ab')) + self.assertTrue(re.fullmatch(r'(?:ab)?+', 'ab')) + self.assertTrue(re.fullmatch(r'(?:ab){1,3}+', 'ab')) + self.assertIsNone(re.fullmatch(r'(?:ab)++', 'abc')) + self.assertIsNone(re.fullmatch(r'(?:ab)*+', 'abc')) + self.assertIsNone(re.fullmatch(r'(?:ab)?+', 'abc')) + self.assertIsNone(re.fullmatch(r'(?:ab){1,3}+', 'abc')) + self.assertTrue(re.fullmatch(r'(?:ab)++c', 'abc')) + self.assertTrue(re.fullmatch(r'(?:ab)*+c', 'abc')) + self.assertTrue(re.fullmatch(r'(?:ab)?+c', 'abc')) + self.assertTrue(re.fullmatch(r'(?:ab){1,3}+c', 'abc')) + + def test_findall_possessive_quantifiers(self): + self.assertEqual(re.findall(r'a++', 'aab'), ['aa']) + self.assertEqual(re.findall(r'a*+', 'aab'), ['aa', '', '']) + self.assertEqual(re.findall(r'a?+', 'aab'), ['a', 'a', '', '']) + self.assertEqual(re.findall(r'a{1,3}+', 'aab'), ['aa']) + + self.assertEqual(re.findall(r'(?:ab)++', 'ababc'), ['abab']) + self.assertEqual(re.findall(r'(?:ab)*+', 'ababc'), ['abab', '', '']) + self.assertEqual(re.findall(r'(?:ab)?+', 'ababc'), ['ab', 'ab', '', '']) + self.assertEqual(re.findall(r'(?:ab){1,3}+', 'ababc'), ['abab']) + + def test_atomic_grouping(self): + """Test Atomic Grouping + Test non-capturing groups of the form (?>...), which does + not maintain any stack point created within the group once the + group is finished being evaluated.""" + pattern1 = re.compile(r'a(?>bc|b)c') + self.assertIsNone(pattern1.match('abc')) + self.assertTrue(pattern1.match('abcc')) + self.assertIsNone(re.match(r'(?>.*).', 'abc')) + self.assertTrue(re.match(r'(?>x)++', 'xxx')) + self.assertTrue(re.match(r'(?>x++)', 'xxx')) + self.assertIsNone(re.match(r'(?>x)++x', 'xxx')) + self.assertIsNone(re.match(r'(?>x++)x', 'xxx')) + + def test_fullmatch_atomic_grouping(self): + self.assertTrue(re.fullmatch(r'(?>a+)', 'a')) + self.assertTrue(re.fullmatch(r'(?>a*)', 'a')) + self.assertTrue(re.fullmatch(r'(?>a?)', 'a')) + self.assertTrue(re.fullmatch(r'(?>a{1,3})', 'a')) + self.assertIsNone(re.fullmatch(r'(?>a+)', 'ab')) + self.assertIsNone(re.fullmatch(r'(?>a*)', 'ab')) + self.assertIsNone(re.fullmatch(r'(?>a?)', 'ab')) + self.assertIsNone(re.fullmatch(r'(?>a{1,3})', 'ab')) + self.assertTrue(re.fullmatch(r'(?>a+)b', 'ab')) + self.assertTrue(re.fullmatch(r'(?>a*)b', 'ab')) + self.assertTrue(re.fullmatch(r'(?>a?)b', 'ab')) + self.assertTrue(re.fullmatch(r'(?>a{1,3})b', 'ab')) + + self.assertTrue(re.fullmatch(r'(?>(?:ab)+)', 'ab')) + self.assertTrue(re.fullmatch(r'(?>(?:ab)*)', 'ab')) + self.assertTrue(re.fullmatch(r'(?>(?:ab)?)', 'ab')) + self.assertTrue(re.fullmatch(r'(?>(?:ab){1,3})', 'ab')) + self.assertIsNone(re.fullmatch(r'(?>(?:ab)+)', 'abc')) + self.assertIsNone(re.fullmatch(r'(?>(?:ab)*)', 'abc')) + self.assertIsNone(re.fullmatch(r'(?>(?:ab)?)', 'abc')) + self.assertIsNone(re.fullmatch(r'(?>(?:ab){1,3})', 'abc')) + self.assertTrue(re.fullmatch(r'(?>(?:ab)+)c', 'abc')) + self.assertTrue(re.fullmatch(r'(?>(?:ab)*)c', 'abc')) + self.assertTrue(re.fullmatch(r'(?>(?:ab)?)c', 'abc')) + self.assertTrue(re.fullmatch(r'(?>(?:ab){1,3})c', 'abc')) + + def test_findall_atomic_grouping(self): + self.assertEqual(re.findall(r'(?>a+)', 'aab'), ['aa']) + self.assertEqual(re.findall(r'(?>a*)', 'aab'), ['aa', '', '']) + self.assertEqual(re.findall(r'(?>a?)', 'aab'), ['a', 'a', '', '']) + self.assertEqual(re.findall(r'(?>a{1,3})', 'aab'), ['aa']) + + self.assertEqual(re.findall(r'(?>(?:ab)+)', 'ababc'), ['abab']) + self.assertEqual(re.findall(r'(?>(?:ab)*)', 'ababc'), ['abab', '', '']) + self.assertEqual(re.findall(r'(?>(?:ab)?)', 'ababc'), ['ab', 'ab', '', '']) + self.assertEqual(re.findall(r'(?>(?:ab){1,3})', 'ababc'), ['abab']) + + def test_bug_gh91616(self): + self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\z', "a.txt")) # reproducer + self.assertTrue(re.fullmatch(r'(?s:(?=(?P<g0>.*?\.))(?P=g0).*)\z', "a.txt")) + + def test_bug_gh100061(self): + # gh-100061 + self.assertEqual(re.match('(?>(?:.(?!D))+)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))++', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))*)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))*+', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))?)', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?:.(?!D))?+', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?>(?:.(?!D)){1,3})', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D)){1,3}+', 'ABCDE').span(), (0, 2)) + # gh-106052 + self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)*+", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)?)", "a").span(), (0, 0)) + self.assertEqual(re.match("(?:ab?c)?+", "a").span(), (0, 0)) + self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; self.assertEqual(re.match('((x)|y|z){3}+', 'xyz').groups(), ('z', 'x'))\n AssertionError: Tuples differ: ('x', 'x') != ('z', 'x') + def test_bug_gh101955(self): + # Possessive quantifier with nested alternative with capture groups + self.assertEqual(re.match('((x)|y|z)*+', 'xyz').groups(), ('z', 'x')) + self.assertEqual(re.match('((x)|y|z){3}+', 'xyz').groups(), ('z', 'x')) + self.assertEqual(re.match('((x)|y|z){3,}+', 'xyz').groups(), ('z', 'x')) + + @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing') + def test_regression_gh94675(self): + pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*' + r'((/[^/\[\n]*(([^\n]|(\[\n]*(]*)*\]))' + r'[^/\[]*)*/))((((//[^\n]*)?[\n])' + r'([\000-\040]|(/\*[^*]*\*+' + r'([^/*]\*+)*/))*)+(?=[^\000-\040);\]}]))') + input_js = '''a(function() { + /////////////////////////////////////////////////////////////////// + });''' + p = multiprocessing.Process(target=pattern.sub, args=('', input_js)) + p.start() + p.join(SHORT_TIMEOUT) + try: + self.assertFalse(p.is_alive(), 'pattern.sub() timed out') + finally: + if p.is_alive(): + p.terminate() + p.join() + + def test_fail(self): + self.assertEqual(re.search(r'12(?!)|3', '123')[0], '3') + + def test_character_set_any(self): + # The union of complementary character sets matches any character + # and is equivalent to "(?s:.)". + s = '1x\n' + for p in r'[\s\S]', r'[\d\D]', r'[\w\W]', r'[\S\s]', r'\s|\S': + with self.subTest(pattern=p): + self.assertEqual(re.findall(p, s), list(s)) + self.assertEqual(re.fullmatch('(?:' + p + ')+', s).group(), s) + + def test_character_set_none(self): + # Negation of the union of complementary character sets does not match + # any character. + s = '1x\n' + for p in r'[^\s\S]', r'[^\d\D]', r'[^\w\W]', r'[^\S\s]': + with self.subTest(pattern=p): + self.assertIsNone(re.search(p, s)) + self.assertIsNone(re.search('(?s:.)' + p, s)) + + def check_interrupt(self, pattern, string, maxcount): + class Interrupt(Exception): + pass + p = re.compile(pattern) + for n in range(maxcount): + try: + p._fail_after(n, Interrupt) + p.match(string) + return n + except Interrupt: + pass + finally: + p._fail_after(-1, None) + + @unittest.skipUnless(hasattr(re.Pattern, '_fail_after'), 'requires debug build') + def test_memory_leaks(self): + self.check_interrupt(r'(.)*:', 'abc:', 100) + self.check_interrupt(r'([^:])*?:', 'abc:', 100) + self.check_interrupt(r'([^:])*+:', 'abc:', 100) + self.check_interrupt(r'(.){2,4}:', 'abc:', 100) + self.check_interrupt(r'([^:]){2,4}?:', 'abc:', 100) + self.check_interrupt(r'([^:]){2,4}+:', 'abc:', 100) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_template_function_and_flag_is_deprecated(self): + return super().test_template_function_and_flag_is_deprecated() + + +def get_debug_out(pat): + with captured_stdout() as out: + re.compile(pat, re.DEBUG) + return out.getvalue() + + +@cpython_only +class DebugTests(unittest.TestCase): + maxDiff = None + + def test_debug_flag(self): + pat = r'(\.)(?:[ch]|py)(?(1)$|: )' + dump = '''\ +SUBPATTERN 1 0 0 + LITERAL 46 +BRANCH + IN + LITERAL 99 + LITERAL 104 +OR + LITERAL 112 + LITERAL 121 +GROUPREF_EXISTS 1 + AT AT_END +ELSE + LITERAL 58 + LITERAL 32 + + 0. INFO 8 0b1 2 5 (to 9) + prefix_skip 0 + prefix [0x2e] ('.') + overlap [0] + 9: MARK 0 +11. LITERAL 0x2e ('.') +13. MARK 1 +15. BRANCH 10 (to 26) +17. IN 6 (to 24) +19. LITERAL 0x63 ('c') +21. LITERAL 0x68 ('h') +23. FAILURE +24: JUMP 9 (to 34) +26: branch 7 (to 33) +27. LITERAL 0x70 ('p') +29. LITERAL 0x79 ('y') +31. JUMP 2 (to 34) +33: FAILURE +34: GROUPREF_EXISTS 0 6 (to 41) +37. AT END +39. JUMP 5 (to 45) +41: LITERAL 0x3a (':') +43. LITERAL 0x20 (' ') +45: SUCCESS +''' + self.assertEqual(get_debug_out(pat), dump) + # Debug output is output again even a second time (bypassing + # the cache -- issue #20426). + self.assertEqual(get_debug_out(pat), dump) + + def test_atomic_group(self): + self.assertEqual(get_debug_out(r'(?>ab?)'), '''\ +ATOMIC_GROUP + LITERAL 97 + MAX_REPEAT 0 1 + LITERAL 98 + + 0. INFO 4 0b0 1 2 (to 5) + 5: ATOMIC_GROUP 11 (to 17) + 7. LITERAL 0x61 ('a') + 9. REPEAT_ONE 6 0 1 (to 16) +13. LITERAL 0x62 ('b') +15. SUCCESS +16: SUCCESS +17: SUCCESS +''') + + def test_possesive_repeat_one(self): + self.assertEqual(get_debug_out(r'a?+'), '''\ +POSSESSIVE_REPEAT 0 1 + LITERAL 97 + + 0. INFO 4 0b0 0 1 (to 5) + 5: POSSESSIVE_REPEAT_ONE 6 0 1 (to 12) + 9. LITERAL 0x61 ('a') +11. SUCCESS +12: SUCCESS +''') + + def test_possesive_repeat(self): + self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ +POSSESSIVE_REPEAT 0 1 + LITERAL 97 + LITERAL 98 + + 0. INFO 4 0b0 0 2 (to 5) + 5: POSSESSIVE_REPEAT 7 0 1 (to 13) + 9. LITERAL 0x61 ('a') +11. LITERAL 0x62 ('b') +13: SUCCESS +14. SUCCESS +''') + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): @@ -2268,8 +2860,7 @@ def test_inline_flags(self): self.check('(?i)pattern', "re.compile('(?i)pattern', re.IGNORECASE)") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unknown_flags(self): self.check_flags('random pattern', 0x123000, "re.compile('random pattern', 0x123000)") @@ -2298,25 +2889,25 @@ def test_long_pattern(self): pattern = 'Very %spattern' % ('long ' * 1000) r = repr(re.compile(pattern)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") + self.assertStartsWith(r, "re.compile('Very long long lon") r = repr(re.compile(pattern, re.I)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") - self.assertEqual(r[-16:], ", re.IGNORECASE)") + self.assertStartsWith(r, "re.compile('Very long long lon") + self.assertEndsWith(r, ", re.IGNORECASE)") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_flags_repr(self): self.assertEqual(repr(re.I), "re.IGNORECASE") self.assertEqual(repr(re.I|re.S|re.X), "re.IGNORECASE|re.DOTALL|re.VERBOSE") self.assertEqual(repr(re.I|re.S|re.X|(1<<20)), "re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000") - self.assertEqual(repr(~re.I), "~re.IGNORECASE") + self.assertEqual( + repr(~re.I), + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DOTALL|re.VERBOSE|re.DEBUG|0x1") self.assertEqual(repr(~(re.I|re.S|re.X)), - "~(re.IGNORECASE|re.DOTALL|re.VERBOSE)") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DEBUG|0x1") self.assertEqual(repr(~(re.I|re.S|re.X|(1<<20))), - "~(re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000)") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DEBUG|0xffe01") class ImplementationTest(unittest.TestCase): @@ -2337,7 +2928,7 @@ def test_immutable(self): tp.foo = 1 def test_overlap_table(self): - f = sre_compile._generate_overlap_table + f = re._compiler._generate_overlap_table self.assertEqual(f(""), []) self.assertEqual(f("a"), [0]) self.assertEqual(f("abcd"), [0, 0, 0, 0]) @@ -2346,8 +2937,8 @@ def test_overlap_table(self): self.assertEqual(f("abcabdac"), [0, 0, 0, 1, 2, 0, 1, 0]) def test_signedness(self): - self.assertGreaterEqual(sre_compile.MAXREPEAT, 0) - self.assertGreaterEqual(sre_compile.MAXGROUPS, 0) + self.assertGreaterEqual(re._compiler.MAXREPEAT, 0) + self.assertGreaterEqual(re._compiler.MAXGROUPS, 0) @cpython_only def test_disallow_instantiation(self): @@ -2357,6 +2948,105 @@ def test_disallow_instantiation(self): pat = re.compile("") check_disallow_instantiation(self, type(pat.scanner(""))) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_deprecated_modules(self): + deprecated = { + 'sre_compile': ['compile', 'error', + 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', + '_compile_info'], + 'sre_constants': ['error', 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', + '_NamedIntConstant'], + 'sre_parse': ['SubPattern', 'parse', + 'SRE_FLAG_IGNORECASE', 'SUBPATTERN', + '_parse_sub'], + } + for name in deprecated: + with self.subTest(module=name): + sys.modules.pop(name, None) + with self.assertWarns(DeprecationWarning) as w: + __import__(name) + self.assertEqual(str(w.warning), + f"module {name!r} is deprecated") + self.assertEqual(w.filename, __file__) + self.assertIn(name, sys.modules) + mod = sys.modules[name] + self.assertEqual(mod.__name__, name) + self.assertEqual(mod.__package__, '') + for attr in deprecated[name]: + self.assertHasAttr(mod, attr) + del sys.modules[name] + + @cpython_only + def test_case_helpers(self): + import _sre + for i in range(128): + c = chr(i) + lo = ord(c.lower()) + self.assertEqual(_sre.ascii_tolower(i), lo) + self.assertEqual(_sre.unicode_tolower(i), lo) + iscased = c in string.ascii_letters + self.assertEqual(_sre.ascii_iscased(i), iscased) + self.assertEqual(_sre.unicode_iscased(i), iscased) + + for i in list(range(128, 0x1000)) + [0x10400, 0x10428]: + c = chr(i) + self.assertEqual(_sre.ascii_tolower(i), i) + if i != 0x0130: + self.assertEqual(_sre.unicode_tolower(i), ord(c.lower())) + iscased = c != c.lower() or c != c.upper() + self.assertFalse(_sre.ascii_iscased(i)) + self.assertEqual(_sre.unicode_iscased(i), + c != c.lower() or c != c.upper()) + + self.assertEqual(_sre.ascii_tolower(0x0130), 0x0130) + self.assertEqual(_sre.unicode_tolower(0x0130), ord('i')) + self.assertFalse(_sre.ascii_iscased(0x0130)) + self.assertTrue(_sre.unicode_iscased(0x0130)) + + @cpython_only + def test_dealloc(self): + # issue 3299: check for segfault in debug build + import _sre + # the overflow limit is different on wide and narrow builds and it + # depends on the definition of SRE_CODE (see sre.h). + # 2**128 should be big enough to overflow on both. For smaller values + # a RuntimeError is raised instead of OverflowError. + long_overflow = 2**128 + self.assertRaises(TypeError, re.finditer, "a", {}) + with self.assertRaises(OverflowError): + _sre.compile("abc", 0, [long_overflow], 0, {}, ()) + with self.assertRaises(TypeError): + _sre.compile({}, 0, [], 0, [], []) + # gh-110590: `TypeError` was overwritten with `OverflowError`: + with self.assertRaises(TypeError): + _sre.compile('', 0, ['abc'], 0, {}, ()) + + @cpython_only + def test_repeat_minmax_overflow_maxrepeat(self): + try: + from _sre import MAXREPEAT + except ImportError: + self.skipTest('requires _sre.MAXREPEAT constant') + string = "x" * 100000 + self.assertIsNone(re.match(r".{%d}" % (MAXREPEAT - 1), string)) + self.assertEqual(re.match(r".{,%d}" % (MAXREPEAT - 1), string).span(), + (0, 100000)) + self.assertIsNone(re.match(r".{%d,}?" % (MAXREPEAT - 1), string)) + self.assertRaises(OverflowError, re.compile, r".{%d}" % MAXREPEAT) + self.assertRaises(OverflowError, re.compile, r".{,%d}" % MAXREPEAT) + self.assertRaises(OverflowError, re.compile, r".{%d,}?" % MAXREPEAT) + + @cpython_only + def test_sre_template_invalid_group_index(self): + # see gh-106524 + import _sre + with self.assertRaises(TypeError) as cm: + _sre.template("", ["", -1, ""]) + self.assertIn("invalid template", str(cm.exception)) + with self.assertRaises(TypeError) as cm: + _sre.template("", ["", (), ""]) + self.assertIn("an integer is required", str(cm.exception)) + class ExternalTests(unittest.TestCase): @@ -2389,7 +3079,7 @@ def test_re_tests(self): with self.subTest(pattern=pattern, string=s): if outcome == SYNTAX_ERROR: # Expected a syntax error - with self.assertRaises(re.error): + with self.assertRaises(re.PatternError): re.compile(pattern) continue diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 57d70fae62e..82939108b12 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -4,29 +4,61 @@ Note: test_regrtest cannot be run twice in parallel. """ +import _colorize import contextlib -import faulthandler +import dataclasses import glob import io +import locale import os.path import platform +import random import re +import shlex +import signal import subprocess import sys import sysconfig import tempfile import textwrap import unittest -from test import libregrtest +import unittest.mock +from xml.etree import ElementTree + from test import support +from test.support import import_helper from test.support import os_helper +from test.libregrtest import cmdline +from test.libregrtest import main +from test.libregrtest import setup from test.libregrtest import utils +from test.libregrtest.filter import get_match_tests, set_match_tests, match_test +from test.libregrtest.result import TestStats +from test.libregrtest.utils import normalize_test_name +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") -Py_DEBUG = hasattr(sys, 'gettotalrefcount') ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' +RESULT_REGEX = ( + 'passed', + 'failed', + 'skipped', + 'interrupted', + 'env changed', + 'timed out', + 'ran no tests', + 'worker non-zero exit code', +) +RESULT_REGEX = fr'(?:{"|".join(RESULT_REGEX)})' + +EXITCODE_BAD_TEST = 2 +EXITCODE_ENV_CHANGED = 3 +EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_RERUN_FAIL = 5 +EXITCODE_INTERRUPTED = 130 TEST_INTERRUPTED = textwrap.dedent(""" from signal import SIGINT, raise_signal @@ -43,9 +75,13 @@ class ParseArgsTestCase(unittest.TestCase): Test regrtest's argument parsing, function _parse_args(). """ + @staticmethod + def parse_args(args): + return cmdline._parse_args(args) + def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): - libregrtest._parse_args(args) + self.parse_args(args) self.assertIn(msg, err.getvalue()) def test_help(self): @@ -53,94 +89,130 @@ def test_help(self): with self.subTest(opt=opt): with support.captured_stdout() as out, \ self.assertRaises(SystemExit): - libregrtest._parse_args([opt]) + self.parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) - @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), - "faulthandler.dump_traceback_later() required") def test_timeout(self): - ns = libregrtest._parse_args(['--timeout', '4.2']) + ns = self.parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) + + # negative, zero and empty string are treated as "no timeout" + for value in ('-1', '0', ''): + with self.subTest(value=value): + ns = self.parse_args([f'--timeout={value}']) + self.assertEqual(ns.timeout, None) + self.checkError(['--timeout'], 'expected one argument') - self.checkError(['--timeout', 'foo'], 'invalid float value') + self.checkError(['--timeout', 'foo'], 'invalid timeout value:') def test_wait(self): - ns = libregrtest._parse_args(['--wait']) + ns = self.parse_args(['--wait']) self.assertTrue(ns.wait) - def test_worker_args(self): - ns = libregrtest._parse_args(['--worker-args', '[[], {}]']) - self.assertEqual(ns.worker_args, '[[], {}]') - self.checkError(['--worker-args'], 'expected one argument') - def test_start(self): for opt in '-S', '--start': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.start, 'foo') self.checkError([opt], 'expected one argument') def test_verbose(self): - ns = libregrtest._parse_args(['-v']) + ns = self.parse_args(['-v']) self.assertEqual(ns.verbose, 1) - ns = libregrtest._parse_args(['-vvv']) + ns = self.parse_args(['-vvv']) self.assertEqual(ns.verbose, 3) - ns = libregrtest._parse_args(['--verbose']) + ns = self.parse_args(['--verbose']) self.assertEqual(ns.verbose, 1) - ns = libregrtest._parse_args(['--verbose'] * 3) + ns = self.parse_args(['--verbose'] * 3) self.assertEqual(ns.verbose, 3) - ns = libregrtest._parse_args([]) + ns = self.parse_args([]) self.assertEqual(ns.verbose, 0) - def test_verbose2(self): - for opt in '-w', '--verbose2': + def test_rerun(self): + for opt in '-w', '--rerun', '--verbose2': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) - self.assertTrue(ns.verbose2) + ns = self.parse_args([opt]) + self.assertTrue(ns.rerun) def test_verbose3(self): for opt in '-W', '--verbose3': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.verbose3) def test_quiet(self): for opt in '-q', '--quiet': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_slowest(self): for opt in '-o', '--slowest': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.print_slow) def test_header(self): - ns = libregrtest._parse_args(['--header']) + ns = self.parse_args(['--header']) self.assertTrue(ns.header) - ns = libregrtest._parse_args(['--verbose']) + ns = self.parse_args(['--verbose']) self.assertTrue(ns.header) def test_randomize(self): - for opt in '-r', '--randomize': + for opt in ('-r', '--randomize'): with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.randomize) + with os_helper.EnvironmentVarGuard() as env: + # with SOURCE_DATE_EPOCH + env['SOURCE_DATE_EPOCH'] = '1697839080' + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertFalse(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, str) + self.assertEqual(regrtest.random_seed, '1697839080') + + # without SOURCE_DATE_EPOCH + del env['SOURCE_DATE_EPOCH'] + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertTrue(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, int) + + def test_no_randomize(self): + ns = self.parse_args([]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize"]) + self.assertIs(ns.randomize, True) + + ns = self.parse_args(["--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize", "--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--no-randomize", "--randomize"]) + self.assertIs(ns.randomize, False) + def test_randseed(self): - ns = libregrtest._parse_args(['--randseed', '12345']) + ns = self.parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) self.assertTrue(ns.randomize) self.checkError(['--randseed'], 'expected one argument') self.checkError(['--randseed', 'foo'], 'invalid int value') + ns = self.parse_args(['--randseed', '12345', '--no-randomize']) + self.assertEqual(ns.random_seed, 12345) + self.assertFalse(ns.randomize) + def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.fromfile, 'foo') self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo', '-s'], "don't go together") @@ -148,46 +220,37 @@ def test_fromfile(self): def test_exclude(self): for opt in '-x', '--exclude': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.exclude) def test_single(self): for opt in '-s', '--single': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_ignore(self): - for opt in '-i', '--ignore': + def test_match(self): + for opt in '-m', '--match': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'pattern']) - self.assertEqual(ns.ignore_tests, ['pattern']) + ns = self.parse_args([opt, 'pattern']) + self.assertEqual(ns.match_tests, [('pattern', True)]) self.checkError([opt], 'expected one argument') - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with open(os_helper.TESTFN, "w") as fp: - print('matchfile1', file=fp) - print('matchfile2', file=fp) - - filename = os.path.abspath(os_helper.TESTFN) - ns = libregrtest._parse_args(['-m', 'match', - '--ignorefile', filename]) - self.assertEqual(ns.ignore_tests, - ['matchfile1', 'matchfile2']) - - def test_match(self): - for opt in '-m', '--match': + for opt in '-i', '--ignore': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, ['pattern']) + ns = self.parse_args([opt, 'pattern']) + self.assertEqual(ns.match_tests, [('pattern', False)]) self.checkError([opt], 'expected one argument') - ns = libregrtest._parse_args(['-m', 'pattern1', - '-m', 'pattern2']) - self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', True)]) + + ns = self.parse_args(['-m', 'pattern1', '-i', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', False)]) + + ns = self.parse_args(['-i', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', False), ('pattern2', True)]) self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, "w") as fp: @@ -195,73 +258,76 @@ def test_match(self): print('matchfile2', file=fp) filename = os.path.abspath(os_helper.TESTFN) - ns = libregrtest._parse_args(['-m', 'match', - '--matchfile', filename]) + ns = self.parse_args(['-m', 'match', '--matchfile', filename]) + self.assertEqual(ns.match_tests, + [('match', True), ('matchfile1', True), ('matchfile2', True)]) + + ns = self.parse_args(['-i', 'match', '--ignorefile', filename]) self.assertEqual(ns.match_tests, - ['match', 'matchfile1', 'matchfile2']) + [('match', False), ('matchfile1', False), ('matchfile2', False)]) def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '-v']) + ns = self.parse_args([opt, '-v']) self.assertTrue(ns.failfast) - ns = libregrtest._parse_args([opt, '-W']) + ns = self.parse_args([opt, '-W']) self.assertTrue(ns.failfast) self.checkError([opt], '-G/--failfast needs either -v or -W') def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'gui,network']) + ns = self.parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) - ns = libregrtest._parse_args([opt, 'gui,none,network']) + ns = self.parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(libregrtest.ALL_RESOURCES) + expected = list(cmdline.ALL_RESOURCES) expected.remove('gui') - ns = libregrtest._parse_args([opt, 'all,-gui']) + ns = self.parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') # all + a resource not part of "all" - ns = libregrtest._parse_args([opt, 'all,tzdata']) + ns = self.parse_args([opt, 'all,tzdata']) self.assertEqual(ns.use_resources, - list(libregrtest.ALL_RESOURCES) + ['tzdata']) + list(cmdline.ALL_RESOURCES) + ['tzdata']) # test another resource which is not part of "all" - ns = libregrtest._parse_args([opt, 'extralargefile']) + ns = self.parse_args([opt, 'extralargefile']) self.assertEqual(ns.use_resources, ['extralargefile']) def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '4G']) + ns = self.parse_args([opt, '4G']) self.assertEqual(ns.memlimit, '4G') self.checkError([opt], 'expected one argument') def test_testdir(self): - ns = libregrtest._parse_args(['--testdir', 'foo']) + ns = self.parse_args(['--testdir', 'foo']) self.assertEqual(ns.testdir, os.path.join(os_helper.SAVEDCWD, 'foo')) self.checkError(['--testdir'], 'expected one argument') def test_runleaks(self): for opt in '-L', '--runleaks': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.runleaks) def test_huntrleaks(self): for opt in '-R', '--huntrleaks': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, ':']) + ns = self.parse_args([opt, ':']) self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) - ns = libregrtest._parse_args([opt, '6:']) + ns = self.parse_args([opt, '6:']) self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) - ns = libregrtest._parse_args([opt, ':3']) + ns = self.parse_args([opt, ':3']) self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) - ns = libregrtest._parse_args([opt, '6:3:leaks.log']) + ns = self.parse_args([opt, '6:3:leaks.log']) self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) self.checkError([opt], 'expected one argument') self.checkError([opt, '6'], @@ -272,23 +338,33 @@ def test_huntrleaks(self): def test_multiprocess(self): for opt in '-j', '--multiprocess': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '2']) + ns = self.parse_args([opt, '2']) self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') - self.checkError([opt, '2', '-T'], "don't go together") - self.checkError([opt, '0', '-T'], "don't go together") - def test_coverage(self): + def test_coverage_sequential(self): + for opt in '-T', '--coverage': + with self.subTest(opt=opt): + with support.captured_stderr() as stderr: + ns = self.parse_args([opt]) + self.assertTrue(ns.trace) + self.assertIn( + "collecting coverage without -j is imprecise", + stderr.getvalue(), + ) + + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_coverage_mp(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt, '-j1']) self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, 'foo']) + ns = self.parse_args([opt, 'foo']) self.assertEqual(ns.coverdir, os.path.join(os_helper.SAVEDCWD, 'foo')) self.checkError([opt], 'expected one argument') @@ -296,13 +372,13 @@ def test_coverdir(self): def test_nocoverdir(self): for opt in '-N', '--nocoverdir': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertIsNone(ns.coverdir) def test_threshold(self): for opt in '-t', '--threshold': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt, '1000']) + ns = self.parse_args([opt, '1000']) self.assertEqual(ns.threshold, 1000) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -311,7 +387,7 @@ def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): with contextlib.redirect_stderr(io.StringIO()) as stderr: - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.nowindows) err = stderr.getvalue() self.assertIn('the --nowindows (-n) option is deprecated', err) @@ -319,39 +395,39 @@ def test_nowindows(self): def test_forever(self): for opt in '-F', '--forever': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + ns = self.parse_args([opt]) self.assertTrue(ns.forever) def test_unrecognized_argument(self): self.checkError(['--xxx'], 'usage:') def test_long_option__partial(self): - ns = libregrtest._parse_args(['--qui']) + ns = self.parse_args(['--qui']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_two_options(self): - ns = libregrtest._parse_args(['--quiet', '--exclude']) + ns = self.parse_args(['--quiet', '--exclude']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertTrue(ns.exclude) def test_option_with_empty_string_value(self): - ns = libregrtest._parse_args(['--start', '']) + ns = self.parse_args(['--start', '']) self.assertEqual(ns.start, '') def test_arg(self): - ns = libregrtest._parse_args(['foo']) + ns = self.parse_args(['foo']) self.assertEqual(ns.args, ['foo']) def test_option_and_arg(self): - ns = libregrtest._parse_args(['--quiet', 'foo']) + ns = self.parse_args(['--quiet', 'foo']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertEqual(ns.args, ['foo']) def test_arg_option_arg(self): - ns = libregrtest._parse_args(['test_unaryop', '-v', 'test_binop']) + ns = self.parse_args(['test_unaryop', '-v', 'test_binop']) self.assertEqual(ns.verbose, 1) self.assertEqual(ns.args, ['test_unaryop', 'test_binop']) @@ -359,6 +435,118 @@ def test_unknown_option(self): self.checkError(['--unknown-option'], 'unrecognized arguments: --unknown-option') + def create_regrtest(self, args): + ns = cmdline._parse_args(args) + + # Check Regrtest attributes which are more reliable than Namespace + # which has an unclear API + with os_helper.EnvironmentVarGuard() as env: + # Ignore SOURCE_DATE_EPOCH env var if it's set + del env['SOURCE_DATE_EPOCH'] + + regrtest = main.Regrtest(ns) + + return regrtest + + def check_ci_mode(self, args, use_resources, + *, rerun=True, randomize=True, output_on_failure=True): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, -1) + self.assertEqual(regrtest.want_rerun, rerun) + self.assertEqual(regrtest.fail_rerun, False) + self.assertEqual(regrtest.randomize, randomize) + self.assertIsInstance(regrtest.random_seed, int) + self.assertTrue(regrtest.fail_env_changed) + self.assertTrue(regrtest.print_slowest) + self.assertEqual(regrtest.output_on_failure, output_on_failure) + self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources)) + return regrtest + + def test_fast_ci(self): + args = ['--fast-ci'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources) + self.assertEqual(regrtest.timeout, 10 * 60) + + def test_fast_ci_python_cmd(self): + args = ['--fast-ci', '--python', 'python -X dev'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources, rerun=False) + self.assertEqual(regrtest.timeout, 10 * 60) + self.assertEqual(regrtest.python_cmd, ('python', '-X', 'dev')) + + def test_fast_ci_resource(self): + # it should be possible to override resources individually + args = ['--fast-ci', '-u-network'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + use_resources.remove('network') + self.check_ci_mode(args, use_resources) + + def test_fast_ci_verbose(self): + args = ['--fast-ci', '--verbose'] + use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources.remove('cpu') + regrtest = self.check_ci_mode(args, use_resources, + output_on_failure=False) + self.assertEqual(regrtest.verbose, True) + + def test_slow_ci(self): + args = ['--slow-ci'] + use_resources = sorted(cmdline.ALL_RESOURCES) + regrtest = self.check_ci_mode(args, use_resources) + self.assertEqual(regrtest.timeout, 20 * 60) + + def test_ci_no_randomize(self): + all_resources = set(cmdline.ALL_RESOURCES) + self.check_ci_mode( + ["--slow-ci", "--no-randomize"], all_resources, randomize=False + ) + self.check_ci_mode( + ["--fast-ci", "--no-randomize"], all_resources - {'cpu'}, randomize=False + ) + + def test_dont_add_python_opts(self): + args = ['--dont-add-python-opts'] + ns = cmdline._parse_args(args) + self.assertFalse(ns._add_python_opts) + + def test_bisect(self): + args = ['--bisect'] + regrtest = self.create_regrtest(args) + self.assertTrue(regrtest.want_bisect) + + def test_verbose3_huntrleaks(self): + args = ['-R', '3:10', '--verbose3'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertIsNotNone(regrtest.hunt_refleak) + self.assertEqual(regrtest.hunt_refleak.warmups, 3) + self.assertEqual(regrtest.hunt_refleak.runs, 10) + self.assertFalse(regrtest.output_on_failure) + + def test_single_process(self): + args = ['-j2', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + + args = ['--fast-ci', '--single-process'] + with support.captured_stderr(): + regrtest = self.create_regrtest(args) + self.assertEqual(regrtest.num_workers, 0) + self.assertTrue(regrtest.single_process) + + +@dataclasses.dataclass(slots=True) +class Rerun: + name: str + match: str | None + success: bool + class BaseTestCase(unittest.TestCase): TEST_UNIQUE_ID = 1 @@ -407,41 +595,61 @@ def regex_search(self, regex, output): self.fail("%r not found in %r" % (regex, output)) return match - def check_line(self, output, regex): - regex = re.compile(r'^' + regex, re.MULTILINE) + def check_line(self, output, pattern, full=False, regex=True): + if not regex: + pattern = re.escape(pattern) + if full: + pattern += '\n' + regex = re.compile(r'^' + pattern, re.MULTILINE) self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = (r'^%s\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' - % (LOG_PREFIX, self.TESTNAME_REGEX)) + regex = (fr'^{LOG_PREFIX}\[ *[0-9]+(?:/ *[0-9]+)*\] ' + fr'({self.TESTNAME_REGEX}) {RESULT_REGEX}') parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) - def check_executed_tests(self, output, tests, skipped=(), failed=(), + def check_executed_tests(self, output, tests, *, stats, + skipped=(), failed=(), env_changed=(), omitted=(), - rerun=(), no_test_ran=(), - randomize=False, interrupted=False, - fail_env_changed=False): + rerun=None, run_no_tests=(), + resource_denied=(), + randomize=False, parallel=False, interrupted=False, + fail_env_changed=False, + forever=False, filtered=False): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): skipped = [skipped] + if isinstance(resource_denied, str): + resource_denied = [resource_denied] if isinstance(failed, str): failed = [failed] if isinstance(env_changed, str): env_changed = [env_changed] if isinstance(omitted, str): omitted = [omitted] - if isinstance(rerun, str): - rerun = [rerun] - if isinstance(no_test_ran, str): - no_test_ran = [no_test_ran] + if isinstance(run_no_tests, str): + run_no_tests = [run_no_tests] + if isinstance(stats, int): + stats = TestStats(stats) + if parallel: + randomize = True + + rerun_failed = [] + if rerun is not None and not env_changed: + failed = [rerun.name] + if not rerun.success: + rerun_failed.append(rerun.name) executed = self.parse_executed_tests(output) + total_tests = list(tests) + if rerun is not None: + total_tests.append(rerun.name) if randomize: - self.assertEqual(set(executed), set(tests), output) + self.assertEqual(set(executed), set(total_tests), output) else: - self.assertEqual(executed, tests, output) + self.assertEqual(executed, total_tests, output) def plural(count): return 's' if count != 1 else '' @@ -457,12 +665,17 @@ def list_regex(line_format, tests): regex = list_regex('%s test%s skipped', skipped) self.check_line(output, regex) + if resource_denied: + regex = list_regex(r'%s test%s skipped \(resource denied\)', resource_denied) + self.check_line(output, regex) + if failed: regex = list_regex('%s test%s failed', failed) self.check_line(output, regex) if env_changed: - regex = list_regex('%s test%s altered the execution environment', + regex = list_regex(r'%s test%s altered the execution environment ' + r'\(env changed\)', env_changed) self.check_line(output, regex) @@ -470,73 +683,120 @@ def list_regex(line_format, tests): regex = list_regex('%s test%s omitted', omitted) self.check_line(output, regex) - if rerun: - regex = list_regex('%s re-run test%s', rerun) + if rerun is not None: + regex = list_regex('%s re-run test%s', [rerun.name]) self.check_line(output, regex) - regex = LOG_PREFIX + r"Re-running failed tests in verbose mode" + regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode" + self.check_line(output, regex) + regex = fr"Re-running {rerun.name} in verbose mode" + if rerun.match: + regex = fr"{regex} \(matching: {rerun.match}\)" self.check_line(output, regex) - for test_name in rerun: - regex = LOG_PREFIX + f"Re-running {test_name} in verbose mode" - self.check_line(output, regex) - if no_test_ran: - regex = list_regex('%s test%s run no tests', no_test_ran) + if run_no_tests: + regex = list_regex('%s test%s run no tests', run_no_tests) self.check_line(output, regex) - good = (len(tests) - len(skipped) - len(failed) - - len(omitted) - len(env_changed) - len(no_test_ran)) + good = (len(tests) - len(skipped) - len(resource_denied) - len(failed) + - len(omitted) - len(env_changed) - len(run_no_tests)) if good: - regex = r'%s test%s OK\.$' % (good, plural(good)) - if not skipped and not failed and good > 1: + regex = r'%s test%s OK\.' % (good, plural(good)) + if not skipped and not failed and (rerun is None or rerun.success) and good > 1: regex = 'All %s' % regex - self.check_line(output, regex) + self.check_line(output, regex, full=True) if interrupted: self.check_line(output, 'Test suite interrupted by signal SIGINT.') - result = [] + # Total tests + text = f'run={stats.tests_run:,}' + if filtered: + text = fr'{text} \(filtered\)' + parts = [text] + if stats.failures: + parts.append(f'failures={stats.failures:,}') + if stats.skipped: + parts.append(f'skipped={stats.skipped:,}') + line = fr'Total tests: {" ".join(parts)}' + self.check_line(output, line, full=True) + + # Total test files + run = len(total_tests) - len(resource_denied) + if rerun is not None: + total_failed = len(rerun_failed) + total_rerun = 1 + else: + total_failed = len(failed) + total_rerun = 0 + if interrupted: + run = 0 + text = f'run={run}' + if not forever: + text = f'{text}/{len(tests)}' + if filtered: + text = fr'{text} \(filtered\)' + report = [text] + for name, ntest in ( + ('failed', total_failed), + ('env_changed', len(env_changed)), + ('skipped', len(skipped)), + ('resource_denied', len(resource_denied)), + ('rerun', total_rerun), + ('run_no_tests', len(run_no_tests)), + ): + if ntest: + report.append(f'{name}={ntest}') + line = fr'Total test files: {" ".join(report)}' + self.check_line(output, line, full=True) + + # Result + state = [] if failed: - result.append('FAILURE') + state.append('FAILURE') elif fail_env_changed and env_changed: - result.append('ENV CHANGED') + state.append('ENV CHANGED') if interrupted: - result.append('INTERRUPTED') - if not any((good, result, failed, interrupted, skipped, + state.append('INTERRUPTED') + if not any((good, failed, interrupted, skipped, env_changed, fail_env_changed)): - result.append("NO TEST RUN") - elif not result: - result.append('SUCCESS') - result = ', '.join(result) - if rerun: - self.check_line(output, 'Tests result: FAILURE') - result = 'FAILURE then %s' % result - - self.check_line(output, 'Tests result: %s' % result) - - def parse_random_seed(self, output): - match = self.regex_search(r'Using random seed ([0-9]+)', output) - randseed = int(match.group(1)) - self.assertTrue(0 <= randseed <= 10000000, randseed) - return randseed + state.append("NO TESTS RAN") + elif not state: + state.append('SUCCESS') + state = ', '.join(state) + if rerun is not None: + new_state = 'SUCCESS' if rerun.success else 'FAILURE' + state = f'{state} then {new_state}' + self.check_line(output, f'Result: {state}', full=True) + + def parse_random_seed(self, output: str) -> str: + match = self.regex_search(r'Using random seed: (.*)', output) + return match.group(1) def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' if 'stderr' not in kw: - kw['stderr'] = subprocess.PIPE + kw['stderr'] = subprocess.STDOUT + + env = kw.pop('env', None) + if env is None: + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + proc = subprocess.run(args, - universal_newlines=True, + text=True, input=input, stdout=subprocess.PIPE, + env=env, **kw) if proc.returncode != exitcode: - msg = ("Command %s failed with exit code %s\n" + msg = ("Command %s failed with exit code %s, but exit code %s expected!\n" "\n" "stdout:\n" "---\n" "%s\n" "---\n" - % (str(args), proc.returncode, proc.stdout)) + % (str(args), proc.returncode, exitcode, proc.stdout)) if proc.stderr: msg += ("\n" "stderr:\n" @@ -547,18 +807,24 @@ def run_command(self, args, input=None, exitcode=0, **kw): self.fail(msg) return proc - def run_python(self, args, **kw): - args = [sys.executable, '-X', 'faulthandler', '-I', *args] - proc = self.run_command(args, **kw) + def run_python(self, args, isolated=True, **kw): + extraargs = [] + if 'uops' in sys._xoptions: + # Pass -X uops along + extraargs.extend(['-X', 'uops']) + cmd = [sys.executable, *extraargs, '-X', 'faulthandler'] + if isolated: + cmd.append('-I') + cmd.extend(args) + proc = self.run_command(cmd, **kw) return proc.stdout class CheckActualTests(BaseTestCase): - """ - Check that regrtest appears to find the expected set of tests. - """ - def test_finds_expected_number_of_tests(self): + """ + Check that regrtest appears to find the expected set of tests. + """ args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests'] output = self.run_python(args) rough_number_of_tests_found = len(output.splitlines()) @@ -578,6 +844,7 @@ def test_finds_expected_number_of_tests(self): f'{", ".join(output.splitlines())}') +@support.force_not_colorized_test_class class ProgramsTestCase(BaseTestCase): """ Test various ways to run the Python test suite. Use options close @@ -595,17 +862,19 @@ def setUp(self): self.python_args = ['-Wd', '-E', '-bb'] self.regrtest_args = ['-uall', '-rwW', '--testdir=%s' % self.tmptestdir] - if hasattr(faulthandler, 'dump_traceback_later'): - self.regrtest_args.extend(('--timeout', '3600', '-j4')) + self.regrtest_args.extend(('--timeout', '3600', '-j4')) if sys.platform == 'win32': self.regrtest_args.append('-n') def check_output(self, output): - self.parse_random_seed(output) - self.check_executed_tests(output, self.tests, randomize=True) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) - def run_tests(self, args): - output = self.run_python(args) + self.check_executed_tests(output, self.tests, + randomize=True, stats=len(self.tests)) + + def run_tests(self, args, env=None, isolated=True): + output = self.run_python(args, env=env, isolated=isolated) self.check_output(output) def test_script_regrtest(self): @@ -615,6 +884,7 @@ def test_script_regrtest(self): args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) + @unittest.skip("TODO: RUSTPYTHON; flaky") def test_module_test(self): # -m test args = [*self.python_args, '-m', 'test', @@ -627,16 +897,14 @@ def test_module_regrtest(self): *self.regrtest_args, *self.tests] self.run_tests(args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; flaky") def test_module_autotest(self): # -m test.autotest args = [*self.python_args, '-m', 'test.autotest', *self.regrtest_args, *self.tests] self.run_tests(args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; flaky") def test_module_from_test_autotest(self): # from test import autotest code = 'from test import autotest' @@ -644,24 +912,18 @@ def test_module_from_test_autotest(self): *self.regrtest_args, *self.tests] self.run_tests(args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; flaky") def test_script_autotest(self): # Lib/test/autotest.py script = os.path.join(self.testdir, 'autotest.py') args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) - @unittest.skipUnless(sysconfig.is_python_build(), - 'run_tests.py script is not installed') - def test_tools_script_run_tests(self): - # Tools/scripts/run_tests.py - script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') - args = [script, *self.regrtest_args, *self.tests] - self.run_tests(args) - def run_batch(self, *args): - proc = self.run_command(args) + proc = self.run_command(args, + # gh-133711: cmd.exe uses the OEM code page + # to display the non-ASCII current directory + errors="backslashreplace") self.check_output(proc.stdout) @unittest.skipUnless(sysconfig.is_python_build(), @@ -673,10 +935,14 @@ def test_tools_buildbot_test(self): test_args = ['--testdir=%s' % self.tmptestdir] if platform.machine() == 'ARM64': test_args.append('-arm64') # ARM 64-bit build + elif platform.machine() == 'ARM': + test_args.append('-arm32') # 32-bit ARM build elif platform.architecture()[0] == '64bit': test_args.append('-x64') # 64-bit build - if not Py_DEBUG: + if not support.Py_DEBUG: test_args.append('+d') # Release build, use python.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + test_args.append('--disable-gil') self.run_batch(script, *test_args, *self.tests) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') @@ -688,13 +954,18 @@ def test_pcbuild_rt(self): rt_args = ["-q"] # Quick, don't run tests twice if platform.machine() == 'ARM64': rt_args.append('-arm64') # ARM 64-bit build + elif platform.machine() == 'ARM': + rt_args.append('-arm32') # 32-bit ARM build elif platform.architecture()[0] == '64bit': rt_args.append('-x64') # 64-bit build - if Py_DEBUG: + if support.Py_DEBUG: rt_args.append('-d') # Debug build, use python_d.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + rt_args.append('--disable-gil') self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) +@support.force_not_colorized_test_class class ArgsTestCase(BaseTestCase): """ Test arguments of the Python test suite. @@ -704,6 +975,40 @@ def run_tests(self, *testargs, **kw): cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] return self.run_python(cmdargs, **kw) + def test_success(self): + code = textwrap.dedent(""" + import unittest + + class PassingTests(unittest.TestCase): + def test_test1(self): + pass + + def test_test2(self): + pass + + def test_test3(self): + pass + """) + tests = [self.create_test(f'ok{i}', code=code) for i in range(1, 6)] + + output = self.run_tests(*tests) + self.check_executed_tests(output, tests, + stats=3 * len(tests)) + + def test_skip(self): + code = textwrap.dedent(""" + import unittest + raise unittest.SkipTest("nope") + """) + test_ok = self.create_test('ok') + test_skip = self.create_test('skip', code=code) + tests = [test_ok, test_skip] + + output = self.run_tests(*tests) + self.check_executed_tests(output, tests, + skipped=[test_skip], + stats=1) + def test_failing_test(self): # test a failing test code = textwrap.dedent(""" @@ -717,8 +1022,9 @@ def test_failing(self): test_failing = self.create_test('failing', code=code) tests = [test_ok, test_failing] - output = self.run_tests(*tests, exitcode=2) - self.check_executed_tests(output, tests, failed=test_failing) + output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, tests, failed=test_failing, + stats=TestStats(2, 1)) def test_resources(self): # test -u command line option @@ -737,17 +1043,19 @@ def test_pass(self): # -u all: 2 resources enabled output = self.run_tests('-u', 'all', *test_names) - self.check_executed_tests(output, test_names) + self.check_executed_tests(output, test_names, stats=2) # -u audio: 1 resource enabled output = self.run_tests('-uaudio', *test_names) self.check_executed_tests(output, test_names, - skipped=tests['network']) + resource_denied=tests['network'], + stats=1) # no option: 0 resources enabled - output = self.run_tests(*test_names) + output = self.run_tests(*test_names, exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, test_names, - skipped=test_names) + resource_denied=test_names, + stats=0) def test_random(self): # test -r and --randseed command line option @@ -758,13 +1066,14 @@ def test_random(self): test = self.create_test('random', code) # first run to get the output with the random seed - output = self.run_tests('-r', test) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN) randseed = self.parse_random_seed(output) match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) test_random = int(match.group(1)) # try to reproduce with the random seed - output = self.run_tests('-r', '--randseed=%s' % randseed, test) + output = self.run_tests('-r', f'--randseed={randseed}', test, + exitcode=EXITCODE_NO_TESTS_RAN) randseed2 = self.parse_random_seed(output) self.assertEqual(randseed2, randseed) @@ -772,6 +1081,35 @@ def test_random(self): test_random2 = int(match.group(1)) self.assertEqual(test_random2, test_random) + # check that random.seed is used by default + output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + + # check SOURCE_DATE_EPOCH (integer) + timestamp = '1697839080' + env = dict(os.environ, SOURCE_DATE_EPOCH=timestamp) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, timestamp) + self.check_line(output, 'TESTRANDOM: 520') + + # check SOURCE_DATE_EPOCH (string) + env = dict(os.environ, SOURCE_DATE_EPOCH='XYZ') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, 'XYZ') + self.check_line(output, 'TESTRANDOM: 22') + + # check SOURCE_DATE_EPOCH (empty string): ignore the env var + env = dict(os.environ, SOURCE_DATE_EPOCH='') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + def test_fromfile(self): # test --fromfile tests = [self.create_test() for index in range(5)] @@ -794,7 +1132,8 @@ def test_fromfile(self): previous = name output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + stats = len(tests) + self.check_executed_tests(output, tests, stats=stats) # test format '[2/7] test_opcodes' with open(filename, "w") as fp: @@ -802,7 +1141,7 @@ def test_fromfile(self): print("[%s/%s] %s" % (index, len(tests), name), file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) # test format 'test_opcodes' with open(filename, "w") as fp: @@ -810,7 +1149,7 @@ def test_fromfile(self): print(name, file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) # test format 'Lib/test/test_opcodes.py' with open(filename, "w") as fp: @@ -818,29 +1157,25 @@ def test_fromfile(self): print('Lib/test/%s.py' % name, file=fp) output = self.run_tests('--fromfile', filename) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=stats) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) - output = self.run_tests(test, exitcode=130) + output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED) self.check_executed_tests(output, test, omitted=test, - interrupted=True) + interrupted=True, stats=0) def test_slowest(self): # test --slowest tests = [self.create_test() for index in range(3)] output = self.run_tests("--slowest", *tests) - self.check_executed_tests(output, tests) + self.check_executed_tests(output, tests, stats=len(tests)) regex = ('10 slowest tests:\n' '(?:- %s: .*\n){%s}' % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_slowest_interrupted(self): # Issue #25373: test --slowest with an interrupted test code = TEST_INTERRUPTED @@ -852,22 +1187,22 @@ def test_slowest_interrupted(self): args = ("--slowest", "-j2", test) else: args = ("--slowest", test) - output = self.run_tests(*args, exitcode=130) + output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED) self.check_executed_tests(output, test, - omitted=test, interrupted=True) + omitted=test, interrupted=True, + stats=0) regex = ('10 slowest tests:\n') self.check_line(output, regex) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Regex didn't match: '^lines +cov% +module +\\(path\\)\\n(?: *[0-9]+ *[0-9]{1,2}\\.[0-9]% *[^ ]+ +\\([^)]+\\)+)+' not found in 'Warning: collecting coverage without -j is imprecise. Configure --with-pydebug and run -m test -T -j for best results.\nUsing random seed: 2780369491\n0:00:00 Run 1 test sequentially in a single process\n0:00:00 [1/1] test_regrtest_coverage\n0:00:00 [1/1] test_regrtest_coverage passed\n\n== Tests result: SUCCESS ==\n\n1 test OK.\n\nTotal duration: 102 ms\nTotal tests: run=1\nTotal test files: run=1/1\nResult: SUCCESS\n' def test_coverage(self): # test --coverage test = self.create_test('coverage') output = self.run_tests("--coverage", test) - self.check_executed_tests(output, [test]) + self.check_executed_tests(output, [test], stats=1) regex = (r'lines +cov% +module +\(path\)\n' - r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') + r'(?: *[0-9]+ *[0-9]{1,2}\.[0-9]% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) def test_wait(self): @@ -894,21 +1229,39 @@ def test_run(self): builtins.__dict__['RUN'] = 1 """) test = self.create_test('forever', code=code) - output = self.run_tests('--forever', test, exitcode=2) - self.check_executed_tests(output, [test]*3, failed=test) - def check_leak(self, code, what): + # --forever + output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [test]*3, failed=test, + stats=TestStats(3, 1), + forever=True) + + # --forever --rerun + output = self.run_tests('--forever', '--rerun', test, exitcode=0) + self.check_executed_tests(output, [test]*3, + rerun=Rerun(test, + match='test_run', + success=True), + stats=TestStats(4, 1), + forever=True) + + @support.requires_jit_disabled + def check_leak(self, code, what, *, run_workers=False): test = self.create_test('huntrleaks', code=code) filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) - output = self.run_tests('--huntrleaks', '3:3:', test, - exitcode=2, + cmd = ['--huntrleaks', '3:3:'] + if run_workers: + cmd.append('-j1') + cmd.append(test) + output = self.run_tests(*cmd, + exitcode=EXITCODE_BAD_TEST, stderr=subprocess.STDOUT) - self.check_executed_tests(output, [test], failed=test) + self.check_executed_tests(output, [test], failed=test, stats=1) - line = 'beginning 6 repetitions\n123456\n......\n' - self.check_line(output, re.escape(line)) + line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n' + self.check_line(output, line) line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) self.assertIn(line2, output) @@ -917,8 +1270,8 @@ def check_leak(self, code, what): reflog = fp.read() self.assertIn(line2, reflog) - @unittest.skipUnless(Py_DEBUG, 'need a debug build') - def test_huntrleaks(self): + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def check_huntrleaks(self, *, run_workers: bool): # test --huntrleaks code = textwrap.dedent(""" import unittest @@ -929,9 +1282,56 @@ class RefLeakTest(unittest.TestCase): def test_leak(self): GLOBAL_LIST.append(object()) """) - self.check_leak(code, 'references') + self.check_leak(code, 'references', run_workers=run_workers) - @unittest.skipUnless(Py_DEBUG, 'need a debug build') + def test_huntrleaks(self): + self.check_huntrleaks(run_workers=False) + + def test_huntrleaks_mp(self): + self.check_huntrleaks(run_workers=True) + + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') + def test_huntrleaks_bisect(self): + # test --huntrleaks --bisect + code = textwrap.dedent(""" + import unittest + + GLOBAL_LIST = [] + + class RefLeakTest(unittest.TestCase): + def test1(self): + pass + + def test2(self): + pass + + def test3(self): + GLOBAL_LIST.append(object()) + + def test4(self): + pass + """) + + test = self.create_test('huntrleaks', code=code) + + filename = 'reflog.txt' + self.addCleanup(os_helper.unlink, filename) + cmd = ['--huntrleaks', '3:3:', '--bisect', test] + output = self.run_tests(*cmd, + exitcode=EXITCODE_BAD_TEST, + stderr=subprocess.STDOUT) + + self.assertIn(f"Bisect {test}", output) + self.assertIn(f"Bisect {test}: exit code 0", output) + + # test3 is the one which leaks + self.assertIn("Bisection completed in", output) + self.assertIn( + "Tests (1):\n" + f"* {test}.RefLeakTest.test3\n", + output) + + @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" @@ -945,7 +1345,6 @@ def test_leak(self): """) self.check_leak(code, 'file descriptors') - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_list_tests(self): # test --list-tests tests = [self.create_test() for i in range(5)] @@ -953,7 +1352,6 @@ def test_list_tests(self): self.assertEqual(output.rstrip().splitlines(), tests) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_list_cases(self): # test --list-cases code = textwrap.dedent(""" @@ -987,16 +1385,14 @@ def test_crashed(self): crash_test = self.create_test(name="crash", code=code) tests = [crash_test] - output = self.run_tests("-j2", *tests, exitcode=2) + output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, tests, failed=crash_test, - randomize=True) + parallel=True, stats=0) def parse_methods(self, output): regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) return [match.group(1) for match in regex.finditer(output)] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ignorefile(self): code = textwrap.dedent(""" import unittest @@ -1011,8 +1407,6 @@ def test_method3(self): def test_method4(self): pass """) - all_methods = ['test_method1', 'test_method2', - 'test_method3', 'test_method4'] testname = self.create_test(code=code) # only run a subset @@ -1074,8 +1468,6 @@ def test_method4(self): subset = ['test_method1', 'test_method3'] self.assertEqual(methods, subset) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_env_changed(self): code = textwrap.dedent(""" import unittest @@ -1088,52 +1480,265 @@ def test_env_changed(self): # don't fail by default output = self.run_tests(testname) - self.check_executed_tests(output, [testname], env_changed=testname) + self.check_executed_tests(output, [testname], + env_changed=testname, stats=1) # fail with --fail-env-changed - output = self.run_tests("--fail-env-changed", testname, exitcode=3) + output = self.run_tests("--fail-env-changed", testname, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=testname, - fail_env_changed=True) + fail_env_changed=True, stats=1) + + # rerun + output = self.run_tests("--rerun", testname) + self.check_executed_tests(output, [testname], + env_changed=testname, + rerun=Rerun(testname, + match=None, + success=True), + stats=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_rerun_fail(self): # FAILURE then FAILURE code = textwrap.dedent(""" import unittest class Tests(unittest.TestCase): - def test_bug(self): - # test always fail + def test_succeed(self): + return + + def test_fail_always(self): + # test that always fails self.fail("bug") """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=2) + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], - failed=testname, rerun=testname) + rerun=Rerun(testname, + "test_fail_always", + success=False), + stats=TestStats(3, 2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_rerun_success(self): # FAILURE then SUCCESS - code = textwrap.dedent(""" - import builtins + marker_filename = os.path.abspath("regrtest_marker_filename") + self.addCleanup(os_helper.unlink, marker_filename) + self.assertFalse(os.path.exists(marker_filename)) + + code = textwrap.dedent(f""" + import os.path import unittest + marker_filename = {marker_filename!r} + class Tests(unittest.TestCase): - failed = False + def test_succeed(self): + return def test_fail_once(self): - if not hasattr(builtins, '_test_failed'): - builtins._test_failed = True + if not os.path.exists(marker_filename): + open(marker_filename, "w").close() self.fail("bug") """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=0) + # FAILURE then SUCCESS => exit code 0 + output = self.run_tests("--rerun", testname, exitcode=0) self.check_executed_tests(output, [testname], - rerun=testname) + rerun=Rerun(testname, + match="test_fail_once", + success=True), + stats=TestStats(3, 1)) + os_helper.unlink(marker_filename) + + # with --fail-rerun, exit code EXITCODE_RERUN_FAIL + # on "FAILURE then SUCCESS" state. + output = self.run_tests("--rerun", "--fail-rerun", testname, + exitcode=EXITCODE_RERUN_FAIL) + self.check_executed_tests(output, [testname], + rerun=Rerun(testname, + match="test_fail_once", + success=True), + stats=TestStats(3, 1)) + os_helper.unlink(marker_filename) + + def test_rerun_setup_class_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + @classmethod + def setUpClass(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="ExampleTests", + success=False), + stats=0) + + def test_rerun_teardown_class_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + @classmethod + def tearDownClass(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="ExampleTests", + success=False), + stats=2) + + def test_rerun_setup_module_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + def setUpModule(): + raise RuntimeError('Fail') + + class ExampleTests(unittest.TestCase): + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match=None, + success=False), + stats=0) + + def test_rerun_teardown_module_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + def tearDownModule(): + raise RuntimeError('Fail') + + class ExampleTests(unittest.TestCase): + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + rerun=Rerun(testname, + match=None, + success=False), + stats=2) + + def test_rerun_setup_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + def setUp(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) + + def test_rerun_teardown_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + def tearDown(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) + + def test_rerun_async_setup_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + raise RuntimeError('Fail') + + async def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) + + def test_rerun_async_teardown_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.IsolatedAsyncioTestCase): + async def asyncTearDown(self): + raise RuntimeError('Fail') + + async def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun=Rerun(testname, + match="test_success", + success=False), + stats=2) def test_no_tests_ran(self): code = textwrap.dedent(""" @@ -1145,8 +1750,11 @@ def test_bug(self): """) testname = self.create_test(code=code) - output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0) - self.check_executed_tests(output, [testname], no_test_ran=testname) + output = self.run_tests(testname, "-m", "nosuchtest", + exitcode=EXITCODE_NO_TESTS_RAN) + self.check_executed_tests(output, [testname], + run_no_tests=testname, + stats=0, filtered=True) def test_no_tests_ran_skip(self): code = textwrap.dedent(""" @@ -1158,8 +1766,9 @@ def test_skipped(self): """) testname = self.create_test(code=code) - output = self.run_tests(testname, exitcode=0) - self.check_executed_tests(output, [testname]) + output = self.run_tests(testname) + self.check_executed_tests(output, [testname], + stats=TestStats(1, skipped=1)) def test_no_tests_ran_multiple_tests_nonexistent(self): code = textwrap.dedent(""" @@ -1172,9 +1781,11 @@ def test_bug(self): testname = self.create_test(code=code) testname2 = self.create_test(code=code) - output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0) + output = self.run_tests(testname, testname2, "-m", "nosuchtest", + exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, [testname, testname2], - no_test_ran=[testname, testname2]) + run_no_tests=[testname, testname2], + stats=0, filtered=True) def test_no_test_ran_some_test_exist_some_not(self): code = textwrap.dedent(""" @@ -1197,10 +1808,14 @@ def test_other_bug(self): output = self.run_tests(testname, testname2, "-m", "nosuchtest", "-m", "test_other_bug", exitcode=0) self.check_executed_tests(output, [testname, testname2], - no_test_ran=[testname]) + run_no_tests=[testname], + stats=1, filtered=True) @support.cpython_only - def test_findleaks(self): + def test_uncollectable(self): + # Skip test if _testcapi is missing + import_helper.import_module('_testcapi') + code = textwrap.dedent(r""" import _testcapi import gc @@ -1220,19 +1835,13 @@ def test_garbage(self): """) testname = self.create_test(code=code) - output = self.run_tests("--fail-env-changed", testname, exitcode=3) - self.check_executed_tests(output, [testname], - env_changed=[testname], - fail_env_changed=True) - - # --findleaks is now basically an alias to --fail-env-changed - output = self.run_tests("--findleaks", testname, exitcode=3) + output = self.run_tests("--fail-env-changed", testname, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], - fail_env_changed=True) + fail_env_changed=True, + stats=1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multiprocessing_timeout(self): code = textwrap.dedent(r""" import time @@ -1254,14 +1863,131 @@ def test_sleep(self): """) testname = self.create_test(code=code) - output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2) + output = self.run_tests("-j2", "--timeout=1.0", testname, + exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], - failed=testname) + failed=testname, stats=0) self.assertRegex(output, re.compile('%s timed out' % testname, re.MULTILINE)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; test_unraisable_exc (test_regrtest_noop39.Tests.test_unraisable_exc) ... ok + def test_unraisable_exc(self): + # --fail-env-changed must catch unraisable exception. + # The exception must be displayed even if sys.stderr is redirected. + code = textwrap.dedent(r""" + import unittest + import weakref + from test.support import captured_stderr + + class MyObject: + pass + + def weakref_callback(obj): + raise Exception("weakref callback bug") + + class Tests(unittest.TestCase): + def test_unraisable_exc(self): + obj = MyObject() + ref = weakref.ref(obj, weakref_callback) + with captured_stderr() as stderr: + # call weakref_callback() which logs + # an unraisable exception + obj = None + self.assertEqual(stderr.getvalue(), '') + """) + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", testname, + exitcode=EXITCODE_ENV_CHANGED) + self.check_executed_tests(output, [testname], + env_changed=[testname], + fail_env_changed=True, + stats=1) + self.assertIn("Warning -- Unraisable exception", output) + self.assertIn("Exception: weakref callback bug", output) + + def test_threading_excepthook(self): + # --fail-env-changed must catch uncaught thread exception. + # The exception must be displayed even if sys.stderr is redirected. + code = textwrap.dedent(r""" + import threading + import unittest + from test.support import captured_stderr + + class MyObject: + pass + + def func_bug(): + raise Exception("bug in thread") + + class Tests(unittest.TestCase): + def test_threading_excepthook(self): + with captured_stderr() as stderr: + thread = threading.Thread(target=func_bug) + thread.start() + thread.join() + self.assertEqual(stderr.getvalue(), '') + """) + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", testname, + exitcode=EXITCODE_ENV_CHANGED) + self.check_executed_tests(output, [testname], + env_changed=[testname], + fail_env_changed=True, + stats=1) + self.assertIn("Warning -- Uncaught thread exception", output) + self.assertIn("Exception: bug in thread", output) + + def test_print_warning(self): + # bpo-45410: The order of messages must be preserved when -W and + # support.print_warning() are used. + code = textwrap.dedent(r""" + import sys + import unittest + from test import support + + class MyObject: + pass + + def func_bug(): + raise Exception("bug in thread") + + class Tests(unittest.TestCase): + def test_print_warning(self): + print("msg1: stdout") + support.print_warning("msg2: print_warning") + # Fail with ENV CHANGED to see print_warning() log + support.environment_altered = True + """) + testname = self.create_test(code=code) + + # Expect an output like: + # + # test_threading_excepthook (test.test_x.Tests) ... msg1: stdout + # Warning -- msg2: print_warning + # ok + regex = (r"test_print_warning.*msg1: stdout\n" + r"Warning -- msg2: print_warning\n" + r"ok\n") + for option in ("-v", "-W"): + with self.subTest(option=option): + cmd = ["--fail-env-changed", option, testname] + output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED) + self.check_executed_tests(output, [testname], + env_changed=[testname], + fail_env_changed=True, + stats=1) + self.assertRegex(output, regex) + + def test_unicode_guard_env(self): + guard = os.environ.get(setup.UNICODE_GUARD_ENV) + self.assertIsNotNone(guard, f"{setup.UNICODE_GUARD_ENV} not set") + if guard.isascii(): + # Skip to signify that the env var value was changed by the user; + # possibly to something ASCII to work around Unicode issues. + self.skipTest("Modified guard") + def test_cleanup(self): dirname = os.path.join(self.tmptestdir, "test_python_123") os.mkdir(dirname) @@ -1277,10 +2003,411 @@ def test_cleanup(self): for name in names: self.assertFalse(os.path.exists(name), name) + @unittest.skip("TODO: RUSTPYTHON; flaky") + @unittest.skipIf(support.is_wasi, + 'checking temp files is not implemented on WASI') + def test_leak_tmp_file(self): + code = textwrap.dedent(r""" + import os.path + import tempfile + import unittest + + class FileTests(unittest.TestCase): + def test_leak_tmp_file(self): + filename = os.path.join(tempfile.gettempdir(), 'mytmpfile') + with open(filename, "wb") as fp: + fp.write(b'content') + """) + testnames = [self.create_test(code=code) for _ in range(3)] + + output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, + exitcode=EXITCODE_ENV_CHANGED) + self.check_executed_tests(output, testnames, + env_changed=testnames, + fail_env_changed=True, + parallel=True, + stats=len(testnames)) + for testname in testnames: + self.assertIn(f"Warning -- {testname} leaked temporary " + f"files (1): mytmpfile", + output) + + def test_worker_decode_error(self): + # gh-109425: Use "backslashreplace" error handler to decode stdout. + if sys.platform == 'win32': + encoding = locale.getencoding() + else: + encoding = sys.stdout.encoding + if encoding is None: + encoding = sys.__stdout__.encoding + if encoding is None: + self.skipTest("cannot get regrtest worker encoding") + + nonascii = bytes(ch for ch in range(128, 256)) + corrupted_output = b"nonascii:%s\n" % (nonascii,) + # gh-108989: On Windows, assertion errors are written in UTF-16: when + # decoded each letter is follow by a NUL character. + assertion_failed = 'Assertion failed: tstate_is_alive(tstate)\n' + corrupted_output += assertion_failed.encode('utf-16-le') + try: + corrupted_output.decode(encoding) + except UnicodeDecodeError: + pass + else: + self.skipTest(f"{encoding} can decode non-ASCII bytes") + + expected_line = corrupted_output.decode(encoding, 'backslashreplace') + + code = textwrap.dedent(fr""" + import sys + import unittest + + class Tests(unittest.TestCase): + def test_pass(self): + pass + + # bytes which cannot be decoded from UTF-8 + corrupted_output = {corrupted_output!a} + sys.stdout.buffer.write(corrupted_output) + sys.stdout.buffer.flush() + """) + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", "-j1", testname) + self.check_executed_tests(output, [testname], + parallel=True, + stats=1) + self.check_line(output, expected_line, regex=False) + + def test_doctest(self): + code = textwrap.dedent(r''' + import doctest + import sys + from test import support + + def my_function(): + """ + Pass: + + >>> 1 + 1 + 2 + + Failure: + + >>> 2 + 3 + 23 + >>> 1 + 1 + 11 + + Skipped test (ignored): + + >>> id(1.0) # doctest: +SKIP + 7948648 + """ + + def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests + ''') + testname = self.create_test(code=code) + + output = self.run_tests("--fail-env-changed", "-v", "-j1", testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + parallel=True, + stats=TestStats(1, 1, 0)) + + def _check_random_seed(self, run_workers: bool): + # gh-109276: When -r/--randomize is used, random.seed() is called + # with the same random seed before running each test file. + code = textwrap.dedent(r''' + import random + import unittest + + class RandomSeedTest(unittest.TestCase): + def test_randint(self): + numbers = [random.randint(0, 1000) for _ in range(10)] + print(f"Random numbers: {numbers}") + ''') + tests = [self.create_test(name=f'test_random{i}', code=code) + for i in range(1, 3+1)] + + random_seed = 856_656_202 + cmd = ["--randomize", f"--randseed={random_seed}"] + if run_workers: + # run as many worker processes than the number of tests + cmd.append(f'-j{len(tests)}') + cmd.extend(tests) + output = self.run_tests(*cmd) + + random.seed(random_seed) + # Make the assumption that nothing consume entropy between libregrest + # setup_tests() which calls random.seed() and RandomSeedTest calling + # random.randint(). + numbers = [random.randint(0, 1000) for _ in range(10)] + expected = f"Random numbers: {numbers}" + + regex = r'^Random numbers: .*$' + matches = re.findall(regex, output, flags=re.MULTILINE) + self.assertEqual(matches, [expected] * len(tests)) + + def test_random_seed(self): + self._check_random_seed(run_workers=False) + + @unittest.skip("TODO: RUSTPYTHON; flaky") + def test_random_seed_workers(self): + self._check_random_seed(run_workers=True) + + @unittest.skip("TODO: RUSTPYTHON; flaky") + def test_python_command(self): + code = textwrap.dedent(r""" + import sys + import unittest + + class WorkerTests(unittest.TestCase): + def test_dev_mode(self): + self.assertTrue(sys.flags.dev_mode) + """) + tests = [self.create_test(code=code) for _ in range(3)] + + # Custom Python command: "python -X dev" + python_cmd = [sys.executable, '-X', 'dev'] + # test.libregrtest.cmdline uses shlex.split() to parse the Python + # command line string + python_cmd = shlex.join(python_cmd) + + output = self.run_tests("--python", python_cmd, "-j0", *tests) + self.check_executed_tests(output, tests, + stats=len(tests), parallel=True) + + def test_unload_tests(self): + # Test that unloading test modules does not break tests + # that import from other tests. + # The test execution order matters for this test. + # Both test_regrtest_a and test_regrtest_c which are executed before + # and after test_regrtest_b import a submodule from the test_regrtest_b + # package and use it in testing. test_regrtest_b itself does not import + # that submodule. + # Previously test_regrtest_c failed because test_regrtest_b.util in + # sys.modules was left after test_regrtest_a (making the import + # statement no-op), but new test_regrtest_b without the util attribute + # was imported for test_regrtest_b. + testdir = os.path.join(os.path.dirname(__file__), + 'regrtestdata', 'import_from_tests') + tests = [f'test_regrtest_{name}' for name in ('a', 'b', 'c')] + args = ['-Wd', '-E', '-bb', '-m', 'test', '--testdir=%s' % testdir, *tests] + output = self.run_python(args) + self.check_executed_tests(output, tests, stats=3) + + def check_add_python_opts(self, option): + # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python + + # Skip test if _testinternalcapi is missing + import_helper.import_module('_testinternalcapi') + + code = textwrap.dedent(r""" + import sys + import unittest + from test import support + try: + from _testcapi import config_get + except ImportError: + config_get = None + + # WASI/WASM buildbots don't use -E option + use_environment = (support.is_emscripten or support.is_wasi) + + class WorkerTests(unittest.TestCase): + @unittest.skipUnless(config_get is None, 'need config_get()') + def test_config(self): + config = config_get() + # -u option + self.assertEqual(config_get('buffered_stdio'), 0) + # -W default option + self.assertTrue(config_get('warnoptions'), ['default']) + # -bb option + self.assertTrue(config_get('bytes_warning'), 2) + # -E option + self.assertTrue(config_get('use_environment'), use_environment) + + def test_python_opts(self): + # -u option + self.assertTrue(sys.__stdout__.write_through) + self.assertTrue(sys.__stderr__.write_through) + + # -W default option + self.assertTrue(sys.warnoptions, ['default']) + + # -bb option + self.assertEqual(sys.flags.bytes_warning, 2) + + # -E option + self.assertEqual(not sys.flags.ignore_environment, + use_environment) + """) + testname = self.create_test(code=code) + + # Use directly subprocess to control the exact command line + cmd = [sys.executable, + "-m", "test", option, + f'--testdir={self.tmptestdir}', + testname] + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + self.assertEqual(proc.returncode, 0, proc) + + def test_add_python_opts(self): + for opt in ("--fast-ci", "--slow-ci"): + with self.subTest(opt=opt): + self.check_add_python_opts(opt) + + # gh-76319: Raising SIGSEGV on Android may not cause a crash. + @unittest.skipIf(support.is_android, + 'raising SIGSEGV on Android is unreliable') + def test_worker_output_on_failure(self): + # Skip test if faulthandler is missing + import_helper.import_module('faulthandler') + + code = textwrap.dedent(r""" + import faulthandler + import unittest + from test import support + + class CrashTests(unittest.TestCase): + def test_crash(self): + print("just before crash!", flush=True) + + with support.SuppressCrashReport(): + faulthandler._sigsegv(True) + """) + testname = self.create_test(code=code) + + # Sanitizers must not handle SIGSEGV (ex: for test_enable_fd()) + env = dict(os.environ) + option = 'handle_segv=0' + support.set_sanitizer_env_var(env, option) + + output = self.run_tests("-j1", testname, + exitcode=EXITCODE_BAD_TEST, + env=env) + self.check_executed_tests(output, testname, + failed=[testname], + stats=0, parallel=True) + if not support.MS_WINDOWS: + exitcode = -int(signal.SIGSEGV) + self.assertIn(f"Exit code {exitcode} (SIGSEGV)", output) + self.check_line(output, "just before crash!", full=True, regex=False) + + def test_verbose3(self): + code = textwrap.dedent(r""" + import unittest + from test import support + + class VerboseTests(unittest.TestCase): + def test_pass(self): + print("SPAM SPAM SPAM") + """) + testname = self.create_test(code=code) + + # Run sequentially + output = self.run_tests("--verbose3", testname) + self.check_executed_tests(output, testname, stats=1) + self.assertNotIn('SPAM SPAM SPAM', output) + + # -R option needs a debug build + if support.Py_DEBUG: + # Check for reference leaks, run in parallel + output = self.run_tests("-R", "3:3", "-j1", "--verbose3", testname) + self.check_executed_tests(output, testname, stats=1, parallel=True) + self.assertNotIn('SPAM SPAM SPAM', output) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: int() argument must be a string, a bytes-like object or a real number, not 'NoneType' + def test_xml(self): + code = textwrap.dedent(r""" + import unittest + + class VerboseTests(unittest.TestCase): + def test_failed(self): + print("abc \x1b def") + self.fail() + """) + testname = self.create_test(code=code) + + # Run sequentially + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + output = self.run_tests(testname, "--junit-xml", filename, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=testname, + stats=TestStats(1, 1, 0)) + + # Test generated XML + with open(filename, encoding="utf8") as fp: + content = fp.read() + + testsuite = ElementTree.fromstring(content) + self.assertEqual(int(testsuite.get('tests')), 1) + self.assertEqual(int(testsuite.get('errors')), 0) + self.assertEqual(int(testsuite.get('failures')), 1) + + testcase = testsuite[0][0] + self.assertEqual(testcase.get('status'), 'run') + self.assertEqual(testcase.get('result'), 'completed') + self.assertGreater(float(testcase.get('time')), 0) + for out in testcase.iter('system-out'): + self.assertEqual(out.text, r"abc \x1b def") + + def test_nonascii(self): + code = textwrap.dedent(r""" + import unittest + + class NonASCIITests(unittest.TestCase): + def test_docstring(self): + '''docstring:\u20ac''' + + def test_subtest(self): + with self.subTest(param='subtest:\u20ac'): + pass + + def test_skip(self): + self.skipTest('skipped:\u20ac') + """) + testname = self.create_test(code=code) + + env = dict(os.environ) + env['PYTHONIOENCODING'] = 'ascii' + + def check(output): + self.check_executed_tests(output, testname, stats=TestStats(3, 0, 1)) + self.assertIn(r'docstring:\u20ac', output) + self.assertIn(r'skipped:\u20ac', output) + + # Run sequentially + output = self.run_tests('-v', testname, env=env, isolated=False) + check(output) + + # Run in parallel + output = self.run_tests('-j1', '-v', testname, env=env, isolated=False) + check(output) + + def test_pgo_exclude(self): + # Get PGO tests + output = self.run_tests('--pgo', '--list-tests') + pgo_tests = output.strip().split() + + # Exclude test_re + output = self.run_tests('--pgo', '--list-tests', '-x', 'test_re') + tests = output.strip().split() + self.assertNotIn('test_re', tests) + self.assertEqual(len(tests), len(pgo_tests) - 1) + class TestUtils(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') @@ -1303,6 +2430,219 @@ def test_format_duration(self): self.assertEqual(utils.format_duration(3 * 3600 + 1), '3 hour 1 sec') + def test_normalize_test_name(self): + normalize = normalize_test_name + self.assertEqual(normalize('test_access (test.test_os.FileTests.test_access)'), + 'test_access') + self.assertEqual(normalize('setUpClass (test.test_os.ChownFileTests)', is_error=True), + 'ChownFileTests') + self.assertEqual(normalize('test_success (test.test_bug.ExampleTests.test_success)', is_error=True), + 'test_success') + self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True)) + self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True)) + + def test_format_resources(self): + format_resources = utils.format_resources + ALL_RESOURCES = utils.ALL_RESOURCES + self.assertEqual( + format_resources(("network",)), + 'resources (1): network') + self.assertEqual( + format_resources(("audio", "decimal", "network")), + 'resources (3): audio,decimal,network') + self.assertEqual( + format_resources(ALL_RESOURCES), + 'resources: all') + self.assertEqual( + format_resources(tuple(name for name in ALL_RESOURCES + if name != "cpu")), + 'resources: all,-cpu') + self.assertEqual( + format_resources((*ALL_RESOURCES, "tzdata")), + 'resources: all,tzdata') + + def test_match_test(self): + class Test: + def __init__(self, test_id): + self.test_id = test_id + + def id(self): + return self.test_id + + # Restore patterns once the test completes + patterns = get_match_tests() + self.addCleanup(set_match_tests, patterns) + + test_access = Test('test.test_os.FileTests.test_access') + test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + test_copy = Test('test.test_shutil.TestCopy.test_copy') + + # Test acceptance + with support.swap_attr(support, '_test_matchers', ()): + # match all + set_match_tests([]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match all using None + set_match_tests(None) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match the full test identifier + set_match_tests([(test_access.id(), True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # match the module name + set_match_tests([('test_os', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + # Test '*' pattern + set_match_tests([('test_*', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # Test case sensitivity + set_match_tests([('filetests', True)]) + self.assertFalse(match_test(test_access)) + set_match_tests([('FileTests', True)]) + self.assertTrue(match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + set_match_tests([('*test_os.*.test_*', True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + # Multiple patterns + set_match_tests([(test_access.id(), True), (test_chdir.id(), True)]) + self.assertTrue(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + set_match_tests([('test_access', True), ('DONTMATCH', True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # Test rejection + with support.swap_attr(support, '_test_matchers', ()): + # match the full test identifier + set_match_tests([(test_access.id(), False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # match the module name + set_match_tests([('test_os', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + # Test '*' pattern + set_match_tests([('test_*', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + + # Test case sensitivity + set_match_tests([('filetests', False)]) + self.assertTrue(match_test(test_access)) + set_match_tests([('FileTests', False)]) + self.assertFalse(match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + set_match_tests([('*test_os.*.test_*', False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + # Multiple patterns + set_match_tests([(test_access.id(), False), (test_chdir.id(), False)]) + self.assertFalse(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + set_match_tests([('test_access', False), ('DONTMATCH', False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + + # Test mixed filters + with support.swap_attr(support, '_test_matchers', ()): + set_match_tests([('*test_os', False), ('test_access', True)]) + self.assertTrue(match_test(test_access)) + self.assertFalse(match_test(test_chdir)) + self.assertTrue(match_test(test_copy)) + + set_match_tests([('*test_os', True), ('test_access', False)]) + self.assertFalse(match_test(test_access)) + self.assertTrue(match_test(test_chdir)) + self.assertFalse(match_test(test_copy)) + + def test_sanitize_xml(self): + sanitize_xml = utils.sanitize_xml + + # escape invalid XML characters + self.assertEqual(sanitize_xml('abc \x1b\x1f def'), + r'abc \x1b\x1f def') + self.assertEqual(sanitize_xml('nul:\x00, bell:\x07'), + r'nul:\x00, bell:\x07') + self.assertEqual(sanitize_xml('surrogate:\uDC80'), + r'surrogate:\udc80') + self.assertEqual(sanitize_xml('illegal \uFFFE and \uFFFF'), + r'illegal \ufffe and \uffff') + + # no escape for valid XML characters + self.assertEqual(sanitize_xml('a\n\tb'), + 'a\n\tb') + self.assertEqual(sanitize_xml('valid t\xe9xt \u20ac'), + 'valid t\xe9xt \u20ac') + + +from test.libregrtest.results import TestResults + + +class TestColorized(unittest.TestCase): + def test_test_result_get_state(self): + # Arrange + green = _colorize.ANSIColors.GREEN + red = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + yellow = _colorize.ANSIColors.YELLOW + + good_results = TestResults() + good_results.good = ["good1", "good2"] + bad_results = TestResults() + bad_results.bad = ["bad1", "bad2"] + no_results = TestResults() + no_results.bad = [] + interrupted_results = TestResults() + interrupted_results.interrupted = True + interrupted_worker_bug = TestResults() + interrupted_worker_bug.interrupted = True + interrupted_worker_bug.worker_bug = True + + for results, expected in ( + (good_results, f"{green}SUCCESS{reset}"), + (bad_results, f"{red}FAILURE{reset}"), + (no_results, f"{yellow}NO TESTS RAN{reset}"), + (interrupted_results, f"{yellow}INTERRUPTED{reset}"), + ( + interrupted_worker_bug, + f"{yellow}INTERRUPTED{reset}, {red}WORKER BUG{reset}", + ), + ): + with self.subTest(results=results, expected=expected): + # Act + with unittest.mock.patch( + "_colorize.can_colorize", return_value=True + ): + result = results.get_state(fail_env_changed=False) + + # Assert + self.assertEqual(result, expected) + if __name__ == '__main__': + setup.setup_process() unittest.main() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index a63c1213c6e..03bf8d8b548 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -92,8 +92,6 @@ def test_multiline_string_parsing(self): output = kill_python(p) self.assertEqual(p.returncode, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_close_stdin(self): user_input = dedent(''' import os diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 611fb9d1e4f..3396b54cc9f 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -3,14 +3,16 @@ Nick Mathewson """ +import annotationlib import sys import os import shutil import importlib import importlib.util import unittest +import textwrap -from test.support import verbose +from test.support import verbose, EqualToForwardRef from test.support.os_helper import create_empty_file from reprlib import repr as r # Don't shadow builtin repr from reprlib import Repr @@ -25,6 +27,29 @@ def nestedTuple(nesting): class ReprTests(unittest.TestCase): + def test_init_kwargs(self): + example_kwargs = { + "maxlevel": 101, + "maxtuple": 102, + "maxlist": 103, + "maxarray": 104, + "maxdict": 105, + "maxset": 106, + "maxfrozenset": 107, + "maxdeque": 108, + "maxstring": 109, + "maxlong": 110, + "maxother": 111, + "fillvalue": "x" * 112, + "indent": "x" * 113, + } + r1 = Repr() + for attr, val in example_kwargs.items(): + setattr(r1, attr, val) + r2 = Repr(**example_kwargs) + for attr in example_kwargs: + self.assertEqual(getattr(r1, attr), getattr(r2, attr), msg=attr) + def test_string(self): eq = self.assertEqual eq(r("abc"), "'abc'") @@ -51,6 +76,13 @@ def test_tuple(self): expected = repr(t3)[:-2] + "...)" eq(r2.repr(t3), expected) + # modified fillvalue: + r3 = Repr() + r3.fillvalue = '+++' + r3.maxtuple = 2 + expected = repr(t3)[:-2] + "+++)" + eq(r3.repr(t3), expected) + def test_container(self): from array import array from collections import deque @@ -118,15 +150,40 @@ def test_frozenset(self): eq(r(frozenset({1, 2, 3, 4, 5, 6})), "frozenset({1, 2, 3, 4, 5, 6})") eq(r(frozenset({1, 2, 3, 4, 5, 6, 7})), "frozenset({1, 2, 3, 4, 5, 6, ...})") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_numbers(self): - eq = self.assertEqual - eq(r(123), repr(123)) - eq(r(123), repr(123)) - eq(r(1.0/3), repr(1.0/3)) - - n = 10**100 - expected = repr(n)[:18] + "..." + repr(n)[-19:] - eq(r(n), expected) + for x in [123, 1.0 / 3]: + self.assertEqual(r(x), repr(x)) + + max_digits = sys.get_int_max_str_digits() + for k in [100, max_digits - 1]: + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + expected = repr(n)[:18] + "..." + repr(n)[-19:] + self.assertEqual(r(n), expected) + + def re_msg(n, d): + return (rf'<{n.__class__.__name__} instance with roughly {d} ' + rf'digits \(limit at {max_digits}\) at 0x[a-f0-9]+>') + + k = max_digits + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + + for k in [max_digits + 1, 2 * max_digits]: + self.assertGreater(k, 100) + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + with self.subTest(f'10 ** {k} - 1', k=k): + n = 10 ** k - 1 + # Here, since math.log10(n) == math.log10(n-1), + # the number of digits of n - 1 is overestimated. + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) def test_instance(self): eq = self.assertEqual @@ -141,26 +198,22 @@ def test_instance(self): eq(r(i3), ("<ClassWithFailingRepr instance at %#x>"%id(i3))) s = r(ClassWithFailingRepr) - self.assertTrue(s.startswith("<class ")) - self.assertTrue(s.endswith(">")) + self.assertStartsWith(s, "<class ") + self.assertEndsWith(s, ">") self.assertIn(s.find("..."), [12, 13]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lambda(self): r = repr(lambda x: x) - self.assertTrue(r.startswith("<function ReprTests.test_lambda.<locals>.<lambda"), r) + self.assertStartsWith(r, "<function ReprTests.test_lambda.<locals>.<lambda") # XXX anonymous functions? see func_repr - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_builtin_function(self): eq = self.assertEqual # Functions eq(repr(hash), '<built-in function hash>') # Methods - self.assertTrue(repr(''.split).startswith( - '<built-in method split of str object at 0x')) + self.assertStartsWith(repr(''.split), + '<built-in method split of str object at 0x') def test_range(self): eq = self.assertEqual @@ -185,8 +238,7 @@ def test_nesting(self): eq(r([[[[[[{}]]]]]]), "[[[[[[{}]]]]]]") eq(r([[[[[[[{}]]]]]]]), "[[[[[[[...]]]]]]]") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cell(self): def get_cell(): x = 42 @@ -223,6 +275,382 @@ def test_unsortable(self): r(y) r(z) + def test_valid_indent(self): + test_cases = [ + { + 'object': (), + 'tests': ( + (dict(indent=None), '()'), + (dict(indent=False), '()'), + (dict(indent=True), '()'), + (dict(indent=0), '()'), + (dict(indent=1), '()'), + (dict(indent=4), '()'), + (dict(indent=4, maxlevel=2), '()'), + (dict(indent=''), '()'), + (dict(indent='-->'), '()'), + (dict(indent='....'), '()'), + ), + }, + { + 'object': '', + 'tests': ( + (dict(indent=None), "''"), + (dict(indent=False), "''"), + (dict(indent=True), "''"), + (dict(indent=0), "''"), + (dict(indent=1), "''"), + (dict(indent=4), "''"), + (dict(indent=4, maxlevel=2), "''"), + (dict(indent=''), "''"), + (dict(indent='-->'), "''"), + (dict(indent='....'), "''"), + ), + }, + { + 'object': [1, 'spam', {'eggs': True, 'ham': []}], + 'tests': ( + (dict(indent=None), '''\ + [1, 'spam', {'eggs': True, 'ham': []}]'''), + (dict(indent=False), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=True), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=0), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=1), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=4), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=4, maxlevel=2), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=''), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent='-->'), '''\ + [ + -->1, + -->'spam', + -->{ + -->-->'eggs': True, + -->-->'ham': [], + -->}, + ]'''), + (dict(indent='....'), '''\ + [ + ....1, + ....'spam', + ....{ + ........'eggs': True, + ........'ham': [], + ....}, + ]'''), + ), + }, + { + 'object': { + 1: 'two', + b'three': [ + (4.5, 6.25), + [set((8, 9)), frozenset((10, 11))], + ], + }, + 'tests': ( + (dict(indent=None), '''\ + {1: 'two', b'three': [(4.5, 6.25), [{8, 9}, frozenset({10, 11})]]}'''), + (dict(indent=False), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.25, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=True), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.25, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=0), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.25, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=1), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.25, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=4), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.25, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=4, maxlevel=2), '''\ + { + 1: 'two', + b'three': [ + (...), + [...], + ], + }'''), + (dict(indent=''), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.25, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent='-->'), '''\ + { + -->1: 'two', + -->b'three': [ + -->-->( + -->-->-->4.5, + -->-->-->6.25, + -->-->), + -->-->[ + -->-->-->{ + -->-->-->-->8, + -->-->-->-->9, + -->-->-->}, + -->-->-->frozenset({ + -->-->-->-->10, + -->-->-->-->11, + -->-->-->}), + -->-->], + -->], + }'''), + (dict(indent='....'), '''\ + { + ....1: 'two', + ....b'three': [ + ........( + ............4.5, + ............6.25, + ........), + ........[ + ............{ + ................8, + ................9, + ............}, + ............frozenset({ + ................10, + ................11, + ............}), + ........], + ....], + }'''), + ), + }, + ] + for test_case in test_cases: + with self.subTest(test_object=test_case['object']): + for repr_settings, expected_repr in test_case['tests']: + with self.subTest(repr_settings=repr_settings): + r = Repr() + for attribute, value in repr_settings.items(): + setattr(r, attribute, value) + resulting_repr = r.repr(test_case['object']) + expected_repr = textwrap.dedent(expected_repr) + self.assertEqual(resulting_repr, expected_repr) + + def test_invalid_indent(self): + test_object = [1, 'spam', {'eggs': True, 'ham': []}] + test_cases = [ + (-1, (ValueError, '[Nn]egative|[Pp]ositive')), + (-4, (ValueError, '[Nn]egative|[Pp]ositive')), + ((), (TypeError, None)), + ([], (TypeError, None)), + ((4,), (TypeError, None)), + ([4,], (TypeError, None)), + (object(), (TypeError, None)), + ] + for indent, (expected_error, expected_msg) in test_cases: + with self.subTest(indent=indent): + r = Repr() + r.indent = indent + expected_msg = expected_msg or f'{type(indent)}' + with self.assertRaisesRegex(expected_error, expected_msg): + r.repr(test_object) + + def test_shadowed_stdlib_array(self): + # Issue #113570: repr() should not be fooled by an array + class array: + def __repr__(self): + return "not array.array" + + self.assertEqual(r(array()), "not array.array") + + def test_shadowed_builtin(self): + # Issue #113570: repr() should not be fooled + # by a shadowed builtin function + class list: + def __repr__(self): + return "not builtins.list" + + self.assertEqual(r(list()), "not builtins.list") + + def test_custom_repr(self): + class MyRepr(Repr): + + def repr_TextIOWrapper(self, obj, level): + if obj.name in {'<stdin>', '<stdout>', '<stderr>'}: + return obj.name + return repr(obj) + + aRepr = MyRepr() + self.assertEqual(aRepr.repr(sys.stdin), "<stdin>") + + def test_custom_repr_class_with_spaces(self): + class TypeWithSpaces: + pass + + t = TypeWithSpaces() + type(t).__name__ = "type with spaces" + self.assertEqual(type(t).__name__, "type with spaces") + + class MyRepr(Repr): + def repr_type_with_spaces(self, obj, level): + return "Type With Spaces" + + + aRepr = MyRepr() + self.assertEqual(aRepr.repr(t), "Type With Spaces") + def write_file(path, text): with open(path, 'w', encoding='ASCII') as fp: fp.write(text) @@ -328,8 +756,8 @@ class baz: importlib.invalidate_caches() from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import baz ibaz = baz.baz() - self.assertTrue(repr(ibaz).startswith( - "<%s.baz object at 0x" % baz.__name__)) + self.assertStartsWith(repr(ibaz), + "<%s.baz object at 0x" % baz.__name__) def test_method(self): self._check_path_limitations('qux') @@ -342,13 +770,13 @@ def amethod(self): pass from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import qux # Unbound methods first r = repr(qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod) - self.assertTrue(r.startswith('<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod'), r) + self.assertStartsWith(r, '<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod') # Bound method next iqux = qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa() r = repr(iqux.amethod) - self.assertTrue(r.startswith( + self.assertStartsWith(r, '<bound method aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod of <%s.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa object at 0x' \ - % (qux.__name__,) ), r) + % (qux.__name__,) ) @unittest.skip('needs a built-in function with a really long name') def test_builtin_function(self): @@ -408,5 +836,39 @@ def test_assigned_attributes(self): for name in assigned: self.assertIs(getattr(wrapper, name), getattr(wrapped, name)) + def test__wrapped__(self): + class X: + def __repr__(self): + return 'X()' + f = __repr__ # save reference to check it later + __repr__ = recursive_repr()(__repr__) + + self.assertIs(X.f, X.__repr__.__wrapped__) + + def test__type_params__(self): + class My: + @recursive_repr() + def __repr__[T: str](self, default: T = '') -> str: + return default + + type_params = My().__repr__.__type_params__ + self.assertEqual(len(type_params), 1) + self.assertEqual(type_params[0].__name__, 'T') + self.assertEqual(type_params[0].__bound__, str) + + def test_annotations(self): + class My: + @recursive_repr() + def __repr__(self, default: undefined = ...): + return default + + annotations = annotationlib.get_annotations( + My.__repr__, format=annotationlib.Format.FORWARDREF + ) + self.assertEqual( + annotations, + {'default': EqualToForwardRef("undefined", owner=My.__repr__)} + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 103d0199d09..d7ff492d0f7 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -2,7 +2,8 @@ import sys import unittest from test import support -from test.support import os_helper, import_helper +from test.support import import_helper +from test.support import os_helper import time resource = import_helper.import_module('resource') @@ -99,6 +100,7 @@ def test_fsize_toobig(self): except (OverflowError, ValueError): pass + @unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage") def test_getrusage(self): self.assertRaises(TypeError, resource.getrusage) self.assertRaises(TypeError, resource.getrusage, 42, 42) @@ -140,7 +142,7 @@ def test_pagesize(self): self.assertIsInstance(pagesize, int) self.assertGreaterEqual(pagesize, 0) - @unittest.skipUnless(sys.platform == 'linux', 'test requires Linux') + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux only') def test_linux_constants(self): for attr in ['MSGQUEUE', 'NICE', 'RTPRIO', 'RTTIME', 'SIGPENDING']: with contextlib.suppress(AttributeError): @@ -177,8 +179,5 @@ def __getitem__(self, key): limits) -def test_main(verbose=None): - support.run_unittest(ResourceTest) - if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 58729a9fea6..b967c7623c5 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -28,9 +28,6 @@ def __gt__(self, other): def __ge__(self, other): return self.x >= other - def __cmp__(self, other): - raise support.TestFailed("Number.__cmp__() should not be called") - def __repr__(self): return "Number(%r)" % (self.x, ) @@ -53,9 +50,6 @@ def __setitem__(self, i, v): def __bool__(self): raise TypeError("Vectors cannot be used in Boolean contexts") - def __cmp__(self, other): - raise support.TestFailed("Vector.__cmp__() should not be called") - def __repr__(self): return "Vector(%r)" % (self.data, ) @@ -221,6 +215,7 @@ def do(bad): self.assertRaises(Exc, func, Bad()) @support.no_tracing + @support.infinite_recursion(25) def test_recursion(self): # Check that comparison for recursive objects fails gracefully from collections import UserList diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 0dc1080ca32..ffadfee2763 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -2,6 +2,7 @@ from unittest.mock import patch import builtins import rlcompleter +from test.support import MISSING_C_DOCSTRINGS class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ @@ -40,21 +41,41 @@ def test_global_matches(self): # test with a customized namespace self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe(']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) self.assertEqual(self.completer.global_matches('eg'), ['egg(']) # XXX: see issue5256 self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe(']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_attr_matches(self): # test with builtins namespace self.assertEqual(self.stdcompleter.attr_matches('str.s'), ['str.{}('.format(x) for x in dir(str) if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) - expected = sorted({'None.%s%s' % (x, '(' if x != '__doc__' else '') - for x in dir(None)}) + + def create_expected_for_none(): + if not MISSING_C_DOCSTRINGS: + parentheses = ('__init_subclass__', '__class__') + else: + # When `--without-doc-strings` is used, `__class__` + # won't have a known signature. + parentheses = ('__init_subclass__',) + + items = set() + for x in dir(None): + if x in parentheses: + items.add(f'None.{x}()') + elif x == '__doc__': + items.add(f'None.{x}') + else: + items.add(f'None.{x}(') + return sorted(items) + + expected = create_expected_for_none() self.assertEqual(self.stdcompleter.attr_matches('None.'), expected) self.assertEqual(self.stdcompleter.attr_matches('None._'), expected) self.assertEqual(self.stdcompleter.attr_matches('None.__'), expected) @@ -64,7 +85,7 @@ def test_attr_matches(self): ['CompleteMe.spam']) self.assertEqual(self.completer.attr_matches('Completeme.egg'), []) self.assertEqual(self.completer.attr_matches('CompleteMe.'), - ['CompleteMe.mro(', 'CompleteMe.spam']) + ['CompleteMe.mro()', 'CompleteMe.spam']) self.assertEqual(self.completer.attr_matches('CompleteMe._'), ['CompleteMe._ham']) matches = self.completer.attr_matches('CompleteMe.__') @@ -81,17 +102,41 @@ def test_attr_matches(self): if x.startswith('s')]) def test_excessive_getattr(self): - # Ensure getattr() is invoked no more than once per attribute + """Ensure getattr() is invoked no more than once per attribute""" + + # note the special case for @property methods below; that is why + # we use __dir__ and __getattr__ in class Foo to create a "magic" + # class attribute 'bar'. This forces `getattr` to call __getattr__ + # (which is doesn't necessarily do). class Foo: calls = 0 + bar = '' + def __getattribute__(self, name): + if name == 'bar': + self.calls += 1 + return None + return super().__getattribute__(name) + + f = Foo() + completer = rlcompleter.Completer(dict(f=f)) + self.assertEqual(completer.complete('f.b', 0), 'f.bar') + self.assertEqual(f.calls, 1) + + def test_property_method_not_called(self): + class Foo: + _bar = 0 + property_called = False + @property def bar(self): - self.calls += 1 - return None + self.property_called = True + return self._bar + f = Foo() completer = rlcompleter.Completer(dict(f=f)) self.assertEqual(completer.complete('f.b', 0), 'f.bar') - self.assertEqual(f.calls, 1) + self.assertFalse(f.property_called) + def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when @@ -114,6 +159,9 @@ def test_complete(self): self.assertEqual(completer.complete('el', 0), 'elif ') self.assertEqual(completer.complete('el', 1), 'else') self.assertEqual(completer.complete('tr', 0), 'try:') + self.assertEqual(completer.complete('_', 0), '_') + self.assertEqual(completer.complete('match', 0), 'match ') + self.assertEqual(completer.complete('case', 0), 'case ') def test_duplicate_globals(self): namespace = { @@ -134,7 +182,7 @@ def test_duplicate_globals(self): # No opening bracket "(" because we overrode the built-in class self.assertEqual(completer.complete('memoryview', 0), 'memoryview') self.assertIsNone(completer.complete('memoryview', 1)) - self.assertEqual(completer.complete('Ellipsis', 0), 'Ellipsis(') + self.assertEqual(completer.complete('Ellipsis', 0), 'Ellipsis()') self.assertIsNone(completer.complete('Ellipsis', 1)) if __name__ == '__main__': diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index b0bed431d4b..e33723cc70c 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -16,6 +16,14 @@ class BaseRobotTest: bad = [] site_maps = None + def __init_subclass__(cls): + super().__init_subclass__() + # Remove tests that do nothing. + if not cls.good: + cls.test_good_urls = None + if not cls.bad: + cls.test_bad_urls = None + def setUp(self): lines = io.StringIO(self.robots_txt).readlines() self.parser = urllib.robotparser.RobotFileParser() @@ -231,9 +239,16 @@ class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * Disallow: /some/path?name=value +Disallow: /another/path? +Disallow: /yet/one/path?name=value&more """ - good = ['/some/path'] - bad = ['/some/path?name=value'] + good = ['/some/path', '/some/path?', + '/some/path%3Fname=value', '/some/path?name%3Dvalue', + '/another/path', '/another/path%3F', + '/yet/one/path?name=value%26more'] + bad = ['/some/path?name=value' + '/another/path?', '/another/path?name=value', + '/yet/one/path?name=value&more'] class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): @@ -249,15 +264,79 @@ class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): bad = ['/some/path'] -class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase): - # normalize the URL first (#17403) +class PercentEncodingTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * -Allow: /some/path? -Disallow: /another/path? - """ - good = ['/some/path?'] - bad = ['/another/path?'] +Disallow: /a1/Z-._~ # unreserved characters +Disallow: /a2/%5A%2D%2E%5F%7E # percent-encoded unreserved characters +Disallow: /u1/%F0%9F%90%8D # percent-encoded ASCII Unicode character +Disallow: /u2/%f0%9f%90%8d +Disallow: /u3/\U0001f40d # raw non-ASCII Unicode character +Disallow: /v1/%F0 # percent-encoded non-ASCII octet +Disallow: /v2/%f0 +Disallow: /v3/\udcf0 # raw non-ASCII octet +Disallow: /p1%xy # raw percent +Disallow: /p2% +Disallow: /p3%25xy # percent-encoded percent +Disallow: /p4%2525xy # double percent-encoded percent +Disallow: /john%20smith # space +Disallow: /john doe +Disallow: /trailingspace%20 +Disallow: /question%3Fq=v # not query +Disallow: /hash%23f # not fragment +Disallow: /dollar%24 +Disallow: /asterisk%2A +Disallow: /sub/dir +Disallow: /slash%2F +Disallow: /query/question?q=%3F +Disallow: /query/raw/question?q=? +Disallow: /query/eq?q%3Dv +Disallow: /query/amp?q=v%26a +""" + good = [ + '/u1/%F0', '/u1/%f0', + '/u2/%F0', '/u2/%f0', + '/u3/%F0', '/u3/%f0', + '/p1%2525xy', '/p2%f0', '/p3%2525xy', '/p4%xy', '/p4%25xy', + '/question?q=v', + '/dollar', '/asterisk', + '/query/eq?q=v', + '/query/amp?q=v&a', + ] + bad = [ + '/a1/Z-._~', '/a1/%5A%2D%2E%5F%7E', + '/a2/Z-._~', '/a2/%5A%2D%2E%5F%7E', + '/u1/%F0%9F%90%8D', '/u1/%f0%9f%90%8d', '/u1/\U0001f40d', + '/u2/%F0%9F%90%8D', '/u2/%f0%9f%90%8d', '/u2/\U0001f40d', + '/u3/%F0%9F%90%8D', '/u3/%f0%9f%90%8d', '/u3/\U0001f40d', + '/v1/%F0', '/v1/%f0', '/v1/\udcf0', '/v1/\U0001f40d', + '/v2/%F0', '/v2/%f0', '/v2/\udcf0', '/v2/\U0001f40d', + '/v3/%F0', '/v3/%f0', '/v3/\udcf0', '/v3/\U0001f40d', + '/p1%xy', '/p1%25xy', + '/p2%', '/p2%25', '/p2%2525', '/p2%xy', + '/p3%xy', '/p3%25xy', + '/p4%2525xy', + '/john%20smith', '/john smith', + '/john%20doe', '/john doe', + '/trailingspace%20', '/trailingspace ', + '/question%3Fq=v', + '/hash#f', '/hash%23f', + '/dollar$', '/dollar%24', + '/asterisk*', '/asterisk%2A', + '/sub/dir', '/sub%2Fdir', + '/slash%2F', '/slash/', + '/query/question?q=?', '/query/question?q=%3F', + '/query/raw/question?q=?', '/query/raw/question?q=%3F', + '/query/eq?q%3Dv', + '/query/amp?q=v%26a', + ] + # other reserved characters + for c in ":/#[]@!$&'()*+,;=": + robots_txt += f'Disallow: /raw{c}\nDisallow: /pc%{ord(c):02X}\n' + bad.append(f'/raw{c}') + bad.append(f'/raw%{ord(c):02X}') + bad.append(f'/pc{c}') + bad.append(f'/pc%{ord(c):02X}') class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): @@ -299,22 +378,17 @@ def test_string_formatting(self): self.assertEqual(str(self.parser), self.expected_output) -class RobotHandler(BaseHTTPRequestHandler): - - def do_GET(self): - self.send_error(403, "Forbidden access") - - def log_message(self, format, *args): - pass - - -class PasswordProtectedSiteTestCase(unittest.TestCase): +@unittest.skipUnless( + support.has_socket_support, + "Socket server requires working socket." +) +class BaseLocalNetworkTestCase: def setUp(self): # clear _opener global variable self.addCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), RobotHandler) + self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) self.t = threading.Thread( name='HTTPServer serving', @@ -331,6 +405,57 @@ def tearDown(self): self.t.join() self.server.server_close() + +SAMPLE_ROBOTS_TXT = b'''\ +User-agent: test_robotparser +Disallow: /utf8/\xf0\x9f\x90\x8d +Disallow: /non-utf8/\xf0 +Disallow: //[spam]/path +''' + + +class LocalNetworkTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(SAMPLE_ROBOTS_TXT) + + def log_message(self, format, *args): + pass + + @threading_helper.reap_threads + def testRead(self): + # Test that reading a weird robots.txt doesn't fail. + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + '/robots.txt' + parser = urllib.robotparser.RobotFileParser() + parser.set_url(robots_url) + parser.read() + # And it can even interpret the weird paths in some reasonable way. + agent = 'test_robotparser' + self.assertTrue(parser.can_fetch(agent, robots_url)) + self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) + self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) + self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) + self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) + self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) + + +class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_error(403, "Forbidden access") + + def log_message(self, format, *args): + pass + @threading_helper.reap_threads def testPasswordProtectedSite(self): addr = self.server.server_address @@ -342,6 +467,7 @@ def testPasswordProtectedSite(self): self.assertFalse(parser.can_fetch("*", robots_url)) +@support.requires_working_socket() class NetworkTestCase(unittest.TestCase): base_url = 'http://www.pythontest.net/' diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index b2fee6207be..cf7fd581ec3 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,9 +12,16 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose +from test.support import ( + force_not_colorized_test_class, + infinite_recursion, + no_tracing, + requires_resource, + requires_subprocess, + verbose, +) from test.support.import_helper import forget, make_legacy_pyc, unload -from test.support.os_helper import create_empty_file, temp_dir +from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -656,13 +663,14 @@ def test_basic_script(self): self._check_script(script_name, "<run_path>", script_name, script_name, expect_spec=False) - def test_basic_script_with_path_object(self): + def test_basic_script_with_pathlike_object(self): with temp_dir() as script_dir: mod_name = 'script' - script_name = pathlib.Path(self._make_test_script(script_dir, - mod_name)) - self._check_script(script_name, "<run_path>", script_name, - script_name, expect_spec=False) + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(FakePath(script_name), "<run_path>", + script_name, + script_name, + expect_spec=False) def test_basic_script_no_suffix(self): with temp_dir() as script_dir: @@ -672,7 +680,6 @@ def test_basic_script_no_suffix(self): self._check_script(script_name, "<run_path>", script_name, script_name, expect_spec=False) - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; weird panic in lz4-flex") def test_script_compiled(self): with temp_dir() as script_dir: mod_name = 'script' @@ -734,6 +741,7 @@ def test_zipfile_error(self): self._check_import_error(zip_name, msg) @no_tracing + @requires_resource('cpu') def test_main_recursion_error(self): with temp_dir() as script_dir, temp_dir() as dummy_dir: mod_name = '__main__' @@ -741,10 +749,9 @@ def test_main_recursion_error(self): "runpy.run_path(%r)\n") % dummy_dir script_name = self._make_test_script(script_dir, mod_name, source) zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) - self.assertRaises(RecursionError, run_path, zip_name) + with infinite_recursion(25): + self.assertRaises(RecursionError, run_path, zip_name) - # TODO: RUSTPYTHON, detect encoding comments in files - @unittest.expectedFailure def test_encoding(self): with temp_dir() as script_dir: filename = os.path.join(script_dir, 'script.py') @@ -757,6 +764,7 @@ def test_encoding(self): self.assertEqual(result['s'], "non-ASCII: h\xe9") +@force_not_colorized_test_class class TestExit(unittest.TestCase): STATUS_CONTROL_C_EXIT = 0xC000013A EXPECTED_CODE = ( @@ -783,16 +791,17 @@ def run(self, *args, **kwargs): ) super().run(*args, **kwargs) - def assertSigInt(self, *args, **kwargs): - proc = subprocess.run(*args, **kwargs, text=True, stderr=subprocess.PIPE) - self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n")) + @requires_subprocess() + def assertSigInt(self, cmd, *args, **kwargs): + # Use -E to ignore PYTHONSAFEPATH + cmd = [sys.executable, '-E', *cmd] + proc = subprocess.run(cmd, *args, **kwargs, text=True, stderr=subprocess.PIPE) + self.assertEndsWith(proc.stderr, "\nKeyboardInterrupt\n") self.assertEqual(proc.returncode, self.EXPECTED_CODE) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_file(self): - self.assertSigInt([sys.executable, self.ham]) + self.assertSigInt([self.ham]) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_file_runpy_run_module(self): tmp = self.ham.parent run_module = tmp / "run_module.py" @@ -804,9 +813,8 @@ def test_pymain_run_file_runpy_run_module(self): """ ) ) - self.assertSigInt([sys.executable, run_module], cwd=tmp) + self.assertSigInt([run_module], cwd=tmp) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_file_runpy_run_module_as_main(self): tmp = self.ham.parent run_module_as_main = tmp / "run_module_as_main.py" @@ -818,28 +826,24 @@ def test_pymain_run_file_runpy_run_module_as_main(self): """ ) ) - self.assertSigInt([sys.executable, run_module_as_main], cwd=tmp) + self.assertSigInt([run_module_as_main], cwd=tmp) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_command_run_module(self): self.assertSigInt( - [sys.executable, "-c", "import runpy; runpy.run_module('ham')"], + ["-c", "import runpy; runpy.run_module('ham')"], cwd=self.ham.parent, ) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_command(self): - self.assertSigInt([sys.executable, "-c", "import ham"], cwd=self.ham.parent) + self.assertSigInt(["-c", "import ham"], cwd=self.ham.parent) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pymain_run_stdin(self): - self.assertSigInt([sys.executable], input="import ham", cwd=self.ham.parent) + self.assertSigInt([], input="import ham", cwd=self.ham.parent) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_module(self): ham = self.ham - self.assertSigInt([sys.executable, "-m", ham.stem], cwd=ham.parent) + self.assertSigInt(["-m", ham.stem], cwd=ham.parent) if __name__ == "__main__": diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py new file mode 100644 index 00000000000..faaf4dd95b6 --- /dev/null +++ b/Lib/test/test_sax.py @@ -0,0 +1,1616 @@ +# regression test for SAX 2.0 + +from xml.sax import make_parser, ContentHandler, \ + SAXException, SAXReaderNotAvailable, SAXParseException +import unittest +from unittest import mock +try: + make_parser() +except SAXReaderNotAvailable: + # don't try to test this module if we cannot create a parser + raise unittest.SkipTest("no XML parsers available") +from xml.sax.saxutils import XMLGenerator, escape, unescape, quoteattr, \ + XMLFilterBase, prepare_input_source +from xml.sax.expatreader import create_parser +from xml.sax.handler import (feature_namespaces, feature_external_ges, + LexicalHandler) +from xml.sax.xmlreader import InputSource, AttributesImpl, AttributesNSImpl +from xml import sax +from io import BytesIO, StringIO +import codecs +import os.path +import pyexpat +import shutil +import sys +from urllib.error import URLError +import urllib.request +from test.support import os_helper +from test.support import findfile, check__all__ +from test.support.os_helper import FakePath, TESTFN + + +TEST_XMLFILE = findfile("test.xml", subdir="xmltestdata") +TEST_XMLFILE_OUT = findfile("test.xml.out", subdir="xmltestdata") +try: + TEST_XMLFILE.encode("utf-8") + TEST_XMLFILE_OUT.encode("utf-8") +except UnicodeEncodeError: + raise unittest.SkipTest("filename is not encodable to utf8") + +supports_nonascii_filenames = True +if not os.path.supports_unicode_filenames: + try: + os_helper.TESTFN_UNICODE.encode(sys.getfilesystemencoding()) + except (UnicodeError, TypeError): + # Either the file system encoding is None, or the file name + # cannot be encoded in the file system encoding. + supports_nonascii_filenames = False +requires_nonascii_filenames = unittest.skipUnless( + supports_nonascii_filenames, + 'Requires non-ascii filenames support') + +ns_uri = "http://www.python.org/xml-ns/saxtest/" + +class XmlTestBase(unittest.TestCase): + def verify_empty_attrs(self, attrs): + self.assertRaises(KeyError, attrs.getValue, "attr") + self.assertRaises(KeyError, attrs.getValueByQName, "attr") + self.assertRaises(KeyError, attrs.getNameByQName, "attr") + self.assertRaises(KeyError, attrs.getQNameByName, "attr") + self.assertRaises(KeyError, attrs.__getitem__, "attr") + self.assertEqual(attrs.getLength(), 0) + self.assertEqual(attrs.getNames(), []) + self.assertEqual(attrs.getQNames(), []) + self.assertEqual(len(attrs), 0) + self.assertNotIn("attr", attrs) + self.assertEqual(list(attrs.keys()), []) + self.assertEqual(attrs.get("attrs"), None) + self.assertEqual(attrs.get("attrs", 25), 25) + self.assertEqual(list(attrs.items()), []) + self.assertEqual(list(attrs.values()), []) + + def verify_empty_nsattrs(self, attrs): + self.assertRaises(KeyError, attrs.getValue, (ns_uri, "attr")) + self.assertRaises(KeyError, attrs.getValueByQName, "ns:attr") + self.assertRaises(KeyError, attrs.getNameByQName, "ns:attr") + self.assertRaises(KeyError, attrs.getQNameByName, (ns_uri, "attr")) + self.assertRaises(KeyError, attrs.__getitem__, (ns_uri, "attr")) + self.assertEqual(attrs.getLength(), 0) + self.assertEqual(attrs.getNames(), []) + self.assertEqual(attrs.getQNames(), []) + self.assertEqual(len(attrs), 0) + self.assertNotIn((ns_uri, "attr"), attrs) + self.assertEqual(list(attrs.keys()), []) + self.assertEqual(attrs.get((ns_uri, "attr")), None) + self.assertEqual(attrs.get((ns_uri, "attr"), 25), 25) + self.assertEqual(list(attrs.items()), []) + self.assertEqual(list(attrs.values()), []) + + def verify_attrs_wattr(self, attrs): + self.assertEqual(attrs.getLength(), 1) + self.assertEqual(attrs.getNames(), ["attr"]) + self.assertEqual(attrs.getQNames(), ["attr"]) + self.assertEqual(len(attrs), 1) + self.assertIn("attr", attrs) + self.assertEqual(list(attrs.keys()), ["attr"]) + self.assertEqual(attrs.get("attr"), "val") + self.assertEqual(attrs.get("attr", 25), "val") + self.assertEqual(list(attrs.items()), [("attr", "val")]) + self.assertEqual(list(attrs.values()), ["val"]) + self.assertEqual(attrs.getValue("attr"), "val") + self.assertEqual(attrs.getValueByQName("attr"), "val") + self.assertEqual(attrs.getNameByQName("attr"), "attr") + self.assertEqual(attrs["attr"], "val") + self.assertEqual(attrs.getQNameByName("attr"), "attr") + + +def xml_str(doc, encoding=None): + if encoding is None: + return doc + return '<?xml version="1.0" encoding="%s"?>\n%s' % (encoding, doc) + +def xml_bytes(doc, encoding, decl_encoding=...): + if decl_encoding is ...: + decl_encoding = encoding + return xml_str(doc, decl_encoding).encode(encoding, 'xmlcharrefreplace') + +def make_xml_file(doc, encoding, decl_encoding=...): + if decl_encoding is ...: + decl_encoding = encoding + with open(TESTFN, 'w', encoding=encoding, errors='xmlcharrefreplace') as f: + f.write(xml_str(doc, decl_encoding)) + + +class ParseTest(unittest.TestCase): + data = '<money value="$\xa3\u20ac\U0001017b">$\xa3\u20ac\U0001017b</money>' + + def tearDown(self): + os_helper.unlink(TESTFN) + + def check_parse(self, f): + from xml.sax import parse + result = StringIO() + parse(f, XMLGenerator(result, 'utf-8')) + self.assertEqual(result.getvalue(), xml_str(self.data, 'utf-8')) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_parse_text(self): + encodings = ('us-ascii', 'iso-8859-1', 'utf-8', + 'utf-16', 'utf-16le', 'utf-16be') + for encoding in encodings: + self.check_parse(StringIO(xml_str(self.data, encoding))) + make_xml_file(self.data, encoding) + with open(TESTFN, 'r', encoding=encoding) as f: + self.check_parse(f) + self.check_parse(StringIO(self.data)) + make_xml_file(self.data, encoding, None) + with open(TESTFN, 'r', encoding=encoding) as f: + self.check_parse(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_parse_bytes(self): + # UTF-8 is default encoding, US-ASCII is compatible with UTF-8, + # UTF-16 is autodetected + encodings = ('us-ascii', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be') + for encoding in encodings: + self.check_parse(BytesIO(xml_bytes(self.data, encoding))) + make_xml_file(self.data, encoding) + self.check_parse(TESTFN) + with open(TESTFN, 'rb') as f: + self.check_parse(f) + self.check_parse(BytesIO(xml_bytes(self.data, encoding, None))) + make_xml_file(self.data, encoding, None) + self.check_parse(TESTFN) + with open(TESTFN, 'rb') as f: + self.check_parse(f) + # accept UTF-8 with BOM + self.check_parse(BytesIO(xml_bytes(self.data, 'utf-8-sig', 'utf-8'))) + make_xml_file(self.data, 'utf-8-sig', 'utf-8') + self.check_parse(TESTFN) + with open(TESTFN, 'rb') as f: + self.check_parse(f) + self.check_parse(BytesIO(xml_bytes(self.data, 'utf-8-sig', None))) + make_xml_file(self.data, 'utf-8-sig', None) + self.check_parse(TESTFN) + with open(TESTFN, 'rb') as f: + self.check_parse(f) + # accept data with declared encoding + self.check_parse(BytesIO(xml_bytes(self.data, 'iso-8859-1'))) + make_xml_file(self.data, 'iso-8859-1') + self.check_parse(TESTFN) + with open(TESTFN, 'rb') as f: + self.check_parse(f) + # fail on non-UTF-8 incompatible data without declared encoding + with self.assertRaises(SAXException): + self.check_parse(BytesIO(xml_bytes(self.data, 'iso-8859-1', None))) + make_xml_file(self.data, 'iso-8859-1', None) + with self.assertRaises(SAXException): + self.check_parse(TESTFN) + with open(TESTFN, 'rb') as f: + with self.assertRaises(SAXException): + self.check_parse(f) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_parse_path_object(self): + make_xml_file(self.data, 'utf-8', None) + self.check_parse(FakePath(TESTFN)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_parse_InputSource(self): + # accept data without declared but with explicitly specified encoding + make_xml_file(self.data, 'iso-8859-1', None) + with open(TESTFN, 'rb') as f: + input = InputSource() + input.setByteStream(f) + input.setEncoding('iso-8859-1') + self.check_parse(input) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_parse_close_source(self): + builtin_open = open + fileobj = None + + def mock_open(*args): + nonlocal fileobj + fileobj = builtin_open(*args) + return fileobj + + with mock.patch('xml.sax.saxutils.open', side_effect=mock_open): + make_xml_file(self.data, 'iso-8859-1', None) + with self.assertRaises(SAXException): + self.check_parse(TESTFN) + self.assertTrue(fileobj.closed) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def check_parseString(self, s): + from xml.sax import parseString + result = StringIO() + parseString(s, XMLGenerator(result, 'utf-8')) + self.assertEqual(result.getvalue(), xml_str(self.data, 'utf-8')) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_parseString_text(self): + encodings = ('us-ascii', 'iso-8859-1', 'utf-8', + 'utf-16', 'utf-16le', 'utf-16be') + for encoding in encodings: + self.check_parseString(xml_str(self.data, encoding)) + self.check_parseString(self.data) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_parseString_bytes(self): + # UTF-8 is default encoding, US-ASCII is compatible with UTF-8, + # UTF-16 is autodetected + encodings = ('us-ascii', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be') + for encoding in encodings: + self.check_parseString(xml_bytes(self.data, encoding)) + self.check_parseString(xml_bytes(self.data, encoding, None)) + # accept UTF-8 with BOM + self.check_parseString(xml_bytes(self.data, 'utf-8-sig', 'utf-8')) + self.check_parseString(xml_bytes(self.data, 'utf-8-sig', None)) + # accept data with declared encoding + self.check_parseString(xml_bytes(self.data, 'iso-8859-1')) + # fail on non-UTF-8 incompatible data without declared encoding + with self.assertRaises(SAXException): + self.check_parseString(xml_bytes(self.data, 'iso-8859-1', None)) + +class MakeParserTest(unittest.TestCase): + def test_make_parser2(self): + # Creating parsers several times in a row should succeed. + # Testing this because there have been failures of this kind + # before. + from xml.sax import make_parser + p = make_parser() + from xml.sax import make_parser + p = make_parser() + from xml.sax import make_parser + p = make_parser() + from xml.sax import make_parser + p = make_parser() + from xml.sax import make_parser + p = make_parser() + from xml.sax import make_parser + p = make_parser() + + def test_make_parser3(self): + # Testing that make_parser can handle different types of + # iterables. + make_parser(['module']) + make_parser(('module', )) + make_parser({'module'}) + make_parser(frozenset({'module'})) + make_parser({'module': None}) + make_parser(iter(['module'])) + + def test_make_parser4(self): + # Testing that make_parser can handle empty iterables. + make_parser([]) + make_parser(tuple()) + make_parser(set()) + make_parser(frozenset()) + make_parser({}) + make_parser(iter([])) + + def test_make_parser5(self): + # Testing that make_parser can handle iterables with more than + # one item. + make_parser(['module1', 'module2']) + make_parser(('module1', 'module2')) + make_parser({'module1', 'module2'}) + make_parser(frozenset({'module1', 'module2'})) + make_parser({'module1': None, 'module2': None}) + make_parser(iter(['module1', 'module2'])) + +# =========================================================================== +# +# saxutils tests +# +# =========================================================================== + +class SaxutilsTest(unittest.TestCase): + # ===== escape + def test_escape_basic(self): + self.assertEqual(escape("Donald Duck & Co"), "Donald Duck &amp; Co") + + def test_escape_all(self): + self.assertEqual(escape("<Donald Duck & Co>"), + "&lt;Donald Duck &amp; Co&gt;") + + def test_escape_extra(self): + self.assertEqual(escape("Hei på deg", {"å" : "&aring;"}), + "Hei p&aring; deg") + + # ===== unescape + def test_unescape_basic(self): + self.assertEqual(unescape("Donald Duck &amp; Co"), "Donald Duck & Co") + + def test_unescape_all(self): + self.assertEqual(unescape("&lt;Donald Duck &amp; Co&gt;"), + "<Donald Duck & Co>") + + def test_unescape_extra(self): + self.assertEqual(unescape("Hei på deg", {"å" : "&aring;"}), + "Hei p&aring; deg") + + def test_unescape_amp_extra(self): + self.assertEqual(unescape("&amp;foo;", {"&foo;": "splat"}), "&foo;") + + # ===== quoteattr + def test_quoteattr_basic(self): + self.assertEqual(quoteattr("Donald Duck & Co"), + '"Donald Duck &amp; Co"') + + def test_single_quoteattr(self): + self.assertEqual(quoteattr('Includes "double" quotes'), + '\'Includes "double" quotes\'') + + def test_double_quoteattr(self): + self.assertEqual(quoteattr("Includes 'single' quotes"), + "\"Includes 'single' quotes\"") + + def test_single_double_quoteattr(self): + self.assertEqual(quoteattr("Includes 'single' and \"double\" quotes"), + "\"Includes 'single' and &quot;double&quot; quotes\"") + + # ===== make_parser + def test_make_parser(self): + # Creating a parser should succeed - it should fall back + # to the expatreader + p = make_parser(['xml.parsers.no_such_parser']) + + +class PrepareInputSourceTest(unittest.TestCase): + + def setUp(self): + self.file = os_helper.TESTFN + with open(self.file, "w") as tmp: + tmp.write("This was read from a file.") + + def tearDown(self): + os_helper.unlink(self.file) + + def make_byte_stream(self): + return BytesIO(b"This is a byte stream.") + + def make_character_stream(self): + return StringIO("This is a character stream.") + + def checkContent(self, stream, content): + self.assertIsNotNone(stream) + self.assertEqual(stream.read(), content) + stream.close() + + + def test_character_stream(self): + # If the source is an InputSource with a character stream, use it. + src = InputSource(self.file) + src.setCharacterStream(self.make_character_stream()) + prep = prepare_input_source(src) + self.assertIsNone(prep.getByteStream()) + self.checkContent(prep.getCharacterStream(), + "This is a character stream.") + + def test_byte_stream(self): + # If the source is an InputSource that does not have a character + # stream but does have a byte stream, use the byte stream. + src = InputSource(self.file) + src.setByteStream(self.make_byte_stream()) + prep = prepare_input_source(src) + self.assertIsNone(prep.getCharacterStream()) + self.checkContent(prep.getByteStream(), + b"This is a byte stream.") + + def test_system_id(self): + # If the source is an InputSource that has neither a character + # stream nor a byte stream, open the system ID. + src = InputSource(self.file) + prep = prepare_input_source(src) + self.assertIsNone(prep.getCharacterStream()) + self.checkContent(prep.getByteStream(), + b"This was read from a file.") + + def test_string(self): + # If the source is a string, use it as a system ID and open it. + prep = prepare_input_source(self.file) + self.assertIsNone(prep.getCharacterStream()) + self.checkContent(prep.getByteStream(), + b"This was read from a file.") + + def test_path_objects(self): + # If the source is a Path object, use it as a system ID and open it. + prep = prepare_input_source(FakePath(self.file)) + self.assertIsNone(prep.getCharacterStream()) + self.checkContent(prep.getByteStream(), + b"This was read from a file.") + + def test_binary_file(self): + # If the source is a binary file-like object, use it as a byte + # stream. + prep = prepare_input_source(self.make_byte_stream()) + self.assertIsNone(prep.getCharacterStream()) + self.checkContent(prep.getByteStream(), + b"This is a byte stream.") + + def test_text_file(self): + # If the source is a text file-like object, use it as a character + # stream. + prep = prepare_input_source(self.make_character_stream()) + self.assertIsNone(prep.getByteStream()) + self.checkContent(prep.getCharacterStream(), + "This is a character stream.") + + +# ===== XMLGenerator + +class XmlgenTest: + def test_xmlgen_basic(self): + result = self.ioclass() + gen = XMLGenerator(result) + gen.startDocument() + gen.startElement("doc", {}) + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml("<doc></doc>")) + + def test_xmlgen_basic_empty(self): + result = self.ioclass() + gen = XMLGenerator(result, short_empty_elements=True) + gen.startDocument() + gen.startElement("doc", {}) + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml("<doc/>")) + + def test_xmlgen_content(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startElement("doc", {}) + gen.characters("huhei") + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml("<doc>huhei</doc>")) + + def test_xmlgen_content_empty(self): + result = self.ioclass() + gen = XMLGenerator(result, short_empty_elements=True) + + gen.startDocument() + gen.startElement("doc", {}) + gen.characters("huhei") + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml("<doc>huhei</doc>")) + + def test_xmlgen_pi(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.processingInstruction("test", "data") + gen.startElement("doc", {}) + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml("<?test data?><doc></doc>")) + + def test_xmlgen_content_escape(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startElement("doc", {}) + gen.characters("<huhei&") + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml("<doc>&lt;huhei&amp;</doc>")) + + def test_xmlgen_attr_escape(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startElement("doc", {"a": '"'}) + gen.startElement("e", {"a": "'"}) + gen.endElement("e") + gen.startElement("e", {"a": "'\""}) + gen.endElement("e") + gen.startElement("e", {"a": "\n\r\t"}) + gen.endElement("e") + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml( + "<doc a='\"'><e a=\"'\"></e>" + "<e a=\"'&quot;\"></e>" + "<e a=\"&#10;&#13;&#9;\"></e></doc>")) + + def test_xmlgen_encoding(self): + encodings = ('iso-8859-15', 'utf-8', 'utf-8-sig', + 'utf-16', 'utf-16be', 'utf-16le', + 'utf-32', 'utf-32be', 'utf-32le') + for encoding in encodings: + result = self.ioclass() + gen = XMLGenerator(result, encoding=encoding) + + gen.startDocument() + gen.startElement("doc", {"a": '\u20ac'}) + gen.characters("\u20ac") + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml('<doc a="\u20ac">\u20ac</doc>', encoding=encoding)) + + def test_xmlgen_unencodable(self): + result = self.ioclass() + gen = XMLGenerator(result, encoding='ascii') + + gen.startDocument() + gen.startElement("doc", {"a": '\u20ac'}) + gen.characters("\u20ac") + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml('<doc a="&#8364;">&#8364;</doc>', encoding='ascii')) + + def test_xmlgen_ignorable(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startElement("doc", {}) + gen.ignorableWhitespace(" ") + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml("<doc> </doc>")) + + def test_xmlgen_ignorable_empty(self): + result = self.ioclass() + gen = XMLGenerator(result, short_empty_elements=True) + + gen.startDocument() + gen.startElement("doc", {}) + gen.ignorableWhitespace(" ") + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml("<doc> </doc>")) + + def test_xmlgen_encoding_bytes(self): + encodings = ('iso-8859-15', 'utf-8', 'utf-8-sig', + 'utf-16', 'utf-16be', 'utf-16le', + 'utf-32', 'utf-32be', 'utf-32le') + for encoding in encodings: + result = self.ioclass() + gen = XMLGenerator(result, encoding=encoding) + + gen.startDocument() + gen.startElement("doc", {"a": '\u20ac'}) + gen.characters("\u20ac".encode(encoding)) + gen.ignorableWhitespace(" ".encode(encoding)) + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml('<doc a="\u20ac">\u20ac </doc>', encoding=encoding)) + + def test_xmlgen_ns(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startPrefixMapping("ns1", ns_uri) + gen.startElementNS((ns_uri, "doc"), "ns1:doc", {}) + # add an unqualified name + gen.startElementNS((None, "udoc"), None, {}) + gen.endElementNS((None, "udoc"), None) + gen.endElementNS((ns_uri, "doc"), "ns1:doc") + gen.endPrefixMapping("ns1") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml( + '<ns1:doc xmlns:ns1="%s"><udoc></udoc></ns1:doc>' % + ns_uri)) + + def test_xmlgen_ns_empty(self): + result = self.ioclass() + gen = XMLGenerator(result, short_empty_elements=True) + + gen.startDocument() + gen.startPrefixMapping("ns1", ns_uri) + gen.startElementNS((ns_uri, "doc"), "ns1:doc", {}) + # add an unqualified name + gen.startElementNS((None, "udoc"), None, {}) + gen.endElementNS((None, "udoc"), None) + gen.endElementNS((ns_uri, "doc"), "ns1:doc") + gen.endPrefixMapping("ns1") + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml( + '<ns1:doc xmlns:ns1="%s"><udoc/></ns1:doc>' % + ns_uri)) + + def test_1463026_1(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startElementNS((None, 'a'), 'a', {(None, 'b'):'c'}) + gen.endElementNS((None, 'a'), 'a') + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml('<a b="c"></a>')) + + def test_1463026_1_empty(self): + result = self.ioclass() + gen = XMLGenerator(result, short_empty_elements=True) + + gen.startDocument() + gen.startElementNS((None, 'a'), 'a', {(None, 'b'):'c'}) + gen.endElementNS((None, 'a'), 'a') + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml('<a b="c"/>')) + + def test_1463026_2(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startPrefixMapping(None, 'qux') + gen.startElementNS(('qux', 'a'), 'a', {}) + gen.endElementNS(('qux', 'a'), 'a') + gen.endPrefixMapping(None) + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml('<a xmlns="qux"></a>')) + + def test_1463026_2_empty(self): + result = self.ioclass() + gen = XMLGenerator(result, short_empty_elements=True) + + gen.startDocument() + gen.startPrefixMapping(None, 'qux') + gen.startElementNS(('qux', 'a'), 'a', {}) + gen.endElementNS(('qux', 'a'), 'a') + gen.endPrefixMapping(None) + gen.endDocument() + + self.assertEqual(result.getvalue(), self.xml('<a xmlns="qux"/>')) + + def test_1463026_3(self): + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startPrefixMapping('my', 'qux') + gen.startElementNS(('qux', 'a'), 'a', {(None, 'b'):'c'}) + gen.endElementNS(('qux', 'a'), 'a') + gen.endPrefixMapping('my') + gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml('<my:a xmlns:my="qux" b="c"></my:a>')) + + def test_1463026_3_empty(self): + result = self.ioclass() + gen = XMLGenerator(result, short_empty_elements=True) + + gen.startDocument() + gen.startPrefixMapping('my', 'qux') + gen.startElementNS(('qux', 'a'), 'a', {(None, 'b'):'c'}) + gen.endElementNS(('qux', 'a'), 'a') + gen.endPrefixMapping('my') + gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml('<my:a xmlns:my="qux" b="c"/>')) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_5027_1(self): + # The xml prefix (as in xml:lang below) is reserved and bound by + # definition to http://www.w3.org/XML/1998/namespace. XMLGenerator had + # a bug whereby a KeyError is raised because this namespace is missing + # from a dictionary. + # + # This test demonstrates the bug by parsing a document. + test_xml = StringIO( + '<?xml version="1.0"?>' + '<a:g1 xmlns:a="http://example.com/ns">' + '<a:g2 xml:lang="en">Hello</a:g2>' + '</a:g1>') + + parser = make_parser() + parser.setFeature(feature_namespaces, True) + result = self.ioclass() + gen = XMLGenerator(result) + parser.setContentHandler(gen) + parser.parse(test_xml) + + self.assertEqual(result.getvalue(), + self.xml( + '<a:g1 xmlns:a="http://example.com/ns">' + '<a:g2 xml:lang="en">Hello</a:g2>' + '</a:g1>')) + + def test_5027_2(self): + # The xml prefix (as in xml:lang below) is reserved and bound by + # definition to http://www.w3.org/XML/1998/namespace. XMLGenerator had + # a bug whereby a KeyError is raised because this namespace is missing + # from a dictionary. + # + # This test demonstrates the bug by direct manipulation of the + # XMLGenerator. + result = self.ioclass() + gen = XMLGenerator(result) + + gen.startDocument() + gen.startPrefixMapping('a', 'http://example.com/ns') + gen.startElementNS(('http://example.com/ns', 'g1'), 'g1', {}) + lang_attr = {('http://www.w3.org/XML/1998/namespace', 'lang'): 'en'} + gen.startElementNS(('http://example.com/ns', 'g2'), 'g2', lang_attr) + gen.characters('Hello') + gen.endElementNS(('http://example.com/ns', 'g2'), 'g2') + gen.endElementNS(('http://example.com/ns', 'g1'), 'g1') + gen.endPrefixMapping('a') + gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml( + '<a:g1 xmlns:a="http://example.com/ns">' + '<a:g2 xml:lang="en">Hello</a:g2>' + '</a:g1>')) + + def test_no_close_file(self): + result = self.ioclass() + def func(out): + gen = XMLGenerator(out) + gen.startDocument() + gen.startElement("doc", {}) + func(result) + self.assertFalse(result.closed) + + def test_xmlgen_fragment(self): + result = self.ioclass() + gen = XMLGenerator(result) + + # Don't call gen.startDocument() + gen.startElement("foo", {"a": "1.0"}) + gen.characters("Hello") + gen.endElement("foo") + gen.startElement("bar", {"b": "2.0"}) + gen.endElement("bar") + # Don't call gen.endDocument() + + self.assertEqual(result.getvalue(), + self.xml('<foo a="1.0">Hello</foo><bar b="2.0"></bar>')[len(self.xml('')):]) + +class StringXmlgenTest(XmlgenTest, unittest.TestCase): + ioclass = StringIO + + def xml(self, doc, encoding='iso-8859-1'): + return '<?xml version="1.0" encoding="%s"?>\n%s' % (encoding, doc) + + test_xmlgen_unencodable = None + +class BytesXmlgenTest(XmlgenTest, unittest.TestCase): + ioclass = BytesIO + + def xml(self, doc, encoding='iso-8859-1'): + return ('<?xml version="1.0" encoding="%s"?>\n%s' % + (encoding, doc)).encode(encoding, 'xmlcharrefreplace') + +class WriterXmlgenTest(BytesXmlgenTest): + class ioclass(list): + write = list.append + closed = False + + def seekable(self): + return True + + def tell(self): + # return 0 at start and not 0 after start + return len(self) + + def getvalue(self): + return b''.join(self) + +class StreamWriterXmlgenTest(XmlgenTest, unittest.TestCase): + def ioclass(self): + raw = BytesIO() + writer = codecs.getwriter('ascii')(raw, 'xmlcharrefreplace') + writer.getvalue = raw.getvalue + return writer + + def xml(self, doc, encoding='iso-8859-1'): + return ('<?xml version="1.0" encoding="%s"?>\n%s' % + (encoding, doc)).encode('ascii', 'xmlcharrefreplace') + +class StreamReaderWriterXmlgenTest(XmlgenTest, unittest.TestCase): + fname = os_helper.TESTFN + '-codecs' + + def ioclass(self): + with self.assertWarns(DeprecationWarning): + writer = codecs.open(self.fname, 'w', encoding='ascii', + errors='xmlcharrefreplace', buffering=0) + def cleanup(): + writer.close() + os_helper.unlink(self.fname) + self.addCleanup(cleanup) + def getvalue(): + # Windows will not let use reopen without first closing + writer.close() + with open(writer.name, 'rb') as f: + return f.read() + writer.getvalue = getvalue + return writer + + def xml(self, doc, encoding='iso-8859-1'): + return ('<?xml version="1.0" encoding="%s"?>\n%s' % + (encoding, doc)).encode('ascii', 'xmlcharrefreplace') + +start = b'<?xml version="1.0" encoding="iso-8859-1"?>\n' + + +class XMLFilterBaseTest(unittest.TestCase): + def test_filter_basic(self): + result = BytesIO() + gen = XMLGenerator(result) + filter = XMLFilterBase() + filter.setContentHandler(gen) + + filter.startDocument() + filter.startElement("doc", {}) + filter.characters("content") + filter.ignorableWhitespace(" ") + filter.endElement("doc") + filter.endDocument() + + self.assertEqual(result.getvalue(), start + b"<doc>content </doc>") + +# =========================================================================== +# +# expatreader tests +# +# =========================================================================== + +with open(TEST_XMLFILE_OUT, 'rb') as f: + xml_test_out = f.read() + +class ExpatReaderTest(XmlTestBase): + + # ===== XMLReader support + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_binary_file(self): + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + with open(TEST_XMLFILE, 'rb') as f: + parser.parse(f) + + self.assertEqual(result.getvalue(), xml_test_out) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_text_file(self): + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + with open(TEST_XMLFILE, 'rt', encoding='iso-8859-1') as f: + parser.parse(f) + + self.assertEqual(result.getvalue(), xml_test_out) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + @requires_nonascii_filenames + def test_expat_binary_file_nonascii(self): + fname = os_helper.TESTFN_UNICODE + shutil.copyfile(TEST_XMLFILE, fname) + self.addCleanup(os_helper.unlink, fname) + + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + parser.parse(open(fname, 'rb')) + + self.assertEqual(result.getvalue(), xml_test_out) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_binary_file_bytes_name(self): + fname = os.fsencode(TEST_XMLFILE) + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + with open(fname, 'rb') as f: + parser.parse(f) + + self.assertEqual(result.getvalue(), xml_test_out) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_binary_file_int_name(self): + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + with open(TEST_XMLFILE, 'rb') as f: + with open(f.fileno(), 'rb', closefd=False) as f2: + parser.parse(f2) + + self.assertEqual(result.getvalue(), xml_test_out) + + # ===== DTDHandler support + + class TestDTDHandler: + + def __init__(self): + self._notations = [] + self._entities = [] + + def notationDecl(self, name, publicId, systemId): + self._notations.append((name, publicId, systemId)) + + def unparsedEntityDecl(self, name, publicId, systemId, ndata): + self._entities.append((name, publicId, systemId, ndata)) + + + class TestEntityRecorder: + def __init__(self): + self.entities = [] + + def resolveEntity(self, publicId, systemId): + self.entities.append((publicId, systemId)) + source = InputSource() + source.setPublicId(publicId) + source.setSystemId(systemId) + return source + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_dtdhandler(self): + parser = create_parser() + handler = self.TestDTDHandler() + parser.setDTDHandler(handler) + + parser.feed('<!DOCTYPE doc [\n') + parser.feed(' <!ENTITY img SYSTEM "expat.gif" NDATA GIF>\n') + parser.feed(' <!NOTATION GIF PUBLIC "-//CompuServe//NOTATION Graphics Interchange Format 89a//EN">\n') + parser.feed(']>\n') + parser.feed('<doc></doc>') + parser.close() + + self.assertEqual(handler._notations, + [("GIF", "-//CompuServe//NOTATION Graphics Interchange Format 89a//EN", None)]) + self.assertEqual(handler._entities, [("img", None, "expat.gif", "GIF")]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_external_dtd_enabled(self): + # clear _opener global variable + self.addCleanup(urllib.request.urlcleanup) + + parser = create_parser() + parser.setFeature(feature_external_ges, True) + resolver = self.TestEntityRecorder() + parser.setEntityResolver(resolver) + + with self.assertRaises(URLError): + parser.feed( + '<!DOCTYPE external SYSTEM "unsupported://non-existing">\n' + ) + self.assertEqual( + resolver.entities, [(None, 'unsupported://non-existing')] + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_external_dtd_default(self): + parser = create_parser() + resolver = self.TestEntityRecorder() + parser.setEntityResolver(resolver) + + parser.feed( + '<!DOCTYPE external SYSTEM "unsupported://non-existing">\n' + ) + parser.feed('<doc />') + parser.close() + self.assertEqual(resolver.entities, []) + + # ===== EntityResolver support + + class TestEntityResolver: + + def resolveEntity(self, publicId, systemId): + inpsrc = InputSource() + inpsrc.setByteStream(BytesIO(b"<entity/>")) + return inpsrc + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_entityresolver_enabled(self): + parser = create_parser() + parser.setFeature(feature_external_ges, True) + parser.setEntityResolver(self.TestEntityResolver()) + result = BytesIO() + parser.setContentHandler(XMLGenerator(result)) + + parser.feed('<!DOCTYPE doc [\n') + parser.feed(' <!ENTITY test SYSTEM "whatever">\n') + parser.feed(']>\n') + parser.feed('<doc>&test;</doc>') + parser.close() + + self.assertEqual(result.getvalue(), start + + b"<doc><entity></entity></doc>") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_entityresolver_default(self): + parser = create_parser() + self.assertEqual(parser.getFeature(feature_external_ges), False) + parser.setEntityResolver(self.TestEntityResolver()) + result = BytesIO() + parser.setContentHandler(XMLGenerator(result)) + + parser.feed('<!DOCTYPE doc [\n') + parser.feed(' <!ENTITY test SYSTEM "whatever">\n') + parser.feed(']>\n') + parser.feed('<doc>&test;</doc>') + parser.close() + + self.assertEqual(result.getvalue(), start + + b"<doc></doc>") + + # ===== Attributes support + + class AttrGatherer(ContentHandler): + + def startElement(self, name, attrs): + self._attrs = attrs + + def startElementNS(self, name, qname, attrs): + self._attrs = attrs + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_attrs_empty(self): + parser = create_parser() + gather = self.AttrGatherer() + parser.setContentHandler(gather) + + parser.feed("<doc/>") + parser.close() + + self.verify_empty_attrs(gather._attrs) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_attrs_wattr(self): + parser = create_parser() + gather = self.AttrGatherer() + parser.setContentHandler(gather) + + parser.feed("<doc attr='val'/>") + parser.close() + + self.verify_attrs_wattr(gather._attrs) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_nsattrs_empty(self): + parser = create_parser(1) + gather = self.AttrGatherer() + parser.setContentHandler(gather) + + parser.feed("<doc/>") + parser.close() + + self.verify_empty_nsattrs(gather._attrs) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_nsattrs_wattr(self): + parser = create_parser(1) + gather = self.AttrGatherer() + parser.setContentHandler(gather) + + parser.feed("<doc xmlns:ns='%s' ns:attr='val'/>" % ns_uri) + parser.close() + + attrs = gather._attrs + + self.assertEqual(attrs.getLength(), 1) + self.assertEqual(attrs.getNames(), [(ns_uri, "attr")]) + self.assertTrue((attrs.getQNames() == [] or + attrs.getQNames() == ["ns:attr"])) + self.assertEqual(len(attrs), 1) + self.assertIn((ns_uri, "attr"), attrs) + self.assertEqual(attrs.get((ns_uri, "attr")), "val") + self.assertEqual(attrs.get((ns_uri, "attr"), 25), "val") + self.assertEqual(list(attrs.items()), [((ns_uri, "attr"), "val")]) + self.assertEqual(list(attrs.values()), ["val"]) + self.assertEqual(attrs.getValue((ns_uri, "attr")), "val") + self.assertEqual(attrs[(ns_uri, "attr")], "val") + + # ===== InputSource support + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_inpsource_filename(self): + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + parser.parse(TEST_XMLFILE) + + self.assertEqual(result.getvalue(), xml_test_out) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_inpsource_sysid(self): + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + parser.parse(InputSource(TEST_XMLFILE)) + + self.assertEqual(result.getvalue(), xml_test_out) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + @requires_nonascii_filenames + def test_expat_inpsource_sysid_nonascii(self): + fname = os_helper.TESTFN_UNICODE + shutil.copyfile(TEST_XMLFILE, fname) + self.addCleanup(os_helper.unlink, fname) + + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + parser.parse(InputSource(fname)) + + self.assertEqual(result.getvalue(), xml_test_out) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_inpsource_byte_stream(self): + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + inpsrc = InputSource() + with open(TEST_XMLFILE, 'rb') as f: + inpsrc.setByteStream(f) + parser.parse(inpsrc) + + self.assertEqual(result.getvalue(), xml_test_out) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_inpsource_character_stream(self): + parser = create_parser() + result = BytesIO() + xmlgen = XMLGenerator(result) + + parser.setContentHandler(xmlgen) + inpsrc = InputSource() + with open(TEST_XMLFILE, 'rt', encoding='iso-8859-1') as f: + inpsrc.setCharacterStream(f) + parser.parse(inpsrc) + + self.assertEqual(result.getvalue(), xml_test_out) + + # ===== IncrementalParser support + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_incremental(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + parser.feed("<doc>") + parser.feed("</doc>") + parser.close() + + self.assertEqual(result.getvalue(), start + b"<doc></doc>") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_incremental_reset(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + parser.feed("<doc>") + parser.feed("text") + + result = BytesIO() + xmlgen = XMLGenerator(result) + parser.setContentHandler(xmlgen) + parser.reset() + + parser.feed("<doc>") + parser.feed("text") + parser.feed("</doc>") + parser.close() + + self.assertEqual(result.getvalue(), start + b"<doc>text</doc>") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + def test_flush_reparse_deferral_enabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in ("<doc", ">"): + parser.feed(chunk) + + self.assertEqual(result.getvalue(), start) # i.e. no elements started + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertTrue(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"<doc>") + + parser.feed("</doc>") + parser.close() + + self.assertEqual(result.getvalue(), start + b"<doc></doc>") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_flush_reparse_deferral_disabled(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + for chunk in ("<doc", ">"): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + parser._parser.SetReparseDeferralEnabled(False) + self.assertEqual(result.getvalue(), start) # i.e. no elements started + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assertFalse(parser._parser.GetReparseDeferralEnabled()) + self.assertEqual(result.getvalue(), start + b"<doc>") + + parser.feed("</doc>") + parser.close() + + self.assertEqual(result.getvalue(), start + b"<doc></doc>") + + # ===== Locator support + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_locator_noinfo(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + + parser.feed("<doc>") + parser.feed("</doc>") + parser.close() + + self.assertEqual(parser.getSystemId(), None) + self.assertEqual(parser.getPublicId(), None) + self.assertEqual(parser.getLineNumber(), 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_locator_withinfo(self): + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + parser.parse(TEST_XMLFILE) + + self.assertEqual(parser.getSystemId(), TEST_XMLFILE) + self.assertEqual(parser.getPublicId(), None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + @requires_nonascii_filenames + def test_expat_locator_withinfo_nonascii(self): + fname = os_helper.TESTFN_UNICODE + shutil.copyfile(TEST_XMLFILE, fname) + self.addCleanup(os_helper.unlink, fname) + + result = BytesIO() + xmlgen = XMLGenerator(result) + parser = create_parser() + parser.setContentHandler(xmlgen) + parser.parse(fname) + + self.assertEqual(parser.getSystemId(), fname) + self.assertEqual(parser.getPublicId(), None) + + +# =========================================================================== +# +# error reporting +# +# =========================================================================== + +class ErrorReportingTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_inpsource_location(self): + parser = create_parser() + parser.setContentHandler(ContentHandler()) # do nothing + source = InputSource() + source.setByteStream(BytesIO(b"<foo bar foobar>")) #ill-formed + name = "a file name" + source.setSystemId(name) + try: + parser.parse(source) + self.fail() + except SAXException as e: + self.assertEqual(e.getSystemId(), name) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_expat_incomplete(self): + parser = create_parser() + parser.setContentHandler(ContentHandler()) # do nothing + self.assertRaises(SAXParseException, parser.parse, StringIO("<foo>")) + self.assertEqual(parser.getColumnNumber(), 5) + self.assertEqual(parser.getLineNumber(), 1) + + def test_sax_parse_exception_str(self): + # pass various values from a locator to the SAXParseException to + # make sure that the __str__() doesn't fall apart when None is + # passed instead of an integer line and column number + # + # use "normal" values for the locator: + str(SAXParseException("message", None, + self.DummyLocator(1, 1))) + # use None for the line number: + str(SAXParseException("message", None, + self.DummyLocator(None, 1))) + # use None for the column number: + str(SAXParseException("message", None, + self.DummyLocator(1, None))) + # use None for both: + str(SAXParseException("message", None, + self.DummyLocator(None, None))) + + class DummyLocator: + def __init__(self, lineno, colno): + self._lineno = lineno + self._colno = colno + + def getPublicId(self): + return "pubid" + + def getSystemId(self): + return "sysid" + + def getLineNumber(self): + return self._lineno + + def getColumnNumber(self): + return self._colno + +# =========================================================================== +# +# xmlreader tests +# +# =========================================================================== + +class XmlReaderTest(XmlTestBase): + + # ===== AttributesImpl + def test_attrs_empty(self): + self.verify_empty_attrs(AttributesImpl({})) + + def test_attrs_wattr(self): + self.verify_attrs_wattr(AttributesImpl({"attr" : "val"})) + + def test_nsattrs_empty(self): + self.verify_empty_nsattrs(AttributesNSImpl({}, {})) + + def test_nsattrs_wattr(self): + attrs = AttributesNSImpl({(ns_uri, "attr") : "val"}, + {(ns_uri, "attr") : "ns:attr"}) + + self.assertEqual(attrs.getLength(), 1) + self.assertEqual(attrs.getNames(), [(ns_uri, "attr")]) + self.assertEqual(attrs.getQNames(), ["ns:attr"]) + self.assertEqual(len(attrs), 1) + self.assertIn((ns_uri, "attr"), attrs) + self.assertEqual(list(attrs.keys()), [(ns_uri, "attr")]) + self.assertEqual(attrs.get((ns_uri, "attr")), "val") + self.assertEqual(attrs.get((ns_uri, "attr"), 25), "val") + self.assertEqual(list(attrs.items()), [((ns_uri, "attr"), "val")]) + self.assertEqual(list(attrs.values()), ["val"]) + self.assertEqual(attrs.getValue((ns_uri, "attr")), "val") + self.assertEqual(attrs.getValueByQName("ns:attr"), "val") + self.assertEqual(attrs.getNameByQName("ns:attr"), (ns_uri, "attr")) + self.assertEqual(attrs[(ns_uri, "attr")], "val") + self.assertEqual(attrs.getQNameByName((ns_uri, "attr")), "ns:attr") + + +class LexicalHandlerTest(unittest.TestCase): + def setUp(self): + self.parser = None + + self.specified_version = '1.0' + self.specified_encoding = 'UTF-8' + self.specified_doctype = 'wish' + self.specified_entity_names = ('nbsp', 'source', 'target') + self.specified_comment = ('Comment in a DTD', + 'Really! You think so?') + self.test_data = StringIO() + self.test_data.write('<?xml version="{}" encoding="{}"?>\n'. + format(self.specified_version, + self.specified_encoding)) + self.test_data.write('<!DOCTYPE {} [\n'. + format(self.specified_doctype)) + self.test_data.write('<!-- {} -->\n'. + format(self.specified_comment[0])) + self.test_data.write('<!ELEMENT {} (to,from,heading,body,footer)>\n'. + format(self.specified_doctype)) + self.test_data.write('<!ELEMENT to (#PCDATA)>\n') + self.test_data.write('<!ELEMENT from (#PCDATA)>\n') + self.test_data.write('<!ELEMENT heading (#PCDATA)>\n') + self.test_data.write('<!ELEMENT body (#PCDATA)>\n') + self.test_data.write('<!ELEMENT footer (#PCDATA)>\n') + self.test_data.write('<!ENTITY {} "&#xA0;">\n'. + format(self.specified_entity_names[0])) + self.test_data.write('<!ENTITY {} "Written by: Alexander.">\n'. + format(self.specified_entity_names[1])) + self.test_data.write('<!ENTITY {} "Hope it gets to: Aristotle.">\n'. + format(self.specified_entity_names[2])) + self.test_data.write(']>\n') + self.test_data.write('<{}>'.format(self.specified_doctype)) + self.test_data.write('<to>Aristotle</to>\n') + self.test_data.write('<from>Alexander</from>\n') + self.test_data.write('<heading>Supplication</heading>\n') + self.test_data.write('<body>Teach me patience!</body>\n') + self.test_data.write('<footer>&{};&{};&{};</footer>\n'. + format(self.specified_entity_names[1], + self.specified_entity_names[0], + self.specified_entity_names[2])) + self.test_data.write('<!-- {} -->\n'.format(self.specified_comment[1])) + self.test_data.write('</{}>\n'.format(self.specified_doctype)) + self.test_data.seek(0) + + # Data received from handlers - to be validated + self.version = None + self.encoding = None + self.standalone = None + self.doctype = None + self.publicID = None + self.systemID = None + self.end_of_dtd = False + self.comments = [] + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_handlers(self): + class TestLexicalHandler(LexicalHandler): + def __init__(self, test_harness, *args, **kwargs): + super().__init__(*args, **kwargs) + self.test_harness = test_harness + + def startDTD(self, doctype, publicID, systemID): + self.test_harness.doctype = doctype + self.test_harness.publicID = publicID + self.test_harness.systemID = systemID + + def endDTD(self): + self.test_harness.end_of_dtd = True + + def comment(self, text): + self.test_harness.comments.append(text) + + self.parser = create_parser() + self.parser.setContentHandler(ContentHandler()) + self.parser.setProperty( + 'http://xml.org/sax/properties/lexical-handler', + TestLexicalHandler(self)) + source = InputSource() + source.setCharacterStream(self.test_data) + self.parser.parse(source) + self.assertEqual(self.doctype, self.specified_doctype) + self.assertIsNone(self.publicID) + self.assertIsNone(self.systemID) + self.assertTrue(self.end_of_dtd) + self.assertEqual(len(self.comments), + len(self.specified_comment)) + self.assertEqual(f' {self.specified_comment[0]} ', self.comments[0]) + + +class CDATAHandlerTest(unittest.TestCase): + def setUp(self): + self.parser = None + self.specified_chars = [] + self.specified_chars.append(('Parseable character data', False)) + self.specified_chars.append(('<> &% - assorted other XML junk.', True)) + self.char_index = 0 # Used to index specified results within handlers + self.test_data = StringIO() + self.test_data.write('<root_doc>\n') + self.test_data.write('<some_pcdata>\n') + self.test_data.write(f'{self.specified_chars[0][0]}\n') + self.test_data.write('</some_pcdata>\n') + self.test_data.write('<some_cdata>\n') + self.test_data.write(f'<![CDATA[{self.specified_chars[1][0]}]]>\n') + self.test_data.write('</some_cdata>\n') + self.test_data.write('</root_doc>\n') + self.test_data.seek(0) + + # Data received from handlers - to be validated + self.chardata = [] + self.in_cdata = False + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetParamEntityParsing' + def test_handlers(self): + class TestLexicalHandler(LexicalHandler): + def __init__(self, test_harness, *args, **kwargs): + super().__init__(*args, **kwargs) + self.test_harness = test_harness + + def startCDATA(self): + self.test_harness.in_cdata = True + + def endCDATA(self): + self.test_harness.in_cdata = False + + class TestCharHandler(ContentHandler): + def __init__(self, test_harness, *args, **kwargs): + super().__init__(*args, **kwargs) + self.test_harness = test_harness + + def characters(self, content): + if content != '\n': + h = self.test_harness + t = h.specified_chars[h.char_index] + h.assertEqual(t[0], content) + h.assertEqual(t[1], h.in_cdata) + h.char_index += 1 + + self.parser = create_parser() + self.parser.setContentHandler(TestCharHandler(self)) + self.parser.setProperty( + 'http://xml.org/sax/properties/lexical-handler', + TestLexicalHandler(self)) + source = InputSource() + source.setCharacterStream(self.test_data) + self.parser.parse(source) + + self.assertFalse(self.in_cdata) + self.assertEqual(self.char_index, 2) + + +class TestModuleAll(unittest.TestCase): + def test_all(self): + extra = ( + 'ContentHandler', + 'ErrorHandler', + 'InputSource', + 'SAXException', + 'SAXNotRecognizedException', + 'SAXNotSupportedException', + 'SAXParseException', + 'SAXReaderNotAvailable', + ) + check__all__(self, sax, extra=extra) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py index 7ae7baae85e..eb52ac7983f 100644 --- a/Lib/test/test_sched.py +++ b/Lib/test/test_sched.py @@ -58,6 +58,7 @@ def test_enterabs(self): scheduler.run() self.assertEqual(l, [0.01, 0.02, 0.03, 0.04, 0.05]) + @threading_helper.requires_working_threading() def test_enter_concurrent(self): q = queue.Queue() fun = q.put @@ -91,10 +92,23 @@ def test_priority(self): l = [] fun = lambda x: l.append(x) scheduler = sched.scheduler(time.time, time.sleep) - for priority in [1, 2, 3, 4, 5]: - z = scheduler.enterabs(0.01, priority, fun, (priority,)) - scheduler.run() - self.assertEqual(l, [1, 2, 3, 4, 5]) + + cases = [ + ([1, 2, 3, 4, 5], [1, 2, 3, 4, 5]), + ([5, 4, 3, 2, 1], [1, 2, 3, 4, 5]), + ([2, 5, 3, 1, 4], [1, 2, 3, 4, 5]), + ([1, 2, 3, 2, 1], [1, 1, 2, 2, 3]), + ] + for priorities, expected in cases: + with self.subTest(priorities=priorities, expected=expected): + for priority in priorities: + scheduler.enterabs(0.01, priority, fun, (priority,)) + scheduler.run() + self.assertEqual(l, expected) + + # Cleanup: + self.assertTrue(scheduler.empty()) + l.clear() def test_cancel(self): l = [] @@ -111,6 +125,7 @@ def test_cancel(self): scheduler.run() self.assertEqual(l, [0.02, 0.03, 0.04]) + @threading_helper.requires_working_threading() def test_cancel_concurrent(self): q = queue.Queue() fun = q.put diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 29a4ac3c162..952afb7e0d3 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -177,6 +177,57 @@ def bar(): self.assertEqual(foo(a=42), 50) self.assertEqual(foo(), 25) + def testCellIsArgAndEscapes(self): + # We need to be sure that a cell passed in as an arg still + # gets wrapped in a new cell if the arg escapes into an + # inner function (closure). + + def external(): + value = 42 + def inner(): + return value + cell, = inner.__closure__ + return cell + cell_ext = external() + + def spam(arg): + def eggs(): + return arg + return eggs + + eggs = spam(cell_ext) + cell_closure, = eggs.__closure__ + cell_eggs = eggs() + + self.assertIs(cell_eggs, cell_ext) + self.assertIsNot(cell_eggs, cell_closure) + + def testCellIsLocalAndEscapes(self): + # We need to be sure that a cell bound to a local still + # gets wrapped in a new cell if the local escapes into an + # inner function (closure). + + def external(): + value = 42 + def inner(): + return value + cell, = inner.__closure__ + return cell + cell_ext = external() + + def spam(arg): + cell = arg + def eggs(): + return cell + return eggs + + eggs = spam(cell_ext) + cell_closure, = eggs.__closure__ + cell_eggs = eggs() + + self.assertIs(cell_eggs, cell_ext) + self.assertIsNot(cell_eggs, cell_closure) + def testRecursion(self): def f(x): @@ -641,10 +692,7 @@ def dec(self): self.assertEqual(c.dec(), 1) self.assertEqual(c.dec(), 0) - # TODO: RUSTPYTHON, figure out how to communicate that `y = 9` should be - # stored as a global rather than a STORE_NAME, even when - # the `global y` is in a nested subscope - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; figure out how to communicate that `y = 9` should be stored as a global rather than a STORE_NAME, even when the `global y` is in a nested subscope def testGlobalInParallelNestedFunctions(self): # A symbol table bug leaked the global statement from one # function to other nested functions in the same block. @@ -731,7 +779,7 @@ class X: class X: locals()["x"] = 43 del x - self.assertFalse(hasattr(X, "x")) + self.assertNotHasAttr(X, "x") self.assertEqual(x, 42) @cpython_only @@ -763,6 +811,30 @@ def dig(self): gc_collect() # For PyPy or other GCs. self.assertIsNone(ref()) + def test_multiple_nesting(self): + # Regression test for https://github.com/python/cpython/issues/121863 + class MultiplyNested: + def f1(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g(_MultiplyNested__arg=2) + + def f2(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g + + inst = MultiplyNested() + with self.assertRaises(TypeError): + inst.f1() + + closure = inst.f2() + with self.assertRaises(TypeError): + closure(_MultiplyNested__arg=2) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_script_helper.py b/Lib/test/test_script_helper.py index e7b54fd7798..4ade2cbc0d4 100644 --- a/Lib/test/test_script_helper.py +++ b/Lib/test/test_script_helper.py @@ -82,7 +82,6 @@ def tearDown(self): # Reset the private cached state. script_helper.__dict__['__cached_interp_requires_environment'] = None - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") @mock.patch('subprocess.check_call') def test_interpreter_requires_environment_true(self, mock_check_call): with mock.patch.dict(os.environ): @@ -92,7 +91,6 @@ def test_interpreter_requires_environment_true(self, mock_check_call): self.assertTrue(script_helper.interpreter_requires_environment()) self.assertEqual(1, mock_check_call.call_count) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") @mock.patch('subprocess.check_call') def test_interpreter_requires_environment_false(self, mock_check_call): with mock.patch.dict(os.environ): @@ -102,7 +100,6 @@ def test_interpreter_requires_environment_false(self, mock_check_call): self.assertFalse(script_helper.interpreter_requires_environment()) self.assertEqual(1, mock_check_call.call_count) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") @mock.patch('subprocess.check_call') def test_interpreter_requires_environment_details(self, mock_check_call): with mock.patch.dict(os.environ): @@ -115,7 +112,6 @@ def test_interpreter_requires_environment_details(self, mock_check_call): self.assertEqual(sys.executable, check_call_command[0]) self.assertIn('-E', check_call_command) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") @mock.patch('subprocess.check_call') def test_interpreter_requires_environment_with_pythonhome(self, mock_check_call): with mock.patch.dict(os.environ): diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py new file mode 100644 index 00000000000..6ce8cd423f7 --- /dev/null +++ b/Lib/test/test_select.py @@ -0,0 +1,107 @@ +import errno +import select +import subprocess +import sys +import textwrap +import unittest +from test import support + +support.requires_working_socket(module=True) + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + try: + select.select([fd], [], [], 0) + except OSError as err: + self.assertEqual(err.errno, errno.EBADF) + else: + self.fail("exception not raised") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: unexpectedly identical: [] + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + @support.requires_fork() + def test_select(self): + code = textwrap.dedent(''' + import time + for i in range(10): + print("testing...", flush=True) + time.sleep(0.050) + ''') + cmd = [sys.executable, '-I', '-c', code] + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + pipe = proc.stdout + for timeout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print(f'timeout = {timeout}') + rfd, wfd, xfd = select.select([pipe], [], [], timeout) + self.assertEqual(wfd, []) + self.assertEqual(xfd, []) + if not rfd: + continue + if rfd == [pipe]: + line = pipe.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select():', + rfd, wfd, xfd) + + # Issue 16230: Crash on select resized list + @unittest.skipIf( + support.is_emscripten, "Emscripten cannot select a fd multiple times." + ) + @unittest.skip("TODO: RUSTPYTHON timed out") + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + self.assertEqual(select.select([], a, []), ([], a[:5], [])) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by poll + def test_disallow_instantiation(self): + support.check_disallow_instantiation(self, type(select.poll())) + + if hasattr(select, 'devpoll'): + support.check_disallow_instantiation(self, type(select.devpoll())) + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index 42c5e8879b0..643775597c5 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -6,8 +6,7 @@ import socket import sys from test import support -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from time import sleep import unittest import unittest.mock @@ -19,6 +18,10 @@ resource = None +if support.is_emscripten or support.is_wasi: + raise unittest.SkipTest("Cannot create socketpair on Emscripten/WASI.") + + if hasattr(socket, 'socketpair'): socketpair = socket.socketpair else: @@ -128,7 +131,6 @@ def test_unregister_after_fd_close_and_reuse(self): s.unregister(r) s.unregister(w) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_unregister_after_socket_close(self): s = self.SELECTOR() self.addCleanup(s.close) @@ -220,6 +222,8 @@ def test_close(self): self.assertRaises(RuntimeError, s.get_key, wr) self.assertRaises(KeyError, mapping.__getitem__, rd) self.assertRaises(KeyError, mapping.__getitem__, wr) + self.assertEqual(mapping.get(rd), None) + self.assertEqual(mapping.get(wr), None) def test_get_key(self): s = self.SELECTOR() @@ -238,13 +242,17 @@ def test_get_map(self): self.addCleanup(s.close) rd, wr = self.make_socketpair() + sentinel = object() keys = s.get_map() self.assertFalse(keys) self.assertEqual(len(keys), 0) self.assertEqual(list(keys), []) + self.assertEqual(keys.get(rd), None) + self.assertEqual(keys.get(rd, sentinel), sentinel) key = s.register(rd, selectors.EVENT_READ, "data") self.assertIn(rd, keys) + self.assertEqual(key, keys.get(rd)) self.assertEqual(key, keys[rd]) self.assertEqual(len(keys), 1) self.assertEqual(list(keys), [rd.fileno()]) @@ -276,6 +284,35 @@ def test_select(self): self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result) + def test_select_read_write(self): + # gh-110038: when a file descriptor is registered for both read and + # write, the two events must be seen on a single call to select(). + s = self.SELECTOR() + self.addCleanup(s.close) + + sock1, sock2 = self.make_socketpair() + sock2.send(b"foo") + my_key = s.register(sock1, selectors.EVENT_READ | selectors.EVENT_WRITE) + + seen_read, seen_write = False, False + result = s.select() + # We get the read and write either in the same result entry or in two + # distinct entries with the same key. + self.assertLessEqual(len(result), 2) + for key, events in result: + self.assertTrue(isinstance(key, selectors.SelectorKey)) + self.assertEqual(key, my_key) + self.assertFalse(events & ~(selectors.EVENT_READ | + selectors.EVENT_WRITE)) + if events & selectors.EVENT_READ: + self.assertFalse(seen_read) + seen_read = True + if events & selectors.EVENT_WRITE: + self.assertFalse(seen_write) + seen_write = True + self.assertTrue(seen_read) + self.assertTrue(seen_write) + def test_context_manager(self): s = self.SELECTOR() self.addCleanup(s.close) @@ -446,6 +483,7 @@ class ScalableSelectorMixIn: # see issue #18963 for why it's skipped on older OS X versions @support.requires_mac_ver(10, 5) @unittest.skipUnless(resource, "Test needs resource module") + @support.requires_resource('cpu') def test_above_fd_setsize(self): # A scalable implementation should have no problem with more than # FD_SETSIZE file descriptors. Since we don't know the value, we just @@ -487,7 +525,7 @@ def test_above_fd_setsize(self): try: fds = s.select() except OSError as e: - if e.errno == errno.EINVAL and sys.platform == 'darwin': + if e.errno == errno.EINVAL and is_apple: # unexplainable errors on macOS don't need to fail the test self.skipTest("Invalid argument error calling poll()") raise diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 4284393ca5e..c68ba49d916 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -1,16 +1,17 @@ -import unittest -from test import support -from test.support import warnings_helper +import collections.abc +import copy import gc -import weakref +import itertools import operator -import copy import pickle -from random import randrange, shuffle +import re +import unittest import warnings -import collections -import collections.abc -import itertools +import weakref +from random import randrange, shuffle +from test import support +from test.support import warnings_helper + class PassThru(Exception): pass @@ -19,6 +20,14 @@ def check_pass_thru(): raise PassThru yield 1 +class CustomHash: + def __init__(self, hash): + self.hash = hash + def __hash__(self): + return self.hash + def __repr__(self): + return f'<CustomHash {self.hash} at {id(self):#x}>' + class BadCmp: def __hash__(self): return 1 @@ -227,14 +236,17 @@ def test_sub_and_super(self): def test_pickling(self): for i in range(pickle.HIGHEST_PROTOCOL + 1): + if type(self.s) not in (set, frozenset): + self.s.x = ['x'] + self.s.z = ['z'] p = pickle.dumps(self.s, i) dup = pickle.loads(p) self.assertEqual(self.s, dup, "%s != %s" % (self.s, dup)) if type(self.s) not in (set, frozenset): - self.s.x = 10 - p = pickle.dumps(self.s, i) - dup = pickle.loads(p) self.assertEqual(self.s.x, dup.x) + self.assertEqual(self.s.z, dup.z) + self.assertNotHasAttr(self.s, 'y') + del self.s.x, self.s.z def test_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -318,8 +330,7 @@ def test_cyclical_repr(self): name = repr(s).partition('(')[0] # strip class name self.assertEqual(repr(s), '%s({%s(...)})' % (name, name)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_do_not_rehash_dict_keys(self): n = 10 d = dict.fromkeys(map(HashCountingInt, range(n))) @@ -339,8 +350,7 @@ def test_do_not_rehash_dict_keys(self): self.assertEqual(sum(elem.hash_count for elem in d), n) self.assertEqual(d3, dict.fromkeys(d, 123)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_container_iterator(self): # Bug #3680: tp_traverse was not implemented for set iterator object class C(object): @@ -353,8 +363,7 @@ class C(object): gc.collect() self.assertTrue(ref() is None, "Cycle was not collected") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.thetype) @@ -430,7 +439,7 @@ def test_remove(self): self.assertRaises(KeyError, self.s.remove, self.thetype(self.word)) def test_remove_keyerror_unpacking(self): - # bug: www.python.org/sf/1576657 + # https://bugs.python.org/issue1576657 for v1 in ['Q', (1,)]: try: self.s.remove(v1) @@ -582,8 +591,6 @@ def test_ixor(self): else: self.assertNotIn(c, self.s) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_inplace_on_self(self): t = self.s.copy() t |= t @@ -640,10 +647,68 @@ def __le__(self, some_set): myset >= myobj self.assertTrue(myobj.le_called) - @unittest.skipUnless(hasattr(set, "test_c_api"), - 'C API test only available in a debug build') - def test_c_api(self): - self.assertEqual(set().test_c_api(), True) + def test_set_membership(self): + myfrozenset = frozenset(range(3)) + myset = {myfrozenset, "abc", 1} + self.assertIn(set(range(3)), myset) + self.assertNotIn(set(range(1)), myset) + myset.discard(set(range(3))) + self.assertEqual(myset, {"abc", 1}) + self.assertRaises(KeyError, myset.remove, set(range(1))) + self.assertRaises(KeyError, myset.remove, set(range(3))) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unhashable_element(self): + myset = {'a'} + elem = [1, 2, 3] + + def check_unhashable_element(): + msg = "cannot use 'list' as a set element (unhashable type: 'list')" + return self.assertRaisesRegex(TypeError, re.escape(msg)) + + with check_unhashable_element(): + elem in myset + with check_unhashable_element(): + myset.add(elem) + with check_unhashable_element(): + myset.discard(elem) + + # Only TypeError exception is overriden, + # other exceptions are left unchanged. + class HashError: + def __hash__(self): + raise KeyError('error') + + elem2 = HashError() + with self.assertRaises(KeyError): + elem2 in myset + with self.assertRaises(KeyError): + myset.add(elem2) + with self.assertRaises(KeyError): + myset.discard(elem2) + + def test_hash_collision_remove_add(self): + self.maxDiff = None + # There should be enough space, so all elements with unique hash + # will be placed in corresponding cells without collision. + n = 64 + elems = [CustomHash(h) for h in range(n)] + # Elements with hash collision. + a = CustomHash(n) + b = CustomHash(n) + elems += [a, b] + s = self.thetype(elems) + self.assertEqual(len(s), len(elems), s) + s.remove(a) + # "a" has been replaced with a dummy. + del elems[n] + self.assertEqual(len(s), len(elems), s) + self.assertEqual(s, set(elems)) + s.add(b) + # "b" should not replace the dummy. + self.assertEqual(len(s), len(elems), s) + self.assertEqual(s, set(elems)) + class SetSubclass(set): pass @@ -652,15 +717,37 @@ class TestSetSubclass(TestSet): thetype = SetSubclass basetype = set -class SetSubclassWithKeywordArgs(set): - def __init__(self, iterable=[], newarg=None): - set.__init__(self, iterable) - -class TestSetSubclassWithKeywordArgs(TestSet): - def test_keywords_in_subclass(self): - 'SF bug #1486663 -- this used to erroneously raise a TypeError' - SetSubclassWithKeywordArgs(newarg=1) + class subclass(set): + pass + u = subclass([1, 2]) + self.assertIs(type(u), subclass) + self.assertEqual(set(u), {1, 2}) + with self.assertRaises(TypeError): + subclass(sequence=()) + + class subclass_with_init(set): + def __init__(self, arg, newarg=None): + super().__init__(arg) + self.newarg = newarg + u = subclass_with_init([1, 2], newarg=3) + self.assertIs(type(u), subclass_with_init) + self.assertEqual(set(u), {1, 2}) + self.assertEqual(u.newarg, 3) + + class subclass_with_new(set): + def __new__(cls, arg, newarg=None): + self = super().__new__(cls, arg) + self.newarg = newarg + return self + u = subclass_with_new([1, 2]) + self.assertIs(type(u), subclass_with_new) + self.assertEqual(set(u), {1, 2}) + self.assertIsNone(u.newarg) + # disallow kwargs in __new__ only (https://bugs.python.org/issue43413#msg402000) + with self.assertRaises(TypeError): + subclass_with_new([1, 2], newarg=3) + class TestFrozenSet(TestJointOps, unittest.TestCase): thetype = frozenset @@ -707,8 +794,6 @@ def test_hash_caching(self): f = self.thetype('abcdcda') self.assertEqual(hash(f), hash(f)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_hash_effectiveness(self): n = 13 hashvalues = set() @@ -744,6 +829,34 @@ class TestFrozenSetSubclass(TestFrozenSet): thetype = FrozenSetSubclass basetype = frozenset + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_keywords_in_subclass(self): + class subclass(frozenset): + pass + u = subclass([1, 2]) + self.assertIs(type(u), subclass) + self.assertEqual(set(u), {1, 2}) + with self.assertRaises(TypeError): + subclass(sequence=()) + + class subclass_with_init(frozenset): + def __init__(self, arg, newarg=None): + self.newarg = newarg + u = subclass_with_init([1, 2], newarg=3) + self.assertIs(type(u), subclass_with_init) + self.assertEqual(set(u), {1, 2}) + self.assertEqual(u.newarg, 3) + + class subclass_with_new(frozenset): + def __new__(cls, arg, newarg=None): + self = super().__new__(cls, arg) + self.newarg = newarg + return self + u = subclass_with_new([1, 2], newarg=3) + self.assertIs(type(u), subclass_with_new) + self.assertEqual(set(u), {1, 2}) + self.assertEqual(u.newarg, 3) + def test_constructor_identity(self): s = self.thetype(range(3)) t = self.thetype(s) @@ -769,6 +882,25 @@ def test_singleton_empty_frozenset(self): # All empty frozenset subclass instances should have different ids self.assertEqual(len(set(map(id, efs))), len(efs)) + +class SetSubclassWithSlots(set): + __slots__ = ('x', 'y', '__dict__') + +class TestSetSubclassWithSlots(unittest.TestCase): + thetype = SetSubclassWithSlots + setUp = TestJointOps.setUp + test_pickling = TestJointOps.test_pickling + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling(self): + return super().test_pickling() + +class FrozenSetSubclassWithSlots(frozenset): + __slots__ = ('x', 'y', '__dict__') + +class TestFrozenSetSubclassWithSlots(TestSetSubclassWithSlots): + thetype = FrozenSetSubclassWithSlots + # Tests taken from test_sets.py ============================================= empty_set = set() @@ -783,8 +915,8 @@ def test_repr(self): def check_repr_against_values(self): text = repr(self.set) - self.assertTrue(text.startswith('{')) - self.assertTrue(text.endswith('}')) + self.assertStartsWith(text, '{') + self.assertEndsWith(text, '}') result = text[1:-1].split(', ') result.sort() @@ -965,8 +1097,7 @@ def test_repr(self): class TestBasicOpsMixedStringBytes(TestBasicOps, unittest.TestCase): def setUp(self): - self._warning_filters = warnings_helper.check_warnings() - self._warning_filters.__enter__() + self.enterContext(warnings_helper.check_warnings()) warnings.simplefilter('ignore', BytesWarning) self.case = "string and bytes set" self.values = ["a", "b", b"a", b"b"] @@ -974,9 +1105,6 @@ def setUp(self): self.dup = set(self.values) self.length = 4 - def tearDown(self): - self._warning_filters.__exit__(None, None, None) - def test_repr(self): self.check_repr_against_values() @@ -1764,6 +1892,7 @@ def test_iter_and_mutate(self): list(si) def test_merge_and_mutate(self): + # gh-141805 class X: def __hash__(self): return hash(0) @@ -1776,6 +1905,33 @@ def __eq__(self, o): s = {0} s.update(other) + def test_hash_collision_concurrent_add(self): + class X: + def __hash__(self): + return 0 + class Y: + flag = False + def __hash__(self): + return 0 + def __eq__(self, other): + if not self.flag: + self.flag = True + s.add(X()) + return self is other + + a = X() + s = set() + s.add(a) + s.add(X()) + s.remove(a) + # Now the set contains a dummy entry followed by an entry + # for an object with hash 0. + s.add(Y()) + # The following operations should not crash. + repr(s) + list(s) + set() | s + class TestOperationsMutating: """Regression test for bpo-46615""" diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index ecc4fffec0d..0bb02ef11f6 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -1,3 +1,10 @@ +import doctest +import traceback +import unittest + +from test.support import BrokenIter + + doctests = """ ########### Tests mostly copied from test_listcomps.py ############ @@ -144,24 +151,49 @@ """ +class SetComprehensionTest(unittest.TestCase): + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + def init_raises(): + try: + {x for x in BrokenIter(init_raises=True)} + except Exception as e: + return e + + def next_raises(): + try: + {x for x in BrokenIter(next_raises=True)} + except Exception as e: + return e + + def iter_raises(): + try: + {x for x in BrokenIter(iter_raises=True)} + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) __test__ = {'doctests' : doctests} -def test_main(verbose=None): - import sys - from test import support - from test import test_setcomps - support.run_doctest(test_setcomps, verbose) - - # verify reference counting - if verbose and hasattr(sys, "gettotalrefcount"): - import gc - counts = [None] * 5 - for i in range(len(counts)): - support.run_doctest(test_setcomps, verbose) - gc.collect() - counts[i] = sys.gettotalrefcount() - print(counts) +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests + if __name__ == "__main__": - test_main(verbose=True) + unittest.main() diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index ac25eee2e52..08c6562f2a2 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -1,7 +1,9 @@ import unittest +import dbm import shelve -import glob -from test import support +import pickle +import os + from test.support import os_helper from collections.abc import MutableMapping from test.test_dbm import dbm_iterator @@ -41,12 +43,8 @@ def copy(self): class TestCase(unittest.TestCase): - - fn = "shelftemp.db" - - def tearDown(self): - for f in glob.glob(self.fn+"*"): - os_helper.unlink(f) + dirname = os_helper.TESTFN + fn = os.path.join(os_helper.TESTFN, "shelftemp.db") def test_close(self): d1 = {} @@ -63,29 +61,34 @@ def test_close(self): else: self.fail('Closed shelf should not find a key') - def test_ascii_file_shelf(self): - s = shelve.open(self.fn, protocol=0) + def test_open_template(self, filename=None, protocol=None): + os.mkdir(self.dirname) + self.addCleanup(os_helper.rmtree, self.dirname) + s = shelve.open(filename=filename if filename is not None else self.fn, + protocol=protocol) try: s['key1'] = (1,2,3,4) self.assertEqual(s['key1'], (1,2,3,4)) finally: s.close() + def test_ascii_file_shelf(self): + self.test_open_template(protocol=0) + def test_binary_file_shelf(self): - s = shelve.open(self.fn, protocol=1) - try: - s['key1'] = (1,2,3,4) - self.assertEqual(s['key1'], (1,2,3,4)) - finally: - s.close() + self.test_open_template(protocol=1) def test_proto2_file_shelf(self): - s = shelve.open(self.fn, protocol=2) - try: - s['key1'] = (1,2,3,4) - self.assertEqual(s['key1'], (1,2,3,4)) - finally: - s.close() + self.test_open_template(protocol=2) + + def test_pathlib_path_file_shelf(self): + self.test_open_template(filename=os_helper.FakePath(self.fn)) + + def test_bytes_path_file_shelf(self): + self.test_open_template(filename=os.fsencode(self.fn)) + + def test_pathlib_bytes_path_file_shelf(self): + self.test_open_template(filename=os_helper.FakePath(os.fsencode(self.fn))) def test_in_memory_shelf(self): d1 = byteskeydict() @@ -160,65 +163,54 @@ def test_with(self): def test_default_protocol(self): with shelve.Shelf({}) as s: - self.assertEqual(s._protocol, 3) + self.assertEqual(s._protocol, pickle.DEFAULT_PROTOCOL) -from test import mapping_tests -class TestShelveBase(mapping_tests.BasicTestMappingProtocol): - fn = "shelftemp.db" - counter = 0 - def __init__(self, *args, **kw): - self._db = [] - mapping_tests.BasicTestMappingProtocol.__init__(self, *args, **kw) +class TestShelveBase: type2test = shelve.Shelf + def _reference(self): return {"key1":"value1", "key2":2, "key3":(1,2,3)} + + +class TestShelveInMemBase(TestShelveBase): def _empty_mapping(self): - if self._in_mem: - x= shelve.Shelf(byteskeydict(), **self._args) - else: - self.counter+=1 - x= shelve.open(self.fn+str(self.counter), **self._args) - self._db.append(x) + return shelve.Shelf(byteskeydict(), **self._args) + + +class TestShelveFileBase(TestShelveBase): + counter = 0 + + def _empty_mapping(self): + self.counter += 1 + x = shelve.open(self.base_path + str(self.counter), **self._args) + self.addCleanup(x.close) return x - def tearDown(self): - for db in self._db: - db.close() - self._db = [] - if not self._in_mem: - for f in glob.glob(self.fn+"*"): - os_helper.unlink(f) - -class TestAsciiFileShelve(TestShelveBase): - _args={'protocol':0} - _in_mem = False -class TestBinaryFileShelve(TestShelveBase): - _args={'protocol':1} - _in_mem = False -class TestProto2FileShelve(TestShelveBase): - _args={'protocol':2} - _in_mem = False -class TestAsciiMemShelve(TestShelveBase): - _args={'protocol':0} - _in_mem = True -class TestBinaryMemShelve(TestShelveBase): - _args={'protocol':1} - _in_mem = True -class TestProto2MemShelve(TestShelveBase): - _args={'protocol':2} - _in_mem = True - -def test_main(): - for module in dbm_iterator(): - support.run_unittest( - TestAsciiFileShelve, - TestBinaryFileShelve, - TestProto2FileShelve, - TestAsciiMemShelve, - TestBinaryMemShelve, - TestProto2MemShelve, - TestCase - ) + + def setUp(self): + dirname = os_helper.TESTFN + os.mkdir(dirname) + self.addCleanup(os_helper.rmtree, dirname) + self.base_path = os.path.join(dirname, "shelftemp.db") + self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) + dbm._defaultmod = self.dbm_mod + + +from test import mapping_tests + +for proto in range(pickle.HIGHEST_PROTOCOL + 1): + bases = (TestShelveInMemBase, mapping_tests.BasicTestMappingProtocol) + name = f'TestProto{proto}MemShelve' + globals()[name] = type(name, bases, + {'_args': {'protocol': proto}}) + bases = (TestShelveFileBase, mapping_tests.BasicTestMappingProtocol) + for dbm_mod in dbm_iterator(): + assert dbm_mod.__name__.startswith('dbm.') + suffix = dbm_mod.__name__[4:] + name = f'TestProto{proto}File_{suffix}Shelve' + globals()[name] = type(name, bases, + {'dbm_mod': dbm_mod, '_args': {'protocol': proto}}) + if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index c3d632a64c8..7c41432b82f 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -3,7 +3,8 @@ import shlex import string import unittest -from unittest import mock +from test.support import cpython_only +from test.support import import_helper # The original test data set was from shellwords, by Hartmut Goebel. @@ -162,19 +163,16 @@ def oldSplit(self, s): tok = lex.get_token() return ret - @mock.patch('sys.stdin', io.StringIO()) - def testSplitNoneDeprecation(self): - with self.assertWarns(DeprecationWarning): + def testSplitNone(self): + with self.assertRaises(ValueError): shlex.split(None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testSplitPosix(self): """Test data splitting with posix parser""" self.splitTest(self.posix_data, comments=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testCompat(self): """Test compatibility interface""" for i in range(len(self.data)): @@ -315,8 +313,7 @@ def testEmptyStringHandling(self): s = shlex.shlex("'')abc", punctuation_chars=True) self.assertEqual(list(s), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testUnicodeHandling(self): """Test punctuation_chars and whitespace_split handle unicode.""" ss = "\u2119\u01b4\u2602\u210c\u00f8\u1f24" @@ -336,6 +333,7 @@ def testQuote(self): unsafe = '"`$\\!' + unicode_sample self.assertEqual(shlex.quote(''), "''") + self.assertEqual(shlex.quote(None), "''") self.assertEqual(shlex.quote(safeunquoted), safeunquoted) self.assertEqual(shlex.quote('test file name'), "'test file name'") for u in unsafe: @@ -344,6 +342,8 @@ def testQuote(self): for u in unsafe: self.assertEqual(shlex.quote("test%s'name'" % u), "'test%s'\"'\"'name'\"'\"''" % u) + self.assertRaises(TypeError, shlex.quote, 42) + self.assertRaises(TypeError, shlex.quote, b"abc") def testJoin(self): for split_command, command in [ @@ -356,8 +356,7 @@ def testJoin(self): joined = shlex.join(split_command) self.assertEqual(joined, command) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testJoinRoundtrip(self): all_data = self.data + self.posix_data for command, *split_command in all_data: @@ -373,6 +372,10 @@ def testPunctuationCharsReadOnly(self): with self.assertRaises(AttributeError): shlex_instance.punctuation_chars = False + @cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports('shlex', {'collections', 're', 'os'}) + # Allow this test to be used with old shlex.py if not getattr(shlex, "split", None): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index fba98972dd7..62c80aab4b3 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -10,7 +10,6 @@ import os.path import errno import functools -import pathlib import subprocess import random import string @@ -34,7 +33,10 @@ from test.support.os_helper import TESTFN, FakePath TESTFN2 = TESTFN + "2" +TESTFN_SRC = TESTFN + "_SRC" +TESTFN_DST = TESTFN + "_DST" MACOS = sys.platform.startswith("darwin") +SOLARIS = sys.platform.startswith("sunos") AIX = sys.platform[:3] == 'aix' try: import grp @@ -48,6 +50,9 @@ except ImportError: _winapi = None +no_chdir = unittest.mock.patch('os.chdir', + side_effect=AssertionError("shouldn't call os.chdir()")) + def _fake_rename(*args, **kwargs): # Pretend the destination path is on a different filesystem. raise OSError(getattr(errno, 'EXDEV', 18), "Invalid cross-device link") @@ -63,16 +68,17 @@ def wrap(*args, **kwargs): os.rename = builtin_rename return wrap -def write_file(path, content, binary=False): +def create_file(path, content=b''): """Write *content* to a file located at *path*. If *path* is a tuple instead of a string, os.path.join will be used to - make a path. If *binary* is true, the file will be opened in binary - mode. + make a path. """ if isinstance(path, tuple): path = os.path.join(*path) - with open(path, 'wb' if binary else 'w') as fp: + if isinstance(content, str): + content = content.encode() + with open(path, 'xb') as fp: fp.write(content) def write_test_file(path, size): @@ -102,7 +108,9 @@ def read_file(path, binary=False): """ if isinstance(path, tuple): path = os.path.join(*path) - with open(path, 'rb' if binary else 'r') as fp: + mode = 'rb' if binary else 'r' + encoding = None if binary else "utf-8" + with open(path, mode, encoding=encoding) as fp: return fp.read() def rlistdir(path): @@ -124,12 +132,12 @@ def supports_file2file_sendfile(): srcname = None dstname = None try: - with tempfile.NamedTemporaryFile("wb", delete=False) as f: + with tempfile.NamedTemporaryFile("wb", dir=os.getcwd(), delete=False) as f: srcname = f.name f.write(b"0123456789") with open(srcname, "rb") as src: - with tempfile.NamedTemporaryFile("wb", delete=False) as dst: + with tempfile.NamedTemporaryFile("wb", dir=os.getcwd(), delete=False) as dst: dstname = dst.name infd = src.fileno() outfd = dst.fileno() @@ -160,42 +168,32 @@ def _maxdataOK(): else: return True -class TestShutil(unittest.TestCase): - - def setUp(self): - super(TestShutil, self).setUp() - self.tempdirs = [] - - def tearDown(self): - super(TestShutil, self).tearDown() - while self.tempdirs: - d = self.tempdirs.pop() - shutil.rmtree(d, os.name in ('nt', 'cygwin')) +class BaseTest: - def mkdtemp(self): + def mkdtemp(self, prefix=None): """Create a temporary directory that will be cleaned up. Returns the path of the directory. """ - basedir = None - if sys.platform == "win32": - basedir = os.path.realpath(os.getcwd()) - d = tempfile.mkdtemp(dir=basedir) - self.tempdirs.append(d) + d = tempfile.mkdtemp(prefix=prefix, dir=os.getcwd()) + self.addCleanup(os_helper.rmtree, d) return d + +class TestRmTree(BaseTest, unittest.TestCase): + def test_rmtree_works_on_bytes(self): tmp = self.mkdtemp() victim = os.path.join(tmp, 'killme') os.mkdir(victim) - write_file(os.path.join(victim, 'somefile'), 'foo') + create_file(os.path.join(victim, 'somefile'), 'foo') victim = os.fsencode(victim) self.assertIsInstance(victim, bytes) shutil.rmtree(victim) @os_helper.skip_unless_symlink - def test_rmtree_fails_on_symlink(self): + def test_rmtree_fails_on_symlink_onerror(self): tmp = self.mkdtemp() dir_ = os.path.join(tmp, 'dir') os.mkdir(dir_) @@ -213,6 +211,25 @@ def onerror(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2][1], OSError) + @os_helper.skip_unless_symlink + def test_rmtree_fails_on_symlink_onexc(self): + tmp = self.mkdtemp() + dir_ = os.path.join(tmp, 'dir') + os.mkdir(dir_) + link = os.path.join(tmp, 'link') + os.symlink(dir_, link) + self.assertRaises(OSError, shutil.rmtree, link) + self.assertTrue(os.path.exists(dir_)) + self.assertTrue(os.path.lexists(link)) + errors = [] + def onexc(*args): + errors.append(args) + shutil.rmtree(link, onexc=onexc) + self.assertEqual(len(errors), 1) + self.assertIs(errors[0][0], os.path.islink) + self.assertEqual(errors[0][1], link) + self.assertIsInstance(errors[0][2], OSError) + @os_helper.skip_unless_symlink def test_rmtree_works_on_symlinks(self): tmp = self.mkdtemp() @@ -222,7 +239,7 @@ def test_rmtree_works_on_symlinks(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') os.symlink(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -236,12 +253,13 @@ def test_rmtree_works_on_symlinks(self): self.assertTrue(os.path.exists(file1)) @unittest.skipUnless(_winapi, 'only relevant on Windows') - def test_rmtree_fails_on_junctions(self): + def test_rmtree_fails_on_junctions_onerror(self): tmp = self.mkdtemp() dir_ = os.path.join(tmp, 'dir') os.mkdir(dir_) link = os.path.join(tmp, 'link') _winapi.CreateJunction(dir_, link) + self.addCleanup(os_helper.unlink, link) self.assertRaises(OSError, shutil.rmtree, link) self.assertTrue(os.path.exists(dir_)) self.assertTrue(os.path.lexists(link)) @@ -254,6 +272,26 @@ def onerror(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2][1], OSError) + @unittest.skipUnless(_winapi, 'only relevant on Windows') + def test_rmtree_fails_on_junctions_onexc(self): + tmp = self.mkdtemp() + dir_ = os.path.join(tmp, 'dir') + os.mkdir(dir_) + link = os.path.join(tmp, 'link') + _winapi.CreateJunction(dir_, link) + self.addCleanup(os_helper.unlink, link) + self.assertRaises(OSError, shutil.rmtree, link) + self.assertTrue(os.path.exists(dir_)) + self.assertTrue(os.path.lexists(link)) + errors = [] + def onexc(*args): + errors.append(args) + shutil.rmtree(link, onexc=onexc) + self.assertEqual(len(errors), 1) + self.assertIs(errors[0][0], os.path.islink) + self.assertEqual(errors[0][1], link) + self.assertIsInstance(errors[0][2], OSError) + @unittest.skipUnless(_winapi, 'only relevant on Windows') def test_rmtree_works_on_junctions(self): tmp = self.mkdtemp() @@ -263,7 +301,7 @@ def test_rmtree_works_on_junctions(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') _winapi.CreateJunction(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -276,29 +314,37 @@ def test_rmtree_works_on_junctions(self): self.assertTrue(os.path.exists(dir3)) self.assertTrue(os.path.exists(file1)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_rmtree_errors(self): # filename is guaranteed not to exist - filename = tempfile.mktemp() + filename = tempfile.mktemp(dir=self.mkdtemp()) self.assertRaises(FileNotFoundError, shutil.rmtree, filename) # test that ignore_errors option is honored shutil.rmtree(filename, ignore_errors=True) # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") + create_file(filename) with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) - # The reason for this rather odd construct is that Windows sprinkles - # a \*.* at the end of file names. But only sometimes on some buildbots - possible_args = [filename, os.path.join(filename, '*.*')] - self.assertIn(cm.exception.filename, possible_args) + self.assertEqual(cm.exception.filename, filename) self.assertTrue(os.path.exists(filename)) # test that ignore_errors option is honored shutil.rmtree(filename, ignore_errors=True) self.assertTrue(os.path.exists(filename)) + + self.assertRaises(TypeError, shutil.rmtree, None) + self.assertRaises(TypeError, shutil.rmtree, None, ignore_errors=True) + exc = TypeError if shutil.rmtree.avoids_symlink_attacks else NotImplementedError + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid') + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid', ignore_errors=True) + + def test_rmtree_errors_onerror(self): + tmpdir = self.mkdtemp() + filename = os.path.join(tmpdir, "tstfile") + create_file(filename) errors = [] def onerror(*args): errors.append(args) @@ -307,17 +353,34 @@ def onerror(*args): self.assertIs(errors[0][0], os.scandir) self.assertEqual(errors[0][1], filename) self.assertIsInstance(errors[0][2][1], NotADirectoryError) - self.assertIn(errors[0][2][1].filename, possible_args) + self.assertEqual(errors[0][2][1].filename, filename) self.assertIs(errors[1][0], os.rmdir) self.assertEqual(errors[1][1], filename) self.assertIsInstance(errors[1][2][1], NotADirectoryError) - self.assertIn(errors[1][2][1].filename, possible_args) + self.assertEqual(errors[1][2][1].filename, filename) + def test_rmtree_errors_onexc(self): + tmpdir = self.mkdtemp() + filename = os.path.join(tmpdir, "tstfile") + create_file(filename) + errors = [] + def onexc(*args): + errors.append(args) + shutil.rmtree(filename, onexc=onexc) + self.assertEqual(len(errors), 2) + self.assertIs(errors[0][0], os.scandir) + self.assertEqual(errors[0][1], filename) + self.assertIsInstance(errors[0][2], NotADirectoryError) + self.assertEqual(errors[0][2].filename, filename) + self.assertIs(errors[1][0], os.rmdir) + self.assertEqual(errors[1][1], filename) + self.assertIsInstance(errors[1][2], NotADirectoryError) + self.assertEqual(errors[1][2].filename, filename) @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") - @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, - "This test can't be run reliably as root (issue #1076467).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod def test_on_error(self): self.errorState = 0 os.mkdir(TESTFN) @@ -364,14 +427,112 @@ def check_args_to_onerror(self, func, arg, exc): else: self.assertIs(func, os.listdir) self.assertIn(arg, [TESTFN, self.child_dir_path]) - self.assertTrue(issubclass(exc[0], OSError)) + self.assertIsSubclass(exc[0], OSError) + self.errorState += 1 + else: + self.assertEqual(func, os.rmdir) + self.assertEqual(arg, TESTFN) + self.assertIsSubclass(exc[0], OSError) + self.errorState = 3 + + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_on_exc(self): + self.errorState = 0 + os.mkdir(TESTFN) + self.addCleanup(shutil.rmtree, TESTFN) + + self.child_file_path = os.path.join(TESTFN, 'a') + self.child_dir_path = os.path.join(TESTFN, 'b') + os_helper.create_empty_file(self.child_file_path) + os.mkdir(self.child_dir_path) + old_dir_mode = os.stat(TESTFN).st_mode + old_child_file_mode = os.stat(self.child_file_path).st_mode + old_child_dir_mode = os.stat(self.child_dir_path).st_mode + # Make unwritable. + new_mode = stat.S_IREAD|stat.S_IEXEC + os.chmod(self.child_file_path, new_mode) + os.chmod(self.child_dir_path, new_mode) + os.chmod(TESTFN, new_mode) + + self.addCleanup(os.chmod, TESTFN, old_dir_mode) + self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode) + self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode) + + shutil.rmtree(TESTFN, onexc=self.check_args_to_onexc) + # Test whether onexc has actually been called. + self.assertEqual(self.errorState, 3, + "Expected call to onexc function did not happen.") + + def check_args_to_onexc(self, func, arg, exc): + # test_rmtree_errors deliberately runs rmtree + # on a directory that is chmod 500, which will fail. + # This function is run when shutil.rmtree fails. + # 99.9% of the time it initially fails to remove + # a file in the directory, so the first time through + # func is os.remove. + # However, some Linux machines running ZFS on + # FUSE experienced a failure earlier in the process + # at os.listdir. The first failure may legally + # be either. + if self.errorState < 2: + if func is os.unlink: + self.assertEqual(arg, self.child_file_path) + elif func is os.rmdir: + self.assertEqual(arg, self.child_dir_path) + else: + self.assertIs(func, os.listdir) + self.assertIn(arg, [TESTFN, self.child_dir_path]) + self.assertTrue(isinstance(exc, OSError)) self.errorState += 1 else: self.assertEqual(func, os.rmdir) self.assertEqual(arg, TESTFN) - self.assertTrue(issubclass(exc[0], OSError)) + self.assertTrue(isinstance(exc, OSError)) self.errorState = 3 + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_both_onerror_and_onexc(self): + onerror_called = False + onexc_called = False + + def onerror(*args): + nonlocal onerror_called + onerror_called = True + + def onexc(*args): + nonlocal onexc_called + onexc_called = True + + os.mkdir(TESTFN) + self.addCleanup(shutil.rmtree, TESTFN) + + self.child_file_path = os.path.join(TESTFN, 'a') + self.child_dir_path = os.path.join(TESTFN, 'b') + os_helper.create_empty_file(self.child_file_path) + os.mkdir(self.child_dir_path) + old_dir_mode = os.stat(TESTFN).st_mode + old_child_file_mode = os.stat(self.child_file_path).st_mode + old_child_dir_mode = os.stat(self.child_dir_path).st_mode + # Make unwritable. + new_mode = stat.S_IREAD|stat.S_IEXEC + os.chmod(self.child_file_path, new_mode) + os.chmod(self.child_dir_path, new_mode) + os.chmod(TESTFN, new_mode) + + self.addCleanup(os.chmod, TESTFN, old_dir_mode) + self.addCleanup(os.chmod, self.child_file_path, old_child_file_mode) + self.addCleanup(os.chmod, self.child_dir_path, old_child_dir_mode) + + shutil.rmtree(TESTFN, onerror=onerror, onexc=onexc) + self.assertTrue(onexc_called) + self.assertFalse(onerror_called) + def test_rmtree_does_not_choke_on_failing_lstat(self): try: orig_lstat = os.lstat @@ -383,373 +544,219 @@ def raiser(fn, *args, **kwargs): os.lstat = raiser os.mkdir(TESTFN) - write_file((TESTFN, 'foo'), 'foo') + create_file((TESTFN, 'foo'), 'foo') shutil.rmtree(TESTFN) finally: os.lstat = orig_lstat - @os_helper.skip_unless_symlink - def test_copymode_follow_symlinks(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') - os.symlink(src, src_link) - os.symlink(dst, dst_link) - os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) - # file to file - os.chmod(dst, stat.S_IRWXO) - self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - shutil.copymode(src, dst) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # On Windows, os.chmod does not follow symlinks (issue #15411) - if os.name != 'nt': - # follow src link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow dst link - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # follow both links - os.chmod(dst, stat.S_IRWXO) - shutil.copymode(src_link, dst_link) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + def test_rmtree_uses_safe_fd_version_if_available(self): + _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= + os.supports_dir_fd and + os.listdir in os.supports_fd and + os.stat in os.supports_follow_symlinks) + if _use_fd_functions: + self.assertTrue(shutil.rmtree.avoids_symlink_attacks) + tmp_dir = self.mkdtemp() + d = os.path.join(tmp_dir, 'a') + os.mkdir(d) + try: + real_open = os.open + class Called(Exception): pass + def _raiser(*args, **kwargs): + raise Called + os.open = _raiser + self.assertRaises(Called, shutil.rmtree, d) + finally: + os.open = real_open + else: + self.assertFalse(shutil.rmtree.avoids_symlink_attacks) - @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod') - @os_helper.skip_unless_symlink - def test_copymode_symlink_to_symlink(self): + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "requires safe rmtree") + def test_rmtree_fails_on_close(self): + # Test that the error handler is called for failed os.close() and that + # os.close() is only called once for a file descriptor. + tmp = self.mkdtemp() + dir1 = os.path.join(tmp, 'dir1') + os.mkdir(dir1) + dir2 = os.path.join(dir1, 'dir2') + os.mkdir(dir2) + def close(fd): + orig_close(fd) + nonlocal close_count + close_count += 1 + raise OSError + + close_count = 0 + with support.swap_attr(os, 'close', close) as orig_close: + with self.assertRaises(OSError): + shutil.rmtree(dir1) + self.assertTrue(os.path.isdir(dir2)) + self.assertEqual(close_count, 2) + + close_count = 0 + errors = [] + def onexc(*args): + errors.append(args) + with support.swap_attr(os, 'close', close) as orig_close: + shutil.rmtree(dir1, onexc=onexc) + self.assertEqual(len(errors), 2) + self.assertIs(errors[0][0], close) + self.assertEqual(errors[0][1], dir2) + self.assertIs(errors[1][0], close) + self.assertEqual(errors[1][1], dir1) + self.assertEqual(close_count, 2) + + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, "dir_fd is not supported") + def test_rmtree_with_dir_fd(self): tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') - os.symlink(src, src_link) - os.symlink(dst, dst_link) - os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) - os.chmod(dst, stat.S_IRWXU) - os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) - # link to link - os.lchmod(dst_link, stat.S_IRWXO) - shutil.copymode(src_link, dst_link, follow_symlinks=False) - self.assertEqual(os.lstat(src_link).st_mode, - os.lstat(dst_link).st_mode) - self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # src link - use chmod - os.lchmod(dst_link, stat.S_IRWXO) - shutil.copymode(src_link, dst, follow_symlinks=False) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - # dst link - use chmod - os.lchmod(dst_link, stat.S_IRWXO) - shutil.copymode(src, dst_link, follow_symlinks=False) - self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - - @unittest.skipIf(hasattr(os, 'lchmod'), 'requires os.lchmod to be missing') - @os_helper.skip_unless_symlink - def test_copymode_symlink_to_symlink_wo_lchmod(self): + victim = 'killme' + fullname = os.path.join(tmp_dir, victim) + dir_fd = os.open(tmp_dir, os.O_RDONLY) + self.addCleanup(os.close, dir_fd) + os.mkdir(fullname) + os.mkdir(os.path.join(fullname, 'subdir')) + create_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') + self.assertTrue(os.path.exists(fullname)) + shutil.rmtree(victim, dir_fd=dir_fd) + self.assertFalse(os.path.exists(fullname)) + + @unittest.skipIf(shutil.rmtree.avoids_symlink_attacks, "dir_fd is supported") + def test_rmtree_with_dir_fd_unsupported(self): tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') - os.symlink(src, src_link) - os.symlink(dst, dst_link) - shutil.copymode(src_link, dst_link, follow_symlinks=False) # silent fail + with self.assertRaises(NotImplementedError): + shutil.rmtree(tmp_dir, dir_fd=0) + self.assertTrue(os.path.exists(tmp_dir)) - @os_helper.skip_unless_symlink - def test_copystat_symlinks(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - dst_link = os.path.join(tmp_dir, 'qux') - write_file(src, 'foo') - src_stat = os.stat(src) - os.utime(src, (src_stat.st_atime, - src_stat.st_mtime - 42.0)) # ensure different mtimes - write_file(dst, 'bar') - self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime) - os.symlink(src, src_link) - os.symlink(dst, dst_link) - if hasattr(os, 'lchmod'): - os.lchmod(src_link, stat.S_IRWXO) - if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): - os.lchflags(src_link, stat.UF_NODUMP) - src_link_stat = os.lstat(src_link) - # follow - if hasattr(os, 'lchmod'): - shutil.copystat(src_link, dst_link, follow_symlinks=True) - self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode) - # don't follow - shutil.copystat(src_link, dst_link, follow_symlinks=False) - dst_link_stat = os.lstat(dst_link) - if os.utime in os.supports_follow_symlinks: - for attr in 'st_atime', 'st_mtime': - # The modification times may be truncated in the new file. - self.assertLessEqual(getattr(src_link_stat, attr), - getattr(dst_link_stat, attr) + 1) - if hasattr(os, 'lchmod'): - self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode) - if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): - self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags) - # tell to follow but dst is not a link - shutil.copystat(src_link, dst, follow_symlinks=False) - self.assertTrue(abs(os.stat(src).st_mtime - os.stat(dst).st_mtime) < - 00000.1) + def test_rmtree_dont_delete_file(self): + # When called on a file instead of a directory, don't delete it. + handle, path = tempfile.mkstemp(dir=self.mkdtemp()) + os.close(handle) + self.assertRaises(NotADirectoryError, shutil.rmtree, path) + os.remove(path) - @unittest.skipUnless(hasattr(os, 'chflags') and - hasattr(errno, 'EOPNOTSUPP') and - hasattr(errno, 'ENOTSUP'), - "requires os.chflags, EOPNOTSUPP & ENOTSUP") - def test_copystat_handles_harmless_chflags_errors(self): - tmpdir = self.mkdtemp() - file1 = os.path.join(tmpdir, 'file1') - file2 = os.path.join(tmpdir, 'file2') - write_file(file1, 'xxx') - write_file(file2, 'xxx') - - def make_chflags_raiser(err): - ex = OSError() - - def _chflags_raiser(path, flags, *, follow_symlinks=True): - ex.errno = err - raise ex - return _chflags_raiser - old_chflags = os.chflags + @os_helper.skip_unless_symlink + def test_rmtree_on_symlink(self): + # bug 1669. + os.mkdir(TESTFN) try: - for err in errno.EOPNOTSUPP, errno.ENOTSUP: - os.chflags = make_chflags_raiser(err) - shutil.copystat(file1, file2) - # assert others errors break it - os.chflags = make_chflags_raiser(errno.EOPNOTSUPP + errno.ENOTSUP) - self.assertRaises(OSError, shutil.copystat, file1, file2) + src = os.path.join(TESTFN, 'cheese') + dst = os.path.join(TESTFN, 'shop') + os.mkdir(src) + os.symlink(src, dst) + self.assertRaises(OSError, shutil.rmtree, dst) + shutil.rmtree(dst, ignore_errors=True) finally: - os.chflags = old_chflags - - @os_helper.skip_unless_xattr - def test_copyxattr(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - write_file(src, 'foo') - dst = os.path.join(tmp_dir, 'bar') - write_file(dst, 'bar') - - # no xattr == no problem - shutil._copyxattr(src, dst) - # common case - os.setxattr(src, 'user.foo', b'42') - os.setxattr(src, 'user.bar', b'43') - shutil._copyxattr(src, dst) - self.assertEqual(sorted(os.listxattr(src)), sorted(os.listxattr(dst))) - self.assertEqual( - os.getxattr(src, 'user.foo'), - os.getxattr(dst, 'user.foo')) - # check errors don't affect other attrs - os.remove(dst) - write_file(dst, 'bar') - os_error = OSError(errno.EPERM, 'EPERM') + shutil.rmtree(TESTFN, ignore_errors=True) - def _raise_on_user_foo(fname, attr, val, **kwargs): - if attr == 'user.foo': - raise os_error - else: - orig_setxattr(fname, attr, val, **kwargs) + @unittest.skipUnless(_winapi, 'only relevant on Windows') + def test_rmtree_on_junction(self): + os.mkdir(TESTFN) try: - orig_setxattr = os.setxattr - os.setxattr = _raise_on_user_foo - shutil._copyxattr(src, dst) - self.assertIn('user.bar', os.listxattr(dst)) + src = os.path.join(TESTFN, 'cheese') + dst = os.path.join(TESTFN, 'shop') + os.mkdir(src) + create_file(os.path.join(src, 'spam')) + _winapi.CreateJunction(src, dst) + self.assertRaises(OSError, shutil.rmtree, dst) + shutil.rmtree(dst, ignore_errors=True) finally: - os.setxattr = orig_setxattr - # the source filesystem not supporting xattrs should be ok, too. - def _raise_on_src(fname, *, follow_symlinks=True): - if fname == src: - raise OSError(errno.ENOTSUP, 'Operation not supported') - return orig_listxattr(fname, follow_symlinks=follow_symlinks) + shutil.rmtree(TESTFN, ignore_errors=True) + + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_rmtree_on_named_pipe(self): + os.mkfifo(TESTFN) try: - orig_listxattr = os.listxattr - os.listxattr = _raise_on_src - shutil._copyxattr(src, dst) + with self.assertRaises(NotADirectoryError): + shutil.rmtree(TESTFN) + self.assertTrue(os.path.exists(TESTFN)) finally: - os.listxattr = orig_listxattr + os.unlink(TESTFN) - # test that shutil.copystat copies xattrs - src = os.path.join(tmp_dir, 'the_original') - srcro = os.path.join(tmp_dir, 'the_original_ro') - write_file(src, src) - write_file(srcro, srcro) - os.setxattr(src, 'user.the_value', b'fiddly') - os.setxattr(srcro, 'user.the_value', b'fiddly') - os.chmod(srcro, 0o444) - dst = os.path.join(tmp_dir, 'the_copy') - dstro = os.path.join(tmp_dir, 'the_copy_ro') - write_file(dst, dst) - write_file(dstro, dstro) - shutil.copystat(src, dst) - shutil.copystat(srcro, dstro) - self.assertEqual(os.getxattr(dst, 'user.the_value'), b'fiddly') - self.assertEqual(os.getxattr(dstro, 'user.the_value'), b'fiddly') + os.mkdir(TESTFN) + os.mkfifo(os.path.join(TESTFN, 'mypipe')) + shutil.rmtree(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) - @os_helper.skip_unless_symlink - @os_helper.skip_unless_xattr - @unittest.skipUnless(hasattr(os, 'geteuid') and os.geteuid() == 0, - 'root privileges required') - def test_copyxattr_symlinks(self): - # On Linux, it's only possible to access non-user xattr for symlinks; - # which in turn require root privileges. This test should be expanded - # as soon as other platforms gain support for extended attributes. - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') - os.symlink(src, src_link) - os.setxattr(src, 'trusted.foo', b'42') - os.setxattr(src_link, 'trusted.foo', b'43', follow_symlinks=False) - dst = os.path.join(tmp_dir, 'bar') - dst_link = os.path.join(tmp_dir, 'qux') - write_file(dst, 'bar') - os.symlink(dst, dst_link) - shutil._copyxattr(src_link, dst_link, follow_symlinks=False) - self.assertEqual(os.getxattr(dst_link, 'trusted.foo', follow_symlinks=False), b'43') - self.assertRaises(OSError, os.getxattr, dst, 'trusted.foo') - shutil._copyxattr(src_link, dst, follow_symlinks=False) - self.assertEqual(os.getxattr(dst, 'trusted.foo'), b'43') + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_rmtree_deleted_race_condition(self): + # bpo-37260 + # + # Test that a file or a directory deleted after it is enumerated + # by scandir() but before unlink() or rmdr() is called doesn't + # generate any errors. + def _onexc(fn, path, exc): + assert fn in (os.rmdir, os.unlink) + if not isinstance(exc, PermissionError): + raise + # Make the parent and the children writeable. + for p, mode in zip(paths, old_modes): + os.chmod(p, mode) + # Remove other dirs except one. + keep = next(p for p in dirs if p != path) + for p in dirs: + if p != keep: + os.rmdir(p) + # Remove other files except one. + keep = next(p for p in files if p != path) + for p in files: + if p != keep: + os.unlink(p) - @os_helper.skip_unless_symlink - def test_copy_symlinks(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') - os.symlink(src, src_link) - if hasattr(os, 'lchmod'): - os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) - # don't follow - shutil.copy(src_link, dst, follow_symlinks=True) - self.assertFalse(os.path.islink(dst)) - self.assertEqual(read_file(src), read_file(dst)) - os.remove(dst) - # follow - shutil.copy(src_link, dst, follow_symlinks=False) - self.assertTrue(os.path.islink(dst)) - self.assertEqual(os.readlink(dst), os.readlink(src_link)) - if hasattr(os, 'lchmod'): - self.assertEqual(os.lstat(src_link).st_mode, - os.lstat(dst).st_mode) + os.mkdir(TESTFN) + paths = [TESTFN] + [os.path.join(TESTFN, f'child{i}') + for i in range(6)] + dirs = paths[1::2] + files = paths[2::2] + for path in dirs: + os.mkdir(path) + for path in files: + create_file(path) + + old_modes = [os.stat(path).st_mode for path in paths] + + # Make the parent and the children non-writeable. + new_mode = stat.S_IREAD|stat.S_IEXEC + for path in reversed(paths): + os.chmod(path, new_mode) - @os_helper.skip_unless_symlink - def test_copy2_symlinks(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') - os.symlink(src, src_link) - if hasattr(os, 'lchmod'): - os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) - if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): - os.lchflags(src_link, stat.UF_NODUMP) - src_stat = os.stat(src) - src_link_stat = os.lstat(src_link) - # follow - shutil.copy2(src_link, dst, follow_symlinks=True) - self.assertFalse(os.path.islink(dst)) - self.assertEqual(read_file(src), read_file(dst)) - os.remove(dst) - # don't follow - shutil.copy2(src_link, dst, follow_symlinks=False) - self.assertTrue(os.path.islink(dst)) - self.assertEqual(os.readlink(dst), os.readlink(src_link)) - dst_stat = os.lstat(dst) - if os.utime in os.supports_follow_symlinks: - for attr in 'st_atime', 'st_mtime': - # The modification times may be truncated in the new file. - self.assertLessEqual(getattr(src_link_stat, attr), - getattr(dst_stat, attr) + 1) - if hasattr(os, 'lchmod'): - self.assertEqual(src_link_stat.st_mode, dst_stat.st_mode) - self.assertNotEqual(src_stat.st_mode, dst_stat.st_mode) - if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): - self.assertEqual(src_link_stat.st_flags, dst_stat.st_flags) + try: + shutil.rmtree(TESTFN, onexc=_onexc) + except: + # Test failed, so cleanup artifacts. + for path, mode in zip(paths, old_modes): + try: + os.chmod(path, mode) + except OSError: + pass + shutil.rmtree(TESTFN) + raise - @os_helper.skip_unless_xattr - def test_copy2_xattr(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'foo') - dst = os.path.join(tmp_dir, 'bar') - write_file(src, 'foo') - os.setxattr(src, 'user.foo', b'42') - shutil.copy2(src, dst) - self.assertEqual( - os.getxattr(src, 'user.foo'), - os.getxattr(dst, 'user.foo')) - os.remove(dst) + def test_rmtree_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = os.path.join(TESTFN, *(['d'] * directory_depth)) + os.makedirs(base) - @os_helper.skip_unless_symlink - def test_copyfile_symlinks(self): - tmp_dir = self.mkdtemp() - src = os.path.join(tmp_dir, 'src') - dst = os.path.join(tmp_dir, 'dst') - dst_link = os.path.join(tmp_dir, 'dst_link') - link = os.path.join(tmp_dir, 'link') - write_file(src, 'foo') - os.symlink(src, link) - # don't follow - shutil.copyfile(link, dst_link, follow_symlinks=False) - self.assertTrue(os.path.islink(dst_link)) - self.assertEqual(os.readlink(link), os.readlink(dst_link)) - # follow - shutil.copyfile(link, dst) - self.assertFalse(os.path.islink(dst)) + with support.infinite_recursion(recursion_limit): + shutil.rmtree(TESTFN) - def test_rmtree_uses_safe_fd_version_if_available(self): - _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= - os.supports_dir_fd and - os.listdir in os.supports_fd and - os.stat in os.supports_follow_symlinks) - if _use_fd_functions: - self.assertTrue(shutil._use_fd_functions) - self.assertTrue(shutil.rmtree.avoids_symlink_attacks) - tmp_dir = self.mkdtemp() - d = os.path.join(tmp_dir, 'a') - os.mkdir(d) - try: - real_rmtree = shutil._rmtree_safe_fd - class Called(Exception): pass - def _raiser(*args, **kwargs): - raise Called - shutil._rmtree_safe_fd = _raiser - self.assertRaises(Called, shutil.rmtree, d) - finally: - shutil._rmtree_safe_fd = real_rmtree - else: - self.assertFalse(shutil._use_fd_functions) - self.assertFalse(shutil.rmtree.avoids_symlink_attacks) - def test_rmtree_dont_delete_file(self): - # When called on a file instead of a directory, don't delete it. - handle, path = tempfile.mkstemp() - os.close(handle) - self.assertRaises(NotADirectoryError, shutil.rmtree, path) - os.remove(path) +class TestCopyTree(BaseTest, unittest.TestCase): def test_copytree_simple(self): - src_dir = tempfile.mkdtemp() - dst_dir = os.path.join(tempfile.mkdtemp(), 'destination') + src_dir = self.mkdtemp() + dst_dir = os.path.join(self.mkdtemp(), 'destination') self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') shutil.copytree(src_dir, dst_dir) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) @@ -762,16 +769,16 @@ def test_copytree_simple(self): self.assertEqual(actual, '456') def test_copytree_dirs_exist_ok(self): - src_dir = tempfile.mkdtemp() - dst_dir = tempfile.mkdtemp() + src_dir = self.mkdtemp() + dst_dir = self.mkdtemp() self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, dst_dir) - write_file((src_dir, 'nonexisting.txt'), '123') + create_file((src_dir, 'nonexisting.txt'), '123') os.mkdir(os.path.join(src_dir, 'existing_dir')) os.mkdir(os.path.join(dst_dir, 'existing_dir')) - write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') - write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') + create_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') + create_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt'))) @@ -794,7 +801,7 @@ def test_copytree_symlinks(self): sub_dir = os.path.join(src_dir, 'sub') os.mkdir(src_dir) os.mkdir(sub_dir) - write_file((src_dir, 'file.txt'), 'foo') + create_file((src_dir, 'file.txt'), 'foo') src_link = os.path.join(sub_dir, 'link') dst_link = os.path.join(dst_dir, 'sub/link') os.symlink(os.path.join(src_dir, 'file.txt'), @@ -822,19 +829,19 @@ def test_copytree_with_exclude(self): # creating data join = os.path.join exists = os.path.exists - src_dir = tempfile.mkdtemp() + src_dir = self.mkdtemp() try: - dst_dir = join(tempfile.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') - write_file((src_dir, 'test.tmp'), '123') + dst_dir = join(self.mkdtemp(), 'destination') + create_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.tmp'), '123') os.mkdir(join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2')) - write_file((src_dir, 'test_dir2', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2', 'subdir')) os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') - write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') + create_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') # testing glob-like patterns try: @@ -893,12 +900,12 @@ def test_copytree_arg_types_of_ignore(self): os.mkdir(join(src_dir)) os.mkdir(join(src_dir, 'test_dir')) os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) - write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') - invokations = [] + invocations = [] def _ignore(src, names): - invokations.append(src) + invocations.append(src) self.assertIsInstance(src, str) self.assertIsInstance(names, list) self.assertEqual(len(names), len(set(names))) @@ -912,7 +919,7 @@ def _ignore(src, names): 'test.txt'))) dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) + shutil.copytree(FakePath(src_dir), dst_dir, ignore=_ignore) self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) @@ -923,21 +930,22 @@ def _ignore(src, names): self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) - self.assertEqual(len(invokations), 9) + self.assertEqual(len(invocations), 9) def test_copytree_retains_permissions(self): - tmp_dir = tempfile.mkdtemp() + tmp_dir = self.mkdtemp() src_dir = os.path.join(tmp_dir, 'source') os.mkdir(src_dir) dst_dir = os.path.join(tmp_dir, 'destination') self.addCleanup(shutil.rmtree, tmp_dir) os.chmod(src_dir, 0o777) - write_file((src_dir, 'permissive.txt'), '123') + create_file((src_dir, 'permissive.txt'), '123') os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777) - write_file((src_dir, 'restrictive.txt'), '456') + create_file((src_dir, 'restrictive.txt'), '456') os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600) restrictive_subdir = tempfile.mkdtemp(dir=src_dir) + self.addCleanup(os_helper.rmtree, restrictive_subdir) os.chmod(restrictive_subdir, 0o600) shutil.copytree(src_dir, dst_dir) @@ -956,8 +964,8 @@ def test_copytree_winerror(self, mock_patch): # When copying to VFAT, copystat() raises OSError. On Windows, the # exception object has a meaningful 'winerror' attribute, but not # on other operating systems. Do not assume 'winerror' is set. - src_dir = tempfile.mkdtemp() - dst_dir = os.path.join(tempfile.mkdtemp(), 'destination') + src_dir = self.mkdtemp() + dst_dir = os.path.join(self.mkdtemp(), 'destination') self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) @@ -975,108 +983,25 @@ def custom_cpfun(a, b): self.assertEqual(b, os.path.join(dst, 'foo')) flag = [] - src = tempfile.mkdtemp() - self.addCleanup(os_helper.rmtree, src) - dst = tempfile.mktemp() - self.addCleanup(os_helper.rmtree, dst) - with open(os.path.join(src, 'foo'), 'w') as f: - f.close() + src = self.mkdtemp() + dst = tempfile.mktemp(dir=self.mkdtemp()) + create_file(os.path.join(src, 'foo')) shutil.copytree(src, dst, copy_function=custom_cpfun) self.assertEqual(len(flag), 1) - @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') - def test_dont_copy_file_onto_link_to_itself(self): - # bug 851123. + # Issue #3002: copyfile and copytree block indefinitely on named pipes + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @os_helper.skip_unless_symlink + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_copytree_named_pipe(self): os.mkdir(TESTFN) - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w') as f: - f.write('cheddar') + subdir = os.path.join(TESTFN, "subdir") + os.mkdir(subdir) + pipe = os.path.join(subdir, "mypipe") try: - os.link(src, dst) - except PermissionError as e: - self.skipTest('os.link(): %s' % e) - self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) - with open(src, 'r') as f: - self.assertEqual(f.read(), 'cheddar') - os.remove(dst) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - @os_helper.skip_unless_symlink - def test_dont_copy_file_onto_symlink_to_itself(self): - # bug 851123. - os.mkdir(TESTFN) - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') - try: - with open(src, 'w') as f: - f.write('cheddar') - # Using `src` here would mean we end up with a symlink pointing - # to TESTFN/TESTFN/cheese, while it should point at - # TESTFN/cheese. - os.symlink('cheese', dst) - self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) - with open(src, 'r') as f: - self.assertEqual(f.read(), 'cheddar') - os.remove(dst) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - @os_helper.skip_unless_symlink - def test_rmtree_on_symlink(self): - # bug 1669. - os.mkdir(TESTFN) - try: - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') - os.mkdir(src) - os.symlink(src, dst) - self.assertRaises(OSError, shutil.rmtree, dst) - shutil.rmtree(dst, ignore_errors=True) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - @unittest.skipUnless(_winapi, 'only relevant on Windows') - def test_rmtree_on_junction(self): - os.mkdir(TESTFN) - try: - src = os.path.join(TESTFN, 'cheese') - dst = os.path.join(TESTFN, 'shop') - os.mkdir(src) - open(os.path.join(src, 'spam'), 'wb').close() - _winapi.CreateJunction(src, dst) - self.assertRaises(OSError, shutil.rmtree, dst) - shutil.rmtree(dst, ignore_errors=True) - finally: - shutil.rmtree(TESTFN, ignore_errors=True) - - # Issue #3002: copyfile and copytree block indefinitely on named pipes - @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') - def test_copyfile_named_pipe(self): - try: - os.mkfifo(TESTFN) - except PermissionError as e: - self.skipTest('os.mkfifo(): %s' % e) - try: - self.assertRaises(shutil.SpecialFileError, - shutil.copyfile, TESTFN, TESTFN2) - self.assertRaises(shutil.SpecialFileError, - shutil.copyfile, __file__, TESTFN) - finally: - os.remove(TESTFN) - - @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') - @os_helper.skip_unless_symlink - def test_copytree_named_pipe(self): - os.mkdir(TESTFN) - try: - subdir = os.path.join(TESTFN, "subdir") - os.mkdir(subdir) - pipe = os.path.join(subdir, "mypipe") - try: - os.mkfifo(pipe) + os.mkfifo(pipe) except PermissionError as e: self.skipTest('os.mkfifo(): %s' % e) try: @@ -1093,12 +1018,11 @@ def test_copytree_named_pipe(self): shutil.rmtree(TESTFN2, ignore_errors=True) def test_copytree_special_func(self): - src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') copied = [] def _copy(src, dst): @@ -1109,19 +1033,25 @@ def _copy(src, dst): @os_helper.skip_unless_symlink def test_copytree_dangling_symlinks(self): - - # a dangling symlink raises an error at the end src_dir = self.mkdtemp() + valid_file = os.path.join(src_dir, 'test.txt') + create_file(valid_file, 'abc') + dir_a = os.path.join(src_dir, 'dir_a') + os.mkdir(dir_a) + for d in src_dir, dir_a: + os.symlink('IDONTEXIST', os.path.join(d, 'broken')) + os.symlink(valid_file, os.path.join(d, 'valid')) + + # A dangling symlink should raise an error. dst_dir = os.path.join(self.mkdtemp(), 'destination') - os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt')) - os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') self.assertRaises(Error, shutil.copytree, src_dir, dst_dir) - # a dangling symlink is ignored with the proper flag + # Dangling symlinks should be ignored with the proper flag. dst_dir = os.path.join(self.mkdtemp(), 'destination2') shutil.copytree(src_dir, dst_dir, ignore_dangling_symlinks=True) - self.assertNotIn('test.txt', os.listdir(dst_dir)) + for root, dirs, files in os.walk(dst_dir): + self.assertNotIn('broken', files) + self.assertIn('valid', files) # a dangling symlink is copied if symlinks=True dst_dir = os.path.join(self.mkdtemp(), 'destination3') @@ -1133,8 +1063,7 @@ def test_copytree_symlink_dir(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') os.mkdir(os.path.join(src_dir, 'real_dir')) - with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'w'): - pass + create_file(os.path.join(src_dir, 'real_dir', 'test.txt')) os.symlink(os.path.join(src_dir, 'real_dir'), os.path.join(src_dir, 'link_to_dir'), target_is_directory=True) @@ -1148,10 +1077,276 @@ def test_copytree_symlink_dir(self): self.assertTrue(os.path.islink(os.path.join(dst_dir, 'link_to_dir'))) self.assertIn('test.txt', os.listdir(os.path.join(dst_dir, 'link_to_dir'))) + def test_copytree_return_value(self): + # copytree returns its destination path. + src_dir = self.mkdtemp() + dst_dir = src_dir + "dest" + self.addCleanup(shutil.rmtree, dst_dir, True) + src = os.path.join(src_dir, 'foo') + create_file(src, 'foo') + rv = shutil.copytree(src_dir, dst_dir) + self.assertEqual(['foo'], os.listdir(rv)) + + def test_copytree_subdirectory(self): + # copytree where dst is a subdirectory of src, see Issue 38688 + base_dir = self.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir, ignore_errors=True) + src_dir = os.path.join(base_dir, "t", "pg") + dst_dir = os.path.join(src_dir, "somevendor", "1.0") + os.makedirs(src_dir) + src = os.path.join(src_dir, 'pol') + create_file(src, 'pol') + rv = shutil.copytree(src_dir, dst_dir) + self.assertEqual(['pol'], os.listdir(rv)) + +class TestCopy(BaseTest, unittest.TestCase): + + ### shutil.copymode + + @os_helper.skip_unless_symlink + def test_copymode_follow_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + dst_link = os.path.join(tmp_dir, 'quux') + create_file(src, 'foo') + create_file(dst, 'foo') + os.symlink(src, src_link) + os.symlink(dst, dst_link) + os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) + # file to file + os.chmod(dst, stat.S_IRWXO) + self.assertNotEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + shutil.copymode(src, dst) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # On Windows, os.chmod does not follow symlinks (issue #15411) + # follow src link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow dst link + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # follow both links + os.chmod(dst, stat.S_IRWXO) + shutil.copymode(src_link, dst_link) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + + @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod') + @os_helper.skip_unless_symlink + def test_copymode_symlink_to_symlink(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + dst_link = os.path.join(tmp_dir, 'quux') + create_file(src, 'foo') + create_file(dst, 'foo') + os.symlink(src, src_link) + os.symlink(dst, dst_link) + os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) + os.chmod(dst, stat.S_IRWXU) + os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) + # link to link + os.lchmod(dst_link, stat.S_IRWXO) + old_mode = os.stat(dst).st_mode + shutil.copymode(src_link, dst_link, follow_symlinks=False) + self.assertEqual(os.lstat(src_link).st_mode, + os.lstat(dst_link).st_mode) + self.assertEqual(os.stat(dst).st_mode, old_mode) + # src link - use chmod + os.lchmod(dst_link, stat.S_IRWXO) + shutil.copymode(src_link, dst, follow_symlinks=False) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + # dst link - use chmod + os.lchmod(dst_link, stat.S_IRWXO) + shutil.copymode(src, dst_link, follow_symlinks=False) + self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) + + @unittest.skipIf(hasattr(os, 'lchmod'), 'requires os.lchmod to be missing') + @os_helper.skip_unless_symlink + def test_copymode_symlink_to_symlink_wo_lchmod(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + dst_link = os.path.join(tmp_dir, 'quux') + create_file(src, 'foo') + create_file(dst, 'foo') + os.symlink(src, src_link) + os.symlink(dst, dst_link) + shutil.copymode(src_link, dst_link, follow_symlinks=False) # silent fail + + ### shutil.copystat + + @os_helper.skip_unless_symlink + def test_copystat_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + dst_link = os.path.join(tmp_dir, 'qux') + create_file(src, 'foo') + src_stat = os.stat(src) + os.utime(src, (src_stat.st_atime, + src_stat.st_mtime - 42.0)) # ensure different mtimes + create_file(dst, 'bar') + self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime) + os.symlink(src, src_link) + os.symlink(dst, dst_link) + if hasattr(os, 'lchmod'): + os.lchmod(src_link, stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(src_link, stat.UF_NODUMP) + src_link_stat = os.lstat(src_link) + # follow + if hasattr(os, 'lchmod'): + shutil.copystat(src_link, dst_link, follow_symlinks=True) + self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode) + # don't follow + shutil.copystat(src_link, dst_link, follow_symlinks=False) + dst_link_stat = os.lstat(dst_link) + if os.utime in os.supports_follow_symlinks: + for attr in 'st_atime', 'st_mtime': + # The modification times may be truncated in the new file. + self.assertLessEqual(getattr(src_link_stat, attr), + getattr(dst_link_stat, attr) + 1) + if hasattr(os, 'lchmod'): + self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode) + if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): + self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags) + # tell to follow but dst is not a link + shutil.copystat(src_link, dst, follow_symlinks=False) + self.assertTrue(abs(os.stat(src).st_mtime - os.stat(dst).st_mtime) < + 00000.1) + + @unittest.skipUnless(hasattr(os, 'chflags') and + hasattr(errno, 'EOPNOTSUPP') and + hasattr(errno, 'ENOTSUP'), + "requires os.chflags, EOPNOTSUPP & ENOTSUP") + def test_copystat_handles_harmless_chflags_errors(self): + tmpdir = self.mkdtemp() + file1 = os.path.join(tmpdir, 'file1') + file2 = os.path.join(tmpdir, 'file2') + create_file(file1, 'xxx') + create_file(file2, 'xxx') + + def make_chflags_raiser(err): + ex = OSError() + + def _chflags_raiser(path, flags, *, follow_symlinks=True): + ex.errno = err + raise ex + return _chflags_raiser + old_chflags = os.chflags + try: + for err in errno.EOPNOTSUPP, errno.ENOTSUP: + os.chflags = make_chflags_raiser(err) + shutil.copystat(file1, file2) + # assert others errors break it + os.chflags = make_chflags_raiser(errno.EOPNOTSUPP + errno.ENOTSUP) + self.assertRaises(OSError, shutil.copystat, file1, file2) + finally: + os.chflags = old_chflags + + ### shutil.copyxattr + + @os_helper.skip_unless_xattr + def test_copyxattr(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + create_file(src, 'foo') + dst = os.path.join(tmp_dir, 'bar') + create_file(dst, 'bar') + + # no xattr == no problem + shutil._copyxattr(src, dst) + # common case + os.setxattr(src, 'user.foo', b'42') + os.setxattr(src, 'user.bar', b'43') + shutil._copyxattr(src, dst) + self.assertEqual(sorted(os.listxattr(src)), sorted(os.listxattr(dst))) + self.assertEqual( + os.getxattr(src, 'user.foo'), + os.getxattr(dst, 'user.foo')) + # check errors don't affect other attrs + os.remove(dst) + create_file(dst, 'bar') + os_error = OSError(errno.EPERM, 'EPERM') + + def _raise_on_user_foo(fname, attr, val, **kwargs): + if attr == 'user.foo': + raise os_error + else: + orig_setxattr(fname, attr, val, **kwargs) + try: + orig_setxattr = os.setxattr + os.setxattr = _raise_on_user_foo + shutil._copyxattr(src, dst) + self.assertIn('user.bar', os.listxattr(dst)) + finally: + os.setxattr = orig_setxattr + # the source filesystem not supporting xattrs should be ok, too. + def _raise_on_src(fname, *, follow_symlinks=True): + if fname == src: + raise OSError(errno.ENOTSUP, 'Operation not supported') + return orig_listxattr(fname, follow_symlinks=follow_symlinks) + try: + orig_listxattr = os.listxattr + os.listxattr = _raise_on_src + shutil._copyxattr(src, dst) + finally: + os.listxattr = orig_listxattr + + # test that shutil.copystat copies xattrs + src = os.path.join(tmp_dir, 'the_original') + srcro = os.path.join(tmp_dir, 'the_original_ro') + create_file(src, src) + create_file(srcro, srcro) + os.setxattr(src, 'user.the_value', b'fiddly') + os.setxattr(srcro, 'user.the_value', b'fiddly') + os.chmod(srcro, 0o444) + dst = os.path.join(tmp_dir, 'the_copy') + dstro = os.path.join(tmp_dir, 'the_copy_ro') + create_file(dst, dst) + create_file(dstro, dstro) + shutil.copystat(src, dst) + shutil.copystat(srcro, dstro) + self.assertEqual(os.getxattr(dst, 'user.the_value'), b'fiddly') + self.assertEqual(os.getxattr(dstro, 'user.the_value'), b'fiddly') + + @os_helper.skip_unless_symlink + @os_helper.skip_unless_xattr + @os_helper.skip_unless_dac_override + def test_copyxattr_symlinks(self): + # On Linux, it's only possible to access non-user xattr for symlinks; + # which in turn require root privileges. This test should be expanded + # as soon as other platforms gain support for extended attributes. + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + src_link = os.path.join(tmp_dir, 'baz') + create_file(src, 'foo') + os.symlink(src, src_link) + os.setxattr(src, 'trusted.foo', b'42') + os.setxattr(src_link, 'trusted.foo', b'43', follow_symlinks=False) + dst = os.path.join(tmp_dir, 'bar') + dst_link = os.path.join(tmp_dir, 'qux') + create_file(dst, 'bar') + os.symlink(dst, dst_link) + shutil._copyxattr(src_link, dst_link, follow_symlinks=False) + self.assertEqual(os.getxattr(dst_link, 'trusted.foo', follow_symlinks=False), b'43') + self.assertRaises(OSError, os.getxattr, dst, 'trusted.foo') + shutil._copyxattr(src_link, dst, follow_symlinks=False) + self.assertEqual(os.getxattr(dst, 'trusted.foo'), b'43') + + ### shutil.copy + def _copy_file(self, method): fname = 'test.txt' tmpdir = self.mkdtemp() - write_file((tmpdir, fname), 'xxx') + create_file((tmpdir, fname), 'xxx') file1 = os.path.join(tmpdir, fname) tmpdir2 = self.mkdtemp() method(file1, tmpdir2) @@ -1164,6 +1359,31 @@ def test_copy(self): self.assertTrue(os.path.exists(file2)) self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode) + @os_helper.skip_unless_symlink + def test_copy_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + create_file(src, 'foo') + os.symlink(src, src_link) + if hasattr(os, 'lchmod'): + os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) + # don't follow + shutil.copy(src_link, dst, follow_symlinks=True) + self.assertFalse(os.path.islink(dst)) + self.assertEqual(read_file(src), read_file(dst)) + os.remove(dst) + # follow + shutil.copy(src_link, dst, follow_symlinks=False) + self.assertTrue(os.path.islink(dst)) + self.assertEqual(os.readlink(dst), os.readlink(src_link)) + if hasattr(os, 'lchmod'): + self.assertEqual(os.lstat(src_link).st_mode, + os.lstat(dst).st_mode) + + ### shutil.copy2 + @unittest.skipUnless(hasattr(os, 'utime'), 'requires os.utime') def test_copy2(self): # Ensure that the copied file exists and has the same mode and @@ -1181,41 +1401,220 @@ def test_copy2(self): self.assertEqual(getattr(file1_stat, 'st_flags'), getattr(file2_stat, 'st_flags')) - @support.requires_zlib - def test_make_tarball(self): - # creating something to tar - root_dir, base_dir = self._create_files('') + @os_helper.skip_unless_symlink + def test_copy2_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + src_link = os.path.join(tmp_dir, 'baz') + create_file(src, 'foo') + os.symlink(src, src_link) + if hasattr(os, 'lchmod'): + os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) + if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): + os.lchflags(src_link, stat.UF_NODUMP) + src_stat = os.stat(src) + src_link_stat = os.lstat(src_link) + # follow + shutil.copy2(src_link, dst, follow_symlinks=True) + self.assertFalse(os.path.islink(dst)) + self.assertEqual(read_file(src), read_file(dst)) + os.remove(dst) + # don't follow + shutil.copy2(src_link, dst, follow_symlinks=False) + self.assertTrue(os.path.islink(dst)) + self.assertEqual(os.readlink(dst), os.readlink(src_link)) + dst_stat = os.lstat(dst) + if os.utime in os.supports_follow_symlinks: + for attr in 'st_atime', 'st_mtime': + # The modification times may be truncated in the new file. + self.assertLessEqual(getattr(src_link_stat, attr), + getattr(dst_stat, attr) + 1) + if hasattr(os, 'lchmod'): + self.assertEqual(src_link_stat.st_mode, dst_stat.st_mode) + self.assertNotEqual(src_stat.st_mode, dst_stat.st_mode) + if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): + self.assertEqual(src_link_stat.st_flags, dst_stat.st_flags) + + @os_helper.skip_unless_xattr + def test_copy2_xattr(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'foo') + dst = os.path.join(tmp_dir, 'bar') + create_file(src, 'foo') + os.setxattr(src, 'user.foo', b'42') + shutil.copy2(src, dst) + self.assertEqual( + os.getxattr(src, 'user.foo'), + os.getxattr(dst, 'user.foo')) + os.remove(dst) + + def test_copy_return_value(self): + # copy and copy2 both return their destination path. + for fn in (shutil.copy, shutil.copy2): + src_dir = self.mkdtemp() + dst_dir = self.mkdtemp() + src = os.path.join(src_dir, 'foo') + create_file(src, 'foo') + rv = fn(src, dst_dir) + self.assertEqual(rv, os.path.join(dst_dir, 'foo')) + rv = fn(src, os.path.join(dst_dir, 'bar')) + self.assertEqual(rv, os.path.join(dst_dir, 'bar')) + + def test_copy_dir(self): + self._test_copy_dir(shutil.copy) + + def test_copy2_dir(self): + self._test_copy_dir(shutil.copy2) + + def _test_copy_dir(self, copy_func): + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + dir2 = self.mkdtemp() + dst = os.path.join(src_dir, 'does_not_exist/') + create_file(src_file, 'foo') + if sys.platform == "win32": + err = PermissionError + else: + err = IsADirectoryError + self.assertRaises(err, copy_func, dir2, src_dir) + + # raise *err* because of src rather than FileNotFoundError because of dst + self.assertRaises(err, copy_func, dir2, dst) + copy_func(src_file, dir2) # should not raise exceptions + + ### shutil.copyfile + + @os_helper.skip_unless_symlink + def test_copyfile_symlinks(self): + tmp_dir = self.mkdtemp() + src = os.path.join(tmp_dir, 'src') + dst = os.path.join(tmp_dir, 'dst') + dst_link = os.path.join(tmp_dir, 'dst_link') + link = os.path.join(tmp_dir, 'link') + create_file(src, 'foo') + os.symlink(src, link) + # don't follow + shutil.copyfile(link, dst_link, follow_symlinks=False) + self.assertTrue(os.path.islink(dst_link)) + self.assertEqual(os.readlink(link), os.readlink(dst_link)) + # follow + shutil.copyfile(link, dst) + self.assertFalse(os.path.islink(dst)) + + @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') + def test_dont_copy_file_onto_link_to_itself(self): + # bug 851123. + os.mkdir(TESTFN) + src = os.path.join(TESTFN, 'cheese') + dst = os.path.join(TESTFN, 'shop') + try: + create_file(src, 'cheddar') + try: + os.link(src, dst) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) + self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) + with open(src, 'r', encoding='utf-8') as f: + self.assertEqual(f.read(), 'cheddar') + os.remove(dst) + finally: + shutil.rmtree(TESTFN, ignore_errors=True) + + @os_helper.skip_unless_symlink + def test_dont_copy_file_onto_symlink_to_itself(self): + # bug 851123. + os.mkdir(TESTFN) + src = os.path.join(TESTFN, 'cheese') + dst = os.path.join(TESTFN, 'shop') + try: + create_file(src, 'cheddar') + # Using `src` here would mean we end up with a symlink pointing + # to TESTFN/TESTFN/cheese, while it should point at + # TESTFN/cheese. + os.symlink('cheese', dst) + self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) + with open(src, 'r', encoding='utf-8') as f: + self.assertEqual(f.read(), 'cheddar') + os.remove(dst) + finally: + shutil.rmtree(TESTFN, ignore_errors=True) + + # Issue #3002: copyfile and copytree block indefinitely on named pipes + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + @unittest.skipIf(sys.platform == "vxworks", + "fifo requires special path on VxWorks") + def test_copyfile_named_pipe(self): + try: + os.mkfifo(TESTFN) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) + try: + self.assertRaises(shutil.SpecialFileError, + shutil.copyfile, TESTFN, TESTFN2) + self.assertRaises(shutil.SpecialFileError, + shutil.copyfile, __file__, TESTFN) + finally: + os.remove(TESTFN) + + def test_copyfile_return_value(self): + # copytree returns its destination path. + src_dir = self.mkdtemp() + dst_dir = self.mkdtemp() + dst_file = os.path.join(dst_dir, 'bar') + src_file = os.path.join(src_dir, 'foo') + create_file(src_file, 'foo') + rv = shutil.copyfile(src_file, dst_file) + self.assertTrue(os.path.exists(rv)) + self.assertEqual(read_file(src_file), read_file(dst_file)) + + def test_copyfile_same_file(self): + # copyfile() should raise SameFileError if the source and destination + # are the same. + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + create_file(src_file, 'foo') + self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) + # But Error should work too, to stay backward compatible. + self.assertRaises(Error, shutil.copyfile, src_file, src_file) + # Make sure file is not corrupted. + self.assertEqual(read_file(src_file), 'foo') + + @unittest.skipIf(MACOS or SOLARIS or _winapi, 'On MACOS, Solaris and Windows the errors are not confusing (though different)') + # gh-92670: The test uses a trailing slash to force the OS consider + # the path as a directory, but on AIX the trailing slash has no effect + # and is considered as a file. + @unittest.skipIf(AIX, 'Not valid on AIX, see gh-92670') + def test_copyfile_nonexistent_dir(self): + # Issue 43219 + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + dst = os.path.join(src_dir, 'does_not_exist/') + create_file(src_file, 'foo') + self.assertRaises(FileNotFoundError, shutil.copyfile, src_file, dst) + + def test_copyfile_copy_dir(self): + # Issue 45234 + # test copy() and copyfile() raising proper exceptions when src and/or + # dst are directories + src_dir = self.mkdtemp() + src_file = os.path.join(src_dir, 'foo') + dir2 = self.mkdtemp() + dst = os.path.join(src_dir, 'does_not_exist/') + create_file(src_file, 'foo') + if sys.platform == "win32": + err = PermissionError + else: + err = IsADirectoryError - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') + self.assertRaises(err, shutil.copyfile, src_dir, dst) + self.assertRaises(err, shutil.copyfile, src_file, src_dir) + self.assertRaises(err, shutil.copyfile, dir2, src_dir) - with os_helper.change_cwd(work_dir): - base_name = os.path.abspath(rel_base_name) - tarball = make_archive(rel_base_name, 'gztar', root_dir, '.') - # check if the compressed tarball was created - self.assertEqual(tarball, base_name + '.tar.gz') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r:gz') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) +class TestArchives(BaseTest, unittest.TestCase): - # trying an uncompressed one - with os_helper.change_cwd(work_dir): - tarball = make_archive(rel_base_name, 'tar', root_dir, '.') - self.assertEqual(tarball, base_name + '.tar') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) + ### shutil.make_archive def _tarinfo(self, path): with tarfile.open(path) as tar: @@ -1228,22 +1627,109 @@ def _create_files(self, base_dir='dist'): root_dir = self.mkdtemp() dist = os.path.join(root_dir, base_dir) os.makedirs(dist, exist_ok=True) - write_file((dist, 'file1'), 'xxx') - write_file((dist, 'file2'), 'xxx') + create_file((dist, 'file1'), 'xxx') + create_file((dist, 'file2'), 'xxx') os.mkdir(os.path.join(dist, 'sub')) - write_file((dist, 'sub', 'file3'), 'xxx') + create_file((dist, 'sub', 'file3'), 'xxx') os.mkdir(os.path.join(dist, 'sub2')) if base_dir: - write_file((root_dir, 'outer'), 'xxx') + create_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir - @support.requires_zlib + @support.requires_zlib() + def test_make_tarfile(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir) + # check if the compressed tarball was created + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'tar', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + # check if the uncompressed tarball was created + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'tar', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist/sub', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_tarfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'gztar') + self.assertEqual(archive, base_name + '.tar.gz') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r:gz') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', base_dir=base_dir) + self.assertEqual(archive, base_name + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + def test_make_tarfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + @support.requires_zlib() @unittest.skipUnless(shutil.which('tar'), 'Need the tar command to run') def test_tarfile_vs_tar(self): root_dir, base_dir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') - tarball = make_archive(base_name, 'gztar', root_dir, base_dir) + with no_chdir: + tarball = make_archive(base_name, 'gztar', root_dir, base_dir) # check if the compressed tarball was created self.assertEqual(tarball, base_name + '.tar.gz') @@ -1252,6 +1738,17 @@ def test_tarfile_vs_tar(self): # now create another tarball using `tar` tarball2 = os.path.join(root_dir, 'archive2.tar') tar_cmd = ['tar', '-cf', 'archive2.tar', base_dir] + if sys.platform == 'darwin': + # macOS tar can include extended attributes, + # ACLs and other mac specific metadata into the + # archive (an recentish version of the OS). + # + # This feature can be disabled with the + # '--no-mac-metadata' option on macOS 11 or + # later. + import platform + if int(platform.mac_ver()[0].split('.')[0]) >= 11: + tar_cmd.insert(1, '--no-mac-metadata') subprocess.check_call(tar_cmd, cwd=root_dir, stdout=subprocess.DEVNULL) @@ -1260,60 +1757,112 @@ def test_tarfile_vs_tar(self): self.assertEqual(self._tarinfo(tarball), self._tarinfo(tarball2)) # trying an uncompressed one - tarball = make_archive(base_name, 'tar', root_dir, base_dir) + with no_chdir: + tarball = make_archive(base_name, 'tar', root_dir, base_dir) self.assertEqual(tarball, base_name + '.tar') self.assertTrue(os.path.isfile(tarball)) # now for a dry_run - tarball = make_archive(base_name, 'tar', root_dir, base_dir, - dry_run=True) + with no_chdir: + tarball = make_archive(base_name, 'tar', root_dir, base_dir, + dry_run=True) self.assertEqual(tarball, base_name + '.tar') self.assertTrue(os.path.isfile(tarball)) - @support.requires_zlib + @support.requires_zlib() def test_make_zipfile(self): - # creating something to zip root_dir, base_dir = self._create_files() - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir): - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir) - - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3', - 'outer']) - - with os_helper.change_cwd(work_dir): - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir, base_dir) - - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3']) - - @support.requires_zlib + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'zip', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'zip', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/sub/', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_zipfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'zip') + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + root_dir, base_dir = self._create_files() + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', base_dir=base_dir) + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_zipfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + @support.requires_zlib() @unittest.skipUnless(shutil.which('zip'), 'Need the zip command to run') def test_zipfile_vs_zip(self): root_dir, base_dir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') - archive = make_archive(base_name, 'zip', root_dir, base_dir) + with no_chdir: + archive = make_archive(base_name, 'zip', root_dir, base_dir) # check if ZIP file was created self.assertEqual(archive, base_name + '.zip') @@ -1333,13 +1882,14 @@ def test_zipfile_vs_zip(self): names2 = zf.namelist() self.assertEqual(sorted(names), sorted(names2)) - @support.requires_zlib + @support.requires_zlib() @unittest.skipUnless(shutil.which('unzip'), 'Need the unzip command to run') def test_unzip_zipfile(self): root_dir, base_dir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') - archive = make_archive(base_name, 'zip', root_dir, base_dir) + with no_chdir: + archive = make_archive(base_name, 'zip', root_dir, base_dir) # check if ZIP file was created self.assertEqual(archive, base_name + '.zip') @@ -1352,7 +1902,10 @@ def test_unzip_zipfile(self): subprocess.check_output(zip_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: details = exc.output.decode(errors="replace") - if 'unrecognized option: t' in details: + if any(message in details for message in [ + 'unrecognized option: t', # BusyBox + 'invalid option -- t', # Android + ]): self.skipTest("unzip doesn't support -t") msg = "{}\n\n**Unzip Output**\n{}" self.fail(msg.format(exc, details)) @@ -1362,7 +1915,7 @@ def test_make_archive(self): base_name = os.path.join(tmpdir, 'archive') self.assertRaises(ValueError, make_archive, base_name, 'xxx') - @support.requires_zlib + @support.requires_zlib() def test_make_archive_owner_group(self): # testing make_archive with owner and group, with various combinations # this works even if there's not gid/uid support @@ -1390,14 +1943,14 @@ def test_make_archive_owner_group(self): self.assertTrue(os.path.isfile(res)) - @support.requires_zlib + @support.requires_zlib() @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") def test_tarfile_root_owner(self): root_dir, base_dir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') group = grp.getgrgid(0)[0] owner = pwd.getpwuid(0)[0] - with os_helper.change_cwd(root_dir): + with os_helper.change_cwd(root_dir), no_chdir: archive_name = make_archive(base_name, 'gztar', root_dir, 'dist', owner=owner, group=group) @@ -1413,33 +1966,79 @@ def test_tarfile_root_owner(self): finally: archive.close() + def test_make_archive_cwd_default(self): + current_dir = os.getcwd() + def archiver(base_name, base_dir, **kw): + self.assertNotIn('root_dir', kw) + self.assertEqual(base_name, 'basename') + self.assertEqual(os.getcwd(), current_dir) + raise RuntimeError() + + register_archive_format('xxx', archiver, [], 'xxx file') + try: + with no_chdir: + with self.assertRaises(RuntimeError): + make_archive('basename', 'xxx') + self.assertEqual(os.getcwd(), current_dir) + finally: + unregister_archive_format('xxx') + def test_make_archive_cwd(self): current_dir = os.getcwd() - def _breaks(*args, **kw): + root_dir = self.mkdtemp() + def archiver(base_name, base_dir, **kw): + self.assertNotIn('root_dir', kw) + self.assertEqual(base_name, os.path.join(current_dir, 'basename')) + self.assertEqual(os.getcwd(), root_dir) raise RuntimeError() + dirs = [] + def _chdir(path): + dirs.append(path) + orig_chdir(path) - register_archive_format('xxx', _breaks, [], 'xxx file') + register_archive_format('xxx', archiver, [], 'xxx file') try: - try: - make_archive('xxx', 'xxx', root_dir=self.mkdtemp()) - except Exception: - pass + with support.swap_attr(os, 'chdir', _chdir) as orig_chdir: + with self.assertRaises(RuntimeError): + make_archive('basename', 'xxx', root_dir=root_dir) + self.assertEqual(os.getcwd(), current_dir) + self.assertEqual(dirs, [root_dir, current_dir]) + finally: + unregister_archive_format('xxx') + + def test_make_archive_cwd_supports_root_dir(self): + current_dir = os.getcwd() + root_dir = self.mkdtemp() + def archiver(base_name, base_dir, **kw): + self.assertEqual(base_name, 'basename') + self.assertEqual(kw['root_dir'], root_dir) + self.assertEqual(os.getcwd(), current_dir) + raise RuntimeError() + archiver.supports_root_dir = True + + register_archive_format('xxx', archiver, [], 'xxx file') + try: + with no_chdir: + with self.assertRaises(RuntimeError): + make_archive('basename', 'xxx', root_dir=root_dir) self.assertEqual(os.getcwd(), current_dir) finally: unregister_archive_format('xxx') def test_make_tarfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() - with os_helper.change_cwd(root_dir): + with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'tar'), 'test.tar') self.assertTrue(os.path.isfile('test.tar')) - @support.requires_zlib + @support.requires_zlib() def test_make_zipfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() - with os_helper.change_cwd(root_dir): + with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'zip'), 'test.zip') self.assertTrue(os.path.isfile('test.zip')) @@ -1459,12 +2058,63 @@ def test_register_archive_format(self): formats = [name for name, params in get_archive_formats()] self.assertNotIn('xxx', formats) - def check_unpack_archive(self, format): - self.check_unpack_archive_with_converter(format, lambda path: path) - self.check_unpack_archive_with_converter(format, pathlib.Path) - self.check_unpack_archive_with_converter(format, FakePath) - - def check_unpack_archive_with_converter(self, format, converter): + def test_make_tarfile_rootdir_nodir(self): + # GH-99203: Test with root_dir is not a real directory. + self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + # root_dir does not exist. + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'tar', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + # root_dir is a file. + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'tar', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + @support.requires_zlib() + def test_make_zipfile_rootdir_nodir(self): + # GH-99203: Test with root_dir is not a real directory. + self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + # root_dir does not exist. + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'zip', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + + # root_dir is a file. + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'zip', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + + ### shutil.unpack_archive + + def check_unpack_archive(self, format, **kwargs): + self.check_unpack_archive_with_converter( + format, lambda path: path, **kwargs) + self.check_unpack_archive_with_converter( + format, FakePath, **kwargs) + self.check_unpack_archive_with_converter(format, FakePath, **kwargs) + + def check_unpack_archive_with_converter(self, format, converter, **kwargs): root_dir, base_dir = self._create_files() expected = rlistdir(root_dir) expected.remove('outer') @@ -1474,36 +2124,49 @@ def check_unpack_archive_with_converter(self, format, converter): # let's try to unpack it now tmpdir2 = self.mkdtemp() - unpack_archive(converter(filename), converter(tmpdir2)) + unpack_archive(converter(filename), converter(tmpdir2), **kwargs) self.assertEqual(rlistdir(tmpdir2), expected) # and again, this time with the format specified tmpdir3 = self.mkdtemp() - unpack_archive(converter(filename), converter(tmpdir3), format=format) + unpack_archive(converter(filename), converter(tmpdir3), format=format, + **kwargs) self.assertEqual(rlistdir(tmpdir3), expected) - self.assertRaises(shutil.ReadError, unpack_archive, converter(TESTFN)) - self.assertRaises(ValueError, unpack_archive, converter(TESTFN), format='xxx') + with self.assertRaises(shutil.ReadError): + unpack_archive(converter(TESTFN), **kwargs) + with self.assertRaises(ValueError): + unpack_archive(converter(TESTFN), format='xxx', **kwargs) + + def check_unpack_tarball(self, format): + self.check_unpack_archive(format, filter='fully_trusted') + self.check_unpack_archive(format, filter='data') def test_unpack_archive_tar(self): - self.check_unpack_archive('tar') + self.check_unpack_tarball('tar') - @support.requires_zlib + @support.requires_zlib() def test_unpack_archive_gztar(self): - self.check_unpack_archive('gztar') + self.check_unpack_tarball('gztar') - @support.requires_bz2 + @support.requires_bz2() def test_unpack_archive_bztar(self): - self.check_unpack_archive('bztar') + self.check_unpack_tarball('bztar') + + @support.requires_zstd() + def test_unpack_archive_zstdtar(self): + self.check_unpack_tarball('zstdtar') - @support.requires_lzma + @support.requires_lzma() @unittest.skipIf(AIX and not _maxdataOK(), "AIX MAXDATA must be 0x20000000 or larger") def test_unpack_archive_xztar(self): - self.check_unpack_archive('xztar') + self.check_unpack_tarball('xztar') - @support.requires_zlib + @support.requires_zlib() def test_unpack_archive_zip(self): self.check_unpack_archive('zip') + with self.assertRaises(TypeError): + self.check_unpack_archive('zip', filter='data') def test_unpack_registry(self): @@ -1531,6 +2194,9 @@ def _boo(filename, extract_dir, extra): unregister_unpack_format('Boo2') self.assertEqual(get_unpack_formats(), formats) + +class TestMisc(BaseTest, unittest.TestCase): + @unittest.skipUnless(hasattr(shutil, 'disk_usage'), "disk_usage not available on this platform") def test_disk_usage(self): @@ -1549,11 +2215,11 @@ def test_disk_usage(self): @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') def test_chown(self): - - # cleaned-up automatically by TestShutil.tearDown method dirname = self.mkdtemp() filename = tempfile.mktemp(dir=dirname) - write_file(filename, 'testing chown function') + linkname = os.path.join(dirname, "chown_link") + create_file(filename, 'testing chown function') + os.symlink(filename, linkname) with self.assertRaises(ValueError): shutil.chown(filename) @@ -1574,7 +2240,7 @@ def test_chown(self): gid = os.getgid() def check_chown(path, uid=None, gid=None): - s = os.stat(filename) + s = os.stat(path) if uid is not None: self.assertEqual(uid, s.st_uid) if gid is not None: @@ -1598,98 +2264,88 @@ def check_chown(path, uid=None, gid=None): shutil.chown(dirname, group=gid) check_chown(dirname, gid=gid) - user = pwd.getpwuid(uid)[0] - group = grp.getgrgid(gid)[0] - shutil.chown(filename, user, group) + try: + user = pwd.getpwuid(uid)[0] + group = grp.getgrgid(gid)[0] + except KeyError: + # On some systems uid/gid cannot be resolved. + pass + else: + shutil.chown(filename, user, group) + check_chown(filename, uid, gid) + shutil.chown(dirname, user, group) + check_chown(dirname, uid, gid) + + dirfd = os.open(dirname, os.O_RDONLY) + self.addCleanup(os.close, dirfd) + basename = os.path.basename(filename) + baselinkname = os.path.basename(linkname) + shutil.chown(basename, uid, gid, dir_fd=dirfd) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, dir_fd=dirfd) + check_chown(filename, uid) + shutil.chown(basename, group=gid, dir_fd=dirfd) + check_chown(filename, gid=gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, uid, gid) + shutil.chown(linkname, uid, follow_symlinks=True) + check_chown(filename, uid) + shutil.chown(baselinkname, group=gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, gid=gid) + shutil.chown(baselinkname, uid, gid, dir_fd=dirfd, follow_symlinks=True) check_chown(filename, uid, gid) - shutil.chown(dirname, user, group) - check_chown(dirname, uid, gid) - - def test_copy_return_value(self): - # copy and copy2 both return their destination path. - for fn in (shutil.copy, shutil.copy2): - src_dir = self.mkdtemp() - dst_dir = self.mkdtemp() - src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') - rv = fn(src, dst_dir) - self.assertEqual(rv, os.path.join(dst_dir, 'foo')) - rv = fn(src, os.path.join(dst_dir, 'bar')) - self.assertEqual(rv, os.path.join(dst_dir, 'bar')) - - def test_copyfile_return_value(self): - # copytree returns its destination path. - src_dir = self.mkdtemp() - dst_dir = self.mkdtemp() - dst_file = os.path.join(dst_dir, 'bar') - src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') - rv = shutil.copyfile(src_file, dst_file) - self.assertTrue(os.path.exists(rv)) - self.assertEqual(read_file(src_file), read_file(dst_file)) - def test_copyfile_same_file(self): - # copyfile() should raise SameFileError if the source and destination - # are the same. - src_dir = self.mkdtemp() - src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') - self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) - # But Error should work too, to stay backward compatible. - self.assertRaises(Error, shutil.copyfile, src_file, src_file) - # Make sure file is not corrupted. - self.assertEqual(read_file(src_file), 'foo') + with self.assertRaises(TypeError): + shutil.chown(filename, uid, dir_fd=dirname) - def test_copytree_return_value(self): - # copytree returns its destination path. - src_dir = self.mkdtemp() - dst_dir = src_dir + "dest" - self.addCleanup(shutil.rmtree, dst_dir, True) - src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') - rv = shutil.copytree(src_dir, dst_dir) - self.assertEqual(['foo'], os.listdir(rv)) + with self.assertRaises(FileNotFoundError): + shutil.chown('missingfile', uid, gid, dir_fd=dirfd) - def test_copytree_subdirectory(self): - # copytree where dst is a subdirectory of src, see Issue 38688 - base_dir = self.mkdtemp() - self.addCleanup(shutil.rmtree, base_dir, ignore_errors=True) - src_dir = os.path.join(base_dir, "t", "pg") - dst_dir = os.path.join(src_dir, "somevendor", "1.0") - os.makedirs(src_dir) - src = os.path.join(src_dir, 'pol') - write_file(src, 'pol') - rv = shutil.copytree(src_dir, dst_dir) - self.assertEqual(['pol'], os.listdir(rv)) + with self.assertRaises(ValueError): + shutil.chown(filename, dir_fd=dirfd) -class TestWhich(unittest.TestCase): +@support.requires_subprocess() +class TestWhich(BaseTest, unittest.TestCase): def setUp(self): - self.temp_dir = tempfile.mkdtemp(prefix="Tmp") - self.addCleanup(shutil.rmtree, self.temp_dir, True) + temp_dir = self.mkdtemp(prefix="Tmp") + base_dir = os.path.join(temp_dir, TESTFN + '-basedir') + os.mkdir(base_dir) + self.dir = os.path.join(base_dir, TESTFN + '-dir') + os.mkdir(self.dir) + self.other_dir = os.path.join(base_dir, TESTFN + '-dir2') + os.mkdir(self.other_dir) # Give the temp_file an ".exe" suffix for all. # It's needed on Windows and not harmful on other platforms. - self.temp_file = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp", - suffix=".Exe") - os.chmod(self.temp_file.name, stat.S_IXUSR) - self.addCleanup(self.temp_file.close) - self.dir, self.file = os.path.split(self.temp_file.name) + self.file = TESTFN + '.Exe' + self.filepath = os.path.join(self.dir, self.file) + self.create_file(self.filepath) self.env_path = self.dir self.curdir = os.curdir self.ext = ".EXE" + to_text_type = staticmethod(os.fsdecode) + + def create_file(self, path): + create_file(path) + os.chmod(path, 0o755) + + def assertNormEqual(self, actual, expected): + self.assertEqual(os.path.normcase(actual), os.path.normcase(expected)) + def test_basic(self): # Given an EXE in a directory, it should be returned. rv = shutil.which(self.file, path=self.dir) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_absolute_cmd(self): # When given the fully qualified path to an executable that exists, # it should be returned. - rv = shutil.which(self.temp_file.name, path=self.temp_dir) - self.assertEqual(rv, self.temp_file.name) + rv = shutil.which(self.filepath, path=self.other_dir) + self.assertEqual(rv, self.filepath) def test_relative_cmd(self): # When given the relative path with a directory part to an executable @@ -1697,31 +2353,54 @@ def test_relative_cmd(self): base_dir, tail_dir = os.path.split(self.dir) relpath = os.path.join(tail_dir, self.file) with os_helper.change_cwd(path=base_dir): - rv = shutil.which(relpath, path=self.temp_dir) + rv = shutil.which(relpath, path=self.other_dir) self.assertEqual(rv, relpath) # But it shouldn't be searched in PATH directories (issue #16957). with os_helper.change_cwd(path=self.dir): rv = shutil.which(relpath, path=base_dir) self.assertIsNone(rv) - def test_cwd(self): + @unittest.skipUnless(sys.platform != "win32", + "test is for non win32") + def test_cwd_non_win32(self): # Issue #16957 + with os_helper.change_cwd(path=self.dir): + rv = shutil.which(self.file, path=self.other_dir) + # non-win32: shouldn't match in the current directory. + self.assertIsNone(rv) + + @unittest.skipUnless(sys.platform == "win32", + "test is for win32") + def test_cwd_win32(self): base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): - rv = shutil.which(self.file, path=base_dir) - if sys.platform == "win32": - # Windows: current directory implicitly on PATH + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + rv = shutil.which(self.file, path=self.other_dir) + # Current directory implicitly on PATH self.assertEqual(rv, os.path.join(self.curdir, self.file)) - else: - # Other platforms: shouldn't match in the current directory. + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=False): + rv = shutil.which(self.file, path=self.other_dir) + # Current directory not on PATH self.assertIsNone(rv) - @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, - 'non-root user required') + @unittest.skipUnless(sys.platform == "win32", + "test is for win32") + def test_cwd_win32_added_before_all_other_path(self): + other_file_path = os.path.join(self.other_dir, self.file) + self.create_file(other_file_path) + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + with os_helper.change_cwd(path=self.dir): + rv = shutil.which(self.file, path=self.other_dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) + with os_helper.change_cwd(path=self.other_dir): + rv = shutil.which(self.file, path=self.dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) + + @os_helper.skip_if_dac_override def test_non_matching_mode(self): # Set the file read-only and ask for writeable files. - os.chmod(self.temp_file.name, stat.S_IREAD) - if os.access(self.temp_file.name, os.W_OK): + os.chmod(self.filepath, stat.S_IREAD) + if os.access(self.filepath, os.W_OK): self.skipTest("can't set the file read-only") rv = shutil.which(self.file, path=self.dir, mode=os.W_OK) self.assertIsNone(rv) @@ -1743,13 +2422,13 @@ def test_pathext_checking(self): # Ask for the file without the ".exe" extension, then ensure that # it gets found properly with the extension. rv = shutil.which(self.file[:-4], path=self.dir) - self.assertEqual(rv, self.temp_file.name[:-4] + self.ext) + self.assertEqual(rv, self.filepath[:-4] + self.ext) def test_environ_path(self): with os_helper.EnvironmentVarGuard() as env: env['PATH'] = self.env_path rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_environ_path_empty(self): # PATH='': no match @@ -1763,12 +2442,9 @@ def test_environ_path_empty(self): self.assertIsNone(rv) def test_environ_path_cwd(self): - expected_cwd = os.path.basename(self.temp_file.name) + expected_cwd = self.file if sys.platform == "win32": - curdir = os.curdir - if isinstance(expected_cwd, bytes): - curdir = os.fsencode(curdir) - expected_cwd = os.path.join(curdir, expected_cwd) + expected_cwd = os.path.join(self.curdir, expected_cwd) # PATH=':': explicitly looks in the current directory with os_helper.EnvironmentVarGuard() as env: @@ -1786,21 +2462,21 @@ def test_environ_path_cwd(self): def test_environ_path_missing(self): with os_helper.EnvironmentVarGuard() as env: - env.pop('PATH', None) + del env['PATH'] # without confstr with unittest.mock.patch('os.confstr', side_effect=ValueError, \ create=True), \ support.swap_attr(os, 'defpath', self.dir): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) # with confstr with unittest.mock.patch('os.confstr', return_value=self.dir, \ create=True), \ support.swap_attr(os, 'defpath', ''): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_empty_path(self): base_dir = os.path.dirname(self.dir) @@ -1812,26 +2488,210 @@ def test_empty_path(self): def test_empty_path_no_PATH(self): with os_helper.EnvironmentVarGuard() as env: - env.pop('PATH', None) + del env['PATH'] rv = shutil.which(self.file) self.assertIsNone(rv) @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext(self): - ext = ".xyz" - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp2", suffix=ext) - os.chmod(temp_filexyz.name, stat.S_IXUSR) - self.addCleanup(temp_filexyz.close) + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] + # Issue 40592: See https://bugs.python.org/issue40592 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_empty_str(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + ';' # note the ; + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_multidot_extension(self): + ext = '.foo.bar' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: env['PATHEXT'] = ext - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_null_extension(self): + cmd = self.to_text_type(TESTFN2) + cmddot = cmd + self.to_text_type('.') + filepath = os.path.join(self.dir, cmd) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = '.xyz' + self.assertIsNone(shutil.which(cmd, path=self.dir)) + self.assertIsNone(shutil.which(cmddot, path=self.dir)) + env['PATHEXT'] = '.xyz;.' # note the . + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + env['PATHEXT'] = '.xyz;..' # multiple dots + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_extension_ends_with_dot(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + dot = self.to_text_type('.') + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + '.' + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) # cmd.exe hangs here + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + self.assertIsNone(shutil.which(cmd + dot, path=self.dir)) + self.assertIsNone(shutil.which(cmdext + dot, path=self.dir)) + + # See GH-75586 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_applied_on_files_in_path(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env["PATH"] = os.fsdecode(self.dir) + env["PATHEXT"] = ext + self.assertEqual(shutil.which(cmd), filepath) + self.assertEqual(shutil.which(cmdext), filepath) + + # See GH-75586 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_win_path_needs_curdir(self): + with unittest.mock.patch('_winapi.NeedCurrentDirectoryForExePath', return_value=True) as need_curdir_mock: + self.assertTrue(shutil._win_path_needs_curdir('dontcare', os.X_OK)) + need_curdir_mock.assert_called_once_with('dontcare') + need_curdir_mock.reset_mock() + self.assertTrue(shutil._win_path_needs_curdir('dontcare', 0)) + need_curdir_mock.assert_not_called() + + with unittest.mock.patch('_winapi.NeedCurrentDirectoryForExePath', return_value=False) as need_curdir_mock: + self.assertFalse(shutil._win_path_needs_curdir('dontcare', os.X_OK)) + need_curdir_mock.assert_called_once_with('dontcare') + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_same_dir_with_pathext_extension(self): + cmd = self.file # with .exe extension + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.dir, cmd2) + self.create_file(other_file_path) + + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + self.assertNormEqual(shutil.which(cmd2, path=self.dir), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=self.dir, mode=os.F_OK), + other_file_path) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_same_dir_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + # pathext match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.dir, cmd) + self.create_file(other_file_path) + + # pathext match if mode contains X_OK + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + other_file_path) + self.assertNormEqual(shutil.which(self.file, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=self.dir, mode=os.F_OK), + self.filepath) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_with_pathext_extension(self): + cmd = self.file # with .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # full match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.other_dir, cmd2) + self.create_file(other_file_path) + + # pathext match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd2, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=search_path, mode=os.F_OK), + other_file_path) + + # full match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.other_dir, cmd) + self.create_file(other_file_path) + + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the second directory + self.assertNormEqual(shutil.which(self.file, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=search_path, mode=os.F_OK), + self.filepath) + + # pathext match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) class TestWhichBytes(TestWhich): @@ -1839,32 +2699,23 @@ def setUp(self): TestWhich.setUp(self) self.dir = os.fsencode(self.dir) self.file = os.fsencode(self.file) - self.temp_file.name = os.fsencode(self.temp_file.name) + self.filepath = os.fsencode(self.filepath) + self.other_dir = os.fsencode(self.other_dir) self.curdir = os.fsencode(self.curdir) self.ext = os.fsencode(self.ext) + to_text_type = staticmethod(os.fsencode) -class TestMove(unittest.TestCase): + +class TestMove(BaseTest, unittest.TestCase): def setUp(self): filename = "foo" - basedir = None - if sys.platform == "win32": - basedir = os.path.realpath(os.getcwd()) - self.src_dir = tempfile.mkdtemp(dir=basedir) - self.dst_dir = tempfile.mkdtemp(dir=basedir) + self.src_dir = self.mkdtemp() + self.dst_dir = self.mkdtemp() self.src_file = os.path.join(self.src_dir, filename) self.dst_file = os.path.join(self.dst_dir, filename) - with open(self.src_file, "wb") as f: - f.write(b"spam") - - def tearDown(self): - for d in (self.src_dir, self.dst_dir): - try: - if d: - shutil.rmtree(d) - except: - pass + create_file(self.src_file, b"spam") def _check_move_file(self, src, dst, real_dst): with open(src, "rb") as f: @@ -1888,6 +2739,16 @@ def test_move_file_to_dir(self): # Move a file inside an existing dir on the same filesystem. self._check_move_file(self.src_file, self.dst_dir, self.dst_file) + def test_move_file_to_dir_pathlike_src(self): + # Move a pathlike file to another location on the same filesystem. + src = FakePath(self.src_file) + self._check_move_file(src, self.dst_dir, self.dst_file) + + def test_move_file_to_dir_pathlike_dst(self): + # Move a file to another pathlike location on the same filesystem. + dst = FakePath(self.dst_dir) + self._check_move_file(self.src_file, dst, self.dst_file) + @mock_rename def test_move_file_other_fs(self): # Move a file to an existing dir on another filesystem. @@ -1900,14 +2761,11 @@ def test_move_file_to_dir_other_fs(self): def test_move_dir(self): # Move a dir to another location on the same filesystem. - dst_dir = tempfile.mktemp() + dst_dir = tempfile.mktemp(dir=self.mkdtemp()) try: self._check_move_dir(self.src_dir, dst_dir, dst_dir) finally: - try: - shutil.rmtree(dst_dir) - except: - pass + os_helper.rmtree(dst_dir) @mock_rename def test_move_dir_other_fs(self): @@ -1935,8 +2793,7 @@ def test_move_dir_altsep_to_dir(self): def test_existing_file_inside_dest_dir(self): # A file with the same name inside the destination dir already exists. - with open(self.dst_file, "wb"): - pass + create_file(self.dst_file) self.assertRaises(shutil.Error, shutil.move, self.src_file, self.dst_dir) def test_dont_move_dir_in_itself(self): @@ -1954,7 +2811,7 @@ def test_destinsrc_false_negative(self): msg='_destinsrc() wrongly concluded that ' 'dst (%s) is not in src (%s)' % (dst, src)) finally: - shutil.rmtree(TESTFN, ignore_errors=True) + os_helper.rmtree(TESTFN) def test_destinsrc_false_positive(self): os.mkdir(TESTFN) @@ -1966,7 +2823,7 @@ def test_destinsrc_false_positive(self): msg='_destinsrc() wrongly concluded that ' 'dst (%s) is in src (%s)' % (dst, src)) finally: - shutil.rmtree(TESTFN, ignore_errors=True) + os_helper.rmtree(TESTFN) @os_helper.skip_unless_symlink @mock_rename @@ -2038,10 +2895,88 @@ def _copy(src, dst): shutil.move(self.src_dir, self.dst_dir, copy_function=_copy) self.assertEqual(len(moved), 3) + def test_move_dir_caseinsensitive(self): + # Renames a folder to the same name + # but a different case. + + self.src_dir = self.mkdtemp() + dst_dir = os.path.join( + os.path.dirname(self.src_dir), + os.path.basename(self.src_dir).upper()) + self.assertNotEqual(self.src_dir, dst_dir) + + try: + shutil.move(self.src_dir, dst_dir) + self.assertTrue(os.path.isdir(dst_dir)) + finally: + os.rmdir(dst_dir) + + # bpo-26791: Check that a symlink to a directory can + # be moved into that directory. + @mock_rename + def _test_move_symlink_to_dir_into_dir(self, dst): + src = os.path.join(self.src_dir, 'linktodir') + dst_link = os.path.join(self.dst_dir, 'linktodir') + os.symlink(self.dst_dir, src, target_is_directory=True) + shutil.move(src, dst) + self.assertTrue(os.path.islink(dst_link)) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertFalse(os.path.exists(src)) + + # Repeat the move operation with the destination + # symlink already in place (should raise shutil.Error). + os.symlink(self.dst_dir, src, target_is_directory=True) + with self.assertRaises(shutil.Error): + shutil.move(src, dst) + self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) + self.assertTrue(os.path.exists(src)) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_dir(self): + self._test_move_symlink_to_dir_into_dir(self.dst_dir) + + @os_helper.skip_unless_symlink + def test_move_symlink_to_dir_into_symlink_to_dir(self): + dst = os.path.join(self.src_dir, 'otherlinktodir') + os.symlink(self.dst_dir, dst, target_is_directory=True) + self._test_move_symlink_to_dir_into_dir(dst) + + @os_helper.skip_unless_dac_override + @unittest.skipUnless(hasattr(os, 'lchflags') + and hasattr(stat, 'SF_IMMUTABLE') + and hasattr(stat, 'UF_OPAQUE'), + 'requires lchflags') + def test_move_dir_permission_denied(self): + # bpo-42782: shutil.move should not create destination directories + # if the source directory cannot be removed. + try: + os.mkdir(TESTFN_SRC) + os.lchflags(TESTFN_SRC, stat.SF_IMMUTABLE) + + # Testing on an empty immutable directory + # TESTFN_DST should not exist if shutil.move failed + self.assertRaises(PermissionError, shutil.move, TESTFN_SRC, TESTFN_DST) + self.assertFalse(TESTFN_DST in os.listdir()) + + # Create a file and keep the directory immutable + os.lchflags(TESTFN_SRC, stat.UF_OPAQUE) + os_helper.create_empty_file(os.path.join(TESTFN_SRC, 'child')) + os.lchflags(TESTFN_SRC, stat.SF_IMMUTABLE) + + # Testing on a non-empty immutable directory + # TESTFN_DST should not exist if shutil.move failed + self.assertRaises(PermissionError, shutil.move, TESTFN_SRC, TESTFN_DST) + self.assertFalse(TESTFN_DST in os.listdir()) + finally: + if os.path.exists(TESTFN_SRC): + os.lchflags(TESTFN_SRC, stat.UF_OPAQUE) + os_helper.rmtree(TESTFN_SRC) + if os.path.exists(TESTFN_DST): + os.lchflags(TESTFN_DST, stat.UF_OPAQUE) + os_helper.rmtree(TESTFN_DST) -class TestCopyFile(unittest.TestCase): - _delete = False +class TestCopyFile(unittest.TestCase): class Faux(object): _entered = False @@ -2061,27 +2996,18 @@ def __exit__(self, exc_type, exc_val, exc_tb): raise OSError("Cannot close") return self._suppress_at_exit - def tearDown(self): - if self._delete: - del shutil.open - - def _set_shutil_open(self, func): - shutil.open = func - self._delete = True - def test_w_source_open_fails(self): def _open(filename, mode='r'): if filename == 'srcfile': raise OSError('Cannot open "srcfile"') assert 0 # shouldn't reach here. - self._set_shutil_open(_open) - - self.assertRaises(OSError, shutil.copyfile, 'srcfile', 'destfile') + with support.swap_attr(shutil, 'open', _open): + with self.assertRaises(OSError): + shutil.copyfile('srcfile', 'destfile') @unittest.skipIf(MACOS, "skipped on macOS") def test_w_dest_open_fails(self): - srcfile = self.Faux() def _open(filename, mode='r'): @@ -2091,9 +3017,8 @@ def _open(filename, mode='r'): raise OSError('Cannot open "destfile"') assert 0 # shouldn't reach here. - self._set_shutil_open(_open) - - shutil.copyfile('srcfile', 'destfile') + with support.swap_attr(shutil, 'open', _open): + shutil.copyfile('srcfile', 'destfile') self.assertTrue(srcfile._entered) self.assertTrue(srcfile._exited_with[0] is OSError) self.assertEqual(srcfile._exited_with[1].args, @@ -2101,7 +3026,6 @@ def _open(filename, mode='r'): @unittest.skipIf(MACOS, "skipped on macOS") def test_w_dest_close_fails(self): - srcfile = self.Faux() destfile = self.Faux(True) @@ -2112,9 +3036,8 @@ def _open(filename, mode='r'): return destfile assert 0 # shouldn't reach here. - self._set_shutil_open(_open) - - shutil.copyfile('srcfile', 'destfile') + with support.swap_attr(shutil, 'open', _open): + shutil.copyfile('srcfile', 'destfile') self.assertTrue(srcfile._entered) self.assertTrue(destfile._entered) self.assertTrue(destfile._raised) @@ -2135,33 +3058,15 @@ def _open(filename, mode='r'): return destfile assert 0 # shouldn't reach here. - self._set_shutil_open(_open) - - self.assertRaises(OSError, - shutil.copyfile, 'srcfile', 'destfile') + with support.swap_attr(shutil, 'open', _open): + with self.assertRaises(OSError): + shutil.copyfile('srcfile', 'destfile') self.assertTrue(srcfile._entered) self.assertTrue(destfile._entered) self.assertFalse(destfile._raised) self.assertTrue(srcfile._exited_with[0] is None) self.assertTrue(srcfile._raised) - def test_move_dir_caseinsensitive(self): - # Renames a folder to the same name - # but a different case. - - self.src_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.src_dir, True) - dst_dir = os.path.join( - os.path.dirname(self.src_dir), - os.path.basename(self.src_dir).upper()) - self.assertNotEqual(self.src_dir, dst_dir) - - try: - shutil.move(self.src_dir, dst_dir) - self.assertTrue(os.path.isdir(dst_dir)) - finally: - os.rmdir(dst_dir) - class TestCopyFileObj(unittest.TestCase): FILESIZE = 2 * 1024 * 1024 @@ -2218,7 +3123,7 @@ def test_win_impl(self): # If file size < 1 MiB memoryview() length must be equal to # the actual file size. - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False) as f: f.write(b'foo') fname = f.name self.addCleanup(os_helper.unlink, fname) @@ -2227,7 +3132,7 @@ def test_win_impl(self): self.assertEqual(m.call_args[0][2], 3) # Empty files should not rely on readinto() variant. - with tempfile.NamedTemporaryFile(delete=False) as f: + with tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False) as f: pass fname = f.name self.addCleanup(os_helper.unlink, fname) @@ -2287,13 +3192,13 @@ def test_regular_copy(self): def test_same_file(self): self.addCleanup(self.reset) with self.get_files() as (src, dst): - with self.assertRaises(Exception): + with self.assertRaises((OSError, _GiveupOnFastCopy)): self.zerocopy_fun(src, src) # Make sure src file is not corrupted. self.assertEqual(read_file(TESTFN, binary=True), self.FILEDATA) def test_non_existent_src(self): - name = tempfile.mktemp() + name = tempfile.mktemp(dir=os.getcwd()) with self.assertRaises(FileNotFoundError) as cm: shutil.copyfile(name, "new") self.assertEqual(cm.exception.filename, name) @@ -2303,8 +3208,7 @@ def test_empty_file(self): dstname = TESTFN + 'dst' self.addCleanup(lambda: os_helper.unlink(srcname)) self.addCleanup(lambda: os_helper.unlink(dstname)) - with open(srcname, "wb"): - pass + create_file(srcname) with open(srcname, "rb") as src: with open(dstname, "wb") as dst: @@ -2318,7 +3222,6 @@ def test_unhandled_exception(self): self.assertRaises(ZeroDivisionError, shutil.copyfile, TESTFN, TESTFN2) - @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON, OSError.error on macOS") def test_exception_on_first_call(self): # Emulate a case where the first call to the zero-copy # function raises an exception in which case the function is @@ -2329,7 +3232,6 @@ def test_exception_on_first_call(self): with self.assertRaises(_GiveupOnFastCopy): self.zerocopy_fun(src, dst) - @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON, OSError.error on macOS") def test_filesystem_full(self): # Emulate a case where filesystem is full and sendfile() fails # on first call. @@ -2339,12 +3241,8 @@ def test_filesystem_full(self): self.assertRaises(OSError, self.zerocopy_fun, src, dst) -@unittest.skipIf(not SUPPORTS_SENDFILE, 'os.sendfile() not supported') -class TestZeroCopySendfile(_ZeroCopyFileTest, unittest.TestCase): - PATCHPOINT = "os.sendfile" - - def zerocopy_fun(self, fsrc, fdst): - return shutil._fastcopy_sendfile(fsrc, fdst) +class _ZeroCopyFileLinuxTest(_ZeroCopyFileTest): + BLOCKSIZE_INDEX = None def test_non_regular_file_src(self): with io.BytesIO(self.FILEDATA) as src: @@ -2365,77 +3263,87 @@ def test_non_regular_file_dst(self): self.assertEqual(dst.read(), self.FILEDATA) def test_exception_on_second_call(self): - def sendfile(*args, **kwargs): + def syscall(*args, **kwargs): if not flag: flag.append(None) - return orig_sendfile(*args, **kwargs) + return orig_syscall(*args, **kwargs) else: raise OSError(errno.EBADF, "yo") flag = [] - orig_sendfile = os.sendfile - with unittest.mock.patch('os.sendfile', create=True, - side_effect=sendfile): + orig_syscall = eval(self.PATCHPOINT) + with unittest.mock.patch(self.PATCHPOINT, create=True, + side_effect=syscall): with self.get_files() as (src, dst): with self.assertRaises(OSError) as cm: - shutil._fastcopy_sendfile(src, dst) + self.zerocopy_fun(src, dst) assert flag self.assertEqual(cm.exception.errno, errno.EBADF) def test_cant_get_size(self): # Emulate a case where src file size cannot be determined. # Internally bufsize will be set to a small value and - # sendfile() will be called repeatedly. + # a system call will be called repeatedly. with unittest.mock.patch('os.fstat', side_effect=OSError) as m: with self.get_files() as (src, dst): - shutil._fastcopy_sendfile(src, dst) + self.zerocopy_fun(src, dst) assert m.called self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA) def test_small_chunks(self): # Force internal file size detection to be smaller than the - # actual file size. We want to force sendfile() to be called + # actual file size. We want to force a system call to be called # multiple times, also in order to emulate a src fd which gets # bigger while it is being copied. mock = unittest.mock.Mock() mock.st_size = 65536 + 1 with unittest.mock.patch('os.fstat', return_value=mock) as m: with self.get_files() as (src, dst): - shutil._fastcopy_sendfile(src, dst) + self.zerocopy_fun(src, dst) assert m.called self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA) def test_big_chunk(self): # Force internal file size detection to be +100MB bigger than - # the actual file size. Make sure sendfile() does not rely on + # the actual file size. Make sure a system call does not rely on # file size value except for (maybe) a better throughput / # performance. mock = unittest.mock.Mock() mock.st_size = self.FILESIZE + (100 * 1024 * 1024) with unittest.mock.patch('os.fstat', return_value=mock) as m: with self.get_files() as (src, dst): - shutil._fastcopy_sendfile(src, dst) + self.zerocopy_fun(src, dst) assert m.called self.assertEqual(read_file(TESTFN2, binary=True), self.FILEDATA) def test_blocksize_arg(self): - with unittest.mock.patch('os.sendfile', + with unittest.mock.patch(self.PATCHPOINT, side_effect=ZeroDivisionError) as m: self.assertRaises(ZeroDivisionError, shutil.copyfile, TESTFN, TESTFN2) - blocksize = m.call_args[0][3] + blocksize = m.call_args[0][self.BLOCKSIZE_INDEX] # Make sure file size and the block size arg passed to # sendfile() are the same. self.assertEqual(blocksize, os.path.getsize(TESTFN)) # ...unless we're dealing with a small file. os_helper.unlink(TESTFN2) - write_file(TESTFN2, b"hello", binary=True) + create_file(TESTFN2, b"hello") self.addCleanup(os_helper.unlink, TESTFN2 + '3') self.assertRaises(ZeroDivisionError, shutil.copyfile, TESTFN2, TESTFN2 + '3') - blocksize = m.call_args[0][3] + blocksize = m.call_args[0][self.BLOCKSIZE_INDEX] self.assertEqual(blocksize, 2 ** 23) + +@unittest.skipIf(not SUPPORTS_SENDFILE, 'os.sendfile() not supported') +@unittest.mock.patch.object(shutil, "_USE_CP_COPY_FILE_RANGE", False) +class TestZeroCopySendfile(_ZeroCopyFileLinuxTest, unittest.TestCase): + PATCHPOINT = "os.sendfile" + BLOCKSIZE_INDEX = 3 + + def zerocopy_fun(self, fsrc, fdst): + return shutil._fastcopy_sendfile(fsrc, fdst) + def test_file2file_not_supported(self): # Emulate a case where sendfile() only support file->socket # fds. In such a case copyfile() is supposed to skip the @@ -2458,6 +3366,29 @@ def test_file2file_not_supported(self): shutil._USE_CP_SENDFILE = True +@unittest.skipUnless(shutil._USE_CP_COPY_FILE_RANGE, "os.copy_file_range() not supported") +class TestZeroCopyCopyFileRange(_ZeroCopyFileLinuxTest, unittest.TestCase): + PATCHPOINT = "os.copy_file_range" + BLOCKSIZE_INDEX = 2 + + def zerocopy_fun(self, fsrc, fdst): + return shutil._fastcopy_copy_file_range(fsrc, fdst) + + def test_empty_file(self): + srcname = f"{TESTFN}src" + dstname = f"{TESTFN}dst" + self.addCleanup(lambda: os_helper.unlink(srcname)) + self.addCleanup(lambda: os_helper.unlink(dstname)) + with open(srcname, "wb"): + pass + + with open(srcname, "rb") as src, open(dstname, "wb") as dst: + # _fastcopy_copy_file_range gives up copying empty files due + # to a bug in older Linux. + with self.assertRaises(shutil._GiveupOnFastCopy): + self.zerocopy_fun(src, dst) + + @unittest.skipIf(not MACOS, 'macOS only') class TestZeroCopyMACOS(_ZeroCopyFileTest, unittest.TestCase): PATCHPOINT = "posix._fcopyfile" @@ -2466,7 +3397,7 @@ def zerocopy_fun(self, src, dst): return shutil._fastcopy_fcopyfile(src, dst, posix._COPYFILE_DATA) -class TermsizeTests(unittest.TestCase): +class TestGetTerminalSize(unittest.TestCase): def test_does_not_crash(self): """Check if get_terminal_size() returns a meaningful value. @@ -2501,6 +3432,7 @@ def test_bad_environ(self): self.assertGreaterEqual(size.lines, 0) @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty") + @support.requires_subprocess() @unittest.skipUnless(hasattr(os, 'get_terminal_size'), 'need os.get_terminal_size()') def test_stty_match(self): @@ -2518,16 +3450,15 @@ def test_stty_match(self): expected = (int(size[1]), int(size[0])) # reversed order with os_helper.EnvironmentVarGuard() as env: - del env['LINES'] - del env['COLUMNS'] + env.unset('LINES', 'COLUMNS') actual = shutil.get_terminal_size() self.assertEqual(expected, actual) + @unittest.skipIf(support.is_wasi, "WASI has no /dev/null") def test_fallback(self): with os_helper.EnvironmentVarGuard() as env: - del env['LINES'] - del env['COLUMNS'] + env.unset('LINES', 'COLUMNS') # sys.__stdout__ has no fileno() with support.swap_attr(sys, '__stdout__', None): @@ -2537,7 +3468,7 @@ def test_fallback(self): # sys.__stdout__ is not a terminal on Unix # or fileno() not in (0, 1, 2) on Windows - with open(os.devnull, 'w') as f, \ + with open(os.devnull, 'w', encoding='utf-8') as f, \ support.swap_attr(sys, '__stdout__', f): size = shutil.get_terminal_size(fallback=(30, 40)) self.assertEqual(size.columns, 30) @@ -2548,10 +3479,10 @@ class PublicAPITests(unittest.TestCase): """Ensures that the correct values are exposed in the public API.""" def test_module_all_attribute(self): - self.assertTrue(hasattr(shutil, '__all__')) + self.assertHasAttr(shutil, '__all__') target_api = ['copyfileobj', 'copyfile', 'copymode', 'copystat', 'copy', 'copy2', 'copytree', 'move', 'rmtree', 'Error', - 'SpecialFileError', 'ExecError', 'make_archive', + 'SpecialFileError', 'make_archive', 'get_archive_formats', 'register_archive_format', 'unregister_archive_format', 'get_unpack_formats', 'register_unpack_format', 'unregister_unpack_format', @@ -2560,6 +3491,8 @@ def test_module_all_attribute(self): if hasattr(os, 'statvfs') or os.name == 'nt': target_api.append('disk_usage') self.assertEqual(set(shutil.__all__), set(target_api)) + with self.assertWarns(DeprecationWarning): + from shutil import ExecError if __name__ == '__main__': diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index e59f6091214..07fc97cb6a1 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1,4 +1,6 @@ +import enum import errno +import functools import inspect import os import random @@ -11,7 +13,9 @@ import time import unittest from test import support -from test.support import os_helper +from test.support import ( + is_apple, is_apple_mobile, os_helper, threading_helper +) from test.support.script_helper import assert_python_ok, spawn_python try: import _testcapi @@ -34,6 +38,32 @@ def test_enums(self): self.assertIsInstance(sig, signal.Signals) self.assertEqual(sys.platform, "win32") + CheckedSignals = enum._old_convert_( + enum.IntEnum, 'Signals', 'signal', + lambda name: + name.isupper() + and (name.startswith('SIG') and not name.startswith('SIG_')) + or name.startswith('CTRL_'), + source=signal, + ) + enum._test_simple_enum(CheckedSignals, signal.Signals) + + CheckedHandlers = enum._old_convert_( + enum.IntEnum, 'Handlers', 'signal', + lambda name: name in ('SIG_DFL', 'SIG_IGN'), + source=signal, + ) + enum._test_simple_enum(CheckedHandlers, signal.Handlers) + + Sigmasks = getattr(signal, 'Sigmasks', None) + if Sigmasks is not None: + CheckedSigmasks = enum._old_convert_( + enum.IntEnum, 'Sigmasks', 'signal', + lambda name: name in ('SIG_BLOCK', 'SIG_UNBLOCK', 'SIG_SETMASK'), + source=signal, + ) + enum._test_simple_enum(CheckedSigmasks, Sigmasks) + def test_functions_module_attr(self): # Issue #27718: If __all__ is not defined all non-builtin functions # should have correct __module__ to be displayed by pydoc. @@ -48,8 +78,9 @@ class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + def create_handler_with_partial(self, argument): + return functools.partial(self.trivial_signal_handler, argument) + def test_out_of_range_signal_number_raises_error(self): self.assertRaises(ValueError, signal.getsignal, 4242) @@ -70,23 +101,45 @@ def test_getsignal(self): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_no_repr_is_called_on_signal_handler(self): + # See https://github.com/python/cpython/issues/112559. + + class MyArgument: + def __init__(self): + self.repr_count = 0 + + def __repr__(self): + self.repr_count += 1 + return super().__repr__() + + argument = MyArgument() + self.assertEqual(0, argument.repr_count) + + handler = self.create_handler_with_partial(argument) + hup = signal.signal(signal.SIGHUP, handler) + self.assertIsInstance(hup, signal.Handlers) + self.assertEqual(signal.getsignal(signal.SIGHUP), handler) + signal.signal(signal.SIGHUP, hup) + self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + self.assertEqual(0, argument.repr_count) + + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124083: strsignal is not supported on NetBSD") def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) self.assertIn("Hangup", signal.strsignal(signal.SIGHUP)) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Issue 3864, unknown if this affects earlier versions of freebsd also def test_interprocess_signal(self): dirname = os.path.dirname(__file__) script = os.path.join(dirname, 'signalinterproctester.py') assert_python_ok(script) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipUnless( + hasattr(signal, "valid_signals"), + "requires signal.valid_signals" + ) def test_valid_signals(self): s = signal.valid_signals() self.assertIsInstance(s, set) @@ -96,7 +149,21 @@ def test_valid_signals(self): self.assertNotIn(signal.NSIG, s) self.assertLess(len(s), signal.NSIG) + # gh-91145: Make sure that all SIGxxx constants exposed by the Python + # signal module have a number in the [0; signal.NSIG-1] range. + for name in dir(signal): + if not name.startswith("SIG"): + continue + if name in {"SIG_IGN", "SIG_DFL"}: + # SIG_IGN and SIG_DFL are pointers + continue + with self.subTest(name=name): + signum = getattr(signal, name) + self.assertGreaterEqual(signum, 0) + self.assertLess(signum, signal.NSIG) + @unittest.skipUnless(sys.executable, "sys.executable required.") + @support.requires_subprocess() def test_keyboard_interrupt_exit_code(self): """KeyboardInterrupt triggers exit via SIGINT.""" process = subprocess.run( @@ -116,8 +183,6 @@ def test_keyboard_interrupt_exit_code(self): @unittest.skipUnless(sys.platform == "win32", "Windows specific") class WindowsSignalTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_valid_signals(self): s = signal.valid_signals() self.assertIsInstance(s, set) @@ -127,8 +192,6 @@ def test_valid_signals(self): self.assertNotIn(signal.NSIG, s) self.assertLess(len(s), signal.NSIG) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue9324(self): # Updated for issue #10003, adding SIGBREAK handler = lambda x, y: None @@ -150,9 +213,8 @@ def test_issue9324(self): with self.assertRaises(ValueError): signal.signal(7, handler) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(sys.executable, "sys.executable required.") + @support.requires_subprocess() def test_keyboard_interrupt_exit_code(self): """KeyboardInterrupt triggers an exit using STATUS_CONTROL_C_EXIT.""" # We don't test via os.kill(os.getpid(), signal.CTRL_C_EVENT) here @@ -178,13 +240,12 @@ def test_invalid_call(self): with self.assertRaises(TypeError): signal.set_wakeup_fd(signal.SIGINT, False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_invalid_fd(self): fd = os_helper.make_bad_fd() self.assertRaises((ValueError, OSError), signal.set_wakeup_fd, fd) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.skipUnless(support.has_socket_support, "needs working sockets.") def test_invalid_socket(self): sock = socket.socket() fd = sock.fileno() @@ -192,7 +253,7 @@ def test_invalid_socket(self): self.assertRaises((ValueError, OSError), signal.set_wakeup_fd, fd) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_set_wakeup_fd_result(self): r1, w1 = os.pipe() self.addCleanup(os.close, r1) @@ -210,7 +271,7 @@ def test_set_wakeup_fd_result(self): self.assertEqual(signal.set_wakeup_fd(-1), w2) self.assertEqual(signal.set_wakeup_fd(-1), -1) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.skipUnless(support.has_socket_support, "needs working sockets.") def test_set_wakeup_fd_socket_result(self): sock1 = socket.socket() self.addCleanup(sock1.close) @@ -230,6 +291,7 @@ def test_set_wakeup_fd_socket_result(self): # On Windows, files are always blocking and Windows does not provide a # function to test if a socket is in non-blocking mode. @unittest.skipIf(sys.platform == "win32", "tests specific to POSIX") + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_set_wakeup_fd_blocking(self): rfd, wfd = os.pipe() self.addCleanup(os.close, rfd) @@ -290,6 +352,7 @@ def check_signum(signals): assert_python_ok('-c', code) @unittest.skipIf(_testcapi is None, 'need _testcapi') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_wakeup_write_error(self): # Issue #16105: write() errors in the C signal handler should not # pass silently. @@ -317,7 +380,7 @@ def handler(signum, frame): except ZeroDivisionError: # An ignored exception should have been printed out on stderr err = err.getvalue() - if ('Exception ignored when trying to write to the signal wakeup fd' + if ('Exception ignored while trying to write to the signal wakeup fd' not in err): raise AssertionError(err) if ('OSError: [Errno %d]' % errno.EBADF) not in err: @@ -506,7 +569,7 @@ def handler(signum, frame): signal.raise_signal(signum) err = err.getvalue() - if ('Exception ignored when trying to {action} to the signal wakeup fd' + if ('Exception ignored while trying to {action} to the signal wakeup fd' not in err): raise AssertionError(err) """.format(action=action) @@ -576,7 +639,7 @@ def handler(signum, frame): "buffer" % written) # By default, we get a warning when a signal arrives - msg = ('Exception ignored when trying to {action} ' + msg = ('Exception ignored while trying to {action} ' 'to the signal wakeup fd') signal.set_wakeup_fd(write.fileno()) @@ -628,9 +691,11 @@ def handler(signum, frame): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") @unittest.skipUnless(hasattr(signal, 'siginterrupt'), "needs signal.siginterrupt()") +@support.requires_subprocess() +@unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") class SiginterruptTest(unittest.TestCase): - def readpipe_interrupted(self, interrupt): + def readpipe_interrupted(self, interrupt, timeout=support.SHORT_TIMEOUT): """Perform a read during which a signal will arrive. Return True if the read is interrupted by the signal and raises an exception. Return False if it returns normally. @@ -678,7 +743,7 @@ def handler(signum, frame): # wait until the child process is loaded and has started first_line = process.stdout.readline() - stdout, stderr = process.communicate(timeout=support.SHORT_TIMEOUT) + stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired: process.kill() return False @@ -690,8 +755,6 @@ def handler(signum, frame): % (exitcode, stdout)) return (exitcode == 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_without_siginterrupt(self): # If a signal handler is installed and siginterrupt is not called # at all, when that signal arrives, it interrupts a syscall that's in @@ -699,8 +762,6 @@ def test_without_siginterrupt(self): interrupted = self.readpipe_interrupted(None) self.assertTrue(interrupted) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_siginterrupt_on(self): # If a signal handler is installed and siginterrupt is called with # a true value for the second argument, when that signal arrives, it @@ -708,11 +769,12 @@ def test_siginterrupt_on(self): interrupted = self.readpipe_interrupted(True) self.assertTrue(interrupted) + @support.requires_resource('walltime') def test_siginterrupt_off(self): # If a signal handler is installed and siginterrupt is called with # a false value for the second argument, when that signal arrives, it # does not interrupt a syscall that's in progress. - interrupted = self.readpipe_interrupted(False) + interrupted = self.readpipe_interrupted(False, timeout=2) self.assertFalse(interrupted) @@ -752,8 +814,6 @@ def sig_prof(self, *args): self.hndl_called = True signal.setitimer(signal.ITIMER_PROF, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_exc(self): # XXX I'm assuming -1 is an invalid itimer, but maybe some platform # defines it ? @@ -763,63 +823,49 @@ def test_itimer_exc(self): self.assertRaises(signal.ItimerError, signal.setitimer, signal.ITIMER_REAL, -1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_real(self): self.itimer = signal.ITIMER_REAL signal.setitimer(self.itimer, 1.0) signal.pause() self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform in ('netbsd5',), + @unittest.skipIf(sys.platform in ('netbsd5',) or is_apple_mobile, 'itimer not reliable (does not mix well with threading) on some BSDs.') def test_itimer_virtual(self): self.itimer = signal.ITIMER_VIRTUAL signal.signal(signal.SIGVTALRM, self.sig_vtalrm) - signal.setitimer(self.itimer, 0.3, 0.2) + signal.setitimer(self.itimer, 0.001, 0.001) - start_time = time.monotonic() - while time.monotonic() - start_time < 60.0: + for _ in support.busy_retry(support.LONG_TIMEOUT): # use up some virtual time by doing real work - _ = pow(12345, 67890, 10000019) + _ = sum(i * i for i in range(10**5)) if signal.getitimer(self.itimer) == (0.0, 0.0): - break # sig_vtalrm handler stopped this itimer - else: # Issue 8424 - self.skipTest("timeout: likely cause: machine too slow or load too " - "high") + # sig_vtalrm handler stopped this itimer + break # virtual itimer should be (0.0, 0.0) now self.assertEqual(signal.getitimer(self.itimer), (0.0, 0.0)) # and the handler should have been called self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_prof(self): self.itimer = signal.ITIMER_PROF signal.signal(signal.SIGPROF, self.sig_prof) signal.setitimer(self.itimer, 0.2, 0.2) - start_time = time.monotonic() - while time.monotonic() - start_time < 60.0: + for _ in support.busy_retry(support.LONG_TIMEOUT): # do some work - _ = pow(12345, 67890, 10000019) + _ = sum(i * i for i in range(10**5)) if signal.getitimer(self.itimer) == (0.0, 0.0): - break # sig_prof handler stopped this itimer - else: # Issue 8424 - self.skipTest("timeout: likely cause: machine too slow or load too " - "high") + # sig_prof handler stopped this itimer + break # profiling itimer should be (0.0, 0.0) now self.assertEqual(signal.getitimer(self.itimer), (0.0, 0.0)) # and the handler should have been called self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_setitimer_tiny(self): # bpo-30807: C setitimer() takes a microsecond-resolution interval. # Check that float -> timeval conversion doesn't round @@ -873,6 +919,7 @@ def handler(signum, frame): @unittest.skipUnless(hasattr(signal, 'pthread_kill'), 'need signal.pthread_kill()') + @threading_helper.requires_working_threading() def test_pthread_kill(self): code = """if 1: import signal @@ -1009,6 +1056,7 @@ def test_sigtimedwait_negative_timeout(self): 'need signal.sigwait()') @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), 'need signal.pthread_sigmask()') + @threading_helper.requires_working_threading() def test_sigwait_thread(self): # Check that calling sigwait() from a thread doesn't suspend the whole # process. A new interpreter is spawned to avoid problems when mixing @@ -1064,6 +1112,7 @@ def test_pthread_sigmask_valid_signals(self): @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), 'need signal.pthread_sigmask()') + @threading_helper.requires_working_threading() def test_pthread_sigmask(self): code = """if 1: import signal @@ -1141,6 +1190,7 @@ def read_sigmask(): @unittest.skipUnless(hasattr(signal, 'pthread_kill'), 'need signal.pthread_kill()') + @threading_helper.requires_working_threading() def test_pthread_kill_main_thread(self): # Test that a signal can be sent to the main thread with pthread_kill() # before any other thread has been created (see issue #12392). @@ -1272,30 +1322,34 @@ def test_stress_delivery_simultaneous(self): def handler(signum, frame): sigs.append(signum) - self.setsig(signal.SIGUSR1, handler) + # On Android, SIGUSR1 is unreliable when used in close proximity to + # another signal – see Android/testbed/app/src/main/python/main.py. + # So we use a different signal. + self.setsig(signal.SIGUSR2, handler) self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL expected_sigs = 0 - deadline = time.monotonic() + support.SHORT_TIMEOUT - while expected_sigs < N: # Hopefully the SIGALRM will be received somewhere during - # initial processing of SIGUSR1. + # initial processing of SIGUSR2. signal.setitimer(signal.ITIMER_REAL, 1e-6 + random.random() * 1e-5) - os.kill(os.getpid(), signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGUSR2) expected_sigs += 2 # Wait for handlers to run to avoid signal coalescing - while len(sigs) < expected_sigs and time.monotonic() < deadline: - time.sleep(1e-5) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(sigs) >= expected_sigs: + break # All ITIMER_REAL signals should have been delivered to the # Python handler self.assertEqual(len(sigs), N, "Some signals were lost") - @unittest.skip("TODO: RUSTPYTHON; hang") + @support.requires_gil_enabled("gh-121065: test is flaky on free-threaded build") + @unittest.skipIf(is_apple, "crashes due to system bug (FB13453490)") @unittest.skipUnless(hasattr(signal, "SIGUSR1"), "test needs SIGUSR1") + @threading_helper.requires_working_threading() def test_stress_modifying_handlers(self): # bpo-43406: race condition between trip_signal() and signal.signal signum = signal.SIGUSR1 @@ -1314,7 +1368,7 @@ def set_interrupts(): num_sent_signals += 1 def cycle_handlers(): - while num_sent_signals < 100: + while num_sent_signals < 100 or num_received_signals < 1: for i in range(20000): # Cycle between a Python-defined and a non-Python handler for handler in [custom_handler, signal.SIG_IGN]: @@ -1347,7 +1401,7 @@ def cycle_handlers(): if not ignored: # Sanity check that some signals were received, but not all self.assertGreater(num_received_signals, 0) - self.assertLess(num_received_signals, num_sent_signals) + self.assertLessEqual(num_received_signals, num_sent_signals) finally: do_stop = True t.join() @@ -1355,14 +1409,10 @@ def cycle_handlers(): class RaiseSignalTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_sigint(self): with self.assertRaises(KeyboardInterrupt): signal.raise_signal(signal.SIGINT) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform != "win32", "Windows specific test") def test_invalid_argument(self): try: @@ -1375,8 +1425,6 @@ def test_invalid_argument(self): else: raise - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_handler(self): is_ok = False def handler(a, b): @@ -1388,6 +1436,22 @@ def handler(a, b): signal.raise_signal(signal.SIGINT) self.assertTrue(is_ok) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test__thread_interrupt_main(self): + # See https://github.com/python/cpython/issues/102397 + code = """if 1: + import _thread + class Foo(): + def __del__(self): + _thread.interrupt_main() + + x = Foo() + """ + + rc, out, err = assert_python_ok('-c', code) + self.assertIn(b'OSError: Signal 2 ignored due to race condition', err) + + class PidfdSignalTest(unittest.TestCase): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index fb9aca659f6..01951e6247b 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -7,22 +7,26 @@ import unittest import test.support from test import support +from test.support.script_helper import assert_python_ok +from test.support import import_helper from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr -from test.support.os_helper import TESTFN, EnvironmentVarGuard, change_cwd +from test.support.os_helper import TESTFN, EnvironmentVarGuard +from test.support.script_helper import spawn_python, kill_python import ast import builtins -import encodings import glob import io import os import re import shutil +import stat import subprocess import sys import sysconfig import tempfile +from textwrap import dedent import urllib.error import urllib.request from unittest import mock @@ -195,6 +199,45 @@ def test_addsitedir(self): finally: pth_file.cleanup() + def test_addsitedir_dotfile(self): + pth_file = PthFile('.dotfile') + pth_file.cleanup(prep=True) + try: + pth_file.create() + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + + @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') + def test_addsitedir_hidden_flags(self): + pth_file = PthFile() + pth_file.cleanup(prep=True) + try: + pth_file.create() + st = os.stat(pth_file.file_path) + os.chflags(pth_file.file_path, st.st_flags | stat.UF_HIDDEN) + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + + @unittest.skipUnless(sys.platform == 'win32', 'test needs Windows') + @support.requires_subprocess() + def test_addsitedir_hidden_file_attribute(self): + pth_file = PthFile() + pth_file.cleanup(prep=True) + try: + pth_file.create() + subprocess.check_call(['attrib', '+H', pth_file.file_path]) + site.addsitedir(pth_file.base_dir, set()) + self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) + self.assertIn(pth_file.base_dir, sys.path) + finally: + pth_file.cleanup() + # This tests _getuserbase, hence the double underline # to distinguish from a test for getuserbase def test__getuserbase(self): @@ -266,8 +309,7 @@ def test_getuserbase(self): with EnvironmentVarGuard() as environ: environ['PYTHONUSERBASE'] = 'xoxo' - self.assertTrue(site.getuserbase().startswith('xoxo'), - site.getuserbase()) + self.assertTrue(site.getuserbase().startswith('xoxo')) @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_getusersitepackages(self): @@ -277,7 +319,7 @@ def test_getusersitepackages(self): # the call sets USER_BASE *and* USER_SITE self.assertEqual(site.USER_SITE, user_site) - self.assertTrue(user_site.startswith(site.USER_BASE), user_site) + self.assertTrue(user_site.startswith(site.USER_BASE)) self.assertEqual(site.USER_BASE, site.getuserbase()) def test_getsitepackages(self): @@ -289,14 +331,14 @@ def test_getsitepackages(self): self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, # XXX: RUSTPYTHON - 'rustpython%d.%d' % sys.version_info[:2], + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', # XXX: RUSTPYTHON - 'rustpython%d.%d' % sys.version_info[:2], + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: @@ -317,16 +359,13 @@ def test_no_home_directory(self): with EnvironmentVarGuard() as environ, \ mock.patch('os.path.expanduser', lambda path: path): - - del environ['PYTHONUSERBASE'] - del environ['APPDATA'] + environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() - self.assertTrue(user_base.startswith('~' + os.sep), - user_base) + self.assertTrue(user_base.startswith('~' + os.sep)) user_site = site.getusersitepackages() - self.assertTrue(user_site.startswith(user_base), user_site) + self.assertTrue(user_site.startswith(user_base)) with mock.patch('os.path.isdir', return_value=False) as mock_isdir, \ mock.patch.object(site, 'addsitedir') as mock_addsitedir, \ @@ -341,6 +380,19 @@ def test_no_home_directory(self): mock_addsitedir.assert_not_called() self.assertFalse(known_paths) + def test_gethistoryfile(self): + filename = 'file' + rc, out, err = assert_python_ok('-c', + f'import site; assert site.gethistoryfile() == "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + + # Check that PYTHON_HISTORY is ignored in isolated mode. + rc, out, err = assert_python_ok('-I', '-c', + f'import site; assert site.gethistoryfile() != "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + def test_trace(self): message = "bla-bla-bla" for verbose, out in (True, message + "\n"), (False, ""): @@ -462,16 +514,54 @@ def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. if "sitecustomize" not in sys.modules: try: - import sitecustomize + import sitecustomize # noqa: F401 except ImportError: pass else: self.fail("sitecustomize not imported automatically") - @test.support.requires_resource('network') - @test.support.system_must_validate_cert + @support.requires_subprocess() + def test_customization_modules_on_startup(self): + mod_names = [ + 'sitecustomize' + ] + + if site.ENABLE_USER_SITE: + mod_names.append('usercustomize') + + temp_dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, temp_dir) + + with EnvironmentVarGuard() as environ: + environ['PYTHONPATH'] = temp_dir + + for module_name in mod_names: + os_helper.rmtree(temp_dir) + os.mkdir(temp_dir) + + customize_path = os.path.join(temp_dir, f'{module_name}.py') + eyecatcher = f'EXECUTED_{module_name}' + + with open(customize_path, 'w') as f: + f.write(f'print("{eyecatcher}")') + + output = subprocess.check_output([sys.executable, '-c', '""']) + self.assertIn(eyecatcher, output.decode('utf-8')) + + # -S blocks any site-packages + output = subprocess.check_output([sys.executable, '-S', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + # -s blocks user site-packages + if 'usercustomize' == module_name: + output = subprocess.check_output([sys.executable, '-s', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') + @test.support.requires_resource('network') + @test.support.system_must_validate_cert def test_license_exists_at_url(self): # This test is a bit fragile since it depends on the format of the # string displayed by license in the absence of a LICENSE file. @@ -487,11 +577,20 @@ def test_license_exists_at_url(self): code = e.code self.assertEqual(code, 200, msg="Can't find " + url) + @support.cpython_only + def test_lazy_imports(self): + import_helper.ensure_lazy_imports("site", [ + "io", + "locale", + "traceback", + "atexit", + "warnings", + "textwrap", + ]) + class StartupImportTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure @support.requires_subprocess() def test_startup_imports(self): # Get sys.path in isolated mode (python3 -I) @@ -581,7 +680,7 @@ def _create_underpth_exe(self, lines, exe_pth=True): _pth_file = os.path.splitext(exe_file)[0] + '._pth' else: _pth_file = os.path.splitext(dll_file)[0] + '._pth' - with open(_pth_file, 'w') as f: + with open(_pth_file, 'w', encoding='utf8') as f: for line in lines: print(line, file=f) return exe_file @@ -608,19 +707,32 @@ def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines): sys_path.append(abs_path) return sys_path - # TODO: RUSTPYTHON - @unittest.expectedFailure + def _get_pth_lines(self, libpath: str, *, import_site: bool): + pth_lines = ['fake-path-name'] + # include 200 lines of `libpath` in _pth lines (or fewer + # if the `libpath` is long enough to get close to 32KB + # see https://github.com/python/cpython/issues/113628) + encoded_libpath_length = len(libpath.encode("utf-8")) + repetitions = min(200, 30000 // encoded_libpath_length) + if repetitions <= 2: + self.skipTest( + f"Python stdlib path is too long ({encoded_libpath_length:,} bytes)") + pth_lines.extend(libpath for _ in range(repetitions)) + pth_lines.extend(['', '# comment']) + if import_site: + pth_lines.append('import site') + return pth_lines + + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_underpth_basic(self): - libpath = test.support.STDLIB_DIR - exe_prefix = os.path.dirname(sys.executable) pth_lines = ['#.', '# ..', *sys.path, '.', '..'] exe_file = self._create_underpth_exe(pth_lines) sys_path = self._calc_sys_path_for_underpth_nosite( os.path.dirname(exe_file), pth_lines) - output = subprocess.check_output([exe_file, '-c', + output = subprocess.check_output([exe_file, '-X', 'utf8', '-c', 'import sys; print("\\n".join(sys.path) if sys.flags.no_site else "")' ], encoding='utf-8', errors='surrogateescape') actual_sys_path = output.rstrip().split('\n') @@ -631,18 +743,12 @@ def test_underpth_basic(self): "sys.path is incorrect" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_underpth_nosite_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - pth_lines = [ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - ] + pth_lines = self._get_pth_lines(libpath, import_site=False) exe_file = self._create_underpth_exe(pth_lines) sys_path = self._calc_sys_path_for_underpth_nosite( os.path.dirname(exe_file), @@ -662,19 +768,13 @@ def test_underpth_nosite_file(self): "sys.path is incorrect" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_underpth_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - exe_file = self._create_underpth_exe([ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - 'import site' - ]) + exe_file = self._create_underpth_exe( + self._get_pth_lines(libpath, import_site=True)) sys_prefix = os.path.dirname(exe_file) env = os.environ.copy() env['PYTHONPATH'] = 'from-env' @@ -689,19 +789,13 @@ def test_underpth_file(self): )], env=env) self.assertTrue(rc, "sys.path is incorrect") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_underpth_dll_file(self): libpath = test.support.STDLIB_DIR exe_prefix = os.path.dirname(sys.executable) - exe_file = self._create_underpth_exe([ - 'fake-path-name', - *[libpath for _ in range(200)], - '', - '# comment', - 'import site' - ], exe_pth=False) + exe_file = self._create_underpth_exe( + self._get_pth_lines(libpath, import_site=True), exe_pth=False) sys_prefix = os.path.dirname(exe_file) env = os.environ.copy() env['PYTHONPATH'] = 'from-env' @@ -717,5 +811,110 @@ def test_underpth_dll_file(self): self.assertTrue(rc, "sys.path is incorrect") +class CommandLineTests(unittest.TestCase): + def exists(self, path): + if path is not None and os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + + def get_excepted_output(self, *args): + if len(args) == 0: + user_base = site.getuserbase() + user_site = site.getusersitepackages() + output = io.StringIO() + output.write("sys.path = [\n") + for dir in sys.path: + output.write(" %r,\n" % (dir,)) + output.write("]\n") + output.write(f"USER_BASE: {user_base} ({self.exists(user_base)})\n") + output.write(f"USER_SITE: {user_site} ({self.exists(user_site)})\n") + output.write(f"ENABLE_USER_SITE: {site.ENABLE_USER_SITE}\n") + return 0, dedent(output.getvalue()).strip() + + buffer = [] + if '--user-base' in args: + buffer.append(site.getuserbase()) + if '--user-site' in args: + buffer.append(site.getusersitepackages()) + + if buffer: + return_code = 3 + if site.ENABLE_USER_SITE: + return_code = 0 + elif site.ENABLE_USER_SITE is False: + return_code = 1 + elif site.ENABLE_USER_SITE is None: + return_code = 2 + output = os.pathsep.join(buffer) + return return_code, os.path.normpath(dedent(output).strip()) + else: + return 10, None + + def invoke_command_line(self, *args): + cmd_args = [] + if sys.flags.no_user_site: + cmd_args.append("-s") + cmd_args.extend(["-m", "site", *args]) + + with EnvironmentVarGuard() as env: + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + proc = spawn_python(*cmd_args, text=True, env=env, + encoding='utf-8', errors='replace') + + output = kill_python(proc) + return_code = proc.returncode + return return_code, os.path.normpath(dedent(output).strip()) + + @support.requires_subprocess() + def test_no_args(self): + return_code, output = self.invoke_command_line() + excepted_return_code, _ = self.get_excepted_output() + self.assertEqual(return_code, excepted_return_code) + lines = output.splitlines() + self.assertEqual(lines[0], "sys.path = [") + self.assertEqual(lines[-4], "]") + excepted_base = f"USER_BASE: '{site.getuserbase()}'" +\ + f" ({self.exists(site.getuserbase())})" + self.assertEqual(lines[-3], excepted_base) + excepted_site = f"USER_SITE: '{site.getusersitepackages()}'" +\ + f" ({self.exists(site.getusersitepackages())})" + self.assertEqual(lines[-2], excepted_site) + self.assertEqual(lines[-1], f"ENABLE_USER_SITE: {site.ENABLE_USER_SITE}") + + @support.requires_subprocess() + def test_unknown_args(self): + return_code, output = self.invoke_command_line("--unknown-arg") + excepted_return_code, _ = self.get_excepted_output("--unknown-arg") + self.assertEqual(return_code, excepted_return_code) + self.assertIn('[--user-base] [--user-site]', output) + + @support.requires_subprocess() + def test_base_arg(self): + return_code, output = self.invoke_command_line("--user-base") + excepted = self.get_excepted_output("--user-base") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + @support.requires_subprocess() + def test_site_arg(self): + return_code, output = self.invoke_command_line("--user-site") + excepted = self.get_excepted_output("--user-site") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + @support.requires_subprocess() + def test_both_args(self): + return_code, output = self.invoke_command_line("--user-base", + "--user-site") + excepted = self.get_excepted_output("--user-base", "--user-site") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_slice.py b/Lib/test/test_slice.py index 53d4c776160..6de7e73c399 100644 --- a/Lib/test/test_slice.py +++ b/Lib/test/test_slice.py @@ -286,8 +286,7 @@ def test_deepcopy(self): self.assertIsNot(s.stop, c.stop) self.assertIsNot(s.step, c.step) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cycle(self): class myobj(): pass o = myobj() diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py deleted file mode 100644 index d2e150d535f..00000000000 --- a/Lib/test/test_smtpd.py +++ /dev/null @@ -1,1018 +0,0 @@ -import unittest -import textwrap -from test import support, mock_socket -from test.support import socket_helper -from test.support import warnings_helper -import socket -import io - -import warnings -with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - import smtpd - import asyncore - - -class DummyServer(smtpd.SMTPServer): - def __init__(self, *args, **kwargs): - smtpd.SMTPServer.__init__(self, *args, **kwargs) - self.messages = [] - if self._decode_data: - self.return_status = 'return status' - else: - self.return_status = b'return status' - - def process_message(self, peer, mailfrom, rcpttos, data, **kw): - self.messages.append((peer, mailfrom, rcpttos, data)) - if data == self.return_status: - return '250 Okish' - if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']: - return '250 SMTPUTF8 message okish' - - -class DummyDispatcherBroken(Exception): - pass - - -class BrokenDummyServer(DummyServer): - def listen(self, num): - raise DummyDispatcherBroken() - - -class SMTPDServerTest(unittest.TestCase): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - - def test_process_message_unimplemented(self): - server = smtpd.SMTPServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) - - def write_line(line): - channel.socket.queue_recv(line) - channel.handle_read() - - write_line(b'HELO example') - write_line(b'MAIL From:eggs@example') - write_line(b'RCPT To:spam@example') - write_line(b'DATA') - self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') - - def test_decode_data_and_enable_SMTPUTF8_raises(self): - self.assertRaises( - ValueError, - smtpd.SMTPServer, - (socket_helper.HOST, 0), - ('b', 0), - enable_SMTPUTF8=True, - decode_data=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - - -class DebuggingServerTest(unittest.TestCase): - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - - def send_data(self, channel, data, enable_SMTPUTF8=False): - def write_line(line): - channel.socket.queue_recv(line) - channel.handle_read() - write_line(b'EHLO example') - if enable_SMTPUTF8: - write_line(b'MAIL From:eggs@example BODY=8BITMIME SMTPUTF8') - else: - write_line(b'MAIL From:eggs@example') - write_line(b'RCPT To:spam@example') - write_line(b'DATA') - write_line(data) - write_line(b'.') - - def test_process_message_with_decode_data_true(self): - server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) - with support.captured_stdout() as s: - self.send_data(channel, b'From: test\n\nhello\n') - stdout = s.getvalue() - self.assertEqual(stdout, textwrap.dedent("""\ - ---------- MESSAGE FOLLOWS ---------- - From: test - X-Peer: peer-address - - hello - ------------ END MESSAGE ------------ - """)) - - def test_process_message_with_decode_data_false(self): - server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) - with support.captured_stdout() as s: - self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') - stdout = s.getvalue() - self.assertEqual(stdout, textwrap.dedent("""\ - ---------- MESSAGE FOLLOWS ---------- - b'From: test' - b'X-Peer: peer-address' - b'' - b'h\\xc3\\xa9llo\\xff' - ------------ END MESSAGE ------------ - """)) - - def test_process_message_with_enable_SMTPUTF8_true(self): - server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), - enable_SMTPUTF8=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) - with support.captured_stdout() as s: - self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') - stdout = s.getvalue() - self.assertEqual(stdout, textwrap.dedent("""\ - ---------- MESSAGE FOLLOWS ---------- - b'From: test' - b'X-Peer: peer-address' - b'' - b'h\\xc3\\xa9llo\\xff' - ------------ END MESSAGE ------------ - """)) - - def test_process_SMTPUTF8_message_with_enable_SMTPUTF8_true(self): - server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), - enable_SMTPUTF8=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) - with support.captured_stdout() as s: - self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n', - enable_SMTPUTF8=True) - stdout = s.getvalue() - self.assertEqual(stdout, textwrap.dedent("""\ - ---------- MESSAGE FOLLOWS ---------- - mail options: ['BODY=8BITMIME', 'SMTPUTF8'] - b'From: test' - b'X-Peer: peer-address' - b'' - b'h\\xc3\\xa9llo\\xff' - ------------ END MESSAGE ------------ - """)) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - - -class TestFamilyDetection(unittest.TestCase): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - - @unittest.skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled") - def test_socket_uses_IPv6(self): - server = smtpd.SMTPServer((socket_helper.HOSTv6, 0), (socket_helper.HOSTv4, 0)) - self.assertEqual(server.socket.family, socket.AF_INET6) - - def test_socket_uses_IPv4(self): - server = smtpd.SMTPServer((socket_helper.HOSTv4, 0), (socket_helper.HOSTv6, 0)) - self.assertEqual(server.socket.family, socket.AF_INET) - - -class TestRcptOptionParsing(unittest.TestCase): - error_response = (b'555 RCPT TO parameters not recognized or not ' - b'implemented\r\n') - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, channel, line): - channel.socket.queue_recv(line) - channel.handle_read() - - def test_params_rejected(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) - self.write_line(channel, b'EHLO example') - self.write_line(channel, b'MAIL from: <foo@example.com> size=20') - self.write_line(channel, b'RCPT to: <foo@example.com> foo=bar') - self.assertEqual(channel.socket.last, self.error_response) - - def test_nothing_accepted(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) - self.write_line(channel, b'EHLO example') - self.write_line(channel, b'MAIL from: <foo@example.com> size=20') - self.write_line(channel, b'RCPT to: <foo@example.com>') - self.assertEqual(channel.socket.last, b'250 OK\r\n') - - -class TestMailOptionParsing(unittest.TestCase): - error_response = (b'555 MAIL FROM parameters not recognized or not ' - b'implemented\r\n') - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, channel, line): - channel.socket.queue_recv(line) - channel.handle_read() - - def test_with_decode_data_true(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0), decode_data=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) - self.write_line(channel, b'EHLO example') - for line in [ - b'MAIL from: <foo@example.com> size=20 SMTPUTF8', - b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=8BITMIME', - b'MAIL from: <foo@example.com> size=20 BODY=UNKNOWN', - b'MAIL from: <foo@example.com> size=20 body=8bitmime', - ]: - self.write_line(channel, line) - self.assertEqual(channel.socket.last, self.error_response) - self.write_line(channel, b'MAIL from: <foo@example.com> size=20') - self.assertEqual(channel.socket.last, b'250 OK\r\n') - - def test_with_decode_data_false(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr) - self.write_line(channel, b'EHLO example') - for line in [ - b'MAIL from: <foo@example.com> size=20 SMTPUTF8', - b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=8BITMIME', - ]: - self.write_line(channel, line) - self.assertEqual(channel.socket.last, self.error_response) - self.write_line( - channel, - b'MAIL from: <foo@example.com> size=20 SMTPUTF8 BODY=UNKNOWN') - self.assertEqual( - channel.socket.last, - b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n') - self.write_line( - channel, b'MAIL from: <foo@example.com> size=20 body=8bitmime') - self.assertEqual(channel.socket.last, b'250 OK\r\n') - - def test_with_enable_smtputf8_true(self): - server = DummyServer((socket_helper.HOST, 0), ('b', 0), enable_SMTPUTF8=True) - conn, addr = server.accept() - channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) - self.write_line(channel, b'EHLO example') - self.write_line( - channel, - b'MAIL from: <foo@example.com> size=20 body=8bitmime smtputf8') - self.assertEqual(channel.socket.last, b'250 OK\r\n') - - -class SMTPDChannelTest(unittest.TestCase): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - decode_data=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_broken_connect(self): - self.assertRaises( - DummyDispatcherBroken, BrokenDummyServer, - (socket_helper.HOST, 0), ('b', 0), decode_data=True) - - def test_decode_data_and_enable_SMTPUTF8_raises(self): - self.assertRaises( - ValueError, smtpd.SMTPChannel, - self.server, self.channel.conn, self.channel.addr, - enable_SMTPUTF8=True, decode_data=True) - - def test_server_accept(self): - self.server.handle_accept() - - def test_missing_data(self): - self.write_line(b'') - self.assertEqual(self.channel.socket.last, - b'500 Error: bad syntax\r\n') - - def test_EHLO(self): - self.write_line(b'EHLO example') - self.assertEqual(self.channel.socket.last, b'250 HELP\r\n') - - def test_EHLO_bad_syntax(self): - self.write_line(b'EHLO') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: EHLO hostname\r\n') - - def test_EHLO_duplicate(self): - self.write_line(b'EHLO example') - self.write_line(b'EHLO example') - self.assertEqual(self.channel.socket.last, - b'503 Duplicate HELO/EHLO\r\n') - - def test_EHLO_HELO_duplicate(self): - self.write_line(b'EHLO example') - self.write_line(b'HELO example') - self.assertEqual(self.channel.socket.last, - b'503 Duplicate HELO/EHLO\r\n') - - def test_HELO(self): - name = smtpd.socket.getfqdn() - self.write_line(b'HELO example') - self.assertEqual(self.channel.socket.last, - '250 {}\r\n'.format(name).encode('ascii')) - - def test_HELO_EHLO_duplicate(self): - self.write_line(b'HELO example') - self.write_line(b'EHLO example') - self.assertEqual(self.channel.socket.last, - b'503 Duplicate HELO/EHLO\r\n') - - def test_HELP(self): - self.write_line(b'HELP') - self.assertEqual(self.channel.socket.last, - b'250 Supported commands: EHLO HELO MAIL RCPT ' + \ - b'DATA RSET NOOP QUIT VRFY\r\n') - - def test_HELP_command(self): - self.write_line(b'HELP MAIL') - self.assertEqual(self.channel.socket.last, - b'250 Syntax: MAIL FROM: <address>\r\n') - - def test_HELP_command_unknown(self): - self.write_line(b'HELP SPAM') - self.assertEqual(self.channel.socket.last, - b'501 Supported commands: EHLO HELO MAIL RCPT ' + \ - b'DATA RSET NOOP QUIT VRFY\r\n') - - def test_HELO_bad_syntax(self): - self.write_line(b'HELO') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: HELO hostname\r\n') - - def test_HELO_duplicate(self): - self.write_line(b'HELO example') - self.write_line(b'HELO example') - self.assertEqual(self.channel.socket.last, - b'503 Duplicate HELO/EHLO\r\n') - - def test_HELO_parameter_rejected_when_extensions_not_enabled(self): - self.extended_smtp = False - self.write_line(b'HELO example') - self.write_line(b'MAIL from:<foo@example.com> SIZE=1234') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM: <address>\r\n') - - def test_MAIL_allows_space_after_colon(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from: <foo@example.com>') - self.assertEqual(self.channel.socket.last, - b'250 OK\r\n') - - def test_extended_MAIL_allows_space_after_colon(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: <foo@example.com> size=20') - self.assertEqual(self.channel.socket.last, - b'250 OK\r\n') - - def test_NOOP(self): - self.write_line(b'NOOP') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_HELO_NOOP(self): - self.write_line(b'HELO example') - self.write_line(b'NOOP') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_NOOP_bad_syntax(self): - self.write_line(b'NOOP hi') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: NOOP\r\n') - - def test_QUIT(self): - self.write_line(b'QUIT') - self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') - - def test_HELO_QUIT(self): - self.write_line(b'HELO example') - self.write_line(b'QUIT') - self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') - - def test_QUIT_arg_ignored(self): - self.write_line(b'QUIT bye bye') - self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') - - def test_bad_state(self): - self.channel.smtp_state = 'BAD STATE' - self.write_line(b'HELO example') - self.assertEqual(self.channel.socket.last, - b'451 Internal confusion\r\n') - - def test_command_too_long(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from: ' + - b'a' * self.channel.command_size_limit + - b'@example') - self.assertEqual(self.channel.socket.last, - b'500 Error: line too long\r\n') - - def test_MAIL_command_limit_extended_with_SIZE(self): - self.write_line(b'EHLO example') - fill_len = self.channel.command_size_limit - len('MAIL from:<@example>') - self.write_line(b'MAIL from:<' + - b'a' * fill_len + - b'@example> SIZE=1234') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - self.write_line(b'MAIL from:<' + - b'a' * (fill_len + 26) + - b'@example> SIZE=1234') - self.assertEqual(self.channel.socket.last, - b'500 Error: line too long\r\n') - - def test_MAIL_command_rejects_SMTPUTF8_by_default(self): - self.write_line(b'EHLO example') - self.write_line( - b'MAIL from: <naive@example.com> BODY=8BITMIME SMTPUTF8') - self.assertEqual(self.channel.socket.last[0:1], b'5') - - def test_data_longer_than_default_data_size_limit(self): - # Hack the default so we don't have to generate so much data. - self.channel.data_size_limit = 1048 - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'A' * self.channel.data_size_limit + - b'A\r\n.') - self.assertEqual(self.channel.socket.last, - b'552 Error: Too much mail data\r\n') - - def test_MAIL_size_parameter(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL FROM:<eggs@example> SIZE=512') - self.assertEqual(self.channel.socket.last, - b'250 OK\r\n') - - def test_MAIL_invalid_size_parameter(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL FROM:<eggs@example> SIZE=invalid') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM: <address> [SP <mail-parameters>]\r\n') - - def test_MAIL_RCPT_unknown_parameters(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL FROM:<eggs@example> ham=green') - self.assertEqual(self.channel.socket.last, - b'555 MAIL FROM parameters not recognized or not implemented\r\n') - - self.write_line(b'MAIL FROM:<eggs@example>') - self.write_line(b'RCPT TO:<eggs@example> ham=green') - self.assertEqual(self.channel.socket.last, - b'555 RCPT TO parameters not recognized or not implemented\r\n') - - def test_MAIL_size_parameter_larger_than_default_data_size_limit(self): - self.channel.data_size_limit = 1048 - self.write_line(b'EHLO example') - self.write_line(b'MAIL FROM:<eggs@example> SIZE=2096') - self.assertEqual(self.channel.socket.last, - b'552 Error: message size exceeds fixed maximum message size\r\n') - - def test_need_MAIL(self): - self.write_line(b'HELO example') - self.write_line(b'RCPT to:spam@example') - self.assertEqual(self.channel.socket.last, - b'503 Error: need MAIL command\r\n') - - def test_MAIL_syntax_HELO(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from eggs@example') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM: <address>\r\n') - - def test_MAIL_syntax_EHLO(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from eggs@example') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM: <address> [SP <mail-parameters>]\r\n') - - def test_MAIL_missing_address(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from:') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: MAIL FROM: <address>\r\n') - - def test_MAIL_chevrons(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from:<eggs@example>') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_MAIL_empty_chevrons(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from:<>') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_MAIL_quoted_localpart(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: <"Fred Blogs"@example.com>') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') - - def test_MAIL_quoted_localpart_no_angles(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: "Fred Blogs"@example.com') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') - - def test_MAIL_quoted_localpart_with_size(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') - - def test_MAIL_quoted_localpart_with_size_no_angles(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') - - def test_nested_MAIL(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL from:eggs@example') - self.write_line(b'MAIL from:spam@example') - self.assertEqual(self.channel.socket.last, - b'503 Error: nested MAIL command\r\n') - - def test_VRFY(self): - self.write_line(b'VRFY eggs@example') - self.assertEqual(self.channel.socket.last, - b'252 Cannot VRFY user, but will accept message and attempt ' + \ - b'delivery\r\n') - - def test_VRFY_syntax(self): - self.write_line(b'VRFY') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: VRFY <address>\r\n') - - def test_EXPN_not_implemented(self): - self.write_line(b'EXPN') - self.assertEqual(self.channel.socket.last, - b'502 EXPN not implemented\r\n') - - def test_no_HELO_MAIL(self): - self.write_line(b'MAIL from:<foo@example.com>') - self.assertEqual(self.channel.socket.last, - b'503 Error: send HELO first\r\n') - - def test_need_RCPT(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last, - b'503 Error: need RCPT command\r\n') - - def test_RCPT_syntax_HELO(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From: eggs@example') - self.write_line(b'RCPT to eggs@example') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: RCPT TO: <address>\r\n') - - def test_RCPT_syntax_EHLO(self): - self.write_line(b'EHLO example') - self.write_line(b'MAIL From: eggs@example') - self.write_line(b'RCPT to eggs@example') - self.assertEqual(self.channel.socket.last, - b'501 Syntax: RCPT TO: <address> [SP <mail-parameters>]\r\n') - - def test_RCPT_lowercase_to_OK(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From: eggs@example') - self.write_line(b'RCPT to: <eggs@example>') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_no_HELO_RCPT(self): - self.write_line(b'RCPT to eggs@example') - self.assertEqual(self.channel.socket.last, - b'503 Error: send HELO first\r\n') - - def test_data_dialog(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.write_line(b'RCPT To:spam@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last, - b'354 End data with <CR><LF>.<CR><LF>\r\n') - self.write_line(b'data\r\nmore\r\n.') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.server.messages, - [(('peer-address', 'peer-port'), - 'eggs@example', - ['spam@example'], - 'data\nmore')]) - - def test_DATA_syntax(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA spam') - self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n') - - def test_no_HELO_DATA(self): - self.write_line(b'DATA spam') - self.assertEqual(self.channel.socket.last, - b'503 Error: send HELO first\r\n') - - def test_data_transparency_section_4_5_2(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'..\r\n.\r\n') - self.assertEqual(self.channel.received_data, '.') - - def test_multiple_RCPT(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'RCPT To:ham@example') - self.write_line(b'DATA') - self.write_line(b'data\r\n.') - self.assertEqual(self.server.messages, - [(('peer-address', 'peer-port'), - 'eggs@example', - ['spam@example','ham@example'], - 'data')]) - - def test_manual_status(self): - # checks that the Channel is able to return a custom status message - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'return status\r\n.') - self.assertEqual(self.channel.socket.last, b'250 Okish\r\n') - - def test_RSET(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'RSET') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.write_line(b'MAIL From:foo@example') - self.write_line(b'RCPT To:eggs@example') - self.write_line(b'DATA') - self.write_line(b'data\r\n.') - self.assertEqual(self.server.messages, - [(('peer-address', 'peer-port'), - 'foo@example', - ['eggs@example'], - 'data')]) - - def test_HELO_RSET(self): - self.write_line(b'HELO example') - self.write_line(b'RSET') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_RSET_syntax(self): - self.write_line(b'RSET hi') - self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n') - - def test_unknown_command(self): - self.write_line(b'UNKNOWN_CMD') - self.assertEqual(self.channel.socket.last, - b'500 Error: command "UNKNOWN_CMD" not ' + \ - b'recognized\r\n') - - def test_attribute_deprecations(self): - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__server - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__server = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__line - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__line = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__state - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__state = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__greeting - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__greeting = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__mailfrom - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__mailfrom = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__rcpttos - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__rcpttos = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__data - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__data = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__fqdn - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__fqdn = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__peer - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__peer = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__conn - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__conn = 'spam' - with warnings_helper.check_warnings(('', DeprecationWarning)): - spam = self.channel._SMTPChannel__addr - with warnings_helper.check_warnings(('', DeprecationWarning)): - self.channel._SMTPChannel__addr = 'spam' - -@unittest.skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled") -class SMTPDChannelIPv6Test(SMTPDChannelTest): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOSTv6, 0), ('b', 0), - decode_data=True) - conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - decode_data=True) - -class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase): - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = self.server.accept() - # Set DATA size limit to 32 bytes for easy testing - self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32, - decode_data=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_data_limit_dialog(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.write_line(b'RCPT To:spam@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last, - b'354 End data with <CR><LF>.<CR><LF>\r\n') - self.write_line(b'data\r\nmore\r\n.') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.assertEqual(self.server.messages, - [(('peer-address', 'peer-port'), - 'eggs@example', - ['spam@example'], - 'data\nmore')]) - - def test_data_limit_dialog_too_much_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - self.write_line(b'RCPT To:spam@example') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last, - b'354 End data with <CR><LF>.<CR><LF>\r\n') - self.write_line(b'This message is longer than 32 bytes\r\n.') - self.assertEqual(self.channel.socket.last, - b'552 Error: Too much mail data\r\n') - - -class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0)) - conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_ascii_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'plain ascii text') - self.write_line(b'.') - self.assertEqual(self.channel.received_data, b'plain ascii text') - - def test_utf8_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') - self.write_line(b'and some plain ascii') - self.write_line(b'.') - self.assertEqual( - self.channel.received_data, - b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87\n' - b'and some plain ascii') - - -class SMTPDChannelWithDecodeDataTrue(unittest.TestCase): - - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), - decode_data=True) - conn, addr = self.server.accept() - # Set decode_data to True - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - decode_data=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_ascii_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'plain ascii text') - self.write_line(b'.') - self.assertEqual(self.channel.received_data, 'plain ascii text') - - def test_utf8_data(self): - self.write_line(b'HELO example') - self.write_line(b'MAIL From:eggs@example') - self.write_line(b'RCPT To:spam@example') - self.write_line(b'DATA') - self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') - self.write_line(b'and some plain ascii') - self.write_line(b'.') - self.assertEqual( - self.channel.received_data, - 'utf8 enriched text: żźć\nand some plain ascii') - - -class SMTPDChannelTestWithEnableSMTPUTF8True(unittest.TestCase): - def setUp(self): - smtpd.socket = asyncore.socket = mock_socket - self.old_debugstream = smtpd.DEBUGSTREAM - self.debug = smtpd.DEBUGSTREAM = io.StringIO() - self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), - enable_SMTPUTF8=True) - conn, addr = self.server.accept() - self.channel = smtpd.SMTPChannel(self.server, conn, addr, - enable_SMTPUTF8=True) - - def tearDown(self): - asyncore.close_all() - asyncore.socket = smtpd.socket = socket - smtpd.DEBUGSTREAM = self.old_debugstream - - def write_line(self, line): - self.channel.socket.queue_recv(line) - self.channel.handle_read() - - def test_MAIL_command_accepts_SMTPUTF8_when_announced(self): - self.write_line(b'EHLO example') - self.write_line( - 'MAIL from: <naïve@example.com> BODY=8BITMIME SMTPUTF8'.encode( - 'utf-8') - ) - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_process_smtputf8_message(self): - self.write_line(b'EHLO example') - for mail_parameters in [b'', b'BODY=8BITMIME SMTPUTF8']: - self.write_line(b'MAIL from: <a@example> ' + mail_parameters) - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'rcpt to:<b@example.com>') - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'data') - self.assertEqual(self.channel.socket.last[0:3], b'354') - self.write_line(b'c\r\n.') - if mail_parameters == b'': - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - else: - self.assertEqual(self.channel.socket.last, - b'250 SMTPUTF8 message okish\r\n') - - def test_utf8_data(self): - self.write_line(b'EHLO example') - self.write_line( - 'MAIL From: naïve@examplé BODY=8BITMIME SMTPUTF8'.encode('utf-8')) - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line('RCPT To:späm@examplé'.encode('utf-8')) - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'DATA') - self.assertEqual(self.channel.socket.last[0:3], b'354') - self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') - self.write_line(b'.') - self.assertEqual( - self.channel.received_data, - b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') - - def test_MAIL_command_limit_extended_with_SIZE_and_SMTPUTF8(self): - self.write_line(b'ehlo example') - fill_len = (512 + 26 + 10) - len('mail from:<@example>') - self.write_line(b'MAIL from:<' + - b'a' * (fill_len + 1) + - b'@example>') - self.assertEqual(self.channel.socket.last, - b'500 Error: line too long\r\n') - self.write_line(b'MAIL from:<' + - b'a' * fill_len + - b'@example>') - self.assertEqual(self.channel.socket.last, b'250 OK\r\n') - - def test_multiple_emails_with_extended_command_length(self): - self.write_line(b'ehlo example') - fill_len = (512 + 26 + 10) - len('mail from:<@example>') - for char in [b'a', b'b', b'c']: - self.write_line(b'MAIL from:<' + char * fill_len + b'a@example>') - self.assertEqual(self.channel.socket.last[0:3], b'500') - self.write_line(b'MAIL from:<' + char * fill_len + b'@example>') - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'rcpt to:<hans@example.com>') - self.assertEqual(self.channel.socket.last[0:3], b'250') - self.write_line(b'data') - self.assertEqual(self.channel.socket.last[0:3], b'354') - self.write_line(b'test\r\n.') - self.assertEqual(self.channel.socket.last[0:3], b'250') - - -class MiscTestCase(unittest.TestCase): - def test__all__(self): - not_exported = { - "program", "Devnull", "DEBUGSTREAM", "NEWLINE", "COMMASPACE", - "DATA_SIZE_DEFAULT", "usage", "Options", "parseargs", - } - support.check__all__(self, smtpd, not_exported=not_exported) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py new file mode 100644 index 00000000000..fb3ea34d766 --- /dev/null +++ b/Lib/test/test_smtplib.py @@ -0,0 +1,1611 @@ +import base64 +import email.mime.text +from email.message import EmailMessage +from email.base64mime import body_encode as encode_base64 +import email.utils +import hashlib +import hmac +import socket +import smtplib +import io +import re +import sys +import time +import select +import errno +import textwrap +import threading + +import unittest +import unittest.mock as mock +from test import support, mock_socket +from test.support import hashlib_helper +from test.support import socket_helper +from test.support import threading_helper +from test.support import asyncore +from test.support import smtpd +from unittest.mock import Mock + + +support.requires_working_socket(module=True) + +HOST = socket_helper.HOST + +if sys.platform == 'darwin': + # select.poll returns a select.POLLHUP at the end of the tests + # on darwin, so just ignore it + def handle_expt(self): + pass + smtpd.SMTPChannel.handle_expt = handle_expt + + +def server(evt, buf, serv): + serv.listen() + evt.set() + try: + conn, addr = serv.accept() + except TimeoutError: + pass + else: + n = 500 + while buf and n > 0: + r, w, e = select.select([], [conn], []) + if w: + sent = conn.send(buf) + buf = buf[sent:] + + n -= 1 + + conn.close() + finally: + serv.close() + evt.set() + +class GeneralTests: + + def setUp(self): + smtplib.socket = mock_socket + self.port = 25 + + def tearDown(self): + smtplib.socket = socket + + # This method is no longer used but is retained for backward compatibility, + # so test to make sure it still works. + def testQuoteData(self): + teststr = "abc\n.jkl\rfoo\r\n..blue" + expected = "abc\r\n..jkl\r\nfoo\r\n...blue" + self.assertEqual(expected, smtplib.quotedata(teststr)) + + def testBasic1(self): + mock_socket.reply_with(b"220 Hola mundo") + # connects + client = self.client(HOST, self.port) + client.close() + + def testSourceAddress(self): + mock_socket.reply_with(b"220 Hola mundo") + # connects + client = self.client(HOST, self.port, + source_address=('127.0.0.1',19876)) + self.assertEqual(client.source_address, ('127.0.0.1', 19876)) + client.close() + + def testBasic2(self): + mock_socket.reply_with(b"220 Hola mundo") + # connects, include port in host name + client = self.client("%s:%s" % (HOST, self.port)) + client.close() + + def testLocalHostName(self): + mock_socket.reply_with(b"220 Hola mundo") + # check that supplied local_hostname is used + client = self.client(HOST, self.port, local_hostname="testhost") + self.assertEqual(client.local_hostname, "testhost") + client.close() + + def testTimeoutDefault(self): + mock_socket.reply_with(b"220 Hola mundo") + self.assertIsNone(mock_socket.getdefaulttimeout()) + mock_socket.setdefaulttimeout(30) + self.assertEqual(mock_socket.getdefaulttimeout(), 30) + try: + client = self.client(HOST, self.port) + finally: + mock_socket.setdefaulttimeout(None) + self.assertEqual(client.sock.gettimeout(), 30) + client.close() + + def testTimeoutNone(self): + mock_socket.reply_with(b"220 Hola mundo") + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + client = self.client(HOST, self.port, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(client.sock.gettimeout()) + client.close() + + def testTimeoutZero(self): + mock_socket.reply_with(b"220 Hola mundo") + with self.assertRaises(ValueError): + self.client(HOST, self.port, timeout=0) + + def testTimeoutValue(self): + mock_socket.reply_with(b"220 Hola mundo") + client = self.client(HOST, self.port, timeout=30) + self.assertEqual(client.sock.gettimeout(), 30) + client.close() + + def test_debuglevel(self): + mock_socket.reply_with(b"220 Hello world") + client = self.client() + client.set_debuglevel(1) + with support.captured_stderr() as stderr: + client.connect(HOST, self.port) + client.close() + expected = re.compile(r"^connect:", re.MULTILINE) + self.assertRegex(stderr.getvalue(), expected) + + def test_debuglevel_2(self): + mock_socket.reply_with(b"220 Hello world") + client = self.client() + client.set_debuglevel(2) + with support.captured_stderr() as stderr: + client.connect(HOST, self.port) + client.close() + expected = re.compile(r"^\d{2}:\d{2}:\d{2}\.\d{6} connect: ", + re.MULTILINE) + self.assertRegex(stderr.getvalue(), expected) + + +class SMTPGeneralTests(GeneralTests, unittest.TestCase): + + client = smtplib.SMTP + + +class LMTPGeneralTests(GeneralTests, unittest.TestCase): + + client = smtplib.LMTP + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), "test requires Unix domain socket") + def testUnixDomainSocketTimeoutDefault(self): + local_host = '/some/local/lmtp/delivery/program' + mock_socket.reply_with(b"220 Hello world") + try: + client = self.client(local_host, self.port) + finally: + mock_socket.setdefaulttimeout(None) + self.assertIsNone(client.sock.gettimeout()) + client.close() + + def testTimeoutZero(self): + super().testTimeoutZero() + local_host = '/some/local/lmtp/delivery/program' + with self.assertRaises(ValueError): + self.client(local_host, timeout=0) + +# Test server thread using the specified SMTP server class +def debugging_server(serv, serv_evt, client_evt): + serv_evt.set() + + try: + if hasattr(select, 'poll'): + poll_fun = asyncore.poll2 + else: + poll_fun = asyncore.poll + + n = 1000 + while asyncore.socket_map and n > 0: + poll_fun(0.01, asyncore.socket_map) + + # when the client conversation is finished, it will + # set client_evt, and it's then ok to kill the server + if client_evt.is_set(): + serv.close() + break + + n -= 1 + + except TimeoutError: + pass + finally: + if not client_evt.is_set(): + # allow some time for the client to read the result + time.sleep(0.5) + serv.close() + asyncore.close_all() + serv_evt.set() + +MSG_BEGIN = '---------- MESSAGE FOLLOWS ----------\n' +MSG_END = '------------ END MESSAGE ------------\n' + +# NOTE: Some SMTP objects in the tests below are created with a non-default +# local_hostname argument to the constructor, since (on some systems) the FQDN +# lookup caused by the default local_hostname sometimes takes so long that the +# test server times out, causing the test to fail. + +# Test behavior of smtpd.DebuggingServer +class DebuggingServerTests(unittest.TestCase): + + maxDiff = None + + def setUp(self): + self.thread_key = threading_helper.threading_setup() + self.real_getfqdn = socket.getfqdn + socket.getfqdn = mock_socket.getfqdn + # temporarily replace sys.stdout to capture DebuggingServer output + self.old_stdout = sys.stdout + self.output = io.StringIO() + sys.stdout = self.output + + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Capture SMTPChannel debug output + self.old_DEBUGSTREAM = smtpd.DEBUGSTREAM + smtpd.DEBUGSTREAM = io.StringIO() + # Pick a random unused port by passing 0 for the port number + self.serv = smtpd.DebuggingServer((HOST, 0), ('nowhere', -1), + decode_data=True) + # Keep a note of what server host and port were assigned + self.host, self.port = self.serv.socket.getsockname()[:2] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + socket.getfqdn = self.real_getfqdn + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + threading_helper.join_thread(self.thread) + # restore sys.stdout + sys.stdout = self.old_stdout + # restore DEBUGSTREAM + smtpd.DEBUGSTREAM.close() + smtpd.DEBUGSTREAM = self.old_DEBUGSTREAM + del self.thread + self.doCleanups() + threading_helper.threading_cleanup(*self.thread_key) + + def get_output_without_xpeer(self): + test_output = self.output.getvalue() + return re.sub(r'(.*?)^X-Peer:\s*\S+\n(.*)', r'\1\2', + test_output, flags=re.MULTILINE|re.DOTALL) + + def testBasic(self): + # connect + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.quit() + + def testSourceAddress(self): + # connect + src_port = socket_helper.find_unused_port() + try: + smtp = smtplib.SMTP(self.host, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT, + source_address=(self.host, src_port)) + self.addCleanup(smtp.close) + self.assertEqual(smtp.source_address, (self.host, src_port)) + self.assertEqual(smtp.local_hostname, 'localhost') + smtp.quit() + except OSError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to source port %d" % src_port) + raise + + def testNOOP(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + expected = (250, b'OK') + self.assertEqual(smtp.noop(), expected) + smtp.quit() + + def testRSET(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + expected = (250, b'OK') + self.assertEqual(smtp.rset(), expected) + smtp.quit() + + def testELHO(self): + # EHLO isn't implemented in DebuggingServer + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + expected = (250, b'\nSIZE 33554432\nHELP') + self.assertEqual(smtp.ehlo(), expected) + smtp.quit() + + def testEXPNNotImplemented(self): + # EXPN isn't implemented in DebuggingServer + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + expected = (502, b'EXPN not implemented') + smtp.putcmd('EXPN') + self.assertEqual(smtp.getreply(), expected) + smtp.quit() + + def test_issue43124_putcmd_escapes_newline(self): + # see: https://bugs.python.org/issue43124 + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with self.assertRaises(ValueError) as exc: + smtp.putcmd('helo\nX-INJECTED') + self.assertIn("prohibited newline characters", str(exc.exception)) + smtp.quit() + + def testVRFY(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + expected = (252, b'Cannot VRFY user, but will accept message ' + \ + b'and attempt delivery') + self.assertEqual(smtp.vrfy('nobody@nowhere.com'), expected) + self.assertEqual(smtp.verify('nobody@nowhere.com'), expected) + smtp.quit() + + def testSecondHELO(self): + # check that a second HELO returns a message that it's a duplicate + # (this behavior is specific to smtpd.SMTPChannel) + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.helo() + expected = (503, b'Duplicate HELO/EHLO') + self.assertEqual(smtp.helo(), expected) + smtp.quit() + + def testHELP(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + self.assertEqual(smtp.help(), b'Supported commands: EHLO HELO MAIL ' + \ + b'RCPT DATA RSET NOOP QUIT VRFY') + smtp.quit() + + def testSend(self): + # connect and send mail + m = 'A test message' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.sendmail('John', 'Sally', m) + # XXX(nnorwitz): this test is flaky and dies with a bad file descriptor + # in asyncore. This sleep might help, but should really be fixed + # properly by using an Event variable. + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + def testSendBinary(self): + m = b'A test message' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.sendmail('John', 'Sally', m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.decode('ascii'), MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + def testSendNeedingDotQuote(self): + # Issue 12283 + m = '.A test\n.mes.sage.' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.sendmail('John', 'Sally', m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + def test_issue43124_escape_localhostname(self): + # see: https://bugs.python.org/issue43124 + # connect and send mail + m = 'wazzuuup\nlinetwo' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='hi\nX-INJECTED', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with self.assertRaises(ValueError) as exc: + smtp.sendmail("hi@me.com", "you@me.com", m) + self.assertIn( + "prohibited newline characters: ehlo hi\\nX-INJECTED", + str(exc.exception), + ) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + debugout = smtpd.DEBUGSTREAM.getvalue() + self.assertNotIn("X-INJECTED", debugout) + + def test_issue43124_escape_options(self): + # see: https://bugs.python.org/issue43124 + # connect and send mail + m = 'wazzuuup\nlinetwo' + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + + self.addCleanup(smtp.close) + smtp.sendmail("hi@me.com", "you@me.com", m) + with self.assertRaises(ValueError) as exc: + smtp.mail("hi@me.com", ["X-OPTION\nX-INJECTED-1", "X-OPTION2\nX-INJECTED-2"]) + msg = str(exc.exception) + self.assertIn("prohibited newline characters", msg) + self.assertIn("X-OPTION\\nX-INJECTED-1 X-OPTION2\\nX-INJECTED-2", msg) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + debugout = smtpd.DEBUGSTREAM.getvalue() + self.assertNotIn("X-OPTION", debugout) + self.assertNotIn("X-OPTION2", debugout) + self.assertNotIn("X-INJECTED-1", debugout) + self.assertNotIn("X-INJECTED-2", debugout) + + def testSendNullSender(self): + m = 'A test message' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.sendmail('<>', 'Sally', m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: <>$", re.MULTILINE) + self.assertRegex(debugout, sender) + + def testSendMessage(self): + m = email.mime.text.MIMEText('A test message') + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.send_message(m, from_addr='John', to_addrs='Sally') + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Remove the X-Peer header that DebuggingServer adds as figuring out + # exactly what IP address format is put there is not easy (and + # irrelevant to our test). Typically 127.0.0.1 or ::1, but it is + # not always the same as socket.gethostbyname(HOST). :( + test_output = self.get_output_without_xpeer() + del m['X-Peer'] + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(test_output, mexpect) + + def testSendMessageWithAddresses(self): + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John' + m['CC'] = 'Sally, Fred' + m['Bcc'] = 'John Root <root@localhost>, "Dinsdale" <warped@silly.walks.com>' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.send_message(m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + # make sure the Bcc header is still in the message. + self.assertEqual(m['Bcc'], 'John Root <root@localhost>, "Dinsdale" ' + '<warped@silly.walks.com>') + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Remove the X-Peer header that DebuggingServer adds. + test_output = self.get_output_without_xpeer() + del m['X-Peer'] + # The Bcc header should not be transmitted. + del m['Bcc'] + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(test_output, mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: foo@bar.com$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('John', 'Sally', 'Fred', 'root@localhost', + 'warped@silly.walks.com'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertRegex(debugout, to_addr) + + def testSendMessageWithSomeAddresses(self): + # Make sure nothing breaks if not all of the three 'to' headers exist + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John, Dinsdale' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.send_message(m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Remove the X-Peer header that DebuggingServer adds. + test_output = self.get_output_without_xpeer() + del m['X-Peer'] + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(test_output, mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: foo@bar.com$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('John', 'Dinsdale'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertRegex(debugout, to_addr) + + def testSendMessageWithSpecifiedAddresses(self): + # Make sure addresses specified in call override those in message. + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John, Dinsdale' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.send_message(m, from_addr='joe@example.com', to_addrs='foo@example.net') + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Remove the X-Peer header that DebuggingServer adds. + test_output = self.get_output_without_xpeer() + del m['X-Peer'] + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(test_output, mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: joe@example.com$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('John', 'Dinsdale'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertNotRegex(debugout, to_addr) + recip = re.compile(r"^recips: .*'foo@example.net'.*$", re.MULTILINE) + self.assertRegex(debugout, recip) + + def testSendMessageWithMultipleFrom(self): + # Sender overrides To + m = email.mime.text.MIMEText('A test message') + m['From'] = 'Bernard, Bianca' + m['Sender'] = 'the_rescuers@Rescue-Aid-Society.com' + m['To'] = 'John, Dinsdale' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.send_message(m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Remove the X-Peer header that DebuggingServer adds. + test_output = self.get_output_without_xpeer() + del m['X-Peer'] + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(test_output, mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: the_rescuers@Rescue-Aid-Society.com$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('John', 'Dinsdale'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertRegex(debugout, to_addr) + + def testSendMessageResent(self): + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John' + m['CC'] = 'Sally, Fred' + m['Bcc'] = 'John Root <root@localhost>, "Dinsdale" <warped@silly.walks.com>' + m['Resent-Date'] = 'Thu, 1 Jan 1970 17:42:00 +0000' + m['Resent-From'] = 'holy@grail.net' + m['Resent-To'] = 'Martha <my_mom@great.cooker.com>, Jeff' + m['Resent-Bcc'] = 'doe@losthope.net' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.send_message(m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # The Resent-Bcc headers are deleted before serialization. + del m['Bcc'] + del m['Resent-Bcc'] + # Remove the X-Peer header that DebuggingServer adds. + test_output = self.get_output_without_xpeer() + del m['X-Peer'] + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(test_output, mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: holy@grail.net$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('my_mom@great.cooker.com', 'Jeff', 'doe@losthope.net'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertRegex(debugout, to_addr) + + def testSendMessageMultipleResentRaises(self): + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John' + m['CC'] = 'Sally, Fred' + m['Bcc'] = 'John Root <root@localhost>, "Dinsdale" <warped@silly.walks.com>' + m['Resent-Date'] = 'Thu, 1 Jan 1970 17:42:00 +0000' + m['Resent-From'] = 'holy@grail.net' + m['Resent-To'] = 'Martha <my_mom@great.cooker.com>, Jeff' + m['Resent-Bcc'] = 'doe@losthope.net' + m['Resent-Date'] = 'Thu, 2 Jan 1970 17:42:00 +0000' + m['Resent-To'] = 'holy@grail.net' + m['Resent-From'] = 'Martha <my_mom@great.cooker.com>, Jeff' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with self.assertRaises(ValueError): + smtp.send_message(m) + smtp.close() + +class NonConnectingTests(unittest.TestCase): + + def testNotConnected(self): + # Test various operations on an unconnected SMTP object that + # should raise exceptions (at present the attempt in SMTP.send + # to reference the nonexistent 'sock' attribute of the SMTP object + # causes an AttributeError) + smtp = smtplib.SMTP() + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.ehlo) + self.assertRaises(smtplib.SMTPServerDisconnected, + smtp.send, 'test msg') + + def testNonnumericPort(self): + # check that non-numeric port raises OSError + self.assertRaises(OSError, smtplib.SMTP, + "localhost", "bogus") + self.assertRaises(OSError, smtplib.SMTP, + "localhost:bogus") + + def testSockAttributeExists(self): + # check that sock attribute is present outside of a connect() call + # (regression test, the previous behavior raised an + # AttributeError: 'SMTP' object has no attribute 'sock') + with smtplib.SMTP() as smtp: + self.assertIsNone(smtp.sock) + + +class DefaultArgumentsTests(unittest.TestCase): + + def setUp(self): + self.msg = EmailMessage() + self.msg['From'] = 'Páolo <főo@bar.com>' + self.smtp = smtplib.SMTP() + self.smtp.ehlo = Mock(return_value=(200, 'OK')) + self.smtp.has_extn, self.smtp.sendmail = Mock(), Mock() + + def testSendMessage(self): + expected_mail_options = ('SMTPUTF8', 'BODY=8BITMIME') + self.smtp.send_message(self.msg) + self.smtp.send_message(self.msg) + self.assertEqual(self.smtp.sendmail.call_args_list[0][0][3], + expected_mail_options) + self.assertEqual(self.smtp.sendmail.call_args_list[1][0][3], + expected_mail_options) + + def testSendMessageWithMailOptions(self): + mail_options = ['STARTTLS'] + expected_mail_options = ('STARTTLS', 'SMTPUTF8', 'BODY=8BITMIME') + self.smtp.send_message(self.msg, None, None, mail_options) + self.assertEqual(mail_options, ['STARTTLS']) + self.assertEqual(self.smtp.sendmail.call_args_list[0][0][3], + expected_mail_options) + + +# test response of client to a non-successful HELO message +class BadHELOServerTests(unittest.TestCase): + + def setUp(self): + smtplib.socket = mock_socket + mock_socket.reply_with(b"199 no hello for you!") + self.old_stdout = sys.stdout + self.output = io.StringIO() + sys.stdout = self.output + self.port = 25 + + def tearDown(self): + smtplib.socket = socket + sys.stdout = self.old_stdout + + def testFailingHELO(self): + self.assertRaises(smtplib.SMTPConnectError, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + +class TooLongLineTests(unittest.TestCase): + respdata = b'250 OK' + (b'.' * smtplib._MAXLINE * 2) + b'\n' + + def setUp(self): + self.thread_key = threading_helper.threading_setup() + self.old_stdout = sys.stdout + self.output = io.StringIO() + sys.stdout = self.output + + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = socket_helper.bind_port(self.sock) + servargs = (self.evt, self.respdata, self.sock) + self.thread = threading.Thread(target=server, args=servargs) + self.thread.start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + sys.stdout = self.old_stdout + threading_helper.join_thread(self.thread) + del self.thread + self.doCleanups() + threading_helper.threading_cleanup(*self.thread_key) + + def testLineTooLong(self): + self.assertRaises(smtplib.SMTPResponseException, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + +sim_users = {'Mr.A@somewhere.com':'John A', + 'Ms.B@xn--fo-fka.com':'Sally B', + 'Mrs.C@somewhereesle.com':'Ruth C', + } + +sim_auth = ('Mr.A@somewhere.com', 'somepassword') +sim_cram_md5_challenge = ('PENCeUxFREJoU0NnbmhNWitOMjNGNn' + 'dAZWx3b29kLmlubm9zb2Z0LmNvbT4=') +sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'], + 'list-2':['Ms.B@xn--fo-fka.com',], + } + +# Simulated SMTP channel & server +class ResponseException(Exception): pass +class SimSMTPChannel(smtpd.SMTPChannel): + + quit_response = None + mail_response = None + rcpt_response = None + data_response = None + rcpt_count = 0 + rset_count = 0 + disconnect = 0 + AUTH = 99 # Add protocol state to enable auth testing. + authenticated_user = None + + def __init__(self, extra_features, *args, **kw): + self._extrafeatures = ''.join( + [ "250-{0}\r\n".format(x) for x in extra_features ]) + self.all_received_lines = [] + super(SimSMTPChannel, self).__init__(*args, **kw) + + # AUTH related stuff. It would be nice if support for this were in smtpd. + def found_terminator(self): + if self.smtp_state == self.AUTH: + line = self._emptystring.join(self.received_lines) + print('Data:', repr(line), file=smtpd.DEBUGSTREAM) + self.received_lines = [] + try: + self.auth_object(line) + except ResponseException as e: + self.smtp_state = self.COMMAND + self.push('%s %s' % (e.smtp_code, e.smtp_error)) + return + self.all_received_lines.append(self.received_lines) + super().found_terminator() + + + def smtp_AUTH(self, arg): + if not self.seen_greeting: + self.push('503 Error: send EHLO first') + return + if not self.extended_smtp or 'AUTH' not in self._extrafeatures: + self.push('500 Error: command "AUTH" not recognized') + return + if self.authenticated_user is not None: + self.push( + '503 Bad sequence of commands: already authenticated') + return + args = arg.split() + if len(args) not in [1, 2]: + self.push('501 Syntax: AUTH <mechanism> [initial-response]') + return + auth_object_name = '_auth_%s' % args[0].lower().replace('-', '_') + try: + self.auth_object = getattr(self, auth_object_name) + except AttributeError: + self.push('504 Command parameter not implemented: unsupported ' + ' authentication mechanism {!r}'.format(auth_object_name)) + return + self.smtp_state = self.AUTH + self.auth_object(args[1] if len(args) == 2 else None) + + def _authenticated(self, user, valid): + if valid: + self.authenticated_user = user + self.push('235 Authentication Succeeded') + else: + self.push('535 Authentication credentials invalid') + self.smtp_state = self.COMMAND + + def _decode_base64(self, string): + return base64.decodebytes(string.encode('ascii')).decode('utf-8') + + def _auth_plain(self, arg=None): + if arg is None: + self.push('334 ') + else: + logpass = self._decode_base64(arg) + try: + *_, user, password = logpass.split('\0') + except ValueError as e: + self.push('535 Splitting response {!r} into user and password' + ' failed: {}'.format(logpass, e)) + return + self._authenticated(user, password == sim_auth[1]) + + def _auth_login(self, arg=None): + if arg is None: + # base64 encoded 'Username:' + self.push('334 VXNlcm5hbWU6') + elif not hasattr(self, '_auth_login_user'): + self._auth_login_user = self._decode_base64(arg) + # base64 encoded 'Password:' + self.push('334 UGFzc3dvcmQ6') + else: + password = self._decode_base64(arg) + self._authenticated(self._auth_login_user, password == sim_auth[1]) + del self._auth_login_user + + def _auth_buggy(self, arg=None): + # This AUTH mechanism will 'trap' client in a neverending 334 + # base64 encoded 'BuGgYbUgGy' + self.push('334 QnVHZ1liVWdHeQ==') + + def _auth_cram_md5(self, arg=None): + if arg is None: + self.push('334 {}'.format(sim_cram_md5_challenge)) + else: + logpass = self._decode_base64(arg) + try: + user, hashed_pass = logpass.split() + except ValueError as e: + self.push('535 Splitting response {!r} into user and password ' + 'failed: {}'.format(logpass, e)) + return + pwd = sim_auth[1].encode('ascii') + msg = self._decode_base64(sim_cram_md5_challenge).encode('ascii') + try: + valid_hashed_pass = hmac.HMAC(pwd, msg, 'md5').hexdigest() + except ValueError: + self.push('504 CRAM-MD5 is not supported') + return + self._authenticated(user, hashed_pass == valid_hashed_pass) + # end AUTH related stuff. + + def smtp_EHLO(self, arg): + resp = ('250-testhost\r\n' + '250-EXPN\r\n' + '250-SIZE 20000000\r\n' + '250-STARTTLS\r\n' + '250-DELIVERBY\r\n') + resp = resp + self._extrafeatures + '250 HELP' + self.push(resp) + self.seen_greeting = arg + self.extended_smtp = True + + def smtp_VRFY(self, arg): + # For max compatibility smtplib should be sending the raw address. + if arg in sim_users: + self.push('250 %s %s' % (sim_users[arg], smtplib.quoteaddr(arg))) + else: + self.push('550 No such user: %s' % arg) + + def smtp_EXPN(self, arg): + list_name = arg.lower() + if list_name in sim_lists: + user_list = sim_lists[list_name] + for n, user_email in enumerate(user_list): + quoted_addr = smtplib.quoteaddr(user_email) + if n < len(user_list) - 1: + self.push('250-%s %s' % (sim_users[user_email], quoted_addr)) + else: + self.push('250 %s %s' % (sim_users[user_email], quoted_addr)) + else: + self.push('550 No access for you!') + + def smtp_QUIT(self, arg): + if self.quit_response is None: + super(SimSMTPChannel, self).smtp_QUIT(arg) + else: + self.push(self.quit_response) + self.close_when_done() + + def smtp_MAIL(self, arg): + if self.mail_response is None: + super().smtp_MAIL(arg) + else: + self.push(self.mail_response) + if self.disconnect: + self.close_when_done() + + def smtp_RCPT(self, arg): + if self.rcpt_response is None: + super().smtp_RCPT(arg) + return + self.rcpt_count += 1 + self.push(self.rcpt_response[self.rcpt_count-1]) + + def smtp_RSET(self, arg): + self.rset_count += 1 + super().smtp_RSET(arg) + + def smtp_DATA(self, arg): + if self.data_response is None: + super().smtp_DATA(arg) + else: + self.push(self.data_response) + + def handle_error(self): + raise + + +class SimSMTPServer(smtpd.SMTPServer): + + channel_class = SimSMTPChannel + + def __init__(self, *args, **kw): + self._extra_features = [] + self._addresses = {} + smtpd.SMTPServer.__init__(self, *args, **kw) + + def handle_accepted(self, conn, addr): + self._SMTPchannel = self.channel_class( + self._extra_features, self, conn, addr, + decode_data=self._decode_data) + + def process_message(self, peer, mailfrom, rcpttos, data): + self._addresses['from'] = mailfrom + self._addresses['tos'] = rcpttos + + def add_feature(self, feature): + self._extra_features.append(feature) + + def handle_error(self): + raise + + +# Test various SMTP & ESMTP commands/behaviors that require a simulated server +# (i.e., something with more features than DebuggingServer) +class SMTPSimTests(unittest.TestCase): + + def setUp(self): + self.thread_key = threading_helper.threading_setup() + self.real_getfqdn = socket.getfqdn + socket.getfqdn = mock_socket.getfqdn + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = SimSMTPServer((HOST, 0), ('nowhere', -1), decode_data=True) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + socket.getfqdn = self.real_getfqdn + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + threading_helper.join_thread(self.thread) + del self.thread + self.doCleanups() + threading_helper.threading_cleanup(*self.thread_key) + + def testBasic(self): + # smoke test + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.quit() + + def testEHLO(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + + # no features should be present before the EHLO + self.assertEqual(smtp.esmtp_features, {}) + + # features expected from the test server + expected_features = {'expn':'', + 'size': '20000000', + 'starttls': '', + 'deliverby': '', + 'help': '', + } + + smtp.ehlo() + self.assertEqual(smtp.esmtp_features, expected_features) + for k in expected_features: + self.assertTrue(smtp.has_extn(k)) + self.assertFalse(smtp.has_extn('unsupported-feature')) + smtp.quit() + + def testVRFY(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + + for addr_spec, name in sim_users.items(): + expected_known = (250, bytes('%s %s' % + (name, smtplib.quoteaddr(addr_spec)), + "ascii")) + self.assertEqual(smtp.vrfy(addr_spec), expected_known) + + u = 'nobody@nowhere.com' + expected_unknown = (550, ('No such user: %s' % u).encode('ascii')) + self.assertEqual(smtp.vrfy(u), expected_unknown) + smtp.quit() + + def testEXPN(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + + for listname, members in sim_lists.items(): + users = [] + for m in members: + users.append('%s %s' % (sim_users[m], smtplib.quoteaddr(m))) + expected_known = (250, bytes('\n'.join(users), "ascii")) + self.assertEqual(smtp.expn(listname), expected_known) + + u = 'PSU-Members-List' + expected_unknown = (550, b'No access for you!') + self.assertEqual(smtp.expn(u), expected_unknown) + smtp.quit() + + def testAUTH_PLAIN(self): + self.serv.add_feature("AUTH PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + resp = smtp.login(sim_auth[0], sim_auth[1]) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def testAUTH_LOGIN(self): + self.serv.add_feature("AUTH LOGIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + resp = smtp.login(sim_auth[0], sim_auth[1]) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def testAUTH_LOGIN_initial_response_ok(self): + self.serv.add_feature("AUTH LOGIN") + with smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) as smtp: + smtp.user, smtp.password = sim_auth + smtp.ehlo("test_auth_login") + resp = smtp.auth("LOGIN", smtp.auth_login, initial_response_ok=True) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + + def testAUTH_LOGIN_initial_response_notok(self): + self.serv.add_feature("AUTH LOGIN") + with smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) as smtp: + smtp.user, smtp.password = sim_auth + smtp.ehlo("test_auth_login") + resp = smtp.auth("LOGIN", smtp.auth_login, initial_response_ok=False) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + + def testAUTH_BUGGY(self): + self.serv.add_feature("AUTH BUGGY") + + def auth_buggy(challenge=None): + self.assertEqual(b"BuGgYbUgGy", challenge) + return "\0" + + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT + ) + try: + smtp.user, smtp.password = sim_auth + smtp.ehlo("test_auth_buggy") + expect = r"^Server AUTH mechanism infinite loop.*" + with self.assertRaisesRegex(smtplib.SMTPException, expect) as cm: + smtp.auth("BUGGY", auth_buggy, initial_response_ok=False) + finally: + smtp.close() + + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def testAUTH_CRAM_MD5(self): + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + resp = smtp.login(sim_auth[0], sim_auth[1]) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + @mock.patch("hmac.HMAC") + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked(self, hmac_constructor): + # CRAM-MD5 is the only "known" method by the server, + # but it is not supported by the client. In particular, + # no challenge will ever be sent. + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + msg = re.escape("No suitable authentication method found.") + with self.assertRaisesRegex(smtplib.SMTPException, msg): + smtp.login(sim_auth[0], sim_auth[1]) + hmac_constructor.assert_not_called() # call has been bypassed + + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked_and_fallback(self): + # Test that PLAIN is tried after CRAM-MD5 failed + self.serv.add_feature("AUTH CRAM-MD5 PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with ( + mock.patch.object(smtp, "auth_cram_md5") as smtp_auth_cram_md5, + mock.patch.object( + smtp, "auth_plain", wraps=smtp.auth_plain + ) as smtp_auth_plain + ): + resp = smtp.login(sim_auth[0], sim_auth[1]) + smtp_auth_plain.assert_called_once() + smtp_auth_cram_md5.assert_not_called() # no call to HMAC constructor + self.assertEqual(resp, (235, b'Authentication Succeeded')) + + @hashlib_helper.requires_hashdigest('md5', openssl=True) + def testAUTH_multiple(self): + # Test that multiple authentication methods are tried. + self.serv.add_feature("AUTH BOGUS PLAIN LOGIN CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + resp = smtp.login(sim_auth[0], sim_auth[1]) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def test_auth_function(self): + supported = {'PLAIN', 'LOGIN'} + try: + hashlib.md5() + except ValueError: + pass + else: + supported.add('CRAM-MD5') + for mechanism in supported: + self.serv.add_feature("AUTH {}".format(mechanism)) + for mechanism in supported: + with self.subTest(mechanism=mechanism): + smtp = smtplib.SMTP(HOST, self.port, + local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.ehlo('foo') + smtp.user, smtp.password = sim_auth[0], sim_auth[1] + method = 'auth_' + mechanism.lower().replace('-', '_') + resp = smtp.auth(mechanism, getattr(smtp, method)) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def test_quit_resets_greeting(self): + smtp = smtplib.SMTP(HOST, self.port, + local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + code, message = smtp.ehlo() + self.assertEqual(code, 250) + self.assertIn('size', smtp.esmtp_features) + smtp.quit() + self.assertNotIn('size', smtp.esmtp_features) + smtp.connect(HOST, self.port) + self.assertNotIn('size', smtp.esmtp_features) + smtp.ehlo_or_helo_if_needed() + self.assertIn('size', smtp.esmtp_features) + smtp.quit() + + def test_with_statement(self): + with smtplib.SMTP(HOST, self.port) as smtp: + code, message = smtp.noop() + self.assertEqual(code, 250) + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo') + with smtplib.SMTP(HOST, self.port) as smtp: + smtp.close() + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo') + + def test_with_statement_QUIT_failure(self): + with self.assertRaises(smtplib.SMTPResponseException) as error: + with smtplib.SMTP(HOST, self.port) as smtp: + smtp.noop() + self.serv._SMTPchannel.quit_response = '421 QUIT FAILED' + self.assertEqual(error.exception.smtp_code, 421) + self.assertEqual(error.exception.smtp_error, b'QUIT FAILED') + + #TODO: add tests for correct AUTH method fallback now that the + #test infrastructure can support it. + + # Issue 17498: make sure _rset does not raise SMTPServerDisconnected exception + def test__rest_from_mail_cmd(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.noop() + self.serv._SMTPchannel.mail_response = '451 Requested action aborted' + self.serv._SMTPchannel.disconnect = True + with self.assertRaises(smtplib.SMTPSenderRefused): + smtp.sendmail('John', 'Sally', 'test message') + self.assertIsNone(smtp.sock) + + # Issue 5713: make sure close, not rset, is called if we get a 421 error + def test_421_from_mail_cmd(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.noop() + self.serv._SMTPchannel.mail_response = '421 closing connection' + with self.assertRaises(smtplib.SMTPSenderRefused): + smtp.sendmail('John', 'Sally', 'test message') + self.assertIsNone(smtp.sock) + self.assertEqual(self.serv._SMTPchannel.rset_count, 0) + + def test_421_from_rcpt_cmd(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.noop() + self.serv._SMTPchannel.rcpt_response = ['250 accepted', '421 closing'] + with self.assertRaises(smtplib.SMTPRecipientsRefused) as r: + smtp.sendmail('John', ['Sally', 'Frank', 'George'], 'test message') + self.assertIsNone(smtp.sock) + self.assertEqual(self.serv._SMTPchannel.rset_count, 0) + self.assertDictEqual(r.exception.args[0], {'Frank': (421, b'closing')}) + + def test_421_from_data_cmd(self): + class MySimSMTPChannel(SimSMTPChannel): + def found_terminator(self): + if self.smtp_state == self.DATA: + self.push('421 closing') + else: + super().found_terminator() + self.serv.channel_class = MySimSMTPChannel + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.noop() + with self.assertRaises(smtplib.SMTPDataError): + smtp.sendmail('John@foo.org', ['Sally@foo.org'], 'test message') + self.assertIsNone(smtp.sock) + self.assertEqual(self.serv._SMTPchannel.rcpt_count, 0) + + def test_smtputf8_NotSupportedError_if_no_server_support(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.ehlo() + self.assertTrue(smtp.does_esmtp) + self.assertFalse(smtp.has_extn('smtputf8')) + self.assertRaises( + smtplib.SMTPNotSupportedError, + smtp.sendmail, + 'John', 'Sally', '', mail_options=['BODY=8BITMIME', 'SMTPUTF8']) + self.assertRaises( + smtplib.SMTPNotSupportedError, + smtp.mail, 'John', options=['BODY=8BITMIME', 'SMTPUTF8']) + + def test_send_unicode_without_SMTPUTF8(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + self.assertRaises(UnicodeEncodeError, smtp.sendmail, 'Alice', 'Böb', '') + self.assertRaises(UnicodeEncodeError, smtp.mail, 'Älice') + + def test_send_message_error_on_non_ascii_addrs_if_no_smtputf8(self): + # This test is located here and not in the SMTPUTF8SimTests + # class because it needs a "regular" SMTP server to work + msg = EmailMessage() + msg['From'] = "Páolo <főo@bar.com>" + msg['To'] = 'Dinsdale' + msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with self.assertRaises(smtplib.SMTPNotSupportedError): + smtp.send_message(msg) + + def test_name_field_not_included_in_envelop_addresses(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + + message = EmailMessage() + message['From'] = email.utils.formataddr(('Michaël', 'michael@example.com')) + message['To'] = email.utils.formataddr(('René', 'rene@example.com')) + + self.assertDictEqual(smtp.send_message(message), {}) + + self.assertEqual(self.serv._addresses['from'], 'michael@example.com') + self.assertEqual(self.serv._addresses['tos'], ['rene@example.com']) + + def test_lowercase_mail_from_rcpt_to(self): + m = 'A test message' + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + + smtp.sendmail('John', 'Sally', m) + + self.assertIn(['mail from:<John> size=14'], self.serv._SMTPchannel.all_received_lines) + self.assertIn(['rcpt to:<Sally>'], self.serv._SMTPchannel.all_received_lines) + + +class SimSMTPUTF8Server(SimSMTPServer): + + def __init__(self, *args, **kw): + # The base SMTP server turns these on automatically, but our test + # server is set up to munge the EHLO response, so we need to provide + # them as well. And yes, the call is to SMTPServer not SimSMTPServer. + self._extra_features = ['SMTPUTF8', '8BITMIME'] + smtpd.SMTPServer.__init__(self, *args, **kw) + + def handle_accepted(self, conn, addr): + self._SMTPchannel = self.channel_class( + self._extra_features, self, conn, addr, + decode_data=self._decode_data, + enable_SMTPUTF8=self.enable_SMTPUTF8, + ) + + def process_message(self, peer, mailfrom, rcpttos, data, mail_options=None, + rcpt_options=None): + self.last_peer = peer + self.last_mailfrom = mailfrom + self.last_rcpttos = rcpttos + self.last_message = data + self.last_mail_options = mail_options + self.last_rcpt_options = rcpt_options + + +class SMTPUTF8SimTests(unittest.TestCase): + + maxDiff = None + + def setUp(self): + self.thread_key = threading_helper.threading_setup() + self.real_getfqdn = socket.getfqdn + socket.getfqdn = mock_socket.getfqdn + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = SimSMTPUTF8Server((HOST, 0), ('nowhere', -1), + decode_data=False, + enable_SMTPUTF8=True) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + socket.getfqdn = self.real_getfqdn + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + threading_helper.join_thread(self.thread) + del self.thread + self.doCleanups() + threading_helper.threading_cleanup(*self.thread_key) + + def test_test_server_supports_extensions(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.ehlo() + self.assertTrue(smtp.does_esmtp) + self.assertTrue(smtp.has_extn('smtputf8')) + + def test_send_unicode_with_SMTPUTF8_via_sendmail(self): + m = '¡a test message containing unicode!'.encode('utf-8') + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.sendmail('Jőhn', 'Sálly', m, + mail_options=['BODY=8BITMIME', 'SMTPUTF8']) + self.assertEqual(self.serv.last_mailfrom, 'Jőhn') + self.assertEqual(self.serv.last_rcpttos, ['Sálly']) + self.assertEqual(self.serv.last_message, m) + self.assertIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) + + def test_send_unicode_with_SMTPUTF8_via_low_level_API(self): + m = '¡a test message containing unicode!'.encode('utf-8') + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + smtp.ehlo() + self.assertEqual( + smtp.mail('Jő', options=['BODY=8BITMIME', 'SMTPUTF8']), + (250, b'OK')) + self.assertEqual(smtp.rcpt('János'), (250, b'OK')) + self.assertEqual(smtp.data(m), (250, b'OK')) + self.assertEqual(self.serv.last_mailfrom, 'Jő') + self.assertEqual(self.serv.last_rcpttos, ['János']) + self.assertEqual(self.serv.last_message, m) + self.assertIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) + + def test_send_message_uses_smtputf8_if_addrs_non_ascii(self): + msg = EmailMessage() + msg['From'] = "Páolo <főo@bar.com>" + msg['To'] = 'Dinsdale' + msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' + # XXX I don't know why I need two \n's here, but this is an existing + # bug (if it is one) and not a problem with the new functionality. + msg.set_content("oh là là, know what I mean, know what I mean?\n\n") + # XXX smtpd converts received /r/n to /n, so we can't easily test that + # we are successfully sending /r/n :(. + expected = textwrap.dedent("""\ + From: Páolo <főo@bar.com> + To: Dinsdale + Subject: Nudge nudge, wink, wink \u1F609 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + MIME-Version: 1.0 + + oh là là, know what I mean, know what I mean? + """) + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + self.assertEqual(smtp.send_message(msg), {}) + self.assertEqual(self.serv.last_mailfrom, 'főo@bar.com') + self.assertEqual(self.serv.last_rcpttos, ['Dinsdale']) + self.assertEqual(self.serv.last_message.decode(), expected) + self.assertIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) + + +EXPECTED_RESPONSE = encode_base64(b'\0psu\0doesnotexist', eol='') + +class SimSMTPAUTHInitialResponseChannel(SimSMTPChannel): + def smtp_AUTH(self, arg): + # RFC 4954's AUTH command allows for an optional initial-response. + # Not all AUTH methods support this; some require a challenge. AUTH + # PLAIN does those, so test that here. See issue #15014. + args = arg.split() + if args[0].lower() == 'plain': + if len(args) == 2: + # AUTH PLAIN <initial-response> with the response base 64 + # encoded. Hard code the expected response for the test. + if args[1] == EXPECTED_RESPONSE: + self.push('235 Ok') + return + self.push('571 Bad authentication') + +class SimSMTPAUTHInitialResponseServer(SimSMTPServer): + channel_class = SimSMTPAUTHInitialResponseChannel + + +class SMTPAUTHInitialResponseSimTests(unittest.TestCase): + def setUp(self): + self.thread_key = threading_helper.threading_setup() + self.real_getfqdn = socket.getfqdn + socket.getfqdn = mock_socket.getfqdn + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = SimSMTPAUTHInitialResponseServer( + (HOST, 0), ('nowhere', -1), decode_data=True) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + socket.getfqdn = self.real_getfqdn + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + threading_helper.join_thread(self.thread) + del self.thread + self.doCleanups() + threading_helper.threading_cleanup(*self.thread_key) + + def testAUTH_PLAIN_initial_response_login(self): + self.serv.add_feature('AUTH PLAIN') + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.login('psu', 'doesnotexist') + smtp.close() + + def testAUTH_PLAIN_initial_response_auth(self): + self.serv.add_feature('AUTH PLAIN') + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + smtp.user = 'psu' + smtp.password = 'doesnotexist' + code, response = smtp.auth('plain', smtp.auth_plain) + smtp.close() + self.assertEqual(code, 235) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py new file mode 100644 index 00000000000..d765746987b --- /dev/null +++ b/Lib/test/test_smtpnet.py @@ -0,0 +1,93 @@ +import unittest +from test import support +from test.support import import_helper +from test.support import socket_helper +import os +import smtplib +import socket + +ssl = import_helper.import_module("ssl") + +support.requires("network") + +SMTP_TEST_SERVER = os.getenv('CPYTHON_TEST_SMTP_SERVER', 'smtp.gmail.com') + +def check_ssl_verifiy(host, port): + context = ssl.create_default_context() + with socket.create_connection((host, port)) as sock: + try: + sock = context.wrap_socket(sock, server_hostname=host) + except Exception: + return False + else: + sock.close() + return True + + +class SmtpTest(unittest.TestCase): + testServer = SMTP_TEST_SERVER + remotePort = 587 + + def test_connect_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP(self.testServer, self.remotePort) + try: + server.starttls(context=context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + + +class SmtpSSLTest(unittest.TestCase): + testServer = SMTP_TEST_SERVER + remotePort = 465 + + def test_connect(self): + support.get_attribute(smtplib, 'SMTP_SSL') + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP_SSL(self.testServer, self.remotePort) + server.ehlo() + server.quit() + + def test_connect_default_port(self): + support.get_attribute(smtplib, 'SMTP_SSL') + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP_SSL(self.testServer) + server.ehlo() + server.quit() + + @support.requires_resource('walltime') + def test_connect_using_sslcontext(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + support.get_attribute(smtplib, 'SMTP_SSL') + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=context) + server.ehlo() + server.quit() + + def test_connect_using_sslcontext_verified(self): + with socket_helper.transient_internet(self.testServer): + can_verify = check_ssl_verifiy(self.testServer, self.remotePort) + if not can_verify: + self.skipTest("SSL certificate can't be verified") + + support.get_attribute(smtplib, 'SMTP_SSL') + context = ssl.create_default_context() + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=context) + server.ehlo() + server.quit() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 35f94a4e226..fe1fc94b69e 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1,33 +1,36 @@ import unittest +from unittest import mock from test import support -from test.support import os_helper -from test.support import socket_helper -from test.support import threading_helper - +from test.support import ( + cpython_only, is_apple, os_helper, refleak_helper, socket_helper, threading_helper +) +from test.support.import_helper import ensure_lazy_imports +import _thread as thread +import array +import contextlib import errno +import gc import io import itertools -import socket -import select -import tempfile -import time -import traceback -import queue -import sys -import os -import platform -import array -import contextlib -from weakref import proxy -import signal import math +import os import pickle -import struct +import platform +import queue import random -import shutil +import re +import select +import signal +import socket import string -import _thread as thread +import struct +import sys +import tempfile import threading +import time +import traceback +import warnings +from weakref import proxy try: import multiprocessing except ImportError: @@ -36,6 +39,12 @@ import fcntl except ImportError: fcntl = None +try: + import _testcapi +except ImportError: + _testcapi = None + +support.requires_working_socket(module=True) HOST = socket_helper.HOST # test unicode string and carriage return @@ -43,12 +52,43 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" +SOLARIS = sys.platform.startswith("sunos") +WSL = "microsoft-standard-WSL" in platform.release() try: import _socket except ImportError: _socket = None +def skipForRefleakHuntinIf(condition, issueref): + if not condition: + def decorator(f): + f.client_skip = lambda f: f + return f + + else: + def decorator(f): + @contextlib.wraps(f) + def wrapper(*args, **kwds): + if refleak_helper.hunting_for_refleaks(): + raise unittest.SkipTest(f"ignore while hunting for refleaks, see {issueref}") + + return f(*args, **kwds) + + def client_skip(f): + @contextlib.wraps(f) + def wrapper(*args, **kwds): + if refleak_helper.hunting_for_refleaks(): + return + + return f(*args, **kwds) + + return wrapper + wrapper.client_skip = client_skip + return wrapper + + return decorator + def get_cid(): if fcntl is None: return None @@ -124,8 +164,8 @@ def _have_socket_qipcrtr(): def _have_socket_vsock(): """Check whether AF_VSOCK sockets are supported on this host.""" - ret = get_cid() is not None - return ret + cid = get_cid() + return (cid is not None) def _have_socket_bluetooth(): @@ -141,6 +181,28 @@ def _have_socket_bluetooth(): return True +def _have_socket_bluetooth_l2cap(): + """Check whether BTPROTO_L2CAP sockets are supported on this host.""" + try: + s = socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) + except (AttributeError, OSError): + return False + else: + s.close() + return True + + +def _have_socket_hyperv(): + """Check whether AF_HYPERV sockets are supported on this host.""" + try: + s = socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) + except (AttributeError, OSError): + return False + else: + s.close() + return True + + @contextlib.contextmanager def socket_setdefaulttimeout(timeout): old_timeout = socket.getdefaulttimeout() @@ -151,6 +213,24 @@ def socket_setdefaulttimeout(timeout): socket.setdefaulttimeout(old_timeout) +@contextlib.contextmanager +def downgrade_malformed_data_warning(): + # This warning happens on macos and win, but does not always happen on linux. + if sys.platform not in {"win32", "darwin"}: + yield + return + + with warnings.catch_warnings(): + # TODO: gh-110012, we should investigate why this warning is happening + # and fix it properly. + warnings.filterwarnings( + action="always", + message="received malformed or improperly-truncated ancillary data", + category=RuntimeWarning, + ) + yield + + HAVE_SOCKET_CAN = _have_socket_can() HAVE_SOCKET_CAN_ISOTP = _have_socket_can_isotp() @@ -165,13 +245,26 @@ def socket_setdefaulttimeout(timeout): HAVE_SOCKET_VSOCK = _have_socket_vsock() -HAVE_SOCKET_UDPLITE = hasattr(socket, "IPPROTO_UDPLITE") +# Older Android versions block UDPLITE with SELinux. +HAVE_SOCKET_UDPLITE = ( + hasattr(socket, "IPPROTO_UDPLITE") + and not (support.is_android and platform.android_ver().api_level < 29)) HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() +HAVE_SOCKET_BLUETOOTH_L2CAP = _have_socket_bluetooth_l2cap() + +HAVE_SOCKET_HYPERV = _have_socket_hyperv() + # Size in bytes of the int type SIZEOF_INT = array.array("i").itemsize +class TestLazyImport(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("socket", {"array", "selectors"}) + + class SocketTCPTest(unittest.TestCase): def setUp(self): @@ -199,24 +292,6 @@ def setUp(self): self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDPLITE) self.port = socket_helper.bind_port(self.serv) -class ThreadSafeCleanupTestCase: - """Subclass of unittest.TestCase with thread-safe cleanup methods. - - This subclass protects the addCleanup() and doCleanups() methods - with a recursive lock. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._cleanup_lock = threading.RLock() - - def addCleanup(self, *args, **kwargs): - with self._cleanup_lock: - return super().addCleanup(*args, **kwargs) - - def doCleanups(self, *args, **kwargs): - with self._cleanup_lock: - return super().doCleanups(*args, **kwargs) class SocketCANTest(unittest.TestCase): @@ -336,9 +411,7 @@ def serverExplicitReady(self): self.server_ready.set() def _setUp(self): - self.wait_threads = threading_helper.wait_threads_exit() - self.wait_threads.__enter__() - self.addCleanup(self.wait_threads.__exit__, None, None, None) + self.enterContext(threading_helper.wait_threads_exit()) self.server_ready = threading.Event() self.client_ready = threading.Event() @@ -485,10 +558,11 @@ def clientTearDown(self): ThreadableTest.clientTearDown(self) @unittest.skipIf(fcntl is None, "need fcntl") +@unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, - "This test can only be run on a virtual guest.") +@unittest.skipUnless(get_cid() != 2, # VMADDR_CID_HOST + "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -501,6 +575,7 @@ def setUp(self): self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) self.serv.listen() self.serverExplicitReady() + self.serv.settimeout(support.LOOPBACK_TIMEOUT) self.conn, self.connaddr = self.serv.accept() self.addCleanup(self.conn.close) @@ -509,10 +584,16 @@ def clientSetUp(self): self.cli = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.cli.close) cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + # gh-119461: Use the local communication address (loopback) + cid = socket.VMADDR_CID_LOCAL self.cli.connect((cid, VSOCKPORT)) def testStream(self): - msg = self.conn.recv(1024) + try: + msg = self.conn.recv(1024) + except PermissionError as exc: + self.skipTest(repr(exc)) self.assertEqual(msg, MSG) def _testStream(self): @@ -557,19 +638,27 @@ class SocketPairTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): unittest.TestCase.__init__(self, methodName=methodName) ThreadableTest.__init__(self) + self.cli = None + self.serv = None + + def socketpair(self): + # To be overridden by some child classes. + return socket.socketpair() def setUp(self): - self.serv, self.cli = socket.socketpair() + self.serv, self.cli = self.socketpair() def tearDown(self): - self.serv.close() + if self.serv: + self.serv.close() self.serv = None def clientSetUp(self): pass def clientTearDown(self): - self.cli.close() + if self.cli: + self.cli.close() self.cli = None ThreadableTest.clientTearDown(self) @@ -591,17 +680,18 @@ class SocketTestBase(unittest.TestCase): def setUp(self): self.serv = self.newSocket() + self.addCleanup(self.close_server) self.bindServer() + def close_server(self): + self.serv.close() + self.serv = None + def bindServer(self): """Bind server socket and set self.serv_addr to its address.""" self.bindSock(self.serv) self.serv_addr = self.serv.getsockname() - def tearDown(self): - self.serv.close() - self.serv = None - class SocketListeningTestMixin(SocketTestBase): """Mixin to listen on the server socket.""" @@ -611,8 +701,7 @@ def setUp(self): self.serv.listen() -class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, - ThreadableTest): +class ThreadedSocketTestMixin(SocketTestBase, ThreadableTest): """Mixin to add client socket and allow client/server tests. Client socket is self.cli and its address is self.cli_addr. See @@ -686,15 +775,10 @@ class UnixSocketTestBase(SocketTestBase): # can't send anything that might be problematic for a privileged # user running the tests. - def setUp(self): - self.dir_path = tempfile.mkdtemp() - self.addCleanup(os.rmdir, self.dir_path) - super().setUp() - def bindSock(self, sock): - path = tempfile.mktemp(dir=self.dir_path) - socket_helper.bind_unix_socket(sock, path) + path = socket_helper.create_unix_domain_name() self.addCleanup(os_helper.unlink, path) + socket_helper.bind_unix_socket(sock, path) class UnixStreamBase(UnixSocketTestBase): """Base class for Unix-domain SOCK_STREAM tests.""" @@ -827,6 +911,13 @@ def requireSocket(*args): class GeneralModuleTests(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; gc.is_tracked not implemented + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_socket_type(self): + self.assertTrue(gc.is_tracked(_socket.socket)) + with self.assertRaisesRegex(TypeError, "immutable"): + _socket.socket.foo = 1 + def test_SocketType_is_socketobject(self): import _socket self.assertTrue(socket.SocketType is _socket.socket) @@ -884,8 +975,7 @@ def testSocketError(self): with self.assertRaises(OSError, msg=msg % 'socket.gaierror'): raise socket.gaierror - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; error message format differs def testSendtoErrors(self): # Testing that sendto doesn't mask failures. See #10169. s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -958,8 +1048,19 @@ def testWindowsSpecificConstants(self): socket.IPPROTO_L2TP socket.IPPROTO_SCTP - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(support.is_wasi, "WASI is missing these methods") + def test_socket_methods(self): + # socket methods that depend on a configure HAVE_ check. They should + # be present on all platforms except WASI. + names = [ + "_accept", "bind", "connect", "connect_ex", "getpeername", + "getsockname", "listen", "recvfrom", "recvfrom_into", "sendto", + "setsockopt", "shutdown" + ] + for name in names: + if not hasattr(socket.socket, name): + self.fail(f"socket method {name} is missing") + @unittest.skipUnless(sys.platform == 'darwin', 'macOS specific test') @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test') def test3542SocketOptions(self): @@ -987,9 +1088,7 @@ def test3542SocketOptions(self): 'IPV6_USE_MIN_MTU', } for opt in opts: - self.assertTrue( - hasattr(socket, opt), f"Missing RFC3542 socket option '{opt}'" - ) + self.assertHasAttr(socket, opt) def testHostnameRes(self): # Testing hostname resolution mechanisms @@ -1021,8 +1120,10 @@ def test_host_resolution(self): def test_host_resolution_bad_address(self): # These are all malformed IP addresses and expected not to resolve to - # any result. But some ISPs, e.g. AWS, may successfully resolve these - # IPs. + # any result. But some ISPs, e.g. AWS and AT&T, may successfully + # resolve these IPs. In particular, AT&T's DNS Error Assist service + # will break this test. See https://bugs.python.org/issue42092 for a + # workaround. explanation = ( "resolving an invalid IP address did not raise OSError; " "can be caused by a broken DNS server" @@ -1056,6 +1157,7 @@ def test_sethostname(self): @unittest.skipUnless(hasattr(socket, 'if_nameindex'), 'socket.if_nameindex() not available.') + @support.skip_android_selinux('if_nameindex') def testInterfaceNameIndex(self): interfaces = socket.if_nameindex() for index, name in interfaces: @@ -1070,17 +1172,39 @@ def testInterfaceNameIndex(self): self.assertIsInstance(_name, str) self.assertEqual(name, _name) + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u32 @unittest.skipUnless(hasattr(socket, 'if_indextoname'), 'socket.if_indextoname() not available.') + @support.skip_android_selinux('if_indextoname') def testInvalidInterfaceIndexToName(self): - self.assertRaises(OSError, socket.if_indextoname, 0) + with self.assertRaises(OSError) as cm: + socket.if_indextoname(0) + self.assertIsNotNone(cm.exception.errno) + + self.assertRaises(ValueError, socket.if_indextoname, -1) + self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + if hasattr(socket, 'if_nameindex'): + indices = dict(socket.if_nameindex()) + for index in indices: + index2 = index + 2**32 + if index2 not in indices: + with self.assertRaises((OverflowError, OSError)): + socket.if_indextoname(index2) + for index in 2**32-1, 2**64-1: + if index not in indices: + with self.assertRaises((OverflowError, OSError)): + socket.if_indextoname(index) @unittest.skipUnless(hasattr(socket, 'if_nametoindex'), 'socket.if_nametoindex() not available.') + @support.skip_android_selinux('if_nametoindex') def testInvalidInterfaceNameToIndex(self): + with self.assertRaises(OSError) as cm: + socket.if_nametoindex("_DEADBEEF") + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(TypeError, socket.if_nametoindex, 0) - self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') @unittest.skipUnless(hasattr(sys, 'getrefcount'), 'test needs sys.getrefcount()') @@ -1116,23 +1240,24 @@ def testNtoH(self): self.assertEqual(swapped & mask, mask) self.assertRaises(OverflowError, func, 1<<34) - @support.cpython_only + @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust u16 def testNtoHErrors(self): - import _testcapi s_good_values = [0, 1, 2, 0xffff] l_good_values = s_good_values + [0xffffffff] - l_bad_values = [-1, -2, 1<<32, 1<<1000] - s_bad_values = ( - l_bad_values + - [_testcapi.INT_MIN-1, _testcapi.INT_MAX+1] + - [1 << 16, _testcapi.INT_MAX] - ) + neg_values = [-1, -2, -(1<<15)-1, -(1<<31)-1, -(1<<63)-1, -1<<1000] + l_bad_values = [1<<32, 1<<1000] + s_bad_values = l_bad_values + [1 << 16, (1<<31)-1, 1<<31] for k in s_good_values: socket.ntohs(k) socket.htons(k) for k in l_good_values: socket.ntohl(k) socket.htonl(k) + for k in neg_values: + self.assertRaises(ValueError, socket.ntohs, k) + self.assertRaises(ValueError, socket.htons, k) + self.assertRaises(ValueError, socket.ntohl, k) + self.assertRaises(ValueError, socket.htonl, k) for k in s_bad_values: self.assertRaises(OverflowError, socket.ntohs, k) self.assertRaises(OverflowError, socket.htons, k) @@ -1145,8 +1270,11 @@ def testGetServBy(self): # Find one service that exists, then check all the related interfaces. # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. - if (sys.platform.startswith(('freebsd', 'netbsd', 'gnukfreebsd')) - or sys.platform in ('linux', 'darwin')): + if ( + sys.platform.startswith( + ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) + or is_apple + ): # avoid the 'echo' service on this platform, as there is an # assumption breaking non-standard port/protocol entry services = ('daytime', 'qotd', 'domain') @@ -1161,9 +1289,8 @@ def testGetServBy(self): else: raise OSError # Try same call with optional protocol omitted - # Issue #26936: Android getservbyname() was broken before API 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue gh-71123: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): port2 = socket.getservbyname(service) eq(port, port2) # Try udp, but don't barf if it doesn't exist @@ -1174,8 +1301,9 @@ def testGetServBy(self): else: eq(udpport, port) # Now make sure the lookup by port returns the same service name - # Issue #26936: Android getservbyport() is broken. - if not support.is_android: + # Issue #26936: when the protocol is omitted, this fails on Android + # before API level 28. + if not (support.is_android and platform.android_ver().api_level < 28): eq(socket.getservbyport(port2), service) eq(socket.getservbyport(port, 'tcp'), service) if udpport is not None: @@ -1378,10 +1506,21 @@ def testStringToIPv6(self): def testSockName(self): # Testing getsockname() - port = socket_helper.find_unused_port() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.addCleanup(sock.close) - sock.bind(("0.0.0.0", port)) + + # Since find_unused_port() is inherently subject to race conditions, we + # call it a couple times if necessary. + for i in itertools.count(): + port = socket_helper.find_unused_port() + try: + sock.bind(("0.0.0.0", port)) + except OSError as e: + if e.errno != errno.EADDRINUSE or i == 5: + raise + else: + break + name = sock.getsockname() # XXX(nnorwitz): http://tinyurl.com/os5jz seems to indicate # it reasonable to get the host's addr in addition to 0.0.0.0. @@ -1462,14 +1601,12 @@ def test_getsockaddrarg(self): break @unittest.skipUnless(os.name == "nt", "Windows specific") - # TODO: RUSTPYTHON, windows ioctls - @unittest.expectedFailure def test_sock_ioctl(self): - self.assertTrue(hasattr(socket.socket, 'ioctl')) - self.assertTrue(hasattr(socket, 'SIO_RCVALL')) - self.assertTrue(hasattr(socket, 'RCVALL_ON')) - self.assertTrue(hasattr(socket, 'RCVALL_OFF')) - self.assertTrue(hasattr(socket, 'SIO_KEEPALIVE_VALS')) + self.assertHasAttr(socket.socket, 'ioctl') + self.assertHasAttr(socket, 'SIO_RCVALL') + self.assertHasAttr(socket, 'RCVALL_ON') + self.assertHasAttr(socket, 'RCVALL_OFF') + self.assertHasAttr(socket, 'SIO_KEEPALIVE_VALS') s = socket.socket() self.addCleanup(s.close) self.assertRaises(ValueError, s.ioctl, -1, None) @@ -1478,8 +1615,6 @@ def test_sock_ioctl(self): @unittest.skipUnless(os.name == "nt", "Windows specific") @unittest.skipUnless(hasattr(socket, 'SIO_LOOPBACK_FAST_PATH'), 'Loopback fast path support required for this test') - # TODO: RUSTPYTHON, AttributeError: 'socket' object has no attribute 'ioctl' - @unittest.expectedFailure def test_sio_loopback_fast_path(self): s = socket.socket() self.addCleanup(s.close) @@ -1493,8 +1628,6 @@ def test_sio_loopback_fast_path(self): raise self.assertRaises(TypeError, s.ioctl, socket.SIO_LOOPBACK_FAST_PATH, None) - # TODO: RUSTPYTHON, AssertionError: '2' != 'AddressFamily.AF_INET' - @unittest.expectedFailure def testGetaddrinfo(self): try: socket.getaddrinfo('localhost', 80) @@ -1515,9 +1648,8 @@ def testGetaddrinfo(self): socket.getaddrinfo('::1', 80) # port can be a string service name such as "http", a numeric # port number or None - # Issue #26936: Android getaddrinfo() was broken before API level 23. - if (not hasattr(sys, 'getandroidapilevel') or - sys.getandroidapilevel() >= 23): + # Issue #26936: this fails on Android before API level 23. + if not (support.is_android and platform.android_ver().api_level < 23): socket.getaddrinfo(HOST, "http") socket.getaddrinfo(HOST, 80) socket.getaddrinfo(HOST, None) @@ -1525,9 +1657,11 @@ def testGetaddrinfo(self): infos = socket.getaddrinfo(HOST, 80, socket.AF_INET, socket.SOCK_STREAM) for family, type, _, _, _ in infos: self.assertEqual(family, socket.AF_INET) - self.assertEqual(str(family), 'AddressFamily.AF_INET') + self.assertEqual(repr(family), '<AddressFamily.AF_INET: %r>' % family.value) + self.assertEqual(str(family), str(family.value)) self.assertEqual(type, socket.SOCK_STREAM) - self.assertEqual(str(type), 'SocketKind.SOCK_STREAM') + self.assertEqual(repr(type), '<SocketKind.SOCK_STREAM: %r>' % type.value) + self.assertEqual(str(type), str(type.value)) infos = socket.getaddrinfo(HOST, None, 0, socket.SOCK_STREAM) for _, socktype, _, _, _ in infos: self.assertEqual(socktype, socket.SOCK_STREAM) @@ -1561,11 +1695,13 @@ def testGetaddrinfo(self): flags=socket.AI_PASSIVE) self.assertEqual(a, b) # Issue #6697. - # XXX RUSTPYTHON TODO: surrogates in str - # self.assertRaises(UnicodeEncodeError, socket.getaddrinfo, 'localhost', '\uD800') + self.assertRaises(UnicodeEncodeError, socket.getaddrinfo, 'localhost', '\uD800') - # Issue 17269: test workaround for OS X platform bug segfault if hasattr(socket, 'AI_NUMERICSERV'): + self.assertRaises(socket.gaierror, socket.getaddrinfo, "localhost", "http", + flags=socket.AI_NUMERICSERV) + + # Issue 17269: test workaround for OS X platform bug segfault try: # The arguments here are undefined and the call may succeed # or fail. All we care here is that it doesn't segfault. @@ -1574,11 +1710,59 @@ def testGetaddrinfo(self): except socket.gaierror: pass + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_getaddrinfo_int_port_overflow(self): + # gh-74895: Test that getaddrinfo does not raise OverflowError on port. + # + # POSIX getaddrinfo() never specify the valid range for "service" + # decimal port number values. For IPv4 and IPv6 they are technically + # unsigned 16-bit values, but the API is protocol agnostic. Which values + # trigger an error from the C library function varies by platform as + # they do not all perform validation. + + # The key here is that we don't want to produce OverflowError as Python + # prior to 3.12 did for ints outside of a [LONG_MIN, LONG_MAX] range. + # Leave the error up to the underlying string based platform C API. + + from _testcapi import ULONG_MAX, LONG_MAX, LONG_MIN + try: + socket.getaddrinfo(None, ULONG_MAX + 1, type=socket.SOCK_STREAM) + except OverflowError: + # Platforms differ as to what values constitute a getaddrinfo() error + # return. Some fail for LONG_MAX+1, others ULONG_MAX+1, and Windows + # silently accepts such huge "port" aka "service" numeric values. + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MAX + 1, type=socket.SOCK_STREAM) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MAX - 0xffff + 1, type=socket.SOCK_STREAM) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + try: + socket.getaddrinfo(None, LONG_MIN - 1, type=socket.SOCK_STREAM) + except OverflowError: + self.fail("Either no error or socket.gaierror expected.") + except socket.gaierror: + pass + + socket.getaddrinfo(None, 0, type=socket.SOCK_STREAM) # No error expected. + socket.getaddrinfo(None, 0xffff, type=socket.SOCK_STREAM) # No error expected. + def test_getnameinfo(self): # only IP addresses are allowed self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) - @unittest.expectedFailureIf(sys.platform != "darwin", "TODO: RUSTPYTHON; socket.gethostbyname_ex") @unittest.skipUnless(support.is_resource_enabled('network'), 'network is not enabled') def test_idna(self): @@ -1633,8 +1817,6 @@ def test_sendall_interrupted(self): def test_sendall_interrupted_with_timeout(self): self.check_sendall_interrupted(True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dealloc_warn(self): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) r = repr(sock) @@ -1722,6 +1904,7 @@ def test_listen_backlog(self): srv.listen() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_listen_backlog_overflow(self): # Issue 15989 import _testcapi @@ -1746,10 +1929,15 @@ def test_getaddrinfo_ipv6_basic(self): ) self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + def test_getfqdn_filter_localhost(self): + self.assertEqual(socket.getfqdn(), socket.getfqdn("0.0.0.0")) + self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") + @support.skip_android_selinux('if_nameindex') def test_getaddrinfo_ipv6_scopeid_symbolic(self): # Just pick up any network interface (Linux, Mac OS X) (ifindex, test_interface) = socket.if_nameindex()[0] @@ -1783,6 +1971,7 @@ def test_getaddrinfo_ipv6_scopeid_numeric(self): @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") + @support.skip_android_selinux('if_nameindex') def test_getnameinfo_ipv6_scopeid_symbolic(self): # Just pick up any network interface. (ifindex, test_interface) = socket.if_nameindex()[0] @@ -1801,16 +1990,15 @@ def test_getnameinfo_ipv6_scopeid_numeric(self): nameinfo = socket.getnameinfo(sockaddr, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV) self.assertEqual(nameinfo, ('ff02::1de:c0:face:8d%' + str(ifindex), '1234')) - # TODO: RUSTPYTHON, AssertionError: '2' != 'AddressFamily.AF_INET' - @unittest.expectedFailure def test_str_for_enums(self): # Make sure that the AF_* and SOCK_* constants have enum-like string # reprs. with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - self.assertEqual(str(s.family), 'AddressFamily.AF_INET') - self.assertEqual(str(s.type), 'SocketKind.SOCK_STREAM') + self.assertEqual(repr(s.family), '<AddressFamily.AF_INET: %r>' % s.family.value) + self.assertEqual(repr(s.type), '<SocketKind.SOCK_STREAM: %r>' % s.type.value) + self.assertEqual(str(s.family), str(s.family.value)) + self.assertEqual(str(s.type), str(s.type.value)) - @unittest.expectedFailureIf(sys.platform.startswith("linux"), "TODO: RUSTPYTHON, AssertionError: 526337 != <SocketKind.SOCK_STREAM: 1>") def test_socket_consistent_sock_type(self): SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) SOCK_CLOEXEC = getattr(socket, 'SOCK_CLOEXEC', 0) @@ -1902,17 +2090,18 @@ def test_socket_fileno(self): self._test_socket_fileno(s, socket.AF_INET6, socket.SOCK_STREAM) if hasattr(socket, "AF_UNIX"): - tmpdir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmpdir) + unix_name = socket_helper.create_unix_domain_name() + self.addCleanup(os_helper.unlink, unix_name) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.addCleanup(s.close) - try: - s.bind(os.path.join(tmpdir, 'socket')) - except PermissionError: - pass - else: - self._test_socket_fileno(s, socket.AF_UNIX, - socket.SOCK_STREAM) + with s: + try: + s.bind(unix_name) + except PermissionError: + pass + else: + self._test_socket_fileno(s, socket.AF_UNIX, + socket.SOCK_STREAM) def test_socket_fileno_rejects_float(self): with self.assertRaises(TypeError): @@ -1956,6 +2145,41 @@ def test_socket_fileno_requires_socket_fd(self): fileno=afile.fileno()) self.assertEqual(cm.exception.errno, errno.ENOTSOCK) + def test_addressfamily_enum(self): + import _socket, enum + CheckedAddressFamily = enum._old_convert_( + enum.IntEnum, 'AddressFamily', 'socket', + lambda C: C.isupper() and C.startswith('AF_'), + source=_socket, + ) + enum._test_simple_enum(CheckedAddressFamily, socket.AddressFamily) + + def test_socketkind_enum(self): + import _socket, enum + CheckedSocketKind = enum._old_convert_( + enum.IntEnum, 'SocketKind', 'socket', + lambda C: C.isupper() and C.startswith('SOCK_'), + source=_socket, + ) + enum._test_simple_enum(CheckedSocketKind, socket.SocketKind) + + def test_msgflag_enum(self): + import _socket, enum + CheckedMsgFlag = enum._old_convert_( + enum.IntFlag, 'MsgFlag', 'socket', + lambda C: C.isupper() and C.startswith('MSG_'), + source=_socket, + ) + enum._test_simple_enum(CheckedMsgFlag, socket.MsgFlag) + + def test_addressinfo_enum(self): + import _socket, enum + CheckedAddressInfo = enum._old_convert_( + enum.IntFlag, 'AddressInfo', 'socket', + lambda C: C.isupper() and C.startswith('AI_'), + source=_socket) + enum._test_simple_enum(CheckedAddressInfo, socket.AddressInfo) + @unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') class BasicCANTest(unittest.TestCase): @@ -1967,8 +2191,6 @@ def testCrucialConstants(self): @unittest.skipUnless(hasattr(socket, "CAN_BCM"), 'socket.CAN_BCM required for this test.') - # TODO: RUSTPYTHON, AttributeError: module 'socket' has no attribute 'CAN_BCM_TX_SETUP' - @unittest.expectedFailure def testBCMConstants(self): socket.CAN_BCM @@ -2009,16 +2231,12 @@ def testCreateBCMSocket(self): with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) as s: pass - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def testBindAny(self): with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: address = ('', ) s.bind(address) self.assertEqual(s.getsockname(), address) - # TODO: RUSTPYTHON, AssertionError: "interface name too long" does not match "bind(): bad family" - @unittest.expectedFailure def testTooLongInterfaceName(self): # most systems limit IFNAMSIZ to 16, take 1024 to be sure with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: @@ -2161,12 +2379,14 @@ def testCreateISOTPSocket(self): with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: pass + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: AF_CAN address must be a tuple (interface,) or (interface, addr) def testTooLongInterfaceName(self): # most systems limit IFNAMSIZ to 16, take 1024 to be sure with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: with self.assertRaisesRegex(OSError, 'interface name too long'): s.bind(('x' * 1024, 1, 2)) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: AF_CAN address must be a tuple (interface,) or (interface, addr) def testBind(self): try: with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: @@ -2188,8 +2408,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.interface = "vcan0" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - J1939 constants not fully implemented @unittest.skipUnless(hasattr(socket, "CAN_J1939"), 'socket.CAN_J1939 required for this test.') def testJ1939Constants(self): @@ -2231,8 +2450,7 @@ def testCreateJ1939Socket(self): with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_J1939) as s: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_CAN J1939 address format not fully implemented def testBind(self): try: with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_J1939) as s: @@ -2371,6 +2589,7 @@ def testVSOCKConstants(self): socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE socket.VMADDR_CID_ANY socket.VMADDR_PORT_ANY + socket.VMADDR_CID_LOCAL socket.VMADDR_CID_HOST socket.VM_SOCKETS_INVALID_VERSION socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID @@ -2406,23 +2625,88 @@ def testSocketBufferSize(self): socket.SO_VM_SOCKETS_BUFFER_MIN_SIZE)) -@unittest.skipUnless(HAVE_SOCKET_BLUETOOTH, +@unittest.skipUnless(hasattr(socket, 'AF_BLUETOOTH'), 'Bluetooth sockets required for this test.') class BasicBluetoothTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'socket' has no attribute 'BTPROTO_RFCOMM' def testBluetoothConstants(self): socket.BDADDR_ANY socket.BDADDR_LOCAL socket.AF_BLUETOOTH socket.BTPROTO_RFCOMM + socket.SOL_RFCOMM + + if sys.platform == "win32": + socket.SO_BTH_ENCRYPT + socket.SO_BTH_MTU + socket.SO_BTH_MTU_MAX + socket.SO_BTH_MTU_MIN if sys.platform != "win32": socket.BTPROTO_HCI socket.SOL_HCI socket.BTPROTO_L2CAP + socket.SOL_L2CAP + socket.BTPROTO_SCO + socket.SOL_SCO + socket.HCI_DATA_DIR + + if sys.platform == "linux": + socket.SOL_BLUETOOTH + socket.HCI_DEV_NONE + socket.HCI_CHANNEL_RAW + socket.HCI_CHANNEL_USER + socket.HCI_CHANNEL_MONITOR + socket.HCI_CHANNEL_CONTROL + socket.HCI_CHANNEL_LOGGING + socket.HCI_TIME_STAMP + socket.BT_SECURITY + socket.BT_SECURITY_SDP + socket.BT_FLUSHABLE + socket.BT_POWER + socket.BT_CHANNEL_POLICY + socket.BT_CHANNEL_POLICY_BREDR_ONLY + if hasattr(socket, 'BT_PHY'): + socket.BT_PHY_BR_1M_1SLOT + if hasattr(socket, 'BT_MODE'): + socket.BT_MODE_BASIC + if hasattr(socket, 'BT_VOICE'): + socket.BT_VOICE_TRANSPARENT + socket.BT_VOICE_CVSD_16BIT + socket.L2CAP_LM + socket.L2CAP_LM_MASTER + socket.L2CAP_LM_AUTH + + if sys.platform in ("linux", "freebsd"): + socket.BDADDR_BREDR + socket.BDADDR_LE_PUBLIC + socket.BDADDR_LE_RANDOM + socket.HCI_FILTER + + if sys.platform.startswith(("freebsd", "netbsd", "dragonfly")): + socket.SO_L2CAP_IMTU + socket.SO_L2CAP_FLUSH + socket.SO_RFCOMM_MTU + socket.SO_RFCOMM_FC_INFO + socket.SO_SCO_MTU + + if sys.platform == "freebsd": + socket.SO_SCO_CONNINFO + + if sys.platform.startswith(("netbsd", "dragonfly")): + socket.SO_HCI_EVT_FILTER + socket.SO_HCI_PKT_FILTER + socket.SO_L2CAP_IQOS + socket.SO_L2CAP_LM + socket.L2CAP_LM_AUTH + socket.SO_RFCOMM_LM + socket.RFCOMM_LM_AUTH + socket.SO_SCO_HANDLE - if not sys.platform.startswith("freebsd"): - socket.BTPROTO_SCO +@unittest.skipUnless(HAVE_SOCKET_BLUETOOTH, + 'Bluetooth sockets required for this test.') +class BluetoothTest(unittest.TestCase): def testCreateRfcommSocket(self): with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM) as s: @@ -2438,12 +2722,245 @@ def testCreateHciSocket(self): with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s: pass - @unittest.skipIf(sys.platform == "win32" or sys.platform.startswith("freebsd"), - "windows and freebsd do not support SCO sockets") + @unittest.skipIf(sys.platform == "win32", "windows does not support SCO sockets") def testCreateScoSocket(self): with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_SCO) as s: pass + @unittest.skipUnless(HAVE_SOCKET_BLUETOOTH_L2CAP, 'Bluetooth L2CAP sockets required for this test') + def testBindLeAttL2capSocket(self): + BDADDR_LE_PUBLIC = support.get_attribute(socket, 'BDADDR_LE_PUBLIC') + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) as f: + # ATT is the only CID allowed in userspace by the Linux kernel + CID_ATT = 4 + f.bind((socket.BDADDR_ANY, 0, CID_ATT, BDADDR_LE_PUBLIC)) + addr = f.getsockname() + self.assertEqual(addr, (socket.BDADDR_ANY, 0, CID_ATT, BDADDR_LE_PUBLIC)) + + @unittest.skipUnless(HAVE_SOCKET_BLUETOOTH_L2CAP, 'Bluetooth L2CAP sockets required for this test') + def testBindLePsmL2capSocket(self): + BDADDR_LE_RANDOM = support.get_attribute(socket, 'BDADDR_LE_RANDOM') + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) as f: + # First user PSM in LE L2CAP + psm = 0x80 + f.bind((socket.BDADDR_ANY, psm, 0, BDADDR_LE_RANDOM)) + addr = f.getsockname() + self.assertEqual(addr, (socket.BDADDR_ANY, psm, 0, BDADDR_LE_RANDOM)) + + @unittest.skipUnless(HAVE_SOCKET_BLUETOOTH_L2CAP, 'Bluetooth L2CAP sockets required for this test') + def testBindBrEdrL2capSocket(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) as f: + # First user PSM in BR/EDR L2CAP + psm = 0x1001 + f.bind((socket.BDADDR_ANY, psm)) + addr = f.getsockname() + self.assertEqual(addr, (socket.BDADDR_ANY, psm)) + + @unittest.skipUnless(HAVE_SOCKET_BLUETOOTH_L2CAP, 'Bluetooth L2CAP sockets required for this test') + def testBadL2capAddr(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_L2CAP) as f: + with self.assertRaises(OSError): + f.bind((socket.BDADDR_ANY, 0, 0, 0, 0)) + with self.assertRaises(OSError): + f.bind((socket.BDADDR_ANY,)) + with self.assertRaises(OSError): + f.bind(socket.BDADDR_ANY) + with self.assertRaises(OSError): + f.bind((socket.BDADDR_ANY.encode(), 0x1001)) + with self.assertRaises(OSError): + f.bind(('\ud812', 0x1001)) + + def testBindRfcommSocket(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM) as s: + channel = 0 + try: + s.bind((socket.BDADDR_ANY, channel)) + except OSError as err: + if sys.platform == 'win32' and err.winerror == 10050: + self.skipTest(str(err)) + raise + addr = s.getsockname() + self.assertEqual(addr, (mock.ANY, channel)) + self.assertRegex(addr[0], r'(?i)[0-9a-f]{2}(?::[0-9a-f]{2}){4}') + if sys.platform != 'win32': + self.assertEqual(addr, (socket.BDADDR_ANY, channel)) + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM) as s: + s.bind(addr) + addr2 = s.getsockname() + self.assertEqual(addr2, addr) + + def testBadRfcommAddr(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, socket.BTPROTO_RFCOMM) as s: + channel = 0 + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY.encode(), channel)) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY,)) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY, channel, 0)) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY + '\0', channel)) + with self.assertRaises(OSError): + s.bind('\ud812') + with self.assertRaises(OSError): + s.bind(('invalid', channel)) + + @unittest.skipUnless(hasattr(socket, 'BTPROTO_HCI'), 'Bluetooth HCI sockets required for this test') + def testBindHciSocket(self): + if sys.platform.startswith(('netbsd', 'dragonfly', 'freebsd')): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s: + s.bind(socket.BDADDR_ANY) + addr = s.getsockname() + self.assertEqual(addr, socket.BDADDR_ANY) + else: + dev = 0 + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s: + try: + s.bind((dev,)) + except OSError as err: + if err.errno in (errno.EINVAL, errno.ENODEV): + self.skipTest(str(err)) + raise + addr = s.getsockname() + self.assertEqual(addr, dev) + + with (self.subTest('integer'), + socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s): + s.bind(dev) + addr = s.getsockname() + self.assertEqual(addr, dev) + + with (self.subTest('channel=HCI_CHANNEL_RAW'), + socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s): + channel = socket.HCI_CHANNEL_RAW + s.bind((dev, channel)) + addr = s.getsockname() + self.assertEqual(addr, dev) + + with (self.subTest('channel=HCI_CHANNEL_USER'), + socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s): + channel = socket.HCI_CHANNEL_USER + try: + s.bind((dev, channel)) + except OSError as err: + # Needs special permissions. + if err.errno in (errno.EPERM, errno.EBUSY, errno.ERFKILL): + self.skipTest(str(err)) + raise + addr = s.getsockname() + self.assertEqual(addr, (dev, channel)) + + @unittest.skipUnless(hasattr(socket, 'BTPROTO_HCI'), 'Bluetooth HCI sockets required for this test') + def testBadHciAddr(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_RAW, socket.BTPROTO_HCI) as s: + if sys.platform.startswith(('netbsd', 'dragonfly', 'freebsd')): + with self.assertRaises(OSError): + s.bind(socket.BDADDR_ANY.encode()) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY,)) + with self.assertRaises(OSError): + s.bind(socket.BDADDR_ANY + '\0') + with self.assertRaises((ValueError, OSError)): + s.bind(socket.BDADDR_ANY + ' '*100) + with self.assertRaises(OSError): + s.bind('\ud812') + with self.assertRaises(OSError): + s.bind('invalid') + with self.assertRaises(OSError): + s.bind(b'invalid') + else: + dev = 0 + with self.assertRaises(OSError): + s.bind(()) + with self.assertRaises(OSError): + s.bind((dev, socket.HCI_CHANNEL_RAW, 0, 0)) + with self.assertRaises(OSError): + s.bind(socket.BDADDR_ANY) + with self.assertRaises(OSError): + s.bind(socket.BDADDR_ANY.encode()) + + @unittest.skipUnless(hasattr(socket, 'BTPROTO_SCO'), 'Bluetooth SCO sockets required for this test') + def testBindScoSocket(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_SCO) as s: + s.bind(socket.BDADDR_ANY) + addr = s.getsockname() + self.assertEqual(addr, socket.BDADDR_ANY) + + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_SCO) as s: + s.bind(socket.BDADDR_ANY.encode()) + addr = s.getsockname() + self.assertEqual(addr, socket.BDADDR_ANY) + + @unittest.skipUnless(hasattr(socket, 'BTPROTO_SCO'), 'Bluetooth SCO sockets required for this test') + def testBadScoAddr(self): + with socket.socket(socket.AF_BLUETOOTH, socket.SOCK_SEQPACKET, socket.BTPROTO_SCO) as s: + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY,)) + with self.assertRaises(OSError): + s.bind((socket.BDADDR_ANY.encode(),)) + with self.assertRaises(ValueError): + s.bind(socket.BDADDR_ANY + '\0') + with self.assertRaises(ValueError): + s.bind(socket.BDADDR_ANY.encode() + b'\0') + with self.assertRaises(UnicodeEncodeError): + s.bind('\ud812') + with self.assertRaises(OSError): + s.bind('invalid') + with self.assertRaises(OSError): + s.bind(b'invalid') + + +@unittest.skipUnless(HAVE_SOCKET_HYPERV, + 'Hyper-V sockets required for this test.') +class BasicHyperVTest(unittest.TestCase): + + def testHyperVConstants(self): + socket.HVSOCKET_CONNECT_TIMEOUT + socket.HVSOCKET_CONNECT_TIMEOUT_MAX + socket.HVSOCKET_CONNECTED_SUSPEND + socket.HVSOCKET_ADDRESS_FLAG_PASSTHRU + socket.HV_GUID_ZERO + socket.HV_GUID_WILDCARD + socket.HV_GUID_BROADCAST + socket.HV_GUID_CHILDREN + socket.HV_GUID_LOOPBACK + socket.HV_GUID_PARENT + + def testCreateHyperVSocketWithUnknownProtoFailure(self): + expected = r"\[WinError 10041\]" + with self.assertRaisesRegex(OSError, expected): + socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM) + + def testCreateHyperVSocketAddrNotTupleFailure(self): + expected = "connect(): AF_HYPERV address must be tuple, not str" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(TypeError, re.escape(expected)): + s.connect(socket.HV_GUID_ZERO) + + def testCreateHyperVSocketAddrNotTupleOf2StrsFailure(self): + expected = "AF_HYPERV address must be a str tuple (vm_id, service_id)" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(TypeError, re.escape(expected)): + s.connect((socket.HV_GUID_ZERO,)) + + def testCreateHyperVSocketAddrNotTupleOfStrsFailure(self): + expected = "AF_HYPERV address must be a str tuple (vm_id, service_id)" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(TypeError, re.escape(expected)): + s.connect((1, 2)) + + def testCreateHyperVSocketAddrVmIdNotValidUUIDFailure(self): + expected = "connect(): AF_HYPERV address vm_id is not a valid UUID string" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(ValueError, re.escape(expected)): + s.connect(("00", socket.HV_GUID_ZERO)) + + def testCreateHyperVSocketAddrServiceIdNotValidUUIDFailure(self): + expected = "connect(): AF_HYPERV address service_id is not a valid UUID string" + with socket.socket(socket.AF_HYPERV, socket.SOCK_STREAM, socket.HV_PROTOCOL_RAW) as s: + with self.assertRaisesRegex(ValueError, re.escape(expected)): + s.connect((socket.HV_GUID_ZERO, "00")) + class BasicTCPTest(SocketConnectedTest): @@ -2522,22 +3039,29 @@ def testDup(self): def _testDup(self): self.serv_conn.send(MSG) - def testShutdown(self): - # Testing shutdown() + def check_shutdown(self): + # Test shutdown() helper msg = self.cli_conn.recv(1024) self.assertEqual(msg, MSG) - # wait for _testShutdown to finish: on OS X, when the server + # wait for _testShutdown[_overflow] to finish: on OS X, when the server # closes the connection the client also becomes disconnected, # and the client's shutdown call will fail. (Issue #4397.) self.done.wait() + def testShutdown(self): + self.check_shutdown() + def _testShutdown(self): self.serv_conn.send(MSG) self.serv_conn.shutdown(2) - testShutdown_overflow = support.cpython_only(testShutdown) + @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def testShutdown_overflow(self): + self.check_shutdown() @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def _testShutdown_overflow(self): import _testcapi self.serv_conn.send(MSG) @@ -2654,7 +3178,7 @@ def _testRecvFromNegative(self): # here assumes that datagram delivery on the local machine will be # reliable. -class SendrecvmsgBase(ThreadSafeCleanupTestCase): +class SendrecvmsgBase: # Base class for sendmsg()/recvmsg() tests. # Time in seconds to wait before considering a test failed, or @@ -3008,7 +3532,7 @@ def _testSendmsgTimeout(self): # Linux supports MSG_DONTWAIT when sending, but in general, it # only works when receiving. Could add other platforms if they # support it too. - @skipWithClientIf(sys.platform not in {"linux"}, + @skipWithClientIf(sys.platform not in {"linux", "android"}, "MSG_DONTWAIT not known to work on this platform when " "sending") def testSendmsgDontWait(self): @@ -3380,6 +3904,10 @@ def testCMSG_SPACE(self): # Test CMSG_SPACE() with various valid and invalid values, # checking the assumptions used by sendmsg(). toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + if SOLARIS and platform.processor() == "sparc": + # On Solaris SPARC, number of bytes returned by socket.CMSG_SPACE + # increases at different lengths; see gh-91214. + toobig -= 3 values = list(range(257)) + list(range(toobig - 257, toobig)) last = socket.CMSG_SPACE(0) @@ -3525,7 +4053,8 @@ def testFDPassCMSG_LEN(self): def _testFDPassCMSG_LEN(self): self.createAndSendFDs(1) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): @@ -3536,7 +4065,8 @@ def testFDPassSeparate(self): maxcmsgs=2) @testFDPassSeparate.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) @@ -3549,7 +4079,8 @@ def _testFDPassSeparate(self): array.array("i", [fd1]))]), len(MSG)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): @@ -3563,7 +4094,8 @@ def testFDPassSeparateMinSpace(self): maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) @testFDPassSeparateMinSpace.client_skip - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) @@ -3587,7 +4119,7 @@ def sendAncillaryIfPossible(self, msg, ancdata): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "see issue #24725") + @unittest.skipIf(is_apple, "skipping, see issue #12958") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. @@ -3661,6 +4193,7 @@ def checkTruncatedHeader(self, result, ignoreflags=0): self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, ignore=ignoreflags) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncNoBufSize(self): # Check that no ancillary data is received when no buffer size # is specified. @@ -3670,26 +4203,32 @@ def testCmsgTruncNoBufSize(self): # received. ignoreflags=socket.MSG_CTRUNC) + @testCmsgTruncNoBufSize.client_skip def _testCmsgTruncNoBufSize(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc0(self): # Check that no ancillary data is received when buffer size is 0. self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), ignoreflags=socket.MSG_CTRUNC) + @testCmsgTrunc0.client_skip def _testCmsgTrunc0(self): self.createAndSendFDs(1) # Check that no ancillary data is returned for various non-zero # (but still too small) buffer sizes. + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + @testCmsgTrunc1.client_skip def _testCmsgTrunc1(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTrunc2Int(self): # The cmsghdr structure has at least three members, two of # which are ints, so we still shouldn't see any ancillary @@ -3697,13 +4236,16 @@ def testCmsgTrunc2Int(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), SIZEOF_INT * 2)) + @testCmsgTrunc2Int.client_skip def _testCmsgTrunc2Int(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0Minus1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), socket.CMSG_LEN(0) - 1)) + @testCmsgTruncLen0Minus1.client_skip def _testCmsgTruncLen0Minus1(self): self.createAndSendFDs(1) @@ -3715,8 +4257,9 @@ def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): # mindata and maxdata bytes when received with buffer size # ancbuf, and that any complete file descriptor numbers are # valid. - msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, - len(MSG), ancbuf) + with downgrade_malformed_data_warning(): # TODO: gh-110012 + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) self.assertEqual(msg, MSG) self.checkRecvmsgAddress(addr, self.cli_addr) self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) @@ -3734,29 +4277,38 @@ def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) self.checkFDs(fds) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + @testCmsgTruncLen0.client_skip def _testCmsgTruncLen0(self): self.createAndSendFDs(1) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen0Plus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + @testCmsgTruncLen0Plus1.client_skip def _testCmsgTruncLen0Plus1(self): self.createAndSendFDs(2) + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), maxdata=SIZEOF_INT) + @testCmsgTruncLen1.client_skip def _testCmsgTruncLen1(self): self.createAndSendFDs(2) + + @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") def testCmsgTruncLen2Minus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, maxdata=(2 * SIZEOF_INT) - 1) + @testCmsgTruncLen2Minus1.client_skip def _testCmsgTruncLen2Minus1(self): self.createAndSendFDs(2) @@ -4058,8 +4610,9 @@ def testSingleCmsgTruncInData(self): self.serv_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_RECVHOPLIMIT, 1) self.misc_event.set() - msg, ancdata, flags, addr = self.doRecvmsg( - self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + with downgrade_malformed_data_warning(): # TODO: gh-110012 + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) self.assertEqual(msg, MSG) self.checkRecvmsgAddress(addr, self.cli_addr) @@ -4162,9 +4715,10 @@ def testSecondCmsgTruncInData(self): self.serv_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_RECVTCLASS, 1) self.misc_event.set() - msg, ancdata, flags, addr = self.doRecvmsg( - self.serv_sock, len(MSG), - socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + with downgrade_malformed_data_warning(): # TODO: gh-110012 + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) self.assertEqual(msg, MSG) self.checkRecvmsgAddress(addr, self.cli_addr) @@ -4520,7 +5074,6 @@ def testInterruptedRecvmsgIntoTimeout(self): @unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), "Don't have signal.alarm or signal.setitimer") class InterruptedSendTimeoutTest(InterruptedTimeoutBase, - ThreadSafeCleanupTestCase, SocketListeningTestMixin, TCPTestBase): # Test interrupting the interruptible send*() methods with signals # when a timeout is set. @@ -4575,15 +5128,13 @@ def testInterruptedSendmsgTimeout(self): class TCPCloserTest(ThreadedTCPSocketTest): - def testClose(self): - conn, addr = self.serv.accept() - conn.close() + conn, _ = self.serv.accept() - sd = self.cli - read, write, err = select.select([sd], [], [], 1.0) - self.assertEqual(read, [sd]) - self.assertEqual(sd.recv(1), b'') + read, _, _ = select.select([conn], [], [], support.SHORT_TIMEOUT) + self.assertEqual(read, [conn]) + self.assertEqual(conn.recv(1), b'x') + conn.close() # Calling close() many times should be safe. conn.close() @@ -4591,7 +5142,10 @@ def testClose(self): def _testClose(self): self.cli.connect((HOST, self.port)) - time.sleep(1.0) + self.cli.send(b'x') + read, _, _ = select.select([self.cli], [], [], support.SHORT_TIMEOUT) + self.assertEqual(read, [self.cli]) + self.assertEqual(self.cli.recv(1), b'') class BasicSocketPairTest(SocketPairTest): @@ -4629,6 +5183,112 @@ def _testSend(self): self.assertEqual(msg, MSG) +class PurePythonSocketPairTest(SocketPairTest): + # Explicitly use socketpair AF_INET or AF_INET6 to ensure that is the + # code path we're using regardless platform is the pure python one where + # `_socket.socketpair` does not exist. (AF_INET does not work with + # _socket.socketpair on many platforms). + def socketpair(self): + # called by super().setUp(). + try: + return socket.socketpair(socket.AF_INET6) + except OSError: + return socket.socketpair(socket.AF_INET) + + # Local imports in this class make for easy security fix backporting. + + def setUp(self): + if hasattr(_socket, "socketpair"): + self._orig_sp = socket.socketpair + # This forces the version using the non-OS provided socketpair + # emulation via an AF_INET socket in Lib/socket.py. + socket.socketpair = socket._fallback_socketpair + else: + # This platform already uses the non-OS provided version. + self._orig_sp = None + super().setUp() + + def tearDown(self): + super().tearDown() + if self._orig_sp is not None: + # Restore the default socket.socketpair definition. + socket.socketpair = self._orig_sp + + def test_recv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _test_recv(self): + self.cli.send(MSG) + + def test_send(self): + self.serv.send(MSG) + + def _test_send(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + + def test_ipv4(self): + cli, srv = socket.socketpair(socket.AF_INET) + cli.close() + srv.close() + + def _test_ipv4(self): + pass + + @unittest.skipIf(not hasattr(_socket, 'IPPROTO_IPV6') or + not hasattr(_socket, 'IPV6_V6ONLY'), + "IPV6_V6ONLY option not supported") + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test') + def test_ipv6(self): + cli, srv = socket.socketpair(socket.AF_INET6) + cli.close() + srv.close() + + def _test_ipv6(self): + pass + + def test_injected_authentication_failure(self): + orig_getsockname = socket.socket.getsockname + inject_sock = None + + def inject_getsocketname(self): + nonlocal inject_sock + sockname = orig_getsockname(self) + # Connect to the listening socket ahead of the + # client socket. + if inject_sock is None: + inject_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + inject_sock.setblocking(False) + try: + inject_sock.connect(sockname[:2]) + except (BlockingIOError, InterruptedError): + pass + inject_sock.setblocking(True) + return sockname + + sock1 = sock2 = None + try: + socket.socket.getsockname = inject_getsocketname + with self.assertRaises(OSError): + sock1, sock2 = socket.socketpair() + finally: + socket.socket.getsockname = orig_getsockname + if inject_sock: + inject_sock.close() + if sock1: # This cleanup isn't needed on a successful test. + sock1.close() + if sock2: + sock2.close() + + def _test_injected_authentication_failure(self): + # No-op. Exists for base class threading infrastructure to call. + # We could refactor this test into its own lesser class along with the + # setUp and tearDown code to construct an ideal; it is simpler to keep + # it here and live with extra overhead one this _one_ failure test. + pass + + class NonBlockingTCPTests(ThreadedTCPSocketTest): def __init__(self, methodName='runTest'): @@ -4676,6 +5336,7 @@ def _testSetBlocking(self): pass @support.cpython_only + @unittest.skipIf(_testcapi is None, "requires _testcapi") def testSetBlocking_overflow(self): # Issue 15989 import _testcapi @@ -4693,8 +5354,6 @@ def testSetBlocking_overflow(self): @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), 'test needs socket.SOCK_NONBLOCK') @support.requires_linux_version(2, 6, 28) - # TODO: RUSTPYTHON, AssertionError: None != 0 - @unittest.expectedFailure def testInitNonBlocking(self): # create a socket with SOCK_NONBLOCK self.serv.close() @@ -4790,6 +5449,39 @@ def _testRecv(self): # send data: recv() will no longer block self.cli.sendall(MSG) + def testLargeTimeout(self): + # gh-126876: Check that a timeout larger than INT_MAX is replaced with + # INT_MAX in the poll() code path. The following assertion must not + # fail: assert(INT_MIN <= ms && ms <= INT_MAX). + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 + + # test recv() with large timeout + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + try: + conn.settimeout(large_timeout) + except OverflowError: + # On Windows, settimeout() fails with OverflowError, whereas + # we want to test recv(). Just give up silently. + return + msg = conn.recv(len(MSG)) + + def _testLargeTimeout(self): + # test sendall() with large timeout + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 + self.cli.connect((HOST, self.port)) + try: + self.cli.settimeout(large_timeout) + except OverflowError: + return + self.cli.sendall(MSG) + class FileObjectClassTestCase(SocketConnectedTest): """Unit tests for the object returned by socket.makefile() @@ -4992,6 +5684,8 @@ def _testMakefileClose(self): self.write_file.write(self.write_msg) self.write_file.flush() + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') def testMakefileCloseSocketDestroy(self): refcount_before = sys.getrefcount(self.cli_conn) self.read_file.close() @@ -5131,6 +5825,7 @@ def mocked_socket_module(self): finally: socket.socket = old_socket + @socket_helper.skip_if_tcp_blackhole def test_connect(self): port = socket_helper.find_unused_port() cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -5139,6 +5834,7 @@ def test_connect(self): cli.connect((HOST, port)) self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + @socket_helper.skip_if_tcp_blackhole def test_create_connection(self): # Issue #9792: errors raised by create_connection() should have # a proper errno attribute. @@ -5163,6 +5859,24 @@ def test_create_connection(self): expected_errnos = socket_helper.get_socket_conn_refused_errs() self.assertIn(cm.exception.errno, expected_errnos) + def test_create_connection_all_errors(self): + port = socket_helper.find_unused_port() + try: + socket.create_connection((HOST, port), all_errors=True) + except ExceptionGroup as e: + eg = e + else: + self.fail('expected connection to fail') + + self.assertIsInstance(eg, ExceptionGroup) + for e in eg.exceptions: + self.assertIsInstance(e, OSError) + + addresses = socket.getaddrinfo( + 'localhost', port, 0, socket.SOCK_STREAM) + # assert that we got an exception for each address + self.assertEqual(len(addresses), len(eg.exceptions)) + def test_create_connection_timeout(self): # Issue #9792: create_connection() should not recast timeout errors # as generic socket errors. @@ -5179,6 +5893,7 @@ def test_create_connection_timeout(self): class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + cli = None def __init__(self, methodName='runTest'): SocketTCPTest.__init__(self, methodName=methodName) @@ -5188,7 +5903,8 @@ def clientSetUp(self): self.source_port = socket_helper.find_unused_port() def clientTearDown(self): - self.cli.close() + if self.cli is not None: + self.cli.close() self.cli = None ThreadableTest.clientTearDown(self) @@ -5323,10 +6039,10 @@ def alarm_handler(signal, frame): self.fail("caught timeout instead of Alarm") except Alarm: pass - except: + except BaseException as e: self.fail("caught other exception instead of Alarm:" " %s(%s):\n%s" % - (sys.exc_info()[:2] + (traceback.format_exc(),))) + (type(e), e, traceback.format_exc())) else: self.fail("nothing caught") finally: @@ -5388,10 +6104,10 @@ def testTimeoutZero(self): class TestExceptions(unittest.TestCase): def testExceptionTree(self): - self.assertTrue(issubclass(OSError, Exception)) - self.assertTrue(issubclass(socket.herror, OSError)) - self.assertTrue(issubclass(socket.gaierror, OSError)) - self.assertTrue(issubclass(socket.timeout, OSError)) + self.assertIsSubclass(OSError, Exception) + self.assertIsSubclass(socket.herror, OSError) + self.assertIsSubclass(socket.gaierror, OSError) + self.assertIsSubclass(socket.timeout, OSError) self.assertIs(socket.error, OSError) self.assertIs(socket.timeout, TimeoutError) @@ -5408,7 +6124,7 @@ def test_setblocking_invalidfd(self): sock.setblocking(False) -@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +@unittest.skipUnless(sys.platform in ('linux', 'android'), 'Linux specific test') class TestLinuxAbstractNamespace(unittest.TestCase): UNIX_PATH_MAX = 108 @@ -5514,8 +6230,6 @@ def testBytesAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - # TODO: RUSTPYTHON, surrogateescape - @unittest.expectedFailure def testSurrogateescapeBind(self): # Test binding to a valid non-ASCII pathname, with the # non-ASCII bytes supplied using surrogateescape encoding. @@ -5535,7 +6249,8 @@ def testUnencodableAddr(self): self.addCleanup(os_helper.unlink, path) self.assertEqual(self.sock.getsockname(), path) - @unittest.skipIf(sys.platform == 'linux', 'Linux specific test') + @unittest.skipIf(sys.platform in ('linux', 'android'), + 'Linux behavior is tested by TestLinuxAbstractNamespace') def testEmptyAddress(self): # Test that binding empty address fails. self.assertRaises(OSError, self.sock.bind, "") @@ -5761,8 +6476,6 @@ class InheritanceTest(unittest.TestCase): @unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), "SOCK_CLOEXEC not defined") @support.requires_linux_version(2, 6, 28) - # TODO: RUSTPYTHON, AssertionError: 524289 != <SocketKind.SOCK_STREAM: 1> - @unittest.expectedFailure def test_SOCK_CLOEXEC(self): with socket.socket(socket.AF_INET, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: @@ -5855,8 +6568,6 @@ def checkNonblock(self, s, nonblock=True, timeout=0.0): self.assertTrue(s.getblocking()) @support.requires_linux_version(2, 6, 28) - # TODO: RUSTPYTHON, AssertionError: 2049 != <SocketKind.SOCK_STREAM: 1> - @unittest.expectedFailure def test_SOCK_NONBLOCK(self): # a lot of it seems silly and redundant, but I wanted to test that # changing back and forth worked ok @@ -5892,7 +6603,6 @@ def test_SOCK_NONBLOCK(self): @unittest.skipUnless(os.name == "nt", "Windows specific") @unittest.skipUnless(multiprocessing, "need multiprocessing") -@unittest.skip("TODO: RUSTPYTHON, socket sharing") class TestSocketSharing(SocketTCPTest): # This must be classmethod and not staticmethod or multiprocessing # won't be able to bootstrap it. @@ -6133,7 +6843,6 @@ def _testCount(self): self.assertEqual(sent, count) self.assertEqual(file.tell(), count) - @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON, killed (for OOM?)") def testCount(self): count = 5000007 conn = self.accept_conn() @@ -6209,7 +6918,6 @@ def _testWithTimeout(self): sent = meth(file) self.assertEqual(sent, self.FILESIZE) - @unittest.skip("TODO: RUSTPYTHON") def testWithTimeout(self): conn = self.accept_conn() data = self.recv_data(conn) @@ -6229,6 +6937,7 @@ def _testWithTimeoutTriggeredSend(self): def testWithTimeoutTriggeredSend(self): conn = self.accept_conn() conn.recv(88192) + # bpo-45212: the wait here needs to be longer than the client-side timeout (0.01s) time.sleep(1) # errors @@ -6269,6 +6978,14 @@ class SendfileUsingSendfileTest(SendfileUsingSendTest): def meth_from_sock(self, sock): return getattr(sock, "_sendfile_use_sendfile") + @unittest.skip("TODO: RUSTPYTHON; os.sendfile count parameter not handled correctly; flaky") + def testCount(self): + return super().testCount() + + @unittest.skip("TODO: RUSTPYTHON; os.sendfile count parameter not handled correctly; flaky") + def testWithTimeout(self): + return super().testWithTimeout() + @unittest.skipUnless(HAVE_SOCKET_ALG, 'AF_ALG required') class LinuxKernelCryptoAPI(unittest.TestCase): @@ -6286,9 +7003,8 @@ def create_alg(self, typ, name): # bpo-31705: On kernel older than 4.5, sendto() failed with ENOKEY, # at least on ppc64le architecture + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented @support.requires_linux_version(4, 5) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def test_sha256(self): expected = bytes.fromhex("ba7816bf8f01cfea414140de5dae2223b00361a396" "177a9cb410ff61f20015ad") @@ -6306,22 +7022,24 @@ def test_sha256(self): op.send(b'') self.assertEqual(op.recv(512), expected) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented def test_hmac_sha1(self): - expected = bytes.fromhex("effcdf6ae5eb2fa2d27416d5f184df9c259a7c79") + # gh-109396: In FIPS mode, Linux 6.5 requires a key + # of at least 112 bits. Use a key of 152 bits. + key = b"Python loves AF_ALG" + data = b"what do ya want for nothing?" + expected = bytes.fromhex("193dbb43c6297b47ea6277ec0ce67119a3f3aa66") with self.create_alg('hash', 'hmac(sha1)') as algo: - algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, b"Jefe") + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) op, _ = algo.accept() with op: - op.sendall(b"what do ya want for nothing?") + op.sendall(data) self.assertEqual(op.recv(512), expected) # Although it should work with 3.19 and newer the test blocks on # Ubuntu 15.10 with Kernel 4.2.0-19. + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented @support.requires_linux_version(4, 3) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def test_aes_cbc(self): key = bytes.fromhex('06a9214036b8a15b512e03d534120006') iv = bytes.fromhex('3dafba429d9eb430b422da802c9fac41') @@ -6362,10 +7080,15 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 9) # see issue29324 - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented + @support.requires_linux_version(4, 9) # see gh-73510 def test_aead_aes_gcm(self): + kernel_version = support._get_kernel_version("Linux") + if kernel_version is not None: + if kernel_version >= (6, 16) and kernel_version < (6, 18): + # See https://github.com/python/cpython/issues/139310. + self.skipTest("upstream Linux kernel issue") + key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') @@ -6427,9 +7150,8 @@ def test_aead_aes_gcm(self): res = op.recv(len(msg) - taglen) self.assertEqual(plain, res[assoclen:]) + @unittest.expectedFailure # TODO: RUSTPYTHON; - AF_ALG not fully implemented @support.requires_linux_version(4, 3) # see test_aes_cbc - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def test_drbg_pr_sha256(self): # deterministic random bit generator, prediction resistance, sha256 with self.create_alg('rng', 'drbg_pr_sha256') as algo: @@ -6440,8 +7162,6 @@ def test_drbg_pr_sha256(self): rn = op.recv(32) self.assertEqual(len(rn), 32) - # TODO: RUSTPYTHON, AttributeError: 'socket' object has no attribute 'sendmsg_afalg' - @unittest.expectedFailure def test_sendmsg_afalg_args(self): sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) with sock: @@ -6460,8 +7180,6 @@ def test_sendmsg_afalg_args(self): with self.assertRaises(TypeError): sock.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, assoclen=-1) - # TODO: RUSTPYTHON, OSError: bind(): bad family - @unittest.expectedFailure def test_length_restriction(self): # bpo-35050, off-by-one error in length check sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) @@ -6485,6 +7203,28 @@ class TestMacOSTCPFlags(unittest.TestCase): def test_tcp_keepalive(self): self.assertTrue(socket.TCP_KEEPALIVE) +@unittest.skipUnless(hasattr(socket, 'TCP_QUICKACK'), 'need socket.TCP_QUICKACK') +class TestQuickackFlag(unittest.TestCase): + def check_set_quickack(self, sock): + # quickack already true by default on some OS distributions + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + if opt: + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 0) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertFalse(opt) + + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK, 1) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_QUICKACK) + self.assertTrue(opt) + + def test_set_quickack(self): + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + self.check_set_quickack(sock) + @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") class TestMSWindowsTCPFlags(unittest.TestCase): @@ -6498,7 +7238,9 @@ class TestMSWindowsTCPFlags(unittest.TestCase): 'TCP_KEEPCNT', # available starting with Windows 10 1709 'TCP_KEEPIDLE', - 'TCP_KEEPINTVL' + 'TCP_KEEPINTVL', + # available starting with Windows 7 / Server 2008 R2 + 'TCP_QUICKACK', } def test_new_tcp_flags(self): @@ -6666,6 +7408,26 @@ def close_fds(fds): self.assertEqual(data, str(index).encode()) +class FreeThreadingTests(unittest.TestCase): + + def test_close_detach_race(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def close(): + for _ in range(1000): + s.close() + + def detach(): + for _ in range(1000): + s.detach() + + t1 = threading.Thread(target=close) + t2 = threading.Thread(target=detach) + + with threading_helper.start_threads([t1, t2]): + pass + + def setUpModule(): thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 113f959ff2d..6235c8e74cf 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -8,7 +8,6 @@ import select import signal import socket -import tempfile import threading import unittest import socketserver @@ -21,6 +20,8 @@ test.support.requires("network") +test.support.requires_working_socket(module=True) + TEST_STR = b"hello world\n" HOST = socket_helper.HOST @@ -28,14 +29,9 @@ HAVE_UNIX_SOCKETS = hasattr(socket, "AF_UNIX") requires_unix_sockets = unittest.skipUnless(HAVE_UNIX_SOCKETS, 'requires Unix sockets') -HAVE_FORKING = hasattr(os, "fork") +HAVE_FORKING = test.support.has_fork_support requires_forking = unittest.skipUnless(HAVE_FORKING, 'requires forking') -def signal_alarm(n): - """Call signal.alarm when it exists (i.e. not on Windows).""" - if hasattr(signal, 'alarm'): - signal.alarm(n) - # Remember real select() to avoid interferences with mocking _real_select = select.select @@ -46,14 +42,6 @@ def receive(sock, n, timeout=test.support.SHORT_TIMEOUT): else: raise RuntimeError("timed out on %r" % (sock,)) -if HAVE_UNIX_SOCKETS and HAVE_FORKING: - class ForkingUnixStreamServer(socketserver.ForkingMixIn, - socketserver.UnixStreamServer): - pass - - class ForkingUnixDatagramServer(socketserver.ForkingMixIn, - socketserver.UnixDatagramServer): - pass @test.support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems @contextlib.contextmanager @@ -75,12 +63,10 @@ class SocketServerTest(unittest.TestCase): """Test all socket servers.""" def setUp(self): - signal_alarm(60) # Kill deadlocks after 60 seconds. self.port_seed = 0 self.test_files = [] def tearDown(self): - signal_alarm(0) # Didn't deadlock. reap_children() for fn in self.test_files: @@ -96,8 +82,7 @@ def pickaddr(self, proto): else: # XXX: We need a way to tell AF_UNIX to pick its own name # like AF_INET provides port==0. - dir = None - fn = tempfile.mktemp(prefix='unix_socket.', dir=dir) + fn = socket_helper.create_unix_domain_name() self.test_files.append(fn) return fn @@ -178,13 +163,11 @@ def dgram_examine(self, proto, addr): buf += data self.assertEqual(buf, TEST_STR) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_TCPServer(self): self.run_server(socketserver.TCPServer, socketserver.StreamRequestHandler, self.stream_examine) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_ThreadingTCPServer(self): self.run_server(socketserver.ThreadingTCPServer, socketserver.StreamRequestHandler, @@ -213,17 +196,15 @@ def test_ThreadingUnixStreamServer(self): @requires_forking def test_ForkingUnixStreamServer(self): with simple_subprocess(self): - self.run_server(ForkingUnixStreamServer, + self.run_server(socketserver.ForkingUnixStreamServer, socketserver.StreamRequestHandler, self.stream_examine) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_UDPServer(self): self.run_server(socketserver.UDPServer, socketserver.DatagramRequestHandler, self.dgram_examine) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_ThreadingUDPServer(self): self.run_server(socketserver.ThreadingUDPServer, socketserver.DatagramRequestHandler, @@ -251,7 +232,7 @@ def test_ThreadingUnixDatagramServer(self): @requires_unix_sockets @requires_forking def test_ForkingUnixDatagramServer(self): - self.run_server(ForkingUnixDatagramServer, + self.run_server(socketserver.ForkingUnixDatagramServer, socketserver.DatagramRequestHandler, self.dgram_examine) @@ -298,7 +279,6 @@ def test_tcpserver_bind_leak(self): socketserver.TCPServer((HOST, -1), socketserver.StreamRequestHandler) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_context_manager(self): with socketserver.TCPServer((HOST, 0), socketserver.StreamRequestHandler) as server: diff --git a/Lib/test/test_sort.py b/Lib/test/test_sort.py index be3d4a8461f..a01f5fc9cee 100644 --- a/Lib/test/test_sort.py +++ b/Lib/test/test_sort.py @@ -128,6 +128,27 @@ def bad_key(x): x = [e for e, i in augmented] # a stable sort of s check("stability", x, s) + def test_small_stability(self): + from itertools import product + from operator import itemgetter + + # Exhaustively test stability across all lists of small lengths + # and only a few distinct elements. + # This can provoke edge cases that randomization is unlikely to find. + # But it can grow very expensive quickly, so don't overdo it. + NELTS = 3 + MAXSIZE = 9 + + pick0 = itemgetter(0) + for length in range(MAXSIZE + 1): + # There are NELTS ** length distinct lists. + for t in product(range(NELTS), repeat=length): + xs = list(zip(t, range(length))) + # Stability forced by index in each element. + forced = sorted(xs) + # Use key= to hide the index from compares. + native = sorted(xs, key=pick0) + self.assertEqual(forced, native) #============================================================================== class TestBugs(unittest.TestCase): @@ -149,7 +170,7 @@ def __lt__(self, other): L = [C() for i in range(50)] self.assertRaises(ValueError, L.sort) - @unittest.skip("TODO: RUSTPYTHON; figure out how to detect sort mutation that doesn't change list length") + @unittest.expectedFailure # TODO: RUSTPYTHON; figure out how to detect sort mutation that doesn't change list length def test_undetected_mutation(self): # Python 2.4a1 did not always detect mutation memorywaster = [] @@ -307,8 +328,7 @@ def test_safe_object_compare(self): for L in float_int_lists: check_against_PyObject_RichCompareBool(self, L) - # XXX RUSTPYTHON: added by us but it seems like an implementation detail - @support.cpython_only + @support.cpython_only # XXX RUSTPYTHON: added by us but it seems like an implementation detail def test_unsafe_object_compare(self): # This test is by ppperry. It ensures that unsafe_object_compare is diff --git a/Lib/test/test_sqlite3/__init__.py b/Lib/test/test_sqlite3/__init__.py index d777fca82da..78a1e2078a5 100644 --- a/Lib/test/test_sqlite3/__init__.py +++ b/Lib/test/test_sqlite3/__init__.py @@ -8,8 +8,7 @@ # Implement the unittest "load tests" protocol. def load_tests(*args): + if verbose: + print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") pkg_dir = os.path.dirname(__file__) return load_package_tests(pkg_dir, *args) - -if verbose: - print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") diff --git a/Lib/test/test_sqlite3/test_backup.py b/Lib/test/test_sqlite3/test_backup.py index fb3a83e3b0e..9d31978b1ad 100644 --- a/Lib/test/test_sqlite3/test_backup.py +++ b/Lib/test/test_sqlite3/test_backup.py @@ -1,6 +1,8 @@ import sqlite3 as sqlite import unittest +from .util import memory_database + class BackupTests(unittest.TestCase): def setUp(self): @@ -32,34 +34,32 @@ def test_bad_target_same_connection(self): self.cx.backup(self.cx) def test_bad_target_closed_connection(self): - bck = sqlite.connect(':memory:') - bck.close() - with self.assertRaises(sqlite.ProgrammingError): - self.cx.backup(bck) + with memory_database() as bck: + bck.close() + with self.assertRaises(sqlite.ProgrammingError): + self.cx.backup(bck) def test_bad_source_closed_connection(self): - bck = sqlite.connect(':memory:') - source = sqlite.connect(":memory:") - source.close() - with self.assertRaises(sqlite.ProgrammingError): - source.backup(bck) + with memory_database() as bck: + source = sqlite.connect(":memory:") + source.close() + with self.assertRaises(sqlite.ProgrammingError): + source.backup(bck) def test_bad_target_in_transaction(self): - bck = sqlite.connect(':memory:') - bck.execute('CREATE TABLE bar (key INTEGER)') - bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) - with self.assertRaises(sqlite.OperationalError) as cm: - self.cx.backup(bck) - if sqlite.sqlite_version_info < (3, 8, 8): - self.assertEqual(str(cm.exception), 'target is in transaction') + with memory_database() as bck: + bck.execute('CREATE TABLE bar (key INTEGER)') + bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) + with self.assertRaises(sqlite.OperationalError) as cm: + self.cx.backup(bck) def test_keyword_only_args(self): with self.assertRaises(TypeError): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, 1) def test_simple(self): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck) self.verify_backup(bck) @@ -69,7 +69,7 @@ def test_progress(self): def progress(status, remaining, total): journal.append(status) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress=progress) self.verify_backup(bck) @@ -83,7 +83,7 @@ def test_progress_all_pages_at_once_1(self): def progress(status, remaining, total): journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, progress=progress) self.verify_backup(bck) @@ -96,18 +96,17 @@ def test_progress_all_pages_at_once_2(self): def progress(status, remaining, total): journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=-1, progress=progress) self.verify_backup(bck) self.assertEqual(len(journal), 1) self.assertEqual(journal[0], 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_callable_progress(self): with self.assertRaises(TypeError) as cm: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress='bar') self.assertEqual(str(cm.exception), 'progress argument must be a callable') @@ -120,7 +119,7 @@ def progress(status, remaining, total): self.cx.commit() journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress=progress) self.verify_backup(bck) @@ -139,17 +138,17 @@ def progress(status, remaining, total): raise SystemError('nearly out of space') with self.assertRaises(SystemError) as err: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, progress=progress) self.assertEqual(str(err.exception), 'nearly out of space') def test_database_source_name(self): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='main') - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='temp') with self.assertRaises(sqlite.OperationalError) as cm: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='non-existing') self.assertIn("unknown database", str(cm.exception)) @@ -157,7 +156,7 @@ def test_database_source_name(self): self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)') self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)]) self.cx.commit() - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='attached_db') self.verify_backup(bck) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 560cd9efc5c..a03d7cbe16b 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -1,52 +1,52 @@ """sqlite3 CLI tests.""" - -import sqlite3 as sqlite -import subprocess -import sys +import sqlite3 import unittest -from test.support import SHORT_TIMEOUT#, requires_subprocess +from sqlite3.__main__ import main as cli from test.support.os_helper import TESTFN, unlink +from test.support import ( + captured_stdout, + captured_stderr, + captured_stdin, + force_not_colorized, +) -# TODO: RUSTPYTHON -#@requires_subprocess() class CommandLineInterface(unittest.TestCase): def _do_test(self, *args, expect_success=True): - with subprocess.Popen( - [sys.executable, "-Xutf8", "-m", "sqlite3", *args], - encoding="utf-8", - bufsize=0, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as proc: - proc.wait() - if expect_success == bool(proc.returncode): - self.fail("".join(proc.stderr)) - stdout = proc.stdout.read() - stderr = proc.stderr.read() - if expect_success: - self.assertEqual(stderr, "") - else: - self.assertEqual(stdout, "") - return stdout, stderr + with ( + captured_stdout() as out, + captured_stderr() as err, + self.assertRaises(SystemExit) as cm + ): + cli(args) + return out.getvalue(), err.getvalue(), cm.exception.code def expect_success(self, *args): - out, _ = self._do_test(*args) + out, err, code = self._do_test(*args) + self.assertEqual(code, 0, + "\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(err, "") return out def expect_failure(self, *args): - _, err = self._do_test(*args, expect_success=False) + out, err, code = self._do_test(*args, expect_success=False) + self.assertNotEqual(code, 0, + "\n".join([f"Unexpected failure: {args=}", out, err])) + self.assertEqual(out, "") return err + @force_not_colorized def test_cli_help(self): out = self.expect_success("-h") - self.assertIn("usage: python -m sqlite3", out) + self.assertIn("usage: ", out) + self.assertIn(" [-h] [-v] [filename] [sql]", out) + self.assertIn("Python sqlite3 CLI", out) def test_cli_version(self): out = self.expect_success("-v") - self.assertIn(sqlite.sqlite_version, out) + self.assertIn(sqlite3.sqlite_version, out) def test_cli_execute_sql(self): out = self.expect_success(":memory:", "select 1") @@ -69,88 +69,126 @@ def test_cli_on_disk_db(self): self.assertIn("(0,)", out) -# TODO: RUSTPYTHON -#@requires_subprocess() class InteractiveSession(unittest.TestCase): - TIMEOUT = SHORT_TIMEOUT / 10. MEMORY_DB_MSG = "Connected to a transient in-memory database" PS1 = "sqlite> " PS2 = "... " - def start_cli(self, *args): - return subprocess.Popen( - [sys.executable, "-Xutf8", "-m", "sqlite3", *args], - encoding="utf-8", - bufsize=0, - stdin=subprocess.PIPE, - # Note: the banner is printed to stderr, the prompt to stdout. - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - def expect_success(self, proc): - proc.wait() - if proc.returncode: - self.fail("".join(proc.stderr)) + def run_cli(self, *args, commands=()): + with ( + captured_stdin() as stdin, + captured_stdout() as stdout, + captured_stderr() as stderr, + self.assertRaises(SystemExit) as cm + ): + for cmd in commands: + stdin.write(cmd + "\n") + stdin.seek(0) + cli(args) + + out = stdout.getvalue() + err = stderr.getvalue() + self.assertEqual(cm.exception.code, 0, + f"Unexpected failure: {args=}\n{out}\n{err}") + return out, err def test_interact(self): - with self.start_cli() as proc: - out, err = proc.communicate(timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) + out, err = self.run_cli() + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 0) def test_interact_quit(self): - with self.start_cli() as proc: - out, err = proc.communicate(input=".quit", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) + out, err = self.run_cli(commands=(".quit",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 0) def test_interact_version(self): - with self.start_cli() as proc: - out, err = proc.communicate(input=".version", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(sqlite.sqlite_version, out) - self.expect_success(proc) + out, err = self.run_cli(commands=(".version",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(sqlite3.sqlite_version + "\n", out) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + self.assertIn(sqlite3.sqlite_version, out) + + def test_interact_empty_source(self): + out, err = self.run_cli(commands=("", " ")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 3) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_dot_commands_unknown(self): + out, err = self.run_cli(commands=(".unknown_command", )) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + self.assertIn("Error", err) + # test "unknown_command" is pointed out in the error message + self.assertIn("unknown_command", err) + + def test_interact_dot_commands_empty(self): + out, err = self.run_cli(commands=(".")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_dot_commands_with_whitespaces(self): + out, err = self.run_cli(commands=(".version ", ". version")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEqual(out.count(sqlite3.sqlite_version + "\n"), 2) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 3) + self.assertEqual(out.count(self.PS2), 0) def test_interact_valid_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="select 1;", - timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn("(1,)", out) - self.expect_success(proc) + out, err = self.run_cli(commands=("SELECT 1;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn("(1,)\n", out) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_incomplete_multiline_sql(self): + out, err = self.run_cli(commands=("SELECT 1",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS2) + self.assertEqual(out.count(self.PS1), 1) + self.assertEqual(out.count(self.PS2), 1) def test_interact_valid_multiline_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="select 1\n;", - timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn(self.PS2, out) - self.assertIn("(1,)", out) - self.expect_success(proc) + out, err = self.run_cli(commands=("SELECT 1\n;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn(self.PS2, out) + self.assertIn("(1,)\n", out) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 1) def test_interact_invalid_sql(self): - with self.start_cli() as proc: - out, err = proc.communicate(input="sel;", timeout=self.TIMEOUT) - self.assertIn(self.MEMORY_DB_MSG, err) - self.assertIn("OperationalError (SQLITE_ERROR)", err) - self.expect_success(proc) + out, err = self.run_cli(commands=("sel;",)) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertIn("OperationalError (SQLITE_ERROR)", err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) def test_interact_on_disk_file(self): self.addCleanup(unlink, TESTFN) - with self.start_cli(TESTFN) as proc: - out, err = proc.communicate(input="create table t(t);", - timeout=self.TIMEOUT) - self.assertIn(TESTFN, err) - self.assertIn(self.PS1, out) - self.expect_success(proc) - with self.start_cli(TESTFN, "select count(t) from t") as proc: - out = proc.stdout.read() - err = proc.stderr.read() - self.assertIn("(0,)", out) - self.expect_success(proc) + + out, err = self.run_cli(TESTFN, commands=("CREATE TABLE t(t);",)) + self.assertIn(TESTFN, err) + self.assertEndsWith(out, self.PS1) + + out, _ = self.run_cli(TESTFN, commands=("SELECT count(t) FROM t;",)) + self.assertIn("(0,)\n", out) if __name__ == "__main__": diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index bbc151fcb78..8962fe00ed8 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -21,6 +21,7 @@ # 3. This notice may not be removed or altered from any source distribution. import contextlib +import functools import os import sqlite3 as sqlite import subprocess @@ -28,33 +29,18 @@ import threading import unittest import urllib.parse +import warnings from test.support import ( - SHORT_TIMEOUT, check_disallow_instantiation,# requires_subprocess, - #is_emscripten, is_wasi -# TODO: RUSTPYTHON + SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess ) -from test.support import threading_helper -# TODO: RUSTPYTHON -#from _testcapi import INT_MAX, ULLONG_MAX +from test.support import gc_collect +from test.support import threading_helper, import_helper from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath - -# Helper for temporary memory databases -def memory_database(*args, **kwargs): - cx = sqlite.connect(":memory:", *args, **kwargs) - return contextlib.closing(cx) - - -# Temporarily limit a database connection parameter -@contextlib.contextmanager -def cx_limit(cx, category=sqlite.SQLITE_LIMIT_SQL_LENGTH, limit=128): - try: - _prev = cx.setlimit(category, limit) - yield limit - finally: - cx.setlimit(category, _prev) +from .util import memory_database, cx_limit +from .util import MemoryDatabaseMixin class ModuleTests(unittest.TestCase): @@ -62,17 +48,6 @@ def test_api_level(self): self.assertEqual(sqlite.apilevel, "2.0", "apilevel is %s, should be 2.0" % sqlite.apilevel) - def test_deprecated_version(self): - msg = "deprecated and will be removed in Python 3.14" - for attr in "version", "version_info": - with self.subTest(attr=attr): - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite, attr) - self.assertEqual(cm.filename, __file__) - with self.assertWarnsRegex(DeprecationWarning, msg) as cm: - getattr(sqlite.dbapi2, attr) - self.assertEqual(cm.filename, __file__) - def test_thread_safety(self): self.assertIn(sqlite.threadsafety, {0, 1, 3}, "threadsafety is %d, should be 0, 1 or 3" % @@ -84,45 +59,34 @@ def test_param_style(self): sqlite.paramstyle) def test_warning(self): - self.assertTrue(issubclass(sqlite.Warning, Exception), - "Warning is not a subclass of Exception") + self.assertIsSubclass(sqlite.Warning, Exception) def test_error(self): - self.assertTrue(issubclass(sqlite.Error, Exception), - "Error is not a subclass of Exception") + self.assertIsSubclass(sqlite.Error, Exception) def test_interface_error(self): - self.assertTrue(issubclass(sqlite.InterfaceError, sqlite.Error), - "InterfaceError is not a subclass of Error") + self.assertIsSubclass(sqlite.InterfaceError, sqlite.Error) def test_database_error(self): - self.assertTrue(issubclass(sqlite.DatabaseError, sqlite.Error), - "DatabaseError is not a subclass of Error") + self.assertIsSubclass(sqlite.DatabaseError, sqlite.Error) def test_data_error(self): - self.assertTrue(issubclass(sqlite.DataError, sqlite.DatabaseError), - "DataError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.DataError, sqlite.DatabaseError) def test_operational_error(self): - self.assertTrue(issubclass(sqlite.OperationalError, sqlite.DatabaseError), - "OperationalError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.OperationalError, sqlite.DatabaseError) def test_integrity_error(self): - self.assertTrue(issubclass(sqlite.IntegrityError, sqlite.DatabaseError), - "IntegrityError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.IntegrityError, sqlite.DatabaseError) def test_internal_error(self): - self.assertTrue(issubclass(sqlite.InternalError, sqlite.DatabaseError), - "InternalError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.InternalError, sqlite.DatabaseError) def test_programming_error(self): - self.assertTrue(issubclass(sqlite.ProgrammingError, sqlite.DatabaseError), - "ProgrammingError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.ProgrammingError, sqlite.DatabaseError) def test_not_supported_error(self): - self.assertTrue(issubclass(sqlite.NotSupportedError, - sqlite.DatabaseError), - "NotSupportedError is not a subclass of DatabaseError") + self.assertIsSubclass(sqlite.NotSupportedError, sqlite.DatabaseError) def test_module_constants(self): consts = [ @@ -167,6 +131,7 @@ def test_module_constants(self): "SQLITE_INTERNAL", "SQLITE_INTERRUPT", "SQLITE_IOERR", + "SQLITE_LIMIT_WORKER_THREADS", "SQLITE_LOCKED", "SQLITE_MISMATCH", "SQLITE_MISUSE", @@ -174,6 +139,7 @@ def test_module_constants(self): "SQLITE_NOMEM", "SQLITE_NOTADB", "SQLITE_NOTFOUND", + "SQLITE_NOTICE", "SQLITE_OK", "SQLITE_PERM", "SQLITE_PRAGMA", @@ -181,6 +147,7 @@ def test_module_constants(self): "SQLITE_RANGE", "SQLITE_READ", "SQLITE_READONLY", + "SQLITE_RECURSIVE", "SQLITE_REINDEX", "SQLITE_ROW", "SQLITE_SAVEPOINT", @@ -189,6 +156,7 @@ def test_module_constants(self): "SQLITE_TOOBIG", "SQLITE_TRANSACTION", "SQLITE_UPDATE", + "SQLITE_WARNING", # Run-time limit categories "SQLITE_LIMIT_LENGTH", "SQLITE_LIMIT_SQL_LENGTH", @@ -202,32 +170,43 @@ def test_module_constants(self): "SQLITE_LIMIT_VARIABLE_NUMBER", "SQLITE_LIMIT_TRIGGER_DEPTH", ] - if sqlite.sqlite_version_info >= (3, 7, 17): - consts += ["SQLITE_NOTICE", "SQLITE_WARNING"] - if sqlite.sqlite_version_info >= (3, 8, 3): - consts.append("SQLITE_RECURSIVE") - if sqlite.sqlite_version_info >= (3, 8, 7): - consts.append("SQLITE_LIMIT_WORKER_THREADS") consts += ["PARSE_DECLTYPES", "PARSE_COLNAMES"] # Extended result codes consts += [ "SQLITE_ABORT_ROLLBACK", + "SQLITE_AUTH_USER", "SQLITE_BUSY_RECOVERY", + "SQLITE_BUSY_SNAPSHOT", + "SQLITE_CANTOPEN_CONVPATH", "SQLITE_CANTOPEN_FULLPATH", "SQLITE_CANTOPEN_ISDIR", "SQLITE_CANTOPEN_NOTEMPDIR", + "SQLITE_CONSTRAINT_CHECK", + "SQLITE_CONSTRAINT_COMMITHOOK", + "SQLITE_CONSTRAINT_FOREIGNKEY", + "SQLITE_CONSTRAINT_FUNCTION", + "SQLITE_CONSTRAINT_NOTNULL", + "SQLITE_CONSTRAINT_PRIMARYKEY", + "SQLITE_CONSTRAINT_ROWID", + "SQLITE_CONSTRAINT_TRIGGER", + "SQLITE_CONSTRAINT_UNIQUE", + "SQLITE_CONSTRAINT_VTAB", "SQLITE_CORRUPT_VTAB", "SQLITE_IOERR_ACCESS", + "SQLITE_IOERR_AUTH", "SQLITE_IOERR_BLOCKED", "SQLITE_IOERR_CHECKRESERVEDLOCK", "SQLITE_IOERR_CLOSE", + "SQLITE_IOERR_CONVPATH", "SQLITE_IOERR_DELETE", "SQLITE_IOERR_DELETE_NOENT", "SQLITE_IOERR_DIR_CLOSE", "SQLITE_IOERR_DIR_FSYNC", "SQLITE_IOERR_FSTAT", "SQLITE_IOERR_FSYNC", + "SQLITE_IOERR_GETTEMPPATH", "SQLITE_IOERR_LOCK", + "SQLITE_IOERR_MMAP", "SQLITE_IOERR_NOMEM", "SQLITE_IOERR_RDLOCK", "SQLITE_IOERR_READ", @@ -239,50 +218,18 @@ def test_module_constants(self): "SQLITE_IOERR_SHORT_READ", "SQLITE_IOERR_TRUNCATE", "SQLITE_IOERR_UNLOCK", + "SQLITE_IOERR_VNODE", "SQLITE_IOERR_WRITE", "SQLITE_LOCKED_SHAREDCACHE", + "SQLITE_NOTICE_RECOVER_ROLLBACK", + "SQLITE_NOTICE_RECOVER_WAL", + "SQLITE_OK_LOAD_PERMANENTLY", "SQLITE_READONLY_CANTLOCK", + "SQLITE_READONLY_DBMOVED", "SQLITE_READONLY_RECOVERY", + "SQLITE_READONLY_ROLLBACK", + "SQLITE_WARNING_AUTOINDEX", ] - if sqlite.sqlite_version_info >= (3, 7, 16): - consts += [ - "SQLITE_CONSTRAINT_CHECK", - "SQLITE_CONSTRAINT_COMMITHOOK", - "SQLITE_CONSTRAINT_FOREIGNKEY", - "SQLITE_CONSTRAINT_FUNCTION", - "SQLITE_CONSTRAINT_NOTNULL", - "SQLITE_CONSTRAINT_PRIMARYKEY", - "SQLITE_CONSTRAINT_TRIGGER", - "SQLITE_CONSTRAINT_UNIQUE", - "SQLITE_CONSTRAINT_VTAB", - "SQLITE_READONLY_ROLLBACK", - ] - if sqlite.sqlite_version_info >= (3, 7, 17): - consts += [ - "SQLITE_IOERR_MMAP", - "SQLITE_NOTICE_RECOVER_ROLLBACK", - "SQLITE_NOTICE_RECOVER_WAL", - ] - if sqlite.sqlite_version_info >= (3, 8, 0): - consts += [ - "SQLITE_BUSY_SNAPSHOT", - "SQLITE_IOERR_GETTEMPPATH", - "SQLITE_WARNING_AUTOINDEX", - ] - if sqlite.sqlite_version_info >= (3, 8, 1): - consts += ["SQLITE_CANTOPEN_CONVPATH", "SQLITE_IOERR_CONVPATH"] - if sqlite.sqlite_version_info >= (3, 8, 2): - consts.append("SQLITE_CONSTRAINT_ROWID") - if sqlite.sqlite_version_info >= (3, 8, 3): - consts.append("SQLITE_READONLY_DBMOVED") - if sqlite.sqlite_version_info >= (3, 8, 7): - consts.append("SQLITE_AUTH_USER") - if sqlite.sqlite_version_info >= (3, 9, 0): - consts.append("SQLITE_IOERR_VNODE") - if sqlite.sqlite_version_info >= (3, 10, 0): - consts.append("SQLITE_IOERR_AUTH") - if sqlite.sqlite_version_info >= (3, 14, 1): - consts.append("SQLITE_OK_LOAD_PERMANENTLY") if sqlite.sqlite_version_info >= (3, 21, 0): consts += [ "SQLITE_IOERR_BEGIN_ATOMIC", @@ -316,7 +263,7 @@ def test_module_constants(self): consts.append("SQLITE_IOERR_CORRUPTFS") for const in consts: with self.subTest(const=const): - self.assertTrue(hasattr(sqlite, const)) + self.assertHasAttr(sqlite, const) def test_error_code_on_exception(self): err_msg = "unable to open database file" @@ -330,10 +277,8 @@ def test_error_code_on_exception(self): sqlite.connect(db) e = cm.exception self.assertEqual(e.sqlite_errorcode, err_code) - self.assertTrue(e.sqlite_errorname.startswith("SQLITE_CANTOPEN")) + self.assertStartsWith(e.sqlite_errorname, "SQLITE_CANTOPEN") - @unittest.skipIf(sqlite.sqlite_version_info <= (3, 7, 16), - "Requires SQLite 3.7.16 or newer") def test_extended_error_code_on_exception(self): with memory_database() as con: with con: @@ -346,12 +291,10 @@ def test_extended_error_code_on_exception(self): sqlite.SQLITE_CONSTRAINT_CHECK) self.assertEqual(exc.sqlite_errorname, "SQLITE_CONSTRAINT_CHECK") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_disallow_instantiation(self): - cx = sqlite.connect(":memory:") - check_disallow_instantiation(self, type(cx("select 1"))) - check_disallow_instantiation(self, sqlite.Blob) + with memory_database() as cx: + check_disallow_instantiation(self, type(cx("select 1"))) + check_disallow_instantiation(self, sqlite.Blob) def test_complete_statement(self): self.assertFalse(sqlite.complete_statement("select t")) @@ -365,6 +308,7 @@ def setUp(self): cu = self.cx.cursor() cu.execute("create table test(id integer primary key, name text)") cu.execute("insert into test(name) values (?)", ("foo",)) + cu.close() def tearDown(self): self.cx.close() @@ -420,8 +364,7 @@ def test_use_after_close(self): with self.cx: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_exceptions(self): # Optional DB-API extension. self.assertEqual(self.cx.Warning, sqlite.Warning) @@ -437,28 +380,28 @@ def test_exceptions(self): def test_in_transaction(self): # Can't use db from setUp because we want to test initial state. - cx = sqlite.connect(":memory:") - cu = cx.cursor() - self.assertEqual(cx.in_transaction, False) - cu.execute("create table transactiontest(id integer primary key, name text)") - self.assertEqual(cx.in_transaction, False) - cu.execute("insert into transactiontest(name) values (?)", ("foo",)) - self.assertEqual(cx.in_transaction, True) - cu.execute("select name from transactiontest where name=?", ["foo"]) - row = cu.fetchone() - self.assertEqual(cx.in_transaction, True) - cx.commit() - self.assertEqual(cx.in_transaction, False) - cu.execute("select name from transactiontest where name=?", ["foo"]) - row = cu.fetchone() - self.assertEqual(cx.in_transaction, False) + with memory_database() as cx: + cu = cx.cursor() + self.assertEqual(cx.in_transaction, False) + cu.execute("create table transactiontest(id integer primary key, name text)") + self.assertEqual(cx.in_transaction, False) + cu.execute("insert into transactiontest(name) values (?)", ("foo",)) + self.assertEqual(cx.in_transaction, True) + cu.execute("select name from transactiontest where name=?", ["foo"]) + row = cu.fetchone() + self.assertEqual(cx.in_transaction, True) + cx.commit() + self.assertEqual(cx.in_transaction, False) + cu.execute("select name from transactiontest where name=?", ["foo"]) + row = cu.fetchone() + self.assertEqual(cx.in_transaction, False) + cu.close() def test_in_transaction_ro(self): with self.assertRaises(AttributeError): self.cx.in_transaction = True - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_exceptions(self): exceptions = [ "DataError", @@ -473,14 +416,13 @@ def test_connection_exceptions(self): ] for exc in exceptions: with self.subTest(exc=exc): - self.assertTrue(hasattr(self.cx, exc)) + self.assertHasAttr(self.cx, exc) self.assertIs(getattr(sqlite, exc), getattr(self.cx, exc)) def test_interrupt_on_closed_db(self): - cx = sqlite.connect(":memory:") - cx.close() + self.cx.close() with self.assertRaises(sqlite.ProgrammingError): - cx.interrupt() + self.cx.interrupt() def test_interrupt(self): self.assertIsNone(self.cx.interrupt()) @@ -547,36 +489,31 @@ def test_connection_init_good_isolation_levels(self): cx.isolation_level = level self.assertEqual(cx.isolation_level, level) - # TODO: RUSTPYTHON - # @unittest.expectedFailure - @unittest.skip("TODO: RUSTPYTHON deadlock") def test_connection_reinit(self): - db = ":memory:" - cx = sqlite.connect(db) - cx.text_factory = bytes - cx.row_factory = sqlite.Row - cu = cx.cursor() - cu.execute("create table foo (bar)") - cu.executemany("insert into foo (bar) values (?)", - ((str(v),) for v in range(4))) - cu.execute("select bar from foo") - - rows = [r for r in cu.fetchmany(2)] - self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) - self.assertEqual([r[0] for r in rows], [b"0", b"1"]) - - cx.__init__(db) - cx.execute("create table foo (bar)") - cx.executemany("insert into foo (bar) values (?)", - ((v,) for v in ("a", "b", "c", "d"))) - - # This uses the old database, old row factory, but new text factory - rows = [r for r in cu.fetchall()] - self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) - self.assertEqual([r[0] for r in rows], ["2", "3"]) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + with memory_database() as cx: + cx.text_factory = bytes + cx.row_factory = sqlite.Row + cu = cx.cursor() + cu.execute("create table foo (bar)") + cu.executemany("insert into foo (bar) values (?)", + ((str(v),) for v in range(4))) + cu.execute("select bar from foo") + + rows = [r for r in cu.fetchmany(2)] + self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) + self.assertEqual([r[0] for r in rows], [b"0", b"1"]) + + cx.__init__(":memory:") + cx.execute("create table foo (bar)") + cx.executemany("insert into foo (bar) values (?)", + ((v,) for v in ("a", "b", "c", "d"))) + + # This uses the old database, old row factory, but new text factory + rows = [r for r in cu.fetchall()] + self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) + self.assertEqual([r[0] for r in rows], ["2", "3"]) + cu.close() + def test_connection_bad_reinit(self): cx = sqlite.connect(":memory:") with cx: @@ -590,12 +527,64 @@ def test_connection_bad_reinit(self): cx.executemany, "insert into t values(?)", ((v,) for v in range(3))) + @unittest.expectedFailure # TODO: RUSTPYTHON SQLITE_DBCONFIG constants not implemented + def test_connection_config(self): + op = sqlite.SQLITE_DBCONFIG_ENABLE_FKEY + with memory_database() as cx: + with self.assertRaisesRegex(ValueError, "unknown"): + cx.getconfig(-1) + + # Toggle and verify. + old = cx.getconfig(op) + new = not old + cx.setconfig(op, new) + self.assertEqual(cx.getconfig(op), new) + + cx.setconfig(op) # defaults to True + self.assertTrue(cx.getconfig(op)) + + # Check that foreign key support was actually enabled. + with cx: + cx.executescript(""" + create table t(t integer primary key); + create table u(u, foreign key(u) references t(t)); + """) + with self.assertRaisesRegex(sqlite.IntegrityError, "constraint"): + cx.execute("insert into u values(0)") + + @unittest.expectedFailure # TODO: RUSTPYTHON deprecation warning not emitted for positional args + def test_connect_positional_arguments(self): + regex = ( + r"Passing more than 1 positional argument to sqlite3.connect\(\)" + " is deprecated. Parameters 'timeout', 'detect_types', " + "'isolation_level', 'check_same_thread', 'factory', " + "'cached_statements' and 'uri' will become keyword-only " + "parameters in Python 3.15." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + cx = sqlite.connect(":memory:", 1.0) + cx.close() + self.assertEqual(cm.filename, __file__) + + @unittest.expectedFailure # TODO: RUSTPYTHON ResourceWarning not emitted + def test_connection_resource_warning(self): + with self.assertWarns(ResourceWarning): + cx = sqlite.connect(":memory:") + del cx + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON Connection signature inspection not working + def test_connection_signature(self): + from inspect import signature + sig = signature(self.cx) + self.assertEqual(str(sig), "(sql, /)") + -@unittest.skip("TODO: RUSTPYHON") class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): self.cx = sqlite.Connection.__new__(sqlite.Connection) + @unittest.skip('TODO: RUSTPYTHON') def test_uninit_operations(self): funcs = ( lambda: self.cx.isolation_level, @@ -620,7 +609,6 @@ def test_serialize_deserialize(self): with cx: cx.execute("create table t(t)") data = cx.serialize() - self.assertEqual(len(data), 8192) # Remove test table, verify that it was removed. with cx: @@ -658,6 +646,14 @@ def test_deserialize_corrupt_database(self): class OpenTests(unittest.TestCase): _sql = "create table test(id integer)" + def test_open_with_bytes_path(self): + path = os.fsencode(TESTFN) + self.addCleanup(unlink, path) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(path)) as cx: + self.assertTrue(os.path.exists(path)) + cx.execute(self._sql) + def test_open_with_path_like_object(self): """ Checks that we can successfully connect to a database using an object that is PathLike, i.e. has __fspath__(). """ @@ -668,15 +664,21 @@ def test_open_with_path_like_object(self): self.assertTrue(os.path.exists(path)) cx.execute(self._sql) + def get_undecodable_path(self): + path = TESTFN_UNDECODABLE + if not path: + self.skipTest("only works if there are undecodable paths") + try: + open(path, 'wb').close() + except OSError: + self.skipTest(f"can't create file with undecodable path {path!r}") + unlink(path) + return path + @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") - # TODO: RUSTPYTHON - # @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(path)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) @@ -716,21 +718,15 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(sys.platform == "darwin", "skipped on macOS") - # TODO: RUSTPYTHON - # @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) uri = "file:" + urllib.parse.quote(path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_factory_database_arg(self): def factory(database, *args, **kwargs): nonlocal database_arg @@ -769,8 +765,6 @@ def test_execute_illegal_sql(self): with self.assertRaises(sqlite.OperationalError): self.cu.execute("select asdf") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_multiple_statements(self): msg = "You can only execute one statement at a time" dataset = ( @@ -793,8 +787,6 @@ def test_execute_multiple_statements(self): with self.assertRaisesRegex(sqlite.ProgrammingError, msg): self.cu.execute(query) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_with_appended_comments(self): dataset = ( "select 1; -- foo bar", @@ -842,15 +834,11 @@ def test_execute_wrong_no_of_args1(self): with self.assertRaises(sqlite.ProgrammingError): self.cu.execute("insert into test(id) values (?)", (17, "Egon")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_wrong_no_of_args2(self): # too little parameters with self.assertRaises(sqlite.ProgrammingError): self.cu.execute("insert into test(id) values (?)") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_wrong_no_of_args3(self): # no parameters, parameters are needed with self.assertRaises(sqlite.ProgrammingError): @@ -887,6 +875,34 @@ def __getitem__(slf, x): with self.assertRaises(ZeroDivisionError): self.cu.execute("select name from test where name=?", L()) + @unittest.expectedFailure # TODO: RUSTPYTHON mixed named and positional parameters not validated + def test_execute_named_param_and_sequence(self): + dataset = ( + ("select :a", (1,)), + ("select :a, ?, ?", (1, 2, 3)), + ("select ?, :b, ?", (1, 2, 3)), + ("select ?, ?, :c", (1, 2, 3)), + ("select :a, :b, ?", (1, 2, 3)), + ) + msg = "Binding.*is a named parameter" + for query, params in dataset: + with self.subTest(query=query, params=params): + with self.assertRaisesRegex(sqlite.ProgrammingError, msg) as cm: + self.cu.execute(query, params) + + def test_execute_indexed_nameless_params(self): + # See gh-117995: "'?1' is considered a named placeholder" + for query, params, expected in ( + ("select ?1, ?2", (1, 2), (1, 2)), + ("select ?2, ?1", (1, 2), (2, 1)), + ): + with self.subTest(query=query, params=params): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + cu = self.cu.execute(query, params) + actual, = cu.fetchall() + self.assertEqual(actual, expected) + def test_execute_too_many_params(self): category = sqlite.SQLITE_LIMIT_VARIABLE_NUMBER msg = "too many SQL variables" @@ -912,15 +928,11 @@ def __missing__(self, key): row = self.cu.fetchone() self.assertEqual(row[0], "foo") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_dict_mapping_too_little_args(self): self.cu.execute("insert into test(name) values ('foo')") with self.assertRaises(sqlite.ProgrammingError): self.cu.execute("select name from test where name=:name and id=:id", {"name": "foo"}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_dict_mapping_no_args(self): self.cu.execute("insert into test(name) values ('foo')") with self.assertRaises(sqlite.ProgrammingError): @@ -963,8 +975,6 @@ def test_rowcount_update_returning(self): self.assertEqual(self.cu.fetchone()[0], 1) self.assertEqual(self.cu.rowcount, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_rowcount_prefixed_with_comment(self): # gh-79579: rowcount is updated even if query is prefixed with comments self.cu.execute(""" @@ -1068,7 +1078,7 @@ def test_array_size(self): # now set to 2 self.cu.arraysize = 2 - # now make the query return 3 rows + # now make the query return 2 rows from a table of 3 rows self.cu.execute("delete from test") self.cu.execute("insert into test(name) values ('A')") self.cu.execute("insert into test(name) values ('B')") @@ -1078,15 +1088,50 @@ def test_array_size(self): self.assertEqual(len(res), 2) + def test_invalid_array_size(self): + UINT32_MAX = (1 << 32) - 1 + setter = functools.partial(setattr, self.cu, 'arraysize') + + self.assertRaises(TypeError, setter, 1.0) + self.assertRaises(ValueError, setter, -3) + self.assertRaises(OverflowError, setter, UINT32_MAX + 1) + def test_fetchmany(self): + # no active SQL statement + res = self.cu.fetchmany() + self.assertEqual(res, []) + res = self.cu.fetchmany(1000) + self.assertEqual(res, []) + + # test default parameter + self.cu.execute("select name from test") + res = self.cu.fetchmany() + self.assertEqual(len(res), 1) + + # test when the number of requested rows exceeds the actual count self.cu.execute("select name from test") res = self.cu.fetchmany(100) self.assertEqual(len(res), 1) res = self.cu.fetchmany(100) self.assertEqual(res, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + # test when size = 0 + self.cu.execute("select name from test") + res = self.cu.fetchmany(0) + self.assertEqual(res, []) + res = self.cu.fetchmany(100) + self.assertEqual(len(res), 1) + res = self.cu.fetchmany(100) + self.assertEqual(res, []) + + def test_invalid_fetchmany(self): + UINT32_MAX = (1 << 32) - 1 + fetchmany = self.cu.fetchmany + + self.assertRaises(TypeError, fetchmany, 1.0) + self.assertRaises(ValueError, fetchmany, -3) + self.assertRaises(OverflowError, fetchmany, UINT32_MAX + 1) + def test_fetchmany_kw_arg(self): """Checks if fetchmany works with keyword arguments""" self.cu.execute("select name from test") @@ -1206,12 +1251,9 @@ def test_blob_seek_and_tell(self): self.blob.seek(-10, SEEK_END) self.assertEqual(self.blob.tell(), 40) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_seek_error(self): msg_oor = "offset out of blob range" msg_orig = "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END" - msg_of = "seek offset results in overflow" dataset = ( (ValueError, msg_oor, lambda: self.blob.seek(1000)), @@ -1223,12 +1265,15 @@ def test_blob_seek_error(self): with self.subTest(exc=exc, msg=msg, fn=fn): self.assertRaisesRegex(exc, msg, fn) + def test_blob_seek_overflow_error(self): # Force overflow errors + msg_of = "seek offset results in overflow" + _testcapi = import_helper.import_module("_testcapi") self.blob.seek(1, SEEK_SET) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_CUR) + self.blob.seek(_testcapi.INT_MAX, SEEK_CUR) with self.assertRaisesRegex(OverflowError, msg_of): - self.blob.seek(INT_MAX, SEEK_END) + self.blob.seek(_testcapi.INT_MAX, SEEK_END) def test_blob_read(self): buf = self.blob.read() @@ -1330,8 +1375,6 @@ def test_blob_set_item_with_offset(self): expected = b"This blob data string is exactly fifty bytes long." self.assertEqual(self.blob.read(), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_slice_buffer_object(self): from array import array self.blob[0:5] = memoryview(b"12345") @@ -1359,22 +1402,16 @@ def test_blob_get_slice_negative_index(self): def test_blob_get_slice_with_skip(self): self.assertEqual(self.blob[0:10:2], b"ti lb") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_slice(self): self.blob[0:5] = b"12345" expected = b"12345" + self.data[5:] actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_empty_slice(self): self.blob[0:0] = b"" self.assertEqual(self.blob[:], self.data) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"12345" actual = self.cx.execute("select b from test").fetchone()[0] @@ -1390,24 +1427,24 @@ def test_blob_mapping_invalid_index_type(self): with self.assertRaisesRegex(TypeError, msg): self.blob["a"] = b"b" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_get_item_error(self): dataset = [len(self.blob), 105, -105] for idx in dataset: with self.subTest(idx=idx): with self.assertRaisesRegex(IndexError, "index out of range"): self.blob[idx] - with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): - self.blob[ULLONG_MAX] # Provoke read error self.cx.execute("update test set b='aaaa' where rowid=1") with self.assertRaises(sqlite.OperationalError): self.blob[0] - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_blob_get_item_error_bigint(self): + _testcapi = import_helper.import_module("_testcapi") + with self.assertRaisesRegex(IndexError, "cannot fit 'int'"): + self.blob[_testcapi.ULLONG_MAX] + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_blob_set_item_error(self): with self.assertRaisesRegex(TypeError, "cannot be interpreted"): self.blob[0] = b"multiple" @@ -1427,8 +1464,6 @@ def test_blob_set_item_error(self): with self.assertRaisesRegex(ValueError, "must be in range"): self.blob[0] = 2**65 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_slice_error(self): with self.assertRaisesRegex(IndexError, "wrong size"): self.blob[5:10] = b"a" @@ -1441,14 +1476,12 @@ def test_blob_set_slice_error(self): with self.assertRaises(BufferError): self.blob[5:10] = memoryview(b"abcde")[::2] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_sequence_not_supported(self): with self.assertRaisesRegex(TypeError, "unsupported operand"): self.blob + self.blob with self.assertRaisesRegex(TypeError, "unsupported operand"): self.blob * 5 - with self.assertRaisesRegex(TypeError, "is not iterable"): + with self.assertRaisesRegex(TypeError, "is not.+iterable"): b"a" in self.blob def test_blob_context_manager(self): @@ -1470,8 +1503,6 @@ class DummyException(Exception): raise DummyException("reraised") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_closed(self): with memory_database() as cx: cx.execute("create table test(b blob)") @@ -1501,8 +1532,6 @@ def test_blob_closed(self): with self.assertRaisesRegex(sqlite.ProgrammingError, msg): blob[0] = b"" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_closed_db_read(self): with memory_database() as cx: cx.execute("create table test(b blob)") @@ -1513,9 +1542,16 @@ def test_blob_closed_db_read(self): "Cannot operate on a closed database", blob.read) + def test_blob_32bit_rowid(self): + # gh-100370: we should not get an OverflowError for 32-bit rowids + with memory_database() as cx: + rowid = 2**32 + cx.execute("create table t(t blob)") + cx.execute("insert into t(rowid, t) values (?, zeroblob(1))", (rowid,)) + cx.blobopen('t', 't', rowid) -# TODO: RUSTPYTHON -# @threading_helper.requires_working_threading() + +@threading_helper.requires_working_threading() class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -1568,8 +1604,7 @@ def test_check_connection_thread(self): with self.subTest(fn=fn): self._run_test(fn) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_check_cursor_thread(self): fns = [ lambda: self.cur.execute("insert into test(name) values('a')"), @@ -1590,12 +1625,12 @@ def run(con, err): except sqlite.Error: err.append("multi-threading not allowed") - con = sqlite.connect(":memory:", check_same_thread=False) - err = [] - t = threading.Thread(target=run, kwargs={"con": con, "err": err}) - t.start() - t.join() - self.assertEqual(len(err), 0, "\n".join(err)) + with memory_database(check_same_thread=False) as con: + err = [] + t = threading.Thread(target=run, kwargs={"con": con, "err": err}) + t.start() + t.join() + self.assertEqual(len(err), 0, "\n".join(err)) class ConstructorTests(unittest.TestCase): @@ -1621,9 +1656,16 @@ def test_binary(self): b = sqlite.Binary(b"\0'") class ExtensionTests(unittest.TestCase): + def setUp(self): + self.con = sqlite.connect(":memory:") + self.cur = self.con.cursor() + + def tearDown(self): + self.cur.close() + self.con.close() + def test_script_string_sql(self): - con = sqlite.connect(":memory:") - cur = con.cursor() + cur = self.cur cur.executescript(""" -- bla bla /* a stupid comment */ @@ -1635,42 +1677,40 @@ def test_script_string_sql(self): self.assertEqual(res, 5) def test_script_syntax_error(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(sqlite.OperationalError): - cur.executescript("create table test(x); asdf; create table test2(x)") + self.cur.executescript(""" + CREATE TABLE test(x); + asdf; + CREATE TABLE test2(x) + """) def test_script_error_normal(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(sqlite.OperationalError): - cur.executescript("create table test(sadfsadfdsa); select foo from hurz;") + self.cur.executescript(""" + CREATE TABLE test(sadfsadfdsa); + SELECT foo FROM hurz; + """) def test_cursor_executescript_as_bytes(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(TypeError): - cur.executescript(b"create table test(foo); insert into test(foo) values (5);") + self.cur.executescript(b""" + CREATE TABLE test(foo); + INSERT INTO test(foo) VALUES (5); + """) def test_cursor_executescript_with_null_characters(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(ValueError): - cur.executescript(""" - create table a(i);\0 - insert into a(i) values (5); - """) + self.cur.executescript(""" + CREATE TABLE a(i);\0 + INSERT INTO a(i) VALUES (5); + """) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cursor_executescript_with_surrogates(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(UnicodeEncodeError): - cur.executescript(""" - create table a(s); - insert into a(s) values ('\ud8ff'); - """) + self.cur.executescript(""" + CREATE TABLE a(s); + INSERT INTO a(s) VALUES ('\ud8ff'); + """) def test_cursor_executescript_too_large_script(self): msg = "query string is too large" @@ -1680,19 +1720,18 @@ def test_cursor_executescript_too_large_script(self): cx.executescript("select 'too large'".ljust(lim+1)) def test_cursor_executescript_tx_control(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("begin") self.assertTrue(con.in_transaction) con.executescript("select 1") self.assertFalse(con.in_transaction) def test_connection_execute(self): - con = sqlite.connect(":memory:") - result = con.execute("select 5").fetchone()[0] + result = self.con.execute("select 5").fetchone()[0] self.assertEqual(result, 5, "Basic test of Connection.execute") def test_connection_executemany(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("create table test(foo)") con.executemany("insert into test(foo) values (?)", [(3,), (4,)]) result = con.execute("select foo from test order by foo").fetchall() @@ -1700,47 +1739,50 @@ def test_connection_executemany(self): self.assertEqual(result[1][0], 4, "Basic test of Connection.executemany") def test_connection_executescript(self): - con = sqlite.connect(":memory:") - con.executescript("create table test(foo); insert into test(foo) values (5);") + con = self.con + con.executescript(""" + CREATE TABLE test(foo); + INSERT INTO test(foo) VALUES (5); + """) result = con.execute("select foo from test").fetchone()[0] self.assertEqual(result, 5, "Basic test of Connection.executescript") + class ClosedConTests(unittest.TestCase): + def check(self, fn, *args, **kwds): + regex = "Cannot operate on a closed database." + with self.assertRaisesRegex(sqlite.ProgrammingError, regex): + fn(*args, **kwds) + + def setUp(self): + self.con = sqlite.connect(":memory:") + self.cur = self.con.cursor() + self.con.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_con_cursor(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - cur = con.cursor() + self.check(self.con.cursor) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_con_commit(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con.commit() + self.check(self.con.commit) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_con_rollback(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con.rollback() + self.check(self.con.rollback) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_cur_execute(self): - con = sqlite.connect(":memory:") - cur = con.cursor() - con.close() - with self.assertRaises(sqlite.ProgrammingError): - cur.execute("select 4") + self.check(self.cur.execute, "select 4") + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_create_function(self): - con = sqlite.connect(":memory:") - con.close() - def f(x): return 17 - with self.assertRaises(sqlite.ProgrammingError): - con.create_function("foo", 1, f) + def f(x): + return 17 + self.check(self.con.create_function, "foo", 1, f) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_create_aggregate(self): - con = sqlite.connect(":memory:") - con.close() class Agg: def __init__(self): pass @@ -1748,38 +1790,28 @@ def step(self, x): pass def finalize(self): return 17 - with self.assertRaises(sqlite.ProgrammingError): - con.create_aggregate("foo", 1, Agg) + self.check(self.con.create_aggregate, "foo", 1, Agg) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_set_authorizer(self): - con = sqlite.connect(":memory:") - con.close() def authorizer(*args): return sqlite.DENY - with self.assertRaises(sqlite.ProgrammingError): - con.set_authorizer(authorizer) + self.check(self.con.set_authorizer, authorizer) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_set_progress_callback(self): - con = sqlite.connect(":memory:") - con.close() - def progress(): pass - with self.assertRaises(sqlite.ProgrammingError): - con.set_progress_handler(progress, 100) + def progress(): + pass + self.check(self.con.set_progress_handler, progress, 100) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for closed connection def test_closed_call(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con() + self.check(self.con) + -class ClosedCurTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure +class ClosedCurTests(MemoryDatabaseMixin, unittest.TestCase): def test_closed(self): - con = sqlite.connect(":memory:") - cur = con.cursor() + cur = self.cx.cursor() cur.close() for method_name in ("execute", "executemany", "executescript", "fetchall", "fetchmany", "fetchone"): @@ -1888,10 +1920,9 @@ def test_on_conflict_replace(self): self.assertEqual(self.cu.fetchall(), [('Very different data!', 'foo')]) -# TODO: RUSTPYTHON -# @requires_subprocess() +@requires_subprocess() class MultiprocessTests(unittest.TestCase): - CONNECTION_TIMEOUT = SHORT_TIMEOUT / 1000. # Defaults to 30 ms + CONNECTION_TIMEOUT = 0 # Disable the busy timeout. def tearDown(self): unlink(TESTFN) @@ -1961,5 +1992,71 @@ def wait(): self.assertEqual(proc.returncode, 0) +class RowTests(unittest.TestCase): + + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.row_factory = sqlite.Row + + def tearDown(self): + self.cx.close() + + def test_row_keys(self): + cu = self.cx.execute("SELECT 1 as first, 2 as second") + row = cu.fetchone() + self.assertEqual(row.keys(), ["first", "second"]) + + def test_row_length(self): + cu = self.cx.execute("SELECT 1, 2, 3") + row = cu.fetchone() + self.assertEqual(len(row), 3) + + def test_row_getitem(self): + cu = self.cx.execute("SELECT 1 as a, 2 as b") + row = cu.fetchone() + self.assertEqual(row[0], 1) + self.assertEqual(row[1], 2) + self.assertEqual(row["a"], 1) + self.assertEqual(row["b"], 2) + for key in "nokey", 4, 1.2: + with self.subTest(key=key): + with self.assertRaises(IndexError): + row[key] + + def test_row_equality(self): + c1 = self.cx.execute("SELECT 1 as a") + r1 = c1.fetchone() + + c2 = self.cx.execute("SELECT 1 as a") + r2 = c2.fetchone() + + self.assertIsNot(r1, r2) + self.assertEqual(r1, r2) + + c3 = self.cx.execute("SELECT 1 as b") + r3 = c3.fetchone() + + self.assertNotEqual(r1, r3) + + @unittest.expectedFailure # TODO: RUSTPYTHON Row with no description fails + def test_row_no_description(self): + cu = self.cx.cursor() + self.assertIsNone(cu.description) + + row = sqlite.Row(cu, ()) + self.assertEqual(row.keys(), []) + with self.assertRaisesRegex(IndexError, "nokey"): + row["nokey"] + + def test_row_is_a_sequence(self): + from collections.abc import Sequence + + cu = self.cx.execute("SELECT 1") + row = cu.fetchone() + + self.assertIsSubclass(sqlite.Row, Sequence) + self.assertIsInstance(row, Sequence) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index ec4a11da8b0..74aacc05c2b 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -1,22 +1,18 @@ # Author: Paul Kippes <kippesp@gmail.com> import unittest -import sqlite3 as sqlite -from .test_dbapi import memory_database +from .util import memory_database +from .util import MemoryDatabaseMixin +from .util import requires_virtual_table -# TODO: RUSTPYTHON -@unittest.expectedFailure -class DumpTests(unittest.TestCase): - def setUp(self): - self.cx = sqlite.connect(":memory:") - self.cu = self.cx.cursor() - def tearDown(self): - self.cx.close() +class DumpTests(MemoryDatabaseMixin, unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON def test_table_dump(self): expected_sqls = [ + "PRAGMA foreign_keys=OFF;", """CREATE TABLE "index"("index" blob);""" , """INSERT INTO "index" VALUES(X'01');""" @@ -27,7 +23,8 @@ def test_table_dump(self): , "CREATE TABLE t1(id integer primary key, s1 text, " \ "t1_i1 integer not null, i2 integer, unique (s1), " \ - "constraint t1_idx1 unique (i2));" + "constraint t1_idx1 unique (i2), " \ + "constraint t1_i1_idx1 unique (t1_i1));" , "INSERT INTO \"t1\" VALUES(1,'foo',10,20);" , @@ -37,6 +34,9 @@ def test_table_dump(self): "t2_i2 integer, primary key (id)," \ "foreign key(t2_i1) references t1(t1_i1));" , + # Foreign key violation. + "INSERT INTO \"t2\" VALUES(1,2,3);" + , "CREATE TRIGGER trigger_1 update of t1_i1 on t1 " \ "begin " \ "update t2 set t2_i1 = new.t1_i1 where t2_i1 = old.t1_i1; " \ @@ -48,11 +48,87 @@ def test_table_dump(self): [self.cu.execute(s) for s in expected_sqls] i = self.cx.iterdump() actual_sqls = [s for s in i] - expected_sqls = ['BEGIN TRANSACTION;'] + expected_sqls + \ - ['COMMIT;'] + expected_sqls = [ + "PRAGMA foreign_keys=OFF;", + "BEGIN TRANSACTION;", + *expected_sqls[1:], + "COMMIT;", + ] [self.assertEqual(expected_sqls[i], actual_sqls[i]) for i in range(len(expected_sqls))] + @unittest.expectedFailure # TODO: RUSTPYTHON iterdump filter parameter not implemented + def test_table_dump_filter(self): + all_table_sqls = [ + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + ] + all_views_sqls = [ + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + ] + # Create database structure. + for sql in [*all_table_sqls, *all_views_sqls]: + self.cu.execute(sql) + # %_table_% matches all tables. + dump_sqls = list(self.cx.iterdump(filter="%_table_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, "COMMIT;"], + ) + # view_% matches all views. + dump_sqls = list(self.cx.iterdump(filter="view_%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_views_sqls, "COMMIT;"], + ) + # %_1 matches tables and views with the _1 suffix. + dump_sqls = list(self.cx.iterdump(filter="%_1")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "test_table_1" ("id_2" INTEGER);""", + """INSERT INTO "test_table_1" VALUES(1);""", + """INSERT INTO "test_table_1" VALUES(2);""", + """CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""", + "COMMIT;" + ], + ) + # some_% matches some_table_2. + dump_sqls = list(self.cx.iterdump(filter="some_%")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", + """INSERT INTO "some_table_2" VALUES(3);""", + """INSERT INTO "some_table_2" VALUES(4);""", + "COMMIT;" + ], + ) + # Only single object. + dump_sqls = list(self.cx.iterdump(filter="view_2")) + self.assertEqual( + dump_sqls, + [ + "BEGIN TRANSACTION;", + """CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""", + "COMMIT;" + ], + ) + # % matches all objects. + dump_sqls = list(self.cx.iterdump(filter="%")) + self.assertEqual( + dump_sqls, + ["BEGIN TRANSACTION;", *all_table_sqls, *all_views_sqls, "COMMIT;"], + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented def test_dump_autoincrement(self): expected = [ 'CREATE TABLE "t1" (id integer primary key autoincrement);', @@ -73,6 +149,7 @@ def test_dump_autoincrement(self): actual = [stmt for stmt in self.cx.iterdump()] self.assertEqual(expected, actual) + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented def test_dump_autoincrement_create_new_db(self): self.cu.execute("BEGIN TRANSACTION") self.cu.execute("CREATE TABLE t1 (id integer primary key autoincrement)") @@ -98,6 +175,7 @@ def test_dump_autoincrement_create_new_db(self): rows = res.fetchall() self.assertEqual(rows[0][0], seq) + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented def test_unorderable_row(self): # iterdump() should be able to cope with unorderable row types (issue #15545) class UnorderableRow: @@ -119,6 +197,44 @@ def __getitem__(self, index): got = list(self.cx.iterdump()) self.assertEqual(expected, got) + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented + def test_dump_custom_row_factory(self): + # gh-118221: iterdump should be able to cope with custom row factories. + def dict_factory(cu, row): + fields = [col[0] for col in cu.description] + return dict(zip(fields, row)) + + self.cx.row_factory = dict_factory + CREATE_TABLE = "CREATE TABLE test(t);" + expected = ["BEGIN TRANSACTION;", CREATE_TABLE, "COMMIT;"] + + self.cu.execute(CREATE_TABLE) + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + self.assertEqual(self.cx.row_factory, dict_factory) + + @unittest.expectedFailure # TODO: RUSTPYTHON _iterdump not implemented + @requires_virtual_table("fts4") + def test_dump_virtual_tables(self): + # gh-64662 + expected = [ + "BEGIN TRANSACTION;", + "PRAGMA writable_schema=ON;", + ("INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table','test','test',0,'CREATE VIRTUAL TABLE test USING fts4(example)');"), + "CREATE TABLE 'test_content'(docid INTEGER PRIMARY KEY, 'c0example');", + "CREATE TABLE 'test_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + ("CREATE TABLE 'test_segdir'(level INTEGER,idx INTEGER,start_block INTEGER," + "leaves_end_block INTEGER,end_block INTEGER,root BLOB,PRIMARY KEY(level, idx));"), + "CREATE TABLE 'test_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + "CREATE TABLE 'test_stat'(id INTEGER PRIMARY KEY, value BLOB);", + "PRAGMA writable_schema=OFF;", + "COMMIT;" + ] + self.cu.execute("CREATE VIRTUAL TABLE test USING fts4(example)") + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index e52c10fe944..2816bd91253 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -24,6 +24,9 @@ import sqlite3 as sqlite from collections.abc import Sequence +from .util import memory_database +from .util import MemoryDatabaseMixin + def dict_factory(cursor, row): d = {} @@ -37,6 +40,7 @@ def __init__(self, *args, **kwargs): self.row_factory = dict_factory class ConnectionFactoryTests(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_factories(self): class DefectFactory(sqlite.Connection): def __init__(self, *args, **kwargs): @@ -45,13 +49,14 @@ class OkFactory(sqlite.Connection): def __init__(self, *args, **kwargs): sqlite.Connection.__init__(self, *args, **kwargs) - for factory in DefectFactory, OkFactory: - with self.subTest(factory=factory): - con = sqlite.connect(":memory:", factory=factory) - self.assertIsInstance(con, factory) + with memory_database(factory=OkFactory) as con: + self.assertIsInstance(con, OkFactory) + regex = "Base Connection.__init__ not called." + with self.assertRaisesRegex(sqlite.ProgrammingError, regex): + with memory_database(factory=DefectFactory) as con: + self.assertIsInstance(con, DefectFactory) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_factory_relayed_call(self): # gh-95132: keyword args must not be passed as positional args class Factory(sqlite.Connection): @@ -59,29 +64,35 @@ def __init__(self, *args, **kwargs): kwargs["isolation_level"] = None super(Factory, self).__init__(*args, **kwargs) - con = sqlite.connect(":memory:", factory=Factory) - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) + with memory_database(factory=Factory) as con: + self.assertIsNone(con.isolation_level) + self.assertIsInstance(con, Factory) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_factory_as_positional_arg(self): class Factory(sqlite.Connection): def __init__(self, *args, **kwargs): super(Factory, self).__init__(*args, **kwargs) - con = sqlite.connect(":memory:", 5.0, 0, None, True, Factory) - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) + regex = ( + r"Passing more than 1 positional argument to _sqlite3.Connection\(\) " + r"is deprecated. Parameters 'timeout', 'detect_types', " + r"'isolation_level', 'check_same_thread', 'factory', " + r"'cached_statements' and 'uri' will become keyword-only " + r"parameters in Python 3.15." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + with memory_database(5.0, 0, None, True, Factory) as con: + self.assertIsNone(con.isolation_level) + self.assertIsInstance(con, Factory) + self.assertEqual(cm.filename, __file__) -class CursorFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class CursorFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def tearDown(self): self.con.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_is_instance(self): cur = self.con.cursor() self.assertIsInstance(cur, sqlite.Cursor) @@ -98,12 +109,9 @@ def test_invalid_factory(self): # invalid callable returning non-cursor self.assertRaises(TypeError, self.con.cursor, lambda con: None) -class RowFactoryTestsBackwardsCompat(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - # TODO: RUSTPYTHON - @unittest.expectedFailure +class RowFactoryTestsBackwardsCompat(MemoryDatabaseMixin, unittest.TestCase): + def test_is_produced_by_factory(self): cur = self.con.cursor(factory=MyCursor) cur.execute("select 4+5 as foo") @@ -111,22 +119,20 @@ def test_is_produced_by_factory(self): self.assertIsInstance(row, dict) cur.close() - def tearDown(self): - self.con.close() -class RowFactoryTests(unittest.TestCase): +class RowFactoryTests(MemoryDatabaseMixin, unittest.TestCase): + def setUp(self): - self.con = sqlite.connect(":memory:") + super().setUp() + self.con.row_factory = sqlite.Row def test_custom_factory(self): self.con.row_factory = lambda cur, row: list(row) row = self.con.execute("select 1, 2").fetchone() self.assertIsInstance(row, list) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_sqlite_row_index(self): - self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as a_1, 2 as b").fetchone() self.assertIsInstance(row, sqlite.Row) @@ -156,10 +162,8 @@ def test_sqlite_row_index(self): with self.assertRaises(IndexError): row[complex()] # index must be int or string - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_sqlite_row_index_unicode(self): - self.con.row_factory = sqlite.Row row = self.con.execute("select 1 as \xff").fetchone() self.assertEqual(row["\xff"], 1) with self.assertRaises(IndexError): @@ -169,7 +173,6 @@ def test_sqlite_row_index_unicode(self): def test_sqlite_row_slice(self): # A sqlite.Row can be sliced like a list. - self.con.row_factory = sqlite.Row row = self.con.execute("select 1, 2, 3, 4").fetchone() self.assertEqual(row[0:0], ()) self.assertEqual(row[0:1], (1,)) @@ -186,30 +189,32 @@ def test_sqlite_row_slice(self): self.assertEqual(row[3:0:-2], (4, 2)) def test_sqlite_row_iter(self): - """Checks if the row object is iterable""" - self.con.row_factory = sqlite.Row + # Checks if the row object is iterable. row = self.con.execute("select 1 as a, 2 as b").fetchone() - for col in row: - pass + + # Is iterable in correct order and produces valid results: + items = [col for col in row] + self.assertEqual(items, [1, 2]) + + # Is iterable the second time: + items = [col for col in row] + self.assertEqual(items, [1, 2]) def test_sqlite_row_as_tuple(self): - """Checks if the row object can be converted to a tuple""" - self.con.row_factory = sqlite.Row + # Checks if the row object can be converted to a tuple. row = self.con.execute("select 1 as a, 2 as b").fetchone() t = tuple(row) self.assertEqual(t, (row['a'], row['b'])) def test_sqlite_row_as_dict(self): - """Checks if the row object can be correctly converted to a dictionary""" - self.con.row_factory = sqlite.Row + # Checks if the row object can be correctly converted to a dictionary. row = self.con.execute("select 1 as a, 2 as b").fetchone() d = dict(row) self.assertEqual(d["a"], row["a"]) self.assertEqual(d["b"], row["b"]) def test_sqlite_row_hash_cmp(self): - """Checks if the row object compares and hashes correctly""" - self.con.row_factory = sqlite.Row + # Checks if the row object compares and hashes correctly. row_1 = self.con.execute("select 1 as a, 2 as b").fetchone() row_2 = self.con.execute("select 1 as a, 2 as b").fetchone() row_3 = self.con.execute("select 1 as a, 3 as b").fetchone() @@ -241,33 +246,30 @@ def test_sqlite_row_hash_cmp(self): self.assertEqual(hash(row_1), hash(row_2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_sqlite_row_as_sequence(self): - """ Checks if the row object can act like a sequence """ - self.con.row_factory = sqlite.Row + # Checks if the row object can act like a sequence. row = self.con.execute("select 1 as a, 2 as b").fetchone() as_tuple = tuple(row) self.assertEqual(list(reversed(row)), list(reversed(as_tuple))) self.assertIsInstance(row, Sequence) + def test_sqlite_row_keys(self): + # Checks if the row object can return a list of columns as strings. + row = self.con.execute("select 1 as a, 2 as b").fetchone() + self.assertEqual(row.keys(), ['a', 'b']) + def test_fake_cursor_class(self): # Issue #24257: Incorrect use of PyObject_IsInstance() caused # segmentation fault. # Issue #27861: Also applies for cursor factory. class FakeCursor(str): __class__ = sqlite.Cursor - self.con.row_factory = sqlite.Row self.assertRaises(TypeError, self.con.cursor, FakeCursor) self.assertRaises(TypeError, sqlite.Row, FakeCursor(), ()) - def tearDown(self): - self.con.close() -class TextFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class TextFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def test_unicode(self): austria = "Österreich" @@ -286,17 +288,19 @@ def test_custom(self): austria = "Österreich" row = self.con.execute("select ?", (austria,)).fetchone() self.assertEqual(type(row[0]), str, "type of row[0] must be unicode") - self.assertTrue(row[0].endswith("reich"), "column must contain original data") + self.assertEndsWith(row[0], "reich", "column must contain original data") - def tearDown(self): - self.con.close() class TextFactoryTestsWithEmbeddedZeroBytes(unittest.TestCase): + def setUp(self): self.con = sqlite.connect(":memory:") self.con.execute("create table test (value text)") self.con.execute("insert into test (value) values (?)", ("a\x00b",)) + def tearDown(self): + self.con.close() + def test_string(self): # text_factory defaults to str row = self.con.execute("select value from test").fetchone() @@ -322,9 +326,6 @@ def test_custom(self): self.assertIs(type(row[0]), bytes) self.assertEqual(row[0], b"a\x00b") - def tearDown(self): - self.con.close() - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 21042b9bf10..c47cfab180d 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -26,34 +26,31 @@ from test.support.os_helper import TESTFN, unlink -from test.test_sqlite3.test_dbapi import memory_database, cx_limit -from test.test_sqlite3.test_userfunctions import with_tracebacks +from .util import memory_database, cx_limit, with_tracebacks +from .util import MemoryDatabaseMixin -class CollationTests(unittest.TestCase): +class CollationTests(MemoryDatabaseMixin, unittest.TestCase): + def test_create_collation_not_string(self): - con = sqlite.connect(":memory:") with self.assertRaises(TypeError): - con.create_collation(None, lambda x, y: (x > y) - (x < y)) + self.con.create_collation(None, lambda x, y: (x > y) - (x < y)) def test_create_collation_not_callable(self): - con = sqlite.connect(":memory:") with self.assertRaises(TypeError) as cm: - con.create_collation("X", 42) + self.con.create_collation("X", 42) self.assertEqual(str(cm.exception), 'parameter must be callable') def test_create_collation_not_ascii(self): - con = sqlite.connect(":memory:") - con.create_collation("collä", lambda x, y: (x > y) - (x < y)) + self.con.create_collation("collä", lambda x, y: (x > y) - (x < y)) def test_create_collation_bad_upper(self): class BadUpperStr(str): def upper(self): return None - con = sqlite.connect(":memory:") mycoll = lambda x, y: -((x > y) - (x < y)) - con.create_collation(BadUpperStr("mycoll"), mycoll) - result = con.execute(""" + self.con.create_collation(BadUpperStr("mycoll"), mycoll) + result = self.con.execute(""" select x from ( select 'a' as x union @@ -68,8 +65,7 @@ def mycoll(x, y): # reverse order return -((x > y) - (x < y)) - con = sqlite.connect(":memory:") - con.create_collation("mycoll", mycoll) + self.con.create_collation("mycoll", mycoll) sql = """ select x from ( select 'a' as x @@ -79,21 +75,20 @@ def mycoll(x, y): select 'c' as x ) order by x collate mycoll """ - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(result, [('c',), ('b',), ('a',)], msg='the expected order was not returned') - con.create_collation("mycoll", None) + self.con.create_collation("mycoll", None) with self.assertRaises(sqlite.OperationalError) as cm: - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') def test_collation_returns_large_integer(self): def mycoll(x, y): # reverse order return -((x > y) - (x < y)) * 2**32 - con = sqlite.connect(":memory:") - con.create_collation("mycoll", mycoll) + self.con.create_collation("mycoll", mycoll) sql = """ select x from ( select 'a' as x @@ -103,7 +98,7 @@ def mycoll(x, y): select 'c' as x ) order by x collate mycoll """ - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(result, [('c',), ('b',), ('a',)], msg="the expected order was not returned") @@ -112,7 +107,7 @@ def test_collation_register_twice(self): Register two different collation functions under the same name. Verify that the last one is actually used. """ - con = sqlite.connect(":memory:") + con = self.con con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) con.create_collation("mycoll", lambda x, y: -((x > y) - (x < y))) result = con.execute(""" @@ -126,25 +121,26 @@ def test_deregister_collation(self): Register a collation, then deregister it. Make sure an error is raised if we try to use it. """ - con = sqlite.connect(":memory:") + con = self.con con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) con.create_collation("mycoll", None) with self.assertRaises(sqlite.OperationalError) as cm: con.execute("select 'a' as x union select 'b' as x order by x collate mycoll") self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') -class ProgressTests(unittest.TestCase): + +class ProgressTests(MemoryDatabaseMixin, unittest.TestCase): + def test_progress_handler_used(self): """ Test that the progress handler is invoked once it is set. """ - con = sqlite.connect(":memory:") progress_calls = [] def progress(): progress_calls.append(None) return 0 - con.set_progress_handler(progress, 1) - con.execute(""" + self.con.set_progress_handler(progress, 1) + self.con.execute(""" create table foo(a, b) """) self.assertTrue(progress_calls) @@ -153,7 +149,7 @@ def test_opcode_count(self): """ Test that the opcode argument is respected. """ - con = sqlite.connect(":memory:") + con = self.con progress_calls = [] def progress(): progress_calls.append(None) @@ -176,11 +172,10 @@ def test_cancel_operation(self): """ Test that returning a non-zero value stops the operation in progress. """ - con = sqlite.connect(":memory:") def progress(): return 1 - con.set_progress_handler(progress, 1) - curs = con.cursor() + self.con.set_progress_handler(progress, 1) + curs = self.con.cursor() self.assertRaises( sqlite.OperationalError, curs.execute, @@ -190,7 +185,7 @@ def test_clear_handler(self): """ Test that setting the progress handler to None clears the previously set handler. """ - con = sqlite.connect(":memory:") + con = self.con action = 0 def progress(): nonlocal action @@ -201,33 +196,47 @@ def progress(): con.execute("select 1 union select 2 union select 3").fetchall() self.assertEqual(action, 0, "progress handler was not cleared") - @with_tracebacks(ZeroDivisionError, name="bad_progress") + @unittest.expectedFailure # TODO: RUSTPYTHON + @with_tracebacks(ZeroDivisionError, msg_regex="bad_progress") def test_error_in_progress_handler(self): - con = sqlite.connect(":memory:") def bad_progress(): 1 / 0 - con.set_progress_handler(bad_progress, 1) + self.con.set_progress_handler(bad_progress, 1) with self.assertRaises(sqlite.OperationalError): - con.execute(""" + self.con.execute(""" create table foo(a, b) """) - @with_tracebacks(ZeroDivisionError, name="bad_progress") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="bad_progress") def test_error_in_progress_handler_result(self): - con = sqlite.connect(":memory:") class BadBool: def __bool__(self): 1 / 0 def bad_progress(): return BadBool() - con.set_progress_handler(bad_progress, 1) + self.con.set_progress_handler(bad_progress, 1) with self.assertRaises(sqlite.OperationalError): - con.execute(""" + self.con.execute(""" create table foo(a, b) """) + @unittest.expectedFailure # TODO: RUSTPYTHON keyword-only arguments not supported for set_progress_handler + def test_progress_handler_keyword_args(self): + regex = ( + r"Passing keyword argument 'progress_handler' to " + r"_sqlite3.Connection.set_progress_handler\(\) is deprecated. " + r"Parameter 'progress_handler' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_progress_handler(progress_handler=lambda: None, n=1) + self.assertEqual(cm.filename, __file__) + + +class TraceCallbackTests(MemoryDatabaseMixin, unittest.TestCase): -class TraceCallbackTests(unittest.TestCase): @contextlib.contextmanager def check_stmt_trace(self, cx, expected): try: @@ -242,12 +251,11 @@ def test_trace_callback_used(self): """ Test that the trace callback is invoked once it is set. """ - con = sqlite.connect(":memory:") traced_statements = [] def trace(statement): traced_statements.append(statement) - con.set_trace_callback(trace) - con.execute("create table foo(a, b)") + self.con.set_trace_callback(trace) + self.con.execute("create table foo(a, b)") self.assertTrue(traced_statements) self.assertTrue(any("create table foo" in stmt for stmt in traced_statements)) @@ -255,7 +263,7 @@ def test_clear_trace_callback(self): """ Test that setting the trace callback to None clears the previously set callback. """ - con = sqlite.connect(":memory:") + con = self.con traced_statements = [] def trace(statement): traced_statements.append(statement) @@ -269,7 +277,7 @@ def test_unicode_content(self): Test that the statement can contain unicode literals. """ unicode_value = '\xf6\xe4\xfc\xd6\xc4\xdc\xdf\u20ac' - con = sqlite.connect(":memory:") + con = self.con traced_statements = [] def trace(statement): traced_statements.append(statement) @@ -317,13 +325,14 @@ def test_trace_expanded_sql(self): cx.execute("create table t(t)") cx.executemany("insert into t values(?)", ((v,) for v in range(3))) + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks( sqlite.DataError, regex="Expanded SQL string exceeds the maximum string length" ) def test_trace_too_much_expanded_sql(self): # If the expanded string is too large, we'll fall back to the - # unexpanded SQL statement (for SQLite 3.14.0 and newer). + # unexpanded SQL statement. # The resulting string length is limited by the runtime limit # SQLITE_LIMIT_LENGTH. template = "select 1 as a where a=" @@ -334,8 +343,6 @@ def test_trace_too_much_expanded_sql(self): unexpanded_query = template + "?" expected = [unexpanded_query] - if sqlite.sqlite_version_info < (3, 14, 0): - expected = [] with self.check_stmt_trace(cx, expected): cx.execute(unexpanded_query, (bad_param,)) @@ -343,12 +350,26 @@ def test_trace_too_much_expanded_sql(self): with self.check_stmt_trace(cx, [expanded_query]): cx.execute(unexpanded_query, (ok_param,)) + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(ZeroDivisionError, regex="division by zero") def test_trace_bad_handler(self): with memory_database() as cx: cx.set_trace_callback(lambda stmt: 5/0) cx.execute("select 1") + @unittest.expectedFailure # TODO: RUSTPYTHON keyword-only arguments not supported for set_trace_callback + def test_trace_keyword_args(self): + regex = ( + r"Passing keyword argument 'trace_callback' to " + r"_sqlite3.Connection.set_trace_callback\(\) is deprecated. " + r"Parameter 'trace_callback' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_trace_callback(trace_callback=lambda: None) + self.assertEqual(cm.filename, __file__) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index dfcf3b11f57..0ebd6d5e9da 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -28,15 +28,12 @@ from test import support from unittest.mock import patch -from test.test_sqlite3.test_dbapi import memory_database, cx_limit +from .util import memory_database, cx_limit +from .util import MemoryDatabaseMixin -class RegressionTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - def tearDown(self): - self.con.close() +class RegressionTests(MemoryDatabaseMixin, unittest.TestCase): def test_pragma_user_version(self): # This used to crash pysqlite because this pragma command returns NULL for the column name @@ -45,28 +42,24 @@ def test_pragma_user_version(self): def test_pragma_schema_version(self): # This still crashed pysqlite <= 2.2.1 - con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES) - try: + with memory_database(detect_types=sqlite.PARSE_COLNAMES) as con: cur = self.con.cursor() cur.execute("pragma schema_version") - finally: - cur.close() - con.close() def test_statement_reset(self): # pysqlite 2.1.0 to 2.2.0 have the problem that not all statements are # reset before a rollback, but only those that are still in the # statement cache. The others are not accessible from the connection object. - con = sqlite.connect(":memory:", cached_statements=5) - cursors = [con.cursor() for x in range(5)] - cursors[0].execute("create table test(x)") - for i in range(10): - cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)]) + with memory_database(cached_statements=5) as con: + cursors = [con.cursor() for x in range(5)] + cursors[0].execute("create table test(x)") + for i in range(10): + cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)]) - for i in range(5): - cursors[i].execute(" " * i + "select x from test") + for i in range(5): + cursors[i].execute(" " * i + "select x from test") - con.rollback() + con.rollback() def test_column_name_with_spaces(self): cur = self.con.cursor() @@ -81,17 +74,15 @@ def test_statement_finalization_on_close_db(self): # cache when closing the database. statements that were still # referenced in cursors weren't closed and could provoke " # "OperationalError: Unable to close due to unfinalised statements". - con = sqlite.connect(":memory:") cursors = [] # default statement cache size is 100 for i in range(105): - cur = con.cursor() + cur = self.con.cursor() cursors.append(cur) cur.execute("select 1 x union select " + str(i)) - con.close() def test_on_conflict_rollback(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("create table foo(x, unique(x) on conflict rollback)") con.execute("insert into foo(x) values (1)") try: @@ -126,16 +117,16 @@ def test_type_map_usage(self): a statement. This test exhibits the problem. """ SELECT = "select * from foo" - con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) - cur = con.cursor() - cur.execute("create table foo(bar timestamp)") - with self.assertWarnsRegex(DeprecationWarning, "adapter"): - cur.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) - cur.execute(SELECT) - cur.execute("drop table foo") - cur.execute("create table foo(bar integer)") - cur.execute("insert into foo(bar) values (5)") - cur.execute(SELECT) + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + cur = con.cursor() + cur.execute("create table foo(bar timestamp)") + with self.assertWarnsRegex(DeprecationWarning, "adapter"): + cur.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) + cur.execute(SELECT) + cur.execute("drop table foo") + cur.execute("create table foo(bar integer)") + cur.execute("insert into foo(bar) values (5)") + cur.execute(SELECT) def test_bind_mutating_list(self): # Issue41662: Crash when mutate a list of parameters during iteration. @@ -144,14 +135,12 @@ def __conform__(self, protocol): parameters.clear() return "..." parameters = [X(), 0] - con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) - con.execute("create table foo(bar X, baz integer)") - # Should not crash - with self.assertRaises(IndexError): - con.execute("insert into foo(bar, baz) values (?, ?)", parameters) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + con.execute("create table foo(bar X, baz integer)") + # Should not crash + with self.assertRaises(IndexError): + con.execute("insert into foo(bar, baz) values (?, ?)", parameters) + def test_error_msg_decode_error(self): # When porting the module to Python 3.0, the error message about # decoding errors disappeared. This verifies they're back again. @@ -175,7 +164,7 @@ def upper(self): def __del__(self): con.isolation_level = "" - con = sqlite.connect(":memory:") + con = self.con con.isolation_level = None for level in "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE": with self.subTest(level=level): @@ -197,8 +186,6 @@ def __del__(self): con.isolation_level = value self.assertEqual(con.isolation_level, "DEFERRED") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cursor_constructor_call_check(self): """ Verifies that cursor methods check whether base class __init__ was @@ -208,8 +195,7 @@ class Cursor(sqlite.Cursor): def __init__(self, con): pass - con = sqlite.connect(":memory:") - cur = Cursor(con) + cur = Cursor(self.con) with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4+5").fetchall() with self.assertRaisesRegex(sqlite.ProgrammingError, @@ -223,8 +209,6 @@ def test_str_subclass(self): class MyStr(str): pass self.con.execute("select ?", (MyStr("abc"),)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_connection_constructor_call_check(self): """ Verifies that connection methods check whether base class __init__ was @@ -244,7 +228,9 @@ def test_auto_commit(self): 2.5.3 introduced a regression so that these could no longer be created. """ - con = sqlite.connect(":memory:", isolation_level=None) + with memory_database(isolation_level=None) as con: + self.assertIsNone(con.isolation_level) + self.assertFalse(con.in_transaction) def test_pragma_autocommit(self): """ @@ -265,8 +251,6 @@ def test_connection_call(self): """ self.assertRaises(TypeError, self.con, b"select 1") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_collation(self): def collation_cb(a, b): return 1 @@ -274,7 +258,7 @@ def collation_cb(a, b): # Lone surrogate cannot be encoded to the default encoding (utf8) "\uDC80", collation_cb) - @unittest.skip("TODO: RUSTPYTHON deadlock") + @unittest.skip('TODO: RUSTPYTHON; recursive cursor use causes lock contention') def test_recursive_cursor_use(self): """ http://bugs.python.org/issue10811 @@ -282,9 +266,7 @@ def test_recursive_cursor_use(self): Recursively using a cursor, such as when reusing it from a generator led to segfaults. Now we catch recursive cursor usage and raise a ProgrammingError. """ - con = sqlite.connect(":memory:") - - cur = con.cursor() + cur = self.con.cursor() cur.execute("create table a (bar)") cur.execute("create table b (baz)") @@ -304,33 +286,33 @@ def test_convert_timestamp_microsecond_padding(self): since the microsecond string "456" actually represents "456000". """ - con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_DECLTYPES) - cur = con.cursor() - cur.execute("CREATE TABLE t (x TIMESTAMP)") + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + cur = con.cursor() + cur.execute("CREATE TABLE t (x TIMESTAMP)") - # Microseconds should be 456000 - cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')") + # Microseconds should be 456000 + cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')") - # Microseconds should be truncated to 123456 - cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')") + # Microseconds should be truncated to 123456 + cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')") - cur.execute("SELECT * FROM t") - with self.assertWarnsRegex(DeprecationWarning, "converter"): - values = [x[0] for x in cur.fetchall()] + cur.execute("SELECT * FROM t") + with self.assertWarnsRegex(DeprecationWarning, "converter"): + values = [x[0] for x in cur.fetchall()] - self.assertEqual(values, [ - datetime.datetime(2012, 4, 4, 15, 6, 0, 456000), - datetime.datetime(2012, 4, 4, 15, 6, 0, 123456), - ]) + self.assertEqual(values, [ + datetime.datetime(2012, 4, 4, 15, 6, 0, 456000), + datetime.datetime(2012, 4, 4, 15, 6, 0, 123456), + ]) + @unittest.expectedFailure # TODO: RUSTPYTHON; error message mismatch def test_invalid_isolation_level_type(self): # isolation level is a string, not an integer - self.assertRaises(TypeError, - sqlite.connect, ":memory:", isolation_level=123) + regex = "isolation_level must be str or None" + with self.assertRaisesRegex(TypeError, regex): + memory_database(isolation_level=123).__enter__() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_null_character(self): # Issue #21147 cur = self.con.cursor() @@ -343,18 +325,14 @@ def test_null_character(self): self.assertRaisesRegex(sqlite.ProgrammingError, "null char", cur.execute, query) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogates(self): - con = sqlite.connect(":memory:") + con = self.con self.assertRaises(UnicodeEncodeError, con, "select '\ud8ff'") self.assertRaises(UnicodeEncodeError, con, "select '\udcff'") cur = con.cursor() self.assertRaises(UnicodeEncodeError, cur.execute, "select '\ud8ff'") self.assertRaises(UnicodeEncodeError, cur.execute, "select '\udcff'") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_large_sql(self): msg = "query string is too large" with memory_database() as cx, cx_limit(cx) as lim: @@ -374,7 +352,7 @@ def test_commit_cursor_reset(self): to return rows multiple times when fetched from cursors after commit. See issues 10513 and 23129 for details. """ - con = sqlite.connect(":memory:") + con = self.con con.executescript(""" create table t(c); create table t2(c); @@ -406,23 +384,19 @@ def test_bpo31770(self): """ def callback(*args): pass - con = sqlite.connect(":memory:") - cur = sqlite.Cursor(con) + cur = sqlite.Cursor(self.con) ref = weakref.ref(cur, callback) - cur.__init__(con) + cur.__init__(self.con) del cur # The interpreter shouldn't crash when ref is collected. del ref support.gc_collect() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_del_isolation_level_segfault(self): with self.assertRaises(AttributeError): del self.con.isolation_level - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bpo37347(self): class Printer: def log(self, *args): @@ -444,6 +418,7 @@ def test_return_empty_bytestring(self): def test_table_lock_cursor_replace_stmt(self): with memory_database() as con: + con = self.con cur = con.cursor() cur.execute("create table t(t)") cur.executemany("insert into t values(?)", @@ -461,10 +436,11 @@ def test_table_lock_cursor_dealloc(self): con.commit() cur = con.execute("select t from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() - @unittest.skip("TODO: RUSTPYTHON deadlock") + @unittest.skip('TODO: RUSTPYTHON; recursive cursor use causes lock contention') def test_table_lock_cursor_non_readonly_select(self): with memory_database() as con: con.execute("create table t(t)") @@ -477,6 +453,7 @@ def dup(v): con.create_function("dup", 1, dup) cur = con.execute("select dup(t) from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() @@ -492,7 +469,7 @@ def test_executescript_step_through_select(self): self.assertEqual(steps, values) -@unittest.skip("TODO: RUSTPYTHON deadlock") +@unittest.skip('TODO: RUSTPYTHON; recursive cursor use causes lock contention') class RecursiveUseOfCursors(unittest.TestCase): # GH-80254: sqlite3 should not segfault for recursive use of cursors. msg = "Recursive use of cursors not allowed" @@ -512,21 +489,21 @@ def tearDown(self): def test_recursive_cursor_init(self): conv = lambda x: self.cur.__init__(self.con) with patch.dict(sqlite.converters, {"INIT": conv}): - self.cur.execute(f'select x as "x [INIT]", x from test') + self.cur.execute('select x as "x [INIT]", x from test') self.assertRaisesRegex(sqlite.ProgrammingError, self.msg, self.cur.fetchall) def test_recursive_cursor_close(self): conv = lambda x: self.cur.close() with patch.dict(sqlite.converters, {"CLOSE": conv}): - self.cur.execute(f'select x as "x [CLOSE]", x from test') + self.cur.execute('select x as "x [CLOSE]", x from test') self.assertRaisesRegex(sqlite.ProgrammingError, self.msg, self.cur.fetchall) def test_recursive_cursor_iter(self): conv = lambda x, l=[]: self.cur.fetchone() if l else l.append(None) with patch.dict(sqlite.converters, {"ITER": conv}): - self.cur.execute(f'select x as "x [ITER]", x from test') + self.cur.execute('select x as "x [ITER]", x from test') self.assertRaisesRegex(sqlite.ProgrammingError, self.msg, self.cur.fetchall) diff --git a/Lib/test/test_sqlite3/test_transactions.py b/Lib/test/test_sqlite3/test_transactions.py index 8835ef76b79..3b57b7f6a08 100644 --- a/Lib/test/test_sqlite3/test_transactions.py +++ b/Lib/test/test_sqlite3/test_transactions.py @@ -22,22 +22,23 @@ import unittest import sqlite3 as sqlite +from contextlib import contextmanager -from test.support import LOOPBACK_TIMEOUT from test.support.os_helper import TESTFN, unlink +from test.support.script_helper import assert_python_ok -from test.test_sqlite3.test_dbapi import memory_database - - -TIMEOUT = LOOPBACK_TIMEOUT / 10 +from .util import memory_database +from .util import MemoryDatabaseMixin class TransactionTests(unittest.TestCase): def setUp(self): - self.con1 = sqlite.connect(TESTFN, timeout=TIMEOUT) + # We can disable the busy handlers, since we control + # the order of SQLite C API operations. + self.con1 = sqlite.connect(TESTFN, timeout=0) self.cur1 = self.con1.cursor() - self.con2 = sqlite.connect(TESTFN, timeout=TIMEOUT) + self.con2 = sqlite.connect(TESTFN, timeout=0) self.cur2 = self.con2.cursor() def tearDown(self): @@ -94,8 +95,6 @@ def test_replace_starts_transaction(self): self.assertEqual(len(res), 1) self.assertEqual(res[0][0], 5) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_toggle_auto_commit(self): self.cur1.execute("create table test(i)") self.cur1.execute("insert into test(i) values (5)") @@ -119,10 +118,8 @@ def test_raise_timeout(self): self.cur2.execute("insert into test(i) values (5)") def test_locking(self): - """ - This tests the improved concurrency with pysqlite 2.3.4. You needed - to roll back con2 before you could commit con1. - """ + # This tests the improved concurrency with pysqlite 2.3.4. You needed + # to roll back con2 before you could commit con1. self.cur1.execute("create table test(i)") self.cur1.execute("insert into test(i) values (5)") with self.assertRaises(sqlite.OperationalError): @@ -132,14 +129,14 @@ def test_locking(self): def test_rollback_cursor_consistency(self): """Check that cursors behave correctly after rollback.""" - con = sqlite.connect(":memory:") - cur = con.cursor() - cur.execute("create table test(x)") - cur.execute("insert into test(x) values (5)") - cur.execute("select 1 union select 2 union select 3") + with memory_database() as con: + cur = con.cursor() + cur.execute("create table test(x)") + cur.execute("insert into test(x) values (5)") + cur.execute("select 1 union select 2 union select 3") - con.rollback() - self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)]) + con.rollback() + self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)]) def test_multiple_cursors_and_iternext(self): # gh-94028: statements are cleared and reset in cursor iternext. @@ -218,10 +215,7 @@ def test_no_duplicate_rows_after_rollback_new_query(self): -class SpecialCommandTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - self.cur = self.con.cursor() +class SpecialCommandTests(MemoryDatabaseMixin, unittest.TestCase): def test_drop_table(self): self.cur.execute("create table test(i)") @@ -233,14 +227,8 @@ def test_pragma(self): self.cur.execute("insert into test(i) values (5)") self.cur.execute("pragma count_changes=1") - def tearDown(self): - self.cur.close() - self.con.close() - -class TransactionalDDL(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class TransactionalDDL(MemoryDatabaseMixin, unittest.TestCase): def test_ddl_does_not_autostart_transaction(self): # For backwards compatibility reasons, DDL statements should not @@ -268,9 +256,6 @@ def test_transactional_ddl(self): with self.assertRaises(sqlite.OperationalError): self.con.execute("select * from test") - def tearDown(self): - self.con.close() - class IsolationLevelFromInit(unittest.TestCase): CREATE = "create table t(t)" @@ -368,5 +353,183 @@ def test_isolation_level_none(self): self.assertEqual(self.traced, [self.QUERY]) +class AutocommitAttribute(unittest.TestCase): + """Test PEP 249-compliant autocommit behaviour.""" + legacy = sqlite.LEGACY_TRANSACTION_CONTROL + + @contextmanager + def check_stmt_trace(self, cx, expected, reset=True): + try: + traced = [] + cx.set_trace_callback(lambda stmt: traced.append(stmt)) + yield + finally: + self.assertEqual(traced, expected) + if reset: + cx.set_trace_callback(None) + + def test_autocommit_default(self): + with memory_database() as cx: + self.assertEqual(cx.autocommit, + sqlite.LEGACY_TRANSACTION_CONTROL) + + def test_autocommit_setget(self): + dataset = ( + True, + False, + sqlite.LEGACY_TRANSACTION_CONTROL, + ) + for mode in dataset: + with self.subTest(mode=mode): + with memory_database(autocommit=mode) as cx: + self.assertEqual(cx.autocommit, mode) + with memory_database() as cx: + cx.autocommit = mode + self.assertEqual(cx.autocommit, mode) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit validation error messages differ + def test_autocommit_setget_invalid(self): + msg = "autocommit must be True, False, or.*LEGACY" + for mode in "a", 12, (), None: + with self.subTest(mode=mode): + with self.assertRaisesRegex(ValueError, msg): + sqlite.connect(":memory:", autocommit=mode) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled(self): + expected = [ + "SELECT 1", + "COMMIT", + "BEGIN", + "ROLLBACK", + "BEGIN", + ] + with memory_database(autocommit=False) as cx: + self.assertTrue(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.execute("SELECT 1") + cx.commit() + cx.rollback() + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled_implicit_rollback(self): + expected = ["ROLLBACK"] + with memory_database(autocommit=False) as cx: + self.assertTrue(cx.in_transaction) + with self.check_stmt_trace(cx, expected, reset=False): + cx.close() + + def test_autocommit_enabled(self): + expected = ["CREATE TABLE t(t)", "INSERT INTO t VALUES(1)"] + with memory_database(autocommit=True) as cx: + self.assertFalse(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.execute("CREATE TABLE t(t)") + cx.execute("INSERT INTO t VALUES(1)") + self.assertFalse(cx.in_transaction) + + def test_autocommit_enabled_txn_ctl(self): + for op in "commit", "rollback": + with self.subTest(op=op): + with memory_database(autocommit=True) as cx: + meth = getattr(cx, op) + self.assertFalse(cx.in_transaction) + with self.check_stmt_trace(cx, []): + meth() # expect this to pass silently + self.assertFalse(cx.in_transaction) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled_then_enabled(self): + expected = ["COMMIT"] + with memory_database(autocommit=False) as cx: + self.assertTrue(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.autocommit = True # should commit + self.assertFalse(cx.in_transaction) + + def test_autocommit_enabled_then_disabled(self): + expected = ["BEGIN"] + with memory_database(autocommit=True) as cx: + self.assertFalse(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.autocommit = False # should begin + self.assertTrue(cx.in_transaction) + + def test_autocommit_explicit_then_disabled(self): + expected = ["BEGIN DEFERRED"] + with memory_database(autocommit=True) as cx: + self.assertFalse(cx.in_transaction) + with self.check_stmt_trace(cx, expected): + cx.execute("BEGIN DEFERRED") + cx.autocommit = False # should now be a no-op + self.assertTrue(cx.in_transaction) + + def test_autocommit_enabled_ctx_mgr(self): + with memory_database(autocommit=True) as cx: + # The context manager is a no-op if autocommit=True + with self.check_stmt_trace(cx, []): + with cx: + self.assertFalse(cx.in_transaction) + self.assertFalse(cx.in_transaction) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled_ctx_mgr(self): + expected = ["COMMIT", "BEGIN"] + with memory_database(autocommit=False) as cx: + with self.check_stmt_trace(cx, expected): + with cx: + self.assertTrue(cx.in_transaction) + self.assertTrue(cx.in_transaction) + + def test_autocommit_compat_ctx_mgr(self): + expected = ["BEGIN ", "INSERT INTO T VALUES(1)", "COMMIT"] + with memory_database(autocommit=self.legacy) as cx: + cx.execute("create table t(t)") + with self.check_stmt_trace(cx, expected): + with cx: + self.assertFalse(cx.in_transaction) + cx.execute("INSERT INTO T VALUES(1)") + self.assertTrue(cx.in_transaction) + self.assertFalse(cx.in_transaction) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_enabled_executescript(self): + expected = ["BEGIN", "SELECT 1"] + with memory_database(autocommit=True) as cx: + with self.check_stmt_trace(cx, expected): + self.assertFalse(cx.in_transaction) + cx.execute("BEGIN") + cx.executescript("SELECT 1") + self.assertTrue(cx.in_transaction) + + @unittest.expectedFailure # TODO: RUSTPYTHON autocommit behavior differs + def test_autocommit_disabled_executescript(self): + expected = ["SELECT 1"] + with memory_database(autocommit=False) as cx: + with self.check_stmt_trace(cx, expected): + self.assertTrue(cx.in_transaction) + cx.executescript("SELECT 1") + self.assertTrue(cx.in_transaction) + + def test_autocommit_compat_executescript(self): + expected = ["BEGIN", "COMMIT", "SELECT 1"] + with memory_database(autocommit=self.legacy) as cx: + with self.check_stmt_trace(cx, expected): + self.assertFalse(cx.in_transaction) + cx.execute("BEGIN") + cx.executescript("SELECT 1") + self.assertFalse(cx.in_transaction) + + def test_autocommit_disabled_implicit_shutdown(self): + # The implicit ROLLBACK should not call back into Python during + # interpreter tear-down. + code = """if 1: + import sqlite3 + cx = sqlite3.connect(":memory:", autocommit=False) + cx.set_trace_callback(print) + """ + assert_python_ok("-c", code, PYTHONIOENCODING="utf-8") + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_types.py b/Lib/test/test_sqlite3/test_types.py index d7631ec9386..66d27d21b8d 100644 --- a/Lib/test/test_sqlite3/test_types.py +++ b/Lib/test/test_sqlite3/test_types.py @@ -95,8 +95,6 @@ def test_too_large_int(self): row = self.cur.fetchone() self.assertIsNone(row) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_string_with_surrogates(self): for value in 0xd8ff, 0xdcff: with self.assertRaises(UnicodeEncodeError): @@ -108,9 +106,9 @@ def test_string_with_surrogates(self): @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @support.bigmemtest(size=2**31, memuse=4, dry_run=False) def test_too_large_string(self, maxsize): - with self.assertRaises(sqlite.InterfaceError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", ('x'*(2**31-1),)) - with self.assertRaises(OverflowError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", ('x'*(2**31),)) self.cur.execute("select 1 from test") row = self.cur.fetchone() @@ -119,9 +117,9 @@ def test_too_large_string(self, maxsize): @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @support.bigmemtest(size=2**31, memuse=3, dry_run=False) def test_too_large_blob(self, maxsize): - with self.assertRaises(sqlite.InterfaceError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", (b'x'*(2**31-1),)) - with self.assertRaises(OverflowError): + with self.assertRaises(sqlite.DataError): self.cur.execute("insert into test(s) values (?)", (b'x'*(2**31),)) self.cur.execute("select 1 from test") row = self.cur.fetchone() @@ -345,8 +343,6 @@ def test_none(self): val = self.cur.fetchone()[0] self.assertEqual(val, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_col_name(self): self.cur.execute("insert into test(x) values (?)", ("xxx",)) self.cur.execute('select x as "x y [bar]" from test') @@ -375,7 +371,6 @@ def test_cursor_description_insert(self): self.assertIsNone(self.cur.description) -@unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "CTEs not supported") class CommonTableExpressionTests(unittest.TestCase): def setUp(self): @@ -437,14 +432,10 @@ def test_missing_adapter(self): with self.assertRaises(sqlite.ProgrammingError): sqlite.adapt(1.) # No float adapter registered - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_missing_protocol(self): with self.assertRaises(sqlite.ProgrammingError): sqlite.adapt(1, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_defect_proto(self): class DefectProto(): def __adapt__(self): @@ -452,8 +443,6 @@ def __adapt__(self): with self.assertRaises(sqlite.ProgrammingError): sqlite.adapt(1., DefectProto) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_defect_self_adapt(self): class DefectSelfAdapt(float): def __conform__(self, _): @@ -527,7 +516,7 @@ def test_sqlite_timestamp(self): self.assertEqual(ts, ts2) def test_sql_timestamp(self): - now = datetime.datetime.utcnow() + now = datetime.datetime.now(tz=datetime.UTC) self.cur.execute("insert into test(ts) values (current_timestamp)") self.cur.execute("select ts from test") with self.assertWarnsRegex(DeprecationWarning, "converter"): diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index d2925feb0f8..3fdde4a26cd 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -21,55 +21,15 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. -import contextlib -import functools -import io -import re import sys import unittest import sqlite3 as sqlite from unittest.mock import Mock, patch -from test.support import bigmemtest, catch_unraisable_exception, gc_collect - -from test.test_sqlite3.test_dbapi import cx_limit - - -def with_tracebacks(exc, regex="", name=""): - """Convenience decorator for testing callback tracebacks.""" - def decorator(func): - _regex = re.compile(regex) if regex else None - @functools.wraps(func) - def wrapper(self, *args, **kwargs): - with catch_unraisable_exception() as cm: - # First, run the test with traceback enabled. - with check_tracebacks(self, cm, exc, _regex, name): - func(self, *args, **kwargs) - - # Then run the test with traceback disabled. - func(self, *args, **kwargs) - return wrapper - return decorator - - -@contextlib.contextmanager -def check_tracebacks(self, cm, exc, regex, obj_name): - """Convenience context manager for testing callback tracebacks.""" - sqlite.enable_callback_tracebacks(True) - try: - buf = io.StringIO() - with contextlib.redirect_stderr(buf): - yield - - # TODO: RUSTPYTHON need unraisable exception - # self.assertEqual(cm.unraisable.exc_type, exc) - # if regex: - # msg = str(cm.unraisable.exc_value) - # self.assertIsNotNone(regex.search(msg)) - # if obj_name: - # self.assertEqual(cm.unraisable.object.__name__, obj_name) - finally: - sqlite.enable_callback_tracebacks(False) +from test.support import bigmemtest, gc_collect + +from .util import cx_limit, memory_database +from .util import with_tracebacks def func_returntext(): @@ -196,7 +156,6 @@ def setUp(self): self.con.create_function("returnblob", 0, func_returnblob) self.con.create_function("returnlonglong", 0, func_returnlonglong) self.con.create_function("returnnan", 0, lambda: float("nan")) - self.con.create_function("returntoolargeint", 0, lambda: 1 << 65) self.con.create_function("return_noncont_blob", 0, lambda: memoryview(b"blob")[::2]) self.con.create_function("raiseexception", 0, func_raiseexception) @@ -211,8 +170,9 @@ def setUp(self): def tearDown(self): self.con.close() + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for invalid num args def test_func_error_on_create(self): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): self.con.create_function("bla", -100, lambda x: 2*x) def test_func_too_many_args(self): @@ -295,12 +255,8 @@ def test_func_return_nan(self): cur.execute("select returnnan()") self.assertIsNone(cur.fetchone()[0]) - def test_func_return_too_large_int(self): - cur = self.con.cursor() - self.assertRaisesRegex(sqlite.DataError, "string or blob too big", - self.con.execute, "select returntoolargeint()") - - @with_tracebacks(ZeroDivisionError, name="func_raiseexception") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="func_raiseexception") def test_func_exception(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -308,14 +264,16 @@ def test_func_exception(self): cur.fetchone() self.assertEqual(str(cm.exception), 'user-defined function raised exception') - @with_tracebacks(MemoryError, name="func_memoryerror") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(MemoryError, msg_regex="func_memoryerror") def test_func_memory_error(self): cur = self.con.cursor() with self.assertRaises(MemoryError): cur.execute("select memoryerror()") cur.fetchone() - @with_tracebacks(OverflowError, name="func_overflowerror") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(OverflowError, msg_regex="func_overflowerror") def test_func_overflow_error(self): cur = self.con.cursor() with self.assertRaises(sqlite.DataError): @@ -337,8 +295,6 @@ def test_nan_float(self): # SQLite has no concept of nan; it is converted to NULL self.assertTrue(cur.fetchone()[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_too_large_int(self): err = "Python int too large to convert to SQLite INTEGER" self.assertRaisesRegex(OverflowError, err, self.con.execute, @@ -350,14 +306,13 @@ def test_non_contiguous_blob(self): self.con.execute, "select spam(?)", (memoryview(b"blob")[::2],)) + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(BufferError, regex="buffer.*contiguous") def test_return_non_contiguous_blob(self): with self.assertRaises(sqlite.OperationalError): cur = self.con.execute("select return_noncont_blob()") cur.fetchone() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_param_surrogates(self): self.assertRaisesRegex(UnicodeEncodeError, "surrogates not allowed", self.con.execute, "select spam(?)", @@ -392,38 +347,22 @@ def append_result(arg): # Regarding deterministic functions: # # Between 3.8.3 and 3.15.0, deterministic functions were only used to - # optimize inner loops, so for those versions we can only test if the - # sqlite machinery has factored out a call or not. From 3.15.0 and onward, - # deterministic functions were permitted in WHERE clauses of partial - # indices, which allows testing based on syntax, iso. the query optimizer. - @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher") + # optimize inner loops. From 3.15.0 and onward, deterministic functions + # were permitted in WHERE clauses of partial indices, which allows testing + # based on syntax, iso. the query optimizer. def test_func_non_deterministic(self): mock = Mock(return_value=None) self.con.create_function("nondeterministic", 0, mock, deterministic=False) - if sqlite.sqlite_version_info < (3, 15, 0): - self.con.execute("select nondeterministic() = nondeterministic()") - self.assertEqual(mock.call_count, 2) - else: - with self.assertRaises(sqlite.OperationalError): - self.con.execute("create index t on test(t) where nondeterministic() is not null") + with self.assertRaises(sqlite.OperationalError): + self.con.execute("create index t on test(t) where nondeterministic() is not null") - @unittest.skipIf(sqlite.sqlite_version_info < (3, 8, 3), "Requires SQLite 3.8.3 or higher") def test_func_deterministic(self): mock = Mock(return_value=None) self.con.create_function("deterministic", 0, mock, deterministic=True) - if sqlite.sqlite_version_info < (3, 15, 0): - self.con.execute("select deterministic() = deterministic()") - self.assertEqual(mock.call_count, 1) - else: - try: - self.con.execute("create index t on test(t) where deterministic() is not null") - except sqlite.OperationalError: - self.fail("Unexpected failure while creating partial index") - - @unittest.skipIf(sqlite.sqlite_version_info >= (3, 8, 3), "SQLite < 3.8.3 needed") - def test_func_deterministic_not_supported(self): - with self.assertRaises(sqlite.NotSupportedError): - self.con.create_function("deterministic", 0, int, deterministic=True) + try: + self.con.execute("create index t on test(t) where deterministic() is not null") + except sqlite.OperationalError: + self.fail("Unexpected failure while creating partial index") def test_func_deterministic_keyword_only(self): with self.assertRaises(TypeError): @@ -432,29 +371,32 @@ def test_func_deterministic_keyword_only(self): def test_function_destructor_via_gc(self): # See bpo-44304: The destructor of the user function can # crash if is called without the GIL from the gc functions - dest = sqlite.connect(':memory:') def md5sum(t): return - dest.create_function("md5", 1, md5sum) - x = dest("create table lang (name, first_appeared)") - del md5sum, dest + with memory_database() as dest: + dest.create_function("md5", 1, md5sum) + x = dest("create table lang (name, first_appeared)") + del md5sum, dest - y = [x] - y.append(y) + y = [x] + y.append(y) - del x,y - gc_collect() + del x,y + gc_collect() + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(OverflowError) def test_func_return_too_large_int(self): cur = self.con.cursor() + msg = "string or blob too big" for value in 2**63, -2**63-1, 2**64: self.con.create_function("largeint", 0, lambda value=value: value) - with self.assertRaises(sqlite.DataError): + with self.assertRaisesRegex(sqlite.DataError, msg): cur.execute("select largeint()") - @with_tracebacks(UnicodeEncodeError, "surrogates not allowed", "chr") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(UnicodeEncodeError, "surrogates not allowed") def test_func_return_text_with_surrogates(self): cur = self.con.cursor() self.con.create_function("pychr", 1, chr) @@ -486,6 +428,30 @@ def test_func_return_illegal_value(self): self.assertRaisesRegex(sqlite.OperationalError, msg, self.con.execute, "select badreturn()") + @unittest.expectedFailure # TODO: RUSTPYTHON deprecation warning not emitted for keyword args + def test_func_keyword_args(self): + regex = ( + r"Passing keyword arguments 'name', 'narg' and 'func' to " + r"_sqlite3.Connection.create_function\(\) is deprecated. " + r"Parameters 'name', 'narg' and 'func' will become " + r"positional-only in Python 3.15." + ) + + def noop(): + return None + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function("noop", 0, func=noop) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function("noop", narg=0, func=noop) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_function(name="noop", narg=0, func=noop) + self.assertEqual(cm.filename, __file__) + class WindowSumInt: def __init__(self): @@ -540,15 +506,20 @@ def setUp(self): """ self.con.create_window_function("sumint", 1, WindowSumInt) + def tearDown(self): + self.cur.close() + self.con.close() + def test_win_sum_int(self): self.cur.execute(self.query % "sumint") self.assertEqual(self.cur.fetchall(), self.expected) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for invalid num args def test_win_error_on_create(self): - self.assertRaises(sqlite.ProgrammingError, - self.con.create_window_function, - "shouldfail", -100, WindowSumInt) + with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): + self.con.create_window_function("shouldfail", -100, WindowSumInt) + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(BadWindow) def test_win_exception_in_method(self): for meth in "__init__", "step", "value", "inverse": @@ -561,17 +532,19 @@ def test_win_exception_in_method(self): self.cur.execute(self.query % name) self.cur.fetchall() + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(BadWindow) def test_win_exception_in_finalize(self): # Note: SQLite does not (as of version 3.38.0) propagate finalize # callback errors to sqlite3_step(); this implies that OperationalError # is _not_ raised. with patch.object(WindowSumInt, "finalize", side_effect=BadWindow): - name = f"exception_in_finalize" + name = "exception_in_finalize" self.con.create_window_function(name, 1, WindowSumInt) self.cur.execute(self.query % name) self.cur.fetchall() + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(AttributeError) def test_win_missing_method(self): class MissingValue: @@ -603,6 +576,7 @@ def finalize(self): return 42 self.cur.execute(self.query % name) self.cur.fetchall() + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented @with_tracebacks(AttributeError) def test_win_missing_finalize(self): # Note: SQLite does not (as of version 3.38.0) propagate finalize @@ -660,6 +634,7 @@ def setUp(self): """) cur.execute("insert into test(t, i, f, n, b) values (?, ?, ?, ?, ?)", ("foo", 5, 3.14, None, memoryview(b"blob"),)) + cur.close() self.con.create_aggregate("nostep", 1, AggrNoStep) self.con.create_aggregate("nofinalize", 1, AggrNoFinalize) @@ -672,15 +647,15 @@ def setUp(self): self.con.create_aggregate("aggtxt", 1, AggrText) def tearDown(self): - #self.cur.close() - #self.con.close() - pass + self.con.close() + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs for invalid num args def test_aggr_error_on_create(self): - with self.assertRaises(sqlite.OperationalError): + with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): self.con.create_function("bla", -100, AggrSum) - @with_tracebacks(AttributeError, name="AggrNoStep") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(AttributeError, msg_regex="AggrNoStep") def test_aggr_no_step(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -695,7 +670,8 @@ def test_aggr_no_finalize(self): cur.execute("select nofinalize(t) from test") val = cur.fetchone()[0] - @with_tracebacks(ZeroDivisionError, name="AggrExceptionInInit") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="AggrExceptionInInit") def test_aggr_exception_in_init(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -703,7 +679,8 @@ def test_aggr_exception_in_init(self): val = cur.fetchone()[0] self.assertEqual(str(cm.exception), "user-defined aggregate's '__init__' method raised error") - @with_tracebacks(ZeroDivisionError, name="AggrExceptionInStep") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="AggrExceptionInStep") def test_aggr_exception_in_step(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -711,7 +688,8 @@ def test_aggr_exception_in_step(self): val = cur.fetchone()[0] self.assertEqual(str(cm.exception), "user-defined aggregate's 'step' method raised error") - @with_tracebacks(ZeroDivisionError, name="AggrExceptionInFinalize") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ZeroDivisionError, msg_regex="AggrExceptionInFinalize") def test_aggr_exception_in_finalize(self): cur = self.con.cursor() with self.assertRaises(sqlite.OperationalError) as cm: @@ -776,6 +754,28 @@ def test_aggr_text(self): val = cur.fetchone()[0] self.assertEqual(val, txt) + @unittest.expectedFailure # TODO: RUSTPYTHON keyword-only arguments not supported for create_aggregate + def test_agg_keyword_args(self): + regex = ( + r"Passing keyword arguments 'name', 'n_arg' and 'aggregate_class' to " + r"_sqlite3.Connection.create_aggregate\(\) is deprecated. " + r"Parameters 'name', 'n_arg' and 'aggregate_class' will become " + r"positional-only in Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate("test", 1, aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate("test", n_arg=1, aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.create_aggregate(name="test", n_arg=0, + aggregate_class=AggrText) + self.assertEqual(cm.filename, __file__) + class AuthorizerTests(unittest.TestCase): @staticmethod @@ -787,8 +787,6 @@ def authorizer_cb(action, arg1, arg2, dbname, source): return sqlite.SQLITE_OK def setUp(self): - # TODO: RUSTPYTHON difference 'prohibited' - self.prohibited = 'not authorized' self.con = sqlite.connect(":memory:") self.con.executescript(""" create table t1 (c1, c2); @@ -803,23 +801,38 @@ def setUp(self): self.con.set_authorizer(self.authorizer_cb) def tearDown(self): - pass + self.con.close() + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs def test_table_access(self): with self.assertRaises(sqlite.DatabaseError) as cm: self.con.execute("select * from t2") - self.assertIn(self.prohibited, str(cm.exception)) + self.assertIn('prohibited', str(cm.exception)) + @unittest.expectedFailure # TODO: RUSTPYTHON error message differs def test_column_access(self): with self.assertRaises(sqlite.DatabaseError) as cm: self.con.execute("select c2 from t1") - self.assertIn(self.prohibited, str(cm.exception)) + self.assertIn('prohibited', str(cm.exception)) def test_clear_authorizer(self): self.con.set_authorizer(None) self.con.execute("select * from t2") self.con.execute("select c2 from t1") + @unittest.expectedFailure # TODO: RUSTPYTHON keyword-only arguments not supported for set_authorizer + def test_authorizer_keyword_args(self): + regex = ( + r"Passing keyword argument 'authorizer_callback' to " + r"_sqlite3.Connection.set_authorizer\(\) is deprecated. " + r"Parameter 'authorizer_callback' will become positional-only in " + r"Python 3.15." + ) + + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + self.con.set_authorizer(authorizer_callback=lambda: None) + self.assertEqual(cm.filename, __file__) + class AuthorizerRaiseExceptionTests(AuthorizerTests): @staticmethod @@ -830,11 +843,13 @@ def authorizer_cb(action, arg1, arg2, dbname, source): raise ValueError return sqlite.SQLITE_OK - @with_tracebacks(ValueError, name="authorizer_cb") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ValueError, msg_regex="authorizer_cb") def test_table_access(self): super().test_table_access() - @with_tracebacks(ValueError, name="authorizer_cb") + @unittest.expectedFailure # TODO: RUSTPYTHON unraisable exception handling not implemented + @with_tracebacks(ValueError, msg_regex="authorizer_cb") def test_column_access(self): super().test_table_access() diff --git a/Lib/test/test_sqlite3/util.py b/Lib/test/test_sqlite3/util.py new file mode 100644 index 00000000000..cccd062160f --- /dev/null +++ b/Lib/test/test_sqlite3/util.py @@ -0,0 +1,89 @@ +import contextlib +import functools +import io +import re +import sqlite3 +import test.support +import unittest + + +# Helper for temporary memory databases +def memory_database(*args, **kwargs): + cx = sqlite3.connect(":memory:", *args, **kwargs) + return contextlib.closing(cx) + + +# Temporarily limit a database connection parameter +@contextlib.contextmanager +def cx_limit(cx, category=sqlite3.SQLITE_LIMIT_SQL_LENGTH, limit=128): + try: + _prev = cx.setlimit(category, limit) + yield limit + finally: + cx.setlimit(category, _prev) + + +def with_tracebacks(exc, regex="", name="", msg_regex=""): + """Convenience decorator for testing callback tracebacks.""" + def decorator(func): + exc_regex = re.compile(regex) if regex else None + _msg_regex = re.compile(msg_regex) if msg_regex else None + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + with test.support.catch_unraisable_exception() as cm: + # First, run the test with traceback enabled. + with check_tracebacks(self, cm, exc, exc_regex, _msg_regex, name): + func(self, *args, **kwargs) + + # Then run the test with traceback disabled. + func(self, *args, **kwargs) + return wrapper + return decorator + + +@contextlib.contextmanager +def check_tracebacks(self, cm, exc, exc_regex, msg_regex, obj_name): + """Convenience context manager for testing callback tracebacks.""" + sqlite3.enable_callback_tracebacks(True) + try: + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + yield + + self.assertEqual(cm.unraisable.exc_type, exc) + if exc_regex: + msg = str(cm.unraisable.exc_value) + self.assertIsNotNone(exc_regex.search(msg), (exc_regex, msg)) + if msg_regex: + msg = cm.unraisable.err_msg + self.assertIsNotNone(msg_regex.search(msg), (msg_regex, msg)) + if obj_name: + self.assertEqual(cm.unraisable.object.__name__, obj_name) + finally: + sqlite3.enable_callback_tracebacks(False) + + +class MemoryDatabaseMixin: + + def setUp(self): + self.con = sqlite3.connect(":memory:") + self.cur = self.con.cursor() + + def tearDown(self): + self.cur.close() + self.con.close() + + @property + def cx(self): + return self.con + + @property + def cu(self): + return self.cur + + +def requires_virtual_table(module): + with memory_database() as cx: + supported = (module,) in list(cx.execute("PRAGMA module_list")) + reason = f"Requires {module!r} virtual table support" + return unittest.skipUnless(supported, reason) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py new file mode 100644 index 00000000000..7cfbe0c97dc --- /dev/null +++ b/Lib/test/test_ssl.py @@ -0,0 +1,5531 @@ +# Test the support for SSL and sockets + +import sys +import unittest +import unittest.mock +from ast import literal_eval +from threading import Thread +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import socket_helper +from test.support import threading_helper +from test.support import warnings_helper +from test.support import asyncore +import array +import re +import socket +import select +import struct +import time +import enum +import gc +import http.client +import os +import errno +import pprint +import urllib.request +import threading +import traceback +import weakref +import platform +import sysconfig +import functools +from contextlib import nullcontext +try: + import ctypes +except ImportError: + ctypes = None + + +ssl = import_helper.import_module("ssl") +import _ssl + +from ssl import Purpose, TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType + +Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = socket_helper.HOST +IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) +PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') + +PROTOCOL_TO_TLS_VERSION = {} +for proto, ver in ( + ("PROTOCOL_SSLv3", "SSLv3"), + ("PROTOCOL_TLSv1", "TLSv1"), + ("PROTOCOL_TLSv1_1", "TLSv1_1"), +): + try: + proto = getattr(ssl, proto) + ver = getattr(ssl.TLSVersion, ver) + except AttributeError: + continue + PROTOCOL_TO_TLS_VERSION[proto] = ver + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), "certdata", *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/certdata/make_ssl_certs.py. +# Other certificates are simply fetched from the internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = os.fsencode(CERTFILE) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = os.fsencode(ONLYCERT) +BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = os.fsencode(CAPATH) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") + +with open(data_file('keycert.pem.reference')) as file: + CERTFILE_INFO = literal_eval(file.read()) + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SINGED_CERTFILE_ONLY = data_file("cert3.pem") +SIGNED_CERTFILE_HOSTNAME = 'localhost' + +with open(data_file('keycert3.pem.reference')) as file: + SIGNED_CERTFILE_INFO = literal_eval(file.read()) + +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNED_CERTFILE2_HOSTNAME = 'fakehostname' +SIGNED_CERTFILE_ECC = data_file("keycertecc.pem") +SIGNED_CERTFILE_ECC_HOSTNAME = 'localhost-ecc' + +# A custom testcase, extracted from `rfc5280::aki::leaf-missing-aki` in x509-limbo: +# The leaf (server) certificate has no AKI, which is forbidden under RFC 5280. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +LEAF_MISSING_AKI_CERTFILE = data_file("leaf-missing-aki.keycert.pem") +LEAF_MISSING_AKI_CERTFILE_HOSTNAME = "example.com" +LEAF_MISSING_AKI_CA = data_file("leaf-missing-aki.ca.pem") + +# Same certificate as pycacert.pem, but without extra text in file +SIGNING_CA = data_file("capath", "ceff1710.0") +# cert with all kinds of subject alt names +ALLSANFILE = data_file("allsans.pem") +IDNSANSFILE = data_file("idnsans.pem") +NOSANFILE = data_file("nosan.pem") +NOSAN_HOSTNAME = 'localhost' + +REMOTE_HOST = "self-signed.pythontest.net" + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +NONEXISTINGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") +TALOS_INVALID_CRLDP = data_file("talos-2019-0758.pem") + +DHFILE = data_file("ffdh3072.pem") +BYTES_DHFILE = os.fsencode(DHFILE) + +# Not defined in all versions of OpenSSL +OP_NO_COMPRESSION = getattr(ssl, "OP_NO_COMPRESSION", 0) +OP_SINGLE_DH_USE = getattr(ssl, "OP_SINGLE_DH_USE", 0) +OP_SINGLE_ECDH_USE = getattr(ssl, "OP_SINGLE_ECDH_USE", 0) +OP_CIPHER_SERVER_PREFERENCE = getattr(ssl, "OP_CIPHER_SERVER_PREFERENCE", 0) +OP_ENABLE_MIDDLEBOX_COMPAT = getattr(ssl, "OP_ENABLE_MIDDLEBOX_COMPAT", 0) + +# Ubuntu has patched OpenSSL and changed behavior of security level 2 +# see https://bugs.python.org/issue41561#msg389003 +def is_ubuntu(): + try: + # Assume that any references of "ubuntu" implies Ubuntu-like distro + # The workaround is not required for 18.04, but doesn't hurt either. + with open("/etc/os-release", encoding="utf-8") as f: + return "ubuntu" in f.read() + except FileNotFoundError: + return False + +if is_ubuntu(): + def seclevel_workaround(*ctxs): + """Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + for ctx in ctxs: + if ( + hasattr(ctx, "minimum_version") and + ctx.minimum_version <= ssl.TLSVersion.TLSv1_1 and + ctx.security_level > 1 + ): + ctx.set_ciphers("@SECLEVEL=1:ALL") +else: + def seclevel_workaround(*ctxs): + pass + + +def has_tls_protocol(protocol): + """Check if a TLS protocol is available and enabled + + :param protocol: enum ssl._SSLMethod member or name + :return: bool + """ + if isinstance(protocol, str): + assert protocol.startswith('PROTOCOL_') + protocol = getattr(ssl, protocol, None) + if protocol is None: + return False + if protocol in { + ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS_SERVER, + ssl.PROTOCOL_TLS_CLIENT + }: + # auto-negotiate protocols are always available + return True + name = protocol.name + return has_tls_version(name[len('PROTOCOL_'):]) + + +@functools.lru_cache +def has_tls_version(version): + """Check if a TLS/SSL version is enabled + + :param version: TLS version name or ssl.TLSVersion member + :return: bool + """ + if isinstance(version, str): + version = ssl.TLSVersion.__members__[version] + + # check compile time flags like ssl.HAS_TLSv1_2 + if not getattr(ssl, f'HAS_{version.name}'): + return False + + if IS_OPENSSL_3_0_0 and version < ssl.TLSVersion.TLSv1_2: + # bpo43791: 3.0.0-alpha14 fails with TLSV1_ALERT_INTERNAL_ERROR + return False + + # check runtime and dynamic crypto policy settings. A TLS version may + # be compiled in but disabled by a policy or config option. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + if ( + hasattr(ctx, 'minimum_version') and + ctx.minimum_version != ssl.TLSVersion.MINIMUM_SUPPORTED and + version < ctx.minimum_version + ): + return False + if ( + hasattr(ctx, 'maximum_version') and + ctx.maximum_version != ssl.TLSVersion.MAXIMUM_SUPPORTED and + version > ctx.maximum_version + ): + return False + + return True + + +def requires_tls_version(version): + """Decorator to skip tests when a required TLS version is not available + + :param version: TLS version name or ssl.TLSVersion member + :return: + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if not has_tls_version(version): + raise unittest.SkipTest(f"{version} is not available.") + else: + return func(*args, **kw) + return wrapper + return decorator + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(sys.exception())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + + +def utc_offset(): #NOTE: ignore issues like #1647654 + # local time = utc time + utc offset + if time.daylight and time.localtime().tm_isdst > 0: + return -time.altzone # seconds + return -time.timezone + + +ignore_deprecation = warnings_helper.ignore_warnings( + category=DeprecationWarning +) + + +def test_wrap_socket(sock, *, + cert_reqs=ssl.CERT_NONE, ca_certs=None, + ciphers=None, certfile=None, keyfile=None, + **kwargs): + if not kwargs.get("server_side"): + kwargs["server_hostname"] = SIGNED_CERTFILE_HOSTNAME + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + else: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + if cert_reqs is not None: + if cert_reqs == ssl.CERT_NONE: + context.check_hostname = False + context.verify_mode = cert_reqs + if ca_certs is not None: + context.load_verify_locations(ca_certs) + if certfile is not None or keyfile is not None: + context.load_cert_chain(certfile, keyfile) + if ciphers is not None: + context.set_ciphers(ciphers) + return context.wrap_socket(sock, **kwargs) + + +USE_SAME_TEST_CONTEXT = False +_TEST_CONTEXT = None + +def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): + """Create context + + client_context, server_context, hostname = testing_context() + """ + global _TEST_CONTEXT + if USE_SAME_TEST_CONTEXT: + if _TEST_CONTEXT is not None: + return _TEST_CONTEXT + + if server_cert == SIGNED_CERTFILE: + hostname = SIGNED_CERTFILE_HOSTNAME + elif server_cert == SIGNED_CERTFILE2: + hostname = SIGNED_CERTFILE2_HOSTNAME + elif server_cert == NOSANFILE: + hostname = NOSAN_HOSTNAME + else: + raise ValueError(server_cert) + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(server_cert) + if server_chain: + server_context.load_verify_locations(SIGNING_CA) + + if USE_SAME_TEST_CONTEXT: + if _TEST_CONTEXT is not None: + _TEST_CONTEXT = client_context, server_context, hostname + + return client_context, server_context, hostname + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + ssl.OP_SINGLE_ECDH_USE + ssl.OP_NO_COMPRESSION + self.assertEqual(ssl.HAS_SNI, True) + self.assertEqual(ssl.HAS_ECDH, True) + self.assertEqual(ssl.HAS_TLSv1_2, True) + self.assertEqual(ssl.HAS_TLSv1_3, True) + ssl.OP_NO_SSLv2 + ssl.OP_NO_SSLv3 + ssl.OP_NO_TLSv1 + ssl.OP_NO_TLSv1_3 + ssl.OP_NO_TLSv1_1 + ssl.OP_NO_TLSv1_2 + self.assertEqual(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv23) + + def test_options(self): + # gh-106687: SSL options values are unsigned integer (uint64_t) + for name in dir(ssl): + if not name.startswith('OP_'): + continue + with self.subTest(option=name): + value = getattr(ssl, name) + self.assertGreaterEqual(value, 0, f"ssl.{name}") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by Certificate + def test_ssl_types(self): + ssl_types = [ + _ssl._SSLContext, + _ssl._SSLSocket, + _ssl.MemoryBIO, + _ssl.Certificate, + _ssl.SSLSession, + _ssl.SSLError, + ] + for ssl_type in ssl_types: + with self.subTest(ssl_type=ssl_type): + with self.assertRaisesRegex(TypeError, "immutable type"): + ssl_type.value = None + support.check_disallow_instantiation(self, _ssl.Certificate) + + def test_private_init(self): + with self.assertRaisesRegex(TypeError, "public constructor"): + with socket.socket() as s: + ssl.SSLSocket(s) + + def test_str_for_enums(self): + # Make sure that the PROTOCOL_* constants have enum-like string + # reprs. + proto = ssl.PROTOCOL_TLS_CLIENT + self.assertEqual(repr(proto), '<_SSLMethod.PROTOCOL_TLS_CLIENT: %r>' % proto.value) + self.assertEqual(str(proto), str(proto.value)) + ctx = ssl.SSLContext(proto) + self.assertIs(ctx.protocol, proto) + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + + # negative num is invalid + self.assertRaises(ValueError, ssl.RAND_bytes, -5) + + ssl.RAND_add("this is a random string", 75.0) + ssl.RAND_add(b"this is a random bytes object", 75.0) + ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) + + def test_parse_cert(self): + self.maxDiff = None + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + self.assertEqual( + ssl._ssl._test_decode_cert(CERTFILE), + CERTFILE_INFO + ) + self.assertEqual( + ssl._ssl._test_decode_cert(SIGNED_CERTFILE), + SIGNED_CERTFILE_INFO + ) + + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2019_5010(self): + p = ssl._ssl._test_decode_cert(TALOS_INVALID_CRLDP) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual( + p, + { + 'issuer': ( + (('countryName', 'UK'),), (('commonName', 'cody-ca'),)), + 'notAfter': 'Jun 14 18:00:58 2028 GMT', + 'notBefore': 'Jun 18 18:00:58 2018 GMT', + 'serialNumber': '02', + 'subject': ((('countryName', 'UK'),), + (('commonName', + 'codenomicon-vm-2.test.lal.cisco.com'),)), + 'subjectAltName': ( + ('DNS', 'codenomicon-vm-2.test.lal.cisco.com'),), + 'version': 3 + } + ) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '<invalid>')) + + self.assertEqual(p['subjectAltName'], san) + + def test_parse_all_sans(self): + p = ssl._ssl._test_decode_cert(ALLSANFILE) + self.assertEqual(p['subjectAltName'], + ( + ('DNS', 'allsans'), + ('othername', '<unsupported>'), + ('othername', '<unsupported>'), + ('email', 'user@example.org'), + ('DNS', 'www.example.org'), + ('DirName', + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'dirname example'),))), + ('URI', 'https://www.python.org/'), + ('IP Address', '127.0.0.1'), + ('IP Address', '0:0:0:0:0:0:0:1'), + ('Registered ID', '1.2.3.4.5') + ) + ) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, int) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 1.1.1 + self.assertGreaterEqual(n, 0x10101000) + # < 4.0 + self.assertLess(n, 0x40000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 1) + self.assertLess(major, 4) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + + libressl_ver = f"LibreSSL {major:d}" + if major >= 3: + # 3.x uses 0xMNN00PP0L + openssl_ver = f"OpenSSL {major:d}.{minor:d}.{patch:d}" + else: + openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" + self.assertStartsWith( + s, (openssl_ver, libressl_ver, "AWS-LC"), + (t, hex(n)) + ) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = test_wrap_socket(s) + wr = weakref.ref(ss) + with warnings_helper.check_warnings(("", ResourceWarning)): + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # OSError raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s) as ss: + self.assertRaises(OSError, ss.recv, 1) + self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) + self.assertRaises(OSError, ss.recvfrom, 1) + self.assertRaises(OSError, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(OSError, ss.send, b'x') + self.assertRaises(OSError, ss.sendto, b'x', ('0.0.0.0', 0)) + self.assertRaises(NotImplementedError, ss.dup) + self.assertRaises(NotImplementedError, ss.sendmsg, + [b'x'], (), 0, ('0.0.0.0', 0)) + self.assertRaises(NotImplementedError, ss.recvmsg, 100) + self.assertRaises(NotImplementedError, ss.recvmsg_into, + [bytearray(100)]) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with test_wrap_socket(s) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_openssl111_deprecations(self): + options = [ + ssl.OP_NO_TLSv1, + ssl.OP_NO_TLSv1_1, + ssl.OP_NO_TLSv1_2, + ssl.OP_NO_TLSv1_3 + ] + protocols = [ + ssl.PROTOCOL_TLSv1, + ssl.PROTOCOL_TLSv1_1, + ssl.PROTOCOL_TLSv1_2, + ssl.PROTOCOL_TLS + ] + versions = [ + ssl.TLSVersion.SSLv3, + ssl.TLSVersion.TLSv1, + ssl.TLSVersion.TLSv1_1, + ] + + for option in options: + with self.subTest(option=option): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertWarns(DeprecationWarning) as cm: + ctx.options |= option + self.assertEqual( + 'ssl.OP_NO_SSL*/ssl.OP_NO_TLS* options are deprecated', + str(cm.warning) + ) + + for protocol in protocols: + if not has_tls_protocol(protocol): + continue + with self.subTest(protocol=protocol): + with self.assertWarns(DeprecationWarning) as cm: + ssl.SSLContext(protocol) + self.assertEqual( + f'ssl.{protocol.name} is deprecated', + str(cm.warning) + ) + + for version in versions: + if not has_tls_version(version): + continue + with self.subTest(version=version): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertWarns(DeprecationWarning) as cm: + ctx.minimum_version = version + version_text = '%s.%s' % (version.__class__.__name__, version.name) + self.assertEqual( + f'ssl.{version_text} is deprecated', + str(cm.warning) + ) + + def bad_cert_test(self, certfile): + """Check that trying to use the given client certificate fails""" + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + "certdata", certfile) + sock = socket.socket() + self.addCleanup(sock.close) + with self.assertRaises(ssl.SSLError): + test_wrap_socket(sock, + certfile=certfile) + + def test_empty_cert(self): + """Wrapping with an empty cert file""" + self.bad_cert_test("nullcert.pem") + + def test_malformed_cert(self): + """Wrapping with a badly formatted certificate (syntax error)""" + self.bad_cert_test("badcert.pem") + + def test_malformed_key(self): + """Wrapping with a badly formatted key (syntax error)""" + self.bad_cert_test("badkey.pem") + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + with socket.socket() as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.create_server(('127.0.0.1', 0)) + c = socket.socket(socket.AF_INET) + c.connect(s.getsockname()) + with test_wrap_socket(c, do_handshake_on_connect=False) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + s.close() + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "<ssl.SSLSocket fd=3, family=2, type=1, proto=0, laddr=('0.0.0.0', 0)>" not found in "unclosed <socket.socket fd=3, family=2, type=1, proto=0, laddr=('0.0.0.0', 0)>" + def test_dealloc_warn(self): + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + r = repr(ss) + with self.assertWarns(ResourceWarning) as cm: + ss = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with os_helper.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (frozenset, set, bool)) + if isinstance(trust, (frozenset, set)): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegex(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegex(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + test_wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + def cert_time_ok(self, timestring, timestamp): + self.assertEqual(ssl.cert_time_to_seconds(timestring), timestamp) + + def cert_time_fail(self, timestring): + with self.assertRaises(ValueError): + ssl.cert_time_to_seconds(timestring) + + @unittest.skipUnless(utc_offset(), + 'local time needs to be different from UTC') + def test_cert_time_to_seconds_timezone(self): + # Issue #19940: ssl.cert_time_to_seconds() returns wrong + # results if local timezone is not UTC + self.cert_time_ok("May 9 00:00:00 2007 GMT", 1178668800.0) + self.cert_time_ok("Jan 5 09:34:43 2018 GMT", 1515144883.0) + + def test_cert_time_to_seconds(self): + timestring = "Jan 5 09:34:43 2018 GMT" + ts = 1515144883.0 + self.cert_time_ok(timestring, ts) + # accept keyword parameter, assert its name + self.assertEqual(ssl.cert_time_to_seconds(cert_time=timestring), ts) + # accept both %e and %d (space or zero generated by strftime) + self.cert_time_ok("Jan 05 09:34:43 2018 GMT", ts) + # case-insensitive + self.cert_time_ok("JaN 5 09:34:43 2018 GmT", ts) + self.cert_time_fail("Jan 5 09:34 2018 GMT") # no seconds + self.cert_time_fail("Jan 5 09:34:43 2018") # no GMT + self.cert_time_fail("Jan 5 09:34:43 2018 UTC") # not GMT timezone + self.cert_time_fail("Jan 35 09:34:43 2018 GMT") # invalid day + self.cert_time_fail("Jon 5 09:34:43 2018 GMT") # invalid month + self.cert_time_fail("Jan 5 24:00:00 2018 GMT") # invalid hour + self.cert_time_fail("Jan 5 09:60:43 2018 GMT") # invalid minute + + newyear_ts = 1230768000.0 + # leap seconds + self.cert_time_ok("Dec 31 23:59:60 2008 GMT", newyear_ts) + # same timestamp + self.cert_time_ok("Jan 1 00:00:00 2009 GMT", newyear_ts) + + self.cert_time_ok("Jan 5 09:34:59 2018 GMT", 1515144899) + # allow 60th second (even if it is not a leap second) + self.cert_time_ok("Jan 5 09:34:60 2018 GMT", 1515144900) + # allow 2nd leap second for compatibility with time.strptime() + self.cert_time_ok("Jan 5 09:34:61 2018 GMT", 1515144901) + self.cert_time_fail("Jan 5 09:34:62 2018 GMT") # invalid seconds + + # no special treatment for the special value: + # 99991231235959Z (rfc 5280) + self.cert_time_ok("Dec 31 23:59:59 9999 GMT", 253402300799.0) + + @support.run_with_locale('LC_ALL', '') + def test_cert_time_to_seconds_locale(self): + # `cert_time_to_seconds()` should be locale independent + + def local_february_name(): + return time.strftime('%b', (1, 2, 3, 4, 5, 6, 0, 0, 0)) + + if local_february_name().lower() == 'feb': + self.skipTest("locale-specific month name needs to be " + "different from C locale") + + # locale-independent + self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) + self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + + def test_connect_ex_error(self): + server = socket.socket(socket.AF_INET) + self.addCleanup(server.close) + port = socket_helper.bind_port(server) # Reserve port but don't listen + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + rc = s.connect_ex((HOST, port)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + + def test_read_write_zero(self): + # empty reads and writes now work, bpo-42854, bpo-31711 + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.send(b""), 0) + + +class ContextTests(unittest.TestCase): + + def test_constructor(self): + for protocol in PROTOCOLS: + if has_tls_protocol(protocol): + with warnings_helper.check_warnings(): + ctx = ssl.SSLContext(protocol) + self.assertEqual(ctx.protocol, protocol) + with warnings_helper.check_warnings(): + ctx = ssl.SSLContext() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @unittest.skipUnless(PY_SSL_DEFAULT_CIPHERS == 1, + "Test applies only to Python default ciphers") + def test_python_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ciphers = ctx.get_ciphers() + for suite in ciphers: + name = suite['name'] + self.assertNotIn("PSK", name) + self.assertNotIn("SRP", name) + self.assertNotIn("MD5", name) + self.assertNotIn("RC4", name) + self.assertNotIn("3DES", name) + + def test_get_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_ciphers('AESGCM') + names = set(d['name'] for d in ctx.get_ciphers()) + expected = { + 'AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'DHE-RSA-AES128-GCM-SHA256', + 'AES256-GCM-SHA384', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'DHE-RSA-AES256-GCM-SHA384', + } + intersection = names.intersection(expected) + self.assertGreaterEqual( + len(intersection), 2, f"\ngot: {sorted(names)}\nexpected: {sorted(expected)}" + ) + + def test_options(self): + # Test default SSLContext options + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + # SSLContext also enables these by default + default |= (OP_NO_COMPRESSION | OP_CIPHER_SERVER_PREFERENCE | + OP_SINGLE_DH_USE | OP_SINGLE_ECDH_USE | + OP_ENABLE_MIDDLEBOX_COMPAT) + self.assertEqual(default, ctx.options) + + # disallow TLSv1 + with warnings_helper.check_warnings(): + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + + # allow TLSv1 + with warnings_helper.check_warnings(): + ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) + self.assertEqual(default, ctx.options) + + # clear all options + ctx.options = 0 + # Ubuntu has OP_NO_SSLv3 forced on by default + self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + + # invalid options + with self.assertRaises(ValueError): + ctx.options = -1 + with self.assertRaises(OverflowError): + ctx.options = 2 ** 100 + with self.assertRaises(TypeError): + ctx.options = "abc" + + def test_verify_mode_protocol(self): + with warnings_helper.check_warnings(): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + + def test_hostname_checks_common_name(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.hostname_checks_common_name) + if ssl.HAS_NEVER_CHECK_COMMON_NAME: + ctx.hostname_checks_common_name = True + self.assertTrue(ctx.hostname_checks_common_name) + ctx.hostname_checks_common_name = False + self.assertFalse(ctx.hostname_checks_common_name) + ctx.hostname_checks_common_name = True + self.assertTrue(ctx.hostname_checks_common_name) + else: + with self.assertRaises(AttributeError): + ctx.hostname_checks_common_name = True + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <TLSVersion.TLSv1_2: 771> not found in {<TLSVersion.TLSv1: 769>, <TLSVersion.TLSv1_1: 770>, <TLSVersion.SSLv3: 768>} + @ignore_deprecation + def test_min_max_version(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like + # Fedora override the setting to TLS 1.0. + minimum_range = { + # stock OpenSSL + ssl.TLSVersion.MINIMUM_SUPPORTED, + # Fedora 29 uses TLS 1.0 by default + ssl.TLSVersion.TLSv1, + # RHEL 8 uses TLS 1.2 by default + ssl.TLSVersion.TLSv1_2 + } + maximum_range = { + # stock OpenSSL + ssl.TLSVersion.MAXIMUM_SUPPORTED, + # Fedora 32 uses TLS 1.3 by default + ssl.TLSVersion.TLSv1_3 + } + + self.assertIn( + ctx.minimum_version, minimum_range + ) + self.assertIn( + ctx.maximum_version, maximum_range + ) + + ctx.minimum_version = ssl.TLSVersion.TLSv1_1 + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + self.assertEqual( + ctx.minimum_version, ssl.TLSVersion.TLSv1_1 + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.TLSv1_2 + ) + + ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + ctx.maximum_version = ssl.TLSVersion.TLSv1 + self.assertEqual( + ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.TLSv1 + ) + + ctx.maximum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED + ) + + ctx.maximum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + self.assertIn( + ctx.maximum_version, + {ssl.TLSVersion.TLSv1, ssl.TLSVersion.TLSv1_1, ssl.TLSVersion.SSLv3} + ) + + ctx.minimum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED + self.assertIn( + ctx.minimum_version, + {ssl.TLSVersion.TLSv1_2, ssl.TLSVersion.TLSv1_3} + ) + + with self.assertRaises(ValueError): + ctx.minimum_version = 42 + + if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) + + self.assertIn( + ctx.minimum_version, minimum_range + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED + ) + with self.assertRaises(ValueError): + ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + with self.assertRaises(ValueError): + ctx.maximum_version = ssl.TLSVersion.TLSv1 + + @unittest.skipUnless( + hasattr(ssl.SSLContext, 'security_level'), + "requires OpenSSL >= 1.1.0" + ) + def test_security_level(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + # The default security callback allows for levels between 0-5 + # with OpenSSL defaulting to 1, however some vendors override the + # default value (e.g. Debian defaults to 2) + security_level_range = { + 0, + 1, # OpenSSL default + 2, # Debian + 3, + 4, + 5, + } + self.assertIn(ctx.security_level, security_level_range) + + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + ctx.verify_flags = ssl.VERIFY_ALLOW_PROXY_CERTS + self.assertEqual(ctx.verify_flags, ssl.VERIFY_ALLOW_PROXY_CERTS) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + key values mismatch # OpenSSL + | + KEY_VALUES_MISMATCH # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.SSLError, regex): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + @threading_helper.requires_working_threading() + def test_load_cert_chain_thread_safety(self): + # gh-134698: _ssl detaches the thread state (and as such, + # releases the GIL and critical sections) around expensive + # OpenSSL calls. Unfortunately, OpenSSL structures aren't + # thread-safe, so executing these calls concurrently led + # to crashes. + ctx = ssl.create_default_context() + + def race(): + ctx.load_cert_chain(CERTFILE) + + threads = [threading.Thread(target=race) for _ in range(8)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(threads): + pass + + self.assertIsNone(cm.exc_value) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read() + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read() + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegex( + ssl.SSLError, + "no start line: cadata does not contain a certificate" + ): + ctx.load_verify_locations(cadata="broken") + with self.assertRaisesRegex( + ssl.SSLError, + "not enough data: cadata does not contain a certificate" + ): + ctx.load_verify_locations(cadata=b"broken") + with self.assertRaises(ssl.SSLError): + ctx.load_verify_locations(cadata=cacert_der + b"A") + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + try: + ctx.load_dh_params(DHFILE) + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(FileNotFoundError) as cm: + ctx.load_dh_params(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + def test_session_stats(self): + for proto in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <SSLContext(protocol=17)> is not None + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': 'Mar 29 12:29:49 2033 GMT', + 'notBefore': 'Mar 30 12:29:49 2003 GMT', + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with os_helper.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + @unittest.skipIf(support.Py_DEBUG, + "Debug build does not share environment between CRTs") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with os_helper.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def _assert_context_options(self, ctx): + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + if OP_NO_COMPRESSION != 0: + self.assertEqual(ctx.options & OP_NO_COMPRESSION, + OP_NO_COMPRESSION) + if OP_SINGLE_DH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_DH_USE, + OP_SINGLE_DH_USE) + if OP_SINGLE_ECDH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_ECDH_USE, + OP_SINGLE_ECDH_USE) + if OP_CIPHER_SERVER_PREFERENCE != 0: + self.assertEqual(ctx.options & OP_CIPHER_SERVER_PREFERENCE, + OP_CIPHER_SERVER_PREFERENCE) + self.assertEqual(ctx.options & ssl.OP_LEGACY_SERVER_CONNECT, + 0 if IS_OPENSSL_3_0_0 else ssl.OP_LEGACY_SERVER_CONNECT) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_PARTIAL_CHAIN, + ssl.VERIFY_X509_PARTIAL_CHAIN) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_STRICT, + ssl.VERIFY_X509_STRICT) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) + + with open(SIGNING_CA) as f: + cadata = f.read() + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self._assert_context_options(ctx) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self._assert_context_options(ctx) + + if has_tls_protocol(ssl.PROTOCOL_TLSv1): + with warnings_helper.check_warnings(): + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + with warnings_helper.check_warnings(): + ctx = ssl._create_stdlib_context( + ssl.PROTOCOL_TLSv1_2, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True + ) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1_2) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + def test_check_hostname(self): + with warnings_helper.check_warnings(): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + # Auto set CERT_REQUIRED + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + # Changing verify_mode does not affect check_hostname + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + # Auto set + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + # keep CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + def test_context_client_server(self): + # PROTOCOL_TLS_CLIENT has sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + # PROTOCOL_TLS_SERVER has different but also sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + def test_context_custom_class(self): + class MySSLSocket(ssl.SSLSocket): + pass + + class MySSLObject(ssl.SSLObject): + pass + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.sslsocket_class = MySSLSocket + ctx.sslobject_class = MySSLObject + + with ctx.wrap_socket(socket.socket(), server_side=True) as sock: + self.assertIsInstance(sock, MySSLSocket) + obj = ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), server_side=True) + self.assertIsInstance(obj, MySSLObject) + + def test_num_tickest(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.num_tickets, 2) + ctx.num_tickets = 1 + self.assertEqual(ctx.num_tickets, 1) + ctx.num_tickets = 0 + self.assertEqual(ctx.num_tickets, 0) + with self.assertRaises(ValueError): + ctx.num_tickets = -1 + with self.assertRaises(TypeError): + ctx.num_tickets = None + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.num_tickets, 2) + with self.assertRaises(ValueError): + ctx.num_tickets = 1 + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + try: + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + + self.assertEqual(cm.exception.library, 'PEM') + regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" + self.assertRegex(cm.exception.reason, regex) + s = str(cm.exception) + self.assertIn("NO_START_LINE", s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with socket.create_server(("127.0.0.1", 0)) as s: + c = socket.create_connection(s.getsockname()) + c.setblocking(False) + with ctx.wrap_socket(c, False, do_handshake_on_connect=False) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertStartsWith(s, "The operation did not complete (read)") + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + + def test_bad_server_hostname(self): + ctx = ssl.create_default_context() + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="") + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname=".example.org") + with self.assertRaises(TypeError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="example.org\x00evil.com") + + +class MemoryBIOTests(unittest.TestCase): + + def test_read_write(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + self.assertEqual(bio.read(), b'') + bio.write(b'foo') + bio.write(b'bar') + self.assertEqual(bio.read(), b'foobar') + self.assertEqual(bio.read(), b'') + bio.write(b'baz') + self.assertEqual(bio.read(2), b'ba') + self.assertEqual(bio.read(1), b'z') + self.assertEqual(bio.read(1), b'') + + def test_eof(self): + bio = ssl.MemoryBIO() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertFalse(bio.eof) + bio.write(b'foo') + self.assertFalse(bio.eof) + bio.write_eof() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(2), b'fo') + self.assertFalse(bio.eof) + self.assertEqual(bio.read(1), b'o') + self.assertTrue(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertTrue(bio.eof) + + def test_pending(self): + bio = ssl.MemoryBIO() + self.assertEqual(bio.pending, 0) + bio.write(b'foo') + self.assertEqual(bio.pending, 3) + for i in range(3): + bio.read(1) + self.assertEqual(bio.pending, 3-i-1) + for i in range(3): + bio.write(b'x') + self.assertEqual(bio.pending, i+1) + bio.read() + self.assertEqual(bio.pending, 0) + + def test_buffer_types(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + bio.write(bytearray(b'bar')) + self.assertEqual(bio.read(), b'bar') + bio.write(memoryview(b'baz')) + self.assertEqual(bio.read(), b'baz') + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + bio.write(memoryview(noncontig_writable)) + + def test_error_types(self): + bio = ssl.MemoryBIO() + self.assertRaises(TypeError, bio.write, 'foo') + self.assertRaises(TypeError, bio.write, None) + self.assertRaises(TypeError, bio.write, True) + self.assertRaises(TypeError, bio.write, 1) + + +class SSLObjectTests(unittest.TestCase): + def test_private_init(self): + bio = ssl.MemoryBIO() + with self.assertRaisesRegex(TypeError, "public constructor"): + ssl.SSLObject(bio, bio) + + def test_unwrap(self): + client_ctx, server_ctx, hostname = testing_context() + c_in = ssl.MemoryBIO() + c_out = ssl.MemoryBIO() + s_in = ssl.MemoryBIO() + s_out = ssl.MemoryBIO() + client = client_ctx.wrap_bio(c_in, c_out, server_hostname=hostname) + server = server_ctx.wrap_bio(s_in, s_out, server_side=True) + + # Loop on the handshake for a bit to get it settled + for _ in range(5): + try: + client.do_handshake() + except ssl.SSLWantReadError: + pass + if c_out.pending: + s_in.write(c_out.read()) + try: + server.do_handshake() + except ssl.SSLWantReadError: + pass + if s_out.pending: + c_in.write(s_out.read()) + # Now the handshakes should be complete (don't raise WantReadError) + client.do_handshake() + server.do_handshake() + + # Now if we unwrap one side unilaterally, it should send close-notify + # and raise WantReadError: + with self.assertRaises(ssl.SSLWantReadError): + client.unwrap() + + # But server.unwrap() does not raise, because it reads the client's + # close-notify: + s_in.write(c_out.read()) + server.unwrap() + + # And now that the client gets the server's close-notify, it doesn't + # raise either. + c_in.write(s_out.read()) + client.unwrap() + +class SimpleBackgroundTests(unittest.TestCase): + """Tests that connect to a simple server running in the background""" + + def setUp(self): + self.server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.server_context.load_cert_chain(SIGNED_CERTFILE) + server = ThreadedEchoServer(context=self.server_context) + self.enterContext(server) + self.server_addr = (HOST, server.port) + + def test_connect(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + self.assertFalse(s.server_side) + + # this should succeed because we specify the root cert + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) as s: + s.connect(self.server_addr) + self.assertTrue(s.getpeercert()) + self.assertFalse(s.server_side) + + def test_connect_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, + s.connect, self.server_addr) + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) + self.addCleanup(s.close) + self.assertEqual(0, s.connect_ex(self.server_addr)) + self.assertTrue(s.getpeercert()) + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.setblocking(False) + rc = s.connect_ex(self.server_addr) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + + def test_connect_with_context(self): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + # Same with a server hostname + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="dummy") as s: + s.connect(self.server_addr) + ctx.verify_mode = ssl.CERT_REQUIRED + # This should succeed because we specify the root cert + ctx.load_verify_locations(SIGNING_CA) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_with_context_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + s = ctx.wrap_socket( + socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME + ) + self.addCleanup(s.close) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, + s.connect, self.server_addr) + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(capath=CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(capath=BYTES_CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_cadata(self): + with open(SIGNING_CA) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + ss.connect(self.server_addr) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + s = socket.socket(socket.AF_INET) + s.connect(self.server_addr) + s.setblocking(False) + s = test_wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + self.addCleanup(s.close) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + _test_get_server_certificate(self, *self.server_addr, cert=SIGNING_CA) + + def test_get_server_certificate_sni(self): + host, port = self.server_addr + server_names = [] + + # We store servername_cb arguments to make sure they match the host + def servername_cb(ssl_sock, server_name, initial_context): + server_names.append(server_name) + self.server_context.set_servername_callback(servername_cb) + + pem = ssl.get_server_certificate((host, port)) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=SIGNING_CA) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port, pem)) + + self.assertEqual(server_names, [host, host]) + + def test_get_server_certificate_fail(self): + # Connection failure crashes ThreadedEchoServer, so run this in an + # independent test method + _test_get_server_certificate_fail(self, *self.server_addr) + + def test_get_server_certificate_timeout(self): + def servername_cb(ssl_sock, server_name, initial_context): + time.sleep(0.2) + self.server_context.set_servername_callback(servername_cb) + + with self.assertRaises(socket.timeout): + ssl.get_server_certificate(self.server_addr, ca_certs=SIGNING_CA, + timeout=0.1) + + def test_ciphers(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(self.server_addr) + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(self.server_addr) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = test_wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(self.server_addr) + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname='localhost') as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + self.assertEqual(len(ctx.get_ca_certs()), 1) + + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx1.load_verify_locations(capath=CAPATH) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx2.load_verify_locations(capath=CAPATH) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s, server_hostname='localhost') as ss: + ss.connect(self.server_addr) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + + def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): + # A simple IO loop. Call func(*args) depending on the error we get + # (WANT_READ or WANT_WRITE) move data between the socket and the BIOs. + timeout = kwargs.get('timeout', support.SHORT_TIMEOUT) + count = 0 + for _ in support.busy_retry(timeout): + errno = None + count += 1 + try: + ret = func(*args) + except ssl.SSLError as e: + if e.errno not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + raise + errno = e.errno + # Get any data from the outgoing BIO irrespective of any error, and + # send it to the socket. + buf = outgoing.read() + sock.sendall(buf) + # If there's no error, we're done. For WANT_READ, we need to get + # data from the socket and put it in the incoming BIO. + if errno is None: + break + elif errno == ssl.SSL_ERROR_WANT_READ: + buf = sock.recv(32768) + if buf: + incoming.write(buf) + else: + incoming.write_eof() + if support.verbose: + sys.stdout.write("Needed %d calls to complete %s().\n" + % (count, func.__name__)) + return ret + + def test_bio_handshake(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.load_verify_locations(SIGNING_CA) + sslobj = ctx.wrap_bio(incoming, outgoing, False, + SIGNED_CERTFILE_HOSTNAME) + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNone(sslobj.version()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # If the server shuts down the TCP connection without sending a + # secure shutdown message, this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + + def test_bio_read_write_data(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'FOO\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf, b'foo\n') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + + def test_transport_eof(self): + client_context, server_context, hostname = testing_context() + with socket.socket(socket.AF_INET) as sock: + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + sslobj = client_context.wrap_bio(incoming, outgoing, + server_hostname=hostname) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + + # Simulate EOF from the transport. + incoming.write_eof() + self.assertRaises(ssl.SSLEOFError, sslobj.read) + + +@support.requires_resource('network') +class NetworkedTests(unittest.TestCase): + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with socket_helper.transient_internet(REMOTE_HOST): + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + elif rc == errno.ENETUNREACH: + self.skipTest("Network unreachable.") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'Needs IPv6') + @support.requires_resource('walltime') + def test_get_server_certificate_ipv6(self): + with socket_helper.transient_internet('ipv6.google.com'): + _test_get_server_certificate(self, 'ipv6.google.com', 443) + _test_get_server_certificate_fail(self, 'ipv6.google.com', 443) + + +def _test_get_server_certificate(test, host, port, cert=None): + pem = ssl.get_server_certificate((host, port)) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + +def _test_get_server_certificate_fail(test, host, port): + with warnings_helper.check_no_resource_warning(test): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + +from test.ssl_servers import make_https_server + +class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(True) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) + except (ConnectionResetError, BrokenPipeError, ConnectionAbortedError) as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL + # tries to send session tickets after handshake. + # https://github.com/openssl/openssl/issues/6342 + # + # ConnectionAbortedError is raised in TLS 1.3 mode, when OpenSSL + # tries to send session tickets after handshake when using WinSock. + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.close() + return False + except (ssl.SSLError, OSError) as e: + # OSError may occur with wrong protocols, e.g. both + # sides use PROTOCOL_TLS_SERVER. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + # + # bpo-31323: Store the exception as string to prevent + # a reference leak: server -> conn_errors -> exception + # -> traceback -> self (ConnectionHandler) -> server + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + + # bpo-44229, bpo-43855, bpo-44237, and bpo-33450: + # Ignore spurious EPROTOTYPE returned by write() on macOS. + # See also http://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ + if e.errno != errno.EPROTOTYPE and sys.platform != "darwin": + self.running = False + self.close() + return False + else: + self.server.shared_ciphers.append(self.sslconn.shared_ciphers()) + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + if cert_binary is None: + sys.stdout.write(" client did not provide a cert\n") + else: + sys.stdout.write(f" cert binary is {len(cert_binary)}b\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + try: + self.sock = self.sslconn.unwrap() + except OSError: + # Many tests shut the TCP connection down + # without an SSL shutdown. This causes + # unwrap() to raise OSError with errno=0! + pass + else: + self.sslconn = None + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + elif stripped == b'PHA': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: initiating post handshake auth\n") + try: + self.sslconn.verify_client_post_handshake() + except ssl.SSLError as e: + self.write(repr(e).encode("us-ascii") + b"\n") + else: + self.write(b"OK\n") + elif stripped == b'HASCERT': + if self.sslconn.getpeercert() is not None: + self.write(b'TRUE\n') + else: + self.write(b'FALSE\n') + elif stripped == b'GETCERT': + cert = self.sslconn.getpeercert() + self.write(repr(cert).encode("us-ascii") + b"\n") + elif stripped == b'VERIFIEDCHAIN': + certs = self.sslconn._sslobj.get_verified_chain() + self.write(len(certs).to_bytes(1, "big") + b"\n") + elif stripped == b'UNVERIFIEDCHAIN': + certs = self.sslconn._sslobj.get_unverified_chain() + self.write(len(certs).to_bytes(1, "big") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except OSError as e: + # handles SSLError and socket errors + if isinstance(e, ConnectionError): + # OpenSSL 1.1.1 sometimes raises + # ConnectionResetError when connection is not + # shut down gracefully. + if self.server.chatty and support.verbose: + print(f" Connection reset by peer: {self.addr}") + + self.close() + self.running = False + return + if self.server.chatty and support.verbose: + handle_error("Test server failure:\n") + try: + self.write(b"ERROR\n") + except OSError: + pass + self.close() + self.running = False + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + alpn_protocols=None, + ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLS_SERVER) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if alpn_protocols: + self.context.set_alpn_protocols(alpn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = socket_helper.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_alpn_protocols = [] + self.shared_ciphers = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + self._in_context = False + + def __enter__(self): + if self._in_context: + raise ValueError('Re-entering ThreadedEchoServer context') + self._in_context = True + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + assert self._in_context + self._in_context = False + self.stop() + self.join() + + def start(self, flag=None): + if not self._in_context: + raise ValueError( + 'ThreadedEchoServer must be used as a context manager') + self.flag = flag + threading.Thread.start(self) + + def run(self): + if not self._in_context: + raise ValueError( + 'ThreadedEchoServer must be used as a context manager') + self.sock.settimeout(1.0) + self.sock.listen(5) + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except TimeoutError as e: + if support.verbose: + sys.stdout.write(f' connection timeout {e!r}\n') + except KeyboardInterrupt: + self.stop() + except BaseException as e: + if support.verbose and self.chatty: + sys.stdout.write( + ' connection handling failed: ' + repr(e) + '\n') + + self.close() + + def close(self): + if self.sock is not None: + self.sock.close() + self.sock = None + + def stop(self): + self.active = False + +class AsyncoreEchoServer(threading.Thread): + + # this one's based on asyncore.dispatcher + + class EchoServer (asyncore.dispatcher): + + class ConnectionHandler(asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = test_wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = socket_helper.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accepted(self, sock_obj, addr): + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + # make sure that ConnectionHandler is removed from socket_map + asyncore.close_all(ignore_all=True) + + def start (self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + +def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None, + session=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=sni_name, session=session) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_alpn_protocol': s.selected_alpn_protocol(), + 'version': s.version(), + 'session_reused': s.session_reused, + 'session': s.session, + }) + s.close() + stats['server_alpn_protocols'] = server.selected_alpn_protocols + stats['server_shared_ciphers'] = server.shared_ciphers + return stats + +def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + + with warnings_helper.check_warnings(): + # ignore Deprecation warnings + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + min_version = PROTOCOL_TO_TLS_VERSION.get(client_protocol, None) + if (min_version is not None + # SSLContext.minimum_version is only available on recent OpenSSL + # (setter added in OpenSSL 1.1.0, getter added in OpenSSL 1.1.1) + and hasattr(server_context, 'minimum_version') + and server_protocol == ssl.PROTOCOL_TLS + and server_context.minimum_version > min_version + ): + # If OpenSSL configuration is strict and requires more recent TLS + # version, we have to change the minimum to test old TLS versions. + with warnings_helper.check_warnings(): + server_context.minimum_version = min_version + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_TLS: + client_context.set_ciphers("ALL") + + seclevel_workaround(server_context, client_context) + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(SIGNED_CERTFILE) + ctx.load_verify_locations(SIGNING_CA) + try: + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except OSError as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) + + +def supports_kx_alias(ctx, aliases): + for cipher in ctx.get_ciphers(): + for alias in aliases: + if f"Kx={alias}" in cipher['description']: + return True + return False + + +class ThreadedTests(unittest.TestCase): + + @support.requires_resource('walltime') + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_SERVER): + server_params_test(client_context=client_context, + server_context=server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + client_context.check_hostname = False + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIn( + 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', + str(e.exception) + ) + + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_SERVER): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=server_context, + chatty=True, connectionchatty=True) + self.assertIn( + 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', + str(e.exception) + ) + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True) + self.assertIn( + 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', + str(e.exception)) + + @unittest.skip("TODO: RUSTPYTHON; flaky") + @unittest.skipUnless(support.Py_GIL_DISABLED, "test is only useful if the GIL is disabled") + def test_ssl_in_multiple_threads(self): + # See GH-124984: OpenSSL is not thread safe. + threads = [] + + warnings_filters = sys.flags.context_aware_warnings + global USE_SAME_TEST_CONTEXT + USE_SAME_TEST_CONTEXT = True + try: + for func in ( + self.test_echo, + self.test_alpn_protocols, + self.test_getpeercert, + self.test_crl_check, + functools.partial( + self.test_check_hostname_idn, + warnings_filters=warnings_filters, + ), + self.test_wrong_cert_tls12, + self.test_wrong_cert_tls13, + ): + # Be careful with the number of threads here. + # Too many can result in failing tests. + for num in range(5): + with self.subTest(func=func, num=num): + threads.append(Thread(target=func)) + + with threading_helper.catch_threading_exception() as cm: + for thread in threads: + with self.subTest(thread=thread): + thread.start() + + for thread in threads: + with self.subTest(thread=thread): + thread.join() + if cm.exc_value is not None: + # Some threads can skip their test + if not isinstance(cm.exc_value, unittest.SkipTest): + raise cm.exc_value + finally: + USE_SAME_TEST_CONTEXT = False + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + do_handshake_on_connect=False, + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(client_context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaisesRegex(ssl.SSLError, regex): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + client_context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: + with self.assertRaisesRegex(ssl.CertificateError, regex): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with socket.socket() as s: + with self.assertRaisesRegex(ValueError, + "check_hostname requires server_hostname"): + client_context.wrap_socket(s) + + @unittest.skipUnless( + ssl.HAS_NEVER_CHECK_COMMON_NAME, "test requires hostname_checks_common_name" + ) + def test_hostname_checks_common_name(self): + client_context, server_context, hostname = testing_context() + assert client_context.hostname_checks_common_name + client_context.hostname_checks_common_name = False + + # default cert has a SAN + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + + client_context, server_context, hostname = testing_context(NOSANFILE) + client_context.hostname_checks_common_name = False + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLCertVerificationError): + s.connect((HOST, server.port)) + + def test_ecc_cert(self): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + client_context.set_ciphers('ECDHE:ECDSA:!NULL:!aRSA') + hostname = SIGNED_CERTFILE_ECC_HOSTNAME + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # load ECC cert + server_context.load_cert_chain(SIGNED_CERTFILE_ECC) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher()[0].split('-') + self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + + @unittest.skipUnless(IS_OPENSSL_3_0_0, + "test requires RFC 5280 check added in OpenSSL 3.0+") + def test_verify_strict(self): + # verification fails by default, since the server cert is non-conforming + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + hostname = LEAF_MISSING_AKI_CERTFILE_HOSTNAME + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # explicitly disabling VERIFY_X509_STRICT allows it to succeed + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + client_context.verify_flags &= ~ssl.VERIFY_X509_STRICT + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_dual_rsa_ecc(self): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + # TODO: fix TLSv1.3 once SSLContext can restrict signature + # algorithms. + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # only ECDSA certs + client_context.set_ciphers('ECDHE:ECDSA:!NULL:!aRSA') + hostname = SIGNED_CERTFILE_ECC_HOSTNAME + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # load ECC and RSA key/cert pairs + server_context.load_cert_chain(SIGNED_CERTFILE_ECC) + server_context.load_cert_chain(SIGNED_CERTFILE) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher()[0].split('-') + self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + + def test_check_hostname_idn(self, warnings_filters=True): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(IDNSANSFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify, when specified in several + # different ways + idn_hostnames = [ + ('könig.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + ('xn--knig-5qa.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + (b'xn--knig-5qa.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + + ('königsgäßchen.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + ('xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + (b'xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + + # ('königsgäßchen.idna2008.pythontest.net', + # 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + ('xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + (b'xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + + ] + for server_hostname, expected_hostname in idn_hostnames: + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname=server_hostname) as s: + self.assertEqual(s.server_hostname, expected_hostname) + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertEqual(s.server_hostname, expected_hostname) + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="python.example.org") as s: + with self.assertRaises(ssl.CertificateError): + s.connect((HOST, server.port)) + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeError), + ): + context.wrap_socket(socket.socket(), server_hostname='.pythontest.net') + + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeDecodeError), + ): + context.wrap_socket( + socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net', + ) + + def test_wrong_cert_tls12(self): + """Connecting when the server rejects the client's certificate + + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with a wrong client certificate fails. + """ + client_context, server_context, hostname = testing_context() + # load client cert that is not signed by trusted CA + client_context.load_cert_chain(CERTFILE) + # require TLS client authentication + server_context.verify_mode = ssl.CERT_REQUIRED + # TLS 1.3 has different handshake + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer( + context=server_context, chatty=True, connectionchatty=True, + ) + + with server, \ + client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + try: + # Expect either an SSL error about the server rejecting + # the connection, or a low-level connection reset (which + # sometimes happens on Windows) + s.connect((HOST, server.port)) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + @requires_tls_version('TLSv1_3') + def test_wrong_cert_tls13(self): + client_context, server_context, hostname = testing_context() + # load client cert that is not signed by trusted CA + client_context.load_cert_chain(CERTFILE) + server_context.verify_mode = ssl.CERT_REQUIRED + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + + server = ThreadedEchoServer( + context=server_context, chatty=True, connectionchatty=True, + ) + with server, \ + client_context.wrap_socket(socket.socket(), + server_hostname=hostname, + suppress_ragged_eofs=False) as s: + s.connect((HOST, server.port)) + with self.assertRaisesRegex( + OSError, + 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA|' + 'closed by the remote host|Connection reset by peer|' + 'Broken pipe' + ): + # TLS 1.3 perform client cert exchange after handshake + s.write(b'data') + s.read(1000) + s.write(b'should have failed already') + s.read(1000) + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = socket_helper.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen() + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with socket.socket() as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = test_wrap_socket(c) + except OSError: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + def test_ssl_cert_verify_error(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + try: + s.connect((HOST, server.port)) + self.fail("Expected connection failure") + except ssl.SSLError as e: + msg = 'unable to get local issuer certificate' + self.assertIsInstance(e, ssl.SSLCertVerificationError) + self.assertEqual(e.verify_code, 20) + self.assertEqual(e.verify_message, msg) + # Allow for flexible libssl error messages. + regex = f"({msg}|CERTIFICATE_VERIFY_FAILED)" + self.assertRegex(repr(e), regex) + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRegex(repr(e), regex) + + def test_PROTOCOL_TLS(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1') + + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_OPTIONAL) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_REQUIRED) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + + # Server with specific SSL options + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + @requires_tls_version('SSLv3') + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + + @requires_tls_version('TLSv1') + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1) + + @requires_tls_version('TLSv1_1') + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + + @requires_tls_version('TLSv1_2') + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') + if has_tls_protocol(ssl.PROTOCOL_TLSv1): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(True) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = test_wrap_socket(s) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using socketserver to create and manage SSL connections.""" + server = make_https_server(self, certfile=SIGNED_CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + # Get this test file itself: + with open(__file__, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = f'https://localhost:{server.port}/test_ssl.py' + context = ssl.create_default_context(cafile=SIGNING_CA) + f = urllib.request.urlopen(url, context=context) + try: + dlen = f.info().get("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = test_wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, expect success?, *args, return value func) + send_methods = [ + ('send', s.send, True, [], len), + ('sendto', s.sendto, False, ["some.address"], len), + ('sendall', s.sendall, True, [], lambda x: None), + ] + # (name, method, whether to expect success, *args) + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = "PREFIX_" + + for (meth_name, send_meth, expect_success, args, + ret_val_meth) in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + ret = send_meth(indata, *args) + msg = "sending with {}".format(meth_name) + self.assertEqual(ret, ret_val_meth(indata), msg=msg) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # read(-1, buffer) is supported, even though read(-1) is not + data = b"data" + s.send(data) + buffer = bytearray(len(data)) + self.assertEqual(s.read(-1, buffer), len(data)) + self.assertEqual(buffer, data) + + # sendall accepts bytes-like objects + if ctypes is not None: + ubyte = ctypes.c_ubyte * len(data) + byteslike = ubyte.from_buffer_copy(data) + s.sendall(byteslike) + self.assertEqual(s.read(), data) + + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.dup) + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, [bytearray(100)]) + s.write(b"over\n") + + self.assertRaises(ValueError, s.recv, -1) + self.assertRaises(ValueError, s.read, -1) + + s.close() + + def test_recv_zero(self): + server = ThreadedEchoServer(CERTFILE) + self.enterContext(server) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + # recv/read(0) should return no data + s.send(b"data") + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.read(0), b"") + self.assertEqual(s.read(), b"data") + + # Should not block if the other end sends no data + s.setblocking(False) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.recv_into(bytearray()), 0) + + def test_recv_into_buffer_protocol_len(self): + server = ThreadedEchoServer(CERTFILE) + self.enterContext(server) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + s.send(b"data") + buf = array.array('I', [0, 0]) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf)[:4], b"data") + + class B(bytearray): + def __len__(self): + 1/0 + s.send(b"data") + buf = B(6) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf), b"data\0\0") + + def test_nonblocking_send(self): + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE) + s.connect((HOST, server.port)) + s.setblocking(False) + + # If we keep sending data, at some point the buffers + # will be full and the call will block + buf = bytearray(8192) + def fill_buffer(): + while True: + s.send(buf) + self.assertRaises((ssl.SSLWantWriteError, + ssl.SSLWantReadError), fill_buffer) + + # Now read all the output and discard it + s.setblocking(True) + s.close() + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = socket_helper.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen() + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegex(TimeoutError, "timed out", + test_wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = test_wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegex(TimeoutError, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + client_ctx, server_ctx, hostname = testing_context() + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = socket_helper.bind_port(server) + server = server_ctx.wrap_socket(server, server_side=True) + self.assertTrue(server.server_side) + + evt = threading.Event() + remote = None + peer = None + def serve(): + nonlocal remote, peer + server.listen() + # Block on the accept and wait on the connection to close. + evt.set() + remote, peer = server.accept() + remote.send(remote.recv(4)) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = client_ctx.wrap_socket( + socket.socket(), server_hostname=hostname + ) + client.connect((hostname, port)) + client.send(b'data') + client.recv() + client_addr = client.getsockname() + client.close() + t.join() + remote.close() + server.close() + # Sanity checks. + self.assertIsInstance(remote, ssl.SSLSocket) + self.assertEqual(peer, client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_no_shared_ciphers(self): + client_context, server_context, hostname = testing_context() + # OpenSSL enables all TLS 1.3 ciphers, enforce TLS 1.2 for test + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # Force different suites on client and server + client_context.set_ciphers("AES128") + server_context.set_ciphers("AES256") + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) + self.assertIn("NO_SHARED_CIPHER", server.conn_errors[0]) + + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + self.assertIs(s.version(), None) + self.assertIs(s._sslobj, None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1.3') + self.assertIs(s._sslobj, None) + self.assertIs(s.version(), None) + + @requires_tls_version('TLSv1_3') + def test_tls1_3(self): + client_context, server_context, hostname = testing_context() + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertIn(s.cipher()[0], { + 'TLS_AES_256_GCM_SHA384', + 'TLS_CHACHA20_POLY1305_SHA256', + 'TLS_AES_128_GCM_SHA256', + }) + self.assertEqual(s.version(), 'TLSv1.3') + + @requires_tls_version('TLSv1_2') + @requires_tls_version('TLSv1') + @ignore_deprecation + def test_min_max_version_tlsv1_2(self): + client_context, server_context, hostname = testing_context() + # client TLSv1.0 to 1.2 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # server only TLSv1.2 + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1.2') + + @requires_tls_version('TLSv1_1') + @ignore_deprecation + def test_min_max_version_tlsv1_1(self): + client_context, server_context, hostname = testing_context() + # client 1.0 to 1.2, server 1.0 to 1.1 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.minimum_version = ssl.TLSVersion.TLSv1 + server_context.maximum_version = ssl.TLSVersion.TLSv1_1 + seclevel_workaround(client_context, server_context) + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1.1') + + @requires_tls_version('TLSv1_2') + @requires_tls_version('TLSv1') + @ignore_deprecation + def test_min_max_version_mismatch(self): + client_context, server_context, hostname = testing_context() + # client 1.0, server 1.2 (mismatch) + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + client_context.maximum_version = ssl.TLSVersion.TLSv1 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + seclevel_workaround(client_context, server_context) + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError) as e: + s.connect((HOST, server.port)) + self.assertRegex(str(e.exception), "(alert|ALERT)") + + @requires_tls_version('SSLv3') + def test_min_max_version_sslv3(self): + client_context, server_context, hostname = testing_context() + server_context.minimum_version = ssl.TLSVersion.SSLv3 + client_context.minimum_version = ssl.TLSVersion.SSLv3 + client_context.maximum_version = ssl.TLSVersion.SSLv3 + seclevel_workaround(client_context, server_context) + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'SSLv3') + + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + client_context, server_context, hostname = testing_context() + # TLSv1.3 defaults to PFS key agreement and no longer has KEA in + # cipher name. + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer(context=server_context, + chatty=True, + connectionchatty=False) + + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write( + " got channel binding data: {0!r}\n".format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + + # now, again + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write( + "got another channel binding data: {0!r}\n".format( + new_cb_data) + ) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + + def test_compression(self): + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_NO_COMPRESSION + server_context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['compression'], None) + + def test_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_no_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options &= ~ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_dh_params(self): + # Check we can get a connection with ephemeral finite-field + # Diffie-Hellman (if supported). + client_context, server_context, hostname = testing_context() + dhe_aliases = {"ADH", "EDH", "DHE"} + if not (supports_kx_alias(client_context, dhe_aliases) + and supports_kx_alias(server_context, dhe_aliases)): + self.skipTest("libssl doesn't support ephemeral DH") + # test scenario needs TLS <= 1.2 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + try: + server_context.load_dh_params(DHFILE) + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + server_context.set_ciphers("kEDH") + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if not dhe_aliases.intersection(parts): + self.fail("Non-DH key exchange: " + cipher[0]) + + def test_ecdh_curve(self): + # server secp384r1, client auto + client_context, server_context, hostname = testing_context() + + server_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + # server auto, client secp384r1 + client_context, server_context, hostname = testing_context() + client_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + # server / client curve mismatch + client_context, server_context, hostname = testing_context() + client_context.set_ecdh_curve("prime256v1") + server_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + with self.assertRaises(ssl.SSLError): + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_selected_alpn_protocol(self): + # selected_alpn_protocol() is None unless ALPN is used. + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['client_alpn_protocol'], None) + + def test_selected_alpn_protocol_if_server_uses_alpn(self): + # selected_alpn_protocol() is None unless ALPN is used by the client. + client_context, server_context, hostname = testing_context() + server_context.set_alpn_protocols(['foo', 'bar']) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['client_alpn_protocol'], None) + + def test_alpn_protocols(self): + server_protocols = ['foo', 'bar', 'milkshake'] + protocol_tests = [ + (['foo', 'bar'], 'foo'), + (['bar', 'foo'], 'foo'), + (['milkshake'], 'milkshake'), + (['http/3.0', 'http/4.0'], None) + ] + for client_protocols, expected in protocol_tests: + client_context, server_context, hostname = testing_context() + server_context.set_alpn_protocols(server_protocols) + client_context.set_alpn_protocols(client_protocols) + + try: + stats = server_params_test(client_context, + server_context, + chatty=True, + connectionchatty=True, + sni_name=hostname) + except ssl.SSLError as e: + stats = e + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_alpn_protocol'] + self.assertEqual(client_result, expected, + msg % (client_result, "client")) + server_result = stats['server_alpn_protocols'][-1] \ + if len(stats['server_alpn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, + msg % (server_result, "server")) + + def test_npn_protocols(self): + assert not ssl.HAS_NPN + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + client_context.check_hostname = False + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, SIGNED_CERTFILE_HOSTNAME) + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, SIGNED_CERTFILE_HOSTNAME) + self.assertEqual(calls, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + TLSV1_ALERT_ACCESS_DENIED + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1/0 + server_context.set_servername_callback(cb_raising) + + with support.catch_unraisable_exception() as catch: + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + + # Allow for flexible libssl error messages. + regex = "(SSLV3_ALERT_HANDSHAKE_FAILURE|NO_PRIVATE_VALUE)" + self.assertRegex(cm.exception.reason, regex) + self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'SSLEOFError' object has no attribute 'reason' + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with support.catch_unraisable_exception() as catch: + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + + + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertEqual(catch.unraisable.exc_type, TypeError) + + def test_shared_ciphers(self): + client_context, server_context, hostname = testing_context() + client_context.set_ciphers("AES128:AES256") + server_context.set_ciphers("AES256:eNULL") + expected_algs = [ + "AES256", "AES-256", + # TLS 1.3 ciphers are always enabled + "TLS_CHACHA20", "TLS_AES", + ] + + stats = server_params_test(client_context, server_context, + sni_name=hostname) + ciphers = stats['server_shared_ciphers'][0] + self.assertGreater(len(ciphers), 0) + for name, tls_version, bits in ciphers: + if not any(alg in name for alg in expected_algs): + self.fail(name) + + def test_read_write_after_close_raises_valuerror(self): + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context, chatty=False) + + with server: + s = client_context.wrap_socket(socket.socket(), + server_hostname=hostname) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + def test_sendfile(self): + TEST_DATA = b"x" * 512 + with open(os_helper.TESTFN, 'wb') as f: + f.write(TEST_DATA) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + with open(os_helper.TESTFN, 'rb') as file: + s.sendfile(file) + self.assertEqual(s.recv(1024), TEST_DATA) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_session(self): + client_context, server_context, hostname = testing_context() + # TODO: sessions aren't compatible with TLSv1.3 yet + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + # first connection without session + stats = server_params_test(client_context, server_context, + sni_name=hostname) + session = stats['session'] + self.assertTrue(session.id) + self.assertGreater(session.time, 0) + self.assertGreater(session.timeout, 0) + self.assertTrue(session.has_ticket) + self.assertGreater(session.ticket_lifetime_hint, 0) + self.assertFalse(stats['session_reused']) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 1) + self.assertEqual(sess_stat['hits'], 0) + + # reuse session + stats = server_params_test(client_context, server_context, + session=session, sni_name=hostname) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 2) + self.assertEqual(sess_stat['hits'], 1) + self.assertTrue(stats['session_reused']) + session2 = stats['session'] + self.assertEqual(session2.id, session.id) + self.assertEqual(session2, session) + self.assertIsNot(session2, session) + self.assertGreaterEqual(session2.time, session.time) + self.assertGreaterEqual(session2.timeout, session.timeout) + + # another one without session + stats = server_params_test(client_context, server_context, + sni_name=hostname) + self.assertFalse(stats['session_reused']) + session3 = stats['session'] + self.assertNotEqual(session3.id, session.id) + self.assertNotEqual(session3, session) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 3) + self.assertEqual(sess_stat['hits'], 1) + + # reuse session again + stats = server_params_test(client_context, server_context, + session=session, sni_name=hostname) + self.assertTrue(stats['session_reused']) + session4 = stats['session'] + self.assertEqual(session4.id, session.id) + self.assertEqual(session4, session) + self.assertGreaterEqual(session4.time, session.time) + self.assertGreaterEqual(session4.timeout, session.timeout) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 4) + self.assertEqual(sess_stat['hits'], 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False != True + def test_session_handling(self): + client_context, server_context, hostname = testing_context() + client_context2, _, _ = testing_context() + + # TODO: session reuse does not work with TLSv1.3 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context2.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # session is None before handshake + self.assertEqual(s.session, None) + self.assertEqual(s.session_reused, None) + s.connect((HOST, server.port)) + session = s.session + self.assertTrue(session) + with self.assertRaises(TypeError) as e: + s.session = object + self.assertEqual(str(e.exception), 'Value is not a SSLSession.') + + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # cannot set session after handshake + with self.assertRaises(ValueError) as e: + s.session = session + self.assertEqual(str(e.exception), + 'Cannot set session after handshake.') + + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # can set session before handshake and before the + # connection was established + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(s.session.id, session.id) + self.assertEqual(s.session, session) + self.assertEqual(s.session_reused, True) + + with client_context2.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # cannot re-use session with a different SSLContext + with self.assertRaises(ValueError) as e: + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(str(e.exception), + 'Session refers to a different SSLContext.') + + @requires_tls_version('TLSv1_2') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') + def test_psk(self): + psk = bytes.fromhex('deadbeef') + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(lambda hint: (None, psk)) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(lambda identity: psk) + + # correct PSK should connect + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + # incorrect PSK should fail + incorrect_psk = bytes.fromhex('cafebabe') + client_context.set_psk_client_callback(lambda hint: (None, incorrect_psk)) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # identity_hint and client_identity should be sent to the other side + identity_hint = 'identity-hint' + client_identity = 'client-identity' + + def client_callback(hint): + self.assertEqual(hint, identity_hint) + return client_identity, psk + + def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + + client_context.set_psk_client_callback(client_callback) + server_context.set_psk_server_callback(server_callback, identity_hint) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + # adding client callback to server or vice versa raises an exception + with self.assertRaisesRegex(ssl.SSLError, 'Cannot add PSK server callback'): + client_context.set_psk_server_callback(server_callback, identity_hint) + with self.assertRaisesRegex(ssl.SSLError, 'Cannot add PSK client callback'): + server_context.set_psk_client_callback(client_callback) + + # test with UTF-8 identities + identity_hint = '身份暗示' # Translation: "Identity hint" + client_identity = '客户身份' # Translation: "Customer identity" + + client_context.set_psk_client_callback(client_callback) + server_context.set_psk_server_callback(server_callback, identity_hint) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + @requires_tls_version('TLSv1_3') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') + def test_psk_tls1_3(self): + psk = bytes.fromhex('deadbeef') + identity_hint = 'identity-hint' + client_identity = 'client-identity' + + def client_callback(hint): + # identity_hint is not sent to the client in TLS 1.3 + self.assertIsNone(hint) + return client_identity, psk + + def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(client_callback) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(server_callback, identity_hint) + + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + @unittest.skip("TODO: RUSTPYTHON; Hangs") + def test_thread_recv_while_main_thread_sends(self): + # GH-137583: Locking was added to calls to send() and recv() on SSL + # socket objects. This seemed fine at the surface level because those + # calls weren't re-entrant, but recv() calls would implicitly mimick + # holding a lock by blocking until it received data. This means that + # if a thread started to infinitely block until data was received, calls + # to send() would deadlock, because it would wait forever on the lock + # that the recv() call held. + data = b"1" * 1024 + event = threading.Event() + def background(sock): + event.set() + received = sock.recv(len(data)) + self.assertEqual(received, data) + + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as sock: + sock.connect((HOST, server.port)) + sock.settimeout(1) + sock.setblocking(1) + # Ensure that the server is ready to accept requests + sock.sendall(b"123") + self.assertEqual(sock.recv(3), b"123") + with threading_helper.catch_threading_exception() as cm: + thread = threading.Thread(target=background, + args=(sock,), daemon=True) + thread.start() + event.wait() + sock.sendall(data) + thread.join() + if cm.exc_value is not None: + raise cm.exc_value + + +@unittest.skipUnless(has_tls_version('TLSv1_3') and ssl.HAS_PHA, + "Test needs TLS 1.3 PHA") +class TestPostHandshakeAuth(unittest.TestCase): + def test_pha_setter(self): + protocols = [ + ssl.PROTOCOL_TLS_SERVER, ssl.PROTOCOL_TLS_CLIENT + ] + for protocol in protocols: + ctx = ssl.SSLContext(protocol) + self.assertEqual(ctx.post_handshake_auth, False) + + ctx.post_handshake_auth = True + self.assertEqual(ctx.post_handshake_auth, True) + + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.post_handshake_auth, True) + + ctx.post_handshake_auth = False + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.post_handshake_auth, False) + + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.post_handshake_auth = True + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + self.assertEqual(ctx.post_handshake_auth, True) + + def test_pha_required(self): + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + # PHA method just returns true when cert is already available + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'GETCERT') + cert_text = s.recv(4096).decode('us-ascii') + self.assertIn('Python Software Foundation CA', cert_text) + + def test_pha_required_nocert(self): + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.post_handshake_auth = True + + def msg_cb(conn, direction, version, content_type, msg_type, data): + if support.verbose and content_type == _TLSContentType.ALERT: + info = (conn, direction, version, content_type, msg_type, data) + sys.stdout.write(f"TLS: {info!r}\n") + + server_context._msg_callback = msg_cb + client_context._msg_callback = msg_cb + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname, + suppress_ragged_eofs=False) as s: + s.connect((HOST, server.port)) + s.write(b'PHA') + # test sometimes fails with EOF error. Test passes as long as + # server aborts connection with an error. + with self.assertRaisesRegex( + OSError, + ('certificate required' + '|EOF occurred' + '|closed by the remote host' + '|Connection reset by peer' + '|Broken pipe') + ): + # receive CertificateRequest + data = s.recv(1024) + self.assertEqual(data, b'OK\n') + + # send empty Certificate + Finish + s.write(b'HASCERT') + + # receive alert + s.recv(1024) + + def test_pha_optional(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + + # check CERT_OPTIONAL + server_context.verify_mode = ssl.CERT_OPTIONAL + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + + def test_pha_optional_nocert(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_OPTIONAL + client_context.post_handshake_auth = True + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + # optional doesn't fail when client does not have a cert + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + + def test_pha_no_pha_client(self): + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_cert_chain(SIGNED_CERTFILE) + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + with self.assertRaisesRegex(ssl.SSLError, 'not server'): + s.verify_client_post_handshake() + s.write(b'PHA') + self.assertIn(b'extension not received', s.recv(1024)) + + def test_pha_no_pha_server(self): + # server doesn't have PHA enabled, cert is requested in handshake + client_context, server_context, hostname = testing_context() + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + # PHA doesn't fail if there is already a cert + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + + def test_pha_not_tls13(self): + # TLS 1.2 + client_context, server_context, hostname = testing_context() + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # PHA fails for TLS != 1.3 + s.write(b'PHA') + self.assertIn(b'WRONG_SSL_VERSION', s.recv(1024)) + + def test_bpo37428_pha_cert_none(self): + # verify that post_handshake_auth does not implicitly enable cert + # validation. + hostname = SIGNED_CERTFILE_HOSTNAME + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + # no cert validation and CA on client side + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(SIGNED_CERTFILE) + server_context.load_verify_locations(SIGNING_CA) + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + # server cert has not been validated + self.assertEqual(s.getpeercert(), {}) + + def test_internal_chain_client(self): + client_context, server_context, hostname = testing_context( + server_chain=False + ) + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + vc = s._sslobj.get_verified_chain() + self.assertEqual(len(vc), 2) + ee, ca = vc + uvc = s._sslobj.get_unverified_chain() + self.assertEqual(len(uvc), 1) + + self.assertEqual(ee, uvc[0]) + self.assertEqual(hash(ee), hash(uvc[0])) + self.assertEqual(repr(ee), repr(uvc[0])) + + self.assertNotEqual(ee, ca) + self.assertNotEqual(hash(ee), hash(ca)) + self.assertNotEqual(repr(ee), repr(ca)) + self.assertNotEqual(ee.get_info(), ca.get_info()) + self.assertIn("CN=localhost", repr(ee)) + self.assertIn("CN=our-ca-server", repr(ca)) + + pem = ee.public_bytes(_ssl.ENCODING_PEM) + der = ee.public_bytes(_ssl.ENCODING_DER) + self.assertIsInstance(pem, str) + self.assertIn("-----BEGIN CERTIFICATE-----", pem) + self.assertIsInstance(der, bytes) + self.assertEqual( + ssl.PEM_cert_to_DER_cert(pem), der + ) + + def test_certificate_chain(self): + client_context, server_context, hostname = testing_context( + server_chain=False + ) + server = ThreadedEchoServer(context=server_context, chatty=False) + + with open(SIGNING_CA) as f: + expected_ca_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with open(SINGED_CERTFILE_ONLY) as f: + expected_ee_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + vc = s.get_verified_chain() + self.assertEqual(len(vc), 2) + + ee, ca = vc + self.assertIsInstance(ee, bytes) + self.assertIsInstance(ca, bytes) + self.assertEqual(expected_ca_cert, ca) + self.assertEqual(expected_ee_cert, ee) + + uvc = s.get_unverified_chain() + self.assertEqual(len(uvc), 1) + self.assertIsInstance(uvc[0], bytes) + + self.assertEqual(ee, uvc[0]) + self.assertNotEqual(ee, ca) + + def test_internal_chain_server(self): + client_context, server_context, hostname = testing_context() + client_context.load_cert_chain(SIGNED_CERTFILE) + server_context.verify_mode = ssl.CERT_REQUIRED + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + s.write(b'VERIFIEDCHAIN\n') + res = s.recv(1024) + self.assertEqual(res, b'\x02\n') + s.write(b'UNVERIFIEDCHAIN\n') + res = s.recv(1024) + self.assertEqual(res, b'\x02\n') + + +HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') +requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') + +class TestSSLDebug(unittest.TestCase): + + def keylog_lines(self, fname=os_helper.TESTFN): + with open(fname) as f: + return len(list(f)) + + @requires_keylog + def test_keylog_defaults(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.keylog_filename, None) + + self.assertFalse(os.path.isfile(os_helper.TESTFN)) + try: + ctx.keylog_filename = os_helper.TESTFN + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + self.assertEqual(ctx.keylog_filename, os_helper.TESTFN) + self.assertTrue(os.path.isfile(os_helper.TESTFN)) + self.assertEqual(self.keylog_lines(), 1) + + ctx.keylog_filename = None + self.assertEqual(ctx.keylog_filename, None) + + with self.assertRaises((IsADirectoryError, PermissionError)): + # Windows raises PermissionError + ctx.keylog_filename = os.path.dirname( + os.path.abspath(os_helper.TESTFN)) + + with self.assertRaises(TypeError): + ctx.keylog_filename = 1 + + @requires_keylog + def test_keylog_filename(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + client_context, server_context, hostname = testing_context() + + try: + client_context.keylog_filename = os_helper.TESTFN + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # header, 5 lines for TLS 1.3 + self.assertEqual(self.keylog_lines(), 6) + + client_context.keylog_filename = None + server_context.keylog_filename = os_helper.TESTFN + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertGreaterEqual(self.keylog_lines(), 11) + + client_context.keylog_filename = os_helper.TESTFN + server_context.keylog_filename = os_helper.TESTFN + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertGreaterEqual(self.keylog_lines(), 21) + + client_context.keylog_filename = None + server_context.keylog_filename = None + + @requires_keylog + @unittest.skipIf(sys.flags.ignore_environment, + "test is not compatible with ignore_environment") + def test_keylog_env(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with unittest.mock.patch.dict(os.environ): + os.environ['SSLKEYLOGFILE'] = os_helper.TESTFN + self.assertEqual(os.environ['SSLKEYLOGFILE'], os_helper.TESTFN) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.keylog_filename, None) + + try: + ctx = ssl.create_default_context() + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + self.assertEqual(ctx.keylog_filename, os_helper.TESTFN) + + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.keylog_filename, os_helper.TESTFN) + + def test_msg_callback(self): + client_context, server_context, hostname = testing_context() + + def msg_cb(conn, direction, version, content_type, msg_type, data): + pass + + self.assertIs(client_context._msg_callback, None) + client_context._msg_callback = msg_cb + self.assertIs(client_context._msg_callback, msg_cb) + with self.assertRaises(TypeError): + client_context._msg_callback = object() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ('read', <TLSVersion.TLSv1_2: 771>, <_TLSContentType.HANDSHAKE: 22>, <_TLSMessageType.SERVER_KEY_EXCHANGE: 12>) not found in [] + def test_msg_callback_tls12(self): + client_context, server_context, hostname = testing_context() + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + msg = [] + + def msg_cb(conn, direction, version, content_type, msg_type, data): + self.assertIsInstance(conn, ssl.SSLSocket) + self.assertIsInstance(data, bytes) + self.assertIn(direction, {'read', 'write'}) + msg.append((direction, version, content_type, msg_type)) + + client_context._msg_callback = msg_cb + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + + self.assertIn( + ("read", TLSVersion.TLSv1_2, _TLSContentType.HANDSHAKE, + _TLSMessageType.SERVER_KEY_EXCHANGE), + msg + ) + self.assertIn( + ("write", TLSVersion.TLSv1_2, _TLSContentType.CHANGE_CIPHER_SPEC, + _TLSMessageType.CHANGE_CIPHER_SPEC), + msg + ) + + def test_msg_callback_deadlock_bpo43577(self): + client_context, server_context, hostname = testing_context() + server_context2 = testing_context()[1] + + def msg_cb(conn, direction, version, content_type, msg_type, data): + pass + + def sni_cb(sock, servername, ctx): + sock.context = server_context2 + + server_context._msg_callback = msg_cb + server_context.sni_callback = sni_cb + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + + +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept, timeout=None): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + if timeout is None: + self.timeout = support.SHORT_TIMEOUT + else: + self.timeout = timeout + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = socket_helper.bind_port(self.listener) + self.listener.settimeout(self.timeout) + self.listener.listen(1) + super().start() + + def run(self): + try: + conn, address = self.listener.accept() + except TimeoutError: + # on timeout, just close the listener + return + finally: + self.listener.close() + + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform in ("linux", "android"): + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + try: + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + finally: + # gh-108342: Explicitly break the reference cycle + err = None + + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(support.SHORT_TIMEOUT): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + self.enterContext(server) # starts it & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + + wrap_error = server.wrap_error + server.wrap_error = None + try: + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + finally: + # gh-108342: Explicitly break the reference cycle + wrap_error = None + server = None + + def test_preauth_data_to_tls_client(self): + server_can_continue_with_wrap_socket = threading.Event() + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + if not server_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT): + print("ERROR: test client took too long") + + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + server_can_continue_with_wrap_socket.set() + + if not client_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT): + self.fail("test server took too long") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + try: + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + finally: + # gh-108342: Explicitly break the reference cycle + with warnings_helper.check_no_resource_warning(self): + wrap_error = None + server = None + + def test_https_client_non_tls_response_ignored(self): + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + # Call clear text HTTP connect(), not the encrypted HTTPS (TLS) + # connect(): wrap_socket() is called manually below. + http.client.HTTPConnection.connect(self) + + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(support.SHORT_TIMEOUT) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + timeout = 2.0 + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder", + timeout=timeout) + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + server.listener.getsockname()[0], + port=server.port, + context=ssl.create_default_context(), + timeout=timeout, + ) + + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with warnings_helper.check_no_resource_warning(self), \ + self.assertRaises(OSError): + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + server.join() + + +class TestEnumerations(unittest.TestCase): + + def test_tlsversion(self): + class CheckedTLSVersion(enum.IntEnum): + MINIMUM_SUPPORTED = _ssl.PROTO_MINIMUM_SUPPORTED + SSLv3 = _ssl.PROTO_SSLv3 + TLSv1 = _ssl.PROTO_TLSv1 + TLSv1_1 = _ssl.PROTO_TLSv1_1 + TLSv1_2 = _ssl.PROTO_TLSv1_2 + TLSv1_3 = _ssl.PROTO_TLSv1_3 + MAXIMUM_SUPPORTED = _ssl.PROTO_MAXIMUM_SUPPORTED + enum._test_simple_enum(CheckedTLSVersion, TLSVersion) + + def test_tlscontenttype(self): + class Checked_TLSContentType(enum.IntEnum): + """Content types (record layer) + + See RFC 8446, section B.1 + """ + CHANGE_CIPHER_SPEC = 20 + ALERT = 21 + HANDSHAKE = 22 + APPLICATION_DATA = 23 + # pseudo content types + HEADER = 0x100 + INNER_CONTENT_TYPE = 0x101 + enum._test_simple_enum(Checked_TLSContentType, _TLSContentType) + + def test_tlsalerttype(self): + class Checked_TLSAlertType(enum.IntEnum): + """Alert types for TLSContentType.ALERT messages + + See RFC 8446, section B.2 + """ + CLOSE_NOTIFY = 0 + UNEXPECTED_MESSAGE = 10 + BAD_RECORD_MAC = 20 + DECRYPTION_FAILED = 21 + RECORD_OVERFLOW = 22 + DECOMPRESSION_FAILURE = 30 + HANDSHAKE_FAILURE = 40 + NO_CERTIFICATE = 41 + BAD_CERTIFICATE = 42 + UNSUPPORTED_CERTIFICATE = 43 + CERTIFICATE_REVOKED = 44 + CERTIFICATE_EXPIRED = 45 + CERTIFICATE_UNKNOWN = 46 + ILLEGAL_PARAMETER = 47 + UNKNOWN_CA = 48 + ACCESS_DENIED = 49 + DECODE_ERROR = 50 + DECRYPT_ERROR = 51 + EXPORT_RESTRICTION = 60 + PROTOCOL_VERSION = 70 + INSUFFICIENT_SECURITY = 71 + INTERNAL_ERROR = 80 + INAPPROPRIATE_FALLBACK = 86 + USER_CANCELED = 90 + NO_RENEGOTIATION = 100 + MISSING_EXTENSION = 109 + UNSUPPORTED_EXTENSION = 110 + CERTIFICATE_UNOBTAINABLE = 111 + UNRECOGNIZED_NAME = 112 + BAD_CERTIFICATE_STATUS_RESPONSE = 113 + BAD_CERTIFICATE_HASH_VALUE = 114 + UNKNOWN_PSK_IDENTITY = 115 + CERTIFICATE_REQUIRED = 116 + NO_APPLICATION_PROTOCOL = 120 + enum._test_simple_enum(Checked_TLSAlertType, _TLSAlertType) + + def test_tlsmessagetype(self): + class Checked_TLSMessageType(enum.IntEnum): + """Message types (handshake protocol) + + See RFC 8446, section B.3 + """ + HELLO_REQUEST = 0 + CLIENT_HELLO = 1 + SERVER_HELLO = 2 + HELLO_VERIFY_REQUEST = 3 + NEWSESSION_TICKET = 4 + END_OF_EARLY_DATA = 5 + HELLO_RETRY_REQUEST = 6 + ENCRYPTED_EXTENSIONS = 8 + CERTIFICATE = 11 + SERVER_KEY_EXCHANGE = 12 + CERTIFICATE_REQUEST = 13 + SERVER_DONE = 14 + CERTIFICATE_VERIFY = 15 + CLIENT_KEY_EXCHANGE = 16 + FINISHED = 20 + CERTIFICATE_URL = 21 + CERTIFICATE_STATUS = 22 + SUPPLEMENTAL_DATA = 23 + KEY_UPDATE = 24 + NEXT_PROTO = 67 + MESSAGE_HASH = 254 + CHANGE_CIPHER_SPEC = 0x0101 + enum._test_simple_enum(Checked_TLSMessageType, _TLSMessageType) + + def test_sslmethod(self): + Checked_SSLMethod = enum._old_convert_( + enum.IntEnum, '_SSLMethod', 'ssl', + lambda name: name.startswith('PROTOCOL_') and name != 'PROTOCOL_SSLv23', + source=ssl._ssl, + ) + # This member is assigned dynamically in `ssl.py`: + Checked_SSLMethod.PROTOCOL_SSLv23 = Checked_SSLMethod.PROTOCOL_TLS + enum._test_simple_enum(Checked_SSLMethod, ssl._SSLMethod) + + def test_options(self): + CheckedOptions = enum._old_convert_( + enum.IntFlag, 'Options', 'ssl', + lambda name: name.startswith('OP_'), + source=ssl._ssl, + ) + enum._test_simple_enum(CheckedOptions, ssl.Options) + + def test_alertdescription(self): + CheckedAlertDescription = enum._old_convert_( + enum.IntEnum, 'AlertDescription', 'ssl', + lambda name: name.startswith('ALERT_DESCRIPTION_'), + source=ssl._ssl, + ) + enum._test_simple_enum(CheckedAlertDescription, ssl.AlertDescription) + + def test_sslerrornumber(self): + Checked_SSLErrorNumber = enum._old_convert_( + enum.IntEnum, 'SSLErrorNumber', 'ssl', + lambda name: name.startswith('SSL_ERROR_'), + source=ssl._ssl, + ) + enum._test_simple_enum(Checked_SSLErrorNumber, ssl.SSLErrorNumber) + + def test_verifyflags(self): + CheckedVerifyFlags = enum._old_convert_( + enum.IntFlag, 'VerifyFlags', 'ssl', + lambda name: name.startswith('VERIFY_'), + source=ssl._ssl, + ) + enum._test_simple_enum(CheckedVerifyFlags, ssl.VerifyFlags) + + def test_verifymode(self): + CheckedVerifyMode = enum._old_convert_( + enum.IntEnum, 'VerifyMode', 'ssl', + lambda name: name.startswith('CERT_'), + source=ssl._ssl, + ) + enum._test_simple_enum(CheckedVerifyMode, ssl.VerifyMode) + + +def setUpModule(): + if support.verbose: + plats = { + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py new file mode 100644 index 00000000000..1e6f69d49e9 --- /dev/null +++ b/Lib/test/test_stable_abi_ctypes.py @@ -0,0 +1,1004 @@ +# Generated by Tools/build/stable_abi.py + +"""Test that all symbols of the Stable ABI are accessible using ctypes +""" + +import sys +import unittest +from test.support.import_helper import import_module +try: + from _testcapi import get_feature_macros +except ImportError: + raise unittest.SkipTest("requires _testcapi") + +feature_macros = get_feature_macros() + +# Stable ABI is incompatible with Py_TRACE_REFS builds due to PyObject +# layout differences. +# See https://github.com/python/cpython/issues/88299#issuecomment-1113366226 +if feature_macros['Py_TRACE_REFS']: + raise unittest.SkipTest("incompatible with Py_TRACE_REFS.") + +ctypes_test = import_module('ctypes') + +class TestStableABIAvailability(unittest.TestCase): + def test_available_symbols(self): + + for symbol_name in SYMBOL_NAMES: + with self.subTest(symbol_name): + ctypes_test.pythonapi[symbol_name] + + def test_feature_macros(self): + self.assertEqual( + set(get_feature_macros()), EXPECTED_FEATURE_MACROS) + + # The feature macros for Windows are used in creating the DLL + # definition, so they must be known on all platforms. + # If we are on Windows, we check that the hardcoded data matches + # the reality. + @unittest.skipIf(sys.platform != "win32", "Windows specific test") + def test_windows_feature_macros(self): + for name, value in WINDOWS_FEATURE_MACROS.items(): + if value != 'maybe': + with self.subTest(name): + self.assertEqual(feature_macros[name], value) + +SYMBOL_NAMES = ( + + "PyAIter_Check", + "PyArg_Parse", + "PyArg_ParseTuple", + "PyArg_ParseTupleAndKeywords", + "PyArg_UnpackTuple", + "PyArg_VaParse", + "PyArg_VaParseTupleAndKeywords", + "PyArg_ValidateKeywordArguments", + "PyBaseObject_Type", + "PyBool_FromLong", + "PyBool_Type", + "PyBuffer_FillContiguousStrides", + "PyBuffer_FillInfo", + "PyBuffer_FromContiguous", + "PyBuffer_GetPointer", + "PyBuffer_IsContiguous", + "PyBuffer_Release", + "PyBuffer_SizeFromFormat", + "PyBuffer_ToContiguous", + "PyByteArrayIter_Type", + "PyByteArray_AsString", + "PyByteArray_Concat", + "PyByteArray_FromObject", + "PyByteArray_FromStringAndSize", + "PyByteArray_Resize", + "PyByteArray_Size", + "PyByteArray_Type", + "PyBytesIter_Type", + "PyBytes_AsString", + "PyBytes_AsStringAndSize", + "PyBytes_Concat", + "PyBytes_ConcatAndDel", + "PyBytes_DecodeEscape", + "PyBytes_FromFormat", + "PyBytes_FromFormatV", + "PyBytes_FromObject", + "PyBytes_FromString", + "PyBytes_FromStringAndSize", + "PyBytes_Repr", + "PyBytes_Size", + "PyBytes_Type", + "PyCFunction_Call", + "PyCFunction_GetFlags", + "PyCFunction_GetFunction", + "PyCFunction_GetSelf", + "PyCFunction_New", + "PyCFunction_NewEx", + "PyCFunction_Type", + "PyCMethod_New", + "PyCallIter_New", + "PyCallIter_Type", + "PyCallable_Check", + "PyCapsule_GetContext", + "PyCapsule_GetDestructor", + "PyCapsule_GetName", + "PyCapsule_GetPointer", + "PyCapsule_Import", + "PyCapsule_IsValid", + "PyCapsule_New", + "PyCapsule_SetContext", + "PyCapsule_SetDestructor", + "PyCapsule_SetName", + "PyCapsule_SetPointer", + "PyCapsule_Type", + "PyClassMethodDescr_Type", + "PyCodec_BackslashReplaceErrors", + "PyCodec_Decode", + "PyCodec_Decoder", + "PyCodec_Encode", + "PyCodec_Encoder", + "PyCodec_IgnoreErrors", + "PyCodec_IncrementalDecoder", + "PyCodec_IncrementalEncoder", + "PyCodec_KnownEncoding", + "PyCodec_LookupError", + "PyCodec_NameReplaceErrors", + "PyCodec_Register", + "PyCodec_RegisterError", + "PyCodec_ReplaceErrors", + "PyCodec_StreamReader", + "PyCodec_StreamWriter", + "PyCodec_StrictErrors", + "PyCodec_Unregister", + "PyCodec_XMLCharRefReplaceErrors", + "PyComplex_FromDoubles", + "PyComplex_ImagAsDouble", + "PyComplex_RealAsDouble", + "PyComplex_Type", + "PyDescr_NewClassMethod", + "PyDescr_NewGetSet", + "PyDescr_NewMember", + "PyDescr_NewMethod", + "PyDictItems_Type", + "PyDictIterItem_Type", + "PyDictIterKey_Type", + "PyDictIterValue_Type", + "PyDictKeys_Type", + "PyDictProxy_New", + "PyDictProxy_Type", + "PyDictRevIterItem_Type", + "PyDictRevIterKey_Type", + "PyDictRevIterValue_Type", + "PyDictValues_Type", + "PyDict_Clear", + "PyDict_Contains", + "PyDict_Copy", + "PyDict_DelItem", + "PyDict_DelItemString", + "PyDict_GetItem", + "PyDict_GetItemRef", + "PyDict_GetItemString", + "PyDict_GetItemStringRef", + "PyDict_GetItemWithError", + "PyDict_Items", + "PyDict_Keys", + "PyDict_Merge", + "PyDict_MergeFromSeq2", + "PyDict_New", + "PyDict_Next", + "PyDict_SetItem", + "PyDict_SetItemString", + "PyDict_Size", + "PyDict_Type", + "PyDict_Update", + "PyDict_Values", + "PyEllipsis_Type", + "PyEnum_Type", + "PyErr_BadArgument", + "PyErr_BadInternalCall", + "PyErr_CheckSignals", + "PyErr_Clear", + "PyErr_Display", + "PyErr_DisplayException", + "PyErr_ExceptionMatches", + "PyErr_Fetch", + "PyErr_Format", + "PyErr_FormatV", + "PyErr_GetExcInfo", + "PyErr_GetHandledException", + "PyErr_GetRaisedException", + "PyErr_GivenExceptionMatches", + "PyErr_NewException", + "PyErr_NewExceptionWithDoc", + "PyErr_NoMemory", + "PyErr_NormalizeException", + "PyErr_Occurred", + "PyErr_Print", + "PyErr_PrintEx", + "PyErr_ProgramText", + "PyErr_ResourceWarning", + "PyErr_Restore", + "PyErr_SetExcInfo", + "PyErr_SetFromErrno", + "PyErr_SetFromErrnoWithFilename", + "PyErr_SetFromErrnoWithFilenameObject", + "PyErr_SetFromErrnoWithFilenameObjects", + "PyErr_SetHandledException", + "PyErr_SetImportError", + "PyErr_SetImportErrorSubclass", + "PyErr_SetInterrupt", + "PyErr_SetInterruptEx", + "PyErr_SetNone", + "PyErr_SetObject", + "PyErr_SetRaisedException", + "PyErr_SetString", + "PyErr_SyntaxLocation", + "PyErr_SyntaxLocationEx", + "PyErr_WarnEx", + "PyErr_WarnExplicit", + "PyErr_WarnFormat", + "PyErr_WriteUnraisable", + "PyEval_AcquireLock", + "PyEval_AcquireThread", + "PyEval_CallFunction", + "PyEval_CallMethod", + "PyEval_CallObjectWithKeywords", + "PyEval_EvalCode", + "PyEval_EvalCodeEx", + "PyEval_EvalFrame", + "PyEval_EvalFrameEx", + "PyEval_GetBuiltins", + "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", + "PyEval_GetFuncDesc", + "PyEval_GetFuncName", + "PyEval_GetGlobals", + "PyEval_GetLocals", + "PyEval_InitThreads", + "PyEval_ReleaseLock", + "PyEval_ReleaseThread", + "PyEval_RestoreThread", + "PyEval_SaveThread", + "PyEval_ThreadsInitialized", + "PyExc_ArithmeticError", + "PyExc_AssertionError", + "PyExc_AttributeError", + "PyExc_BaseException", + "PyExc_BaseExceptionGroup", + "PyExc_BlockingIOError", + "PyExc_BrokenPipeError", + "PyExc_BufferError", + "PyExc_BytesWarning", + "PyExc_ChildProcessError", + "PyExc_ConnectionAbortedError", + "PyExc_ConnectionError", + "PyExc_ConnectionRefusedError", + "PyExc_ConnectionResetError", + "PyExc_DeprecationWarning", + "PyExc_EOFError", + "PyExc_EncodingWarning", + "PyExc_EnvironmentError", + "PyExc_Exception", + "PyExc_FileExistsError", + "PyExc_FileNotFoundError", + "PyExc_FloatingPointError", + "PyExc_FutureWarning", + "PyExc_GeneratorExit", + "PyExc_IOError", + "PyExc_ImportError", + "PyExc_ImportWarning", + "PyExc_IndentationError", + "PyExc_IndexError", + "PyExc_InterruptedError", + "PyExc_IsADirectoryError", + "PyExc_KeyError", + "PyExc_KeyboardInterrupt", + "PyExc_LookupError", + "PyExc_MemoryError", + "PyExc_ModuleNotFoundError", + "PyExc_NameError", + "PyExc_NotADirectoryError", + "PyExc_NotImplementedError", + "PyExc_OSError", + "PyExc_OverflowError", + "PyExc_PendingDeprecationWarning", + "PyExc_PermissionError", + "PyExc_ProcessLookupError", + "PyExc_RecursionError", + "PyExc_ReferenceError", + "PyExc_ResourceWarning", + "PyExc_RuntimeError", + "PyExc_RuntimeWarning", + "PyExc_StopAsyncIteration", + "PyExc_StopIteration", + "PyExc_SyntaxError", + "PyExc_SyntaxWarning", + "PyExc_SystemError", + "PyExc_SystemExit", + "PyExc_TabError", + "PyExc_TimeoutError", + "PyExc_TypeError", + "PyExc_UnboundLocalError", + "PyExc_UnicodeDecodeError", + "PyExc_UnicodeEncodeError", + "PyExc_UnicodeError", + "PyExc_UnicodeTranslateError", + "PyExc_UnicodeWarning", + "PyExc_UserWarning", + "PyExc_ValueError", + "PyExc_Warning", + "PyExc_ZeroDivisionError", + "PyExceptionClass_Name", + "PyException_GetArgs", + "PyException_GetCause", + "PyException_GetContext", + "PyException_GetTraceback", + "PyException_SetArgs", + "PyException_SetCause", + "PyException_SetContext", + "PyException_SetTraceback", + "PyFile_FromFd", + "PyFile_GetLine", + "PyFile_WriteObject", + "PyFile_WriteString", + "PyFilter_Type", + "PyFloat_AsDouble", + "PyFloat_FromDouble", + "PyFloat_FromString", + "PyFloat_GetInfo", + "PyFloat_GetMax", + "PyFloat_GetMin", + "PyFloat_Type", + "PyFrame_GetCode", + "PyFrame_GetLineNumber", + "PyFrozenSet_New", + "PyFrozenSet_Type", + "PyGC_Collect", + "PyGC_Disable", + "PyGC_Enable", + "PyGC_IsEnabled", + "PyGILState_Ensure", + "PyGILState_GetThisThreadState", + "PyGILState_Release", + "PyGetSetDescr_Type", + "PyImport_AddModule", + "PyImport_AddModuleObject", + "PyImport_AddModuleRef", + "PyImport_AppendInittab", + "PyImport_ExecCodeModule", + "PyImport_ExecCodeModuleEx", + "PyImport_ExecCodeModuleObject", + "PyImport_ExecCodeModuleWithPathnames", + "PyImport_GetImporter", + "PyImport_GetMagicNumber", + "PyImport_GetMagicTag", + "PyImport_GetModule", + "PyImport_GetModuleDict", + "PyImport_Import", + "PyImport_ImportFrozenModule", + "PyImport_ImportFrozenModuleObject", + "PyImport_ImportModule", + "PyImport_ImportModuleLevel", + "PyImport_ImportModuleLevelObject", + "PyImport_ImportModuleNoBlock", + "PyImport_ReloadModule", + "PyIndex_Check", + "PyInterpreterState_Clear", + "PyInterpreterState_Delete", + "PyInterpreterState_Get", + "PyInterpreterState_GetDict", + "PyInterpreterState_GetID", + "PyInterpreterState_New", + "PyIter_Check", + "PyIter_Next", + "PyIter_NextItem", + "PyIter_Send", + "PyListIter_Type", + "PyListRevIter_Type", + "PyList_Append", + "PyList_AsTuple", + "PyList_GetItem", + "PyList_GetItemRef", + "PyList_GetSlice", + "PyList_Insert", + "PyList_New", + "PyList_Reverse", + "PyList_SetItem", + "PyList_SetSlice", + "PyList_Size", + "PyList_Sort", + "PyList_Type", + "PyLongRangeIter_Type", + "PyLong_AsDouble", + "PyLong_AsInt", + "PyLong_AsInt32", + "PyLong_AsInt64", + "PyLong_AsLong", + "PyLong_AsLongAndOverflow", + "PyLong_AsLongLong", + "PyLong_AsLongLongAndOverflow", + "PyLong_AsNativeBytes", + "PyLong_AsSize_t", + "PyLong_AsSsize_t", + "PyLong_AsUInt32", + "PyLong_AsUInt64", + "PyLong_AsUnsignedLong", + "PyLong_AsUnsignedLongLong", + "PyLong_AsUnsignedLongLongMask", + "PyLong_AsUnsignedLongMask", + "PyLong_AsVoidPtr", + "PyLong_FromDouble", + "PyLong_FromInt32", + "PyLong_FromInt64", + "PyLong_FromLong", + "PyLong_FromLongLong", + "PyLong_FromNativeBytes", + "PyLong_FromSize_t", + "PyLong_FromSsize_t", + "PyLong_FromString", + "PyLong_FromUInt32", + "PyLong_FromUInt64", + "PyLong_FromUnsignedLong", + "PyLong_FromUnsignedLongLong", + "PyLong_FromUnsignedNativeBytes", + "PyLong_FromVoidPtr", + "PyLong_GetInfo", + "PyLong_Type", + "PyMap_Type", + "PyMapping_Check", + "PyMapping_GetItemString", + "PyMapping_GetOptionalItem", + "PyMapping_GetOptionalItemString", + "PyMapping_HasKey", + "PyMapping_HasKeyString", + "PyMapping_HasKeyStringWithError", + "PyMapping_HasKeyWithError", + "PyMapping_Items", + "PyMapping_Keys", + "PyMapping_Length", + "PyMapping_SetItemString", + "PyMapping_Size", + "PyMapping_Values", + "PyMarshal_ReadObjectFromString", + "PyMarshal_WriteObjectToString", + "PyMem_Calloc", + "PyMem_Free", + "PyMem_Malloc", + "PyMem_RawCalloc", + "PyMem_RawFree", + "PyMem_RawMalloc", + "PyMem_RawRealloc", + "PyMem_Realloc", + "PyMemberDescr_Type", + "PyMember_GetOne", + "PyMember_SetOne", + "PyMemoryView_FromBuffer", + "PyMemoryView_FromMemory", + "PyMemoryView_FromObject", + "PyMemoryView_GetContiguous", + "PyMemoryView_Type", + "PyMethodDescr_Type", + "PyModuleDef_Init", + "PyModuleDef_Type", + "PyModule_Add", + "PyModule_AddFunctions", + "PyModule_AddIntConstant", + "PyModule_AddObject", + "PyModule_AddObjectRef", + "PyModule_AddStringConstant", + "PyModule_AddType", + "PyModule_Create2", + "PyModule_ExecDef", + "PyModule_FromDefAndSpec2", + "PyModule_GetDef", + "PyModule_GetDict", + "PyModule_GetFilename", + "PyModule_GetFilenameObject", + "PyModule_GetName", + "PyModule_GetNameObject", + "PyModule_GetState", + "PyModule_New", + "PyModule_NewObject", + "PyModule_SetDocString", + "PyModule_Type", + "PyNumber_Absolute", + "PyNumber_Add", + "PyNumber_And", + "PyNumber_AsSsize_t", + "PyNumber_Check", + "PyNumber_Divmod", + "PyNumber_Float", + "PyNumber_FloorDivide", + "PyNumber_InPlaceAdd", + "PyNumber_InPlaceAnd", + "PyNumber_InPlaceFloorDivide", + "PyNumber_InPlaceLshift", + "PyNumber_InPlaceMatrixMultiply", + "PyNumber_InPlaceMultiply", + "PyNumber_InPlaceOr", + "PyNumber_InPlacePower", + "PyNumber_InPlaceRemainder", + "PyNumber_InPlaceRshift", + "PyNumber_InPlaceSubtract", + "PyNumber_InPlaceTrueDivide", + "PyNumber_InPlaceXor", + "PyNumber_Index", + "PyNumber_Invert", + "PyNumber_Long", + "PyNumber_Lshift", + "PyNumber_MatrixMultiply", + "PyNumber_Multiply", + "PyNumber_Negative", + "PyNumber_Or", + "PyNumber_Positive", + "PyNumber_Power", + "PyNumber_Remainder", + "PyNumber_Rshift", + "PyNumber_Subtract", + "PyNumber_ToBase", + "PyNumber_TrueDivide", + "PyNumber_Xor", + "PyOS_FSPath", + "PyOS_InputHook", + "PyOS_InterruptOccurred", + "PyOS_double_to_string", + "PyOS_getsig", + "PyOS_mystricmp", + "PyOS_mystrnicmp", + "PyOS_setsig", + "PyOS_snprintf", + "PyOS_string_to_double", + "PyOS_strtol", + "PyOS_strtoul", + "PyOS_vsnprintf", + "PyObject_ASCII", + "PyObject_AsCharBuffer", + "PyObject_AsFileDescriptor", + "PyObject_AsReadBuffer", + "PyObject_AsWriteBuffer", + "PyObject_Bytes", + "PyObject_Call", + "PyObject_CallFunction", + "PyObject_CallFunctionObjArgs", + "PyObject_CallMethod", + "PyObject_CallMethodObjArgs", + "PyObject_CallNoArgs", + "PyObject_CallObject", + "PyObject_Calloc", + "PyObject_CheckBuffer", + "PyObject_CheckReadBuffer", + "PyObject_ClearWeakRefs", + "PyObject_CopyData", + "PyObject_DelAttr", + "PyObject_DelAttrString", + "PyObject_DelItem", + "PyObject_DelItemString", + "PyObject_Dir", + "PyObject_Format", + "PyObject_Free", + "PyObject_GC_Del", + "PyObject_GC_IsFinalized", + "PyObject_GC_IsTracked", + "PyObject_GC_Track", + "PyObject_GC_UnTrack", + "PyObject_GenericGetAttr", + "PyObject_GenericGetDict", + "PyObject_GenericSetAttr", + "PyObject_GenericSetDict", + "PyObject_GetAIter", + "PyObject_GetAttr", + "PyObject_GetAttrString", + "PyObject_GetBuffer", + "PyObject_GetItem", + "PyObject_GetIter", + "PyObject_GetOptionalAttr", + "PyObject_GetOptionalAttrString", + "PyObject_GetTypeData", + "PyObject_HasAttr", + "PyObject_HasAttrString", + "PyObject_HasAttrStringWithError", + "PyObject_HasAttrWithError", + "PyObject_Hash", + "PyObject_HashNotImplemented", + "PyObject_Init", + "PyObject_InitVar", + "PyObject_IsInstance", + "PyObject_IsSubclass", + "PyObject_IsTrue", + "PyObject_Length", + "PyObject_Malloc", + "PyObject_Not", + "PyObject_Realloc", + "PyObject_Repr", + "PyObject_RichCompare", + "PyObject_RichCompareBool", + "PyObject_SelfIter", + "PyObject_SetAttr", + "PyObject_SetAttrString", + "PyObject_SetItem", + "PyObject_Size", + "PyObject_Str", + "PyObject_Type", + "PyObject_Vectorcall", + "PyObject_VectorcallMethod", + "PyProperty_Type", + "PyRangeIter_Type", + "PyRange_Type", + "PyReversed_Type", + "PySeqIter_New", + "PySeqIter_Type", + "PySequence_Check", + "PySequence_Concat", + "PySequence_Contains", + "PySequence_Count", + "PySequence_DelItem", + "PySequence_DelSlice", + "PySequence_Fast", + "PySequence_GetItem", + "PySequence_GetSlice", + "PySequence_In", + "PySequence_InPlaceConcat", + "PySequence_InPlaceRepeat", + "PySequence_Index", + "PySequence_Length", + "PySequence_List", + "PySequence_Repeat", + "PySequence_SetItem", + "PySequence_SetSlice", + "PySequence_Size", + "PySequence_Tuple", + "PySetIter_Type", + "PySet_Add", + "PySet_Clear", + "PySet_Contains", + "PySet_Discard", + "PySet_New", + "PySet_Pop", + "PySet_Size", + "PySet_Type", + "PySlice_AdjustIndices", + "PySlice_GetIndices", + "PySlice_GetIndicesEx", + "PySlice_New", + "PySlice_Type", + "PySlice_Unpack", + "PyState_AddModule", + "PyState_FindModule", + "PyState_RemoveModule", + "PyStructSequence_GetItem", + "PyStructSequence_New", + "PyStructSequence_NewType", + "PyStructSequence_SetItem", + "PyStructSequence_UnnamedField", + "PySuper_Type", + "PySys_AddWarnOption", + "PySys_AddWarnOptionUnicode", + "PySys_AddXOption", + "PySys_Audit", + "PySys_AuditTuple", + "PySys_FormatStderr", + "PySys_FormatStdout", + "PySys_GetObject", + "PySys_GetXOptions", + "PySys_HasWarnOptions", + "PySys_ResetWarnOptions", + "PySys_SetArgv", + "PySys_SetArgvEx", + "PySys_SetObject", + "PySys_SetPath", + "PySys_WriteStderr", + "PySys_WriteStdout", + "PyThreadState_Clear", + "PyThreadState_Delete", + "PyThreadState_DeleteCurrent", + "PyThreadState_Get", + "PyThreadState_GetDict", + "PyThreadState_GetFrame", + "PyThreadState_GetID", + "PyThreadState_GetInterpreter", + "PyThreadState_New", + "PyThreadState_SetAsyncExc", + "PyThreadState_Swap", + "PyThread_GetInfo", + "PyThread_ReInitTLS", + "PyThread_acquire_lock", + "PyThread_acquire_lock_timed", + "PyThread_allocate_lock", + "PyThread_create_key", + "PyThread_delete_key", + "PyThread_delete_key_value", + "PyThread_exit_thread", + "PyThread_free_lock", + "PyThread_get_key_value", + "PyThread_get_stacksize", + "PyThread_get_thread_ident", + "PyThread_init_thread", + "PyThread_release_lock", + "PyThread_set_key_value", + "PyThread_set_stacksize", + "PyThread_start_new_thread", + "PyThread_tss_alloc", + "PyThread_tss_create", + "PyThread_tss_delete", + "PyThread_tss_free", + "PyThread_tss_get", + "PyThread_tss_is_created", + "PyThread_tss_set", + "PyTraceBack_Here", + "PyTraceBack_Print", + "PyTraceBack_Type", + "PyTupleIter_Type", + "PyTuple_GetItem", + "PyTuple_GetSlice", + "PyTuple_New", + "PyTuple_Pack", + "PyTuple_SetItem", + "PyTuple_Size", + "PyTuple_Type", + "PyType_ClearCache", + "PyType_Freeze", + "PyType_FromMetaclass", + "PyType_FromModuleAndSpec", + "PyType_FromSpec", + "PyType_FromSpecWithBases", + "PyType_GenericAlloc", + "PyType_GenericNew", + "PyType_GetBaseByToken", + "PyType_GetFlags", + "PyType_GetFullyQualifiedName", + "PyType_GetModule", + "PyType_GetModuleByDef", + "PyType_GetModuleName", + "PyType_GetModuleState", + "PyType_GetName", + "PyType_GetQualName", + "PyType_GetSlot", + "PyType_GetTypeDataSize", + "PyType_IsSubtype", + "PyType_Modified", + "PyType_Ready", + "PyType_Type", + "PyUnicodeDecodeError_Create", + "PyUnicodeDecodeError_GetEncoding", + "PyUnicodeDecodeError_GetEnd", + "PyUnicodeDecodeError_GetObject", + "PyUnicodeDecodeError_GetReason", + "PyUnicodeDecodeError_GetStart", + "PyUnicodeDecodeError_SetEnd", + "PyUnicodeDecodeError_SetReason", + "PyUnicodeDecodeError_SetStart", + "PyUnicodeEncodeError_GetEncoding", + "PyUnicodeEncodeError_GetEnd", + "PyUnicodeEncodeError_GetObject", + "PyUnicodeEncodeError_GetReason", + "PyUnicodeEncodeError_GetStart", + "PyUnicodeEncodeError_SetEnd", + "PyUnicodeEncodeError_SetReason", + "PyUnicodeEncodeError_SetStart", + "PyUnicodeIter_Type", + "PyUnicodeTranslateError_GetEnd", + "PyUnicodeTranslateError_GetObject", + "PyUnicodeTranslateError_GetReason", + "PyUnicodeTranslateError_GetStart", + "PyUnicodeTranslateError_SetEnd", + "PyUnicodeTranslateError_SetReason", + "PyUnicodeTranslateError_SetStart", + "PyUnicode_Append", + "PyUnicode_AppendAndDel", + "PyUnicode_AsASCIIString", + "PyUnicode_AsCharmapString", + "PyUnicode_AsDecodedObject", + "PyUnicode_AsDecodedUnicode", + "PyUnicode_AsEncodedObject", + "PyUnicode_AsEncodedString", + "PyUnicode_AsEncodedUnicode", + "PyUnicode_AsLatin1String", + "PyUnicode_AsRawUnicodeEscapeString", + "PyUnicode_AsUCS4", + "PyUnicode_AsUCS4Copy", + "PyUnicode_AsUTF16String", + "PyUnicode_AsUTF32String", + "PyUnicode_AsUTF8AndSize", + "PyUnicode_AsUTF8String", + "PyUnicode_AsUnicodeEscapeString", + "PyUnicode_AsWideChar", + "PyUnicode_AsWideCharString", + "PyUnicode_BuildEncodingMap", + "PyUnicode_Compare", + "PyUnicode_CompareWithASCIIString", + "PyUnicode_Concat", + "PyUnicode_Contains", + "PyUnicode_Count", + "PyUnicode_Decode", + "PyUnicode_DecodeASCII", + "PyUnicode_DecodeCharmap", + "PyUnicode_DecodeFSDefault", + "PyUnicode_DecodeFSDefaultAndSize", + "PyUnicode_DecodeLatin1", + "PyUnicode_DecodeLocale", + "PyUnicode_DecodeLocaleAndSize", + "PyUnicode_DecodeRawUnicodeEscape", + "PyUnicode_DecodeUTF16", + "PyUnicode_DecodeUTF16Stateful", + "PyUnicode_DecodeUTF32", + "PyUnicode_DecodeUTF32Stateful", + "PyUnicode_DecodeUTF7", + "PyUnicode_DecodeUTF7Stateful", + "PyUnicode_DecodeUTF8", + "PyUnicode_DecodeUTF8Stateful", + "PyUnicode_DecodeUnicodeEscape", + "PyUnicode_EncodeFSDefault", + "PyUnicode_EncodeLocale", + "PyUnicode_Equal", + "PyUnicode_EqualToUTF8", + "PyUnicode_EqualToUTF8AndSize", + "PyUnicode_FSConverter", + "PyUnicode_FSDecoder", + "PyUnicode_Find", + "PyUnicode_FindChar", + "PyUnicode_Format", + "PyUnicode_FromEncodedObject", + "PyUnicode_FromFormat", + "PyUnicode_FromFormatV", + "PyUnicode_FromObject", + "PyUnicode_FromOrdinal", + "PyUnicode_FromString", + "PyUnicode_FromStringAndSize", + "PyUnicode_FromWideChar", + "PyUnicode_GetDefaultEncoding", + "PyUnicode_GetLength", + "PyUnicode_GetSize", + "PyUnicode_InternFromString", + "PyUnicode_InternImmortal", + "PyUnicode_InternInPlace", + "PyUnicode_IsIdentifier", + "PyUnicode_Join", + "PyUnicode_Partition", + "PyUnicode_RPartition", + "PyUnicode_RSplit", + "PyUnicode_ReadChar", + "PyUnicode_Replace", + "PyUnicode_Resize", + "PyUnicode_RichCompare", + "PyUnicode_Split", + "PyUnicode_Splitlines", + "PyUnicode_Substring", + "PyUnicode_Tailmatch", + "PyUnicode_Translate", + "PyUnicode_Type", + "PyUnicode_WriteChar", + "PyVectorcall_Call", + "PyVectorcall_NARGS", + "PyWeakref_GetObject", + "PyWeakref_GetRef", + "PyWeakref_NewProxy", + "PyWeakref_NewRef", + "PyWrapperDescr_Type", + "PyWrapper_New", + "PyZip_Type", + "Py_AddPendingCall", + "Py_AtExit", + "Py_BuildValue", + "Py_BytesMain", + "Py_CompileString", + "Py_DecRef", + "Py_DecodeLocale", + "Py_EncodeLocale", + "Py_EndInterpreter", + "Py_EnterRecursiveCall", + "Py_Exit", + "Py_FatalError", + "Py_FileSystemDefaultEncodeErrors", + "Py_FileSystemDefaultEncoding", + "Py_Finalize", + "Py_FinalizeEx", + "Py_GenericAlias", + "Py_GenericAliasType", + "Py_GetArgcArgv", + "Py_GetBuildInfo", + "Py_GetCompiler", + "Py_GetConstant", + "Py_GetConstantBorrowed", + "Py_GetCopyright", + "Py_GetExecPrefix", + "Py_GetPath", + "Py_GetPlatform", + "Py_GetPrefix", + "Py_GetProgramFullPath", + "Py_GetProgramName", + "Py_GetPythonHome", + "Py_GetRecursionLimit", + "Py_GetVersion", + "Py_HasFileSystemDefaultEncoding", + "Py_IncRef", + "Py_Initialize", + "Py_InitializeEx", + "Py_Is", + "Py_IsFalse", + "Py_IsFinalizing", + "Py_IsInitialized", + "Py_IsNone", + "Py_IsTrue", + "Py_LeaveRecursiveCall", + "Py_Main", + "Py_MakePendingCalls", + "Py_NewInterpreter", + "Py_NewRef", + "Py_PACK_FULL_VERSION", + "Py_PACK_VERSION", + "Py_REFCNT", + "Py_ReprEnter", + "Py_ReprLeave", + "Py_SetPath", + "Py_SetProgramName", + "Py_SetPythonHome", + "Py_SetRecursionLimit", + "Py_TYPE", + "Py_UTF8Mode", + "Py_VaBuildValue", + "Py_Version", + "Py_XNewRef", + "_PyArg_ParseTupleAndKeywords_SizeT", + "_PyArg_ParseTuple_SizeT", + "_PyArg_Parse_SizeT", + "_PyArg_VaParseTupleAndKeywords_SizeT", + "_PyArg_VaParse_SizeT", + "_PyErr_BadInternalCall", + "_PyObject_CallFunction_SizeT", + "_PyObject_CallMethod_SizeT", + "_PyObject_GC_New", + "_PyObject_GC_NewVar", + "_PyObject_GC_Resize", + "_PyObject_New", + "_PyObject_NewVar", + "_PyState_AddModule", + "_PyThreadState_Init", + "_PyThreadState_Prealloc", + "_PyWeakref_CallableProxyType", + "_PyWeakref_ProxyType", + "_PyWeakref_RefType", + "_Py_BuildValue_SizeT", + "_Py_CheckRecursiveCall", + "_Py_Dealloc", + "_Py_DecRef", + "_Py_EllipsisObject", + "_Py_FalseStruct", + "_Py_IncRef", + "_Py_NoneStruct", + "_Py_NotImplementedStruct", + "_Py_SetRefcnt", + "_Py_SwappedOp", + "_Py_TrueStruct", + "_Py_VaBuildValue_SizeT", +) +if feature_macros['HAVE_FORK']: + SYMBOL_NAMES += ( + 'PyOS_AfterFork', + 'PyOS_AfterFork_Child', + 'PyOS_AfterFork_Parent', + 'PyOS_BeforeFork', + ) +if feature_macros['MS_WINDOWS']: + SYMBOL_NAMES += ( + 'PyErr_SetExcFromWindowsErr', + 'PyErr_SetExcFromWindowsErrWithFilename', + 'PyErr_SetExcFromWindowsErrWithFilenameObject', + 'PyErr_SetExcFromWindowsErrWithFilenameObjects', + 'PyErr_SetFromWindowsErr', + 'PyErr_SetFromWindowsErrWithFilename', + 'PyExc_WindowsError', + 'PyUnicode_AsMBCSString', + 'PyUnicode_DecodeCodePageStateful', + 'PyUnicode_DecodeMBCS', + 'PyUnicode_DecodeMBCSStateful', + 'PyUnicode_EncodeCodePage', + ) +if feature_macros['PY_HAVE_THREAD_NATIVE_ID']: + SYMBOL_NAMES += ( + 'PyThread_get_thread_native_id', + ) +if feature_macros['Py_REF_DEBUG']: + SYMBOL_NAMES += ( + '_Py_NegativeRefcount', + '_Py_RefTotal', + ) +if feature_macros['Py_TRACE_REFS']: + SYMBOL_NAMES += ( + ) +if feature_macros['USE_STACKCHECK']: + SYMBOL_NAMES += ( + 'PyOS_CheckStack', + ) + +EXPECTED_FEATURE_MACROS = set(['HAVE_FORK', + 'MS_WINDOWS', + 'PY_HAVE_THREAD_NATIVE_ID', + 'Py_REF_DEBUG', + 'Py_TRACE_REFS', + 'USE_STACKCHECK']) +WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False, + 'MS_WINDOWS': True, + 'PY_HAVE_THREAD_NATIVE_ID': True, + 'Py_REF_DEBUG': 'maybe', + 'Py_TRACE_REFS': 'maybe', + 'USE_STACKCHECK': 'maybe'} diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index c1edea8491a..a83f7d076f0 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -2,8 +2,7 @@ import os import socket import sys -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from test.support.import_helper import import_fresh_module from test.support.os_helper import TESTFN @@ -15,8 +14,10 @@ class TestFilemode: statmod = None file_flags = {'SF_APPEND', 'SF_ARCHIVED', 'SF_IMMUTABLE', 'SF_NOUNLINK', - 'SF_SNAPSHOT', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', - 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} + 'SF_SNAPSHOT', 'SF_SETTABLE', 'SF_RESTRICTED', 'SF_FIRMLINK', + 'SF_DATALESS', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', + 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE', + 'UF_SETTABLE', 'UF_TRACKED', 'UF_DATAVAULT'} formats = {'S_IFBLK', 'S_IFCHR', 'S_IFDIR', 'S_IFIFO', 'S_IFLNK', 'S_IFREG', 'S_IFSOCK', 'S_IFDOOR', 'S_IFPORT', 'S_IFWHT'} @@ -113,6 +114,7 @@ def assertS_IS(self, name, mode): else: self.assertFalse(func(mode)) + @os_helper.skip_unless_working_chmod def test_mode(self): with open(TESTFN, 'w'): pass @@ -121,8 +123,11 @@ def test_mode(self): st_mode, modestr = self.get_mode() self.assertEqual(modestr, '-rwx------') self.assertS_IS("REG", st_mode) - self.assertEqual(self.statmod.S_IMODE(st_mode), + imode = self.statmod.S_IMODE(st_mode) + self.assertEqual(imode, self.statmod.S_IRWXU) + self.assertEqual(self.statmod.filemode(imode), + '?rwx------') os.chmod(TESTFN, 0o070) st_mode, modestr = self.get_mode() @@ -144,13 +149,26 @@ def test_mode(self): self.assertEqual(modestr, '-r--r--r--') self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) else: + os.chmod(TESTFN, 0o500) + st_mode, modestr = self.get_mode() + self.assertEqual(modestr[:3], '-r-') + self.assertS_IS("REG", st_mode) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) + os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() - self.assertEqual(modestr[:3], '-rw') + self.assertStartsWith(modestr, '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o666) + + def test_filemode_does_not_misclassify_random_bits(self): + # gh-144050 regression test + self.assertEqual(self.statmod.filemode(0o77777)[0], "?") + self.assertEqual(self.statmod.filemode(0o177777)[0], "?") + @os_helper.skip_unless_working_chmod def test_directory(self): os.mkdir(TESTFN) os.chmod(TESTFN, 0o700) @@ -161,7 +179,7 @@ def test_directory(self): else: self.assertEqual(modestr[0], 'd') - @unittest.skipUnless(hasattr(os, 'symlink'), 'os.symlink not available') + @os_helper.skip_unless_symlink def test_link(self): try: os.symlink(os.getcwd(), TESTFN) @@ -227,63 +245,95 @@ def test_module_attributes(self): self.assertTrue(callable(func)) self.assertEqual(func(0), 0) + def test_flags_consistent(self): + self.assertFalse(self.statmod.UF_SETTABLE & self.statmod.SF_SETTABLE) + + for flag in self.file_flags: + if flag.startswith("UF"): + self.assertTrue(getattr(self.statmod, flag) & self.statmod.UF_SETTABLE, f"{flag} not in UF_SETTABLE") + elif is_apple and self.statmod is c_stat and flag == 'SF_DATALESS': + self.assertTrue(self.statmod.SF_DATALESS & self.statmod.SF_SYNTHETIC, "SF_DATALESS not in SF_SYNTHETIC") + self.assertFalse(self.statmod.SF_DATALESS & self.statmod.SF_SETTABLE, "SF_DATALESS in SF_SETTABLE") + else: + self.assertTrue(getattr(self.statmod, flag) & self.statmod.SF_SETTABLE, f"{flag} notin SF_SETTABLE") + @unittest.skipUnless(sys.platform == "win32", "FILE_ATTRIBUTE_* constants are Win32 specific") def test_file_attribute_constants(self): for key, value in sorted(self.file_attributes.items()): - self.assertTrue(hasattr(self.statmod, key), key) + self.assertHasAttr(self.statmod, key) modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) - + @unittest.skipUnless(sys.platform == "darwin", "macOS system check") + def test_macosx_attribute_values(self): + self.assertEqual(self.statmod.UF_SETTABLE, 0x0000ffff) + self.assertEqual(self.statmod.UF_NODUMP, 0x00000001) + self.assertEqual(self.statmod.UF_IMMUTABLE, 0x00000002) + self.assertEqual(self.statmod.UF_APPEND, 0x00000004) + self.assertEqual(self.statmod.UF_OPAQUE, 0x00000008) + self.assertEqual(self.statmod.UF_COMPRESSED, 0x00000020) + self.assertEqual(self.statmod.UF_TRACKED, 0x00000040) + self.assertEqual(self.statmod.UF_DATAVAULT, 0x00000080) + self.assertEqual(self.statmod.UF_HIDDEN, 0x00008000) + + if self.statmod is c_stat: + self.assertEqual(self.statmod.SF_SUPPORTED, 0x009f0000) + self.assertEqual(self.statmod.SF_SETTABLE, 0x3fff0000) + self.assertEqual(self.statmod.SF_SYNTHETIC, 0xc0000000) + else: + self.assertEqual(self.statmod.SF_SETTABLE, 0xffff0000) + self.assertEqual(self.statmod.SF_ARCHIVED, 0x00010000) + self.assertEqual(self.statmod.SF_IMMUTABLE, 0x00020000) + self.assertEqual(self.statmod.SF_APPEND, 0x00040000) + self.assertEqual(self.statmod.SF_RESTRICTED, 0x00080000) + self.assertEqual(self.statmod.SF_NOUNLINK, 0x00100000) + self.assertEqual(self.statmod.SF_FIRMLINK, 0x00800000) + self.assertEqual(self.statmod.SF_DATALESS, 0x40000000) + + self.assertFalse(isinstance(self.statmod.S_IFMT, int)) + self.assertEqual(self.statmod.S_IFIFO, 0o010000) + self.assertEqual(self.statmod.S_IFCHR, 0o020000) + self.assertEqual(self.statmod.S_IFDIR, 0o040000) + self.assertEqual(self.statmod.S_IFBLK, 0o060000) + self.assertEqual(self.statmod.S_IFREG, 0o100000) + self.assertEqual(self.statmod.S_IFLNK, 0o120000) + self.assertEqual(self.statmod.S_IFSOCK, 0o140000) + + if self.statmod is c_stat: + self.assertEqual(self.statmod.S_IFWHT, 0o160000) + + self.assertEqual(self.statmod.S_IRWXU, 0o000700) + self.assertEqual(self.statmod.S_IRUSR, 0o000400) + self.assertEqual(self.statmod.S_IWUSR, 0o000200) + self.assertEqual(self.statmod.S_IXUSR, 0o000100) + self.assertEqual(self.statmod.S_IRWXG, 0o000070) + self.assertEqual(self.statmod.S_IRGRP, 0o000040) + self.assertEqual(self.statmod.S_IWGRP, 0o000020) + self.assertEqual(self.statmod.S_IXGRP, 0o000010) + self.assertEqual(self.statmod.S_IRWXO, 0o000007) + self.assertEqual(self.statmod.S_IROTH, 0o000004) + self.assertEqual(self.statmod.S_IWOTH, 0o000002) + self.assertEqual(self.statmod.S_IXOTH, 0o000001) + self.assertEqual(self.statmod.S_ISUID, 0o004000) + self.assertEqual(self.statmod.S_ISGID, 0o002000) + self.assertEqual(self.statmod.S_ISVTX, 0o001000) + + self.assertNotHasAttr(self.statmod, "S_ISTXT") + self.assertEqual(self.statmod.S_IREAD, self.statmod.S_IRUSR) + self.assertEqual(self.statmod.S_IWRITE, self.statmod.S_IWUSR) + self.assertEqual(self.statmod.S_IEXEC, self.statmod.S_IXUSR) + + + +@unittest.skipIf(c_stat is None, 'need _stat extension') class TestFilemodeCStat(TestFilemode, unittest.TestCase): statmod = c_stat - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_devices(self): - super().test_devices() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_directory(self): - super().test_directory() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_file_attribute_constants(self): - super().test_file_attribute_constants() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_link(self): - super().test_link() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_mode(self): - super().test_mode() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_module_attributes(self): - super().test_module_attributes() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_socket(self): - super().test_socket() - class TestFilemodePyStat(TestFilemode, unittest.TestCase): statmod = py_stat - # TODO: RUSTPYTHON - if sys.platform == "win32": - @unittest.expectedFailure - def test_link(self): - super().test_link() - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 8fcbcf35400..e0b74432c94 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1,4 +1,4 @@ -"""Test suite for statistics module, including helper NumericTestCase and +x = """Test suite for statistics module, including helper NumericTestCase and approx_equal function. """ @@ -9,13 +9,14 @@ import copy import decimal import doctest +import itertools import math import pickle import random import sys import unittest from test import support -from test.support import import_helper +from test.support import import_helper, requires_IEEE_754 from decimal import Decimal from fractions import Fraction @@ -27,6 +28,12 @@ # === Helper functions and class === +# Test copied from Lib/test/test_math.py +# detect evidence of double-rounding: fsum is not always correctly +# rounded on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + def sign(x): """Return -1.0 for negatives, including -0.0, otherwise +1.0.""" return math.copysign(1, x) @@ -638,7 +645,7 @@ def do_test(self, args): def test_numerictestcase_is_testcase(self): # Ensure that NumericTestCase actually is a TestCase. - self.assertTrue(issubclass(NumericTestCase, unittest.TestCase)) + self.assertIsSubclass(NumericTestCase, unittest.TestCase) def test_error_msg_numeric(self): # Test the error message generated for numeric comparisons. @@ -676,40 +683,23 @@ class GlobalsTest(unittest.TestCase): def test_meta(self): # Test for the existence of metadata. for meta in self.expected_metadata: - self.assertTrue(hasattr(self.module, meta), - "%s not present" % meta) + self.assertHasAttr(self.module, meta) def test_check_all(self): # Check everything in __all__ exists and is public. module = self.module for name in module.__all__: # No private names in __all__: - self.assertFalse(name.startswith("_"), + self.assertNotStartsWith(name, "_", 'private name "%s" in __all__' % name) # And anything in __all__ must exist: - self.assertTrue(hasattr(module, name), - 'missing name "%s" in __all__' % name) - + self.assertHasAttr(module, name) -class DocTests(unittest.TestCase): - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -OO and above") - def test_doc_tests(self): - failed, tried = doctest.testmod(statistics, optionflags=doctest.ELLIPSIS) - self.assertGreater(tried, 0) - self.assertEqual(failed, 0) class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): - errmsg = ( - "Expected StatisticsError to be a ValueError, but got a" - " subclass of %r instead." - ) - self.assertTrue(hasattr(statistics, 'StatisticsError')) - self.assertTrue( - issubclass(statistics.StatisticsError, ValueError), - errmsg % statistics.StatisticsError.__base__ - ) + self.assertHasAttr(statistics, 'StatisticsError') + self.assertIsSubclass(statistics.StatisticsError, ValueError) # === Tests for private utility functions === @@ -1039,50 +1029,6 @@ def test_error_msg(self): self.assertEqual(errmsg, msg) -class FindLteqTest(unittest.TestCase): - # Test _find_lteq private function. - - def test_invalid_input_values(self): - for a, x in [ - ([], 1), - ([1, 2], 3), - ([1, 3], 2) - ]: - with self.subTest(a=a, x=x): - with self.assertRaises(ValueError): - statistics._find_lteq(a, x) - - def test_locate_successfully(self): - for a, x, expected_i in [ - ([1, 1, 1, 2, 3], 1, 0), - ([0, 1, 1, 1, 2, 3], 1, 1), - ([1, 2, 3, 3, 3], 3, 2) - ]: - with self.subTest(a=a, x=x): - self.assertEqual(expected_i, statistics._find_lteq(a, x)) - - -class FindRteqTest(unittest.TestCase): - # Test _find_rteq private function. - - def test_invalid_input_values(self): - for a, l, x in [ - ([1], 2, 1), - ([1, 3], 0, 2) - ]: - with self.assertRaises(ValueError): - statistics._find_rteq(a, l, x) - - def test_locate_successfully(self): - for a, l, x, expected_i in [ - ([1, 1, 1, 2, 3], 0, 1, 2), - ([0, 1, 1, 1, 2, 3], 0, 1, 3), - ([1, 2, 3, 3, 3], 0, 3, 4) - ]: - with self.subTest(a=a, l=l, x=x): - self.assertEqual(expected_i, statistics._find_rteq(a, l, x)) - - # === Tests for public functions === class UnivariateCommonMixin: @@ -1117,7 +1063,7 @@ def test_no_inplace_modifications(self): def test_order_doesnt_matter(self): # Test that the order of data points doesn't change the result. - # CAUTION: due to floating point rounding errors, the result actually + # CAUTION: due to floating-point rounding errors, the result actually # may depend on the order. Consider this test representing an ideal. # To avoid this test failing, only test with exact values such as ints # or Fractions. @@ -1210,6 +1156,9 @@ def __pow__(self, other): def __add__(self, other): return type(self)(super().__add__(other)) __radd__ = __add__ + def __mul__(self, other): + return type(self)(super().__mul__(other)) + __rmul__ = __mul__ return (float, Decimal, Fraction, MyFloat) def test_types_conserved(self): @@ -1782,6 +1731,12 @@ def test_repeated_single_value(self): data = [x]*count self.assertEqual(self.func(data), float(x)) + def test_single_value(self): + # Override method from AverageMixin. + # Average of a single value is the value as a float. + for x in (23, 42.5, 1.3e15, Fraction(15, 19), Decimal('0.28')): + self.assertEqual(self.func([x]), float(x)) + def test_odd_fractions(self): # Test median_grouped works with an odd number of Fractions. F = Fraction @@ -1961,6 +1916,27 @@ def test_special_values(self): with self.assertRaises(ValueError): fmean([Inf, -Inf]) + def test_weights(self): + fmean = statistics.fmean + StatisticsError = statistics.StatisticsError + self.assertEqual( + fmean([10, 10, 10, 50], [0.25] * 4), + fmean([10, 10, 10, 50])) + self.assertEqual( + fmean([10, 10, 20], [0.25, 0.25, 0.50]), + fmean([10, 10, 20, 20])) + self.assertEqual( # inputs are iterators + fmean(iter([10, 10, 20]), iter([0.25, 0.25, 0.50])), + fmean([10, 10, 20, 20])) + with self.assertRaises(StatisticsError): + fmean([10, 20, 30], [1, 2]) # unequal lengths + with self.assertRaises(StatisticsError): + fmean(iter([10, 20, 30]), iter([1, 2])) # unequal lengths + with self.assertRaises(StatisticsError): + fmean([10, 20], [-1, 1]) # sum of weights is zero + with self.assertRaises(StatisticsError): + fmean(iter([10, 20]), iter([-1, 1])) # sum of weights is zero + # === Tests for variances and standard deviations === @@ -2029,7 +2005,6 @@ def test_iter_list_same(self): expected = self.func(data) self.assertEqual(self.func(iter(data)), expected) - class TestPVariance(VarianceStdevMixin, NumericTestCase, UnivariateTypeMixin): # Tests for population variance. def setUp(self): @@ -2137,6 +2112,112 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) + def test_gh_140938(self): + # Inputs with inf/nan should raise a ValueError + with self.assertRaises(ValueError): + self.func([1.0, math.inf]) + with self.assertRaises(ValueError): + self.func([1.0, math.nan]) + + +class TestSqrtHelpers(unittest.TestCase): + + def test_integer_sqrt_of_frac_rto(self): + for n, m in itertools.product(range(100), range(1, 1000)): + r = statistics._integer_sqrt_of_frac_rto(n, m) + self.assertIsInstance(r, int) + if r*r*m == n: + # Root is exact + continue + # Inexact, so the root should be odd + self.assertEqual(r&1, 1) + # Verify correct rounding + self.assertTrue(m * (r - 1)**2 < n < m * (r + 1)**2) + + @requires_IEEE_754 + @support.requires_resource('cpu') + def test_float_sqrt_of_frac(self): + + def is_root_correctly_rounded(x: Fraction, root: float) -> bool: + if not x: + return root == 0.0 + + # Extract adjacent representable floats + r_up: float = math.nextafter(root, math.inf) + r_down: float = math.nextafter(root, -math.inf) + assert r_down < root < r_up + + # Convert to fractions for exact arithmetic + frac_root: Fraction = Fraction(root) + half_way_up: Fraction = (frac_root + Fraction(r_up)) / 2 + half_way_down: Fraction = (frac_root + Fraction(r_down)) / 2 + + # Check a closed interval. + # Does not test for a midpoint rounding rule. + return half_way_down ** 2 <= x <= half_way_up ** 2 + + randrange = random.randrange + + for i in range(60_000): + numerator: int = randrange(10 ** randrange(50)) + denonimator: int = randrange(10 ** randrange(50)) + 1 + with self.subTest(numerator=numerator, denonimator=denonimator): + x: Fraction = Fraction(numerator, denonimator) + root: float = statistics._float_sqrt_of_frac(numerator, denonimator) + self.assertTrue(is_root_correctly_rounded(x, root)) + + # Verify that corner cases and error handling match math.sqrt() + self.assertEqual(statistics._float_sqrt_of_frac(0, 1), 0.0) + with self.assertRaises(ValueError): + statistics._float_sqrt_of_frac(-1, 1) + with self.assertRaises(ValueError): + statistics._float_sqrt_of_frac(1, -1) + + # Error handling for zero denominator matches that for Fraction(1, 0) + with self.assertRaises(ZeroDivisionError): + statistics._float_sqrt_of_frac(1, 0) + + # The result is well defined if both inputs are negative + self.assertEqual(statistics._float_sqrt_of_frac(-2, -1), statistics._float_sqrt_of_frac(2, 1)) + + def test_decimal_sqrt_of_frac(self): + root: Decimal + numerator: int + denominator: int + + for root, numerator, denominator in [ + (Decimal('0.4481904599041192673635338663'), 200874688349065940678243576378, 1000000000000000000000000000000), # No adj + (Decimal('0.7924949131383786609961759598'), 628048187350206338833590574929, 1000000000000000000000000000000), # Adj up + (Decimal('0.8500554152289934068192208727'), 722594208960136395984391238251, 1000000000000000000000000000000), # Adj down + ]: + with decimal.localcontext(decimal.DefaultContext): + self.assertEqual(statistics._decimal_sqrt_of_frac(numerator, denominator), root) + + # Confirm expected root with a quad precision decimal computation + with decimal.localcontext(decimal.DefaultContext) as ctx: + ctx.prec *= 4 + high_prec_ratio = Decimal(numerator) / Decimal(denominator) + ctx.rounding = decimal.ROUND_05UP + high_prec_root = high_prec_ratio.sqrt() + with decimal.localcontext(decimal.DefaultContext): + target_root = +high_prec_root + self.assertEqual(root, target_root) + + # Verify that corner cases and error handling match Decimal.sqrt() + self.assertEqual(statistics._decimal_sqrt_of_frac(0, 1), 0.0) + with self.assertRaises(decimal.InvalidOperation): + statistics._decimal_sqrt_of_frac(-1, 1) + with self.assertRaises(decimal.InvalidOperation): + statistics._decimal_sqrt_of_frac(1, -1) + + # Error handling for zero denominator matches that for Fraction(1, 0) + with self.assertRaises(ZeroDivisionError): + statistics._decimal_sqrt_of_frac(1, 0) + + # The result is well defined if both inputs are negative + self.assertEqual(statistics._decimal_sqrt_of_frac(-2, -1), statistics._decimal_sqrt_of_frac(2, 1)) + + class TestStdev(VarianceStdevMixin, NumericTestCase): # Tests for sample standard deviation. def setUp(self): @@ -2151,7 +2232,7 @@ def test_compare_to_variance(self): # Test that stdev is, in fact, the square root of variance. data = [random.uniform(-2, 9) for _ in range(1000)] expected = math.sqrt(statistics.variance(data)) - self.assertEqual(self.func(data), expected) + self.assertAlmostEqual(self.func(data), expected) def test_center_not_at_mean(self): data = (1.0, 2.0) @@ -2219,10 +2300,12 @@ def test_error_cases(self): StatisticsError = statistics.StatisticsError with self.assertRaises(StatisticsError): geometric_mean([]) # empty input - with self.assertRaises(StatisticsError): - geometric_mean([3.5, 0.0, 5.25]) # zero input with self.assertRaises(StatisticsError): geometric_mean([3.5, -4.0, 5.25]) # negative input + with self.assertRaises(StatisticsError): + geometric_mean([0.0, -4.0, 5.25]) # negative input with zero + with self.assertRaises(StatisticsError): + geometric_mean([3.5, -math.inf, 5.25]) # negative infinity with self.assertRaises(StatisticsError): geometric_mean(iter([])) # empty iterator with self.assertRaises(TypeError): @@ -2245,6 +2328,208 @@ def test_special_values(self): with self.assertRaises(ValueError): geometric_mean([Inf, -Inf]) + # Cases with zero + self.assertEqual(geometric_mean([3, 0.0, 5]), 0.0) # Any zero gives a zero + self.assertEqual(geometric_mean([3, -0.0, 5]), 0.0) # Negative zero allowed + self.assertTrue(math.isnan(geometric_mean([0, NaN]))) # NaN beats zero + self.assertTrue(math.isnan(geometric_mean([0, Inf]))) # Because 0.0 * Inf -> NaN + + def test_mixed_int_and_float(self): + # Regression test for b.p.o. issue #28327 + geometric_mean = statistics.geometric_mean + expected_mean = 3.80675409583932 + values = [ + [2, 3, 5, 7], + [2, 3, 5, 7.0], + [2, 3, 5.0, 7.0], + [2, 3.0, 5.0, 7.0], + [2.0, 3.0, 5.0, 7.0], + ] + for v in values: + with self.subTest(v=v): + actual_mean = geometric_mean(v) + self.assertAlmostEqual(actual_mean, expected_mean, places=5) + + +class TestKDE(unittest.TestCase): + + @support.requires_resource('cpu') + def test_kde(self): + kde = statistics.kde + StatisticsError = statistics.StatisticsError + + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # The approximate integral of a PDF should be close to 1.0 + + def integrate(func, low, high, steps=10_000): + "Numeric approximation of a definite function integral." + dx = (high - low) / steps + midpoints = (low + (i + 1/2) * dx for i in range(steps)) + return sum(map(func, midpoints)) * dx + + for kernel in kernels: + with self.subTest(kernel=kernel): + f_hat = kde(sample, h=1.5, kernel=kernel) + area = integrate(f_hat, -20, 20) + self.assertAlmostEqual(area, 1.0, places=4) + + # Check CDF against an integral of the PDF + + data = [3, 5, 10, 12] + h = 2.3 + x = 10.5 + for kernel in kernels: + with self.subTest(kernel=kernel): + cdf = kde(data, h, kernel, cumulative=True) + f_hat = kde(data, h, kernel) + area = integrate(f_hat, -20, x, 100_000) + self.assertAlmostEqual(cdf(x), area, places=4) + + # Check error cases + + with self.assertRaises(StatisticsError): + kde([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde(sample, h=-1.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde(sample, h=1.0, kernel='bogus') # Invalid kernel + with self.assertRaises(TypeError): + kde(sample, 1.0, 'gauss', True) # Positional cumulative argument + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + f_hat = kde(sample, h, kernel) + self.assertEqual(f_hat.__name__, 'pdf') + self.assertIn(kernel, f_hat.__doc__) + self.assertIn(repr(h), f_hat.__doc__) + + # Test closed interval for the support boundaries. + # In particular, 'uniform' should non-zero at the boundaries. + + f_hat = kde([0], 1.0, 'uniform') + self.assertEqual(f_hat(-1.0), 1/2) + self.assertEqual(f_hat(1.0), 1/2) + + # Test online updates to data + + data = [1, 2] + f_hat = kde(data, 5.0, 'triangular') + self.assertEqual(f_hat(100), 0.0) + data.append(100) + self.assertGreater(f_hat(100), 0.0) + + def test_kde_kernel_specs(self): + # White-box test for the kernel formulas in isolation from + # their downstream use in kde() and kde_random() + kernel_specs = statistics._kernel_specs + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + parr = [i/1000 + 5/10000 for i in range(1000)] + for kernel, spec in kernel_specs.items(): + cdf = spec['cdf'] + invcdf = spec['invcdf'] + with self.subTest(kernel=kernel): + for x in xarr: + self.assertAlmostEqual(invcdf(cdf(x)), x, places=6) + for p in parr: + self.assertAlmostEqual(cdf(invcdf(p)), p, places=11) + + @support.requires_resource('cpu') + def test_kde_random(self): + kde_random = statistics.kde_random + StatisticsError = statistics.StatisticsError + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # Smoke test + + for kernel in kernels: + with self.subTest(kernel=kernel): + rand = kde_random(sample, h=1.5, kernel=kernel) + selections = [rand() for i in range(10)] + + # Check error cases + + with self.assertRaises(StatisticsError): + kde_random([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde_random(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde_random(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde_random(sample, h=-1.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde_random(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + rand = kde_random(sample, h, kernel) + self.assertEqual(rand.__name__, 'rand') + self.assertIn(kernel, rand.__doc__) + self.assertIn(repr(h), rand.__doc__) + + # Approximate distribution test: Compare a random sample to the expected distribution + + data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + xarr = [x / 10 for x in range(-100, 250)] + # TODO: RUSTPYTHON - n originally 1_000_000 in CPython implementation but sorting is slow + n = 30_000 + h = 1.75 + dx = 0.1 + + def p_observed(x): + # P(x <= X < x+dx) + i = bisect.bisect_left(big_sample, x) + j = bisect.bisect_left(big_sample, x + dx) + return (j - i) / len(big_sample) + + def p_expected(x): + # P(x <= X < x+dx) + return F_hat(x + dx) - F_hat(x) + + for kernel in kernels: + with self.subTest(kernel=kernel): + + rand = kde_random(data, h, kernel, seed=8675309**2) + big_sample = sorted([rand() for i in range(n)]) + F_hat = statistics.kde(data, h, kernel, cumulative=True) + + for x in xarr: + # TODO : RUSTPYTHON - abs_tol=0.0005 in CPython implementation but smaller `n` increases variance + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.005)) + + # Test online updates to data + + data = [1, 2] + rand = kde_random(data, 5, 'triangular') + self.assertLess(max([rand() for i in range(5000)]), 10) + data.append(100) + self.assertGreater(max(rand() for i in range(5000)), 10) + class TestQuantiles(unittest.TestCase): @@ -2355,6 +2640,11 @@ def f(x): data = random.choices(range(100), k=k) q1, q2, q3 = quantiles(data, method='inclusive') self.assertEqual(q2, statistics.median(data)) + # Base case with a single data point: When estimating quantiles from + # a sample, we want to be able to add one sample point at a time, + # getting increasingly better estimates. + self.assertEqual(quantiles([10], n=4), [10.0, 10.0, 10.0]) + self.assertEqual(quantiles([10], n=4, method='exclusive'), [10.0, 10.0, 10.0]) def test_equal_inputs(self): quantiles = statistics.quantiles @@ -2405,7 +2695,7 @@ def test_error_cases(self): with self.assertRaises(ValueError): quantiles([10, 20, 30], method='X') # method is unknown with self.assertRaises(StatisticsError): - quantiles([10], n=4) # not enough data points + quantiles([], n=4) # not enough data points with self.assertRaises(TypeError): quantiles([10, None, 30], n=4) # data is non-numeric @@ -2464,6 +2754,95 @@ def test_different_scales(self): self.assertAlmostEqual(statistics.correlation(x, y), 1) self.assertAlmostEqual(statistics.covariance(x, y), 0.1) + def test_sqrtprod_helper_function_fundamentals(self): + # Verify that results are close to sqrt(x * y) + for i in range(100): + x = random.expovariate() + y = random.expovariate() + expected = math.sqrt(x * y) + actual = statistics._sqrtprod(x, y) + with self.subTest(x=x, y=y, expected=expected, actual=actual): + self.assertAlmostEqual(expected, actual) + + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + # Test that range extremes avoid underflow and overflow + smallest = sys.float_info.min * sys.float_info.epsilon + self.assertEqual(statistics._sqrtprod(smallest, smallest), smallest) + biggest = sys.float_info.max + self.assertEqual(statistics._sqrtprod(biggest, biggest), biggest) + + # Check special values and the sign of the result + special_values = [0.0, -0.0, 1.0, -1.0, 4.0, -4.0, + math.nan, -math.nan, math.inf, -math.inf] + for x, y in itertools.product(special_values, repeat=2): + try: + expected = math.sqrt(x * y) + except ValueError: + expected = 'ValueError' + try: + actual = statistics._sqrtprod(x, y) + except ValueError: + actual = 'ValueError' + with self.subTest(x=x, y=y, expected=expected, actual=actual): + if isinstance(expected, str) and expected == 'ValueError': + self.assertEqual(actual, 'ValueError') + continue + self.assertIsInstance(actual, float) + if math.isnan(expected): + self.assertTrue(math.isnan(actual)) + continue + self.assertEqual(actual, expected) + self.assertEqual(sign(actual), sign(expected)) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Allow for a weaker sumprod() implementation + def test_sqrtprod_helper_function_improved_accuracy(self): + # Test a known example where accuracy is improved + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + def reference_value(x: float, y: float) -> float: + x = decimal.Decimal(x) + y = decimal.Decimal(y) + with decimal.localcontext() as ctx: + ctx.prec = 200 + return float((x * y).sqrt()) + + # Verify that the new function with improved accuracy + # agrees with a reference value more often than old version. + new_agreements = 0 + old_agreements = 0 + for i in range(10_000): + x = random.expovariate() + y = random.expovariate() + new = statistics._sqrtprod(x, y) + old = math.sqrt(x * y) + ref = reference_value(x, y) + new_agreements += (new == ref) + old_agreements += (old == ref) + self.assertGreater(new_agreements, old_agreements) + + def test_correlation_spearman(self): + # https://statistics.laerd.com/statistical-guides/spearmans-rank-order-correlation-statistical-guide-2.php + # Compare with: + # >>> import scipy.stats.mstats + # >>> scipy.stats.mstats.spearmanr(reading, mathematics) + # SpearmanrResult(correlation=0.6686960980480712, pvalue=0.03450954165178532) + # And Wolfram Alpha gives: 0.668696 + # https://www.wolframalpha.com/input?i=SpearmanRho%5B%7B56%2C+75%2C+45%2C+71%2C+61%2C+64%2C+58%2C+80%2C+76%2C+61%7D%2C+%7B66%2C+70%2C+40%2C+60%2C+65%2C+56%2C+59%2C+77%2C+67%2C+63%7D%5D + reading = [56, 75, 45, 71, 61, 64, 58, 80, 76, 61] + mathematics = [66, 70, 40, 60, 65, 56, 59, 77, 67, 63] + self.assertAlmostEqual(statistics.correlation(reading, mathematics, method='ranked'), + 0.6686960980480712) + + with self.assertRaises(ValueError): + statistics.correlation(reading, mathematics, method='bad_method') class TestLinearRegression(unittest.TestCase): @@ -2487,6 +2866,22 @@ def test_results(self): self.assertAlmostEqual(intercept, true_intercept) self.assertAlmostEqual(slope, true_slope) + def test_proportional(self): + x = [10, 20, 30, 40] + y = [180, 398, 610, 799] + slope, intercept = statistics.linear_regression(x, y, proportional=True) + self.assertAlmostEqual(slope, 20 + 1/150) + self.assertEqual(intercept, 0.0) + + def test_float_output(self): + x = [Fraction(2, 3), Fraction(3, 4)] + y = [Fraction(4, 5), Fraction(5, 6)] + slope, intercept = statistics.linear_regression(x, y) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) + slope, intercept = statistics.linear_regression(x, y, proportional=True) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) class TestNormalDist: @@ -2497,8 +2892,6 @@ class TestNormalDist: # inaccurate. There isn't much we can do about this short of # implementing our own implementations from scratch. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_slots(self): nd = self.module.NormalDist(300, 23) with self.assertRaises(TypeError): @@ -2640,6 +3033,7 @@ def test_cdf(self): self.assertTrue(math.isnan(X.cdf(float('NaN')))) @support.skip_if_pgo_task + @support.requires_resource('cpu') def test_inv_cdf(self): NormalDist = self.module.NormalDist @@ -2697,9 +3091,10 @@ def test_inv_cdf(self): iq.inv_cdf(1.0) # p is one with self.assertRaises(self.module.StatisticsError): iq.inv_cdf(1.1) # p over one - with self.assertRaises(self.module.StatisticsError): - iq = NormalDist(100, 0) # sigma is zero - iq.inv_cdf(0.5) + + # Supported case: + iq = NormalDist(100, 0) # sigma is zero + self.assertEqual(iq.inv_cdf(0.5), 100) # Special values self.assertTrue(math.isnan(Z.inv_cdf(float('NaN')))) @@ -2882,14 +3277,19 @@ def __init__(self, mu, sigma): nd = NormalDist(100, 15) self.assertNotEqual(nd, lnd) - def test_pickle_and_copy(self): + def test_copy(self): nd = self.module.NormalDist(37.5, 5.625) nd1 = copy.copy(nd) self.assertEqual(nd, nd1) nd2 = copy.deepcopy(nd) self.assertEqual(nd, nd2) - nd3 = pickle.loads(pickle.dumps(nd)) - self.assertEqual(nd, nd3) + + def test_pickle(self): + nd = self.module.NormalDist(37.5, 5.625) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + pickled = pickle.loads(pickle.dumps(nd, protocol=proto)) + self.assertEqual(nd, pickled) def test_hashability(self): ND = self.module.NormalDist @@ -2911,7 +3311,7 @@ def setUp(self): def tearDown(self): sys.modules['statistics'] = statistics - + @unittest.skipUnless(c_statistics, 'requires _statistics') class TestNormalDistC(unittest.TestCase, TestNormalDist): @@ -2928,6 +3328,8 @@ def tearDown(self): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) + if sys.float_repr_style == 'short': + tests.addTests(doctest.DocTestSuite(statistics)) return tests diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py new file mode 100644 index 00000000000..3b64033c825 --- /dev/null +++ b/Lib/test/test_str.py @@ -0,0 +1,2805 @@ +""" Test script for the Unicode implementation. + +Written by Marc-Andre Lemburg (mal@lemburg.com). + +(c) Copyright CNRI, All Rights Reserved. NO WARRANTY. + +""" +import _string +import codecs +import datetime +import itertools +import operator +import pickle +import struct +import sys +import textwrap +import unicodedata +import unittest +import warnings +from test.support import warnings_helper +from test import support, string_tests +from test.support.script_helper import assert_python_failure + +try: + import _testcapi +except ImportError: + _testcapi = None + +# Error handling (bad decoder return) +def search_function(encoding): + def decode1(input, errors="strict"): + return 42 # not a tuple + def encode1(input, errors="strict"): + return 42 # not a tuple + def encode2(input, errors="strict"): + return (42, 42) # no unicode + def decode2(input, errors="strict"): + return (42, 42) # no unicode + if encoding=="test.unicode1": + return (encode1, decode1, None, None) + elif encoding=="test.unicode2": + return (encode2, decode2, None, None) + else: + return None + +def duplicate_string(text): + """ + Try to get a fresh clone of the specified text: + new object with a reference count of 1. + + This is a best-effort: latin1 single letters and the empty + string ('') are singletons and cannot be cloned. + """ + return text.encode().decode() + +class StrSubclass(str): + pass + +class OtherStrSubclass(str): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + +class StrTest(string_tests.StringLikeTest, + string_tests.MixinStrUnicodeTest, + unittest.TestCase): + + type2test = str + + def setUp(self): + codecs.register(search_function) + self.addCleanup(codecs.unregister, search_function) + + def checkequalnofix(self, result, object, methodname, *args): + method = getattr(object, methodname) + realresult = method(*args) + self.assertEqual(realresult, result) + self.assertTrue(type(realresult) is type(result)) + + # if the original is returned make sure that + # this doesn't happen with subclasses + if realresult is object: + class usub(str): + def __repr__(self): + return 'usub(%r)' % str.__repr__(self) + object = usub(object) + method = getattr(object, methodname) + realresult = method(*args) + self.assertEqual(realresult, result) + self.assertTrue(object is not realresult) + + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + + def test_literals(self): + self.assertEqual('\xff', '\u00ff') + self.assertEqual('\uffff', '\U0000ffff') + self.assertRaises(SyntaxError, eval, '\'\\Ufffffffe\'') + self.assertRaises(SyntaxError, eval, '\'\\Uffffffff\'') + self.assertRaises(SyntaxError, eval, '\'\\U%08x\'' % 0x110000) + # raw strings should not have unicode escapes + self.assertNotEqual(r"\u0020", " ") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'str'> is not <class 'test.test_str.StrSubclass'> + def test_ascii(self): + self.assertEqual(ascii('abc'), "'abc'") + self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") + self.assertEqual(ascii('ab\\'), "'ab\\\\'") + self.assertEqual(ascii('\\c'), "'\\\\c'") + self.assertEqual(ascii('\\'), "'\\\\'") + self.assertEqual(ascii('\n'), "'\\n'") + self.assertEqual(ascii('\r'), "'\\r'") + self.assertEqual(ascii('\t'), "'\\t'") + self.assertEqual(ascii('\b'), "'\\x08'") + self.assertEqual(ascii("'\""), """'\\'"'""") + self.assertEqual(ascii("'\""), """'\\'"'""") + self.assertEqual(ascii("'"), '''"'"''') + self.assertEqual(ascii('"'), """'"'""") + latin1repr = ( + "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" + "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" + "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" + "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" + "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" + "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" + "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9" + "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7" + "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5" + "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3" + "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1" + "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef" + "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd" + "\\xfe\\xff'") + testrepr = ascii(''.join(map(chr, range(256)))) + self.assertEqual(testrepr, latin1repr) + # Test ascii works on wide unicode escapes without overflow. + self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), + ascii("\U00010000" * 39 + "\uffff" * 4096)) + + self.assertTypedEqual(ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(ascii(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(ascii(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertRaises(TypeError, ascii, WithRepr(b'byte-repr')) + + def test_repr(self): + # Test basic sanity of repr() + self.assertEqual(repr('abc'), "'abc'") + self.assertEqual(repr('ab\\c'), "'ab\\\\c'") + self.assertEqual(repr('ab\\'), "'ab\\\\'") + self.assertEqual(repr('\\c'), "'\\\\c'") + self.assertEqual(repr('\\'), "'\\\\'") + self.assertEqual(repr('\n'), "'\\n'") + self.assertEqual(repr('\r'), "'\\r'") + self.assertEqual(repr('\t'), "'\\t'") + self.assertEqual(repr('\b'), "'\\x08'") + self.assertEqual(repr("'\""), """'\\'"'""") + self.assertEqual(repr("'\""), """'\\'"'""") + self.assertEqual(repr("'"), '''"'"''') + self.assertEqual(repr('"'), """'"'""") + latin1repr = ( + "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" + "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" + "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" + "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" + "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" + "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" + "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" + "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5" + "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3" + "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1" + "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef" + "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd" + "\xfe\xff'") + testrepr = repr(''.join(map(chr, range(256)))) + self.assertEqual(testrepr, latin1repr) + # Test repr works on wide unicode escapes without overflow. + self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), + repr("\U00010000" * 39 + "\uffff" * 4096)) + + self.assertTypedEqual(repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(repr(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(repr(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertRaises(TypeError, repr, WithRepr(b'byte-repr')) + + def test_iterators(self): + # Make sure unicode objects have an __iter__ method + it = "\u1111\u2222\u3333".__iter__() + self.assertEqual(next(it), "\u1111") + self.assertEqual(next(it), "\u2222") + self.assertEqual(next(it), "\u3333") + self.assertRaises(StopIteration, next, it) + + def test_iterators_invocation(self): + cases = [type(iter('abc')), type(iter('🚀'))] + for cls in cases: + with self.subTest(cls=cls): + self.assertRaises(TypeError, cls) + + def test_iteration(self): + cases = ['abc', '🚀🚀🚀', "\u1111\u2222\u3333"] + for case in cases: + with self.subTest(string=case): + self.assertEqual(case, "".join(iter(case))) + + def test_exhausted_iterator(self): + cases = ['abc', '🚀🚀🚀', "\u1111\u2222\u3333"] + for case in cases: + with self.subTest(case=case): + iterator = iter(case) + tuple(iterator) + self.assertRaises(StopIteration, next, iterator) + + def test_pickle_iterator(self): + cases = ['abc', '🚀🚀🚀', "\u1111\u2222\u3333"] + for case in cases: + with self.subTest(case=case): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + it = iter(case) + with self.subTest(proto=proto): + pickled = "".join(pickle.loads(pickle.dumps(it, proto))) + self.assertEqual(case, pickled) + + def test_count(self): + string_tests.StringLikeTest.test_count(self) + # check mixed argument types + self.checkequalnofix(3, 'aaa', 'count', 'a') + self.checkequalnofix(0, 'aaa', 'count', 'b') + self.checkequalnofix(3, 'aaa', 'count', 'a') + self.checkequalnofix(0, 'aaa', 'count', 'b') + self.checkequalnofix(0, 'aaa', 'count', 'b') + self.checkequalnofix(1, 'aaa', 'count', 'a', -1) + self.checkequalnofix(3, 'aaa', 'count', 'a', -10) + self.checkequalnofix(2, 'aaa', 'count', 'a', 0, -1) + self.checkequalnofix(0, 'aaa', 'count', 'a', 0, -10) + # test mixed kinds + self.checkequal(10, '\u0102' + 'a' * 10, 'count', 'a') + self.checkequal(10, '\U00100304' + 'a' * 10, 'count', 'a') + self.checkequal(10, '\U00100304' + '\u0102' * 10, 'count', '\u0102') + self.checkequal(0, 'a' * 10, 'count', '\u0102') + self.checkequal(0, 'a' * 10, 'count', '\U00100304') + self.checkequal(0, '\u0102' * 10, 'count', '\U00100304') + self.checkequal(10, '\u0102' + 'a_' * 10, 'count', 'a_') + self.checkequal(10, '\U00100304' + 'a_' * 10, 'count', 'a_') + self.checkequal(10, '\U00100304' + '\u0102_' * 10, 'count', '\u0102_') + self.checkequal(0, 'a' * 10, 'count', 'a\u0102') + self.checkequal(0, 'a' * 10, 'count', 'a\U00100304') + self.checkequal(0, '\u0102' * 10, 'count', '\u0102\U00100304') + # test subclass + class MyStr(str): + pass + self.checkequal(3, MyStr('aaa'), 'count', 'a') + + def test_find(self): + string_tests.StringLikeTest.test_find(self) + # test implementation details of the memchr fast path + self.checkequal(100, 'a' * 100 + '\u0102', 'find', '\u0102') + self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0201') + self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0120') + self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0220') + self.checkequal(100, 'a' * 100 + '\U00100304', 'find', '\U00100304') + self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00100204') + self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00102004') + # check mixed argument types + self.checkequalnofix(0, 'abcdefghiabc', 'find', 'abc') + self.checkequalnofix(9, 'abcdefghiabc', 'find', 'abc', 1) + self.checkequalnofix(-1, 'abcdefghiabc', 'find', 'def', 4) + + # test utf-8 non-ascii char + self.checkequal(0, 'тест', 'find', 'т') + self.checkequal(3, 'тест', 'find', 'т', 1) + self.checkequal(-1, 'тест', 'find', 'т', 1, 3) + self.checkequal(-1, 'тест', 'find', 'e') # english `e` + # test utf-8 non-ascii slice + self.checkequal(1, 'тест тест', 'find', 'ес') + self.checkequal(1, 'тест тест', 'find', 'ес', 1) + self.checkequal(1, 'тест тест', 'find', 'ес', 1, 3) + self.checkequal(6, 'тест тест', 'find', 'ес', 2) + self.checkequal(-1, 'тест тест', 'find', 'ес', 6, 7) + self.checkequal(-1, 'тест тест', 'find', 'ес', 7) + self.checkequal(-1, 'тест тест', 'find', 'ec') # english `ec` + + self.assertRaises(TypeError, 'hello'.find) + self.assertRaises(TypeError, 'hello'.find, 42) + # test mixed kinds + self.checkequal(100, '\u0102' * 100 + 'a', 'find', 'a') + self.checkequal(100, '\U00100304' * 100 + 'a', 'find', 'a') + self.checkequal(100, '\U00100304' * 100 + '\u0102', 'find', '\u0102') + self.checkequal(-1, 'a' * 100, 'find', '\u0102') + self.checkequal(-1, 'a' * 100, 'find', '\U00100304') + self.checkequal(-1, '\u0102' * 100, 'find', '\U00100304') + self.checkequal(100, '\u0102' * 100 + 'a_', 'find', 'a_') + self.checkequal(100, '\U00100304' * 100 + 'a_', 'find', 'a_') + self.checkequal(100, '\U00100304' * 100 + '\u0102_', 'find', '\u0102_') + self.checkequal(-1, 'a' * 100, 'find', 'a\u0102') + self.checkequal(-1, 'a' * 100, 'find', 'a\U00100304') + self.checkequal(-1, '\u0102' * 100, 'find', '\u0102\U00100304') + + def test_rfind(self): + string_tests.StringLikeTest.test_rfind(self) + # test implementation details of the memrchr fast path + self.checkequal(0, '\u0102' + 'a' * 100 , 'rfind', '\u0102') + self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0201') + self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0120') + self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0220') + self.checkequal(0, '\U00100304' + 'a' * 100, 'rfind', '\U00100304') + self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00100204') + self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00102004') + # check mixed argument types + self.checkequalnofix(9, 'abcdefghiabc', 'rfind', 'abc') + self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '') + self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '') + # test utf-8 non-ascii char + self.checkequal(1, 'тест', 'rfind', 'е') + self.checkequal(1, 'тест', 'rfind', 'е', 1) + self.checkequal(-1, 'тест', 'rfind', 'е', 2) + self.checkequal(-1, 'тест', 'rfind', 'e') # english `e` + # test utf-8 non-ascii slice + self.checkequal(6, 'тест тест', 'rfind', 'ес') + self.checkequal(6, 'тест тест', 'rfind', 'ес', 1) + self.checkequal(1, 'тест тест', 'rfind', 'ес', 1, 3) + self.checkequal(6, 'тест тест', 'rfind', 'ес', 2) + self.checkequal(-1, 'тест тест', 'rfind', 'ес', 6, 7) + self.checkequal(-1, 'тест тест', 'rfind', 'ес', 7) + self.checkequal(-1, 'тест тест', 'rfind', 'ec') # english `ec` + # test mixed kinds + self.checkequal(0, 'a' + '\u0102' * 100, 'rfind', 'a') + self.checkequal(0, 'a' + '\U00100304' * 100, 'rfind', 'a') + self.checkequal(0, '\u0102' + '\U00100304' * 100, 'rfind', '\u0102') + self.checkequal(-1, 'a' * 100, 'rfind', '\u0102') + self.checkequal(-1, 'a' * 100, 'rfind', '\U00100304') + self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304') + self.checkequal(0, '_a' + '\u0102' * 100, 'rfind', '_a') + self.checkequal(0, '_a' + '\U00100304' * 100, 'rfind', '_a') + self.checkequal(0, '_\u0102' + '\U00100304' * 100, 'rfind', '_\u0102') + self.checkequal(-1, 'a' * 100, 'rfind', '\u0102a') + self.checkequal(-1, 'a' * 100, 'rfind', '\U00100304a') + self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304\u0102') + + def test_index(self): + string_tests.StringLikeTest.test_index(self) + self.checkequalnofix(0, 'abcdefghiabc', 'index', '') + self.checkequalnofix(3, 'abcdefghiabc', 'index', 'def') + self.checkequalnofix(0, 'abcdefghiabc', 'index', 'abc') + self.checkequalnofix(9, 'abcdefghiabc', 'index', 'abc', 1) + self.assertRaises(ValueError, 'abcdefghiabc'.index, 'hib') + self.assertRaises(ValueError, 'abcdefghiab'.index, 'abc', 1) + self.assertRaises(ValueError, 'abcdefghi'.index, 'ghi', 8) + self.assertRaises(ValueError, 'abcdefghi'.index, 'ghi', -1) + # test mixed kinds + self.checkequal(100, '\u0102' * 100 + 'a', 'index', 'a') + self.checkequal(100, '\U00100304' * 100 + 'a', 'index', 'a') + self.checkequal(100, '\U00100304' * 100 + '\u0102', 'index', '\u0102') + self.assertRaises(ValueError, ('a' * 100).index, '\u0102') + self.assertRaises(ValueError, ('a' * 100).index, '\U00100304') + self.assertRaises(ValueError, ('\u0102' * 100).index, '\U00100304') + self.checkequal(100, '\u0102' * 100 + 'a_', 'index', 'a_') + self.checkequal(100, '\U00100304' * 100 + 'a_', 'index', 'a_') + self.checkequal(100, '\U00100304' * 100 + '\u0102_', 'index', '\u0102_') + self.assertRaises(ValueError, ('a' * 100).index, 'a\u0102') + self.assertRaises(ValueError, ('a' * 100).index, 'a\U00100304') + self.assertRaises(ValueError, ('\u0102' * 100).index, '\u0102\U00100304') + + def test_rindex(self): + string_tests.StringLikeTest.test_rindex(self) + self.checkequalnofix(12, 'abcdefghiabc', 'rindex', '') + self.checkequalnofix(3, 'abcdefghiabc', 'rindex', 'def') + self.checkequalnofix(9, 'abcdefghiabc', 'rindex', 'abc') + self.checkequalnofix(0, 'abcdefghiabc', 'rindex', 'abc', 0, -1) + + self.assertRaises(ValueError, 'abcdefghiabc'.rindex, 'hib') + self.assertRaises(ValueError, 'defghiabc'.rindex, 'def', 1) + self.assertRaises(ValueError, 'defghiabc'.rindex, 'abc', 0, -1) + self.assertRaises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, 8) + self.assertRaises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, -1) + # test mixed kinds + self.checkequal(0, 'a' + '\u0102' * 100, 'rindex', 'a') + self.checkequal(0, 'a' + '\U00100304' * 100, 'rindex', 'a') + self.checkequal(0, '\u0102' + '\U00100304' * 100, 'rindex', '\u0102') + self.assertRaises(ValueError, ('a' * 100).rindex, '\u0102') + self.assertRaises(ValueError, ('a' * 100).rindex, '\U00100304') + self.assertRaises(ValueError, ('\u0102' * 100).rindex, '\U00100304') + self.checkequal(0, '_a' + '\u0102' * 100, 'rindex', '_a') + self.checkequal(0, '_a' + '\U00100304' * 100, 'rindex', '_a') + self.checkequal(0, '_\u0102' + '\U00100304' * 100, 'rindex', '_\u0102') + self.assertRaises(ValueError, ('a' * 100).rindex, '\u0102a') + self.assertRaises(ValueError, ('a' * 100).rindex, '\U00100304a') + self.assertRaises(ValueError, ('\u0102' * 100).rindex, '\U00100304\u0102') + + def test_maketrans_translate(self): + # these work with plain translate() + self.checkequalnofix('bbbc', 'abababc', 'translate', + {ord('a'): None}) + self.checkequalnofix('iiic', 'abababc', 'translate', + {ord('a'): None, ord('b'): ord('i')}) + self.checkequalnofix('iiix', 'abababc', 'translate', + {ord('a'): None, ord('b'): ord('i'), ord('c'): 'x'}) + self.checkequalnofix('c', 'abababc', 'translate', + {ord('a'): None, ord('b'): ''}) + self.checkequalnofix('xyyx', 'xzx', 'translate', + {ord('z'): 'yy'}) + + # this needs maketrans() + self.checkequalnofix('abababc', 'abababc', 'translate', + {'b': '<i>'}) + tbl = self.type2test.maketrans({'a': None, 'b': '<i>'}) + self.checkequalnofix('<i><i><i>c', 'abababc', 'translate', tbl) + # test alternative way of calling maketrans() + tbl = self.type2test.maketrans('abc', 'xyz', 'd') + self.checkequalnofix('xyzzy', 'abdcdcbdddd', 'translate', tbl) + + # various tests switching from ASCII to latin1 or the opposite; + # same length, remove a letter, or replace with a longer string. + self.assertEqual("[a]".translate(str.maketrans('a', 'X')), + "[X]") + self.assertEqual("[a]".translate(str.maketrans({'a': 'X'})), + "[X]") + self.assertEqual("[a]".translate(str.maketrans({'a': None})), + "[]") + self.assertEqual("[a]".translate(str.maketrans({'a': 'XXX'})), + "[XXX]") + self.assertEqual("[a]".translate(str.maketrans({'a': '\xe9'})), + "[\xe9]") + self.assertEqual('axb'.translate(str.maketrans({'a': None, 'b': '123'})), + "x123") + self.assertEqual('axb'.translate(str.maketrans({'a': None, 'b': '\xe9'})), + "x\xe9") + + # test non-ASCII (don't take the fast-path) + self.assertEqual("[a]".translate(str.maketrans({'a': '<\xe9>'})), + "[<\xe9>]") + self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': 'a'})), + "[a]") + self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': None})), + "[]") + self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': '123'})), + "[123]") + self.assertEqual("[a\xe9]".translate(str.maketrans({'a': '<\u20ac>'})), + "[<\u20ac>\xe9]") + + # invalid Unicode characters + invalid_char = 0x10ffff+1 + for before in "a\xe9\u20ac\U0010ffff": + mapping = str.maketrans({before: invalid_char}) + text = "[%s]" % before + self.assertRaises(ValueError, text.translate, mapping) + + # errors + self.assertRaises(TypeError, self.type2test.maketrans) + self.assertRaises(ValueError, self.type2test.maketrans, 'abc', 'defg') + self.assertRaises(TypeError, self.type2test.maketrans, 2, 'def') + self.assertRaises(TypeError, self.type2test.maketrans, 'abc', 2) + self.assertRaises(TypeError, self.type2test.maketrans, 'abc', 'def', 2) + self.assertRaises(ValueError, self.type2test.maketrans, {'xy': 2}) + self.assertRaises(TypeError, self.type2test.maketrans, {(1,): 2}) + + self.assertRaises(TypeError, 'hello'.translate) + self.assertRaises(TypeError, 'abababc'.translate, 'abc', 'xyz') + + def test_split(self): + string_tests.StringLikeTest.test_split(self) + + # test mixed kinds + for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): + left *= 9 + right *= 9 + for delim in ('c', '\u0102', '\U00010302'): + self.checkequal([left + right], + left + right, 'split', delim) + self.checkequal([left, right], + left + delim + right, 'split', delim) + self.checkequal([left + right], + left + right, 'split', delim * 2) + self.checkequal([left, right], + left + delim * 2 + right, 'split', delim *2) + + def test_rsplit(self): + string_tests.StringLikeTest.test_rsplit(self) + # test mixed kinds + for left, right in ('ba', 'юё', '\u0101\u0100', '\U00010301\U00010300'): + left *= 9 + right *= 9 + for delim in ('c', 'ы', '\u0102', '\U00010302'): + self.checkequal([left + right], + left + right, 'rsplit', delim) + self.checkequal([left, right], + left + delim + right, 'rsplit', delim) + self.checkequal([left + right], + left + right, 'rsplit', delim * 2) + self.checkequal([left, right], + left + delim * 2 + right, 'rsplit', delim *2) + + # Check `None` as well: + self.checkequal([left + right], + left + right, 'rsplit', None) + + def test_partition(self): + string_tests.StringLikeTest.test_partition(self) + # test mixed kinds + self.checkequal(('ABCDEFGH', '', ''), 'ABCDEFGH', 'partition', '\u4200') + for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): + left *= 9 + right *= 9 + for delim in ('c', '\u0102', '\U00010302'): + self.checkequal((left + right, '', ''), + left + right, 'partition', delim) + self.checkequal((left, delim, right), + left + delim + right, 'partition', delim) + self.checkequal((left + right, '', ''), + left + right, 'partition', delim * 2) + self.checkequal((left, delim * 2, right), + left + delim * 2 + right, 'partition', delim * 2) + + def test_rpartition(self): + string_tests.StringLikeTest.test_rpartition(self) + # test mixed kinds + self.checkequal(('', '', 'ABCDEFGH'), 'ABCDEFGH', 'rpartition', '\u4200') + for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): + left *= 9 + right *= 9 + for delim in ('c', '\u0102', '\U00010302'): + self.checkequal(('', '', left + right), + left + right, 'rpartition', delim) + self.checkequal((left, delim, right), + left + delim + right, 'rpartition', delim) + self.checkequal(('', '', left + right), + left + right, 'rpartition', delim * 2) + self.checkequal((left, delim * 2, right), + left + delim * 2 + right, 'rpartition', delim * 2) + + def test_join(self): + string_tests.StringLikeTest.test_join(self) + + class MyWrapper: + def __init__(self, sval): self.sval = sval + def __str__(self): return self.sval + + # mixed arguments + self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd']) + self.checkequalnofix('abcd', '', 'join', ('a', 'b', 'c', 'd')) + self.checkequalnofix('w x y z', ' ', 'join', string_tests.Sequence('wxyz')) + self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd']) + self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd']) + self.checkequalnofix('abcd', '', 'join', ('a', 'b', 'c', 'd')) + self.checkequalnofix('w x y z', ' ', 'join', string_tests.Sequence('wxyz')) + self.checkraises(TypeError, ' ', 'join', ['1', '2', MyWrapper('foo')]) + self.checkraises(TypeError, ' ', 'join', ['1', '2', '3', bytes()]) + self.checkraises(TypeError, ' ', 'join', [1, 2, 3]) + self.checkraises(TypeError, ' ', 'join', ['1', '2', 3]) + + @unittest.skipIf(sys.maxsize > 2**32, + 'needs too much memory on a 64-bit platform') + def test_join_overflow(self): + size = int(sys.maxsize**0.5) + 1 + seq = ('A' * size,) * size + self.assertRaises(OverflowError, ''.join, seq) + + def test_replace(self): + string_tests.StringLikeTest.test_replace(self) + + # method call forwarded from str implementation because of unicode argument + self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1) + self.assertRaises(TypeError, 'replace'.replace, "r", 42) + # test mixed kinds + for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): + left *= 9 + right *= 9 + for delim in ('c', '\u0102', '\U00010302'): + for repl in ('d', '\u0103', '\U00010303'): + self.checkequal(left + right, + left + right, 'replace', delim, repl) + self.checkequal(left + repl + right, + left + delim + right, + 'replace', delim, repl) + self.checkequal(left + right, + left + right, 'replace', delim * 2, repl) + self.checkequal(left + repl + right, + left + delim * 2 + right, + 'replace', delim * 2, repl) + + @support.cpython_only + def test_replace_id(self): + pattern = 'abc' + text = 'abc def' + self.assertIs(text.replace(pattern, pattern), text) + + def test_repeat_id_preserving(self): + a = '123abc1@' + b = '456zyx-+' + self.assertEqual(id(a), id(a)) + self.assertNotEqual(id(a), id(b)) + self.assertNotEqual(id(a), id(a * -4)) + self.assertNotEqual(id(a), id(a * 0)) + self.assertEqual(id(a), id(a * 1)) + self.assertEqual(id(a), id(1 * a)) + self.assertNotEqual(id(a), id(a * 2)) + + class SubStr(str): + pass + + s = SubStr('qwerty()') + self.assertEqual(id(s), id(s)) + self.assertNotEqual(id(s), id(s * -4)) + self.assertNotEqual(id(s), id(s * 0)) + self.assertNotEqual(id(s), id(s * 1)) + self.assertNotEqual(id(s), id(1 * s)) + self.assertNotEqual(id(s), id(s * 2)) + + def test_bytes_comparison(self): + with warnings_helper.check_warnings(): + warnings.simplefilter('ignore', BytesWarning) + self.assertEqual('abc' == b'abc', False) + self.assertEqual('abc' != b'abc', True) + self.assertEqual('abc' == bytearray(b'abc'), False) + self.assertEqual('abc' != bytearray(b'abc'), True) + + def test_comparison(self): + # Comparisons: + self.assertEqual('abc', 'abc') + self.assertTrue('abcd' > 'abc') + self.assertTrue('abc' < 'abcd') + + if 0: + # Move these tests to a Unicode collation module test... + # Testing UTF-16 code point order comparisons... + + # No surrogates, no fixup required. + self.assertTrue('\u0061' < '\u20ac') + # Non surrogate below surrogate value, no fixup required + self.assertTrue('\u0061' < '\ud800\udc02') + + # Non surrogate above surrogate value, fixup required + def test_lecmp(s, s2): + self.assertTrue(s < s2) + + def test_fixup(s): + s2 = '\ud800\udc01' + test_lecmp(s, s2) + s2 = '\ud900\udc01' + test_lecmp(s, s2) + s2 = '\uda00\udc01' + test_lecmp(s, s2) + s2 = '\udb00\udc01' + test_lecmp(s, s2) + s2 = '\ud800\udd01' + test_lecmp(s, s2) + s2 = '\ud900\udd01' + test_lecmp(s, s2) + s2 = '\uda00\udd01' + test_lecmp(s, s2) + s2 = '\udb00\udd01' + test_lecmp(s, s2) + s2 = '\ud800\ude01' + test_lecmp(s, s2) + s2 = '\ud900\ude01' + test_lecmp(s, s2) + s2 = '\uda00\ude01' + test_lecmp(s, s2) + s2 = '\udb00\ude01' + test_lecmp(s, s2) + s2 = '\ud800\udfff' + test_lecmp(s, s2) + s2 = '\ud900\udfff' + test_lecmp(s, s2) + s2 = '\uda00\udfff' + test_lecmp(s, s2) + s2 = '\udb00\udfff' + test_lecmp(s, s2) + + test_fixup('\ue000') + test_fixup('\uff61') + + # Surrogates on both sides, no fixup required + self.assertTrue('\ud800\udc02' < '\ud84d\udc56') + + def test_islower(self): + super().test_islower() + self.checkequalnofix(False, '\u1FFc', 'islower') + self.assertFalse('\u2167'.islower()) + self.assertTrue('\u2177'.islower()) + # non-BMP, uppercase + self.assertFalse('\U00010401'.islower()) + self.assertFalse('\U00010427'.islower()) + # non-BMP, lowercase + self.assertTrue('\U00010429'.islower()) + self.assertTrue('\U0001044E'.islower()) + # non-BMP, non-cased + self.assertFalse('\U0001F40D'.islower()) + self.assertFalse('\U0001F46F'.islower()) + + def test_isupper(self): + super().test_isupper() + self.checkequalnofix(False, '\u1FFc', 'isupper') + self.assertTrue('\u2167'.isupper()) + self.assertFalse('\u2177'.isupper()) + # non-BMP, uppercase + self.assertTrue('\U00010401'.isupper()) + self.assertTrue('\U00010427'.isupper()) + # non-BMP, lowercase + self.assertFalse('\U00010429'.isupper()) + self.assertFalse('\U0001044E'.isupper()) + # non-BMP, non-cased + self.assertFalse('\U0001F40D'.isupper()) + self.assertFalse('\U0001F46F'.isupper()) + + def test_istitle(self): + super().test_istitle() + self.checkequalnofix(True, '\u1FFc', 'istitle') + self.checkequalnofix(True, 'Greek \u1FFcitlecases ...', 'istitle') + + # non-BMP, uppercase + lowercase + self.assertTrue('\U00010401\U00010429'.istitle()) + self.assertTrue('\U00010427\U0001044E'.istitle()) + # apparently there are no titlecased (Lt) non-BMP chars in Unicode 6 + for ch in ['\U00010429', '\U0001044E', '\U0001F40D', '\U0001F46F']: + self.assertFalse(ch.istitle(), '{!a} is not title'.format(ch)) + + def test_isspace(self): + super().test_isspace() + self.checkequalnofix(True, '\u2000', 'isspace') + self.checkequalnofix(True, '\u200a', 'isspace') + self.checkequalnofix(False, '\u2014', 'isspace') + # There are no non-BMP whitespace chars as of Unicode 12. + for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', + '\U0001F40D', '\U0001F46F']: + self.assertFalse(ch.isspace(), '{!a} is not space.'.format(ch)) + + @support.requires_resource('cpu') + def test_isspace_invariant(self): + for codepoint in range(sys.maxunicode + 1): + char = chr(codepoint) + bidirectional = unicodedata.bidirectional(char) + category = unicodedata.category(char) + self.assertEqual(char.isspace(), + (bidirectional in ('WS', 'B', 'S') + or category == 'Zs')) + + def test_isalnum(self): + super().test_isalnum() + for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', + '\U0001D7F6', '\U00011066', '\U000104A0', '\U0001F107']: + self.assertTrue(ch.isalnum(), '{!a} is alnum.'.format(ch)) + + def test_isalpha(self): + super().test_isalpha() + self.checkequalnofix(True, '\u1FFc', 'isalpha') + # non-BMP, cased + self.assertTrue('\U00010401'.isalpha()) + self.assertTrue('\U00010427'.isalpha()) + self.assertTrue('\U00010429'.isalpha()) + self.assertTrue('\U0001044E'.isalpha()) + # non-BMP, non-cased + self.assertFalse('\U0001F40D'.isalpha()) + self.assertFalse('\U0001F46F'.isalpha()) + + def test_isascii(self): + super().test_isascii() + self.assertFalse("\u20ac".isascii()) + self.assertFalse("\U0010ffff".isascii()) + + def test_isdecimal(self): + self.checkequalnofix(False, '', 'isdecimal') + self.checkequalnofix(False, 'a', 'isdecimal') + self.checkequalnofix(True, '0', 'isdecimal') + self.checkequalnofix(False, '\u2460', 'isdecimal') # CIRCLED DIGIT ONE + self.checkequalnofix(False, '\xbc', 'isdecimal') # VULGAR FRACTION ONE QUARTER + self.checkequalnofix(True, '\u0660', 'isdecimal') # ARABIC-INDIC DIGIT ZERO + self.checkequalnofix(True, '0123456789', 'isdecimal') + self.checkequalnofix(False, '0123456789a', 'isdecimal') + + self.checkraises(TypeError, 'abc', 'isdecimal', 42) + + for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', + '\U0001F40D', '\U0001F46F', '\U00011065', '\U0001F107']: + self.assertFalse(ch.isdecimal(), '{!a} is not decimal.'.format(ch)) + for ch in ['\U0001D7F6', '\U00011066', '\U000104A0']: + self.assertTrue(ch.isdecimal(), '{!a} is decimal.'.format(ch)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False != True + def test_isdigit(self): + super().test_isdigit() + self.checkequalnofix(True, '\u2460', 'isdigit') + self.checkequalnofix(False, '\xbc', 'isdigit') + self.checkequalnofix(True, '\u0660', 'isdigit') + + for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', + '\U0001F40D', '\U0001F46F', '\U00011065']: + self.assertFalse(ch.isdigit(), '{!a} is not a digit.'.format(ch)) + for ch in ['\U0001D7F6', '\U00011066', '\U000104A0', '\U0001F107']: + self.assertTrue(ch.isdigit(), '{!a} is a digit.'.format(ch)) + + def test_isnumeric(self): + self.checkequalnofix(False, '', 'isnumeric') + self.checkequalnofix(False, 'a', 'isnumeric') + self.checkequalnofix(True, '0', 'isnumeric') + self.checkequalnofix(True, '\u2460', 'isnumeric') + self.checkequalnofix(True, '\xbc', 'isnumeric') + self.checkequalnofix(True, '\u0660', 'isnumeric') + self.checkequalnofix(True, '0123456789', 'isnumeric') + self.checkequalnofix(False, '0123456789a', 'isnumeric') + + self.assertRaises(TypeError, "abc".isnumeric, 42) + + for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', + '\U0001F40D', '\U0001F46F']: + self.assertFalse(ch.isnumeric(), '{!a} is not numeric.'.format(ch)) + for ch in ['\U00011065', '\U0001D7F6', '\U00011066', + '\U000104A0', '\U0001F107']: + self.assertTrue(ch.isnumeric(), '{!a} is numeric.'.format(ch)) + + def test_isidentifier(self): + self.assertTrue("a".isidentifier()) + self.assertTrue("Z".isidentifier()) + self.assertTrue("_".isidentifier()) + self.assertTrue("b0".isidentifier()) + self.assertTrue("bc".isidentifier()) + self.assertTrue("b_".isidentifier()) + self.assertTrue("µ".isidentifier()) + self.assertTrue("𝔘𝔫𝔦𝔠𝔬𝔡𝔢".isidentifier()) + + self.assertFalse(" ".isidentifier()) + self.assertFalse("[".isidentifier()) + self.assertFalse("©".isidentifier()) + self.assertFalse("0".isidentifier()) + + def test_isprintable(self): + self.assertTrue("".isprintable()) + self.assertTrue(" ".isprintable()) + self.assertTrue("abcdefg".isprintable()) + self.assertFalse("abcdefg\n".isprintable()) + # some defined Unicode character + self.assertTrue("\u0374".isprintable()) + # undefined character + self.assertFalse("\u0378".isprintable()) + # single surrogate character + self.assertFalse("\ud800".isprintable()) + + self.assertTrue('\U0001F46F'.isprintable()) + self.assertFalse('\U000E0020'.isprintable()) + + @support.requires_resource('cpu') + def test_isprintable_invariant(self): + for codepoint in range(sys.maxunicode + 1): + char = chr(codepoint) + category = unicodedata.category(char) + self.assertEqual(char.isprintable(), + category[0] not in ('C', 'Z') + or char == ' ') + + def test_surrogates(self): + for s in ('a\uD800b\uDFFF', 'a\uDFFFb\uD800', + 'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'): + self.assertTrue(s.islower()) + self.assertFalse(s.isupper()) + self.assertFalse(s.istitle()) + for s in ('A\uD800B\uDFFF', 'A\uDFFFB\uD800', + 'A\uD800B\uDFFFA', 'A\uDFFFB\uD800A'): + self.assertFalse(s.islower()) + self.assertTrue(s.isupper()) + self.assertTrue(s.istitle()) + + for meth_name in ('islower', 'isupper', 'istitle'): + meth = getattr(str, meth_name) + for s in ('\uD800', '\uDFFF', '\uD800\uD800', '\uDFFF\uDFFF'): + self.assertFalse(meth(s), '%a.%s() is False' % (s, meth_name)) + + for meth_name in ('isalpha', 'isalnum', 'isdigit', 'isspace', + 'isdecimal', 'isnumeric', + 'isidentifier', 'isprintable'): + meth = getattr(str, meth_name) + for s in ('\uD800', '\uDFFF', '\uD800\uD800', '\uDFFF\uDFFF', + 'a\uD800b\uDFFF', 'a\uDFFFb\uD800', + 'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'): + self.assertFalse(meth(s), '%a.%s() is False' % (s, meth_name)) + + + def test_lower(self): + string_tests.StringLikeTest.test_lower(self) + self.assertEqual('\U00010427'.lower(), '\U0001044F') + self.assertEqual('\U00010427\U00010427'.lower(), + '\U0001044F\U0001044F') + self.assertEqual('\U00010427\U0001044F'.lower(), + '\U0001044F\U0001044F') + self.assertEqual('X\U00010427x\U0001044F'.lower(), + 'x\U0001044Fx\U0001044F') + self.assertEqual('fi'.lower(), 'fi') + self.assertEqual('\u0130'.lower(), '\u0069\u0307') + # Special case for GREEK CAPITAL LETTER SIGMA U+03A3 + self.assertEqual('\u03a3'.lower(), '\u03c3') + self.assertEqual('\u0345\u03a3'.lower(), '\u0345\u03c3') + self.assertEqual('A\u0345\u03a3'.lower(), 'a\u0345\u03c2') + self.assertEqual('A\u0345\u03a3a'.lower(), 'a\u0345\u03c3a') + self.assertEqual('A\u0345\u03a3'.lower(), 'a\u0345\u03c2') + self.assertEqual('A\u03a3\u0345'.lower(), 'a\u03c2\u0345') + self.assertEqual('\u03a3\u0345 '.lower(), '\u03c3\u0345 ') + self.assertEqual('\U0008fffe'.lower(), '\U0008fffe') + self.assertEqual('\u2177'.lower(), '\u2177') + + def test_casefold(self): + self.assertEqual('hello'.casefold(), 'hello') + self.assertEqual('hELlo'.casefold(), 'hello') + self.assertEqual('ß'.casefold(), 'ss') + self.assertEqual('fi'.casefold(), 'fi') + self.assertEqual('\u03a3'.casefold(), '\u03c3') + self.assertEqual('A\u0345\u03a3'.casefold(), 'a\u03b9\u03c3') + self.assertEqual('\u00b5'.casefold(), '\u03bc') + + def test_upper(self): + string_tests.StringLikeTest.test_upper(self) + self.assertEqual('\U0001044F'.upper(), '\U00010427') + self.assertEqual('\U0001044F\U0001044F'.upper(), + '\U00010427\U00010427') + self.assertEqual('\U00010427\U0001044F'.upper(), + '\U00010427\U00010427') + self.assertEqual('X\U00010427x\U0001044F'.upper(), + 'X\U00010427X\U00010427') + self.assertEqual('fi'.upper(), 'FI') + self.assertEqual('\u0130'.upper(), '\u0130') + self.assertEqual('\u03a3'.upper(), '\u03a3') + self.assertEqual('ß'.upper(), 'SS') + self.assertEqual('\u1fd2'.upper(), '\u0399\u0308\u0300') + self.assertEqual('\U0008fffe'.upper(), '\U0008fffe') + self.assertEqual('\u2177'.upper(), '\u2167') + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + def test_capitalize(self): + string_tests.StringLikeTest.test_capitalize(self) + self.assertEqual('\U0001044F'.capitalize(), '\U00010427') + self.assertEqual('\U0001044F\U0001044F'.capitalize(), + '\U00010427\U0001044F') + self.assertEqual('\U00010427\U0001044F'.capitalize(), + '\U00010427\U0001044F') + self.assertEqual('\U0001044F\U00010427'.capitalize(), + '\U00010427\U0001044F') + self.assertEqual('X\U00010427x\U0001044F'.capitalize(), + 'X\U0001044Fx\U0001044F') + self.assertEqual('h\u0130'.capitalize(), 'H\u0069\u0307') + exp = '\u0399\u0308\u0300\u0069\u0307' + self.assertEqual('\u1fd2\u0130'.capitalize(), exp) + self.assertEqual('finnish'.capitalize(), 'Finnish') + self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2') + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + def test_title(self): + super().test_title() + self.assertEqual('\U0001044F'.title(), '\U00010427') + self.assertEqual('\U0001044F\U0001044F'.title(), + '\U00010427\U0001044F') + self.assertEqual('\U0001044F\U0001044F \U0001044F\U0001044F'.title(), + '\U00010427\U0001044F \U00010427\U0001044F') + self.assertEqual('\U00010427\U0001044F \U00010427\U0001044F'.title(), + '\U00010427\U0001044F \U00010427\U0001044F') + self.assertEqual('\U0001044F\U00010427 \U0001044F\U00010427'.title(), + '\U00010427\U0001044F \U00010427\U0001044F') + self.assertEqual('X\U00010427x\U0001044F X\U00010427x\U0001044F'.title(), + 'X\U0001044Fx\U0001044F X\U0001044Fx\U0001044F') + self.assertEqual('fiNNISH'.title(), 'Finnish') + self.assertEqual('A\u03a3 \u1fa1xy'.title(), 'A\u03c2 \u1fa9xy') + self.assertEqual('A\u03a3A'.title(), 'A\u03c3a') + + @unittest.expectedFailure # TODO: RUSTPYTHON; + 𐐧 + def test_swapcase(self): + string_tests.StringLikeTest.test_swapcase(self) + self.assertEqual('\U0001044F'.swapcase(), '\U00010427') + self.assertEqual('\U00010427'.swapcase(), '\U0001044F') + self.assertEqual('\U0001044F\U0001044F'.swapcase(), + '\U00010427\U00010427') + self.assertEqual('\U00010427\U0001044F'.swapcase(), + '\U0001044F\U00010427') + self.assertEqual('\U0001044F\U00010427'.swapcase(), + '\U00010427\U0001044F') + self.assertEqual('X\U00010427x\U0001044F'.swapcase(), + 'x\U0001044FX\U00010427') + self.assertEqual('fi'.swapcase(), 'FI') + self.assertEqual('\u0130'.swapcase(), '\u0069\u0307') + # Special case for GREEK CAPITAL LETTER SIGMA U+03A3 + self.assertEqual('\u03a3'.swapcase(), '\u03c3') + self.assertEqual('\u0345\u03a3'.swapcase(), '\u0399\u03c3') + self.assertEqual('A\u0345\u03a3'.swapcase(), 'a\u0399\u03c2') + self.assertEqual('A\u0345\u03a3a'.swapcase(), 'a\u0399\u03c3A') + self.assertEqual('A\u0345\u03a3'.swapcase(), 'a\u0399\u03c2') + self.assertEqual('A\u03a3\u0345'.swapcase(), 'a\u03c2\u0399') + self.assertEqual('\u03a3\u0345 '.swapcase(), '\u03c3\u0399 ') + self.assertEqual('\u03a3'.swapcase(), '\u03c3') + self.assertEqual('ß'.swapcase(), 'SS') + self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300') + + def test_center(self): + string_tests.StringLikeTest.test_center(self) + self.assertEqual('x'.center(2, '\U0010FFFF'), + 'x\U0010FFFF') + self.assertEqual('x'.center(3, '\U0010FFFF'), + '\U0010FFFFx\U0010FFFF') + self.assertEqual('x'.center(4, '\U0010FFFF'), + '\U0010FFFFx\U0010FFFF\U0010FFFF') + + @unittest.skipUnless(sys.maxsize == 2**31 - 1, "requires 32-bit system") + @support.cpython_only + def test_case_operation_overflow(self): + # Issue #22643 + size = 2**32//12 + 1 + try: + s = "ü" * size + except MemoryError: + self.skipTest('no enough memory (%.0f MiB required)' % (size / 2**20)) + try: + self.assertRaises(OverflowError, s.upper) + finally: + del s + + def test_contains(self): + # Testing Unicode contains method + self.assertIn('a', 'abdb') + self.assertIn('a', 'bdab') + self.assertIn('a', 'bdaba') + self.assertIn('a', 'bdba') + self.assertNotIn('a', 'bdb') + self.assertIn('a', 'bdba') + self.assertIn('a', ('a',1,None)) + self.assertIn('a', (1,None,'a')) + self.assertIn('a', ('a',1,None)) + self.assertIn('a', (1,None,'a')) + self.assertNotIn('a', ('x',1,'y')) + self.assertNotIn('a', ('x',1,None)) + self.assertNotIn('abcd', 'abcxxxx') + self.assertIn('ab', 'abcd') + self.assertIn('ab', 'abc') + self.assertIn('ab', (1,None,'ab')) + self.assertIn('', 'abc') + self.assertIn('', '') + self.assertIn('', 'abc') + self.assertNotIn('\0', 'abc') + self.assertIn('\0', '\0abc') + self.assertIn('\0', 'abc\0') + self.assertIn('a', '\0abc') + self.assertIn('asdf', 'asdf') + self.assertNotIn('asdf', 'asd') + self.assertNotIn('asdf', '') + + self.assertRaises(TypeError, "abc".__contains__) + # test mixed kinds + for fill in ('a', '\u0100', '\U00010300'): + fill *= 9 + for delim in ('c', '\u0102', '\U00010302'): + self.assertNotIn(delim, fill) + self.assertIn(delim, fill + delim) + self.assertNotIn(delim * 2, fill) + self.assertIn(delim * 2, fill + delim * 2) + + def test_issue18183(self): + '\U00010000\U00100000'.lower() + '\U00010000\U00100000'.casefold() + '\U00010000\U00100000'.upper() + '\U00010000\U00100000'.capitalize() + '\U00010000\U00100000'.title() + '\U00010000\U00100000'.swapcase() + '\U00100000'.center(3, '\U00010000') + '\U00100000'.ljust(3, '\U00010000') + '\U00100000'.rjust(3, '\U00010000') + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? + + def test_format(self): + self.assertEqual(''.format(), '') + self.assertEqual('a'.format(), 'a') + self.assertEqual('ab'.format(), 'ab') + self.assertEqual('a{{'.format(), 'a{') + self.assertEqual('a}}'.format(), 'a}') + self.assertEqual('{{b'.format(), '{b') + self.assertEqual('}}b'.format(), '}b') + self.assertEqual('a{{b'.format(), 'a{b') + + # examples from the PEP: + import datetime + self.assertEqual("My name is {0}".format('Fred'), "My name is Fred") + self.assertEqual("My name is {0[name]}".format(dict(name='Fred')), + "My name is Fred") + self.assertEqual("My name is {0} :-{{}}".format('Fred'), + "My name is Fred :-{}") + + d = datetime.date(2007, 8, 18) + self.assertEqual("The year is {0.year}".format(d), + "The year is 2007") + + # classes we'll use for testing + class C: + def __init__(self, x=100): + self._x = x + def __format__(self, spec): + return spec + + class D: + def __init__(self, x): + self.x = x + def __format__(self, spec): + return str(self.x) + + # class with __str__, but no __format__ + class E: + def __init__(self, x): + self.x = x + def __str__(self): + return 'E(' + self.x + ')' + + # class with __repr__, but no __format__ or __str__ + class F: + def __init__(self, x): + self.x = x + def __repr__(self): + return 'F(' + self.x + ')' + + # class with __format__ that forwards to string, for some format_spec's + class G: + def __init__(self, x): + self.x = x + def __str__(self): + return "string is " + self.x + def __format__(self, format_spec): + if format_spec == 'd': + return 'G(' + self.x + ')' + return object.__format__(self, format_spec) + + class I(datetime.date): + def __format__(self, format_spec): + return self.strftime(format_spec) + + class J(int): + def __format__(self, format_spec): + return int.__format__(self * 2, format_spec) + + class M: + def __init__(self, x): + self.x = x + def __repr__(self): + return 'M(' + self.x + ')' + __str__ = None + + class N: + def __init__(self, x): + self.x = x + def __repr__(self): + return 'N(' + self.x + ')' + __format__ = None + + self.assertEqual(''.format(), '') + self.assertEqual('abc'.format(), 'abc') + self.assertEqual('{0}'.format('abc'), 'abc') + self.assertEqual('{0:}'.format('abc'), 'abc') +# self.assertEqual('{ 0 }'.format('abc'), 'abc') + self.assertEqual('X{0}'.format('abc'), 'Xabc') + self.assertEqual('{0}X'.format('abc'), 'abcX') + self.assertEqual('X{0}Y'.format('abc'), 'XabcY') + self.assertEqual('{1}'.format(1, 'abc'), 'abc') + self.assertEqual('X{1}'.format(1, 'abc'), 'Xabc') + self.assertEqual('{1}X'.format(1, 'abc'), 'abcX') + self.assertEqual('X{1}Y'.format(1, 'abc'), 'XabcY') + self.assertEqual('{0}'.format(-15), '-15') + self.assertEqual('{0}{1}'.format(-15, 'abc'), '-15abc') + self.assertEqual('{0}X{1}'.format(-15, 'abc'), '-15Xabc') + self.assertEqual('{{'.format(), '{') + self.assertEqual('}}'.format(), '}') + self.assertEqual('{{}}'.format(), '{}') + self.assertEqual('{{x}}'.format(), '{x}') + self.assertEqual('{{{0}}}'.format(123), '{123}') + self.assertEqual('{{{{0}}}}'.format(), '{{0}}') + self.assertEqual('}}{{'.format(), '}{') + self.assertEqual('}}x{{'.format(), '}x{') + + # weird field names + self.assertEqual("{0[foo-bar]}".format({'foo-bar':'baz'}), 'baz') + self.assertEqual("{0[foo bar]}".format({'foo bar':'baz'}), 'baz') + self.assertEqual("{0[ ]}".format({' ':3}), '3') + + self.assertEqual('{foo._x}'.format(foo=C(20)), '20') + self.assertEqual('{1}{0}'.format(D(10), D(20)), '2010') + self.assertEqual('{0._x.x}'.format(C(D('abc'))), 'abc') + self.assertEqual('{0[0]}'.format(['abc', 'def']), 'abc') + self.assertEqual('{0[1]}'.format(['abc', 'def']), 'def') + self.assertEqual('{0[1][0]}'.format(['abc', ['def']]), 'def') + self.assertEqual('{0[1][0].x}'.format(['abc', [D('def')]]), 'def') + + # strings + self.assertEqual('{0:.3s}'.format('abc'), 'abc') + self.assertEqual('{0:.3s}'.format('ab'), 'ab') + self.assertEqual('{0:.3s}'.format('abcdef'), 'abc') + self.assertEqual('{0:.0s}'.format('abcdef'), '') + self.assertEqual('{0:3.3s}'.format('abc'), 'abc') + self.assertEqual('{0:2.3s}'.format('abc'), 'abc') + self.assertEqual('{0:2.2s}'.format('abc'), 'ab') + self.assertEqual('{0:3.2s}'.format('abc'), 'ab ') + self.assertEqual('{0:x<0s}'.format('result'), 'result') + self.assertEqual('{0:x<5s}'.format('result'), 'result') + self.assertEqual('{0:x<6s}'.format('result'), 'result') + self.assertEqual('{0:x<7s}'.format('result'), 'resultx') + self.assertEqual('{0:x<8s}'.format('result'), 'resultxx') + self.assertEqual('{0: <7s}'.format('result'), 'result ') + self.assertEqual('{0:<7s}'.format('result'), 'result ') + self.assertEqual('{0:>7s}'.format('result'), ' result') + self.assertEqual('{0:>8s}'.format('result'), ' result') + self.assertEqual('{0:^8s}'.format('result'), ' result ') + self.assertEqual('{0:^9s}'.format('result'), ' result ') + self.assertEqual('{0:^10s}'.format('result'), ' result ') + self.assertEqual('{0:8s}'.format('result'), 'result ') + self.assertEqual('{0:0s}'.format('result'), 'result') + self.assertEqual('{0:08s}'.format('result'), 'result00') + self.assertEqual('{0:<08s}'.format('result'), 'result00') + self.assertEqual('{0:>08s}'.format('result'), '00result') + self.assertEqual('{0:^08s}'.format('result'), '0result0') + self.assertEqual('{0:10000}'.format('a'), 'a' + ' ' * 9999) + self.assertEqual('{0:10000}'.format(''), ' ' * 10000) + self.assertEqual('{0:10000000}'.format(''), ' ' * 10000000) + + # issue 12546: use \x00 as a fill character + self.assertEqual('{0:\x00<6s}'.format('foo'), 'foo\x00\x00\x00') + self.assertEqual('{0:\x01<6s}'.format('foo'), 'foo\x01\x01\x01') + self.assertEqual('{0:\x00^6s}'.format('foo'), '\x00foo\x00\x00') + self.assertEqual('{0:^6s}'.format('foo'), ' foo ') + + self.assertEqual('{0:\x00<6}'.format(3), '3\x00\x00\x00\x00\x00') + self.assertEqual('{0:\x01<6}'.format(3), '3\x01\x01\x01\x01\x01') + self.assertEqual('{0:\x00^6}'.format(3), '\x00\x003\x00\x00\x00') + self.assertEqual('{0:<6}'.format(3), '3 ') + + self.assertEqual('{0:\x00<6}'.format(3.25), '3.25\x00\x00') + self.assertEqual('{0:\x01<6}'.format(3.25), '3.25\x01\x01') + self.assertEqual('{0:\x00^6}'.format(3.25), '\x003.25\x00') + self.assertEqual('{0:^6}'.format(3.25), ' 3.25 ') + + self.assertEqual('{0:\x00<12}'.format(3+2.0j), '(3+2j)\x00\x00\x00\x00\x00\x00') + self.assertEqual('{0:\x01<12}'.format(3+2.0j), '(3+2j)\x01\x01\x01\x01\x01\x01') + self.assertEqual('{0:\x00^12}'.format(3+2.0j), '\x00\x00\x00(3+2j)\x00\x00\x00') + self.assertEqual('{0:^12}'.format(3+2.0j), ' (3+2j) ') + + # format specifiers for user defined type + self.assertEqual('{0:abc}'.format(C()), 'abc') + + # !r, !s and !a coercions + self.assertEqual('{0!s}'.format('Hello'), 'Hello') + self.assertEqual('{0!s:}'.format('Hello'), 'Hello') + self.assertEqual('{0!s:15}'.format('Hello'), 'Hello ') + self.assertEqual('{0!s:15s}'.format('Hello'), 'Hello ') + self.assertEqual('{0!r}'.format('Hello'), "'Hello'") + self.assertEqual('{0!r:}'.format('Hello'), "'Hello'") + self.assertEqual('{0!r}'.format(F('Hello')), 'F(Hello)') + self.assertEqual('{0!r}'.format('\u0378'), "'\\u0378'") # nonprintable + self.assertEqual('{0!r}'.format('\u0374'), "'\u0374'") # printable + self.assertEqual('{0!r}'.format(F('\u0374')), 'F(\u0374)') + self.assertEqual('{0!a}'.format('Hello'), "'Hello'") + self.assertEqual('{0!a}'.format('\u0378'), "'\\u0378'") # nonprintable + self.assertEqual('{0!a}'.format('\u0374'), "'\\u0374'") # printable + self.assertEqual('{0!a:}'.format('Hello'), "'Hello'") + self.assertEqual('{0!a}'.format(F('Hello')), 'F(Hello)') + self.assertEqual('{0!a}'.format(F('\u0374')), 'F(\\u0374)') + + # test fallback to object.__format__ + self.assertEqual('{0}'.format({}), '{}') + self.assertEqual('{0}'.format([]), '[]') + self.assertEqual('{0}'.format([1]), '[1]') + + self.assertEqual('{0:d}'.format(G('data')), 'G(data)') + self.assertEqual('{0!s}'.format(G('data')), 'string is data') + + self.assertRaises(TypeError, '{0:^10}'.format, E('data')) + self.assertRaises(TypeError, '{0:^10s}'.format, E('data')) + self.assertRaises(TypeError, '{0:>15s}'.format, G('data')) + + self.assertEqual("{0:date: %Y-%m-%d}".format(I(year=2007, + month=8, + day=27)), + "date: 2007-08-27") + + # test deriving from a builtin type and overriding __format__ + self.assertEqual("{0}".format(J(10)), "20") + + + # string format specifiers + self.assertEqual('{0:}'.format('a'), 'a') + + # computed format specifiers + self.assertEqual("{0:.{1}}".format('hello world', 5), 'hello') + self.assertEqual("{0:.{1}s}".format('hello world', 5), 'hello') + self.assertEqual("{0:.{precision}s}".format('hello world', precision=5), 'hello') + self.assertEqual("{0:{width}.{precision}s}".format('hello world', width=10, precision=5), 'hello ') + self.assertEqual("{0:{width}.{precision}s}".format('hello world', width='10', precision='5'), 'hello ') + + # test various errors + self.assertRaises(ValueError, '{'.format) + self.assertRaises(ValueError, '}'.format) + self.assertRaises(ValueError, 'a{'.format) + self.assertRaises(ValueError, 'a}'.format) + self.assertRaises(ValueError, '{a'.format) + self.assertRaises(ValueError, '}a'.format) + self.assertRaises(IndexError, '{0}'.format) + self.assertRaises(IndexError, '{1}'.format, 'abc') + self.assertRaises(KeyError, '{x}'.format) + self.assertRaises(ValueError, "}{".format) + self.assertRaises(ValueError, "abc{0:{}".format) + self.assertRaises(ValueError, "{0".format) + self.assertRaises(IndexError, "{0.}".format) + self.assertRaises(ValueError, "{0.}".format, 0) + self.assertRaises(ValueError, "{0[}".format) + self.assertRaises(ValueError, "{0[}".format, []) + self.assertRaises(KeyError, "{0]}".format) + self.assertRaises(ValueError, "{0.[]}".format, 0) + self.assertRaises(ValueError, "{0..foo}".format, 0) + self.assertRaises(ValueError, "{0[0}".format, 0) + self.assertRaises(ValueError, "{0[0:foo}".format, 0) + self.assertRaises(KeyError, "{c]}".format) + self.assertRaises(ValueError, "{{ {{{0}}".format, 0) + self.assertRaises(ValueError, "{0}}".format, 0) + self.assertRaises(KeyError, "{foo}".format, bar=3) + self.assertRaises(ValueError, "{0!x}".format, 3) + self.assertRaises(ValueError, "{0!}".format, 0) + self.assertRaises(ValueError, "{0!rs}".format, 0) + self.assertRaises(ValueError, "{!}".format) + self.assertRaises(IndexError, "{:}".format) + self.assertRaises(IndexError, "{:s}".format) + self.assertRaises(IndexError, "{}".format) + big = "23098475029384702983476098230754973209482573" + self.assertRaises(ValueError, ("{" + big + "}").format) + self.assertRaises(ValueError, ("{[" + big + "]}").format, [0]) + + # test number formatter errors: + self.assertRaises(ValueError, '{0:x}'.format, 1j) + self.assertRaises(ValueError, '{0:x}'.format, 1.0) + self.assertRaises(ValueError, '{0:X}'.format, 1j) + self.assertRaises(ValueError, '{0:X}'.format, 1.0) + self.assertRaises(ValueError, '{0:o}'.format, 1j) + self.assertRaises(ValueError, '{0:o}'.format, 1.0) + self.assertRaises(ValueError, '{0:u}'.format, 1j) + self.assertRaises(ValueError, '{0:u}'.format, 1.0) + self.assertRaises(ValueError, '{0:i}'.format, 1j) + self.assertRaises(ValueError, '{0:i}'.format, 1.0) + self.assertRaises(ValueError, '{0:d}'.format, 1j) + self.assertRaises(ValueError, '{0:d}'.format, 1.0) + + # issue 6089 + self.assertRaises(ValueError, "{0[0]x}".format, [None]) + self.assertRaises(ValueError, "{0[0](10)}".format, [None]) + + # can't have a replacement on the field name portion + self.assertRaises(TypeError, '{0[{1}]}'.format, 'abcdefg', 4) + + # exceed maximum recursion depth + self.assertRaises(ValueError, "{0:{1:{2}}}".format, 'abc', 's', '') + self.assertRaises(ValueError, "{0:{1:{2:{3:{4:{5:{6}}}}}}}".format, + 0, 1, 2, 3, 4, 5, 6, 7) + + # string format spec errors + sign_msg = "Sign not allowed in string format specifier" + self.assertRaisesRegex(ValueError, sign_msg, "{0:-s}".format, '') + self.assertRaisesRegex(ValueError, sign_msg, format, "", "-") + space_msg = "Space not allowed in string format specifier" + self.assertRaisesRegex(ValueError, space_msg, "{: }".format, '') + self.assertRaises(ValueError, "{0:=s}".format, '') + + # Alternate formatting is not supported + self.assertRaises(ValueError, format, '', '#') + self.assertRaises(ValueError, format, '', '#20') + + # Non-ASCII + self.assertEqual("{0:s}{1:s}".format("ABC", "\u0410\u0411\u0412"), + 'ABC\u0410\u0411\u0412') + self.assertEqual("{0:.3s}".format("ABC\u0410\u0411\u0412"), + 'ABC') + self.assertEqual("{0:.0s}".format("ABC\u0410\u0411\u0412"), + '') + + self.assertEqual("{[{}]}".format({"{}": 5}), "5") + self.assertEqual("{[{}]}".format({"{}" : "a"}), "a") + self.assertEqual("{[{]}".format({"{" : "a"}), "a") + self.assertEqual("{[}]}".format({"}" : "a"}), "a") + self.assertEqual("{[[]}".format({"[" : "a"}), "a") + self.assertEqual("{[!]}".format({"!" : "a"}), "a") + self.assertRaises(ValueError, "{a{}b}".format, 42) + self.assertRaises(ValueError, "{a{b}".format, 42) + self.assertRaises(ValueError, "{[}".format, 42) + + self.assertEqual("0x{:0{:d}X}".format(0x0,16), "0x0000000000000000") + + # Blocking fallback + m = M('data') + self.assertEqual("{!r}".format(m), 'M(data)') + self.assertRaises(TypeError, "{!s}".format, m) + self.assertRaises(TypeError, "{}".format, m) + n = N('data') + self.assertEqual("{!r}".format(n), 'N(data)') + self.assertEqual("{!s}".format(n), 'N(data)') + self.assertRaises(TypeError, "{}".format, n) + + def test_format_map(self): + self.assertEqual(''.format_map({}), '') + self.assertEqual('a'.format_map({}), 'a') + self.assertEqual('ab'.format_map({}), 'ab') + self.assertEqual('a{{'.format_map({}), 'a{') + self.assertEqual('a}}'.format_map({}), 'a}') + self.assertEqual('{{b'.format_map({}), '{b') + self.assertEqual('}}b'.format_map({}), '}b') + self.assertEqual('a{{b'.format_map({}), 'a{b') + + # using mappings + class Mapping(dict): + def __missing__(self, key): + return key + self.assertEqual('{hello}'.format_map(Mapping()), 'hello') + self.assertEqual('{a} {world}'.format_map(Mapping(a='hello')), 'hello world') + + class InternalMapping: + def __init__(self): + self.mapping = {'a': 'hello'} + def __getitem__(self, key): + return self.mapping[key] + self.assertEqual('{a}'.format_map(InternalMapping()), 'hello') + + + class C: + def __init__(self, x=100): + self._x = x + def __format__(self, spec): + return spec + self.assertEqual('{foo._x}'.format_map({'foo': C(20)}), '20') + + # test various errors + self.assertRaises(TypeError, ''.format_map) + self.assertRaises(TypeError, 'a'.format_map) + + self.assertRaises(ValueError, '{'.format_map, {}) + self.assertRaises(ValueError, '}'.format_map, {}) + self.assertRaises(ValueError, 'a{'.format_map, {}) + self.assertRaises(ValueError, 'a}'.format_map, {}) + self.assertRaises(ValueError, '{a'.format_map, {}) + self.assertRaises(ValueError, '}a'.format_map, {}) + + # issue #12579: can't supply positional params to format_map + self.assertRaises(ValueError, '{}'.format_map, {'a' : 2}) + self.assertRaises(ValueError, '{}'.format_map, 'a') + self.assertRaises(ValueError, '{a} {}'.format_map, {"a" : 2, "b" : 1}) + + class BadMapping: + def __getitem__(self, key): + return 1/0 + self.assertRaises(KeyError, '{a}'.format_map, {}) + self.assertRaises(TypeError, '{a}'.format_map, []) + self.assertRaises(ZeroDivisionError, '{a}'.format_map, BadMapping()) + + def test_format_huge_precision(self): + format_string = ".{}f".format(sys.maxsize + 1) + with self.assertRaises(ValueError): + result = format(2.34, format_string) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_format_huge_width(self): + format_string = "{}f".format(sys.maxsize + 1) + with self.assertRaises(ValueError): + result = format(2.34, format_string) + + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: tuple index out of range + def test_format_huge_item_number(self): + format_string = "{{{}:.6f}}".format(sys.maxsize + 1) + with self.assertRaises(ValueError): + result = format_string.format(2.34) + + def test_format_auto_numbering(self): + class C: + def __init__(self, x=100): + self._x = x + def __format__(self, spec): + return spec + + self.assertEqual('{}'.format(10), '10') + self.assertEqual('{:5}'.format('s'), 's ') + self.assertEqual('{!r}'.format('s'), "'s'") + self.assertEqual('{._x}'.format(C(10)), '10') + self.assertEqual('{[1]}'.format([1, 2]), '2') + self.assertEqual('{[a]}'.format({'a':4, 'b':2}), '4') + self.assertEqual('a{}b{}c'.format(0, 1), 'a0b1c') + + self.assertEqual('a{:{}}b'.format('x', '^10'), 'a x b') + self.assertEqual('a{:{}x}b'.format(20, '#'), 'a0x14b') + + # can't mix and match numbering and auto-numbering + self.assertRaises(ValueError, '{}{1}'.format, 1, 2) + self.assertRaises(ValueError, '{1}{}'.format, 1, 2) + self.assertRaises(ValueError, '{:{1}}'.format, 1, 2) + self.assertRaises(ValueError, '{0:{}}'.format, 1, 2) + + # can mix and match auto-numbering and named + self.assertEqual('{f}{}'.format(4, f='test'), 'test4') + self.assertEqual('{}{f}'.format(4, f='test'), '4test') + self.assertEqual('{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3') + self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g') + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: %x format: an integer is required, not PseudoInt + def test_formatting(self): + string_tests.StringLikeTest.test_formatting(self) + # Testing Unicode formatting strings... + self.assertEqual("%s, %s" % ("abc", "abc"), 'abc, abc') + self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, 2, 3), 'abc, abc, 1, 2.000000, 3.00') + self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, -2, 3), 'abc, abc, 1, -2.000000, 3.00') + self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.5), 'abc, abc, -1, -2.000000, 3.50') + self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.57), 'abc, abc, -1, -2.000000, 3.57') + self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 1003.57), 'abc, abc, -1, -2.000000, 1003.57') + self.assertEqual("%r, %r" % (b"abc", "abc"), "b'abc', 'abc'") + self.assertEqual("%r" % ("\u1234",), "'\u1234'") + self.assertEqual("%a" % ("\u1234",), "'\\u1234'") + self.assertEqual("%(x)s, %(y)s" % {'x':"abc", 'y':"def"}, 'abc, def') + self.assertEqual("%(x)s, %(\xfc)s" % {'x':"abc", '\xfc':"def"}, 'abc, def') + + self.assertEqual('%c' % 0x1234, '\u1234') + self.assertEqual('%c' % 0x21483, '\U00021483') + self.assertRaises(OverflowError, "%c".__mod__, (0x110000,)) + self.assertEqual('%c' % '\U00021483', '\U00021483') + self.assertRaises(TypeError, "%c".__mod__, "aa") + self.assertRaises(ValueError, "%.1\u1032f".__mod__, (1.0/3)) + self.assertRaises(TypeError, "%i".__mod__, "aa") + + # formatting jobs delegated from the string implementation: + self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...') + self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...') + self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...') + self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...') + self.assertEqual('...%(foo)s...' % {'foo':"abc",'def':123}, '...abc...') + self.assertEqual('...%(foo)s...' % {'foo':"abc",'def':123}, '...abc...') + self.assertEqual('...%s...%s...%s...%s...' % (1,2,3,"abc"), '...1...2...3...abc...') + self.assertEqual('...%%...%%s...%s...%s...%s...%s...' % (1,2,3,"abc"), '...%...%s...1...2...3...abc...') + self.assertEqual('...%s...' % "abc", '...abc...') + self.assertEqual('%*s' % (5,'abc',), ' abc') + self.assertEqual('%*s' % (-5,'abc',), 'abc ') + self.assertEqual('%*.*s' % (5,2,'abc',), ' ab') + self.assertEqual('%*.*s' % (5,3,'abc',), ' abc') + self.assertEqual('%i %*.*s' % (10, 5,3,'abc',), '10 abc') + self.assertEqual('%i%s %*.*s' % (10, 3, 5, 3, 'abc',), '103 abc') + self.assertEqual('%c' % 'a', 'a') + class Wrapper: + def __str__(self): + return '\u1234' + self.assertEqual('%s' % Wrapper(), '\u1234') + + # issue 3382 + NAN = float('nan') + INF = float('inf') + self.assertEqual('%f' % NAN, 'nan') + self.assertEqual('%F' % NAN, 'NAN') + self.assertEqual('%f' % INF, 'inf') + self.assertEqual('%F' % INF, 'INF') + + # PEP 393 + self.assertEqual('%.1s' % "a\xe9\u20ac", 'a') + self.assertEqual('%.2s' % "a\xe9\u20ac", 'a\xe9') + + #issue 19995 + class PseudoInt: + def __init__(self, value): + self.value = int(value) + def __int__(self): + return self.value + def __index__(self): + return self.value + class PseudoFloat: + def __init__(self, value): + self.value = float(value) + def __int__(self): + return int(self.value) + pi = PseudoFloat(3.1415) + letter_m = PseudoInt(109) + self.assertEqual('%x' % 42, '2a') + self.assertEqual('%X' % 15, 'F') + self.assertEqual('%o' % 9, '11') + self.assertEqual('%c' % 109, 'm') + self.assertEqual('%x' % letter_m, '6d') + self.assertEqual('%X' % letter_m, '6D') + self.assertEqual('%o' % letter_m, '155') + self.assertEqual('%c' % letter_m, 'm') + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi) + self.assertRaisesRegex(TypeError, '%x format: an integer is required, not complex', operator.mod, '%x', 3j) + self.assertRaisesRegex(TypeError, '%X format: an integer is required, not complex', operator.mod, '%X', 2j) + self.assertRaisesRegex(TypeError, '%o format: an integer is required, not complex', operator.mod, '%o', 1j) + self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) + self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) + self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) + self.assertRaisesRegex(TypeError, r'%c requires an int or a unicode character, not .*\.PseudoFloat', operator.mod, '%c', pi) + + class RaisingNumber: + def __int__(self): + raise RuntimeError('int') # should not be `TypeError` + def __index__(self): + raise RuntimeError('index') # should not be `TypeError` + + rn = RaisingNumber() + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%d', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%i', rn) + self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%u', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%x', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%X', rn) + self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%o', rn) + + def test_formatting_with_enum(self): + # issue18780 + import enum + class Float(float, enum.Enum): + # a mixed-in type will use the name for %s etc. + PI = 3.1415926 + class Int(enum.IntEnum): + # IntEnum uses the value and not the name for %s etc. + IDES = 15 + class Str(enum.StrEnum): + # StrEnum uses the value and not the name for %s etc. + ABC = 'abc' + # Testing Unicode formatting strings... + self.assertEqual("%s, %s" % (Str.ABC, Str.ABC), + 'abc, abc') + self.assertEqual("%s, %s, %d, %i, %u, %f, %5.2f" % + (Str.ABC, Str.ABC, + Int.IDES, Int.IDES, Int.IDES, + Float.PI, Float.PI), + 'abc, abc, 15, 15, 15, 3.141593, 3.14') + + # formatting jobs delegated from the string implementation: + self.assertEqual('...%(foo)s...' % {'foo':Str.ABC}, + '...abc...') + self.assertEqual('...%(foo)r...' % {'foo':Int.IDES}, + '...<Int.IDES: 15>...') + self.assertEqual('...%(foo)s...' % {'foo':Int.IDES}, + '...15...') + self.assertEqual('...%(foo)i...' % {'foo':Int.IDES}, + '...15...') + self.assertEqual('...%(foo)d...' % {'foo':Int.IDES}, + '...15...') + self.assertEqual('...%(foo)u...' % {'foo':Int.IDES, 'def':Float.PI}, + '...15...') + self.assertEqual('...%(foo)f...' % {'foo':Float.PI,'def':123}, + '...3.141593...') + + def test_formatting_huge_precision(self): + format_string = "%.{}f".format(sys.maxsize + 1) + with self.assertRaises(ValueError): + result = format_string % 2.34 + + def test_issue28598_strsubclass_rhs(self): + # A subclass of str with an __rmod__ method should be able to hook + # into the % operator + class SubclassedStr(str): + def __rmod__(self, other): + return 'Success, self.__rmod__({!r}) was called'.format(other) + self.assertEqual('lhs %% %r' % SubclassedStr('rhs'), + "Success, self.__rmod__('lhs %% %r') was called") + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_formatting_huge_precision_c_limits(self): + format_string = "%.{}f".format(_testcapi.INT_MAX + 1) + with self.assertRaises(ValueError): + result = format_string % 2.34 + + def test_formatting_huge_width(self): + format_string = "%{}f".format(sys.maxsize + 1) + with self.assertRaises(ValueError): + result = format_string % 2.34 + + def test_startswith_endswith_errors(self): + for meth in ('foo'.startswith, 'foo'.endswith): + with self.assertRaises(TypeError) as cm: + meth(['f']) + exc = str(cm.exception) + self.assertIn('str', exc) + self.assertIn('tuple', exc) + + @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '') + def test_format_float(self): + # should not format with a comma, but always with C locale + self.assertEqual('1.0', '%.1f' % 1.0) + + def test_constructor(self): + # unicode(obj) tests (this maps to PyObject_Unicode() at C level) + + self.assertEqual( + str('unicode remains unicode'), + 'unicode remains unicode' + ) + + for text in ('ascii', '\xe9', '\u20ac', '\U0010FFFF'): + subclass = StrSubclass(text) + self.assertEqual(str(subclass), text) + self.assertEqual(len(subclass), len(text)) + if text == 'ascii': + self.assertEqual(subclass.encode('ascii'), b'ascii') + self.assertEqual(subclass.encode('utf-8'), b'ascii') + + self.assertEqual( + str('strings are converted to unicode'), + 'strings are converted to unicode' + ) + + class StringCompat: + def __init__(self, x): + self.x = x + def __str__(self): + return self.x + + self.assertEqual( + str(StringCompat('__str__ compatible objects are recognized')), + '__str__ compatible objects are recognized' + ) + + # unicode(obj) is compatible to str(): + + o = StringCompat('unicode(obj) is compatible to str()') + self.assertEqual(str(o), 'unicode(obj) is compatible to str()') + self.assertEqual(str(o), 'unicode(obj) is compatible to str()') + + for obj in (123, 123.45, 123): + self.assertEqual(str(obj), str(str(obj))) + + # unicode(obj, encoding, error) tests (this maps to + # PyUnicode_FromEncodedObject() at C level) + + self.assertRaises( + TypeError, + str, + 'decoding unicode is not supported', + 'utf-8', + 'strict' + ) + + self.assertEqual( + str(b'strings are decoded to unicode', 'utf-8', 'strict'), + 'strings are decoded to unicode' + ) + + self.assertEqual( + str( + memoryview(b'character buffers are decoded to unicode'), + 'utf-8', + 'strict' + ), + 'character buffers are decoded to unicode' + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Pass various keyword argument combinations to the constructor. + def test_constructor_keyword_args(self): + """Pass various keyword argument combinations to the constructor.""" + # The object argument can be passed as a keyword. + self.assertEqual(str(object='foo'), 'foo') + self.assertEqual(str(object=b'foo', encoding='utf-8'), 'foo') + # The errors argument without encoding triggers "decode" mode. + self.assertEqual(str(b'foo', errors='strict'), 'foo') # not "b'foo'" + self.assertEqual(str(object=b'foo', errors='strict'), 'foo') + + @unittest.expectedFailure # TODO: RUSTPYTHON; Check the constructor argument defaults. + def test_constructor_defaults(self): + """Check the constructor argument defaults.""" + # The object argument defaults to '' or b''. + self.assertEqual(str(), '') + self.assertEqual(str(errors='strict'), '') + utf8_cent = '¢'.encode('utf-8') + # The encoding argument defaults to utf-8. + self.assertEqual(str(utf8_cent, errors='strict'), '¢') + # The errors argument defaults to strict. + self.assertRaises(UnicodeDecodeError, str, utf8_cent, encoding='ascii') + + def test_codecs_utf7(self): + utfTests = [ + ('A\u2262\u0391.', b'A+ImIDkQ.'), # RFC2152 example + ('Hi Mom -\u263a-!', b'Hi Mom -+Jjo--!'), # RFC2152 example + ('\u65E5\u672C\u8A9E', b'+ZeVnLIqe-'), # RFC2152 example + ('Item 3 is \u00a31.', b'Item 3 is +AKM-1.'), # RFC2152 example + ('+', b'+-'), + ('+-', b'+--'), + ('+?', b'+-?'), + (r'\?', b'+AFw?'), + ('+?', b'+-?'), + (r'\\?', b'+AFwAXA?'), + (r'\\\?', b'+AFwAXABc?'), + (r'++--', b'+-+---'), + ('\U000abcde', b'+2m/c3g-'), # surrogate pairs + ('/', b'/'), + ] + + for (x, y) in utfTests: + self.assertEqual(x.encode('utf-7'), y) + + # Unpaired surrogates are passed through + self.assertEqual('\uD801'.encode('utf-7'), b'+2AE-') + self.assertEqual('\uD801x'.encode('utf-7'), b'+2AE-x') + self.assertEqual('\uDC01'.encode('utf-7'), b'+3AE-') + self.assertEqual('\uDC01x'.encode('utf-7'), b'+3AE-x') + self.assertEqual(b'+2AE-'.decode('utf-7'), '\uD801') + self.assertEqual(b'+2AE-x'.decode('utf-7'), '\uD801x') + self.assertEqual(b'+3AE-'.decode('utf-7'), '\uDC01') + self.assertEqual(b'+3AE-x'.decode('utf-7'), '\uDC01x') + + self.assertEqual('\uD801\U000abcde'.encode('utf-7'), b'+2AHab9ze-') + self.assertEqual(b'+2AHab9ze-'.decode('utf-7'), '\uD801\U000abcde') + + # Issue #2242: crash on some Windows/MSVC versions + self.assertEqual(b'+\xc1'.decode('utf-7', 'ignore'), '') + + # Direct encoded characters + set_d = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'(),-./:?" + # Optional direct characters + set_o = '!"#$%&*;<=>@[]^_`{|}' + for c in set_d: + self.assertEqual(c.encode('utf7'), c.encode('ascii')) + self.assertEqual(c.encode('ascii').decode('utf7'), c) + for c in set_o: + self.assertEqual(c.encode('ascii').decode('utf7'), c) + + with self.assertRaisesRegex(UnicodeDecodeError, + 'ill-formed sequence'): + b'+@'.decode('utf-7') + + def test_codecs_utf8(self): + self.assertEqual(''.encode('utf-8'), b'') + self.assertEqual('\u20ac'.encode('utf-8'), b'\xe2\x82\xac') + self.assertEqual('\U00010002'.encode('utf-8'), b'\xf0\x90\x80\x82') + self.assertEqual('\U00023456'.encode('utf-8'), b'\xf0\xa3\x91\x96') + self.assertEqual('\ud800'.encode('utf-8', 'surrogatepass'), b'\xed\xa0\x80') + self.assertEqual('\udc00'.encode('utf-8', 'surrogatepass'), b'\xed\xb0\x80') + self.assertEqual(('\U00010002'*10).encode('utf-8'), + b'\xf0\x90\x80\x82'*10) + self.assertEqual( + '\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' + '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' + '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c' + '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067' + '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das' + ' Nunstuck git und'.encode('utf-8'), + b'\xe6\xad\xa3\xe7\xa2\xba\xe3\x81\xab\xe8\xa8\x80\xe3\x81' + b'\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3\xe3\x81\xaf\xe3' + b'\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe' + b'\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' + b'\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8' + b'\xaa\x9e\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81' + b'\xe3\x81\x82\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81' + b'\x9f\xe3\x82\x89\xe3\x82\x81\xe3\x81\xa7\xe3\x81\x99\xe3' + b'\x80\x82\xe5\xae\x9f\xe9\x9a\x9b\xe3\x81\xab\xe3\x81\xaf' + b'\xe3\x80\x8cWenn ist das Nunstuck git und' + ) + + # UTF-8 specific decoding tests + self.assertEqual(str(b'\xf0\xa3\x91\x96', 'utf-8'), '\U00023456' ) + self.assertEqual(str(b'\xf0\x90\x80\x82', 'utf-8'), '\U00010002' ) + self.assertEqual(str(b'\xe2\x82\xac', 'utf-8'), '\u20ac' ) + + # Other possible utf-8 test cases: + # * strict decoding testing for all of the + # UTF8_ERROR cases in PyUnicode_DecodeUTF8 + + def test_utf8_decode_valid_sequences(self): + sequences = [ + # single byte + (b'\x00', '\x00'), (b'a', 'a'), (b'\x7f', '\x7f'), + # 2 bytes + (b'\xc2\x80', '\x80'), (b'\xdf\xbf', '\u07ff'), + # 3 bytes + (b'\xe0\xa0\x80', '\u0800'), (b'\xed\x9f\xbf', '\ud7ff'), + (b'\xee\x80\x80', '\uE000'), (b'\xef\xbf\xbf', '\uffff'), + # 4 bytes + (b'\xF0\x90\x80\x80', '\U00010000'), + (b'\xf4\x8f\xbf\xbf', '\U0010FFFF') + ] + for seq, res in sequences: + self.assertEqual(seq.decode('utf-8'), res) + + + def test_utf8_decode_invalid_sequences(self): + # continuation bytes in a sequence of 2, 3, or 4 bytes + continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)] + # start bytes of a 2-byte sequence equivalent to code points < 0x7F + invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)] + # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF + invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)] + invalid_start_bytes = ( + continuation_bytes + invalid_2B_seq_start_bytes + + invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)] + ) + + for byte in invalid_start_bytes: + self.assertRaises(UnicodeDecodeError, byte.decode, 'utf-8') + + for sb in invalid_2B_seq_start_bytes: + for cb in continuation_bytes: + self.assertRaises(UnicodeDecodeError, (sb+cb).decode, 'utf-8') + + for sb in invalid_4B_seq_start_bytes: + for cb1 in continuation_bytes[:3]: + for cb3 in continuation_bytes[:3]: + self.assertRaises(UnicodeDecodeError, + (sb+cb1+b'\x80'+cb3).decode, 'utf-8') + + for cb in [bytes([x]) for x in range(0x80, 0xA0)]: + self.assertRaises(UnicodeDecodeError, + (b'\xE0'+cb+b'\x80').decode, 'utf-8') + self.assertRaises(UnicodeDecodeError, + (b'\xE0'+cb+b'\xBF').decode, 'utf-8') + # surrogates + for cb in [bytes([x]) for x in range(0xA0, 0xC0)]: + self.assertRaises(UnicodeDecodeError, + (b'\xED'+cb+b'\x80').decode, 'utf-8') + self.assertRaises(UnicodeDecodeError, + (b'\xED'+cb+b'\xBF').decode, 'utf-8') + for cb in [bytes([x]) for x in range(0x80, 0x90)]: + self.assertRaises(UnicodeDecodeError, + (b'\xF0'+cb+b'\x80\x80').decode, 'utf-8') + self.assertRaises(UnicodeDecodeError, + (b'\xF0'+cb+b'\xBF\xBF').decode, 'utf-8') + for cb in [bytes([x]) for x in range(0x90, 0xC0)]: + self.assertRaises(UnicodeDecodeError, + (b'\xF4'+cb+b'\x80\x80').decode, 'utf-8') + self.assertRaises(UnicodeDecodeError, + (b'\xF4'+cb+b'\xBF\xBF').decode, 'utf-8') + + def test_issue127903(self): + # gh-127903: ``_copy_characters`` crashes on DEBUG builds when + # there is nothing to copy. + d = datetime.datetime(2013, 11, 10, 14, 20, 59) + self.assertEqual(d.strftime('%z'), '') + + def test_issue8271(self): + # Issue #8271: during the decoding of an invalid UTF-8 byte sequence, + # only the start byte and the continuation byte(s) are now considered + # invalid, instead of the number of bytes specified by the start byte. + # See https://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (page 95, + # table 3-8, Row 2) for more information about the algorithm used. + FFFD = '\ufffd' + sequences = [ + # invalid start bytes + (b'\x80', FFFD), # continuation byte + (b'\x80\x80', FFFD*2), # 2 continuation bytes + (b'\xc0', FFFD), + (b'\xc0\xc0', FFFD*2), + (b'\xc1', FFFD), + (b'\xc1\xc0', FFFD*2), + (b'\xc0\xc1', FFFD*2), + # with start byte of a 2-byte sequence + (b'\xc2', FFFD), # only the start byte + (b'\xc2\xc2', FFFD*2), # 2 start bytes + (b'\xc2\xc2\xc2', FFFD*3), # 3 start bytes + (b'\xc2\x41', FFFD+'A'), # invalid continuation byte + # with start byte of a 3-byte sequence + (b'\xe1', FFFD), # only the start byte + (b'\xe1\xe1', FFFD*2), # 2 start bytes + (b'\xe1\xe1\xe1', FFFD*3), # 3 start bytes + (b'\xe1\xe1\xe1\xe1', FFFD*4), # 4 start bytes + (b'\xe1\x80', FFFD), # only 1 continuation byte + (b'\xe1\x41', FFFD+'A'), # invalid continuation byte + (b'\xe1\x41\x80', FFFD+'A'+FFFD), # invalid cb followed by valid cb + (b'\xe1\x41\x41', FFFD+'AA'), # 2 invalid continuation bytes + (b'\xe1\x80\x41', FFFD+'A'), # only 1 valid continuation byte + (b'\xe1\x80\xe1\x41', FFFD*2+'A'), # 1 valid and the other invalid + (b'\xe1\x41\xe1\x80', FFFD+'A'+FFFD), # 1 invalid and the other valid + # with start byte of a 4-byte sequence + (b'\xf1', FFFD), # only the start byte + (b'\xf1\xf1', FFFD*2), # 2 start bytes + (b'\xf1\xf1\xf1', FFFD*3), # 3 start bytes + (b'\xf1\xf1\xf1\xf1', FFFD*4), # 4 start bytes + (b'\xf1\xf1\xf1\xf1\xf1', FFFD*5), # 5 start bytes + (b'\xf1\x80', FFFD), # only 1 continuation bytes + (b'\xf1\x80\x80', FFFD), # only 2 continuation bytes + (b'\xf1\x80\x41', FFFD+'A'), # 1 valid cb and 1 invalid + (b'\xf1\x80\x41\x41', FFFD+'AA'), # 1 valid cb and 1 invalid + (b'\xf1\x80\x80\x41', FFFD+'A'), # 2 valid cb and 1 invalid + (b'\xf1\x41\x80', FFFD+'A'+FFFD), # 1 invalid cv and 1 valid + (b'\xf1\x41\x80\x80', FFFD+'A'+FFFD*2), # 1 invalid cb and 2 invalid + (b'\xf1\x41\x80\x41', FFFD+'A'+FFFD+'A'), # 2 invalid cb and 1 invalid + (b'\xf1\x41\x41\x80', FFFD+'AA'+FFFD), # 1 valid cb and 1 invalid + (b'\xf1\x41\xf1\x80', FFFD+'A'+FFFD), + (b'\xf1\x41\x80\xf1', FFFD+'A'+FFFD*2), + (b'\xf1\xf1\x80\x41', FFFD*2+'A'), + (b'\xf1\x41\xf1\xf1', FFFD+'A'+FFFD*2), + # with invalid start byte of a 4-byte sequence (rfc2279) + (b'\xf5', FFFD), # only the start byte + (b'\xf5\xf5', FFFD*2), # 2 start bytes + (b'\xf5\x80', FFFD*2), # only 1 continuation byte + (b'\xf5\x80\x80', FFFD*3), # only 2 continuation byte + (b'\xf5\x80\x80\x80', FFFD*4), # 3 continuation bytes + (b'\xf5\x80\x41', FFFD*2+'A'), # 1 valid cb and 1 invalid + (b'\xf5\x80\x41\xf5', FFFD*2+'A'+FFFD), + (b'\xf5\x41\x80\x80\x41', FFFD+'A'+FFFD*2+'A'), + # with invalid start byte of a 5-byte sequence (rfc2279) + (b'\xf8', FFFD), # only the start byte + (b'\xf8\xf8', FFFD*2), # 2 start bytes + (b'\xf8\x80', FFFD*2), # only one continuation byte + (b'\xf8\x80\x41', FFFD*2 + 'A'), # 1 valid cb and 1 invalid + (b'\xf8\x80\x80\x80\x80', FFFD*5), # invalid 5 bytes seq with 5 bytes + # with invalid start byte of a 6-byte sequence (rfc2279) + (b'\xfc', FFFD), # only the start byte + (b'\xfc\xfc', FFFD*2), # 2 start bytes + (b'\xfc\x80\x80', FFFD*3), # only 2 continuation bytes + (b'\xfc\x80\x80\x80\x80\x80', FFFD*6), # 6 continuation bytes + # invalid start byte + (b'\xfe', FFFD), + (b'\xfe\x80\x80', FFFD*3), + # other sequences + (b'\xf1\x80\x41\x42\x43', '\ufffd\x41\x42\x43'), + (b'\xf1\x80\xff\x42\x43', '\ufffd\ufffd\x42\x43'), + (b'\xf1\x80\xc2\x81\x43', '\ufffd\x81\x43'), + (b'\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64', + '\x61\uFFFD\uFFFD\uFFFD\x62\uFFFD\x63\uFFFD\uFFFD\x64'), + ] + for n, (seq, res) in enumerate(sequences): + self.assertRaises(UnicodeDecodeError, seq.decode, 'utf-8', 'strict') + self.assertEqual(seq.decode('utf-8', 'replace'), res) + self.assertEqual((seq+b'b').decode('utf-8', 'replace'), res+'b') + self.assertEqual(seq.decode('utf-8', 'ignore'), + res.replace('\uFFFD', '')) + + def assertCorrectUTF8Decoding(self, seq, res, err): + """ + Check that an invalid UTF-8 sequence raises a UnicodeDecodeError when + 'strict' is used, returns res when 'replace' is used, and that doesn't + return anything when 'ignore' is used. + """ + with self.assertRaises(UnicodeDecodeError) as cm: + seq.decode('utf-8') + exc = cm.exception + + self.assertIn(err, str(exc)) + self.assertEqual(seq.decode('utf-8', 'replace'), res) + self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'replace'), + 'aaaa' + res + 'bbbb') + res = res.replace('\ufffd', '') + self.assertEqual(seq.decode('utf-8', 'ignore'), res) + self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore'), + 'aaaa' + res + 'bbbb') + + def test_invalid_start_byte(self): + """ + Test that an 'invalid start byte' error is raised when the first byte + is not in the ASCII range or is not a valid start byte of a 2-, 3-, or + 4-bytes sequence. The invalid start byte is replaced with a single + U+FFFD when errors='replace'. + E.g. <80> is a continuation byte and can appear only after a start byte. + """ + FFFD = '\ufffd' + for byte in b'\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF': + self.assertCorrectUTF8Decoding(bytes([byte]), '\ufffd', + 'invalid start byte') + + def test_unexpected_end_of_data(self): + """ + Test that an 'unexpected end of data' error is raised when the string + ends after a start byte of a 2-, 3-, or 4-bytes sequence without having + enough continuation bytes. The incomplete sequence is replaced with a + single U+FFFD when errors='replace'. + E.g. in the sequence <F3 80 80>, F3 is the start byte of a 4-bytes + sequence, but it's followed by only 2 valid continuation bytes and the + last continuation bytes is missing. + Note: the continuation bytes must be all valid, if one of them is + invalid another error will be raised. + """ + sequences = [ + 'C2', 'DF', + 'E0 A0', 'E0 BF', 'E1 80', 'E1 BF', 'EC 80', 'EC BF', + 'ED 80', 'ED 9F', 'EE 80', 'EE BF', 'EF 80', 'EF BF', + 'F0 90', 'F0 BF', 'F0 90 80', 'F0 90 BF', 'F0 BF 80', 'F0 BF BF', + 'F1 80', 'F1 BF', 'F1 80 80', 'F1 80 BF', 'F1 BF 80', 'F1 BF BF', + 'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF', + 'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF' + ] + FFFD = '\ufffd' + for seq in sequences: + self.assertCorrectUTF8Decoding(bytes.fromhex(seq), '\ufffd', + 'unexpected end of data') + + def test_invalid_cb_for_2bytes_seq(self): + """ + Test that an 'invalid continuation byte' error is raised when the + continuation byte of a 2-bytes sequence is invalid. The start byte + is replaced by a single U+FFFD and the second byte is handled + separately when errors='replace'. + E.g. in the sequence <C2 41>, C2 is the start byte of a 2-bytes + sequence, but 41 is not a valid continuation byte because it's the + ASCII letter 'A'. + """ + FFFD = '\ufffd' + FFFDx2 = FFFD * 2 + sequences = [ + ('C2 00', FFFD+'\x00'), ('C2 7F', FFFD+'\x7f'), + ('C2 C0', FFFDx2), ('C2 FF', FFFDx2), + ('DF 00', FFFD+'\x00'), ('DF 7F', FFFD+'\x7f'), + ('DF C0', FFFDx2), ('DF FF', FFFDx2), + ] + for seq, res in sequences: + self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res, + 'invalid continuation byte') + + def test_invalid_cb_for_3bytes_seq(self): + """ + Test that an 'invalid continuation byte' error is raised when the + continuation byte(s) of a 3-bytes sequence are invalid. When + errors='replace', if the first continuation byte is valid, the first + two bytes (start byte + 1st cb) are replaced by a single U+FFFD and the + third byte is handled separately, otherwise only the start byte is + replaced with a U+FFFD and the other continuation bytes are handled + separately. + E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes + sequence, 80 is a valid continuation byte, but 41 is not a valid cb + because it's the ASCII letter 'A'. + Note: when the start byte is E0 or ED, the valid ranges for the first + continuation byte are limited to A0..BF and 80..9F respectively. + Python 2 used to consider all the bytes in range 80..BF valid when the + start byte was ED. This is fixed in Python 3. + """ + FFFD = '\ufffd' + FFFDx2 = FFFD * 2 + sequences = [ + ('E0 00', FFFD+'\x00'), ('E0 7F', FFFD+'\x7f'), ('E0 80', FFFDx2), + ('E0 9F', FFFDx2), ('E0 C0', FFFDx2), ('E0 FF', FFFDx2), + ('E0 A0 00', FFFD+'\x00'), ('E0 A0 7F', FFFD+'\x7f'), + ('E0 A0 C0', FFFDx2), ('E0 A0 FF', FFFDx2), + ('E0 BF 00', FFFD+'\x00'), ('E0 BF 7F', FFFD+'\x7f'), + ('E0 BF C0', FFFDx2), ('E0 BF FF', FFFDx2), ('E1 00', FFFD+'\x00'), + ('E1 7F', FFFD+'\x7f'), ('E1 C0', FFFDx2), ('E1 FF', FFFDx2), + ('E1 80 00', FFFD+'\x00'), ('E1 80 7F', FFFD+'\x7f'), + ('E1 80 C0', FFFDx2), ('E1 80 FF', FFFDx2), + ('E1 BF 00', FFFD+'\x00'), ('E1 BF 7F', FFFD+'\x7f'), + ('E1 BF C0', FFFDx2), ('E1 BF FF', FFFDx2), ('EC 00', FFFD+'\x00'), + ('EC 7F', FFFD+'\x7f'), ('EC C0', FFFDx2), ('EC FF', FFFDx2), + ('EC 80 00', FFFD+'\x00'), ('EC 80 7F', FFFD+'\x7f'), + ('EC 80 C0', FFFDx2), ('EC 80 FF', FFFDx2), + ('EC BF 00', FFFD+'\x00'), ('EC BF 7F', FFFD+'\x7f'), + ('EC BF C0', FFFDx2), ('EC BF FF', FFFDx2), ('ED 00', FFFD+'\x00'), + ('ED 7F', FFFD+'\x7f'), + ('ED A0', FFFDx2), ('ED BF', FFFDx2), # see note ^ + ('ED C0', FFFDx2), ('ED FF', FFFDx2), ('ED 80 00', FFFD+'\x00'), + ('ED 80 7F', FFFD+'\x7f'), ('ED 80 C0', FFFDx2), + ('ED 80 FF', FFFDx2), ('ED 9F 00', FFFD+'\x00'), + ('ED 9F 7F', FFFD+'\x7f'), ('ED 9F C0', FFFDx2), + ('ED 9F FF', FFFDx2), ('EE 00', FFFD+'\x00'), + ('EE 7F', FFFD+'\x7f'), ('EE C0', FFFDx2), ('EE FF', FFFDx2), + ('EE 80 00', FFFD+'\x00'), ('EE 80 7F', FFFD+'\x7f'), + ('EE 80 C0', FFFDx2), ('EE 80 FF', FFFDx2), + ('EE BF 00', FFFD+'\x00'), ('EE BF 7F', FFFD+'\x7f'), + ('EE BF C0', FFFDx2), ('EE BF FF', FFFDx2), ('EF 00', FFFD+'\x00'), + ('EF 7F', FFFD+'\x7f'), ('EF C0', FFFDx2), ('EF FF', FFFDx2), + ('EF 80 00', FFFD+'\x00'), ('EF 80 7F', FFFD+'\x7f'), + ('EF 80 C0', FFFDx2), ('EF 80 FF', FFFDx2), + ('EF BF 00', FFFD+'\x00'), ('EF BF 7F', FFFD+'\x7f'), + ('EF BF C0', FFFDx2), ('EF BF FF', FFFDx2), + ] + for seq, res in sequences: + self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res, + 'invalid continuation byte') + + def test_invalid_cb_for_4bytes_seq(self): + """ + Test that an 'invalid continuation byte' error is raised when the + continuation byte(s) of a 4-bytes sequence are invalid. When + errors='replace',the start byte and all the following valid + continuation bytes are replaced with a single U+FFFD, and all the bytes + starting from the first invalid continuation bytes (included) are + handled separately. + E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes + sequence, 80 is a valid continuation byte, but 41 is not a valid cb + because it's the ASCII letter 'A'. + Note: when the start byte is E0 or ED, the valid ranges for the first + continuation byte are limited to A0..BF and 80..9F respectively. + However, when the start byte is ED, Python 2 considers all the bytes + in range 80..BF valid. This is fixed in Python 3. + """ + FFFD = '\ufffd' + FFFDx2 = FFFD * 2 + sequences = [ + ('F0 00', FFFD+'\x00'), ('F0 7F', FFFD+'\x7f'), ('F0 80', FFFDx2), + ('F0 8F', FFFDx2), ('F0 C0', FFFDx2), ('F0 FF', FFFDx2), + ('F0 90 00', FFFD+'\x00'), ('F0 90 7F', FFFD+'\x7f'), + ('F0 90 C0', FFFDx2), ('F0 90 FF', FFFDx2), + ('F0 BF 00', FFFD+'\x00'), ('F0 BF 7F', FFFD+'\x7f'), + ('F0 BF C0', FFFDx2), ('F0 BF FF', FFFDx2), + ('F0 90 80 00', FFFD+'\x00'), ('F0 90 80 7F', FFFD+'\x7f'), + ('F0 90 80 C0', FFFDx2), ('F0 90 80 FF', FFFDx2), + ('F0 90 BF 00', FFFD+'\x00'), ('F0 90 BF 7F', FFFD+'\x7f'), + ('F0 90 BF C0', FFFDx2), ('F0 90 BF FF', FFFDx2), + ('F0 BF 80 00', FFFD+'\x00'), ('F0 BF 80 7F', FFFD+'\x7f'), + ('F0 BF 80 C0', FFFDx2), ('F0 BF 80 FF', FFFDx2), + ('F0 BF BF 00', FFFD+'\x00'), ('F0 BF BF 7F', FFFD+'\x7f'), + ('F0 BF BF C0', FFFDx2), ('F0 BF BF FF', FFFDx2), + ('F1 00', FFFD+'\x00'), ('F1 7F', FFFD+'\x7f'), ('F1 C0', FFFDx2), + ('F1 FF', FFFDx2), ('F1 80 00', FFFD+'\x00'), + ('F1 80 7F', FFFD+'\x7f'), ('F1 80 C0', FFFDx2), + ('F1 80 FF', FFFDx2), ('F1 BF 00', FFFD+'\x00'), + ('F1 BF 7F', FFFD+'\x7f'), ('F1 BF C0', FFFDx2), + ('F1 BF FF', FFFDx2), ('F1 80 80 00', FFFD+'\x00'), + ('F1 80 80 7F', FFFD+'\x7f'), ('F1 80 80 C0', FFFDx2), + ('F1 80 80 FF', FFFDx2), ('F1 80 BF 00', FFFD+'\x00'), + ('F1 80 BF 7F', FFFD+'\x7f'), ('F1 80 BF C0', FFFDx2), + ('F1 80 BF FF', FFFDx2), ('F1 BF 80 00', FFFD+'\x00'), + ('F1 BF 80 7F', FFFD+'\x7f'), ('F1 BF 80 C0', FFFDx2), + ('F1 BF 80 FF', FFFDx2), ('F1 BF BF 00', FFFD+'\x00'), + ('F1 BF BF 7F', FFFD+'\x7f'), ('F1 BF BF C0', FFFDx2), + ('F1 BF BF FF', FFFDx2), ('F3 00', FFFD+'\x00'), + ('F3 7F', FFFD+'\x7f'), ('F3 C0', FFFDx2), ('F3 FF', FFFDx2), + ('F3 80 00', FFFD+'\x00'), ('F3 80 7F', FFFD+'\x7f'), + ('F3 80 C0', FFFDx2), ('F3 80 FF', FFFDx2), + ('F3 BF 00', FFFD+'\x00'), ('F3 BF 7F', FFFD+'\x7f'), + ('F3 BF C0', FFFDx2), ('F3 BF FF', FFFDx2), + ('F3 80 80 00', FFFD+'\x00'), ('F3 80 80 7F', FFFD+'\x7f'), + ('F3 80 80 C0', FFFDx2), ('F3 80 80 FF', FFFDx2), + ('F3 80 BF 00', FFFD+'\x00'), ('F3 80 BF 7F', FFFD+'\x7f'), + ('F3 80 BF C0', FFFDx2), ('F3 80 BF FF', FFFDx2), + ('F3 BF 80 00', FFFD+'\x00'), ('F3 BF 80 7F', FFFD+'\x7f'), + ('F3 BF 80 C0', FFFDx2), ('F3 BF 80 FF', FFFDx2), + ('F3 BF BF 00', FFFD+'\x00'), ('F3 BF BF 7F', FFFD+'\x7f'), + ('F3 BF BF C0', FFFDx2), ('F3 BF BF FF', FFFDx2), + ('F4 00', FFFD+'\x00'), ('F4 7F', FFFD+'\x7f'), ('F4 90', FFFDx2), + ('F4 BF', FFFDx2), ('F4 C0', FFFDx2), ('F4 FF', FFFDx2), + ('F4 80 00', FFFD+'\x00'), ('F4 80 7F', FFFD+'\x7f'), + ('F4 80 C0', FFFDx2), ('F4 80 FF', FFFDx2), + ('F4 8F 00', FFFD+'\x00'), ('F4 8F 7F', FFFD+'\x7f'), + ('F4 8F C0', FFFDx2), ('F4 8F FF', FFFDx2), + ('F4 80 80 00', FFFD+'\x00'), ('F4 80 80 7F', FFFD+'\x7f'), + ('F4 80 80 C0', FFFDx2), ('F4 80 80 FF', FFFDx2), + ('F4 80 BF 00', FFFD+'\x00'), ('F4 80 BF 7F', FFFD+'\x7f'), + ('F4 80 BF C0', FFFDx2), ('F4 80 BF FF', FFFDx2), + ('F4 8F 80 00', FFFD+'\x00'), ('F4 8F 80 7F', FFFD+'\x7f'), + ('F4 8F 80 C0', FFFDx2), ('F4 8F 80 FF', FFFDx2), + ('F4 8F BF 00', FFFD+'\x00'), ('F4 8F BF 7F', FFFD+'\x7f'), + ('F4 8F BF C0', FFFDx2), ('F4 8F BF FF', FFFDx2) + ] + for seq, res in sequences: + self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res, + 'invalid continuation byte') + + def test_codecs_idna(self): + # Test whether trailing dot is preserved + self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.") + + def test_codecs_errors(self): + # Error handling (encoding) + self.assertRaises(UnicodeError, 'Andr\202 x'.encode, 'ascii') + self.assertRaises(UnicodeError, 'Andr\202 x'.encode, 'ascii','strict') + self.assertEqual('Andr\202 x'.encode('ascii','ignore'), b"Andr x") + self.assertEqual('Andr\202 x'.encode('ascii','replace'), b"Andr? x") + self.assertEqual('Andr\202 x'.encode('ascii', 'replace'), + 'Andr\202 x'.encode('ascii', errors='replace')) + self.assertEqual('Andr\202 x'.encode('ascii', 'ignore'), + 'Andr\202 x'.encode(encoding='ascii', errors='ignore')) + + # Error handling (decoding) + self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii') + self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii', 'strict') + self.assertEqual(str(b'Andr\202 x', 'ascii', 'ignore'), "Andr x") + self.assertEqual(str(b'Andr\202 x', 'ascii', 'replace'), 'Andr\uFFFD x') + self.assertEqual(str(b'\202 x', 'ascii', 'replace'), '\uFFFD x') + + # Error handling (unknown character names) + self.assertEqual(b"\\N{foo}xx".decode("unicode-escape", "ignore"), "xx") + + # Error handling (truncated escape sequence) + self.assertRaises(UnicodeError, b"\\".decode, "unicode-escape") + + self.assertRaises(TypeError, b"hello".decode, "test.unicode1") + self.assertRaises(TypeError, str, b"hello", "test.unicode2") + self.assertRaises(TypeError, "hello".encode, "test.unicode1") + self.assertRaises(TypeError, "hello".encode, "test.unicode2") + + # Error handling (wrong arguments) + self.assertRaises(TypeError, "hello".encode, 42, 42, 42) + + # Error handling (lone surrogate in + # _PyUnicode_TransformDecimalAndSpaceToASCII()) + self.assertRaises(ValueError, int, "\ud800") + self.assertRaises(ValueError, int, "\udf00") + self.assertRaises(ValueError, float, "\ud800") + self.assertRaises(ValueError, float, "\udf00") + self.assertRaises(ValueError, complex, "\ud800") + self.assertRaises(ValueError, complex, "\udf00") + + def test_codecs(self): + # Encoding + self.assertEqual('hello'.encode('ascii'), b'hello') + self.assertEqual('hello'.encode('utf-7'), b'hello') + self.assertEqual('hello'.encode('utf-8'), b'hello') + self.assertEqual('hello'.encode('utf-8'), b'hello') + self.assertEqual('hello'.encode('utf-16-le'), b'h\000e\000l\000l\000o\000') + self.assertEqual('hello'.encode('utf-16-be'), b'\000h\000e\000l\000l\000o') + self.assertEqual('hello'.encode('latin-1'), b'hello') + + # Default encoding is utf-8 + self.assertEqual('\u2603'.encode(), b'\xe2\x98\x83') + + # Roundtrip safety for BMP (just the first 1024 chars) + for c in range(1024): + u = chr(c) + for encoding in ('utf-7', 'utf-8', 'utf-16', 'utf-16-le', + 'utf-16-be', 'raw_unicode_escape', + 'unicode_escape'): + self.assertEqual(str(u.encode(encoding),encoding), u) + + # Roundtrip safety for BMP (just the first 256 chars) + for c in range(256): + u = chr(c) + for encoding in ('latin-1',): + self.assertEqual(str(u.encode(encoding),encoding), u) + + # Roundtrip safety for BMP (just the first 128 chars) + for c in range(128): + u = chr(c) + for encoding in ('ascii',): + self.assertEqual(str(u.encode(encoding),encoding), u) + + # Roundtrip safety for non-BMP (just a few chars) + with warnings.catch_warnings(): + u = '\U00010001\U00020002\U00030003\U00040004\U00050005' + for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', + 'raw_unicode_escape', 'unicode_escape'): + self.assertEqual(str(u.encode(encoding),encoding), u) + + # UTF-8 must be roundtrip safe for all code points + # (except surrogates, which are forbidden). + u = ''.join(map(chr, list(range(0, 0xd800)) + + list(range(0xe000, 0x110000)))) + for encoding in ('utf-8',): + self.assertEqual(str(u.encode(encoding),encoding), u) + + def test_codecs_charmap(self): + # 0-127 + s = bytes(range(128)) + for encoding in ( + 'cp037', 'cp1026', 'cp273', + 'cp437', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850', + 'cp852', 'cp855', 'cp858', 'cp860', 'cp861', 'cp862', + 'cp863', 'cp865', 'cp866', 'cp1125', + 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15', + 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6', + 'iso8859_7', 'iso8859_9', + 'koi8_r', 'koi8_t', 'koi8_u', 'kz1048', 'latin_1', + 'mac_cyrillic', 'mac_latin2', + + 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', + 'cp1256', 'cp1257', 'cp1258', + 'cp856', 'cp857', 'cp864', 'cp869', 'cp874', + + 'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish', + 'cp1006', 'iso8859_8', + + ### These have undefined mappings: + #'cp424', + + ### These fail the round-trip: + #'cp875' + + ): + self.assertEqual(str(s, encoding).encode(encoding), s) + + # 128-255 + s = bytes(range(128, 256)) + for encoding in ( + 'cp037', 'cp1026', 'cp273', + 'cp437', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850', + 'cp852', 'cp855', 'cp858', 'cp860', 'cp861', 'cp862', + 'cp863', 'cp865', 'cp866', 'cp1125', + 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15', + 'iso8859_2', 'iso8859_4', 'iso8859_5', + 'iso8859_9', 'koi8_r', 'koi8_u', 'latin_1', + 'mac_cyrillic', 'mac_latin2', + + ### These have undefined mappings: + #'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', + #'cp1256', 'cp1257', 'cp1258', + #'cp424', 'cp856', 'cp857', 'cp864', 'cp869', 'cp874', + #'iso8859_3', 'iso8859_6', 'iso8859_7', 'koi8_t', 'kz1048', + #'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish', + + ### These fail the round-trip: + #'cp1006', 'cp875', 'iso8859_8', + + ): + self.assertEqual(str(s, encoding).encode(encoding), s) + + def test_concatenation(self): + self.assertEqual(("abc" "def"), "abcdef") + self.assertEqual(("abc" "def"), "abcdef") + self.assertEqual(("abc" "def"), "abcdef") + self.assertEqual(("abc" "def" "ghi"), "abcdefghi") + self.assertEqual(("abc" "def" "ghi"), "abcdefghi") + + def test_ucs4(self): + x = '\U00100000' + y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") + self.assertEqual(x, y) + + y = br'\U00100000' + x = y.decode("raw-unicode-escape").encode("raw-unicode-escape") + self.assertEqual(x, y) + y = br'\U00010000' + x = y.decode("raw-unicode-escape").encode("raw-unicode-escape") + self.assertEqual(x, y) + + try: + br'\U11111111'.decode("raw-unicode-escape") + except UnicodeDecodeError as e: + self.assertEqual(e.start, 0) + self.assertEqual(e.end, 10) + else: + self.fail("Should have raised UnicodeDecodeError") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'str'> is not <class 'test.test_str.StrSubclass'> + def test_conversion(self): + # Make sure __str__() works properly + class StrWithStr(str): + def __new__(cls, value): + self = str.__new__(cls, "") + self.value = value + return self + def __str__(self): + return self.value + + self.assertTypedEqual(str(WithStr('abc')), 'abc') + self.assertTypedEqual(str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr('abc')), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr(StrSubclass('abc'))), + StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr(OtherStrSubclass('abc'))), + StrSubclass('abc')) + + self.assertTypedEqual(str(StrWithStr('abc')), 'abc') + self.assertTypedEqual(str(StrWithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr('abc')), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr(StrSubclass('abc'))), + StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr(OtherStrSubclass('abc'))), + StrSubclass('abc')) + + self.assertTypedEqual(str(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(str(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(StrSubclass(WithRepr('<abc>')), StrSubclass('<abc>')) + self.assertTypedEqual(StrSubclass(WithRepr(StrSubclass('<abc>'))), + StrSubclass('<abc>')) + self.assertTypedEqual(StrSubclass(WithRepr(OtherStrSubclass('<abc>'))), + StrSubclass('<abc>')) + + def test_unicode_repr(self): + class s1: + def __repr__(self): + return '\\n' + + self.assertEqual(repr(s1()), '\\n') + + def test_printable_repr(self): + # printable + self.assertEqual(repr('\U00010000'), "'%c'" % (0x10000,)) + # nonprintable (private use area) + self.assertEqual(repr('\U00100001'), "'\\U00100001'") + + # This test only affects 32-bit platforms because expandtabs can only take + # an int as the max value, not a 64-bit C long. If expandtabs is changed + # to take a 64-bit long, this test should apply to all platforms. + @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4, + 'only applies to 32-bit platforms') + def test_expandtabs_overflows_gracefully(self): + self.assertRaises(OverflowError, 't\tt\t'.expandtabs, sys.maxsize) + + @support.cpython_only + def test_expandtabs_optimization(self): + s = 'abc' + self.assertIs(s.expandtabs(), s) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_raiseMemError(self): + asciifields = "nnb" + compactfields = asciifields + "nP" + ascii_struct_size = support.calcobjsize(asciifields) + compact_struct_size = support.calcobjsize(compactfields) + + for char in ('a', '\xe9', '\u20ac', '\U0010ffff'): + code = ord(char) + if code < 0x80: + char_size = 1 # sizeof(Py_UCS1) + struct_size = ascii_struct_size + elif code < 0x100: + char_size = 1 # sizeof(Py_UCS1) + struct_size = compact_struct_size + elif code < 0x10000: + char_size = 2 # sizeof(Py_UCS2) + struct_size = compact_struct_size + else: + char_size = 4 # sizeof(Py_UCS4) + struct_size = compact_struct_size + # Note: sys.maxsize is half of the actual max allocation because of + # the signedness of Py_ssize_t. Strings of maxlen-1 should in principle + # be allocatable, given enough memory. + maxlen = ((sys.maxsize - struct_size) // char_size) + alloc = lambda: char * maxlen + with self.subTest( + char=char, + struct_size=struct_size, + char_size=char_size + ): + # self-check + self.assertEqual( + sys.getsizeof(char * 42), + struct_size + (char_size * (42 + 1)) + ) + self.assertRaises(MemoryError, alloc) + self.assertRaises(MemoryError, alloc) + + def test_format_subclass(self): + class S(str): + def __str__(self): + return '__str__ overridden' + s = S('xxx') + self.assertEqual("%s" % s, '__str__ overridden') + self.assertEqual("{}".format(s), '__str__ overridden') + + def test_subclass_add(self): + class S(str): + def __add__(self, o): + return "3" + self.assertEqual(S("4") + S("5"), "3") + class S(str): + def __iadd__(self, o): + return "3" + s = S("1") + s += "4" + self.assertEqual(s, "3") + + def test_getnewargs(self): + text = 'abc' + args = text.__getnewargs__() + self.assertIsNot(args[0], text) + self.assertEqual(args[0], text) + self.assertEqual(len(args), 1) + + def test_compare(self): + # Issue #17615 + N = 10 + ascii = 'a' * N + ascii2 = 'z' * N + latin = '\x80' * N + latin2 = '\xff' * N + bmp = '\u0100' * N + bmp2 = '\uffff' * N + astral = '\U00100000' * N + astral2 = '\U0010ffff' * N + strings = ( + ascii, ascii2, + latin, latin2, + bmp, bmp2, + astral, astral2) + for text1, text2 in itertools.combinations(strings, 2): + equal = (text1 is text2) + self.assertEqual(text1 == text2, equal) + self.assertEqual(text1 != text2, not equal) + + if equal: + self.assertTrue(text1 <= text2) + self.assertTrue(text1 >= text2) + + # text1 is text2: duplicate strings to skip the "str1 == str2" + # optimization in unicode_compare_eq() and really compare + # character per character + copy1 = duplicate_string(text1) + copy2 = duplicate_string(text2) + self.assertIsNot(copy1, copy2) + + self.assertTrue(copy1 == copy2) + self.assertFalse(copy1 != copy2) + + self.assertTrue(copy1 <= copy2) + self.assertTrue(copy2 >= copy2) + + self.assertTrue(ascii < ascii2) + self.assertTrue(ascii < latin) + self.assertTrue(ascii < bmp) + self.assertTrue(ascii < astral) + self.assertFalse(ascii >= ascii2) + self.assertFalse(ascii >= latin) + self.assertFalse(ascii >= bmp) + self.assertFalse(ascii >= astral) + + self.assertFalse(latin < ascii) + self.assertTrue(latin < latin2) + self.assertTrue(latin < bmp) + self.assertTrue(latin < astral) + self.assertTrue(latin >= ascii) + self.assertFalse(latin >= latin2) + self.assertFalse(latin >= bmp) + self.assertFalse(latin >= astral) + + self.assertFalse(bmp < ascii) + self.assertFalse(bmp < latin) + self.assertTrue(bmp < bmp2) + self.assertTrue(bmp < astral) + self.assertTrue(bmp >= ascii) + self.assertTrue(bmp >= latin) + self.assertFalse(bmp >= bmp2) + self.assertFalse(bmp >= astral) + + self.assertFalse(astral < ascii) + self.assertFalse(astral < latin) + self.assertFalse(astral < bmp2) + self.assertTrue(astral < astral2) + self.assertTrue(astral >= ascii) + self.assertTrue(astral >= latin) + self.assertTrue(astral >= bmp2) + self.assertFalse(astral >= astral2) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true + def test_free_after_iterating(self): + support.check_free_after_iterating(self, iter, str) + if not support.Py_GIL_DISABLED: + support.check_free_after_iterating(self, reversed, str) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 22 != 10 : _PythonRunResult(rc=22, out=b'', err=b'') + def test_check_encoding_errors(self): + # bpo-37388: str(bytes) and str.decode() must check encoding and errors + # arguments in dev mode + encodings = ('ascii', 'utf8', 'latin1') + invalid = 'Boom, Shaka Laka, Boom!' + code = textwrap.dedent(f''' + import sys + encodings = {encodings!r} + + for data in (b'', b'short string'): + try: + str(data, encoding={invalid!r}) + except LookupError: + pass + else: + sys.exit(21) + + try: + str(data, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(22) + + for encoding in encodings: + try: + str(data, encoding, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(22) + + for data in ('', 'short string'): + try: + data.encode(encoding={invalid!r}) + except LookupError: + pass + else: + sys.exit(23) + + try: + data.encode(errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(24) + + for encoding in encodings: + try: + data.encode(encoding, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(24) + + sys.exit(10) + ''') + proc = assert_python_failure('-X', 'dev', '-c', code) + self.assertEqual(proc.rc, 10, proc) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "str expected at most 3 arguments, got 4" does not match "expected at most 3 arguments, got 4" + def test_str_invalid_call(self): + # too many args + with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): + str("too", "many", "argu", "ments") + with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): + str(1, "", "", 1) + + # no such kw arg + with self.assertRaisesRegex(TypeError, r"str\(\) got an unexpected keyword argument 'test'"): + str(test=1) + + # 'encoding' must be str + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, 1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, encoding=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not bytes"): + str(b"x", b"ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not bytes"): + str(b"x", encoding=b"ascii") + + # 'errors' must be str + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, 1, 1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not int"): + str(1, errors=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not int"): + str(1, "", errors=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not bytes"): + str(b"x", "ascii", b"strict") + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not bytes"): + str(b"x", "ascii", errors=b"strict") + + # both positional and kwarg + with self.assertRaisesRegex(TypeError, r"argument for str\(\) given by name \('encoding'\) and position \(2\)"): + str(b"x", "utf-8", encoding="ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) takes at most 3 arguments \(4 given\)"): + str(b"x", "utf-8", "ignore", encoding="ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) takes at most 3 arguments \(4 given\)"): + str(b"x", "utf-8", "strict", errors="ignore") + + +class StringModuleTest(unittest.TestCase): + def test_formatter_parser(self): + def parse(format): + return list(_string.formatter_parser(format)) + + formatter = parse("prefix {2!s}xxx{0:^+10.3f}{obj.attr!s} {z[0]!s:10}") + self.assertEqual(formatter, [ + ('prefix ', '2', '', 's'), + ('xxx', '0', '^+10.3f', None), + ('', 'obj.attr', '', 's'), + (' ', 'z[0]', '10', 's'), + ]) + + formatter = parse("prefix {} suffix") + self.assertEqual(formatter, [ + ('prefix ', '', '', None), + (' suffix', None, None, None), + ]) + + formatter = parse("str") + self.assertEqual(formatter, [ + ('str', None, None, None), + ]) + + formatter = parse("") + self.assertEqual(formatter, []) + + formatter = parse("{0}") + self.assertEqual(formatter, [ + ('', '0', '', None), + ]) + + self.assertRaises(TypeError, _string.formatter_parser, 1) + + def test_formatter_field_name_split(self): + def split(name): + items = list(_string.formatter_field_name_split(name)) + items[1] = list(items[1]) + return items + self.assertEqual(split("obj"), ["obj", []]) + self.assertEqual(split("obj.arg"), ["obj", [(True, 'arg')]]) + self.assertEqual(split("obj[key]"), ["obj", [(False, 'key')]]) + self.assertEqual(split("obj.arg[key1][key2]"), [ + "obj", + [(True, 'arg'), + (False, 'key1'), + (False, 'key2'), + ]]) + self.assertRaises(TypeError, _string.formatter_field_name_split, 1) + + def test_str_subclass_attr(self): + + name = StrSubclass("name") + name2 = StrSubclass("name2") + class Bag: + pass + + o = Bag() + with self.assertRaises(AttributeError): + delattr(o, name) + setattr(o, name, 1) + self.assertEqual(o.name, 1) + o.name = 2 + self.assertEqual(list(o.__dict__), [name]) + + with self.assertRaises(AttributeError): + delattr(o, name2) + with self.assertRaises(AttributeError): + del o.name2 + setattr(o, name2, 3) + self.assertEqual(o.name2, 3) + o.name2 = 4 + self.assertEqual(list(o.__dict__), [name, name2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index 08ccebb9edb..375f6aaedd8 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -39,7 +39,21 @@ def _update_variables(self, now): if now[3] < 12: self.ampm='(AM|am)' else: self.ampm='(PM|pm)' - self.jan1 = time.localtime(time.mktime((now[0], 1, 1, 0, 0, 0, 0, 1, 0))) + jan1 = time.struct_time( + ( + now.tm_year, # Year + 1, # Month (January) + 1, # Day (1st) + 0, # Hour (0) + 0, # Minute (0) + 0, # Second (0) + -1, # tm_wday (will be determined) + 1, # tm_yday (day 1 of the year) + -1, # tm_isdst (let the system determine) + ) + ) + # use mktime to get the correct tm_wday and tm_isdst values + self.jan1 = time.localtime(time.mktime(jan1)) try: if now[8]: self.tz = time.tzname[1] @@ -54,16 +68,11 @@ def _update_variables(self, now): self.now = now def setUp(self): - try: - import java - java.util.Locale.setDefault(java.util.Locale.US) - except ImportError: - from locale import setlocale, LC_TIME - saved_locale = setlocale(LC_TIME) - setlocale(LC_TIME, 'C') - self.addCleanup(setlocale, LC_TIME, saved_locale) - - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'a Display implementation returned an error unexpectedly: Error'") + from locale import setlocale, LC_TIME + saved_locale = setlocale(LC_TIME) + setlocale(LC_TIME, 'C') + self.addCleanup(setlocale, LC_TIME, saved_locale) + def test_strftime(self): now = time.time() self._update_variables(now) @@ -185,12 +194,10 @@ class Y1900Tests(unittest.TestCase): a date before 1900 is passed with a format string containing "%y" """ - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_y_before_1900(self): # Issue #13674, #19634 t = (1899, 1, 1, 0, 0, 0, 0, 0, 0) - if (sys.platform == "win32" - or sys.platform.startswith(("aix", "sunos", "solaris"))): + if sys.platform.startswith(("aix", "sunos", "solaris")): with self.assertRaises(ValueError): time.strftime("%y", t) else: diff --git a/Lib/test/test_string/__init__.py b/Lib/test/test_string/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_string/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_string/_support.py b/Lib/test/test_string/_support.py new file mode 100644 index 00000000000..cfead782b7d --- /dev/null +++ b/Lib/test/test_string/_support.py @@ -0,0 +1,67 @@ +import unittest +from string.templatelib import Interpolation + + +class TStringBaseCase: + def assertInterpolationEqual(self, i, exp): + """Test Interpolation equality. + + The *i* argument must be an Interpolation instance. + + The *exp* argument must be a tuple of the form + (value, expression, conversion, format_spec) where the final three + items may be omitted and are assumed to be '', None and '' respectively. + """ + if len(exp) == 4: + actual = (i.value, i.expression, i.conversion, i.format_spec) + self.assertEqual(actual, exp) + elif len(exp) == 3: + self.assertEqual((i.value, i.expression, i.conversion), exp) + self.assertEqual(i.format_spec, "") + elif len(exp) == 2: + self.assertEqual((i.value, i.expression), exp) + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + elif len(exp) == 1: + self.assertEqual((i.value,), exp) + self.assertEqual(i.expression, "") + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + + def assertTStringEqual(self, t, strings, interpolations): + """Test template string literal equality. + + The *strings* argument must be a tuple of strings equal to *t.strings*. + + The *interpolations* argument must be a sequence of tuples which are + compared against *t.interpolations*. Each tuple must match the form + described in the `assertInterpolationEqual` method. + """ + self.assertEqual(t.strings, strings) + self.assertEqual(len(t.interpolations), len(interpolations)) + + for i, exp in zip(t.interpolations, interpolations, strict=True): + self.assertInterpolationEqual(i, exp) + + +def convert(value, conversion): + if conversion == "a": + return ascii(value) + elif conversion == "r": + return repr(value) + elif conversion == "s": + return str(value) + return value + + +def fstring(template): + parts = [] + for item in template: + match item: + case str() as s: + parts.append(s) + case Interpolation(value, _, conversion, format_spec): + value = convert(value, conversion) + value = format(value, format_spec) + parts.append(value) + return "".join(parts) diff --git a/Lib/test/test_string.py b/Lib/test/test_string/test_string.py similarity index 95% rename from Lib/test/test_string.py rename to Lib/test/test_string/test_string.py index 2d2e5a67544..5394fe4e12c 100644 --- a/Lib/test/test_string.py +++ b/Lib/test/test_string/test_string.py @@ -1,6 +1,15 @@ import unittest import string from string import Template +import types +from test.support import cpython_only +from test.support.import_helper import ensure_lazy_imports + + +class LazyImportTest(unittest.TestCase): + @cpython_only + def test_lazy_import(self): + ensure_lazy_imports("base64", {"re", "collections"}) class ModuleTest(unittest.TestCase): @@ -101,6 +110,24 @@ def test_index_lookup(self): with self.assertRaises(KeyError): fmt.format("{0[2]}{0[0]}", {}) + def test_auto_numbering_lookup(self): + fmt = string.Formatter() + namespace = types.SimpleNamespace(foo=types.SimpleNamespace(bar='baz')) + widths = [None, types.SimpleNamespace(qux=4)] + self.assertEqual( + fmt.format("{.foo.bar:{[1].qux}}", namespace, widths), 'baz ') + + def test_auto_numbering_reenterability(self): + class ReenteringFormatter(string.Formatter): + def format_field(self, value, format_spec): + if format_spec.isdigit() and int(format_spec) > 0: + return self.format('{:{}}!', value, int(format_spec) - 1) + else: + return super().format_field(value, format_spec) + fmt = ReenteringFormatter() + x = types.SimpleNamespace(a='X') + self.assertEqual(fmt.format('{.a:{}}', x, 3), 'X!!!') + def test_override_get_value(self): class NamespaceFormatter(string.Formatter): def __init__(self, namespace={}): @@ -475,8 +502,6 @@ class PieDelims(Template): self.assertEqual(s.substitute(dict(who='tim', what='ham')), 'tim likes to eat a bag of ham worth $100') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_is_valid(self): eq = self.assertEqual s = Template('$who likes to eat a bag of ${what} worth $$100') @@ -498,8 +523,6 @@ class BadPattern(Template): s = BadPattern('@bag.foo.who likes to eat a bag of @bag.what') self.assertRaises(ValueError, s.is_valid) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_identifiers(self): eq = self.assertEqual raises = self.assertRaises diff --git a/Lib/test/test_string/test_templatelib.py b/Lib/test/test_string/test_templatelib.py new file mode 100644 index 00000000000..1c86717155f --- /dev/null +++ b/Lib/test/test_string/test_templatelib.py @@ -0,0 +1,193 @@ +import pickle +import unittest +from collections.abc import Iterator, Iterable +from string.templatelib import Template, Interpolation, convert + +from test.test_string._support import TStringBaseCase, fstring + + +class TestTemplate(unittest.TestCase, TStringBaseCase): + + def test_common(self): + self.assertEqual(type(t'').__name__, 'Template') + self.assertEqual(type(t'').__qualname__, 'Template') + self.assertEqual(type(t'').__module__, 'string.templatelib') + + a = 'a' + i = t'{a}'.interpolations[0] + self.assertEqual(type(i).__name__, 'Interpolation') + self.assertEqual(type(i).__qualname__, 'Interpolation') + self.assertEqual(type(i).__module__, 'string.templatelib') + + def test_final_types(self): + with self.assertRaisesRegex(TypeError, 'is not an acceptable base type'): + class Sub(Template): ... + + with self.assertRaisesRegex(TypeError, 'is not an acceptable base type'): + class Sub(Interpolation): ... + + def test_basic_creation(self): + # Simple t-string creation + t = t'Hello, world' + self.assertIsInstance(t, Template) + self.assertTStringEqual(t, ('Hello, world',), ()) + self.assertEqual(fstring(t), 'Hello, world') + + # Empty t-string + t = t'' + self.assertTStringEqual(t, ('',), ()) + self.assertEqual(fstring(t), '') + + # Multi-line t-string + t = t"""Hello, +world""" + self.assertEqual(t.strings, ('Hello,\nworld',)) + self.assertEqual(len(t.interpolations), 0) + self.assertEqual(fstring(t), 'Hello,\nworld') + + def test_interpolation_creation(self): + i = Interpolation('Maria', 'name', 'a', 'fmt') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a', 'fmt')) + + i = Interpolation('Maria', 'name', 'a') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a')) + + i = Interpolation('Maria', 'name') + self.assertInterpolationEqual(i, ('Maria', 'name')) + + i = Interpolation('Maria') + self.assertInterpolationEqual(i, ('Maria',)) + + def test_creation_interleaving(self): + # Should add strings on either side + t = Template(Interpolation('Maria', 'name', None, '')) + self.assertTStringEqual(t, ('', ''), [('Maria', 'name')]) + self.assertEqual(fstring(t), 'Maria') + + # Should prepend empty string + t = Template(Interpolation('Maria', 'name', None, ''), ' is my name') + self.assertTStringEqual(t, ('', ' is my name'), [('Maria', 'name')]) + self.assertEqual(fstring(t), 'Maria is my name') + + # Should append empty string + t = Template('Hello, ', Interpolation('Maria', 'name', None, '')) + self.assertTStringEqual(t, ('Hello, ', ''), [('Maria', 'name')]) + self.assertEqual(fstring(t), 'Hello, Maria') + + # Should concatenate strings + t = Template('Hello', ', ', Interpolation('Maria', 'name', None, ''), + '!') + self.assertTStringEqual(t, ('Hello, ', '!'), [('Maria', 'name')]) + self.assertEqual(fstring(t), 'Hello, Maria!') + + # Should add strings on either side and in between + t = Template(Interpolation('Maria', 'name', None, ''), + Interpolation('Python', 'language', None, '')) + self.assertTStringEqual( + t, ('', '', ''), [('Maria', 'name'), ('Python', 'language')] + ) + self.assertEqual(fstring(t), 'MariaPython') + + def test_template_values(self): + t = t'Hello, world' + self.assertEqual(t.values, ()) + + name = "Lys" + t = t'Hello, {name}' + self.assertEqual(t.values, ("Lys",)) + + country = "GR" + age = 0 + t = t'Hello, {name}, {age} from {country}' + self.assertEqual(t.values, ("Lys", 0, "GR")) + + def test_pickle_template(self): + user = 'test' + for template in ( + t'', + t"No values", + t'With inter {user}', + t'With ! {user!r}', + t'With format {1 / 0.3:.2f}', + Template(), + Template('a'), + Template(Interpolation('Nikita', 'name', None, '')), + Template('a', Interpolation('Nikita', 'name', 'r', '')), + ): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto, template=template): + pickled = pickle.dumps(template, protocol=proto) + unpickled = pickle.loads(pickled) + + self.assertEqual(unpickled.values, template.values) + self.assertEqual(fstring(unpickled), fstring(template)) + + def test_pickle_interpolation(self): + for interpolation in ( + Interpolation('Nikita', 'name', None, ''), + Interpolation('Nikita', 'name', 'r', ''), + Interpolation(1/3, 'x', None, '.2f'), + ): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto, interpolation=interpolation): + pickled = pickle.dumps(interpolation, protocol=proto) + unpickled = pickle.loads(pickled) + + self.assertEqual(unpickled.value, interpolation.value) + self.assertEqual(unpickled.expression, interpolation.expression) + self.assertEqual(unpickled.conversion, interpolation.conversion) + self.assertEqual(unpickled.format_spec, interpolation.format_spec) + + +class TemplateIterTests(unittest.TestCase): + def test_abc(self): + self.assertIsInstance(iter(t''), Iterable) + self.assertIsInstance(iter(t''), Iterator) + + def test_final(self): + TemplateIter = type(iter(t'')) + with self.assertRaisesRegex(TypeError, 'is not an acceptable base type'): + class Sub(TemplateIter): ... + + def test_iter(self): + x = 1 + res = list(iter(t'abc {x} yz')) + + self.assertEqual(res[0], 'abc ') + self.assertIsInstance(res[1], Interpolation) + self.assertEqual(res[1].value, 1) + self.assertEqual(res[1].expression, 'x') + self.assertEqual(res[1].conversion, None) + self.assertEqual(res[1].format_spec, '') + self.assertEqual(res[2], ' yz') + + def test_exhausted(self): + # See https://github.com/python/cpython/issues/134119. + template_iter = iter(t"{1}") + self.assertIsInstance(next(template_iter), Interpolation) + self.assertRaises(StopIteration, next, template_iter) + self.assertRaises(StopIteration, next, template_iter) + + +class TestFunctions(unittest.TestCase): + def test_convert(self): + from fractions import Fraction + + for obj in ('Café', None, 3.14, Fraction(1, 2)): + with self.subTest(f'{obj=}'): + self.assertEqual(convert(obj, None), obj) + self.assertEqual(convert(obj, 's'), str(obj)) + self.assertEqual(convert(obj, 'r'), repr(obj)) + self.assertEqual(convert(obj, 'a'), ascii(obj)) + + # Invalid conversion specifier + with self.assertRaises(ValueError): + convert(obj, 'z') + with self.assertRaises(ValueError): + convert(obj, 1) + with self.assertRaises(ValueError): + convert(obj, object()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index 537c8fc5c80..14327d288a4 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -105,32 +105,100 @@ def test_eval_str_incomplete(self): self.assertRaises(SyntaxError, eval, r""" '\U000000' """) self.assertRaises(SyntaxError, eval, r""" '\U0000000' """) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_eval_str_invalid_escape(self): for b in range(1, 128): if b in b"""\n\r"'01234567NU\\abfnrtuvx""": continue - with self.assertWarns(DeprecationWarning): + with self.assertWarns(SyntaxWarning): self.assertEqual(eval(r"'\%c'" % b), '\\' + chr(b)) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always', category=DeprecationWarning) + warnings.simplefilter('always', category=SyntaxWarning) eval("'''\n\\z'''") self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(w[0].filename, '<string>') - self.assertEqual(w[0].lineno, 1) + self.assertEqual(w[0].lineno, 2) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('error', category=DeprecationWarning) + warnings.simplefilter('error', category=SyntaxWarning) with self.assertRaises(SyntaxError) as cm: eval("'''\n\\z'''") exc = cm.exception self.assertEqual(w, []) + self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. ' + r'Did you mean "\\z"? A raw string is also an option.') + self.assertEqual(exc.filename, '<string>') + self.assertEqual(exc.lineno, 2) + self.assertEqual(exc.offset, 1) + + # Check that the warning is raised only once if there are syntax errors + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("'\\e' $") + exc = cm.exception + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, SyntaxWarning) + self.assertRegex(str(w[0].message), 'invalid escape sequence') + self.assertEqual(w[0].filename, '<string>') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_eval_str_invalid_octal_escape(self): + for i in range(0o400, 0o1000): + with self.assertWarns(SyntaxWarning): + self.assertEqual(eval(r"'\%o'" % i), chr(i)) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + eval("'''\n\\407'''") + self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), + r'"\407" is an invalid octal escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\407"? A raw string is also an option.') + self.assertEqual(w[0].filename, '<string>') + self.assertEqual(w[0].lineno, 2) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('error', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("'''\n\\407'''") + exc = cm.exception + self.assertEqual(w, []) + self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. ' + r'Did you mean "\\407"? A raw string is also an option.') self.assertEqual(exc.filename, '<string>') - self.assertEqual(exc.lineno, 1) + self.assertEqual(exc.lineno, 2) self.assertEqual(exc.offset, 1) + def test_invalid_escape_locations_with_offset(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + eval("\"'''''''''''''''''''''invalid\\ Escape\"") + self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), + r'"\ " is an invalid escape sequence. Such sequences ' + r'will not work in the future. Did you mean "\\ "? ' + r'A raw string is also an option.') + self.assertEqual(w[0].filename, '<string>') + self.assertEqual(w[0].lineno, 1) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + eval("\"''Incorrect \\ logic?\"") + self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), + r'"\ " is an invalid escape sequence. Such sequences ' + r'will not work in the future. Did you mean "\\ "? ' + r'A raw string is also an option.') + self.assertEqual(w[0].filename, '<string>') + self.assertEqual(w[0].lineno, 1) + def test_eval_str_raw(self): self.assertEqual(eval(""" r'x' """), 'x') self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01') @@ -157,30 +225,61 @@ def test_eval_bytes_incomplete(self): self.assertRaises(SyntaxError, eval, r""" b'\x' """) self.assertRaises(SyntaxError, eval, r""" b'\x0' """) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_eval_bytes_invalid_escape(self): for b in range(1, 128): if b in b"""\n\r"'01234567\\abfnrtvx""": continue - with self.assertWarns(DeprecationWarning): + with self.assertWarns(SyntaxWarning): self.assertEqual(eval(r"b'\%c'" % b), b'\\' + bytes([b])) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always', category=DeprecationWarning) + warnings.simplefilter('always', category=SyntaxWarning) eval("b'''\n\\z'''") self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), r'"\z" is an invalid escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(w[0].filename, '<string>') - self.assertEqual(w[0].lineno, 1) + self.assertEqual(w[0].lineno, 2) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('error', category=DeprecationWarning) + warnings.simplefilter('error', category=SyntaxWarning) with self.assertRaises(SyntaxError) as cm: eval("b'''\n\\z'''") exc = cm.exception self.assertEqual(w, []) + self.assertEqual(exc.msg, r'"\z" is an invalid escape sequence. ' + r'Did you mean "\\z"? A raw string is also an option.') self.assertEqual(exc.filename, '<string>') - self.assertEqual(exc.lineno, 1) + self.assertEqual(exc.lineno, 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_eval_bytes_invalid_octal_escape(self): + for i in range(0o400, 0o1000): + with self.assertWarns(SyntaxWarning): + self.assertEqual(eval(r"b'\%o'" % i), bytes([i & 0o377])) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + eval("b'''\n\\407'''") + self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), r'"\407" is an invalid octal escape sequence. ' + r'Such sequences will not work in the future. ' + r'Did you mean "\\407"? A raw string is also an option.') + self.assertEqual(w[0].filename, '<string>') + self.assertEqual(w[0].lineno, 2) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('error', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("b'''\n\\407'''") + exc = cm.exception + self.assertEqual(w, []) + self.assertEqual(exc.msg, r'"\407" is an invalid octal escape sequence. ' + r'Did you mean "\\407"? A raw string is also an option.') + self.assertEqual(exc.filename, '<string>') + self.assertEqual(exc.lineno, 2) def test_eval_bytes_raw(self): self.assertEqual(eval(""" br'x' """), b'x') @@ -217,6 +316,13 @@ def test_eval_str_u(self): self.assertRaises(SyntaxError, eval, """ bu'' """) self.assertRaises(SyntaxError, eval, """ ub'' """) + def test_uppercase_prefixes(self): + self.assertEqual(eval(""" B'x' """), b'x') + self.assertEqual(eval(r""" R'\x01' """), r'\x01') + self.assertEqual(eval(r""" BR'\x01' """), br'\x01') + self.assertEqual(eval(""" F'{1+1}' """), f'{1+1}') + self.assertEqual(eval(r""" U'\U0001d120' """), u'\U0001d120') + def check_encoding(self, encoding, extra=""): modname = "xx_" + encoding.replace("-", "_") fn = os.path.join(self.tmpdir, modname + ".py") diff --git a/Lib/test/test_stringprep.py b/Lib/test/test_stringprep.py index 118f3f08678..d4b4a13d0de 100644 --- a/Lib/test/test_stringprep.py +++ b/Lib/test/test_stringprep.py @@ -6,8 +6,6 @@ from stringprep import * class StringprepTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test(self): self.assertTrue(in_table_a1("\u0221")) self.assertFalse(in_table_a1("\u0222")) diff --git a/Lib/test/test_strtod.py b/Lib/test/test_strtod.py index b8b7a9d5026..03c8afa51ef 100644 --- a/Lib/test/test_strtod.py +++ b/Lib/test/test_strtod.py @@ -19,7 +19,7 @@ (?P<int>\d*) # having a (possibly empty) integer part (?:\.(?P<frac>\d*))? # followed by an optional fractional part (?:E(?P<exp>[-+]?\d+))? # and an optional exponent - \Z + \z """, re.VERBOSE | re.IGNORECASE).match # Pure Python version of correctly rounded string->float conversion. @@ -146,7 +146,7 @@ def test_short_halfway_cases(self): digits *= 5 exponent -= 1 - @unittest.skip("TODO: RUSTPYTHON, fails on debug mode, flaky in release mode") + @unittest.skip("TODO: RUSTPYTHON; fails on debug mode, flaky in release mode") def test_halfway_cases(self): # test halfway cases for the round-half-to-even rule for i in range(100 * TEST_SIZE): @@ -173,8 +173,7 @@ def test_halfway_cases(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_boundaries(self): # boundaries expressed as triples (n, e, u), where # n*10**e is an approximation to the boundary value and @@ -195,8 +194,7 @@ def test_boundaries(self): u *= 10 e -= 1 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_underflow_boundary(self): # test values close to 2**-1075, the underflow boundary; similar # to boundary_tests, except that the random error doesn't scale @@ -208,8 +206,7 @@ def test_underflow_boundary(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bigcomp(self): for ndigs in 5, 10, 14, 15, 16, 17, 18, 19, 20, 40, 41, 50: dig10 = 10**ndigs @@ -219,7 +216,6 @@ def test_bigcomp(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON, Incorrectly rounded str->float conversion for -07e-321 @unittest.skip("TODO: RUSTPYTHON; flaky test") def test_parsing(self): # make '0' more likely to be chosen than other digits @@ -288,8 +284,7 @@ def negative_exp(n): self.assertEqual(float(negative_exp(20000)), 1.0) self.assertEqual(float(negative_exp(30000)), 1.0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_particular(self): # inputs that produced crashes or incorrectly rounded results with # previous versions of dtoa.c, for various reasons diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 6cb7b610e5c..733006bb509 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -1,4 +1,5 @@ from collections import abc +from itertools import combinations import array import gc import math @@ -11,12 +12,16 @@ from test import support from test.support import import_helper from test.support.script_helper import assert_python_ok +from test.support.testcase import ComplexesAreIdenticalMixin ISBIGENDIAN = sys.byteorder == "big" integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N' byteorders = '', '@', '=', '<', '>', '!' +INF = float('inf') +NAN = float('nan') + def iter_integer_formats(byteorders=byteorders): for code in integer_codes: for byteorder in byteorders: @@ -33,7 +38,7 @@ def bigendian_to_native(value): else: return string_reverse(value) -class StructTest(unittest.TestCase): +class StructTest(ComplexesAreIdenticalMixin, unittest.TestCase): def test_isbigendian(self): self.assertEqual((struct.pack('=i', 1)[0] == 0), ISBIGENDIAN) @@ -96,6 +101,13 @@ def test_new_features(self): ('10s', b'helloworld', b'helloworld', b'helloworld', 0), ('11s', b'helloworld', b'helloworld\0', b'helloworld\0', 1), ('20s', b'helloworld', b'helloworld'+10*b'\0', b'helloworld'+10*b'\0', 1), + ('0p', b'helloworld', b'', b'', 1), + ('1p', b'helloworld', b'\x00', b'\x00', 1), + ('2p', b'helloworld', b'\x01h', b'\x01h', 1), + ('10p', b'helloworld', b'\x09helloworl', b'\x09helloworl', 1), + ('11p', b'helloworld', b'\x0Ahelloworld', b'\x0Ahelloworld', 0), + ('12p', b'helloworld', b'\x0Ahelloworld\0', b'\x0Ahelloworld\0', 1), + ('20p', b'helloworld', b'\x0Ahelloworld'+9*b'\0', b'\x0Ahelloworld'+9*b'\0', 1), ('b', 7, b'\7', b'\7', 0), ('b', -7, b'\371', b'\371', 0), ('B', 7, b'\7', b'\7', 0), @@ -339,6 +351,7 @@ def assertStructError(func, *args, **kwargs): def test_p_code(self): # Test p ("Pascal string") code. for code, input, expected, expectedback in [ + ('0p', b'abc', b'', b''), ('p', b'abc', b'\x00', b''), ('1p', b'abc', b'\x00', b''), ('2p', b'abc', b'\x01a', b'a'), @@ -352,8 +365,7 @@ def test_p_code(self): (got,) = struct.unpack(code, got) self.assertEqual(got, expectedback) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_705836(self): # SF bug 705836. "<f" and ">f" had a severe rounding bug, where a carry # from the low-order discarded bits could propagate into the exponent @@ -422,56 +434,65 @@ def test_unpack_from(self): self.assertEqual(s.unpack_from(buffer=test_string, offset=2), (b'cd01',)) - def test_pack_into(self): + def _test_pack_into(self, pack_into): test_string = b'Reykjavik rocks, eow!' - writable_buf = array.array('b', b' '*100) - fmt = '21s' - s = struct.Struct(fmt) + writable_buf = memoryview(array.array('b', b' '*100)) # Test without offset - s.pack_into(writable_buf, 0, test_string) + pack_into(writable_buf, 0, test_string) from_buf = writable_buf.tobytes()[:len(test_string)] self.assertEqual(from_buf, test_string) # Test with offset. - s.pack_into(writable_buf, 10, test_string) + pack_into(writable_buf, 10, test_string) from_buf = writable_buf.tobytes()[:len(test_string)+10] self.assertEqual(from_buf, test_string[:10] + test_string) + # Test with negative offset. + pack_into(writable_buf, -30, test_string) + from_buf = writable_buf.tobytes()[-30:-30+len(test_string)] + self.assertEqual(from_buf, test_string) + # Go beyond boundaries. small_buf = array.array('b', b' '*10) - self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 0, - test_string) - self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 2, - test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(small_buf, 0, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, 90, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, -10, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, 150, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, -150, test_string) + + # Test invalid buffer. + self.assertRaises(TypeError, pack_into, b' '*100, 0, test_string) + self.assertRaises(TypeError, pack_into, ' '*100, 0, test_string) + self.assertRaises(TypeError, pack_into, [0]*100, 0, test_string) + self.assertRaises(TypeError, pack_into, None, 0, test_string) + self.assertRaises(TypeError, pack_into, writable_buf[::2], 0, test_string) + self.assertRaises(TypeError, pack_into, writable_buf[::-1], 0, test_string) + + # Test bogus offset (issue bpo-3694) + with self.assertRaises(TypeError): + pack_into(writable_buf, None, test_string) + with self.assertRaises(TypeError): + pack_into(writable_buf, 0.0, test_string) + with self.assertRaises((IndexError, OverflowError)): + pack_into(writable_buf, 2**1000, test_string) + with self.assertRaises((IndexError, OverflowError)): + pack_into(writable_buf, -2**1000, test_string) - # Test bogus offset (issue 3694) - sb = small_buf - self.assertRaises((TypeError, struct.error), struct.pack_into, b'', sb, - None) + @unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object + def test_pack_into(self): + s = struct.Struct('21s') + self._test_pack_into(s.pack_into) + @unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object def test_pack_into_fn(self): - test_string = b'Reykjavik rocks, eow!' - writable_buf = array.array('b', b' '*100) - fmt = '21s' - pack_into = lambda *args: struct.pack_into(fmt, *args) - - # Test without offset. - pack_into(writable_buf, 0, test_string) - from_buf = writable_buf.tobytes()[:len(test_string)] - self.assertEqual(from_buf, test_string) - - # Test with offset. - pack_into(writable_buf, 10, test_string) - from_buf = writable_buf.tobytes()[:len(test_string)+10] - self.assertEqual(from_buf, test_string[:10] + test_string) - - # Go beyond boundaries. - small_buf = array.array('b', b' '*10) - self.assertRaises((ValueError, struct.error), pack_into, small_buf, 0, - test_string) - self.assertRaises((ValueError, struct.error), pack_into, small_buf, 2, - test_string) + pack_into = lambda *args: struct.pack_into('21s', *args) + self._test_pack_into(pack_into) def test_unpack_with_buffer(self): # SF bug 1563759: struct.unpack doesn't support buffer protocol objects @@ -523,6 +544,9 @@ def __bool__(self): for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']: self.assertTrue(struct.unpack('>?', c)[0]) + self.assertTrue(struct.unpack('<?', c)[0]) + self.assertTrue(struct.unpack('=?', c)[0]) + self.assertTrue(struct.unpack('@?', c)[0]) def test_count_overflow(self): hugecount = '{}b'.format(sys.maxsize+1) @@ -582,6 +606,7 @@ def test__sizeof__(self): self.check_sizeof('187s', 1) self.check_sizeof('20p', 1) self.check_sizeof('0s', 1) + self.check_sizeof('0p', 1) self.check_sizeof('0c', 0) def test_boundary_error_message(self): @@ -658,8 +683,7 @@ def test_format_attr(self): s2 = struct.Struct(s.format.encode()) self.assertEqual(s2.format, s.format) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_struct_cleans_up_at_runtime_shutdown(self): code = """if 1: import struct @@ -675,7 +699,7 @@ def __del__(self): rc, stdout, stderr = assert_python_ok("-c", code) self.assertEqual(rc, 0) self.assertEqual(stdout.rstrip(), b"") - self.assertIn(b"Exception ignored in:", stderr) + self.assertIn(b"Exception ignored while calling deallocator", stderr) self.assertIn(b"C.__del__", stderr) def test__struct_reference_cycle_cleaned_up(self): @@ -705,8 +729,6 @@ def test__struct_types_immutable(self): cls.x = 1 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue35714(self): # Embedded null characters should not be allowed in format strings. for s in '\0', '2\0i', b'\0': @@ -714,6 +736,112 @@ def test_issue35714(self): 'embedded null character'): struct.calcsize(s) + @support.cpython_only + def test_issue98248(self): + def test_error_msg(prefix, int_type, is_unsigned): + fmt_str = prefix + int_type + size = struct.calcsize(fmt_str) + if is_unsigned: + max_ = 2 ** (size * 8) - 1 + min_ = 0 + else: + max_ = 2 ** (size * 8 - 1) - 1 + min_ = -2 ** (size * 8 - 1) + error_msg = f"'{int_type}' format requires {min_} <= number <= {max_}" + for number in [int(-1e50), min_ - 1, max_ + 1, int(1e50)]: + with self.subTest(format_str=fmt_str, number=number): + with self.assertRaisesRegex(struct.error, error_msg): + struct.pack(fmt_str, number) + error_msg = "required argument is not an integer" + not_number = "" + with self.subTest(format_str=fmt_str, number=not_number): + with self.assertRaisesRegex(struct.error, error_msg): + struct.pack(fmt_str, not_number) + + for prefix in '@=<>': + for int_type in 'BHILQ': + test_error_msg(prefix, int_type, True) + for int_type in 'bhilq': + test_error_msg(prefix, int_type, False) + + int_type = 'N' + test_error_msg('@', int_type, True) + + int_type = 'n' + test_error_msg('@', int_type, False) + + @support.cpython_only + def test_issue98248_error_propagation(self): + class Div0: + def __index__(self): + 1 / 0 + + def test_error_propagation(fmt_str): + with self.subTest(format_str=fmt_str, exception="ZeroDivisionError"): + with self.assertRaises(ZeroDivisionError): + struct.pack(fmt_str, Div0()) + + for prefix in '@=<>': + for int_type in 'BHILQbhilq': + test_error_propagation(prefix + int_type) + + test_error_propagation('N') + test_error_propagation('n') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_struct_subclass_instantiation(self): + # Regression test for https://github.com/python/cpython/issues/112358 + class MyStruct(struct.Struct): + def __init__(self): + super().__init__('>h') + + my_struct = MyStruct() + self.assertEqual(my_struct.pack(12345), b'\x30\x39') + + def test_repr(self): + s = struct.Struct('=i2H') + self.assertEqual(repr(s), f'Struct({s.format!r})') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_c_complex_round_trip(self): + values = [complex(*_) for _ in combinations([1, -1, 0.0, -0.0, 2, + -3, INF, -INF, NAN], 2)] + for z in values: + for f in ['F', 'D', '>F', '>D', '<F', '<D']: + with self.subTest(z=z, format=f): + round_trip = struct.unpack(f, struct.pack(f, z))[0] + self.assertComplexesAreIdentical(z, round_trip) + + @unittest.skipIf( + support.is_android or support.is_apple_mobile, + "Subinterpreters are not supported on Android and iOS" + ) + def test_endian_table_init_subinterpreters(self): + # Verify that the _struct extension module can be initialized + # concurrently in subinterpreters (gh-140260). + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + raise unittest.SkipTest("InterpreterPoolExecutor not available") + + code = "import struct" + with InterpreterPoolExecutor(max_workers=5) as executor: + results = executor.map(exec, [code] * 5) + self.assertListEqual(list(results), [None] * 5) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at least 1 arguments, got 0 + def test_operations_on_half_initialized_Struct(self): + S = struct.Struct.__new__(struct.Struct) + + spam = array.array('b', b' ') + self.assertRaises(RuntimeError, S.iter_unpack, spam) + self.assertRaises(RuntimeError, S.pack, 1) + self.assertRaises(RuntimeError, S.pack_into, spam, 1) + self.assertRaises(RuntimeError, S.unpack, spam) + self.assertRaises(RuntimeError, S.unpack_from, spam) + self.assertRaises(RuntimeError, getattr, S, 'format') + self.assertEqual(S.size, -1) + class UnpackIteratorTest(unittest.TestCase): """ @@ -741,8 +869,6 @@ def _check_iterator(it): with self.assertRaises(struct.error): s.iter_unpack(b"12") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_uninstantiable(self): iter_unpack_type = type(struct.Struct(">ibcp").iter_unpack(b"")) self.assertRaises(TypeError, iter_unpack_type) @@ -790,6 +916,7 @@ def test_module_func(self): self.assertRaises(StopIteration, next, it) def test_half_float(self): + _testcapi = import_helper.import_module('_testcapi') # Little-endian examples from: # http://en.wikipedia.org/wiki/Half_precision_floating-point_format format_bits_float__cleanRoundtrip_list = [ @@ -834,10 +961,17 @@ def test_half_float(self): # Check that packing produces a bit pattern representing a quiet NaN: # all exponent bits and the msb of the fraction should all be 1. + if _testcapi.nan_msb_is_signaling: + # HP PA RISC and some MIPS CPUs use 0 for quiet, see: + # https://en.wikipedia.org/wiki/NaN#Encoding + expected = 0x7c + else: + expected = 0x7e + packed = struct.pack('<e', math.nan) - self.assertEqual(packed[1] & 0x7e, 0x7e) + self.assertEqual(packed[1] & 0x7e, expected) packed = struct.pack('<e', -math.nan) - self.assertEqual(packed[1] & 0x7e, 0x7e) + self.assertEqual(packed[1] & 0x7e, expected) # Checks for round-to-even behavior format_bits_float__rounding_list = [ diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index 41095f63ad6..a9fe193028e 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -75,8 +75,6 @@ def test_cmp(self): self.assertTrue(t1 >= t2) self.assertTrue(not (t1 != t2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fields(self): t = time.gmtime() self.assertEqual(len(t), t.n_sequence_fields) @@ -129,8 +127,6 @@ def test_match_args(self): 'tm_sec', 'tm_wday', 'tm_yday', 'tm_isdst') self.assertEqual(time.struct_time.__match_args__, expected_args) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_match_args_with_unnamed_fields(self): expected_args = ('st_mode', 'st_ino', 'st_dev', 'st_nlink', 'st_uid', 'st_gid', 'st_size') diff --git a/Lib/test/test_subclassinit.py b/Lib/test/test_subclassinit.py index c007476e004..0d32aa509bd 100644 --- a/Lib/test/test_subclassinit.py +++ b/Lib/test/test_subclassinit.py @@ -134,30 +134,28 @@ class Descriptor: def __set_name__(self, owner, name): 1/0 - with self.assertRaises(RuntimeError) as cm: + with self.assertRaises(ZeroDivisionError) as cm: class NotGoingToWork: attr = Descriptor() - exc = cm.exception - self.assertRegex(str(exc), r'\bNotGoingToWork\b') - self.assertRegex(str(exc), r'\battr\b') - self.assertRegex(str(exc), r'\bDescriptor\b') - self.assertIsInstance(exc.__cause__, ZeroDivisionError) + notes = cm.exception.__notes__ + self.assertRegex(str(notes), r'\bNotGoingToWork\b') + self.assertRegex(str(notes), r'\battr\b') + self.assertRegex(str(notes), r'\bDescriptor\b') def test_set_name_wrong(self): class Descriptor: def __set_name__(self): pass - with self.assertRaises(RuntimeError) as cm: + with self.assertRaises(TypeError) as cm: class NotGoingToWork: attr = Descriptor() - exc = cm.exception - self.assertRegex(str(exc), r'\bNotGoingToWork\b') - self.assertRegex(str(exc), r'\battr\b') - self.assertRegex(str(exc), r'\bDescriptor\b') - self.assertIsInstance(exc.__cause__, TypeError) + notes = cm.exception.__notes__ + self.assertRegex(str(notes), r'\bNotGoingToWork\b') + self.assertRegex(str(notes), r'\battr\b') + self.assertRegex(str(notes), r'\bDescriptor\b') def test_set_name_lookup(self): resolved = [] @@ -232,7 +230,7 @@ def __init__(self, name, bases, namespace, otherarg): super().__init__(name, bases, namespace) with self.assertRaises(TypeError): - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass2(metaclass=MyMeta, otherarg=1): pass class MyMeta(type): @@ -243,10 +241,10 @@ def __init__(self, name, bases, namespace, otherarg): super().__init__(name, bases, namespace) self.otherarg = otherarg - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass3(metaclass=MyMeta, otherarg=1): pass - self.assertEqual(MyClass.otherarg, 1) + self.assertEqual(MyClass3.otherarg, 1) def test_errors_changed_pep487(self): # These tests failed before Python 3.6, PEP 487 @@ -265,10 +263,10 @@ def __new__(cls, name, bases, namespace, otherarg): self.otherarg = otherarg return self - class MyClass(metaclass=MyMeta, otherarg=1): + class MyClass2(metaclass=MyMeta, otherarg=1): pass - self.assertEqual(MyClass.otherarg, 1) + self.assertEqual(MyClass2.otherarg, 1) def test_type(self): t = type('NewClass', (object,), {}) @@ -281,4 +279,3 @@ def test_type(self): if __name__ == "__main__": unittest.main() - diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 0a72ffade8a..824c636dfd1 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1,9 +1,12 @@ import unittest from unittest import mock from test import support +from test.support import check_sanitizer from test.support import import_helper from test.support import os_helper +from test.support import strace_helper from test.support import warnings_helper +from test.support.script_helper import assert_python_ok import subprocess import sys import signal @@ -23,7 +26,6 @@ import gc import textwrap import json -import pathlib from test.support.os_helper import FakePath try: @@ -39,6 +41,10 @@ import grp except ImportError: grp = None +try: + import resource +except ImportError: + resource = None try: import fcntl @@ -156,6 +162,20 @@ def test_call_timeout(self): [sys.executable, "-c", "while True: pass"], timeout=0.1) + def test_timeout_exception(self): + try: + subprocess.run([sys.executable, '-c', 'import time;time.sleep(9)'], timeout = -1) + except subprocess.TimeoutExpired as e: + self.assertIn("-1 seconds", str(e)) + else: + self.fail("Expected TimeoutExpired exception not raised") + try: + subprocess.run([sys.executable, '-c', 'import time;time.sleep(9)'], timeout = 0) + except subprocess.TimeoutExpired as e: + self.assertIn("0 seconds", str(e)) + else: + self.fail("Expected TimeoutExpired exception not raised") + def test_check_call_zero(self): # check_call() function with zero return code rc = subprocess.check_call(ZERO_RETURN_CMD) @@ -272,15 +292,8 @@ def test_check_output_timeout(self): with self.assertRaises(subprocess.TimeoutExpired) as c: output = subprocess.check_output( [sys.executable, "-c", - "import sys, time\n" - "sys.stdout.write('BDFL')\n" - "sys.stdout.flush()\n" - "time.sleep(3600)"], - # Some heavily loaded buildbots (sparc Debian 3.x) require - # this much time to start and print. - timeout=3) - self.fail("Expected TimeoutExpired.") - self.assertEqual(c.exception.output, b'BDFL') + "import time; time.sleep(3600)"], + timeout=0.1) def test_call_kwargs(self): # call() function with keyword args @@ -717,8 +730,9 @@ def test_pipesizes(self): os.close(test_pipe_r) os.close(test_pipe_w) pipesize = pipesize_default // 2 - if pipesize < 512: # the POSIX minimum - raise unittest.SkitTest( + pagesize_default = support.get_pagesize() + if pipesize < pagesize_default: # the POSIX minimum + raise unittest.SkipTest( 'default pipesize too small to perform test.') p = subprocess.Popen( [sys.executable, "-c", @@ -745,31 +759,36 @@ def test_pipesizes(self): @unittest.skipUnless(fcntl and hasattr(fcntl, 'F_GETPIPE_SZ'), 'fcntl.F_GETPIPE_SZ required for test.') def test_pipesize_default(self): - p = subprocess.Popen( + proc = subprocess.Popen( [sys.executable, "-c", 'import sys; sys.stdin.read(); sys.stdout.write("out"); ' 'sys.stderr.write("error!")'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, pipesize=-1) - try: - fp_r, fp_w = os.pipe() + + with proc: try: - default_pipesize = fcntl.fcntl(fp_w, fcntl.F_GETPIPE_SZ) - for fifo in [p.stdin, p.stdout, p.stderr]: - self.assertEqual( - fcntl.fcntl(fifo.fileno(), fcntl.F_GETPIPE_SZ), - default_pipesize) + fp_r, fp_w = os.pipe() + try: + default_read_pipesize = fcntl.fcntl(fp_r, fcntl.F_GETPIPE_SZ) + default_write_pipesize = fcntl.fcntl(fp_w, fcntl.F_GETPIPE_SZ) + finally: + os.close(fp_r) + os.close(fp_w) + + self.assertEqual( + fcntl.fcntl(proc.stdin.fileno(), fcntl.F_GETPIPE_SZ), + default_read_pipesize) + self.assertEqual( + fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETPIPE_SZ), + default_write_pipesize) + self.assertEqual( + fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETPIPE_SZ), + default_write_pipesize) + # On other platforms we cannot test the pipe size (yet). But above + # code using pipesize=-1 should not crash. finally: - os.close(fp_r) - os.close(fp_w) - # On other platforms we cannot test the pipe size (yet). But above - # code using pipesize=-1 should not crash. - p.stdin.close() - p.stdout.close() - p.stderr.close() - finally: - p.kill() - p.wait() + proc.kill() def test_env(self): newenv = os.environ.copy() @@ -782,6 +801,19 @@ def test_env(self): stdout, stderr = p.communicate() self.assertEqual(stdout, b"orange") + @unittest.skipUnless(sys.platform == "win32", "Windows only issue") + def test_win32_duplicate_envs(self): + newenv = os.environ.copy() + newenv["fRUit"] = "cherry" + newenv["fruit"] = "lemon" + newenv["FRUIT"] = "orange" + newenv["frUit"] = "banana" + with subprocess.Popen(["CMD", "/c", "SET", "fruit"], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, _ = p.communicate() + self.assertEqual(stdout.strip(), b"frUit=banana") + # Windows requires at least the SYSTEMROOT environment variable to start # Python @unittest.skipIf(sys.platform == 'win32', @@ -789,6 +821,8 @@ def test_env(self): @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') == 1, 'The Python shared library cannot be loaded ' 'with an empty environment.') + @unittest.skipIf(check_sanitizer(address=True), + 'AddressSanitizer adds to the environment.') def test_empty_env(self): """Verify that env={} is as empty as possible.""" @@ -811,7 +845,26 @@ def is_env_var_to_ignore(n): if not is_env_var_to_ignore(k)] self.assertEqual(child_env_names, []) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, null byte is not checked") + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') == 1, + 'The Python shared library cannot be loaded ' + 'without some system environments.') + @unittest.skipIf(check_sanitizer(address=True), + 'AddressSanitizer adds to the environment.') + def test_one_environment_variable(self): + newenv = {'fruit': 'orange'} + cmd = [sys.executable, '-c', + 'import sys,os;' + 'sys.stdout.write("fruit="+os.getenv("fruit"))'] + if sys.platform == "win32": + cmd = ["CMD", "/c", "SET", "fruit"] + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=newenv) as p: + stdout, stderr = p.communicate() + if p.returncode and support.verbose: + print("STDOUT:", stdout.decode("ascii", "replace")) + print("STDERR:", stderr.decode("ascii", "replace")) + self.assertEqual(p.returncode, 0) + self.assertEqual(stdout.strip(), b"fruit=orange") + def test_invalid_cmd(self): # null character in the command name cmd = sys.executable + '\0' @@ -852,6 +905,19 @@ def test_invalid_env(self): stdout, stderr = p.communicate() self.assertEqual(stdout, b"orange=lemon") + @unittest.skipUnless(sys.platform == "win32", "Windows only issue") + def test_win32_invalid_env(self): + # '=' in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT=VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + + newenv = os.environ.copy() + newenv["==FRUIT"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen(ZERO_RETURN_CMD, env=newenv) + def test_communicate_stdin(self): p = subprocess.Popen([sys.executable, "-c", 'import sys;' @@ -891,6 +957,48 @@ def test_communicate(self): self.assertEqual(stdout, b"banana") self.assertEqual(stderr, b"pineapple") + def test_communicate_memoryview_input(self): + # Test memoryview input with byte elements + test_data = b"Hello, memoryview!" + mv = memoryview(test_data) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(mv) + self.assertEqual(stdout, test_data) + self.assertIsNone(stderr) + + def test_communicate_memoryview_input_nonbyte(self): + # Test memoryview input with non-byte elements (e.g., int32) + # This tests the fix for gh-134453 where non-byte memoryviews + # had incorrect length tracking on POSIX + import array + # Create an array of 32-bit integers that's large enough to trigger + # the chunked writing behavior (> PIPE_BUF) + pipe_buf = getattr(select, 'PIPE_BUF', 512) + # Each 'i' element is 4 bytes, so we need more than pipe_buf/4 elements + # Add some extra to ensure we exceed the buffer size + num_elements = pipe_buf + 1 + test_array = array.array('i', [0x64306f66 for _ in range(num_elements)]) + expected_bytes = test_array.tobytes() + mv = memoryview(test_array) + + p = subprocess.Popen([sys.executable, "-c", + 'import sys; ' + 'data = sys.stdin.buffer.read(); ' + 'sys.stdout.buffer.write(data)'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(mv) + self.assertEqual(stdout, expected_bytes, + msg=f"{len(stdout)=} =? {len(expected_bytes)=}") + self.assertIsNone(stderr) + def test_communicate_timeout(self): p = subprocess.Popen([sys.executable, "-c", 'import sys,os,time;' @@ -926,6 +1034,62 @@ def test_communicate_timeout_large_output(self): (stdout, _) = p.communicate() self.assertEqual(len(stdout), 4 * 64 * 1024) + def test_communicate_timeout_large_input(self): + # Test that timeout is enforced when writing large input to a + # slow-to-read subprocess, and that partial input is preserved + # for continuation after timeout (gh-141473). + # + # This is a regression test for Windows matching POSIX behavior. + # On POSIX, select() is used to multiplex I/O with timeout checking. + # On Windows, stdin writing must also honor the timeout rather than + # blocking indefinitely when the pipe buffer fills. + + # Input larger than typical pipe buffer (4-64KB on Windows) + input_data = b"x" * (128 * 1024) + + p = subprocess.Popen( + [sys.executable, "-c", + "import sys, time; " + "time.sleep(30); " # Don't read stdin for a long time + "sys.stdout.buffer.write(sys.stdin.buffer.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + try: + timeout = 0.2 + start = time.monotonic() + try: + p.communicate(input_data, timeout=timeout) + # If we get here without TimeoutExpired, the timeout was ignored + elapsed = time.monotonic() - start + self.fail( + f"TimeoutExpired not raised. communicate() completed in " + f"{elapsed:.2f}s, but subprocess sleeps for 30s. " + "Stdin writing blocked without enforcing timeout.") + except subprocess.TimeoutExpired: + elapsed = time.monotonic() - start + + # Timeout should occur close to the specified timeout value, + # not after waiting for the subprocess to finish sleeping. + # Allow generous margin for slow CI, but must be well under + # the subprocess sleep time. + self.assertLess(elapsed, 5.0, + f"TimeoutExpired raised after {elapsed:.2f}s; expected ~{timeout}s. " + "Stdin writing blocked without checking timeout.") + + # After timeout, continue communication. The remaining input + # should be sent and we should receive all data back. + stdout, stderr = p.communicate() + + # Verify all input was eventually received by the subprocess + self.assertEqual(len(stdout), len(input_data), + f"Expected {len(input_data)} bytes output but got {len(stdout)}") + self.assertEqual(stdout, input_data) + finally: + p.kill() + p.wait() + # Test for the fd leak reported in http://bugs.python.org/issue2791. def test_communicate_pipe_fd_leak(self): for stdin_pipe in (False, True): @@ -957,7 +1121,6 @@ def test_communicate_returns(self): self.assertEqual(stdout, None) self.assertEqual(stderr, None) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_communicate_pipe_buf(self): # communicate() with writes larger than pipe_buf # This test will probably deadlock rather than fail, if @@ -997,8 +1160,19 @@ def test_writes_before_communicate(self): self.assertEqual(stdout, b"bananasplit") self.assertEqual(stderr, b"") - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_communicate_stdin_closed_before_call(self): + # gh-70560, gh-74389: stdin.close() before communicate() + # should not raise ValueError from stdin.flush() + with subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(0)'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + p.stdin.close() # Close stdin before communicate + # This should not raise ValueError + (stdout, stderr) = p.communicate() + self.assertEqual(p.returncode, 0) + def test_universal_newlines_and_text(self): args = [ sys.executable, "-c", @@ -1038,7 +1212,6 @@ def test_universal_newlines_and_text(self): self.assertEqual(p.stdout.read(), "line4\nline5\nline6\nline7\nline8") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_universal_newlines_communicate(self): # universal newlines through communicate() p = subprocess.Popen([sys.executable, "-c", @@ -1090,7 +1263,6 @@ def test_universal_newlines_communicate_input_none(self): p.communicate() self.assertEqual(p.returncode, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_universal_newlines_communicate_stdin_stdout_stderr(self): # universal newlines through communicate(), with stdin, stdout, stderr p = subprocess.Popen([sys.executable, "-c", @@ -1117,10 +1289,8 @@ def test_universal_newlines_communicate_stdin_stdout_stderr(self): self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) # Python debug build push something like "[42442 refs]\n" # to stderr at exit of subprocess. - self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + self.assertStartsWith(stderr, "eline2\neline6\neline7\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_universal_newlines_communicate_encodings(self): # Check that universal newlines mode works for various encodings, # in particular for encodings in the UTF-16 and UTF-32 families. @@ -1143,8 +1313,6 @@ def test_universal_newlines_communicate_encodings(self): stdout, stderr = popen.communicate(input='') self.assertEqual(stdout, '1\n2\n3\n4') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_communicate_errors(self): for errors, expected in [ ('ignore', ''), @@ -1172,6 +1340,16 @@ def test_no_leaking(self): max_handles = 1026 # too much for most UNIX systems else: max_handles = 2050 # too much for (at least some) Windows setups + if resource: + # And if it is not too much, try to make it too much. + try: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft > 1024: + resource.setrlimit(resource.RLIMIT_NOFILE, (1024, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, + (soft, hard)) + except (OSError, ValueError): + pass handles = [] tmpdir = tempfile.mkdtemp() try: @@ -1186,7 +1364,9 @@ def test_no_leaking(self): else: self.skipTest("failed to reach the file descriptor limit " "(tried %d)" % max_handles) - # Close a couple of them (should be enough for a subprocess) + # Close a couple of them (should be enough for a subprocess). + # Close lower file descriptors, so select() will work. + handles.reverse() for i in range(10): os.close(handles.pop()) # Loop creating some subprocesses. If one of them leaks some fds, @@ -1284,15 +1464,12 @@ def _test_bufsize_equal_one(self, line, expected, universal_newlines): self.assertEqual(p.returncode, 0) self.assertEqual(read_line, expected) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_bufsize_equal_one_text_mode(self): # line is flushed in text mode with bufsize=1. # we should get the full line in return line = "line\n" self._test_bufsize_equal_one(line, line, universal_newlines=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bufsize_equal_one_binary_mode(self): # line is not flushed in binary mode with bufsize=1. # we should get empty response @@ -1300,6 +1477,7 @@ def test_bufsize_equal_one_binary_mode(self): with self.assertWarnsRegex(RuntimeWarning, 'line buffering'): self._test_bufsize_equal_one(line, b'', universal_newlines=False) + @support.requires_resource('cpu') def test_leaking_fds_on_error(self): # see bug #5179: Popen leaks file descriptors to PIPEs if # the child fails to execute; this will eventually exhaust @@ -1363,7 +1541,7 @@ def open_fds(): t = threading.Thread(target=open_fds) t.start() try: - with self.assertRaises(EnvironmentError): + with self.assertRaises(OSError): subprocess.Popen(NONEXISTING_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -1443,7 +1621,7 @@ def test_issue8780(self): "[sys.executable, '-c', 'print(\"Hello World!\")'])", 'assert retcode == 0')) output = subprocess.check_output([sys.executable, '-c', code]) - self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + self.assertStartsWith(output, b'Hello World!') def test_handles_closed_on_exception(self): # If CreateProcess exits with an error, ensure the @@ -1465,7 +1643,6 @@ def test_handles_closed_on_exception(self): self.assertFalse(os.path.exists(ofname)) self.assertFalse(os.path.exists(efname)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_communicate_epipe(self): # Issue 10963: communicate() should hide EPIPE p = subprocess.Popen(ZERO_RETURN_CMD, @@ -1478,9 +1655,6 @@ def test_communicate_epipe(self): p.communicate(b"x" * 2**20) def test_repr(self): - path_cmd = pathlib.Path("my-tool.py") - pathlib_cls = path_cmd.__class__.__name__ - cases = [ ("ls", True, 123, "<Popen: returncode: 123 args: 'ls'>"), ('a' * 100, True, 0, @@ -1488,7 +1662,8 @@ def test_repr(self): (["ls"], False, None, "<Popen: returncode: None args: ['ls']>"), (["ls", '--my-opts', 'a' * 100], False, None, "<Popen: returncode: None args: ['ls', '--my-opts', 'aaaaaaaaaaaaaaaaaaaaaaaa...>"), - (path_cmd, False, 7, f"<Popen: returncode: 7 args: {pathlib_cls}('my-tool.py')>") + (os_helper.FakePath("my-tool.py"), False, 7, + "<Popen: returncode: 7 args: <FakePath 'my-tool.py'>>") ] with unittest.mock.patch.object(subprocess.Popen, '_execute_child'): for cmd, shell, code, sx in cases: @@ -1496,7 +1671,6 @@ def test_repr(self): p.returncode = code self.assertEqual(repr(p), sx) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_communicate_epipe_only_stdin(self): # Issue 10963: communicate() should hide EPIPE p = subprocess.Popen(ZERO_RETURN_CMD, @@ -1505,8 +1679,6 @@ def test_communicate_epipe_only_stdin(self): p.wait() p.communicate(b"x" * 2**20) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), "Requires signal.SIGUSR1") @unittest.skipUnless(hasattr(os, 'kill'), @@ -1556,8 +1728,6 @@ def test_file_not_found_includes_filename(self): subprocess.call(['/opt/nonexistent_binary', 'with', 'some', 'args']) self.assertEqual(c.exception.filename, '/opt/nonexistent_binary') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(mswindows, "behavior currently not supported on Windows") def test_file_not_found_with_bad_cwd(self): with self.assertRaises(FileNotFoundError) as c: @@ -1568,20 +1738,55 @@ def test_class_getitems(self): self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) - @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), - "vfork() not enabled by configure.") - @mock.patch("subprocess._fork_exec") - def test__use_vfork(self, mock_fork_exec): - self.assertTrue(subprocess._USE_VFORK) # The default value regardless. - mock_fork_exec.side_effect = RuntimeError("just testing args") - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - mock_fork_exec.assert_called_once() - self.assertTrue(mock_fork_exec.call_args.args[-1]) - with mock.patch.object(subprocess, '_USE_VFORK', False): - with self.assertRaises(RuntimeError): - subprocess.run([sys.executable, "-c", "pass"]) - self.assertFalse(mock_fork_exec.call_args_list[-1].args[-1]) + @unittest.skipUnless(hasattr(subprocess, '_winapi'), + 'need subprocess._winapi') + def test_wait_negative_timeout(self): + proc = subprocess.Popen(ZERO_RETURN_CMD) + with proc: + patch = mock.patch.object( + subprocess._winapi, + 'WaitForSingleObject', + return_value=subprocess._winapi.WAIT_OBJECT_0) + with patch as mock_wait: + proc.wait(-1) # negative timeout + mock_wait.assert_called_once_with(proc._handle, 0) + proc.returncode = None + + self.assertEqual(proc.wait(), 0) + + def test_post_timeout_communicate_sends_input(self): + """GH-141473 regression test; the stdin pipe must close""" + with subprocess.Popen( + [sys.executable, "-uc", """\ +import sys +while c := sys.stdin.read(512): + sys.stdout.write(c) +print() +"""], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as proc: + try: + data = f"spam{'#'*4096}beans" + proc.communicate( + input=data, + timeout=0, + ) + except subprocess.TimeoutExpired as exc: + pass + # Prior to the bugfix, this would hang as the stdin + # pipe to the child had not been closed. + try: + stdout, stderr = proc.communicate(timeout=15) + except subprocess.TimeoutExpired as exc: + self.fail("communicate() hung waiting on child process that should have seen its stdin pipe close and exit") + self.assertEqual( + proc.returncode, 0, + msg=f"STDERR:\n{stderr}\nSTDOUT:\n{stdout}") + self.assertStartsWith(stdout, "spam") + self.assertIn("beans", stdout) class RunFuncTestCase(BaseTestCase): @@ -1658,17 +1863,9 @@ def test_check_output_stdin_with_input_arg(self): def test_check_output_timeout(self): with self.assertRaises(subprocess.TimeoutExpired) as c: - cp = self.run_python(( - "import sys, time\n" - "sys.stdout.write('BDFL')\n" - "sys.stdout.flush()\n" - "time.sleep(3600)"), - # Some heavily loaded buildbots (sparc Debian 3.x) require - # this much time to start and print. - timeout=3, stdout=subprocess.PIPE) - self.assertEqual(c.exception.output, b'BDFL') - # output is aliased to stdout - self.assertEqual(c.exception.stdout, b'BDFL') + cp = self.run_python( + "import time; time.sleep(3600)", + timeout=0.1, stdout=subprocess.PIPE) def test_run_kwargs(self): newenv = os.environ.copy() @@ -1706,6 +1903,15 @@ def test_run_with_pathlike_path_and_arguments(self): res = subprocess.run(args) self.assertEqual(res.returncode, 57) + @unittest.skipIf(mswindows, "TODO: RUSTPYTHON; empty env block fails nondeterministically") + @unittest.skipUnless(mswindows, "Maybe test trigger a leak on Ubuntu") + def test_run_with_an_empty_env(self): + # gh-105436: fix subprocess.run(..., env={}) broken on Windows + args = [sys.executable, "-c", 'pass'] + # Ignore subprocess errors - we only care that the API doesn't + # raise an OSError + subprocess.run(args, env={}) + def test_capture_output(self): cp = self.run_python(("import sys;" "sys.stdout.write('BDFL'); " @@ -1714,6 +1920,13 @@ def test_capture_output(self): self.assertIn(b'BDFL', cp.stdout) self.assertIn(b'FLUFL', cp.stderr) + def test_stdout_stdout(self): + # run() refuses to accept stdout=STDOUT + with self.assertRaises(ValueError, + msg=("STDOUT can only be used for stderr")): + self.run_python("print('will not be run')", + stdout=subprocess.STDOUT) + def test_stdout_with_capture_output_arg(self): # run() refuses to accept 'stdout' with 'capture_output' tf = tempfile.TemporaryFile() @@ -1758,8 +1971,6 @@ def test_run_with_shell_timeout_and_capture_output(self): msg="TimeoutExpired was delayed! Bad traceback:\n```\n" f"{stacks}```") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encoding_warning(self): code = textwrap.dedent("""\ from subprocess import * @@ -1770,8 +1981,8 @@ def test_encoding_warning(self): capture_output=True) lines = cp.stderr.splitlines() self.assertEqual(len(lines), 2, lines) - self.assertTrue(lines[0].startswith(b"<string>:2: EncodingWarning: ")) - self.assertTrue(lines[1].startswith(b"<string>:3: EncodingWarning: ")) + self.assertStartsWith(lines[0], b"<string>:2: EncodingWarning: ") + self.assertStartsWith(lines[1], b"<string>:3: EncodingWarning: ") def _get_test_grp_name(): @@ -1806,8 +2017,6 @@ def _get_chdir_exception(self): self._nonexistent_dir) return desired_exception - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_cwd(self): """Test error in the child raised in the parent for a bad cwd.""" desired_exception = self._get_chdir_exception() @@ -1823,8 +2032,6 @@ def test_exception_cwd(self): else: self.fail("Expected OSError: %s" % desired_exception) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_bad_executable(self): """Test error in the child raised in the parent for a bad executable.""" desired_exception = self._get_chdir_exception() @@ -1840,8 +2047,6 @@ def test_exception_bad_executable(self): else: self.fail("Expected OSError: %s" % desired_exception) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_bad_args_0(self): """Test error in the child raised in the parent for a bad args[0].""" desired_exception = self._get_chdir_exception() @@ -1906,8 +2111,6 @@ def bad_error(*args): self.assertIn(repr(error_data), str(e.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(not os.path.exists('/proc/self/status'), "need /proc/self/status") def test_restore_signals(self): @@ -1948,8 +2151,6 @@ def test_start_new_session(self): child_sid = int(output) self.assertNotEqual(parent_sid, child_sid) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'setpgid') and hasattr(os, 'getpgid'), 'no setpgid or getpgid on platform') def test_process_group_0(self): @@ -1968,13 +2169,11 @@ def test_process_group_0(self): child_pgid = int(output) self.assertNotEqual(parent_pgid, child_pgid) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'setreuid'), 'no setreuid on platform') def test_user(self): - # For code coverage of the user parameter. We don't care if we get an - # EPERM error from it depending on the test execution environment, that - # still indicates that it was called. + # For code coverage of the user parameter. We don't care if we get a + # permission error from it depending on the test execution environment, + # that still indicates that it was called. uid = os.geteuid() test_users = [65534 if uid != 65534 else 65533, uid] @@ -1998,11 +2197,11 @@ def test_user(self): "import os; print(os.getuid())"], user=user, close_fds=close_fds) - except PermissionError: # (EACCES, EPERM) - pass - except OSError as e: - if e.errno not in (errno.EACCES, errno.EPERM): - raise + except PermissionError as e: # (EACCES, EPERM) + if e.errno == errno.EACCES: + self.assertEqual(e.filename, sys.executable) + else: + self.assertIsNone(e.filename) else: if isinstance(user, str): user_uid = pwd.getpwnam(user).pw_uid @@ -2027,8 +2226,6 @@ def test_user_error(self): with self.assertRaises(ValueError): subprocess.check_call(ZERO_RETURN_CMD, user=65535) - # TODO: RUSTPYTHON, observed gids do not match expected gids - @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'setregid'), 'no setregid() on platform') def test_group(self): gid = os.getegid() @@ -2048,8 +2245,8 @@ def test_group(self): "import os; print(os.getgid())"], group=group, close_fds=close_fds) - except PermissionError: # (EACCES, EPERM) - pass + except PermissionError as e: # (EACCES, EPERM) + self.assertIsNone(e.filename) else: if isinstance(group, str): group_gid = grp.getgrnam(group).gr_gid @@ -2076,14 +2273,18 @@ def test_group_error(self): with self.assertRaises(ValueError): subprocess.check_call(ZERO_RETURN_CMD, group=65535) - # TODO: RUSTPYTHON, observed gids do not match expected gids - @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform') def test_extra_groups(self): gid = os.getegid() group_list = [65534 if gid != 65534 else 65533] + self._test_extra_groups_impl(gid=gid, group_list=group_list) + + @unittest.skipUnless(hasattr(os, 'setgroups'), 'no setgroups() on platform') + def test_extra_groups_empty_list(self): + self._test_extra_groups_impl(gid=os.getegid(), group_list=[]) + + def _test_extra_groups_impl(self, *, gid, group_list): name_group = _get_test_grp_name() - perm_error = False if grp is not None: group_list.append(name_group) @@ -2093,11 +2294,9 @@ def test_extra_groups(self): [sys.executable, "-c", "import os, sys, json; json.dump(os.getgroups(), sys.stdout)"], extra_groups=group_list) - except OSError as ex: - if ex.errno != errno.EPERM: - raise - perm_error = True - + except PermissionError as e: + self.assertIsNone(e.filename) + self.skipTest("setgroup() EPERM; this test may require root.") else: parent_groups = os.getgroups() child_groups = json.loads(output) @@ -2108,12 +2307,16 @@ def test_extra_groups(self): else: desired_gids = group_list - if perm_error: - self.assertEqual(set(child_groups), set(parent_groups)) - else: - self.assertEqual(set(desired_gids), set(child_groups)) + self.assertEqual(set(desired_gids), set(child_groups)) - # make sure we bomb on negative values + if grp is None: + with self.assertRaises(ValueError): + subprocess.check_call(ZERO_RETURN_CMD, + extra_groups=[name_group]) + + # No skip necessary, this test won't make it to a setgroup() call. + @unittest.skip("TODO: RUSTPYTHON; clarify failure condition") + def test_extra_groups_invalid_gid_t_values(self): with self.assertRaises(ValueError): subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[-1]) @@ -2122,18 +2325,6 @@ def test_extra_groups(self): cwd=os.curdir, env=os.environ, extra_groups=[2**64]) - if grp is None: - with self.assertRaises(ValueError): - subprocess.check_call(ZERO_RETURN_CMD, - extra_groups=[name_group]) - - @unittest.skipIf(hasattr(os, 'setgroups'), 'setgroups() available on platform') - def test_extra_groups_error(self): - with self.assertRaises(ValueError): - subprocess.check_call(ZERO_RETURN_CMD, extra_groups=[]) - - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(mswindows or not hasattr(os, 'umask'), 'POSIX umask() is not available.') def test_umask(self): @@ -2185,8 +2376,6 @@ def test_CalledProcessError_str_non_zero(self): error_string = str(err) self.assertIn("non-zero exit status 2.", error_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec(self): # DISCLAIMER: Setting environment variables is *not* a good use # of a preexec_fn. This is merely a test. @@ -2198,8 +2387,6 @@ def test_preexec(self): with p: self.assertEqual(p.stdout.read(), b"apple") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec_exception(self): def raise_it(): raise ValueError("What if two swallows carried a coconut?") @@ -2241,8 +2428,6 @@ def _execute_child(self, *args, **kwargs): for fd in devzero_fds: os.close(fd) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") def test_preexec_errpipe_does_not_double_close_pipes(self): """Issue16140: Don't double close pipes on preexec error.""" @@ -2257,8 +2442,6 @@ def raise_it(): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=raise_it) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec_gc_module_failure(self): # This tests the code that disables garbage collection if the child # process will execute any Python. @@ -2280,8 +2463,6 @@ def test_preexec_gc_module_failure(self): if not enabled: gc.disable() - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( sys.platform == 'darwin', 'setrlimit() seems to fail on OS X') def test_preexec_fork_failure(self): @@ -2692,8 +2873,6 @@ def test_swap_std_fds_with_one_closed(self): for to_fds in itertools.permutations(range(3), 2): self._check_swap_std_fds_with_one_closed(from_fds, to_fds) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogates_error_message(self): def prepare(): raise ValueError("surrogate:\uDCff") @@ -2713,8 +2892,6 @@ def prepare(): else: self.fail("Expected ValueError or subprocess.SubprocessError") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_undecodable_env(self): for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): encoded_value = value.encode("ascii", "surrogateescape") @@ -2835,7 +3012,7 @@ def kill_p2(): p1.stdout.close() p2.stdout.close() - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip("TODO: RUSTPYTHON; flaky test") def test_close_fds(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -2886,7 +3063,7 @@ def test_close_fds(self): @unittest.skipIf(sys.platform.startswith("freebsd") and os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, - "Requires fdescfs mounted on /dev/fd on FreeBSD.") + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_close_fds_when_max_fd_is_lowered(self): """Confirm that issue21618 is fixed (may fail under valgrind).""" fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -2963,11 +3140,11 @@ def test_close_fds_when_max_fd_is_lowered(self): msg="Some fds were left open.") - @unittest.skip("TODO: RUSTPYTHON, flaky test") # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file # descriptor of a pipe closed in the parent process is valid in the # child process according to fstat(), but the mode of the file # descriptor is invalid, and read or write raise an error. + @unittest.skip("TODO: RUSTPYTHON; flaky test") @support.requires_mac_ver(10, 5) def test_pass_fds(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -3171,8 +3348,6 @@ def test_leak_fast_process_del_killed(self): else: self.assertNotIn(ident, [id(o) for o in subprocess._active]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_close_fds_after_preexec(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -3266,7 +3441,7 @@ def __int__(self): 1, 2, 3, 4, True, True, 0, None, None, None, -1, - None, "no vfork") + None) self.assertIn('fds_to_keep', str(c.exception)) finally: if not gc_enabled: @@ -3382,6 +3557,81 @@ def test_communicate_repeated_call_after_stdout_close(self): except subprocess.TimeoutExpired: pass + def test_preexec_at_exit(self): + code = f"""if 1: + import atexit + import subprocess + + def dummy(): + pass + + class AtFinalization: + def __del__(self): + print("OK") + subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) + print("shouldn't be printed") + at_finalization = AtFinalization() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out.strip(), b"OK") + self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) + + @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), + "vfork() not enabled by configure.") + @strace_helper.requires_strace() + @mock.patch("subprocess._USE_POSIX_SPAWN", new=False) + def test_vfork_used_when_expected(self): + # This is a performance regression test to ensure we default to using + # vfork() when possible. + # Technically this test could pass when posix_spawn is used as well + # because libc tends to implement that internally using vfork. But + # that'd just be testing a libc+kernel implementation detail. + + # Are intersted in the system calls: + # clone,clone2,clone3,fork,vfork,exit,exit_group + # Unfortunately using `--trace` with that list to strace fails because + # not all are supported on all platforms (ex. clone2 is ia64 only...) + # So instead use `%process` which is recommended by strace, and contains + # the above. + true_binary = "/bin/true" + strace_args = ["--trace=%process"] + + with self.subTest(name="default_is_vfork"): + vfork_result = strace_helper.strace_python( + f"""\ + import subprocess + subprocess.check_call([{true_binary!r}])""", + strace_args + ) + # Match both vfork() and clone(..., flags=...|CLONE_VFORK|...) + self.assertRegex(vfork_result.event_bytes, br"(?i)vfork") + # Do NOT check that fork() or other clones did not happen. + # If the OS denys the vfork it'll fallback to plain fork(). + + # Test that each individual thing that would disable the use of vfork + # actually disables it. + for sub_name, preamble, sp_kwarg, expect_permission_error in ( + ("preexec", "", "preexec_fn=lambda: None", False), + ("setgid", "", f"group={os.getgid()}", True), + ("setuid", "", f"user={os.getuid()}", True), + ("setgroups", "", "extra_groups=[]", True), + ): + with self.subTest(name=sub_name): + non_vfork_result = strace_helper.strace_python( + f"""\ + import subprocess + {preamble} + try: + subprocess.check_call( + [{true_binary!r}], **dict({sp_kwarg})) + except PermissionError: + if not {expect_permission_error}: + raise""", + strace_args + ) + # Ensure neither vfork() or clone(..., flags=...|CLONE_VFORK|...). + self.assertNotRegex(non_vfork_result.event_bytes, br"(?i)vfork") + @unittest.skipUnless(mswindows, "Windows specific tests") class Win32ProcessTestCase(BaseTestCase): @@ -3475,8 +3725,6 @@ def test_close_fds(self): close_fds=True) self.assertEqual(rc, 47) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_close_fds_with_stdio(self): import msvcrt @@ -3559,8 +3807,6 @@ def test_shell_string(self): with p: self.assertIn(b"physalis", p.stdout.read()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_shell_encodings(self): # Run command through the shell (string) for enc in ['ansi', 'oem']: @@ -3707,7 +3953,6 @@ def popen_via_context_manager(*args, **kwargs): raise KeyboardInterrupt # Test how __exit__ handles ^C. self._test_keyboardinterrupt_no_kill(popen_via_context_manager) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_getoutput(self): self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), @@ -3780,28 +4025,20 @@ def with_spaces(self, *args, **kwargs): "2 [%r, 'ab cd']" % self.fname ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_shell_string_with_spaces(self): # call() function with string argument with spaces on Windows self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, "ab cd"), shell=1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_shell_sequence_with_spaces(self): # call() function with sequence argument with spaces on Windows self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_noshell_string_with_spaces(self): # call() function with string argument with spaces on Windows self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, "ab cd")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_noshell_sequence_with_spaces(self): # call() function with sequence argument with spaces on Windows self.with_spaces([sys.executable, self.fname, "ab cd"]) diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index 90af9da8f9f..d6d08ee53f8 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -1,14 +1,11 @@ """Do a minimal test of all the modules that aren't otherwise tested.""" import importlib -import platform -import sys from test import support from test.support import import_helper from test.support import warnings_helper import unittest class TestUntestedModules(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_untested_modules_can_be_imported(self): untested = ('encodings',) with warnings_helper.check_warnings(quiet=True): @@ -21,38 +18,15 @@ def test_untested_modules_can_be_imported(self): self.fail('{} has tests even though test_sundry claims ' 'otherwise'.format(name)) - import distutils.bcppcompiler - import distutils.ccompiler - import distutils.cygwinccompiler - import distutils.filelist - import distutils.text_file - import distutils.unixccompiler - - import distutils.command.bdist_dumb - if sys.platform.startswith('win') and not platform.win32_is_iot(): - import distutils.command.bdist_msi - import distutils.command.bdist - import distutils.command.bdist_rpm - import distutils.command.build_clib - import distutils.command.build_ext - import distutils.command.build - import distutils.command.clean - import distutils.command.config - import distutils.command.install_data - import distutils.command.install_egg_info - import distutils.command.install_headers - import distutils.command.install_lib - import distutils.command.register - import distutils.command.sdist - import distutils.command.upload - - import html.entities + import html.entities # noqa: F401 try: - import tty # Not available on Windows + # Not available on Windows + import tty # noqa: F401 except ImportError: if support.verbose: print("skipping tty") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index ca8dc6e869d..5548f4c71a2 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -1,6 +1,13 @@ """Unit tests for zero-argument super() & related machinery.""" +import textwrap +import threading import unittest +from unittest.mock import patch +from test.support import import_helper, threading_helper + + +ADAPTIVE_WARMUP_DELAY = 2 class A: @@ -84,41 +91,43 @@ def nested(): self.assertEqual(E().f(), 'AE') - # SyntaxError - # def test_various___class___pathologies(self): - # # See issue #12370 - # class X(A): - # def f(self): - # return super().f() - # __class__ = 413 - # x = X() - # self.assertEqual(x.f(), 'A') - # self.assertEqual(x.__class__, 413) - # class X: - # x = __class__ - # def f(): - # __class__ - # self.assertIs(X.x, type(self)) - # with self.assertRaises(NameError) as e: - # exec("""class X: - # __class__ - # def f(): - # __class__""", globals(), {}) - # self.assertIs(type(e.exception), NameError) # Not UnboundLocalError - # class X: - # global __class__ - # __class__ = 42 - # def f(): - # __class__ - # self.assertEqual(globals()["__class__"], 42) - # del globals()["__class__"] - # self.assertNotIn("__class__", X.__dict__) - # class X: - # nonlocal __class__ - # __class__ = 42 - # def f(): - # __class__ - # self.assertEqual(__class__, 42) + # TODO: RUSTPYTHON; SyntaxError: name '__class__' is assigned to before global declaration + ''' + def test_various___class___pathologies(self): + # See issue #12370 + class X(A): + def f(self): + return super().f() + __class__ = 413 + x = X() + self.assertEqual(x.f(), 'A') + self.assertEqual(x.__class__, 413) + class X: + x = __class__ + def f(): + __class__ + self.assertIs(X.x, type(self)) + with self.assertRaises(NameError) as e: + exec("""class X: + __class__ + def f(): + __class__""", globals(), {}) + self.assertIs(type(e.exception), NameError) # Not UnboundLocalError + class X: + global __class__ + __class__ = 42 + def f(): + __class__ + self.assertEqual(globals()["__class__"], 42) + del globals()["__class__"] + self.assertNotIn("__class__", X.__dict__) + class X: + nonlocal __class__ + __class__ = 42 + def f(): + __class__ + self.assertEqual(__class__, 42) + ''' def test___class___instancemethod(self): # See issue #14857 @@ -162,8 +171,6 @@ def f(): self.assertIs(test_class, A) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test___class___delayed(self): # See issue #23722 test_namespace = None @@ -184,8 +191,7 @@ def f(): B = type("B", (), test_namespace) self.assertIs(B.f(), B) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test___class___mro(self): # See issue #23722 test_class = None @@ -203,8 +209,7 @@ def f(): self.assertIs(test_class, A) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test___classcell___expected_behaviour(self): # See issue #23722 class Meta(type): @@ -236,8 +241,6 @@ def f(self): with self.assertRaises(AttributeError): WithClassRef.__classcell__ - # TODO: RUSTPYTHON - @unittest.expectedFailure def test___classcell___missing(self): # See issue #23722 # Some metaclasses may not pass the original namespace to type.__new__ @@ -292,17 +295,28 @@ def f(self): def test_obscure_super_errors(self): def f(): super() - self.assertRaises(RuntimeError, f) + with self.assertRaisesRegex(RuntimeError, r"no arguments"): + f() + + class C: + def f(): + super() + with self.assertRaisesRegex(RuntimeError, r"no arguments"): + C.f() + def f(x): del x super() - self.assertRaises(RuntimeError, f, None) + with self.assertRaisesRegex(RuntimeError, r"arg\[0\] deleted"): + f(None) + class X: def f(x): nonlocal __class__ del __class__ super() - self.assertRaises(RuntimeError, X().f) + with self.assertRaisesRegex(RuntimeError, r"empty __class__ cell"): + X().f() def test_cell_as_self(self): class X: @@ -326,6 +340,211 @@ def test_super_init_leaks(self): for i in range(1000): super.__init__(sp, int, i) + def test_super_argcount(self): + with self.assertRaisesRegex(TypeError, "expected at most"): + super(int, int, int) + + def test_super_argtype(self): + with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): + super(1, int) + + def test_shadowed_global(self): + source = textwrap.dedent( + """ + class super: + msg = "truly super" + + class C: + def method(self): + return super().msg + """, + ) + with import_helper.ready_to_import(name="shadowed_super", source=source): + import shadowed_super + self.assertEqual(shadowed_super.C().method(), "truly super") + import_helper.unload("shadowed_super") + + def test_shadowed_local(self): + class super: + msg = "quite super" + + class C: + def method(self): + return super().msg + + self.assertEqual(C().method(), "quite super") + + def test_shadowed_dynamic(self): + class MySuper: + msg = "super super" + + class C: + def method(self): + return super().msg + + with patch(f"{__name__}.super", MySuper) as m: + self.assertEqual(C().method(), "super super") + + def test_shadowed_dynamic_two_arg(self): + call_args = [] + class MySuper: + def __init__(self, *args): + call_args.append(args) + msg = "super super" + + class C: + def method(self): + return super(1, 2).msg + + with patch(f"{__name__}.super", MySuper) as m: + self.assertEqual(C().method(), "super super") + self.assertEqual(call_args, [(1, 2)]) + + def test_attribute_error(self): + class C: + def method(self): + return super().msg + + with self.assertRaisesRegex(AttributeError, "'super' object has no attribute 'msg'"): + C().method() + + def test_bad_first_arg(self): + class C: + def method(self): + return super(1, self).method() + + with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): + C().method() + + def test_supercheck_fail(self): + class C: + def method(self, type_, obj): + return super(type_, obj).method() + + c = C() + err_msg = ( + r"super\(type, obj\): obj \({} {}\) is not " + r"an instance or subtype of type \({}\)." + ) + + cases = ( + (int, c, int.__name__, C.__name__, "instance of"), + # obj is instance of type + (C, list(), C.__name__, list.__name__, "instance of"), + # obj is type itself + (C, list, C.__name__, list.__name__, "type"), + ) + + for case in cases: + with self.subTest(case=case): + type_, obj, type_str, obj_str, instance_or_type = case + regex = err_msg.format(instance_or_type, obj_str, type_str) + + with self.assertRaisesRegex(TypeError, regex): + c.method(type_, obj) + + def test_super___class__(self): + class C: + def method(self): + return super().__class__ + + self.assertEqual(C().method(), super) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: type 'super' is not an acceptable base type + def test_super_subclass___class__(self): + class mysuper(super): + pass + + class C: + def method(self): + return mysuper(C, self).__class__ + + self.assertEqual(C().method(), mysuper) + + def test_unusual_getattro(self): + class MyType(type): + pass + + def test(name): + mytype = MyType(name, (MyType,), {}) + super(MyType, type(mytype)).__setattr__(mytype, "bar", 1) + self.assertEqual(mytype.bar, 1) + + for _ in range(ADAPTIVE_WARMUP_DELAY): + test("foo1") + + def test_reassigned_new(self): + class A: + def __new__(cls): + pass + + def __init_subclass__(cls): + if "__new__" not in cls.__dict__: + cls.__new__ = cls.__new__ + + class B(A): + pass + + class C(B): + def __new__(cls): + return super().__new__(cls) + + for _ in range(ADAPTIVE_WARMUP_DELAY): + C() + + def test_mixed_staticmethod_hierarchy(self): + # This test is just a desugared version of `test_reassigned_new` + class A: + @staticmethod + def some(cls, *args, **kwargs): + self.assertFalse(args) + self.assertFalse(kwargs) + + class B(A): + def some(cls, *args, **kwargs): + return super().some(cls, *args, **kwargs) + + class C(B): + @staticmethod + def some(cls): + return super().some(cls) + + for _ in range(ADAPTIVE_WARMUP_DELAY): + C.some(C) + + @threading_helper.requires_working_threading() + def test___class___modification_multithreaded(self): + """ Note: this test isn't actually testing anything on its own. + It requires a sys audithook to be set to crash on older Python. + This should be the case anyways as our test suite sets + an audit hook. + """ + + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(200): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for _ in range(6): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 673160c20bb..37b5543badf 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -1,12 +1,16 @@ +import contextlib import errno import importlib import io +import logging import os import shutil +import signal import socket import stat import subprocess import sys +import sysconfig import tempfile import textwrap import unittest @@ -22,26 +26,51 @@ TESTFN = os_helper.TESTFN +class LogCaptureHandler(logging.StreamHandler): + # Inspired by pytest's caplog + def __init__(self): + super().__init__(io.StringIO()) + self.records = [] + + def emit(self, record) -> None: + self.records.append(record) + super().emit(record) + + def handleError(self, record): + raise + + +@contextlib.contextmanager +def _caplog(): + handler = LogCaptureHandler() + root_logger = logging.getLogger() + root_logger.addHandler(handler) + try: + yield handler + finally: + root_logger.removeHandler(handler) + + class TestSupport(unittest.TestCase): @classmethod def setUpClass(cls): - orig_filter_len = len(warnings.filters) + orig_filter_len = len(warnings._get_filters()) cls._warnings_helper_token = support.ignore_deprecations_from( "test.support.warnings_helper", like=".*used in test_support.*" ) cls._test_support_token = support.ignore_deprecations_from( __name__, like=".*You should NOT be seeing this.*" ) - assert len(warnings.filters) == orig_filter_len + 2 + assert len(warnings._get_filters()) == orig_filter_len + 2 @classmethod def tearDownClass(cls): - orig_filter_len = len(warnings.filters) + orig_filter_len = len(warnings._get_filters()) support.clear_ignored_deprecations( cls._warnings_helper_token, cls._test_support_token, ) - assert len(warnings.filters) == orig_filter_len - 2 + assert len(warnings._get_filters()) == orig_filter_len - 2 def test_ignored_deprecations_are_silent(self): """Test support.ignore_deprecations_from() silences warnings""" @@ -69,7 +98,7 @@ def test_get_original_stdout(self): self.assertEqual(support.get_original_stdout(), sys.stdout) def test_unload(self): - import sched + import sched # noqa: F401 self.assertIn("sched", sys.modules) import_helper.unload("sched") self.assertNotIn("sched", sys.modules) @@ -185,7 +214,7 @@ def test_temp_dir__existing_dir__quiet_true(self): path = os.path.realpath(path) try: - with warnings_helper.check_warnings() as recorder: + with warnings_helper.check_warnings() as recorder, _caplog() as caplog: with os_helper.temp_dir(path, quiet=True) as temp_path: self.assertEqual(path, temp_path) warnings = [str(w.message) for w in recorder.warnings] @@ -194,11 +223,14 @@ def test_temp_dir__existing_dir__quiet_true(self): finally: shutil.rmtree(path) - self.assertEqual(len(warnings), 1, warnings) - warn = warnings[0] - self.assertTrue(warn.startswith(f'tests may fail, unable to create ' - f'temporary directory {path!r}: '), - warn) + self.assertListEqual(warnings, []) + self.assertEqual(len(caplog.records), 1) + record = caplog.records[0] + self.assertStartsWith( + record.getMessage(), + f'tests may fail, unable to create ' + f'temporary directory {path!r}: ' + ) @support.requires_fork() def test_temp_dir__forked_child(self): @@ -258,35 +290,41 @@ def test_change_cwd__non_existent_dir__quiet_true(self): with os_helper.temp_dir() as parent_dir: bad_dir = os.path.join(parent_dir, 'does_not_exist') - with warnings_helper.check_warnings() as recorder: + with warnings_helper.check_warnings() as recorder, _caplog() as caplog: with os_helper.change_cwd(bad_dir, quiet=True) as new_cwd: self.assertEqual(new_cwd, original_cwd) self.assertEqual(os.getcwd(), new_cwd) warnings = [str(w.message) for w in recorder.warnings] - self.assertEqual(len(warnings), 1, warnings) - warn = warnings[0] - self.assertTrue(warn.startswith(f'tests may fail, unable to change ' - f'the current working directory ' - f'to {bad_dir!r}: '), - warn) + self.assertListEqual(warnings, []) + self.assertEqual(len(caplog.records), 1) + record = caplog.records[0] + self.assertStartsWith( + record.getMessage(), + f'tests may fail, unable to change ' + f'the current working directory ' + f'to {bad_dir!r}: ' + ) # Tests for change_cwd() def test_change_cwd__chdir_warning(self): """Check the warning message when os.chdir() fails.""" path = TESTFN + '_does_not_exist' - with warnings_helper.check_warnings() as recorder: + with warnings_helper.check_warnings() as recorder, _caplog() as caplog: with os_helper.change_cwd(path=path, quiet=True): pass messages = [str(w.message) for w in recorder.warnings] - self.assertEqual(len(messages), 1, messages) - msg = messages[0] - self.assertTrue(msg.startswith(f'tests may fail, unable to change ' - f'the current working directory ' - f'to {path!r}: '), - msg) + self.assertListEqual(messages, []) + self.assertEqual(len(caplog.records), 1) + record = caplog.records[0] + self.assertStartsWith( + record.getMessage(), + f'tests may fail, unable to change ' + f'the current working directory ' + f'to {path!r}: ', + ) # Tests for temp_cwd() @@ -310,7 +348,6 @@ def test_temp_cwd__name_none(self): def test_sortdict(self): self.assertEqual(support.sortdict({3:3, 2:2, 1:1}), "{1: 1, 2: 2, 3: 3}") - @unittest.skipIf(sys.platform.startswith("win"), "TODO: RUSTPYTHON; actual c fds on windows") def test_make_bad_fd(self): fd = os_helper.make_bad_fd() with self.assertRaises(OSError) as cm: @@ -370,10 +407,10 @@ class Obj: with support.swap_attr(obj, "y", 5) as y: self.assertEqual(obj.y, 5) self.assertIsNone(y) - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') with support.swap_attr(obj, "y", 5): del obj.y - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') def test_swap_item(self): D = {"x":1} @@ -421,8 +458,7 @@ def test_detect_api_mismatch__ignore(self): self.OtherClass, self.RefClass, ignore=ignore) self.assertEqual(set(), missing_items) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_check__all__(self): extra = {'tempdir'} not_exported = {'template'} @@ -433,10 +469,7 @@ def test_check__all__(self): extra = { 'TextTestResult', - 'findTestCases', - 'getTestCaseNames', 'installHandler', - 'makeSuite', } not_exported = {'load_tests', "TestProgram", "BaseTestSuite"} support.check__all__(self, @@ -529,6 +562,7 @@ def test_args_from_interpreter_flags(self): ['-Wignore', '-X', 'dev'], ['-X', 'faulthandler'], ['-X', 'importtime'], + ['-X', 'importtime=2'], ['-X', 'showrefcount'], ['-X', 'tracemalloc'], ['-X', 'tracemalloc=3'], @@ -552,119 +586,13 @@ def test_optim_args_from_interpreter_flags(self): with self.subTest(opts=opts): self.check_options(opts, 'optim_args_from_interpreter_flags') - def test_match_test(self): - class Test: - def __init__(self, test_id): - self.test_id = test_id - - def id(self): - return self.test_id - - test_access = Test('test.test_os.FileTests.test_access') - test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') - - # Test acceptance - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests([]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the full test identifier - support.set_match_tests([test_access.id()], None) - self.assertTrue(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # match the module name - support.set_match_tests(['test_os'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Test '*' pattern - support.set_match_tests(['test_*'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Test case sensitivity - support.set_match_tests(['filetests'], None) - self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests'], None) - self.assertTrue(support.match_test(test_access)) - - # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*'], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()], None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - support.set_match_tests(['test_access', 'DONTMATCH'], None) - self.assertTrue(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test rejection - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests(ignore_patterns=[]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the full test identifier - support.set_match_tests(None, [test_access.id()]) - self.assertFalse(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match the module name - support.set_match_tests(None, ['test_os']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test '*' pattern - support.set_match_tests(None, ['test_*']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Test case sensitivity - support.set_match_tests(None, ['filetests']) - self.assertTrue(support.match_test(test_access)) - support.set_match_tests(None, ['FileTests']) - self.assertFalse(support.match_test(test_access)) - - # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(None, ['*test_os.*.test_*']) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - # Multiple patterns - support.set_match_tests(None, [test_access.id(), test_chdir.id()]) - self.assertFalse(support.match_test(test_access)) - self.assertFalse(support.match_test(test_chdir)) - - support.set_match_tests(None, ['test_access', 'DONTMATCH']) - self.assertFalse(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - @unittest.skipIf(sys.platform.startswith("win"), "TODO: RUSTPYTHON; os.dup on windows") - @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") + @unittest.skipIf(support.is_apple_mobile, "Unstable on Apple Mobile") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") def test_fd_count(self): - # We cannot test the absolute value of fd_count(): on old Linux - # kernel or glibc versions, os.urandom() keeps a FD open on - # /dev/urandom device and Python has 4 FD opens instead of 3. - # Test is unstable on Emscripten. The platform starts and stops + # We cannot test the absolute value of fd_count(): on old Linux kernel + # or glibc versions, os.urandom() keeps a FD open on /dev/urandom + # device and Python has 4 FD opens instead of 3. Test is unstable on + # Emscripten and Apple Mobile platforms; these platforms start and stop # background threads that use pipes and epoll fds. start = os_helper.fd_count() fd = os.open(__file__, os.O_RDONLY) @@ -686,14 +614,13 @@ def test_print_warning(self): self.check_print_warning("a\nb", 'Warning -- a\nWarning -- b\n') - @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") def test_has_strftime_extensions(self): - if support.is_emscripten or sys.platform == "win32": + if sys.platform == "win32": self.assertFalse(support.has_strftime_extensions) else: self.assertTrue(support.has_strftime_extensions) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - _testinternalcapi module not available def test_get_recursion_depth(self): # test support.get_recursion_depth() code = textwrap.dedent(""" @@ -737,13 +664,15 @@ def test_recursive(depth, limit): """) script_helper.assert_python_ok("-c", code) + @unittest.skip("TODO: RUSTPYTHON; - causes segfault in debug builds") + @support.skip_if_unlimited_stack_size def test_recursion(self): # Test infinite_recursion() and get_recursion_available() functions. def recursive_function(depth): if depth: recursive_function(depth - 1) - for max_depth in (5, 25, 250): + for max_depth in (5, 25, 250, 2500): with support.infinite_recursion(max_depth): available = support.get_recursion_available() @@ -769,7 +698,107 @@ def recursive_function(depth): else: self.fail("RecursionError was not raised") - #self.assertEqual(available, 2) + def test_parse_memlimit(self): + parse = support._parse_memlimit + KiB = 1024 + MiB = KiB * 1024 + GiB = MiB * 1024 + TiB = GiB * 1024 + self.assertEqual(parse('0k'), 0) + self.assertEqual(parse('3k'), 3 * KiB) + self.assertEqual(parse('2.4m'), int(2.4 * MiB)) + self.assertEqual(parse('4g'), int(4 * GiB)) + self.assertEqual(parse('1t'), TiB) + + for limit in ('', '3', '3.5.10k', '10x'): + with self.subTest(limit=limit): + with self.assertRaises(ValueError): + parse(limit) + + def test_set_memlimit(self): + _4GiB = 4 * 1024 ** 3 + TiB = 1024 ** 4 + old_max_memuse = support.max_memuse + old_real_max_memuse = support.real_max_memuse + try: + if sys.maxsize > 2**32: + support.set_memlimit('4g') + self.assertEqual(support.max_memuse, _4GiB) + self.assertEqual(support.real_max_memuse, _4GiB) + + big = 2**100 // TiB + support.set_memlimit(f'{big}t') + self.assertEqual(support.max_memuse, sys.maxsize) + self.assertEqual(support.real_max_memuse, big * TiB) + else: + support.set_memlimit('4g') + self.assertEqual(support.max_memuse, sys.maxsize) + self.assertEqual(support.real_max_memuse, _4GiB) + finally: + support.max_memuse = old_max_memuse + support.real_max_memuse = old_real_max_memuse + + def test_copy_python_src_ignore(self): + # Get source directory + src_dir = sysconfig.get_config_var('abs_srcdir') + if not src_dir: + src_dir = sysconfig.get_config_var('srcdir') + src_dir = os.path.abspath(src_dir) + + # Check that the source code is available + if not os.path.exists(src_dir): + self.skipTest(f"cannot access Python source code directory:" + f" {src_dir!r}") + # Check that the landmark copy_python_src_ignore() expects is available + # (Previously we looked for 'Lib\os.py', which is always present on Windows.) + landmark = os.path.join(src_dir, 'Modules') + if not os.path.exists(landmark): + self.skipTest(f"cannot access Python source code directory:" + f" {landmark!r} landmark is missing") + + # Test support.copy_python_src_ignore() + + # Source code directory + ignored = {'.git', '__pycache__'} + names = os.listdir(src_dir) + self.assertEqual(support.copy_python_src_ignore(src_dir, names), + ignored | {'build'}) + + # Doc/ directory + path = os.path.join(src_dir, 'Doc') + self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), + ignored | {'build', 'venv'}) + + # Another directory + path = os.path.join(src_dir, 'Objects') + self.assertEqual(support.copy_python_src_ignore(path, os.listdir(path)), + ignored) + + def test_get_signal_name(self): + for exitcode, expected in ( + (-int(signal.SIGINT), 'SIGINT'), + (-int(signal.SIGSEGV), 'SIGSEGV'), + (128 + int(signal.SIGABRT), 'SIGABRT'), + (3221225477, "STATUS_ACCESS_VIOLATION"), + (0xC00000FD, "STATUS_STACK_OVERFLOW"), + ): + self.assertEqual(support.get_signal_name(exitcode), expected, + exitcode) + + def test_linked_to_musl(self): + linked = support.linked_to_musl() + self.assertIsNotNone(linked) + if support.is_wasm32: + self.assertTrue(linked) + # The value is cached, so make sure it returns the same value again. + self.assertIs(linked, support.linked_to_musl()) + # The musl version is either triple or just a major version number. + if linked: + self.assertIsInstance(linked, tuple) + self.assertIn(len(linked), (1, 3)) + for v in linked: + self.assertIsInstance(v, int) + # XXX -follows a list of untested API # make_legacy_pyc @@ -782,12 +811,10 @@ def recursive_function(depth): # EnvironmentVarGuard # transient_internet # run_with_locale - # set_memlimit # bigmemtest # precisionbigmemtest # bigaddrspacetest # requires_resource - # run_doctest # threading_cleanup # reap_threads # can_symlink diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ca62b9d6851..1653ab4a718 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -1,9 +1,14 @@ """ Test the API of the symtable module. """ + +import re +import textwrap import symtable import unittest +from test import support +from test.support import os_helper TEST_CODE = """ @@ -11,7 +16,7 @@ glob = 42 some_var = 12 -some_non_assigned_global_var = 11 +some_non_assigned_global_var: int some_assigned_global_var = 11 class Mine: @@ -40,6 +45,129 @@ def foo(): def namespace_test(): pass def namespace_test(): pass + +type Alias = int +type GenericAlias[T] = list[T] + +def generic_spam[T](a): + pass + +class GenericMine[T: int, U: (int, str) = int]: + pass +""" + +TEST_COMPLEX_CLASS_CODE = """ +# The following symbols are defined in ComplexClass +# without being introduced by a 'global' statement. +glob_unassigned_meth: Any +glob_unassigned_meth_pep_695: Any + +glob_unassigned_async_meth: Any +glob_unassigned_async_meth_pep_695: Any + +def glob_assigned_meth(): pass +def glob_assigned_meth_pep_695[T](): pass + +async def glob_assigned_async_meth(): pass +async def glob_assigned_async_meth_pep_695[T](): pass + +# The following symbols are defined in ComplexClass after +# being introduced by a 'global' statement (and therefore +# are not considered as local symbols of ComplexClass). +glob_unassigned_meth_ignore: Any +glob_unassigned_meth_pep_695_ignore: Any + +glob_unassigned_async_meth_ignore: Any +glob_unassigned_async_meth_pep_695_ignore: Any + +def glob_assigned_meth_ignore(): pass +def glob_assigned_meth_pep_695_ignore[T](): pass + +async def glob_assigned_async_meth_ignore(): pass +async def glob_assigned_async_meth_pep_695_ignore[T](): pass + +class ComplexClass: + a_var = 1234 + a_genexpr = (x for x in []) + a_lambda = lambda x: x + + type a_type_alias = int + type a_type_alias_pep_695[T] = list[T] + + class a_class: pass + class a_class_pep_695[T]: pass + + def a_method(self): pass + def a_method_pep_695[T](self): pass + + async def an_async_method(self): pass + async def an_async_method_pep_695[T](self): pass + + @classmethod + def a_classmethod(cls): pass + @classmethod + def a_classmethod_pep_695[T](self): pass + + @classmethod + async def an_async_classmethod(cls): pass + @classmethod + async def an_async_classmethod_pep_695[T](self): pass + + @staticmethod + def a_staticmethod(): pass + @staticmethod + def a_staticmethod_pep_695[T](self): pass + + @staticmethod + async def an_async_staticmethod(): pass + @staticmethod + async def an_async_staticmethod_pep_695[T](self): pass + + # These ones will be considered as methods because of the 'def' although + # they are *not* valid methods at runtime since they are not decorated + # with @staticmethod. + def a_fakemethod(): pass + def a_fakemethod_pep_695[T](): pass + + async def an_async_fakemethod(): pass + async def an_async_fakemethod_pep_695[T](): pass + + # Check that those are still considered as methods + # since they are not using the 'global' keyword. + def glob_unassigned_meth(): pass + def glob_unassigned_meth_pep_695[T](): pass + + async def glob_unassigned_async_meth(): pass + async def glob_unassigned_async_meth_pep_695[T](): pass + + def glob_assigned_meth(): pass + def glob_assigned_meth_pep_695[T](): pass + + async def glob_assigned_async_meth(): pass + async def glob_assigned_async_meth_pep_695[T](): pass + + # The following are not picked as local symbols because they are not + # visible by the class at runtime (this is equivalent to having the + # definitions outside of the class). + global glob_unassigned_meth_ignore + def glob_unassigned_meth_ignore(): pass + global glob_unassigned_meth_pep_695_ignore + def glob_unassigned_meth_pep_695_ignore[T](): pass + + global glob_unassigned_async_meth_ignore + async def glob_unassigned_async_meth_ignore(): pass + global glob_unassigned_async_meth_pep_695_ignore + async def glob_unassigned_async_meth_pep_695_ignore[T](): pass + + global glob_assigned_meth_ignore + def glob_assigned_meth_ignore(): pass + global glob_assigned_meth_pep_695_ignore + def glob_assigned_meth_pep_695_ignore[T](): pass + + global glob_assigned_async_meth_ignore + async def glob_assigned_async_meth_ignore(): pass + global glob_assigned_async_meth_pep_695_ignore + async def glob_assigned_async_meth_pep_695_ignore[T](): pass """ @@ -54,27 +182,57 @@ class SymtableTest(unittest.TestCase): top = symtable.symtable(TEST_CODE, "?", "exec") # These correspond to scopes in TEST_CODE Mine = find_block(top, "Mine") + a_method = find_block(Mine, "a_method") spam = find_block(top, "spam") internal = find_block(spam, "internal") other_internal = find_block(spam, "other_internal") foo = find_block(top, "foo") - + Alias = find_block(top, "Alias") + GenericAlias = find_block(top, "GenericAlias") + # XXX: RUSTPYTHON + # GenericAlias_inner = find_block(GenericAlias, "GenericAlias") + generic_spam = find_block(top, "generic_spam") + # XXX: RUSTPYTHON + # generic_spam_inner = find_block(generic_spam, "generic_spam") + GenericMine = find_block(top, "GenericMine") + # XXX: RUSTPYTHON + # GenericMine_inner = find_block(GenericMine, "GenericMine") + # XXX: RUSTPYTHON + # T = find_block(GenericMine, "T") + # XXX: RUSTPYTHON + # U = find_block(GenericMine, "U") + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_type(self): self.assertEqual(self.top.get_type(), "module") self.assertEqual(self.Mine.get_type(), "class") self.assertEqual(self.a_method.get_type(), "function") self.assertEqual(self.spam.get_type(), "function") self.assertEqual(self.internal.get_type(), "function") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(self.foo.get_type(), "function") + self.assertEqual(self.Alias.get_type(), "type alias") + self.assertEqual(self.GenericAlias.get_type(), "type parameters") + self.assertEqual(self.GenericAlias_inner.get_type(), "type alias") + self.assertEqual(self.generic_spam.get_type(), "type parameters") + self.assertEqual(self.generic_spam_inner.get_type(), "function") + self.assertEqual(self.GenericMine.get_type(), "type parameters") + self.assertEqual(self.GenericMine_inner.get_type(), "class") + self.assertEqual(self.T.get_type(), "type variable") + self.assertEqual(self.U.get_type(), "type variable") + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_id(self): self.assertGreater(self.top.get_id(), 0) self.assertGreater(self.Mine.get_id(), 0) self.assertGreater(self.a_method.get_id(), 0) self.assertGreater(self.spam.get_id(), 0) self.assertGreater(self.internal.get_id(), 0) + self.assertGreater(self.foo.get_id(), 0) + self.assertGreater(self.Alias.get_id(), 0) + self.assertGreater(self.GenericAlias.get_id(), 0) + self.assertGreater(self.generic_spam.get_id(), 0) + self.assertGreater(self.GenericMine.get_id(), 0) def test_optimized(self): self.assertFalse(self.top.is_optimized()) @@ -96,8 +254,7 @@ def test_lineno(self): self.assertEqual(self.top.get_lineno(), 0) self.assertEqual(self.spam.get_lineno(), 14) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_function_info(self): func = self.spam self.assertEqual(sorted(func.get_parameters()), ["a", "b", "kw", "var"]) @@ -106,6 +263,7 @@ def test_function_info(self): self.assertEqual(sorted(func.get_globals()), ["bar", "glob", "some_assigned_global_var"]) self.assertEqual(self.internal.get_frees(), ("x",)) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_globals(self): self.assertTrue(self.spam.lookup("glob").is_global()) self.assertFalse(self.spam.lookup("glob").is_declared_global()) @@ -118,14 +276,14 @@ def test_globals(self): self.assertTrue(self.top.lookup("some_non_assigned_global_var").is_global()) self.assertTrue(self.top.lookup("some_assigned_global_var").is_global()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonlocal(self): self.assertFalse(self.spam.lookup("some_var").is_nonlocal()) self.assertTrue(self.other_internal.lookup("some_var").is_nonlocal()) expected = ("some_var",) self.assertEqual(self.other_internal.get_nonlocals(), expected) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_local(self): self.assertTrue(self.spam.lookup("x").is_local()) self.assertFalse(self.spam.lookup("bar").is_local()) @@ -133,9 +291,11 @@ def test_local(self): self.assertTrue(self.top.lookup("some_non_assigned_global_var").is_local()) self.assertTrue(self.top.lookup("some_assigned_global_var").is_local()) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free(self): self.assertTrue(self.internal.lookup("x").is_free()) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_referenced(self): self.assertTrue(self.internal.lookup("x").is_referenced()) self.assertTrue(self.spam.lookup("internal").is_referenced()) @@ -152,8 +312,7 @@ def test_symbol_lookup(self): self.assertRaises(KeyError, self.top.lookup, "not_here") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_namespaces(self): self.assertTrue(self.top.lookup("Mine").is_namespace()) self.assertTrue(self.Mine.lookup("a_method").is_namespace()) @@ -167,6 +326,10 @@ def test_namespaces(self): self.assertEqual(len(ns_test.get_namespaces()), 2) self.assertRaises(ValueError, ns_test.get_namespace) + ns_test_2 = self.top.lookup("glob") + self.assertEqual(len(ns_test_2.get_namespaces()), 0) + self.assertRaises(ValueError, ns_test_2.get_namespace) + def test_assigned(self): self.assertTrue(self.spam.lookup("x").is_assigned()) self.assertTrue(self.spam.lookup("bar").is_assigned()) @@ -174,14 +337,17 @@ def test_assigned(self): self.assertTrue(self.Mine.lookup("a_method").is_assigned()) self.assertFalse(self.internal.lookup("x").is_assigned()) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') - st2 = st1.get_children()[0] + st2 = st1.get_children()[1] + self.assertEqual(st2.get_type(), "function") self.assertTrue(st2.lookup('x').is_local()) self.assertTrue(st2.lookup('x').is_annotated()) self.assertFalse(st2.lookup('x').is_global()) st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') - st4 = st3.get_children()[0] + st4 = st3.get_children()[1] + self.assertEqual(st4.get_type(), "function") self.assertTrue(st4.lookup('x').is_local()) self.assertFalse(st4.lookup('x').is_annotated()) @@ -199,6 +365,7 @@ def test_annotated(self): ' x: int', 'test', 'exec') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_imported(self): self.assertTrue(self.top.lookup("sys").is_imported()) @@ -208,13 +375,89 @@ def test_name(self): self.assertEqual(self.spam.lookup("x").get_name(), "x") self.assertEqual(self.Mine.get_name(), "Mine") - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_class_info(self): - self.assertEqual(self.Mine.get_methods(), ('a_method',)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_class_get_methods(self): + deprecation_mess = ( + re.escape('symtable.Class.get_methods() is deprecated ' + 'and will be removed in Python 3.16.') + ) + + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(self.Mine.get_methods(), ('a_method',)) + + top = symtable.symtable(TEST_COMPLEX_CLASS_CODE, "?", "exec") + this = find_block(top, "ComplexClass") + + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), ( + 'a_method', 'a_method_pep_695', + 'an_async_method', 'an_async_method_pep_695', + 'a_classmethod', 'a_classmethod_pep_695', + 'an_async_classmethod', 'an_async_classmethod_pep_695', + 'a_staticmethod', 'a_staticmethod_pep_695', + 'an_async_staticmethod', 'an_async_staticmethod_pep_695', + 'a_fakemethod', 'a_fakemethod_pep_695', + 'an_async_fakemethod', 'an_async_fakemethod_pep_695', + 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', + 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', + 'glob_assigned_meth', 'glob_assigned_meth_pep_695', + 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', + )) + + # Test generator expressions that are of type TYPE_FUNCTION + # but will not be reported by get_methods() since they are + # not functions per se. + # + # Other kind of comprehensions such as list, set or dict + # expressions do not have the TYPE_FUNCTION type. + + def check_body(body, expected_methods): + indented = textwrap.indent(body, ' ' * 4) + top = symtable.symtable(f"class A:\n{indented}", "?", "exec") + this = find_block(top, "A") + with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): + self.assertEqual(this.get_methods(), expected_methods) + + # statements with 'genexpr' inside it + GENEXPRS = ( + 'x = (x for x in [])', + 'x = (x async for x in [])', + 'type x[genexpr = (x for x in [])] = (x for x in [])', + 'type x[genexpr = (x async for x in [])] = (x async for x in [])', + 'genexpr = (x for x in [])', + 'genexpr = (x async for x in [])', + 'type genexpr[genexpr = (x for x in [])] = (x for x in [])', + 'type genexpr[genexpr = (x async for x in [])] = (x async for x in [])', + ) + + for gen in GENEXPRS: + # test generator expression + with self.subTest(gen=gen): + check_body(gen, ()) + + # test generator expression + variable named 'genexpr' + with self.subTest(gen=gen, isvar=True): + check_body('\n'.join((gen, 'genexpr = 1')), ()) + check_body('\n'.join(('genexpr = 1', gen)), ()) + + for paramlist in ('()', '(x)', '(x, y)', '(z: T)'): + for func in ( + f'def genexpr{paramlist}:pass', + f'async def genexpr{paramlist}:pass', + f'def genexpr[T]{paramlist}:pass', + f'async def genexpr[T]{paramlist}:pass', + ): + with self.subTest(func=func): + # test function named 'genexpr' + check_body(func, ('genexpr',)) + + for gen in GENEXPRS: + with self.subTest(gen=gen, func=func): + # test generator expression + function named 'genexpr' + check_body('\n'.join((gen, func)), ('genexpr',)) + check_body('\n'.join((func, gen)), ('genexpr',)) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_filename_correct(self): ### Bug tickler: SyntaxError file name correct whether error raised ### while parsing or building symbol table. @@ -230,10 +473,9 @@ def checkfilename(brokencode, offset): checkfilename("def f(x): foo)(", 14) # parse-time checkfilename("def f(x): global x", 11) # symtable-build-time symtable.symtable("pass", b"spam", "exec") - with self.assertWarns(DeprecationWarning), \ - self.assertRaises(TypeError): + with self.assertRaises(TypeError): symtable.symtable("pass", bytearray(b"spam"), "exec") - with self.assertWarns(DeprecationWarning): + with self.assertRaises(TypeError): symtable.symtable("pass", memoryview(b"spam"), "exec") with self.assertRaises(TypeError): symtable.symtable("pass", list(b"spam"), "exec") @@ -247,8 +489,7 @@ def test_single(self): def test_exec(self): symbols = symtable.symtable("def f(x): return x", "?", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bytes(self): top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec") self.assertIsNotNone(find_block(top, "Mine")) @@ -258,12 +499,118 @@ def test_bytes(self): top = symtable.symtable(code, "?", "exec") self.assertIsNotNone(find_block(top, "\u017d")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_symtable_repr(self): self.assertEqual(str(self.top), "<SymbolTable for module ?>") self.assertEqual(str(self.spam), "<Function SymbolTable for spam in ?>") + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_symbol_repr(self): + self.assertEqual(repr(self.spam.lookup("glob")), + "<symbol 'glob': GLOBAL_IMPLICIT, USE>") + self.assertEqual(repr(self.spam.lookup("bar")), + "<symbol 'bar': GLOBAL_EXPLICIT, DEF_GLOBAL|DEF_LOCAL>") + self.assertEqual(repr(self.spam.lookup("a")), + "<symbol 'a': LOCAL, DEF_PARAM>") + self.assertEqual(repr(self.spam.lookup("internal")), + "<symbol 'internal': LOCAL, USE|DEF_LOCAL>") + self.assertEqual(repr(self.spam.lookup("other_internal")), + "<symbol 'other_internal': LOCAL, DEF_LOCAL>") + self.assertEqual(repr(self.internal.lookup("x")), + "<symbol 'x': FREE, USE>") + self.assertEqual(repr(self.other_internal.lookup("some_var")), + "<symbol 'some_var': FREE, USE|DEF_NONLOCAL|DEF_LOCAL>") + self.assertEqual(repr(self.GenericMine.lookup("T")), + "<symbol 'T': LOCAL, DEF_LOCAL|DEF_TYPE_PARAM>") + + st1 = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(repr(st1.lookup("x")), + "<symbol 'x': LOCAL, USE|DEF_LOCAL|DEF_COMP_ITER>") + + st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") + self.assertEqual(repr(st2.lookup("x")), + "<symbol 'x': CELL, DEF_LOCAL|DEF_COMP_ITER|DEF_COMP_CELL>") + + st3 = symtable.symtable("def f():\n" + " x = 1\n" + " class A:\n" + " x = 2\n" + " def method():\n" + " return x\n", + "?", "exec") + # child 0 is for __annotate__ + func_f = st3.get_children()[1] + class_A = func_f.get_children()[0] + self.assertEqual(repr(class_A.lookup('x')), + "<symbol 'x': LOCAL, DEF_LOCAL|DEF_FREE_CLASS>") + + def test_symtable_entry_repr(self): + expected = f"<symtable entry top({self.top.get_id()}), line {self.top.get_lineno()}>" + self.assertEqual(repr(self.top._table), expected) + + def test__symtable_refleak(self): + # Regression test for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'compile_type' AC conversion failed + self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1) + + +class ComprehensionTests(unittest.TestCase): + def get_identifiers_recursive(self, st, res): + res.extend(st.get_identifiers()) + for ch in st.get_children(): + self.get_identifiers_recursive(ch, res) + + def test_loopvar_in_only_one_scope(self): + # ensure that the loop variable appears only once in the symtable + comps = [ + "[x for x in [1]]", + "{x for x in [1]}", + "{x:x*x for x in [1]}", + ] + for comp in comps: + with self.subTest(comp=comp): + st = symtable.symtable(comp, "?", "exec") + ids = [] + self.get_identifiers_recursive(st, ids) + self.assertEqual(len([x for x in ids if x == 'x']), 1) + + +class CommandLineTest(unittest.TestCase): + maxDiff = None + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_file(self): + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + with open(filename, 'w') as f: + f.write(TEST_CODE) + with support.captured_stdout() as stdout: + symtable.main([filename]) + out = stdout.getvalue() + self.assertIn('\n\n', out) + self.assertNotIn('\n\n\n', out) + lines = out.splitlines() + self.assertIn(f"symbol table for module from file {filename!r}:", lines) + self.assertIn(" local symbol 'glob': def_local", lines) + self.assertIn(" global_implicit symbol 'glob': use", lines) + self.assertIn(" local symbol 'spam': def_local", lines) + self.assertIn(" symbol table for function 'spam':", lines) + + def test_stdin(self): + with support.captured_stdin() as stdin: + stdin.write(TEST_CODE) + stdin.seek(0) + with support.captured_stdout() as stdout: + symtable.main([]) + out = stdout.getvalue() + stdin.seek(0) + with support.captured_stdout() as stdout: + symtable.main(['-']) + self.assertEqual(stdout.getvalue(), out) + lines = out.splitlines() + self.assertIn("symbol table for module from file '<stdin>':", lines) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 9726d3cc1e1..98246ac2214 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -259,6 +259,36 @@ Traceback (most recent call last): SyntaxError: invalid syntax +Comprehensions without 'in' keyword: + +>>> [x for x if range(1)] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> tuple(x for x if range(1)) +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x() in a] +Traceback (most recent call last): +SyntaxError: cannot assign to function call + +>>> [x for a, b, (c + 1, d()) in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for a, b, (c + 1, d()) if y] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x+1 in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for x+1, x() in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + Comprehensions creating tuples without parentheses should produce a specialized error message: @@ -322,6 +352,13 @@ Traceback (most recent call last): SyntaxError: invalid syntax +# But prefixes of soft keywords should +# still raise specialized errors + +>>> (mat x) +Traceback (most recent call last): +SyntaxError: invalid syntax. Perhaps you forgot a comma? + From compiler_complex_args(): >>> def f(None=1): @@ -334,7 +371,12 @@ >>> def f(x, y=1, z): ... pass Traceback (most recent call last): -SyntaxError: non-default argument follows default argument +SyntaxError: parameter without a default follows parameter with a default + +>>> def f(x, /, y=1, z): +... pass +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default >>> def f(x, None): ... pass @@ -555,8 +597,16 @@ Traceback (most recent call last): SyntaxError: expected default value expression -# TODO: RUSTPYTHON NameError: name 'PyCF_TYPE_COMMENTS' is not defined ->>> import ast; ast.parse(''' # doctest: +SKIP +>>> lambda a,d=3,c: None +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default + +>>> lambda a,/,d=3,c: None +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default + +>>> # TODO: RUSTPYTHON +>>> import ast; ast.parse(''' # doctest: +SKIP ... def f( ... *, # type: int ... a, # type: int @@ -582,46 +632,31 @@ >>> L = range(10) >>> f(x for x in L) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, 1) # doctest: +SKIP +>>> f(x for x in L, 1) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, y=1) # doctest: +SKIP +>>> f(x for x in L, y=1) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, *[]) # doctest: +SKIP +>>> f(x for x in L, *[]) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, **{}) # doctest: +SKIP +>>> f(x for x in L, **{}) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(L, x for x in L) # doctest: +SKIP +>>> f(L, x for x in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, y for y in L) # doctest: +SKIP +>>> f(x for x in L, y for y in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L,) # doctest: +SKIP +>>> f(x for x in L,) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized >>> f((x for x in L), 1) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -# TODO: RUSTPYTHON TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases ->>> class C(x for x in L): # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> class C(x for x in L): # doctest: +SKIP ... pass Traceback (most recent call last): SyntaxError: invalid syntax @@ -742,7 +777,8 @@ >>> f(x.y=1) Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? ->>> f((x)=2) +>>> # TODO: RUSTPYTHON +>>> f((x)=2) # doctest: +SKIP Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? >>> f(True=1) @@ -757,11 +793,31 @@ >>> f(__debug__=1) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ - -# TODO: RUSTPYTHON NameError: name '__annotations__' is not defined ->>> __debug__: int # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> __debug__: int # doctest: +SKIP Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> f(a=) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(a, b, c=) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(a, b, c=, d) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(*args=[0]) +Traceback (most recent call last): +SyntaxError: cannot assign to iterable argument unpacking +>>> f(a, b, *args=[0]) +Traceback (most recent call last): +SyntaxError: cannot assign to iterable argument unpacking +>>> f(**kwargs={'a': 1}) +Traceback (most recent call last): +SyntaxError: cannot assign to keyword argument unpacking +>>> f(a, b, *args, **kwargs={'a': 1}) +Traceback (most recent call last): +SyntaxError: cannot assign to keyword argument unpacking More set_context(): @@ -989,11 +1045,26 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> def f[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class A ... pass Traceback (most recent call last): SyntaxError: expected ':' + >>> class A[T] + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + + >>> class A[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class R&D: ... pass Traceback (most recent call last): @@ -1153,6 +1224,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -1431,11 +1518,21 @@ Traceback (most recent call last): IndentationError: expected an indented block after function definition on line 1 + >>> def foo[T](x, /, y, *, z=2): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after function definition on line 1 + >>> class Blech(A): ... pass Traceback (most recent call last): IndentationError: expected an indented block after class definition on line 1 + >>> class Blech[T](A): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after class definition on line 1 + >>> match something: ... pass Traceback (most recent call last): @@ -1468,14 +1565,16 @@ Check that an multiple exception types with missing parentheses raise a custom exception - >>> try: + >>> # TODO: RUSTPYTHON + >>> try: # doctest: +SKIP ... pass ... except A, B: ... pass Traceback (most recent call last): SyntaxError: multiple exception types must be parenthesized - >>> try: + >>> # TODO: RUSTPYTHON + >>> try: # doctest: +SKIP ... pass ... except A, B, C: ... pass @@ -1590,30 +1689,113 @@ Traceback (most recent call last): SyntaxError: trailing comma not allowed without surrounding parentheses +>>> import a from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a, b,c from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a,b,c from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. ->>> from t import x,y, and 3 +>>> from t import x,y, and 3 Traceback (most recent call last): SyntaxError: invalid syntax -# TODO: RUSTPYTHON nothing raised. ->>> (): int # doctest: +SKIP +>>> from i import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> from .. import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> (): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> []: int # doctest: +SKIP +>>> []: int Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> (()): int # doctest: +SKIP +>>> (()): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> ([]): int # doctest: +SKIP +>>> ([]): int Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated +# 'not' after operators: + +>>> 3 + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 * not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> ~ not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + not -1 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +# Check that we don't introduce misleading errors +>>> not 1 */ 2 +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not + 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + Corner-cases that used to fail to raise the correct error: >>> def f(*, x=lambda __debug__:0): pass @@ -1634,8 +1816,7 @@ Corner-cases that used to crash: - # TODO: RUSTPYTHON nothing raised. - >>> def f(**__debug__): pass # doctest: +SKIP + >>> def f(**__debug__): pass Traceback (most recent call last): SyntaxError: cannot assign to __debug__ @@ -1649,7 +1830,8 @@ Invalid pattern matching constructs: - >>> match ...: + >>> # TODO: RUSTPYTHON + >>> match ...: # doctest: +SKIP ... case 42 as _: ... ... Traceback (most recent call last): @@ -1749,22 +1931,22 @@ >>> A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[*(1:2)] = 1 Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> del A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[*:] and A[:*] >>> A[*:] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[:*] Traceback (most recent call last): ... @@ -1775,7 +1957,7 @@ >>> A[*] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[**] @@ -1827,10 +2009,246 @@ def f(x: *b) Traceback (most recent call last): ... SyntaxError: invalid syntax + +Invalid bytes literals: + + >>> b"Ā" + Traceback (most recent call last): + ... + b"Ā" + ^^^ + SyntaxError: bytes can only contain ASCII literal characters + + >>> b"абвгде" + Traceback (most recent call last): + ... + b"абвгде" + ^^^^^^^^ + SyntaxError: bytes can only contain ASCII literal characters + + >>> b"abc ъющый" # first 3 letters are ascii + Traceback (most recent call last): + ... + b"abc ъющый" + ^^^^^^^^^^^ + SyntaxError: bytes can only contain ASCII literal characters + +Invalid expressions in type scopes: + + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> async def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> async def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> async def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> type A[T: (x:=3)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> type A[T: ((x:= 3), int)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> type A[T = ((x:=3))] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> def f[T: (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> def f[T: (int, (yield))](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> def f[T = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> def f[*Ts = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> def f[**P = [(yield), int]](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + + >>> type A[T: (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> type A[T: (int, (yield 3))] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> type A[T = (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> type A[T: (await 3)] = int + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within a TypeVar bound + + >>> type A[T: (yield from [])] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (yield 3)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (int, (yield 3))]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> class A[T = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> class A[*Ts = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> class A[**P = [(yield), int]]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + + >>> # TODO: RUSTPYTHON + >>> type A = (x := 3) # doctest: +SKIP + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a type alias + + >>> type A = (yield 3) + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a type alias + + >>> type A = (await 3) + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within a type alias + + >>> type A = (yield from []) + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a type alias + + >>> class A[T]((x := 3)): ... + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within the definition of a generic + + >>> class A[T]((yield 3)): ... + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within the definition of a generic + + >>> class A[T]((await 3)): ... + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within the definition of a generic + + >>> class A[T]((yield from [])): ... + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within the definition of a generic + + >>> f(**x, *y) + Traceback (most recent call last): + SyntaxError: iterable argument unpacking follows keyword argument unpacking + + >>> f(**x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression """ import re import doctest +import textwrap import unittest from test import support @@ -1867,8 +2285,7 @@ def _check_error(self, code, errtext, else: self.fail("compile() did not raise SyntaxError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expression_with_assignment(self): self._check_error( "print(end1 + end2 = ' ')", @@ -1882,6 +2299,7 @@ def test_curly_brace_after_primary_raises_immediately(self): def test_assign_call(self): self._check_error("f() = 1", "assign") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assign_del(self): self._check_error("del (,)", "invalid syntax") self._check_error("del 1", "cannot delete literal") @@ -1933,9 +2351,6 @@ def error2(): """ self._check_error(source, "parameter and nonlocal", lineno=3) - def test_break_outside_loop(self): - self._check_error("break", "outside loop") - def test_yield_outside_function(self): self._check_error("if 0: yield", "outside function") self._check_error("if 0: yield\nelse: x=1", "outside function") @@ -1964,22 +2379,28 @@ def test_return_outside_function(self): "outside function") def test_break_outside_loop(self): - self._check_error("if 0: break", "outside loop") - self._check_error("if 0: break\nelse: x=1", "outside loop") - self._check_error("if 1: pass\nelse: break", "outside loop") - self._check_error("class C:\n if 0: break", "outside loop") + msg = "outside loop" + self._check_error("break", msg, lineno=1) + self._check_error("if 0: break", msg, lineno=1) + self._check_error("if 0: break\nelse: x=1", msg, lineno=1) + self._check_error("if 1: pass\nelse: break", msg, lineno=2) + self._check_error("class C:\n if 0: break", msg, lineno=2) self._check_error("class C:\n if 1: pass\n else: break", - "outside loop") + msg, lineno=3) + self._check_error("with object() as obj:\n break", + msg, lineno=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_continue_outside_loop(self): - self._check_error("if 0: continue", "not properly in loop") - self._check_error("if 0: continue\nelse: x=1", "not properly in loop") - self._check_error("if 1: pass\nelse: continue", "not properly in loop") - self._check_error("class C:\n if 0: continue", "not properly in loop") + msg = "not properly in loop" + self._check_error("if 0: continue", msg, lineno=1) + self._check_error("if 0: continue\nelse: x=1", msg, lineno=1) + self._check_error("if 1: pass\nelse: continue", msg, lineno=2) + self._check_error("class C:\n if 0: continue", msg, lineno=2) self._check_error("class C:\n if 1: pass\n else: continue", - "not properly in loop") + msg, lineno=3) + self._check_error("with object() as obj:\n continue", + msg, lineno=2) def test_unexpected_indent(self): self._check_error("foo()\n bar()\n", "unexpected indent", @@ -1994,31 +2415,41 @@ def test_bad_outdent(self): "unindent does not match .* level", subclass=IndentationError) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last(self): self._check_error("int(base=10, '2')", "positional argument follows keyword argument") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last2(self): self._check_error("int(**{'base': 10}, '2')", "positional argument follows " "keyword argument unpacking") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last3(self): self._check_error("int(**{'base': 10}, *['2'])", "iterable argument unpacking follows " "keyword argument unpacking") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_generator_in_function_call(self): self._check_error("foo(x, y for y in range(3) for z in range(2) if z , p)", "Generator expression must be parenthesized", lineno=1, end_lineno=1, offset=11, end_offset=53) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_then_except_star(self): + self._check_error("try: pass\nexcept ValueError: pass\nexcept* TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=8) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_star_then_except(self): + self._check_error("try: pass\nexcept* ValueError: pass\nexcept TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=7) + def test_empty_line_after_linecont(self): # See issue-40847 s = r"""\ @@ -2063,6 +2494,25 @@ def test_continuation_bad_indentation(self): self.assertRaises(IndentationError, exec, code) + @support.cpython_only + def test_disallowed_type_param_names(self): + # See gh-128632 + + self._check_error(f"class A[__classdict__]: pass", + f"reserved name '__classdict__' cannot be used for type parameter") + self._check_error(f"def f[__classdict__](): pass", + f"reserved name '__classdict__' cannot be used for type parameter") + self._check_error(f"type T[__classdict__] = tuple[__classdict__]", + f"reserved name '__classdict__' cannot be used for type parameter") + + # These compilations are here to make sure __class__, __classcell__ and __classdictcell__ + # don't break in the future like __classdict__ did in this case. + for name in ('__class__', '__classcell__', '__classdictcell__'): + compile(f""" +class A: + class B[{name}]: pass + """, "<testcase>", mode="exec") + @support.cpython_only def test_nested_named_except_blocks(self): code = "" @@ -2073,8 +2523,59 @@ def test_nested_named_except_blocks(self): code += f"{' '*4*12}pass" self._check_error(code, "too many statically nested blocks") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only + def test_with_statement_many_context_managers(self): + # See gh-113297 + + def get_code(n): + code = textwrap.dedent(""" + def bug(): + with ( + a + """) + for i in range(n): + code += f" as a{i}, a\n" + code += "): yield a" + return code + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "<string>", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "<string>", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_barry_as_flufl_with_syntax_errors(self): # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if # is reading the wrong token in the presence of syntax errors later @@ -2092,6 +2593,7 @@ def func2(): """ self._check_error(code, "expected ':'") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_line_continuation_error_position(self): self._check_error(r"a = 3 \ 4", "unexpected character after line continuation character", @@ -2103,6 +2605,7 @@ def test_invalid_line_continuation_error_position(self): "unexpected character after line continuation character", lineno=3, offset=4) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_line_continuation_left_recursive(self): # Check bpo-42218: SyntaxErrors following left-recursive rules # (t_primary_raw in this case) need to be tested explicitly @@ -2111,8 +2614,7 @@ def test_invalid_line_continuation_left_recursive(self): self._check_error("A.\u03bc\\\n", "unexpected EOF while parsing") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_parenthesis(self): for paren in "([{": self._check_error(paren + "1 + 2", f"\\{paren}' was never closed") @@ -2123,10 +2625,39 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") - # TODO: RUSTPYTHON - @unittest.expectedFailure + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + self._check_error("match y:\n case e(e=v,v,", " was never closed") + + # Examples with dencodings + s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' + self._check_error(s, r"'\(' was never closed") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_error_string_literal(self): + + self._check_error("'blech", r"unterminated string literal \(.*\)$") + self._check_error('"blech', r"unterminated string literal \(.*\)$") + self._check_error( + r'"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error( + r'r"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") + self._check_error(b"with(0,,):\n\x01", "invalid non-printable character") def test_match_call_does_not_raise_syntax_error(self): code = """ @@ -2146,6 +2677,7 @@ def case(x): """ compile(code, "<string>", "exec") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiline_compiler_error_points_to_the_end(self): self._check_error( "call(\na=1,\na=1\n)", @@ -2184,7 +2716,8 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") @@ -2193,7 +2726,7 @@ def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" for mode in ["exec", "eval", "single"]: with self.subTest(mode=mode): - with self.assertRaises(MemoryError): + with self.assertRaisesRegex(MemoryError, r"too complex"): compile(source, "<string>", mode) @support.cpython_only diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 82c42006aba..89987aeba6e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1,27 +1,41 @@ import builtins import codecs import gc +import io import locale import operator import os +import random +import socket import struct import subprocess import sys import sysconfig import test.support +from io import StringIO +from unittest import mock from test import support from test.support import os_helper from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.socket_helper import find_unused_port from test.support import threading_helper from test.support import import_helper +from test.support import force_not_colorized +from test.support import SHORT_TIMEOUT +try: + from concurrent import interpreters +except ImportError: + interpreters = None import textwrap import unittest import warnings -# count the number of test runs, used to create unique -# strings to intern in test_intern() -INTERN_NUMRUNS = 0 +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(interpreters is None, + 'subinterpreters required')(meth) + DICT_KEY_STRUCT_FORMAT = 'n2BI2n' @@ -42,7 +56,7 @@ def test_original_displayhook(self): dh(None) self.assertEqual(out.getvalue(), "") - self.assertTrue(not hasattr(builtins, "_")) + self.assertNotHasAttr(builtins, "_") # sys.displayhook() requires arguments self.assertRaises(TypeError, dh) @@ -71,12 +85,22 @@ def baddisplayhook(obj): code = compile("42", "<string>", "single") self.assertRaises(ValueError, eval, code) + def test_gh130163(self): + class X: + def __repr__(self): + sys.stdout = io.StringIO() + support.gc_collect() + return 'foo' + + with support.swap_attr(sys, 'stdout', None): + sys.stdout = io.StringIO() # the only reference + sys.displayhook(X()) # should not crash + + class ActiveExceptionTests(unittest.TestCase): def test_exc_info_no_exception(self): self.assertEqual(sys.exc_info(), (None, None, None)) - # TODO: RUSTPYTHON; AttributeError: module 'sys' has no attribute 'exception' - @unittest.expectedFailure def test_sys_exception_no_exception(self): self.assertEqual(sys.exception(), None) @@ -110,8 +134,6 @@ def f(): self.assertIs(exc_info[1], e) self.assertIs(exc_info[2], e.__traceback__) - # TODO: RUSTPYTHON; AttributeError: module 'sys' has no attribute 'exception' - @unittest.expectedFailure def test_sys_exception_with_exception_instance(self): def f(): raise ValueError(42) @@ -125,8 +147,6 @@ def f(): self.assertIsInstance(e, ValueError) self.assertIs(exc, e) - # TODO: RUSTPYTHON; AttributeError: module 'sys' has no attribute 'exception' - @unittest.expectedFailure def test_sys_exception_with_exception_type(self): def f(): raise ValueError @@ -143,6 +163,7 @@ def f(): class ExceptHookTest(unittest.TestCase): + @force_not_colorized def test_original_excepthook(self): try: raise ValueError(42) @@ -150,12 +171,11 @@ def test_original_excepthook(self): with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) - self.assertTrue(err.getvalue().endswith("ValueError: 42\n")) + self.assertEndsWith(err.getvalue(), "ValueError: 42\n") self.assertRaises(TypeError, sys.__excepthook__) - # TODO: RUSTPYTHON, SyntaxError formatting in arbitrary tracebacks - @unittest.expectedFailure + @force_not_colorized def test_excepthook_bytes_filename(self): # bpo-37467: sys.excepthook() must not crash if a filename # is a bytes string @@ -171,13 +191,12 @@ def test_excepthook_bytes_filename(self): err = err.getvalue() self.assertIn(""" File "b'bytes_filename'", line 123\n""", err) self.assertIn(""" text\n""", err) - self.assertTrue(err.endswith("SyntaxError: msg\n")) + self.assertEndsWith(err, "SyntaxError: msg\n") - # TODO: RUSTPYTHON, print argument error to stderr in sys.excepthook instead of throwing - @unittest.expectedFailure def test_excepthook(self): with test.support.captured_output("stderr") as stderr: - sys.excepthook(1, '1', 1) + with test.support.catch_unraisable_exception(): + sys.excepthook(1, '1', 1) self.assertTrue("TypeError: print_exception(): Exception expected for " \ "value, str found" in stderr.getvalue()) @@ -190,6 +209,7 @@ class SysModuleTest(unittest.TestCase): def tearDown(self): test.support.reap_children() + @unittest.expectedFailure # TODO: RUSTPYTHON; latin-1 codec not registered def test_exit(self): # call with two arguments self.assertRaises(TypeError, sys.exit, 42, 42) @@ -204,6 +224,20 @@ def test_exit(self): self.assertEqual(out, b'') self.assertEqual(err, b'') + # gh-125842: Windows uses 32-bit unsigned integers for exit codes + # so a -1 exit code is sometimes interpreted as 0xffff_ffff. + rc, out, err = assert_python_failure('-c', 'import sys; sys.exit(0xffff_ffff)') + self.assertIn(rc, (-1, 0xff, 0xffff_ffff)) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + # Overflow results in a -1 exit code, which may be converted to 0xff + # or 0xffff_ffff. + rc, out, err = assert_python_failure('-c', 'import sys; sys.exit(2**128)') + self.assertIn(rc, (-1, 0xff, 0xffff_ffff)) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + # call with integer argument with self.assertRaises(SystemExit) as cm: sys.exit(42) @@ -235,8 +269,7 @@ def check_exit_message(code, expected, **env_vars): rc, out, err = assert_python_failure('-c', code, **env_vars) self.assertEqual(rc, 1) self.assertEqual(out, b'') - self.assertTrue(err.startswith(expected), - "%s doesn't start with %s" % (ascii(err), ascii(expected))) + self.assertStartsWith(err, expected) # test that stderr buffer is flushed before the exit message is written # into stderr @@ -246,17 +279,36 @@ def check_exit_message(code, expected, **env_vars): # test that the exit message is written with backslashreplace error # handler to stderr - # TODO: RUSTPYTHON; allow surrogates in strings - # check_exit_message( - # r'import sys; sys.exit("surrogates:\uDCFF")', - # b"surrogates:\\udcff") + check_exit_message( + r'import sys; sys.exit("surrogates:\uDCFF")', + b"surrogates:\\udcff") # test that the unicode message is encoded to the stderr encoding # instead of the default encoding (utf8) - # TODO: RUSTPYTHON; handle PYTHONIOENCODING - # check_exit_message( - # r'import sys; sys.exit("h\xe9")', - # b"h\xe9", PYTHONIOENCODING='latin-1') + check_exit_message( + r'import sys; sys.exit("h\xe9")', + b"h\xe9", PYTHONIOENCODING='latin-1') + + @support.requires_subprocess() + def test_exit_codes_under_repl(self): + # GH-129900: SystemExit, or things that raised it, didn't + # get their return code propagated by the REPL + import tempfile + + exit_ways = [ + "exit", + "__import__('sys').exit", + "raise SystemExit" + ] + + for exitfunc in exit_ways: + for return_code in (0, 123): + with self.subTest(exitfunc=exitfunc, return_code=return_code): + with tempfile.TemporaryFile("w+") as stdin: + stdin.write(f"{exitfunc}({return_code})\n") + stdin.seek(0) + proc = subprocess.run([sys.executable], stdin=stdin) + self.assertEqual(proc.returncode, return_code) def test_getdefaultencoding(self): self.assertRaises(TypeError, sys.getdefaultencoding, 42) @@ -266,8 +318,6 @@ def test_getdefaultencoding(self): # testing sys.settrace() is done in test_sys_settrace.py # testing sys.setprofile() is done in test_sys_setprofile.py - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute 'setswitchinterval' - @unittest.expectedFailure def test_switchinterval(self): self.assertRaises(TypeError, sys.setswitchinterval) self.assertRaises(TypeError, sys.setswitchinterval, "a") @@ -283,21 +333,30 @@ def test_switchinterval(self): finally: sys.setswitchinterval(orig) - def test_recursionlimit(self): + def test_getrecursionlimit(self): + limit = sys.getrecursionlimit() + self.assertIsInstance(limit, int) + self.assertGreater(limit, 1) + self.assertRaises(TypeError, sys.getrecursionlimit, 42) - oldlimit = sys.getrecursionlimit() - self.assertRaises(TypeError, sys.setrecursionlimit) - self.assertRaises(ValueError, sys.setrecursionlimit, -42) - sys.setrecursionlimit(10000) - self.assertEqual(sys.getrecursionlimit(), 10000) - sys.setrecursionlimit(oldlimit) - - @unittest.skipIf(getattr(sys, "_rustpython_debugbuild", False), "TODO: RUSTPYTHON, stack overflow on debug build") + + def test_setrecursionlimit(self): + old_limit = sys.getrecursionlimit() + try: + sys.setrecursionlimit(10_005) + self.assertEqual(sys.getrecursionlimit(), 10_005) + + self.assertRaises(TypeError, sys.setrecursionlimit) + self.assertRaises(ValueError, sys.setrecursionlimit, -42) + finally: + sys.setrecursionlimit(old_limit) + + @unittest.skipIf(getattr(sys, "_rustpython_debugbuild", False), "TODO: RUSTPYTHON; stack overflow on debug build") def test_recursionlimit_recovery(self): if hasattr(sys, 'gettrace') and sys.gettrace(): self.skipTest('fatal error if run with a trace function') - oldlimit = sys.getrecursionlimit() + old_limit = sys.getrecursionlimit() def f(): f() try: @@ -316,38 +375,32 @@ def f(): with self.assertRaises(RecursionError): f() finally: - sys.setrecursionlimit(oldlimit) + sys.setrecursionlimit(old_limit) @test.support.cpython_only - def test_setrecursionlimit_recursion_depth(self): + def test_setrecursionlimit_to_depth(self): # Issue #25274: Setting a low recursion limit must be blocked if the # current recursion depth is already higher than limit. - from _testinternalcapi import get_recursion_depth - - def set_recursion_limit_at_depth(depth, limit): - recursion_depth = get_recursion_depth() - if recursion_depth >= depth: - with self.assertRaises(RecursionError) as cm: - sys.setrecursionlimit(limit) - self.assertRegex(str(cm.exception), - "cannot set the recursion limit to [0-9]+ " - "at the recursion depth [0-9]+: " - "the limit is too low") - else: - set_recursion_limit_at_depth(depth, limit) - - oldlimit = sys.getrecursionlimit() + old_limit = sys.getrecursionlimit() try: - sys.setrecursionlimit(1000) - - for limit in (10, 25, 50, 75, 100, 150, 200): - set_recursion_limit_at_depth(limit, limit) + depth = support.get_recursion_depth() + with self.subTest(limit=sys.getrecursionlimit(), depth=depth): + # depth + 1 is OK + sys.setrecursionlimit(depth + 1) + + # reset the limit to be able to call self.assertRaises() + # context manager + sys.setrecursionlimit(old_limit) + with self.assertRaises(RecursionError) as cm: + sys.setrecursionlimit(depth) + self.assertRegex(str(cm.exception), + "cannot set the recursion limit to [0-9]+ " + "at the recursion depth [0-9]+: " + "the limit is too low") finally: - sys.setrecursionlimit(oldlimit) + sys.setrecursionlimit(old_limit) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getwindowsversion(self): # Raise SkipTest if sys doesn't have getwindowsversion attribute test.support.get_attribute(sys, "getwindowsversion") @@ -378,15 +431,13 @@ def test_getwindowsversion(self): # still has 5 elements maj, min, buildno, plat, csd = sys.getwindowsversion() - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute 'call_tracing' - @unittest.expectedFailure def test_call_tracing(self): self.assertRaises(TypeError, sys.call_tracing, type, 2) @unittest.skipUnless(hasattr(sys, "setdlopenflags"), 'test needs sys.setdlopenflags()') def test_dlopenflags(self): - self.assertTrue(hasattr(sys, "getdlopenflags")) + self.assertHasAttr(sys, "getdlopenflags") self.assertRaises(TypeError, sys.getdlopenflags, 42) oldflags = sys.getdlopenflags() self.assertRaises(TypeError, sys.setdlopenflags) @@ -396,15 +447,21 @@ def test_dlopenflags(self): @test.support.refcount_test def test_refcount(self): - # n here must be a global in order for this test to pass while - # tracing with a python function. Tracing calls PyFrame_FastToLocals - # which will add a copy of any locals to the frame object, causing - # the reference count to increase by 2 instead of 1. + # n here originally had to be a global in order for this test to pass + # while tracing with a python function. Tracing used to call + # PyFrame_FastToLocals, which would add a copy of any locals to the + # frame object, causing the ref count to increase by 2 instead of 1. + # While that no longer happens (due to PEP 667), this test case retains + # its original global-based implementation + # PEP 683's immortal objects also made this point moot, since the + # refcount for None doesn't change anyway. Maybe this test should be + # using a different constant value? (e.g. an integer) global n self.assertRaises(TypeError, sys.getrefcount) c = sys.getrefcount(None) n = None - self.assertEqual(sys.getrefcount(None), c+1) + # Singleton refcnts don't change + self.assertEqual(sys.getrefcount(None), c) del n self.assertEqual(sys.getrefcount(None), c) if hasattr(sys, "gettotalrefcount"): @@ -418,9 +475,27 @@ def test_getframe(self): is sys._getframe().f_code ) + def test_getframemodulename(self): + # Default depth gets ourselves + self.assertEqual(__name__, sys._getframemodulename()) + self.assertEqual("unittest.case", sys._getframemodulename(1)) + i = 0 + f = sys._getframe(i) + while f: + self.assertEqual( + f.f_globals['__name__'], + sys._getframemodulename(i) or '__main__' + ) + i += 1 + f2 = f.f_back + try: + f = sys._getframe(i) + except ValueError: + break + self.assertIs(f, f2) + self.assertIsNone(sys._getframemodulename(i)) + # sys._current_frames() is a CPython-only gimmick. - # XXX RUSTPYTHON: above comment is from original cpython test; not sure why the cpython_only decorator wasn't added - @test.support.cpython_only @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_current_frames(self): @@ -446,49 +521,48 @@ def g456(): t.start() entered_g.wait() - # At this point, t has finished its entered_g.set(), although it's - # impossible to guess whether it's still on that line or has moved on - # to its leave_g.wait(). - self.assertEqual(len(thread_info), 1) - thread_id = thread_info[0] - - d = sys._current_frames() - for tid in d: - self.assertIsInstance(tid, int) - self.assertGreater(tid, 0) - - main_id = threading.get_ident() - self.assertIn(main_id, d) - self.assertIn(thread_id, d) - - # Verify that the captured main-thread frame is _this_ frame. - frame = d.pop(main_id) - self.assertTrue(frame is sys._getframe()) - - # Verify that the captured thread frame is blocked in g456, called - # from f123. This is a little tricky, since various bits of - # threading.py are also in the thread's call stack. - frame = d.pop(thread_id) - stack = traceback.extract_stack(frame) - for i, (filename, lineno, funcname, sourceline) in enumerate(stack): - if funcname == "f123": - break - else: - self.fail("didn't find f123() on thread's call stack") - - self.assertEqual(sourceline, "g456()") + try: + # At this point, t has finished its entered_g.set(), although it's + # impossible to guess whether it's still on that line or has moved on + # to its leave_g.wait(). + self.assertEqual(len(thread_info), 1) + thread_id = thread_info[0] + + d = sys._current_frames() + for tid in d: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + main_id = threading.get_ident() + self.assertIn(main_id, d) + self.assertIn(thread_id, d) + + # Verify that the captured main-thread frame is _this_ frame. + frame = d.pop(main_id) + self.assertTrue(frame is sys._getframe()) + + # Verify that the captured thread frame is blocked in g456, called + # from f123. This is a little tricky, since various bits of + # threading.py are also in the thread's call stack. + frame = d.pop(thread_id) + stack = traceback.extract_stack(frame) + for i, (filename, lineno, funcname, sourceline) in enumerate(stack): + if funcname == "f123": + break + else: + self.fail("didn't find f123() on thread's call stack") - # And the next record must be for g456(). - filename, lineno, funcname, sourceline = stack[i+1] - self.assertEqual(funcname, "g456") - self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"]) + self.assertEqual(sourceline, "g456()") - # Reap the spawned thread. - leave_g.set() - t.join() + # And the next record must be for g456(). + filename, lineno, funcname, sourceline = stack[i+1] + self.assertEqual(funcname, "g456") + self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"]) + finally: + # Reap the spawned thread. + leave_g.set() + t.join() - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute '_current_exceptions' - @unittest.expectedFailure @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_current_exceptions(self): @@ -498,7 +572,7 @@ def test_current_exceptions(self): # Spawn a thread that blocks at a known place. Then the main # thread does sys._current_frames(), and verifies that the frames # returned make sense. - entered_g = threading.Event() + g_raised = threading.Event() leave_g = threading.Event() thread_info = [] # the thread's id @@ -507,55 +581,53 @@ def f123(): def g456(): thread_info.append(threading.get_ident()) - entered_g.set() while True: try: raise ValueError("oops") except ValueError: + g_raised.set() if leave_g.wait(timeout=support.LONG_TIMEOUT): break t = threading.Thread(target=f123) t.start() - entered_g.wait() - - # At this point, t has finished its entered_g.set(), although it's - # impossible to guess whether it's still on that line or has moved on - # to its leave_g.wait(). - self.assertEqual(len(thread_info), 1) - thread_id = thread_info[0] - - d = sys._current_exceptions() - for tid in d: - self.assertIsInstance(tid, int) - self.assertGreater(tid, 0) - - main_id = threading.get_ident() - self.assertIn(main_id, d) - self.assertIn(thread_id, d) - self.assertEqual((None, None, None), d.pop(main_id)) - - # Verify that the captured thread frame is blocked in g456, called - # from f123. This is a little tricky, since various bits of - # threading.py are also in the thread's call stack. - exc_type, exc_value, exc_tb = d.pop(thread_id) - stack = traceback.extract_stack(exc_tb.tb_frame) - for i, (filename, lineno, funcname, sourceline) in enumerate(stack): - if funcname == "f123": - break - else: - self.fail("didn't find f123() on thread's call stack") + g_raised.wait(timeout=support.LONG_TIMEOUT) - self.assertEqual(sourceline, "g456()") + try: + self.assertEqual(len(thread_info), 1) + thread_id = thread_info[0] + + d = sys._current_exceptions() + for tid in d: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + main_id = threading.get_ident() + self.assertIn(main_id, d) + self.assertIn(thread_id, d) + self.assertEqual(None, d.pop(main_id)) + + # Verify that the captured thread frame is blocked in g456, called + # from f123. This is a little tricky, since various bits of + # threading.py are also in the thread's call stack. + exc_value = d.pop(thread_id) + stack = traceback.extract_stack(exc_value.__traceback__.tb_frame) + for i, (filename, lineno, funcname, sourceline) in enumerate(stack): + if funcname == "f123": + break + else: + self.fail("didn't find f123() on thread's call stack") - # And the next record must be for g456(). - filename, lineno, funcname, sourceline = stack[i+1] - self.assertEqual(funcname, "g456") - self.assertTrue(sourceline.startswith("if leave_g.wait(")) + self.assertEqual(sourceline, "g456()") - # Reap the spawned thread. - leave_g.set() - t.join() + # And the next record must be for g456(). + filename, lineno, funcname, sourceline = stack[i+1] + self.assertEqual(funcname, "g456") + self.assertStartsWith(sourceline, ("if leave_g.wait(", "g_raised.set()")) + finally: + # Reap the spawned thread. + leave_g.set() + t.join() def test_attributes(self): self.assertIsInstance(sys.api_version, int) @@ -651,13 +723,15 @@ def test_attributes(self): self.assertIn(sys.float_repr_style, ('short', 'legacy')) if not sys.platform.startswith('win'): self.assertIsInstance(sys.abiflags, str) + else: + self.assertFalse(hasattr(sys, 'abiflags')) def test_thread_info(self): info = sys.thread_info self.assertEqual(len(info), 3) self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) - if sys.platform.startswith(("linux", "freebsd")): + if sys.platform.startswith(("linux", "android", "freebsd")): self.assertEqual(info.name, "pthread") elif sys.platform == "win32": self.assertEqual(info.name, "nt") @@ -680,13 +754,23 @@ def test_43581(self): self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding) def test_intern(self): - global INTERN_NUMRUNS - INTERN_NUMRUNS += 1 + has_is_interned = (test.support.check_impl_detail(cpython=True) + or hasattr(sys, '_is_interned')) self.assertRaises(TypeError, sys.intern) - s = "never interned before" + str(INTERN_NUMRUNS) + self.assertRaises(TypeError, sys.intern, b'abc') + if has_is_interned: + self.assertRaises(TypeError, sys._is_interned) + self.assertRaises(TypeError, sys._is_interned, b'abc') + s = "never interned before" + str(random.randrange(0, 10**9)) self.assertTrue(sys.intern(s) is s) + if has_is_interned: + self.assertIs(sys._is_interned(s), True) s2 = s.swapcase().swapcase() + if has_is_interned: + self.assertIs(sys._is_interned(s2), False) self.assertTrue(sys.intern(s2) is s) + if has_is_interned: + self.assertIs(sys._is_interned(s2), False) # Subclasses of string can't be interned, because they # provide too much opportunity for insane things to happen. @@ -698,6 +782,73 @@ def __hash__(self): return 123 self.assertRaises(TypeError, sys.intern, S("abc")) + if has_is_interned: + self.assertIs(sys._is_interned(S("abc")), False) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_dynamically_allocated(self): + # Implementation detail: Dynamically allocated strings + # are distinct between interpreters + s = "never interned before" + str(random.randrange(0, 10**9)) + t = sys.intern(s) + self.assertIs(t, s) + + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + + assert id(t) != {id(s)}, (id(t), {id(s)}) + assert id(t) != {id(t)}, (id(t), {id(t)}) + ''')) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_statically_allocated(self): + # Implementation detail: Statically allocated strings are shared + # between interpreters. + # See Tools/build/generate_global_objects.py for the list + # of strings that are always statically allocated. + for s in ('__init__', 'CANCELLED', '<module>', 'utf-8', + '{{', '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff', + ): + with self.subTest(s=s): + t = sys.intern(s) + + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + t = sys.intern(s) + assert id(t) == {id(t)}, (id(t), {id(t)}) + ''')) + + @support.cpython_only + @requires_subinterpreters + def test_subinterp_intern_singleton(self): + # Implementation detail: singletons are used for 0- and 1-character + # latin1 strings. + for s in '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff': + with self.subTest(s=s): + interp = interpreters.create() + interp.exec(textwrap.dedent(f''' + import sys + + # set `s`, avoid parser interning & constant folding + s = str({s.encode()!r}, 'utf-8') + + assert id(s) == {id(s)} + t = sys.intern(s) + ''')) + self.assertTrue(sys._is_interned(s)) def test_sys_flags(self): self.assertTrue(sys.flags) @@ -708,7 +859,7 @@ def test_sys_flags(self): "hash_randomization", "isolated", "dev_mode", "utf8_mode", "warn_default_encoding", "safe_path", "int_max_str_digits") for attr in attrs: - self.assertTrue(hasattr(sys.flags, attr), attr) + self.assertHasAttr(sys.flags, attr) attr_type = bool if attr in ("dev_mode", "safe_path") else int self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) self.assertTrue(repr(sys.flags)) @@ -719,12 +870,7 @@ def test_sys_flags(self): def assert_raise_on_new_sys_type(self, sys_attr): # Users are intentionally prevented from creating new instances of # sys.flags, sys.version_info, and sys.getwindowsversion. - arg = sys_attr - attr_type = type(sys_attr) - with self.assertRaises(TypeError): - attr_type(arg) - with self.assertRaises(TypeError): - attr_type.__new__(attr_type, arg) + support.check_disallow_instantiation(self, type(sys_attr), sys_attr) def test_sys_flags_no_instantiation(self): self.assert_raise_on_new_sys_type(sys.flags) @@ -732,8 +878,7 @@ def test_sys_flags_no_instantiation(self): def test_sys_version_info_no_instantiation(self): self.assert_raise_on_new_sys_type(sys.version_info) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError not raised for getwindowsversion instantiation def test_sys_getwindowsversion_no_instantiation(self): # Skip if not being run on Windows. test.support.get_attribute(sys, "getwindowsversion") @@ -741,10 +886,12 @@ def test_sys_getwindowsversion_no_instantiation(self): @test.support.cpython_only def test_clear_type_cache(self): - sys._clear_type_cache() + with self.assertWarnsRegex(DeprecationWarning, + r"sys\._clear_type_cache\(\) is deprecated.*"): + sys._clear_type_cache() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skip("TODO: RUSTPYTHON; cp424 encoding not supported, causes panic") + @force_not_colorized @support.requires_subprocess() def test_ioencoding(self): env = dict(os.environ) @@ -908,14 +1055,12 @@ def check_locale_surrogateescape(self, locale): 'stdout: surrogateescape\n' 'stderr: backslashreplace\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; iso8859_1 codec not registered @support.requires_subprocess() def test_c_locale_surrogateescape(self): self.check_locale_surrogateescape('C') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; iso8859_1 codec not registered @support.requires_subprocess() def test_posix_locale_surrogateescape(self): self.check_locale_surrogateescape('POSIX') @@ -925,10 +1070,11 @@ def test_implementation(self): levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF} - self.assertTrue(hasattr(sys.implementation, 'name')) - self.assertTrue(hasattr(sys.implementation, 'version')) - self.assertTrue(hasattr(sys.implementation, 'hexversion')) - self.assertTrue(hasattr(sys.implementation, 'cache_tag')) + self.assertHasAttr(sys.implementation, 'name') + self.assertHasAttr(sys.implementation, 'version') + self.assertHasAttr(sys.implementation, 'hexversion') + self.assertHasAttr(sys.implementation, 'cache_tag') + self.assertHasAttr(sys.implementation, 'supports_isolated_interpreters') version = sys.implementation.version self.assertEqual(version[:2], (version.major, version.minor)) @@ -942,6 +1088,15 @@ def test_implementation(self): self.assertEqual(sys.implementation.name, sys.implementation.name.lower()) + # https://peps.python.org/pep-0734 + sii = sys.implementation.supports_isolated_interpreters + self.assertIsInstance(sii, bool) + if test.support.check_impl_detail(cpython=True): + if test.support.is_emscripten or test.support.is_wasi: + self.assertFalse(sii) + else: + self.assertTrue(sii) + @test.support.cpython_only def test_debugmallocstats(self): # Test sys._debugmallocstats() @@ -952,14 +1107,10 @@ def test_debugmallocstats(self): # Output of sys._debugmallocstats() depends on configure flags. # The sysconfig vars are not available on Windows. if sys.platform != "win32": - with_freelists = sysconfig.get_config_var("WITH_FREELISTS") with_pymalloc = sysconfig.get_config_var("WITH_PYMALLOC") - if with_freelists: - self.assertIn(b"free PyDictObjects", err) + self.assertIn(b"free PyDictObjects", err) if with_pymalloc: self.assertIn(b'Small block threshold', err) - if not with_freelists and not with_pymalloc: - self.assertFalse(err) # The function has no parameter self.assertRaises(TypeError, sys._debugmallocstats, True) @@ -968,12 +1119,12 @@ def test_debugmallocstats(self): "sys.getallocatedblocks unavailable on this build") def test_getallocatedblocks(self): try: - import _testcapi + import _testinternalcapi except ImportError: with_pymalloc = support.with_pymalloc() else: try: - alloc_name = _testcapi.pymem_getallocatorsname() + alloc_name = _testinternalcapi.pymem_getallocatorsname() except RuntimeError as exc: # "cannot get allocators name" (ex: tracemalloc is used) with_pymalloc = True @@ -990,23 +1141,29 @@ def test_getallocatedblocks(self): # about the underlying implementation: the function might # return 0 or something greater. self.assertGreaterEqual(a, 0) + gc.collect() + b = sys.getallocatedblocks() + self.assertLessEqual(b, a) try: - # While we could imagine a Python session where the number of - # multiple buffer objects would exceed the sharing of references, - # it is unlikely to happen in a normal test run. - self.assertLess(a, sys.gettotalrefcount()) + # The reported blocks will include immortalized strings, but the + # total ref count will not. This will sanity check that among all + # other objects (those eligible for garbage collection) there + # are more references being tracked than allocated blocks. + interned_immortal = sys.getunicodeinternedsize(_only_immortal=True) + self.assertLess(a - interned_immortal, sys.gettotalrefcount()) except AttributeError: # gettotalrefcount() not available pass gc.collect() - b = sys.getallocatedblocks() - self.assertLessEqual(b, a) - gc.collect() c = sys.getallocatedblocks() self.assertIn(c, range(b - 50, b + 50)) - # TODO: RUSTPYTHON, AtExit.__del__ is not invoked because module destruction is missing. - @unittest.expectedFailure + def test_is_gil_enabled(self): + if support.Py_GIL_DISABLED: + self.assertIs(type(sys._is_gil_enabled()), bool) + else: + self.assertTrue(sys._is_gil_enabled()) + def test_is_finalizing(self): self.assertIs(sys.is_finalizing(), False) # Don't use the atexit module because _Py_Finalizing is only set @@ -1028,8 +1185,6 @@ def __del__(self): rc, stdout, stderr = assert_python_ok('-c', code) self.assertEqual(stdout.rstrip(), b'True') - # TODO: RUSTPYTHON, IndexError: list index out of range - @unittest.expectedFailure def test_issue20602(self): # sys.flags and sys.float_info were wiped during shutdown. code = """if 1: @@ -1062,15 +1217,14 @@ def __del__(self): self.assertEqual(stdout.rstrip(), b"") self.assertEqual(stderr.rstrip(), b"") - @unittest.skipUnless(hasattr(sys, 'getandroidapilevel'), - 'need sys.getandroidapilevel()') + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'sys' has no attribute 'getandroidapilevel' + @unittest.skipUnless(sys.platform == "android", "Android only") def test_getandroidapilevel(self): level = sys.getandroidapilevel() self.assertIsInstance(level, int) self.assertGreater(level, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @force_not_colorized @support.requires_subprocess() def test_sys_tracebacklimit(self): code = """if 1: @@ -1091,14 +1245,20 @@ def check(tracebacklimit, expected): traceback = [ b'Traceback (most recent call last):', b' File "<string>", line 8, in <module>', + b' f2()', + b' ~~^^', b' File "<string>", line 6, in f2', + b' f1()', + b' ~~^^', b' File "<string>", line 4, in f1', + b' 1 / 0', + b' ~~^~~', b'ZeroDivisionError: division by zero' ] check(10, traceback) check(3, traceback) - check(2, traceback[:1] + traceback[2:]) - check(1, traceback[:1] + traceback[3:]) + check(2, traceback[:1] + traceback[4:]) + check(1, traceback[:1] + traceback[7:]) check(0, [traceback[-1]]) check(-1, [traceback[-1]]) check(1<<1000, traceback) @@ -1134,15 +1294,11 @@ def test_orig_argv(self): self.assertEqual(proc.stdout.rstrip().splitlines(), expected, proc) - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute 'stdlib_module_names' - @unittest.expectedFailure def test_module_names(self): self.assertIsInstance(sys.stdlib_module_names, frozenset) for name in sys.stdlib_module_names: self.assertIsInstance(name, str) - # TODO: RUSTPYTHON, AttributeError: module 'sys' has no attribute '_stdlib_dir' - @unittest.expectedFailure def test_stdlib_dir(self): os = import_helper.import_fresh_module('os') marker = getattr(os, '__file__', None) @@ -1152,41 +1308,76 @@ def test_stdlib_dir(self): self.assertEqual(os.path.normpath(sys._stdlib_dir), os.path.normpath(expected)) + @unittest.skipUnless(hasattr(sys, 'getobjects'), 'need sys.getobjects()') + def test_getobjects(self): + # sys.getobjects(0) + all_objects = sys.getobjects(0) + self.assertIsInstance(all_objects, list) + self.assertGreater(len(all_objects), 0) + + # sys.getobjects(0, MyType) + class MyType: + pass + size = 100 + my_objects = [MyType() for _ in range(size)] + get_objects = sys.getobjects(0, MyType) + self.assertEqual(len(get_objects), size) + for obj in get_objects: + self.assertIsInstance(obj, MyType) + + # sys.getobjects(3, MyType) + get_objects = sys.getobjects(3, MyType) + self.assertEqual(len(get_objects), 3) + + @unittest.skipUnless(hasattr(sys, '_stats_on'), 'need Py_STATS build') + def test_pystats(self): + # Call the functions, just check that they don't crash + # Cannot save/restore state. + sys._stats_on() + sys._stats_off() + sys._stats_clear() + sys._stats_dump() + + @test.support.cpython_only + @unittest.skipUnless(hasattr(sys, 'abiflags'), 'need sys.abiflags') + def test_disable_gil_abi(self): + self.assertEqual('t' in sys.abiflags, support.Py_GIL_DISABLED) + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): - def write_unraisable_exc(self, exc, err_msg, obj): - import _testcapi - import types - err_msg2 = f"Exception ignored {err_msg}" - try: - _testcapi.write_unraisable_exc(exc, err_msg, obj) - return types.SimpleNamespace(exc_type=type(exc), - exc_value=exc, - exc_traceback=exc.__traceback__, - err_msg=err_msg2, - object=obj) - finally: - # Explicitly break any reference cycle - exc = None - def test_original_unraisablehook(self): - for err_msg in (None, "original hook"): - with self.subTest(err_msg=err_msg): - obj = "an object" - - with test.support.captured_output("stderr") as stderr: - with test.support.swap_attr(sys, 'unraisablehook', - sys.__unraisablehook__): - self.write_unraisable_exc(ValueError(42), err_msg, obj) - - err = stderr.getvalue() - if err_msg is not None: - self.assertIn(f'Exception ignored {err_msg}: {obj!r}\n', err) - else: - self.assertIn(f'Exception ignored in: {obj!r}\n', err) - self.assertIn('Traceback (most recent call last):\n', err) - self.assertIn('ValueError: 42\n', err) + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable, err_formatunraisable + obj = hex + + with support.swap_attr(sys, 'unraisablehook', + sys.__unraisablehook__): + with support.captured_stderr() as stderr: + err_writeunraisable(ValueError(42), obj) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Exception ignored in: {obj!r}') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_writeunraisable(ValueError(42), None) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_formatunraisable(ValueError(42), 'Error in %R', obj) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], f'Error in {obj!r}:') + self.assertEqual(lines[1], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') + + with support.captured_stderr() as stderr: + err_formatunraisable(ValueError(42), None) + lines = stderr.getvalue().splitlines() + self.assertEqual(lines[0], 'Traceback (most recent call last):') + self.assertEqual(lines[-1], 'ValueError: 42') def test_original_unraisablehook_err(self): # bpo-22836: PyErr_WriteUnraisable() should give sensible reports @@ -1226,13 +1417,15 @@ def __del__(self): else: self.assertIn("ValueError", report) self.assertIn("del is broken", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") def test_original_unraisablehook_exception_qualname(self): # See bpo-41031, bpo-45083. # Check that the exception is printed with its qualified name # rather than just classname, and the module names appears # unless it is one of the hard-coded exclusions. + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable class A: class B: class X(Exception): @@ -1244,9 +1437,7 @@ class X(Exception): with test.support.captured_stderr() as stderr, test.support.swap_attr( sys, 'unraisablehook', sys.__unraisablehook__ ): - expected = self.write_unraisable_exc( - A.B.X(), "msg", "obj" - ) + err_writeunraisable(A.B.X(), "obj") report = stderr.getvalue() self.assertIn(A.B.X.__qualname__, report) if moduleName in ['builtins', '__main__']: @@ -1262,34 +1453,46 @@ def test_original_unraisablehook_wrong_type(self): sys.unraisablehook(exc) def test_custom_unraisablehook(self): + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable, err_formatunraisable hook_args = None def hook_func(args): nonlocal hook_args hook_args = args - obj = object() + obj = hex try: with test.support.swap_attr(sys, 'unraisablehook', hook_func): - expected = self.write_unraisable_exc(ValueError(42), - "custom hook", obj) - for attr in "exc_type exc_value exc_traceback err_msg object".split(): - self.assertEqual(getattr(hook_args, attr), - getattr(expected, attr), - (hook_args, expected)) + exc = ValueError(42) + err_writeunraisable(exc, obj) + self.assertIs(hook_args.exc_type, type(exc)) + self.assertIs(hook_args.exc_value, exc) + self.assertIs(hook_args.exc_traceback, exc.__traceback__) + self.assertIsNone(hook_args.err_msg) + self.assertEqual(hook_args.object, obj) + + err_formatunraisable(exc, "custom hook %R", obj) + self.assertIs(hook_args.exc_type, type(exc)) + self.assertIs(hook_args.exc_value, exc) + self.assertIs(hook_args.exc_traceback, exc.__traceback__) + self.assertEqual(hook_args.err_msg, f'custom hook {obj!r}') + self.assertIsNone(hook_args.object) finally: # expected and hook_args contain an exception: break reference cycle expected = None hook_args = None def test_custom_unraisablehook_fail(self): + _testcapi = import_helper.import_module('_testcapi') + from _testcapi import err_writeunraisable + def hook_func(*args): raise Exception("hook_func failed") with test.support.captured_output("stderr") as stderr: with test.support.swap_attr(sys, 'unraisablehook', hook_func): - self.write_unraisable_exc(ValueError(42), - "custom hook fail", None) + err_writeunraisable(ValueError(42), "custom hook fail") err = stderr.getvalue() self.assertIn(f'Exception ignored in sys.unraisablehook: ' @@ -1305,8 +1508,9 @@ class SizeofTest(unittest.TestCase): def setUp(self): self.P = struct.calcsize('P') self.longdigit = sys.int_info.sizeof_digit - import _testinternalcapi + _testinternalcapi = import_helper.import_module("_testinternalcapi") self.gc_headsize = _testinternalcapi.SIZEOF_PYGC_HEAD + self.managed_pre_header_size = _testinternalcapi.SIZEOF_MANAGED_PRE_HEADER check_sizeof = test.support.check_sizeof @@ -1342,7 +1546,7 @@ class OverflowSizeof(int): def __sizeof__(self): return int(self) self.assertEqual(sys.getsizeof(OverflowSizeof(sys.maxsize)), - sys.maxsize + self.gc_headsize) + sys.maxsize + self.gc_headsize + self.managed_pre_header_size) with self.assertRaises(OverflowError): sys.getsizeof(OverflowSizeof(sys.maxsize + 1)) with self.assertRaises(ValueError): @@ -1459,15 +1663,19 @@ class C(object): pass # float check(float(0), size('d')) # sys.floatinfo - check(sys.float_info, vsize('') + self.P * len(sys.float_info)) + check(sys.float_info, self.P + vsize('') + self.P * len(sys.float_info)) # frame def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + if support.Py_GIL_DISABLED: + INTERPRETER_FRAME = '9PihcP' + else: + INTERPRETER_FRAME = '9PhcP' + check(x, size('3PiccPPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass - check(func, size('14Pi')) + check(func, size('16Pi')) class c(): @staticmethod def foo(): @@ -1481,7 +1689,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('P2P4P4c7P2ic??P')) + check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator @@ -1509,7 +1717,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module - check(unittest, size('PnPPP')) + if support.Py_GIL_DISABLED: + check(unittest, size('PPPPPP')) + else: + check(unittest, size('PPPPP')) # None check(None, size('')) # NotImplementedType @@ -1524,9 +1735,15 @@ def delx(self): del self.__x x = property(getx, setx, delx, "") check(x, size('5Pi')) # PyCapsule - # XXX + try: + import _datetime + except ModuleNotFoundError: + pass + else: + check(_datetime.datetime_CAPI, size('6P')) # rangeiterator - check(iter(range(1)), size('4l')) + check(iter(range(1)), size('3l')) + check(iter(range(2**65)), size('3P')) # reverse check(reversed(''), size('nP')) # range @@ -1559,13 +1776,14 @@ def delx(self): del self.__x # super check(super(int), size('3P')) # tuple - check((), vsize('')) - check((1,2,3), vsize('') + 3*self.P) + check((), vsize('') + self.P) + check((1,2,3), vsize('') + self.P + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2nPI13Pl4Pn9Pn12PIP' - s = vsize('2P' + fmt) + fmt = 'P2nPI13Pl4Pn9Pn12PIPc' + s = vsize(fmt) check(int, s) + typeid = 'n' if support.Py_GIL_DISABLED else '' # class s = vsize(fmt + # PyTypeObject '4P' # PyAsyncMethods @@ -1573,8 +1791,9 @@ def delx(self): del self.__x '3P' # PyMappingMethods '10P' # PySequenceMethods '2P' # PyBufferProcs - '6P' - '1P' # Specializer cache + '7P' + '1PIP' # Specializer cache + + typeid # heap type id (free-threaded only) ) class newstyleclass(object): pass # Separate block for PyDictKeysObject with 8 keys and 5 entries @@ -1596,8 +1815,8 @@ class newstyleclass(object): pass '\u0100'*40, '\uffff'*100, '\U00010000'*30, '\U0010ffff'*100] # also update field definitions in test_unicode.test_raiseMemError - asciifields = "nnbP" - compactfields = asciifields + "nPn" + asciifields = "nnb" + compactfields = asciifields + "nP" unicodefields = compactfields + "P" for s in samples: maxchar = ord(max(s)) @@ -1621,11 +1840,15 @@ class newstyleclass(object): pass # TODO: add check that forces layout of unicodefields # weakref import weakref - check(weakref.ref(int), size('2Pn3P')) + if support.Py_GIL_DISABLED: + expected = size('2Pn4P') + else: + expected = size('2Pn3P') + check(weakref.ref(int), expected) # weakproxy # XXX # weakcallableproxy - check(weakref.proxy(int), size('2Pn3P')) + check(weakref.proxy(int), expected) def check_slots(self, obj, base, extra): expected = sys.getsizeof(base) + struct.calcsize(extra) @@ -1667,15 +1890,18 @@ def test_pythontypes(self): check(_ast.AST(), size('P')) try: raise TypeError - except TypeError: - tb = sys.exc_info()[2] + except TypeError as e: + tb = e.__traceback__ # traceback if tb is not None: check(tb, size('2P2i')) # symtable entry # XXX # sys.flags - check(sys.flags, vsize('') + self.P * len(sys.flags)) + # FIXME: The +3 is for the 'gil', 'thread_inherit_context' and + # 'context_aware_warnings' flags and will not be necessary once + # gh-122575 is fixed + check(sys.flags, vsize('') + self.P + self.P * (3 + len(sys.flags))) def test_asyncgen_hooks(self): old = sys.get_asyncgen_hooks() @@ -1683,6 +1909,21 @@ def test_asyncgen_hooks(self): self.assertIsNone(old.finalizer) firstiter = lambda *a: None + finalizer = lambda *a: None + + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer="invalid") + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + + # gh-118473 + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter="invalid", finalizer=finalizer) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + sys.set_asyncgen_hooks(firstiter=firstiter) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1690,7 +1931,6 @@ def test_asyncgen_hooks(self): self.assertIs(hooks.finalizer, None) self.assertIs(hooks[1], None) - finalizer = lambda *a: None sys.set_asyncgen_hooks(finalizer=finalizer) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1719,5 +1959,318 @@ def write(self, s): self.assertEqual(out, b"") self.assertEqual(err, b"") +@test.support.support_remote_exec_only +@test.support.cpython_only +class TestRemoteExec(unittest.TestCase): + def tearDown(self): + test.support.reap_children() + + def _run_remote_exec_test(self, script_code, python_args=None, env=None, + prologue='', + script_path=os_helper.TESTFN + '_remote.py'): + # Create the script that will be remotely executed + self.addCleanup(os_helper.unlink, script_path) + + with open(script_path, 'w') as f: + f.write(script_code) + + # Create and run the target process + target = os_helper.TESTFN + '_target.py' + self.addCleanup(os_helper.unlink, target) + + port = find_unused_port() + + with open(target, 'w') as f: + f.write(f''' +import sys +import time +import socket + +# Connect to the test process +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +sock.connect(('localhost', {port})) + +{prologue} + +# Signal that the process is ready +sock.sendall(b"ready") + +print("Target process running...") + +# Wait for remote script to be executed +# (the execution will happen as the following +# code is processed as soon as the recv call +# unblocks) +sock.recv(1024) + +# Do a bunch of work to give the remote script time to run +x = 0 +for i in range(100): + x += i + +# Write confirmation back +sock.sendall(b"executed") +sock.close() +''') + + # Start the target process and capture its output + cmd = [sys.executable] + if python_args: + cmd.extend(python_args) + cmd.append(target) + + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.bind(('localhost', port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + with subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) as proc: + client_socket = None + try: + # Accept connection from target process + client_socket, _ = server_socket.accept() + server_socket.close() + + response = client_socket.recv(1024) + self.assertEqual(response, b"ready") + + # Try remote exec on the target process + sys.remote_exec(proc.pid, script_path) + + # Signal script to continue + client_socket.sendall(b"continue") + + # Wait for execution confirmation + response = client_socket.recv(1024) + self.assertEqual(response, b"executed") + + # Return output for test verification + stdout, stderr = proc.communicate(timeout=10.0) + return proc.returncode, stdout, stderr + except PermissionError: + self.skipTest("Insufficient permissions to execute code in remote process") + finally: + if client_socket is not None: + client_socket.close() + proc.kill() + proc.terminate() + proc.wait(timeout=SHORT_TIMEOUT) + + def test_remote_exec(self): + """Test basic remote exec functionality""" + script = 'print("Remote script executed successfully!")' + returncode, stdout, stderr = self._run_remote_exec_test(script) + # self.assertEqual(returncode, 0) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + + def test_remote_exec_bytes(self): + script = 'print("Remote script executed successfully!")' + script_path = os.fsencode(os_helper.TESTFN) + b'_bytes_remote.py' + returncode, stdout, stderr = self._run_remote_exec_test(script, + script_path=script_path) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, 'requires undecodable path') + @unittest.skipIf(sys.platform == 'darwin', + 'undecodable paths are not supported on macOS') + def test_remote_exec_undecodable(self): + script = 'print("Remote script executed successfully!")' + script_path = os_helper.TESTFN_UNDECODABLE + b'_undecodable_remote.py' + for script_path in [script_path, os.fsdecode(script_path)]: + returncode, stdout, stderr = self._run_remote_exec_test(script, + script_path=script_path) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + + def test_remote_exec_with_self_process(self): + """Test remote exec with the target process being the same as the test process""" + + code = 'import sys;print("Remote script executed successfully!", file=sys.stderr)' + file = os_helper.TESTFN + '_remote_self.py' + with open(file, 'w') as f: + f.write(code) + self.addCleanup(os_helper.unlink, file) + with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + sys.remote_exec(os.getpid(), os.path.abspath(file)) + print("Done") + self.assertEqual(mock_stderr.getvalue(), "Remote script executed successfully!\n") + self.assertEqual(mock_stdout.getvalue(), "Done\n") + + def test_remote_exec_raises_audit_event(self): + """Test remote exec raises an audit event""" + prologue = '''\ +import sys +def audit_hook(event, arg): + print(f"Audit event: {event}, arg: {arg}".encode("ascii", errors="replace")) +sys.addaudithook(audit_hook) +''' + script = ''' +print("Remote script executed successfully!") +''' + returncode, stdout, stderr = self._run_remote_exec_test(script, prologue=prologue) + self.assertEqual(returncode, 0) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertIn(b"Audit event: cpython.remote_debugger_script, arg: ", stdout) + self.assertEqual(stderr, b"") + + def test_remote_exec_with_exception(self): + """Test remote exec with an exception raised in the target process + + The exception should be raised in the main thread of the target process + but not crash the target process. + """ + script = ''' +raise Exception("Remote script exception") +''' + returncode, stdout, stderr = self._run_remote_exec_test(script) + self.assertEqual(returncode, 0) + self.assertIn(b"Remote script exception", stderr) + self.assertEqual(stdout.strip(), b"Target process running...") + + def test_new_namespace_for_each_remote_exec(self): + """Test that each remote_exec call gets its own namespace.""" + script = textwrap.dedent( + """ + assert globals() is not __import__("__main__").__dict__ + print("Remote script executed successfully!") + """ + ) + returncode, stdout, stderr = self._run_remote_exec_test(script) + self.assertEqual(returncode, 0) + self.assertEqual(stderr, b"") + self.assertIn(b"Remote script executed successfully", stdout) + + def test_remote_exec_disabled_by_env(self): + """Test remote exec is disabled when PYTHON_DISABLE_REMOTE_DEBUG is set""" + env = os.environ.copy() + env['PYTHON_DISABLE_REMOTE_DEBUG'] = '1' + with self.assertRaisesRegex(RuntimeError, "Remote debugging is not enabled in the remote process"): + self._run_remote_exec_test("print('should not run')", env=env) + + def test_remote_exec_disabled_by_xoption(self): + """Test remote exec is disabled with -Xdisable-remote-debug""" + with self.assertRaisesRegex(RuntimeError, "Remote debugging is not enabled in the remote process"): + self._run_remote_exec_test("print('should not run')", python_args=['-Xdisable-remote-debug']) + + def test_remote_exec_invalid_pid(self): + """Test remote exec with invalid process ID""" + with self.assertRaises(OSError): + sys.remote_exec(99999, "print('should not run')") + + def test_remote_exec_invalid_script(self): + """Test remote exec with invalid script type""" + with self.assertRaises(TypeError): + sys.remote_exec(0, None) + with self.assertRaises(TypeError): + sys.remote_exec(0, 123) + + def test_remote_exec_syntax_error(self): + """Test remote exec with syntax error in script""" + script = ''' +this is invalid python code +''' + returncode, stdout, stderr = self._run_remote_exec_test(script) + self.assertEqual(returncode, 0) + self.assertIn(b"SyntaxError", stderr) + self.assertEqual(stdout.strip(), b"Target process running...") + + def test_remote_exec_invalid_script_path(self): + """Test remote exec with invalid script path""" + with self.assertRaises(OSError): + sys.remote_exec(os.getpid(), "invalid_script_path") + + def test_remote_exec_in_process_without_debug_fails_envvar(self): + """Test remote exec in a process without remote debugging enabled""" + script = os_helper.TESTFN + '_remote.py' + self.addCleanup(os_helper.unlink, script) + with open(script, 'w') as f: + f.write('print("Remote script executed successfully!")') + env = os.environ.copy() + env['PYTHON_DISABLE_REMOTE_DEBUG'] = '1' + + _, out, err = assert_python_failure('-c', f'import os, sys; sys.remote_exec(os.getpid(), "{script}")', **env) + self.assertIn(b"Remote debugging is not enabled", err) + self.assertEqual(out, b"") + + def test_remote_exec_in_process_without_debug_fails_xoption(self): + """Test remote exec in a process without remote debugging enabled""" + script = os_helper.TESTFN + '_remote.py' + self.addCleanup(os_helper.unlink, script) + with open(script, 'w') as f: + f.write('print("Remote script executed successfully!")') + + _, out, err = assert_python_failure('-Xdisable-remote-debug', '-c', f'import os, sys; sys.remote_exec(os.getpid(), "{script}")') + self.assertIn(b"Remote debugging is not enabled", err) + self.assertEqual(out, b"") + +class TestSysJIT(unittest.TestCase): + + def test_jit_is_available(self): + available = sys._jit.is_available() + script = f"import sys; assert sys._jit.is_available() is {available}" + assert_python_ok("-c", script, PYTHON_JIT="0") + assert_python_ok("-c", script, PYTHON_JIT="1") + + def test_jit_is_enabled(self): + available = sys._jit.is_available() + script = "import sys; assert sys._jit.is_enabled() is {enabled}" + assert_python_ok("-c", script.format(enabled=False), PYTHON_JIT="0") + assert_python_ok("-c", script.format(enabled=available), PYTHON_JIT="1") + + @unittest.expectedFailure # TODO: RUSTPYTHON; --- + def test_jit_is_active(self): + available = sys._jit.is_available() + script = textwrap.dedent( + """ + import _testcapi + import _testinternalcapi + import sys + + def frame_0_interpreter() -> None: + assert sys._jit.is_active() is False + + def frame_1_interpreter() -> None: + assert sys._jit.is_active() is False + frame_0_interpreter() + assert sys._jit.is_active() is False + + def frame_2_jit(expected: bool) -> None: + # Inlined into the last loop of frame_3_jit: + assert sys._jit.is_active() is expected + # Insert C frame: + _testcapi.pyobject_vectorcall(frame_1_interpreter, None, None) + assert sys._jit.is_active() is expected + + def frame_3_jit() -> None: + # JITs just before the last loop: + for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + # Careful, doing this in the reverse order breaks tracing: + expected = {enabled} and i == _testinternalcapi.TIER2_THRESHOLD + assert sys._jit.is_active() is expected + frame_2_jit(expected) + assert sys._jit.is_active() is expected + + def frame_4_interpreter() -> None: + assert sys._jit.is_active() is False + frame_3_jit() + assert sys._jit.is_active() is False + + assert sys._jit.is_active() is False + frame_4_interpreter() + assert sys._jit.is_active() is False + """ + ) + assert_python_ok("-c", script.format(enabled=False), PYTHON_JIT="0") + assert_python_ok("-c", script.format(enabled=available), PYTHON_JIT="1") + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index d5e3206a5ca..21a09b51926 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -100,8 +100,6 @@ class ProfileHookTestCase(TestCaseBase): def new_watcher(self): return HookWatcher() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple(self): def f(p): pass @@ -110,8 +108,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception(self): def f(p): 1/0 @@ -120,8 +116,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caught_exception(self): def f(p): try: 1/0 @@ -131,8 +125,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caught_nested_exception(self): def f(p): try: 1/0 @@ -142,8 +134,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nested_exception(self): def f(p): 1/0 @@ -155,8 +145,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_in_except_clause(self): def f(p): 1/0 @@ -176,8 +164,6 @@ def g(p): (1, 'return', g_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_propagation(self): def f(p): 1/0 @@ -193,8 +179,6 @@ def g(p): (1, 'return', g_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_raise_twice(self): def f(p): try: 1/0 @@ -204,8 +188,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_raise_reraise(self): def f(p): try: 1/0 @@ -215,8 +197,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_raise(self): def f(p): raise Exception() @@ -225,8 +205,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_distant_exception(self): def f(): 1/0 @@ -255,8 +233,6 @@ def j(p): (1, 'return', j_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_generator(self): def f(): for i in range(2): @@ -279,8 +255,6 @@ def g(p): (1, 'return', g_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_stop_iteration(self): def f(): for i in range(2): @@ -307,8 +281,6 @@ class ProfileSimulatorTestCase(TestCaseBase): def new_watcher(self): return ProfileSimulator(self) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple(self): def f(p): pass @@ -317,8 +289,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_exception(self): def f(p): 1/0 @@ -327,8 +297,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caught_exception(self): def f(p): try: 1/0 @@ -338,8 +306,6 @@ def f(p): (1, 'return', f_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_distant_exception(self): def f(): 1/0 @@ -368,8 +334,6 @@ def j(p): (1, 'return', j_ident), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # bpo-34125: profiling method_descriptor with **kwargs def test_unbound_method(self): kwargs = {} @@ -379,8 +343,6 @@ def f(p): self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test an invalid call (bpo-34126) def test_unbound_method_no_args(self): def f(p): @@ -389,8 +351,6 @@ def f(p): self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test an invalid call (bpo-34126) def test_unbound_method_invalid_args(self): def f(p): @@ -399,8 +359,6 @@ def f(p): self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test an invalid call (bpo-34125) def test_unbound_method_no_keyword_args(self): kwargs = {} @@ -410,8 +368,6 @@ def f(p): self.check_events(f, [(1, 'call', f_ident), (1, 'return', f_ident)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test an invalid call (bpo-34125) def test_unbound_method_invalid_keyword_args(self): kwargs = {} diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a1e4b091a1d..a98b4d22760 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -1218,8 +1218,6 @@ def test_return(self): def test_exception(self): self.run_test_for_event('exception') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_trash_stack(self): def f(): for i in range(5): @@ -1393,23 +1391,17 @@ def test(self): ## The first set of 'jump' tests are for things that are allowed: - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(1, 3, [3]) def test_jump_simple_forwards(output): output.append(1) output.append(2) output.append(3) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(2, 1, [1, 1, 2]) def test_jump_simple_backwards(output): output.append(1) output.append(2) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 5, [2, 5]) def test_jump_out_of_block_forwards(output): for i in 1, 2: @@ -1418,8 +1410,6 @@ def test_jump_out_of_block_forwards(output): output.append(4) output.append(5) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(6, 1, [1, 3, 5, 1, 3, 5, 6, 7]) def test_jump_out_of_block_backwards(output): output.append(1) @@ -1451,8 +1441,6 @@ async def test_jump_out_of_async_for_block_backwards(output): output.append(5) output.append(6) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(1, 2, [3]) def test_jump_to_codeless_line(output): output.append(1) @@ -1465,8 +1453,6 @@ def test_jump_to_same_line(output): output.append(2) output.append(3) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Tests jumping within a finally block, and over one. @jump_test(4, 9, [2, 9]) def test_jump_in_nested_finally(output): @@ -1480,8 +1466,6 @@ def test_jump_in_nested_finally(output): output.append(8) output.append(9) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(6, 7, [2, 7], (ZeroDivisionError, '')) def test_jump_in_nested_finally_2(output): try: @@ -1493,8 +1477,6 @@ def test_jump_in_nested_finally_2(output): output.append(7) output.append(8) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(6, 11, [2, 11], (ZeroDivisionError, '')) def test_jump_in_nested_finally_3(output): try: @@ -1535,8 +1517,6 @@ def test_no_jump_infinite_while_loop(output): output.append(3) output.append(4) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(2, 4, [4, 4]) def test_jump_forwards_into_while_block(output): i = 1 @@ -1545,8 +1525,6 @@ def test_jump_forwards_into_while_block(output): output.append(4) i += 1 - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(5, 3, [3, 3, 3, 5]) def test_jump_backwards_into_while_block(output): i = 1 @@ -1555,8 +1533,6 @@ def test_jump_backwards_into_while_block(output): i += 1 output.append(5) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(2, 3, [1, 3]) def test_jump_forwards_out_of_with_block(output): with tracecontext(output, 1): @@ -1571,8 +1547,6 @@ async def test_jump_forwards_out_of_async_with_block(output): output.append(2) output.append(3) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 1, [1, 2, 1, 2, 3, -2]) def test_jump_backwards_out_of_with_block(output): output.append(1) @@ -1587,8 +1561,6 @@ async def test_jump_backwards_out_of_async_with_block(output): async with asynctracecontext(output, 2): output.append(3) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(2, 5, [5]) def test_jump_forwards_out_of_try_finally_block(output): try: @@ -1597,8 +1569,6 @@ def test_jump_forwards_out_of_try_finally_block(output): output.append(4) output.append(5) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 1, [1, 1, 3, 5]) def test_jump_backwards_out_of_try_finally_block(output): output.append(1) @@ -1607,8 +1577,6 @@ def test_jump_backwards_out_of_try_finally_block(output): finally: output.append(5) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(2, 6, [6]) def test_jump_forwards_out_of_try_except_block(output): try: @@ -1618,8 +1586,6 @@ def test_jump_forwards_out_of_try_except_block(output): raise output.append(6) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 1, [1, 1, 3]) def test_jump_backwards_out_of_try_except_block(output): output.append(1) @@ -1629,8 +1595,6 @@ def test_jump_backwards_out_of_try_except_block(output): output.append(5) raise - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(5, 7, [4, 7, 8]) def test_jump_between_except_blocks(output): try: @@ -1642,8 +1606,6 @@ def test_jump_between_except_blocks(output): output.append(7) output.append(8) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(5, 6, [4, 6, 7]) def test_jump_within_except_block(output): try: @@ -1654,8 +1616,6 @@ def test_jump_within_except_block(output): output.append(6) output.append(7) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(2, 4, [1, 4, 5, -4]) def test_jump_across_with(output): output.append(1) @@ -1674,8 +1634,6 @@ async def test_jump_across_async_with(output): async with asynctracecontext(output, 4): output.append(5) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(4, 5, [1, 3, 5, 6]) def test_jump_out_of_with_block_within_for_block(output): output.append(1) @@ -1696,8 +1654,6 @@ async def test_jump_out_of_async_with_block_within_for_block(output): output.append(5) output.append(6) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(4, 5, [1, 2, 3, 5, -2, 6]) def test_jump_out_of_with_block_within_with_block(output): output.append(1) @@ -1718,8 +1674,6 @@ async def test_jump_out_of_async_with_block_within_with_block(output): output.append(5) output.append(6) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(5, 6, [2, 4, 6, 7]) def test_jump_out_of_with_block_within_finally_block(output): try: @@ -1742,8 +1696,6 @@ async def test_jump_out_of_async_with_block_within_finally_block(output): output.append(6) output.append(7) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(8, 11, [1, 3, 5, 11, 12]) def test_jump_out_of_complex_nested_blocks(output): output.append(1) @@ -1759,8 +1711,6 @@ def test_jump_out_of_complex_nested_blocks(output): output.append(11) output.append(12) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 5, [1, 2, 5]) def test_jump_out_of_with_assignment(output): output.append(1) @@ -1779,8 +1729,6 @@ async def test_jump_out_of_async_with_assignment(output): output.append(4) output.append(5) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 6, [1, 6, 8, 9]) def test_jump_over_return_in_try_finally_block(output): output.append(1) @@ -1793,8 +1741,6 @@ def test_jump_over_return_in_try_finally_block(output): output.append(8) output.append(9) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(5, 8, [1, 3, 8, 10, 11, 13]) def test_jump_over_break_in_try_finally_block(output): output.append(1) @@ -1811,8 +1757,6 @@ def test_jump_over_break_in_try_finally_block(output): break output.append(13) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(1, 7, [7, 8]) def test_jump_over_for_block_before_else(output): output.append(1) @@ -1839,15 +1783,11 @@ async def test_jump_over_async_for_block_before_else(output): # The second set of 'jump' tests are for things that are not allowed: - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(2, 3, [1], (ValueError, 'after')) def test_no_jump_too_far_forwards(output): output.append(1) output.append(2) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(2, -2, [1], (ValueError, 'before')) def test_no_jump_too_far_backwards(output): output.append(1) @@ -1894,8 +1834,6 @@ def test_no_jump_to_except_4(output): output.append(4) raise e - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(1, 3, [], (ValueError, 'into')) def test_no_jump_forwards_into_for_block(output): output.append(1) @@ -1911,8 +1849,6 @@ async def test_no_jump_forwards_into_async_for_block(output): output.append(3) pass - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 2, [2, 2], (ValueError, 'into')) def test_no_jump_backwards_into_for_block(output): for i in 1, 2: @@ -2015,8 +1951,6 @@ def test_no_jump_between_except_blocks_2(output): output.append(7) output.append(8) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(1, 5, [5]) def test_jump_into_finally_block(output): output.append(1) @@ -2025,8 +1959,6 @@ def test_jump_into_finally_block(output): finally: output.append(5) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 6, [2, 6, 7]) def test_jump_into_finally_block_from_try_block(output): try: @@ -2037,8 +1969,6 @@ def test_jump_into_finally_block_from_try_block(output): output.append(6) output.append(7) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(5, 1, [1, 3, 1, 3, 5]) def test_jump_out_of_finally_block(output): output.append(1) @@ -2080,8 +2010,6 @@ def test_no_jump_into_bare_except_block_from_try_block(output): raise output.append(8) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 6, [2], (ValueError, "into an 'except'")) def test_no_jump_into_qualified_except_block_from_try_block(output): try: @@ -2117,8 +2045,6 @@ def test_no_jump_out_of_qualified_except_block(output): output.append(6) output.append(7) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(3, 5, [1, 2, 5, -2]) def test_jump_between_with_blocks(output): output.append(1) @@ -2149,8 +2075,6 @@ def test_no_jump_over_return_out_of_finally_block(output): return output.append(7) - # TODO: RUSTPYTHON - @unittest.expectedFailure @jump_test(7, 4, [1, 6], (ValueError, 'into')) def test_no_jump_into_for_block_before_else(output): output.append(1) @@ -2187,8 +2111,6 @@ def test_no_jump_without_trace_function(self): # triggered. no_jump_without_trace_function() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_large_function(self): d = {} exec("""def f(output): # line 0 @@ -2203,8 +2125,6 @@ def test_large_function(self): f = d['f'] self.run_test(f, 2, 1007, [0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_jump_to_firstlineno(self): # This tests that PDB can jump back to the first line in a # file. See issue #1689458. It can only be triggered in a diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 6db1442980a..7da1326a721 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,33 +1,48 @@ +import platform +import re import unittest import sys import os import subprocess import shutil +import json +import textwrap from copy import copy +from test import support from test.support import ( - captured_stdout, PythonSymlink, requires_subprocess, is_wasi + captured_stdout, + is_android, + is_apple_mobile, + is_wasi, + PythonSymlink, + requires_subprocess, ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) -from test.support.warnings_helper import check_warnings +from test.support.venv import VirtualEnvironmentMixin import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes, _main) + _expand_vars, _get_preferred_schemes, + is_python_build, _PROJECT_BASE) +from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name +import _imp import _osx_support +import _sysconfig HAS_USER_BASE = sysconfig._HAS_USER_BASE -class TestSysConfig(unittest.TestCase): +class TestSysConfig(unittest.TestCase, VirtualEnvironmentMixin): def setUp(self): super(TestSysConfig, self).setUp() + self.maxDiff = None self.sys_path = sys.path[:] # patching os.uname if hasattr(os, 'uname'): @@ -39,8 +54,11 @@ def setUp(self): os.uname = self._get_uname # saving the environment self.name = os.name + self.prefix = sys.prefix + self.exec_prefix = sys.exec_prefix self.platform = sys.platform self.version = sys.version + self._framework = sys._framework self.sep = os.sep self.join = os.path.join self.isabs = os.path.isabs @@ -62,8 +80,11 @@ def tearDown(self): else: del os.uname os.name = self.name + sys.prefix = self.prefix + sys.exec_prefix = self.exec_prefix sys.platform = self.platform sys.version = self.version + sys._framework = self._framework os.sep = self.sep os.path.join = self.join os.path.isabs = self.isabs @@ -137,35 +158,37 @@ def test_get_preferred_schemes(self): # Mac, framework build. os.name = 'posix' sys.platform = 'darwin' - sys._framework = True + sys._framework = "MyPython" self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) - # NOTE: RUSTPYTHON this is hardcoded to 'python', we're set up for failure. - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++ def test_posix_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + f'python{sysconfig._get_python_version_abi()}', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv/ directory + binpath = os.path.join('venv', binpath) + incpath = os.path.join('venv', incpath) + libpath = os.path.join('venv', libpath) - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} + + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv', vars=vars)) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') - self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) + self.assertStartsWith(sysconfig_includedir, incpath + os.sep) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_nt_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress @@ -173,14 +196,19 @@ def test_nt_venv_scheme(self): incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv\ directory + venv = 'venv' + binpath = os.path.join(venv, binpath) + incpath = os.path.join(venv, incpath) + libpath = os.path.join(venv, libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) - self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv', vars=vars)) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): if sys.platform == 'win32': @@ -216,6 +244,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -321,6 +354,13 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'macosx-10.4-%s' % arch) + for macver in range(11, 16): + _osx_support._remove_original_values(get_config_vars()) + get_config_vars()['CFLAGS'] = ('-fno-strict-overflow -Wsign-compare -Wunreachable-code' + '-arch arm64 -fno-common -dynamic -DNDEBUG -g -O3 -Wall') + get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = f"{macver}.0" + self.assertEqual(get_platform(), 'macosx-%d.0-arm64' % macver) + # linux debian sarge os.name = 'posix' sys.version = ('2.3.5 (#1, Jul 4 2007, 17:28:59) ' @@ -331,11 +371,27 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : /Users/youknowone/Projects/RustPython2/include/python3.14t/pyconfig.h @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't distribute header files in the runtime environment") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) @@ -381,8 +437,8 @@ def test_user_similar(self): if name == 'platlib': # Replace "/lib64/python3.11/site-packages" suffix # with "/lib/python3.11/site-packages". - py_version_short = sysconfig.get_python_version() - suffix = f'python{py_version_short}/site-packages' + py_version_abi = sysconfig._get_python_version_abi() + suffix = f'python{py_version_abi}/site-packages' expected = expected.replace(f'/{sys.platlibdir}/{suffix}', f'/lib/{suffix}') self.assertEqual(user_path, expected) @@ -393,8 +449,6 @@ def test_main(self): _main() self.assertTrue(len(output.getvalue().split('\n')) > 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows") def test_ldshared_value(self): ldflags = sysconfig.get_config_var('LDFLAGS') @@ -402,6 +456,31 @@ def test_ldshared_value(self): self.assertIn(ldflags, ldshared) + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") + def test_soabi(self): + soabi = sysconfig.get_config_var('SOABI') + self.assertIn(soabi, _imp.extension_suffixes()[0]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Expected str, not NoneType + def test_library(self): + library = sysconfig.get_config_var('LIBRARY') + ldlibrary = sysconfig.get_config_var('LDLIBRARY') + major, minor = sys.version_info[:2] + abiflags = sysconfig.get_config_var('ABIFLAGS') + if sys.platform.startswith('win'): + self.assertEqual(library, f'python{major}{minor}{abiflags}.dll') + self.assertEqual(library, ldlibrary) + elif is_apple_mobile: + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + self.assertEqual(ldlibrary, f"{framework}.framework/{framework}") + else: + self.assertStartsWith(library, f'libpython{major}.{minor}') + self.assertEndsWith(library, '.a') + if sys.platform == 'darwin' and sys._framework: + self.skipTest('gh-110824: skip LDLIBRARY test for framework build') + else: + self.assertStartsWith(ldlibrary, f'libpython{major}.{minor}') + @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() def test_platform_in_subprocess(self): @@ -448,6 +527,8 @@ def test_platform_in_subprocess(self): @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -460,11 +541,12 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - # <srcdir>/PC/pyconfig.h always exists even if unused on POSIX. - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') - self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) + # <srcdir>/PC/pyconfig.h.in always exists even if unused pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) + if os.name == 'nt': + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already @@ -479,23 +561,16 @@ def test_srcdir_independent_of_cwd(self): srcdir2 = sysconfig.get_config_var('srcdir') self.assertEqual(srcdir, srcdir2) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, 'EXT_SUFFIX required for this test') + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): - import _imp - if not _imp.extension_suffixes(): - self.skipTest("stub loader has no suffixes") vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -505,23 +580,186 @@ def test_triplet_in_ext_suffix(self): expected_suffixes = 'i386-linux-gnu.so', 'x86_64-linux-gnux32.so', 'i386-linux-musl.so' else: # 8 byte pointer size expected_suffixes = 'x86_64-linux-gnu.so', 'x86_64-linux-musl.so' - self.assertTrue(suffix.endswith(expected_suffixes), - f'unexpected suffix {suffix!r}') + self.assertEndsWith(suffix, expected_suffixes) + + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertEndsWith(suffix, f"-{expected_triplet}.so") - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') - self.assertTrue(suffix.endswith('-darwin.so'), suffix) + self.assertEndsWith(suffix, '-darwin.so') + + def test_always_set_py_debug(self): + self.assertIn('Py_DEBUG', sysconfig.get_config_vars()) + Py_DEBUG = sysconfig.get_config_var('Py_DEBUG') + self.assertIn(Py_DEBUG, (0, 1)) + self.assertEqual(Py_DEBUG, support.Py_DEBUG) + + def test_always_set_py_gil_disabled(self): + self.assertIn('Py_GIL_DISABLED', sysconfig.get_config_vars()) + Py_GIL_DISABLED = sysconfig.get_config_var('Py_GIL_DISABLED') + self.assertIn(Py_GIL_DISABLED, (0, 1)) + self.assertEqual(Py_GIL_DISABLED, support.Py_GIL_DISABLED) + + def test_abiflags(self): + # If this test fails on some platforms, maintainers should update the + # test to make it pass, rather than changing the definition of ABIFLAGS. + self.assertIn('abiflags', sysconfig.get_config_vars()) + self.assertIn('ABIFLAGS', sysconfig.get_config_vars()) + abiflags = sysconfig.get_config_var('abiflags') + ABIFLAGS = sysconfig.get_config_var('ABIFLAGS') + self.assertIsInstance(abiflags, str) + self.assertIsInstance(ABIFLAGS, str) + self.assertIn(abiflags, ABIFLAGS) + if os.name == 'nt': + self.assertEqual(abiflags, '') + + if not sys.platform.startswith('win'): + valid_abiflags = ('', 't', 'd', 'td') + else: + # Windows uses '_d' rather than 'd'; see also test_abi_debug below + valid_abiflags = ('', 't', '_d', 't_d') + + self.assertIn(ABIFLAGS, valid_abiflags) + + def test_abi_debug(self): + ABIFLAGS = sysconfig.get_config_var('ABIFLAGS') + if support.Py_DEBUG: + self.assertIn('d', ABIFLAGS) + else: + self.assertNotIn('d', ABIFLAGS) + + # The 'd' flag should always be the last one on Windows. + # On Windows, the debug flag is used differently with a underscore prefix. + # For example, `python{X}.{Y}td` on Unix and `python{X}.{Y}t_d.exe` on Windows. + if support.Py_DEBUG and sys.platform.startswith('win'): + self.assertEndsWith(ABIFLAGS, '_d') + + def test_abi_thread(self): + abi_thread = sysconfig.get_config_var('abi_thread') + ABIFLAGS = sysconfig.get_config_var('ABIFLAGS') + self.assertIsInstance(abi_thread, str) + if support.Py_GIL_DISABLED: + self.assertEqual(abi_thread, 't') + self.assertIn('t', ABIFLAGS) + else: + self.assertEqual(abi_thread, '') + self.assertNotIn('t', ABIFLAGS) + + @requires_subprocess() + @unittest.skipIf(os.name == 'nt', 'TODO: RUSTPYTHON; venv creates python.exe but sys.executable is rustpython.exe') + def test_makefile_overwrites_config_vars(self): + script = textwrap.dedent(""" + import sys, sysconfig + + data = { + 'prefix': sys.prefix, + 'exec_prefix': sys.exec_prefix, + 'base_prefix': sys.base_prefix, + 'base_exec_prefix': sys.base_exec_prefix, + 'config_vars': sysconfig.get_config_vars(), + } + + import json + print(json.dumps(data, indent=2)) + """) + + # We need to run the test inside a virtual environment so that + # sys.prefix/sys.exec_prefix have a different value from the + # prefix/exec_prefix Makefile variables. + with self.venv() as venv: + data = json.loads(venv.run('-c', script).stdout) + + # We expect sysconfig.get_config_vars to correctly reflect sys.prefix/sys.exec_prefix + self.assertEqual(data['prefix'], data['config_vars']['prefix']) + self.assertEqual(data['exec_prefix'], data['config_vars']['exec_prefix']) + # As a sanity check, just make sure sys.prefix/sys.exec_prefix really + # are different from the Makefile values. + # sys.base_prefix/sys.base_exec_prefix should reflect the value of the + # prefix/exec_prefix Makefile variables, so we use them in the comparison. + self.assertNotEqual(data['prefix'], data['base_prefix']) + self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) + + @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX') + @unittest.expectedFailure # TODO: RUSTPYTHON; JSON is generated at build time in CPython + @unittest.skipIf(is_wasi, "_sysconfig-vars JSON file currently isn't available on WASI") + @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS change the prefix') + def test_sysconfigdata_json(self): + if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: + data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] + elif is_python_build(): + data_dir = os.path.join(_PROJECT_BASE, _get_pybuilddir()) + else: + data_dir = sys._stdlib_dir + + json_data_path = os.path.join(data_dir, _get_json_data_name()) + + with open(json_data_path) as f: + json_config_vars = json.load(f) + + system_config_vars = get_config_vars() + + # Keys dependent on uncontrollable external context + ignore_keys = {'userbase'} + # Keys dependent on Python being run outside the build directrory + if sysconfig.is_python_build(): + ignore_keys |= {'srcdir'} + # Keys dependent on the executable location + if os.path.dirname(sys.executable) != system_config_vars['BINDIR']: + ignore_keys |= {'projectbase'} + # Keys dependent on the environment (different inside virtual environments) + if sys.prefix != sys.base_prefix: + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} + # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) + if sysconfig._installation_is_relocated(): + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase', 'srcdir'} + + for key in ignore_keys: + json_config_vars.pop(key, None) + system_config_vars.pop(key, None) + + self.assertEqual(system_config_vars, json_config_vars) + + def test_sysconfig_config_vars_no_prefix_cache(self): + sys.prefix = 'prefix-AAA' + sys.exec_prefix = 'exec-prefix-AAA' + + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) + + sys.prefix = 'prefix-BBB' + sys.exec_prefix = 'exec-prefix-BBB' + + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) + class MakefileTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true : /Users/youknowone/Projects/RustPython2/lib/python3.14t/config-3.14t-aarch64-apple-darwin/Makefile @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_get_makefile_filename(self): makefile = sysconfig.get_makefile_filename() self.assertTrue(os.path.isfile(makefile), makefile) @@ -536,7 +774,7 @@ def test_parse_makefile(self): print("var5=dollar$$5", file=makefile) print("var6=${var3}/lib/python3.5/config-$(VAR2)$(var5)" "-x86_64-linux-gnu", file=makefile) - vars = sysconfig._parse_makefile(TESTFN) + vars = _parse_makefile(TESTFN) self.assertEqual(vars, { 'var1': 'ab42', 'VAR2': 'b42', @@ -547,5 +785,38 @@ def test_parse_makefile(self): }) +class DeprecationTests(unittest.TestCase): + def deprecated(self, removal_version, deprecation_msg=None, error=Exception, error_msg=None): + if sys.version_info >= removal_version: + return self.assertRaises(error, msg=error_msg) + else: + return self.assertWarns(DeprecationWarning, msg=deprecation_msg) + + def test_expand_makefile_vars(self): + with self.deprecated( + removal_version=(3, 16), + deprecation_msg=( + 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' + 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', + ), + error=AttributeError, + error_msg="module 'sysconfig' has no attribute 'expand_makefile_vars'", + ): + sysconfig.expand_makefile_vars('', {}) + + def test_is_python_build_check_home(self): + with self.deprecated( + removal_version=(3, 15), + deprecation_msg=( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + error=TypeError, + error_msg="is_python_build() takes 0 positional arguments but 1 were given", + ): + sysconfig.is_python_build('foo') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_syslog.py b/Lib/test/test_syslog.py index 96945bfd8bf..b378d62e5cf 100644 --- a/Lib/test/test_syslog.py +++ b/Lib/test/test_syslog.py @@ -55,8 +55,6 @@ def test_openlog_noargs(self): syslog.openlog() syslog.syslog('test message from python test_syslog') - # TODO: RUSTPYTHON; AttributeError: module 'sys' has no attribute 'getswitchinterval' - @unittest.expectedFailure @threading_helper.requires_working_threading() def test_syslog_threaded(self): start = threading.Event() diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index 74d3a71a35f..d7a77eb26e4 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -14,7 +14,7 @@ findfile) from test.support.os_helper import unlink -import unittest # XXX: RUSTPYTHON +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests SOURCE_CODES = { @@ -112,9 +112,10 @@ def test_errprint(self): for args, expected in tests: with self.subTest(arguments=args, expected=expected): - with captured_stderr() as stderr: - tabnanny.errprint(*args) - self.assertEqual(stderr.getvalue() , expected) + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + tabnanny.errprint(*args) + self.assertEqual(stderr.getvalue() , expected) class TestNannyNag(TestCase): @@ -199,23 +200,24 @@ def test_correct_directory(self): with TemporaryPyFile(SOURCE_CODES["error_free"], directory=tmp_dir): self.verify_tabnanny_check(tmp_dir) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_when_wrong_indented(self): """A python source code file eligible for raising `IndentationError`.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: err = ('unindent does not match any outer indentation level' ' (<tokenize>, line 3)\n') err = f"{file_path!r}: Indentation Error: {err}" - self.verify_tabnanny_check(file_path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(file_path, err=err) def test_when_tokenize_tokenerror(self): """A python source code file eligible for raising 'tokenize.TokenError'.""" with TemporaryPyFile(SOURCE_CODES["incomplete_expression"]) as file_path: err = "('EOF in multi-line statement', (7, 0))\n" err = f"{file_path!r}: Token Error: {err}" - self.verify_tabnanny_check(file_path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(file_path, err=err) + @unittest.expectedFailure # TODO: RUSTPYTHON; A python source code file eligible for raising `tabnanny.NannyNag`. def test_when_nannynag_error_verbose(self): """A python source code file eligible for raising `tabnanny.NannyNag`. @@ -223,29 +225,27 @@ def test_when_nannynag_error_verbose(self): """ with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: out = f"{file_path!r}: *** Line 3: trouble in tab city! ***\n" - out += "offending line: '\\tprint(\"world\")\\n'\n" - out += "indent not equal e.g. at tab size 1\n" + out += "offending line: '\\tprint(\"world\")'\n" + out += "inconsistent use of tabs and spaces in indentation\n" tabnanny.verbose = 1 self.verify_tabnanny_check(file_path, out=out) + @unittest.expectedFailure # TODO: RUSTPYTHON; A python source code file eligible for raising `tabnanny.NannyNag`. def test_when_nannynag_error(self): """A python source code file eligible for raising `tabnanny.NannyNag`.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: - out = f"{file_path} 3 '\\tprint(\"world\")\\n'\n" + out = f"{file_path} 3 '\\tprint(\"world\")'\n" self.verify_tabnanny_check(file_path, out=out) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_when_no_file(self): """A python file which does not exist actually in system.""" path = 'no_file.py' err = (f"{path!r}: I/O Error: [Errno {errno.ENOENT}] " f"{os.strerror(errno.ENOENT)}: {path!r}\n") - self.verify_tabnanny_check(path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(path, err=err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errored_directory(self): """Directory containing wrongly indented python source code files.""" with tempfile.TemporaryDirectory() as tmp_dir: @@ -259,7 +259,8 @@ def test_errored_directory(self): err = ('unindent does not match any outer indentation level' ' (<tokenize>, line 3)\n') err = f"{e_file!r}: Indentation Error: {err}" - self.verify_tabnanny_check(tmp_dir, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(tmp_dir, err=err) class TestProcessTokens(TestCase): @@ -295,9 +296,12 @@ def test_with_errored_codes_samples(self): class TestCommandLine(TestCase): """Tests command line interface of `tabnanny`.""" - def validate_cmd(self, *args, stdout="", stderr="", partial=False): + def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failure=False): """Common function to assert the behaviour of command line interface.""" - _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) + if expect_failure: + _, out, err = script_helper.assert_python_failure('-m', 'tabnanny', *args) + else: + _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) # Note: The `splitlines()` will solve the problem of CRLF(\r) added # by OS Windows. out = os.fsdecode(out) @@ -312,15 +316,13 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False): self.assertListEqual(out.splitlines(), stdout.splitlines()) self.assertListEqual(err.splitlines(), stderr.splitlines()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_with_errored_file(self): """Should displays error when errored python file is given.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' - ' (<tokenize>, line 3)') - self.validate_cmd(file_path, stderr=stderr) + ' (<string>, line 3)') + self.validate_cmd(file_path, stderr=stderr, expect_failure=True) def test_with_error_free_file(self): """Should not display anything if python file is correctly indented.""" @@ -331,7 +333,7 @@ def test_command_usage(self): """Should display usage on no arguments.""" path = findfile('tabnanny.py') stderr = f"Usage: {path} [-v] file_or_directory ..." - self.validate_cmd(stderr=stderr) + self.validate_cmd(stderr=stderr, expect_failure=True) def test_quiet_flag(self): """Should display less when quite mode is on.""" @@ -339,18 +341,20 @@ def test_quiet_flag(self): stdout = f"{file_path}\n" self.validate_cmd("-q", file_path, stdout=stdout) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_verbose_mode(self): """Should display more error information if verbose mode is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: stdout = textwrap.dedent( - "offending line: '\\tprint(\"world\")\\n'" + "offending line: '\\tprint(\"world\")'" ).strip() self.validate_cmd("-v", path, stdout=stdout, partial=True) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_double_verbose_mode(self): """Should display detailed error information if double verbose is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: stdout = textwrap.dedent( - "offending line: '\\tprint(\"world\")\\n'" + "offending line: '\\tprint(\"world\")'" ).strip() self.validate_cmd("-vv", path, stdout=stdout, partial=True) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 3d6e5a5a97b..d6cbb350428 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1,18 +1,25 @@ +import errno import sys import os import io from hashlib import sha256 -from contextlib import contextmanager +from contextlib import contextmanager, ExitStack from random import Random import pathlib +import shutil +import re +import warnings +import stat import unittest import unittest.mock import tarfile +from test import archiver_tests from test import support from test.support import os_helper from test.support import script_helper +from test.support import warnings_helper # Check for our compression modules. try: @@ -31,18 +38,26 @@ import lzma except ImportError: lzma = None +# XXX: RUSTPYTHON; xz is not supported yet +lzma = None +try: + from compression import zstd +except ImportError: + zstd = None def sha256sum(data): return sha256(data).hexdigest() TEMPDIR = os.path.abspath(os_helper.TESTFN) + "-tardir" tarextdir = TEMPDIR + '-extract-test' -tarname = support.findfile("testtar.tar") +tarname = support.findfile("testtar.tar", subdir="archivetestdata") gzipname = os.path.join(TEMPDIR, "testtar.tar.gz") bz2name = os.path.join(TEMPDIR, "testtar.tar.bz2") xzname = os.path.join(TEMPDIR, "testtar.tar.xz") +zstname = os.path.join(TEMPDIR, "testtar.tar.zst") tmpname = os.path.join(TEMPDIR, "tmp.tar") dotlessname = os.path.join(TEMPDIR, "testtar") +SPACE = b" " sha256_regtype = ( "e09e4bc8b3c9d9177e77256353b36c159f5f040531bbd4b024a8f9b9196c71ce" @@ -83,6 +98,12 @@ class LzmaTest: open = lzma.LZMAFile if lzma else None taropen = tarfile.TarFile.xzopen +@support.requires_zstd() +class ZstdTest: + tarname = zstname + suffix = 'zst' + open = zstd.ZstdFile if zstd else None + taropen = tarfile.TarFile.zstopen class ReadTest(TarTest): @@ -95,6 +116,14 @@ def setUp(self): def tearDown(self): self.tar.close() +class StreamModeTest(ReadTest): + + # Only needs to change how the tarfile is opened to set + # stream mode + def setUp(self): + self.tar = tarfile.open(self.tarname, mode=self.mode, + encoding="iso8859-1", + stream=True) class UstarReadTest(ReadTest, unittest.TestCase): @@ -108,7 +137,7 @@ def test_fileobj_regular_file(self): "regular file extraction failed") def test_fileobj_readlines(self): - self.tar.extract("ustar/regtype", TEMPDIR) + self.tar.extract("ustar/regtype", TEMPDIR, filter='data') tarinfo = self.tar.getmember("ustar/regtype") with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1: lines1 = fobj1.readlines() @@ -126,7 +155,7 @@ def test_fileobj_readlines(self): "fileobj.readlines() failed") def test_fileobj_iter(self): - self.tar.extract("ustar/regtype", TEMPDIR) + self.tar.extract("ustar/regtype", TEMPDIR, filter='data') tarinfo = self.tar.getmember("ustar/regtype") with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1: lines1 = fobj1.readlines() @@ -136,7 +165,8 @@ def test_fileobj_iter(self): "fileobj.__iter__() failed") def test_fileobj_seek(self): - self.tar.extract("ustar/regtype", TEMPDIR) + self.tar.extract("ustar/regtype", TEMPDIR, + filter='data') with open(os.path.join(TEMPDIR, "ustar/regtype"), "rb") as fobj: data = fobj.read() @@ -225,13 +255,19 @@ def test_add_dir_getmember(self): self.add_dir_and_getmember('bar') self.add_dir_and_getmember('a'*101) + @unittest.skipUnless(hasattr(os, "getuid") and hasattr(os, "getgid"), + "Missing getuid or getgid implementation") def add_dir_and_getmember(self, name): + def filter(tarinfo): + tarinfo.uid = tarinfo.gid = 100 + return tarinfo + with os_helper.temp_cwd(): with tarfile.open(tmpname, 'w') as tar: tar.format = tarfile.USTAR_FORMAT try: os.mkdir(name) - tar.add(name) + tar.add(name, filter=filter) finally: os.rmdir(name) with tarfile.open(tmpname) as tar: @@ -249,6 +285,8 @@ class Bz2UstarReadTest(Bz2Test, UstarReadTest): class LzmaUstarReadTest(LzmaTest, UstarReadTest): pass +class ZstdUstarReadTest(ZstdTest, UstarReadTest): + pass class ListTest(ReadTest, unittest.TestCase): @@ -256,8 +294,6 @@ class ListTest(ReadTest, unittest.TestCase): def setUp(self): self.tar = tarfile.open(self.tarname, mode=self.mode) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list(self): tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') with support.swap_attr(sys, 'stdout', tio): @@ -295,8 +331,6 @@ def conv(b): self.assertNotIn(b'link to', out) self.assertNotIn(b'->', out) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_verbose(self): tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') with support.swap_attr(sys, 'stdout', tio): @@ -306,11 +340,23 @@ def test_list_verbose(self): # accessories if verbose flag is being used # ... # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/conttype - # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype + # -rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype + # drwxr-xr-x tarfile/tarfile 0 2003-01-05 15:19:43 ustar/dirtype/ # ... - self.assertRegex(out, (br'\?rw-r--r-- tarfile/tarfile\s+7011 ' - br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d ' - br'ustar/\w+type ?\r?\n') * 2) + # + # Array of values to modify the regex below: + # ((file_type, file_permissions, file_length), ...) + type_perm_lengths = ( + (br'\?', b'rw-r--r--', b'7011'), (b'-', b'rw-r--r--', b'7011'), + (b'd', b'rwxr-xr-x', b'0'), (b'd', b'rwxr-xr-x', b'255'), + (br'\?', b'rw-r--r--', b'0'), (b'l', b'rwxrwxrwx', b'0'), + (b'b', b'rw-rw----', b'3,0'), (b'c', b'rw-rw-rw-', b'1,3'), + (b'p', b'rw-r--r--', b'0')) + self.assertRegex(out, b''.join( + [(tp + (br'%s tarfile/tarfile\s+%s ' % (perm, ln) + + br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d ' + br'ustar/\w+type[/>\sa-z-]*\n')) for tp, perm, ln + in type_perm_lengths])) # Make sure it prints the source of link with verbose flag self.assertIn(b'ustar/symtype -> regtype', out) self.assertIn(b'./ustar/linktest2/symtype -> ../linktest1/regtype', out) @@ -321,8 +367,6 @@ def test_list_verbose(self): self.assertIn(b'pax' + (b'/123' * 125) + b'/longlink link to pax' + (b'/123' * 125) + b'/longname', out) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_members(self): tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') def members(tar): @@ -347,6 +391,8 @@ class Bz2ListTest(Bz2Test, ListTest): class LzmaListTest(LzmaTest, ListTest): pass +class ZstdListTest(ZstdTest, ListTest): + pass class CommonReadTest(ReadTest): @@ -358,7 +404,7 @@ def test_is_tarfile_erroneous(self): self.assertFalse(tarfile.is_tarfile(tmpname)) # is_tarfile works on path-like objects - self.assertFalse(tarfile.is_tarfile(pathlib.Path(tmpname))) + self.assertFalse(tarfile.is_tarfile(os_helper.FakePath(tmpname))) # is_tarfile works on file objects with open(tmpname, "rb") as fobj: @@ -372,7 +418,7 @@ def test_is_tarfile_valid(self): self.assertTrue(tarfile.is_tarfile(self.tarname)) # is_tarfile works on path-like objects - self.assertTrue(tarfile.is_tarfile(pathlib.Path(self.tarname))) + self.assertTrue(tarfile.is_tarfile(os_helper.FakePath(self.tarname))) # is_tarfile works on file objects with open(self.tarname, "rb") as fobj: @@ -382,6 +428,18 @@ def test_is_tarfile_valid(self): with open(self.tarname, "rb") as fobj: self.assertTrue(tarfile.is_tarfile(io.BytesIO(fobj.read()))) + def test_is_tarfile_keeps_position(self): + # Test for issue44289: tarfile.is_tarfile() modifies + # file object's current position + with open(self.tarname, "rb") as fobj: + tarfile.is_tarfile(fobj) + self.assertEqual(fobj.tell(), 0) + + with open(self.tarname, "rb") as fobj: + file_like = io.BytesIO(fobj.read()) + tarfile.is_tarfile(file_like) + self.assertEqual(file_like.tell(), 0) + def test_empty_tarfile(self): # Test for issue6123: Allow opening empty archives. # This test checks if tarfile.open() is able to open an empty tar @@ -455,25 +513,48 @@ def test_premature_end_of_archive(self): t = tar.next() with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"): - tar.extract(t, TEMPDIR) + tar.extract(t, TEMPDIR, filter='data') with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"): tar.extractfile(t).read() - @unittest.skip("TODO: RUSTPYTHON, infinite recursion") def test_length_zero_header(self): # bpo-39017 (CVE-2019-20907): reading a zero-length header should fail # with an exception with self.assertRaisesRegex(tarfile.ReadError, "file could not be opened successfully"): - with tarfile.open(support.findfile('recursion.tar')) as tar: + with tarfile.open(support.findfile('recursion.tar', subdir='archivetestdata')): pass + def test_extractfile_attrs(self): + # gh-74468: TarFile.name must name a file, not a parent archive. + file = self.tar.getmember('ustar/regtype') + with self.tar.extractfile(file) as fobj: + self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) + self.assertIs(fobj.closed, False) + self.assertIs(fobj.closed, True) + self.assertEqual(fobj.name, 'ustar/regtype') + self.assertRaises(AttributeError, fobj.fileno) + self.assertEqual(fobj.mode, 'rb') + self.assertIs(fobj.readable(), True) + self.assertIs(fobj.writable(), False) + if self.is_stream: + self.assertRaises(AttributeError, fobj.seekable) + else: + self.assertIs(fobj.seekable(), True) + + class MiscReadTestBase(CommonReadTest): - def requires_name_attribute(self): - pass + is_stream = False def test_no_name_argument(self): - self.requires_name_attribute() with open(self.tarname, "rb") as fobj: self.assertIsInstance(fobj.name, str) with tarfile.open(fileobj=fobj, mode=self.mode) as tar: @@ -506,7 +587,6 @@ def test_int_name_attribute(self): self.assertIsNone(tar.name) def test_bytes_name_attribute(self): - self.requires_name_attribute() tarname = os.fsencode(self.tarname) with open(tarname, 'rb') as fobj: self.assertIsInstance(fobj.name, bytes) @@ -514,21 +594,23 @@ def test_bytes_name_attribute(self): self.assertIsInstance(tar.name, bytes) self.assertEqual(tar.name, os.path.abspath(fobj.name)) - def test_pathlike_name(self): - tarname = pathlib.Path(self.tarname) + def test_pathlike_name(self, tarname=None): + if tarname is None: + tarname = self.tarname + expected = os.path.abspath(tarname) + tarname = os_helper.FakePath(tarname) with tarfile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with self.taropen(tarname) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) with tarfile.TarFile.open(tarname, mode=self.mode) as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) if self.suffix == '': with tarfile.TarFile(tarname, mode='r') as tar: - self.assertIsInstance(tar.name, str) - self.assertEqual(tar.name, os.path.abspath(os.fspath(tarname))) + self.assertEqual(tar.name, expected) + + def test_pathlike_bytes_name(self): + self.test_pathlike_name(os.fsencode(self.tarname)) def test_illegal_mode_arg(self): with open(tmpname, 'wb'): @@ -611,22 +693,22 @@ def test_find_members(self): def test_extract_hardlink(self): # Test hardlink extraction (e.g. bug #857297). with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar: - tar.extract("ustar/regtype", TEMPDIR) + tar.extract("ustar/regtype", TEMPDIR, filter='data') self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/regtype")) - tar.extract("ustar/lnktype", TEMPDIR) + tar.extract("ustar/lnktype", TEMPDIR, filter='data') self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/lnktype")) with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f: data = f.read() self.assertEqual(sha256sum(data), sha256_regtype) - tar.extract("ustar/symtype", TEMPDIR) + tar.extract("ustar/symtype", TEMPDIR, filter='data') self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/symtype")) with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f: data = f.read() self.assertEqual(sha256sum(data), sha256_regtype) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @os_helper.skip_unless_working_chmod def test_extractall(self): # Test if extractall() correctly restores directory permissions # and times (see issue1735). @@ -635,13 +717,14 @@ def test_extractall(self): os.mkdir(DIR) try: directories = [t for t in tar if t.isdir()] - tar.extractall(DIR, directories) + tar.extractall(DIR, directories, filter='fully_trusted') for tarinfo in directories: path = os.path.join(DIR, tarinfo.name) if sys.platform != "win32": # Win32 has no support for fine grained permissions. self.assertEqual(tarinfo.mode & 0o777, - os.stat(path).st_mode & 0o777) + os.stat(path).st_mode & 0o777, + tarinfo.name) def format_mtime(mtime): if isinstance(mtime, float): return "{} ({})".format(mtime, mtime.hex()) @@ -657,7 +740,25 @@ def format_mtime(mtime): tar.close() os_helper.rmtree(DIR) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @staticmethod + def test_extractall_default_filter(): + # Test that the default filter is now "data", and the other filter types are not used. + DIR = pathlib.Path(TEMPDIR) / "extractall_default_filter" + with ( + os_helper.temp_dir(DIR), + tarfile.open(tarname, encoding="iso8859-1") as tar, + unittest.mock.patch("tarfile.data_filter", wraps=tarfile.data_filter) as mock_data_filter, + unittest.mock.patch("tarfile.tar_filter", wraps=tarfile.tar_filter) as mock_tar_filter, + unittest.mock.patch("tarfile.fully_trusted_filter", wraps=tarfile.fully_trusted_filter) as mock_ft_filter + ): + directories = [t for t in tar if t.isdir()] + tar.extractall(DIR, directories) + + mock_data_filter.assert_called() + mock_ft_filter.assert_not_called() + mock_tar_filter.assert_not_called() + + @os_helper.skip_unless_working_chmod def test_extract_directory(self): dirtype = "ustar/dirtype" DIR = os.path.join(TEMPDIR, "extractdir") @@ -665,7 +766,7 @@ def test_extract_directory(self): try: with tarfile.open(tarname, encoding="iso8859-1") as tar: tarinfo = tar.getmember(dirtype) - tar.extract(tarinfo, path=DIR) + tar.extract(tarinfo, path=DIR, filter='fully_trusted') extracted = os.path.join(DIR, dirtype) self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) if sys.platform != "win32": @@ -673,26 +774,24 @@ def test_extract_directory(self): finally: os_helper.rmtree(DIR) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_extractall_pathlike_name(self): - DIR = pathlib.Path(TEMPDIR) / "extractall" + def test_extractall_pathlike_dir(self): + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: directories = [t for t in tar if t.isdir()] - tar.extractall(DIR, directories) + tar.extractall(os_helper.FakePath(DIR), directories, filter='fully_trusted') for tarinfo in directories: - path = DIR / tarinfo.name + path = os.path.join(DIR, tarinfo.name) self.assertEqual(os.path.getmtime(path), tarinfo.mtime) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - def test_extract_pathlike_name(self): + def test_extract_pathlike_dir(self): dirtype = "ustar/dirtype" - DIR = pathlib.Path(TEMPDIR) / "extractall" + DIR = os.path.join(TEMPDIR, "extractall") with os_helper.temp_dir(DIR), \ tarfile.open(tarname, encoding="iso8859-1") as tar: tarinfo = tar.getmember(dirtype) - tar.extract(tarinfo, path=DIR) - extracted = DIR / dirtype + tar.extract(tarinfo, path=os_helper.FakePath(DIR), filter='fully_trusted') + extracted = os.path.join(DIR, dirtype) self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime) def test_init_close_fobj(self): @@ -731,6 +830,69 @@ def test_zlib_error_does_not_leak(self): with self.assertRaises(tarfile.ReadError): tarfile.open(self.tarname) + def test_next_on_empty_tarfile(self): + fd = io.BytesIO() + tf = tarfile.open(fileobj=fd, mode="w") + tf.close() + + fd.seek(0) + with tarfile.open(fileobj=fd, mode="r|") as tf: + self.assertEqual(tf.next(), None) + + fd.seek(0) + with tarfile.open(fileobj=fd, mode="r") as tf: + self.assertEqual(tf.next(), None) + + def _setup_symlink_to_target(self, temp_dirpath): + target_filepath = os.path.join(temp_dirpath, "target") + ustar_dirpath = os.path.join(temp_dirpath, "ustar") + hardlink_filepath = os.path.join(ustar_dirpath, "lnktype") + with open(target_filepath, "wb") as f: + f.write(b"target") + os.makedirs(ustar_dirpath) + os.symlink(target_filepath, hardlink_filepath) + return target_filepath, hardlink_filepath + + def _assert_on_file_content(self, filepath, digest): + with open(filepath, "rb") as f: + data = f.read() + self.assertEqual(sha256sum(data), digest) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extract_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", DIR, filter="data") + tar.extract("ustar/lnktype", DIR, filter="data") + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extractall_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extractall( + DIR, members=["ustar/regtype", "ustar/lnktype"], filter="data", + ) + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None @@ -739,17 +901,18 @@ class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("BZ2File have no name attribute") + pass class LzmaMiscReadTest(LzmaTest, MiscReadTestBase, unittest.TestCase): - def requires_name_attribute(self): - self.skipTest("LZMAFile have no name attribute") + pass +class ZstdMiscReadTest(ZstdTest, MiscReadTestBase, unittest.TestCase): + pass class StreamReadTest(CommonReadTest, unittest.TestCase): prefix="r|" + is_stream = True def test_read_through(self): # Issue #11224: A poorly designed _FileInFile.read() method @@ -817,6 +980,27 @@ class Bz2StreamReadTest(Bz2Test, StreamReadTest): class LzmaStreamReadTest(LzmaTest, StreamReadTest): pass +class ZstdStreamReadTest(ZstdTest, StreamReadTest): + pass + +class TarStreamModeReadTest(StreamModeTest, unittest.TestCase): + + def test_stream_mode_no_cache(self): + for _ in self.tar: + pass + self.assertEqual(self.tar.members, []) + +class GzipStreamModeReadTest(GzipTest, TarStreamModeReadTest): + pass + +class Bz2StreamModeReadTest(Bz2Test, TarStreamModeReadTest): + pass + +class LzmaStreamModeReadTest(LzmaTest, TarStreamModeReadTest): + pass + +class ZstdStreamModeReadTest(ZstdTest, TarStreamModeReadTest): + pass class DetectReadTest(TarTest, unittest.TestCase): def _testfunc_file(self, name, mode): @@ -879,6 +1063,25 @@ def test_detect_stream_bz2(self): class LzmaDetectReadTest(LzmaTest, DetectReadTest): pass +class ZstdDetectReadTest(ZstdTest, DetectReadTest): + pass + +class GzipBrokenHeaderCorrectException(GzipTest, unittest.TestCase): + """ + See: https://github.com/python/cpython/issues/107396 + """ + def runTest(self): + f = io.BytesIO( + b'\x1f\x8b' # header + b'\x08' # compression method + b'\x04' # flags + b'\0\0\0\0\0\0' # timestamp, compression data, OS ID + b'\0\x01' # size + b'\0\0\0\0\0' # corrupt data (zeros) + ) + with self.assertRaises(tarfile.ReadError): + tarfile.open(fileobj=f, mode='r|gz') + class MemberReadTest(ReadTest, unittest.TestCase): @@ -1028,7 +1231,7 @@ def test_longname_directory(self): os.mkdir(longdir) tar.add(longdir) finally: - os.rmdir(longdir) + os.rmdir(longdir.rstrip("/")) with tarfile.open(tmpname) as tar: self.assertIsNotNone(tar.getmember(longdir)) self.assertIsNotNone(tar.getmember(longdir.removesuffix('/'))) @@ -1047,7 +1250,7 @@ class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase): # an all platforms, and after that a test that will work only on # platforms/filesystems that prove to support sparse files. def _test_sparse_file(self, name): - self.tar.extract(name, TEMPDIR) + self.tar.extract(name, TEMPDIR, filter='data') filename = os.path.join(TEMPDIR, name) with open(filename, "rb") as fobj: data = fobj.read() @@ -1058,19 +1261,15 @@ def _test_sparse_file(self, name): s = os.stat(filename) self.assertLess(s.st_blocks * 512, s.st_size) - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_sparse_file_old(self): self._test_sparse_file("gnu/sparse") - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_sparse_file_00(self): self._test_sparse_file("gnu/sparse-0.0") - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_sparse_file_01(self): self._test_sparse_file("gnu/sparse-0.1") - @unittest.expectedFailureIf(sys.platform == "linux", "TODO: RUSTPYTHON") def test_sparse_file_10(self): self._test_sparse_file("gnu/sparse-1.0") @@ -1082,7 +1281,7 @@ def _fs_supports_holes(): # # The function returns False if page size is larger than 4 KiB. # For example, ppc64 uses pages of 64 KiB. - if sys.platform.startswith("linux"): + if sys.platform.startswith(("linux", "android")): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: @@ -1141,6 +1340,48 @@ def test_pax_number_fields(self): finally: tar.close() + def test_pax_header_bad_formats(self): + # The fields from the pax header have priority over the + # TarInfo. + pax_header_replacements = ( + b" foo=bar\n", + b"0 \n", + b"1 \n", + b"2 \n", + b"3 =\n", + b"4 =a\n", + b"1000000 foo=bar\n", + b"0 foo=bar\n", + b"-12 foo=bar\n", + b"000000000000000000000000036 foo=bar\n", + ) + pax_headers = {"foo": "bar"} + + for replacement in pax_header_replacements: + with self.subTest(header=replacement): + tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT, + encoding="iso8859-1") + try: + t = tarfile.TarInfo() + t.name = "pax" # non-ASCII + t.uid = 1 + t.pax_headers = pax_headers + tar.addfile(t) + finally: + tar.close() + + with open(tmpname, "rb") as f: + data = f.read() + self.assertIn(b"11 foo=bar\n", data) + data = data.replace(b"11 foo=bar\n", replacement) + + with open(tmpname, "wb") as f: + f.truncate() + f.write(data) + + with self.assertRaisesRegex(tarfile.ReadError, r"method tar: ReadError\('invalid header'\)"): + tarfile.open(tmpname, encoding="iso8859-1") + class WriteTestBase(TarTest): # Put all write tests in here that are supposed to be tested @@ -1265,16 +1506,15 @@ def test_ordered_recursion(self): def test_gettarinfo_pathlike_name(self): with tarfile.open(tmpname, self.mode) as tar: - path = pathlib.Path(TEMPDIR) / "file" + path = os.path.join(TEMPDIR, "file") with open(path, "wb") as fobj: fobj.write(b"aaa") - tarinfo = tar.gettarinfo(path) - tarinfo2 = tar.gettarinfo(os.fspath(path)) + tarinfo = tar.gettarinfo(os_helper.FakePath(path)) + tarinfo2 = tar.gettarinfo(path) self.assertIsInstance(tarinfo.name, str) self.assertEqual(tarinfo.name, tarinfo2.name) self.assertEqual(tarinfo.size, 3) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1419,7 +1659,8 @@ def test_extractall_symlinks(self): with tarfile.open(temparchive, errorlevel=2) as tar: # this should not raise OSError: [Errno 17] File exists try: - tar.extractall(path=tempdir) + tar.extractall(path=tempdir, + filter='fully_trusted') except OSError: self.fail("extractall failed with symlinked files") finally: @@ -1463,7 +1704,7 @@ def test_cwd(self): try: for t in tar: if t.name != ".": - self.assertTrue(t.name.startswith("./"), t.name) + self.assertStartsWith(t.name, "./") finally: tar.close() @@ -1477,30 +1718,23 @@ def write(self, data): raise exctype f = BadFile() - with self.assertRaises(exctype): - tar = tarfile.open(tmpname, self.mode, fileobj=f, - format=tarfile.PAX_FORMAT, - pax_headers={'non': 'empty'}) + with ( + warnings_helper.check_no_resource_warning(self), + self.assertRaises(exctype), + ): + tarfile.open(tmpname, self.mode, fileobj=f, + format=tarfile.PAX_FORMAT, + pax_headers={'non': 'empty'}) self.assertFalse(f.closed) + def test_missing_fileobj(self): + with tarfile.open(tmpname, self.mode) as tar: + tarinfo = tar.gettarinfo(tarname) + with self.assertRaises(ValueError): + tar.addfile(tarinfo) + class GzipWriteTest(GzipTest, WriteTest): - # TODO: RUSTPYTHON - def expectedSuccess(test_item): - test_item.__unittest_expecting_failure__ = False - return test_item - - # TODO: RUSTPYTHON - if sys.platform == "win32": - @expectedSuccess - def test_cwd(self): - super().test_cwd() - - # TODO: RUSTPYTHON - if sys.platform == "win32": - @expectedSuccess - def test_symlink_size(self): - super().test_symlink_size() pass @@ -1511,6 +1745,8 @@ class Bz2WriteTest(Bz2Test, WriteTest): class LzmaWriteTest(LzmaTest, WriteTest): pass +class ZstdWriteTest(ZstdTest, WriteTest): + pass class StreamWriteTest(WriteTestBase, unittest.TestCase): @@ -1535,6 +1771,10 @@ def test_stream_padding(self): @unittest.skipUnless(sys.platform != "win32" and hasattr(os, "umask"), "Missing umask implementation") + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "Emscripten's/WASI's umask is a stub." + ) def test_file_mode(self): # Test for issue #8464: Create files with correct # permissions. @@ -1550,6 +1790,16 @@ def test_file_mode(self): finally: os.umask(original_umask) + def test_pathlike_name(self): + expected_name = os.path.abspath(tmpname) + tarpath = os_helper.FakePath(tmpname) + + for func in (tarfile.open, tarfile.TarFile.open): + with self.subTest(): + with func(tarpath, self.mode) as tar: + self.assertEqual(tar.name, expected_name) + os_helper.unlink(tmpname) + class GzipStreamWriteTest(GzipTest, StreamWriteTest): def test_source_directory_not_leaked(self): @@ -1568,6 +1818,78 @@ class Bz2StreamWriteTest(Bz2Test, StreamWriteTest): class LzmaStreamWriteTest(LzmaTest, StreamWriteTest): decompressor = lzma.LZMADecompressor if lzma else None +class ZstdStreamWriteTest(ZstdTest, StreamWriteTest): + decompressor = zstd.ZstdDecompressor if zstd else None + +class _CompressedWriteTest(TarTest): + # This is not actually a standalone test. + # It does not inherit WriteTest because it only makes sense with gz,bz2 + source = (b"And we move to Bristol where they have a special, " + + b"Very Silly candidate") + + def _compressed_tar(self, compresslevel): + fobj = io.BytesIO() + with tarfile.open(tmpname, self.mode, fobj, + compresslevel=compresslevel) as tarfl: + tarfl.addfile(tarfile.TarInfo("foo"), io.BytesIO(self.source)) + return fobj + + def _test_bz2_header(self, compresslevel): + fobj = self._compressed_tar(compresslevel) + self.assertEqual(fobj.getvalue()[0:10], + b"BZh%d1AY&SY" % compresslevel) + + def _test_gz_header(self, compresslevel): + fobj = self._compressed_tar(compresslevel) + self.assertEqual(fobj.getvalue()[:3], b"\x1f\x8b\x08") + +class Bz2CompressWriteTest(Bz2Test, _CompressedWriteTest, unittest.TestCase): + prefix = "w:" + def test_compression_levels(self): + self._test_bz2_header(1) + self._test_bz2_header(5) + self._test_bz2_header(9) + +class Bz2CompressStreamWriteTest(Bz2Test, _CompressedWriteTest, + unittest.TestCase): + prefix = "w|" + def test_compression_levels(self): + self._test_bz2_header(1) + self._test_bz2_header(5) + self._test_bz2_header(9) + +class GzCompressWriteTest(GzipTest, _CompressedWriteTest, unittest.TestCase): + prefix = "w:" + def test_compression_levels(self): + self._test_gz_header(1) + self._test_gz_header(5) + self._test_gz_header(9) + +class GzCompressStreamWriteTest(GzipTest, _CompressedWriteTest, + unittest.TestCase): + prefix = "w|" + def test_compression_levels(self): + self._test_gz_header(1) + self._test_gz_header(5) + self._test_gz_header(9) + +class CompressLevelRaises(unittest.TestCase): + def test_compresslevel_wrong_modes(self): + compresslevel = 5 + fobj = io.BytesIO() + with self.assertRaises(TypeError): + tarfile.open(tmpname, "w:", fobj, compresslevel=compresslevel) + + @support.requires_bz2() + def test_wrong_compresslevels(self): + # BZ2 checks that the compresslevel is in [1,9]. gz does not + fobj = io.BytesIO() + with self.assertRaises(ValueError): + tarfile.open(tmpname, "w:bz2", fobj, compresslevel=0) + with self.assertRaises(ValueError): + tarfile.open(tmpname, "w:bz2", fobj, compresslevel=10) + with self.assertRaises(ValueError): + tarfile.open(tmpname, "w|bz2", fobj, compresslevel=10) class GNUWriteTest(unittest.TestCase): # This testcase checks for correct creation of GNU Longname @@ -1759,10 +2081,10 @@ def test_create_existing_taropen(self): self.assertIn("spameggs42", names[0]) def test_create_pathlike_name(self): - with tarfile.open(pathlib.Path(tmpname), self.mode) as tobj: + with tarfile.open(os_helper.FakePath(tmpname), self.mode) as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -1773,10 +2095,10 @@ def test_create_pathlike_name(self): self.assertIn('spameggs42', names[0]) def test_create_taropen_pathlike_name(self): - with self.taropen(pathlib.Path(tmpname), "x") as tobj: + with self.taropen(os_helper.FakePath(tmpname), "x") as tobj: self.assertIsInstance(tobj.name, str) self.assertEqual(tobj.name, os.path.abspath(tmpname)) - tobj.add(pathlib.Path(self.file_path)) + tobj.add(os_helper.FakePath(self.file_path)) names = tobj.getnames() self.assertEqual(len(names), 1) self.assertIn('spameggs42', names[0]) @@ -1814,6 +2136,14 @@ def test_create_with_preset(self): tobj.add(self.file_path) +class ZstdCreateTest(ZstdTest, CreateTest): + + # Unlike gz and bz2, zstd uses the level keyword instead of compresslevel. + # It does not allow for level to be specified when reading. + def test_create_with_level(self): + with tarfile.open(tmpname, self.mode, level=1) as tobj: + tobj.add(self.file_path) + class CreateWithXModeTest(CreateTest): prefix = "x" @@ -1853,7 +2183,6 @@ def test_add_twice(self): self.assertEqual(tarinfo.type, tarfile.REGTYPE, "add file as regular failed") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_add_hardlink(self): tarinfo = self.tar.gettarinfo(self.bar) self.assertEqual(tarinfo.type, tarfile.LNKTYPE, @@ -2010,8 +2339,6 @@ class UnicodeTest: def test_iso8859_1_filename(self): self._test_unicode_filename("iso8859-1") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_utf7_filename(self): self._test_unicode_filename("utf7") @@ -2092,11 +2419,6 @@ class UstarUnicodeTest(UnicodeTest, unittest.TestCase): format = tarfile.USTAR_FORMAT - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_uname_unicode(self): - super().test_uname_unicode() - # Test whether the utf-8 encoded version of a filename exceeds the 100 # bytes name field limit (every occurrence of '\xff' will be expanded to 2 # bytes). @@ -2176,13 +2498,6 @@ class GNUUnicodeTest(UnicodeTest, unittest.TestCase): format = tarfile.GNU_FORMAT - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_uname_unicode(self): - super().test_uname_unicode() - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_pax_header(self): # Test for issue #8633. GNU tar <= 1.23 creates raw binary fields # without a hdrcharset=BINARY header. @@ -2204,8 +2519,6 @@ class PAXUnicodeTest(UnicodeTest, unittest.TestCase): # PAX_FORMAT ignores encoding in write mode. test_unicode_filename_error = None - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_binary_header(self): # Test a POSIX.1-2008 compatible header with a hdrcharset=BINARY field. for encoding, name in ( @@ -2312,6 +2625,8 @@ class Bz2AppendTest(Bz2Test, AppendTestBase, unittest.TestCase): class LzmaAppendTest(LzmaTest, AppendTestBase, unittest.TestCase): pass +class ZstdAppendTest(ZstdTest, AppendTestBase, unittest.TestCase): + pass class LimitsTest(unittest.TestCase): @@ -2452,10 +2767,8 @@ def test__all__(self): 'SubsequentHeaderError', 'ExFileObject', 'main'} support.check__all__(self, tarfile, not_exported=not_exported) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_useful_error_message_when_modules_missing(self): - fname = os.path.join(os.path.dirname(__file__), 'testtar.tar.xz') + fname = os.path.join(os.path.dirname(__file__), 'archivetestdata', 'testtar.tar.xz') with self.assertRaises(tarfile.ReadError) as excinfo: error = tarfile.CompressionError('lzma module is not available'), with unittest.mock.patch.object(tarfile.TarFile, 'xzopen', side_effect=error): @@ -2466,6 +2779,31 @@ def test_useful_error_message_when_modules_missing(self): str(excinfo.exception), ) + @unittest.skipUnless(os_helper.can_symlink(), 'requires symlink support') + @unittest.skipUnless(hasattr(os, 'chmod'), "missing os.chmod") + @unittest.mock.patch('os.chmod') + def test_deferred_directory_attributes_update(self, mock_chmod): + # Regression test for gh-127987: setting attributes on arbitrary files + tempdir = os.path.join(TEMPDIR, 'test127987') + def mock_chmod_side_effect(path, mode, **kwargs): + target_path = os.path.realpath(path) + if os.path.commonpath([target_path, tempdir]) != tempdir: + raise Exception("should not try to chmod anything outside the destination", target_path) + mock_chmod.side_effect = mock_chmod_side_effect + + outside_tree_dir = os.path.join(TEMPDIR, 'outside_tree_dir') + with ArchiveMaker() as arc: + arc.add('x', symlink_to='.') + arc.add('x', type=tarfile.DIRTYPE, mode='?rwsrwsrwt') + arc.add('x', symlink_to=outside_tree_dir) + + os.makedirs(outside_tree_dir) + try: + arc.open().extractall(path=tempdir, filter='tar') + finally: + os_helper.rmtree(outside_tree_dir) + os_helper.rmtree(tempdir) + class CommandLineTest(unittest.TestCase): @@ -2478,14 +2816,24 @@ def tarfilecmd_failure(self, *args): return script_helper.assert_python_failure('-m', 'tarfile', *args) def make_simple_tarfile(self, tar_name): - files = [support.findfile('tokenize_tests.txt'), + files = [support.findfile('tokenize_tests.txt', + subdir='tokenizedata'), support.findfile('tokenize_tests-no-coding-cookie-' - 'and-utf8-bom-sig-only.txt')] + 'and-utf8-bom-sig-only.txt', + subdir='tokenizedata')] self.addCleanup(os_helper.unlink, tar_name) with tarfile.open(tar_name, 'w') as tf: for tardata in files: tf.add(tardata, arcname=os.path.basename(tardata)) + def make_evil_tarfile(self, tar_name): + self.addCleanup(os_helper.unlink, tar_name) + with tarfile.open(tar_name, 'w') as tf: + benign = tarfile.TarInfo('benign') + tf.addfile(benign, fileobj=io.BytesIO(b'')) + evil = tarfile.TarInfo('../evil') + tf.addfile(evil, fileobj=io.BytesIO(b'')) + def test_bad_use(self): rc, out, err = self.tarfilecmd_failure() self.assertEqual(out, b'') @@ -2510,7 +2858,7 @@ def test_test_command_verbose(self): self.assertIn(b'is a tar archive.\n', out) def test_test_command_invalid_file(self): - zipname = support.findfile('zipdir.zip') + zipname = support.findfile('zipdir.zip', subdir='archivetestdata') rc, out, err = self.tarfilecmd_failure('-t', zipname) self.assertIn(b' is not a tar archive.', err) self.assertEqual(out, b'') @@ -2529,8 +2877,6 @@ def test_test_command_invalid_file(self): finally: os_helper.unlink(tmpname) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_command(self): for tar_name in testtarnames: with support.captured_stdout() as t: @@ -2542,8 +2888,6 @@ def test_list_command(self): PYTHONIOENCODING='ascii') self.assertEqual(out, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_command_verbose(self): for tar_name in testtarnames: with support.captured_stdout() as t: @@ -2556,16 +2900,18 @@ def test_list_command_verbose(self): self.assertEqual(out, expected) def test_list_command_invalid_file(self): - zipname = support.findfile('zipdir.zip') + zipname = support.findfile('zipdir.zip', subdir='archivetestdata') rc, out, err = self.tarfilecmd_failure('-l', zipname) self.assertIn(b' is not a tar archive.', err) self.assertEqual(out, b'') self.assertEqual(rc, 1) def test_create_command(self): - files = [support.findfile('tokenize_tests.txt'), + files = [support.findfile('tokenize_tests.txt', + subdir='tokenizedata'), support.findfile('tokenize_tests-no-coding-cookie-' - 'and-utf8-bom-sig-only.txt')] + 'and-utf8-bom-sig-only.txt', + subdir='tokenizedata')] for opt in '-c', '--create': try: out = self.tarfilecmd(opt, tmpname, *files) @@ -2576,9 +2922,11 @@ def test_create_command(self): os_helper.unlink(tmpname) def test_create_command_verbose(self): - files = [support.findfile('tokenize_tests.txt'), + files = [support.findfile('tokenize_tests.txt', + subdir='tokenizedata'), support.findfile('tokenize_tests-no-coding-cookie-' - 'and-utf8-bom-sig-only.txt')] + 'and-utf8-bom-sig-only.txt', + subdir='tokenizedata')] for opt in '-v', '--verbose': try: out = self.tarfilecmd(opt, '-c', tmpname, *files, @@ -2590,7 +2938,7 @@ def test_create_command_verbose(self): os_helper.unlink(tmpname) def test_create_command_dotless_filename(self): - files = [support.findfile('tokenize_tests.txt')] + files = [support.findfile('tokenize_tests.txt', subdir='tokenizedata')] try: out = self.tarfilecmd('-c', dotlessname, *files) self.assertEqual(out, b'') @@ -2601,7 +2949,7 @@ def test_create_command_dotless_filename(self): def test_create_command_dot_started_filename(self): tar_name = os.path.join(TEMPDIR, ".testtar") - files = [support.findfile('tokenize_tests.txt')] + files = [support.findfile('tokenize_tests.txt', subdir='tokenizedata')] try: out = self.tarfilecmd('-c', tar_name, *files) self.assertEqual(out, b'') @@ -2611,10 +2959,12 @@ def test_create_command_dot_started_filename(self): os_helper.unlink(tar_name) def test_create_command_compressed(self): - files = [support.findfile('tokenize_tests.txt'), + files = [support.findfile('tokenize_tests.txt', + subdir='tokenizedata'), support.findfile('tokenize_tests-no-coding-cookie-' - 'and-utf8-bom-sig-only.txt')] - for filetype in (GzipTest, Bz2Test, LzmaTest): + 'and-utf8-bom-sig-only.txt', + subdir='tokenizedata')] + for filetype in (GzipTest, Bz2Test, LzmaTest, ZstdTest): if not filetype.open: continue try: @@ -2646,6 +2996,25 @@ def test_extract_command_verbose(self): finally: os_helper.rmtree(tarextdir) + def test_extract_command_filter(self): + self.make_evil_tarfile(tmpname) + # Make an inner directory, so the member named '../evil' + # is still extracted into `tarextdir` + destdir = os.path.join(tarextdir, 'dest') + os.mkdir(tarextdir) + try: + with os_helper.temp_cwd(destdir): + self.tarfilecmd_failure('-e', tmpname, + '-v', + '--filter', 'data') + out = self.tarfilecmd('-e', tmpname, + '-v', + '--filter', 'fully_trusted', + PYTHONIOENCODING='utf-8') + self.assertIn(b' file is extracted.', out) + finally: + os_helper.rmtree(tarextdir) + def test_extract_command_different_directory(self): self.make_simple_tarfile(tmpname) try: @@ -2656,7 +3025,7 @@ def test_extract_command_different_directory(self): os_helper.rmtree(tarextdir) def test_extract_command_invalid_file(self): - zipname = support.findfile('zipdir.zip') + zipname = support.findfile('zipdir.zip', subdir='archivetestdata') with os_helper.temp_cwd(tarextdir): rc, out, err = self.tarfilecmd_failure('-e', zipname) self.assertIn(b' is not a tar archive.', err) @@ -2729,7 +3098,7 @@ class LinkEmulationTest(ReadTest, unittest.TestCase): # symbolic or hard links tarfile tries to extract these types of members # as the regular files they point to. def _test_link_extraction(self, name): - self.tar.extract(name, TEMPDIR) + self.tar.extract(name, TEMPDIR, filter='fully_trusted') with open(os.path.join(TEMPDIR, name), "rb") as f: data = f.read() self.assertEqual(sha256sum(data), sha256_regtype) @@ -2861,8 +3230,10 @@ def test_extract_with_numeric_owner(self, mock_geteuid, mock_chmod, mock_chown): with self._setup_test(mock_geteuid) as (tarfl, filename_1, _, filename_2): - tarfl.extract(filename_1, TEMPDIR, numeric_owner=True) - tarfl.extract(filename_2 , TEMPDIR, numeric_owner=True) + tarfl.extract(filename_1, TEMPDIR, numeric_owner=True, + filter='fully_trusted') + tarfl.extract(filename_2 , TEMPDIR, numeric_owner=True, + filter='fully_trusted') # convert to filesystem paths f_filename_1 = os.path.join(TEMPDIR, filename_1) @@ -2880,7 +3251,8 @@ def test_extractall_with_numeric_owner(self, mock_geteuid, mock_chmod, mock_chown): with self._setup_test(mock_geteuid) as (tarfl, filename_1, dirname_1, filename_2): - tarfl.extractall(TEMPDIR, numeric_owner=True) + tarfl.extractall(TEMPDIR, numeric_owner=True, + filter='fully_trusted') # convert to filesystem paths f_filename_1 = os.path.join(TEMPDIR, filename_1) @@ -2905,7 +3277,8 @@ def test_extractall_with_numeric_owner(self, mock_geteuid, mock_chmod, def test_extract_without_numeric_owner(self, mock_geteuid, mock_chmod, mock_chown): with self._setup_test(mock_geteuid) as (tarfl, filename_1, _, _): - tarfl.extract(filename_1, TEMPDIR, numeric_owner=False) + tarfl.extract(filename_1, TEMPDIR, numeric_owner=False, + filter='fully_trusted') # convert to filesystem paths f_filename_1 = os.path.join(TEMPDIR, filename_1) @@ -2919,6 +3292,1535 @@ def test_keyword_only(self, mock_geteuid): tarfl.extract, filename_1, TEMPDIR, False, True) +class ReplaceTests(ReadTest, unittest.TestCase): + def test_replace_name(self): + member = self.tar.getmember('ustar/regtype') + replaced = member.replace(name='misc/other') + self.assertEqual(replaced.name, 'misc/other') + self.assertEqual(member.name, 'ustar/regtype') + self.assertEqual(self.tar.getmember('ustar/regtype').name, + 'ustar/regtype') + + def test_replace_deep(self): + member = self.tar.getmember('pax/regtype1') + replaced = member.replace() + replaced.pax_headers['gname'] = 'not-bar' + self.assertEqual(member.pax_headers['gname'], 'bar') + self.assertEqual( + self.tar.getmember('pax/regtype1').pax_headers['gname'], 'bar') + + def test_replace_shallow(self): + member = self.tar.getmember('pax/regtype1') + replaced = member.replace(deep=False) + replaced.pax_headers['gname'] = 'not-bar' + self.assertEqual(member.pax_headers['gname'], 'not-bar') + self.assertEqual( + self.tar.getmember('pax/regtype1').pax_headers['gname'], 'not-bar') + + def test_replace_all(self): + member = self.tar.getmember('ustar/regtype') + for attr_name in ('name', 'mtime', 'mode', 'linkname', + 'uid', 'gid', 'uname', 'gname'): + with self.subTest(attr_name=attr_name): + replaced = member.replace(**{attr_name: None}) + self.assertEqual(getattr(replaced, attr_name), None) + self.assertNotEqual(getattr(member, attr_name), None) + + def test_replace_internal(self): + member = self.tar.getmember('ustar/regtype') + with self.assertRaises(TypeError): + member.replace(offset=123456789) + + +class NoneInfoExtractTests(ReadTest): + # These mainly check that all kinds of members are extracted successfully + # if some metadata is None. + # Some of the methods do additional spot checks. + + # We also test that the default filters can deal with None. + + extraction_filter = None + + @classmethod + def setUpClass(cls): + tar = tarfile.open(tarname, mode='r', encoding="iso8859-1") + cls.control_dir = pathlib.Path(TEMPDIR) / "extractall_ctrl" + tar.errorlevel = 0 + with ExitStack() as cm: + if cls.extraction_filter is None: + cm.enter_context(warnings.catch_warnings( + action="ignore", category=DeprecationWarning)) + tar.extractall(cls.control_dir, filter=cls.extraction_filter) + tar.close() + cls.control_paths = set( + p.relative_to(cls.control_dir) + for p in pathlib.Path(cls.control_dir).glob('**/*')) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.control_dir) + + def check_files_present(self, directory): + got_paths = set( + p.relative_to(directory) + for p in pathlib.Path(directory).glob('**/*')) + if self.extraction_filter in (None, 'data'): + # The 'data' filter is expected to reject special files + for path in 'ustar/fifotype', 'ustar/blktype', 'ustar/chrtype': + got_paths.discard(pathlib.Path(path)) + self.assertEqual(self.control_paths, got_paths) + + @contextmanager + def extract_with_none(self, *attr_names): + DIR = pathlib.Path(TEMPDIR) / "extractall_none" + self.tar.errorlevel = 0 + for member in self.tar.getmembers(): + for attr_name in attr_names: + setattr(member, attr_name, None) + with os_helper.temp_dir(DIR): + self.tar.extractall(DIR, filter='fully_trusted') + self.check_files_present(DIR) + yield DIR + + def test_extractall_none_mtime(self): + # mtimes of extracted files should be later than 'now' -- the mtime + # of a previously created directory. + now = pathlib.Path(TEMPDIR).stat().st_mtime + with self.extract_with_none('mtime') as DIR: + for path in pathlib.Path(DIR).glob('**/*'): + with self.subTest(path=path): + try: + mtime = path.stat().st_mtime + except OSError: + # Some systems can't stat symlinks, ignore those + if not path.is_symlink(): + raise + else: + self.assertGreaterEqual(path.stat().st_mtime, now) + + def test_extractall_none_mode(self): + # modes of directories and regular files should match the mode + # of a "normally" created directory or regular file + dir_mode = pathlib.Path(TEMPDIR).stat().st_mode + regular_file = pathlib.Path(TEMPDIR) / 'regular_file' + regular_file.write_text('') + regular_file_mode = regular_file.stat().st_mode + with self.extract_with_none('mode') as DIR: + for path in pathlib.Path(DIR).glob('**/*'): + with self.subTest(path=path): + if path.is_dir(): + self.assertEqual(path.stat().st_mode, dir_mode) + elif path.is_file(): + self.assertEqual(path.stat().st_mode, + regular_file_mode) + + def test_extractall_none_uid(self): + with self.extract_with_none('uid'): + pass + + def test_extractall_none_gid(self): + with self.extract_with_none('gid'): + pass + + def test_extractall_none_uname(self): + with self.extract_with_none('uname'): + pass + + def test_extractall_none_gname(self): + with self.extract_with_none('gname'): + pass + + def test_extractall_none_ownership(self): + with self.extract_with_none('uid', 'gid', 'uname', 'gname'): + pass + +class NoneInfoExtractTests_Data(NoneInfoExtractTests, unittest.TestCase): + extraction_filter = 'data' + +class NoneInfoExtractTests_FullyTrusted(NoneInfoExtractTests, + unittest.TestCase): + extraction_filter = 'fully_trusted' + +class NoneInfoExtractTests_Tar(NoneInfoExtractTests, unittest.TestCase): + extraction_filter = 'tar' + +class NoneInfoExtractTests_Default(NoneInfoExtractTests, + unittest.TestCase): + extraction_filter = None + +class NoneInfoTests_Misc(unittest.TestCase): + def test_add(self): + # When addfile() encounters None metadata, it raises a ValueError + bio = io.BytesIO() + for tarformat in (tarfile.USTAR_FORMAT, tarfile.GNU_FORMAT, + tarfile.PAX_FORMAT): + with self.subTest(tarformat=tarformat): + tar = tarfile.open(fileobj=bio, mode='w', format=tarformat) + tarinfo = tar.gettarinfo(tarname) + try: + with open(tarname, 'rb') as f: + tar.addfile(tarinfo, f) + except Exception: + if tarformat == tarfile.USTAR_FORMAT: + # In the old, limited format, adding might fail for + # reasons like the UID being too large + pass + else: + raise + else: + for attr_name in ('mtime', 'mode', 'uid', 'gid', + 'uname', 'gname'): + with self.subTest(attr_name=attr_name): + replaced = tarinfo.replace(**{attr_name: None}) + with self.assertRaisesRegex(ValueError, + f"{attr_name}"): + with open(tarname, 'rb') as f: + tar.addfile(replaced, f) + + def test_list(self): + # Change some metadata to None, then compare list() output + # word-for-word. We want list() to not raise, and to only change + # printout for the affected piece of metadata. + # (n.b.: some contents of the test archive are hardcoded.) + for attr_names in ({'mtime'}, {'mode'}, {'uid'}, {'gid'}, + {'uname'}, {'gname'}, + {'uid', 'uname'}, {'gid', 'gname'}): + with (self.subTest(attr_names=attr_names), + tarfile.open(tarname, encoding="iso8859-1") as tar): + tio_prev = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') + with support.swap_attr(sys, 'stdout', tio_prev): + tar.list() + for member in tar.getmembers(): + for attr_name in attr_names: + setattr(member, attr_name, None) + tio_new = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') + with support.swap_attr(sys, 'stdout', tio_new): + tar.list() + for expected, got in zip(tio_prev.detach().getvalue().split(), + tio_new.detach().getvalue().split()): + if attr_names == {'mtime'} and re.match(rb'2003-01-\d\d', expected): + self.assertEqual(got, b'????-??-??') + elif attr_names == {'mtime'} and re.match(rb'\d\d:\d\d:\d\d', expected): + self.assertEqual(got, b'??:??:??') + elif attr_names == {'mode'} and re.match( + rb'.([r-][w-][x-]){3}', expected): + self.assertEqual(got, b'??????????') + elif attr_names == {'uname'} and expected.startswith( + (b'tarfile/', b'lars/', b'foo/')): + exp_user, exp_group = expected.split(b'/') + got_user, got_group = got.split(b'/') + self.assertEqual(got_group, exp_group) + self.assertRegex(got_user, b'[0-9]+') + elif attr_names == {'gname'} and expected.endswith( + (b'/tarfile', b'/users', b'/bar')): + exp_user, exp_group = expected.split(b'/') + got_user, got_group = got.split(b'/') + self.assertEqual(got_user, exp_user) + self.assertRegex(got_group, b'[0-9]+') + elif attr_names == {'uid'} and expected.startswith( + (b'1000/')): + exp_user, exp_group = expected.split(b'/') + got_user, got_group = got.split(b'/') + self.assertEqual(got_group, exp_group) + self.assertEqual(got_user, b'None') + elif attr_names == {'gid'} and expected.endswith((b'/100')): + exp_user, exp_group = expected.split(b'/') + got_user, got_group = got.split(b'/') + self.assertEqual(got_user, exp_user) + self.assertEqual(got_group, b'None') + elif attr_names == {'uid', 'uname'} and expected.startswith( + (b'tarfile/', b'lars/', b'foo/', b'1000/')): + exp_user, exp_group = expected.split(b'/') + got_user, got_group = got.split(b'/') + self.assertEqual(got_group, exp_group) + self.assertEqual(got_user, b'None') + elif attr_names == {'gname', 'gid'} and expected.endswith( + (b'/tarfile', b'/users', b'/bar', b'/100')): + exp_user, exp_group = expected.split(b'/') + got_user, got_group = got.split(b'/') + self.assertEqual(got_user, exp_user) + self.assertEqual(got_group, b'None') + else: + # In other cases the output should be the same + self.assertEqual(expected, got) + +def _filemode_to_int(mode): + """Inverse of `stat.filemode` (for permission bits) + + Using mode strings rather than numbers makes the later tests more readable. + """ + str_mode = mode[1:] + result = ( + {'r': stat.S_IRUSR, '-': 0}[str_mode[0]] + | {'w': stat.S_IWUSR, '-': 0}[str_mode[1]] + | {'x': stat.S_IXUSR, '-': 0, + 's': stat.S_IXUSR | stat.S_ISUID, + 'S': stat.S_ISUID}[str_mode[2]] + | {'r': stat.S_IRGRP, '-': 0}[str_mode[3]] + | {'w': stat.S_IWGRP, '-': 0}[str_mode[4]] + | {'x': stat.S_IXGRP, '-': 0, + 's': stat.S_IXGRP | stat.S_ISGID, + 'S': stat.S_ISGID}[str_mode[5]] + | {'r': stat.S_IROTH, '-': 0}[str_mode[6]] + | {'w': stat.S_IWOTH, '-': 0}[str_mode[7]] + | {'x': stat.S_IXOTH, '-': 0, + 't': stat.S_IXOTH | stat.S_ISVTX, + 'T': stat.S_ISVTX}[str_mode[8]] + ) + # check we did this right + assert stat.filemode(result)[1:] == mode[1:] + + return result + +class ArchiveMaker: + """Helper to create a tar file with specific contents + + Usage: + + with ArchiveMaker() as t: + t.add('filename', ...) + + with t.open() as tar: + ... # `tar` is now a TarFile with 'filename' in it! + """ + def __init__(self, **kwargs): + self.bio = io.BytesIO() + self.tar_kwargs = dict(kwargs) + + def __enter__(self): + self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio, **self.tar_kwargs) + return self + + def __exit__(self, *exc): + self.tar_w.close() + self.contents = self.bio.getvalue() + self.bio = None + + def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, + mode=None, size=None, content=None, **kwargs): + """Add a member to the test archive. Call within `with`. + + Provides many shortcuts: + - default `type` is based on symlink_to, hardlink_to, and trailing `/` + in name (which is stripped) + - size & content defaults are based on each other + - content can be str or bytes + - mode should be textual ('-rwxrwxrwx') + + (add more! this is unstable internal test-only API) + """ + name = str(name) + tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if content is not None: + if isinstance(content, str): + content = content.encode() + size = len(content) + if size is not None: + tarinfo.size = size + if content is None: + content = bytes(tarinfo.size) + if mode: + tarinfo.mode = _filemode_to_int(mode) + if symlink_to is not None: + type = tarfile.SYMTYPE + tarinfo.linkname = str(symlink_to) + if hardlink_to is not None: + type = tarfile.LNKTYPE + tarinfo.linkname = str(hardlink_to) + if name.endswith('/') and type is None: + type = tarfile.DIRTYPE + if type is not None: + tarinfo.type = type + if tarinfo.isreg(): + fileobj = io.BytesIO(content) + else: + fileobj = None + self.tar_w.addfile(tarinfo, fileobj) + + def open(self, **kwargs): + """Open the resulting archive as TarFile. Call after `with`.""" + bio = io.BytesIO(self.contents) + return tarfile.open(fileobj=bio, **kwargs) + +# Under WASI, `os_helper.can_symlink` is False to make +# `skip_unless_symlink` skip symlink tests. " +# But in the following tests we use can_symlink to *determine* which +# behavior is expected. +# Like other symlink tests, skip these on WASI for now. +if support.is_wasi: + def symlink_test(f): + return unittest.skip("WASI: Skip symlink test for now")(f) +else: + def symlink_test(f): + return f + + +class TestExtractionFilters(unittest.TestCase): + + # A temporary directory for the extraction results. + # All files that "escape" the destination path should still end + # up in this directory. + outerdir = pathlib.Path(TEMPDIR) / 'outerdir' + + # The destination for the extraction, within `outerdir` + destdir = outerdir / 'dest' + + @contextmanager + def check_context(self, tar, filter, *, check_flag=True): + """Extracts `tar` to `self.destdir` and allows checking the result + + If an error occurs, it must be checked using `expect_exception` + + Otherwise, all resulting files must be checked using `expect_file`, + except the destination directory itself and parent directories of + other files. + When checking directories, do so before their contents. + + A file called 'flag' is made in outerdir (i.e. outside destdir) + before extraction; it should not be altered nor should its contents + be read/copied. + """ + with os_helper.temp_dir(self.outerdir): + flag_path = self.outerdir / 'flag' + flag_path.write_text('capture me') + try: + tar.extractall(self.destdir, filter=filter) + except Exception as exc: + self.raised_exception = exc + self.reraise_exception = True + self.expected_paths = set() + else: + self.raised_exception = None + self.reraise_exception = False + self.expected_paths = set(self.outerdir.glob('**/*')) + self.expected_paths.discard(self.destdir) + self.expected_paths.discard(flag_path) + try: + yield self + finally: + tar.close() + if self.reraise_exception: + raise self.raised_exception + self.assertEqual(self.expected_paths, set()) + if check_flag: + self.assertEqual(flag_path.read_text(), 'capture me') + else: + assert filter == 'fully_trusted' + + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None, content=None): + """Check a single file. See check_context.""" + if self.raised_exception: + raise self.raised_exception + # use normpath() rather than resolve() so we don't follow symlinks + path = pathlib.Path(os.path.normpath(self.destdir / name)) + self.assertIn(path, self.expected_paths) + self.expected_paths.remove(path) + if mode is not None and os_helper.can_chmod() and os.name != 'nt': + got = stat.filemode(stat.S_IMODE(path.stat().st_mode)) + self.assertEqual(got, mode) + if type is None and isinstance(name, str) and name.endswith('/'): + type = tarfile.DIRTYPE + if symlink_to is not None: + got = (self.destdir / name).readlink() + expected = pathlib.Path(symlink_to) + # The symlink might be the same (textually) as what we expect, + # but some systems change the link to an equivalent path, so + # we fall back to samefile(). + try: + if expected != got: + self.assertTrue(got.samefile(expected)) + except Exception as e: + # attach a note, so it's shown even if `samefile` fails + e.add_note(f'{expected=}, {got=}') + raise + elif type == tarfile.REGTYPE or type is None: + self.assertTrue(path.is_file()) + elif type == tarfile.DIRTYPE: + self.assertTrue(path.is_dir()) + elif type == tarfile.FIFOTYPE: + self.assertTrue(path.is_fifo()) + elif type == tarfile.SYMTYPE: + self.assertTrue(path.is_symlink()) + else: + raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) + if content is not None: + self.assertEqual(path.read_text(), content) + for parent in path.parents: + self.expected_paths.discard(parent) + + def expect_any_tree(self, name): + """Check a directory; forget about its contents.""" + tree_path = (self.destdir / name).resolve() + self.expect_file(tree_path, type=tarfile.DIRTYPE) + self.expected_paths = { + p for p in self.expected_paths + if tree_path not in p.parents + } + + def expect_exception(self, exc_type, message_re='.'): + with self.assertRaisesRegex(exc_type, message_re): + if self.raised_exception is not None: + raise self.raised_exception + self.reraise_exception = False + return self.raised_exception + + def test_benign_file(self): + with ArchiveMaker() as arc: + arc.add('benign.txt') + for filter in 'fully_trusted', 'tar', 'data': + with self.check_context(arc.open(), filter): + self.expect_file('benign.txt') + + def test_absolute(self): + # Test handling a member with an absolute path + # Inspired by 'absolute1' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add(self.outerdir / 'escaped.evil') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_file('../escaped.evil') + + for filter in 'tar', 'data': + with self.check_context(arc.open(), filter): + if str(self.outerdir).startswith('/'): + # We strip leading slashes, as e.g. GNU tar does + # (without --absolute-filenames). + outerdir_stripped = str(self.outerdir).lstrip('/') + self.expect_file(f'{outerdir_stripped}/escaped.evil') + else: + # On this system, absolute paths don't have leading + # slashes. + # So, there's nothing to strip. We refuse to unpack + # to an absolute path, nonetheless. + self.expect_exception( + tarfile.AbsolutePathError, + """['"].*escaped.evil['"] has an absolute path""") + + @symlink_test + def test_parent_symlink(self): + # Test interplaying symlinks + # Inspired by 'dirsymlink2a' in jwilk/traversal-archives + with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself + arc.add('current', symlink_to='.') + + # effectively points to ./../ + arc.add('parent', symlink_to='current/..') + + arc.add('parent/evil') + + if os_helper.can_symlink(): + with self.check_context(arc.open(), 'fully_trusted'): + if self.raised_exception is not None: + # Windows will refuse to create a file that's a symlink to itself + # (and tarfile doesn't swallow that exception) + self.expect_exception(FileExistsError) + # The other cases will fail with this error too. + # Skip the rest of this test. + return + else: + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='current/..') + self.expect_file('../evil') + + with self.check_context(arc.open(), 'tar'): + self.expect_exception( + tarfile.OutsideDestinationError, + """'parent/evil' would be extracted to ['"].*evil['"], """ + + "which is outside the destination") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.LinkOutsideDestinationError, + """'parent' would link to ['"].*outerdir['"], """ + + "which is outside the destination") + + else: + # No symlink support. The symlinks are ignored. + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_file('parent/evil') + with self.check_context(arc.open(), 'tar'): + self.expect_file('parent/evil') + with self.check_context(arc.open(), 'data'): + self.expect_file('parent/evil') + + @symlink_test + @os_helper.skip_unless_symlink + def test_realpath_limit_attack(self): + # (CVE-2025-4517) + + with ArchiveMaker() as arc: + # populate the symlinks and dirs that expand in os.path.realpath() + # The component length is chosen so that in common cases, the unexpanded + # path fits in PATH_MAX, but it overflows when the final symlink + # is expanded + steps = "abcdefghijklmnop" + if sys.platform == 'win32': + component = 'd' * 25 + elif 'PC_PATH_MAX' in os.pathconf_names: + max_path_len = os.pathconf(self.outerdir.parent, "PC_PATH_MAX") + path_sep_len = 1 + dest_len = len(str(self.destdir)) + path_sep_len + component_len = (max_path_len - dest_len) // (len(steps) + path_sep_len) + component = 'd' * component_len + else: + raise NotImplementedError("Need to guess component length for {sys.platform}") + path = "" + step_path = "" + for i in steps: + arc.add(os.path.join(path, component), type=tarfile.DIRTYPE, + mode='drwxrwxrwx') + arc.add(os.path.join(path, i), symlink_to=component) + path = os.path.join(path, component) + step_path = os.path.join(step_path, i) + # create the final symlink that exceeds PATH_MAX and simply points + # to the top dir. + # this link will never be expanded by + # os.path.realpath(strict=False), nor anything after it. + linkpath = os.path.join(*steps, "l"*254) + parent_segments = [".."] * len(steps) + arc.add(linkpath, symlink_to=os.path.join(*parent_segments)) + # make a symlink outside to keep the tar command happy + arc.add("escape", symlink_to=os.path.join(linkpath, "..")) + # use the symlinks above, that are not checked, to create a hardlink + # to a file outside of the destination path + arc.add("flaglink", hardlink_to=os.path.join("escape", "flag")) + # now that we have the hardlink we can overwrite the file + arc.add("flaglink", content='overwrite') + # we can also create new files as well! + arc.add("escape/newfile", content='new') + + with (self.subTest('fully_trusted'), + self.check_context(arc.open(), filter='fully_trusted', + check_flag=False)): + if sys.platform == 'win32': + self.expect_exception((FileNotFoundError, FileExistsError)) + elif self.raised_exception: + # Cannot symlink/hardlink: tarfile falls back to getmember() + self.expect_exception(KeyError) + # Otherwise, this block should never enter. + else: + self.expect_any_tree(component) + self.expect_file('flaglink', content='overwrite') + self.expect_file('../newfile', content='new') + self.expect_file('escape', type=tarfile.SYMTYPE) + self.expect_file('a', symlink_to=component) + + for filter in 'tar', 'data': + with self.subTest(filter), self.check_context(arc.open(), filter=filter): + exc = self.expect_exception((OSError, KeyError)) + if isinstance(exc, OSError): + if sys.platform == 'win32': + # 3: ERROR_PATH_NOT_FOUND + # 5: ERROR_ACCESS_DENIED + # 206: ERROR_FILENAME_EXCED_RANGE + self.assertIn(exc.winerror, (3, 5, 206)) + else: + self.assertEqual(exc.errno, errno.ENAMETOOLONG) + + @symlink_test + def test_parent_symlink2(self): + # Test interplaying symlinks + # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself + arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` + arc.add('current/parent', symlink_to='..') + + arc.add('parent/evil') + + with self.check_context(arc.open(), 'fully_trusted'): + if os_helper.can_symlink(): + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='..') + self.expect_file('../evil') + else: + self.expect_file('current/') + self.expect_file('parent/evil') + + with self.check_context(arc.open(), 'tar'): + if os_helper.can_symlink(): + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') + + with self.check_context(arc.open(), 'data'): + if os_helper.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') + + @symlink_test + def test_absolute_symlink(self): + # Test symlink to an absolute path + # Inspired by 'dirsymlink' in jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', symlink_to=self.outerdir) + arc.add('parent/evil') + + with self.check_context(arc.open(), 'fully_trusted'): + if os_helper.can_symlink(): + self.expect_file('parent', symlink_to=self.outerdir) + self.expect_file('../evil') + else: + self.expect_file('parent/evil') + + with self.check_context(arc.open(), 'tar'): + if os_helper.can_symlink(): + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + self.expect_file('parent/evil') + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") + + @symlink_test + def test_sly_relative0(self): + # Inspired by 'relative0' in jwilk/traversal-archives + with ArchiveMaker() as arc: + # points to `../../tmp/moo` + arc.add('../moo', symlink_to='..//tmp/moo') + + try: + with self.check_context(arc.open(), filter='fully_trusted'): + if os_helper.can_symlink(): + if isinstance(self.raised_exception, FileExistsError): + # XXX TarFile happens to fail creating a parent + # directory. + # This might be a bug, but fixing it would hurt + # security. + # Note that e.g. GNU `tar` rejects '..' components, + # so you could argue this is an invalid archive and we + # just raise an bad type of exception. + self.expect_exception(FileExistsError) + else: + self.expect_file('../moo', symlink_to='..//tmp/moo') + else: + # The symlink can't be extracted and is ignored + pass + except FileExistsError: + pass + + for filter in 'tar', 'data': + with self.check_context(arc.open(), filter): + self.expect_exception( + tarfile.OutsideDestinationError, + "'../moo' would be extracted to " + + "'.*moo', which is outside " + + "the destination") + + @symlink_test + def test_sly_relative2(self): + # Inspired by 'relative2' in jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('tmp/') + arc.add('tmp/../../moo', symlink_to='tmp/../..//tmp/moo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_file('tmp', type=tarfile.DIRTYPE) + if os_helper.can_symlink(): + self.expect_file('../moo', symlink_to='tmp/../../tmp/moo') + + for filter in 'tar', 'data': + with self.check_context(arc.open(), filter): + self.expect_exception( + tarfile.OutsideDestinationError, + "'tmp/../../moo' would be extracted to " + + """['"].*moo['"], which is outside the """ + + "destination") + + @symlink_test + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + @symlink_test + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to=os.path.join('targetdir', 'target')) + arc.add('linkdir/hardlink2', hardlink_to=os.path.join('linkdir', 'symlink')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + + @symlink_test + def test_sneaky_hardlink_fallback(self): + # (CVE-2025-4330) + # Test that when hardlink extraction falls back to extracting members + # from the archive, the extracted member is (re-)filtered. + with ArchiveMaker() as arc: + # Create a directory structure so the c/escape symlink stays + # inside the path + arc.add("a/t/dummy") + # Create b/ directory + arc.add("b/") + # Point "c" to the bottom of the tree in "a" + arc.add("c", symlink_to=os.path.join("a", "t")) + # link to non-existant location under "a" + arc.add("c/escape", symlink_to=os.path.join("..", "..", + "link_here")) + # Move "c" to point to "b" ("c/escape" no longer exists) + arc.add("c", symlink_to="b") + # Attempt to create a hard link to "c/escape". Since it doesn't + # exist it will attempt to extract "cescape" but at "boom". + arc.add("boom", hardlink_to=os.path.join("c", "escape")) + + with self.check_context(arc.open(), 'data'): + if not os_helper.can_symlink(): + # When 'c/escape' is extracted, 'c' is a regular + # directory, and 'c/escape' *would* point outside + # the destination if symlinks were allowed. + self.expect_exception( + tarfile.LinkOutsideDestinationError) + elif sys.platform == "win32": + # On Windows, 'c/escape' points outside the destination + self.expect_exception(tarfile.LinkOutsideDestinationError) + else: + e = self.expect_exception( + tarfile.LinkFallbackError, + "link 'boom' would be extracted as a copy of " + + "'c/escape', which was rejected") + self.assertIsInstance(e.__cause__, + tarfile.LinkOutsideDestinationError) + for filter in 'tar', 'fully_trusted': + with self.subTest(filter), self.check_context(arc.open(), filter): + if not os_helper.can_symlink(): + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("c/") + else: + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("a/t/escape", symlink_to='../../link_here') + self.expect_file("boom", symlink_to='../../link_here') + self.expect_file("c", symlink_to='b') + + @symlink_test + def test_exfiltration_via_symlink(self): + # (CVE-2025-4138) + # Test changing symlinks that result in a symlink pointing outside + # the extraction directory, unless prevented by 'data' filter's + # normalization. + with ArchiveMaker() as arc: + arc.add("escape", symlink_to=os.path.join('link', 'link', '..', '..', 'link-here')) + arc.add("link", symlink_to='./') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + if os_helper.can_symlink(): + self.expect_file("link", symlink_to='./') + if filter == 'data': + self.expect_file("escape", symlink_to='link-here') + else: + self.expect_file("escape", + symlink_to='link/link/../../link-here') + else: + # Nothing is extracted. + pass + + @symlink_test + def test_chmod_outside_dir(self): + # (CVE-2024-12718) + # Test that members used for delayed updates of directory metadata + # are (re-)filtered. + with ArchiveMaker() as arc: + # "pwn" is a veeeery innocent symlink: + arc.add("a/pwn", symlink_to='.') + # But now "pwn" is also a directory, so it's scheduled to have its + # metadata updated later: + arc.add("a/pwn/", mode='drwxrwxrwx') + # Oops, "pwn" is not so innocent any more: + arc.add("a/pwn", symlink_to='x/../') + # Newly created symlink points to the dest dir, + # so it's OK for the "data" filter. + arc.add('a/x', symlink_to=('../')) + # But now "pwn" points outside the dest dir + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + if not os_helper.can_symlink(): + self.expect_file("a/pwn/") + elif filter == 'data': + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='.') + else: + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='x/../') + if sys.platform != "win32": + st_mode = cc.outerdir.stat().st_mode + self.assertNotEqual(st_mode & 0o777, 0o777) + + def test_link_fallback_normalizes(self): + # Make sure hardlink fallbacks work for non-normalized paths for all + # filters + with ArchiveMaker() as arc: + arc.add("dir/") + arc.add("dir/../afile") + arc.add("link1", hardlink_to='dir/../afile') + arc.add("link2", hardlink_to='dir/../dir/../afile') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + self.expect_file("dir/") + self.expect_file("afile") + self.expect_file("link1") + self.expect_file("link2") + + def test_modes(self): + # Test how file modes are extracted + # (Note that the modes are ignored on platforms without working chmod) + with ArchiveMaker() as arc: + arc.add('all_bits', mode='?rwsrwsrwt') + arc.add('perm_bits', mode='?rwxrwxrwx') + arc.add('exec_group_other', mode='?rw-rwxrwx') + arc.add('read_group_only', mode='?---r-----') + arc.add('no_bits', mode='?---------') + arc.add('dir/', mode='?---rwsrwt') + arc.add('dir_all_bits/', mode='?rwsrwsrwt') + + # On some systems, setting the uid, gid, and/or sticky bit is a no-ops. + # Check which bits we can set, so we can compare tarfile machinery to + # a simple chmod. + tmp_filename = os.path.join(TEMPDIR, "tmp.file") + with open(tmp_filename, 'w'): + pass + try: + new_mode = (os.stat(tmp_filename).st_mode + | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID) + try: + os.chmod(tmp_filename, new_mode) + except OSError as exc: + if exc.errno == getattr(errno, "EFTYPE", 0): + # gh-108948: On FreeBSD, regular users cannot set + # the sticky bit. + self.skipTest("chmod() failed with EFTYPE: " + "regular users cannot set sticky bit") + else: + raise + + got_mode = os.stat(tmp_filename).st_mode + _t_file = 't' if (got_mode & stat.S_ISVTX) else 'x' + _suid_file = 's' if (got_mode & stat.S_ISUID) else 'x' + _sgid_file = 's' if (got_mode & stat.S_ISGID) else 'x' + finally: + os.unlink(tmp_filename) + + os.mkdir(tmp_filename) + new_mode = (os.stat(tmp_filename).st_mode + | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID) + os.chmod(tmp_filename, new_mode) + got_mode = os.stat(tmp_filename).st_mode + _t_dir = 't' if (got_mode & stat.S_ISVTX) else 'x' + _suid_dir = 's' if (got_mode & stat.S_ISUID) else 'x' + _sgid_dir = 's' if (got_mode & stat.S_ISGID) else 'x' + os.rmdir(tmp_filename) + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_file('all_bits', + mode=f'?rw{_suid_file}rw{_sgid_file}rw{_t_file}') + self.expect_file('perm_bits', mode='?rwxrwxrwx') + self.expect_file('exec_group_other', mode='?rw-rwxrwx') + self.expect_file('read_group_only', mode='?---r-----') + self.expect_file('no_bits', mode='?---------') + self.expect_file('dir/', mode=f'?---rw{_sgid_dir}rw{_t_dir}') + self.expect_file('dir_all_bits/', + mode=f'?rw{_suid_dir}rw{_sgid_dir}rw{_t_dir}') + + with self.check_context(arc.open(), 'tar'): + self.expect_file('all_bits', mode='?rwxr-xr-x') + self.expect_file('perm_bits', mode='?rwxr-xr-x') + self.expect_file('exec_group_other', mode='?rw-r-xr-x') + self.expect_file('read_group_only', mode='?---r-----') + self.expect_file('no_bits', mode='?---------') + self.expect_file('dir/', mode='?---r-xr-x') + self.expect_file('dir_all_bits/', mode='?rwxr-xr-x') + + with self.check_context(arc.open(), 'data'): + normal_dir_mode = stat.filemode(stat.S_IMODE( + self.outerdir.stat().st_mode)) + self.expect_file('all_bits', mode='?rwxr-xr-x') + self.expect_file('perm_bits', mode='?rwxr-xr-x') + self.expect_file('exec_group_other', mode='?rw-r--r--') + self.expect_file('read_group_only', mode='?rw-r-----') + self.expect_file('no_bits', mode='?rw-------') + self.expect_file('dir/', mode=normal_dir_mode) + self.expect_file('dir_all_bits/', mode=normal_dir_mode) + + def test_pipe(self): + # Test handling of a special file + with ArchiveMaker() as arc: + arc.add('foo', type=tarfile.FIFOTYPE) + + for filter in 'fully_trusted', 'tar': + with self.check_context(arc.open(), filter): + if hasattr(os, 'mkfifo'): + self.expect_file('foo', type=tarfile.FIFOTYPE) + else: + # The pipe can't be extracted and is skipped. + pass + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.SpecialFileError, + "'foo' is a special file") + + def test_special_files(self): + # Creating device files is tricky. Instead of attempting that let's + # only check the filter result. + for special_type in tarfile.FIFOTYPE, tarfile.CHRTYPE, tarfile.BLKTYPE: + tarinfo = tarfile.TarInfo('foo') + tarinfo.type = special_type + trusted = tarfile.fully_trusted_filter(tarinfo, '') + self.assertIs(trusted, tarinfo) + tar = tarfile.tar_filter(tarinfo, '') + self.assertEqual(tar.type, special_type) + with self.assertRaises(tarfile.SpecialFileError) as cm: + tarfile.data_filter(tarinfo, '') + self.assertIsInstance(cm.exception.tarinfo, tarfile.TarInfo) + self.assertEqual(cm.exception.tarinfo.name, 'foo') + + def test_fully_trusted_filter(self): + # The 'fully_trusted' filter returns the original TarInfo objects. + with tarfile.TarFile.open(tarname) as tar: + for tarinfo in tar.getmembers(): + filtered = tarfile.fully_trusted_filter(tarinfo, '') + self.assertIs(filtered, tarinfo) + + def test_tar_filter(self): + # The 'tar' filter returns TarInfo objects with the same name/type. + # (It can also fail for particularly "evil" input, but we don't have + # that in the test archive.) + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: + for tarinfo in tar.getmembers(): + try: + filtered = tarfile.tar_filter(tarinfo, '') + except UnicodeEncodeError: + continue + self.assertIs(filtered.name, tarinfo.name) + self.assertIs(filtered.type, tarinfo.type) + + def test_data_filter(self): + # The 'data' filter either raises, or returns TarInfo with the same + # name/type. + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: + for tarinfo in tar.getmembers(): + try: + filtered = tarfile.data_filter(tarinfo, '') + except (tarfile.FilterError, UnicodeEncodeError): + continue + self.assertIs(filtered.name, tarinfo.name) + self.assertIs(filtered.type, tarinfo.type) + + @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths') + def test_filter_unencodable(self): + # Sanity check using a valid path. + tarinfo = tarfile.TarInfo(os_helper.TESTFN) + filtered = tarfile.tar_filter(tarinfo, '') + self.assertIs(filtered.name, tarinfo.name) + filtered = tarfile.data_filter(tarinfo, '') + self.assertIs(filtered.name, tarinfo.name) + + tarinfo = tarfile.TarInfo('test\x00') + self.assertRaises(ValueError, tarfile.tar_filter, tarinfo, '') + self.assertRaises(ValueError, tarfile.data_filter, tarinfo, '') + tarinfo = tarfile.TarInfo('\ud800') + self.assertRaises(UnicodeEncodeError, tarfile.tar_filter, tarinfo, '') + self.assertRaises(UnicodeEncodeError, tarfile.data_filter, tarinfo, '') + + @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths') + def test_extract_unencodable(self): + # Create a member with name \xed\xa0\x80 which is UTF-8 encoded + # lone surrogate \ud800. + with ArchiveMaker(encoding='ascii', errors='surrogateescape') as arc: + arc.add('\udced\udca0\udc80') + with os_helper.temp_cwd() as tmp: + tar = arc.open(encoding='utf-8', errors='surrogatepass', + errorlevel=1) + self.assertEqual(tar.getnames(), ['\ud800']) + with self.assertRaises(UnicodeEncodeError): + tar.extractall() + self.assertEqual(os.listdir(), []) + + tar = arc.open(encoding='utf-8', errors='surrogatepass', + errorlevel=0, debug=1) + with support.captured_stderr() as stderr: + tar.extractall() + self.assertEqual(os.listdir(), []) + self.assertIn('tarfile: UnicodeEncodeError ', stderr.getvalue()) + + def test_change_default_filter_on_instance(self): + tar = tarfile.TarFile(tarname, 'r') + def strict_filter(tarinfo, path): + if tarinfo.name == 'ustar/regtype': + return tarinfo + else: + return None + tar.extraction_filter = strict_filter + with self.check_context(tar, None): + self.expect_file('ustar/regtype') + + def test_change_default_filter_on_class(self): + def strict_filter(tarinfo, path): + if tarinfo.name == 'ustar/regtype': + return tarinfo + else: + return None + tar = tarfile.TarFile(tarname, 'r') + with support.swap_attr(tarfile.TarFile, 'extraction_filter', + staticmethod(strict_filter)): + with self.check_context(tar, None): + self.expect_file('ustar/regtype') + + def test_change_default_filter_on_subclass(self): + class TarSubclass(tarfile.TarFile): + def extraction_filter(self, tarinfo, path): + if tarinfo.name == 'ustar/regtype': + return tarinfo + else: + return None + + tar = TarSubclass(tarname, 'r') + with self.check_context(tar, None): + self.expect_file('ustar/regtype') + + def test_change_default_filter_to_string(self): + tar = tarfile.TarFile(tarname, 'r') + tar.extraction_filter = 'data' + with self.check_context(tar, None): + self.expect_exception(TypeError) + + def test_custom_filter(self): + def custom_filter(tarinfo, path): + self.assertIs(path, self.destdir) + if tarinfo.name == 'move_this': + return tarinfo.replace(name='moved') + if tarinfo.name == 'ignore_this': + return None + return tarinfo + + with ArchiveMaker() as arc: + arc.add('move_this') + arc.add('ignore_this') + arc.add('keep') + with self.check_context(arc.open(), custom_filter): + self.expect_file('moved') + self.expect_file('keep') + + def test_bad_filter_name(self): + with ArchiveMaker() as arc: + arc.add('foo') + with self.check_context(arc.open(), 'bad filter name'): + self.expect_exception(ValueError) + + def test_stateful_filter(self): + # Stateful filters should be possible. + # (This doesn't really test tarfile. Rather, it demonstrates + # that third parties can implement a stateful filter.) + class StatefulFilter: + def __enter__(self): + self.num_files_processed = 0 + return self + + def __call__(self, tarinfo, path): + try: + tarinfo = tarfile.data_filter(tarinfo, path) + except tarfile.FilterError: + return None + self.num_files_processed += 1 + return tarinfo + + def __exit__(self, *exc_info): + self.done = True + + with ArchiveMaker() as arc: + arc.add('good') + arc.add('bad', symlink_to='/') + arc.add('good') + with StatefulFilter() as custom_filter: + with self.check_context(arc.open(), custom_filter): + self.expect_file('good') + self.assertEqual(custom_filter.num_files_processed, 2) + self.assertEqual(custom_filter.done, True) + + def test_errorlevel(self): + def extracterror_filter(tarinfo, path): + raise tarfile.ExtractError('failed with ExtractError') + def filtererror_filter(tarinfo, path): + raise tarfile.FilterError('failed with FilterError') + def oserror_filter(tarinfo, path): + raise OSError('failed with OSError') + def tarerror_filter(tarinfo, path): + raise tarfile.TarError('failed with base TarError') + def valueerror_filter(tarinfo, path): + raise ValueError('failed with ValueError') + + with ArchiveMaker() as arc: + arc.add('file') + + # If errorlevel is 0, errors affected by errorlevel are ignored + + with self.check_context(arc.open(errorlevel=0), extracterror_filter): + pass + + with self.check_context(arc.open(errorlevel=0), filtererror_filter): + pass + + with self.check_context(arc.open(errorlevel=0), oserror_filter): + pass + + with self.check_context(arc.open(errorlevel=0), tarerror_filter): + self.expect_exception(tarfile.TarError) + + with self.check_context(arc.open(errorlevel=0), valueerror_filter): + self.expect_exception(ValueError) + + # If 1, all fatal errors are raised + + with self.check_context(arc.open(errorlevel=1), extracterror_filter): + pass + + with self.check_context(arc.open(errorlevel=1), filtererror_filter): + self.expect_exception(tarfile.FilterError) + + with self.check_context(arc.open(errorlevel=1), oserror_filter): + self.expect_exception(OSError) + + with self.check_context(arc.open(errorlevel=1), tarerror_filter): + self.expect_exception(tarfile.TarError) + + with self.check_context(arc.open(errorlevel=1), valueerror_filter): + self.expect_exception(ValueError) + + # If 2, all non-fatal errors are raised as well. + + with self.check_context(arc.open(errorlevel=2), extracterror_filter): + self.expect_exception(tarfile.ExtractError) + + with self.check_context(arc.open(errorlevel=2), filtererror_filter): + self.expect_exception(tarfile.FilterError) + + with self.check_context(arc.open(errorlevel=2), oserror_filter): + self.expect_exception(OSError) + + with self.check_context(arc.open(errorlevel=2), tarerror_filter): + self.expect_exception(tarfile.TarError) + + with self.check_context(arc.open(errorlevel=2), valueerror_filter): + self.expect_exception(ValueError) + + # We only handle ExtractionError, FilterError & OSError specially. + + with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter): + self.expect_exception(TypeError) # errorlevel is not int + + +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = os.path.join(TEMPDIR, "testoverwrite") + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = os.path.join(TEMPDIR, 'tar-with-file.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + p = cls.ar_with_dir = os.path.join(TEMPDIR, 'tar-with-dir.tar') + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + tar.addfile(tar.gettarinfo(os.curdir, 'test')) + + p = os.path.join(TEMPDIR, 'tar-with-implicit-dir.tar') + cls.ar_with_implicit_dir = p + cls.addClassCleanup(os_helper.unlink, p) + with tarfile.open(p, 'w') as tar: + t = tarfile.TarInfo('test/file') + t.size = 10 + tar.addfile(t, io.BytesIO(b'newcontent')) + + def open(self, path): + return tarfile.open(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir, filter='fully_trusted') + + +class OffsetValidationTests(unittest.TestCase): + tarname = tmpname + invalid_posix_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, space, null terminator: 8 bytes + + b"000755" + SPACE + tarfile.NUL + # uid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0011407" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # magic: 6 bytes, version: 2 bytes + + tarfile.POSIX_MAGIC + # uname: 32 bytes + + tarfile.NUL * 32 + # gname: 32 bytes + + tarfile.NUL * 32 + # devmajor, space, null terminator: 8 bytes + + tarfile.NUL * 6 + SPACE + tarfile.NUL + # devminor, space, null terminator: 8 bytes + + tarfile.NUL * 6 + SPACE + tarfile.NUL + # prefix: 155 bytes + + tarfile.NUL * tarfile.LENGTH_PREFIX + # padding: 12 bytes + + tarfile.NUL * 12 + ) + invalid_gnu_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, null terminator: 8 bytes + + b"0000755" + tarfile.NUL + # uid, null terminator: 8 bytes + + b"0000001" + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"0000001" + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0011327" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # magic: 8 bytes + + tarfile.GNU_MAGIC + # uname: 32 bytes + + tarfile.NUL * 32 + # gname: 32 bytes + + tarfile.NUL * 32 + # devmajor, null terminator: 8 bytes + + tarfile.NUL * 8 + # devminor, null terminator: 8 bytes + + tarfile.NUL * 8 + # padding: 167 bytes + + tarfile.NUL * 167 + ) + invalid_v7_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, space, null terminator: 8 bytes + + b"000755" + SPACE + tarfile.NUL + # uid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0010070" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # padding: 255 bytes + + tarfile.NUL * 255 + ) + valid_gnu_header = tarfile.TarInfo("filename").tobuf(tarfile.GNU_FORMAT) + data_block = b"\xff" * tarfile.BLOCKSIZE + + def _write_buffer(self, buffer): + with open(self.tarname, "wb") as f: + f.write(buffer) + + def _get_members(self, ignore_zeros=None): + with open(self.tarname, "rb") as f: + with tarfile.open( + mode="r", fileobj=f, ignore_zeros=ignore_zeros + ) as tar: + return tar.getmembers() + + def _assert_raises_read_error_exception(self): + with self.assertRaisesRegex( + tarfile.ReadError, "file could not be opened successfully" + ): + self._get_members() + + def test_invalid_offset_header_validations(self): + for tar_format, invalid_header in ( + ("posix", self.invalid_posix_header), + ("gnu", self.invalid_gnu_header), + ("v7", self.invalid_v7_header), + ): + with self.subTest(format=tar_format): + self._write_buffer(invalid_header) + self._assert_raises_read_error_exception() + + def test_early_stop_at_invalid_offset_header(self): + buffer = self.valid_gnu_header + self.invalid_gnu_header + self.valid_gnu_header + self._write_buffer(buffer) + members = self._get_members() + self.assertEqual(len(members), 1) + self.assertEqual(members[0].name, "filename") + self.assertEqual(members[0].offset, 0) + + def test_ignore_invalid_archive(self): + # 3 invalid headers with their respective data + buffer = (self.invalid_gnu_header + self.data_block) * 3 + self._write_buffer(buffer) + members = self._get_members(ignore_zeros=True) + self.assertEqual(len(members), 0) + + def test_ignore_invalid_offset_headers(self): + for first_block, second_block, expected_offset in ( + ( + (self.valid_gnu_header), + (self.invalid_gnu_header + self.data_block), + 0, + ), + ( + (self.invalid_gnu_header + self.data_block), + (self.valid_gnu_header), + 1024, + ), + ): + self._write_buffer(first_block + second_block) + members = self._get_members(ignore_zeros=True) + self.assertEqual(len(members), 1) + self.assertEqual(members[0].name, "filename") + self.assertEqual(members[0].offset, expected_offset) + + def setUpModule(): os_helper.unlink(TEMPDIR) os.makedirs(TEMPDIR) @@ -2929,7 +4831,7 @@ def setUpModule(): data = fobj.read() # Create compressed tarfiles. - for c in GzipTest, Bz2Test, LzmaTest: + for c in GzipTest, Bz2Test, LzmaTest, ZstdTest: if c.open: os_helper.unlink(c.tarname) testtarnames.append(c.tarname) diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py deleted file mode 100644 index 41c4fcd4195..00000000000 --- a/Lib/test/test_telnetlib.py +++ /dev/null @@ -1,402 +0,0 @@ -import socket -import selectors -import telnetlib -import threading -import contextlib - -from test import support -from test.support import socket_helper -import unittest - -HOST = socket_helper.HOST - -def server(evt, serv): - serv.listen() - evt.set() - try: - conn, addr = serv.accept() - conn.close() - except TimeoutError: - pass - finally: - serv.close() - -class GeneralTests(unittest.TestCase): - - def setUp(self): - self.evt = threading.Event() - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.settimeout(60) # Safety net. Look issue 11812 - self.port = socket_helper.bind_port(self.sock) - self.thread = threading.Thread(target=server, args=(self.evt,self.sock)) - self.thread.daemon = True - self.thread.start() - self.evt.wait() - - def tearDown(self): - self.thread.join() - del self.thread # Clear out any dangling Thread objects. - - def testBasic(self): - # connects - telnet = telnetlib.Telnet(HOST, self.port) - telnet.sock.close() - - def testContextManager(self): - with telnetlib.Telnet(HOST, self.port) as tn: - self.assertIsNotNone(tn.get_socket()) - self.assertIsNone(tn.get_socket()) - - def testTimeoutDefault(self): - self.assertTrue(socket.getdefaulttimeout() is None) - socket.setdefaulttimeout(30) - try: - telnet = telnetlib.Telnet(HOST, self.port) - finally: - socket.setdefaulttimeout(None) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testTimeoutNone(self): - # None, having other default - self.assertTrue(socket.getdefaulttimeout() is None) - socket.setdefaulttimeout(30) - try: - telnet = telnetlib.Telnet(HOST, self.port, timeout=None) - finally: - socket.setdefaulttimeout(None) - self.assertTrue(telnet.sock.gettimeout() is None) - telnet.sock.close() - - def testTimeoutValue(self): - telnet = telnetlib.Telnet(HOST, self.port, timeout=30) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testTimeoutOpen(self): - telnet = telnetlib.Telnet() - telnet.open(HOST, self.port, timeout=30) - self.assertEqual(telnet.sock.gettimeout(), 30) - telnet.sock.close() - - def testGetters(self): - # Test telnet getter methods - telnet = telnetlib.Telnet(HOST, self.port, timeout=30) - t_sock = telnet.sock - self.assertEqual(telnet.get_socket(), t_sock) - self.assertEqual(telnet.fileno(), t_sock.fileno()) - telnet.sock.close() - -class SocketStub(object): - ''' a socket proxy that re-defines sendall() ''' - def __init__(self, reads=()): - self.reads = list(reads) # Intentionally make a copy. - self.writes = [] - self.block = False - def sendall(self, data): - self.writes.append(data) - def recv(self, size): - out = b'' - while self.reads and len(out) < size: - out += self.reads.pop(0) - if len(out) > size: - self.reads.insert(0, out[size:]) - out = out[:size] - return out - -class TelnetAlike(telnetlib.Telnet): - def fileno(self): - raise NotImplementedError() - def close(self): pass - def sock_avail(self): - return (not self.sock.block) - def msg(self, msg, *args): - with support.captured_stdout() as out: - telnetlib.Telnet.msg(self, msg, *args) - self._messages += out.getvalue() - return - -class MockSelector(selectors.BaseSelector): - - def __init__(self): - self.keys = {} - - @property - def resolution(self): - return 1e-3 - - def register(self, fileobj, events, data=None): - key = selectors.SelectorKey(fileobj, 0, events, data) - self.keys[fileobj] = key - return key - - def unregister(self, fileobj): - return self.keys.pop(fileobj) - - def select(self, timeout=None): - block = False - for fileobj in self.keys: - if isinstance(fileobj, TelnetAlike): - block = fileobj.sock.block - break - if block: - return [] - else: - return [(key, key.events) for key in self.keys.values()] - - def get_map(self): - return self.keys - - -@contextlib.contextmanager -def test_socket(reads): - def new_conn(*ignored): - return SocketStub(reads) - try: - old_conn = socket.create_connection - socket.create_connection = new_conn - yield None - finally: - socket.create_connection = old_conn - return - -def test_telnet(reads=(), cls=TelnetAlike): - ''' return a telnetlib.Telnet object that uses a SocketStub with - reads queued up to be read ''' - for x in reads: - assert type(x) is bytes, x - with test_socket(reads): - telnet = cls('dummy', 0) - telnet._messages = '' # debuglevel output - return telnet - -class ExpectAndReadTestCase(unittest.TestCase): - def setUp(self): - self.old_selector = telnetlib._TelnetSelector - telnetlib._TelnetSelector = MockSelector - def tearDown(self): - telnetlib._TelnetSelector = self.old_selector - -class ReadTests(ExpectAndReadTestCase): - def test_read_until(self): - """ - read_until(expected, timeout=None) - test the blocking version of read_util - """ - want = [b'xxxmatchyyy'] - telnet = test_telnet(want) - data = telnet.read_until(b'match') - self.assertEqual(data, b'xxxmatch', msg=(telnet.cookedq, telnet.rawq, telnet.sock.reads)) - - reads = [b'x' * 50, b'match', b'y' * 50] - expect = b''.join(reads[:-1]) - telnet = test_telnet(reads) - data = telnet.read_until(b'match') - self.assertEqual(data, expect) - - - def test_read_all(self): - """ - read_all() - Read all data until EOF; may block. - """ - reads = [b'x' * 500, b'y' * 500, b'z' * 500] - expect = b''.join(reads) - telnet = test_telnet(reads) - data = telnet.read_all() - self.assertEqual(data, expect) - return - - def test_read_some(self): - """ - read_some() - Read at least one byte or EOF; may block. - """ - # test 'at least one byte' - telnet = test_telnet([b'x' * 500]) - data = telnet.read_some() - self.assertTrue(len(data) >= 1) - # test EOF - telnet = test_telnet() - data = telnet.read_some() - self.assertEqual(b'', data) - - def _read_eager(self, func_name): - """ - read_*_eager() - Read all data available already queued or on the socket, - without blocking. - """ - want = b'x' * 100 - telnet = test_telnet([want]) - func = getattr(telnet, func_name) - telnet.sock.block = True - self.assertEqual(b'', func()) - telnet.sock.block = False - data = b'' - while True: - try: - data += func() - except EOFError: - break - self.assertEqual(data, want) - - def test_read_eager(self): - # read_eager and read_very_eager make the same guarantees - # (they behave differently but we only test the guarantees) - self._read_eager('read_eager') - self._read_eager('read_very_eager') - # NB -- we need to test the IAC block which is mentioned in the - # docstring but not in the module docs - - def read_very_lazy(self): - want = b'x' * 100 - telnet = test_telnet([want]) - self.assertEqual(b'', telnet.read_very_lazy()) - while telnet.sock.reads: - telnet.fill_rawq() - data = telnet.read_very_lazy() - self.assertEqual(want, data) - self.assertRaises(EOFError, telnet.read_very_lazy) - - def test_read_lazy(self): - want = b'x' * 100 - telnet = test_telnet([want]) - self.assertEqual(b'', telnet.read_lazy()) - data = b'' - while True: - try: - read_data = telnet.read_lazy() - data += read_data - if not read_data: - telnet.fill_rawq() - except EOFError: - break - self.assertTrue(want.startswith(data)) - self.assertEqual(data, want) - -class nego_collector(object): - def __init__(self, sb_getter=None): - self.seen = b'' - self.sb_getter = sb_getter - self.sb_seen = b'' - - def do_nego(self, sock, cmd, opt): - self.seen += cmd + opt - if cmd == tl.SE and self.sb_getter: - sb_data = self.sb_getter() - self.sb_seen += sb_data - -tl = telnetlib - -class WriteTests(unittest.TestCase): - '''The only thing that write does is replace each tl.IAC for - tl.IAC+tl.IAC''' - - def test_write(self): - data_sample = [b'data sample without IAC', - b'data sample with' + tl.IAC + b' one IAC', - b'a few' + tl.IAC + tl.IAC + b' iacs' + tl.IAC, - tl.IAC, - b''] - for data in data_sample: - telnet = test_telnet() - telnet.write(data) - written = b''.join(telnet.sock.writes) - self.assertEqual(data.replace(tl.IAC,tl.IAC+tl.IAC), written) - -class OptionTests(unittest.TestCase): - # RFC 854 commands - cmds = [tl.AO, tl.AYT, tl.BRK, tl.EC, tl.EL, tl.GA, tl.IP, tl.NOP] - - def _test_command(self, data): - """ helper for testing IAC + cmd """ - telnet = test_telnet(data) - data_len = len(b''.join(data)) - nego = nego_collector() - telnet.set_option_negotiation_callback(nego.do_nego) - txt = telnet.read_all() - cmd = nego.seen - self.assertTrue(len(cmd) > 0) # we expect at least one command - self.assertIn(cmd[:1], self.cmds) - self.assertEqual(cmd[1:2], tl.NOOPT) - self.assertEqual(data_len, len(txt + cmd)) - nego.sb_getter = None # break the nego => telnet cycle - - def test_IAC_commands(self): - for cmd in self.cmds: - self._test_command([tl.IAC, cmd]) - self._test_command([b'x' * 100, tl.IAC, cmd, b'y'*100]) - self._test_command([b'x' * 10, tl.IAC, cmd, b'y'*10]) - # all at once - self._test_command([tl.IAC + cmd for (cmd) in self.cmds]) - - def test_SB_commands(self): - # RFC 855, subnegotiations portion - send = [tl.IAC + tl.SB + tl.IAC + tl.SE, - tl.IAC + tl.SB + tl.IAC + tl.IAC + tl.IAC + tl.SE, - tl.IAC + tl.SB + tl.IAC + tl.IAC + b'aa' + tl.IAC + tl.SE, - tl.IAC + tl.SB + b'bb' + tl.IAC + tl.IAC + tl.IAC + tl.SE, - tl.IAC + tl.SB + b'cc' + tl.IAC + tl.IAC + b'dd' + tl.IAC + tl.SE, - ] - telnet = test_telnet(send) - nego = nego_collector(telnet.read_sb_data) - telnet.set_option_negotiation_callback(nego.do_nego) - txt = telnet.read_all() - self.assertEqual(txt, b'') - want_sb_data = tl.IAC + tl.IAC + b'aabb' + tl.IAC + b'cc' + tl.IAC + b'dd' - self.assertEqual(nego.sb_seen, want_sb_data) - self.assertEqual(b'', telnet.read_sb_data()) - nego.sb_getter = None # break the nego => telnet cycle - - def test_debuglevel_reads(self): - # test all the various places that self.msg(...) is called - given_a_expect_b = [ - # Telnet.fill_rawq - (b'a', ": recv b''\n"), - # Telnet.process_rawq - (tl.IAC + bytes([88]), ": IAC 88 not recognized\n"), - (tl.IAC + tl.DO + bytes([1]), ": IAC DO 1\n"), - (tl.IAC + tl.DONT + bytes([1]), ": IAC DONT 1\n"), - (tl.IAC + tl.WILL + bytes([1]), ": IAC WILL 1\n"), - (tl.IAC + tl.WONT + bytes([1]), ": IAC WONT 1\n"), - ] - for a, b in given_a_expect_b: - telnet = test_telnet([a]) - telnet.set_debuglevel(1) - txt = telnet.read_all() - self.assertIn(b, telnet._messages) - return - - def test_debuglevel_write(self): - telnet = test_telnet() - telnet.set_debuglevel(1) - telnet.write(b'xxx') - expected = "send b'xxx'\n" - self.assertIn(expected, telnet._messages) - - def test_debug_accepts_str_port(self): - # Issue 10695 - with test_socket([]): - telnet = TelnetAlike('dummy', '0') - telnet._messages = '' - telnet.set_debuglevel(1) - telnet.msg('test') - self.assertRegex(telnet._messages, r'0.*test') - - -class ExpectTests(ExpectAndReadTestCase): - def test_expect(self): - """ - expect(expected, [timeout]) - Read until the expected string has been seen, or a timeout is - hit (default is no timeout); may block. - """ - want = [b'x' * 10, b'match', b'y' * 10] - telnet = test_telnet(want) - (_,_,data) = telnet.expect([b'match']) - self.assertEqual(data, b''.join(want[:-1])) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 1a175fa6544..f01e5dc7fb1 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -11,6 +11,9 @@ import stat import types import weakref +import gc +import shutil +import subprocess from unittest import mock import unittest @@ -60,16 +63,10 @@ def test_infer_return_type_multiples_and_none(self): tempfile._infer_return_type(b'', None, '') def test_infer_return_type_pathlib(self): - self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + self.assertIs(str, tempfile._infer_return_type(os_helper.FakePath('/'))) def test_infer_return_type_pathlike(self): - class Path: - def __init__(self, path): - self.path = path - - def __fspath__(self): - return self.path - + Path = os_helper.FakePath self.assertIs(str, tempfile._infer_return_type(Path('/'))) self.assertIs(bytes, tempfile._infer_return_type(Path(b'/'))) self.assertIs(str, tempfile._infer_return_type('', Path(''))) @@ -90,14 +87,10 @@ class BaseTestCase(unittest.TestCase): b_check = re.compile(br"^[a-z0-9_-]{8}$") def setUp(self): - self._warnings_manager = warnings_helper.check_warnings() - self._warnings_manager.__enter__() + self.enterContext(warnings_helper.check_warnings()) warnings.filterwarnings("ignore", category=RuntimeWarning, message="mktemp", module=__name__) - def tearDown(self): - self._warnings_manager.__exit__(None, None, None) - def nameCheck(self, name, dir, pre, suf): (ndir, nbase) = os.path.split(name) npre = nbase[:len(pre)] @@ -198,8 +191,7 @@ def supports_iter(self): if i == 20: break - @unittest.skipUnless(hasattr(os, 'fork'), - "os.fork is required for this test") + @support.requires_fork() def test_process_awareness(self): # ensure that the random source differs between # child and parent. @@ -290,19 +282,14 @@ def our_candidate_list(): def raise_OSError(*args, **kwargs): raise OSError() - with support.swap_attr(io, "open", raise_OSError): - # test again with failing io.open() + with support.swap_attr(os, "open", raise_OSError): + # test again with failing os.open() with self.assertRaises(FileNotFoundError): tempfile._get_default_tempdir() self.assertEqual(os.listdir(our_temp_directory), []) - def bad_writer(*args, **kwargs): - fp = orig_open(*args, **kwargs) - fp.write = raise_OSError - return fp - - with support.swap_attr(io, "open", bad_writer) as orig_open: - # test again with failing write() + with support.swap_attr(os, "write", raise_OSError): + # test again with failing os.write() with self.assertRaises(FileNotFoundError): tempfile._get_default_tempdir() self.assertEqual(os.listdir(our_temp_directory), []) @@ -341,7 +328,6 @@ def _mock_candidate_names(*names): class TestBadTempdir: - def test_read_only_directory(self): with _inside_empty_temp_dir(): oldmode = mode = os.stat(tempfile.tempdir).st_mode @@ -447,11 +433,12 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir).write(b"blat") - self.do_create(dir=pathlib.Path(dir)).write(b"blat") + self.do_create(dir=os_helper.FakePath(dir)).write(b"blat") finally: support.gc_collect() # For PyPy or other GCs. os.rmdir(dir) + @os_helper.skip_unless_working_chmod def test_file_mode(self): # _mkstemp_inner creates files with the proper mode @@ -465,8 +452,8 @@ def test_file_mode(self): expected = user * (1 + 8 + 64) self.assertEqual(mode, expected) - @support.requires_fork() @unittest.skipUnless(has_spawnl, 'os.spawnl not available') + @support.requires_subprocess() def test_noinherit(self): # _mkstemp_inner file handles are not inherited by child processes @@ -529,11 +516,11 @@ def test_collision_with_existing_file(self): _mock_candidate_names('aaa', 'aaa', 'bbb'): (fd1, name1) = self.make_temp() os.close(fd1) - self.assertTrue(name1.endswith('aaa')) + self.assertEndsWith(name1, 'aaa') (fd2, name2) = self.make_temp() os.close(fd2) - self.assertTrue(name2.endswith('bbb')) + self.assertEndsWith(name2, 'bbb') def test_collision_with_existing_directory(self): # _mkstemp_inner tries another name when a directory with @@ -541,11 +528,11 @@ def test_collision_with_existing_directory(self): with _inside_empty_temp_dir(), \ _mock_candidate_names('aaa', 'aaa', 'bbb'): dir = tempfile.mkdtemp() - self.assertTrue(dir.endswith('aaa')) + self.assertEndsWith(dir, 'aaa') (fd, name) = self.make_temp() os.close(fd) - self.assertTrue(name.endswith('bbb')) + self.assertEndsWith(name, 'bbb') class TestGetTempPrefix(BaseTestCase): @@ -684,7 +671,7 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: self.do_create(dir=dir) - self.do_create(dir=pathlib.Path(dir)) + self.do_create(dir=os_helper.FakePath(dir)) finally: os.rmdir(dir) @@ -785,10 +772,11 @@ def test_choose_directory(self): dir = tempfile.mkdtemp() try: os.rmdir(self.do_create(dir=dir)) - os.rmdir(self.do_create(dir=pathlib.Path(dir))) + os.rmdir(self.do_create(dir=os_helper.FakePath(dir))) finally: os.rmdir(dir) + @os_helper.skip_unless_working_chmod def test_mode(self): # mkdtemp creates directories with the proper mode @@ -806,6 +794,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # Use icacls.exe to extract the users with some level of access + # Main thing we are testing is that the BUILTIN\Users group has + # no access. The exact ACL is going to vary based on which user + # is running the test. + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold() + finally: + os.rmdir(dir) + + dir = dir.casefold() + users = set() + found_user = False + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line.removeprefix(dir).strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".casefold(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists @@ -813,9 +828,9 @@ def test_collision_with_existing_file(self): _mock_candidate_names('aaa', 'aaa', 'bbb'): file = tempfile.NamedTemporaryFile(delete=False) file.close() - self.assertTrue(file.name.endswith('aaa')) + self.assertEndsWith(file.name, 'aaa') dir = tempfile.mkdtemp() - self.assertTrue(dir.endswith('bbb')) + self.assertEndsWith(dir, 'bbb') def test_collision_with_existing_directory(self): # mkdtemp tries another name when a directory with @@ -823,9 +838,9 @@ def test_collision_with_existing_directory(self): with _inside_empty_temp_dir(), \ _mock_candidate_names('aaa', 'aaa', 'bbb'): dir1 = tempfile.mkdtemp() - self.assertTrue(dir1.endswith('aaa')) + self.assertEndsWith(dir1, 'aaa') dir2 = tempfile.mkdtemp() - self.assertTrue(dir2.endswith('bbb')) + self.assertEndsWith(dir2, 'bbb') def test_for_tempdir_is_bytes_issue40701_api_warts(self): orig_tempdir = tempfile.tempdir @@ -853,6 +868,15 @@ def test_for_tempdir_is_bytes_issue40701_api_warts(self): finally: tempfile.tempdir = orig_tempdir + def test_path_is_absolute(self): + # Test that the path returned by mkdtemp with a relative `dir` + # argument is absolute + try: + path = tempfile.mkdtemp(dir=".") + self.assertTrue(os.path.isabs(path)) + finally: + os.rmdir(path) + class TestMktemp(BaseTestCase): """Test mktemp().""" @@ -978,6 +1002,7 @@ def test_del_on_close(self): try: with tempfile.NamedTemporaryFile(dir=dir) as f: f.write(b'blat') + self.assertEqual(os.listdir(dir), []) self.assertFalse(os.path.exists(f.name), "NamedTemporaryFile %s exists after close" % f.name) finally: @@ -1017,18 +1042,104 @@ def use_closed(): pass self.assertRaises(ValueError, use_closed) - def test_no_leak_fd(self): - # Issue #21058: don't leak file descriptor when io.open() fails - closed = [] - os_close = os.close - def close(fd): - closed.append(fd) - os_close(fd) + def test_context_man_not_del_on_close_if_delete_on_close_false(self): + # Issue gh-58451: tempfile.NamedTemporaryFile is not particularly useful + # on Windows + # A NamedTemporaryFile is NOT deleted when closed if + # delete_on_close=False, but is deleted on context manager exit + dir = tempfile.mkdtemp() + try: + with tempfile.NamedTemporaryFile(dir=dir, + delete=True, + delete_on_close=False) as f: + f.write(b'blat') + f_name = f.name + f.close() + with self.subTest(): + # Testing that file is not deleted on close + self.assertTrue(os.path.exists(f.name), + f"NamedTemporaryFile {f.name!r} is incorrectly " + f"deleted on closure when delete_on_close=False") + + with self.subTest(): + # Testing that file is deleted on context manager exit + self.assertFalse(os.path.exists(f.name), + f"NamedTemporaryFile {f.name!r} exists " + f"after context manager exit") + + finally: + os.rmdir(dir) - with mock.patch('os.close', side_effect=close): - with mock.patch('io.open', side_effect=ValueError): - self.assertRaises(ValueError, tempfile.NamedTemporaryFile) - self.assertEqual(len(closed), 1) + def test_context_man_ok_to_delete_manually(self): + # In the case of delete=True, a NamedTemporaryFile can be manually + # deleted in a with-statement context without causing an error. + dir = tempfile.mkdtemp() + try: + with tempfile.NamedTemporaryFile(dir=dir, + delete=True, + delete_on_close=False) as f: + f.write(b'blat') + f.close() + os.unlink(f.name) + + finally: + os.rmdir(dir) + + def test_context_man_not_del_if_delete_false(self): + # A NamedTemporaryFile is not deleted if delete = False + dir = tempfile.mkdtemp() + f_name = "" + try: + # Test that delete_on_close=True has no effect if delete=False. + with tempfile.NamedTemporaryFile(dir=dir, delete=False, + delete_on_close=True) as f: + f.write(b'blat') + f_name = f.name + self.assertTrue(os.path.exists(f.name), + f"NamedTemporaryFile {f.name!r} exists after close") + finally: + os.unlink(f_name) + os.rmdir(dir) + + def test_del_by_finalizer(self): + # A NamedTemporaryFile is deleted when finalized in the case of + # delete=True, delete_on_close=False, and no with-statement is used. + def my_func(dir): + f = tempfile.NamedTemporaryFile(dir=dir, delete=True, + delete_on_close=False) + tmp_name = f.name + f.write(b'blat') + # Testing extreme case, where the file is not explicitly closed + # f.close() + return tmp_name + dir = tempfile.mkdtemp() + try: + with self.assertWarnsRegex( + expected_warning=ResourceWarning, + expected_regex=r"Implicitly cleaning up <_TemporaryFileWrapper file=.*>", + ): + tmp_name = my_func(dir) + support.gc_collect() + self.assertFalse(os.path.exists(tmp_name), + f"NamedTemporaryFile {tmp_name!r} " + f"exists after finalizer ") + finally: + os.rmdir(dir) + + def test_correct_finalizer_work_if_already_deleted(self): + # There should be no error in the case of delete=True, + # delete_on_close=False, no with-statement is used, and the file is + # deleted manually. + def my_func(dir)->str: + f = tempfile.NamedTemporaryFile(dir=dir, delete=True, + delete_on_close=False) + tmp_name = f.name + f.write(b'blat') + f.close() + os.unlink(tmp_name) + return tmp_name + # Make sure that the garbage collector has finalized the file object. + gc.collect() def test_bad_mode(self): dir = tempfile.mkdtemp() @@ -1039,6 +1150,24 @@ def test_bad_mode(self): tempfile.NamedTemporaryFile(mode=2, dir=dir) self.assertEqual(os.listdir(dir), []) + def test_bad_encoding(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with self.assertRaises(LookupError): + tempfile.NamedTemporaryFile('w', encoding='bad-encoding', dir=dir) + self.assertEqual(os.listdir(dir), []) + + def test_unexpected_error(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with mock.patch('tempfile._TemporaryFileWrapper') as mock_ntf, \ + mock.patch('io.open', mock.mock_open()) as mock_open: + mock_ntf.side_effect = KeyboardInterrupt() + with self.assertRaises(KeyboardInterrupt): + tempfile.NamedTemporaryFile(dir=dir) + mock_open().close.assert_called() + self.assertEqual(os.listdir(dir), []) + # How to test the mode and bufsize parameters? class TestSpooledTemporaryFile(BaseTestCase): @@ -1059,6 +1188,31 @@ def test_basic(self): f = self.do_create(max_size=100, pre="a", suf=".txt") self.assertFalse(f._rolled) + def test_is_iobase(self): + # SpooledTemporaryFile should implement io.IOBase + self.assertIsInstance(self.do_create(), io.IOBase) + + def test_iobase_interface(self): + # SpooledTemporaryFile should implement the io.IOBase interface. + # Ensure it has all the required methods and properties. + iobase_attrs = { + # From IOBase + 'fileno', 'seek', 'truncate', 'close', 'closed', '__enter__', + '__exit__', 'flush', 'isatty', '__iter__', '__next__', 'readable', + 'readline', 'readlines', 'seekable', 'tell', 'writable', + 'writelines', + # From BufferedIOBase (binary mode) and TextIOBase (text mode) + 'detach', 'read', 'read1', 'write', 'readinto', 'readinto1', + 'encoding', 'errors', 'newlines', + } + spooledtempfile_attrs = set(dir(tempfile.SpooledTemporaryFile)) + missing_attrs = iobase_attrs - spooledtempfile_attrs + self.assertFalse( + missing_attrs, + 'SpooledTemporaryFile missing attributes from ' + 'IOBase/BufferedIOBase/TextIOBase' + ) + def test_del_on_close(self): # A SpooledTemporaryFile is deleted when closed dir = tempfile.mkdtemp() @@ -1069,11 +1223,37 @@ def test_del_on_close(self): self.assertTrue(f._rolled) filename = f.name f.close() - self.assertFalse(isinstance(filename, str) and os.path.exists(filename), - "SpooledTemporaryFile %s exists after close" % filename) + self.assertEqual(os.listdir(dir), []) + if not isinstance(filename, int): + self.assertFalse(os.path.exists(filename), + "SpooledTemporaryFile %s exists after close" % filename) finally: os.rmdir(dir) + def test_del_unrolled_file(self): + # The unrolled SpooledTemporaryFile should raise a ResourceWarning + # when deleted since the file was not explicitly closed. + f = self.do_create(max_size=10) + f.write(b'foo') + self.assertEqual(f.name, None) # Unrolled so no filename/fd + with self.assertWarns(ResourceWarning): + f.__del__() + + def test_del_rolled_file(self): + # The rolled file should be deleted when the SpooledTemporaryFile + # object is deleted. This should raise a ResourceWarning since the file + # was not explicitly closed. + f = self.do_create(max_size=2) + f.write(b'foo') + name = f.name # This is a fd on posix+cygwin, a filename everywhere else + self.assertTrue(os.path.exists(name)) + with self.assertWarns(ResourceWarning): + f.__del__() + self.assertFalse( + os.path.exists(name), + "Rolled SpooledTemporaryFile (name=%s) exists after delete" % name + ) + def test_rewrite_small(self): # A SpooledTemporaryFile can be written to multiple within the max_size f = self.do_create(max_size=30) @@ -1104,6 +1284,34 @@ def test_writelines(self): buf = f.read() self.assertEqual(buf, b'xyz') + def test_writelines_rollover(self): + # Verify writelines rolls over before exhausting the iterator + f = self.do_create(max_size=2) + + def it(): + yield b'xy' + self.assertFalse(f._rolled) + yield b'z' + self.assertTrue(f._rolled) + + f.writelines(it()) + pos = f.seek(0) + self.assertEqual(pos, 0) + buf = f.read() + self.assertEqual(buf, b'xyz') + + def test_writelines_fast_path(self): + f = self.do_create(max_size=2) + f.write(b'abc') + self.assertTrue(f._rolled) + + f.writelines([b'd', b'e', b'f']) + pos = f.seek(0) + self.assertEqual(pos, 0) + buf = f.read() + self.assertEqual(buf, b'abcdef') + + def test_writelines_sequential(self): # A SpooledTemporaryFile should hold exactly max_size bytes, and roll # over afterward @@ -1187,8 +1395,6 @@ def test_properties(self): with self.assertRaises(AttributeError): f.errors - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_text_mode(self): # Creating a SpooledTemporaryFile with a text mode should produce # a file object reading and writing (Unicode) text strings. @@ -1221,8 +1427,6 @@ def test_text_mode(self): self.assertEqual(f.encoding, "utf-8") self.assertEqual(f.errors, "strict") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_text_newline_and_encoding(self): f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, newline='', encoding='utf-8', @@ -1361,19 +1565,34 @@ def roundtrip(input, *args, **kwargs): roundtrip("\u039B", "w+", encoding="utf-16") roundtrip("foo\r\n", "w+", newline="") - def test_no_leak_fd(self): - # Issue #21058: don't leak file descriptor when io.open() fails - closed = [] - os_close = os.close - def close(fd): - closed.append(fd) - os_close(fd) - - with mock.patch('os.close', side_effect=close): - with mock.patch('io.open', side_effect=ValueError): - self.assertRaises(ValueError, tempfile.TemporaryFile) - self.assertEqual(len(closed), 1) + def test_bad_mode(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with self.assertRaises(ValueError): + tempfile.TemporaryFile(mode='wr', dir=dir) + with self.assertRaises(TypeError): + tempfile.TemporaryFile(mode=2, dir=dir) + self.assertEqual(os.listdir(dir), []) + + def test_bad_encoding(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with self.assertRaises(LookupError): + tempfile.TemporaryFile('w', encoding='bad-encoding', dir=dir) + self.assertEqual(os.listdir(dir), []) + def test_unexpected_error(self): + dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dir) + with mock.patch('tempfile._O_TMPFILE_WORKS', False), \ + mock.patch('os.unlink') as mock_unlink, \ + mock.patch('os.open') as mock_open, \ + mock.patch('os.close') as mock_close: + mock_unlink.side_effect = KeyboardInterrupt() + with self.assertRaises(KeyboardInterrupt): + tempfile.TemporaryFile(dir=dir) + mock_close.assert_called() + self.assertEqual(os.listdir(dir), []) # Helper for test_del_on_shutdown @@ -1418,7 +1637,6 @@ def do_create2(self, path, recurse=1, dirs=1, files=1): with open(os.path.join(path, "test%d.txt" % i), "wb") as f: f.write(b"Hello world!") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_mkdtemp_failure(self): # Check no additional exception if mkdtemp fails # Previously would raise AttributeError instead @@ -1442,7 +1660,7 @@ def test_explicit_cleanup(self): finally: os.rmdir(dir) - def test_explict_cleanup_ignore_errors(self): + def test_explicit_cleanup_ignore_errors(self): """Test that cleanup doesn't return an error when ignoring them.""" with tempfile.TemporaryDirectory() as working_dir: temp_dir = self.do_create( @@ -1466,6 +1684,28 @@ def test_explict_cleanup_ignore_errors(self): temp_path.exists(), f"TemporaryDirectory {temp_path!s} exists after cleanup") + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_explicit_cleanup_correct_error(self): + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create(dir=working_dir) + with open(os.path.join(temp_dir.name, "example.txt"), 'wb'): + # Previously raised NotADirectoryError on some OSes + # (e.g. Windows). See bpo-43153. + with self.assertRaises(PermissionError): + temp_dir.cleanup() + + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_cleanup_with_used_directory(self): + with tempfile.TemporaryDirectory() as working_dir: + temp_dir = self.do_create(dir=working_dir) + subdir = os.path.join(temp_dir.name, "subdir") + os.mkdir(subdir) + with os_helper.change_cwd(subdir): + # Previously raised RecursionError on some OSes + # (e.g. Windows). See bpo-35144. + with self.assertRaises(PermissionError): + temp_dir.cleanup() + @os_helper.skip_unless_symlink def test_cleanup_with_symlink_to_a_directory(self): # cleanup() should not follow symlinks to directories (issue #12464) @@ -1487,6 +1727,104 @@ def test_cleanup_with_symlink_to_a_directory(self): "were deleted") d2.cleanup() + @unittest.skip("TODO: RUSTPYTHON; No such file or directory \"...\"") + @os_helper.skip_unless_symlink + def test_cleanup_with_symlink_modes(self): + # cleanup() should not follow symlinks when fixing mode bits (#91133) + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + for mode in range(8): + mode <<= 6 + with self.subTest(mode=format(mode, '03o')): + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chmod(symlink, mode, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chmod(symlink, mode) + except FileNotFoundError: + pass + os.chmod(d1.name, mode) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chmod(file1, mode) + old_mode = os.stat(file1).st_mode + test(file1, target_is_directory=False) + new_mode = os.stat(file1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + with self.subTest('existing dir'): + os.chmod(dir1, mode) + old_mode = os.stat(dir1).st_mode + test(dir1, target_is_directory=True) + new_mode = os.stat(dir1).st_mode + self.assertEqual(new_mode, old_mode, + '%03o != %03o' % (new_mode, old_mode)) + + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') + @os_helper.skip_unless_symlink + def test_cleanup_with_symlink_flags(self): + # cleanup() should not follow symlinks when fixing flags (#91133) + flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + + with self.do_create(recurse=0) as d2: + file1 = os.path.join(d2, 'file1') + open(file1, 'wb').close() + dir1 = os.path.join(d2, 'dir1') + os.mkdir(dir1) + def test(target, target_is_directory): + d1 = self.do_create(recurse=0) + symlink = os.path.join(d1.name, 'symlink') + os.symlink(target, symlink, + target_is_directory=target_is_directory) + try: + os.chflags(symlink, flags, follow_symlinks=False) + except NotImplementedError: + pass + try: + os.chflags(symlink, flags) + except FileNotFoundError: + pass + os.chflags(d1.name, flags) + d1.cleanup() + self.assertFalse(os.path.exists(d1.name)) + + with self.subTest('nonexisting file'): + test('nonexisting', target_is_directory=False) + with self.subTest('nonexisting dir'): + test('nonexisting', target_is_directory=True) + + with self.subTest('existing file'): + os.chflags(file1, flags) + old_flags = os.stat(file1).st_flags + test(file1, target_is_directory=False) + new_flags = os.stat(file1).st_flags + self.assertEqual(new_flags, old_flags) + + with self.subTest('existing dir'): + os.chflags(dir1, flags) + old_flags = os.stat(dir1).st_flags + test(dir1, target_is_directory=True) + new_flags = os.stat(dir1).st_flags + self.assertEqual(new_flags, old_flags) + @support.cpython_only def test_del_on_collection(self): # A TemporaryDirectory is deleted when garbage collected @@ -1659,9 +1997,27 @@ def test_modes(self): d.cleanup() self.assertFalse(os.path.exists(d.name)) - @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.lchflags') + def check_flags(self, flags): + # skip the test if these flags are not supported (ex: FreeBSD 13) + filename = os_helper.TESTFN + try: + open(filename, "w").close() + try: + os.chflags(filename, flags) + except OSError as exc: + # "OSError: [Errno 45] Operation not supported" + self.skipTest(f"chflags() doesn't support flags " + f"{flags:#b}: {exc}") + else: + os.chflags(filename, 0) + finally: + os_helper.unlink(filename) + + @unittest.skipUnless(hasattr(os, 'chflags'), 'requires os.chflags') def test_flags(self): flags = stat.UF_IMMUTABLE | stat.UF_NOUNLINK + self.check_flags(flags) + d = self.do_create(recurse=3, dirs=2, files=2) with d: # Change files and directories flags recursively. @@ -1672,6 +2028,11 @@ def test_flags(self): d.cleanup() self.assertFalse(os.path.exists(d.name)) + def test_delete_false(self): + with tempfile.TemporaryDirectory(delete=False) as working_dir: + pass + self.assertTrue(os.path.exists(working_dir)) + shutil.rmtree(working_dir) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py new file mode 100644 index 00000000000..e7ebb20b120 --- /dev/null +++ b/Lib/test/test_termios.py @@ -0,0 +1,313 @@ +import errno +import os +import sys +import tempfile +import threading +import unittest +from test import support +from test.support import threading_helper +from test.support.import_helper import import_module + +termios = import_module('termios') + + +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class TestFunctions(unittest.TestCase): + + def setUp(self): + self.master_fd, self.fd = os.openpty() + self.addCleanup(os.close, self.master_fd) + self.stream = self.enterContext(open(self.fd, 'wb', buffering=0)) + tmp = self.enterContext(tempfile.TemporaryFile(mode='wb', buffering=0)) + self.bad_fd = tmp.fileno() + + def assertRaisesTermiosError(self, err, callable, *args): + # Some versions of Android return EACCES when calling termios functions + # on a regular file. + errs = [err] + if sys.platform == 'android' and err == errno.ENOTTY: + errs.append(errno.EACCES) + + with self.assertRaises(termios.error) as cm: + callable(*args) + self.assertIn(cm.exception.args[0], errs) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'FileIO' found. + def test_tcgetattr(self): + attrs = termios.tcgetattr(self.fd) + self.assertIsInstance(attrs, list) + self.assertEqual(len(attrs), 7) + for i in range(6): + self.assertIsInstance(attrs[i], int) + iflag, oflag, cflag, lflag, ispeed, ospeed, cc = attrs + self.assertIsInstance(cc, list) + self.assertEqual(len(cc), termios.NCCS) + for i, x in enumerate(cc): + if ((lflag & termios.ICANON) == 0 and + (i == termios.VMIN or i == termios.VTIME)): + self.assertIsInstance(x, int) + else: + self.assertIsInstance(x, bytes) + self.assertEqual(len(x), 1) + self.assertEqual(termios.tcgetattr(self.stream), attrs) + + @unittest.skip("TODO: RUSTPYTHON segfault") + def test_tcgetattr_errors(self): + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcgetattr, self.bad_fd) + self.assertRaises(ValueError, termios.tcgetattr, -1) + self.assertRaises(OverflowError, termios.tcgetattr, 2**1000) + self.assertRaises(TypeError, termios.tcgetattr, object()) + self.assertRaises(TypeError, termios.tcgetattr) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'FileIO' found. + def test_tcsetattr(self): + attrs = termios.tcgetattr(self.fd) + termios.tcsetattr(self.fd, termios.TCSANOW, attrs) + termios.tcsetattr(self.fd, termios.TCSADRAIN, attrs) + termios.tcsetattr(self.fd, termios.TCSAFLUSH, attrs) + termios.tcsetattr(self.stream, termios.TCSANOW, attrs) + + @unittest.skip("TODO: RUSTPYTHON segfault") + def test_tcsetattr_errors(self): + attrs = termios.tcgetattr(self.fd) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, tuple(attrs)) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs[:-1]) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs + [0]) + for i in range(6): + attrs2 = attrs[:] + attrs2[i] = 2**1000 + self.assertRaises(OverflowError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + attrs2[i] = object() + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs[:-1] + [attrs[-1][:-1]]) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs[:-1] + [attrs[-1] + [b'\0']]) + for i in range(len(attrs[-1])): + attrs2 = attrs[:] + attrs2[-1] = attrs2[-1][:] + attrs2[-1][i] = 2**1000 + self.assertRaises(OverflowError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + attrs2[-1][i] = object() + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + attrs2[-1][i] = b'' + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + attrs2[-1][i] = b'\0\0' + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, object()) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW) + self.assertRaisesTermiosError(errno.EINVAL, termios.tcsetattr, self.fd, -1, attrs) + self.assertRaises(OverflowError, termios.tcsetattr, self.fd, 2**1000, attrs) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, object(), attrs) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcsetattr, self.bad_fd, termios.TCSANOW, attrs) + self.assertRaises(ValueError, termios.tcsetattr, -1, termios.TCSANOW, attrs) + self.assertRaises(OverflowError, termios.tcsetattr, 2**1000, termios.TCSANOW, attrs) + self.assertRaises(TypeError, termios.tcsetattr, object(), termios.TCSANOW, attrs) + self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'FileIO' found. + @support.skip_android_selinux('tcsendbreak') + def test_tcsendbreak(self): + try: + termios.tcsendbreak(self.fd, 1) + except termios.error as exc: + if exc.args[0] == errno.ENOTTY and sys.platform.startswith(('freebsd', "netbsd")): + self.skipTest('termios.tcsendbreak() is not supported ' + 'with pseudo-terminals (?) on this platform') + raise + termios.tcsendbreak(self.stream, 1) + + @unittest.skip("TODO: RUSTPYTHON segfault") + @support.skip_android_selinux('tcsendbreak') + def test_tcsendbreak_errors(self): + self.assertRaises(OverflowError, termios.tcsendbreak, self.fd, 2**1000) + self.assertRaises(TypeError, termios.tcsendbreak, self.fd, 0.0) + self.assertRaises(TypeError, termios.tcsendbreak, self.fd, object()) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcsendbreak, self.bad_fd, 0) + self.assertRaises(ValueError, termios.tcsendbreak, -1, 0) + self.assertRaises(OverflowError, termios.tcsendbreak, 2**1000, 0) + self.assertRaises(TypeError, termios.tcsendbreak, object(), 0) + self.assertRaises(TypeError, termios.tcsendbreak, self.fd) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'FileIO' found. + @support.skip_android_selinux('tcdrain') + def test_tcdrain(self): + termios.tcdrain(self.fd) + termios.tcdrain(self.stream) + + @unittest.skip("TODO: RUSTPYTHON segfault") + @support.skip_android_selinux('tcdrain') + def test_tcdrain_errors(self): + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcdrain, self.bad_fd) + self.assertRaises(ValueError, termios.tcdrain, -1) + self.assertRaises(OverflowError, termios.tcdrain, 2**1000) + self.assertRaises(TypeError, termios.tcdrain, object()) + self.assertRaises(TypeError, termios.tcdrain) + + def test_tcflush(self): + termios.tcflush(self.fd, termios.TCIFLUSH) + termios.tcflush(self.fd, termios.TCOFLUSH) + termios.tcflush(self.fd, termios.TCIOFLUSH) + + @unittest.skip("TODO: RUSTPYTHON segfault") + def test_tcflush_errors(self): + self.assertRaisesTermiosError(errno.EINVAL, termios.tcflush, self.fd, -1) + self.assertRaises(OverflowError, termios.tcflush, self.fd, 2**1000) + self.assertRaises(TypeError, termios.tcflush, self.fd, object()) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcflush, self.bad_fd, termios.TCIFLUSH) + self.assertRaises(ValueError, termios.tcflush, -1, termios.TCIFLUSH) + self.assertRaises(OverflowError, termios.tcflush, 2**1000, termios.TCIFLUSH) + self.assertRaises(TypeError, termios.tcflush, object(), termios.TCIFLUSH) + self.assertRaises(TypeError, termios.tcflush, self.fd) + + def test_tcflush_clear_input_or_output(self): + wfd = self.fd + rfd = self.master_fd + # The data is buffered in the input buffer on Linux, and in + # the output buffer on other platforms. + inbuf = sys.platform in ('linux', 'android') + + os.write(wfd, b'abcdef') + self.assertEqual(os.read(rfd, 2), b'ab') + if inbuf: + # don't flush input + termios.tcflush(rfd, termios.TCOFLUSH) + else: + # don't flush output + termios.tcflush(wfd, termios.TCIFLUSH) + self.assertEqual(os.read(rfd, 2), b'cd') + if inbuf: + # flush input + termios.tcflush(rfd, termios.TCIFLUSH) + else: + # flush output + termios.tcflush(wfd, termios.TCOFLUSH) + os.write(wfd, b'ABCDEF') + self.assertEqual(os.read(rfd, 1024), b'ABCDEF') + + @support.skip_android_selinux('tcflow') + def test_tcflow(self): + termios.tcflow(self.fd, termios.TCOOFF) + termios.tcflow(self.fd, termios.TCOON) + termios.tcflow(self.fd, termios.TCIOFF) + termios.tcflow(self.fd, termios.TCION) + + @unittest.skip("TODO: RUSTPYTHON segfault") + @support.skip_android_selinux('tcflow') + def test_tcflow_errors(self): + self.assertRaisesTermiosError(errno.EINVAL, termios.tcflow, self.fd, -1) + self.assertRaises(OverflowError, termios.tcflow, self.fd, 2**1000) + self.assertRaises(TypeError, termios.tcflow, self.fd, object()) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcflow, self.bad_fd, termios.TCOON) + self.assertRaises(ValueError, termios.tcflow, -1, termios.TCOON) + self.assertRaises(OverflowError, termios.tcflow, 2**1000, termios.TCOON) + self.assertRaises(TypeError, termios.tcflow, object(), termios.TCOON) + self.assertRaises(TypeError, termios.tcflow, self.fd) + + @support.skip_android_selinux('tcflow') + @unittest.skipUnless(sys.platform in ('linux', 'android'), 'only works on Linux') + def test_tcflow_suspend_and_resume_output(self): + wfd = self.fd + rfd = self.master_fd + write_suspended = threading.Event() + write_finished = threading.Event() + + def writer(): + os.write(wfd, b'abc') + self.assertTrue(write_suspended.wait(support.SHORT_TIMEOUT)) + os.write(wfd, b'def') + write_finished.set() + + with threading_helper.start_threads([threading.Thread(target=writer)]): + self.assertEqual(os.read(rfd, 3), b'abc') + try: + try: + termios.tcflow(wfd, termios.TCOOFF) + finally: + write_suspended.set() + self.assertFalse(write_finished.wait(0.5), + 'output was not suspended') + finally: + termios.tcflow(wfd, termios.TCOON) + self.assertTrue(write_finished.wait(support.SHORT_TIMEOUT), + 'output was not resumed') + self.assertEqual(os.read(rfd, 1024), b'def') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'termios' has no attribute 'tcgetwinsize' + def test_tcgetwinsize(self): + size = termios.tcgetwinsize(self.fd) + self.assertIsInstance(size, tuple) + self.assertEqual(len(size), 2) + self.assertIsInstance(size[0], int) + self.assertIsInstance(size[1], int) + self.assertEqual(termios.tcgetwinsize(self.stream), size) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'termios' has no attribute 'tcgetwinsize' + def test_tcgetwinsize_errors(self): + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcgetwinsize, self.bad_fd) + self.assertRaises(ValueError, termios.tcgetwinsize, -1) + self.assertRaises(OverflowError, termios.tcgetwinsize, 2**1000) + self.assertRaises(TypeError, termios.tcgetwinsize, object()) + self.assertRaises(TypeError, termios.tcgetwinsize) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'termios' has no attribute 'tcgetwinsize' + def test_tcsetwinsize(self): + size = termios.tcgetwinsize(self.fd) + termios.tcsetwinsize(self.fd, size) + termios.tcsetwinsize(self.fd, list(size)) + termios.tcsetwinsize(self.stream, size) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'termios' has no attribute 'tcgetwinsize' + def test_tcsetwinsize_errors(self): + size = termios.tcgetwinsize(self.fd) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, size[:-1]) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, size + (0,)) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, object()) + self.assertRaises(OverflowError, termios.tcsetwinsize, self.fd, (size[0], 2**1000)) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, (size[0], float(size[1]))) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, (size[0], object())) + self.assertRaises(OverflowError, termios.tcsetwinsize, self.fd, (2**1000, size[1])) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, (float(size[0]), size[1])) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd, (object(), size[1])) + self.assertRaisesTermiosError(errno.ENOTTY, termios.tcsetwinsize, self.bad_fd, size) + self.assertRaises(ValueError, termios.tcsetwinsize, -1, size) + self.assertRaises(OverflowError, termios.tcsetwinsize, 2**1000, size) + self.assertRaises(TypeError, termios.tcsetwinsize, object(), size) + self.assertRaises(TypeError, termios.tcsetwinsize, self.fd) + + +class TestModule(unittest.TestCase): + def test_constants(self): + self.assertIsInstance(termios.B0, int) + self.assertIsInstance(termios.B38400, int) + self.assertIsInstance(termios.TCSANOW, int) + self.assertIsInstance(termios.TCSADRAIN, int) + self.assertIsInstance(termios.TCSAFLUSH, int) + self.assertIsInstance(termios.TCIFLUSH, int) + self.assertIsInstance(termios.TCOFLUSH, int) + self.assertIsInstance(termios.TCIOFLUSH, int) + self.assertIsInstance(termios.TCOOFF, int) + self.assertIsInstance(termios.TCOON, int) + self.assertIsInstance(termios.TCIOFF, int) + self.assertIsInstance(termios.TCION, int) + self.assertIsInstance(termios.VTIME, int) + self.assertIsInstance(termios.VMIN, int) + self.assertIsInstance(termios.NCCS, int) + self.assertLess(termios.VTIME, termios.NCCS) + self.assertLess(termios.VMIN, termios.NCCS) + + def test_ioctl_constants(self): + # gh-119770: ioctl() constants must be positive + for name in dir(termios): + if not name.startswith('TIO'): + continue + value = getattr(termios, name) + with self.subTest(name=name): + self.assertGreaterEqual(value, 0) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'termios.error'> is a subclass of <class 'OSError'> + def test_exception(self): + self.assertIsSubclass(termios.error, Exception) + self.assertNotIsSubclass(termios.error, OSError) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index dfbc2b93dfc..aca1f427656 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -605,7 +605,7 @@ def test_break_long(self): # bug 1146. Prevent a long word to be wrongly wrapped when the # preceding word is exactly one character shorter than the width self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', 'ialidocious?', @@ -633,7 +633,7 @@ def test_nobreak_long(self): def test_max_lines_long(self): self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', '[...]'], @@ -765,10 +765,67 @@ def test_subsequent_indent(self): # of IndentTestCase! class DedentTestCase(unittest.TestCase): + def test_type_error(self): + with self.assertRaisesRegex(TypeError, "expected str object, not"): + dedent(0) + + with self.assertRaisesRegex(TypeError, "expected str object, not"): + dedent(b'') + def assertUnchanged(self, text): """assert that dedent() has no effect on 'text'""" self.assertEqual(text, dedent(text)) + def test_dedent_whitespace(self): + # The empty string. + text = "" + self.assertUnchanged(text) + + # Only spaces. + text = " " + expect = "" + self.assertEqual(expect, dedent(text)) + + # Only tabs. + text = "\t\t\t\t" + expect = "" + self.assertEqual(expect, dedent(text)) + + # A mixture. + text = " \t \t\t \t " + expect = "" + self.assertEqual(expect, dedent(text)) + + # ASCII whitespace. + text = "\f\n\r\t\v " + expect = "\n" + self.assertEqual(expect, dedent(text)) + + # One newline. + text = "\n" + expect = "\n" + self.assertEqual(expect, dedent(text)) + + # Windows-style newlines. + text = "\r\n" * 5 + expect = "\n" * 5 + self.assertEqual(expect, dedent(text)) + + # Whitespace mixture. + text = " \n\t\n \n\t\t\n\n\n " + expect = "\n\n\n\n\n\n" + self.assertEqual(expect, dedent(text)) + + # Lines consisting only of whitespace are always normalised + text = "a\n \n\t\n" + expect = "a\n\n\n" + self.assertEqual(expect, dedent(text)) + + # Whitespace characters on non-empty lines are retained + text = "a\r\n\r\n\r\n" + expect = "a\r\n\n\n" + self.assertEqual(expect, dedent(text)) + def test_dedent_nomargin(self): # No lines indented. text = "Hello there.\nHow are you?\nOh good, I'm glad." diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index f55cf3656ea..4ae8a833b99 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -105,7 +105,6 @@ def test_nt_and_posix_stack_size(self): thread.stack_size(0) - @unittest.skip("TODO: RUSTPYTHON, weakref destructors") def test__count(self): # Test the _count() function. orig = thread._count() diff --git a/Lib/test/test_threadedtempfile.py b/Lib/test/test_threadedtempfile.py index b088f5baf7b..12feb465dbe 100644 --- a/Lib/test/test_threadedtempfile.py +++ b/Lib/test/test_threadedtempfile.py @@ -50,7 +50,6 @@ def run(self): class ThreadedTempFileTest(unittest.TestCase): - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_main(self): threads = [TempFileGreedy() for i in range(NUM_THREADS)] with threading_helper.start_threads(threads, startEvent.set): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 92ff3dc3809..17693ae093f 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,10 +3,11 @@ """ import test.support -from test.support import threading_helper +from test.support import threading_helper, requires_subprocess, requires_gil_enabled from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support import force_not_colorized import random import sys @@ -20,11 +21,18 @@ import signal import textwrap import traceback +import warnings from unittest import mock from test import lock_tests from test import support +try: + from test.support import interpreters +except ImportError: + interpreters = None + +threading_helper.requires_working_threading(module=True) # Between fork() and exec(), only async-safe functions are allowed (issues # #12316 and #11870), and fork() from a worker thread is known to trigger @@ -32,8 +40,23 @@ # on platforms known to behave badly. platforms_to_skip = ('netbsd5', 'hp-ux11') -# Is Python built with Py_DEBUG macro defined? -Py_DEBUG = hasattr(sys, 'gettotalrefcount') + +def skip_unless_reliable_fork(test): + if not support.has_fork_support: + return unittest.skip("requires working os.fork()")(test) + if sys.platform in platforms_to_skip: + return unittest.skip("due to known OS bug related to thread+fork")(test) + if support.HAVE_ASAN_FORK_BUG: + return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) + if support.check_sanitizer(thread=True): + return unittest.skip("TSAN doesn't support threads after fork")(test) + return test + + +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(interpreters is None, + 'subinterpreters required')(meth) def restore_default_excepthook(testcase): @@ -95,6 +118,7 @@ def tearDown(self): class ThreadTests(BaseTestCase): + maxDiff = 9999 @cpython_only def test_name(self): @@ -123,11 +147,48 @@ def func(): pass thread = threading.Thread(target=func) self.assertEqual(thread.name, "Thread-5 (func)") - @cpython_only - def test_disallow_instantiation(self): - # Ensure that the type disallows instantiation (bpo-43916) - lock = threading.Lock() - test.support.check_disallow_instantiation(self, type(lock)) + def test_args_argument(self): + # bpo-45735: Using list or tuple as *args* in constructor could + # achieve the same effect. + num_list = [1] + num_tuple = (1,) + + str_list = ["str"] + str_tuple = ("str",) + + list_in_tuple = ([1],) + tuple_in_list = [(1,)] + + test_cases = ( + (num_list, lambda arg: self.assertEqual(arg, 1)), + (num_tuple, lambda arg: self.assertEqual(arg, 1)), + (str_list, lambda arg: self.assertEqual(arg, "str")), + (str_tuple, lambda arg: self.assertEqual(arg, "str")), + (list_in_tuple, lambda arg: self.assertEqual(arg, [1])), + (tuple_in_list, lambda arg: self.assertEqual(arg, (1,))) + ) + + for args, target in test_cases: + with self.subTest(target=target, args=args): + t = threading.Thread(target=target, args=args) + t.start() + t.join() + + def test_lock_no_args(self): + threading.Lock() # works + self.assertRaises(TypeError, threading.Lock, 1) + self.assertRaises(TypeError, threading.Lock, a=1) + self.assertRaises(TypeError, threading.Lock, 1, 2, a=1, b=2) + + def test_lock_no_subclass(self): + # Intentionally disallow subclasses of threading.Lock because they have + # never been allowed, so why start now just because the type is public? + with self.assertRaises(TypeError): + class MyLock(threading.Lock): pass + + def test_lock_or_none(self): + import types + self.assertIsInstance(threading.Lock | None, types.UnionType) # Create a bunch of threads, let each do some work, wait until all are # done. @@ -179,8 +240,6 @@ def f(): tid = _thread.start_new_thread(f, ()) done.wait() self.assertEqual(ident[0], tid) - # Kill the "immortal" _DummyThread - del threading._active[ident[0]] # run with a small(ish) thread stack size (256 KiB) def test_various_ops_small_stack(self): @@ -208,11 +267,29 @@ def test_various_ops_large_stack(self): def test_foreign_thread(self): # Check that a "foreign" thread can use the threading module. + dummy_thread = None + error = None def f(mutex): - # Calling current_thread() forces an entry for the foreign - # thread to get made in the threading._active map. - threading.current_thread() - mutex.release() + try: + nonlocal dummy_thread + nonlocal error + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + dummy_thread = threading.current_thread() + tid = dummy_thread.ident + self.assertIn(tid, threading._active) + self.assertIsInstance(dummy_thread, threading._DummyThread) + self.assertIs(threading._active.get(tid), dummy_thread) + # gh-29376 + self.assertTrue( + dummy_thread.is_alive(), + 'Expected _DummyThread to be considered alive.' + ) + self.assertIn('_DummyThread', repr(dummy_thread)) + except BaseException as e: + error = e + finally: + mutex.release() mutex = threading.Lock() mutex.acquire() @@ -220,15 +297,29 @@ def f(mutex): tid = _thread.start_new_thread(f, (mutex,)) # Wait for the thread to finish. mutex.acquire() - self.assertIn(tid, threading._active) - self.assertIsInstance(threading._active[tid], threading._DummyThread) - #Issue 29376 - self.assertTrue(threading._active[tid].is_alive()) - self.assertRegex(repr(threading._active[tid]), '_DummyThread') - del threading._active[tid] + if error is not None: + raise error + self.assertEqual(tid, dummy_thread.ident) + # Issue gh-106236: + with self.assertRaises(RuntimeError): + dummy_thread.join() + dummy_thread._started.clear() + with self.assertRaises(RuntimeError): + dummy_thread.is_alive() + # Busy wait for the following condition: after the thread dies, the + # related dummy thread must be removed from threading._active. + timeout = 5 + timeout_at = time.monotonic() + timeout + while time.monotonic() < timeout_at: + if threading._active.get(dummy_thread.ident) is not dummy_thread: + break + time.sleep(.1) + else: + self.fail('It was expected that the created threading._DummyThread was removed from threading._active.') # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) # exposed at the Python level. This test relies on ctypes to get at it. + @cpython_only def test_PyThreadState_SetAsyncExc(self): ctypes = import_module("ctypes") @@ -317,12 +408,13 @@ def run(self): t.join() # else the thread is still running, and we have no way to kill it + @unittest.skip('TODO: RUSTPYTHON; threading._start_new_thread not exposed') def test_limbo_cleanup(self): # Issue 7481: Failure to start thread should cleanup the limbo map. - def fail_new_thread(*args): + def fail_new_thread(*args, **kwargs): raise threading.ThreadError() - _start_new_thread = threading._start_new_thread - threading._start_new_thread = fail_new_thread + _start_joinable_thread = threading._start_joinable_thread + threading._start_joinable_thread = fail_new_thread try: t = threading.Thread(target=lambda: None) self.assertRaises(threading.ThreadError, t.start) @@ -330,12 +422,17 @@ def fail_new_thread(*args): t in threading._limbo, "Failed to cleanup _limbo map on failure of Thread.start().") finally: - threading._start_new_thread = _start_new_thread + threading._start_joinable_thread = _start_joinable_thread + @unittest.expectedFailure # TODO: RUSTPYTHON; ctypes.pythonapi is not supported def test_finalize_running_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for # example. + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") import_module("ctypes") rc, out, err = assert_python_failure("-c", """if 1: @@ -368,6 +465,11 @@ def waitingThread(): def test_finalize_with_trace(self): # Issue1733757 # Avoid a deadlock when sys.settrace steps into threading._shutdown + if support.check_sanitizer(thread=True): + # the thread running `time.sleep(2)` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + assert_python_ok("-c", """if 1: import sys, threading @@ -390,8 +492,6 @@ def func(frame, event, arg): sys.settrace(func) """) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_join_nondaemon_on_shutdown(self): # Issue 1722344 # Raising SystemExit skipped threading._shutdown @@ -412,8 +512,6 @@ def child(): b"Woke up, sleep function is: <built-in function sleep>") self.assertEqual(err, b"") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_enumerate_after_join(self): # Try hard to trigger #1703448: a thread is still returned in # threading.enumerate() after it has been join()ed. @@ -421,7 +519,7 @@ def test_enumerate_after_join(self): old_interval = sys.getswitchinterval() try: for i in range(1, 100): - sys.setswitchinterval(i * 0.0002) + support.setswitchinterval(i * 0.0002) t = threading.Thread(target=lambda: None) t.start() t.join() @@ -431,6 +529,48 @@ def test_enumerate_after_join(self): finally: sys.setswitchinterval(old_interval) + @support.bigmemtest(size=20, memuse=72*2**20, dry_run=False) + def test_join_from_multiple_threads(self, size): + # Thread.join() should be thread-safe + errors = [] + + def worker(): + time.sleep(0.005) + + def joiner(thread): + try: + thread.join() + except Exception as e: + errors.append(e) + + for N in range(2, 20): + threads = [threading.Thread(target=worker)] + for i in range(N): + threads.append(threading.Thread(target=joiner, + args=(threads[0],))) + for t in threads: + t.start() + time.sleep(0.01) + for t in threads: + t.join() + if errors: + raise errors[0] + + def test_join_with_timeout(self): + lock = _thread.allocate_lock() + lock.acquire() + + def worker(): + lock.acquire() + + thread = threading.Thread(target=worker) + thread.start() + thread.join(timeout=0.01) + assert thread.is_alive() + lock.release() + thread.join() + assert not thread.is_alive() + def test_no_refcycle_through_target(self): class RunSelfFunction(object): def __init__(self, should_raise): @@ -509,41 +649,12 @@ def test_daemon_param(self): t = threading.Thread(daemon=True) self.assertTrue(t.daemon) - @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, exit_handler needs lock._at_fork_reinit') - @unittest.skipUnless(hasattr(os, 'fork'), 'needs os.fork()') - def test_fork_at_exit(self): - # bpo-42350: Calling os.fork() after threading._shutdown() must - # not log an error. - code = textwrap.dedent(""" - import atexit - import os - import sys - from test.support import wait_process - - # Import the threading module to register its "at fork" callback - import threading - - def exit_handler(): - pid = os.fork() - if not pid: - print("child process ok", file=sys.stderr, flush=True) - # child process - else: - wait_process(pid, exitcode=0) - - # exit_handler() will be called after threading._shutdown() - atexit.register(exit_handler) - """) - _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') - self.assertEqual(err.rstrip(), b'child process ok') - - @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + @skip_unless_reliable_fork def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up # the after-fork mechanism. code = """if 1: - import _thread, threading, os, time + import _thread, threading, os, time, warnings def background_thread(evt): # Creates and registers the _DummyThread instance @@ -555,18 +666,23 @@ def background_thread(evt): _thread.start_new_thread(background_thread, (evt,)) evt.wait() assert threading.active_count() == 2, threading.active_count() - if os.fork() == 0: - assert threading.active_count() == 1, threading.active_count() - os._exit(0) - else: - os.wait() + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + os.wait() """ _, out, err = assert_python_ok("-c", code) self.assertEqual(out, b'') self.assertEqual(err, b'') - @unittest.skipUnless(hasattr(sys, 'getswitchinterval'), "TODO: RUSTPYTHON, needs sys.getswitchinterval()") - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skip('TODO: RUSTPYTHON; flaky') + @skip_unless_reliable_fork def test_is_alive_after_fork(self): # Try hard to trigger #18418: is_alive() could sometimes be True on # threads that vanished after a fork. @@ -579,13 +695,15 @@ def test_is_alive_after_fork(self): for i in range(20): t = threading.Thread(target=lambda: None) t.start() - pid = os.fork() - if pid == 0: - os._exit(11 if t.is_alive() else 10) - else: - t.join() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + if (pid := os.fork()) == 0: + os._exit(11 if t.is_alive() else 10) + else: + t.join() - support.wait_process(pid, exitcode=10) + support.wait_process(pid, exitcode=10) def test_main_thread(self): main = threading.main_thread() @@ -600,60 +718,142 @@ def f(): th.start() th.join() - @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork(self): code = """if 1: import os, threading from test import support + ident = threading.get_ident() pid = os.fork() if pid == 0: + print("current ident", threading.get_ident() == ident) main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) + print("main", main.name) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) else: support.wait_process(pid, exitcode=0) """ _, out, err = assert_python_ok("-c", code) data = out.decode().replace('\r', '') self.assertEqual(err, b"") - self.assertEqual(data, "MainThread\nTrue\nTrue\n") - - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + self.assertEqual(data, + "current ident True\n" + "main MainThread\n" + "main ident True\n" + "current is main True\n") + + @unittest.skip("TODO: RUSTPYTHON flaky; process timeout after fork") + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") - @unittest.skipIf(os.name != 'posix', "test needs POSIX semantics") def test_main_thread_after_fork_from_nonmain_thread(self): code = """if 1: - import os, threading, sys + import os, threading, sys, warnings from test import support def func(): + ident = threading.get_ident() + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + pid = os.fork() + if pid == 0: + print("current ident", threading.get_ident() == ident) + main = threading.main_thread() + print("main", main.name, type(main).__name__) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + support.wait_process(pid, exitcode=0) + + th = threading.Thread(target=func) + th.start() + th.join() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err.decode('utf-8'), "") + self.assertEqual(data, + "current ident True\n" + "main Thread-1 (func) Thread\n" + "main ident True\n" + "current is main True\n" + ) + + @skip_unless_reliable_fork + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_foreign_thread(self, create_dummy=False): + code = """if 1: + import os, threading, sys, traceback, _thread + from test import support + + def func(lock): + ident = threading.get_ident() + if %s: + # call current_thread() before fork to allocate DummyThread + current = threading.current_thread() + print("current", current.name, type(current).__name__) + print("ident in _active", ident in threading._active) + # flush before fork, so child won't flush it again + sys.stdout.flush() pid = os.fork() if pid == 0: + print("current ident", threading.get_ident() == ident) main = threading.main_thread() - print(main.name) - print(main.ident == threading.current_thread().ident) - print(main.ident == threading.get_ident()) + print("main", main.name, type(main).__name__) + print("main ident", main.ident == ident) + print("current is main", threading.current_thread() is main) + print("_dangling", [t.name for t in list(threading._dangling)]) # stdout is fully buffered because not a tty, # we have to flush before exit. sys.stdout.flush() + try: + threading._shutdown() + os._exit(0) + except: + traceback.print_exc() + sys.stderr.flush() + os._exit(1) else: - support.wait_process(pid, exitcode=0) - - th = threading.Thread(target=func) - th.start() - th.join() - """ - _, out, err = assert_python_ok("-c", code) + try: + support.wait_process(pid, exitcode=0) + except Exception: + # avoid 'could not acquire lock for + # <_io.BufferedWriter name='<stderr>'> at interpreter shutdown,' + traceback.print_exc() + sys.stderr.flush() + finally: + lock.release() + + join_lock = _thread.allocate_lock() + join_lock.acquire() + th = _thread.start_new_thread(func, (join_lock,)) + join_lock.acquire() + """ % create_dummy + # "DeprecationWarning: This process is multi-threaded, use of fork() + # may lead to deadlocks in the child" + _, out, err = assert_python_ok("-W", "ignore::DeprecationWarning", "-c", code) data = out.decode().replace('\r', '') - self.assertEqual(err, b"") - self.assertEqual(data, "Thread-1 (func)\nTrue\nTrue\n") + self.assertEqual(err.decode(), "") + self.assertEqual(data, + ("current Dummy-1 _DummyThread\n" if create_dummy else "") + + f"ident in _active {create_dummy!s}\n" + + "current ident True\n" + "main MainThread _MainThread\n" + "main ident True\n" + "current is main True\n" + "_dangling ['MainThread']\n") + + def test_main_thread_after_fork_from_dummy_thread(self, create_dummy=False): + self.test_main_thread_after_fork_from_foreign_thread(create_dummy=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_main_thread_during_shutdown(self): # bpo-31516: current_thread() should still point to the main thread # at shutdown @@ -718,41 +918,6 @@ def f(): rc, out, err = assert_python_ok("-c", code) self.assertEqual(err, b"") - def test_tstate_lock(self): - # Test an implementation detail of Thread objects. - started = _thread.allocate_lock() - finish = _thread.allocate_lock() - started.acquire() - finish.acquire() - def f(): - started.release() - finish.acquire() - time.sleep(0.01) - # The tstate lock is None until the thread is started - t = threading.Thread(target=f) - self.assertIs(t._tstate_lock, None) - t.start() - started.acquire() - self.assertTrue(t.is_alive()) - # The tstate lock can't be acquired when the thread is running - # (or suspended). - tstate_lock = t._tstate_lock - self.assertFalse(tstate_lock.acquire(timeout=0), False) - finish.release() - # When the thread ends, the state_lock can be successfully - # acquired. - self.assertTrue(tstate_lock.acquire(timeout=support.SHORT_TIMEOUT), False) - # But is_alive() is still True: we hold _tstate_lock now, which - # prevents is_alive() from knowing the thread's end-of-life C code - # is done. - self.assertTrue(t.is_alive()) - # Let is_alive() find out the C code is done. - tstate_lock.release() - self.assertFalse(t.is_alive()) - # And verify the thread disposed of _tstate_lock. - self.assertIsNone(t._tstate_lock) - t.join() - def test_repr_stopped(self): # Verify that "stopped" shows up in repr(Thread) appropriately. started = _thread.allocate_lock() @@ -800,6 +965,7 @@ def test_BoundedSemaphore_limit(self): @cpython_only def test_frame_tstate_tracing(self): + _testcapi = import_module("_testcapi") # Issue #14432: Crash when a generator is created in a C thread that is # destroyed while the generator is still used. The issue was that a # generator contains a frame, and the frame kept a reference to the @@ -827,7 +993,6 @@ def callback(): threading.settrace(noop_trace) # Create a generator in a C thread which exits after the call - import _testcapi _testcapi.call_in_temporary_c_thread(callback) # Call the generator in a different Python thread, check that the @@ -837,6 +1002,7 @@ def callback(): callback() finally: sys.settrace(old_trace) + threading.settrace(old_trace) def test_gettrace(self): def noop_trace(frame, event, arg): @@ -850,6 +1016,36 @@ def noop_trace(frame, event, arg): finally: threading.settrace(old_trace) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_gettrace_all_threads(self): + def fn(*args): pass + old_trace = threading.gettrace() + first_check = threading.Event() + second_check = threading.Event() + + trace_funcs = [] + def checker(): + trace_funcs.append(sys.gettrace()) + first_check.set() + second_check.wait() + trace_funcs.append(sys.gettrace()) + + try: + t = threading.Thread(target=checker) + t.start() + first_check.wait() + threading.settrace_all_threads(fn) + second_check.set() + t.join() + self.assertEqual(trace_funcs, [None, fn]) + self.assertEqual(threading.gettrace(), fn) + self.assertEqual(sys.gettrace(), fn) + finally: + threading.settrace_all_threads(old_trace) + + self.assertEqual(threading.gettrace(), old_trace) + self.assertEqual(sys.gettrace(), old_trace) + def test_getprofile(self): def fn(*args): pass old_profile = threading.getprofile() @@ -859,32 +1055,36 @@ def fn(*args): pass finally: threading.setprofile(old_profile) - @cpython_only - def test_shutdown_locks(self): - for daemon in (False, True): - with self.subTest(daemon=daemon): - event = threading.Event() - thread = threading.Thread(target=event.wait, daemon=daemon) - - # Thread.start() must add lock to _shutdown_locks, - # but only for non-daemon thread - thread.start() - tstate_lock = thread._tstate_lock - if not daemon: - self.assertIn(tstate_lock, threading._shutdown_locks) - else: - self.assertNotIn(tstate_lock, threading._shutdown_locks) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_getprofile_all_threads(self): + def fn(*args): pass + old_profile = threading.getprofile() + first_check = threading.Event() + second_check = threading.Event() - # unblock the thread and join it - event.set() - thread.join() + profile_funcs = [] + def checker(): + profile_funcs.append(sys.getprofile()) + first_check.set() + second_check.wait() + profile_funcs.append(sys.getprofile()) + + try: + t = threading.Thread(target=checker) + t.start() + first_check.wait() + threading.setprofile_all_threads(fn) + second_check.set() + t.join() + self.assertEqual(profile_funcs, [None, fn]) + self.assertEqual(threading.getprofile(), fn) + self.assertEqual(sys.getprofile(), fn) + finally: + threading.setprofile_all_threads(old_profile) - # Thread._stop() must remove tstate_lock from _shutdown_locks. - # Daemon threads must never add it to _shutdown_locks. - self.assertNotIn(tstate_lock, threading._shutdown_locks) + self.assertEqual(threading.getprofile(), old_profile) + self.assertEqual(sys.getprofile(), old_profile) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_locals_at_exit(self): # bpo-19466: thread locals must not be deleted before destructors # are called @@ -929,16 +1129,6 @@ def noop(): pass threading.Thread(target=noop).start() # Thread.join() is not called - @unittest.skipUnless(Py_DEBUG, 'need debug build (Py_DEBUG)') - def test_debug_deprecation(self): - # bpo-44584: The PYTHONTHREADDEBUG environment variable is deprecated - rc, out, err = assert_python_ok("-Wdefault", "-c", "pass", - PYTHONTHREADDEBUG="1") - msg = (b'DeprecationWarning: The threading debug ' - b'(PYTHONTHREADDEBUG environment variable) ' - b'is deprecated and will be removed in Python 3.12') - self.assertIn(msg, err) - def test_import_from_another_thread(self): # bpo-1596321: If the threading module is first import from a thread # different than the main thread, threading._shutdown() must handle @@ -972,6 +1162,162 @@ def import_threading(): self.assertEqual(out, b'') self.assertEqual(err, b'') + def test_start_new_thread_at_finalization(self): + code = """if 1: + import _thread + + def f(): + print("shouldn't be printed") + + class AtFinalization: + def __del__(self): + print("OK") + _thread.start_new_thread(f, ()) + at_finalization = AtFinalization() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out.strip(), b"OK") + self.assertIn(b"can't create new thread at interpreter shutdown", err) + + def test_start_new_thread_failed(self): + # gh-109746: if Python fails to start newly created thread + # due to failure of underlying PyThread_start_new_thread() call, + # its state should be removed from interpreter' thread states list + # to avoid its double cleanup + try: + from resource import setrlimit, RLIMIT_NPROC + except ImportError as err: + self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD + code = """if 1: + import resource + import _thread + + def f(): + print("shouldn't be printed") + + limits = resource.getrlimit(resource.RLIMIT_NPROC) + [_, hard] = limits + resource.setrlimit(resource.RLIMIT_NPROC, (0, hard)) + + try: + handle = _thread.start_joinable_thread(f) + except RuntimeError: + print('ok') + else: + print('!skip!') + handle.join() + """ + _, out, err = assert_python_ok("-u", "-c", code) + out = out.strip() + if b'!skip!' in out: + self.skipTest('RLIMIT_NPROC had no effect; probably superuser') + self.assertEqual(out, b'ok') + self.assertEqual(err, b'') + + + @skip_unless_reliable_fork + @unittest.skipUnless(hasattr(threading, 'get_native_id'), "test needs threading.get_native_id()") + def test_native_id_after_fork(self): + script = """if True: + import threading + import os + from test import support + + parent_thread_native_id = threading.current_thread().native_id + print(parent_thread_native_id, flush=True) + assert parent_thread_native_id == threading.get_native_id() + childpid = os.fork() + if childpid == 0: + print(threading.current_thread().native_id, flush=True) + assert threading.current_thread().native_id == threading.get_native_id() + else: + try: + assert parent_thread_native_id == threading.current_thread().native_id + assert parent_thread_native_id == threading.get_native_id() + finally: + support.wait_process(childpid, exitcode=0) + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(rc, 0) + self.assertEqual(err, b"") + native_ids = out.strip().splitlines() + self.assertEqual(len(native_ids), 2) + self.assertNotEqual(native_ids[0], native_ids[1]) + + @cpython_only + def test_finalize_daemon_thread_hang(self): + if support.check_sanitizer(thread=True, memory=True): + # the thread running `time.sleep(100)` below will still be alive + # at process exit + self.skipTest( + "https://github.com/python/cpython/issues/124878 - Known" + " race condition that TSAN identifies.") + # gh-87135: tests that daemon threads hang during finalization + script = textwrap.dedent(''' + import os + import sys + import threading + import time + import _testcapi + + lock = threading.Lock() + lock.acquire() + thread_started_event = threading.Event() + def thread_func(): + try: + thread_started_event.set() + _testcapi.finalize_thread_hang(lock.acquire) + finally: + # Control must not reach here. + os._exit(2) + + t = threading.Thread(target=thread_func) + t.daemon = True + t.start() + thread_started_event.wait() + # Sleep to ensure daemon thread is blocked on `lock.acquire` + # + # Note: This test is designed so that in the unlikely case that + # `0.1` seconds is not sufficient time for the thread to become + # blocked on `lock.acquire`, the test will still pass, it just + # won't be properly testing the thread behavior during + # finalization. + time.sleep(0.1) + + def run_during_finalization(): + # Wake up daemon thread + lock.release() + # Sleep to give the daemon thread time to crash if it is going + # to. + # + # Note: If due to an exceptionally slow execution this delay is + # insufficient, the test will still pass but will simply be + # ineffective as a test. + time.sleep(0.1) + # If control reaches here, the test succeeded. + os._exit(0) + + # Replace sys.stderr.flush as a way to run code during finalization + orig_flush = sys.stderr.flush + def do_flush(*args, **kwargs): + orig_flush(*args, **kwargs) + if not sys.is_finalizing: + return + sys.stderr.flush = orig_flush + run_during_finalization() + + sys.stderr.flush = do_flush + + # If the follow exit code is retained, `run_during_finalization` + # did not run. + sys.exit(1) + ''') + assert_python_ok("-c", script) + + @unittest.skip('TODO: RUSTPYTHON; Thread._tstate_lock not implemented') + def test_tstate_lock(self): + return super().test_tstate_lock() + class ThreadJoinOnShutdown(BaseTestCase): @@ -992,8 +1338,6 @@ def joiningfunc(mainthread): data = out.decode().replace('\r', '') self.assertEqual(data, "end of main\nend of thread\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_1_join_on_shutdown(self): # The usual case: on exit, wait for a non-daemon thread script = """if 1: @@ -1006,10 +1350,7 @@ def test_1_join_on_shutdown(self): """ self._run_and_join(script) - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - # TODO: RUSTPYTHON need to fix test_1_join_on_shutdown then this might work - @unittest.expectedFailure + @skip_unless_reliable_fork def test_2_join_in_forked_process(self): # Like the test above, but from a forked interpreter script = """if 1: @@ -1029,9 +1370,8 @@ def test_2_join_in_forked_process(self): """ self._run_and_join(script) - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') + @skip_unless_reliable_fork def test_3_join_in_forked_from_thread(self): # Like the test above, but fork() was called from a worker thread # In the forked process, the main Thread object must be marked as stopped. @@ -1060,10 +1400,16 @@ def worker(): self._run_and_join(script) @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_daemon_threads(self): + @support.bigmemtest(size=40, memuse=70*2**20, dry_run=False) + def test_4_daemon_threads(self, size): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. + if support.check_sanitizer(thread=True): + # some of the threads running `random_io` below will still be alive + # at process exit + self.skipTest("TSAN would report thread leak") + script = """if True: import os import random @@ -1075,8 +1421,9 @@ def test_4_daemon_threads(self): def random_io(): '''Loop for a while sleeping random tiny amounts and doing some I/O.''' + import test.test_threading as mod while True: - with open(os.__file__, 'rb') as in_f: + with open(mod.__file__, 'rb') as in_f: stuff = in_f.read(200) with open(os.devnull, 'wb') as null_f: null_f.write(stuff) @@ -1100,8 +1447,33 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_thread_from_thread(self): + script = """if True: + import threading + import time + + def thread2(): + time.sleep(0.05) + print("OK") + + def thread1(): + time.sleep(0.05) + t2 = threading.Thread(target=thread2) + t2.start() + + t = threading.Thread(target=thread1) + t.start() + # do not join() -- the interpreter waits for non-daemon threads to + # finish. + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"OK") + self.assertEqual(rc, 0) + + # TODO: RUSTPYTHON - parking_lot mutex not fork-safe, child may SIGSEGV + @unittest.skip("TODO: RUSTPYTHON - flaky, parking_lot mutex not fork-safe") + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with # the ad-hoc TLS implementation. @@ -1114,18 +1486,20 @@ def do_fork_and_wait(): else: os._exit(50) - # start a bunch of threads that will fork() child processes - threads = [] - for i in range(16): - t = threading.Thread(target=do_fork_and_wait) - threads.append(t) - t.start() + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() - for t in threads: - t.join() + for t in threads: + t.join() - @unittest.skipUnless(hasattr(sys, '_current_frames'), "TODO: RUSTPYTHON, needs sys._current_frames()") - @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @skip_unless_reliable_fork def test_clear_threads_states_after_fork(self): # Issue #17094: check that threads states are cleared after fork() @@ -1136,18 +1510,22 @@ def test_clear_threads_states_after_fork(self): threads.append(t) t.start() - pid = os.fork() - if pid == 0: - # check that threads states have been cleared - if len(sys._current_frames()) == 1: - os._exit(51) - else: - os._exit(52) - else: - support.wait_process(pid, exitcode=51) - - for t in threads: - t.join() + try: + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(51) + else: + os._exit(52) + else: + support.wait_process(pid, exitcode=51) + finally: + for t in threads: + t.join() class SubinterpThreadingTests(BaseTestCase): @@ -1159,8 +1537,7 @@ def pipe(self): os.set_blocking(r, False) return (r, w) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_threads_join(self): # Non-daemon threads should be joined at subinterpreter shutdown # (issue #18808) @@ -1189,8 +1566,7 @@ def f(): # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_threads_join_2(self): # Same as above, but a delay gets introduced after the thread's # Python code returned but before the thread state is deleted. @@ -1228,8 +1604,47 @@ def f(): # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") + @requires_subinterpreters + def test_threads_join_with_no_main(self): + r_interp, w_interp = self.pipe() + + INTERP = b'I' + FINI = b'F' + DONE = b'D' + + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + os.write({w_interp}, {FINI!r}) + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {DONE!r}) + t = threading.Thread(target=task) + t.start() + + os.write({w_interp}, {INTERP!r}) + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), INTERP) + self.assertEqual(os.read(r_interp, 1), FINI) + self.assertEqual(os.read(r_interp, 1), DONE) + @cpython_only def test_daemon_threads_fatal_error(self): + import_module("_testcapi") subinterp_code = f"""if 1: import os import threading @@ -1251,6 +1666,67 @@ def f(): self.assertIn("Fatal Python error: Py_EndInterpreter: " "not the last thread", err.decode()) + def _check_allowed(self, before_start='', *, + allowed=True, + daemon_allowed=True, + daemon=False, + ): + import_module("_testinternalcapi") + subinterp_code = textwrap.dedent(f""" + import test.support + import threading + def func(): + print('this should not have run!') + t = threading.Thread(target=func, daemon={daemon}) + {before_start} + t.start() + """) + check_multi_interp_extensions = bool(support.Py_GIL_DISABLED) + script = textwrap.dedent(f""" + import test.support + test.support.run_in_subinterp_with_config( + {subinterp_code!r}, + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads={allowed}, + allow_daemon_threads={daemon_allowed}, + check_multi_interp_extensions={check_multi_interp_extensions}, + own_gil=False, + ) + """) + with test.support.SuppressCrashReport(): + _, _, err = assert_python_ok("-c", script) + return err.decode() + + @cpython_only + def test_threads_not_allowed(self): + err = self._check_allowed( + allowed=False, + daemon_allowed=False, + daemon=False, + ) + self.assertIn('RuntimeError', err) + + @cpython_only + def test_daemon_threads_not_allowed(self): + with self.subTest('via Thread()'): + err = self._check_allowed( + allowed=True, + daemon_allowed=False, + daemon=True, + ) + self.assertIn('RuntimeError', err) + + with self.subTest('via Thread.daemon setter'): + err = self._check_allowed( + 't.daemon = True', + allowed=True, + daemon_allowed=False, + daemon=False, + ) + self.assertIn('RuntimeError', err) + class ThreadingExceptionTests(BaseTestCase): # A RuntimeError should be raised if Thread.start() is called @@ -1279,7 +1755,8 @@ def test_releasing_unacquired_lock(self): lock = threading.Lock() self.assertRaises(RuntimeError, lock.release) - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') + @requires_subprocess() def test_recursion_limit(self): # Issue 9670 # test that excessive recursion within a non-main thread causes @@ -1288,13 +1765,6 @@ def test_recursion_limit(self): # for threads script = """if True: import threading - # TODO: RUSTPYTHON - # Following lines set the recursion limit to previous default of 512 - # for the execution of this process. Without this, the test runners - # on Github fail. Ideally, at a future point this should be removed. - import os, sys - if os.getenv("CI"): - sys.setrecursionlimit(512) def recurse(): return recurse() @@ -1399,6 +1869,37 @@ def run(): self.assertEqual(out, b'') self.assertNotIn("Unhandled exception", err.decode()) + def test_print_exception_gh_102056(self): + # This used to crash. See gh-102056. + script = r"""if True: + import time + import threading + import _thread + + def f(): + try: + f() + except RecursionError: + f() + + def g(): + try: + raise ValueError() + except* ValueError: + f() + + def h(): + time.sleep(1) + _thread.interrupt_main() + + t = threading.Thread(target=h) + t.start() + g() + t.join() + """ + + assert_python_failure("-c", script) + def test_bare_raise_in_brand_new_thread(self): def bare_raise(): raise @@ -1436,6 +1937,23 @@ def modify_file(): t.start() t.join() + def test_dummy_thread_on_interpreter_shutdown(self): + # GH-130522: When `threading` held a reference to itself and then a + # _DummyThread() object was created, destruction of the dummy thread + # would emit an unraisable exception at shutdown, due to a lock being + # destroyed. + code = """if True: + import sys + import threading + + threading.x = sys.modules[__name__] + x = threading._DummyThread() + """ + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(out, b"") + self.assertEqual(err, b"") + class ThreadRunFail(threading.Thread): def run(self): @@ -1447,6 +1965,7 @@ def setUp(self): restore_default_excepthook(self) super().setUp() + @force_not_colorized def test_excepthook(self): with support.captured_output("stderr") as stderr: thread = ThreadRunFail(name="excepthook thread") @@ -1460,6 +1979,7 @@ def test_excepthook(self): self.assertIn('ValueError: run failed', stderr) @support.cpython_only + @force_not_colorized def test_excepthook_thread_None(self): # threading.excepthook called with thread=None: log the thread # identifier in this case. @@ -1595,32 +2115,47 @@ class PyRLockTests(lock_tests.RLockTests): class CRLockTests(lock_tests.RLockTests): locktype = staticmethod(threading._CRLock) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON, flaky test") - def test_different_thread(self): - super().test_different_thread() + def test_signature(self): # gh-102029 + with warnings.catch_warnings(record=True) as warnings_log: + threading.RLock() + self.assertEqual(warnings_log, []) + + arg_types = [ + ((1,), {}), + ((), {'a': 1}), + ((1, 2), {'a': 1}), + ] + for args, kwargs in arg_types: + with self.subTest(args=args, kwargs=kwargs): + with self.assertWarns(DeprecationWarning): + threading.RLock(*args, **kwargs) + + # Subtypes with custom `__init__` are allowed (but, not recommended): + class CustomRLock(self.locktype): + def __init__(self, a, *, b) -> None: + super().__init__() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_release_save_unacquired(self): - super().test_release_save_unacquired() + with warnings.catch_warnings(record=True) as warnings_log: + CustomRLock(1, b=2) + self.assertEqual(warnings_log, []) + + @unittest.skip('TODO: RUSTPYTHON; flaky test') + def test_different_thread(self): + return super().test_different_thread() class EventTests(lock_tests.EventTests): eventtype = staticmethod(threading.Event) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reset_internal_locks(): # TODO: RUSTPYTHON; remove this when done - super().test_reset_internal_locks() - class ConditionAsRLockTests(lock_tests.RLockTests): # Condition uses an RLock by default and exports its API. locktype = staticmethod(threading.Condition) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON, flaky test") + def test_recursion_count(self): + self.skipTest("Condition does not expose _recursion_count()") + + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_different_thread(self): - super().test_different_thread() + return super().test_different_thread() class ConditionTests(lock_tests.ConditionTests): condtype = staticmethod(threading.Condition) @@ -1636,8 +2171,6 @@ class BarrierTests(lock_tests.BarrierTests): class MiscTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test__all__(self): restore_default_excepthook(self) @@ -1671,7 +2204,8 @@ def check_interrupt_main_noerror(self, signum): # Restore original handler signal.signal(signum, handler) - @unittest.skip("TODO: RUSTPYTHON; flaky") + @unittest.skip('TODO: RUSTPYTHON; flaky') + @requires_gil_enabled("gh-118433: Flaky due to a longstanding bug") def test_interrupt_main_subthread(self): # Calling start_new_thread with a function that executes interrupt_main # should raise KeyboardInterrupt upon completion. @@ -1730,8 +2264,6 @@ def worker(started, cont, interrupted): class AtexitTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_atexit_output(self): rc, out, err = assert_python_ok("-c", """if True: import threading @@ -1745,8 +2277,6 @@ def run_last(): self.assertFalse(err) self.assertEqual(out.strip(), b'parrot') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_atexit_called_once(self): rc, out, err = assert_python_ok("-c", """if True: import threading @@ -1762,8 +2292,6 @@ def test_atexit_called_once(self): self.assertFalse(err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_atexit_after_shutdown(self): # The only way to do this is by registering an atexit within # an atexit, which is intended to raise an exception. diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index 3443e3875d0..99052de4c7f 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -3,8 +3,8 @@ from doctest import DocTestSuite from test import support from test.support import threading_helper +from test.support.import_helper import import_module import weakref -import gc # Modules under test import _thread @@ -12,6 +12,9 @@ import _threading_local +threading_helper.requires_working_threading(module=True) + + class Weak(object): pass @@ -23,7 +26,7 @@ def target(local, weaklist): class BaseLocalTest: - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_local_refs(self): self._local_refs(20) self._local_refs(50) @@ -182,8 +185,7 @@ class LocalSubclass(self._local): """To test that subclasses behave properly.""" self._test_dict_attribute(LocalSubclass) - # TODO: RUSTPYTHON, cycle detection/collection - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cycle detection/collection def test_cycle_collection(self): class X: pass @@ -197,35 +199,57 @@ class X: self.assertIsNone(wr()) + def test_threading_local_clear_race(self): + # See https://github.com/python/cpython/issues/100892 + + _testcapi = import_module('_testcapi') + _testcapi.call_in_temporary_c_thread(lambda: None, False) + + for _ in range(1000): + _ = threading.local() + + _testcapi.join_temporary_c_thread() + + @support.cpython_only + def test_error(self): + class Loop(self._local): + attr = 1 + + # Trick the "if name == '__dict__':" test of __setattr__() + # to always be true + class NameCompareTrue: + def __eq__(self, other): + return True + + loop = Loop() + with self.assertRaisesRegex(AttributeError, 'Loop.*read-only'): + loop.__setattr__(NameCompareTrue(), 2) + + class ThreadLocalTest(unittest.TestCase, BaseLocalTest): _local = _thread._local - # TODO: RUSTPYTHON, __new__ vs __init__ cooperation - @unittest.expectedFailure - def test_arguments(): - super().test_arguments() - + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by _local + def test_arguments(self): + return super().test_arguments() class PyThreadingLocalTest(unittest.TestCase, BaseLocalTest): _local = _threading_local.local -def test_main(): - suite = unittest.TestSuite() - suite.addTest(DocTestSuite('_threading_local')) - suite.addTest(unittest.makeSuite(ThreadLocalTest)) - suite.addTest(unittest.makeSuite(PyThreadingLocalTest)) +def load_tests(loader, tests, pattern): + tests.addTest(DocTestSuite('_threading_local')) local_orig = _threading_local.local def setUp(test): _threading_local.local = _thread._local def tearDown(test): _threading_local.local = local_orig - suite.addTest(DocTestSuite('_threading_local', - setUp=setUp, tearDown=tearDown) - ) + tests.addTests(DocTestSuite('_threading_local', + setUp=setUp, tearDown=tearDown) + ) + return tests - support.run_unittest(suite) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py new file mode 100644 index 00000000000..bf241ada90e --- /dev/null +++ b/Lib/test/test_threadsignals.py @@ -0,0 +1,237 @@ +"""PyUnit testing that threads honor our signal semantics""" + +import unittest +import signal +import os +import sys +from test.support import threading_helper +import _thread as thread +import time + +if (sys.platform[:3] == 'win'): + raise unittest.SkipTest("Can't test signal on %s" % sys.platform) + +process_pid = os.getpid() +signalled_all=thread.allocate_lock() + +USING_PTHREAD_COND = (sys.thread_info.name == 'pthread' + and sys.thread_info.lock == 'mutex+cond') + +def registerSignals(for_usr1, for_usr2, for_alrm): + usr1 = signal.signal(signal.SIGUSR1, for_usr1) + usr2 = signal.signal(signal.SIGUSR2, for_usr2) + alrm = signal.signal(signal.SIGALRM, for_alrm) + return usr1, usr2, alrm + + +# The signal handler. Just note that the signal occurred and +# from who. +def handle_signals(sig,frame): + signal_blackboard[sig]['tripped'] += 1 + signal_blackboard[sig]['tripped_by'] = thread.get_ident() + +# a function that will be spawned as a separate thread. +def send_signals(): + # We use `raise_signal` rather than `kill` because: + # * It verifies that a signal delivered to a background thread still has + # its Python-level handler called on the main thread. + # * It ensures the signal is handled before the thread exits. + signal.raise_signal(signal.SIGUSR1) + signal.raise_signal(signal.SIGUSR2) + signalled_all.release() + + +@threading_helper.requires_working_threading() +class ThreadSignals(unittest.TestCase): + + def test_signals(self): + with threading_helper.wait_threads_exit(): + # Test signal handling semantics of threads. + # We spawn a thread, have the thread send itself two signals, and + # wait for it to finish. Check that we got both signals + # and that they were run by the main thread. + signalled_all.acquire() + self.spawnSignallingThread() + signalled_all.acquire() + + self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped'], 1) + self.assertEqual( signal_blackboard[signal.SIGUSR1]['tripped_by'], + thread.get_ident()) + self.assertEqual( signal_blackboard[signal.SIGUSR2]['tripped'], 1) + self.assertEqual( signal_blackboard[signal.SIGUSR2]['tripped_by'], + thread.get_ident()) + signalled_all.release() + + def spawnSignallingThread(self): + thread.start_new_thread(send_signals, ()) + + def alarm_interrupt(self, sig, frame): + raise KeyboardInterrupt + + @unittest.skipIf(USING_PTHREAD_COND, + 'POSIX condition variables cannot be interrupted') + @unittest.skipIf(sys.platform.startswith('linux') and + not sys.thread_info.version, + 'Issue 34004: musl does not allow interruption of locks ' + 'by signals.') + # Issue #20564: sem_timedwait() cannot be interrupted on OpenBSD + @unittest.skipIf(sys.platform.startswith('openbsd'), + 'lock cannot be interrupted on OpenBSD') + def test_lock_acquire_interruption(self): + # Mimic receiving a SIGINT (KeyboardInterrupt) with SIGALRM while stuck + # in a deadlock. + # XXX this test can fail when the legacy (non-semaphore) implementation + # of locks is used in thread_pthread.h, see issue #11223. + oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt) + try: + lock = thread.allocate_lock() + lock.acquire() + signal.alarm(1) + t1 = time.monotonic() + self.assertRaises(KeyboardInterrupt, lock.acquire, timeout=5) + dt = time.monotonic() - t1 + # Checking that KeyboardInterrupt was raised is not sufficient. + # We want to assert that lock.acquire() was interrupted because + # of the signal, not that the signal handler was called immediately + # after timeout return of lock.acquire() (which can fool assertRaises). + self.assertLess(dt, 3.0) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, oldalrm) + + @unittest.skipIf(USING_PTHREAD_COND, + 'POSIX condition variables cannot be interrupted') + @unittest.skipIf(sys.platform.startswith('linux') and + not sys.thread_info.version, + 'Issue 34004: musl does not allow interruption of locks ' + 'by signals.') + # Issue #20564: sem_timedwait() cannot be interrupted on OpenBSD + @unittest.skipIf(sys.platform.startswith('openbsd'), + 'lock cannot be interrupted on OpenBSD') + def test_rlock_acquire_interruption(self): + # Mimic receiving a SIGINT (KeyboardInterrupt) with SIGALRM while stuck + # in a deadlock. + # XXX this test can fail when the legacy (non-semaphore) implementation + # of locks is used in thread_pthread.h, see issue #11223. + oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt) + try: + rlock = thread.RLock() + # For reentrant locks, the initial acquisition must be in another + # thread. + def other_thread(): + rlock.acquire() + + with threading_helper.wait_threads_exit(): + thread.start_new_thread(other_thread, ()) + # Wait until we can't acquire it without blocking... + while rlock.acquire(blocking=False): + rlock.release() + time.sleep(0.01) + signal.alarm(1) + t1 = time.monotonic() + self.assertRaises(KeyboardInterrupt, rlock.acquire, timeout=5) + dt = time.monotonic() - t1 + # See rationale above in test_lock_acquire_interruption + self.assertLess(dt, 3.0) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, oldalrm) + + def acquire_retries_on_intr(self, lock): + self.sig_recvd = False + def my_handler(signal, frame): + self.sig_recvd = True + + old_handler = signal.signal(signal.SIGUSR1, my_handler) + try: + def other_thread(): + # Acquire the lock in a non-main thread, so this test works for + # RLocks. + lock.acquire() + # Wait until the main thread is blocked in the lock acquire, and + # then wake it up with this. + time.sleep(0.5) + os.kill(process_pid, signal.SIGUSR1) + # Let the main thread take the interrupt, handle it, and retry + # the lock acquisition. Then we'll let it run. + time.sleep(0.5) + lock.release() + + with threading_helper.wait_threads_exit(): + thread.start_new_thread(other_thread, ()) + # Wait until we can't acquire it without blocking... + while lock.acquire(blocking=False): + lock.release() + time.sleep(0.01) + result = lock.acquire() # Block while we receive a signal. + self.assertTrue(self.sig_recvd) + self.assertTrue(result) + finally: + signal.signal(signal.SIGUSR1, old_handler) + + def test_lock_acquire_retries_on_intr(self): + self.acquire_retries_on_intr(thread.allocate_lock()) + + def test_rlock_acquire_retries_on_intr(self): + self.acquire_retries_on_intr(thread.RLock()) + + def test_interrupted_timed_acquire(self): + # Test to make sure we recompute lock acquisition timeouts when we + # receive a signal. Check this by repeatedly interrupting a lock + # acquire in the main thread, and make sure that the lock acquire times + # out after the right amount of time. + # NOTE: this test only behaves as expected if C signals get delivered + # to the main thread. Otherwise lock.acquire() itself doesn't get + # interrupted and the test trivially succeeds. + self.start = None + self.end = None + self.sigs_recvd = 0 + done = thread.allocate_lock() + done.acquire() + lock = thread.allocate_lock() + lock.acquire() + def my_handler(signum, frame): + self.sigs_recvd += 1 + old_handler = signal.signal(signal.SIGUSR1, my_handler) + try: + def timed_acquire(): + self.start = time.monotonic() + lock.acquire(timeout=0.5) + self.end = time.monotonic() + def send_signals(): + for _ in range(40): + time.sleep(0.02) + os.kill(process_pid, signal.SIGUSR1) + done.release() + + with threading_helper.wait_threads_exit(): + # Send the signals from the non-main thread, since the main thread + # is the only one that can process signals. + thread.start_new_thread(send_signals, ()) + timed_acquire() + # Wait for thread to finish + done.acquire() + # This allows for some timing and scheduling imprecision + self.assertLess(self.end - self.start, 2.0) + self.assertGreater(self.end - self.start, 0.3) + # If the signal is received several times before PyErr_CheckSignals() + # is called, the handler will get called less than 40 times. Just + # check it's been called at least once. + self.assertGreater(self.sigs_recvd, 0) + finally: + signal.signal(signal.SIGUSR1, old_handler) + + +def setUpModule(): + global signal_blackboard + + signal_blackboard = { signal.SIGUSR1 : {'tripped': 0, 'tripped_by': 0 }, + signal.SIGUSR2 : {'tripped': 0, 'tripped_by': 0 }, + signal.SIGALRM : {'tripped': 0, 'tripped_by': 0 } } + + oldsigs = registerSignals(handle_signals, handle_signals, handle_signals) + unittest.addModuleCleanup(registerSignals, *oldsigs) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 4e618d1d90f..f66a37edb4c 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -2,7 +2,6 @@ from test.support import warnings_helper import decimal import enum -import locale import math import platform import sys @@ -14,8 +13,12 @@ import _testcapi except ImportError: _testcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None -from test.support import skip_if_buggy_ucrt_strfptime +from test.support import skip_if_buggy_ucrt_strfptime, SuppressCrashReport # Max year is only limited by the size of C int. SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 @@ -38,6 +41,10 @@ class _PyTime(enum.IntEnum): # Round away from zero ROUND_UP = 3 +# _PyTime_t is int64_t +PyTime_MIN = -2 ** 63 +PyTime_MAX = 2 ** 63 - 1 + # Rounding modes supported by PyTime ROUNDING_MODES = ( # (PyTime rounding method, decimal rounding method) @@ -53,8 +60,6 @@ class TimeTestCase(unittest.TestCase): def setUp(self): self.t = time.time() - # TODO: RUSTPYTHON, AttributeError: module 'time' has no attribute 'altzone' - @unittest.expectedFailure def test_data_attributes(self): time.altzone time.daylight @@ -111,6 +116,7 @@ def test_clock_monotonic(self): 'need time.pthread_getcpuclockid()') @unittest.skipUnless(hasattr(time, 'clock_gettime'), 'need time.clock_gettime()') + @unittest.skipIf(support.is_emscripten, "Fails to find clock") def test_pthread_getcpuclockid(self): clk_id = time.pthread_getcpuclockid(threading.get_ident()) self.assertTrue(type(clk_id) is int) @@ -152,13 +158,32 @@ def test_conversions(self): self.assertEqual(int(time.mktime(time.localtime(self.t))), int(self.t)) - def test_sleep(self): + def test_sleep_exceptions(self): + self.assertRaises(TypeError, time.sleep, []) + self.assertRaises(TypeError, time.sleep, "a") + self.assertRaises(TypeError, time.sleep, complex(0, 0)) + self.assertRaises(ValueError, time.sleep, -2) self.assertRaises(ValueError, time.sleep, -1) - time.sleep(1.2) + self.assertRaises(ValueError, time.sleep, -0.1) + + # Improved exception #81267 + with self.assertRaises(TypeError) as errmsg: + time.sleep([]) + self.assertIn("integer or float", str(errmsg.exception)) + + def test_sleep(self): + for value in [-0.0, 0, 0.0, 1e-100, 1e-9, 1e-6, 1, 1.2]: + with self.subTest(value=value): + time.sleep(value) + + def test_epoch(self): + # bpo-43869: Make sure that Python use the same Epoch on all platforms: + # January 1, 1970, 00:00:00 (UTC). + epoch = time.gmtime(0) + # Only test the date and time, ignore other gmtime() members + self.assertEqual(tuple(epoch)[:6], (1970, 1, 1, 0, 0, 0), epoch) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strftime(self): tt = time.gmtime(self.t) for directive in ('a', 'A', 'b', 'B', 'c', 'd', 'H', 'I', @@ -171,8 +196,44 @@ def test_strftime(self): self.fail('conversion specifier: %r failed.' % format) self.assertRaises(TypeError, time.strftime, b'%S', tt) - # embedded null character - self.assertRaises(ValueError, time.strftime, '%S\0', tt) + + def test_strftime_invalid_format(self): + tt = time.gmtime(self.t) + with SuppressCrashReport(): + for i in range(1, 128): + format = ' %' + chr(i) + with self.subTest(format=format): + try: + time.strftime(format, tt) + except ValueError as exc: + self.assertEqual(str(exc), 'Invalid format string') + + def test_strftime_special(self): + tt = time.gmtime(self.t) + s1 = time.strftime('%c', tt) + s2 = time.strftime('%B', tt) + # gh-52551, gh-78662: Unicode strings should pass through strftime, + # independently from locale. + self.assertEqual(time.strftime('\U0001f40d', tt), '\U0001f40d') + self.assertEqual(time.strftime('\U0001f4bb%c\U0001f40d%B', tt), f'\U0001f4bb{s1}\U0001f40d{s2}') + self.assertEqual(time.strftime('%c\U0001f4bb%B\U0001f40d', tt), f'{s1}\U0001f4bb{s2}\U0001f40d') + # Lone surrogates should pass through. + self.assertEqual(time.strftime('\ud83d', tt), '\ud83d') + self.assertEqual(time.strftime('\udc0d', tt), '\udc0d') + self.assertEqual(time.strftime('\ud83d%c\udc0d%B', tt), f'\ud83d{s1}\udc0d{s2}') + self.assertEqual(time.strftime('%c\ud83d%B\udc0d', tt), f'{s1}\ud83d{s2}\udc0d') + self.assertEqual(time.strftime('%c\udc0d%B\ud83d', tt), f'{s1}\udc0d{s2}\ud83d') + # Surrogate pairs should not recombine. + self.assertEqual(time.strftime('\ud83d\udc0d', tt), '\ud83d\udc0d') + self.assertEqual(time.strftime('%c\ud83d\udc0d%B', tt), f'{s1}\ud83d\udc0d{s2}') + # Surrogate-escaped bytes should not recombine. + self.assertEqual(time.strftime('\udcf0\udc9f\udc90\udc8d', tt), '\udcf0\udc9f\udc90\udc8d') + self.assertEqual(time.strftime('%c\udcf0\udc9f\udc90\udc8d%B', tt), f'{s1}\udcf0\udc9f\udc90\udc8d{s2}') + # gh-124531: The null character should not terminate the format string. + self.assertEqual(time.strftime('\0', tt), '\0') + self.assertEqual(time.strftime('\0'*1000, tt), '\0'*1000) + self.assertEqual(time.strftime('\0%c\0%B', tt), f'\0{s1}\0{s2}') + self.assertEqual(time.strftime('%c\0%B\0', tt), f'{s1}\0{s2}\0') def _bounds_checking(self, func): # Make sure that strftime() checks the bounds of the various parts @@ -231,12 +292,9 @@ def _bounds_checking(self, func): self.assertRaises(ValueError, func, (1900, 1, 1, 0, 0, 0, 0, 367, -1)) - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_strftime_bounding_check(self): self._bounds_checking(lambda tup: time.strftime('', tup)) - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'a Display implementation returned an error unexpectedly: Error'") def test_strftime_format_check(self): # Test that strftime does not crash on invalid format strings # that may trigger a buffer overread. When not triggered, @@ -250,8 +308,6 @@ def test_strftime_format_check(self): except ValueError: pass - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_default_values_for_zero(self): # Make sure that using all zeros uses the proper default # values. No test for daylight savings since strftime() does @@ -262,8 +318,6 @@ def test_default_values_for_zero(self): result = time.strftime("%Y %m %d %H %M %S %w %j", (2000,)+(0,)*8) self.assertEqual(expected, result) - # TODO: RUSTPYTHON - @unittest.expectedFailure @skip_if_buggy_ucrt_strfptime def test_strptime(self): # Should be able to go round-trip from strftime to strptime without @@ -273,6 +327,8 @@ def test_strptime(self): 'j', 'm', 'M', 'p', 'S', 'U', 'w', 'W', 'x', 'X', 'y', 'Y', 'Z', '%'): format = '%' + directive + if directive == 'd': + format += ',%Y' # Avoid GH-70647. strf_output = time.strftime(format, tt) try: time.strptime(strf_output, format) @@ -285,20 +341,22 @@ def test_strptime_bytes(self): self.assertRaises(TypeError, time.strptime, b'2009', "%Y") self.assertRaises(TypeError, time.strptime, '2009', b'%Y') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) with self.assertRaises(ValueError) as e: time.strptime('', '%D') - self.assertIs(e.exception.__suppress_context__, True) - # additional check for IndexError branch (issue #19545) + self.assertTrue(e.exception.__suppress_context__) + # additional check for stray % branch with self.assertRaises(ValueError) as e: - time.strptime('19', '%Y %') - self.assertIs(e.exception.__suppress_context__, True) + time.strptime('%', '%') + self.assertTrue(e.exception.__suppress_context__) + + def test_strptime_leap_year(self): + # GH-70647: warns if parsing a format with a day and no year. + with self.assertWarnsRegex(DeprecationWarning, + r'.*day of month without a year.*'): + time.strptime('02-07 18:28', '%m-%d %H:%M') - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_asctime(self): time.asctime(time.gmtime(self.t)) @@ -314,13 +372,9 @@ def test_asctime(self): self.assertRaises(TypeError, time.asctime, ()) self.assertRaises(TypeError, time.asctime, (0,) * 10) - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_asctime_bounding_check(self): self._bounds_checking(time.asctime) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ctime(self): t = time.mktime((1973, 9, 16, 1, 3, 52, 0, 0, -1)) self.assertEqual(time.ctime(t), 'Sun Sep 16 01:03:52 1973') @@ -420,8 +474,6 @@ def test_insane_timestamps(self): for unreasonable in -1e200, 1e200: self.assertRaises(OverflowError, func, unreasonable) - # TODO: RUSTPYTHON, TypeError: unexpected type NoneType - @unittest.expectedFailure def test_ctime_without_arg(self): # Not sure how to check the values, since the clock could tick # at any time. Make sure these are at least accepted and @@ -429,8 +481,6 @@ def test_ctime_without_arg(self): time.ctime() time.ctime(None) - # TODO: RUSTPYTHON, TypeError: unexpected type NoneType - @unittest.expectedFailure def test_gmtime_without_arg(self): gt0 = time.gmtime() gt1 = time.gmtime(None) @@ -438,8 +488,6 @@ def test_gmtime_without_arg(self): t1 = time.mktime(gt1) self.assertAlmostEqual(t1, t0, delta=0.2) - # TODO: RUSTPYTHON, TypeError: unexpected type NoneType - @unittest.expectedFailure def test_localtime_without_arg(self): lt0 = time.localtime() lt1 = time.localtime(None) @@ -459,7 +507,6 @@ def test_mktime(self): # Issue #13309: passing extreme values to mktime() or localtime() # borks the glibc's internal timezone data. - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'a Display implementation returned an error unexpectedly: Error'") @unittest.skipUnless(platform.libc_ver()[0] != 'glibc', "disabled because of a bug in glibc. Issue #13309") def test_mktime_error(self): @@ -475,8 +522,6 @@ def test_mktime_error(self): pass self.assertEqual(time.strftime('%Z', tt), tzname) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == "win32", "Implement get_clock_info for Windows.") def test_monotonic(self): # monotonic() should not go backward times = [time.monotonic() for n in range(100)] @@ -503,6 +548,12 @@ def test_monotonic(self): def test_perf_counter(self): time.perf_counter() + @unittest.skipIf( + support.is_wasi, "process_time not available on WASI" + ) + @unittest.skipIf( + support.is_emscripten, "process_time present but doesn't exclude sleep" + ) def test_process_time(self): # process_time() should not include time spend during a sleep start = time.process_time() @@ -518,7 +569,7 @@ def test_process_time(self): def test_thread_time(self): if not hasattr(time, 'thread_time'): - if sys.platform.startswith(('linux', 'win')): + if sys.platform.startswith(('linux', 'android', 'win')): self.fail("time.thread_time() should be available on %r" % (sys.platform,)) else: @@ -526,11 +577,10 @@ def test_thread_time(self): # thread_time() should not include time spend during a sleep start = time.thread_time() - time.sleep(0.100) + time.sleep(0.200) stop = time.thread_time() - # use 20 ms because thread_time() has usually a resolution of 15 ms - # on Windows - self.assertLess(stop - start, 0.020) + # gh-143528: use 100 ms to support slow CI + self.assertLess(stop - start, 0.100) info = time.get_clock_info('thread_time') self.assertTrue(info.monotonic) @@ -578,8 +628,9 @@ def test_get_clock_info(self): 'perf_counter', 'process_time', 'time', - 'thread_time', ] + if hasattr(time, 'thread_time'): + clocks.append('thread_time') for name in clocks: with self.subTest(name=name): @@ -598,17 +649,8 @@ def test_get_clock_info(self): class TestLocale(unittest.TestCase): - def setUp(self): - self.oldloc = locale.setlocale(locale.LC_ALL) - - def tearDown(self): - locale.setlocale(locale.LC_ALL, self.oldloc) - + @support.run_with_locale('LC_ALL', 'fr_FR', '') def test_bug_3061(self): - try: - tmp = locale.setlocale(locale.LC_ALL, "fr_FR") - except locale.Error: - self.skipTest('could not set locale.LC_ALL to fr_FR') # This should not cause an exception time.strftime("%B", (2009,2,1,0,0,0,0,0,0)) @@ -619,14 +661,11 @@ class _TestAsctimeYear: def yearstr(self, y): return time.asctime((y,) + (0,) * 8).split()[-1] - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_large_year(self): # Check that it doesn't crash for year > 9999 self.assertEqual(self.yearstr(12345), '12345') self.assertEqual(self.yearstr(123456789), '123456789') -@unittest.skip("TODO: RUSTPYTHON, ValueError: invalid struct_time parameter") class _TestStrftimeYear: # Issue 13305: For years < 1000, the value is not always @@ -634,15 +673,17 @@ class _TestStrftimeYear: # assumes year >= 1900, so it does not specify the number # of digits. - # TODO: RUSTPYTHON - # if time.strftime('%Y', (1,) + (0,) * 8) == '0001': - # _format = '%04d' - # else: - # _format = '%d' + if time.strftime('%Y', (1,) + (0,) * 8) == '0001': + _format = '%04d' + else: + _format = '%d' def yearstr(self, y): return time.strftime('%Y', (y,) + (0,) * 8) + @unittest.skipUnless( + support.has_strftime_extensions, "requires strftime extension" + ) def test_4dyear(self): # Check that we can return the zero padded value. if self._format == '%04d': @@ -653,8 +694,7 @@ def year4d(y): self.test_year('%04d', func=year4d) def skip_if_not_supported(y): - msg = "strftime() is limited to [1; 9999] with Visual Studio" - # Check that it doesn't crash for year > 9999 + msg = f"strftime() does not support year {y} on this platform" try: time.strftime('%Y', (y,) + (0,) * 8) except ValueError: @@ -677,8 +717,6 @@ def test_negative(self): class _Test4dYear: _format = '%d' - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_year(self, fmt=None, func=None): fmt = fmt or self._format func = func or self.yearstr @@ -695,8 +733,6 @@ def test_large_year(self): self.assertEqual(self.yearstr(TIME_MAXYEAR).lstrip('+'), str(TIME_MAXYEAR)) self.assertRaises(OverflowError, self.yearstr, TIME_MAXYEAR + 1) - # TODO: RUSTPYTHON, ValueError: invalid struct_time parameter - @unittest.expectedFailure def test_negative(self): self.assertEqual(self.yearstr(-1), self._format % -1) self.assertEqual(self.yearstr(-1234), '-1234') @@ -713,29 +749,28 @@ def test_negative(self): class TestAsctime4dyear(_TestAsctimeYear, _Test4dYear, unittest.TestCase): pass -# class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): -# pass +class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): + pass class TestPytime(unittest.TestCase): @skip_if_buggy_ucrt_strfptime - @unittest.skip("TODO: RUSTPYTHON, AttributeError: module 'time' has no attribute '_STRUCT_TM_ITEMS'") - # @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. lt = time.localtime() - self.assertTrue(hasattr(lt, "tm_gmtoff")) - self.assertTrue(hasattr(lt, "tm_zone")) + self.assertHasAttr(lt, "tm_gmtoff") + self.assertHasAttr(lt, "tm_zone") # See if the offset and zone are similar to the module # attributes. if lt.tm_gmtoff is None: - self.assertTrue(not hasattr(time, "timezone")) + self.assertNotHasAttr(time, "timezone") else: self.assertEqual(lt.tm_gmtoff, -[time.timezone, time.altzone][lt.tm_isdst]) if lt.tm_zone is None: - self.assertTrue(not hasattr(time, "tzname")) + self.assertNotHasAttr(time, "tzname") else: self.assertEqual(lt.tm_zone, time.tzname[lt.tm_isdst]) @@ -754,17 +789,15 @@ def test_localtime_timezone(self): self.assertEqual(new_lt9, lt) self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) self.assertEqual(new_lt9.tm_zone, lt.tm_zone) - - @unittest.skip("TODO: RUSTPYTHON, AttributeError: module 'time' has no attribute '_STRUCT_TM_ITEMS'") - # @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_strptime_timezone(self): t = time.strptime("UTC", "%Z") self.assertEqual(t.tm_zone, 'UTC') t = time.strptime("+0500", "%z") self.assertEqual(t.tm_gmtoff, 5 * 3600) - - @unittest.skip("TODO: RUSTPYTHON, AttributeError: module 'time' has no attribute '_STRUCT_TM_ITEMS'") - # @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") + + @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_short_times(self): import pickle @@ -776,7 +809,8 @@ def test_short_times(self): self.assertIs(lt.tm_zone, None) -@unittest.skipIf(_testcapi is None, 'need the _testcapi module') +@unittest.skipIf(_testcapi is None, 'need the _testinternalcapi module') +@unittest.skipIf(_testinternalcapi is None, 'need the _testinternalcapi module') class CPyTimeTestCase: """ Base class to test the C _PyTime_t API. @@ -784,7 +818,7 @@ class CPyTimeTestCase: OVERFLOW_SECONDS = None def setUp(self): - from _testcapi import SIZEOF_TIME_T + from _testinternalcapi import SIZEOF_TIME_T bits = SIZEOF_TIME_T * 8 - 1 self.time_t_min = -2 ** bits self.time_t_max = 2 ** bits - 1 @@ -863,7 +897,7 @@ def convert_values(ns_timestamps): # test rounding ns_timestamps = self._rounding_values(use_float) valid_values = convert_values(ns_timestamps) - for time_rnd, decimal_rnd in ROUNDING_MODES : + for time_rnd, decimal_rnd in ROUNDING_MODES: with decimal.localcontext() as context: context.rounding = decimal_rnd @@ -912,36 +946,36 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): OVERFLOW_SECONDS = math.ceil((2**63 + 1) / SEC_TO_NS) def test_FromSeconds(self): - from _testcapi import PyTime_FromSeconds + from _testinternalcapi import _PyTime_FromSeconds - # PyTime_FromSeconds() expects a C int, reject values out of range + # _PyTime_FromSeconds() expects a C int, reject values out of range def c_int_filter(secs): return (_testcapi.INT_MIN <= secs <= _testcapi.INT_MAX) - self.check_int_rounding(lambda secs, rnd: PyTime_FromSeconds(secs), + self.check_int_rounding(lambda secs, rnd: _PyTime_FromSeconds(secs), lambda secs: secs * SEC_TO_NS, value_filter=c_int_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(TypeError): - PyTime_FromSeconds(float('nan')) + _PyTime_FromSeconds(float('nan')) def test_FromSecondsObject(self): - from _testcapi import PyTime_FromSecondsObject + from _testinternalcapi import _PyTime_FromSecondsObject self.check_int_rounding( - PyTime_FromSecondsObject, + _PyTime_FromSecondsObject, lambda secs: secs * SEC_TO_NS) self.check_float_rounding( - PyTime_FromSecondsObject, + _PyTime_FromSecondsObject, lambda ns: self.decimal_round(ns * SEC_TO_NS)) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - PyTime_FromSecondsObject(float('nan'), time_rnd) + _PyTime_FromSecondsObject(float('nan'), time_rnd) def test_AsSecondsDouble(self): from _testcapi import PyTime_AsSecondsDouble @@ -956,11 +990,6 @@ def float_converter(ns): float_converter, NS_TO_SEC) - # test nan - for time_rnd, _ in ROUNDING_MODES: - with self.assertRaises(TypeError): - PyTime_AsSecondsDouble(float('nan')) - def create_decimal_converter(self, denominator): denom = decimal.Decimal(denominator) @@ -971,7 +1000,7 @@ def converter(value): return converter def test_AsTimeval(self): - from _testcapi import PyTime_AsTimeval + from _testinternalcapi import _PyTime_AsTimeval us_converter = self.create_decimal_converter(US_TO_NS) @@ -988,35 +1017,78 @@ def seconds_filter(secs): else: seconds_filter = self.time_t_filter - self.check_int_rounding(PyTime_AsTimeval, + self.check_int_rounding(_PyTime_AsTimeval, timeval_converter, NS_TO_SEC, value_filter=seconds_filter) - @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), - 'need _testcapi.PyTime_AsTimespec') + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimespec'), + 'need _testinternalcapi._PyTime_AsTimespec') def test_AsTimespec(self): - from _testcapi import PyTime_AsTimespec + from _testinternalcapi import _PyTime_AsTimespec def timespec_converter(ns): return divmod(ns, SEC_TO_NS) - self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), + self.check_int_rounding(lambda ns, rnd: _PyTime_AsTimespec(ns), timespec_converter, NS_TO_SEC, value_filter=self.time_t_filter) + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimeval_clamp'), + 'need _testinternalcapi._PyTime_AsTimeval_clamp') + def test_AsTimeval_clamp(self): + from _testinternalcapi import _PyTime_AsTimeval_clamp + + if sys.platform == 'win32': + from _testcapi import LONG_MIN, LONG_MAX + tv_sec_max = LONG_MAX + tv_sec_min = LONG_MIN + else: + tv_sec_max = self.time_t_max + tv_sec_min = self.time_t_min + + for t in (PyTime_MIN, PyTime_MAX): + ts = _PyTime_AsTimeval_clamp(t, _PyTime.ROUND_CEILING) + with decimal.localcontext() as context: + context.rounding = decimal.ROUND_CEILING + us = self.decimal_round(decimal.Decimal(t) / US_TO_NS) + tv_sec, tv_usec = divmod(us, SEC_TO_US) + if tv_sec_max < tv_sec: + tv_sec = tv_sec_max + tv_usec = 0 + elif tv_sec < tv_sec_min: + tv_sec = tv_sec_min + tv_usec = 0 + self.assertEqual(ts, (tv_sec, tv_usec)) + + @unittest.skipUnless(hasattr(_testinternalcapi, '_PyTime_AsTimespec_clamp'), + 'need _testinternalcapi._PyTime_AsTimespec_clamp') + def test_AsTimespec_clamp(self): + from _testinternalcapi import _PyTime_AsTimespec_clamp + + for t in (PyTime_MIN, PyTime_MAX): + ts = _PyTime_AsTimespec_clamp(t) + tv_sec, tv_nsec = divmod(t, NS_TO_SEC) + if self.time_t_max < tv_sec: + tv_sec = self.time_t_max + tv_nsec = 0 + elif tv_sec < self.time_t_min: + tv_sec = self.time_t_min + tv_nsec = 0 + self.assertEqual(ts, (tv_sec, tv_nsec)) + def test_AsMilliseconds(self): - from _testcapi import PyTime_AsMilliseconds + from _testinternalcapi import _PyTime_AsMilliseconds - self.check_int_rounding(PyTime_AsMilliseconds, + self.check_int_rounding(_PyTime_AsMilliseconds, self.create_decimal_converter(MS_TO_NS), NS_TO_SEC) def test_AsMicroseconds(self): - from _testcapi import PyTime_AsMicroseconds + from _testinternalcapi import _PyTime_AsMicroseconds - self.check_int_rounding(PyTime_AsMicroseconds, + self.check_int_rounding(_PyTime_AsMicroseconds, self.create_decimal_converter(US_TO_NS), NS_TO_SEC) @@ -1030,13 +1102,13 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): OVERFLOW_SECONDS = 2 ** 64 def test_object_to_time_t(self): - from _testcapi import pytime_object_to_time_t + from _testinternalcapi import _PyTime_ObjectToTime_t - self.check_int_rounding(pytime_object_to_time_t, + self.check_int_rounding(_PyTime_ObjectToTime_t, lambda secs: secs, value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_time_t, + self.check_float_rounding(_PyTime_ObjectToTime_t, self.decimal_round, value_filter=self.time_t_filter) @@ -1056,36 +1128,36 @@ def converter(secs): return converter def test_object_to_timeval(self): - from _testcapi import pytime_object_to_timeval + from _testinternalcapi import _PyTime_ObjectToTimeval - self.check_int_rounding(pytime_object_to_timeval, + self.check_int_rounding(_PyTime_ObjectToTimeval, lambda secs: (secs, 0), value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_timeval, + self.check_float_rounding(_PyTime_ObjectToTimeval, self.create_converter(SEC_TO_US), value_filter=self.time_t_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - pytime_object_to_timeval(float('nan'), time_rnd) + _PyTime_ObjectToTimeval(float('nan'), time_rnd) def test_object_to_timespec(self): - from _testcapi import pytime_object_to_timespec + from _testinternalcapi import _PyTime_ObjectToTimespec - self.check_int_rounding(pytime_object_to_timespec, + self.check_int_rounding(_PyTime_ObjectToTimespec, lambda secs: (secs, 0), value_filter=self.time_t_filter) - self.check_float_rounding(pytime_object_to_timespec, + self.check_float_rounding(_PyTime_ObjectToTimespec, self.create_converter(SEC_TO_NS), value_filter=self.time_t_filter) # test nan for time_rnd, _ in ROUNDING_MODES: with self.assertRaises(ValueError): - pytime_object_to_timespec(float('nan'), time_rnd) + _PyTime_ObjectToTimespec(float('nan'), time_rnd) @unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") class TestTimeWeaklinking(unittest.TestCase): @@ -1111,11 +1183,11 @@ def test_clock_functions(self): if mac_ver >= (10, 12): for name in clock_names: - self.assertTrue(hasattr(time, name), f"time.{name} is not available") + self.assertHasAttr(time, name) else: for name in clock_names: - self.assertFalse(hasattr(time, name), f"time.{name} is available") + self.assertNotHasAttr(time, name) if __name__ == "__main__": diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index 17e880cb583..2aeebea9f93 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -91,8 +91,6 @@ def test_timer_invalid_setup(self): self.assertRaises(SyntaxError, timeit.Timer, setup='from timeit import *') self.assertRaises(SyntaxError, timeit.Timer, setup=' pass') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_timer_empty_stmt(self): timeit.Timer(stmt='') timeit.Timer(stmt=' \n\t\f') @@ -224,8 +222,8 @@ def test_repeat_function_zero_iters(self): def assert_exc_string(self, exc_string, expected_exc_name): exc_lines = exc_string.splitlines() self.assertGreater(len(exc_lines), 2) - self.assertTrue(exc_lines[0].startswith('Traceback')) - self.assertTrue(exc_lines[-1].startswith(expected_exc_name)) + self.assertStartsWith(exc_lines[0], 'Traceback') + self.assertStartsWith(exc_lines[-1], expected_exc_name) def test_print_exc(self): s = io.StringIO() @@ -299,9 +297,7 @@ def test_main_negative_reps(self): @unittest.skipIf(sys.flags.optimize >= 2, "need __doc__") def test_main_help(self): s = self.run_main(switches=['-h']) - # Note: It's not clear that the trailing space was intended as part of - # the help text, but since it's there, check for it. - self.assertEqual(s, timeit.__doc__ + ' ') + self.assertEqual(s, timeit.__doc__) def test_main_verbose(self): s = self.run_main(switches=['-v']) diff --git a/Lib/test/test_timeout.py b/Lib/test/test_timeout.py index f40c7ee48b0..70a0175d771 100644 --- a/Lib/test/test_timeout.py +++ b/Lib/test/test_timeout.py @@ -71,7 +71,6 @@ def testTypeCheck(self): self.assertRaises(TypeError, self.sock.settimeout, {}) self.assertRaises(TypeError, self.sock.settimeout, 0j) - @unittest.skip("TODO: RUSTPYTHON; crash") def testRangeCheck(self): # Test range checking by settimeout() self.assertRaises(ValueError, self.sock.settimeout, -1) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index e2d2f89454d..394a87c3601 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1,17 +1,22 @@ -from test import support -from test.support import os_helper -from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP, - STRING, ENDMARKER, ENCODING, tok_name, detect_encoding, - open as tokenize_open, Untokenizer, generate_tokens, - NEWLINE) -from io import BytesIO, StringIO +import contextlib +import itertools +import os +import re +import string +import tempfile +import token +import tokenize import unittest +from io import BytesIO, StringIO from textwrap import dedent from unittest import TestCase, mock -from test.test_grammar import (VALID_UNDERSCORE_LITERALS, - INVALID_UNDERSCORE_LITERALS) -import os -import token +from test import support +from test.support import os_helper +from test.support.script_helper import run_test_script, make_script, run_python_until_end +from test.support.numbers import ( + VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS, +) # Converts a source string into a list of textual representation @@ -24,12 +29,12 @@ def stringify_tokens_from_source(token_generator, source_string): missing_trailing_nl = source_string[-1] not in '\r\n' for type, token, start, end, line in token_generator: - if type == ENDMARKER: + if type == tokenize.ENDMARKER: break # Ignore the new line on the last line if the input lacks one - if missing_trailing_nl and type == NEWLINE and end[0] == num_lines: + if missing_trailing_nl and type == tokenize.NEWLINE and end[0] == num_lines: continue - type = tok_name[type] + type = tokenize.tok_name[type] result.append(f" {type:10} {token!r:13} {start} {end}") return result @@ -45,18 +50,37 @@ def check_tokenize(self, s, expected): # Format the tokens in s in a table format. # The ENDMARKER and final NEWLINE are omitted. f = BytesIO(s.encode('utf-8')) - result = stringify_tokens_from_source(tokenize(f.readline), s) + result = stringify_tokens_from_source(tokenize.tokenize(f.readline), s) self.assertEqual(result, [" ENCODING 'utf-8' (0, 0) (0, 0)"] + expected.rstrip().splitlines()) + def test_invalid_readline(self): + def gen(): + yield "sdfosdg" + yield "sdfosdg" + with self.assertRaises(TypeError): + list(tokenize.tokenize(gen().__next__)) + + def gen(): + yield b"sdfosdg" + yield b"sdfosdg" + with self.assertRaises(TypeError): + list(tokenize.generate_tokens(gen().__next__)) + + def gen(): + yield "sdfosdg" + 1/0 + with self.assertRaises(ZeroDivisionError): + list(tokenize.generate_tokens(gen().__next__)) + def test_implicit_newline(self): # Make sure that the tokenizer puts in an implicit NEWLINE # when the input lacks a trailing new line. f = BytesIO("x".encode('utf-8')) - tokens = list(tokenize(f.readline)) - self.assertEqual(tokens[-2].type, NEWLINE) - self.assertEqual(tokens[-1].type, ENDMARKER) + tokens = list(tokenize.tokenize(f.readline)) + self.assertEqual(tokens[-2].type, tokenize.NEWLINE) + self.assertEqual(tokens[-1].type, tokenize.ENDMARKER) def test_basic(self): self.check_tokenize("1 + 1", """\ @@ -83,6 +107,32 @@ def test_basic(self): NEWLINE '\\n' (4, 26) (4, 27) DEDENT '' (5, 0) (5, 0) """) + + self.check_tokenize("if True:\r\n # NL\r\n foo='bar'\r\n\r\n", """\ + NAME 'if' (1, 0) (1, 2) + NAME 'True' (1, 3) (1, 7) + OP ':' (1, 7) (1, 8) + NEWLINE '\\r\\n' (1, 8) (1, 10) + COMMENT '# NL' (2, 4) (2, 8) + NL '\\r\\n' (2, 8) (2, 10) + INDENT ' ' (3, 0) (3, 4) + NAME 'foo' (3, 4) (3, 7) + OP '=' (3, 7) (3, 8) + STRING "\'bar\'" (3, 8) (3, 13) + NEWLINE '\\r\\n' (3, 13) (3, 15) + NL '\\r\\n' (4, 0) (4, 2) + DEDENT '' (5, 0) (5, 0) + """) + + self.check_tokenize("x = 1 + \\\r\n1\r\n", """\ + NAME 'x' (1, 0) (1, 1) + OP '=' (1, 2) (1, 3) + NUMBER '1' (1, 4) (1, 5) + OP '+' (1, 6) (1, 7) + NUMBER '1' (2, 0) (2, 1) + NEWLINE '\\r\\n' (2, 1) (2, 3) + """) + indent_error_file = b"""\ def k(x): x += 2 @@ -91,9 +141,18 @@ def k(x): readline = BytesIO(indent_error_file).readline with self.assertRaisesRegex(IndentationError, "unindent does not match any " - "outer indentation level"): - for tok in tokenize(readline): + "outer indentation level") as e: + for tok in tokenize.tokenize(readline): pass + self.assertEqual(e.exception.lineno, 3) + self.assertEqual(e.exception.filename, '<string>') + self.assertEqual(e.exception.end_lineno, None) + self.assertEqual(e.exception.end_offset, None) + self.assertEqual( + e.exception.msg, + 'unindent does not match any outer indentation level') + self.assertEqual(e.exception.offset, 9) + self.assertEqual(e.exception.text, ' x += 5') def test_int(self): # Ordinary integers and binary operators @@ -177,7 +236,7 @@ def test_long(self): """) def test_float(self): - # Floating point numbers + # Floating-point numbers self.check_tokenize("x = 3.14159", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) @@ -219,8 +278,8 @@ def test_float(self): def test_underscore_literals(self): def number_token(s): f = BytesIO(s.encode('utf-8')) - for toktype, token, start, end, line in tokenize(f.readline): - if toktype == NUMBER: + for toktype, token, start, end, line in tokenize.tokenize(f.readline): + if toktype == tokenize.NUMBER: return token return 'invalid token' for lit in VALID_UNDERSCORE_LITERALS: @@ -228,7 +287,16 @@ def number_token(s): # this won't work with compound complex inputs continue self.assertEqual(number_token(lit), lit) + # Valid cases with extra underscores in the tokenize module + # See gh-105549 for context + extra_valid_cases = {"0_7", "09_99"} for lit in INVALID_UNDERSCORE_LITERALS: + if lit in extra_valid_cases: + continue + try: + number_token(lit) + except tokenize.TokenError: + continue self.assertNotEqual(number_token(lit), lit) def test_string(self): @@ -380,21 +448,175 @@ def test_string(self): STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) """) self.check_tokenize('f"abc"', """\ - STRING 'f"abc"' (1, 0) (1, 6) + FSTRING_START 'f"' (1, 0) (1, 2) + FSTRING_MIDDLE 'abc' (1, 2) (1, 5) + FSTRING_END '"' (1, 5) (1, 6) """) self.check_tokenize('fR"a{b}c"', """\ - STRING 'fR"a{b}c"' (1, 0) (1, 9) + FSTRING_START 'fR"' (1, 0) (1, 3) + FSTRING_MIDDLE 'a' (1, 3) (1, 4) + OP '{' (1, 4) (1, 5) + NAME 'b' (1, 5) (1, 6) + OP '}' (1, 6) (1, 7) + FSTRING_MIDDLE 'c' (1, 7) (1, 8) + FSTRING_END '"' (1, 8) (1, 9) + """) + self.check_tokenize('fR"a{{{b!r}}}c"', """\ + FSTRING_START 'fR"' (1, 0) (1, 3) + FSTRING_MIDDLE 'a{' (1, 3) (1, 5) + OP '{' (1, 6) (1, 7) + NAME 'b' (1, 7) (1, 8) + OP '!' (1, 8) (1, 9) + NAME 'r' (1, 9) (1, 10) + OP '}' (1, 10) (1, 11) + FSTRING_MIDDLE '}' (1, 11) (1, 12) + FSTRING_MIDDLE 'c' (1, 13) (1, 14) + FSTRING_END '"' (1, 14) (1, 15) + """) + self.check_tokenize('f"{{{1+1}}}"', """\ + FSTRING_START 'f"' (1, 0) (1, 2) + FSTRING_MIDDLE '{' (1, 2) (1, 3) + OP '{' (1, 4) (1, 5) + NUMBER '1' (1, 5) (1, 6) + OP '+' (1, 6) (1, 7) + NUMBER '1' (1, 7) (1, 8) + OP '}' (1, 8) (1, 9) + FSTRING_MIDDLE '}' (1, 9) (1, 10) + FSTRING_END '"' (1, 11) (1, 12) + """) + self.check_tokenize('f"""{f\'\'\'{f\'{f"{1+1}"}\'}\'\'\'}"""', """\ + FSTRING_START 'f\"""' (1, 0) (1, 4) + OP '{' (1, 4) (1, 5) + FSTRING_START "f'''" (1, 5) (1, 9) + OP '{' (1, 9) (1, 10) + FSTRING_START "f'" (1, 10) (1, 12) + OP '{' (1, 12) (1, 13) + FSTRING_START 'f"' (1, 13) (1, 15) + OP '{' (1, 15) (1, 16) + NUMBER '1' (1, 16) (1, 17) + OP '+' (1, 17) (1, 18) + NUMBER '1' (1, 18) (1, 19) + OP '}' (1, 19) (1, 20) + FSTRING_END '"' (1, 20) (1, 21) + OP '}' (1, 21) (1, 22) + FSTRING_END "'" (1, 22) (1, 23) + OP '}' (1, 23) (1, 24) + FSTRING_END "'''" (1, 24) (1, 27) + OP '}' (1, 27) (1, 28) + FSTRING_END '\"""' (1, 28) (1, 31) + """) + self.check_tokenize('f""" x\nstr(data, encoding={invalid!r})\n"""', """\ + FSTRING_START 'f\"""' (1, 0) (1, 4) + FSTRING_MIDDLE ' x\\nstr(data, encoding=' (1, 4) (2, 19) + OP '{' (2, 19) (2, 20) + NAME 'invalid' (2, 20) (2, 27) + OP '!' (2, 27) (2, 28) + NAME 'r' (2, 28) (2, 29) + OP '}' (2, 29) (2, 30) + FSTRING_MIDDLE ')\\n' (2, 30) (3, 0) + FSTRING_END '\"""' (3, 0) (3, 3) + """) + self.check_tokenize('f"""123456789\nsomething{None}bad"""', """\ + FSTRING_START 'f\"""' (1, 0) (1, 4) + FSTRING_MIDDLE '123456789\\nsomething' (1, 4) (2, 9) + OP '{' (2, 9) (2, 10) + NAME 'None' (2, 10) (2, 14) + OP '}' (2, 14) (2, 15) + FSTRING_MIDDLE 'bad' (2, 15) (2, 18) + FSTRING_END '\"""' (2, 18) (2, 21) """) self.check_tokenize('f"""abc"""', """\ - STRING 'f\"\"\"abc\"\"\"' (1, 0) (1, 10) + FSTRING_START 'f\"""' (1, 0) (1, 4) + FSTRING_MIDDLE 'abc' (1, 4) (1, 7) + FSTRING_END '\"""' (1, 7) (1, 10) """) self.check_tokenize(r'f"abc\ def"', """\ - STRING 'f"abc\\\\\\ndef"' (1, 0) (2, 4) + FSTRING_START 'f"' (1, 0) (1, 2) + FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 2) (2, 3) + FSTRING_END '"' (2, 3) (2, 4) """) self.check_tokenize(r'Rf"abc\ def"', """\ - STRING 'Rf"abc\\\\\\ndef"' (1, 0) (2, 4) + FSTRING_START 'Rf"' (1, 0) (1, 3) + FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 3) (2, 3) + FSTRING_END '"' (2, 3) (2, 4) + """) + self.check_tokenize("f'some words {a+b:.3f} more words {c+d=} final words'", """\ + FSTRING_START "f'" (1, 0) (1, 2) + FSTRING_MIDDLE 'some words ' (1, 2) (1, 13) + OP '{' (1, 13) (1, 14) + NAME 'a' (1, 14) (1, 15) + OP '+' (1, 15) (1, 16) + NAME 'b' (1, 16) (1, 17) + OP ':' (1, 17) (1, 18) + FSTRING_MIDDLE '.3f' (1, 18) (1, 21) + OP '}' (1, 21) (1, 22) + FSTRING_MIDDLE ' more words ' (1, 22) (1, 34) + OP '{' (1, 34) (1, 35) + NAME 'c' (1, 35) (1, 36) + OP '+' (1, 36) (1, 37) + NAME 'd' (1, 37) (1, 38) + OP '=' (1, 38) (1, 39) + OP '}' (1, 39) (1, 40) + FSTRING_MIDDLE ' final words' (1, 40) (1, 52) + FSTRING_END "'" (1, 52) (1, 53) + """) + self.check_tokenize("""\ +f'''{ +3 +=}'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + OP '{' (1, 4) (1, 5) + NL '\\n' (1, 5) (1, 6) + NUMBER '3' (2, 0) (2, 1) + NL '\\n' (2, 1) (2, 2) + OP '=' (3, 0) (3, 1) + OP '}' (3, 1) (3, 2) + FSTRING_END "'''" (3, 2) (3, 5) + """) + self.check_tokenize("""\ +f'''__{ + x:a +}__'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + FSTRING_MIDDLE '__' (1, 4) (1, 6) + OP '{' (1, 6) (1, 7) + NL '\\n' (1, 7) (1, 8) + NAME 'x' (2, 4) (2, 5) + OP ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'a\\n' (2, 6) (3, 0) + OP '}' (3, 0) (3, 1) + FSTRING_MIDDLE '__' (3, 1) (3, 3) + FSTRING_END "'''" (3, 3) (3, 6) + """) + self.check_tokenize("""\ +f'''__{ + x:a + b + c + d +}__'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + FSTRING_MIDDLE '__' (1, 4) (1, 6) + OP '{' (1, 6) (1, 7) + NL '\\n' (1, 7) (1, 8) + NAME 'x' (2, 4) (2, 5) + OP ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'a\\n b\\n c\\n d\\n' (2, 6) (6, 0) + OP '}' (6, 0) (6, 1) + FSTRING_MIDDLE '__' (6, 1) (6, 3) + FSTRING_END "'''" (6, 3) (6, 6) + """) + + self.check_tokenize("""\ + '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli + aktualni pracownicy, obecni pracownicy''' +""", """\ + INDENT ' ' (1, 0) (1, 4) + STRING "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\\n aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45) + NEWLINE '\\n' (2, 45) (2, 46) + DEDENT '' (3, 0) (3, 0) """) def test_function(self): @@ -945,29 +1167,95 @@ async def bar(): pass DEDENT '' (7, 0) (7, 0) """) + def test_newline_after_parenthesized_block_with_comment(self): + self.check_tokenize('''\ +[ + # A comment here + 1 +] +''', """\ + OP '[' (1, 0) (1, 1) + NL '\\n' (1, 1) (1, 2) + COMMENT '# A comment here' (2, 4) (2, 20) + NL '\\n' (2, 20) (2, 21) + NUMBER '1' (3, 4) (3, 5) + NL '\\n' (3, 5) (3, 6) + OP ']' (4, 0) (4, 1) + NEWLINE '\\n' (4, 1) (4, 2) + """) + + def test_closing_parenthesis_from_different_line(self): + self.check_tokenize("); x", """\ + OP ')' (1, 0) (1, 1) + OP ';' (1, 1) (1, 2) + NAME 'x' (1, 3) (1, 4) + """) + + def test_multiline_non_ascii_fstring(self): + self.check_tokenize("""\ +a = f''' + Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''""", """\ + NAME 'a' (1, 0) (1, 1) + OP '=' (1, 2) (1, 3) + FSTRING_START "f\'\'\'" (1, 4) (1, 8) + FSTRING_MIDDLE '\\n Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68) + FSTRING_END "\'\'\'" (2, 68) (2, 71) + """) + + def test_multiline_non_ascii_fstring_with_expr(self): + self.check_tokenize("""\ +f''' + 🔗 This is a test {test_arg1}🔗 +🔗'''""", """\ + FSTRING_START "f\'\'\'" (1, 0) (1, 4) + FSTRING_MIDDLE '\\n 🔗 This is a test ' (1, 4) (2, 21) + OP '{' (2, 21) (2, 22) + NAME 'test_arg1' (2, 22) (2, 31) + OP '}' (2, 31) (2, 32) + FSTRING_MIDDLE '🔗\\n🔗' (2, 32) (3, 1) + FSTRING_END "\'\'\'" (3, 1) (3, 4) + """) + + # gh-139516, the '\n' is explicit to ensure no trailing whitespace which would invalidate the test + self.check_tokenize('''f"{f(a=lambda: 'à'\n)}"''', """\ + FSTRING_START \'f"\' (1, 0) (1, 2) + OP '{' (1, 2) (1, 3) + NAME 'f' (1, 3) (1, 4) + OP '(' (1, 4) (1, 5) + NAME 'a' (1, 5) (1, 6) + OP '=' (1, 6) (1, 7) + NAME 'lambda' (1, 7) (1, 13) + OP ':' (1, 13) (1, 14) + STRING "\'à\'" (1, 15) (1, 18) + NL '\\n' (1, 18) (1, 19) + OP ')' (2, 0) (2, 1) + OP '}' (2, 1) (2, 2) + FSTRING_END \'"\' (2, 2) (2, 3) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. # The ENDMARKER and final NEWLINE are omitted. f = StringIO(s) - result = stringify_tokens_from_source(generate_tokens(f.readline), s) + result = stringify_tokens_from_source(tokenize.generate_tokens(f.readline), s) self.assertEqual(result, expected.rstrip().splitlines()) def decistmt(s): result = [] - g = tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string + g = tokenize.tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string for toknum, tokval, _, _, _ in g: - if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens + if toknum == tokenize.NUMBER and '.' in tokval: # replace NUMBER tokens result.extend([ - (NAME, 'Decimal'), - (OP, '('), - (STRING, repr(tokval)), - (OP, ')') + (tokenize.NAME, 'Decimal'), + (tokenize.OP, '('), + (tokenize.STRING, repr(tokval)), + (tokenize.OP, ')') ]) else: result.append((toknum, tokval)) - return untokenize(result).decode('utf-8') + return tokenize.untokenize(result).decode('utf-8').strip() class TestMisc(TestCase): @@ -991,6 +1279,13 @@ def test_decistmt(self): self.assertEqual(eval(decistmt(s)), Decimal('-3.217160342717258261933904529E-7')) + def test___all__(self): + expected = token.__all__ + [ + "TokenInfo", "TokenError", "generate_tokens", + "detect_encoding", "untokenize", "open", "tokenize", + ] + self.assertCountEqual(tokenize.__all__, expected) + class TestTokenizerAdheresToPep0263(TestCase): """ @@ -998,8 +1293,9 @@ class TestTokenizerAdheresToPep0263(TestCase): """ def _testFile(self, filename): - path = os.path.join(os.path.dirname(__file__), filename) - TestRoundtrip.check_roundtrip(self, open(path, 'rb')) + path = os.path.join(os.path.dirname(__file__), 'tokenizedata', filename) + with open(path, 'rb') as f: + TestRoundtrip.check_roundtrip(self, f) def test_utf8_coding_cookie_and_no_utf8_bom(self): f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt' @@ -1024,8 +1320,6 @@ def test_utf8_coding_cookie_and_utf8_bom(self): f = 'tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt' self._testFile(f) - # TODO: RUSTPYTHON - @unittest.expectedFailure # "bad_coding.py" and "bad_coding2.py" make the WASM CI fail def test_bad_coding_cookie(self): self.assertRaises(SyntaxError, self._testFile, 'bad_coding.py') self.assertRaises(SyntaxError, self._testFile, 'bad_coding2.py') @@ -1041,33 +1335,18 @@ def readline(): nonlocal first if not first: first = True - return line + yield line else: - return b'' + yield b'' # skip the initial encoding token and the end tokens - tokens = list(_tokenize(readline, encoding='utf-8'))[1:-2] - expected_tokens = [(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')] + tokens = list(tokenize._generate_tokens_from_c_tokenizer(readline().__next__, + encoding='utf-8', + extra_tokens=True))[:-2] + expected_tokens = [tokenize.TokenInfo(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')] self.assertEqual(tokens, expected_tokens, "bytes not decoded with encoding") - def test__tokenize_does_not_decode_with_encoding_none(self): - literal = '"ЉЊЈЁЂ"' - first = False - def readline(): - nonlocal first - if not first: - first = True - return literal - else: - return b'' - - # skip the end tokens - tokens = list(_tokenize(readline, encoding=None))[:-2] - expected_tokens = [(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')] - self.assertEqual(tokens, expected_tokens, - "string not tokenized when encoding is None") - class TestDetectEncoding(TestCase): @@ -1084,24 +1363,63 @@ def readline(): def test_no_bom_no_encoding_cookie(self): lines = ( - b'# something\n', + b'#!/home/\xc3\xa4/bin/python\n', + b'# something \xe2\x82\xac\n', b'print(something)\n', b'do_something(else)\n' ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'utf-8') self.assertEqual(consumed_lines, list(lines[:2])) + def test_no_bom_no_encoding_cookie_first_line_error(self): + lines = ( + b'#!/home/\xa4/bin/python\n\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_no_bom_no_encoding_cookie_second_line_error(self): + lines = ( + b'#!/usr/bin/python\n', + b'# something \xe2\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + def test_bom_no_cookie(self): lines = ( - b'\xef\xbb\xbf# something\n', + b'\xef\xbb\xbf#!/home/\xc3\xa4/bin/python\n', b'print(something)\n', b'do_something(else)\n' ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'utf-8-sig') self.assertEqual(consumed_lines, - [b'# something\n', b'print(something)\n']) + [b'#!/home/\xc3\xa4/bin/python\n', b'print(something)\n']) + + def test_bom_no_cookie_first_line_error(self): + lines = ( + b'\xef\xbb\xbf#!/home/\xa4/bin/python\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_bom_no_cookie_second_line_error(self): + lines = ( + b'\xef\xbb\xbf#!/usr/bin/python\n', + b'# something \xe2\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) def test_cookie_first_line_no_bom(self): lines = ( @@ -1109,7 +1427,7 @@ def test_cookie_first_line_no_bom(self): b'print(something)\n', b'do_something(else)\n' ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'iso-8859-1') self.assertEqual(consumed_lines, [b'# -*- coding: latin-1 -*-\n']) @@ -1119,7 +1437,7 @@ def test_matched_bom_and_cookie_first_line(self): b'print(something)\n', b'do_something(else)\n' ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'utf-8-sig') self.assertEqual(consumed_lines, [b'# coding=utf-8\n']) @@ -1130,7 +1448,7 @@ def test_mismatched_bom_and_cookie_first_line_raises_syntaxerror(self): b'do_something(else)\n' ) readline = self.get_readline(lines) - self.assertRaises(SyntaxError, detect_encoding, readline) + self.assertRaises(SyntaxError, tokenize.detect_encoding, readline) def test_cookie_second_line_no_bom(self): lines = ( @@ -1139,7 +1457,7 @@ def test_cookie_second_line_no_bom(self): b'print(something)\n', b'do_something(else)\n' ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'ascii') expected = [b'#! something\n', b'# vim: set fileencoding=ascii :\n'] self.assertEqual(consumed_lines, expected) @@ -1151,7 +1469,7 @@ def test_matched_bom_and_cookie_second_line(self): b'print(something)\n', b'do_something(else)\n' ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'utf-8-sig') self.assertEqual(consumed_lines, [b'#! something\n', b'f# coding=utf-8\n']) @@ -1164,7 +1482,7 @@ def test_mismatched_bom_and_cookie_second_line_raises_syntaxerror(self): b'do_something(else)\n' ) readline = self.get_readline(lines) - self.assertRaises(SyntaxError, detect_encoding, readline) + self.assertRaises(SyntaxError, tokenize.detect_encoding, readline) def test_cookie_second_line_noncommented_first_line(self): lines = ( @@ -1172,21 +1490,65 @@ def test_cookie_second_line_noncommented_first_line(self): b'# vim: set fileencoding=iso8859-15 :\n', b"print('\xe2\x82\xac')\n" ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'utf-8') expected = [b"print('\xc2\xa3')\n"] self.assertEqual(consumed_lines, expected) - def test_cookie_second_line_commented_first_line(self): + def test_first_non_utf8_coding_line(self): lines = ( - b"#print('\xc2\xa3')\n", - b'# vim: set fileencoding=iso8859-15 :\n', - b"print('\xe2\x82\xac')\n" + b'#coding:iso-8859-15 \xa4\n', + b'print(something)\n' ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'iso8859-15') - expected = [b"#print('\xc2\xa3')\n", b'# vim: set fileencoding=iso8859-15 :\n'] - self.assertEqual(consumed_lines, expected) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_first_utf8_coding_line_error(self): + lines = ( + b'#coding:ascii \xc3\xa4\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_second_non_utf8_coding_line(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:iso-8859-15 \xa4\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_second_utf8_coding_line_error(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:ascii \xc3\xa4\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_non_utf8_shebang(self): + lines = ( + b'#!/home/\xa4/bin/python\n', + b'#coding:iso-8859-15\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_utf8_shebang_error(self): + lines = ( + b'#!/home/\xc3\xa4/bin/python\n', + b'#coding:ascii\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) def test_cookie_second_line_empty_first_line(self): lines = ( @@ -1194,13 +1556,77 @@ def test_cookie_second_line_empty_first_line(self): b'# vim: set fileencoding=iso8859-15 :\n', b"print('\xe2\x82\xac')\n" ) - encoding, consumed_lines = detect_encoding(self.get_readline(lines)) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'iso8859-15') expected = [b'\n', b'# vim: set fileencoding=iso8859-15 :\n'] self.assertEqual(consumed_lines, expected) + def test_cookie_third_line(self): + lines = ( + b'#!/home/\xc3\xa4/bin/python\n', + b'# something\n', + b'# vim: set fileencoding=ascii :\n', + b'print(something)\n', + b'do_something(else)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'utf-8') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_double_coding_line(self): + # If the first line matches the second line is ignored. + lines = ( + b'#coding:iso8859-15\n', + b'#coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_double_coding_same_line(self): + lines = ( + b'#coding:iso8859-15 coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_double_coding_utf8(self): + lines = ( + b'#coding:utf-8\n', + b'#coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'utf-8') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_nul_in_first_coding_line(self): + lines = ( + b'#coding:iso8859-15\x00\n', + b'\n', + b'\n', + b'print(something)\n' + ) + with self.assertRaisesRegex(SyntaxError, + "source code cannot contain null bytes"): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_nul_in_second_coding_line(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:iso8859-15\x00\n', + b'\n', + b'print(something)\n' + ) + with self.assertRaisesRegex(SyntaxError, + "source code cannot contain null bytes"): + tokenize.detect_encoding(self.get_readline(lines)) + def test_latin1_normalization(self): - # See get_normal_name() in tokenizer.c. + # See get_normal_name() in Parser/tokenizer/helpers.c. encodings = ("latin-1", "iso-8859-1", "iso-latin-1", "latin-1-unix", "iso-8859-1-unix", "iso-latin-1-mac") for encoding in encodings: @@ -1211,21 +1637,20 @@ def test_latin1_normalization(self): b"print(things)\n", b"do_something += 4\n") rl = self.get_readline(lines) - found, consumed_lines = detect_encoding(rl) + found, consumed_lines = tokenize.detect_encoding(rl) self.assertEqual(found, "iso-8859-1") def test_syntaxerror_latin1(self): - # Issue 14629: need to raise SyntaxError if the first + # Issue 14629: need to raise TokenError if the first # line(s) have non-UTF-8 characters lines = ( b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S ) readline = self.get_readline(lines) - self.assertRaises(SyntaxError, detect_encoding, readline) - + self.assertRaises(SyntaxError, tokenize.detect_encoding, readline) def test_utf8_normalization(self): - # See get_normal_name() in tokenizer.c. + # See get_normal_name() in Parser/tokenizer/helpers.c. encodings = ("utf-8", "utf-8-mac", "utf-8-unix") for encoding in encodings: for rep in ("-", "_"): @@ -1234,40 +1659,40 @@ def test_utf8_normalization(self): b"# coding: " + enc.encode("ascii") + b"\n", b"1 + 3\n") rl = self.get_readline(lines) - found, consumed_lines = detect_encoding(rl) + found, consumed_lines = tokenize.detect_encoding(rl) self.assertEqual(found, "utf-8") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_short_files(self): readline = self.get_readline((b'print(something)\n',)) - encoding, consumed_lines = detect_encoding(readline) + encoding, consumed_lines = tokenize.detect_encoding(readline) self.assertEqual(encoding, 'utf-8') self.assertEqual(consumed_lines, [b'print(something)\n']) - encoding, consumed_lines = detect_encoding(self.get_readline(())) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(())) self.assertEqual(encoding, 'utf-8') self.assertEqual(consumed_lines, []) readline = self.get_readline((b'\xef\xbb\xbfprint(something)\n',)) - encoding, consumed_lines = detect_encoding(readline) + encoding, consumed_lines = tokenize.detect_encoding(readline) self.assertEqual(encoding, 'utf-8-sig') self.assertEqual(consumed_lines, [b'print(something)\n']) readline = self.get_readline((b'\xef\xbb\xbf',)) - encoding, consumed_lines = detect_encoding(readline) + encoding, consumed_lines = tokenize.detect_encoding(readline) self.assertEqual(encoding, 'utf-8-sig') self.assertEqual(consumed_lines, []) readline = self.get_readline((b'# coding: bad\n',)) - self.assertRaises(SyntaxError, detect_encoding, readline) + self.assertRaises(SyntaxError, tokenize.detect_encoding, readline) def test_false_encoding(self): # Issue 18873: "Encoding" detected in non-comment lines readline = self.get_readline((b'print("#coding=fake")',)) - encoding, consumed_lines = detect_encoding(readline) + encoding, consumed_lines = tokenize.detect_encoding(readline) self.assertEqual(encoding, 'utf-8') self.assertEqual(consumed_lines, [b'print("#coding=fake")']) + @support.thread_unsafe def test_open(self): filename = os_helper.TESTFN + '.py' self.addCleanup(os_helper.unlink, filename) @@ -1277,14 +1702,14 @@ def test_open(self): with open(filename, 'w', encoding=encoding) as fp: print("# coding: %s" % encoding, file=fp) print("print('euro:\u20ac')", file=fp) - with tokenize_open(filename) as fp: + with tokenize.open(filename) as fp: self.assertEqual(fp.encoding, encoding) self.assertEqual(fp.mode, 'r') # test BOM (no coding cookie) with open(filename, 'w', encoding='utf-8-sig') as fp: print("print('euro:\u20ac')", file=fp) - with tokenize_open(filename) as fp: + with tokenize.open(filename) as fp: self.assertEqual(fp.encoding, 'utf-8-sig') self.assertEqual(fp.mode, 'r') @@ -1311,17 +1736,16 @@ def readline(self): ins = Bunk(lines, path) # Make sure lacking a name isn't an issue. del ins.name - detect_encoding(ins.readline) + tokenize.detect_encoding(ins.readline) with self.assertRaisesRegex(SyntaxError, '.*{}'.format(path)): ins = Bunk(lines, path) - detect_encoding(ins.readline) + tokenize.detect_encoding(ins.readline) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_open_error(self): # Issue #23840: open() must close the binary file on error m = BytesIO(b'#coding:xxx') with mock.patch('tokenize._builtin_open', return_value=m): - self.assertRaises(SyntaxError, tokenize_open, 'foobar') + self.assertRaises(SyntaxError, tokenize.open, 'foobar') self.assertTrue(m.closed) @@ -1329,17 +1753,20 @@ class TestTokenize(TestCase): def test_tokenize(self): import tokenize as tokenize_module - encoding = object() + encoding = "utf-8" encoding_used = None def mock_detect_encoding(readline): return encoding, [b'first', b'second'] - def mock__tokenize(readline, encoding): + def mock__tokenize(readline, encoding, **kwargs): nonlocal encoding_used encoding_used = encoding out = [] while True: - next_line = readline() + try: + next_line = readline() + except StopIteration: + return out if next_line: out.append(next_line) continue @@ -1354,16 +1781,16 @@ def mock_readline(): return str(counter).encode() orig_detect_encoding = tokenize_module.detect_encoding - orig__tokenize = tokenize_module._tokenize + orig_c_token = tokenize_module._generate_tokens_from_c_tokenizer tokenize_module.detect_encoding = mock_detect_encoding - tokenize_module._tokenize = mock__tokenize + tokenize_module._generate_tokens_from_c_tokenizer = mock__tokenize try: - results = tokenize(mock_readline) - self.assertEqual(list(results), + results = tokenize.tokenize(mock_readline) + self.assertEqual(list(results)[1:], [b'first', b'second', b'1', b'2', b'3', b'4']) finally: tokenize_module.detect_encoding = orig_detect_encoding - tokenize_module._tokenize = orig__tokenize + tokenize_module._generate_tokens_from_c_tokenizer = orig_c_token self.assertEqual(encoding_used, encoding) @@ -1375,23 +1802,23 @@ def test_oneline_defs(self): buf = '\n'.join(buf) # Test that 500 consequent, one-line defs is OK - toks = list(tokenize(BytesIO(buf.encode('utf-8')).readline)) + toks = list(tokenize.tokenize(BytesIO(buf.encode('utf-8')).readline)) self.assertEqual(toks[-3].string, 'OK') # [-1] is always ENDMARKER # [-2] is always NEWLINE def assertExactTypeEqual(self, opstr, *optypes): - tokens = list(tokenize(BytesIO(opstr.encode('utf-8')).readline)) + tokens = list(tokenize.tokenize(BytesIO(opstr.encode('utf-8')).readline)) num_optypes = len(optypes) self.assertEqual(len(tokens), 3 + num_optypes) - self.assertEqual(tok_name[tokens[0].exact_type], - tok_name[ENCODING]) + self.assertEqual(tokenize.tok_name[tokens[0].exact_type], + tokenize.tok_name[tokenize.ENCODING]) for i in range(num_optypes): - self.assertEqual(tok_name[tokens[i + 1].exact_type], - tok_name[optypes[i]]) - self.assertEqual(tok_name[tokens[1 + num_optypes].exact_type], - tok_name[token.NEWLINE]) - self.assertEqual(tok_name[tokens[2 + num_optypes].exact_type], - tok_name[token.ENDMARKER]) + self.assertEqual(tokenize.tok_name[tokens[i + 1].exact_type], + tokenize.tok_name[optypes[i]]) + self.assertEqual(tokenize.tok_name[tokens[1 + num_optypes].exact_type], + tokenize.tok_name[token.NEWLINE]) + self.assertEqual(tokenize.tok_name[tokens[2 + num_optypes].exact_type], + tokenize.tok_name[token.ENDMARKER]) def test_exact_type(self): self.assertExactTypeEqual('()', token.LPAR, token.RPAR) @@ -1441,11 +1868,11 @@ def test_exact_type(self): self.assertExactTypeEqual('@=', token.ATEQUAL) self.assertExactTypeEqual('a**2+b**2==c**2', - NAME, token.DOUBLESTAR, NUMBER, + tokenize.NAME, token.DOUBLESTAR, tokenize.NUMBER, token.PLUS, - NAME, token.DOUBLESTAR, NUMBER, + tokenize.NAME, token.DOUBLESTAR, tokenize.NUMBER, token.EQEQUAL, - NAME, token.DOUBLESTAR, NUMBER) + tokenize.NAME, token.DOUBLESTAR, tokenize.NUMBER) self.assertExactTypeEqual('{1, 2, 3}', token.LBRACE, token.NUMBER, token.COMMA, @@ -1465,19 +1892,55 @@ def test_pathological_trailing_whitespace(self): def test_comment_at_the_end_of_the_source_without_newline(self): # See http://bugs.python.org/issue44667 source = 'b = 1\n\n#test' - expected_tokens = [token.NAME, token.EQUAL, token.NUMBER, token.NEWLINE, token.NL, token.COMMENT] + expected_tokens = [ + tokenize.TokenInfo(type=token.ENCODING, string='utf-8', start=(0, 0), end=(0, 0), line=''), + tokenize.TokenInfo(type=token.NAME, string='b', start=(1, 0), end=(1, 1), line='b = 1\n'), + tokenize.TokenInfo(type=token.OP, string='=', start=(1, 2), end=(1, 3), line='b = 1\n'), + tokenize.TokenInfo(type=token.NUMBER, string='1', start=(1, 4), end=(1, 5), line='b = 1\n'), + tokenize.TokenInfo(type=token.NEWLINE, string='\n', start=(1, 5), end=(1, 6), line='b = 1\n'), + tokenize.TokenInfo(type=token.NL, string='\n', start=(2, 0), end=(2, 1), line='\n'), + tokenize.TokenInfo(type=token.COMMENT, string='#test', start=(3, 0), end=(3, 5), line='#test'), + tokenize.TokenInfo(type=token.NL, string='', start=(3, 5), end=(3, 6), line='#test'), + tokenize.TokenInfo(type=token.ENDMARKER, string='', start=(4, 0), end=(4, 0), line='') + ] + + tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline)) + self.assertEqual(tokens, expected_tokens) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 869 characters long. Set self.maxDiff to None to see it. + def test_newline_and_space_at_the_end_of_the_source_without_newline(self): + # See https://github.com/python/cpython/issues/105435 + source = 'a\n ' + expected_tokens = [ + tokenize.TokenInfo(token.ENCODING, string='utf-8', start=(0, 0), end=(0, 0), line=''), + tokenize.TokenInfo(token.NAME, string='a', start=(1, 0), end=(1, 1), line='a\n'), + tokenize.TokenInfo(token.NEWLINE, string='\n', start=(1, 1), end=(1, 2), line='a\n'), + tokenize.TokenInfo(token.NL, string='', start=(2, 1), end=(2, 2), line=' '), + tokenize.TokenInfo(token.ENDMARKER, string='', start=(3, 0), end=(3, 0), line='') + ] + + tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline)) + self.assertEqual(tokens, expected_tokens) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'SyntaxError' not found in b'OSError: stream did not contain valid UTF-8\n' + def test_invalid_character_in_fstring_middle(self): + # See gh-103824 + script = b'''F""" + \xe5"""''' + + with os_helper.temp_dir() as temp_dir: + filename = os.path.join(temp_dir, "script.py") + with open(filename, 'wb') as file: + file.write(script) + rs, _ = run_python_until_end(filename) + self.assertIn(b"SyntaxError", rs.err) - tokens = list(tokenize(BytesIO(source.encode('utf-8')).readline)) - self.assertEqual(tok_name[tokens[0].exact_type], tok_name[ENCODING]) - for i in range(6): - self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]]) - self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER]) class UntokenizeTest(TestCase): def test_bad_input_order(self): # raise if previous row - u = Untokenizer() + u = tokenize.Untokenizer() u.prev_row = 2 u.prev_col = 2 with self.assertRaises(ValueError) as cm: @@ -1489,7 +1952,7 @@ def test_bad_input_order(self): def test_backslash_continuation(self): # The problem is that <whitespace>\<newline> leaves no token - u = Untokenizer() + u = tokenize.Untokenizer() u.prev_row = 1 u.prev_col = 1 u.tokens = [] @@ -1501,17 +1964,33 @@ def test_backslash_continuation(self): TestRoundtrip.check_roundtrip(self, 'a\n b\n c\n \\\n c\n') def test_iter_compat(self): - u = Untokenizer() - token = (NAME, 'Hello') - tokens = [(ENCODING, 'utf-8'), token] + u = tokenize.Untokenizer() + token = (tokenize.NAME, 'Hello') + tokens = [(tokenize.ENCODING, 'utf-8'), token] u.compat(token, iter([])) self.assertEqual(u.tokens, ["Hello "]) - u = Untokenizer() + u = tokenize.Untokenizer() self.assertEqual(u.untokenize(iter([token])), 'Hello ') - u = Untokenizer() + u = tokenize.Untokenizer() self.assertEqual(u.untokenize(iter(tokens)), 'Hello ') self.assertEqual(u.encoding, 'utf-8') - self.assertEqual(untokenize(iter(tokens)), b'Hello ') + self.assertEqual(tokenize.untokenize(iter(tokens)), b'Hello ') + + +def contains_ambiguous_backslash(source): + """Return `True` if the source contains a backslash on a + line by itself. For example: + + a = (1 + \\ + ) + + Code like this cannot be untokenized exactly. This is because + the tokenizer does not produce any tokens for the line containing + the backslash and so there is no way to know its indent. + """ + pattern = re.compile(br'\n\s*\\\r?\n') + return pattern.search(source) is not None class TestRoundtrip(TestCase): @@ -1524,6 +2003,9 @@ def check_roundtrip(self, f): tokenize.untokenize(), and the latter tokenized again to 2-tuples. The test fails if the 3 pair tokenizations do not match. + If the source code can be untokenized unambiguously, the + untokenized code must match the original code exactly. + When untokenize bugs are fixed, untokenize with 5-tuples should reproduce code that does not contain a backslash continuation following spaces. A proper test should test this. @@ -1533,21 +2015,38 @@ def check_roundtrip(self, f): code = f.encode('utf-8') else: code = f.read() - f.close() readline = iter(code.splitlines(keepends=True)).__next__ - tokens5 = list(tokenize(readline)) + tokens5 = list(tokenize.tokenize(readline)) tokens2 = [tok[:2] for tok in tokens5] # Reproduce tokens2 from pairs - bytes_from2 = untokenize(tokens2) + bytes_from2 = tokenize.untokenize(tokens2) readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__ - tokens2_from2 = [tok[:2] for tok in tokenize(readline2)] + tokens2_from2 = [tok[:2] for tok in tokenize.tokenize(readline2)] self.assertEqual(tokens2_from2, tokens2) # Reproduce tokens2 from 5-tuples - bytes_from5 = untokenize(tokens5) + bytes_from5 = tokenize.untokenize(tokens5) readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__ - tokens2_from5 = [tok[:2] for tok in tokenize(readline5)] + tokens2_from5 = [tok[:2] for tok in tokenize.tokenize(readline5)] self.assertEqual(tokens2_from5, tokens2) + if not contains_ambiguous_backslash(code): + # The BOM does not produce a token so there is no way to preserve it. + code_without_bom = code.removeprefix(b'\xef\xbb\xbf') + readline = iter(code_without_bom.splitlines(keepends=True)).__next__ + untokenized_code = tokenize.untokenize(tokenize.tokenize(readline)) + self.assertEqual(code_without_bom, untokenized_code) + + def check_line_extraction(self, f): + if isinstance(f, str): + code = f.encode('utf-8') + else: + code = f.read() + readline = iter(code.splitlines(keepends=True)).__next__ + for tok in tokenize.tokenize(readline): + if tok.type in {tokenize.ENCODING, tokenize.ENDMARKER}: + continue + self.assertEqual(tok.string, tok.line[tok.start[1]: tok.end[1]]) + def test_roundtrip(self): # There are some standard formatting practices that are easy to get right. @@ -1563,7 +2062,7 @@ def test_roundtrip(self): self.check_roundtrip("if x == 1 : \n" " print(x)\n") - fn = support.findfile("tokenize_tests.txt") + fn = support.findfile("tokenize_tests.txt", subdir="tokenizedata") with open(fn, 'rb') as f: self.check_roundtrip(f) self.check_roundtrip("if x == 1:\n" @@ -1587,6 +2086,67 @@ def test_roundtrip(self): " print('Can not import' # comment2\n)" "else: print('Loaded')\n") + self.check_roundtrip("f'\\N{EXCLAMATION MARK}'") + self.check_roundtrip(r"f'\\N{SNAKE}'") + self.check_roundtrip(r"f'\\N{{SNAKE}}'") + self.check_roundtrip(r"f'\N{SNAKE}'") + self.check_roundtrip(r"f'\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\N{SNAKE}'") + self.check_roundtrip(r"f'\\\\\\\N{SNAKE}'") + + self.check_roundtrip(r"f'\\N{1}'") + self.check_roundtrip(r"f'\\\\N{2}'") + self.check_roundtrip(r"f'\\\\\\N{3}'") + self.check_roundtrip(r"f'\\\\\\\\N{4}'") + + self.check_roundtrip(r"f'\\N{{'") + self.check_roundtrip(r"f'\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\N{{'") + self.check_roundtrip(r"f'\\\\\\\\N{{'") + + self.check_roundtrip(r"f'\n{{foo}}'") + self.check_roundtrip(r"f'\\n{{foo}}'") + self.check_roundtrip(r"f'\\\n{{foo}}'") + self.check_roundtrip(r"f'\\\\n{{foo}}'") + + self.check_roundtrip(r"f'\t{{foo}}'") + self.check_roundtrip(r"f'\\t{{foo}}'") + self.check_roundtrip(r"f'\\\t{{foo}}'") + self.check_roundtrip(r"f'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\t{{foo}}'") + self.check_roundtrip(r"rf'\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\t{{foo}}'") + self.check_roundtrip(r"rf'\\\\t{{foo}}'") + + self.check_roundtrip(r"rf'\{{foo}}'") + self.check_roundtrip(r"f'\\{{foo}}'") + self.check_roundtrip(r"rf'\\\{{foo}}'") + self.check_roundtrip(r"f'\\\\{{foo}}'") + cases = [ + """ +if 1: + "foo" +"bar" +""", + """ +if 1: + ("foo" + "bar") +""", + """ +if 1: + "foo" + "bar" +""" ] + for case in cases: + self.check_roundtrip(case) + + self.check_roundtrip(r"t'{ {}}'") + self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'") + self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'") + + def test_continuation(self): # Balancing continuation self.check_roundtrip("a = (3,4, \n" @@ -1613,26 +2173,14 @@ def test_string_concatenation(self): # Two string literals on the same line self.check_roundtrip("'' ''") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_random_files(self): # Test roundtrip on random python modules. # pass the '-ucpu' option to process the full directory. import glob, random - fn = support.findfile("tokenize_tests.txt") - tempdir = os.path.dirname(fn) or os.curdir + tempdir = os.path.dirname(__file__) or os.curdir testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py")) - # Tokenize is broken on test_pep3131.py because regular expressions are - # broken on the obscure unicode identifiers in it. *sigh* - # With roundtrip extended to test the 5-tuple mode of untokenize, - # 7 more testfiles fail. Remove them also until the failure is diagnosed. - - testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py")) - for f in ('buffer', 'builtin', 'fileio', 'inspect', 'os', 'platform', 'sys'): - testfiles.remove(os.path.join(tempdir, "test_%s.py") % f) - if not support.is_resource_enabled("cpu"): testfiles = random.sample(testfiles, 10) @@ -1642,12 +2190,13 @@ def test_random_files(self): with open(testfile, 'rb') as f: with self.subTest(file=testfile): self.check_roundtrip(f) + self.check_line_extraction(f) def roundtrip(self, code): if isinstance(code, str): code = code.encode('utf-8') - return untokenize(tokenize(BytesIO(code).readline)).decode('utf-8') + return tokenize.untokenize(tokenize.tokenize(BytesIO(code).readline)).decode('utf-8') def test_indentation_semantics_retained(self): """ @@ -1660,5 +2209,1279 @@ def test_indentation_semantics_retained(self): self.check_roundtrip(code) +class InvalidPythonTests(TestCase): + def test_number_followed_by_name(self): + # See issue #gh-105549 + source = "2sin(x)" + expected_tokens = [ + tokenize.TokenInfo(type=token.NUMBER, string='2', start=(1, 0), end=(1, 1), line='2sin(x)'), + tokenize.TokenInfo(type=token.NAME, string='sin', start=(1, 1), end=(1, 4), line='2sin(x)'), + tokenize.TokenInfo(type=token.OP, string='(', start=(1, 4), end=(1, 5), line='2sin(x)'), + tokenize.TokenInfo(type=token.NAME, string='x', start=(1, 5), end=(1, 6), line='2sin(x)'), + tokenize.TokenInfo(type=token.OP, string=')', start=(1, 6), end=(1, 7), line='2sin(x)'), + tokenize.TokenInfo(type=token.NEWLINE, string='', start=(1, 7), end=(1, 8), line='2sin(x)'), + tokenize.TokenInfo(type=token.ENDMARKER, string='', start=(2, 0), end=(2, 0), line='') + ] + + tokens = list(tokenize.generate_tokens(StringIO(source).readline)) + self.assertEqual(tokens, expected_tokens) + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 855 characters long. Set self.maxDiff to None to see it. + def test_number_starting_with_zero(self): + source = "01234" + expected_tokens = [ + tokenize.TokenInfo(type=token.NUMBER, string='01234', start=(1, 0), end=(1, 5), line='01234'), + tokenize.TokenInfo(type=token.NEWLINE, string='', start=(1, 5), end=(1, 6), line='01234'), + tokenize.TokenInfo(type=token.ENDMARKER, string='', start=(2, 0), end=(2, 0), line='') + ] + + tokens = list(tokenize.generate_tokens(StringIO(source).readline)) + self.assertEqual(tokens, expected_tokens) + +class CTokenizeTest(TestCase): + def check_tokenize(self, s, expected): + # Format the tokens in s in a table format. + # The ENDMARKER and final NEWLINE are omitted. + f = StringIO(s) + with self.subTest(source=s): + result = stringify_tokens_from_source( + tokenize._generate_tokens_from_c_tokenizer(f.readline), s + ) + self.assertEqual(result, expected.rstrip().splitlines()) + + def test_encoding(self): + def readline(encoding): + yield "1+1".encode(encoding) + + expected = [ + tokenize.TokenInfo(type=tokenize.NUMBER, string='1', start=(1, 0), end=(1, 1), line='1+1'), + tokenize.TokenInfo(type=tokenize.OP, string='+', start=(1, 1), end=(1, 2), line='1+1'), + tokenize.TokenInfo(type=tokenize.NUMBER, string='1', start=(1, 2), end=(1, 3), line='1+1'), + tokenize.TokenInfo(type=tokenize.NEWLINE, string='', start=(1, 3), end=(1, 4), line='1+1'), + tokenize.TokenInfo(type=tokenize.ENDMARKER, string='', start=(2, 0), end=(2, 0), line='') + ] + for encoding in ["utf-8", "latin-1", "utf-16"]: + with self.subTest(encoding=encoding): + tokens = list(tokenize._generate_tokens_from_c_tokenizer( + readline(encoding).__next__, + extra_tokens=True, + encoding=encoding, + )) + self.assertEqual(tokens, expected) + + def test_int(self): + + self.check_tokenize('0xff <= 255', """\ + NUMBER '0xff' (1, 0) (1, 4) + LESSEQUAL '<=' (1, 5) (1, 7) + NUMBER '255' (1, 8) (1, 11) + """) + + self.check_tokenize('0b10 <= 255', """\ + NUMBER '0b10' (1, 0) (1, 4) + LESSEQUAL '<=' (1, 5) (1, 7) + NUMBER '255' (1, 8) (1, 11) + """) + + self.check_tokenize('0o123 <= 0O123', """\ + NUMBER '0o123' (1, 0) (1, 5) + LESSEQUAL '<=' (1, 6) (1, 8) + NUMBER '0O123' (1, 9) (1, 14) + """) + + self.check_tokenize('1234567 > ~0x15', """\ + NUMBER '1234567' (1, 0) (1, 7) + GREATER '>' (1, 8) (1, 9) + TILDE '~' (1, 10) (1, 11) + NUMBER '0x15' (1, 11) (1, 15) + """) + + self.check_tokenize('2134568 != 1231515', """\ + NUMBER '2134568' (1, 0) (1, 7) + NOTEQUAL '!=' (1, 8) (1, 10) + NUMBER '1231515' (1, 11) (1, 18) + """) + + self.check_tokenize('(-124561-1) & 200000000', """\ + LPAR '(' (1, 0) (1, 1) + MINUS '-' (1, 1) (1, 2) + NUMBER '124561' (1, 2) (1, 8) + MINUS '-' (1, 8) (1, 9) + NUMBER '1' (1, 9) (1, 10) + RPAR ')' (1, 10) (1, 11) + AMPER '&' (1, 12) (1, 13) + NUMBER '200000000' (1, 14) (1, 23) + """) + + self.check_tokenize('0xdeadbeef != -1', """\ + NUMBER '0xdeadbeef' (1, 0) (1, 10) + NOTEQUAL '!=' (1, 11) (1, 13) + MINUS '-' (1, 14) (1, 15) + NUMBER '1' (1, 15) (1, 16) + """) + + self.check_tokenize('0xdeadc0de & 12345', """\ + NUMBER '0xdeadc0de' (1, 0) (1, 10) + AMPER '&' (1, 11) (1, 12) + NUMBER '12345' (1, 13) (1, 18) + """) + + self.check_tokenize('0xFF & 0x15 | 1234', """\ + NUMBER '0xFF' (1, 0) (1, 4) + AMPER '&' (1, 5) (1, 6) + NUMBER '0x15' (1, 7) (1, 11) + VBAR '|' (1, 12) (1, 13) + NUMBER '1234' (1, 14) (1, 18) + """) + + def test_float(self): + + self.check_tokenize('x = 3.14159', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + NUMBER '3.14159' (1, 4) (1, 11) + """) + + self.check_tokenize('x = 314159.', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + NUMBER '314159.' (1, 4) (1, 11) + """) + + self.check_tokenize('x = .314159', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + NUMBER '.314159' (1, 4) (1, 11) + """) + + self.check_tokenize('x = 3e14159', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + NUMBER '3e14159' (1, 4) (1, 11) + """) + + self.check_tokenize('x = 3E123', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + NUMBER '3E123' (1, 4) (1, 9) + """) + + self.check_tokenize('x+y = 3e-1230', """\ + NAME 'x' (1, 0) (1, 1) + PLUS '+' (1, 1) (1, 2) + NAME 'y' (1, 2) (1, 3) + EQUAL '=' (1, 4) (1, 5) + NUMBER '3e-1230' (1, 6) (1, 13) + """) + + self.check_tokenize('x = 3.14e159', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + NUMBER '3.14e159' (1, 4) (1, 12) + """) + + def test_string(self): + + self.check_tokenize('x = \'\'; y = ""', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + STRING "''" (1, 4) (1, 6) + SEMI ';' (1, 6) (1, 7) + NAME 'y' (1, 8) (1, 9) + EQUAL '=' (1, 10) (1, 11) + STRING '""' (1, 12) (1, 14) + """) + + self.check_tokenize('x = \'"\'; y = "\'"', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + STRING '\\'"\\'' (1, 4) (1, 7) + SEMI ';' (1, 7) (1, 8) + NAME 'y' (1, 9) (1, 10) + EQUAL '=' (1, 11) (1, 12) + STRING '"\\'"' (1, 13) (1, 16) + """) + + self.check_tokenize('x = "doesn\'t "shrink", does it"', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + STRING '"doesn\\'t "' (1, 4) (1, 14) + NAME 'shrink' (1, 14) (1, 20) + STRING '", does it"' (1, 20) (1, 31) + """) + + self.check_tokenize("x = 'abc' + 'ABC'", """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + STRING "'abc'" (1, 4) (1, 9) + PLUS '+' (1, 10) (1, 11) + STRING "'ABC'" (1, 12) (1, 17) + """) + + self.check_tokenize('y = "ABC" + "ABC"', """\ + NAME 'y' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + STRING '"ABC"' (1, 4) (1, 9) + PLUS '+' (1, 10) (1, 11) + STRING '"ABC"' (1, 12) (1, 17) + """) + + self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + STRING "r'abc'" (1, 4) (1, 10) + PLUS '+' (1, 11) (1, 12) + STRING "r'ABC'" (1, 13) (1, 19) + PLUS '+' (1, 20) (1, 21) + STRING "R'ABC'" (1, 22) (1, 28) + PLUS '+' (1, 29) (1, 30) + STRING "R'ABC'" (1, 31) (1, 37) + """) + + self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\ + NAME 'y' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + STRING 'r"abc"' (1, 4) (1, 10) + PLUS '+' (1, 11) (1, 12) + STRING 'r"ABC"' (1, 13) (1, 19) + PLUS '+' (1, 20) (1, 21) + STRING 'R"ABC"' (1, 22) (1, 28) + PLUS '+' (1, 29) (1, 30) + STRING 'R"ABC"' (1, 31) (1, 37) + """) + + self.check_tokenize("u'abc' + U'abc'", """\ + STRING "u'abc'" (1, 0) (1, 6) + PLUS '+' (1, 7) (1, 8) + STRING "U'abc'" (1, 9) (1, 15) + """) + + self.check_tokenize('u"abc" + U"abc"', """\ + STRING 'u"abc"' (1, 0) (1, 6) + PLUS '+' (1, 7) (1, 8) + STRING 'U"abc"' (1, 9) (1, 15) + """) + + self.check_tokenize("b'abc' + B'abc'", """\ + STRING "b'abc'" (1, 0) (1, 6) + PLUS '+' (1, 7) (1, 8) + STRING "B'abc'" (1, 9) (1, 15) + """) + + self.check_tokenize('b"abc" + B"abc"', """\ + STRING 'b"abc"' (1, 0) (1, 6) + PLUS '+' (1, 7) (1, 8) + STRING 'B"abc"' (1, 9) (1, 15) + """) + + self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\ + STRING "br'abc'" (1, 0) (1, 7) + PLUS '+' (1, 8) (1, 9) + STRING "bR'abc'" (1, 10) (1, 17) + PLUS '+' (1, 18) (1, 19) + STRING "Br'abc'" (1, 20) (1, 27) + PLUS '+' (1, 28) (1, 29) + STRING "BR'abc'" (1, 30) (1, 37) + """) + + self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\ + STRING 'br"abc"' (1, 0) (1, 7) + PLUS '+' (1, 8) (1, 9) + STRING 'bR"abc"' (1, 10) (1, 17) + PLUS '+' (1, 18) (1, 19) + STRING 'Br"abc"' (1, 20) (1, 27) + PLUS '+' (1, 28) (1, 29) + STRING 'BR"abc"' (1, 30) (1, 37) + """) + + self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\ + STRING "rb'abc'" (1, 0) (1, 7) + PLUS '+' (1, 8) (1, 9) + STRING "rB'abc'" (1, 10) (1, 17) + PLUS '+' (1, 18) (1, 19) + STRING "Rb'abc'" (1, 20) (1, 27) + PLUS '+' (1, 28) (1, 29) + STRING "RB'abc'" (1, 30) (1, 37) + """) + + self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\ + STRING 'rb"abc"' (1, 0) (1, 7) + PLUS '+' (1, 8) (1, 9) + STRING 'rB"abc"' (1, 10) (1, 17) + PLUS '+' (1, 18) (1, 19) + STRING 'Rb"abc"' (1, 20) (1, 27) + PLUS '+' (1, 28) (1, 29) + STRING 'RB"abc"' (1, 30) (1, 37) + """) + + self.check_tokenize('"a\\\nde\\\nfg"', """\ + STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3) + """) + + self.check_tokenize('u"a\\\nde"', """\ + STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3) + """) + + self.check_tokenize('rb"a\\\nd"', """\ + STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2) + """) + + self.check_tokenize(r'"""a\ +b"""', """\ + STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'u"""a\ +b"""', """\ + STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'rb"""a\ +b\ +c"""', """\ + STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) + """) + + self.check_tokenize(r'"hola\\\r\ndfgf"', """\ + STRING \'"hola\\\\\\\\\\\\r\\\\ndfgf"\' (1, 0) (1, 16) + """) + + self.check_tokenize('f"abc"', """\ + FSTRING_START 'f"' (1, 0) (1, 2) + FSTRING_MIDDLE 'abc' (1, 2) (1, 5) + FSTRING_END '"' (1, 5) (1, 6) + """) + + self.check_tokenize('fR"a{b}c"', """\ + FSTRING_START 'fR"' (1, 0) (1, 3) + FSTRING_MIDDLE 'a' (1, 3) (1, 4) + LBRACE '{' (1, 4) (1, 5) + NAME 'b' (1, 5) (1, 6) + RBRACE '}' (1, 6) (1, 7) + FSTRING_MIDDLE 'c' (1, 7) (1, 8) + FSTRING_END '"' (1, 8) (1, 9) + """) + + self.check_tokenize('f"""abc"""', """\ + FSTRING_START 'f\"""' (1, 0) (1, 4) + FSTRING_MIDDLE 'abc' (1, 4) (1, 7) + FSTRING_END '\"""' (1, 7) (1, 10) + """) + + self.check_tokenize(r'f"abc\ +def"', """\ + FSTRING_START \'f"\' (1, 0) (1, 2) + FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 2) (2, 3) + FSTRING_END '"' (2, 3) (2, 4) + """) + + self.check_tokenize('''\ +f"{ +a}"''', """\ + FSTRING_START 'f"' (1, 0) (1, 2) + LBRACE '{' (1, 2) (1, 3) + NAME 'a' (2, 0) (2, 1) + RBRACE '}' (2, 1) (2, 2) + FSTRING_END '"' (2, 2) (2, 3) + """) + + self.check_tokenize(r'Rf"abc\ +def"', """\ + FSTRING_START 'Rf"' (1, 0) (1, 3) + FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 3) (2, 3) + FSTRING_END '"' (2, 3) (2, 4) + """) + + self.check_tokenize(r'f"hola\\\r\ndfgf"', """\ + FSTRING_START \'f"\' (1, 0) (1, 2) + FSTRING_MIDDLE 'hola\\\\\\\\\\\\r\\\\ndfgf' (1, 2) (1, 16) + FSTRING_END \'"\' (1, 16) (1, 17) + """) + + self.check_tokenize("""\ +f'''__{ + x:a +}__'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + FSTRING_MIDDLE '__' (1, 4) (1, 6) + LBRACE '{' (1, 6) (1, 7) + NAME 'x' (2, 4) (2, 5) + COLON ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'a\\n' (2, 6) (3, 0) + RBRACE '}' (3, 0) (3, 1) + FSTRING_MIDDLE '__' (3, 1) (3, 3) + FSTRING_END "'''" (3, 3) (3, 6) + """) + + self.check_tokenize("""\ +f'''__{ + x:a + b + c + d +}__'''""", """\ + FSTRING_START "f'''" (1, 0) (1, 4) + FSTRING_MIDDLE '__' (1, 4) (1, 6) + LBRACE '{' (1, 6) (1, 7) + NAME 'x' (2, 4) (2, 5) + COLON ':' (2, 5) (2, 6) + FSTRING_MIDDLE 'a\\n b\\n c\\n d\\n' (2, 6) (6, 0) + RBRACE '}' (6, 0) (6, 1) + FSTRING_MIDDLE '__' (6, 1) (6, 3) + FSTRING_END "'''" (6, 3) (6, 6) + """) + + def test_function(self): + + self.check_tokenize('def d22(a, b, c=2, d=2, *k): pass', """\ + NAME 'def' (1, 0) (1, 3) + NAME 'd22' (1, 4) (1, 7) + LPAR '(' (1, 7) (1, 8) + NAME 'a' (1, 8) (1, 9) + COMMA ',' (1, 9) (1, 10) + NAME 'b' (1, 11) (1, 12) + COMMA ',' (1, 12) (1, 13) + NAME 'c' (1, 14) (1, 15) + EQUAL '=' (1, 15) (1, 16) + NUMBER '2' (1, 16) (1, 17) + COMMA ',' (1, 17) (1, 18) + NAME 'd' (1, 19) (1, 20) + EQUAL '=' (1, 20) (1, 21) + NUMBER '2' (1, 21) (1, 22) + COMMA ',' (1, 22) (1, 23) + STAR '*' (1, 24) (1, 25) + NAME 'k' (1, 25) (1, 26) + RPAR ')' (1, 26) (1, 27) + COLON ':' (1, 27) (1, 28) + NAME 'pass' (1, 29) (1, 33) + """) + + self.check_tokenize('def d01v_(a=1, *k, **w): pass', """\ + NAME 'def' (1, 0) (1, 3) + NAME 'd01v_' (1, 4) (1, 9) + LPAR '(' (1, 9) (1, 10) + NAME 'a' (1, 10) (1, 11) + EQUAL '=' (1, 11) (1, 12) + NUMBER '1' (1, 12) (1, 13) + COMMA ',' (1, 13) (1, 14) + STAR '*' (1, 15) (1, 16) + NAME 'k' (1, 16) (1, 17) + COMMA ',' (1, 17) (1, 18) + DOUBLESTAR '**' (1, 19) (1, 21) + NAME 'w' (1, 21) (1, 22) + RPAR ')' (1, 22) (1, 23) + COLON ':' (1, 23) (1, 24) + NAME 'pass' (1, 25) (1, 29) + """) + + self.check_tokenize('def d23(a: str, b: int=3) -> int: pass', """\ + NAME 'def' (1, 0) (1, 3) + NAME 'd23' (1, 4) (1, 7) + LPAR '(' (1, 7) (1, 8) + NAME 'a' (1, 8) (1, 9) + COLON ':' (1, 9) (1, 10) + NAME 'str' (1, 11) (1, 14) + COMMA ',' (1, 14) (1, 15) + NAME 'b' (1, 16) (1, 17) + COLON ':' (1, 17) (1, 18) + NAME 'int' (1, 19) (1, 22) + EQUAL '=' (1, 22) (1, 23) + NUMBER '3' (1, 23) (1, 24) + RPAR ')' (1, 24) (1, 25) + RARROW '->' (1, 26) (1, 28) + NAME 'int' (1, 29) (1, 32) + COLON ':' (1, 32) (1, 33) + NAME 'pass' (1, 34) (1, 38) + """) + + def test_comparison(self): + + self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " + "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\ + NAME 'if' (1, 0) (1, 2) + NUMBER '1' (1, 3) (1, 4) + LESS '<' (1, 5) (1, 6) + NUMBER '1' (1, 7) (1, 8) + GREATER '>' (1, 9) (1, 10) + NUMBER '1' (1, 11) (1, 12) + EQEQUAL '==' (1, 13) (1, 15) + NUMBER '1' (1, 16) (1, 17) + GREATEREQUAL '>=' (1, 18) (1, 20) + NUMBER '5' (1, 21) (1, 22) + LESSEQUAL '<=' (1, 23) (1, 25) + NUMBER '0x15' (1, 26) (1, 30) + LESSEQUAL '<=' (1, 31) (1, 33) + NUMBER '0x12' (1, 34) (1, 38) + NOTEQUAL '!=' (1, 39) (1, 41) + NUMBER '1' (1, 42) (1, 43) + NAME 'and' (1, 44) (1, 47) + NUMBER '5' (1, 48) (1, 49) + NAME 'in' (1, 50) (1, 52) + NUMBER '1' (1, 53) (1, 54) + NAME 'not' (1, 55) (1, 58) + NAME 'in' (1, 59) (1, 61) + NUMBER '1' (1, 62) (1, 63) + NAME 'is' (1, 64) (1, 66) + NUMBER '1' (1, 67) (1, 68) + NAME 'or' (1, 69) (1, 71) + NUMBER '5' (1, 72) (1, 73) + NAME 'is' (1, 74) (1, 76) + NAME 'not' (1, 77) (1, 80) + NUMBER '1' (1, 81) (1, 82) + COLON ':' (1, 82) (1, 83) + NAME 'pass' (1, 84) (1, 88) + """) + + def test_additive(self): + + self.check_tokenize('x = 1 - y + 15 - 1 + 0x124 + z + a[5]', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + NUMBER '1' (1, 4) (1, 5) + MINUS '-' (1, 6) (1, 7) + NAME 'y' (1, 8) (1, 9) + PLUS '+' (1, 10) (1, 11) + NUMBER '15' (1, 12) (1, 14) + MINUS '-' (1, 15) (1, 16) + NUMBER '1' (1, 17) (1, 18) + PLUS '+' (1, 19) (1, 20) + NUMBER '0x124' (1, 21) (1, 26) + PLUS '+' (1, 27) (1, 28) + NAME 'z' (1, 29) (1, 30) + PLUS '+' (1, 31) (1, 32) + NAME 'a' (1, 33) (1, 34) + LSQB '[' (1, 34) (1, 35) + NUMBER '5' (1, 35) (1, 36) + RSQB ']' (1, 36) (1, 37) + """) + + def test_multiplicative(self): + + self.check_tokenize('x = 1//1*1/5*12%0x12@42', """\ + NAME 'x' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + NUMBER '1' (1, 4) (1, 5) + DOUBLESLASH '//' (1, 5) (1, 7) + NUMBER '1' (1, 7) (1, 8) + STAR '*' (1, 8) (1, 9) + NUMBER '1' (1, 9) (1, 10) + SLASH '/' (1, 10) (1, 11) + NUMBER '5' (1, 11) (1, 12) + STAR '*' (1, 12) (1, 13) + NUMBER '12' (1, 13) (1, 15) + PERCENT '%' (1, 15) (1, 16) + NUMBER '0x12' (1, 16) (1, 20) + AT '@' (1, 20) (1, 21) + NUMBER '42' (1, 21) (1, 23) + """) + + def test_unary(self): + + self.check_tokenize('~1 ^ 1 & 1 |1 ^ -1', """\ + TILDE '~' (1, 0) (1, 1) + NUMBER '1' (1, 1) (1, 2) + CIRCUMFLEX '^' (1, 3) (1, 4) + NUMBER '1' (1, 5) (1, 6) + AMPER '&' (1, 7) (1, 8) + NUMBER '1' (1, 9) (1, 10) + VBAR '|' (1, 11) (1, 12) + NUMBER '1' (1, 12) (1, 13) + CIRCUMFLEX '^' (1, 14) (1, 15) + MINUS '-' (1, 16) (1, 17) + NUMBER '1' (1, 17) (1, 18) + """) + + self.check_tokenize('-1*1/1+1*1//1 - ---1**1', """\ + MINUS '-' (1, 0) (1, 1) + NUMBER '1' (1, 1) (1, 2) + STAR '*' (1, 2) (1, 3) + NUMBER '1' (1, 3) (1, 4) + SLASH '/' (1, 4) (1, 5) + NUMBER '1' (1, 5) (1, 6) + PLUS '+' (1, 6) (1, 7) + NUMBER '1' (1, 7) (1, 8) + STAR '*' (1, 8) (1, 9) + NUMBER '1' (1, 9) (1, 10) + DOUBLESLASH '//' (1, 10) (1, 12) + NUMBER '1' (1, 12) (1, 13) + MINUS '-' (1, 14) (1, 15) + MINUS '-' (1, 16) (1, 17) + MINUS '-' (1, 17) (1, 18) + MINUS '-' (1, 18) (1, 19) + NUMBER '1' (1, 19) (1, 20) + DOUBLESTAR '**' (1, 20) (1, 22) + NUMBER '1' (1, 22) (1, 23) + """) + + def test_selector(self): + + self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\ + NAME 'import' (1, 0) (1, 6) + NAME 'sys' (1, 7) (1, 10) + COMMA ',' (1, 10) (1, 11) + NAME 'time' (1, 12) (1, 16) + NEWLINE '' (1, 16) (1, 16) + NAME 'x' (2, 0) (2, 1) + EQUAL '=' (2, 2) (2, 3) + NAME 'sys' (2, 4) (2, 7) + DOT '.' (2, 7) (2, 8) + NAME 'modules' (2, 8) (2, 15) + LSQB '[' (2, 15) (2, 16) + STRING "'time'" (2, 16) (2, 22) + RSQB ']' (2, 22) (2, 23) + DOT '.' (2, 23) (2, 24) + NAME 'time' (2, 24) (2, 28) + LPAR '(' (2, 28) (2, 29) + RPAR ')' (2, 29) (2, 30) + """) + + def test_method(self): + + self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\ + AT '@' (1, 0) (1, 1) + NAME 'staticmethod' (1, 1) (1, 13) + NEWLINE '' (1, 13) (1, 13) + NAME 'def' (2, 0) (2, 3) + NAME 'foo' (2, 4) (2, 7) + LPAR '(' (2, 7) (2, 8) + NAME 'x' (2, 8) (2, 9) + COMMA ',' (2, 9) (2, 10) + NAME 'y' (2, 10) (2, 11) + RPAR ')' (2, 11) (2, 12) + COLON ':' (2, 12) (2, 13) + NAME 'pass' (2, 14) (2, 18) + """) + + def test_tabs(self): + + self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\ + AT '@' (1, 0) (1, 1) + NAME 'staticmethod' (1, 1) (1, 13) + NEWLINE '' (1, 13) (1, 13) + NAME 'def' (2, 0) (2, 3) + NAME 'foo' (2, 4) (2, 7) + LPAR '(' (2, 7) (2, 8) + NAME 'x' (2, 8) (2, 9) + COMMA ',' (2, 9) (2, 10) + NAME 'y' (2, 10) (2, 11) + RPAR ')' (2, 11) (2, 12) + COLON ':' (2, 12) (2, 13) + NAME 'pass' (2, 14) (2, 18) + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_async(self): + + self.check_tokenize('async = 1', """\ + NAME 'async' (1, 0) (1, 5) + EQUAL '=' (1, 6) (1, 7) + NUMBER '1' (1, 8) (1, 9) + """) + + self.check_tokenize('a = (async = 1)', """\ + NAME 'a' (1, 0) (1, 1) + EQUAL '=' (1, 2) (1, 3) + LPAR '(' (1, 4) (1, 5) + NAME 'async' (1, 5) (1, 10) + EQUAL '=' (1, 11) (1, 12) + NUMBER '1' (1, 13) (1, 14) + RPAR ')' (1, 14) (1, 15) + """) + + self.check_tokenize('async()', """\ + NAME 'async' (1, 0) (1, 5) + LPAR '(' (1, 5) (1, 6) + RPAR ')' (1, 6) (1, 7) + """) + + self.check_tokenize('class async(Bar):pass', """\ + NAME 'class' (1, 0) (1, 5) + NAME 'async' (1, 6) (1, 11) + LPAR '(' (1, 11) (1, 12) + NAME 'Bar' (1, 12) (1, 15) + RPAR ')' (1, 15) (1, 16) + COLON ':' (1, 16) (1, 17) + NAME 'pass' (1, 17) (1, 21) + """) + + self.check_tokenize('class async:pass', """\ + NAME 'class' (1, 0) (1, 5) + NAME 'async' (1, 6) (1, 11) + COLON ':' (1, 11) (1, 12) + NAME 'pass' (1, 12) (1, 16) + """) + + self.check_tokenize('await = 1', """\ + NAME 'await' (1, 0) (1, 5) + EQUAL '=' (1, 6) (1, 7) + NUMBER '1' (1, 8) (1, 9) + """) + + self.check_tokenize('foo.async', """\ + NAME 'foo' (1, 0) (1, 3) + DOT '.' (1, 3) (1, 4) + NAME 'async' (1, 4) (1, 9) + """) + + self.check_tokenize('async for a in b: pass', """\ + NAME 'async' (1, 0) (1, 5) + NAME 'for' (1, 6) (1, 9) + NAME 'a' (1, 10) (1, 11) + NAME 'in' (1, 12) (1, 14) + NAME 'b' (1, 15) (1, 16) + COLON ':' (1, 16) (1, 17) + NAME 'pass' (1, 18) (1, 22) + """) + + self.check_tokenize('async with a as b: pass', """\ + NAME 'async' (1, 0) (1, 5) + NAME 'with' (1, 6) (1, 10) + NAME 'a' (1, 11) (1, 12) + NAME 'as' (1, 13) (1, 15) + NAME 'b' (1, 16) (1, 17) + COLON ':' (1, 17) (1, 18) + NAME 'pass' (1, 19) (1, 23) + """) + + self.check_tokenize('async.foo', """\ + NAME 'async' (1, 0) (1, 5) + DOT '.' (1, 5) (1, 6) + NAME 'foo' (1, 6) (1, 9) + """) + + self.check_tokenize('async', """\ + NAME 'async' (1, 0) (1, 5) + """) + + self.check_tokenize('async\n#comment\nawait', """\ + NAME 'async' (1, 0) (1, 5) + NEWLINE '' (1, 5) (1, 5) + NAME 'await' (3, 0) (3, 5) + """) + + self.check_tokenize('async\n...\nawait', """\ + NAME 'async' (1, 0) (1, 5) + NEWLINE '' (1, 5) (1, 5) + ELLIPSIS '...' (2, 0) (2, 3) + NEWLINE '' (2, 3) (2, 3) + NAME 'await' (3, 0) (3, 5) + """) + + self.check_tokenize('async\nawait', """\ + NAME 'async' (1, 0) (1, 5) + NEWLINE '' (1, 5) (1, 5) + NAME 'await' (2, 0) (2, 5) + """) + + self.check_tokenize('foo.async + 1', """\ + NAME 'foo' (1, 0) (1, 3) + DOT '.' (1, 3) (1, 4) + NAME 'async' (1, 4) (1, 9) + PLUS '+' (1, 10) (1, 11) + NUMBER '1' (1, 12) (1, 13) + """) + + self.check_tokenize('async def foo(): pass', """\ + NAME 'async' (1, 0) (1, 5) + NAME 'def' (1, 6) (1, 9) + NAME 'foo' (1, 10) (1, 13) + LPAR '(' (1, 13) (1, 14) + RPAR ')' (1, 14) (1, 15) + COLON ':' (1, 15) (1, 16) + NAME 'pass' (1, 17) (1, 21) + """) + + self.check_tokenize('''\ +async def foo(): + def foo(await): + await = 1 + if 1: + await +async += 1 +''', """\ + NAME 'async' (1, 0) (1, 5) + NAME 'def' (1, 6) (1, 9) + NAME 'foo' (1, 10) (1, 13) + LPAR '(' (1, 13) (1, 14) + RPAR ')' (1, 14) (1, 15) + COLON ':' (1, 15) (1, 16) + NEWLINE '' (1, 16) (1, 16) + INDENT '' (2, -1) (2, -1) + NAME 'def' (2, 2) (2, 5) + NAME 'foo' (2, 6) (2, 9) + LPAR '(' (2, 9) (2, 10) + NAME 'await' (2, 10) (2, 15) + RPAR ')' (2, 15) (2, 16) + COLON ':' (2, 16) (2, 17) + NEWLINE '' (2, 17) (2, 17) + INDENT '' (3, -1) (3, -1) + NAME 'await' (3, 4) (3, 9) + EQUAL '=' (3, 10) (3, 11) + NUMBER '1' (3, 12) (3, 13) + NEWLINE '' (3, 13) (3, 13) + DEDENT '' (4, -1) (4, -1) + NAME 'if' (4, 2) (4, 4) + NUMBER '1' (4, 5) (4, 6) + COLON ':' (4, 6) (4, 7) + NEWLINE '' (4, 7) (4, 7) + INDENT '' (5, -1) (5, -1) + NAME 'await' (5, 4) (5, 9) + NEWLINE '' (5, 9) (5, 9) + DEDENT '' (6, -1) (6, -1) + DEDENT '' (6, -1) (6, -1) + NAME 'async' (6, 0) (6, 5) + PLUSEQUAL '+=' (6, 6) (6, 8) + NUMBER '1' (6, 9) (6, 10) + NEWLINE '' (6, 10) (6, 10) + """) + + self.check_tokenize('async def foo():\n async for i in 1: pass', """\ + NAME 'async' (1, 0) (1, 5) + NAME 'def' (1, 6) (1, 9) + NAME 'foo' (1, 10) (1, 13) + LPAR '(' (1, 13) (1, 14) + RPAR ')' (1, 14) (1, 15) + COLON ':' (1, 15) (1, 16) + NEWLINE '' (1, 16) (1, 16) + INDENT '' (2, -1) (2, -1) + NAME 'async' (2, 2) (2, 7) + NAME 'for' (2, 8) (2, 11) + NAME 'i' (2, 12) (2, 13) + NAME 'in' (2, 14) (2, 16) + NUMBER '1' (2, 17) (2, 18) + COLON ':' (2, 18) (2, 19) + NAME 'pass' (2, 20) (2, 24) + DEDENT '' (2, -1) (2, -1) + """) + + self.check_tokenize('async def foo(async): await', """\ + NAME 'async' (1, 0) (1, 5) + NAME 'def' (1, 6) (1, 9) + NAME 'foo' (1, 10) (1, 13) + LPAR '(' (1, 13) (1, 14) + NAME 'async' (1, 14) (1, 19) + RPAR ')' (1, 19) (1, 20) + COLON ':' (1, 20) (1, 21) + NAME 'await' (1, 22) (1, 27) + """) + + self.check_tokenize('''\ +def f(): + + def baz(): pass + async def bar(): pass + + await = 2''', """\ + NAME 'def' (1, 0) (1, 3) + NAME 'f' (1, 4) (1, 5) + LPAR '(' (1, 5) (1, 6) + RPAR ')' (1, 6) (1, 7) + COLON ':' (1, 7) (1, 8) + NEWLINE '' (1, 8) (1, 8) + INDENT '' (3, -1) (3, -1) + NAME 'def' (3, 2) (3, 5) + NAME 'baz' (3, 6) (3, 9) + LPAR '(' (3, 9) (3, 10) + RPAR ')' (3, 10) (3, 11) + COLON ':' (3, 11) (3, 12) + NAME 'pass' (3, 13) (3, 17) + NEWLINE '' (3, 17) (3, 17) + NAME 'async' (4, 2) (4, 7) + NAME 'def' (4, 8) (4, 11) + NAME 'bar' (4, 12) (4, 15) + LPAR '(' (4, 15) (4, 16) + RPAR ')' (4, 16) (4, 17) + COLON ':' (4, 17) (4, 18) + NAME 'pass' (4, 19) (4, 23) + NEWLINE '' (4, 23) (4, 23) + NAME 'await' (6, 2) (6, 7) + EQUAL '=' (6, 8) (6, 9) + NUMBER '2' (6, 10) (6, 11) + DEDENT '' (6, -1) (6, -1) + """) + + self.check_tokenize('''\ +async def f(): + + def baz(): pass + async def bar(): pass + + await = 2''', """\ + NAME 'async' (1, 0) (1, 5) + NAME 'def' (1, 6) (1, 9) + NAME 'f' (1, 10) (1, 11) + LPAR '(' (1, 11) (1, 12) + RPAR ')' (1, 12) (1, 13) + COLON ':' (1, 13) (1, 14) + NEWLINE '' (1, 14) (1, 14) + INDENT '' (3, -1) (3, -1) + NAME 'def' (3, 2) (3, 5) + NAME 'baz' (3, 6) (3, 9) + LPAR '(' (3, 9) (3, 10) + RPAR ')' (3, 10) (3, 11) + COLON ':' (3, 11) (3, 12) + NAME 'pass' (3, 13) (3, 17) + NEWLINE '' (3, 17) (3, 17) + NAME 'async' (4, 2) (4, 7) + NAME 'def' (4, 8) (4, 11) + NAME 'bar' (4, 12) (4, 15) + LPAR '(' (4, 15) (4, 16) + RPAR ')' (4, 16) (4, 17) + COLON ':' (4, 17) (4, 18) + NAME 'pass' (4, 19) (4, 23) + NEWLINE '' (4, 23) (4, 23) + NAME 'await' (6, 2) (6, 7) + EQUAL '=' (6, 8) (6, 9) + NUMBER '2' (6, 10) (6, 11) + DEDENT '' (6, -1) (6, -1) + """) + + def test_unicode(self): + + self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\ + NAME 'Örter' (1, 0) (1, 5) + EQUAL '=' (1, 6) (1, 7) + STRING "u'places'" (1, 8) (1, 17) + NEWLINE '' (1, 17) (1, 17) + NAME 'grün' (2, 0) (2, 4) + EQUAL '=' (2, 5) (2, 6) + STRING "U'green'" (2, 7) (2, 15) + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_invalid_syntax(self): + def get_tokens(string): + the_string = StringIO(string) + return list(tokenize._generate_tokens_from_c_tokenizer(the_string.readline)) + + for case in [ + "(1+2]", + "(1+2}", + "{1+2]", + "1_", + "1.2_", + "1e2_", + "1e+", + + "\xa0", + "€", + "0b12", + "0b1_2", + "0b2", + "0b1_", + "0b", + "0o18", + "0o1_8", + "0o8", + "0o1_", + "0o", + "0x1_", + "0x", + "1_", + "012", + "1.2_", + "1e2_", + "1e+", + "'sdfsdf", + "'''sdfsdf''", + "("*1000+"a"+")"*1000, + "]", + """\ + f'__{ + x:d + }__'""", + " a\n\x00", + ]: + with self.subTest(case=case): + self.assertRaises(tokenize.TokenError, get_tokens, case) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: IndentationError not raised by <lambda> + @support.skip_wasi_stack_overflow() + def test_max_indent(self): + MAXINDENT = 100 + + def generate_source(indents): + source = ''.join((' ' * x) + 'if True:\n' for x in range(indents)) + source += ' ' * indents + 'pass\n' + return source + + valid = generate_source(MAXINDENT - 1) + the_input = StringIO(valid) + tokens = list(tokenize._generate_tokens_from_c_tokenizer(the_input.readline)) + self.assertEqual(tokens[-2].type, tokenize.DEDENT) + self.assertEqual(tokens[-1].type, tokenize.ENDMARKER) + compile(valid, "<string>", "exec") + + invalid = generate_source(MAXINDENT) + the_input = StringIO(invalid) + self.assertRaises(IndentationError, lambda: list(tokenize._generate_tokens_from_c_tokenizer(the_input.readline))) + self.assertRaises( + IndentationError, compile, invalid, "<string>", "exec" + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; (0, '')] + def test_continuation_lines_indentation(self): + def get_tokens(string): + the_string = StringIO(string) + return [(kind, string) for (kind, string, *_) + in tokenize._generate_tokens_from_c_tokenizer(the_string.readline)] + + code = dedent(""" + def fib(n): + \\ + '''Print a Fibonacci series up to n.''' + \\ + a, b = 0, 1 + """) + + self.check_tokenize(code, """\ + NAME 'def' (2, 0) (2, 3) + NAME 'fib' (2, 4) (2, 7) + LPAR '(' (2, 7) (2, 8) + NAME 'n' (2, 8) (2, 9) + RPAR ')' (2, 9) (2, 10) + COLON ':' (2, 10) (2, 11) + NEWLINE '' (2, 11) (2, 11) + INDENT '' (4, -1) (4, -1) + STRING "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39) + NEWLINE '' (4, 39) (4, 39) + NAME 'a' (6, 0) (6, 1) + COMMA ',' (6, 1) (6, 2) + NAME 'b' (6, 3) (6, 4) + EQUAL '=' (6, 5) (6, 6) + NUMBER '0' (6, 7) (6, 8) + COMMA ',' (6, 8) (6, 9) + NUMBER '1' (6, 10) (6, 11) + NEWLINE '' (6, 11) (6, 11) + DEDENT '' (6, -1) (6, -1) + """) + + code_no_cont = dedent(""" + def fib(n): + '''Print a Fibonacci series up to n.''' + a, b = 0, 1 + """) + + self.assertEqual(get_tokens(code), get_tokens(code_no_cont)) + + code = dedent(""" + pass + \\ + + pass + """) + + self.check_tokenize(code, """\ + NAME 'pass' (2, 0) (2, 4) + NEWLINE '' (2, 4) (2, 4) + NAME 'pass' (5, 0) (5, 4) + NEWLINE '' (5, 4) (5, 4) + """) + + code_no_cont = dedent(""" + pass + pass + """) + + self.assertEqual(get_tokens(code), get_tokens(code_no_cont)) + + code = dedent(""" + if x: + y = 1 + \\ + \\ + \\ + \\ + foo = 1 + """) + + self.check_tokenize(code, """\ + NAME 'if' (2, 0) (2, 2) + NAME 'x' (2, 3) (2, 4) + COLON ':' (2, 4) (2, 5) + NEWLINE '' (2, 5) (2, 5) + INDENT '' (3, -1) (3, -1) + NAME 'y' (3, 4) (3, 5) + EQUAL '=' (3, 6) (3, 7) + NUMBER '1' (3, 8) (3, 9) + NEWLINE '' (3, 9) (3, 9) + NAME 'foo' (8, 4) (8, 7) + EQUAL '=' (8, 8) (8, 9) + NUMBER '1' (8, 10) (8, 11) + NEWLINE '' (8, 11) (8, 11) + DEDENT '' (8, -1) (8, -1) + """) + + code_no_cont = dedent(""" + if x: + y = 1 + foo = 1 + """) + + self.assertEqual(get_tokens(code), get_tokens(code_no_cont)) + + +class CTokenizerBufferTests(unittest.TestCase): + def test_newline_at_the_end_of_buffer(self): + # See issue 99581: Make sure that if we need to add a new line at the + # end of the buffer, we have enough space in the buffer, specially when + # the current line is as long as the buffer space available. + test_script = f"""\ + #coding: latin-1 + #{"a"*10000} + #{"a"*10002}""" + with os_helper.temp_dir() as temp_dir: + file_name = make_script(temp_dir, 'foo', test_script) + run_test_script(file_name) + + +class CommandLineTest(unittest.TestCase): + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + """Dedent *string* and strip it from its surrounding whitespaces. + + This method is used by the other utility functions so that any + string to write or to match against can be freely indented. + """ + return re.sub(r'\s+', ' ', string).strip() + + def set_source(self, content): + with open(self.filename, 'w') as fp: + fp.write(content) + + def invoke_tokenize(self, *flags): + output = StringIO() + with contextlib.redirect_stdout(output): + tokenize._main(args=[*flags, self.filename]) + return self.text_normalize(output.getvalue()) + + def check_output(self, source, expect, *flags): + with self.subTest(source=source, flags=flags): + self.set_source(source) + res = self.invoke_tokenize(*flags) + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + def test_invocation(self): + # test various combinations of parameters + base_flags = ('-e', '--exact') + + self.set_source(''' + def f(): + print(x) + return None + ''') + + for flag in base_flags: + with self.subTest(args=flag): + _ = self.invoke_tokenize(flag) + + with self.assertRaises(SystemExit): + # suppress argparse error message + with contextlib.redirect_stderr(StringIO()): + _ = self.invoke_tokenize('--unknown') + + def test_without_flag(self): + # test 'python -m tokenize source.py' + source = 'a = 1' + expect = ''' + 0,0-0,0: ENCODING 'utf-8' + 1,0-1,1: NAME 'a' + 1,2-1,3: OP '=' + 1,4-1,5: NUMBER '1' + 1,5-1,6: NEWLINE '' + 2,0-2,0: ENDMARKER '' + ''' + self.check_output(source, expect) + + def test_exact_flag(self): + # test 'python -m tokenize -e/--exact source.py' + source = 'a = 1' + expect = ''' + 0,0-0,0: ENCODING 'utf-8' + 1,0-1,1: NAME 'a' + 1,2-1,3: EQUAL '=' + 1,4-1,5: NUMBER '1' + 1,5-1,6: NEWLINE '' + 2,0-2,0: ENDMARKER '' + ''' + for flag in ['-e', '--exact']: + self.check_output(source, expect, flag) + + +class StringPrefixTest(unittest.TestCase): + @staticmethod + def determine_valid_prefixes(): + # Try all lengths until we find a length that has zero valid + # prefixes. This will miss the case where for example there + # are no valid 3 character prefixes, but there are valid 4 + # character prefixes. That seems unlikely. + + single_char_valid_prefixes = set() + + # Find all of the single character string prefixes. Just get + # the lowercase version, we'll deal with combinations of upper + # and lower case later. I'm using this logic just in case + # some uppercase-only prefix is added. + for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase): + try: + eval(f'{letter}""') + single_char_valid_prefixes.add(letter.lower()) + except SyntaxError: + pass + + # This logic assumes that all combinations of valid prefixes only use + # the characters that are valid single character prefixes. That seems + # like a valid assumption, but if it ever changes this will need + # adjusting. + valid_prefixes = set() + for length in itertools.count(): + num_at_this_length = 0 + for prefix in ( + "".join(l) + for l in itertools.combinations(single_char_valid_prefixes, length) + ): + for t in itertools.permutations(prefix): + for u in itertools.product(*[(c, c.upper()) for c in t]): + p = "".join(u) + if p == "not": + # 'not' can never be a string prefix, + # because it's a valid expression: not "" + continue + try: + eval(f'{p}""') + + # No syntax error, so p is a valid string + # prefix. + + valid_prefixes.add(p) + num_at_this_length += 1 + except SyntaxError: + pass + if num_at_this_length == 0: + return valid_prefixes + + + def test_prefixes(self): + # Get the list of defined string prefixes. I don't see an + # obvious documented way of doing this, but probably the best + # thing is to split apart tokenize.StringPrefix. + + # Make sure StringPrefix begins and ends in parens. We're + # assuming it's of the form "(a|b|ab)", if a, b, and cd are + # valid string prefixes. + self.assertEqual(tokenize.StringPrefix[0], '(') + self.assertEqual(tokenize.StringPrefix[-1], ')') + + # Then split apart everything else by '|'. + defined_prefixes = set(tokenize.StringPrefix[1:-1].split('|')) + + # Now compute the actual allowed string prefixes and compare + # to what is defined in the tokenize module. + self.assertEqual(defined_prefixes, self.determine_valid_prefixes()) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tomllib/__main__.py b/Lib/test/test_tomllib/__main__.py index f309c7ec720..dd063653437 100644 --- a/Lib/test/test_tomllib/__main__.py +++ b/Lib/test/test_tomllib/__main__.py @@ -1,6 +1,6 @@ import unittest -from test.test_tomllib import load_tests +from . import load_tests unittest.main() diff --git a/Lib/test/test_tomllib/test_error.py b/Lib/test/test_tomllib/test_error.py index 72446267f04..3a858749285 100644 --- a/Lib/test/test_tomllib/test_error.py +++ b/Lib/test/test_tomllib/test_error.py @@ -39,8 +39,19 @@ def test_invalid_char_quotes(self): tomllib.loads("v = '\n'") self.assertTrue(" '\\n' " in str(exc_info.exception)) + def test_type_error(self): + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(b"v = 1") # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bytes'") + + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(False) # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bool'") + def test_module_name(self): - self.assertEqual(tomllib.TOMLDecodeError().__module__, tomllib.__name__) + self.assertEqual( + tomllib.TOMLDecodeError("", "", 0).__module__, tomllib.__name__ + ) def test_invalid_parse_float(self): def dict_returner(s: str) -> dict: @@ -55,3 +66,33 @@ def list_returner(s: str) -> list: self.assertEqual( str(exc_info.exception), "parse_float must not return dicts or lists" ) + + def test_deprecated_tomldecodeerror(self): + for args in [ + (), + ("err msg",), + (None,), + (None, "doc"), + ("err msg", None), + (None, "doc", None), + ("err msg", "doc", None), + ("one", "two", "three", "four"), + ("one", "two", 3, "four", "five"), + ]: + with self.assertWarns(DeprecationWarning): + e = tomllib.TOMLDecodeError(*args) # type: ignore[arg-type] + self.assertEqual(e.args, args) + + def test_tomldecodeerror(self): + msg = "error parsing" + doc = "v=1\n[table]\nv='val'" + pos = 13 + formatted_msg = "error parsing (at line 3, column 2)" + e = tomllib.TOMLDecodeError(msg, doc, pos) + self.assertEqual(e.args, (formatted_msg,)) + self.assertEqual(str(e), formatted_msg) + self.assertEqual(e.msg, msg) + self.assertEqual(e.doc, doc) + self.assertEqual(e.pos, pos) + self.assertEqual(e.lineno, 3) + self.assertEqual(e.colno, 2) diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index a477a219fd9..118fde24d88 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -5,10 +5,12 @@ import copy import datetime from decimal import Decimal as D +import importlib from pathlib import Path import sys import tempfile import unittest +from test import support from . import tomllib @@ -91,14 +93,34 @@ def test_deepcopy(self): } self.assertEqual(obj_copy, expected_obj) + @support.skip_if_unlimited_stack_size def test_inline_array_recursion_limit(self): - # 465 with default recursion limit - nest_count = int(sys.getrecursionlimit() * 0.465) - recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" - tomllib.loads(recursive_array_toml) + with support.infinite_recursion(max_depth=100): + available = support.get_recursion_available() + nest_count = (available // 2) - 2 + # Add details if the test fails + with self.subTest(limit=sys.getrecursionlimit(), + available=available, + nest_count=nest_count): + recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" + tomllib.loads(recursive_array_toml) + @support.skip_if_unlimited_stack_size def test_inline_table_recursion_limit(self): - # 310 with default recursion limit - nest_count = int(sys.getrecursionlimit() * 0.31) - recursive_table_toml = nest_count * "key = {" + nest_count * "}" - tomllib.loads(recursive_table_toml) + with support.infinite_recursion(max_depth=100): + available = support.get_recursion_available() + nest_count = (available // 3) - 1 + # Add details if the test fails + with self.subTest(limit=sys.getrecursionlimit(), + available=available, + nest_count=nest_count): + recursive_table_toml = nest_count * "key = {" + nest_count * "}" + tomllib.loads(recursive_table_toml) + + def test_types_import(self): + """Test that `_types` module runs. + + The module is for type annotations only, so it is otherwise + never imported by tests. + """ + importlib.import_module(f"{tomllib.__name__}._types") diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py new file mode 100644 index 00000000000..c4395c7c0ad --- /dev/null +++ b/Lib/test/test_tools/__init__.py @@ -0,0 +1,43 @@ +"""Support functions for testing scripts in the Tools directory.""" +import contextlib +import importlib +import os.path +import unittest +from test import support +from test.support import import_helper + + +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + +basepath = os.path.normpath( + os.path.dirname( # <src/install dir> + os.path.dirname( # Lib + os.path.dirname( # test + os.path.dirname(__file__))))) # test_tools + +toolsdir = os.path.join(basepath, 'Tools') +scriptsdir = os.path.join(toolsdir, 'scripts') + +def skip_if_missing(tool=None): + if tool: + tooldir = os.path.join(toolsdir, tool) + else: + tool = 'scripts' + tooldir = scriptsdir + if not os.path.isdir(tooldir): + raise unittest.SkipTest(f'{tool} directory could not be found') + +@contextlib.contextmanager +def imports_under_tool(name, *subdirs): + tooldir = os.path.join(toolsdir, name, *subdirs) + with import_helper.DirsOnSysPath(tooldir) as cm: + yield cm + +def import_tool(toolname): + with import_helper.DirsOnSysPath(scriptsdir): + return importlib.import_module(toolname) + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_tools/__main__.py b/Lib/test/test_tools/__main__.py new file mode 100644 index 00000000000..b6f13e534ed --- /dev/null +++ b/Lib/test/test_tools/__main__.py @@ -0,0 +1,4 @@ +from test.test_tools import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_tools/i18n_data/ascii-escapes.pot b/Lib/test/test_tools/i18n_data/ascii-escapes.pot new file mode 100644 index 00000000000..18d868b6a20 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/ascii-escapes.pot @@ -0,0 +1,45 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: escapes.py:5 +msgid "" +"\"\t\n" +"\r\\" +msgstr "" + +#: escapes.py:8 +msgid "" +"\000\001\002\003\004\005\006\007\010\t\n" +"\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" +msgstr "" + +#: escapes.py:13 +msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" +msgstr "" + +#: escapes.py:17 +msgid "\177" +msgstr "" + +#: escapes.py:20 +msgid "€   ÿ" +msgstr "" + +#: escapes.py:23 +msgid "α ㄱ 𓂀" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/docstrings.pot b/Lib/test/test_tools/i18n_data/docstrings.pot new file mode 100644 index 00000000000..5af1d41422f --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.pot @@ -0,0 +1,40 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: docstrings.py:7 +#, docstring +msgid "" +msgstr "" + +#: docstrings.py:18 +#, docstring +msgid "" +"multiline\n" +" docstring\n" +" " +msgstr "" + +#: docstrings.py:25 +#, docstring +msgid "docstring1" +msgstr "" + +#: docstrings.py:30 +#, docstring +msgid "Hello, {}!" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py new file mode 100644 index 00000000000..85d7f159d37 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.py @@ -0,0 +1,41 @@ +# Test docstring extraction +from gettext import gettext as _ + + +# Empty docstring +def test(x): + """""" + + +# Leading empty line +def test2(x): + + """docstring""" # XXX This should be extracted but isn't. + + +# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`. +def test3(x): + """multiline + docstring + """ + + +# Multiple docstrings - only the first should be extracted +def test4(x): + """docstring1""" + """docstring2""" + + +def test5(x): + """Hello, {}!""".format("world!") # XXX This should not be extracted. + + +# Nested docstrings +def test6(x): + def inner(y): + """nested docstring""" # XXX This should be extracted but isn't. + + +class Outer: + class Inner: + "nested class docstring" # XXX This should be extracted but isn't. diff --git a/Lib/test/test_tools/i18n_data/escapes.pot b/Lib/test/test_tools/i18n_data/escapes.pot new file mode 100644 index 00000000000..2c7899d59da --- /dev/null +++ b/Lib/test/test_tools/i18n_data/escapes.pot @@ -0,0 +1,45 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: escapes.py:5 +msgid "" +"\"\t\n" +"\r\\" +msgstr "" + +#: escapes.py:8 +msgid "" +"\000\001\002\003\004\005\006\007\010\t\n" +"\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" +msgstr "" + +#: escapes.py:13 +msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" +msgstr "" + +#: escapes.py:17 +msgid "\177" +msgstr "" + +#: escapes.py:20 +msgid "\302\200 \302\240 \303\277" +msgstr "" + +#: escapes.py:23 +msgid "\316\261 \343\204\261 \360\223\202\200" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/escapes.py b/Lib/test/test_tools/i18n_data/escapes.py new file mode 100644 index 00000000000..900bd97a70f --- /dev/null +++ b/Lib/test/test_tools/i18n_data/escapes.py @@ -0,0 +1,23 @@ +import gettext as _ + + +# Special characters that are always escaped in the POT file +_('"\t\n\r\\') + +# All ascii characters 0-31 +_('\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n' + '\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15' + '\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f') + +# All ascii characters 32-126 +_(' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ' + '[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~') + +# ascii char 127 +_('\x7f') + +# some characters in the 128-255 range +_('\x80 \xa0 ÿ') + +# some characters >= 256 encoded as 2, 3 and 4 bytes, respectively +_('α ㄱ 𓂀') diff --git a/Lib/test/test_tools/i18n_data/fileloc.pot b/Lib/test/test_tools/i18n_data/fileloc.pot new file mode 100644 index 00000000000..dbd28687a73 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.pot @@ -0,0 +1,35 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: fileloc.py:5 fileloc.py:6 +msgid "foo" +msgstr "" + +#: fileloc.py:9 +msgid "bar" +msgstr "" + +#: fileloc.py:14 fileloc.py:18 +#, docstring +msgid "docstring" +msgstr "" + +#: fileloc.py:22 fileloc.py:26 +#, docstring +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/fileloc.py b/Lib/test/test_tools/i18n_data/fileloc.py new file mode 100644 index 00000000000..c5d4d0595fe --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.py @@ -0,0 +1,26 @@ +# Test file locations +from gettext import gettext as _ + +# Duplicate strings +_('foo') +_('foo') + +# Duplicate strings on the same line should only add one location to the output +_('bar'), _('bar') + + +# Duplicate docstrings +class A: + """docstring""" + + +def f(): + """docstring""" + + +# Duplicate message and docstring +_('baz') + + +def g(): + """baz""" diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot new file mode 100644 index 00000000000..ddfbd18349e --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.pot @@ -0,0 +1,67 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: messages.py:5 +msgid "" +msgstr "" + +#: messages.py:8 messages.py:9 +msgid "parentheses" +msgstr "" + +#: messages.py:12 +msgid "Hello, world!" +msgstr "" + +#: messages.py:15 +msgid "" +"Hello,\n" +" multiline!\n" +msgstr "" + +#: messages.py:29 +msgid "Hello, {}!" +msgstr "" + +#: messages.py:33 +msgid "1" +msgstr "" + +#: messages.py:33 +msgid "2" +msgstr "" + +#: messages.py:34 messages.py:35 +msgid "A" +msgstr "" + +#: messages.py:34 messages.py:35 +msgid "B" +msgstr "" + +#: messages.py:36 +msgid "set" +msgstr "" + +#: messages.py:42 +msgid "nested string" +msgstr "" + +#: messages.py:47 +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py new file mode 100644 index 00000000000..f220294b8d5 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.py @@ -0,0 +1,64 @@ +# Test message extraction +from gettext import gettext as _ + +# Empty string +_("") + +# Extra parentheses +(_("parentheses")) +((_("parentheses"))) + +# Multiline strings +_("Hello, " + "world!") + +_("""Hello, + multiline! +""") + +# Invalid arguments +_() +_(None) +_(1) +_(False) +_(x="kwargs are not allowed") +_("foo", "bar") +_("something", x="something else") + +# .format() +_("Hello, {}!").format("world") # valid +_("Hello, {}!".format("world")) # invalid + +# Nested structures +_("1"), _("2") +arr = [_("A"), _("B")] +obj = {'a': _("A"), 'b': _("B")} +{{{_('set')}}} + + +# Nested functions and classes +def test(): + _("nested string") # XXX This should be extracted but isn't. + [_("nested string")] + + +class Foo: + def bar(self): + return _("baz") + + +def bar(x=_('default value')): # XXX This should be extracted but isn't. + pass + + +def baz(x=[_('default value')]): # XXX This should be extracted but isn't. + pass + + +# Shadowing _() +def _(x): + pass + + +def _(x="don't extract me"): + pass diff --git a/Lib/test/test_tools/msgfmt_data/fuzzy.json b/Lib/test/test_tools/msgfmt_data/fuzzy.json new file mode 100644 index 00000000000..fe51488c706 --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/fuzzy.json @@ -0,0 +1 @@ +[] diff --git a/Lib/test/test_tools/msgfmt_data/fuzzy.mo b/Lib/test/test_tools/msgfmt_data/fuzzy.mo new file mode 100644 index 00000000000..4b144831cf5 Binary files /dev/null and b/Lib/test/test_tools/msgfmt_data/fuzzy.mo differ diff --git a/Lib/test/test_tools/msgfmt_data/fuzzy.po b/Lib/test/test_tools/msgfmt_data/fuzzy.po new file mode 100644 index 00000000000..05e8354948a --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/fuzzy.po @@ -0,0 +1,23 @@ +# Fuzzy translations are not written to the .mo file. +#, fuzzy +msgid "foo" +msgstr "bar" + +# comment +#, fuzzy +msgctxt "abc" +msgid "foo" +msgstr "bar" + +#, fuzzy +# comment +msgctxt "xyz" +msgid "foo" +msgstr "bar" + +#, fuzzy +msgctxt "abc" +msgid "One email sent." +msgid_plural "%d emails sent." +msgstr[0] "One email sent." +msgstr[1] "%d emails sent." diff --git a/Lib/test/test_tools/msgfmt_data/general.json b/Lib/test/test_tools/msgfmt_data/general.json new file mode 100644 index 00000000000..0586113985a --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/general.json @@ -0,0 +1,58 @@ +[ + [ + "", + "Project-Id-Version: PACKAGE VERSION\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME <EMAIL@ADDRESS>\nLanguage-Team: LANGUAGE <LL@li.org>\nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\n" + ], + [ + "\n newlines \n", + "\n translated \n" + ], + [ + "\"escapes\"", + "\"translated\"" + ], + [ + "Multilinestring", + "Multilinetranslation" + ], + [ + "abc\u0004foo", + "bar" + ], + [ + "bar", + "baz" + ], + [ + "xyz\u0004foo", + "bar" + ], + [ + [ + "One email sent.", + 0 + ], + "One email sent." + ], + [ + [ + "One email sent.", + 1 + ], + "%d emails sent." + ], + [ + [ + "abc\u0004One email sent.", + 0 + ], + "One email sent." + ], + [ + [ + "abc\u0004One email sent.", + 1 + ], + "%d emails sent." + ] +] diff --git a/Lib/test/test_tools/msgfmt_data/general.mo b/Lib/test/test_tools/msgfmt_data/general.mo new file mode 100644 index 00000000000..ee905cbb3ec Binary files /dev/null and b/Lib/test/test_tools/msgfmt_data/general.mo differ diff --git a/Lib/test/test_tools/msgfmt_data/general.po b/Lib/test/test_tools/msgfmt_data/general.po new file mode 100644 index 00000000000..8f840426824 --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/general.po @@ -0,0 +1,47 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2024-10-26 18:06+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" +"Language-Team: LANGUAGE <LL@li.org>\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "foo" +msgstr "" + +msgid "bar" +msgstr "baz" + +msgctxt "abc" +msgid "foo" +msgstr "bar" + +# comment +msgctxt "xyz" +msgid "foo" +msgstr "bar" + +msgid "Multiline" +"string" +msgstr "Multiline" +"translation" + +msgid "\"escapes\"" +msgstr "\"translated\"" + +msgid "\n newlines \n" +msgstr "\n translated \n" + +msgid "One email sent." +msgid_plural "%d emails sent." +msgstr[0] "One email sent." +msgstr[1] "%d emails sent." + +msgctxt "abc" +msgid "One email sent." +msgid_plural "%d emails sent." +msgstr[0] "One email sent." +msgstr[1] "%d emails sent." diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py new file mode 100644 index 00000000000..0e7ed67de71 --- /dev/null +++ b/Lib/test/test_tools/test_freeze.py @@ -0,0 +1,37 @@ +"""Sanity-check tests for the "freeze" tool.""" + +import sys +import textwrap +import unittest + +from test import support +from test.support import os_helper +from test.test_tools import imports_under_tool, skip_if_missing + +skip_if_missing('freeze') +with imports_under_tool('freeze', 'test'): + import freeze as helper + +@support.requires_zlib() +@unittest.skipIf(sys.platform.startswith('win'), 'not supported on Windows') +@unittest.skipIf(sys.platform == 'darwin' and sys._framework, + 'not supported for frameworks builds on macOS') +@support.skip_if_buildbot('not all buildbots have enough space') +# gh-103053: Skip test if Python is built with Profile Guided Optimization +# (PGO), since the test is just too slow in this case. +@unittest.skipIf(support.check_cflags_pgo(), + 'test is too slow with PGO') +class TestFreeze(unittest.TestCase): + + @support.requires_resource('cpu') # Building Python is slow + def test_freeze_simple_script(self): + script = textwrap.dedent(""" + import sys + print('running...') + sys.exit(0) + """) + with os_helper.temp_dir() as outdir: + outdir, scriptfile, python = helper.prepare(script, outdir) + executable = helper.freeze(python, scriptfile, outdir) + text = helper.run(executable) + self.assertEqual(text, 'running...') diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py new file mode 100644 index 00000000000..ffa1b1178ed --- /dev/null +++ b/Lib/test/test_tools/test_i18n.py @@ -0,0 +1,444 @@ +"""Tests to cover the Tools/i18n package""" + +import os +import re +import sys +import unittest +from textwrap import dedent +from pathlib import Path + +from test.support.script_helper import assert_python_ok +from test.test_tools import skip_if_missing, toolsdir +from test.support.os_helper import temp_cwd, temp_dir + + +skip_if_missing() + +DATA_DIR = Path(__file__).resolve().parent / 'i18n_data' + + +def normalize_POT_file(pot): + """Normalize the POT creation timestamp, charset and + file locations to make the POT file easier to compare. + + """ + # Normalize the creation date. + date_pattern = re.compile(r'"POT-Creation-Date: .+?\\n"') + header = r'"POT-Creation-Date: 2000-01-01 00:00+0000\\n"' + pot = re.sub(date_pattern, header, pot) + + # Normalize charset to UTF-8 (currently there's no way to specify the output charset). + charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\\n"') + charset = r'"Content-Type: text/plain; charset=UTF-8\\n"' + pot = re.sub(charset_pattern, charset, pot) + + # Normalize file location path separators in case this test is + # running on Windows (which uses '\'). + fileloc_pattern = re.compile(r'#:.+') + + def replace(match): + return match[0].replace(os.sep, "/") + pot = re.sub(fileloc_pattern, replace, pot) + return pot + + +class Test_pygettext(unittest.TestCase): + """Tests for the pygettext.py tool""" + + script = Path(toolsdir, 'i18n', 'pygettext.py') + + def get_header(self, data): + """ utility: return the header of a .po file as a dictionary """ + headers = {} + for line in data.split('\n'): + if not line or line.startswith(('#', 'msgid', 'msgstr')): + continue + line = line.strip('"') + key, val = line.split(':', 1) + headers[key] = val.strip() + return headers + + def get_msgids(self, data): + """ utility: return all msgids in .po file as a list of strings """ + msgids = [] + reading_msgid = False + cur_msgid = [] + for line in data.split('\n'): + if reading_msgid: + if line.startswith('"'): + cur_msgid.append(line.strip('"')) + else: + msgids.append('\n'.join(cur_msgid)) + cur_msgid = [] + reading_msgid = False + continue + if line.startswith('msgid '): + line = line[len('msgid '):] + cur_msgid.append(line.strip('"')) + reading_msgid = True + else: + if reading_msgid: + msgids.append('\n'.join(cur_msgid)) + + return msgids + + def assert_POT_equal(self, expected, actual): + """Check if two POT files are equal""" + self.maxDiff = None + self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual)) + + def extract_from_str(self, module_content, *, args=(), strict=True): + """Return all msgids extracted from module_content.""" + filename = 'test.py' + with temp_cwd(None): + with open(filename, 'w', encoding='utf-8') as fp: + fp.write(module_content) + res = assert_python_ok('-Xutf8', self.script, *args, filename) + if strict: + self.assertEqual(res.err, b'') + with open('messages.pot', encoding='utf-8') as fp: + data = fp.read() + return self.get_msgids(data) + + def extract_docstrings_from_str(self, module_content): + """Return all docstrings extracted from module_content.""" + return self.extract_from_str(module_content, args=('--docstrings',), strict=False) + + def test_header(self): + """Make sure the required fields are in the header, according to: + http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry + """ + with temp_cwd(None) as cwd: + assert_python_ok('-Xutf8', self.script) + with open('messages.pot', encoding='utf-8') as fp: + data = fp.read() + header = self.get_header(data) + + self.assertIn("Project-Id-Version", header) + self.assertIn("POT-Creation-Date", header) + self.assertIn("PO-Revision-Date", header) + self.assertIn("Last-Translator", header) + self.assertIn("Language-Team", header) + self.assertIn("MIME-Version", header) + self.assertIn("Content-Type", header) + self.assertIn("Content-Transfer-Encoding", header) + self.assertIn("Generated-By", header) + + # not clear if these should be required in POT (template) files + #self.assertIn("Report-Msgid-Bugs-To", header) + #self.assertIn("Language", header) + + #"Plural-Forms" is optional + + @unittest.skipIf(sys.platform.startswith('aix'), + 'bpo-29972: broken test on AIX') + def test_POT_Creation_Date(self): + """ Match the date format from xgettext for POT-Creation-Date """ + from datetime import datetime + with temp_cwd(None) as cwd: + assert_python_ok('-Xutf8', self.script) + with open('messages.pot', encoding='utf-8') as fp: + data = fp.read() + header = self.get_header(data) + creationDate = header['POT-Creation-Date'] + + # peel off the escaped newline at the end of string + if creationDate.endswith('\\n'): + creationDate = creationDate[:-len('\\n')] + + # This will raise if the date format does not exactly match. + datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z') + + def test_funcdocstring(self): + for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): + with self.subTest(doc): + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar): + %s + ''' % doc)) + self.assertIn('doc', msgids) + + def test_funcdocstring_bytes(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar): + b"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_funcdocstring_fstring(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar): + f"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_classdocstring(self): + for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): + with self.subTest(doc): + msgids = self.extract_docstrings_from_str(dedent('''\ + class C: + %s + ''' % doc)) + self.assertIn('doc', msgids) + + def test_classdocstring_bytes(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + class C: + b"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_classdocstring_fstring(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + class C: + f"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_moduledocstring(self): + for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): + with self.subTest(doc): + msgids = self.extract_docstrings_from_str(dedent('''\ + %s + ''' % doc)) + self.assertIn('doc', msgids) + + def test_moduledocstring_bytes(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + b"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_moduledocstring_fstring(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_msgid(self): + msgids = self.extract_docstrings_from_str( + '''_("""doc""" r'str' u"ing")''') + self.assertIn('docstring', msgids) + + def test_msgid_bytes(self): + msgids = self.extract_docstrings_from_str('_(b"""doc""")') + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_msgid_fstring(self): + msgids = self.extract_docstrings_from_str('_(f"""doc""")') + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_funcdocstring_annotated_args(self): + """ Test docstrings for functions with annotated args """ + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar: str): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_annotated_return(self): + """ Test docstrings for functions with annotated return type """ + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar) -> str: + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_defvalue_args(self): + """ Test docstring for functions with default arg values """ + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar=()): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_multiple_funcs(self): + """ Test docstring extraction for multiple functions combining + annotated args, annotated return types and default arg values + """ + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo1(bar: tuple=()) -> str: + """doc1""" + + def foo2(bar: List[1:2]) -> (lambda x: x): + """doc2""" + + def foo3(bar: 'func'=lambda x: x) -> {1: 2}: + """doc3""" + ''')) + self.assertIn('doc1', msgids) + self.assertIn('doc2', msgids) + self.assertIn('doc3', msgids) + + def test_classdocstring_early_colon(self): + """ Test docstring extraction for a class with colons occurring within + the parentheses. + """ + msgids = self.extract_docstrings_from_str(dedent('''\ + class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_calls_in_fstrings(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_raw(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + rf"{_('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_nested(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"""{f'{_("foo bar")}'}""" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_attribute(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{obj._('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_with_call_on_call(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{type(str)('foo bar')}" + ''')) + self.assertNotIn('foo bar', msgids) + + def test_calls_in_fstrings_with_format(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo {bar}').format(bar='baz')}" + ''')) + self.assertIn('foo {bar}', msgids) + + def test_calls_in_fstrings_with_wrong_input_1(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(f'foo {bar}')}" + ''')) + self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid]) + + def test_calls_in_fstrings_with_wrong_input_2(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(1)}" + ''')) + self.assertNotIn(1, msgids) + + def test_calls_in_fstring_with_multiple_args(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo', 'bar')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertNotIn('bar', msgids) + + def test_calls_in_fstring_with_keyword_args(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo', bar='baz')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertNotIn('bar', msgids) + self.assertNotIn('baz', msgids) + + def test_calls_in_fstring_with_partially_wrong_expression(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(f'foo') + _('bar')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertIn('bar', msgids) + + def test_function_and_class_names(self): + """Test that function and class names are not mistakenly extracted.""" + msgids = self.extract_from_str(dedent('''\ + def _(x): + pass + + def _(x="foo"): + pass + + async def _(x): + pass + + class _(object): + pass + ''')) + self.assertEqual(msgids, ['']) + + def test_pygettext_output(self): + """Test that the pygettext output exactly matches snapshots.""" + for input_file, output_file, output in extract_from_snapshots(): + with self.subTest(input_file=input_file): + expected = output_file.read_text(encoding='utf-8') + self.assert_POT_equal(expected, output) + + def test_files_list(self): + """Make sure the directories are inspected for source files + bpo-31920 + """ + text1 = 'Text to translate1' + text2 = 'Text to translate2' + text3 = 'Text to ignore' + with temp_cwd(None), temp_dir(None) as sdir: + pymod = Path(sdir, 'pypkg', 'pymod.py') + pymod.parent.mkdir() + pymod.write_text(f'_({text1!r})', encoding='utf-8') + + pymod2 = Path(sdir, 'pkg.py', 'pymod2.py') + pymod2.parent.mkdir() + pymod2.write_text(f'_({text2!r})', encoding='utf-8') + + pymod3 = Path(sdir, 'CVS', 'pymod3.py') + pymod3.parent.mkdir() + pymod3.write_text(f'_({text3!r})', encoding='utf-8') + + assert_python_ok('-Xutf8', self.script, sdir) + data = Path('messages.pot').read_text(encoding='utf-8') + self.assertIn(f'msgid "{text1}"', data) + self.assertIn(f'msgid "{text2}"', data) + self.assertNotIn(text3, data) + + +def extract_from_snapshots(): + snapshots = { + 'messages.py': ('--docstrings',), + 'fileloc.py': ('--docstrings',), + 'docstrings.py': ('--docstrings',), + # == Test character escaping + # Escape ascii and unicode: + 'escapes.py': ('--escape',), + # Escape only ascii and let unicode pass through: + ('escapes.py', 'ascii-escapes.pot'): (), + } + + for filename, args in snapshots.items(): + if isinstance(filename, tuple): + filename, output_file = filename + output_file = DATA_DIR / output_file + input_file = DATA_DIR / filename + else: + input_file = DATA_DIR / filename + output_file = input_file.with_suffix('.pot') + contents = input_file.read_bytes() + with temp_cwd(None): + Path(input_file.name).write_bytes(contents) + assert_python_ok('-Xutf8', Test_pygettext.script, *args, + input_file.name) + yield (input_file, output_file, + Path('messages.pot').read_text(encoding='utf-8')) + + +def update_POT_snapshots(): + for _, output_file, output in extract_from_snapshots(): + output = normalize_POT_file(output) + output_file.write_text(output, encoding='utf-8') + + +if __name__ == '__main__': + # To regenerate POT files + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_POT_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py new file mode 100644 index 00000000000..4c7588d4d93 --- /dev/null +++ b/Lib/test/test_tools/test_makefile.py @@ -0,0 +1,81 @@ +""" +Tests for `Makefile`. +""" + +import os +import unittest +from test import support +import sysconfig + +MAKEFILE = sysconfig.get_makefile_filename() + +if not support.check_impl_detail(cpython=True): + raise unittest.SkipTest('cpython only') +if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE): + raise unittest.SkipTest('Makefile could not be found') + + +class TestMakefile(unittest.TestCase): + def list_test_dirs(self): + result = [] + found_testsubdirs = False + with open(MAKEFILE, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('TESTSUBDIRS='): + found_testsubdirs = True + result.append( + line.removeprefix('TESTSUBDIRS=').replace( + '\\', '', + ).strip(), + ) + continue + if found_testsubdirs: + if '\t' not in line: + break + result.append(line.replace('\\', '').strip()) + return result + + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") + def test_makefile_test_folders(self): + test_dirs = self.list_test_dirs() + idle_test = 'idlelib/idle_test' + self.assertIn(idle_test, test_dirs) + + used = set([idle_test]) + for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR): + dirname = os.path.basename(dirpath) + # Skip temporary dirs: + if dirname == '__pycache__' or dirname.startswith('.'): + dirs.clear() # do not process subfolders + continue + # Skip empty dirs: + if not dirs and not files: + continue + # Skip dirs with hidden-only files: + if files and all( + filename.startswith('.') or filename == '__pycache__' + for filename in files + ): + continue + + relpath = os.path.relpath(dirpath, support.STDLIB_DIR) + with self.subTest(relpath=relpath): + self.assertIn( + relpath, + test_dirs, + msg=( + f"{relpath!r} is not included in the Makefile's list " + "of test directories to install" + ) + ) + used.add(relpath) + + # Don't check the wheel dir when Python is built --with-wheel-pkg-dir + if sysconfig.get_config_var('WHEEL_PKG_DIR'): + test_dirs.remove('test/wheeldata') + used.discard('test/wheeldata') + + # Check that there are no extra entries: + unique_test_dirs = set(test_dirs) + self.assertSetEqual(unique_test_dirs, used) + self.assertEqual(len(test_dirs), len(unique_test_dirs)) diff --git a/Lib/test/test_tools/test_makeunicodedata.py b/Lib/test/test_tools/test_makeunicodedata.py new file mode 100644 index 00000000000..f31375117e2 --- /dev/null +++ b/Lib/test/test_tools/test_makeunicodedata.py @@ -0,0 +1,122 @@ +import unittest +from test.test_tools import skip_if_missing, imports_under_tool +from test import support +from test.support.hypothesis_helper import hypothesis + +st = hypothesis.strategies +given = hypothesis.given +example = hypothesis.example + + +skip_if_missing("unicode") +with imports_under_tool("unicode"): + from dawg import Dawg, build_compression_dawg, lookup, inverse_lookup + + +@st.composite +def char_name_db(draw, min_length=1, max_length=30): + m = draw(st.integers(min_value=min_length, max_value=max_length)) + names = draw( + st.sets(st.text("abcd", min_size=1, max_size=10), min_size=m, max_size=m) + ) + characters = draw(st.sets(st.characters(), min_size=m, max_size=m)) + return list(zip(names, characters)) + + +class TestDawg(unittest.TestCase): + """Tests for the directed acyclic word graph data structure that is used + to store the unicode character names in unicodedata. Tests ported from PyPy + """ + + def test_dawg_direct_simple(self): + dawg = Dawg() + dawg.insert("a", -4) + dawg.insert("c", -2) + dawg.insert("cat", -1) + dawg.insert("catarr", 0) + dawg.insert("catnip", 1) + dawg.insert("zcatnip", 5) + packed, data, inverse = dawg.finish() + + self.assertEqual(lookup(packed, data, b"a"), -4) + self.assertEqual(lookup(packed, data, b"c"), -2) + self.assertEqual(lookup(packed, data, b"cat"), -1) + self.assertEqual(lookup(packed, data, b"catarr"), 0) + self.assertEqual(lookup(packed, data, b"catnip"), 1) + self.assertEqual(lookup(packed, data, b"zcatnip"), 5) + self.assertRaises(KeyError, lookup, packed, data, b"b") + self.assertRaises(KeyError, lookup, packed, data, b"catni") + self.assertRaises(KeyError, lookup, packed, data, b"catnipp") + + self.assertEqual(inverse_lookup(packed, inverse, -4), b"a") + self.assertEqual(inverse_lookup(packed, inverse, -2), b"c") + self.assertEqual(inverse_lookup(packed, inverse, -1), b"cat") + self.assertEqual(inverse_lookup(packed, inverse, 0), b"catarr") + self.assertEqual(inverse_lookup(packed, inverse, 1), b"catnip") + self.assertEqual(inverse_lookup(packed, inverse, 5), b"zcatnip") + self.assertRaises(KeyError, inverse_lookup, packed, inverse, 12) + + def test_forbid_empty_dawg(self): + dawg = Dawg() + self.assertRaises(ValueError, dawg.finish) + + @given(char_name_db()) + @example([("abc", "a"), ("abd", "b")]) + @example( + [ + ("bab", "1"), + ("a", ":"), + ("ad", "@"), + ("b", "<"), + ("aacc", "?"), + ("dab", "D"), + ("aa", "0"), + ("ab", "F"), + ("aaa", "7"), + ("cbd", "="), + ("abad", ";"), + ("ac", "B"), + ("abb", "4"), + ("bb", "2"), + ("aab", "9"), + ("caaaaba", "E"), + ("ca", ">"), + ("bbaaa", "5"), + ("d", "3"), + ("baac", "8"), + ("c", "6"), + ("ba", "A"), + ] + ) + @example( + [ + ("bcdac", "9"), + ("acc", "g"), + ("d", "d"), + ("daabdda", "0"), + ("aba", ";"), + ("c", "6"), + ("aa", "7"), + ("abbd", "c"), + ("badbd", "?"), + ("bbd", "f"), + ("cc", "@"), + ("bb", "8"), + ("daca", ">"), + ("ba", ":"), + ("baac", "3"), + ("dbdddac", "a"), + ("a", "2"), + ("cabd", "b"), + ("b", "="), + ("abd", "4"), + ("adcbd", "5"), + ("abc", "e"), + ("ab", "1"), + ] + ) + def test_dawg(self, data): + # suppress debug prints + with support.captured_stdout() as output: + # it's enough to build it, building will also check the result + build_compression_dawg(data) diff --git a/Lib/test/test_tools/test_msgfmt.py b/Lib/test/test_tools/test_msgfmt.py new file mode 100644 index 00000000000..8cd31680f76 --- /dev/null +++ b/Lib/test/test_tools/test_msgfmt.py @@ -0,0 +1,159 @@ +"""Tests for the Tools/i18n/msgfmt.py tool.""" + +import json +import sys +import unittest +from gettext import GNUTranslations +from pathlib import Path + +from test.support.os_helper import temp_cwd +from test.support.script_helper import assert_python_failure, assert_python_ok +from test.test_tools import skip_if_missing, toolsdir + + +skip_if_missing('i18n') + +data_dir = (Path(__file__).parent / 'msgfmt_data').resolve() +script_dir = Path(toolsdir) / 'i18n' +msgfmt = script_dir / 'msgfmt.py' + + +def compile_messages(po_file, mo_file): + assert_python_ok(msgfmt, '-o', mo_file, po_file) + + +class CompilationTest(unittest.TestCase): + + def test_compilation(self): + self.maxDiff = None + with temp_cwd(): + for po_file in data_dir.glob('*.po'): + with self.subTest(po_file=po_file): + mo_file = po_file.with_suffix('.mo') + with open(mo_file, 'rb') as f: + expected = GNUTranslations(f) + + tmp_mo_file = mo_file.name + compile_messages(po_file, tmp_mo_file) + with open(tmp_mo_file, 'rb') as f: + actual = GNUTranslations(f) + + self.assertDictEqual(actual._catalog, expected._catalog) + + def test_translations(self): + with open(data_dir / 'general.mo', 'rb') as f: + t = GNUTranslations(f) + + self.assertEqual(t.gettext('foo'), 'foo') + self.assertEqual(t.gettext('bar'), 'baz') + self.assertEqual(t.pgettext('abc', 'foo'), 'bar') + self.assertEqual(t.pgettext('xyz', 'foo'), 'bar') + self.assertEqual(t.gettext('Multilinestring'), 'Multilinetranslation') + self.assertEqual(t.gettext('"escapes"'), '"translated"') + self.assertEqual(t.gettext('\n newlines \n'), '\n translated \n') + self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 1), + 'One email sent.') + self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 2), + '%d emails sent.') + self.assertEqual(t.npgettext('abc', 'One email sent.', + '%d emails sent.', 1), + 'One email sent.') + self.assertEqual(t.npgettext('abc', 'One email sent.', + '%d emails sent.', 2), + '%d emails sent.') + + def test_invalid_msgid_plural(self): + with temp_cwd(): + Path('invalid.po').write_text('''\ +msgid_plural "plural" +msgstr[0] "singular" +''') + + res = assert_python_failure(msgfmt, 'invalid.po') + err = res.err.decode('utf-8') + self.assertIn('msgid_plural not preceded by msgid', err) + + def test_plural_without_msgid_plural(self): + with temp_cwd(): + Path('invalid.po').write_text('''\ +msgid "foo" +msgstr[0] "bar" +''') + + res = assert_python_failure(msgfmt, 'invalid.po') + err = res.err.decode('utf-8') + self.assertIn('plural without msgid_plural', err) + + def test_indexed_msgstr_without_msgid_plural(self): + with temp_cwd(): + Path('invalid.po').write_text('''\ +msgid "foo" +msgid_plural "foos" +msgstr "bar" +''') + + res = assert_python_failure(msgfmt, 'invalid.po') + err = res.err.decode('utf-8') + self.assertIn('indexed msgstr required for plural', err) + + def test_generic_syntax_error(self): + with temp_cwd(): + Path('invalid.po').write_text('''\ +"foo" +''') + + res = assert_python_failure(msgfmt, 'invalid.po') + err = res.err.decode('utf-8') + self.assertIn('Syntax error', err) + +class CLITest(unittest.TestCase): + + def test_help(self): + for option in ('--help', '-h'): + res = assert_python_ok(msgfmt, option) + err = res.err.decode('utf-8') + self.assertIn('Generate binary message catalog from textual translation description.', err) + + def test_version(self): + for option in ('--version', '-V'): + res = assert_python_ok(msgfmt, option) + out = res.out.decode('utf-8').strip() + self.assertEqual('msgfmt.py 1.2', out) + + def test_invalid_option(self): + res = assert_python_failure(msgfmt, '--invalid-option') + err = res.err.decode('utf-8') + self.assertIn('Generate binary message catalog from textual translation description.', err) + self.assertIn('option --invalid-option not recognized', err) + + def test_no_input_file(self): + res = assert_python_ok(msgfmt) + err = res.err.decode('utf-8').replace('\r\n', '\n') + self.assertIn('No input file given\n' + "Try `msgfmt --help' for more information.", err) + + def test_nonexistent_file(self): + assert_python_failure(msgfmt, 'nonexistent.po') + + +def update_catalog_snapshots(): + for po_file in data_dir.glob('*.po'): + mo_file = po_file.with_suffix('.mo') + compile_messages(po_file, mo_file) + # Create a human-readable JSON file which is + # easier to review than the binary .mo file. + with open(mo_file, 'rb') as f: + translations = GNUTranslations(f) + catalog_file = po_file.with_suffix('.json') + with open(catalog_file, 'w') as f: + data = translations._catalog.items() + data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0])) + json.dump(data, f, indent=4) + f.write('\n') + + +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_catalog_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_tools/test_reindent.py b/Lib/test/test_tools/test_reindent.py new file mode 100644 index 00000000000..64e31c2b770 --- /dev/null +++ b/Lib/test/test_tools/test_reindent.py @@ -0,0 +1,35 @@ +"""Tests for scripts in the Tools directory. + +This file contains regression tests for some of the scripts found in the +Tools directory of a Python checkout or tarball, such as reindent.py. +""" + +import os +import unittest +from test.support.script_helper import assert_python_ok +from test.support import findfile + +from test.test_tools import toolsdir, skip_if_missing + +skip_if_missing() + +class ReindentTests(unittest.TestCase): + script = os.path.join(toolsdir, 'patchcheck', 'reindent.py') + + def test_noargs(self): + assert_python_ok(self.script) + + def test_help(self): + rc, out, err = assert_python_ok(self.script, '-h') + self.assertEqual(out, b'') + self.assertGreater(err, b'') + + def test_reindent_file_with_bad_encoding(self): + bad_coding_path = findfile('bad_coding.py', subdir='tokenizedata') + rc, out, err = assert_python_ok(self.script, '-r', bad_coding_path) + self.assertEqual(out, b'') + self.assertNotEqual(err, b'') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py new file mode 100644 index 00000000000..d0b702d392c --- /dev/null +++ b/Lib/test/test_tools/test_sundry.py @@ -0,0 +1,30 @@ +"""Tests for scripts in the Tools/scripts directory. + +This file contains extremely basic regression tests for the scripts found in +the Tools directory of a Python checkout or tarball which don't have separate +tests of their own. +""" + +import os +import unittest +from test.support import import_helper + +from test.test_tools import scriptsdir, import_tool, skip_if_missing + +skip_if_missing() + +class TestSundryScripts(unittest.TestCase): + # import logging registers "atfork" functions which keep indirectly the + # logging module dictionary alive. Mock the function to be able to unload + # cleanly the logging module. + @import_helper.mock_register_at_fork + def test_sundry(self, mock_os): + for fn in os.listdir(scriptsdir): + if not fn.endswith('.py'): + continue + name = fn[:-3] + import_tool(name) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index a551a17fc1e..5968becf399 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,11 +1,12 @@ import os from pickle import dump import sys -from test.support import captured_stdout +from test.support import captured_stdout, requires_resource from test.support.os_helper import (TESTFN, rmtree, unlink) from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap import unittest +from types import FunctionType import trace from trace import Trace @@ -129,8 +130,7 @@ def setUp(self): self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) self.my_py_filename = fix_ext_py(__file__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 48): 1} def test_traced_func_linear(self): result = self.tracer.runfunc(traced_func_linear, 2, 5) self.assertEqual(result, 7) @@ -143,8 +143,7 @@ def test_traced_func_linear(self): self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 54): 1} def test_traced_func_loop(self): self.tracer.runfunc(traced_func_loop, 2, 3) @@ -157,8 +156,7 @@ def test_traced_func_loop(self): } self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/tracedmodules/testmod.py', 3): 1} def test_traced_func_importing(self): self.tracer.runfunc(traced_func_importing, 2, 5) @@ -171,8 +169,7 @@ def test_traced_func_importing(self): self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 76): 10} def test_trace_func_generator(self): self.tracer.runfunc(traced_func_calling_generator) @@ -188,8 +185,7 @@ def test_trace_func_generator(self): } self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 87): 1} def test_trace_list_comprehension(self): self.tracer.runfunc(traced_caller_list_comprehension) @@ -197,16 +193,13 @@ def test_trace_list_comprehension(self): firstlineno_called = get_firstlineno(traced_doubler) expected = { (self.my_py_filename, firstlineno_calling + 1): 1, - # List comprehensions work differently in 3.x, so the count - # below changed compared to 2.x. - (self.my_py_filename, firstlineno_calling + 2): 12, + (self.my_py_filename, firstlineno_calling + 2): 11, (self.my_py_filename, firstlineno_calling + 3): 1, (self.my_py_filename, firstlineno_called + 1): 10, } self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 996 characters long. Set self.maxDiff to None to see it. def test_traced_decorated_function(self): self.tracer.runfunc(traced_decorated_function) @@ -226,8 +219,7 @@ def test_traced_decorated_function(self): } self.assertEqual(self.tracer.results().counts, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + {('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 108): 1} def test_linear_methods(self): # XXX todo: later add 'static_method_linear' and 'class_method_linear' # here, once issue1764286 is resolved @@ -251,8 +243,7 @@ def setUp(self): self.my_py_filename = fix_ext_py(__file__) self.addCleanup(sys.settrace, sys.gettrace()) - # TODO: RUSTPYTHON, KeyError: ('Lib/test/test_trace.py', 43) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; KeyError: ('/Users/al03219714/Projects/RustPython4/crates/pylib/Lib/test/test_trace.py', 51) def test_exec_counts(self): self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) code = r'''traced_func_loop(2, 5)''' @@ -287,8 +278,6 @@ def tearDown(self): if self._saved_tracefunc is not None: sys.settrace(self._saved_tracefunc) - # TODO: RUSTPYTHON, gc - @unittest.expectedFailure def test_simple_caller(self): self.tracer.runfunc(traced_func_simple_caller, 1) @@ -306,8 +295,6 @@ def test_arg_errors(self): with self.assertRaises(TypeError): self.tracer.runfunc() - # TODO: RUSTPYTHON, gc - @unittest.expectedFailure def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) @@ -320,8 +307,7 @@ def test_loop_caller_importing(self): } self.assertEqual(self.tracer.results().calledfuncs, expected) - # TODO: RUSTPYTHON, gc - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') def test_inst_method_calling(self): @@ -335,8 +321,6 @@ def test_inst_method_calling(self): } self.assertEqual(self.tracer.results().calledfuncs, expected) - # TODO: RUSTPYTHON, gc - @unittest.expectedFailure def test_traced_decorated_function(self): self.tracer.runfunc(traced_decorated_function) @@ -357,9 +341,9 @@ def setUp(self): self.tracer = Trace(count=0, trace=0, countcallers=1) self.filemod = my_file_and_modname() + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) @@ -387,15 +371,20 @@ def tearDown(self): rmtree(TESTFN) unlink(TESTFN) - def _coverage(self, tracer, - cmd='import test.support, test.test_pprint;' - 'test.support.run_unittest(test.test_pprint.QueryTestCase)'): + DEFAULT_SCRIPT = '''if True: + import unittest + from test.test_pprint import QueryTestCase + loader = unittest.TestLoader() + tests = loader.loadTestsFromTestCase(QueryTestCase) + tests(unittest.TestResult()) + ''' + def _coverage(self, tracer, cmd=DEFAULT_SCRIPT): tracer.run(cmd) r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=TESTFN) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'pprint.py' not found in '' + @requires_resource('cpu') def test_coverage(self): tracer = trace.Trace(trace=0, count=1) with captured_stdout() as stdout: @@ -412,15 +401,14 @@ def test_coverage_ignore(self): libpath = os.path.normpath(os.path.dirname(os.path.dirname(__file__))) # sys.prefix does not work when running from a checkout tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - libpath], trace=0, count=1) + libpath] + sys.path, trace=0, count=1) with captured_stdout() as stdout: self._coverage(tracer) if os.path.exists(TESTFN): files = os.listdir(TESTFN) self.assertEqual(files, ['_importlib.cover']) # Ignore __import__ - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'test.tracedmodules.testmod' not found in {} def test_issue9936(self): tracer = trace.Trace(trace=0, count=1) modname = 'test.tracedmodules.testmod' @@ -436,7 +424,7 @@ def test_issue9936(self): coverage = {} for line in stdout: lines, cov, module = line.split()[:3] - coverage[module] = (int(lines), int(cov[:-1])) + coverage[module] = (float(lines), float(cov[:-1])) # XXX This is needed to run regrtest.py as a script modname = trace._fullmodname(sys.modules[modname].__file__) self.assertIn(modname, coverage) @@ -485,8 +473,7 @@ def tearDown(self): unlink(self.codefile) unlink(self.coverfile) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; --- def test_cover_files_written_no_highlight(self): # Test also that the cover file for the trace module is not created # (issue #34171). @@ -507,8 +494,7 @@ def test_cover_files_written_no_highlight(self): " print('unreachable')\n" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; --- def test_cover_files_written_with_highlight(self): argv = '-m trace --count --missing'.split() + [self.codefile] status, stdout, stderr = assert_python_ok(*argv) @@ -536,8 +522,6 @@ def test_failures(self): *_, stderr = assert_python_failure('-m', 'trace', *args) self.assertIn(message, stderr) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_listfuncs_flag_success(self): filename = TESTFN + '.py' modulename = os.path.basename(TESTFN) @@ -561,8 +545,7 @@ def test_sys_argv_list(self): PYTHONIOENCODING='utf-8') self.assertIn(direct_stdout.strip(), trace_stdout) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'lines cov% module (path)' not found in '' def test_count_and_summary(self): filename = f'{TESTFN}.py' coverfilename = f'{TESTFN}.cover' @@ -585,12 +568,37 @@ def f(): stdout = stdout.decode() self.assertEqual(status, 0) self.assertIn('lines cov% module (path)', stdout) - self.assertIn(f'6 100% {modulename} ({filename})', stdout) + self.assertIn(f'6 100.0% {modulename} ({filename})', stdout) def test_run_as_module(self): assert_python_ok('-m', 'trace', '-l', '--module', 'timeit', '-n', '1') assert_python_failure('-m', 'trace', '-l', '--module', 'not_a_module_zzz') +class TestTrace(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + self.tracer = Trace(count=0, trace=1) + self.filemod = my_file_and_modname() + + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: list index out of range + def test_no_source_file(self): + filename = "<unknown>" + co = traced_func_linear.__code__ + co = co.replace(co_filename=filename) + f = FunctionType(co, globals()) + + with captured_stdout() as out: + self.tracer.runfunc(f, 2, 3) + + out = out.getvalue().splitlines() + firstlineno = get_firstlineno(f) + self.assertIn(f" --- modulename: {self.filemod[1]}, funcname: {f.__code__.co_name}", out[0]) + self.assertIn(f"{filename}({firstlineno + 1})", out[1]) + self.assertIn(f"{filename}({firstlineno + 2})", out[2]) + self.assertIn(f"{filename}({firstlineno + 3})", out[3]) + self.assertIn(f"{filename}({firstlineno + 4})", out[4]) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 20f8085ccf2..e904c149d03 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4,26 +4,60 @@ from io import StringIO import linecache import sys +import types import inspect +import builtins import unittest +import unittest.mock import re +import tempfile +import random +import string from test import support -from test.support import Error, captured_output, cpython_only, ALWAYS_EQ -from test.support.os_helper import TESTFN, unlink -from test.support.script_helper import assert_python_ok +import shutil +from test.support import (Error, captured_output, cpython_only, ALWAYS_EQ, + requires_debug_ranges, has_no_debug_ranges, + requires_subprocess) +from test.support.os_helper import TESTFN, temp_dir, unlink +from test.support.script_helper import assert_python_ok, assert_python_failure, make_script +from test.support.import_helper import forget +from test.support import force_not_colorized, force_not_colorized_test_class + +import json import textwrap - import traceback +from functools import partial +from pathlib import Path +import _colorize +MODULE_PREFIX = f'{__name__}.' if __name__ == '__main__' else '' test_code = namedtuple('code', ['co_filename', 'co_name']) +test_code.co_positions = lambda _: iter([(6, 6, 0, 0)]) test_frame = namedtuple('frame', ['f_code', 'f_globals', 'f_locals']) -test_tb = namedtuple('tb', ['tb_frame', 'tb_lineno', 'tb_next']) +test_tb = namedtuple('tb', ['tb_frame', 'tb_lineno', 'tb_next', 'tb_lasti']) + +color_overrides = {"reset": "z", "filename": "fn", "error_highlight": "E"} +colors = { + color_overrides.get(k, k[0].lower()): v + for k, v in _colorize.default_theme.traceback.items() +} + + +LEVENSHTEIN_DATA_FILE = Path(__file__).parent / 'levenshtein_examples.json' class TracebackCases(unittest.TestCase): # For now, a very minimal set of tests. I want to be sure that # formatting of SyntaxErrors works based on changes for 2.1. + def setUp(self): + super().setUp() + self.colorize = _colorize.COLORIZE + _colorize.COLORIZE = False + + def tearDown(self): + super().tearDown() + _colorize.COLORIZE = self.colorize def get_exception_format(self, func, exc): try: @@ -54,13 +88,11 @@ def syntax_error_bad_indentation2(self): def tokenizer_error_with_caret_range(self): compile("blech ( ", "?", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_caret(self): err = self.get_exception_format(self.syntax_error_with_caret, SyntaxError) self.assertEqual(len(err), 4) - self.assertTrue(err[1].strip() == "return x!") + self.assertEqual(err[1].strip(), "return x!") self.assertIn("^", err[2]) # third line has caret self.assertEqual(err[1].find("!"), err[2].find("^")) # in the right place self.assertEqual(err[2].count("^"), 1) @@ -93,16 +125,81 @@ def test_caret(self): self.assertEqual(err[1].find("("), err[2].find("^")) # in the right place self.assertEqual(err[2].count("^"), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nocaret(self): exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) err = traceback.format_exception_only(SyntaxError, exc) self.assertEqual(len(err), 3) self.assertEqual(err[1].strip(), "bad syntax") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @force_not_colorized + def test_no_caret_with_no_debug_ranges_flag(self): + # Make sure that if `-X no_debug_ranges` is used, there are no carets + # in the traceback. + try: + with open(TESTFN, 'w') as f: + f.write("x = 1 / 0\n") + + _, _, stderr = assert_python_failure( + '-X', 'no_debug_ranges', TESTFN) + + lines = stderr.splitlines() + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0], b'Traceback (most recent call last):') + self.assertIn(b'line 1, in <module>', lines[1]) + self.assertEqual(lines[2], b' x = 1 / 0') + self.assertEqual(lines[3], b'ZeroDivisionError: division by zero') + finally: + unlink(TESTFN) + + def test_no_caret_with_no_debug_ranges_flag_python_traceback(self): + code = textwrap.dedent(""" + import traceback + try: + x = 1 / 0 + except ZeroDivisionError: + traceback.print_exc() + """) + try: + with open(TESTFN, 'w') as f: + f.write(code) + + _, _, stderr = assert_python_ok( + '-X', 'no_debug_ranges', TESTFN) + + lines = stderr.splitlines() + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0], b'Traceback (most recent call last):') + self.assertIn(b'line 4, in <module>', lines[1]) + self.assertEqual(lines[2], b' x = 1 / 0') + self.assertEqual(lines[3], b'ZeroDivisionError: division by zero') + finally: + unlink(TESTFN) + + def test_recursion_error_during_traceback(self): + code = textwrap.dedent(""" + import sys + from weakref import ref + + sys.setrecursionlimit(15) + + def f(): + ref(lambda: 0, []) + f() + + try: + f() + except RecursionError: + pass + """) + try: + with open(TESTFN, 'w') as f: + f.write(code) + + rc, _, _ = assert_python_ok(TESTFN) + self.assertEqual(rc, 0) + finally: + unlink(TESTFN) + def test_bad_indentation(self): err = self.get_exception_format(self.syntax_error_bad_indentation, IndentationError) @@ -129,15 +226,189 @@ def __str__(self): 1/0 err = traceback.format_exception_only(X, X()) self.assertEqual(len(err), 1) - str_value = '<unprintable %s object>' % X.__name__ + str_value = '<exception str() failed>' if X.__module__ in ('__main__', 'builtins'): str_name = X.__qualname__ else: str_name = '.'.join([X.__module__, X.__qualname__]) self.assertEqual(err[0], "%s: %s\n" % (str_name, str_value)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_format_exception_group_without_show_group(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(eg) + self.assertEqual(err, ['ExceptionGroup: A (1 sub-exception)\n']) + + def test_format_exception_group(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ]) + + def test_format_base_exception_group(self): + eg = BaseExceptionGroup('A', [BaseException('B')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'BaseExceptionGroup: A (1 sub-exception)\n', + ' BaseException: B\n', + ]) + + def test_format_exception_group_with_note(self): + exc = ValueError('B') + exc.add_note('Note') + eg = ExceptionGroup('A', [exc]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ' Note\n', + ]) + + def test_format_exception_group_explicit_class(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(ExceptionGroup, eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ]) + + def test_format_exception_group_multiple_exceptions(self): + eg = ExceptionGroup('A', [ValueError('B'), TypeError('C')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (2 sub-exceptions)\n', + ' ValueError: B\n', + ' TypeError: C\n', + ]) + + def test_format_exception_group_multiline_messages(self): + eg = ExceptionGroup('A\n1', [ValueError('B\n2')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n1 (1 sub-exception)\n', + ' ValueError: B\n', + ' 2\n', + ]) + + def test_format_exception_group_multiline2_messages(self): + exc = ValueError('B\n\n2\n') + exc.add_note('\nC\n\n3') + eg = ExceptionGroup('A\n\n1\n', [exc, IndexError('D')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n\n1\n (2 sub-exceptions)\n', + ' ValueError: B\n', + ' \n', + ' 2\n', + ' \n', + ' \n', # first char of `note` + ' C\n', + ' \n', + ' 3\n', # note ends + ' IndexError: D\n', + ]) + + def test_format_exception_group_syntax_error(self): + exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) + eg = ExceptionGroup('A\n1', [exc]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n1 (1 sub-exception)\n', + ' File "x.py", line 23\n', + ' bad syntax\n', + ' SyntaxError: error\n', + ]) + + def test_format_exception_group_nested_with_notes(self): + exc = IndexError('D') + exc.add_note('Note\nmultiline') + eg = ExceptionGroup('A', [ + ValueError('B'), + ExceptionGroup('C', [exc, LookupError('E')]), + TypeError('F'), + ]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (3 sub-exceptions)\n', + ' ValueError: B\n', + ' ExceptionGroup: C (2 sub-exceptions)\n', + ' IndexError: D\n', + ' Note\n', + ' multiline\n', + ' LookupError: E\n', + ' TypeError: F\n', + ]) + + def test_format_exception_group_with_tracebacks(self): + def f(): + try: + 1 / 0 + except ZeroDivisionError as e: + return e + + def g(): + try: + raise TypeError('g') + except TypeError as e: + return e + + eg = ExceptionGroup('A', [ + f(), + ExceptionGroup('B', [g()]), + ]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (2 sub-exceptions)\n', + ' ZeroDivisionError: division by zero\n', + ' ExceptionGroup: B (1 sub-exception)\n', + ' TypeError: g\n', + ]) + + def test_format_exception_group_with_cause(self): + def f(): + try: + try: + 1 / 0 + except ZeroDivisionError: + raise ValueError(0) + except Exception as e: + return e + + eg = ExceptionGroup('A', [f()]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: 0\n', + ]) + + def test_format_exception_group_syntax_error_with_custom_values(self): + # See https://github.com/python/cpython/issues/128894 + for exc in [ + SyntaxError('error', 'abcd'), + SyntaxError('error', [None] * 4), + SyntaxError('error', (1, 2, 3, 4)), + SyntaxError('error', (1, 2, 3, 4)), + SyntaxError('error', (1, 'a', 'b', 2)), + # with end_lineno and end_offset: + SyntaxError('error', 'abcdef'), + SyntaxError('error', [None] * 6), + SyntaxError('error', (1, 2, 3, 4, 5, 6)), + SyntaxError('error', (1, 'a', 'b', 2, 'c', 'd')), + ]: + with self.subTest(exc=exc): + err = traceback.format_exception_only(exc, show_group=True) + # Should not raise an exception: + if exc.lineno is not None: + self.assertEqual(len(err), 2) + self.assertTrue(err[0].startswith(' File')) + else: + self.assertEqual(len(err), 1) + self.assertEqual(err[-1], 'SyntaxError: error\n') + + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range + @requires_subprocess() + @force_not_colorized def test_encoded_file(self): # Test that tracebacks are correctly printed for encoded source files: # - correct line number (Issue2384) @@ -179,15 +450,10 @@ def do_test(firstlines, message, charset, lineno): err_line = "raise RuntimeError('{0}')".format(message_ascii) err_msg = "RuntimeError: {0}".format(message_ascii) - self.assertIn(("line %s" % lineno), stdout[1], - "Invalid line number: {0!r} instead of {1}".format( - stdout[1], lineno)) - self.assertTrue(stdout[2].endswith(err_line), - "Invalid traceback line: {0!r} instead of {1!r}".format( - stdout[2], err_line)) - self.assertTrue(stdout[3] == err_msg, - "Invalid error message: {0!r} instead of {1!r}".format( - stdout[3], err_msg)) + self.assertIn("line %s" % lineno, stdout[1]) + self.assertEndsWith(stdout[2], err_line) + actual_err_msg = stdout[3] + self.assertEqual(actual_err_msg, err_msg) do_test("", "foo", "ascii", 3) for charset in ("ascii", "iso-8859-1", "utf-8", "GBK"): @@ -206,8 +472,6 @@ def do_test(firstlines, message, charset, lineno): # Issue #18960: coding spec should have no effect do_test("x=0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_print_traceback_at_exit(self): # Issue #22599: Ensure that it is possible to use the traceback module # to display an exception at Python exit @@ -219,15 +483,15 @@ class PrintExceptionAtExit(object): def __init__(self): try: x = 1 / 0 - except Exception: - self.exc_info = sys.exc_info() - # self.exc_info[1] (traceback) contains frames: + except Exception as e: + self.exc = e + # self.exc.__traceback__ contains frames: # explicitly clear the reference to self in the current # frame to break a reference cycle self = None def __del__(self): - traceback.print_exception(*self.exc_info) + traceback.print_exception(self.exc) # Keep a reference in the module namespace to call the destructor # when the module is unloaded @@ -236,9 +500,38 @@ def __del__(self): rc, stdout, stderr = assert_python_ok('-c', code) expected = [b'Traceback (most recent call last):', b' File "<string>", line 8, in __init__', + b' x = 1 / 0', + b' ^^^^^', b'ZeroDivisionError: division by zero'] self.assertEqual(stderr.splitlines(), expected) + @cpython_only + def test_lost_io_open(self): + # GH-142737: Display the traceback even if io.open is lost + crasher = textwrap.dedent("""\ + import io + import traceback + # Trigger fallback mode + traceback._print_exception_bltin = None + del io.open + raise RuntimeError("should not crash") + """) + + # Create a temporary script to exercise _Py_FindSourceFile + with temp_dir() as script_dir: + script = make_script( + script_dir=script_dir, + script_basename='tb_test_no_io_open', + source=crasher) + rc, stdout, stderr = assert_python_failure(script) + + self.assertEqual(rc, 1) # Make sure it's not a crash + + expected = [b'Traceback (most recent call last):', + f' File "{script}", line 6, in <module>'.encode(), + b'RuntimeError: should not crash'] + self.assertEqual(stderr.splitlines(), expected) + def test_print_exception(self): output = StringIO() traceback.print_exception( @@ -251,6 +544,12 @@ def test_print_exception_exc(self): traceback.print_exception(Exception("projector"), file=output) self.assertEqual(output.getvalue(), "Exception: projector\n") + def test_print_last(self): + with support.swap_attr(sys, 'last_exc', ValueError(42)): + output = StringIO() + traceback.print_last(file=output) + self.assertEqual(output.getvalue(), "ValueError: 42\n") + def test_format_exception_exc(self): e = Exception("projector") output = traceback.format_exception(e) @@ -259,7 +558,7 @@ def test_format_exception_exc(self): traceback.format_exception(e.__class__, e) with self.assertRaisesRegex(ValueError, 'Both or neither'): traceback.format_exception(e.__class__, tb=e.__traceback__) - with self.assertRaisesRegex(TypeError, 'positional-only'): + with self.assertRaisesRegex(TypeError, 'required positional argument'): traceback.format_exception(exc=e) def test_format_exception_only_exc(self): @@ -293,35 +592,1407 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.print_exception)), ('(exc, /, value=<implicit>, tb=<implicit>, ' - 'limit=None, file=None, chain=True)')) + 'limit=None, file=None, chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception)), ('(exc, /, value=<implicit>, tb=<implicit>, limit=None, ' - 'chain=True)')) + 'chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception_only)), - '(exc, /, value=<implicit>)') + '(exc, /, value=<implicit>, *, show_group=False, **kwargs)') + + +class PurePythonExceptionFormattingMixin: + def get_exception(self, callable, slice_start=0, slice_end=-1): + try: + callable() + except BaseException: + return traceback.format_exc().splitlines()[slice_start:slice_end] + else: + self.fail("No exception thrown.") + + callable_line = get_exception.__code__.co_firstlineno + 2 + + +class CAPIExceptionFormattingMixin: + LEGACY = 0 + + def get_exception(self, callable, slice_start=0, slice_end=-1): + from _testcapi import exception_print + try: + callable() + self.fail("No exception thrown.") + except Exception as e: + with captured_output("stderr") as tbstderr: + exception_print(e, self.LEGACY) + return tbstderr.getvalue().splitlines()[slice_start:slice_end] + + callable_line = get_exception.__code__.co_firstlineno + 3 + +class CAPIExceptionFormattingLegacyMixin(CAPIExceptionFormattingMixin): + LEGACY = 1 + +@requires_debug_ranges() +class TracebackErrorLocationCaretTestBase: + """ + Tests for printing code error expressions as part of PEP 657 + """ + def test_basic_caret(self): + # NOTE: In caret tests, "if True:" is used as a way to force indicator + # display, since the raising expression spans only part of the line. + def f(): + if True: raise ValueError("basic caret tests") + + lineno_f = f.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f\n' + ' if True: raise ValueError("basic caret tests")\n' + ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_f.splitlines()) + + def test_line_with_unicode(self): + # Make sure that even if a line contains multi-byte unicode characters + # the correct carets are printed. + def f_with_unicode(): + if True: raise ValueError("Ĥellö Wörld") + + lineno_f = f_with_unicode.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f_with_unicode\n' + ' if True: raise ValueError("Ĥellö Wörld")\n' + ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_unicode) + self.assertEqual(result_lines, expected_f.splitlines()) + + def test_caret_in_type_annotation(self): + def f_with_type(): + def foo(a: THIS_DOES_NOT_EXIST ) -> int: + return 0 + foo.__annotations__ + + lineno_f = f_with_type.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_type\n' + ' foo.__annotations__\n' + f' File "{__file__}", line {lineno_f+1}, in __annotate__\n' + ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' + ' ^^^^^^^^^^^^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_type) + self.assertEqual(result_lines, expected_f.splitlines()) + + def test_caret_multiline_expression(self): + # Make sure no carets are printed for expressions spanning multiple + # lines. + def f_with_multiline(): + if True: raise ValueError( + "error over multiple lines" + ) + + lineno_f = f_with_multiline.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f_with_multiline\n' + ' if True: raise ValueError(\n' + ' ^^^^^^^^^^^^^^^^^\n' + ' "error over multiple lines"\n' + ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ' )\n' + ' ^' + ) + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + + def test_caret_multiline_expression_syntax_error(self): + # Make sure an expression spanning multiple lines that has + # a syntax error is correctly marked with carets. + code = textwrap.dedent(""" + def foo(*args, **kwargs): + pass + + a, b, c = 1, 2, 3 + + foo(a, z + for z in + range(10), b, c) + """) + + def f_with_multiline(): + # Need to defer the compilation until in self.get_exception(..) + return compile(code, "?", "exec") + + lineno_f = f_with_multiline.__code__.co_firstlineno + + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' return compile(code, "?", "exec")\n' + ' File "?", line 7\n' + ' foo(a, z\n' + ' ^' + ) + + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + + # Check custom error messages covering multiple lines + code = textwrap.dedent(""" + dummy_call( + "dummy value" + foo="bar", + ) + """) + + def f_with_multiline(): + # Need to defer the compilation until in self.get_exception(..) + return compile(code, "?", "exec") + + lineno_f = f_with_multiline.__code__.co_firstlineno + + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' return compile(code, "?", "exec")\n' + ' File "?", line 3\n' + ' "dummy value"\n' + ' ^^^^^^^^^^^^^' + ) + + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + + def test_caret_multiline_expression_bin_op(self): + # Make sure no carets are printed for expressions spanning multiple + # lines. + def f_with_multiline(): + return ( + 2 + 1 / + 0 + ) + + lineno_f = f_with_multiline.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' 2 + 1 /\n' + ' ~~^\n' + ' 0\n' + ' ~' + ) + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + + def test_caret_for_binary_operators(self): + def f_with_binary_operator(): + divisor = 20 + return 10 + divisor / 0 + 30 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' return 10 + divisor / 0 + 30\n' + ' ~~~~~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_with_unicode(self): + def f_with_binary_operator(): + áóí = 20 + return 10 + áóí / 0 + 30 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' return 10 + áóí / 0 + 30\n' + ' ~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_two_char(self): + def f_with_binary_operator(): + divisor = 20 + return 10 + divisor // 0 + 30 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' return 10 + divisor // 0 + 30\n' + ' ~~~~~~~~^^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + a = 1 + b = c = "" + return ( a ) +b + c + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return ( a ) +b + c\n' + ' ~~~~~~~~~~^~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline(self): + def f_with_binary_operator(): + b = 1 + c = "" + a = b \ + +\ + c # test + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' a = b \\\n' + ' ~~~~~~\n' + ' +\\\n' + ' ^~\n' + ' c # test\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline_two_char(self): + def f_with_binary_operator(): + b = 1 + c = "" + a = ( + (b # test + + ) \ + # + + << (c # test + \ + ) # test + ) + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+4}, in f_with_binary_operator\n' + ' (b # test +\n' + ' ~~~~~~~~~~~~\n' + ' ) \\\n' + ' ~~~~\n' + ' # +\n' + ' ~~~\n' + ' << (c # test\n' + ' ^^~~~~~~~~~~~\n' + ' \\\n' + ' ~\n' + ' ) # test\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_binary_operators_multiline_with_unicode(self): + def f_with_binary_operator(): + b = 1 + a = ("ááá" + + "áá") + b + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' a = ("ááá" +\n' + ' ~~~~~~~~\n' + ' "áá") + b\n' + ' ~~~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_subscript(self): + def f_with_subscript(): + some_dict = {'x': {'y': None}} + return some_dict['x']['y']['z'] + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' + " return some_dict['x']['y']['z']\n" + ' ~~~~~~~~~~~~~~~~~~~^^^^^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_subscript_unicode(self): + def f_with_subscript(): + some_dict = {'ó': {'á': {'í': {'theta': 1}}}} + return some_dict['ó']['á']['í']['beta'] + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' + " return some_dict['ó']['á']['í']['beta']\n" + ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_subscript_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + a = [] + b = c = 1 + return b [ a ] + c + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return b [ a ] + c\n' + ' ~~~~~~^^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_subscript_multiline(self): + def f_with_subscript(): + bbbbb = {} + ccc = 1 + ddd = 2 + b = bbbbb \ + [ ccc # test + + + ddd \ + + ] # test + return b + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+4}, in f_with_subscript\n' + ' b = bbbbb \\\n' + ' ~~~~~~~\n' + ' [ ccc # test\n' + ' ^^^^^^^^^^^^^\n' + ' \n' + ' \n' + ' + ddd \\\n' + ' ^^^^^^^^\n' + ' \n' + ' \n' + ' ] # test\n' + ' ^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call(self): + def f_with_call(): + def f1(a): + def f2(b): + raise RuntimeError("fail") + return f2 + return f1("x")("y")("z") + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' + ' return f1("x")("y")("z")\n' + ' ~~~~~~~^^^^^\n' + f' File "{__file__}", line {lineno_f+3}, in f2\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_unicode(self): + def f_with_call(): + def f1(a): + def f2(b): + raise RuntimeError("fail") + return f2 + return f1("ó")("á") + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' + ' return f1("ó")("á")\n' + ' ~~~~~~~^^^^^\n' + f' File "{__file__}", line {lineno_f+3}, in f2\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + def f(a): + raise RuntimeError("fail") + return f ( "x" ) + 2 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return f ( "x" ) + 2\n' + ' ~~~~~~^^^^^^^^^^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_for_call_multiline(self): + def f_with_call(): + class C: + def y(self, a): + def f(b): + raise RuntimeError("fail") + return f + def g(x): + return C() + a = (g(1).y)( + 2 + )(3)(4) + return a + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+8}, in f_with_call\n' + ' a = (g(1).y)(\n' + ' ~~~~~~~~~\n' + ' 2\n' + ' ~\n' + ' )(3)(4)\n' + ' ~^^^\n' + f' File "{__file__}", line {lineno_f+4}, in f\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines(self): + def f(): + x = 1 + if True: x += ( + "a" + + "a" + ) # test + + lineno_f = f.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' if True: x += (\n' + ' ^^^^^^\n' + ' ...<2 lines>...\n' + ' ) # test\n' + ' ^\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines_no_caret(self): + def f(): + x = 1 + x += ( + "a" + + "a" + ) + + lineno_f = f.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' x += (\n' + ' ...<2 lines>...\n' + ' )\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_many_lines_binary_op(self): + def f_with_binary_operator(): + b = 1 + c = "a" + a = ( + b + + b + ) + ( + c + + c + + c + ) + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' a = (\n' + ' ~\n' + ' b +\n' + ' ~~~\n' + ' b\n' + ' ~\n' + ' ) + (\n' + ' ~~^~~\n' + ' c +\n' + ' ~~~\n' + ' ...<2 lines>...\n' + ' )\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_traceback_specialization_with_syntax_error(self): + bytecode = compile("1 / 0 / 1 / 2\n", TESTFN, "exec") + + with open(TESTFN, "w") as file: + # make the file's contents invalid + file.write("1 $ 0 / 1 / 2\n") + self.addCleanup(unlink, TESTFN) + + func = partial(exec, bytecode) + result_lines = self.get_exception(func) + + lineno_f = bytecode.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{TESTFN}", line {lineno_f}, in <module>\n' + " 1 $ 0 / 1 / 2\n" + ' ^^^^^\n' + ) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_traceback_very_long_line(self): + source = "if True: " + "a" * 256 + bytecode = compile(source, TESTFN, "exec") + + with open(TESTFN, "w") as file: + file.write(source) + self.addCleanup(unlink, TESTFN) + + func = partial(exec, bytecode) + result_lines = self.get_exception(func) + + lineno_f = bytecode.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{TESTFN}", line {lineno_f}, in <module>\n' + f' {source}\n' + f' {" "*len("if True: ") + "^"*256}\n' + ) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_secondary_caret_not_elided(self): + # Always show a line's indicators if they include the secondary character. + def f_with_subscript(): + some_dict = {'x': {'y': None}} + some_dict['x']['y']['z'] + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' + " some_dict['x']['y']['z']\n" + ' ~~~~~~~~~~~~~~~~~~~^^^^^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_caret_exception_group(self): + # Notably, this covers whether indicators handle margin strings correctly. + # (Exception groups use margin strings to display vertical indicators.) + # The implementation must account for both "indent" and "margin" offsets. + + def exc(): + if True: raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) + + expected_error = ( + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | callable()\n' + f' | ~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' + f' | if True: raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: eg (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n') + + result_lines = self.get_exception(exc) + self.assertEqual(result_lines, expected_error.splitlines()) + + def assertSpecialized(self, func, expected_specialization): + result_lines = self.get_exception(func) + specialization_line = result_lines[-1] + self.assertEqual(specialization_line.lstrip(), expected_specialization) + + def test_specialization_variations(self): + self.assertSpecialized(lambda: 1/0, + "~^~") + self.assertSpecialized(lambda: 1/0/3, + "~^~") + self.assertSpecialized(lambda: 1 / 0, + "~~^~~") + self.assertSpecialized(lambda: 1 / 0 / 3, + "~~^~~") + self.assertSpecialized(lambda: 1/ 0, + "~^~~") + self.assertSpecialized(lambda: 1/ 0/3, + "~^~~") + self.assertSpecialized(lambda: 1 / 0, + "~~~~~^~~~") + self.assertSpecialized(lambda: 1 / 0 / 5, + "~~~~~^~~~") + self.assertSpecialized(lambda: 1 /0, + "~~^~") + self.assertSpecialized(lambda: 1//0, + "~^^~") + self.assertSpecialized(lambda: 1//0//4, + "~^^~") + self.assertSpecialized(lambda: 1 // 0, + "~~^^~~") + self.assertSpecialized(lambda: 1 // 0 // 4, + "~~^^~~") + self.assertSpecialized(lambda: 1 //0, + "~~^^~") + self.assertSpecialized(lambda: 1// 0, + "~^^~~") + + def test_decorator_application_lineno_correct(self): + def dec_error(func): + raise TypeError + def dec_fine(func): + return func + def applydecs(): + @dec_error + @dec_fine + def g(): pass + result_lines = self.get_exception(applydecs) + lineno_applydescs = applydecs.__code__.co_firstlineno + lineno_dec_error = dec_error.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_applydescs + 1}, in applydecs\n' + ' @dec_error\n' + ' ^^^^^^^^^\n' + f' File "{__file__}", line {lineno_dec_error + 1}, in dec_error\n' + ' raise TypeError\n' + ) + self.assertEqual(result_lines, expected_error.splitlines()) + + def applydecs_class(): + @dec_error + @dec_fine + class A: pass + result_lines = self.get_exception(applydecs_class) + lineno_applydescs_class = applydecs_class.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_applydescs_class + 1}, in applydecs_class\n' + ' @dec_error\n' + ' ^^^^^^^^^\n' + f' File "{__file__}", line {lineno_dec_error + 1}, in dec_error\n' + ' raise TypeError\n' + ) + self.assertEqual(result_lines, expected_error.splitlines()) + + def test_multiline_method_call_a(self): + def f(): + (None + .method + )() + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + " .method", + " ^^^^^^", + ] + self.assertEqual(actual, expected) + + def test_multiline_method_call_b(self): + def f(): + (None. + method + )() + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + " method", + ] + self.assertEqual(actual, expected) + + def test_multiline_method_call_c(self): + def f(): + (None + . method + )() + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + " . method", + " ^^^^^^", + ] + self.assertEqual(actual, expected) + + def test_wide_characters_unicode_with_problematic_byte_offset(self): + def f(): + ｗｉｄｔｈ + + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " ｗｉｄｔｈ", + ] + self.assertEqual(actual, expected) + + + def test_byte_offset_with_wide_characters_middle(self): + def f(): + ｗｉｄｔｈ = 1 + raise ValueError(ｗｉｄｔｈ) + + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + " raise ValueError(ｗｉｄｔｈ)", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_multiline(self): + def f(): + ｗｗｗ = 1 + ｔｈ = 0 + + print(1, ｗｗｗ( + ｔｈ)) + + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 4}, in f", + f" print(1, ｗｗｗ(", + f" ~~~~~~^", + f" ｔｈ))", + f" ^^^^^", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_with_wide_characters_term_highlight(self): + def f(): + 说明说明 = 1 + şçöğıĤellö = 0 # not wide but still non-ascii + return 说明说明 / şçöğıĤellö + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 3}, in f", + f" return 说明说明 / şçöğıĤellö", + f" ~~~~~~~~~^~~~~~~~~~~~", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_with_emojis_term_highlight(self): + def f(): + return "✨🐍" + func_说明说明("📗🚛", + "📗🚛") + "🐍" + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + f' return "✨🐍" + func_说明说明("📗🚛",', + f" ^^^^^^^^^^^^^", + ] + self.assertEqual(actual, expected) + + def test_byte_offset_wide_chars_subscript(self): + def f(): + my_dct = { + "✨🚛✨": { + "说明": { + "🐍🐍🐍": None + } + } + } + return my_dct["✨🚛✨"]["说明"]["🐍"]["说明"]["🐍🐍"] + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 8}, in f", + f' return my_dct["✨🚛✨"]["说明"]["🐍"]["说明"]["🐍🐍"]', + f" ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^", + ] + self.assertEqual(actual, expected) + + def test_memory_error(self): + def f(): + raise MemoryError() + + actual = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f' File "{__file__}", line {self.callable_line}, in get_exception', + ' callable()', + ' ~~~~~~~~^^', + f' File "{__file__}", line {f.__code__.co_firstlineno + 1}, in f', + ' raise MemoryError()'] + self.assertEqual(actual, expected) + + def test_anchors_for_simple_return_statements_are_elided(self): + def g(): + 1/0 + + def f(): + return g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + return g() + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g() + 1", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " 2, 4,", + " 5)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " ~^^^", + " 2, 4,", + " ^^^^^", + " 5) + 1", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def test_anchors_for_simple_assign_statements_are_elided(self): + def g(): + 1/0 + def f(): + x = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 -class TracebackFormatTests(unittest.TestCase): + def f(): + x = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g(1,", + " 2, 3,", + " 4)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + x = y = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g()", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = y = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g(1,", + " ~^^^", + " 2, 3,", + " ^^^^^", + " 4)", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + +class TestKeywordTypoSuggestions(unittest.TestCase): + TYPO_CASES = [ + ("with block ad something:\n pass", "and"), + ("fur a in b:\n pass", "for"), + ("for a in b:\n pass\nelso:\n pass", "else"), + ("whille True:\n pass", "while"), + ("iff x > 5:\n pass", "if"), + ("if x:\n pass\nelseif y:\n pass", "elif"), + ("tyo:\n pass\nexcept y:\n pass", "try"), + ("classe MyClass:\n pass", "class"), + ("impor math", "import"), + ("form x import y", "from"), + ("defn calculate_sum(a, b):\n return a + b", "def"), + ("def foo():\n returm result", "return"), + ("lamda x: x ** 2", "lambda"), + ("def foo():\n yeld i", "yield"), + ("def foo():\n globel counter", "global"), + ("frum math import sqrt", "from"), + ("asynch def fetch_data():\n pass", "async"), + ("async def foo():\n awaid fetch_data()", "await"), + ('raisee ValueError("Error")', "raise"), + ("[x for x\nin range(3)\nof x]", "if"), + ("[123 fur x\nin range(3)\nif x]", "for"), + ("for x im n:\n pass", "in"), + ] + + def test_keyword_suggestions_from_file(self): + with tempfile.TemporaryDirectory() as script_dir: + for i, (code, expected_kw) in enumerate(self.TYPO_CASES): + with self.subTest(typo=expected_kw): + source = textwrap.dedent(code).strip() + script_name = make_script(script_dir, f"script_{i}", source) + rc, stdout, stderr = assert_python_failure(script_name) + stderr_text = stderr.decode('utf-8') + self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + + def test_keyword_suggestions_from_command_string(self): + for code, expected_kw in self.TYPO_CASES: + with self.subTest(typo=expected_kw): + source = textwrap.dedent(code).strip() + rc, stdout, stderr = assert_python_failure('-c', source) + stderr_text = stderr.decode('utf-8') + self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + +@requires_debug_ranges() +@force_not_colorized_test_class +class PurePythonTracebackErrorCaretTests( + PurePythonExceptionFormattingMixin, + TracebackErrorLocationCaretTestBase, + unittest.TestCase, +): + """ + Same set of tests as above using the pure Python implementation of + traceback printing in traceback.py. + """ + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~^^~~'] + def test_caret_for_binary_operators_two_char(self): + return super().test_caret_for_binary_operators_two_char() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~^~~'] + def test_caret_for_binary_operators(self): + return super().test_caret_for_binary_operators() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~^^^^^^^^^'] + def test_caret_for_subscript_with_spaces_and_parenthesis(self): + return super().test_caret_for_subscript_with_spaces_and_parenthesis() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~^~~~~~~~~~~~'] + def test_byte_offset_with_wide_characters_term_highlight(self): + return super().test_byte_offset_with_wide_characters_term_highlight() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~^~'] + def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): + return super().test_caret_for_binary_operators_with_spaces_and_parenthesis() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~~~~~~~~~~^^^^^'] + def test_caret_for_subscript(self): + return super().test_caret_for_subscript() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^'] + def test_byte_offset_wide_chars_subscript(self): + return super().test_byte_offset_wide_chars_subscript() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^'] + def test_caret_for_subscript_unicode(self): + return super().test_caret_for_subscript_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~'] + def test_caret_for_binary_operators_multiline(self): + return super().test_caret_for_binary_operators_multiline() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~^~~'] + def test_caret_for_binary_operators_multiline_with_unicode(self): + return super().test_caret_for_binary_operators_multiline_with_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ^^^^^'] + def test_traceback_specialization_with_syntax_error(self): + return super().test_traceback_specialization_with_syntax_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ^^^^^^^^^^^^^'] + def test_caret_multiline_expression_syntax_error(self): + return super().test_caret_multiline_expression_syntax_error() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~'] + def test_caret_multiline_expression_bin_op(self): + return super().test_caret_multiline_expression_bin_op() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ' ~~~~~~~~~~~~~~~~~~~^^^^^'] + def test_secondary_caret_not_elided(self): + return super().test_secondary_caret_not_elided() + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ~^~ + def test_specialization_variations(self): + return super().test_specialization_variations() + + @unittest.expectedFailure # TODO: RUSTPYTHON; - ' ^^^^'] + def test_multiline_method_call_b(self): + return super().test_multiline_method_call_b() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^ ++ + def test_caret_for_binary_operators_with_unicode(self): + return super().test_caret_for_binary_operators_with_unicode() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++ + def test_multiline_method_call_a(self): + return super().test_multiline_method_call_a() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? +++ + def test_multiline_method_call_c(self): + return super().test_multiline_method_call_c() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + def test_many_lines(self): + return super().test_many_lines() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + def test_many_lines_no_caret(self): + return super().test_many_lines_no_caret() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + + def test_anchors_for_simple_assign_statements_are_elided(self): + return super().test_anchors_for_simple_assign_statements_are_elided() + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + + def test_anchors_for_simple_return_statements_are_elided(self): + return super().test_anchors_for_simple_return_statements_are_elided() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: No exception thrown. + def test_caret_in_type_annotation(self): + return super().test_caret_in_type_annotation() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 652 characters long. Set self.maxDiff to None to see it. + def test_decorator_application_lineno_correct(self): + return super().test_decorator_application_lineno_correct() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 684 characters long. Set self.maxDiff to None to see it. + def test_many_lines_binary_op(self): + return super().test_many_lines_binary_op() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 726 characters long. Set self.maxDiff to None to see it. + def test_caret_for_binary_operators_multiline_two_char(self): + return super().test_caret_for_binary_operators_multiline_two_char() + + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 732 characters long. Set self.maxDiff to None to see it. + def test_caret_for_subscript_multiline(self): + return super().test_caret_for_subscript_multiline() + + +@cpython_only +@requires_debug_ranges() +@force_not_colorized_test_class +class CPythonTracebackErrorCaretTests( + CAPIExceptionFormattingMixin, + TracebackErrorLocationCaretTestBase, + unittest.TestCase, +): + """ + Same set of tests as above but with Python's internal traceback printing. + """ + +@cpython_only +@requires_debug_ranges() +@force_not_colorized_test_class +class CPythonTracebackLegacyErrorCaretTests( + CAPIExceptionFormattingLegacyMixin, + TracebackErrorLocationCaretTestBase, + unittest.TestCase, +): + """ + Same set of tests as above but with Python's legacy internal traceback printing. + """ + + +class TracebackFormatMixin: + DEBUG_RANGES = True def some_exception(self): raise KeyError('blah') + def _filter_debug_ranges(self, expected): + return [line for line in expected if not set(line.strip()) <= set("^~")] + + def _maybe_filter_debug_ranges(self, expected): + if not self.DEBUG_RANGES: + return self._filter_debug_ranges(expected) + return expected + @cpython_only def check_traceback_format(self, cleanup_func=None): from _testcapi import traceback_print try: self.some_exception() - except KeyError: - type_, value, tb = sys.exc_info() + except KeyError as e: + tb = e.__traceback__ if cleanup_func is not None: # Clear the inner frames, not this one cleanup_func(tb.tb_next) traceback_fmt = 'Traceback (most recent call last):\n' + \ ''.join(traceback.format_tb(tb)) + # clear caret lines from traceback_fmt since internal API does + # not emit them + traceback_fmt = "\n".join( + self._filter_debug_ranges(traceback_fmt.splitlines()) + ) + "\n" file_ = StringIO() traceback_print(tb, file_) python_fmt = file_.getvalue() @@ -348,12 +2019,12 @@ def check_traceback_format(self, cleanup_func=None): # Make sure that the traceback is properly indented. tb_lines = python_fmt.splitlines() - self.assertEqual(len(tb_lines), 5) banner = tb_lines[0] - location, source_line = tb_lines[-2:] - self.assertTrue(banner.startswith('Traceback')) - self.assertTrue(location.startswith(' File')) - self.assertTrue(source_line.startswith(' raise')) + self.assertEqual(len(tb_lines), 5) + location, source_line = tb_lines[-2], tb_lines[-1] + self.assertStartsWith(banner, 'Traceback') + self.assertStartsWith(location, ' File') + self.assertStartsWith(source_line, ' raise') def test_traceback_format(self): self.check_traceback_format() @@ -414,12 +2085,16 @@ def f(): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_f+5}, in _check_recursive_traceback_display\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_f+1}, in f\n' ' f()\n' + ' ~^^\n' # XXX: The following line changes depending on whether the tests # are run through the interactive interpreter or with -m # It also varies depending on the platform (stack size) @@ -428,7 +2103,7 @@ def f(): 'RecursionError: maximum recursion depth exceeded\n' ) - expected = result_f.splitlines() + expected = self._maybe_filter_debug_ranges(result_f.splitlines()) actual = stderr_f.getvalue().splitlines() # Check the output text matches expectations @@ -445,7 +2120,7 @@ def f(): # Check a known (limited) number of recursive invocations def g(count=10): if count: - return g(count-1) + return g(count-1) + 1 raise ValueError with captured_output("stderr") as stderr_g: @@ -459,11 +2134,14 @@ def g(count=10): lineno_g = g.__code__.co_firstlineno result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -473,8 +2151,9 @@ def g(count=10): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_g+7}, in _check_recursive_traceback_display\n' ' g()\n' + ' ~^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() self.assertEqual(actual, expected) @@ -497,6 +2176,7 @@ def h(count=10): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_h+7}, in _check_recursive_traceback_display\n' ' h()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' f' File "{__file__}", line {lineno_h+2}, in h\n' @@ -506,8 +2186,9 @@ def h(count=10): ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_h+3}, in h\n' ' g()\n' + ' ~^^\n' ) - expected = (result_h + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((result_h + result_g).splitlines()) actual = stderr_h.getvalue().splitlines() self.assertEqual(actual, expected) @@ -521,21 +2202,25 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' 'ValueError\n' ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+71}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' + ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() self.assertEqual(actual, expected) @@ -549,11 +2234,14 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' ' [Previous line repeated 1 more time]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -561,23 +2249,23 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+99}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+109}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' + ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() self.assertEqual(actual, expected) - def test_recursive_traceback_python(self): - self._check_recursive_traceback_display(traceback.print_exc) - - @cpython_only - def test_recursive_traceback_cpython_internal(self): - from _testcapi import exception_print - def render_exc(): - exc_type, exc_value, exc_tb = sys.exc_info() - exception_print(exc_value) - self._check_recursive_traceback_display(render_exc) + @requires_debug_ranges() + def test_recursive_traceback(self): + if self.DEBUG_RANGES: + self._check_recursive_traceback_display(traceback.print_exc) + else: + from _testcapi import exception_print + def render_exc(): + exception_print(sys.exception()) + self._check_recursive_traceback_display(render_exc) def test_format_stack(self): def fmt(): @@ -606,8 +2294,8 @@ def __eq__(self, other): except UnhashableException: try: raise ex1 - except UnhashableException: - exc_type, exc_val, exc_tb = sys.exc_info() + except UnhashableException as e: + exc_val = e with captured_output("stderr") as stderr_f: exception_print(exc_val) @@ -618,6 +2306,53 @@ def __eq__(self, other): self.assertIn('UnhashableException: ex2', tb[3]) self.assertIn('UnhashableException: ex1', tb[10]) + def deep_eg(self): + e = TypeError(1) + for i in range(2000): + e = ExceptionGroup('eg', [e]) + return e + + @cpython_only + @support.skip_emscripten_stack_overflow() + def test_exception_group_deep_recursion_capi(self): + from _testcapi import exception_print + LIMIT = 75 + eg = self.deep_eg() + with captured_output("stderr") as stderr_f: + with support.infinite_recursion(max_depth=LIMIT): + exception_print(eg) + output = stderr_f.getvalue() + self.assertIn('ExceptionGroup', output) + self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + + @support.skip_emscripten_stack_overflow() + def test_exception_group_deep_recursion_traceback(self): + LIMIT = 75 + eg = self.deep_eg() + with captured_output("stderr") as stderr_f: + with support.infinite_recursion(max_depth=LIMIT): + traceback.print_exception(type(eg), eg, eg.__traceback__) + output = stderr_f.getvalue() + self.assertIn('ExceptionGroup', output) + self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + + @cpython_only + def test_print_exception_bad_type_capi(self): + from _testcapi import exception_print + with captured_output("stderr") as stderr: + with support.catch_unraisable_exception(): + exception_print(42) + self.assertEqual( + stderr.getvalue(), + ('TypeError: print_exception(): ' + 'Exception expected for value, int found\n') + ) + + def test_print_exception_bad_type_python(self): + msg = "Exception expected for value, int found" + with self.assertRaisesRegex(TypeError, msg): + traceback.print_exception(42) + cause_message = ( "\nThe above exception was the direct cause " @@ -630,25 +2365,50 @@ def __eq__(self, other): boundaries = re.compile( '(%s|%s)' % (re.escape(cause_message), re.escape(context_message))) +@force_not_colorized_test_class +class TestTracebackFormat(unittest.TestCase, TracebackFormatMixin): + pass + +@cpython_only +@force_not_colorized_test_class +class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixin): + DEBUG_RANGES = False + def setUp(self) -> None: + self.original_unraisable_hook = sys.unraisablehook + sys.unraisablehook = lambda *args: None + self.original_hook = traceback._print_exception_bltin + traceback._print_exception_bltin = lambda *args: 1/0 + return super().setUp() + + def tearDown(self) -> None: + traceback._print_exception_bltin = self.original_hook + sys.unraisablehook = self.original_unraisable_hook + return super().tearDown() class BaseExceptionReportingTests: def get_exception(self, exception_or_callable): - if isinstance(exception_or_callable, Exception): + if isinstance(exception_or_callable, BaseException): return exception_or_callable try: exception_or_callable() except Exception as e: return e + callable_line = get_exception.__code__.co_firstlineno + 4 + def zero_div(self): 1/0 # In zero_div def check_zero_div(self, msg): lines = msg.splitlines() - self.assertTrue(lines[-3].startswith(' File')) - self.assertIn('1/0 # In zero_div', lines[-2]) - self.assertTrue(lines[-1].startswith('ZeroDivisionError'), lines[-1]) + if has_no_debug_ranges(): + self.assertStartsWith(lines[-3], ' File') + self.assertIn('1/0 # In zero_div', lines[-2]) + else: + self.assertStartsWith(lines[-4], ' File') + self.assertIn('1/0 # In zero_div', lines[-3]) + self.assertStartsWith(lines[-1], 'ZeroDivisionError') def test_simple(self): try: @@ -656,11 +2416,15 @@ def test_simple(self): except ZeroDivisionError as _: e = _ lines = self.get_report(e).splitlines() - self.assertEqual(len(lines), 4) - self.assertTrue(lines[0].startswith('Traceback')) - self.assertTrue(lines[1].startswith(' File')) + if has_no_debug_ranges(): + self.assertEqual(len(lines), 4) + self.assertStartsWith(lines[3], 'ZeroDivisionError') + else: + self.assertEqual(len(lines), 5) + self.assertStartsWith(lines[4], 'ZeroDivisionError') + self.assertStartsWith(lines[0], 'Traceback') + self.assertStartsWith(lines[1], ' File') self.assertIn('1/0 # Marker', lines[2]) - self.assertTrue(lines[3].startswith('ZeroDivisionError')) def test_cause(self): def inner_raise(): @@ -694,16 +2458,16 @@ def test_context_suppression(self): try: try: raise Exception - except: + except Exception: raise ZeroDivisionError from None except ZeroDivisionError as _: e = _ lines = self.get_report(e).splitlines() self.assertEqual(len(lines), 4) - self.assertTrue(lines[0].startswith('Traceback')) - self.assertTrue(lines[1].startswith(' File')) + self.assertStartsWith(lines[3], 'ZeroDivisionError') + self.assertStartsWith(lines[0], 'Traceback') + self.assertStartsWith(lines[1], ' File') self.assertIn('ZeroDivisionError from None', lines[2]) - self.assertTrue(lines[3].startswith('ZeroDivisionError')) def test_cause_and_context(self): # When both a cause and a context are set, only the cause should be @@ -748,8 +2512,6 @@ def outer_raise(): self.assertIn('inner_raise() # Marker', blocks[2]) self.check_zero_div(blocks[2]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_syntax_error_offset_at_eol(self): # See #10186. def e(): @@ -797,6 +2559,171 @@ def test_message_none(self): err = self.get_report(Exception('')) self.assertIn('Exception\n', err) + def test_syntax_error_various_offsets(self): + for offset in range(-5, 10): + for add in [0, 2]: + text = " " * add + "text%d" % offset + expected = [' File "file.py", line 1'] + if offset < 1: + expected.append(" %s" % text.lstrip()) + elif offset <= 6: + expected.append(" %s" % text.lstrip()) + # Set the caret length to match the length of the text minus the offset. + caret_length = max(1, len(text.lstrip()) - offset + 1) + expected.append(" %s%s" % (" " * (offset - 1), "^" * caret_length)) + else: + caret_length = max(1, len(text.lstrip()) - 4) + expected.append(" %s" % text.lstrip()) + expected.append(" %s%s" % (" " * 5, "^" * caret_length)) + expected.append("SyntaxError: msg") + expected.append("") + err = self.get_report(SyntaxError("msg", ("file.py", 1, offset + add, text))) + exp = "\n".join(expected) + self.assertEqual(exp, err) + + def test_exception_with_note(self): + e = ValueError(123) + vanilla = self.get_report(e) + + e.add_note('My Note') + self.assertEqual(self.get_report(e), vanilla + 'My Note\n') + + del e.__notes__ + e.add_note('') + self.assertEqual(self.get_report(e), vanilla + '\n') + + del e.__notes__ + e.add_note('Your Note') + self.assertEqual(self.get_report(e), vanilla + 'Your Note\n') + + del e.__notes__ + self.assertEqual(self.get_report(e), vanilla) + + def test_exception_with_invalid_notes(self): + e = ValueError(123) + vanilla = self.get_report(e) + + # non-sequence __notes__ + class BadThing: + def __str__(self): + return 'bad str' + + def __repr__(self): + return 'bad repr' + + # unprintable, non-sequence __notes__ + class Unprintable: + def __repr__(self): + raise ValueError('bad value') + + e.__notes__ = BadThing() + notes_repr = 'bad repr' + self.assertEqual(self.get_report(e), vanilla + notes_repr + '\n') + + e.__notes__ = Unprintable() + err_msg = '<__notes__ repr() failed>' + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + # non-string item in the __notes__ sequence + e.__notes__ = [BadThing(), 'Final Note'] + bad_note = 'bad str' + self.assertEqual(self.get_report(e), vanilla + bad_note + '\nFinal Note\n') + + # unprintable, non-string item in the __notes__ sequence + e.__notes__ = [Unprintable(), 'Final Note'] + err_msg = '<note str() failed>' + self.assertEqual(self.get_report(e), vanilla + err_msg + '\nFinal Note\n') + + e.__notes__ = "please do not explode me" + err_msg = "'please do not explode me'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + e.__notes__ = b"please do not show me as numbers" + err_msg = "b'please do not show me as numbers'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + # an exception with a broken __getattr__ raising a non expected error + class BrokenException(Exception): + broken = False + def __getattr__(self, name): + if self.broken: + raise ValueError(f'no {name}') + + e = BrokenException(123) + vanilla = self.get_report(e) + e.broken = True + self.assertEqual( + self.get_report(e), + vanilla + "Ignored error getting __notes__: ValueError('no __notes__')\n") + + def test_exception_with_multiple_notes(self): + for e in [ValueError(42), SyntaxError('bad syntax')]: + with self.subTest(e=e): + vanilla = self.get_report(e) + + e.add_note('Note 1') + e.add_note('Note 2') + e.add_note('Note 3') + + self.assertEqual( + self.get_report(e), + vanilla + 'Note 1\n' + 'Note 2\n' + 'Note 3\n') + + del e.__notes__ + e.add_note('Note 4') + del e.__notes__ + e.add_note('Note 5') + e.add_note('Note 6') + + self.assertEqual( + self.get_report(e), + vanilla + 'Note 5\n' + 'Note 6\n') + + def test_exception_qualname(self): + class A: + class B: + class X(Exception): + def __str__(self): + return "I am X" + + err = self.get_report(A.B.X()) + str_value = 'I am X' + str_name = '.'.join([A.B.X.__module__, A.B.X.__qualname__]) + exp = "%s: %s\n" % (str_name, str_value) + self.assertEqual(exp, MODULE_PREFIX + err) + + def test_exception_modulename(self): + class X(Exception): + def __str__(self): + return "I am X" + + for modulename in '__main__', 'builtins', 'some_module': + X.__module__ = modulename + with self.subTest(modulename=modulename): + err = self.get_report(X()) + str_value = 'I am X' + if modulename in ['builtins', '__main__']: + str_name = X.__qualname__ + else: + str_name = '.'.join([X.__module__, X.__qualname__]) + exp = "%s: %s\n" % (str_name, str_value) + self.assertEqual(exp, err) + + def test_exception_angle_bracketed_filename(self): + src = textwrap.dedent(""" + try: + raise ValueError(42) + except Exception as e: + exc = e + """) + + code = compile(src, "<does not exist>", "exec") + g, l = {}, {} + exec(code, g, l) + err = self.get_report(l['exc']) + exp = ' File "<does not exist>", line 3, in <module>\nValueError: 42\n' + self.assertIn(exp, err) + def test_exception_modulename_not_unicode(self): class X(Exception): def __str__(self): @@ -808,41 +2735,457 @@ def __str__(self): exp = f'<unknown>.{X.__qualname__}: I am X\n' self.assertEqual(exp, err) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_syntax_error_various_offsets(self): - for offset in range(-5, 10): - for add in [0, 2]: - text = " "*add + "text%d" % offset - expected = [' File "file.py", line 1'] - if offset < 1: - expected.append(" %s" % text.lstrip()) - elif offset <= 6: - expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*(offset-1))) - else: - expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*5)) - expected.append("SyntaxError: msg") - expected.append("") - err = self.get_report(SyntaxError("msg", ("file.py", 1, offset+add, text))) - exp = "\n".join(expected) - self.assertEqual(exp, err) + def test_exception_bad__str__(self): + class X(Exception): + def __str__(self): + 1/0 + err = self.get_report(X()) + str_value = '<exception str() failed>' + str_name = '.'.join([X.__module__, X.__qualname__]) + self.assertEqual(MODULE_PREFIX + err, f"{str_name}: {str_value}\n") + + + # #### Exception Groups #### + + def test_exception_group_basic(self): + def exc(): + raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) + + expected = ( + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' + f' | raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' + f' | ExceptionGroup: eg (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_exception_group_cause(self): + def exc(): + EG = ExceptionGroup + try: + raise EG("eg1", [ValueError(1), TypeError(2)]) + except Exception as e: + raise EG("eg2", [ValueError(3), TypeError(4)]) from e + + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 3}, in exc\n' + f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' + f' | ExceptionGroup: eg1 (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n' + f'\n' + f'The above exception was the direct cause of the following exception:\n' + f'\n' + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise EG("eg2", [ValueError(3), TypeError(4)]) from e\n' + f' | ExceptionGroup: eg2 (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 3\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 4\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_exception_group_context_with_context(self): + def exc(): + EG = ExceptionGroup + try: + try: + raise EG("eg1", [ValueError(1), TypeError(2)]) + except EG: + raise EG("eg2", [ValueError(3), TypeError(4)]) + except EG: + raise ImportError(5) + + expected = ( + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 4}, in exc\n' + f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' + f' | ExceptionGroup: eg1 (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' + f' | raise EG("eg2", [ValueError(3), TypeError(4)])\n' + f' | ExceptionGroup: eg2 (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 3\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 4\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + f' exception_or_callable()\n' + f' ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' File "{__file__}", line {exc.__code__.co_firstlineno + 8}, in exc\n' + f' raise ImportError(5)\n' + f'ImportError: 5\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_exception_group_nested(self): + def exc(): + EG = ExceptionGroup + VE = ValueError + TE = TypeError + try: + try: + raise EG("nested", [TE(2), TE(3)]) + except Exception as e: + exc = e + raise EG("eg", [VE(1), exc, VE(4)]) + except EG: + raise EG("top", [VE(5)]) + + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n' + f' | raise EG("eg", [VE(1), exc, VE(4)])\n' + f' | ExceptionGroup: eg (3 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' + f' | raise EG("nested", [TE(2), TE(3)])\n' + f' | ExceptionGroup: nested (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | TypeError: 2\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 3\n' + f' +------------------------------------\n' + f' +---------------- 3 ----------------\n' + f' | ValueError: 4\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 11}, in exc\n' + f' | raise EG("top", [VE(5)])\n' + f' | ExceptionGroup: top (1 sub-exception)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 5\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_exception_group_width_limit(self): + excs = [] + for i in range(1000): + excs.append(ValueError(i)) + eg = ExceptionGroup('eg', excs) + + expected = (' | ExceptionGroup: eg (1000 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 0\n' + ' +---------------- 2 ----------------\n' + ' | ValueError: 1\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: 2\n' + ' +---------------- 4 ----------------\n' + ' | ValueError: 3\n' + ' +---------------- 5 ----------------\n' + ' | ValueError: 4\n' + ' +---------------- 6 ----------------\n' + ' | ValueError: 5\n' + ' +---------------- 7 ----------------\n' + ' | ValueError: 6\n' + ' +---------------- 8 ----------------\n' + ' | ValueError: 7\n' + ' +---------------- 9 ----------------\n' + ' | ValueError: 8\n' + ' +---------------- 10 ----------------\n' + ' | ValueError: 9\n' + ' +---------------- 11 ----------------\n' + ' | ValueError: 10\n' + ' +---------------- 12 ----------------\n' + ' | ValueError: 11\n' + ' +---------------- 13 ----------------\n' + ' | ValueError: 12\n' + ' +---------------- 14 ----------------\n' + ' | ValueError: 13\n' + ' +---------------- 15 ----------------\n' + ' | ValueError: 14\n' + ' +---------------- ... ----------------\n' + ' | and 985 more exceptions\n' + ' +------------------------------------\n') + + report = self.get_report(eg) + self.assertEqual(report, expected) + + def test_exception_group_depth_limit(self): + exc = TypeError('bad type') + for i in range(1000): + exc = ExceptionGroup( + f'eg{i}', + [ValueError(i), exc, ValueError(-i)]) + + expected = (' | ExceptionGroup: eg999 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 999\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg998 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 998\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg997 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 997\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg996 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 996\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg995 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 995\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg994 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 994\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg993 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 993\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg992 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 992\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg991 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 991\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg990 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 990\n' + ' +---------------- 2 ----------------\n' + ' | ... (max_group_depth is 10)\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -990\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -991\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -992\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -993\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -994\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -995\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -996\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -997\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -998\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -999\n' + ' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_exception_group_with_notes(self): + def exc(): + try: + excs = [] + for msg in ['bad value', 'terrible value']: + try: + raise ValueError(msg) + except ValueError as e: + e.add_note(f'the {msg}') + excs.append(e) + raise ExceptionGroup("nested", excs) + except ExceptionGroup as e: + e.add_note(('>> Multi line note\n' + '>> Because I am such\n' + '>> an important exception.\n' + '>> empty lines work too\n' + '\n' + '(that was an empty line)')) + raise - def test_format_exception_only_qualname(self): - class A: - class B: - class X(Exception): - def __str__(self): - return "I am X" - pass - err = self.get_report(A.B.X()) - str_value = 'I am X' - str_name = '.'.join([A.B.X.__module__, A.B.X.__qualname__]) - exp = "%s: %s\n" % (str_name, str_value) - self.assertEqual(exp, err) + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n' + f' | raise ExceptionGroup("nested", excs)\n' + f' | ExceptionGroup: nested (2 sub-exceptions)\n' + f' | >> Multi line note\n' + f' | >> Because I am such\n' + f' | >> an important exception.\n' + f' | >> empty lines work too\n' + f' | \n' + f' | (that was an empty line)\n' + f' +-+---------------- 1 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise ValueError(msg)\n' + f' | ValueError: bad value\n' + f' | the bad value\n' + f' +---------------- 2 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise ValueError(msg)\n' + f' | ValueError: terrible value\n' + f' | the terrible value\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_exception_group_with_multiple_notes(self): + def exc(): + try: + excs = [] + for msg in ['bad value', 'terrible value']: + try: + raise ValueError(msg) + except ValueError as e: + e.add_note(f'the {msg}') + e.add_note(f'Goodbye {msg}') + excs.append(e) + raise ExceptionGroup("nested", excs) + except ExceptionGroup as e: + e.add_note(('>> Multi line note\n' + '>> Because I am such\n' + '>> an important exception.\n' + '>> empty lines work too\n' + '\n' + '(that was an empty line)')) + e.add_note('Goodbye!') + raise + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 10}, in exc\n' + f' | raise ExceptionGroup("nested", excs)\n' + f' | ExceptionGroup: nested (2 sub-exceptions)\n' + f' | >> Multi line note\n' + f' | >> Because I am such\n' + f' | >> an important exception.\n' + f' | >> empty lines work too\n' + f' | \n' + f' | (that was an empty line)\n' + f' | Goodbye!\n' + f' +-+---------------- 1 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise ValueError(msg)\n' + f' | ValueError: bad value\n' + f' | the bad value\n' + f' | Goodbye bad value\n' + f' +---------------- 2 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise ValueError(msg)\n' + f' | ValueError: terrible value\n' + f' | the terrible value\n' + f' | Goodbye terrible value\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_exception_group_wrapped_naked(self): + # See gh-128799 + + def exc(): + try: + raise Exception(42) + except* Exception as e: + raise + + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 3}, in exc\n' + f' | except* Exception as e:\n' + f' | raise\n' + f' | ExceptionGroup: (1 sub-exception)\n' + f' +-+---------------- 1 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 2}, in exc\n' + f' | raise Exception(42)\n' + f' | Exception: 42\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_KeyboardInterrupt_at_first_line_of_frame(self): + # see GH-93249 + def f(): + return sys._getframe() + + tb_next = None + frame = f() + lasti = 0 + lineno = f.__code__.co_firstlineno + tb = types.TracebackType(tb_next, frame, lasti, lineno) + + exc = KeyboardInterrupt() + exc.__traceback__ = tb + + expected = (f'Traceback (most recent call last):\n' + f' File "{__file__}", line {lineno}, in f\n' + f' def f():\n' + f'\n' + f'KeyboardInterrupt\n') + + report = self.get_report(exc) + # remove trailing writespace: + report = '\n'.join([l.rstrip() for l in report.split('\n')]) + self.assertEqual(report, expected) + +@force_not_colorized_test_class class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # # This checks reporting through the 'traceback' module, with both @@ -858,7 +3201,12 @@ def get_report(self, e): self.assertEqual(sio.getvalue(), s) return s + @unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 1103 characters long. Set self.maxDiff to None to see it. + def test_exception_group_wrapped_naked(self): + return super().test_exception_group_wrapped_naked() + +@force_not_colorized_test_class class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # # This checks built-in reporting by the interpreter. @@ -908,8 +3256,7 @@ def last_returns_frame4(self): def last_returns_frame5(self): return self.last_returns_frame4() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 not greater than 5 def test_extract_stack(self): frame = self.last_returns_frame5() def extract(**kwargs): @@ -941,8 +3288,8 @@ def assertEqualExcept(actual, expected, ignore): def test_extract_tb(self): try: self.last_raises5() - except Exception: - exc_type, exc_value, tb = sys.exc_info() + except Exception as e: + tb = e.__traceback__ def extract(**kwargs): return traceback.extract_tb(tb, **kwargs) @@ -968,12 +3315,12 @@ def extract(**kwargs): def test_format_exception(self): try: self.last_raises5() - except Exception: - exc_type, exc_value, tb = sys.exc_info() + except Exception as e: + exc = e # [1:-1] to exclude "Traceback (...)" header and # exception type and value def extract(**kwargs): - return traceback.format_exception(exc_type, exc_value, tb, **kwargs)[1:-1] + return traceback.format_exception(exc, **kwargs)[1:-1] with support.swap_attr(sys, 'tracebacklimit', 1000): nolim = extract() @@ -1000,8 +3347,6 @@ class MiscTracebackCases(unittest.TestCase): # Check non-printing functions in traceback module # - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_clear(self): def outer(): middle() @@ -1013,8 +3358,8 @@ def inner(): try: outer() - except: - type_, value, tb = sys.exc_info() + except BaseException as e: + tb = e.__traceback__ # Initial assertion: there's one local in the inner frame. inner_frame = tb.tb_next.tb_next.tb_next.tb_frame @@ -1040,8 +3385,6 @@ def extract(): class TestFrame(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basics(self): linecache.clearcache() linecache.lazycache("f", globals()) @@ -1059,12 +3402,10 @@ def test_basics(self): self.assertNotEqual(f, object()) self.assertEqual(f, ALWAYS_EQ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lazy_lines(self): linecache.clearcache() f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False) - self.assertEqual(None, f._line) + self.assertEqual(None, f._lines) linecache.lazycache("f", globals()) self.assertEqual( '"""Test cases for traceback module"""', @@ -1088,16 +3429,22 @@ class TestStack(unittest.TestCase): def test_walk_stack(self): def deeper(): return list(traceback.walk_stack(None)) - s1 = list(traceback.walk_stack(None)) - s2 = deeper() + s1, s2 = list(traceback.walk_stack(None)), deeper() self.assertEqual(len(s2) - len(s1), 1) self.assertEqual(s2[1:], s1) + def test_walk_innermost_frame(self): + def inner(): + return list(traceback.walk_stack(None)) + frames = inner() + innermost_frame, _ = frames[0] + self.assertEqual(innermost_frame.f_code.co_name, "inner") + def test_walk_tb(self): try: 1/0 - except Exception: - _, _, tb = sys.exc_info() + except Exception as e: + tb = e.__traceback__ s = list(traceback.walk_tb(tb)) self.assertEqual(len(s), 1) @@ -1109,8 +3456,6 @@ def test_extract_stack_limit(self): s = traceback.StackSummary.extract(traceback.walk_stack(None), limit=5) self.assertEqual(len(s), 5) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_extract_stack_lookup_lines(self): linecache.clearcache() linecache.updatecache('/foo.py', globals()) @@ -1120,8 +3465,6 @@ def test_extract_stack_lookup_lines(self): linecache.clearcache() self.assertEqual(s[0].line, "import sys") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_extract_stackup_deferred_lookup_lines(self): linecache.clearcache() c = test_code('/foo.py', 'method') @@ -1153,8 +3496,6 @@ def test_format_smoke(self): [' File "foo.py", line 1, in fred\n line\n'], s.format()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_locals(self): linecache.updatecache('/foo.py', globals()) c = test_code('/foo.py', 'method') @@ -1162,8 +3503,6 @@ def test_locals(self): s = traceback.StackSummary.extract(iter([(f, 6)]), capture_locals=True) self.assertEqual(s[0].locals, {'something': '1'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_locals(self): linecache.updatecache('/foo.py', globals()) c = test_code('/foo.py', 'method') @@ -1171,8 +3510,6 @@ def test_no_locals(self): s = traceback.StackSummary.extract(iter([(f, 6)])) self.assertEqual(s[0].locals, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_format_locals(self): def some_inner(k, v): a = 1 @@ -1189,22 +3526,132 @@ def some_inner(k, v): ' v = 4\n' % (__file__, some_inner.__code__.co_firstlineno + 3) ], s.format()) -class TestTracebackException(unittest.TestCase): + def test_custom_format_frame(self): + class CustomStackSummary(traceback.StackSummary): + def format_frame_summary(self, frame_summary, colorize=False): + return f'{frame_summary.filename}:{frame_summary.lineno}' - def test_smoke(self): - try: + def some_inner(): + return CustomStackSummary.extract( + traceback.walk_stack(None), limit=1) + + s = some_inner() + self.assertEqual( + s.format(), + [f'{__file__}:{some_inner.__code__.co_firstlineno + 1}']) + + def test_dropping_frames(self): + def f(): 1/0 - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) + + def g(): + try: + f() + except Exception as e: + return e.__traceback__ + + tb = g() + + class Skip_G(traceback.StackSummary): + def format_frame_summary(self, frame_summary, colorize=False): + if frame_summary.name == 'g': + return None + return super().format_frame_summary(frame_summary) + + stack = Skip_G.extract( + traceback.walk_tb(tb)).format() + + self.assertEqual(len(stack), 1) + lno = f.__code__.co_firstlineno + 1 + self.assertEqual( + stack[0], + f' File "{__file__}", line {lno}, in f\n 1/0\n' + ) + + def test_summary_should_show_carets(self): + # See: https://github.com/python/cpython/issues/122353 + + # statement to execute and to get a ZeroDivisionError for a traceback + statement = "abcdef = 1 / 0 and 2.0" + colno = statement.index('1 / 0') + end_colno = colno + len('1 / 0') + + # Actual line to use when rendering the traceback + # and whose AST will be extracted (it will be empty). + cached_line = '# this line will be used during rendering' + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as file: + file.write(cached_line) + linecache.updatecache(TESTFN, {}) + + try: + exec(compile(statement, TESTFN, "exec")) + except ZeroDivisionError as exc: + # This is the simplest way to create a StackSummary + # whose FrameSummary items have their column offsets. + s = traceback.TracebackException.from_exception(exc).stack + self.assertIsInstance(s, traceback.StackSummary) + with unittest.mock.patch.object(s, '_should_show_carets', + wraps=s._should_show_carets) as ff: + self.assertEqual(len(s), 2) + self.assertListEqual( + s.format_frame_summary(s[1]).splitlines(), + [ + f' File "{TESTFN}", line 1, in <module>', + f' {cached_line}' + ] + ) + ff.assert_called_with(colno, end_colno, [cached_line], None) + +class Unrepresentable: + def __repr__(self) -> str: + raise Exception("Unrepresentable") + + +# Used in test_dont_swallow_cause_or_context_of_falsey_exception and +# test_dont_swallow_subexceptions_of_falsey_exceptiongroup. +class FalseyException(Exception): + def __bool__(self): + return False + + +class FalseyExceptionGroup(ExceptionGroup): + def __bool__(self): + return False + + +class TestTracebackException(unittest.TestCase): + def do_test_smoke(self, exc, expected_type_str): + try: + raise exc + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(e) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(e.__traceback__)) self.assertEqual(None, exc.__cause__) self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(expected_type_str, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) + + def test_smoke_builtin(self): + self.do_test_smoke(ValueError(42), 'ValueError') + + def test_smoke_user_exception(self): + class MyException(Exception): + pass + + if __name__ == '__main__': + expected = ('TestTracebackException.' + 'test_smoke_user_exception.<locals>.MyException') + else: + expected = ('test.test_traceback.TestTracebackException.' + 'test_smoke_user_exception.<locals>.MyException') + self.do_test_smoke(MyException('bad things happened'), expected) def test_from_exception(self): # Check all the parameters are accepted. @@ -1213,9 +3660,10 @@ def foo(): try: foo() except Exception as e: - exc_info = sys.exc_info() + exc_obj = e + tb = e.__traceback__ self.expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2]), limit=1, lookup_lines=False, + traceback.walk_tb(tb), limit=1, lookup_lines=False, capture_locals=True) self.exc = traceback.TracebackException.from_exception( e, limit=1, lookup_lines=False, capture_locals=True) @@ -1225,68 +3673,72 @@ def foo(): self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) def test_cause(self): try: try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) cause = Exception("cause") raise Exception("uh oh") from cause - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(e) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(e.__traceback__)) exc_cause = traceback.TracebackException(Exception, cause, None) self.assertEqual(exc_cause, exc.__cause__) self.assertEqual(exc_context, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) def test_context(self): try: try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) raise Exception("uh oh") - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(e) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(e.__traceback__)) self.assertEqual(None, exc.__cause__) self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_long_context_chain(self): def f(): try: 1/0 - except: + except ZeroDivisionError: f() try: f() - except RecursionError: - exc_info = sys.exc_info() + except RecursionError as e: + exc_obj = e else: self.fail("Exception not raised") - te = traceback.TracebackException(*exc_info) + te = traceback.TracebackException.from_exception(exc_obj) res = list(te.format()) # many ZeroDiv errors followed by the RecursionError @@ -1304,70 +3756,84 @@ def test_compact_with_cause(self): finally: cause = Exception("cause") raise Exception("uh oh") from cause - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info, compact=True) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(exc_obj, compact=True) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(exc_obj.__traceback__)) exc_cause = traceback.TracebackException(Exception, cause, None) self.assertEqual(exc_cause, exc.__cause__) self.assertEqual(None, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) def test_compact_no_cause(self): try: try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) raise Exception("uh oh") - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info, compact=True) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(e, compact=True) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(exc_obj.__traceback__)) self.assertEqual(None, exc.__cause__) self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) + + def test_no_save_exc_type(self): + try: + 1/0 + except Exception as e: + exc = e + + te = traceback.TracebackException.from_exception( + exc, save_exc_type=False) + with self.assertWarns(DeprecationWarning): + self.assertIsNone(te.exc_type) def test_no_refs_to_exception_and_traceback_objects(self): + exc_obj = None try: 1/0 - except Exception: - exc_info = sys.exc_info() + except Exception as e: + exc_obj = e - refcnt1 = sys.getrefcount(exc_info[1]) - refcnt2 = sys.getrefcount(exc_info[2]) - exc = traceback.TracebackException(*exc_info) - self.assertEqual(sys.getrefcount(exc_info[1]), refcnt1) - self.assertEqual(sys.getrefcount(exc_info[2]), refcnt2) + refcnt1 = sys.getrefcount(exc_obj) + refcnt2 = sys.getrefcount(exc_obj.__traceback__) + exc = traceback.TracebackException.from_exception(exc_obj) + self.assertEqual(sys.getrefcount(exc_obj), refcnt1) + self.assertEqual(sys.getrefcount(exc_obj.__traceback__), refcnt2) def test_comparison_basic(self): try: 1/0 - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) - exc2 = traceback.TracebackException(*exc_info) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(exc_obj) + exc2 = traceback.TracebackException.from_exception(exc_obj) self.assertIsNot(exc, exc2) self.assertEqual(exc, exc2) self.assertNotEqual(exc, object()) self.assertEqual(exc, ALWAYS_EQ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_comparison_params_variations(self): def raise_exc(): try: raise ValueError('bad value') - except: + except ValueError: raise def raise_with_locals(): @@ -1376,28 +3842,28 @@ def raise_with_locals(): try: raise_with_locals() - except Exception: - exc_info = sys.exc_info() + except Exception as e: + exc_obj = e - exc = traceback.TracebackException(*exc_info) - exc1 = traceback.TracebackException(*exc_info, limit=10) - exc2 = traceback.TracebackException(*exc_info, limit=2) + exc = traceback.TracebackException.from_exception(exc_obj) + exc1 = traceback.TracebackException.from_exception(exc_obj, limit=10) + exc2 = traceback.TracebackException.from_exception(exc_obj, limit=2) self.assertEqual(exc, exc1) # limit=10 gets all frames self.assertNotEqual(exc, exc2) # limit=2 truncates the output # locals change the output - exc3 = traceback.TracebackException(*exc_info, capture_locals=True) + exc3 = traceback.TracebackException.from_exception(exc_obj, capture_locals=True) self.assertNotEqual(exc, exc3) # there are no locals in the innermost frame - exc4 = traceback.TracebackException(*exc_info, limit=-1) - exc5 = traceback.TracebackException(*exc_info, limit=-1, capture_locals=True) + exc4 = traceback.TracebackException.from_exception(exc_obj, limit=-1) + exc5 = traceback.TracebackException.from_exception(exc_obj, limit=-1, capture_locals=True) self.assertEqual(exc4, exc5) # there are locals in the next-to-innermost frame - exc6 = traceback.TracebackException(*exc_info, limit=-2) - exc7 = traceback.TracebackException(*exc_info, limit=-2, capture_locals=True) + exc6 = traceback.TracebackException.from_exception(exc_obj, limit=-2) + exc7 = traceback.TracebackException.from_exception(exc_obj, limit=-2, capture_locals=True) self.assertNotEqual(exc6, exc7) def test_comparison_equivalent_exceptions_are_equal(self): @@ -1405,8 +3871,8 @@ def test_comparison_equivalent_exceptions_are_equal(self): for _ in range(2): try: 1/0 - except: - excs.append(traceback.TracebackException(*sys.exc_info())) + except Exception as e: + excs.append(traceback.TracebackException.from_exception(e)) self.assertEqual(excs[0], excs[1]) self.assertEqual(list(excs[0].format()), list(excs[1].format())) @@ -1422,9 +3888,9 @@ def __eq__(self, other): except UnhashableException: try: raise ex1 - except UnhashableException: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) + except UnhashableException as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(exc_obj) formatted = list(exc.format()) self.assertIn('UnhashableException: ex2\n', formatted[2]) self.assertIn('UnhashableException: ex1\n', formatted[6]) @@ -1437,47 +3903,41 @@ def recurse(n): 1/0 try: recurse(10) - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info, limit=5) + except Exception as e: + exc = traceback.TracebackException.from_exception(e, limit=5) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2]), limit=5) + traceback.walk_tb(e.__traceback__), limit=5) self.assertEqual(expected_stack, exc.stack) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lookup_lines(self): linecache.clearcache() e = Exception("uh oh") c = test_code('/foo.py', 'method') f = test_frame(c, None, None) - tb = test_tb(f, 6, None) + tb = test_tb(f, 6, None, 0) exc = traceback.TracebackException(Exception, e, tb, lookup_lines=False) self.assertEqual(linecache.cache, {}) linecache.updatecache('/foo.py', globals()) self.assertEqual(exc.stack[0].line, "import sys") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_locals(self): linecache.updatecache('/foo.py', globals()) e = Exception("uh oh") c = test_code('/foo.py', 'method') - f = test_frame(c, globals(), {'something': 1, 'other': 'string'}) - tb = test_tb(f, 6, None) + f = test_frame(c, globals(), {'something': 1, 'other': 'string', 'unrepresentable': Unrepresentable()}) + tb = test_tb(f, 6, None, 0) exc = traceback.TracebackException( Exception, e, tb, capture_locals=True) self.assertEqual( - exc.stack[0].locals, {'something': '1', 'other': "'string'"}) + exc.stack[0].locals, + {'something': '1', 'other': "'string'", 'unrepresentable': '<local repr() failed>'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_locals(self): linecache.updatecache('/foo.py', globals()) e = Exception("uh oh") c = test_code('/foo.py', 'method') f = test_frame(c, globals(), {'something': 1}) - tb = test_tb(f, 6, None) + tb = test_tb(f, 6, None, 0) exc = traceback.TracebackException(Exception, e, tb) self.assertEqual(exc.stack[0].locals, None) @@ -1487,20 +3947,1126 @@ def test_traceback_header(self): exc = traceback.TracebackException(Exception, Exception("haven"), None) self.assertEqual(list(exc.format()), ["Exception: haven\n"]) + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^ + + @requires_debug_ranges() + def test_print(self): + def f(): + x = 12 + try: + x/0 + except Exception as e: + return e + exc = traceback.TracebackException.from_exception(f(), capture_locals=True) + output = StringIO() + exc.print(file=output) + self.assertEqual( + output.getvalue().split('\n')[-5:], + [' x/0', + ' ~^~', + ' x = 12', + 'ZeroDivisionError: division by zero', + '']) + + def test_dont_swallow_cause_or_context_of_falsey_exception(self): + # see gh-132308: Ensure that __cause__ or __context__ attributes of exceptions + # that evaluate as falsey are included in the output. For falsey term, + # see https://docs.python.org/3/library/stdtypes.html#truth-value-testing. + + try: + raise FalseyException from KeyError + except FalseyException as e: + self.assertIn(cause_message, traceback.format_exception(e)) + + try: + try: + 1/0 + except ZeroDivisionError: + raise FalseyException + except FalseyException as e: + self.assertIn(context_message, traceback.format_exception(e)) + + +class TestTracebackException_ExceptionGroups(unittest.TestCase): + def setUp(self): + super().setUp() + self.eg = self._get_exception_group() + + def _get_exception_group(self): + def f(): + 1/0 + + def g(v): + raise ValueError(v) + + self.lno_f = f.__code__.co_firstlineno + self.lno_g = g.__code__.co_firstlineno + + try: + try: + try: + f() + except Exception as e: + exc1 = e + try: + g(42) + except Exception as e: + exc2 = e + raise ExceptionGroup("eg1", [exc1, exc2]) + except ExceptionGroup as e: + exc3 = e + try: + g(24) + except Exception as e: + exc4 = e + raise ExceptionGroup("eg2", [exc3, exc4]) + except ExceptionGroup as eg: + return eg + self.fail('Exception Not Raised') + + def test_exception_group_construction(self): + eg = self.eg + teg1 = traceback.TracebackException(type(eg), eg, eg.__traceback__) + teg2 = traceback.TracebackException.from_exception(eg) + self.assertIsNot(teg1, teg2) + self.assertEqual(teg1, teg2) + + def test_exception_group_format_exception_only(self): + teg = traceback.TracebackException.from_exception(self.eg) + formatted = ''.join(teg.format_exception_only()).split('\n') + expected = "ExceptionGroup: eg2 (2 sub-exceptions)\n".split('\n') + + self.assertEqual(formatted, expected) + + def test_exception_group_format_exception_onlyi_recursive(self): + teg = traceback.TracebackException.from_exception(self.eg) + formatted = ''.join(teg.format_exception_only(show_group=True)).split('\n') + expected = [ + 'ExceptionGroup: eg2 (2 sub-exceptions)', + ' ExceptionGroup: eg1 (2 sub-exceptions)', + ' ZeroDivisionError: division by zero', + ' ValueError: 42', + ' ValueError: 24', + '' + ] + + self.assertEqual(formatted, expected) + + def test_exception_group_format(self): + teg = traceback.TracebackException.from_exception(self.eg) + + formatted = ''.join(teg.format()).split('\n') + lno_f = self.lno_f + lno_g = self.lno_g + + expected = [ + f' + Exception Group Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+23}, in _get_exception_group', + f' | raise ExceptionGroup("eg2", [exc3, exc4])', + f' | ExceptionGroup: eg2 (2 sub-exceptions)', + f' +-+---------------- 1 ----------------', + f' | Exception Group Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+16}, in _get_exception_group', + f' | raise ExceptionGroup("eg1", [exc1, exc2])', + f' | ExceptionGroup: eg1 (2 sub-exceptions)', + f' +-+---------------- 1 ----------------', + f' | Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+9}, in _get_exception_group', + f' | f()', + f' | ~^^', + f' | File "{__file__}", line {lno_f+1}, in f', + f' | 1/0', + f' | ~^~', + f' | ZeroDivisionError: division by zero', + f' +---------------- 2 ----------------', + f' | Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+13}, in _get_exception_group', + f' | g(42)', + f' | ~^^^^', + f' | File "{__file__}", line {lno_g+1}, in g', + f' | raise ValueError(v)', + f' | ValueError: 42', + f' +------------------------------------', + f' +---------------- 2 ----------------', + f' | Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+20}, in _get_exception_group', + f' | g(24)', + f' | ~^^^^', + f' | File "{__file__}", line {lno_g+1}, in g', + f' | raise ValueError(v)', + f' | ValueError: 24', + f' +------------------------------------', + f''] + + self.assertEqual(formatted, expected) + + def test_max_group_width(self): + excs1 = [] + excs2 = [] + for i in range(3): + excs1.append(ValueError(i)) + for i in range(10): + excs2.append(TypeError(i)) + + EG = ExceptionGroup + eg = EG('eg', [EG('eg1', excs1), EG('eg2', excs2)]) + + teg = traceback.TracebackException.from_exception(eg, max_group_width=2) + formatted = ''.join(teg.format()).split('\n') + + expected = [ + ' | ExceptionGroup: eg (2 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | ExceptionGroup: eg1 (3 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | ValueError: 0', + ' +---------------- 2 ----------------', + ' | ValueError: 1', + ' +---------------- ... ----------------', + ' | and 1 more exception', + ' +------------------------------------', + ' +---------------- 2 ----------------', + ' | ExceptionGroup: eg2 (10 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | TypeError: 0', + ' +---------------- 2 ----------------', + ' | TypeError: 1', + ' +---------------- ... ----------------', + ' | and 8 more exceptions', + ' +------------------------------------', + ''] + + self.assertEqual(formatted, expected) + + def test_max_group_depth(self): + exc = TypeError('bad type') + for i in range(3): + exc = ExceptionGroup('exc', [ValueError(-i), exc, ValueError(i)]) + + teg = traceback.TracebackException.from_exception(exc, max_group_depth=2) + formatted = ''.join(teg.format()).split('\n') + + expected = [ + ' | ExceptionGroup: exc (3 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | ValueError: -2', + ' +---------------- 2 ----------------', + ' | ExceptionGroup: exc (3 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | ValueError: -1', + ' +---------------- 2 ----------------', + ' | ... (max_group_depth is 2)', + ' +---------------- 3 ----------------', + ' | ValueError: 1', + ' +------------------------------------', + ' +---------------- 3 ----------------', + ' | ValueError: 2', + ' +------------------------------------', + ''] + + self.assertEqual(formatted, expected) + + def test_comparison(self): + try: + raise self.eg + except ExceptionGroup as e: + exc = e + for _ in range(5): + try: + raise exc + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(exc_obj) + exc2 = traceback.TracebackException.from_exception(exc_obj) + exc3 = traceback.TracebackException.from_exception(exc_obj, limit=300) + ne = traceback.TracebackException.from_exception(exc_obj, limit=3) + self.assertIsNot(exc, exc2) + self.assertEqual(exc, exc2) + self.assertEqual(exc, exc3) + self.assertNotEqual(exc, ne) + self.assertNotEqual(exc, object()) + self.assertEqual(exc, ALWAYS_EQ) + + def test_dont_swallow_subexceptions_of_falsey_exceptiongroup(self): + # see gh-132308: Ensure that subexceptions of exception groups + # that evaluate as falsey are displayed in the output. For falsey term, + # see https://docs.python.org/3/library/stdtypes.html#truth-value-testing. + + try: + raise FalseyExceptionGroup("Gih", (KeyError(), NameError())) + except Exception as ee: + str_exc = ''.join(traceback.format_exception(ee)) + self.assertIn('+---------------- 1 ----------------', str_exc) + self.assertIn('+---------------- 2 ----------------', str_exc) + + # Test with a falsey exception, in last position, as sub-exceptions. + msg = 'bool' + try: + raise FalseyExceptionGroup("Gah", (KeyError(), FalseyException(msg))) + except Exception as ee: + str_exc = traceback.format_exception(ee) + self.assertIn(f'{FalseyException.__name__}: {msg}', str_exc[-2]) + + +global_for_suggestions = None + + +class SuggestionFormattingTestBase: + def get_suggestion(self, obj, attr_name=None): + if attr_name is not None: + def callable(): + getattr(obj, attr_name) + else: + callable = obj + + result_lines = self.get_exception( + callable, slice_start=-1, slice_end=None + ) + return result_lines[0] + + def test_getattr_suggestions(self): + class Substitution: + noise = more_noise = a = bc = None + blech = None + + class Elimination: + noise = more_noise = a = bc = None + blch = None + + class Addition: + noise = more_noise = a = bc = None + bluchin = None + + class SubstitutionOverElimination: + blach = None + bluc = None + + class SubstitutionOverAddition: + blach = None + bluchi = None + + class EliminationOverAddition: + blucha = None + bluc = None + + class CaseChangeOverSubstitution: + Luch = None + fluch = None + BLuch = None + + for cls, suggestion in [ + (Addition, "'bluchin'?"), + (Substitution, "'blech'?"), + (Elimination, "'blch'?"), + (Addition, "'bluchin'?"), + (SubstitutionOverElimination, "'blach'?"), + (SubstitutionOverAddition, "'blach'?"), + (EliminationOverAddition, "'bluc'?"), + (CaseChangeOverSubstitution, "'BLuch'?"), + ]: + actual = self.get_suggestion(cls(), 'bluch') + self.assertIn(suggestion, actual) + + def test_getattr_suggestions_underscored(self): + class A: + bluch = None + + self.assertIn("'bluch'", self.get_suggestion(A(), 'blach')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + + class B: + _bluch = None + def method(self, name): + getattr(self, name) + + self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) + self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) + self.assertNotIn("'_bluch'", self.get_suggestion(B(), 'bluch')) + + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_blach'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): + class A: + blech = None + + actual = self.get_suggestion(A(), 'somethingverywrong') + self.assertNotIn("blech", actual) + + def test_getattr_error_bad_suggestions_do_not_trigger_for_small_names(self): + class MyClass: + vvv = mom = w = id = pytho = None + + for name in ("b", "v", "m", "py"): + with self.subTest(name=name): + actual = self.get_suggestion(MyClass, name) + self.assertNotIn("Did you mean", actual) + self.assertNotIn("'vvv", actual) + self.assertNotIn("'mom'", actual) + self.assertNotIn("'id'", actual) + self.assertNotIn("'w'", actual) + self.assertNotIn("'pytho'", actual) + + def test_getattr_suggestions_do_not_trigger_for_big_dicts(self): + class A: + blech = None + # A class with a very big __dict__ will not be considered + # for suggestions. + for index in range(2000): + setattr(A, f"index_{index}", None) + + actual = self.get_suggestion(A(), 'bluch') + self.assertNotIn("blech", actual) + + def test_getattr_suggestions_no_args(self): + class A: + blech = None + def __getattr__(self, attr): + raise AttributeError() + + actual = self.get_suggestion(A(), 'bluch') + self.assertIn("blech", actual) + + class A: + blech = None + def __getattr__(self, attr): + raise AttributeError + + actual = self.get_suggestion(A(), 'bluch') + self.assertIn("blech", actual) + + def test_getattr_suggestions_invalid_args(self): + class NonStringifyClass: + __str__ = None + __repr__ = None + + class A: + blech = None + def __getattr__(self, attr): + raise AttributeError(NonStringifyClass()) + + class B: + blech = None + def __getattr__(self, attr): + raise AttributeError("Error", 23) + + class C: + blech = None + def __getattr__(self, attr): + raise AttributeError(23) + + for cls in [A, B, C]: + actual = self.get_suggestion(cls(), 'bluch') + self.assertIn("blech", actual) + + def test_getattr_suggestions_for_same_name(self): + class A: + def __dir__(self): + return ['blech'] + actual = self.get_suggestion(A(), 'blech') + self.assertNotIn("Did you mean", actual) + + def test_attribute_error_with_failing_dict(self): + class T: + bluch = 1 + def __dir__(self): + raise AttributeError("oh no!") + + actual = self.get_suggestion(T(), 'blich') + self.assertNotIn("blech", actual) + self.assertNotIn("oh no!", actual) + + def test_attribute_error_with_non_string_candidates(self): + class T: + bluch = 1 + + instance = T() + instance.__dict__[0] = 1 + actual = self.get_suggestion(instance, 'blich') + self.assertIn("bluch", actual) + + def test_attribute_error_with_bad_name(self): + def raise_attribute_error_with_bad_name(): + raise AttributeError(name=12, obj=23) + + result_lines = self.get_exception( + raise_attribute_error_with_bad_name, slice_start=-1, slice_end=None + ) + self.assertNotIn("?", result_lines[-1]) + + def test_attribute_error_inside_nested_getattr(self): + class A: + bluch = 1 + + class B: + def __getattribute__(self, attr): + a = A() + return a.blich + + actual = self.get_suggestion(B(), 'something') + self.assertIn("Did you mean", actual) + self.assertIn("bluch", actual) + + def make_module(self, code): + tmpdir = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, tmpdir) + + sys.path.append(str(tmpdir)) + self.addCleanup(sys.path.pop) + + mod_name = ''.join(random.choices(string.ascii_letters, k=16)) + module = tmpdir / (mod_name + ".py") + module.write_text(code) + + return mod_name + + def get_import_from_suggestion(self, code, name): + modname = self.make_module(code) + + def callable(): + try: + exec(f"from {modname} import {name}") + except ImportError as e: + raise e from None + except Exception as e: + self.fail(f"Expected ImportError but got {type(e)}") + self.addCleanup(forget, modname) + + result_lines = self.get_exception( + callable, slice_start=-1, slice_end=None + ) + return result_lines[0] + + def test_import_from_suggestions(self): + substitution = textwrap.dedent("""\ + noise = more_noise = a = bc = None + blech = None + """) + + elimination = textwrap.dedent(""" + noise = more_noise = a = bc = None + blch = None + """) + + addition = textwrap.dedent(""" + noise = more_noise = a = bc = None + bluchin = None + """) + + substitutionOverElimination = textwrap.dedent(""" + blach = None + bluc = None + """) + + substitutionOverAddition = textwrap.dedent(""" + blach = None + bluchi = None + """) + + eliminationOverAddition = textwrap.dedent(""" + blucha = None + bluc = None + """) + + caseChangeOverSubstitution = textwrap.dedent(""" + Luch = None + fluch = None + BLuch = None + """) + + for code, suggestion in [ + (addition, "'bluchin'?"), + (substitution, "'blech'?"), + (elimination, "'blch'?"), + (addition, "'bluchin'?"), + (substitutionOverElimination, "'blach'?"), + (substitutionOverAddition, "'blach'?"), + (eliminationOverAddition, "'bluc'?"), + (caseChangeOverSubstitution, "'BLuch'?"), + ]: + actual = self.get_import_from_suggestion(code, 'bluch') + self.assertIn(suggestion, actual) + + def test_import_from_suggestions_underscored(self): + code = "bluch = None" + self.assertIn("'bluch'", self.get_import_from_suggestion(code, 'blach')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_bluch')) + + code = "_bluch = None" + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_blach')) + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + + def test_import_from_suggestions_non_string(self): + modWithNonStringAttr = textwrap.dedent("""\ + globals()[0] = 1 + bluch = 1 + """) + self.assertIn("'bluch'", self.get_import_from_suggestion(modWithNonStringAttr, 'blech')) + + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): + code = "blech = None" + + actual = self.get_suggestion(code, 'somethingverywrong') + self.assertNotIn("blech", actual) + + def test_import_from_error_bad_suggestions_do_not_trigger_for_small_names(self): + code = "vvv = mom = w = id = pytho = None" + + for name in ("b", "v", "m", "py"): + with self.subTest(name=name): + actual = self.get_import_from_suggestion(code, name) + self.assertNotIn("Did you mean", actual) + self.assertNotIn("'vvv'", actual) + self.assertNotIn("'mom'", actual) + self.assertNotIn("'id'", actual) + self.assertNotIn("'w'", actual) + self.assertNotIn("'pytho'", actual) + + def test_import_from_suggestions_do_not_trigger_for_big_namespaces(self): + # A module with lots of names will not be considered for suggestions. + chunks = [f"index_{index} = " for index in range(200)] + chunks.append(" None") + code = " ".join(chunks) + actual = self.get_import_from_suggestion(code, 'bluch') + self.assertNotIn("blech", actual) + + def test_import_from_error_with_bad_name(self): + def raise_attribute_error_with_bad_name(): + raise ImportError(name=12, obj=23, name_from=11) + + result_lines = self.get_exception( + raise_attribute_error_with_bad_name, slice_start=-1, slice_end=None + ) + self.assertNotIn("?", result_lines[-1]) + + def test_name_error_suggestions(self): + def Substitution(): + noise = more_noise = a = bc = None + blech = None + print(bluch) + + def Elimination(): + noise = more_noise = a = bc = None + blch = None + print(bluch) + + def Addition(): + noise = more_noise = a = bc = None + bluchin = None + print(bluch) + + def SubstitutionOverElimination(): + blach = None + bluc = None + print(bluch) + + def SubstitutionOverAddition(): + blach = None + bluchi = None + print(bluch) + + def EliminationOverAddition(): + blucha = None + bluc = None + print(bluch) + + for func, suggestion in [(Substitution, "'blech'?"), + (Elimination, "'blch'?"), + (Addition, "'bluchin'?"), + (EliminationOverAddition, "'blucha'?"), + (SubstitutionOverElimination, "'blach'?"), + (SubstitutionOverAddition, "'blach'?")]: + actual = self.get_suggestion(func) + self.assertIn(suggestion, actual) + + def test_name_error_suggestions_from_globals(self): + def func(): + print(global_for_suggestio) + actual = self.get_suggestion(func) + self.assertIn("'global_for_suggestions'?", actual) + + def test_name_error_suggestions_from_builtins(self): + def func(): + print(ZeroDivisionErrrrr) + actual = self.get_suggestion(func) + self.assertIn("'ZeroDivisionError'?", actual) + + def test_name_error_suggestions_from_builtins_when_builtins_is_module(self): + def func(): + custom_globals = globals().copy() + custom_globals["__builtins__"] = builtins + print(eval("ZeroDivisionErrrrr", custom_globals)) + actual = self.get_suggestion(func) + self.assertIn("'ZeroDivisionError'?", actual) + + def test_name_error_suggestions_with_non_string_candidates(self): + def func(): + abc = 1 + custom_globals = globals().copy() + custom_globals[0] = 1 + print(eval("abv", custom_globals, locals())) + actual = self.get_suggestion(func) + self.assertIn("abc", actual) + + def test_name_error_suggestions_do_not_trigger_for_long_names(self): + def func(): + somethingverywronghehehehehehe = None + print(somethingverywronghe) + actual = self.get_suggestion(func) + self.assertNotIn("somethingverywronghehe", actual) + + def test_name_error_bad_suggestions_do_not_trigger_for_small_names(self): + + def f_b(): + vvv = mom = w = id = pytho = None + b + + def f_v(): + vvv = mom = w = id = pytho = None + v + + def f_m(): + vvv = mom = w = id = pytho = None + m + + def f_py(): + vvv = mom = w = id = pytho = None + py + + for name, func in (("b", f_b), ("v", f_v), ("m", f_m), ("py", f_py)): + with self.subTest(name=name): + actual = self.get_suggestion(func) + self.assertNotIn("you mean", actual) + self.assertNotIn("vvv", actual) + self.assertNotIn("mom", actual) + self.assertNotIn("'id'", actual) + self.assertNotIn("'w'", actual) + self.assertNotIn("'pytho'", actual) + + def test_name_error_suggestions_do_not_trigger_for_too_many_locals(self): + def func(): + # Mutating locals() is unreliable, so we need to do it by hand + a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = a10 = \ + a11 = a12 = a13 = a14 = a15 = a16 = a17 = a18 = a19 = a20 = \ + a21 = a22 = a23 = a24 = a25 = a26 = a27 = a28 = a29 = a30 = \ + a31 = a32 = a33 = a34 = a35 = a36 = a37 = a38 = a39 = a40 = \ + a41 = a42 = a43 = a44 = a45 = a46 = a47 = a48 = a49 = a50 = \ + a51 = a52 = a53 = a54 = a55 = a56 = a57 = a58 = a59 = a60 = \ + a61 = a62 = a63 = a64 = a65 = a66 = a67 = a68 = a69 = a70 = \ + a71 = a72 = a73 = a74 = a75 = a76 = a77 = a78 = a79 = a80 = \ + a81 = a82 = a83 = a84 = a85 = a86 = a87 = a88 = a89 = a90 = \ + a91 = a92 = a93 = a94 = a95 = a96 = a97 = a98 = a99 = a100 = \ + a101 = a102 = a103 = a104 = a105 = a106 = a107 = a108 = a109 = a110 = \ + a111 = a112 = a113 = a114 = a115 = a116 = a117 = a118 = a119 = a120 = \ + a121 = a122 = a123 = a124 = a125 = a126 = a127 = a128 = a129 = a130 = \ + a131 = a132 = a133 = a134 = a135 = a136 = a137 = a138 = a139 = a140 = \ + a141 = a142 = a143 = a144 = a145 = a146 = a147 = a148 = a149 = a150 = \ + a151 = a152 = a153 = a154 = a155 = a156 = a157 = a158 = a159 = a160 = \ + a161 = a162 = a163 = a164 = a165 = a166 = a167 = a168 = a169 = a170 = \ + a171 = a172 = a173 = a174 = a175 = a176 = a177 = a178 = a179 = a180 = \ + a181 = a182 = a183 = a184 = a185 = a186 = a187 = a188 = a189 = a190 = \ + a191 = a192 = a193 = a194 = a195 = a196 = a197 = a198 = a199 = a200 = \ + a201 = a202 = a203 = a204 = a205 = a206 = a207 = a208 = a209 = a210 = \ + a211 = a212 = a213 = a214 = a215 = a216 = a217 = a218 = a219 = a220 = \ + a221 = a222 = a223 = a224 = a225 = a226 = a227 = a228 = a229 = a230 = \ + a231 = a232 = a233 = a234 = a235 = a236 = a237 = a238 = a239 = a240 = \ + a241 = a242 = a243 = a244 = a245 = a246 = a247 = a248 = a249 = a250 = \ + a251 = a252 = a253 = a254 = a255 = a256 = a257 = a258 = a259 = a260 = \ + a261 = a262 = a263 = a264 = a265 = a266 = a267 = a268 = a269 = a270 = \ + a271 = a272 = a273 = a274 = a275 = a276 = a277 = a278 = a279 = a280 = \ + a281 = a282 = a283 = a284 = a285 = a286 = a287 = a288 = a289 = a290 = \ + a291 = a292 = a293 = a294 = a295 = a296 = a297 = a298 = a299 = a300 = \ + a301 = a302 = a303 = a304 = a305 = a306 = a307 = a308 = a309 = a310 = \ + a311 = a312 = a313 = a314 = a315 = a316 = a317 = a318 = a319 = a320 = \ + a321 = a322 = a323 = a324 = a325 = a326 = a327 = a328 = a329 = a330 = \ + a331 = a332 = a333 = a334 = a335 = a336 = a337 = a338 = a339 = a340 = \ + a341 = a342 = a343 = a344 = a345 = a346 = a347 = a348 = a349 = a350 = \ + a351 = a352 = a353 = a354 = a355 = a356 = a357 = a358 = a359 = a360 = \ + a361 = a362 = a363 = a364 = a365 = a366 = a367 = a368 = a369 = a370 = \ + a371 = a372 = a373 = a374 = a375 = a376 = a377 = a378 = a379 = a380 = \ + a381 = a382 = a383 = a384 = a385 = a386 = a387 = a388 = a389 = a390 = \ + a391 = a392 = a393 = a394 = a395 = a396 = a397 = a398 = a399 = a400 = \ + a401 = a402 = a403 = a404 = a405 = a406 = a407 = a408 = a409 = a410 = \ + a411 = a412 = a413 = a414 = a415 = a416 = a417 = a418 = a419 = a420 = \ + a421 = a422 = a423 = a424 = a425 = a426 = a427 = a428 = a429 = a430 = \ + a431 = a432 = a433 = a434 = a435 = a436 = a437 = a438 = a439 = a440 = \ + a441 = a442 = a443 = a444 = a445 = a446 = a447 = a448 = a449 = a450 = \ + a451 = a452 = a453 = a454 = a455 = a456 = a457 = a458 = a459 = a460 = \ + a461 = a462 = a463 = a464 = a465 = a466 = a467 = a468 = a469 = a470 = \ + a471 = a472 = a473 = a474 = a475 = a476 = a477 = a478 = a479 = a480 = \ + a481 = a482 = a483 = a484 = a485 = a486 = a487 = a488 = a489 = a490 = \ + a491 = a492 = a493 = a494 = a495 = a496 = a497 = a498 = a499 = a500 = \ + a501 = a502 = a503 = a504 = a505 = a506 = a507 = a508 = a509 = a510 = \ + a511 = a512 = a513 = a514 = a515 = a516 = a517 = a518 = a519 = a520 = \ + a521 = a522 = a523 = a524 = a525 = a526 = a527 = a528 = a529 = a530 = \ + a531 = a532 = a533 = a534 = a535 = a536 = a537 = a538 = a539 = a540 = \ + a541 = a542 = a543 = a544 = a545 = a546 = a547 = a548 = a549 = a550 = \ + a551 = a552 = a553 = a554 = a555 = a556 = a557 = a558 = a559 = a560 = \ + a561 = a562 = a563 = a564 = a565 = a566 = a567 = a568 = a569 = a570 = \ + a571 = a572 = a573 = a574 = a575 = a576 = a577 = a578 = a579 = a580 = \ + a581 = a582 = a583 = a584 = a585 = a586 = a587 = a588 = a589 = a590 = \ + a591 = a592 = a593 = a594 = a595 = a596 = a597 = a598 = a599 = a600 = \ + a601 = a602 = a603 = a604 = a605 = a606 = a607 = a608 = a609 = a610 = \ + a611 = a612 = a613 = a614 = a615 = a616 = a617 = a618 = a619 = a620 = \ + a621 = a622 = a623 = a624 = a625 = a626 = a627 = a628 = a629 = a630 = \ + a631 = a632 = a633 = a634 = a635 = a636 = a637 = a638 = a639 = a640 = \ + a641 = a642 = a643 = a644 = a645 = a646 = a647 = a648 = a649 = a650 = \ + a651 = a652 = a653 = a654 = a655 = a656 = a657 = a658 = a659 = a660 = \ + a661 = a662 = a663 = a664 = a665 = a666 = a667 = a668 = a669 = a670 = \ + a671 = a672 = a673 = a674 = a675 = a676 = a677 = a678 = a679 = a680 = \ + a681 = a682 = a683 = a684 = a685 = a686 = a687 = a688 = a689 = a690 = \ + a691 = a692 = a693 = a694 = a695 = a696 = a697 = a698 = a699 = a700 = \ + a701 = a702 = a703 = a704 = a705 = a706 = a707 = a708 = a709 = a710 = \ + a711 = a712 = a713 = a714 = a715 = a716 = a717 = a718 = a719 = a720 = \ + a721 = a722 = a723 = a724 = a725 = a726 = a727 = a728 = a729 = a730 = \ + a731 = a732 = a733 = a734 = a735 = a736 = a737 = a738 = a739 = a740 = \ + a741 = a742 = a743 = a744 = a745 = a746 = a747 = a748 = a749 = a750 = \ + a751 = a752 = a753 = a754 = a755 = a756 = a757 = a758 = a759 = a760 = \ + a761 = a762 = a763 = a764 = a765 = a766 = a767 = a768 = a769 = a770 = \ + a771 = a772 = a773 = a774 = a775 = a776 = a777 = a778 = a779 = a780 = \ + a781 = a782 = a783 = a784 = a785 = a786 = a787 = a788 = a789 = a790 = \ + a791 = a792 = a793 = a794 = a795 = a796 = a797 = a798 = a799 = a800 \ + = None + print(a0) + + actual = self.get_suggestion(func) + self.assertNotRegex(actual, r"NameError.*a1") + + def test_name_error_with_custom_exceptions(self): + def func(): + blech = None + raise NameError() + + actual = self.get_suggestion(func) + self.assertNotIn("blech", actual) + + def func(): + blech = None + raise NameError + + actual = self.get_suggestion(func) + self.assertNotIn("blech", actual) + + def test_name_error_with_instance(self): + class A: + def __init__(self): + self.blech = None + def foo(self): + blich = 1 + x = blech + + instance = A() + actual = self.get_suggestion(instance.foo) + self.assertIn("self.blech", actual) + + def test_unbound_local_error_with_instance(self): + class A: + def __init__(self): + self.blech = None + def foo(self): + blich = 1 + x = blech + blech = 1 + + instance = A() + actual = self.get_suggestion(instance.foo) + self.assertNotIn("self.blech", actual) + + def test_unbound_local_error_with_side_effect(self): + # gh-132385 + class A: + def __getattr__(self, key): + if key == 'foo': + raise AttributeError('foo') + if key == 'spam': + raise ValueError('spam') + + def bar(self): + foo + def baz(self): + spam + + suggestion = self.get_suggestion(A().bar) + self.assertNotIn('self.', suggestion) + self.assertIn("'foo'", suggestion) + + suggestion = self.get_suggestion(A().baz) + self.assertNotIn('self.', suggestion) + self.assertIn("'spam'", suggestion) + + def test_unbound_local_error_does_not_match(self): + def func(): + something = 3 + print(somethong) + somethong = 3 + + actual = self.get_suggestion(func) + self.assertNotIn("something", actual) + + def test_name_error_for_stdlib_modules(self): + def func(): + stream = io.StringIO() + + actual = self.get_suggestion(func) + self.assertIn("forget to import 'io'", actual) + + def test_name_error_for_private_stdlib_modules(self): + def func(): + stream = _io.StringIO() + + actual = self.get_suggestion(func) + self.assertIn("forget to import '_io'", actual) + + + +class PurePythonSuggestionFormattingTests( + PurePythonExceptionFormattingMixin, + SuggestionFormattingTestBase, + unittest.TestCase, +): + """ + Same set of tests as above using the pure Python implementation of + traceback printing in traceback.py. + """ + + +@cpython_only +class CPythonSuggestionFormattingTests( + CAPIExceptionFormattingMixin, + SuggestionFormattingTestBase, + unittest.TestCase, +): + """ + Same set of tests as above but with Python's internal traceback printing. + """ + class MiscTest(unittest.TestCase): def test_all(self): expected = set() - denylist = {'print_list'} for name in dir(traceback): - if name.startswith('_') or name in denylist: + if name.startswith('_'): continue module_object = getattr(traceback, name) if getattr(module_object, '__module__', None) == 'traceback': expected.add(name) self.assertCountEqual(traceback.__all__, expected) + def test_levenshtein_distance(self): + # copied from _testinternalcapi.test_edit_cost + # to also exercise the Python implementation + + def CHECK(a, b, expected): + actual = traceback._levenshtein_distance(a, b, 4044) + self.assertEqual(actual, expected) + + CHECK("", "", 0) + CHECK("", "a", 2) + CHECK("a", "A", 1) + CHECK("Apple", "Aple", 2) + CHECK("Banana", "B@n@n@", 6) + CHECK("Cherry", "Cherry!", 2) + CHECK("---0---", "------", 2) + CHECK("abc", "y", 6) + CHECK("aa", "bb", 4) + CHECK("aaaaa", "AAAAA", 5) + CHECK("wxyz", "wXyZ", 2) + CHECK("wxyz", "wXyZ123", 8) + CHECK("Python", "Java", 12) + CHECK("Java", "C#", 8) + CHECK("AbstractFoobarManager", "abstract_foobar_manager", 3+2*2) + CHECK("CPython", "PyPy", 10) + CHECK("CPython", "pypy", 11) + CHECK("AttributeError", "AttributeErrop", 2) + CHECK("AttributeError", "AttributeErrorTests", 10) + CHECK("ABA", "AAB", 4) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: /Users/al03219714/Projects/RustPython/crates/pylib/Lib/test/levenshtein_examples.json is missing. Run `make regen-test-levenshtein` + @support.requires_resource('cpu') + def test_levenshtein_distance_short_circuit(self): + if not LEVENSHTEIN_DATA_FILE.is_file(): + self.fail( + f"{LEVENSHTEIN_DATA_FILE} is missing." + f" Run `make regen-test-levenshtein`" + ) + + with LEVENSHTEIN_DATA_FILE.open("r") as f: + examples = json.load(f) + for a, b, expected in examples: + res1 = traceback._levenshtein_distance(a, b, 1000) + self.assertEqual(res1, expected, msg=(a, b)) + + for threshold in [expected, expected + 1, expected + 2]: + # big enough thresholds shouldn't change the result + res2 = traceback._levenshtein_distance(a, b, threshold) + self.assertEqual(res2, expected, msg=(a, b, threshold)) + + for threshold in range(expected): + # for small thresholds, the only piece of information + # we receive is "strings not close enough". + res3 = traceback._levenshtein_distance(a, b, threshold) + self.assertGreater(res3, threshold, msg=(a, b, threshold)) + + @cpython_only + def test_suggestions_extension(self): + # Check that the C extension is available + import _suggestions + + self.assertEqual( + _suggestions._generate_suggestions( + ["hello", "world"], + "hell" + ), + "hello" + ) + self.assertEqual( + _suggestions._generate_suggestions( + ["hovercraft"], + "eels" + ), + None + ) + + # gh-131936: _generate_suggestions() doesn't accept list subclasses + class MyList(list): + pass + + with self.assertRaises(TypeError): + _suggestions._generate_suggestions(MyList(), "") + + + + +class TestColorizedTraceback(unittest.TestCase): + maxDiff = None + + def test_colorized_traceback(self): + def foo(*args): + x = {'a':{'b': None}} + y = x['a']['b']['c'] + + def baz2(*args): + return (lambda *args: foo(*args))(1,2,3,4) + + def baz1(*args): + return baz2(1,2,3,4) + + def bar(): + return baz1(1, + 2,3 + ,4) + try: + bar() + except Exception as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + lines = "".join(exc.format(colorize=True)) + red = colors["e"] + boldr = colors["E"] + reset = colors["z"] + self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) + self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) + self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) + self.assertIn("return baz2(1,2,3,4)", lines) + self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) + self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) + + def test_colorized_syntax_error(self): + try: + compile("a $ b", "<string>", "exec") + except SyntaxError as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + actual = "".join(exc.format(colorize=True)) + def expected(t, m, fn, l, f, E, e, z): + return "".join( + [ + f' File {fn}"<string>"{z}, line {l}1{z}\n', + f' a {E}${z} b\n', + f' {E}^{z}\n', + f'{t}SyntaxError{z}: {m}invalid syntax{z}\n' + ] + ) + self.assertIn(expected(**colors), actual) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_testcapi' + def test_colorized_traceback_is_the_default(self): + def foo(): + 1/0 + + from _testcapi import exception_print + try: + foo() + self.fail("No exception thrown.") + except Exception as e: + with captured_output("stderr") as tbstderr: + with unittest.mock.patch('_colorize.can_colorize', return_value=True): + exception_print(e) + actual = tbstderr.getvalue().splitlines() + + lno_foo = foo.__code__.co_firstlineno + def expected(t, m, fn, l, f, E, e, z): + return [ + 'Traceback (most recent call last):', + f' File {fn}"{__file__}"{z}, ' + f'line {l}{lno_foo+5}{z}, in {f}test_colorized_traceback_is_the_default{z}', + f' {e}foo{z}{E}(){z}', + f' {e}~~~{z}{E}^^{z}', + f' File {fn}"{__file__}"{z}, ' + f'line {l}{lno_foo+1}{z}, in {f}foo{z}', + f' {e}1{z}{E}/{z}{e}0{z}', + f' {e}~{z}{E}^{z}{e}~{z}', + f'{t}ZeroDivisionError{z}: {m}division by zero{z}', + ] + self.assertEqual(actual, expected(**colors)) + + def test_colorized_traceback_from_exception_group(self): + def foo(): + exceptions = [] + try: + 1 / 0 + except ZeroDivisionError as inner_exc: + exceptions.append(inner_exc) + raise ExceptionGroup("test", exceptions) + + try: + foo() + except Exception as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + + lno_foo = foo.__code__.co_firstlineno + actual = "".join(exc.format(colorize=True)).splitlines() + def expected(t, m, fn, l, f, E, e, z): + return [ + f" + Exception Group Traceback (most recent call last):", + f' | File {fn}"{__file__}"{z}, line {l}{lno_foo+9}{z}, in {f}test_colorized_traceback_from_exception_group{z}', + f' | {e}foo{z}{E}(){z}', + f' | {e}~~~{z}{E}^^{z}', + f" | e = ExceptionGroup('test', [ZeroDivisionError('division by zero')])", + f" | foo = {foo}", + f' | self = <{__name__}.TestColorizedTraceback testMethod=test_colorized_traceback_from_exception_group>', + f' | File {fn}"{__file__}"{z}, line {l}{lno_foo+6}{z}, in {f}foo{z}', + f' | raise ExceptionGroup("test", exceptions)', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {t}ExceptionGroup{z}: {m}test (1 sub-exception){z}', + f' +-+---------------- 1 ----------------', + f' | Traceback (most recent call last):', + f' | File {fn}"{__file__}"{z}, line {l}{lno_foo+3}{z}, in {f}foo{z}', + f' | {e}1 {z}{E}/{z}{e} 0{z}', + f' | {e}~~{z}{E}^{z}{e}~~{z}', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {t}ZeroDivisionError{z}: {m}division by zero{z}', + f' +------------------------------------', + ] + self.assertEqual(actual, expected(**colors)) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py new file mode 100644 index 00000000000..a1f686c8f56 --- /dev/null +++ b/Lib/test/test_tstring.py @@ -0,0 +1,295 @@ +import unittest + +from test.test_string._support import TStringBaseCase, fstring + + +class TestTString(unittest.TestCase, TStringBaseCase): + def test_string_representation(self): + # Test __repr__ + t = t"Hello" + self.assertEqual(repr(t), "Template(strings=('Hello',), interpolations=())") + + name = "Python" + t = t"Hello, {name}" + self.assertEqual(repr(t), + "Template(strings=('Hello, ', ''), " + "interpolations=(Interpolation('Python', 'name', None, ''),))" + ) + + def test_interpolation_basics(self): + # Test basic interpolation + name = "Python" + t = t"Hello, {name}" + self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) + self.assertEqual(fstring(t), "Hello, Python") + + # Multiple interpolations + first = "Python" + last = "Developer" + t = t"{first} {last}" + self.assertTStringEqual( + t, ("", " ", ""), [(first, 'first'), (last, 'last')] + ) + self.assertEqual(fstring(t), "Python Developer") + + # Interpolation with expressions + a = 10 + b = 20 + t = t"Sum: {a + b}" + self.assertTStringEqual(t, ("Sum: ", ""), [(a + b, "a + b")]) + self.assertEqual(fstring(t), "Sum: 30") + + # Interpolation with function + def square(x): + return x * x + t = t"Square: {square(5)}" + self.assertTStringEqual( + t, ("Square: ", ""), [(square(5), "square(5)")] + ) + self.assertEqual(fstring(t), "Square: 25") + + # Test attribute access in expressions + class Person: + def __init__(self, name): + self.name = name + + def upper(self): + return self.name.upper() + + person = Person("Alice") + t = t"Name: {person.name}" + self.assertTStringEqual( + t, ("Name: ", ""), [(person.name, "person.name")] + ) + self.assertEqual(fstring(t), "Name: Alice") + + # Test method calls + t = t"Name: {person.upper()}" + self.assertTStringEqual( + t, ("Name: ", ""), [(person.upper(), "person.upper()")] + ) + self.assertEqual(fstring(t), "Name: ALICE") + + # Test dictionary access + data = {"name": "Bob", "age": 30} + t = t"Name: {data['name']}, Age: {data['age']}" + self.assertTStringEqual( + t, ("Name: ", ", Age: ", ""), + [(data["name"], "data['name']"), (data["age"], "data['age']")], + ) + self.assertEqual(fstring(t), "Name: Bob, Age: 30") + + def test_format_specifiers(self): + # Test basic format specifiers + value = 3.14159 + t = t"Pi: {value:.2f}" + self.assertTStringEqual( + t, ("Pi: ", ""), [(value, "value", None, ".2f")] + ) + self.assertEqual(fstring(t), "Pi: 3.14") + + def test_conversions(self): + # Test !s conversion (str) + obj = object() + t = t"Object: {obj!s}" + self.assertTStringEqual(t, ("Object: ", ""), [(obj, "obj", "s")]) + self.assertEqual(fstring(t), f"Object: {str(obj)}") + + # Test !r conversion (repr) + t = t"Data: {obj!r}" + self.assertTStringEqual(t, ("Data: ", ""), [(obj, "obj", "r")]) + self.assertEqual(fstring(t), f"Data: {repr(obj)}") + + # Test !a conversion (ascii) + text = "Café" + t = t"ASCII: {text!a}" + self.assertTStringEqual(t, ("ASCII: ", ""), [(text, "text", "a")]) + self.assertEqual(fstring(t), f"ASCII: {ascii(text)}") + + # Test !z conversion (error) + num = 1 + with self.assertRaises(SyntaxError): + eval("t'{num!z}'") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++++ + def test_debug_specifier(self): + # Test debug specifier + value = 42 + t = t"Value: {value=}" + self.assertTStringEqual( + t, ("Value: value=", ""), [(value, "value", "r")] + ) + self.assertEqual(fstring(t), "Value: value=42") + + # Test debug specifier with format (conversion default to !r) + t = t"Value: {value=:.2f}" + self.assertTStringEqual( + t, ("Value: value=", ""), [(value, "value", None, ".2f")] + ) + self.assertEqual(fstring(t), "Value: value=42.00") + + # Test debug specifier with conversion + t = t"Value: {value=!s}" + self.assertTStringEqual( + t, ("Value: value=", ""), [(value, "value", "s")] + ) + + # Test white space in debug specifier + t = t"Value: {value = }" + self.assertTStringEqual( + t, ("Value: value = ", ""), [(value, "value", "r")] + ) + self.assertEqual(fstring(t), "Value: value = 42") + + def test_raw_tstrings(self): + path = r"C:\Users" + t = rt"{path}\Documents" + self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")]) + self.assertEqual(fstring(t), r"C:\Users\Documents") + + # Test alternative prefix + t = tr"{path}\Documents" + self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "can only concatenate string.templatelib.Template \(not "str"\) to string.templatelib.Template" does not match "can only concatenate Template (not 'str') to Template" + def test_template_concatenation(self): + # Test template + template + t1 = t"Hello, " + t2 = t"world" + combined = t1 + t2 + self.assertTStringEqual(combined, ("Hello, world",), ()) + self.assertEqual(fstring(combined), "Hello, world") + + # Test template + string + t1 = t"Hello" + expected_msg = 'can only concatenate string.templatelib.Template ' \ + '\\(not "str"\\) to string.templatelib.Template' + with self.assertRaisesRegex(TypeError, expected_msg): + t1 + ", world" + + # Test template + template with interpolation + name = "Python" + t1 = t"Hello, " + t2 = t"{name}" + combined = t1 + t2 + self.assertTStringEqual(combined, ("Hello, ", ""), [(name, "name")]) + self.assertEqual(fstring(combined), "Hello, Python") + + # Test string + template + expected_msg = 'can only concatenate str ' \ + '\\(not "string.templatelib.Template"\\) to str' + with self.assertRaisesRegex(TypeError, expected_msg): + "Hello, " + t"{name}" + + def test_nested_templates(self): + # Test a template inside another template expression + name = "Python" + inner = t"{name}" + t = t"Language: {inner}" + + t_interp = t.interpolations[0] + self.assertEqual(t.strings, ("Language: ", "")) + self.assertEqual(t_interp.value.strings, ("", "")) + self.assertEqual(t_interp.value.interpolations[0].value, name) + self.assertEqual(t_interp.value.interpolations[0].expression, "name") + self.assertEqual(t_interp.value.interpolations[0].conversion, None) + self.assertEqual(t_interp.value.interpolations[0].format_spec, "") + self.assertEqual(t_interp.expression, "inner") + self.assertEqual(t_interp.conversion, None) + self.assertEqual(t_interp.format_spec, "") + + @unittest.expectedFailure # TODO: RUSTPYTHON multiple instances of AssertionError + def test_syntax_errors(self): + for case, err in ( + ("t'", "unterminated t-string literal"), + ("t'''", "unterminated triple-quoted t-string literal"), + ("t''''", "unterminated triple-quoted t-string literal"), + ("t'{", "'{' was never closed"), + ("t'{'", "t-string: expecting '}'"), + ("t'{a'", "t-string: expecting '}'"), + ("t'}'", "t-string: single '}' is not allowed"), + ("t'{}'", "t-string: valid expression required before '}'"), + ("t'{=x}'", "t-string: valid expression required before '='"), + ("t'{!x}'", "t-string: valid expression required before '!'"), + ("t'{:x}'", "t-string: valid expression required before ':'"), + ("t'{x;y}'", "t-string: expecting '=', or '!', or ':', or '}'"), + ("t'{x=y}'", "t-string: expecting '!', or ':', or '}'"), + ("t'{x!s!}'", "t-string: expecting ':' or '}'"), + ("t'{x!s:'", "t-string: expecting '}', or format specs"), + ("t'{x!}'", "t-string: missing conversion character"), + ("t'{x=!}'", "t-string: missing conversion character"), + ("t'{x!z}'", "t-string: invalid conversion character 'z': " + "expected 's', 'r', or 'a'"), + ("t'{lambda:1}'", "t-string: lambda expressions are not allowed " + "without parentheses"), + ("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"), + ("t'{1:d\n}'", "t-string: newlines are not allowed in format specifiers") + ): + with self.subTest(case), self.assertRaisesRegex(SyntaxError, err): + eval(case) + + def test_runtime_errors(self): + # Test missing variables + with self.assertRaises(NameError): + eval("t'Hello, {name}'") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_literal_concatenation(self): + # Test concatenation of t-string literals + t = t"Hello, " t"world" + self.assertTStringEqual(t, ("Hello, world",), ()) + self.assertEqual(fstring(t), "Hello, world") + + # Test concatenation with interpolation + name = "Python" + t = t"Hello, " t"{name}" + self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) + self.assertEqual(fstring(t), "Hello, Python") + + # Test disallowed mix of t-string and string/f-string (incl. bytes) + what = 't' + expected_msg = 'cannot mix t-string literals with string or bytes literals' + for case in ( + "t'{what}-string literal' 'str literal'", + "t'{what}-string literal' u'unicode literal'", + "t'{what}-string literal' f'f-string literal'", + "t'{what}-string literal' r'raw string literal'", + "t'{what}-string literal' rf'raw f-string literal'", + "t'{what}-string literal' b'bytes literal'", + "t'{what}-string literal' br'raw bytes literal'", + "'str literal' t'{what}-string literal'", + "u'unicode literal' t'{what}-string literal'", + "f'f-string literal' t'{what}-string literal'", + "r'raw string literal' t'{what}-string literal'", + "rf'raw f-string literal' t'{what}-string literal'", + "b'bytes literal' t'{what}-string literal'", + "br'raw bytes literal' t'{what}-string literal'", + ): + with self.subTest(case): + with self.assertRaisesRegex(SyntaxError, expected_msg): + eval(case) + + def test_triple_quoted(self): + # Test triple-quoted t-strings + t = t""" + Hello, + world + """ + self.assertTStringEqual( + t, ("\n Hello,\n world\n ",), () + ) + self.assertEqual(fstring(t), "\n Hello,\n world\n ") + + # Test triple-quoted with interpolation + name = "Python" + t = t""" + Hello, + {name} + """ + self.assertTStringEqual( + t, ("\n Hello,\n ", "\n "), [(name, "name")] + ) + self.assertEqual(fstring(t), "\n Hello,\n Python\n ") + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_tty.py b/Lib/test/test_tty.py new file mode 100644 index 00000000000..681772fb519 --- /dev/null +++ b/Lib/test/test_tty.py @@ -0,0 +1,96 @@ +import os +import unittest +from test.support.import_helper import import_module + +termios = import_module('termios') +tty = import_module('tty') + + +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class TestTty(unittest.TestCase): + + def setUp(self): + master_fd, self.fd = os.openpty() + self.addCleanup(os.close, master_fd) + self.stream = self.enterContext(open(self.fd, 'wb', buffering=0)) + self.fd = self.stream.fileno() + self.mode = termios.tcgetattr(self.fd) + self.addCleanup(termios.tcsetattr, self.fd, termios.TCSANOW, self.mode) + self.addCleanup(termios.tcsetattr, self.fd, termios.TCSAFLUSH, self.mode) + + def check_cbreak(self, mode): + self.assertEqual(mode[3] & termios.ECHO, 0) + self.assertEqual(mode[3] & termios.ICANON, 0) + self.assertEqual(mode[6][termios.VMIN], 1) + self.assertEqual(mode[6][termios.VTIME], 0) + + def check_raw(self, mode): + self.check_cbreak(mode) + self.assertEqual(mode[0] & termios.ISTRIP, 0) + self.assertEqual(mode[0] & termios.ICRNL, 0) + self.assertEqual(mode[1] & termios.OPOST, 0) + self.assertEqual(mode[2] & termios.PARENB, termios.CS8 & termios.PARENB) + self.assertEqual(mode[2] & termios.CSIZE, termios.CS8 & termios.CSIZE) + self.assertEqual(mode[2] & termios.CS8, termios.CS8) + self.assertEqual(mode[3] & termios.ECHO, 0) + self.assertEqual(mode[3] & termios.ICANON, 0) + self.assertEqual(mode[3] & termios.ISIG, 0) + self.assertEqual(mode[6][termios.VMIN], 1) + self.assertEqual(mode[6][termios.VTIME], 0) + + def test_cfmakeraw(self): + mode = termios.tcgetattr(self.fd) + self.assertEqual(mode, self.mode) + tty.cfmakeraw(mode) + self.check_raw(mode) + self.assertEqual(mode[4], self.mode[4]) + self.assertEqual(mode[5], self.mode[5]) + + def test_cfmakecbreak(self): + mode = termios.tcgetattr(self.fd) + self.assertEqual(mode, self.mode) + tty.cfmakecbreak(mode) + self.check_cbreak(mode) + self.assertEqual(mode[1], self.mode[1]) + self.assertEqual(mode[2], self.mode[2]) + self.assertEqual(mode[4], self.mode[4]) + self.assertEqual(mode[5], self.mode[5]) + mode[tty.IFLAG] |= termios.ICRNL + tty.cfmakecbreak(mode) + self.assertEqual(mode[tty.IFLAG] & termios.ICRNL, termios.ICRNL, + msg="ICRNL should not be cleared by cbreak") + mode[tty.IFLAG] &= ~termios.ICRNL + tty.cfmakecbreak(mode) + self.assertEqual(mode[tty.IFLAG] & termios.ICRNL, 0, + msg="ICRNL should not be set by cbreak") + + @unittest.expectedFailure # TODO: RUSTPYTHON TypeError: Expected type "int" but "FileIO" found. + def test_setraw(self): + mode0 = termios.tcgetattr(self.fd) + mode1 = tty.setraw(self.fd) + self.assertEqual(mode1, mode0) + mode2 = termios.tcgetattr(self.fd) + self.check_raw(mode2) + mode3 = tty.setraw(self.fd, termios.TCSANOW) + self.assertEqual(mode3, mode2) + tty.setraw(self.stream) + tty.setraw(fd=self.fd, when=termios.TCSANOW) + + @unittest.expectedFailure # TODO: RUSTPYTHON TypeError: Expected type "int" but "FileIO" found. + def test_setcbreak(self): + mode0 = termios.tcgetattr(self.fd) + mode1 = tty.setcbreak(self.fd) + self.assertEqual(mode1, mode0) + mode2 = termios.tcgetattr(self.fd) + self.check_cbreak(mode2) + ICRNL = termios.ICRNL + self.assertEqual(mode2[tty.IFLAG] & ICRNL, mode0[tty.IFLAG] & ICRNL, + msg="ICRNL should not be altered by cbreak") + mode3 = tty.setcbreak(self.fd, termios.TCSANOW) + self.assertEqual(mode3, mode2) + tty.setcbreak(self.stream) + tty.setcbreak(fd=self.fd, when=termios.TCSANOW) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_tuple.py b/Lib/test/test_tuple.py index 26b238e0865..a97ca8c9ad3 100644 --- a/Lib/test/test_tuple.py +++ b/Lib/test/test_tuple.py @@ -42,6 +42,34 @@ def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): tuple(sequence=()) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_keywords_in_subclass(self): + class subclass(tuple): + pass + u = subclass([1, 2]) + self.assertIs(type(u), subclass) + self.assertEqual(list(u), [1, 2]) + with self.assertRaises(TypeError): + subclass(sequence=()) + + class subclass_with_init(tuple): + def __init__(self, arg, newarg=None): + self.newarg = newarg + u = subclass_with_init([1, 2], newarg=3) + self.assertIs(type(u), subclass_with_init) + self.assertEqual(list(u), [1, 2]) + self.assertEqual(u.newarg, 3) + + class subclass_with_new(tuple): + def __new__(cls, arg, newarg=None): + self = super().__new__(cls, arg) + self.newarg = newarg + return self + u = subclass_with_new([1, 2], newarg=3) + self.assertIs(type(u), subclass_with_new) + self.assertEqual(list(u), [1, 2]) + self.assertEqual(u.newarg, 3) + def test_truth(self): super().test_truth() self.assertTrue(not ()) @@ -77,8 +105,6 @@ def f(): # We expect tuples whose base components have deterministic hashes to # have deterministic hashes too - and, indeed, the same hashes across # platforms with hash codes of the same bit width. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_hash_exact(self): def check_one_exact(t, e32, e64): got = hash(t) @@ -265,12 +291,18 @@ def test_repr(self): self.assertEqual(repr(a0), "()") self.assertEqual(repr(a2), "(0, 1, 2)") + # Checks that t is not tracked without any GC collections. + def _not_tracked_instantly(self, t): + self.assertFalse(gc.is_tracked(t), t) + + # Checks that t is not tracked after GC collection. def _not_tracked(self, t): # Nested tuples can take several collections to untrack gc.collect() gc.collect() self.assertFalse(gc.is_tracked(t), t) + # Checks that t continues to be tracked even after GC collection. def _tracked(self, t): self.assertTrue(gc.is_tracked(t), t) gc.collect() @@ -282,13 +314,19 @@ def test_track_literals(self): # Test GC-optimization of tuple literals x, y, z = 1.5, "a", [] - self._not_tracked(()) - self._not_tracked((1,)) - self._not_tracked((1, 2)) - self._not_tracked((1, 2, "a")) - self._not_tracked((1, 2, (None, True, False, ()), int)) - self._not_tracked((object(),)) + # We check that those objects aren't tracked at all. + # It's essential for the GC performance, see gh-139951. + self._not_tracked_instantly(()) + self._not_tracked_instantly((1,)) + self._not_tracked_instantly((1, 2)) + self._not_tracked_instantly((1, 2, "a")) + self._not_tracked_instantly((1, 2) * 5) + self._not_tracked_instantly((12, 10**10, 'a_' * 100)) + self._not_tracked_instantly((object(),)) + self._not_tracked(((1, x), y, (2, 3))) + self._not_tracked((1, 2, (None, True, False, ()), int)) + self._not_tracked((object(), ())) # Tuples with mutable elements are always tracked, even if those # elements are not tracked right now. @@ -318,6 +356,12 @@ def check_track_dynamic(self, tp, always_track): self._tracked(tp(tuple([obj]) for obj in [x, y, z])) self._tracked(tuple(tp([obj]) for obj in [x, y, z])) + t = tp([1, x, y, z]) + self.assertEqual(type(t), tp) + self._tracked(t) + self.assertEqual(type(t[:]), tuple) + self._tracked(t[:]) + @support.cpython_only def test_track_dynamic(self): # Test GC-optimization of dynamically constructed tuples. diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py new file mode 100644 index 00000000000..ee1791bc1d0 --- /dev/null +++ b/Lib/test/test_type_aliases.py @@ -0,0 +1,415 @@ +import pickle +import types +import unittest +from test.support import check_syntax_error, run_code +from test.typinganndata import mod_generics_cache + +from typing import ( + Callable, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, Unpack, get_args, +) + + +class TypeParamsInvalidTest(unittest.TestCase): + def test_name_collisions(self): + check_syntax_error(self, 'type TA1[A, **A] = None', "duplicate type parameter 'A'") + check_syntax_error(self, 'type T[A, *A] = None', "duplicate type parameter 'A'") + check_syntax_error(self, 'type T[*A, **A] = None', "duplicate type parameter 'A'") + + def test_name_non_collision_02(self): + ns = run_code("""type TA1[A] = lambda A: A""") + self.assertIsInstance(ns["TA1"], TypeAliasType) + self.assertTrue(callable(ns["TA1"].__value__)) + self.assertEqual("arg", ns["TA1"].__value__("arg")) + + def test_name_non_collision_03(self): + ns = run_code(""" + class Outer[A]: + type TA1[A] = None + """ + ) + outer_A, = ns["Outer"].__type_params__ + inner_A, = ns["Outer"].TA1.__type_params__ + self.assertIsNot(outer_A, inner_A) + + +class TypeParamsAccessTest(unittest.TestCase): + def test_alias_access_01(self): + ns = run_code("type TA1[A, B] = dict[A, B]") + alias = ns["TA1"] + self.assertIsInstance(alias, TypeAliasType) + self.assertEqual(alias.__type_params__, get_args(alias.__value__)) + + def test_alias_access_02(self): + ns = run_code(""" + type TA1[A, B] = TA1[A, B] | int + """ + ) + alias = ns["TA1"] + self.assertIsInstance(alias, TypeAliasType) + A, B = alias.__type_params__ + self.assertEqual(alias.__value__, alias[A, B] | int) + + def test_alias_access_03(self): + ns = run_code(""" + class Outer[A]: + def inner[B](self): + type TA1[C] = TA1[A, B] | int + return TA1 + """ + ) + cls = ns["Outer"] + A, = cls.__type_params__ + B, = cls.inner.__type_params__ + alias = cls.inner(None) + self.assertIsInstance(alias, TypeAliasType) + alias2 = cls.inner(None) + self.assertIsNot(alias, alias2) + self.assertEqual(len(alias.__type_params__), 1) + + self.assertEqual(alias.__value__, alias[A, B] | int) + + +class TypeParamsAliasValueTest(unittest.TestCase): + def test_alias_value_01(self): + type TA1 = int + + self.assertIsInstance(TA1, TypeAliasType) + self.assertEqual(TA1.__value__, int) + self.assertEqual(TA1.__parameters__, ()) + self.assertEqual(TA1.__type_params__, ()) + + type TA2 = TA1 | str + + self.assertIsInstance(TA2, TypeAliasType) + a, b = TA2.__value__.__args__ + self.assertEqual(a, TA1) + self.assertEqual(b, str) + self.assertEqual(TA2.__parameters__, ()) + self.assertEqual(TA2.__type_params__, ()) + + def test_alias_value_02(self): + class Parent[A]: + type TA1[B] = dict[A, B] + + self.assertIsInstance(Parent.TA1, TypeAliasType) + self.assertEqual(len(Parent.TA1.__parameters__), 1) + self.assertEqual(len(Parent.__parameters__), 1) + a, = Parent.__parameters__ + b, = Parent.TA1.__parameters__ + self.assertEqual(Parent.__type_params__, (a,)) + self.assertEqual(Parent.TA1.__type_params__, (b,)) + self.assertEqual(Parent.TA1.__value__, dict[a, b]) + + def test_alias_value_03(self): + def outer[A](): + type TA1[B] = dict[A, B] + return TA1 + + o = outer() + self.assertIsInstance(o, TypeAliasType) + self.assertEqual(len(o.__parameters__), 1) + self.assertEqual(len(outer.__type_params__), 1) + b = o.__parameters__[0] + self.assertEqual(o.__type_params__, (b,)) + + def test_alias_value_04(self): + def more_generic[T, *Ts, **P](): + type TA[T2, *Ts2, **P2] = tuple[Callable[P, tuple[T, *Ts]], Callable[P2, tuple[T2, *Ts2]]] + return TA + + alias = more_generic() + self.assertIsInstance(alias, TypeAliasType) + T2, Ts2, P2 = alias.__type_params__ + self.assertEqual(alias.__parameters__, (T2, *Ts2, P2)) + T, Ts, P = more_generic.__type_params__ + self.assertEqual(alias.__value__, tuple[Callable[P, tuple[T, *Ts]], Callable[P2, tuple[T2, *Ts2]]]) + + def test_subscripting(self): + type NonGeneric = int + type Generic[A] = dict[A, A] + type VeryGeneric[T, *Ts, **P] = Callable[P, tuple[T, *Ts]] + + with self.assertRaises(TypeError): + NonGeneric[int] + + specialized = Generic[int] + self.assertIsInstance(specialized, types.GenericAlias) + self.assertIs(specialized.__origin__, Generic) + self.assertEqual(specialized.__args__, (int,)) + + specialized2 = VeryGeneric[int, str, float, [bool, range]] + self.assertIsInstance(specialized2, types.GenericAlias) + self.assertIs(specialized2.__origin__, VeryGeneric) + self.assertEqual(specialized2.__args__, (int, str, float, [bool, range])) + + def test_repr(self): + type Simple = int + type VeryGeneric[T, *Ts, **P] = Callable[P, tuple[T, *Ts]] + + self.assertEqual(repr(Simple), "Simple") + self.assertEqual(repr(VeryGeneric), "VeryGeneric") + self.assertEqual(repr(VeryGeneric[int, bytes, str, [float, object]]), + "VeryGeneric[int, bytes, str, [float, object]]") + self.assertEqual(repr(VeryGeneric[int, []]), + "VeryGeneric[int, []]") + self.assertEqual(repr(VeryGeneric[int, [VeryGeneric[int], list[str]]]), + "VeryGeneric[int, [VeryGeneric[int], list[str]]]") + + def test_recursive_repr(self): + type Recursive = Recursive + self.assertEqual(repr(Recursive), "Recursive") + + type X = list[Y] + type Y = list[X] + self.assertEqual(repr(X), "X") + self.assertEqual(repr(Y), "Y") + + type GenericRecursive[X] = list[X | GenericRecursive[X]] + self.assertEqual(repr(GenericRecursive), "GenericRecursive") + self.assertEqual(repr(GenericRecursive[int]), "GenericRecursive[int]") + self.assertEqual(repr(GenericRecursive[GenericRecursive[int]]), + "GenericRecursive[GenericRecursive[int]]") + + def test_raising(self): + type MissingName = list[_My_X] + with self.assertRaisesRegex( + NameError, + "cannot access free variable '_My_X' where it is not associated with a value", + ): + MissingName.__value__ + _My_X = int + self.assertEqual(MissingName.__value__, list[int]) + del _My_X + # Cache should still work: + self.assertEqual(MissingName.__value__, list[int]) + + # Explicit exception: + type ExprException = 1 / 0 + with self.assertRaises(ZeroDivisionError): + ExprException.__value__ + + +class TypeAliasConstructorTest(unittest.TestCase): + def test_basic(self): + TA = TypeAliasType("TA", int) + self.assertEqual(TA.__name__, "TA") + self.assertIs(TA.__value__, int) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + + def test_attributes_with_exec(self): + ns = {} + exec("type TA = int", ns, ns) + TA = ns["TA"] + self.assertEqual(TA.__name__, "TA") + self.assertIs(TA.__value__, int) + self.assertEqual(TA.__type_params__, ()) + self.assertIs(TA.__module__, None) + + def test_generic(self): + T = TypeVar("T") + TA = TypeAliasType("TA", list[T], type_params=(T,)) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[T]) + self.assertEqual(TA.__type_params__, (T,)) + self.assertEqual(TA.__module__, __name__) + self.assertIs(type(TA[int]), types.GenericAlias) + + def test_not_generic(self): + TA = TypeAliasType("TA", list[int], type_params=()) + self.assertEqual(TA.__name__, "TA") + self.assertEqual(TA.__value__, list[int]) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + with self.assertRaisesRegex( + TypeError, + "Only generic type aliases are subscriptable", + ): + TA[int] + + def test_type_params_order_with_defaults(self): + HasNoDefaultT = TypeVar("HasNoDefaultT") + WithDefaultT = TypeVar("WithDefaultT", default=int) + + HasNoDefaultP = ParamSpec("HasNoDefaultP") + WithDefaultP = ParamSpec("WithDefaultP", default=HasNoDefaultP) + + HasNoDefaultTT = TypeVarTuple("HasNoDefaultTT") + WithDefaultTT = TypeVarTuple("WithDefaultTT", default=HasNoDefaultTT) + + for type_params in [ + (HasNoDefaultT, WithDefaultT), + (HasNoDefaultP, WithDefaultP), + (HasNoDefaultTT, WithDefaultTT), + ]: + with self.subTest(type_params=type_params): + TypeAliasType("A", int, type_params=type_params) # ok + + msg = "follows default type parameter" + for type_params in [ + (WithDefaultT, HasNoDefaultT), + (WithDefaultP, HasNoDefaultP), + (WithDefaultTT, HasNoDefaultTT), + (WithDefaultT, HasNoDefaultP), # different types + ]: + with self.subTest(type_params=type_params): + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=type_params) + + def test_expects_type_like(self): + T = TypeVar("T") + + msg = "Expected a type param" + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1,)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(1, 2)) + with self.assertRaisesRegex(TypeError, msg): + TypeAliasType("A", int, type_params=(T, 2)) + + def test_keywords(self): + TA = TypeAliasType(name="TA", value=int) + self.assertEqual(TA.__name__, "TA") + self.assertIs(TA.__value__, int) + self.assertEqual(TA.__type_params__, ()) + self.assertEqual(TA.__module__, __name__) + + def test_errors(self): + with self.assertRaises(TypeError): + TypeAliasType() + with self.assertRaises(TypeError): + TypeAliasType("TA") + with self.assertRaises(TypeError): + TypeAliasType("TA", list, ()) + with self.assertRaises(TypeError): + TypeAliasType("TA", list, type_params=42) + + +class TypeAliasTypeTest(unittest.TestCase): + def test_immutable(self): + with self.assertRaises(TypeError): + TypeAliasType.whatever = "not allowed" + + def test_no_subclassing(self): + with self.assertRaisesRegex(TypeError, "not an acceptable base type"): + class MyAlias(TypeAliasType): + pass + + def test_union(self): + type Alias1 = int + type Alias2 = str + union = Alias1 | Alias2 + self.assertIsInstance(union, types.UnionType) + self.assertEqual(get_args(union), (Alias1, Alias2)) + union2 = Alias1 | list[float] + self.assertIsInstance(union2, types.UnionType) + self.assertEqual(get_args(union2), (Alias1, list[float])) + union3 = list[range] | Alias1 + self.assertIsInstance(union3, types.UnionType) + self.assertEqual(get_args(union3), (list[range], Alias1)) + + def test_module(self): + self.assertEqual(TypeAliasType.__module__, "typing") + type Alias = int + self.assertEqual(Alias.__module__, __name__) + self.assertEqual(mod_generics_cache.Alias.__module__, + mod_generics_cache.__name__) + self.assertEqual(mod_generics_cache.OldStyle.__module__, + mod_generics_cache.__name__) + + def test_unpack(self): + type Alias = tuple[int, int] + unpacked = (*Alias,)[0] + self.assertEqual(unpacked, Unpack[Alias]) + + class Foo[*Ts]: + pass + + x = Foo[str, *Alias] + self.assertEqual(x.__args__, (str, Unpack[Alias])) + + +# All these type aliases are used for pickling tests: +T = TypeVar('T') +type SimpleAlias = int +type RecursiveAlias = dict[str, RecursiveAlias] +type GenericAlias[X] = list[X] +type GenericAliasMultipleTypes[X, Y] = dict[X, Y] +type RecursiveGenericAlias[X] = dict[str, RecursiveAlias[X]] +type BoundGenericAlias[X: int] = set[X] +type ConstrainedGenericAlias[LongName: (str, bytes)] = list[LongName] +type AllTypesAlias[A, *B, **C] = Callable[C, A] | tuple[*B] + + +class TypeAliasPickleTest(unittest.TestCase): + def test_pickling(self): + things_to_test = [ + SimpleAlias, + RecursiveAlias, + + GenericAlias, + GenericAlias[T], + GenericAlias[int], + + GenericAliasMultipleTypes, + GenericAliasMultipleTypes[str, T], + GenericAliasMultipleTypes[T, str], + GenericAliasMultipleTypes[int, str], + + RecursiveGenericAlias, + RecursiveGenericAlias[T], + RecursiveGenericAlias[int], + + BoundGenericAlias, + BoundGenericAlias[int], + BoundGenericAlias[T], + + ConstrainedGenericAlias, + ConstrainedGenericAlias[str], + ConstrainedGenericAlias[T], + + AllTypesAlias, + AllTypesAlias[int, str, T, [T, object]], + + # Other modules: + mod_generics_cache.Alias, + mod_generics_cache.OldStyle, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + type ClassLevel = str + + def test_pickling_local(self): + type A = int + things_to_test = [ + self.ClassLevel, + A, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + with self.assertRaises(pickle.PickleError): + pickle.dumps(thing, protocol=proto) + + +class TypeParamsExoticGlobalsTest(unittest.TestCase): + def test_exec_with_unusual_globals(self): + class customdict(dict): + def __missing__(self, key): + return key + + code = compile("type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["Alias"] + self.assertEqual(Alias.__value__, "undefined") + + code = compile("class A: type Alias = undefined", "test", "exec") + ns = customdict() + exec(code, ns) + Alias = ns["A"].Alias + self.assertEqual(Alias.__value__, "undefined") diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py new file mode 100644 index 00000000000..4c58fade1b4 --- /dev/null +++ b/Lib/test/test_type_annotations.py @@ -0,0 +1,874 @@ +import annotationlib +import inspect +import textwrap +import types +import unittest +from test.support import run_code, check_syntax_error, import_helper, cpython_only +from test.test_inspect import inspect_stringized_annotations + + +class TypeAnnotationTests(unittest.TestCase): + + def test_lazy_create_annotations(self): + # type objects lazy create their __annotations__ dict on demand. + # the annotations dict is stored in type.__dict__ (as __annotations_cache__). + # a freshly created type shouldn't have an annotations dict yet. + foo = type("Foo", (), {}) + for i in range(3): + self.assertFalse("__annotations_cache__" in foo.__dict__) + d = foo.__annotations__ + self.assertTrue("__annotations_cache__" in foo.__dict__) + self.assertEqual(foo.__annotations__, d) + self.assertEqual(foo.__dict__['__annotations_cache__'], d) + del foo.__annotations__ + + def test_setting_annotations(self): + foo = type("Foo", (), {}) + for i in range(3): + self.assertFalse("__annotations_cache__" in foo.__dict__) + d = {'a': int} + foo.__annotations__ = d + self.assertTrue("__annotations_cache__" in foo.__dict__) + self.assertEqual(foo.__annotations__, d) + self.assertEqual(foo.__dict__['__annotations_cache__'], d) + del foo.__annotations__ + + def test_annotations_getset_raises(self): + # builtin types don't have __annotations__ (yet!) + with self.assertRaises(AttributeError): + print(float.__annotations__) + with self.assertRaises(TypeError): + float.__annotations__ = {} + with self.assertRaises(TypeError): + del float.__annotations__ + + # double delete + foo = type("Foo", (), {}) + foo.__annotations__ = {} + del foo.__annotations__ + with self.assertRaises(AttributeError): + del foo.__annotations__ + + def test_annotations_are_created_correctly(self): + class C: + a:int=3 + b:str=4 + self.assertEqual(C.__annotations__, {"a": int, "b": str}) + self.assertTrue("__annotations_cache__" in C.__dict__) + del C.__annotations__ + self.assertFalse("__annotations_cache__" in C.__dict__) + + def test_pep563_annotations(self): + isa = inspect_stringized_annotations + self.assertEqual( + isa.__annotations__, {"a": "int", "b": "str"}, + ) + self.assertEqual( + isa.MyClass.__annotations__, {"a": "int", "b": "str"}, + ) + + def test_explicitly_set_annotations(self): + class C: + __annotations__ = {"what": int} + self.assertEqual(C.__annotations__, {"what": int}) + + def test_explicitly_set_annotate(self): + class C: + __annotate__ = lambda format: {"what": int} + self.assertEqual(C.__annotations__, {"what": int}) + self.assertIsInstance(C.__annotate__, types.FunctionType) + self.assertEqual(C.__annotate__(annotationlib.Format.VALUE), {"what": int}) + + def test_del_annotations_and_annotate(self): + # gh-132285 + called = False + class A: + def __annotate__(format): + nonlocal called + called = True + return {'a': int} + + self.assertEqual(A.__annotations__, {'a': int}) + self.assertTrue(called) + self.assertTrue(A.__annotate__) + + del A.__annotations__ + called = False + + self.assertEqual(A.__annotations__, {}) + self.assertFalse(called) + self.assertIs(A.__annotate__, None) + + def test_descriptor_still_works(self): + class C: + def __init__(self, name=None, bases=None, d=None): + self.my_annotations = None + + @property + def __annotations__(self): + if not hasattr(self, 'my_annotations'): + self.my_annotations = {} + if not isinstance(self.my_annotations, dict): + self.my_annotations = {} + return self.my_annotations + + @__annotations__.setter + def __annotations__(self, value): + if not isinstance(value, dict): + raise ValueError("can only set __annotations__ to a dict") + self.my_annotations = value + + @__annotations__.deleter + def __annotations__(self): + if getattr(self, 'my_annotations', False) is None: + raise AttributeError('__annotations__') + self.my_annotations = None + + c = C() + self.assertEqual(c.__annotations__, {}) + d = {'a':'int'} + c.__annotations__ = d + self.assertEqual(c.__annotations__, d) + with self.assertRaises(ValueError): + c.__annotations__ = 123 + del c.__annotations__ + with self.assertRaises(AttributeError): + del c.__annotations__ + self.assertEqual(c.__annotations__, {}) + + + class D(metaclass=C): + pass + + self.assertEqual(D.__annotations__, {}) + d = {'a':'int'} + D.__annotations__ = d + self.assertEqual(D.__annotations__, d) + with self.assertRaises(ValueError): + D.__annotations__ = 123 + del D.__annotations__ + with self.assertRaises(AttributeError): + del D.__annotations__ + self.assertEqual(D.__annotations__, {}) + + def test_partially_executed_module(self): + partialexe = import_helper.import_fresh_module("test.typinganndata.partialexecution") + self.assertEqual( + partialexe.a.__annotations__, + {"v1": int, "v2": int}, + ) + self.assertEqual(partialexe.b.annos, {"v1": int}) + + @cpython_only + def test_no_cell(self): + # gh-130924: Test that uses of annotations in local scopes do not + # create cell variables. + def f(x): + a: x + return x + + self.assertEqual(f.__code__.co_cellvars, ()) + + +def build_module(code: str, name: str = "top") -> types.ModuleType: + ns = run_code(code) + mod = types.ModuleType(name) + mod.__dict__.update(ns) + return mod + + +class TestSetupAnnotations(unittest.TestCase): + def check(self, code: str): + code = textwrap.dedent(code) + for scope in ("module", "class"): + with self.subTest(scope=scope): + if scope == "class": + code = f"class C:\n{textwrap.indent(code, ' ')}" + ns = run_code(code) + annotations = ns["C"].__annotations__ + else: + annotations = build_module(code).__annotations__ + self.assertEqual(annotations, {"x": int}) + + def test_top_level(self): + self.check("x: int = 1") + + def test_blocks(self): + self.check("if True:\n x: int = 1") + self.check(""" + while True: + x: int = 1 + break + """) + self.check(""" + while False: + pass + else: + x: int = 1 + """) + self.check(""" + for i in range(1): + x: int = 1 + """) + self.check(""" + for i in range(1): + pass + else: + x: int = 1 + """) + + def test_try(self): + self.check(""" + try: + x: int = 1 + except: + pass + """) + self.check(""" + try: + pass + except: + pass + else: + x: int = 1 + """) + self.check(""" + try: + pass + except: + pass + finally: + x: int = 1 + """) + self.check(""" + try: + 1/0 + except: + x: int = 1 + """) + + def test_try_star(self): + self.check(""" + try: + x: int = 1 + except* Exception: + pass + """) + self.check(""" + try: + pass + except* Exception: + pass + else: + x: int = 1 + """) + self.check(""" + try: + pass + except* Exception: + pass + finally: + x: int = 1 + """) + self.check(""" + try: + 1/0 + except* Exception: + x: int = 1 + """) + + def test_match(self): + self.check(""" + match 0: + case 0: + x: int = 1 + """) + + +class AnnotateTests(unittest.TestCase): + """See PEP 649.""" + def test_manual_annotate(self): + def f(): + pass + mod = types.ModuleType("mod") + class X: + pass + + for obj in (f, mod, X): + with self.subTest(obj=obj): + self.check_annotations(obj) + + def check_annotations(self, f): + self.assertEqual(f.__annotations__, {}) + self.assertIs(f.__annotate__, None) + + with self.assertRaisesRegex(TypeError, "__annotate__ must be callable or None"): + f.__annotate__ = 42 + f.__annotate__ = lambda: 42 + with self.assertRaisesRegex(TypeError, r"takes 0 positional arguments but 1 was given"): + print(f.__annotations__) + + f.__annotate__ = lambda x: 42 + with self.assertRaisesRegex(TypeError, r"__annotate__ returned non-dict of type 'int'"): + print(f.__annotations__) + + f.__annotate__ = lambda x: {"x": x} + self.assertEqual(f.__annotations__, {"x": 1}) + + # Setting annotate to None does not invalidate the cached __annotations__ + f.__annotate__ = None + self.assertEqual(f.__annotations__, {"x": 1}) + + # But setting it to a new callable does + f.__annotate__ = lambda x: {"y": x} + self.assertEqual(f.__annotations__, {"y": 1}) + + # Setting f.__annotations__ also clears __annotate__ + f.__annotations__ = {"z": 43} + self.assertIs(f.__annotate__, None) + + def test_user_defined_annotate(self): + class X: + a: int + + def __annotate__(format): + return {"a": str} + self.assertEqual(X.__annotate__(annotationlib.Format.VALUE), {"a": str}) + self.assertEqual(annotationlib.get_annotations(X), {"a": str}) + + mod = build_module( + """ + a: int + def __annotate__(format): + return {"a": str} + """ + ) + self.assertEqual(mod.__annotate__(annotationlib.Format.VALUE), {"a": str}) + self.assertEqual(annotationlib.get_annotations(mod), {"a": str}) + + +class DeferredEvaluationTests(unittest.TestCase): + def test_function(self): + def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_async_function(self): + async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_class(self): + class X: + a: undefined + + with self.assertRaises(NameError): + X.__annotations__ + + undefined = 1 + self.assertEqual(X.__annotations__, {"a": 1}) + + def test_module(self): + ns = run_code("x: undefined = 1") + anno = ns["__annotate__"] + with self.assertRaises(NotImplementedError): + anno(3) + + with self.assertRaises(NameError): + anno(1) + + ns["undefined"] = 1 + self.assertEqual(anno(1), {"x": 1}) + + def test_class_scoping(self): + class Outer: + def meth(self, x: Nested): ... + x: Nested + class Nested: ... + + self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) + self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) + + def test_no_exotic_expressions(self): + preludes = [ + "", + "class X:\n ", + "def f():\n ", + "async def f():\n ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "def func(x: (yield)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "def func(x: (await 42)): ...", "await expression cannot be used within an annotation") + check_syntax_error(self, prelude + "def func(x: [y async for y in x]): ...", "asynchronous comprehension outside of an asynchronous function") + check_syntax_error(self, prelude + "def func(x: {y async for y in x}): ...", "asynchronous comprehension outside of an asynchronous function") + check_syntax_error(self, prelude + "def func(x: {y: y async for y in x}): ...", "asynchronous comprehension outside of an asynchronous function") + + def test_no_exotic_expressions_in_unevaluated_annotations(self): + preludes = [ + "", + "class X: ", + "def f(): ", + "async def f(): ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (__debug__ := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): [y async for y in x]", "asynchronous comprehension outside of an asynchronous function") + check_syntax_error(self, prelude + "(x): {y async for y in x}", "asynchronous comprehension outside of an asynchronous function") + check_syntax_error(self, prelude + "(x): {y: y async for y in x}", "asynchronous comprehension outside of an asynchronous function") + + def test_ignore_non_simple_annotations(self): + ns = run_code("class X: (y): int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int.b: int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int[str]: int") + self.assertEqual(ns["X"].__annotations__, {}) + + def test_generated_annotate(self): + def func(x: int): + pass + class X: + x: int + mod = build_module("x: int") + for obj in (func, X, mod): + with self.subTest(obj=obj): + annotate = obj.__annotate__ + self.assertIsInstance(annotate, types.FunctionType) + self.assertEqual(annotate.__name__, "__annotate__") + with self.assertRaises(NotImplementedError): + annotate(annotationlib.Format.FORWARDREF) + with self.assertRaises(NotImplementedError): + annotate(annotationlib.Format.STRING) + with self.assertRaises(TypeError): + annotate(None) + self.assertEqual(annotate(annotationlib.Format.VALUE), {"x": int}) + + sig = inspect.signature(annotate) + self.assertEqual(sig, inspect.Signature([ + inspect.Parameter("format", inspect.Parameter.POSITIONAL_ONLY) + ])) + + def test_comprehension_in_annotation(self): + # This crashed in an earlier version of the code + ns = run_code("x: [y for y in range(10)]") + self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + + def f(x: int) -> int: pass + """ + ns = run_code(code) + f = ns["f"] + self.assertIsInstance(f.__annotate__, types.FunctionType) + annos = {"x": "int", "return": "int"} + self.assertEqual(f.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(f.__annotations__, annos) + + def test_set_annotations(self): + function_code = textwrap.dedent(""" + def f(x: int): + pass + """) + class_code = textwrap.dedent(""" + class f: + x: int + """) + for future in (False, True): + for label, code in (("function", function_code), ("class", class_code)): + with self.subTest(future=future, label=label): + if future: + code = "from __future__ import annotations\n" + code + ns = run_code(code) + f = ns["f"] + anno = "int" if future else int + self.assertEqual(f.__annotations__, {"x": anno}) + + f.__annotations__ = {"x": str} + self.assertEqual(f.__annotations__, {"x": str}) + + def test_name_clash_with_format(self): + # this test would fail if __annotate__'s parameter was called "format" + # during symbol table construction + code = """ + class format: pass + + def f(x: format): pass + """ + ns = run_code(code) + f = ns["f"] + self.assertEqual(f.__annotations__, {"x": ns["format"]}) + + code = """ + class Outer: + class format: pass + + def meth(self, x: format): ... + """ + ns = run_code(code) + self.assertEqual(ns["Outer"].meth.__annotations__, {"x": ns["Outer"].format}) + + code = """ + def f(format): + def inner(x: format): pass + return inner + res = f("closure var") + """ + ns = run_code(code) + self.assertEqual(ns["res"].__annotations__, {"x": "closure var"}) + + code = """ + def f(x: format): + pass + """ + ns = run_code(code) + # picks up the format() builtin + self.assertEqual(ns["f"].__annotations__, {"x": format}) + + code = """ + def outer(): + def f(x: format): + pass + if False: + class format: pass + return f + f = outer() + """ + ns = run_code(code) + with self.assertRaisesRegex( + NameError, + "cannot access free variable 'format' where it is not associated with a value in enclosing scope", + ): + ns["f"].__annotations__ + + +class ConditionalAnnotationTests(unittest.TestCase): + def check_scopes(self, code, true_annos, false_annos): + for scope in ("class", "module"): + for (cond, expected) in ( + # Constants (so code might get optimized out) + (True, true_annos), (False, false_annos), + # Non-constant expressions + ("not not len", true_annos), ("not len", false_annos), + ): + with self.subTest(scope=scope, cond=cond): + code_to_run = code.format(cond=cond) + if scope == "class": + code_to_run = "class Cls:\n" + textwrap.indent(textwrap.dedent(code_to_run), " " * 4) + ns = run_code(code_to_run) + if scope == "class": + self.assertEqual(ns["Cls"].__annotations__, expected) + else: + self.assertEqual(ns["__annotate__"](annotationlib.Format.VALUE), + expected) + + def test_with(self): + code = """ + class Swallower: + def __enter__(self): + pass + + def __exit__(self, *args): + return True + + with Swallower(): + if {cond}: + about_to_raise: int + raise Exception + in_with: "with" + """ + self.check_scopes(code, {"about_to_raise": int}, {"in_with": "with"}) + + def test_simple_if(self): + code = """ + if {cond}: + in_if: "if" + else: + in_if: "else" + """ + self.check_scopes(code, {"in_if": "if"}, {"in_if": "else"}) + + def test_if_elif(self): + code = """ + if not len: + in_if: "if" + elif {cond}: + in_elif: "elif" + else: + in_else: "else" + """ + self.check_scopes( + code, + {"in_elif": "elif"}, + {"in_else": "else"} + ) + + def test_try(self): + code = """ + try: + if {cond}: + raise Exception + in_try: "try" + except Exception: + in_except: "except" + finally: + in_finally: "finally" + """ + self.check_scopes( + code, + {"in_except": "except", "in_finally": "finally"}, + {"in_try": "try", "in_finally": "finally"} + ) + + def test_try_star(self): + code = """ + try: + if {cond}: + raise Exception + in_try_star: "try" + except* Exception: + in_except_star: "except" + finally: + in_finally: "finally" + """ + self.check_scopes( + code, + {"in_except_star": "except", "in_finally": "finally"}, + {"in_try_star": "try", "in_finally": "finally"} + ) + + def test_while(self): + code = """ + while {cond}: + in_while: "while" + break + else: + in_else: "else" + """ + self.check_scopes( + code, + {"in_while": "while"}, + {"in_else": "else"} + ) + + def test_for(self): + code = """ + for _ in ([1] if {cond} else []): + in_for: "for" + else: + in_else: "else" + """ + self.check_scopes( + code, + {"in_for": "for", "in_else": "else"}, + {"in_else": "else"} + ) + + def test_match(self): + code = """ + match {cond}: + case True: + x: "true" + case False: + x: "false" + """ + self.check_scopes( + code, + {"x": "true"}, + {"x": "false"} + ) + + def test_nesting_override(self): + code = """ + if {cond}: + x: "foo" + if {cond}: + x: "bar" + """ + self.check_scopes( + code, + {"x": "bar"}, + {} + ) + + def test_nesting_outer(self): + code = """ + if {cond}: + outer_before: "outer_before" + if len: + inner_if: "inner_if" + else: + inner_else: "inner_else" + outer_after: "outer_after" + """ + self.check_scopes( + code, + {"outer_before": "outer_before", "inner_if": "inner_if", + "outer_after": "outer_after"}, + {} + ) + + def test_nesting_inner(self): + code = """ + if len: + outer_before: "outer_before" + if {cond}: + inner_if: "inner_if" + else: + inner_else: "inner_else" + outer_after: "outer_after" + """ + self.check_scopes( + code, + {"outer_before": "outer_before", "inner_if": "inner_if", + "outer_after": "outer_after"}, + {"outer_before": "outer_before", "inner_else": "inner_else", + "outer_after": "outer_after"}, + ) + + def test_non_name_annotations(self): + code = """ + before: "before" + if {cond}: + a = "x" + a[0]: int + else: + a = object() + a.b: str + after: "after" + """ + expected = {"before": "before", "after": "after"} + self.check_scopes(code, expected, expected) + + +class RegressionTests(unittest.TestCase): + # gh-132479 + def test_complex_comprehension_inlining(self): + # Test that the various repro cases from the issue don't crash + cases = [ + """ + (unique_name_0): 0 + unique_name_1: ( + 0 + for ( + 0 + for unique_name_2 in 0 + for () in (0 for unique_name_3 in unique_name_4 for unique_name_5 in name_1) + ).name_3 in {0: 0 for name_1 in unique_name_8} + if name_1 + ) + """, + """ + unique_name_0: 0 + unique_name_1: { + 0: 0 + for unique_name_2 in [0 for name_0 in unique_name_4] + if { + 0: 0 + for unique_name_5 in 0 + if name_0 + if ((name_0 for unique_name_8 in unique_name_9) for [] in 0) + } + } + """, + """ + 0[0]: {0 for name_0 in unique_name_1} + unique_name_2: { + 0: (lambda: name_0 for unique_name_4 in unique_name_5) + for unique_name_6 in () + if name_0 + } + """, + ] + for case in cases: + case = textwrap.dedent(case) + compile(case, "<test>", "exec") + + def test_complex_comprehension_inlining_exec(self): + code = """ + unique_name_1 = unique_name_5 = [1] + name_0 = 42 + unique_name_7: {name_0 for name_0 in unique_name_1} + unique_name_2: { + 0: (lambda: name_0 for unique_name_4 in unique_name_5) + for unique_name_6 in [1] + if name_0 + } + """ + mod = build_module(code) + annos = mod.__annotations__ + self.assertEqual(annos.keys(), {"unique_name_7", "unique_name_2"}) + self.assertEqual(annos["unique_name_7"], {True}) + genexp = annos["unique_name_2"][0] + lamb = list(genexp)[0] + self.assertEqual(lamb(), 42) + + # gh-138349 + def test_module_level_annotation_plus_listcomp(self): + cases = [ + """ + def report_error(): + pass + try: + [0 for name_2 in unique_name_0 if (lambda: name_2)] + except: + pass + annotated_name: 0 + """, + """ + class Generic: + pass + try: + [0 for name_2 in unique_name_0 if (0 for unique_name_1 in unique_name_2 for unique_name_3 in name_2)] + except: + pass + annotated_name: 0 + """, + """ + class Generic: + pass + annotated_name: 0 + try: + [0 for name_2 in [[0]] for unique_name_1 in unique_name_2 if (lambda: name_2)] + except: + pass + """, + ] + for code in cases: + with self.subTest(code=code): + mod = build_module(code) + annos = mod.__annotations__ + self.assertEqual(annos, {"annotated_name": 0}) diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index 578c138767d..4f41171b2df 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -1,7 +1,6 @@ import ast import sys import unittest -from test import support funcdef = """\ @@ -67,6 +66,14 @@ def foo(): pass """ +parenthesized_withstmt = """\ +with (a as b): # type: int + pass + +with (a, b): # type: int + pass +""" + vardecl = """\ a = 0 # type: int """ @@ -244,8 +251,7 @@ def parse_all(self, source, minver=lowest, maxver=highest, expected_regex=""): def classic_parse(self, source): return ast.parse(source) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FunctionDef' object has no attribute 'type_comment' def test_funcdef(self): for tree in self.parse_all(funcdef): self.assertEqual(tree.body[0].type_comment, "() -> int") @@ -254,8 +260,7 @@ def test_funcdef(self): self.assertEqual(tree.body[0].type_comment, None) self.assertEqual(tree.body[1].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_asyncdef(self): for tree in self.parse_all(asyncdef, minver=5): self.assertEqual(tree.body[0].type_comment, "() -> int") @@ -264,75 +269,71 @@ def test_asyncdef(self): self.assertEqual(tree.body[0].type_comment, None) self.assertEqual(tree.body[1].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_asyncvar(self): - for tree in self.parse_all(asyncvar, maxver=6): - pass + with self.assertRaises(SyntaxError): + self.classic_parse(asyncvar) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_asynccomp(self): for tree in self.parse_all(asynccomp, minver=6): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_matmul(self): for tree in self.parse_all(matmul, minver=5): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fstring(self): - for tree in self.parse_all(fstring, minver=6): + for tree in self.parse_all(fstring): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_underscorednumber(self): for tree in self.parse_all(underscorednumber, minver=6): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_redundantdef(self): for tree in self.parse_all(redundantdef, maxver=0, expected_regex="^Cannot have two type comments on def"): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FunctionDef' object has no attribute 'type_comment' def test_nonasciidef(self): for tree in self.parse_all(nonasciidef): self.assertEqual(tree.body[0].type_comment, "() -> àçčéñt") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'For' object has no attribute 'type_comment' def test_forstmt(self): for tree in self.parse_all(forstmt): self.assertEqual(tree.body[0].type_comment, "int") tree = self.classic_parse(forstmt) self.assertEqual(tree.body[0].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'With' object has no attribute 'type_comment' def test_withstmt(self): for tree in self.parse_all(withstmt): self.assertEqual(tree.body[0].type_comment, "int") tree = self.classic_parse(withstmt) self.assertEqual(tree.body[0].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'With' object has no attribute 'type_comment' + def test_parenthesized_withstmt(self): + for tree in self.parse_all(parenthesized_withstmt): + self.assertEqual(tree.body[0].type_comment, "int") + self.assertEqual(tree.body[1].type_comment, "int") + tree = self.classic_parse(parenthesized_withstmt) + self.assertEqual(tree.body[0].type_comment, None) + self.assertEqual(tree.body[1].type_comment, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None != 'int' def test_vardecl(self): for tree in self.parse_all(vardecl): self.assertEqual(tree.body[0].type_comment, "int") tree = self.classic_parse(vardecl) self.assertEqual(tree.body[0].type_comment, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + (11, ' whatever')] def test_ignores(self): for tree in self.parse_all(ignores): self.assertEqual( @@ -348,16 +349,15 @@ def test_ignores(self): tree = self.classic_parse(ignores) self.assertEqual(tree.type_ignores, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: SyntaxError not raised : feature_version=(3, 4) def test_longargs(self): - for tree in self.parse_all(longargs): + for tree in self.parse_all(longargs, minver=8): for t in tree.body: # The expected args are encoded in the function name todo = set(t.name[1:]) self.assertEqual(len(t.args.args) + len(t.args.posonlyargs), len(todo) - bool(t.args.vararg) - bool(t.args.kwarg)) - self.assertTrue(t.name.startswith('f'), t.name) + self.assertStartsWith(t.name, 'f') for index, c in enumerate(t.name[1:]): todo.remove(c) if c == 'v': @@ -380,8 +380,7 @@ def test_longargs(self): self.assertIsNone(arg.type_comment, "%s(%s:%r)" % (t.name, arg.arg, arg.type_comment)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Tests for inappropriately-placed type comments. def test_inappropriate_type_comments(self): """Tests for inappropriately-placed type comments. @@ -406,8 +405,7 @@ def check_both_ways(source): check_both_ways("pass # type: ignorewhatever\n") check_both_ways("pass # type: ignoreé\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: mode must be "exec", "eval", "ipython", or "single" def test_func_type_input(self): def parse_func_type_input(source): diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py new file mode 100644 index 00000000000..af26a6a0a01 --- /dev/null +++ b/Lib/test/test_type_params.py @@ -0,0 +1,1468 @@ +import annotationlib +import textwrap +import types +import unittest +import pickle +import weakref +from test.support import check_syntax_error, run_code, run_no_yield_async_fn + +from typing import Generic, NoDefault, Sequence, TypeAliasType, TypeVar, TypeVarTuple, ParamSpec, get_args + + +class TypeParamsInvalidTest(unittest.TestCase): + def test_name_collisions(self): + check_syntax_error(self, 'def func[**A, A](): ...', "duplicate type parameter 'A'") + check_syntax_error(self, 'def func[A, *A](): ...', "duplicate type parameter 'A'") + check_syntax_error(self, 'def func[*A, **A](): ...', "duplicate type parameter 'A'") + + check_syntax_error(self, 'class C[**A, A](): ...', "duplicate type parameter 'A'") + check_syntax_error(self, 'class C[A, *A](): ...', "duplicate type parameter 'A'") + check_syntax_error(self, 'class C[*A, **A](): ...', "duplicate type parameter 'A'") + + def test_name_non_collision_02(self): + ns = run_code("""def func[A](A): return A""") + func = ns["func"] + self.assertEqual(func(1), 1) + A, = func.__type_params__ + self.assertEqual(A.__name__, "A") + + def test_name_non_collision_03(self): + ns = run_code("""def func[A](*A): return A""") + func = ns["func"] + self.assertEqual(func(1), (1,)) + A, = func.__type_params__ + self.assertEqual(A.__name__, "A") + + def test_name_non_collision_04(self): + # Mangled names should not cause a conflict. + ns = run_code(""" + class ClassA: + def func[__A](self, __A): return __A + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(1), 1) + A, = cls.func.__type_params__ + self.assertEqual(A.__name__, "__A") + + def test_name_non_collision_05(self): + ns = run_code(""" + class ClassA: + def func[_ClassA__A](self, __A): return __A + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(1), 1) + A, = cls.func.__type_params__ + self.assertEqual(A.__name__, "_ClassA__A") + + def test_name_non_collision_06(self): + ns = run_code(""" + class ClassA[X]: + def func(self, X): return X + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(1), 1) + X, = cls.__type_params__ + self.assertEqual(X.__name__, "X") + + def test_name_non_collision_07(self): + ns = run_code(""" + class ClassA[X]: + def func(self): + X = 1 + return X + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(), 1) + X, = cls.__type_params__ + self.assertEqual(X.__name__, "X") + + def test_name_non_collision_08(self): + ns = run_code(""" + class ClassA[X]: + def func(self): + return [X for X in [1, 2]] + """ + ) + cls = ns["ClassA"] + self.assertEqual(cls().func(), [1, 2]) + X, = cls.__type_params__ + self.assertEqual(X.__name__, "X") + + def test_name_non_collision_9(self): + ns = run_code(""" + class ClassA[X]: + def func[X](self): + ... + """ + ) + cls = ns["ClassA"] + outer_X, = cls.__type_params__ + inner_X, = cls.func.__type_params__ + self.assertEqual(outer_X.__name__, "X") + self.assertEqual(inner_X.__name__, "X") + self.assertIsNot(outer_X, inner_X) + + def test_name_non_collision_10(self): + ns = run_code(""" + class ClassA[X]: + X: int + """ + ) + cls = ns["ClassA"] + X, = cls.__type_params__ + self.assertEqual(X.__name__, "X") + self.assertIs(cls.__annotations__["X"], int) + + def test_name_non_collision_13(self): + ns = run_code(""" + X = 1 + def outer(): + def inner[X](): + global X + X = 2 + return inner + """ + ) + self.assertEqual(ns["X"], 1) + outer = ns["outer"] + outer()() + self.assertEqual(ns["X"], 2) + + def test_disallowed_expressions(self): + check_syntax_error(self, "type X = (yield)") + check_syntax_error(self, "type X = (yield from x)") + check_syntax_error(self, "type X = (await 42)") + check_syntax_error(self, "async def f(): type X = (yield)") + check_syntax_error(self, "type X = (y := 3)") + check_syntax_error(self, "class X[T: (yield)]: pass") + check_syntax_error(self, "class X[T: (yield from x)]: pass") + check_syntax_error(self, "class X[T: (await 42)]: pass") + check_syntax_error(self, "class X[T: (y := 3)]: pass") + check_syntax_error(self, "class X[T](y := Sequence[T]): pass") + check_syntax_error(self, "def f[T](y: (x := Sequence[T])): pass") + check_syntax_error(self, "class X[T]([(x := 3) for _ in range(2)] and B): pass") + check_syntax_error(self, "def f[T: [(x := 3) for _ in range(2)]](): pass") + check_syntax_error(self, "type T = [(x := 3) for _ in range(2)]") + + def test_incorrect_mro_explicit_object(self): + with self.assertRaisesRegex(TypeError, r"\(MRO\) for bases object, Generic"): + class My[X](object): ... + + +class TypeParamsNonlocalTest(unittest.TestCase): + def test_nonlocal_disallowed_01(self): + code = """ + def outer(): + X = 1 + def inner[X](): + nonlocal X + return X + """ + check_syntax_error(self, code) + + def test_nonlocal_disallowed_02(self): + code = """ + def outer2[T](): + def inner1(): + nonlocal T + """ + check_syntax_error(self, textwrap.dedent(code)) + + def test_nonlocal_disallowed_03(self): + code = """ + class Cls[T]: + nonlocal T + """ + check_syntax_error(self, textwrap.dedent(code)) + + def test_nonlocal_allowed(self): + code = """ + def func[T](): + T = "func" + def inner(): + nonlocal T + T = "inner" + inner() + assert T == "inner" + """ + ns = run_code(code) + func = ns["func"] + T, = func.__type_params__ + self.assertEqual(T.__name__, "T") + + +class TypeParamsAccessTest(unittest.TestCase): + def test_class_access_01(self): + ns = run_code(""" + class ClassA[A, B](dict[A, B]): + ... + """ + ) + cls = ns["ClassA"] + A, B = cls.__type_params__ + self.assertEqual(types.get_original_bases(cls), (dict[A, B], Generic[A, B])) + + def test_class_access_02(self): + ns = run_code(""" + class MyMeta[A, B](type): ... + class ClassA[A, B](metaclass=MyMeta[A, B]): + ... + """ + ) + meta = ns["MyMeta"] + cls = ns["ClassA"] + A1, B1 = meta.__type_params__ + A2, B2 = cls.__type_params__ + self.assertIsNot(A1, A2) + self.assertIsNot(B1, B2) + self.assertIs(type(cls), meta) + + def test_class_access_03(self): + code = """ + def my_decorator(a): + ... + @my_decorator(A) + class ClassA[A, B](): + ... + """ + + with self.assertRaisesRegex(NameError, "name 'A' is not defined"): + run_code(code) + + def test_function_access_01(self): + ns = run_code(""" + def func[A, B](a: dict[A, B]): + ... + """ + ) + func = ns["func"] + A, B = func.__type_params__ + self.assertEqual(func.__annotations__["a"], dict[A, B]) + + def test_function_access_02(self): + code = """ + def func[A](a = list[A]()): + ... + """ + + with self.assertRaisesRegex(NameError, "name 'A' is not defined"): + run_code(code) + + def test_function_access_03(self): + code = """ + def my_decorator(a): + ... + @my_decorator(A) + def func[A](): + ... + """ + + with self.assertRaisesRegex(NameError, "name 'A' is not defined"): + run_code(code) + + def test_method_access_01(self): + ns = run_code(""" + class ClassA: + x = int + def func[T](self, a: x, b: T): + ... + """ + ) + cls = ns["ClassA"] + self.assertIs(cls.func.__annotations__["a"], int) + T, = cls.func.__type_params__ + self.assertIs(cls.func.__annotations__["b"], T) + + def test_nested_access_01(self): + ns = run_code(""" + class ClassA[A]: + def funcB[B](self): + class ClassC[C]: + def funcD[D](self): + return lambda: (A, B, C, D) + return ClassC + """ + ) + cls = ns["ClassA"] + A, = cls.__type_params__ + B, = cls.funcB.__type_params__ + classC = cls().funcB() + C, = classC.__type_params__ + D, = classC.funcD.__type_params__ + self.assertEqual(classC().funcD()(), (A, B, C, D)) + + def test_out_of_scope_01(self): + code = """ + class ClassA[T]: ... + x = T + """ + + with self.assertRaisesRegex(NameError, "name 'T' is not defined"): + run_code(code) + + def test_out_of_scope_02(self): + code = """ + class ClassA[A]: + def funcB[B](self): ... + + x = B + """ + + with self.assertRaisesRegex(NameError, "name 'B' is not defined"): + run_code(code) + + def test_class_scope_interaction_01(self): + ns = run_code(""" + class C: + x = 1 + def method[T](self, arg: x): pass + """) + cls = ns["C"] + self.assertEqual(cls.method.__annotations__["arg"], 1) + + def test_class_scope_interaction_02(self): + ns = run_code(""" + class C: + class Base: pass + class Child[T](Base): pass + """) + cls = ns["C"] + self.assertEqual(cls.Child.__bases__, (cls.Base, Generic)) + T, = cls.Child.__type_params__ + self.assertEqual(types.get_original_bases(cls.Child), (cls.Base, Generic[T])) + + def test_class_deref(self): + ns = run_code(""" + class C[T]: + T = "class" + type Alias = T + """) + cls = ns["C"] + self.assertEqual(cls.Alias.__value__, "class") + + def test_shadowing_nonlocal(self): + ns = run_code(""" + def outer[T](): + T = "outer" + def inner(): + nonlocal T + T = "inner" + return T + return lambda: T, inner + """) + outer = ns["outer"] + T, = outer.__type_params__ + self.assertEqual(T.__name__, "T") + getter, inner = outer() + self.assertEqual(getter(), "outer") + self.assertEqual(inner(), "inner") + self.assertEqual(getter(), "inner") + + def test_reference_previous_typevar(self): + def func[S, T: Sequence[S]](): + pass + + S, T = func.__type_params__ + self.assertEqual(T.__bound__, Sequence[S]) + + def test_super(self): + class Base: + def meth(self): + return "base" + + class Child(Base): + # Having int in the annotation ensures the class gets cells for both + # __class__ and __classdict__ + def meth[T](self, arg: int) -> T: + return super().meth() + "child" + + c = Child() + self.assertEqual(c.meth(1), "basechild") + + def test_type_alias_containing_lambda(self): + type Alias[T] = lambda: T + T, = Alias.__type_params__ + self.assertIs(Alias.__value__(), T) + + def test_class_base_containing_lambda(self): + # Test that scopes nested inside hidden functions work correctly + outer_var = "outer" + class Base[T]: ... + class Child[T](Base[lambda: (int, outer_var, T)]): ... + base, _ = types.get_original_bases(Child) + func, = get_args(base) + T, = Child.__type_params__ + self.assertEqual(func(), (int, "outer", T)) + + def test_comprehension_01(self): + type Alias[T: ([T for T in (T, [1])[1]], T)] = [T for T in T.__name__] + self.assertEqual(Alias.__value__, ["T"]) + T, = Alias.__type_params__ + self.assertEqual(T.__constraints__, ([1], T)) + + def test_comprehension_02(self): + type Alias[T: [lambda: T for T in (T, [1])[1]]] = [lambda: T for T in T.__name__] + func, = Alias.__value__ + self.assertEqual(func(), "T") + T, = Alias.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + + def test_comprehension_03(self): + def F[T: [lambda: T for T in (T, [1])[1]]](): return [lambda: T for T in T.__name__] + func, = F() + self.assertEqual(func(), "T") + T, = F.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + + def test_gen_exp_in_nested_class(self): + code = """ + from test.test_type_params import make_base + + class C[T]: + T = "class" + class Inner(make_base(T for _ in (1,)), make_base(T)): + pass + """ + C = run_code(code)["C"] + T, = C.__type_params__ + base1, base2 = C.Inner.__bases__ + self.assertEqual(list(base1.__arg__), [T]) + self.assertEqual(base2.__arg__, "class") + + def test_gen_exp_in_nested_generic_class(self): + code = """ + from test.test_type_params import make_base + + class C[T]: + T = "class" + class Inner[U](make_base(T for _ in (1,)), make_base(T)): + pass + """ + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(list(base1.__arg__), [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") + + def test_listcomp_in_nested_class(self): + code = """ + from test.test_type_params import make_base + + class C[T]: + T = "class" + class Inner(make_base([T for _ in (1,)]), make_base(T)): + pass + """ + C = run_code(code)["C"] + T, = C.__type_params__ + base1, base2 = C.Inner.__bases__ + self.assertEqual(base1.__arg__, [T]) + self.assertEqual(base2.__arg__, "class") + + def test_listcomp_in_nested_generic_class(self): + code = """ + from test.test_type_params import make_base + + class C[T]: + T = "class" + class Inner[U](make_base([T for _ in (1,)]), make_base(T)): + pass + """ + ns = run_code(code) + inner = ns["C"].Inner + base1, base2, _ = inner.__bases__ + self.assertEqual(base1.__arg__, [ns["C"].__type_params__[0]]) + self.assertEqual(base2.__arg__, "class") + + def test_gen_exp_in_generic_method(self): + code = """ + class C[T]: + T = "class" + def meth[U](x: (T for _ in (1,)), y: T): + pass + """ + ns = run_code(code) + meth = ns["C"].meth + self.assertEqual(list(meth.__annotations__["x"]), [ns["C"].__type_params__[0]]) + self.assertEqual(meth.__annotations__["y"], "class") + + def test_nested_scope_in_generic_alias(self): + code = """ + T = "global" + class C: + T = "class" + {} + """ + cases = [ + "type Alias[T] = (T for _ in (1,))", + "type Alias = (T for _ in (1,))", + "type Alias[T] = [T for _ in (1,)]", + "type Alias = [T for _ in (1,)]", + ] + for case in cases: + with self.subTest(case=case): + ns = run_code(code.format(case)) + alias = ns["C"].Alias + value = list(alias.__value__)[0] + if alias.__type_params__: + self.assertIs(value, alias.__type_params__[0]) + else: + self.assertEqual(value, "global") + + def test_lambda_in_alias_in_class(self): + code = """ + T = "global" + class C: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertEqual(C.Alias.__value__(), "global") + + def test_lambda_in_alias_in_generic_class(self): + code = """ + class C[T]: + T = "class" + type Alias = lambda: T + """ + C = run_code(code)["C"] + self.assertIs(C.Alias.__value__(), C.__type_params__[0]) + + def test_lambda_in_generic_alias_in_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + T = U = "global" + class C: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertEqual(U, "global") + + def test_lambda_in_generic_alias_in_generic_class(self): + # A lambda nested in the alias cannot see the class scope, but can see + # a surrounding annotation scope. + code = """ + class C[T, U]: + T = "class" + U = "class" + type Alias[T] = lambda: (T, U) + """ + C = run_code(code)["C"] + T, U = C.Alias.__value__() + self.assertIs(T, C.Alias.__type_params__[0]) + self.assertIs(U, C.__type_params__[1]) + + def test_type_special_case(self): + # https://github.com/python/cpython/issues/119011 + self.assertEqual(type.__type_params__, ()) + self.assertEqual(object.__type_params__, ()) + + +def make_base(arg): + class Base: + __arg__ = arg + return Base + + +def global_generic_func[T](): + pass + +class GlobalGenericClass[T]: + pass + + +class TypeParamsLazyEvaluationTest(unittest.TestCase): + def test_qualname(self): + class Foo[T]: + pass + + def func[T](): + pass + + self.assertEqual(Foo.__qualname__, "TypeParamsLazyEvaluationTest.test_qualname.<locals>.Foo") + self.assertEqual(func.__qualname__, "TypeParamsLazyEvaluationTest.test_qualname.<locals>.func") + self.assertEqual(global_generic_func.__qualname__, "global_generic_func") + self.assertEqual(GlobalGenericClass.__qualname__, "GlobalGenericClass") + + def test_recursive_class(self): + class Foo[T: Foo, U: (Foo, Foo)]: + pass + + type_params = Foo.__type_params__ + self.assertEqual(len(type_params), 2) + self.assertEqual(type_params[0].__name__, "T") + self.assertIs(type_params[0].__bound__, Foo) + self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[0].__default__, NoDefault) + + self.assertEqual(type_params[1].__name__, "U") + self.assertIs(type_params[1].__bound__, None) + self.assertEqual(type_params[1].__constraints__, (Foo, Foo)) + self.assertIs(type_params[1].__default__, NoDefault) + + def test_evaluation_error(self): + class Foo[T: Undefined, U: (Undefined,)]: + pass + + type_params = Foo.__type_params__ + with self.assertRaises(NameError): + type_params[0].__bound__ + self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[1].__bound__, None) + self.assertIs(type_params[0].__default__, NoDefault) + self.assertIs(type_params[1].__default__, NoDefault) + with self.assertRaises(NameError): + type_params[1].__constraints__ + + Undefined = "defined" + self.assertEqual(type_params[0].__bound__, "defined") + self.assertEqual(type_params[0].__constraints__, ()) + + self.assertIs(type_params[1].__bound__, None) + self.assertEqual(type_params[1].__constraints__, ("defined",)) + + +class TypeParamsClassScopeTest(unittest.TestCase): + def test_alias(self): + class X: + T = int + type U = T + self.assertIs(X.U.__value__, int) + + ns = run_code(""" + glb = "global" + class X: + cls = "class" + type U = (glb, cls) + """) + cls = ns["X"] + self.assertEqual(cls.U.__value__, ("global", "class")) + + def test_bound(self): + class X: + T = int + def foo[U: T](self): ... + self.assertIs(X.foo.__type_params__[0].__bound__, int) + + ns = run_code(""" + glb = "global" + class X: + cls = "class" + def foo[T: glb, U: cls](self): ... + """) + cls = ns["X"] + T, U = cls.foo.__type_params__ + self.assertEqual(T.__bound__, "global") + self.assertEqual(U.__bound__, "class") + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'int'> is not <class 'float'> + def test_modified_later(self): + class X: + T = int + def foo[U: T](self): ... + type Alias = T + X.T = float + self.assertIs(X.foo.__type_params__[0].__bound__, float) + self.assertIs(X.Alias.__value__, float) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + global + def test_binding_uses_global(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + type Alias = x + val = Alias.__value__ + def meth[T: x](self, arg: x): ... + bound = meth.__type_params__[0].__bound__ + annotation = meth.__annotations__["arg"] + x = "class" + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.val, "global") + self.assertEqual(cls.bound, "global") + self.assertEqual(cls.annotation, "global") + + def test_no_binding_uses_nonlocal(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + type Alias = x + val = Alias.__value__ + def meth[T: x](self, arg: x): ... + bound = meth.__type_params__[0].__bound__ + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.val, "nonlocal") + self.assertEqual(cls.bound, "nonlocal") + self.assertEqual(cls.meth.__annotations__["arg"], "nonlocal") + + @unittest.expectedFailure # TODO: RUSTPYTHON; + global + def test_explicit_global(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + global x + type Alias = x + Cls.x = "class" + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.Alias.__value__, "global") + + def test_explicit_global_with_no_static_bound(self): + ns = run_code(""" + def outer(): + class Cls: + global x + type Alias = x + Cls.x = "class" + return Cls + """) + ns["x"] = "global" + cls = ns["outer"]() + self.assertEqual(cls.Alias.__value__, "global") + + @unittest.expectedFailure # TODO: RUSTPYTHON; + global from class + def test_explicit_global_with_assignment(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + global x + type Alias = x + x = "global from class" + Cls.x = "class" + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.Alias.__value__, "global from class") + + def test_explicit_nonlocal(self): + ns = run_code(""" + x = "global" + def outer(): + x = "nonlocal" + class Cls: + nonlocal x + type Alias = x + x = "class" + return Cls + """) + cls = ns["outer"]() + self.assertEqual(cls.Alias.__value__, "class") + + def test_nested_free(self): + ns = run_code(""" + def f(): + T = str + class C: + T = int + class D[U](T): + x = T + return C + """) + C = ns["f"]() + self.assertIn(int, C.D.__bases__) + self.assertIs(C.D.x, str) + + +class DynamicClassTest(unittest.TestCase): + def _set_type_params(self, ns, params): + ns['__type_params__'] = params + + def test_types_new_class_with_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}, + lambda ns: self._set_type_params(ns, (T,))) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, (T,)) + self.assertEqual(Klass.__parameters__, (T,)) + + def test_types_new_class_no_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, ()) # must be explicitly set + self.assertEqual(Klass.__parameters__, (T,)) + + +class TypeParamsManglingTest(unittest.TestCase): + def test_mangling(self): + class Foo[__T]: + param = __T + def meth[__U](self, arg: __T, arg2: __U): + return (__T, __U) + type Alias[__V] = (__T, __V) + + T = Foo.__type_params__[0] + self.assertEqual(T.__name__, "__T") + U = Foo.meth.__type_params__[0] + self.assertEqual(U.__name__, "__U") + V = Foo.Alias.__type_params__[0] + self.assertEqual(V.__name__, "__V") + + anno = Foo.meth.__annotations__ + self.assertIs(anno["arg"], T) + self.assertIs(anno["arg2"], U) + self.assertEqual(Foo().meth(1, 2), (T, U)) + + self.assertEqual(Foo.Alias.__value__, (T, V)) + + def test_no_leaky_mangling_in_module(self): + ns = run_code(""" + __before = "before" + class X[T]: pass + __after = "after" + """) + self.assertEqual(ns["__before"], "before") + self.assertEqual(ns["__after"], "after") + + def test_no_leaky_mangling_in_function(self): + ns = run_code(""" + def f(): + class X[T]: pass + _X_foo = 2 + __foo = 1 + assert locals()['__foo'] == 1 + return __foo + """) + self.assertEqual(ns["f"](), 1) + + def test_no_leaky_mangling_in_class(self): + ns = run_code(""" + class Outer: + __before = "before" + class Inner[T]: + __x = "inner" + __after = "after" + """) + Outer = ns["Outer"] + self.assertEqual(Outer._Outer__before, "before") + self.assertEqual(Outer.Inner._Inner__x, "inner") + self.assertEqual(Outer._Outer__after, "after") + + def test_no_mangling_in_bases(self): + ns = run_code(""" + class __Base: + def __init_subclass__(self, **kwargs): + self.kwargs = kwargs + + class Derived[T](__Base, __kwarg=1): + pass + """) + Derived = ns["Derived"] + self.assertEqual(Derived.__bases__, (ns["__Base"], Generic)) + self.assertEqual(Derived.kwargs, {"__kwarg": 1}) + + def test_no_mangling_in_nested_scopes(self): + ns = run_code(""" + from test.test_type_params import make_base + + class __X: + pass + + class Y[T: __X]( + make_base(lambda: __X), + # doubly nested scope + make_base(lambda: (lambda: __X)), + # list comprehension + make_base([__X for _ in (1,)]), + # genexp + make_base(__X for _ in (1,)), + ): + pass + """) + Y = ns["Y"] + T, = Y.__type_params__ + self.assertIs(T.__bound__, ns["__X"]) + base0 = Y.__bases__[0] + self.assertIs(base0.__arg__(), ns["__X"]) + base1 = Y.__bases__[1] + self.assertIs(base1.__arg__()(), ns["__X"]) + base2 = Y.__bases__[2] + self.assertEqual(base2.__arg__, [ns["__X"]]) + base3 = Y.__bases__[3] + self.assertEqual(list(base3.__arg__), [ns["__X"]]) + + def test_type_params_are_mangled(self): + ns = run_code(""" + from test.test_type_params import make_base + + class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)): + param = __T + """) + Foo = ns["Foo"] + T, U = Foo.__type_params__ + self.assertEqual(T.__name__, "__T") + self.assertEqual(U.__name__, "__U") + self.assertIs(U.__bound__, T) + self.assertIs(Foo.param, T) + + base1, base2, *_ = Foo.__bases__ + self.assertIs(base1.__arg__, T) + self.assertIs(base2.__arg__(), T) + + +class TypeParamsComplexCallsTest(unittest.TestCase): + def test_defaults(self): + # Generic functions with both defaults and kwdefaults trigger a specific code path + # in the compiler. + def func[T](a: T = "a", *, b: T = "b"): + return (a, b) + + T, = func.__type_params__ + self.assertIs(func.__annotations__["a"], T) + self.assertIs(func.__annotations__["b"], T) + self.assertEqual(func(), ("a", "b")) + self.assertEqual(func(1), (1, "b")) + self.assertEqual(func(b=2), ("a", 2)) + + def test_complex_base(self): + class Base: + def __init_subclass__(cls, **kwargs) -> None: + cls.kwargs = kwargs + + kwargs = {"c": 3} + # Base classes with **kwargs trigger a different code path in the compiler. + class C[T](Base, a=1, b=2, **kwargs): + pass + + T, = C.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C.kwargs, {"a": 1, "b": 2, "c": 3}) + self.assertEqual(C.__bases__, (Base, Generic)) + + bases = (Base,) + class C2[T](*bases, **kwargs): + pass + + T, = C2.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C2.kwargs, {"c": 3}) + self.assertEqual(C2.__bases__, (Base, Generic)) + + def test_starargs_base(self): + class C1[T](*()): pass + + T, = C1.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C1.__bases__, (Generic,)) + + class Base: pass + bases = [Base] + class C2[T](*bases): pass + + T, = C2.__type_params__ + self.assertEqual(T.__name__, "T") + self.assertEqual(C2.__bases__, (Base, Generic)) + + +class TypeParamsTraditionalTypeVarsTest(unittest.TestCase): + def test_traditional_01(self): + code = """ + from typing import Generic + class ClassA[T](Generic[T]): ... + """ + + with self.assertRaisesRegex(TypeError, r"Cannot inherit from Generic\[...\] multiple times."): + run_code(code) + + def test_traditional_02(self): + from typing import TypeVar + S = TypeVar("S") + with self.assertRaises(TypeError): + class ClassA[T](dict[T, S]): ... + + def test_traditional_03(self): + # This does not generate a runtime error, but it should be + # flagged as an error by type checkers. + from typing import TypeVar + S = TypeVar("S") + def func[T](a: T, b: S) -> T | S: + return a + + +class TypeParamsTypeVarTest(unittest.TestCase): + def test_typevar_01(self): + def func1[A: str, B: str | int, C: (int, str)](): + return (A, B, C) + + a, b, c = func1() + + self.assertIsInstance(a, TypeVar) + self.assertEqual(a.__bound__, str) + self.assertTrue(a.__infer_variance__) + self.assertFalse(a.__covariant__) + self.assertFalse(a.__contravariant__) + + self.assertIsInstance(b, TypeVar) + self.assertEqual(b.__bound__, str | int) + self.assertTrue(b.__infer_variance__) + self.assertFalse(b.__covariant__) + self.assertFalse(b.__contravariant__) + + self.assertIsInstance(c, TypeVar) + self.assertEqual(c.__bound__, None) + self.assertEqual(c.__constraints__, (int, str)) + self.assertTrue(c.__infer_variance__) + self.assertFalse(c.__covariant__) + self.assertFalse(c.__contravariant__) + + def test_typevar_generator(self): + def get_generator[A](): + def generator1[C](): + yield C + + def generator2[B](): + yield A + yield B + yield from generator1() + return generator2 + + gen = get_generator() + + a, b, c = [x for x in gen()] + + self.assertIsInstance(a, TypeVar) + self.assertEqual(a.__name__, "A") + self.assertIsInstance(b, TypeVar) + self.assertEqual(b.__name__, "B") + self.assertIsInstance(c, TypeVar) + self.assertEqual(c.__name__, "C") + + def test_typevar_coroutine(self): + def get_coroutine[A](): + async def coroutine[B](): + return (A, B) + return coroutine + + co = get_coroutine() + + a, b = run_no_yield_async_fn(co) + + self.assertIsInstance(a, TypeVar) + self.assertEqual(a.__name__, "A") + self.assertIsInstance(b, TypeVar) + self.assertEqual(b.__name__, "B") + + +class TypeParamsTypeVarTupleTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "cannot use bound with TypeVarTuple" does not match "invalid syntax (<test string>, line 1)" + def test_typevartuple_01(self): + code = """def func1[*A: str](): pass""" + check_syntax_error(self, code, "cannot use bound with TypeVarTuple") + code = """def func1[*A: (int, str)](): pass""" + check_syntax_error(self, code, "cannot use constraints with TypeVarTuple") + code = """class X[*A: str]: pass""" + check_syntax_error(self, code, "cannot use bound with TypeVarTuple") + code = """class X[*A: (int, str)]: pass""" + check_syntax_error(self, code, "cannot use constraints with TypeVarTuple") + code = """type X[*A: str] = int""" + check_syntax_error(self, code, "cannot use bound with TypeVarTuple") + code = """type X[*A: (int, str)] = int""" + check_syntax_error(self, code, "cannot use constraints with TypeVarTuple") + + def test_typevartuple_02(self): + def func1[*A](): + return A + + a = func1() + self.assertIsInstance(a, TypeVarTuple) + + +class TypeParamsTypeVarParamSpecTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "cannot use bound with ParamSpec" does not match "invalid syntax (<test string>, line 1)" + def test_paramspec_01(self): + code = """def func1[**A: str](): pass""" + check_syntax_error(self, code, "cannot use bound with ParamSpec") + code = """def func1[**A: (int, str)](): pass""" + check_syntax_error(self, code, "cannot use constraints with ParamSpec") + code = """class X[**A: str]: pass""" + check_syntax_error(self, code, "cannot use bound with ParamSpec") + code = """class X[**A: (int, str)]: pass""" + check_syntax_error(self, code, "cannot use constraints with ParamSpec") + code = """type X[**A: str] = int""" + check_syntax_error(self, code, "cannot use bound with ParamSpec") + code = """type X[**A: (int, str)] = int""" + check_syntax_error(self, code, "cannot use constraints with ParamSpec") + + def test_paramspec_02(self): + def func1[**A](): + return A + + a = func1() + self.assertIsInstance(a, ParamSpec) + self.assertTrue(a.__infer_variance__) + self.assertFalse(a.__covariant__) + self.assertFalse(a.__contravariant__) + + +class TypeParamsTypeParamsDunder(unittest.TestCase): + def test_typeparams_dunder_class_01(self): + class Outer[A, B]: + class Inner[C, D]: + @staticmethod + def get_typeparams(): + return A, B, C, D + + a, b, c, d = Outer.Inner.get_typeparams() + self.assertEqual(Outer.__type_params__, (a, b)) + self.assertEqual(Outer.Inner.__type_params__, (c, d)) + + self.assertEqual(Outer.__parameters__, (a, b)) + self.assertEqual(Outer.Inner.__parameters__, (c, d)) + + def test_typeparams_dunder_class_02(self): + class ClassA: + pass + + self.assertEqual(ClassA.__type_params__, ()) + + def test_typeparams_dunder_class_03(self): + code = """ + class ClassA[A](): + pass + ClassA.__type_params__ = () + params = ClassA.__type_params__ + """ + + ns = run_code(code) + self.assertEqual(ns["params"], ()) + + def test_typeparams_dunder_function_01(self): + def outer[A, B](): + def inner[C, D](): + return A, B, C, D + + return inner + + inner = outer() + a, b, c, d = inner() + self.assertEqual(outer.__type_params__, (a, b)) + self.assertEqual(inner.__type_params__, (c, d)) + + def test_typeparams_dunder_function_02(self): + def func1(): + pass + + self.assertEqual(func1.__type_params__, ()) + + def test_typeparams_dunder_function_03(self): + code = """ + def func[A](): + pass + func.__type_params__ = () + """ + + ns = run_code(code) + self.assertEqual(ns["func"].__type_params__, ()) + + + +# All these type aliases are used for pickling tests: +T = TypeVar('T') +def func1[X](x: X) -> X: ... +def func2[X, Y](x: X | Y) -> X | Y: ... +def func3[X, *Y, **Z](x: X, y: tuple[*Y], z: Z) -> X: ... +def func4[X: int, Y: (bytes, str)](x: X, y: Y) -> X | Y: ... + +class Class1[X]: ... +class Class2[X, Y]: ... +class Class3[X, *Y, **Z]: ... +class Class4[X: int, Y: (bytes, str)]: ... + + +class TypeParamsPickleTest(unittest.TestCase): + def test_pickling_functions(self): + things_to_test = [ + func1, + func2, + func3, + func4, + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + def test_pickling_classes(self): + things_to_test = [ + Class1, + Class1[int], + Class1[T], + + Class2, + Class2[int, T], + Class2[T, int], + Class2[int, str], + + Class3, + Class3[int, T, str, bytes, [float, object, T]], + + Class4, + Class4[int, bytes], + Class4[T, bytes], + Class4[int, T], + Class4[T, T], + ] + for thing in things_to_test: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + self.assertEqual(pickle.loads(pickled), thing) + + for klass in things_to_test: + real_class = getattr(klass, '__origin__', klass) + thing = klass() + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + pickled = pickle.dumps(thing, protocol=proto) + # These instances are not equal, + # but class check is good enough: + self.assertIsInstance(pickle.loads(pickled), real_class) + + +class TypeParamsWeakRefTest(unittest.TestCase): + def test_weakrefs(self): + T = TypeVar('T') + P = ParamSpec('P') + class OldStyle(Generic[T]): + pass + + class NewStyle[T]: + pass + + cases = [ + T, + TypeVar('T', bound=int), + P, + P.args, + P.kwargs, + TypeVarTuple('Ts'), + OldStyle, + OldStyle[int], + OldStyle(), + NewStyle, + NewStyle[int], + NewStyle(), + Generic[T], + ] + for case in cases: + with self.subTest(case=case): + weakref.ref(case) + + +class TypeParamsRuntimeTest(unittest.TestCase): + def test_name_error(self): + # gh-109118: This crashed the interpreter due to a refcounting bug + code = """ + class name_2[name_5]: + class name_4[name_5](name_0): + pass + """ + with self.assertRaises(NameError): + run_code(code) + + # Crashed with a slightly different stack trace + code = """ + class name_2[name_5]: + class name_4[name_5: name_5](name_0): + pass + """ + with self.assertRaises(NameError): + run_code(code) + + def test_broken_class_namespace(self): + code = """ + class WeirdMapping(dict): + def __missing__(self, key): + if key == "T": + raise RuntimeError + raise KeyError(key) + + class Meta(type): + def __prepare__(name, bases): + return WeirdMapping() + + class MyClass[V](metaclass=Meta): + class Inner[U](T): + pass + """ + with self.assertRaises(RuntimeError): + run_code(code) + + +class DefaultsTest(unittest.TestCase): + def test_defaults_on_func(self): + ns = run_code(""" + def func[T=int, **U=float, *V=None](): + pass + """) + + T, U, V = ns["func"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_class(self): + ns = run_code(""" + class C[T=int, **U=float, *V=None]: + pass + """) + + T, U, V = ns["C"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_type_alias(self): + ns = run_code(""" + type Alias[T = int, **U = float, *V = None] = int + """) + + T, U, V = ns["Alias"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_starred_invalid(self): + check_syntax_error(self, "type Alias[T = *int] = int") + check_syntax_error(self, "type Alias[**P = *int] = int") + + def test_starred_typevartuple(self): + ns = run_code(""" + default = tuple[int, str] + type Alias[*Ts = *default] = Ts + """) + + Ts, = ns["Alias"].__type_params__ + self.assertEqual(Ts.__default__, next(iter(ns["default"]))) + + def test_nondefault_after_default(self): + check_syntax_error(self, "def func[T=int, U](): pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "class C[T=int, U]: pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "type A[T=int, U] = int", "non-default type parameter 'U' follows default type parameter") + + def test_lazy_evaluation(self): + ns = run_code(""" + type Alias[T = Undefined, *U = Undefined, **V = Undefined] = int + """) + + T, U, V = ns["Alias"].__type_params__ + + with self.assertRaises(NameError): + T.__default__ + with self.assertRaises(NameError): + U.__default__ + with self.assertRaises(NameError): + V.__default__ + + ns["Undefined"] = "defined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + # Now it is cached + ns["Undefined"] = "redefined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + def test_symtable_key_regression_default(self): + # Test against the bugs that would happen if we used .default_ + # as the key in the symtable. + ns = run_code(""" + type X[T = [T for T in [T]]] = T + """) + + T, = ns["X"].__type_params__ + self.assertEqual(T.__default__, [T]) + + def test_symtable_key_regression_name(self): + # Test against the bugs that would happen if we used .name + # as the key in the symtable. + ns = run_code(""" + type X1[T = A] = T + type X2[T = B] = T + A = "A" + B = "B" + """) + + self.assertEqual(ns["X1"].__type_params__[0].__default__, "A") + self.assertEqual(ns["X2"].__type_params__[0].__default__, "B") + + +class TestEvaluateFunctions(unittest.TestCase): + def test_general(self): + type Alias = int + Alias2 = TypeAliasType("Alias2", int) + def f[T: int = int, **P = int, *Ts = int](): pass + T, P, Ts = f.__type_params__ + T2 = TypeVar("T2", bound=int, default=int) + P2 = ParamSpec("P2", default=int) + Ts2 = TypeVarTuple("Ts2", default=int) + cases = [ + Alias.evaluate_value, + Alias2.evaluate_value, + T.evaluate_bound, + T.evaluate_default, + P.evaluate_default, + Ts.evaluate_default, + T2.evaluate_bound, + T2.evaluate_default, + P2.evaluate_default, + Ts2.evaluate_default, + ] + for case in cases: + with self.subTest(case=case): + self.assertIs(case(1), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.VALUE), int) + self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) + self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') + + def test_constraints(self): + def f[T: (int, str)](): pass + T, = f.__type_params__ + T2 = TypeVar("T2", int, str) + for case in [T, T2]: + with self.subTest(case=case): + self.assertEqual(case.evaluate_constraints(1), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.VALUE), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.FORWARDREF), (int, str)) + self.assertEqual(annotationlib.call_evaluate_function(case.evaluate_constraints, annotationlib.Format.STRING), '(int, str)') + + def test_const_evaluator(self): + T = TypeVar("T", bound=int) + self.assertEqual(repr(T.evaluate_bound), "<constevaluator <class 'int'>>") + + ConstEvaluator = type(T.evaluate_bound) + + with self.assertRaisesRegex(TypeError, r"cannot create '_typing\._ConstEvaluator' instances"): + ConstEvaluator() # This used to segfault. + with self.assertRaisesRegex(TypeError, r"cannot set 'attribute' attribute of immutable type '_typing\._ConstEvaluator'"): + ConstEvaluator.attribute = 1 diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 14975ec0807..ced9e27fed5 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1,19 +1,35 @@ # Python test set -- part 6, built-in types -from test.support import run_with_locale, cpython_only +from test.support import ( + run_with_locale, cpython_only, no_rerun, + MISSING_C_DOCSTRINGS, EqualToForwardRef, check_disallow_instantiation, +) +from test.support.script_helper import assert_python_ok +from test.support.import_helper import import_fresh_module + import collections.abc -from collections import namedtuple +from collections import namedtuple, UserDict import copy +# XXX: RUSTPYTHON +try: + import _datetime +except ImportError: + _datetime = None import gc import inspect import pickle import locale import sys +import textwrap import types import unittest.mock import weakref import typing +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + +c_types = import_fresh_module('types', fresh=['_types']) +py_types = import_fresh_module('types', blocked=['_types']) T = typing.TypeVar("T") @@ -29,6 +45,29 @@ def clear_typing_caches(): class TypesTests(unittest.TestCase): + @unittest.skipUnless(c_types, "TODO: RUSTPYTHON; requires _types module") + def test_names(self): + c_only_names = {'CapsuleType'} + ignored = {'new_class', 'resolve_bases', 'prepare_class', + 'get_original_bases', 'DynamicClassAttribute', 'coroutine'} + + for name in c_types.__all__: + if name not in c_only_names | ignored: + self.assertIs(getattr(c_types, name), getattr(py_types, name)) + + all_names = ignored | { + 'AsyncGeneratorType', 'BuiltinFunctionType', 'BuiltinMethodType', + 'CapsuleType', 'CellType', 'ClassMethodDescriptorType', 'CodeType', + 'CoroutineType', 'EllipsisType', 'FrameType', 'FunctionType', + 'GeneratorType', 'GenericAlias', 'GetSetDescriptorType', + 'LambdaType', 'MappingProxyType', 'MemberDescriptorType', + 'MethodDescriptorType', 'MethodType', 'MethodWrapperType', + 'ModuleType', 'NoneType', 'NotImplementedType', 'SimpleNamespace', + 'TracebackType', 'UnionType', 'WrapperDescriptorType', + } + self.assertEqual(all_names, set(c_types.__all__)) + self.assertEqual(all_names - c_only_names, set(py_types.__all__)) + def test_truth_values(self): if None: self.fail('None is true instead of false') if 0: self.fail('0 is true instead of false') @@ -226,8 +265,8 @@ def test_type_function(self): def test_int__format__(self): def test(i, format_spec, result): # just make sure we have the unified type for integers - assert type(i) == int - assert type(format_spec) == str + self.assertIs(type(i), int) + self.assertIs(type(format_spec), str) self.assertEqual(i.__format__(format_spec), result) test(123456789, 'd', '123456789') @@ -392,9 +431,7 @@ def test(i, format_spec, result): test(123456, "1=20", '11111111111111123456') test(123456, "*=20", '**************123456') - # TODO: RUSTPYTHON - @unittest.expectedFailure - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_float__format__locale(self): # test locale support for __format__ code 'n' @@ -403,7 +440,7 @@ def test_float__format__locale(self): self.assertEqual(locale.format_string('%g', x, grouping=True), format(x, 'n')) self.assertEqual(locale.format_string('%.10g', x, grouping=True), format(x, '.10n')) - @run_with_locale('LC_NUMERIC', 'en_US.UTF8') + @run_with_locale('LC_NUMERIC', 'en_US.UTF8', '') def test_int__format__locale(self): # test locale support for __format__ code 'n' for integers @@ -421,9 +458,6 @@ def test_int__format__locale(self): self.assertEqual(len(format(0, rfmt)), len(format(x, rfmt))) self.assertEqual(len(format(0, lfmt)), len(format(x, lfmt))) self.assertEqual(len(format(0, cfmt)), len(format(x, cfmt))) - - if sys.platform != "darwin": - test_int__format__locale = unittest.expectedFailure(test_int__format__locale) def test_float__format__(self): def test(f, format_spec, result): @@ -490,8 +524,8 @@ def test(f, format_spec, result): # and a number after the decimal. This is tricky, because # a totally empty format specifier means something else. # So, just use a sign flag - test(1e200, '+g', '+1e+200') - test(1e200, '+', '+1e+200') + test(1.25e200, '+g', '+1.25e+200') + test(1.25e200, '+', '+1.25e+200') test(1.1e200, '+g', '+1.1e+200') test(1.1e200, '+', '+1.1e+200') @@ -592,8 +626,6 @@ def test_format_spec_errors(self): for code in 'xXobns': self.assertRaises(ValueError, format, 0, ',' + code) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_internal_sizes(self): self.assertGreater(object.__basicsize__, 0) self.assertGreater(tuple.__itemsize__, 0) @@ -604,8 +636,9 @@ def test_slot_wrapper_types(self): self.assertIsInstance(object.__lt__, types.WrapperDescriptorType) self.assertIsInstance(int.__lt__, types.WrapperDescriptorType) - # TODO: RUSTPYTHON No signature found in builtin method __get__ of 'method_descriptor' objects. - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; No signature found in builtin method __get__ of 'method_descriptor' objects. + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") def test_dunder_get_signature(self): sig = inspect.signature(object.__init__.__get__) self.assertEqual(list(sig.parameters), ["instance", "owner"]) @@ -628,6 +661,26 @@ def test_method_descriptor_types(self): self.assertIsInstance(int.from_bytes, types.BuiltinMethodType) self.assertIsInstance(int.__new__, types.BuiltinMethodType) + @unittest.expectedFailure # TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_queue' + def test_method_descriptor_crash(self): + # gh-132747: The default __get__() implementation in C was unable + # to handle a second argument of None when called from Python + import _io + import io + import _queue + + to_check = [ + # (method, instance) + (_io._TextIOBase.read, io.StringIO()), + (_queue.SimpleQueue.put, _queue.SimpleQueue()), + (str.capitalize, "nobody expects the spanish inquisition") + ] + + for method, instance in to_check: + with self.subTest(method=method, instance=instance): + bound = method.__get__(instance) + self.assertIsInstance(bound, types.BuiltinMethodType) + def test_ellipsis_type(self): self.assertIsInstance(Ellipsis, types.EllipsisType) @@ -645,6 +698,29 @@ def test_traceback_and_frame_types(self): self.assertIsInstance(exc.__traceback__, types.TracebackType) self.assertIsInstance(exc.__traceback__.tb_frame, types.FrameType) + # XXX: RUSTPYTHON + @unittest.skipUnless(_datetime, "requires _datetime module") + def test_capsule_type(self): + self.assertIsInstance(_datetime.datetime_CAPI, types.CapsuleType) + + def test_call_unbound_crash(self): + # GH-131998: The specialized instruction would get tricked into dereferencing + # a bound "self" that didn't exist if subsequently called unbound. + code = """if True: + + def call(part): + [] + ([] + []) + part.pop() + + for _ in range(3): + call(['a']) + try: + call(list) + except TypeError: + pass + """ + assert_python_ok("-c", code) + class UnionTests(unittest.TestCase): @@ -707,15 +783,54 @@ def test_or_types_operator(self): y = int | bool with self.assertRaises(TypeError): x < y - # Check that we don't crash if typing.Union does not have a tuple in __args__ - y = typing.Union[str, int] - y.__args__ = [str, int] - self.assertEqual(x, y) def test_hash(self): self.assertEqual(hash(int | str), hash(str | int)) self.assertEqual(hash(int | str), hash(typing.Union[int, str])) + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual((A | B).__args__, (A, B)) + union1 = A | B + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union1) + + union2 = int | B + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union2) + + union3 = A | int + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union3) + + def test_unhashable_becomes_hashable(self): + is_hashable = False + class UnhashableMeta(type): + def __hash__(self): + if is_hashable: + return 1 + else: + raise TypeError("not hashable") + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + union = A | B + self.assertEqual(union.__args__, (A, B)) + + with self.assertRaisesRegex(TypeError, "not hashable"): + hash(union) + + is_hashable = True + + with self.assertRaisesRegex(TypeError, "union contains 2 unhashable elements"): + hash(union) + def test_instancecheck_and_subclasscheck(self): for x in (int | str, typing.Union[int, str]): with self.subTest(x=x): @@ -723,15 +838,15 @@ def test_instancecheck_and_subclasscheck(self): self.assertIsInstance(True, x) self.assertIsInstance('a', x) self.assertNotIsInstance(None, x) - self.assertTrue(issubclass(int, x)) - self.assertTrue(issubclass(bool, x)) - self.assertTrue(issubclass(str, x)) - self.assertFalse(issubclass(type(None), x)) + self.assertIsSubclass(int, x) + self.assertIsSubclass(bool, x) + self.assertIsSubclass(str, x) + self.assertNotIsSubclass(type(None), x) for x in (int | None, typing.Union[int, None]): with self.subTest(x=x): self.assertIsInstance(None, x) - self.assertTrue(issubclass(type(None), x)) + self.assertIsSubclass(type(None), x) for x in ( int | collections.abc.Mapping, @@ -740,8 +855,8 @@ def test_instancecheck_and_subclasscheck(self): with self.subTest(x=x): self.assertIsInstance({}, x) self.assertNotIsInstance((), x) - self.assertTrue(issubclass(dict, x)) - self.assertFalse(issubclass(list, x)) + self.assertIsSubclass(dict, x) + self.assertNotIsSubclass(list, x) def test_instancecheck_and_subclasscheck_order(self): T = typing.TypeVar('T') @@ -753,7 +868,7 @@ def test_instancecheck_and_subclasscheck_order(self): for x in will_resolve: with self.subTest(x=x): self.assertIsInstance(1, x) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) wont_resolve = ( T | int, @@ -786,13 +901,13 @@ class BadMeta(type): def __subclasscheck__(cls, sub): 1/0 x = int | BadMeta('A', (), {}) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) self.assertRaises(ZeroDivisionError, issubclass, list, x) def test_or_type_operator_with_TypeVar(self): TV = typing.TypeVar('T') - assert TV | str == typing.Union[TV, str] - assert str | TV == typing.Union[str, TV] + self.assertEqual(TV | str, typing.Union[TV, str]) + self.assertEqual(str | TV, typing.Union[str, TV]) self.assertIs((int | TV)[int], int) self.assertIs((TV | int)[int], int) @@ -837,8 +952,6 @@ def test_union_parameter_chaining(self): self.assertEqual((list[T] | list[S])[int, T], list[int] | list[T]) self.assertEqual((list[T] | list[S])[int, int], list[int]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_union_parameter_substitution(self): def eq(actual, expected, typed=True): self.assertEqual(actual, expected) @@ -870,8 +983,6 @@ def eq(actual, expected, typed=True): eq(x[NT], int | NT | bytes) eq(x[S], int | S | bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_union_pickle(self): orig = list[T] | int for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -881,8 +992,6 @@ def test_union_pickle(self): self.assertEqual(loaded.__args__, orig.__args__) self.assertEqual(loaded.__parameters__, orig.__parameters__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_union_copy(self): orig = list[T] | int for copied in (copy.copy(orig), copy.deepcopy(orig)): @@ -902,54 +1011,83 @@ def test_or_type_operator_with_forward(self): ForwardBefore = 'Forward' | T def forward_after(x: ForwardAfter[int]) -> None: ... def forward_before(x: ForwardBefore[int]) -> None: ... - assert typing.get_args(typing.get_type_hints(forward_after)['x']) == (int, Forward) - assert typing.get_args(typing.get_type_hints(forward_before)['x']) == (int, Forward) + self.assertEqual(typing.get_args(typing.get_type_hints(forward_after)['x']), + (int, Forward)) + self.assertEqual(typing.get_args(typing.get_type_hints(forward_before)['x']), + (Forward, int)) def test_or_type_operator_with_Protocol(self): class Proto(typing.Protocol): def meth(self) -> int: ... - assert Proto | str == typing.Union[Proto, str] + self.assertEqual(Proto | str, typing.Union[Proto, str]) def test_or_type_operator_with_Alias(self): - assert list | str == typing.Union[list, str] - assert typing.List | str == typing.Union[typing.List, str] + self.assertEqual(list | str, typing.Union[list, str]) + self.assertEqual(typing.List | str, typing.Union[typing.List, str]) def test_or_type_operator_with_NamedTuple(self): - NT=namedtuple('A', ['B', 'C', 'D']) - assert NT | str == typing.Union[NT,str] + NT = namedtuple('A', ['B', 'C', 'D']) + self.assertEqual(NT | str, typing.Union[NT, str]) def test_or_type_operator_with_TypedDict(self): class Point2D(typing.TypedDict): x: int y: int label: str - assert Point2D | str == typing.Union[Point2D, str] + self.assertEqual(Point2D | str, typing.Union[Point2D, str]) def test_or_type_operator_with_NewType(self): UserId = typing.NewType('UserId', int) - assert UserId | str == typing.Union[UserId, str] + self.assertEqual(UserId | str, typing.Union[UserId, str]) def test_or_type_operator_with_IO(self): - assert typing.IO | str == typing.Union[typing.IO, str] + self.assertEqual(typing.IO | str, typing.Union[typing.IO, str]) def test_or_type_operator_with_SpecialForm(self): - assert typing.Any | str == typing.Union[typing.Any, str] - assert typing.NoReturn | str == typing.Union[typing.NoReturn, str] - assert typing.Optional[int] | str == typing.Union[typing.Optional[int], str] - assert typing.Optional[int] | str == typing.Union[int, str, None] - assert typing.Union[int, bool] | str == typing.Union[int, bool, str] + self.assertEqual(typing.Any | str, typing.Union[typing.Any, str]) + self.assertEqual(typing.NoReturn | str, typing.Union[typing.NoReturn, str]) + self.assertEqual(typing.Optional[int] | str, typing.Union[typing.Optional[int], str]) + self.assertEqual(typing.Optional[int] | str, typing.Union[int, str, None]) + self.assertEqual(typing.Union[int, bool] | str, typing.Union[int, bool, str]) + + def test_or_type_operator_with_Literal(self): + Literal = typing.Literal + self.assertEqual((Literal[1] | Literal[2]).__args__, + (Literal[1], Literal[2])) + + self.assertEqual((Literal[0] | Literal[False]).__args__, + (Literal[0], Literal[False])) + self.assertEqual((Literal[1] | Literal[True]).__args__, + (Literal[1], Literal[True])) + + self.assertEqual(Literal[1] | Literal[1], Literal[1]) + self.assertEqual(Literal['a'] | Literal['a'], Literal['a']) + + import enum + class Ints(enum.IntEnum): + A = 0 + B = 1 + + self.assertEqual(Literal[Ints.A] | Literal[Ints.A], Literal[Ints.A]) + self.assertEqual(Literal[Ints.B] | Literal[Ints.B], Literal[Ints.B]) + + self.assertEqual((Literal[Ints.B] | Literal[Ints.A]).__args__, + (Literal[Ints.B], Literal[Ints.A])) + + self.assertEqual((Literal[0] | Literal[Ints.A]).__args__, + (Literal[0], Literal[Ints.A])) + self.assertEqual((Literal[1] | Literal[Ints.B]).__args__, + (Literal[1], Literal[Ints.B])) def test_or_type_repr(self): - assert repr(int | str) == "int | str" - assert repr((int | str) | list) == "int | str | list" - assert repr(int | (str | list)) == "int | str | list" - assert repr(int | None) == "int | None" - assert repr(int | type(None)) == "int | None" - assert repr(int | typing.GenericAlias(list, int)) == "int | list[int]" - - # TODO: RUSTPYTHON - @unittest.expectedFailure + self.assertEqual(repr(int | str), "int | str") + self.assertEqual(repr((int | str) | list), "int | str | list") + self.assertEqual(repr(int | (str | list)), "int | str | list") + self.assertEqual(repr(int | None), "int | None") + self.assertEqual(repr(int | type(None)), "int | None") + self.assertEqual(repr(int | typing.GenericAlias(list, int)), "int | list[int]") + def test_or_type_operator_with_genericalias(self): a = list[int] b = list[str] @@ -970,9 +1108,14 @@ def __eq__(self, other): return 1 / 0 bt = BadType('bt', (), {}) + bt2 = BadType('bt2', (), {}) # Comparison should fail and errors should propagate out for bad types. + union1 = int | bt + union2 = int | bt2 + with self.assertRaises(ZeroDivisionError): + union1 == union2 with self.assertRaises(ZeroDivisionError): - list[int] | list[bt] + bt | bt2 union_ga = (list[str] | int, collections.abc.Callable[..., str] | int, d | int) @@ -1015,6 +1158,19 @@ def test_or_type_operator_reference_cycle(self): self.assertLessEqual(sys.gettotalrefcount() - before, leeway, msg='Check for union reference leak.') + def test_instantiation(self): + check_disallow_instantiation(self, types.UnionType) + self.assertIs(int, types.UnionType[int]) + self.assertIs(int, types.UnionType[int, int]) + self.assertEqual(int | str, types.UnionType[int, str]) + + for obj in ( + int | typing.ForwardRef("str"), + typing.Union[int, "str"], + ): + self.assertIsInstance(obj, types.UnionType) + self.assertEqual(obj.__args__, (int, EqualToForwardRef("str"))) + class MappingProxyTests(unittest.TestCase): mappingproxy = types.MappingProxyType @@ -1076,8 +1232,6 @@ def __missing__(self, key): self.assertTrue('x' in view) self.assertFalse('y' in view) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_customdict(self): class customdict(dict): def __contains__(self, key): @@ -1206,8 +1360,7 @@ def test_copy(self): self.assertEqual(view['key1'], 70) self.assertEqual(copy['key1'], 27) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: unsupported operand type(s) for |: 'dict' and 'mappingproxy' def test_union(self): mapping = {'a': 0, 'b': 1, 'c': 2} view = self.mappingproxy(mapping) @@ -1224,6 +1377,16 @@ def test_union(self): self.assertDictEqual(mapping, {'a': 0, 'b': 1, 'c': 2}) self.assertDictEqual(other, {'c': 3, 'p': 0}) + def test_hash(self): + class HashableDict(dict): + def __hash__(self): + return 3844817361 + view = self.mappingproxy({'a': 1, 'b': 2}) + self.assertRaises(TypeError, hash, view) + mapping = HashableDict({'a': 1, 'b': 2}) + view = self.mappingproxy(mapping) + self.assertEqual(hash(view), hash(mapping)) + class ClassCreationTests(unittest.TestCase): @@ -1247,7 +1410,7 @@ def test_new_class_basics(self): def test_new_class_subclass(self): C = types.new_class("C", (int,)) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) def test_new_class_meta(self): Meta = self.Meta @@ -1292,7 +1455,7 @@ def func(ns): bases=(int,), kwds=dict(metaclass=Meta, z=2), exec_body=func) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) self.assertIsInstance(C, Meta) self.assertEqual(C.x, 0) self.assertEqual(C.y, 1) @@ -1371,6 +1534,80 @@ class C: pass D = types.new_class('D', (A(), C, B()), {}) self.assertEqual(D.__bases__, (A1, A2, A3, C, B1, B2)) + def test_get_original_bases(self): + T = typing.TypeVar('T') + class A: pass + class B(typing.Generic[T]): pass + class C(B[int]): pass + class D(B[str], float): pass + + self.assertEqual(types.get_original_bases(A), (object,)) + self.assertEqual(types.get_original_bases(B), (typing.Generic[T],)) + self.assertEqual(types.get_original_bases(C), (B[int],)) + self.assertEqual(types.get_original_bases(int), (object,)) + self.assertEqual(types.get_original_bases(D), (B[str], float)) + + class E(list[T]): pass + class F(list[int]): pass + + self.assertEqual(types.get_original_bases(E), (list[T],)) + self.assertEqual(types.get_original_bases(F), (list[int],)) + + class FirstBase(typing.Generic[T]): pass + class SecondBase(typing.Generic[T]): pass + class First(FirstBase[int]): pass + class Second(SecondBase[int]): pass + class G(First, Second): pass + self.assertEqual(types.get_original_bases(G), (First, Second)) + + class First_(typing.Generic[T]): pass + class Second_(typing.Generic[T]): pass + class H(First_, Second_): pass + self.assertEqual(types.get_original_bases(H), (First_, Second_)) + + class ClassBasedNamedTuple(typing.NamedTuple): + x: int + + class GenericNamedTuple(typing.NamedTuple, typing.Generic[T]): + x: T + + CallBasedNamedTuple = typing.NamedTuple("CallBasedNamedTuple", [("x", int)]) + + self.assertIs( + types.get_original_bases(ClassBasedNamedTuple)[0], typing.NamedTuple + ) + self.assertEqual( + types.get_original_bases(GenericNamedTuple), + (typing.NamedTuple, typing.Generic[T]) + ) + self.assertIs( + types.get_original_bases(CallBasedNamedTuple)[0], typing.NamedTuple + ) + + class ClassBasedTypedDict(typing.TypedDict): + x: int + + class GenericTypedDict(typing.TypedDict, typing.Generic[T]): + x: T + + CallBasedTypedDict = typing.TypedDict("CallBasedTypedDict", {"x": int}) + + self.assertIs( + types.get_original_bases(ClassBasedTypedDict)[0], + typing.TypedDict + ) + self.assertEqual( + types.get_original_bases(GenericTypedDict), + (typing.TypedDict, typing.Generic[T]) + ) + self.assertIs( + types.get_original_bases(CallBasedTypedDict)[0], + typing.TypedDict + ) + + with self.assertRaisesRegex(TypeError, "Expected an instance of type"): + types.get_original_bases(object()) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -1631,25 +1868,81 @@ class Model(metaclass=ModelBase): with self.assertRaises(RuntimeWarning): type("SouthPonies", (Model,), {}) + def test_subclass_inherited_slot_update(self): + # gh-132284: Make sure slot update still works after fix. + # Note that after assignment to D.__getitem__ the actual C slot will + # never go back to dict_subscript as it was on class type creation but + # rather be set to slot_mp_subscript, unfortunately there is no way to + # check that here. + + class D(dict): + pass + + d = D({None: None}) + self.assertIs(d[None], None) + D.__getitem__ = lambda self, item: 42 + self.assertEqual(d[None], 42) + D.__getitem__ = dict.__getitem__ + self.assertIs(d[None], None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <class 'tuple'> != <class 'test.test_types.ClassCreationTests.test_tu[41 chars]ass'> + def test_tuple_subclass_as_bases(self): + # gh-132176: it used to crash on using + # tuple subclass for as base classes. + class TupleSubclass(tuple): pass + + typ = type("typ", TupleSubclass((int, object)), {}) + self.assertEqual(typ.__bases__, (int, object)) + self.assertEqual(type(typ.__bases__), TupleSubclass) + class SimpleNamespaceTests(unittest.TestCase): def test_constructor(self): - ns1 = types.SimpleNamespace() - ns2 = types.SimpleNamespace(x=1, y=2) - ns3 = types.SimpleNamespace(**dict(x=1, y=2)) + def check(ns, expected): + self.assertEqual(len(ns.__dict__), len(expected)) + self.assertEqual(vars(ns), expected) + # check order + self.assertEqual(list(vars(ns).items()), list(expected.items())) + for name in expected: + self.assertEqual(getattr(ns, name), expected[name]) + + check(types.SimpleNamespace(), {}) + check(types.SimpleNamespace(x=1, y=2), {'x': 1, 'y': 2}) + check(types.SimpleNamespace(**dict(x=1, y=2)), {'x': 1, 'y': 2}) + check(types.SimpleNamespace({'x': 1, 'y': 2}, x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace([['x', 1], ['y', 2]], x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace(UserDict({'x': 1, 'y': 2}), x=4, z=3), + {'x': 4, 'y': 2, 'z': 3}) + check(types.SimpleNamespace({'x': 1, 'y': 2}), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([['x', 1], ['y', 2]]), {'x': 1, 'y': 2}) + check(types.SimpleNamespace([], x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace({}, x=4, z=3), {'x': 4, 'z': 3}) + check(types.SimpleNamespace([]), {}) + check(types.SimpleNamespace({}), {}) with self.assertRaises(TypeError): - types.SimpleNamespace(1, 2, 3) + types.SimpleNamespace([], []) # too many positional arguments with self.assertRaises(TypeError): - types.SimpleNamespace(**{1: 2}) - - self.assertEqual(len(ns1.__dict__), 0) - self.assertEqual(vars(ns1), {}) - self.assertEqual(len(ns2.__dict__), 2) - self.assertEqual(vars(ns2), {'y': 2, 'x': 1}) - self.assertEqual(len(ns3.__dict__), 2) - self.assertEqual(vars(ns3), {'y': 2, 'x': 1}) + types.SimpleNamespace(1) # not a mapping or iterable + with self.assertRaises(TypeError): + types.SimpleNamespace([1]) # non-iterable + with self.assertRaises(ValueError): + types.SimpleNamespace([['x']]) # not a pair + with self.assertRaises(ValueError): + types.SimpleNamespace([['x', 'y', 'z']]) + with self.assertRaises(TypeError): + types.SimpleNamespace(**{1: 2}) # non-string key + with self.assertRaises(TypeError): + types.SimpleNamespace({1: 2}) + with self.assertRaises(TypeError): + types.SimpleNamespace([[1, 2]]) + with self.assertRaises(TypeError): + types.SimpleNamespace(UserDict({1: 2})) + with self.assertRaises(TypeError): + types.SimpleNamespace([[[], 2]]) # non-hashable key def test_unbound(self): ns1 = vars(types.SimpleNamespace()) @@ -1806,6 +2099,33 @@ def test_pickle(self): self.assertEqual(ns, ns_roundtrip, pname) + def test_replace(self): + ns = types.SimpleNamespace(x=11, y=22) + + ns2 = copy.replace(ns) + self.assertEqual(ns2, ns) + self.assertIsNot(ns2, ns) + self.assertIs(type(ns2), types.SimpleNamespace) + self.assertEqual(vars(ns2), {'x': 11, 'y': 22}) + ns2.x = 3 + self.assertEqual(ns.x, 11) + ns.x = 4 + self.assertEqual(ns2.x, 3) + + self.assertEqual(vars(copy.replace(ns, x=1)), {'x': 1, 'y': 22}) + self.assertEqual(vars(copy.replace(ns, y=2)), {'x': 4, 'y': 2}) + self.assertEqual(vars(copy.replace(ns, x=1, y=2)), {'x': 1, 'y': 2}) + + def test_replace_subclass(self): + class Spam(types.SimpleNamespace): + pass + + spam = Spam(ham=8, eggs=9) + spam2 = copy.replace(spam, ham=5) + + self.assertIs(type(spam2), Spam) + self.assertEqual(vars(spam2), {'ham': 5, 'eggs': 9}) + def test_fake_namespace_compare(self): # Issue #24257: Incorrect use of PyObject_IsInstance() caused # SystemError. @@ -1850,8 +2170,6 @@ def foo(): foo = types.coroutine(foo) self.assertIs(aw, foo()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_async_def(self): # Test that types.coroutine passes 'async def' coroutines # without modification @@ -1935,8 +2253,8 @@ def foo(): return gen self.assertIs(wrapper.__name__, gen.__name__) # Test AttributeErrors - for name in {'gi_running', 'gi_frame', 'gi_code', 'gi_yieldfrom', - 'cr_running', 'cr_frame', 'cr_code', 'cr_await'}: + for name in {'gi_running', 'gi_frame', 'gi_code', 'gi_yieldfrom', 'gi_suspended', + 'cr_running', 'cr_frame', 'cr_code', 'cr_await', 'cr_suspended'}: with self.assertRaises(AttributeError): getattr(wrapper, name) @@ -1945,14 +2263,17 @@ def foo(): return gen gen.gi_frame = object() gen.gi_code = object() gen.gi_yieldfrom = object() + gen.gi_suspended = object() self.assertIs(wrapper.gi_running, gen.gi_running) self.assertIs(wrapper.gi_frame, gen.gi_frame) self.assertIs(wrapper.gi_code, gen.gi_code) self.assertIs(wrapper.gi_yieldfrom, gen.gi_yieldfrom) + self.assertIs(wrapper.gi_suspended, gen.gi_suspended) self.assertIs(wrapper.cr_running, gen.gi_running) self.assertIs(wrapper.cr_frame, gen.gi_frame) self.assertIs(wrapper.cr_code, gen.gi_code) self.assertIs(wrapper.cr_await, gen.gi_yieldfrom) + self.assertIs(wrapper.cr_suspended, gen.gi_suspended) wrapper.close() gen.close.assert_called_once_with() @@ -2059,8 +2380,6 @@ async def corofunc(): else: self.fail('StopIteration was expected') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gen(self): def gen_func(): yield 1 @@ -2073,7 +2392,7 @@ def foo(): return gen self.assertIs(wrapper.__await__(), gen) for name in ('__name__', '__qualname__', 'gi_code', - 'gi_running', 'gi_frame'): + 'gi_running', 'gi_frame', 'gi_suspended'): self.assertIs(getattr(foo(), name), getattr(gen, name)) self.assertIs(foo().cr_code, gen.gi_code) @@ -2087,7 +2406,7 @@ def foo(): return gen wrapper = foo() wrapper.send(None) with self.assertRaisesRegex(Exception, 'ham'): - wrapper.throw(Exception, Exception('ham')) + wrapper.throw(Exception('ham')) # decorate foo second time foo = types.coroutine(foo) @@ -2110,8 +2429,6 @@ def foo(): foo = types.coroutine(foo) self.assertIs(foo(), gencoro) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_genfunc(self): def gen(): yield self.assertIs(types.coroutine(gen), gen) @@ -2138,8 +2455,128 @@ def coro(): self.assertEqual(repr(wrapper), str(wrapper)) self.assertTrue(set(dir(wrapper)).issuperset({ '__await__', '__iter__', '__next__', 'cr_code', 'cr_running', - 'cr_frame', 'gi_code', 'gi_frame', 'gi_running', 'send', - 'close', 'throw'})) + 'cr_frame', 'cr_suspended', 'gi_code', 'gi_frame', 'gi_running', + 'gi_suspended', 'send', 'close', 'throw'})) + + +class FunctionTests(unittest.TestCase): + def test_function_type_defaults(self): + def ex(a, /, b, *, c): + return a + b + c + + func = types.FunctionType( + ex.__code__, {}, "func", (1, 2), None, {'c': 3}, + ) + + self.assertEqual(func(), 6) + self.assertEqual(func.__defaults__, (1, 2)) + self.assertEqual(func.__kwdefaults__, {'c': 3}) + + func = types.FunctionType( + ex.__code__, {}, "func", None, None, None, + ) + self.assertEqual(func.__defaults__, None) + self.assertEqual(func.__kwdefaults__, None) + + def test_function_type_wrong_defaults(self): + def ex(a, /, b, *, c): + return a + b + c + + with self.assertRaisesRegex(TypeError, 'arg 4'): + types.FunctionType( + ex.__code__, {}, "func", 1, None, {'c': 3}, + ) + with self.assertRaisesRegex(TypeError, 'arg 6'): + types.FunctionType( + ex.__code__, {}, "func", None, None, 3, + ) + + +@unittest.skip("TODO: RUSTPYTHON; no subinterpreters yet") +class SubinterpreterTests(unittest.TestCase): + + NUMERIC_METHODS = { + '__abs__', + '__add__', + '__bool__', + '__divmod__', + '__float__', + '__floordiv__', + '__index__', + '__int__', + '__lshift__', + '__mod__', + '__mul__', + '__neg__', + '__pos__', + '__pow__', + '__radd__', + '__rdivmod__', + '__rfloordiv__', + '__rlshift__', + '__rmod__', + '__rmul__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__sub__', + '__truediv__', + } + + @classmethod + def setUpClass(cls): + global interpreters + try: + from concurrent import interpreters + except ModuleNotFoundError: + raise unittest.SkipTest('subinterpreters required') + from test.support import channels # noqa: F401 + cls.create_channel = staticmethod(channels.create) + + @cpython_only + @no_rerun('channels (and queues) might have a refleak; see gh-122199') + def test_static_types_inherited_slots(self): + rch, sch = self.create_channel() + + script = textwrap.dedent(""" + import test.support + results = [] + for cls in test.support.iter_builtin_types(): + for attr, _ in test.support.iter_slot_wrappers(cls): + wrapper = getattr(cls, attr) + res = (cls, attr, wrapper) + results.append(res) + results = tuple((repr(c), a, repr(w)) for c, a, w in results) + sch.send_nowait(results) + """) + def collate_results(raw): + results = {} + for cls, attr, wrapper in raw: + key = cls, attr + assert key not in results, (results, key, wrapper) + results[key] = wrapper + return results + + exec(script) + raw = rch.recv_nowait() + main_results = collate_results(raw) + + interp = interpreters.create() + interp.exec('from concurrent import interpreters') + interp.prepare_main(sch=sch) + interp.exec(script) + raw = rch.recv_nowait() + interp_results = collate_results(raw) + + for key, expected in main_results.items(): + cls, attr = key + with self.subTest(cls=cls, slotattr=attr): + actual = interp_results.pop(key) + self.assertEqual(actual, expected) + self.maxDiff = None + self.assertEqual(interp_results, {}) if __name__ == '__main__': diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index abcc03ce2d1..448d16f1f4a 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1,64 +1,90 @@ +import annotationlib import contextlib import collections +import collections.abc +from collections import defaultdict +from functools import lru_cache, wraps, reduce +import gc +import inspect +import io +import itertools +import operator +import os import pickle import re import sys -from unittest import TestCase, main, skipUnless, skip -# TODO: RUSTPYTHON -import unittest +from unittest import TestCase, main, skip +from unittest.mock import patch from copy import copy, deepcopy -from typing import Any, NoReturn -from typing import TypeVar, AnyStr +from typing import Any, NoReturn, Never, assert_never +from typing import overload, get_overloads, clear_overloads +from typing import TypeVar, TypeVarTuple, Unpack, AnyStr from typing import T, KT, VT # Not in __all__. from typing import Union, Optional, Literal from typing import Tuple, List, Dict, MutableMapping from typing import Callable from typing import Generic, ClassVar, Final, final, Protocol -from typing import cast, runtime_checkable +from typing import assert_type, cast, runtime_checkable from typing import get_type_hints -from typing import get_origin, get_args -from typing import is_typeddict +from typing import get_origin, get_args, get_protocol_members +from typing import override +from typing import is_typeddict, is_protocol +from typing import reveal_type +from typing import dataclass_transform from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NewType -from typing import NamedTuple, TypedDict +from typing import NamedTuple, NotRequired, Required, ReadOnly, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef +from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs -from typing import TypeGuard +from typing import TypeGuard, TypeIs, NoDefault import abc +import textwrap import typing import weakref +import warnings import types -from test import mod_generics_cache -from test import _typed_dict_helper +from test.support import ( + captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper, run_code, + EqualToForwardRef, +) +from test.typinganndata import ( + ann_module695, mod_generics_cache, _typed_dict_helper, + ann_module, ann_module2, ann_module3, ann_module5, ann_module6, ann_module8 +) +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests -class BaseTestCase(TestCase): - def assertIsSubclass(self, cls, class_or_tuple, msg=None): - if not issubclass(cls, class_or_tuple): - message = '%r is not a subclass of %r' % (cls, class_or_tuple) - if msg is not None: - message += ' : %s' % msg - raise self.failureException(message) +CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' +NOT_A_BASE_TYPE = "type 'typing.%s' is not an acceptable base type" +CANNOT_SUBCLASS_INSTANCE = 'Cannot subclass an instance of %s' + - def assertNotIsSubclass(self, cls, class_or_tuple, msg=None): - if issubclass(cls, class_or_tuple): - message = '%r is a subclass of %r' % (cls, class_or_tuple) - if msg is not None: - message += ' : %s' % msg - raise self.failureException(message) +class BaseTestCase(TestCase): def clear_caches(self): for f in typing._cleanups: f() +def all_pickle_protocols(test_func): + """Runs `test_func` with various values for `proto` argument.""" + + @wraps(test_func) + def wrapper(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_proto=proto): + test_func(self, proto=proto) + + return wrapper + + class Employee: pass @@ -81,28 +107,59 @@ def test_any_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, Any) - def test_any_subclass_type_error(self): - with self.assertRaises(TypeError): - issubclass(Employee, Any) - with self.assertRaises(TypeError): - issubclass(Any, Employee) - def test_repr(self): self.assertEqual(repr(Any), 'typing.Any') + class Sub(Any): pass + self.assertEqual( + repr(Sub), + f"<class '{__name__}.AnyTests.test_repr.<locals>.Sub'>", + ) + def test_errors(self): with self.assertRaises(TypeError): - issubclass(42, Any) + isinstance(42, Any) with self.assertRaises(TypeError): Any[int] # Any is not a generic type. - def test_cannot_subclass(self): - with self.assertRaises(TypeError): - class A(Any): - pass - with self.assertRaises(TypeError): - class A(type(Any)): - pass + def test_can_subclass(self): + class Mock(Any): pass + self.assertIsSubclass(Mock, Any) + self.assertIsInstance(Mock(), Mock) + + class Something: pass + self.assertNotIsSubclass(Something, Any) + self.assertNotIsInstance(Something(), Mock) + + class MockSomething(Something, Mock): pass + self.assertIsSubclass(MockSomething, Any) + self.assertIsSubclass(MockSomething, MockSomething) + self.assertIsSubclass(MockSomething, Something) + self.assertIsSubclass(MockSomething, Mock) + ms = MockSomething() + self.assertIsInstance(ms, MockSomething) + self.assertIsInstance(ms, Something) + self.assertIsInstance(ms, Mock) + + def test_subclassing_with_custom_constructor(self): + class Sub(Any): + def __init__(self, *args, **kwargs): pass + # The instantiation must not fail. + Sub(0, s="") + + def test_multiple_inheritance_with_custom_constructors(self): + class Foo: + def __init__(self, x): + self.x = x + + class Bar(Any, Foo): + def __init__(self, x, y): + self.y = y + super().__init__(x) + + b = Bar(1, 2) + self.assertEqual(b.x, 1) + self.assertEqual(b.y, 2) def test_cannot_instantiate(self): with self.assertRaises(TypeError): @@ -117,48 +174,271 @@ def test_any_works_with_alias(self): typing.IO[Any] -class NoReturnTests(BaseTestCase): +class BottomTypeTestsMixin: + bottom_type: ClassVar[Any] + + def test_equality(self): + self.assertEqual(self.bottom_type, self.bottom_type) + self.assertIs(self.bottom_type, self.bottom_type) + self.assertNotEqual(self.bottom_type, None) + + def test_get_origin(self): + self.assertIs(get_origin(self.bottom_type), None) + + def test_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, self.bottom_type) + + def test_subclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(Employee, self.bottom_type) + with self.assertRaises(TypeError): + issubclass(NoReturn, self.bottom_type) - def test_noreturn_instance_type_error(self): + def test_not_generic(self): with self.assertRaises(TypeError): - isinstance(42, NoReturn) + self.bottom_type[int] + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, + 'Cannot subclass ' + re.escape(str(self.bottom_type))): + class A(self.bottom_type): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class B(type(self.bottom_type)): + pass - def test_noreturn_subclass_type_error(self): + def test_cannot_instantiate(self): with self.assertRaises(TypeError): - issubclass(Employee, NoReturn) + self.bottom_type() with self.assertRaises(TypeError): - issubclass(NoReturn, Employee) + type(self.bottom_type)() + + +class NoReturnTests(BottomTypeTestsMixin, BaseTestCase): + bottom_type = NoReturn def test_repr(self): self.assertEqual(repr(NoReturn), 'typing.NoReturn') - def test_not_generic(self): + def test_get_type_hints(self): + def some(arg: NoReturn) -> NoReturn: ... + def some_str(arg: 'NoReturn') -> 'typing.NoReturn': ... + + expected = {'arg': NoReturn, 'return': NoReturn} + for target in [some, some_str]: + with self.subTest(target=target): + self.assertEqual(gth(target), expected) + + def test_not_equality(self): + self.assertNotEqual(NoReturn, Never) + self.assertNotEqual(Never, NoReturn) + + +class NeverTests(BottomTypeTestsMixin, BaseTestCase): + bottom_type = Never + + def test_repr(self): + self.assertEqual(repr(Never), 'typing.Never') + + def test_get_type_hints(self): + def some(arg: Never) -> Never: ... + def some_str(arg: 'Never') -> 'typing.Never': ... + + expected = {'arg': Never, 'return': Never} + for target in [some, some_str]: + with self.subTest(target=target): + self.assertEqual(gth(target), expected) + + +class AssertNeverTests(BaseTestCase): + def test_exception(self): + with self.assertRaises(AssertionError): + assert_never(None) + + value = "some value" + with self.assertRaisesRegex(AssertionError, value): + assert_never(value) + + # Make sure a huge value doesn't get printed in its entirety + huge_value = "a" * 10000 + with self.assertRaises(AssertionError) as cm: + assert_never(huge_value) + self.assertLess( + len(cm.exception.args[0]), + typing._ASSERT_NEVER_REPR_MAX_LENGTH * 2, + ) + + +class SelfTests(BaseTestCase): + def test_equality(self): + self.assertEqual(Self, Self) + self.assertIs(Self, Self) + self.assertNotEqual(Self, None) + + def test_basics(self): + class Foo: + def bar(self) -> Self: ... + class FooStr: + def bar(self) -> 'Self': ... + class FooStrTyping: + def bar(self) -> 'typing.Self': ... + + for target in [Foo, FooStr, FooStrTyping]: + with self.subTest(target=target): + self.assertEqual(gth(target.bar), {'return': Self}) + self.assertIs(get_origin(Self), None) + + def test_repr(self): + self.assertEqual(repr(Self), 'typing.Self') + + def test_cannot_subscript(self): with self.assertRaises(TypeError): - NoReturn[int] + Self[int] def test_cannot_subclass(self): - with self.assertRaises(TypeError): - class A(NoReturn): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(Self)): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Self'): + class D(Self): pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + Self() + with self.assertRaises(TypeError): + type(Self)() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, Self) + with self.assertRaises(TypeError): + issubclass(int, Self) + + def test_alias(self): + # TypeAliases are not actually part of the spec + alias_1 = Tuple[Self, Self] + alias_2 = List[Self] + alias_3 = ClassVar[Self] + self.assertEqual(get_args(alias_1), (Self, Self)) + self.assertEqual(get_args(alias_2), (Self,)) + self.assertEqual(get_args(alias_3), (Self,)) + + +class LiteralStringTests(BaseTestCase): + def test_equality(self): + self.assertEqual(LiteralString, LiteralString) + self.assertIs(LiteralString, LiteralString) + self.assertNotEqual(LiteralString, None) + + def test_basics(self): + class Foo: + def bar(self) -> LiteralString: ... + class FooStr: + def bar(self) -> 'LiteralString': ... + class FooStrTyping: + def bar(self) -> 'typing.LiteralString': ... + + for target in [Foo, FooStr, FooStrTyping]: + with self.subTest(target=target): + self.assertEqual(gth(target.bar), {'return': LiteralString}) + self.assertIs(get_origin(LiteralString), None) + + def test_repr(self): + self.assertEqual(repr(LiteralString), 'typing.LiteralString') + + def test_cannot_subscript(self): with self.assertRaises(TypeError): - class A(type(NoReturn)): + LiteralString[int] + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(LiteralString)): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.LiteralString'): + class D(LiteralString): pass - def test_cannot_instantiate(self): + def test_cannot_init(self): + with self.assertRaises(TypeError): + LiteralString() + with self.assertRaises(TypeError): + type(LiteralString)() + + def test_no_isinstance(self): with self.assertRaises(TypeError): - NoReturn() + isinstance(1, LiteralString) with self.assertRaises(TypeError): - type(NoReturn)() + issubclass(int, LiteralString) + def test_alias(self): + alias_1 = Tuple[LiteralString, LiteralString] + alias_2 = List[LiteralString] + alias_3 = ClassVar[LiteralString] + self.assertEqual(get_args(alias_1), (LiteralString, LiteralString)) + self.assertEqual(get_args(alias_2), (LiteralString,)) + self.assertEqual(get_args(alias_3), (LiteralString,)) -class TypeVarTests(BaseTestCase): +class TypeVarTests(BaseTestCase): def test_basic_plain(self): T = TypeVar('T') # T equals itself. self.assertEqual(T, T) # T is an instance of TypeVar self.assertIsInstance(T, TypeVar) + self.assertEqual(T.__name__, 'T') + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + self.assertEqual(T.__module__, __name__) + + def test_basic_with_exec(self): + ns = {} + exec('from typing import TypeVar; T = TypeVar("T", bound=float)', ns, ns) + T = ns['T'] + self.assertIsInstance(T, TypeVar) + self.assertEqual(T.__name__, 'T') + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, float) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + self.assertIs(T.__module__, None) + + def test_attributes(self): + T_bound = TypeVar('T_bound', bound=int) + self.assertEqual(T_bound.__name__, 'T_bound') + self.assertEqual(T_bound.__constraints__, ()) + self.assertIs(T_bound.__bound__, int) + + T_constraints = TypeVar('T_constraints', int, str) + self.assertEqual(T_constraints.__name__, 'T_constraints') + self.assertEqual(T_constraints.__constraints__, (int, str)) + self.assertIs(T_constraints.__bound__, None) + + T_co = TypeVar('T_co', covariant=True) + self.assertEqual(T_co.__name__, 'T_co') + self.assertIs(T_co.__covariant__, True) + self.assertIs(T_co.__contravariant__, False) + self.assertIs(T_co.__infer_variance__, False) + + T_contra = TypeVar('T_contra', contravariant=True) + self.assertEqual(T_contra.__name__, 'T_contra') + self.assertIs(T_contra.__covariant__, False) + self.assertIs(T_contra.__contravariant__, True) + self.assertIs(T_contra.__infer_variance__, False) + + T_infer = TypeVar('T_infer', infer_variance=True) + self.assertEqual(T_infer.__name__, 'T_infer') + self.assertIs(T_infer.__covariant__, False) + self.assertIs(T_infer.__contravariant__, False) + self.assertIs(T_infer.__infer_variance__, True) def test_typevar_instance_type_error(self): T = TypeVar('T') @@ -197,8 +477,8 @@ def test_or(self): self.assertEqual(X | "x", Union[X, "x"]) self.assertEqual("x" | X, Union["x", X]) # make sure the order is correct - self.assertEqual(get_args(X | "x"), (X, ForwardRef("x"))) - self.assertEqual(get_args("x" | X), (ForwardRef("x"), X)) + self.assertEqual(get_args(X | "x"), (X, EqualToForwardRef("x"))) + self.assertEqual(get_args("x" | X), (EqualToForwardRef("x"), X)) def test_union_constrained(self): A = TypeVar('A', str, bytes) @@ -218,15 +498,13 @@ def test_no_redefinition(self): self.assertNotEqual(TypeVar('T'), TypeVar('T')) self.assertNotEqual(TypeVar('T', int, str), TypeVar('T', int, str)) - def test_cannot_subclass_vars(self): - with self.assertRaises(TypeError): - class V(TypeVar('T')): - pass - - def test_cannot_subclass_var_itself(self): - with self.assertRaises(TypeError): - class V(TypeVar): - pass + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'TypeVar'): + class V(TypeVar): pass + T = TypeVar("T") + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'TypeVar'): + class W(T): pass def test_cannot_instantiate_vars(self): with self.assertRaises(TypeError): @@ -234,9 +512,12 @@ def test_cannot_instantiate_vars(self): def test_bound_errors(self): with self.assertRaises(TypeError): - TypeVar('X', bound=42) + TypeVar('X', bound=Optional) with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) + with self.assertRaisesRegex(TypeError, + r"Bound must be a type\. Got \(1, 2\)\."): + TypeVar('X', bound=(1, 2)) def test_missing__name__(self): # See bpo-39942 @@ -249,169 +530,1767 @@ def test_no_bivariant(self): with self.assertRaises(ValueError): TypeVar('T', covariant=True, contravariant=True) + def test_cannot_combine_explicit_and_infer(self): + with self.assertRaises(ValueError): + TypeVar('T', covariant=True, infer_variance=True) + with self.assertRaises(ValueError): + TypeVar('T', contravariant=True, infer_variance=True) + + def test_var_substitution(self): + T = TypeVar('T') + subst = T.__typing_subst__ + self.assertIs(subst(int), int) + self.assertEqual(subst(list[int]), list[int]) + self.assertEqual(subst(List[int]), List[int]) + self.assertEqual(subst(List), List) + self.assertIs(subst(Any), Any) + self.assertIs(subst(None), type(None)) + self.assertIs(subst(T), T) + self.assertEqual(subst(int|str), int|str) + self.assertEqual(subst(Union[int, str]), Union[int, str]) + def test_bad_var_substitution(self): T = TypeVar('T') - for arg in (), (int, str): + bad_args = ( + (), (int, str), Optional, + Generic, Generic[T], Protocol, Protocol[T], + Final, Final[int], ClassVar, ClassVar[int], + ) + for arg in bad_args: with self.subTest(arg=arg): + with self.assertRaises(TypeError): + T.__typing_subst__(arg) with self.assertRaises(TypeError): List[T][arg] with self.assertRaises(TypeError): list[T][arg] + def test_many_weakrefs(self): + # gh-108295: this used to segfault + for cls in (ParamSpec, TypeVarTuple, TypeVar): + with self.subTest(cls=cls): + vals = weakref.WeakValueDictionary() + + for x in range(10): + vals[x] = cls(str(x)) + del vals + + def test_constructor(self): + T = TypeVar(name="T") + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", bound=type) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, type) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", default=()) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, ()) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", covariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, True) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", contravariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, True) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", infer_variance=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, True) + + +class TypeParameterDefaultsTests(BaseTestCase): + def test_typevar(self): + T = TypeVar('T', default=int) + self.assertEqual(T.__default__, int) + self.assertIs(T.has_default(), True) + self.assertIsInstance(T, TypeVar) -class UnionTests(BaseTestCase): + class A(Generic[T]): ... + Alias = Optional[T] - def test_basics(self): - u = Union[int, float] - self.assertNotEqual(u, Union) + def test_typevar_none(self): + U = TypeVar('U') + U_None = TypeVar('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertIs(U.has_default(), False) + self.assertIs(U_None.__default__, None) + self.assertIs(U_None.has_default(), True) - def test_subclass_error(self): - with self.assertRaises(TypeError): - issubclass(int, Union) - with self.assertRaises(TypeError): - issubclass(Union, int) - with self.assertRaises(TypeError): - issubclass(Union[int, str], int) + class X[T]: ... + T, = X.__type_params__ + self.assertIs(T.__default__, NoDefault) + self.assertIs(T.has_default(), False) - def test_union_any(self): - u = Union[Any] - self.assertEqual(u, Any) - u1 = Union[int, Any] - u2 = Union[Any, int] - u3 = Union[Any, object] - self.assertEqual(u1, u2) - self.assertNotEqual(u1, Any) - self.assertNotEqual(u2, Any) - self.assertNotEqual(u3, Any) + def test_paramspec(self): + P = ParamSpec('P', default=(str, int)) + self.assertEqual(P.__default__, (str, int)) + self.assertIs(P.has_default(), True) + self.assertIsInstance(P, ParamSpec) - def test_union_object(self): - u = Union[object] - self.assertEqual(u, object) - u1 = Union[int, object] - u2 = Union[object, int] - self.assertEqual(u1, u2) - self.assertNotEqual(u1, object) - self.assertNotEqual(u2, object) + class A(Generic[P]): ... + Alias = typing.Callable[P, None] - def test_unordered(self): - u1 = Union[int, float] - u2 = Union[float, int] - self.assertEqual(u1, u2) + P_default = ParamSpec('P_default', default=...) + self.assertIs(P_default.__default__, ...) - def test_single_class_disappears(self): - t = Union[Employee] - self.assertIs(t, Employee) + def test_paramspec_none(self): + U = ParamSpec('U') + U_None = ParamSpec('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertIs(U.has_default(), False) + self.assertIs(U_None.__default__, None) + self.assertIs(U_None.has_default(), True) - def test_base_class_kept(self): - u = Union[Employee, Manager] - self.assertNotEqual(u, Employee) - self.assertIn(Employee, u.__args__) - self.assertIn(Manager, u.__args__) + class X[**P]: ... + P, = X.__type_params__ + self.assertIs(P.__default__, NoDefault) + self.assertIs(P.has_default(), False) - def test_union_union(self): - u = Union[int, float] - v = Union[u, Employee] - self.assertEqual(v, Union[int, float, Employee]) + def test_typevartuple(self): + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + self.assertIs(Ts.has_default(), True) + self.assertIsInstance(Ts, TypeVarTuple) - def test_repr(self): - self.assertEqual(repr(Union), 'typing.Union') - u = Union[Employee, int] - self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) - u = Union[int, Employee] - self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) - T = TypeVar('T') - u = Union[T, int][int] - self.assertEqual(repr(u), repr(int)) - u = Union[List[int], int] - self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') - u = Union[list[int], dict[str, float]] - self.assertEqual(repr(u), 'typing.Union[list[int], dict[str, float]]') - u = Union[int | float] - self.assertEqual(repr(u), 'typing.Union[int, float]') + class A(Generic[Unpack[Ts]]): ... + Alias = Optional[Unpack[Ts]] - u = Union[None, str] - self.assertEqual(repr(u), 'typing.Optional[str]') - u = Union[str, None] - self.assertEqual(repr(u), 'typing.Optional[str]') - u = Union[None, str, int] - self.assertEqual(repr(u), 'typing.Union[NoneType, str, int]') - u = Optional[str] - self.assertEqual(repr(u), 'typing.Optional[str]') + def test_typevartuple_specialization(self): + T = TypeVar("T") + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, Unpack[Ts]]): ... + self.assertEqual(A[float].__args__, (float, str, int)) + self.assertEqual(A[float, range].__args__, (float, range)) + self.assertEqual(A[float, *tuple[int, ...]].__args__, (float, *tuple[int, ...])) + + def test_typevar_and_typevartuple_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, U, Unpack[Ts]]): ... + self.assertEqual(A[int].__args__, (int, float, str, int)) + self.assertEqual(A[int, str].__args__, (int, str, str, int)) + self.assertEqual(A[int, str, range].__args__, (int, str, range)) + self.assertEqual(A[int, str, *tuple[int, ...]].__args__, (int, str, *tuple[int, ...])) - def test_cannot_subclass(self): - with self.assertRaises(TypeError): - class C(Union): - pass - with self.assertRaises(TypeError): - class C(type(Union)): - pass - with self.assertRaises(TypeError): - class C(Union[int, str]): - pass + def test_no_default_after_typevar_tuple(self): + T = TypeVar("T", default=int) + Ts = TypeVarTuple("Ts") + Ts_default = TypeVarTuple("Ts_default", default=Unpack[Tuple[str, int]]) - def test_cannot_instantiate(self): - with self.assertRaises(TypeError): - Union() - with self.assertRaises(TypeError): - type(Union)() - u = Union[int, float] with self.assertRaises(TypeError): - u() + class X(Generic[*Ts, T]): ... + with self.assertRaises(TypeError): - type(u)() + class Y(Generic[*Ts_default, T]): ... - def test_union_generalization(self): - self.assertFalse(Union[str, typing.Iterable[int]] == str) - self.assertFalse(Union[str, typing.Iterable[int]] == typing.Iterable[int]) - self.assertIn(str, Union[str, typing.Iterable[int]].__args__) - self.assertIn(typing.Iterable[int], Union[str, typing.Iterable[int]].__args__) + def test_allow_default_after_non_default_in_alias(self): + T_default = TypeVar('T_default', default=int) + T = TypeVar('T') + Ts = TypeVarTuple('Ts') - def test_union_compare_other(self): - self.assertNotEqual(Union, object) - self.assertNotEqual(Union, Any) - self.assertNotEqual(ClassVar, Union) - self.assertNotEqual(Optional, Union) - self.assertNotEqual([None], Optional) - self.assertNotEqual(Optional, typing.Mapping) - self.assertNotEqual(Optional[typing.MutableMapping], Union) + a1 = Callable[[T_default], T] + self.assertEqual(a1.__args__, (T_default, T)) - def test_optional(self): - o = Optional[int] - u = Union[int, None] - self.assertEqual(o, u) + a2 = dict[T_default, T] + self.assertEqual(a2.__args__, (T_default, T)) - def test_empty(self): - with self.assertRaises(TypeError): - Union[()] + a3 = typing.Dict[T_default, T] + self.assertEqual(a3.__args__, (T_default, T)) - def test_no_eval_union(self): - u = Union[int, str] - def f(x: u): ... - self.assertIs(get_type_hints(f)['x'], u) + a4 = Callable[*Ts, T] + self.assertEqual(a4.__args__, (*Ts, T)) - def test_function_repr_union(self): - def fun() -> int: ... - self.assertEqual(repr(Union[fun, int]), 'typing.Union[fun, int]') + def test_paramspec_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P]): ... + self.assertEqual(A[float].__args__, (float, (str, int))) + self.assertEqual(A[float, [range]].__args__, (float, (range,))) - def test_union_str_pattern(self): - # Shouldn't crash; see http://bugs.python.org/issue25390 - A = Union[str, Pattern] - A + def test_typevar_and_paramspec_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, U, P]): ... + self.assertEqual(A[float].__args__, (float, float, (str, int))) + self.assertEqual(A[float, int].__args__, (float, int, (str, int))) + self.assertEqual(A[float, int, [range]].__args__, (float, int, (range,))) + + def test_paramspec_and_typevar_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, (str, int), float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + + def test_paramspec_and_typevar_specialization_2(self): + T = TypeVar("T") + P = ParamSpec('P', default=...) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, ...) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, ..., float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + + def test_typevartuple_none(self): + U = TypeVarTuple('U') + U_None = TypeVarTuple('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertIs(U.has_default(), False) + self.assertIs(U_None.__default__, None) + self.assertIs(U_None.has_default(), True) + + class X[**Ts]: ... + Ts, = X.__type_params__ + self.assertIs(Ts.__default__, NoDefault) + self.assertIs(Ts.has_default(), False) + + def test_no_default_after_non_default(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') - def test_etree(self): - # See https://github.com/python/typing/issues/229 - # (Only relevant for Python 2.) - from xml.etree.ElementTree import Element + with self.assertRaisesRegex( + TypeError, r"Type parameter ~T without a default follows type parameter with a default" + ): + Test = Generic[DefaultStrT, T] - Union[Element, str] # Shouldn't crash + def test_need_more_params(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + U = TypeVar('U') - def Elem(*args): - return Element(*args) + class A(Generic[T, U, DefaultStrT]): ... + A[int, bool] + A[int, bool, str] - Union[Elem, str] # Nor should this + with self.assertRaisesRegex( + TypeError, r"Too few arguments for .+; actual 1, expected at least 2" + ): + Test = A[int] + def test_pickle(self): + global U, U_co, U_contra, U_default # pickle wants to reference the class by name + U = TypeVar('U') + U_co = TypeVar('U_co', covariant=True) + U_contra = TypeVar('U_contra', contravariant=True) + U_default = TypeVar('U_default', default=int) + for proto in range(pickle.HIGHEST_PROTOCOL): + for typevar in (U, U_co, U_contra, U_default): + z = pickle.loads(pickle.dumps(typevar, proto)) + self.assertEqual(z.__name__, typevar.__name__) + self.assertEqual(z.__covariant__, typevar.__covariant__) + self.assertEqual(z.__contravariant__, typevar.__contravariant__) + self.assertEqual(z.__bound__, typevar.__bound__) + self.assertEqual(z.__default__, typevar.__default__) + + +def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: + """Renders templates with possible combinations of replacements. + + Example 1: Suppose that: + templates = ["dog_breed are awesome", "dog_breed are cool"] + replacements = {"dog_breed": ["Huskies", "Beagles"]} + Then we would return: + [ + ("Huskies are awesome", "Huskies are cool"), + ("Beagles are awesome", "Beagles are cool") + ] + + Example 2: Suppose that: + templates = ["Huskies are word1 but also word2"] + replacements = {"word1": ["playful", "cute"], + "word2": ["feisty", "tiring"]} + Then we would return: + [ + ("Huskies are playful but also feisty"), + ("Huskies are playful but also tiring"), + ("Huskies are cute but also feisty"), + ("Huskies are cute but also tiring") + ] + + Note that if any of the replacements do not occur in any template: + templates = ["Huskies are word1", "Beagles!"] + replacements = {"word1": ["playful", "cute"], + "word2": ["feisty", "tiring"]} + Then we do not generate duplicates, returning: + [ + ("Huskies are playful", "Beagles!"), + ("Huskies are cute", "Beagles!") + ] + """ + # First, build a structure like: + # [ + # [("word1", "playful"), ("word1", "cute")], + # [("word2", "feisty"), ("word2", "tiring")] + # ] + replacement_combos = [] + for original, possible_replacements in replacements.items(): + original_replacement_tuples = [] + for replacement in possible_replacements: + original_replacement_tuples.append((original, replacement)) + replacement_combos.append(original_replacement_tuples) + + # Second, generate rendered templates, including possible duplicates. + rendered_templates = [] + for replacement_combo in itertools.product(*replacement_combos): + # replacement_combo would be e.g. + # [("word1", "playful"), ("word2", "feisty")] + templates_with_replacements = [] + for template in templates: + for original, replacement in replacement_combo: + template = template.replace(original, replacement) + templates_with_replacements.append(template) + rendered_templates.append(tuple(templates_with_replacements)) + + # Finally, remove the duplicates (but keep the order). + rendered_templates_no_duplicates = [] + for x in rendered_templates: + # Inefficient, but should be fine for our purposes. + if x not in rendered_templates_no_duplicates: + rendered_templates_no_duplicates.append(x) + + return rendered_templates_no_duplicates + + +class TemplateReplacementTests(BaseTestCase): + + def test_two_templates_two_replacements_yields_correct_renders(self): + actual = template_replace( + templates=["Cats are word1", "Dogs are word2"], + replacements={ + "word1": ["small", "cute"], + "word2": ["big", "fluffy"], + }, + ) + expected = [ + ("Cats are small", "Dogs are big"), + ("Cats are small", "Dogs are fluffy"), + ("Cats are cute", "Dogs are big"), + ("Cats are cute", "Dogs are fluffy"), + ] + self.assertEqual(actual, expected) + + def test_no_duplicates_if_replacement_not_in_templates(self): + actual = template_replace( + templates=["Cats are word1", "Dogs!"], + replacements={ + "word1": ["small", "cute"], + "word2": ["big", "fluffy"], + }, + ) + expected = [ + ("Cats are small", "Dogs!"), + ("Cats are cute", "Dogs!"), + ] + self.assertEqual(actual, expected) -class TupleTests(BaseTestCase): + +class GenericAliasSubstitutionTests(BaseTestCase): + """Tests for type variable substitution in generic aliases. + + For variadic cases, these tests should be regarded as the source of truth, + since we hadn't realised the full complexity of variadic substitution + at the time of finalizing PEP 646. For full discussion, see + https://github.com/python/cpython/issues/91162. + """ + + def test_one_parameter(self): + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + Ts2 = TypeVarTuple('Ts2') + + class C(Generic[T]): pass + + generics = ['C', 'list', 'List'] + tuple_types = ['tuple', 'Tuple'] + + tests = [ + # Alias # Args # Expected result + ('generic[T]', '[()]', 'TypeError'), + ('generic[T]', '[int]', 'generic[int]'), + ('generic[T]', '[int, str]', 'TypeError'), + ('generic[T]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), + ('generic[T]', '[*tuple_type[int]]', 'generic[int]'), + ('generic[T]', '[*tuple_type[()]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int, str]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int, ...]]', 'TypeError'), + ('generic[T]', '[*Ts]', 'TypeError'), + ('generic[T]', '[T, *Ts]', 'TypeError'), + ('generic[T]', '[*Ts, T]', 'TypeError'), + # Raises TypeError because C is not variadic. + # (If C _were_ variadic, it'd be fine.) + ('C[T, *tuple_type[int, ...]]', '[int]', 'TypeError'), + # Should definitely raise TypeError: list only takes one argument. + ('list[T, *tuple_type[int, ...]]', '[int]', 'list[int, *tuple_type[int, ...]]'), + ('List[T, *tuple_type[int, ...]]', '[int]', 'TypeError'), + # Should raise, because more than one `TypeVarTuple` is not supported. + ('generic[*Ts, *Ts2]', '[int]', 'TypeError'), + ] + + for alias_template, args_template, expected_template in tests: + rendered_templates = template_replace( + templates=[alias_template, args_template, expected_template], + replacements={'generic': generics, 'tuple_type': tuple_types} + ) + for alias_str, args_str, expected_str in rendered_templates: + with self.subTest(alias=alias_str, args=args_str, expected=expected_str): + if expected_str == 'TypeError': + with self.assertRaises(TypeError): + eval(alias_str + args_str) + else: + self.assertEqual( + eval(alias_str + args_str), + eval(expected_str) + ) + + + def test_two_parameters(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + Ts = TypeVarTuple('Ts') + + class C(Generic[T1, T2]): pass + + generics = ['C', 'dict', 'Dict'] + tuple_types = ['tuple', 'Tuple'] + + tests = [ + # Alias # Args # Expected result + ('generic[T1, T2]', '[()]', 'TypeError'), + ('generic[T1, T2]', '[int]', 'TypeError'), + ('generic[T1, T2]', '[int, str]', 'generic[int, str]'), + ('generic[T1, T2]', '[int, str, bool]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int, str, bool]]', 'TypeError'), + + ('generic[T1, T2]', '[int, *tuple_type[str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], str]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[()]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[()], *tuple_type[int, str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[()]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[()], *tuple_type[int]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[float]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[str, float]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[float, bool]]', 'TypeError'), + + ('generic[T1, T2]', '[tuple_type[int, ...]]', 'TypeError'), + ('generic[T1, T2]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), + ('generic[T1, T2]', '[*tuple_type[int, ...]]', 'TypeError'), + ('generic[T1, T2]', '[int, *tuple_type[str, ...]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, ...], str]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), + ('generic[T1, T2]', '[*Ts]', 'TypeError'), + ('generic[T1, T2]', '[T, *Ts]', 'TypeError'), + ('generic[T1, T2]', '[*Ts, T]', 'TypeError'), + # This one isn't technically valid - none of the things that + # `generic` can be (defined in `generics` above) are variadic, so we + # shouldn't really be able to do `generic[T1, *tuple_type[int, ...]]`. + # So even if type checkers shouldn't allow it, we allow it at + # runtime, in accordance with a general philosophy of "Keep the + # runtime lenient so people can experiment with typing constructs". + ('generic[T1, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), + ] + + for alias_template, args_template, expected_template in tests: + rendered_templates = template_replace( + templates=[alias_template, args_template, expected_template], + replacements={'generic': generics, 'tuple_type': tuple_types} + ) + for alias_str, args_str, expected_str in rendered_templates: + with self.subTest(alias=alias_str, args=args_str, expected=expected_str): + if expected_str == 'TypeError': + with self.assertRaises(TypeError): + eval(alias_str + args_str) + else: + self.assertEqual( + eval(alias_str + args_str), + eval(expected_str) + ) + + def test_three_parameters(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + T3 = TypeVar('T3') + + class C(Generic[T1, T2, T3]): pass + + generics = ['C'] + tuple_types = ['tuple', 'Tuple'] + + tests = [ + # Alias # Args # Expected result + ('generic[T1, bool, T2]', '[int, str]', 'generic[int, bool, str]'), + ('generic[T1, bool, T2]', '[*tuple_type[int, str]]', 'generic[int, bool, str]'), + ] + + for alias_template, args_template, expected_template in tests: + rendered_templates = template_replace( + templates=[alias_template, args_template, expected_template], + replacements={'generic': generics, 'tuple_type': tuple_types} + ) + for alias_str, args_str, expected_str in rendered_templates: + with self.subTest(alias=alias_str, args=args_str, expected=expected_str): + if expected_str == 'TypeError': + with self.assertRaises(TypeError): + eval(alias_str + args_str) + else: + self.assertEqual( + eval(alias_str + args_str), + eval(expected_str) + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_variadic_parameters(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + Ts = TypeVarTuple('Ts') + + class C(Generic[*Ts]): pass + + generics = ['C', 'tuple', 'Tuple'] + tuple_types = ['tuple', 'Tuple'] + + tests = [ + # Alias # Args # Expected result + ('generic[*Ts]', '[()]', 'generic[()]'), + ('generic[*Ts]', '[int]', 'generic[int]'), + ('generic[*Ts]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'), + ('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'), + ('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'), + ('generic[*Ts]', '[str, *tuple_type[int, ...], bool]', 'generic[str, *tuple_type[int, ...], bool]'), + ('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), + ('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), + ('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'), + ('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), + + ('generic[*Ts]', '[*Ts]', 'generic[*Ts]'), + ('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'), + ('generic[*Ts]', '[*Ts, T]', 'generic[*Ts, T]'), + ('generic[T, *Ts]', '[()]', 'TypeError'), + ('generic[T, *Ts]', '[int]', 'generic[int]'), + ('generic[T, *Ts]', '[int, str]', 'generic[int, str]'), + ('generic[T, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[list[T], *Ts]', '[()]', 'TypeError'), + ('generic[list[T], *Ts]', '[int]', 'generic[list[int]]'), + ('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'), + ('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'), + + ('generic[*Ts, T]', '[()]', 'TypeError'), + ('generic[*Ts, T]', '[int]', 'generic[int]'), + ('generic[*Ts, T]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, list[T]]', '[()]', 'TypeError'), + ('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'), + ('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'), + ('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'), + + ('generic[T1, T2, *Ts]', '[()]', 'TypeError'), + ('generic[T1, T2, *Ts]', '[int]', 'TypeError'), + ('generic[T1, T2, *Ts]', '[int, str]', 'generic[int, str]'), + ('generic[T1, T2, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[T1, T2, *Ts]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[*Ts, T1, T2]', '[()]', 'TypeError'), + ('generic[*Ts, T1, T2]', '[int]', 'TypeError'), + ('generic[*Ts, T1, T2]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts, T1, T2]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, T1, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[T1, *Ts, T2]', '[()]', 'TypeError'), + ('generic[T1, *Ts, T2]', '[int]', 'TypeError'), + ('generic[T1, *Ts, T2]', '[int, str]', 'generic[int, str]'), + ('generic[T1, *Ts, T2]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[T1, *Ts, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'), + ('generic[T, *Ts]', '[str, *tuple_type[int, ...]]', 'generic[str, *tuple_type[int, ...]]'), + ('generic[T, *Ts]', '[*tuple_type[int, ...], str]', 'generic[int, *tuple_type[int, ...], str]'), + ('generic[*Ts, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], int]'), + ('generic[*Ts, T]', '[str, *tuple_type[int, ...]]', 'generic[str, *tuple_type[int, ...], int]'), + ('generic[*Ts, T]', '[*tuple_type[int, ...], str]', 'generic[*tuple_type[int, ...], str]'), + ('generic[T1, *Ts, T2]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...], int]'), + ('generic[T, str, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, str, *tuple_type[int, ...]]'), + ('generic[*Ts, str, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], str, int]'), + ('generic[list[T], *Ts]', '[*tuple_type[int, ...]]', 'generic[list[int], *tuple_type[int, ...]]'), + ('generic[*Ts, list[T]]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], list[int]]'), + + ('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), + ('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'), + ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'), + ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool, float]', 'TypeError'), + + ('generic[T1, *tuple_type[T2, ...]]', '[int, str]', 'generic[int, *tuple_type[str, ...]]'), + ('generic[*tuple_type[T1, ...], T2]', '[int, str]', 'generic[*tuple_type[int, ...], str]'), + ('generic[T1, *tuple_type[generic[*Ts], ...]]', '[int, str, bool]', 'generic[int, *tuple_type[generic[str, bool], ...]]'), + ('generic[*tuple_type[generic[*Ts], ...], T1]', '[int, str, bool]', 'generic[*tuple_type[generic[int, str], ...], bool]'), + ] + + for alias_template, args_template, expected_template in tests: + rendered_templates = template_replace( + templates=[alias_template, args_template, expected_template], + replacements={'generic': generics, 'tuple_type': tuple_types} + ) + for alias_str, args_str, expected_str in rendered_templates: + with self.subTest(alias=alias_str, args=args_str, expected=expected_str): + if expected_str == 'TypeError': + with self.assertRaises(TypeError): + eval(alias_str + args_str) + else: + self.assertEqual( + eval(alias_str + args_str), + eval(expected_str) + ) + + +class UnpackTests(BaseTestCase): + + def test_accepts_single_type(self): + (*tuple[int],) + Unpack[Tuple[int]] + + def test_dir(self): + dir_items = set(dir(Unpack[Tuple[int]])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + + def test_rejects_multiple_types(self): + with self.assertRaises(TypeError): + Unpack[Tuple[int], Tuple[str]] + # We can't do the equivalent for `*` here - + # *(Tuple[int], Tuple[str]) is just plain tuple unpacking, + # which is valid. + + def test_rejects_multiple_parameterization(self): + with self.assertRaises(TypeError): + (*tuple[int],)[0][tuple[int]] + with self.assertRaises(TypeError): + Unpack[Tuple[int]][Tuple[int]] + + def test_cannot_be_called(self): + with self.assertRaises(TypeError): + Unpack() + + def test_usage_with_kwargs(self): + Movie = TypedDict('Movie', {'name': str, 'year': int}) + def foo(**kwargs: Unpack[Movie]): ... + self.assertEqual(repr(foo.__annotations__['kwargs']), + f"typing.Unpack[{__name__}.Movie]") + + def test_builtin_tuple(self): + Ts = TypeVarTuple("Ts") + + class Old(Generic[*Ts]): ... + class New[*Ts]: ... + + PartOld = Old[int, *Ts] + self.assertEqual(PartOld[str].__args__, (int, str)) + self.assertEqual(PartOld[*tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[Tuple[str]]].__args__, (int, str)) + + PartNew = New[int, *Ts] + self.assertEqual(PartNew[str].__args__, (int, str)) + self.assertEqual(PartNew[*tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[Tuple[str]]].__args__, (int, str)) + + def test_unpack_wrong_type(self): + Ts = TypeVarTuple("Ts") + class Gen[*Ts]: ... + PartGen = Gen[int, *Ts] + + bad_unpack_param = re.escape("Unpack[...] must be used with a tuple type") + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[list[int]]] + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[List[int]]] + + +class TypeVarTupleTests(BaseTestCase): + + def test_name(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(Ts.__name__, 'Ts') + Ts2 = TypeVarTuple('Ts2') + self.assertEqual(Ts2.__name__, 'Ts2') + + def test_module(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(Ts.__module__, __name__) + + def test_exec(self): + ns = {} + exec('from typing import TypeVarTuple; Ts = TypeVarTuple("Ts")', ns) + Ts = ns['Ts'] + self.assertEqual(Ts.__name__, 'Ts') + self.assertIs(Ts.__module__, None) + + def test_instance_is_equal_to_itself(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(Ts, Ts) + + def test_different_instances_are_different(self): + self.assertNotEqual(TypeVarTuple('Ts'), TypeVarTuple('Ts')) + + def test_instance_isinstance_of_typevartuple(self): + Ts = TypeVarTuple('Ts') + self.assertIsInstance(Ts, TypeVarTuple) + + def test_cannot_call_instance(self): + Ts = TypeVarTuple('Ts') + with self.assertRaises(TypeError): + Ts() + + def test_unpacked_typevartuple_is_equal_to_itself(self): + Ts = TypeVarTuple('Ts') + self.assertEqual((*Ts,)[0], (*Ts,)[0]) + self.assertEqual(Unpack[Ts], Unpack[Ts]) + + def test_parameterised_tuple_is_equal_to_itself(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(tuple[*Ts], tuple[*Ts]) + self.assertEqual(Tuple[Unpack[Ts]], Tuple[Unpack[Ts]]) + + def tests_tuple_arg_ordering_matters(self): + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + self.assertNotEqual( + tuple[*Ts1, *Ts2], + tuple[*Ts2, *Ts1], + ) + self.assertNotEqual( + Tuple[Unpack[Ts1], Unpack[Ts2]], + Tuple[Unpack[Ts2], Unpack[Ts1]], + ) + + def test_tuple_args_and_parameters_are_correct(self): + Ts = TypeVarTuple('Ts') + t1 = tuple[*Ts] + self.assertEqual(t1.__args__, (*Ts,)) + self.assertEqual(t1.__parameters__, (Ts,)) + t2 = Tuple[Unpack[Ts]] + self.assertEqual(t2.__args__, (Unpack[Ts],)) + self.assertEqual(t2.__parameters__, (Ts,)) + + def test_var_substitution(self): + Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T2 = TypeVar('T2') + class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass + + for A in G1, G2, Tuple, tuple: + B = A[*Ts] + self.assertEqual(B[()], A[()]) + self.assertEqual(B[float], A[float]) + self.assertEqual(B[float, str], A[float, str]) + + C = A[Unpack[Ts]] + self.assertEqual(C[()], A[()]) + self.assertEqual(C[float], A[float]) + self.assertEqual(C[float, str], A[float, str]) + + D = list[A[*Ts]] + self.assertEqual(D[()], list[A[()]]) + self.assertEqual(D[float], list[A[float]]) + self.assertEqual(D[float, str], list[A[float, str]]) + + E = List[A[Unpack[Ts]]] + self.assertEqual(E[()], List[A[()]]) + self.assertEqual(E[float], List[A[float]]) + self.assertEqual(E[float, str], List[A[float, str]]) + + F = A[T, *Ts, T2] + with self.assertRaises(TypeError): + F[()] + with self.assertRaises(TypeError): + F[float] + self.assertEqual(F[float, str], A[float, str]) + self.assertEqual(F[float, str, int], A[float, str, int]) + self.assertEqual(F[float, str, int, bytes], A[float, str, int, bytes]) + + G = A[T, Unpack[Ts], T2] + with self.assertRaises(TypeError): + G[()] + with self.assertRaises(TypeError): + G[float] + self.assertEqual(G[float, str], A[float, str]) + self.assertEqual(G[float, str, int], A[float, str, int]) + self.assertEqual(G[float, str, int, bytes], A[float, str, int, bytes]) + + H = tuple[list[T], A[*Ts], list[T2]] + with self.assertRaises(TypeError): + H[()] + with self.assertRaises(TypeError): + H[float] + if A != Tuple: + self.assertEqual(H[float, str], + tuple[list[float], A[()], list[str]]) + self.assertEqual(H[float, str, int], + tuple[list[float], A[str], list[int]]) + self.assertEqual(H[float, str, int, bytes], + tuple[list[float], A[str, int], list[bytes]]) + + I = Tuple[List[T], A[Unpack[Ts]], List[T2]] + with self.assertRaises(TypeError): + I[()] + with self.assertRaises(TypeError): + I[float] + if A != Tuple: + self.assertEqual(I[float, str], + Tuple[List[float], A[()], List[str]]) + self.assertEqual(I[float, str, int], + Tuple[List[float], A[str], List[int]]) + self.assertEqual(I[float, str, int, bytes], + Tuple[List[float], A[str, int], List[bytes]]) + + def test_bad_var_substitution(self): + Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T2 = TypeVar('T2') + class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass + + for A in G1, G2, Tuple, tuple: + B = A[Ts] + with self.assertRaises(TypeError): + B[int, str] + + C = A[T, T2] + with self.assertRaises(TypeError): + C[*Ts] + with self.assertRaises(TypeError): + C[Unpack[Ts]] + + B = A[T, *Ts, str, T2] + with self.assertRaises(TypeError): + B[int, *Ts] + with self.assertRaises(TypeError): + B[int, *Ts, *Ts] + + C = A[T, Unpack[Ts], str, T2] + with self.assertRaises(TypeError): + C[int, Unpack[Ts]] + with self.assertRaises(TypeError): + C[int, Unpack[Ts], Unpack[Ts]] + + def test_repr_is_correct(self): + Ts = TypeVarTuple('Ts') + + class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass + + self.assertEqual(repr(Ts), 'Ts') + + self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[Ts]') + self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[Ts]') + + self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[Ts]]') + + self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[Ts]]]') + + def test_variadic_class_repr_is_correct(self): + Ts = TypeVarTuple('Ts') + class A(Generic[*Ts]): pass + class B(Generic[Unpack[Ts]]): pass + + self.assertEndsWith(repr(A[()]), 'A[()]') + self.assertEndsWith(repr(B[()]), 'B[()]') + self.assertEndsWith(repr(A[float]), 'A[float]') + self.assertEndsWith(repr(B[float]), 'B[float]') + self.assertEndsWith(repr(A[float, str]), 'A[float, str]') + self.assertEndsWith(repr(B[float, str]), 'B[float, str]') + + self.assertEndsWith(repr(A[*tuple[int, ...]]), + 'A[*tuple[int, ...]]') + self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]]]), + 'B[typing.Unpack[typing.Tuple[int, ...]]]') + + self.assertEndsWith(repr(A[float, *tuple[int, ...]]), + 'A[float, *tuple[int, ...]]') + self.assertEndsWith(repr(A[float, Unpack[Tuple[int, ...]]]), + 'A[float, typing.Unpack[typing.Tuple[int, ...]]]') + + self.assertEndsWith(repr(A[*tuple[int, ...], str]), + 'A[*tuple[int, ...], str]') + self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]], str]), + 'B[typing.Unpack[typing.Tuple[int, ...]], str]') + + self.assertEndsWith(repr(A[float, *tuple[int, ...], str]), + 'A[float, *tuple[int, ...], str]') + self.assertEndsWith(repr(B[float, Unpack[Tuple[int, ...]], str]), + 'B[float, typing.Unpack[typing.Tuple[int, ...]], str]') + + def test_variadic_class_alias_repr_is_correct(self): + Ts = TypeVarTuple('Ts') + class A(Generic[Unpack[Ts]]): pass + + B = A[*Ts] + self.assertEndsWith(repr(B), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(B[()]), 'A[()]') + self.assertEndsWith(repr(B[float]), 'A[float]') + self.assertEndsWith(repr(B[float, str]), 'A[float, str]') + + C = A[Unpack[Ts]] + self.assertEndsWith(repr(C), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(C[()]), 'A[()]') + self.assertEndsWith(repr(C[float]), 'A[float]') + self.assertEndsWith(repr(C[float, str]), 'A[float, str]') + + D = A[*Ts, int] + self.assertEndsWith(repr(D), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(D[()]), 'A[int]') + self.assertEndsWith(repr(D[float]), 'A[float, int]') + self.assertEndsWith(repr(D[float, str]), 'A[float, str, int]') + + E = A[Unpack[Ts], int] + self.assertEndsWith(repr(E), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(E[()]), 'A[int]') + self.assertEndsWith(repr(E[float]), 'A[float, int]') + self.assertEndsWith(repr(E[float, str]), 'A[float, str, int]') + + F = A[int, *Ts] + self.assertEndsWith(repr(F), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(F[()]), 'A[int]') + self.assertEndsWith(repr(F[float]), 'A[int, float]') + self.assertEndsWith(repr(F[float, str]), 'A[int, float, str]') + + G = A[int, Unpack[Ts]] + self.assertEndsWith(repr(G), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(G[()]), 'A[int]') + self.assertEndsWith(repr(G[float]), 'A[int, float]') + self.assertEndsWith(repr(G[float, str]), 'A[int, float, str]') + + H = A[int, *Ts, str] + self.assertEndsWith(repr(H), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(H[()]), 'A[int, str]') + self.assertEndsWith(repr(H[float]), 'A[int, float, str]') + self.assertEndsWith(repr(H[float, str]), 'A[int, float, str, str]') + + I = A[int, Unpack[Ts], str] + self.assertEndsWith(repr(I), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(I[()]), 'A[int, str]') + self.assertEndsWith(repr(I[float]), 'A[int, float, str]') + self.assertEndsWith(repr(I[float, str]), 'A[int, float, str, str]') + + J = A[*Ts, *tuple[str, ...]] + self.assertEndsWith(repr(J), 'A[typing.Unpack[Ts], *tuple[str, ...]]') + self.assertEndsWith(repr(J[()]), 'A[*tuple[str, ...]]') + self.assertEndsWith(repr(J[float]), 'A[float, *tuple[str, ...]]') + self.assertEndsWith(repr(J[float, str]), 'A[float, str, *tuple[str, ...]]') + + K = A[Unpack[Ts], Unpack[Tuple[str, ...]]] + self.assertEndsWith(repr(K), 'A[typing.Unpack[Ts], typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[()]), 'A[typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[float]), 'A[float, typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[float, str]), 'A[float, str, typing.Unpack[typing.Tuple[str, ...]]]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'TypeVarTuple'): + class C(TypeVarTuple): pass + Ts = TypeVarTuple('Ts') + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'TypeVarTuple'): + class D(Ts): pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class E(type(Unpack)): pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class F(type(*Ts)): pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class G(type(Unpack[Ts])): pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Unpack'): + class H(Unpack): pass + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + class I(*Ts): pass + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + class J(Unpack[Ts]): pass + + def test_variadic_class_args_are_correct(self): + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + class A(Generic[*Ts]): pass + class B(Generic[Unpack[Ts]]): pass + + C = A[()] + D = B[()] + self.assertEqual(C.__args__, ()) + self.assertEqual(D.__args__, ()) + + E = A[int] + F = B[int] + self.assertEqual(E.__args__, (int,)) + self.assertEqual(F.__args__, (int,)) + + G = A[int, str] + H = B[int, str] + self.assertEqual(G.__args__, (int, str)) + self.assertEqual(H.__args__, (int, str)) + + I = A[T] + J = B[T] + self.assertEqual(I.__args__, (T,)) + self.assertEqual(J.__args__, (T,)) + + K = A[*Ts] + L = B[Unpack[Ts]] + self.assertEqual(K.__args__, (*Ts,)) + self.assertEqual(L.__args__, (Unpack[Ts],)) + + M = A[T, *Ts] + N = B[T, Unpack[Ts]] + self.assertEqual(M.__args__, (T, *Ts)) + self.assertEqual(N.__args__, (T, Unpack[Ts])) + + O = A[*Ts, T] + P = B[Unpack[Ts], T] + self.assertEqual(O.__args__, (*Ts, T)) + self.assertEqual(P.__args__, (Unpack[Ts], T)) + + def test_variadic_class_origin_is_correct(self): + Ts = TypeVarTuple('Ts') + + class C(Generic[*Ts]): pass + self.assertIs(C[int].__origin__, C) + self.assertIs(C[T].__origin__, C) + self.assertIs(C[Unpack[Ts]].__origin__, C) + + class D(Generic[Unpack[Ts]]): pass + self.assertIs(D[int].__origin__, D) + self.assertIs(D[T].__origin__, D) + self.assertIs(D[Unpack[Ts]].__origin__, D) + + def test_get_type_hints_on_unpack_args(self): + Ts = TypeVarTuple('Ts') + + def func1(*args: *Ts): pass + self.assertEqual(gth(func1), {'args': Unpack[Ts]}) + + def func2(*args: *tuple[int, str]): pass + self.assertEqual(gth(func2), {'args': Unpack[tuple[int, str]]}) + + class CustomVariadic(Generic[*Ts]): pass + + def func3(*args: *CustomVariadic[int, str]): pass + self.assertEqual(gth(func3), {'args': Unpack[CustomVariadic[int, str]]}) + + def test_get_type_hints_on_unpack_args_string(self): + Ts = TypeVarTuple('Ts') + + def func1(*args: '*Ts'): pass + self.assertEqual(gth(func1, localns={'Ts': Ts}), + {'args': Unpack[Ts]}) + + def func2(*args: '*tuple[int, str]'): pass + self.assertEqual(gth(func2), {'args': Unpack[tuple[int, str]]}) + + class CustomVariadic(Generic[*Ts]): pass + + def func3(*args: '*CustomVariadic[int, str]'): pass + self.assertEqual(gth(func3, localns={'CustomVariadic': CustomVariadic}), + {'args': Unpack[CustomVariadic[int, str]]}) + + def test_tuple_args_are_correct(self): + Ts = TypeVarTuple('Ts') + + self.assertEqual(tuple[*Ts].__args__, (*Ts,)) + self.assertEqual(Tuple[Unpack[Ts]].__args__, (Unpack[Ts],)) + + self.assertEqual(tuple[*Ts, int].__args__, (*Ts, int)) + self.assertEqual(Tuple[Unpack[Ts], int].__args__, (Unpack[Ts], int)) + + self.assertEqual(tuple[int, *Ts].__args__, (int, *Ts)) + self.assertEqual(Tuple[int, Unpack[Ts]].__args__, (int, Unpack[Ts])) + + self.assertEqual(tuple[int, *Ts, str].__args__, + (int, *Ts, str)) + self.assertEqual(Tuple[int, Unpack[Ts], str].__args__, + (int, Unpack[Ts], str)) + + self.assertEqual(tuple[*Ts, int].__args__, (*Ts, int)) + self.assertEqual(Tuple[Unpack[Ts]].__args__, (Unpack[Ts],)) + + def test_callable_args_are_correct(self): + Ts = TypeVarTuple('Ts') + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + # TypeVarTuple in the arguments + + a = Callable[[*Ts], None] + b = Callable[[Unpack[Ts]], None] + self.assertEqual(a.__args__, (*Ts, type(None))) + self.assertEqual(b.__args__, (Unpack[Ts], type(None))) + + c = Callable[[int, *Ts], None] + d = Callable[[int, Unpack[Ts]], None] + self.assertEqual(c.__args__, (int, *Ts, type(None))) + self.assertEqual(d.__args__, (int, Unpack[Ts], type(None))) + + e = Callable[[*Ts, int], None] + f = Callable[[Unpack[Ts], int], None] + self.assertEqual(e.__args__, (*Ts, int, type(None))) + self.assertEqual(f.__args__, (Unpack[Ts], int, type(None))) + + g = Callable[[str, *Ts, int], None] + h = Callable[[str, Unpack[Ts], int], None] + self.assertEqual(g.__args__, (str, *Ts, int, type(None))) + self.assertEqual(h.__args__, (str, Unpack[Ts], int, type(None))) + + # TypeVarTuple as the return + + i = Callable[[None], *Ts] + j = Callable[[None], Unpack[Ts]] + self.assertEqual(i.__args__, (type(None), *Ts)) + self.assertEqual(j.__args__, (type(None), Unpack[Ts])) + + k = Callable[[None], tuple[int, *Ts]] + l = Callable[[None], Tuple[int, Unpack[Ts]]] + self.assertEqual(k.__args__, (type(None), tuple[int, *Ts])) + self.assertEqual(l.__args__, (type(None), Tuple[int, Unpack[Ts]])) + + m = Callable[[None], tuple[*Ts, int]] + n = Callable[[None], Tuple[Unpack[Ts], int]] + self.assertEqual(m.__args__, (type(None), tuple[*Ts, int])) + self.assertEqual(n.__args__, (type(None), Tuple[Unpack[Ts], int])) + + o = Callable[[None], tuple[str, *Ts, int]] + p = Callable[[None], Tuple[str, Unpack[Ts], int]] + self.assertEqual(o.__args__, (type(None), tuple[str, *Ts, int])) + self.assertEqual(p.__args__, (type(None), Tuple[str, Unpack[Ts], int])) + + # TypeVarTuple in both + + q = Callable[[*Ts], *Ts] + r = Callable[[Unpack[Ts]], Unpack[Ts]] + self.assertEqual(q.__args__, (*Ts, *Ts)) + self.assertEqual(r.__args__, (Unpack[Ts], Unpack[Ts])) + + s = Callable[[*Ts1], *Ts2] + u = Callable[[Unpack[Ts1]], Unpack[Ts2]] + self.assertEqual(s.__args__, (*Ts1, *Ts2)) + self.assertEqual(u.__args__, (Unpack[Ts1], Unpack[Ts2])) + + def test_variadic_class_with_duplicate_typevartuples_fails(self): + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + with self.assertRaises(TypeError): + class C(Generic[*Ts1, *Ts1]): pass + with self.assertRaises(TypeError): + class D(Generic[Unpack[Ts1], Unpack[Ts1]]): pass + + with self.assertRaises(TypeError): + class E(Generic[*Ts1, *Ts2, *Ts1]): pass + with self.assertRaises(TypeError): + class F(Generic[Unpack[Ts1], Unpack[Ts2], Unpack[Ts1]]): pass + + def test_type_concatenation_in_variadic_class_argument_list_succeeds(self): + Ts = TypeVarTuple('Ts') + class C(Generic[Unpack[Ts]]): pass + + C[int, *Ts] + C[int, Unpack[Ts]] + + C[*Ts, int] + C[Unpack[Ts], int] + + C[int, *Ts, str] + C[int, Unpack[Ts], str] + + C[int, bool, *Ts, float, str] + C[int, bool, Unpack[Ts], float, str] + + def test_type_concatenation_in_tuple_argument_list_succeeds(self): + Ts = TypeVarTuple('Ts') + + tuple[int, *Ts] + tuple[*Ts, int] + tuple[int, *Ts, str] + tuple[int, bool, *Ts, float, str] + + Tuple[int, Unpack[Ts]] + Tuple[Unpack[Ts], int] + Tuple[int, Unpack[Ts], str] + Tuple[int, bool, Unpack[Ts], float, str] + + def test_variadic_class_definition_using_packed_typevartuple_fails(self): + Ts = TypeVarTuple('Ts') + with self.assertRaises(TypeError): + class C(Generic[Ts]): pass + + def test_variadic_class_definition_using_concrete_types_fails(self): + Ts = TypeVarTuple('Ts') + with self.assertRaises(TypeError): + class F(Generic[*Ts, int]): pass + with self.assertRaises(TypeError): + class E(Generic[Unpack[Ts], int]): pass + + def test_variadic_class_with_2_typevars_accepts_2_or_more_args(self): + Ts = TypeVarTuple('Ts') + T1 = TypeVar('T1') + T2 = TypeVar('T2') + + class A(Generic[T1, T2, *Ts]): pass + A[int, str] + A[int, str, float] + A[int, str, float, bool] + + class B(Generic[T1, T2, Unpack[Ts]]): pass + B[int, str] + B[int, str, float] + B[int, str, float, bool] + + class C(Generic[T1, *Ts, T2]): pass + C[int, str] + C[int, str, float] + C[int, str, float, bool] + + class D(Generic[T1, Unpack[Ts], T2]): pass + D[int, str] + D[int, str, float] + D[int, str, float, bool] + + class E(Generic[*Ts, T1, T2]): pass + E[int, str] + E[int, str, float] + E[int, str, float, bool] + + class F(Generic[Unpack[Ts], T1, T2]): pass + F[int, str] + F[int, str, float] + F[int, str, float, bool] + + def test_variadic_args_annotations_are_correct(self): + Ts = TypeVarTuple('Ts') + + def f(*args: Unpack[Ts]): pass + def g(*args: *Ts): pass + self.assertEqual(f.__annotations__, {'args': Unpack[Ts]}) + self.assertEqual(g.__annotations__, {'args': (*Ts,)[0]}) + + def test_variadic_args_with_ellipsis_annotations_are_correct(self): + def a(*args: *tuple[int, ...]): pass + self.assertEqual(a.__annotations__, + {'args': (*tuple[int, ...],)[0]}) + + def b(*args: Unpack[Tuple[int, ...]]): pass + self.assertEqual(b.__annotations__, + {'args': Unpack[Tuple[int, ...]]}) + + def test_concatenation_in_variadic_args_annotations_are_correct(self): + Ts = TypeVarTuple('Ts') + + # Unpacking using `*`, native `tuple` type + + def a(*args: *tuple[int, *Ts]): pass + self.assertEqual( + a.__annotations__, + {'args': (*tuple[int, *Ts],)[0]}, + ) + + def b(*args: *tuple[*Ts, int]): pass + self.assertEqual( + b.__annotations__, + {'args': (*tuple[*Ts, int],)[0]}, + ) + + def c(*args: *tuple[str, *Ts, int]): pass + self.assertEqual( + c.__annotations__, + {'args': (*tuple[str, *Ts, int],)[0]}, + ) + + def d(*args: *tuple[int, bool, *Ts, float, str]): pass + self.assertEqual( + d.__annotations__, + {'args': (*tuple[int, bool, *Ts, float, str],)[0]}, + ) + + # Unpacking using `Unpack`, `Tuple` type from typing.py + + def e(*args: Unpack[Tuple[int, Unpack[Ts]]]): pass + self.assertEqual( + e.__annotations__, + {'args': Unpack[Tuple[int, Unpack[Ts]]]}, + ) + + def f(*args: Unpack[Tuple[Unpack[Ts], int]]): pass + self.assertEqual( + f.__annotations__, + {'args': Unpack[Tuple[Unpack[Ts], int]]}, + ) + + def g(*args: Unpack[Tuple[str, Unpack[Ts], int]]): pass + self.assertEqual( + g.__annotations__, + {'args': Unpack[Tuple[str, Unpack[Ts], int]]}, + ) + + def h(*args: Unpack[Tuple[int, bool, Unpack[Ts], float, str]]): pass + self.assertEqual( + h.__annotations__, + {'args': Unpack[Tuple[int, bool, Unpack[Ts], float, str]]}, + ) + + def test_variadic_class_same_args_results_in_equalty(self): + Ts = TypeVarTuple('Ts') + class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass + + self.assertEqual(C[int], C[int]) + self.assertEqual(D[int], D[int]) + + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + self.assertEqual( + C[*Ts1], + C[*Ts1], + ) + self.assertEqual( + D[Unpack[Ts1]], + D[Unpack[Ts1]], + ) + + self.assertEqual( + C[*Ts1, *Ts2], + C[*Ts1, *Ts2], + ) + self.assertEqual( + D[Unpack[Ts1], Unpack[Ts2]], + D[Unpack[Ts1], Unpack[Ts2]], + ) + + self.assertEqual( + C[int, *Ts1, *Ts2], + C[int, *Ts1, *Ts2], + ) + self.assertEqual( + D[int, Unpack[Ts1], Unpack[Ts2]], + D[int, Unpack[Ts1], Unpack[Ts2]], + ) + + def test_variadic_class_arg_ordering_matters(self): + Ts = TypeVarTuple('Ts') + class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass + + self.assertNotEqual( + C[int, str], + C[str, int], + ) + self.assertNotEqual( + D[int, str], + D[str, int], + ) + + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + self.assertNotEqual( + C[*Ts1, *Ts2], + C[*Ts2, *Ts1], + ) + self.assertNotEqual( + D[Unpack[Ts1], Unpack[Ts2]], + D[Unpack[Ts2], Unpack[Ts1]], + ) + + def test_variadic_class_arg_typevartuple_identity_matters(self): + Ts = TypeVarTuple('Ts') + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass + + self.assertNotEqual(C[*Ts1], C[*Ts2]) + self.assertNotEqual(D[Unpack[Ts1]], D[Unpack[Ts2]]) + + +class TypeVarTuplePicklingTests(BaseTestCase): + # These are slightly awkward tests to run, because TypeVarTuples are only + # picklable if defined in the global scope. We therefore need to push + # various things defined in these tests into the global scope with `global` + # statements at the start of each test. + + @all_pickle_protocols + def test_pickling_then_unpickling_results_in_same_identity(self, proto): + global global_Ts1 # See explanation at start of class. + global_Ts1 = TypeVarTuple('global_Ts1') + global_Ts2 = pickle.loads(pickle.dumps(global_Ts1, proto)) + self.assertIs(global_Ts1, global_Ts2) + + @all_pickle_protocols + def test_pickling_then_unpickling_unpacked_results_in_same_identity(self, proto): + global global_Ts # See explanation at start of class. + global_Ts = TypeVarTuple('global_Ts') + + unpacked1 = (*global_Ts,)[0] + unpacked2 = pickle.loads(pickle.dumps(unpacked1, proto)) + self.assertIs(unpacked1, unpacked2) + + unpacked3 = Unpack[global_Ts] + unpacked4 = pickle.loads(pickle.dumps(unpacked3, proto)) + self.assertIs(unpacked3, unpacked4) + + @all_pickle_protocols + def test_pickling_then_unpickling_tuple_with_typevartuple_equality( + self, proto + ): + global global_T, global_Ts # See explanation at start of class. + global_T = TypeVar('global_T') + global_Ts = TypeVarTuple('global_Ts') + + tuples = [ + tuple[*global_Ts], + Tuple[Unpack[global_Ts]], + + tuple[T, *global_Ts], + Tuple[T, Unpack[global_Ts]], + + tuple[int, *global_Ts], + Tuple[int, Unpack[global_Ts]], + ] + for t in tuples: + t2 = pickle.loads(pickle.dumps(t, proto)) + self.assertEqual(t, t2) + + +class UnionTests(BaseTestCase): + + def test_basics(self): + u = Union[int, float] + self.assertNotEqual(u, Union) + + def test_union_isinstance(self): + self.assertIsInstance(42, Union[int, str]) + self.assertIsInstance('abc', Union[int, str]) + self.assertNotIsInstance(3.14, Union[int, str]) + self.assertIsInstance(42, Union[int, list[int]]) + self.assertIsInstance(42, Union[int, Any]) + + def test_union_isinstance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Union[str, list[int]]) + with self.assertRaises(TypeError): + isinstance(42, Union[list[int], int]) + with self.assertRaises(TypeError): + isinstance(42, Union[list[int], str]) + with self.assertRaises(TypeError): + isinstance(42, Union[str, Any]) + with self.assertRaises(TypeError): + isinstance(42, Union[Any, int]) + with self.assertRaises(TypeError): + isinstance(42, Union[Any, str]) + + def test_optional_isinstance(self): + self.assertIsInstance(42, Optional[int]) + self.assertIsInstance(None, Optional[int]) + self.assertNotIsInstance('abc', Optional[int]) + + def test_optional_isinstance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Optional[list[int]]) + with self.assertRaises(TypeError): + isinstance(None, Optional[list[int]]) + with self.assertRaises(TypeError): + isinstance(42, Optional[Any]) + with self.assertRaises(TypeError): + isinstance(None, Optional[Any]) + + def test_union_issubclass(self): + self.assertIsSubclass(int, Union[int, str]) + self.assertIsSubclass(str, Union[int, str]) + self.assertNotIsSubclass(float, Union[int, str]) + self.assertIsSubclass(int, Union[int, list[int]]) + self.assertIsSubclass(int, Union[int, Any]) + self.assertNotIsSubclass(int, Union[str, Any]) + self.assertIsSubclass(int, Union[Any, int]) + self.assertNotIsSubclass(int, Union[Any, str]) + + def test_union_issubclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(Union[int, str], int) + with self.assertRaises(TypeError): + issubclass(int, Union[str, list[int]]) + with self.assertRaises(TypeError): + issubclass(int, Union[list[int], int]) + with self.assertRaises(TypeError): + issubclass(int, Union[list[int], str]) + + def test_optional_issubclass(self): + self.assertIsSubclass(int, Optional[int]) + self.assertIsSubclass(type(None), Optional[int]) + self.assertNotIsSubclass(str, Optional[int]) + self.assertIsSubclass(Any, Optional[Any]) + self.assertIsSubclass(type(None), Optional[Any]) + self.assertNotIsSubclass(int, Optional[Any]) + + def test_optional_issubclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(list[int], Optional[list[int]]) + with self.assertRaises(TypeError): + issubclass(type(None), Optional[list[int]]) + with self.assertRaises(TypeError): + issubclass(int, Optional[list[int]]) + + def test_union_any(self): + u = Union[Any] + self.assertEqual(u, Any) + u1 = Union[int, Any] + u2 = Union[Any, int] + u3 = Union[Any, object] + self.assertEqual(u1, u2) + self.assertNotEqual(u1, Any) + self.assertNotEqual(u2, Any) + self.assertNotEqual(u3, Any) + + def test_union_object(self): + u = Union[object] + self.assertEqual(u, object) + u1 = Union[int, object] + u2 = Union[object, int] + self.assertEqual(u1, u2) + self.assertNotEqual(u1, object) + self.assertNotEqual(u2, object) + + def test_unordered(self): + u1 = Union[int, float] + u2 = Union[float, int] + self.assertEqual(u1, u2) + + def test_single_class_disappears(self): + t = Union[Employee] + self.assertIs(t, Employee) + + def test_base_class_kept(self): + u = Union[Employee, Manager] + self.assertNotEqual(u, Employee) + self.assertIn(Employee, u.__args__) + self.assertIn(Manager, u.__args__) + + def test_union_union(self): + u = Union[int, float] + v = Union[u, Employee] + self.assertEqual(v, Union[int, float, Employee]) + + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual(Union[A, B].__args__, (A, B)) + union1 = Union[A, B] + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union1) + + union2 = Union[int, B] + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union2) + + union3 = Union[A, int] + with self.assertRaisesRegex(TypeError, "unhashable type: 'UnhashableMeta'"): + hash(union3) + + def test_repr(self): + u = Union[Employee, int] + self.assertEqual(repr(u), f'{__name__}.Employee | int') + u = Union[int, Employee] + self.assertEqual(repr(u), f'int | {__name__}.Employee') + T = TypeVar('T') + u = Union[T, int][int] + self.assertEqual(repr(u), repr(int)) + u = Union[List[int], int] + self.assertEqual(repr(u), 'typing.List[int] | int') + u = Union[list[int], dict[str, float]] + self.assertEqual(repr(u), 'list[int] | dict[str, float]') + u = Union[int | float] + self.assertEqual(repr(u), 'int | float') + + u = Union[None, str] + self.assertEqual(repr(u), 'None | str') + u = Union[str, None] + self.assertEqual(repr(u), 'str | None') + u = Union[None, str, int] + self.assertEqual(repr(u), 'None | str | int') + u = Optional[str] + self.assertEqual(repr(u), 'str | None') + + def test_dir(self): + dir_items = set(dir(Union[str, int])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, + r"type 'typing\.Union' is not an acceptable base type"): + class C(Union): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass int \| str'): + class E(Union[int, str]): + pass + + def test_cannot_instantiate(self): + with self.assertRaises(TypeError): + Union() + with self.assertRaises(TypeError): + type(Union)() + u = Union[int, float] + with self.assertRaises(TypeError): + u() + with self.assertRaises(TypeError): + type(u)() + + def test_union_generalization(self): + self.assertNotEqual(Union[str, typing.Iterable[int]], str) + self.assertNotEqual(Union[str, typing.Iterable[int]], typing.Iterable[int]) + self.assertIn(str, Union[str, typing.Iterable[int]].__args__) + self.assertIn(typing.Iterable[int], Union[str, typing.Iterable[int]].__args__) + + def test_union_compare_other(self): + self.assertNotEqual(Union, object) + self.assertNotEqual(Union, Any) + self.assertNotEqual(ClassVar, Union) + self.assertNotEqual(Optional, Union) + self.assertNotEqual([None], Optional) + self.assertNotEqual(Optional, typing.Mapping) + self.assertNotEqual(Optional[typing.MutableMapping], Union) + + def test_optional(self): + o = Optional[int] + u = Union[int, None] + self.assertEqual(o, u) + + def test_empty(self): + with self.assertRaises(TypeError): + Union[()] + + def test_no_eval_union(self): + u = Union[int, str] + def f(x: u): ... + self.assertIs(get_type_hints(f)['x'], u) + + def test_function_repr_union(self): + def fun() -> int: ... + self.assertEqual(repr(Union[fun, int]), f'{__name__}.{fun.__qualname__} | int') + + def test_union_str_pattern(self): + # Shouldn't crash; see http://bugs.python.org/issue25390 + A = Union[str, Pattern] + A + + def test_etree(self): + # See https://github.com/python/typing/issues/229 + # (Only relevant for Python 2.) + from xml.etree.ElementTree import Element + + Union[Element, str] # Shouldn't crash + + def Elem(*args): + return Element(*args) + + Union[Elem, str] # Nor should this + + def test_union_of_literals(self): + self.assertEqual(Union[Literal[1], Literal[2]].__args__, + (Literal[1], Literal[2])) + self.assertEqual(Union[Literal[1], Literal[1]], + Literal[1]) + + self.assertEqual(Union[Literal[False], Literal[0]].__args__, + (Literal[False], Literal[0])) + self.assertEqual(Union[Literal[True], Literal[1]].__args__, + (Literal[True], Literal[1])) + + import enum + class Ints(enum.IntEnum): + A = 0 + B = 1 + + self.assertEqual(Union[Literal[Ints.A], Literal[Ints.A]], + Literal[Ints.A]) + self.assertEqual(Union[Literal[Ints.B], Literal[Ints.B]], + Literal[Ints.B]) + + self.assertEqual(Union[Literal[Ints.A], Literal[Ints.B]].__args__, + (Literal[Ints.A], Literal[Ints.B])) + + self.assertEqual(Union[Literal[0], Literal[Ints.A], Literal[False]].__args__, + (Literal[0], Literal[Ints.A], Literal[False])) + self.assertEqual(Union[Literal[1], Literal[Ints.B], Literal[True]].__args__, + (Literal[1], Literal[Ints.B], Literal[True])) + + def test_allow_non_types_in_or(self): + # gh-140348: Test that using | with a Union object allows things that are + # not allowed by is_unionable(). + U1 = Union[int, str] + self.assertEqual(U1 | float, Union[int, str, float]) + self.assertEqual(U1 | "float", Union[int, str, "float"]) + self.assertEqual(float | U1, Union[float, int, str]) + self.assertEqual("float" | U1, Union["float", int, str]) + + +class TupleTests(BaseTestCase): def test_basics(self): with self.assertRaises(TypeError): @@ -476,6 +2355,15 @@ def test_eq_hash(self): self.assertNotEqual(C, Callable[..., int]) self.assertNotEqual(C, Callable) + def test_dir(self): + Callable = self.Callable + dir_items = set(dir(Callable[..., int])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + def test_cannot_instantiate(self): Callable = self.Callable with self.assertRaises(TypeError): @@ -505,13 +2393,13 @@ def test_callable_instance_type_error(self): def f(): pass with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], None]) + isinstance(f, Callable[[], None]) with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], Any]) + isinstance(f, Callable[[], Any]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], None]) + isinstance(None, Callable[[], None]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], Any]) + isinstance(None, Callable[[], Any]) def test_repr(self): Callable = self.Callable @@ -558,14 +2446,29 @@ def test_weakref(self): self.assertEqual(weakref.ref(alias)(), alias) def test_pickle(self): + global T_pickle, P_pickle, TS_pickle # needed for pickling Callable = self.Callable - alias = Callable[[int, str], float] - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - s = pickle.dumps(alias, proto) - loaded = pickle.loads(s) - self.assertEqual(alias.__origin__, loaded.__origin__) - self.assertEqual(alias.__args__, loaded.__args__) - self.assertEqual(alias.__parameters__, loaded.__parameters__) + T_pickle = TypeVar('T_pickle') + P_pickle = ParamSpec('P_pickle') + TS_pickle = TypeVarTuple('TS_pickle') + + samples = [ + Callable[[int, str], float], + Callable[P_pickle, int], + Callable[P_pickle, T_pickle], + Callable[Concatenate[int, P_pickle], int], + Callable[Concatenate[*TS_pickle, P_pickle], int], + ] + for alias in samples: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(alias=alias, proto=proto): + s = pickle.dumps(alias, proto) + loaded = pickle.loads(s) + self.assertEqual(alias.__origin__, loaded.__origin__) + self.assertEqual(alias.__args__, loaded.__args__) + self.assertEqual(alias.__parameters__, loaded.__parameters__) + + del T_pickle, P_pickle, TS_pickle # cleaning up global state def test_var_substitution(self): Callable = self.Callable @@ -574,8 +2477,7 @@ def test_var_substitution(self): C2 = Callable[[KT, T], VT] C3 = Callable[..., T] self.assertEqual(C1[str], Callable[[int, str], str]) - if Callable is typing.Callable: - self.assertEqual(C1[None], Callable[[int, type(None)], type(None)]) + self.assertEqual(C1[None], Callable[[int, type(None)], type(None)]) self.assertEqual(C2[int, float, str], Callable[[int, float], str]) self.assertEqual(C3[int], Callable[..., int]) self.assertEqual(C3[NoReturn], Callable[..., NoReturn]) @@ -592,6 +2494,16 @@ def test_var_substitution(self): self.assertEqual(C5[int, str, float], Callable[[typing.List[int], tuple[str, int], float], int]) + def test_type_subst_error(self): + Callable = self.Callable + P = ParamSpec('P') + T = TypeVar('T') + + pat = "Expected a list of types, an ellipsis, ParamSpec, or Concatenate." + + with self.assertRaisesRegex(TypeError, pat): + Callable[P, T][0, int] + def test_type_erasure(self): Callable = self.Callable class C1(Callable): @@ -649,8 +2561,7 @@ def test_concatenate(self): self.assertEqual(C[[], int], Callable[[int], int]) self.assertEqual(C[Concatenate[str, P2], int], Callable[Concatenate[int, str, P2], int]) - with self.assertRaises(TypeError): - C[..., int] + self.assertEqual(C[..., int], Callable[Concatenate[int, ...], int]) C = Callable[Concatenate[int, P], int] self.assertEqual(repr(C), @@ -661,8 +2572,49 @@ def test_concatenate(self): self.assertEqual(C[[]], Callable[[int], int]) self.assertEqual(C[Concatenate[str, P2]], Callable[Concatenate[int, str, P2], int]) - with self.assertRaises(TypeError): - C[...] + self.assertEqual(C[...], Callable[Concatenate[int, ...], int]) + + def test_nested_paramspec(self): + # Since Callable has some special treatment, we want to be sure + # that substitution works correctly, see gh-103054 + Callable = self.Callable + P = ParamSpec('P') + P2 = ParamSpec('P2') + T = TypeVar('T') + T2 = TypeVar('T2') + Ts = TypeVarTuple('Ts') + class My(Generic[P, T]): + pass + + self.assertEqual(My.__parameters__, (P, T)) + + C1 = My[[int, T2], Callable[P2, T2]] + self.assertEqual(C1.__args__, ((int, T2), Callable[P2, T2])) + self.assertEqual(C1.__parameters__, (T2, P2)) + self.assertEqual(C1[str, [list[int], bytes]], + My[[int, str], Callable[[list[int], bytes], str]]) + + C2 = My[[Callable[[T2], int], list[T2]], str] + self.assertEqual(C2.__args__, ((Callable[[T2], int], list[T2]), str)) + self.assertEqual(C2.__parameters__, (T2,)) + self.assertEqual(C2[list[str]], + My[[Callable[[list[str]], int], list[list[str]]], str]) + + C3 = My[[Callable[P2, T2], T2], T2] + self.assertEqual(C3.__args__, ((Callable[P2, T2], T2), T2)) + self.assertEqual(C3.__parameters__, (P2, T2)) + self.assertEqual(C3[[], int], + My[[Callable[[], int], int], int]) + self.assertEqual(C3[[str, bool], int], + My[[Callable[[str, bool], int], int], int]) + self.assertEqual(C3[[str, bool], T][int], + My[[Callable[[str, bool], int], int], int]) + + C4 = My[[Callable[[int, *Ts, str], T2], T2], T2] + self.assertEqual(C4.__args__, ((Callable[[int, *Ts, str], T2], T2), T2)) + self.assertEqual(C4.__parameters__, (Ts, T2)) + self.assertEqual(C4[bool, bytes, float], + My[[Callable[[int, bool, bytes, str], float], float], float]) def test_errors(self): Callable = self.Callable @@ -676,6 +2628,7 @@ def test_errors(self): with self.assertRaisesRegex(TypeError, "few arguments for"): C1[int] + class TypingCallableTests(BaseCallableTests, BaseTestCase): Callable = typing.Callable @@ -690,20 +2643,6 @@ def test_consistency(self): class CollectionsCallableTests(BaseCallableTests, BaseTestCase): Callable = collections.abc.Callable - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_errors(self): # TODO: RUSTPYTHON, remove when this passes - super().test_errors() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError: 'collections.abc.Callable[__main__.ParamSpec, typing.TypeVar]' != 'collections.abc.Callable[~P, ~T]' - @unittest.expectedFailure - def test_paramspec(self): # TODO: RUSTPYTHON, remove when this passes - super().test_paramspec() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_concatenate(self): # TODO: RUSTPYTHON, remove when this passes - super().test_concatenate() # TODO: RUSTPYTHON, remove when this passes class LiteralTests(BaseTestCase): @@ -718,9 +2657,16 @@ def test_basics(self): Literal[Literal[1, 2], Literal[4, 5]] Literal[b"foo", u"bar"] + def test_enum(self): + import enum + class My(enum.Enum): + A = 'A' + + self.assertEqual(Literal[My.A].__args__, (My.A,)) + def test_illegal_parameters_do_not_raise_runtime_errors(self): # Type checkers should reject these types, but we do not - # raise errors at runtime to maintain maximium flexibility. + # raise errors at runtime to maintain maximum flexibility. Literal[int] Literal[3j + 2, ..., ()] Literal[{"foo": 3, "bar": 4}] @@ -738,6 +2684,14 @@ def test_repr(self): self.assertEqual(repr(Literal[None]), "typing.Literal[None]") self.assertEqual(repr(Literal[1, 2, 3, 3]), "typing.Literal[1, 2, 3]") + def test_dir(self): + dir_items = set(dir(Literal[1, 2, 3])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + def test_cannot_init(self): with self.assertRaises(TypeError): Literal() @@ -799,6 +2753,20 @@ def test_flatten(self): self.assertEqual(l, Literal[1, 2, 3]) self.assertEqual(l.__args__, (1, 2, 3)) + def test_does_not_flatten_enum(self): + import enum + class Ints(enum.IntEnum): + A = 1 + B = 2 + + l = Literal[ + Literal[Ints.A], + Literal[Ints.B], + Literal[1], + Literal[2], + ] + self.assertEqual(l.__args__, (Ints.A, Ints.B, 1, 2)) + XK = TypeVar('XK', str, bytes) XV = TypeVar('XV') @@ -838,6 +2806,7 @@ class Coordinate(Protocol): x: int y: int + @runtime_checkable class Point(Coordinate, Protocol): label: str @@ -883,8 +2852,6 @@ class HasCallProtocol(Protocol): class ProtocolTests(BaseTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_protocol(self): @runtime_checkable class P(Protocol): @@ -907,8 +2874,48 @@ def f(): self.assertNotIsSubclass(types.FunctionType, P) self.assertNotIsInstance(f, P) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_runtime_checkable_generic_non_protocol(self): + # Make sure this doesn't raise AttributeError + with self.assertRaisesRegex( + TypeError, + "@runtime_checkable can be only applied to protocol classes", + ): + @runtime_checkable + class Foo[T]: ... + + def test_runtime_checkable_generic(self): + @runtime_checkable + class Foo[T](Protocol): + def meth(self) -> T: ... + + class Impl: + def meth(self) -> int: ... + + self.assertIsSubclass(Impl, Foo) + + class NotImpl: + def method(self) -> int: ... + + self.assertNotIsSubclass(NotImpl, Foo) + + def test_pep695_generics_can_be_runtime_checkable(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class Bar[T]: + x: T + def __init__(self, x): + self.x = x + + class Capybara[T]: + y: str + def __init__(self, y): + self.y = y + + self.assertIsInstance(Bar(1), HasX) + self.assertNotIsInstance(Capybara('a'), HasX) + def test_everything_implements_empty_protocol(self): @runtime_checkable class Empty(Protocol): @@ -940,10 +2947,10 @@ class BP(Protocol): pass class P(C, Protocol): pass with self.assertRaises(TypeError): - class P(Protocol, C): + class Q(Protocol, C): pass with self.assertRaises(TypeError): - class P(BP, C, Protocol): + class R(BP, C, Protocol): pass class D(BP, C): pass @@ -953,8 +2960,23 @@ class E(C, BP): pass self.assertNotIsInstance(D(), E) self.assertNotIsInstance(E(), D) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_inheritance_from_object(self): + # Inheritance from object is specifically allowed, unlike other nominal classes + class P(Protocol, object): + x: int + + self.assertEqual(typing.get_protocol_members(P), {'x'}) + + class OldGeneric(Protocol, Generic[T], object): + y: T + + self.assertEqual(typing.get_protocol_members(OldGeneric), {'y'}) + + class NewGeneric[T](Protocol, object): + z: T + + self.assertEqual(typing.get_protocol_members(NewGeneric), {'z'}) + def test_no_instantiation(self): class P(Protocol): pass @@ -984,6 +3006,32 @@ class CG(PG[T]): pass with self.assertRaises(TypeError): CG[int](42) + def test_protocol_defining_init_does_not_get_overridden(self): + # check that P.__init__ doesn't get clobbered + # see https://bugs.python.org/issue44807 + + class P(Protocol): + x: int + def __init__(self, x: int) -> None: + self.x = x + class C: pass + + c = C() + P.__init__(c, 1) + self.assertEqual(c.x, 1) + + def test_concrete_class_inheriting_init_from_protocol(self): + class P(Protocol): + x: int + def __init__(self, x: int) -> None: + self.x = x + + class C(P): pass + + c = C(1) + self.assertIsInstance(c, C) + self.assertEqual(c.x, 1) + def test_cannot_instantiate_abstract(self): @runtime_checkable class P(Protocol): @@ -1002,8 +3050,6 @@ def ameth(self) -> int: B() self.assertIsInstance(C(), P) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subprotocols_extending(self): class P1(Protocol): def meth1(self): @@ -1036,8 +3082,6 @@ def meth2(self): self.assertIsInstance(C(), P2) self.assertIsSubclass(C, P2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subprotocols_merging(self): class P1(Protocol): def meth1(self): @@ -1073,107 +3117,589 @@ def meth2(self): self.assertIsInstance(C(), P) self.assertIsSubclass(C, P) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_protocols_issubclass(self): + def test_protocols_issubclass(self): + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + def x(self): ... + + @runtime_checkable + class PG(Protocol[T]): + def x(self): ... + + class BadP(Protocol): + def x(self): ... + + class BadPG(Protocol[T]): + def x(self): ... + + class C: + def x(self): ... + + self.assertIsSubclass(C, P) + self.assertIsSubclass(C, PG) + self.assertIsSubclass(BadP, PG) + + no_subscripted_generics = ( + "Subscripted generics cannot be used with class and instance checks" + ) + + with self.assertRaisesRegex(TypeError, no_subscripted_generics): + issubclass(C, PG[T]) + with self.assertRaisesRegex(TypeError, no_subscripted_generics): + issubclass(C, PG[C]) + + only_runtime_checkable_protocols = ( + "Instance and class checks can only be used with " + "@runtime_checkable protocols" + ) + + with self.assertRaisesRegex(TypeError, only_runtime_checkable_protocols): + issubclass(C, BadP) + with self.assertRaisesRegex(TypeError, only_runtime_checkable_protocols): + issubclass(C, BadPG) + + with self.assertRaisesRegex(TypeError, no_subscripted_generics): + issubclass(P, PG[T]) + with self.assertRaisesRegex(TypeError, no_subscripted_generics): + issubclass(PG, PG[int]) + + only_classes_allowed = r"issubclass\(\) arg 1 must be a class" + + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, P) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, PG) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, BadP) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, BadPG) + + def test_isinstance_against_superproto_doesnt_affect_subproto_instance(self): + @runtime_checkable + class Base(Protocol): + x: int + + @runtime_checkable + class Child(Base, Protocol): + y: str + + class Capybara: + x = 43 + + self.assertIsInstance(Capybara(), Base) + self.assertNotIsInstance(Capybara(), Child) + + def test_implicit_issubclass_between_two_protocols(self): + @runtime_checkable + class CallableMembersProto(Protocol): + def meth(self): ... + + # All the below protocols should be considered "subclasses" + # of CallableMembersProto at runtime, + # even though none of them explicitly subclass CallableMembersProto + + class IdenticalProto(Protocol): + def meth(self): ... + + class SupersetProto(Protocol): + def meth(self): ... + def meth2(self): ... + + class NonCallableMembersProto(Protocol): + meth: Callable[[], None] + + class NonCallableMembersSupersetProto(Protocol): + meth: Callable[[], None] + meth2: Callable[[str, int], bool] + + class MixedMembersProto1(Protocol): + meth: Callable[[], None] + def meth2(self): ... + + class MixedMembersProto2(Protocol): + def meth(self): ... + meth2: Callable[[str, int], bool] + + for proto in ( + IdenticalProto, SupersetProto, NonCallableMembersProto, + NonCallableMembersSupersetProto, MixedMembersProto1, MixedMembersProto2 + ): + with self.subTest(proto=proto.__name__): + self.assertIsSubclass(proto, CallableMembersProto) + + # These two shouldn't be considered subclasses of CallableMembersProto, however, + # since they don't have the `meth` protocol member + + class EmptyProtocol(Protocol): ... + class UnrelatedProtocol(Protocol): + def wut(self): ... + + self.assertNotIsSubclass(EmptyProtocol, CallableMembersProto) + self.assertNotIsSubclass(UnrelatedProtocol, CallableMembersProto) + + # These aren't protocols at all (despite having annotations), + # so they should only be considered subclasses of CallableMembersProto + # if they *actually have an attribute* matching the `meth` member + # (just having an annotation is insufficient) + + class AnnotatedButNotAProtocol: + meth: Callable[[], None] + + class NotAProtocolButAnImplicitSubclass: + def meth(self): pass + + class NotAProtocolButAnImplicitSubclass2: + meth: Callable[[], None] + def meth(self): pass + + class NotAProtocolButAnImplicitSubclass3: + meth: Callable[[], None] + meth2: Callable[[int, str], bool] + def meth(self): pass + def meth2(self, x, y): return True + + self.assertNotIsSubclass(AnnotatedButNotAProtocol, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass2, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass3, CallableMembersProto) + + @unittest.skip("TODO: RUSTPYTHON; (no gc)") + def test_isinstance_checks_not_at_whim_of_gc(self): + self.addCleanup(gc.enable) + gc.disable() + + with self.assertRaisesRegex( + TypeError, + "Protocols can only inherit from other protocols" + ): + class Foo(collections.abc.Mapping, Protocol): + pass + + self.assertNotIsInstance([], collections.abc.Mapping) + + def test_issubclass_and_isinstance_on_Protocol_itself(self): + class C: + def x(self): pass + + self.assertNotIsSubclass(object, Protocol) + self.assertNotIsInstance(object(), Protocol) + + self.assertNotIsSubclass(str, Protocol) + self.assertNotIsInstance('foo', Protocol) + + self.assertNotIsSubclass(C, Protocol) + self.assertNotIsInstance(C(), Protocol) + + only_classes_allowed = r"issubclass\(\) arg 1 must be a class" + + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, Protocol) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass('foo', Protocol) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(C(), Protocol) + + T = TypeVar('T') + + @runtime_checkable + class EmptyProtocol(Protocol): pass + + @runtime_checkable + class SupportsStartsWith(Protocol): + def startswith(self, x: str) -> bool: ... + + @runtime_checkable + class SupportsX(Protocol[T]): + def x(self): ... + + for proto in EmptyProtocol, SupportsStartsWith, SupportsX: + with self.subTest(proto=proto.__name__): + self.assertIsSubclass(proto, Protocol) + + # gh-105237 / PR #105239: + # check that the presence of Protocol subclasses + # where `issubclass(X, <subclass>)` evaluates to True + # doesn't influence the result of `issubclass(X, Protocol)` + + self.assertIsSubclass(object, EmptyProtocol) + self.assertIsInstance(object(), EmptyProtocol) + self.assertNotIsSubclass(object, Protocol) + self.assertNotIsInstance(object(), Protocol) + + self.assertIsSubclass(str, SupportsStartsWith) + self.assertIsInstance('foo', SupportsStartsWith) + self.assertNotIsSubclass(str, Protocol) + self.assertNotIsInstance('foo', Protocol) + + self.assertIsSubclass(C, SupportsX) + self.assertIsInstance(C(), SupportsX) + self.assertNotIsSubclass(C, Protocol) + self.assertNotIsInstance(C(), Protocol) + + def test_protocols_issubclass_non_callable(self): + class C: + x = 1 + + @runtime_checkable + class PNonCall(Protocol): + x = 1 + + non_callable_members_illegal = ( + "Protocols with non-method members don't support issubclass()" + ) + + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(C, PNonCall) + + self.assertIsInstance(C(), PNonCall) + PNonCall.register(C) + + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(C, PNonCall) + + self.assertIsInstance(C(), PNonCall) + + # check that non-protocol subclasses are not affected + class D(PNonCall): ... + + self.assertNotIsSubclass(C, D) + self.assertNotIsInstance(C(), D) + D.register(C) + self.assertIsSubclass(C, D) + self.assertIsInstance(C(), D) + + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(D, PNonCall) + + def test_no_weird_caching_with_issubclass_after_isinstance(self): + @runtime_checkable + class Spam(Protocol): + x: int + + class Eggs: + def __init__(self) -> None: + self.x = 42 + + self.assertIsInstance(Eggs(), Spam) + + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + def test_no_weird_caching_with_issubclass_after_isinstance_2(self): + @runtime_checkable + class Spam(Protocol): + x: int + + class Eggs: ... + + self.assertNotIsInstance(Eggs(), Spam) + + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + def test_no_weird_caching_with_issubclass_after_isinstance_3(self): + @runtime_checkable + class Spam(Protocol): + x: int + + class Eggs: + def __getattr__(self, attr): + if attr == "x": + return 42 + raise AttributeError(attr) + + self.assertNotIsInstance(Eggs(), Spam) + + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + def test_no_weird_caching_with_issubclass_after_isinstance_pep695(self): + @runtime_checkable + class Spam[T](Protocol): + x: T + + class Eggs[T]: + def __init__(self, x: T) -> None: + self.x = x + + self.assertIsInstance(Eggs(42), Spam) + + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + def test_protocols_isinstance(self): + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + def meth(x): ... + + @runtime_checkable + class PG(Protocol[T]): + def meth(x): ... + + @runtime_checkable + class WeirdProto(Protocol): + meth = str.maketrans + + @runtime_checkable + class WeirdProto2(Protocol): + meth = lambda *args, **kwargs: None + + class CustomCallable: + def __call__(self, *args, **kwargs): + pass + + @runtime_checkable + class WeirderProto(Protocol): + meth = CustomCallable() + + class BadP(Protocol): + def meth(x): ... + + class BadPG(Protocol[T]): + def meth(x): ... + + class C: + def meth(x): ... + + class C2: + def __init__(self): + self.meth = lambda: None + + for klass in C, C2: + for proto in P, PG, WeirdProto, WeirdProto2, WeirderProto: + with self.subTest(klass=klass.__name__, proto=proto.__name__): + self.assertIsInstance(klass(), proto) + + no_subscripted_generics = "Subscripted generics cannot be used with class and instance checks" + + with self.assertRaisesRegex(TypeError, no_subscripted_generics): + isinstance(C(), PG[T]) + with self.assertRaisesRegex(TypeError, no_subscripted_generics): + isinstance(C(), PG[C]) + + only_runtime_checkable_msg = ( + "Instance and class checks can only be used " + "with @runtime_checkable protocols" + ) + + with self.assertRaisesRegex(TypeError, only_runtime_checkable_msg): + isinstance(C(), BadP) + with self.assertRaisesRegex(TypeError, only_runtime_checkable_msg): + isinstance(C(), BadPG) + + def test_protocols_isinstance_properties_and_descriptors(self): + class C: + @property + def attr(self): + return 42 + + class CustomDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + class D: + attr = CustomDescriptor() + + # Check that properties set on superclasses + # are still found by the isinstance() logic + class E(C): ... + class F(D): ... + + class Empty: ... + T = TypeVar('T') @runtime_checkable class P(Protocol): - def x(self): ... + @property + def attr(self): ... + + @runtime_checkable + class P1(Protocol): + attr: int @runtime_checkable class PG(Protocol[T]): - def x(self): ... + @property + def attr(self): ... + + @runtime_checkable + class PG1(Protocol[T]): + attr: T + + @runtime_checkable + class MethodP(Protocol): + def attr(self): ... + + @runtime_checkable + class MethodPG(Protocol[T]): + def attr(self) -> T: ... + + for protocol_class in P, P1, PG, PG1, MethodP, MethodPG: + for klass in C, D, E, F: + with self.subTest( + klass=klass.__name__, + protocol_class=protocol_class.__name__ + ): + self.assertIsInstance(klass(), protocol_class) + + with self.subTest(klass="Empty", protocol_class=protocol_class.__name__): + self.assertNotIsInstance(Empty(), protocol_class) class BadP(Protocol): - def x(self): ... + @property + def attr(self): ... + + class BadP1(Protocol): + attr: int class BadPG(Protocol[T]): - def x(self): ... + @property + def attr(self): ... - class C: - def x(self): ... + class BadPG1(Protocol[T]): + attr: T - self.assertIsSubclass(C, P) - self.assertIsSubclass(C, PG) - self.assertIsSubclass(BadP, PG) + cases = ( + PG[T], PG[C], PG1[T], PG1[C], MethodPG[T], + MethodPG[C], BadP, BadP1, BadPG, BadPG1 + ) - with self.assertRaises(TypeError): - issubclass(C, PG[T]) - with self.assertRaises(TypeError): - issubclass(C, PG[C]) - with self.assertRaises(TypeError): - issubclass(C, BadP) - with self.assertRaises(TypeError): - issubclass(C, BadPG) - with self.assertRaises(TypeError): - issubclass(P, PG[T]) - with self.assertRaises(TypeError): - issubclass(PG, PG[int]) + for obj in cases: + for klass in C, D, E, F, Empty: + with self.subTest(klass=klass.__name__, obj=obj): + with self.assertRaises(TypeError): + isinstance(klass(), obj) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_protocols_issubclass_non_callable(self): + def test_protocols_isinstance_not_fooled_by_custom_dir(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class CustomDirWithX: + x = 10 + def __dir__(self): + return [] + + class CustomDirWithoutX: + def __dir__(self): + return ["x"] + + self.assertIsInstance(CustomDirWithX(), HasX) + self.assertNotIsInstance(CustomDirWithoutX(), HasX) + + def test_protocols_isinstance_attribute_access_with_side_effects(self): class C: - x = 1 + @property + def attr(self): + raise AttributeError('no') - @runtime_checkable - class PNonCall(Protocol): - x = 1 + class CustomDescriptor: + def __get__(self, obj, objtype=None): + raise RuntimeError("NO") - with self.assertRaises(TypeError): - issubclass(C, PNonCall) - self.assertIsInstance(C(), PNonCall) - PNonCall.register(C) - with self.assertRaises(TypeError): - issubclass(C, PNonCall) - self.assertIsInstance(C(), PNonCall) + class D: + attr = CustomDescriptor() - # check that non-protocol subclasses are not affected - class D(PNonCall): ... + # Check that properties set on superclasses + # are still found by the isinstance() logic + class E(C): ... + class F(D): ... - self.assertNotIsSubclass(C, D) - self.assertNotIsInstance(C(), D) - D.register(C) - self.assertIsSubclass(C, D) - self.assertIsInstance(C(), D) - with self.assertRaises(TypeError): - issubclass(D, PNonCall) + class WhyWouldYouDoThis: + def __getattr__(self, name): + raise RuntimeError("wut") - def test_protocols_isinstance(self): T = TypeVar('T') @runtime_checkable class P(Protocol): - def meth(x): ... + @property + def attr(self): ... + + @runtime_checkable + class P1(Protocol): + attr: int @runtime_checkable class PG(Protocol[T]): - def meth(x): ... + @property + def attr(self): ... - class BadP(Protocol): - def meth(x): ... + @runtime_checkable + class PG1(Protocol[T]): + attr: T - class BadPG(Protocol[T]): - def meth(x): ... + @runtime_checkable + class MethodP(Protocol): + def attr(self): ... - class C: - def meth(x): ... + @runtime_checkable + class MethodPG(Protocol[T]): + def attr(self) -> T: ... + + for protocol_class in P, P1, PG, PG1, MethodP, MethodPG: + for klass in C, D, E, F: + with self.subTest( + klass=klass.__name__, + protocol_class=protocol_class.__name__ + ): + self.assertIsInstance(klass(), protocol_class) - self.assertIsInstance(C(), P) - self.assertIsInstance(C(), PG) - with self.assertRaises(TypeError): - isinstance(C(), PG[T]) - with self.assertRaises(TypeError): - isinstance(C(), PG[C]) - with self.assertRaises(TypeError): - isinstance(C(), BadP) - with self.assertRaises(TypeError): - isinstance(C(), BadPG) + with self.subTest( + klass="WhyWouldYouDoThis", + protocol_class=protocol_class.__name__ + ): + self.assertNotIsInstance(WhyWouldYouDoThis(), protocol_class) + + def test_protocols_isinstance___slots__(self): + # As per the consensus in https://github.com/python/typing/issues/1367, + # this is desirable behaviour + @runtime_checkable + class HasX(Protocol): + x: int + + class HasNothingButSlots: + __slots__ = ("x",) + + self.assertIsInstance(HasNothingButSlots(), HasX) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_protocols_isinstance_py36(self): class APoint: def __init__(self, x, y, label): @@ -1229,6 +3755,20 @@ def __init__(self, x): self.assertIsInstance(C(1), P) self.assertIsInstance(C(1), PG) + def test_protocols_isinstance_monkeypatching(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class Foo: ... + + f = Foo() + self.assertNotIsInstance(f, HasX) + f.x = 42 + self.assertIsInstance(f, HasX) + del f.x + self.assertNotIsInstance(f, HasX) + def test_protocol_checks_after_subscript(self): class P(Protocol[T]): pass class C(P[T]): pass @@ -1246,8 +3786,6 @@ class D2(C[Any]): pass self.assertIsInstance(D1(), C) self.assertIsSubclass(D2, C) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_protocols_support_register(self): @runtime_checkable class P(Protocol): @@ -1283,8 +3821,6 @@ def __init__(self): self.assertIsInstance(B(), P) self.assertIsInstance(C(), P) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_none_on_callable_blocks_implementation(self): @runtime_checkable class P(Protocol): @@ -1303,8 +3839,6 @@ def __init__(self): self.assertNotIsInstance(B(), P) self.assertNotIsInstance(C(), P) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_non_protocol_subclasses(self): class P(Protocol): x = 1 @@ -1318,10 +3852,10 @@ class NonP(P): class NonPR(PR): pass - class C: + class C(metaclass=abc.ABCMeta): x = 1 - class D: + class D(metaclass=abc.ABCMeta): def meth(self): pass self.assertNotIsInstance(C(), NonP) @@ -1331,6 +3865,33 @@ def meth(self): pass self.assertIsInstance(NonPR(), PR) self.assertIsSubclass(NonPR, PR) + self.assertNotIn("__protocol_attrs__", vars(NonP)) + self.assertNotIn("__protocol_attrs__", vars(NonPR)) + self.assertNotIn("__non_callable_proto_members__", vars(NonP)) + self.assertNotIn("__non_callable_proto_members__", vars(NonPR)) + + self.assertEqual(get_protocol_members(P), {"x"}) + self.assertEqual(get_protocol_members(PR), {"meth"}) + + # the returned object should be immutable, + # and should be a different object to the original attribute + # to prevent users from (accidentally or deliberately) + # mutating the attribute on the original class + self.assertIsInstance(get_protocol_members(P), frozenset) + self.assertIsNot(get_protocol_members(P), P.__protocol_attrs__) + self.assertIsInstance(get_protocol_members(PR), frozenset) + self.assertIsNot(get_protocol_members(PR), P.__protocol_attrs__) + + acceptable_extra_attrs = { + '_is_protocol', '_is_runtime_protocol', '__parameters__', + '__init__', '__annotations__', '__subclasshook__', '__annotate__', + '__annotations_cache__', '__annotate_func__', + } + self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) + self.assertLessEqual( + vars(NonPR).keys(), vars(D).keys() | acceptable_extra_attrs + ) + def test_custom_subclasshook(self): class P(Protocol): x = 1 @@ -1350,15 +3911,68 @@ def __subclasshook__(cls, other): self.assertIsSubclass(OKClass, C) self.assertNotIsSubclass(BadClass, C) + def test_custom_subclasshook_2(self): + @runtime_checkable + class HasX(Protocol): + # The presence of a non-callable member + # would mean issubclass() checks would fail with TypeError + # if it weren't for the custom `__subclasshook__` method + x = 1 + + @classmethod + def __subclasshook__(cls, other): + return hasattr(other, 'x') + + class Empty: pass + + class ImplementsHasX: + x = 1 + + self.assertIsInstance(ImplementsHasX(), HasX) + self.assertNotIsInstance(Empty(), HasX) + self.assertIsSubclass(ImplementsHasX, HasX) + self.assertNotIsSubclass(Empty, HasX) + + # isinstance() and issubclass() checks against this still raise TypeError, + # despite the presence of the custom __subclasshook__ method, + # as it's not decorated with @runtime_checkable + class NotRuntimeCheckable(Protocol): + @classmethod + def __subclasshook__(cls, other): + return hasattr(other, 'x') + + must_be_runtime_checkable = ( + "Instance and class checks can only be used " + "with @runtime_checkable protocols" + ) + + with self.assertRaisesRegex(TypeError, must_be_runtime_checkable): + issubclass(object, NotRuntimeCheckable) + with self.assertRaisesRegex(TypeError, must_be_runtime_checkable): + isinstance(object(), NotRuntimeCheckable) + def test_issubclass_fails_correctly(self): @runtime_checkable - class P(Protocol): + class NonCallableMembers(Protocol): x = 1 + class NotRuntimeCheckable(Protocol): + def callable_member(self) -> int: ... + + @runtime_checkable + class RuntimeCheckable(Protocol): + def callable_member(self) -> int: ... + class C: pass - with self.assertRaises(TypeError): - issubclass(C(), P) + # These three all exercise different code paths, + # but should result in the same error message: + for protocol in NonCallableMembers, NotRuntimeCheckable, RuntimeCheckable: + with self.subTest(proto_name=protocol.__name__): + with self.assertRaisesRegex( + TypeError, r"issubclass\(\) arg 1 must be a class" + ): + issubclass(C(), protocol) def test_defining_generic_protocols(self): T = TypeVar('T') @@ -1380,8 +3994,6 @@ class C(PR[int, T]): pass self.assertIsInstance(C[str](), C) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_defining_generic_protocols_old_style(self): T = TypeVar('T') S = TypeVar('S') @@ -1417,6 +4029,24 @@ def bar(self, x: str) -> str: self.assertIsInstance(Test(), PSub) + def test_pep695_generic_protocol_callable_members(self): + @runtime_checkable + class Foo[T](Protocol): + def meth(self, x: T) -> None: ... + + class Bar[T]: + def meth(self, x: T) -> None: ... + + self.assertIsInstance(Bar(), Foo) + self.assertIsSubclass(Bar, Foo) + + @runtime_checkable + class SupportsTrunc[T](Protocol): + def __trunc__(self) -> T: ... + + self.assertIsInstance(0.0, SupportsTrunc) + self.assertIsSubclass(float, SupportsTrunc) + def test_init_called(self): T = TypeVar('T') @@ -1442,8 +4072,6 @@ class D2(P[T], B): self.assertEqual(D2[int]().test, 'OK') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_new_called(self): T = TypeVar('T') @@ -1467,11 +4095,11 @@ def test_protocols_bad_subscripts(self): with self.assertRaises(TypeError): class P(Protocol[T, T]): pass with self.assertRaises(TypeError): - class P(Protocol[int]): pass + class Q(Protocol[int]): pass with self.assertRaises(TypeError): - class P(Protocol[T], Protocol[S]): pass + class R(Protocol[T], Protocol[S]): pass with self.assertRaises(TypeError): - class P(typing.Mapping[T, S], Protocol[T]): pass + class S(typing.Mapping[T, S], Protocol[T]): pass def test_generic_protocols_repr(self): T = TypeVar('T') @@ -1479,8 +4107,8 @@ def test_generic_protocols_repr(self): class P(Protocol[T, S]): pass - self.assertTrue(repr(P[T, S]).endswith('P[~T, ~S]')) - self.assertTrue(repr(P[int, str]).endswith('P[int, str]')) + self.assertEndsWith(repr(P[T, S]), 'P[~T, ~S]') + self.assertEndsWith(repr(P[int, str]), 'P[int, str]') def test_generic_protocols_eq(self): T = TypeVar('T') @@ -1503,8 +4131,6 @@ class P(Protocol[T]): pass self.assertEqual(P[int].__args__, (int,)) self.assertIs(P[int].__origin__, P) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_generic_protocols_special_from_protocol(self): @runtime_checkable class PR(Protocol): @@ -1522,12 +4148,12 @@ class PG(Protocol[T]): def meth(self): pass - self.assertTrue(P._is_protocol) - self.assertTrue(PR._is_protocol) - self.assertTrue(PG._is_protocol) - self.assertFalse(P._is_runtime_protocol) - self.assertTrue(PR._is_runtime_protocol) - self.assertTrue(PG[int]._is_protocol) + self.assertIs(P._is_protocol, True) + self.assertIs(PR._is_protocol, True) + self.assertIs(PG._is_protocol, True) + self.assertIs(P._is_runtime_protocol, False) + self.assertIs(PR._is_runtime_protocol, True) + self.assertIs(PG[int]._is_protocol, True) self.assertEqual(typing._get_protocol_attrs(P), {'meth'}) self.assertEqual(typing._get_protocol_attrs(PR), {'x'}) self.assertEqual(frozenset(typing._get_protocol_attrs(PG)), @@ -1546,8 +4172,6 @@ class Proto(Protocol): class Concrete(Proto): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_none_treated_correctly(self): @runtime_checkable class P(Protocol): @@ -1574,8 +4198,8 @@ class DI: def __init__(self): self.x = None - self.assertIsInstance(C(), P) - self.assertIsInstance(D(), P) + self.assertIsInstance(CI(), P) + self.assertIsInstance(DI(), P) def test_protocols_in_unions(self): class P(Protocol): @@ -1606,7 +4230,7 @@ class CP(P[int]): self.assertEqual(x.bar, 'abc') self.assertEqual(x.x, 1) self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) - s = pickle.dumps(P) + s = pickle.dumps(P, proto) D = pickle.loads(s) class E: @@ -1614,51 +4238,72 @@ class E: self.assertIsInstance(E(), D) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_runtime_checkable_with_match_args(self): + @runtime_checkable + class P_regular(Protocol): + x: int + y: int + + @runtime_checkable + class P_match(Protocol): + __match_args__ = ('x', 'y') + x: int + y: int + + class Regular: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class WithMatch: + __match_args__ = ('x', 'y', 'z') + def __init__(self, x: int, y: int, z: int): + self.x = x + self.y = y + self.z = z + + class Nope: ... + + self.assertIsInstance(Regular(1, 2), P_regular) + self.assertIsInstance(Regular(1, 2), P_match) + self.assertIsInstance(WithMatch(1, 2, 3), P_regular) + self.assertIsInstance(WithMatch(1, 2, 3), P_match) + self.assertNotIsInstance(Nope(), P_regular) + self.assertNotIsInstance(Nope(), P_match) + def test_supports_int(self): self.assertIsSubclass(int, typing.SupportsInt) self.assertNotIsSubclass(str, typing.SupportsInt) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_supports_float(self): self.assertIsSubclass(float, typing.SupportsFloat) self.assertNotIsSubclass(str, typing.SupportsFloat) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_supports_complex(self): - # Note: complex itself doesn't have __complex__. class C: def __complex__(self): return 0j + self.assertIsSubclass(complex, typing.SupportsComplex) self.assertIsSubclass(C, typing.SupportsComplex) self.assertNotIsSubclass(str, typing.SupportsComplex) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_supports_bytes(self): - # Note: bytes itself doesn't have __bytes__. class B: def __bytes__(self): return b'' + self.assertIsSubclass(bytes, typing.SupportsBytes) self.assertIsSubclass(B, typing.SupportsBytes) self.assertNotIsSubclass(str, typing.SupportsBytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_supports_abs(self): self.assertIsSubclass(float, typing.SupportsAbs) self.assertIsSubclass(int, typing.SupportsAbs) self.assertNotIsSubclass(str, typing.SupportsAbs) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_supports_round(self): issubclass(float, typing.SupportsRound) self.assertIsSubclass(float, typing.SupportsRound) @@ -1669,14 +4314,10 @@ def test_reversible(self): self.assertIsSubclass(list, typing.Reversible) self.assertNotIsSubclass(int, typing.Reversible) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_supports_index(self): self.assertIsSubclass(int, typing.SupportsIndex) self.assertNotIsSubclass(str, typing.SupportsIndex) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bundled_protocol_instance_works(self): self.assertIsInstance(0, typing.SupportsAbs) class C1(typing.SupportsInt): @@ -1687,8 +4328,6 @@ class C2(C1): c = C2() self.assertIsInstance(c, C1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_collections_protocols_allowed(self): @runtime_checkable class Custom(collections.abc.Iterable, Protocol): @@ -1704,14 +4343,70 @@ def close(self): self.assertIsSubclass(B, Custom) self.assertNotIsSubclass(A, Custom) + @runtime_checkable + class ReleasableBuffer(collections.abc.Buffer, Protocol): + def __release_buffer__(self, mv: memoryview) -> None: ... + + class C: pass + class D: + def __buffer__(self, flags: int) -> memoryview: + return memoryview(b'') + def __release_buffer__(self, mv: memoryview) -> None: + pass + + self.assertIsSubclass(D, ReleasableBuffer) + self.assertIsInstance(D(), ReleasableBuffer) + self.assertNotIsSubclass(C, ReleasableBuffer) + self.assertNotIsInstance(C(), ReleasableBuffer) + + def test_io_reader_protocol_allowed(self): + @runtime_checkable + class CustomReader(io.Reader[bytes], Protocol): + def close(self): ... + + class A: pass + class B: + def read(self, sz=-1): + return b"" + def close(self): + pass + + self.assertIsSubclass(B, CustomReader) + self.assertIsInstance(B(), CustomReader) + self.assertNotIsSubclass(A, CustomReader) + self.assertNotIsInstance(A(), CustomReader) + + def test_io_writer_protocol_allowed(self): + @runtime_checkable + class CustomWriter(io.Writer[bytes], Protocol): + def close(self): ... + + class A: pass + class B: + def write(self, b): + pass + def close(self): + pass + + self.assertIsSubclass(B, CustomWriter) + self.assertIsInstance(B(), CustomWriter) + self.assertNotIsSubclass(A, CustomWriter) + self.assertNotIsInstance(A(), CustomWriter) + def test_builtin_protocol_allowlist(self): with self.assertRaises(TypeError): class CustomProtocol(TestCase, Protocol): pass + class CustomPathLikeProtocol(os.PathLike, Protocol): + pass + class CustomContextManager(typing.ContextManager, Protocol): pass + class CustomAsyncIterator(typing.AsyncIterator, Protocol): + pass + def test_non_runtime_protocol_isinstance_check(self): class P(Protocol): x: int @@ -1729,6 +4424,232 @@ def __init__(self): Foo() # Previously triggered RecursionError + def test_get_protocol_members(self): + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(object) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(object()) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(Protocol) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(Generic) + + class P(Protocol): + a: int + def b(self) -> str: ... + @property + def c(self) -> int: ... + + self.assertEqual(get_protocol_members(P), {'a', 'b', 'c'}) + self.assertIsInstance(get_protocol_members(P), frozenset) + self.assertIsNot(get_protocol_members(P), P.__protocol_attrs__) + + class Concrete: + a: int + def b(self) -> str: return "capybara" + @property + def c(self) -> int: return 5 + + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(Concrete) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(Concrete()) + + class ConcreteInherit(P): + a: int = 42 + def b(self) -> str: return "capybara" + @property + def c(self) -> int: return 5 + + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(ConcreteInherit) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(ConcreteInherit()) + + def test_is_protocol(self): + self.assertTrue(is_protocol(Proto)) + self.assertTrue(is_protocol(Point)) + self.assertFalse(is_protocol(Concrete)) + self.assertFalse(is_protocol(Concrete())) + self.assertFalse(is_protocol(Generic)) + self.assertFalse(is_protocol(object)) + + # Protocol is not itself a protocol + self.assertFalse(is_protocol(Protocol)) + + def test_interaction_with_isinstance_checks_on_superclasses_with_ABCMeta(self): + # Ensure the cache is empty, or this test won't work correctly + collections.abc.Sized._abc_registry_clear() + + class Foo(collections.abc.Sized, Protocol): pass + + # gh-105144: this previously raised TypeError + # if a Protocol subclass of Sized had been created + # before any isinstance() checks against Sized + self.assertNotIsInstance(1, collections.abc.Sized) + + def test_interaction_with_isinstance_checks_on_superclasses_with_ABCMeta_2(self): + # Ensure the cache is empty, or this test won't work correctly + collections.abc.Sized._abc_registry_clear() + + class Foo(typing.Sized, Protocol): pass + + # gh-105144: this previously raised TypeError + # if a Protocol subclass of Sized had been created + # before any isinstance() checks against Sized + self.assertNotIsInstance(1, typing.Sized) + + def test_empty_protocol_decorated_with_final(self): + @final + @runtime_checkable + class EmptyProtocol(Protocol): ... + + self.assertIsSubclass(object, EmptyProtocol) + self.assertIsInstance(object(), EmptyProtocol) + + def test_protocol_decorated_with_final_callable_members(self): + @final + @runtime_checkable + class ProtocolWithMethod(Protocol): + def startswith(self, string: str) -> bool: ... + + self.assertIsSubclass(str, ProtocolWithMethod) + self.assertNotIsSubclass(int, ProtocolWithMethod) + self.assertIsInstance('foo', ProtocolWithMethod) + self.assertNotIsInstance(42, ProtocolWithMethod) + + def test_protocol_decorated_with_final_noncallable_members(self): + @final + @runtime_checkable + class ProtocolWithNonCallableMember(Protocol): + x: int + + class Foo: + x = 42 + + only_callable_members_please = ( + r"Protocols with non-method members don't support issubclass()" + ) + + with self.assertRaisesRegex(TypeError, only_callable_members_please): + issubclass(Foo, ProtocolWithNonCallableMember) + + with self.assertRaisesRegex(TypeError, only_callable_members_please): + issubclass(int, ProtocolWithNonCallableMember) + + self.assertIsInstance(Foo(), ProtocolWithNonCallableMember) + self.assertNotIsInstance(42, ProtocolWithNonCallableMember) + + def test_protocol_decorated_with_final_mixed_members(self): + @final + @runtime_checkable + class ProtocolWithMixedMembers(Protocol): + x: int + def method(self) -> None: ... + + class Foo: + x = 42 + def method(self) -> None: ... + + only_callable_members_please = ( + r"Protocols with non-method members don't support issubclass()" + ) + + with self.assertRaisesRegex(TypeError, only_callable_members_please): + issubclass(Foo, ProtocolWithMixedMembers) + + with self.assertRaisesRegex(TypeError, only_callable_members_please): + issubclass(int, ProtocolWithMixedMembers) + + self.assertIsInstance(Foo(), ProtocolWithMixedMembers) + self.assertNotIsInstance(42, ProtocolWithMixedMembers) + + def test_protocol_issubclass_error_message(self): + @runtime_checkable + class Vec2D(Protocol): + x: float + y: float + + def square_norm(self) -> float: + return self.x ** 2 + self.y ** 2 + + self.assertEqual(Vec2D.__protocol_attrs__, {'x', 'y', 'square_norm'}) + expected_error_message = ( + "Protocols with non-method members don't support issubclass()." + " Non-method members: 'x', 'y'." + ) + with self.assertRaisesRegex(TypeError, re.escape(expected_error_message)): + issubclass(int, Vec2D) + + def test_nonruntime_protocol_interaction_with_evil_classproperty(self): + class classproperty: + def __get__(self, instance, type): + raise RuntimeError("NO") + + class Commentable(Protocol): + evil = classproperty() + + # recognised as a protocol attr, + # but not actually accessed by the protocol metaclass + # (which would raise RuntimeError) for non-runtime protocols. + # See gh-113320 + self.assertEqual(get_protocol_members(Commentable), {"evil"}) + + def test_runtime_protocol_interaction_with_evil_classproperty(self): + class CustomError(Exception): pass + + class classproperty: + def __get__(self, instance, type): + raise CustomError + + with self.assertRaises(TypeError) as cm: + @runtime_checkable + class Commentable(Protocol): + evil = classproperty() + + exc = cm.exception + self.assertEqual( + exc.args[0], + "Failed to determine whether protocol member 'evil' is a method member" + ) + self.assertIs(type(exc.__cause__), CustomError) + + def test_isinstance_with_deferred_evaluation_of_annotations(self): + @runtime_checkable + class P(Protocol): + def meth(self): + ... + + class DeferredClass: + x: undefined + + class DeferredClassImplementingP: + x: undefined | int + + def __init__(self): + self.x = 0 + + def meth(self): + ... + + # override meth with a non-method attribute to make it part of __annotations__ instead of __dict__ + class SubProtocol(P, Protocol): + meth: undefined + + + self.assertIsSubclass(SubProtocol, P) + self.assertNotIsInstance(DeferredClass(), P) + self.assertIsInstance(DeferredClassImplementingP(), P) + + def test_deferred_evaluation_of_annotations(self): + class DeferredProto(Protocol): + x: DoesNotExist + self.assertEqual(get_protocol_members(DeferredProto), {"x"}) + self.assertEqual( + annotationlib.get_annotations(DeferredProto, format=annotationlib.Format.STRING), + {'x': 'DoesNotExist'} + ) + class GenericTests(BaseTestCase): @@ -1769,7 +4690,57 @@ class NewGeneric(Generic): ... with self.assertRaises(TypeError): class MyGeneric(Generic[T], Generic[S]): ... with self.assertRaises(TypeError): - class MyGeneric(List[T], Generic[S]): ... + class MyGeneric2(List[T], Generic[S]): ... + with self.assertRaises(TypeError): + Generic[()] + class D(Generic[T]): pass + with self.assertRaises(TypeError): + D[()] + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_generic_init_subclass_not_called_error(self): + notes = ["Note: this exception may have been caused by " + r"'GenericTests.test_generic_init_subclass_not_called_error.<locals>.Base.__init_subclass__' " + "(or the '__init_subclass__' method on a superclass) not calling 'super().__init_subclass__()'"] + + class Base: + def __init_subclass__(cls) -> None: + # Oops, I forgot super().__init_subclass__()! + pass + + with self.subTest(): + class Sub(Base, Generic[T]): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + + with self.subTest(): + class Sub[U](Base): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + + def test_generic_subclass_checks(self): + for typ in [list[int], List[int], + tuple[int, str], Tuple[int, str], + typing.Callable[..., None], + collections.abc.Callable[..., None]]: + with self.subTest(typ=typ): + self.assertRaises(TypeError, issubclass, typ, object) + self.assertRaises(TypeError, issubclass, typ, type) + self.assertRaises(TypeError, issubclass, typ, typ) + self.assertRaises(TypeError, issubclass, object, typ) + + # isinstance is fine: + self.assertTrue(isinstance(typ, object)) + # but, not when the right arg is also a generic: + self.assertRaises(TypeError, isinstance, typ, typ) def test_init(self): T = TypeVar('T') @@ -1826,8 +4797,7 @@ class C(Generic[T]): self.assertNotEqual(Z, Y[int]) self.assertNotEqual(Z, Y[T]) - self.assertTrue(str(Z).endswith( - '.C[typing.Tuple[str, int]]')) + self.assertEndsWith(str(Z), '.C[typing.Tuple[str, int]]') def test_new_repr(self): T = TypeVar('T') @@ -1877,6 +4847,16 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + def test_setattr_exceptions(self): + class Immutable[T]: + def __setattr__(self, key, value): + raise RuntimeError("immutable") + + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `Immutable` instance + # returned by the `Immutable[int]()` call + self.assertIsInstance(Immutable[int](), Immutable) + def test_subscripted_generics_as_proxies(self): T = TypeVar('T') class C(Generic[T]): @@ -2045,13 +5025,12 @@ class A(Generic[T]): self.assertNotEqual(typing.FrozenSet[A[str]], typing.FrozenSet[mod_generics_cache.B.A[str]]) - if sys.version_info[:2] > (3, 2): - self.assertTrue(repr(Tuple[A[str]]).endswith('<locals>.A[str]]')) - self.assertTrue(repr(Tuple[B.A[str]]).endswith('<locals>.B.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) - .endswith('mod_generics_cache.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) - .endswith('mod_generics_cache.B.A[str]]')) + self.assertEndsWith(repr(Tuple[A[str]]), '<locals>.A[str]]') + self.assertEndsWith(repr(Tuple[B.A[str]]), '<locals>.B.A[str]]') + self.assertEndsWith(repr(Tuple[mod_generics_cache.A[str]]), + 'mod_generics_cache.A[str]]') + self.assertEndsWith(repr(Tuple[mod_generics_cache.B.A[str]]), + 'mod_generics_cache.B.A[str]]') def test_extended_generic_rules_eq(self): T = TypeVar('T') @@ -2066,20 +5045,17 @@ def test_extended_generic_rules_eq(self): class Base: ... class Derived(Base): ... self.assertEqual(Union[T, Base][Union[Base, Derived]], Union[Base, Derived]) - with self.assertRaises(TypeError): - Union[T, int][1] - self.assertEqual(Callable[[T], T][KT], Callable[[KT], KT]) self.assertEqual(Callable[..., List[T]][int], Callable[..., List[int]]) def test_extended_generic_rules_repr(self): T = TypeVar('T') self.assertEqual(repr(Union[Tuple, Callable]).replace('typing.', ''), - 'Union[Tuple, Callable]') + 'Tuple | Callable') self.assertEqual(repr(Union[Tuple, Tuple[int]]).replace('typing.', ''), - 'Union[Tuple, Tuple[int]]') + 'Tuple | Tuple[int]') self.assertEqual(repr(Callable[..., Optional[T]][int]).replace('typing.', ''), - 'Callable[..., Optional[int]]') + 'Callable[..., int | None]') self.assertEqual(repr(Callable[[], List[T]][int]).replace('typing.', ''), 'Callable[[], List[int]]') @@ -2109,6 +5085,127 @@ def barfoo(x: AT): ... def barfoo2(x: CT): ... self.assertIs(get_type_hints(barfoo2, globals(), locals())['x'], CT) + def test_generic_pep585_forward_ref(self): + # See https://bugs.python.org/issue41370 + + class C1: + a: list['C1'] + self.assertEqual( + get_type_hints(C1, globals(), locals()), + {'a': list[C1]} + ) + + class C2: + a: dict['C1', list[List[list['C2']]]] + self.assertEqual( + get_type_hints(C2, globals(), locals()), + {'a': dict[C1, list[List[list[C2]]]]} + ) + + # Test stringified annotations + scope = {} + exec(textwrap.dedent(''' + from __future__ import annotations + class C3: + a: List[list["C2"]] + '''), scope) + C3 = scope['C3'] + self.assertEqual(C3.__annotations__['a'], "List[list['C2']]") + self.assertEqual( + get_type_hints(C3, globals(), locals()), + {'a': List[list[C2]]} + ) + + # Test recursive types + X = list["X"] + def f(x: X): ... + self.assertEqual( + get_type_hints(f, globals(), locals()), + {'x': list[list[EqualToForwardRef('X')]]} + ) + + def test_pep695_generic_class_with_future_annotations(self): + original_globals = dict(ann_module695.__dict__) + + hints_for_A = get_type_hints(ann_module695.A) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(hints_for_A["x"], A_type_params[0]) + self.assertEqual(hints_for_A["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(hints_for_A["z"].__args__[0], A_type_params[2]) + + # should not have changed as a result of the get_type_hints() calls! + self.assertEqual(ann_module695.__dict__, original_globals) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + hints_for_B = get_type_hints(ann_module695.B) + self.assertEqual(hints_for_B, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + hints_for_C = get_type_hints(ann_module695.C) + self.assertEqual( + set(hints_for_C.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + hints_for_generic_function = get_type_hints(ann_module695.generic_function) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + hints_for_generic_function.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(hints_for_generic_function["x"], func_t_params[0]) + self.assertEqual(hints_for_generic_function["y"], Unpack[func_t_params[1]]) + self.assertIs(hints_for_generic_function["z"].__origin__, func_t_params[2]) + self.assertIs(hints_for_generic_function["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.generic_function_2).values()), + set(ann_module695.generic_function_2.__type_params__) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + hints_for_generic_method = get_type_hints(ann_module695.D.generic_method) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + hints_for_generic_method, + {"x": params["Foo"], "y": params["Bar"], "return": types.NoneType} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.D.generic_method_2).values()), + set(ann_module695.D.generic_method_2.__type_params__) + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = ann_module695.nested() + + self.assertEqual( + set(results.hints_for_E.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.hints_for_E_meth.values()), + set(results.E.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.hints_for_E_meth.values()), + set(results.E.__type_params__) + ) + self.assertEqual( + set(results.hints_for_E_meth.values()).intersection(results.E.__type_params__), + set() + ) + + self.assertEqual( + set(results.hints_for_generic_func.values()), + set(results.generic_func.__type_params__) + ) + def test_extended_generic_rules_subclassing(self): class T1(Tuple[T, KT]): ... class T2(Tuple[T, ...]): ... @@ -2138,13 +5235,11 @@ def __contains__(self, item): with self.assertRaises(TypeError): issubclass(Tuple[int, ...], typing.Iterable) - def test_fail_with_bare_union(self): + def test_fail_with_special_forms(self): with self.assertRaises(TypeError): - List[Union] + List[Final] with self.assertRaises(TypeError): Tuple[Optional] - with self.assertRaises(TypeError): - ClassVar[ClassVar] with self.assertRaises(TypeError): List[ClassVar[int]] @@ -2170,18 +5265,19 @@ class MyDict(typing.Dict[T, T]): ... class MyDef(typing.DefaultDict[str, T]): ... self.assertIs(MyDef[int]().__class__, MyDef) self.assertEqual(MyDef[int]().__orig_class__, MyDef[int]) - # ChainMap was added in 3.3 - if sys.version_info >= (3, 3): - class MyChain(typing.ChainMap[str, T]): ... - self.assertIs(MyChain[int]().__class__, MyChain) - self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) + class MyChain(typing.ChainMap[str, T]): ... + self.assertIs(MyChain[int]().__class__, MyChain) + self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) for obj in objs: self.assertNotEqual(repr(obj), '') self.assertEqual(obj, obj) - if getattr(obj, '__parameters__', None) and len(obj.__parameters__) == 1: + if (getattr(obj, '__parameters__', None) + and not isinstance(obj, typing.TypeVar) + and isinstance(obj.__parameters__, tuple) + and len(obj.__parameters__) == 1): self.assertEqual(obj[Any].__args__, (Any,)) if isinstance(obj, type): for base in obj.__mro__: @@ -2208,7 +5304,8 @@ class C(B[int]): self.assertEqual(x.bar, 'abc') self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) samples = [Any, Union, Tuple, Callable, ClassVar, - Union[int, str], ClassVar[List], Tuple[int, ...], Callable[[str], bytes], + Union[int, str], ClassVar[List], Tuple[int, ...], Tuple[()], + Callable[[str], bytes], typing.DefaultDict, typing.FrozenSet[int]] for s in samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -2223,37 +5320,60 @@ class C(B[int]): x = pickle.loads(z) self.assertEqual(s, x) + # Test ParamSpec args and kwargs + global PP + PP = ParamSpec('PP') + for thing in [PP.args, PP.kwargs]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(thing, proto)), + thing, + ) + del PP + def test_copy_and_deepcopy(self): T = TypeVar('T') class Node(Generic[T]): ... - things = [Union[T, int], Tuple[T, int], Callable[..., T], Callable[[int], int], + things = [Union[T, int], Tuple[T, int], Tuple[()], + Callable[..., T], Callable[[int], int], Tuple[Any, Any], Node[T], Node[int], Node[Any], typing.Iterable[T], typing.Iterable[Any], typing.Iterable[int], typing.Dict[int, str], typing.Dict[T, Any], ClassVar[int], ClassVar[List[T]], Tuple['T', 'T'], - Union['T', int], List['T'], typing.Mapping['T', int]] - for t in things + [Any]: - self.assertEqual(t, copy(t)) - self.assertEqual(t, deepcopy(t)) + Union['T', int], List['T'], typing.Mapping['T', int], + Union[b"x", b"y"], Any] + for t in things: + with self.subTest(thing=t): + self.assertEqual(t, copy(t)) + self.assertEqual(t, deepcopy(t)) def test_immutability_by_copy_and_pickle(self): # Special forms like Union, Any, etc., generic aliases to containers like List, # Mapping, etc., and type variabcles are considered immutable by copy and pickle. - global TP, TPB, TPV # for pickle + global TP, TPB, TPV, PP # for pickle TP = TypeVar('TP') TPB = TypeVar('TPB', bound=int) TPV = TypeVar('TPV', bytes, str) - for X in [TP, TPB, TPV, List, typing.Mapping, ClassVar, typing.Iterable, + PP = ParamSpec('PP') + for X in [TP, TPB, TPV, PP, + List, typing.Mapping, ClassVar, typing.Iterable, Union, Any, Tuple, Callable]: - self.assertIs(copy(X), X) - self.assertIs(deepcopy(X), X) - self.assertIs(pickle.loads(pickle.dumps(X)), X) + with self.subTest(thing=X): + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertIs(pickle.loads(pickle.dumps(X, proto)), X) + del TP, TPB, TPV, PP + # Check that local type variables are copyable. TL = TypeVar('TL') TLB = TypeVar('TLB', bound=int) TLV = TypeVar('TLV', bytes, str) - for X in [TL, TLB, TLV]: - self.assertIs(copy(X), X) - self.assertIs(deepcopy(X), X) + PL = ParamSpec('PL') + for X in [TL, TLB, TLV, PL]: + with self.subTest(thing=X): + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) def test_copy_generic_instances(self): T = TypeVar('T') @@ -2283,12 +5403,10 @@ def test_weakref_all(self): T = TypeVar('T') things = [Any, Union[T, int], Callable[..., T], Tuple[Any, Any], Optional[List[int]], typing.Mapping[int, str], - typing.re.Match[bytes], typing.Iterable['whatever']] + typing.Match[bytes], typing.Iterable['whatever']] for t in things: self.assertEqual(weakref.ref(t)(), t) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parameterized_slots(self): T = TypeVar('T') class C(Generic[T]): @@ -2308,8 +5426,6 @@ def foo(x: C['C']): ... self.assertEqual(get_type_hints(foo, globals(), locals())['x'], C[C]) self.assertEqual(copy(C[int]), deepcopy(C[int])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parameterized_slots_dict(self): T = TypeVar('T') class D(Generic[T]): @@ -2350,6 +5466,51 @@ class Y(C[int]): self.assertEqual(Y.__qualname__, 'GenericTests.test_repr_2.<locals>.Y') + def test_repr_3(self): + T = TypeVar('T') + T1 = TypeVar('T1') + P = ParamSpec('P') + P2 = ParamSpec('P2') + Ts = TypeVarTuple('Ts') + + class MyCallable(Generic[P, T]): + pass + + class DoubleSpec(Generic[P, P2, T]): + pass + + class TsP(Generic[*Ts, P]): + pass + + object_to_expected_repr = { + MyCallable[P, T]: "MyCallable[~P, ~T]", + MyCallable[Concatenate[T1, P], T]: "MyCallable[typing.Concatenate[~T1, ~P], ~T]", + MyCallable[[], bool]: "MyCallable[[], bool]", + MyCallable[[int], bool]: "MyCallable[[int], bool]", + MyCallable[[int, str], bool]: "MyCallable[[int, str], bool]", + MyCallable[[int, list[int]], bool]: "MyCallable[[int, list[int]], bool]", + MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[Ts], ~P], ~T]", + + DoubleSpec[P2, P, T]: "DoubleSpec[~P2, ~P, ~T]", + DoubleSpec[[int], [str], bool]: "DoubleSpec[[int], [str], bool]", + DoubleSpec[[int, int], [str, str], bool]: "DoubleSpec[[int, int], [str, str], bool]", + + TsP[*Ts, P]: "TsP[typing.Unpack[Ts], ~P]", + TsP[int, str, list[int], []]: "TsP[int, str, list[int], []]", + TsP[int, [str, list[int]]]: "TsP[int, [str, list[int]]]", + + # These lines are just too long to fit: + MyCallable[Concatenate[*Ts, P], int][int, str, [bool, float]]: + "MyCallable[[int, str, bool, float], int]", + } + + for obj, expected_repr in object_to_expected_repr.items(): + with self.subTest(obj=obj, expected_repr=expected_repr): + self.assertRegex( + repr(obj), + fr"^{re.escape(MyCallable.__module__)}.*\.{re.escape(expected_repr)}$", + ) + def test_eq_1(self): self.assertEqual(Generic, Generic) self.assertEqual(Generic[T], Generic[T]) @@ -2387,6 +5548,75 @@ class B(Generic[S]): ... class C(List[int], B): ... self.assertEqual(C.__mro__, (C, list, B, Generic, object)) + def test_multiple_inheritance_non_type_with___mro_entries__(self): + class GoodEntries: + def __mro_entries__(self, bases): + return (object,) + + class A(List[int], GoodEntries()): ... + + self.assertEqual(A.__mro__, (A, list, Generic, object)) + + def test_multiple_inheritance_non_type_without___mro_entries__(self): + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex(TypeError, r"^bases must be types"): + class A(List[int], object()): ... + + def test_multiple_inheritance_non_type_bad___mro_entries__(self): + class BadEntries: + def __mro_entries__(self, bases): + return None + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^__mro_entries__ must return a tuple", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance___mro_entries___returns_non_type(self): + class BadEntries: + def __mro_entries__(self, bases): + return (object(),) + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^bases must be types", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance_with_genericalias(self): + class A(typing.Sized, list[int]): ... + + self.assertEqual( + A.__mro__, + (A, collections.abc.Sized, Generic, list, object), + ) + + def test_multiple_inheritance_with_genericalias_2(self): + T = TypeVar("T") + + class BaseSeq(typing.Sequence[T]): ... + class MySeq(List[T], BaseSeq[T]): ... + + self.assertEqual( + MySeq.__mro__, + ( + MySeq, + list, + BaseSeq, + collections.abc.Sequence, + collections.abc.Reversible, + collections.abc.Collection, + collections.abc.Sized, + collections.abc.Iterable, + collections.abc.Container, + Generic, + object, + ), + ) + def test_init_subclass_super_called(self): class FinalException(Exception): pass @@ -2403,7 +5633,7 @@ class Test(Generic[T], Final): class Subclass(Test): pass with self.assertRaises(FinalException): - class Subclass(Test[int]): + class Subclass2(Test[int]): pass def test_nested(self): @@ -2471,11 +5701,11 @@ class D(C): self.assertEqual(D.__parameters__, ()) - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[int] - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[Any] - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[T] def test_new_with_args(self): @@ -2516,8 +5746,6 @@ def __init__(self, arg): self.assertEqual(c.from_a, 'foo') self.assertEqual(c.from_c, 'foo') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_new_no_args(self): class A(Generic[T]): @@ -2553,406 +5781,435 @@ def test_subclass_special_form(self): for obj in ( ClassVar[int], Final[int], - Union[int, float], - Optional[int], Literal[1, 2], Concatenate[int, ParamSpec("P")], TypeGuard[int], + TypeIs[range], ): with self.subTest(msg=obj): - with self.assertRaisesRegex( - TypeError, f'^{re.escape(f"Cannot subclass {obj!r}")}$' - ): - class Foo(obj): - pass - -class ClassVarTests(BaseTestCase): - - def test_basics(self): - with self.assertRaises(TypeError): - ClassVar[1] - with self.assertRaises(TypeError): - ClassVar[int, str] - with self.assertRaises(TypeError): - ClassVar[int][str] - - def test_repr(self): - self.assertEqual(repr(ClassVar), 'typing.ClassVar') - cv = ClassVar[int] - self.assertEqual(repr(cv), 'typing.ClassVar[int]') - cv = ClassVar[Employee] - self.assertEqual(repr(cv), 'typing.ClassVar[%s.Employee]' % __name__) - - def test_cannot_subclass(self): - with self.assertRaises(TypeError): - class C(type(ClassVar)): - pass - with self.assertRaises(TypeError): - class C(type(ClassVar[int])): - pass - - def test_cannot_init(self): - with self.assertRaises(TypeError): - ClassVar() - with self.assertRaises(TypeError): - type(ClassVar)() - with self.assertRaises(TypeError): - type(ClassVar[Optional[int]])() - - def test_no_isinstance(self): - with self.assertRaises(TypeError): - isinstance(1, ClassVar[int]) - with self.assertRaises(TypeError): - issubclass(int, ClassVar) - -class FinalTests(BaseTestCase): - - def test_basics(self): - Final[int] # OK - with self.assertRaises(TypeError): - Final[1] - with self.assertRaises(TypeError): - Final[int, str] - with self.assertRaises(TypeError): - Final[int][str] - with self.assertRaises(TypeError): - Optional[Final[int]] - - def test_repr(self): - self.assertEqual(repr(Final), 'typing.Final') - cv = Final[int] - self.assertEqual(repr(cv), 'typing.Final[int]') - cv = Final[Employee] - self.assertEqual(repr(cv), 'typing.Final[%s.Employee]' % __name__) - cv = Final[tuple[int]] - self.assertEqual(repr(cv), 'typing.Final[tuple[int]]') - - def test_cannot_subclass(self): - with self.assertRaises(TypeError): - class C(type(Final)): - pass - with self.assertRaises(TypeError): - class C(type(Final[int])): - pass - - def test_cannot_init(self): - with self.assertRaises(TypeError): - Final() - with self.assertRaises(TypeError): - type(Final)() - with self.assertRaises(TypeError): - type(Final[Optional[int]])() - - def test_no_isinstance(self): - with self.assertRaises(TypeError): - isinstance(1, Final[int]) - with self.assertRaises(TypeError): - issubclass(int, Final) - - def test_final_unmodified(self): - def func(x): ... - self.assertIs(func, final(func)) - - -class CastTests(BaseTestCase): - - def test_basics(self): - self.assertEqual(cast(int, 42), 42) - self.assertEqual(cast(float, 42), 42) - self.assertIs(type(cast(float, 42)), int) - self.assertEqual(cast(Any, 42), 42) - self.assertEqual(cast(list, 42), 42) - self.assertEqual(cast(Union[str, float], 42), 42) - self.assertEqual(cast(AnyStr, 42), 42) - self.assertEqual(cast(None, 42), 42) - - def test_errors(self): - # Bogus calls are not expected to fail. - cast(42, 42) - cast('hello', 42) - - -class ForwardRefTests(BaseTestCase): - - def test_basics(self): - - class Node(Generic[T]): - - def __init__(self, label: T): - self.label = label - self.left = self.right = None - - def add_both(self, - left: 'Optional[Node[T]]', - right: 'Node[T]' = None, - stuff: int = None, - blah=None): - self.left = left - self.right = right - - def add_left(self, node: Optional['Node[T]']): - self.add_both(node, None) - - def add_right(self, node: 'Node[T]' = None): - self.add_both(None, node) - - t = Node[int] - both_hints = get_type_hints(t.add_both, globals(), locals()) - self.assertEqual(both_hints['left'], Optional[Node[T]]) - self.assertEqual(both_hints['right'], Optional[Node[T]]) - self.assertEqual(both_hints['left'], both_hints['right']) - self.assertEqual(both_hints['stuff'], Optional[int]) - self.assertNotIn('blah', both_hints) - - left_hints = get_type_hints(t.add_left, globals(), locals()) - self.assertEqual(left_hints['node'], Optional[Node[T]]) - - right_hints = get_type_hints(t.add_right, globals(), locals()) - self.assertEqual(right_hints['node'], Optional[Node[T]]) - - def test_forwardref_instance_type_error(self): - fr = typing.ForwardRef('int') - with self.assertRaises(TypeError): - isinstance(42, fr) - - def test_forwardref_subclass_type_error(self): - fr = typing.ForwardRef('int') - with self.assertRaises(TypeError): - issubclass(int, fr) - - def test_forwardref_only_str_arg(self): - with self.assertRaises(TypeError): - typing.ForwardRef(1) # only `str` type is allowed - - def test_forward_equality(self): - fr = typing.ForwardRef('int') - self.assertEqual(fr, typing.ForwardRef('int')) - self.assertNotEqual(List['int'], List[int]) - self.assertNotEqual(fr, typing.ForwardRef('int', module=__name__)) - frm = typing.ForwardRef('int', module=__name__) - self.assertEqual(frm, typing.ForwardRef('int', module=__name__)) - self.assertNotEqual(frm, typing.ForwardRef('int', module='__other_name__')) - - def test_forward_equality_gth(self): - c1 = typing.ForwardRef('C') - c1_gth = typing.ForwardRef('C') - c2 = typing.ForwardRef('C') - c2_gth = typing.ForwardRef('C') - - class C: - pass - def foo(a: c1_gth, b: c2_gth): - pass + with self.assertRaisesRegex( + TypeError, f'^{re.escape(f"Cannot subclass {obj!r}")}$' + ): + class Foo(obj): + pass - self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': C, 'b': C}) - self.assertEqual(c1, c2) - self.assertEqual(c1, c1_gth) - self.assertEqual(c1_gth, c2_gth) - self.assertEqual(List[c1], List[c1_gth]) - self.assertNotEqual(List[c1], List[C]) - self.assertNotEqual(List[c1_gth], List[C]) - self.assertEqual(Union[c1, c1_gth], Union[c1]) - self.assertEqual(Union[c1, c1_gth, int], Union[c1, int]) - - def test_forward_equality_hash(self): - c1 = typing.ForwardRef('int') - c1_gth = typing.ForwardRef('int') - c2 = typing.ForwardRef('int') - c2_gth = typing.ForwardRef('int') - - def foo(a: c1_gth, b: c2_gth): - pass - get_type_hints(foo, globals(), locals()) + def test_complex_subclasses(self): + T_co = TypeVar("T_co", covariant=True) - self.assertEqual(hash(c1), hash(c2)) - self.assertEqual(hash(c1_gth), hash(c2_gth)) - self.assertEqual(hash(c1), hash(c1_gth)) + class Base(Generic[T_co]): + ... - c3 = typing.ForwardRef('int', module=__name__) - c4 = typing.ForwardRef('int', module='__other_name__') + T = TypeVar("T") - self.assertNotEqual(hash(c3), hash(c1)) - self.assertNotEqual(hash(c3), hash(c1_gth)) - self.assertNotEqual(hash(c3), hash(c4)) - self.assertEqual(hash(c3), hash(typing.ForwardRef('int', module=__name__))) + # see gh-94607: this fails in that bug + class Sub(Base, Generic[T]): + ... - def test_forward_equality_namespace(self): + def test_parameter_detection(self): + self.assertEqual(List[T].__parameters__, (T,)) + self.assertEqual(List[List[T]].__parameters__, (T,)) class A: - pass - def namespace1(): - a = typing.ForwardRef('A') - def fun(x: a): - pass - get_type_hints(fun, globals(), locals()) - return a - - def namespace2(): - a = typing.ForwardRef('A') - - class A: - pass - def fun(x: a): - pass - - get_type_hints(fun, globals(), locals()) - return a + __parameters__ = (T,) + # Bare classes should be skipped + for a in (List, list): + for b in (A, int, TypeVar, TypeVarTuple, ParamSpec, types.GenericAlias, Union): + with self.subTest(generic=a, sub=b): + with self.assertRaisesRegex(TypeError, '.* is not a generic class'): + a[b][str] + # Duck-typing anything that looks like it has __parameters__. + # These tests are optional and failure is okay. + self.assertEqual(List[A()].__parameters__, (T,)) + # C version of GenericAlias + self.assertEqual(list[A()].__parameters__, (T,)) - self.assertEqual(namespace1(), namespace1()) - self.assertNotEqual(namespace1(), namespace2()) - - def test_forward_repr(self): - self.assertEqual(repr(List['int']), "typing.List[ForwardRef('int')]") - - def test_union_forward(self): - - def foo(a: Union['T']): + def test_non_generic_subscript(self): + T = TypeVar('T') + class G(Generic[T]): pass + class A: + __parameters__ = (T,) + + for s in (int, G, A, List, list, + TypeVar, TypeVarTuple, ParamSpec, + types.GenericAlias, Union): + + for t in Tuple, tuple: + with self.subTest(tuple=t, sub=s): + self.assertEqual(t[s, T][int], t[s, int]) + self.assertEqual(t[T, s][int], t[int, s]) + a = t[s] + with self.assertRaises(TypeError): + a[int] + + for c in Callable, collections.abc.Callable: + with self.subTest(callable=c, sub=s): + self.assertEqual(c[[s], T][int], c[[s], int]) + self.assertEqual(c[[T], s][int], c[[int], s]) + a = c[[s], s] + with self.assertRaises(TypeError): + a[int] + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ".+__typing_subst__.+tuple.+int.*" does not match "'TypeAliasType' object is not subscriptable" + def test_return_non_tuple_while_unpacking(self): + # GH-138497: GenericAlias objects didn't ensure that __typing_subst__ actually + # returned a tuple + class EvilTypeVar: + __typing_is_unpacked_typevartuple__ = True + def __typing_prepare_subst__(*_): + return None # any value + def __typing_subst__(*_): + return 42 # not tuple + + evil = EvilTypeVar() + # Create a dummy TypeAlias that will be given the evil generic from + # above. + type type_alias[*_] = 0 + with self.assertRaisesRegex(TypeError, ".+__typing_subst__.+tuple.+int.*"): + type_alias[evil][0] - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': Union[T]}) - - def foo(a: tuple[ForwardRef('T')] | int): - pass - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': tuple[T] | int}) +class ClassVarTests(BaseTestCase): - def test_tuple_forward(self): + def test_basics(self): + with self.assertRaises(TypeError): + ClassVar[int, str] + with self.assertRaises(TypeError): + ClassVar[int][str] - def foo(a: Tuple['T']): - pass + def test_repr(self): + self.assertEqual(repr(ClassVar), 'typing.ClassVar') + cv = ClassVar[int] + self.assertEqual(repr(cv), 'typing.ClassVar[int]') + cv = ClassVar[Employee] + self.assertEqual(repr(cv), 'typing.ClassVar[%s.Employee]' % __name__) - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': Tuple[T]}) + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(ClassVar)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(ClassVar[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.ClassVar'): + class E(ClassVar): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.ClassVar\[int\]'): + class F(ClassVar[int]): + pass - def foo(a: tuple[ForwardRef('T')]): - pass + def test_cannot_init(self): + with self.assertRaises(TypeError): + ClassVar() + with self.assertRaises(TypeError): + type(ClassVar)() + with self.assertRaises(TypeError): + type(ClassVar[Optional[int]])() - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': tuple[T]}) + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, ClassVar[int]) + with self.assertRaises(TypeError): + issubclass(int, ClassVar) - def test_double_forward(self): - def foo(a: 'List[\'int\']'): - pass - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': List[int]}) - def test_forward_recursion_actually(self): - def namespace1(): - a = typing.ForwardRef('A') - A = a - def fun(x: a): pass +class FinalTests(BaseTestCase): - ret = get_type_hints(fun, globals(), locals()) - return a + def test_basics(self): + Final[int] # OK + with self.assertRaises(TypeError): + Final[int, str] + with self.assertRaises(TypeError): + Final[int][str] + with self.assertRaises(TypeError): + Optional[Final[int]] - def namespace2(): - a = typing.ForwardRef('A') - A = a - def fun(x: a): pass + def test_repr(self): + self.assertEqual(repr(Final), 'typing.Final') + cv = Final[int] + self.assertEqual(repr(cv), 'typing.Final[int]') + cv = Final[Employee] + self.assertEqual(repr(cv), 'typing.Final[%s.Employee]' % __name__) + cv = Final[tuple[int]] + self.assertEqual(repr(cv), 'typing.Final[tuple[int]]') - ret = get_type_hints(fun, globals(), locals()) - return a + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(Final)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(Final[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Final'): + class E(Final): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Final\[int\]'): + class F(Final[int]): + pass - def cmp(o1, o2): - return o1 == o2 + def test_cannot_init(self): + with self.assertRaises(TypeError): + Final() + with self.assertRaises(TypeError): + type(Final)() + with self.assertRaises(TypeError): + type(Final[Optional[int]])() - r1 = namespace1() - r2 = namespace2() - self.assertIsNot(r1, r2) - self.assertRaises(RecursionError, cmp, r1, r2) + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, Final[int]) + with self.assertRaises(TypeError): + issubclass(int, Final) - def test_union_forward_recursion(self): - ValueList = List['Value'] - Value = Union[str, ValueList] - class C: - foo: List[Value] - class D: - foo: Union[Value, ValueList] - class E: - foo: Union[List[Value], ValueList] - class F: - foo: Union[Value, List[Value], ValueList] +class FinalDecoratorTests(BaseTestCase): + def test_final_unmodified(self): + def func(x): ... + self.assertIs(func, final(func)) - self.assertEqual(get_type_hints(C, globals(), locals()), get_type_hints(C, globals(), locals())) - self.assertEqual(get_type_hints(C, globals(), locals()), - {'foo': List[Union[str, List[Union[str, List['Value']]]]]}) - self.assertEqual(get_type_hints(D, globals(), locals()), - {'foo': Union[str, List[Union[str, List['Value']]]]}) - self.assertEqual(get_type_hints(E, globals(), locals()), - {'foo': Union[ - List[Union[str, List[Union[str, List['Value']]]]], - List[Union[str, List['Value']]] - ] - }) - self.assertEqual(get_type_hints(F, globals(), locals()), - {'foo': Union[ - str, - List[Union[str, List['Value']]], - List[Union[str, List[Union[str, List['Value']]]]] - ] - }) + def test_dunder_final(self): + @final + def func(): ... + @final + class Cls: ... + self.assertIs(True, func.__final__) + self.assertIs(True, Cls.__final__) + + class Wrapper: + __slots__ = ("func",) + def __init__(self, func): + self.func = func + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + # Check that no error is thrown if the attribute + # is not writable. + @final + @Wrapper + def wrapped(): ... + self.assertIsInstance(wrapped, Wrapper) + self.assertNotHasAttr(wrapped, "__final__") + + class Meta(type): + @property + def __final__(self): return "can't set me" + @final + class WithMeta(metaclass=Meta): ... + self.assertEqual(WithMeta.__final__, "can't set me") + + # Builtin classes throw TypeError if you try to set an + # attribute. + final(int) + self.assertNotHasAttr(int, "__final__") + + # Make sure it works with common builtin decorators + class Methods: + @final + @classmethod + def clsmethod(cls): ... - def test_callable_forward(self): + @final + @staticmethod + def stmethod(): ... - def foo(a: Callable[['T'], 'T']): - pass + # The other order doesn't work because property objects + # don't allow attribute assignment. + @property + @final + def prop(self): ... - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': Callable[[T], T]}) + @final + @lru_cache() + def cached(self): ... - def test_callable_with_ellipsis_forward(self): + # Use getattr_static because the descriptor returns the + # underlying function, which doesn't have __final__. + self.assertIs( + True, + inspect.getattr_static(Methods, "clsmethod").__final__ + ) + self.assertIs( + True, + inspect.getattr_static(Methods, "stmethod").__final__ + ) + self.assertIs(True, Methods.prop.fget.__final__) + self.assertIs(True, Methods.cached.__final__) - def foo(a: 'Callable[..., T]'): - pass - self.assertEqual(get_type_hints(foo, globals(), locals()), - {'a': Callable[..., T]}) +class OverrideDecoratorTests(BaseTestCase): + def test_override(self): + class Base: + def normal_method(self): ... + @classmethod + def class_method_good_order(cls): ... + @classmethod + def class_method_bad_order(cls): ... + @staticmethod + def static_method_good_order(): ... + @staticmethod + def static_method_bad_order(): ... + + class Derived(Base): + @override + def normal_method(self): + return 42 - def test_special_forms_forward(self): + @classmethod + @override + def class_method_good_order(cls): + return 42 + @override + @classmethod + def class_method_bad_order(cls): + return 42 - class C: - a: Annotated['ClassVar[int]', (3, 5)] = 4 - b: Annotated['Final[int]', "const"] = 4 + @staticmethod + @override + def static_method_good_order(): + return 42 + @override + @staticmethod + def static_method_bad_order(): + return 42 - class CF: - b: List['Final[int]'] = 4 + self.assertIsSubclass(Derived, Base) + instance = Derived() + self.assertEqual(instance.normal_method(), 42) + self.assertIs(True, Derived.normal_method.__override__) + self.assertIs(True, instance.normal_method.__override__) + + self.assertEqual(Derived.class_method_good_order(), 42) + self.assertIs(True, Derived.class_method_good_order.__override__) + self.assertEqual(Derived.class_method_bad_order(), 42) + self.assertNotHasAttr(Derived.class_method_bad_order, "__override__") + + self.assertEqual(Derived.static_method_good_order(), 42) + self.assertIs(True, Derived.static_method_good_order.__override__) + self.assertEqual(Derived.static_method_bad_order(), 42) + self.assertNotHasAttr(Derived.static_method_bad_order, "__override__") + + # Base object is not changed: + self.assertNotHasAttr(Base.normal_method, "__override__") + self.assertNotHasAttr(Base.class_method_good_order, "__override__") + self.assertNotHasAttr(Base.class_method_bad_order, "__override__") + self.assertNotHasAttr(Base.static_method_good_order, "__override__") + self.assertNotHasAttr(Base.static_method_bad_order, "__override__") + + def test_property(self): + class Base: + @property + def correct(self) -> int: + return 1 + @property + def wrong(self) -> int: + return 1 + + class Child(Base): + @property + @override + def correct(self) -> int: + return 2 + @override + @property + def wrong(self) -> int: + return 2 + + instance = Child() + self.assertEqual(instance.correct, 2) + self.assertIs(Child.correct.fget.__override__, True) + self.assertEqual(instance.wrong, 2) + self.assertNotHasAttr(Child.wrong, "__override__") + self.assertNotHasAttr(Child.wrong.fset, "__override__") + + def test_silent_failure(self): + class CustomProp: + __slots__ = ('fget',) + def __init__(self, fget): + self.fget = fget + def __get__(self, obj, objtype=None): + return self.fget(obj) + + class WithOverride: + @override # must not fail on object with `__slots__` + @CustomProp + def some(self): + return 1 + + self.assertEqual(WithOverride.some, 1) + self.assertNotHasAttr(WithOverride.some, "__override__") + + def test_multiple_decorators(self): + def with_wraps(f): # similar to `lru_cache` definition + @wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + return wrapper + + class WithOverride: + @override + @with_wraps + def on_top(self, a: int) -> int: + return a + 1 + @with_wraps + @override + def on_bottom(self, a: int) -> int: + return a + 2 + + instance = WithOverride() + self.assertEqual(instance.on_top(1), 2) + self.assertIs(instance.on_top.__override__, True) + self.assertEqual(instance.on_bottom(1), 3) + self.assertIs(instance.on_bottom.__override__, True) - self.assertEqual(get_type_hints(C, globals())['a'], ClassVar[int]) - self.assertEqual(get_type_hints(C, globals())['b'], Final[int]) - with self.assertRaises(TypeError): - get_type_hints(CF, globals()), - def test_syntax_error(self): +class CastTests(BaseTestCase): - with self.assertRaises(SyntaxError): - Generic['/T'] + def test_basics(self): + self.assertEqual(cast(int, 42), 42) + self.assertEqual(cast(float, 42), 42) + self.assertIs(type(cast(float, 42)), int) + self.assertEqual(cast(Any, 42), 42) + self.assertEqual(cast(list, 42), 42) + self.assertEqual(cast(Union[str, float], 42), 42) + self.assertEqual(cast(AnyStr, 42), 42) + self.assertEqual(cast(None, 42), 42) - def test_delayed_syntax_error(self): + def test_errors(self): + # Bogus calls are not expected to fail. + cast(42, 42) + cast('hello', 42) - def foo(a: 'Node[T'): - pass - with self.assertRaises(SyntaxError): - get_type_hints(foo) +class AssertTypeTests(BaseTestCase): - def test_type_error(self): + def test_basics(self): + arg = 42 + self.assertIs(assert_type(arg, int), arg) + self.assertIs(assert_type(arg, str | float), arg) + self.assertIs(assert_type(arg, AnyStr), arg) + self.assertIs(assert_type(arg, None), arg) - def foo(a: Tuple['42']): - pass + def test_errors(self): + # Bogus calls are not expected to fail. + arg = 42 + self.assertIs(assert_type(arg, 42), arg) + self.assertIs(assert_type(arg, 'hello'), arg) - with self.assertRaises(TypeError): - get_type_hints(foo) - def test_name_error(self): +# We need this to make sure that `@no_type_check` respects `__module__` attr: +@no_type_check +class NoTypeCheck_Outer: + Inner = ann_module8.NoTypeCheck_Outer.Inner - def foo(a: 'Noode[T]'): - pass +@no_type_check +class NoTypeCheck_WithFunction: + NoTypeCheck_function = ann_module8.NoTypeCheck_function - with self.assertRaises(NameError): - get_type_hints(foo, locals()) +class NoTypeCheckTests(BaseTestCase): def test_no_type_check(self): @no_type_check @@ -2980,9 +6237,98 @@ def meth(self, x: int): ... @no_type_check class D(C): c = C + # verify that @no_type_check never affects bases self.assertEqual(get_type_hints(C.meth), {'x': int}) + # and never child classes: + class Child(D): + def foo(self, x: int): ... + + self.assertEqual(get_type_hints(Child.foo), {'x': int}) + + def test_no_type_check_nested_types(self): + # See https://bugs.python.org/issue46571 + class Other: + o: int + class B: # Has the same `__name__`` as `A.B` and different `__qualname__` + o: int + @no_type_check + class A: + a: int + class B: + b: int + class C: + c: int + class D: + d: int + + Other = Other + + for klass in [A, A.B, A.B.C, A.D]: + with self.subTest(klass=klass): + self.assertIs(klass.__no_type_check__, True) + self.assertEqual(get_type_hints(klass), {}) + + for not_modified in [Other, B]: + with self.subTest(not_modified=not_modified): + with self.assertRaises(AttributeError): + not_modified.__no_type_check__ + self.assertNotEqual(get_type_hints(not_modified), {}) + + def test_no_type_check_class_and_static_methods(self): + @no_type_check + class Some: + @staticmethod + def st(x: int) -> int: ... + @classmethod + def cl(cls, y: int) -> int: ... + + self.assertIs(Some.st.__no_type_check__, True) + self.assertEqual(get_type_hints(Some.st), {}) + self.assertIs(Some.cl.__no_type_check__, True) + self.assertEqual(get_type_hints(Some.cl), {}) + + def test_no_type_check_other_module(self): + self.assertIs(NoTypeCheck_Outer.__no_type_check__, True) + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_Outer.__no_type_check__ + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_Outer.Inner.__no_type_check__ + + self.assertIs(NoTypeCheck_WithFunction.__no_type_check__, True) + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_function.__no_type_check__ + + def test_no_type_check_foreign_functions(self): + # We should not modify this function: + def some(*args: int) -> int: + ... + + @no_type_check + class A: + some_alias = some + some_class = classmethod(some) + some_static = staticmethod(some) + + with self.assertRaises(AttributeError): + some.__no_type_check__ + self.assertEqual(get_type_hints(some), {'args': int, 'return': int}) + + def test_no_type_check_lambda(self): + @no_type_check + class A: + # Corner case: `lambda` is both an assignment and a function: + bar: Callable[[int], int] = lambda arg: arg + + self.assertIs(A.bar.__no_type_check__, True) + self.assertEqual(get_type_hints(A.bar), {}) + + def test_no_type_check_TypeError(self): + # This simply should not fail with + # `TypeError: can't set attributes of built-in/extension type 'dict'` + no_type_check(dict) + def test_no_type_check_forward_ref_as_string(self): class C: foo: typing.ClassVar[int] = 7 @@ -2997,21 +6343,15 @@ class F: for clazz in [C, D, E, F]: self.assertEqual(get_type_hints(clazz), expected_result) - def test_nested_classvar_fails_forward_ref_check(self): - class E: - foo: 'typing.ClassVar[typing.ClassVar[int]]' = 7 - class F: - foo: ClassVar['ClassVar[int]'] = 7 - - for clazz in [E, F]: - with self.assertRaises(TypeError): - get_type_hints(clazz) - def test_meta_no_type_check(self): - - @no_type_check_decorator - def magic_decorator(func): - return func + depr_msg = ( + "'typing.no_type_check_decorator' is deprecated " + "and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, depr_msg): + @no_type_check_decorator + def magic_decorator(func): + return func self.assertEqual(magic_decorator.__name__, 'magic_decorator') @@ -3032,29 +6372,70 @@ def foo(a: 'whatevers') -> {}: ith = get_type_hints(C().foo) self.assertEqual(ith, {}) - def test_default_globals(self): - code = ("class C:\n" - " def foo(self, a: 'C') -> 'D': pass\n" - "class D:\n" - " def bar(self, b: 'D') -> C: pass\n" - ) - ns = {} - exec(code, ns) - hints = get_type_hints(ns['C'].foo) - self.assertEqual(hints, {'a': ns['C'], 'return': ns['D']}) - def test_final_forward_ref(self): - self.assertEqual(gth(Loop, globals())['attr'], Final[Loop]) - self.assertNotEqual(gth(Loop, globals())['attr'], Final[int]) - self.assertNotEqual(gth(Loop, globals())['attr'], Final) +class InternalsTests(BaseTestCase): + def test_deprecation_for_no_type_params_passed_to__evaluate(self): + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing._eval_type' is deprecated" + ) + ) as cm: + self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) + + self.assertEqual(cm.filename, __file__) + + f = ForwardRef("int") + + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.ForwardRef._evaluate' is deprecated" + ) + ) as cm: + self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) + + self.assertEqual(cm.filename, __file__) + + def test_collect_parameters(self): + typing = import_helper.import_fresh_module("typing") + with self.assertWarnsRegex( + DeprecationWarning, + "The private _collect_parameters function is deprecated" + ) as cm: + typing._collect_parameters + self.assertEqual(cm.filename, __file__) + + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports("typing", { + "warnings", + "inspect", + "re", + "contextlib", + "annotationlib", + }) + + +@lru_cache() +def cached_func(x, y): + return 3 * x + y + + +class MethodHolder: + @classmethod + def clsmethod(cls): ... + @staticmethod + def stmethod(): ... + def method(self): ... class OverloadTests(BaseTestCase): def test_overload_fails(self): - from typing import overload - - with self.assertRaises(RuntimeError): + with self.assertRaises(NotImplementedError): @overload def blah(): @@ -3063,8 +6444,6 @@ def blah(): blah() def test_overload_succeeds(self): - from typing import overload - @overload def blah(): pass @@ -3074,9 +6453,76 @@ def blah(): blah() + @cpython_only # gh-98713 + def test_overload_on_compiled_functions(self): + with patch("typing._overload_registry", + defaultdict(lambda: defaultdict(dict))): + # The registry starts out empty: + self.assertEqual(typing._overload_registry, {}) + + # This should just not fail: + overload(sum) + overload(print) + + # No overloads are recorded (but, it still has a side-effect): + self.assertEqual(typing.get_overloads(sum), []) + self.assertEqual(typing.get_overloads(print), []) + + def set_up_overloads(self): + def blah(): + pass + + overload1 = blah + overload(blah) + + def blah(): + pass + + overload2 = blah + overload(blah) + + def blah(): + pass + + return blah, [overload1, overload2] + + # Make sure we don't clear the global overload registry + @patch("typing._overload_registry", + defaultdict(lambda: defaultdict(dict))) + def test_overload_registry(self): + # The registry starts out empty + self.assertEqual(typing._overload_registry, {}) + + impl, overloads = self.set_up_overloads() + self.assertNotEqual(typing._overload_registry, {}) + self.assertEqual(list(get_overloads(impl)), overloads) + + def some_other_func(): pass + overload(some_other_func) + other_overload = some_other_func + def some_other_func(): pass + self.assertEqual(list(get_overloads(some_other_func)), [other_overload]) + # Unrelated function still has no overloads: + def not_overloaded(): pass + self.assertEqual(list(get_overloads(not_overloaded)), []) + + # Make sure that after we clear all overloads, the registry is + # completely empty. + clear_overloads() + self.assertEqual(typing._overload_registry, {}) + self.assertEqual(get_overloads(impl), []) + + # Querying a function with no overloads shouldn't change the registry. + def the_only_one(): pass + self.assertEqual(get_overloads(the_only_one), []) + self.assertEqual(typing._overload_registry, {}) + + def test_overload_registry_repeated(self): + for _ in range(2): + impl, overloads = self.set_up_overloads() + + self.assertEqual(list(get_overloads(impl)), overloads) -ASYNCIO_TESTS = """ -import asyncio T_a = TypeVar('T_a') @@ -3109,19 +6555,6 @@ async def __aenter__(self) -> int: return 42 async def __aexit__(self, etype, eval, tb): return None -""" - -try: - exec(ASYNCIO_TESTS) -except ImportError: - ASYNCIO = False # multithreading is not enabled -else: - ASYNCIO = True - -# Definitions needed for features introduced in Python 3.6 - -from test import ann_module, ann_module2, ann_module3, ann_module5, ann_module6 -from typing import AsyncContextManager class A: y: float @@ -3168,20 +6601,59 @@ class Point2D(TypedDict): x: int y: int +class Point2DGeneric(Generic[T], TypedDict): + a: T + b: T + class Bar(_typed_dict_helper.Foo, total=False): b: int +class BarGeneric(_typed_dict_helper.FooGeneric[T], total=False): + b: int + class LabelPoint2D(Point2D, Label): ... class Options(TypedDict, total=False): log_level: int log_path: str +class TotalMovie(TypedDict): + title: str + year: NotRequired[int] + +class NontotalMovie(TypedDict, total=False): + title: Required[str] + year: int + +class ParentNontotalMovie(TypedDict, total=False): + title: Required[str] + +class ChildTotalMovie(ParentNontotalMovie): + year: NotRequired[int] + +class ParentDeeplyAnnotatedMovie(TypedDict): + title: Annotated[Annotated[Required[str], "foobar"], "another level"] + +class ChildDeeplyAnnotatedMovie(ParentDeeplyAnnotatedMovie): + year: NotRequired[Annotated[int, 2000]] + +class AnnotatedMovie(TypedDict): + title: Annotated[Required[str], "foobar"] + year: NotRequired[Annotated[int, 2000]] + +class DeeplyAnnotatedMovie(TypedDict): + title: Annotated[Annotated[Required[str], "foobar"], "another level"] + year: NotRequired[Annotated[int, 2000]] + +class WeirdlyQuotedMovie(TypedDict): + title: Annotated['Annotated[Required[str], "foobar"]', "another level"] + year: NotRequired['Annotated[int, 2000]'] + class HasForeignBaseClass(mod_generics_cache.A): some_xrepr: 'XRepr' other_a: 'mod_generics_cache.A' -async def g_with(am: AsyncContextManager[int]): +async def g_with(am: typing.AsyncContextManager[int]): x: int async with am as x: return x @@ -3204,7 +6676,7 @@ def nested(self: 'ForRefExample'): pass -class GetTypeHintTests(BaseTestCase): +class GetTypeHintsTests(BaseTestCase): def test_get_type_hints_from_various_objects(self): # For invalid objects should fail with TypeError (not AttributeError etc). with self.assertRaises(TypeError): @@ -3214,10 +6686,8 @@ def test_get_type_hints_from_various_objects(self): with self.assertRaises(TypeError): gth(None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_type_hints_modules(self): - ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} + ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) @@ -3235,7 +6705,7 @@ def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) - self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) + self.assertEqual(gth(ann_module.M), {'o': type}) self.assertEqual(gth(ann_module.D), {'j': str, 'k': str, 'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.Y), {'z': int}) @@ -3266,8 +6736,8 @@ def test_respect_no_type_check(self): class NoTpCheck: class Inn: def __init__(self, x: 'not a type'): ... - self.assertTrue(NoTpCheck.__no_type_check__) - self.assertTrue(NoTpCheck.Inn.__init__.__no_type_check__) + self.assertIs(NoTpCheck.__no_type_check__, True) + self.assertIs(NoTpCheck.Inn.__init__.__no_type_check__, True) self.assertEqual(gth(ann_module2.NTC.meth), {}) class ABase(Generic[T]): def meth(x: int): ... @@ -3326,122 +6796,413 @@ def foobar(x: List['X']): ... {'x': List[Annotated[int, (1, 10)]]} ) - def foobar(x: list[ForwardRef('X')]): ... - X = Annotated[int, (1, 10)] - self.assertEqual( - get_type_hints(foobar, globals(), locals()), - {'x': list[int]} - ) - self.assertEqual( - get_type_hints(foobar, globals(), locals(), include_extras=True), - {'x': list[Annotated[int, (1, 10)]]} - ) + def foobar(x: list[ForwardRef('X')]): ... + X = Annotated[int, (1, 10)] + self.assertEqual( + get_type_hints(foobar, globals(), locals()), + {'x': list[int]} + ) + self.assertEqual( + get_type_hints(foobar, globals(), locals(), include_extras=True), + {'x': list[Annotated[int, (1, 10)]]} + ) + + BA = Tuple[Annotated[T, (1, 0)], ...] + def barfoo(x: BA): ... + self.assertEqual(get_type_hints(barfoo, globals(), locals())['x'], Tuple[T, ...]) + self.assertEqual( + get_type_hints(barfoo, globals(), locals(), include_extras=True)['x'], + BA + ) + + BA = tuple[Annotated[T, (1, 0)], ...] + def barfoo(x: BA): ... + self.assertEqual(get_type_hints(barfoo, globals(), locals())['x'], tuple[T, ...]) + self.assertEqual( + get_type_hints(barfoo, globals(), locals(), include_extras=True)['x'], + BA + ) + + def barfoo2(x: typing.Callable[..., Annotated[List[T], "const"]], + y: typing.Union[int, Annotated[T, "mutable"]]): ... + self.assertEqual( + get_type_hints(barfoo2, globals(), locals()), + {'x': typing.Callable[..., List[T]], 'y': typing.Union[int, T]} + ) + + BA2 = typing.Callable[..., List[T]] + def barfoo3(x: BA2): ... + self.assertIs( + get_type_hints(barfoo3, globals(), locals(), include_extras=True)["x"], + BA2 + ) + BA3 = typing.Annotated[int | float, "const"] + def barfoo4(x: BA3): ... + self.assertEqual( + get_type_hints(barfoo4, globals(), locals()), + {"x": int | float} + ) + self.assertEqual( + get_type_hints(barfoo4, globals(), locals(), include_extras=True), + {"x": typing.Annotated[int | float, "const"]} + ) + + def test_get_type_hints_annotated_in_union(self): # bpo-46603 + def with_union(x: int | list[Annotated[str, 'meta']]): ... + + self.assertEqual(get_type_hints(with_union), {'x': int | list[str]}) + self.assertEqual( + get_type_hints(with_union, include_extras=True), + {'x': int | list[Annotated[str, 'meta']]}, + ) + + def test_get_type_hints_annotated_refs(self): + + Const = Annotated[T, "Const"] + + class MySet(Generic[T]): + + def __ior__(self, other: "Const[MySet[T]]") -> "MySet[T]": + ... + + def __iand__(self, other: Const["MySet[T]"]) -> "MySet[T]": + ... + + self.assertEqual( + get_type_hints(MySet.__iand__, globals(), locals()), + {'other': MySet[T], 'return': MySet[T]} + ) + + self.assertEqual( + get_type_hints(MySet.__iand__, globals(), locals(), include_extras=True), + {'other': Const[MySet[T]], 'return': MySet[T]} + ) + + self.assertEqual( + get_type_hints(MySet.__ior__, globals(), locals()), + {'other': MySet[T], 'return': MySet[T]} + ) + + def test_get_type_hints_annotated_with_none_default(self): + # See: https://bugs.python.org/issue46195 + def annotated_with_none_default(x: Annotated[int, 'data'] = None): ... + self.assertEqual( + get_type_hints(annotated_with_none_default), + {'x': int}, + ) + self.assertEqual( + get_type_hints(annotated_with_none_default, include_extras=True), + {'x': Annotated[int, 'data']}, + ) + + def test_get_type_hints_classes_str_annotations(self): + class Foo: + y = str + x: 'y' + # This previously raised an error under PEP 563. + self.assertEqual(get_type_hints(Foo), {'x': str}) + + def test_get_type_hints_bad_module(self): + # bpo-41515 + class BadModule: + pass + BadModule.__module__ = 'bad' # Something not in sys.modules + self.assertNotIn('bad', sys.modules) + self.assertEqual(get_type_hints(BadModule), {}) + + def test_get_type_hints_annotated_bad_module(self): + # See https://bugs.python.org/issue44468 + class BadBase: + foo: tuple + class BadType(BadBase): + bar: list + BadType.__module__ = BadBase.__module__ = 'bad' + self.assertNotIn('bad', sys.modules) + self.assertEqual(get_type_hints(BadType), {'foo': tuple, 'bar': list}) + + def test_forward_ref_and_final(self): + # https://bugs.python.org/issue45166 + hints = get_type_hints(ann_module5) + self.assertEqual(hints, {'name': Final[str]}) + + hints = get_type_hints(ann_module5.MyClass) + self.assertEqual(hints, {'value': Final}) + + def test_top_level_class_var(self): + # This is not meaningful but we don't raise for it. + # https://github.com/python/cpython/issues/133959 + hints = get_type_hints(ann_module6) + self.assertEqual(hints, {'wrong': ClassVar[int]}) + + def test_get_type_hints_typeddict(self): + self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(TotalMovie, include_extras=True), { + 'title': str, + 'year': NotRequired[int], + }) + + self.assertEqual(get_type_hints(AnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(AnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(WeirdlyQuotedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(WeirdlyQuotedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated), {'a': int}) + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated, include_extras=True), { + 'a': Annotated[Required[int], "a", "b", "c"] + }) + + self.assertEqual(get_type_hints(ChildTotalMovie), {"title": str, "year": int}) + self.assertEqual(get_type_hints(ChildTotalMovie, include_extras=True), { + "title": Required[str], "year": NotRequired[int] + }) + + self.assertEqual(get_type_hints(ChildDeeplyAnnotatedMovie), {"title": str, "year": int}) + self.assertEqual(get_type_hints(ChildDeeplyAnnotatedMovie, include_extras=True), { + "title": Annotated[Required[str], "foobar", "another level"], + "year": NotRequired[Annotated[int, 2000]] + }) + + def test_get_type_hints_collections_abc_callable(self): + # https://github.com/python/cpython/issues/91621 + P = ParamSpec('P') + def f(x: collections.abc.Callable[[int], int]): ... + def g(x: collections.abc.Callable[..., int]): ... + def h(x: collections.abc.Callable[P, int]): ... + + self.assertEqual(get_type_hints(f), {'x': collections.abc.Callable[[int], int]}) + self.assertEqual(get_type_hints(g), {'x': collections.abc.Callable[..., int]}) + self.assertEqual(get_type_hints(h), {'x': collections.abc.Callable[P, int]}) + + def test_get_type_hints_format(self): + class C: + x: undefined + + with self.assertRaises(NameError): + get_type_hints(C) + + with self.assertRaises(NameError): + get_type_hints(C, format=annotationlib.Format.VALUE) + + annos = get_type_hints(C, format=annotationlib.Format.FORWARDREF) + self.assertIsInstance(annos, dict) + self.assertEqual(list(annos), ['x']) + self.assertIsInstance(annos['x'], annotationlib.ForwardRef) + self.assertEqual(annos['x'].__arg__, 'undefined') + + self.assertEqual(get_type_hints(C, format=annotationlib.Format.STRING), + {'x': 'undefined'}) + # Make sure using an int as format also works: + self.assertEqual(get_type_hints(C, format=4), {'x': 'undefined'}) + + def test_get_type_hints_format_function(self): + def func(x: undefined) -> undefined: ... + + # VALUE + with self.assertRaises(NameError): + get_type_hints(func) + with self.assertRaises(NameError): + get_type_hints(func, format=annotationlib.Format.VALUE) + + # FORWARDREF + self.assertEqual( + get_type_hints(func, format=annotationlib.Format.FORWARDREF), + {'x': EqualToForwardRef('undefined', owner=func), + 'return': EqualToForwardRef('undefined', owner=func)}, + ) + + # STRING + self.assertEqual(get_type_hints(func, format=annotationlib.Format.STRING), + {'x': 'undefined', 'return': 'undefined'}) + + def test_callable_with_ellipsis_forward(self): + + def foo(a: 'Callable[..., T]'): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': Callable[..., T]}) + + def test_special_forms_no_forward(self): + def f(x: ClassVar[int]): + pass + self.assertEqual(get_type_hints(f), {'x': ClassVar[int]}) + + def test_special_forms_forward(self): + + class C: + a: Annotated['ClassVar[int]', (3, 5)] = 4 + b: Annotated['Final[int]', "const"] = 4 + x: 'ClassVar' = 4 + y: 'Final' = 4 + + class CF: + b: List['Final[int]'] = 4 + + self.assertEqual(get_type_hints(C, globals())['a'], ClassVar[int]) + self.assertEqual(get_type_hints(C, globals())['b'], Final[int]) + self.assertEqual(get_type_hints(C, globals())['x'], ClassVar) + self.assertEqual(get_type_hints(C, globals())['y'], Final) + lfi = get_type_hints(CF, globals())['b'] + self.assertIs(get_origin(lfi), list) + self.assertEqual(get_args(lfi), (Final[int],)) + + def test_union_forward_recursion(self): + ValueList = List['Value'] + Value = Union[str, ValueList] + + class C: + foo: List[Value] + class D: + foo: Union[Value, ValueList] + class E: + foo: Union[List[Value], ValueList] + class F: + foo: Union[Value, List[Value], ValueList] + + self.assertEqual(get_type_hints(C, globals(), locals()), get_type_hints(C, globals(), locals())) + self.assertEqual(get_type_hints(C, globals(), locals()), + {'foo': List[Union[str, List[Union[str, List['Value']]]]]}) + self.assertEqual(get_type_hints(D, globals(), locals()), + {'foo': Union[str, List[Union[str, List['Value']]]]}) + self.assertEqual(get_type_hints(E, globals(), locals()), + {'foo': Union[ + List[Union[str, List[Union[str, List['Value']]]]], + List[Union[str, List['Value']]] + ] + }) + self.assertEqual(get_type_hints(F, globals(), locals()), + {'foo': Union[ + str, + List[Union[str, List['Value']]], + List[Union[str, List[Union[str, List['Value']]]]] + ] + }) + + def test_tuple_forward(self): + + def foo(a: Tuple['T']): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': Tuple[T]}) + + def foo(a: tuple[ForwardRef('T')]): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': tuple[T]}) + + def test_double_forward(self): + def foo(a: 'List[\'int\']'): + pass + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': List[int]}) + + def test_union_forward(self): - BA = Tuple[Annotated[T, (1, 0)], ...] - def barfoo(x: BA): ... - self.assertEqual(get_type_hints(barfoo, globals(), locals())['x'], Tuple[T, ...]) - self.assertIs( - get_type_hints(barfoo, globals(), locals(), include_extras=True)['x'], - BA - ) + def foo(a: Union['T']): + pass - BA = tuple[Annotated[T, (1, 0)], ...] - def barfoo(x: BA): ... - self.assertEqual(get_type_hints(barfoo, globals(), locals())['x'], tuple[T, ...]) - self.assertIs( - get_type_hints(barfoo, globals(), locals(), include_extras=True)['x'], - BA - ) + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': Union[T]}) - def barfoo2(x: typing.Callable[..., Annotated[List[T], "const"]], - y: typing.Union[int, Annotated[T, "mutable"]]): ... - self.assertEqual( - get_type_hints(barfoo2, globals(), locals()), - {'x': typing.Callable[..., List[T]], 'y': typing.Union[int, T]} - ) + def foo(a: tuple[ForwardRef('T')] | int): + pass - BA2 = typing.Callable[..., List[T]] - def barfoo3(x: BA2): ... - self.assertIs( - get_type_hints(barfoo3, globals(), locals(), include_extras=True)["x"], - BA2 - ) - BA3 = typing.Annotated[int | float, "const"] - def barfoo4(x: BA3): ... - self.assertEqual( - get_type_hints(barfoo4, globals(), locals()), - {"x": int | float} - ) - self.assertEqual( - get_type_hints(barfoo4, globals(), locals(), include_extras=True), - {"x": typing.Annotated[int | float, "const"]} - ) + self.assertEqual(get_type_hints(foo, globals(), locals()), + {'a': tuple[T] | int}) - def test_get_type_hints_annotated_in_union(self): # bpo-46603 - def with_union(x: int | list[Annotated[str, 'meta']]): ... + def test_default_globals(self): + code = ("class C:\n" + " def foo(self, a: 'C') -> 'D': pass\n" + "class D:\n" + " def bar(self, b: 'D') -> C: pass\n" + ) + ns = {} + exec(code, ns) + hints = get_type_hints(ns['C'].foo) + self.assertEqual(hints, {'a': ns['C'], 'return': ns['D']}) - self.assertEqual(get_type_hints(with_union), {'x': int | list[str]}) - self.assertEqual( - get_type_hints(with_union, include_extras=True), - {'x': int | list[Annotated[str, 'meta']]}, - ) + def test_final_forward_ref(self): + gth = get_type_hints + self.assertEqual(gth(Loop, globals())['attr'], Final[Loop]) + self.assertNotEqual(gth(Loop, globals())['attr'], Final[int]) + self.assertNotEqual(gth(Loop, globals())['attr'], Final) - def test_get_type_hints_annotated_refs(self): + def test_name_error(self): - Const = Annotated[T, "Const"] + def foo(a: 'Noode[T]'): + pass - class MySet(Generic[T]): + with self.assertRaises(NameError): + get_type_hints(foo, locals()) - def __ior__(self, other: "Const[MySet[T]]") -> "MySet[T]": - ... + def test_basics(self): - def __iand__(self, other: Const["MySet[T]"]) -> "MySet[T]": - ... + class Node(Generic[T]): - self.assertEqual( - get_type_hints(MySet.__iand__, globals(), locals()), - {'other': MySet[T], 'return': MySet[T]} - ) + def __init__(self, label: T): + self.label = label + self.left = self.right = None - self.assertEqual( - get_type_hints(MySet.__iand__, globals(), locals(), include_extras=True), - {'other': Const[MySet[T]], 'return': MySet[T]} - ) + def add_both(self, + left: 'Optional[Node[T]]', + right: 'Node[T]' = None, + stuff: int = None, + blah=None): + self.left = left + self.right = right - self.assertEqual( - get_type_hints(MySet.__ior__, globals(), locals()), - {'other': MySet[T], 'return': MySet[T]} - ) + def add_left(self, node: Optional['Node[T]']): + self.add_both(node, None) - def test_get_type_hints_classes_str_annotations(self): - class Foo: - y = str - x: 'y' - # This previously raised an error under PEP 563. - self.assertEqual(get_type_hints(Foo), {'x': str}) + def add_right(self, node: 'Node[T]' = None): + self.add_both(None, node) - def test_get_type_hints_bad_module(self): - # bpo-41515 - class BadModule: - pass - BadModule.__module__ = 'bad' # Something not in sys.modules - self.assertNotIn('bad', sys.modules) - self.assertEqual(get_type_hints(BadModule), {}) + t = Node[int] + both_hints = get_type_hints(t.add_both, globals(), locals()) + self.assertEqual(both_hints['left'], Optional[Node[T]]) + self.assertEqual(both_hints['right'], Node[T]) + self.assertEqual(both_hints['stuff'], int) + self.assertNotIn('blah', both_hints) - def test_get_type_hints_annotated_bad_module(self): - # See https://bugs.python.org/issue44468 - class BadBase: - foo: tuple - class BadType(BadBase): - bar: list - BadType.__module__ = BadBase.__module__ = 'bad' - self.assertNotIn('bad', sys.modules) - self.assertEqual(get_type_hints(BadType), {'foo': tuple, 'bar': list}) + left_hints = get_type_hints(t.add_left, globals(), locals()) + self.assertEqual(left_hints['node'], Optional[Node[T]]) + + right_hints = get_type_hints(t.add_right, globals(), locals()) + self.assertEqual(right_hints['node'], Node[T]) + + def test_stringified_typeddict(self): + ns = run_code( + """ + from __future__ import annotations + from typing import TypedDict + class TD[UniqueT](TypedDict): + a: UniqueT + """ + ) + TD = ns['TD'] + self.assertEqual(TD.__annotations__, {'a': EqualToForwardRef('UniqueT', owner=TD, module=TD.__module__)}) + self.assertEqual(get_type_hints(TD), {'a': TD.__type_params__[0]}) class GetUtilitiesTestCase(TestCase): def test_get_origin(self): T = TypeVar('T') + Ts = TypeVarTuple('Ts') P = ParamSpec('P') class C(Generic[T]): pass self.assertIs(get_origin(C[int]), C) @@ -3460,30 +7221,47 @@ class C(Generic[T]): pass self.assertIs(get_origin(Callable), collections.abc.Callable) self.assertIs(get_origin(list[int]), list) self.assertIs(get_origin(list), None) - self.assertIs(get_origin(list | str), types.UnionType) + self.assertIs(get_origin(list | str), Union) self.assertIs(get_origin(P.args), P) self.assertIs(get_origin(P.kwargs), P) + self.assertIs(get_origin(Required[int]), Required) + self.assertIs(get_origin(NotRequired[int]), NotRequired) + self.assertIs(get_origin((*Ts,)[0]), Unpack) + self.assertIs(get_origin(Unpack[Ts]), Unpack) + self.assertIs(get_origin((*tuple[*Ts],)[0]), tuple) + self.assertIs(get_origin(Unpack[Tuple[Unpack[Ts]]]), Unpack) def test_get_args(self): T = TypeVar('T') class C(Generic[T]): pass self.assertEqual(get_args(C[int]), (int,)) self.assertEqual(get_args(C[T]), (T,)) + self.assertEqual(get_args(typing.SupportsAbs[int]), (int,)) # Protocol + self.assertEqual(get_args(typing.SupportsAbs[T]), (T,)) + self.assertEqual(get_args(Point2DGeneric[int]), (int,)) # TypedDict + self.assertEqual(get_args(Point2DGeneric[T]), (T,)) + self.assertEqual(get_args(T), ()) self.assertEqual(get_args(int), ()) + self.assertEqual(get_args(Any), ()) + self.assertEqual(get_args(Self), ()) + self.assertEqual(get_args(LiteralString), ()) self.assertEqual(get_args(ClassVar[int]), (int,)) self.assertEqual(get_args(Union[int, str]), (int, str)) self.assertEqual(get_args(Literal[42, 43]), (42, 43)) self.assertEqual(get_args(Final[List[int]]), (List[int],)) + self.assertEqual(get_args(Optional[int]), (int, type(None))) + self.assertEqual(get_args(Union[int, None]), (int, type(None))) self.assertEqual(get_args(Union[int, Tuple[T, int]][str]), (int, Tuple[str, int])) self.assertEqual(get_args(typing.Dict[int, Tuple[T, T]][Optional[int]]), (int, Tuple[Optional[int], Optional[int]])) self.assertEqual(get_args(Callable[[], T][int]), ([], int)) self.assertEqual(get_args(Callable[..., int]), (..., int)) + self.assertEqual(get_args(Callable[[int], str]), ([int], str)) self.assertEqual(get_args(Union[int, Callable[[Tuple[T, ...]], str]]), (int, Callable[[Tuple[T, ...]], str])) self.assertEqual(get_args(Tuple[int, ...]), (int, ...)) - self.assertEqual(get_args(Tuple[()]), ((),)) + self.assertEqual(get_args(Tuple[()]), ()) self.assertEqual(get_args(Annotated[T, 'one', 2, ['three']]), (T, 'one', 2, ['three'])) self.assertEqual(get_args(List), ()) self.assertEqual(get_args(Tuple), ()) @@ -3496,26 +7274,148 @@ class C(Generic[T]): pass self.assertEqual(get_args(collections.abc.Callable[[int], str]), get_args(Callable[[int], str])) P = ParamSpec('P') + self.assertEqual(get_args(P), ()) + self.assertEqual(get_args(P.args), ()) + self.assertEqual(get_args(P.kwargs), ()) self.assertEqual(get_args(Callable[P, int]), (P, int)) + self.assertEqual(get_args(collections.abc.Callable[P, int]), (P, int)) self.assertEqual(get_args(Callable[Concatenate[int, P], int]), (Concatenate[int, P], int)) + self.assertEqual(get_args(collections.abc.Callable[Concatenate[int, P], int]), + (Concatenate[int, P], int)) + self.assertEqual(get_args(Concatenate[int, str, P]), (int, str, P)) self.assertEqual(get_args(list | str), (list, str)) + self.assertEqual(get_args(Required[int]), (int,)) + self.assertEqual(get_args(NotRequired[int]), (int,)) + self.assertEqual(get_args(TypeAlias), ()) + self.assertEqual(get_args(TypeGuard[int]), (int,)) + self.assertEqual(get_args(TypeIs[range]), (range,)) + Ts = TypeVarTuple('Ts') + self.assertEqual(get_args(Ts), ()) + self.assertEqual(get_args((*Ts,)[0]), (Ts,)) + self.assertEqual(get_args(Unpack[Ts]), (Ts,)) + self.assertEqual(get_args(tuple[*Ts]), (*Ts,)) + self.assertEqual(get_args(tuple[Unpack[Ts]]), (Unpack[Ts],)) + self.assertEqual(get_args((*tuple[*Ts],)[0]), (*Ts,)) + self.assertEqual(get_args(Unpack[tuple[Unpack[Ts]]]), (tuple[Unpack[Ts]],)) + + +class EvaluateForwardRefTests(BaseTestCase): + def test_evaluate_forward_ref(self): + int_ref = ForwardRef('int') + self.assertIs(typing.evaluate_forward_ref(int_ref), int) + self.assertIs( + typing.evaluate_forward_ref(int_ref, type_params=()), + int, + ) + self.assertIs( + typing.evaluate_forward_ref(int_ref, format=annotationlib.Format.VALUE), + int, + ) + self.assertIs( + typing.evaluate_forward_ref( + int_ref, format=annotationlib.Format.FORWARDREF, + ), + int, + ) + self.assertEqual( + typing.evaluate_forward_ref( + int_ref, format=annotationlib.Format.STRING, + ), + 'int', + ) - def test_forward_ref_and_final(self): - # https://bugs.python.org/issue45166 - hints = get_type_hints(ann_module5) - self.assertEqual(hints, {'name': Final[str]}) + def test_evaluate_forward_ref_undefined(self): + missing = ForwardRef('missing') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(missing) + self.assertIs( + typing.evaluate_forward_ref( + missing, format=annotationlib.Format.FORWARDREF, + ), + missing, + ) + self.assertEqual( + typing.evaluate_forward_ref( + missing, format=annotationlib.Format.STRING, + ), + "missing", + ) - hints = get_type_hints(ann_module5.MyClass) - self.assertEqual(hints, {'value': Final}) + def test_evaluate_forward_ref_nested(self): + ref = ForwardRef("int | list['str']") + self.assertEqual( + typing.evaluate_forward_ref(ref), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.STRING), + "int | list['str']", + ) - def test_top_level_class_var(self): - # https://bugs.python.org/issue45166 - with self.assertRaisesRegex( - TypeError, - r'typing.ClassVar\[int\] is not valid as type argument', - ): - get_type_hints(ann_module6) + why = ForwardRef('"\'str\'"') + self.assertIs(typing.evaluate_forward_ref(why), str) + + def test_evaluate_forward_ref_none(self): + none_ref = ForwardRef('None') + self.assertIs(typing.evaluate_forward_ref(none_ref), None) + + def test_globals(self): + A = "str" + ref = ForwardRef('list[A]') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + self.assertEqual( + typing.evaluate_forward_ref(ref, globals={'A': A}), + list[str], + ) + + def test_owner(self): + ref = ForwardRef("A") + + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + # We default to the globals of `owner`, + # so it no longer raises `NameError` + self.assertIs( + typing.evaluate_forward_ref(ref, owner=Loop), A + ) + + def test_inherited_owner(self): + # owner passed to evaluate_forward_ref + ref = ForwardRef("list['A']") + self.assertEqual( + typing.evaluate_forward_ref(ref, owner=Loop), + list[A], + ) + + # owner set on the ForwardRef + ref = ForwardRef("list['A']", owner=Loop) + self.assertEqual( + typing.evaluate_forward_ref(ref), + list[A], + ) + + def test_partial_evaluation(self): + ref = ForwardRef("list[A]") + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + list[EqualToForwardRef('A')], + ) + + def test_with_module(self): + from test.typinganndata import fwdref_module + + typing.evaluate_forward_ref( + fwdref_module.fw,) class CollectionsAbcTests(BaseTestCase): @@ -3540,27 +7440,17 @@ def test_iterator(self): self.assertIsInstance(it, typing.Iterator) self.assertNotIsInstance(42, typing.Iterator) - @skipUnless(ASYNCIO, 'Python 3.5 and multithreading required') def test_awaitable(self): - ns = {} - exec( - "async def foo() -> typing.Awaitable[int]:\n" - " return await AwaitableWrapper(42)\n", - globals(), ns) - foo = ns['foo'] + async def foo() -> typing.Awaitable[int]: + return await AwaitableWrapper(42) g = foo() self.assertIsInstance(g, typing.Awaitable) self.assertNotIsInstance(foo, typing.Awaitable) g.send(None) # Run foo() till completion, to avoid warning. - @skipUnless(ASYNCIO, 'Python 3.5 and multithreading required') def test_coroutine(self): - ns = {} - exec( - "async def foo():\n" - " return\n", - globals(), ns) - foo = ns['foo'] + async def foo(): + return g = foo() self.assertIsInstance(g, typing.Coroutine) with self.assertRaises(TypeError): @@ -3571,7 +7461,6 @@ def test_coroutine(self): except StopIteration: pass - @skipUnless(ASYNCIO, 'Python 3.5 and multithreading required') def test_async_iterable(self): base_it = range(10) # type: Iterator[int] it = AsyncIteratorWrapper(base_it) @@ -3579,7 +7468,6 @@ def test_async_iterable(self): self.assertIsInstance(it, typing.AsyncIterable) self.assertNotIsInstance(42, typing.AsyncIterable) - @skipUnless(ASYNCIO, 'Python 3.5 and multithreading required') def test_async_iterator(self): base_it = range(10) # type: Iterator[int] it = AsyncIteratorWrapper(base_it) @@ -3625,8 +7513,14 @@ def test_mutablesequence(self): self.assertNotIsInstance((), typing.MutableSequence) def test_bytestring(self): - self.assertIsInstance(b'', typing.ByteString) - self.assertIsInstance(bytearray(b''), typing.ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(b'', typing.ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(bytearray(b''), typing.ByteString) + with self.assertWarns(DeprecationWarning): + class Foo(typing.ByteString): ... + with self.assertWarns(DeprecationWarning): + class Bar(typing.ByteString, typing.Awaitable): ... def test_list(self): self.assertIsSubclass(list, typing.List) @@ -3733,7 +7627,6 @@ class MyOrdDict(typing.OrderedDict[str, int]): self.assertIsSubclass(MyOrdDict, collections.OrderedDict) self.assertNotIsSubclass(collections.OrderedDict, MyOrdDict) - @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') def test_chainmap_instantiation(self): self.assertIs(type(typing.ChainMap()), collections.ChainMap) self.assertIs(type(typing.ChainMap[KT, VT]()), collections.ChainMap) @@ -3741,7 +7634,6 @@ def test_chainmap_instantiation(self): class CM(typing.ChainMap[KT, VT]): ... self.assertIs(type(CM[int, str]()), CM) - @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') def test_chainmap_subclass(self): class MyChainMap(typing.ChainMap[str, int]): @@ -3823,6 +7715,17 @@ def foo(): g = foo() self.assertIsSubclass(type(g), typing.Generator) + def test_generator_default(self): + g1 = typing.Generator[int] + g2 = typing.Generator[int, None, None] + self.assertEqual(get_args(g1), (int, type(None), type(None))) + self.assertEqual(get_args(g1), get_args(g2)) + + g3 = typing.Generator[int, float] + g4 = typing.Generator[int, float, None] + self.assertEqual(get_args(g3), (int, float, type(None))) + self.assertEqual(get_args(g3), get_args(g4)) + def test_no_generator_instantiation(self): with self.assertRaises(TypeError): typing.Generator() @@ -3832,10 +7735,9 @@ def test_no_generator_instantiation(self): typing.Generator[int, int, int]() def test_async_generator(self): - ns = {} - exec("async def f():\n" - " yield 42\n", globals(), ns) - g = ns['f']() + async def f(): + yield 42 + g = f() self.assertIsSubclass(type(g), typing.AsyncGenerator) def test_no_async_generator_instantiation(self): @@ -3846,8 +7748,6 @@ def test_no_async_generator_instantiation(self): with self.assertRaises(TypeError): typing.AsyncGenerator[int, int]() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclassing(self): class MMA(typing.MutableMapping): @@ -3869,7 +7769,7 @@ def __len__(self): return 0 self.assertEqual(len(MMC()), 0) - assert callable(MMC.update) + self.assertTrue(callable(MMC.update)) self.assertIsInstance(MMC(), typing.Mapping) class MMB(typing.MutableMapping[KT, VT]): @@ -3924,9 +7824,8 @@ def asend(self, value): def athrow(self, typ, val=None, tb=None): pass - ns = {} - exec('async def g(): yield 0', globals(), ns) - g = ns['g'] + async def g(): yield 0 + self.assertIsSubclass(G, typing.AsyncGenerator) self.assertIsSubclass(G, typing.AsyncIterable) self.assertIsSubclass(G, collections.abc.AsyncGenerator) @@ -4011,7 +7910,15 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) - @skipUnless(ASYNCIO, 'Python 3.5 required') + def test_contextmanager_type_params(self): + cm1 = typing.ContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.ContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) + + type gen_cm[T1, T2] = typing.ContextManager[T1, T2] + self.assertEqual(get_args(gen_cm.__value__[int, None]), (int, types.NoneType)) + def test_async_contextmanager(self): class NotACM: pass @@ -4023,11 +7930,17 @@ def manager(): cm = manager() self.assertNotIsInstance(cm, typing.AsyncContextManager) - self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int, bool | None)) with self.assertRaises(TypeError): isinstance(42, typing.AsyncContextManager[int]) with self.assertRaises(TypeError): - typing.AsyncContextManager[int, str] + typing.AsyncContextManager[int, str, float] + + def test_asynccontextmanager_type_params(self): + cm1 = typing.AsyncContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.AsyncContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) class TypeTests(BaseTestCase): @@ -4065,16 +7978,24 @@ def foo(a: A) -> Optional[BaseException]: else: return a() - assert isinstance(foo(KeyboardInterrupt), KeyboardInterrupt) - assert foo(None) is None + self.assertIsInstance(foo(KeyboardInterrupt), KeyboardInterrupt) + self.assertIsNone(foo(None)) + + +class TestModules(TestCase): + func_names = ['_idfunc'] + + def test_c_functions(self): + for fname in self.func_names: + self.assertEqual(getattr(typing, fname).__module__, '_typing') class NewTypeTests(BaseTestCase): @classmethod def setUpClass(cls): global UserId - UserId = NewType('UserId', int) - cls.UserName = NewType(cls.__qualname__ + '.UserName', str) + UserId = typing.NewType('UserId', int) + cls.UserName = typing.NewType(cls.__qualname__ + '.UserName', str) @classmethod def tearDownClass(cls): @@ -4082,9 +8003,6 @@ def tearDownClass(cls): del UserId del cls.UserName - def tearDown(self): - self.clear_caches() - def test_basic(self): self.assertIsInstance(UserId(5), int) self.assertIsInstance(self.UserName('Joe'), str) @@ -4100,11 +8018,11 @@ class D(UserId): def test_or(self): for cls in (int, self.UserName): with self.subTest(cls=cls): - self.assertEqual(UserId | cls, Union[UserId, cls]) - self.assertEqual(cls | UserId, Union[cls, UserId]) + self.assertEqual(UserId | cls, typing.Union[UserId, cls]) + self.assertEqual(cls | UserId, typing.Union[cls, UserId]) - self.assertEqual(get_args(UserId | cls), (UserId, cls)) - self.assertEqual(get_args(cls | UserId), (cls, UserId)) + self.assertEqual(typing.get_args(UserId | cls), (UserId, cls)) + self.assertEqual(typing.get_args(cls | UserId), (cls, UserId)) def test_special_attrs(self): self.assertEqual(UserId.__name__, 'UserId') @@ -4125,7 +8043,7 @@ def test_repr(self): f'{__name__}.{self.__class__.__qualname__}.UserName') def test_pickle(self): - UserAge = NewType('UserAge', float) + UserAge = typing.NewType('UserAge', float) for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): pickled = pickle.dumps(UserId, proto) @@ -4145,6 +8063,17 @@ def test_missing__name__(self): ) exec(code, {}) + def test_error_message_when_subclassing(self): + with self.assertRaisesRegex( + TypeError, + re.escape( + "Cannot subclass an instance of NewType. Perhaps you were looking for: " + "`ProUserId = NewType('ProUserId', UserId)`" + ) + ): + class ProUserId(UserId): + ... + class NamedTupleTests(BaseTestCase): class NestedEmployee(NamedTuple): @@ -4167,14 +8096,6 @@ def test_basics(self): self.assertEqual(Emp.__annotations__, collections.OrderedDict([('name', str), ('id', int)])) - def test_namedtuple_pyversion(self): - if sys.version_info[:2] < (3, 6): - with self.assertRaises(TypeError): - NamedTuple('Name', one=int, other=str) - with self.assertRaises(TypeError): - class NotYet(NamedTuple): - whatever = 0 - def test_annotation_usage(self): tim = CoolEmployee('Tim', 9000) self.assertIsInstance(tim, CoolEmployee) @@ -4224,28 +8145,166 @@ class XMethBad2(NamedTuple): def _source(self): return 'no chance for this as well' + def test_annotation_type_check(self): + # These are rejected by _type_check + with self.assertRaises(TypeError): + class X(NamedTuple): + a: Final + with self.assertRaises(TypeError): + class Y(NamedTuple): + a: (1, 2) + + # Conversion by _type_convert + class Z(NamedTuple): + a: None + b: "str" + annos = {'a': type(None), 'b': EqualToForwardRef("str")} + self.assertEqual(Z.__annotations__, annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Z.__annotate__(annotationlib.Format.STRING), {"a": "None", "b": "str"}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + from typing import NamedTuple + class X(NamedTuple): + a: int + b: None + """ + ns = run_code(textwrap.dedent(code)) + X = ns['X'] + self.assertEqual(X.__annotations__, {'a': EqualToForwardRef("int"), 'b': EqualToForwardRef("None")}) + + def test_deferred_annotations(self): + class X(NamedTuple): + y: undefined + + self.assertEqual(X._fields, ('y',)) + with self.assertRaises(NameError): + X.__annotations__ + + undefined = int + self.assertEqual(X.__annotations__, {'y': int}) + def test_multiple_inheritance(self): class A: pass with self.assertRaises(TypeError): class X(NamedTuple, A): x: int + with self.assertRaises(TypeError): + class Y(NamedTuple, tuple): + x: int + with self.assertRaises(TypeError): + class Z(NamedTuple, NamedTuple): + x: int + class B(NamedTuple): + x: int + with self.assertRaises(TypeError): + class C(NamedTuple, B): + y: str + + def test_generic(self): + class X(NamedTuple, Generic[T]): + x: T + self.assertEqual(X.__bases__, (tuple, Generic)) + self.assertEqual(X.__orig_bases__, (NamedTuple, Generic[T])) + self.assertEqual(X.__mro__, (X, tuple, Generic, object)) + + class Y(Generic[T], NamedTuple): + x: T + self.assertEqual(Y.__bases__, (Generic, tuple)) + self.assertEqual(Y.__orig_bases__, (Generic[T], NamedTuple)) + self.assertEqual(Y.__mro__, (Y, Generic, tuple, object)) + + for G in X, Y: + with self.subTest(type=G): + self.assertEqual(G.__parameters__, (T,)) + self.assertEqual(G[T].__args__, (T,)) + self.assertEqual(get_args(G[T]), (T,)) + A = G[int] + self.assertIs(A.__origin__, G) + self.assertEqual(A.__args__, (int,)) + self.assertEqual(get_args(A), (int,)) + self.assertEqual(A.__parameters__, ()) + + a = A(3) + self.assertIs(type(a), G) + self.assertEqual(a.x, 3) + + with self.assertRaises(TypeError): + G[int, str] + + def test_generic_pep695(self): + class X[T](NamedTuple): + x: T + T, = X.__type_params__ + self.assertIsInstance(T, TypeVar) + self.assertEqual(T.__name__, 'T') + self.assertEqual(X.__bases__, (tuple, Generic)) + self.assertEqual(X.__orig_bases__, (NamedTuple, Generic[T])) + self.assertEqual(X.__mro__, (X, tuple, Generic, object)) + self.assertEqual(X.__parameters__, (T,)) + self.assertEqual(X[str].__args__, (str,)) + self.assertEqual(X[str].__parameters__, ()) + + def test_non_generic_subscript(self): + # For backward compatibility, subscription works + # on arbitrary NamedTuple types. + class Group(NamedTuple): + key: T + group: list[T] + A = Group[int] + self.assertEqual(A.__origin__, Group) + self.assertEqual(A.__parameters__, ()) + self.assertEqual(A.__args__, (int,)) + a = A(1, [2]) + self.assertIs(type(a), Group) + self.assertEqual(a, (1, [2])) + + def test_namedtuple_keyword_usage(self): + with self.assertWarnsRegex( + DeprecationWarning, + "Creating NamedTuple classes using keyword arguments is deprecated" + ): + LocalEmployee = NamedTuple("LocalEmployee", name=str, age=int) - def test_namedtuple_keyword_usage(self): - LocalEmployee = NamedTuple("LocalEmployee", name=str, age=int) nick = LocalEmployee('Nick', 25) self.assertIsInstance(nick, tuple) self.assertEqual(nick.name, 'Nick') self.assertEqual(LocalEmployee.__name__, 'LocalEmployee') self.assertEqual(LocalEmployee._fields, ('name', 'age')) self.assertEqual(LocalEmployee.__annotations__, dict(name=str, age=int)) - with self.assertRaises(TypeError): + + with self.assertRaisesRegex( + TypeError, + "Either list of fields or keywords can be provided to NamedTuple, not both" + ): NamedTuple('Name', [('x', int)], y=str) - with self.assertRaises(TypeError): - NamedTuple('Name', x=1, y='a') + + with self.assertRaisesRegex( + TypeError, + "Either list of fields or keywords can be provided to NamedTuple, not both" + ): + NamedTuple('Name', [], y=str) + + with self.assertRaisesRegex( + TypeError, + ( + r"Cannot pass `None` as the 'fields' parameter " + r"and also specify fields using keyword arguments" + ) + ): + NamedTuple('Name', None, x=int) def test_namedtuple_special_keyword_names(self): - NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list) + with self.assertWarnsRegex( + DeprecationWarning, + "Creating NamedTuple classes using keyword arguments is deprecated" + ): + NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list) + self.assertEqual(NT.__name__, 'NT') self.assertEqual(NT._fields, ('cls', 'self', 'typename', 'fields')) a = NT(cls=str, self=42, typename='foo', fields=[('bar', tuple)]) @@ -4255,12 +8314,32 @@ def test_namedtuple_special_keyword_names(self): self.assertEqual(a.fields, [('bar', tuple)]) def test_empty_namedtuple(self): - NT = NamedTuple('NT') + expected_warning = re.escape( + "Failing to pass a value for the 'fields' parameter is deprecated " + "and will be disallowed in Python 3.15. " + "To create a NamedTuple class with 0 fields " + "using the functional syntax, " + "pass an empty list, e.g. `NT1 = NamedTuple('NT1', [])`." + ) + with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): + NT1 = NamedTuple('NT1') + + expected_warning = re.escape( + "Passing `None` as the 'fields' parameter is deprecated " + "and will be disallowed in Python 3.15. " + "To create a NamedTuple class with 0 fields " + "using the functional syntax, " + "pass an empty list, e.g. `NT2 = NamedTuple('NT2', [])`." + ) + with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): + NT2 = NamedTuple('NT2', None) + + NT3 = NamedTuple('NT2', []) class CNT(NamedTuple): pass # empty body - for struct in [NT, CNT]: + for struct in NT1, NT2, NT3, CNT: with self.subTest(struct=struct): self.assertEqual(struct._fields, ()) self.assertEqual(struct._field_defaults, {}) @@ -4270,16 +8349,30 @@ class CNT(NamedTuple): def test_namedtuple_errors(self): with self.assertRaises(TypeError): NamedTuple.__new__() - with self.assertRaises(TypeError): + + with self.assertRaisesRegex( + TypeError, + "missing 1 required positional argument" + ): NamedTuple() - with self.assertRaises(TypeError): + + with self.assertRaisesRegex( + TypeError, + "takes from 1 to 2 positional arguments but 3 were given" + ): NamedTuple('Emp', [('name', str)], None) - with self.assertRaises(ValueError): + + with self.assertRaisesRegex( + ValueError, + "Field names cannot start with an underscore" + ): NamedTuple('Emp', [('_name', str)]) - with self.assertRaises(TypeError): + + with self.assertRaisesRegex( + TypeError, + "missing 1 required positional argument: 'typename'" + ): NamedTuple(typename='Emp', name=str, id=int) - with self.assertRaises(TypeError): - NamedTuple('Emp', fields=[('name', str), ('id', int)]) def test_copy_and_pickle(self): global Emp # pickle wants to reference the class by name @@ -4301,6 +8394,116 @@ def test_copy_and_pickle(self): self.assertEqual(jane2, jane) self.assertIsInstance(jane2, cls) + def test_orig_bases(self): + T = TypeVar('T') + + class SimpleNamedTuple(NamedTuple): + pass + + class GenericNamedTuple(NamedTuple, Generic[T]): + pass + + self.assertEqual(SimpleNamedTuple.__orig_bases__, (NamedTuple,)) + self.assertEqual(GenericNamedTuple.__orig_bases__, (NamedTuple, Generic[T])) + + CallNamedTuple = NamedTuple('CallNamedTuple', []) + + self.assertEqual(CallNamedTuple.__orig_bases__, (NamedTuple,)) + + def test_setname_called_on_values_in_class_dictionary(self): + class Vanilla: + def __set_name__(self, owner, name): + self.name = name + + class Foo(NamedTuple): + attr = Vanilla() + + foo = Foo() + self.assertEqual(len(foo), 0) + self.assertNotIn('attr', Foo._fields) + self.assertIsInstance(foo.attr, Vanilla) + self.assertEqual(foo.attr.name, "attr") + + class Bar(NamedTuple): + attr: Vanilla = Vanilla() + + bar = Bar() + self.assertEqual(len(bar), 1) + self.assertIn('attr', Bar._fields) + self.assertIsInstance(bar.attr, Vanilla) + self.assertEqual(bar.attr.name, "attr") + + def test_setname_raises_the_same_as_on_other_classes(self): + class CustomException(BaseException): pass + + class Annoying: + def __set_name__(self, owner, name): + raise CustomException + + annoying = Annoying() + + with self.assertRaises(CustomException) as cm: + class NormalClass: + attr = annoying + normal_exception = cm.exception + + with self.assertRaises(CustomException) as cm: + class NamedTupleClass(NamedTuple): + attr = annoying + namedtuple_exception = cm.exception + + self.assertIs(type(namedtuple_exception), CustomException) + self.assertIs(type(namedtuple_exception), type(normal_exception)) + + self.assertEqual(len(namedtuple_exception.__notes__), 1) + self.assertEqual( + len(namedtuple_exception.__notes__), len(normal_exception.__notes__) + ) + + expected_note = ( + "Error calling __set_name__ on 'Annoying' instance " + "'attr' in 'NamedTupleClass'" + ) + self.assertEqual(namedtuple_exception.__notes__[0], expected_note) + self.assertEqual( + namedtuple_exception.__notes__[0], + normal_exception.__notes__[0].replace("NormalClass", "NamedTupleClass") + ) + + def test_strange_errors_when_accessing_set_name_itself(self): + class CustomException(Exception): pass + + class Meta(type): + def __getattribute__(self, attr): + if attr == "__set_name__": + raise CustomException + return object.__getattribute__(self, attr) + + class VeryAnnoying(metaclass=Meta): pass + + very_annoying = VeryAnnoying() + + with self.assertRaises(CustomException): + class Foo(NamedTuple): + attr = very_annoying + + def test_super_explicitly_disallowed(self): + expected_message = ( + "uses of super() and __class__ are unsupported " + "in methods of NamedTuple subclasses" + ) + + with self.assertRaises(TypeError, msg=expected_message): + class ThisWontWork(NamedTuple): + def __repr__(self): + return super().__repr__() + + with self.assertRaises(TypeError, msg=expected_message): + class ThisWontWorkEither(NamedTuple): + @property + def name(self): + return __class__.__name__ + class TypedDictTests(BaseTestCase): def test_basics_functional_syntax(self): @@ -4315,35 +8518,16 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__name__, 'Emp') self.assertEqual(Emp.__module__, __name__) self.assertEqual(Emp.__bases__, (dict,)) - self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) - self.assertEqual(Emp.__total__, True) - - def test_basics_keywords_syntax(self): - Emp = TypedDict('Emp', name=str, id=int) - self.assertIsSubclass(Emp, dict) - self.assertIsSubclass(Emp, typing.MutableMapping) - self.assertNotIsSubclass(Emp, collections.abc.Sequence) - jim = Emp(name='Jim', id=1) - self.assertIs(type(jim), dict) - self.assertEqual(jim['name'], 'Jim') - self.assertEqual(jim['id'], 1) - self.assertEqual(Emp.__name__, 'Emp') - self.assertEqual(Emp.__module__, __name__) - self.assertEqual(Emp.__bases__, (dict,)) - self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) + annos = {'name': str, 'id': int} + self.assertEqual(Emp.__annotations__, annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.VALUE), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.FORWARDREF), annos) + self.assertEqual(Emp.__annotate__(annotationlib.Format.STRING), {'name': 'str', 'id': 'int'}) self.assertEqual(Emp.__total__, True) - - def test_typeddict_special_keyword_names(self): - TD = TypedDict("TD", cls=type, self=object, typename=str, _typename=int, fields=list, _fields=dict) - self.assertEqual(TD.__name__, 'TD') - self.assertEqual(TD.__annotations__, {'cls': type, 'self': object, 'typename': str, '_typename': int, 'fields': list, '_fields': dict}) - a = TD(cls=str, self=42, typename='foo', _typename=53, fields=[('bar', tuple)], _fields={'baz', set}) - self.assertEqual(a['cls'], str) - self.assertEqual(a['self'], 42) - self.assertEqual(a['typename'], 'foo') - self.assertEqual(a['_typename'], 53) - self.assertEqual(a['fields'], [('bar', tuple)]) - self.assertEqual(a['_fields'], {'baz', set}) + self.assertEqual(Emp.__required_keys__, {'name', 'id'}) + self.assertIsInstance(Emp.__required_keys__, frozenset) + self.assertEqual(Emp.__optional_keys__, set()) + self.assertIsInstance(Emp.__optional_keys__, frozenset) def test_typeddict_create_errors(self): with self.assertRaises(TypeError): @@ -4352,11 +8536,10 @@ def test_typeddict_create_errors(self): TypedDict() with self.assertRaises(TypeError): TypedDict('Emp', [('name', str)], None) - with self.assertRaises(TypeError): - TypedDict(_typename='Emp', name=str, id=int) + TypedDict(_typename='Emp') with self.assertRaises(TypeError): - TypedDict('Emp', _fields={'name': str, 'id': int}) + TypedDict('Emp', name=str, id=int) def test_typeddict_errors(self): Emp = TypedDict('Emp', {'name': str, 'id': int}) @@ -4368,10 +8551,6 @@ def test_typeddict_errors(self): isinstance(jim, Emp) with self.assertRaises(TypeError): issubclass(dict, Emp) - with self.assertRaises(TypeError): - TypedDict('Hi', x=1) - with self.assertRaises(TypeError): - TypedDict('Hi', [('x', int), ('y', 1)]) with self.assertRaises(TypeError): TypedDict('Hi', [('x', int)], y=int) @@ -4390,7 +8569,7 @@ def test_py36_class_syntax_usage(self): def test_pickle(self): global EmpD # pickle wants to reference the class by name - EmpD = TypedDict('EmpD', name=str, id=int) + EmpD = TypedDict('EmpD', {'name': str, 'id': int}) jane = EmpD({'name': 'jane', 'id': 37}) for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(jane, proto) @@ -4401,8 +8580,19 @@ def test_pickle(self): EmpDnew = pickle.loads(ZZ) self.assertEqual(EmpDnew({'name': 'jane', 'id': 37}), jane) + def test_pickle_generic(self): + point = Point2DGeneric(a=5.0, b=3.0) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + z = pickle.dumps(point, proto) + point2 = pickle.loads(z) + self.assertEqual(point2, point) + self.assertEqual(point2, {'a': 5.0, 'b': 3.0}) + ZZ = pickle.dumps(Point2DGeneric, proto) + Point2DGenericNew = pickle.loads(ZZ) + self.assertEqual(Point2DGenericNew({'a': 5.0, 'b': 3.0}), point) + def test_optional(self): - EmpD = TypedDict('EmpD', name=str, id=int) + EmpD = TypedDict('EmpD', {'name': str, 'id': int}) self.assertEqual(typing.Optional[EmpD], typing.Union[None, EmpD]) self.assertNotEqual(typing.List[EmpD], typing.Tuple[EmpD]) @@ -4413,7 +8603,9 @@ def test_total(self): self.assertEqual(D(x=1), {'x': 1}) self.assertEqual(D.__total__, False) self.assertEqual(D.__required_keys__, frozenset()) + self.assertIsInstance(D.__required_keys__, frozenset) self.assertEqual(D.__optional_keys__, {'x'}) + self.assertIsInstance(D.__optional_keys__, frozenset) self.assertEqual(Options(), {}) self.assertEqual(Options(log_level=2), {'log_level': 2}) @@ -4421,12 +8613,41 @@ def test_total(self): self.assertEqual(Options.__required_keys__, frozenset()) self.assertEqual(Options.__optional_keys__, {'log_level', 'log_path'}) + def test_total_inherits_non_total(self): + class TD1(TypedDict, total=False): + a: int + + self.assertIs(TD1.__total__, False) + + class TD2(TD1): + b: str + + self.assertIs(TD2.__total__, True) + + def test_total_with_assigned_value(self): + class TD(TypedDict): + __total__ = "some_value" + + self.assertIs(TD.__total__, True) + + class TD2(TypedDict, total=True): + __total__ = "some_value" + + self.assertIs(TD2.__total__, True) + + class TD3(TypedDict, total=False): + __total__ = "some value" + + self.assertIs(TD3.__total__, False) + def test_optional_keys(self): class Point2Dor3D(Point2D, total=False): z: int - assert Point2Dor3D.__required_keys__ == frozenset(['x', 'y']) - assert Point2Dor3D.__optional_keys__ == frozenset(['z']) + self.assertEqual(Point2Dor3D.__required_keys__, frozenset(['x', 'y'])) + self.assertIsInstance(Point2Dor3D.__required_keys__, frozenset) + self.assertEqual(Point2Dor3D.__optional_keys__, frozenset(['z'])) + self.assertIsInstance(Point2Dor3D.__optional_keys__, frozenset) def test_keys_inheritance(self): class BaseAnimal(TypedDict): @@ -4439,26 +8660,132 @@ class Animal(BaseAnimal, total=False): class Cat(Animal): fur_color: str - assert BaseAnimal.__required_keys__ == frozenset(['name']) - assert BaseAnimal.__optional_keys__ == frozenset([]) - assert BaseAnimal.__annotations__ == {'name': str} + self.assertEqual(BaseAnimal.__required_keys__, frozenset(['name'])) + self.assertEqual(BaseAnimal.__optional_keys__, frozenset([])) + self.assertEqual(BaseAnimal.__annotations__, {'name': str}) - assert Animal.__required_keys__ == frozenset(['name']) - assert Animal.__optional_keys__ == frozenset(['tail', 'voice']) - assert Animal.__annotations__ == { + self.assertEqual(Animal.__required_keys__, frozenset(['name'])) + self.assertEqual(Animal.__optional_keys__, frozenset(['tail', 'voice'])) + self.assertEqual(Animal.__annotations__, { 'name': str, 'tail': bool, 'voice': str, - } + }) - assert Cat.__required_keys__ == frozenset(['name', 'fur_color']) - assert Cat.__optional_keys__ == frozenset(['tail', 'voice']) - assert Cat.__annotations__ == { + self.assertEqual(Cat.__required_keys__, frozenset(['name', 'fur_color'])) + self.assertEqual(Cat.__optional_keys__, frozenset(['tail', 'voice'])) + self.assertEqual(Cat.__annotations__, { 'fur_color': str, 'name': str, 'tail': bool, 'voice': str, - } + }) + + def test_keys_inheritance_with_same_name(self): + class NotTotal(TypedDict, total=False): + a: int + + class Total(NotTotal): + a: int + + self.assertEqual(NotTotal.__required_keys__, frozenset()) + self.assertEqual(NotTotal.__optional_keys__, frozenset(['a'])) + self.assertEqual(Total.__required_keys__, frozenset(['a'])) + self.assertEqual(Total.__optional_keys__, frozenset()) + + class Base(TypedDict): + a: NotRequired[int] + b: Required[int] + + class Child(Base): + a: Required[int] + b: NotRequired[int] + + self.assertEqual(Base.__required_keys__, frozenset(['b'])) + self.assertEqual(Base.__optional_keys__, frozenset(['a'])) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset(['b'])) + + def test_multiple_inheritance_with_same_key(self): + class Base1(TypedDict): + a: NotRequired[int] + + class Base2(TypedDict): + a: Required[str] + + class Child(Base1, Base2): + pass + + # Last base wins + self.assertEqual(Child.__annotations__, {'a': Required[str]}) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset()) + + def test_inheritance_pep563(self): + def _make_td(future, class_name, annos, base, extra_names=None): + lines = [] + if future: + lines.append('from __future__ import annotations') + lines.append('from typing import TypedDict') + lines.append(f'class {class_name}({base}):') + for name, anno in annos.items(): + lines.append(f' {name}: {anno}') + code = '\n'.join(lines) + ns = run_code(code, extra_names) + return ns[class_name] + + for base_future in (True, False): + for child_future in (True, False): + with self.subTest(base_future=base_future, child_future=child_future): + base = _make_td( + base_future, "Base", {"base": "int"}, "TypedDict" + ) + self.assertIsNotNone(base.__annotate__) + child = _make_td( + child_future, "Child", {"child": "int"}, "Base", {"Base": base} + ) + base_anno = ForwardRef("int", module="builtins", owner=base) if base_future else int + child_anno = ForwardRef("int", module="builtins", owner=child) if child_future else int + self.assertEqual(base.__annotations__, {'base': base_anno}) + self.assertEqual( + child.__annotations__, {'child': child_anno, 'base': base_anno} + ) + + def test_required_notrequired_keys(self): + self.assertEqual(NontotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(NontotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(TotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(TotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(_typed_dict_helper.VeryAnnotated.__required_keys__, + frozenset()) + self.assertEqual(_typed_dict_helper.VeryAnnotated.__optional_keys__, + frozenset({"a"})) + + self.assertEqual(AnnotatedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(AnnotatedMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(WeirdlyQuotedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(WeirdlyQuotedMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(ChildTotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(ChildTotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(ChildDeeplyAnnotatedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(ChildDeeplyAnnotatedMovie.__optional_keys__, + frozenset({"year"})) def test_multiple_inheritance(self): class One(TypedDict): @@ -4548,31 +8875,487 @@ class Wrong(*bases): pass def test_is_typeddict(self): - assert is_typeddict(Point2D) is True - assert is_typeddict(Union[str, int]) is False + self.assertIs(is_typeddict(Point2D), True) + self.assertIs(is_typeddict(Union[str, int]), False) # classes, not instances - assert is_typeddict(Point2D()) is False + self.assertIs(is_typeddict(Point2D()), False) + call_based = TypedDict('call_based', {'a': int}) + self.assertIs(is_typeddict(call_based), True) + self.assertIs(is_typeddict(call_based()), False) + + T = TypeVar("T") + class BarGeneric(TypedDict, Generic[T]): + a: T + self.assertIs(is_typeddict(BarGeneric), True) + self.assertIs(is_typeddict(BarGeneric[int]), False) + self.assertIs(is_typeddict(BarGeneric()), False) + + class NewGeneric[T](TypedDict): + a: T + self.assertIs(is_typeddict(NewGeneric), True) + self.assertIs(is_typeddict(NewGeneric[int]), False) + self.assertIs(is_typeddict(NewGeneric()), False) + + # The TypedDict constructor is not itself a TypedDict + self.assertIs(is_typeddict(TypedDict), False) + + def test_get_type_hints(self): + self.assertEqual( + get_type_hints(Bar), + {'a': typing.Optional[int], 'b': int} + ) + + def test_get_type_hints_generic(self): + self.assertEqual( + get_type_hints(BarGeneric), + {'a': typing.Optional[T], 'b': int} + ) + + class FooBarGeneric(BarGeneric[int]): + c: str + + self.assertEqual( + get_type_hints(FooBarGeneric), + {'a': typing.Optional[T], 'b': int, 'c': str} + ) + + def test_pep695_generic_typeddict(self): + class A[T](TypedDict): + a: T + + T, = A.__type_params__ + self.assertIsInstance(T, TypeVar) + self.assertEqual(T.__name__, 'T') + self.assertEqual(A.__bases__, (Generic, dict)) + self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) + self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) + self.assertEqual(A.__parameters__, (T,)) + self.assertEqual(A[str].__parameters__, ()) + self.assertEqual(A[str].__args__, (str,)) + + def test_generic_inheritance(self): + class A(TypedDict, Generic[T]): + a: T + + self.assertEqual(A.__bases__, (Generic, dict)) + self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) + self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__annotations__, {'a': T}) + self.assertEqual(A.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) + self.assertEqual(A.__parameters__, (T,)) + self.assertEqual(A[str].__parameters__, ()) + self.assertEqual(A[str].__args__, (str,)) + + class A2(Generic[T], TypedDict): + a: T + + self.assertEqual(A2.__bases__, (Generic, dict)) + self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) + self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) + self.assertEqual(A2.__annotations__, {'a': T}) + self.assertEqual(A2.__annotate__(annotationlib.Format.STRING), {'a': 'T'}) + self.assertEqual(A2.__parameters__, (T,)) + self.assertEqual(A2[str].__parameters__, ()) + self.assertEqual(A2[str].__args__, (str,)) + + class B(A[KT], total=False): + b: KT + + self.assertEqual(B.__bases__, (Generic, dict)) + self.assertEqual(B.__orig_bases__, (A[KT],)) + self.assertEqual(B.__mro__, (B, Generic, dict, object)) + self.assertEqual(B.__annotations__, {'a': T, 'b': KT}) + self.assertEqual(B.__annotate__(annotationlib.Format.STRING), {'a': 'T', 'b': 'KT'}) + self.assertEqual(B.__parameters__, (KT,)) + self.assertEqual(B.__total__, False) + self.assertEqual(B.__optional_keys__, frozenset(['b'])) + self.assertEqual(B.__required_keys__, frozenset(['a'])) + + self.assertEqual(B[str].__parameters__, ()) + self.assertEqual(B[str].__args__, (str,)) + self.assertEqual(B[str].__origin__, B) + + class C(B[int]): + c: int + + self.assertEqual(C.__bases__, (Generic, dict)) + self.assertEqual(C.__orig_bases__, (B[int],)) + self.assertEqual(C.__mro__, (C, Generic, dict, object)) + self.assertEqual(C.__parameters__, ()) + self.assertEqual(C.__total__, True) + self.assertEqual(C.__optional_keys__, frozenset(['b'])) + self.assertEqual(C.__required_keys__, frozenset(['a', 'c'])) + self.assertEqual(C.__annotations__, { + 'a': T, + 'b': KT, + 'c': int, + }) + self.assertEqual(C.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) + with self.assertRaises(TypeError): + C[str] + + + class Point3D(Point2DGeneric[T], Generic[T, KT]): + c: KT + + self.assertEqual(Point3D.__bases__, (Generic, dict)) + self.assertEqual(Point3D.__orig_bases__, (Point2DGeneric[T], Generic[T, KT])) + self.assertEqual(Point3D.__mro__, (Point3D, Generic, dict, object)) + self.assertEqual(Point3D.__parameters__, (T, KT)) + self.assertEqual(Point3D.__total__, True) + self.assertEqual(Point3D.__optional_keys__, frozenset()) + self.assertEqual(Point3D.__required_keys__, frozenset(['a', 'b', 'c'])) + self.assertEqual(Point3D.__annotations__, { + 'a': T, + 'b': T, + 'c': KT, + }) + self.assertEqual(Point3D.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'T', + 'c': 'KT', + }) + self.assertEqual(Point3D[int, str].__origin__, Point3D) + + with self.assertRaises(TypeError): + Point3D[int] + + with self.assertRaises(TypeError): + class Point3D(Point2DGeneric[T], Generic[KT]): + c: KT + + def test_implicit_any_inheritance(self): + class A(TypedDict, Generic[T]): + a: T + + class B(A[KT], total=False): + b: KT + + class WithImplicitAny(B): + c: int + + self.assertEqual(WithImplicitAny.__bases__, (Generic, dict,)) + self.assertEqual(WithImplicitAny.__mro__, (WithImplicitAny, Generic, dict, object)) + # Consistent with GenericTests.test_implicit_any + self.assertEqual(WithImplicitAny.__parameters__, ()) + self.assertEqual(WithImplicitAny.__total__, True) + self.assertEqual(WithImplicitAny.__optional_keys__, frozenset(['b'])) + self.assertEqual(WithImplicitAny.__required_keys__, frozenset(['a', 'c'])) + self.assertEqual(WithImplicitAny.__annotations__, { + 'a': T, + 'b': KT, + 'c': int, + }) + self.assertEqual(WithImplicitAny.__annotate__(annotationlib.Format.STRING), { + 'a': 'T', + 'b': 'KT', + 'c': 'int', + }) + with self.assertRaises(TypeError): + WithImplicitAny[str] + + def test_non_generic_subscript(self): + # For backward compatibility, subscription works + # on arbitrary TypedDict types. + class TD(TypedDict): + a: T + A = TD[int] + self.assertEqual(A.__origin__, TD) + self.assertEqual(A.__parameters__, ()) + self.assertEqual(A.__args__, (int,)) + a = A(a = 1) + self.assertIs(type(a), dict) + self.assertEqual(a, {'a': 1}) + + def test_orig_bases(self): + T = TypeVar('T') + + class Parent(TypedDict): + pass + + class Child(Parent): + pass + + class OtherChild(Parent): + pass + + class MixedChild(Child, OtherChild, Parent): + pass + + class GenericParent(TypedDict, Generic[T]): + pass + + class GenericChild(GenericParent[int]): + pass + + class OtherGenericChild(GenericParent[str]): + pass + + class MixedGenericChild(GenericChild, OtherGenericChild, GenericParent[float]): + pass + + class MultipleGenericBases(GenericParent[int], GenericParent[float]): + pass + + CallTypedDict = TypedDict('CallTypedDict', {}) + + self.assertEqual(Parent.__orig_bases__, (TypedDict,)) + self.assertEqual(Child.__orig_bases__, (Parent,)) + self.assertEqual(OtherChild.__orig_bases__, (Parent,)) + self.assertEqual(MixedChild.__orig_bases__, (Child, OtherChild, Parent,)) + self.assertEqual(GenericParent.__orig_bases__, (TypedDict, Generic[T])) + self.assertEqual(GenericChild.__orig_bases__, (GenericParent[int],)) + self.assertEqual(OtherGenericChild.__orig_bases__, (GenericParent[str],)) + self.assertEqual(MixedGenericChild.__orig_bases__, (GenericChild, OtherGenericChild, GenericParent[float])) + self.assertEqual(MultipleGenericBases.__orig_bases__, (GenericParent[int], GenericParent[float])) + self.assertEqual(CallTypedDict.__orig_bases__, (TypedDict,)) + + def test_zero_fields_typeddicts(self): + T1 = TypedDict("T1", {}) + class T2(TypedDict): pass + class T3[tvar](TypedDict): pass + S = TypeVar("S") + class T4(TypedDict, Generic[S]): pass + + expected_warning = re.escape( + "Failing to pass a value for the 'fields' parameter is deprecated " + "and will be disallowed in Python 3.15. " + "To create a TypedDict class with 0 fields " + "using the functional syntax, " + "pass an empty dictionary, e.g. `T5 = TypedDict('T5', {})`." + ) + with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): + T5 = TypedDict('T5') + + expected_warning = re.escape( + "Passing `None` as the 'fields' parameter is deprecated " + "and will be disallowed in Python 3.15. " + "To create a TypedDict class with 0 fields " + "using the functional syntax, " + "pass an empty dictionary, e.g. `T6 = TypedDict('T6', {})`." + ) + with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): + T6 = TypedDict('T6', None) + + for klass in T1, T2, T3, T4, T5, T6: + with self.subTest(klass=klass.__name__): + self.assertEqual(klass.__annotations__, {}) + self.assertEqual(klass.__required_keys__, set()) + self.assertEqual(klass.__optional_keys__, set()) + self.assertIsInstance(klass(), dict) + + def test_readonly_inheritance(self): + class Base1(TypedDict): + a: ReadOnly[int] + + class Child1(Base1): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + class Base2(TypedDict): + a: int + + class Child2(Base2): + b: ReadOnly[str] + + self.assertEqual(Child2.__readonly_keys__, frozenset({'b'})) + self.assertEqual(Child2.__mutable_keys__, frozenset({'a'})) + + def test_cannot_make_mutable_key_readonly(self): + class Base(TypedDict): + a: int + + with self.assertRaises(TypeError): + class Child(Base): + a: ReadOnly[int] + + def test_can_make_readonly_key_mutable(self): + class Base(TypedDict): + a: ReadOnly[int] + + class Child(Base): + a: int + + self.assertEqual(Child.__readonly_keys__, frozenset()) + self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) + + def test_combine_qualifiers(self): + class AllTheThings(TypedDict): + a: Annotated[Required[ReadOnly[int]], "why not"] + b: Required[Annotated[ReadOnly[int], "why not"]] + c: ReadOnly[NotRequired[Annotated[int, "why not"]]] + d: NotRequired[Annotated[int, "why not"]] + + self.assertEqual(AllTheThings.__required_keys__, frozenset({'a', 'b'})) + self.assertEqual(AllTheThings.__optional_keys__, frozenset({'c', 'd'})) + self.assertEqual(AllTheThings.__readonly_keys__, frozenset({'a', 'b', 'c'})) + self.assertEqual(AllTheThings.__mutable_keys__, frozenset({'d'})) + + self.assertEqual( + get_type_hints(AllTheThings, include_extras=False), + {'a': int, 'b': int, 'c': int, 'd': int}, + ) + self.assertEqual( + get_type_hints(AllTheThings, include_extras=True), + { + 'a': Annotated[Required[ReadOnly[int]], 'why not'], + 'b': Required[Annotated[ReadOnly[int], 'why not']], + 'c': ReadOnly[NotRequired[Annotated[int, 'why not']]], + 'd': NotRequired[Annotated[int, 'why not']], + }, + ) + + def test_annotations(self): + # _type_check is applied + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + class X(TypedDict): + a: Final + + # _type_convert is applied + class Y(TypedDict): + a: None + b: "int" + fwdref = EqualToForwardRef('int', module=__name__) + self.assertEqual(Y.__annotations__, {'a': type(None), 'b': fwdref}) + self.assertEqual(Y.__annotate__(annotationlib.Format.FORWARDREF), {'a': type(None), 'b': fwdref}) + + # _type_check is also applied later + class Z(TypedDict): + a: undefined + + with self.assertRaises(NameError): + Z.__annotations__ + + undefined = Final + with self.assertRaisesRegex(TypeError, "Plain typing.Final is not valid as type argument"): + Z.__annotations__ + + undefined = None + self.assertEqual(Z.__annotations__, {'a': type(None)}) + + def test_deferred_evaluation(self): + class A(TypedDict): + x: NotRequired[undefined] + y: ReadOnly[undefined] + z: Required[undefined] + + self.assertEqual(A.__required_keys__, frozenset({'y', 'z'})) + self.assertEqual(A.__optional_keys__, frozenset({'x'})) + self.assertEqual(A.__readonly_keys__, frozenset({'y'})) + self.assertEqual(A.__mutable_keys__, frozenset({'x', 'z'})) + + with self.assertRaises(NameError): + A.__annotations__ + + self.assertEqual( + A.__annotate__(annotationlib.Format.STRING), + {'x': 'NotRequired[undefined]', 'y': 'ReadOnly[undefined]', + 'z': 'Required[undefined]'}, + ) + + +class RequiredTests(BaseTestCase): + + def test_basics(self): + with self.assertRaises(TypeError): + Required[NotRequired] + with self.assertRaises(TypeError): + Required[int, str] + with self.assertRaises(TypeError): + Required[int][str] + + def test_repr(self): + self.assertEqual(repr(Required), 'typing.Required') + cv = Required[int] + self.assertEqual(repr(cv), 'typing.Required[int]') + cv = Required[Employee] + self.assertEqual(repr(cv), f'typing.Required[{__name__}.Employee]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(Required)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(Required[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Required'): + class E(Required): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Required\[int\]'): + class F(Required[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + Required() + with self.assertRaises(TypeError): + type(Required)() + with self.assertRaises(TypeError): + type(Required[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, Required[int]) + with self.assertRaises(TypeError): + issubclass(int, Required) + + +class NotRequiredTests(BaseTestCase): + + def test_basics(self): + with self.assertRaises(TypeError): + NotRequired[Required] + with self.assertRaises(TypeError): + NotRequired[int, str] + with self.assertRaises(TypeError): + NotRequired[int][str] + + def test_repr(self): + self.assertEqual(repr(NotRequired), 'typing.NotRequired') + cv = NotRequired[int] + self.assertEqual(repr(cv), 'typing.NotRequired[int]') + cv = NotRequired[Employee] + self.assertEqual(repr(cv), f'typing.NotRequired[{__name__}.Employee]') - def test_get_type_hints(self): - self.assertEqual( - get_type_hints(Bar), - {'a': typing.Optional[int], 'b': int} - ) + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(NotRequired)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(NotRequired[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.NotRequired'): + class E(NotRequired): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.NotRequired\[int\]'): + class F(NotRequired[int]): + pass - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_non_generic_subscript(self): - # For backward compatibility, subscription works - # on arbitrary TypedDict types. - class TD(TypedDict): - a: T - A = TD[int] - self.assertEqual(A.__origin__, TD) - self.assertEqual(A.__parameters__, ()) - self.assertEqual(A.__args__, (int,)) - a = A(a = 1) - self.assertIs(type(a), dict) - self.assertEqual(a, {'a': 1}) + def test_cannot_init(self): + with self.assertRaises(TypeError): + NotRequired() + with self.assertRaises(TypeError): + type(NotRequired)() + with self.assertRaises(TypeError): + type(NotRequired[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, NotRequired[int]) + with self.assertRaises(TypeError): + issubclass(int, NotRequired) class IOTests(BaseTestCase): @@ -4601,14 +9384,6 @@ def stuff(a: BinaryIO) -> bytes: a = stuff.__annotations__['a'] self.assertEqual(a.__parameters__, ()) - def test_io_submodule(self): - from typing.io import IO, TextIO, BinaryIO, __all__, __name__ - self.assertIs(IO, typing.IO) - self.assertIs(TextIO, typing.TextIO) - self.assertIs(BinaryIO, typing.BinaryIO) - self.assertEqual(set(__all__), set(['IO', 'TextIO', 'BinaryIO'])) - self.assertEqual(__name__, 'typing.io') - class RETests(BaseTestCase): # Much of this is really testing _TypeAlias. @@ -4653,31 +9428,26 @@ def test_repr(self): self.assertEqual(repr(Match[str]), 'typing.Match[str]') self.assertEqual(repr(Match[bytes]), 'typing.Match[bytes]') - def test_re_submodule(self): - from typing.re import Match, Pattern, __all__, __name__ - self.assertIs(Match, typing.Match) - self.assertIs(Pattern, typing.Pattern) - self.assertEqual(set(__all__), set(['Match', 'Pattern'])) - self.assertEqual(__name__, 'typing.re') - - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cannot_subclass(self): - with self.assertRaises(TypeError) as ex: - + with self.assertRaisesRegex( + TypeError, + r"type 're\.Match' is not an acceptable base type", + ): class A(typing.Match): pass - - self.assertEqual(str(ex.exception), - "type 're.Match' is not an acceptable base type") + with self.assertRaisesRegex( + TypeError, + r"type 're\.Pattern' is not an acceptable base type", + ): + class B(typing.Pattern): + pass class AnnotatedTests(BaseTestCase): def test_new(self): with self.assertRaisesRegex( - TypeError, - 'Type Annotated cannot be instantiated', + TypeError, 'Cannot instantiate typing.Annotated', ): Annotated() @@ -4691,12 +9461,91 @@ def test_repr(self): "typing.Annotated[typing.List[int], 4, 5]" ) + def test_dir(self): + dir_items = set(dir(Annotated[int, 4])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + '__metadata__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + def test_flatten(self): A = Annotated[Annotated[int, 4], 5] self.assertEqual(A, Annotated[int, 4, 5]) self.assertEqual(A.__metadata__, (4, 5)) self.assertEqual(A.__origin__, int) + def test_deduplicate_from_union(self): + # Regular: + self.assertEqual(get_args(Annotated[int, 1] | int), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], int]), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[int, 2] | int), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[int, 2], int]), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[str, 1] | int), + (Annotated[int, 1], Annotated[str, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[str, 1], int]), + (Annotated[int, 1], Annotated[str, 1], int)) + + # Duplicates: + self.assertEqual(Annotated[int, 1] | Annotated[int, 1] | int, + Annotated[int, 1] | int) + self.assertEqual(Union[Annotated[int, 1], Annotated[int, 1], int], + Union[Annotated[int, 1], int]) + + # Unhashable metadata: + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[int, set()] | int), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[int, set()], int]), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[str, {}] | int), + (str, Annotated[int, {}], Annotated[str, {}], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[str, {}], int]), + (str, Annotated[int, {}], Annotated[str, {}], int)) + + self.assertEqual(get_args(Annotated[int, 1] | str | Annotated[str, {}] | int), + (Annotated[int, 1], str, Annotated[str, {}], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], str, Annotated[str, {}], int]), + (Annotated[int, 1], str, Annotated[str, {}], int)) + + import dataclasses + @dataclasses.dataclass + class ValueRange: + lo: int + hi: int + v = ValueRange(1, 2) + self.assertEqual(get_args(Annotated[int, v] | None), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Union[Annotated[int, v], None]), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Optional[Annotated[int, v]]), + (Annotated[int, v], types.NoneType)) + + # Unhashable metadata duplicated: + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + Annotated[int, {}] | int) + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + int | Annotated[int, {}]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[Annotated[int, {}], int]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[int, Annotated[int, {}]]) + + def test_order_in_union(self): + expr1 = Annotated[int, 1] | str | Annotated[str, {}] | int + for args in itertools.permutations(get_args(expr1)): + with self.subTest(args=args): + self.assertEqual(expr1, reduce(operator.or_, args)) + + expr2 = Union[Annotated[int, 1], str, Annotated[str, {}], int] + for args in itertools.permutations(get_args(expr2)): + with self.subTest(args=args): + self.assertEqual(expr2, Union[args]) + def test_specialize(self): L = Annotated[List[T], "my decoration"] LI = Annotated[List[int], "my decoration"] @@ -4717,6 +9566,16 @@ def test_hash_eq(self): {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, {Annotated[int, 4, 5], Annotated[T, 4, 5]} ) + # Unhashable `metadata` raises `TypeError`: + a1 = Annotated[int, []] + with self.assertRaises(TypeError): + hash(a1) + + class A: + __hash__ = None + a2 = Annotated[int, A()] + with self.assertRaises(TypeError): + hash(a2) def test_instantiate(self): class C: @@ -4742,6 +9601,17 @@ def test_instantiate_generic(self): self.assertEqual(MyCount([4, 4, 5]), {4: 2, 5: 1}) self.assertEqual(MyCount[int]([4, 4, 5]), {4: 2, 5: 1}) + def test_instantiate_immutable(self): + class C: + def __setattr__(self, key, value): + raise Exception("should be ignored") + + A = Annotated[C, "a decoration"] + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `C` instance + # returned by the `A()` call + self.assertIsInstance(A(), C) + def test_cannot_instantiate_forward(self): A = Annotated["int", (5, 6)] with self.assertRaises(TypeError): @@ -4773,15 +9643,33 @@ class C: self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) - def test_hash_eq(self): - self.assertEqual(len({Annotated[int, 4, 5], Annotated[int, 4, 5]}), 1) - self.assertNotEqual(Annotated[int, 4, 5], Annotated[int, 5, 4]) - self.assertNotEqual(Annotated[int, 4, 5], Annotated[str, 4, 5]) - self.assertNotEqual(Annotated[int, 4], Annotated[int, 4, 4]) - self.assertEqual( - {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, - {Annotated[int, 4, 5], Annotated[T, 4, 5]} - ) + def test_special_forms_nesting(self): + # These are uncommon types and are to ensure runtime + # is lax on validation. See gh-89547 for more context. + class CF: + x: ClassVar[Final[int]] + + class FC: + x: Final[ClassVar[int]] + + class ACF: + x: Annotated[ClassVar[Final[int]], "a decoration"] + + class CAF: + x: ClassVar[Annotated[Final[int], "a decoration"]] + + class AFC: + x: Annotated[Final[ClassVar[int]], "a decoration"] + + class FAC: + x: Final[Annotated[ClassVar[int], "a decoration"]] + + self.assertEqual(get_type_hints(CF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(FC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(ACF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(CAF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(AFC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(FAC, globals())['x'], Final[ClassVar[int]]) def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): @@ -4859,6 +9747,120 @@ def test_subst(self): with self.assertRaises(TypeError): LI[None] + def test_typevar_subst(self): + dec = "a decoration" + Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T1 = TypeVar('T1') + T2 = TypeVar('T2') + + A = Annotated[tuple[*Ts], dec] + self.assertEqual(A[int], Annotated[tuple[int], dec]) + self.assertEqual(A[str, int], Annotated[tuple[str, int], dec]) + with self.assertRaises(TypeError): + Annotated[*Ts, dec] + + B = Annotated[Tuple[Unpack[Ts]], dec] + self.assertEqual(B[int], Annotated[Tuple[int], dec]) + self.assertEqual(B[str, int], Annotated[Tuple[str, int], dec]) + with self.assertRaises(TypeError): + Annotated[Unpack[Ts], dec] + + C = Annotated[tuple[T, *Ts], dec] + self.assertEqual(C[int], Annotated[tuple[int], dec]) + self.assertEqual(C[int, str], Annotated[tuple[int, str], dec]) + self.assertEqual( + C[int, str, float], + Annotated[tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + C[()] + + D = Annotated[Tuple[T, Unpack[Ts]], dec] + self.assertEqual(D[int], Annotated[Tuple[int], dec]) + self.assertEqual(D[int, str], Annotated[Tuple[int, str], dec]) + self.assertEqual( + D[int, str, float], + Annotated[Tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + D[()] + + E = Annotated[tuple[*Ts, T], dec] + self.assertEqual(E[int], Annotated[tuple[int], dec]) + self.assertEqual(E[int, str], Annotated[tuple[int, str], dec]) + self.assertEqual( + E[int, str, float], + Annotated[tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + E[()] + + F = Annotated[Tuple[Unpack[Ts], T], dec] + self.assertEqual(F[int], Annotated[Tuple[int], dec]) + self.assertEqual(F[int, str], Annotated[Tuple[int, str], dec]) + self.assertEqual( + F[int, str, float], + Annotated[Tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + F[()] + + G = Annotated[tuple[T1, *Ts, T2], dec] + self.assertEqual(G[int, str], Annotated[tuple[int, str], dec]) + self.assertEqual( + G[int, str, float], + Annotated[tuple[int, str, float], dec] + ) + self.assertEqual( + G[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec] + ) + with self.assertRaises(TypeError): + G[int] + + H = Annotated[Tuple[T1, Unpack[Ts], T2], dec] + self.assertEqual(H[int, str], Annotated[Tuple[int, str], dec]) + self.assertEqual( + H[int, str, float], + Annotated[Tuple[int, str, float], dec] + ) + self.assertEqual( + H[int, str, bool, float], + Annotated[Tuple[int, str, bool, float], dec] + ) + with self.assertRaises(TypeError): + H[int] + + # Now let's try creating an alias from an alias. + + Ts2 = TypeVarTuple('Ts2') + T3 = TypeVar('T3') + T4 = TypeVar('T4') + + # G is Annotated[tuple[T1, *Ts, T2], dec]. + I = G[T3, *Ts2, T4] + J = G[T3, Unpack[Ts2], T4] + + for x, y in [ + (I, Annotated[tuple[T3, *Ts2, T4], dec]), + (J, Annotated[tuple[T3, Unpack[Ts2], T4], dec]), + (I[int, str], Annotated[tuple[int, str], dec]), + (J[int, str], Annotated[tuple[int, str], dec]), + (I[int, str, float], Annotated[tuple[int, str, float], dec]), + (J[int, str, float], Annotated[tuple[int, str, float], dec]), + (I[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec]), + (J[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec]), + ]: + self.assertEqual(x, y) + + with self.assertRaises(TypeError): + I[int] + with self.assertRaises(TypeError): + J[int] + def test_annotated_in_other_types(self): X = List[Annotated[T, 5]] self.assertEqual(X[int], List[Annotated[int, 5]]) @@ -4868,6 +9870,53 @@ class X(Annotated[int, (1, 10)]): ... self.assertEqual(X.__mro__, (X, int, object), "Annotated should be transparent.") + def test_annotated_cached_with_types(self): + class A(str): ... + class B(str): ... + + field_a1 = Annotated[str, A("X")] + field_a2 = Annotated[str, B("X")] + a1_metadata = field_a1.__metadata__[0] + a2_metadata = field_a2.__metadata__[0] + + self.assertIs(type(a1_metadata), A) + self.assertEqual(a1_metadata, A("X")) + self.assertIs(type(a2_metadata), B) + self.assertEqual(a2_metadata, B("X")) + self.assertIsNot(type(a1_metadata), type(a2_metadata)) + + field_b1 = Annotated[str, A("Y")] + field_b2 = Annotated[str, B("Y")] + b1_metadata = field_b1.__metadata__[0] + b2_metadata = field_b2.__metadata__[0] + + self.assertIs(type(b1_metadata), A) + self.assertEqual(b1_metadata, A("Y")) + self.assertIs(type(b2_metadata), B) + self.assertEqual(b2_metadata, B("Y")) + self.assertIsNot(type(b1_metadata), type(b2_metadata)) + + field_c1 = Annotated[int, 1] + field_c2 = Annotated[int, 1.0] + field_c3 = Annotated[int, True] + + self.assertIs(type(field_c1.__metadata__[0]), int) + self.assertIs(type(field_c2.__metadata__[0]), float) + self.assertIs(type(field_c3.__metadata__[0]), bool) + + def test_forwardref_partial_evaluation(self): + # Test that Annotated partially evaluates if it contains a ForwardRef + # See: https://github.com/python/cpython/issues/137706 + def f(x: Annotated[undefined, '']): pass + + ann = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + + # Test that the attributes are retrievable from the partially evaluated annotation + x_ann = ann['x'] + self.assertIs(get_origin(x_ann), Annotated) + self.assertEqual(x_ann.__origin__, EqualToForwardRef('undefined', owner=f)) + self.assertEqual(x_ann.__metadata__, ('',)) + class TypeAliasTests(BaseTestCase): def test_canonical_usage_with_variable_annotation(self): @@ -4897,12 +9946,13 @@ def test_no_issubclass(self): issubclass(TypeAlias, Employee) def test_cannot_subclass(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeAlias'): class C(TypeAlias): pass with self.assertRaises(TypeError): - class C(type(TypeAlias)): + class D(type(TypeAlias)): pass def test_repr(self): @@ -4919,9 +9969,17 @@ def test_basic_plain(self): P = ParamSpec('P') self.assertEqual(P, P) self.assertIsInstance(P, ParamSpec) + self.assertEqual(P.__name__, 'P') + self.assertEqual(P.__module__, __name__) + + def test_basic_with_exec(self): + ns = {} + exec('from typing import ParamSpec; P = ParamSpec("P")', ns, ns) + P = ns['P'] + self.assertIsInstance(P, ParamSpec) + self.assertEqual(P.__name__, 'P') + self.assertIs(P.__module__, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_valid_uses(self): P = ParamSpec('P') T = TypeVar('T') @@ -5038,32 +10096,208 @@ class X(Generic[P, P2]): self.assertEqual(G1.__args__, ((int, str), (bytes,))) self.assertEqual(G2.__args__, ((int,), (str, bytes))) + def test_typevartuple_and_paramspecs_in_user_generics(self): + Ts = TypeVarTuple("Ts") + P = ParamSpec("P") + + class X(Generic[*Ts, P]): + f: Callable[P, int] + g: Tuple[*Ts] + + G1 = X[int, [bytes]] + self.assertEqual(G1.__args__, (int, (bytes,))) + G2 = X[int, str, [bytes]] + self.assertEqual(G2.__args__, (int, str, (bytes,))) + G3 = X[[bytes]] + self.assertEqual(G3.__args__, ((bytes,),)) + G4 = X[[]] + self.assertEqual(G4.__args__, ((),)) + with self.assertRaises(TypeError): + X[()] + + class Y(Generic[P, *Ts]): + f: Callable[P, int] + g: Tuple[*Ts] + + G1 = Y[[bytes], int] + self.assertEqual(G1.__args__, ((bytes,), int)) + G2 = Y[[bytes], int, str] + self.assertEqual(G2.__args__, ((bytes,), int, str)) + G3 = Y[[bytes]] + self.assertEqual(G3.__args__, ((bytes,),)) + G4 = Y[[]] + self.assertEqual(G4.__args__, ((),)) + with self.assertRaises(TypeError): + Y[()] + + def test_typevartuple_and_paramspecs_in_generic_aliases(self): + P = ParamSpec('P') + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + for C in Callable, collections.abc.Callable: + with self.subTest(generic=C): + A = C[P, Tuple[*Ts]] + B = A[[int, str], bytes, float] + self.assertEqual(B.__args__, (int, str, Tuple[bytes, float])) + + class X(Generic[T, P]): + pass + + A = X[Tuple[*Ts], P] + B = A[bytes, float, [int, str]] + self.assertEqual(B.__args__, (Tuple[bytes, float], (int, str,))) + + class Y(Generic[P, T]): + pass + + A = Y[P, Tuple[*Ts]] + B = A[[int, str], bytes, float] + self.assertEqual(B.__args__, ((int, str,), Tuple[bytes, float])) + + def test_var_substitution(self): + P = ParamSpec("P") + subst = P.__typing_subst__ + self.assertEqual(subst((int, str)), (int, str)) + self.assertEqual(subst([int, str]), (int, str)) + self.assertEqual(subst([None]), (type(None),)) + self.assertIs(subst(...), ...) + self.assertIs(subst(P), P) + self.assertEqual(subst(Concatenate[int, P]), Concatenate[int, P]) + def test_bad_var_substitution(self): T = TypeVar('T') P = ParamSpec('P') bad_args = (42, int, None, T, int|str, Union[int, str]) for arg in bad_args: with self.subTest(arg=arg): + with self.assertRaises(TypeError): + P.__typing_subst__(arg) with self.assertRaises(TypeError): typing.Callable[P, T][arg, str] with self.assertRaises(TypeError): collections.abc.Callable[P, T][arg, str] - def test_no_paramspec_in__parameters__(self): - # ParamSpec should not be found in __parameters__ - # of generics. Usages outside Callable, Concatenate - # and Generic are invalid. - T = TypeVar("T") - P = ParamSpec("P") - self.assertNotIn(P, List[P].__parameters__) - self.assertIn(T, Tuple[T, P].__parameters__) + def test_type_var_subst_for_other_type_vars(self): + T = TypeVar('T') + T2 = TypeVar('T2') + P = ParamSpec('P') + P2 = ParamSpec('P2') + Ts = TypeVarTuple('Ts') + + class Base(Generic[P]): + pass + + A1 = Base[T] + self.assertEqual(A1.__parameters__, (T,)) + self.assertEqual(A1.__args__, ((T,),)) + self.assertEqual(A1[int], Base[int]) + + A2 = Base[[T]] + self.assertEqual(A2.__parameters__, (T,)) + self.assertEqual(A2.__args__, ((T,),)) + self.assertEqual(A2[int], Base[int]) + + A3 = Base[[int, T]] + self.assertEqual(A3.__parameters__, (T,)) + self.assertEqual(A3.__args__, ((int, T),)) + self.assertEqual(A3[str], Base[[int, str]]) + + A4 = Base[[T, int, T2]] + self.assertEqual(A4.__parameters__, (T, T2)) + self.assertEqual(A4.__args__, ((T, int, T2),)) + self.assertEqual(A4[str, bool], Base[[str, int, bool]]) + + A5 = Base[[*Ts, int]] + self.assertEqual(A5.__parameters__, (Ts,)) + self.assertEqual(A5.__args__, ((*Ts, int),)) + self.assertEqual(A5[str, bool], Base[[str, bool, int]]) + + A5_2 = Base[[int, *Ts]] + self.assertEqual(A5_2.__parameters__, (Ts,)) + self.assertEqual(A5_2.__args__, ((int, *Ts),)) + self.assertEqual(A5_2[str, bool], Base[[int, str, bool]]) + + A6 = Base[[T, *Ts]] + self.assertEqual(A6.__parameters__, (T, Ts)) + self.assertEqual(A6.__args__, ((T, *Ts),)) + self.assertEqual(A6[int, str, bool], Base[[int, str, bool]]) + + A7 = Base[[T, T]] + self.assertEqual(A7.__parameters__, (T,)) + self.assertEqual(A7.__args__, ((T, T),)) + self.assertEqual(A7[int], Base[[int, int]]) + + A8 = Base[[T, list[T]]] + self.assertEqual(A8.__parameters__, (T,)) + self.assertEqual(A8.__args__, ((T, list[T]),)) + self.assertEqual(A8[int], Base[[int, list[int]]]) + + A9 = Base[[Tuple[*Ts], *Ts]] + self.assertEqual(A9.__parameters__, (Ts,)) + self.assertEqual(A9.__args__, ((Tuple[*Ts], *Ts),)) + self.assertEqual(A9[int, str], Base[Tuple[int, str], int, str]) + + A10 = Base[P2] + self.assertEqual(A10.__parameters__, (P2,)) + self.assertEqual(A10.__args__, (P2,)) + self.assertEqual(A10[[int, str]], Base[[int, str]]) + + class DoubleP(Generic[P, P2]): + pass + + B1 = DoubleP[P, P2] + self.assertEqual(B1.__parameters__, (P, P2)) + self.assertEqual(B1.__args__, (P, P2)) + self.assertEqual(B1[[int, str], [bool]], DoubleP[[int, str], [bool]]) + self.assertEqual(B1[[], []], DoubleP[[], []]) + + B2 = DoubleP[[int, str], P2] + self.assertEqual(B2.__parameters__, (P2,)) + self.assertEqual(B2.__args__, ((int, str), P2)) + self.assertEqual(B2[[bool, bool]], DoubleP[[int, str], [bool, bool]]) + self.assertEqual(B2[[]], DoubleP[[int, str], []]) + + B3 = DoubleP[P, [bool, bool]] + self.assertEqual(B3.__parameters__, (P,)) + self.assertEqual(B3.__args__, (P, (bool, bool))) + self.assertEqual(B3[[int, str]], DoubleP[[int, str], [bool, bool]]) + self.assertEqual(B3[[]], DoubleP[[], [bool, bool]]) + + B4 = DoubleP[[T, int], [bool, T2]] + self.assertEqual(B4.__parameters__, (T, T2)) + self.assertEqual(B4.__args__, ((T, int), (bool, T2))) + self.assertEqual(B4[str, float], DoubleP[[str, int], [bool, float]]) + + B5 = DoubleP[[*Ts, int], [bool, T2]] + self.assertEqual(B5.__parameters__, (Ts, T2)) + self.assertEqual(B5.__args__, ((*Ts, int), (bool, T2))) + self.assertEqual(B5[str, bytes, float], + DoubleP[[str, bytes, int], [bool, float]]) + + B6 = DoubleP[[T, int], [bool, *Ts]] + self.assertEqual(B6.__parameters__, (T, Ts)) + self.assertEqual(B6.__args__, ((T, int), (bool, *Ts))) + self.assertEqual(B6[str, bytes, float], + DoubleP[[str, int], [bool, bytes, float]]) + + class PandT(Generic[P, T]): + pass - # Test for consistency with builtin generics. - self.assertNotIn(P, list[P].__parameters__) - self.assertIn(T, tuple[T, P].__parameters__) + C1 = PandT[P, T] + self.assertEqual(C1.__parameters__, (P, T)) + self.assertEqual(C1.__args__, (P, T)) + self.assertEqual(C1[[int, str], bool], PandT[[int, str], bool]) - self.assertNotIn(P, (list[P] | int).__parameters__) - self.assertIn(T, (tuple[T, P] | int).__parameters__) + C2 = PandT[[int, T], T] + self.assertEqual(C2.__parameters__, (T,)) + self.assertEqual(C2.__args__, ((int, T), T)) + self.assertEqual(C2[str], PandT[[int, str], str]) + + C3 = PandT[[int, *Ts], T] + self.assertEqual(C3.__parameters__, (Ts, T)) + self.assertEqual(C3.__args__, ((int, *Ts), T)) + self.assertEqual(C3[str, bool, bytes], PandT[[int, str, bool], bytes]) def test_paramspec_in_nested_generics(self): # Although ParamSpec should not be found in __parameters__ of most @@ -5104,6 +10338,24 @@ def test_paramspec_gets_copied(self): self.assertEqual(C2[Concatenate[str, P2]].__parameters__, (P2,)) self.assertEqual(C2[Concatenate[T, P2]].__parameters__, (T, P2)) + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpec'): + class C(ParamSpec): pass + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpecArgs'): + class D(ParamSpecArgs): pass + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpecKwargs'): + class E(ParamSpecKwargs): pass + P = ParamSpec('P') + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'ParamSpec'): + class F(P): pass + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'ParamSpecArgs'): + class G(P.args): pass + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'ParamSpecKwargs'): + class H(P.kwargs): pass + class ConcatenateTests(BaseTestCase): def test_basics(self): @@ -5112,6 +10364,15 @@ class MyClass: ... c = Concatenate[MyClass, P] self.assertNotEqual(c, Concatenate) + def test_dir(self): + P = ParamSpec('P') + dir_items = set(dir(Concatenate[int, P])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + def test_valid_uses(self): P = ParamSpec('P') T = TypeVar('T') @@ -5130,6 +10391,18 @@ def test_valid_uses(self): self.assertEqual(C4.__args__, (Concatenate[int, T, P], T)) self.assertEqual(C4.__parameters__, (T, P)) + def test_invalid_uses(self): + with self.assertRaisesRegex(TypeError, 'Concatenate of no types'): + Concatenate[()] + with self.assertRaisesRegex( + TypeError, + ( + 'The last parameter to Concatenate should be a ' + 'ParamSpec variable or ellipsis' + ), + ): + Concatenate[int] + def test_var_substitution(self): T = TypeVar('T') P = ParamSpec('P') @@ -5140,8 +10413,7 @@ def test_var_substitution(self): self.assertEqual(C[int, []], (int,)) self.assertEqual(C[int, Concatenate[str, P2]], Concatenate[int, str, P2]) - with self.assertRaises(TypeError): - C[int, ...] + self.assertEqual(C[int, ...], Concatenate[int, ...]) C = Concatenate[int, P] self.assertEqual(C[P2], Concatenate[int, P2]) @@ -5149,8 +10421,8 @@ def test_var_substitution(self): self.assertEqual(C[str, float], (int, str, float)) self.assertEqual(C[[]], (int,)) self.assertEqual(C[Concatenate[str, P2]], Concatenate[int, str, P2]) - with self.assertRaises(TypeError): - C[...] + self.assertEqual(C[...], Concatenate[int, ...]) + class TypeGuardTests(BaseTestCase): def test_basics(self): @@ -5159,6 +10431,9 @@ def test_basics(self): def foo(arg) -> TypeGuard[int]: ... self.assertEqual(gth(foo), {'return': TypeGuard[int]}) + with self.assertRaises(TypeError): + TypeGuard[int, str] + def test_repr(self): self.assertEqual(repr(TypeGuard), 'typing.TypeGuard') cv = TypeGuard[int] @@ -5169,11 +10444,19 @@ def test_repr(self): self.assertEqual(repr(cv), 'typing.TypeGuard[tuple[int]]') def test_cannot_subclass(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): class C(type(TypeGuard)): pass - with self.assertRaises(TypeError): - class C(type(TypeGuard[int])): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(TypeGuard[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeGuard'): + class E(TypeGuard): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeGuard\[int\]'): + class F(TypeGuard[int]): pass def test_cannot_init(self): @@ -5191,14 +10474,62 @@ def test_no_isinstance(self): issubclass(int, TypeGuard) +class TypeIsTests(BaseTestCase): + def test_basics(self): + TypeIs[int] # OK + + def foo(arg) -> TypeIs[int]: ... + self.assertEqual(gth(foo), {'return': TypeIs[int]}) + + with self.assertRaises(TypeError): + TypeIs[int, str] + + def test_repr(self): + self.assertEqual(repr(TypeIs), 'typing.TypeIs') + cv = TypeIs[int] + self.assertEqual(repr(cv), 'typing.TypeIs[int]') + cv = TypeIs[Employee] + self.assertEqual(repr(cv), 'typing.TypeIs[%s.Employee]' % __name__) + cv = TypeIs[tuple[int]] + self.assertEqual(repr(cv), 'typing.TypeIs[tuple[int]]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(TypeIs)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(TypeIs[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs'): + class E(TypeIs): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs\[int\]'): + class F(TypeIs[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + TypeIs() + with self.assertRaises(TypeError): + type(TypeIs)() + with self.assertRaises(TypeError): + type(TypeIs[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, TypeIs[int]) + with self.assertRaises(TypeError): + issubclass(int, TypeIs) + + SpecialAttrsP = typing.ParamSpec('SpecialAttrsP') SpecialAttrsT = typing.TypeVar('SpecialAttrsT', int, float, complex) class SpecialAttrsTests(BaseTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_special_attrs(self): cls_to_check = { # ABC classes @@ -5242,7 +10573,7 @@ def test_special_attrs(self): typing.ValuesView: 'ValuesView', # Subscribed ABC classes typing.AbstractSet[Any]: 'AbstractSet', - typing.AsyncContextManager[Any]: 'AsyncContextManager', + typing.AsyncContextManager[Any, Any]: 'AsyncContextManager', typing.AsyncGenerator[Any, Any]: 'AsyncGenerator', typing.AsyncIterable[Any]: 'AsyncIterable', typing.AsyncIterator[Any]: 'AsyncIterator', @@ -5252,7 +10583,7 @@ def test_special_attrs(self): typing.ChainMap[Any, Any]: 'ChainMap', typing.Collection[Any]: 'Collection', typing.Container[Any]: 'Container', - typing.ContextManager[Any]: 'ContextManager', + typing.ContextManager[Any, Any]: 'ContextManager', typing.Coroutine[Any, Any, Any]: 'Coroutine', typing.Counter[Any]: 'Counter', typing.DefaultDict[Any, Any]: 'DefaultDict', @@ -5284,29 +10615,31 @@ def test_special_attrs(self): typing.ClassVar: 'ClassVar', typing.Concatenate: 'Concatenate', typing.Final: 'Final', - typing.ForwardRef: 'ForwardRef', typing.Literal: 'Literal', typing.NewType: 'NewType', typing.NoReturn: 'NoReturn', + typing.Never: 'Never', typing.Optional: 'Optional', typing.TypeAlias: 'TypeAlias', typing.TypeGuard: 'TypeGuard', + typing.TypeIs: 'TypeIs', typing.TypeVar: 'TypeVar', - typing.Union: 'Union', - # Subscribed special forms + typing.Self: 'Self', + # Subscripted special forms typing.Annotated[Any, "Annotation"]: 'Annotated', + typing.Annotated[int, 'Annotation']: 'Annotated', typing.ClassVar[Any]: 'ClassVar', typing.Concatenate[Any, SpecialAttrsP]: 'Concatenate', typing.Final[Any]: 'Final', typing.Literal[Any]: 'Literal', typing.Literal[1, 2]: 'Literal', typing.Literal[True, 2]: 'Literal', - typing.Optional[Any]: 'Optional', + typing.Optional[Any]: 'Union', typing.TypeGuard[Any]: 'TypeGuard', + typing.TypeIs[Any]: 'TypeIs', typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', # Incompatible special forms (tested in test_special_attrs2) - # - typing.ForwardRef('set[Any]') # - typing.NewType('TypeName', Any) # - typing.ParamSpec('SpecialAttrsP') # - typing.TypeVar('T') @@ -5320,25 +10653,14 @@ def test_special_attrs(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(cls, proto) loaded = pickle.loads(s) - self.assertIs(cls, loaded) + if isinstance(cls, Union): + self.assertEqual(cls, loaded) + else: + self.assertIs(cls, loaded) TypeName = typing.NewType('SpecialAttrsTests.TypeName', Any) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_special_attrs2(self): - # Forward refs provide a different introspection API. __name__ and - # __qualname__ make little sense for forward refs as they can store - # complex typing expressions. - fr = typing.ForwardRef('set[Any]') - self.assertFalse(hasattr(fr, '__name__')) - self.assertFalse(hasattr(fr, '__qualname__')) - self.assertEqual(fr.__module__, 'typing') - # Forward refs are currently unpicklable. - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError) as exc: - pickle.dumps(fr, proto) - self.assertEqual(SpecialAttrsTests.TypeName.__name__, 'TypeName') self.assertEqual( SpecialAttrsTests.TypeName.__qualname__, @@ -5359,7 +10681,7 @@ def test_special_attrs2(self): # to the variable name to which it is assigned". Thus, providing # __qualname__ is unnecessary. self.assertEqual(SpecialAttrsT.__name__, 'SpecialAttrsT') - self.assertFalse(hasattr(SpecialAttrsT, '__qualname__')) + self.assertNotHasAttr(SpecialAttrsT, '__qualname__') self.assertEqual(SpecialAttrsT.__module__, __name__) # Module-level type variables are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -5368,7 +10690,7 @@ def test_special_attrs2(self): self.assertIs(SpecialAttrsT, loaded) self.assertEqual(SpecialAttrsP.__name__, 'SpecialAttrsP') - self.assertFalse(hasattr(SpecialAttrsP, '__qualname__')) + self.assertNotHasAttr(SpecialAttrsP, '__qualname__') self.assertEqual(SpecialAttrsP.__module__, __name__) # Module-level ParamSpecs are picklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -5381,10 +10703,153 @@ class Foo(Generic[T]): def bar(self): pass baz = 3 + __magic__ = 4 + # The class attributes of the original class should be visible even # in dir() of the GenericAlias. See bpo-45755. - self.assertIn('bar', dir(Foo[int])) - self.assertIn('baz', dir(Foo[int])) + dir_items = set(dir(Foo[int])) + for required_item in [ + 'bar', 'baz', + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + self.assertNotIn('__magic__', dir_items) + + +class RevealTypeTests(BaseTestCase): + def test_reveal_type(self): + obj = object() + with captured_stderr() as stderr: + self.assertIs(obj, reveal_type(obj)) + self.assertEqual(stderr.getvalue(), "Runtime type is 'object'\n") + + +class DataclassTransformTests(BaseTestCase): + def test_decorator(self): + def create_model(*, frozen: bool = False, kw_only: bool = True): + return lambda cls: cls + + decorated = dataclass_transform(kw_only_default=True, order_default=False)(create_model) + + class CustomerModel: + id: int + + self.assertIs(decorated, create_model) + self.assertEqual( + decorated.__dataclass_transform__, + { + "eq_default": True, + "order_default": False, + "kw_only_default": True, + "frozen_default": False, + "field_specifiers": (), + "kwargs": {}, + } + ) + self.assertIs( + decorated(frozen=True, kw_only=False)(CustomerModel), + CustomerModel + ) + + def test_base_class(self): + class ModelBase: + def __init_subclass__(cls, *, frozen: bool = False): ... + + Decorated = dataclass_transform( + eq_default=True, + order_default=True, + # Arbitrary unrecognized kwargs are accepted at runtime. + make_everything_awesome=True, + )(ModelBase) + + class CustomerModel(Decorated, frozen=True): + id: int + + self.assertIs(Decorated, ModelBase) + self.assertEqual( + Decorated.__dataclass_transform__, + { + "eq_default": True, + "order_default": True, + "kw_only_default": False, + "frozen_default": False, + "field_specifiers": (), + "kwargs": {"make_everything_awesome": True}, + } + ) + self.assertIsSubclass(CustomerModel, Decorated) + + def test_metaclass(self): + class Field: ... + + class ModelMeta(type): + def __new__( + cls, name, bases, namespace, *, init: bool = True, + ): + return super().__new__(cls, name, bases, namespace) + + Decorated = dataclass_transform( + order_default=True, frozen_default=True, field_specifiers=(Field,) + )(ModelMeta) + + class ModelBase(metaclass=Decorated): ... + + class CustomerModel(ModelBase, init=False): + id: int + + self.assertIs(Decorated, ModelMeta) + self.assertEqual( + Decorated.__dataclass_transform__, + { + "eq_default": True, + "order_default": True, + "kw_only_default": False, + "frozen_default": True, + "field_specifiers": (Field,), + "kwargs": {}, + } + ) + self.assertIsInstance(CustomerModel, Decorated) + + +class NoDefaultTests(BaseTestCase): + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(NoDefault, proto) + loaded = pickle.loads(s) + self.assertIs(NoDefault, loaded) + + def test_constructor(self): + self.assertIs(NoDefault, type(NoDefault)()) + with self.assertRaises(TypeError): + type(NoDefault)(1) + + def test_repr(self): + self.assertEqual(repr(NoDefault), 'typing.NoDefault') + + @requires_docstrings + def test_doc(self): + self.assertIsInstance(NoDefault.__doc__, str) + + def test_class(self): + self.assertIs(NoDefault.__class__, type(NoDefault)) + + def test_no_call(self): + with self.assertRaises(TypeError): + NoDefault() + + def test_no_attributes(self): + with self.assertRaises(AttributeError): + NoDefault.foo = 3 + with self.assertRaises(AttributeError): + NoDefault.foo + + # TypeError is consistent with the behavior of NoneType + with self.assertRaises(TypeError): + type(NoDefault).foo = 3 + with self.assertRaises(AttributeError): + type(NoDefault).foo class AllTests(BaseTestCase): @@ -5400,7 +10865,7 @@ def test_all(self): # Context managers. self.assertIn('ContextManager', a) self.assertIn('AsyncContextManager', a) - # Check that io and re are not exported. + # Check that former namespaces io and re are not exported. self.assertNotIn('io', a) self.assertNotIn('re', a) # Spot-check that stdlib modules aren't exported. @@ -5413,6 +10878,10 @@ def test_all(self): self.assertIn('SupportsComplex', a) def test_all_exported_names(self): + # ensure all dynamically created objects are actualised + for name in typing.__all__: + getattr(typing, name) + actual_all = set(typing.__all__) computed_all = { k for k, v in vars(typing).items() @@ -5420,10 +10889,6 @@ def test_all_exported_names(self): if k in actual_all or ( # avoid private names not k.startswith('_') and - # avoid things in the io / re typing submodules - k not in typing.io.__all__ and - k not in typing.re.__all__ and - k not in {'io', 're'} and # there's a few types and metaclasses that aren't exported not k.endswith(('Meta', '_contra', '_co')) and not k.upper() == k and @@ -5434,6 +10899,74 @@ def test_all_exported_names(self): self.assertSetEqual(computed_all, actual_all) +class TypeIterationTests(BaseTestCase): + _UNITERABLE_TYPES = ( + Any, + Union, + Union[str, int], + Union[str, T], + List, + Tuple, + Callable, + Callable[..., T], + Callable[[T], str], + Annotated, + Annotated[T, ''], + ) + + def test_cannot_iterate(self): + expected_error_regex = "object is not iterable" + for test_type in self._UNITERABLE_TYPES: + with self.subTest(type=test_type): + with self.assertRaisesRegex(TypeError, expected_error_regex): + iter(test_type) + with self.assertRaisesRegex(TypeError, expected_error_regex): + list(test_type) + with self.assertRaisesRegex(TypeError, expected_error_regex): + for _ in test_type: + pass + + def test_is_not_instance_of_iterable(self): + for type_to_test in self._UNITERABLE_TYPES: + self.assertNotIsInstance(type_to_test, collections.abc.Iterable) + + +class UnionGenericAliasTests(BaseTestCase): + def test_constructor(self): + # Used e.g. in typer, pydantic + with self.assertWarns(DeprecationWarning): + inst = typing._UnionGenericAlias(typing.Union, (int, str)) + self.assertEqual(inst, int | str) + with self.assertWarns(DeprecationWarning): + # name is accepted but ignored + inst = typing._UnionGenericAlias(typing.Union, (int, None), name="Optional") + self.assertEqual(inst, int | None) + + def test_isinstance(self): + # Used e.g. in pydantic + with self.assertWarns(DeprecationWarning): + self.assertTrue(isinstance(Union[int, str], typing._UnionGenericAlias)) + with self.assertWarns(DeprecationWarning): + self.assertFalse(isinstance(int, typing._UnionGenericAlias)) + + def test_eq(self): + # type(t) == _UnionGenericAlias is used in vyos + with self.assertWarns(DeprecationWarning): + self.assertEqual(Union, typing._UnionGenericAlias) + with self.assertWarns(DeprecationWarning): + self.assertEqual(typing._UnionGenericAlias, typing._UnionGenericAlias) + with self.assertWarns(DeprecationWarning): + self.assertNotEqual(int, typing._UnionGenericAlias) + + def test_hashable(self): + self.assertEqual(hash(typing._UnionGenericAlias), hash(Union)) + + +def load_tests(loader, tests, pattern): + import doctest + tests.addTests(doctest.DocTestSuite(typing)) + return tests + if __name__ == '__main__': main() diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index 6d082a09423..10f262cff12 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -10,6 +10,7 @@ import ast import unittest import unicodedata +import urllib.error from test import support from http.client import HTTPException @@ -102,8 +103,6 @@ def test_cjk_unified_ideographs(self): self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D") self.checkletter("CJK UNIFIED IDEOGRAPH-3134A", "\U0003134A") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bmp_characters(self): for code in range(0x10000): char = chr(code) @@ -117,8 +116,7 @@ def test_misc_symbols(self): self.checkletter("HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK", "\uFF9F") self.checkletter("FULLWIDTH LATIN SMALL LETTER A", "\uFF41") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_aliases(self): # Check that the aliases defined in the NameAliases.txt file work. # This should be updated when new aliases are added or the file @@ -145,8 +143,6 @@ def test_aliases(self): with self.assertRaises(KeyError): unicodedata.ucd_3_2_0.lookup(alias) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_aliases_names_in_pua_range(self): # We are storing aliases in the PUA 15, but their names shouldn't leak for cp in range(0xf0000, 0xf0100): @@ -154,8 +150,6 @@ def test_aliases_names_in_pua_range(self): unicodedata.name(chr(cp)) self.assertEqual(str(cm.exception), 'no such name') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_named_sequences_names_in_pua_range(self): # We are storing named seq in the PUA 15, but their names shouldn't leak for cp in range(0xf0100, 0xf0fff): @@ -163,8 +157,7 @@ def test_named_sequences_names_in_pua_range(self): unicodedata.name(chr(cp)) self.assertEqual(str(cm.exception), 'no such name') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_sequences_sample(self): # Check a few named sequences. See #12753. sequences = [ @@ -181,6 +174,7 @@ def test_named_sequences_sample(self): with self.assertRaises(KeyError): unicodedata.ucd_3_2_0.lookup(seqname) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_sequences_full(self): # Check all the named sequences def check_version(testfile): @@ -191,23 +185,24 @@ def check_version(testfile): try: testdata = support.open_urlresource(url, encoding="utf-8", check=check_version) - except (OSError, HTTPException): - self.skipTest("Could not retrieve " + url) - self.addCleanup(testdata.close) - for line in testdata: - line = line.strip() - if not line or line.startswith('#'): - continue - seqname, codepoints = line.split(';') - codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split()) - self.assertEqual(unicodedata.lookup(seqname), codepoints) - with self.assertRaises(SyntaxError): - self.checkletter(seqname, None) - with self.assertRaises(KeyError): - unicodedata.ucd_3_2_0.lookup(seqname) + except urllib.error.HTTPError as exc: + exc.close() + self.skipTest(f"Could not retrieve {url}: {exc!r}") + except (OSError, HTTPException) as exc: + self.skipTest(f"Could not retrieve {url}: {exc!r}") + with testdata: + for line in testdata: + line = line.strip() + if not line or line.startswith('#'): + continue + seqname, codepoints = line.split(';') + codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split()) + self.assertEqual(unicodedata.lookup(seqname), codepoints) + with self.assertRaises(SyntaxError): + self.checkletter(seqname, None) + with self.assertRaises(KeyError): + unicodedata.ucd_3_2_0.lookup(seqname) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): self.assertRaises(TypeError, unicodedata.name) self.assertRaises(TypeError, unicodedata.name, 'xx') diff --git a/Lib/test/test_unary.py b/Lib/test/test_unary.py index c3c17cc9f61..a45fbf6bd6b 100644 --- a/Lib/test/test_unary.py +++ b/Lib/test/test_unary.py @@ -8,7 +8,6 @@ def test_negative(self): self.assertTrue(-2 == 0 - 2) self.assertEqual(-0, 0) self.assertEqual(--2, 2) - self.assertTrue(-2 == 0 - 2) self.assertTrue(-2.0 == 0 - 2.0) self.assertTrue(-2j == 0 - 2j) @@ -16,15 +15,13 @@ def test_positive(self): self.assertEqual(+2, 2) self.assertEqual(+0, 0) self.assertEqual(++2, 2) - self.assertEqual(+2, 2) self.assertEqual(+2.0, 2.0) self.assertEqual(+2j, 2j) def test_invert(self): - self.assertTrue(-2 == 0 - 2) - self.assertEqual(-0, 0) - self.assertEqual(--2, 2) - self.assertTrue(-2 == 0 - 2) + self.assertTrue(~2 == -(2+1)) + self.assertEqual(~0, -1) + self.assertEqual(~~2, 2) def test_no_overflow(self): nines = "9" * 32 diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py deleted file mode 100644 index 17c9f01cd81..00000000000 --- a/Lib/test/test_unicode.py +++ /dev/null @@ -1,2774 +0,0 @@ -""" Test script for the Unicode implementation. - -Written by Marc-Andre Lemburg (mal@lemburg.com). - -(c) Copyright CNRI, All Rights Reserved. NO WARRANTY. - -""" -import _string -import codecs -import itertools -import operator -import pickle -import struct -import sys -import textwrap -import unicodedata -import unittest -import warnings -from test.support import warnings_helper -from test import support, string_tests -from test.support.script_helper import assert_python_failure - -try: - import _testcapi -except ImportError: - _testcapi = None - -# Error handling (bad decoder return) -def search_function(encoding): - def decode1(input, errors="strict"): - return 42 # not a tuple - def encode1(input, errors="strict"): - return 42 # not a tuple - def encode2(input, errors="strict"): - return (42, 42) # no unicode - def decode2(input, errors="strict"): - return (42, 42) # no unicode - if encoding=="test.unicode1": - return (encode1, decode1, None, None) - elif encoding=="test.unicode2": - return (encode2, decode2, None, None) - else: - return None - -def duplicate_string(text): - """ - Try to get a fresh clone of the specified text: - new object with a reference count of 1. - - This is a best-effort: latin1 single letters and the empty - string ('') are singletons and cannot be cloned. - """ - return text.encode().decode() - -class StrSubclass(str): - pass - -class UnicodeTest(string_tests.CommonTest, - string_tests.MixinStrUnicodeUserStringTest, - string_tests.MixinStrUnicodeTest, - unittest.TestCase): - - type2test = str - - def setUp(self): - codecs.register(search_function) - self.addCleanup(codecs.unregister, search_function) - - def checkequalnofix(self, result, object, methodname, *args): - method = getattr(object, methodname) - realresult = method(*args) - self.assertEqual(realresult, result) - self.assertTrue(type(realresult) is type(result)) - - # if the original is returned make sure that - # this doesn't happen with subclasses - if realresult is object: - class usub(str): - def __repr__(self): - return 'usub(%r)' % str.__repr__(self) - object = usub(object) - method = getattr(object, methodname) - realresult = method(*args) - self.assertEqual(realresult, result) - self.assertTrue(object is not realresult) - - def test_literals(self): - self.assertEqual('\xff', '\u00ff') - self.assertEqual('\uffff', '\U0000ffff') - self.assertRaises(SyntaxError, eval, '\'\\Ufffffffe\'') - self.assertRaises(SyntaxError, eval, '\'\\Uffffffff\'') - self.assertRaises(SyntaxError, eval, '\'\\U%08x\'' % 0x110000) - # raw strings should not have unicode escapes - self.assertNotEqual(r"\u0020", " ") - - def test_ascii(self): - self.assertEqual(ascii('abc'), "'abc'") - self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") - self.assertEqual(ascii('ab\\'), "'ab\\\\'") - self.assertEqual(ascii('\\c'), "'\\\\c'") - self.assertEqual(ascii('\\'), "'\\\\'") - self.assertEqual(ascii('\n'), "'\\n'") - self.assertEqual(ascii('\r'), "'\\r'") - self.assertEqual(ascii('\t'), "'\\t'") - self.assertEqual(ascii('\b'), "'\\x08'") - self.assertEqual(ascii("'\""), """'\\'"'""") - self.assertEqual(ascii("'\""), """'\\'"'""") - self.assertEqual(ascii("'"), '''"'"''') - self.assertEqual(ascii('"'), """'"'""") - latin1repr = ( - "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" - "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" - "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" - "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" - "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" - "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" - "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9" - "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7" - "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5" - "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3" - "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1" - "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef" - "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd" - "\\xfe\\xff'") - testrepr = ascii(''.join(map(chr, range(256)))) - self.assertEqual(testrepr, latin1repr) - # Test ascii works on wide unicode escapes without overflow. - self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), - ascii("\U00010000" * 39 + "\uffff" * 4096)) - - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, ascii, WrongRepr()) - - def test_repr(self): - # Test basic sanity of repr() - self.assertEqual(repr('abc'), "'abc'") - self.assertEqual(repr('ab\\c'), "'ab\\\\c'") - self.assertEqual(repr('ab\\'), "'ab\\\\'") - self.assertEqual(repr('\\c'), "'\\\\c'") - self.assertEqual(repr('\\'), "'\\\\'") - self.assertEqual(repr('\n'), "'\\n'") - self.assertEqual(repr('\r'), "'\\r'") - self.assertEqual(repr('\t'), "'\\t'") - self.assertEqual(repr('\b'), "'\\x08'") - self.assertEqual(repr("'\""), """'\\'"'""") - self.assertEqual(repr("'\""), """'\\'"'""") - self.assertEqual(repr("'"), '''"'"''') - self.assertEqual(repr('"'), """'"'""") - latin1repr = ( - "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r" - "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a" - "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI" - "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f" - "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d" - "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b" - "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" - "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" - "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5" - "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3" - "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1" - "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef" - "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd" - "\xfe\xff'") - testrepr = repr(''.join(map(chr, range(256)))) - self.assertEqual(testrepr, latin1repr) - # Test repr works on wide unicode escapes without overflow. - self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), - repr("\U00010000" * 39 + "\uffff" * 4096)) - - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, repr, WrongRepr()) - - def test_iterators(self): - # Make sure unicode objects have an __iter__ method - it = "\u1111\u2222\u3333".__iter__() - self.assertEqual(next(it), "\u1111") - self.assertEqual(next(it), "\u2222") - self.assertEqual(next(it), "\u3333") - self.assertRaises(StopIteration, next, it) - - def test_iterators_invocation(self): - cases = [type(iter('abc')), type(iter('🚀'))] - for cls in cases: - with self.subTest(cls=cls): - self.assertRaises(TypeError, cls) - - def test_iteration(self): - cases = ['abc', '🚀🚀🚀', "\u1111\u2222\u3333"] - for case in cases: - with self.subTest(string=case): - self.assertEqual(case, "".join(iter(case))) - - def test_exhausted_iterator(self): - cases = ['abc', '🚀🚀🚀', "\u1111\u2222\u3333"] - for case in cases: - with self.subTest(case=case): - iterator = iter(case) - tuple(iterator) - self.assertRaises(StopIteration, next, iterator) - - def test_pickle_iterator(self): - cases = ['abc', '🚀🚀🚀', "\u1111\u2222\u3333"] - for case in cases: - with self.subTest(case=case): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - it = iter(case) - with self.subTest(proto=proto): - pickled = "".join(pickle.loads(pickle.dumps(it, proto))) - self.assertEqual(case, pickled) - - def test_count(self): - string_tests.CommonTest.test_count(self) - # check mixed argument types - self.checkequalnofix(3, 'aaa', 'count', 'a') - self.checkequalnofix(0, 'aaa', 'count', 'b') - self.checkequalnofix(3, 'aaa', 'count', 'a') - self.checkequalnofix(0, 'aaa', 'count', 'b') - self.checkequalnofix(0, 'aaa', 'count', 'b') - self.checkequalnofix(1, 'aaa', 'count', 'a', -1) - self.checkequalnofix(3, 'aaa', 'count', 'a', -10) - self.checkequalnofix(2, 'aaa', 'count', 'a', 0, -1) - self.checkequalnofix(0, 'aaa', 'count', 'a', 0, -10) - # test mixed kinds - self.checkequal(10, '\u0102' + 'a' * 10, 'count', 'a') - self.checkequal(10, '\U00100304' + 'a' * 10, 'count', 'a') - self.checkequal(10, '\U00100304' + '\u0102' * 10, 'count', '\u0102') - self.checkequal(0, 'a' * 10, 'count', '\u0102') - self.checkequal(0, 'a' * 10, 'count', '\U00100304') - self.checkequal(0, '\u0102' * 10, 'count', '\U00100304') - self.checkequal(10, '\u0102' + 'a_' * 10, 'count', 'a_') - self.checkequal(10, '\U00100304' + 'a_' * 10, 'count', 'a_') - self.checkequal(10, '\U00100304' + '\u0102_' * 10, 'count', '\u0102_') - self.checkequal(0, 'a' * 10, 'count', 'a\u0102') - self.checkequal(0, 'a' * 10, 'count', 'a\U00100304') - self.checkequal(0, '\u0102' * 10, 'count', '\u0102\U00100304') - # test subclass - class MyStr(str): - pass - self.checkequal(3, MyStr('aaa'), 'count', 'a') - - def test_find(self): - string_tests.CommonTest.test_find(self) - # test implementation details of the memchr fast path - self.checkequal(100, 'a' * 100 + '\u0102', 'find', '\u0102') - self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0201') - self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0120') - self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0220') - self.checkequal(100, 'a' * 100 + '\U00100304', 'find', '\U00100304') - self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00100204') - self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00102004') - # check mixed argument types - self.checkequalnofix(0, 'abcdefghiabc', 'find', 'abc') - self.checkequalnofix(9, 'abcdefghiabc', 'find', 'abc', 1) - self.checkequalnofix(-1, 'abcdefghiabc', 'find', 'def', 4) - - # test utf-8 non-ascii char - self.checkequal(0, 'тест', 'find', 'т') - self.checkequal(3, 'тест', 'find', 'т', 1) - self.checkequal(-1, 'тест', 'find', 'т', 1, 3) - self.checkequal(-1, 'тест', 'find', 'e') # english `e` - # test utf-8 non-ascii slice - self.checkequal(1, 'тест тест', 'find', 'ес') - self.checkequal(1, 'тест тест', 'find', 'ес', 1) - self.checkequal(1, 'тест тест', 'find', 'ес', 1, 3) - self.checkequal(6, 'тест тест', 'find', 'ес', 2) - self.checkequal(-1, 'тест тест', 'find', 'ес', 6, 7) - self.checkequal(-1, 'тест тест', 'find', 'ес', 7) - self.checkequal(-1, 'тест тест', 'find', 'ec') # english `ec` - - self.assertRaises(TypeError, 'hello'.find) - self.assertRaises(TypeError, 'hello'.find, 42) - # test mixed kinds - self.checkequal(100, '\u0102' * 100 + 'a', 'find', 'a') - self.checkequal(100, '\U00100304' * 100 + 'a', 'find', 'a') - self.checkequal(100, '\U00100304' * 100 + '\u0102', 'find', '\u0102') - self.checkequal(-1, 'a' * 100, 'find', '\u0102') - self.checkequal(-1, 'a' * 100, 'find', '\U00100304') - self.checkequal(-1, '\u0102' * 100, 'find', '\U00100304') - self.checkequal(100, '\u0102' * 100 + 'a_', 'find', 'a_') - self.checkequal(100, '\U00100304' * 100 + 'a_', 'find', 'a_') - self.checkequal(100, '\U00100304' * 100 + '\u0102_', 'find', '\u0102_') - self.checkequal(-1, 'a' * 100, 'find', 'a\u0102') - self.checkequal(-1, 'a' * 100, 'find', 'a\U00100304') - self.checkequal(-1, '\u0102' * 100, 'find', '\u0102\U00100304') - - def test_rfind(self): - string_tests.CommonTest.test_rfind(self) - # test implementation details of the memrchr fast path - self.checkequal(0, '\u0102' + 'a' * 100 , 'rfind', '\u0102') - self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0201') - self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0120') - self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0220') - self.checkequal(0, '\U00100304' + 'a' * 100, 'rfind', '\U00100304') - self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00100204') - self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00102004') - # check mixed argument types - self.checkequalnofix(9, 'abcdefghiabc', 'rfind', 'abc') - self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '') - self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '') - # test utf-8 non-ascii char - self.checkequal(1, 'тест', 'rfind', 'е') - self.checkequal(1, 'тест', 'rfind', 'е', 1) - self.checkequal(-1, 'тест', 'rfind', 'е', 2) - self.checkequal(-1, 'тест', 'rfind', 'e') # english `e` - # test utf-8 non-ascii slice - self.checkequal(6, 'тест тест', 'rfind', 'ес') - self.checkequal(6, 'тест тест', 'rfind', 'ес', 1) - self.checkequal(1, 'тест тест', 'rfind', 'ес', 1, 3) - self.checkequal(6, 'тест тест', 'rfind', 'ес', 2) - self.checkequal(-1, 'тест тест', 'rfind', 'ес', 6, 7) - self.checkequal(-1, 'тест тест', 'rfind', 'ес', 7) - self.checkequal(-1, 'тест тест', 'rfind', 'ec') # english `ec` - # test mixed kinds - self.checkequal(0, 'a' + '\u0102' * 100, 'rfind', 'a') - self.checkequal(0, 'a' + '\U00100304' * 100, 'rfind', 'a') - self.checkequal(0, '\u0102' + '\U00100304' * 100, 'rfind', '\u0102') - self.checkequal(-1, 'a' * 100, 'rfind', '\u0102') - self.checkequal(-1, 'a' * 100, 'rfind', '\U00100304') - self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304') - self.checkequal(0, '_a' + '\u0102' * 100, 'rfind', '_a') - self.checkequal(0, '_a' + '\U00100304' * 100, 'rfind', '_a') - self.checkequal(0, '_\u0102' + '\U00100304' * 100, 'rfind', '_\u0102') - self.checkequal(-1, 'a' * 100, 'rfind', '\u0102a') - self.checkequal(-1, 'a' * 100, 'rfind', '\U00100304a') - self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304\u0102') - - def test_index(self): - string_tests.CommonTest.test_index(self) - self.checkequalnofix(0, 'abcdefghiabc', 'index', '') - self.checkequalnofix(3, 'abcdefghiabc', 'index', 'def') - self.checkequalnofix(0, 'abcdefghiabc', 'index', 'abc') - self.checkequalnofix(9, 'abcdefghiabc', 'index', 'abc', 1) - self.assertRaises(ValueError, 'abcdefghiabc'.index, 'hib') - self.assertRaises(ValueError, 'abcdefghiab'.index, 'abc', 1) - self.assertRaises(ValueError, 'abcdefghi'.index, 'ghi', 8) - self.assertRaises(ValueError, 'abcdefghi'.index, 'ghi', -1) - # test mixed kinds - self.checkequal(100, '\u0102' * 100 + 'a', 'index', 'a') - self.checkequal(100, '\U00100304' * 100 + 'a', 'index', 'a') - self.checkequal(100, '\U00100304' * 100 + '\u0102', 'index', '\u0102') - self.assertRaises(ValueError, ('a' * 100).index, '\u0102') - self.assertRaises(ValueError, ('a' * 100).index, '\U00100304') - self.assertRaises(ValueError, ('\u0102' * 100).index, '\U00100304') - self.checkequal(100, '\u0102' * 100 + 'a_', 'index', 'a_') - self.checkequal(100, '\U00100304' * 100 + 'a_', 'index', 'a_') - self.checkequal(100, '\U00100304' * 100 + '\u0102_', 'index', '\u0102_') - self.assertRaises(ValueError, ('a' * 100).index, 'a\u0102') - self.assertRaises(ValueError, ('a' * 100).index, 'a\U00100304') - self.assertRaises(ValueError, ('\u0102' * 100).index, '\u0102\U00100304') - - def test_rindex(self): - string_tests.CommonTest.test_rindex(self) - self.checkequalnofix(12, 'abcdefghiabc', 'rindex', '') - self.checkequalnofix(3, 'abcdefghiabc', 'rindex', 'def') - self.checkequalnofix(9, 'abcdefghiabc', 'rindex', 'abc') - self.checkequalnofix(0, 'abcdefghiabc', 'rindex', 'abc', 0, -1) - - self.assertRaises(ValueError, 'abcdefghiabc'.rindex, 'hib') - self.assertRaises(ValueError, 'defghiabc'.rindex, 'def', 1) - self.assertRaises(ValueError, 'defghiabc'.rindex, 'abc', 0, -1) - self.assertRaises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, 8) - self.assertRaises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, -1) - # test mixed kinds - self.checkequal(0, 'a' + '\u0102' * 100, 'rindex', 'a') - self.checkequal(0, 'a' + '\U00100304' * 100, 'rindex', 'a') - self.checkequal(0, '\u0102' + '\U00100304' * 100, 'rindex', '\u0102') - self.assertRaises(ValueError, ('a' * 100).rindex, '\u0102') - self.assertRaises(ValueError, ('a' * 100).rindex, '\U00100304') - self.assertRaises(ValueError, ('\u0102' * 100).rindex, '\U00100304') - self.checkequal(0, '_a' + '\u0102' * 100, 'rindex', '_a') - self.checkequal(0, '_a' + '\U00100304' * 100, 'rindex', '_a') - self.checkequal(0, '_\u0102' + '\U00100304' * 100, 'rindex', '_\u0102') - self.assertRaises(ValueError, ('a' * 100).rindex, '\u0102a') - self.assertRaises(ValueError, ('a' * 100).rindex, '\U00100304a') - self.assertRaises(ValueError, ('\u0102' * 100).rindex, '\U00100304\u0102') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_maketrans_translate(self): - # these work with plain translate() - self.checkequalnofix('bbbc', 'abababc', 'translate', - {ord('a'): None}) - self.checkequalnofix('iiic', 'abababc', 'translate', - {ord('a'): None, ord('b'): ord('i')}) - self.checkequalnofix('iiix', 'abababc', 'translate', - {ord('a'): None, ord('b'): ord('i'), ord('c'): 'x'}) - self.checkequalnofix('c', 'abababc', 'translate', - {ord('a'): None, ord('b'): ''}) - self.checkequalnofix('xyyx', 'xzx', 'translate', - {ord('z'): 'yy'}) - - # this needs maketrans() - self.checkequalnofix('abababc', 'abababc', 'translate', - {'b': '<i>'}) - tbl = self.type2test.maketrans({'a': None, 'b': '<i>'}) - self.checkequalnofix('<i><i><i>c', 'abababc', 'translate', tbl) - # test alternative way of calling maketrans() - tbl = self.type2test.maketrans('abc', 'xyz', 'd') - self.checkequalnofix('xyzzy', 'abdcdcbdddd', 'translate', tbl) - - # various tests switching from ASCII to latin1 or the opposite; - # same length, remove a letter, or replace with a longer string. - self.assertEqual("[a]".translate(str.maketrans('a', 'X')), - "[X]") - self.assertEqual("[a]".translate(str.maketrans({'a': 'X'})), - "[X]") - self.assertEqual("[a]".translate(str.maketrans({'a': None})), - "[]") - self.assertEqual("[a]".translate(str.maketrans({'a': 'XXX'})), - "[XXX]") - self.assertEqual("[a]".translate(str.maketrans({'a': '\xe9'})), - "[\xe9]") - self.assertEqual('axb'.translate(str.maketrans({'a': None, 'b': '123'})), - "x123") - self.assertEqual('axb'.translate(str.maketrans({'a': None, 'b': '\xe9'})), - "x\xe9") - - # test non-ASCII (don't take the fast-path) - self.assertEqual("[a]".translate(str.maketrans({'a': '<\xe9>'})), - "[<\xe9>]") - self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': 'a'})), - "[a]") - self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': None})), - "[]") - self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': '123'})), - "[123]") - self.assertEqual("[a\xe9]".translate(str.maketrans({'a': '<\u20ac>'})), - "[<\u20ac>\xe9]") - - # invalid Unicode characters - invalid_char = 0x10ffff+1 - for before in "a\xe9\u20ac\U0010ffff": - mapping = str.maketrans({before: invalid_char}) - text = "[%s]" % before - self.assertRaises(ValueError, text.translate, mapping) - - # errors - self.assertRaises(TypeError, self.type2test.maketrans) - self.assertRaises(ValueError, self.type2test.maketrans, 'abc', 'defg') - self.assertRaises(TypeError, self.type2test.maketrans, 2, 'def') - self.assertRaises(TypeError, self.type2test.maketrans, 'abc', 2) - self.assertRaises(TypeError, self.type2test.maketrans, 'abc', 'def', 2) - self.assertRaises(ValueError, self.type2test.maketrans, {'xy': 2}) - self.assertRaises(TypeError, self.type2test.maketrans, {(1,): 2}) - - self.assertRaises(TypeError, 'hello'.translate) - self.assertRaises(TypeError, 'abababc'.translate, 'abc', 'xyz') - - def test_split(self): - string_tests.CommonTest.test_split(self) - - # test mixed kinds - for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): - left *= 9 - right *= 9 - for delim in ('c', '\u0102', '\U00010302'): - self.checkequal([left + right], - left + right, 'split', delim) - self.checkequal([left, right], - left + delim + right, 'split', delim) - self.checkequal([left + right], - left + right, 'split', delim * 2) - self.checkequal([left, right], - left + delim * 2 + right, 'split', delim *2) - - def test_rsplit(self): - string_tests.CommonTest.test_rsplit(self) - # test mixed kinds - for left, right in ('ba', 'юё', '\u0101\u0100', '\U00010301\U00010300'): - left *= 9 - right *= 9 - for delim in ('c', 'ы', '\u0102', '\U00010302'): - self.checkequal([left + right], - left + right, 'rsplit', delim) - self.checkequal([left, right], - left + delim + right, 'rsplit', delim) - self.checkequal([left + right], - left + right, 'rsplit', delim * 2) - self.checkequal([left, right], - left + delim * 2 + right, 'rsplit', delim *2) - - # Check `None` as well: - self.checkequal([left + right], - left + right, 'rsplit', None) - - def test_partition(self): - string_tests.MixinStrUnicodeUserStringTest.test_partition(self) - # test mixed kinds - self.checkequal(('ABCDEFGH', '', ''), 'ABCDEFGH', 'partition', '\u4200') - for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): - left *= 9 - right *= 9 - for delim in ('c', '\u0102', '\U00010302'): - self.checkequal((left + right, '', ''), - left + right, 'partition', delim) - self.checkequal((left, delim, right), - left + delim + right, 'partition', delim) - self.checkequal((left + right, '', ''), - left + right, 'partition', delim * 2) - self.checkequal((left, delim * 2, right), - left + delim * 2 + right, 'partition', delim * 2) - - def test_rpartition(self): - string_tests.MixinStrUnicodeUserStringTest.test_rpartition(self) - # test mixed kinds - self.checkequal(('', '', 'ABCDEFGH'), 'ABCDEFGH', 'rpartition', '\u4200') - for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): - left *= 9 - right *= 9 - for delim in ('c', '\u0102', '\U00010302'): - self.checkequal(('', '', left + right), - left + right, 'rpartition', delim) - self.checkequal((left, delim, right), - left + delim + right, 'rpartition', delim) - self.checkequal(('', '', left + right), - left + right, 'rpartition', delim * 2) - self.checkequal((left, delim * 2, right), - left + delim * 2 + right, 'rpartition', delim * 2) - - def test_join(self): - string_tests.MixinStrUnicodeUserStringTest.test_join(self) - - class MyWrapper: - def __init__(self, sval): self.sval = sval - def __str__(self): return self.sval - - # mixed arguments - self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd']) - self.checkequalnofix('abcd', '', 'join', ('a', 'b', 'c', 'd')) - self.checkequalnofix('w x y z', ' ', 'join', string_tests.Sequence('wxyz')) - self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd']) - self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd']) - self.checkequalnofix('abcd', '', 'join', ('a', 'b', 'c', 'd')) - self.checkequalnofix('w x y z', ' ', 'join', string_tests.Sequence('wxyz')) - self.checkraises(TypeError, ' ', 'join', ['1', '2', MyWrapper('foo')]) - self.checkraises(TypeError, ' ', 'join', ['1', '2', '3', bytes()]) - self.checkraises(TypeError, ' ', 'join', [1, 2, 3]) - self.checkraises(TypeError, ' ', 'join', ['1', '2', 3]) - - @unittest.skip("TODO: RUSTPYTHON, oom handling") - @unittest.skipIf(sys.maxsize > 2**32, - 'needs too much memory on a 64-bit platform') - def test_join_overflow(self): - size = int(sys.maxsize**0.5) + 1 - seq = ('A' * size,) * size - self.assertRaises(OverflowError, ''.join, seq) - - def test_replace(self): - string_tests.CommonTest.test_replace(self) - - # method call forwarded from str implementation because of unicode argument - self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1) - self.assertRaises(TypeError, 'replace'.replace, "r", 42) - # test mixed kinds - for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): - left *= 9 - right *= 9 - for delim in ('c', '\u0102', '\U00010302'): - for repl in ('d', '\u0103', '\U00010303'): - self.checkequal(left + right, - left + right, 'replace', delim, repl) - self.checkequal(left + repl + right, - left + delim + right, - 'replace', delim, repl) - self.checkequal(left + right, - left + right, 'replace', delim * 2, repl) - self.checkequal(left + repl + right, - left + delim * 2 + right, - 'replace', delim * 2, repl) - - @support.cpython_only - def test_replace_id(self): - pattern = 'abc' - text = 'abc def' - self.assertIs(text.replace(pattern, pattern), text) - - def test_repeat_id_preserving(self): - a = '123abc1@' - b = '456zyx-+' - self.assertEqual(id(a), id(a)) - self.assertNotEqual(id(a), id(b)) - self.assertNotEqual(id(a), id(a * -4)) - self.assertNotEqual(id(a), id(a * 0)) - self.assertEqual(id(a), id(a * 1)) - self.assertEqual(id(a), id(1 * a)) - self.assertNotEqual(id(a), id(a * 2)) - - class SubStr(str): - pass - - s = SubStr('qwerty()') - self.assertEqual(id(s), id(s)) - self.assertNotEqual(id(s), id(s * -4)) - self.assertNotEqual(id(s), id(s * 0)) - self.assertNotEqual(id(s), id(s * 1)) - self.assertNotEqual(id(s), id(1 * s)) - self.assertNotEqual(id(s), id(s * 2)) - - def test_bytes_comparison(self): - with warnings_helper.check_warnings(): - warnings.simplefilter('ignore', BytesWarning) - self.assertEqual('abc' == b'abc', False) - self.assertEqual('abc' != b'abc', True) - self.assertEqual('abc' == bytearray(b'abc'), False) - self.assertEqual('abc' != bytearray(b'abc'), True) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_comparison(self): - # Comparisons: - self.assertEqual('abc', 'abc') - self.assertTrue('abcd' > 'abc') - self.assertTrue('abc' < 'abcd') - - if 0: - # Move these tests to a Unicode collation module test... - # Testing UTF-16 code point order comparisons... - - # No surrogates, no fixup required. - self.assertTrue('\u0061' < '\u20ac') - # Non surrogate below surrogate value, no fixup required - self.assertTrue('\u0061' < '\ud800\udc02') - - # Non surrogate above surrogate value, fixup required - def test_lecmp(s, s2): - self.assertTrue(s < s2) - - def test_fixup(s): - s2 = '\ud800\udc01' - test_lecmp(s, s2) - s2 = '\ud900\udc01' - test_lecmp(s, s2) - s2 = '\uda00\udc01' - test_lecmp(s, s2) - s2 = '\udb00\udc01' - test_lecmp(s, s2) - s2 = '\ud800\udd01' - test_lecmp(s, s2) - s2 = '\ud900\udd01' - test_lecmp(s, s2) - s2 = '\uda00\udd01' - test_lecmp(s, s2) - s2 = '\udb00\udd01' - test_lecmp(s, s2) - s2 = '\ud800\ude01' - test_lecmp(s, s2) - s2 = '\ud900\ude01' - test_lecmp(s, s2) - s2 = '\uda00\ude01' - test_lecmp(s, s2) - s2 = '\udb00\ude01' - test_lecmp(s, s2) - s2 = '\ud800\udfff' - test_lecmp(s, s2) - s2 = '\ud900\udfff' - test_lecmp(s, s2) - s2 = '\uda00\udfff' - test_lecmp(s, s2) - s2 = '\udb00\udfff' - test_lecmp(s, s2) - - test_fixup('\ue000') - test_fixup('\uff61') - - # Surrogates on both sides, no fixup required - self.assertTrue('\ud800\udc02' < '\ud84d\udc56') - - def test_islower(self): - super().test_islower() - self.checkequalnofix(False, '\u1FFc', 'islower') - self.assertFalse('\u2167'.islower()) - self.assertTrue('\u2177'.islower()) - # non-BMP, uppercase - self.assertFalse('\U00010401'.islower()) - self.assertFalse('\U00010427'.islower()) - # non-BMP, lowercase - self.assertTrue('\U00010429'.islower()) - self.assertTrue('\U0001044E'.islower()) - # non-BMP, non-cased - self.assertFalse('\U0001F40D'.islower()) - self.assertFalse('\U0001F46F'.islower()) - - def test_isupper(self): - super().test_isupper() - self.checkequalnofix(False, '\u1FFc', 'isupper') - self.assertTrue('\u2167'.isupper()) - self.assertFalse('\u2177'.isupper()) - # non-BMP, uppercase - self.assertTrue('\U00010401'.isupper()) - self.assertTrue('\U00010427'.isupper()) - # non-BMP, lowercase - self.assertFalse('\U00010429'.isupper()) - self.assertFalse('\U0001044E'.isupper()) - # non-BMP, non-cased - self.assertFalse('\U0001F40D'.isupper()) - self.assertFalse('\U0001F46F'.isupper()) - - def test_istitle(self): - super().test_istitle() - self.checkequalnofix(True, '\u1FFc', 'istitle') - self.checkequalnofix(True, 'Greek \u1FFcitlecases ...', 'istitle') - - # non-BMP, uppercase + lowercase - self.assertTrue('\U00010401\U00010429'.istitle()) - self.assertTrue('\U00010427\U0001044E'.istitle()) - # apparently there are no titlecased (Lt) non-BMP chars in Unicode 6 - for ch in ['\U00010429', '\U0001044E', '\U0001F40D', '\U0001F46F']: - self.assertFalse(ch.istitle(), '{!a} is not title'.format(ch)) - - def test_isspace(self): - super().test_isspace() - self.checkequalnofix(True, '\u2000', 'isspace') - self.checkequalnofix(True, '\u200a', 'isspace') - self.checkequalnofix(False, '\u2014', 'isspace') - # There are no non-BMP whitespace chars as of Unicode 12. - for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', - '\U0001F40D', '\U0001F46F']: - self.assertFalse(ch.isspace(), '{!a} is not space.'.format(ch)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @support.requires_resource('cpu') - def test_isspace_invariant(self): - for codepoint in range(sys.maxunicode + 1): - char = chr(codepoint) - bidirectional = unicodedata.bidirectional(char) - category = unicodedata.category(char) - self.assertEqual(char.isspace(), - (bidirectional in ('WS', 'B', 'S') - or category == 'Zs')) - - def test_isalnum(self): - super().test_isalnum() - for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', - '\U0001D7F6', '\U00011066', '\U000104A0', '\U0001F107']: - self.assertTrue(ch.isalnum(), '{!a} is alnum.'.format(ch)) - - def test_isalpha(self): - super().test_isalpha() - self.checkequalnofix(True, '\u1FFc', 'isalpha') - # non-BMP, cased - self.assertTrue('\U00010401'.isalpha()) - self.assertTrue('\U00010427'.isalpha()) - self.assertTrue('\U00010429'.isalpha()) - self.assertTrue('\U0001044E'.isalpha()) - # non-BMP, non-cased - self.assertFalse('\U0001F40D'.isalpha()) - self.assertFalse('\U0001F46F'.isalpha()) - - def test_isascii(self): - super().test_isascii() - self.assertFalse("\u20ac".isascii()) - self.assertFalse("\U0010ffff".isascii()) - - def test_isdecimal(self): - self.checkequalnofix(False, '', 'isdecimal') - self.checkequalnofix(False, 'a', 'isdecimal') - self.checkequalnofix(True, '0', 'isdecimal') - self.checkequalnofix(False, '\u2460', 'isdecimal') # CIRCLED DIGIT ONE - self.checkequalnofix(False, '\xbc', 'isdecimal') # VULGAR FRACTION ONE QUARTER - self.checkequalnofix(True, '\u0660', 'isdecimal') # ARABIC-INDIC DIGIT ZERO - self.checkequalnofix(True, '0123456789', 'isdecimal') - self.checkequalnofix(False, '0123456789a', 'isdecimal') - - self.checkraises(TypeError, 'abc', 'isdecimal', 42) - - for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', - '\U0001F40D', '\U0001F46F', '\U00011065', '\U0001F107']: - self.assertFalse(ch.isdecimal(), '{!a} is not decimal.'.format(ch)) - for ch in ['\U0001D7F6', '\U00011066', '\U000104A0']: - self.assertTrue(ch.isdecimal(), '{!a} is decimal.'.format(ch)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_isdigit(self): - super().test_isdigit() - self.checkequalnofix(True, '\u2460', 'isdigit') - self.checkequalnofix(False, '\xbc', 'isdigit') - self.checkequalnofix(True, '\u0660', 'isdigit') - - for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', - '\U0001F40D', '\U0001F46F', '\U00011065']: - self.assertFalse(ch.isdigit(), '{!a} is not a digit.'.format(ch)) - for ch in ['\U0001D7F6', '\U00011066', '\U000104A0', '\U0001F107']: - self.assertTrue(ch.isdigit(), '{!a} is a digit.'.format(ch)) - - def test_isnumeric(self): - self.checkequalnofix(False, '', 'isnumeric') - self.checkequalnofix(False, 'a', 'isnumeric') - self.checkequalnofix(True, '0', 'isnumeric') - self.checkequalnofix(True, '\u2460', 'isnumeric') - self.checkequalnofix(True, '\xbc', 'isnumeric') - self.checkequalnofix(True, '\u0660', 'isnumeric') - self.checkequalnofix(True, '0123456789', 'isnumeric') - self.checkequalnofix(False, '0123456789a', 'isnumeric') - - self.assertRaises(TypeError, "abc".isnumeric, 42) - - for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', - '\U0001F40D', '\U0001F46F']: - self.assertFalse(ch.isnumeric(), '{!a} is not numeric.'.format(ch)) - for ch in ['\U00011065', '\U0001D7F6', '\U00011066', - '\U000104A0', '\U0001F107']: - self.assertTrue(ch.isnumeric(), '{!a} is numeric.'.format(ch)) - - def test_isidentifier(self): - self.assertTrue("a".isidentifier()) - self.assertTrue("Z".isidentifier()) - self.assertTrue("_".isidentifier()) - self.assertTrue("b0".isidentifier()) - self.assertTrue("bc".isidentifier()) - self.assertTrue("b_".isidentifier()) - self.assertTrue("µ".isidentifier()) - self.assertTrue("𝔘𝔫𝔦𝔠𝔬𝔡𝔢".isidentifier()) - - self.assertFalse(" ".isidentifier()) - self.assertFalse("[".isidentifier()) - self.assertFalse("©".isidentifier()) - self.assertFalse("0".isidentifier()) - - @support.cpython_only - @support.requires_legacy_unicode_capi() - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_isidentifier_legacy(self): - u = '𝖀𝖓𝖎𝖈𝖔𝖉𝖊' - self.assertTrue(u.isidentifier()) - with warnings_helper.check_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertTrue(_testcapi.unicode_legacy_string(u).isidentifier()) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_isprintable(self): - self.assertTrue("".isprintable()) - self.assertTrue(" ".isprintable()) - self.assertTrue("abcdefg".isprintable()) - self.assertFalse("abcdefg\n".isprintable()) - # some defined Unicode character - self.assertTrue("\u0374".isprintable()) - # undefined character - self.assertFalse("\u0378".isprintable()) - # single surrogate character - self.assertFalse("\ud800".isprintable()) - - self.assertTrue('\U0001F46F'.isprintable()) - self.assertFalse('\U000E0020'.isprintable()) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_surrogates(self): - for s in ('a\uD800b\uDFFF', 'a\uDFFFb\uD800', - 'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'): - self.assertTrue(s.islower()) - self.assertFalse(s.isupper()) - self.assertFalse(s.istitle()) - for s in ('A\uD800B\uDFFF', 'A\uDFFFB\uD800', - 'A\uD800B\uDFFFA', 'A\uDFFFB\uD800A'): - self.assertFalse(s.islower()) - self.assertTrue(s.isupper()) - self.assertTrue(s.istitle()) - - for meth_name in ('islower', 'isupper', 'istitle'): - meth = getattr(str, meth_name) - for s in ('\uD800', '\uDFFF', '\uD800\uD800', '\uDFFF\uDFFF'): - self.assertFalse(meth(s), '%a.%s() is False' % (s, meth_name)) - - for meth_name in ('isalpha', 'isalnum', 'isdigit', 'isspace', - 'isdecimal', 'isnumeric', - 'isidentifier', 'isprintable'): - meth = getattr(str, meth_name) - for s in ('\uD800', '\uDFFF', '\uD800\uD800', '\uDFFF\uDFFF', - 'a\uD800b\uDFFF', 'a\uDFFFb\uD800', - 'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'): - self.assertFalse(meth(s), '%a.%s() is False' % (s, meth_name)) - - - def test_lower(self): - string_tests.CommonTest.test_lower(self) - self.assertEqual('\U00010427'.lower(), '\U0001044F') - self.assertEqual('\U00010427\U00010427'.lower(), - '\U0001044F\U0001044F') - self.assertEqual('\U00010427\U0001044F'.lower(), - '\U0001044F\U0001044F') - self.assertEqual('X\U00010427x\U0001044F'.lower(), - 'x\U0001044Fx\U0001044F') - self.assertEqual('fi'.lower(), 'fi') - self.assertEqual('\u0130'.lower(), '\u0069\u0307') - # Special case for GREEK CAPITAL LETTER SIGMA U+03A3 - self.assertEqual('\u03a3'.lower(), '\u03c3') - self.assertEqual('\u0345\u03a3'.lower(), '\u0345\u03c3') - self.assertEqual('A\u0345\u03a3'.lower(), 'a\u0345\u03c2') - self.assertEqual('A\u0345\u03a3a'.lower(), 'a\u0345\u03c3a') - self.assertEqual('A\u0345\u03a3'.lower(), 'a\u0345\u03c2') - self.assertEqual('A\u03a3\u0345'.lower(), 'a\u03c2\u0345') - self.assertEqual('\u03a3\u0345 '.lower(), '\u03c3\u0345 ') - self.assertEqual('\U0008fffe'.lower(), '\U0008fffe') - self.assertEqual('\u2177'.lower(), '\u2177') - - def test_casefold(self): - self.assertEqual('hello'.casefold(), 'hello') - self.assertEqual('hELlo'.casefold(), 'hello') - self.assertEqual('ß'.casefold(), 'ss') - self.assertEqual('fi'.casefold(), 'fi') - self.assertEqual('\u03a3'.casefold(), '\u03c3') - self.assertEqual('A\u0345\u03a3'.casefold(), 'a\u03b9\u03c3') - self.assertEqual('\u00b5'.casefold(), '\u03bc') - - def test_upper(self): - string_tests.CommonTest.test_upper(self) - self.assertEqual('\U0001044F'.upper(), '\U00010427') - self.assertEqual('\U0001044F\U0001044F'.upper(), - '\U00010427\U00010427') - self.assertEqual('\U00010427\U0001044F'.upper(), - '\U00010427\U00010427') - self.assertEqual('X\U00010427x\U0001044F'.upper(), - 'X\U00010427X\U00010427') - self.assertEqual('fi'.upper(), 'FI') - self.assertEqual('\u0130'.upper(), '\u0130') - self.assertEqual('\u03a3'.upper(), '\u03a3') - self.assertEqual('ß'.upper(), 'SS') - self.assertEqual('\u1fd2'.upper(), '\u0399\u0308\u0300') - self.assertEqual('\U0008fffe'.upper(), '\U0008fffe') - self.assertEqual('\u2177'.upper(), '\u2167') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_capitalize(self): - string_tests.CommonTest.test_capitalize(self) - self.assertEqual('\U0001044F'.capitalize(), '\U00010427') - self.assertEqual('\U0001044F\U0001044F'.capitalize(), - '\U00010427\U0001044F') - self.assertEqual('\U00010427\U0001044F'.capitalize(), - '\U00010427\U0001044F') - self.assertEqual('\U0001044F\U00010427'.capitalize(), - '\U00010427\U0001044F') - self.assertEqual('X\U00010427x\U0001044F'.capitalize(), - 'X\U0001044Fx\U0001044F') - self.assertEqual('h\u0130'.capitalize(), 'H\u0069\u0307') - exp = '\u0399\u0308\u0300\u0069\u0307' - self.assertEqual('\u1fd2\u0130'.capitalize(), exp) - self.assertEqual('finnish'.capitalize(), 'Finnish') - self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_title(self): - super().test_title() - self.assertEqual('\U0001044F'.title(), '\U00010427') - self.assertEqual('\U0001044F\U0001044F'.title(), - '\U00010427\U0001044F') - self.assertEqual('\U0001044F\U0001044F \U0001044F\U0001044F'.title(), - '\U00010427\U0001044F \U00010427\U0001044F') - self.assertEqual('\U00010427\U0001044F \U00010427\U0001044F'.title(), - '\U00010427\U0001044F \U00010427\U0001044F') - self.assertEqual('\U0001044F\U00010427 \U0001044F\U00010427'.title(), - '\U00010427\U0001044F \U00010427\U0001044F') - self.assertEqual('X\U00010427x\U0001044F X\U00010427x\U0001044F'.title(), - 'X\U0001044Fx\U0001044F X\U0001044Fx\U0001044F') - self.assertEqual('fiNNISH'.title(), 'Finnish') - self.assertEqual('A\u03a3 \u1fa1xy'.title(), 'A\u03c2 \u1fa9xy') - self.assertEqual('A\u03a3A'.title(), 'A\u03c3a') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_swapcase(self): - string_tests.CommonTest.test_swapcase(self) - self.assertEqual('\U0001044F'.swapcase(), '\U00010427') - self.assertEqual('\U00010427'.swapcase(), '\U0001044F') - self.assertEqual('\U0001044F\U0001044F'.swapcase(), - '\U00010427\U00010427') - self.assertEqual('\U00010427\U0001044F'.swapcase(), - '\U0001044F\U00010427') - self.assertEqual('\U0001044F\U00010427'.swapcase(), - '\U00010427\U0001044F') - self.assertEqual('X\U00010427x\U0001044F'.swapcase(), - 'x\U0001044FX\U00010427') - self.assertEqual('fi'.swapcase(), 'FI') - self.assertEqual('\u0130'.swapcase(), '\u0069\u0307') - # Special case for GREEK CAPITAL LETTER SIGMA U+03A3 - self.assertEqual('\u03a3'.swapcase(), '\u03c3') - self.assertEqual('\u0345\u03a3'.swapcase(), '\u0399\u03c3') - self.assertEqual('A\u0345\u03a3'.swapcase(), 'a\u0399\u03c2') - self.assertEqual('A\u0345\u03a3a'.swapcase(), 'a\u0399\u03c3A') - self.assertEqual('A\u0345\u03a3'.swapcase(), 'a\u0399\u03c2') - self.assertEqual('A\u03a3\u0345'.swapcase(), 'a\u03c2\u0399') - self.assertEqual('\u03a3\u0345 '.swapcase(), '\u03c3\u0399 ') - self.assertEqual('\u03a3'.swapcase(), '\u03c3') - self.assertEqual('ß'.swapcase(), 'SS') - self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300') - - def test_center(self): - string_tests.CommonTest.test_center(self) - self.assertEqual('x'.center(2, '\U0010FFFF'), - 'x\U0010FFFF') - self.assertEqual('x'.center(3, '\U0010FFFF'), - '\U0010FFFFx\U0010FFFF') - self.assertEqual('x'.center(4, '\U0010FFFF'), - '\U0010FFFFx\U0010FFFF\U0010FFFF') - - @unittest.skipUnless(sys.maxsize == 2**31 - 1, "requires 32-bit system") - @support.cpython_only - def test_case_operation_overflow(self): - # Issue #22643 - size = 2**32//12 + 1 - try: - s = "ü" * size - except MemoryError: - self.skipTest('no enough memory (%.0f MiB required)' % (size / 2**20)) - try: - self.assertRaises(OverflowError, s.upper) - finally: - del s - - def test_contains(self): - # Testing Unicode contains method - self.assertIn('a', 'abdb') - self.assertIn('a', 'bdab') - self.assertIn('a', 'bdaba') - self.assertIn('a', 'bdba') - self.assertNotIn('a', 'bdb') - self.assertIn('a', 'bdba') - self.assertIn('a', ('a',1,None)) - self.assertIn('a', (1,None,'a')) - self.assertIn('a', ('a',1,None)) - self.assertIn('a', (1,None,'a')) - self.assertNotIn('a', ('x',1,'y')) - self.assertNotIn('a', ('x',1,None)) - self.assertNotIn('abcd', 'abcxxxx') - self.assertIn('ab', 'abcd') - self.assertIn('ab', 'abc') - self.assertIn('ab', (1,None,'ab')) - self.assertIn('', 'abc') - self.assertIn('', '') - self.assertIn('', 'abc') - self.assertNotIn('\0', 'abc') - self.assertIn('\0', '\0abc') - self.assertIn('\0', 'abc\0') - self.assertIn('a', '\0abc') - self.assertIn('asdf', 'asdf') - self.assertNotIn('asdf', 'asd') - self.assertNotIn('asdf', '') - - self.assertRaises(TypeError, "abc".__contains__) - # test mixed kinds - for fill in ('a', '\u0100', '\U00010300'): - fill *= 9 - for delim in ('c', '\u0102', '\U00010302'): - self.assertNotIn(delim, fill) - self.assertIn(delim, fill + delim) - self.assertNotIn(delim * 2, fill) - self.assertIn(delim * 2, fill + delim * 2) - - def test_issue18183(self): - '\U00010000\U00100000'.lower() - '\U00010000\U00100000'.casefold() - '\U00010000\U00100000'.upper() - '\U00010000\U00100000'.capitalize() - '\U00010000\U00100000'.title() - '\U00010000\U00100000'.swapcase() - '\U00100000'.center(3, '\U00010000') - '\U00100000'.ljust(3, '\U00010000') - '\U00100000'.rjust(3, '\U00010000') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_format(self): - self.assertEqual(''.format(), '') - self.assertEqual('a'.format(), 'a') - self.assertEqual('ab'.format(), 'ab') - self.assertEqual('a{{'.format(), 'a{') - self.assertEqual('a}}'.format(), 'a}') - self.assertEqual('{{b'.format(), '{b') - self.assertEqual('}}b'.format(), '}b') - self.assertEqual('a{{b'.format(), 'a{b') - - # examples from the PEP: - import datetime - self.assertEqual("My name is {0}".format('Fred'), "My name is Fred") - self.assertEqual("My name is {0[name]}".format(dict(name='Fred')), - "My name is Fred") - self.assertEqual("My name is {0} :-{{}}".format('Fred'), - "My name is Fred :-{}") - - d = datetime.date(2007, 8, 18) - self.assertEqual("The year is {0.year}".format(d), - "The year is 2007") - - # classes we'll use for testing - class C: - def __init__(self, x=100): - self._x = x - def __format__(self, spec): - return spec - - class D: - def __init__(self, x): - self.x = x - def __format__(self, spec): - return str(self.x) - - # class with __str__, but no __format__ - class E: - def __init__(self, x): - self.x = x - def __str__(self): - return 'E(' + self.x + ')' - - # class with __repr__, but no __format__ or __str__ - class F: - def __init__(self, x): - self.x = x - def __repr__(self): - return 'F(' + self.x + ')' - - # class with __format__ that forwards to string, for some format_spec's - class G: - def __init__(self, x): - self.x = x - def __str__(self): - return "string is " + self.x - def __format__(self, format_spec): - if format_spec == 'd': - return 'G(' + self.x + ')' - return object.__format__(self, format_spec) - - class I(datetime.date): - def __format__(self, format_spec): - return self.strftime(format_spec) - - class J(int): - def __format__(self, format_spec): - return int.__format__(self * 2, format_spec) - - class M: - def __init__(self, x): - self.x = x - def __repr__(self): - return 'M(' + self.x + ')' - __str__ = None - - class N: - def __init__(self, x): - self.x = x - def __repr__(self): - return 'N(' + self.x + ')' - __format__ = None - - self.assertEqual(''.format(), '') - self.assertEqual('abc'.format(), 'abc') - self.assertEqual('{0}'.format('abc'), 'abc') - self.assertEqual('{0:}'.format('abc'), 'abc') -# self.assertEqual('{ 0 }'.format('abc'), 'abc') - self.assertEqual('X{0}'.format('abc'), 'Xabc') - self.assertEqual('{0}X'.format('abc'), 'abcX') - self.assertEqual('X{0}Y'.format('abc'), 'XabcY') - self.assertEqual('{1}'.format(1, 'abc'), 'abc') - self.assertEqual('X{1}'.format(1, 'abc'), 'Xabc') - self.assertEqual('{1}X'.format(1, 'abc'), 'abcX') - self.assertEqual('X{1}Y'.format(1, 'abc'), 'XabcY') - self.assertEqual('{0}'.format(-15), '-15') - self.assertEqual('{0}{1}'.format(-15, 'abc'), '-15abc') - self.assertEqual('{0}X{1}'.format(-15, 'abc'), '-15Xabc') - self.assertEqual('{{'.format(), '{') - self.assertEqual('}}'.format(), '}') - self.assertEqual('{{}}'.format(), '{}') - self.assertEqual('{{x}}'.format(), '{x}') - self.assertEqual('{{{0}}}'.format(123), '{123}') - self.assertEqual('{{{{0}}}}'.format(), '{{0}}') - self.assertEqual('}}{{'.format(), '}{') - self.assertEqual('}}x{{'.format(), '}x{') - - # weird field names - self.assertEqual("{0[foo-bar]}".format({'foo-bar':'baz'}), 'baz') - self.assertEqual("{0[foo bar]}".format({'foo bar':'baz'}), 'baz') - self.assertEqual("{0[ ]}".format({' ':3}), '3') - - self.assertEqual('{foo._x}'.format(foo=C(20)), '20') - self.assertEqual('{1}{0}'.format(D(10), D(20)), '2010') - self.assertEqual('{0._x.x}'.format(C(D('abc'))), 'abc') - self.assertEqual('{0[0]}'.format(['abc', 'def']), 'abc') - self.assertEqual('{0[1]}'.format(['abc', 'def']), 'def') - self.assertEqual('{0[1][0]}'.format(['abc', ['def']]), 'def') - self.assertEqual('{0[1][0].x}'.format(['abc', [D('def')]]), 'def') - - # strings - self.assertEqual('{0:.3s}'.format('abc'), 'abc') - self.assertEqual('{0:.3s}'.format('ab'), 'ab') - self.assertEqual('{0:.3s}'.format('abcdef'), 'abc') - self.assertEqual('{0:.0s}'.format('abcdef'), '') - self.assertEqual('{0:3.3s}'.format('abc'), 'abc') - self.assertEqual('{0:2.3s}'.format('abc'), 'abc') - self.assertEqual('{0:2.2s}'.format('abc'), 'ab') - self.assertEqual('{0:3.2s}'.format('abc'), 'ab ') - self.assertEqual('{0:x<0s}'.format('result'), 'result') - self.assertEqual('{0:x<5s}'.format('result'), 'result') - self.assertEqual('{0:x<6s}'.format('result'), 'result') - self.assertEqual('{0:x<7s}'.format('result'), 'resultx') - self.assertEqual('{0:x<8s}'.format('result'), 'resultxx') - self.assertEqual('{0: <7s}'.format('result'), 'result ') - self.assertEqual('{0:<7s}'.format('result'), 'result ') - self.assertEqual('{0:>7s}'.format('result'), ' result') - self.assertEqual('{0:>8s}'.format('result'), ' result') - self.assertEqual('{0:^8s}'.format('result'), ' result ') - self.assertEqual('{0:^9s}'.format('result'), ' result ') - self.assertEqual('{0:^10s}'.format('result'), ' result ') - self.assertEqual('{0:8s}'.format('result'), 'result ') - self.assertEqual('{0:0s}'.format('result'), 'result') - self.assertEqual('{0:08s}'.format('result'), 'result00') - self.assertEqual('{0:<08s}'.format('result'), 'result00') - self.assertEqual('{0:>08s}'.format('result'), '00result') - self.assertEqual('{0:^08s}'.format('result'), '0result0') - self.assertEqual('{0:10000}'.format('a'), 'a' + ' ' * 9999) - self.assertEqual('{0:10000}'.format(''), ' ' * 10000) - self.assertEqual('{0:10000000}'.format(''), ' ' * 10000000) - - # issue 12546: use \x00 as a fill character - self.assertEqual('{0:\x00<6s}'.format('foo'), 'foo\x00\x00\x00') - self.assertEqual('{0:\x01<6s}'.format('foo'), 'foo\x01\x01\x01') - self.assertEqual('{0:\x00^6s}'.format('foo'), '\x00foo\x00\x00') - self.assertEqual('{0:^6s}'.format('foo'), ' foo ') - - self.assertEqual('{0:\x00<6}'.format(3), '3\x00\x00\x00\x00\x00') - self.assertEqual('{0:\x01<6}'.format(3), '3\x01\x01\x01\x01\x01') - self.assertEqual('{0:\x00^6}'.format(3), '\x00\x003\x00\x00\x00') - self.assertEqual('{0:<6}'.format(3), '3 ') - - self.assertEqual('{0:\x00<6}'.format(3.14), '3.14\x00\x00') - self.assertEqual('{0:\x01<6}'.format(3.14), '3.14\x01\x01') - self.assertEqual('{0:\x00^6}'.format(3.14), '\x003.14\x00') - self.assertEqual('{0:^6}'.format(3.14), ' 3.14 ') - - self.assertEqual('{0:\x00<12}'.format(3+2.0j), '(3+2j)\x00\x00\x00\x00\x00\x00') - self.assertEqual('{0:\x01<12}'.format(3+2.0j), '(3+2j)\x01\x01\x01\x01\x01\x01') - self.assertEqual('{0:\x00^12}'.format(3+2.0j), '\x00\x00\x00(3+2j)\x00\x00\x00') - self.assertEqual('{0:^12}'.format(3+2.0j), ' (3+2j) ') - - # format specifiers for user defined type - self.assertEqual('{0:abc}'.format(C()), 'abc') - - # !r, !s and !a coercions - self.assertEqual('{0!s}'.format('Hello'), 'Hello') - self.assertEqual('{0!s:}'.format('Hello'), 'Hello') - self.assertEqual('{0!s:15}'.format('Hello'), 'Hello ') - self.assertEqual('{0!s:15s}'.format('Hello'), 'Hello ') - self.assertEqual('{0!r}'.format('Hello'), "'Hello'") - self.assertEqual('{0!r:}'.format('Hello'), "'Hello'") - self.assertEqual('{0!r}'.format(F('Hello')), 'F(Hello)') - self.assertEqual('{0!r}'.format('\u0378'), "'\\u0378'") # nonprintable - self.assertEqual('{0!r}'.format('\u0374'), "'\u0374'") # printable - self.assertEqual('{0!r}'.format(F('\u0374')), 'F(\u0374)') - self.assertEqual('{0!a}'.format('Hello'), "'Hello'") - self.assertEqual('{0!a}'.format('\u0378'), "'\\u0378'") # nonprintable - self.assertEqual('{0!a}'.format('\u0374'), "'\\u0374'") # printable - self.assertEqual('{0!a:}'.format('Hello'), "'Hello'") - self.assertEqual('{0!a}'.format(F('Hello')), 'F(Hello)') - self.assertEqual('{0!a}'.format(F('\u0374')), 'F(\\u0374)') - - # test fallback to object.__format__ - self.assertEqual('{0}'.format({}), '{}') - self.assertEqual('{0}'.format([]), '[]') - self.assertEqual('{0}'.format([1]), '[1]') - - self.assertEqual('{0:d}'.format(G('data')), 'G(data)') - self.assertEqual('{0!s}'.format(G('data')), 'string is data') - - self.assertRaises(TypeError, '{0:^10}'.format, E('data')) - self.assertRaises(TypeError, '{0:^10s}'.format, E('data')) - self.assertRaises(TypeError, '{0:>15s}'.format, G('data')) - - self.assertEqual("{0:date: %Y-%m-%d}".format(I(year=2007, - month=8, - day=27)), - "date: 2007-08-27") - - # test deriving from a builtin type and overriding __format__ - self.assertEqual("{0}".format(J(10)), "20") - - - # string format specifiers - self.assertEqual('{0:}'.format('a'), 'a') - - # computed format specifiers - self.assertEqual("{0:.{1}}".format('hello world', 5), 'hello') - self.assertEqual("{0:.{1}s}".format('hello world', 5), 'hello') - self.assertEqual("{0:.{precision}s}".format('hello world', precision=5), 'hello') - self.assertEqual("{0:{width}.{precision}s}".format('hello world', width=10, precision=5), 'hello ') - self.assertEqual("{0:{width}.{precision}s}".format('hello world', width='10', precision='5'), 'hello ') - - # test various errors - self.assertRaises(ValueError, '{'.format) - self.assertRaises(ValueError, '}'.format) - self.assertRaises(ValueError, 'a{'.format) - self.assertRaises(ValueError, 'a}'.format) - self.assertRaises(ValueError, '{a'.format) - self.assertRaises(ValueError, '}a'.format) - self.assertRaises(IndexError, '{0}'.format) - self.assertRaises(IndexError, '{1}'.format, 'abc') - self.assertRaises(KeyError, '{x}'.format) - self.assertRaises(ValueError, "}{".format) - self.assertRaises(ValueError, "abc{0:{}".format) - self.assertRaises(ValueError, "{0".format) - self.assertRaises(IndexError, "{0.}".format) - self.assertRaises(ValueError, "{0.}".format, 0) - self.assertRaises(ValueError, "{0[}".format) - self.assertRaises(ValueError, "{0[}".format, []) - self.assertRaises(KeyError, "{0]}".format) - self.assertRaises(ValueError, "{0.[]}".format, 0) - self.assertRaises(ValueError, "{0..foo}".format, 0) - self.assertRaises(ValueError, "{0[0}".format, 0) - self.assertRaises(ValueError, "{0[0:foo}".format, 0) - self.assertRaises(KeyError, "{c]}".format) - self.assertRaises(ValueError, "{{ {{{0}}".format, 0) - self.assertRaises(ValueError, "{0}}".format, 0) - self.assertRaises(KeyError, "{foo}".format, bar=3) - self.assertRaises(ValueError, "{0!x}".format, 3) - self.assertRaises(ValueError, "{0!}".format, 0) - self.assertRaises(ValueError, "{0!rs}".format, 0) - self.assertRaises(ValueError, "{!}".format) - self.assertRaises(IndexError, "{:}".format) - self.assertRaises(IndexError, "{:s}".format) - self.assertRaises(IndexError, "{}".format) - big = "23098475029384702983476098230754973209482573" - self.assertRaises(ValueError, ("{" + big + "}").format) - self.assertRaises(ValueError, ("{[" + big + "]}").format, [0]) - - # test number formatter errors: - self.assertRaises(ValueError, '{0:x}'.format, 1j) - self.assertRaises(ValueError, '{0:x}'.format, 1.0) - self.assertRaises(ValueError, '{0:X}'.format, 1j) - self.assertRaises(ValueError, '{0:X}'.format, 1.0) - self.assertRaises(ValueError, '{0:o}'.format, 1j) - self.assertRaises(ValueError, '{0:o}'.format, 1.0) - self.assertRaises(ValueError, '{0:u}'.format, 1j) - self.assertRaises(ValueError, '{0:u}'.format, 1.0) - self.assertRaises(ValueError, '{0:i}'.format, 1j) - self.assertRaises(ValueError, '{0:i}'.format, 1.0) - self.assertRaises(ValueError, '{0:d}'.format, 1j) - self.assertRaises(ValueError, '{0:d}'.format, 1.0) - - # issue 6089 - self.assertRaises(ValueError, "{0[0]x}".format, [None]) - self.assertRaises(ValueError, "{0[0](10)}".format, [None]) - - # can't have a replacement on the field name portion - self.assertRaises(TypeError, '{0[{1}]}'.format, 'abcdefg', 4) - - # exceed maximum recursion depth - self.assertRaises(ValueError, "{0:{1:{2}}}".format, 'abc', 's', '') - self.assertRaises(ValueError, "{0:{1:{2:{3:{4:{5:{6}}}}}}}".format, - 0, 1, 2, 3, 4, 5, 6, 7) - - # string format spec errors - sign_msg = "Sign not allowed in string format specifier" - self.assertRaisesRegex(ValueError, sign_msg, "{0:-s}".format, '') - self.assertRaisesRegex(ValueError, sign_msg, format, "", "-") - space_msg = "Space not allowed in string format specifier" - self.assertRaisesRegex(ValueError, space_msg, "{: }".format, '') - self.assertRaises(ValueError, "{0:=s}".format, '') - - # Alternate formatting is not supported - self.assertRaises(ValueError, format, '', '#') - self.assertRaises(ValueError, format, '', '#20') - - # Non-ASCII - self.assertEqual("{0:s}{1:s}".format("ABC", "\u0410\u0411\u0412"), - 'ABC\u0410\u0411\u0412') - self.assertEqual("{0:.3s}".format("ABC\u0410\u0411\u0412"), - 'ABC') - self.assertEqual("{0:.0s}".format("ABC\u0410\u0411\u0412"), - '') - - self.assertEqual("{[{}]}".format({"{}": 5}), "5") - self.assertEqual("{[{}]}".format({"{}" : "a"}), "a") - self.assertEqual("{[{]}".format({"{" : "a"}), "a") - self.assertEqual("{[}]}".format({"}" : "a"}), "a") - self.assertEqual("{[[]}".format({"[" : "a"}), "a") - self.assertEqual("{[!]}".format({"!" : "a"}), "a") - self.assertRaises(ValueError, "{a{}b}".format, 42) - self.assertRaises(ValueError, "{a{b}".format, 42) - self.assertRaises(ValueError, "{[}".format, 42) - - self.assertEqual("0x{:0{:d}X}".format(0x0,16), "0x0000000000000000") - - # Blocking fallback - m = M('data') - self.assertEqual("{!r}".format(m), 'M(data)') - self.assertRaises(TypeError, "{!s}".format, m) - self.assertRaises(TypeError, "{}".format, m) - n = N('data') - self.assertEqual("{!r}".format(n), 'N(data)') - self.assertEqual("{!s}".format(n), 'N(data)') - self.assertRaises(TypeError, "{}".format, n) - - def test_format_map(self): - self.assertEqual(''.format_map({}), '') - self.assertEqual('a'.format_map({}), 'a') - self.assertEqual('ab'.format_map({}), 'ab') - self.assertEqual('a{{'.format_map({}), 'a{') - self.assertEqual('a}}'.format_map({}), 'a}') - self.assertEqual('{{b'.format_map({}), '{b') - self.assertEqual('}}b'.format_map({}), '}b') - self.assertEqual('a{{b'.format_map({}), 'a{b') - - # using mappings - class Mapping(dict): - def __missing__(self, key): - return key - self.assertEqual('{hello}'.format_map(Mapping()), 'hello') - self.assertEqual('{a} {world}'.format_map(Mapping(a='hello')), 'hello world') - - class InternalMapping: - def __init__(self): - self.mapping = {'a': 'hello'} - def __getitem__(self, key): - return self.mapping[key] - self.assertEqual('{a}'.format_map(InternalMapping()), 'hello') - - - class C: - def __init__(self, x=100): - self._x = x - def __format__(self, spec): - return spec - self.assertEqual('{foo._x}'.format_map({'foo': C(20)}), '20') - - # test various errors - self.assertRaises(TypeError, ''.format_map) - self.assertRaises(TypeError, 'a'.format_map) - - self.assertRaises(ValueError, '{'.format_map, {}) - self.assertRaises(ValueError, '}'.format_map, {}) - self.assertRaises(ValueError, 'a{'.format_map, {}) - self.assertRaises(ValueError, 'a}'.format_map, {}) - self.assertRaises(ValueError, '{a'.format_map, {}) - self.assertRaises(ValueError, '}a'.format_map, {}) - - # issue #12579: can't supply positional params to format_map - self.assertRaises(ValueError, '{}'.format_map, {'a' : 2}) - self.assertRaises(ValueError, '{}'.format_map, 'a') - self.assertRaises(ValueError, '{a} {}'.format_map, {"a" : 2, "b" : 1}) - - class BadMapping: - def __getitem__(self, key): - return 1/0 - self.assertRaises(KeyError, '{a}'.format_map, {}) - self.assertRaises(TypeError, '{a}'.format_map, []) - self.assertRaises(ZeroDivisionError, '{a}'.format_map, BadMapping()) - - @unittest.skip("TODO: RUSTPYTHON, killed for chewing up RAM") - def test_format_huge_precision(self): - format_string = ".{}f".format(sys.maxsize + 1) - with self.assertRaises(ValueError): - result = format(2.34, format_string) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_format_huge_width(self): - format_string = "{}f".format(sys.maxsize + 1) - with self.assertRaises(ValueError): - result = format(2.34, format_string) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_format_huge_item_number(self): - format_string = "{{{}:.6f}}".format(sys.maxsize + 1) - with self.assertRaises(ValueError): - result = format_string.format(2.34) - - def test_format_auto_numbering(self): - class C: - def __init__(self, x=100): - self._x = x - def __format__(self, spec): - return spec - - self.assertEqual('{}'.format(10), '10') - self.assertEqual('{:5}'.format('s'), 's ') - self.assertEqual('{!r}'.format('s'), "'s'") - self.assertEqual('{._x}'.format(C(10)), '10') - self.assertEqual('{[1]}'.format([1, 2]), '2') - self.assertEqual('{[a]}'.format({'a':4, 'b':2}), '4') - self.assertEqual('a{}b{}c'.format(0, 1), 'a0b1c') - - self.assertEqual('a{:{}}b'.format('x', '^10'), 'a x b') - self.assertEqual('a{:{}x}b'.format(20, '#'), 'a0x14b') - - # can't mix and match numbering and auto-numbering - self.assertRaises(ValueError, '{}{1}'.format, 1, 2) - self.assertRaises(ValueError, '{1}{}'.format, 1, 2) - self.assertRaises(ValueError, '{:{1}}'.format, 1, 2) - self.assertRaises(ValueError, '{0:{}}'.format, 1, 2) - - # can mix and match auto-numbering and named - self.assertEqual('{f}{}'.format(4, f='test'), 'test4') - self.assertEqual('{}{f}'.format(4, f='test'), '4test') - self.assertEqual('{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3') - self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_formatting(self): - string_tests.MixinStrUnicodeUserStringTest.test_formatting(self) - # Testing Unicode formatting strings... - self.assertEqual("%s, %s" % ("abc", "abc"), 'abc, abc') - self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, 2, 3), 'abc, abc, 1, 2.000000, 3.00') - self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, -2, 3), 'abc, abc, 1, -2.000000, 3.00') - self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.5), 'abc, abc, -1, -2.000000, 3.50') - self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.57), 'abc, abc, -1, -2.000000, 3.57') - self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 1003.57), 'abc, abc, -1, -2.000000, 1003.57') - self.assertEqual("%r, %r" % (b"abc", "abc"), "b'abc', 'abc'") - self.assertEqual("%r" % ("\u1234",), "'\u1234'") - self.assertEqual("%a" % ("\u1234",), "'\\u1234'") - self.assertEqual("%(x)s, %(y)s" % {'x':"abc", 'y':"def"}, 'abc, def') - self.assertEqual("%(x)s, %(\xfc)s" % {'x':"abc", '\xfc':"def"}, 'abc, def') - - self.assertEqual('%c' % 0x1234, '\u1234') - self.assertEqual('%c' % 0x21483, '\U00021483') - self.assertRaises(OverflowError, "%c".__mod__, (0x110000,)) - self.assertEqual('%c' % '\U00021483', '\U00021483') - self.assertRaises(TypeError, "%c".__mod__, "aa") - self.assertRaises(ValueError, "%.1\u1032f".__mod__, (1.0/3)) - self.assertRaises(TypeError, "%i".__mod__, "aa") - - # formatting jobs delegated from the string implementation: - self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...') - self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...') - self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...') - self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...') - self.assertEqual('...%(foo)s...' % {'foo':"abc",'def':123}, '...abc...') - self.assertEqual('...%(foo)s...' % {'foo':"abc",'def':123}, '...abc...') - self.assertEqual('...%s...%s...%s...%s...' % (1,2,3,"abc"), '...1...2...3...abc...') - self.assertEqual('...%%...%%s...%s...%s...%s...%s...' % (1,2,3,"abc"), '...%...%s...1...2...3...abc...') - self.assertEqual('...%s...' % "abc", '...abc...') - self.assertEqual('%*s' % (5,'abc',), ' abc') - self.assertEqual('%*s' % (-5,'abc',), 'abc ') - self.assertEqual('%*.*s' % (5,2,'abc',), ' ab') - self.assertEqual('%*.*s' % (5,3,'abc',), ' abc') - self.assertEqual('%i %*.*s' % (10, 5,3,'abc',), '10 abc') - self.assertEqual('%i%s %*.*s' % (10, 3, 5, 3, 'abc',), '103 abc') - self.assertEqual('%c' % 'a', 'a') - class Wrapper: - def __str__(self): - return '\u1234' - self.assertEqual('%s' % Wrapper(), '\u1234') - - # issue 3382 - NAN = float('nan') - INF = float('inf') - self.assertEqual('%f' % NAN, 'nan') - self.assertEqual('%F' % NAN, 'NAN') - self.assertEqual('%f' % INF, 'inf') - self.assertEqual('%F' % INF, 'INF') - - # PEP 393 - self.assertEqual('%.1s' % "a\xe9\u20ac", 'a') - self.assertEqual('%.2s' % "a\xe9\u20ac", 'a\xe9') - - #issue 19995 - class PseudoInt: - def __init__(self, value): - self.value = int(value) - def __int__(self): - return self.value - def __index__(self): - return self.value - class PseudoFloat: - def __init__(self, value): - self.value = float(value) - def __int__(self): - return int(self.value) - pi = PseudoFloat(3.1415) - letter_m = PseudoInt(109) - self.assertEqual('%x' % 42, '2a') - self.assertEqual('%X' % 15, 'F') - self.assertEqual('%o' % 9, '11') - self.assertEqual('%c' % 109, 'm') - self.assertEqual('%x' % letter_m, '6d') - self.assertEqual('%X' % letter_m, '6D') - self.assertEqual('%o' % letter_m, '155') - self.assertEqual('%c' % letter_m, 'm') - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14) - self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11) - self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79) - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi) - self.assertRaisesRegex(TypeError, '%x format: an integer is required, not complex', operator.mod, '%x', 3j) - self.assertRaisesRegex(TypeError, '%X format: an integer is required, not complex', operator.mod, '%X', 2j) - self.assertRaisesRegex(TypeError, '%o format: an integer is required, not complex', operator.mod, '%o', 1j) - self.assertRaisesRegex(TypeError, '%u format: a real number is required, not complex', operator.mod, '%u', 3j) - self.assertRaisesRegex(TypeError, '%i format: a real number is required, not complex', operator.mod, '%i', 2j) - self.assertRaisesRegex(TypeError, '%d format: a real number is required, not complex', operator.mod, '%d', 1j) - self.assertRaisesRegex(TypeError, '%c requires int or char', operator.mod, '%c', pi) - - class RaisingNumber: - def __int__(self): - raise RuntimeError('int') # should not be `TypeError` - def __index__(self): - raise RuntimeError('index') # should not be `TypeError` - - rn = RaisingNumber() - self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%d', rn) - self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%i', rn) - self.assertRaisesRegex(RuntimeError, 'int', operator.mod, '%u', rn) - self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%x', rn) - self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%X', rn) - self.assertRaisesRegex(RuntimeError, 'index', operator.mod, '%o', rn) - - def test_formatting_with_enum(self): - # issue18780 - import enum - class Float(float, enum.Enum): - # a mixed-in type will use the name for %s etc. - PI = 3.1415926 - class Int(enum.IntEnum): - # IntEnum uses the value and not the name for %s etc. - IDES = 15 - class Str(enum.StrEnum): - # StrEnum uses the value and not the name for %s etc. - ABC = 'abc' - # Testing Unicode formatting strings... - self.assertEqual("%s, %s" % (Str.ABC, Str.ABC), - 'abc, abc') - self.assertEqual("%s, %s, %d, %i, %u, %f, %5.2f" % - (Str.ABC, Str.ABC, - Int.IDES, Int.IDES, Int.IDES, - Float.PI, Float.PI), - 'abc, abc, 15, 15, 15, 3.141593, 3.14') - - # formatting jobs delegated from the string implementation: - self.assertEqual('...%(foo)s...' % {'foo':Str.ABC}, - '...abc...') - self.assertEqual('...%(foo)r...' % {'foo':Int.IDES}, - '...<Int.IDES: 15>...') - self.assertEqual('...%(foo)s...' % {'foo':Int.IDES}, - '...15...') - self.assertEqual('...%(foo)i...' % {'foo':Int.IDES}, - '...15...') - self.assertEqual('...%(foo)d...' % {'foo':Int.IDES}, - '...15...') - self.assertEqual('...%(foo)u...' % {'foo':Int.IDES, 'def':Float.PI}, - '...15...') - self.assertEqual('...%(foo)f...' % {'foo':Float.PI,'def':123}, - '...3.141593...') - - def test_formatting_huge_precision(self): - format_string = "%.{}f".format(sys.maxsize + 1) - with self.assertRaises(ValueError): - result = format_string % 2.34 - - def test_issue28598_strsubclass_rhs(self): - # A subclass of str with an __rmod__ method should be able to hook - # into the % operator - class SubclassedStr(str): - def __rmod__(self, other): - return 'Success, self.__rmod__({!r}) was called'.format(other) - self.assertEqual('lhs %% %r' % SubclassedStr('rhs'), - "Success, self.__rmod__('lhs %% %r') was called") - - @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_formatting_huge_precision_c_limits(self): - format_string = "%.{}f".format(_testcapi.INT_MAX + 1) - with self.assertRaises(ValueError): - result = format_string % 2.34 - - def test_formatting_huge_width(self): - format_string = "%{}f".format(sys.maxsize + 1) - with self.assertRaises(ValueError): - result = format_string % 2.34 - - def test_startswith_endswith_errors(self): - for meth in ('foo'.startswith, 'foo'.endswith): - with self.assertRaises(TypeError) as cm: - meth(['f']) - exc = str(cm.exception) - self.assertIn('str', exc) - self.assertIn('tuple', exc) - - @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR') - def test_format_float(self): - # should not format with a comma, but always with C locale - self.assertEqual('1.0', '%.1f' % 1.0) - - def test_constructor(self): - # unicode(obj) tests (this maps to PyObject_Unicode() at C level) - - self.assertEqual( - str('unicode remains unicode'), - 'unicode remains unicode' - ) - - for text in ('ascii', '\xe9', '\u20ac', '\U0010FFFF'): - subclass = StrSubclass(text) - self.assertEqual(str(subclass), text) - self.assertEqual(len(subclass), len(text)) - if text == 'ascii': - self.assertEqual(subclass.encode('ascii'), b'ascii') - self.assertEqual(subclass.encode('utf-8'), b'ascii') - - self.assertEqual( - str('strings are converted to unicode'), - 'strings are converted to unicode' - ) - - class StringCompat: - def __init__(self, x): - self.x = x - def __str__(self): - return self.x - - self.assertEqual( - str(StringCompat('__str__ compatible objects are recognized')), - '__str__ compatible objects are recognized' - ) - - # unicode(obj) is compatible to str(): - - o = StringCompat('unicode(obj) is compatible to str()') - self.assertEqual(str(o), 'unicode(obj) is compatible to str()') - self.assertEqual(str(o), 'unicode(obj) is compatible to str()') - - for obj in (123, 123.45, 123): - self.assertEqual(str(obj), str(str(obj))) - - # unicode(obj, encoding, error) tests (this maps to - # PyUnicode_FromEncodedObject() at C level) - - self.assertRaises( - TypeError, - str, - 'decoding unicode is not supported', - 'utf-8', - 'strict' - ) - - self.assertEqual( - str(b'strings are decoded to unicode', 'utf-8', 'strict'), - 'strings are decoded to unicode' - ) - - self.assertEqual( - str( - memoryview(b'character buffers are decoded to unicode'), - 'utf-8', - 'strict' - ), - 'character buffers are decoded to unicode' - ) - - self.assertRaises(TypeError, str, 42, 42, 42) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_constructor_keyword_args(self): - """Pass various keyword argument combinations to the constructor.""" - # The object argument can be passed as a keyword. - self.assertEqual(str(object='foo'), 'foo') - self.assertEqual(str(object=b'foo', encoding='utf-8'), 'foo') - # The errors argument without encoding triggers "decode" mode. - self.assertEqual(str(b'foo', errors='strict'), 'foo') # not "b'foo'" - self.assertEqual(str(object=b'foo', errors='strict'), 'foo') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_constructor_defaults(self): - """Check the constructor argument defaults.""" - # The object argument defaults to '' or b''. - self.assertEqual(str(), '') - self.assertEqual(str(errors='strict'), '') - utf8_cent = '¢'.encode('utf-8') - # The encoding argument defaults to utf-8. - self.assertEqual(str(utf8_cent, errors='strict'), '¢') - # The errors argument defaults to strict. - self.assertRaises(UnicodeDecodeError, str, utf8_cent, encoding='ascii') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_codecs_utf7(self): - utfTests = [ - ('A\u2262\u0391.', b'A+ImIDkQ.'), # RFC2152 example - ('Hi Mom -\u263a-!', b'Hi Mom -+Jjo--!'), # RFC2152 example - ('\u65E5\u672C\u8A9E', b'+ZeVnLIqe-'), # RFC2152 example - ('Item 3 is \u00a31.', b'Item 3 is +AKM-1.'), # RFC2152 example - ('+', b'+-'), - ('+-', b'+--'), - ('+?', b'+-?'), - (r'\?', b'+AFw?'), - ('+?', b'+-?'), - (r'\\?', b'+AFwAXA?'), - (r'\\\?', b'+AFwAXABc?'), - (r'++--', b'+-+---'), - ('\U000abcde', b'+2m/c3g-'), # surrogate pairs - ('/', b'/'), - ] - - for (x, y) in utfTests: - self.assertEqual(x.encode('utf-7'), y) - - # Unpaired surrogates are passed through - self.assertEqual('\uD801'.encode('utf-7'), b'+2AE-') - self.assertEqual('\uD801x'.encode('utf-7'), b'+2AE-x') - self.assertEqual('\uDC01'.encode('utf-7'), b'+3AE-') - self.assertEqual('\uDC01x'.encode('utf-7'), b'+3AE-x') - self.assertEqual(b'+2AE-'.decode('utf-7'), '\uD801') - self.assertEqual(b'+2AE-x'.decode('utf-7'), '\uD801x') - self.assertEqual(b'+3AE-'.decode('utf-7'), '\uDC01') - self.assertEqual(b'+3AE-x'.decode('utf-7'), '\uDC01x') - - self.assertEqual('\uD801\U000abcde'.encode('utf-7'), b'+2AHab9ze-') - self.assertEqual(b'+2AHab9ze-'.decode('utf-7'), '\uD801\U000abcde') - - # Issue #2242: crash on some Windows/MSVC versions - self.assertEqual(b'+\xc1'.decode('utf-7', 'ignore'), '') - - # Direct encoded characters - set_d = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'(),-./:?" - # Optional direct characters - set_o = '!"#$%&*;<=>@[]^_`{|}' - for c in set_d: - self.assertEqual(c.encode('utf7'), c.encode('ascii')) - self.assertEqual(c.encode('ascii').decode('utf7'), c) - for c in set_o: - self.assertEqual(c.encode('ascii').decode('utf7'), c) - - with self.assertRaisesRegex(UnicodeDecodeError, - 'ill-formed sequence'): - b'+@'.decode('utf-7') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_codecs_utf8(self): - self.assertEqual(''.encode('utf-8'), b'') - self.assertEqual('\u20ac'.encode('utf-8'), b'\xe2\x82\xac') - self.assertEqual('\U00010002'.encode('utf-8'), b'\xf0\x90\x80\x82') - self.assertEqual('\U00023456'.encode('utf-8'), b'\xf0\xa3\x91\x96') - self.assertEqual('\ud800'.encode('utf-8', 'surrogatepass'), b'\xed\xa0\x80') - self.assertEqual('\udc00'.encode('utf-8', 'surrogatepass'), b'\xed\xb0\x80') - self.assertEqual(('\U00010002'*10).encode('utf-8'), - b'\xf0\x90\x80\x82'*10) - self.assertEqual( - '\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' - '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' - '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c' - '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067' - '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das' - ' Nunstuck git und'.encode('utf-8'), - b'\xe6\xad\xa3\xe7\xa2\xba\xe3\x81\xab\xe8\xa8\x80\xe3\x81' - b'\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3\xe3\x81\xaf\xe3' - b'\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe' - b'\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' - b'\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8' - b'\xaa\x9e\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81' - b'\xe3\x81\x82\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81' - b'\x9f\xe3\x82\x89\xe3\x82\x81\xe3\x81\xa7\xe3\x81\x99\xe3' - b'\x80\x82\xe5\xae\x9f\xe9\x9a\x9b\xe3\x81\xab\xe3\x81\xaf' - b'\xe3\x80\x8cWenn ist das Nunstuck git und' - ) - - # UTF-8 specific decoding tests - self.assertEqual(str(b'\xf0\xa3\x91\x96', 'utf-8'), '\U00023456' ) - self.assertEqual(str(b'\xf0\x90\x80\x82', 'utf-8'), '\U00010002' ) - self.assertEqual(str(b'\xe2\x82\xac', 'utf-8'), '\u20ac' ) - - # Other possible utf-8 test cases: - # * strict decoding testing for all of the - # UTF8_ERROR cases in PyUnicode_DecodeUTF8 - - def test_utf8_decode_valid_sequences(self): - sequences = [ - # single byte - (b'\x00', '\x00'), (b'a', 'a'), (b'\x7f', '\x7f'), - # 2 bytes - (b'\xc2\x80', '\x80'), (b'\xdf\xbf', '\u07ff'), - # 3 bytes - (b'\xe0\xa0\x80', '\u0800'), (b'\xed\x9f\xbf', '\ud7ff'), - (b'\xee\x80\x80', '\uE000'), (b'\xef\xbf\xbf', '\uffff'), - # 4 bytes - (b'\xF0\x90\x80\x80', '\U00010000'), - (b'\xf4\x8f\xbf\xbf', '\U0010FFFF') - ] - for seq, res in sequences: - self.assertEqual(seq.decode('utf-8'), res) - - - def test_utf8_decode_invalid_sequences(self): - # continuation bytes in a sequence of 2, 3, or 4 bytes - continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)] - # start bytes of a 2-byte sequence equivalent to code points < 0x7F - invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)] - # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF - invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)] - invalid_start_bytes = ( - continuation_bytes + invalid_2B_seq_start_bytes + - invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)] - ) - - for byte in invalid_start_bytes: - self.assertRaises(UnicodeDecodeError, byte.decode, 'utf-8') - - for sb in invalid_2B_seq_start_bytes: - for cb in continuation_bytes: - self.assertRaises(UnicodeDecodeError, (sb+cb).decode, 'utf-8') - - for sb in invalid_4B_seq_start_bytes: - for cb1 in continuation_bytes[:3]: - for cb3 in continuation_bytes[:3]: - self.assertRaises(UnicodeDecodeError, - (sb+cb1+b'\x80'+cb3).decode, 'utf-8') - - for cb in [bytes([x]) for x in range(0x80, 0xA0)]: - self.assertRaises(UnicodeDecodeError, - (b'\xE0'+cb+b'\x80').decode, 'utf-8') - self.assertRaises(UnicodeDecodeError, - (b'\xE0'+cb+b'\xBF').decode, 'utf-8') - # surrogates - for cb in [bytes([x]) for x in range(0xA0, 0xC0)]: - self.assertRaises(UnicodeDecodeError, - (b'\xED'+cb+b'\x80').decode, 'utf-8') - self.assertRaises(UnicodeDecodeError, - (b'\xED'+cb+b'\xBF').decode, 'utf-8') - for cb in [bytes([x]) for x in range(0x80, 0x90)]: - self.assertRaises(UnicodeDecodeError, - (b'\xF0'+cb+b'\x80\x80').decode, 'utf-8') - self.assertRaises(UnicodeDecodeError, - (b'\xF0'+cb+b'\xBF\xBF').decode, 'utf-8') - for cb in [bytes([x]) for x in range(0x90, 0xC0)]: - self.assertRaises(UnicodeDecodeError, - (b'\xF4'+cb+b'\x80\x80').decode, 'utf-8') - self.assertRaises(UnicodeDecodeError, - (b'\xF4'+cb+b'\xBF\xBF').decode, 'utf-8') - - def test_issue8271(self): - # Issue #8271: during the decoding of an invalid UTF-8 byte sequence, - # only the start byte and the continuation byte(s) are now considered - # invalid, instead of the number of bytes specified by the start byte. - # See https://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (page 95, - # table 3-8, Row 2) for more information about the algorithm used. - FFFD = '\ufffd' - sequences = [ - # invalid start bytes - (b'\x80', FFFD), # continuation byte - (b'\x80\x80', FFFD*2), # 2 continuation bytes - (b'\xc0', FFFD), - (b'\xc0\xc0', FFFD*2), - (b'\xc1', FFFD), - (b'\xc1\xc0', FFFD*2), - (b'\xc0\xc1', FFFD*2), - # with start byte of a 2-byte sequence - (b'\xc2', FFFD), # only the start byte - (b'\xc2\xc2', FFFD*2), # 2 start bytes - (b'\xc2\xc2\xc2', FFFD*3), # 3 start bytes - (b'\xc2\x41', FFFD+'A'), # invalid continuation byte - # with start byte of a 3-byte sequence - (b'\xe1', FFFD), # only the start byte - (b'\xe1\xe1', FFFD*2), # 2 start bytes - (b'\xe1\xe1\xe1', FFFD*3), # 3 start bytes - (b'\xe1\xe1\xe1\xe1', FFFD*4), # 4 start bytes - (b'\xe1\x80', FFFD), # only 1 continuation byte - (b'\xe1\x41', FFFD+'A'), # invalid continuation byte - (b'\xe1\x41\x80', FFFD+'A'+FFFD), # invalid cb followed by valid cb - (b'\xe1\x41\x41', FFFD+'AA'), # 2 invalid continuation bytes - (b'\xe1\x80\x41', FFFD+'A'), # only 1 valid continuation byte - (b'\xe1\x80\xe1\x41', FFFD*2+'A'), # 1 valid and the other invalid - (b'\xe1\x41\xe1\x80', FFFD+'A'+FFFD), # 1 invalid and the other valid - # with start byte of a 4-byte sequence - (b'\xf1', FFFD), # only the start byte - (b'\xf1\xf1', FFFD*2), # 2 start bytes - (b'\xf1\xf1\xf1', FFFD*3), # 3 start bytes - (b'\xf1\xf1\xf1\xf1', FFFD*4), # 4 start bytes - (b'\xf1\xf1\xf1\xf1\xf1', FFFD*5), # 5 start bytes - (b'\xf1\x80', FFFD), # only 1 continuation bytes - (b'\xf1\x80\x80', FFFD), # only 2 continuation bytes - (b'\xf1\x80\x41', FFFD+'A'), # 1 valid cb and 1 invalid - (b'\xf1\x80\x41\x41', FFFD+'AA'), # 1 valid cb and 1 invalid - (b'\xf1\x80\x80\x41', FFFD+'A'), # 2 valid cb and 1 invalid - (b'\xf1\x41\x80', FFFD+'A'+FFFD), # 1 invalid cv and 1 valid - (b'\xf1\x41\x80\x80', FFFD+'A'+FFFD*2), # 1 invalid cb and 2 invalid - (b'\xf1\x41\x80\x41', FFFD+'A'+FFFD+'A'), # 2 invalid cb and 1 invalid - (b'\xf1\x41\x41\x80', FFFD+'AA'+FFFD), # 1 valid cb and 1 invalid - (b'\xf1\x41\xf1\x80', FFFD+'A'+FFFD), - (b'\xf1\x41\x80\xf1', FFFD+'A'+FFFD*2), - (b'\xf1\xf1\x80\x41', FFFD*2+'A'), - (b'\xf1\x41\xf1\xf1', FFFD+'A'+FFFD*2), - # with invalid start byte of a 4-byte sequence (rfc2279) - (b'\xf5', FFFD), # only the start byte - (b'\xf5\xf5', FFFD*2), # 2 start bytes - (b'\xf5\x80', FFFD*2), # only 1 continuation byte - (b'\xf5\x80\x80', FFFD*3), # only 2 continuation byte - (b'\xf5\x80\x80\x80', FFFD*4), # 3 continuation bytes - (b'\xf5\x80\x41', FFFD*2+'A'), # 1 valid cb and 1 invalid - (b'\xf5\x80\x41\xf5', FFFD*2+'A'+FFFD), - (b'\xf5\x41\x80\x80\x41', FFFD+'A'+FFFD*2+'A'), - # with invalid start byte of a 5-byte sequence (rfc2279) - (b'\xf8', FFFD), # only the start byte - (b'\xf8\xf8', FFFD*2), # 2 start bytes - (b'\xf8\x80', FFFD*2), # only one continuation byte - (b'\xf8\x80\x41', FFFD*2 + 'A'), # 1 valid cb and 1 invalid - (b'\xf8\x80\x80\x80\x80', FFFD*5), # invalid 5 bytes seq with 5 bytes - # with invalid start byte of a 6-byte sequence (rfc2279) - (b'\xfc', FFFD), # only the start byte - (b'\xfc\xfc', FFFD*2), # 2 start bytes - (b'\xfc\x80\x80', FFFD*3), # only 2 continuation bytes - (b'\xfc\x80\x80\x80\x80\x80', FFFD*6), # 6 continuation bytes - # invalid start byte - (b'\xfe', FFFD), - (b'\xfe\x80\x80', FFFD*3), - # other sequences - (b'\xf1\x80\x41\x42\x43', '\ufffd\x41\x42\x43'), - (b'\xf1\x80\xff\x42\x43', '\ufffd\ufffd\x42\x43'), - (b'\xf1\x80\xc2\x81\x43', '\ufffd\x81\x43'), - (b'\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64', - '\x61\uFFFD\uFFFD\uFFFD\x62\uFFFD\x63\uFFFD\uFFFD\x64'), - ] - for n, (seq, res) in enumerate(sequences): - self.assertRaises(UnicodeDecodeError, seq.decode, 'utf-8', 'strict') - self.assertEqual(seq.decode('utf-8', 'replace'), res) - self.assertEqual((seq+b'b').decode('utf-8', 'replace'), res+'b') - self.assertEqual(seq.decode('utf-8', 'ignore'), - res.replace('\uFFFD', '')) - - def assertCorrectUTF8Decoding(self, seq, res, err): - """ - Check that an invalid UTF-8 sequence raises a UnicodeDecodeError when - 'strict' is used, returns res when 'replace' is used, and that doesn't - return anything when 'ignore' is used. - """ - with self.assertRaises(UnicodeDecodeError) as cm: - seq.decode('utf-8') - exc = cm.exception - - self.assertIn(err, str(exc)) - self.assertEqual(seq.decode('utf-8', 'replace'), res) - self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'replace'), - 'aaaa' + res + 'bbbb') - res = res.replace('\ufffd', '') - self.assertEqual(seq.decode('utf-8', 'ignore'), res) - self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore'), - 'aaaa' + res + 'bbbb') - - def test_invalid_start_byte(self): - """ - Test that an 'invalid start byte' error is raised when the first byte - is not in the ASCII range or is not a valid start byte of a 2-, 3-, or - 4-bytes sequence. The invalid start byte is replaced with a single - U+FFFD when errors='replace'. - E.g. <80> is a continuation byte and can appear only after a start byte. - """ - FFFD = '\ufffd' - for byte in b'\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF': - self.assertCorrectUTF8Decoding(bytes([byte]), '\ufffd', - 'invalid start byte') - - def test_unexpected_end_of_data(self): - """ - Test that an 'unexpected end of data' error is raised when the string - ends after a start byte of a 2-, 3-, or 4-bytes sequence without having - enough continuation bytes. The incomplete sequence is replaced with a - single U+FFFD when errors='replace'. - E.g. in the sequence <F3 80 80>, F3 is the start byte of a 4-bytes - sequence, but it's followed by only 2 valid continuation bytes and the - last continuation bytes is missing. - Note: the continuation bytes must be all valid, if one of them is - invalid another error will be raised. - """ - sequences = [ - 'C2', 'DF', - 'E0 A0', 'E0 BF', 'E1 80', 'E1 BF', 'EC 80', 'EC BF', - 'ED 80', 'ED 9F', 'EE 80', 'EE BF', 'EF 80', 'EF BF', - 'F0 90', 'F0 BF', 'F0 90 80', 'F0 90 BF', 'F0 BF 80', 'F0 BF BF', - 'F1 80', 'F1 BF', 'F1 80 80', 'F1 80 BF', 'F1 BF 80', 'F1 BF BF', - 'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF', - 'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF' - ] - FFFD = '\ufffd' - for seq in sequences: - self.assertCorrectUTF8Decoding(bytes.fromhex(seq), '\ufffd', - 'unexpected end of data') - - def test_invalid_cb_for_2bytes_seq(self): - """ - Test that an 'invalid continuation byte' error is raised when the - continuation byte of a 2-bytes sequence is invalid. The start byte - is replaced by a single U+FFFD and the second byte is handled - separately when errors='replace'. - E.g. in the sequence <C2 41>, C2 is the start byte of a 2-bytes - sequence, but 41 is not a valid continuation byte because it's the - ASCII letter 'A'. - """ - FFFD = '\ufffd' - FFFDx2 = FFFD * 2 - sequences = [ - ('C2 00', FFFD+'\x00'), ('C2 7F', FFFD+'\x7f'), - ('C2 C0', FFFDx2), ('C2 FF', FFFDx2), - ('DF 00', FFFD+'\x00'), ('DF 7F', FFFD+'\x7f'), - ('DF C0', FFFDx2), ('DF FF', FFFDx2), - ] - for seq, res in sequences: - self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res, - 'invalid continuation byte') - - def test_invalid_cb_for_3bytes_seq(self): - """ - Test that an 'invalid continuation byte' error is raised when the - continuation byte(s) of a 3-bytes sequence are invalid. When - errors='replace', if the first continuation byte is valid, the first - two bytes (start byte + 1st cb) are replaced by a single U+FFFD and the - third byte is handled separately, otherwise only the start byte is - replaced with a U+FFFD and the other continuation bytes are handled - separately. - E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes - sequence, 80 is a valid continuation byte, but 41 is not a valid cb - because it's the ASCII letter 'A'. - Note: when the start byte is E0 or ED, the valid ranges for the first - continuation byte are limited to A0..BF and 80..9F respectively. - Python 2 used to consider all the bytes in range 80..BF valid when the - start byte was ED. This is fixed in Python 3. - """ - FFFD = '\ufffd' - FFFDx2 = FFFD * 2 - sequences = [ - ('E0 00', FFFD+'\x00'), ('E0 7F', FFFD+'\x7f'), ('E0 80', FFFDx2), - ('E0 9F', FFFDx2), ('E0 C0', FFFDx2), ('E0 FF', FFFDx2), - ('E0 A0 00', FFFD+'\x00'), ('E0 A0 7F', FFFD+'\x7f'), - ('E0 A0 C0', FFFDx2), ('E0 A0 FF', FFFDx2), - ('E0 BF 00', FFFD+'\x00'), ('E0 BF 7F', FFFD+'\x7f'), - ('E0 BF C0', FFFDx2), ('E0 BF FF', FFFDx2), ('E1 00', FFFD+'\x00'), - ('E1 7F', FFFD+'\x7f'), ('E1 C0', FFFDx2), ('E1 FF', FFFDx2), - ('E1 80 00', FFFD+'\x00'), ('E1 80 7F', FFFD+'\x7f'), - ('E1 80 C0', FFFDx2), ('E1 80 FF', FFFDx2), - ('E1 BF 00', FFFD+'\x00'), ('E1 BF 7F', FFFD+'\x7f'), - ('E1 BF C0', FFFDx2), ('E1 BF FF', FFFDx2), ('EC 00', FFFD+'\x00'), - ('EC 7F', FFFD+'\x7f'), ('EC C0', FFFDx2), ('EC FF', FFFDx2), - ('EC 80 00', FFFD+'\x00'), ('EC 80 7F', FFFD+'\x7f'), - ('EC 80 C0', FFFDx2), ('EC 80 FF', FFFDx2), - ('EC BF 00', FFFD+'\x00'), ('EC BF 7F', FFFD+'\x7f'), - ('EC BF C0', FFFDx2), ('EC BF FF', FFFDx2), ('ED 00', FFFD+'\x00'), - ('ED 7F', FFFD+'\x7f'), - ('ED A0', FFFDx2), ('ED BF', FFFDx2), # see note ^ - ('ED C0', FFFDx2), ('ED FF', FFFDx2), ('ED 80 00', FFFD+'\x00'), - ('ED 80 7F', FFFD+'\x7f'), ('ED 80 C0', FFFDx2), - ('ED 80 FF', FFFDx2), ('ED 9F 00', FFFD+'\x00'), - ('ED 9F 7F', FFFD+'\x7f'), ('ED 9F C0', FFFDx2), - ('ED 9F FF', FFFDx2), ('EE 00', FFFD+'\x00'), - ('EE 7F', FFFD+'\x7f'), ('EE C0', FFFDx2), ('EE FF', FFFDx2), - ('EE 80 00', FFFD+'\x00'), ('EE 80 7F', FFFD+'\x7f'), - ('EE 80 C0', FFFDx2), ('EE 80 FF', FFFDx2), - ('EE BF 00', FFFD+'\x00'), ('EE BF 7F', FFFD+'\x7f'), - ('EE BF C0', FFFDx2), ('EE BF FF', FFFDx2), ('EF 00', FFFD+'\x00'), - ('EF 7F', FFFD+'\x7f'), ('EF C0', FFFDx2), ('EF FF', FFFDx2), - ('EF 80 00', FFFD+'\x00'), ('EF 80 7F', FFFD+'\x7f'), - ('EF 80 C0', FFFDx2), ('EF 80 FF', FFFDx2), - ('EF BF 00', FFFD+'\x00'), ('EF BF 7F', FFFD+'\x7f'), - ('EF BF C0', FFFDx2), ('EF BF FF', FFFDx2), - ] - for seq, res in sequences: - self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res, - 'invalid continuation byte') - - def test_invalid_cb_for_4bytes_seq(self): - """ - Test that an 'invalid continuation byte' error is raised when the - continuation byte(s) of a 4-bytes sequence are invalid. When - errors='replace',the start byte and all the following valid - continuation bytes are replaced with a single U+FFFD, and all the bytes - starting from the first invalid continuation bytes (included) are - handled separately. - E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes - sequence, 80 is a valid continuation byte, but 41 is not a valid cb - because it's the ASCII letter 'A'. - Note: when the start byte is E0 or ED, the valid ranges for the first - continuation byte are limited to A0..BF and 80..9F respectively. - However, when the start byte is ED, Python 2 considers all the bytes - in range 80..BF valid. This is fixed in Python 3. - """ - FFFD = '\ufffd' - FFFDx2 = FFFD * 2 - sequences = [ - ('F0 00', FFFD+'\x00'), ('F0 7F', FFFD+'\x7f'), ('F0 80', FFFDx2), - ('F0 8F', FFFDx2), ('F0 C0', FFFDx2), ('F0 FF', FFFDx2), - ('F0 90 00', FFFD+'\x00'), ('F0 90 7F', FFFD+'\x7f'), - ('F0 90 C0', FFFDx2), ('F0 90 FF', FFFDx2), - ('F0 BF 00', FFFD+'\x00'), ('F0 BF 7F', FFFD+'\x7f'), - ('F0 BF C0', FFFDx2), ('F0 BF FF', FFFDx2), - ('F0 90 80 00', FFFD+'\x00'), ('F0 90 80 7F', FFFD+'\x7f'), - ('F0 90 80 C0', FFFDx2), ('F0 90 80 FF', FFFDx2), - ('F0 90 BF 00', FFFD+'\x00'), ('F0 90 BF 7F', FFFD+'\x7f'), - ('F0 90 BF C0', FFFDx2), ('F0 90 BF FF', FFFDx2), - ('F0 BF 80 00', FFFD+'\x00'), ('F0 BF 80 7F', FFFD+'\x7f'), - ('F0 BF 80 C0', FFFDx2), ('F0 BF 80 FF', FFFDx2), - ('F0 BF BF 00', FFFD+'\x00'), ('F0 BF BF 7F', FFFD+'\x7f'), - ('F0 BF BF C0', FFFDx2), ('F0 BF BF FF', FFFDx2), - ('F1 00', FFFD+'\x00'), ('F1 7F', FFFD+'\x7f'), ('F1 C0', FFFDx2), - ('F1 FF', FFFDx2), ('F1 80 00', FFFD+'\x00'), - ('F1 80 7F', FFFD+'\x7f'), ('F1 80 C0', FFFDx2), - ('F1 80 FF', FFFDx2), ('F1 BF 00', FFFD+'\x00'), - ('F1 BF 7F', FFFD+'\x7f'), ('F1 BF C0', FFFDx2), - ('F1 BF FF', FFFDx2), ('F1 80 80 00', FFFD+'\x00'), - ('F1 80 80 7F', FFFD+'\x7f'), ('F1 80 80 C0', FFFDx2), - ('F1 80 80 FF', FFFDx2), ('F1 80 BF 00', FFFD+'\x00'), - ('F1 80 BF 7F', FFFD+'\x7f'), ('F1 80 BF C0', FFFDx2), - ('F1 80 BF FF', FFFDx2), ('F1 BF 80 00', FFFD+'\x00'), - ('F1 BF 80 7F', FFFD+'\x7f'), ('F1 BF 80 C0', FFFDx2), - ('F1 BF 80 FF', FFFDx2), ('F1 BF BF 00', FFFD+'\x00'), - ('F1 BF BF 7F', FFFD+'\x7f'), ('F1 BF BF C0', FFFDx2), - ('F1 BF BF FF', FFFDx2), ('F3 00', FFFD+'\x00'), - ('F3 7F', FFFD+'\x7f'), ('F3 C0', FFFDx2), ('F3 FF', FFFDx2), - ('F3 80 00', FFFD+'\x00'), ('F3 80 7F', FFFD+'\x7f'), - ('F3 80 C0', FFFDx2), ('F3 80 FF', FFFDx2), - ('F3 BF 00', FFFD+'\x00'), ('F3 BF 7F', FFFD+'\x7f'), - ('F3 BF C0', FFFDx2), ('F3 BF FF', FFFDx2), - ('F3 80 80 00', FFFD+'\x00'), ('F3 80 80 7F', FFFD+'\x7f'), - ('F3 80 80 C0', FFFDx2), ('F3 80 80 FF', FFFDx2), - ('F3 80 BF 00', FFFD+'\x00'), ('F3 80 BF 7F', FFFD+'\x7f'), - ('F3 80 BF C0', FFFDx2), ('F3 80 BF FF', FFFDx2), - ('F3 BF 80 00', FFFD+'\x00'), ('F3 BF 80 7F', FFFD+'\x7f'), - ('F3 BF 80 C0', FFFDx2), ('F3 BF 80 FF', FFFDx2), - ('F3 BF BF 00', FFFD+'\x00'), ('F3 BF BF 7F', FFFD+'\x7f'), - ('F3 BF BF C0', FFFDx2), ('F3 BF BF FF', FFFDx2), - ('F4 00', FFFD+'\x00'), ('F4 7F', FFFD+'\x7f'), ('F4 90', FFFDx2), - ('F4 BF', FFFDx2), ('F4 C0', FFFDx2), ('F4 FF', FFFDx2), - ('F4 80 00', FFFD+'\x00'), ('F4 80 7F', FFFD+'\x7f'), - ('F4 80 C0', FFFDx2), ('F4 80 FF', FFFDx2), - ('F4 8F 00', FFFD+'\x00'), ('F4 8F 7F', FFFD+'\x7f'), - ('F4 8F C0', FFFDx2), ('F4 8F FF', FFFDx2), - ('F4 80 80 00', FFFD+'\x00'), ('F4 80 80 7F', FFFD+'\x7f'), - ('F4 80 80 C0', FFFDx2), ('F4 80 80 FF', FFFDx2), - ('F4 80 BF 00', FFFD+'\x00'), ('F4 80 BF 7F', FFFD+'\x7f'), - ('F4 80 BF C0', FFFDx2), ('F4 80 BF FF', FFFDx2), - ('F4 8F 80 00', FFFD+'\x00'), ('F4 8F 80 7F', FFFD+'\x7f'), - ('F4 8F 80 C0', FFFDx2), ('F4 8F 80 FF', FFFDx2), - ('F4 8F BF 00', FFFD+'\x00'), ('F4 8F BF 7F', FFFD+'\x7f'), - ('F4 8F BF C0', FFFDx2), ('F4 8F BF FF', FFFDx2) - ] - for seq, res in sequences: - self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res, - 'invalid continuation byte') - - def test_codecs_idna(self): - # Test whether trailing dot is preserved - self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.") - - def test_codecs_errors(self): - # Error handling (encoding) - self.assertRaises(UnicodeError, 'Andr\202 x'.encode, 'ascii') - self.assertRaises(UnicodeError, 'Andr\202 x'.encode, 'ascii','strict') - self.assertEqual('Andr\202 x'.encode('ascii','ignore'), b"Andr x") - self.assertEqual('Andr\202 x'.encode('ascii','replace'), b"Andr? x") - self.assertEqual('Andr\202 x'.encode('ascii', 'replace'), - 'Andr\202 x'.encode('ascii', errors='replace')) - self.assertEqual('Andr\202 x'.encode('ascii', 'ignore'), - 'Andr\202 x'.encode(encoding='ascii', errors='ignore')) - - # Error handling (decoding) - self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii') - self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii', 'strict') - self.assertEqual(str(b'Andr\202 x', 'ascii', 'ignore'), "Andr x") - self.assertEqual(str(b'Andr\202 x', 'ascii', 'replace'), 'Andr\uFFFD x') - self.assertEqual(str(b'\202 x', 'ascii', 'replace'), '\uFFFD x') - - # Error handling (unknown character names) - self.assertEqual(b"\\N{foo}xx".decode("unicode-escape", "ignore"), "xx") - - # Error handling (truncated escape sequence) - self.assertRaises(UnicodeError, b"\\".decode, "unicode-escape") - - self.assertRaises(TypeError, b"hello".decode, "test.unicode1") - self.assertRaises(TypeError, str, b"hello", "test.unicode2") - self.assertRaises(TypeError, "hello".encode, "test.unicode1") - self.assertRaises(TypeError, "hello".encode, "test.unicode2") - - # Error handling (wrong arguments) - self.assertRaises(TypeError, "hello".encode, 42, 42, 42) - - # Error handling (lone surrogate in - # _PyUnicode_TransformDecimalAndSpaceToASCII()) - self.assertRaises(ValueError, int, "\ud800") - self.assertRaises(ValueError, int, "\udf00") - self.assertRaises(ValueError, float, "\ud800") - self.assertRaises(ValueError, float, "\udf00") - self.assertRaises(ValueError, complex, "\ud800") - self.assertRaises(ValueError, complex, "\udf00") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_codecs(self): - # Encoding - self.assertEqual('hello'.encode('ascii'), b'hello') - self.assertEqual('hello'.encode('utf-7'), b'hello') - self.assertEqual('hello'.encode('utf-8'), b'hello') - self.assertEqual('hello'.encode('utf-8'), b'hello') - self.assertEqual('hello'.encode('utf-16-le'), b'h\000e\000l\000l\000o\000') - self.assertEqual('hello'.encode('utf-16-be'), b'\000h\000e\000l\000l\000o') - self.assertEqual('hello'.encode('latin-1'), b'hello') - - # Default encoding is utf-8 - self.assertEqual('\u2603'.encode(), b'\xe2\x98\x83') - - # Roundtrip safety for BMP (just the first 1024 chars) - for c in range(1024): - u = chr(c) - for encoding in ('utf-7', 'utf-8', 'utf-16', 'utf-16-le', - 'utf-16-be', 'raw_unicode_escape', - 'unicode_escape'): - self.assertEqual(str(u.encode(encoding),encoding), u) - - # Roundtrip safety for BMP (just the first 256 chars) - for c in range(256): - u = chr(c) - for encoding in ('latin-1',): - self.assertEqual(str(u.encode(encoding),encoding), u) - - # Roundtrip safety for BMP (just the first 128 chars) - for c in range(128): - u = chr(c) - for encoding in ('ascii',): - self.assertEqual(str(u.encode(encoding),encoding), u) - - # Roundtrip safety for non-BMP (just a few chars) - with warnings.catch_warnings(): - u = '\U00010001\U00020002\U00030003\U00040004\U00050005' - for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', - 'raw_unicode_escape', 'unicode_escape'): - self.assertEqual(str(u.encode(encoding),encoding), u) - - # UTF-8 must be roundtrip safe for all code points - # (except surrogates, which are forbidden). - u = ''.join(map(chr, list(range(0, 0xd800)) + - list(range(0xe000, 0x110000)))) - for encoding in ('utf-8',): - self.assertEqual(str(u.encode(encoding),encoding), u) - - def test_codecs_charmap(self): - # 0-127 - s = bytes(range(128)) - for encoding in ( - 'cp037', 'cp1026', 'cp273', - 'cp437', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850', - 'cp852', 'cp855', 'cp858', 'cp860', 'cp861', 'cp862', - 'cp863', 'cp865', 'cp866', 'cp1125', - 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15', - 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6', - 'iso8859_7', 'iso8859_9', - 'koi8_r', 'koi8_t', 'koi8_u', 'kz1048', 'latin_1', - 'mac_cyrillic', 'mac_latin2', - - 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', - 'cp1256', 'cp1257', 'cp1258', - 'cp856', 'cp857', 'cp864', 'cp869', 'cp874', - - 'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish', - 'cp1006', 'iso8859_8', - - ### These have undefined mappings: - #'cp424', - - ### These fail the round-trip: - #'cp875' - - ): - self.assertEqual(str(s, encoding).encode(encoding), s) - - # 128-255 - s = bytes(range(128, 256)) - for encoding in ( - 'cp037', 'cp1026', 'cp273', - 'cp437', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850', - 'cp852', 'cp855', 'cp858', 'cp860', 'cp861', 'cp862', - 'cp863', 'cp865', 'cp866', 'cp1125', - 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15', - 'iso8859_2', 'iso8859_4', 'iso8859_5', - 'iso8859_9', 'koi8_r', 'koi8_u', 'latin_1', - 'mac_cyrillic', 'mac_latin2', - - ### These have undefined mappings: - #'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', - #'cp1256', 'cp1257', 'cp1258', - #'cp424', 'cp856', 'cp857', 'cp864', 'cp869', 'cp874', - #'iso8859_3', 'iso8859_6', 'iso8859_7', 'koi8_t', 'kz1048', - #'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish', - - ### These fail the round-trip: - #'cp1006', 'cp875', 'iso8859_8', - - ): - self.assertEqual(str(s, encoding).encode(encoding), s) - - def test_concatenation(self): - self.assertEqual(("abc" "def"), "abcdef") - self.assertEqual(("abc" "def"), "abcdef") - self.assertEqual(("abc" "def"), "abcdef") - self.assertEqual(("abc" "def" "ghi"), "abcdefghi") - self.assertEqual(("abc" "def" "ghi"), "abcdefghi") - - def test_ucs4(self): - x = '\U00100000' - y = x.encode("raw-unicode-escape").decode("raw-unicode-escape") - self.assertEqual(x, y) - - y = br'\U00100000' - x = y.decode("raw-unicode-escape").encode("raw-unicode-escape") - self.assertEqual(x, y) - y = br'\U00010000' - x = y.decode("raw-unicode-escape").encode("raw-unicode-escape") - self.assertEqual(x, y) - - try: - br'\U11111111'.decode("raw-unicode-escape") - except UnicodeDecodeError as e: - self.assertEqual(e.start, 0) - self.assertEqual(e.end, 10) - else: - self.fail("Should have raised UnicodeDecodeError") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_conversion(self): - # Make sure __str__() works properly - class ObjectToStr: - def __str__(self): - return "foo" - - class StrSubclassToStr(str): - def __str__(self): - return "foo" - - class StrSubclassToStrSubclass(str): - def __new__(cls, content=""): - return str.__new__(cls, 2*content) - def __str__(self): - return self - - self.assertEqual(str(ObjectToStr()), "foo") - self.assertEqual(str(StrSubclassToStr("bar")), "foo") - s = str(StrSubclassToStrSubclass("foo")) - self.assertEqual(s, "foofoo") - self.assertIs(type(s), StrSubclassToStrSubclass) - s = StrSubclass(StrSubclassToStrSubclass("foo")) - self.assertEqual(s, "foofoo") - self.assertIs(type(s), StrSubclass) - - def test_unicode_repr(self): - class s1: - def __repr__(self): - return '\\n' - - self.assertEqual(repr(s1()), '\\n') - - def test_printable_repr(self): - self.assertEqual(repr('\U00010000'), "'%c'" % (0x10000,)) # printable - self.assertEqual(repr('\U00014000'), "'\\U00014000'") # nonprintable - - # This test only affects 32-bit platforms because expandtabs can only take - # an int as the max value, not a 64-bit C long. If expandtabs is changed - # to take a 64-bit long, this test should apply to all platforms. - @unittest.skip("TODO: RUSTPYTHON, oom handling") - @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4, - 'only applies to 32-bit platforms') - def test_expandtabs_overflows_gracefully(self): - self.assertRaises(OverflowError, 't\tt\t'.expandtabs, sys.maxsize) - - @support.cpython_only - def test_expandtabs_optimization(self): - s = 'abc' - self.assertIs(s.expandtabs(), s) - - @unittest.skip("TODO: RUSTPYTHON, aborted: memory allocation of 9223372036854775759 bytes failed") - def test_raiseMemError(self): - asciifields = "nnb" - compactfields = asciifields + "nP" - ascii_struct_size = support.calcobjsize(asciifields) - compact_struct_size = support.calcobjsize(compactfields) - - for char in ('a', '\xe9', '\u20ac', '\U0010ffff'): - code = ord(char) - if code < 0x80: - char_size = 1 # sizeof(Py_UCS1) - struct_size = ascii_struct_size - elif code < 0x100: - char_size = 1 # sizeof(Py_UCS1) - struct_size = compact_struct_size - elif code < 0x10000: - char_size = 2 # sizeof(Py_UCS2) - struct_size = compact_struct_size - else: - char_size = 4 # sizeof(Py_UCS4) - struct_size = compact_struct_size - # Note: sys.maxsize is half of the actual max allocation because of - # the signedness of Py_ssize_t. Strings of maxlen-1 should in principle - # be allocatable, given enough memory. - maxlen = ((sys.maxsize - struct_size) // char_size) - alloc = lambda: char * maxlen - with self.subTest( - char=char, - struct_size=struct_size, - char_size=char_size - ): - # self-check - self.assertEqual( - sys.getsizeof(char * 42), - struct_size + (char_size * (42 + 1)) - ) - self.assertRaises(MemoryError, alloc) - self.assertRaises(MemoryError, alloc) - - def test_format_subclass(self): - class S(str): - def __str__(self): - return '__str__ overridden' - s = S('xxx') - self.assertEqual("%s" % s, '__str__ overridden') - self.assertEqual("{}".format(s), '__str__ overridden') - - def test_subclass_add(self): - class S(str): - def __add__(self, o): - return "3" - self.assertEqual(S("4") + S("5"), "3") - class S(str): - def __iadd__(self, o): - return "3" - s = S("1") - s += "4" - self.assertEqual(s, "3") - - def test_getnewargs(self): - text = 'abc' - args = text.__getnewargs__() - self.assertIsNot(args[0], text) - self.assertEqual(args[0], text) - self.assertEqual(len(args), 1) - - @support.cpython_only - @support.requires_legacy_unicode_capi() - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_resize(self): - for length in range(1, 100, 7): - # generate a fresh string (refcount=1) - text = 'a' * length + 'b' - - # fill wstr internal field - with self.assertWarns(DeprecationWarning): - abc = _testcapi.getargs_u(text) - self.assertEqual(abc, text) - - # resize text: wstr field must be cleared and then recomputed - text += 'c' - with self.assertWarns(DeprecationWarning): - abcdef = _testcapi.getargs_u(text) - self.assertNotEqual(abc, abcdef) - self.assertEqual(abcdef, text) - - def test_compare(self): - # Issue #17615 - N = 10 - ascii = 'a' * N - ascii2 = 'z' * N - latin = '\x80' * N - latin2 = '\xff' * N - bmp = '\u0100' * N - bmp2 = '\uffff' * N - astral = '\U00100000' * N - astral2 = '\U0010ffff' * N - strings = ( - ascii, ascii2, - latin, latin2, - bmp, bmp2, - astral, astral2) - for text1, text2 in itertools.combinations(strings, 2): - equal = (text1 is text2) - self.assertEqual(text1 == text2, equal) - self.assertEqual(text1 != text2, not equal) - - if equal: - self.assertTrue(text1 <= text2) - self.assertTrue(text1 >= text2) - - # text1 is text2: duplicate strings to skip the "str1 == str2" - # optimization in unicode_compare_eq() and really compare - # character per character - copy1 = duplicate_string(text1) - copy2 = duplicate_string(text2) - self.assertIsNot(copy1, copy2) - - self.assertTrue(copy1 == copy2) - self.assertFalse(copy1 != copy2) - - self.assertTrue(copy1 <= copy2) - self.assertTrue(copy2 >= copy2) - - self.assertTrue(ascii < ascii2) - self.assertTrue(ascii < latin) - self.assertTrue(ascii < bmp) - self.assertTrue(ascii < astral) - self.assertFalse(ascii >= ascii2) - self.assertFalse(ascii >= latin) - self.assertFalse(ascii >= bmp) - self.assertFalse(ascii >= astral) - - self.assertFalse(latin < ascii) - self.assertTrue(latin < latin2) - self.assertTrue(latin < bmp) - self.assertTrue(latin < astral) - self.assertTrue(latin >= ascii) - self.assertFalse(latin >= latin2) - self.assertFalse(latin >= bmp) - self.assertFalse(latin >= astral) - - self.assertFalse(bmp < ascii) - self.assertFalse(bmp < latin) - self.assertTrue(bmp < bmp2) - self.assertTrue(bmp < astral) - self.assertTrue(bmp >= ascii) - self.assertTrue(bmp >= latin) - self.assertFalse(bmp >= bmp2) - self.assertFalse(bmp >= astral) - - self.assertFalse(astral < ascii) - self.assertFalse(astral < latin) - self.assertFalse(astral < bmp2) - self.assertTrue(astral < astral2) - self.assertTrue(astral >= ascii) - self.assertTrue(astral >= latin) - self.assertTrue(astral >= bmp2) - self.assertFalse(astral >= astral2) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_free_after_iterating(self): - support.check_free_after_iterating(self, iter, str) - support.check_free_after_iterating(self, reversed, str) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_check_encoding_errors(self): - # bpo-37388: str(bytes) and str.decode() must check encoding and errors - # arguments in dev mode - encodings = ('ascii', 'utf8', 'latin1') - invalid = 'Boom, Shaka Laka, Boom!' - code = textwrap.dedent(f''' - import sys - encodings = {encodings!r} - - for data in (b'', b'short string'): - try: - str(data, encoding={invalid!r}) - except LookupError: - pass - else: - sys.exit(21) - - try: - str(data, errors={invalid!r}) - except LookupError: - pass - else: - sys.exit(22) - - for encoding in encodings: - try: - str(data, encoding, errors={invalid!r}) - except LookupError: - pass - else: - sys.exit(22) - - for data in ('', 'short string'): - try: - data.encode(encoding={invalid!r}) - except LookupError: - pass - else: - sys.exit(23) - - try: - data.encode(errors={invalid!r}) - except LookupError: - pass - else: - sys.exit(24) - - for encoding in encodings: - try: - data.encode(encoding, errors={invalid!r}) - except LookupError: - pass - else: - sys.exit(24) - - sys.exit(10) - ''') - proc = assert_python_failure('-X', 'dev', '-c', code) - self.assertEqual(proc.rc, 10, proc) - - -class StringModuleTest(unittest.TestCase): - def test_formatter_parser(self): - def parse(format): - return list(_string.formatter_parser(format)) - - formatter = parse("prefix {2!s}xxx{0:^+10.3f}{obj.attr!s} {z[0]!s:10}") - self.assertEqual(formatter, [ - ('prefix ', '2', '', 's'), - ('xxx', '0', '^+10.3f', None), - ('', 'obj.attr', '', 's'), - (' ', 'z[0]', '10', 's'), - ]) - - formatter = parse("prefix {} suffix") - self.assertEqual(formatter, [ - ('prefix ', '', '', None), - (' suffix', None, None, None), - ]) - - formatter = parse("str") - self.assertEqual(formatter, [ - ('str', None, None, None), - ]) - - formatter = parse("") - self.assertEqual(formatter, []) - - formatter = parse("{0}") - self.assertEqual(formatter, [ - ('', '0', '', None), - ]) - - self.assertRaises(TypeError, _string.formatter_parser, 1) - - def test_formatter_field_name_split(self): - def split(name): - items = list(_string.formatter_field_name_split(name)) - items[1] = list(items[1]) - return items - self.assertEqual(split("obj"), ["obj", []]) - self.assertEqual(split("obj.arg"), ["obj", [(True, 'arg')]]) - self.assertEqual(split("obj[key]"), ["obj", [(False, 'key')]]) - self.assertEqual(split("obj.arg[key1][key2]"), [ - "obj", - [(True, 'arg'), - (False, 'key1'), - (False, 'key2'), - ]]) - self.assertRaises(TypeError, _string.formatter_field_name_split, 1) - - def test_str_subclass_attr(self): - - name = StrSubclass("name") - name2 = StrSubclass("name2") - class Bag: - pass - - o = Bag() - with self.assertRaises(AttributeError): - delattr(o, name) - setattr(o, name, 1) - self.assertEqual(o.name, 1) - o.name = 2 - self.assertEqual(list(o.__dict__), [name]) - - with self.assertRaises(AttributeError): - delattr(o, name2) - with self.assertRaises(AttributeError): - del o.name2 - setattr(o, name2, 3) - self.assertEqual(o.name2, 3) - o.name2 = 4 - self.assertEqual(list(o.__dict__), [name, name2]) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_unicode_file.py b/Lib/test/test_unicode_file.py index 80c22c6cdd1..fe25bfe9f88 100644 --- a/Lib/test/test_unicode_file.py +++ b/Lib/test/test_unicode_file.py @@ -110,7 +110,7 @@ def _test_single(self, filename): os.unlink(filename) self.assertTrue(not os.path.exists(filename)) # and again with os.open. - f = os.open(filename, os.O_CREAT) + f = os.open(filename, os.O_CREAT | os.O_WRONLY) os.close(f) try: self._do_single(filename) diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 40955287883..25c16e3a0b7 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -5,7 +5,7 @@ import unittest import warnings from unicodedata import normalize -from test.support import os_helper +from test.support import is_apple, os_helper from test import support @@ -23,13 +23,13 @@ '10_\u1fee\u1ffd', ] -# Mac OS X decomposes Unicode names, using Normal Form D. +# Apple platforms decompose Unicode names, using Normal Form D. # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html # "However, most volume formats do not follow the exact specification for # these normal forms. For example, HFS Plus uses a variant of Normal Form D # in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through # U+2FAFF are not decomposed." -if sys.platform != 'darwin': +if not is_apple: filenames.extend([ # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all different '11_\u0385\u03d3\u03d4', @@ -94,8 +94,6 @@ def _apply_failure(self, fn, filename, "with bad filename in the exception: %a" % (fn.__name__, filename, exc_filename)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_failures(self): # Pass non-existing Unicode filenames all over the place. for name in self.files: @@ -113,8 +111,6 @@ def test_failures(self): else: _listdir_failure = NotADirectoryError - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_open(self): for name in self.files: f = open(name, 'wb') @@ -123,13 +119,11 @@ def test_open(self): os.stat(name) self._apply_failure(os.listdir, name, self._listdir_failure) - # Skip the test on darwin, because darwin does normalize the filename to + # Skip the test on Apple platforms, because they don't normalize the filename to # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( support.is_emscripten or support.is_wasi, "test fails on Emscripten/WASI when host platform is macOS." @@ -148,10 +142,10 @@ def test_normalize(self): self._apply_failure(os.remove, name) self._apply_failure(os.listdir, name) - # Skip the test on darwin, because darwin uses a normalization different + # Skip the test on Apple platforms, because they use a normalization different # than Python NFD normalization: filenames are different even if we use # Python NFD normalization. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') def test_listdir(self): sf0 = set(self.files) with warnings.catch_warnings(): diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index 9e611caf61e..60cfdaabe82 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -2,8 +2,6 @@ class PEP3131Test(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_valid(self): class T: ä = 1 @@ -15,8 +13,6 @@ class T: self.assertEqual(getattr(T, '\u87d2'), 3) self.assertEqual(getattr(T, 'x\U000E0100'), 4) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_non_bmp_normalized(self): 𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1 self.assertIn("Unicode", dir()) @@ -25,7 +21,7 @@ def test_non_bmp_normalized(self): @unittest.expectedFailure def test_invalid(self): try: - from test import badsyntax_3131 + from test.tokenizedata import badsyntax_3131 except SyntaxError as err: self.assertEqual(str(err), "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)") diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index f5b4b6218ef..ceae20e8cb2 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -11,18 +11,22 @@ import sys import unicodedata import unittest -from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation, - ResourceDenied) +from test.support import ( + open_urlresource, + requires_resource, + script_helper, + cpython_only, + check_disallow_instantiation, + force_not_colorized, +) class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = '4739770dd4d0e5f1b1677accfc3552ed3c8ef326' + expectedchecksum = '9e43ee3929471739680c0e705482b4ae1c4122e4' - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + 9e43ee3929471739680c0e705482b4ae1c4122e4 @requires_resource('cpu') def test_method_checksum(self): h = hashlib.sha1() @@ -74,9 +78,9 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = '98d602e1f69d5c5bb8a5910c40bbbad4e18e8370' - # TODO: RUSTPYTHON - @unittest.expectedFailure + expectedchecksum = '23ab09ed4abdf93db23b97359108ed630dd8311d' + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'digit' @requires_resource('cpu') def test_function_checksum(self): data = [] @@ -94,13 +98,13 @@ def test_function_checksum(self): self.db.decomposition(char), str(self.db.mirrored(char)), str(self.db.combining(char)), + unicodedata.east_asian_width(char), + self.db.name(char, ""), ] h.update(''.join(data).encode("ascii")) result = h.hexdigest() self.assertEqual(result, self.expectedchecksum) - # TODO: RUSTPYTHON - @unittest.expectedFailure @requires_resource('cpu') def test_name_inverse_lookup(self): for i in range(sys.maxunicode + 1): @@ -108,8 +112,26 @@ def test_name_inverse_lookup(self): if looked_name := self.db.name(char, None): self.assertEqual(self.db.lookup(looked_name), char) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_no_names_in_pua(self): + puas = [*range(0xe000, 0xf8ff), + *range(0xf0000, 0xfffff), + *range(0x100000, 0x10ffff)] + for i in puas: + char = chr(i) + self.assertRaises(ValueError, self.db.name, char) + + def test_lookup_nonexistant(self): + # just make sure that lookup can fail + for nonexistent in [ + "LATIN SMLL LETR A", + "OPEN HANDS SIGHS", + "DREGS", + "HANDBUG", + "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", + "???", + ]: + self.assertRaises(KeyError, self.db.lookup, nonexistent) + def test_digit(self): self.assertEqual(self.db.digit('A', None), None) self.assertEqual(self.db.digit('9'), 9) @@ -122,8 +144,6 @@ def test_digit(self): self.assertRaises(TypeError, self.db.digit, 'xx') self.assertRaises(ValueError, self.db.digit, 'x') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_numeric(self): self.assertEqual(self.db.numeric('A',None), None) self.assertEqual(self.db.numeric('9'), 9) @@ -137,8 +157,6 @@ def test_numeric(self): self.assertRaises(TypeError, self.db.numeric, 'xx') self.assertRaises(ValueError, self.db.numeric, 'x') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decimal(self): self.assertEqual(self.db.decimal('A',None), None) self.assertEqual(self.db.decimal('9'), 9) @@ -161,8 +179,7 @@ def test_category(self): self.assertRaises(TypeError, self.db.category) self.assertRaises(TypeError, self.db.category, 'xx') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - L def test_bidirectional(self): self.assertEqual(self.db.bidirectional('\uFFFE'), '') self.assertEqual(self.db.bidirectional(' '), 'WS') @@ -172,8 +189,6 @@ def test_bidirectional(self): self.assertRaises(TypeError, self.db.bidirectional) self.assertRaises(TypeError, self.db.bidirectional, 'xx') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decomposition(self): self.assertEqual(self.db.decomposition('\uFFFE'),'') self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034') @@ -181,8 +196,6 @@ def test_decomposition(self): self.assertRaises(TypeError, self.db.decomposition) self.assertRaises(TypeError, self.db.decomposition, 'xx') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mirrored(self): self.assertEqual(self.db.mirrored('\uFFFE'), 0) self.assertEqual(self.db.mirrored('a'), 0) @@ -192,8 +205,6 @@ def test_mirrored(self): self.assertRaises(TypeError, self.db.mirrored) self.assertRaises(TypeError, self.db.mirrored, 'xx') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_combining(self): self.assertEqual(self.db.combining('\uFFFE'), 0) self.assertEqual(self.db.combining('a'), 0) @@ -221,8 +232,7 @@ def test_issue10254(self): b = 'C\u0338' * 20 + '\xC7' self.assertEqual(self.db.normalize('NFC', a), b) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ? + def test_issue29456(self): # Fix #29456 u1176_str_a = '\u1100\u1176\u11a8' @@ -249,8 +259,25 @@ def test_east_asian_width(self): self.assertEqual(eaw('\u2010'), 'A') self.assertEqual(eaw('\U00020000'), 'W') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + W + def test_east_asian_width_unassigned(self): + eaw = self.db.east_asian_width + # unassigned + for char in '\u0530\u0ecf\u10c6\u20fc\uaaca\U000107bd\U000115f2': + self.assertEqual(eaw(char), 'N') + self.assertIs(self.db.name(char, None), None) + + # unassigned but reserved for CJK + for char in '\uFA6E\uFADA\U0002A6E0\U0002FA20\U0003134B\U0003FFFD': + self.assertEqual(eaw(char), 'W') + self.assertIs(self.db.name(char, None), None) + + # private use areas + for char in '\uE000\uF800\U000F0000\U000FFFEE\U00100000\U0010FFF0': + self.assertEqual(eaw(char), 'A') + self.assertIs(self.db.name(char, None), None) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + N def test_east_asian_width_9_0_changes(self): self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N') self.assertEqual(self.db.east_asian_width('\u231a'), 'W') @@ -262,8 +289,8 @@ def test_disallow_instantiation(self): # Ensure that the type disallows instantiation (bpo-43916) check_disallow_instantiation(self, unicodedata.UCD) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; --- + @force_not_colorized def test_failed_import_during_compiling(self): # Issue 4367 # Decoding \N escapes requires the unicodedata module. If it can't be @@ -280,8 +307,6 @@ def test_failed_import_during_compiling(self): "(can't load unicodedata module)" self.assertIn(error, result.err.decode("ascii")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decimal_numeric_consistent(self): # Test that decimal and numeric are consistent, # i.e. if a character has a decimal value, @@ -295,8 +320,6 @@ def test_decimal_numeric_consistent(self): count += 1 self.assertTrue(count >= 10) # should have tested at least the ASCII digits - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_digit_numeric_consistent(self): # Test that digit and numeric are consistent, # i.e. if a character has a digit value, @@ -313,8 +336,7 @@ def test_digit_numeric_consistent(self): def test_bug_1704793(self): self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: False is not true def test_ucd_510(self): import unicodedata # In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0 @@ -326,8 +348,7 @@ def test_ucd_510(self): self.assertTrue("\u1d79".upper()=='\ua77d') self.assertTrue(".".upper()=='.') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @requires_resource('cpu') def test_bug_5828(self): self.assertEqual("\u1d79".lower(), "\u1d79") # Only U+0000 should have U+0000 as its upper/lower/titlecase variant @@ -339,16 +360,13 @@ def test_bug_5828(self): [0] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; + Dž def test_bug_4971(self): # LETTER DZ WITH CARON: DZ, Dz, dz self.assertEqual("\u01c4".title(), "\u01c5") self.assertEqual("\u01c5".title(), "\u01c5") self.assertEqual("\u01c6".title(), "\u01c5") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_linebreak_7643(self): for i in range(0x10000): lines = (chr(i) + 'A').splitlines() @@ -372,6 +390,7 @@ def unistr(data): return "".join([chr(x) for x in data]) @requires_resource('network') + @requires_resource('cpu') def test_normalization(self): TESTDATAFILE = "NormalizationTest.txt" TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}" @@ -457,6 +476,29 @@ def test_bug_834676(self): # Check for bug 834676 unicodedata.normalize('NFC', '\ud55c\uae00') + def test_normalize_return_type(self): + # gh-129569: normalize() return type must always be str + normalize = unicodedata.normalize + + class MyStr(str): + pass + + normalization_forms = ("NFC", "NFKC", "NFD", "NFKD") + input_strings = ( + # normalized strings + "", + "ascii", + # unnormalized strings + "\u1e0b\u0323", + "\u0071\u0307\u0323", + ) + + for form in normalization_forms: + for input_str in input_strings: + with self.subTest(form=form, input_str=input_str): + self.assertIs(type(normalize(form, input_str)), str) + self.assertIs(type(normalize(form, MyStr(input_str))), str) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_unittest.py b/Lib/test/test_unittest.py deleted file mode 100644 index 1079c7df2e5..00000000000 --- a/Lib/test/test_unittest.py +++ /dev/null @@ -1,16 +0,0 @@ -import unittest.test - -from test import support - - -def load_tests(*_): - # used by unittest - return unittest.test.suite() - - -def tearDownModule(): - support.reap_children() - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_unittest/__init__.py b/Lib/test/test_unittest/__init__.py new file mode 100644 index 00000000000..365f26d6438 --- /dev/null +++ b/Lib/test/test_unittest/__init__.py @@ -0,0 +1,7 @@ +import os.path + +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_unittest/__main__.py b/Lib/test/test_unittest/__main__.py new file mode 100644 index 00000000000..0d53bfab847 --- /dev/null +++ b/Lib/test/test_unittest/__main__.py @@ -0,0 +1,5 @@ +import unittest + +from . import load_tests + +unittest.main() diff --git a/Lib/unittest/test/_test_warnings.py b/Lib/test/test_unittest/_test_warnings.py similarity index 84% rename from Lib/unittest/test/_test_warnings.py rename to Lib/test/test_unittest/_test_warnings.py index 5cbfb532ad0..d9f41a4144b 100644 --- a/Lib/unittest/test/_test_warnings.py +++ b/Lib/test/test_unittest/_test_warnings.py @@ -14,21 +14,11 @@ import unittest import warnings + def warnfun(): warnings.warn('rw', RuntimeWarning) class TestWarnings(unittest.TestCase): - # unittest warnings will be printed at most once per type (max one message - # for the fail* methods, and one for the assert* methods) - def test_assert(self): - self.assertEquals(2+2, 4) - self.assertEquals(2*2, 4) - self.assertEquals(2**2, 4) - - def test_fail(self): - self.failUnless(1) - self.failUnless(True) - def test_other_unittest(self): self.assertAlmostEqual(2+2, 4) self.assertNotAlmostEqual(4+4, 2) diff --git a/Lib/unittest/test/dummy.py b/Lib/test/test_unittest/dummy.py similarity index 100% rename from Lib/unittest/test/dummy.py rename to Lib/test/test_unittest/dummy.py diff --git a/Lib/test/test_unittest/support.py b/Lib/test/test_unittest/support.py new file mode 100644 index 00000000000..8c97bf5c729 --- /dev/null +++ b/Lib/test/test_unittest/support.py @@ -0,0 +1,154 @@ +import unittest + + +class TestEquality(object): + """Used as a mixin for TestCase""" + + # Check for a valid __eq__ implementation + def test_eq(self): + for obj_1, obj_2 in self.eq_pairs: + self.assertEqual(obj_1, obj_2) + self.assertEqual(obj_2, obj_1) + + # Check for a valid __ne__ implementation + def test_ne(self): + for obj_1, obj_2 in self.ne_pairs: + self.assertNotEqual(obj_1, obj_2) + self.assertNotEqual(obj_2, obj_1) + +class TestHashing(object): + """Used as a mixin for TestCase""" + + # Check for a valid __hash__ implementation + def test_hash(self): + for obj_1, obj_2 in self.eq_pairs: + try: + if not hash(obj_1) == hash(obj_2): + self.fail("%r and %r do not hash equal" % (obj_1, obj_2)) + except Exception as e: + self.fail("Problem hashing %r and %r: %s" % (obj_1, obj_2, e)) + + for obj_1, obj_2 in self.ne_pairs: + try: + if hash(obj_1) == hash(obj_2): + self.fail("%s and %s hash equal, but shouldn't" % + (obj_1, obj_2)) + except Exception as e: + self.fail("Problem hashing %s and %s: %s" % (obj_1, obj_2, e)) + + +class _BaseLoggingResult(unittest.TestResult): + def __init__(self, log): + self._events = log + super().__init__() + + def startTest(self, test): + self._events.append('startTest') + super().startTest(test) + + def startTestRun(self): + self._events.append('startTestRun') + super().startTestRun() + + def stopTest(self, test): + self._events.append('stopTest') + super().stopTest(test) + + def stopTestRun(self): + self._events.append('stopTestRun') + super().stopTestRun() + + def addFailure(self, *args): + self._events.append('addFailure') + super().addFailure(*args) + + def addSuccess(self, *args): + self._events.append('addSuccess') + super().addSuccess(*args) + + def addError(self, *args): + self._events.append('addError') + super().addError(*args) + + def addSkip(self, *args): + self._events.append('addSkip') + super().addSkip(*args) + + def addExpectedFailure(self, *args): + self._events.append('addExpectedFailure') + super().addExpectedFailure(*args) + + def addUnexpectedSuccess(self, *args): + self._events.append('addUnexpectedSuccess') + super().addUnexpectedSuccess(*args) + + +class LegacyLoggingResult(_BaseLoggingResult): + """ + A legacy TestResult implementation, without an addSubTest method, + which records its method calls. + """ + + @property + def addSubTest(self): + raise AttributeError + + +class LoggingResult(_BaseLoggingResult): + """ + A TestResult implementation which records its method calls. + """ + + def addSubTest(self, test, subtest, err): + if err is None: + self._events.append('addSubTestSuccess') + else: + self._events.append('addSubTestFailure') + super().addSubTest(test, subtest, err) + + +class ResultWithNoStartTestRunStopTestRun(object): + """An object honouring TestResult before startTestRun/stopTestRun.""" + + def __init__(self): + self.failures = [] + self.errors = [] + self.testsRun = 0 + self.skipped = [] + self.expectedFailures = [] + self.unexpectedSuccesses = [] + self.shouldStop = False + + def startTest(self, test): + pass + + def stopTest(self, test): + pass + + def addError(self, test): + pass + + def addFailure(self, test): + pass + + def addSuccess(self, test): + pass + + def wasSuccessful(self): + return True + + +class BufferedWriter: + def __init__(self): + self.result = '' + self.buffer = '' + + def write(self, arg): + self.buffer += arg + + def flush(self): + self.result += self.buffer + self.buffer = '' + + def getvalue(self): + return self.result diff --git a/Lib/unittest/test/test_assertions.py b/Lib/test/test_unittest/test_assertions.py similarity index 96% rename from Lib/unittest/test/test_assertions.py rename to Lib/test/test_unittest/test_assertions.py index a0db3423b86..3d782573d7b 100644 --- a/Lib/unittest/test/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -1,10 +1,11 @@ import datetime +import unittest import warnings import weakref -import unittest -from test.support import gc_collect from itertools import product +from test.support import gc_collect + class Test_Assertions(unittest.TestCase): def test_AlmostEqual(self): @@ -271,20 +272,11 @@ def testAssertDictEqual(self): r"\+ \{'key': 'value'\}$", r"\+ \{'key': 'value'\} : oops$"]) - def testAssertDictContainsSubset(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - - self.assertMessages('assertDictContainsSubset', ({'key': 'value'}, {}), - ["^Missing: 'key'$", "^oops$", - "^Missing: 'key'$", - "^Missing: 'key' : oops$"]) - def testAssertMultiLineEqual(self): self.assertMessages('assertMultiLineEqual', ("", "foo"), - [r"\+ foo$", "^oops$", - r"\+ foo$", - r"\+ foo : oops$"]) + [r"\+ foo\n$", "^oops$", + r"\+ foo\n$", + r"\+ foo\n : oops$"]) def testAssertLess(self): self.assertMessages('assertLess', (2, 1), @@ -395,6 +387,16 @@ def testAssertWarns(self): '^UserWarning not triggered$', '^UserWarning not triggered : oops$']) + def test_assertNotWarns(self): + def warn_future(): + warnings.warn('xyz', FutureWarning, stacklevel=2) + self.assertMessagesCM('_assertNotWarns', (FutureWarning,), + warn_future, + ['^FutureWarning triggered$', + '^oops$', + '^FutureWarning triggered$', + '^FutureWarning triggered : oops$']) + def testAssertWarnsRegex(self): # test error not raised self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'unused regex'), diff --git a/Lib/unittest/test/test_async_case.py b/Lib/test/test_unittest/test_async_case.py similarity index 94% rename from Lib/unittest/test/test_async_case.py rename to Lib/test/test_unittest/test_async_case.py index 98ec4f33453..57228e78f8c 100644 --- a/Lib/unittest/test/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -1,7 +1,9 @@ import asyncio import contextvars import unittest + from test import support +from test.support import force_not_colorized support.requires_working_socket(module=True) @@ -11,7 +13,7 @@ class MyException(Exception): def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCM: @@ -37,11 +39,9 @@ async def __aenter__(self): pass -# TODO: RUSTPYTHON; used by following test suite -# VAR = contextvars.ContextVar('VAR', default=()) +VAR = contextvars.ContextVar('VAR', default=()) -@unittest.skip("TODO: RUSTPYTHON; requires sys.get_coroutine_origin_tracking_depth()") class TestAsyncCase(unittest.TestCase): maxDiff = None @@ -254,6 +254,7 @@ async def on_cleanup(self): test.doCleanups() self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup']) + @force_not_colorized def test_exception_in_tear_clean_up(self): class Test(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): @@ -296,6 +297,7 @@ async def on_cleanup2(self): test.doCleanups() self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup2', 'cleanup1']) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_deprecation_of_return_val_from_test(self): # Issue 41322 - deprecate return of value that is not None from a test class Nothing: @@ -314,18 +316,21 @@ async def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'async_generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Test('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) def test_cleanups_interleave_order(self): events = [] @@ -477,7 +482,20 @@ def test_setup_get_event_loop(self): class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): - asyncio.get_event_loop_policy().get_event_loop() + asyncio.events._get_event_loop_policy().get_event_loop() + + async def test_demo1(self): + pass + + test = TestCase1('test_demo1') + result = test.run() + self.assertTrue(result.wasSuccessful()) + + def test_loop_factory(self): + asyncio.events._set_event_loop_policy(None) + + class TestCase1(unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop async def test_demo1(self): pass @@ -485,6 +503,7 @@ async def test_demo1(self): test = TestCase1('test_demo1') result = test.run() self.assertTrue(result.wasSuccessful()) + self.assertIsNone(support.maybe_get_event_loop_policy()) if __name__ == "__main__": unittest.main() diff --git a/Lib/unittest/test/test_break.py b/Lib/test/test_unittest/test_break.py similarity index 98% rename from Lib/unittest/test/test_break.py rename to Lib/test/test_unittest/test_break.py index 33cbdd2661c..8aa20008ac7 100644 --- a/Lib/unittest/test/test_break.py +++ b/Lib/test/test_unittest/test_break.py @@ -1,10 +1,10 @@ import gc import io import os -import sys import signal -import weakref +import sys import unittest +import weakref from test import support @@ -236,6 +236,7 @@ def __init__(self, catchbreak): self.testRunner = FakeRunner self.test = test self.result = None + self.durations = None p = Program(False) p.runTests() @@ -244,7 +245,8 @@ def __init__(self, catchbreak): 'verbosity': verbosity, 'failfast': failfast, 'tb_locals': False, - 'warnings': None})]) + 'warnings': None, + 'durations': None})]) self.assertEqual(FakeRunner.runArgs, [test]) self.assertEqual(p.result, result) @@ -259,7 +261,8 @@ def __init__(self, catchbreak): 'verbosity': verbosity, 'failfast': failfast, 'tb_locals': False, - 'warnings': None})]) + 'warnings': None, + 'durations': None})]) self.assertEqual(FakeRunner.runArgs, [test]) self.assertEqual(p.result, result) diff --git a/Lib/unittest/test/test_case.py b/Lib/test/test_unittest/test_case.py similarity index 78% rename from Lib/unittest/test/test_case.py rename to Lib/test/test_unittest/test_case.py index e2547f4777a..6e77040c265 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1,26 +1,27 @@ import contextlib import difflib -import pprint +import inspect +import logging import pickle +import pprint import re import sys -import logging +import types +import unittest import warnings import weakref -import inspect -import types - +from collections import UserString from copy import deepcopy -from test import support -import unittest - -from unittest.test.support import ( - TestEquality, TestHashing, LoggingResult, LegacyLoggingResult, - ResultWithNoStartTestRunStopTestRun -) +from test import support from test.support import captured_stderr, gc_collect - +from test.test_unittest.support import ( + LegacyLoggingResult, + LoggingResult, + ResultWithNoStartTestRunStopTestRun, + TestEquality, + TestHashing, +) log_foo = logging.getLogger('foo') log_foobar = logging.getLogger('foo.bar') @@ -54,6 +55,10 @@ def tearDown(self): self.events.append('tearDown') +class List(list): + pass + + class Test_TestCase(unittest.TestCase, TestEquality, TestHashing): ### Set up attributes used by inherited tests @@ -85,7 +90,7 @@ class Test(unittest.TestCase): def runTest(self): raise MyException() def test(self): pass - self.assertEqual(Test().id()[-13:], '.Test.runTest') + self.assertEndsWith(Test().id(), '.Test.runTest') # test that TestCase can be instantiated with no args # primarily for use at the interactive interpreter @@ -106,7 +111,7 @@ class Test(unittest.TestCase): def runTest(self): raise MyException() def test(self): pass - self.assertEqual(Test('test').id()[-10:], '.Test.test') + self.assertEndsWith(Test('test').id(), '.Test.test') # "class TestCase([methodName])" # ... @@ -304,7 +309,8 @@ def defaultTestResult(self): def test(self): pass - Foo('test').run() + with self.assertWarns(RuntimeWarning): + Foo('test').run() def test_deprecation_of_return_val_from_test(self): # Issue 41322 - deprecate return of value that is not None from a test @@ -324,18 +330,40 @@ def test3(self): self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test1', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'int'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test2').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test2', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn("returned 'generator'", str(w.warning)) with self.assertWarns(DeprecationWarning) as w: Foo('test3').run() self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) self.assertIn('test3', str(w.warning)) self.assertEqual(w.filename, __file__) + self.assertIn(f'returned {Nothing.__name__!r}', str(w.warning)) + + def test_deprecation_of_return_val_from_test_async_method(self): + class Foo(unittest.TestCase): + async def test1(self): + return 1 + + with self.assertWarns(DeprecationWarning) as w: + warnings.filterwarnings('ignore', + 'coroutine .* was never awaited', RuntimeWarning) + Foo('test1').run() + support.gc_collect() + self.assertIn('It is deprecated to return a value that is not None', str(w.warning)) + self.assertIn('test1', str(w.warning)) + self.assertEqual(w.filename, __file__) + self.assertIn("returned 'coroutine'", str(w.warning)) + self.assertIn( + 'Maybe you forgot to use IsolatedAsyncioTestCase as the base class?', + str(w.warning), + ) def _check_call_order__subtests(self, result, events, expected_events): class Foo(Test.LoggingTestCase): @@ -677,16 +705,136 @@ def testAssertIsNot(self): self.assertRaises(self.failureException, self.assertIsNot, thing, thing) def testAssertIsInstance(self): - thing = [] + thing = List() self.assertIsInstance(thing, list) - self.assertRaises(self.failureException, self.assertIsInstance, - thing, dict) + self.assertIsInstance(thing, (int, list)) + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int) + self.assertEqual(str(cm.exception), + "[] is not an instance of <class 'int'>") + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, (int, float)) + self.assertEqual(str(cm.exception), + "[] is not an instance of any of (<class 'int'>, <class 'float'>)") + + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertIsInstance(thing, int, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) def testAssertNotIsInstance(self): - thing = [] - self.assertNotIsInstance(thing, dict) - self.assertRaises(self.failureException, self.assertNotIsInstance, - thing, list) + thing = List() + self.assertNotIsInstance(thing, int) + self.assertNotIsInstance(thing, (int, float)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list) + self.assertEqual(str(cm.exception), + "[] is an instance of <class 'list'>") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, (int, list)) + self.assertEqual(str(cm.exception), + "[] is an instance of <class 'list'>") + + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsInstance(thing, list, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertIsSubclass(self): + self.assertIsSubclass(List, list) + self.assertIsSubclass(List, (int, list)) + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int) + self.assertEqual(str(cm.exception), + f"{List!r} is not a subclass of <class 'int'>") + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, (int, float)) + self.assertEqual(str(cm.exception), + f"{List!r} is not a subclass of any of (<class 'int'>, <class 'float'>)") + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(1, int) + self.assertEqual(str(cm.exception), "1 is not a class") + + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertIsSubclass(List, int, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotIsSubclass(self): + self.assertNotIsSubclass(List, int) + self.assertNotIsSubclass(List, (int, float)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list) + self.assertEqual(str(cm.exception), + f"{List!r} is a subclass of <class 'list'>") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, (int, list)) + self.assertEqual(str(cm.exception), + f"{List!r} is a subclass of <class 'list'>") + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(1, int) + self.assertEqual(str(cm.exception), "1 is not a class") + + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list, 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotIsSubclass(List, list, msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertHasAttr(self): + a = List() + a.x = 1 + self.assertHasAttr(a, 'x') + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y') + self.assertEqual(str(cm.exception), + "'List' object has no attribute 'y'") + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(List, 'spam') + self.assertEqual(str(cm.exception), + "type object 'List' has no attribute 'spam'") + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(sys, 'nonexistent') + self.assertEqual(str(cm.exception), + "module 'sys' has no attribute 'nonexistent'") + + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y', 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertHasAttr(a, 'y', msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotHasAttr(self): + a = List() + a.x = 1 + self.assertNotHasAttr(a, 'y') + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x') + self.assertEqual(str(cm.exception), + "'List' object has unexpected attribute 'x'") + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(List, 'append') + self.assertEqual(str(cm.exception), + "type object 'List' has unexpected attribute 'append'") + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(sys, 'modules') + self.assertEqual(str(cm.exception), + "module 'sys' has unexpected attribute 'modules'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x', 'ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotHasAttr(a, 'x', msg='ababahalamaha') + self.assertIn('ababahalamaha', str(cm.exception)) def testAssertIn(self): animals = {'monkey': 'banana', 'cow': 'grass', 'seal': 'fish'} @@ -709,36 +857,6 @@ def testAssertIn(self): self.assertRaises(self.failureException, self.assertNotIn, 'cow', animals) - def testAssertDictContainsSubset(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - - self.assertDictContainsSubset({}, {}) - self.assertDictContainsSubset({}, {'a': 1}) - self.assertDictContainsSubset({'a': 1}, {'a': 1}) - self.assertDictContainsSubset({'a': 1}, {'a': 1, 'b': 2}) - self.assertDictContainsSubset({'a': 1, 'b': 2}, {'a': 1, 'b': 2}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({1: "one"}, {}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'a': 2}, {'a': 1}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'c': 1}, {'a': 1}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'a': 1, 'c': 1}, {'a': 1}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'a': 1, 'c': 1}, {'a': 1}) - - one = ''.join(chr(i) for i in range(255)) - # this used to cause a UnicodeDecodeError constructing the failure msg - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'foo': one}, {'foo': '\uFFFD'}) - def testAssertEqual(self): equal_pairs = [ ((), ()), @@ -1161,6 +1279,8 @@ def testAssertMultiLineEqual(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualSingleLine(self): sample_text = "laden swallows fly slowly" @@ -1177,6 +1297,74 @@ def testAssertEqualSingleLine(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') + + def testAssertEqualwithEmptyString(self): + '''Verify when there is an empty string involved, the diff output + does not treat the empty string as a single empty line. It should + instead be handled as a non-line. + ''' + sample_text = '' + revised_sample_text = 'unladen swallows fly quickly' + sample_text_error = '''\ ++ unladen swallows fly quickly +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') + + def testAssertEqualMultipleLinesMissingNewlineTerminator(self): + '''Verifying format of diff output from assertEqual involving strings + with multiple lines, but missing the terminating newline on both. + ''' + sample_text = 'laden swallows\nfly sloely' + revised_sample_text = 'laden swallows\nfly slowly' + sample_text_error = '''\ + laden swallows +- fly sloely +? ^ ++ fly slowly +? ^ +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') + + def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): + '''Verifying format of diff output from assertEqual involving strings + with multiple lines and mismatched newlines. The output should + include a - on it's own line to indicate the newline difference + between the two strings + ''' + sample_text = 'laden swallows\nfly sloely\n' + revised_sample_text = 'laden swallows\nfly slowly' + sample_text_error = '''\ + laden swallows +- fly sloely +? ^ ++ fly slowly +? ^ +-\x20 +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testEqualityBytesWarning(self): if sys.flags.bytes_warning: @@ -1801,45 +1989,198 @@ def testAssertNoLogsYieldsNone(self): pass self.assertIsNone(value) - def testDeprecatedMethodNames(self): - """ - Test that the deprecated methods raise a DeprecationWarning. See #9424. - """ - old = ( - (self.failIfEqual, (3, 5)), - (self.assertNotEquals, (3, 5)), - (self.failUnlessEqual, (3, 3)), - (self.assertEquals, (3, 3)), - (self.failUnlessAlmostEqual, (2.0, 2.0)), - (self.assertAlmostEquals, (2.0, 2.0)), - (self.failIfAlmostEqual, (3.0, 5.0)), - (self.assertNotAlmostEquals, (3.0, 5.0)), - (self.failUnless, (True,)), - (self.assert_, (True,)), - (self.failUnlessRaises, (TypeError, lambda _: 3.14 + 'spam')), - (self.failIf, (False,)), - (self.assertDictContainsSubset, (dict(a=1, b=2), dict(a=1, b=2, c=3))), - (self.assertRaisesRegexp, (KeyError, 'foo', lambda: {}['foo'])), - (self.assertRegexpMatches, ('bar', 'bar')), - ) - for meth, args in old: - with self.assertWarns(DeprecationWarning): - meth(*args) - - # disable this test for now. When the version where the fail* methods will - # be removed is decided, re-enable it and update the version - def _testDeprecatedFailMethods(self): - """Test that the deprecated fail* methods get removed in 3.x""" - if sys.version_info[:2] < (3, 3): - return + def testAssertStartsWith(self): + self.assertStartsWith('ababahalamaha', 'ababa') + self.assertStartsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertStartsWith(UserString('ababahalamaha'), 'ababa') + self.assertStartsWith(UserString('ababahalamaha'), ('x', 'ababa', 'y')) + self.assertStartsWith(bytearray(b'ababahalamaha'), b'ababa') + self.assertStartsWith(bytearray(b'ababahalamaha'), (b'x', b'ababa', b'y')) + self.assertStartsWith(b'ababahalamaha', bytearray(b'ababa')) + self.assertStartsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'ababa'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't start with 'amaha'") + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', ('x', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't start with any of ('x', 'y')") + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith(b'ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith(b'ababahalamaha', ('amaha', 'ababa')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith([], 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', b'ababa') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', (b'amaha', b'ababa')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertStartsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertStartsWith('ababahalamaha', 'amaha', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotStartsWith(self): + self.assertNotStartsWith('ababahalamaha', 'amaha') + self.assertNotStartsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertNotStartsWith(UserString('ababahalamaha'), 'amaha') + self.assertNotStartsWith(UserString('ababahalamaha'), ('x', 'amaha', 'y')) + self.assertNotStartsWith(bytearray(b'ababahalamaha'), b'amaha') + self.assertNotStartsWith(bytearray(b'ababahalamaha'), (b'x', b'amaha', b'y')) + self.assertNotStartsWith(b'ababahalamaha', bytearray(b'amaha')) + self.assertNotStartsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'amaha'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), + "'ababahalamaha' starts with 'ababa'") + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' starts with 'ababa'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith(b'ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith(b'ababahalamaha', ('amaha', 'ababa')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith([], 'ababa') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', b'ababa') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', (b'amaha', b'ababa')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertNotStartsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotStartsWith('ababahalamaha', 'ababa', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertEndsWith(self): + self.assertEndsWith('ababahalamaha', 'amaha') + self.assertEndsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertEndsWith(UserString('ababahalamaha'), 'amaha') + self.assertEndsWith(UserString('ababahalamaha'), ('x', 'amaha', 'y')) + self.assertEndsWith(bytearray(b'ababahalamaha'), b'amaha') + self.assertEndsWith(bytearray(b'ababahalamaha'), (b'x', b'amaha', b'y')) + self.assertEndsWith(b'ababahalamaha', bytearray(b'amaha')) + self.assertEndsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'amaha'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa') + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't end with 'ababa'") + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', ('x', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' doesn't end with any of ('x', 'y')") + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith(b'ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith(b'ababahalamaha', ('ababa', 'amaha')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith([], 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', b'amaha') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', (b'ababa', b'amaha')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertEndsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertEndsWith('ababahalamaha', 'ababa', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testAssertNotEndsWith(self): + self.assertNotEndsWith('ababahalamaha', 'ababa') + self.assertNotEndsWith('ababahalamaha', ('x', 'ababa', 'y')) + self.assertNotEndsWith(UserString('ababahalamaha'), 'ababa') + self.assertNotEndsWith(UserString('ababahalamaha'), ('x', 'ababa', 'y')) + self.assertNotEndsWith(bytearray(b'ababahalamaha'), b'ababa') + self.assertNotEndsWith(bytearray(b'ababahalamaha'), (b'x', b'ababa', b'y')) + self.assertNotEndsWith(b'ababahalamaha', bytearray(b'ababa')) + self.assertNotEndsWith(b'ababahalamaha', + (bytearray(b'x'), bytearray(b'ababa'), bytearray(b'y'))) + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), + "'ababahalamaha' ends with 'amaha'") + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', ('x', 'amaha', 'y')) + self.assertEqual(str(cm.exception), + "'ababahalamaha' ends with 'amaha'") + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith(b'ababahalamaha', 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith(b'ababahalamaha', ('ababa', 'amaha')) + self.assertEqual(str(cm.exception), 'Expected str, not bytes') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith([], 'amaha') + self.assertEqual(str(cm.exception), 'Expected str, not list') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', b'amaha') + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', (b'ababa', b'amaha')) + self.assertEqual(str(cm.exception), 'Expected bytes, not str') + with self.assertRaises(TypeError): + self.assertNotEndsWith('ababahalamaha', ord('a')) + + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha', 'abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + with self.assertRaises(self.failureException) as cm: + self.assertNotEndsWith('ababahalamaha', 'amaha', msg='abracadabra') + self.assertIn('ababahalamaha', str(cm.exception)) + + def testDeprecatedFailMethods(self): + """Test that the deprecated fail* methods get removed in 3.12""" deprecated_names = [ 'failIfEqual', 'failUnlessEqual', 'failUnlessAlmostEqual', 'failIfAlmostEqual', 'failUnless', 'failUnlessRaises', 'failIf', - 'assertDictContainsSubset', + 'assertNotEquals', 'assertEquals', 'assertAlmostEquals', + 'assertNotAlmostEquals', 'assert_', 'assertDictContainsSubset', + 'assertRaisesRegexp', 'assertRegexpMatches' ] for deprecated_name in deprecated_names: with self.assertRaises(AttributeError): - getattr(self, deprecated_name) # remove these in 3.x + getattr(self, deprecated_name) def testDeepcopy(self): # Issue: 5660 @@ -1956,8 +2297,6 @@ def testNoCycles(self): del case self.assertFalse(wr()) - # TODO: RUSTPYTHON; destructors - @unittest.expectedFailure def test_no_exception_leak(self): # Issue #19880: TestCase.run() should not keep a reference # to the exception diff --git a/Lib/unittest/test/test_discovery.py b/Lib/test/test_unittest/test_discovery.py similarity index 90% rename from Lib/unittest/test/test_discovery.py rename to Lib/test/test_unittest/test_discovery.py index 3b58786ec16..9ed3d04b1f8 100644 --- a/Lib/unittest/test/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -1,16 +1,17 @@ import os.path -from os.path import abspath +import pickle import re import sys import types -import pickle -from test import support -from test.support import import_helper -import test.test_importlib.util - import unittest import unittest.mock -import unittest.test +from importlib._bootstrap_external import NamespaceLoader +from os.path import abspath + +import test.test_unittest +from test import support +from test.support import import_helper +from test.test_importlib import util as test_util class TestableTestProgram(unittest.TestProgram): @@ -396,7 +397,7 @@ def restore_isdir(): self.addCleanup(restore_isdir) _find_tests_args = [] - def _find_tests(start_dir, pattern): + def _find_tests(start_dir, pattern, namespace=None): _find_tests_args.append((start_dir, pattern)) return ['tests'] loader._find_tests = _find_tests @@ -407,10 +408,34 @@ def _find_tests(start_dir, pattern): top_level_dir = os.path.abspath('/foo/bar') start_dir = os.path.abspath('/foo/bar/baz') self.assertEqual(suite, "['tests']") - self.assertEqual(loader._top_level_dir, top_level_dir) + self.assertEqual(loader._top_level_dir, os.path.abspath('/foo')) self.assertEqual(_find_tests_args, [(start_dir, 'pattern')]) self.assertIn(top_level_dir, sys.path) + def test_discover_should_not_persist_top_level_dir_between_calls(self): + original_isfile = os.path.isfile + original_isdir = os.path.isdir + original_sys_path = sys.path[:] + def restore(): + os.path.isfile = original_isfile + os.path.isdir = original_isdir + sys.path[:] = original_sys_path + self.addCleanup(restore) + + os.path.isfile = lambda path: True + os.path.isdir = lambda path: True + loader = unittest.TestLoader() + loader.suiteClass = str + dir = '/foo/bar' + top_level_dir = '/foo' + + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, None) + + loader._top_level_dir = dir2 = '/previous/dir' + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, dir2) + def test_discover_start_dir_is_package_calls_package_load_tests(self): # This test verifies that the package load_tests in a package is indeed # invoked when the start_dir is a package (and not the top level). @@ -789,15 +814,15 @@ def test_discovery_from_dotted_path(self): loader = unittest.TestLoader() tests = [self] - expectedPath = os.path.abspath(os.path.dirname(unittest.test.__file__)) + expectedPath = os.path.abspath(os.path.dirname(test.test_unittest.__file__)) self.wasRun = False - def _find_tests(start_dir, pattern): + def _find_tests(start_dir, pattern, namespace=None): self.wasRun = True self.assertEqual(start_dir, expectedPath) return tests loader._find_tests = _find_tests - suite = loader.discover('unittest.test') + suite = loader.discover('test.test_unittest') self.assertTrue(self.wasRun) self.assertEqual(suite._tests, tests) @@ -825,7 +850,58 @@ def restore(): 'Can not use builtin modules ' 'as dotted module names') + def test_discovery_from_dotted_namespace_packages(self): + loader = unittest.TestLoader() + + package = types.ModuleType('package') + package.__name__ = "tests" + package.__path__ = ['/a', '/b'] + package.__file__ = None + package.__spec__ = types.SimpleNamespace( + name=package.__name__, + loader=NamespaceLoader(package.__name__, package.__path__, None), + submodule_search_locations=['/a', '/b'] + ) + + def _import(packagename, *args, **kwargs): + sys.modules[packagename] = package + return package + + _find_tests_args = [] + def _find_tests(start_dir, pattern, namespace=None): + _find_tests_args.append((start_dir, pattern)) + return ['%s/tests' % start_dir] + + loader._find_tests = _find_tests + loader.suiteClass = list + + with unittest.mock.patch('builtins.__import__', _import): + # Since loader.discover() can modify sys.path, restore it when done. + with import_helper.DirsOnSysPath(): + # Make sure to remove 'package' from sys.modules when done. + with test_util.uncache('package'): + suite = loader.discover('package') + + self.assertEqual(suite, ['/a/tests', '/b/tests']) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_discovery_start_dir_is_namespace(self): + """Subdirectory discovery not affected if start_dir is a namespace pkg.""" + loader = unittest.TestLoader() + with ( + import_helper.DirsOnSysPath(os.path.join(os.path.dirname(__file__))), + test_util.uncache('namespace_test_pkg') + ): + suite = loader.discover('namespace_test_pkg') + self.assertEqual( + {list(suite)[0]._tests[0].__module__ for suite in suite._tests if list(suite)}, + # files under namespace_test_pkg.noop not discovered. + {'namespace_test_pkg.test_foo', 'namespace_test_pkg.bar.test_bar'}, + ) + def test_discovery_failed_discovery(self): + from test.test_importlib import util + loader = unittest.TestLoader() package = types.ModuleType('package') @@ -837,7 +913,7 @@ def _import(packagename, *args, **kwargs): # Since loader.discover() can modify sys.path, restore it when done. with import_helper.DirsOnSysPath(): # Make sure to remove 'package' from sys.modules when done. - with test.test_importlib.util.uncache('package'): + with util.uncache('package'): with self.assertRaises(TypeError) as cm: loader.discover('package') self.assertEqual(str(cm.exception), diff --git a/Lib/unittest/test/test_functiontestcase.py b/Lib/test/test_unittest/test_functiontestcase.py similarity index 99% rename from Lib/unittest/test/test_functiontestcase.py rename to Lib/test/test_unittest/test_functiontestcase.py index 4971729880d..2ebed9564ad 100644 --- a/Lib/unittest/test/test_functiontestcase.py +++ b/Lib/test/test_unittest/test_functiontestcase.py @@ -1,6 +1,6 @@ import unittest -from unittest.test.support import LoggingResult +from test.test_unittest.support import LoggingResult class Test_FunctionTestCase(unittest.TestCase): diff --git a/Lib/unittest/test/test_loader.py b/Lib/test/test_unittest/test_loader.py similarity index 88% rename from Lib/unittest/test/test_loader.py rename to Lib/test/test_unittest/test_loader.py index de2268cda90..0acefccf7f6 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,28 +1,8 @@ import functools import sys import types -import warnings - import unittest -# Decorator used in the deprecation tests to reset the warning registry for -# test isolation and reproducibility. -def warningregistry(func): - def wrapper(*args, **kws): - missing = [] - saved = getattr(warnings, '__warningregistry__', missing).copy() - try: - return func(*args, **kws) - finally: - if saved is missing: - try: - del warnings.__warningregistry__ - except AttributeError: - pass - else: - warnings.__warningregistry__ = saved - return wrapper - class Test_TestLoader(unittest.TestCase): @@ -96,12 +76,28 @@ def runTest(self): loader = unittest.TestLoader() # This has to be false for the test to succeed - self.assertFalse('runTest'.startswith(loader.testMethodPrefix)) + self.assertNotStartsWith('runTest', loader.testMethodPrefix) suite = loader.loadTestsFromTestCase(Foo) self.assertIsInstance(suite, loader.suiteClass) self.assertEqual(list(suite), [Foo('runTest')]) + # "Do not load any tests from `TestCase` class itself." + def test_loadTestsFromTestCase__from_TestCase(self): + loader = unittest.TestLoader() + + suite = loader.loadTestsFromTestCase(unittest.TestCase) + self.assertIsInstance(suite, loader.suiteClass) + self.assertEqual(list(suite), []) + + # "Do not load any tests from `FunctionTestCase` class." + def test_loadTestsFromTestCase__from_FunctionTestCase(self): + loader = unittest.TestLoader() + + suite = loader.loadTestsFromTestCase(unittest.FunctionTestCase) + self.assertIsInstance(suite, loader.suiteClass) + self.assertEqual(list(suite), []) + ################################################################ ### /Tests for TestLoader.loadTestsFromTestCase @@ -123,6 +119,19 @@ def test(self): expected = [loader.suiteClass([MyTestCase('test')])] self.assertEqual(list(suite), expected) + # "This test ensures that internal `TestCase` subclasses are not loaded" + def test_loadTestsFromModule__TestCase_subclass_internals(self): + # See https://github.com/python/cpython/issues/84867 + m = types.ModuleType('m') + # Simulate imported names: + m.TestCase = unittest.TestCase + m.FunctionTestCase = unittest.FunctionTestCase + + loader = unittest.TestLoader() + suite = loader.loadTestsFromModule(m) + self.assertIsInstance(suite, loader.suiteClass) + self.assertEqual(list(suite), []) + # "This method searches `module` for classes derived from TestCase" # # What happens if no tests are found (no TestCase instances)? @@ -174,9 +183,8 @@ class NotAModule(object): self.assertEqual(list(suite), reference) - # Check that loadTestsFromModule honors (or not) a module + # Check that loadTestsFromModule honors a module # with a load_tests function. - @warningregistry def test_loadTestsFromModule__load_tests(self): m = types.ModuleType('m') class MyTestCase(unittest.TestCase): @@ -195,126 +203,13 @@ def load_tests(loader, tests, pattern): suite = loader.loadTestsFromModule(m) self.assertIsInstance(suite, unittest.TestSuite) self.assertEqual(load_tests_args, [loader, suite, None]) - # With Python 3.5, the undocumented and unofficial use_load_tests is - # ignored (and deprecated). - load_tests_args = [] - with warnings.catch_warnings(record=False): - warnings.simplefilter('ignore') - suite = loader.loadTestsFromModule(m, use_load_tests=False) - self.assertEqual(load_tests_args, [loader, suite, None]) - @warningregistry - def test_loadTestsFromModule__use_load_tests_deprecated_positional(self): - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - # The method still works. - loader = unittest.TestLoader() - # use_load_tests=True as a positional argument. - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - suite = loader.loadTestsFromModule(m, False) - self.assertIsInstance(suite, unittest.TestSuite) - # load_tests was still called because use_load_tests is deprecated - # and ignored. - self.assertEqual(load_tests_args, [loader, suite, None]) - # We got a warning. - self.assertIs(w[-1].category, DeprecationWarning) - self.assertEqual(str(w[-1].message), - 'use_load_tests is deprecated and ignored') - - @warningregistry - def test_loadTestsFromModule__use_load_tests_deprecated_keyword(self): - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - # The method still works. - loader = unittest.TestLoader() - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - suite = loader.loadTestsFromModule(m, use_load_tests=False) - self.assertIsInstance(suite, unittest.TestSuite) - # load_tests was still called because use_load_tests is deprecated - # and ignored. - self.assertEqual(load_tests_args, [loader, suite, None]) - # We got a warning. - self.assertIs(w[-1].category, DeprecationWarning) - self.assertEqual(str(w[-1].message), - 'use_load_tests is deprecated and ignored') - - @warningregistry - def test_loadTestsFromModule__too_many_positional_args(self): - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - loader = unittest.TestLoader() - with self.assertRaises(TypeError) as cm, \ - warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - loader.loadTestsFromModule(m, False, 'testme.*') - # We still got the deprecation warning. - self.assertIs(w[-1].category, DeprecationWarning) - self.assertEqual(str(w[-1].message), - 'use_load_tests is deprecated and ignored') - # We also got a TypeError for too many positional arguments. - self.assertEqual(type(cm.exception), TypeError) - self.assertEqual( - str(cm.exception), - 'loadTestsFromModule() takes 1 positional argument but 3 were given') - - @warningregistry - def test_loadTestsFromModule__use_load_tests_other_bad_keyword(self): - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - loader = unittest.TestLoader() - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - with self.assertRaises(TypeError) as cm: - loader.loadTestsFromModule( - m, use_load_tests=False, very_bad=True, worse=False) - self.assertEqual(type(cm.exception), TypeError) - # The error message names the first bad argument alphabetically, - # however use_load_tests (which sorts first) is ignored. - self.assertEqual( - str(cm.exception), - "loadTestsFromModule() got an unexpected keyword argument 'very_bad'") + # In Python 3.12, the undocumented and unofficial use_load_tests has + # been removed. + with self.assertRaises(TypeError): + loader.loadTestsFromModule(m, False) + with self.assertRaises(TypeError): + loader.loadTestsFromModule(m, use_load_tests=False) def test_loadTestsFromModule__pattern(self): m = types.ModuleType('m') @@ -716,7 +611,7 @@ def test_loadTestsFromName__module_not_loaded(self): # We're going to try to load this module as a side-effect, so it # better not be loaded before we try. # - module_name = 'unittest.test.dummy' + module_name = 'test.test_unittest.dummy' sys.modules.pop(module_name, None) loader = unittest.TestLoader() @@ -844,7 +739,7 @@ def test_loadTestsFromNames__unknown_attr_name(self): loader = unittest.TestLoader() suite = loader.loadTestsFromNames( - ['unittest.loader.sdasfasfasdf', 'unittest.test.dummy']) + ['unittest.loader.sdasfasfasdf', 'test.test_unittest.dummy']) error, test = self.check_deferred_error(loader, list(suite)[0]) expected = "module 'unittest.loader' has no attribute 'sdasfasfasdf'" self.assertIn( @@ -1141,7 +1036,7 @@ def test_loadTestsFromNames__module_not_loaded(self): # We're going to try to load this module as a side-effect, so it # better not be loaded before we try. # - module_name = 'unittest.test.dummy' + module_name = 'test.test_unittest.dummy' sys.modules.pop(module_name, None) loader = unittest.TestLoader() @@ -1604,39 +1499,6 @@ def test(self): pass def reverse_three_way_cmp(a, b): return unittest.util.three_way_cmp(b, a) - def test_getTestCaseNames(self): - with self.assertWarns(DeprecationWarning) as w: - tests = unittest.getTestCaseNames(self.MyTestCase, - prefix='check', sortUsing=self.reverse_three_way_cmp, - testNamePatterns=None) - self.assertEqual(w.filename, __file__) - self.assertEqual(tests, ['check_2', 'check_1']) - - def test_makeSuite(self): - with self.assertWarns(DeprecationWarning) as w: - suite = unittest.makeSuite(self.MyTestCase, - prefix='check', sortUsing=self.reverse_three_way_cmp, - suiteClass=self.MyTestSuite) - self.assertEqual(w.filename, __file__) - self.assertIsInstance(suite, self.MyTestSuite) - expected = self.MyTestSuite([self.MyTestCase('check_2'), - self.MyTestCase('check_1')]) - self.assertEqual(suite, expected) - - def test_findTestCases(self): - m = types.ModuleType('m') - m.testcase_1 = self.MyTestCase - - with self.assertWarns(DeprecationWarning) as w: - suite = unittest.findTestCases(m, - prefix='check', sortUsing=self.reverse_three_way_cmp, - suiteClass=self.MyTestSuite) - self.assertEqual(w.filename, __file__) - self.assertIsInstance(suite, self.MyTestSuite) - expected = [self.MyTestSuite([self.MyTestCase('check_2'), - self.MyTestCase('check_1')])] - self.assertEqual(list(suite), expected) - if __name__ == "__main__": unittest.main() diff --git a/Lib/unittest/test/test_program.py b/Lib/test/test_unittest/test_program.py similarity index 76% rename from Lib/unittest/test/test_program.py rename to Lib/test/test_unittest/test_program.py index 26a8550af8f..99c5ec48b67 100644 --- a/Lib/unittest/test/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -1,21 +1,21 @@ -import io - import os -import sys import subprocess -from test import support +import sys import unittest -import unittest.test -from unittest.test.test_result import BufferedWriter + +import test.test_unittest +from test import support +from test.test_unittest.test_result import BufferedWriter +@support.force_not_colorized_test_class class Test_TestProgram(unittest.TestCase): def test_discovery_from_dotted_path(self): loader = unittest.TestLoader() tests = [self] - expectedPath = os.path.abspath(os.path.dirname(unittest.test.__file__)) + expectedPath = os.path.abspath(os.path.dirname(test.test_unittest.__file__)) self.wasRun = False def _find_tests(start_dir, pattern): @@ -23,7 +23,7 @@ def _find_tests(start_dir, pattern): self.assertEqual(start_dir, expectedPath) return tests loader._find_tests = _find_tests - suite = loader.discover('unittest.test') + suite = loader.discover('test.test_unittest') self.assertTrue(self.wasRun) self.assertEqual(suite._tests, tests) @@ -73,15 +73,30 @@ def testExpectedFailure(self): def testUnexpectedSuccess(self): pass - class FooBarLoader(unittest.TestLoader): - """Test loader that returns a suite containing FooBar.""" + class Empty(unittest.TestCase): + pass + + class SetUpClassFailure(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + raise Exception + def testPass(self): + pass + + class TestLoader(unittest.TestLoader): + """Test loader that returns a suite containing the supplied testcase.""" + + def __init__(self, testcase): + self.testcase = testcase + def loadTestsFromModule(self, module): return self.suiteClass( - [self.loadTestsFromTestCase(Test_TestProgram.FooBar)]) + [self.loadTestsFromTestCase(self.testcase)]) def loadTestsFromNames(self, names, module): return self.suiteClass( - [self.loadTestsFromTestCase(Test_TestProgram.FooBar)]) + [self.loadTestsFromTestCase(self.testcase)]) def test_defaultTest_with_string(self): class FakeRunner(object): @@ -93,10 +108,10 @@ def run(self, test): sys.argv = ['faketest'] runner = FakeRunner() program = unittest.TestProgram(testRunner=runner, exit=False, - defaultTest='unittest.test', - testLoader=self.FooBarLoader()) + defaultTest='test.test_unittest', + testLoader=self.TestLoader(self.FooBar)) sys.argv = old_argv - self.assertEqual(('unittest.test',), program.testNames) + self.assertEqual(('test.test_unittest',), program.testNames) def test_defaultTest_with_iterable(self): class FakeRunner(object): @@ -109,10 +124,10 @@ def run(self, test): runner = FakeRunner() program = unittest.TestProgram( testRunner=runner, exit=False, - defaultTest=['unittest.test', 'unittest.test2'], - testLoader=self.FooBarLoader()) + defaultTest=['test.test_unittest', 'test.test_unittest2'], + testLoader=self.TestLoader(self.FooBar)) sys.argv = old_argv - self.assertEqual(['unittest.test', 'unittest.test2'], + self.assertEqual(['test.test_unittest', 'test.test_unittest2'], program.testNames) def test_NonExit(self): @@ -120,48 +135,82 @@ def test_NonExit(self): program = unittest.main(exit=False, argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), - testLoader=self.FooBarLoader()) - self.assertTrue(hasattr(program, 'result')) + testLoader=self.TestLoader(self.FooBar)) + self.assertHasAttr(program, 'result') out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_Exit(self): stream = BufferedWriter() - self.assertRaises( - SystemExit, - unittest.main, - argv=["foobar"], - testRunner=unittest.TextTestRunner(stream=stream), - exit=True, - testLoader=self.FooBarLoader()) + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["foobar"], + testRunner=unittest.TextTestRunner(stream=stream), + exit=True, + testLoader=self.TestLoader(self.FooBar)) + self.assertEqual(cm.exception.code, 1) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) def test_ExitAsDefault(self): stream = BufferedWriter() - self.assertRaises( - SystemExit, - unittest.main, - argv=["foobar"], - testRunner=unittest.TextTestRunner(stream=stream), - testLoader=self.FooBarLoader()) + with self.assertRaises(SystemExit): + unittest.main( + argv=["foobar"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.FooBar)) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) self.assertIn('\nUNEXPECTED SUCCESS: testUnexpectedSuccess ', out) expected = ('\n\nFAILED (failures=1, errors=1, skipped=1, ' 'expected failures=1, unexpected successes=1)\n') - self.assertTrue(out.endswith(expected)) + self.assertEndsWith(out, expected) + + def test_ExitSkippedSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["foobar", "-k", "testSkipped"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.FooBar)) + self.assertEqual(cm.exception.code, 0) + out = stream.getvalue() + expected = '\n\nOK (skipped=1)\n' + self.assertEndsWith(out, expected) + + def test_ExitEmptySuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["empty"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.Empty)) + self.assertEqual(cm.exception.code, 5) + out = stream.getvalue() + self.assertIn('\nNO TESTS RAN\n', out) + + def test_ExitSetUpClassFailureSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["setup_class_failure"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.SetUpClassFailure)) + self.assertEqual(cm.exception.code, 1) + out = stream.getvalue() + self.assertIn("ERROR: setUpClass", out) + self.assertIn("SetUpClassFailure", out) class InitialisableProgram(unittest.TestProgram): @@ -286,6 +335,7 @@ def testRunTestsRunnerClass(self): program.failfast = 'failfast' program.buffer = 'buffer' program.warnings = 'warnings' + program.durations = '5' program.runTests() @@ -293,7 +343,8 @@ def testRunTestsRunnerClass(self): 'failfast': 'failfast', 'buffer': 'buffer', 'tb_locals': False, - 'warnings': 'warnings'}) + 'warnings': 'warnings', + 'durations': '5'}) self.assertEqual(FakeRunner.test, 'test') self.assertIs(program.result, RESULT) @@ -322,7 +373,8 @@ def test_locals(self): 'failfast': False, 'tb_locals': True, 'verbosity': 1, - 'warnings': None}) + 'warnings': None, + 'durations': None}) def testRunTestsOldRunnerClass(self): program = self.program @@ -335,6 +387,7 @@ def testRunTestsOldRunnerClass(self): program.failfast = 'failfast' program.buffer = 'buffer' program.test = 'test' + program.durations = '0' program.runTests() @@ -358,6 +411,7 @@ def fakeInstallHandler(): program = self.program program.catchbreak = True + program.durations = None program.testRunner = FakeRunner @@ -427,8 +481,8 @@ def _join(name): def testParseArgsAbsolutePathsThatCannotBeConverted(self): program = self.program - # even on Windows '/...' is considered absolute by os.path.abspath - argv = ['progname', '/foo/bar/baz.py', '/green/red.py'] + drive = os.path.splitdrive(os.getcwd())[0] + argv = ['progname', f'{drive}/foo/bar/baz.py', f'{drive}/green/red.py'] self._patch_isfile(argv) program.createTests = lambda: None @@ -452,6 +506,7 @@ def testParseArgsSelectedTestNames(self): self.assertEqual(program.testNamePatterns, ['*foo*', '*bar*', '*pat*']) + @unittest.expectedFailureIf(sys.platform != 'win32', 'TODO: RUSTPYTHON') def testSelectedTestNamesFunctionalTest(self): def run_unittest(args): # Use -E to ignore PYTHONSAFEPATH env var @@ -463,14 +518,14 @@ def run_unittest(args): return stderr.decode() t = '_test_warnings' - self.assertIn('Ran 7 tests', run_unittest([t])) - self.assertIn('Ran 7 tests', run_unittest(['-k', 'TestWarnings', t])) - self.assertIn('Ran 7 tests', run_unittest(['discover', '-p', '*_test*', '-k', 'TestWarnings'])) - self.assertIn('Ran 2 tests', run_unittest(['-k', 'f', t])) - self.assertIn('Ran 7 tests', run_unittest(['-k', 't', t])) - self.assertIn('Ran 3 tests', run_unittest(['-k', '*t', t])) - self.assertIn('Ran 7 tests', run_unittest(['-k', '*test_warnings.*Warning*', t])) - self.assertIn('Ran 1 test', run_unittest(['-k', '*test_warnings.*warning*', t])) + self.assertIn('Ran 5 tests', run_unittest([t])) + self.assertIn('Ran 5 tests', run_unittest(['-k', 'TestWarnings', t])) + self.assertIn('Ran 5 tests', run_unittest(['discover', '-p', '*_test*', '-k', 'TestWarnings'])) + self.assertIn('Ran 1 test ', run_unittest(['-k', 'f', t])) + self.assertIn('Ran 5 tests', run_unittest(['-k', 't', t])) + self.assertIn('Ran 2 tests', run_unittest(['-k', '*t', t])) + self.assertIn('Ran 5 tests', run_unittest(['-k', '*test_warnings.*Warning*', t])) + self.assertIn('Ran 1 test ', run_unittest(['-k', '*test_warnings.*warning*', t])) if __name__ == '__main__': diff --git a/Lib/unittest/test/test_result.py b/Lib/test/test_unittest/test_result.py similarity index 98% rename from Lib/unittest/test/test_result.py rename to Lib/test/test_unittest/test_result.py index 9320b0a44b5..c260f90bf03 100644 --- a/Lib/unittest/test/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -1,19 +1,23 @@ import io import sys import textwrap - -from test.support import warnings_helper, captured_stdout, captured_stderr - import traceback import unittest from unittest.util import strclass +from test.support import ( + captured_stdout, + force_not_colorized_test_class, + warnings_helper, +) +from test.test_unittest.support import BufferedWriter + class MockTraceback(object): class TracebackException: def __init__(self, *args, **kwargs): self.capture_locals = kwargs.get('capture_locals', False) - def format(self): + def format(self, **kwargs): result = ['A traceback'] if self.capture_locals: result.append('locals') @@ -33,22 +37,7 @@ def bad_cleanup2(): raise ValueError('bad cleanup2') -class BufferedWriter: - def __init__(self): - self.result = '' - self.buffer = '' - - def write(self, arg): - self.buffer += arg - - def flush(self): - self.result += self.buffer - self.buffer = '' - - def getvalue(self): - return self.result - - +@force_not_colorized_test_class class Test_TestResult(unittest.TestCase): # Note: there are not separate tests for TestResult.wasSuccessful(), # TestResult.errors, TestResult.failures, TestResult.testsRun or @@ -201,7 +190,7 @@ def test_1(self): test = Foo('test_1') try: test.fail("foo") - except: + except AssertionError: exc_info_tuple = sys.exc_info() result = unittest.TestResult() @@ -229,7 +218,7 @@ def test_1(self): def get_exc_info(): try: test.fail("foo") - except: + except AssertionError: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -256,9 +245,9 @@ def get_exc_info(): try: try: test.fail("foo") - except: + except AssertionError: raise ValueError(42) - except: + except ValueError: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -286,7 +275,7 @@ def get_exc_info(): loop.__cause__ = loop loop.__context__ = loop raise loop - except: + except Exception: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -315,7 +304,7 @@ def get_exc_info(): ex1.__cause__ = ex2 ex2.__context__ = ex1 raise C - except: + except Exception: return sys.exc_info() exc_info_tuple = get_exc_info() @@ -360,7 +349,7 @@ def test_1(self): test = Foo('test_1') try: raise TypeError() - except: + except TypeError: exc_info_tuple = sys.exc_info() result = unittest.TestResult() @@ -465,12 +454,14 @@ def testFailFastSetByRunner(self): stream = BufferedWriter() runner = unittest.TextTestRunner(stream=stream, failfast=True) def test(result): + result.testsRun += 1 self.assertTrue(result.failfast) result = runner.run(test) stream.flush() - self.assertTrue(stream.getvalue().endswith('\n\nOK\n')) + self.assertEndsWith(stream.getvalue(), '\n\nOK\n') +@force_not_colorized_test_class class Test_TextTestResult(unittest.TestCase): maxDiff = None @@ -505,8 +496,8 @@ def testGetSubTestDescriptionWithoutDocstringAndParams(self): '(' + __name__ + '.Test_TextTestResult.testGetSubTestDescriptionWithoutDocstringAndParams) ' '(<subtest>)') - def testGetSubTestDescriptionForFalsyValues(self): - expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TextTestResult.testGetSubTestDescriptionForFalsyValues) [%s]' + def testGetSubTestDescriptionForFalseValues(self): + expected = 'testGetSubTestDescriptionForFalseValues (%s.Test_TextTestResult.testGetSubTestDescriptionForFalseValues) [%s]' result = unittest.TextTestResult(None, True, 1) for arg in [0, None, []]: with self.subTest(arg): @@ -772,6 +763,7 @@ def testFoo(self): runner.run(Test('testFoo')) +@force_not_colorized_test_class class TestOutputBuffering(unittest.TestCase): def setUp(self): diff --git a/Lib/unittest/test/test_runner.py b/Lib/test/test_unittest/test_runner.py similarity index 89% rename from Lib/unittest/test/test_runner.py rename to Lib/test/test_unittest/test_runner.py index 50d9ed2fd33..b215a3664d1 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -1,15 +1,17 @@ import io import os -import sys import pickle import subprocess -from test import support - +import sys import unittest from unittest.case import _Outcome -from unittest.test.support import (LoggingResult, - ResultWithNoStartTestRunStopTestRun) +from test import support +from test.test_unittest.support import ( + BufferedWriter, + LoggingResult, + ResultWithNoStartTestRunStopTestRun, +) def resultFactory(*_): @@ -21,6 +23,13 @@ def getRunner(): stream=io.StringIO()) +class CustomError(Exception): + pass + +# For test output compat: +CustomErrorRepr = f"{__name__ + '.' if __name__ != '__main__' else ''}CustomError" + + def runTests(*cases): suite = unittest.TestSuite() for case in cases: @@ -43,7 +52,7 @@ def cleanup(ordering, blowUp=False): ordering.append('cleanup_good') else: ordering.append('cleanup_exc') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestCM: @@ -96,6 +105,7 @@ def cleanup2(*args, **kwargs): self.assertTrue(test.doCleanups()) self.assertEqual(cleanups, [(2, (), {}), (1, (1, 2, 3), dict(four='hello', five='goodbye'))]) + @support.force_not_colorized def testCleanUpWithErrors(self): class TestableTest(unittest.TestCase): def testNothing(self): @@ -105,8 +115,8 @@ def testNothing(self): result = unittest.TestResult() outcome = test._outcome = _Outcome(result=result) - CleanUpExc = Exception('foo') - exc2 = Exception('bar') + CleanUpExc = CustomError('foo') + exc2 = CustomError('bar') def cleanup1(): raise CleanUpExc @@ -122,10 +132,10 @@ def cleanup2(): (_, msg2), (_, msg1) = result.errors self.assertIn('in cleanup1', msg1) self.assertIn('raise CleanUpExc', msg1) - self.assertIn('Exception: foo', msg1) + self.assertIn(f'{CustomErrorRepr}: foo', msg1) self.assertIn('in cleanup2', msg2) self.assertIn('raise exc2', msg2) - self.assertIn('Exception: bar', msg2) + self.assertIn(f'{CustomErrorRepr}: bar', msg2) def testCleanupInRun(self): blowUp = False @@ -136,7 +146,7 @@ def setUp(self): ordering.append('setUp') test.addCleanup(cleanup2) if blowUp: - raise Exception('foo') + raise CustomError('foo') def testNothing(self): ordering.append('test') @@ -239,6 +249,7 @@ def testNothing(self): self.assertEqual(test._cleanups, []) +@support.force_not_colorized_test_class class TestClassCleanup(unittest.TestCase): def test_addClassCleanUp(self): class TestableTest(unittest.TestCase): @@ -277,7 +288,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -303,7 +314,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -343,7 +354,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, @@ -363,10 +374,10 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownClassExc') + raise CustomError('TearDownClassExc') suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -376,7 +387,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -389,16 +400,22 @@ def testNothing(self): pass def cleanup1(): - raise Exception('cleanup1') + raise CustomError('cleanup1') def cleanup2(): - raise Exception('cleanup2') + raise CustomError('cleanup2') TestableTest.addClassCleanup(cleanup1) TestableTest.addClassCleanup(cleanup2) - with self.assertRaises(Exception) as e: - TestableTest.doClassCleanups() - self.assertEqual(e, 'cleanup1') + TestableTest.doClassCleanups() + + self.assertEqual(len(TestableTest.tearDown_exceptions), 2) + + e1, e2 = TestableTest.tearDown_exceptions + self.assertIsInstance(e1[1], CustomError) + self.assertEqual(str(e1[1]), 'cleanup2') + self.assertIsInstance(e2[1], CustomError) + self.assertEqual(str(e2[1]), 'cleanup1') def test_with_errors_addCleanUp(self): ordering = [] @@ -418,7 +435,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) @@ -441,7 +458,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) @@ -457,11 +474,11 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering, blowUp=True) if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -470,7 +487,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) @@ -480,9 +497,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'cleanup_exc']) @@ -491,9 +508,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) @@ -510,11 +527,11 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownExc') + raise CustomError('TearDownExc') result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: TearDownExc') + f'{CustomErrorRepr}: TearDownExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) @@ -574,7 +591,18 @@ def test(self): 'inner setup', 'inner test', 'inner cleanup', 'end outer test', 'outer cleanup']) + def test_run_empty_suite_error_message(self): + class EmptyTest(unittest.TestCase): + pass + + suite = unittest.defaultTestLoader.loadTestsFromTestCase(EmptyTest) + runner = getRunner() + runner.run(suite) + self.assertIn("\nNO TESTS RAN\n", runner.stream.getvalue()) + + +@support.force_not_colorized_test_class class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): module_cleanups = [] @@ -607,7 +635,7 @@ def module_cleanup_good(*args, **kwargs): module_cleanups.append((3, args, kwargs)) def module_cleanup_bad(*args, **kwargs): - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class Module(object): unittest.addModuleCleanup(module_cleanup_good, 1, 2, 3, @@ -617,7 +645,7 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(Exception) as e: + with self.assertRaises(CustomError) as e: unittest.case.doModuleCleanups() self.assertEqual(str(e.exception), 'CleanUpExc') self.assertEqual(unittest.case._module_cleanups, []) @@ -646,7 +674,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception('setUpModule Exc') + raise CustomError('setUpModule Exc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -666,7 +694,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(ordering, ['setUpModule', 'cleanup_good']) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: setUpModule Exc') + f'{CustomErrorRepr}: setUpModule Exc') ordering = [] blowUp = False @@ -686,7 +714,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -697,7 +725,7 @@ def setUpModule(): ordering.append('setUpModule2') unittest.addModuleCleanup(cleanup, ordering) if blowUp2: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule2') @@ -786,7 +814,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestableTest(unittest.TestCase): @classmethod @@ -802,7 +830,7 @@ def tearDownClass(cls): sys.modules['Module'] = Module result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_good']) @@ -842,7 +870,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -860,7 +888,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('TearDownModuleExc') + raise CustomError('TearDownModuleExc') class TestableTest(unittest.TestCase): @classmethod @@ -875,7 +903,7 @@ def tearDownClass(cls): TestableTest.__module__ = 'Module' sys.modules['Module'] = Module suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -886,7 +914,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -965,7 +993,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -995,7 +1023,7 @@ def tearDown(self): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1011,7 +1039,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering, blowUp=True) if module_blow_up: - raise Exception('ModuleExc') + raise CustomError('ModuleExc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -1021,11 +1049,11 @@ class TestableTest(unittest.TestCase): def setUpClass(cls): ordering.append('setUpClass') if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -1037,7 +1065,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'test', 'tearDownClass', 'tearDownModule', @@ -1049,9 +1077,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ModuleExc') + f'{CustomErrorRepr}: ModuleExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] @@ -1060,9 +1088,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'tearDownModule', 'cleanup_exc']) @@ -1072,9 +1100,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) @@ -1176,6 +1204,7 @@ def test_init(self): self.assertTrue(runner.descriptions) self.assertEqual(runner.resultclass, unittest.TextTestResult) self.assertFalse(runner.tb_locals) + self.assertIsNone(runner.durations) def test_multiple_inheritance(self): class AResult(unittest.TestResult): @@ -1267,8 +1296,6 @@ def _makeResult(self): expected = ['startTestRun', 'stopTestRun'] self.assertEqual(events, expected) - # TODO: RUSTPYTHON; fix pickling with io objects - @unittest.expectedFailure def test_pickle_unpickle(self): # Issue #7197: a TextTestRunner should be (un)pickleable. This is # required by test_multiprocessing under Windows (in verbose mode). @@ -1293,6 +1320,7 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + @support.force_not_colorized @support.requires_subprocess() def test_warnings(self): """ @@ -1305,8 +1333,6 @@ def get_parse_out_err(p): return [b.splitlines() for b in p.communicate()] opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(__file__)) - ae_msg = b'Please use assertEqual instead.' - at_msg = b'Please use assertTrue instead.' # no args -> all the warnings are printed, unittest warnings only once p = subprocess.Popen([sys.executable, '-E', '_test_warnings.py'], **opts) @@ -1314,11 +1340,11 @@ def get_parse_out_err(p): out, err = get_parse_out_err(p) self.assertIn(b'OK', err) # check that the total number of warnings in the output is correct - self.assertEqual(len(out), 12) + self.assertEqual(len(out), 10) # check that the numbers of the different kind of warnings is correct for msg in [b'dw', b'iw', b'uw']: self.assertEqual(out.count(msg), 3) - for msg in [ae_msg, at_msg, b'rw']: + for msg in [b'rw']: self.assertEqual(out.count(msg), 1) args_list = ( @@ -1345,11 +1371,9 @@ def get_parse_out_err(p): with p: out, err = get_parse_out_err(p) self.assertIn(b'OK', err) - self.assertEqual(len(out), 14) + self.assertEqual(len(out), 12) for msg in [b'dw', b'iw', b'uw', b'rw']: self.assertEqual(out.count(msg), 3) - for msg in [ae_msg, at_msg]: - self.assertEqual(out.count(msg), 1) def testStdErrLookedUpAtInstantiationTime(self): # see issue 10786 @@ -1368,6 +1392,65 @@ def testSpecifiedStreamUsed(self): runner = unittest.TextTestRunner(f) self.assertTrue(runner.stream.stream is f) + def test_durations(self): + def run(test, *, expect_durations=True): + stream = BufferedWriter() + runner = unittest.TextTestRunner(stream=stream, durations=5, verbosity=2) + result = runner.run(test) + self.assertEqual(result.durations, 5) + stream.flush() + text = stream.getvalue() + regex = r"\n\d+.\d\d\ds" + if expect_durations: + self.assertEqual(len(result.collectedDurations), 1) + self.assertIn('Slowest test durations', text) + self.assertRegex(text, regex) + else: + self.assertEqual(len(result.collectedDurations), 0) + self.assertNotIn('Slowest test durations', text) + self.assertNotRegex(text, regex) + + # success + class Foo(unittest.TestCase): + def test_1(self): + pass + + run(Foo('test_1'), expect_durations=True) + + # failure + class Foo(unittest.TestCase): + def test_1(self): + self.assertEqual(0, 1) + + run(Foo('test_1'), expect_durations=True) + + # error + class Foo(unittest.TestCase): + def test_1(self): + 1 / 0 + + run(Foo('test_1'), expect_durations=True) + + + # error in setUp and tearDown + class Foo(unittest.TestCase): + def setUp(self): + 1 / 0 + tearDown = setUp + def test_1(self): + pass + + run(Foo('test_1'), expect_durations=True) + + # skip (expect no durations) + class Foo(unittest.TestCase): + @unittest.skip("reason") + def test_1(self): + pass + + run(Foo('test_1'), expect_durations=False) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/unittest/test/test_setups.py b/Lib/test/test_unittest/test_setups.py similarity index 99% rename from Lib/unittest/test/test_setups.py rename to Lib/test/test_unittest/test_setups.py index 2df703ed934..2468681003b 100644 --- a/Lib/unittest/test/test_setups.py +++ b/Lib/test/test_unittest/test_setups.py @@ -1,6 +1,5 @@ import io import sys - import unittest diff --git a/Lib/unittest/test/test_skipping.py b/Lib/test/test_unittest/test_skipping.py similarity index 99% rename from Lib/unittest/test/test_skipping.py rename to Lib/test/test_unittest/test_skipping.py index 64ceeae37ef..f5cb860c60b 100644 --- a/Lib/unittest/test/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -1,6 +1,7 @@ import unittest -from unittest.test.support import LoggingResult +from test.support import force_not_colorized +from test.test_unittest.support import LoggingResult class Test_TestSkipping(unittest.TestCase): @@ -293,6 +294,7 @@ def test_die(self): self.assertFalse(result.unexpectedSuccesses) self.assertTrue(result.wasSuccessful()) + @force_not_colorized def test_expected_failure_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure @@ -372,6 +374,7 @@ def test_die(self): self.assertEqual(result.unexpectedSuccesses, [test]) self.assertFalse(result.wasSuccessful()) + @force_not_colorized def test_unexpected_success_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure diff --git a/Lib/unittest/test/test_suite.py b/Lib/test/test_unittest/test_suite.py similarity index 99% rename from Lib/unittest/test/test_suite.py rename to Lib/test/test_unittest/test_suite.py index 0551a169967..11c8c859f3d 100644 --- a/Lib/unittest/test/test_suite.py +++ b/Lib/test/test_unittest/test_suite.py @@ -1,10 +1,9 @@ -import unittest - import gc import sys +import unittest import weakref -from unittest.test.support import LoggingResult, TestEquality +from test.test_unittest.support import LoggingResult, TestEquality ### Support code for Test_TestSuite ################################################################ diff --git a/Lib/test/test_unittest/test_util.py b/Lib/test/test_unittest/test_util.py new file mode 100644 index 00000000000..abadcb96601 --- /dev/null +++ b/Lib/test/test_unittest/test_util.py @@ -0,0 +1,37 @@ +import unittest +from unittest.util import ( + safe_repr, + sorted_list_difference, + unorderable_list_difference, +) + + +class TestUtil(unittest.TestCase): + def test_safe_repr(self): + class RaisingRepr: + def __repr__(self): + raise ValueError("Invalid repr()") + + class LongRepr: + def __repr__(self): + return 'x' * 100 + + safe_repr(RaisingRepr()) + self.assertEqual(safe_repr('foo'), "'foo'") + self.assertEqual(safe_repr(LongRepr(), short=True), 'x'*80 + ' [truncated]...') + + def test_sorted_list_difference(self): + self.assertEqual(sorted_list_difference([], []), ([], [])) + self.assertEqual(sorted_list_difference([1, 2], [2, 3]), ([1], [3])) + self.assertEqual(sorted_list_difference([1, 2], [1, 3]), ([2], [3])) + self.assertEqual(sorted_list_difference([1, 1, 1], [1, 2, 3]), ([], [2, 3])) + self.assertEqual(sorted_list_difference([4], [1, 2, 3, 4]), ([], [1, 2, 3])) + self.assertEqual(sorted_list_difference([1, 1], [2]), ([1], [2])) + self.assertEqual(sorted_list_difference([2], [1, 1]), ([2], [1])) + self.assertEqual(sorted_list_difference([1, 2], [1, 1]), ([2], [])) + + def test_unorderable_list_difference(self): + self.assertEqual(unorderable_list_difference([], []), ([], [])) + self.assertEqual(unorderable_list_difference([1, 2], []), ([2, 1], [])) + self.assertEqual(unorderable_list_difference([], [1, 2]), ([], [1, 2])) + self.assertEqual(unorderable_list_difference([1, 2], [1, 3]), ([2], [3])) diff --git a/Lib/test/test_unittest/testmock/__init__.py b/Lib/test/test_unittest/testmock/__init__.py new file mode 100644 index 00000000000..bc502ef32d2 --- /dev/null +++ b/Lib/test/test_unittest/testmock/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_unittest/testmock/__main__.py b/Lib/test/test_unittest/testmock/__main__.py new file mode 100644 index 00000000000..1e3068b0ddf --- /dev/null +++ b/Lib/test/test_unittest/testmock/__main__.py @@ -0,0 +1,18 @@ +import os +import unittest + + +def load_tests(loader, standard_tests, pattern): + # top level directory cached on loader instance + this_dir = os.path.dirname(__file__) + pattern = pattern or "test*.py" + # We are inside test.test_unittest.testmock, so the top-level is three notches up + top_level_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_dir))) + package_tests = loader.discover(start_dir=this_dir, pattern=pattern, + top_level_dir=top_level_dir) + standard_tests.addTests(package_tests) + return standard_tests + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_unittest/testmock/support.py b/Lib/test/test_unittest/testmock/support.py new file mode 100644 index 00000000000..6c535b7944f --- /dev/null +++ b/Lib/test/test_unittest/testmock/support.py @@ -0,0 +1,27 @@ +target = {'foo': 'FOO'} + + +def is_instance(obj, klass): + """Version of is_instance that doesn't access __class__""" + return issubclass(type(obj), klass) + + +class SomeClass(object): + class_attribute = None + + def wibble(self): pass + + +class X(object): + pass + +# A standin for weurkzeug.local.LocalProxy - issue 119600 +def _inaccessible(*args, **kwargs): + raise AttributeError + + +class OpaqueProxy: + __getattribute__ = _inaccessible + + +g = OpaqueProxy() diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py similarity index 94% rename from Lib/unittest/test/testmock/testasync.py rename to Lib/test/test_unittest/testmock/testasync.py index 90ea72d82b0..81d9c9c55fd 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -218,10 +218,6 @@ def test_create_autospec_instance(self): with self.assertRaises(RuntimeError): create_autospec(async_func, instance=True) - @unittest.skip('Broken test from https://bugs.python.org/issue37251') - def test_create_autospec_awaitable_class(self): - self.assertIsInstance(create_autospec(AwaitableClass), AsyncMock) - def test_create_autospec(self): spec = create_autospec(async_func_args) awaitable = spec(1, 2, c=3) @@ -236,7 +232,9 @@ async def main(): run(main()) self.assertTrue(iscoroutinefunction(spec)) + self.assertTrue(inspect.iscoroutinefunction(spec)) self.assertTrue(asyncio.iscoroutine(awaitable)) + self.assertTrue(inspect.iscoroutine(awaitable)) self.assertEqual(spec.await_count, 1) self.assertEqual(spec.await_args, call(1, 2, c=3)) self.assertEqual(spec.await_args_list, [call(1, 2, c=3)]) @@ -248,6 +246,25 @@ async def main(): with self.assertRaises(AssertionError): spec.assert_any_await(e=1) + def test_autospec_checks_signature(self): + spec = create_autospec(async_func_args) + # signature is not checked when called + awaitable = spec() + self.assertListEqual(spec.mock_calls, []) + + async def main(): + await awaitable + + # but it is checked when awaited + with self.assertRaises(TypeError): + run(main()) + + # _checksig_ raises before running or awaiting the mock + self.assertListEqual(spec.mock_calls, []) + self.assertEqual(spec.await_count, 0) + self.assertIsNone(spec.await_args) + self.assertEqual(spec.await_args_list, []) + spec.assert_not_awaited() def test_patch_with_autospec(self): @@ -257,7 +274,9 @@ async def test_async(): self.assertIsInstance(mock_method.mock, AsyncMock) self.assertTrue(iscoroutinefunction(mock_method)) + self.assertTrue(inspect.iscoroutinefunction(mock_method)) self.assertTrue(asyncio.iscoroutine(awaitable)) + self.assertTrue(inspect.iscoroutine(awaitable)) self.assertTrue(inspect.isawaitable(awaitable)) # Verify the default values during mock setup @@ -308,6 +327,20 @@ def test_spec_normal_methods_on_class_with_mock_seal(self): with self.assertRaises(AttributeError): mock.async_method + def test_spec_async_attributes_instance(self): + async_instance = AsyncClass() + async_instance.async_func_attr = async_func + async_instance.later_async_func_attr = normal_func + + mock_async_instance = Mock(spec_set=async_instance) + + async_instance.later_async_func_attr = async_func + + self.assertIsInstance(mock_async_instance.async_func_attr, AsyncMock) + # only the shape of the spec at the time of mock construction matters + self.assertNotIsInstance(mock_async_instance.later_async_func_attr, AsyncMock) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) @@ -322,6 +355,7 @@ def inner_test(mock_type): with self.subTest(f"test spec kwarg with {mock_type}"): inner_test(mock_type) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_spec_mock_type_positional(self): def inner_test(mock_type): async_mock = mock_type(async_func) @@ -427,9 +461,10 @@ async def addition(self, var): pass self.assertEqual(output, 10) async def test_add_side_effect_exception(self): + class CustomError(Exception): pass async def addition(var): pass - mock = AsyncMock(addition, side_effect=Exception('err')) - with self.assertRaises(Exception): + mock = AsyncMock(addition, side_effect=CustomError('side-effect')) + with self.assertRaisesRegex(CustomError, 'side-effect'): await mock(5) async def test_add_side_effect_coroutine(self): @@ -699,8 +734,6 @@ def __aiter__(self): pass async def __anext__(self): pass - # TODO: RUSTPYTHON; async for - """ def test_aiter_set_return_value(self): mock_iter = AsyncMock(name="tester") mock_iter.__aiter__.return_value = [1, 2, 3] @@ -708,7 +741,6 @@ async def main(): return [i async for i in mock_iter] result = run(main()) self.assertEqual(result, [1, 2, 3]) - """ def test_mock_aiter_and_anext_asyncmock(self): def inner_test(mock_type): @@ -726,8 +758,7 @@ def inner_test(mock_type): with self.subTest(f"test aiter and anext corourtine with {mock_type}"): inner_test(mock_type) - # TODO: RUSTPYTHON; async for - """ + def test_mock_async_for(self): async def iterate(iterator): accumulator = [] @@ -761,7 +792,7 @@ def test_set_return_value_iter(mock_type): with self.subTest(f"set return_value iterator with {mock_type}"): test_set_return_value_iter(mock_type) - """ + class AsyncMockAssert(unittest.TestCase): def setUp(self): @@ -773,6 +804,7 @@ async def _runnable_test(self, *args, **kwargs): async def _await_coroutine(self, coroutine): return await coroutine + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_called_but_not_awaited(self): mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): @@ -813,6 +845,7 @@ def test_assert_called_and_awaited_at_same_time(self): self.mock.assert_called_once() self.mock.assert_awaited_once() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_called_twice_and_awaited_once(self): mock = AsyncMock(AsyncClass) coroutine = mock.async_method() @@ -827,6 +860,7 @@ def test_assert_called_twice_and_awaited_once(self): mock.async_method.assert_awaited() mock.async_method.assert_awaited_once() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_called_once_and_awaited_twice(self): mock = AsyncMock(AsyncClass) coroutine = mock.async_method() @@ -851,6 +885,7 @@ def test_assert_awaited_but_not_called(self): with self.assertRaises(AssertionError): self.mock.assert_called() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_has_calls_not_awaits(self): kalls = [call('foo')] with assertNeverAwaited(self): @@ -859,6 +894,7 @@ def test_assert_has_calls_not_awaits(self): with self.assertRaises(AssertionError): self.mock.assert_has_awaits(kalls) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_has_mock_calls_on_async_mock_no_spec(self): with assertNeverAwaited(self): self.mock() @@ -872,6 +908,7 @@ def test_assert_has_mock_calls_on_async_mock_no_spec(self): mock_kalls = ([call(), call('foo'), call('baz')]) self.assertEqual(self.mock.mock_calls, mock_kalls) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assert_has_mock_calls_on_async_mock_with_spec(self): a_class_mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): @@ -887,6 +924,7 @@ def test_assert_has_mock_calls_on_async_mock_with_spec(self): self.assertEqual(a_class_mock.async_method.mock_calls, method_kalls) self.assertEqual(a_class_mock.mock_calls, mock_kalls) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_async_method_calls_recorded(self): with assertNeverAwaited(self): self.mock.something(3, fish=None) @@ -902,6 +940,7 @@ def test_async_method_calls_recorded(self): [("something", (6,), {'cake': sentinel.Cake})], "method calls not recorded correctly") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_async_arg_lists(self): def assert_attrs(mock): names = ('call_args_list', 'method_calls', 'mock_calls') diff --git a/Lib/unittest/test/testmock/testcallable.py b/Lib/test/test_unittest/testmock/testcallable.py similarity index 98% rename from Lib/unittest/test/testmock/testcallable.py rename to Lib/test/test_unittest/testmock/testcallable.py index 5eadc007049..ca88511f639 100644 --- a/Lib/unittest/test/testmock/testcallable.py +++ b/Lib/test/test_unittest/testmock/testcallable.py @@ -3,7 +3,7 @@ # http://www.voidspace.org.uk/python/mock/ import unittest -from unittest.test.testmock.support import is_instance, X, SomeClass +from test.test_unittest.testmock.support import is_instance, X, SomeClass from unittest.mock import ( Mock, MagicMock, NonCallableMagicMock, diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py similarity index 96% rename from Lib/unittest/test/testmock/testhelpers.py rename to Lib/test/test_unittest/testmock/testhelpers.py index 9e7ec5d62d5..a19f04eb88f 100644 --- a/Lib/unittest/test/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -43,6 +43,7 @@ def test_any_and_datetime(self): mock.assert_called_with(ANY, foo=ANY) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_any_mock_calls_comparison_order(self): mock = Mock() class Foo(object): @@ -952,6 +953,24 @@ def __getattr__(self, attribute): self.assertFalse(hasattr(autospec, '__name__')) + def test_autospec_signature_staticmethod(self): + class Foo: + @staticmethod + def static_method(a, b=10, *, c): pass + + mock = create_autospec(Foo.__dict__['static_method']) + self.assertEqual(inspect.signature(Foo.static_method), inspect.signature(mock)) + + + def test_autospec_signature_classmethod(self): + class Foo: + @classmethod + def class_method(cls, a, b=10, *, c): pass + + mock = create_autospec(Foo.__dict__['class_method']) + self.assertEqual(inspect.signature(Foo.class_method), inspect.signature(mock)) + + def test_spec_inspect_signature(self): def myfunc(x, y): pass @@ -1077,7 +1096,7 @@ def test_propertymock(self): p.stop() - def test_propertymock_returnvalue(self): + def test_propertymock_bare(self): m = MagicMock() p = PropertyMock() type(m).foo = p @@ -1088,6 +1107,35 @@ def test_propertymock_returnvalue(self): self.assertNotIsInstance(returned, PropertyMock) + def test_propertymock_returnvalue(self): + m = MagicMock() + p = PropertyMock(return_value=42) + type(m).foo = p + + returned = m.foo + p.assert_called_once_with() + self.assertEqual(returned, 42) + self.assertNotIsInstance(returned, PropertyMock) + + + def test_propertymock_side_effect(self): + m = MagicMock() + p = PropertyMock(side_effect=ValueError) + type(m).foo = p + + with self.assertRaises(ValueError): + m.foo + p.assert_called_once_with() + + + def test_propertymock_attach(self): + m = Mock() + p = PropertyMock() + type(m).foo = p + m.attach_mock(p, 'foo') + self.assertEqual(m.mock_calls, []) + + class TestCallablePredicate(unittest.TestCase): def test_type(self): diff --git a/Lib/unittest/test/testmock/testmagicmethods.py b/Lib/test/test_unittest/testmock/testmagicmethods.py similarity index 91% rename from Lib/unittest/test/testmock/testmagicmethods.py rename to Lib/test/test_unittest/testmock/testmagicmethods.py index a4feae7e9d3..a8b52ce4871 100644 --- a/Lib/unittest/test/testmock/testmagicmethods.py +++ b/Lib/test/test_unittest/testmock/testmagicmethods.py @@ -331,6 +331,45 @@ def test_magic_methods_fspath(self): self.assertEqual(os.fspath(mock), expected_path) mock.__fspath__.assert_called_once() + def test_magic_mock_does_not_reset_magic_returns(self): + # https://github.com/python/cpython/issues/123934 + for reset in (True, False): + with self.subTest(reset=reset): + mm = MagicMock() + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + for _ in range(3): + # Repeat reset several times to be sure: + mm.reset_mock(return_value=reset) + + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + def test_magic_mock_resets_manual_mocks(self): + mm = MagicMock() + mm.__iter__ = MagicMock(return_value=iter([1])) + mm.custom = MagicMock(return_value=2) + self.assertEqual(list(iter(mm)), [1]) + self.assertEqual(mm.custom(), 2) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) + self.assertIsInstance(mm.custom(), MagicMock) + + def test_magic_mock_resets_manual_mocks_empty_iter(self): + mm = MagicMock() + mm.__iter__.return_value = [] + self.assertEqual(list(iter(mm)), []) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) def test_magic_methods_and_spec(self): class Iterable(object): diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py similarity index 91% rename from Lib/unittest/test/testmock/testmock.py rename to Lib/test/test_unittest/testmock/testmock.py index eaae22e854e..e1b108f81e5 100644 --- a/Lib/unittest/test/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -5,7 +5,7 @@ from test.support import ALWAYS_EQ import unittest -from unittest.test.testmock.support import is_instance +from test.test_unittest.testmock.support import is_instance from unittest import mock from unittest.mock import ( call, DEFAULT, patch, sentinel, @@ -38,6 +38,17 @@ def cmeth(cls, a, b, c, d=None): pass def smeth(a, b, c, d=None): pass +class SomethingElse(object): + def __init__(self): + self._instance = None + + @property + def instance(self): + if not self._instance: + self._instance = 'object' + return self._instance + + class Typos(): autospect = None auto_spec = None @@ -104,6 +115,24 @@ def f(): pass with self.assertRaises(TypeError): mock() + def test_create_autospec_should_be_configurable_by_kwargs(self): + """If kwargs are given to configure mock, the function must configure + the parent mock during initialization.""" + mocked_result = 'mocked value' + class_mock = create_autospec(spec=Something, **{ + 'return_value.meth.side_effect': [ValueError, DEFAULT], + 'return_value.meth.return_value': mocked_result}) + with self.assertRaises(ValueError): + class_mock().meth(a=None, b=None, c=None) + self.assertEqual(class_mock().meth(a=None, b=None, c=None), mocked_result) + # Only the parent mock should be configurable because the user will + # pass kwargs with respect to the parent mock. + self.assertEqual(class_mock().return_value.meth.side_effect, None) + + def test_create_autospec_correctly_handles_name(self): + class X: ... + mock = create_autospec(X, spec_set=True, name="Y") + self.assertEqual(mock._mock_name, "Y") def test_repr(self): mock = Mock(name='foo') @@ -234,6 +263,73 @@ class B(object): with mock.patch('builtins.open', mock.mock_open()): mock.mock_open() # should still be valid with open() mocked + def test_create_autospec_wraps_class(self): + """Autospec a class with wraps & test if the call is passed to the + wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + class_mock = create_autospec(spec=Result, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result (actual instance of "Result") when the mock is called. + class_mock.return_value = mock.DEFAULT + self.assertEqual(class_mock().get_result(), result) + # Autospec should also wrap child attributes of parent. + self.assertEqual(class_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_instance_wraps_class(self): + """Autospec a class instance with wraps & test if the call is passed + to the wrapped object.""" + result = "real result" + + class Result: + @staticmethod + def get_result(): + """This is a static method because when the mocked instance of + 'Result' will call this method, it won't be able to consume + 'self' argument.""" + return result + instance_mock = create_autospec(spec=Result, instance=True, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result from "Result.get_result" when the mocked instance of "Result" + # calls "get_result". + instance_mock.get_result.return_value = mock.DEFAULT + self.assertEqual(instance_mock.get_result(), result) + # Autospec should also wrap child attributes of the instance. + self.assertEqual(instance_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_wraps_function_type(self): + """Autospec a function or a method with wraps & test if the call is + passed to the wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + func_mock = create_autospec(spec=Result.get_result, wraps=Result.get_result) + self.assertEqual(func_mock(Result()), result) + + def test_explicit_return_value_even_if_mock_wraps_object(self): + """If the mock has an explicit return_value set then calls are not + passed to the wrapped object and the return_value is returned instead. + """ + def my_func(): + return None + func_mock = create_autospec(spec=my_func, wraps=my_func) + return_value = "explicit return value" + func_mock.return_value = return_value + self.assertEqual(func_mock(), return_value) + + def test_explicit_parent(self): + parent = Mock() + mock1 = Mock(parent=parent, return_value=None) + mock1(1, 2, 3) + mock2 = Mock(parent=parent, return_value=None) + mock2(4, 5, 6) + + self.assertEqual(parent.mock_calls, [call(1, 2, 3), call(4, 5, 6)]) def test_reset_mock(self): parent = Mock() @@ -603,6 +699,14 @@ def test_wraps_calls(self): real = Mock() mock = Mock(wraps=real) + # If "Mock" wraps an object, just accessing its + # "return_value" ("NonCallableMock.__get_return_value") should not + # trigger its descriptor ("NonCallableMock.__set_return_value") so + # the default "return_value" should always be "sentinel.DEFAULT". + self.assertEqual(mock.return_value, DEFAULT) + # It will not be "sentinel.DEFAULT" if the mock is not wrapping any + # object. + self.assertNotEqual(real.return_value, DEFAULT) self.assertEqual(mock(), real()) real.reset_mock() @@ -1039,7 +1143,7 @@ def test_assert_called_with_failure_message(self): actual = 'not called.' expected = "mock(1, '2', 3, bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), mock.assert_called_with, 1, '2', 3, bar='foo' @@ -1054,7 +1158,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(1, '2', 3, bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, 1, '2', 3, bar='foo' @@ -1064,7 +1168,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, bar='foo' @@ -1074,7 +1178,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(1, 2, 3)" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, 1, 2, 3 @@ -1084,7 +1188,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo()" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth ) @@ -1528,25 +1632,33 @@ def f(x=None): pass mock = Mock(spec=f) mock(1) - with self.assertRaisesRegex( - AssertionError, - '^{}$'.format( - re.escape('Calls not found.\n' - 'Expected: [call()]\n' - 'Actual: [call(1)]'))) as cm: + with self.assertRaises(AssertionError) as cm: mock.assert_has_calls([call()]) + self.assertEqual(str(cm.exception), + 'Calls not found.\n' + 'Expected: [call()]\n' + ' Actual: [call(1)]' + ) self.assertIsNone(cm.exception.__cause__) + uncalled_mock = Mock() + with self.assertRaises(AssertionError) as cm: + uncalled_mock.assert_has_calls([call()]) + self.assertEqual(str(cm.exception), + 'Calls not found.\n' + 'Expected: [call()]\n' + ' Actual: []' + ) + self.assertIsNone(cm.exception.__cause__) - with self.assertRaisesRegex( - AssertionError, - '^{}$'.format( - re.escape( - 'Error processing expected calls.\n' - "Errors: [None, TypeError('too many positional arguments')]\n" - "Expected: [call(), call(1, 2)]\n" - 'Actual: [call(1)]'))) as cm: + with self.assertRaises(AssertionError) as cm: mock.assert_has_calls([call(), call(1, 2)]) + self.assertEqual(str(cm.exception), + 'Error processing expected calls.\n' + "Errors: [None, TypeError('too many positional arguments')]\n" + 'Expected: [call(), call(1, 2)]\n' + ' Actual: [call(1)]' + ) self.assertIsInstance(cm.exception.__cause__, TypeError) def test_assert_any_call(self): @@ -1645,21 +1757,41 @@ def test_mock_unsafe(self): m.aseert_foo_call() with self.assertRaisesRegex(AttributeError, msg): m.assrt_foo_call() + with self.assertRaisesRegex(AttributeError, msg): + m.called_once_with() + with self.assertRaisesRegex(AttributeError, msg): + m.called_once() + with self.assertRaisesRegex(AttributeError, msg): + m.has_calls() + + class Foo(object): + def called_once(self): pass + + def has_calls(self): pass + + m = Mock(spec=Foo) + m.called_once() + m.has_calls() + + m.called_once.assert_called_once() + m.has_calls.assert_called_once() + m = Mock(unsafe=True) m.assert_foo_call() m.assret_foo_call() m.asert_foo_call() m.aseert_foo_call() m.assrt_foo_call() + m.called_once() + m.called_once_with() + m.has_calls() # gh-100739 def test_mock_safe_with_spec(self): class Foo(object): - def assert_bar(self): - pass + def assert_bar(self): pass - def assertSome(self): - pass + def assertSome(self): pass m = Mock(spec=Foo) m.assert_bar() @@ -2231,7 +2363,7 @@ def test_misspelled_arguments(self): class Foo(): one = 'one' # patch, patch.object and create_autospec need to check for misspelled - # arguments explicitly and throw a RuntimError if found. + # arguments explicitly and throw a RuntimeError if found. with self.assertRaises(RuntimeError): with patch(f'{__name__}.Something.meth', autospect=True): pass with self.assertRaises(RuntimeError): @@ -2273,6 +2405,26 @@ class Foo(): f'{__name__}.Typos', autospect=True, set_spec=True, auto_spec=True): pass + def test_property_not_called_with_spec_mock(self): + obj = SomethingElse() + self.assertIsNone(obj._instance, msg='before mock') + mock = Mock(spec=obj) + self.assertIsNone(obj._instance, msg='after mock') + self.assertEqual('object', obj.instance) + + def test_decorated_async_methods_with_spec_mock(self): + class Foo(): + @classmethod + async def class_method(cls): + pass + @staticmethod + async def static_method(): + pass + async def method(self): + pass + mock = Mock(spec=Foo) + for m in (mock.method, mock.class_method, mock.static_method): + self.assertIsInstance(m, AsyncMock) if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/test/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py similarity index 89% rename from Lib/unittest/test/testmock/testpatch.py rename to Lib/test/test_unittest/testmock/testpatch.py index 8ab63a1317d..87424e07e4e 100644 --- a/Lib/unittest/test/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -7,9 +7,11 @@ from collections import OrderedDict import unittest -from unittest.test.testmock import support -from unittest.test.testmock.support import SomeClass, is_instance +import test +from test.test_unittest.testmock import support +from test.test_unittest.testmock.support import SomeClass, is_instance +from test.support.import_helper import DirsOnSysPath from test.test_importlib.util import uncache from unittest.mock import ( NonCallableMock, CallableMixin, sentinel, @@ -669,7 +671,7 @@ def test_patch_dict_decorator_resolution(self): # the new dictionary during function call original = support.target.copy() - @patch.dict('unittest.test.testmock.support.target', {'bar': 'BAR'}) + @patch.dict('test.test_unittest.testmock.support.target', {'bar': 'BAR'}) def test(): self.assertEqual(support.target, {'foo': 'BAZ', 'bar': 'BAR'}) @@ -743,6 +745,54 @@ def test_stop_idempotent(self): self.assertIsNone(patcher.stop()) + def test_exit_idempotent(self): + patcher = patch(foo_name, 'bar', 3) + with patcher: + patcher.stop() + + + def test_second_start_failure(self): + patcher = patch(foo_name, 'bar', 3) + patcher.start() + try: + self.assertRaises(RuntimeError, patcher.start) + finally: + patcher.stop() + + + def test_second_enter_failure(self): + patcher = patch(foo_name, 'bar', 3) + with patcher: + self.assertRaises(RuntimeError, patcher.start) + + + def test_second_start_after_stop(self): + patcher = patch(foo_name, 'bar', 3) + patcher.start() + patcher.stop() + patcher.start() + patcher.stop() + + + def test_property_setters(self): + mock_object = Mock() + mock_bar = mock_object.bar + patcher = patch.object(mock_object, 'bar', 'x') + with patcher: + self.assertEqual(patcher.is_local, False) + self.assertIs(patcher.target, mock_object) + self.assertEqual(patcher.temp_original, mock_bar) + patcher.is_local = True + patcher.target = mock_bar + patcher.temp_original = mock_object + self.assertEqual(patcher.is_local, True) + self.assertIs(patcher.target, mock_bar) + self.assertEqual(patcher.temp_original, mock_object) + # if changes are left intact, they may lead to disruption as shown below (it might be what someone needs though) + self.assertEqual(mock_bar.bar, mock_object) + self.assertEqual(mock_object.bar, 'x') + + def test_patchobject_start_stop(self): original = something patcher = patch.object(PTModule, 'something', 'foo') @@ -996,6 +1046,36 @@ def test_autospec_classmethod(self): method.assert_called_once_with() + def test_autospec_staticmethod_signature(self): + # Patched methods which are decorated with @staticmethod should have the same signature + class Foo: + @staticmethod + def static_method(a, b=10, *, c): pass + + Foo.static_method(1, 2, c=3) + + with patch.object(Foo, 'static_method', autospec=True) as method: + method(1, 2, c=3) + self.assertRaises(TypeError, method) + self.assertRaises(TypeError, method, 1) + self.assertRaises(TypeError, method, 1, 2, 3, c=4) + + + def test_autospec_classmethod_signature(self): + # Patched methods which are decorated with @classmethod should have the same signature + class Foo: + @classmethod + def class_method(cls, a, b=10, *, c): pass + + Foo.class_method(1, 2, c=3) + + with patch.object(Foo, 'class_method', autospec=True) as method: + method(1, 2, c=3) + self.assertRaises(TypeError, method) + self.assertRaises(TypeError, method, 1) + self.assertRaises(TypeError, method, 1, 2, 3, c=4) + + def test_autospec_with_new(self): patcher = patch('%s.function' % __name__, new=3, autospec=True) self.assertRaises(TypeError, patcher.start) @@ -1066,7 +1146,7 @@ def test_new_callable_patch(self): self.assertIsNot(m1, m2) for mock in m1, m2: - self.assertNotCallable(m1) + self.assertNotCallable(mock) def test_new_callable_patch_object(self): @@ -1079,7 +1159,7 @@ def test_new_callable_patch_object(self): self.assertIsNot(m1, m2) for mock in m1, m2: - self.assertNotCallable(m1) + self.assertNotCallable(mock) def test_new_callable_keyword_arguments(self): @@ -1614,7 +1694,7 @@ def test_patch_with_spec_mock_repr(self): def test_patch_nested_autospec_repr(self): - with patch('unittest.test.testmock.support', autospec=True) as m: + with patch('test.test_unittest.testmock.support', autospec=True) as m: self.assertIn(" name='support.SomeClass.wibble()'", repr(m.SomeClass.wibble())) self.assertIn(" name='support.SomeClass().wibble()'", @@ -1698,6 +1778,71 @@ def test(mock): 'exception traceback not propagated') + def test_name_resolution_import_rebinding(self): + # Currently mock.patch uses pkgutil.resolve_name(), but repeat + # similar tests just for the case. + # The same data is also used for testing import in test_import and + # pkgutil.resolve_name() in test_pkgutil. + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + + def test_create_and_specs(self): for kwarg in ('spec', 'spec_set', 'autospec'): p = patch('%s.doesnotexist' % __name__, create=True, @@ -1867,6 +2012,7 @@ def test_patch_and_patch_dict_stopall(self): self.assertEqual(dic2, origdic2) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_attrs(self): def foo(x=0): """TEST""" @@ -1882,7 +2028,7 @@ def foo(x=0): with patch.object(foo, '__module__', "testpatch2"): self.assertEqual(foo.__module__, "testpatch2") - self.assertEqual(foo.__module__, 'unittest.test.testmock.testpatch') + self.assertEqual(foo.__module__, __name__) with patch.object(foo, '__annotations__', dict([('s', 1, )])): self.assertEqual(foo.__annotations__, dict([('s', 1, )])) @@ -1917,16 +2063,16 @@ def test_dotted_but_module_not_loaded(self): # This exercises the AttributeError branch of _dot_lookup. # make sure it's there - import unittest.test.testmock.support + import test.test_unittest.testmock.support # now make sure it's not: with patch.dict('sys.modules'): - del sys.modules['unittest.test.testmock.support'] - del sys.modules['unittest.test.testmock'] - del sys.modules['unittest.test'] - del sys.modules['unittest'] + del sys.modules['test.test_unittest.testmock.support'] + del sys.modules['test.test_unittest.testmock'] + del sys.modules['test.test_unittest'] + del sys.modules['test'] # now make sure we can patch based on a dotted path: - @patch('unittest.test.testmock.support.X') + @patch('test.test_unittest.testmock.support.X') def test(mock): pass test() @@ -1943,11 +2089,18 @@ class Foo: def test_cant_set_kwargs_when_passing_a_mock(self): - @patch('unittest.test.testmock.support.X', new=object(), x=1) + @patch('test.test_unittest.testmock.support.X', new=object(), x=1) def test(): pass with self.assertRaises(TypeError): test() + def test_patch_proxy_object(self): + @patch("test.test_unittest.testmock.support.g", new_callable=MagicMock()) + def test(_): + pass + + test() + if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/test/testmock/testsealable.py b/Lib/test/test_unittest/testmock/testsealable.py similarity index 96% rename from Lib/unittest/test/testmock/testsealable.py rename to Lib/test/test_unittest/testmock/testsealable.py index daba2b49b46..8bf98cfa562 100644 --- a/Lib/unittest/test/testmock/testsealable.py +++ b/Lib/test/test_unittest/testmock/testsealable.py @@ -175,15 +175,12 @@ def test_seal_with_autospec(self): # https://bugs.python.org/issue45156 class Foo: foo = 0 - def bar1(self): - return 1 - def bar2(self): - return 2 + def bar1(self): pass + def bar2(self): pass class Baz: baz = 3 - def ban(self): - return 4 + def ban(self): pass for spec_set in (True, False): with self.subTest(spec_set=spec_set): @@ -200,6 +197,9 @@ def ban(self): self.assertIsInstance(foo.Baz.baz, mock.NonCallableMagicMock) self.assertIsInstance(foo.Baz.ban, mock.MagicMock) + # see gh-91803 + self.assertIsInstance(foo.bar2(), mock.MagicMock) + self.assertEqual(foo.bar1(), 'a') foo.bar1.return_value = 'new_a' self.assertEqual(foo.bar1(), 'new_a') @@ -212,7 +212,7 @@ def ban(self): with self.assertRaises(AttributeError): foo.bar = 1 with self.assertRaises(AttributeError): - foo.bar2() + foo.bar2().x foo.bar2.return_value = 'bar2' self.assertEqual(foo.bar2(), 'bar2') diff --git a/Lib/unittest/test/testmock/testsentinel.py b/Lib/test/test_unittest/testmock/testsentinel.py similarity index 100% rename from Lib/unittest/test/testmock/testsentinel.py rename to Lib/test/test_unittest/testmock/testsentinel.py diff --git a/Lib/test/test_unittest/testmock/testthreadingmock.py b/Lib/test/test_unittest/testmock/testthreadingmock.py new file mode 100644 index 00000000000..a02b532ed44 --- /dev/null +++ b/Lib/test/test_unittest/testmock/testthreadingmock.py @@ -0,0 +1,201 @@ +import time +import unittest +import concurrent.futures + +from test.support import threading_helper +from unittest.mock import patch, ThreadingMock + + +threading_helper.requires_working_threading(module=True) + +VERY_SHORT_TIMEOUT = 0.1 + + +class Something: + def method_1(self): + pass # pragma: no cover + + def method_2(self): + pass # pragma: no cover + + +class TestThreadingMock(unittest.TestCase): + def _call_after_delay(self, func, /, *args, **kwargs): + time.sleep(kwargs.pop("delay")) + func(*args, **kwargs) + + def setUp(self): + self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=5) + + def tearDown(self): + self._executor.shutdown() + + def run_async(self, func, /, *args, delay=0, **kwargs): + self._executor.submit( + self._call_after_delay, func, *args, **kwargs, delay=delay + ) + + def _make_mock(self, *args, **kwargs): + return ThreadingMock(*args, **kwargs) + + def test_spec(self): + waitable_mock = self._make_mock(spec=Something) + + with patch(f"{__name__}.Something", waitable_mock) as m: + something = m() + + self.assertIsInstance(something.method_1, ThreadingMock) + self.assertIsInstance(something.method_1().method_2(), ThreadingMock) + + with self.assertRaises(AttributeError): + m.test + + def test_side_effect(self): + waitable_mock = self._make_mock() + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + something.method_1.side_effect = [1] + + self.assertEqual(something.method_1(), 1) + + def test_instance_check(self): + waitable_mock = self._make_mock() + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + + self.assertIsInstance(something.method_1, ThreadingMock) + self.assertIsInstance(something.method_1().method_2(), ThreadingMock) + + def test_dynamic_child_mocks_are_threading_mocks(self): + waitable_mock = self._make_mock() + self.assertIsInstance(waitable_mock.child, ThreadingMock) + + def test_dynamic_child_mocks_inherit_timeout(self): + mock1 = self._make_mock() + self.assertIs(mock1._mock_wait_timeout, None) + mock2 = self._make_mock(timeout=2) + self.assertEqual(mock2._mock_wait_timeout, 2) + mock3 = self._make_mock(timeout=3) + self.assertEqual(mock3._mock_wait_timeout, 3) + + self.assertIs(mock1.child._mock_wait_timeout, None) + self.assertEqual(mock2.child._mock_wait_timeout, 2) + self.assertEqual(mock3.child._mock_wait_timeout, 3) + + self.assertEqual(mock2.really().__mul__().complex._mock_wait_timeout, 2) + + def test_no_name_clash(self): + waitable_mock = self._make_mock() + waitable_mock._event = "myevent" + waitable_mock.event = "myevent" + waitable_mock.timeout = "mytimeout" + waitable_mock("works") + waitable_mock.wait_until_called() + waitable_mock.wait_until_any_call_with("works") + + def test_patch(self): + waitable_mock = self._make_mock(spec=Something) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + something.method_1() + something.method_1.wait_until_called() + + def test_wait_already_called_success(self): + waitable_mock = self._make_mock(spec=Something) + waitable_mock.method_1() + waitable_mock.method_1.wait_until_called() + waitable_mock.method_1.wait_until_any_call_with() + waitable_mock.method_1.assert_called() + + def test_wait_until_called_success(self): + waitable_mock = self._make_mock(spec=Something) + self.run_async(waitable_mock.method_1, delay=VERY_SHORT_TIMEOUT) + waitable_mock.method_1.wait_until_called() + + def test_wait_until_called_method_timeout(self): + waitable_mock = self._make_mock(spec=Something) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_called(timeout=VERY_SHORT_TIMEOUT) + + def test_wait_until_called_instance_timeout(self): + waitable_mock = self._make_mock(spec=Something, timeout=VERY_SHORT_TIMEOUT) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_called() + + def test_wait_until_called_global_timeout(self): + with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): + ThreadingMock.DEFAULT_TIMEOUT = VERY_SHORT_TIMEOUT + waitable_mock = self._make_mock(spec=Something) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_called() + + def test_wait_until_any_call_with_success(self): + waitable_mock = self._make_mock() + self.run_async(waitable_mock, delay=VERY_SHORT_TIMEOUT) + waitable_mock.wait_until_any_call_with() + + def test_wait_until_any_call_with_instance_timeout(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + with self.assertRaises(AssertionError): + waitable_mock.wait_until_any_call_with() + + def test_wait_until_any_call_global_timeout(self): + with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): + ThreadingMock.DEFAULT_TIMEOUT = VERY_SHORT_TIMEOUT + waitable_mock = self._make_mock() + with self.assertRaises(AssertionError): + waitable_mock.wait_until_any_call_with() + + def test_wait_until_any_call_positional(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + waitable_mock.method_1(1, 2, 3) + waitable_mock.method_1.wait_until_any_call_with(1, 2, 3) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with(2, 3, 1) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with() + + def test_wait_until_any_call_kw(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + waitable_mock.method_1(a=1, b=2) + waitable_mock.method_1.wait_until_any_call_with(a=1, b=2) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with(a=2, b=1) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with() + + def test_magic_methods_success(self): + waitable_mock = self._make_mock() + str(waitable_mock) + waitable_mock.__str__.wait_until_called() + waitable_mock.__str__.assert_called() + + def test_reset_mock_resets_wait(self): + m = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + + with self.assertRaises(AssertionError): + m.wait_until_called() + with self.assertRaises(AssertionError): + m.wait_until_any_call_with() + m() + m.wait_until_called() + m.wait_until_any_call_with() + m.assert_called_once() + + m.reset_mock() + + with self.assertRaises(AssertionError): + m.wait_until_called() + with self.assertRaises(AssertionError): + m.wait_until_any_call_with() + m() + m.wait_until_called() + m.wait_until_any_call_with() + m.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/testmock/testwith.py b/Lib/test/test_unittest/testmock/testwith.py similarity index 97% rename from Lib/unittest/test/testmock/testwith.py rename to Lib/test/test_unittest/testmock/testwith.py index c74d49a63c8..56cb16394fa 100644 --- a/Lib/unittest/test/testmock/testwith.py +++ b/Lib/test/test_unittest/testmock/testwith.py @@ -1,7 +1,7 @@ import unittest from warnings import catch_warnings -from unittest.test.testmock.support import is_instance +from test.test_unittest.testmock.support import is_instance from unittest.mock import MagicMock, Mock, patch, sentinel, mock_open, call @@ -158,7 +158,7 @@ def test_mock_open_context_manager(self): f.read() expected_calls = [call('foo'), call().__enter__(), call().read(), - call().__exit__(None, None, None)] + call().__exit__(None, None, None), call().close()] self.assertEqual(mock.mock_calls, expected_calls) self.assertIs(f, handle) @@ -172,9 +172,9 @@ def test_mock_open_context_manager_multiple_times(self): expected_calls = [ call('foo'), call().__enter__(), call().read(), - call().__exit__(None, None, None), + call().__exit__(None, None, None), call().close(), call('bar'), call().__enter__(), call().read(), - call().__exit__(None, None, None)] + call().__exit__(None, None, None), call().close()] self.assertEqual(mock.mock_calls, expected_calls) def test_explicit_mock(self): diff --git a/Lib/test/test_univnewlines.py b/Lib/test/test_univnewlines.py index b9054918780..ed2e0970bac 100644 --- a/Lib/test/test_univnewlines.py +++ b/Lib/test/test_univnewlines.py @@ -4,7 +4,6 @@ import unittest import os import sys -from test import support from test.support import os_helper diff --git a/Lib/test/test_unpack.py b/Lib/test/test_unpack.py index f5ca1d455b5..305da05b7ce 100644 --- a/Lib/test/test_unpack.py +++ b/Lib/test/test_unpack.py @@ -18,6 +18,13 @@ >>> a == 4 and b == 5 and c == 6 True +Unpack dict + + >>> d = {4: 'four', 5: 'five', 6: 'six'} + >>> a, b, c = d + >>> a == 4 and b == 5 and c == 6 + True + Unpack implied tuple >>> a, b, c = 7, 8, 9 @@ -66,14 +73,14 @@ >>> a, b = t Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 2) + ValueError: too many values to unpack (expected 2, got 3) Unpacking tuple of wrong size >>> a, b = l Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 2) + ValueError: too many values to unpack (expected 2, got 3) Unpacking sequence too short @@ -140,14 +147,59 @@ >>> () = [42] Traceback (most recent call last): ... - ValueError: too many values to unpack (expected 0) + ValueError: too many values to unpack (expected 0, got 1) + +Unpacking a larger iterable should raise ValuleError, but it +should not entirely consume the iterable + >>> it = iter(range(100)) + >>> x, y, z = it + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) + >>> next(it) + 4 + +Unpacking unbalanced dict + + >>> d = {4: 'four', 5: 'five', 6: 'six', 7: 'seven'} + >>> a, b, c = d + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3, got 4) + +Ensure that custom `__len__()` is NOT called when showing the error message + + >>> class LengthTooLong: + ... def __len__(self): + ... return 5 + ... def __getitem__(self, i): + ... return i*2 + ... + >>> x, y, z = LengthTooLong() + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) + +For evil cases like these as well, no actual count to be shown + + >>> class BadLength: + ... def __len__(self): + ... return 1 + ... def __getitem__(self, i): + ... return i*2 + ... + >>> x, y, z = BadLength() + Traceback (most recent call last): + ... + ValueError: too many values to unpack (expected 3) """ __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): - tests.addTest(doctest.DocTestSuite()) + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON + tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # XXX: RUSTPYTHON return tests @@ -162,7 +214,7 @@ def test_extended_oparg_not_ignored(self): ns = {} exec(code, ns) unpack_400 = ns["unpack_400"] - # Warm up the the function for quickening (PEP 659) + # Warm up the function for quickening (PEP 659) for _ in range(30): y = unpack_400(range(400)) self.assertEqual(y, 399) diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py new file mode 100644 index 00000000000..13b789f52dc --- /dev/null +++ b/Lib/test/test_unpack_ex.py @@ -0,0 +1,413 @@ +# Tests for extended unpacking, starred expressions. + +import doctest +import unittest + + +doctests = """ + +Unpack tuple + + >>> t = (1, 2, 3) + >>> a, *b, c = t + >>> a == 1 and b == [2] and c == 3 + True + +Unpack list + + >>> l = [4, 5, 6] + >>> a, *b = l + >>> a == 4 and b == [5, 6] + True + +Unpack implied tuple + + >>> *a, = 7, 8, 9 + >>> a == [7, 8, 9] + True + +Unpack nested implied tuple + + >>> [*[*a]] = [[7,8,9]] + >>> a == [[7,8,9]] + True + +Unpack string... fun! + + >>> a, *b = 'one' + >>> a == 'o' and b == ['n', 'e'] + True + +Unpack long sequence + + >>> a, b, c, *d, e, f, g = range(10) + >>> (a, b, c, d, e, f, g) == (0, 1, 2, [3, 4, 5, 6], 7, 8, 9) + True + +Unpack short sequence + + >>> a, *b, c = (1, 2) + >>> a == 1 and c == 2 and b == [] + True + +Unpack generic sequence + + >>> class Seq: + ... def __getitem__(self, i): + ... if i >= 0 and i < 3: return i + ... raise IndexError + ... + >>> a, *b = Seq() + >>> a == 0 and b == [1, 2] + True + +Unpack in for statement + + >>> for a, *b, c in [(1,2,3), (4,5,6,7)]: + ... print(a, b, c) + ... + 1 [2] 3 + 4 [5, 6] 7 + +Unpack in list + + >>> [a, *b, c] = range(5) + >>> a == 0 and b == [1, 2, 3] and c == 4 + True + +Multiple targets + + >>> a, *b, c = *d, e = range(5) + >>> a == 0 and b == [1, 2, 3] and c == 4 and d == [0, 1, 2, 3] and e == 4 + True + +Assignment unpacking + + >>> a, b, *c = range(5) + >>> a, b, c + (0, 1, [2, 3, 4]) + >>> *a, b, c = a, b, *c + >>> a, b, c + ([0, 1, 2], 3, 4) + +Set display element unpacking + + >>> a = [1, 2, 3] + >>> sorted({1, *a, 0, 4}) + [0, 1, 2, 3, 4] + + >>> {1, *1, 0, 4} + Traceback (most recent call last): + ... + TypeError: 'int' object is not iterable + +Dict display element unpacking + + >>> kwds = {'z': 0, 'w': 12} + >>> sorted({'x': 1, 'y': 2, **kwds}.items()) + [('w', 12), ('x', 1), ('y', 2), ('z', 0)] + + >>> sorted({**{'x': 1}, 'y': 2, **{'z': 3}}.items()) + [('x', 1), ('y', 2), ('z', 3)] + + >>> sorted({**{'x': 1}, 'y': 2, **{'x': 3}}.items()) + [('x', 3), ('y', 2)] + + >>> sorted({**{'x': 1}, **{'x': 3}, 'x': 4}.items()) + [('x', 4)] + + >>> {**{}} + {} + + >>> a = {} + >>> {**a}[0] = 1 + >>> a + {} + + >>> {**1} + Traceback (most recent call last): + ... + TypeError: 'int' object is not a mapping + + >>> {**[]} + Traceback (most recent call last): + ... + TypeError: 'list' object is not a mapping + + >>> len(eval("{" + ", ".join("**{{{}: {}}}".format(i, i) + ... for i in range(1000)) + "}")) + 1000 + + >>> {0:1, **{0:2}, 0:3, 0:4} + {0: 4} + +List comprehension element unpacking + + >>> a, b, c = [0, 1, 2], 3, 4 + >>> [*a, b, c] + [0, 1, 2, 3, 4] + + >>> l = [a, (3, 4), {5}, {6: None}, (i for i in range(7, 10))] + >>> [*item for item in l] + Traceback (most recent call last): + ... + SyntaxError: iterable unpacking cannot be used in comprehension + + >>> [*[0, 1] for i in range(10)] + Traceback (most recent call last): + ... + SyntaxError: iterable unpacking cannot be used in comprehension + + >>> [*'a' for i in range(10)] + Traceback (most recent call last): + ... + SyntaxError: iterable unpacking cannot be used in comprehension + + >>> [*[] for i in range(10)] + Traceback (most recent call last): + ... + SyntaxError: iterable unpacking cannot be used in comprehension + + >>> {**{} for a in [1]} # TODO: RUSTPYTHON # doctest: +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: dict unpacking cannot be used in dict comprehension + +# Pegen is better here. +# Generator expression in function arguments + +# >>> list(*x for x in (range(5) for i in range(3))) +# Traceback (most recent call last): +# ... +# list(*x for x in (range(5) for i in range(3))) +# ^ +# SyntaxError: invalid syntax + + >>> dict(**x for x in [{1:2}]) + Traceback (most recent call last): + ... + dict(**x for x in [{1:2}]) + ^ + SyntaxError: invalid syntax + +Iterable argument unpacking + + >>> print(*[1], *[2], 3) + 1 2 3 + +Make sure that they don't corrupt the passed-in dicts. + + >>> def f(x, y): + ... print(x, y) + ... + >>> original_dict = {'x': 1} + >>> f(**original_dict, y=2) + 1 2 + >>> original_dict + {'x': 1} + +Now for some failures + +Make sure the raised errors are right for keyword argument unpackings + + >>> from collections.abc import MutableMapping + >>> class CrazyDict(MutableMapping): + ... def __init__(self): + ... self.d = {} + ... + ... def __iter__(self): + ... for x in self.d.__iter__(): + ... if x == 'c': + ... self.d['z'] = 10 + ... yield x + ... + ... def __getitem__(self, k): + ... return self.d[k] + ... + ... def __len__(self): + ... return len(self.d) + ... + ... def __setitem__(self, k, v): + ... self.d[k] = v + ... + ... def __delitem__(self, k): + ... del self.d[k] + ... + >>> d = CrazyDict() + >>> d.d = {chr(ord('a') + x): x for x in range(5)} + >>> e = {**d} + Traceback (most recent call last): + ... + RuntimeError: dictionary changed size during iteration + + >>> d.d = {chr(ord('a') + x): x for x in range(5)} + >>> def f(**kwargs): print(kwargs) + >>> f(**d) + Traceback (most recent call last): + ... + RuntimeError: dictionary changed size during iteration + +Overridden parameters + + >>> f(x=5, **{'x': 3}, y=2) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' + + >>> f(**{'x': 3}, x=5, y=2) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' + + >>> f(**{'x': 3}, **{'x': 5}, y=2) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' + + >>> f(x=5, **{'x': 3}, **{'x': 2}) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' + + >>> f(**{1: 3}, **{1: 5}) + Traceback (most recent call last): + ... + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument '1' + +Unpacking non-sequence + + >>> a, *b = 7 + Traceback (most recent call last): + ... + TypeError: cannot unpack non-iterable int object + +Unpacking sequence too short + + >>> a, *b, c, d, e = Seq() + Traceback (most recent call last): + ... + ValueError: not enough values to unpack (expected at least 4, got 3) + +Unpacking sequence too short and target appears last + + >>> a, b, c, d, *e = Seq() + Traceback (most recent call last): + ... + ValueError: not enough values to unpack (expected at least 4, got 3) + +Unpacking a sequence where the test for too long raises a different kind of +error + + >>> class BozoError(Exception): + ... pass + ... + >>> class BadSeq: + ... def __getitem__(self, i): + ... if i >= 0 and i < 3: + ... return i + ... elif i == 3: + ... raise BozoError + ... else: + ... raise IndexError + ... + +Trigger code while not expecting an IndexError (unpack sequence too long, wrong +error) + + >>> a, *b, c, d, e = BadSeq() + Traceback (most recent call last): + ... + test.test_unpack_ex.BozoError + +Now some general starred expressions (all fail). + + >>> a, *b, c, *d, e = range(10) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: multiple starred expressions in assignment + + >>> [*b, *c] = range(10) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: multiple starred expressions in assignment + + >>> a,*b,*c,*d = range(4) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: multiple starred expressions in assignment + + >>> *a = range(10) # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: starred assignment target must be in a list or tuple + + >>> *a # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: can't use starred expression here + + >>> *1 # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: can't use starred expression here + + >>> x = *a # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: can't use starred expression here + + >>> (*x),y = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + + >>> (((*x))),y = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + + >>> z,(*x),y = 1, 2, 4 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + + >>> z,(*x) = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + + >>> ((*x),y) = 1, 2 # TODO: RUSTPYTHON # doctest:+ELLIPSIS +EXPECTED_FAILURE + Traceback (most recent call last): + ... + SyntaxError: cannot use starred expression here + +Some size constraints (all fail.) + + >>> s = ", ".join("a%d" % i for i in range(1<<8)) + ", *rest = range(1<<8 + 1)" + >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: too many expressions in star-unpacking assignment + + >>> s = ", ".join("a%d" % i for i in range(1<<8 + 1)) + ", *rest = range(1<<8 + 2)" + >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS + Traceback (most recent call last): + ... + SyntaxError: too many expressions in star-unpacking assignment + +(there is an additional limit, on the number of expressions after the +'*rest', but it's 1<<24 and testing it takes too much memory.) + +""" + +__test__ = {'doctests' : doctests} + + +def load_tests(loader, tests, pattern): + from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON + tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # XXX: RUSTPYTHON + return tests + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py new file mode 100644 index 00000000000..35e4652a87b --- /dev/null +++ b/Lib/test/test_unparse.py @@ -0,0 +1,1066 @@ +"""Tests for ast.unparse.""" + +import unittest +import test.support +import pathlib +import random +import tokenize +import warnings +import ast +from test.support.ast_helper import ASTTestMixin + + +def read_pyfile(filename): + """Read and return the contents of a Python source file (as a + string), taking into account the file encoding.""" + with tokenize.open(filename) as stream: + return stream.read() + + +for_else = """\ +def f(): + for x in range(10): + break + else: + y = 2 + z = 3 +""" + +while_else = """\ +def g(): + while True: + break + else: + y = 2 + z = 3 +""" + +relative_import = """\ +from . import fred +from .. import barney +from .australia import shrimp as prawns +""" + +nonlocal_ex = """\ +def f(): + x = 1 + def g(): + nonlocal x + x = 2 + y = 7 + def h(): + nonlocal x, y +""" + +# also acts as test for 'except ... as ...' +raise_from = """\ +try: + 1 / 0 +except ZeroDivisionError as e: + raise ArithmeticError from e +""" + +class_decorator = """\ +@f1(arg) +@f2 +class Foo: pass +""" + +elif1 = """\ +if cond1: + suite1 +elif cond2: + suite2 +else: + suite3 +""" + +elif2 = """\ +if cond1: + suite1 +elif cond2: + suite2 +""" + +try_except_finally = """\ +try: + suite1 +except ex1: + suite2 +except ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +try_except_star_finally = """\ +try: + suite1 +except* ex1: + suite2 +except* ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +with_simple = """\ +with f(): + suite1 +""" + +with_as = """\ +with f() as x: + suite1 +""" + +with_two_items = """\ +with f() as x, g() as y: + suite1 +""" + +docstring_prefixes = ( + "", + "class foo:\n ", + "def foo():\n ", + "async def foo():\n ", +) + +class ASTTestCase(ASTTestMixin, unittest.TestCase): + def check_ast_roundtrip(self, code1, **kwargs): + with self.subTest(code1=code1, ast_parse_kwargs=kwargs): + ast1 = ast.parse(code1, **kwargs) + code2 = ast.unparse(ast1) + ast2 = ast.parse(code2, **kwargs) + self.assertASTEqual(ast1, ast2) + + def check_invalid(self, node, raises=ValueError): + with self.subTest(node=node): + self.assertRaises(raises, ast.unparse, node) + + def get_source(self, code1, code2=None, **kwargs): + code2 = code2 or code1 + code1 = ast.unparse(ast.parse(code1, **kwargs)) + return code1, code2 + + def check_src_roundtrip(self, code1, code2=None, **kwargs): + code1, code2 = self.get_source(code1, code2, **kwargs) + with self.subTest(code1=code1, code2=code2): + self.assertEqual(code2, code1) + + def check_src_dont_roundtrip(self, code1, code2=None): + code1, code2 = self.get_source(code1, code2) + with self.subTest(code1=code1, code2=code2): + self.assertNotEqual(code2, code1) + +class UnparseTestCase(ASTTestCase): + # Tests for specific bugs found in earlier versions of unparse + + def test_fstrings(self): + self.check_ast_roundtrip("f'a'") + self.check_ast_roundtrip("f'{{}}'") + self.check_ast_roundtrip("f'{{5}}'") + self.check_ast_roundtrip("f'{{5}}5'") + self.check_ast_roundtrip("f'X{{}}X'") + self.check_ast_roundtrip("f'{a}'") + self.check_ast_roundtrip("f'{ {1:2}}'") + self.check_ast_roundtrip("f'a{a}a'") + self.check_ast_roundtrip("f'a{a}{a}a'") + self.check_ast_roundtrip("f'a{a}a{a}a'") + self.check_ast_roundtrip("f'{a!r}x{a!s}12{{}}{a!a}'") + self.check_ast_roundtrip("f'{a:10}'") + self.check_ast_roundtrip("f'{a:100_000{10}}'") + self.check_ast_roundtrip("f'{a!r:10}'") + self.check_ast_roundtrip("f'{a:a{b}10}'") + self.check_ast_roundtrip( + "f'a{b}{c!s}{d!r}{e!a}{f:a}{g:a{b}}{h!s:a}" + "{j!s:{a}b}{k!s:a{b}c}{l!a:{b}c{d}}{x+y=}'" + ) + + def test_fstrings_special_chars(self): + # See issue 25180 + self.check_ast_roundtrip(r"""f'{f"{0}"*3}'""") + self.check_ast_roundtrip(r"""f'{f"{y}"*3}'""") + self.check_ast_roundtrip("""f''""") + self.check_ast_roundtrip('''f"""'end' "quote\\""""''') + + def test_fstrings_complicated(self): + # See issue 28002 + self.check_ast_roundtrip("""f'''{"'"}'''""") + self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') + self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-'single quote\\'\'\'\'''') + self.check_ast_roundtrip('f"""{\'\'\'\n\'\'\'}"""') + self.check_ast_roundtrip('f"""{g(\'\'\'\n\'\'\')}"""') + self.check_ast_roundtrip('''f"a\\r\\nb"''') + self.check_ast_roundtrip('''f"\\u2028{'x'}"''') + + def test_fstrings_pep701(self): + self.check_ast_roundtrip('f" something { my_dict["key"] } something else "') + self.check_ast_roundtrip('f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}"') + + def test_tstrings(self): + self.check_ast_roundtrip("t'foo'") + self.check_ast_roundtrip("t'foo {bar}'") + self.check_ast_roundtrip("t'foo {bar!s:.2f}'") + self.check_ast_roundtrip("t'{a + b}'") + self.check_ast_roundtrip("t'{a + b:x}'") + self.check_ast_roundtrip("t'{a + b!s}'") + self.check_ast_roundtrip("t'{ {a}}'") + self.check_ast_roundtrip("t'{ {a}=}'") + self.check_ast_roundtrip("t'{{a}}'") + self.check_ast_roundtrip("t''") + self.check_ast_roundtrip('t""') + self.check_ast_roundtrip("t'{(lambda x: x)}'") + self.check_ast_roundtrip("t'{t'{x}'}'") + + def test_tstring_with_nonsensical_str_field(self): + # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.Name(id="test1", ctx=ast.Load()), str="test2", conversion=-1 + ) + ] + ) + ), + "t'{test2}'", + ) + + def test_tstring_with_none_str_field(self): + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="test1"), str=None, conversion=-1)] + ) + ), + "t'{test1}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ + ast.Interpolation( + value=ast.Lambda( + args=ast.arguments(args=[ast.arg(arg="x")]), + body=ast.Name(id="x"), + ), + str=None, + conversion=-1, + ) + ] + ) + ), + "t'{(lambda x: x)}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + # `str` field kept here + [ast.Interpolation(value=ast.Name(id="x"), str="y", conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{y}'}"''', + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="x"), str=None, conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{x}'}"''', + ) + self.assertEqual( + ast.unparse(ast.TemplateStr( + [ast.Interpolation(value=ast.Constant(value="foo"), str=None, conversion=114)] + )), + '''t"{'foo'!r}"''', + ) + + def test_strings(self): + self.check_ast_roundtrip("u'foo'") + self.check_ast_roundtrip("r'foo'") + self.check_ast_roundtrip("b'foo'") + + def test_del_statement(self): + self.check_ast_roundtrip("del x, y, z") + + def test_shifts(self): + self.check_ast_roundtrip("45 << 2") + self.check_ast_roundtrip("13 >> 7") + + def test_for_else(self): + self.check_ast_roundtrip(for_else) + + def test_while_else(self): + self.check_ast_roundtrip(while_else) + + def test_unary_parens(self): + self.check_ast_roundtrip("(-1)**7") + self.check_ast_roundtrip("(-1.)**8") + self.check_ast_roundtrip("(-1j)**6") + self.check_ast_roundtrip("not True or False") + self.check_ast_roundtrip("True or not False") + + def test_integer_parens(self): + self.check_ast_roundtrip("3 .__abs__()") + + def test_huge_float(self): + self.check_ast_roundtrip("1e1000") + self.check_ast_roundtrip("-1e1000") + self.check_ast_roundtrip("1e1000j") + self.check_ast_roundtrip("-1e1000j") + + def test_nan(self): + self.assertASTEqual( + ast.parse(ast.unparse(ast.Constant(value=float('nan')))), + ast.parse('1e1000 - 1e1000') + ) + + def test_min_int(self): + self.check_ast_roundtrip(str(-(2 ** 31))) + self.check_ast_roundtrip(str(-(2 ** 63))) + + def test_imaginary_literals(self): + self.check_ast_roundtrip("7j") + self.check_ast_roundtrip("-7j") + self.check_ast_roundtrip("0j") + self.check_ast_roundtrip("-0j") + + def test_lambda_parentheses(self): + self.check_ast_roundtrip("(lambda: int)()") + + def test_chained_comparisons(self): + self.check_ast_roundtrip("1 < 4 <= 5") + self.check_ast_roundtrip("a is b is c is not d") + + def test_function_arguments(self): + self.check_ast_roundtrip("def f(): pass") + self.check_ast_roundtrip("def f(a): pass") + self.check_ast_roundtrip("def f(b = 2): pass") + self.check_ast_roundtrip("def f(a, b): pass") + self.check_ast_roundtrip("def f(a, b = 2): pass") + self.check_ast_roundtrip("def f(a = 5, b = 2): pass") + self.check_ast_roundtrip("def f(*, a = 1, b = 2): pass") + self.check_ast_roundtrip("def f(*, a = 1, b): pass") + self.check_ast_roundtrip("def f(*, a, b = 2): pass") + self.check_ast_roundtrip("def f(a, b = None, *, c, **kwds): pass") + self.check_ast_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass") + self.check_ast_roundtrip("def f(*args, **kwargs): pass") + + def test_relative_import(self): + self.check_ast_roundtrip(relative_import) + + def test_nonlocal(self): + self.check_ast_roundtrip(nonlocal_ex) + + def test_raise_from(self): + self.check_ast_roundtrip(raise_from) + + def test_bytes(self): + self.check_ast_roundtrip("b'123'") + + def test_annotations(self): + self.check_ast_roundtrip("def f(a : int): pass") + self.check_ast_roundtrip("def f(a: int = 5): pass") + self.check_ast_roundtrip("def f(*args: [int]): pass") + self.check_ast_roundtrip("def f(**kwargs: dict): pass") + self.check_ast_roundtrip("def f() -> None: pass") + + def test_set_literal(self): + self.check_ast_roundtrip("{'a', 'b', 'c'}") + + def test_empty_set(self): + self.assertASTEqual( + ast.parse(ast.unparse(ast.Set(elts=[]))), + ast.parse('{*()}') + ) + + def test_set_comprehension(self): + self.check_ast_roundtrip("{x for x in range(5)}") + + def test_dict_comprehension(self): + self.check_ast_roundtrip("{x: x*x for x in range(10)}") + + def test_class_decorators(self): + self.check_ast_roundtrip(class_decorator) + + def test_class_definition(self): + self.check_ast_roundtrip("class A(metaclass=type, *[], **{}): pass") + + def test_elifs(self): + self.check_ast_roundtrip(elif1) + self.check_ast_roundtrip(elif2) + + def test_try_except_finally(self): + self.check_ast_roundtrip(try_except_finally) + + def test_try_except_star_finally(self): + self.check_ast_roundtrip(try_except_star_finally) + + def test_starred_assignment(self): + self.check_ast_roundtrip("a, *b, c = seq") + self.check_ast_roundtrip("a, (*b, c) = seq") + self.check_ast_roundtrip("a, *b[0], c = seq") + self.check_ast_roundtrip("a, *(b, c) = seq") + + def test_with_simple(self): + self.check_ast_roundtrip(with_simple) + + def test_with_as(self): + self.check_ast_roundtrip(with_as) + + def test_with_two_items(self): + self.check_ast_roundtrip(with_two_items) + + def test_dict_unpacking_in_dict(self): + # See issue 26489 + self.check_ast_roundtrip(r"""{**{'y': 2}, 'x': 1}""") + self.check_ast_roundtrip(r"""{**{'y': 2}, **{'x': 1}}""") + + def test_slices(self): + self.check_ast_roundtrip("a[i]") + self.check_ast_roundtrip("a[i,]") + self.check_ast_roundtrip("a[i, j]") + # The AST for these next two both look like `a[(*a,)]` + self.check_ast_roundtrip("a[(*a,)]") + self.check_ast_roundtrip("a[*a]") + self.check_ast_roundtrip("a[b, *a]") + self.check_ast_roundtrip("a[*a, c]") + self.check_ast_roundtrip("a[b, *a, c]") + self.check_ast_roundtrip("a[*a, *a]") + self.check_ast_roundtrip("a[b, *a, *a]") + self.check_ast_roundtrip("a[*a, b, *a]") + self.check_ast_roundtrip("a[*a, *a, b]") + self.check_ast_roundtrip("a[b, *a, *a, c]") + self.check_ast_roundtrip("a[(a:=b)]") + self.check_ast_roundtrip("a[(a:=b,c)]") + self.check_ast_roundtrip("a[()]") + self.check_ast_roundtrip("a[i:j]") + self.check_ast_roundtrip("a[:j]") + self.check_ast_roundtrip("a[i:]") + self.check_ast_roundtrip("a[i:j:k]") + self.check_ast_roundtrip("a[:j:k]") + self.check_ast_roundtrip("a[i::k]") + self.check_ast_roundtrip("a[i:j,]") + self.check_ast_roundtrip("a[i:j, k]") + + def test_invalid_raise(self): + self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X", ctx=ast.Load()))) + + def test_invalid_fstring_value(self): + self.check_invalid( + ast.JoinedStr( + values=[ + ast.Name(id="test", ctx=ast.Load()), + ast.Constant(value="test") + ] + ) + ) + + def test_fstring_backslash(self): + # valid since Python 3.12 + self.assertEqual(ast.unparse( + ast.FormattedValue( + value=ast.Constant(value="\\\\"), + conversion=-1, + format_spec=None, + ) + ), "{'\\\\\\\\'}") + + def test_invalid_yield_from(self): + self.check_invalid(ast.YieldFrom(value=None)) + + def test_import_from_level_none(self): + tree = ast.ImportFrom(module='mod', names=[ast.alias(name='x')]) + self.assertEqual(ast.unparse(tree), "from mod import x") + tree = ast.ImportFrom(module='mod', names=[ast.alias(name='x')], level=None) + self.assertEqual(ast.unparse(tree), "from mod import x") + + def test_docstrings(self): + docstrings = ( + 'this ends with double quote"', + 'this includes a """triple quote"""', + '\r', + '\\r', + '\t', + '\\t', + '\n', + '\\n', + '\r\\r\t\\t\n\\n', + '""">>> content = \"\"\"blabla\"\"\" <<<"""', + r'foo\n\x00', + "' \\'\\'\\'\"\"\" \"\"\\'\\' \\'", + '🐍⛎𩸽üéş^\\\\X\\\\BB\N{LONG RIGHTWARDS SQUIGGLE ARROW}' + ) + for docstring in docstrings: + # check as Module docstrings for easy testing + self.check_ast_roundtrip(f"'''{docstring}'''") + + def test_constant_tuples(self): + locs = ast.fix_missing_locations + self.check_src_roundtrip( + locs(ast.Module([ast.Expr(ast.Constant(value=(1,)))])), "(1,)") + self.check_src_roundtrip( + locs(ast.Module([ast.Expr(ast.Constant(value=(1, 2, 3)))])), "(1, 2, 3)" + ) + + def test_function_type(self): + for function_type in ( + "() -> int", + "(int, int) -> int", + "(Callable[complex], More[Complex(call.to_typevar())]) -> None" + ): + self.check_ast_roundtrip(function_type, mode="func_type") + + def test_type_comments(self): + for statement in ( + "a = 5 # type:", + "a = 5 # type: int", + "a = 5 # type: int and more", + "def x(): # type: () -> None\n\tpass", + "def x(y): # type: (int) -> None and more\n\tpass", + "async def x(): # type: () -> None\n\tpass", + "async def x(y): # type: (int) -> None and more\n\tpass", + "for x in y: # type: int\n\tpass", + "async for x in y: # type: int\n\tpass", + "with x(): # type: int\n\tpass", + "async with x(): # type: int\n\tpass" + ): + self.check_ast_roundtrip(statement, type_comments=True) + + def test_type_ignore(self): + for statement in ( + "a = 5 # type: ignore", + "a = 5 # type: ignore and more", + "def x(): # type: ignore\n\tpass", + "def x(y): # type: ignore and more\n\tpass", + "async def x(): # type: ignore\n\tpass", + "async def x(y): # type: ignore and more\n\tpass", + "for x in y: # type: ignore\n\tpass", + "async for x in y: # type: ignore\n\tpass", + "with x(): # type: ignore\n\tpass", + "async with x(): # type: ignore\n\tpass" + ): + self.check_ast_roundtrip(statement, type_comments=True) + + def test_unparse_interactive_semicolons(self): + # gh-129598: Fix ast.unparse() when ast.Interactive contains multiple statements + self.check_src_roundtrip("i = 1; 'expr'; raise Exception", mode='single') + self.check_src_roundtrip("i: int = 1; j: float = 0; k += l", mode='single') + combinable = ( + "'expr'", + "(i := 1)", + "import foo", + "from foo import bar", + "i = 1", + "i += 1", + "i: int = 1", + "return i", + "pass", + "break", + "continue", + "del i", + "assert i", + "global i", + "nonlocal j", + "await i", + "yield i", + "yield from i", + "raise i", + "type t[T] = ...", + "i", + ) + for a in combinable: + for b in combinable: + self.check_src_roundtrip(f"{a}; {b}", mode='single') + + def test_unparse_interactive_integrity_1(self): + # rest of unparse_interactive_integrity tests just make sure mode='single' parse and unparse didn't break + self.check_src_roundtrip( + "if i:\n 'expr'\nelse:\n raise Exception", + "if i:\n 'expr'\nelse:\n raise Exception", + mode='single' + ) + self.check_src_roundtrip( + "@decorator1\n@decorator2\ndef func():\n 'docstring'\n i = 1; 'expr'; raise Exception", + '''@decorator1\n@decorator2\ndef func():\n """docstring"""\n i = 1\n 'expr'\n raise Exception''', + mode='single' + ) + self.check_src_roundtrip( + "@decorator1\n@decorator2\nclass cls:\n 'docstring'\n i = 1; 'expr'; raise Exception", + '''@decorator1\n@decorator2\nclass cls:\n """docstring"""\n i = 1\n 'expr'\n raise Exception''', + mode='single' + ) + + def test_unparse_interactive_integrity_2(self): + for statement in ( + "def x():\n pass", + "def x(y):\n pass", + "async def x():\n pass", + "async def x(y):\n pass", + "for x in y:\n pass", + "async for x in y:\n pass", + "with x():\n pass", + "async with x():\n pass", + "def f():\n pass", + "def f(a):\n pass", + "def f(b=2):\n pass", + "def f(a, b):\n pass", + "def f(a, b=2):\n pass", + "def f(a=5, b=2):\n pass", + "def f(*, a=1, b=2):\n pass", + "def f(*, a=1, b):\n pass", + "def f(*, a, b=2):\n pass", + "def f(a, b=None, *, c, **kwds):\n pass", + "def f(a=2, *args, c=5, d, **kwds):\n pass", + "def f(*args, **kwargs):\n pass", + "class cls:\n\n def f(self):\n pass", + "class cls:\n\n def f(self, a):\n pass", + "class cls:\n\n def f(self, b=2):\n pass", + "class cls:\n\n def f(self, a, b):\n pass", + "class cls:\n\n def f(self, a, b=2):\n pass", + "class cls:\n\n def f(self, a=5, b=2):\n pass", + "class cls:\n\n def f(self, *, a=1, b=2):\n pass", + "class cls:\n\n def f(self, *, a=1, b):\n pass", + "class cls:\n\n def f(self, *, a, b=2):\n pass", + "class cls:\n\n def f(self, a, b=None, *, c, **kwds):\n pass", + "class cls:\n\n def f(self, a=2, *args, c=5, d, **kwds):\n pass", + "class cls:\n\n def f(self, *args, **kwargs):\n pass", + ): + self.check_src_roundtrip(statement, mode='single') + + def test_unparse_interactive_integrity_3(self): + for statement in ( + "def x():", + "def x(y):", + "async def x():", + "async def x(y):", + "for x in y:", + "async for x in y:", + "with x():", + "async with x():", + "def f():", + "def f(a):", + "def f(b=2):", + "def f(a, b):", + "def f(a, b=2):", + "def f(a=5, b=2):", + "def f(*, a=1, b=2):", + "def f(*, a=1, b):", + "def f(*, a, b=2):", + "def f(a, b=None, *, c, **kwds):", + "def f(a=2, *args, c=5, d, **kwds):", + "def f(*args, **kwargs):", + ): + src = statement + '\n i=1;j=2' + out = statement + '\n i = 1\n j = 2' + + self.check_src_roundtrip(src, out, mode='single') + + +class CosmeticTestCase(ASTTestCase): + """Test if there are cosmetic issues caused by unnecessary additions""" + + def test_simple_expressions_parens(self): + self.check_src_roundtrip("(a := b)") + self.check_src_roundtrip("await x") + self.check_src_roundtrip("x if x else y") + self.check_src_roundtrip("lambda x: x") + self.check_src_roundtrip("1 + 1") + self.check_src_roundtrip("1 + 2 / 3") + self.check_src_roundtrip("(1 + 2) / 3") + self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2)") + self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2) ** 2") + self.check_src_roundtrip("~x") + self.check_src_roundtrip("x and y") + self.check_src_roundtrip("x and y and z") + self.check_src_roundtrip("x and (y and x)") + self.check_src_roundtrip("(x and y) and z") + self.check_src_roundtrip("(x ** y) ** z ** q") + self.check_src_roundtrip("x >> y") + self.check_src_roundtrip("x << y") + self.check_src_roundtrip("x >> y and x >> z") + self.check_src_roundtrip("x + y - z * q ^ t ** k") + self.check_src_roundtrip("P * V if P and V else n * R * T") + self.check_src_roundtrip("lambda P, V, n: P * V == n * R * T") + self.check_src_roundtrip("flag & (other | foo)") + self.check_src_roundtrip("not x == y") + self.check_src_roundtrip("x == (not y)") + self.check_src_roundtrip("yield x") + self.check_src_roundtrip("yield from x") + self.check_src_roundtrip("call((yield x))") + self.check_src_roundtrip("return x + (yield x)") + + def test_class_bases_and_keywords(self): + self.check_src_roundtrip("class X:\n pass") + self.check_src_roundtrip("class X(A):\n pass") + self.check_src_roundtrip("class X(A, B, C, D):\n pass") + self.check_src_roundtrip("class X(x=y):\n pass") + self.check_src_roundtrip("class X(metaclass=z):\n pass") + self.check_src_roundtrip("class X(x=y, z=d):\n pass") + self.check_src_roundtrip("class X(A, x=y):\n pass") + self.check_src_roundtrip("class X(A, **kw):\n pass") + self.check_src_roundtrip("class X(*args):\n pass") + self.check_src_roundtrip("class X(*args, **kwargs):\n pass") + + def test_fstrings(self): + self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') + self.check_src_roundtrip('''f\'-{f\'\'\'*{f"""+{f".{f'{x}'}."}+"""}*\'\'\'}-\'''') + self.check_src_roundtrip('''f\'-{f\'*{f\'\'\'+{f""".{f"{f'{x}'}"}."""}+\'\'\'}*\'}-\'''') + self.check_src_roundtrip('''f"\\u2028{'x'}"''') + self.check_src_roundtrip(r"f'{x}\n'") + self.check_src_roundtrip('''f"{'\\n'}\\n"''') + self.check_src_roundtrip('''f"{f'{x}\\n'}\\n"''') + + def test_docstrings(self): + docstrings = ( + '"""simple doc string"""', + '''"""A more complex one + with some newlines"""''', + '''"""Foo bar baz + + empty newline"""''', + '"""With some \t"""', + '"""Foo "bar" baz """', + '"""\\r"""', + '""""""', + '"""\'\'\'"""', + '"""\'\'\'\'\'\'"""', + '"""🐍⛎𩸽üéş^\\\\X\\\\BB⟿"""', + '"""end in single \'quote\'"""', + "'''end in double \"quote\"'''", + '"""almost end in double "quote"."""', + ) + + for prefix in docstring_prefixes: + for docstring in docstrings: + self.check_src_roundtrip(f"{prefix}{docstring}") + + def test_docstrings_negative_cases(self): + # Test some cases that involve strings in the children of the + # first node but aren't docstrings to make sure we don't have + # False positives. + docstrings_negative = ( + 'a = """false"""', + '"""false""" + """unless its optimized"""', + '1 + 1\n"""false"""', + 'f"""no, top level but f-fstring"""' + ) + for prefix in docstring_prefixes: + for negative in docstrings_negative: + # this cases should be result with single quote + # rather then triple quoted docstring + src = f"{prefix}{negative}" + self.check_ast_roundtrip(src) + self.check_src_dont_roundtrip(src) + + def test_unary_op_factor(self): + for prefix in ("+", "-", "~"): + self.check_src_roundtrip(f"{prefix}1") + for prefix in ("not",): + self.check_src_roundtrip(f"{prefix} 1") + + def test_slices(self): + self.check_src_roundtrip("a[()]") + self.check_src_roundtrip("a[1]") + self.check_src_roundtrip("a[1, 2]") + # Note that `a[*a]`, `a[*a,]`, and `a[(*a,)]` all evaluate to the same + # thing at runtime and have the same AST, but only `a[*a,]` passes + # this test, because that's what `ast.unparse` produces. + self.check_src_roundtrip("a[*a,]") + self.check_src_roundtrip("a[1, *a]") + self.check_src_roundtrip("a[*a, 2]") + self.check_src_roundtrip("a[1, *a, 2]") + self.check_src_roundtrip("a[*a, *a]") + self.check_src_roundtrip("a[1, *a, *a]") + self.check_src_roundtrip("a[*a, 1, *a]") + self.check_src_roundtrip("a[*a, *a, 1]") + self.check_src_roundtrip("a[1, *a, *a, 2]") + self.check_src_roundtrip("a[1:2, *a]") + self.check_src_roundtrip("a[*a, 1:2]") + + def test_lambda_parameters(self): + self.check_src_roundtrip("lambda: something") + self.check_src_roundtrip("four = lambda: 2 + 2") + self.check_src_roundtrip("lambda x: x * 2") + self.check_src_roundtrip("square = lambda n: n ** 2") + self.check_src_roundtrip("lambda x, y: x + y") + self.check_src_roundtrip("add = lambda x, y: x + y") + self.check_src_roundtrip("lambda x, y, /, z, q, *, u: None") + self.check_src_roundtrip("lambda x, *y, **z: None") + + def test_star_expr_assign_target(self): + for source_type, source in [ + ("single assignment", "{target} = foo"), + ("multiple assignment", "{target} = {target} = bar"), + ("for loop", "for {target} in foo:\n pass"), + ("async for loop", "async for {target} in foo:\n pass") + ]: + for target in [ + "a", + "a,", + "a, b", + "a, *b, c", + "a, (b, c), d", + "a, (b, c, d), *e", + "a, (b, *c, d), e", + "a, (b, *c, (d, e), f), g", + "[a]", + "[a, b]", + "[a, *b, c]", + "[a, [b, c], d]", + "[a, [b, c, d], *e]", + "[a, [b, *c, d], e]", + "[a, [b, *c, [d, e], f], g]", + "a, [b, c], d", + "[a, b, (c, d), (e, f)]", + "a, b, [*c], d, e" + ]: + with self.subTest(source_type=source_type, target=target): + self.check_src_roundtrip(source.format(target=target)) + + def test_star_expr_assign_target_multiple(self): + self.check_src_roundtrip("() = []") + self.check_src_roundtrip("[] = ()") + self.check_src_roundtrip("() = [a] = c, = [d] = e, f = () = g = h") + self.check_src_roundtrip("a = b = c = d") + self.check_src_roundtrip("a, b = c, d = e, f = g") + self.check_src_roundtrip("[a, b] = [c, d] = [e, f] = g") + self.check_src_roundtrip("a, b = [c, d] = e, f = g") + + def test_multiquote_joined_string(self): + self.check_ast_roundtrip("f\"'''{1}\\\"\\\"\\\"\" ") + self.check_ast_roundtrip("""f"'''{1}""\\"" """) + self.check_ast_roundtrip("""f'""\"{1}''' """) + self.check_ast_roundtrip("""f'""\"{1}""\\"' """) + + self.check_ast_roundtrip("""f"'''{"\\n"}""\\"" """) + self.check_ast_roundtrip("""f'""\"{"\\n"}''' """) + self.check_ast_roundtrip("""f'""\"{"\\n"}""\\"' """) + + self.check_ast_roundtrip("""f'''""\"''\\'{"\\n"}''' """) + self.check_ast_roundtrip("""f'''""\"''\\'{"\\n\\"'"}''' """) + self.check_ast_roundtrip("""f'''""\"''\\'{""\"\\n\\"'''""\" '''\\n'''}''' """) + + def test_backslash_in_format_spec(self): + import re + msg = re.escape('"\\ " is an invalid escape sequence. ' + 'Such sequences will not work in the future. ' + 'Did you mean "\\\\ "? A raw string is also an option.') + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\ }" """) + self.check_ast_roundtrip("""f"{x:\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\ }" """) + + with self.assertWarnsRegex(SyntaxWarning, msg): + self.check_ast_roundtrip("""f"{x:\\\\\\ }" """) + self.check_ast_roundtrip("""f"{x:\\\\\\n}" """) + + self.check_ast_roundtrip("""f"{x:\\\\\\\\ }" """) + + def test_quote_in_format_spec(self): + self.check_ast_roundtrip("""f"{x:'}" """) + self.check_ast_roundtrip("""f"{x:\\'}" """) + self.check_ast_roundtrip("""f"{x:\\\\'}" """) + + self.check_ast_roundtrip("""f'\\'{x:"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\"}' """) + self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + + def test_type_params(self): + self.check_ast_roundtrip("type A = int") + self.check_ast_roundtrip("type A[T] = int") + self.check_ast_roundtrip("type A[T: int] = int") + self.check_ast_roundtrip("type A[T = int] = int") + self.check_ast_roundtrip("type A[T: int = int] = int") + self.check_ast_roundtrip("type A[**P] = int") + self.check_ast_roundtrip("type A[**P = int] = int") + self.check_ast_roundtrip("type A[*Ts] = int") + self.check_ast_roundtrip("type A[*Ts = int] = int") + self.check_ast_roundtrip("type A[*Ts = *int] = int") + self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") + self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + + +class ManualASTCreationTestCase(unittest.TestCase): + """Test that AST nodes created without a type_params field unparse correctly.""" + + def test_class(self): + node = ast.ClassDef(name="X", bases=[], keywords=[], body=[ast.Pass()], decorator_list=[]) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "class X:\n pass") + + def test_class_with_type_params(self): + node = ast.ClassDef(name="X", bases=[], keywords=[], body=[ast.Pass()], decorator_list=[], + type_params=[ast.TypeVar("T")]) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "class X[T]:\n pass") + + def test_function(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f():\n pass") + + def test_function_with_type_params(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T")], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T]():\n pass") + + def test_function_with_type_params_and_bound(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T", bound=ast.Name("int", ctx=ast.Load()))], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + + def test_function_with_type_params_and_default(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T = 1, *Ts = *1, **P = 1]():\n pass") + + def test_async_function(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f():\n pass") + + def test_async_function_with_type_params(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), + body=[ast.Pass()], + decorator_list=[], + returns=None, + type_params=[ast.TypeVar("T")], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + + def test_async_function_with_type_params_and_default(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T = 1, *Ts = *1, **P = 1]():\n pass") + + +class DirectoryTestCase(ASTTestCase): + """Test roundtrip behaviour on all files in Lib and Lib/test.""" + + lib_dir = pathlib.Path(__file__).parent / ".." + test_directories = (lib_dir, lib_dir / "test") + run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py", + "test_ast.py", "test_asdl_parser.py", "test_fstring.py", + "test_patma.py", "test_type_alias.py", "test_type_params.py", + "test_tokenize.py", "test_tstring.py"} + + _files_to_test = None + + @classmethod + def files_to_test(cls): + + if cls._files_to_test is not None: + return cls._files_to_test + + items = [ + item.resolve() + for directory in cls.test_directories + for item in directory.glob("*.py") + if not item.name.startswith("bad") + ] + + # Test limited subset of files unless the 'cpu' resource is specified. + if not test.support.is_resource_enabled("cpu"): + + tests_to_run_always = {item for item in items if + item.name in cls.run_always_files} + + items = set(random.sample(items, 10)) + + # Make sure that at least tests that heavily use grammar features are + # always considered in order to reduce the chance of missing something. + items = list(items | tests_to_run_always) + + # bpo-31174: Store the names sample to always test the same files. + # It prevents false alarms when hunting reference leaks. + cls._files_to_test = items + + return items + + def test_files(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', SyntaxWarning) + + for item in self.files_to_test(): + if test.support.verbose: + print(f"Testing {item.absolute()}") + + with self.subTest(filename=item): + source = read_pyfile(item) + self.check_ast_roundtrip(source) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 82f1d9dc2e7..2dd739b77b8 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -7,23 +7,26 @@ import email.message import io import unittest -from unittest.mock import patch from test import support from test.support import os_helper -from test.support import warnings_helper +from test.support import socket_helper +from test.support import control_characters_c0 import os +import socket try: import ssl except ImportError: ssl = None import sys import tempfile -from nturl2path import url2pathname, pathname2url -from base64 import b64encode import collections +if not socket_helper.has_gethostname: + raise unittest.SkipTest("test requires gethostname()") + + def hexescape(char): """Escape char as RFC 2396 specifies""" hex_repr = hex(ord(char))[2:].upper() @@ -31,32 +34,6 @@ def hexescape(char): hex_repr = "0%s" % hex_repr return "%" + hex_repr -# Shortcut for testing FancyURLopener -_urlopener = None - - -def urlopen(url, data=None, proxies=None): - """urlopen(url [, data]) -> open file-like object""" - global _urlopener - if proxies is not None: - opener = urllib.request.FancyURLopener(proxies=proxies) - elif not _urlopener: - opener = FancyURLopener() - _urlopener = opener - else: - opener = _urlopener - if data is None: - return opener.open(url) - else: - return opener.open(url, data) - - -def FancyURLopener(): - with warnings_helper.check_warnings( - ('FancyURLopener style of invoking requests is deprecated.', - DeprecationWarning)): - return urllib.request.FancyURLopener() - def fakehttp(fakedata, mock_close=False): class FakeSocket(io.BytesIO): @@ -115,26 +92,6 @@ def unfakehttp(self): http.client.HTTPConnection = self._connection_class -class FakeFTPMixin(object): - def fakeftp(self): - class FakeFtpWrapper(object): - def __init__(self, user, passwd, host, port, dirs, timeout=None, - persistent=True): - pass - - def retrfile(self, file, type): - return io.BytesIO(), 0 - - def close(self): - pass - - self._ftpwrapper_class = urllib.request.ftpwrapper - urllib.request.ftpwrapper = FakeFtpWrapper - - def unfakeftp(self): - urllib.request.ftpwrapper = self._ftpwrapper_class - - class urlopen_FileTests(unittest.TestCase): """Test urlopen() opening a temporary file. @@ -153,8 +110,8 @@ def setUp(self): finally: f.close() self.pathname = os_helper.TESTFN - self.quoted_pathname = urllib.parse.quote(self.pathname) - self.returned_obj = urlopen("file:%s" % self.quoted_pathname) + self.quoted_pathname = urllib.parse.quote(os.fsencode(self.pathname)) + self.returned_obj = urllib.request.urlopen("file:%s" % self.quoted_pathname) def tearDown(self): """Shut down the open object""" @@ -165,9 +122,7 @@ def test_interface(self): # Make sure object returned by urlopen() has the specified methods for attr in ("read", "readline", "readlines", "fileno", "close", "info", "geturl", "getcode", "__iter__"): - self.assertTrue(hasattr(self.returned_obj, attr), - "object returned by urlopen() lacks %s attribute" % - attr) + self.assertHasAttr(self.returned_obj, attr) def test_read(self): self.assertEqual(self.text, self.returned_obj.read()) @@ -201,7 +156,7 @@ def test_headers(self): self.assertIsInstance(self.returned_obj.headers, email.message.Message) def test_url(self): - self.assertEqual(self.returned_obj.url, self.quoted_pathname) + self.assertEqual(self.returned_obj.url, "file:" + self.quoted_pathname) def test_status(self): self.assertIsNone(self.returned_obj.status) @@ -210,7 +165,7 @@ def test_info(self): self.assertIsInstance(self.returned_obj.info(), email.message.Message) def test_geturl(self): - self.assertEqual(self.returned_obj.geturl(), self.quoted_pathname) + self.assertEqual(self.returned_obj.geturl(), "file:" + self.quoted_pathname) def test_getcode(self): self.assertIsNone(self.returned_obj.getcode()) @@ -227,22 +182,27 @@ def test_iter(self): def test_relativelocalfile(self): self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + def test_remote_authority(self): + # Test for GH-90812. + url = 'file://pythontest.net/foo/bar' + with self.assertRaises(urllib.error.URLError) as e: + urllib.request.urlopen(url) + if os.name == 'nt': + self.assertEqual(e.exception.filename, r'\\pythontest.net\foo\bar') + else: + self.assertEqual(e.exception.reason, 'file:// scheme is supported only on localhost') + class ProxyTests(unittest.TestCase): def setUp(self): # Records changes to env vars - self.env = os_helper.EnvironmentVarGuard() + self.env = self.enterContext(os_helper.EnvironmentVarGuard()) # Delete all proxy related env vars for k in list(os.environ): if 'proxy' in k.lower(): self.env.unset(k) - def tearDown(self): - # Restore all proxy related env vars - self.env.__exit__() - del self.env - def test_getproxies_environment_keep_no_proxies(self): self.env.set('NO_PROXY', 'localhost') proxies = urllib.request.getproxies_environment() @@ -340,13 +300,13 @@ def test_getproxies_environment_prefer_lowercase(self): self.assertEqual('http://somewhere:3128', proxies['http']) -class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): +class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin): """Test urlopen() opening a fake http connection.""" def check_read(self, ver): self.fakehttp(b"HTTP/" + ver + b" 200 OK\r\n\r\nHello!") try: - fp = urlopen("http://python.org/") + fp = urllib.request.urlopen("http://python.org/") self.assertEqual(fp.readline(), b"Hello!") self.assertEqual(fp.readline(), b"") self.assertEqual(fp.geturl(), 'http://python.org/') @@ -367,8 +327,8 @@ def test_url_fragment(self): def test_willclose(self): self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello!") try: - resp = urlopen("http://www.python.org") - self.assertTrue(resp.fp.will_close) + resp = urllib.request.urlopen("http://www.python.org") + self.assertTrue(resp.will_close) finally: self.unfakehttp() @@ -393,9 +353,6 @@ def test_url_path_with_control_char_rejected(self): with self.assertRaisesRegex( InvalidURL, f"contain control.*{escaped_char_repr}"): urllib.request.urlopen(f"https:{schemeless_url}") - # This code path quotes the URL so there is no injection. - resp = urlopen(f"http:{schemeless_url}") - self.assertNotIn(char, resp.geturl()) finally: self.unfakehttp() @@ -417,11 +374,6 @@ def test_url_path_with_newline_header_injection_rejected(self): urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): urllib.request.urlopen(f"https:{schemeless_url}") - # This code path quotes the URL so there is no injection. - resp = urlopen(f"http:{schemeless_url}") - self.assertNotIn(' ', resp.geturl()) - self.assertNotIn('\r', resp.geturl()) - self.assertNotIn('\n', resp.geturl()) finally: self.unfakehttp() @@ -436,9 +388,9 @@ def test_url_host_with_control_char_rejected(self): InvalidURL = http.client.InvalidURL with self.assertRaisesRegex( InvalidURL, f"contain control.*{escaped_char_repr}"): - urlopen(f"http:{schemeless_url}") + urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, f"contain control.*{escaped_char_repr}"): - urlopen(f"https:{schemeless_url}") + urllib.request.urlopen(f"https:{schemeless_url}") finally: self.unfakehttp() @@ -451,9 +403,9 @@ def test_url_host_with_newline_header_injection_rejected(self): InvalidURL = http.client.InvalidURL with self.assertRaisesRegex( InvalidURL, r"contain control.*\\r"): - urlopen(f"http:{schemeless_url}") + urllib.request.urlopen(f"http:{schemeless_url}") with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): - urlopen(f"https:{schemeless_url}") + urllib.request.urlopen(f"https:{schemeless_url}") finally: self.unfakehttp() @@ -477,7 +429,9 @@ def test_read_bogus(self): Content-Type: text/html; charset=iso-8859-1 ''', mock_close=True) try: - self.assertRaises(OSError, urlopen, "http://python.org/") + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen("http://python.org/") + cm.exception.close() finally: self.unfakehttp() @@ -492,22 +446,24 @@ def test_invalid_redirect(self): ''', mock_close=True) try: msg = "Redirection to url 'file:" - with self.assertRaisesRegex(urllib.error.HTTPError, msg): - urlopen("http://python.org/") + with self.assertRaisesRegex(urllib.error.HTTPError, msg) as cm: + urllib.request.urlopen("http://python.org/") + cm.exception.close() finally: self.unfakehttp() def test_redirect_limit_independent(self): # Ticket #12923: make sure independent requests each use their # own retry limit. - for i in range(FancyURLopener().maxtries): + for i in range(urllib.request.HTTPRedirectHandler.max_redirections): self.fakehttp(b'''HTTP/1.1 302 Found Location: file://guidocomputer.athome.com:/python/license Connection: close ''', mock_close=True) try: - self.assertRaises(urllib.error.HTTPError, urlopen, - "http://something") + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen("http://something") + cm.exception.close() finally: self.unfakehttp() @@ -516,96 +472,47 @@ def test_empty_socket(self): # data. (#1680230) self.fakehttp(b'') try: - self.assertRaises(OSError, urlopen, "http://something") + self.assertRaises(OSError, urllib.request.urlopen, "http://something") finally: self.unfakehttp() def test_missing_localfile(self): # Test for #10836 with self.assertRaises(urllib.error.URLError) as e: - urlopen('file://localhost/a/file/which/doesnot/exists.py') + urllib.request.urlopen('file://localhost/a/file/which/doesnot/exists.py') self.assertTrue(e.exception.filename) self.assertTrue(e.exception.reason) def test_file_notexists(self): fd, tmp_file = tempfile.mkstemp() - tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + tmp_file_canon_url = urllib.request.pathname2url(tmp_file, add_scheme=True) + parsed = urllib.parse.urlsplit(tmp_file_canon_url) + tmp_fileurl = parsed._replace(netloc='localhost').geturl() try: self.assertTrue(os.path.exists(tmp_file)) - with urlopen(tmp_fileurl) as fobj: + with urllib.request.urlopen(tmp_fileurl) as fobj: self.assertTrue(fobj) + self.assertEqual(fobj.url, tmp_file_canon_url) finally: os.close(fd) os.unlink(tmp_file) self.assertFalse(os.path.exists(tmp_file)) with self.assertRaises(urllib.error.URLError): - urlopen(tmp_fileurl) + urllib.request.urlopen(tmp_fileurl) def test_ftp_nohost(self): test_ftp_url = 'ftp:///path' with self.assertRaises(urllib.error.URLError) as e: - urlopen(test_ftp_url) + urllib.request.urlopen(test_ftp_url) self.assertFalse(e.exception.filename) self.assertTrue(e.exception.reason) def test_ftp_nonexisting(self): with self.assertRaises(urllib.error.URLError) as e: - urlopen('ftp://localhost/a/file/which/doesnot/exists.py') + urllib.request.urlopen('ftp://localhost/a/file/which/doesnot/exists.py') self.assertFalse(e.exception.filename) self.assertTrue(e.exception.reason) - @patch.object(urllib.request, 'MAXFTPCACHE', 0) - def test_ftp_cache_pruning(self): - self.fakeftp() - try: - urllib.request.ftpcache['test'] = urllib.request.ftpwrapper('user', 'pass', 'localhost', 21, []) - urlopen('ftp://localhost') - finally: - self.unfakeftp() - - def test_userpass_inurl(self): - self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") - try: - fp = urlopen("http://user:pass@python.org/") - self.assertEqual(fp.readline(), b"Hello!") - self.assertEqual(fp.readline(), b"") - self.assertEqual(fp.geturl(), 'http://user:pass@python.org/') - self.assertEqual(fp.getcode(), 200) - finally: - self.unfakehttp() - - def test_userpass_inurl_w_spaces(self): - self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") - try: - userpass = "a b:c d" - url = "http://{}@python.org/".format(userpass) - fakehttp_wrapper = http.client.HTTPConnection - authorization = ("Authorization: Basic %s\r\n" % - b64encode(userpass.encode("ASCII")).decode("ASCII")) - fp = urlopen(url) - # The authorization header must be in place - self.assertIn(authorization, fakehttp_wrapper.buf.decode("UTF-8")) - self.assertEqual(fp.readline(), b"Hello!") - self.assertEqual(fp.readline(), b"") - # the spaces are quoted in URL so no match - self.assertNotEqual(fp.geturl(), url) - self.assertEqual(fp.getcode(), 200) - finally: - self.unfakehttp() - - def test_URLopener_deprecation(self): - with warnings_helper.check_warnings(('',DeprecationWarning)): - urllib.request.URLopener() - - @unittest.skipUnless(ssl, "ssl module required") - def test_cafile_and_context(self): - context = ssl.create_default_context() - with warnings_helper.check_warnings(('', DeprecationWarning)): - with self.assertRaises(ValueError): - urllib.request.urlopen( - "https://localhost", cafile="/nonexistent/path", context=context - ) - class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -636,18 +543,17 @@ def setUp(self): "QOjdAAAAAXNSR0IArs4c6QAAAA9JREFUCNdj%0AYGBg%2BP//PwAGAQL%2BCm8 " "vHgAAAABJRU5ErkJggg%3D%3D%0A%20") - self.text_url_resp = urllib.request.urlopen(self.text_url) - self.text_url_base64_resp = urllib.request.urlopen( - self.text_url_base64) - self.image_url_resp = urllib.request.urlopen(self.image_url) + self.text_url_resp = self.enterContext( + urllib.request.urlopen(self.text_url)) + self.text_url_base64_resp = self.enterContext( + urllib.request.urlopen(self.text_url_base64)) + self.image_url_resp = self.enterContext(urllib.request.urlopen(self.image_url)) def test_interface(self): # Make sure object returned by urlopen() has the specified methods for attr in ("read", "readline", "readlines", "close", "info", "geturl", "getcode", "__iter__"): - self.assertTrue(hasattr(self.text_url_resp, attr), - "object returned by urlopen() lacks %s attribute" % - attr) + self.assertHasAttr(self.text_url_resp, attr) def test_info(self): self.assertIsInstance(self.text_url_resp.info(), email.message.Message) @@ -655,8 +561,10 @@ def test_info(self): [('text/plain', ''), ('charset', 'ISO-8859-1')]) self.assertEqual(self.image_url_resp.info()['content-length'], str(len(self.image))) - self.assertEqual(urllib.request.urlopen("data:,").info().get_params(), + r = urllib.request.urlopen("data:,") + self.assertEqual(r.info().get_params(), [('text/plain', ''), ('charset', 'US-ASCII')]) + r.close() def test_geturl(self): self.assertEqual(self.text_url_resp.geturl(), self.text_url) @@ -683,6 +591,13 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + def test_invalid_mediatype(self): + for c0 in control_characters_c0(): + self.assertRaises(ValueError,urllib.request.urlopen, + f'data:text/html;{c0},data') + for c0 in control_characters_c0(): + self.assertRaises(ValueError,urllib.request.urlopen, + f'data:text/html{c0};base64,ZGF0YQ==') class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" @@ -719,11 +634,7 @@ def tearDown(self): def constructLocalFileUrl(self, filePath): filePath = os.path.abspath(filePath) - try: - filePath.encode("utf-8") - except UnicodeEncodeError: - raise unittest.SkipTest("filePath is not encodable to utf8") - return "file://%s" % urllib.request.pathname2url(filePath) + return urllib.request.pathname2url(filePath, add_scheme=True) def createNewTempFile(self, data=b""): """Creates a new temporary file containing the specified data, @@ -1104,6 +1015,8 @@ def test_unquoting(self): self.assertEqual(result.count('%'), 1, "using unquote(): not all characters escaped: " "%s" % result) + + def test_unquote_rejects_none_and_tuple(self): self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, None) self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, ()) @@ -1526,40 +1439,229 @@ def test_quoting(self): "url2pathname() failed; %s != %s" % (expect, result)) + def test_pathname2url(self): + # Test cases common to Windows and POSIX. + fn = urllib.request.pathname2url + sep = os.path.sep + self.assertEqual(fn(''), '') + self.assertEqual(fn(sep), '///') + self.assertEqual(fn('a'), 'a') + self.assertEqual(fn(f'a{sep}b.c'), 'a/b.c') + self.assertEqual(fn(f'{sep}a{sep}b.c'), '///a/b.c') + self.assertEqual(fn(f'{sep}a{sep}b%#c'), '///a/b%25%23c') + + def test_pathname2url_add_scheme(self): + sep = os.path.sep + subtests = [ + ('', 'file:'), + (sep, 'file:///'), + ('a', 'file:a'), + (f'a{sep}b.c', 'file:a/b.c'), + (f'{sep}a{sep}b.c', 'file:///a/b.c'), + (f'{sep}a{sep}b%#c', 'file:///a/b%25%23c'), + ] + for path, expected_url in subtests: + with self.subTest(path=path): + self.assertEqual( + urllib.request.pathname2url(path, add_scheme=True), expected_url) + @unittest.skipUnless(sys.platform == 'win32', - 'test specific to the nturl2path functions.') - def test_prefixes(self): + 'test specific to Windows pathnames.') + def test_pathname2url_win(self): # Test special prefixes are correctly handled in pathname2url() - given = '\\\\?\\C:\\dir' - expect = '///C:/dir' - result = urllib.request.pathname2url(given) - self.assertEqual(expect, result, - "pathname2url() failed; %s != %s" % - (expect, result)) - given = '\\\\?\\unc\\server\\share\\dir' - expect = '/server/share/dir' - result = urllib.request.pathname2url(given) - self.assertEqual(expect, result, - "pathname2url() failed; %s != %s" % - (expect, result)) - + fn = urllib.request.pathname2url + self.assertEqual(fn('\\\\?\\C:\\dir'), '///C:/dir') + self.assertEqual(fn('\\\\?\\unc\\server\\share\\dir'), '//server/share/dir') + self.assertEqual(fn("C:"), '///C:') + self.assertEqual(fn("C:\\"), '///C:/') + self.assertEqual(fn('c:\\a\\b.c'), '///c:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c'), '///C:/a/b.c') + self.assertEqual(fn('C:\\a\\b.c\\'), '///C:/a/b.c/') + self.assertEqual(fn('C:\\a\\\\b.c'), '///C:/a//b.c') + self.assertEqual(fn('C:\\a\\b%#c'), '///C:/a/b%25%23c') + self.assertEqual(fn('C:\\a\\b\xe9'), '///C:/a/b%C3%A9') + self.assertEqual(fn('C:\\foo\\bar\\spam.foo'), "///C:/foo/bar/spam.foo") + # NTFS alternate data streams + self.assertEqual(fn('C:\\foo:bar'), '///C:/foo%3Abar') + self.assertEqual(fn('foo:bar'), 'foo%3Abar') + # No drive letter + self.assertEqual(fn("\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn("\\\\folder\\test\\"), '//folder/test/') + self.assertEqual(fn("\\\\\\folder\\test\\"), '///folder/test/') + self.assertEqual(fn('\\\\some\\share\\'), '//some/share/') + self.assertEqual(fn('\\\\some\\share\\a\\b.c'), '//some/share/a/b.c') + self.assertEqual(fn('\\\\some\\share\\a\\b%#c\xe9'), '//some/share/a/b%25%23c%C3%A9') + # Alternate path separator + self.assertEqual(fn('C:/a/b.c'), '///C:/a/b.c') + self.assertEqual(fn('//some/share/a/b.c'), '//some/share/a/b.c') + self.assertEqual(fn('//?/C:/dir'), '///C:/dir') + self.assertEqual(fn('//?/unc/server/share/dir'), '//server/share/dir') + # Round-tripping + urls = ['///C:', + '///folder/test/', + '///C:/foo/bar/spam.foo'] + for url in urls: + self.assertEqual(fn(urllib.request.url2pathname(url)), url) + + @unittest.skipIf(sys.platform == 'win32', + 'test specific to POSIX pathnames') + def test_pathname2url_posix(self): + fn = urllib.request.pathname2url + self.assertEqual(fn('//a/b.c'), '////a/b.c') + self.assertEqual(fn('///a/b.c'), '/////a/b.c') + self.assertEqual(fn('////a/b.c'), '//////a/b.c') + + @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') + def test_pathname2url_nonascii(self): + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + url = urllib.parse.quote(os_helper.FS_NONASCII, encoding=encoding, errors=errors) + self.assertEqual(urllib.request.pathname2url(os_helper.FS_NONASCII), url) + + def test_url2pathname(self): + # Test cases common to Windows and POSIX. + fn = urllib.request.url2pathname + sep = os.path.sep + self.assertEqual(fn(''), '') + self.assertEqual(fn('/'), f'{sep}') + self.assertEqual(fn('///'), f'{sep}') + self.assertEqual(fn('////'), f'{sep}{sep}') + self.assertEqual(fn('foo'), 'foo') + self.assertEqual(fn('foo/bar'), f'foo{sep}bar') + self.assertEqual(fn('/foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn('//localhost/foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn('///foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn('////foo/bar'), f'{sep}{sep}foo{sep}bar') + self.assertEqual(fn('data:blah'), 'data:blah') + self.assertEqual(fn('data://blah'), f'data:{sep}{sep}blah') + self.assertEqual(fn('foo?bar'), 'foo') + self.assertEqual(fn('foo#bar'), 'foo') + self.assertEqual(fn('foo?bar=baz'), 'foo') + self.assertEqual(fn('foo?bar#baz'), 'foo') + self.assertEqual(fn('foo%3Fbar'), 'foo?bar') + self.assertEqual(fn('foo%23bar'), 'foo#bar') + self.assertEqual(fn('foo%3Fbar%3Dbaz'), 'foo?bar=baz') + self.assertEqual(fn('foo%3Fbar%23baz'), 'foo?bar#baz') + + def test_url2pathname_require_scheme(self): + sep = os.path.sep + subtests = [ + ('file:', ''), + ('FILE:', ''), + ('FiLe:', ''), + ('file:/', f'{sep}'), + ('file:///', f'{sep}'), + ('file:////', f'{sep}{sep}'), + ('file:foo', 'foo'), + ('file:foo/bar', f'foo{sep}bar'), + ('file:/foo/bar', f'{sep}foo{sep}bar'), + ('file://localhost/foo/bar', f'{sep}foo{sep}bar'), + ('file:///foo/bar', f'{sep}foo{sep}bar'), + ('file:////foo/bar', f'{sep}{sep}foo{sep}bar'), + ('file:data:blah', 'data:blah'), + ('file:data://blah', f'data:{sep}{sep}blah'), + ] + for url, expected_path in subtests: + with self.subTest(url=url): + self.assertEqual( + urllib.request.url2pathname(url, require_scheme=True), + expected_path) + + def test_url2pathname_require_scheme_errors(self): + subtests = [ + '', + ':', + 'foo', + 'http:foo', + 'localfile:foo', + 'data:foo', + 'data:file:foo', + 'data:file://foo', + ] + for url in subtests: + with self.subTest(url=url): + self.assertRaises( + urllib.error.URLError, + urllib.request.url2pathname, + url, require_scheme=True) + + @unittest.skipIf(support.is_emscripten, "Fixed by https://github.com/emscripten-core/emscripten/pull/24593") + def test_url2pathname_resolve_host(self): + fn = urllib.request.url2pathname + sep = os.path.sep + self.assertEqual(fn('//127.0.0.1/foo/bar', resolve_host=True), f'{sep}foo{sep}bar') + self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar'), f'{sep}foo{sep}bar') + self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar', resolve_host=True), f'{sep}foo{sep}bar') @unittest.skipUnless(sys.platform == 'win32', - 'test specific to the urllib.url2path function.') - def test_ntpath(self): - given = ('/C:/', '///C:/', '/C|//') - expect = 'C:\\' - for url in given: - result = urllib.request.url2pathname(url) - self.assertEqual(expect, result, - 'urllib.request..url2pathname() failed; %s != %s' % - (expect, result)) - given = '///C|/path' - expect = 'C:\\path' - result = urllib.request.url2pathname(given) - self.assertEqual(expect, result, - 'urllib.request.url2pathname() failed; %s != %s' % - (expect, result)) + 'test specific to Windows pathnames.') + def test_url2pathname_win(self): + fn = urllib.request.url2pathname + self.assertEqual(fn('/C:/'), 'C:\\') + self.assertEqual(fn('//C:'), 'C:') + self.assertEqual(fn('//C:/'), 'C:\\') + self.assertEqual(fn('//C:\\'), 'C:\\') + self.assertEqual(fn('//C:80/'), 'C:80\\') + self.assertEqual(fn("///C|"), 'C:') + self.assertEqual(fn("///C:"), 'C:') + self.assertEqual(fn('///C:/'), 'C:\\') + self.assertEqual(fn('/C|//'), 'C:\\\\') + self.assertEqual(fn('///C|/path'), 'C:\\path') + # No DOS drive + self.assertEqual(fn("///C/test/"), '\\C\\test\\') + self.assertEqual(fn("////C/test/"), '\\\\C\\test\\') + # DOS drive paths + self.assertEqual(fn('c:/path/to/file'), 'c:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('C:/path/to/file/'), 'C:\\path\\to\\file\\') + self.assertEqual(fn('C:/path/to//file'), 'C:\\path\\to\\\\file') + self.assertEqual(fn('C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('///C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn("///C|/foo/bar/spam.foo"), 'C:\\foo\\bar\\spam.foo') + # Colons in URI + self.assertEqual(fn('///\u00e8|/'), '\u00e8:\\') + self.assertEqual(fn('//host/share/spam.txt:eggs'), '\\\\host\\share\\spam.txt:eggs') + self.assertEqual(fn('///c:/spam.txt:eggs'), 'c:\\spam.txt:eggs') + # UNC paths + self.assertEqual(fn('//server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('/////server/path/to/file'), '\\\\server\\path\\to\\file') + self.assertEqual(fn('//127.0.0.1/path/to/file'), '\\\\127.0.0.1\\path\\to\\file') + # Localhost paths + self.assertEqual(fn('//localhost/C:/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/C|/path/to/file'), 'C:\\path\\to\\file') + self.assertEqual(fn('//localhost/path/to/file'), '\\path\\to\\file') + self.assertEqual(fn('//localhost//server/path/to/file'), '\\\\server\\path\\to\\file') + # Percent-encoded forward slashes are preserved for backwards compatibility + self.assertEqual(fn('C:/foo%2fbar'), 'C:\\foo/bar') + self.assertEqual(fn('//server/share/foo%2fbar'), '\\\\server\\share\\foo/bar') + # Round-tripping + paths = ['C:', + r'\C\test\\', + r'C:\foo\bar\spam.foo'] + for path in paths: + self.assertEqual(fn(urllib.request.pathname2url(path)), path) + + @unittest.skipIf(sys.platform == 'win32', + 'test specific to POSIX pathnames') + def test_url2pathname_posix(self): + fn = urllib.request.url2pathname + self.assertRaises(urllib.error.URLError, fn, '//foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//localhost:/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//:80/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//:/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//c:80/foo/bar') + self.assertRaises(urllib.error.URLError, fn, '//127.0.0.1/foo/bar') + + @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') + def test_url2pathname_nonascii(self): + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + url = os_helper.FS_NONASCII + self.assertEqual(urllib.request.url2pathname(url), os_helper.FS_NONASCII) + url = urllib.parse.quote(url, encoding=encoding, errors=errors) + self.assertEqual(urllib.request.url2pathname(url), os_helper.FS_NONASCII) class Utility_Tests(unittest.TestCase): """Testcase to test the various utility functions in the urllib.""" @@ -1569,56 +1671,6 @@ def test_thishost(self): self.assertIsInstance(urllib.request.thishost(), tuple) -class URLopener_Tests(FakeHTTPMixin, unittest.TestCase): - """Testcase to test the open method of URLopener class.""" - - def test_quoted_open(self): - class DummyURLopener(urllib.request.URLopener): - def open_spam(self, url): - return url - with warnings_helper.check_warnings( - ('DummyURLopener style of invoking requests is deprecated.', - DeprecationWarning)): - self.assertEqual(DummyURLopener().open( - 'spam://example/ /'),'//example/%20/') - - # test the safe characters are not quoted by urlopen - self.assertEqual(DummyURLopener().open( - "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"), - "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/") - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_urlopener_retrieve_file(self): - with os_helper.temp_dir() as tmpdir: - fd, tmpfile = tempfile.mkstemp(dir=tmpdir) - os.close(fd) - fileurl = "file:" + urllib.request.pathname2url(tmpfile) - filename, _ = urllib.request.URLopener().retrieve(fileurl) - # Some buildbots have TEMP folder that uses a lowercase drive letter. - self.assertEqual(os.path.normcase(filename), os.path.normcase(tmpfile)) - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_urlopener_retrieve_remote(self): - url = "http://www.python.org/file.txt" - self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello!") - self.addCleanup(self.unfakehttp) - filename, _ = urllib.request.URLopener().retrieve(url) - self.assertEqual(os.path.splitext(filename)[1], ".txt") - - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_local_file_open(self): - # bpo-35907, CVE-2019-9948: urllib must reject local_file:// scheme - class DummyURLopener(urllib.request.URLopener): - def open_local_file(self, url): - return url - for url in ('local_file://example', 'local-file://example'): - self.assertRaises(OSError, urllib.request.urlopen, url) - self.assertRaises(OSError, urllib.request.URLopener().open, url) - self.assertRaises(OSError, urllib.request.URLopener().retrieve, url) - self.assertRaises(OSError, DummyURLopener().open, url) - self.assertRaises(OSError, DummyURLopener().retrieve, url) - - class RequestTests(unittest.TestCase): """Unit tests for urllib.request.Request.""" @@ -1643,60 +1695,5 @@ def test_with_method_arg(self): self.assertEqual(request.get_method(), 'HEAD') -class URL2PathNameTests(unittest.TestCase): - - def test_converting_drive_letter(self): - self.assertEqual(url2pathname("///C|"), 'C:') - self.assertEqual(url2pathname("///C:"), 'C:') - self.assertEqual(url2pathname("///C|/"), 'C:\\') - - def test_converting_when_no_drive_letter(self): - # cannot end a raw string in \ - self.assertEqual(url2pathname("///C/test/"), r'\\\C\test' '\\') - self.assertEqual(url2pathname("////C/test/"), r'\\C\test' '\\') - - def test_simple_compare(self): - self.assertEqual(url2pathname("///C|/foo/bar/spam.foo"), - r'C:\foo\bar\spam.foo') - - def test_non_ascii_drive_letter(self): - self.assertRaises(IOError, url2pathname, "///\u00e8|/") - - def test_roundtrip_url2pathname(self): - list_of_paths = ['C:', - r'\\\C\test\\', - r'C:\foo\bar\spam.foo' - ] - for path in list_of_paths: - self.assertEqual(url2pathname(pathname2url(path)), path) - -class PathName2URLTests(unittest.TestCase): - - def test_converting_drive_letter(self): - self.assertEqual(pathname2url("C:"), '///C:') - self.assertEqual(pathname2url("C:\\"), '///C:') - - def test_converting_when_no_drive_letter(self): - self.assertEqual(pathname2url(r"\\\folder\test" "\\"), - '/////folder/test/') - self.assertEqual(pathname2url(r"\\folder\test" "\\"), - '////folder/test/') - self.assertEqual(pathname2url(r"\folder\test" "\\"), - '/folder/test/') - - def test_simple_compare(self): - self.assertEqual(pathname2url(r'C:\foo\bar\spam.foo'), - "///C:/foo/bar/spam.foo" ) - - def test_long_drive_letter(self): - self.assertRaises(IOError, pathname2url, "XX:\\") - - def test_roundtrip_pathname2url(self): - list_of_paths = ['///C:', - '/////folder/test/', - '///C:/foo/bar/spam.foo'] - for path in list_of_paths: - self.assertEqual(pathname2url(url2pathname(path)), path) - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 5a02b5db8e5..7d7f2fa00d3 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1,12 +1,14 @@ import unittest from test import support from test.support import os_helper -from test.support import socket_helper +from test.support import requires_subprocess from test.support import warnings_helper from test import test_urllib +from unittest import mock import os import io +import ftplib import socket import array import sys @@ -14,16 +16,20 @@ import subprocess import urllib.request -# The proxy bypass method imported below has logic specific to the OSX -# proxy config data structure but is testable on all platforms. +# The proxy bypass method imported below has logic specific to the +# corresponding system but is testable on all platforms. from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, HTTPPasswordMgrWithPriorAuth, _parse_proxy, + _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) -from urllib.parse import urlparse +from urllib.parse import urlsplit import urllib.error import http.client + +support.requires_working_socket(module=True) + # XXX # Request # CacheFTPHandler (hard to write) @@ -38,10 +44,6 @@ def test___all__(self): context = {} exec('from urllib.%s import *' % module, context) del context['__builtins__'] - if module == 'request' and os.name == 'nt': - u, p = context.pop('url2pathname'), context.pop('pathname2url') - self.assertEqual(u.__module__, 'nturl2path') - self.assertEqual(p.__module__, 'nturl2path') for k, v in context.items(): self.assertEqual(v.__module__, 'urllib.%s' % module, "%r is exposed in 'urllib.%s' but defined in %r" % @@ -483,7 +485,18 @@ def build_test_opener(*handler_instances): return opener -class MockHTTPHandler(urllib.request.BaseHandler): +class MockHTTPHandler(urllib.request.HTTPHandler): + # Very simple mock HTTP handler with no special behavior other than using a mock HTTP connection + + def __init__(self, debuglevel=None): + super(MockHTTPHandler, self).__init__(debuglevel=debuglevel) + self.httpconn = MockHTTPClass() + + def http_open(self, req): + return self.do_open(self.httpconn, req) + + +class MockHTTPHandlerRedirect(urllib.request.BaseHandler): # useful for testing redirections and auth # sends supplied headers and code as first response # sends 200 OK as second response @@ -511,16 +524,17 @@ def http_open(self, req): return MockResponse(200, "OK", msg, "", req.get_full_url()) -class MockHTTPSHandler(urllib.request.AbstractHTTPHandler): - # Useful for testing the Proxy-Authorization request by verifying the - # properties of httpcon +if hasattr(http.client, 'HTTPSConnection'): + class MockHTTPSHandler(urllib.request.HTTPSHandler): + # Useful for testing the Proxy-Authorization request by verifying the + # properties of httpcon - def __init__(self, debuglevel=0): - urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel) - self.httpconn = MockHTTPClass() + def __init__(self, debuglevel=None, context=None, check_hostname=None): + super(MockHTTPSHandler, self).__init__(debuglevel, context, check_hostname) + self.httpconn = MockHTTPClass() - def https_open(self, req): - return self.do_open(self.httpconn, req) + def https_open(self, req): + return self.do_open(self.httpconn, req) class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): @@ -641,8 +655,6 @@ def test_raise(self): self.assertRaises(urllib.error.URLError, o.open, req) self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_http_error(self): # XXX http_error_default # http errors are a special case @@ -666,8 +678,6 @@ def test_http_error(self): self.assertEqual((handler, method_name), got[:2]) self.assertEqual(args, got[2]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_processors(self): # *_request / *_response methods get called appropriately o = OpenerDirector() @@ -704,18 +714,6 @@ def test_processors(self): self.assertIsInstance(args[1], MockResponse) -def sanepathname2url(path): - try: - path.encode("utf-8") - except UnicodeEncodeError: - raise unittest.SkipTest("path is not encodable to utf8") - urlpath = urllib.request.pathname2url(path) - if os.name == "nt" and urlpath.startswith("///"): - urlpath = urlpath[2:] - # XXX don't ask me about the mac... - return urlpath - - class HandlerTests(unittest.TestCase): def test_ftp(self): @@ -742,7 +740,6 @@ def connect_ftp(self, user, passwd, host, port, dirs, self.ftpwrapper = MockFTPWrapper(self.data) return self.ftpwrapper - import ftplib data = "rheum rhaponicum" h = NullFTPHandler(data) h.parent = MockOpener() @@ -765,7 +762,7 @@ def connect_ftp(self, user, passwd, host, port, dirs, ["foo", "bar"], "", None), ("ftp://localhost/baz.gif;type=a", "localhost", ftplib.FTP_PORT, "", "", "A", - [], "baz.gif", None), # XXX really this should guess image/gif + [], "baz.gif", "image/gif"), ]: req = Request(url) req.timeout = None @@ -781,6 +778,29 @@ def connect_ftp(self, user, passwd, host, port, dirs, headers = r.info() self.assertEqual(headers.get("Content-type"), mimetype) self.assertEqual(int(headers["Content-length"]), len(data)) + r.close() + + @support.requires_resource("network") + def test_ftp_error(self): + class ErrorFTPHandler(urllib.request.FTPHandler): + def __init__(self, exception): + self._exception = exception + + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + raise self._exception + + exception = ftplib.error_perm( + "500 OOPS: cannot change directory:/nonexistent") + h = ErrorFTPHandler(exception) + urlopen = urllib.request.build_opener(h).open + try: + urlopen("ftp://www.pythontest.net/") + except urllib.error.URLError as raised: + self.assertEqual(raised.reason, + f"ftp error: {exception.args[0]}") + else: + self.fail("Did not raise ftplib exception") def test_file(self): import email.utils @@ -788,19 +808,22 @@ def test_file(self): o = h.parent = MockOpener() TESTFN = os_helper.TESTFN - urlpath = sanepathname2url(os.path.abspath(TESTFN)) towrite = b"hello, world\n" + canonurl = urllib.request.pathname2url(os.path.abspath(TESTFN), add_scheme=True) + parsed = urlsplit(canonurl) + if parsed.netloc: + raise unittest.SkipTest("non-local working directory") urls = [ - "file://localhost%s" % urlpath, - "file://%s" % urlpath, - "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), + canonurl, + parsed._replace(netloc='localhost').geturl(), + parsed._replace(netloc=socket.gethostbyname('localhost')).geturl(), ] try: localaddr = socket.gethostbyname(socket.gethostname()) except socket.gaierror: localaddr = '' if localaddr: - urls.append("file://%s%s" % (localaddr, urlpath)) + urls.append(parsed._replace(netloc=localaddr).geturl()) for url in urls: f = open(TESTFN, "wb") @@ -825,10 +848,10 @@ def test_file(self): self.assertEqual(headers["Content-type"], "text/plain") self.assertEqual(headers["Content-length"], "13") self.assertEqual(headers["Last-modified"], modified) - self.assertEqual(respurl, url) + self.assertEqual(respurl, canonurl) for url in [ - "file://localhost:80%s" % urlpath, + parsed._replace(netloc='localhost:80').geturl(), "file:///file_does_not_exist.txt", "file://not-a-local-host.com//dir/file.txt", "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), @@ -874,8 +897,6 @@ def test_file(self): self.assertEqual(req.type, "ftp") self.assertEqual(req.type == "ftp", ftp) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_http(self): h = urllib.request.AbstractHTTPHandler() @@ -990,6 +1011,7 @@ def test_http_body_fileobj(self): file_obj.close() + @requires_subprocess() def test_http_body_pipe(self): # A file reading from a pipe. # A pipe cannot be seek'ed. There is no way to determine the @@ -1053,12 +1075,37 @@ def test_http_body_array(self): newreq = h.do_request_(req) self.assertEqual(int(newreq.get_header('Content-length')),16) - def test_http_handler_debuglevel(self): + def test_http_handler_global_debuglevel(self): + with mock.patch.object(http.client.HTTPConnection, 'debuglevel', 6): + o = OpenerDirector() + h = MockHTTPHandler() + o.add_handler(h) + o.open("http://www.example.com") + self.assertEqual(h._debuglevel, 6) + + def test_http_handler_local_debuglevel(self): + o = OpenerDirector() + h = MockHTTPHandler(debuglevel=5) + o.add_handler(h) + o.open("http://www.example.com") + self.assertEqual(h._debuglevel, 5) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_https_handler_global_debuglevel(self): + with mock.patch.object(http.client.HTTPSConnection, 'debuglevel', 7): + o = OpenerDirector() + h = MockHTTPSHandler() + o.add_handler(h) + o.open("https://www.example.com") + self.assertEqual(h._debuglevel, 7) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_https_handler_local_debuglevel(self): o = OpenerDirector() - h = MockHTTPSHandler(debuglevel=1) + h = MockHTTPSHandler(debuglevel=4) o.add_handler(h) o.open("https://www.example.com") - self.assertEqual(h._debuglevel, 1) + self.assertEqual(h._debuglevel, 4) def test_http_doubleslash(self): # Checks the presence of any unnecessary double slash in url does not @@ -1102,13 +1149,13 @@ def test_full_url_setter(self): r = Request('http://example.com') for url in urls: r.full_url = url - parsed = urlparse(url) + parsed = urlsplit(url) self.assertEqual(r.get_full_url(), url) # full_url setter uses splittag to split into components. # splittag sets the fragment as None while urlparse sets it to '' self.assertEqual(r.fragment or '', parsed.fragment) - self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) + self.assertEqual(urlsplit(r.get_full_url()).query, parsed.query) def test_full_url_deleter(self): r = Request('http://www.example.com') @@ -1136,8 +1183,6 @@ def test_fixpath_in_weirdurls(self): self.assertEqual(newreq.host, 'www.python.org') self.assertEqual(newreq.selector, '') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): h = urllib.request.HTTPErrorProcessor() o = h.parent = MockOpener() @@ -1148,23 +1193,21 @@ def test_errors(self): r = MockResponse(200, "OK", {}, "", url) newr = h.http_response(req, r) self.assertIs(r, newr) - self.assertFalse(hasattr(o, "proto")) # o.error not called + self.assertNotHasAttr(o, "proto") # o.error not called r = MockResponse(202, "Accepted", {}, "", url) newr = h.http_response(req, r) self.assertIs(r, newr) - self.assertFalse(hasattr(o, "proto")) # o.error not called + self.assertNotHasAttr(o, "proto") # o.error not called r = MockResponse(206, "Partial content", {}, "", url) newr = h.http_response(req, r) self.assertIs(r, newr) - self.assertFalse(hasattr(o, "proto")) # o.error not called + self.assertNotHasAttr(o, "proto") # o.error not called # anything else calls o.error (and MockOpener returns None, here) r = MockResponse(502, "Bad gateway", {}, "", url) self.assertIsNone(h.http_response(req, r)) self.assertEqual(o.proto, "http") # o.error called self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cookies(self): cj = MockCookieJar() h = urllib.request.HTTPCookieProcessor(cj) @@ -1189,7 +1232,7 @@ def test_redirect(self): o = h.parent = MockOpener() # ordinary redirect behaviour - for code in 301, 302, 303, 307: + for code in 301, 302, 303, 307, 308: for data in None, "blah\nblah\n": method = getattr(h, "http_error_%s" % code) req = Request(from_url, data) @@ -1201,10 +1244,11 @@ def test_redirect(self): try: method(req, MockFile(), code, "Blah", MockHeaders({"location": to_url})) - except urllib.error.HTTPError: - # 307 in response to POST requires user OK - self.assertEqual(code, 307) + except urllib.error.HTTPError as err: + # 307 and 308 in response to POST require user OK + self.assertIn(code, (307, 308)) self.assertIsNotNone(data) + err.close() self.assertEqual(o.req.get_full_url(), to_url) try: self.assertEqual(o.req.get_method(), "GET") @@ -1240,9 +1284,10 @@ def redirect(h, req, url=to_url): while 1: redirect(h, req, "http://example.com/") count = count + 1 - except urllib.error.HTTPError: + except urllib.error.HTTPError as err: # don't stop until max_repeats, because cookies may introduce state self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) + err.close() # detect endless non-repeating chain of redirects req = Request(from_url, origin_req_host="example.com") @@ -1252,9 +1297,10 @@ def redirect(h, req, url=to_url): while 1: redirect(h, req, "http://example.com/%d" % count) count = count + 1 - except urllib.error.HTTPError: + except urllib.error.HTTPError as err: self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_redirections) + err.close() def test_invalid_redirect(self): from_url = "http://example.com/a.html" @@ -1268,9 +1314,11 @@ def test_invalid_redirect(self): for scheme in invalid_schemes: invalid_url = scheme + '://' + schemeless_url - self.assertRaises(urllib.error.HTTPError, h.http_error_302, + with self.assertRaises(urllib.error.HTTPError) as cm: + h.http_error_302( req, MockFile(), 302, "Security Loophole", MockHeaders({"location": invalid_url})) + cm.exception.close() for scheme in valid_schemes: valid_url = scheme + '://' + schemeless_url @@ -1291,8 +1339,6 @@ def test_relative_redirect(self): MockHeaders({"location": valid_url})) self.assertEqual(o.req.get_full_url(), valid_url) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from http.cookiejar import CookieJar @@ -1300,7 +1346,7 @@ def test_cookie_redirect(self): cj = CookieJar() interact_netscape(cj, "http://www.example.com/", "spam=eggs") - hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n") + hh = MockHTTPHandlerRedirect(302, "Location: http://www.cracker.com/\r\n\r\n") hdeh = urllib.request.HTTPDefaultErrorHandler() hrh = urllib.request.HTTPRedirectHandler() cp = urllib.request.HTTPCookieProcessor(cj) @@ -1308,11 +1354,9 @@ def test_cookie_redirect(self): o.open("http://www.example.com/") self.assertFalse(hh.req.has_header("Cookie")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_redirect_fragment(self): redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' - hh = MockHTTPHandler(302, 'Location: ' + redirected_url) + hh = MockHTTPHandlerRedirect(302, 'Location: ' + redirected_url) hdeh = urllib.request.HTTPDefaultErrorHandler() hrh = urllib.request.HTTPRedirectHandler() o = build_test_opener(hh, hdeh, hrh) @@ -1372,10 +1416,17 @@ def http_open(self, req): response = opener.open('http://example.com/') expected = b'GET ' + result + b' ' request = handler.last_buf - self.assertTrue(request.startswith(expected), repr(request)) + self.assertStartsWith(request, expected) + + def test_redirect_head_request(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + req = Request(from_url, method="HEAD") + fp = MockFile() + new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) + self.assertEqual(new_req.get_method(), "HEAD") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy(self): u = "proxy.example.com:3128" for d in dict(http=u), dict(HTTP=u): @@ -1395,7 +1446,8 @@ def test_proxy(self): [tup[0:2] for tup in o.calls]) def test_proxy_no_proxy(self): - os.environ['no_proxy'] = 'python.org' + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env['no_proxy'] = 'python.org' o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) o.add_handler(ph) @@ -1407,10 +1459,10 @@ def test_proxy_no_proxy(self): self.assertEqual(req.host, "www.python.org") o.open(req) self.assertEqual(req.host, "www.python.org") - del os.environ['no_proxy'] def test_proxy_no_proxy_all(self): - os.environ['no_proxy'] = '*' + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env['no_proxy'] = '*' o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) o.add_handler(ph) @@ -1418,10 +1470,7 @@ def test_proxy_no_proxy_all(self): self.assertEqual(req.host, "www.python.org") o.open(req) self.assertEqual(req.host, "www.python.org") - del os.environ['no_proxy'] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_https(self): o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) @@ -1438,6 +1487,7 @@ def test_proxy_https(self): self.assertEqual([(handlers[0], "https_open")], [tup[0:2] for tup in o.calls]) + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') def test_proxy_https_proxy_authorization(self): o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128')) @@ -1461,6 +1511,30 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") + def test_winreg_proxy_bypass(self): + proxy_override = "www.example.com;*.example.net; 192.168.0.1" + proxy_bypass = _proxy_bypass_winreg_override + for host in ("www.example.com", "www.example.net", "192.168.0.1"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expected bypass of %s to be true" % host) + + for host in ("example.com", "www.example.org", "example.net", + "192.168.0.2"): + self.assertFalse(proxy_bypass(host, proxy_override), + "expected bypass of %s to be False" % host) + + # check intranet address bypass + proxy_override = "example.com; <local>" + self.assertTrue(proxy_bypass("example.com", proxy_override), + "expected bypass of %s to be true" % host) + self.assertFalse(proxy_bypass("example.net", proxy_override), + "expected bypass of %s to be False" % host) + for host in ("test", "localhost"): + self.assertTrue(proxy_bypass(host, proxy_override), + "expect <local> to bypass intranet address '%s'" + % host) + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { @@ -1501,7 +1575,7 @@ def check_basic_auth(self, headers, realm): password_manager = MockPasswordManager() auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) body = '\r\n'.join(headers) + '\r\n\r\n' - http_handler = MockHTTPHandler(401, body) + http_handler = MockHTTPHandlerRedirect(401, body) opener.add_handler(auth_handler) opener.add_handler(http_handler) self._test_basic_auth(opener, auth_handler, "Authorization", @@ -1509,8 +1583,6 @@ def check_basic_auth(self, headers, realm): "http://acme.example.com/protected", "http://acme.example.com/protected") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_auth(self): realm = "realm2@example.com" realm2 = "realm2@example.com" @@ -1556,8 +1628,6 @@ def test_basic_auth(self): for challenge in challenges] self.check_basic_auth(headers, realm) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_basic_auth(self): opener = OpenerDirector() ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) @@ -1565,7 +1635,7 @@ def test_proxy_basic_auth(self): password_manager = MockPasswordManager() auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager) realm = "ACME Networks" - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) opener.add_handler(auth_handler) opener.add_handler(http_handler) @@ -1575,15 +1645,13 @@ def test_proxy_basic_auth(self): "proxy.example.com:3128", ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_and_digest_auth_handlers(self): # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* - # response (http://python.org/sf/1479302), where it should instead + # response (https://bugs.python.org/issue1479302), where it should instead # return None to allow another handler (especially # HTTPBasicAuthHandler) to handle the response. - # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must + # Also (https://bugs.python.org/issue14797027, RFC 2617 section 1.2), we must # try digest first (since it's the strongest auth scheme), so we record # order of calls here to check digest comes first: class RecordingOpenerDirector(OpenerDirector): @@ -1611,7 +1679,7 @@ def http_error_401(self, *args, **kwds): digest_handler = TestDigestAuthHandler(password_manager) basic_handler = TestBasicAuthHandler(password_manager) realm = "ACME Networks" - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) opener.add_handler(basic_handler) opener.add_handler(digest_handler) @@ -1631,7 +1699,7 @@ def test_unsupported_auth_digest_handler(self): opener = OpenerDirector() # While using DigestAuthHandler digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 401, 'WWW-Authenticate: Kerberos\r\n\r\n') opener.add_handler(digest_auth_handler) opener.add_handler(http_handler) @@ -1641,7 +1709,7 @@ def test_unsupported_auth_basic_handler(self): # While using BasicAuthHandler opener = OpenerDirector() basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 401, 'WWW-Authenticate: NTLM\r\n\r\n') opener.add_handler(basic_auth_handler) opener.add_handler(http_handler) @@ -1684,8 +1752,6 @@ def _test_basic_auth(self, opener, auth_handler, auth_header, self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_prior_auth_auto_send(self): # Assume already authenticated if is_authenticated=True # for APIs like Github that don't return 401 @@ -1713,8 +1779,6 @@ def test_basic_prior_auth_auto_send(self): # expect request to be sent with auth header self.assertTrue(http_handler.has_auth_header) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_prior_auth_send_after_first_success(self): # Auto send auth header after authentication is successful once @@ -1732,7 +1796,7 @@ def test_basic_prior_auth_send_after_first_success(self): opener = OpenerDirector() opener.add_handler(auth_prior_handler) - http_handler = MockHTTPHandler( + http_handler = MockHTTPHandlerRedirect( 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None) opener.add_handler(http_handler) @@ -1842,14 +1906,21 @@ def test_HTTPError_interface(self): url = code = fp = None hdrs = 'Content-Length: 42' err = urllib.error.HTTPError(url, code, msg, hdrs, fp) - self.assertTrue(hasattr(err, 'reason')) + self.assertHasAttr(err, 'reason') self.assertEqual(err.reason, 'something bad happened') - self.assertTrue(hasattr(err, 'headers')) + self.assertHasAttr(err, 'headers') self.assertEqual(err.headers, 'Content-Length: 42') expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) self.assertEqual(str(err), expected_errmsg) expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) self.assertEqual(repr(err), expected_errmsg) + err.close() + + def test_gh_98778(self): + x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) + self.assertEqual(getattr(x, "__notes__", ()), ()) + self.assertIsInstance(x.fp.read(), bytes) + x.close() def test_parse_proxy(self): parse_proxy_test_cases = [ @@ -1896,10 +1967,38 @@ def test_parse_proxy(self): self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), - def test_unsupported_algorithm(self): - handler = AbstractDigestAuthHandler() + +skip_libssl_fips_mode = unittest.skipIf( + support.is_libssl_fips_mode(), + "conservative skip due to OpenSSL FIPS mode possible algorithm nerfing", +) + + +class TestDigestAuthAlgorithms(unittest.TestCase): + def setUp(self): + self.handler = AbstractDigestAuthHandler() + + @skip_libssl_fips_mode + def test_md5_algorithm(self): + H, KD = self.handler.get_algorithm_impls('MD5') + self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8") + self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732") + + @skip_libssl_fips_mode + def test_sha_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA') + self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de") + + @skip_libssl_fips_mode + def test_sha256_algorithm(self): + H, KD = self.handler.get_algorithm_impls('SHA-256') + self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") + self.assertEqual(KD("foo", "bar"), "a765a8beaa9d561d4c5cbed29d8f4e30870297fdfa9cb7d6e9848a95fec9f937") + + def test_invalid_algorithm(self): with self.assertRaises(ValueError) as exc: - handler.get_algorithm_impls('invalid') + self.handler.get_algorithm_impls('invalid') self.assertEqual( str(exc.exception), "Unsupported digest authentication algorithm 'invalid'" diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 3d5ca2faab7..87da54d5384 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -8,20 +8,22 @@ import unittest import hashlib +from test import support from test.support import hashlib_helper from test.support import threading_helper -from test.support import warnings_helper try: import ssl except ImportError: ssl = None +support.requires_working_socket(module=True) + here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' -CERT_localhost = os.path.join(here, 'keycert.pem') +CERT_localhost = os.path.join(here, 'certdata', 'keycert.pem') # Self-signed cert file for 'fakehostname' -CERT_fakehostname = os.path.join(here, 'keycert2.pem') +CERT_fakehostname = os.path.join(here, 'certdata', 'keycert2.pem') # Loopback http server infrastructure @@ -314,7 +316,9 @@ def test_basic_auth_httperror(self): ah = urllib.request.HTTPBasicAuthHandler() ah.add_password(self.REALM, self.server_url, self.USER, self.INCORRECT_PASSWD) urllib.request.install_opener(urllib.request.build_opener(ah)) - self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, self.server_url) + with self.assertRaises(urllib.error.HTTPError) as cm: + urllib.request.urlopen(self.server_url) + cm.exception.close() @hashlib_helper.requires_hashdigest("md5", openssl=True) @@ -356,23 +360,22 @@ def stop_server(self): self.server.stop() self.server = None - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_proxy_with_bad_password_raises_httperror(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, self.USER, self.PASSWD+"bad") self.digest_auth_handler.set_qop("auth") - self.assertRaises(urllib.error.HTTPError, - self.opener.open, - self.URL) + with self.assertRaises(urllib.error.HTTPError) as cm: + self.opener.open(self.URL) + cm.exception.close() + - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") def test_proxy_with_no_password_raises_httperror(self): self.digest_auth_handler.set_qop("auth") - self.assertRaises(urllib.error.HTTPError, - self.opener.open, - self.URL) + with self.assertRaises(urllib.error.HTTPError) as cm: + self.opener.open(self.URL) + cm.exception.close() - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") def test_proxy_qop_auth_works(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, self.USER, self.PASSWD) @@ -381,7 +384,7 @@ def test_proxy_qop_auth_works(self): while result.read(): pass - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_proxy_qop_auth_int_works_or_throws_urlerror(self): self.proxy_digest_handler.add_password(self.REALM, self.URL, self.USER, self.PASSWD) @@ -506,7 +509,7 @@ def start_https_server(self, responses=None, **kwargs): handler.port = server.port return handler - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_redirection(self): expected_response = b"We got here..." responses = [ @@ -520,7 +523,7 @@ def test_redirection(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/", "/somewhere_else"]) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_chunked(self): expected_response = b"hello world" chunked_start = ( @@ -535,7 +538,7 @@ def test_chunked(self): data = self.urlopen("http://localhost:%s/" % handler.port) self.assertEqual(data, expected_response) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_404(self): expected_response = b"Bad bad bad..." handler = self.start_server([(404, [], expected_response)]) @@ -551,7 +554,7 @@ def test_404(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/weeble"]) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_200(self): expected_response = b"pycon 2008..." handler = self.start_server([(200, [], expected_response)]) @@ -559,7 +562,7 @@ def test_200(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/bizarre"]) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_200_with_parameters(self): expected_response = b"pycon 2008..." handler = self.start_server([(200, [], expected_response)]) @@ -568,47 +571,14 @@ def test_200_with_parameters(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_https(self): handler = self.start_https_server() context = ssl.create_default_context(cafile=CERT_localhost) data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context) self.assertEqual(data, b"we care a bit") - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") - def test_https_with_cafile(self): - handler = self.start_https_server(certfile=CERT_localhost) - with warnings_helper.check_warnings(('', DeprecationWarning)): - # Good cert - data = self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_localhost) - self.assertEqual(data, b"we care a bit") - # Bad cert - with self.assertRaises(urllib.error.URLError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_fakehostname) - # Good cert, but mismatching hostname - handler = self.start_https_server(certfile=CERT_fakehostname) - with self.assertRaises(urllib.error.URLError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cafile=CERT_fakehostname) - - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") - def test_https_with_cadefault(self): - handler = self.start_https_server(certfile=CERT_localhost) - # Self-signed cert should fail verification with system certificate store - with warnings_helper.check_warnings(('', DeprecationWarning)): - with self.assertRaises(urllib.error.URLError) as cm: - self.urlopen("https://localhost:%s/bizarre" % handler.port, - cadefault=True) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_https_sni(self): if ssl is None: self.skipTest("ssl module required") @@ -625,7 +595,7 @@ def cb_sni(ssl_sock, server_name, initial_context): self.urlopen("https://localhost:%s" % handler.port, context=context) self.assertEqual(sni_name, "localhost") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_sending_headers(self): handler = self.start_server() req = urllib.request.Request("http://localhost:%s/" % handler.port, @@ -634,7 +604,7 @@ def test_sending_headers(self): pass self.assertEqual(handler.headers_received["Range"], "bytes=20-39") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_sending_headers_camel(self): handler = self.start_server() req = urllib.request.Request("http://localhost:%s/" % handler.port, @@ -644,16 +614,15 @@ def test_sending_headers_camel(self): self.assertIn("X-Some-Header", handler.headers_received.keys()) self.assertNotIn("X-SoMe-hEader", handler.headers_received.keys()) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_basic(self): handler = self.start_server() with urllib.request.urlopen("http://localhost:%s" % handler.port) as open_url: for attr in ("read", "close", "info", "geturl"): - self.assertTrue(hasattr(open_url, attr), "object returned from " - "urlopen lacks the %s attribute" % attr) + self.assertHasAttr(open_url, attr) self.assertTrue(open_url.read(), "calling 'read' failed") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_info(self): handler = self.start_server() open_url = urllib.request.urlopen( @@ -665,7 +634,7 @@ def test_info(self): "instance of email.message.Message") self.assertEqual(info_obj.get_content_subtype(), "plain") - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_geturl(self): # Make sure same URL as opened is returned by geturl. handler = self.start_server() @@ -674,7 +643,7 @@ def test_geturl(self): url = open_url.geturl() self.assertEqual(url, "http://localhost:%s" % handler.port) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_iteration(self): expected_response = b"pycon 2008..." handler = self.start_server([(200, [], expected_response)]) @@ -682,7 +651,7 @@ def test_iteration(self): for line in data: self.assertEqual(line, expected_response) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_line_iteration(self): lines = [b"We\n", b"got\n", b"here\n", b"verylong " * 8192 + b"\n"] expected_response = b"".join(lines) @@ -695,7 +664,7 @@ def test_line_iteration(self): (index, len(lines[index]), len(line))) self.assertEqual(index + 1, len(lines)) - @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") + def test_issue16464(self): # See https://bugs.python.org/issue16464 # and https://bugs.python.org/issue46648 diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index a7e7c9f0b91..d015267cefd 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -1,10 +1,13 @@ +import contextlib import errno +import sysconfig import unittest +from unittest import mock from test import support from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied -from test.test_urllib2 import sanepathname2url +from test.support.warnings_helper import check_no_resource_warning import os import socket @@ -29,13 +32,6 @@ def wrapped(*args, **kwargs): return _retry_thrice(func, exc, *args, **kwargs) return wrapped -# bpo-35411: FTP tests of test_urllib2net randomly fail -# with "425 Security: Bad IP connecting" on Travis CI -skip_ftp_test_on_travis = unittest.skipIf('TRAVIS' in os.environ, - 'bpo-35411: skip FTP test ' - 'on Travis CI') - - # Connecting to remote hosts is flaky. Make it more robust by retrying # the connection several times. _urlopen_with_retry = _wrap_with_retry_thrice(urllib.request.urlopen, @@ -140,15 +136,54 @@ def setUp(self): # XXX The rest of these tests aren't very good -- they don't check much. # They do sometimes catch some major disasters, though. - @skip_ftp_test_on_travis + @support.requires_resource('walltime') def test_ftp(self): + # Testing the same URL twice exercises the caching in CacheFTPHandler urls = [ + 'ftp://www.pythontest.net/README', 'ftp://www.pythontest.net/README', ('ftp://www.pythontest.net/non-existent-file', None, urllib.error.URLError), ] self._test_urls(urls, self._extra_handlers()) + @support.requires_resource('walltime') + @unittest.skipIf(sysconfig.get_platform() == 'linux-ppc64le', + 'leaks on PPC64LE (gh-140691)') + def test_ftp_no_leak(self): + # gh-140691: When the data connection (but not control connection) + # cannot be made established, we shouldn't leave an open socket object. + + class MockError(OSError): + pass + + orig_create_connection = socket.create_connection + def patched_create_connection(address, *args, **kwargs): + """Simulate REJECTing connections to ports other than 21""" + host, port = address + if port != 21: + raise MockError() + return orig_create_connection(address, *args, **kwargs) + + url = 'ftp://www.pythontest.net/README' + entry = url, None, urllib.error.URLError + no_cache_handlers = [urllib.request.FTPHandler()] + cache_handlers = self._extra_handlers() + with mock.patch('socket.create_connection', patched_create_connection): + with check_no_resource_warning(self): + # Try without CacheFTPHandler + self._test_urls([entry], handlers=no_cache_handlers, + retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (uncached) + self._test_urls([entry], cache_handlers, retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (cached) + self._test_urls([entry], cache_handlers, retry=False) + # Try without the mock: the handler should not use a closed connection + with check_no_resource_warning(self): + self._test_urls([url], cache_handlers, retry=False) + def test_file(self): TESTFN = os_helper.TESTFN f = open(TESTFN, 'w') @@ -156,7 +191,7 @@ def test_file(self): f.write('hi there\n') f.close() urls = [ - 'file:' + sanepathname2url(os.path.abspath(TESTFN)), + urllib.request.pathname2url(os.path.abspath(TESTFN), add_scheme=True), ('file:///nonsensename/etc/passwd', None, urllib.error.URLError), ] @@ -202,6 +237,7 @@ def test_urlwithfrag(self): self.assertEqual(res.geturl(), "http://www.pythontest.net/index.html#frag") + @support.requires_resource('walltime') def test_redirect_url_withfrag(self): redirect_url_with_frag = "http://www.pythontest.net/redir/with_frag/" with socket_helper.transient_internet(redirect_url_with_frag): @@ -260,18 +296,16 @@ def _test_urls(self, urls, handlers, retry=True): else: req = expected_err = None + if expected_err: + context = self.assertRaises(expected_err) + else: + context = contextlib.nullcontext() + with socket_helper.transient_internet(url): - try: + f = None + with context: f = urlopen(url, req, support.INTERNET_TIMEOUT) - # urllib.error.URLError is a subclass of OSError - except OSError as err: - if expected_err: - msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % - (expected_err, url, req, type(err), err)) - self.assertIsInstance(err, expected_err, msg) - else: - raise - else: + if f is not None: try: with time_out, \ socket_peer_reset, \ @@ -340,7 +374,7 @@ def test_http_timeout(self): FTP_HOST = 'ftp://www.pythontest.net/' - @skip_ftp_test_on_travis + @support.requires_resource('walltime') def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) with socket_helper.transient_internet(self.FTP_HOST, timeout=None): @@ -348,7 +382,6 @@ def test_ftp_basic(self): self.addCleanup(u.close) self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) - @skip_ftp_test_on_travis def test_ftp_default_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) with socket_helper.transient_internet(self.FTP_HOST): @@ -360,7 +393,7 @@ def test_ftp_default_timeout(self): socket.setdefaulttimeout(None) self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) - @skip_ftp_test_on_travis + @support.requires_resource('walltime') def test_ftp_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) with socket_helper.transient_internet(self.FTP_HOST): @@ -372,7 +405,7 @@ def test_ftp_no_timeout(self): socket.setdefaulttimeout(None) self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) - @skip_ftp_test_on_travis + @support.requires_resource('walltime') def test_ftp_timeout(self): with socket_helper.transient_internet(self.FTP_HOST): u = _urlopen_with_retry(self.FTP_HOST, timeout=60) diff --git a/Lib/test/test_urllib_response.py b/Lib/test/test_urllib_response.py index 73d2ef0424f..d949fa38bfc 100644 --- a/Lib/test/test_urllib_response.py +++ b/Lib/test/test_urllib_response.py @@ -4,6 +4,11 @@ import tempfile import urllib.response import unittest +from test import support + +if support.is_wasi: + raise unittest.SkipTest("Cannot create socket on WASI") + class TestResponse(unittest.TestCase): @@ -43,6 +48,7 @@ def test_addinfo(self): info = urllib.response.addinfo(self.fp, self.test_headers) self.assertEqual(info.info(), self.test_headers) self.assertEqual(info.headers, self.test_headers) + info.close() def test_addinfourl(self): url = "http://www.python.org" @@ -55,6 +61,7 @@ def test_addinfourl(self): self.assertEqual(infourl.headers, self.test_headers) self.assertEqual(infourl.url, url) self.assertEqual(infourl.status, code) + infourl.close() def tearDown(self): self.sock.close() diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 773101ce41f..1a42c35dc49 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -5,6 +5,7 @@ import contextlib import socket +import urllib.error import urllib.parse import urllib.request import os @@ -70,8 +71,7 @@ def test_basic(self): with self.urlopen(self.url) as open_url: for attr in ("read", "readline", "readlines", "fileno", "close", "info", "geturl"): - self.assertTrue(hasattr(open_url, attr), "object returned from " - "urlopen lacks the %s attribute" % attr) + self.assertHasAttr(open_url, attr) self.assertTrue(open_url.read(), "calling 'read' failed") def test_readlines(self): @@ -101,14 +101,13 @@ def test_getcode(self): # test getcode() with the fancy opener to get 404 error codes URL = self.url + "XXXinvalidXXX" with socket_helper.transient_internet(URL): - with self.assertWarns(DeprecationWarning): - open_url = urllib.request.FancyURLopener().open(URL) - try: - code = open_url.getcode() - finally: - open_url.close() - self.assertEqual(code, 404) + with self.assertRaises(urllib.error.URLError) as e: + with urllib.request.urlopen(URL): + pass + self.assertEqual(e.exception.code, 404) + e.exception.close() + @support.requires_resource('walltime') def test_bad_address(self): # Make sure proper exception is raised when connecting to a bogus # address. @@ -191,6 +190,7 @@ def test_header(self): logo = "http://www.pythontest.net/" + @support.requires_resource('walltime') def test_data_header(self): with self.urlretrieve(self.logo) as (file_location, fileheaders): datevalue = fileheaders.get('Date') diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index af6fe99fb51..b2bde5a9b1d 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -2,6 +2,7 @@ import unicodedata import unittest import urllib.parse +from test import support RFC1808_BASE = "http://a/b/c/d;p?q#f" RFC2396_BASE = "http://a/b/c/d;p?q" @@ -19,6 +20,10 @@ ("=a", [('', 'a')]), ("a", [('a', '')]), ("a=", [('a', '')]), + ("a=b=c", [('a', 'b=c')]), + ("a%3Db=c", [('a=b', 'c')]), + ("a=b&c=d", [('a', 'b'), ('c', 'd')]), + ("a=b%26c=d", [('a', 'b&c=d')]), ("&a=b", [('a', 'b')]), ("a=a+b&b=b+c", [('a', 'a b'), ('b', 'b c')]), ("a=1&a=2", [('a', '1'), ('a', '2')]), @@ -29,6 +34,10 @@ (b"=a", [(b'', b'a')]), (b"a", [(b'a', b'')]), (b"a=", [(b'a', b'')]), + (b"a=b=c", [(b'a', b'b=c')]), + (b"a%3Db=c", [(b'a=b', b'c')]), + (b"a=b&c=d", [(b'a', b'b'), (b'c', b'd')]), + (b"a=b%26c=d", [(b'a', b'b&c=d')]), (b"&a=b", [(b'a', b'b')]), (b"a=a+b&b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1&a=2", [(b'a', b'1'), (b'a', b'2')]), @@ -36,6 +45,14 @@ ("a=a+b;b=b+c", [('a', 'a b;b=b c')]), (b";a=b", [(b';a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b;b=b c')]), + + ("\u0141=\xE9", [('\u0141', '\xE9')]), + ("%C5%81=%C3%A9", [('\u0141', '\xE9')]), + ("%81=%A9", [('\ufffd', '\ufffd')]), + (b"\xc5\x81=\xc3\xa9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"%C5%81=%C3%A9", [(b'\xc5\x81', b'\xc3\xa9')]), + (b"\x81=\xA9", [(b'\x81', b'\xa9')]), + (b"%81=%A9", [(b'\x81', b'\xa9')]), ] # Each parse_qs testcase is a two-tuple that contains @@ -49,6 +66,10 @@ ("=a", {'': ['a']}), ("a", {'a': ['']}), ("a=", {'a': ['']}), + ("a=b=c", {'a': ['b=c']}), + ("a%3Db=c", {'a=b': ['c']}), + ("a=b&c=d", {'a': ['b'], 'c': ['d']}), + ("a=b%26c=d", {'a': ['b&c=d']}), ("&a=b", {'a': ['b']}), ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}), ("a=1&a=2", {'a': ['1', '2']}), @@ -59,6 +80,10 @@ (b"=a", {b'': [b'a']}), (b"a", {b'a': [b'']}), (b"a=", {b'a': [b'']}), + (b"a=b=c", {b'a': [b'b=c']}), + (b"a%3Db=c", {b'a=b': [b'c']}), + (b"a=b&c=d", {b'a': [b'b'], b'c': [b'd']}), + (b"a=b%26c=d", {b'a': [b'b&c=d']}), (b"&a=b", {b'a': [b'b']}), (b"a=a+b&b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1&a=2", {b'a': [b'1', b'2']}), @@ -66,26 +91,37 @@ ("a=a+b;b=b+c", {'a': ['a b;b=b c']}), (b";a=b", {b';a': [b'b']}), (b"a=a+b;b=b+c", {b'a':[ b'a b;b=b c']}), + (b"a=a%E2%80%99b", {b'a': [b'a\xe2\x80\x99b']}), + + ("\u0141=\xE9", {'\u0141': ['\xE9']}), + ("%C5%81=%C3%A9", {'\u0141': ['\xE9']}), + ("%81=%A9", {'\ufffd': ['\ufffd']}), + (b"\xc5\x81=\xc3\xa9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"%C5%81=%C3%A9", {b'\xc5\x81': [b'\xc3\xa9']}), + (b"\x81=\xA9", {b'\x81': [b'\xa9']}), + (b"%81=%A9", {b'\x81': [b'\xa9']}), ] class UrlParseTestCase(unittest.TestCase): - def checkRoundtrips(self, url, parsed, split): + def checkRoundtrips(self, url, parsed, split, url2=None): + if url2 is None: + url2 = url result = urllib.parse.urlparse(url) - self.assertEqual(result, parsed) + self.assertSequenceEqual(result, parsed) t = (result.scheme, result.netloc, result.path, result.params, result.query, result.fragment) - self.assertEqual(t, parsed) + self.assertSequenceEqual(t, parsed) # put it back together and it should be the same result2 = urllib.parse.urlunparse(result) - self.assertEqual(result2, url) - self.assertEqual(result2, result.geturl()) + self.assertSequenceEqual(result2, url2) + self.assertSequenceEqual(result2, result.geturl()) # the result of geturl() is a fixpoint; we can always parse it # again to get the same result: result3 = urllib.parse.urlparse(result.geturl()) self.assertEqual(result3.geturl(), result.geturl()) - self.assertEqual(result3, result) + self.assertSequenceEqual(result3, result) self.assertEqual(result3.scheme, result.scheme) self.assertEqual(result3.netloc, result.netloc) self.assertEqual(result3.path, result.path) @@ -99,18 +135,18 @@ def checkRoundtrips(self, url, parsed, split): # check the roundtrip using urlsplit() as well result = urllib.parse.urlsplit(url) - self.assertEqual(result, split) + self.assertSequenceEqual(result, split) t = (result.scheme, result.netloc, result.path, result.query, result.fragment) - self.assertEqual(t, split) + self.assertSequenceEqual(t, split) result2 = urllib.parse.urlunsplit(result) - self.assertEqual(result2, url) - self.assertEqual(result2, result.geturl()) + self.assertSequenceEqual(result2, url2) + self.assertSequenceEqual(result2, result.geturl()) # check the fixpoint property of re-parsing the result of geturl() result3 = urllib.parse.urlsplit(result.geturl()) self.assertEqual(result3.geturl(), result.geturl()) - self.assertEqual(result3, result) + self.assertSequenceEqual(result3, result) self.assertEqual(result3.scheme, result.scheme) self.assertEqual(result3.netloc, result.netloc) self.assertEqual(result3.path, result.path) @@ -121,30 +157,79 @@ def checkRoundtrips(self, url, parsed, split): self.assertEqual(result3.hostname, result.hostname) self.assertEqual(result3.port, result.port) - def test_qsl(self): - for orig, expect in parse_qsl_test_cases: - result = urllib.parse.parse_qsl(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = [v for v in expect if len(v[1])] - result = urllib.parse.parse_qsl(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) - - def test_qs(self): - for orig, expect in parse_qs_test_cases: - result = urllib.parse.parse_qs(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = {v: expect[v] - for v in expect if len(expect[v][0])} - result = urllib.parse.parse_qs(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) - - def test_roundtrips(self): - str_cases = [ + @support.subTests('orig,expect', parse_qsl_test_cases) + def test_qsl(self, orig, expect): + result = urllib.parse.parse_qsl(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = [v for v in expect if len(v[1])] + result = urllib.parse.parse_qsl(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) + + @support.subTests('orig,expect', parse_qs_test_cases) + def test_qs(self, orig, expect): + result = urllib.parse.parse_qs(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = {v: expect[v] + for v in expect if len(expect[v][0])} + result = urllib.parse.parse_qs(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,parsed,split', [ + ('path/to/file', + ('', '', 'path/to/file', '', '', ''), + ('', '', 'path/to/file', '', '')), + ('/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('//path/to/file', + ('', 'path', '/to/file', '', '', ''), + ('', 'path', '/to/file', '', '')), + ('////path/to/file', + ('', '', '//path/to/file', '', '', ''), + ('', '', '//path/to/file', '', '')), + ('/////path/to/file', + ('', '', '///path/to/file', '', '', ''), + ('', '', '///path/to/file', '', '')), + ('scheme:path/to/file', + ('scheme', '', 'path/to/file', '', '', ''), + ('scheme', '', 'path/to/file', '', '')), + ('scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('scheme://path/to/file', + ('scheme', 'path', '/to/file', '', '', ''), + ('scheme', 'path', '/to/file', '', '')), + ('scheme:////path/to/file', + ('scheme', '', '//path/to/file', '', '', ''), + ('scheme', '', '//path/to/file', '', '')), + ('scheme://///path/to/file', + ('scheme', '', '///path/to/file', '', '', ''), + ('scheme', '', '///path/to/file', '', '')), + ('file:tmp/junk.txt', + ('file', '', 'tmp/junk.txt', '', '', ''), + ('file', '', 'tmp/junk.txt', '', '')), ('file:///tmp/junk.txt', ('file', '', '/tmp/junk.txt', '', '', ''), ('file', '', '/tmp/junk.txt', '', '')), + ('file:////tmp/junk.txt', + ('file', '', '//tmp/junk.txt', '', '', ''), + ('file', '', '//tmp/junk.txt', '', '')), + ('file://///tmp/junk.txt', + ('file', '', '///tmp/junk.txt', '', '', ''), + ('file', '', '///tmp/junk.txt', '', '')), + ('http:tmp/junk.txt', + ('http', '', 'tmp/junk.txt', '', '', ''), + ('http', '', 'tmp/junk.txt', '', '')), + ('http://example.com/tmp/junk.txt', + ('http', 'example.com', '/tmp/junk.txt', '', '', ''), + ('http', 'example.com', '/tmp/junk.txt', '', '')), + ('http:///example.com/tmp/junk.txt', + ('http', '', '/example.com/tmp/junk.txt', '', '', ''), + ('http', '', '/example.com/tmp/junk.txt', '', '')), + ('http:////example.com/tmp/junk.txt', + ('http', '', '//example.com/tmp/junk.txt', '', '', ''), + ('http', '', '//example.com/tmp/junk.txt', '', '')), ('imap://mail.python.org/mbox1', ('imap', 'mail.python.org', '/mbox1', '', '', ''), ('imap', 'mail.python.org', '/mbox1', '', '')), @@ -162,24 +247,68 @@ def test_roundtrips(self): ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/', '', '')), ('git+ssh://git@github.com/user/project.git', - ('git+ssh', 'git@github.com','/user/project.git', - '','',''), - ('git+ssh', 'git@github.com','/user/project.git', - '', '')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - for url, parsed, split in str_cases + bytes_cases: - self.checkRoundtrips(url, parsed, split) - - def test_http_roundtrips(self): - # urllib.parse.urlsplit treats 'http:' as an optimized special case, - # so we test both 'http:' and 'https:' in all the following. - # Three cheers for white box knowledge! - str_cases = [ + ('git+ssh', 'git@github.com','/user/project.git', + '','',''), + ('git+ssh', 'git@github.com','/user/project.git', + '', '')), + ('itms-services://?action=download-manifest&url=https://example.com/app', + ('itms-services', '', '', '', + 'action=download-manifest&url=https://example.com/app', ''), + ('itms-services', '', '', + 'action=download-manifest&url=https://example.com/app', '')), + ('+scheme:path/to/file', + ('', '', '+scheme:path/to/file', '', '', ''), + ('', '', '+scheme:path/to/file', '', '')), + ('sch_me:path/to/file', + ('', '', 'sch_me:path/to/file', '', '', ''), + ('', '', 'sch_me:path/to/file', '', '')), + ('schème:path/to/file', + ('', '', 'schème:path/to/file', '', '', ''), + ('', '', 'schème:path/to/file', '', '')), + ]) + def test_roundtrips(self, bytes, url, parsed, split): + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,url2,parsed,split', [ + ('///path/to/file', + '/path/to/file', + ('', '', '/path/to/file', '', '', ''), + ('', '', '/path/to/file', '', '')), + ('scheme:///path/to/file', + 'scheme:/path/to/file', + ('scheme', '', '/path/to/file', '', '', ''), + ('scheme', '', '/path/to/file', '', '')), + ('file:/tmp/junk.txt', + 'file:///tmp/junk.txt', + ('file', '', '/tmp/junk.txt', '', '', ''), + ('file', '', '/tmp/junk.txt', '', '')), + ('http:/tmp/junk.txt', + 'http:///tmp/junk.txt', + ('http', '', '/tmp/junk.txt', '', '', ''), + ('http', '', '/tmp/junk.txt', '', '')), + ('https:/tmp/junk.txt', + 'https:///tmp/junk.txt', + ('https', '', '/tmp/junk.txt', '', '', ''), + ('https', '', '/tmp/junk.txt', '', '')), + ]) + def test_roundtrips_normalization(self, bytes, url, url2, parsed, split): + if bytes: + url = str_encode(url) + url2 = str_encode(url2) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split, url2) + + @support.subTests('bytes', (False, True)) + @support.subTests('scheme', ('http', 'https')) + @support.subTests('url,parsed,split', [ ('://www.python.org', ('www.python.org', '', '', '', ''), ('www.python.org', '', '', '')), @@ -195,37 +324,42 @@ def test_http_roundtrips(self): ('://a/b/c/d;p?q#f', ('a', '/b/c/d', 'p', 'q', 'f'), ('a', '/b/c/d;p', 'q', 'f')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - str_schemes = ('http', 'https') - bytes_schemes = (b'http', b'https') - str_tests = str_schemes, str_cases - bytes_tests = bytes_schemes, bytes_cases - for schemes, test_cases in (str_tests, bytes_tests): - for scheme in schemes: - for url, parsed, split in test_cases: - url = scheme + url - parsed = (scheme,) + parsed - split = (scheme,) + split - self.checkRoundtrips(url, parsed, split) - - def checkJoin(self, base, relurl, expected): - str_components = (base, relurl, expected) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) - bytes_components = baseb, relurlb, expectedb = [ - x.encode('ascii') for x in str_components] - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) - - def test_unparse_parse(self): - str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] - bytes_cases = [x.encode('ascii') for x in str_cases] - for u in str_cases + bytes_cases: - self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) - self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) + ]) + def test_http_roundtrips(self, bytes, scheme, url, parsed, split): + # urllib.parse.urlsplit treats 'http:' as an optimized special case, + # so we test both 'http:' and 'https:' in all the following. + # Three cheers for white box knowledge! + if bytes: + scheme = str_encode(scheme) + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + url = scheme + url + parsed = (scheme,) + parsed + split = (scheme,) + split + self.checkRoundtrips(url, parsed, split) + + def checkJoin(self, base, relurl, expected, *, relroundtrip=True): + with self.subTest(base=base, relurl=relurl): + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + baseb = base.encode('ascii') + relurlb = relurl.encode('ascii') + expectedb = expected.encode('ascii') + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + + if relroundtrip: + relurl = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurl)) + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + + @support.subTests('bytes', (False, True)) + @support.subTests('u', ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]) + def test_unparse_parse(self, bytes, u): + if bytes: + u = str_encode(u) + self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) + self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) def test_RFC1808(self): # "normal" cases from RFC 1808: @@ -384,8 +518,6 @@ def test_RFC3986(self): def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g:h','g:h') - self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') self.checkJoin(SIMPLE_BASE, 'g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, './g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, 'g/','http://a/b/c/g/') @@ -406,8 +538,6 @@ def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g/./h','http://a/b/c/g/h') self.checkJoin(SIMPLE_BASE, 'g/../h','http://a/b/c/h') self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') - self.checkJoin(SIMPLE_BASE, 'http:?y','http://a/b/c/d?y') self.checkJoin(SIMPLE_BASE, 'http:g?y','http://a/b/c/g?y') self.checkJoin(SIMPLE_BASE, 'http:g?y/./x','http://a/b/c/g?y/./x') self.checkJoin('http:///', '..','http:///') @@ -437,8 +567,127 @@ def test_urljoins(self): # issue 23703: don't duplicate filename self.checkJoin('a', 'b', 'b') - def test_RFC2732(self): - str_cases = [ + # Test with empty (but defined) components. + self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, ';', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, ';?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, ';#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, ';x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, '/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '//#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '//?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, '//;x', 'http://;x') + self.checkJoin(RFC1808_BASE, '///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//v', 'http://v') + # For backward compatibility with RFC1630, the scheme name is allowed + # to be present in a relative reference if it is the same as the base + # URI scheme. + self.checkJoin(RFC1808_BASE, 'http:', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http:#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http:?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http:;', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, 'http:;?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, 'http:;#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, 'http:;x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, 'http:/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http://#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http://?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http://;x', 'http://;x') + self.checkJoin(RFC1808_BASE, 'http:///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://v', 'http://v') + # Different scheme is not ignored. + self.checkJoin(RFC1808_BASE, 'https:', 'https:', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#', 'https:#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#z', 'https:#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?', 'https:?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?y', 'https:?y', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:;', 'https:;') + self.checkJoin(RFC1808_BASE, 'https:;x', 'https:;x') + + def test_urljoins_relative_base(self): + # According to RFC 3986, Section 5.1, a base URI must conform to + # the absolute-URI syntax rule (Section 4.3). But urljoin() lacks + # a context to establish missed components of the relative base URI. + # It still has to return a sensible result for backwards compatibility. + # The following tests are figments of the imagination and artifacts + # of the current implementation that are not based on any standard. + self.checkJoin('', '', '') + self.checkJoin('', '//', '//', relroundtrip=False) + self.checkJoin('', '//v', '//v') + self.checkJoin('', '//v/w', '//v/w') + self.checkJoin('', '/w', '/w') + self.checkJoin('', '///w', '///w', relroundtrip=False) + self.checkJoin('', 'w', 'w') + + self.checkJoin('//', '', '//') + self.checkJoin('//', '//', '//') + self.checkJoin('//', '//v', '//v') + self.checkJoin('//', '//v/w', '//v/w') + self.checkJoin('//', '/w', '///w') + self.checkJoin('//', '///w', '///w') + self.checkJoin('//', 'w', '///w') + + self.checkJoin('//a', '', '//a') + self.checkJoin('//a', '//', '//a') + self.checkJoin('//a', '//v', '//v') + self.checkJoin('//a', '//v/w', '//v/w') + self.checkJoin('//a', '/w', '//a/w') + self.checkJoin('//a', '///w', '//a/w') + self.checkJoin('//a', 'w', '//a/w') + + for scheme in '', 'http:': + self.checkJoin('http:', scheme + '', 'http:') + self.checkJoin('http:', scheme + '//', 'http:') + self.checkJoin('http:', scheme + '//v', 'http://v') + self.checkJoin('http:', scheme + '//v/w', 'http://v/w') + self.checkJoin('http:', scheme + '/w', 'http:/w') + self.checkJoin('http:', scheme + '///w', 'http:/w') + self.checkJoin('http:', scheme + 'w', 'http:/w') + + self.checkJoin('http://', scheme + '', 'http://') + self.checkJoin('http://', scheme + '//', 'http://') + self.checkJoin('http://', scheme + '//v', 'http://v') + self.checkJoin('http://', scheme + '//v/w', 'http://v/w') + self.checkJoin('http://', scheme + '/w', 'http:///w') + self.checkJoin('http://', scheme + '///w', 'http:///w') + self.checkJoin('http://', scheme + 'w', 'http:///w') + + self.checkJoin('http://a', scheme + '', 'http://a') + self.checkJoin('http://a', scheme + '//', 'http://a') + self.checkJoin('http://a', scheme + '//v', 'http://v') + self.checkJoin('http://a', scheme + '//v/w', 'http://v/w') + self.checkJoin('http://a', scheme + '/w', 'http://a/w') + self.checkJoin('http://a', scheme + '///w', 'http://a/w') + self.checkJoin('http://a', scheme + 'w', 'http://a/w') + + self.checkJoin('/b/c', '', '/b/c') + self.checkJoin('/b/c', '//', '/b/c') + self.checkJoin('/b/c', '//v', '//v') + self.checkJoin('/b/c', '//v/w', '//v/w') + self.checkJoin('/b/c', '/w', '/w') + self.checkJoin('/b/c', '///w', '/w') + self.checkJoin('/b/c', 'w', '/b/w') + + self.checkJoin('///b/c', '', '///b/c') + self.checkJoin('///b/c', '//', '///b/c') + self.checkJoin('///b/c', '//v', '//v') + self.checkJoin('///b/c', '//v/w', '//v/w') + self.checkJoin('///b/c', '/w', '///w') + self.checkJoin('///b/c', '///w', '///w') + self.checkJoin('///b/c', 'w', '///b/w') + + @support.subTests('bytes', (False, True)) + @support.subTests('url,hostname,port', [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432), ('http://[::1]:5432/foo/', '::1', 5432), @@ -469,26 +718,28 @@ def test_RFC2732(self): ('http://[::12.34.56.78]:/foo/', '::12.34.56.78', None), ('http://[::ffff:12.34.56.78]:/foo/', '::ffff:12.34.56.78', None), - ] - def _encode(t): - return t[0].encode('ascii'), t[1].encode('ascii'), t[2] - bytes_cases = [_encode(x) for x in str_cases] - for url, hostname, port in str_cases + bytes_cases: - urlparsed = urllib.parse.urlparse(url) - self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port)) - - str_cases = [ + ]) + def test_RFC2732(self, bytes, url, hostname, port): + if bytes: + url = str_encode(url) + hostname = str_encode(hostname) + urlparsed = urllib.parse.urlparse(url) + self.assertEqual((urlparsed.hostname, urlparsed.port), (hostname, port)) + + @support.subTests('bytes', (False, True)) + @support.subTests('invalid_url', [ 'http://::12.34.56.78]/', 'http://[::1/foo/', 'ftp://[::1/foo/bad]/bad', 'http://[::1/foo/bad]/bad', - 'http://[::ffff:12.34.56.78'] - bytes_cases = [x.encode('ascii') for x in str_cases] - for invalid_url in str_cases + bytes_cases: - self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) - - def test_urldefrag(self): - str_cases = [ + 'http://[::ffff:12.34.56.78']) + def test_RFC2732_invalid(self, bytes, invalid_url): + if bytes: + invalid_url = str_encode(invalid_url) + self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,defrag,frag', [ ('http://python.org#frag', 'http://python.org', 'frag'), ('http://python.org', 'http://python.org', ''), ('http://python.org/#frag', 'http://python.org/', 'frag'), @@ -499,16 +750,31 @@ def test_urldefrag(self): ('http://python.org/p?q', 'http://python.org/p?q', ''), (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'), (RFC2396_BASE, 'http://a/b/c/d;p?q', ''), - ] - def _encode(t): - return type(t)(x.encode('ascii') for x in t) - bytes_cases = [_encode(x) for x in str_cases] - for url, defrag, frag in str_cases + bytes_cases: - result = urllib.parse.urldefrag(url) - self.assertEqual(result.geturl(), url) - self.assertEqual(result, (defrag, frag)) - self.assertEqual(result.url, defrag) - self.assertEqual(result.fragment, frag) + ('http://a/b/c;p?q#f', 'http://a/b/c;p?q', 'f'), + ('http://a/b/c;p?q#', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?q', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?#f', 'http://a/b/c;p?', 'f'), + ('http://a/b/c;p#f', 'http://a/b/c;p', 'f'), + ('http://a/b/c;?q#f', 'http://a/b/c;?q', 'f'), + ('http://a/b/c?q#f', 'http://a/b/c?q', 'f'), + ('http:///b/c;p?q#f', 'http:///b/c;p?q', 'f'), + ('http:b/c;p?q#f', 'http:b/c;p?q', 'f'), + ('http:;?q#f', 'http:;?q', 'f'), + ('http:?q#f', 'http:?q', 'f'), + ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), + ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), + ]) + def test_urldefrag(self, bytes, url, defrag, frag): + if bytes: + url = str_encode(url) + defrag = str_encode(defrag) + frag = str_encode(frag) + result = urllib.parse.urldefrag(url) + hash = '#' if isinstance(url, str) else b'#' + self.assertEqual(result.geturl(), url.rstrip(hash)) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') @@ -649,21 +915,94 @@ def test_urlsplit_remove_unsafe_bytes(self): self.assertEqual(p.scheme, "http") self.assertEqual(p.geturl(), "http://www.python.org/javascript:alert('msg')/?query=something#fragment") - def test_attributes_bad_port(self): + def test_urlsplit_strip_url(self): + noise = bytes(range(0, 0x20 + 1)) + base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag" + + url = noise.decode("utf-8") + base_url + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, "http") + self.assertEqual(p.netloc, "User:Pass@www.python.org:080") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "query=yes") + self.assertEqual(p.fragment, "frag") + self.assertEqual(p.username, "User") + self.assertEqual(p.password, "Pass") + self.assertEqual(p.hostname, "www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), base_url) + + url = noise + base_url.encode("utf-8") + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, b"http") + self.assertEqual(p.netloc, b"User:Pass@www.python.org:080") + self.assertEqual(p.path, b"/doc/") + self.assertEqual(p.query, b"query=yes") + self.assertEqual(p.fragment, b"frag") + self.assertEqual(p.username, b"User") + self.assertEqual(p.password, b"Pass") + self.assertEqual(p.hostname, b"www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), base_url.encode("utf-8")) + + # Test that trailing space is preserved as some applications rely on + # this within query strings. + query_spaces_url = "https://www.python.org:88/doc/?query= " + p = urllib.parse.urlsplit(noise.decode("utf-8") + query_spaces_url) + self.assertEqual(p.scheme, "https") + self.assertEqual(p.netloc, "www.python.org:88") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "query= ") + self.assertEqual(p.port, 88) + self.assertEqual(p.geturl(), query_spaces_url) + + p = urllib.parse.urlsplit("www.pypi.org ") + # That "hostname" gets considered a "path" due to the + # trailing space and our existing logic... YUCK... + # and re-assembles via geturl aka unurlsplit into the original. + # django.core.validators.URLValidator (at least through v3.2) relies on + # this, for better or worse, to catch it in a ValidationError via its + # regular expressions. + # Here we test the basic round trip concept of such a trailing space. + self.assertEqual(urllib.parse.urlunsplit(p), "www.pypi.org ") + + # with scheme as cache-key + url = "//www.python.org/" + scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8") + for _ in range(2): + p = urllib.parse.urlsplit(url, scheme=scheme) + self.assertEqual(p.scheme, "https") + self.assertEqual(p.geturl(), "https://www.python.org/") + + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('port', ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६")) + def test_attributes_bad_port(self, bytes, parse, port): """Check handling of invalid ports.""" - for bytes in (False, True): - for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for port in ("foo", "1.5", "-1", "0x10"): - with self.subTest(bytes=bytes, parse=parse, port=port): - netloc = "www.example.net:" + port - url = "http://" + netloc - if bytes: - netloc = netloc.encode("ascii") - url = url.encode("ascii") - p = parse(url) - self.assertEqual(p.netloc, netloc) - with self.assertRaises(ValueError): - p.port + netloc = "www.example.net:" + port + url = "http://" + netloc + "/" + if bytes: + if not (netloc.isascii() and port.isascii()): + self.skipTest('non-ASCII bytes') + netloc = str_encode(netloc) + url = str_encode(url) + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port + + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('scheme', (".", "+", "-", "0", "http&", "६http")) + def test_attributes_bad_scheme(self, bytes, parse, scheme): + """Check handling of invalid schemes.""" + url = scheme + "://www.example.net" + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.scheme, b"" if bytes else "") def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it @@ -775,24 +1114,21 @@ def test_anyscheme(self): self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"), (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'')) - def test_default_scheme(self): + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_default_scheme(self, func): # Exercise the scheme parameter of urlparse() and urlsplit() - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - with self.subTest(function=func): - result = func("http://example.net/", "ftp") - self.assertEqual(result.scheme, "http") - result = func(b"http://example.net/", b"ftp") - self.assertEqual(result.scheme, b"http") - self.assertEqual(func("path", "ftp").scheme, "ftp") - self.assertEqual(func("path", scheme="ftp").scheme, "ftp") - self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") - self.assertEqual(func("path").scheme, "") - self.assertEqual(func(b"path").scheme, b"") - self.assertEqual(func(b"path", "").scheme, b"") - - def test_parse_fragments(self): - # Exercise the allow_fragments parameter of urlparse() and urlsplit() - tests = ( + result = func("http://example.net/", "ftp") + self.assertEqual(result.scheme, "http") + result = func(b"http://example.net/", b"ftp") + self.assertEqual(result.scheme, b"http") + self.assertEqual(func("path", "ftp").scheme, "ftp") + self.assertEqual(func("path", scheme="ftp").scheme, "ftp") + self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") + self.assertEqual(func("path").scheme, "") + self.assertEqual(func(b"path").scheme, b"") + self.assertEqual(func(b"path", "").scheme, b"") + + @support.subTests('url,attr,expected_frag', ( ("http:#frag", "path", "frag"), ("//example.net#frag", "path", "frag"), ("index.html#frag", "path", "frag"), @@ -803,25 +1139,24 @@ def test_parse_fragments(self): ("//abc#@frag", "path", "@frag"), ("//abc:80#@frag", "path", "@frag"), ("//abc#@frag:80", "path", "@frag:80"), - ) - for url, attr, expected_frag in tests: - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - if attr == "params" and func is urllib.parse.urlsplit: - attr = "path" - with self.subTest(url=url, function=func): - result = func(url, allow_fragments=False) - self.assertEqual(result.fragment, "") - self.assertTrue( - getattr(result, attr).endswith("#" + expected_frag)) - self.assertEqual(func(url, "", False).fragment, "") - - result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, expected_frag) - self.assertFalse( - getattr(result, attr).endswith(expected_frag)) - self.assertEqual(func(url, "", True).fragment, - expected_frag) - self.assertEqual(func(url).fragment, expected_frag) + )) + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_parse_fragments(self, url, attr, expected_frag, func): + # Exercise the allow_fragments parameter of urlparse() and urlsplit() + if attr == "params" and func is urllib.parse.urlsplit: + attr = "path" + result = func(url, allow_fragments=False) + self.assertEqual(result.fragment, "") + self.assertEndsWith(getattr(result, attr), + "#" + expected_frag) + self.assertEqual(func(url, "", False).fragment, "") + + result = func(url, allow_fragments=True) + self.assertEqual(result.fragment, expected_frag) + self.assertNotEndsWith(getattr(result, attr), expected_frag) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -847,7 +1182,14 @@ def test_mixed_types_rejected(self): with self.assertRaisesRegex(TypeError, "Cannot mix str"): urllib.parse.urljoin(b"http://python.org", "http://python.org") - def _check_result_type(self, str_type): + @support.subTests('result_type', [ + urllib.parse.DefragResult, + urllib.parse.SplitResult, + urllib.parse.ParseResult, + ]) + def test_result_pairs(self, result_type): + # Check encoding and decoding between result pairs + str_type = result_type num_args = len(str_type._fields) bytes_type = str_type._encoded_counterpart self.assertIs(bytes_type._decoded_counterpart, str_type) @@ -872,16 +1214,6 @@ def _check_result_type(self, str_type): self.assertEqual(str_result.encode(encoding, errors), bytes_args) self.assertEqual(str_result.encode(encoding, errors), bytes_result) - def test_result_pairs(self): - # Check encoding and decoding between result pairs - result_types = [ - urllib.parse.DefragResult, - urllib.parse.SplitResult, - urllib.parse.ParseResult, - ] - for result_type in result_types: - self._check_result_type(result_type) - def test_parse_qs_encoding(self): result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1") self.assertEqual(result, {'key': ['\u0141\xE9']}) @@ -910,11 +1242,10 @@ def test_parse_qsl_encoding(self): def test_parse_qsl_max_num_fields(self): with self.assertRaises(ValueError): - urllib.parse.parse_qs('&'.join(['a=a']*11), max_num_fields=10) - urllib.parse.parse_qs('&'.join(['a=a']*10), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*11), max_num_fields=10) + urllib.parse.parse_qsl('&'.join(['a=a']*10), max_num_fields=10) - def test_parse_qs_separator(self): - parse_qs_semicolon_cases = [ + @support.subTests('orig,expect', [ (";", {}), (";;", {}), (";a=b", {'a': ['b']}), @@ -925,17 +1256,14 @@ def test_parse_qs_separator(self): (b";a=b", {b'a': [b'b']}), (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1;a=2", {b'a': [b'1', b'2']}), - ] - for orig, expect in parse_qs_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qs(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qs(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) - - - def test_parse_qsl_separator(self): - parse_qsl_semicolon_cases = [ + ]) + def test_parse_qs_separator(self, orig, expect): + result = urllib.parse.parse_qs(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qs(orig, separator=b';') + self.assertEqual(result_bytes, expect) + + @support.subTests('orig,expect', [ (";", []), (";;", []), (";a=b", [('a', 'b')]), @@ -946,14 +1274,45 @@ def test_parse_qsl_separator(self): (b";a=b", [(b'a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), - ] - for orig, expect in parse_qsl_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qsl(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qsl(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) - + ]) + def test_parse_qsl_separator(self, orig, expect): + result = urllib.parse.parse_qsl(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qsl(orig, separator=b';') + self.assertEqual(result_bytes, expect) + + def test_parse_qsl_bytes(self): + self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(bytearray(b'a=b')), [(b'a', b'b')]) + self.assertEqual(urllib.parse.parse_qsl(memoryview(b'a=b')), [(b'a', b'b')]) + + def test_parse_qsl_false_value(self): + kwargs = dict(keep_blank_values=True, strict_parsing=True) + for x in '', b'', None, memoryview(b''): + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + for x in 0, 0.0, [], {}: + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(urllib.parse.parse_qsl(x, **kwargs), []) + self.assertEqual(cm.filename, __file__) + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(urllib.parse.parse_qs(x, **kwargs), {}) + self.assertEqual(cm.filename, __file__) + self.assertRaises(ValueError, urllib.parse.parse_qsl, x, separator=1) + + def test_parse_qsl_errors(self): + self.assertRaises(TypeError, urllib.parse.parse_qsl, list(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, iter(b'a=b')) + self.assertRaises(TypeError, urllib.parse.parse_qsl, 1) + self.assertRaises(TypeError, urllib.parse.parse_qsl, object()) + + for separator in '', b'', None, 0, 1, 0.0, 1.5: + with self.assertRaises(ValueError): + urllib.parse.parse_qsl('a=b', separator=separator) + with self.assertRaises(UnicodeEncodeError): + urllib.parse.parse_qsl(b'a=b', separator='\xa6') + with self.assertRaises(UnicodeDecodeError): + urllib.parse.parse_qsl('a=b', separator=b'\xa6') def test_urlencode_sequences(self): # Other tests incidentally urlencode things; test non-covered cases: @@ -985,6 +1344,10 @@ def test_quote_from_bytes(self): self.assertEqual(result, 'archaeological%20arcana') result = urllib.parse.quote_from_bytes(b'') self.assertEqual(result, '') + result = urllib.parse.quote_from_bytes(b'A'*10_000) + self.assertEqual(result, 'A'*10_000) + result = urllib.parse.quote_from_bytes(b'z\x01/ '*253_183) + self.assertEqual(result, 'z%01/%20'*253_183) def test_unquote_to_bytes(self): result = urllib.parse.unquote_to_bytes('abc%20def') @@ -1012,6 +1375,67 @@ def test_issue14072(self): self.assertEqual(p2.scheme, 'tel') self.assertEqual(p2.path, '+31641044153') + def test_invalid_bracketed_hosts(self): + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[important.com:8000]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123r.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v12ae]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v.IP]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v123.]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[v]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af::2309::fae7:1234]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@[0439:23af:2309::fae7:1234:2342:438e:192.0.2.146]/Path?Query') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'Scheme://user@]v6a.ip[/Path') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip]?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip].suffix?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:a1') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:a1') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:1a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:1a') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[::1].suffix:/') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[::1]:?') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://user@prefix.[v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://user@[v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://[v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip]') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://]v6a.ip[') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://]v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip[') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix.[v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip].suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix]v6a.ip[suffix') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://prefix]v6a.ip') + self.assertRaises(ValueError, urllib.parse.urlsplit, 'scheme://v6a.ip[suffix') + + def test_splitting_bracketed_hosts(self): + p1 = urllib.parse.urlsplit('scheme://user@[v6a.ip]:1234/path?query') + self.assertEqual(p1.hostname, 'v6a.ip') + self.assertEqual(p1.username, 'user') + self.assertEqual(p1.path, '/path') + self.assertEqual(p1.port, 1234) + p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') + self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test') + self.assertEqual(p2.username, 'user') + self.assertEqual(p2.path, '/path') + self.assertIs(p2.port, None) + p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') + self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') + self.assertEqual(p3.username, 'user') + self.assertEqual(p3.path, '/path') + def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') @@ -1044,16 +1468,24 @@ def test_telurl_params(self): self.assertEqual(p1.params, 'phone-context=+1-914-555') def test_Quoter_repr(self): - quoter = urllib.parse.Quoter(urllib.parse._ALWAYS_SAFE) + quoter = urllib.parse._Quoter(urllib.parse._ALWAYS_SAFE) self.assertIn('Quoter', repr(quoter)) + def test_clear_cache_for_code_coverage(self): + urllib.parse.clear_cache() + + def test_urllib_parse_getattr_failure(self): + """Test that urllib.parse.__getattr__() fails correctly.""" + with self.assertRaises(AttributeError): + unused = urllib.parse.this_does_not_exist + def test_all(self): expected = [] undocumented = { 'splitattr', 'splithost', 'splitnport', 'splitpasswd', 'splitport', 'splitquery', 'splittag', 'splittype', 'splituser', 'splitvalue', - 'Quoter', 'ResultBase', 'clear_cache', 'to_bytes', 'unwrap', + 'ResultBase', 'clear_cache', 'to_bytes', 'unwrap', } for name in dir(urllib.parse): if name.startswith('_') or name in undocumented: @@ -1063,8 +1495,6 @@ def test_all(self): expected.append(name) self.assertCountEqual(urllib.parse.__all__, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_urlsplit_normalization(self): # Certain characters should never occur in the netloc, # including under normalization. @@ -1073,7 +1503,8 @@ def test_urlsplit_normalization(self): hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars} denorm_chars = [ c for c in map(chr, range(128, sys.maxunicode)) - if (hex_chars & set(unicodedata.decomposition(c).split())) + if unicodedata.decomposition(c) + and (hex_chars & set(unicodedata.decomposition(c).split())) and c not in illegal_chars ] # Sanity check that we found at least one such character @@ -1188,6 +1619,7 @@ def test_splitnport(self): self.assertEqual(splitnport('127.0.0.1', 55), ('127.0.0.1', 55)) self.assertEqual(splitnport('parrot:cheese'), ('parrot', None)) self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None)) + self.assertEqual(splitnport('parrot: +1_0 '), ('parrot', None)) def test_splitquery(self): # Normal cases are exercised by other tests; ensure that we also @@ -1238,15 +1670,15 @@ def test_to_bytes(self): self.assertRaises(UnicodeError, urllib.parse._to_bytes, 'http://www.python.org/medi\u00e6val') - def test_unwrap(self): - for wrapped_url in ('<URL:scheme://host/path>', '<scheme://host/path>', - 'URL:scheme://host/path', 'scheme://host/path'): - url = urllib.parse.unwrap(wrapped_url) - self.assertEqual(url, 'scheme://host/path') + @support.subTests('wrapped_url', + ('<URL:scheme://host/path>', '<scheme://host/path>', + 'URL:scheme://host/path', 'scheme://host/path')) + def test_unwrap(self, wrapped_url): + url = urllib.parse.unwrap(wrapped_url) + self.assertEqual(url, 'scheme://host/path') class DeprecationTest(unittest.TestCase): - def test_splittype_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittype('') @@ -1324,5 +1756,11 @@ def test_to_bytes_deprecation(self): 'urllib.parse.to_bytes() is deprecated as of 3.8') +def str_encode(s): + return s.encode('ascii') + +def tuple_encode(t): + return tuple(str_encode(x) for x in t) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index 483910aaa46..75de9ea252d 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -166,7 +166,7 @@ def test_update(self): def test_missing(self): # Make sure UserDict doesn't have a __missing__ method - self.assertEqual(hasattr(collections.UserDict, "__missing__"), False) + self.assertNotHasAttr(collections.UserDict, "__missing__") # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError @@ -213,6 +213,7 @@ class G(collections.UserDict): else: self.fail("g[42] didn't raise KeyError") + test_repr_deep = mapping_tests.TestHashMappingProtocol.test_repr_deep if __name__ == "__main__": diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index 1ed67dac805..da66ae694d5 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -4,6 +4,7 @@ from test import list_tests import unittest + class UserListTest(list_tests.CommonTest): type2test = UserList @@ -65,5 +66,10 @@ def test_userlist_copy(self): self.assertEqual(u, v) self.assertEqual(type(u), type(v)) + # Decorate existing test with recursion limit, because + # the test is for C structure, but `UserList` is a Python structure. + # test_repr_deep = list_tests.CommonTest.test_repr_deep + test_repr_deep = unittest.skip(list_tests.CommonTest.test_repr_deep) # TODO: RUSTPYTHON; Segfault + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index c0017794e82..74df52f5412 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -7,8 +7,7 @@ from collections import UserString class UserStringTest( - string_tests.CommonTest, - string_tests.MixinStrUnicodeUserStringTest, + string_tests.StringLikeTest, unittest.TestCase ): @@ -53,8 +52,6 @@ def __rmod__(self, other): str3 = ustr3('TEST') self.assertEqual(fmt2 % str3, 'value is TEST') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_default_args(self): self.checkequal(b'hello', 'hello', 'encode') # Check that encoding defaults to utf-8 @@ -62,8 +59,6 @@ def test_encode_default_args(self): # Check that errors defaults to 'strict' self.checkraises(UnicodeError, '\ud800', 'encode') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encode_explicit_none_args(self): self.checkequal(b'hello', 'hello', 'encode', None, None) # Check that encoding defaults to utf-8 diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index b3e3e0bb27f..176a2112718 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -46,8 +46,7 @@ def test_posix_locale(self): out = self.get_output('-c', code, LC_ALL=loc) self.assertEqual(out, '1') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON") def test_xoption(self): code = 'import sys; print(sys.flags.utf8_mode)' diff --git a/Lib/test/test_utf8source.py b/Lib/test/test_utf8source.py index 602b4b69aac..7336cf00a71 100644 --- a/Lib/test/test_utf8source.py +++ b/Lib/test/test_utf8source.py @@ -1,5 +1,3 @@ -# This file is marked as binary in the CVS, to prevent MacCVS from recoding it. - import unittest class PEP3120Test(unittest.TestCase): @@ -14,11 +12,9 @@ def test_pep3120(self): b'\\\xd0\x9f' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badsyntax(self): try: - import test.badsyntax_pep3120 + import test.tokenizedata.badsyntax_pep3120 # noqa: F401 except SyntaxError as msg: msg = str(msg).lower() self.assertTrue('utf-8' in msg) @@ -28,8 +24,6 @@ def test_badsyntax(self): class BuiltinCompileTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure # Issue 3574. def test_latin1(self): # Allow compile() to read Latin-1 source. diff --git a/Lib/test/test_uu.py b/Lib/test/test_uu.py deleted file mode 100644 index 6b0b2f24f55..00000000000 --- a/Lib/test/test_uu.py +++ /dev/null @@ -1,272 +0,0 @@ -""" -Tests for uu module. -Nick Mathewson -""" - -import unittest -from test.support import os_helper - -import os -import stat -import sys -import uu -import io - -plaintext = b"The symbols on top of your keyboard are !@#$%^&*()_+|~\n" - -encodedtext = b"""\ -M5&AE('-Y;6)O;',@;VX@=&]P(&]F('EO=7(@:V5Y8F]A<F0@87)E("% (R0E -*7B8J*"E?*WQ^"@ """ - -# Stolen from io.py -class FakeIO(io.TextIOWrapper): - """Text I/O implementation using an in-memory buffer. - - Can be a used as a drop-in replacement for sys.stdin and sys.stdout. - """ - - # XXX This is really slow, but fully functional - - def __init__(self, initial_value="", encoding="utf-8", - errors="strict", newline="\n"): - super(FakeIO, self).__init__(io.BytesIO(), - encoding=encoding, - errors=errors, - newline=newline) - self._encoding = encoding - self._errors = errors - if initial_value: - if not isinstance(initial_value, str): - initial_value = str(initial_value) - self.write(initial_value) - self.seek(0) - - def getvalue(self): - self.flush() - return self.buffer.getvalue().decode(self._encoding, self._errors) - - -def encodedtextwrapped(mode, filename, backtick=False): - if backtick: - res = (bytes("begin %03o %s\n" % (mode, filename), "ascii") + - encodedtext.replace(b' ', b'`') + b"\n`\nend\n") - else: - res = (bytes("begin %03o %s\n" % (mode, filename), "ascii") + - encodedtext + b"\n \nend\n") - return res - -class UUTest(unittest.TestCase): - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_encode(self): - inp = io.BytesIO(plaintext) - out = io.BytesIO() - uu.encode(inp, out, "t1") - self.assertEqual(out.getvalue(), encodedtextwrapped(0o666, "t1")) - inp = io.BytesIO(plaintext) - out = io.BytesIO() - uu.encode(inp, out, "t1", 0o644) - self.assertEqual(out.getvalue(), encodedtextwrapped(0o644, "t1")) - inp = io.BytesIO(plaintext) - out = io.BytesIO() - uu.encode(inp, out, "t1", backtick=True) - self.assertEqual(out.getvalue(), encodedtextwrapped(0o666, "t1", True)) - with self.assertRaises(TypeError): - uu.encode(inp, out, "t1", 0o644, True) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_decode(self): - for backtick in True, False: - inp = io.BytesIO(encodedtextwrapped(0o666, "t1", backtick=backtick)) - out = io.BytesIO() - uu.decode(inp, out) - self.assertEqual(out.getvalue(), plaintext) - inp = io.BytesIO( - b"UUencoded files may contain many lines,\n" + - b"even some that have 'begin' in them.\n" + - encodedtextwrapped(0o666, "t1", backtick=backtick) - ) - out = io.BytesIO() - uu.decode(inp, out) - self.assertEqual(out.getvalue(), plaintext) - - def test_truncatedinput(self): - inp = io.BytesIO(b"begin 644 t1\n" + encodedtext) - out = io.BytesIO() - try: - uu.decode(inp, out) - self.fail("No exception raised") - except uu.Error as e: - self.assertEqual(str(e), "Truncated input file") - - def test_missingbegin(self): - inp = io.BytesIO(b"") - out = io.BytesIO() - try: - uu.decode(inp, out) - self.fail("No exception raised") - except uu.Error as e: - self.assertEqual(str(e), "No valid begin line found in input file") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_garbage_padding(self): - # Issue #22406 - encodedtext1 = ( - b"begin 644 file\n" - # length 1; bits 001100 111111 111111 111111 - b"\x21\x2C\x5F\x5F\x5F\n" - b"\x20\n" - b"end\n" - ) - encodedtext2 = ( - b"begin 644 file\n" - # length 1; bits 001100 111111 111111 111111 - b"\x21\x2C\x5F\x5F\x5F\n" - b"\x60\n" - b"end\n" - ) - plaintext = b"\x33" # 00110011 - - for encodedtext in encodedtext1, encodedtext2: - with self.subTest("uu.decode()"): - inp = io.BytesIO(encodedtext) - out = io.BytesIO() - uu.decode(inp, out, quiet=True) - self.assertEqual(out.getvalue(), plaintext) - - with self.subTest("uu_codec"): - import codecs - decoded = codecs.decode(encodedtext, "uu_codec") - self.assertEqual(decoded, plaintext) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines_escaped(self): - # Test newlines are escaped with uu.encode - inp = io.BytesIO(plaintext) - out = io.BytesIO() - filename = "test.txt\n\roverflow.txt" - safefilename = b"test.txt\\n\\roverflow.txt" - uu.encode(inp, out, filename) - self.assertIn(safefilename, out.getvalue()) - -class UUStdIOTest(unittest.TestCase): - - def setUp(self): - self.stdin = sys.stdin - self.stdout = sys.stdout - - def tearDown(self): - sys.stdin = self.stdin - sys.stdout = self.stdout - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_encode(self): - sys.stdin = FakeIO(plaintext.decode("ascii")) - sys.stdout = FakeIO() - uu.encode("-", "-", "t1", 0o666) - self.assertEqual(sys.stdout.getvalue(), - encodedtextwrapped(0o666, "t1").decode("ascii")) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_decode(self): - sys.stdin = FakeIO(encodedtextwrapped(0o666, "t1").decode("ascii")) - sys.stdout = FakeIO() - uu.decode("-", "-") - stdout = sys.stdout - sys.stdout = self.stdout - sys.stdin = self.stdin - self.assertEqual(stdout.getvalue(), plaintext.decode("ascii")) - -class UUFileTest(unittest.TestCase): - - def setUp(self): - # uu.encode() supports only ASCII file names - self.tmpin = os_helper.TESTFN_ASCII + "i" - self.tmpout = os_helper.TESTFN_ASCII + "o" - self.addCleanup(os_helper.unlink, self.tmpin) - self.addCleanup(os_helper.unlink, self.tmpout) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_encode(self): - with open(self.tmpin, 'wb') as fin: - fin.write(plaintext) - - with open(self.tmpin, 'rb') as fin: - with open(self.tmpout, 'wb') as fout: - uu.encode(fin, fout, self.tmpin, mode=0o644) - - with open(self.tmpout, 'rb') as fout: - s = fout.read() - self.assertEqual(s, encodedtextwrapped(0o644, self.tmpin)) - - # in_file and out_file as filenames - uu.encode(self.tmpin, self.tmpout, self.tmpin, mode=0o644) - with open(self.tmpout, 'rb') as fout: - s = fout.read() - self.assertEqual(s, encodedtextwrapped(0o644, self.tmpin)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_decode(self): - with open(self.tmpin, 'wb') as f: - f.write(encodedtextwrapped(0o644, self.tmpout)) - - with open(self.tmpin, 'rb') as f: - uu.decode(f) - - with open(self.tmpout, 'rb') as f: - s = f.read() - self.assertEqual(s, plaintext) - # XXX is there an xp way to verify the mode? - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_decode_filename(self): - with open(self.tmpin, 'wb') as f: - f.write(encodedtextwrapped(0o644, self.tmpout)) - - uu.decode(self.tmpin) - - with open(self.tmpout, 'rb') as f: - s = f.read() - self.assertEqual(s, plaintext) - - def test_decodetwice(self): - # Verify that decode() will refuse to overwrite an existing file - with open(self.tmpin, 'wb') as f: - f.write(encodedtextwrapped(0o644, self.tmpout)) - with open(self.tmpin, 'rb') as f: - uu.decode(f) - - with open(self.tmpin, 'rb') as f: - self.assertRaises(uu.Error, uu.decode, f) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_decode_mode(self): - # Verify that decode() will set the given mode for the out_file - expected_mode = 0o444 - with open(self.tmpin, 'wb') as f: - f.write(encodedtextwrapped(expected_mode, self.tmpout)) - - # make file writable again, so it can be removed (Windows only) - self.addCleanup(os.chmod, self.tmpout, expected_mode | stat.S_IWRITE) - - with open(self.tmpin, 'rb') as f: - uu.decode(f) - - self.assertEqual( - stat.S_IMODE(os.stat(self.tmpout).st_mode), - expected_mode - ) - - -if __name__=="__main__": - unittest.main() diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index ee6232ed9eb..0e1a723ce3a 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,16 +1,21 @@ -import unittest -from test import support -from test.support import import_helper import builtins import contextlib import copy +import enum import io import os import pickle +import random import sys +import unittest import weakref +from itertools import product from unittest import mock +from test import support +from test.support import import_helper +from test.support.script_helper import assert_python_ok + py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) @@ -18,7 +23,7 @@ def importable(name): try: __import__(name) return True - except: + except ModuleNotFoundError: return False @@ -31,6 +36,54 @@ def get_command_stdout(command, args): class BaseTestUUID: uuid = None + def test_nil_uuid(self): + nil_uuid = self.uuid.NIL + + s = '00000000-0000-0000-0000-000000000000' + i = 0 + self.assertEqual(nil_uuid, self.uuid.UUID(s)) + self.assertEqual(nil_uuid, self.uuid.UUID(int=i)) + self.assertEqual(nil_uuid.int, i) + self.assertEqual(str(nil_uuid), s) + # The Nil UUID falls within the range of the Apollo NCS variant as per + # RFC 9562. + # See https://www.rfc-editor.org/rfc/rfc9562.html#section-5.9-4 + self.assertEqual(nil_uuid.variant, self.uuid.RESERVED_NCS) + # A version field of all zeros is "Unused" in RFC 9562, but the version + # field also only applies to the 10xx variant, i.e. the variant + # specified in RFC 9562. As such, because the Nil UUID falls under a + # different variant, its version is considered undefined. + # See https://www.rfc-editor.org/rfc/rfc9562.html#table2 + self.assertIsNone(nil_uuid.version) + + def test_max_uuid(self): + max_uuid = self.uuid.MAX + + s = 'ffffffff-ffff-ffff-ffff-ffffffffffff' + i = (1 << 128) - 1 + self.assertEqual(max_uuid, self.uuid.UUID(s)) + self.assertEqual(max_uuid, self.uuid.UUID(int=i)) + self.assertEqual(max_uuid.int, i) + self.assertEqual(str(max_uuid), s) + # The Max UUID falls within the range of the "yet-to-be defined" future + # UUID variant as per RFC 9562. + # See https://www.rfc-editor.org/rfc/rfc9562.html#section-5.10-4 + self.assertEqual(max_uuid.variant, self.uuid.RESERVED_FUTURE) + # A version field of all ones is "Reserved for future definition" in + # RFC 9562, but the version field also only applies to the 10xx + # variant, i.e. the variant specified in RFC 9562. As such, because the + # Max UUID falls under a different variant, its version is considered + # undefined. + # See https://www.rfc-editor.org/rfc/rfc9562.html#table2 + self.assertIsNone(max_uuid.version) + + def test_safe_uuid_enum(self): + class CheckedSafeUUID(enum.Enum): + safe = 0 + unsafe = -1 + unknown = None + enum._test_simple_enum(CheckedSafeUUID, py_uuid.SafeUUID) + def test_UUID(self): equal = self.assertEqual ascending = [] @@ -259,7 +312,7 @@ def test_exceptions(self): # Version number out of range. badvalue(lambda: self.uuid.UUID('00'*16, version=0)) - badvalue(lambda: self.uuid.UUID('00'*16, version=6)) + badvalue(lambda: self.uuid.UUID('00'*16, version=42)) # Integer value out of range. badvalue(lambda: self.uuid.UUID(int=-1)) @@ -522,7 +575,14 @@ def test_uuid1(self): @support.requires_mac_ver(10, 5) @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') def test_uuid1_safe(self): - if not self.uuid._has_uuid_generate_time_safe: + try: + import _uuid + except ImportError: + has_uuid_generate_time_safe = False + else: + has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe + + if not has_uuid_generate_time_safe or not self.uuid._generate_time_safe: self.skipTest('requires uuid_generate_time_safe(3)') u = self.uuid.uuid1() @@ -538,7 +598,6 @@ def mock_generate_time_safe(self, safe_value): """ if os.name != 'posix': self.skipTest('POSIX-only test') - self.uuid._load_system_functions() f = self.uuid._generate_time_safe if f is None: self.skipTest('need uuid._generate_time_safe') @@ -573,8 +632,7 @@ def test_uuid1_bogus_return_value(self): self.assertEqual(u.is_safe, self.uuid.SafeUUID.unknown) def test_uuid1_time(self): - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch.object(self.uuid, 'getnode', return_value=93328246233727), \ mock.patch('time.time_ns', return_value=1545052026752910643), \ @@ -582,8 +640,7 @@ def test_uuid1_time(self): u = self.uuid.uuid1() self.assertEqual(u, self.uuid.UUID('a7a55b92-01fc-11e9-94c5-54e1acf6da7f')) - with mock.patch.object(self.uuid, '_has_uuid_generate_time_safe', False), \ - mock.patch.object(self.uuid, '_generate_time_safe', None), \ + with mock.patch.object(self.uuid, '_generate_time_safe', None), \ mock.patch.object(self.uuid, '_last_timestamp', None), \ mock.patch('time.time_ns', return_value=1545052026752910643): u = self.uuid.uuid1(node=93328246233727, clock_seq=5317) @@ -592,7 +649,22 @@ def test_uuid1_time(self): def test_uuid3(self): equal = self.assertEqual - # Test some known version-3 UUIDs. + # Test some known version-3 UUIDs with name passed as a byte object + for u, v in [(self.uuid.uuid3(self.uuid.NAMESPACE_DNS, b'python.org'), + '6fa459ea-ee8a-3ca4-894e-db77e160355e'), + (self.uuid.uuid3(self.uuid.NAMESPACE_URL, b'http://python.org/'), + '9fe8e8c4-aaa8-32a9-a55c-4535a88b748d'), + (self.uuid.uuid3(self.uuid.NAMESPACE_OID, b'1.3.6.1'), + 'dd1a1cef-13d5-368a-ad82-eca71acd4cd1'), + (self.uuid.uuid3(self.uuid.NAMESPACE_X500, b'c=ca'), + '658d3002-db6b-3040-a1d1-8ddd7d189a4d'), + ]: + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 3) + equal(u, self.uuid.UUID(v)) + equal(str(u), v) + + # Test some known version-3 UUIDs with name passed as a string for u, v in [(self.uuid.uuid3(self.uuid.NAMESPACE_DNS, 'python.org'), '6fa459ea-ee8a-3ca4-894e-db77e160355e'), (self.uuid.uuid3(self.uuid.NAMESPACE_URL, 'http://python.org/'), @@ -624,7 +696,22 @@ def test_uuid4(self): def test_uuid5(self): equal = self.assertEqual - # Test some known version-5 UUIDs. + # Test some known version-5 UUIDs with names given as byte objects + for u, v in [(self.uuid.uuid5(self.uuid.NAMESPACE_DNS, b'python.org'), + '886313e1-3b8a-5372-9b90-0c9aee199e5d'), + (self.uuid.uuid5(self.uuid.NAMESPACE_URL, b'http://python.org/'), + '4c565f0d-3f5a-5890-b41b-20cf47701c5e'), + (self.uuid.uuid5(self.uuid.NAMESPACE_OID, b'1.3.6.1'), + '1447fa61-5277-5fef-a9b3-fbc6e44f4af3'), + (self.uuid.uuid5(self.uuid.NAMESPACE_X500, b'c=ca'), + 'cc957dd1-a972-5349-98cd-874190002798'), + ]: + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 5) + equal(u, self.uuid.UUID(v)) + equal(str(u), v) + + # Test some known version-5 UUIDs with names given as strings for u, v in [(self.uuid.uuid5(self.uuid.NAMESPACE_DNS, 'python.org'), '886313e1-3b8a-5372-9b90-0c9aee199e5d'), (self.uuid.uuid5(self.uuid.NAMESPACE_URL, 'http://python.org/'), @@ -639,6 +726,392 @@ def test_uuid5(self): equal(u, self.uuid.UUID(v)) equal(str(u), v) + def test_uuid6(self): + equal = self.assertEqual + u = self.uuid.uuid6() + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 6) + + fake_nanoseconds = 0x1571_20a1_de1a_c533 + fake_node_value = 0x54e1_acf6_da7f + fake_clock_seq = 0x14c5 + with ( + mock.patch.object(self.uuid, '_last_timestamp_v6', None), + mock.patch.object(self.uuid, 'getnode', return_value=fake_node_value), + mock.patch('time.time_ns', return_value=fake_nanoseconds), + mock.patch('random.getrandbits', return_value=fake_clock_seq) + ): + u = self.uuid.uuid6() + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 6) + + # 32 (top) | 16 (mid) | 12 (low) == 60 (timestamp) + equal(u.time, 0x1e901fca_7a55_b92) + equal(u.fields[0], 0x1e901fca) # 32 top bits of time + equal(u.fields[1], 0x7a55) # 16 mid bits of time + # 4 bits of version + 12 low bits of time + equal((u.fields[2] >> 12) & 0xf, 6) + equal((u.fields[2] & 0xfff), 0xb92) + # 2 bits of variant + 6 high bits of clock_seq + equal((u.fields[3] >> 6) & 0xf, 2) + equal(u.fields[3] & 0x3f, fake_clock_seq >> 8) + # 8 low bits of clock_seq + equal(u.fields[4], fake_clock_seq & 0xff) + equal(u.fields[5], fake_node_value) + + def test_uuid6_uniqueness(self): + # Test that UUIDv6-generated values are unique. + + # Unlike UUIDv8, only 62 bits can be randomized for UUIDv6. + # In practice, however, it remains unlikely to generate two + # identical UUIDs for the same 60-bit timestamp if neither + # the node ID nor the clock sequence is specified. + uuids = {self.uuid.uuid6() for _ in range(1000)} + self.assertEqual(len(uuids), 1000) + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {6}) + + timestamp = 0x1ec9414c_232a_b00 + fake_nanoseconds = (timestamp - 0x1b21dd21_3814_000) * 100 + + with mock.patch('time.time_ns', return_value=fake_nanoseconds): + def gen(): + with mock.patch.object(self.uuid, '_last_timestamp_v6', None): + return self.uuid.uuid6(node=0, clock_seq=None) + + # By the birthday paradox, sampling N = 1024 UUIDs with identical + # node IDs and timestamps results in duplicates with probability + # close to 1 (not having a duplicate happens with probability of + # order 1E-15) since only the 14-bit clock sequence is randomized. + N = 1024 + uuids = {gen() for _ in range(N)} + self.assertSetEqual({u.node for u in uuids}, {0}) + self.assertSetEqual({u.time for u in uuids}, {timestamp}) + self.assertLess(len(uuids), N, 'collision property does not hold') + + def test_uuid6_node(self): + # Make sure the given node ID appears in the UUID. + # + # Note: when no node ID is specified, the same logic as for UUIDv1 + # is applied to UUIDv6. In particular, there is no need to test that + # getnode() correctly returns positive integers of exactly 48 bits + # since this is done in test_uuid1_eui64(). + self.assertLessEqual(self.uuid.uuid6().node.bit_length(), 48) + + self.assertEqual(self.uuid.uuid6(0).node, 0) + + # tests with explicit values + max_node = 0xffff_ffff_ffff + self.assertEqual(self.uuid.uuid6(max_node).node, max_node) + big_node = 0xE_1234_5678_ABCD # 52-bit node + res_node = 0x0_1234_5678_ABCD # truncated to 48 bits + self.assertEqual(self.uuid.uuid6(big_node).node, res_node) + + # randomized tests + for _ in range(10): + # node with > 48 bits is truncated + for b in [24, 48, 72]: + node = (1 << (b - 1)) | random.getrandbits(b) + with self.subTest(node=node, bitlen=b): + self.assertEqual(node.bit_length(), b) + u = self.uuid.uuid6(node=node) + self.assertEqual(u.node, node & 0xffff_ffff_ffff) + + def test_uuid6_clock_seq(self): + # Make sure the supplied clock sequence appears in the UUID. + # + # For UUIDv6, clock sequence bits are stored from bit 48 to bit 62, + # with the convention that the least significant bit is bit 0 and + # the most significant bit is bit 127. + get_clock_seq = lambda u: (u.int >> 48) & 0x3fff + + u = self.uuid.uuid6() + self.assertLessEqual(get_clock_seq(u).bit_length(), 14) + + # tests with explicit values + big_clock_seq = 0xffff # 16-bit clock sequence + res_clock_seq = 0x3fff # truncated to 14 bits + u = self.uuid.uuid6(clock_seq=big_clock_seq) + self.assertEqual(get_clock_seq(u), res_clock_seq) + + # some randomized tests + for _ in range(10): + # clock_seq with > 14 bits is truncated + for b in [7, 14, 28]: + node = random.getrandbits(48) + clock_seq = (1 << (b - 1)) | random.getrandbits(b) + with self.subTest(node=node, clock_seq=clock_seq, bitlen=b): + self.assertEqual(clock_seq.bit_length(), b) + u = self.uuid.uuid6(node=node, clock_seq=clock_seq) + self.assertEqual(get_clock_seq(u), clock_seq & 0x3fff) + + def test_uuid6_test_vectors(self): + equal = self.assertEqual + # https://www.rfc-editor.org/rfc/rfc9562#name-test-vectors + # (separators are put at the 12th and 28th bits) + timestamp = 0x1ec9414c_232a_b00 + fake_nanoseconds = (timestamp - 0x1b21dd21_3814_000) * 100 + # https://www.rfc-editor.org/rfc/rfc9562#name-example-of-a-uuidv6-value + node = 0x9f6bdeced846 + clock_seq = (3 << 12) | 0x3c8 + + with ( + mock.patch.object(self.uuid, '_last_timestamp_v6', None), + mock.patch('time.time_ns', return_value=fake_nanoseconds) + ): + u = self.uuid.uuid6(node=node, clock_seq=clock_seq) + equal(str(u).upper(), '1EC9414C-232A-6B00-B3C8-9F6BDECED846') + # 32 16 4 12 2 14 48 + # time_hi | time_mid | ver | time_lo | var | clock_seq | node + equal(u.time, timestamp) + equal(u.int & 0xffff_ffff_ffff, node) + equal((u.int >> 48) & 0x3fff, clock_seq) + equal((u.int >> 62) & 0x3, 0b10) + equal((u.int >> 64) & 0xfff, 0xb00) + equal((u.int >> 76) & 0xf, 0x6) + equal((u.int >> 80) & 0xffff, 0x232a) + equal((u.int >> 96) & 0xffff_ffff, 0x1ec9_414c) + + def test_uuid7(self): + equal = self.assertEqual + u = self.uuid.uuid7() + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 7) + + # 1 Jan 2023 12:34:56.123_456_789 + timestamp_ns = 1672533296_123_456_789 # ns precision + timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + + for _ in range(100): + counter_hi = random.getrandbits(11) + counter_lo = random.getrandbits(30) + counter = (counter_hi << 30) | counter_lo + + tail = random.getrandbits(32) + # effective number of bits is 32 + 30 + 11 = 73 + random_bits = counter << 32 | tail + + # set all remaining MSB of fake random bits to 1 to ensure that + # the implementation correctly removes them + random_bits = (((1 << 7) - 1) << 73) | random_bits + random_data = random_bits.to_bytes(10) + + with ( + mock.patch.multiple( + self.uuid, + _last_timestamp_v7=None, + _last_counter_v7=0, + ), + mock.patch('time.time_ns', return_value=timestamp_ns), + mock.patch('os.urandom', return_value=random_data) as urand + ): + u = self.uuid.uuid7() + urand.assert_called_once_with(10) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 7) + + equal(self.uuid._last_timestamp_v7, timestamp_ms) + equal(self.uuid._last_counter_v7, counter) + + unix_ts_ms = timestamp_ms & 0xffff_ffff_ffff + equal(u.time, unix_ts_ms) + equal((u.int >> 80) & 0xffff_ffff_ffff, unix_ts_ms) + + equal((u.int >> 75) & 1, 0) # check that the MSB is 0 + equal((u.int >> 64) & 0xfff, counter_hi) + equal((u.int >> 32) & 0x3fff_ffff, counter_lo) + equal(u.int & 0xffff_ffff, tail) + + def test_uuid7_uniqueness(self): + # Test that UUIDv7-generated values are unique. + # + # While UUIDv8 has an entropy of 122 bits, those 122 bits may not + # necessarily be sampled from a PRNG. On the other hand, UUIDv7 + # uses os.urandom() as a PRNG which features better randomness. + N = 1000 + uuids = {self.uuid.uuid7() for _ in range(N)} + self.assertEqual(len(uuids), N) + + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {7}) + + def test_uuid7_monotonicity(self): + equal = self.assertEqual + + us = [self.uuid.uuid7() for _ in range(10_000)] + equal(us, sorted(us)) + + with mock.patch.multiple( + self.uuid, + _last_timestamp_v7=0, + _last_counter_v7=0, + ): + # 1 Jan 2023 12:34:56.123_456_789 + timestamp_ns = 1672533296_123_456_789 # ns precision + timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + + # counter_{hi,lo} are chosen so that "counter + 1" does not overflow + counter_hi = random.getrandbits(11) + counter_lo = random.getrandbits(29) + counter = (counter_hi << 30) | counter_lo + self.assertLess(counter + 1, 0x3ff_ffff_ffff) + + tail = random.getrandbits(32) + random_bits = counter << 32 | tail + random_data = random_bits.to_bytes(10) + + with ( + mock.patch('time.time_ns', return_value=timestamp_ns), + mock.patch('os.urandom', return_value=random_data) as urand + ): + u1 = self.uuid.uuid7() + urand.assert_called_once_with(10) + equal(self.uuid._last_timestamp_v7, timestamp_ms) + equal(self.uuid._last_counter_v7, counter) + equal(u1.time, timestamp_ms) + equal((u1.int >> 64) & 0xfff, counter_hi) + equal((u1.int >> 32) & 0x3fff_ffff, counter_lo) + equal(u1.int & 0xffff_ffff, tail) + + # 1 Jan 2023 12:34:56.123_457_032 (same millisecond but not same ns) + next_timestamp_ns = 1672533296_123_457_032 + next_timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + equal(timestamp_ms, next_timestamp_ms) + + next_tail_bytes = os.urandom(4) + next_fail = int.from_bytes(next_tail_bytes) + + with ( + mock.patch('time.time_ns', return_value=next_timestamp_ns), + mock.patch('os.urandom', return_value=next_tail_bytes) as urand + ): + u2 = self.uuid.uuid7() + urand.assert_called_once_with(4) + # same milli-second + equal(self.uuid._last_timestamp_v7, timestamp_ms) + # 42-bit counter advanced by 1 + equal(self.uuid._last_counter_v7, counter + 1) + equal(u2.time, timestamp_ms) + equal((u2.int >> 64) & 0xfff, counter_hi) + equal((u2.int >> 32) & 0x3fff_ffff, counter_lo + 1) + equal(u2.int & 0xffff_ffff, next_fail) + + self.assertLess(u1, u2) + + def test_uuid7_timestamp_backwards(self): + equal = self.assertEqual + # 1 Jan 2023 12:34:56.123_456_789 + timestamp_ns = 1672533296_123_456_789 # ns precision + timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + fake_last_timestamp_v7 = timestamp_ms + 1 + + # counter_{hi,lo} are chosen so that "counter + 1" does not overflow + counter_hi = random.getrandbits(11) + counter_lo = random.getrandbits(29) + counter = (counter_hi << 30) | counter_lo + self.assertLess(counter + 1, 0x3ff_ffff_ffff) + + tail_bytes = os.urandom(4) + tail = int.from_bytes(tail_bytes) + + with ( + mock.patch.multiple( + self.uuid, + _last_timestamp_v7=fake_last_timestamp_v7, + _last_counter_v7=counter, + ), + mock.patch('time.time_ns', return_value=timestamp_ns), + mock.patch('os.urandom', return_value=tail_bytes) as urand + ): + u = self.uuid.uuid7() + urand.assert_called_once_with(4) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 7) + equal(self.uuid._last_timestamp_v7, fake_last_timestamp_v7 + 1) + unix_ts_ms = (fake_last_timestamp_v7 + 1) & 0xffff_ffff_ffff + equal(u.time, unix_ts_ms) + equal((u.int >> 80) & 0xffff_ffff_ffff, unix_ts_ms) + # 42-bit counter advanced by 1 + equal(self.uuid._last_counter_v7, counter + 1) + equal((u.int >> 64) & 0xfff, counter_hi) + # 42-bit counter advanced by 1 (counter_hi is untouched) + equal((u.int >> 32) & 0x3fff_ffff, counter_lo + 1) + equal(u.int & 0xffff_ffff, tail) + + def test_uuid7_overflow_counter(self): + equal = self.assertEqual + # 1 Jan 2023 12:34:56.123_456_789 + timestamp_ns = 1672533296_123_456_789 # ns precision + timestamp_ms, _ = divmod(timestamp_ns, 1_000_000) + + new_counter_hi = random.getrandbits(11) + new_counter_lo = random.getrandbits(30) + new_counter = (new_counter_hi << 30) | new_counter_lo + + tail = random.getrandbits(32) + random_bits = (new_counter << 32) | tail + random_data = random_bits.to_bytes(10) + + with ( + mock.patch.multiple( + self.uuid, + _last_timestamp_v7=timestamp_ms, + # same timestamp, but force an overflow on the counter + _last_counter_v7=0x3ff_ffff_ffff, + ), + mock.patch('time.time_ns', return_value=timestamp_ns), + mock.patch('os.urandom', return_value=random_data) as urand + ): + u = self.uuid.uuid7() + urand.assert_called_with(10) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 7) + # timestamp advanced due to overflow + equal(self.uuid._last_timestamp_v7, timestamp_ms + 1) + unix_ts_ms = (timestamp_ms + 1) & 0xffff_ffff_ffff + equal(u.time, unix_ts_ms) + equal((u.int >> 80) & 0xffff_ffff_ffff, unix_ts_ms) + # counter overflowed, so we picked a new one + equal(self.uuid._last_counter_v7, new_counter) + equal((u.int >> 64) & 0xfff, new_counter_hi) + equal((u.int >> 32) & 0x3fff_ffff, new_counter_lo) + equal(u.int & 0xffff_ffff, tail) + + def test_uuid8(self): + equal = self.assertEqual + u = self.uuid.uuid8() + + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 8) + + for (_, hi, mid, lo) in product( + range(10), # repeat 10 times + [None, 0, random.getrandbits(48)], + [None, 0, random.getrandbits(12)], + [None, 0, random.getrandbits(62)], + ): + u = self.uuid.uuid8(hi, mid, lo) + equal(u.variant, self.uuid.RFC_4122) + equal(u.version, 8) + if hi is not None: + equal((u.int >> 80) & 0xffffffffffff, hi) + if mid is not None: + equal((u.int >> 64) & 0xfff, mid) + if lo is not None: + equal(u.int & 0x3fffffffffffffff, lo) + + def test_uuid8_uniqueness(self): + # Test that UUIDv8-generated values are unique (up to a negligible + # probability of failure). There are 122 bits of entropy and assuming + # that the underlying mt-19937-based random generator is sufficiently + # good, it is unlikely to have a collision of two UUIDs. + N = 1000 + uuids = {self.uuid.uuid8() for _ in range(N)} + self.assertEqual(len(uuids), N) + + versions = {u.version for u in uuids} + self.assertSetEqual(versions, {8}) + @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates @@ -667,13 +1140,144 @@ def test_uuid_weakref(self): weak = weakref.ref(strong) self.assertIs(strong, weak()) -class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): + +class CommandLineTestCases: + uuid = None # to be defined in subclasses + + def do_test_standalone_uuid(self, version): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + output = stdout.getvalue().strip() + u = self.uuid.UUID(output) + self.assertEqual(output, str(u)) + self.assertEqual(u.version, version) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid1"]) + def test_cli_uuid1(self): + self.do_test_standalone_uuid(1) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns"]) + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_namespace_required_for_uuid3(self, mock_err): + with self.assertRaises(SystemExit) as cm: + self.uuid.main() + + # Check that exception code is the same as argparse.ArgumentParser.error + self.assertEqual(cm.exception.code, 2) + self.assertIn("error: Incorrect number of arguments", mock_err.getvalue()) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-N", "python.org"]) + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_name_required_for_uuid3(self, mock_err): + with self.assertRaises(SystemExit) as cm: + self.uuid.main() + # Check that exception code is the same as argparse.ArgumentParser.error + self.assertEqual(cm.exception.code, 2) + self.assertIn("error: Incorrect number of arguments", mock_err.getvalue()) + + @mock.patch.object(sys, "argv", [""]) + def test_cli_uuid4_outputted_with_no_args(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output uuid should be in the format of uuid4 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 4) + + @mock.patch.object(sys, "argv", ["", "-C", "3"]) + def test_cli_uuid4_outputted_with_count(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip().splitlines() + + # Check that 3 UUIDs in the format of uuid4 have been generated + self.assertEqual(len(output), 3) + for o in output: + uuid_output = self.uuid.UUID(o) + self.assertEqual(uuid_output.version, 4) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", "@dns", "-N", "python.org"]) + def test_cli_uuid3_ouputted_with_valid_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid5 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 3) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid5", "-n", "@dns", "-N", "python.org"]) + def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid5 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 5) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) + def test_cli_uuid6(self): + self.do_test_standalone_uuid(6) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid7"]) + def test_cli_uuid7(self): + self.do_test_standalone_uuid(7) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid8"]) + def test_cli_uuid8(self): + self.do_test_standalone_uuid(8) + + +class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid + @unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid + def check_has_stable_libuuid_extractable_node(self): + if not self.uuid._has_stable_extractable_node: + self.skipTest("libuuid cannot deduce MAC address") + + @unittest.skipUnless(os.name == 'posix', 'POSIX only') + def test_unix_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._unix_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + + @unittest.skipUnless(os.name == 'nt', 'Windows only') + def test_windows_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._windll_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + class BaseTestInternals: _uuid = py_uuid diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 94d626598ba..19b12070531 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -5,21 +5,31 @@ Licensed to the PSF under a contributor agreement. """ +import contextlib import ensurepip import os import os.path +import pathlib import re import shutil import struct import subprocess import sys +import sysconfig import tempfile -from test.support import (captured_stdout, captured_stderr, requires_zlib, - skip_if_broken_multiprocessing_synchronize) -from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) +import shlex +from test.support import (captured_stdout, captured_stderr, + skip_if_broken_multiprocessing_synchronize, verbose, + requires_subprocess, is_android, is_apple_mobile, + is_emscripten, is_wasi, + requires_venv_with_pip, TEST_HOME_DIR, + requires_resource, copy_python_src_ignore) +from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, + TESTFN, FakePath) +from test.support.testcase import ExtraAssertions import unittest import venv -from unittest.mock import patch +from unittest.mock import patch, Mock try: import ctypes @@ -33,18 +43,29 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on this platform") + +@requires_subprocess() def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding=encoding) + env={**os.environ, "PYTHONHOME": ""}) out, err = p.communicate() if p.returncode: + if verbose and err: + print(err.decode(encoding or 'utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) + if encoding: + return ( + out.decode(encoding, 'backslashreplace'), + err.decode(encoding, 'backslashreplace'), + ) return out, err -class BaseTest(unittest.TestCase): +class BaseTest(unittest.TestCase, ExtraAssertions): """Base class for venv tests.""" maxDiff = 80 * 50 @@ -56,7 +77,7 @@ def setUp(self): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}') self.include = 'include' executable = sys._base_executable self.exe = os.path.split(executable)[-1] @@ -70,6 +91,13 @@ def setUp(self): def tearDown(self): rmtree(self.env_dir) + def envpy(self, *, real_env_dir=False): + if real_env_dir: + env_dir = os.path.realpath(self.env_dir) + else: + env_dir = self.env_dir + return os.path.join(env_dir, self.bindir, self.exe) + def run_with_capture(self, func, *args, **kwargs): with captured_stdout() as output: with captured_stderr() as error: @@ -91,12 +119,27 @@ def isdir(self, *args): fn = self.get_env_file(*args) self.assertTrue(os.path.isdir(fn)) - def test_defaults(self): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_defaults_with_str_path(self): """ - Test the create function with default arguments. + Test the create function with default arguments and a str path. """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) + self._check_output_of_default_create() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_defaults_with_pathlike(self): + """ + Test the create function with default arguments and a path-like path. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, FakePath(self.env_dir)) + self._check_output_of_default_create() + + def _check_output_of_default_create(self): self.isdir(self.bindir) self.isdir(self.include) self.isdir(*self.lib) @@ -112,6 +155,12 @@ def test_defaults(self): executable = sys._base_executable path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) + self.assertIn('executable = %s' % + os.path.realpath(sys.executable), data) + copies = '' if os.name=='nt' else ' --copies' + cmd = (f'command = {sys.executable} -m venv{copies} --without-pip ' + f'--without-scm-ignore-files {self.env_dir}') + self.assertIn(cmd, data) fn = self.get_env_file(self.bindir, self.exe) if not os.path.exists(fn): # diagnostics for Windows buildbot failures bd = self.get_env_file(self.bindir) @@ -119,6 +168,39 @@ def test_defaults(self): print(' %r' % os.listdir(bd)) self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) + def test_config_file_command_key(self): + options = [ + (None, None, None), # Default case. + ('--copies', 'symlinks', False), + ('--without-pip', 'with_pip', False), + ('--system-site-packages', 'system_site_packages', True), + ('--clear', 'clear', True), + ('--upgrade', 'upgrade', True), + ('--upgrade-deps', 'upgrade_deps', True), + ('--prompt="foobar"', 'prompt', 'foobar'), + ('--without-scm-ignore-files', 'scm_ignore_files', frozenset()), + ] + for opt, attr, value in options: + with self.subTest(opt=opt, attr=attr, value=value): + rmtree(self.env_dir) + if not attr: + kwargs = {} + else: + kwargs = {attr: value} + b = venv.EnvBuilder(**kwargs) + b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps + b._setup_pip = Mock() # avoid pip setup + self.run_with_capture(b.create, self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + if not attr or opt.endswith('git'): + for opt in ('--system-site-packages', '--clear', '--upgrade', + '--upgrade-deps', '--prompt'): + self.assertNotRegex(data, rf'command = .* {opt}') + elif os.name=='nt' and attr=='symlinks': + pass + else: + self.assertRegex(data, rf'command = .* {opt}') + def test_prompt(self): env_name = os.path.split(self.env_dir)[1] @@ -127,7 +209,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % env_name) + self.assertEqual(context.prompt, env_name) self.assertNotIn("prompt = ", data) rmtree(self.env_dir) @@ -135,7 +217,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(My prompt) ') + self.assertEqual(context.prompt, 'My prompt') self.assertIn("prompt = 'My prompt'\n", data) rmtree(self.env_dir) @@ -144,13 +226,19 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % cwd) + self.assertEqual(context.prompt, cwd) self.assertIn("prompt = '%s'\n" % cwd, data) def test_upgrade_dependencies(self): builder = venv.EnvBuilder() - bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' + bin_path = 'bin' python_exe = os.path.split(sys.executable)[1] + if sys.platform == 'win32': + bin_path = 'Scripts' + if os.path.normcase(os.path.splitext(python_exe)[0]).endswith('_d'): + python_exe = 'python_d.exe' + else: + python_exe = 'python.exe' with tempfile.TemporaryDirectory() as fake_env_dir: expect_exe = os.path.normcase( os.path.join(fake_env_dir, bin_path, python_exe) @@ -158,7 +246,7 @@ def test_upgrade_dependencies(self): if sys.platform == 'win32': expect_exe = os.path.normcase(os.path.realpath(expect_exe)) - def pip_cmd_checker(cmd): + def pip_cmd_checker(cmd, **kwargs): cmd[0] = os.path.normcase(cmd[0]) self.assertEqual( cmd, @@ -169,12 +257,11 @@ def pip_cmd_checker(cmd): 'install', '--upgrade', 'pip', - 'setuptools' ] ) fake_context = builder.ensure_directories(fake_env_dir) - with patch('venv.subprocess.check_call', pip_cmd_checker): + with patch('venv.subprocess.check_output', pip_cmd_checker): builder.upgrade_dependencies(fake_context) @requireVenvCreate @@ -185,8 +272,7 @@ def test_prefixes(self): # check a venv's prefixes rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(self.env_dir, self.bindir, self.exe) - cmd = [envpy, '-c', None] + cmd = [self.envpy(), '-c', None] for prefix, expected in ( ('prefix', self.env_dir), ('exec_prefix', self.env_dir), @@ -194,7 +280,76 @@ def test_prefixes(self): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode()) + self.assertEqual(pathlib.Path(out.strip().decode()), + pathlib.Path(expected), prefix) + + @requireVenvCreate + def test_sysconfig(self): + """ + Test that the sysconfig functions work in a virtual environment. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, symlinks=False) + cmd = [self.envpy(), '-c', None] + for call, expected in ( + # installation scheme + ('get_preferred_scheme("prefix")', 'venv'), + ('get_default_scheme()', 'venv'), + # build environment + ('is_python_build()', str(sysconfig.is_python_build())), + ('get_makefile_filename()', sysconfig.get_makefile_filename()), + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): + with self.subTest(call): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + + @requireVenvCreate + @unittest.skipUnless(can_symlink(), 'Needs symlinks') + def test_sysconfig_symlinks(self): + """ + Test that the sysconfig functions work in a virtual environment. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, symlinks=True) + cmd = [self.envpy(), '-c', None] + for call, expected in ( + # installation scheme + ('get_preferred_scheme("prefix")', 'venv'), + ('get_default_scheme()', 'venv'), + # build environment + ('is_python_build()', str(sysconfig.is_python_build())), + ('get_makefile_filename()', sysconfig.get_makefile_filename()), + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): + with self.subTest(call): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + # HACK: Test fails on POSIX with unversioned binary (PR gh-113033) + #('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) if sys.platform == 'win32': ENV_SUBDIRS = ( @@ -259,6 +414,8 @@ def test_unoverwritable_fails(self): self.assertRaises((ValueError, OSError), venv.create, self.env_dir) self.clear_directory(self.env_dir) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_upgrade(self): """ Test upgrading an existing environment directory. @@ -321,8 +478,7 @@ def test_executable(self): """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) + envpy = self.envpy(real_env_dir=True) out, err = check_output([envpy, '-c', 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) @@ -335,12 +491,89 @@ def test_executable_symlinks(self): rmtree(self.env_dir) builder = venv.EnvBuilder(clear=True, symlinks=True) builder.create(self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) + envpy = self.envpy(real_env_dir=True) out, err = check_output([envpy, '-c', 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_bash(self): + """ + Test that the template strings are quoted properly (bash) + """ + rmtree(self.env_dir) + bash = shutil.which('bash') + if bash is None: + self.skipTest('bash required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate') + test_script = os.path.join(self.env_dir, 'test_special_chars.sh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([bash, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + @unittest.skipIf(sys.platform.startswith('netbsd'), + "NetBSD csh fails with quoted special chars; see gh-139308") + def test_special_chars_csh(self): + """ + Test that the template strings are quoted properly (csh) + """ + rmtree(self.env_dir) + csh = shutil.which('tcsh') or shutil.which('csh') + if csh is None: + self.skipTest('csh required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.csh') + test_script = os.path.join(self.env_dir, 'test_special_chars.csh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([csh, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings on Windows + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_special_chars_windows(self): + """ + Test that the template strings are quoted properly on Windows + """ + rmtree(self.env_dir) + env_name = "'&&^$e" + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_special_chars.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + f'"{activate}" & ' + f'{self.exe} -c "import sys; print(sys.executable)" & ' + f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & ' + 'deactivate') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') def test_unicode_in_batch_file(self): """ @@ -351,13 +584,27 @@ def test_unicode_in_batch_file(self): builder = venv.EnvBuilder(clear=True) builder.create(env_dir) activate = os.path.join(env_dir, self.bindir, 'activate.bat') - envpy = os.path.join(env_dir, self.bindir, self.exe) out, err = check_output( [activate, '&', self.exe, '-c', 'print(0)'], encoding='oem', ) self.assertEqual(out.strip(), '0') + @unittest.skipUnless(os.name == 'nt' and can_symlink(), + 'symlinks on Windows') + def test_failed_symlink(self): + """ + Test handling of failed symlinks on Windows. + """ + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + with patch('os.symlink') as mock_symlink: + mock_symlink.side_effect = OSError() + builder = venv.EnvBuilder(clear=True, symlinks=True) + _, err = self.run_with_capture(builder.create, env_dir) + filepath_regex = r"'[A-Z]:\\\\(?:[^\\\\]+\\\\)*[^\\\\]+'" + self.assertRegex(err, rf"Unable to symlink {filepath_regex} to {filepath_regex}") + @requireVenvCreate def test_multiprocessing(self): """ @@ -370,15 +617,25 @@ def test_multiprocessing(self): rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'from multiprocessing import Pool; ' 'pool = Pool(1); ' 'print(pool.apply_async("Python".lower).get(3)); ' 'pool.terminate()']) self.assertEqual(out.strip(), "python".encode()) + @requireVenvCreate + def test_multiprocessing_recursion(self): + """ + Test that the multiprocessing is able to spawn itself + """ + skip_if_broken_multiprocessing_synchronize() + + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py') + subprocess.check_call([self.envpy(real_env_dir=True), "-I", script]) + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') def test_deactivate_with_strict_bash_opts(self): bash = shutil.which("bash") @@ -404,19 +661,250 @@ def test_macos_env(self): builder = venv.EnvBuilder() builder.create(self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'import os; print("__PYVENV_LAUNCHER__" in os.environ)']) self.assertEqual(out.strip(), 'False'.encode()) + def test_pathsep_error(self): + """ + Test that venv creation fails when the target directory contains + the path separator. + """ + rmtree(self.env_dir) + bad_itempath = self.env_dir + os.pathsep + self.assertRaises(ValueError, venv.create, bad_itempath) + self.assertRaises(ValueError, venv.create, FakePath(bad_itempath)) + + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + @requireVenvCreate + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_zippath_from_non_installed_posix(self): + """ + Test that when create venv from non-installed python, the zip path + value is as expected. + """ + rmtree(self.env_dir) + # First try to create a non-installed python. It's not a real full + # functional non-installed python, but enough for this test. + platlibdir = sys.platlibdir + non_installed_dir = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(rmtree, non_installed_dir) + bindir = os.path.join(non_installed_dir, self.bindir) + os.mkdir(bindir) + shutil.copy2(sys.executable, bindir) + libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) + os.makedirs(libdir) + landmark = os.path.join(libdir, "os.py") + abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}" + zip_landmark = os.path.join(non_installed_dir, + platlibdir, + stdlib_zip) + additional_pythonpath_for_non_installed = [] + + # Copy stdlib files to the non-installed python so venv can + # correctly calculate the prefix. + for eachpath in sys.path: + if eachpath.endswith(".zip"): + if os.path.isfile(eachpath): + shutil.copyfile( + eachpath, + os.path.join(non_installed_dir, platlibdir)) + elif os.path.isfile(os.path.join(eachpath, "os.py")): + names = os.listdir(eachpath) + ignored_names = copy_python_src_ignore(eachpath, names) + for name in names: + if name in ignored_names: + continue + if name == "site-packages": + continue + fn = os.path.join(eachpath, name) + if os.path.isfile(fn): + shutil.copy(fn, libdir) + elif os.path.isdir(fn): + shutil.copytree(fn, os.path.join(libdir, name), + ignore=copy_python_src_ignore) + else: + additional_pythonpath_for_non_installed.append( + eachpath) + cmd = [os.path.join(non_installed_dir, self.bindir, self.exe), + "-m", + "venv", + "--without-pip", + "--without-scm-ignore-files", + self.env_dir] + # Our fake non-installed python is not fully functional because + # it cannot find the extensions. Set PYTHONPATH so it can run the + # venv module correctly. + pythonpath = os.pathsep.join( + additional_pythonpath_for_non_installed) + # For python built with shared enabled. We need to set + # LD_LIBRARY_PATH so the non-installed python can find and link + # libpython.so + ld_library_path = sysconfig.get_config_var("LIBDIR") + if not ld_library_path or sysconfig.is_python_build(): + ld_library_path = os.path.abspath(os.path.dirname(sys.executable)) + if sys.platform == 'darwin': + ld_library_path_env = "DYLD_LIBRARY_PATH" + else: + ld_library_path_env = "LD_LIBRARY_PATH" + child_env = { + "PYTHONPATH": pythonpath, + ld_library_path_env: ld_library_path, + } + if asan_options := os.environ.get("ASAN_OPTIONS"): + # prevent https://github.com/python/cpython/issues/104839 + child_env["ASAN_OPTIONS"] = asan_options + subprocess.check_call(cmd, env=child_env) + # Now check the venv created from the non-installed python has + # correct zip path in pythonpath. + cmd = [self.envpy(), '-S', '-c', 'import sys; print(sys.path)'] + out, err = check_output(cmd) + self.assertTrue(zip_landmark.encode() in out) + + @requireVenvCreate + def test_activate_shell_script_has_no_dos_newlines(self): + """ + Test that the `activate` shell script contains no CR LF. + This is relevant for Cygwin, as the Windows build might have + converted line endings accidentally. + """ + venv_dir = pathlib.Path(self.env_dir) + rmtree(venv_dir) + [[scripts_dir], *_] = self.ENV_SUBDIRS + script_path = venv_dir / scripts_dir / "activate" + venv.create(venv_dir) + with open(script_path, 'rb') as script: + for i, line in enumerate(script, 1): + error_message = f"CR LF found in line {i}" + self.assertFalse(line.endswith(b'\r\n'), error_message) + + @requireVenvCreate + def test_scm_ignore_files_git(self): + """ + Test that a .gitignore file is created when "git" is specified. + The file should contain a `*\n` line. + """ + self.run_with_capture(venv.create, self.env_dir, + scm_ignore_files={'git'}) + file_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', file_lines) + + @requireVenvCreate + def test_create_scm_ignore_files_multiple(self): + """ + Test that ``scm_ignore_files`` can work with multiple SCMs. + """ + bzrignore_name = ".bzrignore" + contents = "# For Bazaar.\n*\n" + + class BzrEnvBuilder(venv.EnvBuilder): + def create_bzr_ignore_file(self, context): + gitignore_path = os.path.join(context.env_dir, bzrignore_name) + with open(gitignore_path, 'w', encoding='utf-8') as file: + file.write(contents) + + builder = BzrEnvBuilder(scm_ignore_files={'git', 'bzr'}) + self.run_with_capture(builder.create, self.env_dir) + + gitignore_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', gitignore_lines) + + bzrignore = self.get_text_file_contents(bzrignore_name) + self.assertEqual(bzrignore, contents) + + @requireVenvCreate + def test_create_scm_ignore_files_empty(self): + """ + Test that no default ignore files are created when ``scm_ignore_files`` + is empty. + """ + # scm_ignore_files is set to frozenset() by default. + self.run_with_capture(venv.create, self.env_dir) + with self.assertRaises(FileNotFoundError): + self.get_text_file_contents('.gitignore') + + self.assertIn("--without-scm-ignore-files", + self.get_text_file_contents('pyvenv.cfg')) + + @requireVenvCreate + def test_cli_with_scm_ignore_files(self): + """ + Test that default SCM ignore files are created by default via the CLI. + """ + self.run_with_capture(venv.main, ['--without-pip', self.env_dir]) + + gitignore_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', gitignore_lines) + + @requireVenvCreate + def test_cli_without_scm_ignore_files(self): + """ + Test that ``--without-scm-ignore-files`` doesn't create SCM ignore files. + """ + args = ['--without-pip', '--without-scm-ignore-files', self.env_dir] + self.run_with_capture(venv.main, args) + + with self.assertRaises(FileNotFoundError): + self.get_text_file_contents('.gitignore') + + def test_venv_same_path(self): + same_path = venv.EnvBuilder._same_path + if sys.platform == 'win32': + # Case-insensitive, and handles short/long names + tests = [ + (True, TESTFN, TESTFN), + (True, TESTFN.lower(), TESTFN.upper()), + ] + import _winapi + # ProgramFiles is the most reliable path that will have short/long + progfiles = os.getenv('ProgramFiles') + if progfiles: + tests = [ + *tests, + (True, progfiles, progfiles), + (True, _winapi.GetShortPathName(progfiles), _winapi.GetLongPathName(progfiles)), + ] + else: + # Just a simple case-sensitive comparison + tests = [ + (True, TESTFN, TESTFN), + (False, TESTFN.lower(), TESTFN.upper()), + ] + for r, path1, path2 in tests: + with self.subTest(f"{path1}-{path2}"): + if r: + self.assertTrue(same_path(path1, path2)) + else: + self.assertFalse(same_path(path1, path2)) + + # gh-126084: venvwlauncher should run pythonw, not python + @requireVenvCreate + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_venvwlauncher(self): + """ + Test that the GUI launcher runs the GUI python. + """ + rmtree(self.env_dir) + venv.create(self.env_dir) + exename = self.exe + # Retain the debug suffix if present + if "python" in exename and not "pythonw" in exename: + exename = exename.replace("python", "pythonw") + envpyw = os.path.join(self.env_dir, self.bindir, exename) + try: + subprocess.check_call([envpyw, "-c", "import sys; " + "assert sys._base_executable.endswith('%s')" % exename]) + except subprocess.CalledProcessError: + self.fail("venvwlauncher.exe did not run %s" % exename) + + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" def assert_pip_not_installed(self): - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'try:\n import pip\nexcept ImportError:\n print("OK")']) # We force everything to text, so unittest gives the detailed diff # if we get unexpected results @@ -478,20 +966,14 @@ def do_test_with_pip(self, system_site_packages): # Actually run the create command with all that unhelpful # config in place to ensure we ignore it - try: + with self.nicer_error(): self.run_with_capture(venv.create, self.env_dir, system_site_packages=system_site_packages, with_pip=True) - except subprocess.CalledProcessError as exc: - # The output this produces can be a little hard to read, - # but at least it has all the details - details = exc.output.decode(errors="replace") - msg = "{}\n\n**Subprocess Output**\n{}" - self.fail(msg.format(exc, details)) # Ensure pip is available in the virtual environment - envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe) # Ignore DeprecationWarning since pip code is not part of Python - out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', + out, err = check_output([self.envpy(real_env_dir=True), + '-W', 'ignore::DeprecationWarning', '-W', 'ignore::ImportWarning', '-I', '-m', 'pip', '--version']) # We force everything to text, so unittest gives the detailed diff @@ -508,13 +990,14 @@ def do_test_with_pip(self, system_site_packages): # Check the private uninstall command provided for the Windows # installers works (at least in a virtual environment) with EnvironmentVarGuard() as envvars: - # It seems ensurepip._uninstall calls subprocesses which do not - # inherit the interpreter settings. - envvars["PYTHONWARNINGS"] = "ignore" - out, err = check_output([envpy, - '-W', 'ignore::DeprecationWarning', - '-W', 'ignore::ImportWarning', '-I', - '-m', 'ensurepip._uninstall']) + with self.nicer_error(): + # It seems ensurepip._uninstall calls subprocesses which do not + # inherit the interpreter settings. + envvars["PYTHONWARNINGS"] = "ignore" + out, err = check_output([self.envpy(real_env_dir=True), + '-W', 'ignore::DeprecationWarning', + '-W', 'ignore::ImportWarning', '-I', + '-m', 'ensurepip._uninstall']) # We force everything to text, so unittest gives the detailed diff # if we get unexpected results err = err.decode("latin-1") # Force to text, prevent decoding errors @@ -527,25 +1010,51 @@ def do_test_with_pip(self, system_site_packages): err = re.sub("^(WARNING: )?The directory .* or its parent directory " "is not owned or is not writable by the current user.*$", "", err, flags=re.MULTILINE) + # Ignore warning about missing optional module: + try: + import ssl + except ImportError: + err = re.sub( + "^WARNING: Disabling truststore since ssl support is missing$", + "", + err, flags=re.MULTILINE) self.assertEqual(err.rstrip(), "") # Being fairly specific regarding the expected behaviour for the # initial bundling phase in Python 3.4. If the output changes in # future pip versions, this test can likely be relaxed further. out = out.decode("latin-1") # Force to text, prevent decoding errors self.assertIn("Successfully uninstalled pip", out) - self.assertIn("Successfully uninstalled setuptools", out) # Check pip is now gone from the virtual environment. This only # applies in the system_site_packages=False case, because in the # other case, pip may still be available in the system site-packages if not system_site_packages: self.assert_pip_not_installed() - # Issue #26610: pip/pep425tags.py requires ctypes - @unittest.skipUnless(ctypes, 'pip requires ctypes') - @requires_zlib() + @contextlib.contextmanager + def nicer_error(self): + """ + Capture output from a failed subprocess for easier debugging. + + The output this handler produces can be a little hard to read, + but at least it has all the details. + """ + try: + yield + except subprocess.CalledProcessError as exc: + out = (exc.output or b'').decode(errors="replace") + err = (exc.stderr or b'').decode(errors="replace") + self.fail( + f"{exc}\n\n" + f"**Subprocess Output**\n{out}\n\n" + f"**Subprocess Error**\n{err}" + ) + + @requires_venv_with_pip() + @requires_resource('cpu') def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py new file mode 100644 index 00000000000..eae885a6130 --- /dev/null +++ b/Lib/test/test_wait3.py @@ -0,0 +1,53 @@ +"""This test checks for correct wait3() behavior. +""" + +import os +import subprocess +import sys +import unittest +from test.fork_wait import ForkWait +from test import support + +if not support.has_fork_support: + raise unittest.SkipTest("requires working os.fork()") + +if not hasattr(os, 'wait3'): + raise unittest.SkipTest("os.wait3 not defined") + +class Wait3Test(ForkWait): + def wait_impl(self, cpid, *, exitcode): + # This many iterations can be required, since some previously run + # tests (e.g. test_ctypes) could have spawned a lot of children + # very quickly. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + # wait3() shouldn't hang, but some of the buildbots seem to hang + # in the forking tests. This is an attempt to fix the problem. + spid, status, rusage = os.wait3(os.WNOHANG) + if spid == cpid: + break + + self.assertEqual(spid, cpid) + self.assertEqual(os.waitstatus_to_exitcode(status), exitcode) + self.assertTrue(rusage) + + def test_wait3_rusage_initialized(self): + # Ensure a successful wait3() call where no child was ready to report + # its exit status does not return uninitialized memory in the rusage + # structure. See bpo-36279. + args = [sys.executable, '-c', 'import sys; sys.stdin.read()'] + proc = subprocess.Popen(args, stdin=subprocess.PIPE) + try: + pid, status, rusage = os.wait3(os.WNOHANG) + self.assertEqual(0, pid) + self.assertEqual(0, status) + self.assertEqual(0, sum(rusage)) + finally: + proc.stdin.close() + proc.wait() + + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py new file mode 100644 index 00000000000..67afab1d6f2 --- /dev/null +++ b/Lib/test/test_wait4.py @@ -0,0 +1,38 @@ +"""This test checks for correct wait4() behavior. +""" + +import os +import sys +import unittest +from test.fork_wait import ForkWait +from test import support + +# If either of these do not exist, skip this test. +if not support.has_fork_support: + raise unittest.SkipTest("requires working os.fork()") + +support.get_attribute(os, 'wait4') + + +class Wait4Test(ForkWait): + def wait_impl(self, cpid, *, exitcode): + option = os.WNOHANG + if sys.platform.startswith('aix'): + # Issue #11185: wait4 is broken on AIX and will always return 0 + # with WNOHANG. + option = 0 + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + # wait4() shouldn't hang, but some of the buildbots seem to hang + # in the forking tests. This is an attempt to fix the problem. + spid, status, rusage = os.wait4(cpid, option) + if spid == cpid: + break + self.assertEqual(spid, cpid) + self.assertEqual(os.waitstatus_to_exitcode(status), exitcode) + self.assertTrue(rusage) + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py new file mode 100644 index 00000000000..71dc20d0b59 --- /dev/null +++ b/Lib/test/test_warnings/__init__.py @@ -0,0 +1,2216 @@ +from contextlib import contextmanager +import linecache +import os +import importlib +import inspect +from io import StringIO +import re +import sys +import textwrap +import types +from typing import overload, get_overloads +import unittest +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import warnings_helper +from test.support import force_not_colorized +from test.support.script_helper import assert_python_ok, assert_python_failure + +from test.test_warnings.data import package_helper +from test.test_warnings.data import stacklevel as warning_tests + +import warnings as original_warnings +from warnings import deprecated + + +py_warnings = import_helper.import_fresh_module('_py_warnings') +py_warnings._set_module(py_warnings) + +c_warnings = import_helper.import_fresh_module( + "warnings", fresh=["_warnings", "_py_warnings"] +) +c_warnings._set_module(c_warnings) + +@contextmanager +def warnings_state(module): + """Use a specific warnings implementation in warning_tests.""" + global __warningregistry__ + for to_clear in (sys, warning_tests): + try: + to_clear.__warningregistry__.clear() + except AttributeError: + pass + try: + __warningregistry__.clear() + except NameError: + pass + original_warnings = warning_tests.warnings + if module._use_context: + saved_context, context = module._new_context() + else: + original_filters = module.filters + module.filters = original_filters[:] + try: + module.simplefilter("once") + warning_tests.warnings = module + yield + finally: + warning_tests.warnings = original_warnings + if module._use_context: + module._set_context(saved_context) + else: + module.filters = original_filters + + +class TestWarning(Warning): + pass + + +class BaseTest: + + """Basic bookkeeping required for testing.""" + + def setUp(self): + self.old_unittest_module = unittest.case.warnings + # The __warningregistry__ needs to be in a pristine state for tests + # to work properly. + if '__warningregistry__' in globals(): + del globals()['__warningregistry__'] + if hasattr(warning_tests, '__warningregistry__'): + del warning_tests.__warningregistry__ + if hasattr(sys, '__warningregistry__'): + del sys.__warningregistry__ + # The 'warnings' module must be explicitly set so that the proper + # interaction between _warnings and 'warnings' can be controlled. + sys.modules['warnings'] = self.module + # Ensure that unittest.TestCase.assertWarns() uses the same warnings + # module than warnings.catch_warnings(). Otherwise, + # warnings.catch_warnings() will be unable to remove the added filter. + unittest.case.warnings = self.module + super(BaseTest, self).setUp() + + def tearDown(self): + sys.modules['warnings'] = original_warnings + unittest.case.warnings = self.old_unittest_module + super(BaseTest, self).tearDown() + +class PublicAPITests(BaseTest): + + """Ensures that the correct values are exposed in the + public API. + """ + + def test_module_all_attribute(self): + self.assertHasAttr(self.module, '__all__') + target_api = ["warn", "warn_explicit", "showwarning", + "formatwarning", "filterwarnings", "simplefilter", + "resetwarnings", "catch_warnings", "deprecated"] + self.assertSetEqual(set(self.module.__all__), + set(target_api)) + +class CPublicAPITests(PublicAPITests, unittest.TestCase): + module = c_warnings + +class PyPublicAPITests(PublicAPITests, unittest.TestCase): + module = py_warnings + +class FilterTests(BaseTest): + + """Testing the filtering functionality.""" + + def test_error(self): + with self.module.catch_warnings() as w: + self.module.resetwarnings() + self.module.filterwarnings("error", category=UserWarning) + self.assertRaises(UserWarning, self.module.warn, + "FilterTests.test_error") + + def test_error_after_default(self): + with self.module.catch_warnings() as w: + self.module.resetwarnings() + message = "FilterTests.test_ignore_after_default" + def f(): + self.module.warn(message, UserWarning) + + with support.captured_stderr() as stderr: + f() + stderr = stderr.getvalue() + self.assertIn("UserWarning: FilterTests.test_ignore_after_default", + stderr) + self.assertIn("self.module.warn(message, UserWarning)", + stderr) + + self.module.filterwarnings("error", category=UserWarning) + self.assertRaises(UserWarning, f) + + def test_ignore(self): + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings("ignore", category=UserWarning) + self.module.warn("FilterTests.test_ignore", UserWarning) + self.assertEqual(len(w), 0) + self.assertEqual(list(__warningregistry__), ['version']) + + def test_ignore_after_default(self): + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + message = "FilterTests.test_ignore_after_default" + def f(): + self.module.warn(message, UserWarning) + f() + self.module.filterwarnings("ignore", category=UserWarning) + f() + f() + self.assertEqual(len(w), 1) + + def test_always_and_all(self): + for mode in {"always", "all"}: + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings(mode, category=UserWarning) + message = "FilterTests.test_always_and_all" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) + + def test_always_and_all_after_default(self): + for mode in {"always", "all"}: + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + message = "FilterTests.test_always_and_all_after_ignore" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 1) + self.module.filterwarnings(mode, category=UserWarning) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 3) + self.assertEqual(w[-1].message.args[0], message) + + def test_default(self): + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings("default", category=UserWarning) + message = UserWarning("FilterTests.test_default") + for x in range(2): + self.module.warn(message, UserWarning) + if x == 0: + self.assertEqual(w[-1].message, message) + del w[:] + elif x == 1: + self.assertEqual(len(w), 0) + else: + raise ValueError("loop variant unhandled") + + def test_module(self): + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings("module", category=UserWarning) + message = UserWarning("FilterTests.test_module") + self.module.warn(message, UserWarning) + self.assertEqual(w[-1].message, message) + del w[:] + self.module.warn(message, UserWarning) + self.assertEqual(len(w), 0) + + def test_once(self): + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings("once", category=UserWarning) + message = UserWarning("FilterTests.test_once") + self.module.warn_explicit(message, UserWarning, "__init__.py", + 42) + self.assertEqual(w[-1].message, message) + del w[:] + self.module.warn_explicit(message, UserWarning, "__init__.py", + 13) + self.assertEqual(len(w), 0) + self.module.warn_explicit(message, UserWarning, "test_warnings2.py", + 42) + self.assertEqual(len(w), 0) + + def test_filter_module(self): + MS_WINDOWS = (sys.platform == 'win32') + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'package\.module\z') + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='package.module') + self.assertEqual(len(w), 1) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module='package') + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='package.module') + self.assertEqual(len(w), 1) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='other.package.module') + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/otherpackage/module.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'/path/to/package/module\z') + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + self.assertEqual(len(w), 2) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/PATH/TO/PACKAGE/MODULE', 42) + if MS_WINDOWS: + if self.module is py_warnings: + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.PY', 42) + self.assertEqual(len(w), 3) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module/__init__.py', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.pyw', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'\path\to\package\module', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'/path/to/package/__init__\z') + self.module.warn_explicit('msg', UserWarning, '/path/to/package/__init__.py', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/__init__', 42) + self.assertEqual(len(w), 2) + + if MS_WINDOWS: + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'C:\\path\\to\\package\\module\z') + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.py', 42) + self.assertEqual(len(w), 2) + if self.module is py_warnings: + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PY', 42) + self.assertEqual(len(w), 3) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.pyw', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\PATH\TO\PACKAGE\MODULE', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:/path/to/package/module', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module\__init__.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'<unknown>\z') + self.module.warn_explicit('msg', UserWarning, '', 42) + self.assertEqual(len(w), 1) + + def test_module_globals(self): + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter("always", UserWarning) + + # bpo-33509: module_globals=None must not crash + self.module.warn_explicit('msg', UserWarning, "filename", 42, + module_globals=None) + self.assertEqual(len(w), 1) + + # Invalid module_globals type + with self.assertRaises(TypeError): + self.module.warn_explicit('msg', UserWarning, "filename", 42, + module_globals=True) + self.assertEqual(len(w), 1) + + # Empty module_globals + self.module.warn_explicit('msg', UserWarning, "filename", 42, + module_globals={}) + self.assertEqual(len(w), 2) + + def test_inheritance(self): + with self.module.catch_warnings() as w: + self.module.resetwarnings() + self.module.filterwarnings("error", category=Warning) + self.assertRaises(UserWarning, self.module.warn, + "FilterTests.test_inheritance", UserWarning) + + def test_ordering(self): + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings("ignore", category=UserWarning) + self.module.filterwarnings("error", category=UserWarning, + append=True) + del w[:] + try: + self.module.warn("FilterTests.test_ordering", UserWarning) + except UserWarning: + self.fail("order handling for actions failed") + self.assertEqual(len(w), 0) + + def test_filterwarnings(self): + # Test filterwarnings(). + # Implicitly also tests resetwarnings(). + with self.module.catch_warnings(record=True) as w: + self.module.filterwarnings("error", "", Warning, "", 0) + self.assertRaises(UserWarning, self.module.warn, 'convert to error') + + self.module.resetwarnings() + text = 'handle normally' + self.module.warn(text) + self.assertEqual(str(w[-1].message), text) + self.assertIs(w[-1].category, UserWarning) + + self.module.filterwarnings("ignore", "", Warning, "", 0) + text = 'filtered out' + self.module.warn(text) + self.assertNotEqual(str(w[-1].message), text) + + self.module.resetwarnings() + self.module.filterwarnings("error", "hex*", Warning, "", 0) + self.assertRaises(UserWarning, self.module.warn, 'hex/oct') + text = 'nonmatching text' + self.module.warn(text) + self.assertEqual(str(w[-1].message), text) + self.assertIs(w[-1].category, UserWarning) + + def test_message_matching(self): + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter("ignore", UserWarning) + self.module.filterwarnings("error", "match", UserWarning) + self.assertRaises(UserWarning, self.module.warn, "match") + self.assertRaises(UserWarning, self.module.warn, "match prefix") + self.module.warn("suffix match") + self.assertEqual(w, []) + self.module.warn("something completely different") + self.assertEqual(w, []) + + def test_mutate_filter_list(self): + class X: + def match(self, a): + L[:] = [] + + L = [("default",X(),UserWarning,X(),0) for i in range(2)] + with self.module.catch_warnings(record=True) as w: + self.module.filters = L + self.module.warn_explicit(UserWarning("b"), None, "f.py", 42) + self.assertEqual(str(w[-1].message), "b") + + def test_filterwarnings_duplicate_filters(self): + with self.module.catch_warnings(): + self.module.resetwarnings() + self.module.filterwarnings("error", category=UserWarning) + self.assertEqual(len(self.module._get_filters()), 1) + self.module.filterwarnings("ignore", category=UserWarning) + self.module.filterwarnings("error", category=UserWarning) + self.assertEqual( + len(self.module._get_filters()), 2, + "filterwarnings inserted duplicate filter" + ) + self.assertEqual( + self.module._get_filters()[0][0], "error", + "filterwarnings did not promote filter to " + "the beginning of list" + ) + + def test_simplefilter_duplicate_filters(self): + with self.module.catch_warnings(): + self.module.resetwarnings() + self.module.simplefilter("error", category=UserWarning) + self.assertEqual(len(self.module._get_filters()), 1) + self.module.simplefilter("ignore", category=UserWarning) + self.module.simplefilter("error", category=UserWarning) + self.assertEqual( + len(self.module._get_filters()), 2, + "simplefilter inserted duplicate filter" + ) + self.assertEqual( + self.module._get_filters()[0][0], "error", + "simplefilter did not promote filter to the beginning of list" + ) + + def test_append_duplicate(self): + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.simplefilter("ignore") + self.module.simplefilter("error", append=True) + self.module.simplefilter("ignore", append=True) + self.module.warn("test_append_duplicate", category=UserWarning) + self.assertEqual(len(self.module._get_filters()), 2, + "simplefilter inserted duplicate filter" + ) + self.assertEqual(len(w), 0, + "appended duplicate changed order of filters" + ) + + def test_argument_validation(self): + with self.assertRaises(ValueError): + self.module.filterwarnings(action='foo') + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', message=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', category=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', category=int) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', module=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', lineno=int) + with self.assertRaises(ValueError): + self.module.filterwarnings('ignore', lineno=-1) + with self.assertRaises(ValueError): + self.module.simplefilter(action='foo') + with self.assertRaises(TypeError): + self.module.simplefilter('ignore', lineno=int) + with self.assertRaises(ValueError): + self.module.simplefilter('ignore', lineno=-1) + + def test_catchwarnings_with_simplefilter_ignore(self): + with self.module.catch_warnings(module=self.module): + self.module.resetwarnings() + self.module.simplefilter("error") + with self.module.catch_warnings(action="ignore"): + self.module.warn("This will be ignored") + + def test_catchwarnings_with_simplefilter_error(self): + with self.module.catch_warnings(): + self.module.resetwarnings() + with self.module.catch_warnings( + action="error", category=FutureWarning + ): + with support.captured_stderr() as stderr: + error_msg = "Other types of warnings are not errors" + self.module.warn(error_msg) + self.assertRaises(FutureWarning, + self.module.warn, FutureWarning("msg")) + stderr = stderr.getvalue() + self.assertIn(error_msg, stderr) + +class CFilterTests(FilterTests, unittest.TestCase): + module = c_warnings + +class PyFilterTests(FilterTests, unittest.TestCase): + module = py_warnings + + +class WarnTests(BaseTest): + + """Test warnings.warn() and warnings.warn_explicit().""" + + def test_message(self): + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter("once") + for i in range(4): + text = 'multi %d' %i # Different text on each call. + self.module.warn(text) + self.assertEqual(str(w[-1].message), text) + self.assertIs(w[-1].category, UserWarning) + + # Issue 3639 + def test_warn_nonstandard_types(self): + # warn() should handle non-standard types without issue. + for ob in (Warning, None, 42): + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter("once") + self.module.warn(ob) + # Don't directly compare objects since + # ``Warning() != Warning()``. + self.assertEqual(str(w[-1].message), str(UserWarning(ob))) + + def test_filename(self): + with warnings_state(self.module): + with self.module.catch_warnings(record=True) as w: + warning_tests.inner("spam1") + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + warning_tests.outer("spam2") + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + + def test_stacklevel(self): + # Test stacklevel argument + # make sure all messages are different, so the warning won't be skipped + with warnings_state(self.module): + with self.module.catch_warnings(record=True) as w: + warning_tests.inner("spam3", stacklevel=1) + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + warning_tests.outer("spam4", stacklevel=1) + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + + warning_tests.inner("spam5", stacklevel=2) + self.assertEqual(os.path.basename(w[-1].filename), + "__init__.py") + warning_tests.outer("spam6", stacklevel=2) + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + warning_tests.outer("spam6.5", stacklevel=3) + self.assertEqual(os.path.basename(w[-1].filename), + "__init__.py") + + warning_tests.inner("spam7", stacklevel=9999) + self.assertEqual(os.path.basename(w[-1].filename), + "<sys>") + + @unittest.expectedFailure # TODO: RUSTPYTHON; + /Users/al03219714/Projects/RustPython1/crates/pylib/Lib/test/test_warnings/__init__.py + def test_stacklevel_import(self): + # Issue #24305: With stacklevel=2, module-level warnings should work. + import_helper.unload('test.test_warnings.data.import_warning') + with warnings_state(self.module): + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('always') + import test.test_warnings.data.import_warning # noqa: F401 + self.assertEqual(len(w), 1) + self.assertEqual(w[0].filename, __file__) + + def test_skip_file_prefixes(self): + with warnings_state(self.module): + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('always') + + # Warning never attributed to the data/ package. + package_helper.inner_api( + "inner_api", stacklevel=2, + warnings_module=warning_tests.warnings) + self.assertEqual(w[-1].filename, __file__) + warning_tests.package("package api", stacklevel=2) + self.assertEqual(w[-1].filename, __file__) + self.assertEqual(w[-2].filename, w[-1].filename) + # Low stacklevels are overridden to 2 behavior. + warning_tests.package("package api 1", stacklevel=1) + self.assertEqual(w[-1].filename, __file__) + warning_tests.package("package api 0", stacklevel=0) + self.assertEqual(w[-1].filename, __file__) + warning_tests.package("package api -99", stacklevel=-99) + self.assertEqual(w[-1].filename, __file__) + + # The stacklevel still goes up out of the package. + warning_tests.package("prefix02", stacklevel=3) + self.assertIn("unittest", w[-1].filename) + + def test_skip_file_prefixes_file_path(self): + # see: gh-126209 + with warnings_state(self.module): + skipped = warning_tests.__file__ + with self.module.catch_warnings(record=True) as w: + warning_tests.outer("msg", skip_file_prefixes=(skipped,)) + + self.assertEqual(len(w), 1) + self.assertNotEqual(w[-1].filename, skipped) + + def test_skip_file_prefixes_type_errors(self): + with warnings_state(self.module): + warn = warning_tests.warnings.warn + with self.assertRaises(TypeError): + warn("msg", skip_file_prefixes=[]) + with self.assertRaises(TypeError): + warn("msg", skip_file_prefixes=(b"bytes",)) + with self.assertRaises(TypeError): + warn("msg", skip_file_prefixes="a sequence of strs") + + def test_exec_filename(self): + filename = "<warnings-test>" + codeobj = compile(("import warnings\n" + "warnings.warn('hello', UserWarning)"), + filename, "exec") + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter("always", category=UserWarning) + exec(codeobj) + self.assertEqual(w[0].filename, filename) + + def test_warn_explicit_non_ascii_filename(self): + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings("always", category=UserWarning) + filenames = ["nonascii\xe9\u20ac", "surrogate\udc80"] + for filename in filenames: + try: + os.fsencode(filename) + except UnicodeEncodeError: + continue + self.module.warn_explicit("text", UserWarning, filename, 1) + self.assertEqual(w[-1].filename, filename) + + def test_warn_explicit_type_errors(self): + # warn_explicit() should error out gracefully if it is given objects + # of the wrong types. + # lineno is expected to be an integer. + self.assertRaises(TypeError, self.module.warn_explicit, + None, UserWarning, None, None) + # Either 'message' needs to be an instance of Warning or 'category' + # needs to be a subclass. + self.assertRaises(TypeError, self.module.warn_explicit, + None, None, None, 1) + # 'registry' must be a dict or None. + self.assertRaises((TypeError, AttributeError), + self.module.warn_explicit, + None, Warning, None, 1, registry=42) + + def test_bad_str(self): + # issue 6415 + # Warnings instance with a bad format string for __str__ should not + # trigger a bus error. + class BadStrWarning(Warning): + """Warning with a bad format string for __str__.""" + def __str__(self): + return ("A bad formatted string %(err)" % + {"err" : "there is no %(err)s"}) + + with self.assertRaises(ValueError): + self.module.warn(BadStrWarning()) + + def test_warning_classes(self): + class MyWarningClass(Warning): + pass + + class NonWarningSubclass: + pass + + # passing a non-subclass of Warning should raise a TypeError + with self.assertRaises(TypeError) as cm: + self.module.warn('bad warning category', '') + self.assertIn('category must be a Warning subclass, not ', + str(cm.exception)) + + with self.assertRaises(TypeError) as cm: + self.module.warn('bad warning category', NonWarningSubclass) + self.assertIn('category must be a Warning subclass, not ', + str(cm.exception)) + + # check that warning instances also raise a TypeError + with self.assertRaises(TypeError) as cm: + self.module.warn('bad warning category', MyWarningClass()) + self.assertIn('category must be a Warning subclass, not ', + str(cm.exception)) + + with self.module.catch_warnings(): + self.module.resetwarnings() + self.module.filterwarnings('default') + with self.assertWarns(MyWarningClass) as cm: + self.module.warn('good warning category', MyWarningClass) + self.assertEqual('good warning category', str(cm.warning)) + + with self.assertWarns(UserWarning) as cm: + self.module.warn('good warning category', None) + self.assertEqual('good warning category', str(cm.warning)) + + with self.assertWarns(MyWarningClass) as cm: + self.module.warn('good warning category', MyWarningClass) + self.assertIsInstance(cm.warning, Warning) + + def check_module_globals(self, module_globals): + with self.module.catch_warnings(record=True) as w: + self.module.filterwarnings('default') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, UserWarning) + self.assertEqual(str(w[0].message), 'eggs') + + def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with self.module.catch_warnings(record=True) as w: + self.module.filterwarnings('always') + with self.assertRaisesRegex(errtype, re.escape(errmsg)): + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 0) + + def check_module_globals_deprecated(self, module_globals, msg): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with self.module.catch_warnings(record=True) as w: + self.module.filterwarnings('always') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 2) + self.assertEqual(w[0].category, DeprecationWarning) + self.assertEqual(str(w[0].message), msg) + self.assertEqual(w[1].category, UserWarning) + self.assertEqual(str(w[1].message), 'eggs') + + def test_gh86298_no_loader_and_no_spec(self): + self.check_module_globals({'__name__': 'bar'}) + + def test_gh86298_loader_is_none_and_no_spec(self): + self.check_module_globals({'__name__': 'bar', '__loader__': None}) + + def test_gh86298_no_loader_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_loader_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, + '__spec__': types.SimpleNamespace(loader=None)}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_spec_is_none(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec_loader(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_and_spec_loader_disagree(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace(loader=object())}, + 'Module globals; __loader__ != __spec__.loader') + + def test_gh86298_no_loader_and_no_spec_loader(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader', AttributeError) + + def test_gh86298_no_loader_with_spec_loader_okay(self): + self.check_module_globals( + {'__name__': 'bar', + '__spec__': types.SimpleNamespace(loader=object())}) + +class CWarnTests(WarnTests, unittest.TestCase): + module = c_warnings + + # As an early adopter, we sanity check the + # test.import_helper.import_fresh_module utility function + def test_accelerated(self): + self.assertIsNot(original_warnings, self.module) + self.assertNotHasAttr(self.module.warn, '__code__') + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 + def test_gh86298_loader_and_spec_loader_disagree(self): + return super().test_gh86298_loader_and_spec_loader_disagree() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 + def test_gh86298_no_spec(self): + return super().test_gh86298_no_spec() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 + def test_gh86298_no_spec_loader(self): + return super().test_gh86298_no_spec_loader() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 1 != 2 + def test_gh86298_spec_is_none(self): + return super().test_gh86298_spec_is_none() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: AttributeError not raised + def test_gh86298_no_loader_and_no_spec_loader(self): + return super().test_gh86298_no_loader_and_no_spec_loader() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_gh86298_loader_is_none_and_spec_is_none(self): + return super().test_gh86298_loader_is_none_and_spec_is_none() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_gh86298_loader_is_none_and_spec_loader_is_none(self): + return super().test_gh86298_loader_is_none_and_spec_loader_is_none() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised + def test_gh86298_no_loader_and_spec_is_none(self): + return super().test_gh86298_no_loader_and_spec_is_none() + +class PyWarnTests(WarnTests, unittest.TestCase): + module = py_warnings + + # As an early adopter, we sanity check the + # test.import_helper.import_fresh_module utility function + def test_pure_python(self): + self.assertIsNot(original_warnings, self.module) + self.assertHasAttr(self.module.warn, '__code__') + + +class WCmdLineTests(BaseTest): + + def test_improper_input(self): + # Uses the private _setoption() function to test the parsing + # of command-line warning arguments + with self.module.catch_warnings(): + self.assertRaises(self.module._OptionError, + self.module._setoption, '1:2:3:4:5:6') + self.assertRaises(self.module._OptionError, + self.module._setoption, 'bogus::Warning') + self.assertRaises(self.module._OptionError, + self.module._setoption, 'ignore:2::4:-5') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::123') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::123abc') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::===') + with self.assertRaisesRegex(self.module._OptionError, 'Wärning'): + self.module._setoption('ignore::Wärning') + self.module._setoption('error::Warning::0') + self.assertRaises(UserWarning, self.module.warn, 'convert to error') + + def test_import_from_module(self): + with self.module.catch_warnings(): + self.module._setoption('ignore::Warning') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::TestWarning') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::test.test_warnings.bogus') + self.module._setoption('error::test.test_warnings.TestWarning') + with self.assertRaises(TestWarning): + self.module.warn('test warning', TestWarning) + + +class CWCmdLineTests(WCmdLineTests, unittest.TestCase): + module = c_warnings + + +class PyWCmdLineTests(WCmdLineTests, unittest.TestCase): + module = py_warnings + + def test_improper_option(self): + # Same as above, but check that the message is printed out when + # the interpreter is executed. This also checks that options are + # actually parsed at all. + rc, out, err = assert_python_ok("-Wxxx", "-c", "pass") + self.assertIn(b"Invalid -W option ignored: invalid action: 'xxx'", err) + + def test_warnings_bootstrap(self): + # Check that the warnings module does get loaded when -W<some option> + # is used (see issue #10372 for an example of silent bootstrap failure). + rc, out, err = assert_python_ok("-Wi", "-c", + "import sys; sys.modules['warnings'].warn('foo', RuntimeWarning)") + # '-Wi' was observed + self.assertFalse(out.strip()) + self.assertNotIn(b'RuntimeWarning', err) + + +class _WarningsTests(BaseTest, unittest.TestCase): + + """Tests specific to the _warnings module.""" + + module = c_warnings + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: UserWarning not raised by warn + def test_filter(self): + # Everything should function even if 'filters' is not in warnings. + with self.module.catch_warnings() as w: + self.module.filterwarnings("error", "", Warning, "", 0) + self.assertRaises(UserWarning, self.module.warn, + 'convert to error') + del self.module.filters + self.assertRaises(UserWarning, self.module.warn, + 'convert to error') + + def test_onceregistry(self): + # Replacing or removing the onceregistry should be okay. + global __warningregistry__ + message = UserWarning('onceregistry test') + try: + original_registry = self.module.onceregistry + __warningregistry__ = {} + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + self.module.filterwarnings("once", category=UserWarning) + self.module.warn_explicit(message, UserWarning, "file", 42) + self.assertEqual(w[-1].message, message) + del w[:] + self.module.warn_explicit(message, UserWarning, "file", 42) + self.assertEqual(len(w), 0) + # Test the resetting of onceregistry. + self.module.onceregistry = {} + __warningregistry__ = {} + self.module.warn('onceregistry test') + self.assertEqual(w[-1].message.args, message.args) + # Removal of onceregistry is okay. + del w[:] + del self.module.onceregistry + __warningregistry__ = {} + self.module.warn_explicit(message, UserWarning, "file", 42) + self.assertEqual(len(w), 0) + finally: + self.module.onceregistry = original_registry + + def test_default_action(self): + # Replacing or removing defaultaction should be okay. + message = UserWarning("defaultaction test") + original = self.module.defaultaction + try: + with self.module.catch_warnings(record=True) as w: + self.module.resetwarnings() + registry = {} + self.module.warn_explicit(message, UserWarning, "<test>", 42, + registry=registry) + self.assertEqual(w[-1].message, message) + self.assertEqual(len(w), 1) + # One actual registry key plus the "version" key + self.assertEqual(len(registry), 2) + self.assertIn("version", registry) + del w[:] + # Test removal. + del self.module.defaultaction + __warningregistry__ = {} + registry = {} + self.module.warn_explicit(message, UserWarning, "<test>", 43, + registry=registry) + self.assertEqual(w[-1].message, message) + self.assertEqual(len(w), 1) + self.assertEqual(len(registry), 2) + del w[:] + # Test setting. + self.module.defaultaction = "ignore" + __warningregistry__ = {} + registry = {} + self.module.warn_explicit(message, UserWarning, "<test>", 44, + registry=registry) + self.assertEqual(len(w), 0) + finally: + self.module.defaultaction = original + + def test_showwarning_missing(self): + # Test that showwarning() missing is okay. + if self.module._use_context: + # If _use_context is true, the warnings module does not + # override/restore showwarning() + return + text = 'del showwarning test' + with self.module.catch_warnings(): + self.module.filterwarnings("always", category=UserWarning) + del self.module.showwarning + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + self.assertIn(text, result) + + def test_showwarnmsg_missing(self): + # Test that _showwarnmsg() missing is okay. + text = 'del _showwarnmsg test' + with self.module.catch_warnings(): + self.module.filterwarnings("always", category=UserWarning) + + show = self.module._showwarnmsg + try: + del self.module._showwarnmsg + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + finally: + self.module._showwarnmsg = show + self.assertIn(text, result) + + def test_showwarning_not_callable(self): + orig = self.module.showwarning + try: + with self.module.catch_warnings(): + self.module.filterwarnings("always", category=UserWarning) + self.module.showwarning = print + with support.captured_output('stdout'): + self.module.warn('Warning!') + self.module.showwarning = 23 + self.assertRaises(TypeError, self.module.warn, "Warning!") + finally: + self.module.showwarning = orig + + def test_show_warning_output(self): + # With showwarning() missing, make sure that output is okay. + orig = self.module.showwarning + try: + text = 'test show_warning' + with self.module.catch_warnings(): + self.module.filterwarnings("always", category=UserWarning) + del self.module.showwarning + with support.captured_output('stderr') as stream: + warning_tests.inner(text) + result = stream.getvalue() + self.assertEqual(result.count('\n'), 2, + "Too many newlines in %r" % result) + first_line, second_line = result.split('\n', 1) + expected_file = os.path.splitext(warning_tests.__file__)[0] + '.py' + first_line_parts = first_line.rsplit(':', 3) + path, line, warning_class, message = first_line_parts + line = int(line) + self.assertEqual(expected_file, path) + self.assertEqual(warning_class, ' ' + UserWarning.__name__) + self.assertEqual(message, ' ' + text) + expected_line = ' ' + linecache.getline(path, line).strip() + '\n' + assert expected_line + self.assertEqual(second_line, expected_line) + finally: + self.module.showwarning = orig + + def test_filename_none(self): + # issue #12467: race condition if a warning is emitted at shutdown + globals_dict = globals() + oldfile = globals_dict['__file__'] + try: + catch = self.module.catch_warnings(record=True) + with catch as w: + self.module.filterwarnings("always", category=UserWarning) + globals_dict['__file__'] = None + self.module.warn('test', UserWarning) + self.assertTrue(len(w)) + finally: + globals_dict['__file__'] = oldfile + + def test_stderr_none(self): + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stderr = None; " + "import warnings; warnings.simplefilter('always'); " + "warnings.warn('Warning!')") + self.assertEqual(stdout, b'') + self.assertNotIn(b'Warning!', stderr) + self.assertNotIn(b'Error', stderr) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'int' object is not iterable + def test_issue31285(self): + # warn_explicit() should neither raise a SystemError nor cause an + # assertion failure, in case the return value of get_source() has a + # bad splitlines() method. + get_source_called = [] + def get_module_globals(*, splitlines_ret_val): + class BadSource(str): + def splitlines(self): + return splitlines_ret_val + + class BadLoader: + def get_source(self, fullname): + get_source_called.append(splitlines_ret_val) + return BadSource('spam') + + loader = BadLoader() + spec = importlib.machinery.ModuleSpec('foobar', loader) + return {'__loader__': loader, + '__spec__': spec, + '__name__': 'foobar'} + + + wmod = self.module + with wmod.catch_warnings(): + wmod.filterwarnings('default', category=UserWarning) + + linecache.clearcache() + with support.captured_stderr() as stderr: + wmod.warn_explicit( + 'foo', UserWarning, 'bar', 1, + module_globals=get_module_globals(splitlines_ret_val=42)) + self.assertIn('UserWarning: foo', stderr.getvalue()) + self.assertEqual(get_source_called, [42]) + + linecache.clearcache() + with support.swap_attr(wmod, '_showwarnmsg', None): + del wmod._showwarnmsg + with support.captured_stderr() as stderr: + wmod.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=get_module_globals(splitlines_ret_val=[42])) + self.assertIn('UserWarning: eggs', stderr.getvalue()) + self.assertEqual(get_source_called, [42, [42]]) + linecache.clearcache() + + @support.cpython_only + def test_issue31411(self): + # warn_explicit() shouldn't raise a SystemError in case + # warnings.onceregistry isn't a dictionary. + wmod = self.module + with wmod.catch_warnings(): + wmod.filterwarnings('once') + with support.swap_attr(wmod, 'onceregistry', None): + with self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1, registry=None) + + @support.cpython_only + def test_issue31416(self): + # warn_explicit() shouldn't cause an assertion failure in case of a + # bad warnings.filters or warnings.defaultaction. + wmod = self.module + with wmod.catch_warnings(): + wmod._get_filters()[:] = [(None, None, Warning, None, 0)] + with self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1) + + wmod._get_filters()[:] = [] + with support.swap_attr(wmod, 'defaultaction', None), \ + self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1) + + @support.cpython_only + def test_issue31566(self): + # warn() shouldn't cause an assertion failure in case of a bad + # __name__ global. + with self.module.catch_warnings(): + self.module.filterwarnings('error', category=UserWarning) + with support.swap_item(globals(), '__name__', b'foo'), \ + support.swap_item(globals(), '__file__', None): + self.assertRaises(UserWarning, self.module.warn, 'bar') + + +class WarningsDisplayTests(BaseTest): + + """Test the displaying of warnings and the ability to overload functions + related to displaying warnings.""" + + def test_formatwarning(self): + message = "msg" + category = Warning + file_name = os.path.splitext(warning_tests.__file__)[0] + '.py' + line_num = 5 + file_line = linecache.getline(file_name, line_num).strip() + format = "%s:%s: %s: %s\n %s\n" + expect = format % (file_name, line_num, category.__name__, message, + file_line) + self.assertEqual(expect, self.module.formatwarning(message, + category, file_name, line_num)) + # Test the 'line' argument. + file_line += " for the win!" + expect = format % (file_name, line_num, category.__name__, message, + file_line) + self.assertEqual(expect, self.module.formatwarning(message, + category, file_name, line_num, file_line)) + + def test_showwarning(self): + file_name = os.path.splitext(warning_tests.__file__)[0] + '.py' + line_num = 3 + expected_file_line = linecache.getline(file_name, line_num).strip() + message = 'msg' + category = Warning + file_object = StringIO() + expect = self.module.formatwarning(message, category, file_name, + line_num) + self.module.showwarning(message, category, file_name, line_num, + file_object) + self.assertEqual(file_object.getvalue(), expect) + # Test 'line' argument. + expected_file_line += "for the win!" + expect = self.module.formatwarning(message, category, file_name, + line_num, expected_file_line) + file_object = StringIO() + self.module.showwarning(message, category, file_name, line_num, + file_object, expected_file_line) + self.assertEqual(expect, file_object.getvalue()) + + def test_formatwarning_override(self): + # bpo-35178: Test that a custom formatwarning function gets the 'line' + # argument as a positional argument, and not only as a keyword argument + def myformatwarning(message, category, filename, lineno, text): + return f'm={message}:c={category}:f={filename}:l={lineno}:t={text}' + + file_name = os.path.splitext(warning_tests.__file__)[0] + '.py' + line_num = 3 + file_line = linecache.getline(file_name, line_num).strip() + message = 'msg' + category = Warning + file_object = StringIO() + expected = f'm={message}:c={category}:f={file_name}:l={line_num}' + \ + f':t={file_line}' + with support.swap_attr(self.module, 'formatwarning', myformatwarning): + self.module.showwarning(message, category, file_name, line_num, + file_object, file_line) + self.assertEqual(file_object.getvalue(), expected) + + +class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): + module = c_warnings + +class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): + module = py_warnings + + @unittest.expectedFailure # TODO: RUSTPYTHON; + ResourceWarning: Enable tracemalloc to get the object allocation traceback + def test_tracemalloc(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + + with open(os_helper.TESTFN, 'w', encoding="utf-8") as fp: + fp.write(textwrap.dedent(""" + def func(): + f = open(__file__, "rb") + # Emit ResourceWarning + f = None + + func() + """)) + + def run(*args): + res = assert_python_ok(*args, PYTHONIOENCODING='utf-8') + stderr = res.err.decode('utf-8', 'replace') + stderr = '\n'.join(stderr.splitlines()) + + # normalize newlines + stderr = re.sub('<.*>', '<...>', stderr) + return stderr + + # tracemalloc disabled + filename = os.path.abspath(os_helper.TESTFN) + stderr = run('-Wd', os_helper.TESTFN) + expected = textwrap.dedent(f''' + {filename}:5: ResourceWarning: unclosed file <...> + f = None + ResourceWarning: Enable tracemalloc to get the object allocation traceback + ''').strip() + self.assertEqual(stderr, expected) + + # tracemalloc enabled + stderr = run('-Wd', '-X', 'tracemalloc=2', os_helper.TESTFN) + expected = textwrap.dedent(f''' + {filename}:5: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call last): + File "{filename}", lineno 7 + func() + File "{filename}", lineno 3 + f = open(__file__, "rb") + ''').strip() + self.assertEqual(stderr, expected) + + +class CatchWarningTests(BaseTest): + + """Test catch_warnings().""" + + def test_catch_warnings_restore(self): + if self.module._use_context: + return # test disabled if using context vars + wmod = self.module + orig_filters = wmod.filters + orig_showwarning = wmod.showwarning + # Ensure both showwarning and filters are restored when recording + with wmod.catch_warnings(record=True): + wmod.filters = wmod.showwarning = object() + self.assertIs(wmod.filters, orig_filters) + self.assertIs(wmod.showwarning, orig_showwarning) + # Same test, but with recording disabled + with wmod.catch_warnings(record=False): + wmod.filters = wmod.showwarning = object() + self.assertIs(wmod.filters, orig_filters) + self.assertIs(wmod.showwarning, orig_showwarning) + + def test_catch_warnings_recording(self): + wmod = self.module + # Ensure warnings are recorded when requested + with wmod.catch_warnings(record=True) as w: + self.assertEqual(w, []) + self.assertIs(type(w), list) + wmod.simplefilter("always") + wmod.warn("foo") + self.assertEqual(str(w[-1].message), "foo") + wmod.warn("bar") + self.assertEqual(str(w[-1].message), "bar") + self.assertEqual(str(w[0].message), "foo") + self.assertEqual(str(w[1].message), "bar") + del w[:] + self.assertEqual(w, []) + # Ensure warnings are not recorded when not requested + orig_showwarning = wmod.showwarning + with wmod.catch_warnings(record=False) as w: + self.assertIsNone(w) + self.assertIs(wmod.showwarning, orig_showwarning) + + def test_catch_warnings_reentry_guard(self): + wmod = self.module + # Ensure catch_warnings is protected against incorrect usage + x = wmod.catch_warnings(record=True) + self.assertRaises(RuntimeError, x.__exit__) + with x: + self.assertRaises(RuntimeError, x.__enter__) + # Same test, but with recording disabled + x = wmod.catch_warnings(record=False) + self.assertRaises(RuntimeError, x.__exit__) + with x: + self.assertRaises(RuntimeError, x.__enter__) + + def test_catch_warnings_defaults(self): + wmod = self.module + orig_filters = wmod._get_filters() + orig_showwarning = wmod.showwarning + # Ensure default behaviour is not to record warnings + with wmod.catch_warnings() as w: + self.assertIsNone(w) + self.assertIs(wmod.showwarning, orig_showwarning) + self.assertIsNot(wmod._get_filters(), orig_filters) + self.assertIs(wmod._get_filters(), orig_filters) + if wmod is sys.modules['warnings']: + # Ensure the default module is this one + with wmod.catch_warnings() as w: + self.assertIsNone(w) + self.assertIs(wmod.showwarning, orig_showwarning) + self.assertIsNot(wmod._get_filters(), orig_filters) + self.assertIs(wmod._get_filters(), orig_filters) + + def test_record_override_showwarning_before(self): + # Issue #28835: If warnings.showwarning() was overridden, make sure + # that catch_warnings(record=True) overrides it again. + if self.module._use_context: + # If _use_context is true, the warnings module does not restore + # showwarning() + return + text = "This is a warning" + wmod = self.module + my_log = [] + + def my_logger(message, category, filename, lineno, file=None, line=None): + nonlocal my_log + my_log.append(message) + + # Override warnings.showwarning() before calling catch_warnings() + with support.swap_attr(wmod, 'showwarning', my_logger): + with wmod.catch_warnings(record=True) as log: + self.assertIsNot(wmod.showwarning, my_logger) + + wmod.simplefilter("always") + wmod.warn(text) + + self.assertIs(wmod.showwarning, my_logger) + + self.assertEqual(len(log), 1, log) + self.assertEqual(log[0].message.args[0], text) + self.assertEqual(my_log, []) + + def test_record_override_showwarning_inside(self): + # Issue #28835: It is possible to override warnings.showwarning() + # in the catch_warnings(record=True) context manager. + if self.module._use_context: + # If _use_context is true, the warnings module does not restore + # showwarning() + return + text = "This is a warning" + wmod = self.module + my_log = [] + + def my_logger(message, category, filename, lineno, file=None, line=None): + nonlocal my_log + my_log.append(message) + + with wmod.catch_warnings(record=True) as log: + wmod.simplefilter("always") + wmod.showwarning = my_logger + wmod.warn(text) + + self.assertEqual(len(my_log), 1, my_log) + self.assertEqual(my_log[0].args[0], text) + self.assertEqual(log, []) + + def test_check_warnings(self): + # Explicit tests for the test.support convenience wrapper + wmod = self.module + if wmod is not sys.modules['warnings']: + self.skipTest('module to test is not loaded warnings module') + with warnings_helper.check_warnings(quiet=False) as w: + self.assertEqual(w.warnings, []) + wmod.simplefilter("always") + wmod.warn("foo") + self.assertEqual(str(w.message), "foo") + wmod.warn("bar") + self.assertEqual(str(w.message), "bar") + self.assertEqual(str(w.warnings[0].message), "foo") + self.assertEqual(str(w.warnings[1].message), "bar") + w.reset() + self.assertEqual(w.warnings, []) + + with warnings_helper.check_warnings(): + # defaults to quiet=True without argument + pass + with warnings_helper.check_warnings(('foo', UserWarning)): + wmod.warn("foo") + + with self.assertRaises(AssertionError): + with warnings_helper.check_warnings(('', RuntimeWarning)): + # defaults to quiet=False with argument + pass + with self.assertRaises(AssertionError): + with warnings_helper.check_warnings(('foo', RuntimeWarning)): + wmod.warn("foo") + +class CCatchWarningTests(CatchWarningTests, unittest.TestCase): + module = c_warnings + +class PyCatchWarningTests(CatchWarningTests, unittest.TestCase): + module = py_warnings + + +class EnvironmentVariableTests(BaseTest): + + def test_single_warning(self): + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning", + PYTHONDEVMODE="") + self.assertEqual(stdout, b"['ignore::DeprecationWarning']") + + def test_comma_separated_warnings(self): + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning,ignore::UnicodeWarning", + PYTHONDEVMODE="") + self.assertEqual(stdout, + b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + + @force_not_colorized + def test_envvar_and_command_line(self): + rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning", + PYTHONDEVMODE="") + self.assertEqual(stdout, + b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + + @force_not_colorized + def test_conflicting_envvar_and_command_line(self): + rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c", + "import sys, warnings; sys.stdout.write(str(sys.warnoptions)); " + "warnings.warn('Message', DeprecationWarning)", + PYTHONWARNINGS="default::DeprecationWarning", + PYTHONDEVMODE="") + self.assertEqual(stdout, + b"['default::DeprecationWarning', 'error::DeprecationWarning']") + self.assertEqual(stderr.splitlines(), + [b"Traceback (most recent call last):", + b" File \"<string>\", line 1, in <module>", + b' import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w' + b"arn('Message', DeprecationWarning)", + b' ~~~~~~~~~~' + b'~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', + b"DeprecationWarning: Message"]) + + def test_default_filter_configuration(self): + pure_python_api = self.module is py_warnings + if support.Py_DEBUG: + expected_default_filters = [] + else: + if pure_python_api: + main_module_filter = re.compile("__main__") + else: + main_module_filter = "__main__" + expected_default_filters = [ + ('default', None, DeprecationWarning, main_module_filter, 0), + ('ignore', None, DeprecationWarning, None, 0), + ('ignore', None, PendingDeprecationWarning, None, 0), + ('ignore', None, ImportWarning, None, 0), + ('ignore', None, ResourceWarning, None, 0), + ] + expected_output = [str(f).encode() for f in expected_default_filters] + + if pure_python_api: + # Disable the warnings acceleration module in the subprocess + code = "import sys; sys.modules.pop('warnings', None); sys.modules['_warnings'] = None; " + else: + code = "" + code += "import warnings; [print(f) for f in warnings._get_filters()]" + + rc, stdout, stderr = assert_python_ok("-c", code, __isolated=True) + stdout_lines = [line.strip() for line in stdout.splitlines()] + self.maxDiff = None + self.assertEqual(stdout_lines, expected_output) + + + @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii', + 'requires non-ascii filesystemencoding') + def test_nonascii(self): + PYTHONWARNINGS="ignore:DeprecationWarning" + os_helper.FS_NONASCII + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONIOENCODING="utf-8", + PYTHONWARNINGS=PYTHONWARNINGS, + PYTHONDEVMODE="") + self.assertEqual(stdout, str([PYTHONWARNINGS]).encode()) + +class CEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): + module = c_warnings + +class PyEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): + module = py_warnings + + +class LocksTest(unittest.TestCase): + @support.cpython_only + @unittest.skipUnless(c_warnings, 'C module is required') + def test_release_lock_no_lock(self): + with self.assertRaisesRegex( + RuntimeError, + 'cannot release un-acquired lock', + ): + c_warnings._release_lock() + + +class _DeprecatedTest(BaseTest, unittest.TestCase): + + """Test _deprecated().""" + + module = original_warnings + + def test_warning(self): + version = (3, 11, 0, "final", 0) + test = [(4, 12), (4, 11), (4, 0), (3, 12)] + for remove in test: + msg = rf".*test_warnings.*{remove[0]}\.{remove[1]}" + filter = msg, DeprecationWarning + with self.subTest(remove=remove): + with warnings_helper.check_warnings(filter, quiet=False): + self.module._deprecated("test_warnings", remove=remove, + _version=version) + + version = (3, 11, 0, "alpha", 0) + msg = r".*test_warnings.*3\.11" + with warnings_helper.check_warnings((msg, DeprecationWarning), quiet=False): + self.module._deprecated("test_warnings", remove=(3, 11), + _version=version) + + def test_RuntimeError(self): + version = (3, 11, 0, "final", 0) + test = [(2, 0), (2, 12), (3, 10)] + for remove in test: + with self.subTest(remove=remove): + with self.assertRaises(RuntimeError): + self.module._deprecated("test_warnings", remove=remove, + _version=version) + for level in ["beta", "candidate", "final"]: + version = (3, 11, 0, level, 0) + with self.subTest(releaselevel=level): + with self.assertRaises(RuntimeError): + self.module._deprecated("test_warnings", remove=(3, 11), + _version=version) + + +class BootstrapTest(unittest.TestCase): + + def test_issue_8766(self): + # "import encodings" emits a warning whereas the warnings is not loaded + # or not completely loaded (warnings imports indirectly encodings by + # importing linecache) yet + with os_helper.temp_cwd() as cwd, os_helper.temp_cwd('encodings'): + # encodings loaded by initfsencoding() + assert_python_ok('-c', 'pass', PYTHONPATH=cwd) + + # Use -W to load warnings module at startup + assert_python_ok('-c', 'pass', '-W', 'always', PYTHONPATH=cwd) + + +class FinalizationTest(unittest.TestCase): + def test_finalization(self): + # Issue #19421: warnings.warn() should not crash + # during Python finalization + code = """ +import warnings +warn = warnings.warn + +class A: + def __del__(self): + warn("test") + +a=A() + """ + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(err.decode().rstrip(), + '<string>:7: UserWarning: test') + + def test_late_resource_warning(self): + # Issue #21925: Emitting a ResourceWarning late during the Python + # shutdown must be logged. + + expected = b"<sys>:0: ResourceWarning: unclosed file " + + # don't import the warnings module + # (_warnings will try to import it) + code = "f = open(%a)" % __file__ + rc, out, err = assert_python_ok("-Wd", "-c", code) + self.assertStartsWith(err, expected) + + # import the warnings module + code = "import warnings; f = open(%a)" % __file__ + rc, out, err = assert_python_ok("-Wd", "-c", code) + self.assertStartsWith(err, expected) + + +class AsyncTests(BaseTest): + """Verifies that the catch_warnings() context manager behaves + as expected when used inside async co-routines. This requires + that the context_aware_warnings flag is enabled, so that + the context manager uses a context variable. + """ + + def setUp(self): + super().setUp() + self.module.resetwarnings() + + @unittest.skipIf(not sys.flags.context_aware_warnings, + "requires context aware warnings") + def test_async_context(self): + import asyncio + + # Events to force the execution interleaving we want. + step_a1 = asyncio.Event() + step_a2 = asyncio.Event() + step_b1 = asyncio.Event() + step_b2 = asyncio.Event() + + async def run_a(): + with self.module.catch_warnings(record=True) as w: + await step_a1.wait() + # The warning emitted here should be caught be the enclosing + # context manager. + self.module.warn('run_a warning', UserWarning) + step_b1.set() + await step_a2.wait() + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'run_a warning') + step_b2.set() + + async def run_b(): + with self.module.catch_warnings(record=True) as w: + step_a1.set() + await step_b1.wait() + # The warning emitted here should be caught be the enclosing + # context manager. + self.module.warn('run_b warning', UserWarning) + step_a2.set() + await step_b2.wait() + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'run_b warning') + + async def run_tasks(): + await asyncio.gather(run_a(), run_b()) + + asyncio.run(run_tasks()) + + @unittest.skipIf(not sys.flags.context_aware_warnings, + "requires context aware warnings") + def test_async_task_inherit(self): + """Check that a new asyncio task inherits warnings context from the + coroutine that spawns it. + """ + import asyncio + + step1 = asyncio.Event() + step2 = asyncio.Event() + + async def run_child1(): + await step1.wait() + # This should be recorded by the run_parent() catch_warnings + # context. + self.module.warn('child warning', UserWarning) + step2.set() + + async def run_child2(): + # This establishes a new catch_warnings() context. The + # run_child1() task should still be using the context from + # run_parent() if context-aware warnings are enabled. + with self.module.catch_warnings(record=True) as w: + step1.set() + await step2.wait() + + async def run_parent(): + with self.module.catch_warnings(record=True) as w: + await asyncio.gather(run_child1(), run_child2()) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'child warning') + + asyncio.run(run_parent()) + + +class CAsyncTests(AsyncTests, unittest.TestCase): + module = c_warnings + + +class PyAsyncTests(AsyncTests, unittest.TestCase): + module = py_warnings + + +class ThreadTests(BaseTest): + """Verifies that the catch_warnings() context manager behaves as + expected when used within threads. This requires that both the + context_aware_warnings flag and thread_inherit_context flags are enabled. + """ + + ENABLE_THREAD_TESTS = (sys.flags.context_aware_warnings and + sys.flags.thread_inherit_context) + + def setUp(self): + super().setUp() + self.module.resetwarnings() + + @unittest.skipIf(not ENABLE_THREAD_TESTS, + "requires thread-safe warnings flags") + def test_threaded_context(self): + import threading + + barrier = threading.Barrier(2, timeout=2) + + def run_a(): + with self.module.catch_warnings(record=True) as w: + barrier.wait() + # The warning emitted here should be caught be the enclosing + # context manager. + self.module.warn('run_a warning', UserWarning) + barrier.wait() + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'run_a warning') + # Should be caught be the catch_warnings() context manager of run_threads() + self.module.warn('main warning', UserWarning) + + def run_b(): + with self.module.catch_warnings(record=True) as w: + barrier.wait() + # The warning emitted here should be caught be the enclosing + # context manager. + barrier.wait() + self.module.warn('run_b warning', UserWarning) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].message.args[0], 'run_b warning') + # Should be caught be the catch_warnings() context manager of run_threads() + self.module.warn('main warning', UserWarning) + + def run_threads(): + threads = [ + threading.Thread(target=run_a), + threading.Thread(target=run_b), + ] + with self.module.catch_warnings(record=True) as w: + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertEqual(len(w), 2) + self.assertEqual(w[0].message.args[0], 'main warning') + self.assertEqual(w[1].message.args[0], 'main warning') + + run_threads() + + +class CThreadTests(ThreadTests, unittest.TestCase): + module = c_warnings + + +class PyThreadTests(ThreadTests, unittest.TestCase): + module = py_warnings + + +class DeprecatedTests(PyPublicAPITests): + def test_dunder_deprecated(self): + @deprecated("A will go away soon") + class A: + pass + + self.assertEqual(A.__deprecated__, "A will go away soon") + self.assertIsInstance(A, type) + + @deprecated("b will go away soon") + def b(): + pass + + self.assertEqual(b.__deprecated__, "b will go away soon") + self.assertIsInstance(b, types.FunctionType) + + @overload + @deprecated("no more ints") + def h(x: int) -> int: ... + @overload + def h(x: str) -> str: ... + def h(x): + return x + + overloads = get_overloads(h) + self.assertEqual(len(overloads), 2) + self.assertEqual(overloads[0].__deprecated__, "no more ints") + + def test_class(self): + @deprecated("A will go away soon") + class A: + pass + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + A() + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + with self.assertRaises(TypeError): + A(42) + + def test_class_with_init(self): + @deprecated("HasInit will go away soon") + class HasInit: + def __init__(self, x): + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasInit will go away soon"): + instance = HasInit(42) + self.assertEqual(instance.x, 42) + + def test_class_with_new(self): + has_new_called = False + + @deprecated("HasNew will go away soon") + class HasNew: + def __new__(cls, x): + nonlocal has_new_called + has_new_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasNew will go away soon"): + instance = HasNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(has_new_called) + + def test_class_with_inherited_new(self): + new_base_called = False + + class NewBase: + def __new__(cls, x): + nonlocal new_base_called + new_base_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + @deprecated("HasInheritedNew will go away soon") + class HasInheritedNew(NewBase): + pass + + with self.assertWarnsRegex(DeprecationWarning, "HasInheritedNew will go away soon"): + instance = HasInheritedNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_base_called) + + def test_class_with_new_but_no_init(self): + new_called = False + + @deprecated("HasNewNoInit will go away soon") + class HasNewNoInit: + def __new__(cls, x): + nonlocal new_called + new_called = True + obj = super().__new__(cls) + obj.x = x + return obj + + with self.assertWarnsRegex(DeprecationWarning, "HasNewNoInit will go away soon"): + instance = HasNewNoInit(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_called) + + def test_mixin_class(self): + @deprecated("Mixin will go away soon") + class Mixin: + pass + + class Base: + def __init__(self, a) -> None: + self.a = a + + with self.assertWarnsRegex(DeprecationWarning, "Mixin will go away soon"): + class Child(Base, Mixin): + pass + + instance = Child(42) + self.assertEqual(instance.a, 42) + + def test_do_not_shadow_user_arguments(self): + new_called = False + new_called_cls = None + + @deprecated("MyMeta will go away soon") + class MyMeta(type): + def __new__(mcs, name, bases, attrs, cls=None): + nonlocal new_called, new_called_cls + new_called = True + new_called_cls = cls + return super().__new__(mcs, name, bases, attrs) + + with self.assertWarnsRegex(DeprecationWarning, "MyMeta will go away soon"): + class Foo(metaclass=MyMeta, cls='haha'): + pass + + self.assertTrue(new_called) + self.assertEqual(new_called_cls, 'haha') + + def test_existing_init_subclass(self): + @deprecated("C will go away soon") + class C: + def __init_subclass__(cls) -> None: + cls.inited = True + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C): + pass + + self.assertTrue(D.inited) + self.assertIsInstance(D(), D) # no deprecation + + def test_existing_init_subclass_in_base(self): + class Base: + def __init_subclass__(cls, x) -> None: + cls.inited = x + + @deprecated("C will go away soon") + class C(Base, x=42): + pass + + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C, x=3): + pass + + self.assertEqual(D.inited, 3) + + def test_existing_init_subclass_in_sibling_base(self): + @deprecated("A will go away soon") + class A: + pass + class B: + def __init_subclass__(cls, x): + super().__init_subclass__() + cls.inited = x + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + class C(A, B, x=42): + pass + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + class D(B, A, x=42): + pass + self.assertEqual(D.inited, 42) + + def test_init_subclass_has_correct_cls(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_init_subclass_with_explicit_classmethod(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + @classmethod + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_function(self): + @deprecated("b will go away soon") + def b(): + pass + + with self.assertWarnsRegex(DeprecationWarning, "b will go away soon"): + b() + + def test_method(self): + class Capybara: + @deprecated("x will go away soon") + def x(self): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x() + + def test_property(self): + class Capybara: + @property + @deprecated("x will go away soon") + def x(self): + pass + + @property + def no_more_setting(self): + return 42 + + @no_more_setting.setter + @deprecated("no more setting") + def no_more_setting(self, value): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + self.assertEqual(instance.no_more_setting, 42) + + with self.assertWarnsRegex(DeprecationWarning, "no more setting"): + instance.no_more_setting = 42 + + def test_category(self): + @deprecated("c will go away soon", category=RuntimeWarning) + def c(): + pass + + with self.assertWarnsRegex(RuntimeWarning, "c will go away soon"): + c() + + def test_turn_off_warnings(self): + @deprecated("d will go away soon", category=None) + def d(): + pass + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + d() + + def test_only_strings_allowed(self): + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'type'" + ): + @deprecated + class Foo: ... + + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'function'" + ): + @deprecated + def foo(): ... + + def test_no_retained_references_to_wrapper_instance(self): + @deprecated('depr') + def d(): pass + + self.assertFalse(any( + isinstance(cell.cell_contents, deprecated) for cell in d.__closure__ + )) + + def test_inspect(self): + @deprecated("depr") + def sync(): + pass + + @deprecated("depr") + async def coro(): + pass + + class Cls: + @deprecated("depr") + def sync(self): + pass + + @deprecated("depr") + async def coro(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(sync)) + self.assertTrue(inspect.iscoroutinefunction(coro)) + self.assertFalse(inspect.iscoroutinefunction(Cls.sync)) + self.assertTrue(inspect.iscoroutinefunction(Cls.coro)) + + def test_inspect_class_signature(self): + class Cls1: # no __init__ or __new__ + pass + + class Cls2: # __new__ only + def __new__(cls, x, y): + return super().__new__(cls) + + class Cls3: # __init__ only + def __init__(self, x, y): + pass + + class Cls4: # __new__ and __init__ + def __new__(cls, x, y): + return super().__new__(cls) + + def __init__(self, x, y): + pass + + class Cls5(Cls1): # inherits no __init__ or __new__ + pass + + class Cls6(Cls2): # inherits __new__ only + pass + + class Cls7(Cls3): # inherits __init__ only + pass + + class Cls8(Cls4): # inherits __new__ and __init__ + pass + + # The `@deprecated` decorator will update the class in-place. + # Test the child classes first. + for cls in reversed((Cls1, Cls2, Cls3, Cls4, Cls5, Cls6, Cls7, Cls8)): + with self.subTest(f'class {cls.__name__} signature'): + try: + original_signature = inspect.signature(cls) + except ValueError: + original_signature = None + try: + original_new_signature = inspect.signature(cls.__new__) + except ValueError: + original_new_signature = None + + deprecated_cls = deprecated("depr")(cls) + + try: + deprecated_signature = inspect.signature(deprecated_cls) + except ValueError: + deprecated_signature = None + self.assertEqual(original_signature, deprecated_signature) + + try: + deprecated_new_signature = inspect.signature(deprecated_cls.__new__) + except ValueError: + deprecated_new_signature = None + self.assertEqual(original_new_signature, deprecated_new_signature) + + +def setUpModule(): + py_warnings.onceregistry.clear() + c_warnings.onceregistry.clear() + + +tearDownModule = setUpModule + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_warnings/__main__.py b/Lib/test/test_warnings/__main__.py new file mode 100644 index 00000000000..44e52ec0704 --- /dev/null +++ b/Lib/test/test_warnings/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_warnings') diff --git a/Lib/test/test_warnings/data/import_warning.py b/Lib/test/test_warnings/data/import_warning.py new file mode 100644 index 00000000000..32daec11404 --- /dev/null +++ b/Lib/test/test_warnings/data/import_warning.py @@ -0,0 +1,3 @@ +import warnings + +warnings.warn('module-level warning', DeprecationWarning, stacklevel=2) diff --git a/Lib/test/test_warnings/data/package_helper.py b/Lib/test/test_warnings/data/package_helper.py new file mode 100644 index 00000000000..c22a4f6405c --- /dev/null +++ b/Lib/test/test_warnings/data/package_helper.py @@ -0,0 +1,10 @@ +# helper to the helper for testing skip_file_prefixes. + +import os + +package_path = os.path.dirname(__file__) + +def inner_api(message, *, stacklevel, warnings_module): + warnings_module.warn( + message, stacklevel=stacklevel, + skip_file_prefixes=(package_path,)) diff --git a/Lib/test/test_warnings/data/stacklevel.py b/Lib/test/test_warnings/data/stacklevel.py new file mode 100644 index 00000000000..fe36242d3d2 --- /dev/null +++ b/Lib/test/test_warnings/data/stacklevel.py @@ -0,0 +1,17 @@ +# Helper module for testing stacklevel and skip_file_prefixes arguments +# of warnings.warn() + +import warnings +from test.test_warnings.data import package_helper + + +def outer(message, stacklevel=1, skip_file_prefixes=()): + inner(message, stacklevel, skip_file_prefixes) + +def inner(message, stacklevel=1, skip_file_prefixes=()): + warnings.warn(message, stacklevel=stacklevel, + skip_file_prefixes=skip_file_prefixes) + +def package(message, *, stacklevel): + package_helper.inner_api(message, stacklevel=stacklevel, + warnings_module=warnings) diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py new file mode 100644 index 00000000000..346a343761a --- /dev/null +++ b/Lib/test/test_wave.py @@ -0,0 +1,236 @@ +import unittest +from test import audiotests +from test import support +import io +import os +import struct +import sys +import wave + + +class WaveTest(audiotests.AudioWriteTests, + audiotests.AudioTestsWithSourceFile): + module = wave + + +class WavePCM8Test(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm8.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 1 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 827F CB80 B184 0088 4B86 C883 3F81 837E 387A 3473 A96B 9A66 \ + 6D64 4662 8E60 6F60 D762 7B68 936F 5877 177B 757C 887B 5F7B \ + 917A BE7B 3C7C E67F 4F84 C389 418E D192 6E97 0296 FF94 0092 \ + C98E D28D 6F8F 4E8F 648C E38A 888A AB8B D18E 0B91 368E C48A \ + """) + + +class WavePCM16Test(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm16.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 2 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 022EFFEA 4B5C00F9 311404EF 80DC0843 CBDF06B2 48AA03F3 BFE701B2 036BFE7C \ + B857FA3E B4B2F34F 2999EBCA 1A5FE6D7 EDFCE491 C626E279 0E05E0B8 EF27E02D \ + 5754E275 FB31E843 1373EF89 D827F72C 978BFB7A F5F7FC11 0866FB9C DF30FB42 \ + 117FFA36 3EE4FB5D BC75FCB6 66D5FF5F CF16040E 43220978 C1BC0EC8 511F12A4 \ + EEDF1755 82061666 7FFF1446 80001296 499C0EB2 52BA0DB9 EFB70F5C CE400FBC \ + E4B50CEB 63440A5A 08CA0A1F 2BBA0B0B 51460E47 8BCB113C B6F50EEA 44150A59 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 2) + + +class WavePCM24Test(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm24.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 3 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 022D65FFEB9D 4B5A0F00FA54 3113C304EE2B 80DCD6084303 \ + CBDEC006B261 48A99803F2F8 BFE82401B07D 036BFBFE7B5D \ + B85756FA3EC9 B4B055F3502B 299830EBCB62 1A5CA7E6D99A \ + EDFA3EE491BD C625EBE27884 0E05A9E0B6CF EF2929E02922 \ + 5758D8E27067 FB3557E83E16 1377BFEF8402 D82C5BF7272A \ + 978F16FB7745 F5F865FC1013 086635FB9C4E DF30FCFB40EE \ + 117FE0FA3438 3EE6B8FB5AC3 BC77A3FCB2F4 66D6DAFF5F32 \ + CF13B9041275 431D69097A8C C1BB600EC74E 5120B912A2BA \ + EEDF641754C0 8207001664B7 7FFFFF14453F 8000001294E6 \ + 499C1B0EB3B2 52B73E0DBCA0 EFB2B20F5FD8 CE3CDB0FBE12 \ + E4B49C0CEA2D 6344A80A5A7C 08C8FE0A1FFE 2BB9860B0A0E \ + 51486F0E44E1 8BCC64113B05 B6F4EC0EEB36 4413170A5B48 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 3) + + +class WavePCM24ExtTest(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm24-ext.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 3 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 022D65FFEB9D 4B5A0F00FA54 3113C304EE2B 80DCD6084303 \ + CBDEC006B261 48A99803F2F8 BFE82401B07D 036BFBFE7B5D \ + B85756FA3EC9 B4B055F3502B 299830EBCB62 1A5CA7E6D99A \ + EDFA3EE491BD C625EBE27884 0E05A9E0B6CF EF2929E02922 \ + 5758D8E27067 FB3557E83E16 1377BFEF8402 D82C5BF7272A \ + 978F16FB7745 F5F865FC1013 086635FB9C4E DF30FCFB40EE \ + 117FE0FA3438 3EE6B8FB5AC3 BC77A3FCB2F4 66D6DAFF5F32 \ + CF13B9041275 431D69097A8C C1BB600EC74E 5120B912A2BA \ + EEDF641754C0 8207001664B7 7FFFFF14453F 8000001294E6 \ + 499C1B0EB3B2 52B73E0DBCA0 EFB2B20F5FD8 CE3CDB0FBE12 \ + E4B49C0CEA2D 6344A80A5A7C 08C8FE0A1FFE 2BB9860B0A0E \ + 51486F0E44E1 8BCC64113B05 B6F4EC0EEB36 4413170A5B48 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 3) + + +class WavePCM32Test(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm32.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 4 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 022D65BCFFEB9D92 4B5A0F8000FA549C 3113C34004EE2BC0 80DCD680084303E0 \ + CBDEC0C006B26140 48A9980003F2F8FC BFE8248001B07D92 036BFB60FE7B5D34 \ + B8575600FA3EC920 B4B05500F3502BC0 29983000EBCB6240 1A5CA7A0E6D99A60 \ + EDFA3E80E491BD40 C625EB80E27884A0 0E05A9A0E0B6CFE0 EF292940E0292280 \ + 5758D800E2706700 FB3557D8E83E1640 1377BF00EF840280 D82C5B80F7272A80 \ + 978F1600FB774560 F5F86510FC101364 086635A0FB9C4E20 DF30FC40FB40EE28 \ + 117FE0A0FA3438B0 3EE6B840FB5AC3F0 BC77A380FCB2F454 66D6DA80FF5F32B4 \ + CF13B980041275B0 431D6980097A8C00 C1BB60000EC74E00 5120B98012A2BAA0 \ + EEDF64C01754C060 820700001664B780 7FFFFFFF14453F40 800000001294E6E0 \ + 499C1B000EB3B270 52B73E000DBCA020 EFB2B2E00F5FD880 CE3CDB400FBE1270 \ + E4B49CC00CEA2D90 6344A8800A5A7CA0 08C8FE800A1FFEE0 2BB986C00B0A0E00 \ + 51486F800E44E190 8BCC6480113B0580 B6F4EC000EEB3630 441317800A5B48A0 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 4) + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + not_exported = {'WAVE_FORMAT_PCM', 'WAVE_FORMAT_EXTENSIBLE', 'KSDATAFORMAT_SUBTYPE_PCM'} + support.check__all__(self, wave, not_exported=not_exported) + + def test_read_deprecations(self): + filename = support.findfile('pluck-pcm8.wav', subdir='audiodata') + with wave.open(filename) as reader: + with self.assertWarns(DeprecationWarning): + with self.assertRaises(wave.Error): + reader.getmark('mark') + with self.assertWarns(DeprecationWarning): + self.assertIsNone(reader.getmarkers()) + + def test_write_deprecations(self): + with io.BytesIO(b'') as tmpfile: + with wave.open(tmpfile, 'wb') as writer: + writer.setnchannels(1) + writer.setsampwidth(1) + writer.setframerate(1) + writer.setcomptype('NONE', 'not compressed') + + with self.assertWarns(DeprecationWarning): + with self.assertRaises(wave.Error): + writer.setmark(0, 0, 'mark') + with self.assertWarns(DeprecationWarning): + with self.assertRaises(wave.Error): + writer.getmark('mark') + with self.assertWarns(DeprecationWarning): + self.assertIsNone(writer.getmarkers()) + + +class WaveLowLevelTest(unittest.TestCase): + + def test_read_no_chunks(self): + b = b'SPAM' + with self.assertRaises(EOFError): + wave.open(io.BytesIO(b)) + + def test_read_no_riff_chunk(self): + b = b'SPAM' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, + 'file does not start with RIFF id'): + wave.open(io.BytesIO(b)) + + def test_read_not_wave(self): + b = b'RIFF' + struct.pack('<L', 4) + b'SPAM' + with self.assertRaisesRegex(wave.Error, + 'not a WAVE file'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 4) + b'WAVE' + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_data_chunk(self): + b = b'RIFF' + struct.pack('<L', 28) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 8) + with self.assertRaisesRegex(wave.Error, + 'fmt chunk and/or data chunk missing'): + wave.open(io.BytesIO(b)) + + def test_read_no_fmt_chunk(self): + b = b'RIFF' + struct.pack('<L', 12) + b'WAVE' + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'data chunk before fmt chunk'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_form(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 2, 1, 11025, 11025, 1, 1) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'unknown format: 2'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_number_of_channels(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 0, 11025, 11025, 1, 8) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad # of channels'): + wave.open(io.BytesIO(b)) + + def test_read_wrong_sample_width(self): + b = b'RIFF' + struct.pack('<L', 36) + b'WAVE' + b += b'fmt ' + struct.pack('<LHHLLHH', 16, 1, 1, 11025, 11025, 1, 0) + b += b'data' + struct.pack('<L', 0) + with self.assertRaisesRegex(wave.Error, 'bad sample width'): + wave.open(io.BytesIO(b)) + + def test_open_in_write_raises(self): + # gh-136523: Wave_write.__del__ should not throw + with support.catch_unraisable_exception() as cm: + with self.assertRaises(OSError): + wave.open(os.curdir, "wb") + support.gc_collect() + self.assertIsNone(cm.unraisable) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 751002bfc9b..cd0bdacaaf0 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1,5 +1,6 @@ import gc import sys +import doctest import unittest import collections import weakref @@ -9,10 +10,14 @@ import threading import time import random +import textwrap from test import support from test.support import script_helper, ALWAYS_EQ from test.support import gc_collect +from test.support import import_helper +from test.support import threading_helper +from test.support import is_wasi, Py_DEBUG # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -77,7 +82,7 @@ def callback(self, ref): @contextlib.contextmanager -def collect_in_thread(period=0.0001): +def collect_in_thread(period=0.005): """ Ensure GC collections happen in a different thread, at a high frequency. """ @@ -114,6 +119,49 @@ def test_basic_ref(self): del o repr(wr) + @support.cpython_only + def test_ref_repr(self): + obj = C() + ref = weakref.ref(obj) + regex = ( + rf"<weakref at 0x[0-9a-fA-F]+; " + rf"to '{'' if __name__ == '__main__' else C.__module__ + '.'}{C.__qualname__}' " + rf"at 0x[0-9a-fA-F]+>" + ) + self.assertRegex(repr(ref), regex) + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'<weakref at 0x[0-9a-fA-F]+; dead>') + + # test type with __name__ + class WithName: + @property + def __name__(self): + return "custom_name" + + obj2 = WithName() + ref2 = weakref.ref(obj2) + regex = ( + rf"<weakref at 0x[0-9a-fA-F]+; " + rf"to '{'' if __name__ == '__main__' else WithName.__module__ + '.'}" + rf"{WithName.__qualname__}' " + rf"at 0x[0-9a-fA-F]+ +\(custom_name\)>" + ) + self.assertRegex(repr(ref2), regex) + + def test_repr_failure_gh99184(self): + class MyConfig(dict): + def __getattr__(self, x): + return self[x] + + obj = MyConfig(offset=5) + obj_weakref = weakref.ref(obj) + + self.assertIn('MyConfig', repr(obj_weakref)) + self.assertIn('MyConfig', str(obj_weakref)) + def test_basic_callback(self): self.check_basic_callback(C) self.check_basic_callback(create_function) @@ -121,7 +169,7 @@ def test_basic_callback(self): @support.cpython_only def test_cfunction(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") create_cfunction = _testcapi.create_cfunction f = create_cfunction() wr = weakref.ref(f) @@ -182,6 +230,22 @@ def check(proxy): self.assertRaises(ReferenceError, bool, ref3) self.assertEqual(self.cbcalled, 2) + @support.cpython_only + def test_proxy_repr(self): + obj = C() + ref = weakref.proxy(obj, self.callback) + regex = ( + rf"<weakproxy at 0x[0-9a-fA-F]+; " + rf"to '{'' if __name__ == '__main__' else C.__module__ + '.'}{C.__qualname__}' " + rf"at 0x[0-9a-fA-F]+>" + ) + self.assertRegex(repr(ref), regex) + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'<weakproxy at 0x[0-9a-fA-F]+; dead>') + def check_basic_ref(self, factory): o = factory() ref = weakref.ref(o) @@ -225,8 +289,7 @@ def test_ref_reuse(self): self.assertEqual(weakref.getweakrefcount(o), 1, "wrong weak ref count for object after deleting proxy") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_reuse(self): o = C() proxy1 = weakref.proxy(o) @@ -274,8 +337,6 @@ def __bytes__(self): self.assertIn("__bytes__", dir(weakref.proxy(instance))) self.assertEqual(bytes(weakref.proxy(instance)), b"bytes") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_index(self): class C: def __index__(self): @@ -284,8 +345,6 @@ def __index__(self): p = weakref.proxy(o) self.assertEqual(operator.index(p), 10) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_div(self): class C: def __floordiv__(self, other): @@ -298,8 +357,6 @@ def __ifloordiv__(self, other): p //= 5 self.assertEqual(p, 21) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_matmul(self): class C: def __matmul__(self, other): @@ -323,13 +380,11 @@ def __imatmul__(self, other): # was not honored, and was broken in different ways for # PyWeakref_NewRef() and PyWeakref_NewProxy(). (Two tests.) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_shared_ref_without_callback(self): self.check_shared_without_callback(weakref.ref) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_shared_proxy_without_callback(self): self.check_shared_without_callback(weakref.proxy) @@ -351,8 +406,7 @@ def check_shared_without_callback(self, makeref): p2 = makeref(o) self.assertIs(p1, p2, "callbacks were None, NULL in the C API") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callable_proxy(self): o = Callable() ref1 = weakref.proxy(o) @@ -382,7 +436,7 @@ def check_proxy(self, o, proxy): self.assertEqual(proxy.foo, 2, "proxy does not reflect attribute modification") del o.foo - self.assertFalse(hasattr(proxy, 'foo'), + self.assertNotHasAttr(proxy, 'foo', "proxy does not reflect attribute removal") proxy.foo = 1 @@ -392,7 +446,7 @@ def check_proxy(self, o, proxy): self.assertEqual(o.foo, 2, "object does not reflect attribute modification via proxy") del proxy.foo - self.assertFalse(hasattr(o, 'foo'), + self.assertNotHasAttr(o, 'foo', "object does not reflect attribute removal via proxy") def test_proxy_deletion(self): @@ -447,8 +501,6 @@ def __iter__(self): # Calls proxy.__next__ self.assertEqual(list(weak_it), [4, 5, 6]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_bad_next(self): # bpo-44720: PyIter_Next() shouldn't be called if the reference # isn't an iterator. @@ -538,8 +590,6 @@ def test_getweakrefs(self): self.assertEqual(weakref.getweakrefs(1), [], "list of refs does not match for int") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_newstyle_number_ops(self): class F(float): pass @@ -613,7 +663,7 @@ class C(object): # deallocation of c2. del c2 - def test_callback_in_cycle_1(self): + def test_callback_in_cycle(self): import gc class J(object): @@ -653,40 +703,11 @@ def acallback(self, ignore): del I, J, II gc.collect() - def test_callback_in_cycle_2(self): + def test_callback_reachable_one_way(self): import gc - # This is just like test_callback_in_cycle_1, except that II is an - # old-style class. The symptom is different then: an instance of an - # old-style class looks in its own __dict__ first. 'J' happens to - # get cleared from I.__dict__ before 'wr', and 'J' was never in II's - # __dict__, so the attribute isn't found. The difference is that - # the old-style II doesn't have a NULL __mro__ (it doesn't have any - # __mro__), so no segfault occurs. Instead it got: - # test_callback_in_cycle_2 (__main__.ReferencesTestCase) ... - # Exception exceptions.AttributeError: - # "II instance has no attribute 'J'" in <bound method II.acallback - # of <?.II instance at 0x00B9B4B8>> ignored - - class J(object): - pass - - class II: - def acallback(self, ignore): - self.J - - I = II() - I.J = J - I.wr = weakref.ref(J, I.acallback) - - del I, J, II - gc.collect() - - def test_callback_in_cycle_3(self): - import gc - - # This one broke the first patch that fixed the last two. In this - # case, the objects reachable from the callback aren't also reachable + # This one broke the first patch that fixed the previous test. In this case, + # the objects reachable from the callback aren't also reachable # from the object (c1) *triggering* the callback: you can get to # c1 from c2, but not vice-versa. The result was that c2's __dict__ # got tp_clear'ed by the time the c2.cb callback got invoked. @@ -706,10 +727,10 @@ def cb(self, ignore): del c1, c2 gc.collect() - def test_callback_in_cycle_4(self): + def test_callback_different_classes(self): import gc - # Like test_callback_in_cycle_3, except c2 and c1 have different + # Like test_callback_reachable_one_way, except c2 and c1 have different # classes. c2's class (C) isn't reachable from c1 then, so protecting # objects reachable from the dying object (c1) isn't enough to stop # c2's class (C) from getting tp_clear'ed before c2.cb is invoked. @@ -734,8 +755,7 @@ class D: del c1, c2, C, D gc.collect() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_in_cycle_resurrection(self): import gc @@ -782,8 +802,7 @@ def C_went_away(ignore): gc.collect() self.assertEqual(alist, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callbacks_on_callback(self): import gc @@ -822,13 +841,9 @@ def cb(self, ignore): gc.collect() self.assertEqual(alist, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gc_during_ref_creation(self): self.check_gc_during_creation(weakref.ref) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gc_during_proxy_creation(self): self.check_gc_during_creation(weakref.proxy) @@ -869,8 +884,6 @@ def __del__(self): w = Target() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_init(self): # Issue 3634 # <weakref to class>.__init__() doesn't check errors correctly @@ -958,6 +971,7 @@ def test_hashing(self): self.assertEqual(hash(a), hash(42)) self.assertRaises(TypeError, hash, b) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_trashcan_16602(self): # Issue #16602: when a weakref's target was part of a long # deallocation chain, the trashcan mechanism could delay clearing @@ -979,8 +993,7 @@ def cb(wparent): del root gc.collect() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_attribute(self): x = Object(1) callback = lambda ref: None @@ -990,8 +1003,7 @@ def test_callback_attribute(self): ref2 = weakref.ref(x) self.assertIsNone(ref2.__callback__) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_attribute_after_deletion(self): x = Object(1) ref = weakref.ref(x, self.callback) @@ -1015,11 +1027,34 @@ def __del__(self): pass del x support.gc_collect() + @support.cpython_only + def test_no_memory_when_clearing(self): + # gh-118331: Make sure we do not raise an exception from the destructor + # when clearing weakrefs if allocating the intermediate tuple fails. + code = textwrap.dedent(""" + import _testcapi + import weakref + + class TestObj: + pass + + def callback(obj): + pass + + obj = TestObj() + # The choice of 50 is arbitrary, but must be large enough to ensure + # the allocation won't be serviced by the free list. + wrs = [weakref.ref(obj, callback) for _ in range(50)] + _testcapi.set_nomemory(0) + del obj + """).strip() + res, _ = script_helper.run_python_until_end("-c", code) + stderr = res.err.decode("ascii", "backslashreplace") + self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + class SubclassableWeakrefTestCase(TestBase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_refs(self): class MyRef(weakref.ref): def __init__(self, ob, callback=None, value=42): @@ -1038,8 +1073,6 @@ def __call__(self): self.assertIsNone(mr()) self.assertTrue(mr.called) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_refs_dont_replace_standard_refs(self): class MyRef(weakref.ref): pass @@ -1068,8 +1101,6 @@ class MyRef(weakref.ref): self.assertIn(r1, refs) self.assertIn(r2, refs) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_refs_with_slots(self): class MyRef(weakref.ref): __slots__ = "slot1", "slot2" @@ -1085,7 +1116,7 @@ def meth(self): self.assertEqual(r.slot1, "abc") self.assertEqual(r.slot2, "def") self.assertEqual(r.meth(), "abcdef") - self.assertFalse(hasattr(r, "__dict__")) + self.assertNotHasAttr(r, "__dict__") def test_subclass_refs_with_cycle(self): """Confirm https://bugs.python.org/issue3100 is fixed.""" @@ -1267,6 +1298,12 @@ class MappingTestCase(TestBase): COUNT = 10 + if support.check_sanitizer(thread=True) and support.Py_GIL_DISABLED: + # Reduce iteration count to get acceptable latency + NUM_THREADED_ITERATIONS = 1000 + else: + NUM_THREADED_ITERATIONS = 100000 + def check_len_cycles(self, dict_type, cons): N = 20 items = [RefCycle() for i in range(N)] @@ -1287,13 +1324,9 @@ def check_len_cycles(self, dict_type, cons): self.assertIn(n1, (0, 1)) self.assertEqual(n2, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keyed_len_cycles(self): self.check_len_cycles(weakref.WeakKeyDictionary, lambda k: (k, 1)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_valued_len_cycles(self): self.check_len_cycles(weakref.WeakValueDictionary, lambda k: (1, k)) @@ -1321,13 +1354,9 @@ def check_len_race(self, dict_type, cons): self.assertGreaterEqual(n2, 0) self.assertLessEqual(n2, n1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keyed_len_race(self): self.check_len_race(weakref.WeakKeyDictionary, lambda k: (k, 1)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_valued_len_race(self): self.check_len_race(weakref.WeakValueDictionary, lambda k: (1, k)) @@ -1828,8 +1857,6 @@ def test_weak_valued_delitem(self): self.assertEqual(len(d), 1) self.assertEqual(list(d.items()), [('something else', o2)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_weak_keyed_bad_delitem(self): d = weakref.WeakKeyDictionary() o = Object('1') @@ -1898,34 +1925,58 @@ def test_make_weak_keyed_dict_repr(self): dict = weakref.WeakKeyDictionary() self.assertRegex(repr(dict), '<WeakKeyDictionary at 0x.*>') + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") + @threading_helper.requires_working_threading() def test_threaded_weak_valued_setdefault(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): x = d.setdefault(10, RefCycle()) self.assertIsNot(x, None) # we never put None in there! del x + @threading_helper.requires_working_threading() def test_threaded_weak_valued_pop(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): d[10] = RefCycle() x = d.pop(10, 10) self.assertIsNot(x, None) # we never put None in there! + @unittest.skip("TODO: RUSTPYTHON; race condition between GC and WeakValueDictionary callback") + @threading_helper.requires_working_threading() def test_threaded_weak_valued_consistency(self): # Issue #28427: old keys should not remove new values from # WeakValueDictionary when collecting from another thread. d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(200000): + for i in range(2 * self.NUM_THREADED_ITERATIONS): o = RefCycle() d[10] = o # o is still alive, so the dict can't be empty self.assertEqual(len(d), 1) o = None # lose ref + @support.cpython_only + def test_weak_valued_consistency(self): + # A single-threaded, deterministic repro for issue #28427: old keys + # should not remove new values from WeakValueDictionary. This relies on + # an implementation detail of CPython's WeakValueDictionary (its + # underlying dictionary of KeyedRefs) to reproduce the issue. + d = weakref.WeakValueDictionary() + with support.disable_gc(): + d[10] = RefCycle() + # Keep the KeyedRef alive after it's replaced so that GC will invoke + # the callback. + wr = d.data[10] + # Replace the value with something that isn't cyclic garbage + o = RefCycle() + d[10] = o + # Trigger GC, which will invoke the callback for `wr` + gc.collect() + self.assertEqual(len(d), 1) + def check_threaded_weak_dict_copy(self, type_, deepcopy): # `type_` should be either WeakKeyDictionary or WeakValueDictionary. # `deepcopy` should be either True or False. @@ -1987,21 +2038,33 @@ def pop_and_collect(lst): if exc: raise exc[0] + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, False) + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_deepcopy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, False) + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (malloc corruption)") + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_deepcopy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. @@ -2184,7 +2247,6 @@ def error(): assert f3.atexit == True assert f4.atexit == True - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_atexit(self): prog = ('from test.test_weakref import FinalizeTestCase;'+ 'FinalizeTestCase.run_in_child()') @@ -2194,6 +2256,18 @@ def test_atexit(self): self.assertTrue(b'ZeroDivisionError' in err) +class ModuleTestCase(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_names(self): + for name in ('ReferenceType', 'ProxyType', 'CallableProxyType', + 'WeakMethod', 'WeakSet', 'WeakKeyDictionary', 'WeakValueDictionary'): + obj = getattr(weakref, name) + if name != 'WeakSet': + self.assertEqual(obj.__module__, 'weakref') + self.assertEqual(obj.__name__, name) + self.assertEqual(obj.__qualname__, name) + + libreftest = """ Doctest for examples in the library reference: weakref.rst >>> from test.support import gc_collect @@ -2282,19 +2356,10 @@ def test_atexit(self): __test__ = {'libreftest' : libreftest} -def test_main(): - support.run_unittest( - ReferencesTestCase, - WeakMethodTestCase, - MappingTestCase, - WeakValueDictionaryTestCase, - WeakKeyDictionaryTestCase, - SubclassableWeakrefTestCase, - FinalizeTestCase, - ) - # TODO: RUSTPYTHON - # support.run_doctest(sys.modules[__name__]) +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index b93ecfdb796..76e8e5c8ab7 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -44,7 +44,7 @@ def setUp(self): def test_methods(self): weaksetmethods = dir(WeakSet) for method in dir(set): - if method == 'test_c_api' or method.startswith('_'): + if method.startswith('_'): continue self.assertIn(method, weaksetmethods, "WeakSet missing method " + method) @@ -203,8 +203,6 @@ def test_constructor_identity(self): t = WeakSet(s) self.assertNotEqual(id(s), id(t)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_hash(self): self.assertRaises(TypeError, hash, self.s) @@ -324,8 +322,6 @@ def test_ixor(self): else: self.assertNotIn(c, self.s) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_inplace_on_self(self): t = self.s.copy() t |= t @@ -407,8 +403,6 @@ def testcontext(): s.clear() self.assertEqual(len(s), 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_len_cycles(self): N = 20 items = [RefCycle() for i in range(N)] @@ -429,8 +423,6 @@ def test_len_cycles(self): self.assertIn(n1, (0, 1)) self.assertEqual(n2, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_len_race(self): # Extended sanity checks for len() in the face of cyclic collection self.addCleanup(gc.set_threshold, *gc.get_threshold()) @@ -462,8 +454,6 @@ def test_abc(self): self.assertIsInstance(self.s, Set) self.assertIsInstance(self.s, MutableSet) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copying(self): for cls in WeakSet, WeakSetWithSlots: s = cls(self.items) diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 673cc995d3f..4fcbc5c2e59 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -1,15 +1,22 @@ -import webbrowser -import unittest import os -import sys +import re +import shlex import subprocess -from unittest import mock +import sys +import unittest +import webbrowser from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper +from test.support import requires_subprocess +from test.support import threading_helper +from unittest import mock +# The webbrowser module uses threading locks +threading_helper.requires_working_threading(module=True) -URL = 'http://www.example.com' +URL = 'https://www.example.com' CMD_NAME = 'test' @@ -22,6 +29,7 @@ def wait(self, seconds=None): return 0 +@requires_subprocess() class CommandTestMixin: def _test(self, meth, *, args=[URL], kw={}, options, arguments): @@ -92,10 +100,19 @@ def test_open_new_tab(self): options=[], arguments=[URL]) + def test_open_bad_new_parameter(self): + with self.assertRaisesRegex(webbrowser.Error, + re.escape("Bad 'new' parameter to open(); " + "expected 0, 1, or 2, got 999")): + self._test('open', + options=[], + arguments=[URL], + kw=dict(new=999)) -class MozillaCommandTest(CommandTestMixin, unittest.TestCase): - browser_class = webbrowser.Mozilla +class EdgeCommandTest(CommandTestMixin, unittest.TestCase): + + browser_class = webbrowser.Edge def test_open(self): self._test('open', @@ -109,43 +126,43 @@ def test_open_with_autoraise_false(self): def test_open_new(self): self._test('open_new', - options=[], - arguments=['-new-window', URL]) + options=['--new-window'], + arguments=[URL]) def test_open_new_tab(self): self._test('open_new_tab', options=[], - arguments=['-new-tab', URL]) + arguments=[URL]) -class NetscapeCommandTest(CommandTestMixin, unittest.TestCase): +class MozillaCommandTest(CommandTestMixin, unittest.TestCase): - browser_class = webbrowser.Netscape + browser_class = webbrowser.Mozilla def test_open(self): self._test('open', - options=['-raise', '-remote'], - arguments=['openURL({})'.format(URL)]) + options=[], + arguments=[URL]) def test_open_with_autoraise_false(self): self._test('open', kw=dict(autoraise=False), - options=['-noraise', '-remote'], - arguments=['openURL({})'.format(URL)]) + options=[], + arguments=[URL]) def test_open_new(self): self._test('open_new', - options=['-raise', '-remote'], - arguments=['openURL({},new-window)'.format(URL)]) + options=[], + arguments=['-new-window', URL]) def test_open_new_tab(self): self._test('open_new_tab', - options=['-raise', '-remote'], - arguments=['openURL({},new-tab)'.format(URL)]) + options=[], + arguments=['-new-tab', URL]) -class GaleonCommandTest(CommandTestMixin, unittest.TestCase): +class EpiphanyCommandTest(CommandTestMixin, unittest.TestCase): - browser_class = webbrowser.Galeon + browser_class = webbrowser.Epiphany def test_open(self): self._test('open', @@ -199,22 +216,89 @@ class ELinksCommandTest(CommandTestMixin, unittest.TestCase): def test_open(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_with_autoraise_false(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_new(self): self._test('open_new', options=['-remote'], - arguments=['openURL({},new-window)'.format(URL)]) + arguments=[f'openURL({URL},new-window)']) def test_open_new_tab(self): self._test('open_new_tab', options=['-remote'], - arguments=['openURL({},new-tab)'.format(URL)]) + arguments=[f'openURL({URL},new-tab)']) + + +@unittest.skipUnless(sys.platform == "ios", "Test only applicable to iOS") +class IOSBrowserTest(unittest.TestCase): + def _obj_ref(self, *args): + # Construct a string representation of the arguments that can be used + # as a proxy for object instance references + return "|".join(str(a) for a in args) + + @unittest.skipIf(getattr(webbrowser, "objc", None) is None, + "iOS Webbrowser tests require ctypes") + def setUp(self): + # Intercept the objc library. Wrap the calls to get the + # references to classes and selectors to return strings, and + # wrap msgSend to return stringified object references + self.orig_objc = webbrowser.objc + + webbrowser.objc = mock.Mock() + webbrowser.objc.objc_getClass = lambda cls: f"C#{cls.decode()}" + webbrowser.objc.sel_registerName = lambda sel: f"S#{sel.decode()}" + webbrowser.objc.objc_msgSend.side_effect = self._obj_ref + + def tearDown(self): + webbrowser.objc = self.orig_objc + + def _test(self, meth, **kwargs): + # The browser always gets focus, there's no concept of separate browser + # windows, and there's no API-level control over creating a new tab. + # Therefore, all calls to webbrowser are effectively the same. + getattr(webbrowser, meth)(URL, **kwargs) + + # The ObjC String version of the URL is created with UTF-8 encoding + url_string_args = [ + "C#NSString", + "S#stringWithCString:encoding:", + b'https://www.example.com', + 4, + ] + # The NSURL version of the URL is created from that string + url_obj_args = [ + "C#NSURL", + "S#URLWithString:", + self._obj_ref(*url_string_args), + ] + # The openURL call is invoked on the shared application + shared_app_args = ["C#UIApplication", "S#sharedApplication"] + + # Verify that the last call is the one that opens the URL. + webbrowser.objc.objc_msgSend.assert_called_with( + self._obj_ref(*shared_app_args), + "S#openURL:options:completionHandler:", + self._obj_ref(*url_obj_args), + None, + None + ) + + def test_open(self): + self._test('open') + + def test_open_with_autoraise_false(self): + self._test('open', autoraise=False) + + def test_open_new(self): + self._test('open_new') + + def test_open_new_tab(self): + self._test('open_new_tab') class BrowserRegistrationTest(unittest.TestCase): @@ -269,6 +353,16 @@ def test_register_default(self): def test_register_preferred(self): self._check_registration(preferred=True) + @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") + def test_no_xdg_settings_on_macOS(self): + # On macOS webbrowser should not use xdg-settings to + # look for X11 based browsers (for those users with + # XQuartz installed) + with mock.patch("subprocess.check_output") as ck_o: + webbrowser.register_standard_browsers() + + ck_o.assert_not_called() + class ImportTest(unittest.TestCase): def test_register(self): @@ -294,29 +388,38 @@ def test_get(self): webbrowser.get('fakebrowser') self.assertIsNotNone(webbrowser._tryorder) + @unittest.skipIf(" " in sys.executable, "test assumes no space in path (GH-114452)") def test_synthesize(self): webbrowser = import_helper.import_fresh_module('webbrowser') name = os.path.basename(sys.executable).lower() webbrowser.register(name, None, webbrowser.GenericBrowser(name)) webbrowser.get(sys.executable) + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: browser = webbrowser.get().name - except (webbrowser.Error, AttributeError) as err: + except webbrowser.Error as err: self.skipTest(str(err)) with os_helper.EnvironmentVarGuard() as env: env["BROWSER"] = browser webbrowser = import_helper.import_fresh_module('webbrowser') webbrowser.get() + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment_preferred(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: webbrowser.get() least_preferred_browser = webbrowser.get(webbrowser._tryorder[-1]).name - except (webbrowser.Error, AttributeError, IndexError) as err: + except (webbrowser.Error, IndexError) as err: self.skipTest(str(err)) with os_helper.EnvironmentVarGuard() as env: @@ -330,5 +433,74 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) -if __name__=='__main__': +class CliTest(unittest.TestCase): + def test_parse_args(self): + for command, url, new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + args = webbrowser.parse_args(shlex.split(command)) + + self.assertEqual(args.url, url) + self.assertEqual(args.new_win, new_win) + + def test_parse_args_error(self): + for command in [ + # Arguments must not both be given + "https://example.com -n -t", + "https://example.com --new-window --new-tab", + "https://example.com -n --new-tab", + "https://example.com --new-window -t", + ]: + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split(command)) + self.assertIn( + 'error: argument -t/--new-tab: not allowed with argument -n/--new-window', + stderr.getvalue(), + ) + + # Ensure ambiguous shortening fails + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split("https://example.com --new")) + self.assertIn( + 'error: ambiguous option: --new could match --new-window, --new-tab', + stderr.getvalue() + ) + + def test_main(self): + for command, expected_url, expected_new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + with ( + mock.patch("webbrowser.open", return_value=None) as mock_open, + mock.patch("builtins.print", return_value=None), + ): + webbrowser.main(shlex.split(command)) + mock_open.assert_called_once_with(expected_url, expected_new_win) + + +if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py new file mode 100644 index 00000000000..e64208330ad --- /dev/null +++ b/Lib/test/test_winapi.py @@ -0,0 +1,158 @@ +# Test the Windows-only _winapi module + +import os +import pathlib +import re +import unittest +from test.support import import_helper, os_helper + +_winapi = import_helper.import_module('_winapi', required_on=['win']) + +MAXIMUM_WAIT_OBJECTS = 64 +MAXIMUM_BATCHED_WAIT_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) ** 2 + +class WinAPIBatchedWaitForMultipleObjectsTests(unittest.TestCase): + def _events_waitall_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + + # Ensure no errors raised when all are triggered + for e in evts: + _winapi.SetEvent(e) + try: + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + except TimeoutError: + self.fail("expected wait to complete immediately") + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Replace events with invalid handles to make sure we fail + for i in chosen: + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + evts[i] = old_evt + + + def _events_waitany_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Trigger one by one. They are auto-reset events, so will only trigger once + for i in chosen: + with self.subTest(f"trigger event {i} of {len(evts)}"): + _winapi.SetEvent(evts[i]) + triggered = _winapi.BatchedWaitForMultipleObjects(evts, False, 10000) + self.assertSetEqual(set(triggered), {i}) + + # Trigger all at once. This may require multiple calls + for i in chosen: + _winapi.SetEvent(evts[i]) + triggered = set() + while len(triggered) < len(chosen): + triggered.update(_winapi.BatchedWaitForMultipleObjects(evts, False, 10000)) + self.assertSetEqual(triggered, set(chosen)) + + # Replace events with invalid handles to make sure we fail + for i in chosen: + with self.subTest(f"corrupt event {i} of {len(evts)}"): + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + evts[i] = old_evt + + + def test_few_events_waitall(self): + self._events_waitall_test(16) + + def test_many_events_waitall(self): + self._events_waitall_test(256) + + def test_max_events_waitall(self): + self._events_waitall_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + + def test_few_events_waitany(self): + self._events_waitany_test(16) + + def test_many_events_waitany(self): + self._events_waitany_test(256) + + def test_max_events_waitany(self): + self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + +class WinAPITests(unittest.TestCase): + def test_getlongpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")).parents[-1] / "PROGRA~1" + if not os.path.isdir(testfn): + raise unittest.SkipTest("require x:\\PROGRA~1 to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetLongPathName(testfn) + + actual = _winapi.GetLongPathName(os.fsdecode(testfn)) + + # Can't assume that PROGRA~1 expands to any particular variation, so + # ensure it matches any one of them. + candidates = set(testfn.parent.glob("Progra*")) + self.assertIn(pathlib.Path(actual), candidates) + + def test_getshortpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")) + if not os.path.isdir(testfn): + raise unittest.SkipTest("require '%ProgramFiles%' to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetShortPathName(testfn) + + actual = _winapi.GetShortPathName(os.fsdecode(testfn)) + + # Should contain "PROGRA~" but we can't predict the number + self.assertIsNotNone(re.match(r".\:\\PROGRA~\d", actual.upper()), actual) + + def test_namedpipe(self): + pipe_name = rf"\\.\pipe\LOCAL\{os_helper.TESTFN}" + + # Pipe does not exist, so this raises + with self.assertRaises(FileNotFoundError): + _winapi.WaitNamedPipe(pipe_name, 0) + + pipe = _winapi.CreateNamedPipe( + pipe_name, + _winapi.PIPE_ACCESS_DUPLEX, + 8, # 8=PIPE_REJECT_REMOTE_CLIENTS + 2, # two instances available + 32, 32, 0, 0) + self.addCleanup(_winapi.CloseHandle, pipe) + + # Pipe instance is available, so this passes + _winapi.WaitNamedPipe(pipe_name, 0) + + with open(pipe_name, 'w+b') as pipe2: + # No instances available, so this times out + # (WinError 121 does not get mapped to TimeoutError) + with self.assertRaises(OSError): + _winapi.WaitNamedPipe(pipe_name, 0) + + _winapi.WriteFile(pipe, b'testdata') + self.assertEqual(b'testdata', pipe2.read(8)) + + self.assertEqual((b'', 0), _winapi.PeekNamedPipe(pipe, 8)[:2]) + pipe2.write(b'testdata') + pipe2.flush() + self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py new file mode 100644 index 00000000000..1bae884ed9a --- /dev/null +++ b/Lib/test/test_winconsoleio.py @@ -0,0 +1,244 @@ +'''Tests for WindowsConsoleIO +''' + +import io +import os +import sys +import tempfile +import unittest +from test.support import os_helper, requires_resource + +if sys.platform != 'win32': + raise unittest.SkipTest("test only relevant on win32") + +from _testconsole import write_input + +ConIO = io._WindowsConsoleIO + +class WindowsConsoleIOTests(unittest.TestCase): + def test_abc(self): + self.assertIsSubclass(ConIO, io.RawIOBase) + self.assertNotIsSubclass(ConIO, io.BufferedIOBase) + self.assertNotIsSubclass(ConIO, io.TextIOBase) + + def test_open_fd(self): + self.assertRaisesRegex(ValueError, + "negative file descriptor", ConIO, -1) + + with tempfile.TemporaryFile() as tmpfile: + fd = tmpfile.fileno() + # Windows 10: "Cannot open non-console file" + # Earlier: "Cannot open console output buffer for reading" + self.assertRaisesRegex(ValueError, + "Cannot open (console|non-console file)", ConIO, fd) + + try: + f = ConIO(0) + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertEqual(0, f.fileno()) + f.close() # multiple close should not crash + f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass + + try: + f = ConIO(1, 'w') + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertEqual(1, f.fileno()) + f.close() + f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass + + try: + f = ConIO(2, 'w') + except ValueError: + # cannot open console because it's not a real console + pass + else: + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertEqual(2, f.fileno()) + f.close() + f.close() + + def test_open_name(self): + self.assertRaises(ValueError, ConIO, sys.executable) + + f = ConIO("CON") + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() # multiple close should not crash + f.close() + + f = ConIO('CONIN$') + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() + f.close() + + f = ConIO('CONOUT$', 'w') + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertIsNotNone(f.fileno()) + f.close() + f.close() + + # bpo-45354: Windows 11 changed MS-DOS device name handling + if sys.getwindowsversion()[:3] < (10, 0, 22000): + f = open('C:/con', 'rb', buffering=0) + self.assertIsInstance(f, ConIO) + f.close() + + def test_subclass_repr(self): + class TestSubclass(ConIO): + pass + + f = TestSubclass("CON") + with f: + self.assertIn(TestSubclass.__name__, repr(f)) + + self.assertIn(TestSubclass.__name__, repr(f)) + + @unittest.skipIf(sys.getwindowsversion()[:2] <= (6, 1), + "test does not work on Windows 7 and earlier") + def test_conin_conout_names(self): + f = open(r'\\.\conin$', 'rb', buffering=0) + self.assertIsInstance(f, ConIO) + f.close() + + f = open('//?/conout$', 'wb', buffering=0) + self.assertIsInstance(f, ConIO) + f.close() + + def test_conout_path(self): + temp_path = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, temp_path) + + conout_path = os.path.join(temp_path, 'CONOUT$') + + with open(conout_path, 'wb', buffering=0) as f: + # bpo-45354: Windows 11 changed MS-DOS device name handling + if (6, 1) < sys.getwindowsversion()[:3] < (10, 0, 22000): + self.assertIsInstance(f, ConIO) + else: + self.assertNotIsInstance(f, ConIO) + + def test_write_empty_data(self): + with ConIO('CONOUT$', 'w') as f: + self.assertEqual(f.write(b''), 0) + + @requires_resource('console') + def test_write(self): + testcases = [] + with ConIO('CONOUT$', 'w') as f: + for a in [ + b'', + b'abc', + b'\xc2\xa7\xe2\x98\x83\xf0\x9f\x90\x8d', + b'\xff'*10, + ]: + for b in b'\xc2\xa7', b'\xe2\x98\x83', b'\xf0\x9f\x90\x8d': + testcases.append(a + b) + for i in range(1, len(b)): + data = a + b[:i] + testcases.append(data + b'z') + testcases.append(data + b'\xff') + # incomplete multibyte sequence + with self.subTest(data=data): + self.assertEqual(f.write(data), len(a)) + for data in testcases: + with self.subTest(data=data): + self.assertEqual(f.write(data), len(data)) + + def assertStdinRoundTrip(self, text): + stdin = open('CONIN$', 'r') + old_stdin = sys.stdin + try: + sys.stdin = stdin + write_input( + stdin.buffer.raw, + (text + '\r\n').encode('utf-16-le', 'surrogatepass') + ) + actual = input() + finally: + sys.stdin = old_stdin + self.assertEqual(actual, text) + + @requires_resource('console') + def test_input(self): + # ASCII + self.assertStdinRoundTrip('abc123') + # Non-ASCII + self.assertStdinRoundTrip('ϼўТλФЙ') + # Combining characters + self.assertStdinRoundTrip('A͏B ﬖ̳AA̝') + + # bpo-38325 + @unittest.skipIf(True, "Handling Non-BMP characters is broken") + def test_input_nonbmp(self): + # Non-BMP + self.assertStdinRoundTrip('\U00100000\U0010ffff\U0010fffd') + + @requires_resource('console') + def test_partial_reads(self): + # Test that reading less than 1 full character works when stdin + # contains multibyte UTF-8 sequences + source = 'ϼўТλФЙ\r\n'.encode('utf-16-le') + expected = 'ϼўТλФЙ\r\n'.encode('utf-8') + for read_count in range(1, 16): + with open('CONIN$', 'rb', buffering=0) as stdin: + write_input(stdin, source) + + actual = b'' + while not actual.endswith(b'\n'): + b = stdin.read(read_count) + actual += b + + self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + + # bpo-38325 + @unittest.skipIf(True, "Handling Non-BMP characters is broken") + def test_partial_surrogate_reads(self): + # Test that reading less than 1 full character works when stdin + # contains surrogate pairs that cannot be decoded to UTF-8 without + # reading an extra character. + source = '\U00101FFF\U00101001\r\n'.encode('utf-16-le') + expected = '\U00101FFF\U00101001\r\n'.encode('utf-8') + for read_count in range(1, 16): + with open('CONIN$', 'rb', buffering=0) as stdin: + write_input(stdin, source) + + actual = b'' + while not actual.endswith(b'\n'): + b = stdin.read(read_count) + actual += b + + self.assertEqual(actual, expected, 'stdin.read({})'.format(read_count)) + + @requires_resource('console') + def test_ctrl_z(self): + with open('CONIN$', 'rb', buffering=0) as stdin: + source = '\xC4\x1A\r\n'.encode('utf-16-le') + expected = '\xC4'.encode('utf-8') + write_input(stdin, source) + a, b = stdin.read(1), stdin.readall() + self.assertEqual(expected[0:1], a) + self.assertEqual(expected[1:], b) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py new file mode 100644 index 00000000000..1bc830c02c3 --- /dev/null +++ b/Lib/test/test_winreg.py @@ -0,0 +1,557 @@ +# Test the windows specific win32reg module. +# Only win32reg functions not hit here: FlushKey, LoadKey and SaveKey + +import gc +import os, sys, errno +import itertools +import threading +import unittest +from platform import machine, win32_edition +from test.support import cpython_only, import_helper + +# Do this first so test will be skipped if module doesn't exist +import_helper.import_module('winreg', required_on=['win']) +# Now import everything +from winreg import * + +try: + REMOTE_NAME = sys.argv[sys.argv.index("--remote")+1] +except (IndexError, ValueError): + REMOTE_NAME = None + +# tuple of (major, minor) +WIN_VER = sys.getwindowsversion()[:2] +# Some tests should only run on 64-bit architectures where WOW64 will be. +WIN64_MACHINE = True if machine() == "AMD64" else False + +# Starting with Windows 7 and Windows Server 2008 R2, WOW64 no longer uses +# registry reflection and formerly reflected keys are shared instead. +# Windows 7 and Windows Server 2008 R2 are version 6.1. Due to this, some +# tests are only valid up until 6.1 +HAS_REFLECTION = True if WIN_VER < (6, 1) else False + +# Use a per-process key to prevent concurrent test runs (buildbot!) from +# stomping on each other. +test_key_base = "Python Test Key [%d] - Delete Me" % (os.getpid(),) +test_key_name = "SOFTWARE\\" + test_key_base +# On OS'es that support reflection we should test with a reflected key +test_reflect_key_name = "SOFTWARE\\Classes\\" + test_key_base + +test_data = [ + ("Int Value", 45, REG_DWORD), + ("Qword Value", 0x1122334455667788, REG_QWORD), + ("String Val", "A string value", REG_SZ), + ("StringExpand", "The path is %path%", REG_EXPAND_SZ), + ("Multi-string", ["Lots", "of", "string", "values"], REG_MULTI_SZ), + ("Multi-nul", ["", "", "", ""], REG_MULTI_SZ), + ("Raw Data", b"binary\x00data", REG_BINARY), + ("Big String", "x"*(2**14-1), REG_SZ), + ("Big Binary", b"x"*(2**14), REG_BINARY), + # Two and three kanjis, meaning: "Japan" and "Japanese". + ("Japanese 日本", "日本語", REG_SZ), +] + + +@cpython_only +class HeapTypeTests(unittest.TestCase): + def test_have_gc(self): + self.assertTrue(gc.is_tracked(HKEYType)) + + def test_immutable(self): + with self.assertRaisesRegex(TypeError, "immutable"): + HKEYType.foo = "bar" + + +class BaseWinregTests(unittest.TestCase): + + def setUp(self): + # Make sure that the test key is absent when the test + # starts. + self.delete_tree(HKEY_CURRENT_USER, test_key_name) + + def delete_tree(self, root, subkey): + try: + hkey = OpenKey(root, subkey, 0, KEY_ALL_ACCESS) + except OSError: + # subkey does not exist + return + while True: + try: + subsubkey = EnumKey(hkey, 0) + except OSError: + # no more subkeys + break + self.delete_tree(hkey, subsubkey) + CloseKey(hkey) + DeleteKey(root, subkey) + + def _write_test_data(self, root_key, subkeystr="sub_key", + CreateKey=CreateKey): + # Set the default value for this key. + SetValue(root_key, test_key_name, REG_SZ, "Default value") + key = CreateKey(root_key, test_key_name) + self.assertTrue(key.handle != 0) + # Create a sub-key + sub_key = CreateKey(key, subkeystr) + # Give the sub-key some named values + + for value_name, value_data, value_type in test_data: + SetValueEx(sub_key, value_name, 0, value_type, value_data) + + # Check we wrote as many items as we thought. + nkeys, nvalues, since_mod = QueryInfoKey(key) + self.assertEqual(nkeys, 1, "Not the correct number of sub keys") + self.assertEqual(nvalues, 1, "Not the correct number of values") + nkeys, nvalues, since_mod = QueryInfoKey(sub_key) + self.assertEqual(nkeys, 0, "Not the correct number of sub keys") + self.assertEqual(nvalues, len(test_data), + "Not the correct number of values") + # Close this key this way... + # (but before we do, copy the key as an integer - this allows + # us to test that the key really gets closed). + int_sub_key = int(sub_key) + CloseKey(sub_key) + try: + QueryInfoKey(int_sub_key) + self.fail("It appears the CloseKey() function does " + "not close the actual key!") + except OSError: + pass + # ... and close that key that way :-) + int_key = int(key) + key.Close() + try: + QueryInfoKey(int_key) + self.fail("It appears the key.Close() function " + "does not close the actual key!") + except OSError: + pass + def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): + # Check we can get default value for this key. + val = QueryValue(root_key, test_key_name) + self.assertEqual(val, "Default value", + "Registry didn't give back the correct value") + + key = OpenKey(root_key, test_key_name) + # Read the sub-keys + with OpenKey(key, subkeystr) as sub_key: + # Check I can enumerate over the values. + index = 0 + while 1: + try: + data = EnumValue(sub_key, index) + except OSError: + break + self.assertEqual(data in test_data, True, + "Didn't read back the correct test data") + index = index + 1 + self.assertEqual(index, len(test_data), + "Didn't read the correct number of items") + # Check I can directly access each item + for value_name, value_data, value_type in test_data: + read_val, read_typ = QueryValueEx(sub_key, value_name) + self.assertEqual(read_val, value_data, + "Could not directly read the value") + self.assertEqual(read_typ, value_type, + "Could not directly read the value") + sub_key.Close() + # Enumerate our main key. + read_val = EnumKey(key, 0) + self.assertEqual(read_val, subkeystr, "Read subkey value wrong") + try: + EnumKey(key, 1) + self.fail("Was able to get a second key when I only have one!") + except OSError: + pass + + key.Close() + + def _delete_test_data(self, root_key, subkeystr="sub_key"): + key = OpenKey(root_key, test_key_name, 0, KEY_ALL_ACCESS) + sub_key = OpenKey(key, subkeystr, 0, KEY_ALL_ACCESS) + # It is not necessary to delete the values before deleting + # the key (although subkeys must not exist). We delete them + # manually just to prove we can :-) + for value_name, value_data, value_type in test_data: + DeleteValue(sub_key, value_name) + + nkeys, nvalues, since_mod = QueryInfoKey(sub_key) + self.assertEqual(nkeys, 0, "subkey not empty before delete") + self.assertEqual(nvalues, 0, "subkey not empty before delete") + sub_key.Close() + DeleteKey(key, subkeystr) + + try: + # Shouldn't be able to delete it twice! + DeleteKey(key, subkeystr) + self.fail("Deleting the key twice succeeded") + except OSError: + pass + key.Close() + DeleteKey(root_key, test_key_name) + # Opening should now fail! + try: + key = OpenKey(root_key, test_key_name) + self.fail("Could open the non-existent key") + except OSError: # Use this error name this time + pass + + def _test_all(self, root_key, subkeystr="sub_key"): + self._write_test_data(root_key, subkeystr) + self._read_test_data(root_key, subkeystr) + self._delete_test_data(root_key, subkeystr) + + def _test_named_args(self, key, sub_key): + with CreateKeyEx(key=key, sub_key=sub_key, reserved=0, + access=KEY_ALL_ACCESS) as ckey: + self.assertTrue(ckey.handle != 0) + + with OpenKeyEx(key=key, sub_key=sub_key, reserved=0, + access=KEY_ALL_ACCESS) as okey: + self.assertTrue(okey.handle != 0) + + +class LocalWinregTests(BaseWinregTests): + + def test_registry_works(self): + self._test_all(HKEY_CURRENT_USER) + self._test_all(HKEY_CURRENT_USER, "日本-subkey") + + def test_registry_works_extended_functions(self): + # Substitute the regular CreateKey and OpenKey calls with their + # extended counterparts. + # Note: DeleteKeyEx is not used here because it is platform dependent + cke = lambda key, sub_key: CreateKeyEx(key, sub_key, 0, KEY_ALL_ACCESS) + self._write_test_data(HKEY_CURRENT_USER, CreateKey=cke) + + oke = lambda key, sub_key: OpenKeyEx(key, sub_key, 0, KEY_READ) + self._read_test_data(HKEY_CURRENT_USER, OpenKey=oke) + + self._delete_test_data(HKEY_CURRENT_USER) + + def test_named_arguments(self): + self._test_named_args(HKEY_CURRENT_USER, test_key_name) + # Use the regular DeleteKey to clean up + # DeleteKeyEx takes named args and is tested separately + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_connect_registry_to_local_machine_works(self): + # perform minimal ConnectRegistry test which just invokes it + h = ConnectRegistry(None, HKEY_LOCAL_MACHINE) + self.assertNotEqual(h.handle, 0) + h.Close() + self.assertEqual(h.handle, 0) + + def test_nonexistent_remote_registry(self): + connect = lambda: ConnectRegistry("abcdefghijkl", HKEY_CURRENT_USER) + self.assertRaises(OSError, connect) + + def testExpandEnvironmentStrings(self): + r = ExpandEnvironmentStrings("%windir%\\test") + self.assertEqual(type(r), str) + self.assertEqual(r, os.environ["windir"] + "\\test") + + def test_context_manager(self): + # ensure that the handle is closed if an exception occurs + try: + with ConnectRegistry(None, HKEY_LOCAL_MACHINE) as h: + self.assertNotEqual(h.handle, 0) + raise OSError + except OSError: + self.assertEqual(h.handle, 0) + + def test_changing_value(self): + # Issue2810: A race condition in 2.6 and 3.1 may cause + # EnumValue or QueryValue to raise "WindowsError: More data is + # available" + done = False + + class VeryActiveThread(threading.Thread): + def run(self): + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + use_short = True + long_string = 'x'*2000 + while not done: + s = 'x' if use_short else long_string + use_short = not use_short + SetValue(key, 'changing_value', REG_SZ, s) + + thread = VeryActiveThread() + thread.start() + try: + with CreateKey(HKEY_CURRENT_USER, + test_key_name+'\\changing_value') as key: + for _ in range(1000): + num_subkeys, num_values, t = QueryInfoKey(key) + for i in range(num_values): + name = EnumValue(key, i) + QueryValue(key, name[0]) + finally: + done = True + thread.join() + DeleteKey(HKEY_CURRENT_USER, test_key_name+'\\changing_value') + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_queryvalueex_race_condition(self): + # gh-142282: QueryValueEx could read garbage buffer under race + # condition when another thread changes the value size + done = False + ready = threading.Event() + values = [b'ham', b'spam'] + + class WriterThread(threading.Thread): + def run(self): + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + values_iter = itertools.cycle(values) + while not done: + val = next(values_iter) + SetValueEx(key, 'test_value', 0, REG_BINARY, val) + ready.set() + + thread = WriterThread() + thread.start() + try: + ready.wait() + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + for _ in range(1000): + result, typ = QueryValueEx(key, 'test_value') + # The result must be one of the written values, + # not garbage data from uninitialized buffer + self.assertIn(result, values) + finally: + done = True + thread.join() + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_long_key(self): + # Issue2810, in 2.6 and 3.1 when the key name was exactly 256 + # characters, EnumKey raised "WindowsError: More data is + # available" + name = 'x'*256 + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + SetValue(key, name, REG_SZ, 'x') + num_subkeys, num_values, t = QueryInfoKey(key) + EnumKey(key, 0) + finally: + DeleteKey(HKEY_CURRENT_USER, '\\'.join((test_key_name, name))) + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_dynamic_key(self): + # Issue2810, when the value is dynamically generated, these + # raise "WindowsError: More data is available" in 2.6 and 3.1 + try: + EnumValue(HKEY_PERFORMANCE_DATA, 0) + except OSError as e: + if e.errno in (errno.EPERM, errno.EACCES): + self.skipTest("access denied to registry key " + "(are you running in a non-interactive session?)") + raise + QueryValueEx(HKEY_PERFORMANCE_DATA, "") + + # Reflection requires XP x64/Vista at a minimum. XP doesn't have this stuff + # or DeleteKeyEx so make sure their use raises NotImplementedError + @unittest.skipUnless(WIN_VER < (5, 2), "Requires Windows XP") + def test_reflection_unsupported(self): + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + + key = OpenKey(HKEY_CURRENT_USER, test_key_name) + self.assertNotEqual(key.handle, 0) + + with self.assertRaises(NotImplementedError): + DisableReflectionKey(key) + with self.assertRaises(NotImplementedError): + EnableReflectionKey(key) + with self.assertRaises(NotImplementedError): + QueryReflectionKey(key) + with self.assertRaises(NotImplementedError): + DeleteKeyEx(HKEY_CURRENT_USER, test_key_name) + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_setvalueex_value_range(self): + # Test for Issue #14420, accept proper ranges for SetValueEx. + # Py2Reg, which gets called by SetValueEx, was using PyLong_AsLong, + # thus raising OverflowError. The implementation now uses + # PyLong_AsUnsignedLong to match DWORD's size. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + SetValueEx(ck, "test_name", None, REG_DWORD, 0x80000000) + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_setvalueex_negative_one_check(self): + # Test for Issue #43984, check -1 was not set by SetValueEx. + # Py2Reg, which gets called by SetValueEx, wasn't checking return + # value by PyLong_AsUnsignedLong, thus setting -1 as value in the registry. + # The implementation now checks PyLong_AsUnsignedLong return value to assure + # the value set was not -1. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + with self.assertRaises(OverflowError): + SetValueEx(ck, "test_name_dword", None, REG_DWORD, -1) + SetValueEx(ck, "test_name_qword", None, REG_QWORD, -1) + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_dword") + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_qword") + + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_queryvalueex_return_value(self): + # Test for Issue #16759, return unsigned int from QueryValueEx. + # Reg2Py, which gets called by QueryValueEx, was returning a value + # generated by PyLong_FromLong. The implementation now uses + # PyLong_FromUnsignedLong to match DWORD's size. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + test_val = 0x80000000 + SetValueEx(ck, "test_name", None, REG_DWORD, test_val) + ret_val, ret_type = QueryValueEx(ck, "test_name") + self.assertEqual(ret_type, REG_DWORD) + self.assertEqual(ret_val, test_val) + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_setvalueex_crash_with_none_arg(self): + # Test for Issue #21151, segfault when None is passed to SetValueEx + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + test_val = None + SetValueEx(ck, "test_name", 0, REG_BINARY, test_val) + ret_val, ret_type = QueryValueEx(ck, "test_name") + self.assertEqual(ret_type, REG_BINARY) + self.assertEqual(ret_val, test_val) + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_read_string_containing_null(self): + # Test for issue 25778: REG_SZ should not contain null characters + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + test_val = "A string\x00 with a null" + SetValueEx(ck, "test_name", 0, REG_SZ, test_val) + ret_val, ret_type = QueryValueEx(ck, "test_name") + self.assertEqual(ret_type, REG_SZ) + self.assertEqual(ret_val, "A string") + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + +@unittest.skipUnless(REMOTE_NAME, "Skipping remote registry tests") +class RemoteWinregTests(BaseWinregTests): + + def test_remote_registry_works(self): + remote_key = ConnectRegistry(REMOTE_NAME, HKEY_CURRENT_USER) + self._test_all(remote_key) + + +@unittest.skipUnless(WIN64_MACHINE, "x64 specific registry tests") +class Win64WinregTests(BaseWinregTests): + + def test_named_arguments(self): + self._test_named_args(HKEY_CURRENT_USER, test_key_name) + # Clean up and also exercise the named arguments + DeleteKeyEx(key=HKEY_CURRENT_USER, sub_key=test_key_name, + access=KEY_ALL_ACCESS, reserved=0) + + @unittest.skipIf(win32_edition() in ('WindowsCoreHeadless', 'IoTEdgeOS'), "APIs not available on WindowsCoreHeadless") + def test_reflection_functions(self): + # Test that we can call the query, enable, and disable functions + # on a key which isn't on the reflection list with no consequences. + with OpenKey(HKEY_LOCAL_MACHINE, "Software") as key: + # HKLM\Software is redirected but not reflected in all OSes + self.assertTrue(QueryReflectionKey(key)) + self.assertIsNone(EnableReflectionKey(key)) + self.assertIsNone(DisableReflectionKey(key)) + self.assertTrue(QueryReflectionKey(key)) + + @unittest.skipUnless(HAS_REFLECTION, "OS doesn't support reflection") + def test_reflection(self): + # Test that we can create, open, and delete keys in the 32-bit + # area. Because we are doing this in a key which gets reflected, + # test the differences of 32 and 64-bit keys before and after the + # reflection occurs (ie. when the created key is closed). + try: + with CreateKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_ALL_ACCESS | KEY_WOW64_32KEY) as created_key: + self.assertNotEqual(created_key.handle, 0) + + # The key should now be available in the 32-bit area + with OpenKey(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_ALL_ACCESS | KEY_WOW64_32KEY) as key: + self.assertNotEqual(key.handle, 0) + + # Write a value to what currently is only in the 32-bit area + SetValueEx(created_key, "", 0, REG_SZ, "32KEY") + + # The key is not reflected until created_key is closed. + # The 64-bit version of the key should not be available yet. + open_fail = lambda: OpenKey(HKEY_CURRENT_USER, + test_reflect_key_name, 0, + KEY_READ | KEY_WOW64_64KEY) + self.assertRaises(OSError, open_fail) + + # Now explicitly open the 64-bit version of the key + with OpenKey(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_ALL_ACCESS | KEY_WOW64_64KEY) as key: + self.assertNotEqual(key.handle, 0) + # Make sure the original value we set is there + self.assertEqual("32KEY", QueryValue(key, "")) + # Set a new value, which will get reflected to 32-bit + SetValueEx(key, "", 0, REG_SZ, "64KEY") + + # Reflection uses a "last-writer wins policy, so the value we set + # on the 64-bit key should be the same on 32-bit + with OpenKey(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_READ | KEY_WOW64_32KEY) as key: + self.assertEqual("64KEY", QueryValue(key, "")) + finally: + DeleteKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, + KEY_WOW64_32KEY, 0) + + @unittest.skipUnless(HAS_REFLECTION, "OS doesn't support reflection") + def test_disable_reflection(self): + # Make use of a key which gets redirected and reflected + try: + with CreateKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_ALL_ACCESS | KEY_WOW64_32KEY) as created_key: + # QueryReflectionKey returns whether or not the key is disabled + disabled = QueryReflectionKey(created_key) + self.assertEqual(type(disabled), bool) + # HKCU\Software\Classes is reflected by default + self.assertFalse(disabled) + + DisableReflectionKey(created_key) + self.assertTrue(QueryReflectionKey(created_key)) + + # The key is now closed and would normally be reflected to the + # 64-bit area, but let's make sure that didn't happen. + open_fail = lambda: OpenKeyEx(HKEY_CURRENT_USER, + test_reflect_key_name, 0, + KEY_READ | KEY_WOW64_64KEY) + self.assertRaises(OSError, open_fail) + + # Make sure the 32-bit key is actually there + with OpenKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_READ | KEY_WOW64_32KEY) as key: + self.assertNotEqual(key.handle, 0) + finally: + DeleteKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, + KEY_WOW64_32KEY, 0) + + def test_exception_numbers(self): + with self.assertRaises(FileNotFoundError) as ctx: + QueryValue(HKEY_CLASSES_ROOT, 'some_value_that_does_not_exist') + + +if __name__ == "__main__": + if not REMOTE_NAME: + print("Remote registry calls can be tested using", + "'test_winreg.py --remote \\\\machine_name'") + unittest.main() diff --git a/Lib/test/test_winsound.py b/Lib/test/test_winsound.py new file mode 100644 index 00000000000..9724d830ade --- /dev/null +++ b/Lib/test/test_winsound.py @@ -0,0 +1,187 @@ +# Ridiculously simple test of the winsound module for Windows. + +import functools +import os +import time +import unittest + +from test import support +from test.support import import_helper +from test.support import os_helper + + +support.requires('audio') +winsound = import_helper.import_module('winsound') + + +# Unless we actually have an ear in the room, we have no idea whether a sound +# actually plays, and it's incredibly flaky trying to figure out if a sound +# even *should* play. Instead of guessing, just call the function and assume +# it either passed or raised the RuntimeError we expect in case of failure. +def sound_func(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + ret = func(*args, **kwargs) + except RuntimeError as e: + if support.verbose: + print(func.__name__, 'failed:', e) + else: + if support.verbose: + print(func.__name__, 'returned') + return ret + return wrapper + + +safe_Beep = sound_func(winsound.Beep) +safe_MessageBeep = sound_func(winsound.MessageBeep) +safe_PlaySound = sound_func(winsound.PlaySound) + + +class BeepTest(unittest.TestCase): + + def test_errors(self): + self.assertRaises(TypeError, winsound.Beep) + self.assertRaises(ValueError, winsound.Beep, 36, 75) + self.assertRaises(ValueError, winsound.Beep, 32768, 75) + + def test_extremes(self): + safe_Beep(37, 75) + safe_Beep(32767, 75) + + def test_increasingfrequency(self): + for i in range(100, 2000, 100): + safe_Beep(i, 75) + + def test_keyword_args(self): + safe_Beep(duration=75, frequency=2000) + + +class MessageBeepTest(unittest.TestCase): + + def tearDown(self): + time.sleep(0.5) + + def test_default(self): + self.assertRaises(TypeError, winsound.MessageBeep, "bad") + self.assertRaises(TypeError, winsound.MessageBeep, 42, 42) + safe_MessageBeep() + + def test_ok(self): + safe_MessageBeep(winsound.MB_OK) + + def test_asterisk(self): + safe_MessageBeep(winsound.MB_ICONASTERISK) + + def test_exclamation(self): + safe_MessageBeep(winsound.MB_ICONEXCLAMATION) + + def test_hand(self): + safe_MessageBeep(winsound.MB_ICONHAND) + + def test_question(self): + safe_MessageBeep(winsound.MB_ICONQUESTION) + + def test_error(self): + safe_MessageBeep(winsound.MB_ICONERROR) + + def test_information(self): + safe_MessageBeep(winsound.MB_ICONINFORMATION) + + def test_stop(self): + safe_MessageBeep(winsound.MB_ICONSTOP) + + def test_warning(self): + safe_MessageBeep(winsound.MB_ICONWARNING) + + def test_keyword_args(self): + safe_MessageBeep(type=winsound.MB_OK) + + +class PlaySoundTest(unittest.TestCase): + + def test_errors(self): + self.assertRaises(TypeError, winsound.PlaySound) + self.assertRaises(TypeError, winsound.PlaySound, "bad", "bad") + self.assertRaises( + RuntimeError, + winsound.PlaySound, + "none", winsound.SND_ASYNC | winsound.SND_MEMORY + ) + self.assertRaises(TypeError, winsound.PlaySound, b"bad", 0) + self.assertRaises(TypeError, winsound.PlaySound, "bad", + winsound.SND_MEMORY) + self.assertRaises(TypeError, winsound.PlaySound, 1, 0) + # embedded null character + self.assertRaises(ValueError, winsound.PlaySound, 'bad\0', 0) + + def test_keyword_args(self): + safe_PlaySound(flags=winsound.SND_ALIAS, sound="SystemExit") + + def test_snd_memory(self): + with open(support.findfile('pluck-pcm8.wav', + subdir='audiodata'), 'rb') as f: + audio_data = f.read() + safe_PlaySound(audio_data, winsound.SND_MEMORY) + audio_data = bytearray(audio_data) + safe_PlaySound(audio_data, winsound.SND_MEMORY) + + def test_snd_filename(self): + fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') + safe_PlaySound(fn, winsound.SND_FILENAME | winsound.SND_NODEFAULT) + + def test_snd_filepath(self): + fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') + path = os_helper.FakePath(fn) + safe_PlaySound(path, winsound.SND_FILENAME | winsound.SND_NODEFAULT) + + def test_snd_filepath_as_bytes(self): + fn = support.findfile('pluck-pcm8.wav', subdir='audiodata') + self.assertRaises( + TypeError, + winsound.PlaySound, + os_helper.FakePath(os.fsencode(fn)), + winsound.SND_FILENAME | winsound.SND_NODEFAULT + ) + + def test_aliases(self): + aliases = [ + "SystemAsterisk", + "SystemExclamation", + "SystemExit", + "SystemHand", + "SystemQuestion", + ] + for alias in aliases: + with self.subTest(alias=alias): + safe_PlaySound(alias, winsound.SND_ALIAS) + + def test_alias_fallback(self): + safe_PlaySound('!"$%&/(#+*', winsound.SND_ALIAS) + + def test_alias_nofallback(self): + safe_PlaySound('!"$%&/(#+*', winsound.SND_ALIAS | winsound.SND_NODEFAULT) + + def test_stopasync(self): + safe_PlaySound( + 'SystemQuestion', + winsound.SND_ALIAS | winsound.SND_ASYNC | winsound.SND_LOOP + ) + time.sleep(0.5) + safe_PlaySound('SystemQuestion', winsound.SND_ALIAS | winsound.SND_NOSTOP) + # Issue 8367: PlaySound(None, winsound.SND_PURGE) + # does not raise on systems without a sound card. + winsound.PlaySound(None, winsound.SND_PURGE) + + def test_sound_sentry(self): + safe_PlaySound("SystemExit", winsound.SND_ALIAS | winsound.SND_SENTRY) + + def test_sound_sync(self): + safe_PlaySound("SystemExit", winsound.SND_ALIAS | winsound.SND_SYNC) + + def test_sound_system(self): + safe_PlaySound("SystemExit", winsound.SND_ALIAS | winsound.SND_SYSTEM) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index 07522bda6a5..68ee3725088 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -1,15 +1,27 @@ -"""Unit tests for the with statement specified in PEP 343.""" +"""Unit tests for the 'with/async with' statements specified in PEP 343/492.""" __author__ = "Mike Bland" __email__ = "mbland at acm dot org" +import re import sys +import traceback import unittest from collections import deque from contextlib import _GeneratorContextManager, contextmanager, nullcontext +def do_with(obj): + with obj: + pass + + +async def do_async_with(obj): + async with obj: + pass + + class MockContextManager(_GeneratorContextManager): def __init__(self, *args): super().__init__(*args) @@ -79,11 +91,11 @@ def __exit__(self, *exc_info): try: if mgr.__exit__(*ex): ex = (None, None, None) - except: - ex = sys.exc_info() + except BaseException as e: + ex = (type(e), e, e.__traceback__) self.entered = None if ex is not exc_info: - raise ex[0](ex[1]).with_traceback(ex[2]) + raise ex class MockNested(Nested): @@ -109,34 +121,77 @@ def fooNotDeclared(): with foo: pass self.assertRaises(NameError, fooNotDeclared) - def testEnterAttributeError1(self): - class LacksEnter(object): - def __exit__(self, type, value, traceback): - pass + def testEnterAttributeError(self): + class LacksEnter: + def __exit__(self, type, value, traceback): ... - def fooLacksEnter(): - foo = LacksEnter() - with foo: pass - self.assertRaisesRegex(TypeError, 'the context manager', fooLacksEnter) - - def testEnterAttributeError2(self): - class LacksEnterAndExit(object): - pass - - def fooLacksEnterAndExit(): - foo = LacksEnterAndExit() - with foo: pass - self.assertRaisesRegex(TypeError, 'the context manager', fooLacksEnterAndExit) + with self.assertRaisesRegex(TypeError, re.escape(( + "object does not support the context manager protocol " + "(missed __enter__ method)" + ))): + do_with(LacksEnter()) def testExitAttributeError(self): - class LacksExit(object): - def __enter__(self): - pass - - def fooLacksExit(): - foo = LacksExit() - with foo: pass - self.assertRaisesRegex(TypeError, 'the context manager.*__exit__', fooLacksExit) + class LacksExit: + def __enter__(self): ... + + msg = re.escape(( + "object does not support the context manager protocol " + "(missed __exit__ method)" + )) + # a missing __exit__ is reported missing before a missing __enter__ + with self.assertRaisesRegex(TypeError, msg): + do_with(object()) + with self.assertRaisesRegex(TypeError, msg): + do_with(LacksExit()) + + def testWithForAsyncManager(self): + class AsyncManager: + async def __aenter__(self): ... + async def __aexit__(self, type, value, traceback): ... + + with self.assertRaisesRegex(TypeError, re.escape(( + "object does not support the context manager protocol " + "(missed __exit__ method) but it supports the asynchronous " + "context manager protocol. Did you mean to use 'async with'?" + ))): + do_with(AsyncManager()) + + def testAsyncEnterAttributeError(self): + class LacksAsyncEnter: + async def __aexit__(self, type, value, traceback): ... + + with self.assertRaisesRegex(TypeError, re.escape(( + "object does not support the asynchronous context manager protocol " + "(missed __aenter__ method)" + ))): + do_async_with(LacksAsyncEnter()).send(None) + + def testAsyncExitAttributeError(self): + class LacksAsyncExit: + async def __aenter__(self): ... + + msg = re.escape(( + "object does not support the asynchronous context manager protocol " + "(missed __aexit__ method)" + )) + # a missing __aexit__ is reported missing before a missing __aenter__ + with self.assertRaisesRegex(TypeError, msg): + do_async_with(object()).send(None) + with self.assertRaisesRegex(TypeError, msg): + do_async_with(LacksAsyncExit()).send(None) + + def testAsyncWithForSyncManager(self): + class SyncManager: + def __enter__(self): ... + def __exit__(self, type, value, traceback): ... + + with self.assertRaisesRegex(TypeError, re.escape(( + "object does not support the asynchronous context manager protocol " + "(missed __aexit__ method) but it supports the context manager " + "protocol. Did you mean to use 'with'?" + ))): + do_async_with(SyncManager()).send(None) def assertRaisesSyntaxError(self, codestr): def shouldRaiseSyntaxError(s): @@ -170,7 +225,10 @@ def __exit__(self, *args): def shouldThrow(): ct = EnterThrows() self.foo = None - with ct as self.foo: + # Ruff complains that we're redefining `self.foo` here, + # but the whole point of the test is to check that `self.foo` + # is *not* redefined (because `__enter__` raises) + with ct as self.foo: # noqa: F811 pass self.assertRaises(RuntimeError, shouldThrow) self.assertEqual(self.foo, None) @@ -186,6 +244,7 @@ def shouldThrow(): pass self.assertRaises(RuntimeError, shouldThrow) + class ContextmanagerAssertionMixin(object): def setUp(self): @@ -251,7 +310,6 @@ def testInlineGeneratorBoundSyntax(self): self.assertAfterWithGeneratorInvariantsNoError(foo) def testInlineGeneratorBoundToExistingVariable(self): - foo = None with mock_contextmanager_generator() as foo: self.assertInWithGeneratorInvariants(foo) self.assertAfterWithGeneratorInvariantsNoError(foo) @@ -621,7 +679,7 @@ def testSingleComplexTarget(self): class C: pass blah = C() with mock_contextmanager_generator() as blah.foo: - self.assertEqual(hasattr(blah, "foo"), True) + self.assertHasAttr(blah, "foo") def testMultipleComplexTargets(self): class C: @@ -716,7 +774,7 @@ def testExceptionInExprList(self): try: with self.Dummy() as a, self.InitRaises(): pass - except: + except RuntimeError: pass self.assertTrue(a.enter_called) self.assertTrue(a.exit_called) @@ -749,5 +807,49 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FrameSummary' object has no attribute 'end_lineno' + def testExceptionLocation(self): + # The location of an exception raised from + # __init__, __enter__ or __exit__ of a context + # manager should be just the context manager expression, + # pinpointing the precise context manager in case there + # is more than one. + + def init_raises(): + try: + with self.Dummy(), self.InitRaises() as cm, self.Dummy() as d: + pass + except Exception as e: + return e + + def enter_raises(): + try: + with self.EnterRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + def exit_raises(): + try: + with self.ExitRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "self.InitRaises()"), + (enter_raises, "self.EnterRaises()"), + (exit_raises, "self.ExitRaises()"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py new file mode 100644 index 00000000000..90eb40439d4 --- /dev/null +++ b/Lib/test/test_wmi.py @@ -0,0 +1,89 @@ +# Test the internal _wmi module on Windows +# This is used by the platform module, and potentially others + +import unittest +from test import support +from test.support import import_helper + + +# Do this first so test will be skipped if module doesn't exist +_wmi = import_helper.import_module('_wmi', required_on=['win']) + + +def wmi_exec_query(query): + # gh-112278: WMI maybe slow response when first call. + for _ in support.sleeping_retry(support.LONG_TIMEOUT): + try: + return _wmi.exec_query(query) + except BrokenPipeError: + pass + # retry on pipe error + except WindowsError as exc: + if exc.winerror != 258: + raise + # retry on timeout + + +class WmiTests(unittest.TestCase): + def test_wmi_query_os_version(self): + r = wmi_exec_query("SELECT Version FROM Win32_OperatingSystem").split("\0") + self.assertEqual(1, len(r)) + k, eq, v = r[0].partition("=") + self.assertEqual("=", eq, r[0]) + self.assertEqual("Version", k, r[0]) + # Best we can check for the version is that it's digits, dot, digits, anything + # Otherwise, we are likely checking the result of the query against itself + self.assertRegex(v, r"\d+\.\d+.+$", r[0]) + + def test_wmi_query_repeated(self): + # Repeated queries should not break + for _ in range(10): + self.test_wmi_query_os_version() + + def test_wmi_query_error(self): + # Invalid queries fail with OSError + try: + wmi_exec_query("SELECT InvalidColumnName FROM InvalidTableName") + except OSError as ex: + if ex.winerror & 0xFFFFFFFF == 0x80041010: + # This is the expected error code. All others should fail the test + return + self.fail("Expected OSError") + + def test_wmi_query_repeated_error(self): + for _ in range(10): + self.test_wmi_query_error() + + def test_wmi_query_not_select(self): + # Queries other than SELECT are blocked to avoid potential exploits + with self.assertRaises(ValueError): + wmi_exec_query("not select, just in case someone tries something") + + @support.requires_resource('cpu') + def test_wmi_query_overflow(self): + # Ensure very big queries fail + # Test multiple times to ensure consistency + for _ in range(2): + with self.assertRaises(OSError): + wmi_exec_query("SELECT * FROM CIM_DataFile") + + def test_wmi_query_multiple_rows(self): + # Multiple instances should have an extra null separator + r = wmi_exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") + self.assertNotStartsWith(r, "\0") + self.assertNotEndsWith(r, "\0") + it = iter(r.split("\0")) + try: + while True: + self.assertRegex(next(it), r"ProcessId=\d+") + self.assertEqual("", next(it)) + except StopIteration: + pass + + def test_wmi_query_threads(self): + from concurrent.futures import ThreadPoolExecutor + query = "SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000" + with ThreadPoolExecutor(4) as pool: + task = [pool.submit(wmi_exec_query, query) for _ in range(32)] + for t in task: + self.assertRegex(t.result(), "ProcessId=") diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index b89e181f9f8..d546e3ef219 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -1,6 +1,6 @@ from unittest import mock from test import support -from test.support import warnings_helper +from test.support import socket_helper from test.test_httpservers import NoLogRequestHandler from unittest import TestCase from wsgiref.util import setup_testing_defaults @@ -80,41 +80,26 @@ def run_amock(app=hello_app, data=b"GET / HTTP/1.0\n\n"): return out.getvalue(), err.getvalue() -def compare_generic_iter(make_it,match): - """Utility to compare a generic 2.1/2.2+ iterator with an iterable - If running under Python 2.2+, this tests the iterator using iter()/next(), - as well as __getitem__. 'make_it' must be a function returning a fresh +def compare_generic_iter(make_it, match): + """Utility to compare a generic iterator with an iterable + + This tests the iterator using iter()/next(). + 'make_it' must be a function returning a fresh iterator to be tested (since this may test the iterator twice).""" it = make_it() - n = 0 + if not iter(it) is it: + raise AssertionError for item in match: - if not it[n]==item: raise AssertionError - n+=1 - try: - it[n] - except IndexError: - pass - else: - raise AssertionError("Too many items from __getitem__",it) - + if not next(it) == item: + raise AssertionError try: - iter, StopIteration - except NameError: + next(it) + except StopIteration: pass else: - # Only test iter mode under 2.2+ - it = make_it() - if not iter(it) is it: raise AssertionError - for item in match: - if not next(it) == item: raise AssertionError - try: - next(it) - except StopIteration: - pass - else: - raise AssertionError("Too many items from .__next__()", it) + raise AssertionError("Too many items from .__next__()", it) class IntegrationTests(TestCase): @@ -152,7 +137,7 @@ def test_environ(self): def test_request_length(self): out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") self.assertEqual(out.splitlines()[0], - b"HTTP/1.0 414 Request-URI Too Long") + b"HTTP/1.0 414 URI Too Long") def test_validated_hello(self): out, err = run_amock(validator(hello_app)) @@ -264,7 +249,7 @@ def app(environ, start_response): class WsgiHandler(NoLogRequestHandler, WSGIRequestHandler): pass - server = make_server(support.HOST, 0, app, handler_class=WsgiHandler) + server = make_server(socket_helper.HOST, 0, app, handler_class=WsgiHandler) self.addCleanup(server.server_close) interrupted = threading.Event() @@ -339,7 +324,6 @@ def checkReqURI(self,uri,query=1,**kw): util.setup_testing_defaults(kw) self.assertEqual(util.request_uri(kw,query),uri) - @warnings_helper.ignore_warnings(category=DeprecationWarning) def checkFW(self,text,size,match): def make_it(text=text,size=size): @@ -358,15 +342,6 @@ def make_it(text=text,size=size): it.close() self.assertTrue(it.filelike.closed) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_filewrapper_getitem_deprecation(self): - wrapper = util.FileWrapper(StringIO('foobar'), 3) - with self.assertWarnsRegex(DeprecationWarning, - r'Use iterator protocol instead'): - # This should have returned 'bar'. - self.assertEqual(wrapper[1], 'foo') - def testSimpleShifts(self): self.checkShift('','/', '', '/', '') self.checkShift('','/x', 'x', '/x', '') @@ -473,6 +448,10 @@ def testHopByHop(self): for alt in hop, hop.title(), hop.upper(), hop.lower(): self.assertFalse(util.is_hop_by_hop(alt)) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_filewrapper_getitem_deprecation(self): + return super().test_filewrapper_getitem_deprecation() + class HeaderTests(TestCase): def testMappingInterface(self): @@ -581,7 +560,7 @@ def testEnviron(self): # Test handler.environ as a dict expected = {} setup_testing_defaults(expected) - # Handler inherits os_environ variables which are not overriden + # Handler inherits os_environ variables which are not overridden # by SimpleHandler.add_cgi_vars() (SimpleHandler.base_env) for key, value in os_environ.items(): if key not in expected: @@ -821,8 +800,6 @@ def flush(self): b"Hello, world!", written) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testClientConnectionTerminations(self): environ = {"SERVER_PROTOCOL": "HTTP/1.0"} for exception in ( @@ -841,8 +818,6 @@ def write(self, b): self.assertFalse(stderr.getvalue()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def testDontResetInternalStateOnException(self): class CustomException(ValueError): pass diff --git a/Lib/test/test_xdrlib.py b/Lib/test/test_xdrlib.py deleted file mode 100644 index 5f62649f6e3..00000000000 --- a/Lib/test/test_xdrlib.py +++ /dev/null @@ -1,81 +0,0 @@ -import unittest - -import xdrlib - -class XDRTest(unittest.TestCase): - - def test_xdr(self): - p = xdrlib.Packer() - - s = b'hello world' - a = [b'what', b'is', b'hapnin', b'doctor'] - - p.pack_int(42) - p.pack_int(-17) - p.pack_uint(9) - p.pack_bool(True) - p.pack_bool(False) - p.pack_uhyper(45) - p.pack_float(1.9) - p.pack_double(1.9) - p.pack_string(s) - p.pack_list(range(5), p.pack_uint) - p.pack_array(a, p.pack_string) - - # now verify - data = p.get_buffer() - up = xdrlib.Unpacker(data) - - self.assertEqual(up.get_position(), 0) - - self.assertEqual(up.unpack_int(), 42) - self.assertEqual(up.unpack_int(), -17) - self.assertEqual(up.unpack_uint(), 9) - self.assertTrue(up.unpack_bool() is True) - - # remember position - pos = up.get_position() - self.assertTrue(up.unpack_bool() is False) - - # rewind and unpack again - up.set_position(pos) - self.assertTrue(up.unpack_bool() is False) - - self.assertEqual(up.unpack_uhyper(), 45) - self.assertAlmostEqual(up.unpack_float(), 1.9) - self.assertAlmostEqual(up.unpack_double(), 1.9) - self.assertEqual(up.unpack_string(), s) - self.assertEqual(up.unpack_list(up.unpack_uint), list(range(5))) - self.assertEqual(up.unpack_array(up.unpack_string), a) - up.done() - self.assertRaises(EOFError, up.unpack_uint) - -class ConversionErrorTest(unittest.TestCase): - - def setUp(self): - self.packer = xdrlib.Packer() - - def assertRaisesConversion(self, *args): - self.assertRaises(xdrlib.ConversionError, *args) - - def test_pack_int(self): - self.assertRaisesConversion(self.packer.pack_int, 'string') - - def test_pack_uint(self): - self.assertRaisesConversion(self.packer.pack_uint, 'string') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_float(self): - self.assertRaisesConversion(self.packer.pack_float, 'string') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_double(self): - self.assertRaisesConversion(self.packer.pack_double, 'string') - - def test_uhyper(self): - self.assertRaisesConversion(self.packer.pack_uhyper, 'string') - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_xml_dom_minicompat.py b/Lib/test/test_xml_dom_minicompat.py index c90a01d2e42..3b03dfc5399 100644 --- a/Lib/test/test_xml_dom_minicompat.py +++ b/Lib/test/test_xml_dom_minicompat.py @@ -82,8 +82,6 @@ def test_nodelist___radd__(self): node_list = [1, 2] + NodeList([3, 4]) self.assertEqual(node_list, NodeList([1, 2, 3, 4])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nodelist_pickle_roundtrip(self): # Test pickling and unpickling of a NodeList. diff --git a/Lib/test/test_xml_dom_xmlbuilder.py b/Lib/test/test_xml_dom_xmlbuilder.py new file mode 100644 index 00000000000..5f5f2eb328d --- /dev/null +++ b/Lib/test/test_xml_dom_xmlbuilder.py @@ -0,0 +1,88 @@ +import io +import unittest +from http import client +from test.test_httplib import FakeSocket +from unittest import mock +from xml.dom import getDOMImplementation, minidom, xmlbuilder + +SMALL_SAMPLE = b"""<?xml version="1.0"?> +<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xdc="http://www.xml.com/books"> +<!-- A comment --> +<title>Introduction to XSL</title> +<hr/> +<p><xdc:author xdc:attrib="prefixed attribute" attrib="other attrib">A. Namespace</xdc:author></p> +</html>""" + + +class XMLBuilderTest(unittest.TestCase): + def test_entity_resolver(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + resolver = xmlbuilder.DOMEntityResolver() + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + source = resolver.resolveEntity(None, "http://example.com/2000/svg") + + self.assertIsInstance(source, xmlbuilder.DOMInputSource) + self.assertIsNone(source.publicId) + self.assertEqual(source.systemId, "http://example.com/2000/svg") + self.assertEqual(source.baseURI, "http://example.com/2000/") + self.assertEqual(source.encoding, "utf-8") + self.assertIs(source.byteStream, response) + + self.assertIsNone(source.characterStream) + self.assertIsNone(source.stringData) + + def test_builder(self): + imp = getDOMImplementation() + self.assertIsInstance(imp, xmlbuilder.DOMImplementationLS) + + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + self.assertIsInstance(builder, xmlbuilder.DOMBuilder) + + def test_parse_uri(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + + imp = getDOMImplementation() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + document = builder.parseURI("http://example.com/2000/svg") + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) + + def test_parse_with_systemId(self): + response = io.BytesIO(SMALL_SAMPLE) + + with mock.patch("urllib.request.urlopen") as mock_open: + mock_open.return_value = response + + imp = getDOMImplementation() + source = imp.createDOMInputSource() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + source.systemId = "http://example.com/2000/svg" + document = builder.parse(source) + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 1a681d5a7ca..f9b4f3abcad 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -13,13 +13,16 @@ import operator import os import pickle +import pyexpat import sys import textwrap import types import unittest +import unittest.mock as mock import warnings import weakref +from contextlib import nullcontext from functools import partial from itertools import product, islice from test import support @@ -120,6 +123,21 @@ </foo> """ +def is_python_implementation(): + assert ET is not None, "ET must be initialized" + assert pyET is not None, "pyET must be initialized" + return ET is pyET + + +def equal_wrapper(cls): + """Mock cls.__eq__ to check whether it has been called or not. + + The behaviour of cls.__eq__ (side-effects included) is left as is. + """ + eq = cls.__eq__ + return mock.patch.object(cls, "__eq__", autospec=True, wraps=eq) + + def checkwarnings(*filters, quiet=False): def decorator(test): def newtest(*args, **kwargs): @@ -137,9 +155,9 @@ class ModuleTest(unittest.TestCase): def test_sanity(self): # Import sanity. - from xml.etree import ElementTree - from xml.etree import ElementInclude - from xml.etree import ElementPath + from xml.etree import ElementTree # noqa: F401 + from xml.etree import ElementInclude # noqa: F401 + from xml.etree import ElementPath # noqa: F401 def test_all(self): names = ("xml.etree.ElementTree", "_elementtree") @@ -200,43 +218,50 @@ class ElementTreeTest(unittest.TestCase): def serialize_check(self, elem, expected): self.assertEqual(serialize(elem), expected) + def test_constructor(self): + # Test constructor behavior. + + with self.assertRaises(TypeError): + tree = ET.ElementTree("") + with self.assertRaises(TypeError): + tree = ET.ElementTree(ET.ElementTree()) + + def test_setroot(self): + # Test _setroot behavior. + + tree = ET.ElementTree() + element = ET.Element("tag") + tree._setroot(element) + self.assertEqual(tree.getroot().tag, "tag") + self.assertEqual(tree.getroot(), element) + + # Test behavior with an invalid root element + + tree = ET.ElementTree() + with self.assertRaises(TypeError): + tree._setroot("") + with self.assertRaises(TypeError): + tree._setroot(ET.ElementTree()) + with self.assertRaises(TypeError): + tree._setroot(None) + def test_interface(self): # Test element tree interface. - def check_string(string): - len(string) - for char in string: - self.assertEqual(len(char), 1, - msg="expected one-character string, got %r" % char) - new_string = string + "" - new_string = string + " " - string[:0] - - def check_mapping(mapping): - len(mapping) - keys = mapping.keys() - items = mapping.items() - for key in keys: - item = mapping[key] - mapping["key"] = "value" - self.assertEqual(mapping["key"], "value", - msg="expected value string, got %r" % mapping["key"]) - def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") direlem = dir(element) for attr in 'tag', 'attrib', 'text', 'tail': - self.assertTrue(hasattr(element, attr), - msg='no %s member' % attr) + self.assertHasAttr(element, attr) self.assertIn(attr, direlem, msg='no %s visible by dir' % attr) - check_string(element.tag) - check_mapping(element.attrib) + self.assertIsInstance(element.tag, str) + self.assertIsInstance(element.attrib, dict) if element.text is not None: - check_string(element.text) + self.assertIsInstance(element.text, str) if element.tail is not None: - check_string(element.tail) + self.assertIsInstance(element.tail, str) for elem in element: check_element(elem) @@ -252,7 +277,7 @@ def check_element(element): # Make sure all standard element methods exist. def check_method(method): - self.assertTrue(hasattr(method, '__call__'), + self.assertHasAttr(method, '__call__', msg="%s not callable" % method) check_method(element.append) @@ -313,8 +338,6 @@ def test_set_attribute(self): element.attrib = {'A': 'B', 'C': 'D'} self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simpleops(self): # Basic method sanity checks. @@ -347,9 +370,9 @@ def test_simpleops(self): self.serialize_check(element, '<tag key="value"><subtag /></tag>') # 4 element.remove(subelement) self.serialize_check(element, '<tag key="value" />') # 5 - with self.assertRaises(ValueError) as cm: + with self.assertRaisesRegex(ValueError, + r'Element\.remove\(.+\): element not found'): element.remove(subelement) - self.assertEqual(str(cm.exception), 'list.remove(x): x not in list') self.serialize_check(element, '<tag key="value" />') # 6 element[0:0] = [subelement, subelement, subelement] self.serialize_check(element[1], '<subtag />') @@ -359,8 +382,6 @@ def test_simpleops(self): self.serialize_check(element, '<tag key="value"><subtag /><subtag /></tag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cdata(self): # Test CDATA handling (etc). @@ -371,8 +392,7 @@ def test_cdata(self): self.serialize_check(ET.XML("<tag><![CDATA[hello]]></tag>"), '<tag>hello</tag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_file_init(self): stringfile = io.BytesIO(SAMPLE_XML.encode("utf-8")) tree = ET.ElementTree(file=stringfile) @@ -384,14 +404,13 @@ def test_file_init(self): self.assertEqual(tree.find("element/../empty-element").tag, 'empty-element') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_path_cache(self): # Check that the path cache behaves sanely. from xml.etree import ElementPath elem = ET.XML(SAMPLE_XML) + ElementPath._cache.clear() for i in range(10): ET.ElementTree(elem).find('./'+str(i)) cache_len_10 = len(ElementPath._cache) for i in range(10): ET.ElementTree(elem).find('./'+str(i)) @@ -401,8 +420,6 @@ def test_path_cache(self): for i in range(600): ET.ElementTree(elem).find('./'+str(i)) self.assertLess(len(ElementPath._cache), 500) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copy(self): # Test copy handling (etc). @@ -415,8 +432,6 @@ def test_copy(self): self.serialize_check(e2, '<tag>hello<bar /></tag>') self.serialize_check(e3, '<tag>hello<foo /></tag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attrib(self): # Test attribute handling. @@ -493,8 +508,7 @@ def test_makeelement(self): elem[:] = tuple([subelem]) self.serialize_check(elem, '<tag><subtag key="value" /></tag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parsefile(self): # Test parsing from file. @@ -540,8 +554,7 @@ def test_parsefile(self): ' <empty-element />\n' '</root>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parseliteral(self): element = ET.XML("<html><body>text</body></html>") self.assertEqual(ET.tostring(element, encoding='unicode'), @@ -564,128 +577,6 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_iterparse(self): - # Test iterparse interface. - - iterparse = ET.iterparse - - context = iterparse(SIMPLE_XMLFILE) - action, elem = next(context) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - context = iterparse(SIMPLE_NS_XMLFILE) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', '{namespace}element'), - ('end', '{namespace}element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ]) - - events = () - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = () - context = iterparse(SIMPLE_XMLFILE, events=events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = ("start", "end") - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('start', 'root'), - ('start', 'element'), - ('end', 'element'), - ('start', 'element'), - ('end', 'element'), - ('start', 'empty-element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - - events = ("start", "end", "start-ns", "end-ns") - context = iterparse(SIMPLE_NS_XMLFILE, events) - self.assertEqual([(action, elem.tag) if action in ("start", "end") - else (action, elem) - for action, elem in context], [ - ('start-ns', ('', 'namespace')), - ('start', '{namespace}root'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}empty-element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ('end-ns', None), - ]) - - events = ('start-ns', 'end-ns') - context = iterparse(io.StringIO(r"<root xmlns=''/>"), events) - res = [action for action, elem in context] - self.assertEqual(res, ['start-ns', 'end-ns']) - - events = ("start", "end", "bogus") - with open(SIMPLE_XMLFILE, "rb") as f: - with self.assertRaises(ValueError) as cm: - iterparse(f, events) - self.assertFalse(f.closed) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError) as cm: - iterparse(SIMPLE_XMLFILE, events) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - del cm - - source = io.BytesIO( - b"<?xml version='1.0' encoding='iso-8859-1'?>\n" - b"<body xmlns='http://&#233;ffbot.org/ns'\n" - b" xmlns:cl\xe9='http://effbot.org/ns'>text</body>\n") - events = ("start-ns",) - context = iterparse(source, events) - self.assertEqual([(action, elem) for action, elem in context], [ - ('start-ns', ('', 'http://\xe9ffbot.org/ns')), - ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), - ]) - - source = io.StringIO("<document />junk") - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - - self.addCleanup(os_helper.unlink, TESTFN) - with open(TESTFN, "wb") as f: - f.write(b"<document />junk") - it = iterparse(TESTFN) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - del cm, it - - # Not exhausting the iterator still closes the resource (bpo-43292) - with warnings_helper.check_no_resource_warning(self): - it = iterparse(TESTFN) - del it - - with self.assertRaises(FileNotFoundError): - iterparse("nonexistent") - def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -703,8 +594,7 @@ def test_writefile(self): elem[0] = ET.PI("key", "value") self.serialize_check(elem, 'text<?key value?><subtag>subtext</subtag>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_custom_builder(self): # Test parser w. custom builder. @@ -766,8 +656,7 @@ def end_ns(self, prefix): ('end-ns', ''), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_custom_builder_only_end_ns(self): class Builder(list): def end_ns(self, prefix): @@ -790,8 +679,6 @@ def end_ns(self, prefix): ('end-ns', ''), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_initialize_parser_without_target(self): # Explicit None parser = ET.XMLParser(target=None) @@ -801,8 +688,7 @@ def test_initialize_parser_without_target(self): parser2 = ET.XMLParser() self.assertIsInstance(parser2.target, ET.TreeBuilder) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_children(self): # Test Element children iteration @@ -840,16 +726,12 @@ def test_children(self): elem.clear() self.assertEqual(list(elem), []) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_writestring(self): elem = ET.XML("<html><body>text</body></html>") self.assertEqual(ET.tostring(elem), b'<html><body>text</body></html>') elem = ET.fromstring("<html><body>text</body></html>") self.assertEqual(ET.tostring(elem), b'<html><body>text</body></html>') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_indent(self): elem = ET.XML("<root></root>") ET.indent(elem) @@ -894,8 +776,6 @@ def test_indent(self): b'</html>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_indent_space(self): elem = ET.XML("<html><body><p>pre<br/>post</p><p>text</p></body></html>") ET.indent(elem, space='\t') @@ -921,8 +801,6 @@ def test_indent_space(self): b'</html>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_indent_space_caching(self): elem = ET.XML("<html><body><p>par</p><p>text</p><p><br/></p><p /></body></html>") ET.indent(elem) @@ -939,8 +817,6 @@ def test_indent_space_caching(self): len({id(el.tail) for el in elem.iter()}), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_indent_level(self): elem = ET.XML("<html><body><p>pre<br/>post</p><p>text</p></body></html>") with self.assertRaises(ValueError): @@ -973,8 +849,6 @@ def test_indent_level(self): b' </html>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostring_default_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -986,8 +860,6 @@ def test_tostring_default_namespace(self): '<body xmlns="http://effbot.org/ns"><tag /></body>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostring_default_namespace_different_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -995,16 +867,12 @@ def test_tostring_default_namespace_different_namespace(self): '<ns1:body xmlns="foobar" xmlns:ns1="http://effbot.org/ns"><ns1:tag /></ns1:body>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostring_default_namespace_original_no_namespace(self): elem = ET.XML('<body><tag/></body>') EXPECTED_MSG = '^cannot use non-qualified names with default_namespace option$' with self.assertRaisesRegex(ValueError, EXPECTED_MSG): ET.tostring(elem, encoding='unicode', default_namespace='foobar') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostring_no_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1012,8 +880,6 @@ def test_tostring_no_xml_declaration(self): '<body><tag /></body>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostring_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1021,8 +887,6 @@ def test_tostring_xml_declaration(self): b"<?xml version='1.0' encoding='utf8'?>\n<body><tag /></body>" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostring_xml_declaration_unicode_encoding(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1030,8 +894,6 @@ def test_tostring_xml_declaration_unicode_encoding(self): "<?xml version='1.0' encoding='utf-8'?>\n<body><tag /></body>" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostring_xml_declaration_cases(self): elem = ET.XML('<body><tag>ø</tag></body>') TESTCASES = [ @@ -1076,8 +938,6 @@ def test_tostring_xml_declaration_cases(self): expected_retval ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostringlist_default_namespace(self): elem = ET.XML('<body xmlns="http://effbot.org/ns"><tag/></body>') self.assertEqual( @@ -1089,8 +949,6 @@ def test_tostringlist_default_namespace(self): '<body xmlns="http://effbot.org/ns"><tag /></body>' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostringlist_xml_declaration(self): elem = ET.XML('<body><tag/></body>') self.assertEqual( @@ -1110,8 +968,7 @@ def test_tostringlist_xml_declaration(self): self.assertRegex(stringlist[0], r"^<\?xml version='1.0' encoding='.+'?>") self.assertEqual(['<body', '>', '<tag', ' />', '</body>'], stringlist[1:]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoding(self): def check(encoding, body=''): xml = ("<?xml version='1.0' encoding='%s'?><xml>%s</xml>" % @@ -1171,8 +1028,6 @@ def bxml(encoding): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_methods(self): # Test serialization methods. @@ -1188,8 +1043,6 @@ def test_methods(self): '<html><link><script>1 < 2</script></html>\n') self.assertEqual(serialize(e, method="text"), '1 < 2\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue18347(self): e = ET.XML('<html><CamelCase>text</CamelCase></html>') self.assertEqual(serialize(e), @@ -1197,8 +1050,7 @@ def test_issue18347(self): self.assertEqual(serialize(e, method="html"), '<html><CamelCase>text</CamelCase></html>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_entity(self): # Test entity handling. @@ -1236,8 +1088,7 @@ def test_entity(self): self.assertEqual(str(cm.exception), 'undefined entity &entity;: line 4, column 10') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_namespace(self): # Test namespace issues. @@ -1336,8 +1187,6 @@ def test_qname(self): self.assertNotEqual(q1, 'ns:tag') self.assertEqual(q1, '{ns}tag') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_doctype_public(self): # Test PUBLIC doctype. @@ -1404,8 +1253,6 @@ def check(p, expected, namespaces=None): {'': 'http://www.w3.org/2001/XMLSchema', 'ns': 'http://www.w3.org/2001/XMLSchema'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_processinginstruction(self): # Test ProcessingInstruction directly @@ -1422,13 +1269,12 @@ def test_processinginstruction(self): b"<?xml version='1.0' encoding='latin-1'?>\n" b"<?test <testing&>\xe3?>") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_html_empty_elems_serialization(self): # issue 15970 # from http://www.w3.org/TR/html401/index/elements.html - for element in ['AREA', 'BASE', 'BASEFONT', 'BR', 'COL', 'FRAME', 'HR', - 'IMG', 'INPUT', 'ISINDEX', 'LINK', 'META', 'PARAM']: + for element in ['AREA', 'BASE', 'BASEFONT', 'BR', 'COL', 'EMBED', 'FRAME', + 'HR', 'IMG', 'INPUT', 'ISINDEX', 'LINK', 'META', 'PARAM', + 'SOURCE', 'TRACK', 'WBR']: for elem in [element, element.lower()]: expected = '<%s>' % elem serialized = serialize(ET.XML('<%s />' % elem), method='html') @@ -1453,8 +1299,7 @@ def test_tree_write_attribute_order(self): self.assertEqual(serialize(root, method='html'), '<cirriculum status="public" company="example"></cirriculum>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_attlist_default(self): # Test default attribute values; See BPO 42151. root = ET.fromstring(ATTLIST_XML) @@ -1462,14 +1307,253 @@ def test_attlist_default(self): {'{http://www.w3.org/XML/1998/namespace}lang': 'eng'}) +class IterparseTest(unittest.TestCase): + # Test iterparse interface. + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_basic(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(it.root) + action, elem = next(it) + self.assertIsNone(it.root) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + it.close() + + it = iterparse(SIMPLE_NS_XMLFILE) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', '{namespace}element'), + ('end', '{namespace}element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ]) + it.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_external_file(self): + with open(SIMPLE_XMLFILE, 'rb') as source: + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_events(self): + iterparse = ET.iterparse + + events = () + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = () + it = iterparse(SIMPLE_XMLFILE, events=events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = ("start", "end") + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'root'), + ('start', 'element'), + ('end', 'element'), + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + it.close() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_namespace_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "start-ns", "end-ns") + it = iterparse(SIMPLE_NS_XMLFILE, events) + self.assertEqual([(action, elem.tag) if action in ("start", "end") + else (action, elem) + for action, elem in it], [ + ('start-ns', ('', 'namespace')), + ('start', '{namespace}root'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}empty-element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ('end-ns', None), + ]) + it.close() + + events = ('start-ns', 'end-ns') + it = iterparse(io.BytesIO(br"<root xmlns=''/>"), events) + res = [action for action, elem in it] + self.assertEqual(res, ['start-ns', 'end-ns']) + it.close() + + def test_unknown_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "bogus") + with open(SIMPLE_XMLFILE, "rb") as f: + with self.assertRaises(ValueError) as cm: + iterparse(f, events) + self.assertFalse(f.closed) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError) as cm: + iterparse(SIMPLE_XMLFILE, events) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + del cm + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_non_utf8(self): + source = io.BytesIO( + b"<?xml version='1.0' encoding='iso-8859-1'?>\n" + b"<body xmlns='http://&#233;ffbot.org/ns'\n" + b" xmlns:cl\xe9='http://effbot.org/ns'>text</body>\n") + events = ("start-ns",) + it = ET.iterparse(source, events) + self.assertEqual([(action, elem) for action, elem in it], [ + ('start-ns', ('', 'http://\xe9ffbot.org/ns')), + ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), + ]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_parsing_error(self): + source = io.BytesIO(b"<document />junk") + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + + def test_nonexistent_file(self): + with self.assertRaises(FileNotFoundError): + ET.iterparse("nonexistent") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_resource_warnings_not_exhausted(self): + # Not exhausting the iterator still closes the underlying file (bpo-43292) + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_resource_warnings_failed_iteration(self): + self.addCleanup(os_helper.unlink, TESTFN) + with open(TESTFN, "wb") as f: + f.write(b"<document />junk") + + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + + def test_resource_warnings_exhausted(self): + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + list(it) + del it + gc_collect() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_close_not_exhausted(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + def test_close_exhausted(self): + iterparse = ET.iterparse + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + class XMLPullParserTest(unittest.TestCase): - def _feed(self, parser, data, chunk_size=None): + def _feed(self, parser, data, chunk_size=None, flush=False): if chunk_size is None: parser.feed(data) else: for i in range(0, len(data), chunk_size): parser.feed(data[i:i+chunk_size]) + if flush: + parser.flush() def assert_events(self, parser, expected, max_events=None): self.assertEqual( @@ -1487,33 +1571,41 @@ def assert_event_tags(self, parser, expected, max_events=None): self.assertEqual([(action, elem.tag) for action, elem in events], expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_simple_xml(self): - for chunk_size in (None, 1, 5): - with self.subTest(chunk_size=chunk_size): - parser = ET.XMLPullParser() - self.assert_event_tags(parser, []) - self._feed(parser, "<!-- comment -->\n", chunk_size) - self.assert_event_tags(parser, []) - self._feed(parser, - "<root>\n <element key='value'>text</element", - chunk_size) - self.assert_event_tags(parser, []) - self._feed(parser, ">\n", chunk_size) - self.assert_event_tags(parser, [('end', 'element')]) - self._feed(parser, "<element>text</element>tail\n", chunk_size) - self._feed(parser, "<empty-element/>\n", chunk_size) - self.assert_event_tags(parser, [ - ('end', 'element'), - ('end', 'empty-element'), - ]) - self._feed(parser, "</root>\n", chunk_size) - self.assert_event_tags(parser, [('end', 'root')]) - self.assertIsNone(parser.close()) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_simple_xml(self, chunk_size=None, flush=False): + parser = ET.XMLPullParser() + self.assert_event_tags(parser, []) + self._feed(parser, "<!-- comment -->\n", chunk_size, flush) + self.assert_event_tags(parser, []) + self._feed(parser, + "<root>\n <element key='value'>text</element", + chunk_size, flush) + self.assert_event_tags(parser, []) + self._feed(parser, ">\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'element')]) + self._feed(parser, "<element>text</element>tail\n", chunk_size, flush) + self._feed(parser, "<empty-element/>\n", chunk_size, flush) + self.assert_event_tags(parser, [ + ('end', 'element'), + ('end', 'empty-element'), + ]) + self._feed(parser, "</root>\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'root')]) + self.assertIsNone(parser.close()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_simple_xml_chunk_1(self): + self.test_simple_xml(chunk_size=1, flush=True) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_simple_xml_chunk_5(self): + self.test_simple_xml(chunk_size=5, flush=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_simple_xml_chunk_22(self): + self.test_simple_xml(chunk_size=22) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_feed_while_iterating(self): parser = ET.XMLPullParser() it = parser.read_events() @@ -1526,8 +1618,7 @@ def test_feed_while_iterating(self): with self.assertRaises(StopIteration): next(it) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_xml_with_ns(self): parser = ET.XMLPullParser() self.assert_event_tags(parser, []) @@ -1549,8 +1640,7 @@ def test_simple_xml_with_ns(self): self.assert_event_tags(parser, [('end', '{namespace}root')]) self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events(self): parser = ET.XMLPullParser(events=('start-ns', 'end-ns')) self._feed(parser, "<!-- comment -->\n") @@ -1566,8 +1656,7 @@ def test_ns_events(self): self.assertEqual(list(parser.read_events()), [('end-ns', None)]) self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events_start(self): parser = ET.XMLPullParser(events=('start-ns', 'start', 'end')) self._feed(parser, "<tag xmlns='abc' xmlns:p='xyz'>\n") @@ -1591,8 +1680,7 @@ def test_ns_events_start(self): ('end', '{abc}tag'), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events_start_end(self): parser = ET.XMLPullParser(events=('start-ns', 'start', 'end', 'end-ns')) self._feed(parser, "<tag xmlns='abc' xmlns:p='xyz'>\n") @@ -1620,8 +1708,7 @@ def test_ns_events_start_end(self): ('end-ns', None), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events(self): parser = ET.XMLPullParser(events=()) self._feed(parser, "<root/>\n") @@ -1668,8 +1755,7 @@ def test_events(self): self._feed(parser, "</root>") self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_comment(self): parser = ET.XMLPullParser(events=('start', 'comment', 'end')) self._feed(parser, "<!-- text here -->\n") @@ -1689,8 +1775,7 @@ def test_events_comment(self): self._feed(parser, "<!-- text here -->\n") self.assert_events(parser, [('comment', (ET.Comment, ' text here '))]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_pi(self): parser = ET.XMLPullParser(events=('start', 'pi', 'end')) self._feed(parser, "<?pitarget?>\n") @@ -1699,8 +1784,6 @@ def test_events_pi(self): self._feed(parser, "<?pitarget some text ?>\n") self.assert_events(parser, [('pi', (ET.PI, 'pitarget some text '))]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_events_sequence(self): # Test that events can be some sequence that's not just a tuple or list eventset = {'end', 'start'} @@ -1720,12 +1803,64 @@ def __next__(self): self._feed(parser, "<foo>bar</foo>") self.assert_event_tags(parser, [('start', 'foo'), ('end', 'foo')]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + with self.assertRaisesRegex(ValueError, "unknown event 'bogus'"): + ET.XMLPullParser(events=(x.decode() for x in (b'start', b'end', b'bogus'))) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + def test_flush_reparse_deferral_enabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in ("<doc", ">"): + parser.feed(chunk) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + parser.feed("</doc>") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_flush_reparse_deferral_disabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in ("<doc", ">"): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + if not ET is pyET: + self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' + 'methods not available in C') + parser._parser._parser.SetReparseDeferralEnabled(False) + self.assert_event_tags(parser, []) # i.e. no elements started + + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("</doc>") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) # # xinclude tests (samples from appendix C of the xinclude specification) @@ -1882,8 +2017,7 @@ def _my_loader(self, href, parse): else: return None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_default(self): from xml.etree import ElementInclude doc = self.xinclude_loader('default.xml') @@ -1898,8 +2032,7 @@ def test_xinclude_default(self): '</root>\n' '</document>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude(self): from xml.etree import ElementInclude @@ -1964,8 +2097,7 @@ def test_xinclude(self): ' </ns0:include>\n' '</div>') # C5 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_repeated(self): from xml.etree import ElementInclude @@ -1973,8 +2105,7 @@ def test_xinclude_repeated(self): ElementInclude.include(document, self.xinclude_loader) self.assertEqual(1+4*2, len(document.findall(".//p"))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_failures(self): from xml.etree import ElementInclude @@ -2079,8 +2210,7 @@ def check(elem): elem.set("123", 123) check(elem) # attribute value - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit25(self): # typo in ElementTree.findtext @@ -2089,8 +2219,6 @@ def test_bug_xmltoolkit25(self): self.assertEqual(tree.findtext("tag"), 'text') self.assertEqual(tree.findtext("section/tag"), 'subtext') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_xmltoolkit28(self): # .//tag causes exceptions @@ -2098,8 +2226,6 @@ def test_bug_xmltoolkit28(self): self.assertEqual(summarize_list(tree.findall(".//thead")), []) self.assertEqual(summarize_list(tree.findall(".//tbody")), ['tbody']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_xmltoolkitX1(self): # dump() doesn't flush the output buffer @@ -2108,8 +2234,7 @@ def test_bug_xmltoolkitX1(self): ET.dump(tree) self.assertEqual(stdout.getvalue(), '<doc><table><tbody /></table></doc>\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit39(self): # non-ascii element and attribute names doesn't work @@ -2135,8 +2260,6 @@ def test_bug_xmltoolkit39(self): self.assertEqual(ET.tostring(tree, "utf-8"), b'<tag \xc3\xa4ttr="v\xc3\xa4lue" />') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_xmltoolkit54(self): # problems handling internally defined entities @@ -2146,8 +2269,7 @@ def test_bug_xmltoolkit54(self): b'<doc>&#33328;</doc>') self.assertEqual(serialize(e), '<doc>\u8230</doc>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit55(self): # make sure we're reporting the first error, not the last @@ -2157,8 +2279,6 @@ def test_bug_xmltoolkit55(self): self.assertEqual(str(cm.exception), 'undefined entity &ldots;: line 1, column 36') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_xmltoolkit60(self): # Handle crash in stream source. @@ -2168,8 +2288,7 @@ def read(self, x): self.assertRaises(OSError, ET.parse, ExceptionFile()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit62(self): # Don't crash when using custom entities. @@ -2187,8 +2306,6 @@ def test_bug_xmltoolkit62(self): self.assertEqual(t.find('.//paragraph').text, 'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.gettrace(), "Skips under coverage.") def test_bug_xmltoolkit63(self): # Check reference leak. @@ -2204,8 +2321,7 @@ def xmltoolkit63(): xmltoolkit63() self.assertEqual(sys.getrefcount(None), count) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_200708_newline(self): # Preserve newlines in attributes. @@ -2217,8 +2333,6 @@ def test_bug_200708_newline(self): self.assertEqual(ET.tostring(ET.XML(ET.tostring(e))), b'<SomeTag text="def _f():&#10; return 3&#10;" />') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_200708_close(self): # Test default builder. parser = ET.XMLParser() # default @@ -2256,8 +2370,6 @@ def test_bug_200709_default_namespace(self): self.assertEqual(str(cm.exception), 'cannot use non-qualified names with default_namespace option') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_200709_register_namespace(self): e = ET.Element("{http://namespace.invalid/does/not/exist/}title") self.assertEqual(ET.tostring(e), @@ -2313,8 +2425,6 @@ def test_bug_1534630(self): e = bob.close() self.assertEqual(serialize(e), '<tag />') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue6233(self): e = ET.XML(b"<?xml version='1.0' encoding='utf-8'?>" b'<body>t\xc3\xa3g</body>') @@ -2327,8 +2437,7 @@ def test_issue6233(self): b"<?xml version='1.0' encoding='ascii'?>\n" b'<body>t&#227;g</body>') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue6565(self): elem = ET.XML("<body><tag/></body>") self.assertEqual(summarize_list(elem), ['tag']) @@ -2374,8 +2483,7 @@ def __bool__(self): self.assertIsInstance(e[0].tail, str) self.assertEqual(e[0].tail, 'changed') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lost_elem(self): # Issue #25902: Borrowed element can disappear class Tag: @@ -2401,8 +2509,7 @@ def check_expat224_utf8_bug(self, text): root = ET.XML(xml) self.assertEqual(root.get('b'), text.decode('utf-8')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expat224_utf8_bug(self): # bpo-31170: Expat 2.2.3 had a bug in its UTF-8 decoder. # Check that Expat 2.2.4 fixed the bug. @@ -2415,8 +2522,7 @@ def test_expat224_utf8_bug(self): text = b'x' + b'\xc3\xa0' * 1024 self.check_expat224_utf8_bug(text) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expat224_utf8_bug_file(self): with open(UTF8_BUG_XMLFILE, 'rb') as fp: raw = fp.read() @@ -2434,6 +2540,22 @@ def test_39495_treebuilder_start(self): self.assertRaises(TypeError, ET.TreeBuilder().start, "tag") self.assertRaises(TypeError, ET.TreeBuilder().start, "tag", None) + def test_issue123213_correct_extend_exception(self): + # Does not hide the internal exception when extending the element + self.assertRaises(ZeroDivisionError, ET.Element('tag').extend, + (1/0 for i in range(2))) + + # Still raises the TypeError when extending with a non-iterable + self.assertRaises(TypeError, ET.Element('tag').extend, None) + + # Preserves the TypeError message when extending with a generator + def f(): + raise TypeError("mymessage") + + self.assertRaisesRegex( + TypeError, 'mymessage', + ET.Element('tag').extend, (f() for i in range(2))) + # -------------------------------------------------------------------- @@ -2468,35 +2590,6 @@ def test___init__(self): self.assertIsNot(element_foo.attrib, attrib) self.assertNotEqual(element_foo.attrib, attrib) - def test_copy(self): - # Only run this test if Element.copy() is defined. - if "copy" not in dir(ET.Element): - raise unittest.SkipTest("Element.copy() not present") - - element_foo = ET.Element("foo", { "zix": "wyp" }) - element_foo.append(ET.Element("bar", { "baz": "qix" })) - - with self.assertWarns(DeprecationWarning): - element_foo2 = element_foo.copy() - - # elements are not the same - self.assertIsNot(element_foo2, element_foo) - - # string attributes are equal - self.assertEqual(element_foo2.tag, element_foo.tag) - self.assertEqual(element_foo2.text, element_foo.text) - self.assertEqual(element_foo2.tail, element_foo.tail) - - # number of children is the same - self.assertEqual(len(element_foo2), len(element_foo)) - - # children are the same - for (child1, child2) in itertools.zip_longest(element_foo, element_foo2): - self.assertIs(child1, child2) - - # attrib is a copy - self.assertEqual(element_foo2.attrib, element_foo.attrib) - def test___copy__(self): element_foo = ET.Element("foo", { "zix": "wyp" }) element_foo.append(ET.Element("bar", { "baz": "qix" })) @@ -2579,8 +2672,7 @@ def __deepcopy__(self, memo): e[:] = [E('bar')] self.assertRaises(TypeError, copy.deepcopy, e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cyclic_gc(self): class Dummy: pass @@ -2653,8 +2745,6 @@ def test_pickle(self): self.assertEqual(len(e2), 2) self.assertEqualElements(e, e2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickle_issue18997(self): for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): for dumper, loader in product(self.modules, repeat=2): @@ -2662,8 +2752,7 @@ def test_pickle_issue18997(self): <group><dogs>4</dogs> </group>""" e1 = dumper.fromstring(XMLTEXT) - if hasattr(e1, '__getstate__'): - self.assertEqual(e1.__getstate__()['tag'], 'group') + self.assertEqual(e1.__getstate__()['tag'], 'group') e2 = self.pickleRoundTrip(e1, 'xml.etree.ElementTree', dumper, loader, proto) self.assertEqual(e2.tag, 'group') @@ -2671,6 +2760,7 @@ def test_pickle_issue18997(self): class BadElementTest(ElementTestCase, unittest.TestCase): + def test_extend_mutable_list(self): class X: @property @@ -2709,20 +2799,170 @@ class Y(X, ET.Element): e = ET.Element('foo') e.extend(L) - @unittest.skip("TODO: RUSTPYTHON, hangs") - def test_remove_with_mutating(self): - class X(ET.Element): + def test_remove_with_clear_assume_missing(self): + # gh-126033: Check that a concurrent clear() for an assumed-to-be + # missing element does not make the interpreter crash. + self.do_test_remove_with_clear(raises=True) + + def test_remove_with_clear_assume_existing(self): + # gh-126033: Check that a concurrent clear() for an assumed-to-be + # existing element does not make the interpreter crash. + self.do_test_remove_with_clear(raises=False) + + def do_test_remove_with_clear(self, *, raises): + + # Until the discrepency between "del root[:]" and "root.clear()" is + # resolved, we need to keep two tests. Previously, using "del root[:]" + # did not crash with the reproducer of gh-126033 while "root.clear()" + # did. + + class E(ET.Element): + """Local class to be able to mock E.__eq__ for introspection.""" + + class X(E): def __eq__(self, o): - del e[:] - return False - e = ET.Element('foo') - e.extend([X('bar')]) - self.assertRaises(ValueError, e.remove, ET.Element('baz')) + del root[:] + return not raises - e = ET.Element('foo') - e.extend([ET.Element('bar')]) - self.assertRaises(ValueError, e.remove, X('baz')) + class Y(E): + def __eq__(self, o): + root.clear() + return not raises + + if raises: + get_checker_context = lambda: self.assertRaises(ValueError) + else: + get_checker_context = nullcontext + + self.assertIs(E.__eq__, object.__eq__) + + for Z, side_effect in [(X, 'del root[:]'), (Y, 'root.clear()')]: + self.enterContext(self.subTest(side_effect=side_effect)) + + # test removing R() from [U()] + for R, U, description in [ + (E, Z, "remove missing E() from [Z()]"), + (Z, E, "remove missing Z() from [E()]"), + (Z, Z, "remove missing Z() from [Z()]"), + ]: + with self.subTest(description): + root = E('top') + root.extend([U('one')]) + with get_checker_context(): + root.remove(R('missing')) + + # test removing R() from [U(), V()] + cases = self.cases_for_remove_missing_with_mutations(E, Z) + for R, U, V, description in cases: + with self.subTest(description): + root = E('top') + root.extend([U('one'), V('two')]) + with get_checker_context(): + root.remove(R('missing')) + + # Test removing root[0] from [Z()]. + # + # Since we call root.remove() with root[0], Z.__eq__() + # will not be called (we branch on the fast Py_EQ path). + with self.subTest("remove root[0] from [Z()]"): + root = E('top') + root.append(Z('rem')) + with equal_wrapper(E) as f, equal_wrapper(Z) as g: + root.remove(root[0]) + f.assert_not_called() + g.assert_not_called() + + # Test removing root[1] (of type R) from [U(), R()]. + is_special = is_python_implementation() and raises and Z is Y + if is_python_implementation() and raises and Z is Y: + # In pure Python, using root.clear() sets the children + # list to [] without calling list.clear(). + # + # For this reason, the call to root.remove() first + # checks root[0] and sets the children list to [] + # since either root[0] or root[1] is an evil element. + # + # Since checking root[1] still uses the old reference + # to the children list, PyObject_RichCompareBool() branches + # to the fast Py_EQ path and Y.__eq__() is called exactly + # once (when checking root[0]). + continue + else: + cases = self.cases_for_remove_existing_with_mutations(E, Z) + for R, U, description in cases: + with self.subTest(description): + root = E('top') + root.extend([U('one'), R('rem')]) + with get_checker_context(): + root.remove(root[1]) + + def test_remove_with_mutate_root_assume_missing(self): + # gh-126033: Check that a concurrent mutation for an assumed-to-be + # missing element does not make the interpreter crash. + self.do_test_remove_with_mutate_root(raises=True) + + def test_remove_with_mutate_root_assume_existing(self): + # gh-126033: Check that a concurrent mutation for an assumed-to-be + # existing element does not make the interpreter crash. + self.do_test_remove_with_mutate_root(raises=False) + + def do_test_remove_with_mutate_root(self, *, raises): + E = ET.Element + + class Z(E): + def __eq__(self, o): + del root[0] + return not raises + if raises: + get_checker_context = lambda: self.assertRaises(ValueError) + else: + get_checker_context = nullcontext + + # test removing R() from [U(), V()] + cases = self.cases_for_remove_missing_with_mutations(E, Z) + for R, U, V, description in cases: + with self.subTest(description): + root = E('top') + root.extend([U('one'), V('two')]) + with get_checker_context(): + root.remove(R('missing')) + + # test removing root[1] (of type R) from [U(), R()] + cases = self.cases_for_remove_existing_with_mutations(E, Z) + for R, U, description in cases: + with self.subTest(description): + root = E('top') + root.extend([U('one'), R('rem')]) + with get_checker_context(): + root.remove(root[1]) + + def cases_for_remove_missing_with_mutations(self, E, Z): + # Cases for removing R() from [U(), V()]. + # The case U = V = R = E is not interesting as there is no mutation. + for U, V in [(E, Z), (Z, E), (Z, Z)]: + description = (f"remove missing {E.__name__}() from " + f"[{U.__name__}(), {V.__name__}()]") + yield E, U, V, description + + for U, V in [(E, E), (E, Z), (Z, E), (Z, Z)]: + description = (f"remove missing {Z.__name__}() from " + f"[{U.__name__}(), {V.__name__}()]") + yield Z, U, V, description + + def cases_for_remove_existing_with_mutations(self, E, Z): + # Cases for removing root[1] (of type R) from [U(), R()]. + # The case U = R = E is not interesting as there is no mutation. + for U, R, description in [ + (E, Z, "remove root[1] from [E(), Z()]"), + (Z, E, "remove root[1] from [Z(), E()]"), + (Z, Z, "remove root[1] from [Z(), Z()]"), + ]: + description = (f"remove root[1] (of type {R.__name__}) " + f"from [{U.__name__}(), {R.__name__}()]") + yield R, U, description + + @support.infinite_recursion(25) def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo') @@ -2770,32 +3010,72 @@ def __del__(self): elem = b.close() self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') - def test_subscr(self): - # Issue #27863 + def test_subscr_with_clear(self): + # See https://github.com/python/cpython/issues/143200. + self.do_test_subscr_with_mutating_slice(use_clear_method=True) + + def test_subscr_with_delete(self): + # See https://github.com/python/cpython/issues/72050. + self.do_test_subscr_with_mutating_slice(use_clear_method=False) + + def do_test_subscr_with_mutating_slice(self, *, use_clear_method): class X: + def __init__(self, i=0): + self.i = i def __index__(self): - del e[:] - return 1 + if use_clear_method: + e.clear() + else: + del e[:] + return self.i - e = ET.Element('elem') - e.append(ET.Element('child')) - e[:X()] # shouldn't crash + for s in self.get_mutating_slices(X, 10): + with self.subTest(s): + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[s] # shouldn't crash - e.append(ET.Element('child')) - e[0:10:X()] # shouldn't crash + def test_ass_subscr_with_mutating_slice(self): + # See https://github.com/python/cpython/issues/72050 + # and https://github.com/python/cpython/issues/143200. - def test_ass_subscr(self): - # Issue #27863 class X: + def __init__(self, i=0): + self.i = i def __index__(self): e[:] = [] - return 1 + return self.i + + for s in self.get_mutating_slices(X, 10): + with self.subTest(s): + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[s] = [] # shouldn't crash + + def get_mutating_slices(self, index_class, n_children): + self.assertGreaterEqual(n_children, 10) + return [ + slice(index_class(), None, None), + slice(index_class(2), None, None), + slice(None, index_class(), None), + slice(None, index_class(2), None), + slice(0, 2, index_class(1)), + slice(0, 2, index_class(2)), + slice(0, n_children, index_class(1)), + slice(0, n_children, index_class(2)), + slice(0, 2 * n_children, index_class(1)), + slice(0, 2 * n_children, index_class(2)), + ] - e = ET.Element('elem') - for _ in range(10): - e.insert(0, ET.Element('child')) + def test_ass_subscr_with_mutating_iterable_value(self): + class V: + def __iter__(self): + e.clear() + return iter([ET.Element('a'), ET.Element('b')]) - e[0:10:X()] = [] # shouldn't crash + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[:] = V() def test_treebuilder_start(self): # Issue #27863 @@ -2821,21 +3101,83 @@ def element_factory(x, y): del b gc_collect() + def test_deepcopy_clear(self): + # Prevent crashes when __deepcopy__() clears the children list. + # See https://github.com/python/cpython/issues/133009. + class X(ET.Element): + def __deepcopy__(self, memo): + root.clear() + return self + + root = ET.Element('a') + evil = X('x') + root.extend([evil, ET.Element('y')]) + if is_python_implementation(): + # Mutating a list over which we iterate raises an error. + self.assertRaises(RuntimeError, copy.deepcopy, root) + else: + c = copy.deepcopy(root) + # In the C implementation, we can still copy the evil element. + self.assertListEqual(list(c), [evil]) + + def test_deepcopy_grow(self): + # Prevent crashes when __deepcopy__() mutates the children list. + # See https://github.com/python/cpython/issues/133009. + a = ET.Element('a') + b = ET.Element('b') + c = ET.Element('c') + + class X(ET.Element): + def __deepcopy__(self, memo): + root.append(a) + root.append(b) + return self + + root = ET.Element('top') + evil1, evil2 = X('1'), X('2') + root.extend([evil1, c, evil2]) + children = list(copy.deepcopy(root)) + # mock deep copies + self.assertIs(children[0], evil1) + self.assertIs(children[2], evil2) + # true deep copies + self.assertEqual(children[1].tag, c.tag) + self.assertEqual([c.tag for c in children[3:]], + [a.tag, b.tag, a.tag, b.tag]) -class MutatingElementPath(str): + +class MutationDeleteElementPath(str): def __new__(cls, elem, *args): self = str.__new__(cls, *args) self.elem = elem return self + def __eq__(self, o): del self.elem[:] return True -MutatingElementPath.__hash__ = str.__hash__ + + __hash__ = str.__hash__ + + +class MutationClearElementPath(str): + def __new__(cls, elem, *args): + self = str.__new__(cls, *args) + self.elem = elem + return self + + def __eq__(self, o): + self.elem.clear() + return True + + __hash__ = str.__hash__ + class BadElementPath(str): def __eq__(self, o): raise 1/0 -BadElementPath.__hash__ = str.__hash__ + + __hash__ = str.__hash__ + class BadElementPathTest(ElementTestCase, unittest.TestCase): def setUp(self): @@ -2850,9 +3192,11 @@ def tearDown(self): super().tearDown() def test_find_with_mutating(self): - e = ET.Element('foo') - e.extend([ET.Element('bar')]) - e.find(MutatingElementPath(e, 'x')) + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + e.extend([ET.Element('bar')]) + e.find(cls(e, 'x')) def test_find_with_error(self): e = ET.Element('foo') @@ -2863,9 +3207,11 @@ def test_find_with_error(self): pass def test_findtext_with_mutating(self): - e = ET.Element('foo') - e.extend([ET.Element('bar')]) - e.findtext(MutatingElementPath(e, 'x')) + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + e.extend([ET.Element('bar')]) + e.findtext(cls(e, 'x')) def test_findtext_with_error(self): e = ET.Element('foo') @@ -2875,10 +3221,26 @@ def test_findtext_with_error(self): except ZeroDivisionError: pass + def test_findtext_with_falsey_text_attribute(self): + root_elem = ET.Element('foo') + sub_elem = ET.SubElement(root_elem, 'bar') + falsey = ["", 0, False, [], (), {}] + for val in falsey: + sub_elem.text = val + self.assertEqual(root_elem.findtext('./bar'), val) + + def test_findtext_with_none_text_attribute(self): + root_elem = ET.Element('foo') + sub_elem = ET.SubElement(root_elem, 'bar') + sub_elem.text = None + self.assertEqual(root_elem.findtext('./bar'), '') + def test_findall_with_mutating(self): - e = ET.Element('foo') - e.extend([ET.Element('bar')]) - e.findall(MutatingElementPath(e, 'x')) + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + e.extend([ET.Element('bar')]) + e.findall(cls(e, 'x')) def test_findall_with_error(self): e = ET.Element('foo') @@ -2946,8 +3308,7 @@ class MyElement(ET.Element): class ElementFindTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_simple(self): e = ET.XML(SAMPLE_XML) self.assertEqual(e.find('tag').tag, 'tag') @@ -2971,8 +3332,7 @@ def test_find_simple(self): # Issue #16922 self.assertEqual(ET.XML('<tag><empty /></tag>').findtext('empty'), '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_xpath(self): LINEAR_XML = ''' <body> @@ -2995,8 +3355,7 @@ def test_find_xpath(self): self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()-0]') self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()+1]') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall(self): e = ET.XML(SAMPLE_XML) e[2] = ET.XML(SAMPLE_SECTION) @@ -3109,8 +3468,6 @@ def test_findall(self): self.assertEqual(summarize_list(e.findall(".//tag[. = 'subtext']")), ['tag', 'tag']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_test_find_with_ns(self): e = ET.XML(SAMPLE_XML_NS) self.assertEqual(summarize_list(e.findall('tag')), []) @@ -3121,8 +3478,6 @@ def test_test_find_with_ns(self): summarize_list(e.findall(".//{http://effbot.org/ns}tag")), ['{http://effbot.org/ns}tag'] * 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_findall_different_nsmaps(self): root = ET.XML(''' <a xmlns:x="X" xmlns:y="Y"> @@ -3140,8 +3495,6 @@ def test_findall_different_nsmaps(self): self.assertEqual(len(root.findall(".//xx:b", namespaces=nsmap)), 2) self.assertEqual(len(root.findall(".//b", namespaces=nsmap)), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_findall_wildcard(self): root = ET.XML(''' <a xmlns:x="X" xmlns:y="Y"> @@ -3186,15 +3539,12 @@ def test_findall_wildcard(self): self.assertEqual(summarize_list(root.findall(".//{}b")), summarize_list(root.findall(".//b"))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_find(self): e = ET.XML(SAMPLE_XML) with self.assertRaisesRegex(SyntaxError, 'cannot use absolute path'): e.findall('/tag') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_through_ElementTree(self): e = ET.XML(SAMPLE_XML) self.assertEqual(ET.ElementTree(e).find('tag').tag, 'tag') @@ -3214,8 +3564,6 @@ class ElementIterTest(unittest.TestCase): def _ilist(self, elem, tag=None): return summarize_list(elem.iter(tag)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic(self): doc = ET.XML("<html><body>this is a <i>paragraph</i>.</body>..</html>") self.assertEqual(self._ilist(doc), ['html', 'body', 'i']) @@ -3233,8 +3581,7 @@ def test_basic(self): # With an explicit parser too (issue #9708) sourcefile = serialize(doc, to_string=False) parser = ET.XMLParser(target=ET.TreeBuilder()) - self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], - 'end') + self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], 'end') tree = ET.ElementTree(None) self.assertRaises(AttributeError, tree.iter) @@ -3265,8 +3612,6 @@ def test_corners(self): del a[1] self.assertEqual(self._ilist(a), ['a', 'd']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_iter_by_tag(self): doc = ET.XML(''' <document> @@ -3331,8 +3676,6 @@ def _check_sample1_element(self, e): self.assertEqual(child.tail, 'tail') self.assertEqual(child.attrib, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dummy_builder(self): class BaseDummyBuilder: def close(self): @@ -3380,8 +3723,6 @@ def test_treebuilder_pi(self): self.assertEqual(b.pi('target'), (len('target'), None)) self.assertEqual(b.pi('pitarget', ' text '), (len('pitarget'), ' text ')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_late_tail(self): # Issue #37399: The tail of an ignored comment could overwrite the text before it. class TreeBuilderSubclass(ET.TreeBuilder): @@ -3406,8 +3747,7 @@ class TreeBuilderSubclass(ET.TreeBuilder): a = parser.close() self.assertEqual(a.text, "texttail") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_late_tail_mix_pi_comments(self): # Issue #37399: The tail of an ignored comment could overwrite the text before it. # Test appending tails to comments/pis. @@ -3444,16 +3784,12 @@ class TreeBuilderSubclass(ET.TreeBuilder): self.assertEqual(a[0].tail, 'tail') self.assertEqual(a.text, "text\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_treebuilder_elementfactory_none(self): parser = ET.XMLParser(target=ET.TreeBuilder(element_factory=None)) parser.feed(self.sample1) e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass(self): class MyTreeBuilder(ET.TreeBuilder): def foobar(self, x): @@ -3468,8 +3804,6 @@ def foobar(self, x): e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_comment_pi(self): class MyTreeBuilder(ET.TreeBuilder): def foobar(self, x): @@ -3485,8 +3819,6 @@ def foobar(self, x): e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_element_factory(self): lst = [] def myfactory(tag, attrib): @@ -3510,15 +3842,11 @@ def _check_element_factory_class(self, cls): self.assertIsInstance(e, cls) self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_element_factory_subclass(self): class MyElement(ET.Element): pass self._check_element_factory_class(MyElement) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_element_factory_pure_python_subclass(self): # Mimic SimpleTAL's behaviour (issue #16089): both versions of # TreeBuilder should be able to cope with a subclass of the @@ -3532,8 +3860,7 @@ class MyElement(base, ValueError): pass self._check_element_factory_class(MyElement) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype(self): class DoctypeParser: _doctype = None @@ -3551,8 +3878,6 @@ def close(self): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_builder_lookup_errors(self): class RaisingBuilder: def __init__(self, raise_in=None, what=ValueError): @@ -3593,16 +3918,12 @@ def _check_sample_element(self, e): self.assertEqual(e[0].tag, 'line') self.assertEqual(e[0].text, '22') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_constructor_args(self): parser2 = ET.XMLParser(encoding='utf-8', target=ET.TreeBuilder()) parser2.feed(self.sample1) self._check_sample_element(parser2.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass(self): class MyParser(ET.XMLParser): pass @@ -3610,8 +3931,6 @@ class MyParser(ET.XMLParser): parser.feed(self.sample1) self._check_sample_element(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_doctype_warning(self): with warnings.catch_warnings(): warnings.simplefilter('error', DeprecationWarning) @@ -3619,8 +3938,7 @@ def test_doctype_warning(self): parser.feed(self.sample2) parser.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_doctype(self): _doctype = None class MyParserWithDoctype(ET.XMLParser): @@ -3651,8 +3969,6 @@ def doctype(self, name, pubid, system): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_inherited_doctype(self): '''Ensure that ordinary usage is not deprecated (Issue 19176)''' with warnings.catch_warnings(): @@ -3664,8 +3980,7 @@ class MyParserWithoutDoctype(ET.XMLParser): parser.feed(self.sample2) parser.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_string(self): parser = ET.XMLParser(target=ET.TreeBuilder()) parser.feed(self.sample3) @@ -3676,8 +3991,6 @@ def test_parse_string(self): class NamespaceParseTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_find_with_namespace(self): nsmap = {'h': 'hello', 'f': 'foo'} doc = ET.fromstring(SAMPLE_XML_NS_ELEMS) @@ -3836,10 +4149,24 @@ def test_setslice_negative_steps(self): e[1::-sys.maxsize<<64] = [ET.Element('d')] self.assertEqual(self._subelem_tags(e), ['a0', 'd', 'a2', 'a3']) + def test_issue123213_setslice_exception(self): + e = ET.Element('tag') + # Does not hide the internal exception when assigning to the element + with self.assertRaises(ZeroDivisionError): + e[:1] = (1/0 for i in range(2)) + + # Still raises the TypeError when assigning with a non-iterable + with self.assertRaises(TypeError): + e[:1] = None + + # Preserve the original TypeError message when assigning. + def f(): + raise TypeError("mymessage") + + with self.assertRaisesRegex(TypeError, 'mymessage'): + e[:1] = (f() for i in range(2)) class IOTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encoding(self): # Test encoding issues. elem = ET.Element("tag") @@ -3909,8 +4236,6 @@ def test_encoding(self): ("<?xml version='1.0' encoding='%s'?>\n" "<tag key=\"åöö&lt;&gt;\" />" % enc).encode(enc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_filename(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -3918,8 +4243,6 @@ def test_write_to_filename(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>&#248;</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_filename_with_encoding(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -3933,8 +4256,6 @@ def test_write_to_filename_with_encoding(self): b'''<?xml version='1.0' encoding='ISO-8859-1'?>\n''' b'''<site>\xf8</site>''')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_filename_as_unicode(self): self.addCleanup(os_helper.unlink, TESTFN) with open(TESTFN, 'w') as f: @@ -3946,8 +4267,6 @@ def test_write_to_filename_as_unicode(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b"<site>\xc3\xb8</site>") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_text_file(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -3969,8 +4288,6 @@ def test_write_to_text_file(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>\xf8</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_binary_file(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -3980,8 +4297,6 @@ def test_write_to_binary_file(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''<site>&#248;</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_binary_file_with_encoding(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -3999,8 +4314,6 @@ def test_write_to_binary_file_with_encoding(self): b'''<?xml version='1.0' encoding='ISO-8859-1'?>\n''' b'''<site>\xf8</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_binary_file_with_bom(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) @@ -4021,32 +4334,24 @@ def test_write_to_binary_file_with_bom(self): '''<?xml version='1.0' encoding='utf-16'?>\n''' '''<site>\xf8</site>'''.encode("utf-16")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_read_from_stringio(self): tree = ET.ElementTree() stream = io.StringIO('''<?xml version="1.0"?><site></site>''') tree.parse(stream) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_stringio(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) stream = io.StringIO() tree.write(stream, encoding='unicode') self.assertEqual(stream.getvalue(), '''<site>\xf8</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_read_from_bytesio(self): tree = ET.ElementTree() raw = io.BytesIO(b'''<?xml version="1.0"?><site></site>''') tree.parse(raw) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_bytesio(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) raw = io.BytesIO() @@ -4056,8 +4361,6 @@ def test_write_to_bytesio(self): class dummy: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_read_from_user_text_reader(self): stream = io.StringIO('''<?xml version="1.0"?><site></site>''') reader = self.dummy() @@ -4066,8 +4369,6 @@ def test_read_from_user_text_reader(self): tree.parse(reader) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_user_text_writer(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) stream = io.StringIO() @@ -4076,8 +4377,6 @@ def test_write_to_user_text_writer(self): tree.write(writer, encoding='unicode') self.assertEqual(stream.getvalue(), '''<site>\xf8</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_read_from_user_binary_reader(self): raw = io.BytesIO(b'''<?xml version="1.0"?><site></site>''') reader = self.dummy() @@ -4087,8 +4386,6 @@ def test_read_from_user_binary_reader(self): self.assertEqual(tree.getroot().tag, 'site') tree = ET.ElementTree() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_user_binary_writer(self): tree = ET.ElementTree(ET.XML('''<site>\xf8</site>''')) raw = io.BytesIO() @@ -4097,8 +4394,6 @@ def test_write_to_user_binary_writer(self): tree.write(writer) self.assertEqual(raw.getvalue(), b'''<site>&#248;</site>''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_write_to_user_binary_writer_with_bom(self): tree = ET.ElementTree(ET.XML('''<site />''')) raw = io.BytesIO() @@ -4111,8 +4406,6 @@ def test_write_to_user_binary_writer_with_bom(self): '''<?xml version='1.0' encoding='utf-16'?>\n''' '''<site />'''.encode("utf-16")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tostringlist_invariant(self): root = ET.fromstring('<tag>foo</tag>') self.assertEqual( @@ -4122,8 +4415,6 @@ def test_tostringlist_invariant(self): ET.tostring(root, 'utf-16'), b''.join(ET.tostringlist(root, 'utf-16'))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_short_empty_elements(self): root = ET.fromstring('<tag>a<x />b<y></y>c</tag>') self.assertEqual( @@ -4147,15 +4438,13 @@ def _get_error(self, s): except ET.ParseError as e: return e - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_position(self): self.assertEqual(self._get_error('foo').position, (1, 0)) self.assertEqual(self._get_error('<tag>&foo;</tag>').position, (1, 5)) self.assertEqual(self._get_error('foobar<').position, (1, 6)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_code(self): import xml.parsers.expat.errors as ERRORS self.assertEqual(self._get_error('foo').code, @@ -4163,8 +4452,6 @@ def test_error_code(self): class KeywordArgsTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test various issues with keyword arguments passed to ET.Element # constructor and methods def test_issue14818(self): @@ -4201,12 +4488,11 @@ def test_issue14818(self): # -------------------------------------------------------------------- class NoAcceleratorTest(unittest.TestCase): - def setUp(self): - if not pyET: + @classmethod + def setUpClass(cls): + if ET is not pyET: raise unittest.SkipTest('only for the Python version') - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test that the C accelerator was not imported for pyET def test_correct_import_pyET(self): # The type of methods defined in Python code is types.FunctionType, @@ -4215,6 +4501,26 @@ def test_correct_import_pyET(self): self.assertIsInstance(pyET.Element.__init__, types.FunctionType) self.assertIsInstance(pyET.XMLParser.__init__, types.FunctionType) +# -------------------------------------------------------------------- + +class BoolTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_warning(self): + e = ET.fromstring('<a style="new"></a>') + msg = ( + r"Testing an element's truth value will always return True in " + r"future versions. " + r"Use specific 'len\(elem\)' or 'elem is not None' test instead.") + with self.assertWarnsRegex(DeprecationWarning, msg): + result = bool(e) + # Emulate prior behavior for now + self.assertIs(result, False) + + # Element with children + ET.SubElement(e, 'b') + with self.assertWarnsRegex(DeprecationWarning, msg): + new_result = bool(e) + self.assertIs(new_result, True) # -------------------------------------------------------------------- @@ -4228,8 +4534,7 @@ class C14NTest(unittest.TestCase): # # simple roundtrip tests (from c14n.py) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_roundtrip(self): # Basics self.assertEqual(c14n_roundtrip("<doc/>"), '<doc></doc>') @@ -4270,8 +4575,7 @@ def test_simple_roundtrip(self): xml = '<X xmlns="http://nps/a"><Y xmlns:b="http://nsp/b" b:targets="abc,xyz"></Y></X>' self.assertEqual(c14n_roundtrip(xml), xml) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_c14n_exclusion(self): xml = textwrap.dedent("""\ <root xmlns:x="http://example.com/x"> @@ -4352,8 +4656,7 @@ def test_c14n_exclusion(self): # note that this uses generated C14N versions of the standard ET.write # output, not roundtripped C14N (see above). - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xml_c14n2(self): datadir = findfile("c14n-20", subdir="xmltestdata") full_path = partial(os.path.join, datadir) @@ -4456,8 +4759,7 @@ def get_option(config, option_name, default=None): # -------------------------------------------------------------------- - -def test_main(module=None): +def setUpModule(module=None): # When invoked without a module, runs the Python ET tests by loading pyET. # Otherwise, uses the given module as the ET. global pyET @@ -4469,62 +4771,30 @@ def test_main(module=None): global ET ET = module - test_classes = [ - ModuleTest, - ElementSlicingTest, - BasicElementTest, - BadElementTest, - BadElementPathTest, - ElementTreeTest, - IOTest, - ParseErrorTest, - XIncludeTest, - ElementTreeTypeTest, - ElementFindTest, - ElementIterTest, - TreeBuilderTest, - XMLParserTest, - XMLPullParserTest, - BugsTest, - KeywordArgsTest, - C14NTest, - ] - - # These tests will only run for the pure-Python version that doesn't import - # _elementtree. We can't use skipUnless here, because pyET is filled in only - # after the module is loaded. - if pyET is not ET: - test_classes.extend([ - NoAcceleratorTest, - ]) + # don't interfere with subsequent tests + def cleanup(): + global ET, pyET + ET = pyET = None + unittest.addModuleCleanup(cleanup) # Provide default namespace mapping and path cache. from xml.etree import ElementPath nsmap = ET.register_namespace._namespace_map # Copy the default namespace mapping nsmap_copy = nsmap.copy() + unittest.addModuleCleanup(nsmap.update, nsmap_copy) + unittest.addModuleCleanup(nsmap.clear) + # Copy the path cache (should be empty) path_cache = ElementPath._cache + unittest.addModuleCleanup(setattr, ElementPath, "_cache", path_cache) ElementPath._cache = path_cache.copy() + # Align the Comment/PI factories. if hasattr(ET, '_set_factories'): old_factories = ET._set_factories(ET.Comment, ET.PI) - else: - old_factories = None - - try: - support.run_unittest(*test_classes) - finally: - from xml.etree import ElementPath - # Restore mapping and path cache - nsmap.clear() - nsmap.update(nsmap_copy) - ElementPath._cache = path_cache - if old_factories is not None: - ET._set_factories(*old_factories) - # don't interfere with subsequent tests - ET = pyET = None + unittest.addModuleCleanup(ET._set_factories, *old_factories) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py new file mode 100644 index 00000000000..270b9d6da8e --- /dev/null +++ b/Lib/test/test_xml_etree_c.py @@ -0,0 +1,280 @@ +# xml.etree test for cElementTree +import io +import struct +from test import support +from test.support.import_helper import import_fresh_module +import types +import unittest + +cET = import_fresh_module('xml.etree.ElementTree', + fresh=['_elementtree']) +cET_alias = import_fresh_module('xml.etree.cElementTree', + fresh=['_elementtree', 'xml.etree'], + deprecated=True) + + +@unittest.skipUnless(cET, 'requires _elementtree') +class MiscTests(unittest.TestCase): + # Issue #8651. + @support.bigmemtest(size=support._2G + 100, memuse=1, dry_run=False) + def test_length_overflow(self, size): + data = b'x' * size + parser = cET.XMLParser() + try: + self.assertRaises(OverflowError, parser.feed, data) + finally: + data = None + + def test_del_attribute(self): + element = cET.Element('tag') + + element.tag = 'TAG' + with self.assertRaises(AttributeError): + del element.tag + self.assertEqual(element.tag, 'TAG') + + with self.assertRaises(AttributeError): + del element.text + self.assertIsNone(element.text) + element.text = 'TEXT' + with self.assertRaises(AttributeError): + del element.text + self.assertEqual(element.text, 'TEXT') + + with self.assertRaises(AttributeError): + del element.tail + self.assertIsNone(element.tail) + element.tail = 'TAIL' + with self.assertRaises(AttributeError): + del element.tail + self.assertEqual(element.tail, 'TAIL') + + with self.assertRaises(AttributeError): + del element.attrib + self.assertEqual(element.attrib, {}) + element.attrib = {'A': 'B', 'C': 'D'} + with self.assertRaises(AttributeError): + del element.attrib + self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) + + @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() + def test_trashcan(self): + # If this test fails, it will most likely die via segfault. + e = root = cET.Element('root') + for i in range(200000): + e = cET.SubElement(e, 'x') + del e + del root + support.gc_collect() + + def test_parser_ref_cycle(self): + # bpo-31499: xmlparser_dealloc() crashed with a segmentation fault when + # xmlparser_gc_clear() was called previously by the garbage collector, + # when the parser was part of a reference cycle. + + def parser_ref_cycle(): + parser = cET.XMLParser() + # Create a reference cycle using an exception to keep the frame + # alive, so the parser will be destroyed by the garbage collector + try: + raise ValueError + except ValueError as exc: + err = exc + + # Create a parser part of reference cycle + parser_ref_cycle() + # Trigger an explicit garbage collection to break the reference cycle + # and so destroy the parser + support.gc_collect() + + def test_bpo_31728(self): + # A crash or an assertion failure shouldn't happen, in case garbage + # collection triggers a call to clear() or a reading of text or tail, + # while a setter or clear() or __setstate__() is already running. + elem = cET.Element('elem') + class X: + def __del__(self): + elem.text + elem.tail + elem.clear() + + elem.text = X() + elem.clear() # shouldn't crash + + elem.tail = X() + elem.clear() # shouldn't crash + + elem.text = X() + elem.text = X() # shouldn't crash + elem.clear() + + elem.tail = X() + elem.tail = X() # shouldn't crash + elem.clear() + + elem.text = X() + elem.__setstate__({'tag': 42}) # shouldn't cause an assertion failure + elem.clear() + + elem.tail = X() + elem.__setstate__({'tag': 42}) # shouldn't cause an assertion failure + + @support.cpython_only + def test_uninitialized_parser(self): + # The interpreter shouldn't crash in case of calling methods or + # accessing attributes of uninitialized XMLParser objects. + parser = cET.XMLParser.__new__(cET.XMLParser) + self.assertRaises(ValueError, parser.close) + self.assertRaises(ValueError, parser.feed, 'foo') + class MockFile: + def read(*args): + return '' + self.assertRaises(ValueError, parser._parse_whole, MockFile()) + self.assertRaises(ValueError, parser._setevents, None) + self.assertIsNone(parser.entity) + self.assertIsNone(parser.target) + + def test_setstate_leaks(self): + # Test reference leaks + elem = cET.Element.__new__(cET.Element) + for i in range(100): + elem.__setstate__({'tag': 'foo', 'attrib': {'bar': 42}, + '_children': [cET.Element('child')], + 'text': 'text goes here', + 'tail': 'opposite of head'}) + + self.assertEqual(elem.tag, 'foo') + self.assertEqual(elem.text, 'text goes here') + self.assertEqual(elem.tail, 'opposite of head') + self.assertEqual(list(elem.attrib.items()), [('bar', 42)]) + self.assertEqual(len(elem), 1) + self.assertEqual(elem[0].tag, 'child') + + def test_iterparse_leaks(self): + # Test reference leaks in TreeBuilder (issue #35502). + # The test is written to be executed in the hunting reference leaks + # mode. + XML = '<a></a></b>' + parser = cET.iterparse(io.StringIO(XML)) + next(parser) + del parser + support.gc_collect() + + def test_xmlpullparser_leaks(self): + # Test reference leaks in TreeBuilder (issue #35502). + # The test is written to be executed in the hunting reference leaks + # mode. + XML = '<a></a></b>' + parser = cET.XMLPullParser() + parser.feed(XML) + del parser + support.gc_collect() + + def test_dict_disappearing_during_get_item(self): + # test fix for seg fault reported in issue 27946 + class X: + def __hash__(self): + e.attrib = {} # this frees e->extra->attrib + [{i: i} for i in range(1000)] # exhaust the dict keys cache + return 13 + + e = cET.Element("elem", {1: 2}) + r = e.get(X()) + self.assertIsNone(r) + + @support.cpython_only + def test_immutable_types(self): + root = cET.fromstring('<a></a>') + dataset = ( + cET.Element, + cET.TreeBuilder, + cET.XMLParser, + type(root.iter()), + ) + for tp in dataset: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = 1 + + @support.cpython_only + def test_disallow_instantiation(self): + root = cET.fromstring('<a></a>') + iter_type = type(root.iter()) + support.check_disallow_instantiation(self, iter_type) + + +@unittest.skipUnless(cET, 'requires _elementtree') +class TestAliasWorking(unittest.TestCase): + # Test that the cET alias module is alive + def test_alias_working(self): + e = cET_alias.Element('foo') + self.assertEqual(e.tag, 'foo') + + +@unittest.skipUnless(cET, 'requires _elementtree') +@support.cpython_only +class TestAcceleratorImported(unittest.TestCase): + # Test that the C accelerator was imported, as expected + def test_correct_import_cET(self): + # SubElement is a function so it retains _elementtree as its module. + self.assertEqual(cET.SubElement.__module__, '_elementtree') + + def test_correct_import_cET_alias(self): + self.assertEqual(cET_alias.SubElement.__module__, '_elementtree') + + def test_parser_comes_from_C(self): + # The type of methods defined in Python code is types.FunctionType, + # while the type of methods defined inside _elementtree is + # <class 'wrapper_descriptor'> + self.assertNotIsInstance(cET.Element.__init__, types.FunctionType) + + +@unittest.skipUnless(cET, 'requires _elementtree') +@support.cpython_only +class SizeofTest(unittest.TestCase): + def setUp(self): + self.elementsize = support.calcobjsize('5P') + # extra + self.extra = struct.calcsize('PnnP4P') + + check_sizeof = support.check_sizeof + + def test_element(self): + e = cET.Element('a') + self.check_sizeof(e, self.elementsize) + + def test_element_with_attrib(self): + e = cET.Element('a', href='about:') + self.check_sizeof(e, self.elementsize + self.extra) + + def test_element_with_children(self): + e = cET.Element('a') + for i in range(5): + cET.SubElement(e, 'span') + # should have space for 8 children now + self.check_sizeof(e, self.elementsize + self.extra + + struct.calcsize('8P')) + + +def install_tests(): + # Test classes should have __module__ referring to this module. + from test import test_xml_etree + for name, base in vars(test_xml_etree).items(): + if isinstance(base, type) and issubclass(base, unittest.TestCase): + class Temp(base): + pass + Temp.__name__ = Temp.__qualname__ = name + Temp.__module__ = __name__ + assert name not in globals() + globals()[name] = Temp + +install_tests() + +def setUpModule(): + from test import test_xml_etree + test_xml_etree.setUpModule(module=cET) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 25a4be0e039..bb2a1ef0904 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -25,6 +25,8 @@ except ImportError: gzip = None +support.requires_working_socket(module=True) + alist = [{'astring': 'foo@bar.baz.spam', 'afloat': 7283.43, 'anint': 2**20, @@ -45,15 +47,11 @@ class XMLRPCTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_load(self): dump = xmlrpclib.dumps((alist,)) load = xmlrpclib.loads(dump) self.assertEqual(alist, load[0][0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_bare_datetime(self): # This checks that an unwrapped datetime.date object can be handled # by the marshalling code. This can't be done via test_dump_load() @@ -88,8 +86,6 @@ def test_dump_bare_datetime(self): self.assertIsNone(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_datetime_before_1900(self): # same as before but with a date before 1900 dt = datetime.datetime(1, 2, 10, 11, 41, 23) @@ -108,8 +104,6 @@ def test_datetime_before_1900(self): self.assertIs(type(newdt), xmlrpclib.DateTime) self.assertIsNone(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_1164912 (self): d = xmlrpclib.DateTime() ((new_d,), dummy) = xmlrpclib.loads(xmlrpclib.dumps((d,), @@ -120,8 +114,6 @@ def test_bug_1164912 (self): s = xmlrpclib.dumps((new_d,), methodresponse=True) self.assertIsInstance(s, str) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_newstyle_class(self): class T(object): pass @@ -187,8 +179,6 @@ def dummy_write(s): m.dump_double(xmlrpclib.MAXINT + 42, dummy_write) m.dump_double(xmlrpclib.MININT - 42, dummy_write) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_none(self): value = alist + [None] arg1 = (alist + [None],) @@ -197,8 +187,7 @@ def test_dump_none(self): xmlrpclib.loads(strg)[0][0]) self.assertRaises(TypeError, xmlrpclib.dumps, (arg1,)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_encoding(self): value = {'key\u20ac\xa4': 'value\u20ac\xa4'} @@ -220,8 +209,6 @@ def test_dump_encoding(self): self.assertEqual(xmlrpclib.loads(strg)[0][0], value) self.assertEqual(xmlrpclib.loads(strg)[1], methodname) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_bytes(self): sample = b"my dog has fleas" self.assertEqual(sample, xmlrpclib.Binary(sample)) @@ -241,8 +228,7 @@ def test_dump_bytes(self): self.assertIs(type(newvalue), xmlrpclib.Binary) self.assertIsNone(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_loads_unsupported(self): ResponseError = xmlrpclib.ResponseError data = '<params><param><value><spam/></value></param></params>' @@ -265,8 +251,6 @@ def check_loads(self, s, value, **kwargs): self.assertIs(type(newvalue), type(value)) self.assertIsNone(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_load_standard_types(self): check = self.check_loads check('string', 'string') @@ -294,8 +278,7 @@ def test_load_standard_types(self): '<member><name>a</name><value><int>1</int></value></member>' '</struct>', {'a': 1, 'b': 2}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_load_extension_types(self): check = self.check_loads check('<nil/>', None) @@ -309,6 +292,16 @@ def test_load_extension_types(self): check('<bigdecimal>9876543210.0123456789</bigdecimal>', decimal.Decimal('9876543210.0123456789')) + def test_limit_int(self): + check = self.check_loads + maxdigits = 5000 + with support.adjust_int_max_str_digits(maxdigits): + s = '1' * (maxdigits + 1) + with self.assertRaises(ValueError): + check(f'<int>{s}</int>', None) + with self.assertRaises(ValueError): + check(f'<biginteger>{s}</biginteger>', None) + def test_get_host_info(self): # see bug #3613, this raised a TypeError transp = xmlrpc.client.Transport() @@ -318,7 +311,7 @@ def test_get_host_info(self): def test_ssl_presence(self): try: - import ssl + import ssl # noqa: F401 except ImportError: has_ssl = False else: @@ -330,8 +323,6 @@ def test_ssl_presence(self): except OSError: self.assertTrue(has_ssl) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_keepalive_disconnect(self): class RequestHandler(http.server.BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -471,8 +462,6 @@ def test_repr(self): self.assertEqual(repr(f), "<Fault 42: 'Test Fault'>") self.assertEqual(repr(f), str(f)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dump_fault(self): f = xmlrpclib.Fault(42, 'Test Fault') s = xmlrpclib.dumps((f,)) @@ -518,10 +507,16 @@ def test_time_struct(self): self.assertEqual(str(t), time.strftime("%Y%m%dT%H:%M:%S", d)) def test_datetime_datetime(self): + # naive (no tzinfo) d = datetime.datetime(2007,1,2,3,4,5) t = xmlrpclib.DateTime(d) self.assertEqual(str(t), '20070102T03:04:05') + # aware (with tzinfo): the timezone is ignored + d = datetime.datetime(2023, 6, 12, 13, 30, tzinfo=datetime.UTC) + t = xmlrpclib.DateTime(d) + self.assertEqual(str(t), '20230612T13:30:00') + def test_repr(self): d = datetime.datetime(2007,1,2,3,4,5) t = xmlrpclib.DateTime(d) @@ -814,8 +809,6 @@ def tearDown(self): xmlrpc.server.SimpleXMLRPCServer._send_traceback_header = False class SimpleServerTestCase(BaseServerTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple1(self): try: p = xmlrpclib.ServerProxy(URL) @@ -826,8 +819,6 @@ def test_simple1(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nonascii(self): start_string = 'P\N{LATIN SMALL LETTER Y WITH CIRCUMFLEX}t' end_string = 'h\N{LATIN SMALL LETTER O WITH HORN}n' @@ -841,8 +832,7 @@ def test_nonascii(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_client_encoding(self): start_string = '\u20ac' end_string = '\xa4' @@ -857,8 +847,6 @@ def test_client_encoding(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nonascii_methodname(self): try: p = xmlrpclib.ServerProxy(URL, encoding='ascii') @@ -879,8 +867,6 @@ def test_404(self): self.assertEqual(response.status, 404) self.assertEqual(response.reason, 'Not Found') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_introspection1(self): expected_methods = set(['pow', 'div', 'my_function', 'add', 'têšt', 'system.listMethods', 'system.methodHelp', @@ -897,8 +883,6 @@ def test_introspection1(self): self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_introspection2(self): try: # test _methodHelp() @@ -911,8 +895,6 @@ def test_introspection2(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure @make_request_and_skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_introspection3(self): @@ -927,8 +909,6 @@ def test_introspection3(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_introspection4(self): # the SimpleXMLRPCServer doesn't support signatures, but # at least check that we can try making the call @@ -942,8 +922,6 @@ def test_introspection4(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multicall(self): try: p = xmlrpclib.ServerProxy(URL) @@ -961,8 +939,6 @@ def test_multicall(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_non_existing_multicall(self): try: p = xmlrpclib.ServerProxy(URL) @@ -984,8 +960,6 @@ def test_non_existing_multicall(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dotted_attribute(self): # Raises an AttributeError because private methods are not allowed. self.assertRaises(AttributeError, @@ -996,16 +970,12 @@ def test_dotted_attribute(self): # This avoids waiting for the socket timeout. self.test_simple1() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_allow_dotted_names_true(self): # XXX also need allow_dotted_names_false test. server = xmlrpclib.ServerProxy("http://%s:%d/RPC2" % (ADDR, PORT)) data = server.Fixture.getData() self.assertEqual(data, '42') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicode_host(self): server = xmlrpclib.ServerProxy("http://%s:%d/RPC2" % (ADDR, PORT)) self.assertEqual(server.add("a", "\xe9"), "a\xe9") @@ -1020,8 +990,6 @@ def test_partial_post(self): 'Accept-Encoding: identity\r\n' 'Content-Length: 0\r\n\r\n'.encode('ascii')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_context_manager(self): with xmlrpclib.ServerProxy(URL) as server: server.add(2, 3) @@ -1030,8 +998,6 @@ def test_context_manager(self): self.assertEqual(server('transport')._connection, (None, None)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_context_manager_method_error(self): try: with xmlrpclib.ServerProxy(URL) as server: @@ -1047,8 +1013,7 @@ class SimpleServerEncodingTestCase(BaseServerTestCase): def threadFunc(evt, numrequests, requestHandler=None, encoding=None): http_server(evt, numrequests, requestHandler, 'iso-8859-15') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_server_encoding(self): start_string = '\u20ac' end_string = '\xa4' @@ -1067,70 +1032,57 @@ def test_server_encoding(self): class MultiPathServerTestCase(BaseServerTestCase): threadFunc = staticmethod(http_multi_server) request_count = 2 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_path1(self): p = xmlrpclib.ServerProxy(URL+"/foo") self.assertEqual(p.pow(6,8), 6**8) self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_path2(self): p = xmlrpclib.ServerProxy(URL+"/foo/bar") self.assertEqual(p.add(6,8), 6+8) self.assertRaises(xmlrpclib.Fault, p.pow, 6, 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_path3(self): p = xmlrpclib.ServerProxy(URL+"/is/broken") self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_invalid_path(self): p = xmlrpclib.ServerProxy(URL+"/invalid") self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_path_query_fragment(self): p = xmlrpclib.ServerProxy(URL+"/foo?k=v#frag") self.assertEqual(p.test(), "/foo?k=v#frag") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_path_fragment(self): p = xmlrpclib.ServerProxy(URL+"/foo#frag") self.assertEqual(p.test(), "/foo#frag") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_path_query(self): p = xmlrpclib.ServerProxy(URL+"/foo?k=v") self.assertEqual(p.test(), "/foo?k=v") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_empty_path(self): p = xmlrpclib.ServerProxy(URL) self.assertEqual(p.test(), "/RPC2") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_root_path(self): p = xmlrpclib.ServerProxy(URL + "/") self.assertEqual(p.test(), "/") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_empty_path_query(self): p = xmlrpclib.ServerProxy(URL + "?k=v") self.assertEqual(p.test(), "?k=v") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_resource('walltime') def test_empty_path_fragment(self): p = xmlrpclib.ServerProxy(URL + "#frag") self.assertEqual(p.test(), "#frag") @@ -1163,8 +1115,6 @@ def setUp(self): #A test case that verifies that a server using the HTTP/1.1 keep-alive mechanism #does indeed serve subsequent requests on the same connection class KeepaliveServerTestCase1(BaseKeepaliveServerTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_two(self): p = xmlrpclib.ServerProxy(URL) #do three requests. @@ -1181,7 +1131,6 @@ def test_two(self): self.assertGreaterEqual(len(self.RequestHandler.myRequests[-1]), 2) -@unittest.skip("TODO: RUSTPYTHON, appears to hang") #test special attribute access on the serverproxy, through the __call__ #function. class KeepaliveServerTestCase2(BaseKeepaliveServerTestCase): @@ -1248,8 +1197,6 @@ def send_content(self, connection, body): def setUp(self): BaseServerTestCase.setUp(self) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gzip_request(self): t = self.Transport() t.encode_threshold = None @@ -1273,8 +1220,6 @@ def test_bad_gzip_request(self): p.pow(6, 8) p("close")() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gzip_response(self): t = self.Transport() p = xmlrpclib.ServerProxy(URL, transport=t) @@ -1333,8 +1278,6 @@ def assertContainsAdditionalHeaders(self, headers, additional): for key, value in additional.items(): self.assertEqual(headers.get(key), value) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_header(self): p = xmlrpclib.ServerProxy(URL, headers=[('X-Test', 'foo')]) self.assertEqual(p.pow(6, 8), 6**8) @@ -1342,8 +1285,6 @@ def test_header(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {'X-Test': 'foo'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_header_many(self): p = xmlrpclib.ServerProxy( URL, headers=[('X-Test', 'foo'), ('X-Test-Second', 'bar')]) @@ -1353,8 +1294,6 @@ def test_header_many(self): self.assertContainsAdditionalHeaders( headers, {'X-Test': 'foo', 'X-Test-Second': 'bar'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_header_empty(self): p = xmlrpclib.ServerProxy(URL, headers=[]) self.assertEqual(p.pow(6, 8), 6**8) @@ -1362,8 +1301,6 @@ def test_header_empty(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_header_tuple(self): p = xmlrpclib.ServerProxy(URL, headers=(('X-Test', 'foo'),)) self.assertEqual(p.pow(6, 8), 6**8) @@ -1371,8 +1308,6 @@ def test_header_tuple(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {'X-Test': 'foo'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_header_items(self): p = xmlrpclib.ServerProxy(URL, headers={'X-Test': 'foo'}.items()) self.assertEqual(p.pow(6, 8), 6**8) @@ -1431,8 +1366,6 @@ def tearDown(self): default_class = http.client.HTTPMessage xmlrpc.server.SimpleXMLRPCRequestHandler.MessageClass = default_class - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic(self): # check that flag is false by default flagval = xmlrpc.server.SimpleXMLRPCServer._send_traceback_header @@ -1527,8 +1460,6 @@ def test_cgi_get(self): self.assertEqual(message, 'Bad Request') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cgi_xmlrpc_response(self): data = """<?xml version='1.0'?> <methodCall> @@ -1574,8 +1505,6 @@ def test_cgi_xmlrpc_response(self): class UseBuiltinTypesTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_use_builtin_types(self): # SimpleXMLRPCDispatcher.__init__ accepts use_builtin_types, which # makes all dispatch of binary data as bytes instances, and all diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index 97acfd54139..7028a606217 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -538,8 +538,7 @@ def g(): "finishing g", ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_broken_getattr_handling(self): """ Test subiterator with a broken getattr implementation @@ -787,8 +786,6 @@ def outer(): repr(value), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_throwing_GeneratorExit_into_subgen_that_returns(self): """ Test throwing GeneratorExit into a subgenerator that @@ -819,8 +816,6 @@ def g(): "Enter f", ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_throwing_GeneratorExit_into_subgenerator_that_yields(self): """ Test throwing GeneratorExit into a subgenerator that @@ -887,8 +882,7 @@ def g(): yield from () self.assertRaises(StopIteration, next, g()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_delegating_generators_claim_to_be_running(self): # Check with basic iteration def one(): @@ -904,6 +898,7 @@ def two(): yield 2 g1 = one() self.assertEqual(list(g1), [0, 1, 2, 3]) + # Check with send g1 = one() res = [next(g1)] @@ -913,6 +908,9 @@ def two(): except StopIteration: pass self.assertEqual(res, [0, 1, 2, 3]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Lists differ: [0, 1, 2] != [0, 1, 2, 3] + def test_delegating_generators_claim_to_be_running_with_throw(self): # Check with throw class MyErr(Exception): pass @@ -949,8 +947,10 @@ def two(): except: self.assertEqual(res, [0, 1, 2, 3]) raise + + def test_delegating_generators_claim_to_be_running_with_close(self): # Check with close - class MyIt(object): + class MyIt: def __iter__(self): return self def __next__(self): @@ -1057,6 +1057,550 @@ def outer(): g.send((1, 2, 3, 4)) self.assertEqual(v, (1, 2, 3, 4)) +class TestInterestingEdgeCases(unittest.TestCase): + + def assert_stop_iteration(self, iterator): + with self.assertRaises(StopIteration) as caught: + next(iterator) + self.assertIsNone(caught.exception.value) + self.assertIsNone(caught.exception.__context__) + + def assert_generator_raised_stop_iteration(self): + return self.assertRaisesRegex(RuntimeError, r"^generator raised StopIteration$") + + def assert_generator_ignored_generator_exit(self): + return self.assertRaisesRegex(RuntimeError, r"^generator ignored GeneratorExit$") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_close_and_throw_work(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + yield yielded_first + yield yielded_second + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + def test_close_and_throw_raise_generator_exit(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + # GeneratorExit is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = GeneratorExit() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + # The raised GeneratorExit is suppressed, but the thrown one + # propagates. This is consistent with PEP 380: + # https://peps.python.org/pep-0380/#proposal + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = StopIteration() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = BaseException() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = Exception() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised + def test_close_and_throw_raise_stop_iteration(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.close() + self.assertIs(caught.exception.__context__, raised) + self.assertIsInstance(caught.exception.__context__.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = GeneratorExit() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = BaseException() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = Exception() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + def test_close_and_throw_raise_base_exception(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + with self.assertRaises(BaseException) as caught: + g.close() + self.assertIs(caught.exception, raised) + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = GeneratorExit() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = StopIteration() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = BaseException() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = Exception() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + def test_close_and_throw_raise_exception(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + with self.assertRaises(Exception) as caught: + g.close() + self.assertIs(caught.exception, raised) + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = GeneratorExit() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = StopIteration() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = BaseException() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = Exception() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None is not StopIteration() + def test_close_and_throw_yield(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + finally: + yield yielded_second + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + # No chaining happens. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assert_generator_ignored_generator_exit() as caught: + g.close() + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + # No chaining happens. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assert_generator_ignored_generator_exit() as caught: + g.throw(thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + self.assertEqual(g.throw(thrown), yielded_second) + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + next(g) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + self.assertEqual(g.throw(thrown), yielded_second) + with self.assertRaises(BaseException) as caught: + next(g) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + self.assertEqual(g.throw(thrown), yielded_second) + with self.assertRaises(Exception) as caught: + next(g) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + def test_close_and_throw_return(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + except: + pass + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + # StopIteration is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + # StopIteration is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + def test_throws_in_iter(self): + # See GH-126366: NULL pointer dereference if __iter__ + # threw an exception. + class Silly: + def __iter__(self): + raise RuntimeError("nobody expects the spanish inquisition") + + def my_generator(): + yield from Silly() + + with self.assertRaisesRegex(RuntimeError, "nobody expects the spanish inquisition"): + next(iter(my_generator())) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index 84ccb1bf1f0..8fb0a68deba 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -9,6 +9,7 @@ import zipapp import zipfile from test.support import requires_zlib +from test.support import os_helper from unittest.mock import patch @@ -54,6 +55,22 @@ def test_create_archive_with_subdirs(self): self.assertIn('foo/', z.namelist()) self.assertIn('bar/', z.namelist()) + def test_create_sorted_archive(self): + # Test that zipapps order their files by name + source = self.tmpdir / 'source' + source.mkdir() + (source / 'zed.py').touch() + (source / 'bin').mkdir() + (source / 'bin' / 'qux').touch() + (source / 'bin' / 'baz').touch() + (source / '__main__.py').touch() + target = io.BytesIO() + zipapp.create_archive(str(source), target) + target.seek(0) + with zipfile.ZipFile(target, 'r') as zf: + self.assertEqual(zf.namelist(), + ["__main__.py", "bin/", "bin/baz", "bin/qux", "zed.py"]) + def test_create_archive_with_filter(self): # Test packing a directory and using filter to specify # which files to include. @@ -72,6 +89,43 @@ def skip_pyc_files(path): self.assertIn('test.py', z.namelist()) self.assertNotIn('test.pyc', z.namelist()) + def test_create_archive_self_insertion(self): + # When creating an archive, we shouldn't + # include the archive in the list of files to add. + source = self.tmpdir + (source / '__main__.py').touch() + (source / 'test.py').touch() + target = self.tmpdir / 'target.pyz' + + zipapp.create_archive(source, target) + with zipfile.ZipFile(target, 'r') as z: + self.assertEqual(len(z.namelist()), 2) + self.assertIn('__main__.py', z.namelist()) + self.assertIn('test.py', z.namelist()) + + def test_target_overwrites_source_file(self): + # The target cannot be one of the files to add. + source = self.tmpdir + (source / '__main__.py').touch() + target = source / 'target.pyz' + target.touch() + + with self.assertRaises(zipapp.ZipAppError): + zipapp.create_archive(source, target) + + def test_target_overwrites_filtered_source_file(self): + # If there's a filter that excludes the target, + # the overwrite check shouldn't trigger. + source = self.tmpdir + (source / '__main__.py').touch() + target = source / 'target.pyz' + target.touch() + pyz_filter = lambda p: not p.match('*.pyz') + zipapp.create_archive(source, target, filter=pyz_filter) + with zipfile.ZipFile(target, 'r') as z: + self.assertEqual(len(z.namelist()), 1) + self.assertIn('__main__.py', z.namelist()) + def test_create_archive_filter_exclude_dir(self): # Test packing a directory and using a filter to exclude a # subdirectory (ensures that the path supplied to include @@ -205,7 +259,7 @@ def test_pack_to_fileobj(self): (source / '__main__.py').touch() target = io.BytesIO() zipapp.create_archive(str(source), target, interpreter='python') - self.assertTrue(target.getvalue().startswith(b'#!python\n')) + self.assertStartsWith(target.getvalue(), b'#!python\n') def test_read_shebang(self): # Test that we can read the shebang line correctly. @@ -246,16 +300,17 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(source), str(target), interpreter='python') new_target = io.BytesIO() zipapp.create_archive(str(target), new_target, interpreter='python2.7') - self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) + self.assertStartsWith(new_target.getvalue(), b'#!python2.7\n') - def test_read_from_pathobj(self): - # Test that we can copy an archive using a pathlib.Path object + def test_read_from_pathlike_obj(self): + # Test that we can copy an archive using a path-like object # for the source. source = self.tmpdir / 'source' source.mkdir() (source / '__main__.py').touch() - target1 = self.tmpdir / 'target1.pyz' - target2 = self.tmpdir / 'target2.pyz' + source = os_helper.FakePath(str(source)) + target1 = os_helper.FakePath(str(self.tmpdir / 'target1.pyz')) + target2 = os_helper.FakePath(str(self.tmpdir / 'target2.pyz')) zipapp.create_archive(source, target1, interpreter='python') zipapp.create_archive(target1, target2, interpreter='python2.7') self.assertEqual(zipapp.get_interpreter(target2), 'python2.7') @@ -271,7 +326,7 @@ def test_read_from_fileobj(self): new_target = io.BytesIO() temp_archive.seek(0) zipapp.create_archive(temp_archive, new_target, interpreter='python2.7') - self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) + self.assertStartsWith(new_target.getvalue(), b'#!python2.7\n') def test_remove_shebang(self): # Test that we can remove the shebang from a file. @@ -301,6 +356,7 @@ def test_content_of_copied_archive(self): # (Unix only) tests that archives with shebang lines are made executable @unittest.skipIf(sys.platform == 'win32', 'Windows does not support an executable bit') + @os_helper.skip_unless_working_chmod def test_shebang_is_executable(self): # Test that an archive with a shebang line is made executable. source = self.tmpdir / 'source' @@ -310,8 +366,6 @@ def test_shebang_is_executable(self): zipapp.create_archive(str(source), str(target), interpreter='python') self.assertTrue(target.stat().st_mode & stat.S_IEXEC) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == 'win32', 'Windows does not support an executable bit') def test_no_shebang_is_not_executable(self): diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py deleted file mode 100644 index fd4a3918e65..00000000000 --- a/Lib/test/test_zipfile.py +++ /dev/null @@ -1,3172 +0,0 @@ -import array -import contextlib -import importlib.util -import io -import itertools -import os -import pathlib -import posixpath -import string -import struct -import subprocess -import sys -import time -import unittest -import unittest.mock as mock -import zipfile -import functools - - -from tempfile import TemporaryFile -from random import randint, random, randbytes - -from test.support import script_helper -from test.support import (findfile, requires_zlib, requires_bz2, - requires_lzma, captured_stdout) -from test.support.os_helper import TESTFN, unlink, rmtree, temp_dir, temp_cwd - - -TESTFN2 = TESTFN + "2" -TESTFNDIR = TESTFN + "d" -FIXEDTEST_SIZE = 1000 -DATAFILES_DIR = 'zipfile_datafiles' - -SMALL_TEST_DATA = [('_ziptest1', '1q2w3e4r5t'), - ('ziptest2dir/_ziptest2', 'qawsedrftg'), - ('ziptest2dir/ziptest3dir/_ziptest3', 'azsxdcfvgb'), - ('ziptest2dir/ziptest3dir/ziptest4dir/_ziptest3', '6y7u8i9o0p')] - -def get_files(test): - yield TESTFN2 - with TemporaryFile() as f: - yield f - test.assertFalse(f.closed) - with io.BytesIO() as f: - yield f - test.assertFalse(f.closed) - -class AbstractTestsWithSourceFile: - @classmethod - def setUpClass(cls): - cls.line_gen = [bytes("Zipfile test line %d. random float: %f\n" % - (i, random()), "ascii") - for i in range(FIXEDTEST_SIZE)] - cls.data = b''.join(cls.line_gen) - - def setUp(self): - # Make a source file with some lines - with open(TESTFN, "wb") as fp: - fp.write(self.data) - - def make_test_archive(self, f, compression, compresslevel=None): - kwargs = {'compression': compression, 'compresslevel': compresslevel} - # Create the ZIP archive - with zipfile.ZipFile(f, "w", **kwargs) as zipfp: - zipfp.write(TESTFN, "another.name") - zipfp.write(TESTFN, TESTFN) - zipfp.writestr("strfile", self.data) - with zipfp.open('written-open-w', mode='w') as f: - for line in self.line_gen: - f.write(line) - - def zip_test(self, f, compression, compresslevel=None): - self.make_test_archive(f, compression, compresslevel) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r", compression) as zipfp: - self.assertEqual(zipfp.read(TESTFN), self.data) - self.assertEqual(zipfp.read("another.name"), self.data) - self.assertEqual(zipfp.read("strfile"), self.data) - - # Print the ZIP directory - fp = io.StringIO() - zipfp.printdir(file=fp) - directory = fp.getvalue() - lines = directory.splitlines() - self.assertEqual(len(lines), 5) # Number of files + header - - self.assertIn('File Name', lines[0]) - self.assertIn('Modified', lines[0]) - self.assertIn('Size', lines[0]) - - fn, date, time_, size = lines[1].split() - self.assertEqual(fn, 'another.name') - self.assertTrue(time.strptime(date, '%Y-%m-%d')) - self.assertTrue(time.strptime(time_, '%H:%M:%S')) - self.assertEqual(size, str(len(self.data))) - - # Check the namelist - names = zipfp.namelist() - self.assertEqual(len(names), 4) - self.assertIn(TESTFN, names) - self.assertIn("another.name", names) - self.assertIn("strfile", names) - self.assertIn("written-open-w", names) - - # Check infolist - infos = zipfp.infolist() - names = [i.filename for i in infos] - self.assertEqual(len(names), 4) - self.assertIn(TESTFN, names) - self.assertIn("another.name", names) - self.assertIn("strfile", names) - self.assertIn("written-open-w", names) - for i in infos: - self.assertEqual(i.file_size, len(self.data)) - - # check getinfo - for nm in (TESTFN, "another.name", "strfile", "written-open-w"): - info = zipfp.getinfo(nm) - self.assertEqual(info.filename, nm) - self.assertEqual(info.file_size, len(self.data)) - - # Check that testzip doesn't raise an exception - zipfp.testzip() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_basic(self): - for f in get_files(self): - self.zip_test(f, self.compression) - - def zip_open_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r", compression) as zipfp: - zipdata1 = [] - with zipfp.open(TESTFN) as zipopen1: - while True: - read_data = zipopen1.read(256) - if not read_data: - break - zipdata1.append(read_data) - - zipdata2 = [] - with zipfp.open("another.name") as zipopen2: - while True: - read_data = zipopen2.read(256) - if not read_data: - break - zipdata2.append(read_data) - - self.assertEqual(b''.join(zipdata1), self.data) - self.assertEqual(b''.join(zipdata2), self.data) - - def test_open(self): - for f in get_files(self): - self.zip_open_test(f, self.compression) - - def test_open_with_pathlike(self): - path = pathlib.Path(TESTFN2) - self.zip_open_test(path, self.compression) - with zipfile.ZipFile(path, "r", self.compression) as zipfp: - self.assertIsInstance(zipfp.filename, str) - - def zip_random_open_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r", compression) as zipfp: - zipdata1 = [] - with zipfp.open(TESTFN) as zipopen1: - while True: - read_data = zipopen1.read(randint(1, 1024)) - if not read_data: - break - zipdata1.append(read_data) - - self.assertEqual(b''.join(zipdata1), self.data) - - def test_random_open(self): - for f in get_files(self): - self.zip_random_open_test(f, self.compression) - - def zip_read1_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp, \ - zipfp.open(TESTFN) as zipopen: - zipdata = [] - while True: - read_data = zipopen.read1(-1) - if not read_data: - break - zipdata.append(read_data) - - self.assertEqual(b''.join(zipdata), self.data) - - def test_read1(self): - for f in get_files(self): - self.zip_read1_test(f, self.compression) - - def zip_read1_10_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp, \ - zipfp.open(TESTFN) as zipopen: - zipdata = [] - while True: - read_data = zipopen.read1(10) - self.assertLessEqual(len(read_data), 10) - if not read_data: - break - zipdata.append(read_data) - - self.assertEqual(b''.join(zipdata), self.data) - - def test_read1_10(self): - for f in get_files(self): - self.zip_read1_10_test(f, self.compression) - - def zip_readline_read_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp, \ - zipfp.open(TESTFN) as zipopen: - data = b'' - while True: - read = zipopen.readline() - if not read: - break - data += read - - read = zipopen.read(100) - if not read: - break - data += read - - self.assertEqual(data, self.data) - - def test_readline_read(self): - # Issue #7610: calls to readline() interleaved with calls to read(). - for f in get_files(self): - self.zip_readline_read_test(f, self.compression) - - def zip_readline_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp: - with zipfp.open(TESTFN) as zipopen: - for line in self.line_gen: - linedata = zipopen.readline() - self.assertEqual(linedata, line) - - def test_readline(self): - for f in get_files(self): - self.zip_readline_test(f, self.compression) - - def zip_readlines_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp: - with zipfp.open(TESTFN) as zipopen: - ziplines = zipopen.readlines() - for line, zipline in zip(self.line_gen, ziplines): - self.assertEqual(zipline, line) - - def test_readlines(self): - for f in get_files(self): - self.zip_readlines_test(f, self.compression) - - def zip_iterlines_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r") as zipfp: - with zipfp.open(TESTFN) as zipopen: - for line, zipline in zip(self.line_gen, zipopen): - self.assertEqual(zipline, line) - - def test_iterlines(self): - for f in get_files(self): - self.zip_iterlines_test(f, self.compression) - - def test_low_compression(self): - """Check for cases where compressed data is larger than original.""" - # Create the ZIP archive - with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipfp: - zipfp.writestr("strfile", '12') - - # Get an open object for strfile - with zipfile.ZipFile(TESTFN2, "r", self.compression) as zipfp: - with zipfp.open("strfile") as openobj: - self.assertEqual(openobj.read(1), b'1') - self.assertEqual(openobj.read(1), b'2') - - def test_writestr_compression(self): - zipfp = zipfile.ZipFile(TESTFN2, "w") - zipfp.writestr("b.txt", "hello world", compress_type=self.compression) - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, self.compression) - - def test_writestr_compresslevel(self): - zipfp = zipfile.ZipFile(TESTFN2, "w", compresslevel=1) - zipfp.writestr("a.txt", "hello world", compress_type=self.compression) - zipfp.writestr("b.txt", "hello world", compress_type=self.compression, - compresslevel=2) - - # Compression level follows the constructor. - a_info = zipfp.getinfo('a.txt') - self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info._compresslevel, 1) - - # Compression level is overridden. - b_info = zipfp.getinfo('b.txt') - self.assertEqual(b_info.compress_type, self.compression) - self.assertEqual(b_info._compresslevel, 2) - - def test_read_return_size(self): - # Issue #9837: ZipExtFile.read() shouldn't return more bytes - # than requested. - for test_size in (1, 4095, 4096, 4097, 16384): - file_size = test_size + 1 - junk = randbytes(file_size) - with zipfile.ZipFile(io.BytesIO(), "w", self.compression) as zipf: - zipf.writestr('foo', junk) - with zipf.open('foo', 'r') as fp: - buf = fp.read(test_size) - self.assertEqual(len(buf), test_size) - - def test_truncated_zipfile(self): - fp = io.BytesIO() - with zipfile.ZipFile(fp, mode='w') as zipf: - zipf.writestr('strfile', self.data, compress_type=self.compression) - end_offset = fp.tell() - zipfiledata = fp.getvalue() - - fp = io.BytesIO(zipfiledata) - with zipfile.ZipFile(fp) as zipf: - with zipf.open('strfile') as zipopen: - fp.truncate(end_offset - 20) - with self.assertRaises(EOFError): - zipopen.read() - - fp = io.BytesIO(zipfiledata) - with zipfile.ZipFile(fp) as zipf: - with zipf.open('strfile') as zipopen: - fp.truncate(end_offset - 20) - with self.assertRaises(EOFError): - while zipopen.read(100): - pass - - fp = io.BytesIO(zipfiledata) - with zipfile.ZipFile(fp) as zipf: - with zipf.open('strfile') as zipopen: - fp.truncate(end_offset - 20) - with self.assertRaises(EOFError): - while zipopen.read1(100): - pass - - def test_repr(self): - fname = 'file.name' - for f in get_files(self): - with zipfile.ZipFile(f, 'w', self.compression) as zipfp: - zipfp.write(TESTFN, fname) - r = repr(zipfp) - self.assertIn("mode='w'", r) - - with zipfile.ZipFile(f, 'r') as zipfp: - r = repr(zipfp) - if isinstance(f, str): - self.assertIn('filename=%r' % f, r) - else: - self.assertIn('file=%r' % f, r) - self.assertIn("mode='r'", r) - r = repr(zipfp.getinfo(fname)) - self.assertIn('filename=%r' % fname, r) - self.assertIn('filemode=', r) - self.assertIn('file_size=', r) - if self.compression != zipfile.ZIP_STORED: - self.assertIn('compress_type=', r) - self.assertIn('compress_size=', r) - with zipfp.open(fname) as zipopen: - r = repr(zipopen) - self.assertIn('name=%r' % fname, r) - self.assertIn("mode='r'", r) - if self.compression != zipfile.ZIP_STORED: - self.assertIn('compress_type=', r) - self.assertIn('[closed]', repr(zipopen)) - self.assertIn('[closed]', repr(zipfp)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_compresslevel_basic(self): - for f in get_files(self): - self.zip_test(f, self.compression, compresslevel=9) - - def test_per_file_compresslevel(self): - """Check that files within a Zip archive can have different - compression levels.""" - with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: - zipfp.write(TESTFN, 'compress_1') - zipfp.write(TESTFN, 'compress_9', compresslevel=9) - one_info = zipfp.getinfo('compress_1') - nine_info = zipfp.getinfo('compress_9') - self.assertEqual(one_info._compresslevel, 1) - self.assertEqual(nine_info._compresslevel, 9) - - def test_writing_errors(self): - class BrokenFile(io.BytesIO): - def write(self, data): - nonlocal count - if count is not None: - if count == stop: - raise OSError - count += 1 - super().write(data) - - stop = 0 - while True: - testfile = BrokenFile() - count = None - with zipfile.ZipFile(testfile, 'w', self.compression) as zipfp: - with zipfp.open('file1', 'w') as f: - f.write(b'data1') - count = 0 - try: - with zipfp.open('file2', 'w') as f: - f.write(b'data2') - except OSError: - stop += 1 - else: - break - finally: - count = None - with zipfile.ZipFile(io.BytesIO(testfile.getvalue())) as zipfp: - self.assertEqual(zipfp.namelist(), ['file1']) - self.assertEqual(zipfp.read('file1'), b'data1') - - with zipfile.ZipFile(io.BytesIO(testfile.getvalue())) as zipfp: - self.assertEqual(zipfp.namelist(), ['file1', 'file2']) - self.assertEqual(zipfp.read('file1'), b'data1') - self.assertEqual(zipfp.read('file2'), b'data2') - - - def tearDown(self): - unlink(TESTFN) - unlink(TESTFN2) - - -class StoredTestsWithSourceFile(AbstractTestsWithSourceFile, - unittest.TestCase): - compression = zipfile.ZIP_STORED - test_low_compression = None - - def zip_test_writestr_permissions(self, f, compression): - # Make sure that writestr and open(... mode='w') create files with - # mode 0600, when they are passed a name rather than a ZipInfo - # instance. - - self.make_test_archive(f, compression) - with zipfile.ZipFile(f, "r") as zipfp: - zinfo = zipfp.getinfo('strfile') - self.assertEqual(zinfo.external_attr, 0o600 << 16) - - zinfo2 = zipfp.getinfo('written-open-w') - self.assertEqual(zinfo2.external_attr, 0o600 << 16) - - def test_writestr_permissions(self): - for f in get_files(self): - self.zip_test_writestr_permissions(f, zipfile.ZIP_STORED) - - def test_absolute_arcnames(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - zipfp.write(TESTFN, "/absolute") - - with zipfile.ZipFile(TESTFN2, "r", zipfile.ZIP_STORED) as zipfp: - self.assertEqual(zipfp.namelist(), ["absolute"]) - - def test_append_to_zip_file(self): - """Test appending to an existing zipfile.""" - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - zipfp.write(TESTFN, TESTFN) - - with zipfile.ZipFile(TESTFN2, "a", zipfile.ZIP_STORED) as zipfp: - zipfp.writestr("strfile", self.data) - self.assertEqual(zipfp.namelist(), [TESTFN, "strfile"]) - - def test_append_to_non_zip_file(self): - """Test appending to an existing file that is not a zipfile.""" - # NOTE: this test fails if len(d) < 22 because of the first - # line "fpin.seek(-22, 2)" in _EndRecData - data = b'I am not a ZipFile!'*10 - with open(TESTFN2, 'wb') as f: - f.write(data) - - with zipfile.ZipFile(TESTFN2, "a", zipfile.ZIP_STORED) as zipfp: - zipfp.write(TESTFN, TESTFN) - - with open(TESTFN2, 'rb') as f: - f.seek(len(data)) - with zipfile.ZipFile(f, "r") as zipfp: - self.assertEqual(zipfp.namelist(), [TESTFN]) - self.assertEqual(zipfp.read(TESTFN), self.data) - with open(TESTFN2, 'rb') as f: - self.assertEqual(f.read(len(data)), data) - zipfiledata = f.read() - with io.BytesIO(zipfiledata) as bio, zipfile.ZipFile(bio) as zipfp: - self.assertEqual(zipfp.namelist(), [TESTFN]) - self.assertEqual(zipfp.read(TESTFN), self.data) - - def test_read_concatenated_zip_file(self): - with io.BytesIO() as bio: - with zipfile.ZipFile(bio, 'w', zipfile.ZIP_STORED) as zipfp: - zipfp.write(TESTFN, TESTFN) - zipfiledata = bio.getvalue() - data = b'I am not a ZipFile!'*10 - with open(TESTFN2, 'wb') as f: - f.write(data) - f.write(zipfiledata) - - with zipfile.ZipFile(TESTFN2) as zipfp: - self.assertEqual(zipfp.namelist(), [TESTFN]) - self.assertEqual(zipfp.read(TESTFN), self.data) - - def test_append_to_concatenated_zip_file(self): - with io.BytesIO() as bio: - with zipfile.ZipFile(bio, 'w', zipfile.ZIP_STORED) as zipfp: - zipfp.write(TESTFN, TESTFN) - zipfiledata = bio.getvalue() - data = b'I am not a ZipFile!'*1000000 - with open(TESTFN2, 'wb') as f: - f.write(data) - f.write(zipfiledata) - - with zipfile.ZipFile(TESTFN2, 'a') as zipfp: - self.assertEqual(zipfp.namelist(), [TESTFN]) - zipfp.writestr('strfile', self.data) - - with open(TESTFN2, 'rb') as f: - self.assertEqual(f.read(len(data)), data) - zipfiledata = f.read() - with io.BytesIO(zipfiledata) as bio, zipfile.ZipFile(bio) as zipfp: - self.assertEqual(zipfp.namelist(), [TESTFN, 'strfile']) - self.assertEqual(zipfp.read(TESTFN), self.data) - self.assertEqual(zipfp.read('strfile'), self.data) - - def test_ignores_newline_at_end(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - zipfp.write(TESTFN, TESTFN) - with open(TESTFN2, 'a', encoding='utf-8') as f: - f.write("\r\n\00\00\00") - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - self.assertIsInstance(zipfp, zipfile.ZipFile) - - def test_ignores_stuff_appended_past_comments(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - zipfp.comment = b"this is a comment" - zipfp.write(TESTFN, TESTFN) - with open(TESTFN2, 'a', encoding='utf-8') as f: - f.write("abcdef\r\n") - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - self.assertIsInstance(zipfp, zipfile.ZipFile) - self.assertEqual(zipfp.comment, b"this is a comment") - - def test_write_default_name(self): - """Check that calling ZipFile.write without arcname specified - produces the expected result.""" - with zipfile.ZipFile(TESTFN2, "w") as zipfp: - zipfp.write(TESTFN) - with open(TESTFN, "rb") as f: - self.assertEqual(zipfp.read(TESTFN), f.read()) - - def test_io_on_closed_zipextfile(self): - fname = "somefile.txt" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: - zipfp.writestr(fname, "bogus") - - with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: - with zipfp.open(fname) as fid: - fid.close() - self.assertRaises(ValueError, fid.read) - self.assertRaises(ValueError, fid.seek, 0) - self.assertRaises(ValueError, fid.tell) - self.assertRaises(ValueError, fid.readable) - self.assertRaises(ValueError, fid.seekable) - - def test_write_to_readonly(self): - """Check that trying to call write() on a readonly ZipFile object - raises a ValueError.""" - with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: - zipfp.writestr("somefile.txt", "bogus") - - with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: - self.assertRaises(ValueError, zipfp.write, TESTFN) - - with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: - with self.assertRaises(ValueError): - zipfp.open(TESTFN, mode='w') - - def test_add_file_before_1980(self): - # Set atime and mtime to 1970-01-01 - os.utime(TESTFN, (0, 0)) - with zipfile.ZipFile(TESTFN2, "w") as zipfp: - self.assertRaises(ValueError, zipfp.write, TESTFN) - - with zipfile.ZipFile(TESTFN2, "w", strict_timestamps=False) as zipfp: - zipfp.write(TESTFN) - zinfo = zipfp.getinfo(TESTFN) - self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) - - def test_add_file_after_2107(self): - # Set atime and mtime to 2108-12-30 - ts = 4386268800 - try: - time.localtime(ts) - except OverflowError: - self.skipTest(f'time.localtime({ts}) raises OverflowError') - try: - os.utime(TESTFN, (ts, ts)) - except OverflowError: - self.skipTest('Host fs cannot set timestamp to required value.') - - mtime_ns = os.stat(TESTFN).st_mtime_ns - if mtime_ns != (4386268800 * 10**9): - # XFS filesystem is limited to 32-bit timestamp, but the syscall - # didn't fail. Moreover, there is a VFS bug which returns - # a cached timestamp which is different than the value on disk. - # - # Test st_mtime_ns rather than st_mtime to avoid rounding issues. - # - # https://bugzilla.redhat.com/show_bug.cgi?id=1795576 - # https://bugs.python.org/issue39460#msg360952 - self.skipTest(f"Linux VFS/XFS kernel bug detected: {mtime_ns=}") - - with zipfile.ZipFile(TESTFN2, "w") as zipfp: - self.assertRaises(struct.error, zipfp.write, TESTFN) - - with zipfile.ZipFile(TESTFN2, "w", strict_timestamps=False) as zipfp: - zipfp.write(TESTFN) - zinfo = zipfp.getinfo(TESTFN) - self.assertEqual(zinfo.date_time, (2107, 12, 31, 23, 59, 59)) - - -@requires_zlib() -class DeflateTestsWithSourceFile(AbstractTestsWithSourceFile, - unittest.TestCase): - compression = zipfile.ZIP_DEFLATED - - def test_per_file_compression(self): - """Check that files within a Zip archive can have different - compression options.""" - with zipfile.ZipFile(TESTFN2, "w") as zipfp: - zipfp.write(TESTFN, 'storeme', zipfile.ZIP_STORED) - zipfp.write(TESTFN, 'deflateme', zipfile.ZIP_DEFLATED) - sinfo = zipfp.getinfo('storeme') - dinfo = zipfp.getinfo('deflateme') - self.assertEqual(sinfo.compress_type, zipfile.ZIP_STORED) - self.assertEqual(dinfo.compress_type, zipfile.ZIP_DEFLATED) - -@requires_bz2() -class Bzip2TestsWithSourceFile(AbstractTestsWithSourceFile, - unittest.TestCase): - compression = zipfile.ZIP_BZIP2 - -@requires_lzma() -class LzmaTestsWithSourceFile(AbstractTestsWithSourceFile, - unittest.TestCase): - compression = zipfile.ZIP_LZMA - - -class AbstractTestZip64InSmallFiles: - # These tests test the ZIP64 functionality without using large files, - # see test_zipfile64 for proper tests. - - @classmethod - def setUpClass(cls): - line_gen = (bytes("Test of zipfile line %d." % i, "ascii") - for i in range(0, FIXEDTEST_SIZE)) - cls.data = b'\n'.join(line_gen) - - def setUp(self): - self._limit = zipfile.ZIP64_LIMIT - self._filecount_limit = zipfile.ZIP_FILECOUNT_LIMIT - zipfile.ZIP64_LIMIT = 1000 - zipfile.ZIP_FILECOUNT_LIMIT = 9 - - # Make a source file with some lines - with open(TESTFN, "wb") as fp: - fp.write(self.data) - - def zip_test(self, f, compression): - # Create the ZIP archive - with zipfile.ZipFile(f, "w", compression, allowZip64=True) as zipfp: - zipfp.write(TESTFN, "another.name") - zipfp.write(TESTFN, TESTFN) - zipfp.writestr("strfile", self.data) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r", compression) as zipfp: - self.assertEqual(zipfp.read(TESTFN), self.data) - self.assertEqual(zipfp.read("another.name"), self.data) - self.assertEqual(zipfp.read("strfile"), self.data) - - # Print the ZIP directory - fp = io.StringIO() - zipfp.printdir(fp) - - directory = fp.getvalue() - lines = directory.splitlines() - self.assertEqual(len(lines), 4) # Number of files + header - - self.assertIn('File Name', lines[0]) - self.assertIn('Modified', lines[0]) - self.assertIn('Size', lines[0]) - - fn, date, time_, size = lines[1].split() - self.assertEqual(fn, 'another.name') - self.assertTrue(time.strptime(date, '%Y-%m-%d')) - self.assertTrue(time.strptime(time_, '%H:%M:%S')) - self.assertEqual(size, str(len(self.data))) - - # Check the namelist - names = zipfp.namelist() - self.assertEqual(len(names), 3) - self.assertIn(TESTFN, names) - self.assertIn("another.name", names) - self.assertIn("strfile", names) - - # Check infolist - infos = zipfp.infolist() - names = [i.filename for i in infos] - self.assertEqual(len(names), 3) - self.assertIn(TESTFN, names) - self.assertIn("another.name", names) - self.assertIn("strfile", names) - for i in infos: - self.assertEqual(i.file_size, len(self.data)) - - # check getinfo - for nm in (TESTFN, "another.name", "strfile"): - info = zipfp.getinfo(nm) - self.assertEqual(info.filename, nm) - self.assertEqual(info.file_size, len(self.data)) - - # Check that testzip doesn't raise an exception - zipfp.testzip() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_basic(self): - for f in get_files(self): - self.zip_test(f, self.compression) - - def test_too_many_files(self): - # This test checks that more than 64k files can be added to an archive, - # and that the resulting archive can be read properly by ZipFile - zipf = zipfile.ZipFile(TESTFN, "w", self.compression, - allowZip64=True) - zipf.debug = 100 - numfiles = 15 - for i in range(numfiles): - zipf.writestr("foo%08d" % i, "%d" % (i**3 % 57)) - self.assertEqual(len(zipf.namelist()), numfiles) - zipf.close() - - zipf2 = zipfile.ZipFile(TESTFN, "r", self.compression) - self.assertEqual(len(zipf2.namelist()), numfiles) - for i in range(numfiles): - content = zipf2.read("foo%08d" % i).decode('ascii') - self.assertEqual(content, "%d" % (i**3 % 57)) - zipf2.close() - - def test_too_many_files_append(self): - zipf = zipfile.ZipFile(TESTFN, "w", self.compression, - allowZip64=False) - zipf.debug = 100 - numfiles = 9 - for i in range(numfiles): - zipf.writestr("foo%08d" % i, "%d" % (i**3 % 57)) - self.assertEqual(len(zipf.namelist()), numfiles) - with self.assertRaises(zipfile.LargeZipFile): - zipf.writestr("foo%08d" % numfiles, b'') - self.assertEqual(len(zipf.namelist()), numfiles) - zipf.close() - - zipf = zipfile.ZipFile(TESTFN, "a", self.compression, - allowZip64=False) - zipf.debug = 100 - self.assertEqual(len(zipf.namelist()), numfiles) - with self.assertRaises(zipfile.LargeZipFile): - zipf.writestr("foo%08d" % numfiles, b'') - self.assertEqual(len(zipf.namelist()), numfiles) - zipf.close() - - zipf = zipfile.ZipFile(TESTFN, "a", self.compression, - allowZip64=True) - zipf.debug = 100 - self.assertEqual(len(zipf.namelist()), numfiles) - numfiles2 = 15 - for i in range(numfiles, numfiles2): - zipf.writestr("foo%08d" % i, "%d" % (i**3 % 57)) - self.assertEqual(len(zipf.namelist()), numfiles2) - zipf.close() - - zipf2 = zipfile.ZipFile(TESTFN, "r", self.compression) - self.assertEqual(len(zipf2.namelist()), numfiles2) - for i in range(numfiles2): - content = zipf2.read("foo%08d" % i).decode('ascii') - self.assertEqual(content, "%d" % (i**3 % 57)) - zipf2.close() - - def tearDown(self): - zipfile.ZIP64_LIMIT = self._limit - zipfile.ZIP_FILECOUNT_LIMIT = self._filecount_limit - unlink(TESTFN) - unlink(TESTFN2) - - -class StoredTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, - unittest.TestCase): - compression = zipfile.ZIP_STORED - - def large_file_exception_test(self, f, compression): - with zipfile.ZipFile(f, "w", compression, allowZip64=False) as zipfp: - self.assertRaises(zipfile.LargeZipFile, - zipfp.write, TESTFN, "another.name") - - def large_file_exception_test2(self, f, compression): - with zipfile.ZipFile(f, "w", compression, allowZip64=False) as zipfp: - self.assertRaises(zipfile.LargeZipFile, - zipfp.writestr, "another.name", self.data) - - def test_large_file_exception(self): - for f in get_files(self): - self.large_file_exception_test(f, zipfile.ZIP_STORED) - self.large_file_exception_test2(f, zipfile.ZIP_STORED) - - def test_absolute_arcnames(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED, - allowZip64=True) as zipfp: - zipfp.write(TESTFN, "/absolute") - - with zipfile.ZipFile(TESTFN2, "r", zipfile.ZIP_STORED) as zipfp: - self.assertEqual(zipfp.namelist(), ["absolute"]) - - def test_append(self): - # Test that appending to the Zip64 archive doesn't change - # extra fields of existing entries. - with zipfile.ZipFile(TESTFN2, "w", allowZip64=True) as zipfp: - zipfp.writestr("strfile", self.data) - with zipfile.ZipFile(TESTFN2, "r", allowZip64=True) as zipfp: - zinfo = zipfp.getinfo("strfile") - extra = zinfo.extra - with zipfile.ZipFile(TESTFN2, "a", allowZip64=True) as zipfp: - zipfp.writestr("strfile2", self.data) - with zipfile.ZipFile(TESTFN2, "r", allowZip64=True) as zipfp: - zinfo = zipfp.getinfo("strfile") - self.assertEqual(zinfo.extra, extra) - - def make_zip64_file( - self, file_size_64_set=False, file_size_extra=False, - compress_size_64_set=False, compress_size_extra=False, - header_offset_64_set=False, header_offset_extra=False, - ): - """Generate bytes sequence for a zip with (incomplete) zip64 data. - - The actual values (not the zip 64 0xffffffff values) stored in the file - are: - file_size: 8 - compress_size: 8 - header_offset: 0 - """ - actual_size = 8 - actual_header_offset = 0 - local_zip64_fields = [] - central_zip64_fields = [] - - file_size = actual_size - if file_size_64_set: - file_size = 0xffffffff - if file_size_extra: - local_zip64_fields.append(actual_size) - central_zip64_fields.append(actual_size) - file_size = struct.pack("<L", file_size) - - compress_size = actual_size - if compress_size_64_set: - compress_size = 0xffffffff - if compress_size_extra: - local_zip64_fields.append(actual_size) - central_zip64_fields.append(actual_size) - compress_size = struct.pack("<L", compress_size) - - header_offset = actual_header_offset - if header_offset_64_set: - header_offset = 0xffffffff - if header_offset_extra: - central_zip64_fields.append(actual_header_offset) - header_offset = struct.pack("<L", header_offset) - - local_extra = struct.pack( - '<HH' + 'Q'*len(local_zip64_fields), - 0x0001, - 8*len(local_zip64_fields), - *local_zip64_fields - ) - - central_extra = struct.pack( - '<HH' + 'Q'*len(central_zip64_fields), - 0x0001, - 8*len(central_zip64_fields), - *central_zip64_fields - ) - - central_dir_size = struct.pack('<Q', 58 + 8 * len(central_zip64_fields)) - offset_to_central_dir = struct.pack('<Q', 50 + 8 * len(local_zip64_fields)) - - local_extra_length = struct.pack("<H", 4 + 8 * len(local_zip64_fields)) - central_extra_length = struct.pack("<H", 4 + 8 * len(central_zip64_fields)) - - filename = b"test.txt" - content = b"test1234" - filename_length = struct.pack("<H", len(filename)) - zip64_contents = ( - # Local file header - b"PK\x03\x04\x14\x00\x00\x00\x00\x00\x00\x00!\x00\x9e%\xf5\xaf" - + compress_size - + file_size - + filename_length - + local_extra_length - + filename - + local_extra - + content - # Central directory: - + b"PK\x01\x02-\x03-\x00\x00\x00\x00\x00\x00\x00!\x00\x9e%\xf5\xaf" - + compress_size - + file_size - + filename_length - + central_extra_length - + b"\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01" - + header_offset - + filename - + central_extra - # Zip64 end of central directory - + b"PK\x06\x06,\x00\x00\x00\x00\x00\x00\x00-\x00-" - + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" - + b"\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00" - + central_dir_size - + offset_to_central_dir - # Zip64 end of central directory locator - + b"PK\x06\x07\x00\x00\x00\x00l\x00\x00\x00\x00\x00\x00\x00\x01" - + b"\x00\x00\x00" - # end of central directory - + b"PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00:\x00\x00\x002\x00" - + b"\x00\x00\x00\x00" - ) - return zip64_contents - - def test_bad_zip64_extra(self): - """Missing zip64 extra records raises an exception. - - There are 4 fields that the zip64 format handles (the disk number is - not used in this module and so is ignored here). According to the zip - spec: - The order of the fields in the zip64 extended - information record is fixed, but the fields MUST - only appear if the corresponding Local or Central - directory record field is set to 0xFFFF or 0xFFFFFFFF. - - If the zip64 extra content doesn't contain enough entries for the - number of fields marked with 0xFFFF or 0xFFFFFFFF, we raise an error. - This test mismatches the length of the zip64 extra field and the number - of fields set to indicate the presence of zip64 data. - """ - # zip64 file size present, no fields in extra, expecting one, equals - # missing file size. - missing_file_size_extra = self.make_zip64_file( - file_size_64_set=True, - ) - with self.assertRaises(zipfile.BadZipFile) as e: - zipfile.ZipFile(io.BytesIO(missing_file_size_extra)) - self.assertIn('file size', str(e.exception).lower()) - - # zip64 file size present, zip64 compress size present, one field in - # extra, expecting two, equals missing compress size. - missing_compress_size_extra = self.make_zip64_file( - file_size_64_set=True, - file_size_extra=True, - compress_size_64_set=True, - ) - with self.assertRaises(zipfile.BadZipFile) as e: - zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) - self.assertIn('compress size', str(e.exception).lower()) - - # zip64 compress size present, no fields in extra, expecting one, - # equals missing compress size. - missing_compress_size_extra = self.make_zip64_file( - compress_size_64_set=True, - ) - with self.assertRaises(zipfile.BadZipFile) as e: - zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) - self.assertIn('compress size', str(e.exception).lower()) - - # zip64 file size present, zip64 compress size present, zip64 header - # offset present, two fields in extra, expecting three, equals missing - # header offset - missing_header_offset_extra = self.make_zip64_file( - file_size_64_set=True, - file_size_extra=True, - compress_size_64_set=True, - compress_size_extra=True, - header_offset_64_set=True, - ) - with self.assertRaises(zipfile.BadZipFile) as e: - zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) - self.assertIn('header offset', str(e.exception).lower()) - - # zip64 compress size present, zip64 header offset present, one field - # in extra, expecting two, equals missing header offset - missing_header_offset_extra = self.make_zip64_file( - file_size_64_set=False, - compress_size_64_set=True, - compress_size_extra=True, - header_offset_64_set=True, - ) - with self.assertRaises(zipfile.BadZipFile) as e: - zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) - self.assertIn('header offset', str(e.exception).lower()) - - # zip64 file size present, zip64 header offset present, one field in - # extra, expecting two, equals missing header offset - missing_header_offset_extra = self.make_zip64_file( - file_size_64_set=True, - file_size_extra=True, - compress_size_64_set=False, - header_offset_64_set=True, - ) - with self.assertRaises(zipfile.BadZipFile) as e: - zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) - self.assertIn('header offset', str(e.exception).lower()) - - # zip64 header offset present, no fields in extra, expecting one, - # equals missing header offset - missing_header_offset_extra = self.make_zip64_file( - file_size_64_set=False, - compress_size_64_set=False, - header_offset_64_set=True, - ) - with self.assertRaises(zipfile.BadZipFile) as e: - zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) - self.assertIn('header offset', str(e.exception).lower()) - - def test_generated_valid_zip64_extra(self): - # These values are what is set in the make_zip64_file method. - expected_file_size = 8 - expected_compress_size = 8 - expected_header_offset = 0 - expected_content = b"test1234" - - # Loop through the various valid combinations of zip64 masks - # present and extra fields present. - params = ( - {"file_size_64_set": True, "file_size_extra": True}, - {"compress_size_64_set": True, "compress_size_extra": True}, - {"header_offset_64_set": True, "header_offset_extra": True}, - ) - - for r in range(1, len(params) + 1): - for combo in itertools.combinations(params, r): - kwargs = {} - for c in combo: - kwargs.update(c) - with zipfile.ZipFile(io.BytesIO(self.make_zip64_file(**kwargs))) as zf: - zinfo = zf.infolist()[0] - self.assertEqual(zinfo.file_size, expected_file_size) - self.assertEqual(zinfo.compress_size, expected_compress_size) - self.assertEqual(zinfo.header_offset, expected_header_offset) - self.assertEqual(zf.read(zinfo), expected_content) - - -@requires_zlib() -class DeflateTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, - unittest.TestCase): - compression = zipfile.ZIP_DEFLATED - -@requires_bz2() -class Bzip2TestZip64InSmallFiles(AbstractTestZip64InSmallFiles, - unittest.TestCase): - compression = zipfile.ZIP_BZIP2 - -@requires_lzma() -class LzmaTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, - unittest.TestCase): - compression = zipfile.ZIP_LZMA - - -class AbstractWriterTests: - - def tearDown(self): - unlink(TESTFN2) - - def test_close_after_close(self): - data = b'content' - with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: - w = zipf.open('test', 'w') - w.write(data) - w.close() - self.assertTrue(w.closed) - w.close() - self.assertTrue(w.closed) - self.assertEqual(zipf.read('test'), data) - - def test_write_after_close(self): - data = b'content' - with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: - w = zipf.open('test', 'w') - w.write(data) - w.close() - self.assertTrue(w.closed) - self.assertRaises(ValueError, w.write, b'') - self.assertEqual(zipf.read('test'), data) - - def test_issue44439(self): - q = array.array('Q', [1, 2, 3, 4, 5]) - LENGTH = len(q) * q.itemsize - with zipfile.ZipFile(io.BytesIO(), 'w', self.compression) as zip: - with zip.open('data', 'w') as data: - self.assertEqual(data.write(q), LENGTH) - self.assertEqual(zip.getinfo('data').file_size, LENGTH) - -class StoredWriterTests(AbstractWriterTests, unittest.TestCase): - compression = zipfile.ZIP_STORED - -@requires_zlib() -class DeflateWriterTests(AbstractWriterTests, unittest.TestCase): - compression = zipfile.ZIP_DEFLATED - -@requires_bz2() -class Bzip2WriterTests(AbstractWriterTests, unittest.TestCase): - compression = zipfile.ZIP_BZIP2 - -@requires_lzma() -class LzmaWriterTests(AbstractWriterTests, unittest.TestCase): - compression = zipfile.ZIP_LZMA - - -class PyZipFileTests(unittest.TestCase): - def assertCompiledIn(self, name, namelist): - if name + 'o' not in namelist: - self.assertIn(name + 'c', namelist) - - def requiresWriteAccess(self, path): - # effective_ids unavailable on windows - if not os.access(path, os.W_OK, - effective_ids=os.access in os.supports_effective_ids): - self.skipTest('requires write access to the installed location') - filename = os.path.join(path, 'test_zipfile.try') - try: - fd = os.open(filename, os.O_WRONLY | os.O_CREAT) - os.close(fd) - except Exception: - self.skipTest('requires write access to the installed location') - unlink(filename) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_write_pyfile(self): - self.requiresWriteAccess(os.path.dirname(__file__)) - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - fn = __file__ - if fn.endswith('.pyc'): - path_split = fn.split(os.sep) - if os.altsep is not None: - path_split.extend(fn.split(os.altsep)) - if '__pycache__' in path_split: - fn = importlib.util.source_from_cache(fn) - else: - fn = fn[:-1] - - zipfp.writepy(fn) - - bn = os.path.basename(fn) - self.assertNotIn(bn, zipfp.namelist()) - self.assertCompiledIn(bn, zipfp.namelist()) - - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - fn = __file__ - if fn.endswith('.pyc'): - fn = fn[:-1] - - zipfp.writepy(fn, "testpackage") - - bn = "%s/%s" % ("testpackage", os.path.basename(fn)) - self.assertNotIn(bn, zipfp.namelist()) - self.assertCompiledIn(bn, zipfp.namelist()) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_write_python_package(self): - import email - packagedir = os.path.dirname(email.__file__) - self.requiresWriteAccess(packagedir) - - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(packagedir) - - # Check for a couple of modules at different levels of the - # hierarchy - names = zipfp.namelist() - self.assertCompiledIn('email/__init__.py', names) - self.assertCompiledIn('email/mime/text.py', names) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_write_filtered_python_package(self): - import test - packagedir = os.path.dirname(test.__file__) - self.requiresWriteAccess(packagedir) - - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - - # first make sure that the test folder gives error messages - # (on the badsyntax_... files) - with captured_stdout() as reportSIO: - zipfp.writepy(packagedir) - reportStr = reportSIO.getvalue() - self.assertTrue('SyntaxError' in reportStr) - - # then check that the filter works on the whole package - with captured_stdout() as reportSIO: - zipfp.writepy(packagedir, filterfunc=lambda whatever: False) - reportStr = reportSIO.getvalue() - self.assertTrue('SyntaxError' not in reportStr) - - # then check that the filter works on individual files - def filter(path): - return not os.path.basename(path).startswith("bad") - with captured_stdout() as reportSIO, self.assertWarns(UserWarning): - zipfp.writepy(packagedir, filterfunc=filter) - reportStr = reportSIO.getvalue() - if reportStr: - print(reportStr) - self.assertTrue('SyntaxError' not in reportStr) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_write_with_optimization(self): - import email - packagedir = os.path.dirname(email.__file__) - self.requiresWriteAccess(packagedir) - optlevel = 1 if __debug__ else 0 - ext = '.pyc' - - with TemporaryFile() as t, \ - zipfile.PyZipFile(t, "w", optimize=optlevel) as zipfp: - zipfp.writepy(packagedir) - - names = zipfp.namelist() - self.assertIn('email/__init__' + ext, names) - self.assertIn('email/mime/text' + ext, names) - - def test_write_python_directory(self): - os.mkdir(TESTFN2) - try: - with open(os.path.join(TESTFN2, "mod1.py"), "w", encoding='utf-8') as fp: - fp.write("print(42)\n") - - with open(os.path.join(TESTFN2, "mod2.py"), "w", encoding='utf-8') as fp: - fp.write("print(42 * 42)\n") - - with open(os.path.join(TESTFN2, "mod2.txt"), "w", encoding='utf-8') as fp: - fp.write("bla bla bla\n") - - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(TESTFN2) - - names = zipfp.namelist() - self.assertCompiledIn('mod1.py', names) - self.assertCompiledIn('mod2.py', names) - self.assertNotIn('mod2.txt', names) - - finally: - rmtree(TESTFN2) - - def test_write_python_directory_filtered(self): - os.mkdir(TESTFN2) - try: - with open(os.path.join(TESTFN2, "mod1.py"), "w", encoding='utf-8') as fp: - fp.write("print(42)\n") - - with open(os.path.join(TESTFN2, "mod2.py"), "w", encoding='utf-8') as fp: - fp.write("print(42 * 42)\n") - - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(TESTFN2, filterfunc=lambda fn: - not fn.endswith('mod2.py')) - - names = zipfp.namelist() - self.assertCompiledIn('mod1.py', names) - self.assertNotIn('mod2.py', names) - - finally: - rmtree(TESTFN2) - - def test_write_non_pyfile(self): - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - with open(TESTFN, 'w', encoding='utf-8') as f: - f.write('most definitely not a python file') - self.assertRaises(RuntimeError, zipfp.writepy, TESTFN) - unlink(TESTFN) - - def test_write_pyfile_bad_syntax(self): - os.mkdir(TESTFN2) - try: - with open(os.path.join(TESTFN2, "mod1.py"), "w", encoding='utf-8') as fp: - fp.write("Bad syntax in python file\n") - - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - # syntax errors are printed to stdout - with captured_stdout() as s: - zipfp.writepy(os.path.join(TESTFN2, "mod1.py")) - - self.assertIn("SyntaxError", s.getvalue()) - - # as it will not have compiled the python file, it will - # include the .py file not .pyc - names = zipfp.namelist() - self.assertIn('mod1.py', names) - self.assertNotIn('mod1.pyc', names) - - finally: - rmtree(TESTFN2) - - def test_write_pathlike(self): - os.mkdir(TESTFN2) - try: - with open(os.path.join(TESTFN2, "mod1.py"), "w", encoding='utf-8') as fp: - fp.write("print(42)\n") - - with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: - zipfp.writepy(pathlib.Path(TESTFN2) / "mod1.py") - names = zipfp.namelist() - self.assertCompiledIn('mod1.py', names) - finally: - rmtree(TESTFN2) - - -class ExtractTests(unittest.TestCase): - - def make_test_file(self): - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) - - def test_extract(self): - with temp_cwd(): - self.make_test_file() - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - writtenfile = zipfp.extract(fpath) - - # make sure it was written to the right place - correctfile = os.path.join(os.getcwd(), fpath) - correctfile = os.path.normpath(correctfile) - - self.assertEqual(writtenfile, correctfile) - - # make sure correct data is in correct file - with open(writtenfile, "rb") as f: - self.assertEqual(fdata.encode(), f.read()) - - unlink(writtenfile) - - def _test_extract_with_target(self, target): - self.make_test_file() - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - writtenfile = zipfp.extract(fpath, target) - - # make sure it was written to the right place - correctfile = os.path.join(target, fpath) - correctfile = os.path.normpath(correctfile) - self.assertTrue(os.path.samefile(writtenfile, correctfile), (writtenfile, target)) - - # make sure correct data is in correct file - with open(writtenfile, "rb") as f: - self.assertEqual(fdata.encode(), f.read()) - - unlink(writtenfile) - - unlink(TESTFN2) - - def test_extract_with_target(self): - with temp_dir() as extdir: - self._test_extract_with_target(extdir) - - def test_extract_with_target_pathlike(self): - with temp_dir() as extdir: - self._test_extract_with_target(pathlib.Path(extdir)) - - def test_extract_all(self): - with temp_cwd(): - self.make_test_file() - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - zipfp.extractall() - for fpath, fdata in SMALL_TEST_DATA: - outfile = os.path.join(os.getcwd(), fpath) - - with open(outfile, "rb") as f: - self.assertEqual(fdata.encode(), f.read()) - - unlink(outfile) - - def _test_extract_all_with_target(self, target): - self.make_test_file() - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - zipfp.extractall(target) - for fpath, fdata in SMALL_TEST_DATA: - outfile = os.path.join(target, fpath) - - with open(outfile, "rb") as f: - self.assertEqual(fdata.encode(), f.read()) - - unlink(outfile) - - unlink(TESTFN2) - - def test_extract_all_with_target(self): - with temp_dir() as extdir: - self._test_extract_all_with_target(extdir) - - def test_extract_all_with_target_pathlike(self): - with temp_dir() as extdir: - self._test_extract_all_with_target(pathlib.Path(extdir)) - - def check_file(self, filename, content): - self.assertTrue(os.path.isfile(filename)) - with open(filename, 'rb') as f: - self.assertEqual(f.read(), content) - - def test_sanitize_windows_name(self): - san = zipfile.ZipFile._sanitize_windows_name - # Passing pathsep in allows this test to work regardless of platform. - self.assertEqual(san(r',,?,C:,foo,bar/z', ','), r'_,C_,foo,bar/z') - self.assertEqual(san(r'a\b,c<d>e|f"g?h*i', ','), r'a\b,c_d_e_f_g_h_i') - self.assertEqual(san('../../foo../../ba..r', '/'), r'foo/ba..r') - - def test_extract_hackers_arcnames_common_cases(self): - common_hacknames = [ - ('../foo/bar', 'foo/bar'), - ('foo/../bar', 'foo/bar'), - ('foo/../../bar', 'foo/bar'), - ('foo/bar/..', 'foo/bar'), - ('./../foo/bar', 'foo/bar'), - ('/foo/bar', 'foo/bar'), - ('/foo/../bar', 'foo/bar'), - ('/foo/../../bar', 'foo/bar'), - ] - self._test_extract_hackers_arcnames(common_hacknames) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf(os.path.sep != '\\', 'Requires \\ as path separator.') - def test_extract_hackers_arcnames_windows_only(self): - """Test combination of path fixing and windows name sanitization.""" - windows_hacknames = [ - (r'..\foo\bar', 'foo/bar'), - (r'..\/foo\/bar', 'foo/bar'), - (r'foo/\..\/bar', 'foo/bar'), - (r'foo\/../\bar', 'foo/bar'), - (r'C:foo/bar', 'foo/bar'), - (r'C:/foo/bar', 'foo/bar'), - (r'C://foo/bar', 'foo/bar'), - (r'C:\foo\bar', 'foo/bar'), - (r'//conky/mountpoint/foo/bar', 'foo/bar'), - (r'\\conky\mountpoint\foo\bar', 'foo/bar'), - (r'///conky/mountpoint/foo/bar', 'conky/mountpoint/foo/bar'), - (r'\\\conky\mountpoint\foo\bar', 'conky/mountpoint/foo/bar'), - (r'//conky//mountpoint/foo/bar', 'conky/mountpoint/foo/bar'), - (r'\\conky\\mountpoint\foo\bar', 'conky/mountpoint/foo/bar'), - (r'//?/C:/foo/bar', 'foo/bar'), - (r'\\?\C:\foo\bar', 'foo/bar'), - (r'C:/../C:/foo/bar', 'C_/foo/bar'), - (r'a:b\c<d>e|f"g?h*i', 'b/c_d_e_f_g_h_i'), - ('../../foo../../ba..r', 'foo/ba..r'), - ] - self._test_extract_hackers_arcnames(windows_hacknames) - - @unittest.skipIf(os.path.sep != '/', r'Requires / as path separator.') - def test_extract_hackers_arcnames_posix_only(self): - posix_hacknames = [ - ('//foo/bar', 'foo/bar'), - ('../../foo../../ba..r', 'foo../ba..r'), - (r'foo/..\bar', r'foo/..\bar'), - ] - self._test_extract_hackers_arcnames(posix_hacknames) - - def _test_extract_hackers_arcnames(self, hacknames): - for arcname, fixedname in hacknames: - content = b'foobar' + arcname.encode() - with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_STORED) as zipfp: - zinfo = zipfile.ZipInfo() - # preserve backslashes - zinfo.filename = arcname - zinfo.external_attr = 0o600 << 16 - zipfp.writestr(zinfo, content) - - arcname = arcname.replace(os.sep, "/") - targetpath = os.path.join('target', 'subdir', 'subsub') - correctfile = os.path.join(targetpath, *fixedname.split('/')) - - with zipfile.ZipFile(TESTFN2, 'r') as zipfp: - writtenfile = zipfp.extract(arcname, targetpath) - self.assertEqual(writtenfile, correctfile, - msg='extract %r: %r != %r' % - (arcname, writtenfile, correctfile)) - self.check_file(correctfile, content) - rmtree('target') - - with zipfile.ZipFile(TESTFN2, 'r') as zipfp: - zipfp.extractall(targetpath) - self.check_file(correctfile, content) - rmtree('target') - - correctfile = os.path.join(os.getcwd(), *fixedname.split('/')) - - with zipfile.ZipFile(TESTFN2, 'r') as zipfp: - writtenfile = zipfp.extract(arcname) - self.assertEqual(writtenfile, correctfile, - msg="extract %r" % arcname) - self.check_file(correctfile, content) - rmtree(fixedname.split('/')[0]) - - with zipfile.ZipFile(TESTFN2, 'r') as zipfp: - zipfp.extractall() - self.check_file(correctfile, content) - rmtree(fixedname.split('/')[0]) - - unlink(TESTFN2) - - -class OtherTests(unittest.TestCase): - def test_open_via_zip_info(self): - # Create the ZIP archive - with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: - zipfp.writestr("name", "foo") - with self.assertWarns(UserWarning): - zipfp.writestr("name", "bar") - self.assertEqual(zipfp.namelist(), ["name"] * 2) - - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - infos = zipfp.infolist() - data = b"" - for info in infos: - with zipfp.open(info) as zipopen: - data += zipopen.read() - self.assertIn(data, {b"foobar", b"barfoo"}) - data = b"" - for info in infos: - data += zipfp.read(info) - self.assertIn(data, {b"foobar", b"barfoo"}) - - def test_writestr_extended_local_header_issue1202(self): - with zipfile.ZipFile(TESTFN2, 'w') as orig_zip: - for data in 'abcdefghijklmnop': - zinfo = zipfile.ZipInfo(data) - zinfo.flag_bits |= 0x08 # Include an extended local header. - orig_zip.writestr(zinfo, data) - - def test_close(self): - """Check that the zipfile is closed after the 'with' block.""" - with zipfile.ZipFile(TESTFN2, "w") as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) - self.assertIsNotNone(zipfp.fp, 'zipfp is not open') - self.assertIsNone(zipfp.fp, 'zipfp is not closed') - - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - self.assertIsNotNone(zipfp.fp, 'zipfp is not open') - self.assertIsNone(zipfp.fp, 'zipfp is not closed') - - def test_close_on_exception(self): - """Check that the zipfile is closed if an exception is raised in the - 'with' block.""" - with zipfile.ZipFile(TESTFN2, "w") as zipfp: - for fpath, fdata in SMALL_TEST_DATA: - zipfp.writestr(fpath, fdata) - - try: - with zipfile.ZipFile(TESTFN2, "r") as zipfp2: - raise zipfile.BadZipFile() - except zipfile.BadZipFile: - self.assertIsNone(zipfp2.fp, 'zipfp is not closed') - - def test_unsupported_version(self): - # File has an extract_version of 120 - data = (b'PK\x03\x04x\x00\x00\x00\x00\x00!p\xa1@\x00\x00\x00\x00\x00\x00' - b'\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00xPK\x01\x02x\x03x\x00\x00\x00\x00' - b'\x00!p\xa1@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00\x00xPK\x05\x06' - b'\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x1f\x00\x00\x00\x00\x00') - - self.assertRaises(NotImplementedError, zipfile.ZipFile, - io.BytesIO(data), 'r') - - @requires_zlib() - def test_read_unicode_filenames(self): - # bug #10801 - fname = findfile('zip_cp437_header.zip') - with zipfile.ZipFile(fname) as zipfp: - for name in zipfp.namelist(): - zipfp.open(name).close() - - def test_write_unicode_filenames(self): - with zipfile.ZipFile(TESTFN, "w") as zf: - zf.writestr("foo.txt", "Test for unicode filename") - zf.writestr("\xf6.txt", "Test for unicode filename") - self.assertIsInstance(zf.infolist()[0].filename, str) - - with zipfile.ZipFile(TESTFN, "r") as zf: - self.assertEqual(zf.filelist[0].filename, "foo.txt") - self.assertEqual(zf.filelist[1].filename, "\xf6.txt") - - def test_read_after_write_unicode_filenames(self): - with zipfile.ZipFile(TESTFN2, 'w') as zipfp: - zipfp.writestr('приклад', b'sample') - self.assertEqual(zipfp.read('приклад'), b'sample') - - def test_exclusive_create_zip_file(self): - """Test exclusive creating a new zipfile.""" - unlink(TESTFN2) - filename = 'testfile.txt' - content = b'hello, world. this is some content.' - with zipfile.ZipFile(TESTFN2, "x", zipfile.ZIP_STORED) as zipfp: - zipfp.writestr(filename, content) - with self.assertRaises(FileExistsError): - zipfile.ZipFile(TESTFN2, "x", zipfile.ZIP_STORED) - with zipfile.ZipFile(TESTFN2, "r") as zipfp: - self.assertEqual(zipfp.namelist(), [filename]) - self.assertEqual(zipfp.read(filename), content) - - def test_create_non_existent_file_for_append(self): - if os.path.exists(TESTFN): - os.unlink(TESTFN) - - filename = 'testfile.txt' - content = b'hello, world. this is some content.' - - try: - with zipfile.ZipFile(TESTFN, 'a') as zf: - zf.writestr(filename, content) - except OSError: - self.fail('Could not append data to a non-existent zip file.') - - self.assertTrue(os.path.exists(TESTFN)) - - with zipfile.ZipFile(TESTFN, 'r') as zf: - self.assertEqual(zf.read(filename), content) - - def test_close_erroneous_file(self): - # This test checks that the ZipFile constructor closes the file object - # it opens if there's an error in the file. If it doesn't, the - # traceback holds a reference to the ZipFile object and, indirectly, - # the file object. - # On Windows, this causes the os.unlink() call to fail because the - # underlying file is still open. This is SF bug #412214. - # - with open(TESTFN, "w", encoding="utf-8") as fp: - fp.write("this is not a legal zip file\n") - try: - zf = zipfile.ZipFile(TESTFN) - except zipfile.BadZipFile: - pass - - def test_is_zip_erroneous_file(self): - """Check that is_zipfile() correctly identifies non-zip files.""" - # - passing a filename - with open(TESTFN, "w", encoding='utf-8') as fp: - fp.write("this is not a legal zip file\n") - self.assertFalse(zipfile.is_zipfile(TESTFN)) - # - passing a path-like object - self.assertFalse(zipfile.is_zipfile(pathlib.Path(TESTFN))) - # - passing a file object - with open(TESTFN, "rb") as fp: - self.assertFalse(zipfile.is_zipfile(fp)) - # - passing a file-like object - fp = io.BytesIO() - fp.write(b"this is not a legal zip file\n") - self.assertFalse(zipfile.is_zipfile(fp)) - fp.seek(0, 0) - self.assertFalse(zipfile.is_zipfile(fp)) - - def test_damaged_zipfile(self): - """Check that zipfiles with missing bytes at the end raise BadZipFile.""" - # - Create a valid zip file - fp = io.BytesIO() - with zipfile.ZipFile(fp, mode="w") as zipf: - zipf.writestr("foo.txt", b"O, for a Muse of Fire!") - zipfiledata = fp.getvalue() - - # - Now create copies of it missing the last N bytes and make sure - # a BadZipFile exception is raised when we try to open it - for N in range(len(zipfiledata)): - fp = io.BytesIO(zipfiledata[:N]) - self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, fp) - - def test_is_zip_valid_file(self): - """Check that is_zipfile() correctly identifies zip files.""" - # - passing a filename - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - zipf.writestr("foo.txt", b"O, for a Muse of Fire!") - - self.assertTrue(zipfile.is_zipfile(TESTFN)) - # - passing a file object - with open(TESTFN, "rb") as fp: - self.assertTrue(zipfile.is_zipfile(fp)) - fp.seek(0, 0) - zip_contents = fp.read() - # - passing a file-like object - fp = io.BytesIO() - fp.write(zip_contents) - self.assertTrue(zipfile.is_zipfile(fp)) - fp.seek(0, 0) - self.assertTrue(zipfile.is_zipfile(fp)) - - def test_non_existent_file_raises_OSError(self): - # make sure we don't raise an AttributeError when a partially-constructed - # ZipFile instance is finalized; this tests for regression on SF tracker - # bug #403871. - - # The bug we're testing for caused an AttributeError to be raised - # when a ZipFile instance was created for a file that did not - # exist; the .fp member was not initialized but was needed by the - # __del__() method. Since the AttributeError is in the __del__(), - # it is ignored, but the user should be sufficiently annoyed by - # the message on the output that regression will be noticed - # quickly. - self.assertRaises(OSError, zipfile.ZipFile, TESTFN) - - def test_empty_file_raises_BadZipFile(self): - f = open(TESTFN, 'w', encoding='utf-8') - f.close() - self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN) - - with open(TESTFN, 'w', encoding='utf-8') as fp: - fp.write("short file") - self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_negative_central_directory_offset_raises_BadZipFile(self): - # Zip file containing an empty EOCD record - buffer = bytearray(b'PK\x05\x06' + b'\0'*18) - - # Set the size of the central directory bytes to become 1, - # causing the central directory offset to become negative - for dirsize in 1, 2**32-1: - buffer[12:16] = struct.pack('<L', dirsize) - f = io.BytesIO(buffer) - self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, f) - - def test_closed_zip_raises_ValueError(self): - """Verify that testzip() doesn't swallow inappropriate exceptions.""" - data = io.BytesIO() - with zipfile.ZipFile(data, mode="w") as zipf: - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - - # This is correct; calling .read on a closed ZipFile should raise - # a ValueError, and so should calling .testzip. An earlier - # version of .testzip would swallow this exception (and any other) - # and report that the first file in the archive was corrupt. - self.assertRaises(ValueError, zipf.read, "foo.txt") - self.assertRaises(ValueError, zipf.open, "foo.txt") - self.assertRaises(ValueError, zipf.testzip) - self.assertRaises(ValueError, zipf.writestr, "bogus.txt", "bogus") - with open(TESTFN, 'w', encoding='utf-8') as f: - f.write('zipfile test data') - self.assertRaises(ValueError, zipf.write, TESTFN) - - def test_bad_constructor_mode(self): - """Check that bad modes passed to ZipFile constructor are caught.""" - self.assertRaises(ValueError, zipfile.ZipFile, TESTFN, "q") - - def test_bad_open_mode(self): - """Check that bad modes passed to ZipFile.open are caught.""" - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - - with zipfile.ZipFile(TESTFN, mode="r") as zipf: - # read the data to make sure the file is there - zipf.read("foo.txt") - self.assertRaises(ValueError, zipf.open, "foo.txt", "q") - # universal newlines support is removed - self.assertRaises(ValueError, zipf.open, "foo.txt", "U") - self.assertRaises(ValueError, zipf.open, "foo.txt", "rU") - - def test_read0(self): - """Check that calling read(0) on a ZipExtFile object returns an empty - string and doesn't advance file pointer.""" - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - # read the data to make sure the file is there - with zipf.open("foo.txt") as f: - for i in range(FIXEDTEST_SIZE): - self.assertEqual(f.read(0), b'') - - self.assertEqual(f.read(), b"O, for a Muse of Fire!") - - def test_open_non_existent_item(self): - """Check that attempting to call open() for an item that doesn't - exist in the archive raises a RuntimeError.""" - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - self.assertRaises(KeyError, zipf.open, "foo.txt", "r") - - def test_bad_compression_mode(self): - """Check that bad compression methods passed to ZipFile.open are - caught.""" - self.assertRaises(NotImplementedError, zipfile.ZipFile, TESTFN, "w", -1) - - def test_unsupported_compression(self): - # data is declared as shrunk, but actually deflated - data = (b'PK\x03\x04.\x00\x00\x00\x01\x00\xe4C\xa1@\x00\x00\x00' - b'\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00x\x03\x00PK\x01' - b'\x02.\x03.\x00\x00\x00\x01\x00\xe4C\xa1@\x00\x00\x00\x00\x02\x00\x00' - b'\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - b'\x80\x01\x00\x00\x00\x00xPK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00' - b'/\x00\x00\x00!\x00\x00\x00\x00\x00') - with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: - self.assertRaises(NotImplementedError, zipf.open, 'x') - - def test_null_byte_in_filename(self): - """Check that a filename containing a null byte is properly - terminated.""" - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - zipf.writestr("foo.txt\x00qqq", b"O, for a Muse of Fire!") - self.assertEqual(zipf.namelist(), ['foo.txt']) - - def test_struct_sizes(self): - """Check that ZIP internal structure sizes are calculated correctly.""" - self.assertEqual(zipfile.sizeEndCentDir, 22) - self.assertEqual(zipfile.sizeCentralDir, 46) - self.assertEqual(zipfile.sizeEndCentDir64, 56) - self.assertEqual(zipfile.sizeEndCentDir64Locator, 20) - - def test_comments(self): - """Check that comments on the archive are handled properly.""" - - # check default comment is empty - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - self.assertEqual(zipf.comment, b'') - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - - with zipfile.ZipFile(TESTFN, mode="r") as zipfr: - self.assertEqual(zipfr.comment, b'') - - # check a simple short comment - comment = b'Bravely taking to his feet, he beat a very brave retreat.' - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - zipf.comment = comment - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - with zipfile.ZipFile(TESTFN, mode="r") as zipfr: - self.assertEqual(zipf.comment, comment) - - # check a comment of max length - comment2 = ''.join(['%d' % (i**3 % 10) for i in range((1 << 16)-1)]) - comment2 = comment2.encode("ascii") - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - zipf.comment = comment2 - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - - with zipfile.ZipFile(TESTFN, mode="r") as zipfr: - self.assertEqual(zipfr.comment, comment2) - - # check a comment that is too long is truncated - with zipfile.ZipFile(TESTFN, mode="w") as zipf: - with self.assertWarns(UserWarning): - zipf.comment = comment2 + b'oops' - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - with zipfile.ZipFile(TESTFN, mode="r") as zipfr: - self.assertEqual(zipfr.comment, comment2) - - # check that comments are correctly modified in append mode - with zipfile.ZipFile(TESTFN,mode="w") as zipf: - zipf.comment = b"original comment" - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - with zipfile.ZipFile(TESTFN,mode="a") as zipf: - zipf.comment = b"an updated comment" - with zipfile.ZipFile(TESTFN,mode="r") as zipf: - self.assertEqual(zipf.comment, b"an updated comment") - - # check that comments are correctly shortened in append mode - # and the file is indeed truncated - with zipfile.ZipFile(TESTFN,mode="w") as zipf: - zipf.comment = b"original comment that's longer" - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - original_zip_size = os.path.getsize(TESTFN) - with zipfile.ZipFile(TESTFN,mode="a") as zipf: - zipf.comment = b"shorter comment" - self.assertTrue(original_zip_size > os.path.getsize(TESTFN)) - with zipfile.ZipFile(TESTFN,mode="r") as zipf: - self.assertEqual(zipf.comment, b"shorter comment") - - def test_unicode_comment(self): - with zipfile.ZipFile(TESTFN, "w", zipfile.ZIP_STORED) as zipf: - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - with self.assertRaises(TypeError): - zipf.comment = "this is an error" - - def test_change_comment_in_empty_archive(self): - with zipfile.ZipFile(TESTFN, "a", zipfile.ZIP_STORED) as zipf: - self.assertFalse(zipf.filelist) - zipf.comment = b"this is a comment" - with zipfile.ZipFile(TESTFN, "r") as zipf: - self.assertEqual(zipf.comment, b"this is a comment") - - def test_change_comment_in_nonempty_archive(self): - with zipfile.ZipFile(TESTFN, "w", zipfile.ZIP_STORED) as zipf: - zipf.writestr("foo.txt", "O, for a Muse of Fire!") - with zipfile.ZipFile(TESTFN, "a", zipfile.ZIP_STORED) as zipf: - self.assertTrue(zipf.filelist) - zipf.comment = b"this is a comment" - with zipfile.ZipFile(TESTFN, "r") as zipf: - self.assertEqual(zipf.comment, b"this is a comment") - - def test_empty_zipfile(self): - # Check that creating a file in 'w' or 'a' mode and closing without - # adding any files to the archives creates a valid empty ZIP file - zipf = zipfile.ZipFile(TESTFN, mode="w") - zipf.close() - try: - zipf = zipfile.ZipFile(TESTFN, mode="r") - except zipfile.BadZipFile: - self.fail("Unable to create empty ZIP file in 'w' mode") - - zipf = zipfile.ZipFile(TESTFN, mode="a") - zipf.close() - try: - zipf = zipfile.ZipFile(TESTFN, mode="r") - except: - self.fail("Unable to create empty ZIP file in 'a' mode") - - def test_open_empty_file(self): - # Issue 1710703: Check that opening a file with less than 22 bytes - # raises a BadZipFile exception (rather than the previously unhelpful - # OSError) - f = open(TESTFN, 'w', encoding='utf-8') - f.close() - self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN, 'r') - - def test_create_zipinfo_before_1980(self): - self.assertRaises(ValueError, - zipfile.ZipInfo, 'seventies', (1979, 1, 1, 0, 0, 0)) - - def test_create_empty_zipinfo_repr(self): - """Before bpo-26185, repr() on empty ZipInfo object was failing.""" - zi = zipfile.ZipInfo(filename="empty") - self.assertEqual(repr(zi), "<ZipInfo filename='empty' file_size=0>") - - def test_create_empty_zipinfo_default_attributes(self): - """Ensure all required attributes are set.""" - zi = zipfile.ZipInfo() - self.assertEqual(zi.orig_filename, "NoName") - self.assertEqual(zi.filename, "NoName") - self.assertEqual(zi.date_time, (1980, 1, 1, 0, 0, 0)) - self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) - self.assertEqual(zi.comment, b"") - self.assertEqual(zi.extra, b"") - self.assertIn(zi.create_system, (0, 3)) - self.assertEqual(zi.create_version, zipfile.DEFAULT_VERSION) - self.assertEqual(zi.extract_version, zipfile.DEFAULT_VERSION) - self.assertEqual(zi.reserved, 0) - self.assertEqual(zi.flag_bits, 0) - self.assertEqual(zi.volume, 0) - self.assertEqual(zi.internal_attr, 0) - self.assertEqual(zi.external_attr, 0) - - # Before bpo-26185, both were missing - self.assertEqual(zi.file_size, 0) - self.assertEqual(zi.compress_size, 0) - - def test_zipfile_with_short_extra_field(self): - """If an extra field in the header is less than 4 bytes, skip it.""" - zipdata = ( - b'PK\x03\x04\x14\x00\x00\x00\x00\x00\x93\x9b\xad@\x8b\x9e' - b'\xd9\xd3\x01\x00\x00\x00\x01\x00\x00\x00\x03\x00\x03\x00ab' - b'c\x00\x00\x00APK\x01\x02\x14\x03\x14\x00\x00\x00\x00' - b'\x00\x93\x9b\xad@\x8b\x9e\xd9\xd3\x01\x00\x00\x00\x01\x00\x00' - b'\x00\x03\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00' - b'\x00\x00\x00abc\x00\x00PK\x05\x06\x00\x00\x00\x00' - b'\x01\x00\x01\x003\x00\x00\x00%\x00\x00\x00\x00\x00' - ) - with zipfile.ZipFile(io.BytesIO(zipdata), 'r') as zipf: - # testzip returns the name of the first corrupt file, or None - self.assertIsNone(zipf.testzip()) - - def test_open_conflicting_handles(self): - # It's only possible to open one writable file handle at a time - msg1 = b"It's fun to charter an accountant!" - msg2 = b"And sail the wide accountant sea" - msg3 = b"To find, explore the funds offshore" - with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_STORED) as zipf: - with zipf.open('foo', mode='w') as w2: - w2.write(msg1) - with zipf.open('bar', mode='w') as w1: - with self.assertRaises(ValueError): - zipf.open('handle', mode='w') - with self.assertRaises(ValueError): - zipf.open('foo', mode='r') - with self.assertRaises(ValueError): - zipf.writestr('str', 'abcde') - with self.assertRaises(ValueError): - zipf.write(__file__, 'file') - with self.assertRaises(ValueError): - zipf.close() - w1.write(msg2) - with zipf.open('baz', mode='w') as w2: - w2.write(msg3) - - with zipfile.ZipFile(TESTFN2, 'r') as zipf: - self.assertEqual(zipf.read('foo'), msg1) - self.assertEqual(zipf.read('bar'), msg2) - self.assertEqual(zipf.read('baz'), msg3) - self.assertEqual(zipf.namelist(), ['foo', 'bar', 'baz']) - - def test_seek_tell(self): - # Test seek functionality - txt = b"Where's Bruce?" - bloc = txt.find(b"Bruce") - # Check seek on a file - with zipfile.ZipFile(TESTFN, "w") as zipf: - zipf.writestr("foo.txt", txt) - with zipfile.ZipFile(TESTFN, "r") as zipf: - with zipf.open("foo.txt", "r") as fp: - fp.seek(bloc, os.SEEK_SET) - self.assertEqual(fp.tell(), bloc) - fp.seek(-bloc, os.SEEK_CUR) - self.assertEqual(fp.tell(), 0) - fp.seek(bloc, os.SEEK_CUR) - self.assertEqual(fp.tell(), bloc) - self.assertEqual(fp.read(5), txt[bloc:bloc+5]) - fp.seek(0, os.SEEK_END) - self.assertEqual(fp.tell(), len(txt)) - fp.seek(0, os.SEEK_SET) - self.assertEqual(fp.tell(), 0) - # Check seek on memory file - data = io.BytesIO() - with zipfile.ZipFile(data, mode="w") as zipf: - zipf.writestr("foo.txt", txt) - with zipfile.ZipFile(data, mode="r") as zipf: - with zipf.open("foo.txt", "r") as fp: - fp.seek(bloc, os.SEEK_SET) - self.assertEqual(fp.tell(), bloc) - fp.seek(-bloc, os.SEEK_CUR) - self.assertEqual(fp.tell(), 0) - fp.seek(bloc, os.SEEK_CUR) - self.assertEqual(fp.tell(), bloc) - self.assertEqual(fp.read(5), txt[bloc:bloc+5]) - fp.seek(0, os.SEEK_END) - self.assertEqual(fp.tell(), len(txt)) - fp.seek(0, os.SEEK_SET) - self.assertEqual(fp.tell(), 0) - - @requires_bz2() - def test_decompress_without_3rd_party_library(self): - data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - zip_file = io.BytesIO(data) - with zipfile.ZipFile(zip_file, 'w', compression=zipfile.ZIP_BZIP2) as zf: - zf.writestr('a.txt', b'a') - with mock.patch('zipfile.bz2', None): - with zipfile.ZipFile(zip_file) as zf: - self.assertRaises(RuntimeError, zf.extract, 'a.txt') - - def tearDown(self): - unlink(TESTFN) - unlink(TESTFN2) - - -class AbstractBadCrcTests: - def test_testzip_with_bad_crc(self): - """Tests that files with bad CRCs return their name from testzip.""" - zipdata = self.zip_with_bad_crc - - with zipfile.ZipFile(io.BytesIO(zipdata), mode="r") as zipf: - # testzip returns the name of the first corrupt file, or None - self.assertEqual('afile', zipf.testzip()) - - def test_read_with_bad_crc(self): - """Tests that files with bad CRCs raise a BadZipFile exception when read.""" - zipdata = self.zip_with_bad_crc - - # Using ZipFile.read() - with zipfile.ZipFile(io.BytesIO(zipdata), mode="r") as zipf: - self.assertRaises(zipfile.BadZipFile, zipf.read, 'afile') - - # Using ZipExtFile.read() - with zipfile.ZipFile(io.BytesIO(zipdata), mode="r") as zipf: - with zipf.open('afile', 'r') as corrupt_file: - self.assertRaises(zipfile.BadZipFile, corrupt_file.read) - - # Same with small reads (in order to exercise the buffering logic) - with zipfile.ZipFile(io.BytesIO(zipdata), mode="r") as zipf: - with zipf.open('afile', 'r') as corrupt_file: - corrupt_file.MIN_READ_SIZE = 2 - with self.assertRaises(zipfile.BadZipFile): - while corrupt_file.read(2): - pass - - -class StoredBadCrcTests(AbstractBadCrcTests, unittest.TestCase): - compression = zipfile.ZIP_STORED - zip_with_bad_crc = ( - b'PK\003\004\024\0\0\0\0\0 \213\212;:r' - b'\253\377\f\0\0\0\f\0\0\0\005\0\0\000af' - b'ilehello,AworldP' - b'K\001\002\024\003\024\0\0\0\0\0 \213\212;:' - b'r\253\377\f\0\0\0\f\0\0\0\005\0\0\0\0' - b'\0\0\0\0\0\0\0\200\001\0\0\0\000afi' - b'lePK\005\006\0\0\0\0\001\0\001\0003\000' - b'\0\0/\0\0\0\0\0') - -@requires_zlib() -class DeflateBadCrcTests(AbstractBadCrcTests, unittest.TestCase): - compression = zipfile.ZIP_DEFLATED - zip_with_bad_crc = ( - b'PK\x03\x04\x14\x00\x00\x00\x08\x00n}\x0c=FA' - b'KE\x10\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00af' - b'ile\xcbH\xcd\xc9\xc9W(\xcf/\xcaI\xc9\xa0' - b'=\x13\x00PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00n' - b'}\x0c=FAKE\x10\x00\x00\x00n\x00\x00\x00\x05' - b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00' - b'\x00afilePK\x05\x06\x00\x00\x00\x00\x01\x00' - b'\x01\x003\x00\x00\x003\x00\x00\x00\x00\x00') - -@requires_bz2() -class Bzip2BadCrcTests(AbstractBadCrcTests, unittest.TestCase): - compression = zipfile.ZIP_BZIP2 - zip_with_bad_crc = ( - b'PK\x03\x04\x14\x03\x00\x00\x0c\x00nu\x0c=FA' - b'KE8\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00af' - b'ileBZh91AY&SY\xd4\xa8\xca' - b'\x7f\x00\x00\x0f\x11\x80@\x00\x06D\x90\x80 \x00 \xa5' - b'P\xd9!\x03\x03\x13\x13\x13\x89\xa9\xa9\xc2u5:\x9f' - b'\x8b\xb9"\x9c(HjTe?\x80PK\x01\x02\x14' - b'\x03\x14\x03\x00\x00\x0c\x00nu\x0c=FAKE8' - b'\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00' - b'\x00 \x80\x80\x81\x00\x00\x00\x00afilePK' - b'\x05\x06\x00\x00\x00\x00\x01\x00\x01\x003\x00\x00\x00[\x00' - b'\x00\x00\x00\x00') - -@requires_lzma() -class LzmaBadCrcTests(AbstractBadCrcTests, unittest.TestCase): - compression = zipfile.ZIP_LZMA - zip_with_bad_crc = ( - b'PK\x03\x04\x14\x03\x00\x00\x0e\x00nu\x0c=FA' - b'KE\x1b\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00af' - b'ile\t\x04\x05\x00]\x00\x00\x00\x04\x004\x19I' - b'\xee\x8d\xe9\x17\x89:3`\tq!.8\x00PK' - b'\x01\x02\x14\x03\x14\x03\x00\x00\x0e\x00nu\x0c=FA' - b'KE\x1b\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00\x00\x00' - b'\x00\x00\x00\x00 \x80\x80\x81\x00\x00\x00\x00afil' - b'ePK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x003\x00\x00' - b'\x00>\x00\x00\x00\x00\x00') - - -class DecryptionTests(unittest.TestCase): - """Check that ZIP decryption works. Since the library does not - support encryption at the moment, we use a pre-generated encrypted - ZIP file.""" - - data = ( - b'PK\x03\x04\x14\x00\x01\x00\x00\x00n\x92i.#y\xef?&\x00\x00\x00\x1a\x00' - b'\x00\x00\x08\x00\x00\x00test.txt\xfa\x10\xa0gly|\xfa-\xc5\xc0=\xf9y' - b'\x18\xe0\xa8r\xb3Z}Lg\xbc\xae\xf9|\x9b\x19\xe4\x8b\xba\xbb)\x8c\xb0\xdbl' - b'PK\x01\x02\x14\x00\x14\x00\x01\x00\x00\x00n\x92i.#y\xef?&\x00\x00\x00' - b'\x1a\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x01\x00 \x00\xb6\x81' - b'\x00\x00\x00\x00test.txtPK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x006\x00' - b'\x00\x00L\x00\x00\x00\x00\x00' ) - data2 = ( - b'PK\x03\x04\x14\x00\t\x00\x08\x00\xcf}38xu\xaa\xb2\x14\x00\x00\x00\x00\x02' - b'\x00\x00\x04\x00\x15\x00zeroUT\t\x00\x03\xd6\x8b\x92G\xda\x8b\x92GUx\x04' - b'\x00\xe8\x03\xe8\x03\xc7<M\xb5a\xceX\xa3Y&\x8b{oE\xd7\x9d\x8c\x98\x02\xc0' - b'PK\x07\x08xu\xaa\xb2\x14\x00\x00\x00\x00\x02\x00\x00PK\x01\x02\x17\x03' - b'\x14\x00\t\x00\x08\x00\xcf}38xu\xaa\xb2\x14\x00\x00\x00\x00\x02\x00\x00' - b'\x04\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00ze' - b'roUT\x05\x00\x03\xd6\x8b\x92GUx\x00\x00PK\x05\x06\x00\x00\x00\x00\x01' - b'\x00\x01\x00?\x00\x00\x00[\x00\x00\x00\x00\x00' ) - - plain = b'zipfile.py encryption test' - plain2 = b'\x00'*512 - - def setUp(self): - with open(TESTFN, "wb") as fp: - fp.write(self.data) - self.zip = zipfile.ZipFile(TESTFN, "r") - with open(TESTFN2, "wb") as fp: - fp.write(self.data2) - self.zip2 = zipfile.ZipFile(TESTFN2, "r") - - def tearDown(self): - self.zip.close() - os.unlink(TESTFN) - self.zip2.close() - os.unlink(TESTFN2) - - def test_no_password(self): - # Reading the encrypted file without password - # must generate a RunTime exception - self.assertRaises(RuntimeError, self.zip.read, "test.txt") - self.assertRaises(RuntimeError, self.zip2.read, "zero") - - def test_bad_password(self): - self.zip.setpassword(b"perl") - self.assertRaises(RuntimeError, self.zip.read, "test.txt") - self.zip2.setpassword(b"perl") - self.assertRaises(RuntimeError, self.zip2.read, "zero") - - @requires_zlib() - def test_good_password(self): - self.zip.setpassword(b"python") - self.assertEqual(self.zip.read("test.txt"), self.plain) - self.zip2.setpassword(b"12345") - self.assertEqual(self.zip2.read("zero"), self.plain2) - - def test_unicode_password(self): - self.assertRaises(TypeError, self.zip.setpassword, "unicode") - self.assertRaises(TypeError, self.zip.read, "test.txt", "python") - self.assertRaises(TypeError, self.zip.open, "test.txt", pwd="python") - self.assertRaises(TypeError, self.zip.extract, "test.txt", pwd="python") - - def test_seek_tell(self): - self.zip.setpassword(b"python") - txt = self.plain - test_word = b'encryption' - bloc = txt.find(test_word) - bloc_len = len(test_word) - with self.zip.open("test.txt", "r") as fp: - fp.seek(bloc, os.SEEK_SET) - self.assertEqual(fp.tell(), bloc) - fp.seek(-bloc, os.SEEK_CUR) - self.assertEqual(fp.tell(), 0) - fp.seek(bloc, os.SEEK_CUR) - self.assertEqual(fp.tell(), bloc) - self.assertEqual(fp.read(bloc_len), txt[bloc:bloc+bloc_len]) - - # Make sure that the second read after seeking back beyond - # _readbuffer returns the same content (ie. rewind to the start of - # the file to read forward to the required position). - old_read_size = fp.MIN_READ_SIZE - fp.MIN_READ_SIZE = 1 - fp._readbuffer = b'' - fp._offset = 0 - fp.seek(0, os.SEEK_SET) - self.assertEqual(fp.tell(), 0) - fp.seek(bloc, os.SEEK_CUR) - self.assertEqual(fp.read(bloc_len), txt[bloc:bloc+bloc_len]) - fp.MIN_READ_SIZE = old_read_size - - fp.seek(0, os.SEEK_END) - self.assertEqual(fp.tell(), len(txt)) - fp.seek(0, os.SEEK_SET) - self.assertEqual(fp.tell(), 0) - - # Read the file completely to definitely call any eof integrity - # checks (crc) and make sure they still pass. - fp.read() - - -class AbstractTestsWithRandomBinaryFiles: - @classmethod - def setUpClass(cls): - datacount = randint(16, 64)*1024 + randint(1, 1024) - cls.data = b''.join(struct.pack('<f', random()*randint(-1000, 1000)) - for i in range(datacount)) - - def setUp(self): - # Make a source file with some lines - with open(TESTFN, "wb") as fp: - fp.write(self.data) - - def tearDown(self): - unlink(TESTFN) - unlink(TESTFN2) - - def make_test_archive(self, f, compression): - # Create the ZIP archive - with zipfile.ZipFile(f, "w", compression) as zipfp: - zipfp.write(TESTFN, "another.name") - zipfp.write(TESTFN, TESTFN) - - def zip_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r", compression) as zipfp: - testdata = zipfp.read(TESTFN) - self.assertEqual(len(testdata), len(self.data)) - self.assertEqual(testdata, self.data) - self.assertEqual(zipfp.read("another.name"), self.data) - - def test_read(self): - for f in get_files(self): - self.zip_test(f, self.compression) - - def zip_open_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r", compression) as zipfp: - zipdata1 = [] - with zipfp.open(TESTFN) as zipopen1: - while True: - read_data = zipopen1.read(256) - if not read_data: - break - zipdata1.append(read_data) - - zipdata2 = [] - with zipfp.open("another.name") as zipopen2: - while True: - read_data = zipopen2.read(256) - if not read_data: - break - zipdata2.append(read_data) - - testdata1 = b''.join(zipdata1) - self.assertEqual(len(testdata1), len(self.data)) - self.assertEqual(testdata1, self.data) - - testdata2 = b''.join(zipdata2) - self.assertEqual(len(testdata2), len(self.data)) - self.assertEqual(testdata2, self.data) - - def test_open(self): - for f in get_files(self): - self.zip_open_test(f, self.compression) - - def zip_random_open_test(self, f, compression): - self.make_test_archive(f, compression) - - # Read the ZIP archive - with zipfile.ZipFile(f, "r", compression) as zipfp: - zipdata1 = [] - with zipfp.open(TESTFN) as zipopen1: - while True: - read_data = zipopen1.read(randint(1, 1024)) - if not read_data: - break - zipdata1.append(read_data) - - testdata = b''.join(zipdata1) - self.assertEqual(len(testdata), len(self.data)) - self.assertEqual(testdata, self.data) - - def test_random_open(self): - for f in get_files(self): - self.zip_random_open_test(f, self.compression) - - -class StoredTestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles, - unittest.TestCase): - compression = zipfile.ZIP_STORED - -@requires_zlib() -class DeflateTestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles, - unittest.TestCase): - compression = zipfile.ZIP_DEFLATED - -@requires_bz2() -class Bzip2TestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles, - unittest.TestCase): - compression = zipfile.ZIP_BZIP2 - -@requires_lzma() -class LzmaTestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles, - unittest.TestCase): - compression = zipfile.ZIP_LZMA - - -# Provide the tell() method but not seek() -class Tellable: - def __init__(self, fp): - self.fp = fp - self.offset = 0 - - def write(self, data): - n = self.fp.write(data) - self.offset += n - return n - - def tell(self): - return self.offset - - def flush(self): - self.fp.flush() - -class Unseekable: - def __init__(self, fp): - self.fp = fp - - def write(self, data): - return self.fp.write(data) - - def flush(self): - self.fp.flush() - -class UnseekableTests(unittest.TestCase): - def test_writestr(self): - for wrapper in (lambda f: f), Tellable, Unseekable: - with self.subTest(wrapper=wrapper): - f = io.BytesIO() - f.write(b'abc') - bf = io.BufferedWriter(f) - with zipfile.ZipFile(wrapper(bf), 'w', zipfile.ZIP_STORED) as zipfp: - zipfp.writestr('ones', b'111') - zipfp.writestr('twos', b'222') - self.assertEqual(f.getvalue()[:5], b'abcPK') - with zipfile.ZipFile(f, mode='r') as zipf: - with zipf.open('ones') as zopen: - self.assertEqual(zopen.read(), b'111') - with zipf.open('twos') as zopen: - self.assertEqual(zopen.read(), b'222') - - def test_write(self): - for wrapper in (lambda f: f), Tellable, Unseekable: - with self.subTest(wrapper=wrapper): - f = io.BytesIO() - f.write(b'abc') - bf = io.BufferedWriter(f) - with zipfile.ZipFile(wrapper(bf), 'w', zipfile.ZIP_STORED) as zipfp: - self.addCleanup(unlink, TESTFN) - with open(TESTFN, 'wb') as f2: - f2.write(b'111') - zipfp.write(TESTFN, 'ones') - with open(TESTFN, 'wb') as f2: - f2.write(b'222') - zipfp.write(TESTFN, 'twos') - self.assertEqual(f.getvalue()[:5], b'abcPK') - with zipfile.ZipFile(f, mode='r') as zipf: - with zipf.open('ones') as zopen: - self.assertEqual(zopen.read(), b'111') - with zipf.open('twos') as zopen: - self.assertEqual(zopen.read(), b'222') - - def test_open_write(self): - for wrapper in (lambda f: f), Tellable, Unseekable: - with self.subTest(wrapper=wrapper): - f = io.BytesIO() - f.write(b'abc') - bf = io.BufferedWriter(f) - with zipfile.ZipFile(wrapper(bf), 'w', zipfile.ZIP_STORED) as zipf: - with zipf.open('ones', 'w') as zopen: - zopen.write(b'111') - with zipf.open('twos', 'w') as zopen: - zopen.write(b'222') - self.assertEqual(f.getvalue()[:5], b'abcPK') - with zipfile.ZipFile(f) as zipf: - self.assertEqual(zipf.read('ones'), b'111') - self.assertEqual(zipf.read('twos'), b'222') - - -@requires_zlib() -class TestsWithMultipleOpens(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.data1 = b'111' + randbytes(10000) - cls.data2 = b'222' + randbytes(10000) - - def make_test_archive(self, f): - # Create the ZIP archive - with zipfile.ZipFile(f, "w", zipfile.ZIP_DEFLATED) as zipfp: - zipfp.writestr('ones', self.data1) - zipfp.writestr('twos', self.data2) - - def test_same_file(self): - # Verify that (when the ZipFile is in control of creating file objects) - # multiple open() calls can be made without interfering with each other. - for f in get_files(self): - self.make_test_archive(f) - with zipfile.ZipFile(f, mode="r") as zipf: - with zipf.open('ones') as zopen1, zipf.open('ones') as zopen2: - data1 = zopen1.read(500) - data2 = zopen2.read(500) - data1 += zopen1.read() - data2 += zopen2.read() - self.assertEqual(data1, data2) - self.assertEqual(data1, self.data1) - - def test_different_file(self): - # Verify that (when the ZipFile is in control of creating file objects) - # multiple open() calls can be made without interfering with each other. - for f in get_files(self): - self.make_test_archive(f) - with zipfile.ZipFile(f, mode="r") as zipf: - with zipf.open('ones') as zopen1, zipf.open('twos') as zopen2: - data1 = zopen1.read(500) - data2 = zopen2.read(500) - data1 += zopen1.read() - data2 += zopen2.read() - self.assertEqual(data1, self.data1) - self.assertEqual(data2, self.data2) - - def test_interleaved(self): - # Verify that (when the ZipFile is in control of creating file objects) - # multiple open() calls can be made without interfering with each other. - for f in get_files(self): - self.make_test_archive(f) - with zipfile.ZipFile(f, mode="r") as zipf: - with zipf.open('ones') as zopen1: - data1 = zopen1.read(500) - with zipf.open('twos') as zopen2: - data2 = zopen2.read(500) - data1 += zopen1.read() - data2 += zopen2.read() - self.assertEqual(data1, self.data1) - self.assertEqual(data2, self.data2) - - def test_read_after_close(self): - for f in get_files(self): - self.make_test_archive(f) - with contextlib.ExitStack() as stack: - with zipfile.ZipFile(f, 'r') as zipf: - zopen1 = stack.enter_context(zipf.open('ones')) - zopen2 = stack.enter_context(zipf.open('twos')) - data1 = zopen1.read(500) - data2 = zopen2.read(500) - data1 += zopen1.read() - data2 += zopen2.read() - self.assertEqual(data1, self.data1) - self.assertEqual(data2, self.data2) - - def test_read_after_write(self): - for f in get_files(self): - with zipfile.ZipFile(f, 'w', zipfile.ZIP_DEFLATED) as zipf: - zipf.writestr('ones', self.data1) - zipf.writestr('twos', self.data2) - with zipf.open('ones') as zopen1: - data1 = zopen1.read(500) - self.assertEqual(data1, self.data1[:500]) - with zipfile.ZipFile(f, 'r') as zipf: - data1 = zipf.read('ones') - data2 = zipf.read('twos') - self.assertEqual(data1, self.data1) - self.assertEqual(data2, self.data2) - - def test_write_after_read(self): - for f in get_files(self): - with zipfile.ZipFile(f, "w", zipfile.ZIP_DEFLATED) as zipf: - zipf.writestr('ones', self.data1) - with zipf.open('ones') as zopen1: - zopen1.read(500) - zipf.writestr('twos', self.data2) - with zipfile.ZipFile(f, 'r') as zipf: - data1 = zipf.read('ones') - data2 = zipf.read('twos') - self.assertEqual(data1, self.data1) - self.assertEqual(data2, self.data2) - - def test_many_opens(self): - # Verify that read() and open() promptly close the file descriptor, - # and don't rely on the garbage collector to free resources. - self.make_test_archive(TESTFN2) - with zipfile.ZipFile(TESTFN2, mode="r") as zipf: - for x in range(100): - zipf.read('ones') - with zipf.open('ones') as zopen1: - pass - with open(os.devnull, "rb") as f: - self.assertLess(f.fileno(), 100) - - def test_write_while_reading(self): - with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_DEFLATED) as zipf: - zipf.writestr('ones', self.data1) - with zipfile.ZipFile(TESTFN2, 'a', zipfile.ZIP_DEFLATED) as zipf: - with zipf.open('ones', 'r') as r1: - data1 = r1.read(500) - with zipf.open('twos', 'w') as w1: - w1.write(self.data2) - data1 += r1.read() - self.assertEqual(data1, self.data1) - with zipfile.ZipFile(TESTFN2) as zipf: - self.assertEqual(zipf.read('twos'), self.data2) - - def tearDown(self): - unlink(TESTFN2) - - -class TestWithDirectory(unittest.TestCase): - def setUp(self): - os.mkdir(TESTFN2) - - def test_extract_dir(self): - with zipfile.ZipFile(findfile("zipdir.zip")) as zipf: - zipf.extractall(TESTFN2) - self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) - self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) - self.assertTrue(os.path.exists(os.path.join(TESTFN2, "a", "b", "c"))) - - def test_bug_6050(self): - # Extraction should succeed if directories already exist - os.mkdir(os.path.join(TESTFN2, "a")) - self.test_extract_dir() - - def test_write_dir(self): - dirpath = os.path.join(TESTFN2, "x") - os.mkdir(dirpath) - mode = os.stat(dirpath).st_mode & 0xFFFF - with zipfile.ZipFile(TESTFN, "w") as zipf: - zipf.write(dirpath) - zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("/x/")) - self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) - zipf.write(dirpath, "y") - zinfo = zipf.filelist[1] - self.assertTrue(zinfo.filename, "y/") - self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) - with zipfile.ZipFile(TESTFN, "r") as zipf: - zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("/x/")) - self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) - zinfo = zipf.filelist[1] - self.assertTrue(zinfo.filename, "y/") - self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) - target = os.path.join(TESTFN2, "target") - os.mkdir(target) - zipf.extractall(target) - self.assertTrue(os.path.isdir(os.path.join(target, "y"))) - self.assertEqual(len(os.listdir(target)), 2) - - def test_writestr_dir(self): - os.mkdir(os.path.join(TESTFN2, "x")) - with zipfile.ZipFile(TESTFN, "w") as zipf: - zipf.writestr("x/", b'') - zinfo = zipf.filelist[0] - self.assertEqual(zinfo.filename, "x/") - self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) - with zipfile.ZipFile(TESTFN, "r") as zipf: - zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("x/")) - self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) - target = os.path.join(TESTFN2, "target") - os.mkdir(target) - zipf.extractall(target) - self.assertTrue(os.path.isdir(os.path.join(target, "x"))) - self.assertEqual(os.listdir(target), ["x"]) - - def tearDown(self): - rmtree(TESTFN2) - if os.path.exists(TESTFN): - unlink(TESTFN) - - -class ZipInfoTests(unittest.TestCase): - def test_from_file(self): - zi = zipfile.ZipInfo.from_file(__file__) - self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') - self.assertFalse(zi.is_dir()) - self.assertEqual(zi.file_size, os.path.getsize(__file__)) - - def test_from_file_pathlike(self): - zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) - self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') - self.assertFalse(zi.is_dir()) - self.assertEqual(zi.file_size, os.path.getsize(__file__)) - - def test_from_file_bytes(self): - zi = zipfile.ZipInfo.from_file(os.fsencode(__file__), 'test') - self.assertEqual(posixpath.basename(zi.filename), 'test') - self.assertFalse(zi.is_dir()) - self.assertEqual(zi.file_size, os.path.getsize(__file__)) - - def test_from_file_fileno(self): - with open(__file__, 'rb') as f: - zi = zipfile.ZipInfo.from_file(f.fileno(), 'test') - self.assertEqual(posixpath.basename(zi.filename), 'test') - self.assertFalse(zi.is_dir()) - self.assertEqual(zi.file_size, os.path.getsize(__file__)) - - def test_from_dir(self): - dirpath = os.path.dirname(os.path.abspath(__file__)) - zi = zipfile.ZipInfo.from_file(dirpath, 'stdlib_tests') - self.assertEqual(zi.filename, 'stdlib_tests/') - self.assertTrue(zi.is_dir()) - self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) - self.assertEqual(zi.file_size, 0) - - -class CommandLineTest(unittest.TestCase): - - def zipfilecmd(self, *args, **kwargs): - rc, out, err = script_helper.assert_python_ok('-m', 'zipfile', *args, - **kwargs) - return out.replace(os.linesep.encode(), b'\n') - - def zipfilecmd_failure(self, *args): - return script_helper.assert_python_failure('-m', 'zipfile', *args) - - def test_bad_use(self): - rc, out, err = self.zipfilecmd_failure() - self.assertEqual(out, b'') - self.assertIn(b'usage', err.lower()) - self.assertIn(b'error', err.lower()) - self.assertIn(b'required', err.lower()) - rc, out, err = self.zipfilecmd_failure('-l', '') - self.assertEqual(out, b'') - self.assertNotEqual(err.strip(), b'') - - def test_test_command(self): - zip_name = findfile('zipdir.zip') - for opt in '-t', '--test': - out = self.zipfilecmd(opt, zip_name) - self.assertEqual(out.rstrip(), b'Done testing') - zip_name = findfile('testtar.tar') - rc, out, err = self.zipfilecmd_failure('-t', zip_name) - self.assertEqual(out, b'') - - def test_list_command(self): - zip_name = findfile('zipdir.zip') - t = io.StringIO() - with zipfile.ZipFile(zip_name, 'r') as tf: - tf.printdir(t) - expected = t.getvalue().encode('ascii', 'backslashreplace') - for opt in '-l', '--list': - out = self.zipfilecmd(opt, zip_name, - PYTHONIOENCODING='ascii:backslashreplace') - self.assertEqual(out, expected) - - @requires_zlib() - def test_create_command(self): - self.addCleanup(unlink, TESTFN) - with open(TESTFN, 'w', encoding='utf-8') as f: - f.write('test 1') - os.mkdir(TESTFNDIR) - self.addCleanup(rmtree, TESTFNDIR) - with open(os.path.join(TESTFNDIR, 'file.txt'), 'w', encoding='utf-8') as f: - f.write('test 2') - files = [TESTFN, TESTFNDIR] - namelist = [TESTFN, TESTFNDIR + '/', TESTFNDIR + '/file.txt'] - for opt in '-c', '--create': - try: - out = self.zipfilecmd(opt, TESTFN2, *files) - self.assertEqual(out, b'') - with zipfile.ZipFile(TESTFN2) as zf: - self.assertEqual(zf.namelist(), namelist) - self.assertEqual(zf.read(namelist[0]), b'test 1') - self.assertEqual(zf.read(namelist[2]), b'test 2') - finally: - unlink(TESTFN2) - - def test_extract_command(self): - zip_name = findfile('zipdir.zip') - for opt in '-e', '--extract': - with temp_dir() as extdir: - out = self.zipfilecmd(opt, zip_name, extdir) - self.assertEqual(out, b'') - with zipfile.ZipFile(zip_name) as zf: - for zi in zf.infolist(): - path = os.path.join(extdir, - zi.filename.replace('/', os.sep)) - if zi.is_dir(): - self.assertTrue(os.path.isdir(path)) - else: - self.assertTrue(os.path.isfile(path)) - with open(path, 'rb') as f: - self.assertEqual(f.read(), zf.read(zi)) - - -class TestExecutablePrependedZip(unittest.TestCase): - """Test our ability to open zip files with an executable prepended.""" - - def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='ziptestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='ziptestdata') - - def _test_zip_works(self, name): - # bpo28494 sanity check: ensure is_zipfile works on these. - self.assertTrue(zipfile.is_zipfile(name), - f'is_zipfile failed on {name}') - # Ensure we can operate on these via ZipFile. - with zipfile.ZipFile(name) as zipfp: - for n in zipfp.namelist(): - data = zipfp.read(n) - self.assertIn(b'FAVORITE_NUMBER', data) - - def test_read_zip_with_exe_prepended(self): - self._test_zip_works(self.exe_zip) - - def test_read_zip64_with_exe_prepended(self): - self._test_zip_works(self.exe_zip64) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipUnless(sys.executable, 'sys.executable required.') - @unittest.skipUnless(os.access('/bin/bash', os.X_OK), - 'Test relies on #!/bin/bash working.') - def test_execute_zip2(self): - output = subprocess.check_output([self.exe_zip, sys.executable]) - self.assertIn(b'number in executable: 5', output) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipUnless(sys.executable, 'sys.executable required.') - @unittest.skipUnless(os.access('/bin/bash', os.X_OK), - 'Test relies on #!/bin/bash working.') - def test_execute_zip64(self): - output = subprocess.check_output([self.exe_zip64, sys.executable]) - self.assertIn(b'number in executable: 5', output) - - -# Poor man's technique to consume a (smallish) iterable. -consume = tuple - - -# from jaraco.itertools 5.0 -class jaraco: - class itertools: - class Counter: - def __init__(self, i): - self.count = 0 - self._orig_iter = iter(i) - - def __iter__(self): - return self - - def __next__(self): - result = next(self._orig_iter) - self.count += 1 - return result - - -def add_dirs(zf): - """ - Given a writable zip file zf, inject directory entries for - any directories implied by the presence of children. - """ - for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): - zf.writestr(name, b"") - return zf - - -def build_alpharep_fixture(): - """ - Create a zip file with this structure: - - . - ├── a.txt - ├── b - │ ├── c.txt - │ ├── d - │ │ └── e.txt - │ └── f.txt - └── g - └── h - └── i.txt - - This fixture has the following key characteristics: - - - a file at the root (a) - - a file two levels deep (b/d/e) - - multiple files in a directory (b/c, b/f) - - a directory containing only a directory (g/h) - - "alpha" because it uses alphabet - "rep" because it's a representative example - """ - data = io.BytesIO() - zf = zipfile.ZipFile(data, "w") - zf.writestr("a.txt", b"content of a") - zf.writestr("b/c.txt", b"content of c") - zf.writestr("b/d/e.txt", b"content of e") - zf.writestr("b/f.txt", b"content of f") - zf.writestr("g/h/i.txt", b"content of i") - zf.filename = "alpharep.zip" - return zf - - -def pass_alpharep(meth): - """ - Given a method, wrap it in a for loop that invokes method - with each subtest. - """ - - @functools.wraps(meth) - def wrapper(self): - for alpharep in self.zipfile_alpharep(): - meth(self, alpharep=alpharep) - - return wrapper - - -class TestPath(unittest.TestCase): - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - def zipfile_alpharep(self): - with self.subTest(): - yield build_alpharep_fixture() - with self.subTest(): - yield add_dirs(build_alpharep_fixture()) - - def zipfile_ondisk(self, alpharep): - tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) - buffer = alpharep.fp - alpharep.close() - path = tmpdir / alpharep.filename - with path.open("wb") as strm: - strm.write(buffer.getvalue()) - return path - - @pass_alpharep - def test_iterdir_and_types(self, alpharep): - root = zipfile.Path(alpharep) - assert root.is_dir() - a, b, g = root.iterdir() - assert a.is_file() - assert b.is_dir() - assert g.is_dir() - c, f, d = b.iterdir() - assert c.is_file() and f.is_file() - (e,) = d.iterdir() - assert e.is_file() - (h,) = g.iterdir() - (i,) = h.iterdir() - assert i.is_file() - - @pass_alpharep - def test_is_file_missing(self, alpharep): - root = zipfile.Path(alpharep) - assert not root.joinpath('missing.txt').is_file() - - @pass_alpharep - def test_iterdir_on_file(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - with self.assertRaises(ValueError): - a.iterdir() - - @pass_alpharep - def test_subdir_is_dir(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'b').is_dir() - assert (root / 'b/').is_dir() - assert (root / 'g').is_dir() - assert (root / 'g/').is_dir() - - @pass_alpharep - def test_open(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - with a.open(encoding="utf-8") as strm: - data = strm.read() - assert data == "content of a" - - def test_open_write(self): - """ - If the zipfile is open for write, it should be possible to - write bytes or text to it. - """ - zf = zipfile.Path(zipfile.ZipFile(io.BytesIO(), mode='w')) - with zf.joinpath('file.bin').open('wb') as strm: - strm.write(b'binary contents') - with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: - strm.write('text file') - - def test_open_extant_directory(self): - """ - Attempting to open a directory raises IsADirectoryError. - """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) - with self.assertRaises(IsADirectoryError): - zf.joinpath('b').open() - - @pass_alpharep - def test_open_binary_invalid_args(self, alpharep): - root = zipfile.Path(alpharep) - with self.assertRaises(ValueError): - root.joinpath('a.txt').open('rb', encoding='utf-8') - with self.assertRaises(ValueError): - root.joinpath('a.txt').open('rb', 'utf-8') - - def test_open_missing_directory(self): - """ - Attempting to open a missing directory raises FileNotFoundError. - """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) - with self.assertRaises(FileNotFoundError): - zf.joinpath('z').open() - - @pass_alpharep - def test_read(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - assert a.read_text(encoding="utf-8") == "content of a" - assert a.read_bytes() == b"content of a" - - @pass_alpharep - def test_joinpath(self, alpharep): - root = zipfile.Path(alpharep) - a = root.joinpath("a.txt") - assert a.is_file() - e = root.joinpath("b").joinpath("d").joinpath("e.txt") - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_joinpath_multiple(self, alpharep): - root = zipfile.Path(alpharep) - e = root.joinpath("b", "d", "e.txt") - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_traverse_truediv(self, alpharep): - root = zipfile.Path(alpharep) - a = root / "a.txt" - assert a.is_file() - e = root / "b" / "d" / "e.txt" - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_traverse_simplediv(self, alpharep): - """ - Disable the __future__.division when testing traversal. - """ - code = compile( - source="zipfile.Path(alpharep) / 'a'", - filename="(test)", - mode="eval", - dont_inherit=True, - ) - eval(code) - - @pass_alpharep - def test_pathlike_construction(self, alpharep): - """ - zipfile.Path should be constructable from a path-like object - """ - zipfile_ondisk = self.zipfile_ondisk(alpharep) - pathlike = pathlib.Path(str(zipfile_ondisk)) - zipfile.Path(pathlike) - - @pass_alpharep - def test_traverse_pathlike(self, alpharep): - root = zipfile.Path(alpharep) - root / pathlib.Path("a") - - @pass_alpharep - def test_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'a').parent.at == '' - assert (root / 'a' / 'b').parent.at == 'a/' - - @pass_alpharep - def test_dir_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'b').parent.at == '' - assert (root / 'b/').parent.at == '' - - @pass_alpharep - def test_missing_dir_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'missing dir/').parent.at == '' - - @pass_alpharep - def test_mutability(self, alpharep): - """ - If the underlying zipfile is changed, the Path object should - reflect that change. - """ - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - alpharep.writestr('foo.txt', 'foo') - alpharep.writestr('bar/baz.txt', 'baz') - assert any(child.name == 'foo.txt' for child in root.iterdir()) - assert (root / 'foo.txt').read_text(encoding="utf-8") == 'foo' - (baz,) = (root / 'bar').iterdir() - assert baz.read_text(encoding="utf-8") == 'baz' - - HUGE_ZIPFILE_NUM_ENTRIES = 2 ** 13 - - def huge_zipfile(self): - """Create a read-only zipfile with a huge number of entries entries.""" - strm = io.BytesIO() - zf = zipfile.ZipFile(strm, "w") - for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): - zf.writestr(entry, entry) - zf.mode = 'r' - return zf - - def test_joinpath_constant_time(self): - """ - Ensure joinpath on items in zipfile is linear time. - """ - root = zipfile.Path(self.huge_zipfile()) - entries = jaraco.itertools.Counter(root.iterdir()) - for entry in entries: - entry.joinpath('suffix') - # Check the file iterated all items - assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES - - # @func_timeout.func_set_timeout(3) - def test_implied_dirs_performance(self): - data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)] - zipfile.CompleteDirs._implied_dirs(data) - - @pass_alpharep - def test_read_does_not_close(self, alpharep): - alpharep = self.zipfile_ondisk(alpharep) - with zipfile.ZipFile(alpharep) as file: - for rep in range(2): - zipfile.Path(file, 'a.txt').read_text(encoding="utf-8") - - @pass_alpharep - def test_subclass(self, alpharep): - class Subclass(zipfile.Path): - pass - - root = Subclass(alpharep) - assert isinstance(root / 'b', Subclass) - - @pass_alpharep - def test_filename(self, alpharep): - root = zipfile.Path(alpharep) - assert root.filename == pathlib.Path('alpharep.zip') - - @pass_alpharep - def test_root_name(self, alpharep): - """ - The name of the root should be the name of the zipfile - """ - root = zipfile.Path(alpharep) - assert root.name == 'alpharep.zip' == root.filename.name - - @pass_alpharep - def test_root_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert root.parent == pathlib.Path('.') - root.root.filename = 'foo/bar.zip' - assert root.parent == pathlib.Path('foo') - - @pass_alpharep - def test_root_unnamed(self, alpharep): - """ - It is an error to attempt to get the name - or parent of an unnamed zipfile. - """ - alpharep.filename = None - root = zipfile.Path(alpharep) - with self.assertRaises(TypeError): - root.name - with self.assertRaises(TypeError): - root.parent - - # .name and .parent should still work on subs - sub = root / "b" - assert sub.name == "b" - assert sub.parent - - @pass_alpharep - def test_inheritance(self, alpharep): - cls = type('PathChild', (zipfile.Path,), {}) - for alpharep in self.zipfile_alpharep(): - file = cls(alpharep).joinpath('some dir').parent - assert isinstance(file, cls) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_zipfile/__init__.py b/Lib/test/test_zipfile/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_zipfile/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_zipfile/__main__.py b/Lib/test/test_zipfile/__main__.py new file mode 100644 index 00000000000..e25ac946edf --- /dev/null +++ b/Lib/test/test_zipfile/__main__.py @@ -0,0 +1,7 @@ +import unittest + +from . import load_tests # noqa: F401 + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_zipfile/_path/__init__.py b/Lib/test/test_zipfile/_path/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/test_zipfile/_path/_functools.py b/Lib/test/test_zipfile/_path/_functools.py new file mode 100644 index 00000000000..75f2b20e06d --- /dev/null +++ b/Lib/test/test_zipfile/_path/_functools.py @@ -0,0 +1,9 @@ +import functools + + +# from jaraco.functools 3.5.2 +def compose(*funcs): + def compose_two(f1, f2): + return lambda *args, **kwargs: f1(f2(*args, **kwargs)) + + return functools.reduce(compose_two, funcs) diff --git a/Lib/test/test_zipfile/_path/_itertools.py b/Lib/test/test_zipfile/_path/_itertools.py new file mode 100644 index 00000000000..f735dd21733 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_itertools.py @@ -0,0 +1,79 @@ +import itertools +from collections import deque +from itertools import islice + + +# from jaraco.itertools 6.3.0 +class Counter: + """ + Wrap an iterable in an object that stores the count of items + that pass through it. + + >>> items = Counter(range(20)) + >>> items.count + 0 + >>> values = list(items) + >>> items.count + 20 + """ + + def __init__(self, i): + self.count = 0 + self.iter = zip(itertools.count(1), i) + + def __iter__(self): + return self + + def __next__(self): + self.count, result = next(self.iter) + return result + + +# from more_itertools v8.13.0 +def always_iterable(obj, base_type=(str, bytes)): + if obj is None: + return iter(()) + + if (base_type is not None) and isinstance(obj, base_type): + return iter((obj,)) + + try: + return iter(obj) + except TypeError: + return iter((obj,)) + + +# from more_itertools v9.0.0 +def consume(iterator, n=None): + """Advance *iterable* by *n* steps. If *n* is ``None``, consume it + entirely. + Efficiently exhausts an iterator without returning values. Defaults to + consuming the whole iterator, but an optional second argument may be + provided to limit consumption. + >>> i = (x for x in range(10)) + >>> next(i) + 0 + >>> consume(i, 3) + >>> next(i) + 4 + >>> consume(i) + >>> next(i) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + If the iterator has fewer items remaining than the provided limit, the + whole iterator will be consumed. + >>> i = (x for x in range(3)) + >>> consume(i, 5) + >>> next(i) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + """ + # Use functions that consume iterators at C speed. + if n is None: + # feed the entire iterator into a zero-length deque + deque(iterator, maxlen=0) + else: + # advance to the empty slice starting at position n + next(islice(iterator, n, n), None) diff --git a/Lib/test/test_zipfile/_path/_support.py b/Lib/test/test_zipfile/_path/_support.py new file mode 100644 index 00000000000..1afdf3b3a77 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_support.py @@ -0,0 +1,9 @@ +import importlib +import unittest + + +def import_or_skip(name): + try: + return importlib.import_module(name) + except ImportError: # pragma: no cover + raise unittest.SkipTest(f'Unable to import {name}') diff --git a/Lib/test/test_zipfile/_path/_test_params.py b/Lib/test/test_zipfile/_path/_test_params.py new file mode 100644 index 00000000000..00a9eaf2f99 --- /dev/null +++ b/Lib/test/test_zipfile/_path/_test_params.py @@ -0,0 +1,39 @@ +import functools +import types + +from ._itertools import always_iterable + + +def parameterize(names, value_groups): + """ + Decorate a test method to run it as a set of subtests. + + Modeled after pytest.parametrize. + """ + + def decorator(func): + @functools.wraps(func) + def wrapped(self): + for values in value_groups: + resolved = map(Invoked.eval, always_iterable(values)) + params = dict(zip(always_iterable(names), resolved)) + with self.subTest(**params): + func(self, **params) + + return wrapped + + return decorator + + +class Invoked(types.SimpleNamespace): + """ + Wrap a function to be invoked for each usage. + """ + + @classmethod + def wrap(cls, func): + return cls(func=func) + + @classmethod + def eval(cls, cand): + return cand.func() if isinstance(cand, cls) else cand diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py new file mode 100644 index 00000000000..7c108fc6ab8 --- /dev/null +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -0,0 +1,105 @@ +import io +import itertools +import math +import re +import string +import unittest +import zipfile + +from ._functools import compose +from ._itertools import consume +from ._support import import_or_skip + +big_o = import_or_skip('big_o') +pytest = import_or_skip('pytest') + + +class TestComplexity(unittest.TestCase): + @pytest.mark.flaky + def test_implied_dirs_performance(self): + best, others = big_o.big_o( + compose(consume, zipfile._path.CompleteDirs._implied_dirs), + lambda size: [ + '/'.join(string.ascii_lowercase + str(n)) for n in range(size) + ], + max_n=1000, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + def make_zip_path(self, depth=1, width=1) -> zipfile.Path: + """ + Construct a Path with width files at every level of depth. + """ + zf = zipfile.ZipFile(io.BytesIO(), mode='w') + pairs = itertools.product(self.make_deep_paths(depth), self.make_names(width)) + for path, name in pairs: + zf.writestr(f"{path}{name}.txt", b'') + zf.filename = "big un.zip" + return zipfile.Path(zf) + + @classmethod + def make_names(cls, width, letters=string.ascii_lowercase): + """ + >>> list(TestComplexity.make_names(1)) + ['a'] + >>> list(TestComplexity.make_names(2)) + ['a', 'b'] + >>> list(TestComplexity.make_names(30)) + ['aa', 'ab', ..., 'bd'] + >>> list(TestComplexity.make_names(17124)) + ['aaa', 'aab', ..., 'zip'] + """ + # determine how many products are needed to produce width + n_products = max(1, math.ceil(math.log(width, len(letters)))) + inputs = (letters,) * n_products + combinations = itertools.product(*inputs) + names = map(''.join, combinations) + return itertools.islice(names, width) + + @classmethod + def make_deep_paths(cls, depth): + return map(cls.make_deep_path, range(depth)) + + @classmethod + def make_deep_path(cls, depth): + return ''.join(('d/',) * depth) + + def test_baseline_regex_complexity(self): + best, others = big_o.big_o( + lambda path: re.fullmatch(r'[^/]*\\.txt', path), + self.make_deep_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Constant + + @pytest.mark.flaky + def test_glob_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + self.make_zip_path, + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + @pytest.mark.flaky + def test_glob_width(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(width=size), + max_n=100, + min_n=1, + ) + assert best <= big_o.complexities.Linear + + @pytest.mark.flaky + def test_glob_width_and_depth(self): + best, others = big_o.big_o( + lambda path: consume(path.glob('*.txt')), + lambda size: self.make_zip_path(depth=size, width=size), + max_n=10, + min_n=1, + ) + assert best <= big_o.complexities.Linear diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py new file mode 100644 index 00000000000..f34251bc93c --- /dev/null +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -0,0 +1,691 @@ +import contextlib +import io +import itertools +import pathlib +import pickle +import stat +import sys +import time +import unittest +import zipfile +import zipfile._path + +from test.support.os_helper import FakePath, temp_dir + +from ._functools import compose +from ._itertools import Counter +from ._test_params import Invoked, parameterize + + +class jaraco: + class itertools: + Counter = Counter + + +def _make_link(info: zipfile.ZipInfo): # type: ignore[name-defined] + info.external_attr |= stat.S_IFLNK << 16 + + +def build_alpharep_fixture(): + """ + Create a zip file with this structure: + + . + ├── a.txt + ├── n.txt (-> a.txt) + ├── b + │ ├── c.txt + │ ├── d + │ │ └── e.txt + │ └── f.txt + ├── g + │ └── h + │ └── i.txt + └── j + ├── k.bin + ├── l.baz + └── m.bar + + This fixture has the following key characteristics: + + - a file at the root (a) + - a file two levels deep (b/d/e) + - multiple files in a directory (b/c, b/f) + - a directory containing only a directory (g/h) + - a directory with files of different extensions (j/klm) + - a symlink (n) pointing to (a) + + "alpha" because it uses alphabet + "rep" because it's a representative example + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("a.txt", b"content of a") + zf.writestr("b/c.txt", b"content of c") + zf.writestr("b/d/e.txt", b"content of e") + zf.writestr("b/f.txt", b"content of f") + zf.writestr("g/h/i.txt", b"content of i") + zf.writestr("j/k.bin", b"content of k") + zf.writestr("j/l.baz", b"content of l") + zf.writestr("j/m.bar", b"content of m") + zf.writestr("n.txt", b"a.txt") + _make_link(zf.infolist()[-1]) + + zf.filename = "alpharep.zip" + return zf + + +alpharep_generators = [ + Invoked.wrap(build_alpharep_fixture), + Invoked.wrap(compose(zipfile._path.CompleteDirs.inject, build_alpharep_fixture)), +] + +pass_alpharep = parameterize(['alpharep'], alpharep_generators) + + +class TestPath(unittest.TestCase): + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + def zipfile_ondisk(self, alpharep): + tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) + buffer = alpharep.fp + alpharep.close() + path = tmpdir / alpharep.filename + with path.open("wb") as strm: + strm.write(buffer.getvalue()) + return path + + @pass_alpharep + def test_iterdir_and_types(self, alpharep): + root = zipfile.Path(alpharep) + assert root.is_dir() + a, n, b, g, j = root.iterdir() + assert a.is_file() + assert b.is_dir() + assert g.is_dir() + c, f, d = b.iterdir() + assert c.is_file() and f.is_file() + (e,) = d.iterdir() + assert e.is_file() + (h,) = g.iterdir() + (i,) = h.iterdir() + assert i.is_file() + + @pass_alpharep + def test_is_file_missing(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.joinpath('missing.txt').is_file() + + @pass_alpharep + def test_iterdir_on_file(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + with self.assertRaises(ValueError): + a.iterdir() + + @pass_alpharep + def test_subdir_is_dir(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'b').is_dir() + assert (root / 'b/').is_dir() + assert (root / 'g').is_dir() + assert (root / 'g/').is_dir() + + @pass_alpharep + def test_open(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + with a.open(encoding="utf-8") as strm: + data = strm.read() + self.assertEqual(data, "content of a") + with a.open('r', "utf-8") as strm: # not a kw, no gh-101144 TypeError + data = strm.read() + self.assertEqual(data, "content of a") + + def test_open_encoding_utf16(self): + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.writestr("path/16.txt", "This was utf-16".encode("utf-16")) + zf.filename = "test_open_utf16.zip" + root = zipfile.Path(zf) + (path,) = root.iterdir() + u16 = path.joinpath("16.txt") + with u16.open('r', "utf-16") as strm: + data = strm.read() + assert data == "This was utf-16" + with u16.open(encoding="utf-16") as strm: + data = strm.read() + assert data == "This was utf-16" + + def test_open_encoding_errors(self): + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.writestr("path/bad-utf8.bin", b"invalid utf-8: \xff\xff.") + zf.filename = "test_read_text_encoding_errors.zip" + root = zipfile.Path(zf) + (path,) = root.iterdir() + u16 = path.joinpath("bad-utf8.bin") + + # encoding= as a positional argument for gh-101144. + data = u16.read_text("utf-8", errors="ignore") + assert data == "invalid utf-8: ." + with u16.open("r", "utf-8", errors="surrogateescape") as f: + assert f.read() == "invalid utf-8: \udcff\udcff." + + # encoding= both positional and keyword is an error; gh-101144. + with self.assertRaisesRegex(TypeError, "encoding"): + data = u16.read_text("utf-8", encoding="utf-8") + + # both keyword arguments work. + with u16.open("r", encoding="utf-8", errors="strict") as f: + # error during decoding with wrong codec. + with self.assertRaises(UnicodeDecodeError): + f.read() + + @unittest.skipIf( + not getattr(sys.flags, 'warn_default_encoding', 0), + "Requires warn_default_encoding", + ) + @pass_alpharep + def test_encoding_warnings(self, alpharep): + """EncodingWarning must blame the read_text and open calls.""" + assert sys.flags.warn_default_encoding + root = zipfile.Path(alpharep) + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) + root.joinpath("a.txt").read_text() + assert __file__ == wc.filename + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) + root.joinpath("a.txt").open("r").close() + assert __file__ == wc.filename + + def test_open_write(self): + """ + If the zipfile is open for write, it should be possible to + write bytes or text to it. + """ + zf = zipfile.Path(zipfile.ZipFile(io.BytesIO(), mode='w')) + with zf.joinpath('file.bin').open('wb') as strm: + strm.write(b'binary contents') + with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: + strm.write('text file') + + @pass_alpharep + def test_open_extant_directory(self, alpharep): + """ + Attempting to open a directory raises IsADirectoryError. + """ + zf = zipfile.Path(alpharep) + with self.assertRaises(IsADirectoryError): + zf.joinpath('b').open() + + @pass_alpharep + def test_open_binary_invalid_args(self, alpharep): + root = zipfile.Path(alpharep) + with self.assertRaises(ValueError): + root.joinpath('a.txt').open('rb', encoding='utf-8') + with self.assertRaises(ValueError): + root.joinpath('a.txt').open('rb', 'utf-8') + + @pass_alpharep + def test_open_missing_directory(self, alpharep): + """ + Attempting to open a missing directory raises FileNotFoundError. + """ + zf = zipfile.Path(alpharep) + with self.assertRaises(FileNotFoundError): + zf.joinpath('z').open() + + @pass_alpharep + def test_read(self, alpharep): + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + assert a.read_text(encoding="utf-8") == "content of a" + # Also check positional encoding arg (gh-101144). + assert a.read_text("utf-8") == "content of a" + assert a.read_bytes() == b"content of a" + + @pass_alpharep + def test_joinpath(self, alpharep): + root = zipfile.Path(alpharep) + a = root.joinpath("a.txt") + assert a.is_file() + e = root.joinpath("b").joinpath("d").joinpath("e.txt") + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_joinpath_multiple(self, alpharep): + root = zipfile.Path(alpharep) + e = root.joinpath("b", "d", "e.txt") + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_traverse_truediv(self, alpharep): + root = zipfile.Path(alpharep) + a = root / "a.txt" + assert a.is_file() + e = root / "b" / "d" / "e.txt" + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_pathlike_construction(self, alpharep): + """ + zipfile.Path should be constructable from a path-like object + """ + zipfile_ondisk = self.zipfile_ondisk(alpharep) + pathlike = FakePath(str(zipfile_ondisk)) + root = zipfile.Path(pathlike) + root.root.close() + + @pass_alpharep + def test_traverse_pathlike(self, alpharep): + root = zipfile.Path(alpharep) + root / FakePath("a") + + @pass_alpharep + def test_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'a').parent.at == '' + assert (root / 'a' / 'b').parent.at == 'a/' + + @pass_alpharep + def test_dir_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'b').parent.at == '' + assert (root / 'b/').parent.at == '' + + @pass_alpharep + def test_missing_dir_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'missing dir/').parent.at == '' + + @pass_alpharep + def test_mutability(self, alpharep): + """ + If the underlying zipfile is changed, the Path object should + reflect that change. + """ + root = zipfile.Path(alpharep) + a, n, b, g, j = root.iterdir() + alpharep.writestr('foo.txt', 'foo') + alpharep.writestr('bar/baz.txt', 'baz') + assert any(child.name == 'foo.txt' for child in root.iterdir()) + assert (root / 'foo.txt').read_text(encoding="utf-8") == 'foo' + (baz,) = (root / 'bar').iterdir() + assert baz.read_text(encoding="utf-8") == 'baz' + + HUGE_ZIPFILE_NUM_ENTRIES = 2**13 + + def huge_zipfile(self): + """Create a read-only zipfile with a huge number of entries entries.""" + strm = io.BytesIO() + zf = zipfile.ZipFile(strm, "w") + for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): + zf.writestr(entry, entry) + zf.mode = 'r' + return zf + + def test_joinpath_constant_time(self): + """ + Ensure joinpath on items in zipfile is linear time. + """ + root = zipfile.Path(self.huge_zipfile()) + entries = jaraco.itertools.Counter(root.iterdir()) + for entry in entries: + entry.joinpath('suffix') + # Check the file iterated all items + assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES + + @pass_alpharep + def test_read_does_not_close(self, alpharep): + alpharep = self.zipfile_ondisk(alpharep) + with zipfile.ZipFile(alpharep) as file: + for rep in range(2): + zipfile.Path(file, 'a.txt').read_text(encoding="utf-8") + + @pass_alpharep + def test_subclass(self, alpharep): + class Subclass(zipfile.Path): + pass + + root = Subclass(alpharep) + assert isinstance(root / 'b', Subclass) + + @pass_alpharep + def test_filename(self, alpharep): + root = zipfile.Path(alpharep) + assert root.filename == pathlib.Path('alpharep.zip') + + @pass_alpharep + def test_root_name(self, alpharep): + """ + The name of the root should be the name of the zipfile + """ + root = zipfile.Path(alpharep) + assert root.name == 'alpharep.zip' == root.filename.name + + @pass_alpharep + def test_root_on_disk(self, alpharep): + """ + The name/stem of the root should match the zipfile on disk. + + This condition must hold across platforms. + """ + root = zipfile.Path(self.zipfile_ondisk(alpharep)) + assert root.name == 'alpharep.zip' == root.filename.name + assert root.stem == 'alpharep' == root.filename.stem + root.root.close() + + @pass_alpharep + def test_suffix(self, alpharep): + """ + The suffix of the root should be the suffix of the zipfile. + The suffix of each nested file is the final component's last suffix, if any. + Includes the leading period, just like pathlib.Path. + """ + root = zipfile.Path(alpharep) + assert root.suffix == '.zip' == root.filename.suffix + + b = root / "b.txt" + assert b.suffix == ".txt" + + c = root / "c" / "filename.tar.gz" + assert c.suffix == ".gz" + + d = root / "d" + assert d.suffix == "" + + @pass_alpharep + def test_suffixes(self, alpharep): + """ + The suffix of the root should be the suffix of the zipfile. + The suffix of each nested file is the final component's last suffix, if any. + Includes the leading period, just like pathlib.Path. + """ + root = zipfile.Path(alpharep) + assert root.suffixes == ['.zip'] == root.filename.suffixes + + b = root / 'b.txt' + assert b.suffixes == ['.txt'] + + c = root / 'c' / 'filename.tar.gz' + assert c.suffixes == ['.tar', '.gz'] + + d = root / 'd' + assert d.suffixes == [] + + e = root / '.hgrc' + assert e.suffixes == [] + + @pass_alpharep + def test_suffix_no_filename(self, alpharep): + alpharep.filename = None + root = zipfile.Path(alpharep) + assert root.joinpath('example').suffix == "" + assert root.joinpath('example').suffixes == [] + + @pass_alpharep + def test_stem(self, alpharep): + """ + The final path component, without its suffix + """ + root = zipfile.Path(alpharep) + assert root.stem == 'alpharep' == root.filename.stem + + b = root / "b.txt" + assert b.stem == "b" + + c = root / "c" / "filename.tar.gz" + assert c.stem == "filename.tar" + + d = root / "d" + assert d.stem == "d" + + assert (root / ".gitignore").stem == ".gitignore" + + @pass_alpharep + def test_root_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert root.parent == pathlib.Path('.') + root.root.filename = 'foo/bar.zip' + assert root.parent == pathlib.Path('foo') + + @pass_alpharep + def test_root_unnamed(self, alpharep): + """ + It is an error to attempt to get the name + or parent of an unnamed zipfile. + """ + alpharep.filename = None + root = zipfile.Path(alpharep) + with self.assertRaises(TypeError): + root.name + with self.assertRaises(TypeError): + root.parent + + # .name and .parent should still work on subs + sub = root / "b" + assert sub.name == "b" + assert sub.parent + + @pass_alpharep + def test_match_and_glob(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.match("*.txt") + + assert list(root.glob("b/c.*")) == [zipfile.Path(alpharep, "b/c.txt")] + assert list(root.glob("b/*.txt")) == [ + zipfile.Path(alpharep, "b/c.txt"), + zipfile.Path(alpharep, "b/f.txt"), + ] + + @pass_alpharep + def test_glob_recursive(self, alpharep): + root = zipfile.Path(alpharep) + files = root.glob("**/*.txt") + assert all(each.match("*.txt") for each in files) + + assert list(root.glob("**/*.txt")) == list(root.rglob("*.txt")) + + @pass_alpharep + def test_glob_dirs(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('b')) == [zipfile.Path(alpharep, "b/")] + assert list(root.glob('b*')) == [zipfile.Path(alpharep, "b/")] + + @pass_alpharep + def test_glob_subdir(self, alpharep): + root = zipfile.Path(alpharep) + assert list(root.glob('g/h')) == [zipfile.Path(alpharep, "g/h/")] + assert list(root.glob('g*/h*')) == [zipfile.Path(alpharep, "g/h/")] + + @pass_alpharep + def test_glob_subdirs(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*/i.txt")) == [] + assert list(root.rglob("*/i.txt")) == [zipfile.Path(alpharep, "g/h/i.txt")] + + @pass_alpharep + def test_glob_does_not_overmatch_dot(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("*.xt")) == [] + + @pass_alpharep + def test_glob_single_char(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("a?txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[.]txt")) == [zipfile.Path(alpharep, "a.txt")] + assert list(root.glob("a[?]txt")) == [] + + @pass_alpharep + def test_glob_chars(self, alpharep): + root = zipfile.Path(alpharep) + + assert list(root.glob("j/?.b[ai][nz]")) == [ + zipfile.Path(alpharep, "j/k.bin"), + zipfile.Path(alpharep, "j/l.baz"), + ] + + def test_glob_empty(self): + root = zipfile.Path(zipfile.ZipFile(io.BytesIO(), 'w')) + with self.assertRaises(ValueError): + root.glob('') + + @pass_alpharep + def test_eq_hash(self, alpharep): + root = zipfile.Path(alpharep) + assert root == zipfile.Path(alpharep) + + assert root != (root / "a.txt") + assert (root / "a.txt") == (root / "a.txt") + + root = zipfile.Path(alpharep) + assert root in {root} + + @pass_alpharep + def test_is_symlink(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.joinpath('a.txt').is_symlink() + assert root.joinpath('n.txt').is_symlink() + + @pass_alpharep + def test_relative_to(self, alpharep): + root = zipfile.Path(alpharep) + relative = root.joinpath("b", "c.txt").relative_to(root / "b") + assert str(relative) == "c.txt" + + relative = root.joinpath("b", "d", "e.txt").relative_to(root / "b") + assert str(relative) == "d/e.txt" + + @pass_alpharep + def test_inheritance(self, alpharep): + cls = type('PathChild', (zipfile.Path,), {}) + file = cls(alpharep).joinpath('some dir').parent + assert isinstance(file, cls) + + @parameterize( + ['alpharep', 'path_type', 'subpath'], + itertools.product( + alpharep_generators, + [str, FakePath], + ['', 'b/'], + ), + ) + def test_pickle(self, alpharep, path_type, subpath): + zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) + root = zipfile.Path(zipfile_ondisk, at=subpath) + saved_1 = pickle.dumps(root) + root.root.close() + restored_1 = pickle.loads(saved_1) + first, *rest = restored_1.iterdir() + assert first.read_text(encoding='utf-8').startswith('content of ') + restored_1.root.close() + + @pass_alpharep + def test_extract_orig_with_implied_dirs(self, alpharep): + """ + A zip file wrapped in a Path should extract even with implied dirs. + """ + source_path = self.zipfile_ondisk(alpharep) + zf = zipfile.ZipFile(source_path) + # wrap the zipfile for its side effect + zipfile.Path(zf) + zf.extractall(source_path.parent) + zf.close() + + @pass_alpharep + def test_getinfo_missing(self, alpharep): + """ + Validate behavior of getinfo on original zipfile after wrapping. + """ + zipfile.Path(alpharep) + with self.assertRaises(KeyError): + alpharep.getinfo('does-not-exist') + + def test_malformed_paths(self): + """ + Path should handle malformed paths gracefully. + + Paths with leading slashes are not visible. + + Paths with dots are treated like regular files. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("/one-slash.txt", b"content") + zf.writestr("//two-slash.txt", b"content") + zf.writestr("../parent.txt", b"content") + zf.filename = '' + root = zipfile.Path(zf) + assert list(map(str, root.iterdir())) == ['../'] + assert root.joinpath('..').joinpath('parent.txt').read_bytes() == b'content' + + def test_unsupported_names(self): + """ + Path segments with special characters are readable. + + On some platforms or file systems, characters like + ``:`` and ``?`` are not allowed, but they are valid + in the zip file. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("path?", b"content") + zf.writestr("V: NMS.flac", b"fLaC...") + zf.filename = '' + root = zipfile.Path(zf) + contents = root.iterdir() + assert next(contents).name == 'path?' + assert next(contents).name == 'V: NMS.flac' + assert root.joinpath('V: NMS.flac').read_bytes() == b"fLaC..." + + def test_backslash_not_separator(self): + """ + In a zip file, backslashes are not separators. + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr(DirtyZipInfo.for_name("foo\\bar", zf), b"content") + zf.filename = '' + root = zipfile.Path(zf) + (first,) = root.iterdir() + assert not first.is_dir() + assert first.name == 'foo\\bar' + + @pass_alpharep + def test_interface(self, alpharep): + from importlib.resources.abc import Traversable + + zf = zipfile.Path(alpharep) + assert isinstance(zf, Traversable) + + +class DirtyZipInfo(zipfile.ZipInfo): + """ + Bypass name sanitization. + """ + + def __init__(self, filename, *args, **kwargs): + super().__init__(filename, *args, **kwargs) + self.filename = filename + + @classmethod + def for_name(cls, name, archive): + """ + Construct the same way that ZipFile.writestr does. + + TODO: extract this functionality and re-use + """ + self = cls(filename=name, date_time=time.localtime(time.time())[:6]) + self.compress_type = archive.compression + self.compress_level = archive.compresslevel + if self.filename.endswith('/'): # pragma: no cover + self.external_attr = 0o40775 << 16 # drwxrwxr-x + self.external_attr |= 0x10 # MS-DOS directory flag + else: + self.external_attr = 0o600 << 16 # ?rw------- + return self diff --git a/Lib/test/test_zipfile/_path/write-alpharep.py b/Lib/test/test_zipfile/_path/write-alpharep.py new file mode 100644 index 00000000000..7418391abad --- /dev/null +++ b/Lib/test/test_zipfile/_path/write-alpharep.py @@ -0,0 +1,3 @@ +from . import test_path + +__name__ == '__main__' and test_path.build_alpharep_fixture().extractall('alpharep') diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py new file mode 100644 index 00000000000..63413d7b944 --- /dev/null +++ b/Lib/test/test_zipfile/test_core.py @@ -0,0 +1,3738 @@ +import _pyio +import array +import contextlib +import importlib.util +import io +import itertools +import os +import posixpath +import struct +import subprocess +import sys +import time +import unittest +import unittest.mock as mock +import zipfile + + +from tempfile import TemporaryFile +from random import randint, random, randbytes + +from test import archiver_tests +from test.support import script_helper +from test.support import ( + findfile, requires_zlib, requires_bz2, requires_lzma, + captured_stdout, captured_stderr, requires_subprocess +) +from test.support.os_helper import ( + TESTFN, unlink, rmtree, temp_dir, temp_cwd, fd_count, FakePath +) + + +TESTFN2 = TESTFN + "2" +TESTFNDIR = TESTFN + "d" +FIXEDTEST_SIZE = 1000 +DATAFILES_DIR = 'zipfile_datafiles' + +SMALL_TEST_DATA = [('_ziptest1', '1q2w3e4r5t'), + ('ziptest2dir/_ziptest2', 'qawsedrftg'), + ('ziptest2dir/ziptest3dir/_ziptest3', 'azsxdcfvgb'), + ('ziptest2dir/ziptest3dir/ziptest4dir/_ziptest3', '6y7u8i9o0p')] + +def get_files(test): + yield TESTFN2 + with TemporaryFile() as f: + yield f + test.assertFalse(f.closed) + with io.BytesIO() as f: + yield f + test.assertFalse(f.closed) + +class AbstractTestsWithSourceFile: + @classmethod + def setUpClass(cls): + cls.line_gen = [bytes("Zipfile test line %d. random float: %f\n" % + (i, random()), "ascii") + for i in range(FIXEDTEST_SIZE)] + cls.data = b''.join(cls.line_gen) + + def setUp(self): + # Make a source file with some lines + with open(TESTFN, "wb") as fp: + fp.write(self.data) + + def make_test_archive(self, f, compression, compresslevel=None): + kwargs = {'compression': compression, 'compresslevel': compresslevel} + # Create the ZIP archive + with zipfile.ZipFile(f, "w", **kwargs) as zipfp: + zipfp.write(TESTFN, "another.name") + zipfp.write(TESTFN, TESTFN) + zipfp.writestr("strfile", self.data) + with zipfp.open('written-open-w', mode='w') as f: + for line in self.line_gen: + f.write(line) + + def zip_test(self, f, compression, compresslevel=None): + self.make_test_archive(f, compression, compresslevel) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r", compression) as zipfp: + self.assertEqual(zipfp.read(TESTFN), self.data) + self.assertEqual(zipfp.read("another.name"), self.data) + self.assertEqual(zipfp.read("strfile"), self.data) + + # Print the ZIP directory + fp = io.StringIO() + zipfp.printdir(file=fp) + directory = fp.getvalue() + lines = directory.splitlines() + self.assertEqual(len(lines), 5) # Number of files + header + + self.assertIn('File Name', lines[0]) + self.assertIn('Modified', lines[0]) + self.assertIn('Size', lines[0]) + + fn, date, time_, size = lines[1].split() + self.assertEqual(fn, 'another.name') + self.assertTrue(time.strptime(date, '%Y-%m-%d')) + self.assertTrue(time.strptime(time_, '%H:%M:%S')) + self.assertEqual(size, str(len(self.data))) + + # Check the namelist + names = zipfp.namelist() + self.assertEqual(len(names), 4) + self.assertIn(TESTFN, names) + self.assertIn("another.name", names) + self.assertIn("strfile", names) + self.assertIn("written-open-w", names) + + # Check infolist + infos = zipfp.infolist() + names = [i.filename for i in infos] + self.assertEqual(len(names), 4) + self.assertIn(TESTFN, names) + self.assertIn("another.name", names) + self.assertIn("strfile", names) + self.assertIn("written-open-w", names) + for i in infos: + self.assertEqual(i.file_size, len(self.data)) + + # check getinfo + for nm in (TESTFN, "another.name", "strfile", "written-open-w"): + info = zipfp.getinfo(nm) + self.assertEqual(info.filename, nm) + self.assertEqual(info.file_size, len(self.data)) + + # Check that testzip thinks the archive is ok + # (it returns None if all contents could be read properly) + self.assertIsNone(zipfp.testzip()) + + def test_basic(self): + for f in get_files(self): + self.zip_test(f, self.compression) + + def zip_open_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r", compression) as zipfp: + zipdata1 = [] + with zipfp.open(TESTFN) as zipopen1: + while True: + read_data = zipopen1.read(256) + if not read_data: + break + zipdata1.append(read_data) + + zipdata2 = [] + with zipfp.open("another.name") as zipopen2: + while True: + read_data = zipopen2.read(256) + if not read_data: + break + zipdata2.append(read_data) + + self.assertEqual(b''.join(zipdata1), self.data) + self.assertEqual(b''.join(zipdata2), self.data) + + def test_open(self): + for f in get_files(self): + self.zip_open_test(f, self.compression) + + def test_open_with_pathlike(self): + path = FakePath(TESTFN2) + self.zip_open_test(path, self.compression) + with zipfile.ZipFile(path, "r", self.compression) as zipfp: + self.assertIsInstance(zipfp.filename, str) + + def zip_random_open_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r", compression) as zipfp: + zipdata1 = [] + with zipfp.open(TESTFN) as zipopen1: + while True: + read_data = zipopen1.read(randint(1, 1024)) + if not read_data: + break + zipdata1.append(read_data) + + self.assertEqual(b''.join(zipdata1), self.data) + + def test_random_open(self): + for f in get_files(self): + self.zip_random_open_test(f, self.compression) + + def zip_read1_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r") as zipfp, \ + zipfp.open(TESTFN) as zipopen: + zipdata = [] + while True: + read_data = zipopen.read1(-1) + if not read_data: + break + zipdata.append(read_data) + + self.assertEqual(b''.join(zipdata), self.data) + + def test_read1(self): + for f in get_files(self): + self.zip_read1_test(f, self.compression) + + def zip_read1_10_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r") as zipfp, \ + zipfp.open(TESTFN) as zipopen: + zipdata = [] + while True: + read_data = zipopen.read1(10) + self.assertLessEqual(len(read_data), 10) + if not read_data: + break + zipdata.append(read_data) + + self.assertEqual(b''.join(zipdata), self.data) + + def test_read1_10(self): + for f in get_files(self): + self.zip_read1_10_test(f, self.compression) + + def zip_readline_read_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r") as zipfp, \ + zipfp.open(TESTFN) as zipopen: + data = b'' + while True: + read = zipopen.readline() + if not read: + break + data += read + + read = zipopen.read(100) + if not read: + break + data += read + + self.assertEqual(data, self.data) + + def test_readline_read(self): + # Issue #7610: calls to readline() interleaved with calls to read(). + for f in get_files(self): + self.zip_readline_read_test(f, self.compression) + + def zip_readline_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r") as zipfp: + with zipfp.open(TESTFN) as zipopen: + for line in self.line_gen: + linedata = zipopen.readline() + self.assertEqual(linedata, line) + + def test_readline(self): + for f in get_files(self): + self.zip_readline_test(f, self.compression) + + def zip_readlines_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r") as zipfp: + with zipfp.open(TESTFN) as zipopen: + ziplines = zipopen.readlines() + for line, zipline in zip(self.line_gen, ziplines): + self.assertEqual(zipline, line) + + def test_readlines(self): + for f in get_files(self): + self.zip_readlines_test(f, self.compression) + + def zip_iterlines_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r") as zipfp: + with zipfp.open(TESTFN) as zipopen: + for line, zipline in zip(self.line_gen, zipopen): + self.assertEqual(zipline, line) + + def test_iterlines(self): + for f in get_files(self): + self.zip_iterlines_test(f, self.compression) + + def test_low_compression(self): + """Check for cases where compressed data is larger than original.""" + # Create the ZIP archive + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipfp: + zipfp.writestr("strfile", '12') + + # Get an open object for strfile + with zipfile.ZipFile(TESTFN2, "r", self.compression) as zipfp: + with zipfp.open("strfile") as openobj: + self.assertEqual(openobj.read(1), b'1') + self.assertEqual(openobj.read(1), b'2') + + def test_writestr_compression(self): + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.writestr("b.txt", "hello world", compress_type=self.compression) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, self.compression) + + def test_writestr_compresslevel(self): + with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: + zipfp.writestr("a.txt", "hello world", compress_type=self.compression) + zipfp.writestr("b.txt", "hello world", compress_type=self.compression, + compresslevel=2) + + # Compression level follows the constructor. + a_info = zipfp.getinfo('a.txt') + self.assertEqual(a_info.compress_type, self.compression) + self.assertEqual(a_info.compress_level, 1) + + # Compression level is overridden. + b_info = zipfp.getinfo('b.txt') + self.assertEqual(b_info.compress_type, self.compression) + self.assertEqual(b_info._compresslevel, 2) + + def test_read_return_size(self): + # Issue #9837: ZipExtFile.read() shouldn't return more bytes + # than requested. + for test_size in (1, 4095, 4096, 4097, 16384): + file_size = test_size + 1 + junk = randbytes(file_size) + with zipfile.ZipFile(io.BytesIO(), "w", self.compression) as zipf: + zipf.writestr('foo', junk) + with zipf.open('foo', 'r') as fp: + buf = fp.read(test_size) + self.assertEqual(len(buf), test_size) + + def test_truncated_zipfile(self): + fp = io.BytesIO() + with zipfile.ZipFile(fp, mode='w') as zipf: + zipf.writestr('strfile', self.data, compress_type=self.compression) + end_offset = fp.tell() + zipfiledata = fp.getvalue() + + fp = io.BytesIO(zipfiledata) + with zipfile.ZipFile(fp) as zipf: + with zipf.open('strfile') as zipopen: + fp.truncate(end_offset - 20) + with self.assertRaises(EOFError): + zipopen.read() + + fp = io.BytesIO(zipfiledata) + with zipfile.ZipFile(fp) as zipf: + with zipf.open('strfile') as zipopen: + fp.truncate(end_offset - 20) + with self.assertRaises(EOFError): + while zipopen.read(100): + pass + + fp = io.BytesIO(zipfiledata) + with zipfile.ZipFile(fp) as zipf: + with zipf.open('strfile') as zipopen: + fp.truncate(end_offset - 20) + with self.assertRaises(EOFError): + while zipopen.read1(100): + pass + + def test_repr(self): + fname = 'file.name' + for f in get_files(self): + with zipfile.ZipFile(f, 'w', self.compression) as zipfp: + zipfp.write(TESTFN, fname) + r = repr(zipfp) + self.assertIn("mode='w'", r) + + with zipfile.ZipFile(f, 'r') as zipfp: + r = repr(zipfp) + if isinstance(f, str): + self.assertIn('filename=%r' % f, r) + else: + self.assertIn('file=%r' % f, r) + self.assertIn("mode='r'", r) + r = repr(zipfp.getinfo(fname)) + self.assertIn('filename=%r' % fname, r) + self.assertIn('filemode=', r) + self.assertIn('file_size=', r) + if self.compression != zipfile.ZIP_STORED: + self.assertIn('compress_type=', r) + self.assertIn('compress_size=', r) + with zipfp.open(fname) as zipopen: + r = repr(zipopen) + self.assertIn('name=%r' % fname, r) + if self.compression != zipfile.ZIP_STORED: + self.assertIn('compress_type=', r) + self.assertIn('[closed]', repr(zipopen)) + self.assertIn('[closed]', repr(zipfp)) + + def test_compresslevel_basic(self): + for f in get_files(self): + self.zip_test(f, self.compression, compresslevel=9) + + def test_per_file_compresslevel(self): + """Check that files within a Zip archive can have different + compression levels.""" + with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: + zipfp.write(TESTFN, 'compress_1') + zipfp.write(TESTFN, 'compress_9', compresslevel=9) + one_info = zipfp.getinfo('compress_1') + nine_info = zipfp.getinfo('compress_9') + self.assertEqual(one_info._compresslevel, 1) + self.assertEqual(nine_info.compress_level, 9) + + def test_writing_errors(self): + class BrokenFile(io.BytesIO): + def write(self, data): + nonlocal count + if count is not None: + if count == stop: + raise OSError + count += 1 + super().write(data) + + stop = 0 + while True: + testfile = BrokenFile() + count = None + with zipfile.ZipFile(testfile, 'w', self.compression) as zipfp: + with zipfp.open('file1', 'w') as f: + f.write(b'data1') + count = 0 + try: + with zipfp.open('file2', 'w') as f: + f.write(b'data2') + except OSError: + stop += 1 + else: + break + finally: + count = None + with zipfile.ZipFile(io.BytesIO(testfile.getvalue())) as zipfp: + self.assertEqual(zipfp.namelist(), ['file1']) + self.assertEqual(zipfp.read('file1'), b'data1') + + with zipfile.ZipFile(io.BytesIO(testfile.getvalue())) as zipfp: + self.assertEqual(zipfp.namelist(), ['file1', 'file2']) + self.assertEqual(zipfp.read('file1'), b'data1') + self.assertEqual(zipfp.read('file2'), b'data2') + + def test_zipextfile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'rb') + self.assertIs(fid.readable(), True) + self.assertIs(fid.writable(), False) + self.assertIs(fid.seekable(), True) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'rb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertRaises(ValueError, fid.readable) + self.assertIs(fid.writable(), False) + self.assertRaises(ValueError, fid.seekable) + + def tearDown(self): + unlink(TESTFN) + unlink(TESTFN2) + + +class StoredTestsWithSourceFile(AbstractTestsWithSourceFile, + unittest.TestCase): + compression = zipfile.ZIP_STORED + test_low_compression = None + + def zip_test_writestr_permissions(self, f, compression): + # Make sure that writestr and open(... mode='w') create files with + # mode 0600, when they are passed a name rather than a ZipInfo + # instance. + + self.make_test_archive(f, compression) + with zipfile.ZipFile(f, "r") as zipfp: + zinfo = zipfp.getinfo('strfile') + self.assertEqual(zinfo.external_attr, 0o600 << 16) + + zinfo2 = zipfp.getinfo('written-open-w') + self.assertEqual(zinfo2.external_attr, 0o600 << 16) + + def test_writestr_permissions(self): + for f in get_files(self): + self.zip_test_writestr_permissions(f, zipfile.ZIP_STORED) + + def test_absolute_arcnames(self): + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: + zipfp.write(TESTFN, "/absolute") + + with zipfile.ZipFile(TESTFN2, "r", zipfile.ZIP_STORED) as zipfp: + self.assertEqual(zipfp.namelist(), ["absolute"]) + + def test_append_to_zip_file(self): + """Test appending to an existing zipfile.""" + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: + zipfp.write(TESTFN, TESTFN) + + with zipfile.ZipFile(TESTFN2, "a", zipfile.ZIP_STORED) as zipfp: + zipfp.writestr("strfile", self.data) + self.assertEqual(zipfp.namelist(), [TESTFN, "strfile"]) + + def test_append_to_non_zip_file(self): + """Test appending to an existing file that is not a zipfile.""" + # NOTE: this test fails if len(d) < 22 because of the first + # line "fpin.seek(-22, 2)" in _EndRecData + data = b'I am not a ZipFile!'*10 + with open(TESTFN2, 'wb') as f: + f.write(data) + + with zipfile.ZipFile(TESTFN2, "a", zipfile.ZIP_STORED) as zipfp: + zipfp.write(TESTFN, TESTFN) + + with open(TESTFN2, 'rb') as f: + f.seek(len(data)) + with zipfile.ZipFile(f, "r") as zipfp: + self.assertEqual(zipfp.namelist(), [TESTFN]) + self.assertEqual(zipfp.read(TESTFN), self.data) + with open(TESTFN2, 'rb') as f: + self.assertEqual(f.read(len(data)), data) + zipfiledata = f.read() + with io.BytesIO(zipfiledata) as bio, zipfile.ZipFile(bio) as zipfp: + self.assertEqual(zipfp.namelist(), [TESTFN]) + self.assertEqual(zipfp.read(TESTFN), self.data) + + def test_read_concatenated_zip_file(self): + with io.BytesIO() as bio: + with zipfile.ZipFile(bio, 'w', zipfile.ZIP_STORED) as zipfp: + zipfp.write(TESTFN, TESTFN) + zipfiledata = bio.getvalue() + data = b'I am not a ZipFile!'*10 + with open(TESTFN2, 'wb') as f: + f.write(data) + f.write(zipfiledata) + + with zipfile.ZipFile(TESTFN2) as zipfp: + self.assertEqual(zipfp.namelist(), [TESTFN]) + self.assertEqual(zipfp.read(TESTFN), self.data) + + def test_append_to_concatenated_zip_file(self): + with io.BytesIO() as bio: + with zipfile.ZipFile(bio, 'w', zipfile.ZIP_STORED) as zipfp: + zipfp.write(TESTFN, TESTFN) + zipfiledata = bio.getvalue() + data = b'I am not a ZipFile!'*1000000 + with open(TESTFN2, 'wb') as f: + f.write(data) + f.write(zipfiledata) + + with zipfile.ZipFile(TESTFN2, 'a') as zipfp: + self.assertEqual(zipfp.namelist(), [TESTFN]) + zipfp.writestr('strfile', self.data) + + with open(TESTFN2, 'rb') as f: + self.assertEqual(f.read(len(data)), data) + zipfiledata = f.read() + with io.BytesIO(zipfiledata) as bio, zipfile.ZipFile(bio) as zipfp: + self.assertEqual(zipfp.namelist(), [TESTFN, 'strfile']) + self.assertEqual(zipfp.read(TESTFN), self.data) + self.assertEqual(zipfp.read('strfile'), self.data) + + def test_ignores_newline_at_end(self): + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: + zipfp.write(TESTFN, TESTFN) + with open(TESTFN2, 'a', encoding='utf-8') as f: + f.write("\r\n\00\00\00") + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + self.assertIsInstance(zipfp, zipfile.ZipFile) + + def test_ignores_stuff_appended_past_comments(self): + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: + zipfp.comment = b"this is a comment" + zipfp.write(TESTFN, TESTFN) + with open(TESTFN2, 'a', encoding='utf-8') as f: + f.write("abcdef\r\n") + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + self.assertIsInstance(zipfp, zipfile.ZipFile) + self.assertEqual(zipfp.comment, b"this is a comment") + + def test_write_default_name(self): + """Check that calling ZipFile.write without arcname specified + produces the expected result.""" + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.write(TESTFN) + with open(TESTFN, "rb") as f: + self.assertEqual(zipfp.read(TESTFN), f.read()) + + def test_io_on_closed_zipextfile(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + zipfp.writestr(fname, "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with zipfp.open(fname) as fid: + fid.close() + self.assertIs(fid.closed, True) + self.assertRaises(ValueError, fid.read) + self.assertRaises(ValueError, fid.seek, 0) + self.assertRaises(ValueError, fid.tell) + + def test_write_to_readonly(self): + """Check that trying to call write() on a readonly ZipFile object + raises a ValueError.""" + with zipfile.ZipFile(TESTFN2, mode="w") as zipfp: + zipfp.writestr("somefile.txt", "bogus") + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + self.assertRaises(ValueError, zipfp.write, TESTFN) + + with zipfile.ZipFile(TESTFN2, mode="r") as zipfp: + with self.assertRaises(ValueError): + zipfp.open(TESTFN, mode='w') + + def test_add_file_before_1980(self): + # Set atime and mtime to 1970-01-01 + os.utime(TESTFN, (0, 0)) + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + self.assertRaises(ValueError, zipfp.write, TESTFN) + + with zipfile.ZipFile(TESTFN2, "w", strict_timestamps=False) as zipfp: + zipfp.write(TESTFN) + zinfo = zipfp.getinfo(TESTFN) + self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) + + def test_add_file_after_2107(self): + # Set atime and mtime to 2108-12-30 + ts = 4386268800 + try: + time.localtime(ts) + except OverflowError: + self.skipTest(f'time.localtime({ts}) raises OverflowError') + try: + os.utime(TESTFN, (ts, ts)) + except OverflowError: + self.skipTest('Host fs cannot set timestamp to required value.') + + mtime_ns = os.stat(TESTFN).st_mtime_ns + if mtime_ns != (4386268800 * 10**9): + # XFS filesystem is limited to 32-bit timestamp, but the syscall + # didn't fail. Moreover, there is a VFS bug which returns + # a cached timestamp which is different than the value on disk. + # + # Test st_mtime_ns rather than st_mtime to avoid rounding issues. + # + # https://bugzilla.redhat.com/show_bug.cgi?id=1795576 + # https://bugs.python.org/issue39460#msg360952 + self.skipTest(f"Linux VFS/XFS kernel bug detected: {mtime_ns=}") + + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + self.assertRaises(struct.error, zipfp.write, TESTFN) + + with zipfile.ZipFile(TESTFN2, "w", strict_timestamps=False) as zipfp: + zipfp.write(TESTFN) + zinfo = zipfp.getinfo(TESTFN) + self.assertEqual(zinfo.date_time, (2107, 12, 31, 23, 59, 59)) + + +@requires_zlib() +class DeflateTestsWithSourceFile(AbstractTestsWithSourceFile, + unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + + def test_per_file_compression(self): + """Check that files within a Zip archive can have different + compression options.""" + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.write(TESTFN, 'storeme', zipfile.ZIP_STORED) + zipfp.write(TESTFN, 'deflateme', zipfile.ZIP_DEFLATED) + sinfo = zipfp.getinfo('storeme') + dinfo = zipfp.getinfo('deflateme') + self.assertEqual(sinfo.compress_type, zipfile.ZIP_STORED) + self.assertEqual(dinfo.compress_type, zipfile.ZIP_DEFLATED) + +@requires_bz2() +class Bzip2TestsWithSourceFile(AbstractTestsWithSourceFile, + unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma() +class LzmaTestsWithSourceFile(AbstractTestsWithSourceFile, + unittest.TestCase): + compression = zipfile.ZIP_LZMA + + +class AbstractTestZip64InSmallFiles: + # These tests test the ZIP64 functionality without using large files, + # see test_zipfile64 for proper tests. + + @classmethod + def setUpClass(cls): + line_gen = (bytes("Test of zipfile line %d." % i, "ascii") + for i in range(0, FIXEDTEST_SIZE)) + cls.data = b'\n'.join(line_gen) + + def setUp(self): + self._limit = zipfile.ZIP64_LIMIT + self._filecount_limit = zipfile.ZIP_FILECOUNT_LIMIT + zipfile.ZIP64_LIMIT = 1000 + zipfile.ZIP_FILECOUNT_LIMIT = 9 + + # Make a source file with some lines + with open(TESTFN, "wb") as fp: + fp.write(self.data) + + def zip_test(self, f, compression): + # Create the ZIP archive + with zipfile.ZipFile(f, "w", compression, allowZip64=True) as zipfp: + zipfp.write(TESTFN, "another.name") + zipfp.write(TESTFN, TESTFN) + zipfp.writestr("strfile", self.data) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r", compression) as zipfp: + self.assertEqual(zipfp.read(TESTFN), self.data) + self.assertEqual(zipfp.read("another.name"), self.data) + self.assertEqual(zipfp.read("strfile"), self.data) + + # Print the ZIP directory + fp = io.StringIO() + zipfp.printdir(fp) + + directory = fp.getvalue() + lines = directory.splitlines() + self.assertEqual(len(lines), 4) # Number of files + header + + self.assertIn('File Name', lines[0]) + self.assertIn('Modified', lines[0]) + self.assertIn('Size', lines[0]) + + fn, date, time_, size = lines[1].split() + self.assertEqual(fn, 'another.name') + self.assertTrue(time.strptime(date, '%Y-%m-%d')) + self.assertTrue(time.strptime(time_, '%H:%M:%S')) + self.assertEqual(size, str(len(self.data))) + + # Check the namelist + names = zipfp.namelist() + self.assertEqual(len(names), 3) + self.assertIn(TESTFN, names) + self.assertIn("another.name", names) + self.assertIn("strfile", names) + + # Check infolist + infos = zipfp.infolist() + names = [i.filename for i in infos] + self.assertEqual(len(names), 3) + self.assertIn(TESTFN, names) + self.assertIn("another.name", names) + self.assertIn("strfile", names) + for i in infos: + self.assertEqual(i.file_size, len(self.data)) + + # check getinfo + for nm in (TESTFN, "another.name", "strfile"): + info = zipfp.getinfo(nm) + self.assertEqual(info.filename, nm) + self.assertEqual(info.file_size, len(self.data)) + + # Check that testzip thinks the archive is valid + self.assertIsNone(zipfp.testzip()) + + def test_basic(self): + for f in get_files(self): + self.zip_test(f, self.compression) + + def test_too_many_files(self): + # This test checks that more than 64k files can be added to an archive, + # and that the resulting archive can be read properly by ZipFile + zipf = zipfile.ZipFile(TESTFN, "w", self.compression, + allowZip64=True) + zipf.debug = 100 + numfiles = 15 + for i in range(numfiles): + zipf.writestr("foo%08d" % i, "%d" % (i**3 % 57)) + self.assertEqual(len(zipf.namelist()), numfiles) + zipf.close() + + zipf2 = zipfile.ZipFile(TESTFN, "r", self.compression) + self.assertEqual(len(zipf2.namelist()), numfiles) + for i in range(numfiles): + content = zipf2.read("foo%08d" % i).decode('ascii') + self.assertEqual(content, "%d" % (i**3 % 57)) + zipf2.close() + + def test_too_many_files_append(self): + zipf = zipfile.ZipFile(TESTFN, "w", self.compression, + allowZip64=False) + zipf.debug = 100 + numfiles = 9 + for i in range(numfiles): + zipf.writestr("foo%08d" % i, "%d" % (i**3 % 57)) + self.assertEqual(len(zipf.namelist()), numfiles) + with self.assertRaises(zipfile.LargeZipFile): + zipf.writestr("foo%08d" % numfiles, b'') + self.assertEqual(len(zipf.namelist()), numfiles) + zipf.close() + + zipf = zipfile.ZipFile(TESTFN, "a", self.compression, + allowZip64=False) + zipf.debug = 100 + self.assertEqual(len(zipf.namelist()), numfiles) + with self.assertRaises(zipfile.LargeZipFile): + zipf.writestr("foo%08d" % numfiles, b'') + self.assertEqual(len(zipf.namelist()), numfiles) + zipf.close() + + zipf = zipfile.ZipFile(TESTFN, "a", self.compression, + allowZip64=True) + zipf.debug = 100 + self.assertEqual(len(zipf.namelist()), numfiles) + numfiles2 = 15 + for i in range(numfiles, numfiles2): + zipf.writestr("foo%08d" % i, "%d" % (i**3 % 57)) + self.assertEqual(len(zipf.namelist()), numfiles2) + zipf.close() + + zipf2 = zipfile.ZipFile(TESTFN, "r", self.compression) + self.assertEqual(len(zipf2.namelist()), numfiles2) + for i in range(numfiles2): + content = zipf2.read("foo%08d" % i).decode('ascii') + self.assertEqual(content, "%d" % (i**3 % 57)) + zipf2.close() + + def tearDown(self): + zipfile.ZIP64_LIMIT = self._limit + zipfile.ZIP_FILECOUNT_LIMIT = self._filecount_limit + unlink(TESTFN) + unlink(TESTFN2) + + +class StoredTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, + unittest.TestCase): + compression = zipfile.ZIP_STORED + + def large_file_exception_test(self, f, compression): + with zipfile.ZipFile(f, "w", compression, allowZip64=False) as zipfp: + self.assertRaises(zipfile.LargeZipFile, + zipfp.write, TESTFN, "another.name") + + def large_file_exception_test2(self, f, compression): + with zipfile.ZipFile(f, "w", compression, allowZip64=False) as zipfp: + self.assertRaises(zipfile.LargeZipFile, + zipfp.writestr, "another.name", self.data) + + def test_large_file_exception(self): + for f in get_files(self): + self.large_file_exception_test(f, zipfile.ZIP_STORED) + self.large_file_exception_test2(f, zipfile.ZIP_STORED) + + def test_absolute_arcnames(self): + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED, + allowZip64=True) as zipfp: + zipfp.write(TESTFN, "/absolute") + + with zipfile.ZipFile(TESTFN2, "r", zipfile.ZIP_STORED) as zipfp: + self.assertEqual(zipfp.namelist(), ["absolute"]) + + def test_append(self): + # Test that appending to the Zip64 archive doesn't change + # extra fields of existing entries. + with zipfile.ZipFile(TESTFN2, "w", allowZip64=True) as zipfp: + zipfp.writestr("strfile", self.data) + with zipfile.ZipFile(TESTFN2, "r", allowZip64=True) as zipfp: + zinfo = zipfp.getinfo("strfile") + extra = zinfo.extra + with zipfile.ZipFile(TESTFN2, "a", allowZip64=True) as zipfp: + zipfp.writestr("strfile2", self.data) + with zipfile.ZipFile(TESTFN2, "r", allowZip64=True) as zipfp: + zinfo = zipfp.getinfo("strfile") + self.assertEqual(zinfo.extra, extra) + + def make_zip64_file( + self, file_size_64_set=False, file_size_extra=False, + compress_size_64_set=False, compress_size_extra=False, + header_offset_64_set=False, header_offset_extra=False, + extensible_data=b'', + end_of_central_dir_size=None, offset_to_end_of_central_dir=None, + ): + """Generate bytes sequence for a zip with (incomplete) zip64 data. + + The actual values (not the zip 64 0xffffffff values) stored in the file + are: + file_size: 8 + compress_size: 8 + header_offset: 0 + """ + actual_size = 8 + actual_header_offset = 0 + local_zip64_fields = [] + central_zip64_fields = [] + + file_size = actual_size + if file_size_64_set: + file_size = 0xffffffff + if file_size_extra: + local_zip64_fields.append(actual_size) + central_zip64_fields.append(actual_size) + file_size = struct.pack("<L", file_size) + + compress_size = actual_size + if compress_size_64_set: + compress_size = 0xffffffff + if compress_size_extra: + local_zip64_fields.append(actual_size) + central_zip64_fields.append(actual_size) + compress_size = struct.pack("<L", compress_size) + + header_offset = actual_header_offset + if header_offset_64_set: + header_offset = 0xffffffff + if header_offset_extra: + central_zip64_fields.append(actual_header_offset) + header_offset = struct.pack("<L", header_offset) + + local_extra = struct.pack( + '<HH' + 'Q'*len(local_zip64_fields), + 0x0001, + 8*len(local_zip64_fields), + *local_zip64_fields + ) + + central_extra = struct.pack( + '<HH' + 'Q'*len(central_zip64_fields), + 0x0001, + 8*len(central_zip64_fields), + *central_zip64_fields + ) + + central_dir_size = struct.pack('<Q', 58 + 8 * len(central_zip64_fields)) + offset_to_central_dir = struct.pack('<Q', 50 + 8 * len(local_zip64_fields)) + if end_of_central_dir_size is None: + end_of_central_dir_size = 44 + len(extensible_data) + if offset_to_end_of_central_dir is None: + offset_to_end_of_central_dir = (108 + + 8 * len(local_zip64_fields) + + 8 * len(central_zip64_fields)) + + local_extra_length = struct.pack("<H", 4 + 8 * len(local_zip64_fields)) + central_extra_length = struct.pack("<H", 4 + 8 * len(central_zip64_fields)) + + filename = b"test.txt" + content = b"test1234" + filename_length = struct.pack("<H", len(filename)) + zip64_contents = ( + # Local file header + b"PK\x03\x04\x14\x00\x00\x00\x00\x00\x00\x00!\x00\x9e%\xf5\xaf" + + compress_size + + file_size + + filename_length + + local_extra_length + + filename + + local_extra + + content + # Central directory: + + b"PK\x01\x02-\x03-\x00\x00\x00\x00\x00\x00\x00!\x00\x9e%\xf5\xaf" + + compress_size + + file_size + + filename_length + + central_extra_length + + b"\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01" + + header_offset + + filename + + central_extra + # Zip64 end of central directory + + b"PK\x06\x06" + + struct.pack('<Q', end_of_central_dir_size) + + b"-\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + + b"\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00" + + central_dir_size + + offset_to_central_dir + + extensible_data + # Zip64 end of central directory locator + + b"PK\x06\x07\x00\x00\x00\x00" + + struct.pack('<Q', offset_to_end_of_central_dir) + + b"\x01\x00\x00\x00" + # end of central directory + + b"PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00:\x00\x00\x002\x00" + + b"\x00\x00\x00\x00" + ) + return zip64_contents + + def test_bad_zip64_extra(self): + """Missing zip64 extra records raises an exception. + + There are 4 fields that the zip64 format handles (the disk number is + not used in this module and so is ignored here). According to the zip + spec: + The order of the fields in the zip64 extended + information record is fixed, but the fields MUST + only appear if the corresponding Local or Central + directory record field is set to 0xFFFF or 0xFFFFFFFF. + + If the zip64 extra content doesn't contain enough entries for the + number of fields marked with 0xFFFF or 0xFFFFFFFF, we raise an error. + This test mismatches the length of the zip64 extra field and the number + of fields set to indicate the presence of zip64 data. + """ + # zip64 file size present, no fields in extra, expecting one, equals + # missing file size. + missing_file_size_extra = self.make_zip64_file( + file_size_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_file_size_extra)) + self.assertIn('file size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_file_size_extra))) + + # zip64 file size present, zip64 compress size present, one field in + # extra, expecting two, equals missing compress size. + missing_compress_size_extra = self.make_zip64_file( + file_size_64_set=True, + file_size_extra=True, + compress_size_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) + self.assertIn('compress size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra))) + + # zip64 compress size present, no fields in extra, expecting one, + # equals missing compress size. + missing_compress_size_extra = self.make_zip64_file( + compress_size_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) + self.assertIn('compress size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra))) + + # zip64 file size present, zip64 compress size present, zip64 header + # offset present, two fields in extra, expecting three, equals missing + # header offset + missing_header_offset_extra = self.make_zip64_file( + file_size_64_set=True, + file_size_extra=True, + compress_size_64_set=True, + compress_size_extra=True, + header_offset_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) + self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) + + # zip64 compress size present, zip64 header offset present, one field + # in extra, expecting two, equals missing header offset + missing_header_offset_extra = self.make_zip64_file( + file_size_64_set=False, + compress_size_64_set=True, + compress_size_extra=True, + header_offset_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) + self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) + + # zip64 file size present, zip64 header offset present, one field in + # extra, expecting two, equals missing header offset + missing_header_offset_extra = self.make_zip64_file( + file_size_64_set=True, + file_size_extra=True, + compress_size_64_set=False, + header_offset_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) + self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) + + # zip64 header offset present, no fields in extra, expecting one, + # equals missing header offset + missing_header_offset_extra = self.make_zip64_file( + file_size_64_set=False, + compress_size_64_set=False, + header_offset_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) + self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) + + def test_bad_zip64_end_of_central_dir(self): + zipdata = self.make_zip64_file(end_of_central_dir_size=0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(end_of_central_dir_size=100) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(offset_to_end_of_central_dir=0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(offset_to_end_of_central_dir=1000) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*locator'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + def test_zip64_end_of_central_dir_record_not_found(self): + zipdata = self.make_zip64_file() + zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4) + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file( + extensible_data=b'\xca\xfe\x04\x00\x00\x00data') + zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4) + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + def test_zip64_extensible_data(self): + # These values are what is set in the make_zip64_file method. + expected_file_size = 8 + expected_compress_size = 8 + expected_header_offset = 0 + expected_content = b"test1234" + + zipdata = self.make_zip64_file( + extensible_data=b'\xca\xfe\x04\x00\x00\x00data') + with zipfile.ZipFile(io.BytesIO(zipdata)) as zf: + zinfo = zf.infolist()[0] + self.assertEqual(zinfo.file_size, expected_file_size) + self.assertEqual(zinfo.compress_size, expected_compress_size) + self.assertEqual(zinfo.header_offset, expected_header_offset) + self.assertEqual(zf.read(zinfo), expected_content) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(zipdata))) + + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(b'prepended' + zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(b'prepended' + zipdata))) + + def test_generated_valid_zip64_extra(self): + # These values are what is set in the make_zip64_file method. + expected_file_size = 8 + expected_compress_size = 8 + expected_header_offset = 0 + expected_content = b"test1234" + + # Loop through the various valid combinations of zip64 masks + # present and extra fields present. + params = ( + {"file_size_64_set": True, "file_size_extra": True}, + {"compress_size_64_set": True, "compress_size_extra": True}, + {"header_offset_64_set": True, "header_offset_extra": True}, + ) + + for r in range(1, len(params) + 1): + for combo in itertools.combinations(params, r): + kwargs = {} + for c in combo: + kwargs.update(c) + with zipfile.ZipFile(io.BytesIO(self.make_zip64_file(**kwargs))) as zf: + zinfo = zf.infolist()[0] + self.assertEqual(zinfo.file_size, expected_file_size) + self.assertEqual(zinfo.compress_size, expected_compress_size) + self.assertEqual(zinfo.header_offset, expected_header_offset) + self.assertEqual(zf.read(zinfo), expected_content) + + def test_force_zip64(self): + """Test that forcing zip64 extensions correctly notes this in the zip file""" + + # GH-103861 describes an issue where forcing a small file to use zip64 + # extensions would add a zip64 extra record, but not change the data + # sizes to 0xFFFFFFFF to indicate to the extractor that the zip64 + # record should be read. Additionally, it would not set the required + # version to indicate that zip64 extensions are required to extract it. + # This test replicates the situation and reads the raw data to specifically ensure: + # - The required extract version is always >= ZIP64_VERSION + # - The compressed and uncompressed size in the file headers are both + # 0xFFFFFFFF (ie. point to zip64 record) + # - The zip64 record is provided and has the correct sizes in it + # Other aspects of the zip are checked as well, but verifying the above is the main goal. + # Because this is hard to verify by parsing the data as a zip, the raw + # bytes are checked to ensure that they line up with the zip spec. + # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + # The relevant sections for this test are: + # - 4.3.7 for local file header + # - 4.5.3 for zip64 extra field + + data = io.BytesIO() + with zipfile.ZipFile(data, mode="w", allowZip64=True) as zf: + with zf.open("text.txt", mode="w", force_zip64=True) as zi: + zi.write(b"_") + + zipdata = data.getvalue() + + # pull out and check zip information + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQx4s", zipdata[:63]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual(flags, 0) # no flags + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, 1) # uncompressed size + self.assertEqual(ex_csize, 1) # compressed size + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + + z = zipfile.ZipFile(io.BytesIO(zipdata)) + zinfos = z.infolist() + self.assertEqual(len(zinfos), 1) + self.assertGreaterEqual(zinfos[0].extract_version, zipfile.ZIP64_VERSION) # requires zip64 to extract + + def test_unseekable_zip_unknown_filesize(self): + """Test that creating a zip with/without seeking will raise a RuntimeError if zip64 was required but not used""" + + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=True) as zf: + with zf.open("text.txt", mode="w", force_zip64=False) as zi: + zi.write(b"_" * (zipfile.ZIP64_LIMIT + 1)) + + self.assertRaises(RuntimeError, make_zip, io.BytesIO()) + self.assertRaises(RuntimeError, make_zip, Unseekable(io.BytesIO())) + + def test_zip64_required_not_allowed_fail(self): + """Test that trying to add a large file to a zip that doesn't allow zip64 extensions fails on add""" + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=False) as zf: + # pretend zipfile.ZipInfo.from_file was used to get the name and filesize + info = zipfile.ZipInfo("text.txt") + info.file_size = zipfile.ZIP64_LIMIT + 1 + zf.open(info, mode="w") + + self.assertRaises(zipfile.LargeZipFile, make_zip, io.BytesIO()) + self.assertRaises(zipfile.LargeZipFile, make_zip, Unseekable(io.BytesIO())) + + def test_unseekable_zip_known_filesize(self): + """Test that creating a zip without seeking will use zip64 extensions if the file size is provided up-front""" + + # This test ensures that the zip will use a zip64 data descriptor (same + # as a regular data descriptor except the sizes are 8 bytes instead of + # 4) record to communicate the size of a file if the zip is being + # written to an unseekable stream. + # Because this sort of thing is hard to verify by parsing the data back + # in as a zip, this test looks at the raw bytes created to ensure that + # the correct data has been generated. + # The spec for this can be found at: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + # The relevant sections for this test are: + # - 4.3.7 for local file header + # - 4.3.9 for the data descriptor + # - 4.5.3 for zip64 extra field + + file_size = zipfile.ZIP64_LIMIT + 1 + + def make_zip(fp): + with zipfile.ZipFile(fp, mode="w", allowZip64=True) as zf: + # pretend zipfile.ZipInfo.from_file was used to get the name and filesize + info = zipfile.ZipInfo("text.txt") + info.file_size = file_size + with zf.open(info, mode="w", force_zip64=False) as zi: + zi.write(b"_" * file_size) + return fp + + # check seekable file information + seekable_data = make_zip(io.BytesIO()).getvalue() + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, + cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQ{}x4s".format(file_size), seekable_data[:62 + file_size]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual(flags, 0) # no flags set + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, file_size) # uncompressed size + self.assertEqual(ex_csize, file_size) # compressed size + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + + # check unseekable file information + unseekable_data = make_zip(Unseekable(io.BytesIO())).fp.getvalue() + ( + header, vers, os, flags, comp, csize, usize, fn_len, + ex_total_len, filename, ex_id, ex_len, ex_usize, ex_csize, + dd_header, dd_usize, dd_csize, cd_sig + ) = struct.unpack("<4sBBHH8xIIHH8shhQQ{}x4s4xQQ4s".format(file_size), unseekable_data[:86 + file_size]) + + self.assertEqual(header, b"PK\x03\x04") # local file header + self.assertGreaterEqual(vers, zipfile.ZIP64_VERSION) # requires zip64 to extract + self.assertEqual(os, 0) # compatible with MS-DOS + self.assertEqual("{:b}".format(flags), "1000") # streaming flag set + self.assertEqual(comp, 0) # compression method = stored + self.assertEqual(csize, 0xFFFFFFFF) # sizes are in zip64 extra + self.assertEqual(usize, 0xFFFFFFFF) + self.assertEqual(fn_len, 8) # filename len + self.assertEqual(ex_total_len, 20) # size of extra records + self.assertEqual(ex_id, 1) # Zip64 extra record + self.assertEqual(ex_len, 16) # 16 bytes of data + self.assertEqual(ex_usize, 0) # uncompressed size - 0 to defer to data descriptor + self.assertEqual(ex_csize, 0) # compressed size - 0 to defer to data descriptor + self.assertEqual(dd_header, b"PK\07\x08") # data descriptor + self.assertEqual(dd_usize, file_size) # file size (8 bytes because zip64) + self.assertEqual(dd_csize, file_size) # compressed size (8 bytes because zip64) + self.assertEqual(cd_sig, b"PK\x01\x02") # ensure the central directory header is next + + +@requires_zlib() +class DeflateTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, + unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2() +class Bzip2TestZip64InSmallFiles(AbstractTestZip64InSmallFiles, + unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma() +class LzmaTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, + unittest.TestCase): + compression = zipfile.ZIP_LZMA + + +class AbstractWriterTests: + + def tearDown(self): + unlink(TESTFN2) + + def test_close_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + w.close() + self.assertTrue(w.closed) + self.assertEqual(zipf.read('test'), data) + + def test_write_after_close(self): + data = b'content' + with zipfile.ZipFile(TESTFN2, "w", self.compression) as zipf: + w = zipf.open('test', 'w') + w.write(data) + w.close() + self.assertTrue(w.closed) + self.assertRaises(ValueError, w.write, b'') + self.assertEqual(zipf.read('test'), data) + + def test_issue44439(self): + q = array.array('Q', [1, 2, 3, 4, 5]) + LENGTH = len(q) * q.itemsize + with zipfile.ZipFile(io.BytesIO(), 'w', self.compression) as zip: + with zip.open('data', 'w') as data: + self.assertEqual(data.write(q), LENGTH) + self.assertEqual(zip.getinfo('data').file_size, LENGTH) + + def test_zipwritefile_attrs(self): + fname = "somefile.txt" + with zipfile.ZipFile(TESTFN2, mode="w", compression=self.compression) as zipfp: + with zipfp.open(fname, 'w') as fid: + self.assertEqual(fid.name, fname) + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertEqual(fid.mode, 'wb') + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + self.assertIs(fid.closed, False) + self.assertIs(fid.closed, True) + self.assertEqual(fid.name, fname) + self.assertEqual(fid.mode, 'wb') + self.assertRaises(io.UnsupportedOperation, fid.fileno) + self.assertIs(fid.readable(), False) + self.assertIs(fid.writable(), True) + self.assertIs(fid.seekable(), False) + +class StoredWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_STORED + +@requires_zlib() +class DeflateWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2() +class Bzip2WriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma() +class LzmaWriterTests(AbstractWriterTests, unittest.TestCase): + compression = zipfile.ZIP_LZMA + + +class PyZipFileTests(unittest.TestCase): + def assertCompiledIn(self, name, namelist): + if name + 'o' not in namelist: + self.assertIn(name + 'c', namelist) + + def requiresWriteAccess(self, path): + # effective_ids unavailable on windows + if not os.access(path, os.W_OK, + effective_ids=os.access in os.supports_effective_ids): + self.skipTest('requires write access to the installed location') + filename = os.path.join(path, 'test_zipfile.try') + try: + fd = os.open(filename, os.O_WRONLY | os.O_CREAT) + os.close(fd) + except Exception: + self.skipTest('requires write access to the installed location') + unlink(filename) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_write_pyfile(self): + self.requiresWriteAccess(os.path.dirname(__file__)) + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + fn = __file__ + if fn.endswith('.pyc'): + path_split = fn.split(os.sep) + if os.altsep is not None: + path_split.extend(fn.split(os.altsep)) + if '__pycache__' in path_split: + fn = importlib.util.source_from_cache(fn) + else: + fn = fn[:-1] + + zipfp.writepy(fn) + + bn = os.path.basename(fn) + self.assertNotIn(bn, zipfp.namelist()) + self.assertCompiledIn(bn, zipfp.namelist()) + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + fn = __file__ + if fn.endswith('.pyc'): + fn = fn[:-1] + + zipfp.writepy(fn, "testpackage") + + bn = "%s/%s" % ("testpackage", os.path.basename(fn)) + self.assertNotIn(bn, zipfp.namelist()) + self.assertCompiledIn(bn, zipfp.namelist()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_write_python_package(self): + import email + packagedir = os.path.dirname(email.__file__) + self.requiresWriteAccess(packagedir) + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(packagedir) + + # Check for a couple of modules at different levels of the + # hierarchy + names = zipfp.namelist() + self.assertCompiledIn('email/__init__.py', names) + self.assertCompiledIn('email/mime/text.py', names) + + @unittest.expectedFailure # TODO: RUSTPYTHON; - AttributeError: module 'os' has no attribute 'supports_effective_ids' + def test_write_filtered_python_package(self): + import test + packagedir = os.path.dirname(test.__file__) + self.requiresWriteAccess(packagedir) + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + + # first make sure that the test folder gives error messages + # (on the badsyntax_... files) + with captured_stdout() as reportSIO: + zipfp.writepy(packagedir) + reportStr = reportSIO.getvalue() + self.assertTrue('SyntaxError' in reportStr) + + # then check that the filter works on the whole package + with captured_stdout() as reportSIO: + zipfp.writepy(packagedir, filterfunc=lambda whatever: False) + reportStr = reportSIO.getvalue() + self.assertTrue('SyntaxError' not in reportStr) + + # then check that the filter works on individual files + def filter(path): + return not os.path.basename(path).startswith("bad") + with captured_stdout() as reportSIO, self.assertWarns(UserWarning): + zipfp.writepy(packagedir, filterfunc=filter) + reportStr = reportSIO.getvalue() + if reportStr: + print(reportStr) + self.assertTrue('SyntaxError' not in reportStr) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_write_with_optimization(self): + import email + packagedir = os.path.dirname(email.__file__) + self.requiresWriteAccess(packagedir) + optlevel = 1 if __debug__ else 0 + ext = '.pyc' + + with TemporaryFile() as t, \ + zipfile.PyZipFile(t, "w", optimize=optlevel) as zipfp: + zipfp.writepy(packagedir) + + names = zipfp.namelist() + self.assertIn('email/__init__' + ext, names) + self.assertIn('email/mime/text' + ext, names) + + def test_write_python_directory(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w", encoding='utf-8') as fp: + fp.write("print(42)\n") + + with open(os.path.join(TESTFN2, "mod2.py"), "w", encoding='utf-8') as fp: + fp.write("print(42 * 42)\n") + + with open(os.path.join(TESTFN2, "mod2.txt"), "w", encoding='utf-8') as fp: + fp.write("bla bla bla\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(TESTFN2) + + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + self.assertCompiledIn('mod2.py', names) + self.assertNotIn('mod2.txt', names) + + finally: + rmtree(TESTFN2) + + def test_write_python_directory_filtered(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w", encoding='utf-8') as fp: + fp.write("print(42)\n") + + with open(os.path.join(TESTFN2, "mod2.py"), "w", encoding='utf-8') as fp: + fp.write("print(42 * 42)\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(TESTFN2, filterfunc=lambda fn: + not fn.endswith('mod2.py')) + + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + self.assertNotIn('mod2.py', names) + + finally: + rmtree(TESTFN2) + + def test_write_non_pyfile(self): + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + with open(TESTFN, 'w', encoding='utf-8') as f: + f.write('most definitely not a python file') + self.assertRaises(RuntimeError, zipfp.writepy, TESTFN) + unlink(TESTFN) + + def test_write_pyfile_bad_syntax(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w", encoding='utf-8') as fp: + fp.write("Bad syntax in python file\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + # syntax errors are printed to stdout + with captured_stdout() as s: + zipfp.writepy(os.path.join(TESTFN2, "mod1.py")) + + self.assertIn("SyntaxError", s.getvalue()) + + # as it will not have compiled the python file, it will + # include the .py file not .pyc + names = zipfp.namelist() + self.assertIn('mod1.py', names) + self.assertNotIn('mod1.pyc', names) + + finally: + rmtree(TESTFN2) + + def test_write_pathlike(self): + os.mkdir(TESTFN2) + try: + with open(os.path.join(TESTFN2, "mod1.py"), "w", encoding='utf-8') as fp: + fp.write("print(42)\n") + + with TemporaryFile() as t, zipfile.PyZipFile(t, "w") as zipfp: + zipfp.writepy(FakePath(os.path.join(TESTFN2, "mod1.py"))) + names = zipfp.namelist() + self.assertCompiledIn('mod1.py', names) + finally: + rmtree(TESTFN2) + + +class ExtractTests(unittest.TestCase): + + def make_test_file(self): + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + zipfp.writestr(fpath, fdata) + + def test_extract(self): + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + writtenfile = zipfp.extract(fpath) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fpath) + correctfile = os.path.normpath(correctfile) + + self.assertEqual(writtenfile, correctfile) + + # make sure correct data is in correct file + with open(writtenfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(writtenfile) + + def _test_extract_with_target(self, target): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + writtenfile = zipfp.extract(fpath, target) + + # make sure it was written to the right place + correctfile = os.path.join(target, fpath) + correctfile = os.path.normpath(correctfile) + self.assertTrue(os.path.samefile(writtenfile, correctfile), (writtenfile, target)) + + # make sure correct data is in correct file + with open(writtenfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(writtenfile) + + unlink(TESTFN2) + + def test_extract_with_target(self): + with temp_dir() as extdir: + self._test_extract_with_target(extdir) + + def test_extract_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_with_target(FakePath(extdir)) + + def test_extract_all(self): + with temp_cwd(): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + zipfp.extractall() + for fpath, fdata in SMALL_TEST_DATA: + outfile = os.path.join(os.getcwd(), fpath) + + with open(outfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(outfile) + + def _test_extract_all_with_target(self, target): + self.make_test_file() + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + zipfp.extractall(target) + for fpath, fdata in SMALL_TEST_DATA: + outfile = os.path.join(target, fpath) + + with open(outfile, "rb") as f: + self.assertEqual(fdata.encode(), f.read()) + + unlink(outfile) + + unlink(TESTFN2) + + def test_extract_all_with_target(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(extdir) + + def test_extract_all_with_target_pathlike(self): + with temp_dir() as extdir: + self._test_extract_all_with_target(FakePath(extdir)) + + def check_file(self, filename, content): + self.assertTrue(os.path.isfile(filename)) + with open(filename, 'rb') as f: + self.assertEqual(f.read(), content) + + def test_sanitize_windows_name(self): + san = zipfile.ZipFile._sanitize_windows_name + # Passing pathsep in allows this test to work regardless of platform. + self.assertEqual(san(r',,?,C:,foo,bar/z', ','), r'_,C_,foo,bar/z') + self.assertEqual(san(r'a\b,c<d>e|f"g?h*i', ','), r'a\b,c_d_e_f_g_h_i') + self.assertEqual(san('../../foo../../ba..r', '/'), r'foo/ba..r') + self.assertEqual(san(' / /foo / /ba r', '/'), r'foo/ba r') + self.assertEqual(san(' . /. /foo ./ . /. ./ba .r', '/'), r'foo/ba .r') + + def test_extract_hackers_arcnames_common_cases(self): + common_hacknames = [ + ('../foo/bar', 'foo/bar'), + ('foo/../bar', 'foo/bar'), + ('foo/../../bar', 'foo/bar'), + ('foo/bar/..', 'foo/bar'), + ('./../foo/bar', 'foo/bar'), + ('/foo/bar', 'foo/bar'), + ('/foo/../bar', 'foo/bar'), + ('/foo/../../bar', 'foo/bar'), + ] + self._test_extract_hackers_arcnames(common_hacknames) + + @unittest.skipIf(os.path.sep != '\\', 'Requires \\ as path separator.') + def test_extract_hackers_arcnames_windows_only(self): + """Test combination of path fixing and windows name sanitization.""" + windows_hacknames = [ + (r'..\foo\bar', 'foo/bar'), + (r'..\/foo\/bar', 'foo/bar'), + (r'foo/\..\/bar', 'foo/bar'), + (r'foo\/../\bar', 'foo/bar'), + (r'C:foo/bar', 'foo/bar'), + (r'C:/foo/bar', 'foo/bar'), + (r'C://foo/bar', 'foo/bar'), + (r'C:\foo\bar', 'foo/bar'), + (r'//conky/mountpoint/foo/bar', 'foo/bar'), + (r'\\conky\mountpoint\foo\bar', 'foo/bar'), + (r'///conky/mountpoint/foo/bar', 'mountpoint/foo/bar'), + (r'\\\conky\mountpoint\foo\bar', 'mountpoint/foo/bar'), + (r'//conky//mountpoint/foo/bar', 'mountpoint/foo/bar'), + (r'\\conky\\mountpoint\foo\bar', 'mountpoint/foo/bar'), + (r'//?/C:/foo/bar', 'foo/bar'), + (r'\\?\C:\foo\bar', 'foo/bar'), + (r'C:/../C:/foo/bar', 'C_/foo/bar'), + (r'a:b\c<d>e|f"g?h*i', 'b/c_d_e_f_g_h_i'), + ('../../foo../../ba..r', 'foo/ba..r'), + ] + self._test_extract_hackers_arcnames(windows_hacknames) + + @unittest.skipIf(os.path.sep != '/', r'Requires / as path separator.') + def test_extract_hackers_arcnames_posix_only(self): + posix_hacknames = [ + ('//foo/bar', 'foo/bar'), + ('../../foo../../ba..r', 'foo../ba..r'), + (r'foo/..\bar', r'foo/..\bar'), + ] + self._test_extract_hackers_arcnames(posix_hacknames) + + def _test_extract_hackers_arcnames(self, hacknames): + for arcname, fixedname in hacknames: + content = b'foobar' + arcname.encode() + with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_STORED) as zipfp: + zinfo = zipfile.ZipInfo() + # preserve backslashes + zinfo.filename = arcname + zinfo.external_attr = 0o600 << 16 + zipfp.writestr(zinfo, content) + + arcname = arcname.replace(os.sep, "/") + targetpath = os.path.join('target', 'subdir', 'subsub') + correctfile = os.path.join(targetpath, *fixedname.split('/')) + + with zipfile.ZipFile(TESTFN2, 'r') as zipfp: + writtenfile = zipfp.extract(arcname, targetpath) + self.assertEqual(writtenfile, correctfile, + msg='extract %r: %r != %r' % + (arcname, writtenfile, correctfile)) + self.check_file(correctfile, content) + rmtree('target') + + with zipfile.ZipFile(TESTFN2, 'r') as zipfp: + zipfp.extractall(targetpath) + self.check_file(correctfile, content) + rmtree('target') + + correctfile = os.path.join(os.getcwd(), *fixedname.split('/')) + + with zipfile.ZipFile(TESTFN2, 'r') as zipfp: + writtenfile = zipfp.extract(arcname) + self.assertEqual(writtenfile, correctfile, + msg="extract %r" % arcname) + self.check_file(correctfile, content) + rmtree(fixedname.split('/')[0]) + + with zipfile.ZipFile(TESTFN2, 'r') as zipfp: + zipfp.extractall() + self.check_file(correctfile, content) + rmtree(fixedname.split('/')[0]) + + unlink(TESTFN2) + + +class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): + testdir = TESTFN + + @classmethod + def setUpClass(cls): + p = cls.ar_with_file = TESTFN + '-with-file.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test', b'newcontent') + + p = cls.ar_with_dir = TESTFN + '-with-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.mkdir('test') + + p = cls.ar_with_implicit_dir = TESTFN + '-with-implicit-dir.zip' + cls.addClassCleanup(unlink, p) + with zipfile.ZipFile(p, 'w') as zipfp: + zipfp.writestr('test/file', b'newcontent') + + def open(self, path): + return zipfile.ZipFile(path, 'r') + + def extractall(self, ar): + ar.extractall(self.testdir) + + +class OtherTests(unittest.TestCase): + def test_open_via_zip_info(self): + # Create the ZIP archive + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: + zipfp.writestr("name", "foo") + with self.assertWarns(UserWarning): + zipfp.writestr("name", "bar") + self.assertEqual(zipfp.namelist(), ["name"] * 2) + + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + infos = zipfp.infolist() + data = b"" + for info in infos: + with zipfp.open(info) as zipopen: + data += zipopen.read() + self.assertIn(data, {b"foobar", b"barfoo"}) + data = b"" + for info in infos: + data += zipfp.read(info) + self.assertIn(data, {b"foobar", b"barfoo"}) + + def test_writestr_extended_local_header_issue1202(self): + with zipfile.ZipFile(TESTFN2, 'w') as orig_zip: + for data in 'abcdefghijklmnop': + zinfo = zipfile.ZipInfo(data) + zinfo.flag_bits |= zipfile._MASK_USE_DATA_DESCRIPTOR # Include an extended local header. + orig_zip.writestr(zinfo, data) + + def test_close(self): + """Check that the zipfile is closed after the 'with' block.""" + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + zipfp.writestr(fpath, fdata) + self.assertIsNotNone(zipfp.fp, 'zipfp is not open') + self.assertIsNone(zipfp.fp, 'zipfp is not closed') + + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + self.assertIsNotNone(zipfp.fp, 'zipfp is not open') + self.assertIsNone(zipfp.fp, 'zipfp is not closed') + + def test_close_on_exception(self): + """Check that the zipfile is closed if an exception is raised in the + 'with' block.""" + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + for fpath, fdata in SMALL_TEST_DATA: + zipfp.writestr(fpath, fdata) + + try: + with zipfile.ZipFile(TESTFN2, "r") as zipfp2: + raise zipfile.BadZipFile() + except zipfile.BadZipFile: + self.assertIsNone(zipfp2.fp, 'zipfp is not closed') + + def test_unsupported_version(self): + # File has an extract_version of 120 + data = (b'PK\x03\x04x\x00\x00\x00\x00\x00!p\xa1@\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00xPK\x01\x02x\x03x\x00\x00\x00\x00' + b'\x00!p\xa1@\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00\x00xPK\x05\x06' + b'\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x1f\x00\x00\x00\x00\x00') + + self.assertRaises(NotImplementedError, zipfile.ZipFile, + io.BytesIO(data), 'r') + + @requires_zlib() + def test_read_unicode_filenames(self): + # bug #10801 + fname = findfile('zip_cp437_header.zip', subdir='archivetestdata') + with zipfile.ZipFile(fname) as zipfp: + for name in zipfp.namelist(): + zipfp.open(name).close() + + def test_write_unicode_filenames(self): + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.writestr("foo.txt", "Test for unicode filename") + zf.writestr("\xf6.txt", "Test for unicode filename") + self.assertIsInstance(zf.infolist()[0].filename, str) + + with zipfile.ZipFile(TESTFN, "r") as zf: + self.assertEqual(zf.filelist[0].filename, "foo.txt") + self.assertEqual(zf.filelist[1].filename, "\xf6.txt") + + def create_zipfile_with_extra_data(self, filename, extra_data_name): + with zipfile.ZipFile(TESTFN, mode='w') as zf: + filename_encoded = filename.encode("utf-8") + # create a ZipInfo object with Unicode path extra field + zip_info = zipfile.ZipInfo(filename) + + tag_for_unicode_path = b'\x75\x70' + version_of_unicode_path = b'\x01' + + import zlib + filename_crc = struct.pack('<L', zlib.crc32(filename_encoded)) + + extra_data = version_of_unicode_path + filename_crc + extra_data_name + tsize = len(extra_data).to_bytes(2, 'little') + + zip_info.extra = tag_for_unicode_path + tsize + extra_data + + # add the file to the ZIP archive + zf.writestr(zip_info, b'Hello World!') + + @requires_zlib() + def test_read_zipfile_containing_unicode_path_extra_field(self): + self.create_zipfile_with_extra_data("이름.txt", "이름.txt".encode("utf-8")) + with zipfile.ZipFile(TESTFN, "r") as zf: + self.assertEqual(zf.filelist[0].filename, "이름.txt") + + @requires_zlib() + def test_read_zipfile_warning(self): + self.create_zipfile_with_extra_data("이름.txt", b"") + with self.assertWarns(UserWarning): + zipfile.ZipFile(TESTFN, "r").close() + + @requires_zlib() + def test_read_zipfile_error(self): + self.create_zipfile_with_extra_data("이름.txt", b"\xff") + with self.assertRaises(zipfile.BadZipfile): + zipfile.ZipFile(TESTFN, "r").close() + + def test_read_after_write_unicode_filenames(self): + with zipfile.ZipFile(TESTFN2, 'w') as zipfp: + zipfp.writestr('приклад', b'sample') + self.assertEqual(zipfp.read('приклад'), b'sample') + + def test_exclusive_create_zip_file(self): + """Test exclusive creating a new zipfile.""" + unlink(TESTFN2) + filename = 'testfile.txt' + content = b'hello, world. this is some content.' + with zipfile.ZipFile(TESTFN2, "x", zipfile.ZIP_STORED) as zipfp: + zipfp.writestr(filename, content) + with self.assertRaises(FileExistsError): + zipfile.ZipFile(TESTFN2, "x", zipfile.ZIP_STORED) + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + self.assertEqual(zipfp.namelist(), [filename]) + self.assertEqual(zipfp.read(filename), content) + + def test_create_non_existent_file_for_append(self): + if os.path.exists(TESTFN): + os.unlink(TESTFN) + + filename = 'testfile.txt' + content = b'hello, world. this is some content.' + + try: + with zipfile.ZipFile(TESTFN, 'a') as zf: + zf.writestr(filename, content) + except OSError: + self.fail('Could not append data to a non-existent zip file.') + + self.assertTrue(os.path.exists(TESTFN)) + + with zipfile.ZipFile(TESTFN, 'r') as zf: + self.assertEqual(zf.read(filename), content) + + def test_close_erroneous_file(self): + # This test checks that the ZipFile constructor closes the file object + # it opens if there's an error in the file. If it doesn't, the + # traceback holds a reference to the ZipFile object and, indirectly, + # the file object. + # On Windows, this causes the os.unlink() call to fail because the + # underlying file is still open. This is SF bug #412214. + # + with open(TESTFN, "w", encoding="utf-8") as fp: + fp.write("this is not a legal zip file\n") + try: + zf = zipfile.ZipFile(TESTFN) + except zipfile.BadZipFile: + pass + + def test_is_zip_erroneous_file(self): + """Check that is_zipfile() correctly identifies non-zip files.""" + # - passing a filename + with open(TESTFN, "w", encoding='utf-8') as fp: + fp.write("this is not a legal zip file\n") + self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a path-like object + self.assertFalse(zipfile.is_zipfile(FakePath(TESTFN))) + # - passing a file object + with open(TESTFN, "rb") as fp: + self.assertFalse(zipfile.is_zipfile(fp)) + # - passing a file-like object + fp = io.BytesIO() + fp.write(b"this is not a legal zip file\n") + self.assertFalse(zipfile.is_zipfile(fp)) + fp.seek(0, 0) + self.assertFalse(zipfile.is_zipfile(fp)) + + def test_damaged_zipfile(self): + """Check that zipfiles with missing bytes at the end raise BadZipFile.""" + # - Create a valid zip file + fp = io.BytesIO() + with zipfile.ZipFile(fp, mode="w") as zipf: + zipf.writestr("foo.txt", b"O, for a Muse of Fire!") + zipfiledata = fp.getvalue() + + # - Now create copies of it missing the last N bytes and make sure + # a BadZipFile exception is raised when we try to open it + for N in range(len(zipfiledata)): + fp = io.BytesIO(zipfiledata[:N]) + self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, fp) + + def test_is_zip_valid_file(self): + """Check that is_zipfile() correctly identifies zip files.""" + # - passing a filename + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + zipf.writestr("foo.txt", b"O, for a Muse of Fire!") + + self.assertTrue(zipfile.is_zipfile(TESTFN)) + # - passing a file object + with open(TESTFN, "rb") as fp: + self.assertTrue(zipfile.is_zipfile(fp)) + fp.seek(0, 0) + zip_contents = fp.read() + # - passing a file-like object + fp = io.BytesIO() + fp.write(zip_contents) + self.assertTrue(zipfile.is_zipfile(fp)) + fp.seek(0, 0) + self.assertTrue(zipfile.is_zipfile(fp)) + + def test_non_existent_file_raises_OSError(self): + # make sure we don't raise an AttributeError when a partially-constructed + # ZipFile instance is finalized; this tests for regression on SF tracker + # bug #403871. + + # The bug we're testing for caused an AttributeError to be raised + # when a ZipFile instance was created for a file that did not + # exist; the .fp member was not initialized but was needed by the + # __del__() method. Since the AttributeError is in the __del__(), + # it is ignored, but the user should be sufficiently annoyed by + # the message on the output that regression will be noticed + # quickly. + self.assertRaises(OSError, zipfile.ZipFile, TESTFN) + + def test_empty_file_raises_BadZipFile(self): + f = open(TESTFN, 'w', encoding='utf-8') + f.close() + self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN) + + with open(TESTFN, 'w', encoding='utf-8') as fp: + fp.write("short file") + self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN) + + def test_negative_central_directory_offset_raises_BadZipFile(self): + # Zip file containing an empty EOCD record + buffer = bytearray(b'PK\x05\x06' + b'\0'*18) + + # Set the size of the central directory bytes to become 1, + # causing the central directory offset to become negative + for dirsize in 1, 2**32-1: + buffer[12:16] = struct.pack('<L', dirsize) + f = io.BytesIO(buffer) + self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, f) + + def test_closed_zip_raises_ValueError(self): + """Verify that testzip() doesn't swallow inappropriate exceptions.""" + data = io.BytesIO() + with zipfile.ZipFile(data, mode="w") as zipf: + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + + # This is correct; calling .read on a closed ZipFile should raise + # a ValueError, and so should calling .testzip. An earlier + # version of .testzip would swallow this exception (and any other) + # and report that the first file in the archive was corrupt. + self.assertRaises(ValueError, zipf.read, "foo.txt") + self.assertRaises(ValueError, zipf.open, "foo.txt") + self.assertRaises(ValueError, zipf.testzip) + self.assertRaises(ValueError, zipf.writestr, "bogus.txt", "bogus") + with open(TESTFN, 'w', encoding='utf-8') as f: + f.write('zipfile test data') + self.assertRaises(ValueError, zipf.write, TESTFN) + + def test_bad_constructor_mode(self): + """Check that bad modes passed to ZipFile constructor are caught.""" + self.assertRaises(ValueError, zipfile.ZipFile, TESTFN, "q") + + def test_bad_open_mode(self): + """Check that bad modes passed to ZipFile.open are caught.""" + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + # read the data to make sure the file is there + zipf.read("foo.txt") + self.assertRaises(ValueError, zipf.open, "foo.txt", "q") + # universal newlines support is removed + self.assertRaises(ValueError, zipf.open, "foo.txt", "U") + self.assertRaises(ValueError, zipf.open, "foo.txt", "rU") + + def test_read0(self): + """Check that calling read(0) on a ZipExtFile object returns an empty + string and doesn't advance file pointer.""" + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + # read the data to make sure the file is there + with zipf.open("foo.txt") as f: + for i in range(FIXEDTEST_SIZE): + self.assertEqual(f.read(0), b'') + + self.assertEqual(f.read(), b"O, for a Muse of Fire!") + + def test_open_non_existent_item(self): + """Check that attempting to call open() for an item that doesn't + exist in the archive raises a RuntimeError.""" + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + self.assertRaises(KeyError, zipf.open, "foo.txt", "r") + + def test_bad_compression_mode(self): + """Check that bad compression methods passed to ZipFile.open are + caught.""" + self.assertRaises(NotImplementedError, zipfile.ZipFile, TESTFN, "w", -1) + + def test_unsupported_compression(self): + # data is declared as shrunk, but actually deflated + data = (b'PK\x03\x04.\x00\x00\x00\x01\x00\xe4C\xa1@\x00\x00\x00' + b'\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00x\x03\x00PK\x01' + b'\x02.\x03.\x00\x00\x00\x01\x00\xe4C\xa1@\x00\x00\x00\x00\x02\x00\x00' + b'\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x80\x01\x00\x00\x00\x00xPK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00' + b'/\x00\x00\x00!\x00\x00\x00\x00\x00') + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertRaises(NotImplementedError, zipf.open, 'x') + + def test_null_byte_in_filename(self): + """Check that a filename containing a null byte is properly + terminated.""" + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + zipf.writestr("foo.txt\x00qqq", b"O, for a Muse of Fire!") + self.assertEqual(zipf.namelist(), ['foo.txt']) + + def test_struct_sizes(self): + """Check that ZIP internal structure sizes are calculated correctly.""" + self.assertEqual(zipfile.sizeEndCentDir, 22) + self.assertEqual(zipfile.sizeCentralDir, 46) + self.assertEqual(zipfile.sizeEndCentDir64, 56) + self.assertEqual(zipfile.sizeEndCentDir64Locator, 20) + + def test_comments(self): + """Check that comments on the archive are handled properly.""" + + # check default comment is empty + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + self.assertEqual(zipf.comment, b'') + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + + with zipfile.ZipFile(TESTFN, mode="r") as zipfr: + self.assertEqual(zipfr.comment, b'') + + # check a simple short comment + comment = b'Bravely taking to his feet, he beat a very brave retreat.' + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + zipf.comment = comment + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + with zipfile.ZipFile(TESTFN, mode="r") as zipfr: + self.assertEqual(zipf.comment, comment) + + # check a comment of max length + comment2 = ''.join(['%d' % (i**3 % 10) for i in range((1 << 16)-1)]) + comment2 = comment2.encode("ascii") + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + zipf.comment = comment2 + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + + with zipfile.ZipFile(TESTFN, mode="r") as zipfr: + self.assertEqual(zipfr.comment, comment2) + + # check a comment that is too long is truncated + with zipfile.ZipFile(TESTFN, mode="w") as zipf: + with self.assertWarns(UserWarning): + zipf.comment = comment2 + b'oops' + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + with zipfile.ZipFile(TESTFN, mode="r") as zipfr: + self.assertEqual(zipfr.comment, comment2) + + # check that comments are correctly modified in append mode + with zipfile.ZipFile(TESTFN,mode="w") as zipf: + zipf.comment = b"original comment" + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + with zipfile.ZipFile(TESTFN,mode="a") as zipf: + zipf.comment = b"an updated comment" + with zipfile.ZipFile(TESTFN,mode="r") as zipf: + self.assertEqual(zipf.comment, b"an updated comment") + + # check that comments are correctly shortened in append mode + # and the file is indeed truncated + with zipfile.ZipFile(TESTFN,mode="w") as zipf: + zipf.comment = b"original comment that's longer" + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + original_zip_size = os.path.getsize(TESTFN) + with zipfile.ZipFile(TESTFN,mode="a") as zipf: + zipf.comment = b"shorter comment" + self.assertTrue(original_zip_size > os.path.getsize(TESTFN)) + with zipfile.ZipFile(TESTFN,mode="r") as zipf: + self.assertEqual(zipf.comment, b"shorter comment") + + def test_unicode_comment(self): + with zipfile.ZipFile(TESTFN, "w", zipfile.ZIP_STORED) as zipf: + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + with self.assertRaises(TypeError): + zipf.comment = "this is an error" + + def test_change_comment_in_empty_archive(self): + with zipfile.ZipFile(TESTFN, "a", zipfile.ZIP_STORED) as zipf: + self.assertFalse(zipf.filelist) + zipf.comment = b"this is a comment" + with zipfile.ZipFile(TESTFN, "r") as zipf: + self.assertEqual(zipf.comment, b"this is a comment") + + def test_change_comment_in_nonempty_archive(self): + with zipfile.ZipFile(TESTFN, "w", zipfile.ZIP_STORED) as zipf: + zipf.writestr("foo.txt", "O, for a Muse of Fire!") + with zipfile.ZipFile(TESTFN, "a", zipfile.ZIP_STORED) as zipf: + self.assertTrue(zipf.filelist) + zipf.comment = b"this is a comment" + with zipfile.ZipFile(TESTFN, "r") as zipf: + self.assertEqual(zipf.comment, b"this is a comment") + + def test_empty_zipfile(self): + # Check that creating a file in 'w' or 'a' mode and closing without + # adding any files to the archives creates a valid empty ZIP file + zipf = zipfile.ZipFile(TESTFN, mode="w") + zipf.close() + try: + zipf = zipfile.ZipFile(TESTFN, mode="r") + except zipfile.BadZipFile: + self.fail("Unable to create empty ZIP file in 'w' mode") + zipf.close() + + zipf = zipfile.ZipFile(TESTFN, mode="a") + zipf.close() + try: + zipf = zipfile.ZipFile(TESTFN, mode="r") + except: + self.fail("Unable to create empty ZIP file in 'a' mode") + zipf.close() + + def test_open_empty_file(self): + # Issue 1710703: Check that opening a file with less than 22 bytes + # raises a BadZipFile exception (rather than the previously unhelpful + # OSError) + f = open(TESTFN, 'w', encoding='utf-8') + f.close() + self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN, 'r') + + def test_create_zipinfo_before_1980(self): + self.assertRaises(ValueError, + zipfile.ZipInfo, 'seventies', (1979, 1, 1, 0, 0, 0)) + + def test_create_empty_zipinfo_repr(self): + """Before bpo-26185, repr() on empty ZipInfo object was failing.""" + zi = zipfile.ZipInfo(filename="empty") + self.assertEqual(repr(zi), "<ZipInfo filename='empty' file_size=0>") + + def test_create_empty_zipinfo_default_attributes(self): + """Ensure all required attributes are set.""" + zi = zipfile.ZipInfo() + self.assertEqual(zi.orig_filename, "NoName") + self.assertEqual(zi.filename, "NoName") + self.assertEqual(zi.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + self.assertEqual(zi.comment, b"") + self.assertEqual(zi.extra, b"") + self.assertIn(zi.create_system, (0, 3)) + self.assertEqual(zi.create_version, zipfile.DEFAULT_VERSION) + self.assertEqual(zi.extract_version, zipfile.DEFAULT_VERSION) + self.assertEqual(zi.reserved, 0) + self.assertEqual(zi.flag_bits, 0) + self.assertEqual(zi.volume, 0) + self.assertEqual(zi.internal_attr, 0) + self.assertEqual(zi.external_attr, 0) + + # Before bpo-26185, both were missing + self.assertEqual(zi.file_size, 0) + self.assertEqual(zi.compress_size, 0) + + def test_zipfile_with_short_extra_field(self): + """If an extra field in the header is less than 4 bytes, skip it.""" + zipdata = ( + b'PK\x03\x04\x14\x00\x00\x00\x00\x00\x93\x9b\xad@\x8b\x9e' + b'\xd9\xd3\x01\x00\x00\x00\x01\x00\x00\x00\x03\x00\x03\x00ab' + b'c\x00\x00\x00APK\x01\x02\x14\x03\x14\x00\x00\x00\x00' + b'\x00\x93\x9b\xad@\x8b\x9e\xd9\xd3\x01\x00\x00\x00\x01\x00\x00' + b'\x00\x03\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00' + b'\x00\x00\x00abc\x00\x00PK\x05\x06\x00\x00\x00\x00' + b'\x01\x00\x01\x003\x00\x00\x00%\x00\x00\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(zipdata), 'r') as zipf: + # testzip returns the name of the first corrupt file, or None + self.assertIsNone(zipf.testzip()) + + def test_open_conflicting_handles(self): + # It's only possible to open one writable file handle at a time + msg1 = b"It's fun to charter an accountant!" + msg2 = b"And sail the wide accountant sea" + msg3 = b"To find, explore the funds offshore" + with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_STORED) as zipf: + with zipf.open('foo', mode='w') as w2: + w2.write(msg1) + with zipf.open('bar', mode='w') as w1: + with self.assertRaises(ValueError): + zipf.open('handle', mode='w') + with self.assertRaises(ValueError): + zipf.open('foo', mode='r') + with self.assertRaises(ValueError): + zipf.writestr('str', 'abcde') + with self.assertRaises(ValueError): + zipf.write(__file__, 'file') + with self.assertRaises(ValueError): + zipf.close() + w1.write(msg2) + with zipf.open('baz', mode='w') as w2: + w2.write(msg3) + + with zipfile.ZipFile(TESTFN2, 'r') as zipf: + self.assertEqual(zipf.read('foo'), msg1) + self.assertEqual(zipf.read('bar'), msg2) + self.assertEqual(zipf.read('baz'), msg3) + self.assertEqual(zipf.namelist(), ['foo', 'bar', 'baz']) + + def test_seek_tell(self): + # Test seek functionality + txt = b"Where's Bruce?" + bloc = txt.find(b"Bruce") + # Check seek on a file + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("foo.txt", txt) + with zipfile.ZipFile(TESTFN, "r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.seek(bloc, os.SEEK_SET) + self.assertEqual(fp.tell(), bloc) + fp.seek(-bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), 0) + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), bloc) + self.assertEqual(fp.read(5), txt[bloc:bloc+5]) + self.assertEqual(fp.tell(), bloc + 5) + fp.seek(0, os.SEEK_END) + self.assertEqual(fp.tell(), len(txt)) + fp.seek(0, os.SEEK_SET) + self.assertEqual(fp.tell(), 0) + # Check seek on memory file + data = io.BytesIO() + with zipfile.ZipFile(data, mode="w") as zipf: + zipf.writestr("foo.txt", txt) + with zipfile.ZipFile(data, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.seek(bloc, os.SEEK_SET) + self.assertEqual(fp.tell(), bloc) + fp.seek(-bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), 0) + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), bloc) + self.assertEqual(fp.read(5), txt[bloc:bloc+5]) + self.assertEqual(fp.tell(), bloc + 5) + fp.seek(0, os.SEEK_END) + self.assertEqual(fp.tell(), len(txt)) + fp.seek(0, os.SEEK_SET) + self.assertEqual(fp.tell(), 0) + + def test_read_after_seek(self): + # Issue 102956: Make sure seek(x, os.SEEK_CUR) doesn't break read() + txt = b"Charge men!" + bloc = txt.find(b"men") + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("foo.txt", txt) + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.read(-1), b'men!') + with zipfile.ZipFile(TESTFN, mode="r") as zipf: + with zipf.open("foo.txt", "r") as fp: + fp.read(6) + fp.seek(1, os.SEEK_CUR) + self.assertEqual(fp.read(-1), b'men!') + + def test_uncompressed_interleaved_seek_read(self): + # gh-127847: Make sure the position in the archive is correct + # in the special case of seeking in a ZIP_STORED entry. + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("a.txt", "123") + zipf.writestr("b.txt", "456") + with zipfile.ZipFile(TESTFN, "r") as zipf: + with zipf.open("a.txt", "r") as a, zipf.open("b.txt", "r") as b: + self.assertEqual(a.read(1), b"1") + self.assertEqual(b.seek(1), 1) + self.assertEqual(b.read(1), b"5") + + @requires_bz2() + def test_decompress_without_3rd_party_library(self): + data = b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + zip_file = io.BytesIO(data) + with zipfile.ZipFile(zip_file, 'w', compression=zipfile.ZIP_BZIP2) as zf: + zf.writestr('a.txt', b'a') + with mock.patch('zipfile.bz2', None): + with zipfile.ZipFile(zip_file) as zf: + self.assertRaises(RuntimeError, zf.extract, 'a.txt') + + @requires_zlib() + def test_full_overlap_different_names(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00b\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + self.assertEqual(len(zipf.read('b')), 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'): + zipf.read('a') + + @requires_zlib() + def test_full_overlap_different_names2(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'): + zipf.read('b') + with self.assertWarnsRegex(UserWarning, 'Overlapped entries') as cm: + self.assertEqual(len(zipf.read('a')), 1033) + self.assertEqual(cm.filename, __file__) + + @requires_zlib() + def test_full_overlap_same_name(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'a']) + self.assertEqual(len(zipf.infolist()), 2) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + self.assertEqual(len(zipf.read('a')), 1033) + self.assertEqual(len(zipf.read(zi)), 1033) + self.assertEqual(len(zipf.read(zipf.infolist()[1])), 1033) + with self.assertWarnsRegex(UserWarning, 'Overlapped entries') as cm: + self.assertEqual(len(zipf.read(zipf.infolist()[0])), 1033) + self.assertEqual(cm.filename, __file__) + with self.assertWarnsRegex(UserWarning, 'Overlapped entries') as cm: + zipf.open(zipf.infolist()[0]).close() + self.assertEqual(cm.filename, __file__) + + @requires_zlib() + def test_quoted_overlap(self): + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' + b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' + b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\' + b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0' + b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00' + b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00' + b'\x00S\x00\x00\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a', 'b']) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 52) + self.assertEqual(zi.file_size, 1064) + zi = zipf.getinfo('b') + self.assertEqual(zi.header_offset, 36) + self.assertEqual(zi.compress_size, 16) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zipf.read('a') + self.assertEqual(len(zipf.read('b')), 1033) + + @requires_zlib() + def test_overlap_with_central_dir(self): + data = ( + b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' + b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\xb4\x81\x00\x00\x00\x00aP' + b'K\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a']) + self.assertEqual(len(zipf.infolist()), 1) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 11) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number'): + zipf.read('a') + + @requires_zlib() + def test_overlap_with_archive_comment(self): + data = ( + b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' + b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\xb4\x81E\x00\x00\x00aP' + b'K\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x00' + b'\x00\x00\x00*\x00' + b'PK\x03\x04\x14\x00\x00\x00\x08\x00G_|Z\xe2\x1e' + b'8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00aK' + b'L\x1c\x05\xa3`\x14\x8cx\x00\x00' + ) + with zipfile.ZipFile(io.BytesIO(data), 'r') as zipf: + self.assertEqual(zipf.namelist(), ['a']) + self.assertEqual(len(zipf.infolist()), 1) + zi = zipf.getinfo('a') + self.assertEqual(zi.header_offset, 69) + self.assertEqual(zi.compress_size, 11) + self.assertEqual(zi.file_size, 1033) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zipf.read('a') + + def tearDown(self): + unlink(TESTFN) + unlink(TESTFN2) + + +class AbstractBadCrcTests: + def test_testzip_with_bad_crc(self): + """Tests that files with bad CRCs return their name from testzip.""" + zipdata = self.zip_with_bad_crc + + with zipfile.ZipFile(io.BytesIO(zipdata), mode="r") as zipf: + # testzip returns the name of the first corrupt file, or None + self.assertEqual('afile', zipf.testzip()) + + def test_read_with_bad_crc(self): + """Tests that files with bad CRCs raise a BadZipFile exception when read.""" + zipdata = self.zip_with_bad_crc + + # Using ZipFile.read() + with zipfile.ZipFile(io.BytesIO(zipdata), mode="r") as zipf: + self.assertRaises(zipfile.BadZipFile, zipf.read, 'afile') + + # Using ZipExtFile.read() + with zipfile.ZipFile(io.BytesIO(zipdata), mode="r") as zipf: + with zipf.open('afile', 'r') as corrupt_file: + self.assertRaises(zipfile.BadZipFile, corrupt_file.read) + + # Same with small reads (in order to exercise the buffering logic) + with zipfile.ZipFile(io.BytesIO(zipdata), mode="r") as zipf: + with zipf.open('afile', 'r') as corrupt_file: + corrupt_file.MIN_READ_SIZE = 2 + with self.assertRaises(zipfile.BadZipFile): + while corrupt_file.read(2): + pass + + +class StoredBadCrcTests(AbstractBadCrcTests, unittest.TestCase): + compression = zipfile.ZIP_STORED + zip_with_bad_crc = ( + b'PK\003\004\024\0\0\0\0\0 \213\212;:r' + b'\253\377\f\0\0\0\f\0\0\0\005\0\0\000af' + b'ilehello,AworldP' + b'K\001\002\024\003\024\0\0\0\0\0 \213\212;:' + b'r\253\377\f\0\0\0\f\0\0\0\005\0\0\0\0' + b'\0\0\0\0\0\0\0\200\001\0\0\0\000afi' + b'lePK\005\006\0\0\0\0\001\0\001\0003\000' + b'\0\0/\0\0\0\0\0') + +@requires_zlib() +class DeflateBadCrcTests(AbstractBadCrcTests, unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + zip_with_bad_crc = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00n}\x0c=FA' + b'KE\x10\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00af' + b'ile\xcbH\xcd\xc9\xc9W(\xcf/\xcaI\xc9\xa0' + b'=\x13\x00PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00n' + b'}\x0c=FAKE\x10\x00\x00\x00n\x00\x00\x00\x05' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00' + b'\x00afilePK\x05\x06\x00\x00\x00\x00\x01\x00' + b'\x01\x003\x00\x00\x003\x00\x00\x00\x00\x00') + +@requires_bz2() +class Bzip2BadCrcTests(AbstractBadCrcTests, unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + zip_with_bad_crc = ( + b'PK\x03\x04\x14\x03\x00\x00\x0c\x00nu\x0c=FA' + b'KE8\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00af' + b'ileBZh91AY&SY\xd4\xa8\xca' + b'\x7f\x00\x00\x0f\x11\x80@\x00\x06D\x90\x80 \x00 \xa5' + b'P\xd9!\x03\x03\x13\x13\x13\x89\xa9\xa9\xc2u5:\x9f' + b'\x8b\xb9"\x9c(HjTe?\x80PK\x01\x02\x14' + b'\x03\x14\x03\x00\x00\x0c\x00nu\x0c=FAKE8' + b'\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00 \x80\x80\x81\x00\x00\x00\x00afilePK' + b'\x05\x06\x00\x00\x00\x00\x01\x00\x01\x003\x00\x00\x00[\x00' + b'\x00\x00\x00\x00') + +@requires_lzma() +class LzmaBadCrcTests(AbstractBadCrcTests, unittest.TestCase): + compression = zipfile.ZIP_LZMA + zip_with_bad_crc = ( + b'PK\x03\x04\x14\x03\x00\x00\x0e\x00nu\x0c=FA' + b'KE\x1b\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00af' + b'ile\t\x04\x05\x00]\x00\x00\x00\x04\x004\x19I' + b'\xee\x8d\xe9\x17\x89:3`\tq!.8\x00PK' + b'\x01\x02\x14\x03\x14\x03\x00\x00\x0e\x00nu\x0c=FA' + b'KE\x1b\x00\x00\x00n\x00\x00\x00\x05\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00 \x80\x80\x81\x00\x00\x00\x00afil' + b'ePK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x003\x00\x00' + b'\x00>\x00\x00\x00\x00\x00') + + +class DecryptionTests(unittest.TestCase): + """Check that ZIP decryption works. Since the library does not + support encryption at the moment, we use a pre-generated encrypted + ZIP file.""" + + data = ( + b'PK\x03\x04\x14\x00\x01\x00\x00\x00n\x92i.#y\xef?&\x00\x00\x00\x1a\x00' + b'\x00\x00\x08\x00\x00\x00test.txt\xfa\x10\xa0gly|\xfa-\xc5\xc0=\xf9y' + b'\x18\xe0\xa8r\xb3Z}Lg\xbc\xae\xf9|\x9b\x19\xe4\x8b\xba\xbb)\x8c\xb0\xdbl' + b'PK\x01\x02\x14\x00\x14\x00\x01\x00\x00\x00n\x92i.#y\xef?&\x00\x00\x00' + b'\x1a\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00\x00\x01\x00 \x00\xb6\x81' + b'\x00\x00\x00\x00test.txtPK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x006\x00' + b'\x00\x00L\x00\x00\x00\x00\x00' ) + data2 = ( + b'PK\x03\x04\x14\x00\t\x00\x08\x00\xcf}38xu\xaa\xb2\x14\x00\x00\x00\x00\x02' + b'\x00\x00\x04\x00\x15\x00zeroUT\t\x00\x03\xd6\x8b\x92G\xda\x8b\x92GUx\x04' + b'\x00\xe8\x03\xe8\x03\xc7<M\xb5a\xceX\xa3Y&\x8b{oE\xd7\x9d\x8c\x98\x02\xc0' + b'PK\x07\x08xu\xaa\xb2\x14\x00\x00\x00\x00\x02\x00\x00PK\x01\x02\x17\x03' + b'\x14\x00\t\x00\x08\x00\xcf}38xu\xaa\xb2\x14\x00\x00\x00\x00\x02\x00\x00' + b'\x04\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00ze' + b'roUT\x05\x00\x03\xd6\x8b\x92GUx\x00\x00PK\x05\x06\x00\x00\x00\x00\x01' + b'\x00\x01\x00?\x00\x00\x00[\x00\x00\x00\x00\x00' ) + + plain = b'zipfile.py encryption test' + plain2 = b'\x00'*512 + + def setUp(self): + with open(TESTFN, "wb") as fp: + fp.write(self.data) + self.zip = zipfile.ZipFile(TESTFN, "r") + with open(TESTFN2, "wb") as fp: + fp.write(self.data2) + self.zip2 = zipfile.ZipFile(TESTFN2, "r") + + def tearDown(self): + self.zip.close() + os.unlink(TESTFN) + self.zip2.close() + os.unlink(TESTFN2) + + def test_no_password(self): + # Reading the encrypted file without password + # must generate a RunTime exception + self.assertRaises(RuntimeError, self.zip.read, "test.txt") + self.assertRaises(RuntimeError, self.zip2.read, "zero") + + def test_bad_password(self): + self.zip.setpassword(b"perl") + self.assertRaises(RuntimeError, self.zip.read, "test.txt") + self.zip2.setpassword(b"perl") + self.assertRaises(RuntimeError, self.zip2.read, "zero") + + @requires_zlib() + def test_good_password(self): + self.zip.setpassword(b"python") + self.assertEqual(self.zip.read("test.txt"), self.plain) + self.zip2.setpassword(b"12345") + self.assertEqual(self.zip2.read("zero"), self.plain2) + + def test_unicode_password(self): + expected_msg = "pwd: expected bytes, got str" + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.setpassword("unicode") + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.read("test.txt", "python") + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.open("test.txt", pwd="python") + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.extract("test.txt", pwd="python") + + with self.assertRaisesRegex(TypeError, expected_msg): + self.zip.pwd = "python" + self.zip.open("test.txt") + + def test_seek_tell(self): + self.zip.setpassword(b"python") + txt = self.plain + test_word = b'encryption' + bloc = txt.find(test_word) + bloc_len = len(test_word) + with self.zip.open("test.txt", "r") as fp: + fp.seek(bloc, os.SEEK_SET) + self.assertEqual(fp.tell(), bloc) + fp.seek(-bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), 0) + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), bloc) + self.assertEqual(fp.read(bloc_len), txt[bloc:bloc+bloc_len]) + + # Make sure that the second read after seeking back beyond + # _readbuffer returns the same content (ie. rewind to the start of + # the file to read forward to the required position). + old_read_size = fp.MIN_READ_SIZE + fp.MIN_READ_SIZE = 1 + fp._readbuffer = b'' + fp._offset = 0 + fp.seek(0, os.SEEK_SET) + self.assertEqual(fp.tell(), 0) + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.read(bloc_len), txt[bloc:bloc+bloc_len]) + fp.MIN_READ_SIZE = old_read_size + + fp.seek(0, os.SEEK_END) + self.assertEqual(fp.tell(), len(txt)) + fp.seek(0, os.SEEK_SET) + self.assertEqual(fp.tell(), 0) + + # Read the file completely to definitely call any eof integrity + # checks (crc) and make sure they still pass. + fp.read() + + +class AbstractTestsWithRandomBinaryFiles: + @classmethod + def setUpClass(cls): + datacount = randint(16, 64)*1024 + randint(1, 1024) + cls.data = b''.join(struct.pack('<f', random()*randint(-1000, 1000)) + for i in range(datacount)) + + def setUp(self): + # Make a source file with some lines + with open(TESTFN, "wb") as fp: + fp.write(self.data) + + def tearDown(self): + unlink(TESTFN) + unlink(TESTFN2) + + def make_test_archive(self, f, compression): + # Create the ZIP archive + with zipfile.ZipFile(f, "w", compression) as zipfp: + zipfp.write(TESTFN, "another.name") + zipfp.write(TESTFN, TESTFN) + + def zip_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r", compression) as zipfp: + testdata = zipfp.read(TESTFN) + self.assertEqual(len(testdata), len(self.data)) + self.assertEqual(testdata, self.data) + self.assertEqual(zipfp.read("another.name"), self.data) + + def test_read(self): + for f in get_files(self): + self.zip_test(f, self.compression) + + def zip_open_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r", compression) as zipfp: + zipdata1 = [] + with zipfp.open(TESTFN) as zipopen1: + while True: + read_data = zipopen1.read(256) + if not read_data: + break + zipdata1.append(read_data) + + zipdata2 = [] + with zipfp.open("another.name") as zipopen2: + while True: + read_data = zipopen2.read(256) + if not read_data: + break + zipdata2.append(read_data) + + testdata1 = b''.join(zipdata1) + self.assertEqual(len(testdata1), len(self.data)) + self.assertEqual(testdata1, self.data) + + testdata2 = b''.join(zipdata2) + self.assertEqual(len(testdata2), len(self.data)) + self.assertEqual(testdata2, self.data) + + def test_open(self): + for f in get_files(self): + self.zip_open_test(f, self.compression) + + def zip_random_open_test(self, f, compression): + self.make_test_archive(f, compression) + + # Read the ZIP archive + with zipfile.ZipFile(f, "r", compression) as zipfp: + zipdata1 = [] + with zipfp.open(TESTFN) as zipopen1: + while True: + read_data = zipopen1.read(randint(1, 1024)) + if not read_data: + break + zipdata1.append(read_data) + + testdata = b''.join(zipdata1) + self.assertEqual(len(testdata), len(self.data)) + self.assertEqual(testdata, self.data) + + def test_random_open(self): + for f in get_files(self): + self.zip_random_open_test(f, self.compression) + + +class StoredTestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles, + unittest.TestCase): + compression = zipfile.ZIP_STORED + +@requires_zlib() +class DeflateTestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles, + unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2() +class Bzip2TestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles, + unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma() +class LzmaTestsWithRandomBinaryFiles(AbstractTestsWithRandomBinaryFiles, + unittest.TestCase): + compression = zipfile.ZIP_LZMA + + +# Provide the tell() method but not seek() +class Tellable: + def __init__(self, fp): + self.fp = fp + self.offset = 0 + + def write(self, data): + n = self.fp.write(data) + self.offset += n + return n + + def tell(self): + return self.offset + + def flush(self): + self.fp.flush() + +class Unseekable: + def __init__(self, fp): + self.fp = fp + + def write(self, data): + return self.fp.write(data) + + def flush(self): + self.fp.flush() + +class UnseekableTests(unittest.TestCase): + def test_writestr(self): + for wrapper in (lambda f: f), Tellable, Unseekable: + with self.subTest(wrapper=wrapper): + f = io.BytesIO() + f.write(b'abc') + bf = io.BufferedWriter(f) + with zipfile.ZipFile(wrapper(bf), 'w', zipfile.ZIP_STORED) as zipfp: + zipfp.writestr('ones', b'111') + zipfp.writestr('twos', b'222') + self.assertEqual(f.getvalue()[:5], b'abcPK') + with zipfile.ZipFile(f, mode='r') as zipf: + with zipf.open('ones') as zopen: + self.assertEqual(zopen.read(), b'111') + with zipf.open('twos') as zopen: + self.assertEqual(zopen.read(), b'222') + + def test_write(self): + for wrapper in (lambda f: f), Tellable, Unseekable: + with self.subTest(wrapper=wrapper): + f = io.BytesIO() + f.write(b'abc') + bf = io.BufferedWriter(f) + with zipfile.ZipFile(wrapper(bf), 'w', zipfile.ZIP_STORED) as zipfp: + self.addCleanup(unlink, TESTFN) + with open(TESTFN, 'wb') as f2: + f2.write(b'111') + zipfp.write(TESTFN, 'ones') + with open(TESTFN, 'wb') as f2: + f2.write(b'222') + zipfp.write(TESTFN, 'twos') + self.assertEqual(f.getvalue()[:5], b'abcPK') + with zipfile.ZipFile(f, mode='r') as zipf: + with zipf.open('ones') as zopen: + self.assertEqual(zopen.read(), b'111') + with zipf.open('twos') as zopen: + self.assertEqual(zopen.read(), b'222') + + def test_open_write(self): + for wrapper in (lambda f: f), Tellable, Unseekable: + with self.subTest(wrapper=wrapper): + f = io.BytesIO() + f.write(b'abc') + bf = io.BufferedWriter(f) + with zipfile.ZipFile(wrapper(bf), 'w', zipfile.ZIP_STORED) as zipf: + with zipf.open('ones', 'w') as zopen: + zopen.write(b'111') + with zipf.open('twos', 'w') as zopen: + zopen.write(b'222') + self.assertEqual(f.getvalue()[:5], b'abcPK') + with zipfile.ZipFile(f) as zipf: + self.assertEqual(zipf.read('ones'), b'111') + self.assertEqual(zipf.read('twos'), b'222') + + +@requires_zlib() +class TestsWithMultipleOpens(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.data1 = b'111' + randbytes(10000) + cls.data2 = b'222' + randbytes(10000) + + def make_test_archive(self, f): + # Create the ZIP archive + with zipfile.ZipFile(f, "w", zipfile.ZIP_DEFLATED) as zipfp: + zipfp.writestr('ones', self.data1) + zipfp.writestr('twos', self.data2) + + def test_same_file(self): + # Verify that (when the ZipFile is in control of creating file objects) + # multiple open() calls can be made without interfering with each other. + for f in get_files(self): + self.make_test_archive(f) + with zipfile.ZipFile(f, mode="r") as zipf: + with zipf.open('ones') as zopen1, zipf.open('ones') as zopen2: + data1 = zopen1.read(500) + data2 = zopen2.read(500) + data1 += zopen1.read() + data2 += zopen2.read() + self.assertEqual(data1, data2) + self.assertEqual(data1, self.data1) + + def test_different_file(self): + # Verify that (when the ZipFile is in control of creating file objects) + # multiple open() calls can be made without interfering with each other. + for f in get_files(self): + self.make_test_archive(f) + with zipfile.ZipFile(f, mode="r") as zipf: + with zipf.open('ones') as zopen1, zipf.open('twos') as zopen2: + data1 = zopen1.read(500) + data2 = zopen2.read(500) + data1 += zopen1.read() + data2 += zopen2.read() + self.assertEqual(data1, self.data1) + self.assertEqual(data2, self.data2) + + def test_interleaved(self): + # Verify that (when the ZipFile is in control of creating file objects) + # multiple open() calls can be made without interfering with each other. + for f in get_files(self): + self.make_test_archive(f) + with zipfile.ZipFile(f, mode="r") as zipf: + with zipf.open('ones') as zopen1: + data1 = zopen1.read(500) + with zipf.open('twos') as zopen2: + data2 = zopen2.read(500) + data1 += zopen1.read() + data2 += zopen2.read() + self.assertEqual(data1, self.data1) + self.assertEqual(data2, self.data2) + + def test_read_after_close(self): + for f in get_files(self): + self.make_test_archive(f) + with contextlib.ExitStack() as stack: + with zipfile.ZipFile(f, 'r') as zipf: + zopen1 = stack.enter_context(zipf.open('ones')) + zopen2 = stack.enter_context(zipf.open('twos')) + data1 = zopen1.read(500) + data2 = zopen2.read(500) + data1 += zopen1.read() + data2 += zopen2.read() + self.assertEqual(data1, self.data1) + self.assertEqual(data2, self.data2) + + def test_read_after_write(self): + for f in get_files(self): + with zipfile.ZipFile(f, 'w', zipfile.ZIP_DEFLATED) as zipf: + zipf.writestr('ones', self.data1) + zipf.writestr('twos', self.data2) + with zipf.open('ones') as zopen1: + data1 = zopen1.read(500) + self.assertEqual(data1, self.data1[:500]) + with zipfile.ZipFile(f, 'r') as zipf: + data1 = zipf.read('ones') + data2 = zipf.read('twos') + self.assertEqual(data1, self.data1) + self.assertEqual(data2, self.data2) + + def test_write_after_read(self): + for f in get_files(self): + with zipfile.ZipFile(f, "w", zipfile.ZIP_DEFLATED) as zipf: + zipf.writestr('ones', self.data1) + with zipf.open('ones') as zopen1: + zopen1.read(500) + zipf.writestr('twos', self.data2) + with zipfile.ZipFile(f, 'r') as zipf: + data1 = zipf.read('ones') + data2 = zipf.read('twos') + self.assertEqual(data1, self.data1) + self.assertEqual(data2, self.data2) + + def test_many_opens(self): + # Verify that read() and open() promptly close the file descriptor, + # and don't rely on the garbage collector to free resources. + startcount = fd_count() + self.make_test_archive(TESTFN2) + with zipfile.ZipFile(TESTFN2, mode="r") as zipf: + for x in range(100): + zipf.read('ones') + with zipf.open('ones') as zopen1: + pass + self.assertEqual(startcount, fd_count()) + + def test_write_while_reading(self): + with zipfile.ZipFile(TESTFN2, 'w', zipfile.ZIP_DEFLATED) as zipf: + zipf.writestr('ones', self.data1) + with zipfile.ZipFile(TESTFN2, 'a', zipfile.ZIP_DEFLATED) as zipf: + with zipf.open('ones', 'r') as r1: + data1 = r1.read(500) + with zipf.open('twos', 'w') as w1: + w1.write(self.data2) + data1 += r1.read() + self.assertEqual(data1, self.data1) + with zipfile.ZipFile(TESTFN2) as zipf: + self.assertEqual(zipf.read('twos'), self.data2) + + def tearDown(self): + unlink(TESTFN2) + + +class TestWithDirectory(unittest.TestCase): + def setUp(self): + os.mkdir(TESTFN2) + + def test_extract_dir(self): + with zipfile.ZipFile(findfile("zipdir.zip", subdir="archivetestdata")) as zipf: + zipf.extractall(TESTFN2) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) + self.assertTrue(os.path.exists(os.path.join(TESTFN2, "a", "b", "c"))) + + def test_bug_6050(self): + # Extraction should succeed if directories already exist + os.mkdir(os.path.join(TESTFN2, "a")) + self.test_extract_dir() + + def test_extract_dir_backslash(self): + zfname = findfile("zipdir_backslash.zip", subdir="archivetestdata") + with zipfile.ZipFile(zfname) as zipf: + zipf.extractall(TESTFN2) + if os.name == 'nt': + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "a", "b"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a", "b", "c"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d"))) + self.assertTrue(os.path.isdir(os.path.join(TESTFN2, "d", "e"))) + else: + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "a\\b\\c"))) + self.assertTrue(os.path.isfile(os.path.join(TESTFN2, "d\\e\\"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "a"))) + self.assertFalse(os.path.exists(os.path.join(TESTFN2, "d"))) + + def test_write_dir(self): + dirpath = os.path.join(TESTFN2, "x") + os.mkdir(dirpath) + mode = os.stat(dirpath).st_mode & 0xFFFF + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.write(dirpath) + zinfo = zipf.filelist[0] + self.assertTrue(zinfo.filename.endswith("/x/")) + self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) + zipf.write(dirpath, "y") + zinfo = zipf.filelist[1] + self.assertTrue(zinfo.filename, "y/") + self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) + with zipfile.ZipFile(TESTFN, "r") as zipf: + zinfo = zipf.filelist[0] + self.assertTrue(zinfo.filename.endswith("/x/")) + self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) + zinfo = zipf.filelist[1] + self.assertTrue(zinfo.filename, "y/") + self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) + target = os.path.join(TESTFN2, "target") + os.mkdir(target) + zipf.extractall(target) + self.assertTrue(os.path.isdir(os.path.join(target, "y"))) + self.assertEqual(len(os.listdir(target)), 2) + + def test_writestr_dir(self): + os.mkdir(os.path.join(TESTFN2, "x")) + with zipfile.ZipFile(TESTFN, "w") as zipf: + zipf.writestr("x/", b'') + zinfo = zipf.filelist[0] + self.assertEqual(zinfo.filename, "x/") + self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) + with zipfile.ZipFile(TESTFN, "r") as zipf: + zinfo = zipf.filelist[0] + self.assertTrue(zinfo.filename.endswith("x/")) + self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) + target = os.path.join(TESTFN2, "target") + os.mkdir(target) + zipf.extractall(target) + self.assertTrue(os.path.isdir(os.path.join(target, "x"))) + self.assertEqual(os.listdir(target), ["x"]) + + def test_mkdir(self): + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.mkdir("directory") + zinfo = zf.filelist[0] + self.assertEqual(zinfo.filename, "directory/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + zf.mkdir("directory2/") + zinfo = zf.filelist[1] + self.assertEqual(zinfo.filename, "directory2/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + zf.mkdir("directory3", mode=0o777) + zinfo = zf.filelist[2] + self.assertEqual(zinfo.filename, "directory3/") + self.assertEqual(zinfo.external_attr, (0o40777 << 16) | 0x10) + + old_zinfo = zipfile.ZipInfo("directory4/") + old_zinfo.external_attr = (0o40777 << 16) | 0x10 + old_zinfo.CRC = 0 + old_zinfo.file_size = 0 + old_zinfo.compress_size = 0 + zf.mkdir(old_zinfo) + new_zinfo = zf.filelist[3] + self.assertEqual(old_zinfo.filename, "directory4/") + self.assertEqual(old_zinfo.external_attr, new_zinfo.external_attr) + + target = os.path.join(TESTFN2, "target") + os.mkdir(target) + zf.extractall(target) + self.assertEqual(set(os.listdir(target)), {"directory", "directory2", "directory3", "directory4"}) + + def test_create_directory_with_write(self): + with zipfile.ZipFile(TESTFN, "w") as zf: + zf.writestr(zipfile.ZipInfo('directory/'), '') + + zinfo = zf.filelist[0] + self.assertEqual(zinfo.filename, "directory/") + + directory = os.path.join(TESTFN2, "directory2") + os.mkdir(directory) + mode = os.stat(directory).st_mode & 0xFFFF + zf.write(directory, arcname="directory2/") + zinfo = zf.filelist[1] + self.assertEqual(zinfo.filename, "directory2/") + self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) + + target = os.path.join(TESTFN2, "target") + os.mkdir(target) + zf.extractall(target) + + self.assertEqual(set(os.listdir(target)), {"directory", "directory2"}) + + def test_root_folder_in_zipfile(self): + """ + gh-112795: Some tools or self constructed codes will add '/' folder to + the zip file, this is a strange behavior, but we should support it. + """ + in_memory_file = io.BytesIO() + zf = zipfile.ZipFile(in_memory_file, "w") + zf.mkdir('/') + zf.writestr('./a.txt', 'aaa') + zf.extractall(TESTFN2) + + def tearDown(self): + rmtree(TESTFN2) + if os.path.exists(TESTFN): + unlink(TESTFN) + + +class ZipInfoTests(unittest.TestCase): + def test_from_file(self): + zi = zipfile.ZipInfo.from_file(__file__) + self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_pathlike(self): + zi = zipfile.ZipInfo.from_file(FakePath(__file__)) + self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_bytes(self): + zi = zipfile.ZipInfo.from_file(os.fsencode(__file__), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_file_fileno(self): + with open(__file__, 'rb') as f: + zi = zipfile.ZipInfo.from_file(f.fileno(), 'test') + self.assertEqual(posixpath.basename(zi.filename), 'test') + self.assertFalse(zi.is_dir()) + self.assertEqual(zi.file_size, os.path.getsize(__file__)) + + def test_from_dir(self): + dirpath = os.path.dirname(os.path.abspath(__file__)) + zi = zipfile.ZipInfo.from_file(dirpath, 'stdlib_tests') + self.assertEqual(zi.filename, 'stdlib_tests/') + self.assertTrue(zi.is_dir()) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + self.assertEqual(zi.file_size, 0) + + def test_compresslevel_property(self): + zinfo = zipfile.ZipInfo("xxx") + self.assertFalse(zinfo._compresslevel) + self.assertFalse(zinfo.compress_level) + zinfo._compresslevel = 99 # test the legacy @property.setter + self.assertEqual(zinfo.compress_level, 99) + self.assertEqual(zinfo._compresslevel, 99) + zinfo.compress_level = 8 + self.assertEqual(zinfo.compress_level, 8) + self.assertEqual(zinfo._compresslevel, 8) + + +class CommandLineTest(unittest.TestCase): + + def zipfilecmd(self, *args, **kwargs): + rc, out, err = script_helper.assert_python_ok('-m', 'zipfile', *args, + **kwargs) + return out.replace(os.linesep.encode(), b'\n') + + def zipfilecmd_failure(self, *args): + return script_helper.assert_python_failure('-m', 'zipfile', *args) + + def test_bad_use(self): + rc, out, err = self.zipfilecmd_failure() + self.assertEqual(out, b'') + self.assertIn(b'usage', err.lower()) + self.assertIn(b'error', err.lower()) + self.assertIn(b'required', err.lower()) + rc, out, err = self.zipfilecmd_failure('-l', '') + self.assertEqual(out, b'') + self.assertNotEqual(err.strip(), b'') + + def test_test_command(self): + zip_name = findfile('zipdir.zip', subdir='archivetestdata') + for opt in '-t', '--test': + out = self.zipfilecmd(opt, zip_name) + self.assertEqual(out.rstrip(), b'Done testing') + zip_name = findfile('testtar.tar') + rc, out, err = self.zipfilecmd_failure('-t', zip_name) + self.assertEqual(out, b'') + + def test_list_command(self): + zip_name = findfile('zipdir.zip', subdir='archivetestdata') + t = io.StringIO() + with zipfile.ZipFile(zip_name, 'r') as tf: + tf.printdir(t) + expected = t.getvalue().encode('ascii', 'backslashreplace') + for opt in '-l', '--list': + out = self.zipfilecmd(opt, zip_name, + PYTHONIOENCODING='ascii:backslashreplace') + self.assertEqual(out, expected) + + @requires_zlib() + def test_create_command(self): + self.addCleanup(unlink, TESTFN) + with open(TESTFN, 'w', encoding='utf-8') as f: + f.write('test 1') + os.mkdir(TESTFNDIR) + self.addCleanup(rmtree, TESTFNDIR) + with open(os.path.join(TESTFNDIR, 'file.txt'), 'w', encoding='utf-8') as f: + f.write('test 2') + files = [TESTFN, TESTFNDIR] + namelist = [TESTFN, TESTFNDIR + '/', TESTFNDIR + '/file.txt'] + for opt in '-c', '--create': + try: + out = self.zipfilecmd(opt, TESTFN2, *files) + self.assertEqual(out, b'') + with zipfile.ZipFile(TESTFN2) as zf: + self.assertEqual(zf.namelist(), namelist) + self.assertEqual(zf.read(namelist[0]), b'test 1') + self.assertEqual(zf.read(namelist[2]), b'test 2') + finally: + unlink(TESTFN2) + + def test_extract_command(self): + zip_name = findfile('zipdir.zip', subdir='archivetestdata') + for opt in '-e', '--extract': + with temp_dir() as extdir: + out = self.zipfilecmd(opt, zip_name, extdir) + self.assertEqual(out, b'') + with zipfile.ZipFile(zip_name) as zf: + for zi in zf.infolist(): + path = os.path.join(extdir, + zi.filename.replace('/', os.sep)) + if zi.is_dir(): + self.assertTrue(os.path.isdir(path)) + else: + self.assertTrue(os.path.isfile(path)) + with open(path, 'rb') as f: + self.assertEqual(f.read(), zf.read(zi)) + + +class TestExecutablePrependedZip(unittest.TestCase): + """Test our ability to open zip files with an executable prepended.""" + + def setUp(self): + self.exe_zip = findfile('exe_with_zip', subdir='archivetestdata') + self.exe_zip64 = findfile('exe_with_z64', subdir='archivetestdata') + + def _test_zip_works(self, name): + # bpo28494 sanity check: ensure is_zipfile works on these. + self.assertTrue(zipfile.is_zipfile(name), + f'is_zipfile failed on {name}') + # Ensure we can operate on these via ZipFile. + with zipfile.ZipFile(name) as zipfp: + for n in zipfp.namelist(): + data = zipfp.read(n) + self.assertIn(b'FAVORITE_NUMBER', data) + + def test_read_zip_with_exe_prepended(self): + self._test_zip_works(self.exe_zip) + + def test_read_zip64_with_exe_prepended(self): + self._test_zip_works(self.exe_zip64) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(sys.executable, 'sys.executable required.') + @unittest.skipUnless(os.access('/bin/bash', os.X_OK), + 'Test relies on #!/bin/bash working.') + @requires_subprocess() + def test_execute_zip2(self): + output = subprocess.check_output([self.exe_zip, sys.executable]) + self.assertIn(b'number in executable: 5', output) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(sys.executable, 'sys.executable required.') + @unittest.skipUnless(os.access('/bin/bash', os.X_OK), + 'Test relies on #!/bin/bash working.') + @requires_subprocess() + def test_execute_zip64(self): + output = subprocess.check_output([self.exe_zip64, sys.executable]) + self.assertIn(b'number in executable: 5', output) + + +@unittest.skip("TODO: RUSTPYTHON shift_jis encoding unsupported") +class EncodedMetadataTests(unittest.TestCase): + file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three' + file_content = [ + "This is pure ASCII.\n".encode('ascii'), + # This is modern Japanese. (UTF-8) + "\u3053\u308c\u306f\u73fe\u4ee3\u7684\u65e5\u672c\u8a9e\u3067\u3059\u3002\n".encode('utf-8'), + # TODO RUSTPYTHON + # Uncomment when Shift JIS is supported + # This is obsolete Japanese. (Shift JIS) + # "\u3053\u308c\u306f\u53e4\u3044\u65e5\u672c\u8a9e\u3067\u3059\u3002\n".encode('shift_jis'), + ] + + def setUp(self): + self.addCleanup(unlink, TESTFN) + # Create .zip of 3 members with Han names encoded in Shift JIS. + # Each name is 1 Han character encoding to 2 bytes in Shift JIS. + # The ASCII names are arbitrary as long as they are length 2 and + # not otherwise contained in the zip file. + # Data elements are encoded bytes (ascii, utf-8, shift_jis). + placeholders = ["n1", "n2"] + self.file_names[2:] + with zipfile.ZipFile(TESTFN, mode="w") as tf: + for temp, content in zip(placeholders, self.file_content): + tf.writestr(temp, content, zipfile.ZIP_STORED) + # Hack in the Shift JIS names with flag bit 11 (UTF-8) unset. + with open(TESTFN, "rb") as tf: + data = tf.read() + for name, temp in zip(self.file_names, placeholders[:2]): + data = data.replace(temp.encode('ascii'), + name.encode('shift_jis')) + with open(TESTFN, "wb") as tf: + tf.write(data) + + def _test_read(self, zipfp, expected_names, expected_content): + # Check the namelist + names = zipfp.namelist() + self.assertEqual(sorted(names), sorted(expected_names)) + + # Check infolist + infos = zipfp.infolist() + names = [zi.filename for zi in infos] + self.assertEqual(sorted(names), sorted(expected_names)) + + # check getinfo + for name, content in zip(expected_names, expected_content): + info = zipfp.getinfo(name) + self.assertEqual(info.filename, name) + self.assertEqual(info.file_size, len(content)) + self.assertEqual(zipfp.read(name), content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_with_metadata_encoding(self): + # Read the ZIP archive with correct metadata_encoding + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: + self._test_read(zipfp, self.file_names, self.file_content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_without_metadata_encoding(self): + # Read the ZIP archive without metadata_encoding + expected_names = [name.encode('shift_jis').decode('cp437') + for name in self.file_names[:2]] + self.file_names[2:] + with zipfile.ZipFile(TESTFN, "r") as zipfp: + self._test_read(zipfp, expected_names, self.file_content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_with_incorrect_metadata_encoding(self): + # Read the ZIP archive with incorrect metadata_encoding + expected_names = [name.encode('shift_jis').decode('koi8-u') + for name in self.file_names[:2]] + self.file_names[2:] + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='koi8-u') as zipfp: + self._test_read(zipfp, expected_names, self.file_content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_with_unsuitable_metadata_encoding(self): + # Read the ZIP archive with metadata_encoding unsuitable for + # decoding metadata + with self.assertRaises(UnicodeDecodeError): + zipfile.ZipFile(TESTFN, "r", metadata_encoding='ascii') + with self.assertRaises(UnicodeDecodeError): + zipfile.ZipFile(TESTFN, "r", metadata_encoding='utf-8') + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_read_after_append(self): + newname = '\u56db' # Han 'four' + expected_names = [name.encode('shift_jis').decode('cp437') + for name in self.file_names[:2]] + self.file_names[2:] + expected_names.append(newname) + expected_content = (*self.file_content, b"newcontent") + + with zipfile.ZipFile(TESTFN, "a") as zipfp: + zipfp.writestr(newname, "newcontent") + self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) + + with zipfile.ZipFile(TESTFN, "r") as zipfp: + self._test_read(zipfp, expected_names, expected_content) + + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: + self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) + for i, (name, content) in enumerate(zip(expected_names, expected_content)): + info = zipfp.getinfo(name) + self.assertEqual(info.filename, name) + self.assertEqual(info.file_size, len(content)) + if i < 2: + with self.assertRaises(zipfile.BadZipFile): + zipfp.read(name) + else: + self.assertEqual(zipfp.read(name), content) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_write_with_metadata_encoding(self): + ZF = zipfile.ZipFile + for mode in ("w", "x", "a"): + with self.assertRaisesRegex(ValueError, + "^metadata_encoding is only"): + ZF("nonesuch.zip", mode, metadata_encoding="shift_jis") + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_cli_with_metadata_encoding(self): + errmsg = "Non-conforming encodings not supported with -c." + args = ["--metadata-encoding=shift_jis", "-c", "nonesuch", "nonesuch"] + with captured_stdout() as stdout: + with captured_stderr() as stderr: + self.assertRaises(SystemExit, zipfile.main, args) + self.assertEqual(stdout.getvalue(), "") + self.assertIn(errmsg, stderr.getvalue()) + + with captured_stdout() as stdout: + zipfile.main(["--metadata-encoding=shift_jis", "-t", TESTFN]) + listing = stdout.getvalue() + + with captured_stdout() as stdout: + zipfile.main(["--metadata-encoding=shift_jis", "-l", TESTFN]) + listing = stdout.getvalue() + for name in self.file_names: + self.assertIn(name, listing) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + def test_cli_with_metadata_encoding_extract(self): + os.mkdir(TESTFN2) + self.addCleanup(rmtree, TESTFN2) + # Depending on locale, extracted file names can be not encodable + # with the filesystem encoding. + for fn in self.file_names: + try: + os.stat(os.path.join(TESTFN2, fn)) + except OSError: + pass + except UnicodeEncodeError: + self.skipTest(f'cannot encode file name {fn!a}') + + zipfile.main(["--metadata-encoding=shift_jis", "-e", TESTFN, TESTFN2]) + listing = os.listdir(TESTFN2) + for name in self.file_names: + self.assertIn(name, listing) + + +class StripExtraTests(unittest.TestCase): + # Note: all of the "z" characters are technically invalid, but up + # to 3 bytes at the end of the extra will be passed through as they + # are too short to encode a valid extra. + + ZIP64_EXTRA = 1 + + def test_no_data(self): + s = struct.Struct("<HH") + a = s.pack(self.ZIP64_EXTRA, 0) + b = s.pack(2, 0) + c = s.pack(3, 0) + + self.assertEqual(b'', zipfile._Extra.strip(a, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(b, (self.ZIP64_EXTRA,))) + self.assertEqual( + b+b"z", zipfile._Extra.strip(b+b"z", (self.ZIP64_EXTRA,))) + + self.assertEqual(b+c, zipfile._Extra.strip(a+b+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+a+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+c+a, (self.ZIP64_EXTRA,))) + + def test_with_data(self): + s = struct.Struct("<HH") + a = s.pack(self.ZIP64_EXTRA, 1) + b"a" + b = s.pack(2, 2) + b"bb" + c = s.pack(3, 3) + b"ccc" + + self.assertEqual(b"", zipfile._Extra.strip(a, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(b, (self.ZIP64_EXTRA,))) + self.assertEqual( + b+b"z", zipfile._Extra.strip(b+b"z", (self.ZIP64_EXTRA,))) + + self.assertEqual(b+c, zipfile._Extra.strip(a+b+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+a+c, (self.ZIP64_EXTRA,))) + self.assertEqual(b+c, zipfile._Extra.strip(b+c+a, (self.ZIP64_EXTRA,))) + + def test_multiples(self): + s = struct.Struct("<HH") + a = s.pack(self.ZIP64_EXTRA, 1) + b"a" + b = s.pack(2, 2) + b"bb" + + self.assertEqual(b"", zipfile._Extra.strip(a+a, (self.ZIP64_EXTRA,))) + self.assertEqual(b"", zipfile._Extra.strip(a+a+a, (self.ZIP64_EXTRA,))) + self.assertEqual( + b"z", zipfile._Extra.strip(a+a+b"z", (self.ZIP64_EXTRA,))) + self.assertEqual( + b+b"z", zipfile._Extra.strip(a+a+b+b"z", (self.ZIP64_EXTRA,))) + + self.assertEqual(b, zipfile._Extra.strip(a+a+b, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(a+b+a, (self.ZIP64_EXTRA,))) + self.assertEqual(b, zipfile._Extra.strip(b+a+a, (self.ZIP64_EXTRA,))) + + def test_too_short(self): + self.assertEqual(b"", zipfile._Extra.strip(b"", (self.ZIP64_EXTRA,))) + self.assertEqual(b"z", zipfile._Extra.strip(b"z", (self.ZIP64_EXTRA,))) + self.assertEqual( + b"zz", zipfile._Extra.strip(b"zz", (self.ZIP64_EXTRA,))) + self.assertEqual( + b"zzz", zipfile._Extra.strip(b"zzz", (self.ZIP64_EXTRA,))) + + +class StatIO(_pyio.BytesIO): + """Buffer which remembers the number of bytes that were read.""" + + def __init__(self): + super().__init__() + self.bytes_read = 0 + + def read(self, size=-1): + bs = super().read(size) + self.bytes_read += len(bs) + return bs + + +class StoredZipExtFileRandomReadTest(unittest.TestCase): + """Tests whether an uncompressed, unencrypted zip entry can be randomly + seek and read without reading redundant bytes.""" + def test_stored_seek_and_read(self): + + sio = StatIO() + # 20000 bytes + txt = b'0123456789' * 2000 + + # The seek length must be greater than ZipExtFile.MIN_READ_SIZE + # as `ZipExtFile._read2()` reads in blocks of this size and we + # need to seek out of the buffered data + read_buffer_size = zipfile.ZipExtFile.MIN_READ_SIZE + self.assertGreaterEqual(10002, read_buffer_size) # for forward seek test + self.assertGreaterEqual(5003, read_buffer_size) # for backward seek test + # The read length must be less than MIN_READ_SIZE, since we assume that + # only 1 block is read in the test. + read_length = 100 + self.assertGreaterEqual(read_buffer_size, read_length) # for read() calls + + with zipfile.ZipFile(sio, "w", compression=zipfile.ZIP_STORED) as zipf: + zipf.writestr("foo.txt", txt) + + # check random seek and read on a file + with zipfile.ZipFile(sio, "r") as zipf: + with zipf.open("foo.txt", "r") as fp: + # Test this optimized read hasn't rewound and read from the + # start of the file (as in the case of the unoptimized path) + + # forward seek + old_count = sio.bytes_read + forward_seek_len = 10002 + current_pos = 0 + fp.seek(forward_seek_len, os.SEEK_CUR) + current_pos += forward_seek_len + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(fp._left, fp._compress_left) + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) + self.assertEqual(fp._left, fp._compress_left) + read_count = sio.bytes_read - old_count + self.assertLessEqual(read_count, read_buffer_size) + + # backward seek + old_count = sio.bytes_read + backward_seek_len = 5003 + fp.seek(-backward_seek_len, os.SEEK_CUR) + current_pos -= backward_seek_len + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(fp._left, fp._compress_left) + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(fp.tell(), current_pos) + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) + self.assertEqual(fp._left, fp._compress_left) + read_count = sio.bytes_read - old_count + self.assertLessEqual(read_count, read_buffer_size) + + # eof flags test + fp.seek(0, os.SEEK_END) + fp.seek(12345, os.SEEK_SET) + current_pos = 12345 + arr = fp.read(read_length) + current_pos += read_length + self.assertEqual(arr, txt[current_pos - read_length:current_pos]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index 0947013afbc..2e1affe0252 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -11,7 +11,7 @@ 'test requires loads of disk-space bytes and a long time to run' ) -import zipfile, os, unittest +import zipfile, unittest import time import sys @@ -32,10 +32,6 @@ def setUp(self): line_gen = ("Test of zipfile line %d." % i for i in range(1000000)) self.data = '\n'.join(line_gen).encode('ascii') - # And write it to a file. - with open(TESTFN, "wb") as fp: - fp.write(self.data) - def zipTest(self, f, compression): # Create the ZIP archive. with zipfile.ZipFile(f, "w", compression) as zipfp: @@ -67,6 +63,9 @@ def zipTest(self, f, compression): (num, filecount)), file=sys.__stdout__) sys.__stdout__.flush() + # Check that testzip thinks the archive is valid + self.assertIsNone(zipfp.testzip()) + def testStored(self): # Try the temp file first. If we do TESTFN2 first, then it hogs # gigabytes of disk space for the duration of the test. @@ -85,9 +84,7 @@ def testDeflated(self): self.zipTest(TESTFN2, zipfile.ZIP_DEFLATED) def tearDown(self): - for fname in TESTFN, TESTFN2: - if os.path.exists(fname): - os.remove(fname) + os_helper.unlink(TESTFN2) class OtherTests(unittest.TestCase): diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index b291d530169..d448e3df5d7 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1,8 +1,10 @@ import sys import os import marshal +import glob import importlib import importlib.util +import re import struct import time import unittest @@ -50,10 +52,14 @@ def module_path_to_dotted_name(path): TESTMOD = "ziptestmodule" +TESTMOD2 = "ziptestmodule2" +TESTMOD3 = "ziptestmodule3" TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" +TESTPACK3 = "ziptestpackage3" TEMP_DIR = os.path.abspath("junk95142") TEMP_ZIP = os.path.abspath("junk95142.zip") +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data") pyc_file = importlib.util.cache_from_source(TESTMOD + '.py') pyc_ext = '.pyc' @@ -92,8 +98,10 @@ def makeTree(self, files, dirName=TEMP_DIR): # defined by files under the directory dirName. self.addCleanup(os_helper.rmtree, dirName) - for name, (mtime, data) in files.items(): - path = os.path.join(dirName, name) + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + path = os.path.join(dirName, *name.split('/')) if path[-1] == os.sep: if not os.path.isdir(path): os.makedirs(path) @@ -104,22 +112,18 @@ def makeTree(self, files, dirName=TEMP_DIR): with open(path, 'wb') as fp: fp.write(data) - def makeZip(self, files, zipName=TEMP_ZIP, **kw): + def makeZip(self, files, zipName=TEMP_ZIP, *, + comment=None, file_comment=None, stuff=None, prefix='', **kw): # Create a zip archive based set of modules/packages - # defined by files in the zip file zipName. If the - # key 'stuff' exists in kw it is prepended to the archive. + # defined by files in the zip file zipName. + # If stuff is not None, it is prepended to the archive. self.addCleanup(os_helper.unlink, zipName) - with ZipFile(zipName, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - z.writestr(zinfo, data) - comment = kw.get("comment", None) + with ZipFile(zipName, "w", compression=self.compression) as z: + self.writeZip(z, files, file_comment=file_comment, prefix=prefix) if comment is not None: z.comment = comment - stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile with open(zipName, "rb") as f: @@ -128,20 +132,47 @@ def makeZip(self, files, zipName=TEMP_ZIP, **kw): f.write(stuff) f.write(data) + def writeZip(self, z, files, *, file_comment=None, prefix=''): + for name, data in files.items(): + if isinstance(data, tuple): + mtime, data = data + else: + mtime = NOW + name = name.replace(os.sep, '/') + zinfo = ZipInfo(prefix + name, time.localtime(mtime)) + zinfo.compress_type = self.compression + if file_comment is not None: + zinfo.comment = file_comment + if data is None: + zinfo.CRC = 0 + z.mkdir(zinfo) + else: + assert name[-1] != '/' + z.writestr(zinfo, data) + + def getZip64Files(self): + # This is the simplest way to make zipfile generate the zip64 EOCD block + return {f"f{n}.py": test_src for n in range(65537)} + def doTest(self, expected_ext, files, *modules, **kw): + if 'prefix' not in kw: + kw['prefix'] = 'pre/fix/' self.makeZip(files, **kw) + self.doTestWithPreBuiltZip(expected_ext, *modules, **kw) - sys.path.insert(0, TEMP_ZIP) + def doTestWithPreBuiltZip(self, expected_ext, *modules, + call=None, prefix='', **kw): + zip_path = os.path.join(TEMP_ZIP, *prefix.split('/')[:-1]) + sys.path.insert(0, zip_path) mod = importlib.import_module(".".join(modules)) - call = kw.get('call') if call is not None: call(mod) if expected_ext: file = mod.get_file() - self.assertEqual(file, os.path.join(TEMP_ZIP, + self.assertEqual(file, os.path.join(zip_path, *modules) + expected_ext) def testAFakeZlib(self): @@ -155,7 +186,8 @@ def testAFakeZlib(self): # zlib.decompress function object, after which the problem being # tested here wouldn't be a problem anymore... # (Hence the 'A' in the test method name: to make it the first - # item in a list sorted by name, like unittest.makeSuite() does.) + # item in a list sorted by name, like + # unittest.TestLoader.getTestCaseNames() does.) # # This test fails on platforms on which the zlib module is # statically linked, but the problem it tests for can't @@ -166,7 +198,7 @@ def testAFakeZlib(self): self.skipTest('zlib is a builtin module') if "zlib" in sys.modules: del sys.modules["zlib"] - files = {"zlib.py": (NOW, test_src)} + files = {"zlib.py": test_src} try: self.doTest(".py", files, "zlib") except ImportError: @@ -177,16 +209,16 @@ def testAFakeZlib(self): self.fail("expected test to raise ImportError") def testPy(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD) def testPyc(self): - files = {TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testBoth(self): - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTMOD) def testUncheckedHashBasedPyc(self): @@ -219,22 +251,22 @@ def check(mod): self.doTest(None, files, TESTMOD, call=check) def testEmptyPy(self): - files = {TESTMOD + ".py": (NOW, "")} + files = {TESTMOD + ".py": ""} self.doTest(None, files, TESTMOD) def testBadMagic(self): # make pyc magic word invalid, forcing loading from .py badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badmagic_pyc} self.doTest(".py", files, TESTMOD) def testBadMagic2(self): # make pyc magic word invalid, causing an ImportError badmagic_pyc = bytearray(test_pyc) badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit - files = {TESTMOD + pyc_ext: (NOW, badmagic_pyc)} + files = {TESTMOD + pyc_ext: badmagic_pyc} try: self.doTest(".py", files, TESTMOD) self.fail("This should not be reached") @@ -247,22 +279,22 @@ def testBadMTime(self): # flip the second bit -- not the first as that one isn't stored in the # .py's mtime in the zip archive. badtime_pyc[11] ^= 0x02 - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, badtime_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: badtime_pyc} self.doTest(".py", files, TESTMOD) def test2038MTime(self): # Make sure we can handle mtimes larger than what a 32-bit signed number # can hold. twenty_thirty_eight_pyc = make_pyc(test_co, 2**32 - 1, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, twenty_thirty_eight_pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: twenty_thirty_eight_pyc} self.doTest(".py", files, TESTMOD) def testPackage(self): packdir = TESTPACK + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTMOD) def testSubPackage(self): @@ -270,9 +302,9 @@ def testSubPackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) def testSubNamespacePackage(self): @@ -281,29 +313,104 @@ def testSubNamespacePackage(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep # The first two files are just directory entries (so have no data). - files = {packdir: (NOW, ""), - packdir2: (NOW, ""), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir: None, + packdir2: None, + packdir2 + TESTMOD + pyc_ext: test_pyc} self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD) + def testPackageExplicitDirectories(self): + # Test explicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.writestr('a/__init__.py', test_src) + z.mkdir('a/b') + z.writestr('a/b/__init__.py', test_src) + z.mkdir('a/b/c') + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testPackageImplicitDirectories(self): + # Test explicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/__init__.py', test_src) + z.writestr('a/b/__init__.py', test_src) + z.writestr('a/b/c/__init__.py', test_src) + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile='__init__.py') + + def testNamespacePackageExplicitDirectories(self): + # Test implicit namespace packages with explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def testNamespacePackageImplicitDirectories(self): + # Test implicit namespace packages without explicit directory entries. + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.writestr('a/b/c/d.py', test_src) + self._testPackage(initfile=None) + + def _testPackage(self, initfile): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'a')) + if initfile is None: + # XXX Should it work? + self.assertRaises(zipimport.ZipImportError, zi.is_package, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_source, 'b') + self.assertRaises(zipimport.ZipImportError, zi.get_code, 'b') + else: + self.assertTrue(zi.is_package('b')) + self.assertEqual(zi.get_source('b'), test_src) + self.assertEqual(zi.get_code('b').co_filename, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + + sys.path.insert(0, TEMP_ZIP) + self.assertNotIn('a', sys.modules) + + mod = importlib.import_module(f'a.b') + self.assertIn('a', sys.modules) + self.assertIs(sys.modules['a.b'], mod) + if initfile is None: + self.assertIsNone(mod.__file__) + else: + self.assertEqual(mod.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', initfile)) + self.assertEqual(len(mod.__path__), 1, mod.__path__) + self.assertEqual(mod.__path__[0], os.path.join(TEMP_ZIP, 'a', 'b')) + + mod2 = importlib.import_module(f'a.b.c.d') + self.assertIn('a.b.c', sys.modules) + self.assertIn('a.b.c.d', sys.modules) + self.assertIs(sys.modules['a.b.c.d'], mod2) + self.assertIs(mod.c.d, mod2) + self.assertEqual(mod2.__file__, + os.path.join(TEMP_ZIP, 'a', 'b', 'c', 'd.py')) + def testMixedNamespacePackage(self): # Test implicit namespace packages spread between a # real filesystem and a zip archive. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) @@ -336,8 +443,8 @@ def testMixedNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-3]) @@ -354,13 +461,13 @@ def testMixedNamespacePackage(self): self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual(os.path.basename(TEMP_DIR), mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testNamespacePackage(self): @@ -368,22 +475,22 @@ def testNamespacePackage(self): # archives. packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - packdir3 = packdir2 + TESTPACK + '3' + os.sep - files1 = {packdir: (NOW, ""), - packdir + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir3: (NOW, ""), - packdir3 + TESTMOD + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + '3' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + packdir3 = packdir2 + TESTPACK3 + os.sep + files1 = {packdir: None, + packdir + TESTMOD + pyc_ext: test_pyc, + packdir2: None, + packdir3: None, + packdir3 + TESTMOD + pyc_ext: test_pyc, + packdir2 + TESTMOD3 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip1 = os.path.abspath("path1.zip") self.makeZip(files1, zip1) - files2 = {packdir: (NOW, ""), - packdir + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2: (NOW, ""), - packdir2 + TESTMOD + '2' + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files2 = {packdir: None, + packdir + TESTMOD2 + pyc_ext: test_pyc, + packdir2: None, + packdir2 + TESTMOD2 + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} zip2 = os.path.abspath("path2.zip") self.makeZip(files2, zip2) @@ -412,8 +519,8 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((TESTPACK, TESTMOD))) self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3]) - # And TESTPACK/(TESTMOD + '2') only exists in path2. - mod = importlib.import_module('.'.join((TESTPACK, TESTMOD + '2'))) + # And TESTPACK/(TESTMOD2) only exists in path2. + mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2))) self.assertEqual("path2.zip", mod.__file__.split(os.sep)[-3]) # One level deeper... @@ -428,29 +535,22 @@ def testNamespacePackage(self): mod = importlib.import_module('.'.join((subpkg, TESTMOD))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # subpkg.TESTMOD + '2' only exists in zip2. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '2'))) + # subpkg.TESTMOD2 only exists in zip2. + mod = importlib.import_module('.'.join((subpkg, TESTMOD2))) self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4]) - # Finally subpkg.TESTMOD + '3' only exists in zip1. - mod = importlib.import_module('.'.join((subpkg, TESTMOD + '3'))) + # Finally subpkg.TESTMOD3 only exists in zip1. + mod = importlib.import_module('.'.join((subpkg, TESTMOD3))) self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4]) def testZipImporterMethods(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) self.assertEqual(zi.archive, TEMP_ZIP) @@ -459,12 +559,6 @@ def testZipImporterMethods(self): # PEP 302 with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) - find_mod = zi.find_module('spam') - self.assertIsNotNone(find_mod) - self.assertIsInstance(find_mod, zipimport.zipimporter) - self.assertFalse(find_mod.is_package('spam')) - load_mod = find_mod.load_module('spam') - self.assertEqual(find_mod.get_filename('spam'), load_mod.__file__) mod = zi.load_module(TESTPACK) self.assertEqual(zi.get_filename(TESTPACK), mod.__file__) @@ -512,58 +606,70 @@ def testZipImporterMethods(self): def testInvalidateCaches(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc), - "spam" + pyc_ext: (NOW, test_pyc)} - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(zi._files.keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Check that the file information remains accurate after reloading zi.invalidate_caches() - self.assertEqual(zi._files.keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) # Add a new file to the ZIP archive - newfile = {"spam2" + pyc_ext: (NOW, test_pyc)} + newfile = {"spam2" + pyc_ext: test_pyc} files.update(newfile) - with ZipFile(TEMP_ZIP, "a") as z: - for name, (mtime, data) in newfile.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"spam" - z.writestr(zinfo, data) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") # Check that we can detect the new file after invalidating the cache zi.invalidate_caches() - self.assertEqual(zi._files.keys(), files.keys()) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) spec = zi.find_spec('spam2') self.assertIsNotNone(spec) self.assertIsInstance(spec.loader, zipimport.zipimporter) # Check that the cached data is removed if the file is deleted os.remove(TEMP_ZIP) zi.invalidate_caches() - self.assertFalse(zi._files) + self.assertFalse(zi._get_files()) self.assertIsNone(zipimport._zip_directory_cache.get(zi.archive)) self.assertIsNone(zi.find_spec("name_does_not_matter")) - def testZipImporterMethodsInSubDirectory(self): + def testInvalidateCachesWithMultipleZipimports(self): packdir = TESTPACK + os.sep packdir2 = packdir + TESTPACK2 + os.sep - files = {packdir2 + "__init__" + pyc_ext: (NOW, test_pyc), - packdir2 + TESTMOD + pyc_ext: (NOW, test_pyc)} + files = {packdir + "__init__" + pyc_ext: test_pyc, + packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc, + "spam" + pyc_ext: test_pyc} + extra_files = [packdir, packdir2] + self.makeZip(files, file_comment=b"spam") - self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - for name, (mtime, data) in files.items(): - zinfo = ZipInfo(name, time.localtime(mtime)) - zinfo.compress_type = self.compression - zinfo.comment = b"eggs" - z.writestr(zinfo, data) + zi = zipimport.zipimporter(TEMP_ZIP) + self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files])) + # Zipimporter for the same path. + zi2 = zipimport.zipimporter(TEMP_ZIP) + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) + # Add a new file to the ZIP archive to make the cache wrong. + newfile = {"spam2" + pyc_ext: test_pyc} + files.update(newfile) + with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z: + self.writeZip(z, newfile, file_comment=b"spam") + # Invalidate the cache of the first zipimporter. + zi.invalidate_caches() + # Check that the second zipimporter detects the new file and isn't using a stale cache. + self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files])) + spec = zi2.find_spec('spam2') + self.assertIsNotNone(spec) + self.assertIsInstance(spec.loader, zipimport.zipimporter) + + def testZipImporterMethodsInSubDirectory(self): + packdir = TESTPACK + os.sep + packdir2 = packdir + TESTPACK2 + os.sep + files = {packdir2 + "__init__" + pyc_ext: test_pyc, + packdir2 + TESTMOD + pyc_ext: test_pyc} + self.makeZip(files, file_comment=b"eggs") zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir) self.assertEqual(zi.archive, TEMP_ZIP) @@ -585,16 +691,6 @@ def testZipImporterMethodsInSubDirectory(self): pkg_path = TEMP_ZIP + os.sep + packdir + TESTPACK2 zi2 = zipimport.zipimporter(pkg_path) - # PEP 302 - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - find_mod_dotted = zi2.find_module(TESTMOD) - self.assertIsNotNone(find_mod_dotted) - self.assertIsInstance(find_mod_dotted, zipimport.zipimporter) - self.assertFalse(zi2.is_package(TESTMOD)) - load_mod = find_mod_dotted.load_module(TESTMOD) - self.assertEqual( - find_mod_dotted.get_filename(TESTMOD), load_mod.__file__) # PEP 451 spec = zi2.find_spec(TESTMOD) @@ -619,17 +715,33 @@ def testZipImporterMethodsInSubDirectory(self): self.assertIsNone(loader.get_source(mod_name)) self.assertEqual(loader.get_filename(mod_name), mod.__file__) - def testGetData(self): + def testGetDataExplicitDirectories(self): self.addCleanup(os_helper.unlink, TEMP_ZIP) - with ZipFile(TEMP_ZIP, "w") as z: - z.compression = self.compression - name = "testdata.dat" - data = bytes(x for x in range(256)) - z.writestr(name, data) - - zi = zipimport.zipimporter(TEMP_ZIP) - self.assertEqual(data, zi.get_data(name)) - self.assertIn('zipimporter object', repr(zi)) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + z.mkdir('a') + z.mkdir('a/b') + z.mkdir('a/b/c') + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def testGetDataImplicitDirectories(self): + self.addCleanup(os_helper.unlink, TEMP_ZIP) + with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z: + data = bytes(range(256)) + z.writestr('a/b/c/testdata.dat', data) + self._testGetData() + + def _testGetData(self): + zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'ignored')) + pathname = os.path.join('a', 'b', 'c', 'testdata.dat') + data = bytes(range(256)) + self.assertEqual(zi.get_data(pathname), data) + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, pathname)), data) + self.assertEqual(zi.get_data(os.path.join('a', 'b', '')), b'') + self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, 'a', 'b', '')), b'') + self.assertRaises(OSError, zi.get_data, os.path.join('a', 'b')) + self.assertRaises(OSError, zi.get_data, os.path.join(TEMP_ZIP, 'a', 'b')) def testImporterAttr(self): src = """if 1: # indent hack @@ -638,9 +750,9 @@ def get_file(): if __loader__.get_data("some.data") != b"some data": raise AssertionError("bad data")\n""" pyc = make_pyc(compile(src, "<???>", "exec"), NOW, len(src)) - files = {TESTMOD + pyc_ext: (NOW, pyc), - "some.data": (NOW, "some data")} - self.doTest(pyc_ext, files, TESTMOD) + files = {TESTMOD + pyc_ext: pyc, + "some.data": "some data"} + self.doTest(pyc_ext, files, TESTMOD, prefix='') def testDefaultOptimizationLevel(self): # zipimport should use the default optimization level (#28131) @@ -648,17 +760,20 @@ def testDefaultOptimizationLevel(self): def test(val): assert(val) return val\n""" - files = {TESTMOD + '.py': (NOW, src)} + files = {TESTMOD + '.py': src} self.makeZip(files) sys.path.insert(0, TEMP_ZIP) mod = importlib.import_module(TESTMOD) self.assertEqual(mod.test(1), 1) - self.assertRaises(AssertionError, mod.test, False) + if __debug__: + self.assertRaises(AssertionError, mod.test, False) + else: + self.assertEqual(mod.test(0), 0) def testImport_WithStuff(self): # try importing from a zipfile which contains additional # stuff at the beginning of the file - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"Some Stuff"*31) @@ -666,18 +781,18 @@ def assertModuleSource(self, module): self.assertEqual(inspect.getsource(module), test_src) def testGetSource(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, call=self.assertModuleSource) def testGetCompiledSource(self): pyc = make_pyc(compile(test_src, "<???>", "exec"), NOW, len(test_src)) - files = {TESTMOD + ".py": (NOW, test_src), - TESTMOD + pyc_ext: (NOW, pyc)} + files = {TESTMOD + ".py": test_src, + TESTMOD + pyc_ext: pyc} self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource) def runDoctest(self, callback): - files = {TESTMOD + ".py": (NOW, test_src), - "xyz.txt": (NOW, ">>> log.append(True)\n")} + files = {TESTMOD + ".py": test_src, + "xyz.txt": ">>> log.append(True)\n"} self.doTest(".py", files, TESTMOD, call=callback) def doDoctestFile(self, module): @@ -720,54 +835,179 @@ def doTraceback(self, module): s = io.StringIO() print_tb(tb, 1, s) - self.assertTrue(s.getvalue().endswith(raise_src)) + self.assertEndsWith(s.getvalue(), + ' def do_raise(): raise TypeError\n' + '' if support.has_no_debug_ranges() else + ' ^^^^^^^^^^^^^^^\n' + ) else: raise AssertionError("This ought to be impossible") + @unittest.expectedFailure # TODO: RUSTPYTHON; empty caret lines from equal col/end_col def testTraceback(self): - files = {TESTMOD + ".py": (NOW, raise_src)} + files = {TESTMOD + ".py": raise_src} self.doTest(None, files, TESTMOD, call=self.doTraceback) @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") def testUnencodable(self): filename = os_helper.TESTFN_UNENCODABLE + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) spec = zipimport.zipimporter(filename).find_spec(TESTMOD) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) def testBytesPath(self): filename = os_helper.TESTFN + ".zip" - self.addCleanup(os_helper.unlink, filename) - with ZipFile(filename, "w") as z: - zinfo = ZipInfo(TESTMOD + ".py", time.localtime(NOW)) - zinfo.compress_type = self.compression - z.writestr(zinfo, test_src) + self.makeZip({TESTMOD + ".py": test_src}, filename) zipimport.zipimporter(filename) - zipimport.zipimporter(os.fsencode(filename)) + with self.assertRaises(TypeError): + zipimport.zipimporter(os.fsencode(filename)) with self.assertRaises(TypeError): zipimport.zipimporter(bytearray(os.fsencode(filename))) with self.assertRaises(TypeError): zipimport.zipimporter(memoryview(os.fsencode(filename))) def testComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"comment") def testBeginningCruftAndComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi") def testLargestPossibleComment(self): - files = {TESTMOD + ".py": (NOW, test_src)} + files = {TESTMOD + ".py": test_src} self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1)) + @support.requires_resource('cpu') + def testZip64(self): + files = self.getZip64Files() + self.doTest(".py", files, "f6") + + @support.requires_resource('cpu') + def testZip64CruftAndComment(self): + files = self.getZip64Files() + self.doTest(".py", files, "f65536", comment=b"c" * ((1 << 16) - 1)) + + @unittest.skip("TODO: RUSTPYTHON; (intermittent success/failures); ValueError: name=\"RustPython/crates/pylib/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part\" does not fit expected pattern.") + def testZip64LargeFile(self): + support.requires( + "largefile", + f"test generates files >{0xFFFFFFFF} bytes and takes a long time " + "to run" + ) + + # N.B.: We do a lot of gymnastics below in the ZIP_STORED case to save + # and reconstruct a sparse zip on systems that support sparse files. + # Instead of creating a ~8GB zip file mainly consisting of null bytes + # for every run of the test, we create the zip once and save off the + # non-null portions of the resulting file as data blobs with offsets + # that allow re-creating the zip file sparsely. This drops disk space + # usage to ~9KB for the ZIP_STORED case and drops that test time by ~2 + # orders of magnitude. For the ZIP_DEFLATED case, however, we bite the + # bullet. The resulting zip file is ~8MB of non-null data; so the sparse + # trick doesn't work and would result in that full ~8MB zip data file + # being checked in to source control. + parts_glob = f"sparse-zip64-c{self.compression:d}-0x*.part" + full_parts_glob = os.path.join(TEST_DATA_DIR, parts_glob) + pre_built_zip_parts = glob.glob(full_parts_glob) + + self.addCleanup(os_helper.unlink, TEMP_ZIP) + if not pre_built_zip_parts: + if self.compression != ZIP_STORED: + support.requires( + "cpu", + "test requires a lot of CPU for compression." + ) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + f.write(b"data") + f.write(os.linesep.encode()) + f.seek(0xffff_ffff, os.SEEK_CUR) + f.write(os.linesep.encode()) + os.utime(os_helper.TESTFN, (0.0, 0.0)) + with ZipFile( + TEMP_ZIP, + "w", + compression=self.compression, + strict_timestamps=False + ) as z: + z.write(os_helper.TESTFN, "data1") + z.writestr( + ZipInfo("module.py", (1980, 1, 1, 0, 0, 0)), test_src + ) + z.write(os_helper.TESTFN, "data2") + + # This "works" but relies on the zip format having a non-empty + # final page due to the trailing central directory to wind up with + # the correct length file. + def make_sparse_zip_parts(name): + empty_page = b"\0" * 4096 + with open(name, "rb") as f: + part = None + try: + while True: + offset = f.tell() + data = f.read(len(empty_page)) + if not data: + break + if data != empty_page: + if not part: + part_fullname = os.path.join( + TEST_DATA_DIR, + f"sparse-zip64-c{self.compression:d}-" + f"{offset:#011x}.part", + ) + os.makedirs( + os.path.dirname(part_fullname), + exist_ok=True + ) + part = open(part_fullname, "wb") + print("Created", part_fullname) + part.write(data) + else: + if part: + part.close() + part = None + finally: + if part: + part.close() + + if self.compression == ZIP_STORED: + print(f"Creating sparse parts to check in into {TEST_DATA_DIR}:") + make_sparse_zip_parts(TEMP_ZIP) + + else: + def extract_offset(name): + if m := re.search(r"-(0x[0-9a-f]{9})\.part$", name): + return int(m.group(1), base=16) + raise ValueError(f"{name=} does not fit expected pattern.") + offset_parts = [(extract_offset(n), n) for n in pre_built_zip_parts] + with open(TEMP_ZIP, "wb") as f: + for offset, part_fn in sorted(offset_parts): + with open(part_fn, "rb") as part: + f.seek(offset, os.SEEK_SET) + f.write(part.read()) + # Confirm that the reconstructed zip file works and looks right. + with ZipFile(TEMP_ZIP, "r") as z: + self.assertEqual( + z.getinfo("module.py").date_time, (1980, 1, 1, 0, 0, 0) + ) + self.assertEqual( + z.read("module.py"), test_src.encode(), + msg=f"Recreate {full_parts_glob}, unexpected contents." + ) + def assertDataEntry(name): + zinfo = z.getinfo(name) + self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertGreater(zinfo.file_size, 0xffff_ffff) + assertDataEntry("data1") + assertDataEntry("data2") + + self.doTestWithPreBuiltZip(".py", "module") + @support.requires_zlib() class CompressedZipImportTestCase(UncompressedZipImportTestCase): @@ -799,6 +1039,7 @@ def testEmptyFile(self): os_helper.create_empty_file(TESTMOD) self.assertZipFailure(TESTMOD) + @unittest.skipIf(support.is_wasi, "mode 000 not supported.") def testFileUnreadable(self): os_helper.unlink(TESTMOD) fd = os.open(TESTMOD, os.O_CREAT, 000) @@ -842,7 +1083,6 @@ def _testBogusZipFile(self): self.assertRaises(TypeError, z.get_source, None) error = zipimport.ZipImportError - self.assertIsNone(z.find_module('abc')) self.assertIsNone(z.find_spec('abc')) with warnings.catch_warnings(): diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 026a5abc251..bb1366cb21c 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -3,11 +3,10 @@ from test.support import import_helper import binascii import copy -import os import pickle import random import sys -from test.support import bigmemtest, _1G, _4G +from test.support import bigmemtest, _1G, _4G, is_s390x zlib = import_helper.import_module('zlib') @@ -19,8 +18,24 @@ hasattr(zlib.decompressobj(), "copy"), 'requires Decompress.copy()') -# bpo-46623: On s390x, when a hardware accelerator is used, using different -# ways to compress data with zlib can produce different compressed data. + +def _zlib_runtime_version_tuple(zlib_version=zlib.ZLIB_RUNTIME_VERSION): + # Register "1.2.3" as "1.2.3.0" + # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" + v = zlib_version.split('-', 1)[0].split('.') + if len(v) < 4: + v.append('0') + elif not v[-1].isnumeric(): + v[-1] = '0' + return tuple(map(int, v)) + + +ZLIB_RUNTIME_VERSION_TUPLE = _zlib_runtime_version_tuple() + + +# bpo-46623: When a hardware accelerator is used (currently only on s390x), +# using different ways to compress data with zlib can produce different +# compressed data. # Simplified test_pair() code: # # def func1(data): @@ -43,16 +58,13 @@ # # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # -# Make the assumption that s390x always has an accelerator to simplify the skip -# condition. Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') +# To simplify the skip condition, make the assumption that s390x always has an +# accelerator, and nothing else has it. +HW_ACCELERATED = is_s390x class VersionTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_library_version(self): # Test that the major version of the actual library in use matches the # major version that we were compiled against. We can't guarantee that @@ -214,12 +226,14 @@ def test_keywords(self): bufsize=zlib.DEF_BUF_SIZE), HAMLET_SCENE) - @skip_on_s390x def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 x = zlib.compress(data) - self.assertEqual(zlib.compress(bytearray(data)), x) + # With hardware acceleration, the compressed bytes + # might not be identical. + if not HW_ACCELERATED: + self.assertEqual(zlib.compress(bytearray(data)), x) for ob in x, bytearray(x): self.assertEqual(zlib.decompress(ob), data) @@ -265,10 +279,7 @@ def test_64bit_compress(self, size): class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test compression object - @skip_on_s390x def test_pair(self): # straightforward compress/decompress objects datasrc = HAMLET_SCENE * 128 @@ -279,7 +290,10 @@ def test_pair(self): x1 = co.compress(data) x2 = co.flush() self.assertRaises(zlib.error, co.flush) # second flush should not work - self.assertEqual(x1 + x2, datazip) + # With hardware acceleration, the compressed bytes might not + # be identical. + if not HW_ACCELERATED: + self.assertEqual(x1 + x2, datazip) for v1, v2 in ((x1, x2), (bytearray(x1), bytearray(x2))): dco = zlib.decompressobj() y1 = dco.decompress(v1 + v2) @@ -288,8 +302,6 @@ def test_pair(self): self.assertIsInstance(dco.unconsumed_tail, bytes) self.assertIsInstance(dco.unused_data, bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_keywords(self): level = 2 method = zlib.DEFLATED @@ -447,8 +459,6 @@ def test_decompressmaxlen(self, flush=False): def test_decompressmaxlenflush(self): self.test_decompressmaxlen(flush=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_maxlenmisc(self): # Misc tests of max_length dco = zlib.decompressobj() @@ -479,17 +489,15 @@ def test_clear_unconsumed_tail(self): ddata += dco.decompress(dco.unconsumed_tail) self.assertEqual(dco.unconsumed_tail, b"") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Z_BLOCK support in flate2 def test_flushes(self): # Test flush() with the various options, using all the # different levels in order to provide more variations. sync_opt = ['Z_NO_FLUSH', 'Z_SYNC_FLUSH', 'Z_FULL_FLUSH', 'Z_PARTIAL_FLUSH'] - ver = tuple(int(v) for v in zlib.ZLIB_RUNTIME_VERSION.split('.')) # Z_BLOCK has a known failure prior to 1.2.5.3 - if ver >= (1, 2, 5, 3): + if ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 5, 3): sync_opt.append('Z_BLOCK') sync_opt = [getattr(zlib, opt) for opt in sync_opt @@ -498,20 +506,16 @@ def test_flushes(self): for sync in sync_opt: for level in range(10): - try: + with self.subTest(sync=sync, level=level): obj = zlib.compressobj( level ) a = obj.compress( data[:3000] ) b = obj.flush( sync ) c = obj.compress( data[3000:] ) d = obj.flush() - except: - print("Error for flush mode={}, level={}" - .format(sync, level)) - raise - self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), - data, ("Decompress failed: flush " - "mode=%i, level=%i") % (sync, level)) - del obj + self.assertEqual(zlib.decompress(b''.join([a,b,c,d])), + data, ("Decompress failed: flush " + "mode=%i, level=%i") % (sync, level)) + del obj @unittest.skipUnless(hasattr(zlib, 'Z_SYNC_FLUSH'), 'requires zlib.Z_SYNC_FLUSH') @@ -526,18 +530,7 @@ def test_odd_flush(self): # Try 17K of data # generate random data stream - try: - # In 2.3 and later, WichmannHill is the RNG of the bug report - gen = random.WichmannHill() - except AttributeError: - try: - # 2.2 called it Random - gen = random.Random() - except AttributeError: - # others might simply have a single RNG - gen = random - gen.seed(1) - data = gen.randbytes(17 * 1024) + data = random.randbytes(17 * 1024) # compress, sync-flush, and decompress first = co.compress(data) @@ -557,8 +550,6 @@ def test_empty_flush(self): dco = zlib.decompressobj() self.assertEqual(dco.flush(), b"") # Returns nothing - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dictionary(self): h = HAMLET_SCENE # Build a simulated dictionary out of the words in HAMLET. @@ -575,8 +566,6 @@ def test_dictionary(self): dco = zlib.decompressobj() self.assertRaises(zlib.error, dco.decompress, cd) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dictionary_streaming(self): # This simulates the reuse of a compressor object for compressing # several separate data streams. @@ -649,8 +638,6 @@ def test_decompress_unused_data(self): self.assertEqual(dco.unconsumed_tail, b'') self.assertEqual(dco.unused_data, remainder) - # TODO: RUSTPYTHON - @unittest.expectedFailure # issue27164 def test_decompress_raw_with_dictionary(self): zdict = b'abcdefghijklmnopqrstuvwxyz' @@ -758,15 +745,11 @@ def test_baddecompresscopy(self): self.assertRaises(ValueError, copy.copy, d) self.assertRaises(ValueError, copy.deepcopy, d) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compresspickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises((TypeError, pickle.PicklingError)): pickle.dumps(zlib.compressobj(zlib.Z_BEST_COMPRESSION), proto) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decompresspickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises((TypeError, pickle.PicklingError)): @@ -826,20 +809,10 @@ def test_large_unconsumed_tail(self, size): finally: comp = uncomp = data = None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; wbits=0 support in flate2 def test_wbits(self): # wbits=0 only supported since zlib v1.2.3.5 - # Register "1.2.3" as "1.2.3.0" - # or "1.2.0-linux","1.2.0.f","1.2.0.f-linux" - v = zlib.ZLIB_RUNTIME_VERSION.split('-', 1)[0].split('.') - if len(v) < 4: - v.append('0') - elif not v[-1].isnumeric(): - v[-1] = '0' - - v = tuple(map(int, v)) - supports_wbits_0 = v >= (1, 2, 3, 5) + supports_wbits_0 = ZLIB_RUNTIME_VERSION_TUPLE >= (1, 2, 3, 5) co = zlib.compressobj(level=1, wbits=15) zlib15 = co.compress(HAMLET_SCENE) + co.flush() @@ -966,6 +939,178 @@ def choose_lines(source, number, seed=None, generator=random): """ +class ZlibDecompressorTest(unittest.TestCase): + # Test adopted from test_bz2.py + TEXT = HAMLET_SCENE + DATA = zlib.compress(HAMLET_SCENE) + BAD_DATA = b"Not a valid deflate block" + BIG_TEXT = DATA * ((128 * 1024 // len(DATA)) + 1) + BIG_DATA = zlib.compress(BIG_TEXT) + + def test_Constructor(self): + self.assertRaises(TypeError, zlib._ZlibDecompressor, "ASDA") + self.assertRaises(TypeError, zlib._ZlibDecompressor, -15, "notbytes") + self.assertRaises(TypeError, zlib._ZlibDecompressor, -15, b"bytes", 5) + + def testDecompress(self): + zlibd = zlib._ZlibDecompressor() + self.assertRaises(TypeError, zlibd.decompress) + text = zlibd.decompress(self.DATA) + self.assertEqual(text, self.TEXT) + + def testDecompressChunks10(self): + zlibd = zlib._ZlibDecompressor() + text = b'' + n = 0 + while True: + str = self.DATA[n*10:(n+1)*10] + if not str: + break + text += zlibd.decompress(str) + n += 1 + self.assertEqual(text, self.TEXT) + + def testDecompressUnusedData(self): + zlibd = zlib._ZlibDecompressor() + unused_data = b"this is unused data" + text = zlibd.decompress(self.DATA+unused_data) + self.assertEqual(text, self.TEXT) + self.assertEqual(zlibd.unused_data, unused_data) + + def testEOFError(self): + zlibd = zlib._ZlibDecompressor() + text = zlibd.decompress(self.DATA) + self.assertRaises(EOFError, zlibd.decompress, b"anything") + self.assertRaises(EOFError, zlibd.decompress, b"") + + @support.skip_if_pgo_task + @bigmemtest(size=_4G + 100, memuse=3.3) + def testDecompress4G(self, size): + # "Test zlib._ZlibDecompressor.decompress() with >4GiB input" + blocksize = min(10 * 1024 * 1024, size) + block = random.randbytes(blocksize) + try: + data = block * ((size-1) // blocksize + 1) + compressed = zlib.compress(data) + zlibd = zlib._ZlibDecompressor() + decompressed = zlibd.decompress(compressed) + self.assertTrue(decompressed == data) + finally: + data = None + compressed = None + decompressed = None + + def testPickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(zlib._ZlibDecompressor(), proto) + + def testDecompressorChunksMaxsize(self): + zlibd = zlib._ZlibDecompressor() + max_length = 100 + out = [] + + # Feed some input + len_ = len(self.BIG_DATA) - 64 + out.append(zlibd.decompress(self.BIG_DATA[:len_], + max_length=max_length)) + self.assertFalse(zlibd.needs_input) + self.assertEqual(len(out[-1]), max_length) + + # Retrieve more data without providing more input + out.append(zlibd.decompress(b'', max_length=max_length)) + self.assertFalse(zlibd.needs_input) + self.assertEqual(len(out[-1]), max_length) + + # Retrieve more data while providing more input + out.append(zlibd.decompress(self.BIG_DATA[len_:], + max_length=max_length)) + self.assertLessEqual(len(out[-1]), max_length) + + # Retrieve remaining uncompressed data + while not zlibd.eof: + out.append(zlibd.decompress(b'', max_length=max_length)) + self.assertLessEqual(len(out[-1]), max_length) + + out = b"".join(out) + self.assertEqual(out, self.BIG_TEXT) + self.assertEqual(zlibd.unused_data, b"") + + def test_decompressor_inputbuf_1(self): + # Test reusing input buffer after moving existing + # contents to beginning + zlibd = zlib._ZlibDecompressor() + out = [] + + # Create input buffer and fill it + self.assertEqual(zlibd.decompress(self.DATA[:100], + max_length=0), b'') + + # Retrieve some results, freeing capacity at beginning + # of input buffer + out.append(zlibd.decompress(b'', 2)) + + # Add more data that fits into input buffer after + # moving existing data to beginning + out.append(zlibd.decompress(self.DATA[100:105], 15)) + + # Decompress rest of data + out.append(zlibd.decompress(self.DATA[105:])) + self.assertEqual(b''.join(out), self.TEXT) + + def test_decompressor_inputbuf_2(self): + # Test reusing input buffer by appending data at the + # end right away + zlibd = zlib._ZlibDecompressor() + out = [] + + # Create input buffer and empty it + self.assertEqual(zlibd.decompress(self.DATA[:200], + max_length=0), b'') + out.append(zlibd.decompress(b'')) + + # Fill buffer with new data + out.append(zlibd.decompress(self.DATA[200:280], 2)) + + # Append some more data, not enough to require resize + out.append(zlibd.decompress(self.DATA[280:300], 2)) + + # Decompress rest of data + out.append(zlibd.decompress(self.DATA[300:])) + self.assertEqual(b''.join(out), self.TEXT) + + def test_decompressor_inputbuf_3(self): + # Test reusing input buffer after extending it + + zlibd = zlib._ZlibDecompressor() + out = [] + + # Create almost full input buffer + out.append(zlibd.decompress(self.DATA[:200], 5)) + + # Add even more data to it, requiring resize + out.append(zlibd.decompress(self.DATA[200:300], 5)) + + # Decompress rest of data + out.append(zlibd.decompress(self.DATA[300:])) + self.assertEqual(b''.join(out), self.TEXT) + + def test_failure(self): + zlibd = zlib._ZlibDecompressor() + self.assertRaises(Exception, zlibd.decompress, self.BAD_DATA * 30) + # Previously, a second call could crash due to internal inconsistency + self.assertRaises(Exception, zlibd.decompress, self.BAD_DATA * 30) + + @support.refcount_test + def test_refleaks_in___init__(self): + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + zlibd = zlib._ZlibDecompressor() + refs_before = gettotalrefcount() + for i in range(100): + zlibd.__init__() + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + class CustomInt: def __index__(self): return 100 diff --git a/Lib/test/test_zoneinfo/__init__.py b/Lib/test/test_zoneinfo/__init__.py new file mode 100644 index 00000000000..4b16ecc3115 --- /dev/null +++ b/Lib/test/test_zoneinfo/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_zoneinfo/__main__.py b/Lib/test/test_zoneinfo/__main__.py new file mode 100644 index 00000000000..5cc4e055d5e --- /dev/null +++ b/Lib/test/test_zoneinfo/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_zoneinfo') diff --git a/Lib/test/test_zoneinfo/_support.py b/Lib/test/test_zoneinfo/_support.py new file mode 100644 index 00000000000..5a76c163fb7 --- /dev/null +++ b/Lib/test/test_zoneinfo/_support.py @@ -0,0 +1,100 @@ +import contextlib +import functools +import sys +import threading +import unittest +from test.support.import_helper import import_fresh_module + +OS_ENV_LOCK = threading.Lock() +TZPATH_LOCK = threading.Lock() +TZPATH_TEST_LOCK = threading.Lock() + + +def call_once(f): + """Decorator that ensures a function is only ever called once.""" + lock = threading.Lock() + cached = functools.lru_cache(None)(f) + + @functools.wraps(f) + def inner(): + with lock: + return cached() + + return inner + + +@call_once +def get_modules(): + """Retrieve two copies of zoneinfo: pure Python and C accelerated. + + Because this function manipulates the import system in a way that might + be fragile or do unexpected things if it is run many times, it uses a + `call_once` decorator to ensure that this is only ever called exactly + one time — in other words, when using this function you will only ever + get one copy of each module rather than a fresh import each time. + """ + import zoneinfo as c_module + + py_module = import_fresh_module("zoneinfo", blocked=["_zoneinfo"]) + + return py_module, c_module + + +@contextlib.contextmanager +def set_zoneinfo_module(module): + """Make sure sys.modules["zoneinfo"] refers to `module`. + + This is necessary because `pickle` will refuse to serialize + an type calling itself `zoneinfo.ZoneInfo` unless `zoneinfo.ZoneInfo` + refers to the same object. + """ + + NOT_PRESENT = object() + old_zoneinfo = sys.modules.get("zoneinfo", NOT_PRESENT) + sys.modules["zoneinfo"] = module + yield + if old_zoneinfo is not NOT_PRESENT: + sys.modules["zoneinfo"] = old_zoneinfo + else: # pragma: nocover + sys.modules.pop("zoneinfo") + + +class ZoneInfoTestBase(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.klass = cls.module.ZoneInfo + super().setUpClass() + + @contextlib.contextmanager + def tzpath_context(self, tzpath, block_tzdata=True, lock=TZPATH_LOCK): + def pop_tzdata_modules(): + tzdata_modules = {} + for modname in list(sys.modules): + if modname.split(".", 1)[0] != "tzdata": # pragma: nocover + continue + + tzdata_modules[modname] = sys.modules.pop(modname) + + return tzdata_modules + + with lock: + if block_tzdata: + # In order to fully exclude tzdata from the path, we need to + # clear the sys.modules cache of all its contents — setting the + # root package to None is not enough to block direct access of + # already-imported submodules (though it will prevent new + # imports of submodules). + tzdata_modules = pop_tzdata_modules() + sys.modules["tzdata"] = None + + old_path = self.module.TZPATH + try: + self.module.reset_tzpath(tzpath) + yield + finally: + if block_tzdata: + sys.modules.pop("tzdata") + for modname, module in tzdata_modules.items(): + sys.modules[modname] = module + + self.module.reset_tzpath(old_path) diff --git a/Lib/test/test_zoneinfo/data/update_test_data.py b/Lib/test/test_zoneinfo/data/update_test_data.py new file mode 100644 index 00000000000..f531ab316a1 --- /dev/null +++ b/Lib/test/test_zoneinfo/data/update_test_data.py @@ -0,0 +1,122 @@ +""" +Script to automatically generate a JSON file containing time zone information. + +This is done to allow "pinning" a small subset of the tzdata in the tests, +since we are testing properties of a file that may be subject to change. For +example, the behavior in the far future of any given zone is likely to change, +but "does this give the right answer for this file in 2040" is still an +important property to test. + +This must be run from a computer with zoneinfo data installed. +""" +from __future__ import annotations + +import base64 +import functools +import json +import lzma +import pathlib +import textwrap +import typing + +import zoneinfo + +KEYS = [ + "Africa/Abidjan", + "Africa/Casablanca", + "America/Los_Angeles", + "America/Santiago", + "Asia/Tokyo", + "Australia/Sydney", + "Europe/Dublin", + "Europe/Lisbon", + "Europe/London", + "Pacific/Kiritimati", + "UTC", +] + +TEST_DATA_LOC = pathlib.Path(__file__).parent + + +@functools.lru_cache(maxsize=None) +def get_zoneinfo_path() -> pathlib.Path: + """Get the first zoneinfo directory on TZPATH containing the "UTC" zone.""" + key = "UTC" + for path in map(pathlib.Path, zoneinfo.TZPATH): + if (path / key).exists(): + return path + else: + raise OSError("Cannot find time zone data.") + + +def get_zoneinfo_metadata() -> typing.Dict[str, str]: + path = get_zoneinfo_path() + + tzdata_zi = path / "tzdata.zi" + if not tzdata_zi.exists(): + # tzdata.zi is necessary to get the version information + raise OSError("Time zone data does not include tzdata.zi.") + + with open(tzdata_zi, "r") as f: + version_line = next(f) + + _, version = version_line.strip().rsplit(" ", 1) + + if ( + not version[0:4].isdigit() + or len(version) < 5 + or not version[4:].isalpha() + ): + raise ValueError( + "Version string should be YYYYx, " + + "where YYYY is the year and x is a letter; " + + f"found: {version}" + ) + + return {"version": version} + + +def get_zoneinfo(key: str) -> bytes: + path = get_zoneinfo_path() + + with open(path / key, "rb") as f: + return f.read() + + +def encode_compressed(data: bytes) -> typing.List[str]: + compressed_zone = lzma.compress(data) + raw = base64.b85encode(compressed_zone) + + raw_data_str = raw.decode("utf-8") + + data_str = textwrap.wrap(raw_data_str, width=70) + return data_str + + +def load_compressed_keys() -> typing.Dict[str, typing.List[str]]: + output = {key: encode_compressed(get_zoneinfo(key)) for key in KEYS} + + return output + + +def update_test_data(fname: str = "zoneinfo_data.json") -> None: + TEST_DATA_LOC.mkdir(exist_ok=True, parents=True) + + # Annotation required: https://github.com/python/mypy/issues/8772 + json_kwargs: typing.Dict[str, typing.Any] = dict( + indent=2, sort_keys=True, + ) + + compressed_keys = load_compressed_keys() + metadata = get_zoneinfo_metadata() + output = { + "metadata": metadata, + "data": compressed_keys, + } + + with open(TEST_DATA_LOC / fname, "w") as f: + json.dump(output, f, **json_kwargs) + + +if __name__ == "__main__": + update_test_data() diff --git a/Lib/test/test_zoneinfo/data/zoneinfo_data.json b/Lib/test/test_zoneinfo/data/zoneinfo_data.json new file mode 100644 index 00000000000..ec4414a0cde --- /dev/null +++ b/Lib/test/test_zoneinfo/data/zoneinfo_data.json @@ -0,0 +1,190 @@ +{ + "data": { + "Africa/Abidjan": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j-~f{VGF<>F7KxBg5R*{Ksocg8-YYVul=v7vZzaHN", + "uC=da5UI2rH18c!OnjV{y4u(+A!!VBKmY&$ORw>7UO^(500B;v0RR91bXh%WvBYQl0ssI2", + "00dcD" + ], + "Africa/Casablanca": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j;0b&Kz+C_;7KxBg5R*{N&yjMUR~;C-fDaSOU;q-~", + "FqW+4{YBjbcw}`a!dW>b)R2-0a+uwf`P3{_Y@HuCz}S$J$ZJ>R_V<~|Fk>sgX4=%0vUrh-", + "lt@YP^Wrus;j?`Th#xRPzf<<~Hp4DH^gZX>d{+WOp~HNu8!{uWu}&XphAd{j1;rB4|9?R!", + "pqruAFUMt8#*WcrVS{;kLlY(cJRV$w?d2car%R<ALOSO?^`4;ZZtI)%f^^G^>s>q9BgTU4", + "Ht-tQKZ7Z`9QqOb?R#b%z?rk>!CkH7jy3wja4NG2q)H}fNRKg8v{);Em;K3Cncf4C6&Oaj", + "V+DbX%o4+)CV3+e!Lm6dutu(0BQpH1T?W(~cQtKV*^_Pdx!LirjpTs?Bmt@vktjLq4;)O!", + "rrly=c*rwTwMJFd0I57`hgkc?=nyI4RZf9W$6DCWugmf&)wk^tWH17owj=#PGH7Xv-?9$j", + "njwDlkOE+BFNR9YXEmBpO;rqEw=e2IR-8^(W;8ma?M3JVd($2T>IW+0tk|Gm8>ftukRQ9J", + "8k3brzqMnVyjsLI-CKneFa)Lxvp_a<CkQEd#(pMA^rr}rBNElGA=*!M)puBdoErR9{kWL@", + "w=svMc6eZ^-(vQZrV<u^PY#nOIUDJ8%A&;BUVlY9=;@i2j2J1_`P>q40f}0J3VVoWL5rox", + "`Kptivcp}o5xA^@>qNI%?zo=Yj4AMV?kbAA)j(1%)+Pp)bSn+7Yk`M{oE}L-Z!G6<Dgq&*", + "(C-mFJfbEGDH5M^vBr65rcnsx*~|Em_GeU#B)(+T!|MG-nxj0@IPbp-nHejH3~>OMr5G+h", + "p)$3Lg{ono{4cN>Vr&>L4kXH;_VnBL5U!LgzqE%P7QQ*<E!guRW2SE@ayq@)G2nXqA2tGo", + "QIgc6>tue}O`3(TZ0`aKn&~8trOQ-rBXCp)f@P6RMO4l0+;b|5-pk9_ryNh}Zc*v%mvz_#", + "yd<xXt%~gT90dn4e{Ac<baL-)Y{L7&5G($I$>6fjB0g9{MmMnu8bG%#C~ugXK^S^k@?ab#", + "O|aE>dDTt4s4n69(~@t~!wniV%g<uWQat_i6>7khFx~I*4>Y|V$4j5%KPF*-FyKIi@!Ho&", + "x8QQsksYt8)D+W)Ni!=G`ogSu^vLL-l#7A7=iIAKL2SuZk9F}NfNk86VI)9WZE?%2wC-ya", + "F~z#Qsq)LH0|_D8^5fU8X%GeQ4TB>R-dlziA&tZe&1ada208!$nk`7bOFO2S00G<w{Sp8G", + "{cR_IvBYQl0ssI200dcD" + ], + "America/Los_Angeles": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j;0qH3OkDsf7KxBg5R*;z{h&-RlhRYu$%jt%!jv+I", + "JxhE=%W1?wYb!37Rb?(rgwFIAQI{L#8r*zy!$TMtER_1(vn(Zix^{AVB1(jwr$iL6h0Z!2", + "8Gb~UW@0~e512{Z%8}Qzdnjl~wJ1{c2>`Z@1A~t&lyL{p{eM{5)QGf7Mo5FW9==mlyXJt2", + "UwpntR7H0eSq!(aYq#aqUz&RM*tvuMI)AsM?K3-dV3-TT{t)!Iy#JTo=tXkzAM9~j2YbiO", + "ls3(H8Dc>Y|D1aqL51vjLbpYG;GvGTQB4bXuJ%mA;(B4eUpu$$@zv2vVcq-Y)VKbzp^tei", + "uzy}R{Luv<C;_cPe*n$Z<jeC9ogWF9=1mvvUYXS>DjpuVb`79O+CBmg{Wx!bvx$eu4zRE&", + "PehMb=&G<9$>iZ|bFE)0=4I?KLFGBC0I(0_svgw0%FiMsT%koo*!nEYc6GY@QnU}&4Isg;", + "l=|khi(!VaiSE2=Ny`&&tpi~~;{$u<GHlsr3Ze!iYsU205RFKsLnrXwOL?Mq08xffgS{6h", + "E|figx+&N%wbO}re@|}$l;g_6J-Wl%j|qev8A<T?NJ)`;2neGi_DHE4ET*W!c*ggPAgU+L", + "E9=bH7;maCUikw^R)UM;TdVvNkQ;FGgN=yQER`SZ1nOgPXr0LCebLety&}kVdmVmB=8eSg", + "td!1%p=a2wooIL!Da}OPXvKBfRo?YxqS>N}%f|7mBhAy;<Er2&_LfND#qXN~Mkgf!@4VFA", + "Hr%$c)wrKA2cJYWK2>s3YT^sy!$eG~?`9mNJC9@4Bac_p^BZh)Yd_rWW5qh-?tKY(>5VHO", + "L*iT8P@wCavLj^yYbnDR+4ukhS+xPrpl)iqB?u)bj9a2aW==g6G3lCJd>(+Blf<d4CF%7u", + "tlBUDki}J-!_Dy}5S(MrxSXy~$Z+hgH3P^<<w7D72L7I-R%H3(xm&q_DXxkp$owLTS6Wzk", + "hc3nn;laROa3)6hl&gH#)2Lif8fZe$@CdeJ-Zn&*>r)~^40F4f>cRZ^UF;RibfZ>0m73hR", + "C{$vTfC(STN`g7(B<=Z2556{}0`?p&|Akkst!4Xy4OT;A@c$XTUI3FRRjy*KA7uC56FD)z", + "^X{WV*sr(w!c$W357o!&eLO2wTDNOyw@gf(&R<<LF_3URI4=Ei`-%dM3T66j#9!aG7&b_@", + "g1-9vo?DzXZ5vGaf~w__p_@_X?OdvQ_r5bvy2hpESTf+{p?jL+!~!{g8-<-5$@d8EZV&-5", + "@a|;^1gB*R-~{EHFA-td_G2bt;~Y}>t;=-Tu1TV{>%8ZVATC9tjD8|(&`$9YHvZ9bVe#>w", + "|8c;Tg|xE&)`*}LwM*E}q}q8^Qja%p`_U)*5DdLI9O@!e=3jFjOCrCq28b_bb;s>%D#iJB", + "CWJi{JH!Js;6nfayos$kq^OEX00HO-lokL0!mqm{vBYQl0ssI200dcD" + ], + "America/Santiago": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j;0fRZ<6QtM7KxBg84(fsEAUJ$J{f-TXlPEUec5Ee", + "n+hsD4lC(QYax=JdSpoyje8%VM`GW}<Unz6IOY4=y66tfqG2X4E8xIJQ(~?r{`L~T!sI~o", + "VBl7Ao!R1A76Y8P6Y<TfwVHf@sl@S-D4OuAy5mq0MKJZ>{bJ8@y$A8O&*$pw{(f~Os#}2w", + "eX6^Rgi$IT%n^V^85L>$_c7{cB^#ogV=rHBJGiz-RQNFGK?gdPi|q)j`&8)}KJ{qo6dixa", + "9@yYyVg+%lo0nO+Tw0-w2hJ%mafy<Co(;L+24CYl&?rN0mrh90nxG?%1&Ed@za`Yd>WL)|", + ")<o0dZL-*?RFtH7dAv%G*O%l?qvq!0F5C?K#_ZoT{P$77IMoj3&8w3f&n36zquu~s`s0T)", + ";>?W6Bi%FWuGPA1Dru$XR4SZANsAthU2EoKH<MU4wYvUTlZGcLIDR+hSik>F6oEtKq`rwP", + "(VNegnI_NI%;ma$)wj{k!@KFB30Yo)IOr<QX7IQ@TBq9d;e3QAtYU?$PS-WoaiqwFrg4PR", + "A->l>)$)D|+(5h&+%2vuwGuy^@S8FT^s21V5};>VA9Iu;?8bHz#r<;JtfZDI1(FT@edh0#", + "MYW$A1qkMGIwTZqqdYNE3gl#zp&NbL9Mp=voqN|;?gqR&4$)1`znddtEyuKS*^nMMD=0^>", + "7^z6-C4P67UWOXuMBubP>j6i~03aR@jD^-Y`JSYu#Yp0P8dLLJ0QOPE8=BoiuRX59YW7xg", + "WiexjHX%&0?`ZQCdxCdL^qd1v@kOjQKaWo2Y1++~LcA%FTq?5o<?(jL(_Uo}I}k_Fwflcr", + "aovwSR_(ILA6li<iBLPQ0#rEet;W-*54kj#sZEGK*tAF{)HNkn#&Hc5`#eaRF;N#$<xQU?", + "E%zm?2+b5Ho>%}fX1-RIvlB)1#iTNomGnUL=nM!>Ix|AGtON7!F1O?53kqlC2o-`ZGw*+s", + "NM$^9znsIJMwlgscE`|O3|;BRgsQMYm~`uv+nvuv`nigRa}X=BX=A5Sw$)WEklF7&c>_~$", + "zJ(m--bqXgiN^w-U=BJH9C0Qro(x90zo@rK;&TJ$nI@&k$ORgOb2<MjjIhYfr;pFUGdMd!", + "0d&bOvyq3AZPCez8E(XSg2hBu2A&^k?w|1u8v3JE>s%gWbc}ok_27)Eoku~Fq|B-Ps+4J_", + "HPJMLJ2^_)cOU$p&3kNAlrV!)%~6r$BJ>OOi~=-<6byle{?zd4J{NG}o8tw|+#ZNLcpNwk", + "TuPE~sbJB8_RZb2DopStO+Wwux~F#S59zm%00I98;S&G=b(j+6vBYQl0ssI200dcD" + ], + "Asia/Tokyo": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j-~luMgIxeB7KxBg5R*;y?l4Rl4neXH3cv!OtfK@h", + "KZzauI)S!FSDREPhhBS6Fb$&Vv#7%;?Te|>pF^0HBr&z_Tk<%vMW_QqjevRZOp8XVFgP<8", + "TkT#`9H&0Ua;gT1#rZLV0HqbAKK;_z@nO;6t0L<i8TZ+%T<;ci2bYSG1u!mUSO5S3XcbN8", + "dIxbZ00Ex?wE_SDJu@vkvBYQl0ssI200dcD" + ], + "Australia/Sydney": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j;0T)o7+nA=7KxBg5R*_t6jS5T`_Ull(nK1_YY;k%", + ";_YdTuU3*!K)eKg@^kzjAtbo@Jd|KGai=Q%%sX5FI?*?LG!|m9cKH5~IEwI=PAr_Yc}w35", + ">}hOdk<>TdUa07R(LPI6@!GU$ty4=mwqHG-XVe*n(Yvgdlr+FqIU18!osi)48t~eWX8)&L", + "G)Ud^0zz@*AF+2r7E}N<P$kOfo*88g)_bOO?7N1Jr|HJyg+HXc7f4}?%Dur3w|~JU?<x4K", + "%RRC~q_D87;UyN{nLRu!fEqKeRR*U$vs>f9Y72K~o-T%}D&z%}#7g<qim`EbfhF7ntyAiP", + "%LFNc&!$@Kv)Olyf&Y9%(#SkM+%yI}S%b+@ZM2dH7DpmndGMIda<(`#E9q|?H(HzClx+l;", + "M?IEz1eF}r?}ay!V9?9rKD^-ayjE@wUMD$2kC!iwH`n=eVrJPmJyNKaW`LdJ68&u;2nF1K", + "kZjKCY_A<>2br?oH6ZiYH^%>J3D)TPKV(JY*bwjuw5=DsPB@~CrR<E_U_fJTF9ufU%!cXK", + "_4uM#!%%Q1e1G~{E}~vGVE0{Kxecm^NjtJM`c8EFHFTiUIVl@YUD8F+s!u8jz~6hte@oa|", + "qayb*^Lwd(etNmBro;aXQjkY8g(*`_JQ0%{V3QP2l!GGQ7D+v&k_PK0F(?f{GziU5>OZeN", + "x>A*H&CHrWt0`EP`m!F%waepl#|w#&`XgVc?~2M3uw$fGX~tf_Il!q#Aa<*8xlzQ2+7r6Z", + "^;Laa9F(WB_O&Dy2r>~@kSi16W{=6+i5GV=Uq~KX*~&HUN4oz7*O(gXIr}sDVcD`Ikgw#|", + "50ssal8s)Qy;?YGCf;*UKKKN!T4!Kqy_G;7<gSrPK{)5#a>PfQapugqvVBKy12v3TVH^L2", + "0?#5*VP~MOYfe$h`*L!7@tiW|_^X1N%<}`7YahiUYtMu5XwmOf3?dr+@zXHwW`z}ZDqZlT", + "<2Cs(<1%M!i6o&VK89BY0J7HPIo;O62s=|IbV^@y$N&#<x=a876<(U>=>i^F00FcHoDl#3", + "Mdv&xvBYQl0ssI200dcD" + ], + "Europe/Dublin": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j;0>b$_+0=h7KxBg5R*;&J77#T_U2R5sleVWFDmK~", + "Kzj5oh@`<njquRZ&tJIS(cXp1>QKHvW^6V{jU-w>qg1tSt0c^vh;?qAqA0%t?;#S~6U8Qi", + "v&f1s9IH#g$m1k1a#3+lylw4mwT4QnEUUQdwg+xnEcBlgu31bAVabn41OMZVLGz6NDwG%X", + "uQar!b>GI{qSahE`AG}$kRWbuI~JCt;38)Xwbb~Qggs55t+MAHIxgDxzTJ;2xXx99+qCy4", + "45kC#v_l8fx|G&jlVvaciR<-wwf22l%4(t@S6tnX39#_K(4S0fu$FUs$isu<UOJYm|4)2i", + "aEpsajn@}B#rnY=Cg_TXsm-A)*adXV&$klNTn3n{XXlaquu}6m{k%oRmY0Yyhlj*<W{D5m", + "22}OiqnwHT!tnK`wPqx?wiF%v{ipTrOkcJ5P@7OC4(-l`*&SB$Wd4Vf8gn?>d<i@%mP*e*", + "ttDj`9M1;9$YV@dhT)DVcwdq(Ly~KDm_&KL?{_mFwwYtJqRZBk)i1FVQy!40w_KyAg?hIA", + "=_{(3#S0eWsF8f%_4Zza$4@$lSmov+Huyn$vP^zJ|8-<C3#q#0kEs9cNg^xUR(m?wEWt-D", + "GctAh2nIo~fz%$m$I41=b_WuJ6M9g#A9_Epwqw{d0B|vzmg#_y<=_>9IKzCXB<o`d)**5V", + "6g!<<Jw1n5TrN-$)aYz4cLsTmpsUf-6L7ix+kk>78NkARYq@9Dc0TGkhz);NtM_SSzEffN", + "l{2^*CKGdp52h!52A)6q9fUSltXF{T*Ehc9Q7u8!W7pE(Fv$D$cKUAt6wY=DA1mGgxC*VX", + "q_If3G#FY6-Voj`fIKk`0}Cc72_SD{v>468LV{pyBI33^p0E?}RwDA6Pkq--C~0jF&Z@Pv", + "!dx_1SN_)jwz@P$(oK%P!Tk9?fRjK88yxhxlcFtTjjZ$DYssSsa#ufYrR+}}nKS+r384o~", + "!Uw$nwTbF~qgRsgr0N#d@KIinx%<pnyQ!|>hQB(SJyjJtDtIy(%mDm}ZBGN}dV6K~om|=U", + "VGkbciQ=^$_14|gT21!YQ)@y*Rd0i_lS6gtPBE9+ah%WIJPwzUTjIr+J1XckkmA!6WE16%", + "CVAl{Dn&-)=G$Bjh?bh0$Xt1UDcgXJjXzzojuw0>paV~?Sa`VN3FysqF<S*L0RYSAY3jt(", + "8wCD04RfyEcP(RNT%x7k(7m-9H3{zuQ`RZy-Rz%*&dldDVFF+TwSAPO1wRX^5W5@xJ9{vW", + "w?rc^NH({%Ie<rxKqSVy!Le-_`U&@W_(D+>xTzfKVAu*ucq#+m=|KSSMvp_#@-lwd+q*ue", + "FQ^5<D+|jLr?k{O39i8AX2Qb^zi9A<7XD1y!-W2|0Hk8JVkN;gl><|<0R-u4qYMbRqzSn&", + "Q7jSuvc%b+EZc%>nI(+&0Tl1Y>a6v4`uNFD-7$QrhHgS7Wnv~rDgfH;rQw3+m`LJxoM4v#", + "gK@?|B{RHJ*VxZgk#!p<_&-sjxOda0YaiJ1UnG41VPv(Et%ElzKRMcO$AfgU+Xnwg5p2_+", + "NrnZ1WfEj^fmHd^sx@%JWKkh#zaK0ox%rdP)zUmGZZnqmZ_9L=%6R8ibJH0bOT$AGhDo6{", + "fJ?;_U;D|^>5by2ul@i4Zf()InfFN}00EQ=q#FPL>RM>svBYQl0ssI200dcD" + ], + "Europe/Lisbon": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j;0=rf*IfWA7KxBg5R*;*X|PN+G3LqthM?xgkNUN_", + ")gCt1Sc%YT6^TTomk4yVHXeyvQj8}l<;q&s7K}#Vnc8lII1?)AHh$*>OKUU4S;*h>v*ep0", + "xTi1cK2{aY*|2D*-~K<;-{_W+r@NvZ7-|NZv($ek_C%VfP0xjWeZP#CPXD`IKkakjh(kUd", + "&H)m;^Q(jGjIyiyrcUMtOP)u3A>sw6ux;Bmp3x$4QvQKMx5TrCx_!$srWQuXNs&`9=^IY1", + "yc&C31!sQh7P=Mk*#6x8Z@5^%ehR8UW<EvzdWer9z;R6PrdUaWab3G>$OWw0KMw}P1ycI^", + "4eh12oBUOV?S>n*d!+EM@>x#9PZD12iD=zaC;7`8dTfkU_6d}OZvSFSbGgXeKw}XyX@D=(", + ")D0!^DBGr8pXWBT$S-yhLP>Z3ys^VW<kSQr?{jhl<+{Fki;mTI=&Stgy$rttN?ulQM$lDr", + "G7))C7Dx=J6V-e^(Qk|r;f~TvIw1KqRIC{8f^jPy#blstV{-&2a}ZJe!Zr2c_R4NT)L@bs", + "+gRRm6Wn)VWVNHeK*TEV=f#2KZqu%y?mTx#EfRiK0)TG7$$~=LGxx@0D|lS2up|oCON{YQ", + "oN5-H$!_n-Kx2*=RO!epEX>3}RQ6{NGGVJG6vf*MH93vvNW6yLjie1;{4tVhg-KnSf|G`!", + "Z;j$7gJ1ows~RD=@n7I6aFd8rOR_7Y?E-$clI%1o5gA@O!KPa^(8^iFFeFykI-+z>E$mvp", + "E_h`vbHPjqkLs`Dn-0FV`R@z|h!S(Lb;M&|Exr<u8#s-T(>!biY`%bfp$6`hK;GDhdP|^Q", + "*Ty*}1d41K>H2B{jrjE9aFK>yAQJBX9CD%-384S;0fw`PlprHGS`^b$oS-`I4VH7ji8ou-", + "g|060jfb1XcxiInT0oO<S+<vh^)XY;lr@|IeXj}%k;}|kSlDGaYidk^zB|gEYaet~F%QYd", + "f7pbnQKLZ0o7=kso86doS;J@aQ>oeR7#%e5Ug5#KW)nV<Rc;|LjUDdhk8*dYJQwYN?hzH%", + "0<XB$!(rpf2nxaL22M`L4pKx>SRvLHNe$SQHM@2)`S9L7>RL@<XAlxVQfb2=%lcu!h+Um0", + "Q+Z=itevTFy}-Jl<g5crK55BF`VsoPH~qP3QrG%YtrD#s{=gA7p)QI<i=EwY(cel8`B=#u", + "Yq<K;4T(QBF_GvrYueSk*}gfrCSg22+YH-1N<WYkp|DA-P-&va<Xu<}^yafJKlzezB-lS{", + "a++P_^gYmgrc9FO-K3s~`jAcqVV!k?NV2IFV^86`cr>Qx%fmm7?3u7P5TywFQ}C@S(pq}|", + "eLPT{C^{<0Q?uU&kSVd%!~8q3;Z0s3OqzF`$HRkePL5Ywgiwn{R(<RY8ut&RJ;$?J*w*n)", + ">zi+jmOBFrVpW;)@UsU#%$8BcV#h@}m$#!Fglo&bwb78aYqOG_W7h{eb(+39&-mk4EIXq_", + "_`30=8sfA3=!3TO_TyS5X22~?6nKngZ|bq=grdq=9X)3xAkA42L!~rmS)n3w-~;lgz%Fhn", + "(?rXdp2ho~9?wmVs2JwVt~?@FVD%`tN69{(i3oQa;O0<Hp#T5?$WIy3h`IlL00Hv}jT-;}", + "Z2tpNvBYQl0ssI200dcD" + ], + "Europe/London": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j;0`|pJ6!-O7KxBg5R*;$9DqzW!kQs3DZt(=0_!m1", + "4wvE`6N%Vj#u6PS_3S?~(2)&xn8}2}3Wr#kG8n2!x8>$E$lF&~Y#_H6bu6(BiwblJ>;-Fs", + "gA$Y$*?=X)n1pFkKn}F~`>=4)+LLQk?L*P!bhAm0;`N~z3QbUIyVrm%kOZ(n1JJsm0pyb8", + "!GV{d*C!9KXv;4v<seWRpo=ZZxGf)-5Qsn$3dw`uhF)+6#mgUoNF-Y2jN73pVhdTs*p0`Z", + "AbnT1puEtudB{Nul>D4Q>-k#+x(!V5L@w5M>v2V5<gcLskF+p`aGTSn{sY8^@MUc;2o{&V", + "R!$180N}BtfYKS)i9w=!<~&l?1Cv^PWs!&a9{s(35^yqGU$72DKX|IkRtDblB>a`B>t(|B", + "|Fqr4^-{S*%Ep~ojUtx_CRbSQ(uFwu2=KH)Q@EBs@ZqRXn4mU;B!68;;IQs3Ub=n&UU%*m", + "k&zwD36&JSwsN(%k&x?H+tN^6)23c`I0=5^N_R0~1>tsFZ`^`3z~rXSXT&qcwa#n!%+Z#P", + "PG}(D^_CCILXnF|GKwabBh*xFS?4rwGo2vtJUwzrbv_$5PO+`?$l{H-jGB@X%S!OAhw;D4", + "XFycN3!XqQ&EorJOD3>~^U%Luw!jF<;6_q-f-S|6<EHry?%{@fuyH`_+D%uTA@g0$5e!Yi", + "P1vQuevyS;jE(-R>{cQDfZ2(4Xf1MMLr1=SA=MwVf2%Pp%VP;jn)|5Tf!-DbUGn%I-r<KG", + "4jJ(Y#L-fJUpUb$yNfvhX*iqWZoG7T*WUfE6iQD9_^EWqExH`rc&jJ<o^E8-mM10WrZ_Vv", + "xx9nj<vMlEt*KfP*pyth!c_AKnrKtQTACX08#{pioAFnDq!53+h*hO^f*yrWjg0u2pUcgv", + "UlpEZ9G_dlhlW1J^h@gTt7{KPL2mRal;1juJ3Q8-!GXO#IPzT4ciJ-nB+nkphssM}Q7IAT", + "pM}AT%y(J!78F?>kYaH7?$$O!t)wwClAisr3eUoeB^~T=U*_P~Y2*KdnO87>B!19sV=xZ5", + "yApq26RxgqA|*tmsvtL#OhcF(C<0EGWHP)BF<g*iSWicU6k1<Ps?BQ$IWg-#s2uF-qXgJ_", + "!H_mZIMx*L%&a*_6;_trMCULk0ZYM<hfJlYBddHwRyYUDu3!C_lJZWTQ?c-R&@9054pj0k", + "kQ{Xi{A$&)&b#^G*}8w^qE5i<@aDxaJQs2E$W)AIqUXO{gQ;U8|FA%BD~sORzq44)AntUu", + "QHBO{{Pi<EpK!$x4(~7w)la!dN=M@L_j};6|5G&QfuO~2?Q7996z)78fqW<D#8tKNV(*qc", + "mfA>l?h)_*7!{LoJiv%RsOs!q->n+DcV%9~B@Rb<ISu%16c5H-7zQIq+SuS+s<lQOWK5+C", + "d*>C_1G_1g6`Yd~8|%-=2l~oGN!~TVv2Bnk>7wW8L@^?vX$f3AiT)(4nrCuTm9%(XC6Nai", + "E(;}7&=YZagjAN$O-cN;1u{dTkElmB0GT$|Wa)QMmKrx<|LCJ9qlUoFsUbD^H^6_8(w<0{", + "ftj&O1~p_%lh5z;zNV&sP<T$*OgK)_0B#JDtXOkhC;Bo7h)#RUy;vBiVLN-T$*7t*t9@ey", + "3Woa&24QZ_z38BQ@A(A<(9n@%R?}B`7%w2wowt~UU;bAlqCzr(H$M5t==jGIqMqCsE=Jwa", + "$3P+3^&|~i28@=d_u6Cgthe(Lq(wxKpdSDL|7X6Un<nrt00Gwuz#ISo`BbmvvBYQl0ssI2", + "00dcD" + ], + "Pacific/Kiritimati": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j-~jCaVO;<!7KxBg5R*{K!`A|q%C5j6({{dSEy5>+", + "NF2>iK{8KMUf+)<-)VxXbLxD(alL}N$AT-ogNbJSMMYeX+Z{jS)b8TK^PB=FxyBxzfmFto", + "eo0R`a(%NO?#aEH9|?Cv00000NIsFh6BW2800DjO0RR918Pu^`vBYQl0ssI200dcD" + ], + "UTC": [ + "{Wp48S^xk9=GL@E0stWa761SMbT8$j-~e#|9bEt_7KxBg5R*|3h1|xhHLji!C57qW6L*|H", + "pEErm00000ygu;I+>V)?00B92fhY-(AGY&-0RR9100dcD" + ] + }, + "metadata": { + "version": "2020a" + } +} \ No newline at end of file diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py new file mode 100644 index 00000000000..85877c984c8 --- /dev/null +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -0,0 +1,2344 @@ +from __future__ import annotations + +import base64 +import contextlib +import dataclasses +import importlib.metadata +import io +import json +import os +import pathlib +import pickle +import re +import shutil +import struct +import tempfile +import unittest +from datetime import date, datetime, time, timedelta, timezone +from functools import cached_property + +from test.support import MISSING_C_DOCSTRINGS +from test.support.os_helper import EnvironmentVarGuard, FakePath +from test.test_zoneinfo import _support as test_support +from test.test_zoneinfo._support import TZPATH_TEST_LOCK, ZoneInfoTestBase +from test.support.import_helper import import_module, CleanImport +from test.support.script_helper import assert_python_ok + +lzma = import_module('lzma') +py_zoneinfo, c_zoneinfo = test_support.get_modules() + +try: + importlib.metadata.metadata("tzdata") + HAS_TZDATA_PKG = True +except importlib.metadata.PackageNotFoundError: + HAS_TZDATA_PKG = False + +ZONEINFO_DATA = None +ZONEINFO_DATA_V1 = None +TEMP_DIR = None +DATA_DIR = pathlib.Path(__file__).parent / "data" +ZONEINFO_JSON = DATA_DIR / "zoneinfo_data.json" +DRIVE = os.path.splitdrive('x:')[0] + +# Useful constants +ZERO = timedelta(0) +ONE_H = timedelta(hours=1) + + +def setUpModule(): + global TEMP_DIR + global ZONEINFO_DATA + global ZONEINFO_DATA_V1 + + TEMP_DIR = pathlib.Path(tempfile.mkdtemp(prefix="zoneinfo")) + ZONEINFO_DATA = ZoneInfoData(ZONEINFO_JSON, TEMP_DIR / "v2") + ZONEINFO_DATA_V1 = ZoneInfoData(ZONEINFO_JSON, TEMP_DIR / "v1", v1=True) + + +def tearDownModule(): + shutil.rmtree(TEMP_DIR) + + +class CustomError(Exception): + pass + + +class TzPathUserMixin: + """ + Adds a setUp() and tearDown() to make TZPATH manipulations thread-safe. + + Any tests that require manipulation of the TZPATH global are necessarily + thread unsafe, so we will acquire a lock and reset the TZPATH variable + to the default state before each test and release the lock after the test + is through. + """ + + @property + def tzpath(self): # pragma: nocover + return None + + @property + def block_tzdata(self): + return True + + def setUp(self): + with contextlib.ExitStack() as stack: + stack.enter_context( + self.tzpath_context( + self.tzpath, + block_tzdata=self.block_tzdata, + lock=TZPATH_TEST_LOCK, + ) + ) + self.addCleanup(stack.pop_all().close) + + super().setUp() + + +class DatetimeSubclassMixin: + """ + Replaces all ZoneTransition transition dates with a datetime subclass. + """ + + class DatetimeSubclass(datetime): + @classmethod + def from_datetime(cls, dt): + return cls( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + tzinfo=dt.tzinfo, + fold=dt.fold, + ) + + def load_transition_examples(self, key): + transition_examples = super().load_transition_examples(key) + for zt in transition_examples: + dt = zt.transition + new_dt = self.DatetimeSubclass.from_datetime(dt) + new_zt = dataclasses.replace(zt, transition=new_dt) + yield new_zt + + +class ZoneInfoTest(TzPathUserMixin, ZoneInfoTestBase): + module = py_zoneinfo + class_name = "ZoneInfo" + + def setUp(self): + super().setUp() + + # This is necessary because various subclasses pull from different + # data sources (e.g. tzdata, V1 files, etc). + self.klass.clear_cache() + + @property + def zoneinfo_data(self): + return ZONEINFO_DATA + + @property + def tzpath(self): + return [self.zoneinfo_data.tzpath] + + def zone_from_key(self, key): + return self.klass(key) + + def zones(self): + return ZoneDumpData.transition_keys() + + def fixed_offset_zones(self): + return ZoneDumpData.fixed_offset_zones() + + def load_transition_examples(self, key): + return ZoneDumpData.load_transition_examples(key) + + def test_str(self): + # Zones constructed with a key must have str(zone) == key + for key in self.zones(): + with self.subTest(key): + zi = self.zone_from_key(key) + + self.assertEqual(str(zi), key) + + # Zones with no key constructed should have str(zone) == repr(zone) + file_key = self.zoneinfo_data.keys[0] + file_path = self.zoneinfo_data.path_from_key(file_key) + + with open(file_path, "rb") as f: + with self.subTest(test_name="Repr test", path=file_path): + zi_ff = self.klass.from_file(f) + self.assertEqual(str(zi_ff), repr(zi_ff)) + + def test_repr(self): + # The repr is not guaranteed, but I think we can insist that it at + # least contain the name of the class. + key = next(iter(self.zones())) + + zi = self.klass(key) + class_name = self.class_name + with self.subTest(name="from key"): + self.assertRegex(repr(zi), class_name) + + file_key = self.zoneinfo_data.keys[0] + file_path = self.zoneinfo_data.path_from_key(file_key) + with open(file_path, "rb") as f: + zi_ff = self.klass.from_file(f, key=file_key) + + with self.subTest(name="from file with key"): + self.assertRegex(repr(zi_ff), class_name) + + with open(file_path, "rb") as f: + zi_ff_nk = self.klass.from_file(f) + + with self.subTest(name="from file without key"): + self.assertRegex(repr(zi_ff_nk), class_name) + + def test_key_attribute(self): + key = next(iter(self.zones())) + + def from_file_nokey(key): + with open(self.zoneinfo_data.path_from_key(key), "rb") as f: + return self.klass.from_file(f) + + constructors = ( + ("Primary constructor", self.klass, key), + ("no_cache", self.klass.no_cache, key), + ("from_file", from_file_nokey, None), + ) + + for msg, constructor, expected in constructors: + zi = constructor(key) + + # Ensure that the key attribute is set to the input to ``key`` + with self.subTest(msg): + self.assertEqual(zi.key, expected) + + # Ensure that the key attribute is read-only + with self.subTest(f"{msg}: readonly"): + with self.assertRaises(AttributeError): + zi.key = "Some/Value" + + def test_bad_keys(self): + bad_keys = [ + "Eurasia/Badzone", # Plausible but does not exist + "BZQ", + "America.Los_Angeles", + "🇨🇦", # Non-ascii + "America/New\ud800York", # Contains surrogate character + "Europe", # Is a directory, see issue gh-85702 + ] + + for bad_key in bad_keys: + with self.assertRaises(self.module.ZoneInfoNotFoundError): + self.klass(bad_key) + + def test_bad_keys_paths(self): + bad_keys = [ + "/America/Los_Angeles", # Absolute path + "America/Los_Angeles/", # Trailing slash - not normalized + "../zoneinfo/America/Los_Angeles", # Traverses above TZPATH + "America/../America/Los_Angeles", # Not normalized + "America/./Los_Angeles", + ] + + for bad_key in bad_keys: + with self.assertRaises(ValueError): + self.klass(bad_key) + + def test_bad_zones(self): + bad_zones = [ + b"", # Empty file + b"AAAA3" + b" " * 15, # Bad magic + # Truncated V2 file (should not loop indefinitely) + b"TZif2" + (b"\x00" * 39) + b"TZif2" + (b"\x00" * 39) + b"\n" + b"Part", + ] + + for bad_zone in bad_zones: + fobj = io.BytesIO(bad_zone) + with self.assertRaises(ValueError): + self.klass.from_file(fobj) + + def test_fromutc_errors(self): + key = next(iter(self.zones())) + zone = self.zone_from_key(key) + + bad_values = [ + (datetime(2019, 1, 1, tzinfo=timezone.utc), ValueError), + (datetime(2019, 1, 1), ValueError), + (date(2019, 1, 1), TypeError), + (time(0), TypeError), + (0, TypeError), + ("2019-01-01", TypeError), + ] + + for val, exc_type in bad_values: + with self.subTest(val=val): + with self.assertRaises(exc_type): + zone.fromutc(val) + + def test_utc(self): + zi = self.klass("UTC") + dt = datetime(2020, 1, 1, tzinfo=zi) + + self.assertEqual(dt.utcoffset(), ZERO) + self.assertEqual(dt.dst(), ZERO) + self.assertEqual(dt.tzname(), "UTC") + + def test_unambiguous(self): + test_cases = [] + for key in self.zones(): + for zone_transition in self.load_transition_examples(key): + test_cases.append( + ( + key, + zone_transition.transition - timedelta(days=2), + zone_transition.offset_before, + ) + ) + + test_cases.append( + ( + key, + zone_transition.transition + timedelta(days=2), + zone_transition.offset_after, + ) + ) + + for key, dt, offset in test_cases: + with self.subTest(key=key, dt=dt, offset=offset): + tzi = self.zone_from_key(key) + dt = dt.replace(tzinfo=tzi) + + self.assertEqual(dt.tzname(), offset.tzname, dt) + self.assertEqual(dt.utcoffset(), offset.utcoffset, dt) + self.assertEqual(dt.dst(), offset.dst, dt) + + def test_folds_and_gaps(self): + test_cases = [] + for key in self.zones(): + tests = {"folds": [], "gaps": []} + for zt in self.load_transition_examples(key): + if zt.fold: + test_group = tests["folds"] + elif zt.gap: + test_group = tests["gaps"] + else: + # Assign a random variable here to disable the peephole + # optimizer so that coverage can see this line. + # See bpo-2506 for more information. + no_peephole_opt = None + continue + + # Cases are of the form key, dt, fold, offset + dt = zt.anomaly_start - timedelta(seconds=1) + test_group.append((dt, 0, zt.offset_before)) + test_group.append((dt, 1, zt.offset_before)) + + dt = zt.anomaly_start + test_group.append((dt, 0, zt.offset_before)) + test_group.append((dt, 1, zt.offset_after)) + + dt = zt.anomaly_start + timedelta(seconds=1) + test_group.append((dt, 0, zt.offset_before)) + test_group.append((dt, 1, zt.offset_after)) + + dt = zt.anomaly_end - timedelta(seconds=1) + test_group.append((dt, 0, zt.offset_before)) + test_group.append((dt, 1, zt.offset_after)) + + dt = zt.anomaly_end + test_group.append((dt, 0, zt.offset_after)) + test_group.append((dt, 1, zt.offset_after)) + + dt = zt.anomaly_end + timedelta(seconds=1) + test_group.append((dt, 0, zt.offset_after)) + test_group.append((dt, 1, zt.offset_after)) + + for grp, test_group in tests.items(): + test_cases.append(((key, grp), test_group)) + + for (key, grp), tests in test_cases: + with self.subTest(key=key, grp=grp): + tzi = self.zone_from_key(key) + + for dt, fold, offset in tests: + dt = dt.replace(fold=fold, tzinfo=tzi) + + self.assertEqual(dt.tzname(), offset.tzname, dt) + self.assertEqual(dt.utcoffset(), offset.utcoffset, dt) + self.assertEqual(dt.dst(), offset.dst, dt) + + def test_folds_from_utc(self): + for key in self.zones(): + zi = self.zone_from_key(key) + with self.subTest(key=key): + for zt in self.load_transition_examples(key): + if not zt.fold: + continue + + dt_utc = zt.transition_utc + dt_before_utc = dt_utc - timedelta(seconds=1) + dt_after_utc = dt_utc + timedelta(seconds=1) + + dt_before = dt_before_utc.astimezone(zi) + self.assertEqual(dt_before.fold, 0, (dt_before, dt_utc)) + + dt_after = dt_after_utc.astimezone(zi) + self.assertEqual(dt_after.fold, 1, (dt_after, dt_utc)) + + def test_time_variable_offset(self): + # self.zones() only ever returns variable-offset zones + for key in self.zones(): + zi = self.zone_from_key(key) + t = time(11, 15, 1, 34471, tzinfo=zi) + + with self.subTest(key=key): + self.assertIs(t.tzname(), None) + self.assertIs(t.utcoffset(), None) + self.assertIs(t.dst(), None) + + def test_time_fixed_offset(self): + for key, offset in self.fixed_offset_zones(): + zi = self.zone_from_key(key) + + t = time(11, 15, 1, 34471, tzinfo=zi) + + with self.subTest(key=key): + self.assertEqual(t.tzname(), offset.tzname) + self.assertEqual(t.utcoffset(), offset.utcoffset) + self.assertEqual(t.dst(), offset.dst) + + def test_cache_exception(self): + class Incomparable(str): + eq_called = False + def __eq__(self, other): + self.eq_called = True + raise CustomError + __hash__ = str.__hash__ + + key = "America/Los_Angeles" + tz1 = self.klass(key) + key = Incomparable(key) + try: + tz2 = self.klass(key) + except CustomError: + self.assertTrue(key.eq_called) + else: + self.assertFalse(key.eq_called) + self.assertIs(tz2, tz1) + + +class CZoneInfoTest(ZoneInfoTest): + module = c_zoneinfo + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signatures(self): + """Ensure that C module has valid method signatures.""" + import inspect + + must_have_signatures = ( + self.klass.clear_cache, + self.klass.no_cache, + self.klass.from_file, + ) + for method in must_have_signatures: + with self.subTest(method=method): + inspect.Signature.from_callable(method) + + def test_fold_mutate(self): + """Test that fold isn't mutated when no change is necessary. + + The underlying C API is capable of mutating datetime objects, and + may rely on the fact that addition of a datetime object returns a + new datetime; this test ensures that the input datetime to fromutc + is not mutated. + """ + + def to_subclass(dt): + class SameAddSubclass(type(dt)): + def __add__(self, other): + if other == timedelta(0): + return self + + return super().__add__(other) # pragma: nocover + + return SameAddSubclass( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + fold=dt.fold, + tzinfo=dt.tzinfo, + ) + + subclass = [False, True] + + key = "Europe/London" + zi = self.zone_from_key(key) + for zt in self.load_transition_examples(key): + if zt.fold and zt.offset_after.utcoffset == ZERO: + example = zt.transition_utc.replace(tzinfo=zi) + break + + for subclass in [False, True]: + if subclass: + dt = to_subclass(example) + else: + dt = example + + with self.subTest(subclass=subclass): + dt_fromutc = zi.fromutc(dt) + + self.assertEqual(dt_fromutc.fold, 1) + self.assertEqual(dt.fold, 0) + + +class ZoneInfoDatetimeSubclassTest(DatetimeSubclassMixin, ZoneInfoTest): + pass + + +class CZoneInfoDatetimeSubclassTest(DatetimeSubclassMixin, CZoneInfoTest): + pass + + +class ZoneInfoSubclassTest(ZoneInfoTest): + @classmethod + def setUpClass(cls): + super().setUpClass() + + class ZISubclass(cls.klass): + pass + + cls.class_name = "ZISubclass" + cls.parent_klass = cls.klass + cls.klass = ZISubclass + + def test_subclass_own_cache(self): + base_obj = self.parent_klass("Europe/London") + sub_obj = self.klass("Europe/London") + + self.assertIsNot(base_obj, sub_obj) + self.assertIsInstance(base_obj, self.parent_klass) + self.assertIsInstance(sub_obj, self.klass) + + +class CZoneInfoSubclassTest(ZoneInfoSubclassTest): + module = c_zoneinfo + + +class ZoneInfoV1Test(ZoneInfoTest): + @property + def zoneinfo_data(self): + return ZONEINFO_DATA_V1 + + def load_transition_examples(self, key): + # We will discard zdump examples outside the range epoch +/- 2**31, + # because they are not well-supported in Version 1 files. + epoch = datetime(1970, 1, 1) + max_offset_32 = timedelta(seconds=2 ** 31) + min_dt = epoch - max_offset_32 + max_dt = epoch + max_offset_32 + + for zt in ZoneDumpData.load_transition_examples(key): + if min_dt <= zt.transition <= max_dt: + yield zt + + +class CZoneInfoV1Test(ZoneInfoV1Test): + module = c_zoneinfo + + +@unittest.skipIf( + not HAS_TZDATA_PKG, "Skipping tzdata-specific tests: tzdata not installed" +) +class TZDataTests(ZoneInfoTest): + """ + Runs all the ZoneInfoTest tests, but against the tzdata package + + NOTE: The ZoneDumpData has frozen test data, but tzdata will update, so + some of the tests (particularly those related to the far future) may break + in the event that the time zone policies in the relevant time zones change. + """ + + @property + def tzpath(self): + return [] + + @property + def block_tzdata(self): + return False + + def zone_from_key(self, key): + return self.klass(key=key) + + +@unittest.skipIf( + not HAS_TZDATA_PKG, "Skipping tzdata-specific tests: tzdata not installed" +) +class CTZDataTests(TZDataTests): + module = c_zoneinfo + + +class WeirdZoneTest(ZoneInfoTestBase): + module = py_zoneinfo + + def test_one_transition(self): + LMT = ZoneOffset("LMT", -timedelta(hours=6, minutes=31, seconds=2)) + STD = ZoneOffset("STD", -timedelta(hours=6)) + + transitions = [ + ZoneTransition(datetime(1883, 6, 9, 14), LMT, STD), + ] + + after = "STD6" + + zf = self.construct_zone(transitions, after) + zi = self.klass.from_file(zf) + + dt0 = datetime(1883, 6, 9, 1, tzinfo=zi) + dt1 = datetime(1883, 6, 10, 1, tzinfo=zi) + + for dt, offset in [(dt0, LMT), (dt1, STD)]: + with self.subTest(name="local", dt=dt): + self.assertEqual(dt.tzname(), offset.tzname) + self.assertEqual(dt.utcoffset(), offset.utcoffset) + self.assertEqual(dt.dst(), offset.dst) + + dts = [ + ( + datetime(1883, 6, 9, 1, tzinfo=zi), + datetime(1883, 6, 9, 7, 31, 2, tzinfo=timezone.utc), + ), + ( + datetime(2010, 4, 1, 12, tzinfo=zi), + datetime(2010, 4, 1, 18, tzinfo=timezone.utc), + ), + ] + + for dt_local, dt_utc in dts: + with self.subTest(name="fromutc", dt=dt_local): + dt_actual = dt_utc.astimezone(zi) + self.assertEqual(dt_actual, dt_local) + + dt_utc_actual = dt_local.astimezone(timezone.utc) + self.assertEqual(dt_utc_actual, dt_utc) + + def test_one_zone_dst(self): + DST = ZoneOffset("DST", ONE_H, ONE_H) + transitions = [ + ZoneTransition(datetime(1970, 1, 1), DST, DST), + ] + + after = "STD0DST-1,0/0,J365/25" + + zf = self.construct_zone(transitions, after) + zi = self.klass.from_file(zf) + + dts = [ + datetime(1900, 3, 1), + datetime(1965, 9, 12), + datetime(1970, 1, 1), + datetime(2010, 11, 3), + datetime(2040, 1, 1), + ] + + for dt in dts: + dt = dt.replace(tzinfo=zi) + with self.subTest(dt=dt): + self.assertEqual(dt.tzname(), DST.tzname) + self.assertEqual(dt.utcoffset(), DST.utcoffset) + self.assertEqual(dt.dst(), DST.dst) + + def test_no_tz_str(self): + STD = ZoneOffset("STD", ONE_H, ZERO) + DST = ZoneOffset("DST", 2 * ONE_H, ONE_H) + + transitions = [] + for year in range(1996, 2000): + transitions.append( + ZoneTransition(datetime(year, 3, 1, 2), STD, DST) + ) + transitions.append( + ZoneTransition(datetime(year, 11, 1, 2), DST, STD) + ) + + after = "" + + zf = self.construct_zone(transitions, after) + + # According to RFC 8536, local times after the last transition time + # with an empty TZ string are unspecified. We will go with "hold the + # last transition", but the most we should promise is "doesn't crash." + zi = self.klass.from_file(zf) + + cases = [ + (datetime(1995, 1, 1), STD), + (datetime(1996, 4, 1), DST), + (datetime(1996, 11, 2), STD), + (datetime(2001, 1, 1), STD), + ] + + for dt, offset in cases: + dt = dt.replace(tzinfo=zi) + with self.subTest(dt=dt): + self.assertEqual(dt.tzname(), offset.tzname) + self.assertEqual(dt.utcoffset(), offset.utcoffset) + self.assertEqual(dt.dst(), offset.dst) + + # Test that offsets return None when using a datetime.time + t = time(0, tzinfo=zi) + with self.subTest("Testing datetime.time"): + self.assertIs(t.tzname(), None) + self.assertIs(t.utcoffset(), None) + self.assertIs(t.dst(), None) + + def test_tz_before_only(self): + # From RFC 8536 Section 3.2: + # + # If there are no transitions, local time for all timestamps is + # specified by the TZ string in the footer if present and nonempty; + # otherwise, it is specified by time type 0. + + offsets = [ + ZoneOffset("STD", ZERO, ZERO), + ZoneOffset("DST", ONE_H, ONE_H), + ] + + for offset in offsets: + # Phantom transition to set time type 0. + transitions = [ + ZoneTransition(None, offset, offset), + ] + + after = "" + + zf = self.construct_zone(transitions, after) + zi = self.klass.from_file(zf) + + dts = [ + datetime(1900, 1, 1), + datetime(1970, 1, 1), + datetime(2000, 1, 1), + ] + + for dt in dts: + dt = dt.replace(tzinfo=zi) + with self.subTest(offset=offset, dt=dt): + self.assertEqual(dt.tzname(), offset.tzname) + self.assertEqual(dt.utcoffset(), offset.utcoffset) + self.assertEqual(dt.dst(), offset.dst) + + def test_empty_zone(self): + zf = self.construct_zone([], "") + + with self.assertRaises(ValueError): + self.klass.from_file(zf) + + def test_zone_very_large_timestamp(self): + """Test when a transition is in the far past or future. + + Particularly, this is a concern if something: + + 1. Attempts to call ``datetime.timestamp`` for a datetime outside + of ``[datetime.min, datetime.max]``. + 2. Attempts to construct a timedelta outside of + ``[timedelta.min, timedelta.max]``. + + This actually occurs "in the wild", as some time zones on Ubuntu (at + least as of 2020) have an initial transition added at ``-2**58``. + """ + + LMT = ZoneOffset("LMT", timedelta(seconds=-968)) + GMT = ZoneOffset("GMT", ZERO) + + transitions = [ + (-(1 << 62), LMT, LMT), + ZoneTransition(datetime(1912, 1, 1), LMT, GMT), + ((1 << 62), GMT, GMT), + ] + + after = "GMT0" + + zf = self.construct_zone(transitions, after) + zi = self.klass.from_file(zf, key="Africa/Abidjan") + + offset_cases = [ + (datetime.min, LMT), + (datetime.max, GMT), + (datetime(1911, 12, 31), LMT), + (datetime(1912, 1, 2), GMT), + ] + + for dt_naive, offset in offset_cases: + dt = dt_naive.replace(tzinfo=zi) + with self.subTest(name="offset", dt=dt, offset=offset): + self.assertEqual(dt.tzname(), offset.tzname) + self.assertEqual(dt.utcoffset(), offset.utcoffset) + self.assertEqual(dt.dst(), offset.dst) + + utc_cases = [ + (datetime.min, datetime.min + timedelta(seconds=968)), + (datetime(1898, 12, 31, 23, 43, 52), datetime(1899, 1, 1)), + ( + datetime(1911, 12, 31, 23, 59, 59, 999999), + datetime(1912, 1, 1, 0, 16, 7, 999999), + ), + (datetime(1912, 1, 1, 0, 16, 8), datetime(1912, 1, 1, 0, 16, 8)), + (datetime(1970, 1, 1), datetime(1970, 1, 1)), + (datetime.max, datetime.max), + ] + + for naive_dt, naive_dt_utc in utc_cases: + dt = naive_dt.replace(tzinfo=zi) + dt_utc = naive_dt_utc.replace(tzinfo=timezone.utc) + + self.assertEqual(dt_utc.astimezone(zi), dt) + self.assertEqual(dt, dt_utc) + + def test_fixed_offset_phantom_transition(self): + UTC = ZoneOffset("UTC", ZERO, ZERO) + + transitions = [ZoneTransition(datetime(1970, 1, 1), UTC, UTC)] + + after = "UTC0" + zf = self.construct_zone(transitions, after) + zi = self.klass.from_file(zf, key="UTC") + + dt = datetime(2020, 1, 1, tzinfo=zi) + with self.subTest("datetime.datetime"): + self.assertEqual(dt.tzname(), UTC.tzname) + self.assertEqual(dt.utcoffset(), UTC.utcoffset) + self.assertEqual(dt.dst(), UTC.dst) + + t = time(0, tzinfo=zi) + with self.subTest("datetime.time"): + self.assertEqual(t.tzname(), UTC.tzname) + self.assertEqual(t.utcoffset(), UTC.utcoffset) + self.assertEqual(t.dst(), UTC.dst) + + def construct_zone(self, transitions, after=None, version=3): + # These are not used for anything, so we're not going to include + # them for now. + isutc = [] + isstd = [] + leap_seconds = [] + + offset_lists = [[], []] + trans_times_lists = [[], []] + trans_idx_lists = [[], []] + + v1_range = (-(2 ** 31), 2 ** 31) + v2_range = (-(2 ** 63), 2 ** 63) + ranges = [v1_range, v2_range] + + def zt_as_tuple(zt): + # zt may be a tuple (timestamp, offset_before, offset_after) or + # a ZoneTransition object — this is to allow the timestamp to be + # values that are outside the valid range for datetimes but still + # valid 64-bit timestamps. + if isinstance(zt, tuple): + return zt + + if zt.transition: + trans_time = int(zt.transition_utc.timestamp()) + else: + trans_time = None + + return (trans_time, zt.offset_before, zt.offset_after) + + transitions = sorted(map(zt_as_tuple, transitions), key=lambda x: x[0]) + + for zt in transitions: + trans_time, offset_before, offset_after = zt + + for v, (dt_min, dt_max) in enumerate(ranges): + offsets = offset_lists[v] + trans_times = trans_times_lists[v] + trans_idx = trans_idx_lists[v] + + if trans_time is not None and not ( + dt_min <= trans_time <= dt_max + ): + continue + + if offset_before not in offsets: + offsets.append(offset_before) + + if offset_after not in offsets: + offsets.append(offset_after) + + if trans_time is not None: + trans_times.append(trans_time) + trans_idx.append(offsets.index(offset_after)) + + isutcnt = len(isutc) + isstdcnt = len(isstd) + leapcnt = len(leap_seconds) + + zonefile = io.BytesIO() + + time_types = ("l", "q") + for v in range(min((version, 2))): + offsets = offset_lists[v] + trans_times = trans_times_lists[v] + trans_idx = trans_idx_lists[v] + time_type = time_types[v] + + # Translate the offsets into something closer to the C values + abbrstr = bytearray() + ttinfos = [] + + for offset in offsets: + utcoff = int(offset.utcoffset.total_seconds()) + isdst = bool(offset.dst) + abbrind = len(abbrstr) + + ttinfos.append((utcoff, isdst, abbrind)) + abbrstr += offset.tzname.encode("ascii") + b"\x00" + abbrstr = bytes(abbrstr) + + typecnt = len(offsets) + timecnt = len(trans_times) + charcnt = len(abbrstr) + + # Write the header + zonefile.write(b"TZif") + zonefile.write(b"%d" % version) + zonefile.write(b" " * 15) + zonefile.write( + struct.pack( + ">6l", isutcnt, isstdcnt, leapcnt, timecnt, typecnt, charcnt + ) + ) + + # Now the transition data + zonefile.write(struct.pack(f">{timecnt}{time_type}", *trans_times)) + zonefile.write(struct.pack(f">{timecnt}B", *trans_idx)) + + for ttinfo in ttinfos: + zonefile.write(struct.pack(">lbb", *ttinfo)) + + zonefile.write(bytes(abbrstr)) + + # Now the metadata and leap seconds + zonefile.write(struct.pack(f"{isutcnt}b", *isutc)) + zonefile.write(struct.pack(f"{isstdcnt}b", *isstd)) + zonefile.write(struct.pack(f">{leapcnt}l", *leap_seconds)) + + # Finally we write the TZ string if we're writing a Version 2+ file + if v > 0: + zonefile.write(b"\x0A") + zonefile.write(after.encode("ascii")) + zonefile.write(b"\x0A") + + zonefile.seek(0) + return zonefile + + +class CWeirdZoneTest(WeirdZoneTest): + module = c_zoneinfo + + +class TZStrTest(ZoneInfoTestBase): + module = py_zoneinfo + + NORMAL = 0 + FOLD = 1 + GAP = 2 + + @classmethod + def setUpClass(cls): + super().setUpClass() + + cls._populate_test_cases() + cls.populate_tzstr_header() + + @classmethod + def populate_tzstr_header(cls): + out = bytearray() + # The TZif format always starts with a Version 1 file followed by + # the Version 2+ file. In this case, we have no transitions, just + # the tzstr in the footer, so up to the footer, the files are + # identical and we can just write the same file twice in a row. + for _ in range(2): + out += b"TZif" # Magic value + out += b"3" # Version + out += b" " * 15 # Reserved + + # We will not write any of the manual transition parts + out += struct.pack(">6l", 0, 0, 0, 0, 0, 0) + + cls._tzif_header = bytes(out) + + def zone_from_tzstr(self, tzstr): + """Creates a zoneinfo file following a POSIX rule.""" + zonefile = io.BytesIO(self._tzif_header) + zonefile.seek(0, 2) + + # Write the footer + zonefile.write(b"\x0A") + zonefile.write(tzstr.encode("ascii")) + zonefile.write(b"\x0A") + + zonefile.seek(0) + + return self.klass.from_file(zonefile, key=tzstr) + + def test_tzstr_localized(self): + for tzstr, cases in self.test_cases.items(): + with self.subTest(tzstr=tzstr): + zi = self.zone_from_tzstr(tzstr) + + for dt_naive, offset, _ in cases: + dt = dt_naive.replace(tzinfo=zi) + + with self.subTest(tzstr=tzstr, dt=dt, offset=offset): + self.assertEqual(dt.tzname(), offset.tzname) + self.assertEqual(dt.utcoffset(), offset.utcoffset) + self.assertEqual(dt.dst(), offset.dst) + + def test_tzstr_from_utc(self): + for tzstr, cases in self.test_cases.items(): + with self.subTest(tzstr=tzstr): + zi = self.zone_from_tzstr(tzstr) + + for dt_naive, offset, dt_type in cases: + if dt_type == self.GAP: + continue # Cannot create a gap from UTC + + dt_utc = (dt_naive - offset.utcoffset).replace( + tzinfo=timezone.utc + ) + + # Check that we can go UTC -> Our zone + dt_act = dt_utc.astimezone(zi) + dt_exp = dt_naive.replace(tzinfo=zi) + + self.assertEqual(dt_act, dt_exp) + + if dt_type == self.FOLD: + self.assertEqual(dt_act.fold, dt_naive.fold, dt_naive) + else: + self.assertEqual(dt_act.fold, 0) + + # Now check that we can go our zone -> UTC + dt_act = dt_exp.astimezone(timezone.utc) + + self.assertEqual(dt_act, dt_utc) + + def test_extreme_tzstr(self): + tzstrs = [ + # Extreme offset hour + "AAA24", + "AAA+24", + "AAA-24", + "AAA24BBB,J60/2,J300/2", + "AAA+24BBB,J60/2,J300/2", + "AAA-24BBB,J60/2,J300/2", + "AAA4BBB24,J60/2,J300/2", + "AAA4BBB+24,J60/2,J300/2", + "AAA4BBB-24,J60/2,J300/2", + # Extreme offset minutes + "AAA4:00BBB,J60/2,J300/2", + "AAA4:59BBB,J60/2,J300/2", + "AAA4BBB5:00,J60/2,J300/2", + "AAA4BBB5:59,J60/2,J300/2", + # Extreme offset seconds + "AAA4:00:00BBB,J60/2,J300/2", + "AAA4:00:59BBB,J60/2,J300/2", + "AAA4BBB5:00:00,J60/2,J300/2", + "AAA4BBB5:00:59,J60/2,J300/2", + # Extreme total offset + "AAA24:59:59BBB5,J60/2,J300/2", + "AAA-24:59:59BBB5,J60/2,J300/2", + "AAA4BBB24:59:59,J60/2,J300/2", + "AAA4BBB-24:59:59,J60/2,J300/2", + # Extreme months + "AAA4BBB,M12.1.1/2,M1.1.1/2", + "AAA4BBB,M1.1.1/2,M12.1.1/2", + # Extreme weeks + "AAA4BBB,M1.5.1/2,M1.1.1/2", + "AAA4BBB,M1.1.1/2,M1.5.1/2", + # Extreme weekday + "AAA4BBB,M1.1.6/2,M2.1.1/2", + "AAA4BBB,M1.1.1/2,M2.1.6/2", + # Extreme numeric offset + "AAA4BBB,0/2,20/2", + "AAA4BBB,0/2,0/14", + "AAA4BBB,20/2,365/2", + "AAA4BBB,365/2,365/14", + # Extreme julian offset + "AAA4BBB,J1/2,J20/2", + "AAA4BBB,J1/2,J1/14", + "AAA4BBB,J20/2,J365/2", + "AAA4BBB,J365/2,J365/14", + # Extreme transition hour + "AAA4BBB,J60/167,J300/2", + "AAA4BBB,J60/+167,J300/2", + "AAA4BBB,J60/-167,J300/2", + "AAA4BBB,J60/2,J300/167", + "AAA4BBB,J60/2,J300/+167", + "AAA4BBB,J60/2,J300/-167", + # Extreme transition minutes + "AAA4BBB,J60/2:00,J300/2", + "AAA4BBB,J60/2:59,J300/2", + "AAA4BBB,J60/2,J300/2:00", + "AAA4BBB,J60/2,J300/2:59", + # Extreme transition seconds + "AAA4BBB,J60/2:00:00,J300/2", + "AAA4BBB,J60/2:00:59,J300/2", + "AAA4BBB,J60/2,J300/2:00:00", + "AAA4BBB,J60/2,J300/2:00:59", + # Extreme total transition time + "AAA4BBB,J60/167:59:59,J300/2", + "AAA4BBB,J60/-167:59:59,J300/2", + "AAA4BBB,J60/2,J300/167:59:59", + "AAA4BBB,J60/2,J300/-167:59:59", + ] + + for tzstr in tzstrs: + with self.subTest(tzstr=tzstr): + self.zone_from_tzstr(tzstr) + + def test_invalid_tzstr(self): + invalid_tzstrs = [ + "PST8PDT", # DST but no transition specified + "+11", # Unquoted alphanumeric + "GMT,M3.2.0/2,M11.1.0/3", # Transition rule but no DST + "GMT0+11,M3.2.0/2,M11.1.0/3", # Unquoted alphanumeric in DST + "PST8PDT,M3.2.0/2", # Only one transition rule + # Invalid offset hours + "AAA168", + "AAA+168", + "AAA-168", + "AAA168BBB,J60/2,J300/2", + "AAA+168BBB,J60/2,J300/2", + "AAA-168BBB,J60/2,J300/2", + "AAA4BBB168,J60/2,J300/2", + "AAA4BBB+168,J60/2,J300/2", + "AAA4BBB-168,J60/2,J300/2", + # Invalid offset minutes + "AAA4:0BBB,J60/2,J300/2", + "AAA4:100BBB,J60/2,J300/2", + "AAA4BBB5:0,J60/2,J300/2", + "AAA4BBB5:100,J60/2,J300/2", + # Invalid offset seconds + "AAA4:00:0BBB,J60/2,J300/2", + "AAA4:00:100BBB,J60/2,J300/2", + "AAA4BBB5:00:0,J60/2,J300/2", + "AAA4BBB5:00:100,J60/2,J300/2", + # Completely invalid dates + "AAA4BBB,M1443339,M11.1.0/3", + "AAA4BBB,M3.2.0/2,0349309483959c", + "AAA4BBB,,J300/2", + "AAA4BBB,z,J300/2", + "AAA4BBB,J60/2,", + "AAA4BBB,J60/2,z", + # Invalid months + "AAA4BBB,M13.1.1/2,M1.1.1/2", + "AAA4BBB,M1.1.1/2,M13.1.1/2", + "AAA4BBB,M0.1.1/2,M1.1.1/2", + "AAA4BBB,M1.1.1/2,M0.1.1/2", + # Invalid weeks + "AAA4BBB,M1.6.1/2,M1.1.1/2", + "AAA4BBB,M1.1.1/2,M1.6.1/2", + # Invalid weekday + "AAA4BBB,M1.1.7/2,M2.1.1/2", + "AAA4BBB,M1.1.1/2,M2.1.7/2", + # Invalid numeric offset + "AAA4BBB,-1/2,20/2", + "AAA4BBB,1/2,-1/2", + "AAA4BBB,367,20/2", + "AAA4BBB,1/2,367/2", + # Invalid julian offset + "AAA4BBB,J0/2,J20/2", + "AAA4BBB,J20/2,J366/2", + # Invalid transition time + "AAA4BBB,J60/2/3,J300/2", + "AAA4BBB,J60/2,J300/2/3", + # Invalid transition hour + "AAA4BBB,J60/168,J300/2", + "AAA4BBB,J60/+168,J300/2", + "AAA4BBB,J60/-168,J300/2", + "AAA4BBB,J60/2,J300/168", + "AAA4BBB,J60/2,J300/+168", + "AAA4BBB,J60/2,J300/-168", + # Invalid transition minutes + "AAA4BBB,J60/2:0,J300/2", + "AAA4BBB,J60/2:100,J300/2", + "AAA4BBB,J60/2,J300/2:0", + "AAA4BBB,J60/2,J300/2:100", + # Invalid transition seconds + "AAA4BBB,J60/2:00:0,J300/2", + "AAA4BBB,J60/2:00:100,J300/2", + "AAA4BBB,J60/2,J300/2:00:0", + "AAA4BBB,J60/2,J300/2:00:100", + ] + + for invalid_tzstr in invalid_tzstrs: + with self.subTest(tzstr=invalid_tzstr): + # Not necessarily a guaranteed property, but we should show + # the problematic TZ string if that's the cause of failure. + tzstr_regex = re.escape(invalid_tzstr) + with self.assertRaisesRegex(ValueError, tzstr_regex): + self.zone_from_tzstr(invalid_tzstr) + + @classmethod + def _populate_test_cases(cls): + # This method uses a somewhat unusual style in that it populates the + # test cases for each tzstr by using a decorator to automatically call + # a function that mutates the current dictionary of test cases. + # + # The population of the test cases is done in individual functions to + # give each set of test cases its own namespace in which to define + # its offsets (this way we don't have to worry about variable reuse + # causing problems if someone makes a typo). + # + # The decorator for calling is used to make it more obvious that each + # function is actually called (if it's not decorated, it's not called). + def call(f): + """Decorator to call the addition methods. + + This will call a function which adds at least one new entry into + the `cases` dictionary. The decorator will also assert that + something was added to the dictionary. + """ + prev_len = len(cases) + f() + assert len(cases) > prev_len, "Function did not add a test case!" + + NORMAL = cls.NORMAL + FOLD = cls.FOLD + GAP = cls.GAP + + cases = {} + + @call + def _add(): + # Transition to EDT on the 2nd Sunday in March at 4 AM, and + # transition back on the first Sunday in November at 3AM + tzstr = "EST5EDT,M3.2.0/4:00,M11.1.0/3:00" + + EST = ZoneOffset("EST", timedelta(hours=-5), ZERO) + EDT = ZoneOffset("EDT", timedelta(hours=-4), ONE_H) + + cases[tzstr] = ( + (datetime(2019, 3, 9), EST, NORMAL), + (datetime(2019, 3, 10, 3, 59), EST, NORMAL), + (datetime(2019, 3, 10, 4, 0, fold=0), EST, GAP), + (datetime(2019, 3, 10, 4, 0, fold=1), EDT, GAP), + (datetime(2019, 3, 10, 4, 1, fold=0), EST, GAP), + (datetime(2019, 3, 10, 4, 1, fold=1), EDT, GAP), + (datetime(2019, 11, 2), EDT, NORMAL), + (datetime(2019, 11, 3, 1, 59, fold=1), EDT, NORMAL), + (datetime(2019, 11, 3, 2, 0, fold=0), EDT, FOLD), + (datetime(2019, 11, 3, 2, 0, fold=1), EST, FOLD), + (datetime(2020, 3, 8, 3, 59), EST, NORMAL), + (datetime(2020, 3, 8, 4, 0, fold=0), EST, GAP), + (datetime(2020, 3, 8, 4, 0, fold=1), EDT, GAP), + (datetime(2020, 11, 1, 1, 59, fold=1), EDT, NORMAL), + (datetime(2020, 11, 1, 2, 0, fold=0), EDT, FOLD), + (datetime(2020, 11, 1, 2, 0, fold=1), EST, FOLD), + ) + + @call + def _add(): + # Transition to BST happens on the last Sunday in March at 1 AM GMT + # and the transition back happens the last Sunday in October at 2AM BST + tzstr = "GMT0BST-1,M3.5.0/1:00,M10.5.0/2:00" + + GMT = ZoneOffset("GMT", ZERO, ZERO) + BST = ZoneOffset("BST", ONE_H, ONE_H) + + cases[tzstr] = ( + (datetime(2019, 3, 30), GMT, NORMAL), + (datetime(2019, 3, 31, 0, 59), GMT, NORMAL), + (datetime(2019, 3, 31, 2, 0), BST, NORMAL), + (datetime(2019, 10, 26), BST, NORMAL), + (datetime(2019, 10, 27, 0, 59, fold=1), BST, NORMAL), + (datetime(2019, 10, 27, 1, 0, fold=0), BST, GAP), + (datetime(2019, 10, 27, 2, 0, fold=1), GMT, GAP), + (datetime(2020, 3, 29, 0, 59), GMT, NORMAL), + (datetime(2020, 3, 29, 2, 0), BST, NORMAL), + (datetime(2020, 10, 25, 0, 59, fold=1), BST, NORMAL), + (datetime(2020, 10, 25, 1, 0, fold=0), BST, FOLD), + (datetime(2020, 10, 25, 2, 0, fold=1), GMT, NORMAL), + ) + + @call + def _add(): + # Austrialian time zone - DST start is chronologically first + tzstr = "AEST-10AEDT,M10.1.0/2,M4.1.0/3" + + AEST = ZoneOffset("AEST", timedelta(hours=10), ZERO) + AEDT = ZoneOffset("AEDT", timedelta(hours=11), ONE_H) + + cases[tzstr] = ( + (datetime(2019, 4, 6), AEDT, NORMAL), + (datetime(2019, 4, 7, 1, 59), AEDT, NORMAL), + (datetime(2019, 4, 7, 1, 59, fold=1), AEDT, NORMAL), + (datetime(2019, 4, 7, 2, 0, fold=0), AEDT, FOLD), + (datetime(2019, 4, 7, 2, 1, fold=0), AEDT, FOLD), + (datetime(2019, 4, 7, 2, 0, fold=1), AEST, FOLD), + (datetime(2019, 4, 7, 2, 1, fold=1), AEST, FOLD), + (datetime(2019, 4, 7, 3, 0, fold=0), AEST, NORMAL), + (datetime(2019, 4, 7, 3, 0, fold=1), AEST, NORMAL), + (datetime(2019, 10, 5, 0), AEST, NORMAL), + (datetime(2019, 10, 6, 1, 59), AEST, NORMAL), + (datetime(2019, 10, 6, 2, 0, fold=0), AEST, GAP), + (datetime(2019, 10, 6, 2, 0, fold=1), AEDT, GAP), + (datetime(2019, 10, 6, 3, 0), AEDT, NORMAL), + ) + + @call + def _add(): + # Irish time zone - negative DST + tzstr = "IST-1GMT0,M10.5.0,M3.5.0/1" + + GMT = ZoneOffset("GMT", ZERO, -ONE_H) + IST = ZoneOffset("IST", ONE_H, ZERO) + + cases[tzstr] = ( + (datetime(2019, 3, 30), GMT, NORMAL), + (datetime(2019, 3, 31, 0, 59), GMT, NORMAL), + (datetime(2019, 3, 31, 2, 0), IST, NORMAL), + (datetime(2019, 10, 26), IST, NORMAL), + (datetime(2019, 10, 27, 0, 59, fold=1), IST, NORMAL), + (datetime(2019, 10, 27, 1, 0, fold=0), IST, FOLD), + (datetime(2019, 10, 27, 1, 0, fold=1), GMT, FOLD), + (datetime(2019, 10, 27, 2, 0, fold=1), GMT, NORMAL), + (datetime(2020, 3, 29, 0, 59), GMT, NORMAL), + (datetime(2020, 3, 29, 2, 0), IST, NORMAL), + (datetime(2020, 10, 25, 0, 59, fold=1), IST, NORMAL), + (datetime(2020, 10, 25, 1, 0, fold=0), IST, FOLD), + (datetime(2020, 10, 25, 2, 0, fold=1), GMT, NORMAL), + ) + + @call + def _add(): + # Pacific/Kosrae: Fixed offset zone with a quoted numerical tzname + tzstr = "<+11>-11" + + cases[tzstr] = ( + ( + datetime(2020, 1, 1), + ZoneOffset("+11", timedelta(hours=11)), + NORMAL, + ), + ) + + @call + def _add(): + # Quoted STD and DST, transitions at 24:00 + tzstr = "<-04>4<-03>,M9.1.6/24,M4.1.6/24" + + M04 = ZoneOffset("-04", timedelta(hours=-4)) + M03 = ZoneOffset("-03", timedelta(hours=-3), ONE_H) + + cases[tzstr] = ( + (datetime(2020, 5, 1), M04, NORMAL), + (datetime(2020, 11, 1), M03, NORMAL), + ) + + @call + def _add(): + # Permanent daylight saving time is modeled with transitions at 0/0 + # and J365/25, as mentioned in RFC 8536 Section 3.3.1 + tzstr = "EST5EDT,0/0,J365/25" + + EDT = ZoneOffset("EDT", timedelta(hours=-4), ONE_H) + + cases[tzstr] = ( + (datetime(2019, 1, 1), EDT, NORMAL), + (datetime(2019, 6, 1), EDT, NORMAL), + (datetime(2019, 12, 31, 23, 59, 59, 999999), EDT, NORMAL), + (datetime(2020, 1, 1), EDT, NORMAL), + (datetime(2020, 3, 1), EDT, NORMAL), + (datetime(2020, 6, 1), EDT, NORMAL), + (datetime(2020, 12, 31, 23, 59, 59, 999999), EDT, NORMAL), + (datetime(2400, 1, 1), EDT, NORMAL), + (datetime(2400, 3, 1), EDT, NORMAL), + (datetime(2400, 12, 31, 23, 59, 59, 999999), EDT, NORMAL), + ) + + @call + def _add(): + # Transitions on March 1st and November 1st of each year + tzstr = "AAA3BBB,J60/12,J305/12" + + AAA = ZoneOffset("AAA", timedelta(hours=-3)) + BBB = ZoneOffset("BBB", timedelta(hours=-2), ONE_H) + + cases[tzstr] = ( + (datetime(2019, 1, 1), AAA, NORMAL), + (datetime(2019, 2, 28), AAA, NORMAL), + (datetime(2019, 3, 1, 11, 59), AAA, NORMAL), + (datetime(2019, 3, 1, 12, fold=0), AAA, GAP), + (datetime(2019, 3, 1, 12, fold=1), BBB, GAP), + (datetime(2019, 3, 1, 13), BBB, NORMAL), + (datetime(2019, 11, 1, 10, 59), BBB, NORMAL), + (datetime(2019, 11, 1, 11, fold=0), BBB, FOLD), + (datetime(2019, 11, 1, 11, fold=1), AAA, FOLD), + (datetime(2019, 11, 1, 12), AAA, NORMAL), + (datetime(2019, 12, 31, 23, 59, 59, 999999), AAA, NORMAL), + (datetime(2020, 1, 1), AAA, NORMAL), + (datetime(2020, 2, 29), AAA, NORMAL), + (datetime(2020, 3, 1, 11, 59), AAA, NORMAL), + (datetime(2020, 3, 1, 12, fold=0), AAA, GAP), + (datetime(2020, 3, 1, 12, fold=1), BBB, GAP), + (datetime(2020, 3, 1, 13), BBB, NORMAL), + (datetime(2020, 11, 1, 10, 59), BBB, NORMAL), + (datetime(2020, 11, 1, 11, fold=0), BBB, FOLD), + (datetime(2020, 11, 1, 11, fold=1), AAA, FOLD), + (datetime(2020, 11, 1, 12), AAA, NORMAL), + (datetime(2020, 12, 31, 23, 59, 59, 999999), AAA, NORMAL), + ) + + @call + def _add(): + # Taken from America/Godthab, this rule has a transition on the + # Saturday before the last Sunday of March and October, at 22:00 + # and 23:00, respectively. This is encoded with negative start + # and end transition times. + tzstr = "<-03>3<-02>,M3.5.0/-2,M10.5.0/-1" + + N03 = ZoneOffset("-03", timedelta(hours=-3)) + N02 = ZoneOffset("-02", timedelta(hours=-2), ONE_H) + + cases[tzstr] = ( + (datetime(2020, 3, 27), N03, NORMAL), + (datetime(2020, 3, 28, 21, 59, 59), N03, NORMAL), + (datetime(2020, 3, 28, 22, fold=0), N03, GAP), + (datetime(2020, 3, 28, 22, fold=1), N02, GAP), + (datetime(2020, 3, 28, 23), N02, NORMAL), + (datetime(2020, 10, 24, 21), N02, NORMAL), + (datetime(2020, 10, 24, 22, fold=0), N02, FOLD), + (datetime(2020, 10, 24, 22, fold=1), N03, FOLD), + (datetime(2020, 10, 24, 23), N03, NORMAL), + ) + + @call + def _add(): + # Transition times with minutes and seconds + tzstr = "AAA3BBB,M3.2.0/01:30,M11.1.0/02:15:45" + + AAA = ZoneOffset("AAA", timedelta(hours=-3)) + BBB = ZoneOffset("BBB", timedelta(hours=-2), ONE_H) + + cases[tzstr] = ( + (datetime(2012, 3, 11, 1, 0), AAA, NORMAL), + (datetime(2012, 3, 11, 1, 30, fold=0), AAA, GAP), + (datetime(2012, 3, 11, 1, 30, fold=1), BBB, GAP), + (datetime(2012, 3, 11, 2, 30), BBB, NORMAL), + (datetime(2012, 11, 4, 1, 15, 44, 999999), BBB, NORMAL), + (datetime(2012, 11, 4, 1, 15, 45, fold=0), BBB, FOLD), + (datetime(2012, 11, 4, 1, 15, 45, fold=1), AAA, FOLD), + (datetime(2012, 11, 4, 2, 15, 45), AAA, NORMAL), + ) + + cls.test_cases = cases + + +class CTZStrTest(TZStrTest): + module = c_zoneinfo + + +class ZoneInfoCacheTest(TzPathUserMixin, ZoneInfoTestBase): + module = py_zoneinfo + + def setUp(self): + self.klass.clear_cache() + super().setUp() + + @property + def zoneinfo_data(self): + return ZONEINFO_DATA + + @property + def tzpath(self): + return [self.zoneinfo_data.tzpath] + + def test_ephemeral_zones(self): + self.assertIs( + self.klass("America/Los_Angeles"), self.klass("America/Los_Angeles") + ) + + def test_strong_refs(self): + tz0 = self.klass("Australia/Sydney") + tz1 = self.klass("Australia/Sydney") + + self.assertIs(tz0, tz1) + + def test_no_cache(self): + + tz0 = self.klass("Europe/Lisbon") + tz1 = self.klass.no_cache("Europe/Lisbon") + + self.assertIsNot(tz0, tz1) + + def test_cache_reset_tzpath(self): + """Test that the cache persists when tzpath has been changed. + + The PEP specifies that as long as a reference exists to one zone + with a given key, the primary constructor must continue to return + the same object. + """ + zi0 = self.klass("America/Los_Angeles") + with self.tzpath_context([]): + zi1 = self.klass("America/Los_Angeles") + + self.assertIs(zi0, zi1) + + def test_clear_cache_explicit_none(self): + la0 = self.klass("America/Los_Angeles") + self.klass.clear_cache(only_keys=None) + la1 = self.klass("America/Los_Angeles") + + self.assertIsNot(la0, la1) + + def test_clear_cache_one_key(self): + """Tests that you can clear a single key from the cache.""" + la0 = self.klass("America/Los_Angeles") + dub0 = self.klass("Europe/Dublin") + + self.klass.clear_cache(only_keys=["America/Los_Angeles"]) + + la1 = self.klass("America/Los_Angeles") + dub1 = self.klass("Europe/Dublin") + + self.assertIsNot(la0, la1) + self.assertIs(dub0, dub1) + + def test_clear_cache_two_keys(self): + la0 = self.klass("America/Los_Angeles") + dub0 = self.klass("Europe/Dublin") + tok0 = self.klass("Asia/Tokyo") + + self.klass.clear_cache( + only_keys=["America/Los_Angeles", "Europe/Dublin"] + ) + + la1 = self.klass("America/Los_Angeles") + dub1 = self.klass("Europe/Dublin") + tok1 = self.klass("Asia/Tokyo") + + self.assertIsNot(la0, la1) + self.assertIsNot(dub0, dub1) + self.assertIs(tok0, tok1) + + def test_clear_cache_refleak(self): + class Stringy(str): + allow_comparisons = True + def __eq__(self, other): + if not self.allow_comparisons: + raise CustomError + return super().__eq__(other) + __hash__ = str.__hash__ + + key = Stringy("America/Los_Angeles") + self.klass(key) + key.allow_comparisons = False + try: + # Note: This is try/except rather than assertRaises because + # there is no guarantee that the key is even still in the cache, + # or that the key for the cache is the original `key` object. + self.klass.clear_cache(only_keys="America/Los_Angeles") + except CustomError: + pass + + def test_weak_cache_descriptor_use_after_free(self): + class BombDescriptor: + def __get__(self, obj, owner): + return {} + + class EvilZoneInfo(self.klass): + pass + + # Must be set after the class creation. + EvilZoneInfo._weak_cache = BombDescriptor() + + key = "America/Los_Angeles" + zone1 = EvilZoneInfo(key) + self.assertEqual(str(zone1), key) + + EvilZoneInfo.clear_cache() + zone2 = EvilZoneInfo(key) + self.assertEqual(str(zone2), key) + self.assertIsNot(zone2, zone1) + + +class CZoneInfoCacheTest(ZoneInfoCacheTest): + module = c_zoneinfo + + +class ZoneInfoPickleTest(TzPathUserMixin, ZoneInfoTestBase): + module = py_zoneinfo + + def setUp(self): + self.klass.clear_cache() + + with contextlib.ExitStack() as stack: + stack.enter_context(test_support.set_zoneinfo_module(self.module)) + self.addCleanup(stack.pop_all().close) + + super().setUp() + + @property + def zoneinfo_data(self): + return ZONEINFO_DATA + + @property + def tzpath(self): + return [self.zoneinfo_data.tzpath] + + def test_cache_hit(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + zi_in = self.klass("Europe/Dublin") + pkl = pickle.dumps(zi_in, protocol=proto) + zi_rt = pickle.loads(pkl) + + with self.subTest(test="Is non-pickled ZoneInfo"): + self.assertIs(zi_in, zi_rt) + + zi_rt2 = pickle.loads(pkl) + with self.subTest(test="Is unpickled ZoneInfo"): + self.assertIs(zi_rt, zi_rt2) + + def test_cache_miss(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + zi_in = self.klass("Europe/Dublin") + pkl = pickle.dumps(zi_in, protocol=proto) + + del zi_in + self.klass.clear_cache() # Induce a cache miss + zi_rt = pickle.loads(pkl) + zi_rt2 = pickle.loads(pkl) + + self.assertIs(zi_rt, zi_rt2) + + def test_no_cache(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + zi_no_cache = self.klass.no_cache("Europe/Dublin") + + pkl = pickle.dumps(zi_no_cache, protocol=proto) + zi_rt = pickle.loads(pkl) + + with self.subTest(test="Not the pickled object"): + self.assertIsNot(zi_rt, zi_no_cache) + + zi_rt2 = pickle.loads(pkl) + with self.subTest(test="Not a second unpickled object"): + self.assertIsNot(zi_rt, zi_rt2) + + zi_cache = self.klass("Europe/Dublin") + with self.subTest(test="Not a cached object"): + self.assertIsNot(zi_rt, zi_cache) + + def test_from_file(self): + key = "Europe/Dublin" + with open(self.zoneinfo_data.path_from_key(key), "rb") as f: + zi_nokey = self.klass.from_file(f) + + f.seek(0) + zi_key = self.klass.from_file(f, key=key) + + test_cases = [ + (zi_key, "ZoneInfo with key"), + (zi_nokey, "ZoneInfo without key"), + ] + + for zi, test_name in test_cases: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(test_name=test_name, proto=proto): + with self.assertRaises(pickle.PicklingError): + pickle.dumps(zi, protocol=proto) + + def test_pickle_after_from_file(self): + # This may be a bit of paranoia, but this test is to ensure that no + # global state is maintained in order to handle the pickle cache and + # from_file behavior, and that it is possible to interweave the + # constructors of each of these and pickling/unpickling without issues. + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + key = "Europe/Dublin" + zi = self.klass(key) + + pkl_0 = pickle.dumps(zi, protocol=proto) + zi_rt_0 = pickle.loads(pkl_0) + self.assertIs(zi, zi_rt_0) + + with open(self.zoneinfo_data.path_from_key(key), "rb") as f: + zi_ff = self.klass.from_file(f, key=key) + + pkl_1 = pickle.dumps(zi, protocol=proto) + zi_rt_1 = pickle.loads(pkl_1) + self.assertIs(zi, zi_rt_1) + + with self.assertRaises(pickle.PicklingError): + pickle.dumps(zi_ff, protocol=proto) + + pkl_2 = pickle.dumps(zi, protocol=proto) + zi_rt_2 = pickle.loads(pkl_2) + self.assertIs(zi, zi_rt_2) + + +class CZoneInfoPickleTest(ZoneInfoPickleTest): + module = c_zoneinfo + + +class CallingConventionTest(ZoneInfoTestBase): + """Tests for functions with restricted calling conventions.""" + + module = py_zoneinfo + + @property + def zoneinfo_data(self): + return ZONEINFO_DATA + + def test_from_file(self): + with open(self.zoneinfo_data.path_from_key("UTC"), "rb") as f: + with self.assertRaises(TypeError): + self.klass.from_file(fobj=f) + + def test_clear_cache(self): + with self.assertRaises(TypeError): + self.klass.clear_cache(["UTC"]) + + +class CCallingConventionTest(CallingConventionTest): + module = c_zoneinfo + + +class TzPathTest(TzPathUserMixin, ZoneInfoTestBase): + module = py_zoneinfo + + @staticmethod + @contextlib.contextmanager + def python_tzpath_context(value): + with EnvironmentVarGuard() as env: + env["PYTHONTZPATH"] = value + yield + + def test_env_variable(self): + """Tests that the environment variable works with reset_tzpath.""" + new_paths = [ + ("", []), + (f"{DRIVE}/etc/zoneinfo", [f"{DRIVE}/etc/zoneinfo"]), + (f"{DRIVE}/a/b/c{os.pathsep}{DRIVE}/d/e/f", [f"{DRIVE}/a/b/c", f"{DRIVE}/d/e/f"]), + ] + + for new_path_var, expected_result in new_paths: + with self.python_tzpath_context(new_path_var): + with self.subTest(tzpath=new_path_var): + self.module.reset_tzpath() + tzpath = self.module.TZPATH + self.assertSequenceEqual(tzpath, expected_result) + + def test_env_variable_relative_paths(self): + test_cases = [ + [("path/to/somewhere",), ()], + [ + (f"{DRIVE}/usr/share/zoneinfo", "path/to/somewhere",), + (f"{DRIVE}/usr/share/zoneinfo",), + ], + [("../relative/path",), ()], + [ + (f"{DRIVE}/usr/share/zoneinfo", "../relative/path",), + (f"{DRIVE}/usr/share/zoneinfo",), + ], + [("path/to/somewhere", "../relative/path",), ()], + [ + ( + f"{DRIVE}/usr/share/zoneinfo", + "path/to/somewhere", + "../relative/path", + ), + (f"{DRIVE}/usr/share/zoneinfo",), + ], + ] + + for input_paths, expected_paths in test_cases: + path_var = os.pathsep.join(input_paths) + with self.python_tzpath_context(path_var): + with self.subTest("warning", path_var=path_var): + # Note: Per PEP 615 the warning is implementation-defined + # behavior, other implementations need not warn. + with self.assertWarns(self.module.InvalidTZPathWarning) as w: + self.module.reset_tzpath() + self.assertEqual(w.warnings[0].filename, __file__) + + tzpath = self.module.TZPATH + with self.subTest("filtered", path_var=path_var): + self.assertSequenceEqual(tzpath, expected_paths) + + @unittest.expectedFailure # TODO: RUSTPYTHON; + /home/runner/work/RustPython/RustPython/crates/pylib/Lib/test/test_zoneinfo/test_zoneinfo.py + def test_env_variable_relative_paths_warning_location(self): + path_var = "path/to/somewhere" + + with self.python_tzpath_context(path_var): + with CleanImport("zoneinfo", "zoneinfo._tzpath"): + with self.assertWarns(RuntimeWarning) as w: + import zoneinfo + InvalidTZPathWarning = zoneinfo.InvalidTZPathWarning + self.assertIsInstance(w.warnings[0].message, InvalidTZPathWarning) + # It should represent the current file: + self.assertEqual(w.warnings[0].filename, __file__) + + def test_reset_tzpath_kwarg(self): + self.module.reset_tzpath(to=[f"{DRIVE}/a/b/c"]) + + self.assertSequenceEqual(self.module.TZPATH, (f"{DRIVE}/a/b/c",)) + + def test_reset_tzpath_relative_paths(self): + bad_values = [ + ("path/to/somewhere",), + ("/usr/share/zoneinfo", "path/to/somewhere",), + ("../relative/path",), + ("/usr/share/zoneinfo", "../relative/path",), + ("path/to/somewhere", "../relative/path",), + ("/usr/share/zoneinfo", "path/to/somewhere", "../relative/path",), + (FakePath("path/to/somewhere"),) + ] + for input_paths in bad_values: + with self.subTest(input_paths=input_paths): + with self.assertRaises(ValueError): + self.module.reset_tzpath(to=input_paths) + + def test_tzpath_type_error(self): + bad_values = [ + "/etc/zoneinfo:/usr/share/zoneinfo", + b"/etc/zoneinfo:/usr/share/zoneinfo", + 0, + (b"/bytes/path", "/valid/path"), + (FakePath(b"/bytes/path"),), + (0,), + ] + + for bad_value in bad_values: + with self.subTest(value=bad_value): + with self.assertRaises(TypeError): + self.module.reset_tzpath(bad_value) + + def test_tzpath_attribute(self): + tzpath_0 = [f"{DRIVE}/one", f"{DRIVE}/two"] + tzpath_1 = [f"{DRIVE}/three"] + tzpath_pathlike = (FakePath(f"{DRIVE}/usr/share/zoneinfo"),) + + with self.tzpath_context(tzpath_0): + query_0 = self.module.TZPATH + + with self.tzpath_context(tzpath_1): + query_1 = self.module.TZPATH + + with self.tzpath_context(tzpath_pathlike): + query_pathlike = self.module.TZPATH + + self.assertSequenceEqual(tzpath_0, query_0) + self.assertSequenceEqual(tzpath_1, query_1) + self.assertSequenceEqual(tuple([os.fspath(p) for p in tzpath_pathlike]), query_pathlike) + + +class CTzPathTest(TzPathTest): + module = c_zoneinfo + + +class TestModule(ZoneInfoTestBase): + module = py_zoneinfo + + @property + def zoneinfo_data(self): + return ZONEINFO_DATA + + @cached_property + def _UTC_bytes(self): + zone_file = self.zoneinfo_data.path_from_key("UTC") + with open(zone_file, "rb") as f: + return f.read() + + def touch_zone(self, key, tz_root): + """Creates a valid TZif file at key under the zoneinfo root tz_root. + + tz_root must exist, but all folders below that will be created. + """ + if not os.path.exists(tz_root): + raise FileNotFoundError(f"{tz_root} does not exist.") + + root_dir, *tail = key.rsplit("/", 1) + if tail: # If there's no tail, then the first component isn't a dir + os.makedirs(os.path.join(tz_root, root_dir), exist_ok=True) + + zonefile_path = os.path.join(tz_root, key) + with open(zonefile_path, "wb") as f: + f.write(self._UTC_bytes) + + def test_getattr_error(self): + with self.assertRaises(AttributeError): + self.module.NOATTRIBUTE + + @unittest.expectedFailure # TODO: RUSTPYTHON; dir(self.module) should at least contain everything in __all__. + def test_dir_contains_all(self): + """dir(self.module) should at least contain everything in __all__.""" + module_all_set = set(self.module.__all__) + module_dir_set = set(dir(self.module)) + + difference = module_all_set - module_dir_set + + self.assertFalse(difference) + + def test_dir_unique(self): + """Test that there are no duplicates in dir(self.module)""" + module_dir = dir(self.module) + module_unique = set(module_dir) + + self.assertCountEqual(module_dir, module_unique) + + def test_available_timezones(self): + with self.tzpath_context([self.zoneinfo_data.tzpath]): + self.assertTrue(self.zoneinfo_data.keys) # Sanity check + + available_keys = self.module.available_timezones() + zoneinfo_keys = set(self.zoneinfo_data.keys) + + # If tzdata is not present, zoneinfo_keys == available_keys, + # otherwise it should be a subset. + union = zoneinfo_keys & available_keys + self.assertEqual(zoneinfo_keys, union) + + def test_available_timezones_weirdzone(self): + with tempfile.TemporaryDirectory() as td: + # Make a fictional zone at "Mars/Olympus_Mons" + self.touch_zone("Mars/Olympus_Mons", td) + + with self.tzpath_context([td]): + available_keys = self.module.available_timezones() + self.assertIn("Mars/Olympus_Mons", available_keys) + + def test_folder_exclusions(self): + expected = { + "America/Los_Angeles", + "America/Santiago", + "America/Indiana/Indianapolis", + "UTC", + "Europe/Paris", + "Europe/London", + "Asia/Tokyo", + "Australia/Sydney", + } + + base_tree = list(expected) + posix_tree = [f"posix/{x}" for x in base_tree] + right_tree = [f"right/{x}" for x in base_tree] + + cases = [ + ("base_tree", base_tree), + ("base_and_posix", base_tree + posix_tree), + ("base_and_right", base_tree + right_tree), + ("all_trees", base_tree + right_tree + posix_tree), + ] + + with tempfile.TemporaryDirectory() as td: + for case_name, tree in cases: + tz_root = os.path.join(td, case_name) + os.mkdir(tz_root) + + for key in tree: + self.touch_zone(key, tz_root) + + with self.tzpath_context([tz_root]): + with self.subTest(case_name): + actual = self.module.available_timezones() + self.assertEqual(actual, expected) + + def test_exclude_posixrules(self): + expected = { + "America/New_York", + "Europe/London", + } + + tree = list(expected) + ["posixrules"] + + with tempfile.TemporaryDirectory() as td: + for key in tree: + self.touch_zone(key, td) + + with self.tzpath_context([td]): + actual = self.module.available_timezones() + self.assertEqual(actual, expected) + + +class CTestModule(TestModule): + module = c_zoneinfo + + +class MiscTests(unittest.TestCase): + def test_pydatetime(self): + # Test that zoneinfo works if the C implementation of datetime + # is not available and the Python implementation of datetime is used. + # The Python implementation of zoneinfo should be used in thet case. + # + # Run the test in a subprocess, as importing _zoneinfo with + # _datettime disabled causes crash in the previously imported + # _zoneinfo. + assert_python_ok('-c', '''if 1: + import sys + sys.modules['_datetime'] = None + import datetime + import zoneinfo + tzinfo = zoneinfo.ZoneInfo('Europe/London') + datetime.datetime(2025, 10, 26, 2, 0, tzinfo=tzinfo) + ''', + PYTHONTZPATH=str(ZONEINFO_DATA.tzpath)) + + +class ExtensionBuiltTest(unittest.TestCase): + """Smoke test to ensure that the C and Python extensions are both tested. + + Because the intention is for the Python and C versions of ZoneInfo to + behave identically, these tests necessarily rely on implementation details, + so the tests may need to be adjusted if the implementations change. Do not + rely on these tests as an indication of stable properties of these classes. + """ + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: type object 'ZoneInfo' has unexpected attribute '_weak_cache' + def test_cache_location(self): + # The pure Python version stores caches on attributes, but the C + # extension stores them in C globals (at least for now) + self.assertNotHasAttr(c_zoneinfo.ZoneInfo, "_weak_cache") + self.assertHasAttr(py_zoneinfo.ZoneInfo, "_weak_cache") + + def test_gc_tracked(self): + import gc + + self.assertTrue(gc.is_tracked(py_zoneinfo.ZoneInfo)) + self.assertTrue(gc.is_tracked(c_zoneinfo.ZoneInfo)) + + +@dataclasses.dataclass(frozen=True) +class ZoneOffset: + tzname: str + utcoffset: timedelta + dst: timedelta = ZERO + + +@dataclasses.dataclass(frozen=True) +class ZoneTransition: + transition: datetime + offset_before: ZoneOffset + offset_after: ZoneOffset + + @property + def transition_utc(self): + return (self.transition - self.offset_before.utcoffset).replace( + tzinfo=timezone.utc + ) + + @property + def fold(self): + """Whether this introduces a fold""" + return self.offset_before.utcoffset > self.offset_after.utcoffset + + @property + def gap(self): + """Whether this introduces a gap""" + return self.offset_before.utcoffset < self.offset_after.utcoffset + + @property + def delta(self): + return self.offset_after.utcoffset - self.offset_before.utcoffset + + @property + def anomaly_start(self): + if self.fold: + return self.transition + self.delta + else: + return self.transition + + @property + def anomaly_end(self): + if not self.fold: + return self.transition + self.delta + else: + return self.transition + + +class ZoneInfoData: + def __init__(self, source_json, tzpath, v1=False): + self.tzpath = pathlib.Path(tzpath) + self.keys = [] + self.v1 = v1 + self._populate_tzpath(source_json) + + def path_from_key(self, key): + return self.tzpath / key + + def _populate_tzpath(self, source_json): + with open(source_json, "rb") as f: + zoneinfo_dict = json.load(f) + + zoneinfo_data = zoneinfo_dict["data"] + + for key, value in zoneinfo_data.items(): + self.keys.append(key) + raw_data = self._decode_text(value) + + if self.v1: + data = self._convert_to_v1(raw_data) + else: + data = raw_data + + destination = self.path_from_key(key) + destination.parent.mkdir(exist_ok=True, parents=True) + with open(destination, "wb") as f: + f.write(data) + + def _decode_text(self, contents): + raw_data = b"".join(map(str.encode, contents)) + decoded = base64.b85decode(raw_data) + + return lzma.decompress(decoded) + + def _convert_to_v1(self, contents): + assert contents[0:4] == b"TZif", "Invalid TZif data found!" + version = int(contents[4:5]) + + header_start = 4 + 16 + header_end = header_start + 24 # 6l == 24 bytes + assert version >= 2, "Version 1 file found: no conversion necessary" + isutcnt, isstdcnt, leapcnt, timecnt, typecnt, charcnt = struct.unpack( + ">6l", contents[header_start:header_end] + ) + + file_size = ( + timecnt * 5 + + typecnt * 6 + + charcnt + + leapcnt * 8 + + isstdcnt + + isutcnt + ) + file_size += header_end + out = b"TZif" + b"\x00" + contents[5:file_size] + + assert ( + contents[file_size : (file_size + 4)] == b"TZif" + ), "Version 2 file not truncated at Version 2 header" + + return out + + +class ZoneDumpData: + @classmethod + def transition_keys(cls): + return cls._get_zonedump().keys() + + @classmethod + def load_transition_examples(cls, key): + return cls._get_zonedump()[key] + + @classmethod + def fixed_offset_zones(cls): + if not cls._FIXED_OFFSET_ZONES: + cls._populate_fixed_offsets() + + return cls._FIXED_OFFSET_ZONES.items() + + @classmethod + def _get_zonedump(cls): + if not cls._ZONEDUMP_DATA: + cls._populate_zonedump_data() + return cls._ZONEDUMP_DATA + + @classmethod + def _populate_fixed_offsets(cls): + cls._FIXED_OFFSET_ZONES = { + "UTC": ZoneOffset("UTC", ZERO, ZERO), + } + + @classmethod + def _populate_zonedump_data(cls): + def _Africa_Abidjan(): + LMT = ZoneOffset("LMT", timedelta(seconds=-968)) + GMT = ZoneOffset("GMT", ZERO) + + return [ + ZoneTransition(datetime(1912, 1, 1), LMT, GMT), + ] + + def _Africa_Casablanca(): + P00_s = ZoneOffset("+00", ZERO, ZERO) + P01_d = ZoneOffset("+01", ONE_H, ONE_H) + P00_d = ZoneOffset("+00", ZERO, -ONE_H) + P01_s = ZoneOffset("+01", ONE_H, ZERO) + + return [ + # Morocco sometimes pauses DST during Ramadan + ZoneTransition(datetime(2018, 3, 25, 2), P00_s, P01_d), + ZoneTransition(datetime(2018, 5, 13, 3), P01_d, P00_s), + ZoneTransition(datetime(2018, 6, 17, 2), P00_s, P01_d), + # On October 28th Morocco set standard time to +01, + # with negative DST only during Ramadan + ZoneTransition(datetime(2018, 10, 28, 3), P01_d, P01_s), + ZoneTransition(datetime(2019, 5, 5, 3), P01_s, P00_d), + ZoneTransition(datetime(2019, 6, 9, 2), P00_d, P01_s), + ] + + def _America_Los_Angeles(): + LMT = ZoneOffset("LMT", timedelta(seconds=-28378), ZERO) + PST = ZoneOffset("PST", timedelta(hours=-8), ZERO) + PDT = ZoneOffset("PDT", timedelta(hours=-7), ONE_H) + PWT = ZoneOffset("PWT", timedelta(hours=-7), ONE_H) + PPT = ZoneOffset("PPT", timedelta(hours=-7), ONE_H) + + return [ + ZoneTransition(datetime(1883, 11, 18, 12, 7, 2), LMT, PST), + ZoneTransition(datetime(1918, 3, 31, 2), PST, PDT), + ZoneTransition(datetime(1918, 3, 31, 2), PST, PDT), + ZoneTransition(datetime(1918, 10, 27, 2), PDT, PST), + # Transition to Pacific War Time + ZoneTransition(datetime(1942, 2, 9, 2), PST, PWT), + # Transition from Pacific War Time to Pacific Peace Time + ZoneTransition(datetime(1945, 8, 14, 16), PWT, PPT), + ZoneTransition(datetime(1945, 9, 30, 2), PPT, PST), + ZoneTransition(datetime(2015, 3, 8, 2), PST, PDT), + ZoneTransition(datetime(2015, 11, 1, 2), PDT, PST), + # After 2038: Rules continue indefinitely + ZoneTransition(datetime(2450, 3, 13, 2), PST, PDT), + ZoneTransition(datetime(2450, 11, 6, 2), PDT, PST), + ] + + def _America_Santiago(): + LMT = ZoneOffset("LMT", timedelta(seconds=-16966), ZERO) + SMT = ZoneOffset("SMT", timedelta(seconds=-16966), ZERO) + N05 = ZoneOffset("-05", timedelta(seconds=-18000), ZERO) + N04 = ZoneOffset("-04", timedelta(seconds=-14400), ZERO) + N03 = ZoneOffset("-03", timedelta(seconds=-10800), ONE_H) + + return [ + ZoneTransition(datetime(1890, 1, 1), LMT, SMT), + ZoneTransition(datetime(1910, 1, 10), SMT, N05), + ZoneTransition(datetime(1916, 7, 1), N05, SMT), + ZoneTransition(datetime(2008, 3, 30), N03, N04), + ZoneTransition(datetime(2008, 10, 12), N04, N03), + ZoneTransition(datetime(2040, 4, 8), N03, N04), + ZoneTransition(datetime(2040, 9, 2), N04, N03), + ] + + def _Asia_Tokyo(): + JST = ZoneOffset("JST", timedelta(seconds=32400), ZERO) + JDT = ZoneOffset("JDT", timedelta(seconds=36000), ONE_H) + + # Japan had DST from 1948 to 1951, and it was unusual in that + # the transition from DST to STD occurred at 25:00, and is + # denominated as such in the time zone database + return [ + ZoneTransition(datetime(1948, 5, 2), JST, JDT), + ZoneTransition(datetime(1948, 9, 12, 1), JDT, JST), + ZoneTransition(datetime(1951, 9, 9, 1), JDT, JST), + ] + + def _Australia_Sydney(): + LMT = ZoneOffset("LMT", timedelta(seconds=36292), ZERO) + AEST = ZoneOffset("AEST", timedelta(seconds=36000), ZERO) + AEDT = ZoneOffset("AEDT", timedelta(seconds=39600), ONE_H) + + return [ + ZoneTransition(datetime(1895, 2, 1), LMT, AEST), + ZoneTransition(datetime(1917, 1, 1, 0, 1), AEST, AEDT), + ZoneTransition(datetime(1917, 3, 25, 2), AEDT, AEST), + ZoneTransition(datetime(2012, 4, 1, 3), AEDT, AEST), + ZoneTransition(datetime(2012, 10, 7, 2), AEST, AEDT), + ZoneTransition(datetime(2040, 4, 1, 3), AEDT, AEST), + ZoneTransition(datetime(2040, 10, 7, 2), AEST, AEDT), + ] + + def _Europe_Dublin(): + LMT = ZoneOffset("LMT", timedelta(seconds=-1500), ZERO) + DMT = ZoneOffset("DMT", timedelta(seconds=-1521), ZERO) + IST_0 = ZoneOffset("IST", timedelta(seconds=2079), ONE_H) + GMT_0 = ZoneOffset("GMT", ZERO, ZERO) + BST = ZoneOffset("BST", ONE_H, ONE_H) + GMT_1 = ZoneOffset("GMT", ZERO, -ONE_H) + IST_1 = ZoneOffset("IST", ONE_H, ZERO) + + return [ + ZoneTransition(datetime(1880, 8, 2, 0), LMT, DMT), + ZoneTransition(datetime(1916, 5, 21, 2), DMT, IST_0), + ZoneTransition(datetime(1916, 10, 1, 3), IST_0, GMT_0), + ZoneTransition(datetime(1917, 4, 8, 2), GMT_0, BST), + ZoneTransition(datetime(2016, 3, 27, 1), GMT_1, IST_1), + ZoneTransition(datetime(2016, 10, 30, 2), IST_1, GMT_1), + ZoneTransition(datetime(2487, 3, 30, 1), GMT_1, IST_1), + ZoneTransition(datetime(2487, 10, 26, 2), IST_1, GMT_1), + ] + + def _Europe_Lisbon(): + WET = ZoneOffset("WET", ZERO, ZERO) + WEST = ZoneOffset("WEST", ONE_H, ONE_H) + CET = ZoneOffset("CET", ONE_H, ZERO) + CEST = ZoneOffset("CEST", timedelta(seconds=7200), ONE_H) + + return [ + ZoneTransition(datetime(1992, 3, 29, 1), WET, WEST), + ZoneTransition(datetime(1992, 9, 27, 2), WEST, CET), + ZoneTransition(datetime(1993, 3, 28, 2), CET, CEST), + ZoneTransition(datetime(1993, 9, 26, 3), CEST, CET), + ZoneTransition(datetime(1996, 3, 31, 2), CET, WEST), + ZoneTransition(datetime(1996, 10, 27, 2), WEST, WET), + ] + + def _Europe_London(): + LMT = ZoneOffset("LMT", timedelta(seconds=-75), ZERO) + GMT = ZoneOffset("GMT", ZERO, ZERO) + BST = ZoneOffset("BST", ONE_H, ONE_H) + + return [ + ZoneTransition(datetime(1847, 12, 1), LMT, GMT), + ZoneTransition(datetime(2005, 3, 27, 1), GMT, BST), + ZoneTransition(datetime(2005, 10, 30, 2), BST, GMT), + ZoneTransition(datetime(2043, 3, 29, 1), GMT, BST), + ZoneTransition(datetime(2043, 10, 25, 2), BST, GMT), + ] + + def _Pacific_Kiritimati(): + LMT = ZoneOffset("LMT", timedelta(seconds=-37760), ZERO) + N1040 = ZoneOffset("-1040", timedelta(seconds=-38400), ZERO) + N10 = ZoneOffset("-10", timedelta(seconds=-36000), ZERO) + P14 = ZoneOffset("+14", timedelta(seconds=50400), ZERO) + + # This is literally every transition in Christmas Island history + return [ + ZoneTransition(datetime(1901, 1, 1), LMT, N1040), + ZoneTransition(datetime(1979, 10, 1), N1040, N10), + # They skipped December 31, 1994 + ZoneTransition(datetime(1994, 12, 31), N10, P14), + ] + + cls._ZONEDUMP_DATA = { + "Africa/Abidjan": _Africa_Abidjan(), + "Africa/Casablanca": _Africa_Casablanca(), + "America/Los_Angeles": _America_Los_Angeles(), + "America/Santiago": _America_Santiago(), + "Australia/Sydney": _Australia_Sydney(), + "Asia/Tokyo": _Asia_Tokyo(), + "Europe/Dublin": _Europe_Dublin(), + "Europe/Lisbon": _Europe_Lisbon(), + "Europe/London": _Europe_London(), + "Pacific/Kiritimati": _Pacific_Kiritimati(), + } + + _ZONEDUMP_DATA = None + _FIXED_OFFSET_ZONES = None + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py new file mode 100644 index 00000000000..c00815e2fd4 --- /dev/null +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -0,0 +1,370 @@ +import contextlib +import datetime +import os +import pickle +import unittest +import zoneinfo + +from test.support.hypothesis_helper import hypothesis + +import test.test_zoneinfo._support as test_support + +ZoneInfoTestBase = test_support.ZoneInfoTestBase + +py_zoneinfo, c_zoneinfo = test_support.get_modules() + +UTC = datetime.timezone.utc +MIN_UTC = datetime.datetime.min.replace(tzinfo=UTC) +MAX_UTC = datetime.datetime.max.replace(tzinfo=UTC) +ZERO = datetime.timedelta(0) + + +def _valid_keys(): + """Get available time zones, including posix/ and right/ directories.""" + from importlib import resources + + available_zones = sorted(zoneinfo.available_timezones()) + TZPATH = zoneinfo.TZPATH + + def valid_key(key): + for root in TZPATH: + key_file = os.path.join(root, key) + if os.path.exists(key_file): + return True + + components = key.split("/") + package_name = ".".join(["tzdata.zoneinfo"] + components[:-1]) + resource_name = components[-1] + + try: + return resources.files(package_name).joinpath(resource_name).is_file() + except ModuleNotFoundError: + return False + + # This relies on the fact that dictionaries maintain insertion order — for + # shrinking purposes, it is preferable to start with the standard version, + # then move to the posix/ version, then to the right/ version. + out_zones = {"": available_zones} + for prefix in ["posix", "right"]: + prefix_out = [] + for key in available_zones: + prefix_key = f"{prefix}/{key}" + if valid_key(prefix_key): + prefix_out.append(prefix_key) + + out_zones[prefix] = prefix_out + + output = [] + for keys in out_zones.values(): + output.extend(keys) + + return output + + +VALID_KEYS = _valid_keys() +if not VALID_KEYS: + raise unittest.SkipTest("No time zone data available") + + +def valid_keys(): + return hypothesis.strategies.sampled_from(VALID_KEYS) + + +KEY_EXAMPLES = [ + "Africa/Abidjan", + "Africa/Casablanca", + "America/Los_Angeles", + "America/Santiago", + "Asia/Tokyo", + "Australia/Sydney", + "Europe/Dublin", + "Europe/Lisbon", + "Europe/London", + "Pacific/Kiritimati", + "UTC", +] + + +def add_key_examples(f): + for key in KEY_EXAMPLES: + f = hypothesis.example(key)(f) + return f + + +class ZoneInfoTest(ZoneInfoTestBase): + module = py_zoneinfo + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_str(self, key): + zi = self.klass(key) + self.assertEqual(str(zi), key) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_key(self, key): + zi = self.klass(key) + + self.assertEqual(zi.key, key) + + @hypothesis.given( + dt=hypothesis.strategies.one_of( + hypothesis.strategies.datetimes(), hypothesis.strategies.times() + ) + ) + @hypothesis.example(dt=datetime.datetime.min) + @hypothesis.example(dt=datetime.datetime.max) + @hypothesis.example(dt=datetime.datetime(1970, 1, 1)) + @hypothesis.example(dt=datetime.datetime(2039, 1, 1)) + @hypothesis.example(dt=datetime.time(0)) + @hypothesis.example(dt=datetime.time(12, 0)) + @hypothesis.example(dt=datetime.time(23, 59, 59, 999999)) + def test_utc(self, dt): + zi = self.klass("UTC") + dt_zi = dt.replace(tzinfo=zi) + + self.assertEqual(dt_zi.utcoffset(), ZERO) + self.assertEqual(dt_zi.dst(), ZERO) + self.assertEqual(dt_zi.tzname(), "UTC") + + +class CZoneInfoTest(ZoneInfoTest): + module = c_zoneinfo + + +class ZoneInfoPickleTest(ZoneInfoTestBase): + module = py_zoneinfo + + def setUp(self): + with contextlib.ExitStack() as stack: + stack.enter_context(test_support.set_zoneinfo_module(self.module)) + self.addCleanup(stack.pop_all().close) + + super().setUp() + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_cache(self, key): + zi = self.klass(key) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) + + self.assertIs(zi, zi_rt) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_no_cache(self, key): + zi = self.klass.no_cache(key) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) + + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_cache_multiple_rounds(self, key): + """Test that pickle/unpickle is idempotent.""" + zi_0 = self.klass(key) + pkl_str_0 = pickle.dumps(zi_0) + zi_1 = pickle.loads(pkl_str_0) + pkl_str_1 = pickle.dumps(zi_1) + zi_2 = pickle.loads(pkl_str_1) + pkl_str_2 = pickle.dumps(zi_2) + + self.assertEqual(pkl_str_0, pkl_str_1) + self.assertEqual(pkl_str_1, pkl_str_2) + + self.assertIs(zi_0, zi_1) + self.assertIs(zi_0, zi_2) + self.assertIs(zi_1, zi_2) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_no_cache_multiple_rounds(self, key): + """Test that pickle/unpickle is idempotent.""" + zi_cache = self.klass(key) + + zi_0 = self.klass.no_cache(key) + pkl_str_0 = pickle.dumps(zi_0) + zi_1 = pickle.loads(pkl_str_0) + pkl_str_1 = pickle.dumps(zi_1) + zi_2 = pickle.loads(pkl_str_1) + pkl_str_2 = pickle.dumps(zi_2) + + self.assertEqual(pkl_str_0, pkl_str_1) + self.assertEqual(pkl_str_1, pkl_str_2) + + self.assertIsNot(zi_0, zi_1) + self.assertIsNot(zi_0, zi_2) + self.assertIsNot(zi_1, zi_2) + + self.assertIsNot(zi_0, zi_cache) + self.assertIsNot(zi_1, zi_cache) + self.assertIsNot(zi_2, zi_cache) + + +class CZoneInfoPickleTest(ZoneInfoPickleTest): + module = c_zoneinfo + + +class ZoneInfoCacheTest(ZoneInfoTestBase): + module = py_zoneinfo + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_cache(self, key): + zi_0 = self.klass(key) + zi_1 = self.klass(key) + + self.assertIs(zi_0, zi_1) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_no_cache(self, key): + zi_0 = self.klass.no_cache(key) + zi_1 = self.klass.no_cache(key) + + self.assertIsNot(zi_0, zi_1) + + +class CZoneInfoCacheTest(ZoneInfoCacheTest): + klass = c_zoneinfo.ZoneInfo + + +class PythonCConsistencyTest(unittest.TestCase): + """Tests that the C and Python versions do the same thing.""" + + def _is_ambiguous(self, dt): + return dt.replace(fold=not dt.fold).utcoffset() == dt.utcoffset() + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime.min, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.max, key="America/New_York") + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime(2020, 6, 1), key="Europe/Paris") + def test_same_str(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + self.assertEqual(str(py_dt), str(c_dt)) + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 2, 5), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 8, 12), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2040, 1, 1), key="Africa/Casablanca") + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime(2040, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime.min, key="Asia/Tokyo") + @hypothesis.example(dt=datetime.datetime.max, key="Asia/Tokyo") + def test_same_offsets_and_names(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + self.assertEqual(py_dt.tzname(), c_dt.tzname()) + self.assertEqual(py_dt.utcoffset(), c_dt.utcoffset()) + self.assertEqual(py_dt.dst(), c_dt.dst()) + + @hypothesis.given( + dt=hypothesis.strategies.datetimes(timezones=hypothesis.strategies.just(UTC)), + key=valid_keys(), + ) + @hypothesis.example(dt=MIN_UTC, key="Asia/Tokyo") + @hypothesis.example(dt=MAX_UTC, key="Asia/Tokyo") + @hypothesis.example(dt=MIN_UTC, key="America/New_York") + @hypothesis.example(dt=MAX_UTC, key="America/New_York") + @hypothesis.example( + dt=datetime.datetime(2006, 10, 29, 5, 15, tzinfo=UTC), + key="America/New_York", + ) + def test_same_from_utc(self, dt, key): + py_zi = py_zoneinfo.ZoneInfo(key) + c_zi = c_zoneinfo.ZoneInfo(key) + + # Convert to UTC: This can overflow, but we just care about consistency + py_overflow_exc = None + c_overflow_exc = None + try: + py_dt = dt.astimezone(py_zi) + except OverflowError as e: + py_overflow_exc = e + + try: + c_dt = dt.astimezone(c_zi) + except OverflowError as e: + c_overflow_exc = e + + if (py_overflow_exc is not None) != (c_overflow_exc is not None): + raise py_overflow_exc or c_overflow_exc # pragma: nocover + + if py_overflow_exc is not None: + return # Consistently raises the same exception + + # PEP 495 says that an inter-zone comparison between ambiguous + # datetimes is always False. + if py_dt != c_dt: + self.assertEqual( + self._is_ambiguous(py_dt), + self._is_ambiguous(c_dt), + (py_dt, c_dt), + ) + + self.assertEqual(py_dt.tzname(), c_dt.tzname()) + self.assertEqual(py_dt.utcoffset(), c_dt.utcoffset()) + self.assertEqual(py_dt.dst(), c_dt.dst()) + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime.max, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.min, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.min, key="Asia/Tokyo") + @hypothesis.example(dt=datetime.datetime.max, key="Asia/Tokyo") + def test_same_to_utc(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + # Convert from UTC: Overflow OK if it happens in both implementations + py_overflow_exc = None + c_overflow_exc = None + try: + py_utc = py_dt.astimezone(UTC) + except OverflowError as e: + py_overflow_exc = e + + try: + c_utc = c_dt.astimezone(UTC) + except OverflowError as e: + c_overflow_exc = e + + if (py_overflow_exc is not None) != (c_overflow_exc is not None): + raise py_overflow_exc or c_overflow_exc # pragma: nocover + + if py_overflow_exc is not None: + return # Consistently raises the same exception + + self.assertEqual(py_utc, c_utc) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_cross_module_pickle(self, key): + py_zi = py_zoneinfo.ZoneInfo(key) + c_zi = c_zoneinfo.ZoneInfo(key) + + with test_support.set_zoneinfo_module(py_zoneinfo): + py_pkl = pickle.dumps(py_zi) + + with test_support.set_zoneinfo_module(c_zoneinfo): + c_pkl = pickle.dumps(c_zi) + + with test_support.set_zoneinfo_module(c_zoneinfo): + # Python → C + py_to_c_zi = pickle.loads(py_pkl) + self.assertIs(py_to_c_zi, c_zi) + + with test_support.set_zoneinfo_module(py_zoneinfo): + # C → Python + c_to_py_zi = pickle.loads(c_pkl) + self.assertIs(c_to_py_zi, py_zi) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py new file mode 100644 index 00000000000..cf618534add --- /dev/null +++ b/Lib/test/test_zstd.py @@ -0,0 +1,2802 @@ +import array +import gc +import io +import pathlib +import random +import re +import os +import unittest +import tempfile +import threading + +from test.support.import_helper import import_module +from test.support import threading_helper +from test.support import _1M +from test.support import Py_GIL_DISABLED + +_zstd = import_module("_zstd") +zstd = import_module("compression.zstd") + +from compression.zstd import ( + open, + compress, + decompress, + ZstdCompressor, + ZstdDecompressor, + ZstdDict, + ZstdError, + zstd_version, + zstd_version_info, + COMPRESSION_LEVEL_DEFAULT, + get_frame_info, + get_frame_size, + finalize_dict, + train_dict, + CompressionParameter, + DecompressionParameter, + Strategy, + ZstdFile, +) + +_1K = 1024 +_130_1K = 130 * _1K +DICT_SIZE1 = 3*_1K + +DAT_130K_D = None +DAT_130K_C = None + +DECOMPRESSED_DAT = None +COMPRESSED_DAT = None + +DECOMPRESSED_100_PLUS_32KB = None +COMPRESSED_100_PLUS_32KB = None + +SKIPPABLE_FRAME = None + +THIS_FILE_BYTES = None +THIS_FILE_STR = None +COMPRESSED_THIS_FILE = None + +COMPRESSED_BOGUS = None + +SAMPLES = None + +TRAINED_DICT = None + +# Cannot be deferred to setup as it is used to check whether or not to skip +# tests +try: + SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) +except Exception: + SUPPORT_MULTITHREADING = False + +C_INT_MIN = -(2**31) +C_INT_MAX = (2**31) - 1 + + +def setUpModule(): + # uncompressed size 130KB, more than a zstd block. + # with a frame epilogue, 4 bytes checksum. + global DAT_130K_D + DAT_130K_D = bytes([random.randint(0, 127) for _ in range(130*_1K)]) + + global DAT_130K_C + DAT_130K_C = compress(DAT_130K_D, options={CompressionParameter.checksum_flag:1}) + + global DECOMPRESSED_DAT + DECOMPRESSED_DAT = b'abcdefg123456' * 1000 + + global COMPRESSED_DAT + COMPRESSED_DAT = compress(DECOMPRESSED_DAT) + + global DECOMPRESSED_100_PLUS_32KB + DECOMPRESSED_100_PLUS_32KB = b'a' * (100 + 32*_1K) + + global COMPRESSED_100_PLUS_32KB + COMPRESSED_100_PLUS_32KB = compress(DECOMPRESSED_100_PLUS_32KB) + + global SKIPPABLE_FRAME + SKIPPABLE_FRAME = (0x184D2A50).to_bytes(4, byteorder='little') + \ + (32*_1K).to_bytes(4, byteorder='little') + \ + b'a' * (32*_1K) + + global THIS_FILE_BYTES, THIS_FILE_STR + with io.open(os.path.abspath(__file__), 'rb') as f: + THIS_FILE_BYTES = f.read() + THIS_FILE_BYTES = re.sub(rb'\r?\n', rb'\n', THIS_FILE_BYTES) + THIS_FILE_STR = THIS_FILE_BYTES.decode('utf-8') + + global COMPRESSED_THIS_FILE + COMPRESSED_THIS_FILE = compress(THIS_FILE_BYTES) + + global COMPRESSED_BOGUS + COMPRESSED_BOGUS = DECOMPRESSED_DAT + + # dict data + words = [b'red', b'green', b'yellow', b'black', b'withe', b'blue', + b'lilac', b'purple', b'navy', b'glod', b'silver', b'olive', + b'dog', b'cat', b'tiger', b'lion', b'fish', b'bird'] + lst = [] + for i in range(300): + sample = [b'%s = %d' % (random.choice(words), random.randrange(100)) + for j in range(20)] + sample = b'\n'.join(sample) + + lst.append(sample) + global SAMPLES + SAMPLES = lst + assert len(SAMPLES) > 10 + + global TRAINED_DICT + TRAINED_DICT = train_dict(SAMPLES, 3*_1K) + assert len(TRAINED_DICT.dict_content) <= 3*_1K + + +class FunctionsTestCase(unittest.TestCase): + + def test_version(self): + s = ".".join((str(i) for i in zstd_version_info)) + self.assertEqual(s, zstd_version) + + def test_compressionLevel_values(self): + min, max = CompressionParameter.compression_level.bounds() + self.assertIs(type(COMPRESSION_LEVEL_DEFAULT), int) + self.assertIs(type(min), int) + self.assertIs(type(max), int) + self.assertLess(min, max) + + def test_roundtrip_default(self): + raw_dat = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6] + dat1 = compress(raw_dat) + dat2 = decompress(dat1) + self.assertEqual(dat2, raw_dat) + + def test_roundtrip_level(self): + raw_dat = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6] + level_min, level_max = CompressionParameter.compression_level.bounds() + + for level in range(max(-20, level_min), level_max + 1): + dat1 = compress(raw_dat, level) + dat2 = decompress(dat1) + self.assertEqual(dat2, raw_dat) + + def test_get_frame_info(self): + # no dict + info = get_frame_info(COMPRESSED_100_PLUS_32KB[:20]) + self.assertEqual(info.decompressed_size, 32 * _1K + 100) + self.assertEqual(info.dictionary_id, 0) + + # use dict + dat = compress(b"a" * 345, zstd_dict=TRAINED_DICT) + info = get_frame_info(dat) + self.assertEqual(info.decompressed_size, 345) + self.assertEqual(info.dictionary_id, TRAINED_DICT.dict_id) + + with self.assertRaisesRegex(ZstdError, "not less than the frame header"): + get_frame_info(b"aaaaaaaaaaaaaa") + + def test_get_frame_size(self): + size = get_frame_size(COMPRESSED_100_PLUS_32KB) + self.assertEqual(size, len(COMPRESSED_100_PLUS_32KB)) + + with self.assertRaisesRegex(ZstdError, "not less than this complete frame"): + get_frame_size(b"aaaaaaaaaaaaaa") + + def test_decompress_2x130_1K(self): + decompressed_size = get_frame_info(DAT_130K_C).decompressed_size + self.assertEqual(decompressed_size, _130_1K) + + dat = decompress(DAT_130K_C + DAT_130K_C) + self.assertEqual(len(dat), 2 * _130_1K) + + +class CompressorTestCase(unittest.TestCase): + + def test_simple_compress_bad_args(self): + # ZstdCompressor + self.assertRaises(TypeError, ZstdCompressor, []) + self.assertRaises(TypeError, ZstdCompressor, level=3.14) + self.assertRaises(TypeError, ZstdCompressor, level="abc") + self.assertRaises(TypeError, ZstdCompressor, options=b"abc") + + self.assertRaises(TypeError, ZstdCompressor, zstd_dict=123) + self.assertRaises(TypeError, ZstdCompressor, zstd_dict=b"abcd1234") + self.assertRaises(TypeError, ZstdCompressor, zstd_dict={1: 2, 3: 4}) + + # valid range for compression level is [-(1<<17), 22] + msg = r'illegal compression level {}; the valid range is \[-?\d+, -?\d+\]' + with self.assertRaisesRegex(ValueError, msg.format(C_INT_MAX)): + ZstdCompressor(C_INT_MAX) + with self.assertRaisesRegex(ValueError, msg.format(C_INT_MIN)): + ZstdCompressor(C_INT_MIN) + msg = r'illegal compression level; the valid range is \[-?\d+, -?\d+\]' + with self.assertRaisesRegex(ValueError, msg): + ZstdCompressor(level=-(2**1000)) + with self.assertRaisesRegex(ValueError, msg): + ZstdCompressor(level=2**1000) + + with self.assertRaises(ValueError): + ZstdCompressor(options={CompressionParameter.window_log: 100}) + with self.assertRaises(ValueError): + ZstdCompressor(options={3333: 100}) + + # Method bad arguments + zc = ZstdCompressor() + self.assertRaises(TypeError, zc.compress) + self.assertRaises((TypeError, ValueError), zc.compress, b"foo", b"bar") + self.assertRaises(TypeError, zc.compress, "str") + self.assertRaises((TypeError, ValueError), zc.flush, b"foo") + self.assertRaises(TypeError, zc.flush, b"blah", 1) + + self.assertRaises(ValueError, zc.compress, b'', -1) + self.assertRaises(ValueError, zc.compress, b'', 3) + self.assertRaises(ValueError, zc.flush, zc.CONTINUE) # 0 + self.assertRaises(ValueError, zc.flush, 3) + + zc.compress(b'') + zc.compress(b'', zc.CONTINUE) + zc.compress(b'', zc.FLUSH_BLOCK) + zc.compress(b'', zc.FLUSH_FRAME) + empty = zc.flush() + zc.flush(zc.FLUSH_BLOCK) + zc.flush(zc.FLUSH_FRAME) + + def test_compress_parameters(self): + d = {CompressionParameter.compression_level : 10, + + CompressionParameter.window_log : 12, + CompressionParameter.hash_log : 10, + CompressionParameter.chain_log : 12, + CompressionParameter.search_log : 12, + CompressionParameter.min_match : 4, + CompressionParameter.target_length : 12, + CompressionParameter.strategy : Strategy.lazy, + + CompressionParameter.enable_long_distance_matching : 1, + CompressionParameter.ldm_hash_log : 12, + CompressionParameter.ldm_min_match : 11, + CompressionParameter.ldm_bucket_size_log : 5, + CompressionParameter.ldm_hash_rate_log : 12, + + CompressionParameter.content_size_flag : 1, + CompressionParameter.checksum_flag : 1, + CompressionParameter.dict_id_flag : 0, + + CompressionParameter.nb_workers : 2 if SUPPORT_MULTITHREADING else 0, + CompressionParameter.job_size : 5*_1M if SUPPORT_MULTITHREADING else 0, + CompressionParameter.overlap_log : 9 if SUPPORT_MULTITHREADING else 0, + } + ZstdCompressor(options=d) + + d1 = d.copy() + # larger than signed int + d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MAX + with self.assertRaises(ValueError): + ZstdCompressor(options=d1) + # smaller than signed int + d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MIN + with self.assertRaises(ValueError): + ZstdCompressor(options=d1) + + # out of bounds compression level + level_min, level_max = CompressionParameter.compression_level.bounds() + with self.assertRaises(ValueError): + compress(b'', level_max+1) + with self.assertRaises(ValueError): + compress(b'', level_min-1) + with self.assertRaises(ValueError): + compress(b'', 2**1000) + with self.assertRaises(ValueError): + compress(b'', -(2**1000)) + with self.assertRaises(ValueError): + compress(b'', options={ + CompressionParameter.compression_level: level_max+1}) + with self.assertRaises(ValueError): + compress(b'', options={ + CompressionParameter.compression_level: level_min-1}) + + # zstd lib doesn't support MT compression + if not SUPPORT_MULTITHREADING: + with self.assertRaises(ValueError): + ZstdCompressor(options={CompressionParameter.nb_workers:4}) + with self.assertRaises(ValueError): + ZstdCompressor(options={CompressionParameter.job_size:4}) + with self.assertRaises(ValueError): + ZstdCompressor(options={CompressionParameter.overlap_log:4}) + + # out of bounds error msg + option = {CompressionParameter.window_log:100} + with self.assertRaisesRegex( + ValueError, + "compression parameter 'window_log' received an illegal value 100; " + r'the valid range is \[-?\d+, -?\d+\]', + ): + compress(b'', options=option) + + def test_unknown_compression_parameter(self): + KEY = 100001234 + option = {CompressionParameter.compression_level: 10, + KEY: 200000000} + pattern = rf"invalid compression parameter 'unknown parameter \(key {KEY}\)'" + with self.assertRaisesRegex(ValueError, pattern): + ZstdCompressor(options=option) + + @unittest.skipIf(not SUPPORT_MULTITHREADING, + "zstd build doesn't support multi-threaded compression") + def test_zstd_multithread_compress(self): + size = 40*_1M + b = THIS_FILE_BYTES * (size // len(THIS_FILE_BYTES)) + + options = {CompressionParameter.compression_level : 4, + CompressionParameter.nb_workers : 2} + + # compress() + dat1 = compress(b, options=options) + dat2 = decompress(dat1) + self.assertEqual(dat2, b) + + # ZstdCompressor + c = ZstdCompressor(options=options) + dat1 = c.compress(b, c.CONTINUE) + dat2 = c.compress(b, c.FLUSH_BLOCK) + dat3 = c.compress(b, c.FLUSH_FRAME) + dat4 = decompress(dat1+dat2+dat3) + self.assertEqual(dat4, b * 3) + + # ZstdFile + with ZstdFile(io.BytesIO(), 'w', options=options) as f: + f.write(b) + + def test_compress_flushblock(self): + point = len(THIS_FILE_BYTES) // 2 + + c = ZstdCompressor() + self.assertEqual(c.last_mode, c.FLUSH_FRAME) + dat1 = c.compress(THIS_FILE_BYTES[:point]) + self.assertEqual(c.last_mode, c.CONTINUE) + dat1 += c.compress(THIS_FILE_BYTES[point:], c.FLUSH_BLOCK) + self.assertEqual(c.last_mode, c.FLUSH_BLOCK) + dat2 = c.flush() + pattern = "Compressed data ended before the end-of-stream marker" + with self.assertRaisesRegex(ZstdError, pattern): + decompress(dat1) + + dat3 = decompress(dat1 + dat2) + + self.assertEqual(dat3, THIS_FILE_BYTES) + + def test_compress_flushframe(self): + # test compress & decompress + point = len(THIS_FILE_BYTES) // 2 + + c = ZstdCompressor() + + dat1 = c.compress(THIS_FILE_BYTES[:point]) + self.assertEqual(c.last_mode, c.CONTINUE) + + dat1 += c.compress(THIS_FILE_BYTES[point:], c.FLUSH_FRAME) + self.assertEqual(c.last_mode, c.FLUSH_FRAME) + + nt = get_frame_info(dat1) + self.assertEqual(nt.decompressed_size, None) # no content size + + dat2 = decompress(dat1) + + self.assertEqual(dat2, THIS_FILE_BYTES) + + # single .FLUSH_FRAME mode has content size + c = ZstdCompressor() + dat = c.compress(THIS_FILE_BYTES, mode=c.FLUSH_FRAME) + self.assertEqual(c.last_mode, c.FLUSH_FRAME) + + nt = get_frame_info(dat) + self.assertEqual(nt.decompressed_size, len(THIS_FILE_BYTES)) + + def test_compress_empty(self): + # output empty content frame + self.assertNotEqual(compress(b''), b'') + + c = ZstdCompressor() + self.assertNotEqual(c.compress(b'', c.FLUSH_FRAME), b'') + + def test_set_pledged_input_size(self): + DAT = DECOMPRESSED_100_PLUS_32KB + CHUNK_SIZE = len(DAT) // 3 + + # wrong value + c = ZstdCompressor() + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(-300) + # overflow + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64) + # ZSTD_CONTENTSIZE_ERROR is invalid + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-2) + # ZSTD_CONTENTSIZE_UNKNOWN should use None + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-1) + + # check valid values are settable + c.set_pledged_input_size(2**63) + c.set_pledged_input_size(2**64-3) + + # check that zero means empty frame + c = ZstdCompressor(level=1) + c.set_pledged_input_size(0) + c.compress(b'') + dat = c.flush() + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, 0) + + + # wrong mode + c = ZstdCompressor(level=1) + c.compress(b'123456') + self.assertEqual(c.last_mode, c.CONTINUE) + with self.assertRaisesRegex(ValueError, + r'last_mode == FLUSH_FRAME'): + c.set_pledged_input_size(300) + + # None value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(None) + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, None) + + # correct value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + chunks = [] + posi = 0 + while posi < len(DAT): + dat = c.compress(DAT[posi:posi+CHUNK_SIZE]) + posi += CHUNK_SIZE + chunks.append(dat) + + dat = c.flush() + chunks.append(dat) + chunks = b''.join(chunks) + + ret = get_frame_info(chunks) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(chunks), DAT) + + c.set_pledged_input_size(len(DAT)) # the second frame + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(dat), DAT) + + # not enough data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)+1) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.flush() + + # too much data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.compress(b'extra', ZstdCompressor.FLUSH_FRAME) + + # content size not set if content_size_flag == 0 + c = ZstdCompressor(options={CompressionParameter.content_size_flag: 0}) + c.set_pledged_input_size(10) + dat1 = c.compress(b"hello") + dat2 = c.compress(b"world") + dat3 = c.flush() + frame_data = get_frame_info(dat1 + dat2 + dat3) + self.assertIsNone(frame_data.decompressed_size) + + +class DecompressorTestCase(unittest.TestCase): + + def test_simple_decompress_bad_args(self): + # ZstdDecompressor + self.assertRaises(TypeError, ZstdDecompressor, ()) + self.assertRaises(TypeError, ZstdDecompressor, zstd_dict=123) + self.assertRaises(TypeError, ZstdDecompressor, zstd_dict=b'abc') + self.assertRaises(TypeError, ZstdDecompressor, zstd_dict={1:2, 3:4}) + + self.assertRaises(TypeError, ZstdDecompressor, options=123) + self.assertRaises(TypeError, ZstdDecompressor, options='abc') + self.assertRaises(TypeError, ZstdDecompressor, options=b'abc') + + with self.assertRaises(ValueError): + ZstdDecompressor(options={C_INT_MAX: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={C_INT_MIN: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={0: C_INT_MAX}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={2**1000: 100}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={-(2**1000): 100}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={0: -(2**1000)}) + + with self.assertRaises(ValueError): + ZstdDecompressor(options={DecompressionParameter.window_log_max: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={3333: 100}) + + empty = compress(b'') + lzd = ZstdDecompressor() + self.assertRaises(TypeError, lzd.decompress) + self.assertRaises(TypeError, lzd.decompress, b"foo", b"bar") + self.assertRaises(TypeError, lzd.decompress, "str") + lzd.decompress(empty) + + def test_decompress_parameters(self): + d = {DecompressionParameter.window_log_max : 15} + ZstdDecompressor(options=d) + + d1 = d.copy() + # larger than signed int + d1[DecompressionParameter.window_log_max] = 2**1000 + with self.assertRaises(OverflowError): + ZstdDecompressor(None, d1) + # smaller than signed int + d1[DecompressionParameter.window_log_max] = -(2**1000) + with self.assertRaises(OverflowError): + ZstdDecompressor(None, d1) + + d1[DecompressionParameter.window_log_max] = C_INT_MAX + with self.assertRaises(ValueError): + ZstdDecompressor(None, d1) + d1[DecompressionParameter.window_log_max] = C_INT_MIN + with self.assertRaises(ValueError): + ZstdDecompressor(None, d1) + + # out of bounds error msg + options = {DecompressionParameter.window_log_max:100} + with self.assertRaisesRegex( + ValueError, + "decompression parameter 'window_log_max' received an illegal value 100; " + r'the valid range is \[-?\d+, -?\d+\]', + ): + decompress(b'', options=options) + + # out of bounds deecompression parameter + options[DecompressionParameter.window_log_max] = C_INT_MAX + with self.assertRaises(ValueError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = C_INT_MIN + with self.assertRaises(ValueError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = 2**1000 + with self.assertRaises(OverflowError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = -(2**1000) + with self.assertRaises(OverflowError): + decompress(b'', options=options) + + def test_unknown_decompression_parameter(self): + KEY = 100001234 + options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1], + KEY: 200000000} + pattern = rf"invalid decompression parameter 'unknown parameter \(key {KEY}\)'" + with self.assertRaisesRegex(ValueError, pattern): + ZstdDecompressor(options=options) + + def test_decompress_epilogue_flags(self): + # DAT_130K_C has a 4 bytes checksum at frame epilogue + + # full unlimited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + with self.assertRaises(EOFError): + dat = d.decompress(b'') + + # full limited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C, _130_1K) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + with self.assertRaises(EOFError): + dat = d.decompress(b'', 0) + + # [:-4] unlimited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-4]) + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.needs_input) + + dat = d.decompress(b'') + self.assertEqual(len(dat), 0) + self.assertTrue(d.needs_input) + + # [:-4] limited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-4], _130_1K) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + dat = d.decompress(b'', 0) + self.assertEqual(len(dat), 0) + self.assertFalse(d.needs_input) + + # [:-3] unlimited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-3]) + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.needs_input) + + dat = d.decompress(b'') + self.assertEqual(len(dat), 0) + self.assertTrue(d.needs_input) + + # [:-3] limited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-3], _130_1K) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + dat = d.decompress(b'', 0) + self.assertEqual(len(dat), 0) + self.assertFalse(d.needs_input) + + # [:-1] unlimited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-1]) + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.needs_input) + + dat = d.decompress(b'') + self.assertEqual(len(dat), 0) + self.assertTrue(d.needs_input) + + # [:-1] limited + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-1], _130_1K) + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.needs_input) + + dat = d.decompress(b'', 0) + self.assertEqual(len(dat), 0) + self.assertFalse(d.needs_input) + + def test_decompressor_arg(self): + zd = ZstdDict(b'12345678', is_raw=True) + + with self.assertRaises(TypeError): + d = ZstdDecompressor(zstd_dict={}) + + with self.assertRaises(TypeError): + d = ZstdDecompressor(options=zd) + + ZstdDecompressor() + ZstdDecompressor(zd, {}) + ZstdDecompressor(zstd_dict=zd, options={DecompressionParameter.window_log_max:25}) + + def test_decompressor_1(self): + # empty + d = ZstdDecompressor() + dat = d.decompress(b'') + + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + + # 130_1K full + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C) + + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + + # 130_1K full, limit output + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C, _130_1K) + + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + + # 130_1K, without 4 bytes checksum + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-4]) + + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + + # above, limit output + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C[:-4], _130_1K) + + self.assertEqual(len(dat), _130_1K) + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + + # full, unused_data + TRAIL = b'89234893abcd' + d = ZstdDecompressor() + dat = d.decompress(DAT_130K_C + TRAIL, _130_1K) + + self.assertEqual(len(dat), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, TRAIL) + + def test_decompressor_chunks_read_300(self): + TRAIL = b'89234893abcd' + DAT = DAT_130K_C + TRAIL + d = ZstdDecompressor() + + bi = io.BytesIO(DAT) + lst = [] + while True: + if d.needs_input: + dat = bi.read(300) + if not dat: + break + else: + raise Exception('should not get here') + + ret = d.decompress(dat) + lst.append(ret) + if d.eof: + break + + ret = b''.join(lst) + + self.assertEqual(len(ret), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data + bi.read(), TRAIL) + + def test_decompressor_chunks_read_3(self): + TRAIL = b'89234893' + DAT = DAT_130K_C + TRAIL + d = ZstdDecompressor() + + bi = io.BytesIO(DAT) + lst = [] + while True: + if d.needs_input: + dat = bi.read(3) + if not dat: + break + else: + dat = b'' + + ret = d.decompress(dat, 1) + lst.append(ret) + if d.eof: + break + + ret = b''.join(lst) + + self.assertEqual(len(ret), _130_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data + bi.read(), TRAIL) + + + def test_decompress_empty(self): + with self.assertRaises(ZstdError): + decompress(b'') + + d = ZstdDecompressor() + self.assertEqual(d.decompress(b''), b'') + self.assertFalse(d.eof) + + def test_decompress_empty_content_frame(self): + DAT = compress(b'') + # decompress + self.assertGreaterEqual(len(DAT), 4) + self.assertEqual(decompress(DAT), b'') + + with self.assertRaises(ZstdError): + decompress(DAT[:-1]) + + # ZstdDecompressor + d = ZstdDecompressor() + dat = d.decompress(DAT) + self.assertEqual(dat, b'') + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + d = ZstdDecompressor() + dat = d.decompress(DAT[:-1]) + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + +class DecompressorFlagsTestCase(unittest.TestCase): + + @classmethod + def setUpClass(cls): + options = {CompressionParameter.checksum_flag:1} + c = ZstdCompressor(options=options) + + cls.DECOMPRESSED_42 = b'a'*42 + cls.FRAME_42 = c.compress(cls.DECOMPRESSED_42, c.FLUSH_FRAME) + + cls.DECOMPRESSED_60 = b'a'*60 + cls.FRAME_60 = c.compress(cls.DECOMPRESSED_60, c.FLUSH_FRAME) + + cls.FRAME_42_60 = cls.FRAME_42 + cls.FRAME_60 + cls.DECOMPRESSED_42_60 = cls.DECOMPRESSED_42 + cls.DECOMPRESSED_60 + + cls._130_1K = 130*_1K + + c = ZstdCompressor() + cls.UNKNOWN_FRAME_42 = c.compress(cls.DECOMPRESSED_42) + c.flush() + cls.UNKNOWN_FRAME_60 = c.compress(cls.DECOMPRESSED_60) + c.flush() + cls.UNKNOWN_FRAME_42_60 = cls.UNKNOWN_FRAME_42 + cls.UNKNOWN_FRAME_60 + + cls.TRAIL = b'12345678abcdefg!@#$%^&*()_+|' + + def test_function_decompress(self): + + self.assertEqual(len(decompress(COMPRESSED_100_PLUS_32KB)), 100+32*_1K) + + # 1 frame + self.assertEqual(decompress(self.FRAME_42), self.DECOMPRESSED_42) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_42), self.DECOMPRESSED_42) + + pattern = r"Compressed data ended before the end-of-stream marker" + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.FRAME_42[:1]) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.FRAME_42[:-4]) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.FRAME_42[:-1]) + + # 2 frames + self.assertEqual(decompress(self.FRAME_42_60), self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_42_60), self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(self.FRAME_42 + self.UNKNOWN_FRAME_60), + self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_42 + self.FRAME_60), + self.DECOMPRESSED_42_60) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.FRAME_42_60[:-4]) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(self.UNKNOWN_FRAME_42_60[:-1]) + + # 130_1K + self.assertEqual(decompress(DAT_130K_C), DAT_130K_D) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(DAT_130K_C[:-4]) + + with self.assertRaisesRegex(ZstdError, pattern): + decompress(DAT_130K_C[:-1]) + + # Unknown frame descriptor + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(b'aaaaaaaaa') + + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(self.FRAME_42 + b'aaaaaaaaa') + + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(self.UNKNOWN_FRAME_42_60 + b'aaaaaaaaa') + + # doesn't match checksum + checksum = DAT_130K_C[-4:] + if checksum[0] == 255: + wrong_checksum = bytes([254]) + checksum[1:] + else: + wrong_checksum = bytes([checksum[0]+1]) + checksum[1:] + + dat = DAT_130K_C[:-4] + wrong_checksum + + with self.assertRaisesRegex(ZstdError, "doesn't match checksum"): + decompress(dat) + + def test_function_skippable(self): + self.assertEqual(decompress(SKIPPABLE_FRAME), b'') + self.assertEqual(decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME), b'') + + # 1 frame + 2 skippable + self.assertEqual(len(decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME + DAT_130K_C)), + self._130_1K) + + self.assertEqual(len(decompress(DAT_130K_C + SKIPPABLE_FRAME + SKIPPABLE_FRAME)), + self._130_1K) + + self.assertEqual(len(decompress(SKIPPABLE_FRAME + DAT_130K_C + SKIPPABLE_FRAME)), + self._130_1K) + + # unknown size + self.assertEqual(decompress(SKIPPABLE_FRAME + self.UNKNOWN_FRAME_60), + self.DECOMPRESSED_60) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_60 + SKIPPABLE_FRAME), + self.DECOMPRESSED_60) + + # 2 frames + 1 skippable + self.assertEqual(decompress(self.FRAME_42 + SKIPPABLE_FRAME + self.FRAME_60), + self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(SKIPPABLE_FRAME + self.FRAME_42_60), + self.DECOMPRESSED_42_60) + + self.assertEqual(decompress(self.UNKNOWN_FRAME_42_60 + SKIPPABLE_FRAME), + self.DECOMPRESSED_42_60) + + # incomplete + with self.assertRaises(ZstdError): + decompress(SKIPPABLE_FRAME[:1]) + + with self.assertRaises(ZstdError): + decompress(SKIPPABLE_FRAME[:-1]) + + with self.assertRaises(ZstdError): + decompress(self.FRAME_42 + SKIPPABLE_FRAME[:-1]) + + # Unknown frame descriptor + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(b'aaaaaaaaa' + SKIPPABLE_FRAME) + + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(SKIPPABLE_FRAME + b'aaaaaaaaa') + + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME + b'aaaaaaaaa') + + def test_decompressor_1(self): + # empty 1 + d = ZstdDecompressor() + + dat = d.decompress(b'') + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(b'', 0) + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(COMPRESSED_100_PLUS_32KB + b'a') + self.assertEqual(dat, DECOMPRESSED_100_PLUS_32KB) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'a') + self.assertEqual(d.unused_data, b'a') # twice + + # empty 2 + d = ZstdDecompressor() + + dat = d.decompress(b'', 0) + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(b'') + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(COMPRESSED_100_PLUS_32KB + b'a') + self.assertEqual(dat, DECOMPRESSED_100_PLUS_32KB) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'a') + self.assertEqual(d.unused_data, b'a') # twice + + # 1 frame + d = ZstdDecompressor() + dat = d.decompress(self.FRAME_42) + + self.assertEqual(dat, self.DECOMPRESSED_42) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + with self.assertRaises(EOFError): + d.decompress(b'') + + # 1 frame, trail + d = ZstdDecompressor() + dat = d.decompress(self.FRAME_42 + self.TRAIL) + + self.assertEqual(dat, self.DECOMPRESSED_42) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, self.TRAIL) + self.assertEqual(d.unused_data, self.TRAIL) # twice + + # 1 frame, 32_1K + temp = compress(b'a'*(32*_1K)) + d = ZstdDecompressor() + dat = d.decompress(temp, 32*_1K) + + self.assertEqual(dat, b'a'*(32*_1K)) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + with self.assertRaises(EOFError): + d.decompress(b'') + + # 1 frame, 32_1K+100, trail + d = ZstdDecompressor() + dat = d.decompress(COMPRESSED_100_PLUS_32KB+self.TRAIL, 100) # 100 bytes + + self.assertEqual(len(dat), 100) + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + + dat = d.decompress(b'') # 32_1K + + self.assertEqual(len(dat), 32*_1K) + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, self.TRAIL) + self.assertEqual(d.unused_data, self.TRAIL) # twice + + with self.assertRaises(EOFError): + d.decompress(b'') + + # incomplete 1 + d = ZstdDecompressor() + dat = d.decompress(self.FRAME_60[:1]) + + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # incomplete 2 + d = ZstdDecompressor() + + dat = d.decompress(self.FRAME_60[:-4]) + self.assertEqual(dat, self.DECOMPRESSED_60) + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # incomplete 3 + d = ZstdDecompressor() + + dat = d.decompress(self.FRAME_60[:-1]) + self.assertEqual(dat, self.DECOMPRESSED_60) + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + + # incomplete 4 + d = ZstdDecompressor() + + dat = d.decompress(self.FRAME_60[:-4], 60) + self.assertEqual(dat, self.DECOMPRESSED_60) + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(b'') + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # Unknown frame descriptor + d = ZstdDecompressor() + with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"): + d.decompress(b'aaaaaaaaa') + + def test_decompressor_skippable(self): + # 1 skippable + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME) + + self.assertEqual(dat, b'') + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # 1 skippable, max_length=0 + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME, 0) + + self.assertEqual(dat, b'') + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # 1 skippable, trail + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME + self.TRAIL) + + self.assertEqual(dat, b'') + self.assertTrue(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, self.TRAIL) + self.assertEqual(d.unused_data, self.TRAIL) # twice + + # incomplete + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME[:-1]) + + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + # incomplete + d = ZstdDecompressor() + dat = d.decompress(SKIPPABLE_FRAME[:-1], 0) + + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertFalse(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + dat = d.decompress(b'') + + self.assertEqual(dat, b'') + self.assertFalse(d.eof) + self.assertTrue(d.needs_input) + self.assertEqual(d.unused_data, b'') + self.assertEqual(d.unused_data, b'') # twice + + + +class ZstdDictTestCase(unittest.TestCase): + + def test_is_raw(self): + # must be passed as a keyword argument + with self.assertRaises(TypeError): + ZstdDict(bytes(8), True) + + # content < 8 + b = b'1234567' + with self.assertRaises(ValueError): + ZstdDict(b) + + # content == 8 + b = b'12345678' + zd = ZstdDict(b, is_raw=True) + self.assertEqual(zd.dict_id, 0) + + temp = compress(b'aaa12345678', level=3, zstd_dict=zd) + self.assertEqual(b'aaa12345678', decompress(temp, zd)) + + # is_raw == False + b = b'12345678abcd' + with self.assertRaises(ValueError): + ZstdDict(b) + + # read only attributes + with self.assertRaises(AttributeError): + zd.dict_content = b + + with self.assertRaises(AttributeError): + zd.dict_id = 10000 + + # ZstdDict arguments + zd = ZstdDict(TRAINED_DICT.dict_content, is_raw=False) + self.assertNotEqual(zd.dict_id, 0) + + zd = ZstdDict(TRAINED_DICT.dict_content, is_raw=True) + self.assertNotEqual(zd.dict_id, 0) # note this assertion + + with self.assertRaises(TypeError): + ZstdDict("12345678abcdef", is_raw=True) + with self.assertRaises(TypeError): + ZstdDict(TRAINED_DICT) + + # invalid parameter + with self.assertRaises(TypeError): + ZstdDict(desk333=345) + + def test_invalid_dict(self): + DICT_MAGIC = 0xEC30A437.to_bytes(4, byteorder='little') + dict_content = DICT_MAGIC + b'abcdefghighlmnopqrstuvwxyz' + + # corrupted + zd = ZstdDict(dict_content, is_raw=False) + with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'): + ZstdCompressor(zstd_dict=zd.as_digested_dict) + with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'): + ZstdDecompressor(zd) + + # wrong type + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 1, 2)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, -1)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 3)) + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, -2**1000)) + + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd, 1, 2)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd, -1)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd, 3)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, -2**1000)) + + def test_train_dict(self): + TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1) + ZstdDict(TRAINED_DICT.dict_content, is_raw=False) + + self.assertNotEqual(TRAINED_DICT.dict_id, 0) + self.assertGreater(len(TRAINED_DICT.dict_content), 0) + self.assertLessEqual(len(TRAINED_DICT.dict_content), DICT_SIZE1) + self.assertTrue(re.match(r'^<ZstdDict dict_id=\d+ dict_size=\d+>$', str(TRAINED_DICT))) + + # compress/decompress + c = ZstdCompressor(zstd_dict=TRAINED_DICT) + for sample in SAMPLES: + dat1 = compress(sample, zstd_dict=TRAINED_DICT) + dat2 = decompress(dat1, TRAINED_DICT) + self.assertEqual(sample, dat2) + + dat1 = c.compress(sample) + dat1 += c.flush() + dat2 = decompress(dat1, TRAINED_DICT) + self.assertEqual(sample, dat2) + + def test_finalize_dict(self): + DICT_SIZE2 = 200*_1K + C_LEVEL = 6 + + try: + dic2 = finalize_dict(TRAINED_DICT, SAMPLES, DICT_SIZE2, C_LEVEL) + except NotImplementedError: + # < v1.4.5 at compile-time, >= v.1.4.5 at run-time + return + + self.assertNotEqual(dic2.dict_id, 0) + self.assertGreater(len(dic2.dict_content), 0) + self.assertLessEqual(len(dic2.dict_content), DICT_SIZE2) + + # compress/decompress + c = ZstdCompressor(C_LEVEL, zstd_dict=dic2) + for sample in SAMPLES: + dat1 = compress(sample, C_LEVEL, zstd_dict=dic2) + dat2 = decompress(dat1, dic2) + self.assertEqual(sample, dat2) + + dat1 = c.compress(sample) + dat1 += c.flush() + dat2 = decompress(dat1, dic2) + self.assertEqual(sample, dat2) + + # dict mismatch + self.assertNotEqual(TRAINED_DICT.dict_id, dic2.dict_id) + + dat1 = compress(SAMPLES[0], zstd_dict=TRAINED_DICT) + with self.assertRaises(ZstdError): + decompress(dat1, dic2) + + def test_train_dict_arguments(self): + with self.assertRaises(ValueError): + train_dict([], 100*_1K) + + with self.assertRaises(ValueError): + train_dict(SAMPLES, -100) + + with self.assertRaises(ValueError): + train_dict(SAMPLES, 0) + + def test_finalize_dict_arguments(self): + with self.assertRaises(TypeError): + finalize_dict({1:2}, (b'aaa', b'bbb'), 100*_1K, 2) + + with self.assertRaises(ValueError): + finalize_dict(TRAINED_DICT, [], 100*_1K, 2) + + with self.assertRaises(ValueError): + finalize_dict(TRAINED_DICT, SAMPLES, -100, 2) + + with self.assertRaises(ValueError): + finalize_dict(TRAINED_DICT, SAMPLES, 0, 2) + + def test_train_dict_c(self): + # argument wrong type + with self.assertRaises(TypeError): + _zstd.train_dict({}, (), 100) + with self.assertRaises(TypeError): + _zstd.train_dict(bytearray(), (), 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', 99, 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', [], 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', (), 100.1) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', (99.1,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (4, -1), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (2,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (99,), 100) + + # size > size_t + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (2**1000,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (-2**1000,), 100) + + # dict_size <= 0 + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (), 0) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (), -1) + + with self.assertRaises(ZstdError): + _zstd.train_dict(b'', (), 1) + + def test_finalize_dict_c(self): + with self.assertRaises(TypeError): + _zstd.finalize_dict(1, 2, 3, 4, 5) + + # argument wrong type + with self.assertRaises(TypeError): + _zstd.finalize_dict({}, b'', (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(bytearray(TRAINED_DICT.dict_content), b'', (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, bytearray(), (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', [], 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1) + + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (4, -1), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (2,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (99,), 100, 5) + + # size > size_t + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**1000,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (-2**1000,), 100, 5) + + # dict_size <= 0 + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -1, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 2**1000, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -2**1000, 5) + + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 2**1000) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, -2**1000) + + with self.assertRaises(ZstdError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5) + + def test_train_buffer_protocol_samples(self): + def _nbytes(dat): + if isinstance(dat, (bytes, bytearray)): + return len(dat) + return memoryview(dat).nbytes + + # prepare samples + chunk_lst = [] + wrong_size_lst = [] + correct_size_lst = [] + for _ in range(300): + arr = array.array('Q', [random.randint(0, 20) for i in range(20)]) + chunk_lst.append(arr) + correct_size_lst.append(_nbytes(arr)) + wrong_size_lst.append(len(arr)) + concatenation = b''.join(chunk_lst) + + # wrong size list + with self.assertRaisesRegex(ValueError, + "The samples size tuple doesn't match the concatenation's size"): + _zstd.train_dict(concatenation, tuple(wrong_size_lst), 100*_1K) + + # correct size list + _zstd.train_dict(concatenation, tuple(correct_size_lst), 3*_1K) + + # wrong size list + with self.assertRaisesRegex(ValueError, + "The samples size tuple doesn't match the concatenation's size"): + _zstd.finalize_dict(TRAINED_DICT.dict_content, + concatenation, tuple(wrong_size_lst), 300*_1K, 5) + + # correct size list + _zstd.finalize_dict(TRAINED_DICT.dict_content, + concatenation, tuple(correct_size_lst), 300*_1K, 5) + + def test_as_prefix(self): + # V1 + V1 = THIS_FILE_BYTES + zd = ZstdDict(V1, is_raw=True) + + # V2 + mid = len(V1) // 2 + V2 = V1[:mid] + \ + (b'a' if V1[mid] != int.from_bytes(b'a') else b'b') + \ + V1[mid+1:] + + # compress + dat = compress(V2, zstd_dict=zd.as_prefix) + self.assertEqual(get_frame_info(dat).dictionary_id, 0) + + # decompress + self.assertEqual(decompress(dat, zd.as_prefix), V2) + + # use wrong prefix + zd2 = ZstdDict(SAMPLES[0], is_raw=True) + try: + decompressed = decompress(dat, zd2.as_prefix) + except ZstdError: # expected + pass + else: + self.assertNotEqual(decompressed, V2) + + # read only attribute + with self.assertRaises(AttributeError): + zd.as_prefix = b'1234' + + def test_as_digested_dict(self): + zd = TRAINED_DICT + + # test .as_digested_dict + dat = compress(SAMPLES[0], zstd_dict=zd.as_digested_dict) + self.assertEqual(decompress(dat, zd.as_digested_dict), SAMPLES[0]) + with self.assertRaises(AttributeError): + zd.as_digested_dict = b'1234' + + # test .as_undigested_dict + dat = compress(SAMPLES[0], zstd_dict=zd.as_undigested_dict) + self.assertEqual(decompress(dat, zd.as_undigested_dict), SAMPLES[0]) + with self.assertRaises(AttributeError): + zd.as_undigested_dict = b'1234' + + def test_advanced_compression_parameters(self): + options = {CompressionParameter.compression_level: 6, + CompressionParameter.window_log: 20, + CompressionParameter.enable_long_distance_matching: 1} + + # automatically select + dat = compress(SAMPLES[0], options=options, zstd_dict=TRAINED_DICT) + self.assertEqual(decompress(dat, TRAINED_DICT), SAMPLES[0]) + + # explicitly select + dat = compress(SAMPLES[0], options=options, zstd_dict=TRAINED_DICT.as_digested_dict) + self.assertEqual(decompress(dat, TRAINED_DICT), SAMPLES[0]) + + def test_len(self): + self.assertEqual(len(TRAINED_DICT), len(TRAINED_DICT.dict_content)) + self.assertIn(str(len(TRAINED_DICT)), str(TRAINED_DICT)) + +class FileTestCase(unittest.TestCase): + def setUp(self): + self.DECOMPRESSED_42 = b'a'*42 + self.FRAME_42 = compress(self.DECOMPRESSED_42) + + def test_init(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + pass + with ZstdFile(io.BytesIO(), "w") as f: + pass + with ZstdFile(io.BytesIO(), "x") as f: + pass + with ZstdFile(io.BytesIO(), "a") as f: + pass + + with ZstdFile(io.BytesIO(), "w", level=12) as f: + pass + with ZstdFile(io.BytesIO(), "w", options={CompressionParameter.checksum_flag:1}) as f: + pass + with ZstdFile(io.BytesIO(), "w", options={}) as f: + pass + with ZstdFile(io.BytesIO(), "w", level=20, zstd_dict=TRAINED_DICT) as f: + pass + + with ZstdFile(io.BytesIO(), "r", options={DecompressionParameter.window_log_max:25}) as f: + pass + with ZstdFile(io.BytesIO(), "r", options={}, zstd_dict=TRAINED_DICT) as f: + pass + + def test_init_with_PathLike_filename(self): + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + with ZstdFile(filename, "a") as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + with ZstdFile(filename) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + + with ZstdFile(filename, "a") as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + with ZstdFile(filename) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB * 2) + + os.remove(filename) + + def test_init_with_filename(self): + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + with ZstdFile(filename) as f: + pass + with ZstdFile(filename, "w") as f: + pass + with ZstdFile(filename, "a") as f: + pass + + os.remove(filename) + + def test_init_mode(self): + bi = io.BytesIO() + + with ZstdFile(bi, "r"): + pass + with ZstdFile(bi, "rb"): + pass + with ZstdFile(bi, "w"): + pass + with ZstdFile(bi, "wb"): + pass + with ZstdFile(bi, "a"): + pass + with ZstdFile(bi, "ab"): + pass + + def test_init_with_x_mode(self): + with tempfile.NamedTemporaryFile() as tmp_f: + filename = pathlib.Path(tmp_f.name) + + for mode in ("x", "xb"): + with ZstdFile(filename, mode): + pass + with self.assertRaises(FileExistsError): + with ZstdFile(filename, mode): + pass + os.remove(filename) + + def test_init_bad_mode(self): + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), (3, "x")) + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "xt") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "x+") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rx") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "wx") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rt") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r+") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "wt") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "w+") + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rw") + + with self.assertRaisesRegex(TypeError, + r"not be a CompressionParameter"): + ZstdFile(io.BytesIO(), 'rb', + options={CompressionParameter.compression_level:5}) + with self.assertRaisesRegex(TypeError, + r"not be a DecompressionParameter"): + ZstdFile(io.BytesIO(), 'wb', + options={DecompressionParameter.window_log_max:21}) + + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r", level=12) + + def test_init_bad_check(self): + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(), "w", level='asd') + # CHECK_UNKNOWN and anything above CHECK_ID_MAX should be invalid. + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(), "w", options={999:9999}) + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(), "w", options={CompressionParameter.window_log:99}) + + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r", options=33) + + with self.assertRaises(OverflowError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), + options={DecompressionParameter.window_log_max:2**31}) + + with self.assertRaises(ValueError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), + options={444:333}) + + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), zstd_dict={1:2}) + + with self.assertRaises(TypeError): + ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), zstd_dict=b'dict123456') + + def test_init_close_fp(self): + # get a temp file name + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + tmp_f.write(DAT_130K_C) + filename = tmp_f.name + + with self.assertRaises(TypeError): + ZstdFile(filename, options={'a':'b'}) + + # for PyPy + gc.collect() + + os.remove(filename) + + def test_close(self): + with io.BytesIO(COMPRESSED_100_PLUS_32KB) as src: + f = ZstdFile(src) + f.close() + # ZstdFile.close() should not close the underlying file object. + self.assertFalse(src.closed) + # Try closing an already-closed ZstdFile. + f.close() + self.assertFalse(src.closed) + + # Test with a real file on disk, opened directly by ZstdFile. + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + f = ZstdFile(filename) + fp = f._fp + f.close() + # Here, ZstdFile.close() *should* close the underlying file object. + self.assertTrue(fp.closed) + # Try closing an already-closed ZstdFile. + f.close() + + os.remove(filename) + + def test_closed(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertFalse(f.closed) + f.read() + self.assertFalse(f.closed) + finally: + f.close() + self.assertTrue(f.closed) + + f = ZstdFile(io.BytesIO(), "w") + try: + self.assertFalse(f.closed) + finally: + f.close() + self.assertTrue(f.closed) + + def test_fileno(self): + # 1 + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertRaises(io.UnsupportedOperation, f.fileno) + finally: + f.close() + self.assertRaises(ValueError, f.fileno) + + # 2 + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + f = ZstdFile(filename) + try: + self.assertEqual(f.fileno(), f._fp.fileno()) + self.assertIsInstance(f.fileno(), int) + finally: + f.close() + self.assertRaises(ValueError, f.fileno) + + os.remove(filename) + + # 3, no .fileno() method + class C: + def read(self, size=-1): + return b'123' + with ZstdFile(C(), 'rb') as f: + with self.assertRaisesRegex(AttributeError, r'fileno'): + f.fileno() + + def test_name(self): + # 1 + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + with self.assertRaises(AttributeError): + f.name + finally: + f.close() + with self.assertRaises(ValueError): + f.name + + # 2 + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + filename = pathlib.Path(tmp_f.name) + + f = ZstdFile(filename) + try: + self.assertEqual(f.name, f._fp.name) + self.assertIsInstance(f.name, str) + finally: + f.close() + with self.assertRaises(ValueError): + f.name + + os.remove(filename) + + # 3, no .filename property + class C: + def read(self, size=-1): + return b'123' + with ZstdFile(C(), 'rb') as f: + with self.assertRaisesRegex(AttributeError, r'name'): + f.name + + def test_seekable(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertTrue(f.seekable()) + f.read() + self.assertTrue(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + f = ZstdFile(io.BytesIO(), "w") + try: + self.assertFalse(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + src = io.BytesIO(COMPRESSED_100_PLUS_32KB) + src.seekable = lambda: False + f = ZstdFile(src) + try: + self.assertFalse(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + def test_readable(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertTrue(f.readable()) + f.read() + self.assertTrue(f.readable()) + finally: + f.close() + self.assertRaises(ValueError, f.readable) + + f = ZstdFile(io.BytesIO(), "w") + try: + self.assertFalse(f.readable()) + finally: + f.close() + self.assertRaises(ValueError, f.readable) + + def test_writable(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + try: + self.assertFalse(f.writable()) + f.read() + self.assertFalse(f.writable()) + finally: + f.close() + self.assertRaises(ValueError, f.writable) + + f = ZstdFile(io.BytesIO(), "w") + try: + self.assertTrue(f.writable()) + finally: + f.close() + self.assertRaises(ValueError, f.writable) + + def test_read_0(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + self.assertEqual(f.read(0), b"") + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), + options={DecompressionParameter.window_log_max:20}) as f: + self.assertEqual(f.read(0), b"") + + # empty file + with ZstdFile(io.BytesIO(b'')) as f: + self.assertEqual(f.read(0), b"") + with self.assertRaises(EOFError): + f.read(10) + + with ZstdFile(io.BytesIO(b'')) as f: + with self.assertRaises(EOFError): + f.read(10) + + def test_read_10(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + chunks = [] + while True: + result = f.read(10) + if not result: + break + self.assertLessEqual(len(result), 10) + chunks.append(result) + self.assertEqual(b"".join(chunks), DECOMPRESSED_100_PLUS_32KB) + + def test_read_multistream(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 5)) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB * 5) + + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB + SKIPPABLE_FRAME)) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB + COMPRESSED_DAT)) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB + DECOMPRESSED_DAT) + + def test_read_incomplete(self): + with ZstdFile(io.BytesIO(DAT_130K_C[:-200])) as f: + self.assertRaises(EOFError, f.read) + + # Trailing data isn't a valid compressed stream + with ZstdFile(io.BytesIO(self.FRAME_42 + b'12345')) as f: + self.assertRaises(ZstdError, f.read) + + with ZstdFile(io.BytesIO(SKIPPABLE_FRAME + b'12345')) as f: + self.assertRaises(ZstdError, f.read) + + def test_read_truncated(self): + # Drop stream epilogue: 4 bytes checksum + truncated = DAT_130K_C[:-4] + with ZstdFile(io.BytesIO(truncated)) as f: + self.assertRaises(EOFError, f.read) + + with ZstdFile(io.BytesIO(truncated)) as f: + # this is an important test, make sure it doesn't raise EOFError. + self.assertEqual(f.read(130*_1K), DAT_130K_D) + with self.assertRaises(EOFError): + f.read(1) + + # Incomplete header + for i in range(1, 20): + with ZstdFile(io.BytesIO(truncated[:i])) as f: + self.assertRaises(EOFError, f.read, 1) + + def test_read_bad_args(self): + f = ZstdFile(io.BytesIO(COMPRESSED_DAT)) + f.close() + self.assertRaises(ValueError, f.read) + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(ValueError, f.read) + with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f: + self.assertRaises(TypeError, f.read, float()) + + def test_read_bad_data(self): + with ZstdFile(io.BytesIO(COMPRESSED_BOGUS)) as f: + self.assertRaises(ZstdError, f.read) + + def test_read_exception(self): + class C: + def read(self, size=-1): + raise OSError + with ZstdFile(C()) as f: + with self.assertRaises(OSError): + f.read(10) + + def test_read1(self): + with ZstdFile(io.BytesIO(DAT_130K_C)) as f: + blocks = [] + while True: + result = f.read1() + if not result: + break + blocks.append(result) + self.assertEqual(b"".join(blocks), DAT_130K_D) + self.assertEqual(f.read1(), b"") + + def test_read1_0(self): + with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f: + self.assertEqual(f.read1(0), b"") + + def test_read1_10(self): + with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f: + blocks = [] + while True: + result = f.read1(10) + if not result: + break + blocks.append(result) + self.assertEqual(b"".join(blocks), DECOMPRESSED_DAT) + self.assertEqual(f.read1(), b"") + + def test_read1_multistream(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 5)) as f: + blocks = [] + while True: + result = f.read1() + if not result: + break + blocks.append(result) + self.assertEqual(b"".join(blocks), DECOMPRESSED_100_PLUS_32KB * 5) + self.assertEqual(f.read1(), b"") + + def test_read1_bad_args(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + f.close() + self.assertRaises(ValueError, f.read1) + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(ValueError, f.read1) + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + self.assertRaises(TypeError, f.read1, None) + + def test_readinto(self): + arr = array.array("I", range(100)) + self.assertEqual(len(arr), 100) + self.assertEqual(len(arr) * arr.itemsize, 400) + ba = bytearray(300) + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + # 0 length output buffer + self.assertEqual(f.readinto(ba[0:0]), 0) + + # use correct length for buffer protocol object + self.assertEqual(f.readinto(arr), 400) + self.assertEqual(arr.tobytes(), DECOMPRESSED_100_PLUS_32KB[:400]) + + # normal readinto + self.assertEqual(f.readinto(ba), 300) + self.assertEqual(ba, DECOMPRESSED_100_PLUS_32KB[400:700]) + + def test_peek(self): + with ZstdFile(io.BytesIO(DAT_130K_C)) as f: + result = f.peek() + self.assertGreater(len(result), 0) + self.assertTrue(DAT_130K_D.startswith(result)) + self.assertEqual(f.read(), DAT_130K_D) + with ZstdFile(io.BytesIO(DAT_130K_C)) as f: + result = f.peek(10) + self.assertGreater(len(result), 0) + self.assertTrue(DAT_130K_D.startswith(result)) + self.assertEqual(f.read(), DAT_130K_D) + + def test_peek_bad_args(self): + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(ValueError, f.peek) + + def test_iterator(self): + with io.BytesIO(THIS_FILE_BYTES) as f: + lines = f.readlines() + compressed = compress(THIS_FILE_BYTES) + + # iter + with ZstdFile(io.BytesIO(compressed)) as f: + self.assertListEqual(list(iter(f)), lines) + + # readline + with ZstdFile(io.BytesIO(compressed)) as f: + for line in lines: + self.assertEqual(f.readline(), line) + self.assertEqual(f.readline(), b'') + self.assertEqual(f.readline(), b'') + + # readlines + with ZstdFile(io.BytesIO(compressed)) as f: + self.assertListEqual(f.readlines(), lines) + + def test_decompress_limited(self): + _ZSTD_DStreamInSize = 128*_1K + 3 + + bomb = compress(b'\0' * int(2e6), level=10) + self.assertLess(len(bomb), _ZSTD_DStreamInSize) + + decomp = ZstdFile(io.BytesIO(bomb)) + self.assertEqual(decomp.read(1), b'\0') + + # BufferedReader uses 128 KiB buffer in __init__.py + max_decomp = 128*_1K + self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, + "Excessive amount of data was decompressed") + + def test_write(self): + raw_data = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6] + with io.BytesIO() as dst: + with ZstdFile(dst, "w") as f: + f.write(raw_data) + + comp = ZstdCompressor() + expected = comp.compress(raw_data) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + with io.BytesIO() as dst: + with ZstdFile(dst, "w", level=12) as f: + f.write(raw_data) + + comp = ZstdCompressor(12) + expected = comp.compress(raw_data) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + with io.BytesIO() as dst: + with ZstdFile(dst, "w", options={CompressionParameter.checksum_flag:1}) as f: + f.write(raw_data) + + comp = ZstdCompressor(options={CompressionParameter.checksum_flag:1}) + expected = comp.compress(raw_data) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + with io.BytesIO() as dst: + options = {CompressionParameter.compression_level:-5, + CompressionParameter.checksum_flag:1} + with ZstdFile(dst, "w", + options=options) as f: + f.write(raw_data) + + comp = ZstdCompressor(options=options) + expected = comp.compress(raw_data) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + def test_write_empty_frame(self): + # .FLUSH_FRAME generates an empty content frame + c = ZstdCompressor() + self.assertNotEqual(c.flush(c.FLUSH_FRAME), b'') + self.assertNotEqual(c.flush(c.FLUSH_FRAME), b'') + + # don't generate empty content frame + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + pass + self.assertEqual(bo.getvalue(), b'') + + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.flush(f.FLUSH_FRAME) + self.assertEqual(bo.getvalue(), b'') + + # if .write(b''), generate empty content frame + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.write(b'') + self.assertNotEqual(bo.getvalue(), b'') + + # has an empty content frame + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.flush(f.FLUSH_BLOCK) + self.assertNotEqual(bo.getvalue(), b'') + + def test_write_empty_block(self): + # If no internal data, .FLUSH_BLOCK return b''. + c = ZstdCompressor() + self.assertEqual(c.flush(c.FLUSH_BLOCK), b'') + self.assertNotEqual(c.compress(b'123', c.FLUSH_BLOCK), + b'') + self.assertEqual(c.flush(c.FLUSH_BLOCK), b'') + self.assertEqual(c.compress(b''), b'') + self.assertEqual(c.compress(b''), b'') + self.assertEqual(c.flush(c.FLUSH_BLOCK), b'') + + # mode = .last_mode + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.write(b'123') + f.flush(f.FLUSH_BLOCK) + fp_pos = f._fp.tell() + self.assertNotEqual(fp_pos, 0) + f.flush(f.FLUSH_BLOCK) + self.assertEqual(f._fp.tell(), fp_pos) + + # mode != .last_mode + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.flush(f.FLUSH_BLOCK) + self.assertEqual(f._fp.tell(), 0) + f.write(b'') + f.flush(f.FLUSH_BLOCK) + self.assertEqual(f._fp.tell(), 0) + + def test_write_101(self): + with io.BytesIO() as dst: + with ZstdFile(dst, "w") as f: + for start in range(0, len(THIS_FILE_BYTES), 101): + f.write(THIS_FILE_BYTES[start:start+101]) + + comp = ZstdCompressor() + expected = comp.compress(THIS_FILE_BYTES) + comp.flush() + self.assertEqual(dst.getvalue(), expected) + + def test_write_append(self): + def comp(data): + comp = ZstdCompressor() + return comp.compress(data) + comp.flush() + + part1 = THIS_FILE_BYTES[:_1K] + part2 = THIS_FILE_BYTES[_1K:1536] + part3 = THIS_FILE_BYTES[1536:] + expected = b"".join(comp(x) for x in (part1, part2, part3)) + with io.BytesIO() as dst: + with ZstdFile(dst, "w") as f: + f.write(part1) + with ZstdFile(dst, "a") as f: + f.write(part2) + with ZstdFile(dst, "a") as f: + f.write(part3) + self.assertEqual(dst.getvalue(), expected) + + def test_write_bad_args(self): + f = ZstdFile(io.BytesIO(), "w") + f.close() + self.assertRaises(ValueError, f.write, b"foo") + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r") as f: + self.assertRaises(ValueError, f.write, b"bar") + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(TypeError, f.write, None) + self.assertRaises(TypeError, f.write, "text") + self.assertRaises(TypeError, f.write, 789) + + def test_writelines(self): + def comp(data): + comp = ZstdCompressor() + return comp.compress(data) + comp.flush() + + with io.BytesIO(THIS_FILE_BYTES) as f: + lines = f.readlines() + with io.BytesIO() as dst: + with ZstdFile(dst, "w") as f: + f.writelines(lines) + expected = comp(THIS_FILE_BYTES) + self.assertEqual(dst.getvalue(), expected) + + def test_seek_forward(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(555) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[555:]) + + def test_seek_forward_across_streams(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 2)) as f: + f.seek(len(DECOMPRESSED_100_PLUS_32KB) + 123) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[123:]) + + def test_seek_forward_relative_to_current(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.read(100) + f.seek(1236, 1) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[1336:]) + + def test_seek_forward_relative_to_end(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(-555, 2) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[-555:]) + + def test_seek_backward(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.read(1001) + f.seek(211) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[211:]) + + def test_seek_backward_across_streams(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 2)) as f: + f.read(len(DECOMPRESSED_100_PLUS_32KB) + 333) + f.seek(737) + self.assertEqual(f.read(), + DECOMPRESSED_100_PLUS_32KB[737:] + DECOMPRESSED_100_PLUS_32KB) + + def test_seek_backward_relative_to_end(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(-150, 2) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[-150:]) + + def test_seek_past_end(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(len(DECOMPRESSED_100_PLUS_32KB) + 9001) + self.assertEqual(f.tell(), len(DECOMPRESSED_100_PLUS_32KB)) + self.assertEqual(f.read(), b"") + + def test_seek_past_start(self): + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + f.seek(-88) + self.assertEqual(f.tell(), 0) + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + + def test_seek_bad_args(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + f.close() + self.assertRaises(ValueError, f.seek, 0) + with ZstdFile(io.BytesIO(), "w") as f: + self.assertRaises(ValueError, f.seek, 0) + with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f: + self.assertRaises(ValueError, f.seek, 0, 3) + # io.BufferedReader raises TypeError instead of ValueError + self.assertRaises((TypeError, ValueError), f.seek, 9, ()) + self.assertRaises(TypeError, f.seek, None) + self.assertRaises(TypeError, f.seek, b"derp") + + def test_seek_not_seekable(self): + class C(io.BytesIO): + def seekable(self): + return False + obj = C(COMPRESSED_100_PLUS_32KB) + with ZstdFile(obj, 'r') as f: + d = f.read(1) + self.assertFalse(f.seekable()) + with self.assertRaisesRegex(io.UnsupportedOperation, + 'File or stream is not seekable'): + f.seek(0) + d += f.read() + self.assertEqual(d, DECOMPRESSED_100_PLUS_32KB) + + def test_tell(self): + with ZstdFile(io.BytesIO(DAT_130K_C)) as f: + pos = 0 + while True: + self.assertEqual(f.tell(), pos) + result = f.read(random.randint(171, 189)) + if not result: + break + pos += len(result) + self.assertEqual(f.tell(), len(DAT_130K_D)) + with ZstdFile(io.BytesIO(), "w") as f: + for pos in range(0, len(DAT_130K_D), 143): + self.assertEqual(f.tell(), pos) + f.write(DAT_130K_D[pos:pos+143]) + self.assertEqual(f.tell(), len(DAT_130K_D)) + + def test_tell_bad_args(self): + f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) + f.close() + self.assertRaises(ValueError, f.tell) + + def test_file_dict(self): + # default + bi = io.BytesIO() + with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with ZstdFile(bi, zstd_dict=TRAINED_DICT) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + # .as_(un)digested_dict + bi = io.BytesIO() + with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT.as_digested_dict) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with ZstdFile(bi, zstd_dict=TRAINED_DICT.as_undigested_dict) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + def test_file_prefix(self): + bi = io.BytesIO() + with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT.as_prefix) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with ZstdFile(bi, zstd_dict=TRAINED_DICT.as_prefix) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + def test_UnsupportedOperation(self): + # 1 + with ZstdFile(io.BytesIO(), 'r') as f: + with self.assertRaises(io.UnsupportedOperation): + f.write(b'1234') + + # 2 + class T: + def read(self, size): + return b'a' * size + + with self.assertRaises(TypeError): # on creation + with ZstdFile(T(), 'w') as f: + pass + + # 3 + with ZstdFile(io.BytesIO(), 'w') as f: + with self.assertRaises(io.UnsupportedOperation): + f.read(100) + with self.assertRaises(io.UnsupportedOperation): + f.seek(100) + self.assertEqual(f.closed, True) + with self.assertRaises(ValueError): + f.readable() + with self.assertRaises(ValueError): + f.tell() + with self.assertRaises(ValueError): + f.read(100) + + def test_read_readinto_readinto1(self): + lst = [] + with ZstdFile(io.BytesIO(COMPRESSED_THIS_FILE*5)) as f: + while True: + method = random.randint(0, 2) + size = random.randint(0, 300) + + if method == 0: + dat = f.read(size) + if not dat and size: + break + lst.append(dat) + elif method == 1: + ba = bytearray(size) + read_size = f.readinto(ba) + if read_size == 0 and size: + break + lst.append(bytes(ba[:read_size])) + elif method == 2: + ba = bytearray(size) + read_size = f.readinto1(ba) + if read_size == 0 and size: + break + lst.append(bytes(ba[:read_size])) + self.assertEqual(b''.join(lst), THIS_FILE_BYTES*5) + + def test_zstdfile_flush(self): + # closed + f = ZstdFile(io.BytesIO(), 'w') + f.close() + with self.assertRaises(ValueError): + f.flush() + + # read + with ZstdFile(io.BytesIO(), 'r') as f: + # does nothing for read-only stream + f.flush() + + # write + DAT = b'abcd' + bi = io.BytesIO() + with ZstdFile(bi, 'w') as f: + self.assertEqual(f.write(DAT), len(DAT)) + self.assertEqual(f.tell(), len(DAT)) + self.assertEqual(bi.tell(), 0) # not enough for a block + + self.assertEqual(f.flush(), None) + self.assertEqual(f.tell(), len(DAT)) + self.assertGreater(bi.tell(), 0) # flushed + + # write, no .flush() method + class C: + def write(self, b): + return len(b) + with ZstdFile(C(), 'w') as f: + self.assertEqual(f.write(DAT), len(DAT)) + self.assertEqual(f.tell(), len(DAT)) + + self.assertEqual(f.flush(), None) + self.assertEqual(f.tell(), len(DAT)) + + def test_zstdfile_flush_mode(self): + self.assertEqual(ZstdFile.FLUSH_BLOCK, ZstdCompressor.FLUSH_BLOCK) + self.assertEqual(ZstdFile.FLUSH_FRAME, ZstdCompressor.FLUSH_FRAME) + with self.assertRaises(AttributeError): + ZstdFile.CONTINUE + + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + # flush block + self.assertEqual(f.write(b'123'), 3) + self.assertIsNone(f.flush(f.FLUSH_BLOCK)) + p1 = bo.tell() + # mode == .last_mode, should return + self.assertIsNone(f.flush()) + p2 = bo.tell() + self.assertEqual(p1, p2) + # flush frame + self.assertEqual(f.write(b'456'), 3) + self.assertIsNone(f.flush(mode=f.FLUSH_FRAME)) + # flush frame + self.assertEqual(f.write(b'789'), 3) + self.assertIsNone(f.flush(f.FLUSH_FRAME)) + p1 = bo.tell() + # mode == .last_mode, should return + self.assertIsNone(f.flush(f.FLUSH_FRAME)) + p2 = bo.tell() + self.assertEqual(p1, p2) + self.assertEqual(decompress(bo.getvalue()), b'123456789') + + bo = io.BytesIO() + with ZstdFile(bo, 'w') as f: + f.write(b'123') + with self.assertRaisesRegex(ValueError, r'\.FLUSH_.*?\.FLUSH_'): + f.flush(ZstdCompressor.CONTINUE) + with self.assertRaises(ValueError): + f.flush(-1) + with self.assertRaises(ValueError): + f.flush(123456) + with self.assertRaises(TypeError): + f.flush(node=ZstdCompressor.CONTINUE) + with self.assertRaises((TypeError, ValueError)): + f.flush('FLUSH_FRAME') + with self.assertRaises(TypeError): + f.flush(b'456', f.FLUSH_BLOCK) + + def test_zstdfile_truncate(self): + with ZstdFile(io.BytesIO(), 'w') as f: + with self.assertRaises(io.UnsupportedOperation): + f.truncate(200) + + def test_zstdfile_iter_issue45475(self): + lines = [l for l in ZstdFile(io.BytesIO(COMPRESSED_THIS_FILE))] + self.assertGreater(len(lines), 0) + + def test_append_new_file(self): + with tempfile.NamedTemporaryFile(delete=True) as tmp_f: + filename = tmp_f.name + + with ZstdFile(filename, 'a') as f: + pass + self.assertTrue(os.path.isfile(filename)) + + os.remove(filename) + +class OpenTestCase(unittest.TestCase): + + def test_binary_modes(self): + with open(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rb") as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + with io.BytesIO() as bio: + with open(bio, "wb") as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + file_data = decompress(bio.getvalue()) + self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB) + with open(bio, "ab") as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + file_data = decompress(bio.getvalue()) + self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB * 2) + + def test_text_modes(self): + # empty input + with self.assertRaises(EOFError): + with open(io.BytesIO(b''), "rt", encoding="utf-8", newline='\n') as reader: + for _ in reader: + pass + + # read + uncompressed = THIS_FILE_STR.replace(os.linesep, "\n") + with open(io.BytesIO(COMPRESSED_THIS_FILE), "rt", encoding="utf-8") as f: + self.assertEqual(f.read(), uncompressed) + + with io.BytesIO() as bio: + # write + with open(bio, "wt", encoding="utf-8") as f: + f.write(uncompressed) + file_data = decompress(bio.getvalue()).decode("utf-8") + self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed) + # append + with open(bio, "at", encoding="utf-8") as f: + f.write(uncompressed) + file_data = decompress(bio.getvalue()).decode("utf-8") + self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed * 2) + + def test_bad_params(self): + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + TESTFN = pathlib.Path(tmp_f.name) + + with self.assertRaises(ValueError): + open(TESTFN, "") + with self.assertRaises(ValueError): + open(TESTFN, "rbt") + with self.assertRaises(ValueError): + open(TESTFN, "rb", encoding="utf-8") + with self.assertRaises(ValueError): + open(TESTFN, "rb", errors="ignore") + with self.assertRaises(ValueError): + open(TESTFN, "rb", newline="\n") + + os.remove(TESTFN) + + def test_option(self): + options = {DecompressionParameter.window_log_max:25} + with open(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rb", options=options) as f: + self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB) + + options = {CompressionParameter.compression_level:12} + with io.BytesIO() as bio: + with open(bio, "wb", options=options) as f: + f.write(DECOMPRESSED_100_PLUS_32KB) + file_data = decompress(bio.getvalue()) + self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB) + + def test_encoding(self): + uncompressed = THIS_FILE_STR.replace(os.linesep, "\n") + + with io.BytesIO() as bio: + with open(bio, "wt", encoding="utf-16-le") as f: + f.write(uncompressed) + file_data = decompress(bio.getvalue()).decode("utf-16-le") + self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed) + bio.seek(0) + with open(bio, "rt", encoding="utf-16-le") as f: + self.assertEqual(f.read().replace(os.linesep, "\n"), uncompressed) + + def test_encoding_error_handler(self): + with io.BytesIO(compress(b"foo\xffbar")) as bio: + with open(bio, "rt", encoding="ascii", errors="ignore") as f: + self.assertEqual(f.read(), "foobar") + + def test_newline(self): + # Test with explicit newline (universal newline mode disabled). + text = THIS_FILE_STR.replace(os.linesep, "\n") + with io.BytesIO() as bio: + with open(bio, "wt", encoding="utf-8", newline="\n") as f: + f.write(text) + bio.seek(0) + with open(bio, "rt", encoding="utf-8", newline="\r") as f: + self.assertEqual(f.readlines(), [text]) + + def test_x_mode(self): + with tempfile.NamedTemporaryFile(delete=False) as tmp_f: + TESTFN = pathlib.Path(tmp_f.name) + + for mode in ("x", "xb", "xt"): + os.remove(TESTFN) + + if mode == "xt": + encoding = "utf-8" + else: + encoding = None + with open(TESTFN, mode, encoding=encoding): + pass + with self.assertRaises(FileExistsError): + with open(TESTFN, mode): + pass + + os.remove(TESTFN) + + def test_open_dict(self): + # default + bi = io.BytesIO() + with open(bi, 'w', zstd_dict=TRAINED_DICT) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with open(bi, zstd_dict=TRAINED_DICT) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + # .as_(un)digested_dict + bi = io.BytesIO() + with open(bi, 'w', zstd_dict=TRAINED_DICT.as_digested_dict) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with open(bi, zstd_dict=TRAINED_DICT.as_undigested_dict) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + # invalid dictionary + bi = io.BytesIO() + with self.assertRaisesRegex(TypeError, 'zstd_dict'): + open(bi, 'w', zstd_dict={1:2, 2:3}) + + with self.assertRaisesRegex(TypeError, 'zstd_dict'): + open(bi, 'w', zstd_dict=b'1234567890') + + def test_open_prefix(self): + bi = io.BytesIO() + with open(bi, 'w', zstd_dict=TRAINED_DICT.as_prefix) as f: + f.write(SAMPLES[0]) + bi.seek(0) + with open(bi, zstd_dict=TRAINED_DICT.as_prefix) as f: + dat = f.read() + self.assertEqual(dat, SAMPLES[0]) + + def test_buffer_protocol(self): + # don't use len() for buffer protocol objects + arr = array.array("i", range(1000)) + LENGTH = len(arr) * arr.itemsize + + with open(io.BytesIO(), "wb") as f: + self.assertEqual(f.write(arr), LENGTH) + self.assertEqual(f.tell(), LENGTH) + +class FreeThreadingMethodTests(unittest.TestCase): + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_compress_locking(self): + input = b'a'* (16*_1K) + num_threads = 8 + + # gh-136394: the first output of .compress() includes the frame header + # we run the first .compress() call outside of the threaded portion + # to make the test order-independent + + comp = ZstdCompressor() + parts = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] + for _ in range(num_threads): + res = comp.compress(input, ZstdCompressor.FLUSH_BLOCK) + if res: + parts.append(res) + rest1 = comp.flush() + expected = b''.join(parts) + rest1 + + comp = ZstdCompressor() + output = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] + def run_method(method, input_data, output_data): + res = method(input_data, ZstdCompressor.FLUSH_BLOCK) + if res: + output_data.append(res) + threads = [] + + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(comp.compress, input, output)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + rest2 = comp.flush() + self.assertEqual(rest1, rest2) + actual = b''.join(output) + rest2 + self.assertEqual(expected, actual) + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_decompress_locking(self): + input = compress(b'a'* (16*_1K)) + num_threads = 8 + # to ensure we decompress over multiple calls, set maxsize + window_size = _1K * 16//num_threads + + decomp = ZstdDecompressor() + parts = [] + for _ in range(num_threads): + res = decomp.decompress(input, window_size) + if res: + parts.append(res) + expected = b''.join(parts) + + comp = ZstdDecompressor() + output = [] + def run_method(method, input_data, output_data): + res = method(input_data, window_size) + if res: + output_data.append(res) + threads = [] + + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(comp.decompress, input, output)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + actual = b''.join(output) + self.assertEqual(expected, actual) + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_compress_shared_dict(self): + num_threads = 8 + + def run_method(b): + level = threading.get_ident() % 4 + # sync threads to increase chance of contention on + # capsule storing dictionary levels + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_decompress_shared_dict(self): + num_threads = 8 + + def run_method(b): + # sync threads to increase chance of contention on + # decompression dictionary + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/tokenizedata/__init__.py b/Lib/test/tokenizedata/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/tokenizedata/bad_coding.py b/Lib/test/tokenizedata/bad_coding.py new file mode 100644 index 00000000000..971b0a8f3d6 --- /dev/null +++ b/Lib/test/tokenizedata/bad_coding.py @@ -0,0 +1 @@ +# -*- coding: uft-8 -*- diff --git a/Lib/test/tokenizedata/bad_coding2.py b/Lib/test/tokenizedata/bad_coding2.py new file mode 100644 index 00000000000..bb2bb7e1e75 --- /dev/null +++ b/Lib/test/tokenizedata/bad_coding2.py @@ -0,0 +1,2 @@ +#coding: utf8 +print('我') diff --git a/Lib/test/tokenizedata/badsyntax_3131.py b/Lib/test/tokenizedata/badsyntax_3131.py new file mode 100644 index 00000000000..901d3744ca0 --- /dev/null +++ b/Lib/test/tokenizedata/badsyntax_3131.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +€ = 2 diff --git a/Lib/test/tokenizedata/badsyntax_pep3120.py b/Lib/test/tokenizedata/badsyntax_pep3120.py new file mode 100644 index 00000000000..d14b4c96ede --- /dev/null +++ b/Lib/test/tokenizedata/badsyntax_pep3120.py @@ -0,0 +1 @@ +print("bse") diff --git a/Lib/test/tokenizedata/coding20731.py b/Lib/test/tokenizedata/coding20731.py new file mode 100644 index 00000000000..b0e227ad110 --- /dev/null +++ b/Lib/test/tokenizedata/coding20731.py @@ -0,0 +1,4 @@ +#coding:latin1 + + + diff --git a/Lib/test/tokenizedata/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt new file mode 100644 index 00000000000..1b5335b64ed --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt @@ -0,0 +1,13 @@ +# -*- coding: latin1 -*- +# IMPORTANT: this file has the utf-8 BOM signature '\xef\xbb\xbf' +# at the start of it. Make sure this is preserved if any changes +# are made! Also note that the coding cookie above conflicts with +# the presence of a utf-8 BOM signature -- this is intended. + +# Arbitrary encoded utf-8 text (stolen from test_doctest2.py). +x = 'ЉЊЈЁЂ' +def y(): + """ + And again in a comment. ЉЊЈЁЂ + """ + pass diff --git a/Lib/test/tokenizedata/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt b/Lib/test/tokenizedata/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt new file mode 100644 index 00000000000..23fd2168ae5 --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt @@ -0,0 +1,11 @@ +# IMPORTANT: this file has the utf-8 BOM signature '\xef\xbb\xbf' +# at the start of it. Make sure this is preserved if any changes +# are made! + +# Arbitrary encoded utf-8 text (stolen from test_doctest2.py). +x = 'ЉЊЈЁЂ' +def y(): + """ + And again in a comment. ЉЊЈЁЂ + """ + pass diff --git a/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt new file mode 100644 index 00000000000..04561e48472 --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# IMPORTANT: unlike the other test_tokenize-*.txt files, this file +# does NOT have the utf-8 BOM signature '\xef\xbb\xbf' at the start +# of it. Make sure this is not added inadvertently by your editor +# if any changes are made to this file! + +# Arbitrary encoded utf-8 text (stolen from test_doctest2.py). +x = 'ЉЊЈЁЂ' +def y(): + """ + And again in a comment. ЉЊЈЁЂ + """ + pass diff --git a/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt new file mode 100644 index 00000000000..4b20ff6ad6d --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# IMPORTANT: this file has the utf-8 BOM signature '\xef\xbb\xbf' +# at the start of it. Make sure this is preserved if any changes +# are made! + +# Arbitrary encoded utf-8 text (stolen from test_doctest2.py). +x = 'ЉЊЈЁЂ' +def y(): + """ + And again in a comment. ЉЊЈЁЂ + """ + pass diff --git a/Lib/test/tokenizedata/tokenize_tests.txt b/Lib/test/tokenizedata/tokenize_tests.txt new file mode 100644 index 00000000000..c4f5a58a946 --- /dev/null +++ b/Lib/test/tokenizedata/tokenize_tests.txt @@ -0,0 +1,189 @@ +# Tests for the 'tokenize' module. +# Large bits stolen from test_grammar.py. + +# Comments +"#" +#' +#" +#\ + # + # abc +'''# +#''' + +x = 1 # + +# Balancing continuation + +a = (3, 4, + 5, 6) +y = [3, 4, + 5] +z = {'a':5, + 'b':6} +x = (len(repr(y)) + 5*x - a[ + 3 ] + - x + len({ + } + ) + ) + +# Backslash means line continuation: +x = 1 \ ++ 1 + +# Backslash does not means continuation in comments :\ +x = 0 + +# Ordinary integers +0xff != 255 +0o377 != 255 +2147483647 != 0o17777777777 +-2147483647-1 != 0o20000000000 +0o37777777777 != -1 +0xffffffff != -1; 0o37777777777 != -1; -0o1234567 == 0O001234567; 0b10101 == 0B00010101 + +# Long integers +x = 0 +x = 0 +x = 0xffffffffffffffff +x = 0xffffffffffffffff +x = 0o77777777777777777 +x = 0B11101010111111111 +x = 123456789012345678901234567890 +x = 123456789012345678901234567890 + +# Floating-point numbers +x = 3.14 +x = 314. +x = 0.314 +# XXX x = 000.314 +x = .314 +x = 3e14 +x = 3E14 +x = 3e-14 +x = 3e+14 +x = 3.e14 +x = .3e14 +x = 3.1e4 + +# String literals +x = ''; y = ""; +x = '\''; y = "'"; +x = '"'; y = "\""; +x = "doesn't \"shrink\" does it" +y = 'doesn\'t "shrink" does it' +x = "does \"shrink\" doesn't it" +y = 'does "shrink" doesn\'t it' +x = """ +The "quick" +brown fox +jumps over +the 'lazy' dog. +""" +y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n' +y = ''' +The "quick" +brown fox +jumps over +the 'lazy' dog. +'''; +y = "\n\ +The \"quick\"\n\ +brown fox\n\ +jumps over\n\ +the 'lazy' dog.\n\ +"; +y = '\n\ +The \"quick\"\n\ +brown fox\n\ +jumps over\n\ +the \'lazy\' dog.\n\ +'; +x = r'\\' + R'\\' +x = r'\'' + '' +y = r''' +foo bar \\ +baz''' + R''' +foo''' +y = r"""foo +bar \\ baz +""" + R'''spam +''' +x = b'abc' + B'ABC' +y = b"abc" + B"ABC" +x = br'abc' + Br'ABC' + bR'ABC' + BR'ABC' +y = br"abc" + Br"ABC" + bR"ABC" + BR"ABC" +x = rb'abc' + rB'ABC' + Rb'ABC' + RB'ABC' +y = rb"abc" + rB"ABC" + Rb"ABC" + RB"ABC" +x = br'\\' + BR'\\' +x = rb'\\' + RB'\\' +x = br'\'' + '' +x = rb'\'' + '' +y = br''' +foo bar \\ +baz''' + BR''' +foo''' +y = Br"""foo +bar \\ baz +""" + bR'''spam +''' +y = rB"""foo +bar \\ baz +""" + Rb'''spam +''' + +# Indentation +if 1: + x = 2 +if 1: + x = 2 +if 1: + while 0: + if 0: + x = 2 + x = 2 +if 0: + if 2: + while 0: + if 1: + x = 2 + +# Operators + +def d22(a, b, c=1, d=2): pass +def d01v(a=1, *restt, **restd): pass + +(x, y) != ({'a':1}, {'b':2}) + +# comparison +if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 != 1 in 1 not in 1 is 1 is not 1: pass + +# binary +x = 1 & 1 +x = 1 ^ 1 +x = 1 | 1 + +# shift +x = 1 << 1 >> 1 + +# additive +x = 1 - 1 + 1 - 1 + 1 + +# multiplicative +x = 1 / 1 * 1 % 1 + +# unary +x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 +x = -1*1/1 + 1*1 - ---1*1 + +# selector +import sys, time +x = sys.modules['time'].time() + +@staticmethod +def foo(): pass + +@staticmethod +def foo(x:1)->1: pass + diff --git a/Lib/test/translationdata/argparse/msgids.txt b/Lib/test/translationdata/argparse/msgids.txt new file mode 100644 index 00000000000..97416aad520 --- /dev/null +++ b/Lib/test/translationdata/argparse/msgids.txt @@ -0,0 +1,40 @@ + (default: %(default)s) +%(heading)s: +%(prog)s: error: %(message)s\n +%(prog)s: warning: %(message)s\n +%r is not callable +'required' is an invalid argument for positionals +.__call__() not defined +ambiguous option: %(option)s could match %(matches)s +argument "-" with mode %r +argument %(argument_name)s: %(message)s +argument '%(argument_name)s' is deprecated +can't open '%(filename)s': %(error)s +cannot have multiple subparser arguments +cannot merge actions - two groups are named %r +command '%(parser_name)s' is deprecated +conflicting subparser alias: %s +conflicting subparser: %s +dest= is required for options like %r +expected at least one argument +expected at most one argument +expected one argument +ignored explicit argument %r +invalid %(type)s value: %(value)r +invalid choice: %(value)r (choose from %(choices)s) +invalid conflict_resolution value: %r +invalid option string %(option)r: must start with a character %(prefix_chars)r +mutually exclusive arguments must be optional +not allowed with argument %s +one of the arguments %s is required +option '%(option)s' is deprecated +options +positional arguments +show program's version number and exit +show this help message and exit +subcommands +the following arguments are required: %s +unexpected option string: %s +unknown parser %(parser_name)r (choices: %(choices)s) +unrecognized arguments: %s +usage: \ No newline at end of file diff --git a/Lib/test/translationdata/getopt/msgids.txt b/Lib/test/translationdata/getopt/msgids.txt new file mode 100644 index 00000000000..1ffab1f31ab --- /dev/null +++ b/Lib/test/translationdata/getopt/msgids.txt @@ -0,0 +1,6 @@ +option -%s not recognized +option -%s requires argument +option --%s must not have an argument +option --%s not a unique prefix +option --%s not recognized +option --%s requires argument \ No newline at end of file diff --git a/Lib/test/translationdata/optparse/msgids.txt b/Lib/test/translationdata/optparse/msgids.txt new file mode 100644 index 00000000000..ac5317c736a --- /dev/null +++ b/Lib/test/translationdata/optparse/msgids.txt @@ -0,0 +1,14 @@ +%prog [options] +%s option does not take a value +Options +Usage +Usage: %s\n +ambiguous option: %s (%s?) +complex +floating-point +integer +no such option: %s +option %s: invalid %s value: %r +option %s: invalid choice: %r (choose from %s) +show program's version number and exit +show this help message and exit \ No newline at end of file diff --git a/Lib/test/typinganndata/_typed_dict_helper.py b/Lib/test/typinganndata/_typed_dict_helper.py new file mode 100644 index 00000000000..9df0ede7d40 --- /dev/null +++ b/Lib/test/typinganndata/_typed_dict_helper.py @@ -0,0 +1,30 @@ +"""Used to test `get_type_hints()` on a cross-module inherited `TypedDict` class + +This script uses future annotations to postpone a type that won't be available +on the module inheriting from to `Foo`. The subclass in the other module should +look something like this: + + class Bar(_typed_dict_helper.Foo, total=False): + b: int + +In addition, it uses multiple levels of Annotated to test the interaction +between the __future__ import, Annotated, and Required. +""" + +from __future__ import annotations + +from typing import Annotated, Generic, Optional, Required, TypedDict, TypeVar + + +OptionalIntType = Optional[int] + +class Foo(TypedDict): + a: OptionalIntType + +T = TypeVar("T") + +class FooGeneric(TypedDict, Generic[T]): + a: Optional[T] + +class VeryAnnotated(TypedDict, total=False): + a: Annotated[Annotated[Annotated[Required[int], "a"], "b"], "c"] diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b5834..e1a1792cb4a 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/test/typinganndata/ann_module695.py b/Lib/test/typinganndata/ann_module695.py new file mode 100644 index 00000000000..b6f3b06bd50 --- /dev/null +++ b/Lib/test/typinganndata/ann_module695.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import Callable + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: *Ts, z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +def nested(): + from types import SimpleNamespace + from typing import get_type_hints + + Eggs = bytes + Spam = memoryview + + + class E[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + return SimpleNamespace( + E=E, + hints_for_E=get_type_hints(E), + hints_for_E_meth=get_type_hints(E.generic_method), + generic_func=generic_function, + hints_for_generic_func=get_type_hints(generic_function) + ) diff --git a/Lib/test/typinganndata/fwdref_module.py b/Lib/test/typinganndata/fwdref_module.py new file mode 100644 index 00000000000..7347a7a4245 --- /dev/null +++ b/Lib/test/typinganndata/fwdref_module.py @@ -0,0 +1,6 @@ +from typing import ForwardRef + +MyList = list[int] +MyDict = dict[str, 'MyList'] + +fw = ForwardRef('MyDict', module=__name__) diff --git a/Lib/test/typinganndata/mod_generics_cache.py b/Lib/test/typinganndata/mod_generics_cache.py new file mode 100644 index 00000000000..6c1ee2fec83 --- /dev/null +++ b/Lib/test/typinganndata/mod_generics_cache.py @@ -0,0 +1,24 @@ +"""Module for testing the behavior of generics across different modules.""" + +from typing import TypeVar, Generic, Optional, TypeAliasType + +default_a: Optional['A'] = None +default_b: Optional['B'] = None + +T = TypeVar('T') + + +class A(Generic[T]): + some_b: 'B' + + +class B(Generic[T]): + class A(Generic[T]): + pass + + my_inner_a1: 'B.A' + my_inner_a2: A + my_outer_a: 'A' # unless somebody calls get_type_hints with localns=B.__dict__ + +type Alias = int +OldStyle = TypeAliasType("OldStyle", int) diff --git a/Lib/test/typinganndata/partialexecution/__init__.py b/Lib/test/typinganndata/partialexecution/__init__.py new file mode 100644 index 00000000000..c39074ea84b --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/__init__.py @@ -0,0 +1 @@ +from . import a diff --git a/Lib/test/typinganndata/partialexecution/a.py b/Lib/test/typinganndata/partialexecution/a.py new file mode 100644 index 00000000000..ed0b8dcbd55 --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/a.py @@ -0,0 +1,5 @@ +v1: int + +from . import b + +v2: int diff --git a/Lib/test/typinganndata/partialexecution/b.py b/Lib/test/typinganndata/partialexecution/b.py new file mode 100644 index 00000000000..36b8d2e52a3 --- /dev/null +++ b/Lib/test/typinganndata/partialexecution/b.py @@ -0,0 +1,3 @@ +from . import a + +annos = a.__annotations__ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part new file mode 100644 index 00000000000..c6beae8e255 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part new file mode 100644 index 00000000000..74ab03b4648 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part differ diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part new file mode 100644 index 00000000000..9769a404f67 Binary files /dev/null and b/Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part differ diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 841de9baecf..41366fbf443 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -2,7 +2,7 @@ """ # Copyright (C) 1999-2001 Gregory P. Ward. -# Copyright (C) 2002, 2003 Python Software Foundation. +# Copyright (C) 2002 Python Software Foundation. # Written by Greg Ward <gward@python.net> import re @@ -63,10 +63,7 @@ class TextWrapper: Append to the last line of truncated text. """ - unicode_whitespace_trans = {} - uspace = ord(' ') - for x in _whitespace: - unicode_whitespace_trans[ord(x)] = uspace + unicode_whitespace_trans = dict.fromkeys(map(ord, _whitespace), ord(' ')) # This funky little regex is just the trick for splitting # text up into word-wrappable chunks. E.g. @@ -89,7 +86,7 @@ class TextWrapper: -(?: (?<=%(lt)s{2}-) | (?<=%(lt)s-%(lt)s-)) (?= %(lt)s -? %(lt)s) | # end of word - (?=%(ws)s|\Z) + (?=%(ws)s|\z) | # em-dash (?<=%(wp)s) (?=-{2,}\w) ) @@ -110,7 +107,7 @@ class TextWrapper: sentence_end_re = re.compile(r'[a-z]' # lowercase letter r'[\.\!\?]' # sentence-ending punct. r'[\"\']?' # optional end-of-quote - r'\Z') # end of chunk + r'\z') # end of chunk def __init__(self, width=70, @@ -214,7 +211,7 @@ def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. - if self.break_long_words: + if self.break_long_words and space_left > 0: end = space_left chunk = reversed_chunks[-1] if self.break_on_hyphens and len(chunk) > space_left: @@ -416,9 +413,6 @@ def shorten(text, width, **kwargs): # -- Loosely related functionality ------------------------------------- -_whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE) -_leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE) - def dedent(text): """Remove any common leading whitespace from every line in `text`. @@ -432,42 +426,22 @@ def dedent(text): Entirely blank lines are normalized to a newline character. """ - # Look for the longest leading string of spaces and tabs common to - # all lines. - margin = None - text = _whitespace_only_re.sub('', text) - indents = _leading_whitespace_re.findall(text) - for indent in indents: - if margin is None: - margin = indent - - # Current line more deeply indented than previous winner: - # no change (previous winner is still on top). - elif indent.startswith(margin): - pass - - # Current line consistent with and no deeper than previous winner: - # it's the new winner. - elif margin.startswith(indent): - margin = indent - - # Find the largest common whitespace between current line and previous - # winner. - else: - for i, (x, y) in enumerate(zip(margin, indent)): - if x != y: - margin = margin[:i] - break + try: + lines = text.split('\n') + except (AttributeError, TypeError): + msg = f'expected str object, not {type(text).__qualname__!r}' + raise TypeError(msg) from None - # sanity check (testing/debugging only) - if 0 and margin: - for line in text.split("\n"): - assert not line or line.startswith(margin), \ - "line = %r, margin = %r" % (line, margin) + # Get length of leading whitespace, inspired by ``os.path.commonprefix()``. + non_blank_lines = [l for l in lines if l and not l.isspace()] + l1 = min(non_blank_lines, default='') + l2 = max(non_blank_lines, default='') + margin = 0 + for margin, c in enumerate(l1): + if c != l2[margin] or c not in ' \t': + break - if margin: - text = re.sub(r'(?m)^' + margin, '', text) - return text + return '\n'.join([l[margin:] if not l.isspace() else '' for l in lines]) def indent(text, prefix, predicate=None): @@ -478,14 +452,21 @@ def indent(text, prefix, predicate=None): it will default to adding 'prefix' to all non-empty lines that do not consist solely of whitespace characters. """ + prefixed_lines = [] if predicate is None: - def predicate(line): - return line.strip() - - def prefixed_lines(): + # str.splitlines(keepends=True) doesn't produce the empty string, + # so we need to use `str.isspace()` rather than a truth test. + # Inlining the predicate leads to a ~30% performance improvement. + for line in text.splitlines(True): + if not line.isspace(): + prefixed_lines.append(prefix) + prefixed_lines.append(line) + else: for line in text.splitlines(True): - yield (prefix + line if predicate(line) else line) - return ''.join(prefixed_lines()) + if predicate(line): + prefixed_lines.append(prefix) + prefixed_lines.append(line) + return ''.join(prefixed_lines) if __name__ == "__main__": diff --git a/Lib/threading.py b/Lib/threading.py index 668126523d5..c03b0b5370c 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -3,11 +3,11 @@ import os as _os import sys as _sys import _thread -import functools +import _contextvars from time import monotonic as _time from _weakrefset import WeakSet -from itertools import islice as _islice, count as _count +from itertools import count as _count try: from _collections import deque as _deque except ImportError: @@ -28,19 +28,30 @@ 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size', - 'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile'] + 'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile', + 'setprofile_all_threads','settrace_all_threads'] # Rename some stuff so "from threading import *" is safe -_start_new_thread = _thread.start_new_thread +_start_joinable_thread = _thread.start_joinable_thread +_daemon_threads_allowed = _thread.daemon_threads_allowed _allocate_lock = _thread.allocate_lock -_set_sentinel = _thread._set_sentinel +_LockType = _thread.LockType +_thread_shutdown = _thread._shutdown +_make_thread_handle = _thread._make_thread_handle +_ThreadHandle = _thread._ThreadHandle get_ident = _thread.get_ident +_get_main_thread_ident = _thread._get_main_thread_ident +_is_main_interpreter = _thread._is_main_interpreter try: get_native_id = _thread.get_native_id _HAVE_THREAD_NATIVE_ID = True __all__.append('get_native_id') except AttributeError: _HAVE_THREAD_NATIVE_ID = False +try: + _set_name = _thread.set_name +except AttributeError: + _set_name = None ThreadError = _thread.error try: _CRLock = _thread.RLock @@ -49,6 +60,13 @@ TIMEOUT_MAX = _thread.TIMEOUT_MAX del _thread +# get thread-local implementation, either from the thread +# module, or from the python fallback + +try: + from _thread import _local as local +except ImportError: + from _threading_local import local # Support for profile and trace hooks @@ -60,11 +78,20 @@ def setprofile(func): The func will be passed to sys.setprofile() for each thread, before its run() method is called. - """ global _profile_hook _profile_hook = func +def setprofile_all_threads(func): + """Set a profile function for all threads started from the threading module + and all Python threads that are currently executing. + + The func will be passed to sys.setprofile() for each thread, before its + run() method is called. + """ + setprofile(func) + _sys._setprofileallthreads(func) + def getprofile(): """Get the profiler function as set by threading.setprofile().""" return _profile_hook @@ -74,18 +101,27 @@ def settrace(func): The func will be passed to sys.settrace() for each thread, before its run() method is called. - """ global _trace_hook _trace_hook = func +def settrace_all_threads(func): + """Set a trace function for all threads started from the threading module + and all Python threads that are currently executing. + + The func will be passed to sys.settrace() for each thread, before its run() + method is called. + """ + settrace(func) + _sys._settraceallthreads(func) + def gettrace(): """Get the trace function as set by threading.settrace().""" return _trace_hook # Synchronization classes -Lock = _allocate_lock +Lock = _LockType def RLock(*args, **kwargs): """Factory function that returns a new reentrant lock. @@ -96,6 +132,13 @@ def RLock(*args, **kwargs): acquired it. """ + if args or kwargs: + import warnings + warnings.warn( + 'Passing arguments to RLock is deprecated and will be removed in 3.15', + DeprecationWarning, + stacklevel=2, + ) if _CRLock is None: return _PyRLock(*args, **kwargs) return _CRLock(*args, **kwargs) @@ -122,7 +165,7 @@ def __repr__(self): except KeyError: pass return "<%s %s.%s object owner=%r count=%d at %s>" % ( - "locked" if self._block.locked() else "unlocked", + "locked" if self.locked() else "unlocked", self.__class__.__module__, self.__class__.__qualname__, owner, @@ -199,6 +242,10 @@ def release(self): def __exit__(self, t, v, tb): self.release() + def locked(self): + """Return whether this object is locked.""" + return self._block.locked() + # Internal methods used by condition variables def _acquire_restore(self, state): @@ -218,6 +265,13 @@ def _release_save(self): def _is_owned(self): return self._owner == get_ident() + # Internal method used for reentrancy checks + + def _recursion_count(self): + if self._owner != get_ident(): + return 0 + return self._count + _PyRLock = _RLock @@ -237,24 +291,19 @@ def __init__(self, lock=None): if lock is None: lock = RLock() self._lock = lock - # Export the lock's acquire() and release() methods + # Export the lock's acquire(), release(), and locked() methods self.acquire = lock.acquire self.release = lock.release + self.locked = lock.locked # If the lock defines _release_save() and/or _acquire_restore(), # these override the default implementations (which just call # release() and acquire() on the lock). Ditto for _is_owned(). - try: + if hasattr(lock, '_release_save'): self._release_save = lock._release_save - except AttributeError: - pass - try: + if hasattr(lock, '_acquire_restore'): self._acquire_restore = lock._acquire_restore - except AttributeError: - pass - try: + if hasattr(lock, '_is_owned'): self._is_owned = lock._is_owned - except AttributeError: - pass self._waiters = _deque() def _at_fork_reinit(self): @@ -297,7 +346,7 @@ def wait(self, timeout=None): awakened or timed out, it re-acquires the lock and returns. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). When the underlying lock is an RLock, it is not released using its @@ -425,6 +474,11 @@ def __init__(self, value=1): self._cond = Condition(Lock()) self._value = value + def __repr__(self): + cls = self.__class__ + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" value={self._value}>") + def acquire(self, blocking=True, timeout=None): """Acquire a semaphore, decrementing the internal counter by one. @@ -483,8 +537,7 @@ def release(self, n=1): raise ValueError('n must be one or more') with self._cond: self._value += n - for i in range(n): - self._cond.notify() + self._cond.notify(n) def __exit__(self, t, v, tb): self.release() @@ -508,9 +561,14 @@ class BoundedSemaphore(Semaphore): """ def __init__(self, value=1): - Semaphore.__init__(self, value) + super().__init__(value) self._initial_value = value + def __repr__(self): + cls = self.__class__ + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" value={self._value}/{self._initial_value}>") + def release(self, n=1): """Release a semaphore, incrementing the internal counter by one or more. @@ -527,8 +585,7 @@ def release(self, n=1): if self._value + n > self._initial_value: raise ValueError("Semaphore released too many times") self._value += n - for i in range(n): - self._cond.notify() + self._cond.notify(n) class Event: @@ -546,8 +603,13 @@ def __init__(self): self._cond = Condition(Lock()) self._flag = False + def __repr__(self): + cls = self.__class__ + status = 'set' if self._flag else 'unset' + return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: {status}>" + def _at_fork_reinit(self): - # Private method called by Thread._reset_internal_locks() + # Private method called by Thread._after_fork() self._cond._at_fork_reinit() def is_set(self): @@ -557,7 +619,7 @@ def is_set(self): def isSet(self): """Return true if and only if the internal flag is true. - This method is deprecated, use notify_all() instead. + This method is deprecated, use is_set() instead. """ import warnings @@ -594,11 +656,12 @@ def wait(self, timeout=None): the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). This method returns the internal flag on exit, so it will always return - True except if a timeout is given and the operation times out. + ``True`` except if a timeout is given and the operation times out, when + it will return ``False``. """ with self._cond: @@ -637,6 +700,8 @@ def __init__(self, parties, action=None, timeout=None): default for all subsequent 'wait()' calls. """ + if parties < 1: + raise ValueError("parties must be >= 1") self._cond = Condition(Lock()) self._action = action self._timeout = timeout @@ -644,6 +709,13 @@ def __init__(self, parties, action=None, timeout=None): self._state = 0 # 0 filling, 1 draining, -1 resetting, -2 broken self._count = 0 + def __repr__(self): + cls = self.__class__ + if self.broken: + return f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}: broken>" + return (f"<{cls.__module__}.{cls.__qualname__} at {id(self):#x}:" + f" waiters={self.n_waiting}/{self.parties}>") + def wait(self, timeout=None): """Wait for the barrier. @@ -791,25 +863,6 @@ def _newname(name_template): _limbo = {} _dangling = WeakSet() -# Set of Thread._tstate_lock locks of non-daemon threads used by _shutdown() -# to wait until all Python thread states get deleted: -# see Thread._set_tstate_lock(). -_shutdown_locks_lock = _allocate_lock() -_shutdown_locks = set() - -def _maintain_shutdown_locks(): - """ - Drop any shutdown locks that don't correspond to running threads anymore. - - Calling this from time to time avoids an ever-growing _shutdown_locks - set when Thread objects are not joined explicitly. See bpo-37788. - - This must be called with _shutdown_locks_lock acquired. - """ - # If a lock was released, the corresponding thread has exited - to_remove = [lock for lock in _shutdown_locks if not lock.locked()] - _shutdown_locks.difference_update(to_remove) - # Main class for threads @@ -825,7 +878,7 @@ class Thread: _initialized = False def __init__(self, group=None, target=None, name=None, - args=(), kwargs=None, *, daemon=None): + args=(), kwargs=None, *, daemon=None, context=None): """This constructor should always be called with keyword arguments. Arguments are: *group* should be None; reserved for future extension when a ThreadGroup @@ -837,11 +890,19 @@ class is implemented. *name* is the thread name. By default, a unique name is constructed of the form "Thread-N" where N is a small decimal number. - *args* is the argument tuple for the target invocation. Defaults to (). + *args* is a list or tuple of arguments for the target invocation. Defaults to (). *kwargs* is a dictionary of keyword arguments for the target invocation. Defaults to {}. + *context* is the contextvars.Context value to use for the thread. + The default value is None, which means to check + sys.flags.thread_inherit_context. If that flag is true, use a copy + of the context of the caller. If false, use an empty context. To + explicitly start with an empty context, pass a new instance of + contextvars.Context(). To explicitly start with a copy of the current + context, pass the value from contextvars.copy_context(). + If a subclass overrides the constructor, it must make sure to invoke the base class constructor (Thread.__init__()) before doing anything else to the thread. @@ -866,15 +927,17 @@ class is implemented. self._args = args self._kwargs = kwargs if daemon is not None: + if daemon and not _daemon_threads_allowed(): + raise RuntimeError('daemon threads are disabled in this (sub)interpreter') self._daemonic = daemon else: self._daemonic = current_thread().daemon + self._context = context self._ident = None if _HAVE_THREAD_NATIVE_ID: self._native_id = None - self._tstate_lock = None + self._os_thread_handle = _ThreadHandle() self._started = Event() - self._is_stopped = False self._initialized = True # Copy of sys.stderr used by self._invoke_excepthook() self._stderr = _sys.stderr @@ -882,30 +945,26 @@ class is implemented. # For debugging and _after_fork() _dangling.add(self) - def _reset_internal_locks(self, is_alive): - # private! Called by _after_fork() to reset our internal locks as - # they may be in an invalid state leading to a deadlock or crash. + def _after_fork(self, new_ident=None): + # Private! Called by threading._after_fork(). self._started._at_fork_reinit() - if is_alive: - # bpo-42350: If the fork happens when the thread is already stopped - # (ex: after threading._shutdown() has been called), _tstate_lock - # is None. Do nothing in this case. - if self._tstate_lock is not None: - self._tstate_lock._at_fork_reinit() - self._tstate_lock.acquire() + if new_ident is not None: + # This thread is alive. + self._ident = new_ident + assert self._os_thread_handle.ident == new_ident + if _HAVE_THREAD_NATIVE_ID: + self._set_native_id() else: - # The thread isn't alive after fork: it doesn't have a tstate - # anymore. - self._is_stopped = True - self._tstate_lock = None + # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() + # already marked our handle done. + pass def __repr__(self): assert self._initialized, "Thread.__init__() was not called" status = "initial" if self._started.is_set(): status = "started" - self.is_alive() # easy way to get ._is_stopped set when appropriate - if self._is_stopped: + if self._os_thread_handle.is_done(): status = "stopped" if self._daemonic: status += " daemon" @@ -931,13 +990,25 @@ def start(self): with _active_limbo_lock: _limbo[self] = self + + if self._context is None: + # No context provided + if _sys.flags.thread_inherit_context: + # start with a copy of the context of the caller + self._context = _contextvars.copy_context() + else: + # start with an empty context + self._context = _contextvars.Context() + try: - _start_new_thread(self._bootstrap, ()) + # Start joinable thread + _start_joinable_thread(self._bootstrap, handle=self._os_thread_handle, + daemon=self.daemon) except Exception: with _active_limbo_lock: del _limbo[self] raise - self._started.wait() + self._started.wait() # Will set ident and native_id def run(self): """Method representing the thread's activity. @@ -983,25 +1054,20 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() - def _set_tstate_lock(self): - """ - Set a lock object which will be released by the interpreter when - the underlying thread state (see pystate.h) gets deleted. - """ - self._tstate_lock = _set_sentinel() - self._tstate_lock.acquire() - - if not self.daemon: - with _shutdown_locks_lock: - _maintain_shutdown_locks() - _shutdown_locks.add(self._tstate_lock) + def _set_os_name(self): + if _set_name is None or not self._name: + return + try: + _set_name(self._name) + except OSError: + pass def _bootstrap_inner(self): try: self._set_ident() - self._set_tstate_lock() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() + self._set_os_name() self._started.set() with _active_limbo_lock: _active[self._ident] = self @@ -1013,44 +1079,11 @@ def _bootstrap_inner(self): _sys.setprofile(_profile_hook) try: - self.run() + self._context.run(self.run) except: self._invoke_excepthook(self) finally: - with _active_limbo_lock: - try: - # We don't call self._delete() because it also - # grabs _active_limbo_lock. - del _active[get_ident()] - except: - pass - - def _stop(self): - # After calling ._stop(), .is_alive() returns False and .join() returns - # immediately. ._tstate_lock must be released before calling ._stop(). - # - # Normal case: C code at the end of the thread's life - # (release_sentinel in _threadmodule.c) releases ._tstate_lock, and - # that's detected by our ._wait_for_tstate_lock(), called by .join() - # and .is_alive(). Any number of threads _may_ call ._stop() - # simultaneously (for example, if multiple threads are blocked in - # .join() calls), and they're not serialized. That's harmless - - # they'll just make redundant rebindings of ._is_stopped and - # ._tstate_lock. Obscure: we rebind ._tstate_lock last so that the - # "assert self._is_stopped" in ._wait_for_tstate_lock() always works - # (the assert is executed only if ._tstate_lock is None). - # - # Special case: _main_thread releases ._tstate_lock via this - # module's _shutdown() function. - lock = self._tstate_lock - if lock is not None: - assert not lock.locked() - self._is_stopped = True - self._tstate_lock = None - if not self.daemon: - with _shutdown_locks_lock: - # Remove our lock and other released locks from _shutdown_locks - _maintain_shutdown_locks() + self._delete() def _delete(self): "Remove current thread from the dict of currently running threads." @@ -1069,7 +1102,7 @@ def join(self, timeout=None): or until the optional timeout occurs. When the timeout argument is present and not None, it should be a - floating point number specifying a timeout for the operation in seconds + floating-point number specifying a timeout for the operation in seconds (or fractions thereof). As join() always returns None, you must call is_alive() after join() to decide whether a timeout happened -- if the thread is still alive, the join() call timed out. @@ -1092,39 +1125,12 @@ def join(self, timeout=None): if self is current_thread(): raise RuntimeError("cannot join current thread") - if timeout is None: - self._wait_for_tstate_lock() - else: - # the behavior of a negative timeout isn't documented, but - # historically .join(timeout=x) for x<0 has acted as if timeout=0 - self._wait_for_tstate_lock(timeout=max(timeout, 0)) - - def _wait_for_tstate_lock(self, block=True, timeout=-1): - # Issue #18808: wait for the thread state to be gone. - # At the end of the thread's life, after all knowledge of the thread - # is removed from C data structures, C code releases our _tstate_lock. - # This method passes its arguments to _tstate_lock.acquire(). - # If the lock is acquired, the C code is done, and self._stop() is - # called. That sets ._is_stopped to True, and ._tstate_lock to None. - lock = self._tstate_lock - if lock is None: - # already determined that the C code is done - assert self._is_stopped - return + # the behavior of a negative timeout isn't documented, but + # historically .join(timeout=x) for x<0 has acted as if timeout=0 + if timeout is not None: + timeout = max(timeout, 0) - try: - if lock.acquire(block, timeout): - lock.release() - self._stop() - except: - if lock.locked(): - # bpo-45274: lock.acquire() acquired the lock, but the function - # was interrupted with an exception before reaching the - # lock.release(). It can happen if a signal handler raises an - # exception, like CTRL+C which raises KeyboardInterrupt. - lock.release() - self._stop() - raise + self._os_thread_handle.join(timeout) @property def name(self): @@ -1141,6 +1147,8 @@ def name(self): def name(self, name): assert self._initialized, "Thread.__init__() not called" self._name = str(name) + if get_ident() == self._ident: + self._set_os_name() @property def ident(self): @@ -1175,10 +1183,7 @@ def is_alive(self): """ assert self._initialized, "Thread.__init__() not called" - if self._is_stopped or not self._started.is_set(): - return False - self._wait_for_tstate_lock(False) - return not self._is_stopped + return self._started.is_set() and not self._os_thread_handle.is_done() @property def daemon(self): @@ -1199,6 +1204,8 @@ def daemon(self): def daemon(self, daemonic): if not self._initialized: raise RuntimeError("Thread.__init__() not called") + if daemonic and not _daemon_threads_allowed(): + raise RuntimeError('daemon threads are disabled in this interpreter') if self._started.is_set(): raise RuntimeError("cannot set daemon status of active thread") self._daemonic = daemonic @@ -1385,19 +1392,45 @@ class _MainThread(Thread): def __init__(self): Thread.__init__(self, name="MainThread", daemon=False) - self._set_tstate_lock() self._started.set() - self._set_ident() + self._ident = _get_main_thread_ident() + self._os_thread_handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self +# Helper thread-local instance to detect when a _DummyThread +# is collected. Not a part of the public API. +_thread_local_info = local() + + +class _DeleteDummyThreadOnDel: + ''' + Helper class to remove a dummy thread from threading._active on __del__. + ''' + + def __init__(self, dummy_thread): + self._dummy_thread = dummy_thread + self._tident = dummy_thread.ident + # Put the thread on a thread local variable so that when + # the related thread finishes this instance is collected. + # + # Note: no other references to this instance may be created. + # If any client code creates a reference to this instance, + # the related _DummyThread will be kept forever! + _thread_local_info._track_dummy_thread_ref = self + + def __del__(self, _active_limbo_lock=_active_limbo_lock, _active=_active): + with _active_limbo_lock: + if _active.get(self._tident) is self._dummy_thread: + _active.pop(self._tident, None) + + # Dummy thread class to represent threads not started here. -# These aren't garbage collected when they die, nor can they be waited for. -# If they invoke anything in threading.py that calls current_thread(), they -# leave an entry in the _active dict forever after. +# These should be added to `_active` and removed automatically +# when they die, although they can't be waited for. # Their purpose is to return *something* from current_thread(). # They are marked as daemon threads so we won't wait for them # when we exit (conform previous semantics). @@ -1405,24 +1438,31 @@ def __init__(self): class _DummyThread(Thread): def __init__(self): - Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True) - + Thread.__init__(self, name=_newname("Dummy-%d"), + daemon=_daemon_threads_allowed()) self._started.set() self._set_ident() + self._os_thread_handle = _make_thread_handle(self._ident) if _HAVE_THREAD_NATIVE_ID: self._set_native_id() with _active_limbo_lock: _active[self._ident] = self - - def _stop(self): - pass + _DeleteDummyThreadOnDel(self) def is_alive(self): - assert not self._is_stopped and self._started.is_set() - return True + if not self._os_thread_handle.is_done() and self._started.is_set(): + return True + raise RuntimeError("thread is not alive") def join(self, timeout=None): - assert False, "cannot join a dummy thread" + raise RuntimeError("cannot join a dummy thread") + + def _after_fork(self, new_ident=None): + if new_ident is not None: + self.__class__ = _MainThread + self._name = 'MainThread' + self._daemonic = False + Thread._after_fork(self, new_ident=new_ident) # Global API functions @@ -1457,6 +1497,8 @@ def active_count(): enumerate(). """ + # NOTE: if the logic in here ever changes, update Modules/posixmodule.c + # warn_about_fork_with_threads() to match. with _active_limbo_lock: return len(_active) + len(_limbo) @@ -1503,8 +1545,7 @@ def _register_atexit(func, *arg, **kwargs): if _SHUTTING_DOWN: raise RuntimeError("can't register atexit after shutdown") - call = functools.partial(func, *arg, **kwargs) - _threading_atexits.append(call) + _threading_atexits.append(lambda: func(*arg, **kwargs)) from _thread import stack_size @@ -1519,12 +1560,11 @@ def _shutdown(): """ Wait until the Python thread state of all non-daemon threads get deleted. """ - # Obscure: other threads may be waiting to join _main_thread. That's - # dubious, but some code does it. We can't wait for C code to release - # the main thread's tstate_lock - that won't happen until the interpreter - # is nearly dead. So we release it here. Note that just calling _stop() - # isn't enough: other threads may already be waiting on _tstate_lock. - if _main_thread._is_stopped: + # Obscure: other threads may be waiting to join _main_thread. That's + # dubious, but some code does it. We can't wait for it to be marked as done + # normally - that won't happen until the interpreter is nearly dead. So + # mark it done here. + if _main_thread._os_thread_handle.is_done() and _is_main_interpreter(): # _shutdown() was already called return @@ -1536,39 +1576,11 @@ def _shutdown(): for atexit_call in reversed(_threading_atexits): atexit_call() - # Main thread - if _main_thread.ident == get_ident(): - tlock = _main_thread._tstate_lock - # The main thread isn't finished yet, so its thread state lock can't - # have been released. - assert tlock is not None - assert tlock.locked() - tlock.release() - _main_thread._stop() - else: - # bpo-1596321: _shutdown() must be called in the main thread. - # If the threading module was not imported by the main thread, - # _main_thread is the thread which imported the threading module. - # In this case, ignore _main_thread, similar behavior than for threads - # spawned by C libraries or using _thread.start_new_thread(). - pass - - # Join all non-deamon threads - while True: - with _shutdown_locks_lock: - locks = list(_shutdown_locks) - _shutdown_locks.clear() - - if not locks: - break - - for lock in locks: - # mimic Thread.join() - lock.acquire() - lock.release() - - # new threads can be spawned while we were waiting for the other - # threads to complete + if _is_main_interpreter(): + _main_thread._os_thread_handle._set_done() + + # Wait for all non-daemon threads to exit. + _thread_shutdown() def main_thread(): @@ -1577,16 +1589,9 @@ def main_thread(): In normal conditions, the main thread is the thread from which the Python interpreter was started. """ + # XXX Figure this out for subinterpreters. (See gh-75698.) return _main_thread -# get thread-local implementation, either from the thread -# module, or from the python fallback - -try: - from _thread import _local as local -except ImportError: - from _threading_local import local - def _after_fork(): """ @@ -1595,7 +1600,6 @@ def _after_fork(): # Reset _active_limbo_lock, in case we forked while the lock was held # by another (non-forked) thread. http://bugs.python.org/issue874900 global _active_limbo_lock, _main_thread - global _shutdown_locks_lock, _shutdown_locks _active_limbo_lock = RLock() # fork() only copied the current thread; clear references to others. @@ -1611,10 +1615,6 @@ def _after_fork(): _main_thread = current - # reset _shutdown() locks: threads re-register their _tstate_lock below - _shutdown_locks_lock = _allocate_lock() - _shutdown_locks = set() - with _active_limbo_lock: # Dangling thread instances must still have their locks reset, # because someone may join() them. @@ -1624,16 +1624,13 @@ def _after_fork(): # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if thread is current: - # There is only one active thread. We reset the ident to - # its new value since it can have changed. - thread._reset_internal_locks(True) + # This is the one and only active thread. ident = get_ident() - thread._ident = ident + thread._after_fork(new_ident=ident) new_active[ident] = thread else: # All the others are already stopped. - thread._reset_internal_locks(False) - thread._stop() + thread._after_fork() _limbo.clear() _active.clear() diff --git a/Lib/timeit.py b/Lib/timeit.py old mode 100755 new mode 100644 index f323e65572d..e767f018782 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """Tool for measuring execution time of small code snippets. This module avoids a number of common traps for measuring execution @@ -46,13 +44,12 @@ timeit(string, string) -> float repeat(string, string) -> list default_timer() -> float - """ import gc +import itertools import sys import time -import itertools __all__ = ["Timer", "timeit", "repeat", "default_timer"] @@ -77,9 +74,11 @@ def inner(_it, _timer{init}): return _t1 - _t0 """ + def reindent(src, indent): """Helper to reindent a multi-line statement.""" - return src.replace("\n", "\n" + " "*indent) + return src.replace("\n", "\n" + " " * indent) + class Timer: """Class for timing execution speed of small code snippets. @@ -166,22 +165,20 @@ def timeit(self, number=default_number): To be precise, this executes the setup statement once, and then returns the time it takes to execute the main statement - a number of times, as a float measured in seconds. The + a number of times, as float seconds if using the default timer. The argument is the number of times through the loop, defaulting to one million. The main statement, the setup statement and the timer function to be used are passed to the constructor. """ it = itertools.repeat(None, number) - # XXX RUSTPYTHON TODO: gc module implementation - # gcold = gc.isenabled() - # gc.disable() - # try: - # timing = self.inner(it, self.timer) - # finally: - # if gcold: - # gc.enable() - # return timing - return self.inner(it, self.timer) + gcold = gc.isenabled() + gc.disable() + try: + timing = self.inner(it, self.timer) + finally: + if gcold: + gc.enable() + return timing def repeat(self, repeat=default_repeat, number=default_number): """Call timeit() a few times. @@ -230,16 +227,19 @@ def autorange(self, callback=None): return (number, time_taken) i *= 10 + def timeit(stmt="pass", setup="pass", timer=default_timer, number=default_number, globals=None): """Convenience function to create Timer object and call timeit method.""" return Timer(stmt, setup, timer, globals).timeit(number) + def repeat(stmt="pass", setup="pass", timer=default_timer, repeat=default_repeat, number=default_number, globals=None): """Convenience function to create Timer object and call repeat method.""" return Timer(stmt, setup, timer, globals).repeat(repeat, number) + def main(args=None, *, _wrap_timer=None): """Main program, used when run as a script. @@ -261,10 +261,9 @@ def main(args=None, *, _wrap_timer=None): args = sys.argv[1:] import getopt try: - opts, args = getopt.getopt(args, "n:u:s:r:tcpvh", + opts, args = getopt.getopt(args, "n:u:s:r:pvh", ["number=", "setup=", "repeat=", - "time", "clock", "process", - "verbose", "unit=", "help"]) + "process", "verbose", "unit=", "help"]) except getopt.error as err: print(err) print("use -h/--help for command line help") @@ -272,7 +271,7 @@ def main(args=None, *, _wrap_timer=None): timer = default_timer stmt = "\n".join(args) or "pass" - number = 0 # auto-determine + number = 0 # auto-determine setup = [] repeat = default_repeat verbose = 0 @@ -289,7 +288,7 @@ def main(args=None, *, _wrap_timer=None): time_unit = a else: print("Unrecognized unit. Please select nsec, usec, msec, or sec.", - file=sys.stderr) + file=sys.stderr) return 2 if o in ("-r", "--repeat"): repeat = int(a) @@ -302,7 +301,7 @@ def main(args=None, *, _wrap_timer=None): precision += 1 verbose += 1 if o in ("-h", "--help"): - print(__doc__, end=' ') + print(__doc__, end="") return 0 setup = "\n".join(setup) or "pass" @@ -323,7 +322,7 @@ def callback(number, time_taken): msg = "{num} loop{s} -> {secs:.{prec}g} secs" plural = (number != 1) print(msg.format(num=number, s='s' if plural else '', - secs=time_taken, prec=precision)) + secs=time_taken, prec=precision)) try: number, _ = t.autorange(callback) except: @@ -374,5 +373,6 @@ def format_time(dt): UserWarning, '', 0) return None + if __name__ == "__main__": sys.exit(main()) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py new file mode 100644 index 00000000000..df3b936ccd0 --- /dev/null +++ b/Lib/tkinter/__init__.py @@ -0,0 +1,4986 @@ +"""Wrapper functions for Tcl/Tk. + +Tkinter provides classes which allow the display, positioning and +control of widgets. Toplevel widgets are Tk and Toplevel. Other +widgets are Frame, Label, Entry, Text, Canvas, Button, Radiobutton, +Checkbutton, Scale, Listbox, Scrollbar, OptionMenu, Spinbox +LabelFrame and PanedWindow. + +Properties of the widgets are specified with keyword arguments. +Keyword arguments have the same name as the corresponding resource +under Tk. + +Widgets are positioned with one of the geometry managers Place, Pack +or Grid. These managers can be called with methods place, pack, grid +available in every Widget. + +Actions are bound to events by resources (e.g. keyword argument +command) or with the method bind. + +Example (Hello, World): +import tkinter +from tkinter.constants import * +tk = tkinter.Tk() +frame = tkinter.Frame(tk, relief=RIDGE, borderwidth=2) +frame.pack(fill=BOTH,expand=1) +label = tkinter.Label(frame, text="Hello, World") +label.pack(fill=X, expand=1) +button = tkinter.Button(frame,text="Exit",command=tk.destroy) +button.pack(side=BOTTOM) +tk.mainloop() +""" + +import collections +import enum +import sys +import types + +import _tkinter # If this fails your Python may not be configured for Tk +TclError = _tkinter.TclError +from tkinter.constants import * +import re + +wantobjects = 1 +_debug = False # set to True to print executed Tcl/Tk commands + +TkVersion = float(_tkinter.TK_VERSION) +TclVersion = float(_tkinter.TCL_VERSION) + +READABLE = _tkinter.READABLE +WRITABLE = _tkinter.WRITABLE +EXCEPTION = _tkinter.EXCEPTION + + +_magic_re = re.compile(r'([\\{}])') +_space_re = re.compile(r'([\s])', re.ASCII) + + +def _join(value): + """Internal function.""" + return ' '.join(map(_stringify, value)) + + +def _stringify(value): + """Internal function.""" + if isinstance(value, (list, tuple)): + if len(value) == 1: + value = _stringify(value[0]) + if _magic_re.search(value): + value = '{%s}' % value + else: + value = '{%s}' % _join(value) + else: + if isinstance(value, bytes): + value = str(value, 'latin1') + else: + value = str(value) + if not value: + value = '{}' + elif _magic_re.search(value): + # add '\' before special characters and spaces + value = _magic_re.sub(r'\\\1', value) + value = value.replace('\n', r'\n') + value = _space_re.sub(r'\\\1', value) + if value[0] == '"': + value = '\\' + value + elif value[0] == '"' or _space_re.search(value): + value = '{%s}' % value + return value + + +def _flatten(seq): + """Internal function.""" + res = () + for item in seq: + if isinstance(item, (tuple, list)): + res = res + _flatten(item) + elif item is not None: + res = res + (item,) + return res + + +try: _flatten = _tkinter._flatten +except AttributeError: pass + + +def _cnfmerge(cnfs): + """Internal function.""" + if isinstance(cnfs, dict): + return cnfs + elif isinstance(cnfs, (type(None), str)): + return cnfs + else: + cnf = {} + for c in _flatten(cnfs): + try: + cnf.update(c) + except (AttributeError, TypeError) as msg: + print("_cnfmerge: fallback due to:", msg) + for k, v in c.items(): + cnf[k] = v + return cnf + + +try: _cnfmerge = _tkinter._cnfmerge +except AttributeError: pass + + +def _splitdict(tk, v, cut_minus=True, conv=None): + """Return a properly formatted dict built from Tcl list pairs. + + If cut_minus is True, the supposed '-' prefix will be removed from + keys. If conv is specified, it is used to convert values. + + Tcl list is expected to contain an even number of elements. + """ + t = tk.splitlist(v) + if len(t) % 2: + raise RuntimeError('Tcl list representing a dict is expected ' + 'to contain an even number of elements') + it = iter(t) + dict = {} + for key, value in zip(it, it): + key = str(key) + if cut_minus and key[0] == '-': + key = key[1:] + if conv: + value = conv(value) + dict[key] = value + return dict + +class _VersionInfoType(collections.namedtuple('_VersionInfoType', + ('major', 'minor', 'micro', 'releaselevel', 'serial'))): + def __str__(self): + if self.releaselevel == 'final': + return f'{self.major}.{self.minor}.{self.micro}' + else: + return f'{self.major}.{self.minor}{self.releaselevel[0]}{self.serial}' + +def _parse_version(version): + import re + m = re.fullmatch(r'(\d+)\.(\d+)([ab.])(\d+)', version) + major, minor, releaselevel, serial = m.groups() + major, minor, serial = int(major), int(minor), int(serial) + if releaselevel == '.': + micro = serial + serial = 0 + releaselevel = 'final' + else: + micro = 0 + releaselevel = {'a': 'alpha', 'b': 'beta'}[releaselevel] + return _VersionInfoType(major, minor, micro, releaselevel, serial) + + +@enum._simple_enum(enum.StrEnum) +class EventType: + KeyPress = '2' + Key = KeyPress + KeyRelease = '3' + ButtonPress = '4' + Button = ButtonPress + ButtonRelease = '5' + Motion = '6' + Enter = '7' + Leave = '8' + FocusIn = '9' + FocusOut = '10' + Keymap = '11' # undocumented + Expose = '12' + GraphicsExpose = '13' # undocumented + NoExpose = '14' # undocumented + Visibility = '15' + Create = '16' + Destroy = '17' + Unmap = '18' + Map = '19' + MapRequest = '20' + Reparent = '21' + Configure = '22' + ConfigureRequest = '23' + Gravity = '24' + ResizeRequest = '25' + Circulate = '26' + CirculateRequest = '27' + Property = '28' + SelectionClear = '29' # undocumented + SelectionRequest = '30' # undocumented + Selection = '31' # undocumented + Colormap = '32' + ClientMessage = '33' # undocumented + Mapping = '34' # undocumented + VirtualEvent = '35' # undocumented + Activate = '36' + Deactivate = '37' + MouseWheel = '38' + + +class Event: + """Container for the properties of an event. + + Instances of this type are generated if one of the following events occurs: + + KeyPress, KeyRelease - for keyboard events + ButtonPress, ButtonRelease, Motion, Enter, Leave, MouseWheel - for mouse events + Visibility, Unmap, Map, Expose, FocusIn, FocusOut, Circulate, + Colormap, Gravity, Reparent, Property, Destroy, Activate, + Deactivate - for window events. + + If a callback function for one of these events is registered + using bind, bind_all, bind_class, or tag_bind, the callback is + called with an Event as first argument. It will have the + following attributes (in braces are the event types for which + the attribute is valid): + + serial - serial number of event + num - mouse button pressed (ButtonPress, ButtonRelease) + focus - whether the window has the focus (Enter, Leave) + height - height of the exposed window (Configure, Expose) + width - width of the exposed window (Configure, Expose) + keycode - keycode of the pressed key (KeyPress, KeyRelease) + state - state of the event as a number (ButtonPress, ButtonRelease, + Enter, KeyPress, KeyRelease, + Leave, Motion) + state - state as a string (Visibility) + time - when the event occurred + x - x-position of the mouse + y - y-position of the mouse + x_root - x-position of the mouse on the screen + (ButtonPress, ButtonRelease, KeyPress, KeyRelease, Motion) + y_root - y-position of the mouse on the screen + (ButtonPress, ButtonRelease, KeyPress, KeyRelease, Motion) + char - pressed character (KeyPress, KeyRelease) + send_event - see X/Windows documentation + keysym - keysym of the event as a string (KeyPress, KeyRelease) + keysym_num - keysym of the event as a number (KeyPress, KeyRelease) + type - type of the event as a number + widget - widget in which the event occurred + delta - delta of wheel movement (MouseWheel) + """ + + def __repr__(self): + attrs = {k: v for k, v in self.__dict__.items() if v != '??'} + if not self.char: + del attrs['char'] + elif self.char != '??': + attrs['char'] = repr(self.char) + if not getattr(self, 'send_event', True): + del attrs['send_event'] + if self.state == 0: + del attrs['state'] + elif isinstance(self.state, int): + state = self.state + mods = ('Shift', 'Lock', 'Control', + 'Mod1', 'Mod2', 'Mod3', 'Mod4', 'Mod5', + 'Button1', 'Button2', 'Button3', 'Button4', 'Button5') + s = [] + for i, n in enumerate(mods): + if state & (1 << i): + s.append(n) + state = state & ~((1<< len(mods)) - 1) + if state or not s: + s.append(hex(state)) + attrs['state'] = '|'.join(s) + if self.delta == 0: + del attrs['delta'] + # widget usually is known + # serial and time are not very interesting + # keysym_num duplicates keysym + # x_root and y_root mostly duplicate x and y + keys = ('send_event', + 'state', 'keysym', 'keycode', 'char', + 'num', 'delta', 'focus', + 'x', 'y', 'width', 'height') + return '<%s event%s>' % ( + getattr(self.type, 'name', self.type), + ''.join(' %s=%s' % (k, attrs[k]) for k in keys if k in attrs) + ) + + +_support_default_root = True +_default_root = None + + +def NoDefaultRoot(): + """Inhibit setting of default root window. + + Call this function to inhibit that the first instance of + Tk is used for windows without an explicit parent window. + """ + global _support_default_root, _default_root + _support_default_root = False + # Delete, so any use of _default_root will immediately raise an exception. + # Rebind before deletion, so repeated calls will not fail. + _default_root = None + del _default_root + + +def _get_default_root(what=None): + if not _support_default_root: + raise RuntimeError("No master specified and tkinter is " + "configured to not support default root") + if _default_root is None: + if what: + raise RuntimeError(f"Too early to {what}: no default root window") + root = Tk() + assert _default_root is root + return _default_root + + +def _get_temp_root(): + global _support_default_root + if not _support_default_root: + raise RuntimeError("No master specified and tkinter is " + "configured to not support default root") + root = _default_root + if root is None: + assert _support_default_root + _support_default_root = False + root = Tk() + _support_default_root = True + assert _default_root is None + root.withdraw() + root._temporary = True + return root + + +def _destroy_temp_root(master): + if getattr(master, '_temporary', False): + try: + master.destroy() + except TclError: + pass + + +def _tkerror(err): + """Internal function.""" + pass + + +def _exit(code=0): + """Internal function. Calling it will raise the exception SystemExit.""" + try: + code = int(code) + except ValueError: + pass + raise SystemExit(code) + + +_varnum = 0 + + +class Variable: + """Class to define value holders for e.g. buttons. + + Subclasses StringVar, IntVar, DoubleVar, BooleanVar are specializations + that constrain the type of the value returned from get().""" + _default = "" + _tk = None + _tclCommands = None + + def __init__(self, master=None, value=None, name=None): + """Construct a variable + + MASTER can be given as master widget. + VALUE is an optional value (defaults to "") + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + # check for type of NAME parameter to override weird error message + # raised from Modules/_tkinter.c:SetVar like: + # TypeError: setvar() takes exactly 3 arguments (2 given) + if name is not None and not isinstance(name, str): + raise TypeError("name must be a string") + global _varnum + if master is None: + master = _get_default_root('create variable') + self._root = master._root() + self._tk = master.tk + if name: + self._name = name + else: + self._name = 'PY_VAR' + repr(_varnum) + _varnum += 1 + if value is not None: + self.initialize(value) + elif not self._tk.getboolean(self._tk.call("info", "exists", self._name)): + self.initialize(self._default) + + def __del__(self): + """Unset the variable in Tcl.""" + if self._tk is None: + return + if self._tk.getboolean(self._tk.call("info", "exists", self._name)): + self._tk.globalunsetvar(self._name) + if self._tclCommands is not None: + for name in self._tclCommands: + self._tk.deletecommand(name) + self._tclCommands = None + + def __str__(self): + """Return the name of the variable in Tcl.""" + return self._name + + def set(self, value): + """Set the variable to VALUE.""" + return self._tk.globalsetvar(self._name, value) + + initialize = set + + def get(self): + """Return value of variable.""" + return self._tk.globalgetvar(self._name) + + def _register(self, callback): + f = CallWrapper(callback, None, self._root).__call__ + cbname = repr(id(f)) + try: + callback = callback.__func__ + except AttributeError: + pass + try: + cbname = cbname + callback.__name__ + except AttributeError: + pass + self._tk.createcommand(cbname, f) + if self._tclCommands is None: + self._tclCommands = [] + self._tclCommands.append(cbname) + return cbname + + def trace_add(self, mode, callback): + """Define a trace callback for the variable. + + Mode is one of "read", "write", "unset", or a list or tuple of + such strings. + Callback must be a function which is called when the variable is + read, written or unset. + + Return the name of the callback. + """ + cbname = self._register(callback) + self._tk.call('trace', 'add', 'variable', + self._name, mode, (cbname,)) + return cbname + + def trace_remove(self, mode, cbname): + """Delete the trace callback for a variable. + + Mode is one of "read", "write", "unset" or a list or tuple of + such strings. Must be same as were specified in trace_add(). + cbname is the name of the callback returned from trace_add(). + """ + self._tk.call('trace', 'remove', 'variable', + self._name, mode, cbname) + for m, ca in self.trace_info(): + if self._tk.splitlist(ca)[0] == cbname: + break + else: + self._tk.deletecommand(cbname) + try: + self._tclCommands.remove(cbname) + except ValueError: + pass + + def trace_info(self): + """Return all trace callback information.""" + splitlist = self._tk.splitlist + return [(splitlist(k), v) for k, v in map(splitlist, + splitlist(self._tk.call('trace', 'info', 'variable', self._name)))] + + def trace_variable(self, mode, callback): + """Define a trace callback for the variable. + + MODE is one of "r", "w", "u" for read, write, undefine. + CALLBACK must be a function which is called when + the variable is read, written or undefined. + + Return the name of the callback. + + This deprecated method wraps a deprecated Tcl method that will + likely be removed in the future. Use trace_add() instead. + """ + # TODO: Add deprecation warning + cbname = self._register(callback) + self._tk.call("trace", "variable", self._name, mode, cbname) + return cbname + + trace = trace_variable + + def trace_vdelete(self, mode, cbname): + """Delete the trace callback for a variable. + + MODE is one of "r", "w", "u" for read, write, undefine. + CBNAME is the name of the callback returned from trace_variable or trace. + + This deprecated method wraps a deprecated Tcl method that will + likely be removed in the future. Use trace_remove() instead. + """ + # TODO: Add deprecation warning + self._tk.call("trace", "vdelete", self._name, mode, cbname) + cbname = self._tk.splitlist(cbname)[0] + for m, ca in self.trace_info(): + if self._tk.splitlist(ca)[0] == cbname: + break + else: + self._tk.deletecommand(cbname) + try: + self._tclCommands.remove(cbname) + except ValueError: + pass + + def trace_vinfo(self): + """Return all trace callback information. + + This deprecated method wraps a deprecated Tcl method that will + likely be removed in the future. Use trace_info() instead. + """ + # TODO: Add deprecation warning + return [self._tk.splitlist(x) for x in self._tk.splitlist( + self._tk.call("trace", "vinfo", self._name))] + + def __eq__(self, other): + if not isinstance(other, Variable): + return NotImplemented + return (self._name == other._name + and self.__class__.__name__ == other.__class__.__name__ + and self._tk == other._tk) + + +class StringVar(Variable): + """Value holder for strings variables.""" + _default = "" + + def __init__(self, master=None, value=None, name=None): + """Construct a string variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to "") + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return value of variable as string.""" + value = self._tk.globalgetvar(self._name) + if isinstance(value, str): + return value + return str(value) + + +class IntVar(Variable): + """Value holder for integer variables.""" + _default = 0 + + def __init__(self, master=None, value=None, name=None): + """Construct an integer variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to 0) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return the value of the variable as an integer.""" + value = self._tk.globalgetvar(self._name) + try: + return self._tk.getint(value) + except (TypeError, TclError): + return int(self._tk.getdouble(value)) + + +class DoubleVar(Variable): + """Value holder for float variables.""" + _default = 0.0 + + def __init__(self, master=None, value=None, name=None): + """Construct a float variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to 0.0) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def get(self): + """Return the value of the variable as a float.""" + return self._tk.getdouble(self._tk.globalgetvar(self._name)) + + +class BooleanVar(Variable): + """Value holder for boolean variables.""" + _default = False + + def __init__(self, master=None, value=None, name=None): + """Construct a boolean variable. + + MASTER can be given as master widget. + VALUE is an optional value (defaults to False) + NAME is an optional Tcl name (defaults to PY_VARnum). + + If NAME matches an existing variable and VALUE is omitted + then the existing value is retained. + """ + Variable.__init__(self, master, value, name) + + def set(self, value): + """Set the variable to VALUE.""" + return self._tk.globalsetvar(self._name, self._tk.getboolean(value)) + + initialize = set + + def get(self): + """Return the value of the variable as a bool.""" + try: + return self._tk.getboolean(self._tk.globalgetvar(self._name)) + except TclError: + raise ValueError("invalid literal for getboolean()") + + +def mainloop(n=0): + """Run the main loop of Tcl.""" + _get_default_root('run the main loop').tk.mainloop(n) + + +getint = int + +getdouble = float + + +def getboolean(s): + """Convert Tcl object to True or False.""" + try: + return _get_default_root('use getboolean()').tk.getboolean(s) + except TclError: + raise ValueError("invalid literal for getboolean()") + + +# Methods defined on both toplevel and interior widgets + +class Misc: + """Internal class. + + Base class which defines methods common for interior widgets.""" + + # used for generating child widget names + _last_child_ids = None + + # XXX font command? + _tclCommands = None + + def destroy(self): + """Internal function. + + Delete all Tcl commands created for + this widget in the Tcl interpreter.""" + if self._tclCommands is not None: + for name in self._tclCommands: + self.tk.deletecommand(name) + self._tclCommands = None + + def deletecommand(self, name): + """Internal function. + + Delete the Tcl command provided in NAME.""" + self.tk.deletecommand(name) + try: + self._tclCommands.remove(name) + except ValueError: + pass + + def tk_strictMotif(self, boolean=None): + """Set Tcl internal variable, whether the look and feel + should adhere to Motif. + + A parameter of 1 means adhere to Motif (e.g. no color + change if mouse passes over slider). + Returns the set value.""" + return self.tk.getboolean(self.tk.call( + 'set', 'tk_strictMotif', boolean)) + + def tk_bisque(self): + """Change the color scheme to light brown as used in Tk 3.6 and before.""" + self.tk.call('tk_bisque') + + def tk_setPalette(self, *args, **kw): + """Set a new color scheme for all widget elements. + + A single color as argument will cause that all colors of Tk + widget elements are derived from this. + Alternatively several keyword parameters and its associated + colors can be given. The following keywords are valid: + activeBackground, foreground, selectColor, + activeForeground, highlightBackground, selectBackground, + background, highlightColor, selectForeground, + disabledForeground, insertBackground, troughColor.""" + self.tk.call(('tk_setPalette',) + + _flatten(args) + _flatten(list(kw.items()))) + + def wait_variable(self, name='PY_VAR'): + """Wait until the variable is modified. + + A parameter of type IntVar, StringVar, DoubleVar or + BooleanVar must be given.""" + self.tk.call('tkwait', 'variable', name) + waitvar = wait_variable # XXX b/w compat + + def wait_window(self, window=None): + """Wait until a WIDGET is destroyed. + + If no parameter is given self is used.""" + if window is None: + window = self + self.tk.call('tkwait', 'window', window._w) + + def wait_visibility(self, window=None): + """Wait until the visibility of a WIDGET changes + (e.g. it appears). + + If no parameter is given self is used.""" + if window is None: + window = self + self.tk.call('tkwait', 'visibility', window._w) + + def setvar(self, name='PY_VAR', value='1'): + """Set Tcl variable NAME to VALUE.""" + self.tk.setvar(name, value) + + def getvar(self, name='PY_VAR'): + """Return value of Tcl variable NAME.""" + return self.tk.getvar(name) + + def getint(self, s): + try: + return self.tk.getint(s) + except TclError as exc: + raise ValueError(str(exc)) + + def getdouble(self, s): + try: + return self.tk.getdouble(s) + except TclError as exc: + raise ValueError(str(exc)) + + def getboolean(self, s): + """Return a boolean value for Tcl boolean values true and false given as parameter.""" + try: + return self.tk.getboolean(s) + except TclError: + raise ValueError("invalid literal for getboolean()") + + def focus_set(self): + """Direct input focus to this widget. + + If the application currently does not have the focus + this widget will get the focus if the application gets + the focus through the window manager.""" + self.tk.call('focus', self._w) + focus = focus_set # XXX b/w compat? + + def focus_force(self): + """Direct input focus to this widget even if the + application does not have the focus. Use with + caution!""" + self.tk.call('focus', '-force', self._w) + + def focus_get(self): + """Return the widget which has currently the focus in the + application. + + Use focus_displayof to allow working with several + displays. Return None if application does not have + the focus.""" + name = self.tk.call('focus') + if name == 'none' or not name: return None + return self._nametowidget(name) + + def focus_displayof(self): + """Return the widget which has currently the focus on the + display where this widget is located. + + Return None if the application does not have the focus.""" + name = self.tk.call('focus', '-displayof', self._w) + if name == 'none' or not name: return None + return self._nametowidget(name) + + def focus_lastfor(self): + """Return the widget which would have the focus if top level + for this widget gets the focus from the window manager.""" + name = self.tk.call('focus', '-lastfor', self._w) + if name == 'none' or not name: return None + return self._nametowidget(name) + + def tk_focusFollowsMouse(self): + """The widget under mouse will get automatically focus. Can not + be disabled easily.""" + self.tk.call('tk_focusFollowsMouse') + + def tk_focusNext(self): + """Return the next widget in the focus order which follows + widget which has currently the focus. + + The focus order first goes to the next child, then to + the children of the child recursively and then to the + next sibling which is higher in the stacking order. A + widget is omitted if it has the takefocus resource set + to 0.""" + name = self.tk.call('tk_focusNext', self._w) + if not name: return None + return self._nametowidget(name) + + def tk_focusPrev(self): + """Return previous widget in the focus order. See tk_focusNext for details.""" + name = self.tk.call('tk_focusPrev', self._w) + if not name: return None + return self._nametowidget(name) + + def after(self, ms, func=None, *args): + """Call function once after given time. + + MS specifies the time in milliseconds. FUNC gives the + function which shall be called. Additional parameters + are given as parameters to the function call. Return + identifier to cancel scheduling with after_cancel.""" + if func is None: + # I'd rather use time.sleep(ms*0.001) + self.tk.call('after', ms) + return None + else: + def callit(): + try: + func(*args) + finally: + try: + self.deletecommand(name) + except TclError: + pass + try: + callit.__name__ = func.__name__ + except AttributeError: + # Required for callable classes (bpo-44404) + callit.__name__ = type(func).__name__ + name = self._register(callit) + return self.tk.call('after', ms, name) + + def after_idle(self, func, *args): + """Call FUNC once if the Tcl main loop has no event to + process. + + Return an identifier to cancel the scheduling with + after_cancel.""" + return self.after('idle', func, *args) + + def after_cancel(self, id): + """Cancel scheduling of function identified with ID. + + Identifier returned by after or after_idle must be + given as first parameter. + """ + if not id: + raise ValueError('id must be a valid identifier returned from ' + 'after or after_idle') + try: + data = self.tk.call('after', 'info', id) + script = self.tk.splitlist(data)[0] + self.deletecommand(script) + except TclError: + pass + self.tk.call('after', 'cancel', id) + + def after_info(self, id=None): + """Return information about existing event handlers. + + With no argument, return a tuple of the identifiers for all existing + event handlers created by the after and after_idle commands for this + interpreter. If id is supplied, it specifies an existing handler; id + must have been the return value from some previous call to after or + after_idle and it must not have triggered yet or been canceled. If the + id doesn't exist, a TclError is raised. Otherwise, the return value is + a tuple containing (script, type) where script is a reference to the + function to be called by the event handler and type is either 'idle' + or 'timer' to indicate what kind of event handler it is. + """ + return self.tk.splitlist(self.tk.call('after', 'info', id)) + + def bell(self, displayof=0): + """Ring a display's bell.""" + self.tk.call(('bell',) + self._displayof(displayof)) + + def tk_busy_cget(self, option): + """Return the value of busy configuration option. + + The widget must have been previously made busy by + tk_busy_hold(). Option may have any of the values accepted by + tk_busy_hold(). + """ + return self.tk.call('tk', 'busy', 'cget', self._w, '-'+option) + busy_cget = tk_busy_cget + + def tk_busy_configure(self, cnf=None, **kw): + """Query or modify the busy configuration options. + + The widget must have been previously made busy by + tk_busy_hold(). Options may have any of the values accepted by + tk_busy_hold(). + + Please note that the option database is referenced by the widget + name or class. For example, if a Frame widget with name "frame" + is to be made busy, the busy cursor can be specified for it by + either call: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + """ + if kw: + cnf = _cnfmerge((cnf, kw)) + elif cnf: + cnf = _cnfmerge(cnf) + if cnf is None: + return self._getconfigure( + 'tk', 'busy', 'configure', self._w) + if isinstance(cnf, str): + return self._getconfigure1( + 'tk', 'busy', 'configure', self._w, '-'+cnf) + self.tk.call('tk', 'busy', 'configure', self._w, *self._options(cnf)) + busy_config = busy_configure = tk_busy_config = tk_busy_configure + + def tk_busy_current(self, pattern=None): + """Return a list of widgets that are currently busy. + + If a pattern is given, only busy widgets whose path names match + a pattern are returned. + """ + return [self._nametowidget(x) for x in + self.tk.splitlist(self.tk.call( + 'tk', 'busy', 'current', pattern))] + busy_current = tk_busy_current + + def tk_busy_forget(self): + """Make this widget no longer busy. + + User events will again be received by the widget. + """ + self.tk.call('tk', 'busy', 'forget', self._w) + busy_forget = tk_busy_forget + + def tk_busy_hold(self, **kw): + """Make this widget appear busy. + + The specified widget and its descendants will be blocked from + user interactions. Normally update() should be called + immediately afterward to insure that the hold operation is in + effect before the application starts its processing. + + The only supported configuration option is: + + cursor: the cursor to be displayed when the widget is made + busy. + """ + self.tk.call('tk', 'busy', 'hold', self._w, *self._options(kw)) + busy = busy_hold = tk_busy = tk_busy_hold + + def tk_busy_status(self): + """Return True if the widget is busy, False otherwise.""" + return self.tk.getboolean(self.tk.call( + 'tk', 'busy', 'status', self._w)) + busy_status = tk_busy_status + + # Clipboard handling: + def clipboard_get(self, **kw): + """Retrieve data from the clipboard on window's display. + + The window keyword defaults to the root window of the Tkinter + application. + + The type keyword specifies the form in which the data is + to be returned and should be an atom name such as STRING + or FILE_NAME. Type defaults to STRING, except on X11, where the default + is to try UTF8_STRING and fall back to STRING. + + This command is equivalent to: + + selection_get(CLIPBOARD) + """ + if 'type' not in kw and self._windowingsystem == 'x11': + try: + kw['type'] = 'UTF8_STRING' + return self.tk.call(('clipboard', 'get') + self._options(kw)) + except TclError: + del kw['type'] + return self.tk.call(('clipboard', 'get') + self._options(kw)) + + def clipboard_clear(self, **kw): + """Clear the data in the Tk clipboard. + + A widget specified for the optional displayof keyword + argument specifies the target display.""" + if 'displayof' not in kw: kw['displayof'] = self._w + self.tk.call(('clipboard', 'clear') + self._options(kw)) + + def clipboard_append(self, string, **kw): + """Append STRING to the Tk clipboard. + + A widget specified at the optional displayof keyword + argument specifies the target display. The clipboard + can be retrieved with selection_get.""" + if 'displayof' not in kw: kw['displayof'] = self._w + self.tk.call(('clipboard', 'append') + self._options(kw) + + ('--', string)) + # XXX grab current w/o window argument + + def grab_current(self): + """Return widget which has currently the grab in this application + or None.""" + name = self.tk.call('grab', 'current', self._w) + if not name: return None + return self._nametowidget(name) + + def grab_release(self): + """Release grab for this widget if currently set.""" + self.tk.call('grab', 'release', self._w) + + def grab_set(self): + """Set grab for this widget. + + A grab directs all events to this and descendant + widgets in the application.""" + self.tk.call('grab', 'set', self._w) + + def grab_set_global(self): + """Set global grab for this widget. + + A global grab directs all events to this and + descendant widgets on the display. Use with caution - + other applications do not get events anymore.""" + self.tk.call('grab', 'set', '-global', self._w) + + def grab_status(self): + """Return None, "local" or "global" if this widget has + no, a local or a global grab.""" + status = self.tk.call('grab', 'status', self._w) + if status == 'none': status = None + return status + + def option_add(self, pattern, value, priority = None): + """Set a VALUE (second parameter) for an option + PATTERN (first parameter). + + An optional third parameter gives the numeric priority + (defaults to 80).""" + self.tk.call('option', 'add', pattern, value, priority) + + def option_clear(self): + """Clear the option database. + + It will be reloaded if option_add is called.""" + self.tk.call('option', 'clear') + + def option_get(self, name, className): + """Return the value for an option NAME for this widget + with CLASSNAME. + + Values with higher priority override lower values.""" + return self.tk.call('option', 'get', self._w, name, className) + + def option_readfile(self, fileName, priority = None): + """Read file FILENAME into the option database. + + An optional second parameter gives the numeric + priority.""" + self.tk.call('option', 'readfile', fileName, priority) + + def selection_clear(self, **kw): + """Clear the current X selection.""" + if 'displayof' not in kw: kw['displayof'] = self._w + self.tk.call(('selection', 'clear') + self._options(kw)) + + def selection_get(self, **kw): + """Return the contents of the current X selection. + + A keyword parameter selection specifies the name of + the selection and defaults to PRIMARY. A keyword + parameter displayof specifies a widget on the display + to use. A keyword parameter type specifies the form of data to be + fetched, defaulting to STRING except on X11, where UTF8_STRING is tried + before STRING.""" + if 'displayof' not in kw: kw['displayof'] = self._w + if 'type' not in kw and self._windowingsystem == 'x11': + try: + kw['type'] = 'UTF8_STRING' + return self.tk.call(('selection', 'get') + self._options(kw)) + except TclError: + del kw['type'] + return self.tk.call(('selection', 'get') + self._options(kw)) + + def selection_handle(self, command, **kw): + """Specify a function COMMAND to call if the X + selection owned by this widget is queried by another + application. + + This function must return the contents of the + selection. The function will be called with the + arguments OFFSET and LENGTH which allows the chunking + of very long selections. The following keyword + parameters can be provided: + selection - name of the selection (default PRIMARY), + type - type of the selection (e.g. STRING, FILE_NAME).""" + name = self._register(command) + self.tk.call(('selection', 'handle') + self._options(kw) + + (self._w, name)) + + def selection_own(self, **kw): + """Become owner of X selection. + + A keyword parameter selection specifies the name of + the selection (default PRIMARY).""" + self.tk.call(('selection', 'own') + + self._options(kw) + (self._w,)) + + def selection_own_get(self, **kw): + """Return owner of X selection. + + The following keyword parameter can + be provided: + selection - name of the selection (default PRIMARY), + type - type of the selection (e.g. STRING, FILE_NAME).""" + if 'displayof' not in kw: kw['displayof'] = self._w + name = self.tk.call(('selection', 'own') + self._options(kw)) + if not name: return None + return self._nametowidget(name) + + def send(self, interp, cmd, *args): + """Send Tcl command CMD to different interpreter INTERP to be executed.""" + return self.tk.call(('send', interp, cmd) + args) + + def lower(self, belowThis=None): + """Lower this widget in the stacking order.""" + self.tk.call('lower', self._w, belowThis) + + def tkraise(self, aboveThis=None): + """Raise this widget in the stacking order.""" + self.tk.call('raise', self._w, aboveThis) + + lift = tkraise + + def info_patchlevel(self): + """Returns the exact version of the Tcl library.""" + patchlevel = self.tk.call('info', 'patchlevel') + return _parse_version(patchlevel) + + def winfo_atom(self, name, displayof=0): + """Return integer which represents atom NAME.""" + args = ('winfo', 'atom') + self._displayof(displayof) + (name,) + return self.tk.getint(self.tk.call(args)) + + def winfo_atomname(self, id, displayof=0): + """Return name of atom with identifier ID.""" + args = ('winfo', 'atomname') \ + + self._displayof(displayof) + (id,) + return self.tk.call(args) + + def winfo_cells(self): + """Return number of cells in the colormap for this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'cells', self._w)) + + def winfo_children(self): + """Return a list of all widgets which are children of this widget.""" + result = [] + for child in self.tk.splitlist( + self.tk.call('winfo', 'children', self._w)): + try: + # Tcl sometimes returns extra windows, e.g. for + # menus; those need to be skipped + result.append(self._nametowidget(child)) + except KeyError: + pass + return result + + def winfo_class(self): + """Return window class name of this widget.""" + return self.tk.call('winfo', 'class', self._w) + + def winfo_colormapfull(self): + """Return True if at the last color request the colormap was full.""" + return self.tk.getboolean( + self.tk.call('winfo', 'colormapfull', self._w)) + + def winfo_containing(self, rootX, rootY, displayof=0): + """Return the widget which is at the root coordinates ROOTX, ROOTY.""" + args = ('winfo', 'containing') \ + + self._displayof(displayof) + (rootX, rootY) + name = self.tk.call(args) + if not name: return None + return self._nametowidget(name) + + def winfo_depth(self): + """Return the number of bits per pixel.""" + return self.tk.getint(self.tk.call('winfo', 'depth', self._w)) + + def winfo_exists(self): + """Return true if this widget exists.""" + return self.tk.getint( + self.tk.call('winfo', 'exists', self._w)) + + def winfo_fpixels(self, number): + """Return the number of pixels for the given distance NUMBER + (e.g. "3c") as float.""" + return self.tk.getdouble(self.tk.call( + 'winfo', 'fpixels', self._w, number)) + + def winfo_geometry(self): + """Return geometry string for this widget in the form "widthxheight+X+Y".""" + return self.tk.call('winfo', 'geometry', self._w) + + def winfo_height(self): + """Return height of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'height', self._w)) + + def winfo_id(self): + """Return identifier ID for this widget.""" + return int(self.tk.call('winfo', 'id', self._w), 0) + + def winfo_interps(self, displayof=0): + """Return the name of all Tcl interpreters for this display.""" + args = ('winfo', 'interps') + self._displayof(displayof) + return self.tk.splitlist(self.tk.call(args)) + + def winfo_ismapped(self): + """Return true if this widget is mapped.""" + return self.tk.getint( + self.tk.call('winfo', 'ismapped', self._w)) + + def winfo_manager(self): + """Return the window manager name for this widget.""" + return self.tk.call('winfo', 'manager', self._w) + + def winfo_name(self): + """Return the name of this widget.""" + return self.tk.call('winfo', 'name', self._w) + + def winfo_parent(self): + """Return the name of the parent of this widget.""" + return self.tk.call('winfo', 'parent', self._w) + + def winfo_pathname(self, id, displayof=0): + """Return the pathname of the widget given by ID.""" + if isinstance(id, int): + id = hex(id) + args = ('winfo', 'pathname') \ + + self._displayof(displayof) + (id,) + return self.tk.call(args) + + def winfo_pixels(self, number): + """Rounded integer value of winfo_fpixels.""" + return self.tk.getint( + self.tk.call('winfo', 'pixels', self._w, number)) + + def winfo_pointerx(self): + """Return the x coordinate of the pointer on the root window.""" + return self.tk.getint( + self.tk.call('winfo', 'pointerx', self._w)) + + def winfo_pointerxy(self): + """Return a tuple of x and y coordinates of the pointer on the root window.""" + return self._getints( + self.tk.call('winfo', 'pointerxy', self._w)) + + def winfo_pointery(self): + """Return the y coordinate of the pointer on the root window.""" + return self.tk.getint( + self.tk.call('winfo', 'pointery', self._w)) + + def winfo_reqheight(self): + """Return requested height of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'reqheight', self._w)) + + def winfo_reqwidth(self): + """Return requested width of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'reqwidth', self._w)) + + def winfo_rgb(self, color): + """Return a tuple of integer RGB values in range(65536) for color in this widget.""" + return self._getints( + self.tk.call('winfo', 'rgb', self._w, color)) + + def winfo_rootx(self): + """Return x coordinate of upper left corner of this widget on the + root window.""" + return self.tk.getint( + self.tk.call('winfo', 'rootx', self._w)) + + def winfo_rooty(self): + """Return y coordinate of upper left corner of this widget on the + root window.""" + return self.tk.getint( + self.tk.call('winfo', 'rooty', self._w)) + + def winfo_screen(self): + """Return the screen name of this widget.""" + return self.tk.call('winfo', 'screen', self._w) + + def winfo_screencells(self): + """Return the number of the cells in the colormap of the screen + of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'screencells', self._w)) + + def winfo_screendepth(self): + """Return the number of bits per pixel of the root window of the + screen of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'screendepth', self._w)) + + def winfo_screenheight(self): + """Return the number of pixels of the height of the screen of this widget + in pixel.""" + return self.tk.getint( + self.tk.call('winfo', 'screenheight', self._w)) + + def winfo_screenmmheight(self): + """Return the number of pixels of the height of the screen of + this widget in mm.""" + return self.tk.getint( + self.tk.call('winfo', 'screenmmheight', self._w)) + + def winfo_screenmmwidth(self): + """Return the number of pixels of the width of the screen of + this widget in mm.""" + return self.tk.getint( + self.tk.call('winfo', 'screenmmwidth', self._w)) + + def winfo_screenvisual(self): + """Return one of the strings directcolor, grayscale, pseudocolor, + staticcolor, staticgray, or truecolor for the default + colormodel of this screen.""" + return self.tk.call('winfo', 'screenvisual', self._w) + + def winfo_screenwidth(self): + """Return the number of pixels of the width of the screen of + this widget in pixel.""" + return self.tk.getint( + self.tk.call('winfo', 'screenwidth', self._w)) + + def winfo_server(self): + """Return information of the X-Server of the screen of this widget in + the form "XmajorRminor vendor vendorVersion".""" + return self.tk.call('winfo', 'server', self._w) + + def winfo_toplevel(self): + """Return the toplevel widget of this widget.""" + return self._nametowidget(self.tk.call( + 'winfo', 'toplevel', self._w)) + + def winfo_viewable(self): + """Return true if the widget and all its higher ancestors are mapped.""" + return self.tk.getint( + self.tk.call('winfo', 'viewable', self._w)) + + def winfo_visual(self): + """Return one of the strings directcolor, grayscale, pseudocolor, + staticcolor, staticgray, or truecolor for the + colormodel of this widget.""" + return self.tk.call('winfo', 'visual', self._w) + + def winfo_visualid(self): + """Return the X identifier for the visual for this widget.""" + return self.tk.call('winfo', 'visualid', self._w) + + def winfo_visualsavailable(self, includeids=False): + """Return a list of all visuals available for the screen + of this widget. + + Each item in the list consists of a visual name (see winfo_visual), a + depth and if includeids is true is given also the X identifier.""" + data = self.tk.call('winfo', 'visualsavailable', self._w, + 'includeids' if includeids else None) + data = [self.tk.splitlist(x) for x in self.tk.splitlist(data)] + return [self.__winfo_parseitem(x) for x in data] + + def __winfo_parseitem(self, t): + """Internal function.""" + return t[:1] + tuple(map(self.__winfo_getint, t[1:])) + + def __winfo_getint(self, x): + """Internal function.""" + return int(x, 0) + + def winfo_vrootheight(self): + """Return the height of the virtual root window associated with this + widget in pixels. If there is no virtual root window return the + height of the screen.""" + return self.tk.getint( + self.tk.call('winfo', 'vrootheight', self._w)) + + def winfo_vrootwidth(self): + """Return the width of the virtual root window associated with this + widget in pixel. If there is no virtual root window return the + width of the screen.""" + return self.tk.getint( + self.tk.call('winfo', 'vrootwidth', self._w)) + + def winfo_vrootx(self): + """Return the x offset of the virtual root relative to the root + window of the screen of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'vrootx', self._w)) + + def winfo_vrooty(self): + """Return the y offset of the virtual root relative to the root + window of the screen of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'vrooty', self._w)) + + def winfo_width(self): + """Return the width of this widget.""" + return self.tk.getint( + self.tk.call('winfo', 'width', self._w)) + + def winfo_x(self): + """Return the x coordinate of the upper left corner of this widget + in the parent.""" + return self.tk.getint( + self.tk.call('winfo', 'x', self._w)) + + def winfo_y(self): + """Return the y coordinate of the upper left corner of this widget + in the parent.""" + return self.tk.getint( + self.tk.call('winfo', 'y', self._w)) + + def update(self): + """Enter event loop until all pending events have been processed by Tcl.""" + self.tk.call('update') + + def update_idletasks(self): + """Enter event loop until all idle callbacks have been called. This + will update the display of windows but not process events caused by + the user.""" + self.tk.call('update', 'idletasks') + + def bindtags(self, tagList=None): + """Set or get the list of bindtags for this widget. + + With no argument return the list of all bindtags associated with + this widget. With a list of strings as argument the bindtags are + set to this list. The bindtags determine in which order events are + processed (see bind).""" + if tagList is None: + return self.tk.splitlist( + self.tk.call('bindtags', self._w)) + else: + self.tk.call('bindtags', self._w, tagList) + + def _bind(self, what, sequence, func, add, needcleanup=1): + """Internal function.""" + if isinstance(func, str): + self.tk.call(what + (sequence, func)) + elif func: + funcid = self._register(func, self._substitute, + needcleanup) + cmd = ('%sif {"[%s %s]" == "break"} break\n' + % + (add and '+' or '', + funcid, self._subst_format_str)) + self.tk.call(what + (sequence, cmd)) + return funcid + elif sequence: + return self.tk.call(what + (sequence,)) + else: + return self.tk.splitlist(self.tk.call(what)) + + def bind(self, sequence=None, func=None, add=None): + """Bind to this widget at event SEQUENCE a call to function FUNC. + + SEQUENCE is a string of concatenated event + patterns. An event pattern is of the form + <MODIFIER-MODIFIER-TYPE-DETAIL> where MODIFIER is one + of Control, Mod2, M2, Shift, Mod3, M3, Lock, Mod4, M4, + Button1, B1, Mod5, M5 Button2, B2, Meta, M, Button3, + B3, Alt, Button4, B4, Double, Button5, B5 Triple, + Mod1, M1. TYPE is one of Activate, Enter, Map, + ButtonPress, Button, Expose, Motion, ButtonRelease + FocusIn, MouseWheel, Circulate, FocusOut, Property, + Colormap, Gravity Reparent, Configure, KeyPress, Key, + Unmap, Deactivate, KeyRelease Visibility, Destroy, + Leave and DETAIL is the button number for ButtonPress, + ButtonRelease and DETAIL is the Keysym for KeyPress and + KeyRelease. Examples are + <Control-Button-1> for pressing Control and mouse button 1 or + <Alt-A> for pressing A and the Alt key (KeyPress can be omitted). + An event pattern can also be a virtual event of the form + <<AString>> where AString can be arbitrary. This + event can be generated by event_generate. + If events are concatenated they must appear shortly + after each other. + + FUNC will be called if the event sequence occurs with an + instance of Event as argument. If the return value of FUNC is + "break" no further bound function is invoked. + + An additional boolean parameter ADD specifies whether FUNC will + be called additionally to the other bound function or whether + it will replace the previous function. + + Bind will return an identifier to allow deletion of the bound function with + unbind without memory leak. + + If FUNC or SEQUENCE is omitted the bound function or list + of bound events are returned.""" + + return self._bind(('bind', self._w), sequence, func, add) + + def unbind(self, sequence, funcid=None): + """Unbind for this widget the event SEQUENCE. + + If FUNCID is given, only unbind the function identified with FUNCID + and also delete the corresponding Tcl command. + + Otherwise destroy the current binding for SEQUENCE, leaving SEQUENCE + unbound. + """ + self._unbind(('bind', self._w, sequence), funcid) + + def _unbind(self, what, funcid=None): + if funcid is None: + self.tk.call(*what, '') + else: + lines = self.tk.call(what).split('\n') + prefix = f'if {{"[{funcid} ' + keep = '\n'.join(line for line in lines + if not line.startswith(prefix)) + if not keep.strip(): + keep = '' + self.tk.call(*what, keep) + self.deletecommand(funcid) + + def bind_all(self, sequence=None, func=None, add=None): + """Bind to all widgets at an event SEQUENCE a call to function FUNC. + An additional boolean parameter ADD specifies whether FUNC will + be called additionally to the other bound function or whether + it will replace the previous function. See bind for the return value.""" + return self._root()._bind(('bind', 'all'), sequence, func, add, True) + + def unbind_all(self, sequence): + """Unbind for all widgets for event SEQUENCE all functions.""" + self._root()._unbind(('bind', 'all', sequence)) + + def bind_class(self, className, sequence=None, func=None, add=None): + """Bind to widgets with bindtag CLASSNAME at event + SEQUENCE a call of function FUNC. An additional + boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or + whether it will replace the previous function. See bind for + the return value.""" + + return self._root()._bind(('bind', className), sequence, func, add, True) + + def unbind_class(self, className, sequence): + """Unbind for all widgets with bindtag CLASSNAME for event SEQUENCE + all functions.""" + self._root()._unbind(('bind', className, sequence)) + + def mainloop(self, n=0): + """Call the mainloop of Tk.""" + self.tk.mainloop(n) + + def quit(self): + """Quit the Tcl interpreter. All widgets will be destroyed.""" + self.tk.quit() + + def _getints(self, string): + """Internal function.""" + if string: + return tuple(map(self.tk.getint, self.tk.splitlist(string))) + + def _getdoubles(self, string): + """Internal function.""" + if string: + return tuple(map(self.tk.getdouble, self.tk.splitlist(string))) + + def _getboolean(self, string): + """Internal function.""" + if string: + return self.tk.getboolean(string) + + def _displayof(self, displayof): + """Internal function.""" + if displayof: + return ('-displayof', displayof) + if displayof is None: + return ('-displayof', self._w) + return () + + @property + def _windowingsystem(self): + """Internal function.""" + try: + return self._root()._windowingsystem_cached + except AttributeError: + ws = self._root()._windowingsystem_cached = \ + self.tk.call('tk', 'windowingsystem') + return ws + + def _options(self, cnf, kw = None): + """Internal function.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + else: + cnf = _cnfmerge(cnf) + res = () + for k, v in cnf.items(): + if v is not None: + if k[-1] == '_': k = k[:-1] + if callable(v): + v = self._register(v) + elif isinstance(v, (tuple, list)): + nv = [] + for item in v: + if isinstance(item, int): + nv.append(str(item)) + elif isinstance(item, str): + nv.append(_stringify(item)) + else: + break + else: + v = ' '.join(nv) + res = res + ('-'+k, v) + return res + + def nametowidget(self, name): + """Return the Tkinter instance of a widget identified by + its Tcl name NAME.""" + name = str(name).split('.') + w = self + + if not name[0]: + w = w._root() + name = name[1:] + + for n in name: + if not n: + break + w = w.children[n] + + return w + + _nametowidget = nametowidget + + def _register(self, func, subst=None, needcleanup=1): + """Return a newly created Tcl function. If this + function is called, the Python function FUNC will + be executed. An optional function SUBST can + be given which will be executed before FUNC.""" + f = CallWrapper(func, subst, self).__call__ + name = repr(id(f)) + try: + func = func.__func__ + except AttributeError: + pass + try: + name = name + func.__name__ + except AttributeError: + pass + self.tk.createcommand(name, f) + if needcleanup: + if self._tclCommands is None: + self._tclCommands = [] + self._tclCommands.append(name) + return name + + register = _register + + def _root(self): + """Internal function.""" + w = self + while w.master is not None: w = w.master + return w + _subst_format = ('%#', '%b', '%f', '%h', '%k', + '%s', '%t', '%w', '%x', '%y', + '%A', '%E', '%K', '%N', '%W', '%T', '%X', '%Y', '%D') + _subst_format_str = " ".join(_subst_format) + + def _substitute(self, *args): + """Internal function.""" + if len(args) != len(self._subst_format): return args + getboolean = self.tk.getboolean + + getint = self.tk.getint + def getint_event(s): + """Tk changed behavior in 8.4.2, returning "??" rather more often.""" + try: + return getint(s) + except (ValueError, TclError): + return s + + if any(isinstance(s, tuple) for s in args): + args = [s[0] if isinstance(s, tuple) and len(s) == 1 else s + for s in args] + nsign, b, f, h, k, s, t, w, x, y, A, E, K, N, W, T, X, Y, D = args + # Missing: (a, c, d, m, o, v, B, R) + e = Event() + # serial field: valid for all events + # number of button: ButtonPress and ButtonRelease events only + # height field: Configure, ConfigureRequest, Create, + # ResizeRequest, and Expose events only + # keycode field: KeyPress and KeyRelease events only + # time field: "valid for events that contain a time field" + # width field: Configure, ConfigureRequest, Create, ResizeRequest, + # and Expose events only + # x field: "valid for events that contain an x field" + # y field: "valid for events that contain a y field" + # keysym as decimal: KeyPress and KeyRelease events only + # x_root, y_root fields: ButtonPress, ButtonRelease, KeyPress, + # KeyRelease, and Motion events + e.serial = getint(nsign) + e.num = getint_event(b) + try: e.focus = getboolean(f) + except TclError: pass + e.height = getint_event(h) + e.keycode = getint_event(k) + e.state = getint_event(s) + e.time = getint_event(t) + e.width = getint_event(w) + e.x = getint_event(x) + e.y = getint_event(y) + e.char = A + try: e.send_event = getboolean(E) + except TclError: pass + e.keysym = K + e.keysym_num = getint_event(N) + try: + e.type = EventType(T) + except ValueError: + try: + e.type = EventType(str(T)) # can be int + except ValueError: + e.type = T + try: + e.widget = self._nametowidget(W) + except KeyError: + e.widget = W + e.x_root = getint_event(X) + e.y_root = getint_event(Y) + try: + e.delta = getint(D) + except (ValueError, TclError): + e.delta = 0 + return (e,) + + def _report_exception(self): + """Internal function.""" + exc, val, tb = sys.exc_info() + root = self._root() + root.report_callback_exception(exc, val, tb) + + def _getconfigure(self, *args): + """Call Tcl configure command and return the result as a dict.""" + cnf = {} + for x in self.tk.splitlist(self.tk.call(*args)): + x = self.tk.splitlist(x) + cnf[x[0][1:]] = (x[0][1:],) + x[1:] + return cnf + + def _getconfigure1(self, *args): + x = self.tk.splitlist(self.tk.call(*args)) + return (x[0][1:],) + x[1:] + + def _configure(self, cmd, cnf, kw): + """Internal function.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + elif cnf: + cnf = _cnfmerge(cnf) + if cnf is None: + return self._getconfigure(_flatten((self._w, cmd))) + if isinstance(cnf, str): + return self._getconfigure1(_flatten((self._w, cmd, '-'+cnf))) + self.tk.call(_flatten((self._w, cmd)) + self._options(cnf)) + # These used to be defined in Widget: + + def configure(self, cnf=None, **kw): + """Configure resources of a widget. + + The values for resources are specified as keyword + arguments. To get an overview about + the allowed keyword arguments call the method keys. + """ + return self._configure('configure', cnf, kw) + + config = configure + + def cget(self, key): + """Return the resource value for a KEY given as string.""" + return self.tk.call(self._w, 'cget', '-' + key) + + __getitem__ = cget + + def __setitem__(self, key, value): + self.configure({key: value}) + + def keys(self): + """Return a list of all resource names of this widget.""" + splitlist = self.tk.splitlist + return [splitlist(x)[0][1:] for x in + splitlist(self.tk.call(self._w, 'configure'))] + + def __str__(self): + """Return the window path name of this widget.""" + return self._w + + def __repr__(self): + return '<%s.%s object %s>' % ( + self.__class__.__module__, self.__class__.__qualname__, self._w) + + # Pack methods that apply to the master + _noarg_ = ['_noarg_'] + + def pack_propagate(self, flag=_noarg_): + """Set or get the status for propagation of geometry information. + + A boolean argument specifies whether the geometry information + of the slaves will determine the size of this widget. If no argument + is given the current setting will be returned. + """ + if flag is Misc._noarg_: + return self._getboolean(self.tk.call( + 'pack', 'propagate', self._w)) + else: + self.tk.call('pack', 'propagate', self._w, flag) + + propagate = pack_propagate + + def pack_slaves(self): + """Return a list of all slaves of this widget + in its packing order.""" + return [self._nametowidget(x) for x in + self.tk.splitlist( + self.tk.call('pack', 'slaves', self._w))] + + slaves = pack_slaves + + # Place method that applies to the master + def place_slaves(self): + """Return a list of all slaves of this widget + in its packing order.""" + return [self._nametowidget(x) for x in + self.tk.splitlist( + self.tk.call( + 'place', 'slaves', self._w))] + + # Grid methods that apply to the master + + def grid_anchor(self, anchor=None): # new in Tk 8.5 + """The anchor value controls how to place the grid within the + master when no row/column has any weight. + + The default anchor is nw.""" + self.tk.call('grid', 'anchor', self._w, anchor) + + anchor = grid_anchor + + def grid_bbox(self, column=None, row=None, col2=None, row2=None): + """Return a tuple of integer coordinates for the bounding + box of this widget controlled by the geometry manager grid. + + If COLUMN, ROW is given the bounding box applies from + the cell with row and column 0 to the specified + cell. If COL2 and ROW2 are given the bounding box + starts at that cell. + + The returned integers specify the offset of the upper left + corner in the master widget and the width and height. + """ + args = ('grid', 'bbox', self._w) + if column is not None and row is not None: + args = args + (column, row) + if col2 is not None and row2 is not None: + args = args + (col2, row2) + return self._getints(self.tk.call(*args)) or None + + bbox = grid_bbox + + def _gridconvvalue(self, value): + if isinstance(value, (str, _tkinter.Tcl_Obj)): + try: + svalue = str(value) + if not svalue: + return None + elif '.' in svalue: + return self.tk.getdouble(svalue) + else: + return self.tk.getint(svalue) + except (ValueError, TclError): + pass + return value + + def _grid_configure(self, command, index, cnf, kw): + """Internal function.""" + if isinstance(cnf, str) and not kw: + if cnf[-1:] == '_': + cnf = cnf[:-1] + if cnf[:1] != '-': + cnf = '-'+cnf + options = (cnf,) + else: + options = self._options(cnf, kw) + if not options: + return _splitdict( + self.tk, + self.tk.call('grid', command, self._w, index), + conv=self._gridconvvalue) + res = self.tk.call( + ('grid', command, self._w, index) + + options) + if len(options) == 1: + return self._gridconvvalue(res) + + def grid_columnconfigure(self, index, cnf={}, **kw): + """Configure column INDEX of a grid. + + Valid resources are minsize (minimum size of the column), + weight (how much does additional space propagate to this column) + and pad (how much space to let additionally).""" + return self._grid_configure('columnconfigure', index, cnf, kw) + + columnconfigure = grid_columnconfigure + + def grid_location(self, x, y): + """Return a tuple of column and row which identify the cell + at which the pixel at position X and Y inside the master + widget is located.""" + return self._getints( + self.tk.call( + 'grid', 'location', self._w, x, y)) or None + + def grid_propagate(self, flag=_noarg_): + """Set or get the status for propagation of geometry information. + + A boolean argument specifies whether the geometry information + of the slaves will determine the size of this widget. If no argument + is given, the current setting will be returned. + """ + if flag is Misc._noarg_: + return self._getboolean(self.tk.call( + 'grid', 'propagate', self._w)) + else: + self.tk.call('grid', 'propagate', self._w, flag) + + def grid_rowconfigure(self, index, cnf={}, **kw): + """Configure row INDEX of a grid. + + Valid resources are minsize (minimum size of the row), + weight (how much does additional space propagate to this row) + and pad (how much space to let additionally).""" + return self._grid_configure('rowconfigure', index, cnf, kw) + + rowconfigure = grid_rowconfigure + + def grid_size(self): + """Return a tuple of the number of column and rows in the grid.""" + return self._getints( + self.tk.call('grid', 'size', self._w)) or None + + size = grid_size + + def grid_slaves(self, row=None, column=None): + """Return a list of all slaves of this widget + in its packing order.""" + args = () + if row is not None: + args = args + ('-row', row) + if column is not None: + args = args + ('-column', column) + return [self._nametowidget(x) for x in + self.tk.splitlist(self.tk.call( + ('grid', 'slaves', self._w) + args))] + + # Support for the "event" command, new in Tk 4.2. + # By Case Roole. + + def event_add(self, virtual, *sequences): + """Bind a virtual event VIRTUAL (of the form <<Name>>) + to an event SEQUENCE such that the virtual event is triggered + whenever SEQUENCE occurs.""" + args = ('event', 'add', virtual) + sequences + self.tk.call(args) + + def event_delete(self, virtual, *sequences): + """Unbind a virtual event VIRTUAL from SEQUENCE.""" + args = ('event', 'delete', virtual) + sequences + self.tk.call(args) + + def event_generate(self, sequence, **kw): + """Generate an event SEQUENCE. Additional + keyword arguments specify parameter of the event + (e.g. x, y, rootx, rooty).""" + args = ('event', 'generate', self._w, sequence) + for k, v in kw.items(): + args = args + ('-%s' % k, str(v)) + self.tk.call(args) + + def event_info(self, virtual=None): + """Return a list of all virtual events or the information + about the SEQUENCE bound to the virtual event VIRTUAL.""" + return self.tk.splitlist( + self.tk.call('event', 'info', virtual)) + + # Image related commands + + def image_names(self): + """Return a list of all existing image names.""" + return self.tk.splitlist(self.tk.call('image', 'names')) + + def image_types(self): + """Return a list of all available image types (e.g. photo bitmap).""" + return self.tk.splitlist(self.tk.call('image', 'types')) + + +class CallWrapper: + """Internal class. Stores function to call when some user + defined Tcl function is called e.g. after an event occurred.""" + + def __init__(self, func, subst, widget): + """Store FUNC, SUBST and WIDGET as members.""" + self.func = func + self.subst = subst + self.widget = widget + + def __call__(self, *args): + """Apply first function SUBST to arguments, than FUNC.""" + try: + if self.subst: + args = self.subst(*args) + return self.func(*args) + except SystemExit: + raise + except: + self.widget._report_exception() + + +class XView: + """Mix-in class for querying and changing the horizontal position + of a widget's window.""" + + def xview(self, *args): + """Query and change the horizontal position of the view.""" + res = self.tk.call(self._w, 'xview', *args) + if not args: + return self._getdoubles(res) + + def xview_moveto(self, fraction): + """Adjusts the view in the window so that FRACTION of the + total width of the canvas is off-screen to the left.""" + self.tk.call(self._w, 'xview', 'moveto', fraction) + + def xview_scroll(self, number, what): + """Shift the x-view according to NUMBER which is measured in "units" + or "pages" (WHAT).""" + self.tk.call(self._w, 'xview', 'scroll', number, what) + + +class YView: + """Mix-in class for querying and changing the vertical position + of a widget's window.""" + + def yview(self, *args): + """Query and change the vertical position of the view.""" + res = self.tk.call(self._w, 'yview', *args) + if not args: + return self._getdoubles(res) + + def yview_moveto(self, fraction): + """Adjusts the view in the window so that FRACTION of the + total height of the canvas is off-screen to the top.""" + self.tk.call(self._w, 'yview', 'moveto', fraction) + + def yview_scroll(self, number, what): + """Shift the y-view according to NUMBER which is measured in + "units" or "pages" (WHAT).""" + self.tk.call(self._w, 'yview', 'scroll', number, what) + + +class Wm: + """Provides functions for the communication with the window manager.""" + + def wm_aspect(self, + minNumer=None, minDenom=None, + maxNumer=None, maxDenom=None): + """Instruct the window manager to set the aspect ratio (width/height) + of this widget to be between MINNUMER/MINDENOM and MAXNUMER/MAXDENOM. Return a tuple + of the actual values if no argument is given.""" + return self._getints( + self.tk.call('wm', 'aspect', self._w, + minNumer, minDenom, + maxNumer, maxDenom)) + + aspect = wm_aspect + + def wm_attributes(self, *args, return_python_dict=False, **kwargs): + """Return or sets platform specific attributes. + + When called with a single argument return_python_dict=True, + return a dict of the platform specific attributes and their values. + When called without arguments or with a single argument + return_python_dict=False, return a tuple containing intermixed + attribute names with the minus prefix and their values. + + When called with a single string value, return the value for the + specific option. When called with keyword arguments, set the + corresponding attributes. + """ + if not kwargs: + if not args: + res = self.tk.call('wm', 'attributes', self._w) + if return_python_dict: + return _splitdict(self.tk, res) + else: + return self.tk.splitlist(res) + if len(args) == 1 and args[0] is not None: + option = args[0] + if option[0] == '-': + # TODO: deprecate + option = option[1:] + return self.tk.call('wm', 'attributes', self._w, '-' + option) + # TODO: deprecate + return self.tk.call('wm', 'attributes', self._w, *args) + elif args: + raise TypeError('wm_attribute() options have been specified as ' + 'positional and keyword arguments') + else: + self.tk.call('wm', 'attributes', self._w, *self._options(kwargs)) + + attributes = wm_attributes + + def wm_client(self, name=None): + """Store NAME in WM_CLIENT_MACHINE property of this widget. Return + current value.""" + return self.tk.call('wm', 'client', self._w, name) + + client = wm_client + + def wm_colormapwindows(self, *wlist): + """Store list of window names (WLIST) into WM_COLORMAPWINDOWS property + of this widget. This list contains windows whose colormaps differ from their + parents. Return current list of widgets if WLIST is empty.""" + if len(wlist) > 1: + wlist = (wlist,) # Tk needs a list of windows here + args = ('wm', 'colormapwindows', self._w) + wlist + if wlist: + self.tk.call(args) + else: + return [self._nametowidget(x) + for x in self.tk.splitlist(self.tk.call(args))] + + colormapwindows = wm_colormapwindows + + def wm_command(self, value=None): + """Store VALUE in WM_COMMAND property. It is the command + which shall be used to invoke the application. Return current + command if VALUE is None.""" + return self.tk.call('wm', 'command', self._w, value) + + command = wm_command + + def wm_deiconify(self): + """Deiconify this widget. If it was never mapped it will not be mapped. + On Windows it will raise this widget and give it the focus.""" + return self.tk.call('wm', 'deiconify', self._w) + + deiconify = wm_deiconify + + def wm_focusmodel(self, model=None): + """Set focus model to MODEL. "active" means that this widget will claim + the focus itself, "passive" means that the window manager shall give + the focus. Return current focus model if MODEL is None.""" + return self.tk.call('wm', 'focusmodel', self._w, model) + + focusmodel = wm_focusmodel + + def wm_forget(self, window): # new in Tk 8.5 + """The window will be unmapped from the screen and will no longer + be managed by wm. toplevel windows will be treated like frame + windows once they are no longer managed by wm, however, the menu + option configuration will be remembered and the menus will return + once the widget is managed again.""" + self.tk.call('wm', 'forget', window) + + forget = wm_forget + + def wm_frame(self): + """Return identifier for decorative frame of this widget if present.""" + return self.tk.call('wm', 'frame', self._w) + + frame = wm_frame + + def wm_geometry(self, newGeometry=None): + """Set geometry to NEWGEOMETRY of the form =widthxheight+x+y. Return + current value if None is given.""" + return self.tk.call('wm', 'geometry', self._w, newGeometry) + + geometry = wm_geometry + + def wm_grid(self, + baseWidth=None, baseHeight=None, + widthInc=None, heightInc=None): + """Instruct the window manager that this widget shall only be + resized on grid boundaries. WIDTHINC and HEIGHTINC are the width and + height of a grid unit in pixels. BASEWIDTH and BASEHEIGHT are the + number of grid units requested in Tk_GeometryRequest.""" + return self._getints(self.tk.call( + 'wm', 'grid', self._w, + baseWidth, baseHeight, widthInc, heightInc)) + + grid = wm_grid + + def wm_group(self, pathName=None): + """Set the group leader widgets for related widgets to PATHNAME. Return + the group leader of this widget if None is given.""" + return self.tk.call('wm', 'group', self._w, pathName) + + group = wm_group + + def wm_iconbitmap(self, bitmap=None, default=None): + """Set bitmap for the iconified widget to BITMAP. Return + the bitmap if None is given. + + Under Windows, the DEFAULT parameter can be used to set the icon + for the widget and any descendants that don't have an icon set + explicitly. DEFAULT can be the relative path to a .ico file + (example: root.iconbitmap(default='myicon.ico') ). See Tk + documentation for more information.""" + if default is not None: + return self.tk.call('wm', 'iconbitmap', self._w, '-default', default) + else: + return self.tk.call('wm', 'iconbitmap', self._w, bitmap) + + iconbitmap = wm_iconbitmap + + def wm_iconify(self): + """Display widget as icon.""" + return self.tk.call('wm', 'iconify', self._w) + + iconify = wm_iconify + + def wm_iconmask(self, bitmap=None): + """Set mask for the icon bitmap of this widget. Return the + mask if None is given.""" + return self.tk.call('wm', 'iconmask', self._w, bitmap) + + iconmask = wm_iconmask + + def wm_iconname(self, newName=None): + """Set the name of the icon for this widget. Return the name if + None is given.""" + return self.tk.call('wm', 'iconname', self._w, newName) + + iconname = wm_iconname + + def wm_iconphoto(self, default=False, *args): # new in Tk 8.5 + """Sets the titlebar icon for this window based on the named photo + images passed through args. If default is True, this is applied to + all future created toplevels as well. + + The data in the images is taken as a snapshot at the time of + invocation. If the images are later changed, this is not reflected + to the titlebar icons. Multiple images are accepted to allow + different images sizes to be provided. The window manager may scale + provided icons to an appropriate size. + + On Windows, the images are packed into a Windows icon structure. + This will override an icon specified to wm_iconbitmap, and vice + versa. + + On X, the images are arranged into the _NET_WM_ICON X property, + which most modern window managers support. An icon specified by + wm_iconbitmap may exist simultaneously. + + On Macintosh, this currently does nothing.""" + if default: + self.tk.call('wm', 'iconphoto', self._w, "-default", *args) + else: + self.tk.call('wm', 'iconphoto', self._w, *args) + + iconphoto = wm_iconphoto + + def wm_iconposition(self, x=None, y=None): + """Set the position of the icon of this widget to X and Y. Return + a tuple of the current values of X and X if None is given.""" + return self._getints(self.tk.call( + 'wm', 'iconposition', self._w, x, y)) + + iconposition = wm_iconposition + + def wm_iconwindow(self, pathName=None): + """Set widget PATHNAME to be displayed instead of icon. Return the current + value if None is given.""" + return self.tk.call('wm', 'iconwindow', self._w, pathName) + + iconwindow = wm_iconwindow + + def wm_manage(self, widget): # new in Tk 8.5 + """The widget specified will become a stand alone top-level window. + The window will be decorated with the window managers title bar, + etc.""" + self.tk.call('wm', 'manage', widget) + + manage = wm_manage + + def wm_maxsize(self, width=None, height=None): + """Set max WIDTH and HEIGHT for this widget. If the window is gridded + the values are given in grid units. Return the current values if None + is given.""" + return self._getints(self.tk.call( + 'wm', 'maxsize', self._w, width, height)) + + maxsize = wm_maxsize + + def wm_minsize(self, width=None, height=None): + """Set min WIDTH and HEIGHT for this widget. If the window is gridded + the values are given in grid units. Return the current values if None + is given.""" + return self._getints(self.tk.call( + 'wm', 'minsize', self._w, width, height)) + + minsize = wm_minsize + + def wm_overrideredirect(self, boolean=None): + """Instruct the window manager to ignore this widget + if BOOLEAN is given with 1. Return the current value if None + is given.""" + return self._getboolean(self.tk.call( + 'wm', 'overrideredirect', self._w, boolean)) + + overrideredirect = wm_overrideredirect + + def wm_positionfrom(self, who=None): + """Instruct the window manager that the position of this widget shall + be defined by the user if WHO is "user", and by its own policy if WHO is + "program".""" + return self.tk.call('wm', 'positionfrom', self._w, who) + + positionfrom = wm_positionfrom + + def wm_protocol(self, name=None, func=None): + """Bind function FUNC to command NAME for this widget. + Return the function bound to NAME if None is given. NAME could be + e.g. "WM_SAVE_YOURSELF" or "WM_DELETE_WINDOW".""" + if callable(func): + command = self._register(func) + else: + command = func + return self.tk.call( + 'wm', 'protocol', self._w, name, command) + + protocol = wm_protocol + + def wm_resizable(self, width=None, height=None): + """Instruct the window manager whether this width can be resized + in WIDTH or HEIGHT. Both values are boolean values.""" + return self.tk.call('wm', 'resizable', self._w, width, height) + + resizable = wm_resizable + + def wm_sizefrom(self, who=None): + """Instruct the window manager that the size of this widget shall + be defined by the user if WHO is "user", and by its own policy if WHO is + "program".""" + return self.tk.call('wm', 'sizefrom', self._w, who) + + sizefrom = wm_sizefrom + + def wm_state(self, newstate=None): + """Query or set the state of this widget as one of normal, icon, + iconic (see wm_iconwindow), withdrawn, or zoomed (Windows only).""" + return self.tk.call('wm', 'state', self._w, newstate) + + state = wm_state + + def wm_title(self, string=None): + """Set the title of this widget.""" + return self.tk.call('wm', 'title', self._w, string) + + title = wm_title + + def wm_transient(self, master=None): + """Instruct the window manager that this widget is transient + with regard to widget MASTER.""" + return self.tk.call('wm', 'transient', self._w, master) + + transient = wm_transient + + def wm_withdraw(self): + """Withdraw this widget from the screen such that it is unmapped + and forgotten by the window manager. Re-draw it with wm_deiconify.""" + return self.tk.call('wm', 'withdraw', self._w) + + withdraw = wm_withdraw + + +class Tk(Misc, Wm): + """Toplevel widget of Tk which represents mostly the main window + of an application. It has an associated Tcl interpreter.""" + _w = '.' + + def __init__(self, screenName=None, baseName=None, className='Tk', + useTk=True, sync=False, use=None): + """Return a new top level widget on screen SCREENNAME. A new Tcl interpreter will + be created. BASENAME will be used for the identification of the profile file (see + readprofile). + It is constructed from sys.argv[0] without extensions if None is given. CLASSNAME + is the name of the widget class.""" + self.master = None + self.children = {} + self._tkloaded = False + # to avoid recursions in the getattr code in case of failure, we + # ensure that self.tk is always _something_. + self.tk = None + if baseName is None: + import os + # TODO: RUSTPYTHON + # baseName = os.path.basename(sys.argv[0]) + baseName = "" # sys.argv[0] + baseName, ext = os.path.splitext(baseName) + if ext not in ('.py', '.pyc'): + baseName = baseName + ext + interactive = False + self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use) + if _debug: + self.tk.settrace(_print_command) + if useTk: + self._loadtk() + if not sys.flags.ignore_environment: + # Issue #16248: Honor the -E flag to avoid code injection. + self.readprofile(baseName, className) + + def loadtk(self): + if not self._tkloaded: + self.tk.loadtk() + self._loadtk() + + def _loadtk(self): + self._tkloaded = True + global _default_root + # Version sanity checks + tk_version = self.tk.getvar('tk_version') + if tk_version != _tkinter.TK_VERSION: + raise RuntimeError("tk.h version (%s) doesn't match libtk.a version (%s)" + % (_tkinter.TK_VERSION, tk_version)) + # Under unknown circumstances, tcl_version gets coerced to float + tcl_version = str(self.tk.getvar('tcl_version')) + if tcl_version != _tkinter.TCL_VERSION: + raise RuntimeError("tcl.h version (%s) doesn't match libtcl.a version (%s)" \ + % (_tkinter.TCL_VERSION, tcl_version)) + # Create and register the tkerror and exit commands + # We need to inline parts of _register here, _ register + # would register differently-named commands. + if self._tclCommands is None: + self._tclCommands = [] + self.tk.createcommand('tkerror', _tkerror) + self.tk.createcommand('exit', _exit) + self._tclCommands.append('tkerror') + self._tclCommands.append('exit') + if _support_default_root and _default_root is None: + _default_root = self + self.protocol("WM_DELETE_WINDOW", self.destroy) + + def destroy(self): + """Destroy this and all descendants widgets. This will + end the application of this Tcl interpreter.""" + for c in list(self.children.values()): c.destroy() + self.tk.call('destroy', self._w) + Misc.destroy(self) + global _default_root + if _support_default_root and _default_root is self: + _default_root = None + + def readprofile(self, baseName, className): + """Internal function. It reads .BASENAME.tcl and .CLASSNAME.tcl into + the Tcl Interpreter and calls exec on the contents of .BASENAME.py and + .CLASSNAME.py if such a file exists in the home directory.""" + import os + if 'HOME' in os.environ: home = os.environ['HOME'] + else: home = os.curdir + class_tcl = os.path.join(home, '.%s.tcl' % className) + class_py = os.path.join(home, '.%s.py' % className) + base_tcl = os.path.join(home, '.%s.tcl' % baseName) + base_py = os.path.join(home, '.%s.py' % baseName) + dir = {'self': self} + exec('from tkinter import *', dir) + if os.path.isfile(class_tcl): + self.tk.call('source', class_tcl) + if os.path.isfile(class_py): + exec(open(class_py).read(), dir) + if os.path.isfile(base_tcl): + self.tk.call('source', base_tcl) + if os.path.isfile(base_py): + exec(open(base_py).read(), dir) + + def report_callback_exception(self, exc, val, tb): + """Report callback exception on sys.stderr. + + Applications may want to override this internal function, and + should when sys.stderr is None.""" + import traceback + print("Exception in Tkinter callback", file=sys.stderr) + sys.last_exc = val + sys.last_type = exc + sys.last_value = val + sys.last_traceback = tb + traceback.print_exception(exc, val, tb) + + def __getattr__(self, attr): + "Delegate attribute access to the interpreter object" + return getattr(self.tk, attr) + + +def _print_command(cmd, *, file=sys.stderr): + # Print executed Tcl/Tk commands. + assert isinstance(cmd, tuple) + cmd = _join(cmd) + print(cmd, file=file) + + +# Ideally, the classes Pack, Place and Grid disappear, the +# pack/place/grid methods are defined on the Widget class, and +# everybody uses w.pack_whatever(...) instead of Pack.whatever(w, +# ...), with pack(), place() and grid() being short for +# pack_configure(), place_configure() and grid_columnconfigure(), and +# forget() being short for pack_forget(). As a practical matter, I'm +# afraid that there is too much code out there that may be using the +# Pack, Place or Grid class, so I leave them intact -- but only as +# backwards compatibility features. Also note that those methods that +# take a master as argument (e.g. pack_propagate) have been moved to +# the Misc class (which now incorporates all methods common between +# toplevel and interior widgets). Again, for compatibility, these are +# copied into the Pack, Place or Grid class. + + +def Tcl(screenName=None, baseName=None, className='Tk', useTk=False): + return Tk(screenName, baseName, className, useTk) + + +class Pack: + """Geometry manager Pack. + + Base class to use the methods pack_* in every widget.""" + + def pack_configure(self, cnf={}, **kw): + """Pack a widget in the parent widget. Use as options: + after=widget - pack it after you have packed widget + anchor=NSEW (or subset) - position widget according to + given direction + before=widget - pack it before you will pack widget + expand=bool - expand widget if parent size grows + fill=NONE or X or Y or BOTH - fill widget if widget grows + in=master - use master to contain this widget + in_=master - see 'in' option description + ipadx=amount - add internal padding in x direction + ipady=amount - add internal padding in y direction + padx=amount - add padding in x direction + pady=amount - add padding in y direction + side=TOP or BOTTOM or LEFT or RIGHT - where to add this widget. + """ + self.tk.call( + ('pack', 'configure', self._w) + + self._options(cnf, kw)) + + pack = configure = config = pack_configure + + def pack_forget(self): + """Unmap this widget and do not use it for the packing order.""" + self.tk.call('pack', 'forget', self._w) + + forget = pack_forget + + def pack_info(self): + """Return information about the packing options + for this widget.""" + d = _splitdict(self.tk, self.tk.call('pack', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d + + info = pack_info + propagate = pack_propagate = Misc.pack_propagate + slaves = pack_slaves = Misc.pack_slaves + + +class Place: + """Geometry manager Place. + + Base class to use the methods place_* in every widget.""" + + def place_configure(self, cnf={}, **kw): + """Place a widget in the parent widget. Use as options: + in=master - master relative to which the widget is placed + in_=master - see 'in' option description + x=amount - locate anchor of this widget at position x of master + y=amount - locate anchor of this widget at position y of master + relx=amount - locate anchor of this widget between 0.0 and 1.0 + relative to width of master (1.0 is right edge) + rely=amount - locate anchor of this widget between 0.0 and 1.0 + relative to height of master (1.0 is bottom edge) + anchor=NSEW (or subset) - position anchor according to given direction + width=amount - width of this widget in pixel + height=amount - height of this widget in pixel + relwidth=amount - width of this widget between 0.0 and 1.0 + relative to width of master (1.0 is the same width + as the master) + relheight=amount - height of this widget between 0.0 and 1.0 + relative to height of master (1.0 is the same + height as the master) + bordermode="inside" or "outside" - whether to take border width of + master widget into account + """ + self.tk.call( + ('place', 'configure', self._w) + + self._options(cnf, kw)) + + place = configure = config = place_configure + + def place_forget(self): + """Unmap this widget.""" + self.tk.call('place', 'forget', self._w) + + forget = place_forget + + def place_info(self): + """Return information about the placing options + for this widget.""" + d = _splitdict(self.tk, self.tk.call('place', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d + + info = place_info + slaves = place_slaves = Misc.place_slaves + + +class Grid: + """Geometry manager Grid. + + Base class to use the methods grid_* in every widget.""" + # Thanks to Masazumi Yoshikawa (yosikawa@isi.edu) + + def grid_configure(self, cnf={}, **kw): + """Position a widget in the parent widget in a grid. Use as options: + column=number - use cell identified with given column (starting with 0) + columnspan=number - this widget will span several columns + in=master - use master to contain this widget + in_=master - see 'in' option description + ipadx=amount - add internal padding in x direction + ipady=amount - add internal padding in y direction + padx=amount - add padding in x direction + pady=amount - add padding in y direction + row=number - use cell identified with given row (starting with 0) + rowspan=number - this widget will span several rows + sticky=NSEW - if cell is larger on which sides will this + widget stick to the cell boundary + """ + self.tk.call( + ('grid', 'configure', self._w) + + self._options(cnf, kw)) + + grid = configure = config = grid_configure + bbox = grid_bbox = Misc.grid_bbox + columnconfigure = grid_columnconfigure = Misc.grid_columnconfigure + + def grid_forget(self): + """Unmap this widget.""" + self.tk.call('grid', 'forget', self._w) + + forget = grid_forget + + def grid_remove(self): + """Unmap this widget but remember the grid options.""" + self.tk.call('grid', 'remove', self._w) + + def grid_info(self): + """Return information about the options + for positioning this widget in a grid.""" + d = _splitdict(self.tk, self.tk.call('grid', 'info', self._w)) + if 'in' in d: + d['in'] = self.nametowidget(d['in']) + return d + + info = grid_info + location = grid_location = Misc.grid_location + propagate = grid_propagate = Misc.grid_propagate + rowconfigure = grid_rowconfigure = Misc.grid_rowconfigure + size = grid_size = Misc.grid_size + slaves = grid_slaves = Misc.grid_slaves + + +class BaseWidget(Misc): + """Internal class.""" + + def _setup(self, master, cnf): + """Internal function. Sets up information about children.""" + if master is None: + master = _get_default_root() + self.master = master + self.tk = master.tk + name = None + if 'name' in cnf: + name = cnf['name'] + del cnf['name'] + if not name: + name = self.__class__.__name__.lower() + if name[-1].isdigit(): + name += "!" # Avoid duplication when calculating names below + if master._last_child_ids is None: + master._last_child_ids = {} + count = master._last_child_ids.get(name, 0) + 1 + master._last_child_ids[name] = count + if count == 1: + name = '!%s' % (name,) + else: + name = '!%s%d' % (name, count) + self._name = name + if master._w=='.': + self._w = '.' + name + else: + self._w = master._w + '.' + name + self.children = {} + if self._name in self.master.children: + self.master.children[self._name].destroy() + self.master.children[self._name] = self + + def __init__(self, master, widgetName, cnf={}, kw={}, extra=()): + """Construct a widget with the parent widget MASTER, a name WIDGETNAME + and appropriate options.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + self.widgetName = widgetName + self._setup(master, cnf) + if self._tclCommands is None: + self._tclCommands = [] + classes = [(k, v) for k, v in cnf.items() if isinstance(k, type)] + for k, v in classes: + del cnf[k] + self.tk.call( + (widgetName, self._w) + extra + self._options(cnf)) + for k, v in classes: + k.configure(self, v) + + def destroy(self): + """Destroy this and all descendants widgets.""" + for c in list(self.children.values()): c.destroy() + self.tk.call('destroy', self._w) + if self._name in self.master.children: + del self.master.children[self._name] + Misc.destroy(self) + + def _do(self, name, args=()): + # XXX Obsolete -- better use self.tk.call directly! + return self.tk.call((self._w, name) + args) + + +class Widget(BaseWidget, Pack, Place, Grid): + """Internal class. + + Base class for a widget which can be positioned with the geometry managers + Pack, Place or Grid.""" + pass + + +class Toplevel(BaseWidget, Wm): + """Toplevel widget, e.g. for dialogs.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a toplevel widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, class, + colormap, container, cursor, height, highlightbackground, + highlightcolor, highlightthickness, menu, relief, screen, takefocus, + use, visual, width.""" + if kw: + cnf = _cnfmerge((cnf, kw)) + extra = () + for wmkey in ['screen', 'class_', 'class', 'visual', + 'colormap']: + if wmkey in cnf: + val = cnf[wmkey] + # TBD: a hack needed because some keys + # are not valid as keyword arguments + if wmkey[-1] == '_': opt = '-'+wmkey[:-1] + else: opt = '-'+wmkey + extra = extra + (opt, val) + del cnf[wmkey] + BaseWidget.__init__(self, master, 'toplevel', cnf, {}, extra) + root = self._root() + self.iconname(root.iconname()) + self.title(root.title()) + self.protocol("WM_DELETE_WINDOW", self.destroy) + + +class Button(Widget): + """Button widget.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a button widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, activeforeground, anchor, + background, bitmap, borderwidth, cursor, + disabledforeground, font, foreground + highlightbackground, highlightcolor, + highlightthickness, image, justify, + padx, pady, relief, repeatdelay, + repeatinterval, takefocus, text, + textvariable, underline, wraplength + + WIDGET-SPECIFIC OPTIONS + + command, compound, default, height, + overrelief, state, width + """ + Widget.__init__(self, master, 'button', cnf, kw) + + def flash(self): + """Flash the button. + + This is accomplished by redisplaying + the button several times, alternating between active and + normal colors. At the end of the flash the button is left + in the same normal/active state as when the command was + invoked. This command is ignored if the button's state is + disabled. + """ + self.tk.call(self._w, 'flash') + + def invoke(self): + """Invoke the command associated with the button. + + The return value is the return value from the command, + or an empty string if there is no command associated with + the button. This command is ignored if the button's state + is disabled. + """ + return self.tk.call(self._w, 'invoke') + + +class Canvas(Widget, XView, YView): + """Canvas widget to display graphical elements like lines or text.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a canvas widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, closeenough, + confine, cursor, height, highlightbackground, highlightcolor, + highlightthickness, insertbackground, insertborderwidth, + insertofftime, insertontime, insertwidth, offset, relief, + scrollregion, selectbackground, selectborderwidth, selectforeground, + state, takefocus, width, xscrollcommand, xscrollincrement, + yscrollcommand, yscrollincrement.""" + Widget.__init__(self, master, 'canvas', cnf, kw) + + def addtag(self, *args): + """Internal function.""" + self.tk.call((self._w, 'addtag') + args) + + def addtag_above(self, newtag, tagOrId): + """Add tag NEWTAG to all items above TAGORID.""" + self.addtag(newtag, 'above', tagOrId) + + def addtag_all(self, newtag): + """Add tag NEWTAG to all items.""" + self.addtag(newtag, 'all') + + def addtag_below(self, newtag, tagOrId): + """Add tag NEWTAG to all items below TAGORID.""" + self.addtag(newtag, 'below', tagOrId) + + def addtag_closest(self, newtag, x, y, halo=None, start=None): + """Add tag NEWTAG to item which is closest to pixel at X, Y. + If several match take the top-most. + All items closer than HALO are considered overlapping (all are + closest). If START is specified the next below this tag is taken.""" + self.addtag(newtag, 'closest', x, y, halo, start) + + def addtag_enclosed(self, newtag, x1, y1, x2, y2): + """Add tag NEWTAG to all items in the rectangle defined + by X1,Y1,X2,Y2.""" + self.addtag(newtag, 'enclosed', x1, y1, x2, y2) + + def addtag_overlapping(self, newtag, x1, y1, x2, y2): + """Add tag NEWTAG to all items which overlap the rectangle + defined by X1,Y1,X2,Y2.""" + self.addtag(newtag, 'overlapping', x1, y1, x2, y2) + + def addtag_withtag(self, newtag, tagOrId): + """Add tag NEWTAG to all items with TAGORID.""" + self.addtag(newtag, 'withtag', tagOrId) + + def bbox(self, *args): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle + which encloses all items with tags specified as arguments.""" + return self._getints( + self.tk.call((self._w, 'bbox') + args)) or None + + def tag_unbind(self, tagOrId, sequence, funcid=None): + """Unbind for all items with TAGORID for event SEQUENCE the + function identified with FUNCID.""" + self._unbind((self._w, 'bind', tagOrId, sequence), funcid) + + def tag_bind(self, tagOrId, sequence=None, func=None, add=None): + """Bind to all items with TAGORID at event SEQUENCE a call to function FUNC. + + An additional boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or whether it will + replace the previous function. See bind for the return value.""" + return self._bind((self._w, 'bind', tagOrId), + sequence, func, add) + + def canvasx(self, screenx, gridspacing=None): + """Return the canvas x coordinate of pixel position SCREENX rounded + to nearest multiple of GRIDSPACING units.""" + return self.tk.getdouble(self.tk.call( + self._w, 'canvasx', screenx, gridspacing)) + + def canvasy(self, screeny, gridspacing=None): + """Return the canvas y coordinate of pixel position SCREENY rounded + to nearest multiple of GRIDSPACING units.""" + return self.tk.getdouble(self.tk.call( + self._w, 'canvasy', screeny, gridspacing)) + + def coords(self, *args): + """Return a list of coordinates for the item given in ARGS.""" + args = _flatten(args) + return [self.tk.getdouble(x) for x in + self.tk.splitlist( + self.tk.call((self._w, 'coords') + args))] + + def _create(self, itemType, args, kw): # Args: (val, val, ..., cnf={}) + """Internal function.""" + args = _flatten(args) + cnf = args[-1] + if isinstance(cnf, (dict, tuple)): + args = args[:-1] + else: + cnf = {} + return self.tk.getint(self.tk.call( + self._w, 'create', itemType, + *(args + self._options(cnf, kw)))) + + def create_arc(self, *args, **kw): + """Create arc shaped region with coordinates x1,y1,x2,y2.""" + return self._create('arc', args, kw) + + def create_bitmap(self, *args, **kw): + """Create bitmap with coordinates x1,y1.""" + return self._create('bitmap', args, kw) + + def create_image(self, *args, **kw): + """Create image item with coordinates x1,y1.""" + return self._create('image', args, kw) + + def create_line(self, *args, **kw): + """Create line with coordinates x1,y1,...,xn,yn.""" + return self._create('line', args, kw) + + def create_oval(self, *args, **kw): + """Create oval with coordinates x1,y1,x2,y2.""" + return self._create('oval', args, kw) + + def create_polygon(self, *args, **kw): + """Create polygon with coordinates x1,y1,...,xn,yn.""" + return self._create('polygon', args, kw) + + def create_rectangle(self, *args, **kw): + """Create rectangle with coordinates x1,y1,x2,y2.""" + return self._create('rectangle', args, kw) + + def create_text(self, *args, **kw): + """Create text with coordinates x1,y1.""" + return self._create('text', args, kw) + + def create_window(self, *args, **kw): + """Create window with coordinates x1,y1,x2,y2.""" + return self._create('window', args, kw) + + def dchars(self, *args): + """Delete characters of text items identified by tag or id in ARGS (possibly + several times) from FIRST to LAST character (including).""" + self.tk.call((self._w, 'dchars') + args) + + def delete(self, *args): + """Delete items identified by all tag or ids contained in ARGS.""" + self.tk.call((self._w, 'delete') + args) + + def dtag(self, *args): + """Delete tag or id given as last arguments in ARGS from items + identified by first argument in ARGS.""" + self.tk.call((self._w, 'dtag') + args) + + def find(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'find') + args)) or () + + def find_above(self, tagOrId): + """Return items above TAGORID.""" + return self.find('above', tagOrId) + + def find_all(self): + """Return all items.""" + return self.find('all') + + def find_below(self, tagOrId): + """Return all items below TAGORID.""" + return self.find('below', tagOrId) + + def find_closest(self, x, y, halo=None, start=None): + """Return item which is closest to pixel at X, Y. + If several match take the top-most. + All items closer than HALO are considered overlapping (all are + closest). If START is specified the next below this tag is taken.""" + return self.find('closest', x, y, halo, start) + + def find_enclosed(self, x1, y1, x2, y2): + """Return all items in rectangle defined + by X1,Y1,X2,Y2.""" + return self.find('enclosed', x1, y1, x2, y2) + + def find_overlapping(self, x1, y1, x2, y2): + """Return all items which overlap the rectangle + defined by X1,Y1,X2,Y2.""" + return self.find('overlapping', x1, y1, x2, y2) + + def find_withtag(self, tagOrId): + """Return all items with TAGORID.""" + return self.find('withtag', tagOrId) + + def focus(self, *args): + """Set focus to the first item specified in ARGS.""" + return self.tk.call((self._w, 'focus') + args) + + def gettags(self, *args): + """Return tags associated with the first item specified in ARGS.""" + return self.tk.splitlist( + self.tk.call((self._w, 'gettags') + args)) + + def icursor(self, *args): + """Set cursor at position POS in the item identified by TAGORID. + In ARGS TAGORID must be first.""" + self.tk.call((self._w, 'icursor') + args) + + def index(self, *args): + """Return position of cursor as integer in item specified in ARGS.""" + return self.tk.getint(self.tk.call((self._w, 'index') + args)) + + def insert(self, *args): + """Insert TEXT in item TAGORID at position POS. ARGS must + be TAGORID POS TEXT.""" + self.tk.call((self._w, 'insert') + args) + + def itemcget(self, tagOrId, option): + """Return the resource value for an OPTION for item TAGORID.""" + return self.tk.call( + (self._w, 'itemcget') + (tagOrId, '-'+option)) + + def itemconfigure(self, tagOrId, cnf=None, **kw): + """Configure resources of an item TAGORID. + + The values for resources are specified as keyword + arguments. To get an overview about + the allowed keyword arguments call the method without arguments. + """ + return self._configure(('itemconfigure', tagOrId), cnf, kw) + + itemconfig = itemconfigure + + # lower, tkraise/lift hide Misc.lower, Misc.tkraise/lift, + # so the preferred name for them is tag_lower, tag_raise + # (similar to tag_bind, and similar to the Text widget); + # unfortunately can't delete the old ones yet (maybe in 1.6) + def tag_lower(self, *args): + """Lower an item TAGORID given in ARGS + (optional below another item).""" + self.tk.call((self._w, 'lower') + args) + + lower = tag_lower + + def move(self, *args): + """Move an item TAGORID given in ARGS.""" + self.tk.call((self._w, 'move') + args) + + def moveto(self, tagOrId, x='', y=''): + """Move the items given by TAGORID in the canvas coordinate + space so that the first coordinate pair of the bottommost + item with tag TAGORID is located at position (X,Y). + X and Y may be the empty string, in which case the + corresponding coordinate will be unchanged. All items matching + TAGORID remain in the same positions relative to each other.""" + self.tk.call(self._w, 'moveto', tagOrId, x, y) + + def postscript(self, cnf={}, **kw): + """Print the contents of the canvas to a postscript + file. Valid options: colormap, colormode, file, fontmap, + height, pageanchor, pageheight, pagewidth, pagex, pagey, + rotate, width, x, y.""" + return self.tk.call((self._w, 'postscript') + + self._options(cnf, kw)) + + def tag_raise(self, *args): + """Raise an item TAGORID given in ARGS + (optional above another item).""" + self.tk.call((self._w, 'raise') + args) + + lift = tkraise = tag_raise + + def scale(self, *args): + """Scale item TAGORID with XORIGIN, YORIGIN, XSCALE, YSCALE.""" + self.tk.call((self._w, 'scale') + args) + + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + + def scan_dragto(self, x, y, gain=10): + """Adjust the view of the canvas to GAIN times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y, gain) + + def select_adjust(self, tagOrId, index): + """Adjust the end of the selection near the cursor of an item TAGORID to index.""" + self.tk.call(self._w, 'select', 'adjust', tagOrId, index) + + def select_clear(self): + """Clear the selection if it is in this widget.""" + self.tk.call(self._w, 'select', 'clear') + + def select_from(self, tagOrId, index): + """Set the fixed end of a selection in item TAGORID to INDEX.""" + self.tk.call(self._w, 'select', 'from', tagOrId, index) + + def select_item(self): + """Return the item which has the selection.""" + return self.tk.call(self._w, 'select', 'item') or None + + def select_to(self, tagOrId, index): + """Set the variable end of a selection in item TAGORID to INDEX.""" + self.tk.call(self._w, 'select', 'to', tagOrId, index) + + def type(self, tagOrId): + """Return the type of the item TAGORID.""" + return self.tk.call(self._w, 'type', tagOrId) or None + + +_checkbutton_count = 0 + +class Checkbutton(Widget): + """Checkbutton widget which is either in on- or off-state.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a checkbutton widget with the parent MASTER. + + Valid resource names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, command, cursor, + disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, image, + indicatoron, justify, offvalue, onvalue, padx, pady, relief, + selectcolor, selectimage, state, takefocus, text, textvariable, + underline, variable, width, wraplength.""" + Widget.__init__(self, master, 'checkbutton', cnf, kw) + + def _setup(self, master, cnf): + # Because Checkbutton defaults to a variable with the same name as + # the widget, Checkbutton default names must be globally unique, + # not just unique within the parent widget. + if not cnf.get('name'): + global _checkbutton_count + name = self.__class__.__name__.lower() + _checkbutton_count += 1 + # To avoid collisions with ttk.Checkbutton, use the different + # name template. + cnf['name'] = f'!{name}-{_checkbutton_count}' + super()._setup(master, cnf) + + def deselect(self): + """Put the button in off-state.""" + self.tk.call(self._w, 'deselect') + + def flash(self): + """Flash the button.""" + self.tk.call(self._w, 'flash') + + def invoke(self): + """Toggle the button and invoke a command if given as resource.""" + return self.tk.call(self._w, 'invoke') + + def select(self): + """Put the button in on-state.""" + self.tk.call(self._w, 'select') + + def toggle(self): + """Toggle the button.""" + self.tk.call(self._w, 'toggle') + + +class Entry(Widget, XView): + """Entry widget which allows displaying simple text.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct an entry widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, cursor, + exportselection, fg, font, foreground, highlightbackground, + highlightcolor, highlightthickness, insertbackground, + insertborderwidth, insertofftime, insertontime, insertwidth, + invalidcommand, invcmd, justify, relief, selectbackground, + selectborderwidth, selectforeground, show, state, takefocus, + textvariable, validate, validatecommand, vcmd, width, + xscrollcommand.""" + Widget.__init__(self, master, 'entry', cnf, kw) + + def delete(self, first, last=None): + """Delete text from FIRST to LAST (not included).""" + self.tk.call(self._w, 'delete', first, last) + + def get(self): + """Return the text.""" + return self.tk.call(self._w, 'get') + + def icursor(self, index): + """Insert cursor at INDEX.""" + self.tk.call(self._w, 'icursor', index) + + def index(self, index): + """Return position of cursor.""" + return self.tk.getint(self.tk.call( + self._w, 'index', index)) + + def insert(self, index, string): + """Insert STRING at INDEX.""" + self.tk.call(self._w, 'insert', index, string) + + def scan_mark(self, x): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x) + + def scan_dragto(self, x): + """Adjust the view of the canvas to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x) + + def selection_adjust(self, index): + """Adjust the end of the selection near the cursor to INDEX.""" + self.tk.call(self._w, 'selection', 'adjust', index) + + select_adjust = selection_adjust + + def selection_clear(self): + """Clear the selection if it is in this widget.""" + self.tk.call(self._w, 'selection', 'clear') + + select_clear = selection_clear + + def selection_from(self, index): + """Set the fixed end of a selection to INDEX.""" + self.tk.call(self._w, 'selection', 'from', index) + + select_from = selection_from + + def selection_present(self): + """Return True if there are characters selected in the entry, False + otherwise.""" + return self.tk.getboolean( + self.tk.call(self._w, 'selection', 'present')) + + select_present = selection_present + + def selection_range(self, start, end): + """Set the selection from START to END (not included).""" + self.tk.call(self._w, 'selection', 'range', start, end) + + select_range = selection_range + + def selection_to(self, index): + """Set the variable end of a selection to INDEX.""" + self.tk.call(self._w, 'selection', 'to', index) + + select_to = selection_to + + +class Frame(Widget): + """Frame widget which may contain other widgets and can have a 3D border.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a frame widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, class, + colormap, container, cursor, height, highlightbackground, + highlightcolor, highlightthickness, relief, takefocus, visual, width.""" + cnf = _cnfmerge((cnf, kw)) + extra = () + if 'class_' in cnf: + extra = ('-class', cnf['class_']) + del cnf['class_'] + elif 'class' in cnf: + extra = ('-class', cnf['class']) + del cnf['class'] + Widget.__init__(self, master, 'frame', cnf, {}, extra) + + +class Label(Widget): + """Label widget which can display text and bitmaps.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a label widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, activeforeground, anchor, + background, bitmap, borderwidth, cursor, + disabledforeground, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, image, justify, + padx, pady, relief, takefocus, text, + textvariable, underline, wraplength + + WIDGET-SPECIFIC OPTIONS + + height, state, width + + """ + Widget.__init__(self, master, 'label', cnf, kw) + + +class Listbox(Widget, XView, YView): + """Listbox widget which can display a list of strings.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a listbox widget with the parent MASTER. + + Valid resource names: background, bd, bg, borderwidth, cursor, + exportselection, fg, font, foreground, height, highlightbackground, + highlightcolor, highlightthickness, relief, selectbackground, + selectborderwidth, selectforeground, selectmode, setgrid, takefocus, + width, xscrollcommand, yscrollcommand, listvariable.""" + Widget.__init__(self, master, 'listbox', cnf, kw) + + def activate(self, index): + """Activate item identified by INDEX.""" + self.tk.call(self._w, 'activate', index) + + def bbox(self, index): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle + which encloses the item identified by the given index.""" + return self._getints(self.tk.call(self._w, 'bbox', index)) or None + + def curselection(self): + """Return the indices of currently selected item.""" + return self._getints(self.tk.call(self._w, 'curselection')) or () + + def delete(self, first, last=None): + """Delete items from FIRST to LAST (included).""" + self.tk.call(self._w, 'delete', first, last) + + def get(self, first, last=None): + """Get list of items from FIRST to LAST (included).""" + if last is not None: + return self.tk.splitlist(self.tk.call( + self._w, 'get', first, last)) + else: + return self.tk.call(self._w, 'get', first) + + def index(self, index): + """Return index of item identified with INDEX.""" + i = self.tk.call(self._w, 'index', index) + if i == 'none': return None + return self.tk.getint(i) + + def insert(self, index, *elements): + """Insert ELEMENTS at INDEX.""" + self.tk.call((self._w, 'insert', index) + elements) + + def nearest(self, y): + """Get index of item which is nearest to y coordinate Y.""" + return self.tk.getint(self.tk.call( + self._w, 'nearest', y)) + + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + + def scan_dragto(self, x, y): + """Adjust the view of the listbox to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y) + + def see(self, index): + """Scroll such that INDEX is visible.""" + self.tk.call(self._w, 'see', index) + + def selection_anchor(self, index): + """Set the fixed end oft the selection to INDEX.""" + self.tk.call(self._w, 'selection', 'anchor', index) + + select_anchor = selection_anchor + + def selection_clear(self, first, last=None): + """Clear the selection from FIRST to LAST (included).""" + self.tk.call(self._w, + 'selection', 'clear', first, last) + + select_clear = selection_clear + + def selection_includes(self, index): + """Return True if INDEX is part of the selection.""" + return self.tk.getboolean(self.tk.call( + self._w, 'selection', 'includes', index)) + + select_includes = selection_includes + + def selection_set(self, first, last=None): + """Set the selection from FIRST to LAST (included) without + changing the currently selected elements.""" + self.tk.call(self._w, 'selection', 'set', first, last) + + select_set = selection_set + + def size(self): + """Return the number of elements in the listbox.""" + return self.tk.getint(self.tk.call(self._w, 'size')) + + def itemcget(self, index, option): + """Return the resource value for an ITEM and an OPTION.""" + return self.tk.call( + (self._w, 'itemcget') + (index, '-'+option)) + + def itemconfigure(self, index, cnf=None, **kw): + """Configure resources of an ITEM. + + The values for resources are specified as keyword arguments. + To get an overview about the allowed keyword arguments + call the method without arguments. + Valid resource names: background, bg, foreground, fg, + selectbackground, selectforeground.""" + return self._configure(('itemconfigure', index), cnf, kw) + + itemconfig = itemconfigure + + +class Menu(Widget): + """Menu widget which allows displaying menu bars, pull-down menus and pop-up menus.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct menu widget with the parent MASTER. + + Valid resource names: activebackground, activeborderwidth, + activeforeground, background, bd, bg, borderwidth, cursor, + disabledforeground, fg, font, foreground, postcommand, relief, + selectcolor, takefocus, tearoff, tearoffcommand, title, type.""" + Widget.__init__(self, master, 'menu', cnf, kw) + + def tk_popup(self, x, y, entry=""): + """Post the menu at position X,Y with entry ENTRY.""" + self.tk.call('tk_popup', self._w, x, y, entry) + + def activate(self, index): + """Activate entry at INDEX.""" + self.tk.call(self._w, 'activate', index) + + def add(self, itemType, cnf={}, **kw): + """Internal function.""" + self.tk.call((self._w, 'add', itemType) + + self._options(cnf, kw)) + + def add_cascade(self, cnf={}, **kw): + """Add hierarchical menu item.""" + self.add('cascade', cnf or kw) + + def add_checkbutton(self, cnf={}, **kw): + """Add checkbutton menu item.""" + self.add('checkbutton', cnf or kw) + + def add_command(self, cnf={}, **kw): + """Add command menu item.""" + self.add('command', cnf or kw) + + def add_radiobutton(self, cnf={}, **kw): + """Add radio menu item.""" + self.add('radiobutton', cnf or kw) + + def add_separator(self, cnf={}, **kw): + """Add separator.""" + self.add('separator', cnf or kw) + + def insert(self, index, itemType, cnf={}, **kw): + """Internal function.""" + self.tk.call((self._w, 'insert', index, itemType) + + self._options(cnf, kw)) + + def insert_cascade(self, index, cnf={}, **kw): + """Add hierarchical menu item at INDEX.""" + self.insert(index, 'cascade', cnf or kw) + + def insert_checkbutton(self, index, cnf={}, **kw): + """Add checkbutton menu item at INDEX.""" + self.insert(index, 'checkbutton', cnf or kw) + + def insert_command(self, index, cnf={}, **kw): + """Add command menu item at INDEX.""" + self.insert(index, 'command', cnf or kw) + + def insert_radiobutton(self, index, cnf={}, **kw): + """Add radio menu item at INDEX.""" + self.insert(index, 'radiobutton', cnf or kw) + + def insert_separator(self, index, cnf={}, **kw): + """Add separator at INDEX.""" + self.insert(index, 'separator', cnf or kw) + + def delete(self, index1, index2=None): + """Delete menu items between INDEX1 and INDEX2 (included).""" + if index2 is None: + index2 = index1 + + num_index1, num_index2 = self.index(index1), self.index(index2) + if (num_index1 is None) or (num_index2 is None): + num_index1, num_index2 = 0, -1 + + for i in range(num_index1, num_index2 + 1): + if 'command' in self.entryconfig(i): + c = str(self.entrycget(i, 'command')) + if c: + self.deletecommand(c) + self.tk.call(self._w, 'delete', index1, index2) + + def entrycget(self, index, option): + """Return the resource value of a menu item for OPTION at INDEX.""" + return self.tk.call(self._w, 'entrycget', index, '-' + option) + + def entryconfigure(self, index, cnf=None, **kw): + """Configure a menu item at INDEX.""" + return self._configure(('entryconfigure', index), cnf, kw) + + entryconfig = entryconfigure + + def index(self, index): + """Return the index of a menu item identified by INDEX.""" + i = self.tk.call(self._w, 'index', index) + return None if i in ('', 'none') else self.tk.getint(i) # GH-103685. + + def invoke(self, index): + """Invoke a menu item identified by INDEX and execute + the associated command.""" + return self.tk.call(self._w, 'invoke', index) + + def post(self, x, y): + """Display a menu at position X,Y.""" + self.tk.call(self._w, 'post', x, y) + + def type(self, index): + """Return the type of the menu item at INDEX.""" + return self.tk.call(self._w, 'type', index) + + def unpost(self): + """Unmap a menu.""" + self.tk.call(self._w, 'unpost') + + def xposition(self, index): # new in Tk 8.5 + """Return the x-position of the leftmost pixel of the menu item + at INDEX.""" + return self.tk.getint(self.tk.call(self._w, 'xposition', index)) + + def yposition(self, index): + """Return the y-position of the topmost pixel of the menu item at INDEX.""" + return self.tk.getint(self.tk.call( + self._w, 'yposition', index)) + + +class Menubutton(Widget): + """Menubutton widget, obsolete since Tk8.0.""" + + def __init__(self, master=None, cnf={}, **kw): + Widget.__init__(self, master, 'menubutton', cnf, kw) + + +class Message(Widget): + """Message widget to display multiline text. Obsolete since Label does it too.""" + + def __init__(self, master=None, cnf={}, **kw): + Widget.__init__(self, master, 'message', cnf, kw) + + +class Radiobutton(Widget): + """Radiobutton widget which shows only one of several buttons in on-state.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a radiobutton widget with the parent MASTER. + + Valid resource names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, command, cursor, + disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, image, + indicatoron, justify, padx, pady, relief, selectcolor, selectimage, + state, takefocus, text, textvariable, underline, value, variable, + width, wraplength.""" + Widget.__init__(self, master, 'radiobutton', cnf, kw) + + def deselect(self): + """Put the button in off-state.""" + + self.tk.call(self._w, 'deselect') + + def flash(self): + """Flash the button.""" + self.tk.call(self._w, 'flash') + + def invoke(self): + """Toggle the button and invoke a command if given as resource.""" + return self.tk.call(self._w, 'invoke') + + def select(self): + """Put the button in on-state.""" + self.tk.call(self._w, 'select') + + +class Scale(Widget): + """Scale widget which can display a numerical scale.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a scale widget with the parent MASTER. + + Valid resource names: activebackground, background, bigincrement, bd, + bg, borderwidth, command, cursor, digits, fg, font, foreground, from, + highlightbackground, highlightcolor, highlightthickness, label, + length, orient, relief, repeatdelay, repeatinterval, resolution, + showvalue, sliderlength, sliderrelief, state, takefocus, + tickinterval, to, troughcolor, variable, width.""" + Widget.__init__(self, master, 'scale', cnf, kw) + + def get(self): + """Get the current value as integer or float.""" + value = self.tk.call(self._w, 'get') + try: + return self.tk.getint(value) + except (ValueError, TypeError, TclError): + return self.tk.getdouble(value) + + def set(self, value): + """Set the value to VALUE.""" + self.tk.call(self._w, 'set', value) + + def coords(self, value=None): + """Return a tuple (X,Y) of the point along the centerline of the + trough that corresponds to VALUE or the current value if None is + given.""" + + return self._getints(self.tk.call(self._w, 'coords', value)) + + def identify(self, x, y): + """Return where the point X,Y lies. Valid return values are "slider", + "though1" and "though2".""" + return self.tk.call(self._w, 'identify', x, y) + + +class Scrollbar(Widget): + """Scrollbar widget which displays a slider at a certain position.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a scrollbar widget with the parent MASTER. + + Valid resource names: activebackground, activerelief, + background, bd, bg, borderwidth, command, cursor, + elementborderwidth, highlightbackground, + highlightcolor, highlightthickness, jump, orient, + relief, repeatdelay, repeatinterval, takefocus, + troughcolor, width.""" + Widget.__init__(self, master, 'scrollbar', cnf, kw) + + def activate(self, index=None): + """Marks the element indicated by index as active. + The only index values understood by this method are "arrow1", + "slider", or "arrow2". If any other value is specified then no + element of the scrollbar will be active. If index is not specified, + the method returns the name of the element that is currently active, + or None if no element is active.""" + return self.tk.call(self._w, 'activate', index) or None + + def delta(self, deltax, deltay): + """Return the fractional change of the scrollbar setting if it + would be moved by DELTAX or DELTAY pixels.""" + return self.tk.getdouble( + self.tk.call(self._w, 'delta', deltax, deltay)) + + def fraction(self, x, y): + """Return the fractional value which corresponds to a slider + position of X,Y.""" + return self.tk.getdouble(self.tk.call(self._w, 'fraction', x, y)) + + def identify(self, x, y): + """Return the element under position X,Y as one of + "arrow1","slider","arrow2" or "".""" + return self.tk.call(self._w, 'identify', x, y) + + def get(self): + """Return the current fractional values (upper and lower end) + of the slider position.""" + return self._getdoubles(self.tk.call(self._w, 'get')) + + def set(self, first, last): + """Set the fractional values of the slider position (upper and + lower ends as value between 0 and 1).""" + self.tk.call(self._w, 'set', first, last) + + +class Text(Widget, XView, YView): + """Text widget which can display text in various forms.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a text widget with the parent MASTER. + + STANDARD OPTIONS + + background, borderwidth, cursor, + exportselection, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, insertbackground, + insertborderwidth, insertofftime, + insertontime, insertwidth, padx, pady, + relief, selectbackground, + selectborderwidth, selectforeground, + setgrid, takefocus, + xscrollcommand, yscrollcommand, + + WIDGET-SPECIFIC OPTIONS + + autoseparators, height, maxundo, + spacing1, spacing2, spacing3, + state, tabs, undo, width, wrap, + + """ + Widget.__init__(self, master, 'text', cnf, kw) + + def bbox(self, index): + """Return a tuple of (x,y,width,height) which gives the bounding + box of the visible part of the character at the given index.""" + return self._getints( + self.tk.call(self._w, 'bbox', index)) or None + + def compare(self, index1, op, index2): + """Return whether between index INDEX1 and index INDEX2 the + relation OP is satisfied. OP is one of <, <=, ==, >=, >, or !=.""" + return self.tk.getboolean(self.tk.call( + self._w, 'compare', index1, op, index2)) + + def count(self, index1, index2, *options, return_ints=False): # new in Tk 8.5 + """Counts the number of relevant things between the two indices. + + If INDEX1 is after INDEX2, the result will be a negative number + (and this holds for each of the possible options). + + The actual items which are counted depends on the options given. + The result is a tuple of integers, one for the result of each + counting option given, if more than one option is specified or + return_ints is false (default), otherwise it is an integer. + Valid counting options are "chars", "displaychars", + "displayindices", "displaylines", "indices", "lines", "xpixels" + and "ypixels". The default value, if no option is specified, is + "indices". There is an additional possible option "update", + which if given then all subsequent options ensure that any + possible out of date information is recalculated. + """ + options = ['-%s' % arg for arg in options] + res = self.tk.call(self._w, 'count', *options, index1, index2) + if not isinstance(res, int): + res = self._getints(res) + if len(res) == 1: + res, = res + if not return_ints: + if not res: + res = None + elif len(options) <= 1: + res = (res,) + return res + + def debug(self, boolean=None): + """Turn on the internal consistency checks of the B-Tree inside the text + widget according to BOOLEAN.""" + if boolean is None: + return self.tk.getboolean(self.tk.call(self._w, 'debug')) + self.tk.call(self._w, 'debug', boolean) + + def delete(self, index1, index2=None): + """Delete the characters between INDEX1 and INDEX2 (not included).""" + self.tk.call(self._w, 'delete', index1, index2) + + def dlineinfo(self, index): + """Return tuple (x,y,width,height,baseline) giving the bounding box + and baseline position of the visible part of the line containing + the character at INDEX.""" + return self._getints(self.tk.call(self._w, 'dlineinfo', index)) + + def dump(self, index1, index2=None, command=None, **kw): + """Return the contents of the widget between index1 and index2. + + The type of contents returned in filtered based on the keyword + parameters; if 'all', 'image', 'mark', 'tag', 'text', or 'window' are + given and true, then the corresponding items are returned. The result + is a list of triples of the form (key, value, index). If none of the + keywords are true then 'all' is used by default. + + If the 'command' argument is given, it is called once for each element + of the list of triples, with the values of each triple serving as the + arguments to the function. In this case the list is not returned.""" + args = [] + func_name = None + result = None + if not command: + # Never call the dump command without the -command flag, since the + # output could involve Tcl quoting and would be a pain to parse + # right. Instead just set the command to build a list of triples + # as if we had done the parsing. + result = [] + def append_triple(key, value, index, result=result): + result.append((key, value, index)) + command = append_triple + try: + if not isinstance(command, str): + func_name = command = self._register(command) + args += ["-command", command] + for key in kw: + if kw[key]: args.append("-" + key) + args.append(index1) + if index2: + args.append(index2) + self.tk.call(self._w, "dump", *args) + return result + finally: + if func_name: + self.deletecommand(func_name) + + ## new in tk8.4 + def edit(self, *args): + """Internal method + + This method controls the undo mechanism and + the modified flag. The exact behavior of the + command depends on the option argument that + follows the edit argument. The following forms + of the command are currently supported: + + edit_modified, edit_redo, edit_reset, edit_separator + and edit_undo + + """ + return self.tk.call(self._w, 'edit', *args) + + def edit_modified(self, arg=None): + """Get or Set the modified flag + + If arg is not specified, returns the modified + flag of the widget. The insert, delete, edit undo and + edit redo commands or the user can set or clear the + modified flag. If boolean is specified, sets the + modified flag of the widget to arg. + """ + return self.edit("modified", arg) + + def edit_redo(self): + """Redo the last undone edit + + When the undo option is true, reapplies the last + undone edits provided no other edits were done since + then. Generates an error when the redo stack is empty. + Does nothing when the undo option is false. + """ + return self.edit("redo") + + def edit_reset(self): + """Clears the undo and redo stacks + """ + return self.edit("reset") + + def edit_separator(self): + """Inserts a separator (boundary) on the undo stack. + + Does nothing when the undo option is false + """ + return self.edit("separator") + + def edit_undo(self): + """Undoes the last edit action + + If the undo option is true. An edit action is defined + as all the insert and delete commands that are recorded + on the undo stack in between two separators. Generates + an error when the undo stack is empty. Does nothing + when the undo option is false + """ + return self.edit("undo") + + def get(self, index1, index2=None): + """Return the text from INDEX1 to INDEX2 (not included).""" + return self.tk.call(self._w, 'get', index1, index2) + # (Image commands are new in 8.0) + + def image_cget(self, index, option): + """Return the value of OPTION of an embedded image at INDEX.""" + if option[:1] != "-": + option = "-" + option + if option[-1:] == "_": + option = option[:-1] + return self.tk.call(self._w, "image", "cget", index, option) + + def image_configure(self, index, cnf=None, **kw): + """Configure an embedded image at INDEX.""" + return self._configure(('image', 'configure', index), cnf, kw) + + def image_create(self, index, cnf={}, **kw): + """Create an embedded image at INDEX.""" + return self.tk.call( + self._w, "image", "create", index, + *self._options(cnf, kw)) + + def image_names(self): + """Return all names of embedded images in this widget.""" + return self.tk.call(self._w, "image", "names") + + def index(self, index): + """Return the index in the form line.char for INDEX.""" + return str(self.tk.call(self._w, 'index', index)) + + def insert(self, index, chars, *args): + """Insert CHARS before the characters at INDEX. An additional + tag can be given in ARGS. Additional CHARS and tags can follow in ARGS.""" + self.tk.call((self._w, 'insert', index, chars) + args) + + def mark_gravity(self, markName, direction=None): + """Change the gravity of a mark MARKNAME to DIRECTION (LEFT or RIGHT). + Return the current value if None is given for DIRECTION.""" + return self.tk.call( + (self._w, 'mark', 'gravity', markName, direction)) + + def mark_names(self): + """Return all mark names.""" + return self.tk.splitlist(self.tk.call( + self._w, 'mark', 'names')) + + def mark_set(self, markName, index): + """Set mark MARKNAME before the character at INDEX.""" + self.tk.call(self._w, 'mark', 'set', markName, index) + + def mark_unset(self, *markNames): + """Delete all marks in MARKNAMES.""" + self.tk.call((self._w, 'mark', 'unset') + markNames) + + def mark_next(self, index): + """Return the name of the next mark after INDEX.""" + return self.tk.call(self._w, 'mark', 'next', index) or None + + def mark_previous(self, index): + """Return the name of the previous mark before INDEX.""" + return self.tk.call(self._w, 'mark', 'previous', index) or None + + def peer_create(self, newPathName, cnf={}, **kw): # new in Tk 8.5 + """Creates a peer text widget with the given newPathName, and any + optional standard configuration options. By default the peer will + have the same start and end line as the parent widget, but + these can be overridden with the standard configuration options.""" + self.tk.call(self._w, 'peer', 'create', newPathName, + *self._options(cnf, kw)) + + def peer_names(self): # new in Tk 8.5 + """Returns a list of peers of this widget (this does not include + the widget itself).""" + return self.tk.splitlist(self.tk.call(self._w, 'peer', 'names')) + + def replace(self, index1, index2, chars, *args): # new in Tk 8.5 + """Replaces the range of characters between index1 and index2 with + the given characters and tags specified by args. + + See the method insert for some more information about args, and the + method delete for information about the indices.""" + self.tk.call(self._w, 'replace', index1, index2, chars, *args) + + def scan_mark(self, x, y): + """Remember the current X, Y coordinates.""" + self.tk.call(self._w, 'scan', 'mark', x, y) + + def scan_dragto(self, x, y): + """Adjust the view of the text to 10 times the + difference between X and Y and the coordinates given in + scan_mark.""" + self.tk.call(self._w, 'scan', 'dragto', x, y) + + def search(self, pattern, index, stopindex=None, + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, elide=None): + """Search PATTERN beginning from INDEX until STOPINDEX. + Return the index of the first character of a match or an + empty string.""" + args = [self._w, 'search'] + if forwards: args.append('-forwards') + if backwards: args.append('-backwards') + if exact: args.append('-exact') + if regexp: args.append('-regexp') + if nocase: args.append('-nocase') + if elide: args.append('-elide') + if count: args.append('-count'); args.append(count) + if pattern and pattern[0] == '-': args.append('--') + args.append(pattern) + args.append(index) + if stopindex: args.append(stopindex) + return str(self.tk.call(tuple(args))) + + def see(self, index): + """Scroll such that the character at INDEX is visible.""" + self.tk.call(self._w, 'see', index) + + def tag_add(self, tagName, index1, *args): + """Add tag TAGNAME to all characters between INDEX1 and index2 in ARGS. + Additional pairs of indices may follow in ARGS.""" + self.tk.call( + (self._w, 'tag', 'add', tagName, index1) + args) + + def tag_unbind(self, tagName, sequence, funcid=None): + """Unbind for all characters with TAGNAME for event SEQUENCE the + function identified with FUNCID.""" + return self._unbind((self._w, 'tag', 'bind', tagName, sequence), funcid) + + def tag_bind(self, tagName, sequence, func, add=None): + """Bind to all characters with TAGNAME at event SEQUENCE a call to function FUNC. + + An additional boolean parameter ADD specifies whether FUNC will be + called additionally to the other bound function or whether it will + replace the previous function. See bind for the return value.""" + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + + def _tag_bind(self, tagName, sequence=None, func=None, add=None): + # For tests only + return self._bind((self._w, 'tag', 'bind', tagName), + sequence, func, add) + + def tag_cget(self, tagName, option): + """Return the value of OPTION for tag TAGNAME.""" + if option[:1] != '-': + option = '-' + option + if option[-1:] == '_': + option = option[:-1] + return self.tk.call(self._w, 'tag', 'cget', tagName, option) + + def tag_configure(self, tagName, cnf=None, **kw): + """Configure a tag TAGNAME.""" + return self._configure(('tag', 'configure', tagName), cnf, kw) + + tag_config = tag_configure + + def tag_delete(self, *tagNames): + """Delete all tags in TAGNAMES.""" + self.tk.call((self._w, 'tag', 'delete') + tagNames) + + def tag_lower(self, tagName, belowThis=None): + """Change the priority of tag TAGNAME such that it is lower + than the priority of BELOWTHIS.""" + self.tk.call(self._w, 'tag', 'lower', tagName, belowThis) + + def tag_names(self, index=None): + """Return a list of all tag names.""" + return self.tk.splitlist( + self.tk.call(self._w, 'tag', 'names', index)) + + def tag_nextrange(self, tagName, index1, index2=None): + """Return a list of start and end index for the first sequence of + characters between INDEX1 and INDEX2 which all have tag TAGNAME. + The text is searched forward from INDEX1.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'nextrange', tagName, index1, index2)) + + def tag_prevrange(self, tagName, index1, index2=None): + """Return a list of start and end index for the first sequence of + characters between INDEX1 and INDEX2 which all have tag TAGNAME. + The text is searched backwards from INDEX1.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'prevrange', tagName, index1, index2)) + + def tag_raise(self, tagName, aboveThis=None): + """Change the priority of tag TAGNAME such that it is higher + than the priority of ABOVETHIS.""" + self.tk.call( + self._w, 'tag', 'raise', tagName, aboveThis) + + def tag_ranges(self, tagName): + """Return a list of ranges of text which have tag TAGNAME.""" + return self.tk.splitlist(self.tk.call( + self._w, 'tag', 'ranges', tagName)) + + def tag_remove(self, tagName, index1, index2=None): + """Remove tag TAGNAME from all characters between INDEX1 and INDEX2.""" + self.tk.call( + self._w, 'tag', 'remove', tagName, index1, index2) + + def window_cget(self, index, option): + """Return the value of OPTION of an embedded window at INDEX.""" + if option[:1] != '-': + option = '-' + option + if option[-1:] == '_': + option = option[:-1] + return self.tk.call(self._w, 'window', 'cget', index, option) + + def window_configure(self, index, cnf=None, **kw): + """Configure an embedded window at INDEX.""" + return self._configure(('window', 'configure', index), cnf, kw) + + window_config = window_configure + + def window_create(self, index, cnf={}, **kw): + """Create a window at INDEX.""" + self.tk.call( + (self._w, 'window', 'create', index) + + self._options(cnf, kw)) + + def window_names(self): + """Return all names of embedded windows in this widget.""" + return self.tk.splitlist( + self.tk.call(self._w, 'window', 'names')) + + def yview_pickplace(self, *what): + """Obsolete function, use see.""" + self.tk.call((self._w, 'yview', '-pickplace') + what) + + +class _setit: + """Internal class. It wraps the command in the widget OptionMenu.""" + + def __init__(self, var, value, callback=None): + self.__value = value + self.__var = var + self.__callback = callback + + def __call__(self, *args): + self.__var.set(self.__value) + if self.__callback is not None: + self.__callback(self.__value, *args) + + +class OptionMenu(Menubutton): + """OptionMenu which allows the user to select a value from a menu.""" + + def __init__(self, master, variable, value, *values, **kwargs): + """Construct an optionmenu widget with the parent MASTER, with + the resource textvariable set to VARIABLE, the initially selected + value VALUE, the other menu values VALUES and an additional + keyword argument command.""" + kw = {"borderwidth": 2, "textvariable": variable, + "indicatoron": 1, "relief": RAISED, "anchor": "c", + "highlightthickness": 2} + Widget.__init__(self, master, "menubutton", kw) + self.widgetName = 'tk_optionMenu' + menu = self.__menu = Menu(self, name="menu", tearoff=0) + self.menuname = menu._w + # 'command' is the only supported keyword + callback = kwargs.get('command') + if 'command' in kwargs: + del kwargs['command'] + if kwargs: + raise TclError('unknown option -'+next(iter(kwargs))) + menu.add_command(label=value, + command=_setit(variable, value, callback)) + for v in values: + menu.add_command(label=v, + command=_setit(variable, v, callback)) + self["menu"] = menu + + def __getitem__(self, name): + if name == 'menu': + return self.__menu + return Widget.__getitem__(self, name) + + def destroy(self): + """Destroy this widget and the associated menu.""" + Menubutton.destroy(self) + self.__menu = None + + +class Image: + """Base class for images.""" + _last_id = 0 + + def __init__(self, imgtype, name=None, cnf={}, master=None, **kw): + self.name = None + if master is None: + master = _get_default_root('create image') + self.tk = getattr(master, 'tk', master) + if not name: + Image._last_id += 1 + name = "pyimage%r" % (Image._last_id,) # tk itself would use image<x> + if kw and cnf: cnf = _cnfmerge((cnf, kw)) + elif kw: cnf = kw + options = () + for k, v in cnf.items(): + options = options + ('-'+k, v) + self.tk.call(('image', 'create', imgtype, name,) + options) + self.name = name + + def __str__(self): return self.name + + def __del__(self): + if self.name: + try: + self.tk.call('image', 'delete', self.name) + except TclError: + # May happen if the root was destroyed + pass + + def __setitem__(self, key, value): + self.tk.call(self.name, 'configure', '-'+key, value) + + def __getitem__(self, key): + return self.tk.call(self.name, 'configure', '-'+key) + + def configure(self, **kw): + """Configure the image.""" + res = () + for k, v in _cnfmerge(kw).items(): + if v is not None: + if k[-1] == '_': k = k[:-1] + res = res + ('-'+k, v) + self.tk.call((self.name, 'config') + res) + + config = configure + + def height(self): + """Return the height of the image.""" + return self.tk.getint( + self.tk.call('image', 'height', self.name)) + + def type(self): + """Return the type of the image, e.g. "photo" or "bitmap".""" + return self.tk.call('image', 'type', self.name) + + def width(self): + """Return the width of the image.""" + return self.tk.getint( + self.tk.call('image', 'width', self.name)) + + +class PhotoImage(Image): + """Widget which can display images in PGM, PPM, GIF, PNG format.""" + + def __init__(self, name=None, cnf={}, master=None, **kw): + """Create an image with NAME. + + Valid resource names: data, format, file, gamma, height, palette, + width.""" + Image.__init__(self, 'photo', name, cnf, master, **kw) + + def blank(self): + """Display a transparent image.""" + self.tk.call(self.name, 'blank') + + def cget(self, option): + """Return the value of OPTION.""" + return self.tk.call(self.name, 'cget', '-' + option) + # XXX config + + def __getitem__(self, key): + return self.tk.call(self.name, 'cget', '-' + key) + + def copy(self, *, from_coords=None, zoom=None, subsample=None): + """Return a new PhotoImage with the same image as this widget. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + """ + destImage = PhotoImage(master=self.tk) + destImage.copy_replace(self, from_coords=from_coords, + zoom=zoom, subsample=subsample) + return destImage + + def zoom(self, x, y='', *, from_coords=None): + """Return a new PhotoImage with the same image as this widget + but zoom it with a factor of X in the X direction and Y in the Y + direction. If Y is not given, the default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. + """ + if y=='': y=x + return self.copy(zoom=(x, y), from_coords=from_coords) + + def subsample(self, x, y='', *, from_coords=None): + """Return a new PhotoImage based on the same image as this widget + but use only every Xth or Yth pixel. If Y is not given, the + default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. + """ + if y=='': y=x + return self.copy(subsample=(x, y), from_coords=from_coords) + + def copy_replace(self, sourceImage, *, from_coords=None, to=None, shrink=False, + zoom=None, subsample=None, compositingrule=None): + """Copy a region from the source image (which must be a PhotoImage) to + this image, possibly with pixel zooming and/or subsampling. If no + options are specified, this command copies the whole of the source + image into this image, starting at coordinates (0, 0). + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + The TO option specifies a rectangular sub-region of the destination + image to be affected. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is (x1,y1) plus the size of the source region + (after subsampling and zooming, if specified). If x2 and y2 are + specified, the source region will be replicated if necessary to fill + the destination region in a tiled fashion. + + If SHRINK is true, the size of the destination image should be + reduced, if necessary, so that the region being copied into is at + the bottom-right corner of the image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + + The COMPOSITINGRULE option specifies how transparent pixels in the + source image are combined with the destination image. When a + compositing rule of 'overlay' is set, the old contents of the + destination image are visible, as if the source image were printed + on a piece of transparent film and placed over the top of the + destination. When a compositing rule of 'set' is set, the old + contents of the destination image are discarded and the source image + is used as-is. The default compositing rule is 'overlay'. + """ + options = [] + if from_coords is not None: + options.extend(('-from', *from_coords)) + if to is not None: + options.extend(('-to', *to)) + if shrink: + options.append('-shrink') + if zoom is not None: + if not isinstance(zoom, (tuple, list)): + zoom = (zoom,) + options.extend(('-zoom', *zoom)) + if subsample is not None: + if not isinstance(subsample, (tuple, list)): + subsample = (subsample,) + options.extend(('-subsample', *subsample)) + if compositingrule: + options.extend(('-compositingrule', compositingrule)) + self.tk.call(self.name, 'copy', sourceImage, *options) + + def get(self, x, y): + """Return the color (red, green, blue) of the pixel at X,Y.""" + return self.tk.call(self.name, 'get', x, y) + + def put(self, data, to=None): + """Put row formatted colors to image starting from + position TO, e.g. image.put("{red green} {blue yellow}", to=(4,6))""" + args = (self.name, 'put', data) + if to: + if to[0] == '-to': + to = to[1:] + args = args + ('-to',) + tuple(to) + self.tk.call(args) + + def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False): + """Reads image data from the file named FILENAME into the image. + + The FORMAT option specifies the format of the image data in the + file. + + The FROM_COORDS option specifies a rectangular sub-region of the image + file data to be copied to the destination image. It must be a tuple + or a list of 1 to 4 integers (x1, y1, x2, y2). (x1, y1) and + (x2, y2) specify diagonally opposite corners of the rectangle. If + x2 and y2 are not specified, the default value is the bottom-right + corner of the source image. The default, if this option is not + specified, is the whole of the image in the image file. + + The TO option specifies the coordinates of the top-left corner of + the region of the image into which data from filename are to be + read. The default is (0, 0). + + If SHRINK is true, the size of the destination image will be + reduced, if necessary, so that the region into which the image file + data are read is at the bottom-right corner of the image. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if shrink: + options += ('-shrink',) + if to is not None: + options += ('-to', *to) + self.tk.call(self.name, 'read', filename, *options) + + def write(self, filename, format=None, from_coords=None, *, + background=None, grayscale=False): + """Writes image data from the image to a file named FILENAME. + + The FORMAT option specifies the name of the image file format + handler to be used to write the data to the file. If this option + is not given, the format is guessed from the file extension. + + The FROM_COORDS option specifies a rectangular region of the image + to be written to the image file. It must be a tuple or a list of 1 + to 4 integers (x1, y1, x2, y2). If only x1 and y1 are specified, + the region extends from (x1,y1) to the bottom-right corner of the + image. If all four coordinates are given, they specify diagonally + opposite corners of the rectangular region. The default, if this + option is not given, is the whole image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + self.tk.call(self.name, 'write', filename, *options) + + def data(self, format=None, *, from_coords=None, + background=None, grayscale=False): + """Returns image data. + + The FORMAT option specifies the name of the image file format + handler to be used. If this option is not given, this method uses + a format that consists of a tuple (one element per row) of strings + containings space separated (one element per pixel/column) colors + in “#RRGGBB” format (where RR is a pair of hexadecimal digits for + the red channel, GG for green, and BB for blue). + + The FROM_COORDS option specifies a rectangular region of the image + to be returned. It must be a tuple or a list of 1 to 4 integers + (x1, y1, x2, y2). If only x1 and y1 are specified, the region + extends from (x1,y1) to the bottom-right corner of the image. If + all four coordinates are given, they specify diagonally opposite + corners of the rectangular region, including (x1, y1) and excluding + (x2, y2). The default, if this option is not given, is the whole + image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + data = self.tk.call(self.name, 'data', *options) + if isinstance(data, str): # For wantobjects = 0. + if format is None: + data = self.tk.splitlist(data) + else: + data = bytes(data, 'latin1') + return data + + def transparency_get(self, x, y): + """Return True if the pixel at x,y is transparent.""" + return self.tk.getboolean(self.tk.call( + self.name, 'transparency', 'get', x, y)) + + def transparency_set(self, x, y, boolean): + """Set the transparency of the pixel at x,y.""" + self.tk.call(self.name, 'transparency', 'set', x, y, boolean) + + +class BitmapImage(Image): + """Widget which can display images in XBM format.""" + + def __init__(self, name=None, cnf={}, master=None, **kw): + """Create a bitmap with NAME. + + Valid resource names: background, data, file, foreground, maskdata, maskfile.""" + Image.__init__(self, 'bitmap', name, cnf, master, **kw) + + +def image_names(): + tk = _get_default_root('use image_names()').tk + return tk.splitlist(tk.call('image', 'names')) + + +def image_types(): + tk = _get_default_root('use image_types()').tk + return tk.splitlist(tk.call('image', 'types')) + + +class Spinbox(Widget, XView): + """spinbox widget.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a spinbox widget with the parent MASTER. + + STANDARD OPTIONS + + activebackground, background, borderwidth, + cursor, exportselection, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, insertbackground, + insertborderwidth, insertofftime, + insertontime, insertwidth, justify, relief, + repeatdelay, repeatinterval, + selectbackground, selectborderwidth + selectforeground, takefocus, textvariable + xscrollcommand. + + WIDGET-SPECIFIC OPTIONS + + buttonbackground, buttoncursor, + buttondownrelief, buttonuprelief, + command, disabledbackground, + disabledforeground, format, from, + invalidcommand, increment, + readonlybackground, state, to, + validate, validatecommand values, + width, wrap, + """ + Widget.__init__(self, master, 'spinbox', cnf, kw) + + def bbox(self, index): + """Return a tuple of X1,Y1,X2,Y2 coordinates for a + rectangle which encloses the character given by index. + + The first two elements of the list give the x and y + coordinates of the upper-left corner of the screen + area covered by the character (in pixels relative + to the widget) and the last two elements give the + width and height of the character, in pixels. The + bounding box may refer to a region outside the + visible area of the window. + """ + return self._getints(self.tk.call(self._w, 'bbox', index)) or None + + def delete(self, first, last=None): + """Delete one or more elements of the spinbox. + + First is the index of the first character to delete, + and last is the index of the character just after + the last one to delete. If last isn't specified it + defaults to first+1, i.e. a single character is + deleted. This command returns an empty string. + """ + return self.tk.call(self._w, 'delete', first, last) + + def get(self): + """Returns the spinbox's string""" + return self.tk.call(self._w, 'get') + + def icursor(self, index): + """Alter the position of the insertion cursor. + + The insertion cursor will be displayed just before + the character given by index. Returns an empty string + """ + return self.tk.call(self._w, 'icursor', index) + + def identify(self, x, y): + """Returns the name of the widget at position x, y + + Return value is one of: none, buttondown, buttonup, entry + """ + return self.tk.call(self._w, 'identify', x, y) + + def index(self, index): + """Returns the numerical index corresponding to index + """ + return self.tk.call(self._w, 'index', index) + + def insert(self, index, s): + """Insert string s at index + + Returns an empty string. + """ + return self.tk.call(self._w, 'insert', index, s) + + def invoke(self, element): + """Causes the specified element to be invoked + + The element could be buttondown or buttonup + triggering the action associated with it. + """ + return self.tk.call(self._w, 'invoke', element) + + def scan(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'scan') + args)) or () + + def scan_mark(self, x): + """Records x and the current view in the spinbox window; + + used in conjunction with later scan dragto commands. + Typically this command is associated with a mouse button + press in the widget. It returns an empty string. + """ + return self.scan("mark", x) + + def scan_dragto(self, x): + """Compute the difference between the given x argument + and the x argument to the last scan mark command + + It then adjusts the view left or right by 10 times the + difference in x-coordinates. This command is typically + associated with mouse motion events in the widget, to + produce the effect of dragging the spinbox at high speed + through the window. The return value is an empty string. + """ + return self.scan("dragto", x) + + def selection(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'selection') + args)) or () + + def selection_adjust(self, index): + """Locate the end of the selection nearest to the character + given by index, + + Then adjust that end of the selection to be at index + (i.e including but not going beyond index). The other + end of the selection is made the anchor point for future + select to commands. If the selection isn't currently in + the spinbox, then a new selection is created to include + the characters between index and the most recent selection + anchor point, inclusive. + """ + return self.selection("adjust", index) + + def selection_clear(self): + """Clear the selection + + If the selection isn't in this widget then the + command has no effect. + """ + return self.selection("clear") + + def selection_element(self, element=None): + """Sets or gets the currently selected element. + + If a spinbutton element is specified, it will be + displayed depressed. + """ + return self.tk.call(self._w, 'selection', 'element', element) + + def selection_from(self, index): + """Set the fixed end of a selection to INDEX.""" + self.selection('from', index) + + def selection_present(self): + """Return True if there are characters selected in the spinbox, False + otherwise.""" + return self.tk.getboolean( + self.tk.call(self._w, 'selection', 'present')) + + def selection_range(self, start, end): + """Set the selection from START to END (not included).""" + self.selection('range', start, end) + + def selection_to(self, index): + """Set the variable end of a selection to INDEX.""" + self.selection('to', index) + +########################################################################### + + +class LabelFrame(Widget): + """labelframe widget.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a labelframe widget with the parent MASTER. + + STANDARD OPTIONS + + borderwidth, cursor, font, foreground, + highlightbackground, highlightcolor, + highlightthickness, padx, pady, relief, + takefocus, text + + WIDGET-SPECIFIC OPTIONS + + background, class, colormap, container, + height, labelanchor, labelwidget, + visual, width + """ + Widget.__init__(self, master, 'labelframe', cnf, kw) + +######################################################################## + + +class PanedWindow(Widget): + """panedwindow widget.""" + + def __init__(self, master=None, cnf={}, **kw): + """Construct a panedwindow widget with the parent MASTER. + + STANDARD OPTIONS + + background, borderwidth, cursor, height, + orient, relief, width + + WIDGET-SPECIFIC OPTIONS + + handlepad, handlesize, opaqueresize, + sashcursor, sashpad, sashrelief, + sashwidth, showhandle, + """ + Widget.__init__(self, master, 'panedwindow', cnf, kw) + + def add(self, child, **kw): + """Add a child widget to the panedwindow in a new pane. + + The child argument is the name of the child widget + followed by pairs of arguments that specify how to + manage the windows. The possible options and values + are the ones accepted by the paneconfigure method. + """ + self.tk.call((self._w, 'add', child) + self._options(kw)) + + def remove(self, child): + """Remove the pane containing child from the panedwindow + + All geometry management options for child will be forgotten. + """ + self.tk.call(self._w, 'forget', child) + + forget = remove + + def identify(self, x, y): + """Identify the panedwindow component at point x, y + + If the point is over a sash or a sash handle, the result + is a two element list containing the index of the sash or + handle, and a word indicating whether it is over a sash + or a handle, such as {0 sash} or {2 handle}. If the point + is over any other part of the panedwindow, the result is + an empty list. + """ + return self.tk.call(self._w, 'identify', x, y) + + def proxy(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'proxy') + args)) or () + + def proxy_coord(self): + """Return the x and y pair of the most recent proxy location + """ + return self.proxy("coord") + + def proxy_forget(self): + """Remove the proxy from the display. + """ + return self.proxy("forget") + + def proxy_place(self, x, y): + """Place the proxy at the given x and y coordinates. + """ + return self.proxy("place", x, y) + + def sash(self, *args): + """Internal function.""" + return self._getints( + self.tk.call((self._w, 'sash') + args)) or () + + def sash_coord(self, index): + """Return the current x and y pair for the sash given by index. + + Index must be an integer between 0 and 1 less than the + number of panes in the panedwindow. The coordinates given are + those of the top left corner of the region containing the sash. + pathName sash dragto index x y This command computes the + difference between the given coordinates and the coordinates + given to the last sash coord command for the given sash. It then + moves that sash the computed difference. The return value is the + empty string. + """ + return self.sash("coord", index) + + def sash_mark(self, index): + """Records x and y for the sash given by index; + + Used in conjunction with later dragto commands to move the sash. + """ + return self.sash("mark", index) + + def sash_place(self, index, x, y): + """Place the sash given by index at the given coordinates + """ + return self.sash("place", index, x, y) + + def panecget(self, child, option): + """Query a management option for window. + + Option may be any value allowed by the paneconfigure subcommand + """ + return self.tk.call( + (self._w, 'panecget') + (child, '-'+option)) + + def paneconfigure(self, tagOrId, cnf=None, **kw): + """Query or modify the management options for window. + + If no option is specified, returns a list describing all + of the available options for pathName. If option is + specified with no value, then the command returns a list + describing the one named option (this list will be identical + to the corresponding sublist of the value returned if no + option is specified). If one or more option-value pairs are + specified, then the command modifies the given widget + option(s) to have the given value(s); in this case the + command returns an empty string. The following options + are supported: + + after window + Insert the window after the window specified. window + should be the name of a window already managed by pathName. + before window + Insert the window before the window specified. window + should be the name of a window already managed by pathName. + height size + Specify a height for the window. The height will be the + outer dimension of the window including its border, if + any. If size is an empty string, or if -height is not + specified, then the height requested internally by the + window will be used initially; the height may later be + adjusted by the movement of sashes in the panedwindow. + Size may be any value accepted by Tk_GetPixels. + minsize n + Specifies that the size of the window cannot be made + less than n. This constraint only affects the size of + the widget in the paned dimension -- the x dimension + for horizontal panedwindows, the y dimension for + vertical panedwindows. May be any value accepted by + Tk_GetPixels. + padx n + Specifies a non-negative value indicating how much + extra space to leave on each side of the window in + the X-direction. The value may have any of the forms + accepted by Tk_GetPixels. + pady n + Specifies a non-negative value indicating how much + extra space to leave on each side of the window in + the Y-direction. The value may have any of the forms + accepted by Tk_GetPixels. + sticky style + If a window's pane is larger than the requested + dimensions of the window, this option may be used + to position (or stretch) the window within its pane. + Style is a string that contains zero or more of the + characters n, s, e or w. The string can optionally + contains spaces or commas, but they are ignored. Each + letter refers to a side (north, south, east, or west) + that the window will "stick" to. If both n and s + (or e and w) are specified, the window will be + stretched to fill the entire height (or width) of + its cavity. + width size + Specify a width for the window. The width will be + the outer dimension of the window including its + border, if any. If size is an empty string, or + if -width is not specified, then the width requested + internally by the window will be used initially; the + width may later be adjusted by the movement of sashes + in the panedwindow. Size may be any value accepted by + Tk_GetPixels. + + """ + if cnf is None and not kw: + return self._getconfigure(self._w, 'paneconfigure', tagOrId) + if isinstance(cnf, str) and not kw: + return self._getconfigure1( + self._w, 'paneconfigure', tagOrId, '-'+cnf) + self.tk.call((self._w, 'paneconfigure', tagOrId) + + self._options(cnf, kw)) + + paneconfig = paneconfigure + + def panes(self): + """Returns an ordered list of the child panes.""" + return self.tk.splitlist(self.tk.call(self._w, 'panes')) + +# Test: + + +def _test(): + root = Tk() + text = "This is Tcl/Tk %s" % root.globalgetvar('tk_patchLevel') + text += "\nThis should be a cedilla: \xe7" + label = Label(root, text=text) + label.pack() + test = Button(root, text="Click me!", + command=lambda root=root: root.test.configure( + text="[%s]" % root.test['text'])) + test.pack() + root.test = test + quit = Button(root, text="QUIT", command=root.destroy) + quit.pack() + # The following three commands are needed so the window pops + # up on top on Windows... + root.iconify() + root.update() + root.deiconify() + root.mainloop() + + +__all__ = [name for name, obj in globals().items() + if not name.startswith('_') and not isinstance(obj, types.ModuleType) + and name not in {'wantobjects'}] + +if __name__ == '__main__': + _test() diff --git a/Lib/tkinter/__main__.py b/Lib/tkinter/__main__.py new file mode 100644 index 00000000000..757880d439c --- /dev/null +++ b/Lib/tkinter/__main__.py @@ -0,0 +1,7 @@ +"""Main entry point""" + +import sys +if sys.argv[0].endswith("__main__.py"): + sys.argv[0] = "python -m tkinter" +from . import _test as main +main() diff --git a/Lib/tkinter/colorchooser.py b/Lib/tkinter/colorchooser.py new file mode 100644 index 00000000000..e2fb69dba92 --- /dev/null +++ b/Lib/tkinter/colorchooser.py @@ -0,0 +1,86 @@ +# tk common color chooser dialogue +# +# this module provides an interface to the native color dialogue +# available in Tk 4.2 and newer. +# +# written by Fredrik Lundh, May 1997 +# +# fixed initialcolor handling in August 1998 +# + + +from tkinter.commondialog import Dialog + +__all__ = ["Chooser", "askcolor"] + + +class Chooser(Dialog): + """Create a dialog for the tk_chooseColor command. + + Args: + master: The master widget for this dialog. If not provided, + defaults to options['parent'] (if defined). + options: Dictionary of options for the tk_chooseColor call. + initialcolor: Specifies the selected color when the + dialog is first displayed. This can be a tk color + string or a 3-tuple of ints in the range (0, 255) + for an RGB triplet. + parent: The parent window of the color dialog. The + color dialog is displayed on top of this. + title: A string for the title of the dialog box. + """ + + command = "tk_chooseColor" + + def _fixoptions(self): + """Ensure initialcolor is a tk color string. + + Convert initialcolor from a RGB triplet to a color string. + """ + try: + color = self.options["initialcolor"] + if isinstance(color, tuple): + # Assume an RGB triplet. + self.options["initialcolor"] = "#%02x%02x%02x" % color + except KeyError: + pass + + def _fixresult(self, widget, result): + """Adjust result returned from call to tk_chooseColor. + + Return both an RGB tuple of ints in the range (0, 255) and the + tk color string in the form #rrggbb. + """ + # Result can be many things: an empty tuple, an empty string, or + # a _tkinter.Tcl_Obj, so this somewhat weird check handles that. + if not result or not str(result): + return None, None # canceled + + # To simplify application code, the color chooser returns + # an RGB tuple together with the Tk color string. + r, g, b = widget.winfo_rgb(result) + return (r//256, g//256, b//256), str(result) + + +# +# convenience stuff + +def askcolor(color=None, **options): + """Display dialog window for selection of a color. + + Convenience wrapper for the Chooser class. Displays the color + chooser dialog with color as the initial value. + """ + + if color: + options = options.copy() + options["initialcolor"] = color + + return Chooser(**options).show() + + +# -------------------------------------------------------------------- +# test stuff + +if __name__ == "__main__": + print("color", askcolor()) diff --git a/Lib/tkinter/commondialog.py b/Lib/tkinter/commondialog.py new file mode 100644 index 00000000000..86f5387e001 --- /dev/null +++ b/Lib/tkinter/commondialog.py @@ -0,0 +1,53 @@ +# base class for tk common dialogues +# +# this module provides a base class for accessing the common +# dialogues available in Tk 4.2 and newer. use filedialog, +# colorchooser, and messagebox to access the individual +# dialogs. +# +# written by Fredrik Lundh, May 1997 +# + +__all__ = ["Dialog"] + +from tkinter import _get_temp_root, _destroy_temp_root + + +class Dialog: + + command = None + + def __init__(self, master=None, **options): + if master is None: + master = options.get('parent') + self.master = master + self.options = options + + def _fixoptions(self): + pass # hook + + def _fixresult(self, widget, result): + return result # hook + + def show(self, **options): + + # update instance options + for k, v in options.items(): + self.options[k] = v + + self._fixoptions() + + master = self.master + if master is None: + master = _get_temp_root() + try: + self._test_callback(master) # The function below is replaced for some tests. + s = master.tk.call(self.command, *master._options(self.options)) + s = self._fixresult(master, s) + finally: + _destroy_temp_root(master) + + return s + + def _test_callback(self, master): + pass diff --git a/Lib/tkinter/constants.py b/Lib/tkinter/constants.py new file mode 100644 index 00000000000..63eee33d24d --- /dev/null +++ b/Lib/tkinter/constants.py @@ -0,0 +1,110 @@ +# Symbolic constants for Tk + +# Booleans +NO=FALSE=OFF=0 +YES=TRUE=ON=1 + +# -anchor and -sticky +N='n' +S='s' +W='w' +E='e' +NW='nw' +SW='sw' +NE='ne' +SE='se' +NS='ns' +EW='ew' +NSEW='nsew' +CENTER='center' + +# -fill +NONE='none' +X='x' +Y='y' +BOTH='both' + +# -side +LEFT='left' +TOP='top' +RIGHT='right' +BOTTOM='bottom' + +# -relief +RAISED='raised' +SUNKEN='sunken' +FLAT='flat' +RIDGE='ridge' +GROOVE='groove' +SOLID = 'solid' + +# -orient +HORIZONTAL='horizontal' +VERTICAL='vertical' + +# -tabs +NUMERIC='numeric' + +# -wrap +CHAR='char' +WORD='word' + +# -align +BASELINE='baseline' + +# -bordermode +INSIDE='inside' +OUTSIDE='outside' + +# Special tags, marks and insert positions +SEL='sel' +SEL_FIRST='sel.first' +SEL_LAST='sel.last' +END='end' +INSERT='insert' +CURRENT='current' +ANCHOR='anchor' +ALL='all' # e.g. Canvas.delete(ALL) + +# Text widget and button states +NORMAL='normal' +DISABLED='disabled' +ACTIVE='active' +# Canvas state +HIDDEN='hidden' + +# Menu item types +CASCADE='cascade' +CHECKBUTTON='checkbutton' +COMMAND='command' +RADIOBUTTON='radiobutton' +SEPARATOR='separator' + +# Selection modes for list boxes +SINGLE='single' +BROWSE='browse' +MULTIPLE='multiple' +EXTENDED='extended' + +# Activestyle for list boxes +# NONE='none' is also valid +DOTBOX='dotbox' +UNDERLINE='underline' + +# Various canvas styles +PIESLICE='pieslice' +CHORD='chord' +ARC='arc' +FIRST='first' +LAST='last' +BUTT='butt' +PROJECTING='projecting' +ROUND='round' +BEVEL='bevel' +MITER='miter' + +# Arguments to xview/yview +MOVETO='moveto' +SCROLL='scroll' +UNITS='units' +PAGES='pages' diff --git a/Lib/tkinter/dialog.py b/Lib/tkinter/dialog.py new file mode 100644 index 00000000000..36ae6c277cb --- /dev/null +++ b/Lib/tkinter/dialog.py @@ -0,0 +1,49 @@ +# dialog.py -- Tkinter interface to the tk_dialog script. + +from tkinter import _cnfmerge, Widget, TclError, Button, Pack + +__all__ = ["Dialog"] + +DIALOG_ICON = 'questhead' + + +class Dialog(Widget): + def __init__(self, master=None, cnf={}, **kw): + cnf = _cnfmerge((cnf, kw)) + self.widgetName = '__dialog__' + self._setup(master, cnf) + self.num = self.tk.getint( + self.tk.call( + 'tk_dialog', self._w, + cnf['title'], cnf['text'], + cnf['bitmap'], cnf['default'], + *cnf['strings'])) + try: Widget.destroy(self) + except TclError: pass + + def destroy(self): pass + + +def _test(): + d = Dialog(None, {'title': 'File Modified', + 'text': + 'File "Python.h" has been modified' + ' since the last time it was saved.' + ' Do you want to save it before' + ' exiting the application.', + 'bitmap': DIALOG_ICON, + 'default': 0, + 'strings': ('Save File', + 'Discard Changes', + 'Return to Editor')}) + print(d.num) + + +if __name__ == '__main__': + t = Button(None, {'text': 'Test', + 'command': _test, + Pack: {}}) + q = Button(None, {'text': 'Quit', + 'command': t.quit, + Pack: {}}) + t.mainloop() diff --git a/Lib/tkinter/dnd.py b/Lib/tkinter/dnd.py new file mode 100644 index 00000000000..acec61ba71f --- /dev/null +++ b/Lib/tkinter/dnd.py @@ -0,0 +1,324 @@ +"""Drag-and-drop support for Tkinter. + +This is very preliminary. I currently only support dnd *within* one +application, between different windows (or within the same window). + +I am trying to make this as generic as possible -- not dependent on +the use of a particular widget or icon type, etc. I also hope that +this will work with Pmw. + +To enable an object to be dragged, you must create an event binding +for it that starts the drag-and-drop process. Typically, you should +bind <ButtonPress> to a callback function that you write. The function +should call Tkdnd.dnd_start(source, event), where 'source' is the +object to be dragged, and 'event' is the event that invoked the call +(the argument to your callback function). Even though this is a class +instantiation, the returned instance should not be stored -- it will +be kept alive automatically for the duration of the drag-and-drop. + +When a drag-and-drop is already in process for the Tk interpreter, the +call is *ignored*; this normally averts starting multiple simultaneous +dnd processes, e.g. because different button callbacks all +dnd_start(). + +The object is *not* necessarily a widget -- it can be any +application-specific object that is meaningful to potential +drag-and-drop targets. + +Potential drag-and-drop targets are discovered as follows. Whenever +the mouse moves, and at the start and end of a drag-and-drop move, the +Tk widget directly under the mouse is inspected. This is the target +widget (not to be confused with the target object, yet to be +determined). If there is no target widget, there is no dnd target +object. If there is a target widget, and it has an attribute +dnd_accept, this should be a function (or any callable object). The +function is called as dnd_accept(source, event), where 'source' is the +object being dragged (the object passed to dnd_start() above), and +'event' is the most recent event object (generally a <Motion> event; +it can also be <ButtonPress> or <ButtonRelease>). If the dnd_accept() +function returns something other than None, this is the new dnd target +object. If dnd_accept() returns None, or if the target widget has no +dnd_accept attribute, the target widget's parent is considered as the +target widget, and the search for a target object is repeated from +there. If necessary, the search is repeated all the way up to the +root widget. If none of the target widgets can produce a target +object, there is no target object (the target object is None). + +The target object thus produced, if any, is called the new target +object. It is compared with the old target object (or None, if there +was no old target widget). There are several cases ('source' is the +source object, and 'event' is the most recent event object): + +- Both the old and new target objects are None. Nothing happens. + +- The old and new target objects are the same object. Its method +dnd_motion(source, event) is called. + +- The old target object was None, and the new target object is not +None. The new target object's method dnd_enter(source, event) is +called. + +- The new target object is None, and the old target object is not +None. The old target object's method dnd_leave(source, event) is +called. + +- The old and new target objects differ and neither is None. The old +target object's method dnd_leave(source, event), and then the new +target object's method dnd_enter(source, event) is called. + +Once this is done, the new target object replaces the old one, and the +Tk mainloop proceeds. The return value of the methods mentioned above +is ignored; if they raise an exception, the normal exception handling +mechanisms take over. + +The drag-and-drop processes can end in two ways: a final target object +is selected, or no final target object is selected. When a final +target object is selected, it will always have been notified of the +potential drop by a call to its dnd_enter() method, as described +above, and possibly one or more calls to its dnd_motion() method; its +dnd_leave() method has not been called since the last call to +dnd_enter(). The target is notified of the drop by a call to its +method dnd_commit(source, event). + +If no final target object is selected, and there was an old target +object, its dnd_leave(source, event) method is called to complete the +dnd sequence. + +Finally, the source object is notified that the drag-and-drop process +is over, by a call to source.dnd_end(target, event), specifying either +the selected target object, or None if no target object was selected. +The source object can use this to implement the commit action; this is +sometimes simpler than to do it in the target's dnd_commit(). The +target's dnd_commit() method could then simply be aliased to +dnd_leave(). + +At any time during a dnd sequence, the application can cancel the +sequence by calling the cancel() method on the object returned by +dnd_start(). This will call dnd_leave() if a target is currently +active; it will never call dnd_commit(). + +""" + +import tkinter + +__all__ = ["dnd_start", "DndHandler"] + + +# The factory function + +def dnd_start(source, event): + h = DndHandler(source, event) + if h.root is not None: + return h + else: + return None + + +# The class that does the work + +class DndHandler: + + root = None + + def __init__(self, source, event): + if event.num > 5: + return + root = event.widget._root() + try: + root.__dnd + return # Don't start recursive dnd + except AttributeError: + root.__dnd = self + self.root = root + self.source = source + self.target = None + self.initial_button = button = event.num + self.initial_widget = widget = event.widget + self.release_pattern = "<B%d-ButtonRelease-%d>" % (button, button) + self.save_cursor = widget['cursor'] or "" + widget.bind(self.release_pattern, self.on_release) + widget.bind("<Motion>", self.on_motion) + widget['cursor'] = "hand2" + + def __del__(self): + root = self.root + self.root = None + if root is not None: + try: + del root.__dnd + except AttributeError: + pass + + def on_motion(self, event): + x, y = event.x_root, event.y_root + target_widget = self.initial_widget.winfo_containing(x, y) + source = self.source + new_target = None + while target_widget is not None: + try: + attr = target_widget.dnd_accept + except AttributeError: + pass + else: + new_target = attr(source, event) + if new_target is not None: + break + target_widget = target_widget.master + old_target = self.target + if old_target is new_target: + if old_target is not None: + old_target.dnd_motion(source, event) + else: + if old_target is not None: + self.target = None + old_target.dnd_leave(source, event) + if new_target is not None: + new_target.dnd_enter(source, event) + self.target = new_target + + def on_release(self, event): + self.finish(event, 1) + + def cancel(self, event=None): + self.finish(event, 0) + + def finish(self, event, commit=0): + target = self.target + source = self.source + widget = self.initial_widget + root = self.root + try: + del root.__dnd + self.initial_widget.unbind(self.release_pattern) + self.initial_widget.unbind("<Motion>") + widget['cursor'] = self.save_cursor + self.target = self.source = self.initial_widget = self.root = None + if target is not None: + if commit: + target.dnd_commit(source, event) + else: + target.dnd_leave(source, event) + finally: + source.dnd_end(target, event) + + +# ---------------------------------------------------------------------- +# The rest is here for testing and demonstration purposes only! + +class Icon: + + def __init__(self, name): + self.name = name + self.canvas = self.label = self.id = None + + def attach(self, canvas, x=10, y=10): + if canvas is self.canvas: + self.canvas.coords(self.id, x, y) + return + if self.canvas is not None: + self.detach() + if canvas is None: + return + label = tkinter.Label(canvas, text=self.name, + borderwidth=2, relief="raised") + id = canvas.create_window(x, y, window=label, anchor="nw") + self.canvas = canvas + self.label = label + self.id = id + label.bind("<ButtonPress>", self.press) + + def detach(self): + canvas = self.canvas + if canvas is None: + return + id = self.id + label = self.label + self.canvas = self.label = self.id = None + canvas.delete(id) + label.destroy() + + def press(self, event): + if dnd_start(self, event): + # where the pointer is relative to the label widget: + self.x_off = event.x + self.y_off = event.y + # where the widget is relative to the canvas: + self.x_orig, self.y_orig = self.canvas.coords(self.id) + + def move(self, event): + x, y = self.where(self.canvas, event) + self.canvas.coords(self.id, x, y) + + def putback(self): + self.canvas.coords(self.id, self.x_orig, self.y_orig) + + def where(self, canvas, event): + # where the corner of the canvas is relative to the screen: + x_org = canvas.winfo_rootx() + y_org = canvas.winfo_rooty() + # where the pointer is relative to the canvas widget: + x = event.x_root - x_org + y = event.y_root - y_org + # compensate for initial pointer offset + return x - self.x_off, y - self.y_off + + def dnd_end(self, target, event): + pass + + +class Tester: + + def __init__(self, root): + self.top = tkinter.Toplevel(root) + self.canvas = tkinter.Canvas(self.top, width=100, height=100) + self.canvas.pack(fill="both", expand=1) + self.canvas.dnd_accept = self.dnd_accept + + def dnd_accept(self, source, event): + return self + + def dnd_enter(self, source, event): + self.canvas.focus_set() # Show highlight border + x, y = source.where(self.canvas, event) + x1, y1, x2, y2 = source.canvas.bbox(source.id) + dx, dy = x2-x1, y2-y1 + self.dndid = self.canvas.create_rectangle(x, y, x+dx, y+dy) + self.dnd_motion(source, event) + + def dnd_motion(self, source, event): + x, y = source.where(self.canvas, event) + x1, y1, x2, y2 = self.canvas.bbox(self.dndid) + self.canvas.move(self.dndid, x-x1, y-y1) + + def dnd_leave(self, source, event): + self.top.focus_set() # Hide highlight border + self.canvas.delete(self.dndid) + self.dndid = None + + def dnd_commit(self, source, event): + self.dnd_leave(source, event) + x, y = source.where(self.canvas, event) + source.attach(self.canvas, x, y) + + +def test(): + root = tkinter.Tk() + root.geometry("+1+1") + tkinter.Button(command=root.quit, text="Quit").pack() + t1 = Tester(root) + t1.top.geometry("+1+60") + t2 = Tester(root) + t2.top.geometry("+120+60") + t3 = Tester(root) + t3.top.geometry("+240+60") + i1 = Icon("ICON1") + i2 = Icon("ICON2") + i3 = Icon("ICON3") + i1.attach(t1.canvas) + i2.attach(t2.canvas) + i3.attach(t3.canvas) + root.mainloop() + + +if __name__ == '__main__': + test() diff --git a/Lib/tkinter/filedialog.py b/Lib/tkinter/filedialog.py new file mode 100644 index 00000000000..e2eff98e601 --- /dev/null +++ b/Lib/tkinter/filedialog.py @@ -0,0 +1,492 @@ +"""File selection dialog classes. + +Classes: + +- FileDialog +- LoadFileDialog +- SaveFileDialog + +This module also presents tk common file dialogues, it provides interfaces +to the native file dialogues available in Tk 4.2 and newer, and the +directory dialogue available in Tk 8.3 and newer. +These interfaces were written by Fredrik Lundh, May 1997. +""" +__all__ = ["FileDialog", "LoadFileDialog", "SaveFileDialog", + "Open", "SaveAs", "Directory", + "askopenfilename", "asksaveasfilename", "askopenfilenames", + "askopenfile", "askopenfiles", "asksaveasfile", "askdirectory"] + +import fnmatch +import os +from tkinter import ( + Frame, LEFT, YES, BOTTOM, Entry, TOP, Button, Tk, X, + Toplevel, RIGHT, Y, END, Listbox, BOTH, Scrollbar, +) +from tkinter.dialog import Dialog +from tkinter import commondialog +from tkinter.simpledialog import _setup_dialog + + +dialogstates = {} + + +class FileDialog: + + """Standard file selection dialog -- no checks on selected file. + + Usage: + + d = FileDialog(master) + fname = d.go(dir_or_file, pattern, default, key) + if fname is None: ...canceled... + else: ...open file... + + All arguments to go() are optional. + + The 'key' argument specifies a key in the global dictionary + 'dialogstates', which keeps track of the values for the directory + and pattern arguments, overriding the values passed in (it does + not keep track of the default argument!). If no key is specified, + the dialog keeps no memory of previous state. Note that memory is + kept even when the dialog is canceled. (All this emulates the + behavior of the Macintosh file selection dialogs.) + + """ + + title = "File Selection Dialog" + + def __init__(self, master, title=None): + if title is None: title = self.title + self.master = master + self.directory = None + + self.top = Toplevel(master) + self.top.title(title) + self.top.iconname(title) + _setup_dialog(self.top) + + self.botframe = Frame(self.top) + self.botframe.pack(side=BOTTOM, fill=X) + + self.selection = Entry(self.top) + self.selection.pack(side=BOTTOM, fill=X) + self.selection.bind('<Return>', self.ok_event) + + self.filter = Entry(self.top) + self.filter.pack(side=TOP, fill=X) + self.filter.bind('<Return>', self.filter_command) + + self.midframe = Frame(self.top) + self.midframe.pack(expand=YES, fill=BOTH) + + self.filesbar = Scrollbar(self.midframe) + self.filesbar.pack(side=RIGHT, fill=Y) + self.files = Listbox(self.midframe, exportselection=0, + yscrollcommand=(self.filesbar, 'set')) + self.files.pack(side=RIGHT, expand=YES, fill=BOTH) + btags = self.files.bindtags() + self.files.bindtags(btags[1:] + btags[:1]) + self.files.bind('<ButtonRelease-1>', self.files_select_event) + self.files.bind('<Double-ButtonRelease-1>', self.files_double_event) + self.filesbar.config(command=(self.files, 'yview')) + + self.dirsbar = Scrollbar(self.midframe) + self.dirsbar.pack(side=LEFT, fill=Y) + self.dirs = Listbox(self.midframe, exportselection=0, + yscrollcommand=(self.dirsbar, 'set')) + self.dirs.pack(side=LEFT, expand=YES, fill=BOTH) + self.dirsbar.config(command=(self.dirs, 'yview')) + btags = self.dirs.bindtags() + self.dirs.bindtags(btags[1:] + btags[:1]) + self.dirs.bind('<ButtonRelease-1>', self.dirs_select_event) + self.dirs.bind('<Double-ButtonRelease-1>', self.dirs_double_event) + + self.ok_button = Button(self.botframe, + text="OK", + command=self.ok_command) + self.ok_button.pack(side=LEFT) + self.filter_button = Button(self.botframe, + text="Filter", + command=self.filter_command) + self.filter_button.pack(side=LEFT, expand=YES) + self.cancel_button = Button(self.botframe, + text="Cancel", + command=self.cancel_command) + self.cancel_button.pack(side=RIGHT) + + self.top.protocol('WM_DELETE_WINDOW', self.cancel_command) + # XXX Are the following okay for a general audience? + self.top.bind('<Alt-w>', self.cancel_command) + self.top.bind('<Alt-W>', self.cancel_command) + + def go(self, dir_or_file=os.curdir, pattern="*", default="", key=None): + if key and key in dialogstates: + self.directory, pattern = dialogstates[key] + else: + dir_or_file = os.path.expanduser(dir_or_file) + if os.path.isdir(dir_or_file): + self.directory = dir_or_file + else: + self.directory, default = os.path.split(dir_or_file) + self.set_filter(self.directory, pattern) + self.set_selection(default) + self.filter_command() + self.selection.focus_set() + self.top.wait_visibility() # window needs to be visible for the grab + self.top.grab_set() + self.how = None + self.master.mainloop() # Exited by self.quit(how) + if key: + directory, pattern = self.get_filter() + if self.how: + directory = os.path.dirname(self.how) + dialogstates[key] = directory, pattern + self.top.destroy() + return self.how + + def quit(self, how=None): + self.how = how + self.master.quit() # Exit mainloop() + + def dirs_double_event(self, event): + self.filter_command() + + def dirs_select_event(self, event): + dir, pat = self.get_filter() + subdir = self.dirs.get('active') + dir = os.path.normpath(os.path.join(self.directory, subdir)) + self.set_filter(dir, pat) + + def files_double_event(self, event): + self.ok_command() + + def files_select_event(self, event): + file = self.files.get('active') + self.set_selection(file) + + def ok_event(self, event): + self.ok_command() + + def ok_command(self): + self.quit(self.get_selection()) + + def filter_command(self, event=None): + dir, pat = self.get_filter() + try: + names = os.listdir(dir) + except OSError: + self.master.bell() + return + self.directory = dir + self.set_filter(dir, pat) + names.sort() + subdirs = [os.pardir] + matchingfiles = [] + for name in names: + fullname = os.path.join(dir, name) + if os.path.isdir(fullname): + subdirs.append(name) + elif fnmatch.fnmatch(name, pat): + matchingfiles.append(name) + self.dirs.delete(0, END) + for name in subdirs: + self.dirs.insert(END, name) + self.files.delete(0, END) + for name in matchingfiles: + self.files.insert(END, name) + head, tail = os.path.split(self.get_selection()) + if tail == os.curdir: tail = '' + self.set_selection(tail) + + def get_filter(self): + filter = self.filter.get() + filter = os.path.expanduser(filter) + if filter[-1:] == os.sep or os.path.isdir(filter): + filter = os.path.join(filter, "*") + return os.path.split(filter) + + def get_selection(self): + file = self.selection.get() + file = os.path.expanduser(file) + return file + + def cancel_command(self, event=None): + self.quit() + + def set_filter(self, dir, pat): + if not os.path.isabs(dir): + try: + pwd = os.getcwd() + except OSError: + pwd = None + if pwd: + dir = os.path.join(pwd, dir) + dir = os.path.normpath(dir) + self.filter.delete(0, END) + self.filter.insert(END, os.path.join(dir or os.curdir, pat or "*")) + + def set_selection(self, file): + self.selection.delete(0, END) + self.selection.insert(END, os.path.join(self.directory, file)) + + +class LoadFileDialog(FileDialog): + + """File selection dialog which checks that the file exists.""" + + title = "Load File Selection Dialog" + + def ok_command(self): + file = self.get_selection() + if not os.path.isfile(file): + self.master.bell() + else: + self.quit(file) + + +class SaveFileDialog(FileDialog): + + """File selection dialog which checks that the file may be created.""" + + title = "Save File Selection Dialog" + + def ok_command(self): + file = self.get_selection() + if os.path.exists(file): + if os.path.isdir(file): + self.master.bell() + return + d = Dialog(self.top, + title="Overwrite Existing File Question", + text="Overwrite existing file %r?" % (file,), + bitmap='questhead', + default=1, + strings=("Yes", "Cancel")) + if d.num != 0: + return + else: + head, tail = os.path.split(file) + if not os.path.isdir(head): + self.master.bell() + return + self.quit(file) + + +# For the following classes and modules: +# +# options (all have default values): +# +# - defaultextension: added to filename if not explicitly given +# +# - filetypes: sequence of (label, pattern) tuples. the same pattern +# may occur with several patterns. use "*" as pattern to indicate +# all files. +# +# - initialdir: initial directory. preserved by dialog instance. +# +# - initialfile: initial file (ignored by the open dialog). preserved +# by dialog instance. +# +# - parent: which window to place the dialog on top of +# +# - title: dialog title +# +# - multiple: if true user may select more than one file +# +# options for the directory chooser: +# +# - initialdir, parent, title: see above +# +# - mustexist: if true, user must pick an existing directory +# + + +class _Dialog(commondialog.Dialog): + + def _fixoptions(self): + try: + # make sure "filetypes" is a tuple + self.options["filetypes"] = tuple(self.options["filetypes"]) + except KeyError: + pass + + def _fixresult(self, widget, result): + if result: + # keep directory and filename until next time + # convert Tcl path objects to strings + try: + result = result.string + except AttributeError: + # it already is a string + pass + path, file = os.path.split(result) + self.options["initialdir"] = path + self.options["initialfile"] = file + self.filename = result # compatibility + return result + + +# +# file dialogs + +class Open(_Dialog): + "Ask for a filename to open" + + command = "tk_getOpenFile" + + def _fixresult(self, widget, result): + if isinstance(result, tuple): + # multiple results: + result = tuple([getattr(r, "string", r) for r in result]) + if result: + path, file = os.path.split(result[0]) + self.options["initialdir"] = path + # don't set initialfile or filename, as we have multiple of these + return result + if not widget.tk.wantobjects() and "multiple" in self.options: + # Need to split result explicitly + return self._fixresult(widget, widget.tk.splitlist(result)) + return _Dialog._fixresult(self, widget, result) + + +class SaveAs(_Dialog): + "Ask for a filename to save as" + + command = "tk_getSaveFile" + + +# the directory dialog has its own _fix routines. +class Directory(commondialog.Dialog): + "Ask for a directory" + + command = "tk_chooseDirectory" + + def _fixresult(self, widget, result): + if result: + # convert Tcl path objects to strings + try: + result = result.string + except AttributeError: + # it already is a string + pass + # keep directory until next time + self.options["initialdir"] = result + self.directory = result # compatibility + return result + +# +# convenience stuff + + +def askopenfilename(**options): + "Ask for a filename to open" + + return Open(**options).show() + + +def asksaveasfilename(**options): + "Ask for a filename to save as" + + return SaveAs(**options).show() + + +def askopenfilenames(**options): + """Ask for multiple filenames to open + + Returns a list of filenames or empty list if + cancel button selected + """ + options["multiple"]=1 + return Open(**options).show() + +# FIXME: are the following perhaps a bit too convenient? + + +def askopenfile(mode = "r", **options): + "Ask for a filename to open, and returned the opened file" + + filename = Open(**options).show() + if filename: + return open(filename, mode) + return None + + +def askopenfiles(mode = "r", **options): + """Ask for multiple filenames and return the open file + objects + + returns a list of open file objects or an empty list if + cancel selected + """ + + files = askopenfilenames(**options) + if files: + ofiles=[] + for filename in files: + ofiles.append(open(filename, mode)) + files=ofiles + return files + + +def asksaveasfile(mode = "w", **options): + "Ask for a filename to save as, and returned the opened file" + + filename = SaveAs(**options).show() + if filename: + return open(filename, mode) + return None + + +def askdirectory (**options): + "Ask for a directory, and return the file name" + return Directory(**options).show() + + +# -------------------------------------------------------------------- +# test stuff + +def test(): + """Simple test program.""" + root = Tk() + root.withdraw() + fd = LoadFileDialog(root) + loadfile = fd.go(key="test") + fd = SaveFileDialog(root) + savefile = fd.go(key="test") + print(loadfile, savefile) + + # Since the file name may contain non-ASCII characters, we need + # to find an encoding that likely supports the file name, and + # displays correctly on the terminal. + + # Start off with UTF-8 + enc = "utf-8" + + # See whether CODESET is defined + try: + import locale + locale.setlocale(locale.LC_ALL,'') + enc = locale.nl_langinfo(locale.CODESET) + except (ImportError, AttributeError): + pass + + # dialog for opening files + + openfilename=askopenfilename(filetypes=[("all files", "*")]) + try: + fp=open(openfilename,"r") + fp.close() + except BaseException as exc: + print("Could not open File: ") + print(exc) + + print("open", openfilename.encode(enc)) + + # dialog for saving files + + saveasfilename=asksaveasfilename() + print("saveas", saveasfilename.encode(enc)) + + +if __name__ == '__main__': + test() diff --git a/Lib/tkinter/font.py b/Lib/tkinter/font.py new file mode 100644 index 00000000000..3e24e28ef58 --- /dev/null +++ b/Lib/tkinter/font.py @@ -0,0 +1,239 @@ +# Tkinter font wrapper +# +# written by Fredrik Lundh, February 1998 +# + +import itertools +import tkinter + +__version__ = "0.9" +__all__ = ["NORMAL", "ROMAN", "BOLD", "ITALIC", + "nametofont", "Font", "families", "names"] + +# weight/slant +NORMAL = "normal" +ROMAN = "roman" +BOLD = "bold" +ITALIC = "italic" + + +def nametofont(name, root=None): + """Given the name of a tk named font, returns a Font representation. + """ + return Font(name=name, exists=True, root=root) + + +class Font: + """Represents a named font. + + Constructor options are: + + font -- font specifier (name, system font, or (family, size, style)-tuple) + name -- name to use for this font configuration (defaults to a unique name) + exists -- does a named font by this name already exist? + Creates a new named font if False, points to the existing font if True. + Raises _tkinter.TclError if the assertion is false. + + the following are ignored if font is specified: + + family -- font 'family', e.g. Courier, Times, Helvetica + size -- font size in points + weight -- font thickness: NORMAL, BOLD + slant -- font slant: ROMAN, ITALIC + underline -- font underlining: false (0), true (1) + overstrike -- font strikeout: false (0), true (1) + + """ + + counter = itertools.count(1) + + def _set(self, kw): + options = [] + for k, v in kw.items(): + options.append("-"+k) + options.append(str(v)) + return tuple(options) + + def _get(self, args): + options = [] + for k in args: + options.append("-"+k) + return tuple(options) + + def _mkdict(self, args): + options = {} + for i in range(0, len(args), 2): + options[args[i][1:]] = args[i+1] + return options + + def __init__(self, root=None, font=None, name=None, exists=False, + **options): + if root is None: + root = tkinter._get_default_root('use font') + tk = getattr(root, 'tk', root) + if font: + # get actual settings corresponding to the given font + font = tk.splitlist(tk.call("font", "actual", font)) + else: + font = self._set(options) + if not name: + name = "font" + str(next(self.counter)) + self.name = name + + if exists: + self.delete_font = False + # confirm font exists + if self.name not in tk.splitlist(tk.call("font", "names")): + raise tkinter._tkinter.TclError( + "named font %s does not already exist" % (self.name,)) + # if font config info supplied, apply it + if font: + tk.call("font", "configure", self.name, *font) + else: + # create new font (raises TclError if the font exists) + tk.call("font", "create", self.name, *font) + self.delete_font = True + self._tk = tk + self._split = tk.splitlist + self._call = tk.call + + def __str__(self): + return self.name + + def __repr__(self): + return f"<{self.__class__.__module__}.{self.__class__.__qualname__}" \ + f" object {self.name!r}>" + + def __eq__(self, other): + if not isinstance(other, Font): + return NotImplemented + return self.name == other.name and self._tk == other._tk + + def __getitem__(self, key): + return self.cget(key) + + def __setitem__(self, key, value): + self.configure(**{key: value}) + + def __del__(self): + try: + if self.delete_font: + self._call("font", "delete", self.name) + except Exception: + pass + + def copy(self): + "Return a distinct copy of the current font" + return Font(self._tk, **self.actual()) + + def actual(self, option=None, displayof=None): + "Return actual font attributes" + args = () + if displayof: + args = ('-displayof', displayof) + if option: + args = args + ('-' + option, ) + return self._call("font", "actual", self.name, *args) + else: + return self._mkdict( + self._split(self._call("font", "actual", self.name, *args))) + + def cget(self, option): + "Get font attribute" + return self._call("font", "config", self.name, "-"+option) + + def config(self, **options): + "Modify font attributes" + if options: + self._call("font", "config", self.name, + *self._set(options)) + else: + return self._mkdict( + self._split(self._call("font", "config", self.name))) + + configure = config + + def measure(self, text, displayof=None): + "Return text width" + args = (text,) + if displayof: + args = ('-displayof', displayof, text) + return self._tk.getint(self._call("font", "measure", self.name, *args)) + + def metrics(self, *options, **kw): + """Return font metrics. + + For best performance, create a dummy widget + using this font before calling this method.""" + args = () + displayof = kw.pop('displayof', None) + if displayof: + args = ('-displayof', displayof) + if options: + args = args + self._get(options) + return self._tk.getint( + self._call("font", "metrics", self.name, *args)) + else: + res = self._split(self._call("font", "metrics", self.name, *args)) + options = {} + for i in range(0, len(res), 2): + options[res[i][1:]] = self._tk.getint(res[i+1]) + return options + + +def families(root=None, displayof=None): + "Get font families (as a tuple)" + if root is None: + root = tkinter._get_default_root('use font.families()') + args = () + if displayof: + args = ('-displayof', displayof) + return root.tk.splitlist(root.tk.call("font", "families", *args)) + + +def names(root=None): + "Get names of defined fonts (as a tuple)" + if root is None: + root = tkinter._get_default_root('use font.names()') + return root.tk.splitlist(root.tk.call("font", "names")) + + +# -------------------------------------------------------------------- +# test stuff + +if __name__ == "__main__": + + root = tkinter.Tk() + + # create a font + f = Font(family="times", size=30, weight=NORMAL) + + print(f.actual()) + print(f.actual("family")) + print(f.actual("weight")) + + print(f.config()) + print(f.cget("family")) + print(f.cget("weight")) + + print(names()) + + print(f.measure("hello"), f.metrics("linespace")) + + print(f.metrics(displayof=root)) + + f = Font(font=("Courier", 20, "bold")) + print(f.measure("hello"), f.metrics("linespace", displayof=root)) + + w = tkinter.Label(root, text="Hello, world", font=f) + w.pack() + + w = tkinter.Button(root, text="Quit!", command=root.destroy) + w.pack() + + fb = Font(font=w["font"]).copy() + fb.config(weight=BOLD) + + w.config(font=fb) + + tkinter.mainloop() diff --git a/Lib/tkinter/messagebox.py b/Lib/tkinter/messagebox.py new file mode 100644 index 00000000000..5f0343b660c --- /dev/null +++ b/Lib/tkinter/messagebox.py @@ -0,0 +1,146 @@ +# tk common message boxes +# +# this module provides an interface to the native message boxes +# available in Tk 4.2 and newer. +# +# written by Fredrik Lundh, May 1997 +# + +# +# options (all have default values): +# +# - default: which button to make default (one of the reply codes) +# +# - icon: which icon to display (see below) +# +# - message: the message to display +# +# - parent: which window to place the dialog on top of +# +# - title: dialog title +# +# - type: dialog type; that is, which buttons to display (see below) +# + +from tkinter.commondialog import Dialog + +__all__ = ["showinfo", "showwarning", "showerror", + "askquestion", "askokcancel", "askyesno", + "askyesnocancel", "askretrycancel"] + +# +# constants + +# icons +ERROR = "error" +INFO = "info" +QUESTION = "question" +WARNING = "warning" + +# types +ABORTRETRYIGNORE = "abortretryignore" +OK = "ok" +OKCANCEL = "okcancel" +RETRYCANCEL = "retrycancel" +YESNO = "yesno" +YESNOCANCEL = "yesnocancel" + +# replies +ABORT = "abort" +RETRY = "retry" +IGNORE = "ignore" +OK = "ok" +CANCEL = "cancel" +YES = "yes" +NO = "no" + + +# +# message dialog class + +class Message(Dialog): + "A message box" + + command = "tk_messageBox" + + +# +# convenience stuff + +# Rename _icon and _type options to allow overriding them in options +def _show(title=None, message=None, _icon=None, _type=None, **options): + if _icon and "icon" not in options: options["icon"] = _icon + if _type and "type" not in options: options["type"] = _type + if title: options["title"] = title + if message: options["message"] = message + res = Message(**options).show() + # In some Tcl installations, yes/no is converted into a boolean. + if isinstance(res, bool): + if res: + return YES + return NO + # In others we get a Tcl_Obj. + return str(res) + + +def showinfo(title=None, message=None, **options): + "Show an info message" + return _show(title, message, INFO, OK, **options) + + +def showwarning(title=None, message=None, **options): + "Show a warning message" + return _show(title, message, WARNING, OK, **options) + + +def showerror(title=None, message=None, **options): + "Show an error message" + return _show(title, message, ERROR, OK, **options) + + +def askquestion(title=None, message=None, **options): + "Ask a question" + return _show(title, message, QUESTION, YESNO, **options) + + +def askokcancel(title=None, message=None, **options): + "Ask if operation should proceed; return true if the answer is ok" + s = _show(title, message, QUESTION, OKCANCEL, **options) + return s == OK + + +def askyesno(title=None, message=None, **options): + "Ask a question; return true if the answer is yes" + s = _show(title, message, QUESTION, YESNO, **options) + return s == YES + + +def askyesnocancel(title=None, message=None, **options): + "Ask a question; return true if the answer is yes, None if cancelled." + s = _show(title, message, QUESTION, YESNOCANCEL, **options) + # s might be a Tcl index object, so convert it to a string + s = str(s) + if s == CANCEL: + return None + return s == YES + + +def askretrycancel(title=None, message=None, **options): + "Ask if operation should be retried; return true if the answer is yes" + s = _show(title, message, WARNING, RETRYCANCEL, **options) + return s == RETRY + + +# -------------------------------------------------------------------- +# test stuff + +if __name__ == "__main__": + + print("info", showinfo("Spam", "Egg Information")) + print("warning", showwarning("Spam", "Egg Warning")) + print("error", showerror("Spam", "Egg Alert")) + print("question", askquestion("Spam", "Question?")) + print("proceed", askokcancel("Spam", "Proceed?")) + print("yes/no", askyesno("Spam", "Got it?")) + print("yes/no/cancel", askyesnocancel("Spam", "Want it?")) + print("try again", askretrycancel("Spam", "Try again?")) diff --git a/Lib/tkinter/scrolledtext.py b/Lib/tkinter/scrolledtext.py new file mode 100644 index 00000000000..4f9a8815b61 --- /dev/null +++ b/Lib/tkinter/scrolledtext.py @@ -0,0 +1,56 @@ +"""A ScrolledText widget feels like a text widget but also has a +vertical scroll bar on its right. (Later, options may be added to +add a horizontal bar as well, to make the bars disappear +automatically when not needed, to move them to the other side of the +window, etc.) + +Configuration options are passed to the Text widget. +A Frame widget is inserted between the master and the text, to hold +the Scrollbar widget. +Most methods calls are inherited from the Text widget; Pack, Grid and +Place methods are redirected to the Frame widget however. +""" + +from tkinter import Frame, Text, Scrollbar, Pack, Grid, Place +from tkinter.constants import RIGHT, LEFT, Y, BOTH + +__all__ = ['ScrolledText'] + + +class ScrolledText(Text): + def __init__(self, master=None, **kw): + self.frame = Frame(master) + self.vbar = Scrollbar(self.frame) + self.vbar.pack(side=RIGHT, fill=Y) + + kw.update({'yscrollcommand': self.vbar.set}) + Text.__init__(self, self.frame, **kw) + self.pack(side=LEFT, fill=BOTH, expand=True) + self.vbar['command'] = self.yview + + # Copy geometry methods of self.frame without overriding Text + # methods -- hack! + text_meths = vars(Text).keys() + methods = vars(Pack).keys() | vars(Grid).keys() | vars(Place).keys() + methods = methods.difference(text_meths) + + for m in methods: + if m[0] != '_' and m != 'config' and m != 'configure': + setattr(self, m, getattr(self.frame, m)) + + def __str__(self): + return str(self.frame) + + +def example(): + from tkinter.constants import END + + stext = ScrolledText(bg='white', height=10) + stext.insert(END, __doc__) + stext.pack(fill=BOTH, side=LEFT, expand=True) + stext.focus_set() + stext.mainloop() + + +if __name__ == "__main__": + example() diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py new file mode 100644 index 00000000000..6e5b025a9f9 --- /dev/null +++ b/Lib/tkinter/simpledialog.py @@ -0,0 +1,441 @@ +# +# An Introduction to Tkinter +# +# Copyright (c) 1997 by Fredrik Lundh +# +# This copyright applies to Dialog, askinteger, askfloat and asktring +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +"""This modules handles dialog boxes. + +It contains the following public symbols: + +SimpleDialog -- A simple but flexible modal dialog box + +Dialog -- a base class for dialogs + +askinteger -- get an integer from the user + +askfloat -- get a float from the user + +askstring -- get a string from the user +""" + +from tkinter import * +from tkinter import _get_temp_root, _destroy_temp_root +from tkinter import messagebox + + +class SimpleDialog: + + def __init__(self, master, + text='', buttons=[], default=None, cancel=None, + title=None, class_=None): + if class_: + self.root = Toplevel(master, class_=class_) + else: + self.root = Toplevel(master) + if title: + self.root.title(title) + self.root.iconname(title) + + _setup_dialog(self.root) + + self.message = Message(self.root, text=text, aspect=400) + self.message.pack(expand=1, fill=BOTH) + self.frame = Frame(self.root) + self.frame.pack() + self.num = default + self.cancel = cancel + self.default = default + self.root.bind('<Return>', self.return_event) + for num in range(len(buttons)): + s = buttons[num] + b = Button(self.frame, text=s, + command=(lambda self=self, num=num: self.done(num))) + if num == default: + b.config(relief=RIDGE, borderwidth=8) + b.pack(side=LEFT, fill=BOTH, expand=1) + self.root.protocol('WM_DELETE_WINDOW', self.wm_delete_window) + self.root.transient(master) + _place_window(self.root, master) + + def go(self): + self.root.wait_visibility() + self.root.grab_set() + self.root.mainloop() + self.root.destroy() + return self.num + + def return_event(self, event): + if self.default is None: + self.root.bell() + else: + self.done(self.default) + + def wm_delete_window(self): + if self.cancel is None: + self.root.bell() + else: + self.done(self.cancel) + + def done(self, num): + self.num = num + self.root.quit() + + +class Dialog(Toplevel): + + '''Class to open dialogs. + + This class is intended as a base class for custom dialogs + ''' + + def __init__(self, parent, title = None): + '''Initialize a dialog. + + Arguments: + + parent -- a parent window (the application window) + + title -- the dialog title + ''' + master = parent + if master is None: + master = _get_temp_root() + + Toplevel.__init__(self, master) + + self.withdraw() # remain invisible for now + # If the parent is not viewable, don't + # make the child transient, or else it + # would be opened withdrawn + if parent is not None and parent.winfo_viewable(): + self.transient(parent) + + if title: + self.title(title) + + _setup_dialog(self) + + self.parent = parent + + self.result = None + + body = Frame(self) + self.initial_focus = self.body(body) + body.pack(padx=5, pady=5) + + self.buttonbox() + + if self.initial_focus is None: + self.initial_focus = self + + self.protocol("WM_DELETE_WINDOW", self.cancel) + + _place_window(self, parent) + + self.initial_focus.focus_set() + + # wait for window to appear on screen before calling grab_set + self.wait_visibility() + self.grab_set() + self.wait_window(self) + + def destroy(self): + '''Destroy the window''' + self.initial_focus = None + Toplevel.destroy(self) + _destroy_temp_root(self.master) + + # + # construction hooks + + def body(self, master): + '''create dialog body. + + return widget that should have initial focus. + This method should be overridden, and is called + by the __init__ method. + ''' + pass + + def buttonbox(self): + '''add standard button box. + + override if you do not want the standard buttons + ''' + + box = Frame(self) + + w = Button(box, text="OK", width=10, command=self.ok, default=ACTIVE) + w.pack(side=LEFT, padx=5, pady=5) + w = Button(box, text="Cancel", width=10, command=self.cancel) + w.pack(side=LEFT, padx=5, pady=5) + + self.bind("<Return>", self.ok) + self.bind("<Escape>", self.cancel) + + box.pack() + + # + # standard button semantics + + def ok(self, event=None): + + if not self.validate(): + self.initial_focus.focus_set() # put focus back + return + + self.withdraw() + self.update_idletasks() + + try: + self.apply() + finally: + self.cancel() + + def cancel(self, event=None): + + # put focus back to the parent window + if self.parent is not None: + self.parent.focus_set() + self.destroy() + + # + # command hooks + + def validate(self): + '''validate the data + + This method is called automatically to validate the data before the + dialog is destroyed. By default, it always validates OK. + ''' + + return 1 # override + + def apply(self): + '''process the data + + This method is called automatically to process the data, *after* + the dialog is destroyed. By default, it does nothing. + ''' + + pass # override + + +# Place a toplevel window at the center of parent or screen +# It is a Python implementation of ::tk::PlaceWindow. +def _place_window(w, parent=None): + w.wm_withdraw() # Remain invisible while we figure out the geometry + w.update_idletasks() # Actualize geometry information + + minwidth = w.winfo_reqwidth() + minheight = w.winfo_reqheight() + maxwidth = w.winfo_vrootwidth() + maxheight = w.winfo_vrootheight() + if parent is not None and parent.winfo_ismapped(): + x = parent.winfo_rootx() + (parent.winfo_width() - minwidth) // 2 + y = parent.winfo_rooty() + (parent.winfo_height() - minheight) // 2 + vrootx = w.winfo_vrootx() + vrooty = w.winfo_vrooty() + x = min(x, vrootx + maxwidth - minwidth) + x = max(x, vrootx) + y = min(y, vrooty + maxheight - minheight) + y = max(y, vrooty) + if w._windowingsystem == 'aqua': + # Avoid the native menu bar which sits on top of everything. + y = max(y, 22) + else: + x = (w.winfo_screenwidth() - minwidth) // 2 + y = (w.winfo_screenheight() - minheight) // 2 + + w.wm_maxsize(maxwidth, maxheight) + w.wm_geometry('+%d+%d' % (x, y)) + w.wm_deiconify() # Become visible at the desired location + + +def _setup_dialog(w): + if w._windowingsystem == "aqua": + w.tk.call("::tk::unsupported::MacWindowStyle", "style", + w, "moveableModal", "") + elif w._windowingsystem == "x11": + w.wm_attributes(type="dialog") + +# -------------------------------------------------------------------- +# convenience dialogues + +class _QueryDialog(Dialog): + + def __init__(self, title, prompt, + initialvalue=None, + minvalue = None, maxvalue = None, + parent = None): + + self.prompt = prompt + self.minvalue = minvalue + self.maxvalue = maxvalue + + self.initialvalue = initialvalue + + Dialog.__init__(self, parent, title) + + def destroy(self): + self.entry = None + Dialog.destroy(self) + + def body(self, master): + + w = Label(master, text=self.prompt, justify=LEFT) + w.grid(row=0, padx=5, sticky=W) + + self.entry = Entry(master, name="entry") + self.entry.grid(row=1, padx=5, sticky=W+E) + + if self.initialvalue is not None: + self.entry.insert(0, self.initialvalue) + self.entry.select_range(0, END) + + return self.entry + + def validate(self): + try: + result = self.getresult() + except ValueError: + messagebox.showwarning( + "Illegal value", + self.errormessage + "\nPlease try again", + parent = self + ) + return 0 + + if self.minvalue is not None and result < self.minvalue: + messagebox.showwarning( + "Too small", + "The allowed minimum value is %s. " + "Please try again." % self.minvalue, + parent = self + ) + return 0 + + if self.maxvalue is not None and result > self.maxvalue: + messagebox.showwarning( + "Too large", + "The allowed maximum value is %s. " + "Please try again." % self.maxvalue, + parent = self + ) + return 0 + + self.result = result + + return 1 + + +class _QueryInteger(_QueryDialog): + errormessage = "Not an integer." + + def getresult(self): + return self.getint(self.entry.get()) + + +def askinteger(title, prompt, **kw): + '''get an integer from the user + + Arguments: + + title -- the dialog title + prompt -- the label text + **kw -- see SimpleDialog class + + Return value is an integer + ''' + d = _QueryInteger(title, prompt, **kw) + return d.result + + +class _QueryFloat(_QueryDialog): + errormessage = "Not a floating-point value." + + def getresult(self): + return self.getdouble(self.entry.get()) + + +def askfloat(title, prompt, **kw): + '''get a float from the user + + Arguments: + + title -- the dialog title + prompt -- the label text + **kw -- see SimpleDialog class + + Return value is a float + ''' + d = _QueryFloat(title, prompt, **kw) + return d.result + + +class _QueryString(_QueryDialog): + def __init__(self, *args, **kw): + if "show" in kw: + self.__show = kw["show"] + del kw["show"] + else: + self.__show = None + _QueryDialog.__init__(self, *args, **kw) + + def body(self, master): + entry = _QueryDialog.body(self, master) + if self.__show is not None: + entry.configure(show=self.__show) + return entry + + def getresult(self): + return self.entry.get() + + +def askstring(title, prompt, **kw): + '''get a string from the user + + Arguments: + + title -- the dialog title + prompt -- the label text + **kw -- see SimpleDialog class + + Return value is a string + ''' + d = _QueryString(title, prompt, **kw) + return d.result + + +if __name__ == '__main__': + + def test(): + root = Tk() + def doit(root=root): + d = SimpleDialog(root, + text="This is a test dialog. " + "Would this have been an actual dialog, " + "the buttons below would have been glowing " + "in soft pink light.\n" + "Do you believe this?", + buttons=["Yes", "No", "Cancel"], + default=0, + cancel=2, + title="Test Dialog") + print(d.go()) + print(askinteger("Spam", "Egg count", initialvalue=12*12)) + print(askfloat("Spam", "Egg weight\n(in tons)", minvalue=1, + maxvalue=100)) + print(askstring("Spam", "Egg label")) + t = Button(root, text='Test', command=doit) + t.pack() + q = Button(root, text='Quit', command=t.quit) + q.pack() + t.mainloop() + + test() diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py new file mode 100644 index 00000000000..073b3ae2079 --- /dev/null +++ b/Lib/tkinter/ttk.py @@ -0,0 +1,1647 @@ +"""Ttk wrapper. + +This module provides classes to allow using Tk themed widget set. + +Ttk is based on a revised and enhanced version of +TIP #48 (http://tip.tcl.tk/48) specified style engine. + +Its basic idea is to separate, to the extent possible, the code +implementing a widget's behavior from the code implementing its +appearance. Widget class bindings are primarily responsible for +maintaining the widget state and invoking callbacks, all aspects +of the widgets appearance lies at Themes. +""" + +__version__ = "0.3.1" + +__author__ = "Guilherme Polo <ggpolo@gmail.com>" + +__all__ = ["Button", "Checkbutton", "Combobox", "Entry", "Frame", "Label", + "Labelframe", "LabelFrame", "Menubutton", "Notebook", "Panedwindow", + "PanedWindow", "Progressbar", "Radiobutton", "Scale", "Scrollbar", + "Separator", "Sizegrip", "Spinbox", "Style", "Treeview", + # Extensions + "LabeledScale", "OptionMenu", + # functions + "tclobjs_to_py", "setup_master"] + +import tkinter +from tkinter import _flatten, _join, _stringify, _splitdict + + +def _format_optvalue(value, script=False): + """Internal function.""" + if script: + # if caller passes a Tcl script to tk.call, all the values need to + # be grouped into words (arguments to a command in Tcl dialect) + value = _stringify(value) + elif isinstance(value, (list, tuple)): + value = _join(value) + return value + +def _format_optdict(optdict, script=False, ignore=None): + """Formats optdict to a tuple to pass it to tk.call. + + E.g. (script=False): + {'foreground': 'blue', 'padding': [1, 2, 3, 4]} returns: + ('-foreground', 'blue', '-padding', '1 2 3 4')""" + + opts = [] + for opt, value in optdict.items(): + if not ignore or opt not in ignore: + opts.append("-%s" % opt) + if value is not None: + opts.append(_format_optvalue(value, script)) + + return _flatten(opts) + +def _mapdict_values(items): + # each value in mapdict is expected to be a sequence, where each item + # is another sequence containing a state (or several) and a value + # E.g. (script=False): + # [('active', 'selected', 'grey'), ('focus', [1, 2, 3, 4])] + # returns: + # ['active selected', 'grey', 'focus', [1, 2, 3, 4]] + opt_val = [] + for *state, val in items: + if len(state) == 1: + # if it is empty (something that evaluates to False), then + # format it to Tcl code to denote the "normal" state + state = state[0] or '' + else: + # group multiple states + state = ' '.join(state) # raise TypeError if not str + opt_val.append(state) + if val is not None: + opt_val.append(val) + return opt_val + +def _format_mapdict(mapdict, script=False): + """Formats mapdict to pass it to tk.call. + + E.g. (script=False): + {'expand': [('active', 'selected', 'grey'), ('focus', [1, 2, 3, 4])]} + + returns: + + ('-expand', '{active selected} grey focus {1, 2, 3, 4}')""" + + opts = [] + for opt, value in mapdict.items(): + opts.extend(("-%s" % opt, + _format_optvalue(_mapdict_values(value), script))) + + return _flatten(opts) + +def _format_elemcreate(etype, script=False, *args, **kw): + """Formats args and kw according to the given element factory etype.""" + specs = () + opts = () + if etype == "image": # define an element based on an image + # first arg should be the default image name + iname = args[0] + # next args, if any, are statespec/value pairs which is almost + # a mapdict, but we just need the value + imagespec = (iname, *_mapdict_values(args[1:])) + if script: + specs = (imagespec,) + else: + specs = (_join(imagespec),) + opts = _format_optdict(kw, script) + + if etype == "vsapi": + # define an element whose visual appearance is drawn using the + # Microsoft Visual Styles API which is responsible for the + # themed styles on Windows XP and Vista. + # Availability: Tk 8.6, Windows XP and Vista. + if len(args) < 3: + class_name, part_id = args + statemap = (((), 1),) + else: + class_name, part_id, statemap = args + specs = (class_name, part_id, tuple(_mapdict_values(statemap))) + opts = _format_optdict(kw, script) + + elif etype == "from": # clone an element + # it expects a themename and optionally an element to clone from, + # otherwise it will clone {} (empty element) + specs = (args[0],) # theme name + if len(args) > 1: # elementfrom specified + opts = (_format_optvalue(args[1], script),) + + if script: + specs = _join(specs) + opts = ' '.join(opts) + return specs, opts + else: + return *specs, opts + + +def _format_layoutlist(layout, indent=0, indent_size=2): + """Formats a layout list so we can pass the result to ttk::style + layout and ttk::style settings. Note that the layout doesn't have to + be a list necessarily. + + E.g.: + [("Menubutton.background", None), + ("Menubutton.button", {"children": + [("Menubutton.focus", {"children": + [("Menubutton.padding", {"children": + [("Menubutton.label", {"side": "left", "expand": 1})] + })] + })] + }), + ("Menubutton.indicator", {"side": "right"}) + ] + + returns: + + Menubutton.background + Menubutton.button -children { + Menubutton.focus -children { + Menubutton.padding -children { + Menubutton.label -side left -expand 1 + } + } + } + Menubutton.indicator -side right""" + script = [] + + for layout_elem in layout: + elem, opts = layout_elem + opts = opts or {} + fopts = ' '.join(_format_optdict(opts, True, ("children",))) + head = "%s%s%s" % (' ' * indent, elem, (" %s" % fopts) if fopts else '') + + if "children" in opts: + script.append(head + " -children {") + indent += indent_size + newscript, indent = _format_layoutlist(opts['children'], indent, + indent_size) + script.append(newscript) + indent -= indent_size + script.append('%s}' % (' ' * indent)) + else: + script.append(head) + + return '\n'.join(script), indent + +def _script_from_settings(settings): + """Returns an appropriate script, based on settings, according to + theme_settings definition to be used by theme_settings and + theme_create.""" + script = [] + # a script will be generated according to settings passed, which + # will then be evaluated by Tcl + for name, opts in settings.items(): + # will format specific keys according to Tcl code + if opts.get('configure'): # format 'configure' + s = ' '.join(_format_optdict(opts['configure'], True)) + script.append("ttk::style configure %s %s;" % (name, s)) + + if opts.get('map'): # format 'map' + s = ' '.join(_format_mapdict(opts['map'], True)) + script.append("ttk::style map %s %s;" % (name, s)) + + if 'layout' in opts: # format 'layout' which may be empty + if not opts['layout']: + s = 'null' # could be any other word, but this one makes sense + else: + s, _ = _format_layoutlist(opts['layout']) + script.append("ttk::style layout %s {\n%s\n}" % (name, s)) + + if opts.get('element create'): # format 'element create' + eopts = opts['element create'] + etype = eopts[0] + + # find where args end, and where kwargs start + argc = 1 # etype was the first one + while argc < len(eopts) and not hasattr(eopts[argc], 'items'): + argc += 1 + + elemargs = eopts[1:argc] + elemkw = eopts[argc] if argc < len(eopts) and eopts[argc] else {} + specs, eopts = _format_elemcreate(etype, True, *elemargs, **elemkw) + + script.append("ttk::style element create %s %s %s %s" % ( + name, etype, specs, eopts)) + + return '\n'.join(script) + +def _list_from_statespec(stuple): + """Construct a list from the given statespec tuple according to the + accepted statespec accepted by _format_mapdict.""" + if isinstance(stuple, str): + return stuple + result = [] + it = iter(stuple) + for state, val in zip(it, it): + if hasattr(state, 'typename'): # this is a Tcl object + state = str(state).split() + elif isinstance(state, str): + state = state.split() + elif not isinstance(state, (tuple, list)): + state = (state,) + if hasattr(val, 'typename'): + val = str(val) + result.append((*state, val)) + + return result + +def _list_from_layouttuple(tk, ltuple): + """Construct a list from the tuple returned by ttk::layout, this is + somewhat the reverse of _format_layoutlist.""" + ltuple = tk.splitlist(ltuple) + res = [] + + indx = 0 + while indx < len(ltuple): + name = ltuple[indx] + opts = {} + res.append((name, opts)) + indx += 1 + + while indx < len(ltuple): # grab name's options + opt, val = ltuple[indx:indx + 2] + if not opt.startswith('-'): # found next name + break + + opt = opt[1:] # remove the '-' from the option + indx += 2 + + if opt == 'children': + val = _list_from_layouttuple(tk, val) + + opts[opt] = val + + return res + +def _val_or_dict(tk, options, *args): + """Format options then call Tk command with args and options and return + the appropriate result. + + If no option is specified, a dict is returned. If an option is + specified with the None value, the value for that option is returned. + Otherwise, the function just sets the passed options and the caller + shouldn't be expecting a return value anyway.""" + options = _format_optdict(options) + res = tk.call(*(args + options)) + + if len(options) % 2: # option specified without a value, return its value + return res + + return _splitdict(tk, res, conv=_tclobj_to_py) + +def _convert_stringval(value): + """Converts a value to, hopefully, a more appropriate Python object.""" + value = str(value) + try: + value = int(value) + except (ValueError, TypeError): + pass + + return value + +def _to_number(x): + if isinstance(x, str): + if '.' in x: + x = float(x) + else: + x = int(x) + return x + +def _tclobj_to_py(val): + """Return value converted from Tcl object to Python object.""" + if val and hasattr(val, '__len__') and not isinstance(val, str): + if getattr(val[0], 'typename', None) == 'StateSpec': + val = _list_from_statespec(val) + else: + val = list(map(_convert_stringval, val)) + + elif hasattr(val, 'typename'): # some other (single) Tcl object + val = _convert_stringval(val) + + return val + +def tclobjs_to_py(adict): + """Returns adict with its values converted from Tcl objects to Python + objects.""" + for opt, val in adict.items(): + adict[opt] = _tclobj_to_py(val) + + return adict + +def setup_master(master=None): + """If master is not None, itself is returned. If master is None, + the default master is returned if there is one, otherwise a new + master is created and returned. + + If it is not allowed to use the default root and master is None, + RuntimeError is raised.""" + if master is None: + master = tkinter._get_default_root() + return master + + +class Style(object): + """Manipulate style database.""" + + _name = "ttk::style" + + def __init__(self, master=None): + master = setup_master(master) + self.master = master + self.tk = self.master.tk + + + def configure(self, style, query_opt=None, **kw): + """Query or sets the default value of the specified option(s) in + style. + + Each key in kw is an option and each value is either a string or + a sequence identifying the value for that option.""" + if query_opt is not None: + kw[query_opt] = None + result = _val_or_dict(self.tk, kw, self._name, "configure", style) + if result or query_opt: + return result + + + def map(self, style, query_opt=None, **kw): + """Query or sets dynamic values of the specified option(s) in + style. + + Each key in kw is an option and each value should be a list or a + tuple (usually) containing statespecs grouped in tuples, or list, + or something else of your preference. A statespec is compound of + one or more states and then a value.""" + if query_opt is not None: + result = self.tk.call(self._name, "map", style, '-%s' % query_opt) + return _list_from_statespec(self.tk.splitlist(result)) + + result = self.tk.call(self._name, "map", style, *_format_mapdict(kw)) + return {k: _list_from_statespec(self.tk.splitlist(v)) + for k, v in _splitdict(self.tk, result).items()} + + + def lookup(self, style, option, state=None, default=None): + """Returns the value specified for option in style. + + If state is specified it is expected to be a sequence of one + or more states. If the default argument is set, it is used as + a fallback value in case no specification for option is found.""" + state = ' '.join(state) if state else '' + + return self.tk.call(self._name, "lookup", style, '-%s' % option, + state, default) + + + def layout(self, style, layoutspec=None): + """Define the widget layout for given style. If layoutspec is + omitted, return the layout specification for given style. + + layoutspec is expected to be a list or an object different than + None that evaluates to False if you want to "turn off" that style. + If it is a list (or tuple, or something else), each item should be + a tuple where the first item is the layout name and the second item + should have the format described below: + + LAYOUTS + + A layout can contain the value None, if takes no options, or + a dict of options specifying how to arrange the element. + The layout mechanism uses a simplified version of the pack + geometry manager: given an initial cavity, each element is + allocated a parcel. Valid options/values are: + + side: whichside + Specifies which side of the cavity to place the + element; one of top, right, bottom or left. If + omitted, the element occupies the entire cavity. + + sticky: nswe + Specifies where the element is placed inside its + allocated parcel. + + children: [sublayout... ] + Specifies a list of elements to place inside the + element. Each element is a tuple (or other sequence) + where the first item is the layout name, and the other + is a LAYOUT.""" + lspec = None + if layoutspec: + lspec = _format_layoutlist(layoutspec)[0] + elif layoutspec is not None: # will disable the layout ({}, '', etc) + lspec = "null" # could be any other word, but this may make sense + # when calling layout(style) later + + return _list_from_layouttuple(self.tk, + self.tk.call(self._name, "layout", style, lspec)) + + + def element_create(self, elementname, etype, *args, **kw): + """Create a new element in the current theme of given etype.""" + *specs, opts = _format_elemcreate(etype, False, *args, **kw) + self.tk.call(self._name, "element", "create", elementname, etype, + *specs, *opts) + + + def element_names(self): + """Returns the list of elements defined in the current theme.""" + return tuple(n.lstrip('-') for n in self.tk.splitlist( + self.tk.call(self._name, "element", "names"))) + + + def element_options(self, elementname): + """Return the list of elementname's options.""" + return tuple(o.lstrip('-') for o in self.tk.splitlist( + self.tk.call(self._name, "element", "options", elementname))) + + + def theme_create(self, themename, parent=None, settings=None): + """Creates a new theme. + + It is an error if themename already exists. If parent is + specified, the new theme will inherit styles, elements and + layouts from the specified parent theme. If settings are present, + they are expected to have the same syntax used for theme_settings.""" + script = _script_from_settings(settings) if settings else '' + + if parent: + self.tk.call(self._name, "theme", "create", themename, + "-parent", parent, "-settings", script) + else: + self.tk.call(self._name, "theme", "create", themename, + "-settings", script) + + + def theme_settings(self, themename, settings): + """Temporarily sets the current theme to themename, apply specified + settings and then restore the previous theme. + + Each key in settings is a style and each value may contain the + keys 'configure', 'map', 'layout' and 'element create' and they + are expected to have the same format as specified by the methods + configure, map, layout and element_create respectively.""" + script = _script_from_settings(settings) + self.tk.call(self._name, "theme", "settings", themename, script) + + + def theme_names(self): + """Returns a list of all known themes.""" + return self.tk.splitlist(self.tk.call(self._name, "theme", "names")) + + + def theme_use(self, themename=None): + """If themename is None, returns the theme in use, otherwise, set + the current theme to themename, refreshes all widgets and emits + a <<ThemeChanged>> event.""" + if themename is None: + # Starting on Tk 8.6, checking this global is no longer needed + # since it allows doing self.tk.call(self._name, "theme", "use") + return self.tk.eval("return $ttk::currentTheme") + + # using "ttk::setTheme" instead of "ttk::style theme use" causes + # the variable currentTheme to be updated, also, ttk::setTheme calls + # "ttk::style theme use" in order to change theme. + self.tk.call("ttk::setTheme", themename) + + +class Widget(tkinter.Widget): + """Base class for Tk themed widgets.""" + + def __init__(self, master, widgetname, kw=None): + """Constructs a Ttk Widget with the parent master. + + STANDARD OPTIONS + + class, cursor, takefocus, style + + SCROLLABLE WIDGET OPTIONS + + xscrollcommand, yscrollcommand + + LABEL WIDGET OPTIONS + + text, textvariable, underline, image, compound, width + + WIDGET STATES + + active, disabled, focus, pressed, selected, background, + readonly, alternate, invalid + """ + master = setup_master(master) + tkinter.Widget.__init__(self, master, widgetname, kw=kw) + + + def identify(self, x, y): + """Returns the name of the element at position x, y, or the empty + string if the point does not lie within any element. + + x and y are pixel coordinates relative to the widget.""" + return self.tk.call(self._w, "identify", x, y) + + + def instate(self, statespec, callback=None, *args, **kw): + """Test the widget's state. + + If callback is not specified, returns True if the widget state + matches statespec and False otherwise. If callback is specified, + then it will be invoked with *args, **kw if the widget state + matches statespec. statespec is expected to be a sequence.""" + ret = self.tk.getboolean( + self.tk.call(self._w, "instate", ' '.join(statespec))) + if ret and callback is not None: + return callback(*args, **kw) + + return ret + + + def state(self, statespec=None): + """Modify or inquire widget state. + + Widget state is returned if statespec is None, otherwise it is + set according to the statespec flags and then a new state spec + is returned indicating which flags were changed. statespec is + expected to be a sequence.""" + if statespec is not None: + statespec = ' '.join(statespec) + + return self.tk.splitlist(str(self.tk.call(self._w, "state", statespec))) + + +class Button(Widget): + """Ttk Button widget, displays a textual label and/or image, and + evaluates a command when pressed.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Button widget with the parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, default, width + """ + Widget.__init__(self, master, "ttk::button", kw) + + + def invoke(self): + """Invokes the command associated with the button.""" + return self.tk.call(self._w, "invoke") + + +class Checkbutton(Widget): + """Ttk Checkbutton widget which is either in on- or off-state.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Checkbutton widget with the parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, offvalue, onvalue, variable + """ + Widget.__init__(self, master, "ttk::checkbutton", kw) + + + def invoke(self): + """Toggles between the selected and deselected states and + invokes the associated command. If the widget is currently + selected, sets the option variable to the offvalue option + and deselects the widget; otherwise, sets the option variable + to the option onvalue. + + Returns the result of the associated command.""" + return self.tk.call(self._w, "invoke") + + +class Entry(Widget, tkinter.Entry): + """Ttk Entry widget displays a one-line text string and allows that + string to be edited by the user.""" + + def __init__(self, master=None, widget=None, **kw): + """Constructs a Ttk Entry widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, xscrollcommand + + WIDGET-SPECIFIC OPTIONS + + exportselection, invalidcommand, justify, show, state, + textvariable, validate, validatecommand, width + + VALIDATION MODES + + none, key, focus, focusin, focusout, all + """ + Widget.__init__(self, master, widget or "ttk::entry", kw) + + + def bbox(self, index): + """Return a tuple of (x, y, width, height) which describes the + bounding box of the character given by index.""" + return self._getints(self.tk.call(self._w, "bbox", index)) + + + def identify(self, x, y): + """Returns the name of the element at position x, y, or the + empty string if the coordinates are outside the window.""" + return self.tk.call(self._w, "identify", x, y) + + + def validate(self): + """Force revalidation, independent of the conditions specified + by the validate option. Returns False if validation fails, True + if it succeeds. Sets or clears the invalid state accordingly.""" + return self.tk.getboolean(self.tk.call(self._w, "validate")) + + +class Combobox(Entry): + """Ttk Combobox widget combines a text field with a pop-down list of + values.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Combobox widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + exportselection, justify, height, postcommand, state, + textvariable, values, width + """ + Entry.__init__(self, master, "ttk::combobox", **kw) + + + def current(self, newindex=None): + """If newindex is supplied, sets the combobox value to the + element at position newindex in the list of values. Otherwise, + returns the index of the current value in the list of values + or -1 if the current value does not appear in the list.""" + if newindex is None: + res = self.tk.call(self._w, "current") + if res == '': + return -1 + return self.tk.getint(res) + return self.tk.call(self._w, "current", newindex) + + + def set(self, value): + """Sets the value of the combobox to value.""" + self.tk.call(self._w, "set", value) + + +class Frame(Widget): + """Ttk Frame widget is a container, used to group other widgets + together.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Frame with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + borderwidth, relief, padding, width, height + """ + Widget.__init__(self, master, "ttk::frame", kw) + + +class Label(Widget): + """Ttk Label widget displays a textual label and/or image.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Label with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, style, takefocus, text, + textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + anchor, background, font, foreground, justify, padding, + relief, text, wraplength + """ + Widget.__init__(self, master, "ttk::label", kw) + + +class Labelframe(Widget): + """Ttk Labelframe widget is a container used to group other widgets + together. It has an optional label, which may be a plain text string + or another widget.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Labelframe with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + labelanchor, text, underline, padding, labelwidget, width, + height + """ + Widget.__init__(self, master, "ttk::labelframe", kw) + +LabelFrame = Labelframe # tkinter name compatibility + + +class Menubutton(Widget): + """Ttk Menubutton widget displays a textual label and/or image, and + displays a menu when pressed.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Menubutton with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + direction, menu + """ + Widget.__init__(self, master, "ttk::menubutton", kw) + + +class Notebook(Widget): + """Ttk Notebook widget manages a collection of windows and displays + a single one at a time. Each child window is associated with a tab, + which the user may select to change the currently-displayed window.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Notebook with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + height, padding, width + + TAB OPTIONS + + state, sticky, padding, text, image, compound, underline + + TAB IDENTIFIERS (tab_id) + + The tab_id argument found in several methods may take any of + the following forms: + + * An integer between zero and the number of tabs + * The name of a child window + * A positional specification of the form "@x,y", which + defines the tab + * The string "current", which identifies the + currently-selected tab + * The string "end", which returns the number of tabs (only + valid for method index) + """ + Widget.__init__(self, master, "ttk::notebook", kw) + + + def add(self, child, **kw): + """Adds a new tab to the notebook. + + If window is currently managed by the notebook but hidden, it is + restored to its previous position.""" + self.tk.call(self._w, "add", child, *(_format_optdict(kw))) + + + def forget(self, tab_id): + """Removes the tab specified by tab_id, unmaps and unmanages the + associated window.""" + self.tk.call(self._w, "forget", tab_id) + + + def hide(self, tab_id): + """Hides the tab specified by tab_id. + + The tab will not be displayed, but the associated window remains + managed by the notebook and its configuration remembered. Hidden + tabs may be restored with the add command.""" + self.tk.call(self._w, "hide", tab_id) + + + def identify(self, x, y): + """Returns the name of the tab element at position x, y, or the + empty string if none.""" + return self.tk.call(self._w, "identify", x, y) + + + def index(self, tab_id): + """Returns the numeric index of the tab specified by tab_id, or + the total number of tabs if tab_id is the string "end".""" + return self.tk.getint(self.tk.call(self._w, "index", tab_id)) + + + def insert(self, pos, child, **kw): + """Inserts a pane at the specified position. + + pos is either the string end, an integer index, or the name of + a managed child. If child is already managed by the notebook, + moves it to the specified position.""" + self.tk.call(self._w, "insert", pos, child, *(_format_optdict(kw))) + + + def select(self, tab_id=None): + """Selects the specified tab. + + The associated child window will be displayed, and the + previously-selected window (if different) is unmapped. If tab_id + is omitted, returns the widget name of the currently selected + pane.""" + return self.tk.call(self._w, "select", tab_id) + + + def tab(self, tab_id, option=None, **kw): + """Query or modify the options of the specific tab_id. + + If kw is not given, returns a dict of the tab option values. If option + is specified, returns the value of that option. Otherwise, sets the + options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "tab", tab_id) + + + def tabs(self): + """Returns a list of windows managed by the notebook.""" + return self.tk.splitlist(self.tk.call(self._w, "tabs") or ()) + + + def enable_traversal(self): + """Enable keyboard traversal for a toplevel window containing + this notebook. + + This will extend the bindings for the toplevel window containing + this notebook as follows: + + Control-Tab: selects the tab following the currently selected + one + + Shift-Control-Tab: selects the tab preceding the currently + selected one + + Alt-K: where K is the mnemonic (underlined) character of any + tab, will select that tab. + + Multiple notebooks in a single toplevel may be enabled for + traversal, including nested notebooks. However, notebook traversal + only works properly if all panes are direct children of the + notebook.""" + # The only, and good, difference I see is about mnemonics, which works + # after calling this method. Control-Tab and Shift-Control-Tab always + # works (here at least). + self.tk.call("ttk::notebook::enableTraversal", self._w) + + +class Panedwindow(Widget, tkinter.PanedWindow): + """Ttk Panedwindow widget displays a number of subwindows, stacked + either vertically or horizontally.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Panedwindow with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient, width, height + + PANE OPTIONS + + weight + """ + Widget.__init__(self, master, "ttk::panedwindow", kw) + + + forget = tkinter.PanedWindow.forget # overrides Pack.forget + + + def insert(self, pos, child, **kw): + """Inserts a pane at the specified positions. + + pos is either the string end, and integer index, or the name + of a child. If child is already managed by the paned window, + moves it to the specified position.""" + self.tk.call(self._w, "insert", pos, child, *(_format_optdict(kw))) + + + def pane(self, pane, option=None, **kw): + """Query or modify the options of the specified pane. + + pane is either an integer index or the name of a managed subwindow. + If kw is not given, returns a dict of the pane option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "pane", pane) + + + def sashpos(self, index, newpos=None): + """If newpos is specified, sets the position of sash number index. + + May adjust the positions of adjacent sashes to ensure that + positions are monotonically increasing. Sash positions are further + constrained to be between 0 and the total size of the widget. + + Returns the new position of sash number index.""" + return self.tk.getint(self.tk.call(self._w, "sashpos", index, newpos)) + +PanedWindow = Panedwindow # tkinter name compatibility + + +class Progressbar(Widget): + """Ttk Progressbar widget shows the status of a long-running + operation. They can operate in two modes: determinate mode shows the + amount completed relative to the total amount of work to be done, and + indeterminate mode provides an animated display to let the user know + that something is happening.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Progressbar with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient, length, mode, maximum, value, variable, phase + """ + Widget.__init__(self, master, "ttk::progressbar", kw) + + + def start(self, interval=None): + """Begin autoincrement mode: schedules a recurring timer event + that calls method step every interval milliseconds. + + interval defaults to 50 milliseconds (20 steps/second) if omitted.""" + self.tk.call(self._w, "start", interval) + + + def step(self, amount=None): + """Increments the value option by amount. + + amount defaults to 1.0 if omitted.""" + self.tk.call(self._w, "step", amount) + + + def stop(self): + """Stop autoincrement mode: cancels any recurring timer event + initiated by start.""" + self.tk.call(self._w, "stop") + + +class Radiobutton(Widget): + """Ttk Radiobutton widgets are used in groups to show or change a + set of mutually-exclusive options.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Radiobutton with parent master. + + STANDARD OPTIONS + + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width + + WIDGET-SPECIFIC OPTIONS + + command, value, variable + """ + Widget.__init__(self, master, "ttk::radiobutton", kw) + + + def invoke(self): + """Sets the option variable to the option value, selects the + widget, and invokes the associated command. + + Returns the result of the command, or an empty string if + no command is specified.""" + return self.tk.call(self._w, "invoke") + + +class Scale(Widget, tkinter.Scale): + """Ttk Scale widget is typically used to control the numeric value of + a linked variable that varies uniformly over some range.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Scale with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + command, from, length, orient, to, value, variable + """ + Widget.__init__(self, master, "ttk::scale", kw) + + + def configure(self, cnf=None, **kw): + """Modify or query scale options. + + Setting a value for any of the "from", "from_" or "to" options + generates a <<RangeChanged>> event.""" + retval = Widget.configure(self, cnf, **kw) + if not isinstance(cnf, (type(None), str)): + kw.update(cnf) + if any(['from' in kw, 'from_' in kw, 'to' in kw]): + self.event_generate('<<RangeChanged>>') + return retval + + + def get(self, x=None, y=None): + """Get the current value of the value option, or the value + corresponding to the coordinates x, y if they are specified. + + x and y are pixel coordinates relative to the scale widget + origin.""" + return self.tk.call(self._w, 'get', x, y) + + +class Scrollbar(Widget, tkinter.Scrollbar): + """Ttk Scrollbar controls the viewport of a scrollable widget.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Scrollbar with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + command, orient + """ + Widget.__init__(self, master, "ttk::scrollbar", kw) + + +class Separator(Widget): + """Ttk Separator widget displays a horizontal or vertical separator + bar.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Separator with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus + + WIDGET-SPECIFIC OPTIONS + + orient + """ + Widget.__init__(self, master, "ttk::separator", kw) + + +class Sizegrip(Widget): + """Ttk Sizegrip allows the user to resize the containing toplevel + window by pressing and dragging the grip.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Sizegrip with parent master. + + STANDARD OPTIONS + + class, cursor, state, style, takefocus + """ + Widget.__init__(self, master, "ttk::sizegrip", kw) + + +class Spinbox(Entry): + """Ttk Spinbox is an Entry with increment and decrement arrows + + It is commonly used for number entry or to select from a list of + string values. + """ + + def __init__(self, master=None, **kw): + """Construct a Ttk Spinbox widget with the parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, validate, + validatecommand, xscrollcommand, invalidcommand + + WIDGET-SPECIFIC OPTIONS + + to, from_, increment, values, wrap, format, command + """ + Entry.__init__(self, master, "ttk::spinbox", **kw) + + + def set(self, value): + """Sets the value of the Spinbox to value.""" + self.tk.call(self._w, "set", value) + + +class Treeview(Widget, tkinter.XView, tkinter.YView): + """Ttk Treeview widget displays a hierarchical collection of items. + + Each item has a textual label, an optional image, and an optional list + of data values. The data values are displayed in successive columns + after the tree label.""" + + def __init__(self, master=None, **kw): + """Construct a Ttk Treeview with parent master. + + STANDARD OPTIONS + + class, cursor, style, takefocus, xscrollcommand, + yscrollcommand + + WIDGET-SPECIFIC OPTIONS + + columns, displaycolumns, height, padding, selectmode, show + + ITEM OPTIONS + + text, image, values, open, tags + + TAG OPTIONS + + foreground, background, font, image + """ + Widget.__init__(self, master, "ttk::treeview", kw) + + + def bbox(self, item, column=None): + """Returns the bounding box (relative to the treeview widget's + window) of the specified item in the form x y width height. + + If column is specified, returns the bounding box of that cell. + If the item is not visible (i.e., if it is a descendant of a + closed item or is scrolled offscreen), returns an empty string.""" + return self._getints(self.tk.call(self._w, "bbox", item, column)) or '' + + + def get_children(self, item=None): + """Returns a tuple of children belonging to item. + + If item is not specified, returns root children.""" + return self.tk.splitlist( + self.tk.call(self._w, "children", item or '') or ()) + + + def set_children(self, item, *newchildren): + """Replaces item's child with newchildren. + + Children present in item that are not present in newchildren + are detached from tree. No items in newchildren may be an + ancestor of item.""" + self.tk.call(self._w, "children", item, newchildren) + + + def column(self, column, option=None, **kw): + """Query or modify the options for the specified column. + + If kw is not given, returns a dict of the column option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "column", column) + + + def delete(self, *items): + """Delete all specified items and all their descendants. The root + item may not be deleted.""" + self.tk.call(self._w, "delete", items) + + + def detach(self, *items): + """Unlinks all of the specified items from the tree. + + The items and all of their descendants are still present, and may + be reinserted at another point in the tree, but will not be + displayed. The root item may not be detached.""" + self.tk.call(self._w, "detach", items) + + + def exists(self, item): + """Returns True if the specified item is present in the tree, + False otherwise.""" + return self.tk.getboolean(self.tk.call(self._w, "exists", item)) + + + def focus(self, item=None): + """If item is specified, sets the focus item to item. Otherwise, + returns the current focus item, or '' if there is none.""" + return self.tk.call(self._w, "focus", item) + + + def heading(self, column, option=None, **kw): + """Query or modify the heading options for the specified column. + + If kw is not given, returns a dict of the heading option values. If + option is specified then the value for that option is returned. + Otherwise, sets the options to the corresponding values. + + Valid options/values are: + text: text + The text to display in the column heading + image: image_name + Specifies an image to display to the right of the column + heading + anchor: anchor + Specifies how the heading text should be aligned. One of + the standard Tk anchor values + command: callback + A callback to be invoked when the heading label is + pressed. + + To configure the tree column heading, call this with column = "#0" """ + cmd = kw.get('command') + if cmd and not isinstance(cmd, str): + # callback not registered yet, do it now + kw['command'] = self.master.register(cmd, self._substitute) + + if option is not None: + kw[option] = None + + return _val_or_dict(self.tk, kw, self._w, 'heading', column) + + + def identify(self, component, x, y): + """Returns a description of the specified component under the + point given by x and y, or the empty string if no such component + is present at that position.""" + return self.tk.call(self._w, "identify", component, x, y) + + + def identify_row(self, y): + """Returns the item ID of the item at position y.""" + return self.identify("row", 0, y) + + + def identify_column(self, x): + """Returns the data column identifier of the cell at position x. + + The tree column has ID #0.""" + return self.identify("column", x, 0) + + + def identify_region(self, x, y): + """Returns one of: + + heading: Tree heading area. + separator: Space between two columns headings; + tree: The tree area. + cell: A data cell. + + * Availability: Tk 8.6""" + return self.identify("region", x, y) + + + def identify_element(self, x, y): + """Returns the element at position x, y. + + * Availability: Tk 8.6""" + return self.identify("element", x, y) + + + def index(self, item): + """Returns the integer index of item within its parent's list + of children.""" + return self.tk.getint(self.tk.call(self._w, "index", item)) + + + def insert(self, parent, index, iid=None, **kw): + """Creates a new item and return the item identifier of the newly + created item. + + parent is the item ID of the parent item, or the empty string + to create a new top-level item. index is an integer, or the value + end, specifying where in the list of parent's children to insert + the new item. If index is less than or equal to zero, the new node + is inserted at the beginning, if index is greater than or equal to + the current number of children, it is inserted at the end. If iid + is specified, it is used as the item identifier, iid must not + already exist in the tree. Otherwise, a new unique identifier + is generated.""" + opts = _format_optdict(kw) + if iid is not None: + res = self.tk.call(self._w, "insert", parent, index, + "-id", iid, *opts) + else: + res = self.tk.call(self._w, "insert", parent, index, *opts) + + return res + + + def item(self, item, option=None, **kw): + """Query or modify the options for the specified item. + + If no options are given, a dict with options/values for the item + is returned. If option is specified then the value for that option + is returned. Otherwise, sets the options to the corresponding + values as given by kw.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "item", item) + + + def move(self, item, parent, index): + """Moves item to position index in parent's list of children. + + It is illegal to move an item under one of its descendants. If + index is less than or equal to zero, item is moved to the + beginning, if greater than or equal to the number of children, + it is moved to the end. If item was detached it is reattached.""" + self.tk.call(self._w, "move", item, parent, index) + + reattach = move # A sensible method name for reattaching detached items + + + def next(self, item): + """Returns the identifier of item's next sibling, or '' if item + is the last child of its parent.""" + return self.tk.call(self._w, "next", item) + + + def parent(self, item): + """Returns the ID of the parent of item, or '' if item is at the + top level of the hierarchy.""" + return self.tk.call(self._w, "parent", item) + + + def prev(self, item): + """Returns the identifier of item's previous sibling, or '' if + item is the first child of its parent.""" + return self.tk.call(self._w, "prev", item) + + + def see(self, item): + """Ensure that item is visible. + + Sets all of item's ancestors open option to True, and scrolls + the widget if necessary so that item is within the visible + portion of the tree.""" + self.tk.call(self._w, "see", item) + + + def selection(self): + """Returns the tuple of selected items.""" + return self.tk.splitlist(self.tk.call(self._w, "selection")) + + + def _selection(self, selop, items): + if len(items) == 1 and isinstance(items[0], (tuple, list)): + items = items[0] + + self.tk.call(self._w, "selection", selop, items) + + + def selection_set(self, *items): + """The specified items becomes the new selection.""" + self._selection("set", items) + + + def selection_add(self, *items): + """Add all of the specified items to the selection.""" + self._selection("add", items) + + + def selection_remove(self, *items): + """Remove all of the specified items from the selection.""" + self._selection("remove", items) + + + def selection_toggle(self, *items): + """Toggle the selection state of each specified item.""" + self._selection("toggle", items) + + + def set(self, item, column=None, value=None): + """Query or set the value of given item. + + With one argument, return a dictionary of column/value pairs + for the specified item. With two arguments, return the current + value of the specified column. With three arguments, set the + value of given column in given item to the specified value.""" + res = self.tk.call(self._w, "set", item, column, value) + if column is None and value is None: + return _splitdict(self.tk, res, + cut_minus=False, conv=_tclobj_to_py) + else: + return res + + + def tag_bind(self, tagname, sequence=None, callback=None): + """Bind a callback for the given event sequence to the tag tagname. + When an event is delivered to an item, the callbacks for each + of the item's tags option are called.""" + self._bind((self._w, "tag", "bind", tagname), sequence, callback, add=0) + + + def tag_configure(self, tagname, option=None, **kw): + """Query or modify the options for the specified tagname. + + If kw is not given, returns a dict of the option settings for tagname. + If option is specified, returns the value for that option for the + specified tagname. Otherwise, sets the options to the corresponding + values for the given tagname.""" + if option is not None: + kw[option] = None + return _val_or_dict(self.tk, kw, self._w, "tag", "configure", + tagname) + + + def tag_has(self, tagname, item=None): + """If item is specified, returns 1 or 0 depending on whether the + specified item has the given tagname. Otherwise, returns a list of + all items which have the specified tag. + + * Availability: Tk 8.6""" + if item is None: + return self.tk.splitlist( + self.tk.call(self._w, "tag", "has", tagname)) + else: + return self.tk.getboolean( + self.tk.call(self._w, "tag", "has", tagname, item)) + + +# Extensions + +class LabeledScale(Frame): + """A Ttk Scale widget with a Ttk Label widget indicating its + current value. + + The Ttk Scale can be accessed through instance.scale, and Ttk Label + can be accessed through instance.label""" + + def __init__(self, master=None, variable=None, from_=0, to=10, **kw): + """Construct a horizontal LabeledScale with parent master, a + variable to be associated with the Ttk Scale widget and its range. + If variable is not specified, a tkinter.IntVar is created. + + WIDGET-SPECIFIC OPTIONS + + compound: 'top' or 'bottom' + Specifies how to display the label relative to the scale. + Defaults to 'top'. + """ + self._label_top = kw.pop('compound', 'top') == 'top' + + Frame.__init__(self, master, **kw) + self._variable = variable or tkinter.IntVar(master) + self._variable.set(from_) + self._last_valid = from_ + + self.label = Label(self) + self.scale = Scale(self, variable=self._variable, from_=from_, to=to) + self.scale.bind('<<RangeChanged>>', self._adjust) + + # position scale and label according to the compound option + scale_side = 'bottom' if self._label_top else 'top' + label_side = 'top' if scale_side == 'bottom' else 'bottom' + self.scale.pack(side=scale_side, fill='x') + # Dummy required to make frame correct height + dummy = Label(self) + dummy.pack(side=label_side) + dummy.lower() + self.label.place(anchor='n' if label_side == 'top' else 's') + + # update the label as scale or variable changes + self.__tracecb = self._variable.trace_add('write', self._adjust) + self.bind('<Configure>', self._adjust) + self.bind('<Map>', self._adjust) + + + def destroy(self): + """Destroy this widget and possibly its associated variable.""" + try: + self._variable.trace_remove('write', self.__tracecb) + except AttributeError: + pass + else: + del self._variable + super().destroy() + self.label = None + self.scale = None + + + def _adjust(self, *args): + """Adjust the label position according to the scale.""" + def adjust_label(): + self.update_idletasks() # "force" scale redraw + + x, y = self.scale.coords() + if self._label_top: + y = self.scale.winfo_y() - self.label.winfo_reqheight() + else: + y = self.scale.winfo_reqheight() + self.label.winfo_reqheight() + + self.label.place_configure(x=x, y=y) + + from_ = _to_number(self.scale['from']) + to = _to_number(self.scale['to']) + if to < from_: + from_, to = to, from_ + newval = self._variable.get() + if not from_ <= newval <= to: + # value outside range, set value back to the last valid one + self.value = self._last_valid + return + + self._last_valid = newval + self.label['text'] = newval + self.after_idle(adjust_label) + + @property + def value(self): + """Return current scale value.""" + return self._variable.get() + + @value.setter + def value(self, val): + """Set new scale value.""" + self._variable.set(val) + + +class OptionMenu(Menubutton): + """Themed OptionMenu, based after tkinter's OptionMenu, which allows + the user to select a value from a menu.""" + + def __init__(self, master, variable, default=None, *values, **kwargs): + """Construct a themed OptionMenu widget with master as the parent, + the resource textvariable set to variable, the initially selected + value specified by the default parameter, the menu values given by + *values and additional keywords. + + WIDGET-SPECIFIC OPTIONS + + style: stylename + Menubutton style. + direction: 'above', 'below', 'left', 'right', or 'flush' + Menubutton direction. + command: callback + A callback that will be invoked after selecting an item. + """ + kw = {'textvariable': variable, 'style': kwargs.pop('style', None), + 'direction': kwargs.pop('direction', None)} + Menubutton.__init__(self, master, **kw) + self['menu'] = tkinter.Menu(self, tearoff=False) + + self._variable = variable + self._callback = kwargs.pop('command', None) + if kwargs: + raise tkinter.TclError('unknown option -%s' % ( + next(iter(kwargs.keys())))) + + self.set_menu(default, *values) + + + def __getitem__(self, item): + if item == 'menu': + return self.nametowidget(Menubutton.__getitem__(self, item)) + + return Menubutton.__getitem__(self, item) + + + def set_menu(self, default=None, *values): + """Build a new menu of radiobuttons with *values and optionally + a default value.""" + menu = self['menu'] + menu.delete(0, 'end') + for val in values: + menu.add_radiobutton(label=val, + command=( + None if self._callback is None + else lambda val=val: self._callback(val) + ), + variable=self._variable) + + if default: + self._variable.set(default) + + + def destroy(self): + """Destroy this widget and its associated variable.""" + try: + del self._variable + except AttributeError: + pass + super().destroy() diff --git a/Lib/token.py b/Lib/token.py index 493bf042650..f61723cc09d 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" -# Auto-generated by Tools/scripts/generate_token.py +# Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 @@ -57,17 +58,23 @@ RARROW = 51 ELLIPSIS = 52 COLONEQUAL = 53 -OP = 54 -AWAIT = 55 -ASYNC = 56 -TYPE_IGNORE = 57 -TYPE_COMMENT = 58 +EXCLAMATION = 54 +OP = 55 +TYPE_IGNORE = 56 +TYPE_COMMENT = 57 +SOFT_KEYWORD = 58 +FSTRING_START = 59 +FSTRING_MIDDLE = 60 +FSTRING_END = 61 +TSTRING_START = 62 +TSTRING_MIDDLE = 63 +TSTRING_END = 64 +COMMENT = 65 +NL = 66 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 59 -COMMENT = 60 -NL = 61 -ENCODING = 62 -N_TOKENS = 63 +ERRORTOKEN = 67 +ENCODING = 68 +N_TOKENS = 69 # Special definitions for cooperation with parser NT_OFFSET = 256 @@ -77,6 +84,7 @@ __all__.extend(tok_name.values()) EXACT_TOKEN_TYPES = { + '!': EXCLAMATION, '!=': NOTEQUAL, '%': PERCENT, '%=': PERCENTEQUAL, @@ -126,11 +134,11 @@ '~': TILDE, } -def ISTERMINAL(x): +def ISTERMINAL(x: int) -> bool: return x < NT_OFFSET -def ISNONTERMINAL(x): +def ISNONTERMINAL(x: int) -> bool: return x >= NT_OFFSET -def ISEOF(x): +def ISEOF(x: int) -> bool: return x == ENDMARKER diff --git a/Lib/tokenize.py b/Lib/tokenize.py index d72968e4250..1f31258ce36 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -24,10 +24,7 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' 'Michael Foord') -try: - from builtins import open as _builtin_open -except ImportError: - pass +from builtins import open as _builtin_open from codecs import lookup, BOM_UTF8 import collections import functools @@ -37,13 +34,14 @@ import sys from token import * from token import EXACT_TOKEN_TYPES +import _tokenize -cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +cookie_re = re.compile(br'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) import token __all__ = token.__all__ + ["tokenize", "generate_tokens", "detect_encoding", - "untokenize", "TokenInfo"] + "untokenize", "TokenInfo", "open", "TokenError"] del token class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): @@ -88,7 +86,7 @@ def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. - _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr'] # if we add binary f-strings, add: ['fb', 'fbr'] result = {''} for prefix in _valid_string_prefixes: @@ -134,7 +132,7 @@ def _compile(expr): group("'", r'\\\r?\n'), StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' + group('"', r'\\\r?\n')) -PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) +PseudoExtras = group(r'\\\r?\n|\z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) # For a given string prefix plus quotes, endpats maps it to a regex @@ -146,6 +144,7 @@ def _compile(expr): endpats[_prefix + '"'] = Double endpats[_prefix + "'''"] = Single3 endpats[_prefix + '"""'] = Double3 +del _prefix # A set of all of the single and triple quoted string prefixes, # including the opening quotes. @@ -156,13 +155,12 @@ def _compile(expr): single_quoted.add(u) for u in (t + '"""', t + "'''"): triple_quoted.add(u) +del t, u tabsize = 8 class TokenError(Exception): pass -class StopTokenizing(Exception): pass - class Untokenizer: @@ -170,6 +168,8 @@ def __init__(self): self.tokens = [] self.prev_row = 1 self.prev_col = 0 + self.prev_type = None + self.prev_line = "" self.encoding = None def add_whitespace(self, start): @@ -177,14 +177,51 @@ def add_whitespace(self, start): if row < self.prev_row or row == self.prev_row and col < self.prev_col: raise ValueError("start ({},{}) precedes previous end ({},{})" .format(row, col, self.prev_row, self.prev_col)) - row_offset = row - self.prev_row - if row_offset: - self.tokens.append("\\\n" * row_offset) - self.prev_col = 0 + self.add_backslash_continuation(start) col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) + def add_backslash_continuation(self, start): + """Add backslash continuation characters if the row has increased + without encountering a newline token. + + This also inserts the correct amount of whitespace before the backslash. + """ + row = start[0] + row_offset = row - self.prev_row + if row_offset == 0: + return + + newline = '\r\n' if self.prev_line.endswith('\r\n') else '\n' + line = self.prev_line.rstrip('\\\r\n') + ws = ''.join(_itertools.takewhile(str.isspace, reversed(line))) + self.tokens.append(ws + f"\\{newline}" * row_offset) + self.prev_col = 0 + + def escape_brackets(self, token): + characters = [] + consume_until_next_bracket = False + for character in token: + if character == "}": + if consume_until_next_bracket: + consume_until_next_bracket = False + else: + characters.append(character) + if character == "{": + n_backslashes = sum( + 1 for char in _itertools.takewhile( + "\\".__eq__, + characters[-2::-1] + ) + ) + if n_backslashes % 2 == 0 or characters[-1] != "N": + characters.append(character) + else: + consume_until_next_bracket = True + characters.append(character) + return "".join(characters) + def untokenize(self, iterable): it = iter(iterable) indents = [] @@ -214,12 +251,22 @@ def untokenize(self, iterable): self.tokens.append(indent) self.prev_col = len(indent) startline = False + elif tok_type in {FSTRING_MIDDLE, TSTRING_MIDDLE}: + if '{' in token or '}' in token: + token = self.escape_brackets(token) + last_line = token.splitlines()[-1] + end_line, end_col = end + extra_chars = last_line.count("{{") + last_line.count("}}") + end = (end_line, end_col + extra_chars) + self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end if tok_type in (NEWLINE, NL): self.prev_row += 1 self.prev_col = 0 + self.prev_type = tok_type + self.prev_line = line return "".join(self.tokens) def compat(self, token, iterable): @@ -227,6 +274,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False + in_fstring_or_tstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -245,6 +293,10 @@ def compat(self, token, iterable): else: prevstring = False + if toknum in {FSTRING_START, TSTRING_START}: + in_fstring_or_tstring += 1 + elif toknum in {FSTRING_END, TSTRING_END}: + in_fstring_or_tstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -256,7 +308,19 @@ def compat(self, token, iterable): elif startline and indents: toks_append(indents[-1]) startline = False + elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}: + tokval = self.escape_brackets(tokval) + + # Insert a space between two consecutive brackets if we are in an f-string or t-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring_or_tstring: + tokval = ' ' + tokval + + # Insert a space between two consecutive f-strings + if toknum in (STRING, FSTRING_START) and self.prev_type in (STRING, FSTRING_END): + self.tokens.append(" ") + toks_append(tokval) + self.prev_type = toknum def untokenize(iterable): @@ -268,16 +332,10 @@ def untokenize(iterable): with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited input: - # Output bytes will tokenize back to the input - t1 = [tok[:2] for tok in tokenize(f.readline)] - newcode = untokenize(t1) - readline = BytesIO(newcode).readline - t2 = [tok[:2] for tok in tokenize(readline)] - assert t1 == t2 + The result is guaranteed to tokenize back to match the input so + that the conversion is lossless and round-trips are assured. + The guarantee applies only to the token type and token string as + the spacing between tokens (column positions) may change. """ ut = Untokenizer() out = ut.untokenize(iterable) @@ -287,7 +345,7 @@ def untokenize(iterable): def _get_normal_name(orig_enc): - """Imitates get_normal_name in tokenizer.c.""" + """Imitates get_normal_name in Parser/tokenizer/helpers.c.""" # Only care about the first 12 characters. enc = orig_enc[:12].lower().replace("_", "-") if enc == "utf-8" or enc.startswith("utf-8-"): @@ -327,22 +385,23 @@ def read_or_stop(): except StopIteration: return b'' - def find_cookie(line): + def check(line, encoding): + # Check if the line matches the encoding. + if 0 in line: + raise SyntaxError("source code cannot contain null bytes") try: - # Decode as UTF-8. Either the line is an encoding declaration, - # in which case it should be pure ASCII, or it must be UTF-8 - # per default encoding. - line_string = line.decode('utf-8') + line.decode(encoding) except UnicodeDecodeError: msg = "invalid or missing encoding declaration" if filename is not None: msg = '{} for {!r}'.format(msg, filename) raise SyntaxError(msg) - match = cookie_re.match(line_string) + def find_cookie(line): + match = cookie_re.match(line) if not match: return None - encoding = _get_normal_name(match.group(1)) + encoding = _get_normal_name(match.group(1).decode()) try: codec = lookup(encoding) except LookupError: @@ -375,18 +434,23 @@ def find_cookie(line): encoding = find_cookie(first) if encoding: + check(first, encoding) return encoding, [first] if not blank_re.match(first): + check(first, default) return default, [first] second = read_or_stop() if not second: + check(first, default) return default, [first] encoding = find_cookie(second) if encoding: + check(first + second, encoding) return encoding, [first, second] + check(first + second, default) return default, [first, second] @@ -405,7 +469,6 @@ def open(filename): buffer.close() raise - def tokenize(readline): """ The tokenize() generator requires one argument, readline, which @@ -426,193 +489,13 @@ def tokenize(readline): which tells you which encoding was used to decode the bytes stream. """ encoding, consumed = detect_encoding(readline) - empty = _itertools.repeat(b"") - rl_gen = _itertools.chain(consumed, iter(readline, b""), empty) - return _tokenize(rl_gen.__next__, encoding) - - -def _tokenize(readline, encoding): - lnum = parenlev = continued = 0 - numchars = '0123456789' - contstr, needcont = '', 0 - contline = None - indents = [0] - + rl_gen = _itertools.chain(consumed, iter(readline, b"")) if encoding is not None: if encoding == "utf-8-sig": # BOM will already have been stripped. encoding = "utf-8" yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') - last_line = b'' - line = b'' - while True: # loop over lines in stream - try: - # We capture the value of the line variable here because - # readline uses the empty string '' to signal end of input, - # hence `line` itself will always be overwritten at the end - # of this loop. - last_line = line - line = readline() - except StopIteration: - line = b'' - - if encoding is not None: - line = line.decode(encoding) - lnum += 1 - pos, max = 0, len(line) - - if contstr: # continued string - if not line: - raise TokenError("EOF in multi-line string", strstart) - endmatch = endprog.match(line) - if endmatch: - pos = end = endmatch.end(0) - yield TokenInfo(STRING, contstr + line[:end], - strstart, (lnum, end), contline + line) - contstr, needcont = '', 0 - contline = None - elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': - yield TokenInfo(ERRORTOKEN, contstr + line, - strstart, (lnum, len(line)), contline) - contstr = '' - contline = None - continue - else: - contstr = contstr + line - contline = contline + line - continue - - elif parenlev == 0 and not continued: # new statement - if not line: break - column = 0 - while pos < max: # measure leading whitespace - if line[pos] == ' ': - column += 1 - elif line[pos] == '\t': - column = (column//tabsize + 1)*tabsize - elif line[pos] == '\f': - column = 0 - else: - break - pos += 1 - if pos == max: - break - - if line[pos] in '#\r\n': # skip comments or blank lines - if line[pos] == '#': - comment_token = line[pos:].rstrip('\r\n') - yield TokenInfo(COMMENT, comment_token, - (lnum, pos), (lnum, pos + len(comment_token)), line) - pos += len(comment_token) - - yield TokenInfo(NL, line[pos:], - (lnum, pos), (lnum, len(line)), line) - continue - - if column > indents[-1]: # count indents or dedents - indents.append(column) - yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) - while column < indents[-1]: - if column not in indents: - raise IndentationError( - "unindent does not match any outer indentation level", - ("<tokenize>", lnum, pos, line)) - indents = indents[:-1] - - yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) - - else: # continued statement - if not line: - raise TokenError("EOF in multi-line statement", (lnum, 0)) - continued = 0 - - while pos < max: - pseudomatch = _compile(PseudoToken).match(line, pos) - if pseudomatch: # scan for tokens - start, end = pseudomatch.span(1) - spos, epos, pos = (lnum, start), (lnum, end), end - if start == end: - continue - token, initial = line[start:end], line[start] - - if (initial in numchars or # ordinary number - (initial == '.' and token != '.' and token != '...')): - yield TokenInfo(NUMBER, token, spos, epos, line) - elif initial in '\r\n': - if parenlev > 0: - yield TokenInfo(NL, token, spos, epos, line) - else: - yield TokenInfo(NEWLINE, token, spos, epos, line) - - elif initial == '#': - assert not token.endswith("\n") - yield TokenInfo(COMMENT, token, spos, epos, line) - - elif token in triple_quoted: - endprog = _compile(endpats[token]) - endmatch = endprog.match(line, pos) - if endmatch: # all on one line - pos = endmatch.end(0) - token = line[start:pos] - yield TokenInfo(STRING, token, spos, (lnum, pos), line) - else: - strstart = (lnum, start) # multiple lines - contstr = line[start:] - contline = line - break - - # Check up to the first 3 chars of the token to see if - # they're in the single_quoted set. If so, they start - # a string. - # We're using the first 3, because we're looking for - # "rb'" (for example) at the start of the token. If - # we switch to longer prefixes, this needs to be - # adjusted. - # Note that initial == token[:1]. - # Also note that single quote checking must come after - # triple quote checking (above). - elif (initial in single_quoted or - token[:2] in single_quoted or - token[:3] in single_quoted): - if token[-1] == '\n': # continued string - strstart = (lnum, start) - # Again, using the first 3 chars of the - # token. This is looking for the matching end - # regex for the correct type of quote - # character. So it's really looking for - # endpats["'"] or endpats['"'], by trying to - # skip string prefix characters, if any. - endprog = _compile(endpats.get(initial) or - endpats.get(token[1]) or - endpats.get(token[2])) - contstr, needcont = line[start:], 1 - contline = line - break - else: # ordinary string - yield TokenInfo(STRING, token, spos, epos, line) - - elif initial.isidentifier(): # ordinary name - yield TokenInfo(NAME, token, spos, epos, line) - elif initial == '\\': # continued stmt - continued = 1 - else: - if initial in '([{': - parenlev += 1 - elif initial in ')]}': - parenlev -= 1 - yield TokenInfo(OP, token, spos, epos, line) - else: - yield TokenInfo(ERRORTOKEN, line[pos], - (lnum, pos), (lnum, pos+1), line) - pos += 1 - - # Add an implicit NEWLINE if the input doesn't end in one - if last_line and last_line[-1] not in '\r\n' and not last_line.strip().startswith("#"): - yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '') - for indent in indents[1:]: # pop remaining indent levels - yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') - yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') - + yield from _generate_tokens_from_c_tokenizer(rl_gen.__next__, encoding, extra_tokens=True) def generate_tokens(readline): """Tokenize a source reading Python code as unicode strings. @@ -620,9 +503,9 @@ def generate_tokens(readline): This has the same API as tokenize(), except that it expects the *readline* callable to return str objects instead of bytes. """ - return _tokenize(readline, None) + return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True) -def main(): +def _main(args=None): import argparse # Helper error handling routines @@ -641,13 +524,13 @@ def error(message, filename=None, location=None): sys.exit(1) # Parse the arguments and options - parser = argparse.ArgumentParser(prog='python -m tokenize') + parser = argparse.ArgumentParser(color=True) parser.add_argument(dest='filename', nargs='?', metavar='filename.py', help='the file to tokenize; defaults to stdin') parser.add_argument('-e', '--exact', dest='exact', action='store_true', help='display token names using the exact type') - args = parser.parse_args() + args = parser.parse_args(args) try: # Tokenize the input @@ -657,7 +540,9 @@ def error(message, filename=None, location=None): tokens = list(tokenize(f.readline)) else: filename = "<stdin>" - tokens = _tokenize(sys.stdin.readline, None) + tokens = _generate_tokens_from_c_tokenizer( + sys.stdin.readline, extra_tokens=True) + # Output the tokenization for token in tokens: @@ -683,5 +568,31 @@ def error(message, filename=None, location=None): perror("unexpected error: %s" % err) raise +def _transform_msg(msg): + """Transform error messages from the C tokenizer into the Python tokenize + + The C tokenizer is more picky than the Python one, so we need to massage + the error messages a bit for backwards compatibility. + """ + if "unterminated triple-quoted string literal" in msg: + return "EOF in multi-line string" + return msg + +def _generate_tokens_from_c_tokenizer(source, encoding=None, extra_tokens=False): + """Tokenize a source reading Python code as unicode strings using the internal C tokenizer""" + if encoding is None: + it = _tokenize.TokenizerIter(source, extra_tokens=extra_tokens) + else: + it = _tokenize.TokenizerIter(source, encoding=encoding, extra_tokens=extra_tokens) + try: + for info in it: + yield TokenInfo._make(info) + except SyntaxError as e: + if type(e) != SyntaxError: + raise e from None + msg = _transform_msg(e.msg) + raise TokenError(msg, (e.lineno, e.offset)) from None + + if __name__ == "__main__": - main() + _main() diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 45ca7a89630..3ee47aa9e0a 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -4,10 +4,7 @@ from __future__ import annotations -from collections.abc import Iterable -import string from types import MappingProxyType -from typing import Any, BinaryIO, NamedTuple from ._re import ( RE_DATETIME, @@ -17,7 +14,13 @@ match_to_localtime, match_to_number, ) -from ._types import Key, ParseFloat, Pos + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import IO, Any + + from ._types import Key, ParseFloat, Pos ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) @@ -33,9 +36,11 @@ TOML_WS = frozenset(" \t") TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") -BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") +BARE_KEY_CHARS = frozenset( + "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "-_" +) KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") -HEXDIGIT_CHARS = frozenset(string.hexdigits) +HEXDIGIT_CHARS = frozenset("abcdef" "ABCDEF" "0123456789") BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( { @@ -50,11 +55,73 @@ ) +class DEPRECATED_DEFAULT: + """Sentinel to be used as default arg during deprecation + period of TOMLDecodeError's free-form arguments.""" + + class TOMLDecodeError(ValueError): - """An error raised if a document is not valid TOML.""" + """An error raised if a document is not valid TOML. + + Adds the following attributes to ValueError: + msg: The unformatted error message + doc: The TOML document being parsed + pos: The index of doc where parsing failed + lineno: The line corresponding to pos + colno: The column corresponding to pos + """ + + def __init__( + self, + msg: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + doc: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + pos: Pos = DEPRECATED_DEFAULT, # type: ignore[assignment] + *args: Any, + ): + if ( + args + or not isinstance(msg, str) + or not isinstance(doc, str) + or not isinstance(pos, int) + ): + import warnings + + warnings.warn( + "Free-form arguments for TOMLDecodeError are deprecated. " + "Please set 'msg' (str), 'doc' (str) and 'pos' (int) arguments only.", + DeprecationWarning, + stacklevel=2, + ) + if pos is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = pos, *args + if doc is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = doc, *args + if msg is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = msg, *args + ValueError.__init__(self, *args) + return + + lineno = doc.count("\n", 0, pos) + 1 + if lineno == 1: + colno = pos + 1 + else: + colno = pos - doc.rindex("\n", 0, pos) + + if pos >= len(doc): + coord_repr = "end of document" + else: + coord_repr = f"line {lineno}, column {colno}" + errmsg = f"{msg} (at {coord_repr})" + ValueError.__init__(self, errmsg) + + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno -def load(fp: BinaryIO, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: +def load(fp: IO[bytes], /, *, parse_float: ParseFloat = float) -> dict[str, Any]: """Parse TOML from a binary file object.""" b = fp.read() try: @@ -71,9 +138,14 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n # The spec allows converting "\r\n" to "\n", even in string # literals. Let's do so to simplify parsing. - src = s.replace("\r\n", "\n") + try: + src = s.replace("\r\n", "\n") + except (AttributeError, TypeError): + raise TypeError( + f"Expected str object, not '{type(s).__qualname__}'" + ) from None pos = 0 - out = Output(NestedDict(), Flags()) + out = Output() header: Key = () parse_float = make_safe_parse_float(parse_float) @@ -113,7 +185,7 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n pos, header = create_dict_rule(src, pos, out) pos = skip_chars(src, pos, TOML_WS) elif char != "#": - raise suffixed_err(src, pos, "Invalid statement") + raise TOMLDecodeError("Invalid statement", src, pos) # 3. Skip comment pos = skip_comment(src, pos) @@ -124,8 +196,8 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n except IndexError: break if char != "\n": - raise suffixed_err( - src, pos, "Expected newline or end of document after a statement" + raise TOMLDecodeError( + "Expected newline or end of document after a statement", src, pos ) pos += 1 @@ -142,7 +214,7 @@ class Flags: EXPLICIT_NEST = 1 def __init__(self) -> None: - self._flags: dict[str, dict] = {} + self._flags: dict[str, dict[Any, Any]] = {} self._pending_flags: set[tuple[Key, int]] = set() def add_pending(self, key: Key, flag: int) -> None: @@ -200,7 +272,7 @@ def get_or_create_nest( key: Key, *, access_lists: bool = True, - ) -> dict: + ) -> dict[str, Any]: cont: Any = self.dict for k in key: if k not in cont: @@ -210,7 +282,7 @@ def get_or_create_nest( cont = cont[-1] if not isinstance(cont, dict): raise KeyError("There is no nest behind this key") - return cont + return cont # type: ignore[no-any-return] def append_nest_to_list(self, key: Key) -> None: cont = self.get_or_create_nest(key[:-1]) @@ -224,9 +296,10 @@ def append_nest_to_list(self, key: Key) -> None: cont[last_key] = [{}] -class Output(NamedTuple): - data: NestedDict - flags: Flags +class Output: + def __init__(self) -> None: + self.data = NestedDict() + self.flags = Flags() def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: @@ -251,12 +324,12 @@ def skip_until( except ValueError: new_pos = len(src) if error_on_eof: - raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None + raise TOMLDecodeError(f"Expected {expect!r}", src, new_pos) from None if not error_on.isdisjoint(src[pos:new_pos]): while src[pos] not in error_on: pos += 1 - raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") + raise TOMLDecodeError(f"Found invalid character {src[pos]!r}", src, pos) return new_pos @@ -287,15 +360,17 @@ def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot declare {key} twice") + raise TOMLDecodeError(f"Cannot declare {key} twice", src, pos) out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) try: out.data.get_or_create_nest(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]", pos): - raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") + raise TOMLDecodeError( + "Expected ']' at the end of a table declaration", src, pos + ) return pos + 1, key @@ -305,7 +380,7 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) # Free the namespace now that it points to another empty list item... out.flags.unset_all(key) # ...but this key precisely is still prohibited from table declaration @@ -313,10 +388,12 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: try: out.data.append_nest_to_list(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]]", pos): - raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") + raise TOMLDecodeError( + "Expected ']]' at the end of an array declaration", src, pos + ) return pos + 2, key @@ -331,22 +408,22 @@ def key_value_rule( for cont_key in relative_path_cont_keys: # Check that dotted key syntax does not redefine an existing table if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): - raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") + raise TOMLDecodeError(f"Cannot redefine namespace {cont_key}", src, pos) # Containers in the relative path can't be opened with the table syntax or # dotted key/value syntax in following table sections. out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) if out.flags.is_(abs_key_parent, Flags.FROZEN): - raise suffixed_err( - src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" + raise TOMLDecodeError( + f"Cannot mutate immutable namespace {abs_key_parent}", src, pos ) try: nest = out.data.get_or_create_nest(abs_key_parent) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, "Cannot overwrite a value") + raise TOMLDecodeError("Cannot overwrite a value", src, pos) # Mark inline table and array namespaces recursively immutable if isinstance(value, (dict, list)): out.flags.set(header + key, Flags.FROZEN, recursive=True) @@ -363,7 +440,7 @@ def parse_key_value_pair( except IndexError: char = None if char != "=": - raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") + raise TOMLDecodeError("Expected '=' after a key in a key/value pair", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS) pos, value = parse_value(src, pos, parse_float) @@ -401,7 +478,7 @@ def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: return parse_literal_str(src, pos) if char == '"': return parse_one_line_basic_str(src, pos) - raise suffixed_err(src, pos, "Invalid initial character for a key part") + raise TOMLDecodeError("Invalid initial character for a key part", src, pos) def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: @@ -409,9 +486,9 @@ def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: return parse_basic_str(src, pos, multiline=False) -def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list]: +def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list[Any]]: pos += 1 - array: list = [] + array: list[Any] = [] pos = skip_comments_and_array_ws(src, pos) if src.startswith("]", pos): @@ -425,7 +502,7 @@ def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list] if c == "]": return pos + 1, array if c != ",": - raise suffixed_err(src, pos, "Unclosed array") + raise TOMLDecodeError("Unclosed array", src, pos) pos += 1 pos = skip_comments_and_array_ws(src, pos) @@ -433,7 +510,7 @@ def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list] return pos + 1, array -def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, dict]: +def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, dict[str, Any]]: pos += 1 nested_dict = NestedDict() flags = Flags() @@ -445,20 +522,20 @@ def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos pos, key, value = parse_key_value_pair(src, pos, parse_float) key_parent, key_stem = key[:-1], key[-1] if flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) try: nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") + raise TOMLDecodeError(f"Duplicate inline table key {key_stem!r}", src, pos) nest[key_stem] = value pos = skip_chars(src, pos, TOML_WS) c = src[pos : pos + 1] if c == "}": return pos + 1, nested_dict.dict if c != ",": - raise suffixed_err(src, pos, "Unclosed inline table") + raise TOMLDecodeError("Unclosed inline table", src, pos) if isinstance(value, (dict, list)): flags.set(key, Flags.FROZEN, recursive=True) pos += 1 @@ -480,7 +557,7 @@ def parse_basic_str_escape( except IndexError: return pos, "" if char != "\n": - raise suffixed_err(src, pos, "Unescaped '\\' in a string") + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) return pos, "" @@ -491,7 +568,7 @@ def parse_basic_str_escape( try: return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] except KeyError: - raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) from None def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: @@ -501,11 +578,13 @@ def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: hex_str = src[pos : pos + hex_len] if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): - raise suffixed_err(src, pos, "Invalid hex value") + raise TOMLDecodeError("Invalid hex value", src, pos) pos += hex_len hex_int = int(hex_str, 16) if not is_unicode_scalar_value(hex_int): - raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") + raise TOMLDecodeError( + "Escaped character is not a Unicode scalar value", src, pos + ) return pos, chr(hex_int) @@ -562,7 +641,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: try: char = src[pos] except IndexError: - raise suffixed_err(src, pos, "Unterminated string") from None + raise TOMLDecodeError("Unterminated string", src, pos) from None if char == '"': if not multiline: return pos + 1, result + src[start_pos:pos] @@ -577,7 +656,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: start_pos = pos continue if char in error_on: - raise suffixed_err(src, pos, f"Illegal character {char!r}") + raise TOMLDecodeError(f"Illegal character {char!r}", src, pos) pos += 1 @@ -625,7 +704,7 @@ def parse_value( # noqa: C901 try: datetime_obj = match_to_datetime(datetime_match) except ValueError as e: - raise suffixed_err(src, pos, "Invalid date or datetime") from e + raise TOMLDecodeError("Invalid date or datetime", src, pos) from e return datetime_match.end(), datetime_obj localtime_match = RE_LOCALTIME.match(src, pos) if localtime_match: @@ -646,24 +725,7 @@ def parse_value( # noqa: C901 if first_four in {"-inf", "+inf", "-nan", "+nan"}: return pos + 4, parse_float(first_four) - raise suffixed_err(src, pos, "Invalid value") - - -def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: - """Return a `TOMLDecodeError` where error message is suffixed with - coordinates in source.""" - - def coord_repr(src: str, pos: Pos) -> str: - if pos >= len(src): - return "end of document" - line = src.count("\n", 0, pos) + 1 - if line == 1: - column = pos + 1 - else: - column = pos - src.rindex("\n", 0, pos) - return f"line {line}, column {column}" - - return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") + raise TOMLDecodeError("Invalid value", src, pos) def is_unicode_scalar_value(codepoint: int) -> bool: @@ -679,7 +741,7 @@ def make_safe_parse_float(parse_float: ParseFloat) -> ParseFloat: instead of returning illegal types. """ # The default `float` callable never returns illegal types. Optimize it. - if parse_float is float: # type: ignore[comparison-overlap] + if parse_float is float: return float def safe_parse_float(float_str: str) -> Any: diff --git a/Lib/tomllib/_re.py b/Lib/tomllib/_re.py index 994bb7493fd..eb8beb19747 100644 --- a/Lib/tomllib/_re.py +++ b/Lib/tomllib/_re.py @@ -7,9 +7,12 @@ from datetime import date, datetime, time, timedelta, timezone, tzinfo from functools import lru_cache import re -from typing import Any -from ._types import ParseFloat +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Any + + from ._types import ParseFloat # E.g. # - 00:32:00.999999 @@ -49,7 +52,7 @@ ) -def match_to_datetime(match: re.Match) -> datetime | date: +def match_to_datetime(match: re.Match[str]) -> datetime | date: """Convert a `RE_DATETIME` match to `datetime.datetime` or `datetime.date`. Raises ValueError if the match does not correspond to a valid date @@ -84,6 +87,9 @@ def match_to_datetime(match: re.Match) -> datetime | date: return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) +# No need to limit cache size. This is only ever called on input +# that matched RE_DATETIME, so there is an implicit bound of +# 24 (hours) * 60 (minutes) * 2 (offset direction) = 2880. @lru_cache(maxsize=None) def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: sign = 1 if sign_str == "+" else -1 @@ -95,13 +101,13 @@ def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: ) -def match_to_localtime(match: re.Match) -> time: +def match_to_localtime(match: re.Match[str]) -> time: hour_str, minute_str, sec_str, micros_str = match.groups() micros = int(micros_str.ljust(6, "0")) if micros_str else 0 return time(int(hour_str), int(minute_str), int(sec_str), micros) -def match_to_number(match: re.Match, parse_float: ParseFloat) -> Any: +def match_to_number(match: re.Match[str], parse_float: ParseFloat) -> Any: if match.group("floatpart"): return parse_float(match.group()) return int(match.group(), 0) diff --git a/Lib/tomllib/mypy.ini b/Lib/tomllib/mypy.ini new file mode 100644 index 00000000000..1761dce4556 --- /dev/null +++ b/Lib/tomllib/mypy.ini @@ -0,0 +1,17 @@ +# Config file for running mypy on tomllib. +# Run mypy by invoking `mypy --config-file Lib/tomllib/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/tomllib +mypy_path = $MYPY_CONFIG_FILE_DIR/../../Misc/mypy +explicit_package_bases = True +python_version = 3.12 +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code +strict = True +strict_bytes = True +local_partial_types = True +warn_unreachable = True diff --git a/Lib/trace.py b/Lib/trace.py old mode 100755 new mode 100644 index 213e46517d6..cf8817f4383 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # portions copyright 2001, Autonomous Zones Industries, Inc., all rights... # err... reserved and offered to the public under the terms of the # Python 2.2 license. @@ -49,6 +47,7 @@ """ __all__ = ['Trace', 'CoverageResults'] +import io import linecache import os import sys @@ -201,7 +200,8 @@ def update(self, other): for key in other_callers: callers[key] = 1 - def write_results(self, show_missing=True, summary=False, coverdir=None): + def write_results(self, show_missing=True, summary=False, coverdir=None, *, + ignore_missing_files=False): """ Write the coverage results. @@ -210,6 +210,9 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): :param coverdir: If None, the results of each module are placed in its directory, otherwise it is included in the directory specified. + :param ignore_missing_files: If True, counts for files that no longer + exist are silently ignored. Otherwise, a missing file + will raise a FileNotFoundError. """ if self.calledfuncs: print() @@ -252,13 +255,15 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): if filename.endswith(".pyc"): filename = filename[:-1] + if ignore_missing_files and not os.path.isfile(filename): + continue + if coverdir is None: dir = os.path.dirname(os.path.abspath(filename)) modulename = _modname(filename) else: dir = coverdir - if not os.path.exists(dir): - os.makedirs(dir) + os.makedirs(dir, exist_ok=True) modulename = _fullmodname(filename) # If desired, get a list of the line numbers which represent @@ -274,15 +279,13 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): n_hits, n_lines = self.write_results_file(coverpath, source, lnotab, count, encoding) if summary and n_lines: - percent = int(100 * n_hits / n_lines) - sums[modulename] = n_lines, percent, modulename, filename - + sums[modulename] = n_lines, n_hits, modulename, filename if summary and sums: print("lines cov% module (path)") for m in sorted(sums): - n_lines, percent, modulename, filename = sums[m] - print("%5d %3d%% %s (%s)" % sums[m]) + n_lines, n_hits, modulename, filename = sums[m] + print(f"{n_lines:5d} {n_hits/n_lines:.1%} {modulename} ({filename})") if self.outfile: # try and store counts and module info into self.outfile @@ -396,7 +399,7 @@ def __init__(self, count=1, trace=1, countfuncs=0, countcallers=0, @param countfuncs true iff it should just output a list of (filename, modulename, funcname,) for functions that were called at least once; This overrides - `count' and `trace' + 'count' and 'trace' @param ignoremods a list of the names of modules to ignore @param ignoredirs a list of the names of directories to ignore all of the (recursive) contents of @@ -528,7 +531,7 @@ def globaltrace_countfuncs(self, frame, why, arg): def globaltrace_lt(self, frame, why, arg): """Handler for call events. - If the code block being entered is to be ignored, returns `None', + If the code block being entered is to be ignored, returns 'None', else returns self.localtrace. """ if why == 'call': @@ -559,8 +562,12 @@ def localtrace_trace_and_count(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_trace(self, frame, why, arg): @@ -572,8 +579,12 @@ def localtrace_trace(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_count(self, frame, why, arg): @@ -593,7 +604,7 @@ def results(self): def main(): import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument('--version', action='version', version='trace 2.0') grp = parser.add_argument_group('Main options', @@ -716,7 +727,7 @@ def parse_ignore_dir(s): sys.argv = [opts.progname, *opts.arguments] sys.path[0] = os.path.dirname(opts.progname) - with open(opts.progname, 'rb') as fp: + with io.open_code(opts.progname) as fp: code = compile(fp.read(), opts.progname, 'exec') # try to emulate __main__ namespace as much as possible globs = { diff --git a/Lib/traceback.py b/Lib/traceback.py index d6a010f4157..5a34a2b87b6 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,21 +1,31 @@ """Extract, format and print information about Python stack traces.""" -import collections +import collections.abc import itertools import linecache import sys +import textwrap +import warnings +import codeop +import keyword +import tokenize +import io +import _colorize + +from contextlib import suppress __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', 'format_tb', 'print_exc', 'format_exc', 'print_exception', 'print_last', 'print_stack', 'print_tb', 'clear_frames', 'FrameSummary', 'StackSummary', 'TracebackException', - 'walk_stack', 'walk_tb'] + 'walk_stack', 'walk_tb', 'print_list'] # # Formatting and printing lists of traceback lines. # + def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or extract_stack() as a formatted stack trace to the given file.""" @@ -69,7 +79,8 @@ def extract_tb(tb, limit=None): trace. The line is a string with leading and trailing whitespace stripped; if the source is not available it is None. """ - return StackSummary.extract(walk_tb(tb), limit=limit) + return StackSummary._extract_from_extended_frame_gen( + _walk_tb_with_full_positions(tb), limit=limit) # # Exception formatting and output. @@ -95,14 +106,18 @@ def _parse_value_tb(exc, value, tb): raise ValueError("Both or neither of value and tb must be given") if value is tb is _sentinel: if exc is not None: - return exc, exc.__traceback__ + if isinstance(exc, BaseException): + return exc, exc.__traceback__ + + raise TypeError(f'Exception expected for value, ' + f'{type(exc).__name__} found') else: return None, None return value, tb def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - file=None, chain=True): + file=None, chain=True, **kwargs): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -113,16 +128,23 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ occurred with a caret on the next line indicating the approximate position of the error. """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) - if file is None: - file = sys.stderr te = TracebackException(type(value), value, tb, limit=limit, compact=True) - for line in te.format(chain=chain): - print(line, file=file, end="") + te.print(file=file, chain=chain, colorize=colorize) + + +BUILTIN_EXCEPTION_LIMIT = object() + + +def _print_exception_bltin(exc, /): + file = sys.stderr if sys.stderr is not None else sys.__stderr__ + colorize = _colorize.can_colorize(file=file) + return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -131,64 +153,77 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) -def format_exception_only(exc, /, value=_sentinel): +def format_exception_only(exc, /, value=_sentinel, *, show_group=False, **kwargs): """Format the exception part of a traceback. The return value is a list of strings, each ending in a newline. - Normally, the list contains a single string; however, for - SyntaxError exceptions, it contains several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the list. + The list contains the exception's message, which is + normally a single string; however, for :exc:`SyntaxError` exceptions, it + contains several lines that (when printed) display detailed information + about where the syntax error occurred. Following the message, the list + contains the exception's ``__notes__``. + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. """ + colorize = kwargs.get("colorize", False) if value is _sentinel: value = exc te = TracebackException(type(value), value, None, compact=True) - return list(te.format_exception_only()) + return list(te.format_exception_only(show_group=show_group, colorize=colorize)) # -- not official API but folk probably use these two functions. -def _format_final_exc_line(etype, value): - valuestr = _some_str(value) +def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize=False): + valuestr = _safe_string(value, 'exception') + end_char = "\n" if insert_final_newline else "" + if colorize: + theme = _colorize.get_theme(force_color=True).traceback + else: + theme = _colorize.get_theme(force_no_color=True).traceback if value is None or not valuestr: - line = "%s\n" % etype + line = f"{theme.type}{etype}{theme.reset}{end_char}" else: - line = "%s: %s\n" % (etype, valuestr) + line = f"{theme.type}{etype}{theme.reset}: {theme.message}{valuestr}{theme.reset}{end_char}" return line -def _some_str(value): + +def _safe_string(value, what, func=str): try: - return str(value) + return func(value) except: - return '<unprintable %s object>' % type(value).__name__ + return f'<{what} {func.__name__}() failed>' # -- def print_exc(limit=None, file=None, chain=True): - """Shorthand for 'print_exception(*sys.exc_info(), limit, file)'.""" - print_exception(*sys.exc_info(), limit=limit, file=file, chain=chain) + """Shorthand for 'print_exception(sys.exception(), limit=limit, file=file, chain=chain)'.""" + print_exception(sys.exception(), limit=limit, file=file, chain=chain) def format_exc(limit=None, chain=True): """Like print_exc() but return a string.""" - return "".join(format_exception(*sys.exc_info(), limit=limit, chain=chain)) + return "".join(format_exception(sys.exception(), limit=limit, chain=chain)) def print_last(limit=None, file=None, chain=True): - """This is a shorthand for 'print_exception(sys.last_type, - sys.last_value, sys.last_traceback, limit, file)'.""" - if not hasattr(sys, "last_type"): + """This is a shorthand for 'print_exception(sys.last_exc, limit=limit, file=file, chain=chain)'.""" + if not hasattr(sys, "last_exc") and not hasattr(sys, "last_type"): raise ValueError("no last exception") - print_exception(sys.last_type, sys.last_value, sys.last_traceback, - limit, file, chain) + + if hasattr(sys, "last_exc"): + print_exception(sys.last_exc, limit=limit, file=file, chain=chain) + else: + print_exception(sys.last_type, sys.last_value, sys.last_traceback, + limit=limit, file=file, chain=chain) + # # Printing and Extracting Stacks. @@ -241,7 +276,7 @@ def clear_frames(tb): class FrameSummary: - """A single frame from a traceback. + """Information about a single frame from a traceback. - :attr:`filename` The filename for the frame. - :attr:`lineno` The line within filename for the frame that was @@ -254,10 +289,12 @@ class FrameSummary: mapping the name to the repr() of the variable. """ - __slots__ = ('filename', 'lineno', 'name', '_line', 'locals') + __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno', + 'name', '_lines', '_lines_dedented', 'locals', '_code') def __init__(self, filename, lineno, name, *, lookup_line=True, - locals=None, line=None): + locals=None, line=None, + end_lineno=None, colno=None, end_colno=None, **kwargs): """Construct a FrameSummary. :param lookup_line: If True, `linecache` is consulted for the source @@ -269,11 +306,17 @@ def __init__(self, filename, lineno, name, *, lookup_line=True, """ self.filename = filename self.lineno = lineno + self.end_lineno = lineno if end_lineno is None else end_lineno + self.colno = colno + self.end_colno = end_colno self.name = name - self._line = line + self._code = kwargs.get("_code") + self._lines = line + self._lines_dedented = None if lookup_line: self.line - self.locals = {k: repr(v) for k, v in locals.items()} if locals else None + self.locals = {k: _safe_string(v, 'local', func=repr) + for k, v in locals.items()} if locals else None def __eq__(self, other): if isinstance(other, FrameSummary): @@ -298,13 +341,43 @@ def __repr__(self): def __len__(self): return 4 + def _set_lines(self): + if ( + self._lines is None + and self.lineno is not None + and self.end_lineno is not None + ): + lines = [] + for lineno in range(self.lineno, self.end_lineno + 1): + # treat errors (empty string) and empty lines (newline) as the same + line = linecache.getline(self.filename, lineno).rstrip() + if not line and self._code is not None and self.filename.startswith("<"): + line = linecache._getline_from_code(self._code, lineno).rstrip() + lines.append(line) + self._lines = "\n".join(lines) + "\n" + + @property + def _original_lines(self): + # Returns the line as-is from the source, without modifying whitespace. + self._set_lines() + return self._lines + + @property + def _dedented_lines(self): + # Returns _original_lines, but dedented + self._set_lines() + if self._lines_dedented is None and self._lines is not None: + self._lines_dedented = textwrap.dedent(self._lines) + return self._lines_dedented + @property def line(self): - if self._line is None: - if self.lineno is None: - return None - self._line = linecache.getline(self.filename, self.lineno) - return self._line.strip() + self._set_lines() + if self._lines is None: + return None + # return only the first line, stripped + return self._lines.partition("\n")[0].strip() + def walk_stack(f): """Walk a stack yielding the frame and line number for each frame. @@ -313,10 +386,14 @@ def walk_stack(f): current stack is used. Usually used with StackSummary.extract. """ if f is None: - f = sys._getframe().f_back.f_back - while f is not None: - yield f, f.f_lineno - f = f.f_back + f = sys._getframe().f_back + + def walk_stack_generator(frame): + while frame is not None: + yield frame, frame.f_lineno + frame = frame.f_back + + return walk_stack_generator(f) def walk_tb(tb): @@ -330,18 +407,40 @@ def walk_tb(tb): tb = tb.tb_next +def _walk_tb_with_full_positions(tb): + # Internal version of walk_tb that yields full code positions including + # end line and column information. + while tb is not None: + positions = _get_code_position(tb.tb_frame.f_code, tb.tb_lasti) + # Yield tb_lineno when co_positions does not have a line number to + # maintain behavior with walk_tb. + if positions[0] is None: + yield tb.tb_frame, (tb.tb_lineno, ) + positions[1:] + else: + yield tb.tb_frame, positions + tb = tb.tb_next + + +def _get_code_position(code, instruction_index): + if instruction_index < 0: + return (None, None, None, None) + positions_gen = code.co_positions() + return next(itertools.islice(positions_gen, instruction_index // 2, None)) + + _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. + class StackSummary(list): - """A stack of frames.""" + """A list of FrameSummary objects, representing a stack of frames.""" @classmethod def extract(klass, frame_gen, *, limit=None, lookup_lines=True, capture_locals=False): """Create a StackSummary from a traceback or stack object. - :param frame_gen: A generator that yields (frame, lineno) tuples to - include in the stack. + :param frame_gen: A generator that yields (frame, lineno) tuples + whose summaries are to be included in the stack. :param limit: None to include all frames or the number of frames to include. :param lookup_lines: If True, lookup lines for each frame immediately, @@ -349,23 +448,41 @@ def extract(klass, frame_gen, *, limit=None, lookup_lines=True, :param capture_locals: If True, the local variables from each frame will be captured as object representations into the FrameSummary. """ - if limit is None: + def extended_frame_gen(): + for f, lineno in frame_gen: + yield f, (lineno, None, None, None) + + return klass._extract_from_extended_frame_gen( + extended_frame_gen(), limit=limit, lookup_lines=lookup_lines, + capture_locals=capture_locals) + + @classmethod + def _extract_from_extended_frame_gen(klass, frame_gen, *, limit=None, + lookup_lines=True, capture_locals=False): + # Same as extract but operates on a frame generator that yields + # (frame, (lineno, end_lineno, colno, end_colno)) in the stack. + # Only lineno is required, the remaining fields can be None if the + # information is not available. + builtin_limit = limit is BUILTIN_EXCEPTION_LIMIT + if limit is None or builtin_limit: limit = getattr(sys, 'tracebacklimit', None) if limit is not None and limit < 0: limit = 0 if limit is not None: - if limit >= 0: + if builtin_limit: + frame_gen = tuple(frame_gen) + frame_gen = frame_gen[len(frame_gen) - limit:] + elif limit >= 0: frame_gen = itertools.islice(frame_gen, limit) else: frame_gen = collections.deque(frame_gen, maxlen=-limit) result = klass() fnames = set() - for f, lineno in frame_gen: + for f, (lineno, end_lineno, colno, end_colno) in frame_gen: co = f.f_code filename = co.co_filename name = co.co_name - fnames.add(filename) linecache.lazycache(filename, f.f_globals) # Must defer line lookups until we have called checkcache. @@ -373,10 +490,16 @@ def extract(klass, frame_gen, *, limit=None, lookup_lines=True, f_locals = f.f_locals else: f_locals = None - result.append(FrameSummary( - filename, lineno, name, lookup_line=False, locals=f_locals)) + result.append( + FrameSummary(filename, lineno, name, + lookup_line=False, locals=f_locals, + end_lineno=end_lineno, colno=colno, end_colno=end_colno, + _code=f.f_code, + ) + ) for filename in fnames: linecache.checkcache(filename) + # If immediate lookup was desired, trigger lookups now. if lookup_lines: for f in result: @@ -402,7 +525,224 @@ def from_list(klass, a_list): result.append(FrameSummary(filename, lineno, name, line=line)) return result - def format(self): + def format_frame_summary(self, frame_summary, **kwargs): + """Format the lines for a single FrameSummary. + + Returns a string representing one frame involved in the stack. This + gets called for every frame to be printed in the stack summary. + """ + colorize = kwargs.get("colorize", False) + row = [] + filename = frame_summary.filename + if frame_summary.filename.startswith("<stdin-") and frame_summary.filename.endswith('>'): + filename = "<stdin>" + if colorize: + theme = _colorize.get_theme(force_color=True).traceback + else: + theme = _colorize.get_theme(force_no_color=True).traceback + row.append( + ' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( + theme.filename, + filename, + theme.reset, + theme.line_no, + frame_summary.lineno, + theme.reset, + theme.frame, + frame_summary.name, + theme.reset, + ) + ) + if frame_summary._dedented_lines and frame_summary._dedented_lines.strip(): + if ( + frame_summary.colno is None or + frame_summary.end_colno is None + ): + # only output first line if column information is missing + row.append(textwrap.indent(frame_summary.line, ' ') + "\n") + else: + # get first and last line + all_lines_original = frame_summary._original_lines.splitlines() + first_line = all_lines_original[0] + # assume all_lines_original has enough lines (since we constructed it) + last_line = all_lines_original[frame_summary.end_lineno - frame_summary.lineno] + + # character index of the start/end of the instruction + start_offset = _byte_offset_to_character_offset(first_line, frame_summary.colno) + end_offset = _byte_offset_to_character_offset(last_line, frame_summary.end_colno) + + all_lines = frame_summary._dedented_lines.splitlines()[ + :frame_summary.end_lineno - frame_summary.lineno + 1 + ] + + # adjust start/end offset based on dedent + dedent_characters = len(first_line) - len(all_lines[0]) + start_offset = max(0, start_offset - dedent_characters) + end_offset = max(0, end_offset - dedent_characters) + + # When showing this on a terminal, some of the non-ASCII characters + # might be rendered as double-width characters, so we need to take + # that into account when calculating the length of the line. + dp_start_offset = _display_width(all_lines[0], offset=start_offset) + dp_end_offset = _display_width(all_lines[-1], offset=end_offset) + + # get exact code segment corresponding to the instruction + segment = "\n".join(all_lines) + segment = segment[start_offset:len(segment) - (len(all_lines[-1]) - end_offset)] + + # attempt to parse for anchors + anchors = None + show_carets = False + with suppress(Exception): + anchors = _extract_caret_anchors_from_line_segment(segment) + show_carets = self._should_show_carets(start_offset, end_offset, all_lines, anchors) + + result = [] + + # only display first line, last line, and lines around anchor start/end + significant_lines = {0, len(all_lines) - 1} + + anchors_left_end_offset = 0 + anchors_right_start_offset = 0 + primary_char = "^" + secondary_char = "^" + if anchors: + anchors_left_end_offset = anchors.left_end_offset + anchors_right_start_offset = anchors.right_start_offset + # computed anchor positions do not take start_offset into account, + # so account for it here + if anchors.left_end_lineno == 0: + anchors_left_end_offset += start_offset + if anchors.right_start_lineno == 0: + anchors_right_start_offset += start_offset + + # account for display width + anchors_left_end_offset = _display_width( + all_lines[anchors.left_end_lineno], offset=anchors_left_end_offset + ) + anchors_right_start_offset = _display_width( + all_lines[anchors.right_start_lineno], offset=anchors_right_start_offset + ) + + primary_char = anchors.primary_char + secondary_char = anchors.secondary_char + significant_lines.update( + range(anchors.left_end_lineno - 1, anchors.left_end_lineno + 2) + ) + significant_lines.update( + range(anchors.right_start_lineno - 1, anchors.right_start_lineno + 2) + ) + + # remove bad line numbers + significant_lines.discard(-1) + significant_lines.discard(len(all_lines)) + + def output_line(lineno): + """output all_lines[lineno] along with carets""" + result.append(all_lines[lineno] + "\n") + if not show_carets: + return + num_spaces = len(all_lines[lineno]) - len(all_lines[lineno].lstrip()) + carets = [] + num_carets = dp_end_offset if lineno == len(all_lines) - 1 else _display_width(all_lines[lineno]) + # compute caret character for each position + for col in range(num_carets): + if col < num_spaces or (lineno == 0 and col < dp_start_offset): + # before first non-ws char of the line, or before start of instruction + carets.append(' ') + elif anchors and ( + lineno > anchors.left_end_lineno or + (lineno == anchors.left_end_lineno and col >= anchors_left_end_offset) + ) and ( + lineno < anchors.right_start_lineno or + (lineno == anchors.right_start_lineno and col < anchors_right_start_offset) + ): + # within anchors + carets.append(secondary_char) + else: + carets.append(primary_char) + if colorize: + # Replace the previous line with a red version of it only in the parts covered + # by the carets. + line = result[-1] + colorized_line_parts = [] + colorized_carets_parts = [] + + for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): + caret_group = list(group) + if color == "^": + colorized_line_parts.append(theme.error_highlight + "".join(char for char, _ in caret_group) + theme.reset) + colorized_carets_parts.append(theme.error_highlight + "".join(caret for _, caret in caret_group) + theme.reset) + elif color == "~": + colorized_line_parts.append(theme.error_range + "".join(char for char, _ in caret_group) + theme.reset) + colorized_carets_parts.append(theme.error_range + "".join(caret for _, caret in caret_group) + theme.reset) + else: + colorized_line_parts.append("".join(char for char, _ in caret_group)) + colorized_carets_parts.append("".join(caret for _, caret in caret_group)) + + colorized_line = "".join(colorized_line_parts) + colorized_carets = "".join(colorized_carets_parts) + result[-1] = colorized_line + result.append(colorized_carets + "\n") + else: + result.append("".join(carets) + "\n") + + # display significant lines + sig_lines_list = sorted(significant_lines) + for i, lineno in enumerate(sig_lines_list): + if i: + linediff = lineno - sig_lines_list[i - 1] + if linediff == 2: + # 1 line in between - just output it + output_line(lineno - 1) + elif linediff > 2: + # > 1 line in between - abbreviate + result.append(f"...<{linediff - 1} lines>...\n") + output_line(lineno) + + row.append( + textwrap.indent(textwrap.dedent("".join(result)), ' ', lambda line: True) + ) + if frame_summary.locals: + for name, value in sorted(frame_summary.locals.items()): + row.append(' {name} = {value}\n'.format(name=name, value=value)) + + return ''.join(row) + + def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): + with suppress(SyntaxError, ImportError): + import ast + tree = ast.parse('\n'.join(all_lines)) + if not tree.body: + return False + statement = tree.body[0] + value = None + def _spawns_full_line(value): + return ( + value.lineno == 1 + and value.end_lineno == len(all_lines) + and value.col_offset == start_offset + and value.end_col_offset == end_offset + ) + match statement: + case ast.Return(value=ast.Call()): + if isinstance(statement.value.func, ast.Name): + value = statement.value + case ast.Assign(value=ast.Call()): + if ( + len(statement.targets) == 1 and + isinstance(statement.targets[0], ast.Name) + ): + value = statement.value + if value is not None and _spawns_full_line(value): + return False + if anchors: + return True + if all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + return True + return False + + def format(self, **kwargs): """Format the stack ready for printing. Returns a list of strings ready for printing. Each string in the @@ -414,37 +754,34 @@ def format(self): repetitions are shown, followed by a summary line stating the exact number of further repetitions. """ + colorize = kwargs.get("colorize", False) result = [] last_file = None last_line = None last_name = None count = 0 - for frame in self: - if (last_file is None or last_file != frame.filename or - last_line is None or last_line != frame.lineno or - last_name is None or last_name != frame.name): + for frame_summary in self: + formatted_frame = self.format_frame_summary(frame_summary, colorize=colorize) + if formatted_frame is None: + continue + if (last_file is None or last_file != frame_summary.filename or + last_line is None or last_line != frame_summary.lineno or + last_name is None or last_name != frame_summary.name): if count > _RECURSIVE_CUTOFF: count -= _RECURSIVE_CUTOFF result.append( f' [Previous line repeated {count} more ' f'time{"s" if count > 1 else ""}]\n' ) - last_file = frame.filename - last_line = frame.lineno - last_name = frame.name + last_file = frame_summary.filename + last_line = frame_summary.lineno + last_name = frame_summary.name count = 0 count += 1 if count > _RECURSIVE_CUTOFF: continue - row = [] - row.append(' File "{}", line {}, in {}\n'.format( - frame.filename, frame.lineno, frame.name)) - if frame.line: - row.append(' {}\n'.format(frame.line.strip())) - if frame.locals: - for name, value in sorted(frame.locals.items()): - row.append(' {name} = {value}\n'.format(name=name, value=value)) - result.append(''.join(row)) + result.append(formatted_frame) + if count > _RECURSIVE_CUTOFF: count -= _RECURSIVE_CUTOFF result.append( @@ -454,6 +791,216 @@ def format(self): return result +def _byte_offset_to_character_offset(str, offset): + as_utf8 = str.encode('utf-8') + return len(as_utf8[:offset].decode("utf-8", errors="replace")) + + +_Anchors = collections.namedtuple( + "_Anchors", + [ + "left_end_lineno", + "left_end_offset", + "right_start_lineno", + "right_start_offset", + "primary_char", + "secondary_char", + ], + defaults=["~", "^"] +) + +def _extract_caret_anchors_from_line_segment(segment): + """ + Given source code `segment` corresponding to a FrameSummary, determine: + - for binary ops, the location of the binary op + - for indexing and function calls, the location of the brackets. + `segment` is expected to be a valid Python expression. + """ + import ast + + try: + # Without parentheses, `segment` is parsed as a statement. + # Binary ops, subscripts, and calls are expressions, so + # we can wrap them with parentheses to parse them as + # (possibly multi-line) expressions. + # e.g. if we try to highlight the addition in + # x = ( + # a + + # b + # ) + # then we would ast.parse + # a + + # b + # which is not a valid statement because of the newline. + # Adding brackets makes it a valid expression. + # ( + # a + + # b + # ) + # Line locations will be different than the original, + # which is taken into account later on. + tree = ast.parse(f"(\n{segment}\n)") + except SyntaxError: + return None + + if len(tree.body) != 1: + return None + + lines = segment.splitlines() + + def normalize(lineno, offset): + """Get character index given byte offset""" + return _byte_offset_to_character_offset(lines[lineno], offset) + + def next_valid_char(lineno, col): + """Gets the next valid character index in `lines`, if + the current location is not valid. Handles empty lines. + """ + while lineno < len(lines) and col >= len(lines[lineno]): + col = 0 + lineno += 1 + assert lineno < len(lines) and col < len(lines[lineno]) + return lineno, col + + def increment(lineno, col): + """Get the next valid character index in `lines`.""" + col += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def nextline(lineno, col): + """Get the next valid character at least on the next line""" + col = 0 + lineno += 1 + lineno, col = next_valid_char(lineno, col) + return lineno, col + + def increment_until(lineno, col, stop): + """Get the next valid non-"\\#" character that satisfies the `stop` predicate""" + while True: + ch = lines[lineno][col] + if ch in "\\#": + lineno, col = nextline(lineno, col) + elif not stop(ch): + lineno, col = increment(lineno, col) + else: + break + return lineno, col + + def setup_positions(expr, force_valid=True): + """Get the lineno/col position of the end of `expr`. If `force_valid` is True, + forces the position to be a valid character (e.g. if the position is beyond the + end of the line, move to the next line) + """ + # -2 since end_lineno is 1-indexed and because we added an extra + # bracket + newline to `segment` when calling ast.parse + lineno = expr.end_lineno - 2 + col = normalize(lineno, expr.end_col_offset) + return next_valid_char(lineno, col) if force_valid else (lineno, col) + + statement = tree.body[0] + match statement: + case ast.Expr(expr): + match expr: + case ast.BinOp(): + # ast gives these locations for BinOp subexpressions + # ( left_expr ) + ( right_expr ) + # left^^^^^ right^^^^^ + lineno, col = setup_positions(expr.left) + + # First operator character is the first non-space/')' character + lineno, col = increment_until(lineno, col, lambda x: not x.isspace() and x != ')') + + # binary op is 1 or 2 characters long, on the same line, + # before the right subexpression + right_col = col + 1 + if ( + right_col < len(lines[lineno]) + and ( + # operator char should not be in the right subexpression + expr.right.lineno - 2 > lineno or + right_col < normalize(expr.right.lineno - 2, expr.right.col_offset) + ) + and not (ch := lines[lineno][right_col]).isspace() + and ch not in "\\#" + ): + right_col += 1 + + # right_col can be invalid since it is exclusive + return _Anchors(lineno, col, lineno, right_col) + case ast.Subscript(): + # ast gives these locations for value and slice subexpressions + # ( value_expr ) [ slice_expr ] + # value^^^^^ slice^^^^^ + # subscript^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.value) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '[') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + case ast.Call(): + # ast gives these locations for function call expressions + # ( func_expr ) (args, kwargs) + # func^^^^^ + # call^^^^^^^^^^^^^^^^^^^^^^^^ + + # find left bracket + left_lineno, left_col = setup_positions(expr.func) + left_lineno, left_col = increment_until(left_lineno, left_col, lambda x: x == '(') + # find right bracket (final character of expression) + right_lineno, right_col = setup_positions(expr, force_valid=False) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + + return None + +_WIDE_CHAR_SPECIFIERS = "WF" + +def _display_width(line, offset=None): + """Calculate the extra amount of width space the given source + code segment might take if it were to be displayed on a fixed + width output device. Supports wide unicode characters and emojis.""" + + if offset is None: + offset = len(line) + + # Fast track for ASCII-only strings + if line.isascii(): + return offset + + import unicodedata + + return sum( + 2 if unicodedata.east_asian_width(char) in _WIDE_CHAR_SPECIFIERS else 1 + for char in line[:offset] + ) + + + +class _ExceptionPrintContext: + def __init__(self): + self.seen = set() + self.exception_group_depth = 0 + self.need_close = False + + def indent(self): + return ' ' * (2 * self.exception_group_depth) + + def emit(self, text_gen, margin_char=None): + if margin_char is None: + margin_char = '|' + indent_str = self.indent() + if self.exception_group_depth: + indent_str += margin_char + ' ' + + if isinstance(text_gen, str): + yield textwrap.indent(text_gen, indent_str, lambda line: True) + else: + for text in text_gen: + yield textwrap.indent(text, indent_str, lambda line: True) + + class TracebackException: """An exception ready for rendering. @@ -461,16 +1008,24 @@ class TracebackException: to this intermediary form to ensure that no references are held, while still being able to fully print or format it. + max_group_width and max_group_depth control the formatting of exception + groups. The depth refers to the nesting level of the group, and the width + refers to the size of a single exception group's exceptions array. The + formatted output is truncated when either limit is exceeded. + Use `from_exception` to create TracebackException instances from exception objects, or the constructor to create TracebackException instances from individual components. - :attr:`__cause__` A TracebackException of the original *__cause__*. - :attr:`__context__` A TracebackException of the original *__context__*. + - :attr:`exceptions` For exception groups - a list of TracebackException + instances for the nested *exceptions*. ``None`` for other exceptions. - :attr:`__suppress_context__` The *__suppress_context__* value from the original exception. - :attr:`stack` A `StackSummary` representing the traceback. - - :attr:`exc_type` The class of the original traceback. + - :attr:`exc_type` (deprecated) The class of the original traceback. + - :attr:`exc_type_str` String display of exc_type - :attr:`filename` For syntax errors - the filename where the error occurred. - :attr:`lineno` For syntax errors - the linenumber where the error @@ -481,14 +1036,14 @@ class TracebackException: occurred. - :attr:`offset` For syntax errors - the offset into the text where the error occurred. - - :attr:`end_offset` For syntax errors - the offset into the text where the - error occurred. Can be `None` if not present. + - :attr:`end_offset` For syntax errors - the end offset into the text where + the error occurred. Can be `None` if not present. - :attr:`msg` For syntax errors - the compiler error message. """ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, - _seen=None): + max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. @@ -498,14 +1053,34 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _seen = set() _seen.add(id(exc_value)) - # TODO: locals. - self.stack = StackSummary.extract( - walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines, + self.max_group_width = max_group_width + self.max_group_depth = max_group_depth + + self.stack = StackSummary._extract_from_extended_frame_gen( + _walk_tb_with_full_positions(exc_traceback), + limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals) - self.exc_type = exc_type + + self._exc_type = exc_type if save_exc_type else None + # Capture now to permit freeing resources: only complication is in the # unofficial API _format_final_exc_line - self._str = _some_str(exc_value) + self._str = _safe_string(exc_value, 'exception') + try: + self.__notes__ = getattr(exc_value, '__notes__', None) + except Exception as e: + self.__notes__ = [ + f'Ignored error getting __notes__: {_safe_string(e, '__notes__', repr)}'] + + self._is_syntax_error = False + self._have_exc_type = exc_type is not None + if exc_type is not None: + self.exc_type_qualname = exc_type.__qualname__ + self.exc_type_module = exc_type.__module__ + else: + self.exc_type_qualname = None + self.exc_type_module = None + if exc_type and issubclass(exc_type, SyntaxError): # Handle SyntaxError's specially self.filename = exc_value.filename @@ -517,6 +1092,27 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, self.offset = exc_value.offset self.end_offset = exc_value.end_offset self.msg = exc_value.msg + self._is_syntax_error = True + self._exc_metadata = getattr(exc_value, "_metadata", None) + elif exc_type and issubclass(exc_type, ImportError) and \ + getattr(exc_value, "name_from", None) is not None: + wrong_name = getattr(exc_value, "name_from", None) + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + self._str += f". Did you mean: '{suggestion}'?" + elif exc_type and issubclass(exc_type, (NameError, AttributeError)) and \ + getattr(exc_value, "name", None) is not None: + wrong_name = getattr(exc_value, "name", None) + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + self._str += f". Did you mean: '{suggestion}'?" + if issubclass(exc_type, NameError): + wrong_name = getattr(exc_value, "name", None) + if wrong_name is not None and wrong_name in sys.stdlib_module_names: + if suggestion: + self._str += f" Or did you forget to import '{wrong_name}'?" + else: + self._str += f". Did you forget to import '{wrong_name}'?" if lookup_lines: self._load_lines() self.__suppress_context__ = \ @@ -528,7 +1124,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, queue = [(self, exc_value)] while queue: te, e = queue.pop() - if (e and e.__cause__ is not None + if (e is not None and e.__cause__ is not None and id(e.__cause__) not in _seen): cause = TracebackException( type(e.__cause__), @@ -537,6 +1133,8 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) else: cause = None @@ -547,7 +1145,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, not e.__suppress_context__) else: need_context = True - if (e and e.__context__ is not None + if (e is not None and e.__context__ is not None and need_context and id(e.__context__) not in _seen): context = TracebackException( type(e.__context__), @@ -556,21 +1154,62 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, limit=limit, lookup_lines=lookup_lines, capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, _seen=_seen) else: context = None + + if e is not None and isinstance(e, BaseExceptionGroup): + exceptions = [] + for exc in e.exceptions: + texc = TracebackException( + type(exc), + exc, + exc.__traceback__, + limit=limit, + lookup_lines=lookup_lines, + capture_locals=capture_locals, + max_group_width=max_group_width, + max_group_depth=max_group_depth, + _seen=_seen) + exceptions.append(texc) + else: + exceptions = None + te.__cause__ = cause te.__context__ = context + te.exceptions = exceptions if cause: queue.append((te.__cause__, e.__cause__)) if context: queue.append((te.__context__, e.__context__)) + if exceptions: + queue.extend(zip(te.exceptions, e.exceptions)) @classmethod def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) + @property + def exc_type(self): + warnings.warn('Deprecated in 3.13. Use exc_type_str instead.', + DeprecationWarning, stacklevel=2) + return self._exc_type + + @property + def exc_type_str(self): + if not self._have_exc_type: + return None + stype = self.exc_type_qualname + smod = self.exc_type_module + if smod not in ("__main__", "builtins"): + if not isinstance(smod, str): + smod = "<unknown>" + stype = smod + '.' + stype + return stype + def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" for frame in self.stack: @@ -584,72 +1223,249 @@ def __eq__(self, other): def __str__(self): return self._str - def format_exception_only(self): + def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): """Format the exception part of the traceback. The return value is a generator of strings, each ending in a newline. - Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the output. + Generator yields the exception message. + For :exc:`SyntaxError` exceptions, it + also yields (before the exception message) + several lines that (when printed) + display detailed information about where the syntax error occurred. + Following the message, generator also yields + all the exception's ``__notes__``. + + When *show_group* is ``True``, and the exception is an instance of + :exc:`BaseExceptionGroup`, the nested exceptions are included as + well, recursively, with indentation relative to their nesting depth. """ - if self.exc_type is None: - yield _format_final_exc_line(None, self._str) + colorize = kwargs.get("colorize", False) + + indent = 3 * _depth * ' ' + if not self._have_exc_type: + yield indent + _format_final_exc_line(None, self._str, colorize=colorize) return - stype = self.exc_type.__qualname__ - smod = self.exc_type.__module__ - if smod not in ("__main__", "builtins"): - if not isinstance(smod, str): - smod = "<unknown>" - stype = smod + '.' + stype + stype = self.exc_type_str + if not self._is_syntax_error: + if _depth > 0: + # Nested exceptions needs correct handling of multiline messages. + formatted = _format_final_exc_line( + stype, self._str, insert_final_newline=False, colorize=colorize + ).split('\n') + yield from [ + indent + l + '\n' + for l in formatted + ] + else: + yield _format_final_exc_line(stype, self._str, colorize=colorize) + else: + yield from [indent + l for l in self._format_syntax_error(stype, colorize=colorize)] + + if ( + isinstance(self.__notes__, collections.abc.Sequence) + and not isinstance(self.__notes__, (str, bytes)) + ): + for note in self.__notes__: + note = _safe_string(note, 'note') + yield from [indent + l + '\n' for l in note.split('\n')] + elif self.__notes__ is not None: + yield indent + "{}\n".format(_safe_string(self.__notes__, '__notes__', func=repr)) + + if self.exceptions and show_group: + for ex in self.exceptions: + yield from ex.format_exception_only(show_group=show_group, _depth=_depth+1, colorize=colorize) + + def _find_keyword_typos(self): + assert self._is_syntax_error + try: + import _suggestions + except ImportError: + _suggestions = None + + # Only try to find keyword typos if there is no custom message + if self.msg != "invalid syntax" and "Perhaps you forgot a comma" not in self.msg: + return + + if not self._exc_metadata: + return - if not issubclass(self.exc_type, SyntaxError): - yield _format_final_exc_line(stype, self._str) + line, offset, source = self._exc_metadata + end_line = int(self.lineno) if self.lineno is not None else 0 + lines = None + from_filename = False + + if source is None: + if self.filename: + try: + with open(self.filename) as f: + lines = f.read().splitlines() + except Exception: + line, end_line, offset = 0,1,0 + else: + from_filename = True + lines = lines if lines is not None else self.text.splitlines() else: - yield from self._format_syntax_error(stype) + lines = source.splitlines() + + error_code = lines[line -1 if line > 0 else 0:end_line] + error_code = textwrap.dedent('\n'.join(error_code)) + + # Do not continue if the source is too large + if len(error_code) > 1024: + return + + error_lines = error_code.splitlines() + tokens = tokenize.generate_tokens(io.StringIO(error_code).readline) + tokens_left_to_process = 10 + import difflib + for token in tokens: + start, end = token.start, token.end + if token.type != tokenize.NAME: + continue + # Only consider NAME tokens on the same line as the error + the_end = end_line if line == 0 else end_line + 1 + if from_filename and token.start[0]+line != the_end: + continue + wrong_name = token.string + if wrong_name in keyword.kwlist: + continue - def _format_syntax_error(self, stype): + # Limit the number of valid tokens to consider to not spend + # to much time in this function + tokens_left_to_process -= 1 + if tokens_left_to_process < 0: + break + # Limit the number of possible matches to try + max_matches = 3 + matches = [] + if _suggestions is not None: + suggestion = _suggestions._generate_suggestions(keyword.kwlist, wrong_name) + if suggestion: + matches.append(suggestion) + matches.extend(difflib.get_close_matches(wrong_name, keyword.kwlist, n=max_matches, cutoff=0.5)) + matches = matches[:max_matches] + for suggestion in matches: + if not suggestion or suggestion == wrong_name: + continue + # Try to replace the token with the keyword + the_lines = error_lines.copy() + the_line = the_lines[start[0] - 1][:] + chars = list(the_line) + chars[token.start[1]:token.end[1]] = suggestion + the_lines[start[0] - 1] = ''.join(chars) + code = '\n'.join(the_lines) + + # Check if it works + try: + codeop.compile_command(code, symbol="exec", flags=codeop.PyCF_ONLY_AST) + except SyntaxError: + continue + + # Keep token.line but handle offsets correctly + self.text = token.line + self.offset = token.start[1] + 1 + self.end_offset = token.end[1] + 1 + self.lineno = start[0] + self.end_lineno = end[0] + self.msg = f"invalid syntax. Did you mean '{suggestion}'?" + return + + + def _format_syntax_error(self, stype, **kwargs): """Format SyntaxError exceptions (internal helper).""" # Show exactly where the problem was found. + colorize = kwargs.get("colorize", False) + if colorize: + theme = _colorize.get_theme(force_color=True).traceback + else: + theme = _colorize.get_theme(force_no_color=True).traceback filename_suffix = '' if self.lineno is not None: - yield ' File "{}", line {}\n'.format( - self.filename or "<string>", self.lineno) + yield ' File {}"{}"{}, line {}{}{}\n'.format( + theme.filename, + self.filename or "<string>", + theme.reset, + theme.line_no, + self.lineno, + theme.reset, + ) elif self.filename is not None: filename_suffix = ' ({})'.format(self.filename) text = self.text - if text is not None: + if isinstance(text, str): # text = " foo\n" # rtext = " foo" # ltext = "foo" + with suppress(Exception): + self._find_keyword_typos() + text = self.text rtext = text.rstrip('\n') ltext = rtext.lstrip(' \n\f') spaces = len(rtext) - len(ltext) - yield ' {}\n'.format(ltext) - - if self.offset is not None: + if self.offset is None: + yield ' {}\n'.format(ltext) + elif isinstance(self.offset, int): offset = self.offset - end_offset = self.end_offset if self.end_offset not in {None, 0} else offset - if offset == end_offset or end_offset == -1: + if self.lineno == self.end_lineno: + end_offset = ( + self.end_offset + if ( + isinstance(self.end_offset, int) + and self.end_offset != 0 + ) + else offset + ) + else: + end_offset = len(rtext) + 1 + + if self.text and offset > len(self.text): + offset = len(rtext) + 1 + if self.text and end_offset > len(self.text): + end_offset = len(rtext) + 1 + if offset >= end_offset or end_offset < 0: end_offset = offset + 1 # Convert 1-based column offset to 0-based index into stripped text colno = offset - 1 - spaces end_colno = end_offset - 1 - spaces + caretspace = ' ' if colno >= 0: # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno]) - yield ' {}{}'.format("".join(caretspace), ('^' * (end_colno - colno) + "\n")) + start_color = end_color = "" + if colorize: + # colorize from colno to end_colno + ltext = ( + ltext[:colno] + + theme.error_highlight + ltext[colno:end_colno] + theme.reset + + ltext[end_colno:] + ) + start_color = theme.error_highlight + end_color = theme.reset + yield ' {}\n'.format(ltext) + yield ' {}{}{}{}\n'.format( + "".join(caretspace), + start_color, + ('^' * (end_colno - colno)), + end_color, + ) + else: + yield ' {}\n'.format(ltext) msg = self.msg or "<no detail available>" - yield "{}: {}{}\n".format(stype, msg, filename_suffix) - - def format(self, *, chain=True): + yield "{}{}{}: {}{}{}{}\n".format( + theme.type, + stype, + theme.reset, + theme.message, + msg, + theme.reset, + filename_suffix, + ) + + def format(self, *, chain=True, _ctx=None, **kwargs): """Format the exception. If chain is not *True*, *__cause__* and *__context__* will not be formatted. @@ -661,11 +1477,14 @@ def format(self, *, chain=True): The message indicating which exception occurred is always the last string in the output. """ + colorize = kwargs.get("colorize", False) + if _ctx is None: + _ctx = _ExceptionPrintContext() output = [] exc = self - while exc: - if chain: + if chain: + while exc: if exc.__cause__ is not None: chained_msg = _cause_message chained_exc = exc.__cause__ @@ -679,14 +1498,246 @@ def format(self, *, chain=True): output.append((chained_msg, exc)) exc = chained_exc - else: - output.append((None, exc)) - exc = None + else: + output.append((None, exc)) for msg, exc in reversed(output): if msg is not None: - yield msg - if exc.stack: - yield 'Traceback (most recent call last):\n' - yield from exc.stack.format() - yield from exc.format_exception_only() + yield from _ctx.emit(msg) + if exc.exceptions is None: + if exc.stack: + yield from _ctx.emit('Traceback (most recent call last):\n') + yield from _ctx.emit(exc.stack.format(colorize=colorize)) + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) + elif _ctx.exception_group_depth > self.max_group_depth: + # exception group, but depth exceeds limit + yield from _ctx.emit( + f"... (max_group_depth is {self.max_group_depth})\n") + else: + # format exception group + is_toplevel = (_ctx.exception_group_depth == 0) + if is_toplevel: + _ctx.exception_group_depth += 1 + + if exc.stack: + yield from _ctx.emit( + 'Exception Group Traceback (most recent call last):\n', + margin_char = '+' if is_toplevel else None) + yield from _ctx.emit(exc.stack.format(colorize=colorize)) + + yield from _ctx.emit(exc.format_exception_only(colorize=colorize)) + num_excs = len(exc.exceptions) + if num_excs <= self.max_group_width: + n = num_excs + else: + n = self.max_group_width + 1 + _ctx.need_close = False + for i in range(n): + last_exc = (i == n-1) + if last_exc: + # The closing frame may be added by a recursive call + _ctx.need_close = True + + if self.max_group_width is not None: + truncated = (i >= self.max_group_width) + else: + truncated = False + title = f'{i+1}' if not truncated else '...' + yield (_ctx.indent() + + ('+-' if i==0 else ' ') + + f'+---------------- {title} ----------------\n') + _ctx.exception_group_depth += 1 + if not truncated: + yield from exc.exceptions[i].format(chain=chain, _ctx=_ctx, colorize=colorize) + else: + remaining = num_excs - self.max_group_width + plural = 's' if remaining > 1 else '' + yield from _ctx.emit( + f"and {remaining} more exception{plural}\n") + + if last_exc and _ctx.need_close: + yield (_ctx.indent() + + "+------------------------------------\n") + _ctx.need_close = False + _ctx.exception_group_depth -= 1 + + if is_toplevel: + assert _ctx.exception_group_depth == 1 + _ctx.exception_group_depth = 0 + + + def print(self, *, file=None, chain=True, **kwargs): + """Print the result of self.format(chain=chain) to 'file'.""" + colorize = kwargs.get("colorize", False) + if file is None: + file = sys.stderr + for line in self.format(chain=chain, colorize=colorize): + print(line, file=file, end="") + + +_MAX_CANDIDATE_ITEMS = 750 +_MAX_STRING_SIZE = 40 +_MOVE_COST = 2 +_CASE_COST = 1 + + +def _substitution_cost(ch_a, ch_b): + if ch_a == ch_b: + return 0 + if ch_a.lower() == ch_b.lower(): + return _CASE_COST + return _MOVE_COST + + +def _compute_suggestion_error(exc_value, tb, wrong_name): + if wrong_name is None or not isinstance(wrong_name, str): + return None + if isinstance(exc_value, AttributeError): + obj = exc_value.obj + try: + try: + d = dir(obj) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(obj.__class__.__dict__.keys()) + list(obj.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) + hide_underscored = (wrong_name[:1] != '_') + if hide_underscored and tb is not None: + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + if 'self' in frame.f_locals and frame.f_locals['self'] is obj: + hide_underscored = False + if hide_underscored: + d = [x for x in d if x[:1] != '_'] + except Exception: + return None + elif isinstance(exc_value, ImportError): + try: + mod = __import__(exc_value.name) + try: + d = dir(mod) + except TypeError: # Attributes are unsortable, e.g. int and str + d = list(mod.__dict__.keys()) + d = sorted([x for x in d if isinstance(x, str)]) + if wrong_name[:1] != '_': + d = [x for x in d if x[:1] != '_'] + except Exception: + return None + else: + assert isinstance(exc_value, NameError) + # find most recent frame + if tb is None: + return None + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + d = ( + list(frame.f_locals) + + list(frame.f_globals) + + list(frame.f_builtins) + ) + d = [x for x in d if isinstance(x, str)] + + # Check first if we are in a method and the instance + # has the wrong name as attribute + if 'self' in frame.f_locals: + self = frame.f_locals['self'] + try: + has_wrong_name = hasattr(self, wrong_name) + except Exception: + has_wrong_name = False + if has_wrong_name: + return f"self.{wrong_name}" + + try: + import _suggestions + except ImportError: + pass + else: + return _suggestions._generate_suggestions(d, wrong_name) + + # Compute closest match + + if len(d) > _MAX_CANDIDATE_ITEMS: + return None + wrong_name_len = len(wrong_name) + if wrong_name_len > _MAX_STRING_SIZE: + return None + best_distance = wrong_name_len + suggestion = None + for possible_name in d: + if possible_name == wrong_name: + # A missing attribute is "found". Don't suggest it (see GH-88821). + continue + # No more than 1/3 of the involved characters should need changed. + max_distance = (len(possible_name) + wrong_name_len + 3) * _MOVE_COST // 6 + # Don't take matches we've already beaten. + max_distance = min(max_distance, best_distance - 1) + current_distance = _levenshtein_distance(wrong_name, possible_name, max_distance) + if current_distance > max_distance: + continue + if not suggestion or current_distance < best_distance: + suggestion = possible_name + best_distance = current_distance + return suggestion + + +def _levenshtein_distance(a, b, max_cost): + # A Python implementation of Python/suggestions.c:levenshtein_distance. + + # Both strings are the same + if a == b: + return 0 + + # Trim away common affixes + pre = 0 + while a[pre:] and b[pre:] and a[pre] == b[pre]: + pre += 1 + a = a[pre:] + b = b[pre:] + post = 0 + while a[:post or None] and b[:post or None] and a[post-1] == b[post-1]: + post -= 1 + a = a[:post or None] + b = b[:post or None] + if not a or not b: + return _MOVE_COST * (len(a) + len(b)) + if len(a) > _MAX_STRING_SIZE or len(b) > _MAX_STRING_SIZE: + return max_cost + 1 + + # Prefer shorter buffer + if len(b) < len(a): + a, b = b, a + + # Quick fail when a match is impossible + if (len(b) - len(a)) * _MOVE_COST > max_cost: + return max_cost + 1 + + # Instead of producing the whole traditional len(a)-by-len(b) + # matrix, we can update just one row in place. + # Initialize the buffer row + row = list(range(_MOVE_COST, _MOVE_COST * (len(a) + 1), _MOVE_COST)) + + result = 0 + for bindex in range(len(b)): + bchar = b[bindex] + distance = result = bindex * _MOVE_COST + minimum = sys.maxsize + for index in range(len(a)): + # 1) Previous distance in this row is cost(b[:b_index], a[:index]) + substitute = distance + _substitution_cost(bchar, a[index]) + # 2) cost(b[:b_index], a[:index+1]) from previous row + distance = row[index] + # 3) existing result is cost(b[:b_index+1], a[index]) + + insert_delete = min(result, distance) + _MOVE_COST + result = min(insert_delete, substitute) + + # cost(b[:b_index+1], a[:index+1]) + row[index] = result + if result < minimum: + minimum = result + if minimum > max_cost: + # Everything in this row is too big, so bail early. + return max_cost + 1 + return result diff --git a/Lib/tty.py b/Lib/tty.py index a72eb675545..5a49e040042 100644 --- a/Lib/tty.py +++ b/Lib/tty.py @@ -4,9 +4,9 @@ from termios import * -__all__ = ["setraw", "setcbreak"] +__all__ = ["cfmakeraw", "cfmakecbreak", "setraw", "setcbreak"] -# Indexes for termios list. +# Indices for termios list. IFLAG = 0 OFLAG = 1 CFLAG = 2 @@ -15,22 +15,59 @@ OSPEED = 5 CC = 6 -def setraw(fd, when=TCSAFLUSH): - """Put terminal into a raw mode.""" - mode = tcgetattr(fd) - mode[IFLAG] = mode[IFLAG] & ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON) - mode[OFLAG] = mode[OFLAG] & ~(OPOST) - mode[CFLAG] = mode[CFLAG] & ~(CSIZE | PARENB) - mode[CFLAG] = mode[CFLAG] | CS8 - mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON | IEXTEN | ISIG) +def cfmakeraw(mode): + """Make termios mode raw.""" + # Clear all POSIX.1-2017 input mode flags. + # See chapter 11 "General Terminal Interface" + # of POSIX.1-2017 Base Definitions. + mode[IFLAG] &= ~(IGNBRK | BRKINT | IGNPAR | PARMRK | INPCK | ISTRIP | + INLCR | IGNCR | ICRNL | IXON | IXANY | IXOFF) + + # Do not post-process output. + mode[OFLAG] &= ~OPOST + + # Disable parity generation and detection; clear character size mask; + # let character size be 8 bits. + mode[CFLAG] &= ~(PARENB | CSIZE) + mode[CFLAG] |= CS8 + + # Clear all POSIX.1-2017 local mode flags. + mode[LFLAG] &= ~(ECHO | ECHOE | ECHOK | ECHONL | ICANON | + IEXTEN | ISIG | NOFLSH | TOSTOP) + + # POSIX.1-2017, 11.1.7 Non-Canonical Mode Input Processing, + # Case B: MIN>0, TIME=0 + # A pending read shall block until MIN (here 1) bytes are received, + # or a signal is received. + mode[CC] = list(mode[CC]) mode[CC][VMIN] = 1 mode[CC][VTIME] = 0 - tcsetattr(fd, when, mode) -def setcbreak(fd, when=TCSAFLUSH): - """Put terminal into a cbreak mode.""" - mode = tcgetattr(fd) - mode[LFLAG] = mode[LFLAG] & ~(ECHO | ICANON) +def cfmakecbreak(mode): + """Make termios mode cbreak.""" + # Do not echo characters; disable canonical input. + mode[LFLAG] &= ~(ECHO | ICANON) + + # POSIX.1-2017, 11.1.7 Non-Canonical Mode Input Processing, + # Case B: MIN>0, TIME=0 + # A pending read shall block until MIN (here 1) bytes are received, + # or a signal is received. + mode[CC] = list(mode[CC]) mode[CC][VMIN] = 1 mode[CC][VTIME] = 0 - tcsetattr(fd, when, mode) + +def setraw(fd, when=TCSAFLUSH): + """Put terminal into raw mode.""" + mode = tcgetattr(fd) + new = list(mode) + cfmakeraw(new) + tcsetattr(fd, when, new) + return mode + +def setcbreak(fd, when=TCSAFLUSH): + """Put terminal into cbreak mode.""" + mode = tcgetattr(fd) + new = list(mode) + cfmakecbreak(new) + tcsetattr(fd, when, new) + return mode diff --git a/Lib/types.py b/Lib/types.py index 4dab6ddce0b..fa6324fb434 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -1,68 +1,79 @@ """ Define names for built-in types that aren't directly accessible as a builtin. """ -import sys # Iterators in Python aren't a matter of type but of protocol. A large # and changing number of builtin types implement *some* flavor of # iterator. Don't check the type! Use hasattr to check for both # "__iter__" and "__next__" attributes instead. -def _f(): pass -FunctionType = type(_f) -LambdaType = type(lambda: None) # Same as FunctionType -CodeType = type(_f.__code__) -MappingProxyType = type(type.__dict__) -SimpleNamespace = type(sys.implementation) +try: + from _types import * +except ImportError: + import sys -def _cell_factory(): - a = 1 - def f(): - nonlocal a - return f.__closure__[0] -CellType = type(_cell_factory()) + def _f(): pass + FunctionType = type(_f) + LambdaType = type(lambda: None) # Same as FunctionType + CodeType = type(_f.__code__) + MappingProxyType = type(type.__dict__) + SimpleNamespace = type(sys.implementation) -def _g(): - yield 1 -GeneratorType = type(_g()) + def _cell_factory(): + a = 1 + def f(): + nonlocal a + return f.__closure__[0] + CellType = type(_cell_factory()) -async def _c(): pass -_c = _c() -CoroutineType = type(_c) -_c.close() # Prevent ResourceWarning + def _g(): + yield 1 + GeneratorType = type(_g()) -async def _ag(): - yield -_ag = _ag() -AsyncGeneratorType = type(_ag) + async def _c(): pass + _c = _c() + CoroutineType = type(_c) + _c.close() # Prevent ResourceWarning -class _C: - def _m(self): pass -MethodType = type(_C()._m) + async def _ag(): + yield + _ag = _ag() + AsyncGeneratorType = type(_ag) -BuiltinFunctionType = type(len) -BuiltinMethodType = type([].append) # Same as BuiltinFunctionType + class _C: + def _m(self): pass + MethodType = type(_C()._m) -WrapperDescriptorType = type(object.__init__) -MethodWrapperType = type(object().__str__) -MethodDescriptorType = type(str.join) -ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) + BuiltinFunctionType = type(len) + BuiltinMethodType = type([].append) # Same as BuiltinFunctionType -ModuleType = type(sys) + WrapperDescriptorType = type(object.__init__) + MethodWrapperType = type(object().__str__) + MethodDescriptorType = type(str.join) + ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) -try: - raise TypeError -except TypeError: - tb = sys.exc_info()[2] - TracebackType = type(tb) - FrameType = type(tb.tb_frame) - tb = None; del tb + ModuleType = type(sys) + + try: + raise TypeError + except TypeError as exc: + TracebackType = type(exc.__traceback__) + FrameType = type(exc.__traceback__.tb_frame) + + GetSetDescriptorType = type(FunctionType.__code__) + MemberDescriptorType = type(FunctionType.__globals__) + + GenericAlias = type(list[int]) + UnionType = type(int | str) -# For Jython, the following two types are identical -GetSetDescriptorType = type(FunctionType.__code__) -MemberDescriptorType = type(FunctionType.__globals__) + EllipsisType = type(Ellipsis) + NoneType = type(None) + NotImplementedType = type(NotImplemented) -del sys, _f, _g, _C, _c, _ag # Not for export + # CapsuleType cannot be accessed from pure Python, + # so there is no fallback definition. + + del sys, _f, _g, _C, _c, _ag, _cell_factory # Not for export # Provide a PEP 3115 compliant mechanism for class creation @@ -82,7 +93,7 @@ def resolve_bases(bases): updated = False shift = 0 for i, base in enumerate(bases): - if isinstance(base, type) and not isinstance(base, GenericAlias): + if isinstance(base, type): continue if not hasattr(base, "__mro_entries__"): continue @@ -146,6 +157,35 @@ def _calculate_meta(meta, bases): "of the metaclasses of all its bases") return winner + +def get_original_bases(cls, /): + """Return the class's "original" bases prior to modification by `__mro_entries__`. + + Examples:: + + from typing import TypeVar, Generic, NamedTuple, TypedDict + + T = TypeVar("T") + class Foo(Generic[T]): ... + class Bar(Foo[int], float): ... + class Baz(list[str]): ... + Eggs = NamedTuple("Eggs", [("a", int), ("b", str)]) + Spam = TypedDict("Spam", {"a": int, "b": str}) + + assert get_original_bases(Bar) == (Foo[int], float) + assert get_original_bases(Baz) == (list[str],) + assert get_original_bases(Eggs) == (NamedTuple,) + assert get_original_bases(Spam) == (TypedDict,) + assert get_original_bases(int) == (object,) + """ + try: + return cls.__dict__.get("__orig_bases__", cls.__bases__) + except AttributeError: + raise TypeError( + f"Expected an instance of type, not {type(cls).__name__!r}" + ) from None + + class DynamicClassAttribute: """Route attribute access on a class to __getattr__. @@ -158,7 +198,7 @@ class DynamicClassAttribute: attributes on the class with the same name. (Enum used this between Python versions 3.4 - 3.9 .) - Subclass from this to use a different method of accessing virtual atributes + Subclass from this to use a different method of accessing virtual attributes and still be treated properly by the inspect module. (Enum uses this since Python 3.10 .) @@ -234,10 +274,14 @@ def gi_running(self): @property def gi_yieldfrom(self): return self.__wrapped.gi_yieldfrom + @property + def gi_suspended(self): + return self.__wrapped.gi_suspended cr_code = gi_code cr_frame = gi_frame cr_running = gi_running cr_await = gi_yieldfrom + cr_suspended = gi_suspended def __next__(self): return next(self.__wrapped) def __iter__(self): @@ -252,8 +296,7 @@ def coroutine(func): if not callable(func): raise TypeError('types.coroutine() expects a callable') - # XXX RUSTPYTHON TODO: iterable coroutine - if (False and func.__class__ is FunctionType and + if (func.__class__ is FunctionType and getattr(func, '__code__', None).__class__ is CodeType): co_flags = func.__code__.co_flags @@ -298,11 +341,4 @@ def wrapped(*args, **kwargs): return wrapped -GenericAlias = type(list[int]) -UnionType = type(int | str) - -EllipsisType = type(Ellipsis) -NoneType = type(None) -NotImplementedType = type(NotImplemented) - -__all__ = [n for n in globals() if n[:1] != '_'] +__all__ = [n for n in globals() if not n.startswith('_')] # for pydoc diff --git a/Lib/typing.py b/Lib/typing.py index 086d0f3f959..380211183a4 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1,34 +1,46 @@ """ -The typing module: Support for gradual typing as defined by PEP 484. - -At large scale, the structure of the module is following: -* Imports and exports, all public names should be explicitly added to __all__. -* Internal helper functions: these should never be used in code outside this module. -* _SpecialForm and its instances (special forms): - Any, NoReturn, ClassVar, Union, Optional, Concatenate -* Classes whose instances can be type arguments in addition to types: - ForwardRef, TypeVar and ParamSpec -* The core of internal generics API: _GenericAlias and _VariadicGenericAlias, the latter is - currently only used by Tuple and Callable. All subscripted types like X[int], Union[int, str], - etc., are instances of either of these classes. -* The public counterpart of the generics API consists of two classes: Generic and Protocol. -* Public helper functions: get_type_hints, overload, cast, no_type_check, - no_type_check_decorator. -* Generic aliases for collections.abc ABCs and few additional protocols. +The typing module: Support for gradual typing as defined by PEP 484 and subsequent PEPs. + +Among other things, the module includes the following: +* Generic, Protocol, and internal machinery to support generic aliases. + All subscripted types like X[int], Union[int, str] are generic aliases. +* Various "special forms" that have unique meanings in type annotations: + NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others. +* Classes whose instances can be type arguments to generic classes and functions: + TypeVar, ParamSpec, TypeVarTuple. +* Public helper functions: get_type_hints, overload, cast, final, and others. +* Several protocols to support duck-typing: + SupportsFloat, SupportsIndex, SupportsAbs, and others. * Special types: NewType, NamedTuple, TypedDict. -* Wrapper submodules for re and io related types. +* Deprecated aliases for builtin types and collections.abc ABCs. + +Any name not present in __all__ is an implementation detail +that may be changed without notice. Use at your own risk! """ from abc import abstractmethod, ABCMeta import collections +from collections import defaultdict import collections.abc -import contextlib +import copyreg import functools import operator -import re as stdlib_re # Avoid confusion with the re we export. import sys import types -from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias +from types import GenericAlias + +from _typing import ( + _idfunc, + TypeVar, + ParamSpec, + TypeVarTuple, + ParamSpecArgs, + ParamSpecKwargs, + TypeAliasType, + Generic, + Union, + NoDefault, +) # Please keep __all__ alphabetized within each category. __all__ = [ @@ -48,6 +60,7 @@ 'Tuple', 'Type', 'TypeVar', + 'TypeVarTuple', 'Union', # ABCs (from collections.abc). @@ -109,47 +122,72 @@ # One-off things. 'AnyStr', + 'assert_type', + 'assert_never', 'cast', + 'clear_overloads', + 'dataclass_transform', + 'evaluate_forward_ref', 'final', 'get_args', 'get_origin', + 'get_overloads', + 'get_protocol_members', 'get_type_hints', + 'is_protocol', 'is_typeddict', + 'LiteralString', + 'Never', 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoDefault', 'NoReturn', + 'NotRequired', 'overload', + 'override', 'ParamSpecArgs', 'ParamSpecKwargs', + 'ReadOnly', + 'Required', + 'reveal_type', 'runtime_checkable', + 'Self', 'Text', 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TypeIs', + 'TypeAliasType', + 'Unpack', ] -# The pseudo-submodules 're' and 'io' are part of the public -# namespace, but excluded from __all__ because they might stomp on -# legitimate imports of those modules. +class _LazyAnnotationLib: + def __getattr__(self, attr): + global _lazy_annotationlib + import annotationlib + _lazy_annotationlib = annotationlib + return getattr(annotationlib, attr) + +_lazy_annotationlib = _LazyAnnotationLib() -def _type_convert(arg, module=None, *, allow_special_forms=False): +def _type_convert(arg, module=None, *, allow_special_forms=False, owner=None): """For converting None to type(None), and strings to ForwardRef.""" if arg is None: return type(None) if isinstance(arg, str): - return ForwardRef(arg, module=module, is_class=allow_special_forms) + return _make_forward_ref(arg, module=module, is_class=allow_special_forms, owner=owner) return arg -def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False): +def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False, owner=None): """Check that the argument is a type, and return it (internal helper). As a special case, accept None and return type(None) instead. Also wrap strings into ForwardRef instances. Consider several corner cases, for example plain special forms like Union are not valid, while Union[int, str] is OK, etc. - The msg argument is a human-readable error message, e.g:: + The msg argument is a human-readable error message, e.g.:: "Union[arg, ...]: arg should be a type." @@ -161,18 +199,17 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if is_argument: invalid_generic_forms += (Final,) - arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms) + arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms, owner=owner) if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") - if arg in (Any, NoReturn, Final, TypeAlias): + if arg in (Any, LiteralString, NoReturn, Never, Self, TypeAlias): + return arg + if allow_special_forms and arg in (ClassVar, Final): return arg if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol): raise TypeError(f"Plain {arg} is not valid as type argument") - if isinstance(arg, (type, TypeVar, ForwardRef, types.UnionType, ParamSpec, - ParamSpecArgs, ParamSpecKwargs)): - return arg - if not callable(arg): + if type(arg) is tuple: raise TypeError(f"{msg} Got {arg!r:.100}.") return arg @@ -182,6 +219,30 @@ def _is_param_expr(arg): (tuple, list, ParamSpec, _ConcatenateGenericAlias)) +def _should_unflatten_callable_args(typ, args): + """Internal helper for munging collections.abc.Callable's __args__. + + The canonical representation for a Callable's __args__ flattens the + argument types, see https://github.com/python/cpython/issues/86361. + + For example:: + + >>> import collections.abc + >>> P = ParamSpec('P') + >>> collections.abc.Callable[[int, int], str].__args__ == (int, int, str) + True + >>> collections.abc.Callable[P, str].__args__ == (P, str) + True + + As a result, if we need to reconstruct the Callable from its __args__, + we need to unflatten it. + """ + return ( + typ.__origin__ is collections.abc.Callable + and not (len(args) == 2 and _is_param_expr(args[0])) + ) + + def _type_repr(obj): """Return the repr() of an object, special-casing types (internal helper). @@ -190,99 +251,119 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - if isinstance(obj, types.GenericAlias): - return repr(obj) - if isinstance(obj, type): - if obj.__module__ == 'builtins': - return obj.__qualname__ - return f'{obj.__module__}.{obj.__qualname__}' - if obj is ...: - return('...') - if isinstance(obj, types.FunctionType): - return obj.__name__ - return repr(obj) + if isinstance(obj, tuple): + # Special case for `repr` of types with `ParamSpec`: + return '[' + ', '.join(_type_repr(t) for t in obj) + ']' + return _lazy_annotationlib.type_repr(obj) + +def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): + """Collect all type parameters in args + in order of first appearance (lexicographic order). -def _collect_type_vars(types_, typevar_types=None): - """Collect all type variable contained - in types in order of first appearance (lexicographic order). For example:: + For example:: - _collect_type_vars((T, List[S, T])) == (T, S) + >>> P = ParamSpec('P') + >>> T = TypeVar('T') + >>> _collect_type_parameters((T, Callable[P, T])) + (~T, ~P) """ - if typevar_types is None: - typevar_types = TypeVar - tvars = [] - for t in types_: - if isinstance(t, typevar_types) and t not in tvars: - tvars.append(t) - if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): - tvars.extend([t for t in t.__parameters__ if t not in tvars]) - return tuple(tvars) + # required type parameter cannot appear after parameter with default + default_encountered = False + # or after TypeVarTuple + type_var_tuple_encountered = False + parameters = [] + for t in args: + if isinstance(t, type): + # We don't want __parameters__ descriptor of a bare Python class. + pass + elif isinstance(t, tuple): + # `t` might be a tuple, when `ParamSpec` is substituted with + # `[T, int]`, or `[int, *Ts]`, etc. + for x in t: + for collected in _collect_type_parameters([x]): + if collected not in parameters: + parameters.append(collected) + elif hasattr(t, '__typing_subst__'): + if t not in parameters: + if enforce_default_ordering: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') + + parameters.append(t) + else: + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True + for x in getattr(t, '__parameters__', ()): + if x not in parameters: + parameters.append(x) + return tuple(parameters) -def _check_generic(cls, parameters, elen): +def _check_generic_specialization(cls, arguments): """Check correct count for parameters of a generic cls (internal helper). + This gives a nice error message in case of count mismatch. """ - if not elen: + expected_len = len(cls.__parameters__) + if not expected_len: raise TypeError(f"{cls} is not a generic class") - alen = len(parameters) - if alen != elen: - raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" - f" actual {alen}, expected {elen}") - -def _prepare_paramspec_params(cls, params): - """Prepares the parameters for a Generic containing ParamSpec - variables (internal helper). - """ - # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. - if (len(cls.__parameters__) == 1 - and params and not _is_param_expr(params[0])): - assert isinstance(cls.__parameters__[0], ParamSpec) - return (params,) - else: - _check_generic(cls, params, len(cls.__parameters__)) - _params = [] - # Convert lists to tuples to help other libraries cache the results. - for p, tvar in zip(params, cls.__parameters__): - if isinstance(tvar, ParamSpec) and isinstance(p, list): - p = tuple(p) - _params.append(p) - return tuple(_params) - -def _deduplicate(params): - # Weed out strict duplicates, preserving the first of each occurrence. - all_params = set(params) - if len(all_params) < len(params): - new_params = [] - for t in params: - if t in all_params: - new_params.append(t) - all_params.remove(t) - params = new_params - assert not all_params, all_params - return params + actual_len = len(arguments) + if actual_len != expected_len: + # deal with defaults + if actual_len < expected_len: + # If the parameter at index `actual_len` in the parameters list + # has a default, then all parameters after it must also have + # one, because we validated as much in _collect_type_parameters(). + # That means that no error needs to be raised here, despite + # the number of arguments being passed not matching the number + # of parameters: all parameters that aren't explicitly + # specialized in this call are parameters with default values. + if cls.__parameters__[actual_len].has_default(): + return + + expected_len -= sum(p.has_default() for p in cls.__parameters__) + expect_val = f"at least {expected_len}" + else: + expect_val = expected_len + raise TypeError(f"Too {'many' if actual_len > expected_len else 'few'} arguments" + f" for {cls}; actual {actual_len}, expected {expect_val}") -def _remove_dups_flatten(parameters): - """An internal helper for Union creation and substitution: flatten Unions - among parameters, then remove duplicates. - """ - # Flatten out Union[Union[...], ...]. - params = [] - for p in parameters: - if isinstance(p, (_UnionGenericAlias, types.UnionType)): - params.extend(p.__args__) - elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: - params.extend(p[1:]) - else: - params.append(p) - return tuple(_deduplicate(params)) +def _unpack_args(*args): + newargs = [] + for arg in args: + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs is not None and not (subargs and subargs[-1] is ...): + newargs.extend(subargs) + else: + newargs.append(arg) + return newargs +def _deduplicate(params, *, unhashable_fallback=False): + # Weed out strict duplicates, preserving the first of each occurrence. + try: + return dict.fromkeys(params) + except TypeError: + if not unhashable_fallback: + raise + # Happens for cases like `Annotated[dict, {'x': IntValidator()}]` + new_unhashable = [] + for t in params: + if t not in new_unhashable: + new_unhashable.append(t) + return new_unhashable def _flatten_literal_params(parameters): - """An internal helper for Literal creation: flatten Literals among parameters""" + """Internal helper for Literal creation: flatten Literals among parameters.""" params = [] for p in parameters: if isinstance(p, _LiteralGenericAlias): @@ -293,20 +374,29 @@ def _flatten_literal_params(parameters): _cleanups = [] +_caches = {} def _tp_cache(func=None, /, *, typed=False): - """Internal wrapper caching __getitem__ of generic types with a fallback to - original function for non-hashable arguments. + """Internal wrapper caching __getitem__ of generic types. + + For non-hashable arguments, the original function is used as a fallback. """ def decorator(func): - cached = functools.lru_cache(typed=typed)(func) - _cleanups.append(cached.cache_clear) + # The callback 'inner' references the newly created lru_cache + # indirectly by performing a lookup in the global '_caches' dictionary. + # This breaks a reference that can be problematic when combined with + # C API extensions that leak references to types. See GH-98253. + + cache = functools.lru_cache(typed=typed)(func) + _caches[func] = cache + _cleanups.append(cache.cache_clear) + del cache @functools.wraps(func) def inner(*args, **kwds): try: - return cached(*args, **kwds) + return _caches[func](*args, **kwds) except TypeError: pass # All real errors (not unhashable args) are raised below. return func(*args, **kwds) @@ -317,21 +407,86 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, recursive_guard=frozenset()): + +def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: + import warnings + + depr_message = ( + f"Failing to pass a value to the 'type_params' parameter " + f"of {funcname!r} is deprecated, as it leads to incorrect behaviour " + f"when calling {funcname} on a stringified annotation " + f"that references a PEP 695 type parameter. " + f"It will be disallowed in Python 3.15." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) + + +class _Sentinel: + __slots__ = () + def __repr__(self): + return '<sentinel>' + + +_sentinel = _Sentinel() + + +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), + format=None, owner=None, parent_fwdref=None, prefer_fwd_module=False): """Evaluate all forward references in the given type t. + For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ - if isinstance(t, ForwardRef): - return t._evaluate(globalns, localns, recursive_guard) - if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): - ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__) + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing._eval_type") + type_params = () + if isinstance(t, _lazy_annotationlib.ForwardRef): + # If the forward_ref has __forward_module__ set, evaluate() infers the globals + # from the module, and it will probably pick better than the globals we have here. + # We do this only for calls from get_type_hints() (which opts in through the + # prefer_fwd_module flag), so that the default behavior remains more straightforward. + if prefer_fwd_module and t.__forward_module__ is not None: + globalns = None + # If there are type params on the owner, we need to add them back, because + # annotationlib won't. + if owner_type_params := getattr(owner, "__type_params__", None): + globalns = getattr( + sys.modules.get(t.__forward_module__, None), "__dict__", None + ) + if globalns is not None: + globalns = dict(globalns) + for type_param in owner_type_params: + globalns[type_param.__name__] = type_param + return evaluate_forward_ref(t, globals=globalns, locals=localns, + type_params=type_params, owner=owner, + _recursive_guard=recursive_guard, format=format) + if isinstance(t, (_GenericAlias, GenericAlias, Union)): + if isinstance(t, GenericAlias): + args = tuple( + _make_forward_ref(arg, parent_fwdref=parent_fwdref) if isinstance(arg, str) else arg + for arg in t.__args__ + ) + is_unpacked = t.__unpacked__ + if _should_unflatten_callable_args(t, args): + t = t.__origin__[(args[:-1], args[-1])] + else: + t = t.__origin__[args] + if is_unpacked: + t = Unpack[t] + + ev_args = tuple( + _eval_type( + a, globalns, localns, type_params, recursive_guard=recursive_guard, + format=format, owner=owner, prefer_fwd_module=prefer_fwd_module, + ) + for a in t.__args__ + ) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): return GenericAlias(t.__origin__, ev_args) - if isinstance(t, types.UnionType): + if isinstance(t, Union): return functools.reduce(operator.or_, ev_args) else: return t.copy_with(ev_args) @@ -339,28 +494,36 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): class _Final: - """Mixin to prohibit subclassing""" + """Mixin to prohibit subclassing.""" __slots__ = ('__weakref__',) - def __init_subclass__(self, /, *args, **kwds): + def __init_subclass__(cls, /, *args, **kwds): if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") -class _Immutable: - """Mixin to indicate that object should not be copied.""" - __slots__ = () - def __copy__(self): - return self +class _NotIterable: + """Mixin to prevent iteration, without being compatible with Iterable. + + That is, we could do:: - def __deepcopy__(self, memo): - return self + def __iter__(self): raise TypeError() + + But this would make users of this mixin duck type-compatible with + collections.abc.Iterable - isinstance(foo, Iterable) would be True. + + Luckily, we can instead prevent iteration by setting __iter__ to None, which + is treated specially. + """ + + __slots__ = () + __iter__ = None # Internal indicator of special typing constructs. # See __doc__ instance attribute for specific docs. -class _SpecialForm(_Final, _root=True): +class _SpecialForm(_Final, _NotIterable, _root=True): __slots__ = ('_name', '__doc__', '_getitem') def __init__(self, getitem): @@ -403,15 +566,26 @@ def __getitem__(self, parameters): return self._getitem(self, parameters) -class _LiteralSpecialForm(_SpecialForm, _root=True): +class _TypedCacheSpecialForm(_SpecialForm, _root=True): def __getitem__(self, parameters): if not isinstance(parameters, tuple): parameters = (parameters,) return self._getitem(self, *parameters) -@_SpecialForm -def Any(self, parameters): +class _AnyMeta(type): + def __instancecheck__(self, obj): + if self is Any: + raise TypeError("typing.Any cannot be used with isinstance()") + return super().__instancecheck__(obj) + + def __repr__(self): + if self is Any: + return "typing.Any" + return super().__repr__() # respect to subclasses + + +class Any(metaclass=_AnyMeta): """Special type indicating an unconstrained type. - Any is compatible with every type. @@ -420,43 +594,128 @@ def Any(self, parameters): Note that all the above statements are true from the point of view of static type checkers. At runtime, Any should not be used with instance - or class checks. + checks. """ - raise TypeError(f"{self} is not subscriptable") + + def __new__(cls, *args, **kwargs): + if cls is Any: + raise TypeError("Any cannot be instantiated") + return super().__new__(cls) + @_SpecialForm def NoReturn(self, parameters): """Special type indicating functions that never return. + + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + NoReturn can also be used as a bottom type, a type that + has no values. Starting in Python 3.11, the Never type should + be used for this concept instead. Type checkers should treat the two + equivalently. + """ + raise TypeError(f"{self} is not subscriptable") + +# This is semantically identical to NoReturn, but it is implemented +# separately so that type checkers can distinguish between the two +# if they want. +@_SpecialForm +def Never(self, parameters): + """The bottom type, a type that has no members. + + This can be used to define a function that should never be + called, or a function that never returns:: + + from typing import Never + + def never_call_me(arg: Never) -> None: + pass + + def int_or_str(arg: int | str) -> None: + never_call_me(arg) # type checker error + match arg: + case int(): + print("It's an int") + case str(): + print("It's a str") + case _: + never_call_me(arg) # OK, arg is of type Never + """ + raise TypeError(f"{self} is not subscriptable") + + +@_SpecialForm +def Self(self, parameters): + """Used to spell the type of "self" in classes. + Example:: - from typing import NoReturn + from typing import Self - def stop() -> NoReturn: - raise Exception('no way') + class Foo: + def return_self(self) -> Self: + ... + return self + + This is especially useful for: + - classmethods that are used as alternative constructors + - annotating an `__enter__` method which returns self + """ + raise TypeError(f"{self} is not subscriptable") + + +@_SpecialForm +def LiteralString(self, parameters): + """Represents an arbitrary literal string. + + Example:: + + from typing import LiteralString + + def run_query(sql: LiteralString) -> None: + ... - This type is invalid in other positions, e.g., ``List[NoReturn]`` - will fail in static type checkers. + def caller(arbitrary_string: str, literal_string: LiteralString) -> None: + run_query("SELECT * FROM students") # OK + run_query(literal_string) # OK + run_query("SELECT * FROM " + literal_string) # OK + run_query(arbitrary_string) # type checker error + run_query( # type checker error + f"SELECT * FROM students WHERE name = {arbitrary_string}" + ) + + Only string literals and other LiteralStrings are compatible + with LiteralString. This provides a tool to help prevent + security issues such as SQL injection. """ raise TypeError(f"{self} is not subscriptable") + @_SpecialForm def ClassVar(self, parameters): """Special type construct to mark class variables. An annotation wrapped in ClassVar indicates that a given attribute is intended to be used as a class variable and - should not be set on instances of that class. Usage:: + should not be set on instances of that class. - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable + Usage:: + + class Starship: + stats: ClassVar[dict[str, int]] = {} # class variable + damage: int = 10 # instance variable ClassVar accepts only types and cannot be further subscribed. Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -464,89 +723,47 @@ def Final(self, parameters): """Special typing construct to indicate final names to type checkers. A final name cannot be re-assigned or overridden in a subclass. - For example: - MAX_SIZE: Final = 9000 - MAX_SIZE += 1 # Error reported by type checker + For example:: + + MAX_SIZE: Final = 9000 + MAX_SIZE += 1 # Error reported by type checker - class Connection: - TIMEOUT: Final[int] = 10 + class Connection: + TIMEOUT: Final[int] = 10 - class FastConnector(Connection): - TIMEOUT = 1 # Error reported by type checker + class FastConnector(Connection): + TIMEOUT = 1 # Error reported by type checker There is no runtime checking of these properties. """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) -@_SpecialForm -def Union(self, parameters): - """Union type; Union[X, Y] means either X or Y. - - To define a union, use e.g. Union[int, str]. Details: - - The arguments must be types and there must be at least one. - - None as an argument is a special case and is replaced by - type(None). - - Unions of unions are flattened, e.g.:: - - Union[Union[int, str], float] == Union[int, str, float] - - - Unions of a single argument vanish, e.g.:: - - Union[int] == int # The constructor actually returns int - - - Redundant arguments are skipped, e.g.:: - - Union[int, str, int] == Union[int, str] - - - When comparing unions, the argument order is ignored, e.g.:: - - Union[int, str] == Union[str, int] - - - You cannot subclass or instantiate a union. - - You can use Optional[X] as a shorthand for Union[X, None]. - """ - if parameters == (): - raise TypeError("Cannot take a Union of no types.") - if not isinstance(parameters, tuple): - parameters = (parameters,) - msg = "Union[arg, ...]: each arg must be a type." - parameters = tuple(_type_check(p, msg) for p in parameters) - parameters = _remove_dups_flatten(parameters) - if len(parameters) == 1: - return parameters[0] - if len(parameters) == 2 and type(None) in parameters: - return _UnionGenericAlias(self, parameters, name="Optional") - return _UnionGenericAlias(self, parameters) - @_SpecialForm def Optional(self, parameters): - """Optional type. - - Optional[X] is equivalent to Union[X, None]. - """ + """Optional[X] is equivalent to Union[X, None].""" arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] -@_LiteralSpecialForm +@_TypedCacheSpecialForm @_tp_cache(typed=True) def Literal(self, *parameters): """Special typing form to define literal types (a.k.a. value types). This form can be used to indicate to type checkers that the corresponding variable or function parameter has a value equivalent to the provided - literal (or one of several literals): + literal (or one of several literals):: - def validate_simple(data: Any) -> Literal[True]: # always returns True - ... + def validate_simple(data: Any) -> Literal[True]: # always returns True + ... - MODE = Literal['r', 'rb', 'w', 'wb'] - def open_helper(file: str, mode: MODE) -> str: - ... + MODE = Literal['r', 'rb', 'w', 'wb'] + def open_helper(file: str, mode: MODE) -> str: + ... - open_helper('/some/path', 'r') # Passes type check - open_helper('/other/path', 'typo') # Error in type checker + open_helper('/some/path', 'r') # Passes type check + open_helper('/other/path', 'typo') # Error in type checker Literal[...] cannot be subclassed. At runtime, an arbitrary value is allowed as type argument to Literal[...], but type checkers may @@ -566,7 +783,9 @@ def open_helper(file: str, mode: MODE) -> str: @_SpecialForm def TypeAlias(self, parameters): - """Special marker indicating that an assignment should + """Special form for marking type aliases. + + Use TypeAlias to indicate that an assignment should be recognized as a proper type alias definition by type checkers. @@ -581,13 +800,15 @@ def TypeAlias(self, parameters): @_SpecialForm def Concatenate(self, parameters): - """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a - higher order function which adds, removes or transforms parameters of a - callable. + """Special form for annotating higher-order functions. + + ``Concatenate`` can be used in conjunction with ``ParamSpec`` and + ``Callable`` to represent a higher-order function which adds, removes or + transforms the parameters of a callable. For example:: - Callable[Concatenate[int, P], int] + Callable[Concatenate[int, P], int] See PEP 612 for detailed information. """ @@ -595,56 +816,62 @@ def Concatenate(self, parameters): raise TypeError("Cannot take a Concatenate of no types.") if not isinstance(parameters, tuple): parameters = (parameters,) - if not isinstance(parameters[-1], ParamSpec): + if not (parameters[-1] is ... or isinstance(parameters[-1], ParamSpec)): raise TypeError("The last parameter to Concatenate should be a " - "ParamSpec variable.") + "ParamSpec variable or ellipsis.") msg = "Concatenate[arg, ...]: each arg must be a type." parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1]) - return _ConcatenateGenericAlias(self, parameters, - _typevar_types=(TypeVar, ParamSpec), - _paramspec_tvars=True) + return _ConcatenateGenericAlias(self, parameters) @_SpecialForm def TypeGuard(self, parameters): - """Special typing form used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. + """Special typing construct for marking user-defined type predicate functions. + + ``TypeGuard`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeGuard`` only accepts a single type argument. At runtime, functions marked this way should return a boolean. ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard". + conditional expression here is sometimes referred to as a "type predicate". Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeGuard[...]`` or + ``TypeIs[...]`` as its return type to alert static type checkers to + this intention. ``TypeGuard`` should be used over ``TypeIs`` when narrowing + from an incompatible type (e.g., ``list[object]`` to ``list[int]``) or when + the function does not return ``True`` for all instances of the narrowed type. - Using ``-> TypeGuard`` tells the static type checker that for a given - function: + Using ``-> TypeGuard[NarrowedType]`` tells the static type checker that + for a given function: 1. The return value is a boolean. 2. If the return value is ``True``, the type of its argument - is the type inside ``TypeGuard``. + is ``NarrowedType``. + + For example:: - For example:: + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: + '''Determines whether all objects in the list are strings''' + return all(isinstance(x, str) for x in val) - def is_str(val: Union[str, float]): - # "isinstance" type guard - if isinstance(val, str): - # Type of ``val`` is narrowed to ``str`` - ... - else: - # Else, type of ``val`` is narrowed to ``float``. - ... + def func1(val: list[object]): + if is_str_list(val): + # Type of ``val`` is narrowed to ``list[str]``. + print(" ".join(val)) + else: + # Type of ``val`` remains as ``list[object]``. + print("Not a list of strings!") Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower form of ``TypeA`` (it can even be a wider form) and this may lead to type-unsafe results. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter is not - a subtype of the former, since ``List`` is invariant. The responsibility of - writing type-safe type guards is left to the user. + narrowing ``list[object]`` to ``list[str]`` even though the latter is not + a subtype of the former, since ``list`` is invariant. The responsibility of + writing type-safe type predicates is left to the user. ``TypeGuard`` also works with type variables. For more information, see PEP 647 (User-Defined Type Guards). @@ -653,297 +880,348 @@ def is_str(val: Union[str, float]): return _GenericAlias(self, (item,)) -class ForwardRef(_Final, _root=True): - """Internal wrapper to hold a forward reference.""" - - __slots__ = ('__forward_arg__', '__forward_code__', - '__forward_evaluated__', '__forward_value__', - '__forward_is_argument__', '__forward_is_class__', - '__forward_module__') - - def __init__(self, arg, is_argument=True, module=None, *, is_class=False): - if not isinstance(arg, str): - raise TypeError(f"Forward reference must be a string -- got {arg!r}") - try: - code = compile(arg, '<string>', 'eval') - except SyntaxError: - raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") - self.__forward_arg__ = arg - self.__forward_code__ = code - self.__forward_evaluated__ = False - self.__forward_value__ = None - self.__forward_is_argument__ = is_argument - self.__forward_is_class__ = is_class - self.__forward_module__ = module - - def _evaluate(self, globalns, localns, recursive_guard): - if self.__forward_arg__ in recursive_guard: - return self - if not self.__forward_evaluated__ or localns is not globalns: - if globalns is None and localns is None: - globalns = localns = {} - elif globalns is None: - globalns = localns - elif localns is None: - localns = globalns - if self.__forward_module__ is not None: - globalns = getattr( - sys.modules.get(self.__forward_module__, None), '__dict__', globalns - ) - type_ = _type_check( - eval(self.__forward_code__, globalns, localns), - "Forward references must evaluate to types.", - is_argument=self.__forward_is_argument__, - allow_special_forms=self.__forward_is_class__, - ) - self.__forward_value__ = _eval_type( - type_, globalns, localns, recursive_guard | {self.__forward_arg__} - ) - self.__forward_evaluated__ = True - return self.__forward_value__ - - def __eq__(self, other): - if not isinstance(other, ForwardRef): - return NotImplemented - if self.__forward_evaluated__ and other.__forward_evaluated__: - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_module__ == other.__forward_module__) - - def __hash__(self): - return hash((self.__forward_arg__, self.__forward_module__)) - - def __repr__(self): - return f'ForwardRef({self.__forward_arg__!r})' - -class _TypeVarLike: - """Mixin for TypeVar-like types (TypeVar and ParamSpec).""" - def __init__(self, bound, covariant, contravariant): - """Used to setup TypeVars and ParamSpec's bound, covariant and - contravariant attributes. - """ - if covariant and contravariant: - raise ValueError("Bivariant types are not supported.") - self.__covariant__ = bool(covariant) - self.__contravariant__ = bool(contravariant) - if bound: - self.__bound__ = _type_check(bound, "Bound must be a type.") - else: - self.__bound__ = None - - def __or__(self, right): - return Union[self, right] - - def __ror__(self, left): - return Union[left, self] - - def __repr__(self): - if self.__covariant__: - prefix = '+' - elif self.__contravariant__: - prefix = '-' - else: - prefix = '~' - return prefix + self.__name__ - - def __reduce__(self): - return self.__name__ - - -class TypeVar( _Final, _Immutable, _TypeVarLike, _root=True): - """Type variable. - - Usage:: +@_SpecialForm +def TypeIs(self, parameters): + """Special typing construct for marking user-defined type predicate functions. - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and accept + at least one argument. - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows: + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type predicate". - def repeat(x: T, n: int) -> List[T]: - '''Return a list containing n references to x.''' - return [x]*n + Sometimes it would be convenient to use a user-defined boolean function + as a type predicate. Such a function should use ``TypeIs[...]`` or + ``TypeGuard[...]`` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. - def longest(x: A, y: A) -> A: - '''Return the longest of two strings.''' - return x if len(x) >= len(y) else y + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for + a given function: - The latter example's signature is essentially the overloading - of (str, str) -> str and (bytes, bytes) -> bytes. Also note - that if the arguments are instances of some subclass of str, - the return type is still plain str. + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and + ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. - At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError. + For example:: - Type variables defined with covariant=True or contravariant=True - can be used to declare covariant or contravariant generic types. - See PEP 484 for more details. By default generic types are invariant - in all type variables. + from typing import assert_type, final, TypeIs - Type variables can be introspected. e.g.: + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass - T.__name__ == 'T' - T.__constraints__ == () - T.__covariant__ == False - T.__contravariant__ = False - A.__constraints__ == (str, bytes) + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) - Note that only type variables defined in global scope can be pickled. + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + ``TypeIs`` also works with type variables. For more information, see + PEP 742 (Narrowing types with ``TypeIs``). """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _GenericAlias(self, (item,)) - __slots__ = ('__name__', '__bound__', '__constraints__', - '__covariant__', '__contravariant__', '__dict__') - - def __init__(self, name, *constraints, bound=None, - covariant=False, contravariant=False): - self.__name__ = name - super().__init__(bound, covariant, contravariant) - if constraints and bound is not None: - raise TypeError("Constraints cannot be combined with bound=...") - if constraints and len(constraints) == 1: - raise TypeError("A single constraint is not allowed") - msg = "TypeVar(name, constraint, ...): constraints must be types." - self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling - except (AttributeError, ValueError): - def_mod = None - if def_mod != 'typing': - self.__module__ = def_mod - - -class ParamSpecArgs(_Final, _Immutable, _root=True): - """The args for a ParamSpec object. - - Given a ParamSpec object P, P.args is an instance of ParamSpecArgs. - - ParamSpecArgs objects have a reference back to their ParamSpec: - P.args.__origin__ is P +def _make_forward_ref(code, *, parent_fwdref=None, **kwargs): + if parent_fwdref is not None: + if parent_fwdref.__forward_module__ is not None: + kwargs['module'] = parent_fwdref.__forward_module__ + if parent_fwdref.__owner__ is not None: + kwargs['owner'] = parent_fwdref.__owner__ + forward_ref = _lazy_annotationlib.ForwardRef(code, **kwargs) + # For compatibility, eagerly compile the forwardref's code. + forward_ref.__forward_code__ + return forward_ref + + +def evaluate_forward_ref( + forward_ref, + *, + owner=None, + globals=None, + locals=None, + type_params=None, + format=None, + _recursive_guard=frozenset(), +): + """Evaluate a forward reference as a type hint. + + This is similar to calling the ForwardRef.evaluate() method, + but unlike that method, evaluate_forward_ref() also + recursively evaluates forward references nested within the type hint. + + *forward_ref* must be an instance of ForwardRef. *owner*, if given, + should be the object that holds the annotations that the forward reference + derived from, such as a module, class object, or function. It is used to + infer the namespaces to use for looking up names. *globals* and *locals* + can also be explicitly given to provide the global and local namespaces. + *type_params* is a tuple of type parameters that are in scope when + evaluating the forward reference. This parameter should be provided (though + it may be an empty tuple) if *owner* is not given and the forward reference + does not already have an owner set. *format* specifies the format of the + annotation and is a member of the annotationlib.Format enum, defaulting to + VALUE. - This type is meant for runtime introspection and has no special meaning to - static type checkers. """ - def __init__(self, origin): - self.__origin__ = origin + if format == _lazy_annotationlib.Format.STRING: + return forward_ref.__forward_arg__ + if forward_ref.__forward_arg__ in _recursive_guard: + return forward_ref + + if format is None: + format = _lazy_annotationlib.Format.VALUE + value = forward_ref.evaluate(globals=globals, locals=locals, + type_params=type_params, owner=owner, format=format) + + if (isinstance(value, _lazy_annotationlib.ForwardRef) + and format == _lazy_annotationlib.Format.FORWARDREF): + return value + + if isinstance(value, str): + value = _make_forward_ref(value, module=forward_ref.__forward_module__, + owner=owner or forward_ref.__owner__, + is_argument=forward_ref.__forward_is_argument__, + is_class=forward_ref.__forward_is_class__) + if owner is None: + owner = forward_ref.__owner__ + return _eval_type( + value, + globals, + locals, + type_params, + recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, + format=format, + owner=owner, + parent_fwdref=forward_ref, + ) + + +def _is_unpacked_typevartuple(x: Any) -> bool: + # Need to check 'is True' here + # See: https://github.com/python/cpython/issues/137706 + return ((not isinstance(x, type)) and + getattr(x, '__typing_is_unpacked_typevartuple__', False) is True) + + +def _is_typevar_like(x: Any) -> bool: + return isinstance(x, (TypeVar, ParamSpec)) or _is_unpacked_typevartuple(x) + + +def _typevar_subst(self, arg): + msg = "Parameters to generic types must be types." + arg = _type_check(arg, msg, is_argument=True) + if ((isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack) or + (isinstance(arg, GenericAlias) and getattr(arg, '__unpacked__', False))): + raise TypeError(f"{arg} is not valid as type argument") + return arg - def __repr__(self): - return f"{self.__origin__.__name__}.args" - def __eq__(self, other): - if not isinstance(other, ParamSpecArgs): - return NotImplemented - return self.__origin__ == other.__origin__ +def _typevartuple_prepare_subst(self, alias, args): + params = alias.__parameters__ + typevartuple_index = params.index(self) + for param in params[typevartuple_index + 1:]: + if isinstance(param, TypeVarTuple): + raise TypeError(f"More than one TypeVarTuple parameter in {alias}") + + alen = len(args) + plen = len(params) + left = typevartuple_index + right = plen - typevartuple_index - 1 + var_tuple_index = None + fillarg = None + for k, arg in enumerate(args): + if not isinstance(arg, type): + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs and len(subargs) == 2 and subargs[-1] is ...: + if var_tuple_index is not None: + raise TypeError("More than one unpacked arbitrary-length tuple argument") + var_tuple_index = k + fillarg = subargs[0] + if var_tuple_index is not None: + left = min(left, var_tuple_index) + right = min(right, alen - var_tuple_index - 1) + elif left + right > alen: + raise TypeError(f"Too few arguments for {alias};" + f" actual {alen}, expected at least {plen-1}") + if left == alen - right and self.has_default(): + replacement = _unpack_args(self.__default__) + else: + replacement = args[left: alen - right] + + return ( + *args[:left], + *([fillarg]*(typevartuple_index - left)), + replacement, + *([fillarg]*(plen - right - left - typevartuple_index - 1)), + *args[alen - right:], + ) + + +def _paramspec_subst(self, arg): + if isinstance(arg, (list, tuple)): + arg = tuple(_type_check(a, "Expected a type.") for a in arg) + elif not _is_param_expr(arg): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {arg}") + return arg -class ParamSpecKwargs(_Final, _Immutable, _root=True): - """The kwargs for a ParamSpec object. +def _paramspec_prepare_subst(self, alias, args): + params = alias.__parameters__ + i = params.index(self) + if i == len(args) and self.has_default(): + args = (*args, self.__default__) + if i >= len(args): + raise TypeError(f"Too few arguments for {alias}") + # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. + if len(params) == 1 and not _is_param_expr(args[0]): + assert i == 0 + args = (args,) + # Convert lists to tuples to help other libraries cache the results. + elif isinstance(args[i], list): + args = (*args[:i], tuple(args[i]), *args[i+1:]) + return args - Given a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs. - ParamSpecKwargs objects have a reference back to their ParamSpec: +@_tp_cache +def _generic_class_getitem(cls, args): + """Parameterizes a generic class. - P.kwargs.__origin__ is P + At least, parameterizing a generic class is the *main* thing this method + does. For example, for some generic class `Foo`, this is called when we + do `Foo[int]` - there, with `cls=Foo` and `args=int`. - This type is meant for runtime introspection and has no special meaning to - static type checkers. + However, note that this method is also called when defining generic + classes in the first place with `class Foo(Generic[T]): ...`. """ - def __init__(self, origin): - self.__origin__ = origin - - def __repr__(self): - return f"{self.__origin__.__name__}.kwargs" - - def __eq__(self, other): - if not isinstance(other, ParamSpecKwargs): - return NotImplemented - return self.__origin__ == other.__origin__ - - -class ParamSpec(_Final, _Immutable, _TypeVarLike, _root=True): - """Parameter specification variable. + if not isinstance(args, tuple): + args = (args,) - Usage:: + args = tuple(_type_convert(p) for p in args) + is_generic_or_protocol = cls in (Generic, Protocol) - P = ParamSpec('P') - - Parameter specification variables exist primarily for the benefit of static - type checkers. They are used to forward the parameter types of one - callable to another callable, a pattern commonly found in higher order - functions and decorators. They are only valid when used in ``Concatenate``, - or as the first argument to ``Callable``, or as parameters for user-defined - Generics. See class Generic for more information on generic types. An - example for annotating a decorator:: - - T = TypeVar('T') - P = ParamSpec('P') - - def add_logging(f: Callable[P, T]) -> Callable[P, T]: - '''A type-safe decorator to add logging to a function.''' - def inner(*args: P.args, **kwargs: P.kwargs) -> T: - logging.info(f'{f.__name__} was called') - return f(*args, **kwargs) - return inner - - @add_logging - def add_two(x: float, y: float) -> float: - '''Add two numbers together.''' - return x + y - - Parameter specification variables defined with covariant=True or - contravariant=True can be used to declare covariant or contravariant - generic types. These keyword arguments are valid, but their actual semantics - are yet to be decided. See PEP 612 for details. - - Parameter specification variables can be introspected. e.g.: - - P.__name__ == 'T' - P.__bound__ == None - P.__covariant__ == False - P.__contravariant__ == False - - Note that only parameter specification variables defined in global scope can - be pickled. - """ + if is_generic_or_protocol: + # Generic and Protocol can only be subscripted with unique type variables. + if not args: + raise TypeError( + f"Parameter list to {cls.__qualname__}[...] cannot be empty" + ) + if not all(_is_typevar_like(p) for p in args): + raise TypeError( + f"Parameters to {cls.__name__}[...] must all be type variables " + f"or parameter specification variables.") + if len(set(args)) != len(args): + raise TypeError( + f"Parameters to {cls.__name__}[...] must all be unique") + else: + # Subscripting a regular Generic subclass. + try: + parameters = cls.__parameters__ + except AttributeError as e: + init_subclass = getattr(cls, '__init_subclass__', None) + if init_subclass not in {None, Generic.__init_subclass__}: + e.add_note( + f"Note: this exception may have been caused by " + f"{init_subclass.__qualname__!r} (or the " + f"'__init_subclass__' method on a superclass) not " + f"calling 'super().__init_subclass__()'" + ) + raise + for param in parameters: + prepare = getattr(param, '__typing_prepare_subst__', None) + if prepare is not None: + args = prepare(cls, args) + _check_generic_specialization(cls, args) - __slots__ = ('__name__', '__bound__', '__covariant__', '__contravariant__', - '__dict__') + new_args = [] + for param, new_arg in zip(parameters, args): + if isinstance(param, TypeVarTuple): + new_args.extend(new_arg) + else: + new_args.append(new_arg) + args = tuple(new_args) - @property - def args(self): - return ParamSpecArgs(self) + return _GenericAlias(cls, args) - @property - def kwargs(self): - return ParamSpecKwargs(self) - def __init__(self, name, *, bound=None, covariant=False, contravariant=False): - self.__name__ = name - super().__init__(bound, covariant, contravariant) - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - def_mod = None - if def_mod != 'typing': - self.__module__ = def_mod +def _generic_init_subclass(cls, *args, **kwargs): + super(Generic, cls).__init_subclass__(*args, **kwargs) + tvars = [] + if '__orig_bases__' in cls.__dict__: + error = Generic in cls.__orig_bases__ + else: + error = (Generic in cls.__bases__ and + cls.__name__ != 'Protocol' and + type(cls) != _TypedDictMeta) + if error: + raise TypeError("Cannot inherit from plain Generic") + if '__orig_bases__' in cls.__dict__: + tvars = _collect_type_parameters(cls.__orig_bases__) + # Look for Generic[T1, ..., Tn]. + # If found, tvars must be a subset of it. + # If not found, tvars is it. + # Also check for and reject plain Generic, + # and reject multiple Generic[...]. + gvars = None + for base in cls.__orig_bases__: + if (isinstance(base, _GenericAlias) and + base.__origin__ is Generic): + if gvars is not None: + raise TypeError( + "Cannot inherit from Generic[...] multiple times.") + gvars = base.__parameters__ + if gvars is not None: + tvarset = set(tvars) + gvarset = set(gvars) + if not tvarset <= gvarset: + s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) + s_args = ', '.join(str(g) for g in gvars) + raise TypeError(f"Some type variables ({s_vars}) are" + f" not listed in Generic[{s_args}]") + tvars = gvars + cls.__parameters__ = tuple(tvars) def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') class _BaseGenericAlias(_Final, _root=True): - """The central part of internal API. + """The central part of the internal API. This represents a generic version of type 'origin' with type arguments 'params'. There are two kind of these aliases: user defined and special. The special ones are wrappers around builtin collections and ABCs in collections.abc. These must - have 'name' always set. If 'inst' is False, then the alias can't be instantiated, + have 'name' always set. If 'inst' is False, then the alias can't be instantiated; this is used by e.g. typing.List and typing.Dict. """ + def __init__(self, origin, *, inst=True, name=None): self._inst = inst self._name = name @@ -957,7 +1235,9 @@ def __call__(self, *args, **kwargs): result = self.__origin__(*args, **kwargs) try: result.__orig_class__ = self - except AttributeError: + # Some objects raise TypeError (or something even more exotic) + # if you try to set attributes on them; we guard against that here + except Exception: pass return result @@ -965,9 +1245,29 @@ def __mro_entries__(self, bases): res = [] if self.__origin__ not in bases: res.append(self.__origin__) + + # Check if any base that occurs after us in `bases` is either itself a + # subclass of Generic, or something which will add a subclass of Generic + # to `__bases__` via its `__mro_entries__`. If not, add Generic + # ourselves. The goal is to ensure that Generic (or a subclass) will + # appear exactly once in the final bases tuple. If we let it appear + # multiple times, we risk "can't form a consistent MRO" errors. i = bases.index(self) for b in bases[i+1:]: - if isinstance(b, _BaseGenericAlias) or issubclass(b, Generic): + if isinstance(b, _BaseGenericAlias): + break + if not isinstance(b, type): + meth = getattr(b, "__mro_entries__", None) + new_bases = meth(bases) if meth else None + if ( + isinstance(new_bases, tuple) and + any( + isinstance(b2, type) and issubclass(b2, Generic) + for b2 in new_bases + ) + ): + break + elif issubclass(b, Generic): break else: res.append(Generic) @@ -984,8 +1284,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', - '_typevar_types', '_paramspec_tvars'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', '_defaults'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1001,6 +1300,7 @@ def __dir__(self): return list(set(super().__dir__() + [attr for attr in dir(self.__origin__) if not _is_dunder(attr)])) + # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: # * __parameters__ is a tuple of unique free type parameters of a generic @@ -1013,18 +1313,42 @@ def __dir__(self): class _GenericAlias(_BaseGenericAlias, _root=True): - def __init__(self, origin, params, *, inst=True, name=None, - _typevar_types=TypeVar, - _paramspec_tvars=False): + # The type of parameterized generics. + # + # That is, for example, `type(List[int])` is `_GenericAlias`. + # + # Objects which are instances of this class include: + # * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. + # * Note that native container types, e.g. `tuple`, `list`, use + # `types.GenericAlias` instead. + # * Parameterized classes: + # class C[T]: pass + # # C[int] is a _GenericAlias + # * `Callable` aliases, generic `Callable` aliases, and + # parameterized `Callable` aliases: + # T = TypeVar('T') + # # _CallableGenericAlias inherits from _GenericAlias. + # A = Callable[[], None] # _CallableGenericAlias + # B = Callable[[T], None] # _CallableGenericAlias + # C = B[int] # _CallableGenericAlias + # * Parameterized `Final`, `ClassVar`, `TypeGuard`, and `TypeIs`: + # # All _GenericAlias + # Final[int] + # ClassVar[float] + # TypeGuard[bool] + # TypeIs[range] + + def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) - if not isinstance(params, tuple): - params = (params,) + if not isinstance(args, tuple): + args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else - () if a is _TypingEmpty else - a for a in params) - self.__parameters__ = _collect_type_vars(params, typevar_types=_typevar_types) - self._typevar_types = _typevar_types - self._paramspec_tvars = _paramspec_tvars + a for a in args) + enforce_default_ordering = origin in (Generic, Protocol) + self.__parameters__ = _collect_type_parameters( + args, + enforce_default_ordering=enforce_default_ordering, + ) if not name: self.__module__ = origin.__module__ @@ -1044,53 +1368,140 @@ def __ror__(self, left): return Union[left, self] @_tp_cache - def __getitem__(self, params): + def __getitem__(self, args): + # Parameterizes an already-parameterized object. + # + # For example, we arrive here doing something like: + # T1 = TypeVar('T1') + # T2 = TypeVar('T2') + # T3 = TypeVar('T3') + # class A(Generic[T1]): pass + # B = A[T2] # B is a _GenericAlias + # C = B[T3] # Invokes _GenericAlias.__getitem__ + # + # We also arrive here when parameterizing a generic `Callable` alias: + # T = TypeVar('T') + # C = Callable[[T], None] + # C[int] # Invokes _GenericAlias.__getitem__ + if self.__origin__ in (Generic, Protocol): # Can't subscript Generic[...] or Protocol[...]. raise TypeError(f"Cannot subscript already-subscripted {self}") - if not isinstance(params, tuple): - params = (params,) - params = tuple(_type_convert(p) for p in params) - if (self._paramspec_tvars - and any(isinstance(t, ParamSpec) for t in self.__parameters__)): - params = _prepare_paramspec_params(self, params) - else: - _check_generic(self, params, len(self.__parameters__)) + if not self.__parameters__: + raise TypeError(f"{self} is not a generic class") - subst = dict(zip(self.__parameters__, params)) + # Preprocess `args`. + if not isinstance(args, tuple): + args = (args,) + args = _unpack_args(*(_type_convert(p) for p in args)) + new_args = self._determine_new_args(args) + r = self.copy_with(new_args) + return r + + def _determine_new_args(self, args): + # Determines new __args__ for __getitem__. + # + # For example, suppose we had: + # T1 = TypeVar('T1') + # T2 = TypeVar('T2') + # class A(Generic[T1, T2]): pass + # T3 = TypeVar('T3') + # B = A[int, T3] + # C = B[str] + # `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. + # Unfortunately, this is harder than it looks, because if `T3` is + # anything more exotic than a plain `TypeVar`, we need to consider + # edge cases. + + params = self.__parameters__ + # In the example above, this would be {T3: str} + for param in params: + prepare = getattr(param, '__typing_prepare_subst__', None) + if prepare is not None: + args = prepare(self, args) + alen = len(args) + plen = len(params) + if alen != plen: + raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};" + f" actual {alen}, expected {plen}") + new_arg_by_param = dict(zip(params, args)) + return tuple(self._make_substitution(self.__args__, new_arg_by_param)) + + def _make_substitution(self, args, new_arg_by_param): + """Create a list of new type arguments.""" new_args = [] - for arg in self.__args__: - if isinstance(arg, self._typevar_types): - if isinstance(arg, ParamSpec): - arg = subst[arg] - if not _is_param_expr(arg): - raise TypeError(f"Expected a list of types, an ellipsis, " - f"ParamSpec, or Concatenate. Got {arg}") + for old_arg in args: + if isinstance(old_arg, type): + new_args.append(old_arg) + continue + + substfunc = getattr(old_arg, '__typing_subst__', None) + if substfunc: + new_arg = substfunc(new_arg_by_param[old_arg]) + else: + subparams = getattr(old_arg, '__parameters__', ()) + if not subparams: + new_arg = old_arg else: - arg = subst[arg] - elif isinstance(arg, (_GenericAlias, GenericAlias, types.UnionType)): - subparams = arg.__parameters__ - if subparams: - subargs = tuple(subst[x] for x in subparams) - arg = arg[subargs] - # Required to flatten out the args for CallableGenericAlias - if self.__origin__ == collections.abc.Callable and isinstance(arg, tuple): - new_args.extend(arg) + subargs = [] + for x in subparams: + if isinstance(x, TypeVarTuple): + subargs.extend(new_arg_by_param[x]) + else: + subargs.append(new_arg_by_param[x]) + new_arg = old_arg[tuple(subargs)] + + if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple): + # Consider the following `Callable`. + # C = Callable[[int], str] + # Here, `C.__args__` should be (int, str) - NOT ([int], str). + # That means that if we had something like... + # P = ParamSpec('P') + # T = TypeVar('T') + # C = Callable[P, T] + # D = C[[int, str], float] + # ...we need to be careful; `new_args` should end up as + # `(int, str, float)` rather than `([int, str], float)`. + new_args.extend(new_arg) + elif _is_unpacked_typevartuple(old_arg): + # Consider the following `_GenericAlias`, `B`: + # class A(Generic[*Ts]): ... + # B = A[T, *Ts] + # If we then do: + # B[float, int, str] + # The `new_arg` corresponding to `T` will be `float`, and the + # `new_arg` corresponding to `*Ts` will be `(int, str)`. We + # should join all these types together in a flat list + # `(float, int, str)` - so again, we should `extend`. + new_args.extend(new_arg) + elif isinstance(old_arg, tuple): + # Corner case: + # P = ParamSpec('P') + # T = TypeVar('T') + # class Base(Generic[P]): ... + # Can be substituted like this: + # X = Base[[int, T]] + # In this case, `old_arg` will be a tuple: + new_args.append( + tuple(self._make_substitution(old_arg, new_arg_by_param)), + ) else: - new_args.append(arg) - return self.copy_with(tuple(new_args)) + new_args.append(new_arg) + return new_args - def copy_with(self, params): - return self.__class__(self.__origin__, params, name=self._name, inst=self._inst, - _typevar_types=self._typevar_types, - _paramspec_tvars=self._paramspec_tvars) + def copy_with(self, args): + return self.__class__(self.__origin__, args, name=self._name, inst=self._inst) def __repr__(self): if self._name: name = 'typing.' + self._name else: name = _type_repr(self.__origin__) - args = ", ".join([_type_repr(a) for a in self.__args__]) + if self.__args__: + args = ", ".join([_type_repr(a) for a in self.__args__]) + else: + # To ensure the repr is eval-able. + args = "()" return f'{name}[{args}]' def __reduce__(self): @@ -1118,21 +1529,25 @@ def __mro_entries__(self, bases): return () return (self.__origin__,) + def __iter__(self): + yield Unpack[self] + # _nparams is the number of accepted parameters, e.g. 0 for Hashable, # 1 for List and 2 for Dict. It may be -1 if variable number of # parameters are accepted (needs custom __getitem__). -class _SpecialGenericAlias(_BaseGenericAlias, _root=True): - def __init__(self, origin, nparams, *, inst=True, name=None): +class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): + def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): if name is None: name = origin.__name__ super().__init__(origin, inst=inst, name=name) self._nparams = nparams + self._defaults = defaults if origin.__module__ == 'builtins': - self.__doc__ = f'A generic version of {origin.__qualname__}.' + self.__doc__ = f'Deprecated alias to {origin.__qualname__}.' else: - self.__doc__ = f'A generic version of {origin.__module__}.{origin.__qualname__}.' + self.__doc__ = f'Deprecated alias to {origin.__module__}.{origin.__qualname__}.' @_tp_cache def __getitem__(self, params): @@ -1140,7 +1555,22 @@ def __getitem__(self, params): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params, self._nparams) + if (self._defaults + and len(params) < self._nparams + and len(params) + len(self._defaults) >= self._nparams + ): + params = (*params, *self._defaults[len(params) - self._nparams:]) + actual_len = len(params) + + if actual_len != self._nparams: + if self._defaults: + expected = f"at least {self._nparams - len(self._defaults)}" + else: + expected = str(self._nparams) + if not self._nparams: + raise TypeError(f"{self} is not a generic class") + raise TypeError(f"Too {'many' if actual_len > self._nparams else 'few'} arguments for {self};" + f" actual {actual_len}, expected {expected}") return self.copy_with(params) def copy_with(self, params): @@ -1166,7 +1596,23 @@ def __or__(self, right): def __ror__(self, left): return Union[left, self] -class _CallableGenericAlias(_GenericAlias, _root=True): + +class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): + def __init__( + self, origin, nparams, *, removal_version, inst=True, name=None + ): + super().__init__(origin, nparams, inst=inst, name=name) + self._removal_version = removal_version + + def __instancecheck__(self, inst): + import warnings + warnings._deprecated( + f"{self.__module__}.{self._name}", remove=self._removal_version + ) + return super().__instancecheck__(inst) + + +class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' args = self.__args__ @@ -1186,9 +1632,7 @@ def __reduce__(self): class _CallableType(_SpecialGenericAlias, _root=True): def copy_with(self, params): return _CallableGenericAlias(self.__origin__, params, - name=self._name, inst=self._inst, - _typevar_types=(TypeVar, ParamSpec), - _paramspec_tvars=True) + name=self._name, inst=self._inst) def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: @@ -1221,51 +1665,52 @@ def __getitem_inner__(self, params): class _TupleType(_SpecialGenericAlias, _root=True): @_tp_cache def __getitem__(self, params): - if params == (): - return self.copy_with((_TypingEmpty,)) if not isinstance(params, tuple): params = (params,) - if len(params) == 2 and params[1] is ...: + if len(params) >= 2 and params[-1] is ...: msg = "Tuple[t, ...]: t must be a type." - p = _type_check(params[0], msg) - return self.copy_with((p, _TypingEllipsis)) + params = tuple(_type_check(p, msg) for p in params[:-1]) + return self.copy_with((*params, _TypingEllipsis)) msg = "Tuple[t0, t1, ...]: each t must be a type." params = tuple(_type_check(p, msg) for p in params) return self.copy_with(params) -class _UnionGenericAlias(_GenericAlias, _root=True): - def copy_with(self, params): - return Union[params] +class _UnionGenericAliasMeta(type): + def __instancecheck__(self, inst: object) -> bool: + import warnings + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) + return isinstance(inst, Union) + + def __subclasscheck__(self, inst: type) -> bool: + import warnings + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) + return issubclass(inst, Union) def __eq__(self, other): - if not isinstance(other, (_UnionGenericAlias, types.UnionType)): - return NotImplemented - return set(self.__args__) == set(other.__args__) + import warnings + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) + if other is _UnionGenericAlias or other is Union: + return True + return NotImplemented def __hash__(self): - return hash(frozenset(self.__args__)) + return hash(Union) - def __repr__(self): - args = self.__args__ - if len(args) == 2: - if args[0] is type(None): - return f'typing.Optional[{_type_repr(args[1])}]' - elif args[1] is type(None): - return f'typing.Optional[{_type_repr(args[0])}]' - return super().__repr__() - def __instancecheck__(self, obj): - return self.__subclasscheck__(type(obj)) +class _UnionGenericAlias(metaclass=_UnionGenericAliasMeta): + """Compatibility hack. - def __subclasscheck__(self, cls): - for arg in self.__args__: - if issubclass(cls, arg): - return True + A class named _UnionGenericAlias used to be used to implement + typing.Union. This class exists to serve as a shim to preserve + the meaning of some code that used to use _UnionGenericAlias + directly. - def __reduce__(self): - func, (origin, args) = super().__reduce__() - return func, (Union, args) + """ + def __new__(cls, self_cls, parameters, /, *, name=None): + import warnings + warnings._deprecated("_UnionGenericAlias", remove=(3, 17)) + return Union[parameters] def _value_and_type_iter(parameters): @@ -1273,7 +1718,6 @@ def _value_and_type_iter(parameters): class _LiteralGenericAlias(_GenericAlias, _root=True): - def __eq__(self, other): if not isinstance(other, _LiteralGenericAlias): return NotImplemented @@ -1290,118 +1734,109 @@ def copy_with(self, params): return (*params[:-1], *params[-1]) if isinstance(params[-1], _ConcatenateGenericAlias): params = (*params[:-1], *params[-1].__args__) - elif not isinstance(params[-1], ParamSpec): - raise TypeError("The last parameter to Concatenate should be a " - "ParamSpec variable.") return super().copy_with(params) -class Generic: - """Abstract base class for generic types. +@_SpecialForm +def Unpack(self, parameters): + """Type unpack operator. - A generic type is typically declared by inheriting from - this class parameterized with one or more type variables. - For example, a generic mapping type might be defined as:: + The type unpack operator takes the child types from some container type, + such as `tuple[int, str]` or a `TypeVarTuple`, and 'pulls them out'. - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. + For example:: - This class can then be used as follows:: + # For some generic class `Foo`: + Foo[Unpack[tuple[int, str]]] # Equivalent to Foo[int, str] - def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: - try: - return mapping[key] - except KeyError: - return default - """ - __slots__ = () - _is_protocol = False + Ts = TypeVarTuple('Ts') + # Specifies that `Bar` is generic in an arbitrary number of types. + # (Think of `Ts` as a tuple of an arbitrary number of individual + # `TypeVar`s, which the `Unpack` is 'pulling out' directly into the + # `Generic[]`.) + class Bar(Generic[Unpack[Ts]]): ... + Bar[int] # Valid + Bar[int, str] # Also valid - @_tp_cache - def __class_getitem__(cls, params): - if not isinstance(params, tuple): - params = (params,) - if not params and cls is not Tuple: - raise TypeError( - f"Parameter list to {cls.__qualname__}[...] cannot be empty") - params = tuple(_type_convert(p) for p in params) - if cls in (Generic, Protocol): - # Generic and Protocol can only be subscripted with unique type variables. - if not all(isinstance(p, (TypeVar, ParamSpec)) for p in params): - raise TypeError( - f"Parameters to {cls.__name__}[...] must all be type variables " - f"or parameter specification variables.") - if len(set(params)) != len(params): - raise TypeError( - f"Parameters to {cls.__name__}[...] must all be unique") - else: - # Subscripting a regular Generic subclass. - if any(isinstance(t, ParamSpec) for t in cls.__parameters__): - params = _prepare_paramspec_params(cls, params) - else: - _check_generic(cls, params, len(cls.__parameters__)) - return _GenericAlias(cls, params, - _typevar_types=(TypeVar, ParamSpec), - _paramspec_tvars=True) + From Python 3.11, this can also be done using the `*` operator:: - def __init_subclass__(cls, *args, **kwargs): - super().__init_subclass__(*args, **kwargs) - tvars = [] - if '__orig_bases__' in cls.__dict__: - error = Generic in cls.__orig_bases__ - else: - error = Generic in cls.__bases__ and cls.__name__ != 'Protocol' - if error: - raise TypeError("Cannot inherit from plain Generic") - if '__orig_bases__' in cls.__dict__: - tvars = _collect_type_vars(cls.__orig_bases__, (TypeVar, ParamSpec)) - # Look for Generic[T1, ..., Tn]. - # If found, tvars must be a subset of it. - # If not found, tvars is it. - # Also check for and reject plain Generic, - # and reject multiple Generic[...]. - gvars = None - for base in cls.__orig_bases__: - if (isinstance(base, _GenericAlias) and - base.__origin__ is Generic): - if gvars is not None: - raise TypeError( - "Cannot inherit from Generic[...] multiple types.") - gvars = base.__parameters__ - if gvars is not None: - tvarset = set(tvars) - gvarset = set(gvars) - if not tvarset <= gvarset: - s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) - s_args = ', '.join(str(g) for g in gvars) - raise TypeError(f"Some type variables ({s_vars}) are" - f" not listed in Generic[{s_args}]") - tvars = gvars - cls.__parameters__ = tuple(tvars) - - -class _TypingEmpty: - """Internal placeholder for () or []. Used by TupleMeta and CallableMeta - to allow empty list/tuple in specific places, without allowing them - to sneak in where prohibited. + Foo[*tuple[int, str]] + class Bar(Generic[*Ts]): ... + + And from Python 3.12, it can be done using built-in syntax for generics:: + + Foo[*tuple[int, str]] + class Bar[*Ts]: ... + + The operator can also be used along with a `TypedDict` to annotate + `**kwargs` in a function signature:: + + class Movie(TypedDict): + name: str + year: int + + # This function expects two keyword arguments - *name* of type `str` and + # *year* of type `int`. + def foo(**kwargs: Unpack[Movie]): ... + + Note that there is only some runtime checking of this operator. Not + everything the runtime allows may be accepted by static type checkers. + + For more information, see PEPs 646 and 692. """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _UnpackGenericAlias(origin=self, args=(item,)) + + +class _UnpackGenericAlias(_GenericAlias, _root=True): + def __repr__(self): + # `Unpack` only takes one argument, so __args__ should contain only + # a single item. + return f'typing.Unpack[{_type_repr(self.__args__[0])}]' + + def __getitem__(self, args): + if self.__typing_is_unpacked_typevartuple__: + return args + return super().__getitem__(args) + + @property + def __typing_unpacked_tuple_args__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + arg, = self.__args__ + if isinstance(arg, (_GenericAlias, types.GenericAlias)): + if arg.__origin__ is not tuple: + raise TypeError("Unpack[...] must be used with a tuple type") + return arg.__args__ + return None + + @property + def __typing_is_unpacked_typevartuple__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + return isinstance(self.__args__[0], TypeVarTuple) class _TypingEllipsis: """Internal placeholder for ... (ellipsis).""" -_TYPING_INTERNALS = ['__parameters__', '__orig_bases__', '__orig_class__', - '_is_protocol', '_is_runtime_protocol'] +_TYPING_INTERNALS = frozenset({ + '__parameters__', '__orig_bases__', '__orig_class__', + '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', + '__non_callable_proto_members__', '__type_params__', +}) -_SPECIAL_NAMES = ['__abstractmethods__', '__annotations__', '__dict__', '__doc__', - '__init__', '__module__', '__new__', '__slots__', - '__subclasshook__', '__weakref__', '__class_getitem__'] +_SPECIAL_NAMES = frozenset({ + '__abstractmethods__', '__annotations__', '__dict__', '__doc__', + '__init__', '__module__', '__new__', '__slots__', + '__subclasshook__', '__weakref__', '__class_getitem__', + '__match_args__', '__static_attributes__', '__firstlineno__', + '__annotate__', '__annotate_func__', '__annotations_cache__', +}) # These special attributes will be not collected as protocol members. -EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS + _SPECIAL_NAMES + ['_MutableMapping__marker'] +EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS | _SPECIAL_NAMES | {'_MutableMapping__marker'} def _get_protocol_attrs(cls): @@ -1412,20 +1847,21 @@ def _get_protocol_attrs(cls): """ attrs = set() for base in cls.__mro__[:-1]: # without object - if base.__name__ in ('Protocol', 'Generic'): + if base.__name__ in {'Protocol', 'Generic'}: continue - annotations = getattr(base, '__annotations__', {}) - for attr in list(base.__dict__.keys()) + list(annotations.keys()): + try: + annotations = base.__annotations__ + except Exception: + # Only go through annotationlib to handle deferred annotations if we need to + annotations = _lazy_annotationlib.get_annotations( + base, format=_lazy_annotationlib.Format.FORWARDREF + ) + for attr in (*base.__dict__, *annotations): if not attr.startswith('_abc_') and attr not in EXCLUDED_ATTRIBUTES: attrs.add(attr) return attrs -def _is_callable_members_only(cls): - # PEP 544 prohibits using issubclass() with protocols that have non-method members. - return all(callable(getattr(cls, attr, None)) for attr in _get_protocol_attrs(cls)) - - def _no_init_or_replace_init(self, *args, **kwargs): cls = type(self) @@ -1456,59 +1892,200 @@ def _no_init_or_replace_init(self, *args, **kwargs): def _caller(depth=1, default='__main__'): + try: + return sys._getframemodulename(depth + 1) or default + except AttributeError: # For platforms without _getframemodulename() + pass try: return sys._getframe(depth + 1).f_globals.get('__name__', default) except (AttributeError, ValueError): # For platforms without _getframe() - return None - + pass + return None -def _allow_reckless_class_checks(depth=3): +def _allow_reckless_class_checks(depth=2): """Allow instance and class checks for special stdlib modules. The abc and functools modules indiscriminately call isinstance() and issubclass() on the whole MRO of a user class, which may contain protocols. """ - try: - return sys._getframe(depth).f_globals['__name__'] in ['abc', 'functools'] - except (AttributeError, ValueError): # For platforms without _getframe(). - return True + return _caller(depth) in {'abc', 'functools', None} _PROTO_ALLOWLIST = { 'collections.abc': [ 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', - 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', + 'AsyncIterator', 'Hashable', 'Sized', 'Container', 'Collection', + 'Reversible', 'Buffer', ], 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], + 'io': ['Reader', 'Writer'], + 'os': ['PathLike'], } +@functools.cache +def _lazy_load_getattr_static(): + # Import getattr_static lazily so as not to slow down the import of typing.py + # Cache the result so we don't slow down _ProtocolMeta.__instancecheck__ unnecessarily + from inspect import getattr_static + return getattr_static + + +_cleanups.append(_lazy_load_getattr_static.cache_clear) + +def _pickle_psargs(psargs): + return ParamSpecArgs, (psargs.__origin__,) + +copyreg.pickle(ParamSpecArgs, _pickle_psargs) + +def _pickle_pskwargs(pskwargs): + return ParamSpecKwargs, (pskwargs.__origin__,) + +copyreg.pickle(ParamSpecKwargs, _pickle_pskwargs) + +del _pickle_psargs, _pickle_pskwargs + + +# Preload these once, as globals, as a micro-optimisation. +# This makes a significant difference to the time it takes +# to do `isinstance()`/`issubclass()` checks +# against runtime-checkable protocols with only one callable member. +_abc_instancecheck = ABCMeta.__instancecheck__ +_abc_subclasscheck = ABCMeta.__subclasscheck__ + + +def _type_check_issubclass_arg_1(arg): + """Raise TypeError if `arg` is not an instance of `type` + in `issubclass(arg, <protocol>)`. + + In most cases, this is verified by type.__subclasscheck__. + Checking it again unnecessarily would slow down issubclass() checks, + so, we don't perform this check unless we absolutely have to. + + For various error paths, however, + we want to ensure that *this* error message is shown to the user + where relevant, rather than a typing.py-specific error message. + """ + if not isinstance(arg, type): + # Same error message as for issubclass(1, int). + raise TypeError('issubclass() arg 1 must be a class') + + class _ProtocolMeta(ABCMeta): - # This metaclass is really unfortunate and exists only because of - # the lack of __instancehook__. + # This metaclass is somewhat unfortunate, + # but is necessary for several reasons... + def __new__(mcls, name, bases, namespace, /, **kwargs): + if name == "Protocol" and bases == (Generic,): + pass + elif Protocol in bases: + for base in bases: + if not ( + base in {object, Generic} + or base.__name__ in _PROTO_ALLOWLIST.get(base.__module__, []) + or ( + issubclass(base, Generic) + and getattr(base, "_is_protocol", False) + ) + ): + raise TypeError( + f"Protocols can only inherit from other protocols, " + f"got {base!r}" + ) + return super().__new__(mcls, name, bases, namespace, **kwargs) + + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + if getattr(cls, "_is_protocol", False): + cls.__protocol_attrs__ = _get_protocol_attrs(cls) + + def __subclasscheck__(cls, other): + if cls is Protocol: + return type.__subclasscheck__(cls, other) + if ( + getattr(cls, '_is_protocol', False) + and not _allow_reckless_class_checks() + ): + if not getattr(cls, '_is_runtime_protocol', False): + _type_check_issubclass_arg_1(other) + raise TypeError( + "Instance and class checks can only be used with " + "@runtime_checkable protocols" + ) + if ( + # this attribute is set by @runtime_checkable: + cls.__non_callable_proto_members__ + and cls.__dict__.get("__subclasshook__") is _proto_hook + ): + _type_check_issubclass_arg_1(other) + non_method_attrs = sorted(cls.__non_callable_proto_members__) + raise TypeError( + "Protocols with non-method members don't support issubclass()." + f" Non-method members: {str(non_method_attrs)[1:-1]}." + ) + return _abc_subclasscheck(cls, other) + def __instancecheck__(cls, instance): # We need this method for situations where attributes are # assigned in __init__. + if cls is Protocol: + return type.__instancecheck__(cls, instance) + if not getattr(cls, "_is_protocol", False): + # i.e., it's a concrete subclass of a protocol + return _abc_instancecheck(cls, instance) + if ( - getattr(cls, '_is_protocol', False) and not getattr(cls, '_is_runtime_protocol', False) and - not _allow_reckless_class_checks(depth=2) + not _allow_reckless_class_checks() ): raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") - if ((not getattr(cls, '_is_protocol', False) or - _is_callable_members_only(cls)) and - issubclass(instance.__class__, cls)): + if _abc_instancecheck(cls, instance): return True - if cls._is_protocol: - if all(hasattr(instance, attr) and - # All *methods* can be blocked by setting them to None. - (not callable(getattr(cls, attr, None)) or - getattr(instance, attr) is not None) - for attr in _get_protocol_attrs(cls)): - return True - return super().__instancecheck__(instance) + + getattr_static = _lazy_load_getattr_static() + for attr in cls.__protocol_attrs__: + try: + val = getattr_static(instance, attr) + except AttributeError: + break + # this attribute is set by @runtime_checkable: + if val is None and attr not in cls.__non_callable_proto_members__: + break + else: + return True + + return False + + +@classmethod +def _proto_hook(cls, other): + if not cls.__dict__.get('_is_protocol', False): + return NotImplemented + + for attr in cls.__protocol_attrs__: + for base in other.__mro__: + # Check if the members appears in the class dictionary... + if attr in base.__dict__: + if base.__dict__[attr] is None: + return NotImplemented + break + + # ...or in annotations, if it is a sub-protocol. + if issubclass(other, Generic) and getattr(other, "_is_protocol", False): + # We avoid the slower path through annotationlib here because in most + # cases it should be unnecessary. + try: + annos = base.__annotations__ + except Exception: + annos = _lazy_annotationlib.get_annotations( + base, format=_lazy_annotationlib.Format.FORWARDREF + ) + if attr in annos: + break + else: + return NotImplemented + return True class Protocol(Generic, metaclass=_ProtocolMeta): @@ -1521,7 +2098,9 @@ def meth(self) -> int: ... Such classes are primarily used with static type checkers that recognize - structural subtyping (static duck-typing), for example:: + structural subtyping (static duck-typing). + + For example:: class C: def meth(self) -> int: @@ -1537,91 +2116,47 @@ def func(x: Proto) -> int: only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as:: - class GenProto(Protocol[T]): + class GenProto[T](Protocol): def meth(self) -> T: ... """ + __slots__ = () _is_protocol = True _is_runtime_protocol = False def __init_subclass__(cls, *args, **kwargs): - super().__init_subclass__(*args, **kwargs) - - # Determine if this is a protocol or a concrete subclass. - if not cls.__dict__.get('_is_protocol', False): - cls._is_protocol = any(b is Protocol for b in cls.__bases__) - - # Set (or override) the protocol subclass hook. - def _proto_hook(other): - if not cls.__dict__.get('_is_protocol', False): - return NotImplemented - - # First, perform various sanity checks. - if not getattr(cls, '_is_runtime_protocol', False): - if _allow_reckless_class_checks(): - return NotImplemented - raise TypeError("Instance and class checks can only be used with" - " @runtime_checkable protocols") - if not _is_callable_members_only(cls): - if _allow_reckless_class_checks(): - return NotImplemented - raise TypeError("Protocols with non-method members" - " don't support issubclass()") - if not isinstance(other, type): - # Same error message as for issubclass(1, int). - raise TypeError('issubclass() arg 1 must be a class') - - # Second, perform the actual structural compatibility check. - for attr in _get_protocol_attrs(cls): - for base in other.__mro__: - # Check if the members appears in the class dictionary... - if attr in base.__dict__: - if base.__dict__[attr] is None: - return NotImplemented - break - - # ...or in annotations, if it is a sub-protocol. - annotations = getattr(base, '__annotations__', {}) - if (isinstance(annotations, collections.abc.Mapping) and - attr in annotations and - issubclass(other, Generic) and other._is_protocol): - break - else: - return NotImplemented - return True + super().__init_subclass__(*args, **kwargs) + + # Determine if this is a protocol or a concrete subclass. + if not cls.__dict__.get('_is_protocol', False): + cls._is_protocol = any(b is Protocol for b in cls.__bases__) + # Set (or override) the protocol subclass hook. if '__subclasshook__' not in cls.__dict__: cls.__subclasshook__ = _proto_hook - # We have nothing more to do for non-protocols... - if not cls._is_protocol: - return + # Prohibit instantiation for protocol classes + if cls._is_protocol and cls.__init__ is Protocol.__init__: + cls.__init__ = _no_init_or_replace_init - # ... otherwise check consistency of bases, and prohibit instantiation. - for base in cls.__bases__: - if not (base in (object, Generic) or - base.__module__ in _PROTO_ALLOWLIST and - base.__name__ in _PROTO_ALLOWLIST[base.__module__] or - issubclass(base, Generic) and base._is_protocol): - raise TypeError('Protocols can only inherit from other' - ' protocols, got %r' % base) - cls.__init__ = _no_init_or_replace_init - -class _AnnotatedAlias(_GenericAlias, _root=True): +class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): """Runtime representation of an annotated type. At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' - with extra annotations. The alias behaves like a normal typing alias, - instantiating is the same as instantiating the underlying type, binding + with extra metadata. The alias behaves like a normal typing alias. + Instantiating is the same as instantiating the underlying type; binding it to types is also the same. + + The metadata itself is stored in a '__metadata__' attribute as a tuple. """ + def __init__(self, origin, metadata): if isinstance(origin, _AnnotatedAlias): metadata = origin.__metadata__ + metadata origin = origin.__origin__ - super().__init__(origin, origin) + super().__init__(origin, origin, name='Annotated') self.__metadata__ = metadata def copy_with(self, params): @@ -1654,9 +2189,14 @@ def __getattr__(self, attr): return 'Annotated' return super().__getattr__(attr) + def __mro_entries__(self, bases): + return (self.__origin__,) + -class Annotated: - """Add context specific metadata to a type. +@_TypedCacheSpecialForm +@_tp_cache(typed=True) +def Annotated(self, *params): + """Add context-specific metadata to a type. Example: Annotated[int, runtime_check.Unsigned] indicates to the hypothetical runtime_check module that this type is an unsigned int. @@ -1668,44 +2208,51 @@ class Annotated: Details: - It's an error to call `Annotated` with less than two arguments. - - Nested Annotated are flattened:: + - Access the metadata via the ``__metadata__`` attribute:: + + assert Annotated[int, '$'].__metadata__ == ('$',) - Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3] + - Nested Annotated types are flattened:: + + assert Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3] - Instantiating an annotated type is equivalent to instantiating the underlying type:: - Annotated[C, Ann1](5) == C(5) + assert Annotated[C, Ann1](5) == C(5) - Annotated can be used as a generic type alias:: - Optimized = Annotated[T, runtime.Optimize()] - Optimized[int] == Annotated[int, runtime.Optimize()] + type Optimized[T] = Annotated[T, runtime.Optimize()] + # type checker will treat Optimized[int] + # as equivalent to Annotated[int, runtime.Optimize()] - OptimizedList = Annotated[List[T], runtime.Optimize()] - OptimizedList[int] == Annotated[List[int], runtime.Optimize()] - """ + type OptimizedList[T] = Annotated[list[T], runtime.Optimize()] + # type checker will treat OptimizedList[int] + # as equivalent to Annotated[list[int], runtime.Optimize()] - __slots__ = () + - Annotated cannot be used with an unpacked TypeVarTuple:: - def __new__(cls, *args, **kwargs): - raise TypeError("Type Annotated cannot be instantiated.") + type Variadic[*Ts] = Annotated[*Ts, Ann1] # NOT valid - @_tp_cache - def __class_getitem__(cls, params): - if not isinstance(params, tuple) or len(params) < 2: - raise TypeError("Annotated[...] should be used " - "with at least two arguments (a type and an " - "annotation).") - msg = "Annotated[t, ...]: t must be a type." - origin = _type_check(params[0], msg, allow_special_forms=True) - metadata = tuple(params[1:]) - return _AnnotatedAlias(origin, metadata) + This would be equivalent to:: - def __init_subclass__(cls, *args, **kwargs): - raise TypeError( - "Cannot subclass {}.Annotated".format(cls.__module__) - ) + Annotated[T1, T2, T3, ..., Ann1] + + where T1, T2 etc. are TypeVars, which would be invalid, because + only one type should be passed to Annotated. + """ + if len(params) < 2: + raise TypeError("Annotated[...] should be used " + "with at least two arguments (a type and an " + "annotation).") + if _is_unpacked_typevartuple(params[0]): + raise TypeError("Annotated[...] should not be used with an " + "unpacked TypeVarTuple") + msg = "Annotated[t, ...]: t must be a type." + origin = _type_check(params[0], msg, allow_special_forms=True) + metadata = tuple(params[1:]) + return _AnnotatedAlias(origin, metadata) def runtime_checkable(cls): @@ -1715,6 +2262,7 @@ def runtime_checkable(cls): Raise TypeError if applied to a non-protocol class. This allows a simple-minded structural check very similar to one trick ponies in collections.abc such as Iterable. + For example:: @runtime_checkable @@ -1726,10 +2274,26 @@ def close(self): ... Warning: this will check only the presence of the required methods, not their type signatures! """ - if not issubclass(cls, Generic) or not cls._is_protocol: + if not issubclass(cls, Generic) or not getattr(cls, '_is_protocol', False): raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + # See gh-113320 for why we compute this attribute here, + # rather than in `_ProtocolMeta.__init__` + cls.__non_callable_proto_members__ = set() + for attr in cls.__protocol_attrs__: + try: + is_callable = callable(getattr(cls, attr, None)) + except Exception as e: + raise TypeError( + f"Failed to determine whether protocol member {attr!r} " + "is a method member" + ) from e + else: + if not is_callable: + cls.__non_callable_proto_members__.add(attr) return cls @@ -1744,37 +2308,28 @@ def cast(typ, val): return val -def _get_defaults(func): - """Internal helper to extract the default arguments, by name.""" - try: - code = func.__code__ - except AttributeError: - # Some built-in functions don't have __code__, __defaults__, etc. - return {} - pos_count = code.co_argcount - arg_names = code.co_varnames - arg_names = arg_names[:pos_count] - defaults = func.__defaults__ or () - kwdefaults = func.__kwdefaults__ - res = dict(kwdefaults) if kwdefaults else {} - pos_offset = pos_count - len(defaults) - for name, value in zip(arg_names[pos_offset:], defaults): - assert name not in res - res[name] = value - return res +def assert_type(val, typ, /): + """Ask a static type checker to confirm that the value is of the given type. + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. -_allowed_types = (types.FunctionType, types.BuiltinFunctionType, - types.MethodType, types.ModuleType, - WrapperDescriptorType, MethodWrapperType, MethodDescriptorType) + When a static type checker encounters a call to assert_type(), it + emits an error if the value is not of the specified type:: + + def greet(name: str) -> None: + assert_type(name, str) # OK + assert_type(name, int) # type checker error + """ + return val -def get_type_hints(obj, globalns=None, localns=None, include_extras=False): +def get_type_hints(obj, globalns=None, localns=None, include_extras=False, + *, format=None): """Return type hints for an object. This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals, adds Optional[t] if a - default value equal to None is set and recursively replaces all + forward references encoded as string literals and recursively replaces all 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations @@ -1801,20 +2356,23 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): - If two dict arguments are passed, they specify globals and locals, respectively. """ - if getattr(obj, '__no_type_check__', None): return {} + Format = _lazy_annotationlib.Format + if format is None: + format = Format.VALUE # Classes require a special treatment. if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): + ann = _lazy_annotationlib.get_annotations(base, format=format) + if format == Format.STRING: + hints.update(ann) + continue if globalns is None: base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) - if isinstance(ann, types.GetSetDescriptorType): - ann = {} base_locals = dict(vars(base)) if localns is None else localns if localns is None and globalns is None: # This is surprising, but required. Before Python 3.10, @@ -1824,14 +2382,33 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): # *base_globals* first rather than *base_locals*. # This only affects ForwardRefs. base_globals, base_locals = base_locals, base_globals + type_params = base.__type_params__ + base_globals, base_locals = _add_type_params_to_scope( + type_params, base_globals, base_locals, True) for name, value in ann.items(): + if isinstance(value, str): + value = _make_forward_ref(value, is_argument=False, is_class=True) + value = _eval_type(value, base_globals, base_locals, (), + format=format, owner=obj, prefer_fwd_module=True) if value is None: value = type(None) - if isinstance(value, str): - value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals) hints[name] = value - return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} + if include_extras or format == Format.STRING: + return hints + else: + return {k: _strip_annotations(t) for k, t in hints.items()} + + hints = _lazy_annotationlib.get_annotations(obj, format=format) + if ( + not hints + and not isinstance(obj, types.ModuleType) + and not callable(obj) + and not hasattr(obj, '__annotations__') + and not hasattr(obj, '__annotate__') + ): + raise TypeError(f"{obj!r} is not a module, class, or callable.") + if format == Format.STRING: + return hints if globalns is None: if isinstance(obj, types.ModuleType): @@ -1846,39 +2423,44 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): localns = globalns elif localns is None: localns = globalns - hints = getattr(obj, '__annotations__', None) - if hints is None: - # Return empty annotations for something that _could_ have them. - if isinstance(obj, _allowed_types): - return {} - else: - raise TypeError('{!r} is not a module, class, method, ' - 'or function.'.format(obj)) - defaults = _get_defaults(obj) - hints = dict(hints) + type_params = getattr(obj, "__type_params__", ()) + globalns, localns = _add_type_params_to_scope(type_params, globalns, localns, False) for name, value in hints.items(): - if value is None: - value = type(None) if isinstance(value, str): # class-level forward refs were handled above, this must be either # a module-level annotation or a function argument annotation - value = ForwardRef( + value = _make_forward_ref( value, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - value = _eval_type(value, globalns, localns) - if name in defaults and defaults[name] is None: - value = Optional[value] + value = _eval_type(value, globalns, localns, (), format=format, owner=obj, prefer_fwd_module=True) + if value is None: + value = type(None) hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} +# Add type parameters to the globals and locals scope. This is needed for +# compatibility. +def _add_type_params_to_scope(type_params, globalns, localns, is_class): + if not type_params: + return globalns, localns + globalns = dict(globalns) + localns = dict(localns) + for param in type_params: + if not is_class or param.__name__ not in globalns: + globalns[param.__name__] = param + localns.pop(param.__name__, None) + return globalns, localns + + def _strip_annotations(t): - """Strips the annotations from a given type. - """ + """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): return _strip_annotations(t.__origin__) + if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired, ReadOnly): + return _strip_annotations(t.__args__[0]) if isinstance(t, _GenericAlias): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: @@ -1889,7 +2471,7 @@ def _strip_annotations(t): if stripped_args == t.__args__: return t return GenericAlias(t.__origin__, stripped_args) - if isinstance(t, types.UnionType): + if isinstance(t, Union): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: return t @@ -1901,17 +2483,20 @@ def _strip_annotations(t): def get_origin(tp): """Get the unsubscripted version of a type. - This supports generic types, Callable, Tuple, Union, Literal, Final, ClassVar - and Annotated. Return None for unsupported types. Examples:: - - get_origin(Literal[42]) is Literal - get_origin(int) is None - get_origin(ClassVar[int]) is ClassVar - get_origin(Generic) is Generic - get_origin(Generic[T]) is Generic - get_origin(Union[T, int]) is Union - get_origin(List[Tuple[T, T]][int]) == list - get_origin(P.args) is P + This supports generic types, Callable, Tuple, Union, Literal, Final, ClassVar, + Annotated, and others. Return None for unsupported types. + + Examples:: + + >>> P = ParamSpec('P') + >>> assert get_origin(Literal[42]) is Literal + >>> assert get_origin(int) is None + >>> assert get_origin(ClassVar[int]) is ClassVar + >>> assert get_origin(Generic) is Generic + >>> assert get_origin(Generic[T]) is Generic + >>> assert get_origin(Union[T, int]) is Union + >>> assert get_origin(List[Tuple[T, T]][int]) is list + >>> assert get_origin(P.args) is P """ if isinstance(tp, _AnnotatedAlias): return Annotated @@ -1920,8 +2505,8 @@ def get_origin(tp): return tp.__origin__ if tp is Generic: return Generic - if isinstance(tp, types.UnionType): - return types.UnionType + if isinstance(tp, Union): + return Union return None @@ -1929,40 +2514,74 @@ def get_args(tp): """Get type arguments with all substitutions performed. For unions, basic simplifications used by Union constructor are performed. + Examples:: - get_args(Dict[str, int]) == (str, int) - get_args(int) == () - get_args(Union[int, Union[T, int], str][int]) == (int, str) - get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) - get_args(Callable[[], T][int]) == ([], int) + + >>> T = TypeVar('T') + >>> assert get_args(Dict[str, int]) == (str, int) + >>> assert get_args(int) == () + >>> assert get_args(Union[int, Union[T, int], str][int]) == (int, str) + >>> assert get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) + >>> assert get_args(Callable[[], T][int]) == ([], int) """ if isinstance(tp, _AnnotatedAlias): return (tp.__origin__,) + tp.__metadata__ if isinstance(tp, (_GenericAlias, GenericAlias)): res = tp.__args__ - if (tp.__origin__ is collections.abc.Callable - and not (len(res) == 2 and _is_param_expr(res[0]))): + if _should_unflatten_callable_args(tp, res): res = (list(res[:-1]), res[-1]) return res - if isinstance(tp, types.UnionType): + if isinstance(tp, Union): return tp.__args__ return () def is_typeddict(tp): - """Check if an annotation is a TypedDict class + """Check if an annotation is a TypedDict class. For example:: - class Film(TypedDict): - title: str - year: int - is_typeddict(Film) # => True - is_typeddict(Union[list, str]) # => False + >>> from typing import TypedDict + >>> class Film(TypedDict): + ... title: str + ... year: int + ... + >>> is_typeddict(Film) + True + >>> is_typeddict(dict) + False """ return isinstance(tp, _TypedDictMeta) +_ASSERT_NEVER_REPR_MAX_LENGTH = 100 + + +def assert_never(arg: Never, /) -> Never: + """Statically assert that a line of code is unreachable. + + Example:: + + def int_or_str(arg: int | str) -> None: + match arg: + case int(): + print("It's an int") + case str(): + print("It's a str") + case _: + assert_never(arg) + + If a type checker finds that a call to assert_never() is + reachable, it will emit an error. + + At runtime, this throws an exception when called. + """ + value = repr(arg) + if len(value) > _ASSERT_NEVER_REPR_MAX_LENGTH: + value = value[:_ASSERT_NEVER_REPR_MAX_LENGTH] + '...' + raise AssertionError(f"Expected code to be unreachable, but got: {value}") + + def no_type_check(arg): """Decorator to indicate that annotations are not type hints. @@ -1973,13 +2592,23 @@ def no_type_check(arg): This mutates the function(s) or class(es) in place. """ if isinstance(arg, type): - arg_attrs = arg.__dict__.copy() - for attr, val in arg.__dict__.items(): - if val in arg.__bases__ + (arg,): - arg_attrs.pop(attr) - for obj in arg_attrs.values(): + for key in dir(arg): + obj = getattr(arg, key) + if ( + not hasattr(obj, '__qualname__') + or obj.__qualname__ != f'{arg.__qualname__}.{obj.__name__}' + or getattr(obj, '__module__', None) != arg.__module__ + ): + # We only modify objects that are defined in this type directly. + # If classes / methods are nested in multiple layers, + # we will modify them when processing their direct holders. + continue + # Instance, class, and static methods: if isinstance(obj, types.FunctionType): obj.__no_type_check__ = True + if isinstance(obj, types.MethodType): + obj.__func__.__no_type_check__ = True + # Nested types: if isinstance(obj, type): no_type_check(obj) try: @@ -1995,7 +2624,8 @@ def no_type_check_decorator(decorator): This wraps the decorator with something that wraps the decorated function in @no_type_check. """ - + import warnings + warnings._deprecated("typing.no_type_check_decorator", remove=(3, 15)) @functools.wraps(decorator) def wrapped_decorator(*args, **kwds): func = decorator(*args, **kwds) @@ -2014,63 +2644,107 @@ def _overload_dummy(*args, **kwds): "by an implementation that is not @overload-ed.") +# {module: {qualname: {firstlineno: func}}} +_overload_registry = defaultdict(functools.partial(defaultdict, dict)) + + def overload(func): """Decorator for overloaded functions/methods. In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. For example: + function in a row, each decorated with @overload. + + For example:: - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... + @overload + def utf8(value: None) -> None: ... + @overload + def utf8(value: bytes) -> bytes: ... + @overload + def utf8(value: str) -> bytes: ... In a non-stub file (i.e. a regular .py file), do the same but follow it with an implementation. The implementation should *not* - be decorated with @overload. For example: - - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... - def utf8(value): - # implementation goes here + be decorated with @overload:: + + @overload + def utf8(value: None) -> None: ... + @overload + def utf8(value: bytes) -> bytes: ... + @overload + def utf8(value: str) -> bytes: ... + def utf8(value): + ... # implementation goes here + + The overloads for a function can be retrieved at runtime using the + get_overloads() function. """ + # classmethod and staticmethod + f = getattr(func, "__func__", func) + try: + _overload_registry[f.__module__][f.__qualname__][f.__code__.co_firstlineno] = func + except AttributeError: + # Not a normal function; ignore. + pass return _overload_dummy +def get_overloads(func): + """Return all defined overloads for *func* as a sequence.""" + # classmethod and staticmethod + f = getattr(func, "__func__", func) + if f.__module__ not in _overload_registry: + return [] + mod_dict = _overload_registry[f.__module__] + if f.__qualname__ not in mod_dict: + return [] + return list(mod_dict[f.__qualname__].values()) + + +def clear_overloads(): + """Clear all overloads in the registry.""" + _overload_registry.clear() + + def final(f): - """A decorator to indicate final methods and final classes. + """Decorator to indicate final methods and final classes. Use this decorator to indicate to type checkers that the decorated method cannot be overridden, and decorated class cannot be subclassed. - For example: - - class Base: - @final - def done(self) -> None: - ... - class Sub(Base): - def done(self) -> None: # Error reported by type checker + + For example:: + + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker ... - @final - class Leaf: - ... - class Other(Leaf): # Error reported by type checker - ... + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... - There is no runtime checking of these properties. + There is no runtime checking of these properties. The decorator + attempts to set the ``__final__`` attribute to ``True`` on the decorated + object to allow runtime introspection. """ + try: + f.__final__ = True + except (AttributeError, TypeError): + # Skip the attribute silently if it is not writable. + # AttributeError happens if the object has __slots__ or a + # read-only property, TypeError if it's a builtin class. + pass return f -# Some unconstrained type variables. These are used by the container types. -# (These are not for export.) +# Some unconstrained type variables. These were initially used by the container types. +# They were never meant for export and are now unused, but we keep them around to +# avoid breaking compatibility with users who import them. T = TypeVar('T') # Any type. KT = TypeVar('KT') # Key type. VT = TypeVar('VT') # Value type. @@ -2081,6 +2755,7 @@ class Other(Leaf): # Error reported by type checker # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) + # A useful type variable with constraints. This represents string types. # (This one *is* for export!) AnyStr = TypeVar('AnyStr', bytes, str) @@ -2102,13 +2777,17 @@ class Other(Leaf): # Error reported by type checker Collection = _alias(collections.abc.Collection, 1) Callable = _CallableType(collections.abc.Callable, 2) Callable.__doc__ = \ - """Callable type; Callable[[int], str] is a function of (int) -> str. + """Deprecated alias to collections.abc.Callable. + + Callable[[int], str] signifies a function that takes a single + parameter of type int and returns a str. The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or ellipsis; the return type must be a single type. + values: the argument list and the return type. + The argument list must be a list of types, a ParamSpec, + Concatenate or ellipsis. The return type must be a single type. - There is no syntax to indicate optional or keyword arguments, + There is no syntax to indicate optional or keyword arguments; such function types are rarely used as callback types. """ AbstractSet = _alias(collections.abc.Set, 1, name='AbstractSet') @@ -2118,11 +2797,15 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _alias(collections.abc.ByteString, 0) # Not generic +ByteString = _DeprecatedGenericAlias( + collections.abc.ByteString, 0, removal_version=(3, 17) # Not generic. +) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ - """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. + """Deprecated alias to builtins.tuple. + + Tuple[X, Y] is the cross-product type of X and Y. Example: Tuple[T1, T2] is a tuple of two elements corresponding to type variables T1 and T2. Tuple[int, float, str] is a tuple @@ -2138,36 +2821,34 @@ class Other(Leaf): # Error reported by type checker KeysView = _alias(collections.abc.KeysView, 1) ItemsView = _alias(collections.abc.ItemsView, 2) ValuesView = _alias(collections.abc.ValuesView, 1) -ContextManager = _alias(contextlib.AbstractContextManager, 1, name='ContextManager') -AsyncContextManager = _alias(contextlib.AbstractAsyncContextManager, 1, name='AsyncContextManager') Dict = _alias(dict, 2, inst=False, name='Dict') DefaultDict = _alias(collections.defaultdict, 2, name='DefaultDict') OrderedDict = _alias(collections.OrderedDict, 2) Counter = _alias(collections.Counter, 1) ChainMap = _alias(collections.ChainMap, 2) -Generator = _alias(collections.abc.Generator, 3) -AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2) +Generator = _alias(collections.abc.Generator, 3, defaults=(types.NoneType, types.NoneType)) +AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2, defaults=(types.NoneType,)) Type = _alias(type, 1, inst=False, name='Type') Type.__doc__ = \ - """A special construct usable to annotate class objects. + """Deprecated alias to builtins.type. + builtins.type or typing.Type can be used to annotate class objects. For example, suppose we have the following classes:: - class User: ... # Abstract base for User classes - class BasicUser(User): ... - class ProUser(User): ... - class TeamUser(User): ... + class User: ... # Abstract base for User classes + class BasicUser(User): ... + class ProUser(User): ... + class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - U = TypeVar('U', bound=User) - def new_user(user_class: Type[U]) -> U: - user = user_class() - # (Here we could write the user object to a database) - return user + def new_user[U](user_class: Type[U]) -> U: + user = user_class() + # (Here we could write the user object to a database) + return user - joe = new_user(BasicUser) + joe = new_user(BasicUser) At this point the type checker knows that joe has type BasicUser. """ @@ -2176,6 +2857,7 @@ def new_user(user_class: Type[U]) -> U: @runtime_checkable class SupportsInt(Protocol): """An ABC with one abstract method __int__.""" + __slots__ = () @abstractmethod @@ -2186,6 +2868,7 @@ def __int__(self) -> int: @runtime_checkable class SupportsFloat(Protocol): """An ABC with one abstract method __float__.""" + __slots__ = () @abstractmethod @@ -2196,6 +2879,7 @@ def __float__(self) -> float: @runtime_checkable class SupportsComplex(Protocol): """An ABC with one abstract method __complex__.""" + __slots__ = () @abstractmethod @@ -2206,6 +2890,7 @@ def __complex__(self) -> complex: @runtime_checkable class SupportsBytes(Protocol): """An ABC with one abstract method __bytes__.""" + __slots__ = () @abstractmethod @@ -2216,6 +2901,7 @@ def __bytes__(self) -> bytes: @runtime_checkable class SupportsIndex(Protocol): """An ABC with one abstract method __index__.""" + __slots__ = () @abstractmethod @@ -2224,50 +2910,94 @@ def __index__(self) -> int: @runtime_checkable -class SupportsAbs(Protocol[T_co]): +class SupportsAbs[T](Protocol): """An ABC with one abstract method __abs__ that is covariant in its return type.""" + __slots__ = () @abstractmethod - def __abs__(self) -> T_co: + def __abs__(self) -> T: pass @runtime_checkable -class SupportsRound(Protocol[T_co]): +class SupportsRound[T](Protocol): """An ABC with one abstract method __round__ that is covariant in its return type.""" + __slots__ = () @abstractmethod - def __round__(self, ndigits: int = 0) -> T_co: + def __round__(self, ndigits: int = 0) -> T: pass -def _make_nmtuple(name, types, module, defaults = ()): - fields = [n for n, t in types] - types = {n: _type_check(t, f"field {n} annotation must be a type") - for n, t in types} +def _make_nmtuple(name, fields, annotate_func, module, defaults = ()): nm_tpl = collections.namedtuple(name, fields, defaults=defaults, module=module) - nm_tpl.__annotations__ = nm_tpl.__new__.__annotations__ = types + nm_tpl.__annotate__ = nm_tpl.__new__.__annotate__ = annotate_func return nm_tpl +def _make_eager_annotate(types): + checked_types = {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in types.items()} + def annotate(format): + match format: + case _lazy_annotationlib.Format.VALUE | _lazy_annotationlib.Format.FORWARDREF: + return checked_types + case _lazy_annotationlib.Format.STRING: + return _lazy_annotationlib.annotations_to_string(types) + case _: + raise NotImplementedError(format) + return annotate + + # attributes prohibited to set in NamedTuple class syntax _prohibited = frozenset({'__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_make', '_replace', '_asdict', '_source'}) -_special = frozenset({'__module__', '__name__', '__annotations__'}) +_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__', + '__annotate_func__', '__annotations_cache__'}) class NamedTupleMeta(type): - def __new__(cls, typename, bases, ns): - assert bases[0] is _NamedTuple - types = ns.get('__annotations__', {}) + assert _NamedTuple in bases + if "__classcell__" in ns: + raise TypeError( + "uses of super() and __class__ are unsupported in methods of NamedTuple subclasses") + for base in bases: + if base is not _NamedTuple and base is not Generic: + raise TypeError( + 'can only inherit from a NamedTuple type and Generic') + bases = tuple(tuple if base is _NamedTuple else base for base in bases) + if "__annotations__" in ns: + types = ns["__annotations__"] + field_names = list(types) + annotate = _make_eager_annotate(types) + elif (original_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: + types = _lazy_annotationlib.call_annotate_function( + original_annotate, _lazy_annotationlib.Format.FORWARDREF) + field_names = list(types) + + # For backward compatibility, type-check all the types at creation time + for typ in types.values(): + _type_check(typ, "field annotation must be a type") + + def annotate(format): + annos = _lazy_annotationlib.call_annotate_function( + original_annotate, format) + if format != _lazy_annotationlib.Format.STRING: + return {key: _type_check(val, f"field {key} annotation must be a type") + for key, val in annos.items()} + return annos + else: + # Empty NamedTuple + field_names = [] + annotate = lambda format: {} default_names = [] - for field_name in types: + for field_name in field_names: if field_name in ns: default_names.append(field_name) elif default_names: @@ -2275,22 +3005,43 @@ def __new__(cls, typename, bases, ns): f"cannot follow default field" f"{'s' if len(default_names) > 1 else ''} " f"{', '.join(default_names)}") - nm_tpl = _make_nmtuple(typename, types.items(), + nm_tpl = _make_nmtuple(typename, field_names, annotate, defaults=[ns[n] for n in default_names], module=ns['__module__']) + nm_tpl.__bases__ = bases + if Generic in bases: + class_getitem = _generic_class_getitem + nm_tpl.__class_getitem__ = classmethod(class_getitem) # update from user namespace without overriding special namedtuple attributes - for key in ns: + for key, val in ns.items(): if key in _prohibited: raise AttributeError("Cannot overwrite NamedTuple attribute " + key) - elif key not in _special and key not in nm_tpl._fields: - setattr(nm_tpl, key, ns[key]) + elif key not in _special: + if key not in nm_tpl._fields: + setattr(nm_tpl, key, val) + try: + set_name = type(val).__set_name__ + except AttributeError: + pass + else: + try: + set_name(val, nm_tpl, key) + except BaseException as e: + e.add_note( + f"Error calling __set_name__ on {type(val).__name__!r} " + f"instance {key!r} in {typename!r}" + ) + raise + + if Generic in bases: + nm_tpl.__init_subclass__() return nm_tpl -def NamedTuple(typename, fields=None, /, **kwargs): +def NamedTuple(typename, fields=_sentinel, /, **kwargs): """Typed version of namedtuple. - Usage in Python versions >= 3.6:: + Usage:: class Employee(NamedTuple): name: str @@ -2303,39 +3054,90 @@ class Employee(NamedTuple): The resulting class has an extra __annotations__ attribute, giving a dict that maps field names to types. (The field names are also in the _fields attribute, which is part of the namedtuple API.) - Alternative equivalent keyword syntax is also accepted:: - - Employee = NamedTuple('Employee', name=str, id=int) - - In Python versions <= 3.5 use:: + An alternative equivalent functional syntax is also accepted:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) """ - if fields is None: - fields = kwargs.items() + if fields is _sentinel: + if kwargs: + deprecated_thing = "Creating NamedTuple classes using keyword arguments" + deprecation_msg = ( + "{name} is deprecated and will be disallowed in Python {remove}. " + "Use the class-based or functional syntax instead." + ) + else: + deprecated_thing = "Failing to pass a value for the 'fields' parameter" + example = f"`{typename} = NamedTuple({typename!r}, [])`" + deprecation_msg = ( + "{name} is deprecated and will be disallowed in Python {remove}. " + "To create a NamedTuple class with 0 fields " + "using the functional syntax, " + "pass an empty list, e.g. " + ) + example + "." + elif fields is None: + if kwargs: + raise TypeError( + "Cannot pass `None` as the 'fields' parameter " + "and also specify fields using keyword arguments" + ) + else: + deprecated_thing = "Passing `None` as the 'fields' parameter" + example = f"`{typename} = NamedTuple({typename!r}, [])`" + deprecation_msg = ( + "{name} is deprecated and will be disallowed in Python {remove}. " + "To create a NamedTuple class with 0 fields " + "using the functional syntax, " + "pass an empty list, e.g. " + ) + example + "." elif kwargs: raise TypeError("Either list of fields or keywords" " can be provided to NamedTuple, not both") - try: - module = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - module = None - return _make_nmtuple(typename, fields, module=module) + if fields is _sentinel or fields is None: + import warnings + warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) + fields = kwargs.items() + types = {n: _type_check(t, f"field {n} annotation must be a type") + for n, t in fields} + field_names = [n for n, _ in fields] + + nt = _make_nmtuple(typename, field_names, _make_eager_annotate(types), module=_caller()) + nt.__orig_bases__ = (NamedTuple,) + return nt _NamedTuple = type.__new__(NamedTupleMeta, 'NamedTuple', (), {}) def _namedtuple_mro_entries(bases): - if len(bases) > 1: - raise TypeError("Multiple inheritance with NamedTuple is not supported") - assert bases[0] is NamedTuple + assert NamedTuple in bases return (_NamedTuple,) NamedTuple.__mro_entries__ = _namedtuple_mro_entries +def _get_typeddict_qualifiers(annotation_type): + while True: + annotation_origin = get_origin(annotation_type) + if annotation_origin is Annotated: + annotation_args = get_args(annotation_type) + if annotation_args: + annotation_type = annotation_args[0] + else: + break + elif annotation_origin is Required: + yield Required + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is NotRequired: + yield NotRequired + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is ReadOnly: + yield ReadOnly + (annotation_type,) = get_args(annotation_type) + else: + break + + class _TypedDictMeta(type): def __new__(cls, name, bases, ns, total=True): - """Create new typed dict class object. + """Create a new typed dict class object. This method is called when TypedDict is subclassed, or when TypedDict is instantiated. This way @@ -2343,38 +3145,120 @@ def __new__(cls, name, bases, ns, total=True): Subclasses and instances of TypedDict return actual dictionaries. """ for base in bases: - if type(base) is not _TypedDictMeta: + if type(base) is not _TypedDictMeta and base is not Generic: raise TypeError('cannot inherit from both a TypedDict type ' 'and a non-TypedDict base class') - tp_dict = type.__new__(_TypedDictMeta, name, (dict,), ns) - annotations = {} - own_annotations = ns.get('__annotations__', {}) - own_annotation_keys = set(own_annotations.keys()) + if any(issubclass(b, Generic) for b in bases): + generic_base = (Generic,) + else: + generic_base = () + + ns_annotations = ns.pop('__annotations__', None) + + tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns) + + if not hasattr(tp_dict, '__orig_bases__'): + tp_dict.__orig_bases__ = bases + + if ns_annotations is not None: + own_annotate = None + own_annotations = ns_annotations + elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: + own_annotations = _lazy_annotationlib.call_annotate_function( + own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict + ) + else: + own_annotate = None + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - own_annotations = { - n: _type_check(tp, msg, module=tp_dict.__module__) + own_checked_annotations = { + n: _type_check(tp, msg, owner=tp_dict, module=tp_dict.__module__) for n, tp in own_annotations.items() } required_keys = set() optional_keys = set() + readonly_keys = set() + mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) - required_keys.update(base.__dict__.get('__required_keys__', ())) - optional_keys.update(base.__dict__.get('__optional_keys__', ())) + base_required = base.__dict__.get('__required_keys__', set()) + required_keys |= base_required + optional_keys -= base_required + + base_optional = base.__dict__.get('__optional_keys__', set()) + required_keys -= base_optional + optional_keys |= base_optional + + readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) + mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) + + for annotation_key, annotation_type in own_checked_annotations.items(): + qualifiers = set(_get_typeddict_qualifiers(annotation_type)) + if Required in qualifiers: + is_required = True + elif NotRequired in qualifiers: + is_required = False + else: + is_required = total - annotations.update(own_annotations) - if total: - required_keys.update(own_annotation_keys) - else: - optional_keys.update(own_annotation_keys) + if is_required: + required_keys.add(annotation_key) + optional_keys.discard(annotation_key) + else: + optional_keys.add(annotation_key) + required_keys.discard(annotation_key) + + if ReadOnly in qualifiers: + if annotation_key in mutable_keys: + raise TypeError( + f"Cannot override mutable key {annotation_key!r}" + " with read-only key" + ) + readonly_keys.add(annotation_key) + else: + mutable_keys.add(annotation_key) + readonly_keys.discard(annotation_key) + + assert required_keys.isdisjoint(optional_keys), ( + f"Required keys overlap with optional keys in {name}:" + f" {required_keys=}, {optional_keys=}" + ) + + def __annotate__(format): + annos = {} + for base in bases: + if base is Generic: + continue + base_annotate = base.__annotate__ + if base_annotate is None: + continue + base_annos = _lazy_annotationlib.call_annotate_function( + base_annotate, format, owner=base) + annos.update(base_annos) + if own_annotate is not None: + own = _lazy_annotationlib.call_annotate_function( + own_annotate, format, owner=tp_dict) + if format != _lazy_annotationlib.Format.STRING: + own = { + n: _type_check(tp, msg, module=tp_dict.__module__) + for n, tp in own.items() + } + elif format == _lazy_annotationlib.Format.STRING: + own = _lazy_annotationlib.annotations_to_string(own_annotations) + elif format in (_lazy_annotationlib.Format.FORWARDREF, _lazy_annotationlib.Format.VALUE): + own = own_checked_annotations + else: + raise NotImplementedError(format) + annos.update(own) + return annos - tp_dict.__annotations__ = annotations + tp_dict.__annotate__ = __annotate__ tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) - if not hasattr(tp_dict, '__total__'): - tp_dict.__total__ = total + tp_dict.__readonly_keys__ = frozenset(readonly_keys) + tp_dict.__mutable_keys__ = frozenset(mutable_keys) + tp_dict.__total__ = total return tp_dict __call__ = dict # static method @@ -2386,72 +3270,164 @@ def __subclasscheck__(cls, other): __instancecheck__ = __subclasscheck__ -def TypedDict(typename, fields=None, /, *, total=True, **kwargs): +def TypedDict(typename, fields=_sentinel, /, *, total=True): """A simple typed namespace. At runtime it is equivalent to a plain dict. - TypedDict creates a dictionary type that expects all of its + TypedDict creates a dictionary type such that a type checker will expect all instances to have a certain set of keys, where each key is associated with a value of a consistent type. This expectation - is not checked at runtime but is only enforced by type checkers. - Usage:: - - class Point2D(TypedDict): - x: int - y: int - label: str + is not checked at runtime. - a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK - b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + Usage:: - assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + >>> class Point2D(TypedDict): + ... x: int + ... y: int + ... label: str + ... + >>> a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK + >>> b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check + >>> Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + True The type info can be accessed via the Point2D.__annotations__ dict, and the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. - TypedDict supports two additional equivalent forms:: + TypedDict supports an additional equivalent form:: - Point2D = TypedDict('Point2D', x=int, y=int, label=str) Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality. - Usage:: + to override this by specifying totality:: - class point2D(TypedDict, total=False): + class Point2D(TypedDict, total=False): x: int y: int - This means that a point2D TypedDict can have any of the keys omitted.A type + This means that a Point2D TypedDict can have any of the keys omitted. A type checker is only expected to support a literal False or True as the value of the total argument. True is the default, and makes all items defined in the class body be required. - The class syntax is only supported in Python 3.6+, while two other - syntax forms work for Python 2.7 and 3.2+ + The Required and NotRequired special forms can also be used to mark + individual keys as being required or not required:: + + class Point2D(TypedDict): + x: int # the "x" key must always be present (Required is the default) + y: NotRequired[int] # the "y" key can be omitted + + See PEP 655 for more details on Required and NotRequired. + + The ReadOnly special form can be used + to mark individual keys as immutable for type checkers:: + + class DatabaseUser(TypedDict): + id: ReadOnly[int] # the "id" key must not be modified + username: str # the "username" key can be changed + """ - if fields is None: - fields = kwargs - elif kwargs: - raise TypeError("TypedDict takes either a dict or keyword arguments," - " but not both") + if fields is _sentinel or fields is None: + import warnings + + if fields is _sentinel: + deprecated_thing = "Failing to pass a value for the 'fields' parameter" + else: + deprecated_thing = "Passing `None` as the 'fields' parameter" + + example = f"`{typename} = TypedDict({typename!r}, {{{{}}}})`" + deprecation_msg = ( + "{name} is deprecated and will be disallowed in Python {remove}. " + "To create a TypedDict class with 0 fields " + "using the functional syntax, " + "pass an empty dictionary, e.g. " + ) + example + "." + warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) + fields = {} ns = {'__annotations__': dict(fields)} - try: + module = _caller() + if module is not None: # Setting correct module is necessary to make typed dict classes pickleable. - ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass + ns['__module__'] = module - return _TypedDictMeta(typename, (), ns, total=total) + td = _TypedDictMeta(typename, (), ns, total=total) + td.__orig_bases__ = (TypedDict,) + return td _TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {}) TypedDict.__mro_entries__ = lambda bases: (_TypedDict,) +@_SpecialForm +def Required(self, parameters): + """Special typing construct to mark a TypedDict key as required. + + This is mainly useful for total=False TypedDicts. + + For example:: + + class Movie(TypedDict, total=False): + title: Required[str] + year: int + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + + There is no runtime checking that a required key is actually provided + when instantiating a related TypedDict. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + +@_SpecialForm +def NotRequired(self, parameters): + """Special typing construct to mark a TypedDict key as potentially missing. + + For example:: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + +@_SpecialForm +def ReadOnly(self, parameters): + """A special typing construct to mark an item of a TypedDict as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1992 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + class NewType: - """NewType creates simple unique types with almost zero - runtime overhead. NewType(name, tp) is considered a subtype of tp + """NewType creates simple unique types with almost zero runtime overhead. + + NewType(name, tp) is considered a subtype of tp by static type checkers. At runtime, NewType(name, tp) returns - a dummy callable that simply returns its argument. Usage:: + a dummy callable that simply returns its argument. + + Usage:: UserId = NewType('UserId', int) @@ -2466,6 +3442,8 @@ def name_by_id(user_id: UserId) -> str: num = UserId(5) + 1 # type: int """ + __call__ = _idfunc + def __init__(self, name, tp): self.__qualname__ = name if '.' in name: @@ -2476,12 +3454,24 @@ def __init__(self, name, tp): if def_mod != 'typing': self.__module__ = def_mod + def __mro_entries__(self, bases): + # We defined __mro_entries__ to get a better error message + # if a user attempts to subclass a NewType instance. bpo-46170 + superclass_name = self.__name__ + + class Dummy: + def __init_subclass__(cls): + subclass_name = cls.__name__ + raise TypeError( + f"Cannot subclass an instance of NewType. Perhaps you were looking for: " + f"`{subclass_name} = NewType({subclass_name!r}, {superclass_name})`" + ) + + return (Dummy,) + def __repr__(self): return f'{self.__module__}.{self.__qualname__}' - def __call__(self, x): - return x - def __reduce__(self): return self.__qualname__ @@ -2559,7 +3549,7 @@ def readline(self, limit: int = -1) -> AnyStr: pass @abstractmethod - def readlines(self, hint: int = -1) -> List[AnyStr]: + def readlines(self, hint: int = -1) -> list[AnyStr]: pass @abstractmethod @@ -2575,7 +3565,7 @@ def tell(self) -> int: pass @abstractmethod - def truncate(self, size: int = None) -> int: + def truncate(self, size: int | None = None) -> int: pass @abstractmethod @@ -2587,11 +3577,11 @@ def write(self, s: AnyStr) -> int: pass @abstractmethod - def writelines(self, lines: List[AnyStr]) -> None: + def writelines(self, lines: list[AnyStr]) -> None: pass @abstractmethod - def __enter__(self) -> 'IO[AnyStr]': + def __enter__(self) -> IO[AnyStr]: pass @abstractmethod @@ -2605,11 +3595,11 @@ class BinaryIO(IO[bytes]): __slots__ = () @abstractmethod - def write(self, s: Union[bytes, bytearray]) -> int: + def write(self, s: bytes | bytearray) -> int: pass @abstractmethod - def __enter__(self) -> 'BinaryIO': + def __enter__(self) -> BinaryIO: pass @@ -2630,7 +3620,7 @@ def encoding(self) -> str: @property @abstractmethod - def errors(self) -> Optional[str]: + def errors(self) -> str | None: pass @property @@ -2644,32 +3634,221 @@ def newlines(self) -> Any: pass @abstractmethod - def __enter__(self) -> 'TextIO': + def __enter__(self) -> TextIO: + pass + + +def reveal_type[T](obj: T, /) -> T: + """Ask a static type checker to reveal the inferred type of an expression. + + When a static type checker encounters a call to ``reveal_type()``, + it will emit the inferred type of the argument:: + + x: int = 1 + reveal_type(x) + + Running a static type checker (e.g., mypy) on this example + will produce output similar to 'Revealed type is "builtins.int"'. + + At runtime, the function prints the runtime type of the + argument and returns the argument unchanged. + """ + print(f"Runtime type is {type(obj).__name__!r}", file=sys.stderr) + return obj + + +class _IdentityCallable(Protocol): + def __call__[T](self, arg: T, /) -> T: + ... + + +def dataclass_transform( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + frozen_default: bool = False, + field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), + **kwargs: Any, +) -> _IdentityCallable: + """Decorator to mark an object as providing dataclass-like behaviour. + + The decorator can be applied to a function, class, or metaclass. + + Example usage with a decorator function:: + + @dataclass_transform() + def create_model[T](cls: type[T]) -> type[T]: + ... + return cls + + @create_model + class CustomerModel: + id: int + name: str + + On a base class:: + + @dataclass_transform() + class ModelBase: ... + + class CustomerModel(ModelBase): + id: int + name: str + + On a metaclass:: + + @dataclass_transform() + class ModelMeta(type): ... + + class ModelBase(metaclass=ModelMeta): ... + + class CustomerModel(ModelBase): + id: int + name: str + + The ``CustomerModel`` classes defined above will + be treated by type checkers similarly to classes created with + ``@dataclasses.dataclass``. + For example, type checkers will assume these classes have + ``__init__`` methods that accept ``id`` and ``name``. + + The arguments to this decorator can be used to customize this behavior: + - ``eq_default`` indicates whether the ``eq`` parameter is assumed to be + ``True`` or ``False`` if it is omitted by the caller. + - ``order_default`` indicates whether the ``order`` parameter is + assumed to be True or False if it is omitted by the caller. + - ``kw_only_default`` indicates whether the ``kw_only`` parameter is + assumed to be True or False if it is omitted by the caller. + - ``frozen_default`` indicates whether the ``frozen`` parameter is + assumed to be True or False if it is omitted by the caller. + - ``field_specifiers`` specifies a static list of supported classes + or functions that describe fields, similar to ``dataclasses.field()``. + - Arbitrary other keyword arguments are accepted in order to allow for + possible future extensions. + + At runtime, this decorator records its arguments in the + ``__dataclass_transform__`` attribute on the decorated object. + It has no other runtime effect. + + See PEP 681 for more details. + """ + def decorator(cls_or_fn): + cls_or_fn.__dataclass_transform__ = { + "eq_default": eq_default, + "order_default": order_default, + "kw_only_default": kw_only_default, + "frozen_default": frozen_default, + "field_specifiers": field_specifiers, + "kwargs": kwargs, + } + return cls_or_fn + return decorator + + +type _Func = Callable[..., Any] + + +def override[F: _Func](method: F, /) -> F: + """Indicate that a method is intended to override a method in a base class. + + Usage:: + + class Base: + def method(self) -> None: + pass + + class Child(Base): + @override + def method(self) -> None: + super().method() + + When this decorator is applied to a method, the type checker will + validate that it overrides a method or attribute with the same name on a + base class. This helps prevent bugs that may occur when a base class is + changed without an equivalent change to a child class. + + There is no runtime checking of this property. The decorator attempts to + set the ``__override__`` attribute to ``True`` on the decorated object to + allow runtime introspection. + + See PEP 698 for details. + """ + try: + method.__override__ = True + except (AttributeError, TypeError): + # Skip the attribute silently if it is not writable. + # AttributeError happens if the object has __slots__ or a + # read-only property, TypeError if it's a builtin class. pass + return method + +def is_protocol(tp: type, /) -> bool: + """Return True if the given type is a Protocol. -class io: - """Wrapper namespace for IO generic classes.""" + Example:: + + >>> from typing import Protocol, is_protocol + >>> class P(Protocol): + ... def a(self) -> str: ... + ... b: int + >>> is_protocol(P) + True + >>> is_protocol(int) + False + """ + return ( + isinstance(tp, type) + and getattr(tp, '_is_protocol', False) + and tp != Protocol + ) - __all__ = ['IO', 'TextIO', 'BinaryIO'] - IO = IO - TextIO = TextIO - BinaryIO = BinaryIO +def get_protocol_members(tp: type, /) -> frozenset[str]: + """Return the set of members defined in a Protocol. -io.__name__ = __name__ + '.io' -sys.modules[io.__name__] = io + Example:: -Pattern = _alias(stdlib_re.Pattern, 1) -Match = _alias(stdlib_re.Match, 1) + >>> from typing import Protocol, get_protocol_members + >>> class P(Protocol): + ... def a(self) -> str: ... + ... b: int + >>> get_protocol_members(P) == frozenset({'a', 'b'}) + True -class re: - """Wrapper namespace for re type aliases.""" + Raise a TypeError for arguments that are not Protocols. + """ + if not is_protocol(tp): + raise TypeError(f'{tp!r} is not a Protocol') + return frozenset(tp.__protocol_attrs__) - __all__ = ['Pattern', 'Match'] - Pattern = Pattern - Match = Match +def __getattr__(attr): + """Improve the import time of the typing module. -re.__name__ = __name__ + '.re' -sys.modules[re.__name__] = re + Soft-deprecated objects which are costly to create + are only created on-demand here. + """ + if attr == "ForwardRef": + obj = _lazy_annotationlib.ForwardRef + elif attr in {"Pattern", "Match"}: + import re + obj = _alias(getattr(re, attr), 1) + elif attr in {"ContextManager", "AsyncContextManager"}: + import contextlib + obj = _alias(getattr(contextlib, f"Abstract{attr}"), 2, name=attr, defaults=(bool | None,)) + elif attr == "_collect_parameters": + import warnings + + depr_message = ( + "The private _collect_parameters function is deprecated and will be" + " removed in a future version of Python. Any use of private functions" + " is discouraged and may break in the future." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) + obj = _collect_type_parameters + else: + raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") + globals()[attr] = obj + return obj diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index 7ab3f5e87b4..b049402eed7 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -27,7 +27,7 @@ def testMultiply(self): http://docs.python.org/library/unittest.html Copyright (c) 1999-2003 Steve Purcell -Copyright (c) 2003-2010 Python Software Foundation +Copyright (c) 2003 Python Software Foundation This module is free software, and you may redistribute it and/or modify it under the same terms as Python itself, so long as this copyright message and disclaimer are retained in their original form. @@ -51,36 +51,33 @@ def testMultiply(self): 'registerResult', 'removeResult', 'removeHandler', 'addModuleCleanup', 'doModuleCleanups', 'enterModuleContext'] -# Expose obsolete functions for backwards compatibility -# bpo-5846: Deprecated in Python 3.11, scheduled for removal in Python 3.13. -__all__.extend(['getTestCaseNames', 'makeSuite', 'findTestCases']) - __unittest = True -from .result import TestResult -from .case import (addModuleCleanup, TestCase, FunctionTestCase, SkipTest, skip, - skipIf, skipUnless, expectedFailure, doModuleCleanups, - enterModuleContext) -from .suite import BaseTestSuite, TestSuite +from .case import ( + FunctionTestCase, + SkipTest, + TestCase, + addModuleCleanup, + doModuleCleanups, + enterModuleContext, + expectedFailure, + skip, + skipIf, + skipUnless, +) from .loader import TestLoader, defaultTestLoader -from .main import TestProgram, main -from .runner import TextTestRunner, TextTestResult -from .signals import installHandler, registerResult, removeResult, removeHandler -# IsolatedAsyncioTestCase will be imported lazily. -from .loader import makeSuite, getTestCaseNames, findTestCases - -# deprecated -_TextTestResult = TextTestResult - +from .main import TestProgram, main # noqa: F401 +from .result import TestResult +from .runner import TextTestResult, TextTestRunner +from .signals import ( + installHandler, + registerResult, + removeHandler, + removeResult, +) +from .suite import BaseTestSuite, TestSuite # noqa: F401 -# There are no tests here, so don't try to run anything discovered from -# introspecting the symbols (e.g. FunctionTestCase). Instead, all our -# tests come from within unittest.test. -def load_tests(loader, tests, pattern): - import os.path - # top level directory cached on loader instance - this_dir = os.path.dirname(__file__) - return loader.discover(start_dir=this_dir, pattern=pattern) +# IsolatedAsyncioTestCase will be imported lazily. # Lazy import of IsolatedAsyncioTestCase from .async_case @@ -98,6 +95,7 @@ def __getattr__(name): raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + # XXX: RUSTPYTHON # This is very useful to reduce platform difference boilerplates in tests. def expectedFailureIf(condition, reason): @@ -111,4 +109,4 @@ def expectedFailureIf(condition, reason): # Even more useful because most of them are windows only. def expectedFailureIfWindows(reason): import sys - return expectedFailureIf(sys.platform == 'win32', reason) + return expectedFailureIf(sys.platform == 'win32', reason) \ No newline at end of file diff --git a/Lib/unittest/__main__.py b/Lib/unittest/__main__.py index e5876f569b5..50111190eee 100644 --- a/Lib/unittest/__main__.py +++ b/Lib/unittest/__main__.py @@ -1,6 +1,7 @@ """Main entry point""" import sys + if sys.argv[0].endswith("__main__.py"): import os.path # We change sys.argv[0] to make help message more useful diff --git a/Lib/unittest/_log.py b/Lib/unittest/_log.py index 94868e5bb95..c61abb15745 100644 --- a/Lib/unittest/_log.py +++ b/Lib/unittest/_log.py @@ -1,9 +1,8 @@ -import logging import collections +import logging from .case import _BaseTestCaseContext - _LoggingWatcher = collections.namedtuple("_LoggingWatcher", ["records", "output"]) diff --git a/Lib/unittest/async_case.py b/Lib/unittest/async_case.py index bd2a4711560..a1c0d6c368c 100644 --- a/Lib/unittest/async_case.py +++ b/Lib/unittest/async_case.py @@ -5,6 +5,7 @@ from .case import TestCase +__unittest = True class IsolatedAsyncioTestCase(TestCase): # Names intentionally have a long prefix @@ -25,12 +26,15 @@ class IsolatedAsyncioTestCase(TestCase): # them inside the same task. # Note: the test case modifies event loop policy if the policy was not instantiated - # yet. + # yet, unless loop_factory=asyncio.EventLoop is set. # asyncio.get_event_loop_policy() creates a default policy on demand but never # returns None # I believe this is not an issue in user level tests but python itself for testing # should reset a policy in every test module # by calling asyncio.set_event_loop_policy(None) in tearDownModule() + # or set loop_factory=asyncio.EventLoop + + loop_factory = None def __init__(self, methodName='runTest'): super().__init__(methodName) @@ -71,9 +75,17 @@ async def enterAsyncContext(self, cm): enter = cls.__aenter__ exit = cls.__aexit__ except AttributeError: - raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " - f"not support the asynchronous context manager protocol" - ) from None + msg = (f"'{cls.__module__}.{cls.__qualname__}' object does " + "not support the asynchronous context manager protocol") + try: + cls.__enter__ + cls.__exit__ + except AttributeError: + pass + else: + msg += (" but it supports the context manager protocol. " + "Did you mean to use enterContext()?") + raise TypeError(msg) from None result = await enter(cm) self.addAsyncCleanup(exit, cm, None, None, None) return result @@ -87,9 +99,13 @@ def _callSetUp(self): self._callAsync(self.asyncSetUp) def _callTestMethod(self, method): - if self._callMaybeAsync(method) is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=4) + result = self._callMaybeAsync(method) + if result is not None: + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})', + ) + warnings.warn(msg, DeprecationWarning, stacklevel=4) def _callTearDown(self): self._callAsync(self.asyncTearDown) @@ -118,7 +134,7 @@ def _callMaybeAsync(self, func, /, *args, **kwargs): def _setupAsyncioRunner(self): assert self._asyncioRunner is None, 'asyncio runner is already initialized' - runner = asyncio.Runner(debug=True) + runner = asyncio.Runner(debug=True, loop_factory=self.loop_factory) self._asyncioRunner = runner def _tearDownAsyncioRunner(self): diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index c4aa2d77219..b09836d6747 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -1,19 +1,25 @@ """Test case implementation""" -import sys -import functools +import collections +import contextlib import difflib +import functools import pprint import re -import warnings -import collections -import contextlib +import sys +import time import traceback import types +import warnings from . import result -from .util import (strclass, safe_repr, _count_diff_all_purpose, - _count_diff_hashable, _common_shorten_repr) +from .util import ( + _common_shorten_repr, + _count_diff_all_purpose, + _count_diff_hashable, + safe_repr, + strclass, +) __unittest = True @@ -110,8 +116,17 @@ def _enter_context(cm, addcleanup): enter = cls.__enter__ exit = cls.__exit__ except AttributeError: - raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " - f"not support the context manager protocol") from None + msg = (f"'{cls.__module__}.{cls.__qualname__}' object does " + "not support the context manager protocol") + try: + cls.__aenter__ + cls.__aexit__ + except AttributeError: + pass + else: + msg += (" but it supports the asynchronous context manager " + "protocol. Did you mean to use enterAsyncContext()?") + raise TypeError(msg) from None result = enter(cm) addcleanup(exit, cm, None, None, None) return result @@ -331,6 +346,23 @@ def __exit__(self, exc_type, exc_value, tb): self._raiseFailure("{} not triggered".format(exc_name)) +class _AssertNotWarnsContext(_AssertWarnsContext): + + def __exit__(self, exc_type, exc_value, tb): + self.warnings_manager.__exit__(exc_type, exc_value, tb) + if exc_type is not None: + # let unexpected exceptions pass through + return + try: + exc_name = self.expected.__name__ + except AttributeError: + exc_name = str(self.expected) + for m in self.warnings: + w = m.message + if isinstance(w, self.expected): + self._raiseFailure(f"{exc_name} triggered") + + class _OrderedChainMap(collections.ChainMap): def __iter__(self): seen = set() @@ -572,13 +604,31 @@ def _addUnexpectedSuccess(self, result): else: addUnexpectedSuccess(self) + def _addDuration(self, result, elapsed): + try: + addDuration = result.addDuration + except AttributeError: + warnings.warn("TestResult has no addDuration method", + RuntimeWarning) + else: + addDuration(self, elapsed) + def _callSetUp(self): self.setUp() def _callTestMethod(self, method): - if method() is not None: - warnings.warn(f'It is deprecated to return a value that is not None from a ' - f'test case ({method})', DeprecationWarning, stacklevel=3) + result = method() + if result is not None: + import inspect + msg = ( + f'It is deprecated to return a value that is not None ' + f'from a test case ({method} returned {type(result).__name__!r})' + ) + if inspect.iscoroutine(result): + msg += ( + '. Maybe you forgot to use IsolatedAsyncioTestCase as the base class?' + ) + warnings.warn(msg, DeprecationWarning, stacklevel=3) def _callTearDown(self): self.tearDown() @@ -612,6 +662,7 @@ def run(self, result=None): getattr(testMethod, "__unittest_expecting_failure__", False) ) outcome = _Outcome(result) + start_time = time.perf_counter() try: self._outcome = outcome @@ -625,6 +676,7 @@ def run(self, result=None): with outcome.testPartExecutor(self): self._callTearDown() self.doCleanups() + self._addDuration(result, (time.perf_counter() - start_time)) if outcome.success: if expecting_failure: @@ -799,6 +851,11 @@ def assertWarns(self, expected_warning, *args, **kwargs): context = _AssertWarnsContext(expected_warning, self) return context.handle('assertWarns', args, kwargs) + def _assertNotWarns(self, expected_warning, *args, **kwargs): + """The opposite of assertWarns. Private due to low demand.""" + context = _AssertNotWarnsContext(expected_warning, self) + return context.handle('_assertNotWarns', args, kwargs) + def assertLogs(self, logger=None, level=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to @@ -1171,35 +1228,6 @@ def assertDictEqual(self, d1, d2, msg=None): standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) - def assertDictContainsSubset(self, subset, dictionary, msg=None): - """Checks whether dictionary is a superset of subset.""" - warnings.warn('assertDictContainsSubset is deprecated', - DeprecationWarning) - missing = [] - mismatched = [] - for key, value in subset.items(): - if key not in dictionary: - missing.append(key) - elif value != dictionary[key]: - mismatched.append('%s, expected: %s, actual: %s' % - (safe_repr(key), safe_repr(value), - safe_repr(dictionary[key]))) - - if not (missing or mismatched): - return - - standardMsg = '' - if missing: - standardMsg = 'Missing: %s' % ','.join(safe_repr(m) for m in - missing) - if mismatched: - if standardMsg: - standardMsg += '; ' - standardMsg += 'Mismatched values: %s' % ','.join(mismatched) - - self.fail(self._formatMessage(msg, standardMsg)) - - def assertCountEqual(self, first, second, msg=None): """Asserts that two iterables have the same elements, the same number of times, without regard to order. @@ -1234,19 +1262,34 @@ def assertCountEqual(self, first, second, msg=None): def assertMultiLineEqual(self, first, second, msg=None): """Assert that two multi-line strings are equal.""" - self.assertIsInstance(first, str, 'First argument is not a string') - self.assertIsInstance(second, str, 'Second argument is not a string') + self.assertIsInstance(first, str, "First argument is not a string") + self.assertIsInstance(second, str, "Second argument is not a string") if first != second: - # don't use difflib if the strings are too long + # Don't use difflib if the strings are too long if (len(first) > self._diffThreshold or len(second) > self._diffThreshold): self._baseAssertEqual(first, second, msg) - firstlines = first.splitlines(keepends=True) - secondlines = second.splitlines(keepends=True) - if len(firstlines) == 1 and first.strip('\r\n') == first: - firstlines = [first + '\n'] - secondlines = [second + '\n'] + + # Append \n to both strings if either is missing the \n. + # This allows the final ndiff to show the \n difference. The + # exception here is if the string is empty, in which case no + # \n should be added + first_presplit = first + second_presplit = second + if first and second: + if first[-1] != '\n' or second[-1] != '\n': + first_presplit += '\n' + second_presplit += '\n' + elif second and second[-1] != '\n': + second_presplit += '\n' + elif first and first[-1] != '\n': + first_presplit += '\n' + + firstlines = first_presplit.splitlines(keepends=True) + secondlines = second_presplit.splitlines(keepends=True) + + # Generate the message and diff, then raise the exception standardMsg = '%s != %s' % _common_shorten_repr(first, second) diff = '\n' + ''.join(difflib.ndiff(firstlines, secondlines)) standardMsg = self._truncateMessage(standardMsg, diff) @@ -1292,13 +1335,71 @@ def assertIsInstance(self, obj, cls, msg=None): """Same as self.assertTrue(isinstance(obj, cls)), with a nicer default message.""" if not isinstance(obj, cls): - standardMsg = '%s is not an instance of %r' % (safe_repr(obj), cls) + if isinstance(cls, tuple): + standardMsg = f'{safe_repr(obj)} is not an instance of any of {cls!r}' + else: + standardMsg = f'{safe_repr(obj)} is not an instance of {cls!r}' self.fail(self._formatMessage(msg, standardMsg)) def assertNotIsInstance(self, obj, cls, msg=None): """Included for symmetry with assertIsInstance.""" if isinstance(obj, cls): - standardMsg = '%s is an instance of %r' % (safe_repr(obj), cls) + if isinstance(cls, tuple): + for x in cls: + if isinstance(obj, x): + cls = x + break + standardMsg = f'{safe_repr(obj)} is an instance of {cls!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertIsSubclass(self, cls, superclass, msg=None): + try: + if issubclass(cls, superclass): + return + except TypeError: + if not isinstance(cls, type): + self.fail(self._formatMessage(msg, f'{cls!r} is not a class')) + raise + if isinstance(superclass, tuple): + standardMsg = f'{cls!r} is not a subclass of any of {superclass!r}' + else: + standardMsg = f'{cls!r} is not a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotIsSubclass(self, cls, superclass, msg=None): + try: + if not issubclass(cls, superclass): + return + except TypeError: + if not isinstance(cls, type): + self.fail(self._formatMessage(msg, f'{cls!r} is not a class')) + raise + if isinstance(superclass, tuple): + for x in superclass: + if issubclass(cls, x): + superclass = x + break + standardMsg = f'{cls!r} is a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertHasAttr(self, obj, name, msg=None): + if not hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has no attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has no attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has no attribute {name!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotHasAttr(self, obj, name, msg=None): + if hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has unexpected attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has unexpected attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has unexpected attribute {name!r}' self.fail(self._formatMessage(msg, standardMsg)) def assertRaisesRegex(self, expected_exception, expected_regex, @@ -1362,27 +1463,80 @@ def assertNotRegex(self, text, unexpected_regex, msg=None): msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) + def _tail_type_check(self, s, tails, msg): + if not isinstance(tails, tuple): + tails = (tails,) + for tail in tails: + if isinstance(tail, str): + if not isinstance(s, str): + self.fail(self._formatMessage(msg, + f'Expected str, not {type(s).__name__}')) + elif isinstance(tail, (bytes, bytearray)): + if not isinstance(s, (bytes, bytearray)): + self.fail(self._formatMessage(msg, + f'Expected bytes, not {type(s).__name__}')) + + def assertStartsWith(self, s, prefix, msg=None): + try: + if s.startswith(prefix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, prefix, msg) + raise + a = safe_repr(s, short=True) + b = safe_repr(prefix) + if isinstance(prefix, tuple): + standardMsg = f"{a} doesn't start with any of {b}" + else: + standardMsg = f"{a} doesn't start with {b}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotStartsWith(self, s, prefix, msg=None): + try: + if not s.startswith(prefix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, prefix, msg) + raise + if isinstance(prefix, tuple): + for x in prefix: + if s.startswith(x): + prefix = x + break + a = safe_repr(s, short=True) + b = safe_repr(prefix) + self.fail(self._formatMessage(msg, f"{a} starts with {b}")) - def _deprecate(original_func): - def deprecated_func(*args, **kwargs): - warnings.warn( - 'Please use {0} instead.'.format(original_func.__name__), - DeprecationWarning, 2) - return original_func(*args, **kwargs) - return deprecated_func - - # see #9424 - failUnlessEqual = assertEquals = _deprecate(assertEqual) - failIfEqual = assertNotEquals = _deprecate(assertNotEqual) - failUnlessAlmostEqual = assertAlmostEquals = _deprecate(assertAlmostEqual) - failIfAlmostEqual = assertNotAlmostEquals = _deprecate(assertNotAlmostEqual) - failUnless = assert_ = _deprecate(assertTrue) - failUnlessRaises = _deprecate(assertRaises) - failIf = _deprecate(assertFalse) - assertRaisesRegexp = _deprecate(assertRaisesRegex) - assertRegexpMatches = _deprecate(assertRegex) - assertNotRegexpMatches = _deprecate(assertNotRegex) + def assertEndsWith(self, s, suffix, msg=None): + try: + if s.endswith(suffix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, suffix, msg) + raise + a = safe_repr(s, short=True) + b = safe_repr(suffix) + if isinstance(suffix, tuple): + standardMsg = f"{a} doesn't end with any of {b}" + else: + standardMsg = f"{a} doesn't end with {b}" + self.fail(self._formatMessage(msg, standardMsg)) + def assertNotEndsWith(self, s, suffix, msg=None): + try: + if not s.endswith(suffix): + return + except (AttributeError, TypeError): + self._tail_type_check(s, suffix, msg) + raise + if isinstance(suffix, tuple): + for x in suffix: + if s.endswith(x): + suffix = x + break + a = safe_repr(s, short=True) + b = safe_repr(suffix) + self.fail(self._formatMessage(msg, f"{a} ends with {b}")) class FunctionTestCase(TestCase): diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 7e6ce2f224b..fa8d647ad8a 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -1,13 +1,11 @@ """Loading unittests.""" +import functools import os import re import sys import traceback import types -import functools -import warnings - from fnmatch import fnmatch, fnmatchcase from . import case, suite, util @@ -57,9 +55,7 @@ def testSkipped(self): TestClass = type("ModuleSkipped", (case.TestCase,), attrs) return suiteClass((TestClass(methodname),)) -def _jython_aware_splitext(path): - if path.lower().endswith('$py.class'): - return path[:-9] +def _splitext(path): return os.path.splitext(path)[0] @@ -87,40 +83,26 @@ def loadTestsFromTestCase(self, testCaseClass): raise TypeError("Test cases should not be derived from " "TestSuite. Maybe you meant to derive from " "TestCase?") - testCaseNames = self.getTestCaseNames(testCaseClass) - if not testCaseNames and hasattr(testCaseClass, 'runTest'): - testCaseNames = ['runTest'] + if testCaseClass in (case.TestCase, case.FunctionTestCase): + # We don't load any tests from base types that should not be loaded. + testCaseNames = [] + else: + testCaseNames = self.getTestCaseNames(testCaseClass) + if not testCaseNames and hasattr(testCaseClass, 'runTest'): + testCaseNames = ['runTest'] loaded_suite = self.suiteClass(map(testCaseClass, testCaseNames)) return loaded_suite - # XXX After Python 3.5, remove backward compatibility hacks for - # use_load_tests deprecation via *args and **kws. See issue 16662. - def loadTestsFromModule(self, module, *args, pattern=None, **kws): + def loadTestsFromModule(self, module, *, pattern=None): """Return a suite of all test cases contained in the given module""" - # This method used to take an undocumented and unofficial - # use_load_tests argument. For backward compatibility, we still - # accept the argument (which can also be the first position) but we - # ignore it and issue a deprecation warning if it's present. - if len(args) > 0 or 'use_load_tests' in kws: - warnings.warn('use_load_tests is deprecated and ignored', - DeprecationWarning) - kws.pop('use_load_tests', None) - if len(args) > 1: - # Complain about the number of arguments, but don't forget the - # required `module` argument. - complaint = len(args) + 1 - raise TypeError('loadTestsFromModule() takes 1 positional argument but {} were given'.format(complaint)) - if len(kws) != 0: - # Since the keyword arguments are unsorted (see PEP 468), just - # pick the alphabetically sorted first argument to complain about, - # if multiple were given. At least the error message will be - # predictable. - complaint = sorted(kws)[0] - raise TypeError("loadTestsFromModule() got an unexpected keyword argument '{}'".format(complaint)) tests = [] for name in dir(module): obj = getattr(module, name) - if isinstance(obj, type) and issubclass(obj, case.TestCase): + if ( + isinstance(obj, type) + and issubclass(obj, case.TestCase) + and obj not in (case.TestCase, case.FunctionTestCase) + ): tests.append(self.loadTestsFromTestCase(obj)) load_tests = getattr(module, 'load_tests', None) @@ -189,7 +171,11 @@ def loadTestsFromName(self, name, module=None): if isinstance(obj, types.ModuleType): return self.loadTestsFromModule(obj) - elif isinstance(obj, type) and issubclass(obj, case.TestCase): + elif ( + isinstance(obj, type) + and issubclass(obj, case.TestCase) + and obj not in (case.TestCase, case.FunctionTestCase) + ): return self.loadTestsFromTestCase(obj) elif (isinstance(obj, types.FunctionType) and isinstance(parent, type) and @@ -267,6 +253,7 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): Paths are sorted before being imported to ensure reproducible execution order even on filesystems with non-alphabetical ordering like ext3/4. """ + original_top_level_dir = self._top_level_dir set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: # make top_level_dir optional if called from load_tests in a package @@ -286,6 +273,8 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): self._top_level_dir = top_level_dir is_not_importable = False + is_namespace = False + tests = [] if os.path.isdir(os.path.abspath(start_dir)): start_dir = os.path.abspath(start_dir) if start_dir != top_level_dir: @@ -298,12 +287,25 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): is_not_importable = True else: the_module = sys.modules[start_dir] - top_part = start_dir.split('.')[0] - try: - start_dir = os.path.abspath( - os.path.dirname((the_module.__file__))) - except AttributeError: - if the_module.__name__ in sys.builtin_module_names: + if not hasattr(the_module, "__file__") or the_module.__file__ is None: + # look for namespace packages + try: + spec = the_module.__spec__ + except AttributeError: + spec = None + + if spec and spec.submodule_search_locations is not None: + is_namespace = True + + for path in the_module.__path__: + if (not set_implicit_top and + not path.startswith(top_level_dir)): + continue + self._top_level_dir = \ + (path.split(the_module.__name__ + .replace(".", os.path.sep))[0]) + tests.extend(self._find_tests(path, pattern, namespace=True)) + elif the_module.__name__ in sys.builtin_module_names: # builtin module raise TypeError('Can not use builtin modules ' 'as dotted module names') from None @@ -312,14 +314,28 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None): f"don't know how to discover from {the_module!r}" ) from None + else: + top_part = start_dir.split('.')[0] + start_dir = os.path.abspath(os.path.dirname((the_module.__file__))) + if set_implicit_top: - self._top_level_dir = self._get_directory_containing_module(top_part) + if not is_namespace: + if sys.modules[top_part].__file__ is None: + self._top_level_dir = os.path.dirname(the_module.__file__) + if self._top_level_dir not in sys.path: + sys.path.insert(0, self._top_level_dir) + else: + self._top_level_dir = \ + self._get_directory_containing_module(top_part) sys.path.remove(top_level_dir) if is_not_importable: raise ImportError('Start directory is not importable: %r' % start_dir) - tests = list(self._find_tests(start_dir, pattern)) + if not is_namespace: + tests = list(self._find_tests(start_dir, pattern)) + + self._top_level_dir = original_top_level_dir return self.suiteClass(tests) def _get_directory_containing_module(self, module_name): @@ -337,7 +353,7 @@ def _get_directory_containing_module(self, module_name): def _get_name_from_path(self, path): if path == self._top_level_dir: return '.' - path = _jython_aware_splitext(os.path.normpath(path)) + path = _splitext(os.path.normpath(path)) _relpath = os.path.relpath(path, self._top_level_dir) assert not os.path.isabs(_relpath), "Path must be within the project" @@ -354,7 +370,7 @@ def _match_path(self, path, full_path, pattern): # override this method to use alternative matching strategy return fnmatch(path, pattern) - def _find_tests(self, start_dir, pattern): + def _find_tests(self, start_dir, pattern, namespace=False): """Used by discovery. Yields test suites it loads.""" # Handle the __init__ in this package name = self._get_name_from_path(start_dir) @@ -363,7 +379,8 @@ def _find_tests(self, start_dir, pattern): if name != '.' and name not in self._loading_packages: # name is in self._loading_packages while we have called into # loadTestsFromModule with name. - tests, should_recurse = self._find_test_path(start_dir, pattern) + tests, should_recurse = self._find_test_path( + start_dir, pattern, namespace) if tests is not None: yield tests if not should_recurse: @@ -374,7 +391,8 @@ def _find_tests(self, start_dir, pattern): paths = sorted(os.listdir(start_dir)) for path in paths: full_path = os.path.join(start_dir, path) - tests, should_recurse = self._find_test_path(full_path, pattern) + tests, should_recurse = self._find_test_path( + full_path, pattern, False) if tests is not None: yield tests if should_recurse: @@ -382,11 +400,11 @@ def _find_tests(self, start_dir, pattern): name = self._get_name_from_path(full_path) self._loading_packages.add(name) try: - yield from self._find_tests(full_path, pattern) + yield from self._find_tests(full_path, pattern, False) finally: self._loading_packages.discard(name) - def _find_test_path(self, full_path, pattern): + def _find_test_path(self, full_path, pattern, namespace=False): """Used by discovery. Loads tests from a single file, or a directories' __init__.py when @@ -415,13 +433,13 @@ def _find_test_path(self, full_path, pattern): else: mod_file = os.path.abspath( getattr(module, '__file__', full_path)) - realpath = _jython_aware_splitext( + realpath = _splitext( os.path.realpath(mod_file)) - fullpath_noext = _jython_aware_splitext( + fullpath_noext = _splitext( os.path.realpath(full_path)) if realpath.lower() != fullpath_noext.lower(): module_dir = os.path.dirname(realpath) - mod_name = _jython_aware_splitext( + mod_name = _splitext( os.path.basename(full_path)) expected_dir = os.path.dirname(full_path) msg = ("%r module incorrectly imported from %r. Expected " @@ -430,7 +448,8 @@ def _find_test_path(self, full_path, pattern): msg % (mod_name, module_dir, expected_dir)) return self.loadTestsFromModule(module, pattern=pattern), False elif os.path.isdir(full_path): - if not os.path.isfile(os.path.join(full_path, '__init__.py')): + if (not namespace and + not os.path.isfile(os.path.join(full_path, '__init__.py'))): return None, False load_tests = None @@ -462,47 +481,3 @@ def _find_test_path(self, full_path, pattern): defaultTestLoader = TestLoader() - - -# These functions are considered obsolete for long time. -# They will be removed in Python 3.13. - -def _makeLoader(prefix, sortUsing, suiteClass=None, testNamePatterns=None): - loader = TestLoader() - loader.sortTestMethodsUsing = sortUsing - loader.testMethodPrefix = prefix - loader.testNamePatterns = testNamePatterns - if suiteClass: - loader.suiteClass = suiteClass - return loader - -def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp, testNamePatterns=None): - import warnings - warnings.warn( - "unittest.getTestCaseNames() is deprecated and will be removed in Python 3.13. " - "Please use unittest.TestLoader.getTestCaseNames() instead.", - DeprecationWarning, stacklevel=2 - ) - return _makeLoader(prefix, sortUsing, testNamePatterns=testNamePatterns).getTestCaseNames(testCaseClass) - -def makeSuite(testCaseClass, prefix='test', sortUsing=util.three_way_cmp, - suiteClass=suite.TestSuite): - import warnings - warnings.warn( - "unittest.makeSuite() is deprecated and will be removed in Python 3.13. " - "Please use unittest.TestLoader.loadTestsFromTestCase() instead.", - DeprecationWarning, stacklevel=2 - ) - return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromTestCase( - testCaseClass) - -def findTestCases(module, prefix='test', sortUsing=util.three_way_cmp, - suiteClass=suite.TestSuite): - import warnings - warnings.warn( - "unittest.findTestCases() is deprecated and will be removed in Python 3.13. " - "Please use unittest.TestLoader.loadTestsFromModule() instead.", - DeprecationWarning, stacklevel=2 - ) - return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromModule(\ - module) diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 046fbd3a45d..1855fccf336 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -1,14 +1,14 @@ """Unittest main program""" -import sys import argparse import os -import warnings +import sys from . import loader, runner from .signals import installHandler __unittest = True +_NO_TESTS_EXITCODE = 5 MAIN_EXAMPLES = """\ Examples: @@ -66,7 +66,8 @@ class TestProgram(object): def __init__(self, module='__main__', defaultTest=None, argv=None, testRunner=None, testLoader=loader.defaultTestLoader, exit=True, verbosity=1, failfast=None, catchbreak=None, - buffer=None, warnings=None, *, tb_locals=False): + buffer=None, warnings=None, *, tb_locals=False, + durations=None): if isinstance(module, str): self.module = __import__(module) for part in module.split('.')[1:]: @@ -82,6 +83,7 @@ def __init__(self, module='__main__', defaultTest=None, argv=None, self.verbosity = verbosity self.buffer = buffer self.tb_locals = tb_locals + self.durations = durations if warnings is None and not sys.warnoptions: # even if DeprecationWarnings are ignored by default # print them anyway unless other warnings settings are @@ -101,16 +103,6 @@ def __init__(self, module='__main__', defaultTest=None, argv=None, self.parseArgs(argv) self.runTests() - def usageExit(self, msg=None): - warnings.warn("TestProgram.usageExit() is deprecated and will be" - " removed in Python 3.13", DeprecationWarning) - if msg: - print(msg) - if self._discovery_parser is None: - self._initArgParsers() - self._print_help() - sys.exit(2) - def _print_help(self, *args, **kwargs): if self.module is None: print(self._main_parser.format_help()) @@ -178,6 +170,9 @@ def _getParentArgParser(self): parser.add_argument('--locals', dest='tb_locals', action='store_true', help='Show local variables in tracebacks') + parser.add_argument('--durations', dest='durations', type=int, + default=None, metavar="N", + help='Show the N slowest test cases (N=0 for all)') if self.failfast is None: parser.add_argument('-f', '--failfast', dest='failfast', action='store_true', @@ -202,7 +197,7 @@ def _getParentArgParser(self): return parser def _getMainArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent]) + parser = argparse.ArgumentParser(parents=[parent], color=True) parser.prog = self.progName parser.print_help = self._print_help @@ -213,7 +208,7 @@ def _getMainArgParser(self, parent): return parser def _getDiscoveryArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent]) + parser = argparse.ArgumentParser(parents=[parent], color=True) parser.prog = '%s discover' % self.progName parser.epilog = ('For test discovery all test modules must be ' 'importable from the top level directory of the ' @@ -258,9 +253,10 @@ def runTests(self): failfast=self.failfast, buffer=self.buffer, warnings=self.warnings, - tb_locals=self.tb_locals) + tb_locals=self.tb_locals, + durations=self.durations) except TypeError: - # didn't accept the tb_locals argument + # didn't accept the tb_locals or durations argument testRunner = self.testRunner(verbosity=self.verbosity, failfast=self.failfast, buffer=self.buffer, @@ -273,6 +269,12 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - sys.exit(not self.result.wasSuccessful()) + if not self.result.wasSuccessful(): + sys.exit(1) + elif self.result.testsRun == 0 and len(self.result.skipped) == 0: + sys.exit(_NO_TESTS_EXITCODE) + else: + sys.exit(0) + main = TestProgram diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index fa0bd9131a2..1089dcb11f1 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -14,6 +14,7 @@ 'call', 'create_autospec', 'AsyncMock', + 'ThreadingMock', 'FILTER_DIR', 'NonCallableMock', 'NonCallableMagicMock', @@ -24,18 +25,20 @@ import asyncio +import builtins import contextlib -import io import inspect +import io +import pkgutil import pprint import sys -import builtins -import pkgutil -from asyncio import iscoroutinefunction -from types import CodeType, ModuleType, MethodType -from unittest.util import safe_repr -from functools import wraps, partial +import threading +from dataclasses import fields, is_dataclass +from functools import partial, wraps +from inspect import iscoroutinefunction from threading import RLock +from types import CodeType, MethodType, ModuleType +from unittest.util import safe_repr class InvalidSpecError(Exception): @@ -98,6 +101,12 @@ def _get_signature_object(func, as_instance, eat_self): func = func.__init__ # Skip the `self` argument in __init__ eat_self = True + elif isinstance(func, (classmethod, staticmethod)): + if isinstance(func, classmethod): + # Skip the `cls` argument of a class method + eat_self = True + # Use the original decorated method to extract the correct function signature + func = func.__func__ elif not isinstance(func, FunctionTypes): # If we really want to model an instance of the passed type, # __call__ should be looked up, not __init__. @@ -198,6 +207,28 @@ def checksig(*args, **kwargs): _setup_func(funcopy, mock, sig) return funcopy +def _set_async_signature(mock, original, instance=False, is_async_mock=False): + # creates an async function with signature (*args, **kwargs) that delegates to a + # mock. It still does signature checking by calling a lambda with the same + # signature as the original. + + skipfirst = isinstance(original, type) + func, sig = _get_signature_object(original, instance, skipfirst) + def checksig(*args, **kwargs): + sig.bind(*args, **kwargs) + _copy_func_details(func, checksig) + + name = original.__name__ + context = {'_checksig_': checksig, 'mock': mock} + src = """async def %s(*args, **kwargs): + _checksig_(*args, **kwargs) + return await mock(*args, **kwargs)""" % name + exec (src, context) + funcopy = context[name] + _setup_func(funcopy, mock, sig) + _setup_async_mock(funcopy) + return funcopy + def _setup_func(funcopy, mock, sig): funcopy.mock = mock @@ -411,15 +442,18 @@ class NonCallableMock(Base): # necessary. _lock = RLock() - def __new__(cls, /, *args, **kw): + def __new__( + cls, spec=None, wraps=None, name=None, spec_set=None, + parent=None, _spec_state=None, _new_name='', _new_parent=None, + _spec_as_instance=False, _eat_self=None, unsafe=False, **kwargs + ): # every instance has its own class # so we can create magic methods on the # class without stomping on other mocks bases = (cls,) if not issubclass(cls, AsyncMockMixin): # Check if spec is an async object or function - bound_args = _MOCK_SIG.bind_partial(cls, *args, **kw).arguments - spec_arg = bound_args.get('spec_set', bound_args.get('spec')) + spec_arg = spec_set or spec if spec_arg is not None and _is_async_obj(spec_arg): bases = (AsyncMockMixin, cls) new = type(cls.__name__, bases, {'__doc__': cls.__doc__}) @@ -505,10 +539,6 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_signature = None _spec_asyncs = [] - for attr in dir(spec): - if iscoroutinefunction(getattr(spec, attr, None)): - _spec_asyncs.append(attr) - if spec is not None and not _is_list(spec): if isinstance(spec, type): _spec_class = spec @@ -518,7 +548,19 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_as_instance, _eat_self) _spec_signature = res and res[1] - spec = dir(spec) + spec_list = dir(spec) + + for attr in spec_list: + static_attr = inspect.getattr_static(spec, attr, None) + unwrapped_attr = static_attr + try: + unwrapped_attr = inspect.unwrap(unwrapped_attr) + except ValueError: + pass + if iscoroutinefunction(unwrapped_attr): + _spec_asyncs.append(attr) + + spec = spec_list __dict__ = self.__dict__ __dict__['_spec_class'] = _spec_class @@ -527,12 +569,17 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, __dict__['_mock_methods'] = spec __dict__['_spec_asyncs'] = _spec_asyncs + def _mock_extend_spec_methods(self, spec_methods): + methods = self.__dict__.get('_mock_methods') or [] + methods.extend(spec_methods) + self.__dict__['_mock_methods'] = methods + def __get_return_value(self): ret = self._mock_return_value if self._mock_delegate is not None: ret = self._mock_delegate.return_value - if ret is DEFAULT: + if ret is DEFAULT and self._mock_wraps is None: ret = self._get_child_mock( _new_parent=self, _new_name='()' ) @@ -587,7 +634,9 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None,*, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, + return_value: bool = False, + side_effect: bool = False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -648,7 +697,7 @@ def __getattr__(self, name): elif _is_magic(name): raise AttributeError(name) if not self._mock_unsafe and (not self._mock_methods or name not in self._mock_methods): - if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')): + if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')) or name in _ATTRIB_DENY_LIST: raise AttributeError( f"{name!r} is not a valid assertion. Use a spec " f"for the mock if {name!r} is meant to be an attribute.") @@ -789,6 +838,9 @@ def __setattr__(self, name, value): mock_name = f'{self._extract_mock_name()}.{name}' raise AttributeError(f'Cannot set {mock_name}') + if isinstance(value, PropertyMock): + self.__dict__[name] = value + return return object.__setattr__(self, name, value) @@ -816,7 +868,7 @@ def _format_mock_call_signature(self, args, kwargs): def _format_mock_failure_message(self, args, kwargs, action='call'): - message = 'expected %s not found.\nExpected: %s\nActual: %s' + message = 'expected %s not found.\nExpected: %s\n Actual: %s' expected_string = self._format_mock_call_signature(args, kwargs) call_args = self.call_args actual_string = self._format_mock_call_signature(*call_args) @@ -919,7 +971,7 @@ def assert_called_with(self, /, *args, **kwargs): if self.call_args is None: expected = self._format_mock_call_signature(args, kwargs) actual = 'not called.' - error_message = ('expected call not found.\nExpected: %s\nActual: %s' + error_message = ('expected call not found.\nExpected: %s\n Actual: %s' % (expected, actual)) raise AssertionError(error_message) @@ -969,8 +1021,8 @@ def assert_has_calls(self, calls, any_order=False): for e in expected]) raise AssertionError( f'{problem}\n' - f'Expected: {_CallList(calls)}' - f'{self._calls_repr(prefix="Actual").rstrip(".")}' + f'Expected: {_CallList(calls)}\n' + f' Actual: {safe_repr(self.mock_calls)}' ) from cause return @@ -1044,7 +1096,7 @@ def _get_child_mock(self, /, **kw): return klass(**kw) - def _calls_repr(self, prefix="Calls"): + def _calls_repr(self): """Renders self.mock_calls as a string. Example: "\nCalls: [call(1), call(2)]." @@ -1054,10 +1106,15 @@ def _calls_repr(self, prefix="Calls"): """ if not self.mock_calls: return "" - return f"\n{prefix}: {safe_repr(self.mock_calls)}." + return f"\nCalls: {safe_repr(self.mock_calls)}." -_MOCK_SIG = inspect.signature(NonCallableMock.__init__) +# Denylist for forbidden attribute names in safe mode +_ATTRIB_DENY_LIST = frozenset({ + name.removeprefix("assert_") + for name in dir(NonCallableMock) + if name.startswith("assert_") +}) class _AnyComparer(list): @@ -1188,6 +1245,9 @@ def _execute_mock_call(self, /, *args, **kwargs): if self._mock_return_value is not DEFAULT: return self.return_value + if self._mock_delegate and self._mock_delegate.return_value is not DEFAULT: + return self.return_value + if self._mock_wraps is not None: return self._mock_wraps(*args, **kwargs) @@ -1229,9 +1289,11 @@ class or instance) that acts as the specification for the mock object. If `return_value` attribute. * `unsafe`: By default, accessing any attribute whose name starts with - *assert*, *assret*, *asert*, *aseert* or *assrt* will raise an - AttributeError. Passing `unsafe=True` will allow access to - these attributes. + *assert*, *assret*, *asert*, *aseert*, or *assrt* raises an AttributeError. + Additionally, an AttributeError is raised when accessing + attributes that match the name of an assertion method without the prefix + `assert_`, e.g. accessing `called_once` instead of `assert_called_once`. + Passing `unsafe=True` will allow access to these attributes. * `wraps`: Item for the mock object to wrap. If `wraps` is not None then calling the Mock will pass the call through to the wrapped object @@ -1303,6 +1365,7 @@ def __init__( self.autospec = autospec self.kwargs = kwargs self.additional_patchers = [] + self.is_started = False def copy(self): @@ -1415,6 +1478,9 @@ def get_original(self): def __enter__(self): """Perform the patch.""" + if self.is_started: + raise RuntimeError("Patch is already started") + new, spec, spec_set = self.new, self.spec, self.spec_set autospec, kwargs = self.autospec, self.kwargs new_callable = self.new_callable @@ -1457,13 +1523,12 @@ def __enter__(self): if isinstance(original, type): # If we're patching out a class and there is a spec inherit = True - if spec is None and _is_async_obj(original): - Klass = AsyncMock - else: - Klass = MagicMock - _kwargs = {} + + # Determine the Klass to use if new_callable is not None: Klass = new_callable + elif spec is None and _is_async_obj(original): + Klass = AsyncMock elif spec is not None or spec_set is not None: this_spec = spec if spec_set is not None: @@ -1476,7 +1541,12 @@ def __enter__(self): Klass = AsyncMock elif not_callable: Klass = NonCallableMagicMock + else: + Klass = MagicMock + else: + Klass = MagicMock + _kwargs = {} if spec is not None: _kwargs['spec'] = spec if spec_set is not None: @@ -1542,6 +1612,7 @@ def __enter__(self): self.temp_original = original self.is_local = local self._exit_stack = contextlib.ExitStack() + self.is_started = True try: setattr(self.target, self.attribute, new_attr) if self.attribute_name is not None: @@ -1561,6 +1632,9 @@ def __enter__(self): def __exit__(self, *exc_info): """Undo the patch.""" + if not self.is_started: + return + if self.is_local and self.temp_original is not DEFAULT: setattr(self.target, self.attribute, self.temp_original) else: @@ -1577,6 +1651,7 @@ def __exit__(self, *exc_info): del self.target exit_stack = self._exit_stack del self._exit_stack + self.is_started = False return exit_stack.__exit__(*exc_info) @@ -1697,7 +1772,7 @@ def patch( the patch is undone. If `new` is omitted, then the target is replaced with an - `AsyncMock if the patched object is an async function or a + `AsyncMock` if the patched object is an async function or a `MagicMock` otherwise. If `patch` is used as a decorator and `new` is omitted, the created mock is passed in as an extra argument to the decorated function. If `patch` is used as a context manager the created @@ -1771,7 +1846,8 @@ def patch( class _patch_dict(object): """ Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. + to its original state after the test, where the restored dictionary is + a copy of the dictionary as it was before the test. `in_dict` can be a dictionary or a mapping like container. If it is a mapping then it must at least support getting, setting and deleting items @@ -2107,8 +2183,6 @@ def _mock_set_magics(self): if getattr(self, "_mock_methods", None) is not None: these_magics = orig_magics.intersection(self._mock_methods) - - remove_magics = set() remove_magics = orig_magics - these_magics for entry in remove_magics: @@ -2138,10 +2212,8 @@ def mock_add_spec(self, spec, spec_set=False): class AsyncMagicMixin(MagicMixin): - def __init__(self, /, *args, **kw): - self._mock_set_magics() # make magic work for kwargs in init - _safe_super(AsyncMagicMixin, self).__init__(*args, **kw) - self._mock_set_magics() # fix magic broken by upper level init + pass + class MagicMock(MagicMixin, Mock): """ @@ -2163,6 +2235,17 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() + def reset_mock(self, /, *args, return_value: bool = False, **kwargs): + if ( + return_value + and self._mock_name + and _is_magic(self._mock_name) + ): + # Don't reset return values for magic methods, + # otherwise `m.__str__` will start + # to return `MagicMock` instances, instead of `str` instances. + return_value = False + super().reset_mock(*args, return_value=return_value, **kwargs) class MagicProxy(Base): @@ -2183,6 +2266,13 @@ def __get__(self, obj, _type=None): return self.create_mock() +try: + _CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) + _CODE_ATTRS = dir(CodeType) +except ValueError: + _CODE_SIG = None + + class AsyncMockMixin(Base): await_count = _delegating_property('await_count') await_args = _delegating_property('await_args') @@ -2200,8 +2290,21 @@ def __init__(self, /, *args, **kwargs): self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() - code_mock = NonCallableMock(spec_set=CodeType) - code_mock.co_flags = inspect.CO_COROUTINE + if _CODE_SIG: + code_mock = NonCallableMock(spec_set=_CODE_ATTRS) + code_mock.__dict__["_spec_class"] = CodeType + code_mock.__dict__["_spec_signature"] = _CODE_SIG + else: + code_mock = NonCallableMock(spec_set=CodeType) + code_mock.co_flags = ( + inspect.CO_COROUTINE + + inspect.CO_VARARGS + + inspect.CO_VARKEYWORDS + ) + code_mock.co_argcount = 0 + code_mock.co_varnames = ('args', 'kwargs') + code_mock.co_posonlyargcount = 0 + code_mock.co_kwonlyargcount = 0 self.__dict__['__code__'] = code_mock self.__dict__['__name__'] = 'AsyncMock' self.__dict__['__defaults__'] = tuple() @@ -2379,7 +2482,7 @@ class AsyncMock(AsyncMockMixin, AsyncMagicMixin, Mock): recognized as an async function, and the result of a call is an awaitable: >>> mock = AsyncMock() - >>> iscoroutinefunction(mock) + >>> inspect.iscoroutinefunction(mock) True >>> inspect.isawaitable(mock()) True @@ -2669,6 +2772,16 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, f'[object={spec!r}]') is_async_func = _is_async_func(spec) _kwargs = {'spec': spec} + + entries = [(entry, _missing) for entry in dir(spec)] + if is_type and instance and is_dataclass(spec): + is_dataclass_spec = True + dataclass_fields = fields(spec) + entries.extend((f.name, f.type) for f in dataclass_fields) + dataclass_spec_list = [f.name for f in dataclass_fields] + else: + is_dataclass_spec = False + if spec_set: _kwargs = {'spec_set': spec} elif spec is None: @@ -2679,6 +2792,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if not unsafe: _check_spec_arg_typos(kwargs) + _name = kwargs.pop('name', _name) + _new_name = _name + if _parent is None: + # for a top level object no _new_name should be set + _new_name = '' + _kwargs.update(kwargs) Klass = MagicMock @@ -2696,33 +2815,32 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, elif is_type and instance and not _instance_callable(spec): Klass = NonCallableMagicMock - _name = _kwargs.pop('name', _name) - - _new_name = _name - if _parent is None: - # for a top level object no _new_name should be set - _new_name = '' - mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name, name=_name, **_kwargs) + if is_dataclass_spec: + mock._mock_extend_spec_methods(dataclass_spec_list) if isinstance(spec, FunctionTypes): # should only happen at the top level because we don't # recurse for functions - mock = _set_signature(mock, spec) if is_async_func: - _setup_async_mock(mock) + mock = _set_async_signature(mock, spec) + else: + mock = _set_signature(mock, spec) else: _check_signature(spec, mock, is_type, instance) if _parent is not None and not instance: _parent._mock_children[_name] = mock + # Pop wraps from kwargs because it must not be passed to configure_mock. + wrapped = kwargs.pop('wraps', None) if is_type and not instance and 'return_value' not in kwargs: mock.return_value = create_autospec(spec, spec_set, instance=True, - _name='()', _parent=mock) + _name='()', _parent=mock, + wraps=wrapped) - for entry in dir(spec): + for entry, original in entries: if _is_magic(entry): # MagicMock already does the useful magic methods for us continue @@ -2736,14 +2854,18 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # AttributeError on being fetched? # we could be resilient against it, or catch and propagate the # exception when the attribute is fetched from the mock - try: - original = getattr(spec, entry) - except AttributeError: - continue + if original is _missing: + try: + original = getattr(spec, entry) + except AttributeError: + continue - kwargs = {'spec': original} + child_kwargs = {'spec': original} + # Wrap child attributes also. + if wrapped and hasattr(wrapped, entry): + child_kwargs.update(wraps=original) if spec_set: - kwargs = {'spec_set': original} + child_kwargs = {'spec_set': original} if not isinstance(original, FunctionTypes): new = _SpecState(original, spec_set, mock, entry, instance) @@ -2754,15 +2876,15 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, parent = mock.mock skipfirst = _must_skip(spec, entry, is_type) - kwargs['_eat_self'] = skipfirst + child_kwargs['_eat_self'] = skipfirst if iscoroutinefunction(original): child_klass = AsyncMock else: child_klass = MagicMock new = child_klass(parent=parent, name=entry, _new_name=entry, - _new_parent=parent, - **kwargs) + _new_parent=parent, **child_kwargs) mock._mock_children[entry] = new + new.return_value = child_klass() _check_signature(original, new, skipfirst=skipfirst) # so functions created with _set_signature become instance attributes, @@ -2771,6 +2893,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # setting as an instance attribute? if isinstance(new, FunctionTypes): setattr(mock, entry, new) + # kwargs are passed with respect to the parent mock so, they are not used + # for creating return_value of the parent mock. So, this condition + # should be true only for the parent mock if kwargs are given. + if _is_instance_mock(mock) and kwargs: + mock.configure_mock(**kwargs) return mock @@ -2876,6 +3003,9 @@ def _next_side_effect(): return handle.readline.return_value return next(_state[0]) + def _exit_side_effect(exctype, excinst, exctb): + handle.close() + global file_spec if file_spec is None: import _io @@ -2902,6 +3032,7 @@ def _next_side_effect(): handle.readlines.side_effect = _readlines_side_effect handle.__iter__.side_effect = _iter_side_effect handle.__next__.side_effect = _next_side_effect + handle.__exit__.side_effect = _exit_side_effect def reset_data(*args, **kwargs): _state[0] = _to_stream(read_data) @@ -2934,6 +3065,96 @@ def __set__(self, obj, val): self(val) +_timeout_unset = sentinel.TIMEOUT_UNSET + +class ThreadingMixin(Base): + + DEFAULT_TIMEOUT = None + + def _get_child_mock(self, /, **kw): + if isinstance(kw.get("parent"), ThreadingMixin): + kw["timeout"] = kw["parent"]._mock_wait_timeout + elif isinstance(kw.get("_new_parent"), ThreadingMixin): + kw["timeout"] = kw["_new_parent"]._mock_wait_timeout + return super()._get_child_mock(**kw) + + def __init__(self, *args, timeout=_timeout_unset, **kwargs): + super().__init__(*args, **kwargs) + if timeout is _timeout_unset: + timeout = self.DEFAULT_TIMEOUT + self.__dict__["_mock_event"] = threading.Event() # Event for any call + self.__dict__["_mock_calls_events"] = [] # Events for each of the calls + self.__dict__["_mock_calls_events_lock"] = threading.Lock() + self.__dict__["_mock_wait_timeout"] = timeout + + def reset_mock(self, /, *args, **kwargs): + """ + See :func:`.Mock.reset_mock()` + """ + super().reset_mock(*args, **kwargs) + self.__dict__["_mock_event"] = threading.Event() + self.__dict__["_mock_calls_events"] = [] + + def __get_event(self, expected_args, expected_kwargs): + with self._mock_calls_events_lock: + for args, kwargs, event in self._mock_calls_events: + if (args, kwargs) == (expected_args, expected_kwargs): + return event + new_event = threading.Event() + self._mock_calls_events.append((expected_args, expected_kwargs, new_event)) + return new_event + + def _mock_call(self, *args, **kwargs): + ret_value = super()._mock_call(*args, **kwargs) + + call_event = self.__get_event(args, kwargs) + call_event.set() + + self._mock_event.set() + + return ret_value + + def wait_until_called(self, *, timeout=_timeout_unset): + """Wait until the mock object is called. + + `timeout` - time to wait for in seconds, waits forever otherwise. + Defaults to the constructor provided timeout. + Use None to block undefinetively. + """ + if timeout is _timeout_unset: + timeout = self._mock_wait_timeout + if not self._mock_event.wait(timeout=timeout): + msg = (f"{self._mock_name or 'mock'} was not called before" + f" timeout({timeout}).") + raise AssertionError(msg) + + def wait_until_any_call_with(self, *args, **kwargs): + """Wait until the mock object is called with given args. + + Waits for the timeout in seconds provided in the constructor. + """ + event = self.__get_event(args, kwargs) + if not event.wait(timeout=self._mock_wait_timeout): + expected_string = self._format_mock_call_signature(args, kwargs) + raise AssertionError(f'{expected_string} call not found') + + +class ThreadingMock(ThreadingMixin, MagicMixin, Mock): + """ + A mock that can be used to wait until on calls happening + in a different thread. + + The constructor can take a `timeout` argument which + controls the timeout in seconds for all `wait` calls of the mock. + + You can change the default timeout of all instances via the + `ThreadingMock.DEFAULT_TIMEOUT` attribute. + + If no timeout is set, it will block undefinetively. + """ + pass + + def seal(mock): """Disable the automatic generation of child mocks. diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 5ca4c23238b..8eafb3891c9 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -3,9 +3,9 @@ import io import sys import traceback +from functools import wraps from . import util -from functools import wraps __unittest = True @@ -43,6 +43,7 @@ def __init__(self, stream=None, descriptions=None, verbosity=None): self.skipped = [] self.expectedFailures = [] self.unexpectedSuccesses = [] + self.collectedDurations = [] self.shouldStop = False self.buffer = False self.tb_locals = False @@ -157,6 +158,17 @@ def addUnexpectedSuccess(self, test): """Called when a test was expected to fail, but succeed.""" self.unexpectedSuccesses.append(test) + def addDuration(self, test, elapsed): + """Called when a test finished to run, regardless of its outcome. + *test* is the test case corresponding to the test method. + *elapsed* is the time represented in seconds, and it includes the + execution of cleanup functions. + """ + # support for a TextTestRunner using an old TestResult class + if hasattr(self, "collectedDurations"): + # Pass test repr and not the test object itself to avoid resources leak + self.collectedDurations.append((str(test), elapsed)) + def wasSuccessful(self): """Tells whether or not this result was a success.""" # The hasattr check is for test_result's OldResult test. That @@ -177,7 +189,10 @@ def _exc_info_to_string(self, err, test): tb_e = traceback.TracebackException( exctype, value, tb, capture_locals=self.tb_locals, compact=True) - msgLines = list(tb_e.format()) + from _colorize import can_colorize + + colorize = hasattr(self, "stream") and can_colorize(file=self.stream) + msgLines = list(tb_e.format(colorize=colorize)) if self.buffer: output = sys.stdout.getvalue() diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index cb452c7adef..5f22d91aebd 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -4,6 +4,8 @@ import time import warnings +from _colorize import get_theme + from . import result from .case import _SubTest from .signals import registerResult @@ -13,18 +15,18 @@ class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" - def __init__(self,stream): + def __init__(self, stream): self.stream = stream def __getattr__(self, attr): if attr in ('stream', '__getstate__'): raise AttributeError(attr) - return getattr(self.stream,attr) + return getattr(self.stream, attr) def writeln(self, arg=None): if arg: self.write(arg) - self.write('\n') # text-mode streams translate to \r\n if needed + self.write('\n') # text-mode streams translate to \r\n if needed class TextTestResult(result.TestResult): @@ -35,13 +37,17 @@ class TextTestResult(result.TestResult): separator1 = '=' * 70 separator2 = '-' * 70 - def __init__(self, stream, descriptions, verbosity): + def __init__(self, stream, descriptions, verbosity, *, durations=None): + """Construct a TextTestResult. Subclasses should accept **kwargs + to ensure compatibility as the interface changes.""" super(TextTestResult, self).__init__(stream, descriptions, verbosity) self.stream = stream self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions + self._theme = get_theme(tty_file=stream).unittest self._newline = True + self.durations = durations def getDescription(self, test): doc_first_line = test.shortDescription() @@ -73,86 +79,100 @@ def _write_status(self, test, status): def addSubTest(self, test, subtest, err): if err is not None: + t = self._theme if self.showAll: if issubclass(err[0], subtest.failureException): - self._write_status(subtest, "FAIL") + self._write_status(subtest, f"{t.fail}FAIL{t.reset}") else: - self._write_status(subtest, "ERROR") + self._write_status(subtest, f"{t.fail}ERROR{t.reset}") elif self.dots: if issubclass(err[0], subtest.failureException): - self.stream.write('F') + self.stream.write(f"{t.fail}F{t.reset}") else: - self.stream.write('E') + self.stream.write(f"{t.fail}E{t.reset}") self.stream.flush() super(TextTestResult, self).addSubTest(test, subtest, err) def addSuccess(self, test): super(TextTestResult, self).addSuccess(test) + t = self._theme if self.showAll: - self._write_status(test, "ok") + self._write_status(test, f"{t.passed}ok{t.reset}") elif self.dots: - self.stream.write('.') + self.stream.write(f"{t.passed}.{t.reset}") self.stream.flush() def addError(self, test, err): super(TextTestResult, self).addError(test, err) + t = self._theme if self.showAll: - self._write_status(test, "ERROR") + self._write_status(test, f"{t.fail}ERROR{t.reset}") elif self.dots: - self.stream.write('E') + self.stream.write(f"{t.fail}E{t.reset}") self.stream.flush() def addFailure(self, test, err): super(TextTestResult, self).addFailure(test, err) + t = self._theme if self.showAll: - self._write_status(test, "FAIL") + self._write_status(test, f"{t.fail}FAIL{t.reset}") elif self.dots: - self.stream.write('F') + self.stream.write(f"{t.fail}F{t.reset}") self.stream.flush() def addSkip(self, test, reason): super(TextTestResult, self).addSkip(test, reason) + t = self._theme if self.showAll: - self._write_status(test, "skipped {0!r}".format(reason)) + self._write_status(test, f"{t.warn}skipped{t.reset} {reason!r}") elif self.dots: - self.stream.write("s") + self.stream.write(f"{t.warn}s{t.reset}") self.stream.flush() def addExpectedFailure(self, test, err): super(TextTestResult, self).addExpectedFailure(test, err) + t = self._theme if self.showAll: - self.stream.writeln("expected failure") + self.stream.writeln(f"{t.warn}expected failure{t.reset}") self.stream.flush() elif self.dots: - self.stream.write("x") + self.stream.write(f"{t.warn}x{t.reset}") self.stream.flush() def addUnexpectedSuccess(self, test): super(TextTestResult, self).addUnexpectedSuccess(test) + t = self._theme if self.showAll: - self.stream.writeln("unexpected success") + self.stream.writeln(f"{t.fail}unexpected success{t.reset}") self.stream.flush() elif self.dots: - self.stream.write("u") + self.stream.write(f"{t.fail}u{t.reset}") self.stream.flush() def printErrors(self): + t = self._theme if self.dots or self.showAll: self.stream.writeln() self.stream.flush() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - unexpectedSuccesses = getattr(self, 'unexpectedSuccesses', ()) + self.printErrorList(f"{t.fail}ERROR{t.reset}", self.errors) + self.printErrorList(f"{t.fail}FAIL{t.reset}", self.failures) + unexpectedSuccesses = getattr(self, "unexpectedSuccesses", ()) if unexpectedSuccesses: self.stream.writeln(self.separator1) for test in unexpectedSuccesses: - self.stream.writeln(f"UNEXPECTED SUCCESS: {self.getDescription(test)}") + self.stream.writeln( + f"{t.fail}UNEXPECTED SUCCESS{t.fail_info}: " + f"{self.getDescription(test)}{t.reset}" + ) self.stream.flush() def printErrorList(self, flavour, errors): + t = self._theme for test, err in errors: self.stream.writeln(self.separator1) - self.stream.writeln("%s: %s" % (flavour,self.getDescription(test))) + self.stream.writeln( + f"{flavour}{t.fail_info}: {self.getDescription(test)}{t.reset}" + ) self.stream.writeln(self.separator2) self.stream.writeln("%s" % err) self.stream.flush() @@ -168,7 +188,7 @@ class TextTestRunner(object): def __init__(self, stream=None, descriptions=True, verbosity=1, failfast=False, buffer=False, resultclass=None, warnings=None, - *, tb_locals=False): + *, tb_locals=False, durations=None): """Construct a TextTestRunner. Subclasses should accept **kwargs to ensure compatibility as the @@ -182,12 +202,41 @@ def __init__(self, stream=None, descriptions=True, verbosity=1, self.failfast = failfast self.buffer = buffer self.tb_locals = tb_locals + self.durations = durations self.warnings = warnings if resultclass is not None: self.resultclass = resultclass def _makeResult(self): - return self.resultclass(self.stream, self.descriptions, self.verbosity) + try: + return self.resultclass(self.stream, self.descriptions, + self.verbosity, durations=self.durations) + except TypeError: + # didn't accept the durations argument + return self.resultclass(self.stream, self.descriptions, + self.verbosity) + + def _printDurations(self, result): + if not result.collectedDurations: + return + ls = sorted(result.collectedDurations, key=lambda x: x[1], + reverse=True) + if self.durations > 0: + ls = ls[:self.durations] + self.stream.writeln("Slowest test durations") + if hasattr(result, 'separator2'): + self.stream.writeln(result.separator2) + hidden = False + for test, elapsed in ls: + if self.verbosity < 2 and elapsed < 0.001: + hidden = True + continue + self.stream.writeln("%-10s %s" % ("%.3fs" % elapsed, test)) + if hidden: + self.stream.writeln("\n(durations < 0.001s were hidden; " + "use -v to show these durations)") + else: + self.stream.writeln("") def run(self, test): "Run the given test case or test suite." @@ -200,16 +249,7 @@ def run(self, test): if self.warnings: # if self.warnings is set, use it to filter all the warnings warnings.simplefilter(self.warnings) - # if the filter is 'default' or 'always', special-case the - # warnings from the deprecated unittest methods to show them - # no more than once per module, because they can be fairly - # noisy. The -Wd and -Wa flags can be used to bypass this - # only when self.warnings is None. - if self.warnings in ['default', 'always']: - warnings.filterwarnings('module', - category=DeprecationWarning, - message=r'Please use assert\w+ instead.') - startTime = time.perf_counter() + start_time = time.perf_counter() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() @@ -219,17 +259,21 @@ def run(self, test): stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() - stopTime = time.perf_counter() - timeTaken = stopTime - startTime + stop_time = time.perf_counter() + time_taken = stop_time - start_time result.printErrors() + if self.durations is not None: + self._printDurations(result) + if hasattr(result, 'separator2'): self.stream.writeln(result.separator2) + run = result.testsRun self.stream.writeln("Ran %d test%s in %.3fs" % - (run, run != 1 and "s" or "", timeTaken)) + (run, run != 1 and "s" or "", time_taken)) self.stream.writeln() - expectedFails = unexpectedSuccesses = skipped = 0 + expected_fails = unexpected_successes = skipped = 0 try: results = map(len, (result.expectedFailures, result.unexpectedSuccesses, @@ -237,24 +281,30 @@ def run(self, test): except AttributeError: pass else: - expectedFails, unexpectedSuccesses, skipped = results + expected_fails, unexpected_successes, skipped = results infos = [] + t = get_theme(tty_file=self.stream).unittest + if not result.wasSuccessful(): - self.stream.write("FAILED") + self.stream.write(f"{t.fail_info}FAILED{t.reset}") failed, errored = len(result.failures), len(result.errors) if failed: - infos.append("failures=%d" % failed) + infos.append(f"{t.fail_info}failures={failed}{t.reset}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"{t.fail_info}errors={errored}{t.reset}") + elif run == 0 and not skipped: + self.stream.write(f"{t.warn}NO TESTS RAN{t.reset}") else: - self.stream.write("OK") + self.stream.write(f"{t.passed}OK{t.reset}") if skipped: - infos.append("skipped=%d" % skipped) - if expectedFails: - infos.append("expected failures=%d" % expectedFails) - if unexpectedSuccesses: - infos.append("unexpected successes=%d" % unexpectedSuccesses) + infos.append(f"{t.warn}skipped={skipped}{t.reset}") + if expected_fails: + infos.append(f"{t.warn}expected failures={expected_fails}{t.reset}") + if unexpected_successes: + infos.append( + f"{t.fail}unexpected successes={unexpected_successes}{t.reset}" + ) if infos: self.stream.writeln(" (%s)" % (", ".join(infos),)) else: diff --git a/Lib/unittest/signals.py b/Lib/unittest/signals.py index e6a5fc52439..4e654c2c5db 100644 --- a/Lib/unittest/signals.py +++ b/Lib/unittest/signals.py @@ -1,6 +1,5 @@ import signal import weakref - from functools import wraps __unittest = True diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py index 6f45b6fe5f6..3c40176f070 100644 --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -2,8 +2,7 @@ import sys -from . import case -from . import util +from . import case, util __unittest = True diff --git a/Lib/unittest/test/__init__.py b/Lib/unittest/test/__init__.py deleted file mode 100644 index 143f4ab5a3d..00000000000 --- a/Lib/unittest/test/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import sys -import unittest - - -here = os.path.dirname(__file__) -loader = unittest.defaultTestLoader - -def suite(): - suite = unittest.TestSuite() - for fn in os.listdir(here): - if fn.startswith("test") and fn.endswith(".py"): - modname = "unittest.test." + fn[:-3] - try: - __import__(modname) - except unittest.SkipTest: - continue - module = sys.modules[modname] - suite.addTest(loader.loadTestsFromModule(module)) - suite.addTest(loader.loadTestsFromName('unittest.test.testmock')) - return suite - - -if __name__ == "__main__": - unittest.main(defaultTest="suite") diff --git a/Lib/unittest/test/__main__.py b/Lib/unittest/test/__main__.py deleted file mode 100644 index 44d0591e847..00000000000 --- a/Lib/unittest/test/__main__.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import unittest - - -def load_tests(loader, standard_tests, pattern): - # top level directory cached on loader instance - this_dir = os.path.dirname(__file__) - pattern = pattern or "test_*.py" - # We are inside unittest.test, so the top-level is two notches up - top_level_dir = os.path.dirname(os.path.dirname(this_dir)) - package_tests = loader.discover(start_dir=this_dir, pattern=pattern, - top_level_dir=top_level_dir) - standard_tests.addTests(package_tests) - return standard_tests - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/unittest/test/support.py b/Lib/unittest/test/support.py deleted file mode 100644 index 529265304f2..00000000000 --- a/Lib/unittest/test/support.py +++ /dev/null @@ -1,138 +0,0 @@ -import unittest - - -class TestEquality(object): - """Used as a mixin for TestCase""" - - # Check for a valid __eq__ implementation - def test_eq(self): - for obj_1, obj_2 in self.eq_pairs: - self.assertEqual(obj_1, obj_2) - self.assertEqual(obj_2, obj_1) - - # Check for a valid __ne__ implementation - def test_ne(self): - for obj_1, obj_2 in self.ne_pairs: - self.assertNotEqual(obj_1, obj_2) - self.assertNotEqual(obj_2, obj_1) - -class TestHashing(object): - """Used as a mixin for TestCase""" - - # Check for a valid __hash__ implementation - def test_hash(self): - for obj_1, obj_2 in self.eq_pairs: - try: - if not hash(obj_1) == hash(obj_2): - self.fail("%r and %r do not hash equal" % (obj_1, obj_2)) - except Exception as e: - self.fail("Problem hashing %r and %r: %s" % (obj_1, obj_2, e)) - - for obj_1, obj_2 in self.ne_pairs: - try: - if hash(obj_1) == hash(obj_2): - self.fail("%s and %s hash equal, but shouldn't" % - (obj_1, obj_2)) - except Exception as e: - self.fail("Problem hashing %s and %s: %s" % (obj_1, obj_2, e)) - - -class _BaseLoggingResult(unittest.TestResult): - def __init__(self, log): - self._events = log - super().__init__() - - def startTest(self, test): - self._events.append('startTest') - super().startTest(test) - - def startTestRun(self): - self._events.append('startTestRun') - super().startTestRun() - - def stopTest(self, test): - self._events.append('stopTest') - super().stopTest(test) - - def stopTestRun(self): - self._events.append('stopTestRun') - super().stopTestRun() - - def addFailure(self, *args): - self._events.append('addFailure') - super().addFailure(*args) - - def addSuccess(self, *args): - self._events.append('addSuccess') - super().addSuccess(*args) - - def addError(self, *args): - self._events.append('addError') - super().addError(*args) - - def addSkip(self, *args): - self._events.append('addSkip') - super().addSkip(*args) - - def addExpectedFailure(self, *args): - self._events.append('addExpectedFailure') - super().addExpectedFailure(*args) - - def addUnexpectedSuccess(self, *args): - self._events.append('addUnexpectedSuccess') - super().addUnexpectedSuccess(*args) - - -class LegacyLoggingResult(_BaseLoggingResult): - """ - A legacy TestResult implementation, without an addSubTest method, - which records its method calls. - """ - - @property - def addSubTest(self): - raise AttributeError - - -class LoggingResult(_BaseLoggingResult): - """ - A TestResult implementation which records its method calls. - """ - - def addSubTest(self, test, subtest, err): - if err is None: - self._events.append('addSubTestSuccess') - else: - self._events.append('addSubTestFailure') - super().addSubTest(test, subtest, err) - - -class ResultWithNoStartTestRunStopTestRun(object): - """An object honouring TestResult before startTestRun/stopTestRun.""" - - def __init__(self): - self.failures = [] - self.errors = [] - self.testsRun = 0 - self.skipped = [] - self.expectedFailures = [] - self.unexpectedSuccesses = [] - self.shouldStop = False - - def startTest(self, test): - pass - - def stopTest(self, test): - pass - - def addError(self, test): - pass - - def addFailure(self, test): - pass - - def addSuccess(self, test): - pass - - def wasSuccessful(self): - return True diff --git a/Lib/unittest/test/testmock/__init__.py b/Lib/unittest/test/testmock/__init__.py deleted file mode 100644 index 661b577259b..00000000000 --- a/Lib/unittest/test/testmock/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import sys -import unittest - - -here = os.path.dirname(__file__) -loader = unittest.defaultTestLoader - -def load_tests(*args): - suite = unittest.TestSuite() - # TODO: RUSTPYTHON; allow objects to be mocked better - return suite - for fn in os.listdir(here): - if fn.startswith("test") and fn.endswith(".py"): - modname = "unittest.test.testmock." + fn[:-3] - __import__(modname) - module = sys.modules[modname] - suite.addTest(loader.loadTestsFromModule(module)) - return suite diff --git a/Lib/unittest/test/testmock/__main__.py b/Lib/unittest/test/testmock/__main__.py deleted file mode 100644 index 45c633a4ee4..00000000000 --- a/Lib/unittest/test/testmock/__main__.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import unittest - - -def load_tests(loader, standard_tests, pattern): - # top level directory cached on loader instance - this_dir = os.path.dirname(__file__) - pattern = pattern or "test*.py" - # We are inside unittest.test.testmock, so the top-level is three notches up - top_level_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_dir))) - package_tests = loader.discover(start_dir=this_dir, pattern=pattern, - top_level_dir=top_level_dir) - standard_tests.addTests(package_tests) - return standard_tests - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/unittest/test/testmock/support.py b/Lib/unittest/test/testmock/support.py deleted file mode 100644 index 49986d65dc4..00000000000 --- a/Lib/unittest/test/testmock/support.py +++ /dev/null @@ -1,16 +0,0 @@ -target = {'foo': 'FOO'} - - -def is_instance(obj, klass): - """Version of is_instance that doesn't access __class__""" - return issubclass(type(obj), klass) - - -class SomeClass(object): - class_attribute = None - - def wibble(self): pass - - -class X(object): - pass diff --git a/Lib/unittest/util.py b/Lib/unittest/util.py index 050eaed0b3f..b81b6a4219b 100644 --- a/Lib/unittest/util.py +++ b/Lib/unittest/util.py @@ -1,6 +1,6 @@ """Various utility functions.""" -from collections import namedtuple, Counter +from collections import Counter, namedtuple from os.path import commonprefix __unittest = True diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index 8cd901f13f8..a9cd1ecadd6 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -10,7 +10,7 @@ an application may want to handle an exception like a regular response. """ - +import io import urllib.response __all__ = ['URLError', 'HTTPError', 'ContentTooShortError'] @@ -42,12 +42,9 @@ def __init__(self, url, code, msg, hdrs, fp): self.hdrs = hdrs self.fp = fp self.filename = url - # The addinfourl classes depend on fp being a valid file - # object. In some cases, the HTTPError may not have a valid - # file object. If this happens, the simplest workaround is to - # not initialize the base classes. - if fp is not None: - self.__super_init(fp, hdrs, url, code) + if fp is None: + fp = io.BytesIO() + self.__super_init(fp, hdrs, url, code) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index b35997bc00c..67d9bbea0d3 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -25,13 +25,19 @@ scenarios for parsing, and for backward compatibility purposes, some parsing quirks from older RFCs are retained. The testcases in test_urlparse.py provides a good indicator of parsing behavior. + +The WHATWG URL Parser spec should also be considered. We are not compliant with +it either due to existing user code API behavior expectations (Hyrum's Law). +It serves as a useful guide when making changes. """ +from collections import namedtuple +import functools +import math import re -import sys import types -import collections import warnings +import ipaddress __all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", "urlsplit", "urlunsplit", "urlencode", "parse_qs", @@ -46,18 +52,18 @@ uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', 'wais', 'file', 'https', 'shttp', 'mms', - 'prospero', 'rtsp', 'rtspu', 'sftp', + 'prospero', 'rtsp', 'rtsps', 'rtspu', 'sftp', 'svn', 'svn+ssh', 'ws', 'wss'] uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', 'imap', 'wais', 'file', 'mms', 'https', 'shttp', - 'snews', 'prospero', 'rtsp', 'rtspu', 'rsync', + 'snews', 'prospero', 'rtsp', 'rtsps', 'rtspu', 'rsync', 'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', - 'ws', 'wss'] + 'ws', 'wss', 'itms-services'] uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', - 'https', 'shttp', 'rtsp', 'rtspu', 'sip', 'sips', - 'mms', 'sftp', 'tel'] + 'https', 'shttp', 'rtsp', 'rtsps', 'rtspu', 'sip', + 'sips', 'mms', 'sftp', 'tel'] # These are not actually used anymore, but should stay for backwards # compatibility. (They are undocumented, but have a public-looking name.) @@ -66,7 +72,7 @@ 'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', - 'gopher', 'rtsp', 'rtspu', 'sip', 'sips'] + 'gopher', 'rtsp', 'rtsps', 'rtspu', 'sip', 'sips'] uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', 'nntp', 'wais', 'https', 'shttp', 'snews', @@ -78,18 +84,17 @@ '0123456789' '+-.') +# Leading and trailing C0 control and space to be stripped per WHATWG spec. +# == "".join([chr(i) for i in range(0, 0x20 + 1)]) +_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ' + # Unsafe bytes to be removed per WHATWG spec _UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] -# XXX: Consider replacing with functools.lru_cache -MAX_CACHE_SIZE = 20 -_parse_cache = {} - def clear_cache(): - """Clear the parse cache and the quoters cache.""" - _parse_cache.clear() - _safe_quoters.clear() - + """Clear internal performance caches. Undocumented; some tests want it.""" + urlsplit.cache_clear() + _byte_quoter_factory.cache_clear() # Helpers for bytes handling # For 3.2, we deliberately require applications that @@ -171,12 +176,11 @@ def hostname(self): def port(self): port = self._hostinfo[1] if port is not None: - try: - port = int(port, 10) - except ValueError: - message = f'Port could not be cast to integer value as {port!r}' - raise ValueError(message) from None - if not ( 0 <= port <= 65535): + if port.isdigit() and port.isascii(): + port = int(port) + else: + raise ValueError(f"Port could not be cast to integer value as {port!r}") + if not (0 <= port <= 65535): raise ValueError("Port out of range 0-65535") return port @@ -243,13 +247,11 @@ def _hostinfo(self): return hostname, port -from collections import namedtuple - -_DefragResultBase = namedtuple('DefragResult', 'url fragment') +_DefragResultBase = namedtuple('_DefragResultBase', 'url fragment') _SplitResultBase = namedtuple( - 'SplitResult', 'scheme netloc path query fragment') + '_SplitResultBase', 'scheme netloc path query fragment') _ParseResultBase = namedtuple( - 'ParseResult', 'scheme netloc path params query fragment') + '_ParseResultBase', 'scheme netloc path params query fragment') _DefragResultBase.__doc__ = """ DefragResult(url, fragment) @@ -390,20 +392,23 @@ def urlparse(url, scheme='', allow_fragments=True): Note that % escapes are not expanded. """ url, scheme, _coerce_result = _coerce_args(url, scheme) - splitresult = urlsplit(url, scheme, allow_fragments) - scheme, netloc, url, query, fragment = splitresult - if scheme in uses_params and ';' in url: - url, params = _splitparams(url) - else: - params = '' - result = ParseResult(scheme, netloc, url, params, query, fragment) + scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) + result = ParseResult(scheme or '', netloc or '', url, params or '', query or '', fragment or '') return _coerce_result(result) -def _splitparams(url): +def _urlparse(url, scheme=None, allow_fragments=True): + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + if (scheme or '') in uses_params and ';' in url: + url, params = _splitparams(url, allow_none=True) + else: + params = None + return (scheme, netloc, url, params, query, fragment) + +def _splitparams(url, allow_none=False): if '/' in url: i = url.find(';', url.rfind('/')) if i < 0: - return url, '' + return url, None if allow_none else '' else: i = url.find(';') return url[:i], url[i+1:] @@ -434,6 +439,37 @@ def _checknetloc(netloc): raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") +def _check_bracketed_netloc(netloc): + # Note that this function must mirror the splitting + # done in NetlocResultMixins._hostinfo(). + hostname_and_port = netloc.rpartition('@')[2] + before_bracket, have_open_br, bracketed = hostname_and_port.partition('[') + if have_open_br: + # No data is allowed before a bracket. + if before_bracket: + raise ValueError("Invalid IPv6 URL") + hostname, _, port = bracketed.partition(']') + # No data is allowed after the bracket but before the port delimiter. + if port and not port.startswith(":"): + raise ValueError("Invalid IPv6 URL") + else: + hostname, _, port = hostname_and_port.partition(':') + _check_bracketed_host(hostname) + +# Valid bracketed hosts are defined in +# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ +def _check_bracketed_host(hostname): + if hostname.startswith('v'): + if not re.match(r"\Av[a-fA-F0-9]+\..+\z", hostname): + raise ValueError(f"IPvFuture address is invalid") + else: + ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4 + if isinstance(ip, ipaddress.IPv4Address): + raise ValueError(f"An IPv4 address cannot be in brackets") + +# typed=True avoids BytesWarnings being emitted during cache key +# comparison since this API supports both bytes and str input. +@functools.lru_cache(typed=True) def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: <scheme>://<netloc>/<path>?<query>#<fragment> @@ -456,40 +492,43 @@ def urlsplit(url, scheme='', allow_fragments=True): """ url, scheme, _coerce_result = _coerce_args(url, scheme) + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + v = SplitResult(scheme or '', netloc or '', url, query or '', fragment or '') + return _coerce_result(v) +def _urlsplit(url, scheme=None, allow_fragments=True): + # Only lstrip url as some applications rely on preserving trailing space. + # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) + url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) for b in _UNSAFE_URL_BYTES_TO_REMOVE: url = url.replace(b, "") - scheme = scheme.replace(b, "") + if scheme is not None: + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + scheme = scheme.replace(b, "") allow_fragments = bool(allow_fragments) - key = url, scheme, allow_fragments, type(url), type(scheme) - cached = _parse_cache.get(key, None) - if cached: - return _coerce_result(cached) - if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth - clear_cache() - netloc = query = fragment = '' + netloc = query = fragment = None i = url.find(':') - if i > 0: + if i > 0 and url[0].isascii() and url[0].isalpha(): for c in url[:i]: if c not in scheme_chars: break else: scheme, url = url[:i].lower(), url[i+1:] - if url[:2] == '//': netloc, url = _splitnetloc(url, 2) if (('[' in netloc and ']' not in netloc) or (']' in netloc and '[' not in netloc)): raise ValueError("Invalid IPv6 URL") + if '[' in netloc and ']' in netloc: + _check_bracketed_netloc(netloc) if allow_fragments and '#' in url: url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) _checknetloc(netloc) - v = SplitResult(scheme, netloc, url, query, fragment) - _parse_cache[key] = v - return _coerce_result(v) + return (scheme, netloc, url, query, fragment) def urlunparse(components): """Put a parsed URL back together again. This may result in a @@ -498,9 +537,15 @@ def urlunparse(components): (the draft states that these are equivalent).""" scheme, netloc, url, params, query, fragment, _coerce_result = ( _coerce_args(*components)) + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None if params: url = "%s;%s" % (url, params) - return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment))) + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) def urlunsplit(components): """Combine the elements of a tuple as returned by urlsplit() into a @@ -510,16 +555,27 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc or (scheme and scheme in uses_netloc and url[:2] != '//'): + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) + +def _urlunsplit(scheme, netloc, url, query, fragment): + if netloc is not None: if url and url[:1] != '/': url = '/' + url - url = '//' + (netloc or '') + url + url = '//' + netloc + url + elif url[:2] == '//': + url = '//' + url if scheme: url = scheme + ':' + url - if query: + if query is not None: url = url + '?' + query - if fragment: + if fragment is not None: url = url + '#' + fragment - return _coerce_result(url) + return url def urljoin(base, url, allow_fragments=True): """Join a base URL and a possibly relative URL to form an absolute @@ -530,26 +586,29 @@ def urljoin(base, url, allow_fragments=True): return base base, url, _coerce_result = _coerce_args(base, url) - bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) - scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) - - if scheme != bscheme or scheme not in uses_relative: + bscheme, bnetloc, bpath, bquery, bfragment = \ + _urlsplit(base, None, allow_fragments) + scheme, netloc, path, query, fragment = \ + _urlsplit(url, None, allow_fragments) + + if scheme is None: + scheme = bscheme + if scheme != bscheme or (scheme and scheme not in uses_relative): return _coerce_result(url) - if scheme in uses_netloc: + if not scheme or scheme in uses_netloc: if netloc: - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) netloc = bnetloc - if not path and not params: + if not path: path = bpath - params = bparams - if not query: + if query is None: query = bquery - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + if fragment is None: + fragment = bfragment + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) base_parts = bpath.split('/') if base_parts[-1] != '': @@ -586,8 +645,8 @@ def urljoin(base, url, allow_fragments=True): # then we need to append the trailing '/' resolved_path.append('') - return _coerce_result(urlunparse((scheme, netloc, '/'.join( - resolved_path) or '/', params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, '/'.join( + resolved_path) or '/', query, fragment)) def urldefrag(url): @@ -599,18 +658,21 @@ def urldefrag(url): """ url, _coerce_result = _coerce_args(url) if '#' in url: - s, n, p, a, q, frag = urlparse(url) - defrag = urlunparse((s, n, p, a, q, '')) + s, n, p, q, frag = _urlsplit(url) + defrag = _urlunsplit(s, n, p, q, None) else: frag = '' defrag = url - return _coerce_result(DefragResult(defrag, frag)) + return _coerce_result(DefragResult(defrag, frag or '')) _hexdig = '0123456789ABCDEFabcdef' _hextobyte = None def unquote_to_bytes(string): """unquote_to_bytes('abc%20def') -> b'abc def'.""" + return bytes(_unquote_impl(string)) + +def _unquote_impl(string: bytes | bytearray | str) -> bytes | bytearray: # Note: strings are encoded as UTF-8. This is only an issue if it contains # unescaped non-ASCII characters, which URIs should not. if not string: @@ -622,8 +684,8 @@ def unquote_to_bytes(string): bits = string.split(b'%') if len(bits) == 1: return string - res = [bits[0]] - append = res.append + res = bytearray(bits[0]) + append = res.extend # Delay the initialization of the table to not waste memory # if the function is never called global _hextobyte @@ -637,10 +699,20 @@ def unquote_to_bytes(string): except KeyError: append(b'%') append(item) - return b''.join(res) + return res _asciire = re.compile('([\x00-\x7f]+)') +def _generate_unquoted_parts(string, encoding, errors): + previous_match_end = 0 + for ascii_match in _asciire.finditer(string): + start, end = ascii_match.span() + yield string[previous_match_end:start] # Non-ASCII + # The ascii_match[1] group == string[start:end]. + yield _unquote_impl(ascii_match[1]).decode(encoding, errors) + previous_match_end = end + yield string[previous_match_end:] # Non-ASCII tail + def unquote(string, encoding='utf-8', errors='replace'): """Replace %xx escapes by their single-character equivalent. The optional encoding and errors parameters specify how to decode percent-encoded @@ -652,21 +724,16 @@ def unquote(string, encoding='utf-8', errors='replace'): unquote('abc%20def') -> 'abc def'. """ if isinstance(string, bytes): - return unquote_to_bytes(string).decode(encoding, errors) + return _unquote_impl(string).decode(encoding, errors) if '%' not in string: + # Is it a string-like object? string.split return string if encoding is None: encoding = 'utf-8' if errors is None: errors = 'replace' - bits = _asciire.split(string) - res = [bits[0]] - append = res.append - for i in range(1, len(bits), 2): - append(unquote_to_bytes(bits[i]).decode(encoding, errors)) - append(bits[i + 1]) - return ''.join(res) + return ''.join(_generate_unquoted_parts(string, encoding, errors)) def parse_qs(qs, keep_blank_values=False, strict_parsing=False, @@ -702,7 +769,8 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, parsed_result = {} pairs = parse_qsl(qs, keep_blank_values, strict_parsing, encoding=encoding, errors=errors, - max_num_fields=max_num_fields, separator=separator) + max_num_fields=max_num_fields, separator=separator, + _stacklevel=2) for name, value in pairs: if name in parsed_result: parsed_result[name].append(value) @@ -712,7 +780,7 @@ def parse_qs(qs, keep_blank_values=False, strict_parsing=False, def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): + encoding='utf-8', errors='replace', max_num_fields=None, separator='&', *, _stacklevel=1): """Parse a query given as a string argument. Arguments: @@ -740,11 +808,37 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, Returns a list, as G-d intended. """ - qs, _coerce_result = _coerce_args(qs) - separator, _ = _coerce_args(separator) - - if not separator or (not isinstance(separator, (str, bytes))): + if not separator or not isinstance(separator, (str, bytes)): raise ValueError("Separator must be of type string or bytes.") + if isinstance(qs, str): + if not isinstance(separator, str): + separator = str(separator, 'ascii') + eq = '=' + def _unquote(s): + return unquote_plus(s, encoding=encoding, errors=errors) + elif qs is None: + return [] + else: + try: + # Use memoryview() to reject integers and iterables, + # acceptable by the bytes constructor. + qs = bytes(memoryview(qs)) + except TypeError: + if not qs: + warnings.warn(f"Accepting {type(qs).__name__} objects with " + f"false value in urllib.parse.parse_qsl() is " + f"deprecated as of 3.14", + DeprecationWarning, stacklevel=_stacklevel + 1) + return [] + raise + if isinstance(separator, str): + separator = bytes(separator, 'ascii') + eq = b'=' + def _unquote(s): + return unquote_to_bytes(s.replace(b'+', b' ')) + + if not qs: + return [] # If max_num_fields is defined then check that the number of fields # is less than max_num_fields. This prevents a memory exhaustion DOS @@ -756,25 +850,14 @@ def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, r = [] for name_value in qs.split(separator): - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: + if name_value or strict_parsing: + name, has_eq, value = name_value.partition(eq) + if not has_eq and strict_parsing: raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = nv[0].replace('+', ' ') - name = unquote(name, encoding=encoding, errors=errors) - name = _coerce_result(name) - value = nv[1].replace('+', ' ') - value = unquote(value, encoding=encoding, errors=errors) - value = _coerce_result(value) - r.append((name, value)) + if value or keep_blank_values: + name = _unquote(name) + value = _unquote(value) + r.append((name, value)) return r def unquote_plus(string, encoding='utf-8', errors='replace'): @@ -791,23 +874,22 @@ def unquote_plus(string, encoding='utf-8', errors='replace'): b'0123456789' b'_.-~') _ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) -_safe_quoters = {} -class Quoter(collections.defaultdict): - """A mapping from bytes (in range(0,256)) to strings. + +class _Quoter(dict): + """A mapping from bytes numbers (in range(0,256)) to strings. String values are percent-encoded byte values, unless the key < 128, and - in the "safe" set (either the specified safe set, or default set). + in either of the specified safe set, or the always safe set. """ - # Keeps a cache internally, using defaultdict, for efficiency (lookups + # Keeps a cache internally, via __missing__, for efficiency (lookups # of cached keys don't call Python code at all). def __init__(self, safe): """safe: bytes object.""" self.safe = _ALWAYS_SAFE.union(safe) def __repr__(self): - # Without this, will just display as a defaultdict - return "<%s %r>" % (self.__class__.__name__, dict(self)) + return f"<Quoter {dict(self)!r}>" def __missing__(self, b): # Handle a cache miss. Store quoted string in cache and return. @@ -886,6 +968,11 @@ def quote_plus(string, safe='', encoding=None, errors=None): string = quote(string, safe + space, encoding, errors) return string.replace(' ', '+') +# Expectation: A typical program is unlikely to create more than 5 of these. +@functools.lru_cache +def _byte_quoter_factory(safe): + return _Quoter(safe).__getitem__ + def quote_from_bytes(bs, safe='/'): """Like quote(), but accepts a bytes object rather than a str, and does not perform string-to-bytes encoding. It always returns an ASCII string. @@ -899,14 +986,19 @@ def quote_from_bytes(bs, safe='/'): # Normalize 'safe' by converting to bytes and removing non-ASCII chars safe = safe.encode('ascii', 'ignore') else: + # List comprehensions are faster than generator expressions. safe = bytes([c for c in safe if c < 128]) if not bs.rstrip(_ALWAYS_SAFE_BYTES + safe): return bs.decode() - try: - quoter = _safe_quoters[safe] - except KeyError: - _safe_quoters[safe] = quoter = Quoter(safe).__getitem__ - return ''.join([quoter(char) for char in bs]) + quoter = _byte_quoter_factory(safe) + if (bs_len := len(bs)) < 200_000: + return ''.join(map(quoter, bs)) + else: + # This saves memory - https://github.com/python/cpython/issues/95865 + chunk_size = math.isqrt(bs_len) + chunks = [''.join(map(quoter, bs[i:i+chunk_size])) + for i in range(0, bs_len, chunk_size)] + return ''.join(chunks) def urlencode(query, doseq=False, safe='', encoding=None, errors=None, quote_via=quote_plus): @@ -939,10 +1031,9 @@ def urlencode(query, doseq=False, safe='', encoding=None, errors=None, # but that's a minor nit. Since the original implementation # allowed empty dicts that type of behavior probably should be # preserved for consistency - except TypeError: - ty, va, tb = sys.exc_info() + except TypeError as err: raise TypeError("not a valid non-string sequence " - "or mapping object").with_traceback(tb) + "or mapping object") from err l = [] if not doseq: @@ -1125,15 +1216,15 @@ def splitnport(host, defport=-1): def _splitnport(host, defport=-1): """Split host and port, returning numeric port. Return given default port if no ':' found; defaults to -1. - Return numerical port if a valid number are found after ':'. + Return numerical port if a valid number is found after ':'. Return None if ':' but not a valid number.""" host, delim, port = host.rpartition(':') if not delim: host = port elif port: - try: + if port.isdigit() and port.isascii(): nport = int(port) - except ValueError: + else: nport = None return host, nport return host, defport diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index a0ef60b30de..8d7470a2273 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -11,8 +11,8 @@ Handlers needed to open the requested URL. For example, the HTTPHandler performs HTTP GET and POST requests and deals with non-error returns. The HTTPRedirectHandler automatically deals with -HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler -deals with digest authentication. +HTTP 301, 302, 303, 307, and 308 redirect errors, and the +HTTPDigestAuthHandler deals with digest authentication. urlopen(url, data=None) -- Basic usage is the same as original urllib. pass the url and optionally data to post to an HTTP URL, and @@ -83,33 +83,31 @@ import base64 import bisect +import contextlib import email import hashlib import http.client import io import os -import posixpath import re import socket import string import sys import time import tempfile -import contextlib -import warnings from urllib.error import URLError, HTTPError, ContentTooShortError from urllib.parse import ( urlparse, urlsplit, urljoin, unwrap, quote, unquote, _splittype, _splithost, _splitport, _splituser, _splitpasswd, - _splitattr, _splitquery, _splitvalue, _splittag, _to_bytes, + _splitattr, _splitvalue, _splittag, unquote_to_bytes, urlunparse) from urllib.response import addinfourl, addclosehook # check for SSL try: - import ssl + import ssl # noqa: F401 except ImportError: _have_ssl = False else: @@ -129,7 +127,7 @@ 'urlopen', 'install_opener', 'build_opener', 'pathname2url', 'url2pathname', 'getproxies', # Legacy interface - 'urlretrieve', 'urlcleanup', 'URLopener', 'FancyURLopener', + 'urlretrieve', 'urlcleanup', ] # used in User-Agent header sent @@ -137,7 +135,7 @@ _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - *, cafile=None, capath=None, cadefault=False, context=None): + *, context=None): '''Open the URL url, which can be either a string or a Request object. *data* must be an object specifying additional data to be sent to @@ -155,14 +153,6 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, If *context* is specified, it must be a ssl.SSLContext instance describing the various SSL options. See HTTPSConnection for more details. - The optional *cafile* and *capath* parameters specify a set of trusted CA - certificates for HTTPS requests. cafile should point to a single file - containing a bundle of CA certificates, whereas capath should point to a - directory of hashed certificate files. More information can be found in - ssl.SSLContext.load_verify_locations(). - - The *cadefault* parameter is ignored. - This function always returns an object which can work as a context manager and has the properties url, headers, and status. @@ -174,8 +164,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, the reason phrase returned by the server --- instead of the response headers as it is specified in the documentation for HTTPResponse. - For FTP, file, and data URLs and requests explicitly handled by legacy - URLopener and FancyURLopener classes, this function returns a + For FTP, file, and data URLs, this function returns a urllib.response.addinfourl object. Note that None may be returned if no handler handles the request (though @@ -188,25 +177,7 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ''' global _opener - if cafile or capath or cadefault: - import warnings - warnings.warn("cafile, capath and cadefault are deprecated, use a " - "custom context instead.", DeprecationWarning, 2) - if context is not None: - raise ValueError( - "You can't pass both context and any of cafile, capath, and " - "cadefault" - ) - if not _have_ssl: - raise ValueError('SSL support not available') - context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, - cafile=cafile, - capath=capath) - # send ALPN extension to indicate HTTP/1.1 protocol - context.set_alpn_protocols(['http/1.1']) - https_handler = HTTPSHandler(context=context) - opener = build_opener(https_handler) - elif context: + if context: https_handler = HTTPSHandler(context=context) opener = build_opener(https_handler) elif _opener is None: @@ -266,10 +237,7 @@ def urlretrieve(url, filename=None, reporthook=None, data=None): if reporthook: reporthook(blocknum, bs, size) - while True: - block = fp.read(bs) - if not block: - break + while block := fp.read(bs): read += len(block) tfp.write(block) blocknum += 1 @@ -661,7 +629,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): but another Handler might. """ m = req.get_method() - if (not (code in (301, 302, 303, 307) and m in ("GET", "HEAD") + if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD") or code in (301, 302, 303) and m == "POST")): raise HTTPError(req.full_url, code, msg, headers, fp) @@ -680,6 +648,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS} return Request(newurl, + method="HEAD" if m == "HEAD" else "GET", headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True) @@ -748,7 +717,7 @@ def http_error_302(self, req, fp, code, msg, headers): return self.parent.open(new, timeout=req.timeout) - http_error_301 = http_error_303 = http_error_307 = http_error_302 + http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302 inf_msg = "The HTTP server returned a redirect error that would " \ "lead to an infinite loop.\n" \ @@ -907,9 +876,9 @@ def find_user_password(self, realm, authuri): class HTTPPasswordMgrWithPriorAuth(HTTPPasswordMgrWithDefaultRealm): - def __init__(self, *args, **kwargs): + def __init__(self): self.authenticated = {} - super().__init__(*args, **kwargs) + super().__init__() def add_password(self, realm, uri, user, passwd, is_authenticated=False): self.update_authenticated(uri, is_authenticated) @@ -969,6 +938,7 @@ def _parse_realm(self, header): for mo in AbstractBasicAuthHandler.rx.finditer(header): scheme, quote, realm = mo.groups() if quote not in ['"', "'"]: + import warnings warnings.warn("Basic Auth Realm was unquoted", UserWarning, 3) @@ -1078,7 +1048,7 @@ def http_error_407(self, req, fp, code, msg, headers): class AbstractDigestAuthHandler: - # Digest authentication is specified in RFC 2617. + # Digest authentication is specified in RFC 2617/7616. # XXX The client does not inspect the Authentication-Info header # in a successful response. @@ -1206,11 +1176,14 @@ def get_authorization(self, req, chal): return base def get_algorithm_impls(self, algorithm): + # algorithm names taken from RFC 7616 Section 6.1 # lambdas assume digest modules are imported at the top level if algorithm == 'MD5': H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest() - elif algorithm == 'SHA': + elif algorithm == 'SHA': # non-standard, retained for compatibility. H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest() + elif algorithm == 'SHA-256': + H = lambda x: hashlib.sha256(x.encode("ascii")).hexdigest() # XXX MD5-sess else: raise ValueError("Unsupported digest authentication " @@ -1255,8 +1228,8 @@ def http_error_407(self, req, fp, code, msg, headers): class AbstractHTTPHandler(BaseHandler): - def __init__(self, debuglevel=0): - self._debuglevel = debuglevel + def __init__(self, debuglevel=None): + self._debuglevel = debuglevel if debuglevel is not None else http.client.HTTPConnection.debuglevel def set_http_debuglevel(self, level): self._debuglevel = level @@ -1382,14 +1355,19 @@ def http_open(self, req): class HTTPSHandler(AbstractHTTPHandler): - def __init__(self, debuglevel=0, context=None, check_hostname=None): + def __init__(self, debuglevel=None, context=None, check_hostname=None): + debuglevel = debuglevel if debuglevel is not None else http.client.HTTPSConnection.debuglevel AbstractHTTPHandler.__init__(self, debuglevel) + if context is None: + http_version = http.client.HTTPSConnection._http_vsn + context = http.client._create_https_context(http_version) + if check_hostname is not None: + context.check_hostname = check_hostname self._context = context - self._check_hostname = check_hostname def https_open(self, req): return self.do_open(http.client.HTTPSConnection, req, - context=self._context, check_hostname=self._check_hostname) + context=self._context) https_request = AbstractHTTPHandler.do_request_ @@ -1472,16 +1450,6 @@ def parse_http_list(s): return [part.strip() for part in res] class FileHandler(BaseHandler): - # Use local file or FTP depending on form of URL - def file_open(self, req): - url = req.selector - if url[:2] == '//' and url[2:3] != '/' and (req.host and - req.host != 'localhost'): - if not req.host in self.get_names(): - raise URLError("file:// scheme is supported only on localhost") - else: - return self.open_local_file(req) - # names for the localhost names = None def get_names(self): @@ -1498,35 +1466,41 @@ def get_names(self): def open_local_file(self, req): import email.utils import mimetypes - host = req.host - filename = req.selector - localfile = url2pathname(filename) + localfile = url2pathname(req.full_url, require_scheme=True, resolve_host=True) try: stats = os.stat(localfile) size = stats.st_size modified = email.utils.formatdate(stats.st_mtime, usegmt=True) - mtype = mimetypes.guess_type(filename)[0] + mtype = mimetypes.guess_file_type(localfile)[0] headers = email.message_from_string( 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % (mtype or 'text/plain', size, modified)) - if host: - host, port = _splitport(host) - if not host or \ - (not port and _safe_gethostbyname(host) in self.get_names()): - if host: - origurl = 'file://' + host + filename - else: - origurl = 'file://' + filename - return addinfourl(open(localfile, 'rb'), headers, origurl) + origurl = pathname2url(localfile, add_scheme=True) + return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: - raise URLError(exp) - raise URLError('file not on local host') + raise URLError(exp, exp.filename) + + file_open = open_local_file -def _safe_gethostbyname(host): +def _is_local_authority(authority, resolve): + # Compare hostnames + if not authority or authority == 'localhost': + return True try: - return socket.gethostbyname(host) - except socket.gaierror: - return None + hostname = socket.gethostname() + except (socket.gaierror, AttributeError): + pass + else: + if authority == hostname: + return True + # Compare IP addresses + if not resolve: + return False + try: + address = socket.gethostbyname(authority) + except (socket.gaierror, AttributeError, UnicodeEncodeError): + return False + return address in FileHandler().get_names() class FTPHandler(BaseHandler): def ftp_open(self, req): @@ -1561,6 +1535,7 @@ def ftp_open(self, req): dirs, file = dirs[:-1], dirs[-1] if dirs and not dirs[0]: dirs = dirs[1:] + fw = None try: fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' @@ -1578,9 +1553,12 @@ def ftp_open(self, req): headers += "Content-length: %d\n" % retrlen headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) - except ftplib.all_errors as exp: - exc = URLError('ftp error: %r' % exp) - raise exc.with_traceback(sys.exc_info()[2]) + except Exception as exp: + if fw is not None and not fw.keepalive: + fw.close() + if isinstance(exp, ftplib.all_errors): + raise URLError(f"ftp error: {exp}") from exp + raise def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, @@ -1604,14 +1582,15 @@ def setMaxConns(self, m): def connect_ftp(self, user, passwd, host, port, dirs, timeout): key = user, host, port, '/'.join(dirs), timeout - if key in self.cache: - self.timeout[key] = time.time() + self.delay - else: - self.cache[key] = ftpwrapper(user, passwd, host, port, - dirs, timeout) - self.timeout[key] = time.time() + self.delay + conn = self.cache.get(key) + if conn is None or not conn.keepalive: + if conn is not None: + conn.close() + conn = self.cache[key] = ftpwrapper(user, passwd, host, port, + dirs, timeout) + self.timeout[key] = time.time() + self.delay self.check_cache() - return self.cache[key] + return conn def check_cache(self): # first check for old ones @@ -1655,6 +1634,11 @@ def data_open(self, req): scheme, data = url.split(":",1) mediatype, data = data.split(",",1) + # Disallow control characters within mediatype. + if re.search(r"[\x00-\x1F\x7F]", mediatype): + raise ValueError( + "Control characters not allowed in data: mediatype") + # even base64 encoded data URLs might be quoted so unquote in any case: data = unquote_to_bytes(data) if mediatype.endswith(";base64"): @@ -1670,683 +1654,80 @@ def data_open(self, req): return addinfourl(io.BytesIO(data), headers, url) -# Code move from the old urllib module - -MAXFTPCACHE = 10 # Trim the ftp cache beyond this size - -# Helper for non-unix systems -if os.name == 'nt': - from nturl2path import url2pathname, pathname2url -else: - def url2pathname(pathname): - """OS-specific conversion from a relative URL of the 'file' scheme - to a file system path; not recommended for general use.""" - return unquote(pathname) - - def pathname2url(pathname): - """OS-specific conversion from a file system path to a relative URL - of the 'file' scheme; not recommended for general use.""" - return quote(pathname) - - -ftpcache = {} - +# Code moved from the old urllib module -class URLopener: - """Class to open URLs. - This is a class rather than just a subroutine because we may need - more than one set of global protocol-specific options. - Note -- this is a base class for those who don't want the - automatic handling of errors type 302 (relocated) and 401 - (authorization needed).""" +def url2pathname(url, *, require_scheme=False, resolve_host=False): + """Convert the given file URL to a local file system path. - __tempfiles = None + The 'file:' scheme prefix must be omitted unless *require_scheme* + is set to true. - version = "Python-urllib/%s" % __version__ - - # Constructor - def __init__(self, proxies=None, **x509): - msg = "%(class)s style of invoking requests is deprecated. " \ - "Use newer urlopen functions/methods" % {'class': self.__class__.__name__} - warnings.warn(msg, DeprecationWarning, stacklevel=3) - if proxies is None: - proxies = getproxies() - assert hasattr(proxies, 'keys'), "proxies must be a mapping" - self.proxies = proxies - self.key_file = x509.get('key_file') - self.cert_file = x509.get('cert_file') - self.addheaders = [('User-Agent', self.version), ('Accept', '*/*')] - self.__tempfiles = [] - self.__unlink = os.unlink # See cleanup() - self.tempcache = None - # Undocumented feature: if you assign {} to tempcache, - # it is used to cache files retrieved with - # self.retrieve(). This is not enabled by default - # since it does not work for changing documents (and I - # haven't got the logic to check expiration headers - # yet). - self.ftpcache = ftpcache - # Undocumented feature: you can use a different - # ftp cache by assigning to the .ftpcache member; - # in case you want logically independent URL openers - # XXX This is not threadsafe. Bah. - - def __del__(self): - self.close() - - def close(self): - self.cleanup() - - def cleanup(self): - # This code sometimes runs when the rest of this module - # has already been deleted, so it can't use any globals - # or import anything. - if self.__tempfiles: - for file in self.__tempfiles: - try: - self.__unlink(file) - except OSError: - pass - del self.__tempfiles[:] - if self.tempcache: - self.tempcache.clear() - - def addheader(self, *args): - """Add a header to be used by the HTTP interface only - e.g. u.addheader('Accept', 'sound/basic')""" - self.addheaders.append(args) - - # External interface - def open(self, fullurl, data=None): - """Use URLopener().open(file) instead of open(file, 'r').""" - fullurl = unwrap(_to_bytes(fullurl)) - fullurl = quote(fullurl, safe="%/:=&?~#+!$,;'@()*[]|") - if self.tempcache and fullurl in self.tempcache: - filename, headers = self.tempcache[fullurl] - fp = open(filename, 'rb') - return addinfourl(fp, headers, fullurl) - urltype, url = _splittype(fullurl) - if not urltype: - urltype = 'file' - if urltype in self.proxies: - proxy = self.proxies[urltype] - urltype, proxyhost = _splittype(proxy) - host, selector = _splithost(proxyhost) - url = (host, fullurl) # Signal special case to open_*() - else: - proxy = None - name = 'open_' + urltype - self.type = urltype - name = name.replace('-', '_') - if not hasattr(self, name) or name == 'open_local_file': - if proxy: - return self.open_unknown_proxy(proxy, fullurl, data) - else: - return self.open_unknown(fullurl, data) - try: - if data is None: - return getattr(self, name)(url) - else: - return getattr(self, name)(url, data) - except (HTTPError, URLError): - raise - except OSError as msg: - raise OSError('socket error', msg).with_traceback(sys.exc_info()[2]) - - def open_unknown(self, fullurl, data=None): - """Overridable interface to open unknown URL type.""" - type, url = _splittype(fullurl) - raise OSError('url error', 'unknown url type', type) - - def open_unknown_proxy(self, proxy, fullurl, data=None): - """Overridable interface to open unknown URL type.""" - type, url = _splittype(fullurl) - raise OSError('url error', 'invalid proxy for %s' % type, proxy) - - # External interface - def retrieve(self, url, filename=None, reporthook=None, data=None): - """retrieve(url) returns (filename, headers) for a local object - or (tempfilename, headers) for a remote object.""" - url = unwrap(_to_bytes(url)) - if self.tempcache and url in self.tempcache: - return self.tempcache[url] - type, url1 = _splittype(url) - if filename is None and (not type or type == 'file'): - try: - fp = self.open_local_file(url1) - hdrs = fp.info() - fp.close() - return url2pathname(_splithost(url1)[1]), hdrs - except OSError: - pass - fp = self.open(url, data) - try: - headers = fp.info() - if filename: - tfp = open(filename, 'wb') - else: - garbage, path = _splittype(url) - garbage, path = _splithost(path or "") - path, garbage = _splitquery(path or "") - path, garbage = _splitattr(path or "") - suffix = os.path.splitext(path)[1] - (fd, filename) = tempfile.mkstemp(suffix) - self.__tempfiles.append(filename) - tfp = os.fdopen(fd, 'wb') - try: - result = filename, headers - if self.tempcache is not None: - self.tempcache[url] = result - bs = 1024*8 - size = -1 - read = 0 - blocknum = 0 - if "content-length" in headers: - size = int(headers["Content-Length"]) - if reporthook: - reporthook(blocknum, bs, size) - while 1: - block = fp.read(bs) - if not block: - break - read += len(block) - tfp.write(block) - blocknum += 1 - if reporthook: - reporthook(blocknum, bs, size) - finally: - tfp.close() - finally: - fp.close() - - # raise exception if actual size does not match content-length header - if size >= 0 and read < size: - raise ContentTooShortError( - "retrieval incomplete: got only %i out of %i bytes" - % (read, size), result) - - return result - - # Each method named open_<type> knows how to open that type of URL - - def _open_generic_http(self, connection_factory, url, data): - """Make an HTTP connection using connection_class. - - This is an internal method that should be called from - open_http() or open_https(). - - Arguments: - - connection_factory should take a host name and return an - HTTPConnection instance. - - url is the url to retrieval or a host, relative-path pair. - - data is payload for a POST request or None. - """ - - user_passwd = None - proxy_passwd= None - if isinstance(url, str): - host, selector = _splithost(url) - if host: - user_passwd, host = _splituser(host) - host = unquote(host) - realhost = host - else: - host, selector = url - # check whether the proxy contains authorization information - proxy_passwd, host = _splituser(host) - # now we proceed with the url we want to obtain - urltype, rest = _splittype(selector) - url = rest - user_passwd = None - if urltype.lower() != 'http': - realhost = None - else: - realhost, rest = _splithost(rest) - if realhost: - user_passwd, realhost = _splituser(realhost) - if user_passwd: - selector = "%s://%s%s" % (urltype, realhost, rest) - if proxy_bypass(realhost): - host = realhost - - if not host: raise OSError('http error', 'no host given') - - if proxy_passwd: - proxy_passwd = unquote(proxy_passwd) - proxy_auth = base64.b64encode(proxy_passwd.encode()).decode('ascii') - else: - proxy_auth = None - - if user_passwd: - user_passwd = unquote(user_passwd) - auth = base64.b64encode(user_passwd.encode()).decode('ascii') - else: - auth = None - http_conn = connection_factory(host) - headers = {} - if proxy_auth: - headers["Proxy-Authorization"] = "Basic %s" % proxy_auth - if auth: - headers["Authorization"] = "Basic %s" % auth - if realhost: - headers["Host"] = realhost - - # Add Connection:close as we don't support persistent connections yet. - # This helps in closing the socket and avoiding ResourceWarning - - headers["Connection"] = "close" - - for header, value in self.addheaders: - headers[header] = value - - if data is not None: - headers["Content-Type"] = "application/x-www-form-urlencoded" - http_conn.request("POST", selector, data, headers) - else: - http_conn.request("GET", selector, headers=headers) - - try: - response = http_conn.getresponse() - except http.client.BadStatusLine: - # something went wrong with the HTTP status line - raise URLError("http protocol error: bad status line") - - # According to RFC 2616, "2xx" code indicates that the client's - # request was successfully received, understood, and accepted. - if 200 <= response.status < 300: - return addinfourl(response, response.msg, "http:" + url, - response.status) - else: - return self.http_error( - url, response.fp, - response.status, response.reason, response.msg, data) - - def open_http(self, url, data=None): - """Use HTTP protocol.""" - return self._open_generic_http(http.client.HTTPConnection, url, data) - - def http_error(self, url, fp, errcode, errmsg, headers, data=None): - """Handle http errors. - - Derived class can override this, or provide specific handlers - named http_error_DDD where DDD is the 3-digit error code.""" - # First check if there's a specific handler for this error - name = 'http_error_%d' % errcode - if hasattr(self, name): - method = getattr(self, name) - if data is None: - result = method(url, fp, errcode, errmsg, headers) - else: - result = method(url, fp, errcode, errmsg, headers, data) - if result: return result - return self.http_error_default(url, fp, errcode, errmsg, headers) - - def http_error_default(self, url, fp, errcode, errmsg, headers): - """Default error handler: close the connection and raise OSError.""" - fp.close() - raise HTTPError(url, errcode, errmsg, headers, None) - - if _have_ssl: - def _https_connection(self, host): - return http.client.HTTPSConnection(host, - key_file=self.key_file, - cert_file=self.cert_file) - - def open_https(self, url, data=None): - """Use HTTPS protocol.""" - return self._open_generic_http(self._https_connection, url, data) - - def open_file(self, url): - """Use local file or FTP depending on form of URL.""" - if not isinstance(url, str): - raise URLError('file error: proxy support for file protocol currently not implemented') - if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/': - raise ValueError("file:// scheme is supported only on localhost") - else: - return self.open_local_file(url) - - def open_local_file(self, url): - """Use local file.""" - import email.utils - import mimetypes - host, file = _splithost(url) - localname = url2pathname(file) - try: - stats = os.stat(localname) - except OSError as e: - raise URLError(e.strerror, e.filename) - size = stats.st_size - modified = email.utils.formatdate(stats.st_mtime, usegmt=True) - mtype = mimetypes.guess_type(url)[0] - headers = email.message_from_string( - 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' % - (mtype or 'text/plain', size, modified)) - if not host: - urlfile = file - if file[:1] == '/': - urlfile = 'file://' + file - return addinfourl(open(localname, 'rb'), headers, urlfile) - host, port = _splitport(host) - if (not port - and socket.gethostbyname(host) in ((localhost(),) + thishost())): - urlfile = file - if file[:1] == '/': - urlfile = 'file://' + file - elif file[:2] == './': - raise ValueError("local file url may start with / or file:. Unknown url of type: %s" % url) - return addinfourl(open(localname, 'rb'), headers, urlfile) - raise URLError('local file error: not on local host') - - def open_ftp(self, url): - """Use FTP protocol.""" - if not isinstance(url, str): - raise URLError('ftp error: proxy support for ftp protocol currently not implemented') - import mimetypes - host, path = _splithost(url) - if not host: raise URLError('ftp error: no host given') - host, port = _splitport(host) - user, host = _splituser(host) - if user: user, passwd = _splitpasswd(user) - else: passwd = None - host = unquote(host) - user = unquote(user or '') - passwd = unquote(passwd or '') - host = socket.gethostbyname(host) - if not port: - import ftplib - port = ftplib.FTP_PORT - else: - port = int(port) - path, attrs = _splitattr(path) - path = unquote(path) - dirs = path.split('/') - dirs, file = dirs[:-1], dirs[-1] - if dirs and not dirs[0]: dirs = dirs[1:] - if dirs and not dirs[0]: dirs[0] = '/' - key = user, host, port, '/'.join(dirs) - # XXX thread unsafe! - if len(self.ftpcache) > MAXFTPCACHE: - # Prune the cache, rather arbitrarily - for k in list(self.ftpcache): - if k != key: - v = self.ftpcache[k] - del self.ftpcache[k] - v.close() - try: - if key not in self.ftpcache: - self.ftpcache[key] = \ - ftpwrapper(user, passwd, host, port, dirs) - if not file: type = 'D' - else: type = 'I' - for attr in attrs: - attr, value = _splitvalue(attr) - if attr.lower() == 'type' and \ - value in ('a', 'A', 'i', 'I', 'd', 'D'): - type = value.upper() - (fp, retrlen) = self.ftpcache[key].retrfile(file, type) - mtype = mimetypes.guess_type("ftp:" + url)[0] - headers = "" - if mtype: - headers += "Content-Type: %s\n" % mtype - if retrlen is not None and retrlen >= 0: - headers += "Content-Length: %d\n" % retrlen - headers = email.message_from_string(headers) - return addinfourl(fp, headers, "ftp:" + url) - except ftperrors() as exp: - raise URLError('ftp error %r' % exp).with_traceback(sys.exc_info()[2]) - - def open_data(self, url, data=None): - """Use "data" URL.""" - if not isinstance(url, str): - raise URLError('data error: proxy support for data protocol currently not implemented') - # ignore POSTed data - # - # syntax of data URLs: - # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data - # mediatype := [ type "/" subtype ] *( ";" parameter ) - # data := *urlchar - # parameter := attribute "=" value - try: - [type, data] = url.split(',', 1) - except ValueError: - raise OSError('data error', 'bad data URL') - if not type: - type = 'text/plain;charset=US-ASCII' - semi = type.rfind(';') - if semi >= 0 and '=' not in type[semi:]: - encoding = type[semi+1:] - type = type[:semi] - else: - encoding = '' - msg = [] - msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT', - time.gmtime(time.time()))) - msg.append('Content-type: %s' % type) - if encoding == 'base64': - # XXX is this encoding/decoding ok? - data = base64.decodebytes(data.encode('ascii')).decode('latin-1') - else: - data = unquote(data) - msg.append('Content-Length: %d' % len(data)) - msg.append('') - msg.append(data) - msg = '\n'.join(msg) - headers = email.message_from_string(msg) - f = io.StringIO(msg) - #f.fileno = None # needed for addinfourl - return addinfourl(f, headers, url) - - -class FancyURLopener(URLopener): - """Derived class with handlers for errors we can handle (perhaps).""" - - def __init__(self, *args, **kwargs): - URLopener.__init__(self, *args, **kwargs) - self.auth_cache = {} - self.tries = 0 - self.maxtries = 10 - - def http_error_default(self, url, fp, errcode, errmsg, headers): - """Default error handling -- don't raise an exception.""" - return addinfourl(fp, headers, "http:" + url, errcode) - - def http_error_302(self, url, fp, errcode, errmsg, headers, data=None): - """Error 302 -- relocated (temporarily).""" - self.tries += 1 - try: - if self.maxtries and self.tries >= self.maxtries: - if hasattr(self, "http_error_500"): - meth = self.http_error_500 - else: - meth = self.http_error_default - return meth(url, fp, 500, - "Internal Server Error: Redirect Recursion", - headers) - result = self.redirect_internal(url, fp, errcode, errmsg, - headers, data) - return result - finally: - self.tries = 0 - - def redirect_internal(self, url, fp, errcode, errmsg, headers, data): - if 'location' in headers: - newurl = headers['location'] - elif 'uri' in headers: - newurl = headers['uri'] - else: - return - fp.close() - - # In case the server sent a relative URL, join with original: - newurl = urljoin(self.type + ":" + url, newurl) - - urlparts = urlparse(newurl) - - # For security reasons, we don't allow redirection to anything other - # than http, https and ftp. - - # We are using newer HTTPError with older redirect_internal method - # This older method will get deprecated in 3.3 - - if urlparts.scheme not in ('http', 'https', 'ftp', ''): - raise HTTPError(newurl, errcode, - errmsg + - " Redirection to url '%s' is not allowed." % newurl, - headers, fp) - - return self.open(newurl) - - def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): - """Error 301 -- also relocated (permanently).""" - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - - def http_error_303(self, url, fp, errcode, errmsg, headers, data=None): - """Error 303 -- also relocated (essentially identical to 302).""" - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - - def http_error_307(self, url, fp, errcode, errmsg, headers, data=None): - """Error 307 -- relocated, but turn POST into error.""" - if data is None: - return self.http_error_302(url, fp, errcode, errmsg, headers, data) - else: - return self.http_error_default(url, fp, errcode, errmsg, headers) - - def http_error_401(self, url, fp, errcode, errmsg, headers, data=None, - retry=False): - """Error 401 -- authentication required. - This function supports Basic authentication only.""" - if 'www-authenticate' not in headers: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - stuff = headers['www-authenticate'] - match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff) - if not match: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - scheme, realm = match.groups() - if scheme.lower() != 'basic': - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - if not retry: - URLopener.http_error_default(self, url, fp, errcode, errmsg, - headers) - name = 'retry_' + self.type + '_basic_auth' - if data is None: - return getattr(self,name)(url, realm) - else: - return getattr(self,name)(url, realm, data) - - def http_error_407(self, url, fp, errcode, errmsg, headers, data=None, - retry=False): - """Error 407 -- proxy authentication required. - This function supports Basic authentication only.""" - if 'proxy-authenticate' not in headers: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - stuff = headers['proxy-authenticate'] - match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff) - if not match: - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - scheme, realm = match.groups() - if scheme.lower() != 'basic': - URLopener.http_error_default(self, url, fp, - errcode, errmsg, headers) - if not retry: - URLopener.http_error_default(self, url, fp, errcode, errmsg, - headers) - name = 'retry_proxy_' + self.type + '_basic_auth' - if data is None: - return getattr(self,name)(url, realm) - else: - return getattr(self,name)(url, realm, data) - - def retry_proxy_http_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - newurl = 'http://' + host + selector - proxy = self.proxies['http'] - urltype, proxyhost = _splittype(proxy) - proxyhost, proxyselector = _splithost(proxyhost) - i = proxyhost.find('@') + 1 - proxyhost = proxyhost[i:] - user, passwd = self.get_user_passwd(proxyhost, realm, i) - if not (user or passwd): return None - proxyhost = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), proxyhost) - self.proxies['http'] = 'http://' + proxyhost + proxyselector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_proxy_https_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - newurl = 'https://' + host + selector - proxy = self.proxies['https'] - urltype, proxyhost = _splittype(proxy) - proxyhost, proxyselector = _splithost(proxyhost) - i = proxyhost.find('@') + 1 - proxyhost = proxyhost[i:] - user, passwd = self.get_user_passwd(proxyhost, realm, i) - if not (user or passwd): return None - proxyhost = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), proxyhost) - self.proxies['https'] = 'https://' + proxyhost + proxyselector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_http_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - i = host.find('@') + 1 - host = host[i:] - user, passwd = self.get_user_passwd(host, realm, i) - if not (user or passwd): return None - host = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), host) - newurl = 'http://' + host + selector - if data is None: - return self.open(newurl) - else: - return self.open(newurl, data) - - def retry_https_basic_auth(self, url, realm, data=None): - host, selector = _splithost(url) - i = host.find('@') + 1 - host = host[i:] - user, passwd = self.get_user_passwd(host, realm, i) - if not (user or passwd): return None - host = "%s:%s@%s" % (quote(user, safe=''), - quote(passwd, safe=''), host) - newurl = 'https://' + host + selector - if data is None: - return self.open(newurl) + The URL authority may be resolved with gethostbyname() if + *resolve_host* is set to true. + """ + if not require_scheme: + url = 'file:' + url + scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment. + if scheme != 'file': + raise URLError("URL is missing a 'file:' scheme") + if os.name == 'nt': + if authority[1:2] == ':': + # e.g. file://c:/file.txt + url = authority + url + elif not _is_local_authority(authority, resolve_host): + # e.g. file://server/share/file.txt + url = '//' + authority + url + elif url[:3] == '///': + # e.g. file://///server/share/file.txt + url = url[1:] else: - return self.open(newurl, data) - - def get_user_passwd(self, host, realm, clear_cache=0): - key = realm + '@' + host.lower() - if key in self.auth_cache: - if clear_cache: - del self.auth_cache[key] - else: - return self.auth_cache[key] - user, passwd = self.prompt_user_passwd(host, realm) - if user or passwd: self.auth_cache[key] = (user, passwd) - return user, passwd - - def prompt_user_passwd(self, host, realm): - """Override this in a GUI environment!""" - import getpass - try: - user = input("Enter username for %s at %s: " % (realm, host)) - passwd = getpass.getpass("Enter password for %s in %s at %s: " % - (user, realm, host)) - return user, passwd - except KeyboardInterrupt: - print() - return None, None + if url[:1] == '/' and url[2:3] in (':', '|'): + # Skip past extra slash before DOS drive in URL path. + url = url[1:] + if url[1:2] == '|': + # Older URLs use a pipe after a drive letter + url = url[:1] + ':' + url[2:] + url = url.replace('/', '\\') + elif not _is_local_authority(authority, resolve_host): + raise URLError("file:// scheme is supported only on localhost") + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + return unquote(url, encoding=encoding, errors=errors) + + +def pathname2url(pathname, *, add_scheme=False): + """Convert the given local file system path to a file URL. + + The 'file:' scheme prefix is omitted unless *add_scheme* + is set to true. + """ + if os.name == 'nt': + pathname = pathname.replace('\\', '/') + encoding = sys.getfilesystemencoding() + errors = sys.getfilesystemencodeerrors() + scheme = 'file:' if add_scheme else '' + drive, root, tail = os.path.splitroot(pathname) + if drive: + # First, clean up some special forms. We are going to sacrifice the + # additional information anyway + if drive[:4] == '//?/': + drive = drive[4:] + if drive[:4].upper() == 'UNC/': + drive = '//' + drive[4:] + if drive[1:] == ':': + # DOS drive specified. Add three slashes to the start, producing + # an authority section with a zero-length authority, and a path + # section starting with a single slash. + drive = '///' + drive + drive = quote(drive, encoding=encoding, errors=errors, safe='/:') + elif root: + # Add explicitly empty authority to absolute path. If the path + # starts with exactly one slash then this change is mostly + # cosmetic, but if it begins with two or more slashes then this + # avoids interpreting the path as a URL authority. + root = '//' + root + tail = quote(tail, encoding=encoding, errors=errors) + return scheme + drive + root + tail # Utility functions @@ -2436,8 +1817,7 @@ def retrfile(self, file, type): conn, retrlen = self.ftp.ntransfercmd(cmd) except ftplib.error_perm as reason: if str(reason)[:3] != '550': - raise URLError('ftp error: %r' % reason).with_traceback( - sys.exc_info()[2]) + raise URLError(f'ftp error: {reason}') from reason if not conn: # Set transfer mode to ASCII! self.ftp.voidcmd('TYPE A') @@ -2464,7 +1844,13 @@ def retrfile(self, file, type): return (ftpobj, retrlen) def endtransfer(self): + if not self.busy: + return self.busy = 0 + try: + self.ftp.voidresp() + except ftperrors(): + pass def close(self): self.keepalive = False @@ -2489,31 +1875,35 @@ def getproxies_environment(): """Return a dictionary of scheme -> proxy server URL mappings. Scan the environment for variables named <scheme>_proxy; - this seems to be the standard convention. If you need a - different way, you can pass a proxies dictionary to the - [Fancy]URLopener constructor. - + this seems to be the standard convention. """ - proxies = {} # in order to prefer lowercase variables, process environment in # two passes: first matches any, second pass matches lowercase only - for name, value in os.environ.items(): - name = name.lower() - if value and name[-6:] == '_proxy': - proxies[name[:-6]] = value + + # select only environment variables which end in (after making lowercase) _proxy + proxies = {} + environment = [] + for name in os.environ: + # fast screen underscore position before more expensive case-folding + if len(name) > 5 and name[-6] == "_" and name[-5:].lower() == "proxy": + value = os.environ[name] + proxy_name = name[:-6].lower() + environment.append((name, value, proxy_name)) + if value: + proxies[proxy_name] = value # CVE-2016-1000110 - If we are running as CGI script, forget HTTP_PROXY # (non-all-lowercase) as it may be set from the web server by a "Proxy:" # header from the client # If "proxy" is lowercase, it will still be used thanks to the next block if 'REQUEST_METHOD' in os.environ: proxies.pop('http', None) - for name, value in os.environ.items(): + for name, value, proxy_name in environment: + # not case-folded, checking here for lower-case env vars only if name[-6:] == '_proxy': - name = name.lower() if value: - proxies[name[:-6]] = value + proxies[proxy_name] = value else: - proxies.pop(name[:-6], None) + proxies.pop(proxy_name, None) return proxies def proxy_bypass_environment(host, proxies=None): @@ -2566,6 +1956,7 @@ def _proxy_bypass_macosx_sysconf(host, proxy_settings): } """ from fnmatch import fnmatch + from ipaddress import AddressValueError, IPv4Address hostonly, port = _splitport(host) @@ -2582,20 +1973,17 @@ def ip2num(ipAddr): return True hostIP = None + try: + hostIP = int(IPv4Address(hostonly)) + except AddressValueError: + pass for value in proxy_settings.get('exceptions', ()): # Items in the list are strings like these: *.local, 169.254/16 if not value: continue m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value) - if m is not None: - if hostIP is None: - try: - hostIP = socket.gethostbyname(hostonly) - hostIP = ip2num(hostIP) - except OSError: - continue - + if m is not None and hostIP is not None: base = ip2num(m.group(1)) mask = m.group(2) if mask is None: @@ -2618,6 +2006,31 @@ def ip2num(ipAddr): return False +# Same as _proxy_bypass_macosx_sysconf, testable on all platforms +def _proxy_bypass_winreg_override(host, override): + """Return True if the host should bypass the proxy server. + + The proxy override list is obtained from the Windows + Internet settings proxy override registry value. + + An example of a proxy override value is: + "www.example.com;*.example.net; 192.168.0.1" + """ + from fnmatch import fnmatch + + host, _ = _splitport(host) + proxy_override = override.split(';') + for test in proxy_override: + test = test.strip() + # "<local>" should bypass the proxy server for all intranet addresses + if test == '<local>': + if '.' not in host: + return True + elif fnmatch(host, test): + return True + return False + + if sys.platform == 'darwin': from _scproxy import _get_proxy_settings, _get_proxies @@ -2716,7 +2129,7 @@ def proxy_bypass_registry(host): import winreg except ImportError: # Std modules, so should be around - but you never know! - return 0 + return False try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') @@ -2726,40 +2139,10 @@ def proxy_bypass_registry(host): 'ProxyOverride')[0]) # ^^^^ Returned as Unicode but problems if not converted to ASCII except OSError: - return 0 + return False if not proxyEnable or not proxyOverride: - return 0 - # try to make a host list from name and IP address. - rawHost, port = _splitport(host) - host = [rawHost] - try: - addr = socket.gethostbyname(rawHost) - if addr != rawHost: - host.append(addr) - except OSError: - pass - try: - fqdn = socket.getfqdn(rawHost) - if fqdn != rawHost: - host.append(fqdn) - except OSError: - pass - # make a check value list from the registry entry: replace the - # '<local>' string by the localhost entry and the corresponding - # canonical entry. - proxyOverride = proxyOverride.split(';') - # now check if we match one of the registry values. - for test in proxyOverride: - if test == '<local>': - if '.' not in rawHost: - return 1 - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char - for val in host: - if re.match(test, val, re.I): - return 1 - return 0 + return False + return _proxy_bypass_winreg_override(host, proxyOverride) def proxy_bypass(host): """Return True, if host should be bypassed. diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index c58565e3945..4009fd6b58f 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -11,6 +11,8 @@ """ import collections +import re +import urllib.error import urllib.parse import urllib.request @@ -19,6 +21,19 @@ RequestRate = collections.namedtuple("RequestRate", "requests seconds") +def normalize(path): + unquoted = urllib.parse.unquote(path, errors='surrogateescape') + return urllib.parse.quote(unquoted, errors='surrogateescape') + +def normalize_path(path): + path, sep, query = path.partition('?') + path = normalize(path) + if sep: + query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + + class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -54,7 +69,7 @@ def modified(self): def set_url(self, url): """Sets the URL referring to a robots.txt file.""" self.url = url - self.host, self.path = urllib.parse.urlparse(url)[1:3] + self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" @@ -65,9 +80,10 @@ def read(self): self.disallow_all = True elif err.code >= 400 and err.code < 500: self.allow_all = True + err.close() else: raw = f.read() - self.parse(raw.decode("utf-8").splitlines()) + self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) def _add_entry(self, entry): if "*" in entry.useragents: @@ -111,7 +127,7 @@ def parse(self, lines): line = line.split(':', 1) if len(line) == 2: line[0] = line[0].strip().lower() - line[1] = urllib.parse.unquote(line[1].strip()) + line[1] = line[1].strip() if line[0] == "user-agent": if state == 2: self._add_entry(entry) @@ -165,10 +181,11 @@ def can_fetch(self, useragent, url): return False # search for given user agent matches # the first match counts - parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url)) - url = urllib.parse.urlunparse(('','',parsed_url.path, - parsed_url.params,parsed_url.query, parsed_url.fragment)) - url = urllib.parse.quote(url) + # TODO: The private API is used in order to preserve an empty query. + # This is temporary until the public API starts supporting this feature. + parsed_url = urllib.parse._urlsplit(url, '') + url = urllib.parse._urlunsplit(None, None, *parsed_url[2:]) + url = normalize_path(url) if not url: url = "/" for entry in self.entries: @@ -211,7 +228,6 @@ def __str__(self): entries = entries + [self.default_entry] return '\n\n'.join(map(str, entries)) - class RuleLine: """A rule line is a single "Allow:" (allowance==True) or "Disallow:" (allowance==False) followed by a path.""" @@ -219,8 +235,7 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True - path = urllib.parse.urlunparse(urllib.parse.urlparse(path)) - self.path = urllib.parse.quote(path) + self.path = normalize_path(path) self.allowance = allowance def applies_to(self, filename): @@ -266,7 +281,7 @@ def applies_to(self, useragent): def allowance(self, filename): """Preconditions: - our agent applies to this entry - - filename is URL decoded""" + - filename is URL encoded""" for line in self.rulelines: if line.applies_to(filename): return line.allowance diff --git a/Lib/uu.py b/Lib/uu.py deleted file mode 100755 index d68d29374a8..00000000000 --- a/Lib/uu.py +++ /dev/null @@ -1,199 +0,0 @@ -#! /usr/bin/env python3 - -# Copyright 1994 by Lance Ellinghouse -# Cathedral City, California Republic, United States of America. -# All Rights Reserved -# Permission to use, copy, modify, and distribute this software and its -# documentation for any purpose and without fee is hereby granted, -# provided that the above copyright notice appear in all copies and that -# both that copyright notice and this permission notice appear in -# supporting documentation, and that the name of Lance Ellinghouse -# not be used in advertising or publicity pertaining to distribution -# of the software without specific, written prior permission. -# LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -# FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE -# FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# -# Modified by Jack Jansen, CWI, July 1995: -# - Use binascii module to do the actual line-by-line conversion -# between ascii and binary. This results in a 1000-fold speedup. The C -# version is still 5 times faster, though. -# - Arguments more compliant with python standard - -"""Implementation of the UUencode and UUdecode functions. - -encode(in_file, out_file [,name, mode]) -decode(in_file [, out_file, mode]) -""" - -import binascii -import os -import sys - -__all__ = ["Error", "encode", "decode"] - -class Error(Exception): - pass - -def encode(in_file, out_file, name=None, mode=None): - """Uuencode file""" - # - # If in_file is a pathname open it and change defaults - # - opened_files = [] - try: - if in_file == '-': - in_file = sys.stdin.buffer - elif isinstance(in_file, str): - if name is None: - name = os.path.basename(in_file) - if mode is None: - try: - mode = os.stat(in_file).st_mode - except AttributeError: - pass - in_file = open(in_file, 'rb') - opened_files.append(in_file) - # - # Open out_file if it is a pathname - # - if out_file == '-': - out_file = sys.stdout.buffer - elif isinstance(out_file, str): - out_file = open(out_file, 'wb') - opened_files.append(out_file) - # - # Set defaults for name and mode - # - if name is None: - name = '-' - if mode is None: - mode = 0o666 - # - # Write the data - # - out_file.write(('begin %o %s\n' % ((mode & 0o777), name)).encode("ascii")) - data = in_file.read(45) - while len(data) > 0: - out_file.write(binascii.b2a_uu(data)) - data = in_file.read(45) - out_file.write(b' \nend\n') - finally: - for f in opened_files: - f.close() - - -def decode(in_file, out_file=None, mode=None, quiet=False): - """Decode uuencoded file""" - # - # Open the input file, if needed. - # - opened_files = [] - if in_file == '-': - in_file = sys.stdin.buffer - elif isinstance(in_file, str): - in_file = open(in_file, 'rb') - opened_files.append(in_file) - - try: - # - # Read until a begin is encountered or we've exhausted the file - # - while True: - hdr = in_file.readline() - if not hdr: - raise Error('No valid begin line found in input file') - if not hdr.startswith(b'begin'): - continue - hdrfields = hdr.split(b' ', 2) - if len(hdrfields) == 3 and hdrfields[0] == b'begin': - try: - int(hdrfields[1], 8) - break - except ValueError: - pass - if out_file is None: - # If the filename isn't ASCII, what's up with that?!? - out_file = hdrfields[2].rstrip(b' \t\r\n\f').decode("ascii") - if os.path.exists(out_file): - raise Error('Cannot overwrite existing file: %s' % out_file) - if mode is None: - mode = int(hdrfields[1], 8) - # - # Open the output file - # - if out_file == '-': - out_file = sys.stdout.buffer - elif isinstance(out_file, str): - fp = open(out_file, 'wb') - try: - os.path.chmod(out_file, mode) - except AttributeError: - pass - out_file = fp - opened_files.append(out_file) - # - # Main decoding loop - # - s = in_file.readline() - while s and s.strip(b' \t\r\n\f') != b'end': - try: - data = binascii.a2b_uu(s) - except binascii.Error as v: - # Workaround for broken uuencoders by /Fredrik Lundh - nbytes = (((s[0]-32) & 63) * 4 + 5) // 3 - data = binascii.a2b_uu(s[:nbytes]) - if not quiet: - sys.stderr.write("Warning: %s\n" % v) - out_file.write(data) - s = in_file.readline() - if not s: - raise Error('Truncated input file') - finally: - for f in opened_files: - f.close() - -def test(): - """uuencode/uudecode main program""" - - import optparse - parser = optparse.OptionParser(usage='usage: %prog [-d] [-t] [input [output]]') - parser.add_option('-d', '--decode', dest='decode', help='Decode (instead of encode)?', default=False, action='store_true') - parser.add_option('-t', '--text', dest='text', help='data is text, encoded format unix-compatible text?', default=False, action='store_true') - - (options, args) = parser.parse_args() - if len(args) > 2: - parser.error('incorrect number of arguments') - sys.exit(1) - - # Use the binary streams underlying stdin/stdout - input = sys.stdin.buffer - output = sys.stdout.buffer - if len(args) > 0: - input = args[0] - if len(args) > 1: - output = args[1] - - if options.decode: - if options.text: - if isinstance(output, str): - output = open(output, 'wb') - else: - print(sys.argv[0], ': cannot do -t to stdout') - sys.exit(1) - decode(input, output) - else: - if options.text: - if isinstance(input, str): - input = open(input, 'rb') - else: - print(sys.argv[0], ': cannot do -t from stdin') - sys.exit(1) - encode(input, output) - -if __name__ == '__main__': - test() diff --git a/Lib/uuid.py b/Lib/uuid.py index e4298253c21..313f2fc46cb 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -1,8 +1,12 @@ -r"""UUID objects (universally unique identifiers) according to RFC 4122. +r"""UUID objects (universally unique identifiers) according to RFC 4122/9562. -This module provides immutable UUID objects (class UUID) and the functions -uuid1(), uuid3(), uuid4(), uuid5() for generating version 1, 3, 4, and 5 -UUIDs as specified in RFC 4122. +This module provides immutable UUID objects (class UUID) and functions for +generating UUIDs corresponding to a specific UUID version as specified in +RFC 4122/9562, e.g., uuid1() for UUID version 1, uuid3() for UUID version 3, +and so on. + +Note that UUID version 2 is deliberately omitted as it is outside the scope +of the RFC. If all you want is a unique ID, you should probably call uuid1() or uuid4(). Note that uuid1() may compromise privacy since it creates a UUID containing @@ -42,26 +46,36 @@ # make a UUID from a 16-byte string >>> uuid.UUID(bytes=x.bytes) UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') + + # get the Nil UUID + >>> uuid.NIL + UUID('00000000-0000-0000-0000-000000000000') + + # get the Max UUID + >>> uuid.MAX + UUID('ffffffff-ffff-ffff-ffff-ffffffffffff') """ import os import sys +import time -from enum import Enum +from enum import Enum, _simple_enum __author__ = 'Ka-Ping Yee <ping@zesty.ca>' # The recognized platforms - known behaviors -if sys.platform in ('win32', 'darwin'): - _AIX = _LINUX = False -elif sys.platform in ('emscripten', 'wasi'): # XXX: RUSTPYTHON; patched to support those platforms +if sys.platform in {'win32', 'darwin', 'emscripten', 'wasi'}: _AIX = _LINUX = False +elif sys.platform == 'linux': + _LINUX = True + _AIX = False else: import platform _platform_system = platform.system() _AIX = _platform_system == 'AIX' - _LINUX = _platform_system == 'Linux' + _LINUX = _platform_system in ('Linux', 'Android') _MAC_DELIM = b':' _MAC_OMITS_LEADING_ZEROES = False @@ -77,12 +91,26 @@ bytes_ = bytes # The built-in bytes type -class SafeUUID(Enum): +@_simple_enum(Enum) +class SafeUUID: safe = 0 unsafe = -1 unknown = None +_UINT_128_MAX = (1 << 128) - 1 +# 128-bit mask to clear the variant and version bits of a UUID integral value +_RFC_4122_CLEARFLAGS_MASK = ~((0xf000 << 64) | (0xc000 << 48)) +# RFC 4122 variant bits and version bits to activate on a UUID integral value. +_RFC_4122_VERSION_1_FLAGS = ((1 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_3_FLAGS = ((3 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_4_FLAGS = ((4 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_5_FLAGS = ((5 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_6_FLAGS = ((6 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_7_FLAGS = ((7 << 76) | (0x8000 << 48)) +_RFC_4122_VERSION_8_FLAGS = ((8 << 76) | (0x8000 << 48)) + + class UUID: """Instances of the UUID class represent UUIDs as specified in RFC 4122. UUID objects are immutable, hashable, and usable as dictionary keys. @@ -106,7 +134,16 @@ class UUID: fields a tuple of the six integer fields of the UUID, which are also available as six individual attributes - and two derived attributes: + and two derived attributes. Those attributes are not + always relevant to all UUID versions: + + The 'time_*' attributes are only relevant to version 1. + + The 'clock_seq*' and 'node' attributes are only relevant + to versions 1 and 6. + + The 'time' attribute is only relevant to versions 1, 6 + and 7. time_low the first 32 bits of the UUID time_mid the next 16 bits of the UUID @@ -115,19 +152,20 @@ class UUID: clock_seq_low the next 8 bits of the UUID node the last 48 bits of the UUID - time the 60-bit timestamp + time the 60-bit timestamp for UUIDv1/v6, + or the 48-bit timestamp for UUIDv7 clock_seq the 14-bit sequence number hex the UUID as a 32-character hexadecimal string int the UUID as a 128-bit integer - urn the UUID as a URN as specified in RFC 4122 + urn the UUID as a URN as specified in RFC 4122/9562 variant the UUID variant (one of the constants RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, or RESERVED_FUTURE) - version the UUID version number (1 through 5, meaningful only + version the UUID version number (1 through 8, meaningful only when the variant is RFC_4122) is_safe An enum indicating whether the UUID has been generated in @@ -172,57 +210,69 @@ def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None, if [hex, bytes, bytes_le, fields, int].count(None) != 4: raise TypeError('one of the hex, bytes, bytes_le, fields, ' 'or int arguments must be given') - if hex is not None: + if int is not None: + pass + elif hex is not None: hex = hex.replace('urn:', '').replace('uuid:', '') hex = hex.strip('{}').replace('-', '') if len(hex) != 32: raise ValueError('badly formed hexadecimal UUID string') int = int_(hex, 16) - if bytes_le is not None: + elif bytes_le is not None: if len(bytes_le) != 16: raise ValueError('bytes_le is not a 16-char string') + assert isinstance(bytes_le, bytes_), repr(bytes_le) bytes = (bytes_le[4-1::-1] + bytes_le[6-1:4-1:-1] + bytes_le[8-1:6-1:-1] + bytes_le[8:]) - if bytes is not None: + int = int_.from_bytes(bytes) # big endian + elif bytes is not None: if len(bytes) != 16: raise ValueError('bytes is not a 16-char string') assert isinstance(bytes, bytes_), repr(bytes) - int = int_.from_bytes(bytes, byteorder='big') - if fields is not None: + int = int_.from_bytes(bytes) # big endian + elif fields is not None: if len(fields) != 6: raise ValueError('fields is not a 6-tuple') (time_low, time_mid, time_hi_version, clock_seq_hi_variant, clock_seq_low, node) = fields - if not 0 <= time_low < 1<<32: + if not 0 <= time_low < (1 << 32): raise ValueError('field 1 out of range (need a 32-bit value)') - if not 0 <= time_mid < 1<<16: + if not 0 <= time_mid < (1 << 16): raise ValueError('field 2 out of range (need a 16-bit value)') - if not 0 <= time_hi_version < 1<<16: + if not 0 <= time_hi_version < (1 << 16): raise ValueError('field 3 out of range (need a 16-bit value)') - if not 0 <= clock_seq_hi_variant < 1<<8: + if not 0 <= clock_seq_hi_variant < (1 << 8): raise ValueError('field 4 out of range (need an 8-bit value)') - if not 0 <= clock_seq_low < 1<<8: + if not 0 <= clock_seq_low < (1 << 8): raise ValueError('field 5 out of range (need an 8-bit value)') - if not 0 <= node < 1<<48: + if not 0 <= node < (1 << 48): raise ValueError('field 6 out of range (need a 48-bit value)') clock_seq = (clock_seq_hi_variant << 8) | clock_seq_low int = ((time_low << 96) | (time_mid << 80) | (time_hi_version << 64) | (clock_seq << 48) | node) - if int is not None: - if not 0 <= int < 1<<128: - raise ValueError('int is out of range (need a 128-bit value)') + if not 0 <= int <= _UINT_128_MAX: + raise ValueError('int is out of range (need a 128-bit value)') if version is not None: - if not 1 <= version <= 5: + if not 1 <= version <= 8: raise ValueError('illegal version number') - # Set the variant to RFC 4122. - int &= ~(0xc000 << 48) - int |= 0x8000 << 48 + # clear the variant and the version number bits + int &= _RFC_4122_CLEARFLAGS_MASK + # Set the variant to RFC 4122/9562. + int |= 0x8000_0000_0000_0000 # (0x8000 << 48) # Set the version number. - int &= ~(0xf000 << 64) int |= version << 76 object.__setattr__(self, 'int', int) object.__setattr__(self, 'is_safe', is_safe) + @classmethod + def _from_int(cls, value): + """Create a UUID from an integer *value*. Internal use only.""" + assert 0 <= value <= _UINT_128_MAX, repr(value) + self = object.__new__(cls) + object.__setattr__(self, 'int', value) + object.__setattr__(self, 'is_safe', SafeUUID.unknown) + return self + def __getstate__(self): d = {'int': self.int} if self.is_safe != SafeUUID.unknown: @@ -279,13 +329,12 @@ def __setattr__(self, name, value): raise TypeError('UUID objects are immutable') def __str__(self): - hex = '%032x' % self.int - return '%s-%s-%s-%s-%s' % ( - hex[:8], hex[8:12], hex[12:16], hex[16:20], hex[20:]) + x = self.hex + return f'{x[:8]}-{x[8:12]}-{x[12:16]}-{x[16:20]}-{x[20:]}' @property def bytes(self): - return self.int.to_bytes(16, 'big') + return self.int.to_bytes(16) # big endian @property def bytes_le(self): @@ -320,8 +369,22 @@ def clock_seq_low(self): @property def time(self): - return (((self.time_hi_version & 0x0fff) << 48) | - (self.time_mid << 32) | self.time_low) + if self.version == 6: + # time_hi (32) | time_mid (16) | ver (4) | time_lo (12) | ... (64) + time_hi = self.int >> 96 + time_lo = (self.int >> 64) & 0x0fff + return time_hi << 28 | (self.time_mid << 12) | time_lo + elif self.version == 7: + # unix_ts_ms (48) | ... (80) + return self.int >> 80 + else: + # time_lo (32) | time_mid (16) | ver (4) | time_hi (12) | ... (64) + # + # For compatibility purposes, we do not warn or raise when the + # version is not 1 (timestamp is irrelevant to other versions). + time_hi = (self.int >> 64) & 0x0fff + time_lo = self.int >> 96 + return time_hi << 48 | (self.time_mid << 32) | time_lo @property def clock_seq(self): @@ -334,7 +397,7 @@ def node(self): @property def hex(self): - return '%032x' % self.int + return self.bytes.hex() @property def urn(self): @@ -353,7 +416,7 @@ def variant(self): @property def version(self): - # The version bits are only meaningful for RFC 4122 UUIDs. + # The version bits are only meaningful for RFC 4122/9562 UUIDs. if self.variant == RFC_4122: return int((self.int >> 76) & 0xf) @@ -372,7 +435,12 @@ def _get_command_stdout(command, *args): # for are actually localized, but in theory some system could do so.) env = dict(os.environ) env['LC_ALL'] = 'C' - proc = subprocess.Popen((executable,) + args, + # Empty strings will be quoted by popen so we should just omit it + if args != ('',): + command = (executable, *args) + else: + command = (executable,) + proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) @@ -397,7 +465,7 @@ def _get_command_stdout(command, *args): # over locally administered ones since the former are globally unique, but # we'll return the first of the latter found if that's all the machine has. # -# See https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local +# See https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local_(U/L_bit) def _is_universal(mac): return not (mac & (1 << 41)) @@ -512,7 +580,7 @@ def _ifconfig_getnode(): mac = _find_mac_near_keyword('ifconfig', args, keywords, lambda i: i+1) if mac: return mac - return None + return None def _ip_getnode(): """Get the hardware address on Unix by running ip.""" @@ -525,6 +593,8 @@ def _ip_getnode(): def _arp_getnode(): """Get the hardware address on Unix by running arp.""" import os, socket + if not hasattr(socket, "gethostbyname"): + return None try: ip_addr = socket.gethostbyname(socket.gethostname()) except OSError: @@ -558,60 +628,48 @@ def _netstat_getnode(): # This works on AIX and might work on Tru64 UNIX. return _find_mac_under_heading('netstat', '-ian', b'Address') -def _ipconfig_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - -def _netbios_getnode(): - """[DEPRECATED] Get the hardware address on Windows.""" - # bpo-40501: UuidCreateSequential() is now the only supported approach - return _windll_getnode() - # Import optional C extension at toplevel, to help disabling it when testing try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) + _has_stable_extractable_node = _uuid.has_stable_extractable_node _UuidCreate = getattr(_uuid, "UuidCreate", None) - _has_uuid_generate_time_safe = _uuid.has_uuid_generate_time_safe except ImportError: _uuid = None _generate_time_safe = None + _has_stable_extractable_node = False _UuidCreate = None - _has_uuid_generate_time_safe = None - - -def _load_system_functions(): - """[DEPRECATED] Platform-specific functions loaded at import time""" def _unix_getnode(): """Get the hardware address on Unix using the _uuid extension module.""" - if _generate_time_safe: + if _generate_time_safe and _has_stable_extractable_node: uuid_time, _ = _generate_time_safe() return UUID(bytes=uuid_time).node def _windll_getnode(): """Get the hardware address on Windows using the _uuid extension module.""" - if _UuidCreate: + if _UuidCreate and _has_stable_extractable_node: uuid_bytes = _UuidCreate() return UUID(bytes_le=uuid_bytes).node def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or - # pseudo-randomly generated value may be used; see Section 4.5. The - # multicast bit must be set in such addresses, in order that they will - # never conflict with addresses obtained from network cards." + # RFC 9562, §6.10-3 says that + # + # Implementations MAY elect to obtain a 48-bit cryptographic-quality + # random number as per Section 6.9 to use as the Node ID. [...] [and] + # implementations MUST set the least significant bit of the first octet + # of the Node ID to 1. This bit is the unicast or multicast bit, which + # will never be set in IEEE 802 addresses obtained from network cards. # # The "multicast bit" of a MAC address is defined to be "the least # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # - # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast - import random - return random.getrandbits(48) | (1 << 40) + # See https://en.wikipedia.org/w/index.php?title=MAC_address&oldid=1128764812#Universal_vs._local_(U/L_bit) + return int.from_bytes(os.urandom(6)) | (1 << 40) # _OS_GETTERS, when known, are targeted for a specific OS or platform. @@ -682,7 +740,6 @@ def uuid1(node=None, clock_seq=None): return UUID(bytes=uuid_time, is_safe=is_safe) global _last_timestamp - import time nanoseconds = time.time_ns() # 0x01b21dd213814000 is the number of 100-ns intervals between the # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. @@ -705,22 +762,234 @@ def uuid1(node=None, clock_seq=None): def uuid3(namespace, name): """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" - from hashlib import md5 - digest = md5( - namespace.bytes + bytes(name, "utf-8"), - usedforsecurity=False - ).digest() - return UUID(bytes=digest[:16], version=3) + if isinstance(name, str): + name = bytes(name, "utf-8") + import hashlib + h = hashlib.md5(namespace.bytes + name, usedforsecurity=False) + int_uuid_3 = int.from_bytes(h.digest()) + int_uuid_3 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_3 |= _RFC_4122_VERSION_3_FLAGS + return UUID._from_int(int_uuid_3) def uuid4(): """Generate a random UUID.""" - return UUID(bytes=os.urandom(16), version=4) + int_uuid_4 = int.from_bytes(os.urandom(16)) + int_uuid_4 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_4 |= _RFC_4122_VERSION_4_FLAGS + return UUID._from_int(int_uuid_4) def uuid5(namespace, name): """Generate a UUID from the SHA-1 hash of a namespace UUID and a name.""" - from hashlib import sha1 - hash = sha1(namespace.bytes + bytes(name, "utf-8")).digest() - return UUID(bytes=hash[:16], version=5) + if isinstance(name, str): + name = bytes(name, "utf-8") + import hashlib + h = hashlib.sha1(namespace.bytes + name, usedforsecurity=False) + int_uuid_5 = int.from_bytes(h.digest()[:16]) + int_uuid_5 &= _RFC_4122_CLEARFLAGS_MASK + int_uuid_5 |= _RFC_4122_VERSION_5_FLAGS + return UUID._from_int(int_uuid_5) + + +_last_timestamp_v6 = None + +def uuid6(node=None, clock_seq=None): + """Similar to :func:`uuid1` but where fields are ordered differently + for improved DB locality. + + More precisely, given a 60-bit timestamp value as specified for UUIDv1, + for UUIDv6 the first 48 most significant bits are stored first, followed + by the 4-bit version (same position), followed by the remaining 12 bits + of the original 60-bit timestamp. + """ + global _last_timestamp_v6 + import time + nanoseconds = time.time_ns() + # 0x01b21dd213814000 is the number of 100-ns intervals between the + # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. + timestamp = nanoseconds // 100 + 0x01b21dd213814000 + if _last_timestamp_v6 is not None and timestamp <= _last_timestamp_v6: + timestamp = _last_timestamp_v6 + 1 + _last_timestamp_v6 = timestamp + if clock_seq is None: + import random + clock_seq = random.getrandbits(14) # instead of stable storage + time_hi_and_mid = (timestamp >> 12) & 0xffff_ffff_ffff + time_lo = timestamp & 0x0fff # keep 12 bits and clear version bits + clock_s = clock_seq & 0x3fff # keep 14 bits and clear variant bits + if node is None: + node = getnode() + # --- 32 + 16 --- -- 4 -- -- 12 -- -- 2 -- -- 14 --- 48 + # time_hi_and_mid | version | time_lo | variant | clock_seq | node + int_uuid_6 = time_hi_and_mid << 80 + int_uuid_6 |= time_lo << 64 + int_uuid_6 |= clock_s << 48 + int_uuid_6 |= node & 0xffff_ffff_ffff + # by construction, the variant and version bits are already cleared + int_uuid_6 |= _RFC_4122_VERSION_6_FLAGS + return UUID._from_int(int_uuid_6) + + +_last_timestamp_v7 = None +_last_counter_v7 = 0 # 42-bit counter + +def _uuid7_get_counter_and_tail(): + rand = int.from_bytes(os.urandom(10)) + # 42-bit counter with MSB set to 0 + counter = (rand >> 32) & 0x1ff_ffff_ffff + # 32-bit random data + tail = rand & 0xffff_ffff + return counter, tail + + +def uuid7(): + """Generate a UUID from a Unix timestamp in milliseconds and random bits. + + UUIDv7 objects feature monotonicity within a millisecond. + """ + # --- 48 --- -- 4 -- --- 12 --- -- 2 -- --- 30 --- - 32 - + # unix_ts_ms | version | counter_hi | variant | counter_lo | random + # + # 'counter = counter_hi | counter_lo' is a 42-bit counter constructed + # with Method 1 of RFC 9562, §6.2, and its MSB is set to 0. + # + # 'random' is a 32-bit random value regenerated for every new UUID. + # + # If multiple UUIDs are generated within the same millisecond, the LSB + # of 'counter' is incremented by 1. When overflowing, the timestamp is + # advanced and the counter is reset to a random 42-bit integer with MSB + # set to 0. + + global _last_timestamp_v7 + global _last_counter_v7 + + nanoseconds = time.time_ns() + timestamp_ms = nanoseconds // 1_000_000 + + if _last_timestamp_v7 is None or timestamp_ms > _last_timestamp_v7: + counter, tail = _uuid7_get_counter_and_tail() + else: + if timestamp_ms < _last_timestamp_v7: + timestamp_ms = _last_timestamp_v7 + 1 + # advance the 42-bit counter + counter = _last_counter_v7 + 1 + if counter > 0x3ff_ffff_ffff: + # advance the 48-bit timestamp + timestamp_ms += 1 + counter, tail = _uuid7_get_counter_and_tail() + else: + # 32-bit random data + tail = int.from_bytes(os.urandom(4)) + + unix_ts_ms = timestamp_ms & 0xffff_ffff_ffff + counter_msbs = counter >> 30 + # keep 12 counter's MSBs and clear variant bits + counter_hi = counter_msbs & 0x0fff + # keep 30 counter's LSBs and clear version bits + counter_lo = counter & 0x3fff_ffff + # ensure that the tail is always a 32-bit integer (by construction, + # it is already the case, but future interfaces may allow the user + # to specify the random tail) + tail &= 0xffff_ffff + + int_uuid_7 = unix_ts_ms << 80 + int_uuid_7 |= counter_hi << 64 + int_uuid_7 |= counter_lo << 32 + int_uuid_7 |= tail + # by construction, the variant and version bits are already cleared + int_uuid_7 |= _RFC_4122_VERSION_7_FLAGS + res = UUID._from_int(int_uuid_7) + + # defer global update until all computations are done + _last_timestamp_v7 = timestamp_ms + _last_counter_v7 = counter + return res + + +def uuid8(a=None, b=None, c=None): + """Generate a UUID from three custom blocks. + + * 'a' is the first 48-bit chunk of the UUID (octets 0-5); + * 'b' is the mid 12-bit chunk (octets 6-7); + * 'c' is the last 62-bit chunk (octets 8-15). + + When a value is not specified, a pseudo-random value is generated. + """ + if a is None: + import random + a = random.getrandbits(48) + if b is None: + import random + b = random.getrandbits(12) + if c is None: + import random + c = random.getrandbits(62) + int_uuid_8 = (a & 0xffff_ffff_ffff) << 80 + int_uuid_8 |= (b & 0xfff) << 64 + int_uuid_8 |= c & 0x3fff_ffff_ffff_ffff + # by construction, the variant and version bits are already cleared + int_uuid_8 |= _RFC_4122_VERSION_8_FLAGS + return UUID._from_int(int_uuid_8) + + +def main(): + """Run the uuid command line interface.""" + uuid_funcs = { + "uuid1": uuid1, + "uuid3": uuid3, + "uuid4": uuid4, + "uuid5": uuid5, + "uuid6": uuid6, + "uuid7": uuid7, + "uuid8": uuid8, + } + uuid_namespace_funcs = ("uuid3", "uuid5") + namespaces = { + "@dns": NAMESPACE_DNS, + "@url": NAMESPACE_URL, + "@oid": NAMESPACE_OID, + "@x500": NAMESPACE_X500 + } + + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Generate a UUID using the selected UUID function.", + color=True, + ) + parser.add_argument("-u", "--uuid", + choices=uuid_funcs.keys(), + default="uuid4", + help="function to generate the UUID") + parser.add_argument("-n", "--namespace", + choices=["any UUID", *namespaces.keys()], + help="uuid3/uuid5 only: " + "a UUID, or a well-known predefined UUID addressed " + "by namespace name") + parser.add_argument("-N", "--name", + help="uuid3/uuid5 only: " + "name used as part of generating the UUID") + parser.add_argument("-C", "--count", metavar="NUM", type=int, default=1, + help="generate NUM fresh UUIDs") + + args = parser.parse_args() + uuid_func = uuid_funcs[args.uuid] + namespace = args.namespace + name = args.name + + if args.uuid in uuid_namespace_funcs: + if not namespace or not name: + parser.error( + "Incorrect number of arguments. " + f"{args.uuid} requires a namespace and a name. " + "Run 'python -m uuid -h' for more information." + ) + namespace = namespaces[namespace] if namespace in namespaces else UUID(namespace) + for _ in range(args.count): + print(uuid_func(namespace, name)) + else: + for _ in range(args.count): + print(uuid_func()) + # The following standard UUIDs are for use with uuid3() or uuid5(). @@ -728,3 +997,11 @@ def uuid5(namespace, name): NAMESPACE_URL = UUID('6ba7b811-9dad-11d1-80b4-00c04fd430c8') NAMESPACE_OID = UUID('6ba7b812-9dad-11d1-80b4-00c04fd430c8') NAMESPACE_X500 = UUID('6ba7b814-9dad-11d1-80b4-00c04fd430c8') + +# RFC 9562 Sections 5.9 and 5.10 define the special Nil and Max UUID formats. + +NIL = UUID('00000000-0000-0000-0000-000000000000') +MAX = UUID('ffffffff-ffff-ffff-ffff-ffffffffffff') + +if __name__ == "__main__": + main() diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 6f1af294ae6..f7a6d261401 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -11,9 +11,10 @@ import sys import sysconfig import types +import shlex -CORE_VENV_DEPS = ('pip', 'setuptools') +CORE_VENV_DEPS = ('pip',) logger = logging.getLogger(__name__) @@ -41,20 +42,24 @@ class EnvBuilder: environment :param prompt: Alternative terminal prefix for the environment. :param upgrade_deps: Update the base venv modules to the latest on PyPI + :param scm_ignore_files: Create ignore files for the SCMs specified by the + iterable. """ def __init__(self, system_site_packages=False, clear=False, symlinks=False, upgrade=False, with_pip=False, prompt=None, - upgrade_deps=False): + upgrade_deps=False, *, scm_ignore_files=frozenset()): self.system_site_packages = system_site_packages self.clear = clear self.symlinks = symlinks self.upgrade = upgrade self.with_pip = with_pip + self.orig_prompt = prompt if prompt == '.': # see bpo-38901 prompt = os.path.basename(os.getcwd()) self.prompt = prompt self.upgrade_deps = upgrade_deps + self.scm_ignore_files = frozenset(map(str.lower, scm_ignore_files)) def create(self, env_dir): """ @@ -65,6 +70,8 @@ def create(self, env_dir): """ env_dir = os.path.abspath(env_dir) context = self.ensure_directories(env_dir) + for scm in self.scm_ignore_files: + getattr(self, f"create_{scm}_ignore_file")(context) # See issue 24875. We need system_site_packages to be False # until after pip is installed. true_system_site_packages = self.system_site_packages @@ -92,6 +99,42 @@ def clear_directory(self, path): elif os.path.isdir(fn): shutil.rmtree(fn) + def _venv_path(self, env_dir, name): + vars = { + 'base': env_dir, + 'platbase': env_dir, + 'installed_base': env_dir, + 'installed_platbase': env_dir, + } + return sysconfig.get_path(name, scheme='venv', vars=vars) + + @classmethod + def _same_path(cls, path1, path2): + """Check whether two paths appear the same. + + Whether they refer to the same file is irrelevant; we're testing for + whether a human reader would look at the path string and easily tell + that they're the same file. + """ + if sys.platform == 'win32': + if os.path.normcase(path1) == os.path.normcase(path2): + return True + # gh-90329: Don't display a warning for short/long names + import _winapi + try: + path1 = _winapi.GetLongPathName(os.fsdecode(path1)) + except OSError: + pass + try: + path2 = _winapi.GetLongPathName(os.fsdecode(path2)) + except OSError: + pass + if os.path.normcase(path1) == os.path.normcase(path2): + return True + return False + else: + return path1 == path2 + def ensure_directories(self, env_dir): """ Create the directories for the environment. @@ -106,31 +149,38 @@ def create_if_needed(d): elif os.path.islink(d) or os.path.isfile(d): raise ValueError('Unable to create directory %r' % d) + if os.pathsep in os.fspath(env_dir): + raise ValueError(f'Refusing to create a venv in {env_dir} because ' + f'it contains the PATH separator {os.pathsep}.') if os.path.exists(env_dir) and self.clear: self.clear_directory(env_dir) context = types.SimpleNamespace() context.env_dir = env_dir context.env_name = os.path.split(env_dir)[1] - prompt = self.prompt if self.prompt is not None else context.env_name - context.prompt = '(%s) ' % prompt + context.prompt = self.prompt if self.prompt is not None else context.env_name create_if_needed(env_dir) executable = sys._base_executable + if not executable: # see gh-96861 + raise ValueError('Unable to determine path to the running ' + 'Python interpreter. Provide an explicit path or ' + 'check that your PATH environment variable is ' + 'correctly set.') dirname, exename = os.path.split(os.path.abspath(executable)) + if sys.platform == 'win32': + # Always create the simplest name in the venv. It will either be a + # link back to executable, or a copy of the appropriate launcher + _d = '_d' if os.path.splitext(exename)[0].endswith('_d') else '' + exename = f'python{_d}.exe' context.executable = executable context.python_dir = dirname context.python_exe = exename - if sys.platform == 'win32': - binname = 'Scripts' - incpath = 'Include' - libpath = os.path.join(env_dir, 'Lib', 'site-packages') - else: - binname = 'bin' - incpath = 'include' - libpath = os.path.join(env_dir, 'lib', - 'python%d.%d' % sys.version_info[:2], - 'site-packages') - context.inc_path = path = os.path.join(env_dir, incpath) - create_if_needed(path) + binpath = self._venv_path(env_dir, 'scripts') + incpath = self._venv_path(env_dir, 'include') + libpath = self._venv_path(env_dir, 'purelib') + + context.inc_path = incpath + create_if_needed(incpath) + context.lib_path = libpath create_if_needed(libpath) # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX if ((sys.maxsize > 2**32) and (os.name == 'posix') and @@ -138,8 +188,8 @@ def create_if_needed(d): link_path = os.path.join(env_dir, 'lib64') if not os.path.exists(link_path): # Issue #21643 os.symlink('lib', link_path) - context.bin_path = binpath = os.path.join(env_dir, binname) - context.bin_name = binname + context.bin_path = binpath + context.bin_name = os.path.relpath(binpath, env_dir) context.env_exe = os.path.join(binpath, exename) create_if_needed(binpath) # Assign and update the command to use when launching the newly created @@ -149,7 +199,7 @@ def create_if_needed(d): # bpo-45337: Fix up env_exec_cmd to account for file system redirections. # Some redirects only apply to CreateFile and not CreateProcess real_env_exe = os.path.realpath(context.env_exe) - if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): + if not self._same_path(real_env_exe, context.env_exe): logger.warning('Actual environment location may have moved due to ' 'redirects, links or junctions.\n' ' Requested location: "%s"\n' @@ -178,86 +228,84 @@ def create_configuration(self, context): f.write('version = %d.%d.%d\n' % sys.version_info[:3]) if self.prompt is not None: f.write(f'prompt = {self.prompt!r}\n') - - if os.name != 'nt': - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - force_copy = not self.symlinks - if not force_copy: - try: - if not os.path.islink(dst): # can't link to itself! - if relative_symlinks_ok: - assert os.path.dirname(src) == os.path.dirname(dst) - os.symlink(os.path.basename(src), dst) - else: - os.symlink(src, dst) - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - force_copy = True - if force_copy: - shutil.copyfile(src, dst) - else: - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - bad_src = os.path.lexists(src) and not os.path.exists(src) - if self.symlinks and not bad_src and not os.path.islink(dst): - try: + f.write('executable = %s\n' % os.path.realpath(sys.executable)) + args = [] + nt = os.name == 'nt' + if nt and self.symlinks: + args.append('--symlinks') + if not nt and not self.symlinks: + args.append('--copies') + if not self.with_pip: + args.append('--without-pip') + if self.system_site_packages: + args.append('--system-site-packages') + if self.clear: + args.append('--clear') + if self.upgrade: + args.append('--upgrade') + if self.upgrade_deps: + args.append('--upgrade-deps') + if self.orig_prompt is not None: + args.append(f'--prompt="{self.orig_prompt}"') + if not self.scm_ignore_files: + args.append('--without-scm-ignore-files') + + args.append(context.env_dir) + args = ' '.join(args) + f.write(f'command = {sys.executable} -m venv {args}\n') + + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + (Unused on Windows, because we can't just copy a failed symlink file: we + switch to a different set of files instead.) + """ + assert os.name != 'nt' + force_copy = not self.symlinks + if not force_copy: + try: + if not os.path.islink(dst): # can't link to itself! if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) else: os.symlink(src, dst) - return - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - - # On Windows, we rewrite symlinks to our base python.exe into - # copies of venvlauncher.exe - basename, ext = os.path.splitext(os.path.basename(src)) - srcfn = os.path.join(os.path.dirname(__file__), - "scripts", - "nt", - basename + ext) - # Builds or venv's from builds need to remap source file - # locations, as we do not put them into Lib/venv/scripts - if sysconfig.is_python_build(True) or not os.path.isfile(srcfn): - if basename.endswith('_d'): - ext = '_d' + ext - basename = basename[:-2] - if basename == 'python': - basename = 'venvlauncher' - elif basename == 'pythonw': - basename = 'venvwlauncher' - src = os.path.join(os.path.dirname(src), basename + ext) - else: - src = srcfn - if not os.path.exists(src): - if not bad_src: - logger.warning('Unable to copy %r', src) - return - + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + force_copy = True + if force_copy: shutil.copyfile(src, dst) - def setup_python(self, context): + def create_git_ignore_file(self, context): """ - Set up a Python executable in the environment. + Create a .gitignore file in the environment directory. - :param context: The information for the environment creation request - being processed. + The contents of the file cause the entire environment directory to be + ignored by git. """ - binpath = context.bin_path - path = context.env_exe - copier = self.symlink_or_copy - dirname = context.python_dir - if os.name != 'nt': + gitignore_path = os.path.join(context.env_dir, '.gitignore') + with open(gitignore_path, 'w', encoding='utf-8') as file: + file.write('# Created by venv; ' + 'see https://docs.python.org/3/library/venv.html\n') + file.write('*\n') + + if os.name != 'nt': + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + path = context.env_exe + copier = self.symlink_or_copy + dirname = context.python_dir copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): + for suffix in ('python', 'python3', + f'python3.{sys.version_info[1]}'): path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -265,32 +313,107 @@ def setup_python(self, context): copier(context.env_exe, path, relative_symlinks_ok=True) if not os.path.islink(path): os.chmod(path, 0o755) - else: - if self.symlinks: - # For symlinking, we need a complete copy of the root directory - # If symlinks fail, you'll get unnecessary copies of files, but - # we assume that if you've opted into symlinks on Windows then - # you know what you're doing. - suffixes = [ - f for f in os.listdir(dirname) if - os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') - ] - if sysconfig.is_python_build(True): - suffixes = [ - f for f in suffixes if - os.path.normcase(f).startswith(('python', 'vcruntime')) - ] + + else: + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + dirname = context.python_dir + exename = os.path.basename(context.env_exe) + exe_stem = os.path.splitext(exename)[0] + exe_d = '_d' if os.path.normcase(exe_stem).endswith('_d') else '' + if sysconfig.is_python_build(): + scripts = dirname else: - suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'} - base_exe = os.path.basename(context.env_exe) - suffixes.add(base_exe) + scripts = os.path.join(os.path.dirname(__file__), + 'scripts', 'nt') + if not sysconfig.get_config_var("Py_GIL_DISABLED"): + python_exe = os.path.join(dirname, f'python{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlauncher{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlauncher{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + else: + exe_t = f'3.{sys.version_info[1]}t' + python_exe = os.path.join(dirname, f'python{exe_t}{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_t}{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlaunchert{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlaunchert{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + + do_copies = True + if self.symlinks: + do_copies = False + # For symlinking, we need all the DLLs to be available alongside + # the executables. + link_sources.update({ + f: os.path.join(dirname, f) for f in os.listdir(dirname) + if os.path.normcase(f).startswith(('python', 'vcruntime')) + and os.path.normcase(os.path.splitext(f)[1]) == '.dll' + }) + + to_unlink = [] + for dest, src in link_sources.items(): + dest = os.path.join(binpath, dest) + try: + os.symlink(src, dest) + to_unlink.append(dest) + except OSError: + logger.warning('Unable to symlink %r to %r', src, dest) + do_copies = True + for f in to_unlink: + try: + os.unlink(f) + except OSError: + logger.warning('Failed to clean up symlink %r', + f) + logger.warning('Retrying with copies') + break - for suffix in suffixes: - src = os.path.join(dirname, suffix) - if os.path.lexists(src): - copier(src, os.path.join(binpath, suffix)) + if do_copies: + for dest, src in copy_sources.items(): + dest = os.path.join(binpath, dest) + try: + shutil.copy2(src, dest) + except OSError: + logger.warning('Unable to copy %r to %r', src, dest) - if sysconfig.is_python_build(True): + if sysconfig.is_python_build(): # copy init.tcl for root, dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: @@ -303,14 +426,25 @@ def setup_python(self, context): shutil.copyfile(src, dst) break + def _call_new_python(self, context, *py_args, **kwargs): + """Executes the newly created Python using safe-ish options""" + # gh-98251: We do not want to just use '-I' because that masks + # legitimate user preferences (such as not writing bytecode). All we + # really need is to ensure that the path variables do not overrule + # normal venv handling. + args = [context.env_exec_cmd, *py_args] + kwargs['env'] = env = os.environ.copy() + env['VIRTUAL_ENV'] = context.env_dir + env.pop('PYTHONHOME', None) + env.pop('PYTHONPATH', None) + kwargs['cwd'] = context.env_dir + kwargs['executable'] = context.env_exec_cmd + subprocess.check_output(args, **kwargs) + def _setup_pip(self, context): """Installs or upgrades pip in a virtual environment""" - # We run ensurepip in isolated mode to avoid side effects from - # environment vars, the current directory and anything else - # intended for the global Python environment - cmd = [context.env_exec_cmd, '-Im', 'ensurepip', '--upgrade', - '--default-pip'] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) + self._call_new_python(context, '-m', 'ensurepip', '--upgrade', + '--default-pip', stderr=subprocess.STDOUT) def setup_scripts(self, context): """ @@ -348,11 +482,41 @@ def replace_variables(self, text, context): :param context: The information for the environment creation request being processed. """ - text = text.replace('__VENV_DIR__', context.env_dir) - text = text.replace('__VENV_NAME__', context.env_name) - text = text.replace('__VENV_PROMPT__', context.prompt) - text = text.replace('__VENV_BIN_NAME__', context.bin_name) - text = text.replace('__VENV_PYTHON__', context.env_exe) + replacements = { + '__VENV_DIR__': context.env_dir, + '__VENV_NAME__': context.env_name, + '__VENV_PROMPT__': context.prompt, + '__VENV_BIN_NAME__': context.bin_name, + '__VENV_PYTHON__': context.env_exe, + } + + def quote_ps1(s): + """ + This should satisfy PowerShell quoting rules [1], unless the quoted + string is passed directly to Windows native commands [2]. + [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules + [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters + """ + s = s.replace("'", "''") + return f"'{s}'" + + def quote_bat(s): + return s + + # gh-124651: need to quote the template strings properly + quote = shlex.quote + script_path = context.script_path + if script_path.endswith('.ps1'): + quote = quote_ps1 + elif script_path.endswith('.bat'): + quote = quote_bat + else: + # fallbacks to POSIX shell compliant quote + quote = shlex.quote + + replacements = {key: quote(s) for key, s in replacements.items()} + for key, quoted in replacements.items(): + text = text.replace(key, quoted) return text def install_scripts(self, context, path): @@ -370,15 +534,22 @@ def install_scripts(self, context, path): """ binpath = context.bin_path plen = len(path) + if os.name == 'nt': + def skip_file(f): + f = os.path.normcase(f) + return (f.startswith(('python', 'venv')) + and f.endswith(('.exe', '.pdb'))) + else: + def skip_file(f): + return False for root, dirs, files in os.walk(path): - if root == path: # at top-level, remove irrelevant dirs + if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: if d not in ('common', os.name): dirs.remove(d) - continue # ignore files in top level + continue # ignore files in top level for f in files: - if (os.name == 'nt' and f.startswith('python') - and f.endswith(('.exe', '.pdb'))): + if skip_file(f): continue srcfile = os.path.join(root, f) suffix = root[plen:].split(os.sep)[2:] @@ -389,116 +560,122 @@ def install_scripts(self, context, path): if not os.path.exists(dstdir): os.makedirs(dstdir) dstfile = os.path.join(dstdir, f) + if os.name == 'nt' and srcfile.endswith(('.exe', '.pdb')): + shutil.copy2(srcfile, dstfile) + continue with open(srcfile, 'rb') as f: data = f.read() - if not srcfile.endswith(('.exe', '.pdb')): - try: - data = data.decode('utf-8') - data = self.replace_variables(data, context) - data = data.encode('utf-8') - except UnicodeError as e: - data = None - logger.warning('unable to copy script %r, ' - 'may be binary: %s', srcfile, e) - if data is not None: + try: + context.script_path = srcfile + new_data = ( + self.replace_variables(data.decode('utf-8'), context) + .encode('utf-8') + ) + except UnicodeError as e: + logger.warning('unable to copy script %r, ' + 'may be binary: %s', srcfile, e) + continue + if new_data == data: + shutil.copy2(srcfile, dstfile) + else: with open(dstfile, 'wb') as f: - f.write(data) + f.write(new_data) shutil.copymode(srcfile, dstfile) def upgrade_dependencies(self, context): logger.debug( f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' ) - cmd = [context.env_exec_cmd, '-m', 'pip', 'install', '--upgrade'] - cmd.extend(CORE_VENV_DEPS) - subprocess.check_call(cmd) + self._call_new_python(context, '-m', 'pip', 'install', '--upgrade', + *CORE_VENV_DEPS) def create(env_dir, system_site_packages=False, clear=False, - symlinks=False, with_pip=False, prompt=None, upgrade_deps=False): + symlinks=False, with_pip=False, prompt=None, upgrade_deps=False, + *, scm_ignore_files=frozenset()): """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip, - prompt=prompt, upgrade_deps=upgrade_deps) + prompt=prompt, upgrade_deps=upgrade_deps, + scm_ignore_files=scm_ignore_files) builder.create(env_dir) + def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with Python >= 3.3') + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.', + epilog='Once an environment has been ' + 'created, you may wish to ' + 'activate it, e.g. by ' + 'sourcing an activate script ' + 'in its bin directory.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory to create the environment in.') + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.', - epilog='Once an environment has been ' - 'created, you may wish to ' - 'activate it, e.g. by ' - 'sourcing an activate script ' - 'in its bin directory.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory to create the environment in.') - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - group = parser.add_mutually_exclusive_group() - group.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - group.add_argument('--copies', default=not use_symlinks, - action='store_false', dest='symlinks', - help='Try to use copies rather than symlinks, ' - 'even when symlinks are the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'environment directory if it ' - 'already exists, before ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the environment ' - 'directory to use this version ' - 'of Python, assuming Python ' - 'has been upgraded in-place.') - parser.add_argument('--without-pip', dest='with_pip', - default=True, action='store_false', - help='Skips installing or upgrading pip in the ' - 'virtual environment (pip is bootstrapped ' - 'by default)') - parser.add_argument('--prompt', - help='Provides an alternative prompt prefix for ' - 'this environment.') - parser.add_argument('--upgrade-deps', default=False, action='store_true', - dest='upgrade_deps', - help='Upgrade core dependencies: {} to the latest ' - 'version in PyPI'.format( - ' '.join(CORE_VENV_DEPS))) - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = EnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - with_pip=options.with_pip, - prompt=options.prompt, - upgrade_deps=options.upgrade_deps) - for d in options.dirs: - builder.create(d) + use_symlinks = True + group = parser.add_mutually_exclusive_group() + group.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + group.add_argument('--copies', default=not use_symlinks, + action='store_false', dest='symlinks', + help='Try to use copies rather than symlinks, ' + 'even when symlinks are the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'environment directory if it ' + 'already exists, before ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the environment ' + 'directory to use this version ' + 'of Python, assuming Python ' + 'has been upgraded in-place.') + parser.add_argument('--without-pip', dest='with_pip', + default=True, action='store_false', + help='Skips installing or upgrading pip in the ' + 'virtual environment (pip is bootstrapped ' + 'by default)') + parser.add_argument('--prompt', + help='Provides an alternative prompt prefix for ' + 'this environment.') + parser.add_argument('--upgrade-deps', default=False, action='store_true', + dest='upgrade_deps', + help=f'Upgrade core dependencies ({", ".join(CORE_VENV_DEPS)}) ' + 'to the latest version in PyPI') + parser.add_argument('--without-scm-ignore-files', dest='scm_ignore_files', + action='store_const', const=frozenset(), + default=frozenset(['git']), + help='Skips adding SCM ignore files to the environment ' + 'directory (Git is supported by default).') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = EnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + with_pip=options.with_pip, + prompt=options.prompt, + upgrade_deps=options.upgrade_deps, + scm_ignore_files=options.scm_ignore_files) + for d in options.dirs: + builder.create(d) + if __name__ == '__main__': rc = 1 diff --git a/Lib/venv/__main__.py b/Lib/venv/__main__.py index 912423e4a78..88f55439dc2 100644 --- a/Lib/venv/__main__.py +++ b/Lib/venv/__main__.py @@ -6,5 +6,5 @@ main() rc = 0 except Exception as e: - print('Error: %s' % e, file=sys.stderr) + print('Error:', e, file=sys.stderr) sys.exit(rc) diff --git a/Lib/venv/scripts/common/Activate.ps1 b/Lib/venv/scripts/common/Activate.ps1 index b49d77ba44b..16ba5290fae 100644 --- a/Lib/venv/scripts/common/Activate.ps1 +++ b/Lib/venv/scripts/common/Activate.ps1 @@ -219,6 +219,8 @@ deactivate -nondestructive # that there is an activated venv. $env:VIRTUAL_ENV = $VenvDir +$env:VIRTUAL_ENV_PROMPT = $Prompt + if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Verbose "Setting prompt to '$Prompt'" @@ -233,7 +235,6 @@ if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " _OLD_VIRTUAL_PROMPT } - $env:VIRTUAL_ENV_PROMPT = $Prompt } # Clear PYTHONHOME diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 6fbc2b8801d..70673a265d4 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -1,5 +1,5 @@ # This file must be used with "source bin/activate" *from bash* -# you cannot run it directly +# You cannot run it directly deactivate () { # reset old environment variables @@ -14,12 +14,10 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi + # Call hash to forget past locations. Without forgetting + # past locations the $PATH changes we made may not be respected. + # See "man bash" for more details. hash is usually a builtin of your shell + hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -38,13 +36,27 @@ deactivate () { # unset irrelevant variables deactivate nondestructive -VIRTUAL_ENV="__VENV_DIR__" -export VIRTUAL_ENV +# on Windows, a path can contain colons and backslashes and has to be converted: +case "$(uname)" in + CYGWIN*|MSYS*|MINGW*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS and MINGW + # and to /cygdrive/d/path/to/venv on Cygwin + VIRTUAL_ENV=$(cygpath __VENV_DIR__) + export VIRTUAL_ENV + ;; + *) + # use the path as-is + export VIRTUAL_ENV=__VENV_DIR__ + ;; +esac _OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +PATH="$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" export PATH +VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +export VIRTUAL_ENV_PROMPT + # unset PYTHONHOME if set # this will fail if PYTHONHOME is set to the empty string (which is bad anyway) # could use `if (set -u; : $PYTHONHOME) ;` in bash @@ -55,15 +67,10 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="__VENV_PROMPT__${PS1:-}" + PS1="("__VENV_PROMPT__") ${PS1:-}" export PS1 - VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" - export VIRTUAL_ENV_PROMPT fi -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting +# Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi +hash -r 2> /dev/null diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/common/activate.fish similarity index 76% rename from Lib/venv/scripts/posix/activate.fish rename to Lib/venv/scripts/common/activate.fish index e40a1d71489..284a7469c99 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/common/activate.fish @@ -1,5 +1,5 @@ # This file must be used with "source <venv>/bin/activate.fish" *from fish* -# (https://fishshell.com/); you cannot run it directly. +# (https://fishshell.com/). You cannot run it directly. function deactivate -d "Exit virtual environment and return to normal shell environment" # reset old environment variables @@ -13,10 +13,13 @@ function deactivate -d "Exit virtual environment and return to normal shell env end if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt + # prevents error when using nested fish instances (Issue #93858) + if functions -q _old_fish_prompt + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end end set -e VIRTUAL_ENV @@ -30,10 +33,11 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV "__VENV_DIR__" +set -gx VIRTUAL_ENV __VENV_DIR__ set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH +set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH +set -gx VIRTUAL_ENV_PROMPT __VENV_PROMPT__ # Unset PYTHONHOME if set. if set -q PYTHONHOME @@ -53,7 +57,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s(%s)%s " (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . @@ -62,5 +66,4 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" end set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" end diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 5daa45afc9f..9ac5c20b477 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -8,15 +8,15 @@ if defined _OLD_CODEPAGE ( "%SystemRoot%\System32\chcp.com" 65001 > nul ) -set VIRTUAL_ENV=__VENV_DIR__ +set "VIRTUAL_ENV=__VENV_DIR__" if not defined PROMPT set PROMPT=$P$G if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -set _OLD_VIRTUAL_PROMPT=%PROMPT% -set PROMPT=__VENV_PROMPT__%PROMPT% +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=(__VENV_PROMPT__) %PROMPT%" if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% set PYTHONHOME= @@ -24,8 +24,8 @@ set PYTHONHOME= if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% -set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% -set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" +set "VIRTUAL_ENV_PROMPT=__VENV_PROMPT__" :END if defined _OLD_CODEPAGE ( diff --git a/Lib/venv/scripts/nt/venvlauncher.exe b/Lib/venv/scripts/nt/venvlauncher.exe new file mode 100644 index 00000000000..c6863b56e57 Binary files /dev/null and b/Lib/venv/scripts/nt/venvlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvlaunchert.exe b/Lib/venv/scripts/nt/venvlaunchert.exe new file mode 100644 index 00000000000..c12a7a869f4 Binary files /dev/null and b/Lib/venv/scripts/nt/venvlaunchert.exe differ diff --git a/Lib/venv/scripts/nt/venvwlauncher.exe b/Lib/venv/scripts/nt/venvwlauncher.exe new file mode 100644 index 00000000000..d0d3733266f Binary files /dev/null and b/Lib/venv/scripts/nt/venvwlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvwlaunchert.exe b/Lib/venv/scripts/nt/venvwlaunchert.exe new file mode 100644 index 00000000000..9456a9e9b4a Binary files /dev/null and b/Lib/venv/scripts/nt/venvwlaunchert.exe differ diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index d6f697c55ed..2a3fa835476 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -1,5 +1,6 @@ # This file must be used with "source bin/activate.csh" *from csh*. # You cannot run it directly. + # Created by Davide Di Blasi <davidedb@gmail.com>. # Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com> @@ -8,17 +9,17 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV "__VENV_DIR__" +setenv VIRTUAL_ENV __VENV_DIR__ set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" +setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "__VENV_PROMPT__$prompt" - setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" + set prompt = "("__VENV_PROMPT__") $prompt:q" endif alias pydoc python -m pydoc diff --git a/Lib/warnings.py b/Lib/warnings.py index 7d8c4400127..6759857d909 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -1,580 +1,99 @@ -"""Python part of the warnings subsystem.""" - import sys +__all__ = [ + "warn", + "warn_explicit", + "showwarning", + "formatwarning", + "filterwarnings", + "simplefilter", + "resetwarnings", + "catch_warnings", + "deprecated", +] + +from _py_warnings import ( + WarningMessage, + _DEPRECATED_MSG, + _OptionError, + _add_filter, + _deprecated, + _filters_mutated, + _filters_mutated_lock_held, + _filters_version, + _formatwarning_orig, + _formatwarnmsg, + _formatwarnmsg_impl, + _get_context, + _get_filters, + _getaction, + _getcategory, + _is_filename_to_skip, + _is_internal_filename, + _is_internal_frame, + _lock, + _new_context, + _next_external_frame, + _processoptions, + _set_context, + _set_module, + _setoption, + _setup_defaults, + _showwarning_orig, + _showwarnmsg, + _showwarnmsg_impl, + _use_context, + _warn_unawaited_coroutine, + _warnings_context, + catch_warnings, + defaultaction, + deprecated, + filters, + filterwarnings, + formatwarning, + onceregistry, + resetwarnings, + showwarning, + simplefilter, + warn, + warn_explicit, +) -__all__ = ["warn", "warn_explicit", "showwarning", - "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings"] - -def showwarning(message, category, filename, lineno, file=None, line=None): - """Hook to write a warning to a file; replace if you like.""" - msg = WarningMessage(message, category, filename, lineno, file, line) - _showwarnmsg_impl(msg) - -def formatwarning(message, category, filename, lineno, line=None): - """Function to format a warning the standard way.""" - msg = WarningMessage(message, category, filename, lineno, None, line) - return _formatwarnmsg_impl(msg) - -def _showwarnmsg_impl(msg): - file = msg.file - if file is None: - file = sys.stderr - if file is None: - # sys.stderr is None when run with pythonw.exe: - # warnings get lost - return - text = _formatwarnmsg(msg) - try: - file.write(text) - except OSError: - # the file (probably stderr) is invalid - this warning gets lost. - pass - -def _formatwarnmsg_impl(msg): - category = msg.category.__name__ - s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n" - - if msg.line is None: - try: - import linecache - line = linecache.getline(msg.filename, msg.lineno) - except Exception: - # When a warning is logged during Python shutdown, linecache - # and the import machinery don't work anymore - line = None - linecache = None - else: - line = msg.line - if line: - line = line.strip() - s += " %s\n" % line - - if msg.source is not None: - try: - import tracemalloc - # Logging a warning should not raise a new exception: - # catch Exception, not only ImportError and RecursionError. - except Exception: - # don't suggest to enable tracemalloc if it's not available - tracing = True - tb = None - else: - tracing = tracemalloc.is_tracing() - try: - tb = tracemalloc.get_object_traceback(msg.source) - except Exception: - # When a warning is logged during Python shutdown, tracemalloc - # and the import machinery don't work anymore - tb = None - - if tb is not None: - s += 'Object allocated at (most recent call last):\n' - for frame in tb: - s += (' File "%s", lineno %s\n' - % (frame.filename, frame.lineno)) - - try: - if linecache is not None: - line = linecache.getline(frame.filename, frame.lineno) - else: - line = None - except Exception: - line = None - if line: - line = line.strip() - s += ' %s\n' % line - elif not tracing: - s += (f'{category}: Enable tracemalloc to get the object ' - f'allocation traceback\n') - return s - -# Keep a reference to check if the function was replaced -_showwarning_orig = showwarning - -def _showwarnmsg(msg): - """Hook to write a warning to a file; replace if you like.""" - try: - sw = showwarning - except NameError: - pass - else: - if sw is not _showwarning_orig: - # warnings.showwarning() was replaced - if not callable(sw): - raise TypeError("warnings.showwarning() must be set to a " - "function or method") - - sw(msg.message, msg.category, msg.filename, msg.lineno, - msg.file, msg.line) - return - _showwarnmsg_impl(msg) - -# Keep a reference to check if the function was replaced -_formatwarning_orig = formatwarning - -def _formatwarnmsg(msg): - """Function to format a warning the standard way.""" - try: - fw = formatwarning - except NameError: - pass - else: - if fw is not _formatwarning_orig: - # warnings.formatwarning() was replaced - return fw(msg.message, msg.category, - msg.filename, msg.lineno, msg.line) - return _formatwarnmsg_impl(msg) - -def filterwarnings(action, message="", category=Warning, module="", lineno=0, - append=False): - """Insert an entry into the list of warnings filters (at the front). - - 'action' -- one of "error", "ignore", "always", "default", "module", - or "once" - 'message' -- a regex that the warning message must match - 'category' -- a class that the warning must be a subclass of - 'module' -- a regex that the module name must match - 'lineno' -- an integer line number, 0 matches all warnings - 'append' -- if true, append to the list of filters - """ - assert action in ("error", "ignore", "always", "default", "module", - "once"), "invalid action: %r" % (action,) - assert isinstance(message, str), "message must be a string" - assert isinstance(category, type), "category must be a class" - assert issubclass(category, Warning), "category must be a Warning subclass" - assert isinstance(module, str), "module must be a string" - assert isinstance(lineno, int) and lineno >= 0, \ - "lineno must be an int >= 0" - - if message or module: - import re - - if message: - message = re.compile(message, re.I) - else: - message = None - if module: - module = re.compile(module) - else: - module = None - - _add_filter(action, message, category, module, lineno, append=append) - -def simplefilter(action, category=Warning, lineno=0, append=False): - """Insert a simple entry into the list of warnings filters (at the front). - - A simple filter matches all modules and messages. - 'action' -- one of "error", "ignore", "always", "default", "module", - or "once" - 'category' -- a class that the warning must be a subclass of - 'lineno' -- an integer line number, 0 matches all warnings - 'append' -- if true, append to the list of filters - """ - assert action in ("error", "ignore", "always", "default", "module", - "once"), "invalid action: %r" % (action,) - assert isinstance(lineno, int) and lineno >= 0, \ - "lineno must be an int >= 0" - _add_filter(action, None, category, None, lineno, append=append) - -def _add_filter(*item, append): - # Remove possible duplicate filters, so new one will be placed - # in correct place. If append=True and duplicate exists, do nothing. - if not append: - try: - filters.remove(item) - except ValueError: - pass - filters.insert(0, item) - else: - if item not in filters: - filters.append(item) - _filters_mutated() - -def resetwarnings(): - """Clear the list of warning filters, so that no filters are active.""" - filters[:] = [] - _filters_mutated() - -class _OptionError(Exception): - """Exception used by option processing helpers.""" - pass - -# Helper to process -W options passed via sys.warnoptions -def _processoptions(args): - for arg in args: - try: - _setoption(arg) - except _OptionError as msg: - print("Invalid -W option ignored:", msg, file=sys.stderr) - -# Helper for _processoptions() -def _setoption(arg): - parts = arg.split(':') - if len(parts) > 5: - raise _OptionError("too many fields (max 5): %r" % (arg,)) - while len(parts) < 5: - parts.append('') - action, message, category, module, lineno = [s.strip() - for s in parts] - action = _getaction(action) - category = _getcategory(category) - if message or module: - import re - if message: - message = re.escape(message) - if module: - module = re.escape(module) + r'\Z' - if lineno: - try: - lineno = int(lineno) - if lineno < 0: - raise ValueError - except (ValueError, OverflowError): - raise _OptionError("invalid lineno %r" % (lineno,)) from None - else: - lineno = 0 - filterwarnings(action, message, category, module, lineno) - -# Helper for _setoption() -def _getaction(action): - if not action: - return "default" - if action == "all": return "always" # Alias - for a in ('default', 'always', 'ignore', 'module', 'once', 'error'): - if a.startswith(action): - return a - raise _OptionError("invalid action: %r" % (action,)) - -# Helper for _setoption() -def _getcategory(category): - if not category: - return Warning - if '.' not in category: - import builtins as m - klass = category - else: - module, _, klass = category.rpartition('.') - try: - m = __import__(module, None, None, [klass]) - except ImportError: - raise _OptionError("invalid module name: %r" % (module,)) from None - try: - cat = getattr(m, klass) - except AttributeError: - raise _OptionError("unknown warning category: %r" % (category,)) from None - if not issubclass(cat, Warning): - raise _OptionError("invalid warning category: %r" % (category,)) - return cat - - -def _is_internal_frame(frame): - """Signal whether the frame is an internal CPython implementation detail.""" - filename = frame.f_code.co_filename - return 'importlib' in filename and '_bootstrap' in filename - - -def _next_external_frame(frame): - """Find the next frame that doesn't involve CPython internals.""" - frame = frame.f_back - while frame is not None and _is_internal_frame(frame): - frame = frame.f_back - return frame - - -# Code typically replaced by _warnings -def warn(message, category=None, stacklevel=1, source=None): - """Issue a warning, or maybe ignore it or raise an exception.""" - # Check if message is already a Warning object - if isinstance(message, Warning): - category = message.__class__ - # Check category argument - if category is None: - category = UserWarning - if not (isinstance(category, type) and issubclass(category, Warning)): - raise TypeError("category must be a Warning subclass, " - "not '{:s}'".format(type(category).__name__)) - # Get context information - try: - if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)): - # If frame is too small to care or if the warning originated in - # internal code, then do not try to hide any frames. - frame = sys._getframe(stacklevel) - else: - frame = sys._getframe(1) - # Look for one frame less since the above line starts us off. - for x in range(stacklevel-1): - frame = _next_external_frame(frame) - if frame is None: - raise ValueError - except ValueError: - globals = sys.__dict__ - filename = "sys" - lineno = 1 - else: - globals = frame.f_globals - filename = frame.f_code.co_filename - lineno = frame.f_lineno - if '__name__' in globals: - module = globals['__name__'] - else: - module = "<string>" - registry = globals.setdefault("__warningregistry__", {}) - warn_explicit(message, category, filename, lineno, module, registry, - globals, source) - -def warn_explicit(message, category, filename, lineno, - module=None, registry=None, module_globals=None, - source=None): - lineno = int(lineno) - if module is None: - module = filename or "<unknown>" - if module[-3:].lower() == ".py": - module = module[:-3] # XXX What about leading pathname? - if registry is None: - registry = {} - if registry.get('version', 0) != _filters_version: - registry.clear() - registry['version'] = _filters_version - if isinstance(message, Warning): - text = str(message) - category = message.__class__ - else: - text = message - message = category(message) - key = (text, category, lineno) - # Quick test for common case - if registry.get(key): - return - # Search the filters - for item in filters: - action, msg, cat, mod, ln = item - if ((msg is None or msg.match(text)) and - issubclass(category, cat) and - (mod is None or mod.match(module)) and - (ln == 0 or lineno == ln)): - break - else: - action = defaultaction - # Early exit actions - if action == "ignore": - return - - # Prime the linecache for formatting, in case the - # "file" is actually in a zipfile or something. - import linecache - linecache.getlines(filename, module_globals) - - if action == "error": - raise message - # Other actions - if action == "once": - registry[key] = 1 - oncekey = (text, category) - if onceregistry.get(oncekey): - return - onceregistry[oncekey] = 1 - elif action == "always": - pass - elif action == "module": - registry[key] = 1 - altkey = (text, category, 0) - if registry.get(altkey): - return - registry[altkey] = 1 - elif action == "default": - registry[key] = 1 - else: - # Unrecognized actions are errors - raise RuntimeError( - "Unrecognized action (%r) in warnings.filters:\n %s" % - (action, item)) - # Print message and context - msg = WarningMessage(message, category, filename, lineno, source) - _showwarnmsg(msg) - - -class WarningMessage(object): - - _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line", "source") - - def __init__(self, message, category, filename, lineno, file=None, - line=None, source=None): - self.message = message - self.category = category - self.filename = filename - self.lineno = lineno - self.file = file - self.line = line - self.source = source - self._category_name = category.__name__ if category else None - - def __str__(self): - return ("{message : %r, category : %r, filename : %r, lineno : %s, " - "line : %r}" % (self.message, self._category_name, - self.filename, self.lineno, self.line)) - - -class catch_warnings(object): - - """A context manager that copies and restores the warnings filter upon - exiting the context. - - The 'record' argument specifies whether warnings should be captured by a - custom implementation of warnings.showwarning() and be appended to a list - returned by the context manager. Otherwise None is returned by the context - manager. The objects appended to the list are arguments whose attributes - mirror the arguments to showwarning(). - - The 'module' argument is to specify an alternative module to the module - named 'warnings' and imported under that name. This argument is only useful - when testing the warnings module itself. - - If the 'action' argument is not None, the remaining arguments are passed - to warnings.simplefilter() as if it were called immediately on entering the - context. - """ - - def __init__(self, *, record=False, module=None, - action=None, category=Warning, lineno=0, append=False): - """Specify whether to record warnings and if an alternative module - should be used other than sys.modules['warnings']. - - For compatibility with Python 3.0, please consider all arguments to be - keyword-only. - - """ - self._record = record - self._module = sys.modules['warnings'] if module is None else module - self._entered = False - if action is None: - self._filter = None - else: - self._filter = (action, category, lineno, append) - - def __repr__(self): - args = [] - if self._record: - args.append("record=True") - if self._module is not sys.modules['warnings']: - args.append("module=%r" % self._module) - name = type(self).__name__ - return "%s(%s)" % (name, ", ".join(args)) - - def __enter__(self): - if self._entered: - raise RuntimeError("Cannot enter %r twice" % self) - self._entered = True - self._filters = self._module.filters - self._module.filters = self._filters[:] - self._module._filters_mutated() - self._showwarning = self._module.showwarning - self._showwarnmsg_impl = self._module._showwarnmsg_impl - if self._filter is not None: - simplefilter(*self._filter) - if self._record: - log = [] - self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig - return log - else: - return None - - def __exit__(self, *exc_info): - if not self._entered: - raise RuntimeError("Cannot exit %r without entering first" % self) - self._module.filters = self._filters - self._module._filters_mutated() - self._module.showwarning = self._showwarning - self._module._showwarnmsg_impl = self._showwarnmsg_impl - - -_DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" - -def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): - """Warn that *name* is deprecated or should be removed. - - RuntimeError is raised if *remove* specifies a major/minor tuple older than - the current Python version or the same version but past the alpha. - - The *message* argument is formatted with *name* and *remove* as a Python - version (e.g. "3.11"). - - """ - remove_formatted = f"{remove[0]}.{remove[1]}" - if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"): - msg = f"{name!r} was slated for removal after Python {remove_formatted} alpha" - raise RuntimeError(msg) - else: - msg = message.format(name=name, remove=remove_formatted) - warn(msg, DeprecationWarning, stacklevel=3) - - -# Private utility function called by _PyErr_WarnUnawaitedCoroutine -def _warn_unawaited_coroutine(coro): - msg_lines = [ - f"coroutine '{coro.__qualname__}' was never awaited\n" - ] - if coro.cr_origin is not None: - import linecache, traceback - def extract(): - for filename, lineno, funcname in reversed(coro.cr_origin): - line = linecache.getline(filename, lineno) - yield (filename, lineno, funcname, line) - msg_lines.append("Coroutine created at (most recent call last)\n") - msg_lines += traceback.format_list(list(extract())) - msg = "".join(msg_lines).rstrip("\n") - # Passing source= here means that if the user happens to have tracemalloc - # enabled and tracking where the coroutine was created, the warning will - # contain that traceback. This does mean that if they have *both* - # coroutine origin tracking *and* tracemalloc enabled, they'll get two - # partially-redundant tracebacks. If we wanted to be clever we could - # probably detect this case and avoid it, but for now we don't bother. - warn(msg, category=RuntimeWarning, stacklevel=2, source=coro) - - -# filters contains a sequence of filter 5-tuples -# The components of the 5-tuple are: -# - an action: error, ignore, always, default, module, or once -# - a compiled regex that must match the warning message -# - a class representing the warning category -# - a compiled regex that must match the module that is being warned -# - a line number for the line being warning, or 0 to mean any line -# If either if the compiled regexs are None, match anything. try: - from _warnings import (filters, _defaultaction, _onceregistry, - warn, warn_explicit, _filters_mutated) - defaultaction = _defaultaction - onceregistry = _onceregistry + # Try to use the C extension, this will replace some parts of the + # _py_warnings implementation imported above. + from _warnings import ( + _acquire_lock, + _defaultaction as defaultaction, + _filters_mutated_lock_held, + _onceregistry as onceregistry, + _release_lock, + _warnings_context, + filters, + warn, + warn_explicit, + ) + _warnings_defaults = True -except ImportError: - filters = [] - defaultaction = "default" - onceregistry = {} - _filters_version = 1 + class _Lock: + def __enter__(self): + _acquire_lock() + return self - def _filters_mutated(): - global _filters_version - _filters_version += 1 + def __exit__(self, *args): + _release_lock() + _lock = _Lock() +except ImportError: _warnings_defaults = False # Module initialization +_set_module(sys.modules[__name__]) _processoptions(sys.warnoptions) if not _warnings_defaults: - # Several warning categories are ignored by default in regular builds - if not hasattr(sys, 'gettotalrefcount'): - filterwarnings("default", category=DeprecationWarning, - module="__main__", append=1) - simplefilter("ignore", category=DeprecationWarning, append=1) - simplefilter("ignore", category=PendingDeprecationWarning, append=1) - simplefilter("ignore", category=ImportWarning, append=1) - simplefilter("ignore", category=ResourceWarning, append=1) + _setup_defaults() del _warnings_defaults +del _setup_defaults diff --git a/Lib/wave.py b/Lib/wave.py new file mode 100644 index 00000000000..b8476e26486 --- /dev/null +++ b/Lib/wave.py @@ -0,0 +1,665 @@ +"""Stuff to parse WAVE files. + +Usage. + +Reading WAVE files: + f = wave.open(file, 'r') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods read(), seek(), and close(). +When the setpos() and rewind() methods are not used, the seek() +method is not necessary. + +This returns an instance of a class with the following public methods: + getnchannels() -- returns number of audio channels (1 for + mono, 2 for stereo) + getsampwidth() -- returns sample width in bytes + getframerate() -- returns sampling frequency + getnframes() -- returns number of audio frames + getcomptype() -- returns compression type ('NONE' for linear samples) + getcompname() -- returns human-readable version of + compression type ('not compressed' linear samples) + getparams() -- returns a namedtuple consisting of all of the + above in the above order + getmarkers() -- returns None (for compatibility with the + old aifc module) + getmark(id) -- raises an error since the mark does not + exist (for compatibility with the old aifc module) + readframes(n) -- returns at most n frames of audio + rewind() -- rewind to the beginning of the audio stream + setpos(pos) -- seek to the specified position + tell() -- return the current position + close() -- close the instance (make it unusable) +The position returned by tell() and the position given to setpos() +are compatible and have nothing to do with the actual position in the +file. +The close() method is called automatically when the class instance +is destroyed. + +Writing WAVE files: + f = wave.open(file, 'w') +where file is either the name of a file or an open file pointer. +The open file pointer must have methods write(), tell(), seek(), and +close(). + +This returns an instance of a class with the following public methods: + setnchannels(n) -- set the number of channels + setsampwidth(n) -- set the sample width + setframerate(n) -- set the frame rate + setnframes(n) -- set the number of frames + setcomptype(type, name) + -- set the compression type and the + human-readable compression type + setparams(tuple) + -- set all parameters at once + tell() -- return current position in output file + writeframesraw(data) + -- write audio frames without patching up the + file header + writeframes(data) + -- write audio frames and patch up the file header + close() -- patch up the file header and close the + output file +You should set the parameters before the first writeframesraw or +writeframes. The total number of frames does not need to be set, +but when it is set to the correct value, the header does not have to +be patched up. +It is best to first set all parameters, perhaps possibly the +compression type, and then write audio frames using writeframesraw. +When all frames have been written, either call writeframes(b'') or +close() to patch up the sizes in the header. +The close() method is called automatically when the class instance +is destroyed. +""" + +from collections import namedtuple +import builtins +import struct +import sys + + +__all__ = ["open", "Error", "Wave_read", "Wave_write"] + +class Error(Exception): + pass + +WAVE_FORMAT_PCM = 0x0001 +WAVE_FORMAT_EXTENSIBLE = 0xFFFE +# Derived from uuid.UUID("00000001-0000-0010-8000-00aa00389b71").bytes_le +KSDATAFORMAT_SUBTYPE_PCM = b'\x01\x00\x00\x00\x00\x00\x10\x00\x80\x00\x00\xaa\x008\x9bq' + +_array_fmts = None, 'b', 'h', None, 'i' + +_wave_params = namedtuple('_wave_params', + 'nchannels sampwidth framerate nframes comptype compname') + + +def _byteswap(data, width): + swapped_data = bytearray(len(data)) + + for i in range(0, len(data), width): + for j in range(width): + swapped_data[i + width - 1 - j] = data[i + j] + + return bytes(swapped_data) + + +class _Chunk: + def __init__(self, file, align=True, bigendian=True, inclheader=False): + self.closed = False + self.align = align # whether to align to word (2-byte) boundaries + if bigendian: + strflag = '>' + else: + strflag = '<' + self.file = file + self.chunkname = file.read(4) + if len(self.chunkname) < 4: + raise EOFError + try: + self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0] + except struct.error: + raise EOFError from None + if inclheader: + self.chunksize = self.chunksize - 8 # subtract header + self.size_read = 0 + try: + self.offset = self.file.tell() + except (AttributeError, OSError): + self.seekable = False + else: + self.seekable = True + + def getname(self): + """Return the name (ID) of the current chunk.""" + return self.chunkname + + def close(self): + if not self.closed: + try: + self.skip() + finally: + self.closed = True + + def seek(self, pos, whence=0): + """Seek to specified position into the chunk. + Default position is 0 (start of chunk). + If the file is not seekable, this will result in an error. + """ + + if self.closed: + raise ValueError("I/O operation on closed file") + if not self.seekable: + raise OSError("cannot seek") + if whence == 1: + pos = pos + self.size_read + elif whence == 2: + pos = pos + self.chunksize + if pos < 0 or pos > self.chunksize: + raise RuntimeError + self.file.seek(self.offset + pos, 0) + self.size_read = pos + + def tell(self): + if self.closed: + raise ValueError("I/O operation on closed file") + return self.size_read + + def read(self, size=-1): + """Read at most size bytes from the chunk. + If size is omitted or negative, read until the end + of the chunk. + """ + + if self.closed: + raise ValueError("I/O operation on closed file") + if self.size_read >= self.chunksize: + return b'' + if size < 0: + size = self.chunksize - self.size_read + if size > self.chunksize - self.size_read: + size = self.chunksize - self.size_read + data = self.file.read(size) + self.size_read = self.size_read + len(data) + if self.size_read == self.chunksize and \ + self.align and \ + (self.chunksize & 1): + dummy = self.file.read(1) + self.size_read = self.size_read + len(dummy) + return data + + def skip(self): + """Skip the rest of the chunk. + If you are not interested in the contents of the chunk, + this method should be called so that the file points to + the start of the next chunk. + """ + + if self.closed: + raise ValueError("I/O operation on closed file") + if self.seekable: + try: + n = self.chunksize - self.size_read + # maybe fix alignment + if self.align and (self.chunksize & 1): + n = n + 1 + self.file.seek(n, 1) + self.size_read = self.size_read + n + return + except OSError: + pass + while self.size_read < self.chunksize: + n = min(8192, self.chunksize - self.size_read) + dummy = self.read(n) + if not dummy: + raise EOFError + + +class Wave_read: + """Variables used in this class: + + These variables are available to the user though appropriate + methods of this class: + _file -- the open file with methods read(), close(), and seek() + set through the __init__() method + _nchannels -- the number of audio channels + available through the getnchannels() method + _nframes -- the number of audio frames + available through the getnframes() method + _sampwidth -- the number of bytes per audio sample + available through the getsampwidth() method + _framerate -- the sampling frequency + available through the getframerate() method + _comptype -- the AIFF-C compression type ('NONE' if AIFF) + available through the getcomptype() method + _compname -- the human-readable AIFF-C compression type + available through the getcomptype() method + _soundpos -- the position in the audio stream + available through the tell() method, set through the + setpos() method + + These variables are used internally only: + _fmt_chunk_read -- 1 iff the FMT chunk has been read + _data_seek_needed -- 1 iff positioned correctly in audio + file for readframes() + _data_chunk -- instantiation of a chunk class for the DATA chunk + _framesize -- size of one frame in the file + """ + + def initfp(self, file): + self._convert = None + self._soundpos = 0 + self._file = _Chunk(file, bigendian = 0) + if self._file.getname() != b'RIFF': + raise Error('file does not start with RIFF id') + if self._file.read(4) != b'WAVE': + raise Error('not a WAVE file') + self._fmt_chunk_read = 0 + self._data_chunk = None + while 1: + self._data_seek_needed = 1 + try: + chunk = _Chunk(self._file, bigendian = 0) + except EOFError: + break + chunkname = chunk.getname() + if chunkname == b'fmt ': + self._read_fmt_chunk(chunk) + self._fmt_chunk_read = 1 + elif chunkname == b'data': + if not self._fmt_chunk_read: + raise Error('data chunk before fmt chunk') + self._data_chunk = chunk + self._nframes = chunk.chunksize // self._framesize + self._data_seek_needed = 0 + break + chunk.skip() + if not self._fmt_chunk_read or not self._data_chunk: + raise Error('fmt chunk and/or data chunk missing') + + def __init__(self, f): + self._i_opened_the_file = None + if isinstance(f, str): + f = builtins.open(f, 'rb') + self._i_opened_the_file = f + # else, assume it is an open file object already + try: + self.initfp(f) + except: + if self._i_opened_the_file: + f.close() + raise + + def __del__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + # + # User visible methods. + # + def getfp(self): + return self._file + + def rewind(self): + self._data_seek_needed = 1 + self._soundpos = 0 + + def close(self): + self._file = None + file = self._i_opened_the_file + if file: + self._i_opened_the_file = None + file.close() + + def tell(self): + return self._soundpos + + def getnchannels(self): + return self._nchannels + + def getnframes(self): + return self._nframes + + def getsampwidth(self): + return self._sampwidth + + def getframerate(self): + return self._framerate + + def getcomptype(self): + return self._comptype + + def getcompname(self): + return self._compname + + def getparams(self): + return _wave_params(self.getnchannels(), self.getsampwidth(), + self.getframerate(), self.getnframes(), + self.getcomptype(), self.getcompname()) + + def getmarkers(self): + import warnings + warnings._deprecated("Wave_read.getmarkers", remove=(3, 15)) + return None + + def getmark(self, id): + import warnings + warnings._deprecated("Wave_read.getmark", remove=(3, 15)) + raise Error('no marks') + + def setpos(self, pos): + if pos < 0 or pos > self._nframes: + raise Error('position not in range') + self._soundpos = pos + self._data_seek_needed = 1 + + def readframes(self, nframes): + if self._data_seek_needed: + self._data_chunk.seek(0, 0) + pos = self._soundpos * self._framesize + if pos: + self._data_chunk.seek(pos, 0) + self._data_seek_needed = 0 + if nframes == 0: + return b'' + data = self._data_chunk.read(nframes * self._framesize) + if self._sampwidth != 1 and sys.byteorder == 'big': + data = _byteswap(data, self._sampwidth) + if self._convert and data: + data = self._convert(data) + self._soundpos = self._soundpos + len(data) // (self._nchannels * self._sampwidth) + return data + + # + # Internal methods. + # + + def _read_fmt_chunk(self, chunk): + try: + wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from('<HHLLH', chunk.read(14)) + except struct.error: + raise EOFError from None + if wFormatTag != WAVE_FORMAT_PCM and wFormatTag != WAVE_FORMAT_EXTENSIBLE: + raise Error('unknown format: %r' % (wFormatTag,)) + try: + sampwidth = struct.unpack_from('<H', chunk.read(2))[0] + except struct.error: + raise EOFError from None + if wFormatTag == WAVE_FORMAT_EXTENSIBLE: + try: + cbSize, wValidBitsPerSample, dwChannelMask = struct.unpack_from('<HHL', chunk.read(8)) + # Read the entire UUID from the chunk + SubFormat = chunk.read(16) + if len(SubFormat) < 16: + raise EOFError + except struct.error: + raise EOFError from None + if SubFormat != KSDATAFORMAT_SUBTYPE_PCM: + try: + import uuid + subformat_msg = f'unknown extended format: {uuid.UUID(bytes_le=SubFormat)}' + except Exception: + subformat_msg = 'unknown extended format' + raise Error(subformat_msg) + self._sampwidth = (sampwidth + 7) // 8 + if not self._sampwidth: + raise Error('bad sample width') + if not self._nchannels: + raise Error('bad # of channels') + self._framesize = self._nchannels * self._sampwidth + self._comptype = 'NONE' + self._compname = 'not compressed' + + +class Wave_write: + """Variables used in this class: + + These variables are user settable through appropriate methods + of this class: + _file -- the open file with methods write(), close(), tell(), seek() + set through the __init__() method + _comptype -- the AIFF-C compression type ('NONE' in AIFF) + set through the setcomptype() or setparams() method + _compname -- the human-readable AIFF-C compression type + set through the setcomptype() or setparams() method + _nchannels -- the number of audio channels + set through the setnchannels() or setparams() method + _sampwidth -- the number of bytes per audio sample + set through the setsampwidth() or setparams() method + _framerate -- the sampling frequency + set through the setframerate() or setparams() method + _nframes -- the number of audio frames written to the header + set through the setnframes() or setparams() method + + These variables are used internally only: + _datalength -- the size of the audio samples written to the header + _nframeswritten -- the number of frames actually written + _datawritten -- the size of the audio samples actually written + """ + + _file = None + + def __init__(self, f): + self._i_opened_the_file = None + if isinstance(f, str): + f = builtins.open(f, 'wb') + self._i_opened_the_file = f + try: + self.initfp(f) + except: + if self._i_opened_the_file: + f.close() + raise + + def initfp(self, file): + self._file = file + self._convert = None + self._nchannels = 0 + self._sampwidth = 0 + self._framerate = 0 + self._nframes = 0 + self._nframeswritten = 0 + self._datawritten = 0 + self._datalength = 0 + self._headerwritten = False + + def __del__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + # + # User visible methods. + # + def setnchannels(self, nchannels): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + if nchannels < 1: + raise Error('bad # of channels') + self._nchannels = nchannels + + def getnchannels(self): + if not self._nchannels: + raise Error('number of channels not set') + return self._nchannels + + def setsampwidth(self, sampwidth): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + if sampwidth < 1 or sampwidth > 4: + raise Error('bad sample width') + self._sampwidth = sampwidth + + def getsampwidth(self): + if not self._sampwidth: + raise Error('sample width not set') + return self._sampwidth + + def setframerate(self, framerate): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + if framerate <= 0: + raise Error('bad frame rate') + self._framerate = int(round(framerate)) + + def getframerate(self): + if not self._framerate: + raise Error('frame rate not set') + return self._framerate + + def setnframes(self, nframes): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + self._nframes = nframes + + def getnframes(self): + return self._nframeswritten + + def setcomptype(self, comptype, compname): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + if comptype not in ('NONE',): + raise Error('unsupported compression type') + self._comptype = comptype + self._compname = compname + + def getcomptype(self): + return self._comptype + + def getcompname(self): + return self._compname + + def setparams(self, params): + nchannels, sampwidth, framerate, nframes, comptype, compname = params + if self._datawritten: + raise Error('cannot change parameters after starting to write') + self.setnchannels(nchannels) + self.setsampwidth(sampwidth) + self.setframerate(framerate) + self.setnframes(nframes) + self.setcomptype(comptype, compname) + + def getparams(self): + if not self._nchannels or not self._sampwidth or not self._framerate: + raise Error('not all parameters set') + return _wave_params(self._nchannels, self._sampwidth, self._framerate, + self._nframes, self._comptype, self._compname) + + def setmark(self, id, pos, name): + import warnings + warnings._deprecated("Wave_write.setmark", remove=(3, 15)) + raise Error('setmark() not supported') + + def getmark(self, id): + import warnings + warnings._deprecated("Wave_write.getmark", remove=(3, 15)) + raise Error('no marks') + + def getmarkers(self): + import warnings + warnings._deprecated("Wave_write.getmarkers", remove=(3, 15)) + return None + + def tell(self): + return self._nframeswritten + + def writeframesraw(self, data): + if not isinstance(data, (bytes, bytearray)): + data = memoryview(data).cast('B') + self._ensure_header_written(len(data)) + nframes = len(data) // (self._sampwidth * self._nchannels) + if self._convert: + data = self._convert(data) + if self._sampwidth != 1 and sys.byteorder == 'big': + data = _byteswap(data, self._sampwidth) + self._file.write(data) + self._datawritten += len(data) + self._nframeswritten = self._nframeswritten + nframes + + def writeframes(self, data): + self.writeframesraw(data) + if self._datalength != self._datawritten: + self._patchheader() + + def close(self): + try: + if self._file: + self._ensure_header_written(0) + if self._datalength != self._datawritten: + self._patchheader() + self._file.flush() + finally: + self._file = None + file = self._i_opened_the_file + if file: + self._i_opened_the_file = None + file.close() + + # + # Internal methods. + # + + def _ensure_header_written(self, datasize): + if not self._headerwritten: + if not self._nchannels: + raise Error('# channels not specified') + if not self._sampwidth: + raise Error('sample width not specified') + if not self._framerate: + raise Error('sampling rate not specified') + self._write_header(datasize) + + def _write_header(self, initlength): + assert not self._headerwritten + self._file.write(b'RIFF') + if not self._nframes: + self._nframes = initlength // (self._nchannels * self._sampwidth) + self._datalength = self._nframes * self._nchannels * self._sampwidth + try: + self._form_length_pos = self._file.tell() + except (AttributeError, OSError): + self._form_length_pos = None + self._file.write(struct.pack('<L4s4sLHHLLHH4s', + 36 + self._datalength, b'WAVE', b'fmt ', 16, + WAVE_FORMAT_PCM, self._nchannels, self._framerate, + self._nchannels * self._framerate * self._sampwidth, + self._nchannels * self._sampwidth, + self._sampwidth * 8, b'data')) + if self._form_length_pos is not None: + self._data_length_pos = self._file.tell() + self._file.write(struct.pack('<L', self._datalength)) + self._headerwritten = True + + def _patchheader(self): + assert self._headerwritten + if self._datawritten == self._datalength: + return + curpos = self._file.tell() + self._file.seek(self._form_length_pos, 0) + self._file.write(struct.pack('<L', 36 + self._datawritten)) + self._file.seek(self._data_length_pos, 0) + self._file.write(struct.pack('<L', self._datawritten)) + self._file.seek(curpos, 0) + self._datalength = self._datawritten + + +def open(f, mode=None): + if mode is None: + if hasattr(f, 'mode'): + mode = f.mode + else: + mode = 'rb' + if mode in ('r', 'rb'): + return Wave_read(f) + elif mode in ('w', 'wb'): + return Wave_write(f) + else: + raise Error("mode must be 'r', 'rb', 'w', or 'wb'") diff --git a/Lib/weakref.py b/Lib/weakref.py index 994ea8aa37d..94e4278143c 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -2,7 +2,7 @@ This module is an implementation of PEP 205: -https://www.python.org/dev/peps/pep-0205/ +https://peps.python.org/pep-0205/ """ # Naming convention: Variables named "wr" are weak reference objects; @@ -19,7 +19,7 @@ ReferenceType, _remove_dead_weakref) -from _weakrefset import WeakSet, _IterationGuard +from _weakrefset import WeakSet import _collections_abc # Import after _weakref to avoid circular import. import sys @@ -33,7 +33,6 @@ "WeakSet", "WeakMethod", "finalize"] -_collections_abc.Set.register(WeakSet) _collections_abc.MutableSet.register(WeakSet) class WeakMethod(ref): @@ -106,34 +105,14 @@ def __init__(self, other=(), /, **kw): def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(wr.key) - else: - # Atomic removal is necessary since this function - # can be called asynchronously by the GC - _atomic_removal(self.data, wr.key) + # Atomic removal is necessary since this function + # can be called asynchronously by the GC + _atomic_removal(self.data, wr.key) self._remove = remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() self.data = {} self.update(other, **kw) - def _commit_removals(self, _atomic_removal=_remove_dead_weakref): - pop = self._pending_removals.pop - d = self.data - # We shouldn't encounter any KeyError, because this method should - # always be called *before* mutating the dict. - while True: - try: - key = pop() - except IndexError: - return - _atomic_removal(d, key) - def __getitem__(self, key): - if self._pending_removals: - self._commit_removals() o = self.data[key]() if o is None: raise KeyError(key) @@ -141,18 +120,12 @@ def __getitem__(self, key): return o def __delitem__(self, key): - if self._pending_removals: - self._commit_removals() del self.data[key] def __len__(self): - if self._pending_removals: - self._commit_removals() return len(self.data) def __contains__(self, key): - if self._pending_removals: - self._commit_removals() try: o = self.data[key]() except KeyError: @@ -163,38 +136,28 @@ def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) def __setitem__(self, key, value): - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(value, self._remove, key) def copy(self): - if self._pending_removals: - self._commit_removals() new = WeakValueDictionary() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[key] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[key] = o return new __copy__ = copy def __deepcopy__(self, memo): from copy import deepcopy - if self._pending_removals: - self._commit_removals() new = self.__class__() - with _IterationGuard(self): - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[deepcopy(key, memo)] = o + for key, wr in self.data.copy().items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o return new def get(self, key, default=None): - if self._pending_removals: - self._commit_removals() try: wr = self.data[key] except KeyError: @@ -208,21 +171,15 @@ def get(self, key, default=None): return o def items(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - v = wr() - if v is not None: - yield k, v + for k, wr in self.data.copy().items(): + v = wr() + if v is not None: + yield k, v def keys(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for k, wr in self.data.items(): - if wr() is not None: - yield k + for k, wr in self.data.copy().items(): + if wr() is not None: + yield k __iter__ = keys @@ -236,23 +193,15 @@ def itervaluerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - yield from self.data.values() + yield from self.data.copy().values() def values(self): - if self._pending_removals: - self._commit_removals() - with _IterationGuard(self): - for wr in self.data.values(): - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy().values(): + obj = wr() + if obj is not None: + yield obj def popitem(self): - if self._pending_removals: - self._commit_removals() while True: key, wr = self.data.popitem() o = wr() @@ -260,8 +209,6 @@ def popitem(self): return key, o def pop(self, key, *args): - if self._pending_removals: - self._commit_removals() try: o = self.data.pop(key)() except KeyError: @@ -280,16 +227,12 @@ def setdefault(self, key, default=None): except KeyError: o = None if o is None: - if self._pending_removals: - self._commit_removals() self.data[key] = KeyedRef(default, self._remove, key) return default else: return o def update(self, other=None, /, **kwargs): - if self._pending_removals: - self._commit_removals() d = self.data if other is not None: if not hasattr(other, "items"): @@ -309,9 +252,7 @@ def valuerefs(self): keep the values around longer than needed. """ - if self._pending_removals: - self._commit_removals() - return list(self.data.values()) + return list(self.data.copy().values()) def __ior__(self, other): self.update(other) @@ -370,57 +311,22 @@ def __init__(self, dict=None): def remove(k, selfref=ref(self)): self = selfref() if self is not None: - if self._iterating: - self._pending_removals.append(k) - else: - try: - del self.data[k] - except KeyError: - pass + try: + del self.data[k] + except KeyError: + pass self._remove = remove - # A list of dead weakrefs (keys to be removed) - self._pending_removals = [] - self._iterating = set() - self._dirty_len = False if dict is not None: self.update(dict) - def _commit_removals(self): - # NOTE: We don't need to call this method before mutating the dict, - # because a dead weakref never compares equal to a live weakref, - # even if they happened to refer to equal objects. - # However, it means keys may already have been removed. - pop = self._pending_removals.pop - d = self.data - while True: - try: - key = pop() - except IndexError: - return - - try: - del d[key] - except KeyError: - pass - - def _scrub_removals(self): - d = self.data - self._pending_removals = [k for k in self._pending_removals if k in d] - self._dirty_len = False - def __delitem__(self, key): - self._dirty_len = True del self.data[ref(key)] def __getitem__(self, key): return self.data[ref(key)] def __len__(self): - if self._dirty_len and self._pending_removals: - # self._pending_removals may still contain keys which were - # explicitly removed, we have to scrub them (see issue #21173). - self._scrub_removals() - return len(self.data) - len(self._pending_removals) + return len(self.data) def __repr__(self): return "<%s at %#x>" % (self.__class__.__name__, id(self)) @@ -430,11 +336,10 @@ def __setitem__(self, key, value): def copy(self): new = WeakKeyDictionary() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = value + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = value return new __copy__ = copy @@ -442,11 +347,10 @@ def copy(self): def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() - with _IterationGuard(self): - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = deepcopy(value, memo) + for key, value in self.data.copy().items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) return new def get(self, key, default=None): @@ -460,26 +364,23 @@ def __contains__(self, key): return wr in self.data def items(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - key = wr() - if key is not None: - yield key, value + for wr, value in self.data.copy().items(): + key = wr() + if key is not None: + yield key, value def keys(self): - with _IterationGuard(self): - for wr in self.data: - obj = wr() - if obj is not None: - yield obj + for wr in self.data.copy(): + obj = wr() + if obj is not None: + yield obj __iter__ = keys def values(self): - with _IterationGuard(self): - for wr, value in self.data.items(): - if wr() is not None: - yield value + for wr, value in self.data.copy().items(): + if wr() is not None: + yield value def keyrefs(self): """Return a list of weak references to the keys. @@ -494,7 +395,6 @@ def keyrefs(self): return list(self.data) def popitem(self): - self._dirty_len = True while True: key, value = self.data.popitem() o = key() @@ -502,7 +402,6 @@ def popitem(self): return o, value def pop(self, key, *args): - self._dirty_len = True return self.data.pop(ref(key), *args) def setdefault(self, key, default=None): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index ec3cece48c9..2f9555ad60d 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -11,14 +11,17 @@ __all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] + class Error(Exception): pass + _lock = threading.RLock() _browsers = {} # Dictionary of available browser controllers _tryorder = None # Preference order of available browsers _os_preferred_browser = None # The preferred browser + def register(name, klass, instance=None, *, preferred=False): """Register a browser connector.""" with _lock: @@ -29,11 +32,12 @@ def register(name, klass, instance=None, *, preferred=False): # Preferred browsers go to the front of the list. # Need to match to the default browser returned by xdg-settings, which # may be of the form e.g. "firefox.desktop". - if preferred or (_os_preferred_browser and name in _os_preferred_browser): + if preferred or (_os_preferred_browser and f'{name}.desktop' == _os_preferred_browser): _tryorder.insert(0, name) else: _tryorder.append(name) + def get(using=None): """Return a browser launcher instance appropriate for the environment.""" if _tryorder is None: @@ -64,6 +68,7 @@ def get(using=None): return command[0]() raise Error("could not locate runnable browser") + # Please note: the following definition hides a builtin function. # It is recommended one does "import webbrowser" and uses webbrowser.open(url) # instead of "from webbrowser import *". @@ -76,6 +81,9 @@ def open(url, new=0, autoraise=True): - 1: a new browser window. - 2: a new browser page ("tab"). If possible, autoraise raises the window (the default) or not. + + If opening the browser succeeds, return True. + If there is a problem, return False. """ if _tryorder is None: with _lock: @@ -87,6 +95,7 @@ def open(url, new=0, autoraise=True): return True return False + def open_new(url): """Open url in a new window of the default browser. @@ -94,6 +103,7 @@ def open_new(url): """ return open(url, 1) + def open_new_tab(url): """Open url in a new page ("tab") of the default browser. @@ -136,7 +146,7 @@ def _synthesize(browser, *, preferred=False): # General parent classes -class BaseBrowser(object): +class BaseBrowser: """Parent class for all browsers. Do not use directly.""" args = ['%s'] @@ -197,7 +207,7 @@ def open(self, url, new=0, autoraise=True): else: p = subprocess.Popen(cmdline, close_fds=True, start_new_session=True) - return (p.poll() is None) + return p.poll() is None except OSError: return False @@ -225,7 +235,8 @@ def _invoke(self, args, remote, autoraise, url=None): # use autoraise argument only for remote invocation autoraise = int(autoraise) opt = self.raise_opts[autoraise] - if opt: raise_opt = [opt] + if opt: + raise_opt = [opt] cmdline = [self.name] + raise_opt + args @@ -266,8 +277,8 @@ def open(self, url, new=0, autoraise=True): else: action = self.remote_action_newtab else: - raise Error("Bad 'new' parameter to open(); " + - "expected 0, 1, or 2, got %s" % new) + raise Error("Bad 'new' parameter to open(); " + f"expected 0, 1, or 2, got {new}") args = [arg.replace("%s", url).replace("%action", action) for arg in self.remote_args] @@ -291,19 +302,8 @@ class Mozilla(UnixBrowser): background = True -class Netscape(UnixBrowser): - """Launcher class for Netscape browser.""" - - raise_opts = ["-noraise", "-raise"] - remote_args = ['-remote', 'openURL(%s%action)'] - remote_action = "" - remote_action_newwin = ",new-window" - remote_action_newtab = ",new-tab" - background = True - - -class Galeon(UnixBrowser): - """Launcher class for Galeon/Epiphany browsers.""" +class Epiphany(UnixBrowser): + """Launcher class for Epiphany browser.""" raise_opts = ["-noraise", ""] remote_args = ['%action', '%s'] @@ -313,7 +313,7 @@ class Galeon(UnixBrowser): class Chrome(UnixBrowser): - "Launcher class for Google Chrome browser." + """Launcher class for Google Chrome browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -321,11 +321,12 @@ class Chrome(UnixBrowser): remote_action_newtab = "" background = True + Chromium = Chrome class Opera(UnixBrowser): - "Launcher class for Opera browser." + """Launcher class for Opera browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -335,7 +336,7 @@ class Opera(UnixBrowser): class Elinks(UnixBrowser): - "Launcher class for Elinks browsers." + """Launcher class for Elinks browsers.""" remote_args = ['-remote', 'openURL(%s%action)'] remote_action = "" @@ -398,54 +399,17 @@ def open(self, url, new=0, autoraise=True): except OSError: return False else: - return (p.poll() is None) - - -class Grail(BaseBrowser): - # There should be a way to maintain a connection to Grail, but the - # Grail remote control protocol doesn't really allow that at this - # point. It probably never will! - def _find_grail_rc(self): - import glob - import pwd - import socket - import tempfile - tempdir = os.path.join(tempfile.gettempdir(), - ".grail-unix") - user = pwd.getpwuid(os.getuid())[0] - filename = os.path.join(glob.escape(tempdir), glob.escape(user) + "-*") - maybes = glob.glob(filename) - if not maybes: - return None - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - for fn in maybes: - # need to PING each one until we find one that's live - try: - s.connect(fn) - except OSError: - # no good; attempt to clean it out, but don't fail: - try: - os.unlink(fn) - except OSError: - pass - else: - return s + return p.poll() is None - def _remote(self, action): - s = self._find_grail_rc() - if not s: - return 0 - s.send(action) - s.close() - return 1 - def open(self, url, new=0, autoraise=True): - sys.audit("webbrowser.open", url) - if new: - ok = self._remote("LOADNEW " + url) - else: - ok = self._remote("LOAD " + url) - return ok +class Edge(UnixBrowser): + """Launcher class for Microsoft Edge browser.""" + + remote_args = ['%action', '%s'] + remote_action = "" + remote_action_newwin = "--new-window" + remote_action_newtab = "" + background = True # @@ -461,47 +425,44 @@ def register_X_browsers(): if shutil.which("xdg-open"): register("xdg-open", None, BackgroundBrowser("xdg-open")) + # Opens an appropriate browser for the URL scheme according to + # freedesktop.org settings (GNOME, KDE, XFCE, etc.) + if shutil.which("gio"): + register("gio", None, BackgroundBrowser(["gio", "open", "--", "%s"])) + + xdg_desktop = os.getenv("XDG_CURRENT_DESKTOP", "").split(":") + # The default GNOME3 browser - if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gvfs-open"): + if (("GNOME" in xdg_desktop or + "GNOME_DESKTOP_SESSION_ID" in os.environ) and + shutil.which("gvfs-open")): register("gvfs-open", None, BackgroundBrowser("gvfs-open")) - # The default GNOME browser - if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gnome-open"): - register("gnome-open", None, BackgroundBrowser("gnome-open")) - # The default KDE browser - if "KDE_FULL_SESSION" in os.environ and shutil.which("kfmclient"): + if (("KDE" in xdg_desktop or + "KDE_FULL_SESSION" in os.environ) and + shutil.which("kfmclient")): register("kfmclient", Konqueror, Konqueror("kfmclient")) + # Common symbolic link for the default X11 browser if shutil.which("x-www-browser"): register("x-www-browser", None, BackgroundBrowser("x-www-browser")) # The Mozilla browsers - for browser in ("firefox", "iceweasel", "iceape", "seamonkey"): + for browser in ("firefox", "iceweasel", "seamonkey", "mozilla-firefox", + "mozilla"): if shutil.which(browser): register(browser, None, Mozilla(browser)) - # The Netscape and old Mozilla browsers - for browser in ("mozilla-firefox", - "mozilla-firebird", "firebird", - "mozilla", "netscape"): - if shutil.which(browser): - register(browser, None, Netscape(browser)) - # Konqueror/kfm, the KDE browser. if shutil.which("kfm"): register("kfm", Konqueror, Konqueror("kfm")) elif shutil.which("konqueror"): register("konqueror", Konqueror, Konqueror("konqueror")) - # Gnome's Galeon and Epiphany - for browser in ("galeon", "epiphany"): - if shutil.which(browser): - register(browser, None, Galeon(browser)) - - # Skipstone, another Gtk/Mozilla based browser - if shutil.which("skipstone"): - register("skipstone", None, BackgroundBrowser("skipstone")) + # Gnome's Epiphany + if shutil.which("epiphany"): + register("epiphany", None, Epiphany("epiphany")) # Google Chrome/Chromium browsers for browser in ("google-chrome", "chrome", "chromium", "chromium-browser"): @@ -512,13 +473,9 @@ def register_X_browsers(): if shutil.which("opera"): register("opera", None, Opera("opera")) - # Next, Mosaic -- old but still in use. - if shutil.which("mosaic"): - register("mosaic", None, BackgroundBrowser("mosaic")) + if shutil.which("microsoft-edge"): + register("microsoft-edge", None, Edge("microsoft-edge")) - # Grail, the Python browser. Does anybody still use it? - if shutil.which("grail"): - register("grail", Grail, None) def register_standard_browsers(): global _tryorder @@ -532,6 +489,9 @@ def register_standard_browsers(): # OS X can use below Unix support (but we prefer using the OS X # specific stuff) + if sys.platform == "ios": + register("iosbrowser", None, IOSBrowser(), preferred=True) + if sys.platform == "serenityos": # SerenityOS webbrowser, simply called "Browser". register("Browser", None, BackgroundBrowser("Browser")) @@ -540,21 +500,33 @@ def register_standard_browsers(): # First try to use the default Windows browser register("windows-default", WindowsDefault) - # Detect some common Windows browsers, fallback to IE - iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), - "Internet Explorer\\IEXPLORE.EXE") - for browser in ("firefox", "firebird", "seamonkey", "mozilla", - "netscape", "opera", iexplore): + # Detect some common Windows browsers, fallback to Microsoft Edge + # location in 64-bit Windows + edge64 = os.path.join(os.environ.get("PROGRAMFILES(x86)", "C:\\Program Files (x86)"), + "Microsoft\\Edge\\Application\\msedge.exe") + # location in 32-bit Windows + edge32 = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), + "Microsoft\\Edge\\Application\\msedge.exe") + for browser in ("firefox", "seamonkey", "mozilla", "chrome", + "opera", edge64, edge32): if shutil.which(browser): register(browser, None, BackgroundBrowser(browser)) + if shutil.which("MicrosoftEdge.exe"): + register("microsoft-edge", None, Edge("MicrosoftEdge.exe")) else: # Prefer X browsers if present - if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): + # + # NOTE: Do not check for X11 browser on macOS, + # XQuartz installation sets a DISPLAY environment variable and will + # autostart when someone tries to access the display. Mac users in + # general don't need an X11 browser. + if sys.platform != "darwin" and (os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")): try: cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() - except (FileNotFoundError, subprocess.CalledProcessError, PermissionError, NotADirectoryError) : + except (FileNotFoundError, subprocess.CalledProcessError, + PermissionError, NotADirectoryError): pass else: global _os_preferred_browser @@ -564,14 +536,15 @@ def register_standard_browsers(): # Also try console browsers if os.environ.get("TERM"): + # Common symbolic link for the default text-based browser if shutil.which("www-browser"): register("www-browser", None, GenericBrowser("www-browser")) - # The Links/elinks browsers <http://artax.karlin.mff.cuni.cz/~mikulas/links/> + # The Links/elinks browsers <http://links.twibright.com/> if shutil.which("links"): register("links", None, GenericBrowser("links")) if shutil.which("elinks"): register("elinks", None, Elinks("elinks")) - # The Lynx browser <http://lynx.isc.org/>, <http://lynx.browser.org/> + # The Lynx browser <https://lynx.invisible-island.net/>, <http://lynx.browser.org/> if shutil.which("lynx"): register("lynx", None, GenericBrowser("lynx")) # The w3m browser <http://w3m.sourceforge.net/> @@ -613,105 +586,125 @@ def open(self, url, new=0, autoraise=True): return True # -# Platform support for MacOS +# Platform support for macOS # if sys.platform == 'darwin': - # Adapted from patch submitted to SourceForge by Steven J. Burr - class MacOSX(BaseBrowser): - """Launcher class for Aqua browsers on Mac OS X - - Optionally specify a browser name on instantiation. Note that this - will not work for Aqua browsers if the user has moved the application - package after installation. - - If no browser is specified, the default browser, as specified in the - Internet System Preferences panel, will be used. - """ - def __init__(self, name): - self.name = name + class MacOSXOSAScript(BaseBrowser): + def __init__(self, name='default'): + super().__init__(name) def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) - assert "'" not in url - # hack for local urls - if not ':' in url: - url = 'file:'+url - - # new must be 0 or 1 - new = int(bool(new)) - if self.name == "default": - # User called open, open_new or get without a browser parameter - script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser + url = url.replace('"', '%22') + if self.name == 'default': + script = f'open location "{url}"' # opens in default browser else: - # User called get and chose a browser - if self.name == "OmniWeb": - toWindow = "" - else: - # Include toWindow parameter of OpenURL command for browsers - # that support it. 0 == new window; -1 == existing - toWindow = "toWindow %d" % (new - 1) - cmd = 'OpenURL "%s"' % url.replace('"', '%22') - script = '''tell application "%s" - activate - %s %s - end tell''' % (self.name, cmd, toWindow) - # Open pipe to AppleScript through osascript command + script = f''' + tell application "{self.name}" + activate + open location "{url}" + end + ''' + osapipe = os.popen("osascript", "w") if osapipe is None: return False - # Write script to osascript's stdin + osapipe.write(script) rc = osapipe.close() return not rc - class MacOSXOSAScript(BaseBrowser): - def __init__(self, name): - self._name = name +# +# Platform support for iOS +# +if sys.platform == "ios": + from _ios_support import objc + if objc: + # If objc exists, we know ctypes is also importable. + from ctypes import c_void_p, c_char_p, c_ulong + class IOSBrowser(BaseBrowser): def open(self, url, new=0, autoraise=True): - if self._name == 'default': - script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser - else: - script = ''' - tell application "%s" - activate - open location "%s" - end - '''%(self._name, url.replace('"', '%22')) - - osapipe = os.popen("osascript", "w") - if osapipe is None: + sys.audit("webbrowser.open", url) + # If ctypes isn't available, we can't open a browser + if objc is None: return False - osapipe.write(script) - rc = osapipe.close() - return not rc + # All the messages in this call return object references. + objc.objc_msgSend.restype = c_void_p + + # This is the equivalent of: + # NSString url_string = + # [NSString stringWithCString:url.encode("utf-8") + # encoding:NSUTF8StringEncoding]; + NSString = objc.objc_getClass(b"NSString") + constructor = objc.sel_registerName(b"stringWithCString:encoding:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_char_p, c_ulong] + url_string = objc.objc_msgSend( + NSString, + constructor, + url.encode("utf-8"), + 4, # NSUTF8StringEncoding = 4 + ) + + # Create an NSURL object representing the URL + # This is the equivalent of: + # NSURL *nsurl = [NSURL URLWithString:url]; + NSURL = objc.objc_getClass(b"NSURL") + urlWithString_ = objc.sel_registerName(b"URLWithString:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_void_p] + ns_url = objc.objc_msgSend(NSURL, urlWithString_, url_string) + + # Get the shared UIApplication instance + # This code is the equivalent of: + # UIApplication shared_app = [UIApplication sharedApplication] + UIApplication = objc.objc_getClass(b"UIApplication") + sharedApplication = objc.sel_registerName(b"sharedApplication") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + shared_app = objc.objc_msgSend(UIApplication, sharedApplication) + + # Open the URL on the shared application + # This code is the equivalent of: + # [shared_app openURL:ns_url + # options:NIL + # completionHandler:NIL]; + openURL_ = objc.sel_registerName(b"openURL:options:completionHandler:") + objc.objc_msgSend.argtypes = [ + c_void_p, c_void_p, c_void_p, c_void_p, c_void_p + ] + # Method returns void + objc.objc_msgSend.restype = None + objc.objc_msgSend(shared_app, openURL_, ns_url, None, None) + return True -def main(): - import getopt - usage = """Usage: %s [-n | -t] url - -n: open new window - -t: open new tab""" % sys.argv[0] - try: - opts, args = getopt.getopt(sys.argv[1:], 'ntd') - except getopt.error as msg: - print(msg, file=sys.stderr) - print(usage, file=sys.stderr) - sys.exit(1) - new_win = 0 - for o, a in opts: - if o == '-n': new_win = 1 - elif o == '-t': new_win = 2 - if len(args) != 1: - print(usage, file=sys.stderr) - sys.exit(1) - - url = args[0] - open(url, new_win) + +def parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser(description="Open URL in a web browser.") + parser.add_argument("url", help="URL to open") + + group = parser.add_mutually_exclusive_group() + group.add_argument("-n", "--new-window", action="store_const", + const=1, default=0, dest="new_win", + help="open new window") + group.add_argument("-t", "--new-tab", action="store_const", + const=2, default=0, dest="new_win", + help="open new tab") + + args = parser.parse_args(arg_list) + + return args + + +def main(arg_list: list[str] | None = None): + args = parse_args(arg_list) + + open(args.url, args.new_win) print("\a") + if __name__ == "__main__": main() diff --git a/Lib/wsgiref/__init__.py b/Lib/wsgiref/__init__.py index 1efbba01a30..59ee48fddec 100644 --- a/Lib/wsgiref/__init__.py +++ b/Lib/wsgiref/__init__.py @@ -13,6 +13,8 @@ * validate -- validation wrapper that sits between an app and a server to detect errors in either +* types -- collection of WSGI-related types for static type checking + To-Do: * cgi_gateway -- Run WSGI apps under CGI (pending a deployment standard) diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py index f4300b831a4..cafe872c7aa 100644 --- a/Lib/wsgiref/handlers.py +++ b/Lib/wsgiref/handlers.py @@ -136,6 +136,10 @@ def run(self, application): self.setup_environ() self.result = application(self.environ, self.start_response) self.finish_response() + except (ConnectionAbortedError, BrokenPipeError, ConnectionResetError): + # We expect the client to close the connection abruptly from time + # to time. + return except: try: self.handle_error() @@ -179,7 +183,16 @@ def finish_response(self): for data in self.result: self.write(data) self.finish_content() - finally: + except: + # Call close() on the iterable returned by the WSGI application + # in case of an exception. + if hasattr(self.result, 'close'): + self.result.close() + raise + else: + # We only call close() when no exception is raised, because it + # will set status, result, headers, and environ fields to None. + # See bpo-29183 for more details. self.close() @@ -215,8 +228,7 @@ def start_response(self, status, headers,exc_info=None): if exc_info: try: if self.headers_sent: - # Re-raise original exception if headers sent - raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) + raise finally: exc_info = None # avoid dangling circular ref elif self.headers is not None: @@ -225,18 +237,25 @@ def start_response(self, status, headers,exc_info=None): self.status = status self.headers = self.headers_class(headers) status = self._convert_string_type(status, "Status") - assert len(status)>=4,"Status must be at least 4 characters" - assert status[:3].isdigit(), "Status message must begin w/3-digit code" - assert status[3]==" ", "Status message must have a space after code" + self._validate_status(status) if __debug__: for name, val in headers: name = self._convert_string_type(name, "Header name") val = self._convert_string_type(val, "Header value") - assert not is_hop_by_hop(name),"Hop-by-hop headers not allowed" + assert not is_hop_by_hop(name),\ + f"Hop-by-hop header, '{name}: {val}', not allowed" return self.write + def _validate_status(self, status): + if len(status) < 4: + raise AssertionError("Status must be at least 4 characters") + if not status[:3].isdigit(): + raise AssertionError("Status message must begin w/3-digit code") + if status[3] != " ": + raise AssertionError("Status message must have a space after code") + def _convert_string_type(self, value, title): """Convert/check value type.""" if type(value) is str: @@ -456,10 +475,7 @@ def _write(self,data): from warnings import warn warn("SimpleHandler.stdout.write() should not do partial writes", DeprecationWarning) - while True: - data = data[result:] - if not data: - break + while data := data[result:]: result = self.stdout.write(data) def _flush(self): diff --git a/Lib/wsgiref/simple_server.py b/Lib/wsgiref/simple_server.py index f71563a5ae0..a0f2397fcf0 100644 --- a/Lib/wsgiref/simple_server.py +++ b/Lib/wsgiref/simple_server.py @@ -84,10 +84,6 @@ def get_environ(self): env['PATH_INFO'] = urllib.parse.unquote(path, 'iso-8859-1') env['QUERY_STRING'] = query - - host = self.address_string() - if host != self.client_address[0]: - env['REMOTE_HOST'] = host env['REMOTE_ADDR'] = self.client_address[0] if self.headers.get('content-type') is None: @@ -127,7 +123,8 @@ def handle(self): return handler = ServerHandler( - self.rfile, self.wfile, self.get_stderr(), self.get_environ() + self.rfile, self.wfile, self.get_stderr(), self.get_environ(), + multithread=False, ) handler.request_handler = self # backpointer for logging handler.run(self.server.get_app()) diff --git a/Lib/wsgiref/types.py b/Lib/wsgiref/types.py new file mode 100644 index 00000000000..ef0aead5b28 --- /dev/null +++ b/Lib/wsgiref/types.py @@ -0,0 +1,54 @@ +"""WSGI-related types for static type checking""" + +from collections.abc import Callable, Iterable, Iterator +from types import TracebackType +from typing import Any, Protocol, TypeAlias + +__all__ = [ + "StartResponse", + "WSGIEnvironment", + "WSGIApplication", + "InputStream", + "ErrorStream", + "FileWrapper", +] + +_ExcInfo: TypeAlias = tuple[type[BaseException], BaseException, TracebackType] +_OptExcInfo: TypeAlias = _ExcInfo | tuple[None, None, None] + +class StartResponse(Protocol): + """start_response() callable as defined in PEP 3333""" + def __call__( + self, + status: str, + headers: list[tuple[str, str]], + exc_info: _OptExcInfo | None = ..., + /, + ) -> Callable[[bytes], object]: ... + +WSGIEnvironment: TypeAlias = dict[str, Any] +WSGIApplication: TypeAlias = Callable[[WSGIEnvironment, StartResponse], + Iterable[bytes]] + +class InputStream(Protocol): + """WSGI input stream as defined in PEP 3333""" + def read(self, size: int = ..., /) -> bytes: ... + def readline(self, size: int = ..., /) -> bytes: ... + def readlines(self, hint: int = ..., /) -> list[bytes]: ... + def __iter__(self) -> Iterator[bytes]: ... + +class ErrorStream(Protocol): + """WSGI error stream as defined in PEP 3333""" + def flush(self) -> object: ... + def write(self, s: str, /) -> object: ... + def writelines(self, seq: list[str], /) -> object: ... + +class _Readable(Protocol): + def read(self, size: int = ..., /) -> bytes: ... + # Optional: def close(self) -> object: ... + +class FileWrapper(Protocol): + """WSGI file wrapper as defined in PEP 3333""" + def __call__( + self, file: _Readable, block_size: int = ..., /, + ) -> Iterable[bytes]: ... diff --git a/Lib/wsgiref/util.py b/Lib/wsgiref/util.py index 516fe898d01..63b92331737 100644 --- a/Lib/wsgiref/util.py +++ b/Lib/wsgiref/util.py @@ -4,7 +4,7 @@ __all__ = [ 'FileWrapper', 'guess_scheme', 'application_uri', 'request_uri', - 'shift_path_info', 'setup_testing_defaults', + 'shift_path_info', 'setup_testing_defaults', 'is_hop_by_hop', ] @@ -17,12 +17,6 @@ def __init__(self, filelike, blksize=8192): if hasattr(filelike,'close'): self.close = filelike.close - def __getitem__(self,key): - data = self.filelike.read(self.blksize) - if data: - return data - raise IndexError - def __iter__(self): return self @@ -155,9 +149,9 @@ def setup_testing_defaults(environ): _hoppish = { - 'connection':1, 'keep-alive':1, 'proxy-authenticate':1, - 'proxy-authorization':1, 'te':1, 'trailers':1, 'transfer-encoding':1, - 'upgrade':1 + 'connection', 'keep-alive', 'proxy-authenticate', + 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', + 'upgrade' }.__contains__ def is_hop_by_hop(header_name): diff --git a/Lib/wsgiref/validate.py b/Lib/wsgiref/validate.py index 6107dcd7a4d..1a1853cd63a 100644 --- a/Lib/wsgiref/validate.py +++ b/Lib/wsgiref/validate.py @@ -1,6 +1,6 @@ # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) -# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php -# Also licenced under the Apache License, 2.0: http://opensource.org/licenses/apache2.0.php +# Licensed under the MIT license: https://opensource.org/licenses/mit-license.php +# Also licenced under the Apache License, 2.0: https://opensource.org/licenses/apache2.0.php # Licensed to PSF under a Contributor Agreement """ Middleware to check for obedience to the WSGI specification. @@ -77,7 +77,7 @@ * That wsgi.input is used properly: - - .read() is called with zero or one argument + - .read() is called with exactly one argument - That it returns a string @@ -137,7 +137,7 @@ def validator(application): """ When applied between a WSGI server and a WSGI application, this - middleware will check for WSGI compliancy on a number of levels. + middleware will check for WSGI compliance on a number of levels. This middleware does not modify the request or response in any way, but will raise an AssertionError if anything seems off (except for a failure to close the application iterator, which @@ -214,10 +214,7 @@ def readlines(self, *args): return lines def __iter__(self): - while 1: - line = self.readline() - if not line: - return + while line := self.readline(): yield line def close(self): @@ -390,7 +387,6 @@ def check_headers(headers): assert_(type(headers) is list, "Headers (%r) must be of type list: %r" % (headers, type(headers))) - header_names = {} for item in headers: assert_(type(item) is tuple, "Individual headers (%r) must be of type tuple: %r" @@ -403,7 +399,6 @@ def check_headers(headers): "The Status header cannot be used; it conflicts with CGI " "script, and HTTP status is not given through headers " "(value: %r)." % value) - header_names[name.lower()] = None assert_('\n' not in name and ':' not in name, "Header names may not contain ':' or '\\n': %r" % name) assert_(header_re.search(name), "Bad header name: %r" % name) diff --git a/Lib/xdrlib.py b/Lib/xdrlib.py deleted file mode 100644 index d6e1aeb5272..00000000000 --- a/Lib/xdrlib.py +++ /dev/null @@ -1,241 +0,0 @@ -"""Implements (a subset of) Sun XDR -- eXternal Data Representation. - -See: RFC 1014 - -""" - -import struct -from io import BytesIO -from functools import wraps - -__all__ = ["Error", "Packer", "Unpacker", "ConversionError"] - -# exceptions -class Error(Exception): - """Exception class for this module. Use: - - except xdrlib.Error as var: - # var has the Error instance for the exception - - Public ivars: - msg -- contains the message - - """ - def __init__(self, msg): - self.msg = msg - def __repr__(self): - return repr(self.msg) - def __str__(self): - return str(self.msg) - - -class ConversionError(Error): - pass - -def raise_conversion_error(function): - """ Wrap any raised struct.errors in a ConversionError. """ - - @wraps(function) - def result(self, value): - try: - return function(self, value) - except struct.error as e: - raise ConversionError(e.args[0]) from None - return result - - -class Packer: - """Pack various data representations into a buffer.""" - - def __init__(self): - self.reset() - - def reset(self): - self.__buf = BytesIO() - - def get_buffer(self): - return self.__buf.getvalue() - # backwards compatibility - get_buf = get_buffer - - @raise_conversion_error - def pack_uint(self, x): - self.__buf.write(struct.pack('>L', x)) - - @raise_conversion_error - def pack_int(self, x): - self.__buf.write(struct.pack('>l', x)) - - pack_enum = pack_int - - def pack_bool(self, x): - if x: self.__buf.write(b'\0\0\0\1') - else: self.__buf.write(b'\0\0\0\0') - - def pack_uhyper(self, x): - try: - self.pack_uint(x>>32 & 0xffffffff) - except (TypeError, struct.error) as e: - raise ConversionError(e.args[0]) from None - try: - self.pack_uint(x & 0xffffffff) - except (TypeError, struct.error) as e: - raise ConversionError(e.args[0]) from None - - pack_hyper = pack_uhyper - - @raise_conversion_error - def pack_float(self, x): - self.__buf.write(struct.pack('>f', x)) - - @raise_conversion_error - def pack_double(self, x): - self.__buf.write(struct.pack('>d', x)) - - def pack_fstring(self, n, s): - if n < 0: - raise ValueError('fstring size must be nonnegative') - data = s[:n] - n = ((n+3)//4)*4 - data = data + (n - len(data)) * b'\0' - self.__buf.write(data) - - pack_fopaque = pack_fstring - - def pack_string(self, s): - n = len(s) - self.pack_uint(n) - self.pack_fstring(n, s) - - pack_opaque = pack_string - pack_bytes = pack_string - - def pack_list(self, list, pack_item): - for item in list: - self.pack_uint(1) - pack_item(item) - self.pack_uint(0) - - def pack_farray(self, n, list, pack_item): - if len(list) != n: - raise ValueError('wrong array size') - for item in list: - pack_item(item) - - def pack_array(self, list, pack_item): - n = len(list) - self.pack_uint(n) - self.pack_farray(n, list, pack_item) - - - -class Unpacker: - """Unpacks various data representations from the given buffer.""" - - def __init__(self, data): - self.reset(data) - - def reset(self, data): - self.__buf = data - self.__pos = 0 - - def get_position(self): - return self.__pos - - def set_position(self, position): - self.__pos = position - - def get_buffer(self): - return self.__buf - - def done(self): - if self.__pos < len(self.__buf): - raise Error('unextracted data remains') - - def unpack_uint(self): - i = self.__pos - self.__pos = j = i+4 - data = self.__buf[i:j] - if len(data) < 4: - raise EOFError - return struct.unpack('>L', data)[0] - - def unpack_int(self): - i = self.__pos - self.__pos = j = i+4 - data = self.__buf[i:j] - if len(data) < 4: - raise EOFError - return struct.unpack('>l', data)[0] - - unpack_enum = unpack_int - - def unpack_bool(self): - return bool(self.unpack_int()) - - def unpack_uhyper(self): - hi = self.unpack_uint() - lo = self.unpack_uint() - return int(hi)<<32 | lo - - def unpack_hyper(self): - x = self.unpack_uhyper() - if x >= 0x8000000000000000: - x = x - 0x10000000000000000 - return x - - def unpack_float(self): - i = self.__pos - self.__pos = j = i+4 - data = self.__buf[i:j] - if len(data) < 4: - raise EOFError - return struct.unpack('>f', data)[0] - - def unpack_double(self): - i = self.__pos - self.__pos = j = i+8 - data = self.__buf[i:j] - if len(data) < 8: - raise EOFError - return struct.unpack('>d', data)[0] - - def unpack_fstring(self, n): - if n < 0: - raise ValueError('fstring size must be nonnegative') - i = self.__pos - j = i + (n+3)//4*4 - if j > len(self.__buf): - raise EOFError - self.__pos = j - return self.__buf[i:i+n] - - unpack_fopaque = unpack_fstring - - def unpack_string(self): - n = self.unpack_uint() - return self.unpack_fstring(n) - - unpack_opaque = unpack_string - unpack_bytes = unpack_string - - def unpack_list(self, unpack_item): - list = [] - while 1: - x = self.unpack_uint() - if x == 0: break - if x != 1: - raise ConversionError('0 or 1 expected, got %r' % (x,)) - item = unpack_item() - list.append(item) - return list - - def unpack_farray(self, n, unpack_item): - list = [] - for i in range(n): - list.append(unpack_item()) - return list - - def unpack_array(self, unpack_item): - n = self.unpack_uint() - return self.unpack_farray(n, unpack_item) diff --git a/Lib/xml/dom/__init__.py b/Lib/xml/dom/__init__.py index 97cf9a64299..dd7fb996afd 100644 --- a/Lib/xml/dom/__init__.py +++ b/Lib/xml/dom/__init__.py @@ -137,4 +137,4 @@ class UserDataHandler: EMPTY_NAMESPACE = None EMPTY_PREFIX = None -from .domreg import getDOMImplementation, registerDOMImplementation +from .domreg import getDOMImplementation, registerDOMImplementation # noqa: F401 diff --git a/Lib/xml/dom/expatbuilder.py b/Lib/xml/dom/expatbuilder.py index 199c22d0af3..7dd667bf3fb 100644 --- a/Lib/xml/dom/expatbuilder.py +++ b/Lib/xml/dom/expatbuilder.py @@ -200,10 +200,7 @@ def parseFile(self, file): parser = self.getParser() first_buffer = True try: - while 1: - buffer = file.read(16*1024) - if not buffer: - break + while buffer := file.read(16*1024): parser.Parse(buffer, False) if first_buffer and self.document.documentElement: self._setup_subset(buffer) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index d09ef5e7d03..16b33b90184 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -292,20 +292,29 @@ def _append_child(self, node): childNodes.append(node) node.parentNode = self -def _in_document(node): - # return True iff node is part of a document tree - while node is not None: - if node.nodeType == Node.DOCUMENT_NODE: - return True - node = node.parentNode - return False -def _write_data(writer, data): +def _write_data(writer, text, attr): "Writes datachars to writer." - if data: - data = data.replace("&", "&amp;").replace("<", "&lt;"). \ - replace("\"", "&quot;").replace(">", "&gt;") - writer.write(data) + if not text: + return + # See the comments in ElementTree.py for behavior and + # implementation details. + if "&" in text: + text = text.replace("&", "&amp;") + if "<" in text: + text = text.replace("<", "&lt;") + if ">" in text: + text = text.replace(">", "&gt;") + if attr: + if '"' in text: + text = text.replace('"', "&quot;") + if "\r" in text: + text = text.replace("\r", "&#13;") + if "\n" in text: + text = text.replace("\n", "&#10;") + if "\t" in text: + text = text.replace("\t", "&#9;") + writer.write(text) def _get_elements_by_tagName_helper(parent, name, rc): for node in parent.childNodes: @@ -355,9 +364,12 @@ class Attr(Node): def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, prefix=None): self.ownerElement = None + self.ownerDocument = None self._name = qName self.namespaceURI = namespaceURI self._prefix = prefix + if localName is not None: + self._localName = localName self.childNodes = NodeList() # Add the single child node that represents the value of the attr @@ -678,6 +690,7 @@ class Element(Node): def __init__(self, tagName, namespaceURI=EMPTY_NAMESPACE, prefix=None, localName=None): + self.ownerDocument = None self.parentNode = None self.tagName = self.nodeName = tagName self.prefix = prefix @@ -881,7 +894,7 @@ def writexml(self, writer, indent="", addindent="", newl=""): for a_name in attrs.keys(): writer.write(" %s=\"" % a_name) - _write_data(writer, attrs[a_name].value) + _write_data(writer, attrs[a_name].value, True) writer.write("\"") if self.childNodes: writer.write(">") @@ -1110,7 +1123,7 @@ def splitText(self, offset): return newText def writexml(self, writer, indent="", addindent="", newl=""): - _write_data(writer, "%s%s%s" % (indent, self.data, newl)) + _write_data(writer, "%s%s%s" % (indent, self.data, newl), False) # DOM Level 3 (WD 9 April 2002) @@ -1537,7 +1550,7 @@ def _clear_id_cache(node): if node.nodeType == Node.DOCUMENT_NODE: node._id_cache.clear() node._id_search_stack = None - elif _in_document(node): + elif node.ownerDocument: node.ownerDocument._id_cache.clear() node.ownerDocument._id_search_stack= None diff --git a/Lib/xml/dom/xmlbuilder.py b/Lib/xml/dom/xmlbuilder.py index 8a200263497..a8852625a2f 100644 --- a/Lib/xml/dom/xmlbuilder.py +++ b/Lib/xml/dom/xmlbuilder.py @@ -189,7 +189,7 @@ def parse(self, input): options.filter = self.filter options.errorHandler = self.errorHandler fp = input.byteStream - if fp is None and options.systemId: + if fp is None and input.systemId: import urllib.request fp = urllib.request.urlopen(input.systemId) return self._parse_bytestream(fp, options) @@ -247,10 +247,12 @@ def _create_opener(self): def _guess_media_encoding(self, source): info = source.byteStream.info() - if "Content-Type" in info: - for param in info.getplist(): - if param.startswith("charset="): - return param.split("=", 1)[1].lower() + # import email.message + # assert isinstance(info, email.message.Message) + charset = info.get_param('charset') + if charset is not None: + return charset.lower() + return None class DOMInputSource(object): diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 40a9b222924..986e6c3bbe9 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -79,8 +79,8 @@ class LimitedRecursiveIncludeError(FatalIncludeError): # @param parse Parse mode. Either "xml" or "text". # @param encoding Optional text encoding (UTF-8 by default for "text"). # @return The expanded resource. If the parse mode is "xml", this -# is an ElementTree instance. If the parse mode is "text", this -# is a Unicode string. If the loader fails, it can return None +# is an Element instance. If the parse mode is "text", this +# is a string. If the loader fails, it can return None # or raise an OSError exception. # @throws OSError If the loader fails to load the resource. @@ -98,7 +98,7 @@ def default_loader(href, parse, encoding=None): ## # Expand XInclude directives. # -# @param elem Root element. +# @param elem Root Element or any ElementTree of a tree to be expanded # @param loader Optional resource loader. If omitted, it defaults # to {@link default_loader}. If given, it should be a callable # that implements the same interface as <b>default_loader</b>. @@ -106,12 +106,13 @@ def default_loader(href, parse, encoding=None): # relative include file references. # @param max_depth The maximum number of recursive inclusions. # Limited to reduce the risk of malicious content explosion. -# Pass a negative value to disable the limitation. +# Pass None to disable the limitation. # @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. # @throws FatalIncludeError If the function fails to include a given # resource, or if the tree contains malformed XInclude elements. -# @throws IOError If the function fails to load a given resource. -# @returns the node or its replacement if it was an XInclude node +# @throws OSError If the function fails to load a given resource. +# @throws ValueError If negative {@link max_depth} is passed. +# @returns None. Modifies tree pointed by {@link elem} def include(elem, loader=None, base_url=None, max_depth=DEFAULT_MAX_INCLUSION_DEPTH): diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index cd3c354d081..dc6bd28c031 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -416,6 +416,8 @@ def findall(elem, path, namespaces=None): def findtext(elem, path, default=None, namespaces=None): try: elem = next(iterfind(elem, path, namespaces)) - return elem.text or "" + if elem.text is None: + return "" + return elem.text except StopIteration: return default diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 2503d9ee76a..dafe5b1b8a0 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -99,6 +99,7 @@ import collections import collections.abc import contextlib +import weakref from . import ElementPath @@ -188,19 +189,6 @@ def makeelement(self, tag, attrib): """ return self.__class__(tag, attrib) - def copy(self): - """Return copy of current element. - - This creates a shallow copy. Subelements will be shared with the - original tree. - - """ - warnings.warn( - "elem.copy() is deprecated. Use copy.copy(elem) instead.", - DeprecationWarning - ) - return self.__copy__() - def __copy__(self): elem = self.makeelement(self.tag, self.attrib) elem.text = self.text @@ -213,9 +201,10 @@ def __len__(self): def __bool__(self): warnings.warn( - "The behavior of this method will change in future versions. " + "Testing an element's truth value will always return True in " + "future versions. " "Use specific 'len(elem)' or 'elem is not None' test instead.", - FutureWarning, stacklevel=2 + DeprecationWarning, stacklevel=2 ) return len(self._children) != 0 # emulate old behaviour, for now @@ -278,7 +267,11 @@ def remove(self, subelement): """ # assert iselement(element) - self._children.remove(subelement) + try: + self._children.remove(subelement) + except ValueError: + # to align the error message with the C implementation + raise ValueError("Element.remove(x): element not found") from None def find(self, path, namespaces=None): """Find first matching element by tag name or path. @@ -534,7 +527,9 @@ class ElementTree: """ def __init__(self, element=None, file=None): - # assert element is None or iselement(element) + if element is not None and not iselement(element): + raise TypeError('expected an Element, not %s' % + type(element).__name__) self._root = element # first node if file: self.parse(file) @@ -550,7 +545,9 @@ def _setroot(self, element): with the given element. Use with care! """ - # assert iselement(element) + if not iselement(element): + raise TypeError('expected an Element, not %s' + % type(element).__name__) self._root = element def parse(self, source, parser=None): @@ -579,10 +576,7 @@ def parse(self, source, parser=None): # it with chunks. self._root = parser._parse_whole(source) return self._root - while True: - data = source.read(65536) - if not data: - break + while data := source.read(65536): parser.feed(data) self._root = parser.close() return self._root @@ -719,6 +713,8 @@ def write(self, file_or_filename, of start/end tags """ + if self._root is None: + raise TypeError('ElementTree not initialized') if not method: method = "xml" elif method not in _serialize: @@ -911,13 +907,9 @@ def _serialize_xml(write, elem, qnames, namespaces, if elem.tail: write(_escape_cdata(elem.tail)) -HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr", - "img", "input", "isindex", "link", "meta", "param") - -try: - HTML_EMPTY = set(HTML_EMPTY) -except NameError: - pass +HTML_EMPTY = {"area", "base", "basefont", "br", "col", "embed", "frame", "hr", + "img", "input", "isindex", "link", "meta", "param", "source", + "track", "wbr"} def _serialize_html(write, elem, qnames, namespaces, **kwargs): tag = elem.tag @@ -1242,13 +1234,14 @@ def iterparse(source, events=None, parser=None): # parser argument of iterparse is removed, this can be killed. pullparser = XMLPullParser(events=events, _parser=parser) - def iterator(source): + if not hasattr(source, "read"): + source = open(source, "rb") + close_source = True + else: close_source = False + + def iterator(source): try: - if not hasattr(source, "read"): - source = open(source, "rb") - close_source = True - yield None while True: yield from pullparser.read_events() # load event buffer @@ -1258,18 +1251,30 @@ def iterator(source): pullparser.feed(data) root = pullparser._close_and_return_root() yield from pullparser.read_events() - it.root = root + it = wr() + if it is not None: + it.root = root finally: if close_source: source.close() + gen = iterator(source) class IterParseIterator(collections.abc.Iterator): - __next__ = iterator(source).__next__ + __next__ = gen.__next__ + def close(self): + if close_source: + source.close() + gen.close() + + def __del__(self): + # TODO: Emit a ResourceWarning if it was not explicitly closed. + # (When the close() method will be supported in all maintained Python versions.) + if close_source: + source.close() + it = IterParseIterator() it.root = None - del iterator, IterParseIterator - - next(it) + wr = weakref.ref(it) return it @@ -1325,6 +1330,11 @@ def read_events(self): else: yield event + def flush(self): + if self._parser is None: + raise ValueError("flush() called after end of stream") + self._parser.flush() + def XML(text, parser=None): """Parse XML document from string constant. @@ -1731,6 +1741,15 @@ def close(self): del self.parser, self._parser del self.target, self._target + def flush(self): + was_enabled = self.parser.GetReparseDeferralEnabled() + try: + self.parser.SetReparseDeferralEnabled(False) + self.parser.Parse(b"", False) + except self._error as v: + self._raiseerror(v) + finally: + self.parser.SetReparseDeferralEnabled(was_enabled) # -------------------------------------------------------------------- # C14N 2.0 diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index 17b75879eba..fe4582c6f8b 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -21,9 +21,9 @@ from .xmlreader import InputSource from .handler import ContentHandler, ErrorHandler -from ._exceptions import SAXException, SAXNotRecognizedException, \ - SAXParseException, SAXNotSupportedException, \ - SAXReaderNotAvailable +from ._exceptions import (SAXException, SAXNotRecognizedException, + SAXParseException, SAXNotSupportedException, + SAXReaderNotAvailable) def parse(source, handler, errorHandler=ErrorHandler()): @@ -55,16 +55,12 @@ def parseString(string, handler, errorHandler=ErrorHandler()): # tell modulefinder that importing sax potentially imports expatreader _false = 0 if _false: - import xml.sax.expatreader + import xml.sax.expatreader # noqa: F401 import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: default_parser_list = os.environ["PY_SAX_PARSER"].split(",") -del os - -_key = "python.xml.sax.parser" -if sys.platform[:4] == "java" and sys.registry.containsKey(_key): - default_parser_list = sys.registry.getProperty(_key).split(",") +del os, sys def make_parser(parser_list=()): @@ -93,15 +89,12 @@ def make_parser(parser_list=()): # --- Internal utility methods used by make_parser -if sys.platform[ : 4] == "java": - def _create_parser(parser_name): - from org.python.core import imp - drv_module = imp.importName(parser_name, 0, globals()) - return drv_module.create_parser() +def _create_parser(parser_name): + drv_module = __import__(parser_name,{},{},['create_parser']) + return drv_module.create_parser() -else: - def _create_parser(parser_name): - drv_module = __import__(parser_name,{},{},['create_parser']) - return drv_module.create_parser() -del sys +__all__ = ['ContentHandler', 'ErrorHandler', 'InputSource', 'SAXException', + 'SAXNotRecognizedException', 'SAXNotSupportedException', + 'SAXParseException', 'SAXReaderNotAvailable', + 'default_parser_list', 'make_parser', 'parse', 'parseString'] diff --git a/Lib/xml/sax/_exceptions.py b/Lib/xml/sax/_exceptions.py index a9b2ba35c6a..f292dc3a8e5 100644 --- a/Lib/xml/sax/_exceptions.py +++ b/Lib/xml/sax/_exceptions.py @@ -1,8 +1,4 @@ """Different kinds of SAX Exceptions""" -import sys -if sys.platform[:4] == "java": - from java.lang import Exception -del sys # ===== SAXEXCEPTION ===== diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index e334ac9fea0..ba3c1e98517 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -12,12 +12,6 @@ from xml.sax.handler import feature_string_interning from xml.sax.handler import property_xml_string, property_interning_dict -# xml.parsers.expat does not raise ImportError in Jython -import sys -if sys.platform[:4] == "java": - raise SAXReaderNotAvailable("expat not available in Java", None) -del sys - try: from xml.parsers import expat except ImportError: @@ -220,6 +214,20 @@ def feed(self, data, isFinal=False): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def flush(self): + if self._parser is None: + return + + was_enabled = self._parser.GetReparseDeferralEnabled() + try: + self._parser.SetReparseDeferralEnabled(False) + self._parser.Parse(b"", False) + except expat.error as e: + exc = SAXParseException(expat.ErrorString(e.code), e, self) + self._err_handler.fatalError(exc) + finally: + self._parser.SetReparseDeferralEnabled(was_enabled) + def _close_source(self): source = self._source try: diff --git a/Lib/xml/sax/handler.py b/Lib/xml/sax/handler.py index e8d417e5194..3183c3fe96d 100644 --- a/Lib/xml/sax/handler.py +++ b/Lib/xml/sax/handler.py @@ -371,7 +371,7 @@ def startDTD(self, name, public_id, system_id): name is the name of the document element type, public_id the public identifier of the DTD (or None if none were supplied) - and system_id the system identfier of the external subset (or + and system_id the system identifier of the external subset (or None if none were supplied).""" def endDTD(self): diff --git a/Lib/xml/sax/xmlreader.py b/Lib/xml/sax/xmlreader.py index 716f2284041..e906121d23b 100644 --- a/Lib/xml/sax/xmlreader.py +++ b/Lib/xml/sax/xmlreader.py @@ -120,10 +120,8 @@ def parse(self, source): file = source.getCharacterStream() if file is None: file = source.getByteStream() - buffer = file.read(self._bufsize) - while buffer: + while buffer := file.read(self._bufsize): self.feed(buffer) - buffer = file.read(self._bufsize) self.close() def feed(self, data): diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index 121e44023c1..f441376d09c 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -135,8 +135,7 @@ from decimal import Decimal import http.client import urllib.parse -# XXX RUSTPYTHON TODO: pyexpat -# from xml.parsers import expat +from xml.parsers import expat import errno from io import BytesIO try: @@ -246,41 +245,15 @@ def __repr__(self): ## # Backwards compatibility - boolean = Boolean = bool -## -# Wrapper for XML-RPC DateTime values. This converts a time value to -# the format used by XML-RPC. -# <p> -# The value can be given as a datetime object, as a string in the -# format "yyyymmddThh:mm:ss", as a 9-item time tuple (as returned by -# time.localtime()), or an integer value (as returned by time.time()). -# The wrapper uses time.localtime() to convert an integer to a time -# tuple. -# -# @param value The time, given as a datetime object, an ISO 8601 string, -# a time tuple, or an integer time value. - -# Issue #13305: different format codes across platforms -_day0 = datetime(1, 1, 1) -def _try(fmt): - try: - return _day0.strftime(fmt) == '0001' - except ValueError: - return False -if _try('%Y'): # Mac OS X - def _iso8601_format(value): - return value.strftime("%Y%m%dT%H:%M:%S") -elif _try('%4Y'): # Linux - def _iso8601_format(value): - return value.strftime("%4Y%m%dT%H:%M:%S") -else: - def _iso8601_format(value): - return value.strftime("%Y%m%dT%H:%M:%S").zfill(17) -del _day0 -del _try +def _iso8601_format(value): + if value.tzinfo is not None: + # XML-RPC only uses the naive portion of the datetime + value = value.replace(tzinfo=None) + # XML-RPC doesn't use '-' separator in the date part + return value.isoformat(timespec='seconds').replace('-', '') def _strftime(value): @@ -851,9 +824,9 @@ def __init__(self, results): def __getitem__(self, i): item = self.results[i] - if type(item) == type({}): + if isinstance(item, dict): raise Fault(item['faultCode'], item['faultString']) - elif type(item) == type([]): + elif isinstance(item, list): return item[0] else: raise ValueError("unexpected type in multicall result") @@ -1340,10 +1313,7 @@ def parse_response(self, response): p, u = self.getparser() - while 1: - data = stream.read(1024) - if not data: - break + while data := stream.read(1024): if self.verbose: print("body:", repr(data)) p.feed(data) diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 69a260f5b12..3e6871157d0 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -239,7 +239,7 @@ def register_multicall_functions(self): see http://www.xmlrpc.com/discuss/msgReader$1208""" - self.funcs.update({'system.multicall' : self.system_multicall}) + self.funcs['system.multicall'] = self.system_multicall def _marshaled_dispatch(self, data, dispatch_method = None, path = None): """Dispatches an XML-RPC method from marshalled (XML) data. @@ -268,17 +268,11 @@ def _marshaled_dispatch(self, data, dispatch_method = None, path = None): except Fault as fault: response = dumps(fault, allow_none=self.allow_none, encoding=self.encoding) - except: - # report exception back to server - exc_type, exc_value, exc_tb = sys.exc_info() - try: - response = dumps( - Fault(1, "%s:%s" % (exc_type, exc_value)), - encoding=self.encoding, allow_none=self.allow_none, - ) - finally: - # Break reference cycle - exc_type = exc_value = exc_tb = None + except BaseException as exc: + response = dumps( + Fault(1, "%s:%s" % (type(exc), exc)), + encoding=self.encoding, allow_none=self.allow_none, + ) return response.encode(self.encoding, 'xmlcharrefreplace') @@ -368,16 +362,11 @@ def system_multicall(self, call_list): {'faultCode' : fault.faultCode, 'faultString' : fault.faultString} ) - except: - exc_type, exc_value, exc_tb = sys.exc_info() - try: - results.append( - {'faultCode' : 1, - 'faultString' : "%s:%s" % (exc_type, exc_value)} - ) - finally: - # Break reference cycle - exc_type = exc_value = exc_tb = None + except BaseException as exc: + results.append( + {'faultCode' : 1, + 'faultString' : "%s:%s" % (type(exc), exc)} + ) return results def _dispatch(self, method, params): @@ -440,7 +429,7 @@ class SimpleXMLRPCRequestHandler(BaseHTTPRequestHandler): # Class attribute listing the accessible path components; # paths not on this list will result in a 404 error. - rpc_paths = ('/', '/RPC2') + rpc_paths = ('/', '/RPC2', '/pydoc.css') #if not None, encode responses larger than this, if possible encode_threshold = 1400 #a common MTU @@ -589,6 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True + allow_reuse_port = False # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception @@ -634,19 +624,14 @@ def _marshaled_dispatch(self, data, dispatch_method = None, path = None): try: response = self.dispatchers[path]._marshaled_dispatch( data, dispatch_method, path) - except: + except BaseException as exc: # report low level exception back to server # (each dispatcher should have handled their own # exceptions) - exc_type, exc_value = sys.exc_info()[:2] - try: - response = dumps( - Fault(1, "%s:%s" % (exc_type, exc_value)), - encoding=self.encoding, allow_none=self.allow_none) - response = response.encode(self.encoding, 'xmlcharrefreplace') - finally: - # Break reference cycle - exc_type = exc_value = None + response = dumps( + Fault(1, "%s:%s" % (type(exc), exc)), + encoding=self.encoding, allow_none=self.allow_none) + response = response.encode(self.encoding, 'xmlcharrefreplace') return response class CGIXMLRPCRequestHandler(SimpleXMLRPCDispatcher): @@ -736,9 +721,7 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): r'RFC[- ]?(\d+)|' r'PEP[- ]?(\d+)|' r'(self\.)?((?:\w|\.)+))\b') - while 1: - match = pattern.search(text, here) - if not match: break + while match := pattern.search(text, here): start, end = match.span() results.append(escape(text[here:start])) @@ -747,10 +730,10 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): url = escape(all).replace('"', '&quot;') results.append('<a href="%s">%s</a>' % (url, url)) elif rfc: - url = 'http://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) + url = 'https://www.rfc-editor.org/rfc/rfc%d.txt' % int(rfc) results.append('<a href="%s">%s</a>' % (url, escape(all))) elif pep: - url = 'https://www.python.org/dev/peps/pep-%04d/' % int(pep) + url = 'https://peps.python.org/pep-%04d/' % int(pep) results.append('<a href="%s">%s</a>' % (url, escape(all))) elif text[end:end+1] == '(': results.append(self.namelink(name, methods, funcs, classes)) @@ -801,7 +784,7 @@ def docserver(self, server_name, package_documentation, methods): server_name = self.escape(server_name) head = '<big><big><strong>%s</strong></big></big>' % server_name - result = self.heading(head, '#ffffff', '#7799ee') + result = self.heading(head) doc = self.markup(package_documentation, self.preformat, fdict) doc = doc and '<tt>%s</tt>' % doc @@ -812,10 +795,25 @@ def docserver(self, server_name, package_documentation, methods): for key, value in method_items: contents.append(self.docroutine(value, key, funcs=fdict)) result = result + self.bigsection( - 'Methods', '#ffffff', '#eeaa77', ''.join(contents)) + 'Methods', 'functions', ''.join(contents)) return result + + def page(self, title, contents): + """Format an HTML page.""" + css_path = "/pydoc.css" + css_link = ( + '<link rel="stylesheet" type="text/css" href="%s">' % + css_path) + return '''\ +<!DOCTYPE> +<html lang="en"> +<head> +<meta charset="utf-8"> +<title>Python: %s</title> +%s</head><body>%s</body></html>''' % (title, css_link, contents) + class XMLRPCDocGenerator: """Generates documentation for an XML-RPC server. @@ -907,6 +905,12 @@ class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): for documentation. """ + def _get_css(self, url): + path_here = os.path.dirname(os.path.realpath(__file__)) + css_path = os.path.join(path_here, "..", "pydoc_data", "_pydoc.css") + with open(css_path, mode="rb") as fp: + return fp.read() + def do_GET(self): """Handles the HTTP GET request. @@ -918,9 +922,15 @@ def do_GET(self): self.report_404() return - response = self.server.generate_html_documentation().encode('utf-8') + if self.path.endswith('.css'): + content_type = 'text/css' + response = self._get_css(self.path) + else: + content_type = 'text/html' + response = self.server.generate_html_documentation().encode('utf-8') + self.send_response(200) - self.send_header("Content-type", "text/html") + self.send_header('Content-Type', '%s; charset=UTF-8' % content_type) self.send_header("Content-length", str(len(response))) self.end_headers() self.wfile.write(response) diff --git a/Lib/zipapp.py b/Lib/zipapp.py index ce77632516c..7a4ef96ea0f 100644 --- a/Lib/zipapp.py +++ b/Lib/zipapp.py @@ -131,15 +131,40 @@ def create_archive(source, target=None, interpreter=None, main=None, elif not hasattr(target, 'write'): target = pathlib.Path(target) + # Create the list of files to add to the archive now, in case + # the target is being created in the source directory - we + # don't want the target being added to itself + files_to_add = {} + for path in sorted(source.rglob('*')): + relative_path = path.relative_to(source) + if filter is None or filter(relative_path): + files_to_add[path] = relative_path + + # The target cannot be in the list of files to add. If it were, we'd + # end up overwriting the source file and writing the archive into + # itself, which is an error. We therefore check for that case and + # provide a helpful message for the user. + + # Note that we only do a simple path equality check. This won't + # catch every case, but it will catch the common case where the + # source is the CWD and the target is a file in the CWD. More + # thorough checks don't provide enough value to justify the extra + # cost. + + # If target is a file-like object, it will simply fail to compare + # equal to any of the entries in files_to_add, so there's no need + # to add a special check for that. + if target in files_to_add: + raise ZipAppError( + f"The target archive {target} overwrites one of the source files.") + with _maybe_open(target, 'wb') as fd: _write_file_prefix(fd, interpreter) compression = (zipfile.ZIP_DEFLATED if compressed else zipfile.ZIP_STORED) with zipfile.ZipFile(fd, 'w', compression=compression) as z: - for child in source.rglob('*'): - arcname = child.relative_to(source) - if filter is None or filter(arcname): - z.write(child, arcname.as_posix()) + for path, relative_path in files_to_add.items(): + z.write(path, relative_path.as_posix()) if main_py: z.writestr('__main__.py', main_py.encode('utf-8')) @@ -162,7 +187,7 @@ def main(args=None): """ import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(color=True) parser.add_argument('--output', '-o', default=None, help="The name of the output archive. " "Required if SOURCE is an archive.") diff --git a/Lib/zipfile.py b/Lib/zipfile.py deleted file mode 100644 index ef1eb47f9f1..00000000000 --- a/Lib/zipfile.py +++ /dev/null @@ -1,2483 +0,0 @@ -""" -Read and write ZIP files. - -XXX references to utf-8 need further investigation. -""" -import binascii -import importlib.util -import io -import itertools -try: - import os -except ImportError: - import _dummy_os as os -import posixpath -try: - import shutil -except ImportError: - pass -import stat -import struct -import sys -try: - import threading -except ImportError: - import dummy_threading as threading -import time -import contextlib -import pathlib - -try: - import zlib # We may need its compression method - crc32 = zlib.crc32 -except ImportError: - zlib = None - crc32 = binascii.crc32 - -try: - import bz2 # We may need its compression method -except ImportError: - bz2 = None - -try: - import lzma # We may need its compression method -except ImportError: - lzma = None - -__all__ = ["BadZipFile", "BadZipfile", "error", - "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", - "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile", - "Path"] - -class BadZipFile(Exception): - pass - - -class LargeZipFile(Exception): - """ - Raised when writing a zipfile, the zipfile requires ZIP64 extensions - and those extensions are disabled. - """ - -error = BadZipfile = BadZipFile # Pre-3.2 compatibility names - - -ZIP64_LIMIT = (1 << 31) - 1 -ZIP_FILECOUNT_LIMIT = (1 << 16) - 1 -ZIP_MAX_COMMENT = (1 << 16) - 1 - -# constants for Zip file compression methods -ZIP_STORED = 0 -ZIP_DEFLATED = 8 -ZIP_BZIP2 = 12 -ZIP_LZMA = 14 -# Other ZIP compression methods not supported - -DEFAULT_VERSION = 20 -ZIP64_VERSION = 45 -BZIP2_VERSION = 46 -LZMA_VERSION = 63 -# we recognize (but not necessarily support) all features up to that version -MAX_EXTRACT_VERSION = 63 - -# Below are some formats and associated data for reading/writing headers using -# the struct module. The names and structures of headers/records are those used -# in the PKWARE description of the ZIP file format: -# http://www.pkware.com/documents/casestudies/APPNOTE.TXT -# (URL valid as of January 2008) - -# The "end of central directory" structure, magic number, size, and indices -# (section V.I in the format document) -structEndArchive = b"<4s4H2LH" -stringEndArchive = b"PK\005\006" -sizeEndCentDir = struct.calcsize(structEndArchive) - -_ECD_SIGNATURE = 0 -_ECD_DISK_NUMBER = 1 -_ECD_DISK_START = 2 -_ECD_ENTRIES_THIS_DISK = 3 -_ECD_ENTRIES_TOTAL = 4 -_ECD_SIZE = 5 -_ECD_OFFSET = 6 -_ECD_COMMENT_SIZE = 7 -# These last two indices are not part of the structure as defined in the -# spec, but they are used internally by this module as a convenience -_ECD_COMMENT = 8 -_ECD_LOCATION = 9 - -# The "central directory" structure, magic number, size, and indices -# of entries in the structure (section V.F in the format document) -structCentralDir = "<4s4B4HL2L5H2L" -stringCentralDir = b"PK\001\002" -sizeCentralDir = struct.calcsize(structCentralDir) - -# indexes of entries in the central directory structure -_CD_SIGNATURE = 0 -_CD_CREATE_VERSION = 1 -_CD_CREATE_SYSTEM = 2 -_CD_EXTRACT_VERSION = 3 -_CD_EXTRACT_SYSTEM = 4 -_CD_FLAG_BITS = 5 -_CD_COMPRESS_TYPE = 6 -_CD_TIME = 7 -_CD_DATE = 8 -_CD_CRC = 9 -_CD_COMPRESSED_SIZE = 10 -_CD_UNCOMPRESSED_SIZE = 11 -_CD_FILENAME_LENGTH = 12 -_CD_EXTRA_FIELD_LENGTH = 13 -_CD_COMMENT_LENGTH = 14 -_CD_DISK_NUMBER_START = 15 -_CD_INTERNAL_FILE_ATTRIBUTES = 16 -_CD_EXTERNAL_FILE_ATTRIBUTES = 17 -_CD_LOCAL_HEADER_OFFSET = 18 - -# The "local file header" structure, magic number, size, and indices -# (section V.A in the format document) -structFileHeader = "<4s2B4HL2L2H" -stringFileHeader = b"PK\003\004" -sizeFileHeader = struct.calcsize(structFileHeader) - -_FH_SIGNATURE = 0 -_FH_EXTRACT_VERSION = 1 -_FH_EXTRACT_SYSTEM = 2 -_FH_GENERAL_PURPOSE_FLAG_BITS = 3 -_FH_COMPRESSION_METHOD = 4 -_FH_LAST_MOD_TIME = 5 -_FH_LAST_MOD_DATE = 6 -_FH_CRC = 7 -_FH_COMPRESSED_SIZE = 8 -_FH_UNCOMPRESSED_SIZE = 9 -_FH_FILENAME_LENGTH = 10 -_FH_EXTRA_FIELD_LENGTH = 11 - -# The "Zip64 end of central directory locator" structure, magic number, and size -structEndArchive64Locator = "<4sLQL" -stringEndArchive64Locator = b"PK\x06\x07" -sizeEndCentDir64Locator = struct.calcsize(structEndArchive64Locator) - -# The "Zip64 end of central directory" record, magic number, size, and indices -# (section V.G in the format document) -structEndArchive64 = "<4sQ2H2L4Q" -stringEndArchive64 = b"PK\x06\x06" -sizeEndCentDir64 = struct.calcsize(structEndArchive64) - -_CD64_SIGNATURE = 0 -_CD64_DIRECTORY_RECSIZE = 1 -_CD64_CREATE_VERSION = 2 -_CD64_EXTRACT_VERSION = 3 -_CD64_DISK_NUMBER = 4 -_CD64_DISK_NUMBER_START = 5 -_CD64_NUMBER_ENTRIES_THIS_DISK = 6 -_CD64_NUMBER_ENTRIES_TOTAL = 7 -_CD64_DIRECTORY_SIZE = 8 -_CD64_OFFSET_START_CENTDIR = 9 - -_DD_SIGNATURE = 0x08074b50 - -_EXTRA_FIELD_STRUCT = struct.Struct('<HH') - -def _strip_extra(extra, xids): - # Remove Extra Fields with specified IDs. - unpack = _EXTRA_FIELD_STRUCT.unpack - modified = False - buffer = [] - start = i = 0 - while i + 4 <= len(extra): - xid, xlen = unpack(extra[i : i + 4]) - j = i + 4 + xlen - if xid in xids: - if i != start: - buffer.append(extra[start : i]) - start = j - modified = True - i = j - if not modified: - return extra - return b''.join(buffer) - -def _check_zipfile(fp): - try: - if _EndRecData(fp): - return True # file has correct magic number - except OSError: - pass - return False - -def is_zipfile(filename): - """Quickly see if a file is a ZIP file by checking the magic number. - - The filename argument may be a file or file-like object too. - """ - result = False - try: - if hasattr(filename, "read"): - result = _check_zipfile(fp=filename) - else: - with open(filename, "rb") as fp: - result = _check_zipfile(fp) - except OSError: - pass - return result - -def _EndRecData64(fpin, offset, endrec): - """ - Read the ZIP64 end-of-archive records and use that to update endrec - """ - try: - fpin.seek(offset - sizeEndCentDir64Locator, 2) - except OSError: - # If the seek fails, the file is not large enough to contain a ZIP64 - # end-of-archive record, so just return the end record we were given. - return endrec - - data = fpin.read(sizeEndCentDir64Locator) - if len(data) != sizeEndCentDir64Locator: - return endrec - sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) - if sig != stringEndArchive64Locator: - return endrec - - if diskno != 0 or disks > 1: - raise BadZipFile("zipfiles that span multiple disks are not supported") - - # Assume no 'zip64 extensible data' - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) - data = fpin.read(sizeEndCentDir64) - if len(data) != sizeEndCentDir64: - return endrec - sig, sz, create_version, read_version, disk_num, disk_dir, \ - dircount, dircount2, dirsize, diroffset = \ - struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec - - # Update the original endrec using data from the ZIP64 record - endrec[_ECD_SIGNATURE] = sig - endrec[_ECD_DISK_NUMBER] = disk_num - endrec[_ECD_DISK_START] = disk_dir - endrec[_ECD_ENTRIES_THIS_DISK] = dircount - endrec[_ECD_ENTRIES_TOTAL] = dircount2 - endrec[_ECD_SIZE] = dirsize - endrec[_ECD_OFFSET] = diroffset - return endrec - - -def _EndRecData(fpin): - """Return data from the "End of Central Directory" record, or None. - - The data is a list of the nine items in the ZIP "End of central dir" - record followed by a tenth item, the file seek offset of this record.""" - - # Determine file size - fpin.seek(0, 2) - filesize = fpin.tell() - - # Check to see if this is ZIP file with no archive comment (the - # "end of central directory" structure should be the last item in the - # file if this is the case). - try: - fpin.seek(-sizeEndCentDir, 2) - except OSError: - return None - data = fpin.read() - if (len(data) == sizeEndCentDir and - data[0:4] == stringEndArchive and - data[-2:] == b"\000\000"): - # the signature is correct and there's no comment, unpack structure - endrec = struct.unpack(structEndArchive, data) - endrec=list(endrec) - - # Append a blank comment and record start offset - endrec.append(b"") - endrec.append(filesize - sizeEndCentDir) - - # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) - - # Either this is not a ZIP file, or it is a ZIP file with an archive - # comment. Search the end of the file for the "end of central directory" - # record signature. The comment is the last item in the ZIP file and may be - # up to 64K long. It is assumed that the "end of central directory" magic - # number does not appear in the comment. - maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) - fpin.seek(maxCommentStart, 0) - data = fpin.read() - start = data.rfind(stringEndArchive) - if start >= 0: - # found the magic number; attempt to unpack and interpret - recData = data[start:start+sizeEndCentDir] - if len(recData) != sizeEndCentDir: - # Zip file is corrupted. - return None - endrec = list(struct.unpack(structEndArchive, recData)) - commentSize = endrec[_ECD_COMMENT_SIZE] #as claimed by the zip file - comment = data[start+sizeEndCentDir:start+sizeEndCentDir+commentSize] - endrec.append(comment) - endrec.append(maxCommentStart + start) - - # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) - - # Unable to find a valid end of central directory structure - return None - - -class ZipInfo (object): - """Class with attributes describing each file in the ZIP archive.""" - - __slots__ = ( - 'orig_filename', - 'filename', - 'date_time', - 'compress_type', - '_compresslevel', - 'comment', - 'extra', - 'create_system', - 'create_version', - 'extract_version', - 'reserved', - 'flag_bits', - 'volume', - 'internal_attr', - 'external_attr', - 'header_offset', - 'CRC', - 'compress_size', - 'file_size', - '_raw_time', - ) - - def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): - self.orig_filename = filename # Original file name in archive - - # Terminate the file name at the first null byte. Null bytes in file - # names are used as tricks by viruses in archives. - null_byte = filename.find(chr(0)) - if null_byte >= 0: - filename = filename[0:null_byte] - # This is used to ensure paths in generated ZIP files always use - # forward slashes as the directory separator, as required by the - # ZIP format specification. - if os.sep != "/" and os.sep in filename: - filename = filename.replace(os.sep, "/") - - self.filename = filename # Normalized file name - self.date_time = date_time # year, month, day, hour, min, sec - - if date_time[0] < 1980: - raise ValueError('ZIP does not support timestamps before 1980') - - # Standard values: - self.compress_type = ZIP_STORED # Type of compression for the file - self._compresslevel = None # Level for the compressor - self.comment = b"" # Comment for each file - self.extra = b"" # ZIP extra data - if sys.platform == 'win32': - self.create_system = 0 # System which created ZIP archive - else: - # Assume everything else is unix-y - self.create_system = 3 # System which created ZIP archive - self.create_version = DEFAULT_VERSION # Version which created ZIP archive - self.extract_version = DEFAULT_VERSION # Version needed to extract archive - self.reserved = 0 # Must be zero - self.flag_bits = 0 # ZIP flag bits - self.volume = 0 # Volume number of file header - self.internal_attr = 0 # Internal attributes - self.external_attr = 0 # External file attributes - self.compress_size = 0 # Size of the compressed file - self.file_size = 0 # Size of the uncompressed file - # Other attributes are set by class ZipFile: - # header_offset Byte offset to the file header - # CRC CRC-32 of the uncompressed file - - def __repr__(self): - result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] - if self.compress_type != ZIP_STORED: - result.append(' compress_type=%s' % - compressor_names.get(self.compress_type, - self.compress_type)) - hi = self.external_attr >> 16 - lo = self.external_attr & 0xFFFF - if hi: - result.append(' filemode=%r' % stat.filemode(hi)) - if lo: - result.append(' external_attr=%#x' % lo) - isdir = self.is_dir() - if not isdir or self.file_size: - result.append(' file_size=%r' % self.file_size) - if ((not isdir or self.compress_size) and - (self.compress_type != ZIP_STORED or - self.file_size != self.compress_size)): - result.append(' compress_size=%r' % self.compress_size) - result.append('>') - return ''.join(result) - - def FileHeader(self, zip64=None): - """Return the per-file header as a bytes object.""" - dt = self.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - if self.flag_bits & 0x08: - # Set these to zero because we write them after the file data - CRC = compress_size = file_size = 0 - else: - CRC = self.CRC - compress_size = self.compress_size - file_size = self.file_size - - extra = self.extra - - min_version = 0 - if zip64 is None: - zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT - if zip64: - fmt = '<HHQQ' - extra = extra + struct.pack(fmt, - 1, struct.calcsize(fmt)-4, file_size, compress_size) - if file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT: - if not zip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - # File is larger than what fits into a 4 byte integer, - # fall back to the ZIP64 extension - file_size = 0xffffffff - compress_size = 0xffffffff - min_version = ZIP64_VERSION - - if self.compress_type == ZIP_BZIP2: - min_version = max(BZIP2_VERSION, min_version) - elif self.compress_type == ZIP_LZMA: - min_version = max(LZMA_VERSION, min_version) - - self.extract_version = max(min_version, self.extract_version) - self.create_version = max(min_version, self.create_version) - filename, flag_bits = self._encodeFilenameFlags() - header = struct.pack(structFileHeader, stringFileHeader, - self.extract_version, self.reserved, flag_bits, - self.compress_type, dostime, dosdate, CRC, - compress_size, file_size, - len(filename), len(extra)) - return header + filename + extra - - def _encodeFilenameFlags(self): - try: - return self.filename.encode('ascii'), self.flag_bits - except UnicodeEncodeError: - return self.filename.encode('utf-8'), self.flag_bits | 0x800 - - def _decodeExtra(self): - # Try to decode the extra field. - extra = self.extra - unpack = struct.unpack - while len(extra) >= 4: - tp, ln = unpack('<HH', extra[:4]) - if ln+4 > len(extra): - raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln)) - if tp == 0x0001: - data = extra[4:ln+4] - # ZIP64 extension (large files and/or large archives) - try: - if self.file_size in (0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF): - field = "File size" - self.file_size, = unpack('<Q', data[:8]) - data = data[8:] - if self.compress_size == 0xFFFF_FFFF: - field = "Compress size" - self.compress_size, = unpack('<Q', data[:8]) - data = data[8:] - if self.header_offset == 0xFFFF_FFFF: - field = "Header offset" - self.header_offset, = unpack('<Q', data[:8]) - except struct.error: - raise BadZipFile(f"Corrupt zip64 extra field. " - f"{field} not found.") from None - - extra = extra[ln+4:] - - @classmethod - def from_file(cls, filename, arcname=None, *, strict_timestamps=True): - """Construct an appropriate ZipInfo for a file on the filesystem. - - filename should be the path to a file or directory on the filesystem. - - arcname is the name which it will have within the archive (by default, - this will be the same as filename, but without a drive letter and with - leading path separators removed). - """ - if isinstance(filename, os.PathLike): - filename = os.fspath(filename) - st = os.stat(filename) - isdir = stat.S_ISDIR(st.st_mode) - mtime = time.localtime(st.st_mtime) - date_time = mtime[0:6] - if not strict_timestamps and date_time[0] < 1980: - date_time = (1980, 1, 1, 0, 0, 0) - elif not strict_timestamps and date_time[0] > 2107: - date_time = (2107, 12, 31, 23, 59, 59) - # Create ZipInfo instance to store file information - if arcname is None: - arcname = filename - arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) - while arcname[0] in (os.sep, os.altsep): - arcname = arcname[1:] - if isdir: - arcname += '/' - zinfo = cls(arcname, date_time) - zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 # Unix attributes - if isdir: - zinfo.file_size = 0 - zinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - zinfo.file_size = st.st_size - - return zinfo - - def is_dir(self): - """Return True if this archive member is a directory.""" - return self.filename[-1] == '/' - - -# ZIP encryption uses the CRC32 one-byte primitive for scrambling some -# internal keys. We noticed that a direct implementation is faster than -# relying on binascii.crc32(). - -_crctable = None -def _gen_crc(crc): - for j in range(8): - if crc & 1: - crc = (crc >> 1) ^ 0xEDB88320 - else: - crc >>= 1 - return crc - -# ZIP supports a password-based form of encryption. Even though known -# plaintext attacks have been found against it, it is still useful -# to be able to get data out of such a file. -# -# Usage: -# zd = _ZipDecrypter(mypwd) -# plain_bytes = zd(cypher_bytes) - -def _ZipDecrypter(pwd): - key0 = 305419896 - key1 = 591751049 - key2 = 878082192 - - global _crctable - if _crctable is None: - _crctable = list(map(_gen_crc, range(256))) - crctable = _crctable - - def crc32(ch, crc): - """Compute the CRC32 primitive on one byte.""" - return (crc >> 8) ^ crctable[(crc ^ ch) & 0xFF] - - def update_keys(c): - nonlocal key0, key1, key2 - key0 = crc32(c, key0) - key1 = (key1 + (key0 & 0xFF)) & 0xFFFFFFFF - key1 = (key1 * 134775813 + 1) & 0xFFFFFFFF - key2 = crc32(key1 >> 24, key2) - - for p in pwd: - update_keys(p) - - def decrypter(data): - """Decrypt a bytes object.""" - result = bytearray() - append = result.append - for c in data: - k = key2 | 2 - c ^= ((k * (k^1)) >> 8) & 0xFF - update_keys(c) - append(c) - return bytes(result) - - return decrypter - - -class LZMACompressor: - - def __init__(self): - self._comp = None - - def _init(self): - props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) - self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[ - lzma._decode_filter_properties(lzma.FILTER_LZMA1, props) - ]) - return struct.pack('<BBH', 9, 4, len(props)) + props - - def compress(self, data): - if self._comp is None: - return self._init() + self._comp.compress(data) - return self._comp.compress(data) - - def flush(self): - if self._comp is None: - return self._init() + self._comp.flush() - return self._comp.flush() - - -class LZMADecompressor: - - def __init__(self): - self._decomp = None - self._unconsumed = b'' - self.eof = False - - def decompress(self, data): - if self._decomp is None: - self._unconsumed += data - if len(self._unconsumed) <= 4: - return b'' - psize, = struct.unpack('<H', self._unconsumed[2:4]) - if len(self._unconsumed) <= 4 + psize: - return b'' - - self._decomp = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=[ - lzma._decode_filter_properties(lzma.FILTER_LZMA1, - self._unconsumed[4:4 + psize]) - ]) - data = self._unconsumed[4 + psize:] - del self._unconsumed - - result = self._decomp.decompress(data) - self.eof = self._decomp.eof - return result - - -compressor_names = { - 0: 'store', - 1: 'shrink', - 2: 'reduce', - 3: 'reduce', - 4: 'reduce', - 5: 'reduce', - 6: 'implode', - 7: 'tokenize', - 8: 'deflate', - 9: 'deflate64', - 10: 'implode', - 12: 'bzip2', - 14: 'lzma', - 18: 'terse', - 19: 'lz77', - 97: 'wavpack', - 98: 'ppmd', -} - -def _check_compression(compression): - if compression == ZIP_STORED: - pass - elif compression == ZIP_DEFLATED: - if not zlib: - raise RuntimeError( - "Compression requires the (missing) zlib module") - elif compression == ZIP_BZIP2: - if not bz2: - raise RuntimeError( - "Compression requires the (missing) bz2 module") - elif compression == ZIP_LZMA: - if not lzma: - raise RuntimeError( - "Compression requires the (missing) lzma module") - else: - raise NotImplementedError("That compression method is not supported") - - -def _get_compressor(compress_type, compresslevel=None): - if compress_type == ZIP_DEFLATED: - if compresslevel is not None: - return zlib.compressobj(compresslevel, zlib.DEFLATED, -15) - return zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) - elif compress_type == ZIP_BZIP2: - if compresslevel is not None: - return bz2.BZ2Compressor(compresslevel) - return bz2.BZ2Compressor() - # compresslevel is ignored for ZIP_LZMA - elif compress_type == ZIP_LZMA: - return LZMACompressor() - else: - return None - - -def _get_decompressor(compress_type): - _check_compression(compress_type) - if compress_type == ZIP_STORED: - return None - elif compress_type == ZIP_DEFLATED: - return zlib.decompressobj(-15) - elif compress_type == ZIP_BZIP2: - return bz2.BZ2Decompressor() - elif compress_type == ZIP_LZMA: - return LZMADecompressor() - else: - descr = compressor_names.get(compress_type) - if descr: - raise NotImplementedError("compression type %d (%s)" % (compress_type, descr)) - else: - raise NotImplementedError("compression type %d" % (compress_type,)) - - -class _SharedFile: - def __init__(self, file, pos, close, lock, writing): - self._file = file - self._pos = pos - self._close = close - self._lock = lock - self._writing = writing - self.seekable = file.seekable - - def tell(self): - return self._pos - - def seek(self, offset, whence=0): - with self._lock: - if self._writing(): - raise ValueError("Can't reposition in the ZIP file while " - "there is an open writing handle on it. " - "Close the writing handle before trying to read.") - self._file.seek(offset, whence) - self._pos = self._file.tell() - return self._pos - - def read(self, n=-1): - with self._lock: - if self._writing(): - raise ValueError("Can't read from the ZIP file while there " - "is an open writing handle on it. " - "Close the writing handle before trying to read.") - self._file.seek(self._pos) - data = self._file.read(n) - self._pos = self._file.tell() - return data - - def close(self): - if self._file is not None: - fileobj = self._file - self._file = None - self._close(fileobj) - -# Provide the tell method for unseekable stream -class _Tellable: - def __init__(self, fp): - self.fp = fp - self.offset = 0 - - def write(self, data): - n = self.fp.write(data) - self.offset += n - return n - - def tell(self): - return self.offset - - def flush(self): - self.fp.flush() - - def close(self): - self.fp.close() - - -class ZipExtFile(io.BufferedIOBase): - """File-like object for reading an archive member. - Is returned by ZipFile.open(). - """ - - # Max size supported by decompressor. - MAX_N = 1 << 31 - 1 - - # Read from compressed files in 4k blocks. - MIN_READ_SIZE = 4096 - - # Chunk size to read during seek - MAX_SEEK_READ = 1 << 24 - - def __init__(self, fileobj, mode, zipinfo, pwd=None, - close_fileobj=False): - self._fileobj = fileobj - self._pwd = pwd - self._close_fileobj = close_fileobj - - self._compress_type = zipinfo.compress_type - self._compress_left = zipinfo.compress_size - self._left = zipinfo.file_size - - self._decompressor = _get_decompressor(self._compress_type) - - self._eof = False - self._readbuffer = b'' - self._offset = 0 - - self.newlines = None - - self.mode = mode - self.name = zipinfo.filename - - if hasattr(zipinfo, 'CRC'): - self._expected_crc = zipinfo.CRC - self._running_crc = crc32(b'') - else: - self._expected_crc = None - - self._seekable = False - try: - if fileobj.seekable(): - self._orig_compress_start = fileobj.tell() - self._orig_compress_size = zipinfo.compress_size - self._orig_file_size = zipinfo.file_size - self._orig_start_crc = self._running_crc - self._seekable = True - except AttributeError: - pass - - self._decrypter = None - if pwd: - if zipinfo.flag_bits & 0x8: - # compare against the file type from extended local headers - check_byte = (zipinfo._raw_time >> 8) & 0xff - else: - # compare against the CRC otherwise - check_byte = (zipinfo.CRC >> 24) & 0xff - h = self._init_decrypter() - if h != check_byte: - raise RuntimeError("Bad password for file %r" % zipinfo.orig_filename) - - - def _init_decrypter(self): - self._decrypter = _ZipDecrypter(self._pwd) - # The first 12 bytes in the cypher stream is an encryption header - # used to strengthen the algorithm. The first 11 bytes are - # completely random, while the 12th contains the MSB of the CRC, - # or the MSB of the file time depending on the header type - # and is used to check the correctness of the password. - header = self._fileobj.read(12) - self._compress_left -= 12 - return self._decrypter(header)[11] - - def __repr__(self): - result = ['<%s.%s' % (self.__class__.__module__, - self.__class__.__qualname__)] - if not self.closed: - result.append(' name=%r mode=%r' % (self.name, self.mode)) - if self._compress_type != ZIP_STORED: - result.append(' compress_type=%s' % - compressor_names.get(self._compress_type, - self._compress_type)) - else: - result.append(' [closed]') - result.append('>') - return ''.join(result) - - def readline(self, limit=-1): - """Read and return a line from the stream. - - If limit is specified, at most limit bytes will be read. - """ - - if limit < 0: - # Shortcut common case - newline found in buffer. - i = self._readbuffer.find(b'\n', self._offset) + 1 - if i > 0: - line = self._readbuffer[self._offset: i] - self._offset = i - return line - - return io.BufferedIOBase.readline(self, limit) - - def peek(self, n=1): - """Returns buffered bytes without advancing the position.""" - if n > len(self._readbuffer) - self._offset: - chunk = self.read(n) - if len(chunk) > self._offset: - self._readbuffer = chunk + self._readbuffer[self._offset:] - self._offset = 0 - else: - self._offset -= len(chunk) - - # Return up to 512 bytes to reduce allocation overhead for tight loops. - return self._readbuffer[self._offset: self._offset + 512] - - def readable(self): - if self.closed: - raise ValueError("I/O operation on closed file.") - return True - - def read(self, n=-1): - """Read and return up to n bytes. - If the argument is omitted, None, or negative, data is read and returned until EOF is reached. - """ - if self.closed: - raise ValueError("read from closed file.") - if n is None or n < 0: - buf = self._readbuffer[self._offset:] - self._readbuffer = b'' - self._offset = 0 - while not self._eof: - buf += self._read1(self.MAX_N) - return buf - - end = n + self._offset - if end < len(self._readbuffer): - buf = self._readbuffer[self._offset:end] - self._offset = end - return buf - - n = end - len(self._readbuffer) - buf = self._readbuffer[self._offset:] - self._readbuffer = b'' - self._offset = 0 - while n > 0 and not self._eof: - data = self._read1(n) - if n < len(data): - self._readbuffer = data - self._offset = n - buf += data[:n] - break - buf += data - n -= len(data) - return buf - - def _update_crc(self, newdata): - # Update the CRC using the given data. - if self._expected_crc is None: - # No need to compute the CRC if we don't have a reference value - return - self._running_crc = crc32(newdata, self._running_crc) - # Check the CRC if we're at the end of the file - if self._eof and self._running_crc != self._expected_crc: - raise BadZipFile("Bad CRC-32 for file %r" % self.name) - - def read1(self, n): - """Read up to n bytes with at most one read() system call.""" - - if n is None or n < 0: - buf = self._readbuffer[self._offset:] - self._readbuffer = b'' - self._offset = 0 - while not self._eof: - data = self._read1(self.MAX_N) - if data: - buf += data - break - return buf - - end = n + self._offset - if end < len(self._readbuffer): - buf = self._readbuffer[self._offset:end] - self._offset = end - return buf - - n = end - len(self._readbuffer) - buf = self._readbuffer[self._offset:] - self._readbuffer = b'' - self._offset = 0 - if n > 0: - while not self._eof: - data = self._read1(n) - if n < len(data): - self._readbuffer = data - self._offset = n - buf += data[:n] - break - if data: - buf += data - break - return buf - - def _read1(self, n): - # Read up to n compressed bytes with at most one read() system call, - # decrypt and decompress them. - if self._eof or n <= 0: - return b'' - - # Read from file. - if self._compress_type == ZIP_DEFLATED: - ## Handle unconsumed data. - data = self._decompressor.unconsumed_tail - if n > len(data): - data += self._read2(n - len(data)) - else: - data = self._read2(n) - - if self._compress_type == ZIP_STORED: - self._eof = self._compress_left <= 0 - elif self._compress_type == ZIP_DEFLATED: - n = max(n, self.MIN_READ_SIZE) - data = self._decompressor.decompress(data, n) - self._eof = (self._decompressor.eof or - self._compress_left <= 0 and - not self._decompressor.unconsumed_tail) - if self._eof: - data += self._decompressor.flush() - else: - data = self._decompressor.decompress(data) - self._eof = self._decompressor.eof or self._compress_left <= 0 - - data = data[:self._left] - self._left -= len(data) - if self._left <= 0: - self._eof = True - self._update_crc(data) - return data - - def _read2(self, n): - if self._compress_left <= 0: - return b'' - - n = max(n, self.MIN_READ_SIZE) - n = min(n, self._compress_left) - - data = self._fileobj.read(n) - self._compress_left -= len(data) - if not data: - raise EOFError - - if self._decrypter is not None: - data = self._decrypter(data) - return data - - def close(self): - try: - if self._close_fileobj: - self._fileobj.close() - finally: - super().close() - - def seekable(self): - if self.closed: - raise ValueError("I/O operation on closed file.") - return self._seekable - - def seek(self, offset, whence=0): - if self.closed: - raise ValueError("seek on closed file.") - if not self._seekable: - raise io.UnsupportedOperation("underlying stream is not seekable") - curr_pos = self.tell() - if whence == 0: # Seek from start of file - new_pos = offset - elif whence == 1: # Seek from current position - new_pos = curr_pos + offset - elif whence == 2: # Seek from EOF - new_pos = self._orig_file_size + offset - else: - raise ValueError("whence must be os.SEEK_SET (0), " - "os.SEEK_CUR (1), or os.SEEK_END (2)") - - if new_pos > self._orig_file_size: - new_pos = self._orig_file_size - - if new_pos < 0: - new_pos = 0 - - read_offset = new_pos - curr_pos - buff_offset = read_offset + self._offset - - if buff_offset >= 0 and buff_offset < len(self._readbuffer): - # Just move the _offset index if the new position is in the _readbuffer - self._offset = buff_offset - read_offset = 0 - elif read_offset < 0: - # Position is before the current position. Reset the ZipExtFile - self._fileobj.seek(self._orig_compress_start) - self._running_crc = self._orig_start_crc - self._compress_left = self._orig_compress_size - self._left = self._orig_file_size - self._readbuffer = b'' - self._offset = 0 - self._decompressor = _get_decompressor(self._compress_type) - self._eof = False - read_offset = new_pos - if self._decrypter is not None: - self._init_decrypter() - - while read_offset > 0: - read_len = min(self.MAX_SEEK_READ, read_offset) - self.read(read_len) - read_offset -= read_len - - return self.tell() - - def tell(self): - if self.closed: - raise ValueError("tell on closed file.") - if not self._seekable: - raise io.UnsupportedOperation("underlying stream is not seekable") - filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset - return filepos - - -class _ZipWriteFile(io.BufferedIOBase): - def __init__(self, zf, zinfo, zip64): - self._zinfo = zinfo - self._zip64 = zip64 - self._zipfile = zf - self._compressor = _get_compressor(zinfo.compress_type, - zinfo._compresslevel) - self._file_size = 0 - self._compress_size = 0 - self._crc = 0 - - @property - def _fileobj(self): - return self._zipfile.fp - - def writable(self): - return True - - def write(self, data): - if self.closed: - raise ValueError('I/O operation on closed file.') - - # Accept any data that supports the buffer protocol - if isinstance(data, (bytes, bytearray)): - nbytes = len(data) - else: - data = memoryview(data) - nbytes = data.nbytes - self._file_size += nbytes - - self._crc = crc32(data, self._crc) - if self._compressor: - data = self._compressor.compress(data) - self._compress_size += len(data) - self._fileobj.write(data) - return nbytes - - def close(self): - if self.closed: - return - try: - super().close() - # Flush any data from the compressor, and update header info - if self._compressor: - buf = self._compressor.flush() - self._compress_size += len(buf) - self._fileobj.write(buf) - self._zinfo.compress_size = self._compress_size - else: - self._zinfo.compress_size = self._file_size - self._zinfo.CRC = self._crc - self._zinfo.file_size = self._file_size - - # Write updated header info - if self._zinfo.flag_bits & 0x08: - # Write CRC and file sizes after the file data - fmt = '<LLQQ' if self._zip64 else '<LLLL' - self._fileobj.write(struct.pack(fmt, _DD_SIGNATURE, self._zinfo.CRC, - self._zinfo.compress_size, self._zinfo.file_size)) - self._zipfile.start_dir = self._fileobj.tell() - else: - if not self._zip64: - if self._file_size > ZIP64_LIMIT: - raise RuntimeError( - 'File size unexpectedly exceeded ZIP64 limit') - if self._compress_size > ZIP64_LIMIT: - raise RuntimeError( - 'Compressed size unexpectedly exceeded ZIP64 limit') - # Seek backwards and write file header (which will now include - # correct CRC and file sizes) - - # Preserve current position in file - self._zipfile.start_dir = self._fileobj.tell() - self._fileobj.seek(self._zinfo.header_offset) - self._fileobj.write(self._zinfo.FileHeader(self._zip64)) - self._fileobj.seek(self._zipfile.start_dir) - - # Successfully written: Add file to our caches - self._zipfile.filelist.append(self._zinfo) - self._zipfile.NameToInfo[self._zinfo.filename] = self._zinfo - finally: - self._zipfile._writing = False - - - -class ZipFile: - """ Class with methods to open, read, write, close, list zip files. - - z = ZipFile(file, mode="r", compression=ZIP_STORED, allowZip64=True, - compresslevel=None) - - file: Either the path to the file, or a file-like object. - If it is a path, the file will be opened and closed by ZipFile. - mode: The mode can be either read 'r', write 'w', exclusive create 'x', - or append 'a'. - compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), - ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma). - allowZip64: if True ZipFile will create files with ZIP64 extensions when - needed, otherwise it will raise an exception when this would - be necessary. - compresslevel: None (default for the given compression type) or an integer - specifying the level to pass to the compressor. - When using ZIP_STORED or ZIP_LZMA this keyword has no effect. - When using ZIP_DEFLATED integers 0 through 9 are accepted. - When using ZIP_BZIP2 integers 1 through 9 are accepted. - - """ - - fp = None # Set here since __del__ checks it - _windows_illegal_name_trans_table = None - - def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, - compresslevel=None, *, strict_timestamps=True): - """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', - or append 'a'.""" - if mode not in ('r', 'w', 'x', 'a'): - raise ValueError("ZipFile requires mode 'r', 'w', 'x', or 'a'") - - _check_compression(compression) - - self._allowZip64 = allowZip64 - self._didModify = False - self.debug = 0 # Level of printing: 0 through 3 - self.NameToInfo = {} # Find file info given name - self.filelist = [] # List of ZipInfo instances for archive - self.compression = compression # Method of compression - self.compresslevel = compresslevel - self.mode = mode - self.pwd = None - self._comment = b'' - self._strict_timestamps = strict_timestamps - - # Check if we were passed a file-like object - if isinstance(file, os.PathLike): - file = os.fspath(file) - if isinstance(file, str): - # No, it's a filename - self._filePassed = 0 - self.filename = file - modeDict = {'r' : 'rb', 'w': 'w+b', 'x': 'x+b', 'a' : 'r+b', - 'r+b': 'w+b', 'w+b': 'wb', 'x+b': 'xb'} - filemode = modeDict[mode] - while True: - try: - self.fp = io.open(file, filemode) - except OSError: - if filemode in modeDict: - filemode = modeDict[filemode] - continue - raise - break - else: - self._filePassed = 1 - self.fp = file - self.filename = getattr(file, 'name', None) - self._fileRefCnt = 1 - self._lock = threading.RLock() - self._seekable = True - self._writing = False - - try: - if mode == 'r': - self._RealGetContents() - elif mode in ('w', 'x'): - # set the modified flag so central directory gets written - # even if no files are added to the archive - self._didModify = True - try: - self.start_dir = self.fp.tell() - except (AttributeError, OSError): - self.fp = _Tellable(self.fp) - self.start_dir = 0 - self._seekable = False - else: - # Some file-like objects can provide tell() but not seek() - try: - self.fp.seek(self.start_dir) - except (AttributeError, OSError): - self._seekable = False - elif mode == 'a': - try: - # See if file is a zip file - self._RealGetContents() - # seek to start of directory and overwrite - self.fp.seek(self.start_dir) - except BadZipFile: - # file is not a zip file, just append - self.fp.seek(0, 2) - - # set the modified flag so central directory gets written - # even if no files are added to the archive - self._didModify = True - self.start_dir = self.fp.tell() - else: - raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") - except: - fp = self.fp - self.fp = None - self._fpclose(fp) - raise - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def __repr__(self): - result = ['<%s.%s' % (self.__class__.__module__, - self.__class__.__qualname__)] - if self.fp is not None: - if self._filePassed: - result.append(' file=%r' % self.fp) - elif self.filename is not None: - result.append(' filename=%r' % self.filename) - result.append(' mode=%r' % self.mode) - else: - result.append(' [closed]') - result.append('>') - return ''.join(result) - - def _RealGetContents(self): - """Read in the table of contents for the ZIP file.""" - fp = self.fp - try: - endrec = _EndRecData(fp) - except OSError: - raise BadZipFile("File is not a zip file") - if not endrec: - raise BadZipFile("File is not a zip file") - if self.debug > 1: - print(endrec) - size_cd = endrec[_ECD_SIZE] # bytes in central directory - offset_cd = endrec[_ECD_OFFSET] # offset of central directory - self._comment = endrec[_ECD_COMMENT] # archive comment - - # "concat" is zero, unless zip was concatenated to another file - concat = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) - - if self.debug > 2: - inferred = concat + offset_cd - print("given, inferred, offset", offset_cd, inferred, concat) - # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + concat - if self.start_dir < 0: - raise BadZipFile("Bad offset for central directory") - fp.seek(self.start_dir, 0) - data = fp.read(size_cd) - fp = io.BytesIO(data) - total = 0 - while total < size_cd: - centdir = fp.read(sizeCentralDir) - if len(centdir) != sizeCentralDir: - raise BadZipFile("Truncated central directory") - centdir = struct.unpack(structCentralDir, centdir) - if centdir[_CD_SIGNATURE] != stringCentralDir: - raise BadZipFile("Bad magic number for central directory") - if self.debug > 2: - print(centdir) - filename = fp.read(centdir[_CD_FILENAME_LENGTH]) - flags = centdir[5] - if flags & 0x800: - # UTF-8 file names extension - filename = filename.decode('utf-8') - else: - # Historical ZIP filename encoding - filename = filename.decode('cp437') - # Create ZipInfo instance to store file information - x = ZipInfo(filename) - x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) - x.comment = fp.read(centdir[_CD_COMMENT_LENGTH]) - x.header_offset = centdir[_CD_LOCAL_HEADER_OFFSET] - (x.create_version, x.create_system, x.extract_version, x.reserved, - x.flag_bits, x.compress_type, t, d, - x.CRC, x.compress_size, x.file_size) = centdir[1:12] - if x.extract_version > MAX_EXTRACT_VERSION: - raise NotImplementedError("zip file version %.1f" % - (x.extract_version / 10)) - x.volume, x.internal_attr, x.external_attr = centdir[15:18] - # Convert date/time code to (year, month, day, hour, min, sec) - x._raw_time = t - x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, - t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) - - x._decodeExtra() - x.header_offset = x.header_offset + concat - self.filelist.append(x) - self.NameToInfo[x.filename] = x - - # update total bytes read from central directory - total = (total + sizeCentralDir + centdir[_CD_FILENAME_LENGTH] - + centdir[_CD_EXTRA_FIELD_LENGTH] - + centdir[_CD_COMMENT_LENGTH]) - - if self.debug > 2: - print("total", total) - - - def namelist(self): - """Return a list of file names in the archive.""" - return [data.filename for data in self.filelist] - - def infolist(self): - """Return a list of class ZipInfo instances for files in the - archive.""" - return self.filelist - - def printdir(self, file=None): - """Print a table of contents for the zip file.""" - print("%-46s %19s %12s" % ("File Name", "Modified ", "Size"), - file=file) - for zinfo in self.filelist: - date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time[:6] - print("%-46s %s %12d" % (zinfo.filename, date, zinfo.file_size), - file=file) - - def testzip(self): - """Read all the files and check the CRC.""" - chunk_size = 2 ** 20 - for zinfo in self.filelist: - try: - # Read by chunks, to avoid an OverflowError or a - # MemoryError with very large embedded files. - with self.open(zinfo.filename, "r") as f: - while f.read(chunk_size): # Check CRC-32 - pass - except BadZipFile: - return zinfo.filename - - def getinfo(self, name): - """Return the instance of ZipInfo given 'name'.""" - info = self.NameToInfo.get(name) - if info is None: - raise KeyError( - 'There is no item named %r in the archive' % name) - - return info - - def setpassword(self, pwd): - """Set default password for encrypted files.""" - if pwd and not isinstance(pwd, bytes): - raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) - if pwd: - self.pwd = pwd - else: - self.pwd = None - - @property - def comment(self): - """The comment text associated with the ZIP file.""" - return self._comment - - @comment.setter - def comment(self, comment): - if not isinstance(comment, bytes): - raise TypeError("comment: expected bytes, got %s" % type(comment).__name__) - # check for valid comment length - if len(comment) > ZIP_MAX_COMMENT: - import warnings - warnings.warn('Archive comment is too long; truncating to %d bytes' - % ZIP_MAX_COMMENT, stacklevel=2) - comment = comment[:ZIP_MAX_COMMENT] - self._comment = comment - self._didModify = True - - def read(self, name, pwd=None): - """Return file bytes for name.""" - with self.open(name, "r", pwd) as fp: - return fp.read() - - def open(self, name, mode="r", pwd=None, *, force_zip64=False): - """Return file-like object for 'name'. - - name is a string for the file name within the ZIP file, or a ZipInfo - object. - - mode should be 'r' to read a file already in the ZIP file, or 'w' to - write to a file newly added to the archive. - - pwd is the password to decrypt files (only used for reading). - - When writing, if the file size is not known in advance but may exceed - 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large - files. If the size is known in advance, it is best to pass a ZipInfo - instance for name, with zinfo.file_size set. - """ - if mode not in {"r", "w"}: - raise ValueError('open() requires mode "r" or "w"') - if pwd and not isinstance(pwd, bytes): - raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) - if pwd and (mode == "w"): - raise ValueError("pwd is only supported for reading files") - if not self.fp: - raise ValueError( - "Attempt to use ZIP archive that was already closed") - - # Make sure we have an info object - if isinstance(name, ZipInfo): - # 'name' is already an info object - zinfo = name - elif mode == 'w': - zinfo = ZipInfo(name) - zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel - else: - # Get info object for name - zinfo = self.getinfo(name) - - if mode == 'w': - return self._open_to_write(zinfo, force_zip64=force_zip64) - - if self._writing: - raise ValueError("Can't read from the ZIP file while there " - "is an open writing handle on it. " - "Close the writing handle before trying to read.") - - # Open for reading: - self._fileRefCnt += 1 - zef_file = _SharedFile(self.fp, zinfo.header_offset, - self._fpclose, self._lock, lambda: self._writing) - try: - # Skip the file header: - fheader = zef_file.read(sizeFileHeader) - if len(fheader) != sizeFileHeader: - raise BadZipFile("Truncated file header") - fheader = struct.unpack(structFileHeader, fheader) - if fheader[_FH_SIGNATURE] != stringFileHeader: - raise BadZipFile("Bad magic number for file header") - - fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) - if fheader[_FH_EXTRA_FIELD_LENGTH]: - zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) - - if zinfo.flag_bits & 0x20: - # Zip 2.7: compressed patched data - raise NotImplementedError("compressed patched data (flag bit 5)") - - if zinfo.flag_bits & 0x40: - # strong encryption - raise NotImplementedError("strong encryption (flag bit 6)") - - if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & 0x800: - # UTF-8 filename - fname_str = fname.decode("utf-8") - else: - fname_str = fname.decode("cp437") - - if fname_str != zinfo.orig_filename: - raise BadZipFile( - 'File name in directory %r and header %r differ.' - % (zinfo.orig_filename, fname)) - - # check for encrypted flag & handle password - is_encrypted = zinfo.flag_bits & 0x1 - if is_encrypted: - if not pwd: - pwd = self.pwd - if not pwd: - raise RuntimeError("File %r is encrypted, password " - "required for extraction" % name) - else: - pwd = None - - return ZipExtFile(zef_file, mode, zinfo, pwd, True) - except: - zef_file.close() - raise - - def _open_to_write(self, zinfo, force_zip64=False): - if force_zip64 and not self._allowZip64: - raise ValueError( - "force_zip64 is True, but allowZip64 was False when opening " - "the ZIP file." - ) - if self._writing: - raise ValueError("Can't write to the ZIP file while there is " - "another write handle open on it. " - "Close the first handle before opening another.") - - # Size and CRC are overwritten with correct data after processing the file - zinfo.compress_size = 0 - zinfo.CRC = 0 - - zinfo.flag_bits = 0x00 - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 - if not self._seekable: - zinfo.flag_bits |= 0x08 - - if not zinfo.external_attr: - zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- - - # Compressed size can be larger than uncompressed size - zip64 = self._allowZip64 and \ - (force_zip64 or zinfo.file_size * 1.05 > ZIP64_LIMIT) - - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() - - self._writecheck(zinfo) - self._didModify = True - - self.fp.write(zinfo.FileHeader(zip64)) - - self._writing = True - return _ZipWriteFile(self, zinfo, zip64) - - def extract(self, member, path=None, pwd=None): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. - """ - if path is None: - path = os.getcwd() - else: - path = os.fspath(path) - - return self._extract_member(member, path, pwd) - - def extractall(self, path=None, members=None, pwd=None): - """Extract all members from the archive to the current working - directory. `path' specifies a different directory to extract to. - `members' is optional and must be a subset of the list returned - by namelist(). - """ - if members is None: - members = self.namelist() - - if path is None: - path = os.getcwd() - else: - path = os.fspath(path) - - for zipinfo in members: - self._extract_member(zipinfo, path, pwd) - - @classmethod - def _sanitize_windows_name(cls, arcname, pathsep): - """Replace bad characters and remove trailing dots from parts.""" - table = cls._windows_illegal_name_trans_table - if not table: - illegal = ':<>|"?*' - table = str.maketrans(illegal, '_' * len(illegal)) - cls._windows_illegal_name_trans_table = table - arcname = arcname.translate(table) - # remove trailing dots - arcname = (x.rstrip('.') for x in arcname.split(pathsep)) - # rejoin, removing empty parts. - arcname = pathsep.join(x for x in arcname if x) - return arcname - - def _extract_member(self, member, targetpath, pwd): - """Extract the ZipInfo object 'member' to a physical - file on the path targetpath. - """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - - # build the destination pathname, replacing - # forward slashes to platform specific separators. - arcname = member.filename.replace('/', os.path.sep) - - if os.path.altsep: - arcname = arcname.replace(os.path.altsep, os.path.sep) - # interpret absolute pathname as relative, remove drive letter or - # UNC path, redundant separators, "." and ".." components. - arcname = os.path.splitdrive(arcname)[1] - invalid_path_parts = ('', os.path.curdir, os.path.pardir) - arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) - if x not in invalid_path_parts) - if os.path.sep == '\\': - # filter illegal characters on Windows - arcname = self._sanitize_windows_name(arcname, os.path.sep) - - targetpath = os.path.join(targetpath, arcname) - targetpath = os.path.normpath(targetpath) - - # Create all upper directories if necessary. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) - - if member.is_dir(): - if not os.path.isdir(targetpath): - os.mkdir(targetpath) - return targetpath - - with self.open(member, pwd=pwd) as source, \ - open(targetpath, "wb") as target: - shutil.copyfileobj(source, target) - - return targetpath - - def _writecheck(self, zinfo): - """Check for errors before writing a file to the archive.""" - if zinfo.filename in self.NameToInfo: - import warnings - warnings.warn('Duplicate name: %r' % zinfo.filename, stacklevel=3) - if self.mode not in ('w', 'x', 'a'): - raise ValueError("write() requires mode 'w', 'x', or 'a'") - if not self.fp: - raise ValueError( - "Attempt to write ZIP archive that was already closed") - _check_compression(zinfo.compress_type) - if not self._allowZip64: - requires_zip64 = None - if len(self.filelist) >= ZIP_FILECOUNT_LIMIT: - requires_zip64 = "Files count" - elif zinfo.file_size > ZIP64_LIMIT: - requires_zip64 = "Filesize" - elif zinfo.header_offset > ZIP64_LIMIT: - requires_zip64 = "Zipfile size" - if requires_zip64: - raise LargeZipFile(requires_zip64 + - " would require ZIP64 extensions") - - def write(self, filename, arcname=None, - compress_type=None, compresslevel=None): - """Put the bytes from filename into the archive under the name - arcname.""" - if not self.fp: - raise ValueError( - "Attempt to write to ZIP archive that was already closed") - if self._writing: - raise ValueError( - "Can't write to ZIP archive while an open writing handle exists" - ) - - zinfo = ZipInfo.from_file(filename, arcname, - strict_timestamps=self._strict_timestamps) - - if zinfo.is_dir(): - zinfo.compress_size = 0 - zinfo.CRC = 0 - else: - if compress_type is not None: - zinfo.compress_type = compress_type - else: - zinfo.compress_type = self.compression - - if compresslevel is not None: - zinfo._compresslevel = compresslevel - else: - zinfo._compresslevel = self.compresslevel - - if zinfo.is_dir(): - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() # Start of header bytes - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 - - self._writecheck(zinfo) - self._didModify = True - - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - self.fp.write(zinfo.FileHeader(False)) - self.start_dir = self.fp.tell() - else: - with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: - shutil.copyfileobj(src, dest, 1024*8) - - def writestr(self, zinfo_or_arcname, data, - compress_type=None, compresslevel=None): - """Write a file into the archive. The contents is 'data', which - may be either a 'str' or a 'bytes' instance; if it is a 'str', - it is encoded as UTF-8 first. - 'zinfo_or_arcname' is either a ZipInfo instance or - the name of the file in the archive.""" - if isinstance(data, str): - data = data.encode("utf-8") - if not isinstance(zinfo_or_arcname, ZipInfo): - zinfo = ZipInfo(filename=zinfo_or_arcname, - date_time=time.localtime(time.time())[:6]) - zinfo.compress_type = self.compression - zinfo._compresslevel = self.compresslevel - if zinfo.filename[-1] == '/': - zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - zinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - zinfo.external_attr = 0o600 << 16 # ?rw------- - else: - zinfo = zinfo_or_arcname - - if not self.fp: - raise ValueError( - "Attempt to write to ZIP archive that was already closed") - if self._writing: - raise ValueError( - "Can't write to ZIP archive while an open writing handle exists." - ) - - if compress_type is not None: - zinfo.compress_type = compress_type - - if compresslevel is not None: - zinfo._compresslevel = compresslevel - - zinfo.file_size = len(data) # Uncompressed size - with self._lock: - with self.open(zinfo, mode='w') as dest: - dest.write(data) - - def __del__(self): - """Call the "close()" method in case the user forgot.""" - self.close() - - def close(self): - """Close the file, and for mode 'w', 'x' and 'a' write the ending - records.""" - if self.fp is None: - return - - if self._writing: - raise ValueError("Can't close the ZIP file while there is " - "an open writing handle on it. " - "Close the writing handle before closing the zip.") - - try: - if self.mode in ('w', 'x', 'a') and self._didModify: # write ending records - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - self._write_end_record() - finally: - fp = self.fp - self.fp = None - self._fpclose(fp) - - def _write_end_record(self): - for zinfo in self.filelist: # write central directory - dt = zinfo.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - extra = [] - if zinfo.file_size > ZIP64_LIMIT \ - or zinfo.compress_size > ZIP64_LIMIT: - extra.append(zinfo.file_size) - extra.append(zinfo.compress_size) - file_size = 0xffffffff - compress_size = 0xffffffff - else: - file_size = zinfo.file_size - compress_size = zinfo.compress_size - - if zinfo.header_offset > ZIP64_LIMIT: - extra.append(zinfo.header_offset) - header_offset = 0xffffffff - else: - header_offset = zinfo.header_offset - - extra_data = zinfo.extra - min_version = 0 - if extra: - # Append a ZIP64 field to the extra's - extra_data = _strip_extra(extra_data, (1,)) - extra_data = struct.pack( - '<HH' + 'Q'*len(extra), - 1, 8*len(extra), *extra) + extra_data - - min_version = ZIP64_VERSION - - if zinfo.compress_type == ZIP_BZIP2: - min_version = max(BZIP2_VERSION, min_version) - elif zinfo.compress_type == ZIP_LZMA: - min_version = max(LZMA_VERSION, min_version) - - extract_version = max(min_version, zinfo.extract_version) - create_version = max(min_version, zinfo.create_version) - filename, flag_bits = zinfo._encodeFilenameFlags() - centdir = struct.pack(structCentralDir, - stringCentralDir, create_version, - zinfo.create_system, extract_version, zinfo.reserved, - flag_bits, zinfo.compress_type, dostime, dosdate, - zinfo.CRC, compress_size, file_size, - len(filename), len(extra_data), len(zinfo.comment), - 0, zinfo.internal_attr, zinfo.external_attr, - header_offset) - self.fp.write(centdir) - self.fp.write(filename) - self.fp.write(extra_data) - self.fp.write(zinfo.comment) - - pos2 = self.fp.tell() - # Write end-of-zip-archive record - centDirCount = len(self.filelist) - centDirSize = pos2 - self.start_dir - centDirOffset = self.start_dir - requires_zip64 = None - if centDirCount > ZIP_FILECOUNT_LIMIT: - requires_zip64 = "Files count" - elif centDirOffset > ZIP64_LIMIT: - requires_zip64 = "Central directory offset" - elif centDirSize > ZIP64_LIMIT: - requires_zip64 = "Central directory size" - if requires_zip64: - # Need to write the ZIP64 end-of-archive records - if not self._allowZip64: - raise LargeZipFile(requires_zip64 + - " would require ZIP64 extensions") - zip64endrec = struct.pack( - structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, - centDirSize, centDirOffset) - self.fp.write(zip64endrec) - - zip64locrec = struct.pack( - structEndArchive64Locator, - stringEndArchive64Locator, 0, pos2, 1) - self.fp.write(zip64locrec) - centDirCount = min(centDirCount, 0xFFFF) - centDirSize = min(centDirSize, 0xFFFFFFFF) - centDirOffset = min(centDirOffset, 0xFFFFFFFF) - - endrec = struct.pack(structEndArchive, stringEndArchive, - 0, 0, centDirCount, centDirCount, - centDirSize, centDirOffset, len(self._comment)) - self.fp.write(endrec) - self.fp.write(self._comment) - if self.mode == "a": - self.fp.truncate() - self.fp.flush() - - def _fpclose(self, fp): - assert self._fileRefCnt > 0 - self._fileRefCnt -= 1 - if not self._fileRefCnt and not self._filePassed: - fp.close() - - -class PyZipFile(ZipFile): - """Class to create ZIP archives with Python library files and packages.""" - - def __init__(self, file, mode="r", compression=ZIP_STORED, - allowZip64=True, optimize=-1): - ZipFile.__init__(self, file, mode=mode, compression=compression, - allowZip64=allowZip64) - self._optimize = optimize - - def writepy(self, pathname, basename="", filterfunc=None): - """Add all files from "pathname" to the ZIP archive. - - If pathname is a package directory, search the directory and - all package subdirectories recursively for all *.py and enter - the modules into the archive. If pathname is a plain - directory, listdir *.py and enter all modules. Else, pathname - must be a Python *.py file and the module will be put into the - archive. Added modules are always module.pyc. - This method will compile the module.py into module.pyc if - necessary. - If filterfunc(pathname) is given, it is called with every argument. - When it is False, the file or directory is skipped. - """ - pathname = os.fspath(pathname) - if filterfunc and not filterfunc(pathname): - if self.debug: - label = 'path' if os.path.isdir(pathname) else 'file' - print('%s %r skipped by filterfunc' % (label, pathname)) - return - dir, name = os.path.split(pathname) - if os.path.isdir(pathname): - initname = os.path.join(pathname, "__init__.py") - if os.path.isfile(initname): - # This is a package directory, add it - if basename: - basename = "%s/%s" % (basename, name) - else: - basename = name - if self.debug: - print("Adding package in", pathname, "as", basename) - fname, arcname = self._get_codename(initname[0:-3], basename) - if self.debug: - print("Adding", arcname) - self.write(fname, arcname) - dirlist = sorted(os.listdir(pathname)) - dirlist.remove("__init__.py") - # Add all *.py files and package subdirectories - for filename in dirlist: - path = os.path.join(pathname, filename) - root, ext = os.path.splitext(filename) - if os.path.isdir(path): - if os.path.isfile(os.path.join(path, "__init__.py")): - # This is a package directory, add it - self.writepy(path, basename, - filterfunc=filterfunc) # Recursive call - elif ext == ".py": - if filterfunc and not filterfunc(path): - if self.debug: - print('file %r skipped by filterfunc' % path) - continue - fname, arcname = self._get_codename(path[0:-3], - basename) - if self.debug: - print("Adding", arcname) - self.write(fname, arcname) - else: - # This is NOT a package directory, add its files at top level - if self.debug: - print("Adding files from directory", pathname) - for filename in sorted(os.listdir(pathname)): - path = os.path.join(pathname, filename) - root, ext = os.path.splitext(filename) - if ext == ".py": - if filterfunc and not filterfunc(path): - if self.debug: - print('file %r skipped by filterfunc' % path) - continue - fname, arcname = self._get_codename(path[0:-3], - basename) - if self.debug: - print("Adding", arcname) - self.write(fname, arcname) - else: - if pathname[-3:] != ".py": - raise RuntimeError( - 'Files added with writepy() must end with ".py"') - fname, arcname = self._get_codename(pathname[0:-3], basename) - if self.debug: - print("Adding file", arcname) - self.write(fname, arcname) - - def _get_codename(self, pathname, basename): - """Return (filename, archivename) for the path. - - Given a module name path, return the correct file path and - archive name, compiling if necessary. For example, given - /python/lib/string, return (/python/lib/string.pyc, string). - """ - def _compile(file, optimize=-1): - import py_compile - if self.debug: - print("Compiling", file) - try: - py_compile.compile(file, doraise=True, optimize=optimize) - except py_compile.PyCompileError as err: - print(err.msg) - return False - return True - - file_py = pathname + ".py" - file_pyc = pathname + ".pyc" - pycache_opt0 = importlib.util.cache_from_source(file_py, optimization='') - pycache_opt1 = importlib.util.cache_from_source(file_py, optimization=1) - pycache_opt2 = importlib.util.cache_from_source(file_py, optimization=2) - if self._optimize == -1: - # legacy mode: use whatever file is present - if (os.path.isfile(file_pyc) and - os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): - # Use .pyc file. - arcname = fname = file_pyc - elif (os.path.isfile(pycache_opt0) and - os.stat(pycache_opt0).st_mtime >= os.stat(file_py).st_mtime): - # Use the __pycache__/*.pyc file, but write it to the legacy pyc - # file name in the archive. - fname = pycache_opt0 - arcname = file_pyc - elif (os.path.isfile(pycache_opt1) and - os.stat(pycache_opt1).st_mtime >= os.stat(file_py).st_mtime): - # Use the __pycache__/*.pyc file, but write it to the legacy pyc - # file name in the archive. - fname = pycache_opt1 - arcname = file_pyc - elif (os.path.isfile(pycache_opt2) and - os.stat(pycache_opt2).st_mtime >= os.stat(file_py).st_mtime): - # Use the __pycache__/*.pyc file, but write it to the legacy pyc - # file name in the archive. - fname = pycache_opt2 - arcname = file_pyc - else: - # Compile py into PEP 3147 pyc file. - if _compile(file_py): - if sys.flags.optimize == 0: - fname = pycache_opt0 - elif sys.flags.optimize == 1: - fname = pycache_opt1 - else: - fname = pycache_opt2 - arcname = file_pyc - else: - fname = arcname = file_py - else: - # new mode: use given optimization level - if self._optimize == 0: - fname = pycache_opt0 - arcname = file_pyc - else: - arcname = file_pyc - if self._optimize == 1: - fname = pycache_opt1 - elif self._optimize == 2: - fname = pycache_opt2 - else: - msg = "invalid value for 'optimize': {!r}".format(self._optimize) - raise ValueError(msg) - if not (os.path.isfile(fname) and - os.stat(fname).st_mtime >= os.stat(file_py).st_mtime): - if not _compile(file_py, optimize=self._optimize): - fname = arcname = file_py - archivename = os.path.split(arcname)[1] - if basename: - archivename = "%s/%s" % (basename, archivename) - return (fname, archivename) - - -def _parents(path): - """ - Given a path with elements separated by - posixpath.sep, generate all parents of that path. - - >>> list(_parents('b/d')) - ['b'] - >>> list(_parents('/b/d/')) - ['/b'] - >>> list(_parents('b/d/f/')) - ['b/d', 'b'] - >>> list(_parents('b')) - [] - >>> list(_parents('')) - [] - """ - return itertools.islice(_ancestry(path), 1, None) - - -def _ancestry(path): - """ - Given a path with elements separated by - posixpath.sep, generate all elements of that path - - >>> list(_ancestry('b/d')) - ['b/d', 'b'] - >>> list(_ancestry('/b/d/')) - ['/b/d', '/b'] - >>> list(_ancestry('b/d/f/')) - ['b/d/f', 'b/d', 'b'] - >>> list(_ancestry('b')) - ['b'] - >>> list(_ancestry('')) - [] - """ - path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: - yield path - path, tail = posixpath.split(path) - - -_dedupe = dict.fromkeys -"""Deduplicate an iterable in original order""" - - -def _difference(minuend, subtrahend): - """ - Return items in minuend not in subtrahend, retaining order - with O(1) lookup. - """ - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - -class CompleteDirs(ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. - """ - - @staticmethod - def _implied_dirs(names): - parents = itertools.chain.from_iterable(map(_parents, names)) - as_dirs = (p + posixpath.sep for p in parents) - return _dedupe(_difference(as_dirs, names)) - - def namelist(self): - names = super(CompleteDirs, self).namelist() - return names + list(self._implied_dirs(names)) - - def _name_set(self): - return set(self.namelist()) - - def resolve_dir(self, name): - """ - If the name represents a directory, return that name - as a directory (with the trailing slash). - """ - names = self._name_set() - dirname = name + '/' - dir_match = name not in names and dirname in names - return dirname if dir_match else name - - @classmethod - def make(cls, source): - """ - Given a source (filename or zipfile), return an - appropriate CompleteDirs subclass. - """ - if isinstance(source, CompleteDirs): - return source - - if not isinstance(source, ZipFile): - return cls(source) - - # Only allow for FastLookup when supplied zipfile is read-only - if 'r' not in source.mode: - cls = CompleteDirs - - source.__class__ = cls - return source - - -class FastLookup(CompleteDirs): - """ - ZipFile subclass to ensure implicit - dirs exist and are resolved rapidly. - """ - - def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super(FastLookup, self).namelist() - return self.__names - - def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() - return self.__lookup - - -class Path: - """ - A pathlib-compatible interface for zip files. - - Consider a zip file with this structure:: - - . - ├── a.txt - └── b - ├── c.txt - └── d - └── e.txt - - >>> data = io.BytesIO() - >>> zf = ZipFile(data, 'w') - >>> zf.writestr('a.txt', 'content of a') - >>> zf.writestr('b/c.txt', 'content of c') - >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'mem/abcde.zip' - - Path accepts the zipfile object itself or a filename - - >>> root = Path(zf) - - From there, several path operations are available. - - Directory iteration (including the zip file itself): - - >>> a, b = root.iterdir() - >>> a - Path('mem/abcde.zip', 'a.txt') - >>> b - Path('mem/abcde.zip', 'b/') - - name property: - - >>> b.name - 'b' - - join with divide operator: - - >>> c = b / 'c.txt' - >>> c - Path('mem/abcde.zip', 'b/c.txt') - >>> c.name - 'c.txt' - - Read text: - - >>> c.read_text() - 'content of c' - - existence: - - >>> c.exists() - True - >>> (b / 'missing.txt').exists() - False - - Coercion to string: - - >>> import os - >>> str(c).replace(os.sep, posixpath.sep) - 'mem/abcde.zip/b/c.txt' - - At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. - - >>> root.name - 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) - 'mem' - """ - - __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" - - def __init__(self, root, at=""): - """ - Construct a Path from a ZipFile or filename. - - Note: When the source is an existing ZipFile object, - its type (__class__) will be mutated to a - specialized type. If the caller wishes to retain the - original type, the caller should either create a - separate ZipFile object or pass a filename. - """ - self.root = FastLookup.make(root) - self.at = at - - def open(self, mode='r', *args, pwd=None, **kwargs): - """ - Open this entry as text or binary following the semantics - of ``pathlib.Path.open()`` by passing arguments through - to io.TextIOWrapper(). - """ - if self.is_dir(): - raise IsADirectoryError(self) - zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': - raise FileNotFoundError(self) - stream = self.root.open(self.at, zip_mode, pwd=pwd) - if 'b' in mode: - if args or kwargs: - raise ValueError("encoding args invalid for binary operation") - return stream - else: - kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) - return io.TextIOWrapper(stream, *args, **kwargs) - - @property - def name(self): - return pathlib.Path(self.at).name or self.filename.name - - @property - def filename(self): - return pathlib.Path(self.root.filename).joinpath(self.at) - - def read_text(self, *args, **kwargs): - kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) - with self.open('r', *args, **kwargs) as strm: - return strm.read() - - def read_bytes(self): - with self.open('rb') as strm: - return strm.read() - - def _is_child(self, path): - return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") - - def _next(self, at): - return self.__class__(self.root, at) - - def is_dir(self): - return not self.at or self.at.endswith("/") - - def is_file(self): - return self.exists() and not self.is_dir() - - def exists(self): - return self.at in self.root._name_set() - - def iterdir(self): - if not self.is_dir(): - raise ValueError("Can't listdir a file") - subs = map(self._next, self.root.namelist()) - return filter(self._is_child, subs) - - def __str__(self): - return posixpath.join(self.root.filename, self.at) - - def __repr__(self): - return self.__repr.format(self=self) - - def joinpath(self, *other): - next = posixpath.join(self.at, *other) - return self._next(self.root.resolve_dir(next)) - - __truediv__ = joinpath - - @property - def parent(self): - if not self.at: - return self.filename.parent - parent_at = posixpath.dirname(self.at.rstrip('/')) - if parent_at: - parent_at += '/' - return self._next(parent_at) - - -def main(args=None): - import argparse - - description = 'A simple command-line interface for zipfile module.' - parser = argparse.ArgumentParser(description=description) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='<zipfile>', - help='Show listing of a zipfile') - group.add_argument('-e', '--extract', nargs=2, - metavar=('<zipfile>', '<output_dir>'), - help='Extract zipfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('<name>', '<file>'), - help='Create zipfile from sources') - group.add_argument('-t', '--test', metavar='<zipfile>', - help='Test if a zipfile is valid') - args = parser.parse_args(args) - - if args.test is not None: - src = args.test - with ZipFile(src, 'r') as zf: - badfile = zf.testzip() - if badfile: - print("The following enclosed file is corrupted: {!r}".format(badfile)) - print("Done testing") - - elif args.list is not None: - src = args.list - with ZipFile(src, 'r') as zf: - zf.printdir() - - elif args.extract is not None: - src, curdir = args.extract - with ZipFile(src, 'r') as zf: - zf.extractall(curdir) - - elif args.create is not None: - zip_name = args.create.pop(0) - files = args.create - - def addToZip(zf, path, zippath): - if os.path.isfile(path): - zf.write(path, zippath, ZIP_DEFLATED) - elif os.path.isdir(path): - if zippath: - zf.write(path, zippath) - for nm in sorted(os.listdir(path)): - addToZip(zf, - os.path.join(path, nm), os.path.join(zippath, nm)) - # else: ignore - - with ZipFile(zip_name, 'w') as zf: - for path in files: - zippath = os.path.basename(path) - if not zippath: - zippath = os.path.basename(os.path.dirname(path)) - if zippath in ('', os.curdir, os.pardir): - zippath = '' - addToZip(zf, path, zippath) - - -if __name__ == "__main__": - main() diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py new file mode 100644 index 00000000000..ac2332e5846 --- /dev/null +++ b/Lib/zipfile/__init__.py @@ -0,0 +1,2435 @@ +""" +Read and write ZIP files. + +XXX references to utf-8 need further investigation. +""" +import binascii +import importlib.util +import io +import os +import shutil +import stat +import struct +import sys +import threading +import time + +try: + import zlib # We may need its compression method + crc32 = zlib.crc32 +except ImportError: + zlib = None + crc32 = binascii.crc32 + +try: + import bz2 # We may need its compression method +except ImportError: + bz2 = None + +try: + import lzma # We may need its compression method +except ImportError: + lzma = None + +try: + from compression import zstd # We may need its compression method +except ImportError: + zstd = None + +__all__ = ["BadZipFile", "BadZipfile", "error", + "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", + "ZIP_ZSTANDARD", "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", + "LargeZipFile", "Path"] + +class BadZipFile(Exception): + pass + + +class LargeZipFile(Exception): + """ + Raised when writing a zipfile, the zipfile requires ZIP64 extensions + and those extensions are disabled. + """ + +error = BadZipfile = BadZipFile # Pre-3.2 compatibility names + + +ZIP64_LIMIT = (1 << 31) - 1 +ZIP_FILECOUNT_LIMIT = (1 << 16) - 1 +ZIP_MAX_COMMENT = (1 << 16) - 1 + +# constants for Zip file compression methods +ZIP_STORED = 0 +ZIP_DEFLATED = 8 +ZIP_BZIP2 = 12 +ZIP_LZMA = 14 +ZIP_ZSTANDARD = 93 +# Other ZIP compression methods not supported + +DEFAULT_VERSION = 20 +ZIP64_VERSION = 45 +BZIP2_VERSION = 46 +LZMA_VERSION = 63 +ZSTANDARD_VERSION = 63 +# we recognize (but not necessarily support) all features up to that version +MAX_EXTRACT_VERSION = 63 + +# Below are some formats and associated data for reading/writing headers using +# the struct module. The names and structures of headers/records are those used +# in the PKWARE description of the ZIP file format: +# http://www.pkware.com/documents/casestudies/APPNOTE.TXT +# (URL valid as of January 2008) + +# The "end of central directory" structure, magic number, size, and indices +# (section V.I in the format document) +structEndArchive = b"<4s4H2LH" +stringEndArchive = b"PK\005\006" +sizeEndCentDir = struct.calcsize(structEndArchive) + +_ECD_SIGNATURE = 0 +_ECD_DISK_NUMBER = 1 +_ECD_DISK_START = 2 +_ECD_ENTRIES_THIS_DISK = 3 +_ECD_ENTRIES_TOTAL = 4 +_ECD_SIZE = 5 +_ECD_OFFSET = 6 +_ECD_COMMENT_SIZE = 7 +# These last two indices are not part of the structure as defined in the +# spec, but they are used internally by this module as a convenience +_ECD_COMMENT = 8 +_ECD_LOCATION = 9 + +# The "central directory" structure, magic number, size, and indices +# of entries in the structure (section V.F in the format document) +structCentralDir = "<4s4B4HL2L5H2L" +stringCentralDir = b"PK\001\002" +sizeCentralDir = struct.calcsize(structCentralDir) + +# indexes of entries in the central directory structure +_CD_SIGNATURE = 0 +_CD_CREATE_VERSION = 1 +_CD_CREATE_SYSTEM = 2 +_CD_EXTRACT_VERSION = 3 +_CD_EXTRACT_SYSTEM = 4 +_CD_FLAG_BITS = 5 +_CD_COMPRESS_TYPE = 6 +_CD_TIME = 7 +_CD_DATE = 8 +_CD_CRC = 9 +_CD_COMPRESSED_SIZE = 10 +_CD_UNCOMPRESSED_SIZE = 11 +_CD_FILENAME_LENGTH = 12 +_CD_EXTRA_FIELD_LENGTH = 13 +_CD_COMMENT_LENGTH = 14 +_CD_DISK_NUMBER_START = 15 +_CD_INTERNAL_FILE_ATTRIBUTES = 16 +_CD_EXTERNAL_FILE_ATTRIBUTES = 17 +_CD_LOCAL_HEADER_OFFSET = 18 + +# General purpose bit flags +# Zip Appnote: 4.4.4 general purpose bit flag: (2 bytes) +_MASK_ENCRYPTED = 1 << 0 +# Bits 1 and 2 have different meanings depending on the compression used. +_MASK_COMPRESS_OPTION_1 = 1 << 1 +# _MASK_COMPRESS_OPTION_2 = 1 << 2 +# _MASK_USE_DATA_DESCRIPTOR: If set, crc-32, compressed size and uncompressed +# size are zero in the local header and the real values are written in the data +# descriptor immediately following the compressed data. +_MASK_USE_DATA_DESCRIPTOR = 1 << 3 +# Bit 4: Reserved for use with compression method 8, for enhanced deflating. +# _MASK_RESERVED_BIT_4 = 1 << 4 +_MASK_COMPRESSED_PATCH = 1 << 5 +_MASK_STRONG_ENCRYPTION = 1 << 6 +# _MASK_UNUSED_BIT_7 = 1 << 7 +# _MASK_UNUSED_BIT_8 = 1 << 8 +# _MASK_UNUSED_BIT_9 = 1 << 9 +# _MASK_UNUSED_BIT_10 = 1 << 10 +_MASK_UTF_FILENAME = 1 << 11 +# Bit 12: Reserved by PKWARE for enhanced compression. +# _MASK_RESERVED_BIT_12 = 1 << 12 +# _MASK_ENCRYPTED_CENTRAL_DIR = 1 << 13 +# Bit 14, 15: Reserved by PKWARE +# _MASK_RESERVED_BIT_14 = 1 << 14 +# _MASK_RESERVED_BIT_15 = 1 << 15 + +# The "local file header" structure, magic number, size, and indices +# (section V.A in the format document) +structFileHeader = "<4s2B4HL2L2H" +stringFileHeader = b"PK\003\004" +sizeFileHeader = struct.calcsize(structFileHeader) + +_FH_SIGNATURE = 0 +_FH_EXTRACT_VERSION = 1 +_FH_EXTRACT_SYSTEM = 2 +_FH_GENERAL_PURPOSE_FLAG_BITS = 3 +_FH_COMPRESSION_METHOD = 4 +_FH_LAST_MOD_TIME = 5 +_FH_LAST_MOD_DATE = 6 +_FH_CRC = 7 +_FH_COMPRESSED_SIZE = 8 +_FH_UNCOMPRESSED_SIZE = 9 +_FH_FILENAME_LENGTH = 10 +_FH_EXTRA_FIELD_LENGTH = 11 + +# The "Zip64 end of central directory locator" structure, magic number, and size +structEndArchive64Locator = "<4sLQL" +stringEndArchive64Locator = b"PK\x06\x07" +sizeEndCentDir64Locator = struct.calcsize(structEndArchive64Locator) + +# The "Zip64 end of central directory" record, magic number, size, and indices +# (section V.G in the format document) +structEndArchive64 = "<4sQ2H2L4Q" +stringEndArchive64 = b"PK\x06\x06" +sizeEndCentDir64 = struct.calcsize(structEndArchive64) + +_CD64_SIGNATURE = 0 +_CD64_DIRECTORY_RECSIZE = 1 +_CD64_CREATE_VERSION = 2 +_CD64_EXTRACT_VERSION = 3 +_CD64_DISK_NUMBER = 4 +_CD64_DISK_NUMBER_START = 5 +_CD64_NUMBER_ENTRIES_THIS_DISK = 6 +_CD64_NUMBER_ENTRIES_TOTAL = 7 +_CD64_DIRECTORY_SIZE = 8 +_CD64_OFFSET_START_CENTDIR = 9 + +_DD_SIGNATURE = 0x08074b50 + + +class _Extra(bytes): + FIELD_STRUCT = struct.Struct('<HH') + + def __new__(cls, val, id=None): + return super().__new__(cls, val) + + def __init__(self, val, id=None): + self.id = id + + @classmethod + def read_one(cls, raw): + try: + xid, xlen = cls.FIELD_STRUCT.unpack(raw[:4]) + except struct.error: + xid = None + xlen = 0 + return cls(raw[:4+xlen], xid), raw[4+xlen:] + + @classmethod + def split(cls, data): + # use memoryview for zero-copy slices + rest = memoryview(data) + while rest: + extra, rest = _Extra.read_one(rest) + yield extra + + @classmethod + def strip(cls, data, xids): + """Remove Extra fields with specified IDs.""" + return b''.join( + ex + for ex in cls.split(data) + if ex.id not in xids + ) + + +def _check_zipfile(fp): + try: + endrec = _EndRecData(fp) + if endrec: + if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0: + return True # Empty zipfiles are still zipfiles + elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]: + # Central directory is on the same disk + fp.seek(sum(_handle_prepended_data(endrec))) + if endrec[_ECD_SIZE] >= sizeCentralDir: + data = fp.read(sizeCentralDir) # CD is where we expect it to be + if len(data) == sizeCentralDir: + centdir = struct.unpack(structCentralDir, data) # CD is the right size + if centdir[_CD_SIGNATURE] == stringCentralDir: + return True # First central directory entry has correct magic number + except OSError: + pass + return False + +def is_zipfile(filename): + """Quickly see if a file is a ZIP file by checking the magic number. + + The filename argument may be a file or file-like object too. + """ + result = False + try: + if hasattr(filename, "read"): + pos = filename.tell() + result = _check_zipfile(fp=filename) + filename.seek(pos) + else: + with open(filename, "rb") as fp: + result = _check_zipfile(fp) + except (OSError, BadZipFile): + pass + return result + +def _handle_prepended_data(endrec, debug=0): + size_cd = endrec[_ECD_SIZE] # bytes in central directory + offset_cd = endrec[_ECD_OFFSET] # offset of central directory + + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd + + if debug > 2: + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) + + return offset_cd, concat + +def _EndRecData64(fpin, offset, endrec): + """ + Read the ZIP64 end-of-archive records and use that to update endrec + """ + offset -= sizeEndCentDir64Locator + if offset < 0: + # The file is not large enough to contain a ZIP64 + # end-of-archive record, so just return the end record we were given. + return endrec + fpin.seek(offset) + data = fpin.read(sizeEndCentDir64Locator) + if len(data) != sizeEndCentDir64Locator: + raise OSError("Unknown I/O error") + sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) + if sig != stringEndArchive64Locator: + return endrec + + if diskno != 0 or disks > 1: + raise BadZipFile("zipfiles that span multiple disks are not supported") + + offset -= sizeEndCentDir64 + if reloff > offset: + raise BadZipFile("Corrupt zip64 end of central directory locator") + # First, check the assumption that there is no prepended data. + fpin.seek(reloff) + extrasz = offset - reloff + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64) and reloff != offset: + # Since we already have seen the Zip64 EOCD Locator, it's + # possible we got here because there is prepended data. + # Assume no 'zip64 extensible data' + fpin.seek(offset) + extrasz = 0 + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64): + raise BadZipFile("Zip64 end of central directory record not found") + + sig, sz, create_version, read_version, disk_num, disk_dir, \ + dircount, dircount2, dirsize, diroffset = \ + struct.unpack(structEndArchive64, data) + if (diroffset + dirsize != reloff or + sz + 12 != sizeEndCentDir64 + extrasz): + raise BadZipFile("Corrupt zip64 end of central directory record") + + # Update the original endrec using data from the ZIP64 record + endrec[_ECD_SIGNATURE] = sig + endrec[_ECD_DISK_NUMBER] = disk_num + endrec[_ECD_DISK_START] = disk_dir + endrec[_ECD_ENTRIES_THIS_DISK] = dircount + endrec[_ECD_ENTRIES_TOTAL] = dircount2 + endrec[_ECD_SIZE] = dirsize + endrec[_ECD_OFFSET] = diroffset + endrec[_ECD_LOCATION] = offset - extrasz + return endrec + + +def _EndRecData(fpin): + """Return data from the "End of Central Directory" record, or None. + + The data is a list of the nine items in the ZIP "End of central dir" + record followed by a tenth item, the file seek offset of this record.""" + + # Determine file size + fpin.seek(0, 2) + filesize = fpin.tell() + + # Check to see if this is ZIP file with no archive comment (the + # "end of central directory" structure should be the last item in the + # file if this is the case). + try: + fpin.seek(-sizeEndCentDir, 2) + except OSError: + return None + data = fpin.read(sizeEndCentDir) + if (len(data) == sizeEndCentDir and + data[0:4] == stringEndArchive and + data[-2:] == b"\000\000"): + # the signature is correct and there's no comment, unpack structure + endrec = struct.unpack(structEndArchive, data) + endrec=list(endrec) + + # Append a blank comment and record start offset + endrec.append(b"") + endrec.append(filesize - sizeEndCentDir) + + # Try to read the "Zip64 end of central directory" structure + return _EndRecData64(fpin, filesize - sizeEndCentDir, endrec) + + # Either this is not a ZIP file, or it is a ZIP file with an archive + # comment. Search the end of the file for the "end of central directory" + # record signature. The comment is the last item in the ZIP file and may be + # up to 64K long. It is assumed that the "end of central directory" magic + # number does not appear in the comment. + maxCommentStart = max(filesize - ZIP_MAX_COMMENT - sizeEndCentDir, 0) + fpin.seek(maxCommentStart, 0) + data = fpin.read(ZIP_MAX_COMMENT + sizeEndCentDir) + start = data.rfind(stringEndArchive) + if start >= 0: + # found the magic number; attempt to unpack and interpret + recData = data[start:start+sizeEndCentDir] + if len(recData) != sizeEndCentDir: + # Zip file is corrupted. + return None + endrec = list(struct.unpack(structEndArchive, recData)) + commentSize = endrec[_ECD_COMMENT_SIZE] #as claimed by the zip file + comment = data[start+sizeEndCentDir:start+sizeEndCentDir+commentSize] + endrec.append(comment) + endrec.append(maxCommentStart + start) + + # Try to read the "Zip64 end of central directory" structure + return _EndRecData64(fpin, maxCommentStart + start, endrec) + + # Unable to find a valid end of central directory structure + return None + +def _sanitize_filename(filename): + """Terminate the file name at the first null byte and + ensure paths always use forward slashes as the directory separator.""" + + # Terminate the file name at the first null byte. Null bytes in file + # names are used as tricks by viruses in archives. + null_byte = filename.find(chr(0)) + if null_byte >= 0: + filename = filename[0:null_byte] + # This is used to ensure paths in generated ZIP files always use + # forward slashes as the directory separator, as required by the + # ZIP format specification. + if os.sep != "/" and os.sep in filename: + filename = filename.replace(os.sep, "/") + if os.altsep and os.altsep != "/" and os.altsep in filename: + filename = filename.replace(os.altsep, "/") + return filename + + +class ZipInfo: + """Class with attributes describing each file in the ZIP archive.""" + + __slots__ = ( + 'orig_filename', + 'filename', + 'date_time', + 'compress_type', + 'compress_level', + 'comment', + 'extra', + 'create_system', + 'create_version', + 'extract_version', + 'reserved', + 'flag_bits', + 'volume', + 'internal_attr', + 'external_attr', + 'header_offset', + 'CRC', + 'compress_size', + 'file_size', + '_raw_time', + '_end_offset', + ) + + def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): + self.orig_filename = filename # Original file name in archive + + # Terminate the file name at the first null byte and + # ensure paths always use forward slashes as the directory separator. + filename = _sanitize_filename(filename) + + self.filename = filename # Normalized file name + self.date_time = date_time # year, month, day, hour, min, sec + + if date_time[0] < 1980: + raise ValueError('ZIP does not support timestamps before 1980') + + # Standard values: + self.compress_type = ZIP_STORED # Type of compression for the file + self.compress_level = None # Level for the compressor + self.comment = b"" # Comment for each file + self.extra = b"" # ZIP extra data + if sys.platform == 'win32': + self.create_system = 0 # System which created ZIP archive + else: + # Assume everything else is unix-y + self.create_system = 3 # System which created ZIP archive + self.create_version = DEFAULT_VERSION # Version which created ZIP archive + self.extract_version = DEFAULT_VERSION # Version needed to extract archive + self.reserved = 0 # Must be zero + self.flag_bits = 0 # ZIP flag bits + self.volume = 0 # Volume number of file header + self.internal_attr = 0 # Internal attributes + self.external_attr = 0 # External file attributes + self.compress_size = 0 # Size of the compressed file + self.file_size = 0 # Size of the uncompressed file + self._end_offset = None # Start of the next local header or central directory + # Other attributes are set by class ZipFile: + # header_offset Byte offset to the file header + # CRC CRC-32 of the uncompressed file + + # Maintain backward compatibility with the old protected attribute name. + @property + def _compresslevel(self): + return self.compress_level + + @_compresslevel.setter + def _compresslevel(self, value): + self.compress_level = value + + def __repr__(self): + result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] + if self.compress_type != ZIP_STORED: + result.append(' compress_type=%s' % + compressor_names.get(self.compress_type, + self.compress_type)) + hi = self.external_attr >> 16 + lo = self.external_attr & 0xFFFF + if hi: + result.append(' filemode=%r' % stat.filemode(hi)) + if lo: + result.append(' external_attr=%#x' % lo) + isdir = self.is_dir() + if not isdir or self.file_size: + result.append(' file_size=%r' % self.file_size) + if ((not isdir or self.compress_size) and + (self.compress_type != ZIP_STORED or + self.file_size != self.compress_size)): + result.append(' compress_size=%r' % self.compress_size) + result.append('>') + return ''.join(result) + + def FileHeader(self, zip64=None): + """Return the per-file header as a bytes object. + + When the optional zip64 arg is None rather than a bool, we will + decide based upon the file_size and compress_size, if known, + False otherwise. + """ + dt = self.date_time + dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] + dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + if self.flag_bits & _MASK_USE_DATA_DESCRIPTOR: + # Set these to zero because we write them after the file data + CRC = compress_size = file_size = 0 + else: + CRC = self.CRC + compress_size = self.compress_size + file_size = self.file_size + + extra = self.extra + + min_version = 0 + if zip64 is None: + # We always explicitly pass zip64 within this module.... This + # remains for anyone using ZipInfo.FileHeader as a public API. + zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT + if zip64: + fmt = '<HHQQ' + extra = extra + struct.pack(fmt, + 1, struct.calcsize(fmt)-4, file_size, compress_size) + file_size = 0xffffffff + compress_size = 0xffffffff + min_version = ZIP64_VERSION + + if self.compress_type == ZIP_BZIP2: + min_version = max(BZIP2_VERSION, min_version) + elif self.compress_type == ZIP_LZMA: + min_version = max(LZMA_VERSION, min_version) + elif self.compress_type == ZIP_ZSTANDARD: + min_version = max(ZSTANDARD_VERSION, min_version) + + self.extract_version = max(min_version, self.extract_version) + self.create_version = max(min_version, self.create_version) + filename, flag_bits = self._encodeFilenameFlags() + header = struct.pack(structFileHeader, stringFileHeader, + self.extract_version, self.reserved, flag_bits, + self.compress_type, dostime, dosdate, CRC, + compress_size, file_size, + len(filename), len(extra)) + return header + filename + extra + + def _encodeFilenameFlags(self): + try: + return self.filename.encode('ascii'), self.flag_bits + except UnicodeEncodeError: + return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME + + def _decodeExtra(self, filename_crc): + # Try to decode the extra field. + extra = self.extra + unpack = struct.unpack + while len(extra) >= 4: + tp, ln = unpack('<HH', extra[:4]) + if ln+4 > len(extra): + raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln)) + if tp == 0x0001: + data = extra[4:ln+4] + # ZIP64 extension (large files and/or large archives) + try: + if self.file_size in (0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF): + field = "File size" + self.file_size, = unpack('<Q', data[:8]) + data = data[8:] + if self.compress_size == 0xFFFF_FFFF: + field = "Compress size" + self.compress_size, = unpack('<Q', data[:8]) + data = data[8:] + if self.header_offset == 0xFFFF_FFFF: + field = "Header offset" + self.header_offset, = unpack('<Q', data[:8]) + except struct.error: + raise BadZipFile(f"Corrupt zip64 extra field. " + f"{field} not found.") from None + elif tp == 0x7075: + data = extra[4:ln+4] + # Unicode Path Extra Field + try: + up_version, up_name_crc = unpack('<BL', data[:5]) + if up_version == 1 and up_name_crc == filename_crc: + up_unicode_name = data[5:].decode('utf-8') + if up_unicode_name: + self.filename = _sanitize_filename(up_unicode_name) + else: + import warnings + warnings.warn("Empty unicode path extra field (0x7075)", stacklevel=2) + except struct.error as e: + raise BadZipFile("Corrupt unicode path extra field (0x7075)") from e + except UnicodeDecodeError as e: + raise BadZipFile('Corrupt unicode path extra field (0x7075): invalid utf-8 bytes') from e + + extra = extra[ln+4:] + + @classmethod + def from_file(cls, filename, arcname=None, *, strict_timestamps=True): + """Construct an appropriate ZipInfo for a file on the filesystem. + + filename should be the path to a file or directory on the filesystem. + + arcname is the name which it will have within the archive (by default, + this will be the same as filename, but without a drive letter and with + leading path separators removed). + """ + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) + st = os.stat(filename) + isdir = stat.S_ISDIR(st.st_mode) + mtime = time.localtime(st.st_mtime) + date_time = mtime[0:6] + if not strict_timestamps and date_time[0] < 1980: + date_time = (1980, 1, 1, 0, 0, 0) + elif not strict_timestamps and date_time[0] > 2107: + date_time = (2107, 12, 31, 23, 59, 59) + # Create ZipInfo instance to store file information + if arcname is None: + arcname = filename + arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) + while arcname[0] in (os.sep, os.altsep): + arcname = arcname[1:] + if isdir: + arcname += '/' + zinfo = cls(arcname, date_time) + zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 # Unix attributes + if isdir: + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 # MS-DOS directory flag + else: + zinfo.file_size = st.st_size + + return zinfo + + def _for_archive(self, archive): + """Resolve suitable defaults from the archive. + + Resolve the date_time, compression attributes, and external attributes + to suitable defaults as used by :method:`ZipFile.writestr`. + + Return self. + """ + # gh-91279: Set the SOURCE_DATE_EPOCH to a specific timestamp + epoch = os.environ.get('SOURCE_DATE_EPOCH') + get_time = int(epoch) if epoch else time.time() + self.date_time = time.localtime(get_time)[:6] + + self.compress_type = archive.compression + self.compress_level = archive.compresslevel + if self.filename.endswith('/'): # pragma: no cover + self.external_attr = 0o40775 << 16 # drwxrwxr-x + self.external_attr |= 0x10 # MS-DOS directory flag + else: + self.external_attr = 0o600 << 16 # ?rw------- + return self + + def is_dir(self): + """Return True if this archive member is a directory.""" + if self.filename.endswith('/'): + return True + # The ZIP format specification requires to use forward slashes + # as the directory separator, but in practice some ZIP files + # created on Windows can use backward slashes. For compatibility + # with the extraction code which already handles this: + if os.path.altsep: + return self.filename.endswith((os.path.sep, os.path.altsep)) + return False + + +# ZIP encryption uses the CRC32 one-byte primitive for scrambling some +# internal keys. We noticed that a direct implementation is faster than +# relying on binascii.crc32(). + +_crctable = None +def _gen_crc(crc): + for j in range(8): + if crc & 1: + crc = (crc >> 1) ^ 0xEDB88320 + else: + crc >>= 1 + return crc + +# ZIP supports a password-based form of encryption. Even though known +# plaintext attacks have been found against it, it is still useful +# to be able to get data out of such a file. +# +# Usage: +# zd = _ZipDecrypter(mypwd) +# plain_bytes = zd(cypher_bytes) + +def _ZipDecrypter(pwd): + key0 = 305419896 + key1 = 591751049 + key2 = 878082192 + + global _crctable + if _crctable is None: + _crctable = list(map(_gen_crc, range(256))) + crctable = _crctable + + def crc32(ch, crc): + """Compute the CRC32 primitive on one byte.""" + return (crc >> 8) ^ crctable[(crc ^ ch) & 0xFF] + + def update_keys(c): + nonlocal key0, key1, key2 + key0 = crc32(c, key0) + key1 = (key1 + (key0 & 0xFF)) & 0xFFFFFFFF + key1 = (key1 * 134775813 + 1) & 0xFFFFFFFF + key2 = crc32(key1 >> 24, key2) + + for p in pwd: + update_keys(p) + + def decrypter(data): + """Decrypt a bytes object.""" + result = bytearray() + append = result.append + for c in data: + k = key2 | 2 + c ^= ((k * (k^1)) >> 8) & 0xFF + update_keys(c) + append(c) + return bytes(result) + + return decrypter + + +class LZMACompressor: + + def __init__(self): + self._comp = None + + def _init(self): + props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) + self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[ + lzma._decode_filter_properties(lzma.FILTER_LZMA1, props) + ]) + return struct.pack('<BBH', 9, 4, len(props)) + props + + def compress(self, data): + if self._comp is None: + return self._init() + self._comp.compress(data) + return self._comp.compress(data) + + def flush(self): + if self._comp is None: + return self._init() + self._comp.flush() + return self._comp.flush() + + +class LZMADecompressor: + + def __init__(self): + self._decomp = None + self._unconsumed = b'' + self.eof = False + + def decompress(self, data): + if self._decomp is None: + self._unconsumed += data + if len(self._unconsumed) <= 4: + return b'' + psize, = struct.unpack('<H', self._unconsumed[2:4]) + if len(self._unconsumed) <= 4 + psize: + return b'' + + self._decomp = lzma.LZMADecompressor(lzma.FORMAT_RAW, filters=[ + lzma._decode_filter_properties(lzma.FILTER_LZMA1, + self._unconsumed[4:4 + psize]) + ]) + data = self._unconsumed[4 + psize:] + del self._unconsumed + + result = self._decomp.decompress(data) + self.eof = self._decomp.eof + return result + + +compressor_names = { + 0: 'store', + 1: 'shrink', + 2: 'reduce', + 3: 'reduce', + 4: 'reduce', + 5: 'reduce', + 6: 'implode', + 7: 'tokenize', + 8: 'deflate', + 9: 'deflate64', + 10: 'implode', + 12: 'bzip2', + 14: 'lzma', + 18: 'terse', + 19: 'lz77', + 93: 'zstd', + 97: 'wavpack', + 98: 'ppmd', +} + +def _check_compression(compression): + if compression == ZIP_STORED: + pass + elif compression == ZIP_DEFLATED: + if not zlib: + raise RuntimeError( + "Compression requires the (missing) zlib module") + elif compression == ZIP_BZIP2: + if not bz2: + raise RuntimeError( + "Compression requires the (missing) bz2 module") + elif compression == ZIP_LZMA: + if not lzma: + raise RuntimeError( + "Compression requires the (missing) lzma module") + elif compression == ZIP_ZSTANDARD: + if not zstd: + raise RuntimeError( + "Compression requires the (missing) compression.zstd module") + else: + raise NotImplementedError("That compression method is not supported") + + +def _get_compressor(compress_type, compresslevel=None): + if compress_type == ZIP_DEFLATED: + if compresslevel is not None: + return zlib.compressobj(compresslevel, zlib.DEFLATED, -15) + return zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15) + elif compress_type == ZIP_BZIP2: + if compresslevel is not None: + return bz2.BZ2Compressor(compresslevel) + return bz2.BZ2Compressor() + # compresslevel is ignored for ZIP_LZMA + elif compress_type == ZIP_LZMA: + return LZMACompressor() + elif compress_type == ZIP_ZSTANDARD: + return zstd.ZstdCompressor(level=compresslevel) + else: + return None + + +def _get_decompressor(compress_type): + _check_compression(compress_type) + if compress_type == ZIP_STORED: + return None + elif compress_type == ZIP_DEFLATED: + return zlib.decompressobj(-15) + elif compress_type == ZIP_BZIP2: + return bz2.BZ2Decompressor() + elif compress_type == ZIP_LZMA: + return LZMADecompressor() + elif compress_type == ZIP_ZSTANDARD: + return zstd.ZstdDecompressor() + else: + descr = compressor_names.get(compress_type) + if descr: + raise NotImplementedError("compression type %d (%s)" % (compress_type, descr)) + else: + raise NotImplementedError("compression type %d" % (compress_type,)) + + +class _SharedFile: + def __init__(self, file, pos, close, lock, writing): + self._file = file + self._pos = pos + self._close = close + self._lock = lock + self._writing = writing + self.seekable = file.seekable + + def tell(self): + return self._pos + + def seek(self, offset, whence=0): + with self._lock: + if self._writing(): + raise ValueError("Can't reposition in the ZIP file while " + "there is an open writing handle on it. " + "Close the writing handle before trying to read.") + if whence == os.SEEK_CUR: + self._file.seek(self._pos + offset) + else: + self._file.seek(offset, whence) + self._pos = self._file.tell() + return self._pos + + def read(self, n=-1): + with self._lock: + if self._writing(): + raise ValueError("Can't read from the ZIP file while there " + "is an open writing handle on it. " + "Close the writing handle before trying to read.") + self._file.seek(self._pos) + data = self._file.read(n) + self._pos = self._file.tell() + return data + + def close(self): + if self._file is not None: + fileobj = self._file + self._file = None + self._close(fileobj) + +# Provide the tell method for unseekable stream +class _Tellable: + def __init__(self, fp): + self.fp = fp + self.offset = 0 + + def write(self, data): + n = self.fp.write(data) + self.offset += n + return n + + def tell(self): + return self.offset + + def flush(self): + self.fp.flush() + + def close(self): + self.fp.close() + + +class ZipExtFile(io.BufferedIOBase): + """File-like object for reading an archive member. + Is returned by ZipFile.open(). + """ + + # Max size supported by decompressor. + MAX_N = 1 << 31 - 1 + + # Read from compressed files in 4k blocks. + MIN_READ_SIZE = 4096 + + # Chunk size to read during seek + MAX_SEEK_READ = 1 << 24 + + def __init__(self, fileobj, mode, zipinfo, pwd=None, + close_fileobj=False): + self._fileobj = fileobj + self._pwd = pwd + self._close_fileobj = close_fileobj + + self._compress_type = zipinfo.compress_type + self._compress_left = zipinfo.compress_size + self._left = zipinfo.file_size + + self._decompressor = _get_decompressor(self._compress_type) + + self._eof = False + self._readbuffer = b'' + self._offset = 0 + + self.newlines = None + + self.mode = mode + self.name = zipinfo.filename + + if hasattr(zipinfo, 'CRC'): + self._expected_crc = zipinfo.CRC + self._running_crc = crc32(b'') + else: + self._expected_crc = None + + self._seekable = False + try: + if fileobj.seekable(): + self._orig_compress_start = fileobj.tell() + self._orig_compress_size = zipinfo.compress_size + self._orig_file_size = zipinfo.file_size + self._orig_start_crc = self._running_crc + self._orig_crc = self._expected_crc + self._seekable = True + except AttributeError: + pass + + self._decrypter = None + if pwd: + if zipinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR: + # compare against the file type from extended local headers + check_byte = (zipinfo._raw_time >> 8) & 0xff + else: + # compare against the CRC otherwise + check_byte = (zipinfo.CRC >> 24) & 0xff + h = self._init_decrypter() + if h != check_byte: + raise RuntimeError("Bad password for file %r" % zipinfo.orig_filename) + + + def _init_decrypter(self): + self._decrypter = _ZipDecrypter(self._pwd) + # The first 12 bytes in the cypher stream is an encryption header + # used to strengthen the algorithm. The first 11 bytes are + # completely random, while the 12th contains the MSB of the CRC, + # or the MSB of the file time depending on the header type + # and is used to check the correctness of the password. + header = self._fileobj.read(12) + self._compress_left -= 12 + return self._decrypter(header)[11] + + def __repr__(self): + result = ['<%s.%s' % (self.__class__.__module__, + self.__class__.__qualname__)] + if not self.closed: + result.append(' name=%r' % (self.name,)) + if self._compress_type != ZIP_STORED: + result.append(' compress_type=%s' % + compressor_names.get(self._compress_type, + self._compress_type)) + else: + result.append(' [closed]') + result.append('>') + return ''.join(result) + + def readline(self, limit=-1): + """Read and return a line from the stream. + + If limit is specified, at most limit bytes will be read. + """ + + if limit < 0: + # Shortcut common case - newline found in buffer. + i = self._readbuffer.find(b'\n', self._offset) + 1 + if i > 0: + line = self._readbuffer[self._offset: i] + self._offset = i + return line + + return io.BufferedIOBase.readline(self, limit) + + def peek(self, n=1): + """Returns buffered bytes without advancing the position.""" + if n > len(self._readbuffer) - self._offset: + chunk = self.read(n) + if len(chunk) > self._offset: + self._readbuffer = chunk + self._readbuffer[self._offset:] + self._offset = 0 + else: + self._offset -= len(chunk) + + # Return up to 512 bytes to reduce allocation overhead for tight loops. + return self._readbuffer[self._offset: self._offset + 512] + + def readable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") + return True + + def read(self, n=-1): + """Read and return up to n bytes. + If the argument is omitted, None, or negative, data is read and returned until EOF is reached. + """ + if self.closed: + raise ValueError("read from closed file.") + if n is None or n < 0: + buf = self._readbuffer[self._offset:] + self._readbuffer = b'' + self._offset = 0 + while not self._eof: + buf += self._read1(self.MAX_N) + return buf + + end = n + self._offset + if end < len(self._readbuffer): + buf = self._readbuffer[self._offset:end] + self._offset = end + return buf + + n = end - len(self._readbuffer) + buf = self._readbuffer[self._offset:] + self._readbuffer = b'' + self._offset = 0 + while n > 0 and not self._eof: + data = self._read1(n) + if n < len(data): + self._readbuffer = data + self._offset = n + buf += data[:n] + break + buf += data + n -= len(data) + return buf + + def _update_crc(self, newdata): + # Update the CRC using the given data. + if self._expected_crc is None: + # No need to compute the CRC if we don't have a reference value + return + self._running_crc = crc32(newdata, self._running_crc) + # Check the CRC if we're at the end of the file + if self._eof and self._running_crc != self._expected_crc: + raise BadZipFile("Bad CRC-32 for file %r" % self.name) + + def read1(self, n): + """Read up to n bytes with at most one read() system call.""" + + if n is None or n < 0: + buf = self._readbuffer[self._offset:] + self._readbuffer = b'' + self._offset = 0 + while not self._eof: + data = self._read1(self.MAX_N) + if data: + buf += data + break + return buf + + end = n + self._offset + if end < len(self._readbuffer): + buf = self._readbuffer[self._offset:end] + self._offset = end + return buf + + n = end - len(self._readbuffer) + buf = self._readbuffer[self._offset:] + self._readbuffer = b'' + self._offset = 0 + if n > 0: + while not self._eof: + data = self._read1(n) + if n < len(data): + self._readbuffer = data + self._offset = n + buf += data[:n] + break + if data: + buf += data + break + return buf + + def _read1(self, n): + # Read up to n compressed bytes with at most one read() system call, + # decrypt and decompress them. + if self._eof or n <= 0: + return b'' + + # Read from file. + if self._compress_type == ZIP_DEFLATED: + ## Handle unconsumed data. + data = self._decompressor.unconsumed_tail + if n > len(data): + data += self._read2(n - len(data)) + else: + data = self._read2(n) + + if self._compress_type == ZIP_STORED: + self._eof = self._compress_left <= 0 + elif self._compress_type == ZIP_DEFLATED: + n = max(n, self.MIN_READ_SIZE) + data = self._decompressor.decompress(data, n) + self._eof = (self._decompressor.eof or + self._compress_left <= 0 and + not self._decompressor.unconsumed_tail) + if self._eof: + data += self._decompressor.flush() + else: + data = self._decompressor.decompress(data) + self._eof = self._decompressor.eof or self._compress_left <= 0 + + data = data[:self._left] + self._left -= len(data) + if self._left <= 0: + self._eof = True + self._update_crc(data) + return data + + def _read2(self, n): + if self._compress_left <= 0: + return b'' + + n = max(n, self.MIN_READ_SIZE) + n = min(n, self._compress_left) + + data = self._fileobj.read(n) + self._compress_left -= len(data) + if not data: + raise EOFError + + if self._decrypter is not None: + data = self._decrypter(data) + return data + + def close(self): + try: + if self._close_fileobj: + self._fileobj.close() + finally: + super().close() + + def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") + return self._seekable + + def seek(self, offset, whence=os.SEEK_SET): + if self.closed: + raise ValueError("seek on closed file.") + if not self._seekable: + raise io.UnsupportedOperation("underlying stream is not seekable") + curr_pos = self.tell() + if whence == os.SEEK_SET: + new_pos = offset + elif whence == os.SEEK_CUR: + new_pos = curr_pos + offset + elif whence == os.SEEK_END: + new_pos = self._orig_file_size + offset + else: + raise ValueError("whence must be os.SEEK_SET (0), " + "os.SEEK_CUR (1), or os.SEEK_END (2)") + + if new_pos > self._orig_file_size: + new_pos = self._orig_file_size + + if new_pos < 0: + new_pos = 0 + + read_offset = new_pos - curr_pos + buff_offset = read_offset + self._offset + + if buff_offset >= 0 and buff_offset < len(self._readbuffer): + # Just move the _offset index if the new position is in the _readbuffer + self._offset = buff_offset + read_offset = 0 + # Fast seek uncompressed unencrypted file + elif self._compress_type == ZIP_STORED and self._decrypter is None and read_offset != 0: + # disable CRC checking after first seeking - it would be invalid + self._expected_crc = None + # seek actual file taking already buffered data into account + read_offset -= len(self._readbuffer) - self._offset + self._fileobj.seek(read_offset, os.SEEK_CUR) + self._left -= read_offset + self._compress_left -= read_offset + self._eof = self._left <= 0 + read_offset = 0 + # flush read buffer + self._readbuffer = b'' + self._offset = 0 + elif read_offset < 0: + # Position is before the current position. Reset the ZipExtFile + self._fileobj.seek(self._orig_compress_start) + self._running_crc = self._orig_start_crc + self._expected_crc = self._orig_crc + self._compress_left = self._orig_compress_size + self._left = self._orig_file_size + self._readbuffer = b'' + self._offset = 0 + self._decompressor = _get_decompressor(self._compress_type) + self._eof = False + read_offset = new_pos + if self._decrypter is not None: + self._init_decrypter() + + while read_offset > 0: + read_len = min(self.MAX_SEEK_READ, read_offset) + self.read(read_len) + read_offset -= read_len + + return self.tell() + + def tell(self): + if self.closed: + raise ValueError("tell on closed file.") + if not self._seekable: + raise io.UnsupportedOperation("underlying stream is not seekable") + filepos = self._orig_file_size - self._left - len(self._readbuffer) + self._offset + return filepos + + +class _ZipWriteFile(io.BufferedIOBase): + def __init__(self, zf, zinfo, zip64): + self._zinfo = zinfo + self._zip64 = zip64 + self._zipfile = zf + self._compressor = _get_compressor(zinfo.compress_type, + zinfo.compress_level) + self._file_size = 0 + self._compress_size = 0 + self._crc = 0 + + @property + def _fileobj(self): + return self._zipfile.fp + + @property + def name(self): + return self._zinfo.filename + + @property + def mode(self): + return 'wb' + + def writable(self): + return True + + def write(self, data): + if self.closed: + raise ValueError('I/O operation on closed file.') + + # Accept any data that supports the buffer protocol + if isinstance(data, (bytes, bytearray)): + nbytes = len(data) + else: + data = memoryview(data) + nbytes = data.nbytes + self._file_size += nbytes + + self._crc = crc32(data, self._crc) + if self._compressor: + data = self._compressor.compress(data) + self._compress_size += len(data) + self._fileobj.write(data) + return nbytes + + def close(self): + if self.closed: + return + try: + super().close() + # Flush any data from the compressor, and update header info + if self._compressor: + buf = self._compressor.flush() + self._compress_size += len(buf) + self._fileobj.write(buf) + self._zinfo.compress_size = self._compress_size + else: + self._zinfo.compress_size = self._file_size + self._zinfo.CRC = self._crc + self._zinfo.file_size = self._file_size + + if not self._zip64: + if self._file_size > ZIP64_LIMIT: + raise RuntimeError("File size too large, try using force_zip64") + if self._compress_size > ZIP64_LIMIT: + raise RuntimeError("Compressed size too large, try using force_zip64") + + # Write updated header info + if self._zinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR: + # Write CRC and file sizes after the file data + fmt = '<LLQQ' if self._zip64 else '<LLLL' + self._fileobj.write(struct.pack(fmt, _DD_SIGNATURE, self._zinfo.CRC, + self._zinfo.compress_size, self._zinfo.file_size)) + self._zipfile.start_dir = self._fileobj.tell() + else: + # Seek backwards and write file header (which will now include + # correct CRC and file sizes) + + # Preserve current position in file + self._zipfile.start_dir = self._fileobj.tell() + self._fileobj.seek(self._zinfo.header_offset) + self._fileobj.write(self._zinfo.FileHeader(self._zip64)) + self._fileobj.seek(self._zipfile.start_dir) + + # Successfully written: Add file to our caches + self._zipfile.filelist.append(self._zinfo) + self._zipfile.NameToInfo[self._zinfo.filename] = self._zinfo + finally: + self._zipfile._writing = False + + + +class ZipFile: + """ Class with methods to open, read, write, close, list zip files. + + z = ZipFile(file, mode="r", compression=ZIP_STORED, allowZip64=True, + compresslevel=None) + + file: Either the path to the file, or a file-like object. + If it is a path, the file will be opened and closed by ZipFile. + mode: The mode can be either read 'r', write 'w', exclusive create 'x', + or append 'a'. + compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), + ZIP_BZIP2 (requires bz2), ZIP_LZMA (requires lzma), or + ZIP_ZSTANDARD (requires compression.zstd). + allowZip64: if True ZipFile will create files with ZIP64 extensions when + needed, otherwise it will raise an exception when this would + be necessary. + compresslevel: None (default for the given compression type) or an integer + specifying the level to pass to the compressor. + When using ZIP_STORED or ZIP_LZMA this keyword has no effect. + When using ZIP_DEFLATED integers 0 through 9 are accepted. + When using ZIP_BZIP2 integers 1 through 9 are accepted. + When using ZIP_ZSTANDARD integers -7 though 22 are common, + see the CompressionParameter enum in compression.zstd for + details. + + """ + + fp = None # Set here since __del__ checks it + _windows_illegal_name_trans_table = None + + def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, + compresslevel=None, *, strict_timestamps=True, metadata_encoding=None): + """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', + or append 'a'.""" + if mode not in ('r', 'w', 'x', 'a'): + raise ValueError("ZipFile requires mode 'r', 'w', 'x', or 'a'") + + _check_compression(compression) + + self._allowZip64 = allowZip64 + self._didModify = False + self.debug = 0 # Level of printing: 0 through 3 + self.NameToInfo = {} # Find file info given name + self.filelist = [] # List of ZipInfo instances for archive + self.compression = compression # Method of compression + self.compresslevel = compresslevel + self.mode = mode + self.pwd = None + self._comment = b'' + self._strict_timestamps = strict_timestamps + self.metadata_encoding = metadata_encoding + + # Check that we don't try to write with nonconforming codecs + if self.metadata_encoding and mode != 'r': + raise ValueError( + "metadata_encoding is only supported for reading files") + + # Check if we were passed a file-like object + if isinstance(file, os.PathLike): + file = os.fspath(file) + if isinstance(file, str): + # No, it's a filename + self._filePassed = 0 + self.filename = file + modeDict = {'r' : 'rb', 'w': 'w+b', 'x': 'x+b', 'a' : 'r+b', + 'r+b': 'w+b', 'w+b': 'wb', 'x+b': 'xb'} + filemode = modeDict[mode] + while True: + try: + self.fp = io.open(file, filemode) + except OSError: + if filemode in modeDict: + filemode = modeDict[filemode] + continue + raise + break + else: + self._filePassed = 1 + self.fp = file + self.filename = getattr(file, 'name', None) + self._fileRefCnt = 1 + self._lock = threading.RLock() + self._seekable = True + self._writing = False + + try: + if mode == 'r': + self._RealGetContents() + elif mode in ('w', 'x'): + # set the modified flag so central directory gets written + # even if no files are added to the archive + self._didModify = True + try: + self.start_dir = self.fp.tell() + except (AttributeError, OSError): + self.fp = _Tellable(self.fp) + self.start_dir = 0 + self._seekable = False + else: + # Some file-like objects can provide tell() but not seek() + try: + self.fp.seek(self.start_dir) + except (AttributeError, OSError): + self._seekable = False + elif mode == 'a': + try: + # See if file is a zip file + self._RealGetContents() + # seek to start of directory and overwrite + self.fp.seek(self.start_dir) + except BadZipFile: + # file is not a zip file, just append + self.fp.seek(0, 2) + + # set the modified flag so central directory gets written + # even if no files are added to the archive + self._didModify = True + self.start_dir = self.fp.tell() + else: + raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") + except: + fp = self.fp + self.fp = None + self._fpclose(fp) + raise + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def __repr__(self): + result = ['<%s.%s' % (self.__class__.__module__, + self.__class__.__qualname__)] + if self.fp is not None: + if self._filePassed: + result.append(' file=%r' % self.fp) + elif self.filename is not None: + result.append(' filename=%r' % self.filename) + result.append(' mode=%r' % self.mode) + else: + result.append(' [closed]') + result.append('>') + return ''.join(result) + + def _RealGetContents(self): + """Read in the table of contents for the ZIP file.""" + fp = self.fp + try: + endrec = _EndRecData(fp) + except OSError: + raise BadZipFile("File is not a zip file") + if not endrec: + raise BadZipFile("File is not a zip file") + if self.debug > 1: + print(endrec) + self._comment = endrec[_ECD_COMMENT] # archive comment + + offset_cd, concat = _handle_prepended_data(endrec, self.debug) + + # self.start_dir: Position of start of central directory + self.start_dir = offset_cd + concat + + if self.start_dir < 0: + raise BadZipFile("Bad offset for central directory") + fp.seek(self.start_dir, 0) + size_cd = endrec[_ECD_SIZE] + data = fp.read(size_cd) + fp = io.BytesIO(data) + total = 0 + while total < size_cd: + centdir = fp.read(sizeCentralDir) + if len(centdir) != sizeCentralDir: + raise BadZipFile("Truncated central directory") + centdir = struct.unpack(structCentralDir, centdir) + if centdir[_CD_SIGNATURE] != stringCentralDir: + raise BadZipFile("Bad magic number for central directory") + if self.debug > 2: + print(centdir) + filename = fp.read(centdir[_CD_FILENAME_LENGTH]) + orig_filename_crc = crc32(filename) + flags = centdir[_CD_FLAG_BITS] + if flags & _MASK_UTF_FILENAME: + # UTF-8 file names extension + filename = filename.decode('utf-8') + else: + # Historical ZIP filename encoding + filename = filename.decode(self.metadata_encoding or 'cp437') + # Create ZipInfo instance to store file information + x = ZipInfo(filename) + x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) + x.comment = fp.read(centdir[_CD_COMMENT_LENGTH]) + x.header_offset = centdir[_CD_LOCAL_HEADER_OFFSET] + (x.create_version, x.create_system, x.extract_version, x.reserved, + x.flag_bits, x.compress_type, t, d, + x.CRC, x.compress_size, x.file_size) = centdir[1:12] + if x.extract_version > MAX_EXTRACT_VERSION: + raise NotImplementedError("zip file version %.1f" % + (x.extract_version / 10)) + x.volume, x.internal_attr, x.external_attr = centdir[15:18] + # Convert date/time code to (year, month, day, hour, min, sec) + x._raw_time = t + x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, + t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) + x._decodeExtra(orig_filename_crc) + x.header_offset = x.header_offset + concat + self.filelist.append(x) + self.NameToInfo[x.filename] = x + + # update total bytes read from central directory + total = (total + sizeCentralDir + centdir[_CD_FILENAME_LENGTH] + + centdir[_CD_EXTRA_FIELD_LENGTH] + + centdir[_CD_COMMENT_LENGTH]) + + if self.debug > 2: + print("total", total) + + end_offset = self.start_dir + for zinfo in reversed(sorted(self.filelist, + key=lambda zinfo: zinfo.header_offset)): + zinfo._end_offset = end_offset + end_offset = zinfo.header_offset + + def namelist(self): + """Return a list of file names in the archive.""" + return [data.filename for data in self.filelist] + + def infolist(self): + """Return a list of class ZipInfo instances for files in the + archive.""" + return self.filelist + + def printdir(self, file=None): + """Print a table of contents for the zip file.""" + print("%-46s %19s %12s" % ("File Name", "Modified ", "Size"), + file=file) + for zinfo in self.filelist: + date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time[:6] + print("%-46s %s %12d" % (zinfo.filename, date, zinfo.file_size), + file=file) + + def testzip(self): + """Read all the files and check the CRC. + + Return None if all files could be read successfully, or the name + of the offending file otherwise.""" + chunk_size = 2 ** 20 + for zinfo in self.filelist: + try: + # Read by chunks, to avoid an OverflowError or a + # MemoryError with very large embedded files. + with self.open(zinfo.filename, "r") as f: + while f.read(chunk_size): # Check CRC-32 + pass + except BadZipFile: + return zinfo.filename + + def getinfo(self, name): + """Return the instance of ZipInfo given 'name'.""" + info = self.NameToInfo.get(name) + if info is None: + raise KeyError( + 'There is no item named %r in the archive' % name) + + return info + + def setpassword(self, pwd): + """Set default password for encrypted files.""" + if pwd and not isinstance(pwd, bytes): + raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) + if pwd: + self.pwd = pwd + else: + self.pwd = None + + @property + def comment(self): + """The comment text associated with the ZIP file.""" + return self._comment + + @comment.setter + def comment(self, comment): + if not isinstance(comment, bytes): + raise TypeError("comment: expected bytes, got %s" % type(comment).__name__) + # check for valid comment length + if len(comment) > ZIP_MAX_COMMENT: + import warnings + warnings.warn('Archive comment is too long; truncating to %d bytes' + % ZIP_MAX_COMMENT, stacklevel=2) + comment = comment[:ZIP_MAX_COMMENT] + self._comment = comment + self._didModify = True + + def read(self, name, pwd=None): + """Return file bytes for name. 'pwd' is the password to decrypt + encrypted files.""" + with self.open(name, "r", pwd) as fp: + return fp.read() + + def open(self, name, mode="r", pwd=None, *, force_zip64=False): + """Return file-like object for 'name'. + + name is a string for the file name within the ZIP file, or a ZipInfo + object. + + mode should be 'r' to read a file already in the ZIP file, or 'w' to + write to a file newly added to the archive. + + pwd is the password to decrypt files (only used for reading). + + When writing, if the file size is not known in advance but may exceed + 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large + files. If the size is known in advance, it is best to pass a ZipInfo + instance for name, with zinfo.file_size set. + """ + if mode not in {"r", "w"}: + raise ValueError('open() requires mode "r" or "w"') + if pwd and (mode == "w"): + raise ValueError("pwd is only supported for reading files") + if not self.fp: + raise ValueError( + "Attempt to use ZIP archive that was already closed") + + # Make sure we have an info object + if isinstance(name, ZipInfo): + # 'name' is already an info object + zinfo = name + elif mode == 'w': + zinfo = ZipInfo(name) + zinfo.compress_type = self.compression + zinfo.compress_level = self.compresslevel + else: + # Get info object for name + zinfo = self.getinfo(name) + + if mode == 'w': + return self._open_to_write(zinfo, force_zip64=force_zip64) + + if self._writing: + raise ValueError("Can't read from the ZIP file while there " + "is an open writing handle on it. " + "Close the writing handle before trying to read.") + + # Open for reading: + self._fileRefCnt += 1 + zef_file = _SharedFile(self.fp, zinfo.header_offset, + self._fpclose, self._lock, lambda: self._writing) + try: + # Skip the file header: + fheader = zef_file.read(sizeFileHeader) + if len(fheader) != sizeFileHeader: + raise BadZipFile("Truncated file header") + fheader = struct.unpack(structFileHeader, fheader) + if fheader[_FH_SIGNATURE] != stringFileHeader: + raise BadZipFile("Bad magic number for file header") + + fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) + if fheader[_FH_EXTRA_FIELD_LENGTH]: + zef_file.seek(fheader[_FH_EXTRA_FIELD_LENGTH], whence=1) + + if zinfo.flag_bits & _MASK_COMPRESSED_PATCH: + # Zip 2.7: compressed patched data + raise NotImplementedError("compressed patched data (flag bit 5)") + + if zinfo.flag_bits & _MASK_STRONG_ENCRYPTION: + # strong encryption + raise NotImplementedError("strong encryption (flag bit 6)") + + if fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] & _MASK_UTF_FILENAME: + # UTF-8 filename + fname_str = fname.decode("utf-8") + else: + fname_str = fname.decode(self.metadata_encoding or "cp437") + + if fname_str != zinfo.orig_filename: + raise BadZipFile( + 'File name in directory %r and header %r differ.' + % (zinfo.orig_filename, fname)) + + if (zinfo._end_offset is not None and + zef_file.tell() + zinfo.compress_size > zinfo._end_offset): + if zinfo._end_offset == zinfo.header_offset: + import warnings + warnings.warn( + f"Overlapped entries: {zinfo.orig_filename!r} " + f"(possible zip bomb)", + skip_file_prefixes=(os.path.dirname(__file__),)) + else: + raise BadZipFile( + f"Overlapped entries: {zinfo.orig_filename!r} " + f"(possible zip bomb)") + + # check for encrypted flag & handle password + is_encrypted = zinfo.flag_bits & _MASK_ENCRYPTED + if is_encrypted: + if not pwd: + pwd = self.pwd + if pwd and not isinstance(pwd, bytes): + raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) + if not pwd: + raise RuntimeError("File %r is encrypted, password " + "required for extraction" % name) + else: + pwd = None + + return ZipExtFile(zef_file, mode + 'b', zinfo, pwd, True) + except: + zef_file.close() + raise + + def _open_to_write(self, zinfo, force_zip64=False): + if force_zip64 and not self._allowZip64: + raise ValueError( + "force_zip64 is True, but allowZip64 was False when opening " + "the ZIP file." + ) + if self._writing: + raise ValueError("Can't write to the ZIP file while there is " + "another write handle open on it. " + "Close the first handle before opening another.") + + # Size and CRC are overwritten with correct data after processing the file + zinfo.compress_size = 0 + zinfo.CRC = 0 + + zinfo.flag_bits = 0x00 + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 + if not self._seekable: + zinfo.flag_bits |= _MASK_USE_DATA_DESCRIPTOR + + if not zinfo.external_attr: + zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- + + # Compressed size can be larger than uncompressed size + zip64 = force_zip64 or (zinfo.file_size * 1.05 > ZIP64_LIMIT) + if not self._allowZip64 and zip64: + raise LargeZipFile("Filesize would require ZIP64 extensions") + + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() + + self._writecheck(zinfo) + self._didModify = True + + self.fp.write(zinfo.FileHeader(zip64)) + + self._writing = True + return _ZipWriteFile(self, zinfo, zip64) + + def extract(self, member, path=None, pwd=None): + """Extract a member from the archive to the current working directory, + using its full name. Its file information is extracted as accurately + as possible. 'member' may be a filename or a ZipInfo object. You can + specify a different directory using 'path'. You can specify the + password to decrypt the file using 'pwd'. + """ + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + + return self._extract_member(member, path, pwd) + + def extractall(self, path=None, members=None, pwd=None): + """Extract all members from the archive to the current working + directory. 'path' specifies a different directory to extract to. + 'members' is optional and must be a subset of the list returned + by namelist(). You can specify the password to decrypt all files + using 'pwd'. + """ + if members is None: + members = self.namelist() + + if path is None: + path = os.getcwd() + else: + path = os.fspath(path) + + for zipinfo in members: + self._extract_member(zipinfo, path, pwd) + + @classmethod + def _sanitize_windows_name(cls, arcname, pathsep): + """Replace bad characters and remove trailing dots from parts.""" + table = cls._windows_illegal_name_trans_table + if not table: + illegal = ':<>|"?*' + table = str.maketrans(illegal, '_' * len(illegal)) + cls._windows_illegal_name_trans_table = table + arcname = arcname.translate(table) + # remove trailing dots and spaces + arcname = (x.rstrip(' .') for x in arcname.split(pathsep)) + # rejoin, removing empty parts. + arcname = pathsep.join(x for x in arcname if x) + return arcname + + def _extract_member(self, member, targetpath, pwd): + """Extract the ZipInfo object 'member' to a physical + file on the path targetpath. + """ + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + + # build the destination pathname, replacing + # forward slashes to platform specific separators. + arcname = member.filename.replace('/', os.path.sep) + + if os.path.altsep: + arcname = arcname.replace(os.path.altsep, os.path.sep) + # interpret absolute pathname as relative, remove drive letter or + # UNC path, redundant separators, "." and ".." components. + arcname = os.path.splitdrive(arcname)[1] + invalid_path_parts = ('', os.path.curdir, os.path.pardir) + arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) + if x not in invalid_path_parts) + if os.path.sep == '\\': + # filter illegal characters on Windows + arcname = self._sanitize_windows_name(arcname, os.path.sep) + + if not arcname and not member.is_dir(): + raise ValueError("Empty filename.") + + targetpath = os.path.join(targetpath, arcname) + targetpath = os.path.normpath(targetpath) + + # Create all upper directories if necessary. + upperdirs = os.path.dirname(targetpath) + if upperdirs and not os.path.exists(upperdirs): + os.makedirs(upperdirs, exist_ok=True) + + if member.is_dir(): + if not os.path.isdir(targetpath): + try: + os.mkdir(targetpath) + except FileExistsError: + if not os.path.isdir(targetpath): + raise + return targetpath + + with self.open(member, pwd=pwd) as source, \ + open(targetpath, "wb") as target: + shutil.copyfileobj(source, target) + + return targetpath + + def _writecheck(self, zinfo): + """Check for errors before writing a file to the archive.""" + if zinfo.filename in self.NameToInfo: + import warnings + warnings.warn('Duplicate name: %r' % zinfo.filename, stacklevel=3) + if self.mode not in ('w', 'x', 'a'): + raise ValueError("write() requires mode 'w', 'x', or 'a'") + if not self.fp: + raise ValueError( + "Attempt to write ZIP archive that was already closed") + _check_compression(zinfo.compress_type) + if not self._allowZip64: + requires_zip64 = None + if len(self.filelist) >= ZIP_FILECOUNT_LIMIT: + requires_zip64 = "Files count" + elif zinfo.file_size > ZIP64_LIMIT: + requires_zip64 = "Filesize" + elif zinfo.header_offset > ZIP64_LIMIT: + requires_zip64 = "Zipfile size" + if requires_zip64: + raise LargeZipFile(requires_zip64 + + " would require ZIP64 extensions") + + def write(self, filename, arcname=None, + compress_type=None, compresslevel=None): + """Put the bytes from filename into the archive under the name + arcname.""" + if not self.fp: + raise ValueError( + "Attempt to write to ZIP archive that was already closed") + if self._writing: + raise ValueError( + "Can't write to ZIP archive while an open writing handle exists" + ) + + zinfo = ZipInfo.from_file(filename, arcname, + strict_timestamps=self._strict_timestamps) + + if zinfo.is_dir(): + zinfo.compress_size = 0 + zinfo.CRC = 0 + self.mkdir(zinfo) + else: + if compress_type is not None: + zinfo.compress_type = compress_type + else: + zinfo.compress_type = self.compression + + if compresslevel is not None: + zinfo.compress_level = compresslevel + else: + zinfo.compress_level = self.compresslevel + + with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: + shutil.copyfileobj(src, dest, 1024*8) + + def writestr(self, zinfo_or_arcname, data, + compress_type=None, compresslevel=None): + """Write a file into the archive. The contents is 'data', which + may be either a 'str' or a 'bytes' instance; if it is a 'str', + it is encoded as UTF-8 first. + 'zinfo_or_arcname' is either a ZipInfo instance or + the name of the file in the archive.""" + if isinstance(data, str): + data = data.encode("utf-8") + if isinstance(zinfo_or_arcname, ZipInfo): + zinfo = zinfo_or_arcname + else: + zinfo = ZipInfo(zinfo_or_arcname)._for_archive(self) + + if not self.fp: + raise ValueError( + "Attempt to write to ZIP archive that was already closed") + if self._writing: + raise ValueError( + "Can't write to ZIP archive while an open writing handle exists." + ) + + if compress_type is not None: + zinfo.compress_type = compress_type + + if compresslevel is not None: + zinfo.compress_level = compresslevel + + zinfo.file_size = len(data) # Uncompressed size + with self._lock: + with self.open(zinfo, mode='w') as dest: + dest.write(data) + + def mkdir(self, zinfo_or_directory_name, mode=511): + """Creates a directory inside the zip archive.""" + if isinstance(zinfo_or_directory_name, ZipInfo): + zinfo = zinfo_or_directory_name + if not zinfo.is_dir(): + raise ValueError("The given ZipInfo does not describe a directory") + elif isinstance(zinfo_or_directory_name, str): + directory_name = zinfo_or_directory_name + if not directory_name.endswith("/"): + directory_name += "/" + zinfo = ZipInfo(directory_name) + zinfo.compress_size = 0 + zinfo.CRC = 0 + zinfo.external_attr = ((0o40000 | mode) & 0xFFFF) << 16 + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 + else: + raise TypeError("Expected type str or ZipInfo") + + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + zinfo.header_offset = self.fp.tell() # Start of header bytes + if zinfo.compress_type == ZIP_LZMA: + # Compressed data includes an end-of-stream (EOS) marker + zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 + + self._writecheck(zinfo) + self._didModify = True + + self.filelist.append(zinfo) + self.NameToInfo[zinfo.filename] = zinfo + self.fp.write(zinfo.FileHeader(False)) + self.start_dir = self.fp.tell() + + def __del__(self): + """Call the "close()" method in case the user forgot.""" + self.close() + + def close(self): + """Close the file, and for mode 'w', 'x' and 'a' write the ending + records.""" + if self.fp is None: + return + + if self._writing: + raise ValueError("Can't close the ZIP file while there is " + "an open writing handle on it. " + "Close the writing handle before closing the zip.") + + try: + if self.mode in ('w', 'x', 'a') and self._didModify: # write ending records + with self._lock: + if self._seekable: + self.fp.seek(self.start_dir) + self._write_end_record() + finally: + fp = self.fp + self.fp = None + self._fpclose(fp) + + def _write_end_record(self): + for zinfo in self.filelist: # write central directory + dt = zinfo.date_time + dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] + dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + extra = [] + if zinfo.file_size > ZIP64_LIMIT \ + or zinfo.compress_size > ZIP64_LIMIT: + extra.append(zinfo.file_size) + extra.append(zinfo.compress_size) + file_size = 0xffffffff + compress_size = 0xffffffff + else: + file_size = zinfo.file_size + compress_size = zinfo.compress_size + + if zinfo.header_offset > ZIP64_LIMIT: + extra.append(zinfo.header_offset) + header_offset = 0xffffffff + else: + header_offset = zinfo.header_offset + + extra_data = zinfo.extra + min_version = 0 + if extra: + # Append a ZIP64 field to the extra's + extra_data = _Extra.strip(extra_data, (1,)) + extra_data = struct.pack( + '<HH' + 'Q'*len(extra), + 1, 8*len(extra), *extra) + extra_data + + min_version = ZIP64_VERSION + + if zinfo.compress_type == ZIP_BZIP2: + min_version = max(BZIP2_VERSION, min_version) + elif zinfo.compress_type == ZIP_LZMA: + min_version = max(LZMA_VERSION, min_version) + elif zinfo.compress_type == ZIP_ZSTANDARD: + min_version = max(ZSTANDARD_VERSION, min_version) + + extract_version = max(min_version, zinfo.extract_version) + create_version = max(min_version, zinfo.create_version) + filename, flag_bits = zinfo._encodeFilenameFlags() + centdir = struct.pack(structCentralDir, + stringCentralDir, create_version, + zinfo.create_system, extract_version, zinfo.reserved, + flag_bits, zinfo.compress_type, dostime, dosdate, + zinfo.CRC, compress_size, file_size, + len(filename), len(extra_data), len(zinfo.comment), + 0, zinfo.internal_attr, zinfo.external_attr, + header_offset) + self.fp.write(centdir) + self.fp.write(filename) + self.fp.write(extra_data) + self.fp.write(zinfo.comment) + + pos2 = self.fp.tell() + # Write end-of-zip-archive record + centDirCount = len(self.filelist) + centDirSize = pos2 - self.start_dir + centDirOffset = self.start_dir + requires_zip64 = None + if centDirCount > ZIP_FILECOUNT_LIMIT: + requires_zip64 = "Files count" + elif centDirOffset > ZIP64_LIMIT: + requires_zip64 = "Central directory offset" + elif centDirSize > ZIP64_LIMIT: + requires_zip64 = "Central directory size" + if requires_zip64: + # Need to write the ZIP64 end-of-archive records + if not self._allowZip64: + raise LargeZipFile(requires_zip64 + + " would require ZIP64 extensions") + zip64endrec = struct.pack( + structEndArchive64, stringEndArchive64, + sizeEndCentDir64 - 12, 45, 45, 0, 0, centDirCount, centDirCount, + centDirSize, centDirOffset) + self.fp.write(zip64endrec) + + zip64locrec = struct.pack( + structEndArchive64Locator, + stringEndArchive64Locator, 0, pos2, 1) + self.fp.write(zip64locrec) + centDirCount = min(centDirCount, 0xFFFF) + centDirSize = min(centDirSize, 0xFFFFFFFF) + centDirOffset = min(centDirOffset, 0xFFFFFFFF) + + endrec = struct.pack(structEndArchive, stringEndArchive, + 0, 0, centDirCount, centDirCount, + centDirSize, centDirOffset, len(self._comment)) + self.fp.write(endrec) + self.fp.write(self._comment) + if self.mode == "a": + self.fp.truncate() + self.fp.flush() + + def _fpclose(self, fp): + assert self._fileRefCnt > 0 + self._fileRefCnt -= 1 + if not self._fileRefCnt and not self._filePassed: + fp.close() + + +class PyZipFile(ZipFile): + """Class to create ZIP archives with Python library files and packages.""" + + def __init__(self, file, mode="r", compression=ZIP_STORED, + allowZip64=True, optimize=-1): + ZipFile.__init__(self, file, mode=mode, compression=compression, + allowZip64=allowZip64) + self._optimize = optimize + + def writepy(self, pathname, basename="", filterfunc=None): + """Add all files from "pathname" to the ZIP archive. + + If pathname is a package directory, search the directory and + all package subdirectories recursively for all *.py and enter + the modules into the archive. If pathname is a plain + directory, listdir *.py and enter all modules. Else, pathname + must be a Python *.py file and the module will be put into the + archive. Added modules are always module.pyc. + This method will compile the module.py into module.pyc if + necessary. + If filterfunc(pathname) is given, it is called with every argument. + When it is False, the file or directory is skipped. + """ + pathname = os.fspath(pathname) + if filterfunc and not filterfunc(pathname): + if self.debug: + label = 'path' if os.path.isdir(pathname) else 'file' + print('%s %r skipped by filterfunc' % (label, pathname)) + return + dir, name = os.path.split(pathname) + if os.path.isdir(pathname): + initname = os.path.join(pathname, "__init__.py") + if os.path.isfile(initname): + # This is a package directory, add it + if basename: + basename = "%s/%s" % (basename, name) + else: + basename = name + if self.debug: + print("Adding package in", pathname, "as", basename) + fname, arcname = self._get_codename(initname[0:-3], basename) + if self.debug: + print("Adding", arcname) + self.write(fname, arcname) + dirlist = sorted(os.listdir(pathname)) + dirlist.remove("__init__.py") + # Add all *.py files and package subdirectories + for filename in dirlist: + path = os.path.join(pathname, filename) + root, ext = os.path.splitext(filename) + if os.path.isdir(path): + if os.path.isfile(os.path.join(path, "__init__.py")): + # This is a package directory, add it + self.writepy(path, basename, + filterfunc=filterfunc) # Recursive call + elif ext == ".py": + if filterfunc and not filterfunc(path): + if self.debug: + print('file %r skipped by filterfunc' % path) + continue + fname, arcname = self._get_codename(path[0:-3], + basename) + if self.debug: + print("Adding", arcname) + self.write(fname, arcname) + else: + # This is NOT a package directory, add its files at top level + if self.debug: + print("Adding files from directory", pathname) + for filename in sorted(os.listdir(pathname)): + path = os.path.join(pathname, filename) + root, ext = os.path.splitext(filename) + if ext == ".py": + if filterfunc and not filterfunc(path): + if self.debug: + print('file %r skipped by filterfunc' % path) + continue + fname, arcname = self._get_codename(path[0:-3], + basename) + if self.debug: + print("Adding", arcname) + self.write(fname, arcname) + else: + if pathname[-3:] != ".py": + raise RuntimeError( + 'Files added with writepy() must end with ".py"') + fname, arcname = self._get_codename(pathname[0:-3], basename) + if self.debug: + print("Adding file", arcname) + self.write(fname, arcname) + + def _get_codename(self, pathname, basename): + """Return (filename, archivename) for the path. + + Given a module name path, return the correct file path and + archive name, compiling if necessary. For example, given + /python/lib/string, return (/python/lib/string.pyc, string). + """ + def _compile(file, optimize=-1): + import py_compile + if self.debug: + print("Compiling", file) + try: + py_compile.compile(file, doraise=True, optimize=optimize) + except py_compile.PyCompileError as err: + print(err.msg) + return False + return True + + file_py = pathname + ".py" + file_pyc = pathname + ".pyc" + pycache_opt0 = importlib.util.cache_from_source(file_py, optimization='') + pycache_opt1 = importlib.util.cache_from_source(file_py, optimization=1) + pycache_opt2 = importlib.util.cache_from_source(file_py, optimization=2) + if self._optimize == -1: + # legacy mode: use whatever file is present + if (os.path.isfile(file_pyc) and + os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): + # Use .pyc file. + arcname = fname = file_pyc + elif (os.path.isfile(pycache_opt0) and + os.stat(pycache_opt0).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_opt0 + arcname = file_pyc + elif (os.path.isfile(pycache_opt1) and + os.stat(pycache_opt1).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_opt1 + arcname = file_pyc + elif (os.path.isfile(pycache_opt2) and + os.stat(pycache_opt2).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_opt2 + arcname = file_pyc + else: + # Compile py into PEP 3147 pyc file. + if _compile(file_py): + if sys.flags.optimize == 0: + fname = pycache_opt0 + elif sys.flags.optimize == 1: + fname = pycache_opt1 + else: + fname = pycache_opt2 + arcname = file_pyc + else: + fname = arcname = file_py + else: + # new mode: use given optimization level + if self._optimize == 0: + fname = pycache_opt0 + arcname = file_pyc + else: + arcname = file_pyc + if self._optimize == 1: + fname = pycache_opt1 + elif self._optimize == 2: + fname = pycache_opt2 + else: + msg = "invalid value for 'optimize': {!r}".format(self._optimize) + raise ValueError(msg) + if not (os.path.isfile(fname) and + os.stat(fname).st_mtime >= os.stat(file_py).st_mtime): + if not _compile(file_py, optimize=self._optimize): + fname = arcname = file_py + archivename = os.path.split(arcname)[1] + if basename: + archivename = "%s/%s" % (basename, archivename) + return (fname, archivename) + + +def main(args=None): + import argparse + + description = 'A simple command-line interface for zipfile module.' + parser = argparse.ArgumentParser(description=description, color=True) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-l', '--list', metavar='<zipfile>', + help='Show listing of a zipfile') + group.add_argument('-e', '--extract', nargs=2, + metavar=('<zipfile>', '<output_dir>'), + help='Extract zipfile into target dir') + group.add_argument('-c', '--create', nargs='+', + metavar=('<name>', '<file>'), + help='Create zipfile from sources') + group.add_argument('-t', '--test', metavar='<zipfile>', + help='Test if a zipfile is valid') + parser.add_argument('--metadata-encoding', metavar='<encoding>', + help='Specify encoding of member names for -l, -e and -t') + args = parser.parse_args(args) + + encoding = args.metadata_encoding + + if args.test is not None: + src = args.test + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + badfile = zf.testzip() + if badfile: + print("The following enclosed file is corrupted: {!r}".format(badfile)) + print("Done testing") + + elif args.list is not None: + src = args.list + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.printdir() + + elif args.extract is not None: + src, curdir = args.extract + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.extractall(curdir) + + elif args.create is not None: + if encoding: + print("Non-conforming encodings not supported with -c.", + file=sys.stderr) + sys.exit(1) + + zip_name = args.create.pop(0) + files = args.create + + def addToZip(zf, path, zippath): + if os.path.isfile(path): + zf.write(path, zippath, ZIP_DEFLATED) + elif os.path.isdir(path): + if zippath: + zf.write(path, zippath) + for nm in sorted(os.listdir(path)): + addToZip(zf, + os.path.join(path, nm), os.path.join(zippath, nm)) + # else: ignore + + with ZipFile(zip_name, 'w') as zf: + for path in files: + zippath = os.path.basename(path) + if not zippath: + zippath = os.path.basename(os.path.dirname(path)) + if zippath in ('', os.curdir, os.pardir): + zippath = '' + addToZip(zf, path, zippath) + + +from ._path import ( # noqa: E402 + Path, + + # used privately for tests + CompleteDirs, # noqa: F401 +) diff --git a/Lib/zipfile/__main__.py b/Lib/zipfile/__main__.py new file mode 100644 index 00000000000..868d99efc3c --- /dev/null +++ b/Lib/zipfile/__main__.py @@ -0,0 +1,4 @@ +from . import main + +if __name__ == "__main__": + main() diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py new file mode 100644 index 00000000000..80f5d607731 --- /dev/null +++ b/Lib/zipfile/_path/__init__.py @@ -0,0 +1,452 @@ +""" +A Path-like interface for zipfiles. + +This codebase is shared between zipfile.Path in the stdlib +and zipp in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" + +import contextlib +import io +import itertools +import pathlib +import posixpath +import re +import stat +import sys +import zipfile + +from .glob import Translator + +__all__ = ['Path'] + + +def _parents(path): + """ + Given a path with elements separated by + posixpath.sep, generate all parents of that path. + + >>> list(_parents('b/d')) + ['b'] + >>> list(_parents('/b/d/')) + ['/b'] + >>> list(_parents('b/d/f/')) + ['b/d', 'b'] + >>> list(_parents('b')) + [] + >>> list(_parents('')) + [] + """ + return itertools.islice(_ancestry(path), 1, None) + + +def _ancestry(path): + """ + Given a path with elements separated by + posixpath.sep, generate all elements of that path. + + >>> list(_ancestry('b/d')) + ['b/d', 'b'] + >>> list(_ancestry('/b/d/')) + ['/b/d', '/b'] + >>> list(_ancestry('b/d/f/')) + ['b/d/f', 'b/d', 'b'] + >>> list(_ancestry('b')) + ['b'] + >>> list(_ancestry('')) + [] + + Multiple separators are treated like a single. + + >>> list(_ancestry('//b//d///f//')) + ['//b//d///f', '//b//d', '//b'] + """ + path = path.rstrip(posixpath.sep) + while path.rstrip(posixpath.sep): + yield path + path, tail = posixpath.split(path) + + +_dedupe = dict.fromkeys +"""Deduplicate an iterable in original order""" + + +def _difference(minuend, subtrahend): + """ + Return items in minuend not in subtrahend, retaining order + with O(1) lookup. + """ + return itertools.filterfalse(set(subtrahend).__contains__, minuend) + + +class InitializedState: + """ + Mix-in to save the initialization state for pickling. + """ + + def __init__(self, *args, **kwargs): + self.__args = args + self.__kwargs = kwargs + super().__init__(*args, **kwargs) + + def __getstate__(self): + return self.__args, self.__kwargs + + def __setstate__(self, state): + args, kwargs = state + super().__init__(*args, **kwargs) + + +class CompleteDirs(InitializedState, zipfile.ZipFile): + """ + A ZipFile subclass that ensures that implied directories + are always included in the namelist. + + >>> list(CompleteDirs._implied_dirs(['foo/bar.txt', 'foo/bar/baz.txt'])) + ['foo/', 'foo/bar/'] + >>> list(CompleteDirs._implied_dirs(['foo/bar.txt', 'foo/bar/baz.txt', 'foo/bar/'])) + ['foo/'] + """ + + @staticmethod + def _implied_dirs(names): + parents = itertools.chain.from_iterable(map(_parents, names)) + as_dirs = (p + posixpath.sep for p in parents) + return _dedupe(_difference(as_dirs, names)) + + def namelist(self): + names = super().namelist() + return names + list(self._implied_dirs(names)) + + def _name_set(self): + return set(self.namelist()) + + def resolve_dir(self, name): + """ + If the name represents a directory, return that name + as a directory (with the trailing slash). + """ + names = self._name_set() + dirname = name + '/' + dir_match = name not in names and dirname in names + return dirname if dir_match else name + + def getinfo(self, name): + """ + Supplement getinfo for implied dirs. + """ + try: + return super().getinfo(name) + except KeyError: + if not name.endswith('/') or name not in self._name_set(): + raise + return zipfile.ZipInfo(filename=name) + + @classmethod + def make(cls, source): + """ + Given a source (filename or zipfile), return an + appropriate CompleteDirs subclass. + """ + if isinstance(source, CompleteDirs): + return source + + if not isinstance(source, zipfile.ZipFile): + return cls(source) + + # Only allow for FastLookup when supplied zipfile is read-only + if 'r' not in source.mode: + cls = CompleteDirs + + source.__class__ = cls + return source + + @classmethod + def inject(cls, zf: zipfile.ZipFile) -> zipfile.ZipFile: + """ + Given a writable zip file zf, inject directory entries for + any directories implied by the presence of children. + """ + for name in cls._implied_dirs(zf.namelist()): + zf.writestr(name, b"") + return zf + + +class FastLookup(CompleteDirs): + """ + ZipFile subclass to ensure implicit + dirs exist and are resolved rapidly. + """ + + def namelist(self): + with contextlib.suppress(AttributeError): + return self.__names + self.__names = super().namelist() + return self.__names + + def _name_set(self): + with contextlib.suppress(AttributeError): + return self.__lookup + self.__lookup = super()._name_set() + return self.__lookup + +def _extract_text_encoding(encoding=None, *args, **kwargs): + # compute stack level so that the caller of the caller sees any warning. + is_pypy = sys.implementation.name == 'pypy' + # PyPy no longer special cased after 7.3.19 (or maybe 7.3.18) + # See jaraco/zipp#143 + is_old_pypi = is_pypy and sys.pypy_version_info < (7, 3, 19) + stack_level = 3 + is_old_pypi + return io.text_encoding(encoding, stack_level), args, kwargs + + +class Path: + """ + A :class:`importlib.resources.abc.Traversable` interface for zip files. + + Implements many of the features users enjoy from + :class:`pathlib.Path`. + + Consider a zip file with this structure:: + + . + ├── a.txt + └── b + ├── c.txt + └── d + └── e.txt + + >>> data = io.BytesIO() + >>> zf = ZipFile(data, 'w') + >>> zf.writestr('a.txt', 'content of a') + >>> zf.writestr('b/c.txt', 'content of c') + >>> zf.writestr('b/d/e.txt', 'content of e') + >>> zf.filename = 'mem/abcde.zip' + + Path accepts the zipfile object itself or a filename + + >>> path = Path(zf) + + From there, several path operations are available. + + Directory iteration (including the zip file itself): + + >>> a, b = path.iterdir() + >>> a + Path('mem/abcde.zip', 'a.txt') + >>> b + Path('mem/abcde.zip', 'b/') + + name property: + + >>> b.name + 'b' + + join with divide operator: + + >>> c = b / 'c.txt' + >>> c + Path('mem/abcde.zip', 'b/c.txt') + >>> c.name + 'c.txt' + + Read text: + + >>> c.read_text(encoding='utf-8') + 'content of c' + + existence: + + >>> c.exists() + True + >>> (b / 'missing.txt').exists() + False + + Coercion to string: + + >>> import os + >>> str(c).replace(os.sep, posixpath.sep) + 'mem/abcde.zip/b/c.txt' + + At the root, ``name``, ``filename``, and ``parent`` + resolve to the zipfile. + + >>> str(path) + 'mem/abcde.zip/' + >>> path.name + 'abcde.zip' + >>> path.filename == pathlib.Path('mem/abcde.zip') + True + >>> str(path.parent) + 'mem' + + If the zipfile has no filename, such attributes are not + valid and accessing them will raise an Exception. + + >>> zf.filename = None + >>> path.name + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.filename + Traceback (most recent call last): + ... + TypeError: ... + + >>> path.parent + Traceback (most recent call last): + ... + TypeError: ... + + # workaround python/cpython#106763 + >>> pass + """ + + __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" + + def __init__(self, root, at=""): + """ + Construct a Path from a ZipFile or filename. + + Note: When the source is an existing ZipFile object, + its type (__class__) will be mutated to a + specialized type. If the caller wishes to retain the + original type, the caller should either create a + separate ZipFile object or pass a filename. + """ + self.root = FastLookup.make(root) + self.at = at + + def __eq__(self, other): + """ + >>> Path(zipfile.ZipFile(io.BytesIO(), 'w')) == 'foo' + False + """ + if self.__class__ is not other.__class__: + return NotImplemented + return (self.root, self.at) == (other.root, other.at) + + def __hash__(self): + return hash((self.root, self.at)) + + def open(self, mode='r', *args, pwd=None, **kwargs): + """ + Open this entry as text or binary following the semantics + of ``pathlib.Path.open()`` by passing arguments through + to io.TextIOWrapper(). + """ + if self.is_dir(): + raise IsADirectoryError(self) + zip_mode = mode[0] + if zip_mode == 'r' and not self.exists(): + raise FileNotFoundError(self) + stream = self.root.open(self.at, zip_mode, pwd=pwd) + if 'b' in mode: + if args or kwargs: + raise ValueError("encoding args invalid for binary operation") + return stream + # Text mode: + encoding, args, kwargs = _extract_text_encoding(*args, **kwargs) + return io.TextIOWrapper(stream, encoding, *args, **kwargs) + + def _base(self): + return pathlib.PurePosixPath(self.at) if self.at else self.filename + + @property + def name(self): + return self._base().name + + @property + def suffix(self): + return self._base().suffix + + @property + def suffixes(self): + return self._base().suffixes + + @property + def stem(self): + return self._base().stem + + @property + def filename(self): + return pathlib.Path(self.root.filename).joinpath(self.at) + + def read_text(self, *args, **kwargs): + encoding, args, kwargs = _extract_text_encoding(*args, **kwargs) + with self.open('r', encoding, *args, **kwargs) as strm: + return strm.read() + + def read_bytes(self): + with self.open('rb') as strm: + return strm.read() + + def _is_child(self, path): + return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") + + def _next(self, at): + return self.__class__(self.root, at) + + def is_dir(self): + return not self.at or self.at.endswith("/") + + def is_file(self): + return self.exists() and not self.is_dir() + + def exists(self): + return self.at in self.root._name_set() + + def iterdir(self): + if not self.is_dir(): + raise ValueError("Can't listdir a file") + subs = map(self._next, self.root.namelist()) + return filter(self._is_child, subs) + + def match(self, path_pattern): + return pathlib.PurePosixPath(self.at).match(path_pattern) + + def is_symlink(self): + """ + Return whether this path is a symlink. + """ + info = self.root.getinfo(self.at) + mode = info.external_attr >> 16 + return stat.S_ISLNK(mode) + + def glob(self, pattern): + if not pattern: + raise ValueError(f"Unacceptable pattern: {pattern!r}") + + prefix = re.escape(self.at) + tr = Translator(seps='/') + matches = re.compile(prefix + tr.translate(pattern)).fullmatch + return map(self._next, filter(matches, self.root.namelist())) + + def rglob(self, pattern): + return self.glob(f'**/{pattern}') + + def relative_to(self, other, *extra): + return posixpath.relpath(str(self), str(other.joinpath(*extra))) + + def __str__(self): + return posixpath.join(self.root.filename, self.at) + + def __repr__(self): + return self.__repr.format(self=self) + + def joinpath(self, *other): + next = posixpath.join(self.at, *other) + return self._next(self.root.resolve_dir(next)) + + __truediv__ = joinpath + + @property + def parent(self): + if not self.at: + return self.filename.parent + parent_at = posixpath.dirname(self.at.rstrip('/')) + if parent_at: + parent_at += '/' + return self._next(parent_at) diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py new file mode 100644 index 00000000000..bd2839304b7 --- /dev/null +++ b/Lib/zipfile/_path/glob.py @@ -0,0 +1,113 @@ +import os +import re + +_default_seps = os.sep + str(os.altsep) * bool(os.altsep) + + +class Translator: + """ + >>> Translator('xyz') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + + >>> Translator('') + Traceback (most recent call last): + ... + AssertionError: Invalid separators + """ + + seps: str + + def __init__(self, seps: str = _default_seps): + assert seps and set(seps) <= set(_default_seps), "Invalid separators" + self.seps = seps + + def translate(self, pattern): + """ + Given a glob pattern, produce a regex that matches it. + """ + return self.extend(self.match_dirs(self.translate_core(pattern))) + + def extend(self, pattern): + r""" + Extend regex for pattern-wide concerns. + + Apply '(?s:)' to create a non-matching group that + matches newlines (valid on Unix). + + Append '\z' to imply fullmatch even when match is used. + """ + return rf'(?s:{pattern})\z' + + def match_dirs(self, pattern): + """ + Ensure that zipfile.Path directory names are matched. + + zipfile.Path directory names always end in a slash. + """ + return rf'{pattern}[/]?' + + def translate_core(self, pattern): + r""" + Given a glob pattern, produce a regex that matches it. + + >>> t = Translator() + >>> t.translate_core('*.txt').replace('\\\\', '') + '[^/]*\\.txt' + >>> t.translate_core('a?txt') + 'a[^/]txt' + >>> t.translate_core('**/*').replace('\\\\', '') + '.*/[^/][^/]*' + """ + self.restrict_rglob(pattern) + return ''.join(map(self.replace, separate(self.star_not_empty(pattern)))) + + def replace(self, match): + """ + Perform the replacements for a match from :func:`separate`. + """ + return match.group('set') or ( + re.escape(match.group(0)) + .replace('\\*\\*', r'.*') + .replace('\\*', rf'[^{re.escape(self.seps)}]*') + .replace('\\?', r'[^/]') + ) + + def restrict_rglob(self, pattern): + """ + Raise ValueError if ** appears in anything but a full path segment. + + >>> Translator().translate('**foo') + Traceback (most recent call last): + ... + ValueError: ** must appear alone in a path segment + """ + seps_pattern = rf'[{re.escape(self.seps)}]+' + segments = re.split(seps_pattern, pattern) + if any('**' in segment and segment != '**' for segment in segments): + raise ValueError("** must appear alone in a path segment") + + def star_not_empty(self, pattern): + """ + Ensure that * will not match an empty segment. + """ + + def handle_segment(match): + segment = match.group(0) + return '?*' if segment == '*' else segment + + not_seps_pattern = rf'[^{re.escape(self.seps)}]+' + return re.sub(not_seps_pattern, handle_segment, pattern) + + +def separate(pattern): + """ + Separate out character sets to avoid translating their contents. + + >>> [m.group(0) for m in separate('*.txt')] + ['*.txt'] + >>> [m.group(0) for m in separate('a[?]txt')] + ['a', '[?]', 'txt'] + """ + return re.finditer(r'([^\[]+)|(?P<set>[\[].*?[\]])|([\[][^\]]*$)', pattern) diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 25eaee9c0f2..444c9dd11d8 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -1,11 +1,9 @@ """zipimport provides support for importing Python modules from Zip archives. -This module exports three objects: +This module exports two objects: - zipimporter: a class; its constructor takes a path to a Zip archive. - ZipImportError: exception raised by zipimporter objects. It's a subclass of ImportError, so it can be caught as ImportError, too. -- _zip_directory_cache: a dict, mapping archive paths to zip directory - info dicts, as used in zipimporter._files. It is usually not needed to use the zipimport module explicitly; it is used by the builtin import mechanism for sys.path items that are paths @@ -15,14 +13,13 @@ #from importlib import _bootstrap_external #from importlib import _bootstrap # for _verbose_message import _frozen_importlib_external as _bootstrap_external -from _frozen_importlib_external import _unpack_uint16, _unpack_uint32 +from _frozen_importlib_external import _unpack_uint16, _unpack_uint32, _unpack_uint64 import _frozen_importlib as _bootstrap # for _verbose_message import _imp # for check_hash_based_pycs import _io # for open import marshal # for loads import sys # for modules import time # for mktime -import _warnings # For warn() __all__ = ['ZipImportError', 'zipimporter'] @@ -40,8 +37,14 @@ class ZipImportError(ImportError): _module_type = type(sys) END_CENTRAL_DIR_SIZE = 22 -STRING_END_ARCHIVE = b'PK\x05\x06' +END_CENTRAL_DIR_SIZE_64 = 56 +END_CENTRAL_DIR_LOCATOR_SIZE_64 = 20 +STRING_END_ARCHIVE = b'PK\x05\x06' # standard EOCD signature +STRING_END_LOCATOR_64 = b'PK\x06\x07' # Zip64 EOCD Locator signature +STRING_END_ZIP_64 = b'PK\x06\x06' # Zip64 EOCD signature MAX_COMMENT_LEN = (1 << 16) - 1 +MAX_UINT32 = 0xffffffff +ZIP64_EXTRA_TAG = 0x1 class zipimporter(_bootstrap_external._LoaderBasics): """zipimporter(archivepath) -> zipimporter object @@ -63,8 +66,7 @@ class zipimporter(_bootstrap_external._LoaderBasics): # if found, or else read it from the archive. def __init__(self, path): if not isinstance(path, str): - import os - path = os.fsdecode(path) + raise TypeError(f"expected str, not {type(path)!r}") if not path: raise ZipImportError('archive path is empty', path=path) if alt_path_sep: @@ -89,12 +91,8 @@ def __init__(self, path): raise ZipImportError('not a Zip file', path=path) break - try: - files = _zip_directory_cache[path] - except KeyError: - files = _read_directory(path) - _zip_directory_cache[path] = files - self._files = files + if path not in _zip_directory_cache: + _zip_directory_cache[path] = _read_directory(path) self.archive = path # a prefix directory following the ZIP file path. self.prefix = _bootstrap_external._path_join(*prefix[::-1]) @@ -102,64 +100,6 @@ def __init__(self, path): self.prefix += path_sep - # Check whether we can satisfy the import of the module named by - # 'fullname', or whether it could be a portion of a namespace - # package. Return self if we can load it, a string containing the - # full path if it's a possible namespace portion, None if we - # can't load it. - def find_loader(self, fullname, path=None): - """find_loader(fullname, path=None) -> self, str or None. - - Search for a module specified by 'fullname'. 'fullname' must be the - fully qualified (dotted) module name. It returns the zipimporter - instance itself if the module was found, a string containing the - full path name if it's possibly a portion of a namespace package, - or None otherwise. The optional 'path' argument is ignored -- it's - there for compatibility with the importer protocol. - - Deprecated since Python 3.10. Use find_spec() instead. - """ - _warnings.warn("zipimporter.find_loader() is deprecated and slated for " - "removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - mi = _get_module_info(self, fullname) - if mi is not None: - # This is a module or package. - return self, [] - - # Not a module or regular package. See if this is a directory, and - # therefore possibly a portion of a namespace package. - - # We're only interested in the last path component of fullname - # earlier components are recorded in self.prefix. - modpath = _get_module_path(self, fullname) - if _is_dir(self, modpath): - # This is possibly a portion of a namespace - # package. Return the string representing its path, - # without a trailing separator. - return None, [f'{self.archive}{path_sep}{modpath}'] - - return None, [] - - - # Check whether we can satisfy the import of the module named by - # 'fullname'. Return self if we can, None if we can't. - def find_module(self, fullname, path=None): - """find_module(fullname, path=None) -> self or None. - - Search for a module specified by 'fullname'. 'fullname' must be the - fully qualified (dotted) module name. It returns the zipimporter - instance itself if the module was found, or None if it wasn't. - The optional 'path' argument is ignored -- it's there for compatibility - with the importer protocol. - - Deprecated since Python 3.10. Use find_spec() instead. - """ - _warnings.warn("zipimporter.find_module() is deprecated and slated for " - "removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - return self.find_loader(fullname, path)[0] - def find_spec(self, fullname, target=None): """Create a ModuleSpec for the specified module. @@ -211,9 +151,11 @@ def get_data(self, pathname): key = pathname[len(self.archive + path_sep):] try: - toc_entry = self._files[key] + toc_entry = self._get_files()[key] except KeyError: raise OSError(0, '', key) + if toc_entry is None: + return b'' return _get_data(self.archive, toc_entry) @@ -248,7 +190,7 @@ def get_source(self, fullname): fullpath = f'{path}.py' try: - toc_entry = self._files[fullpath] + toc_entry = self._get_files()[fullpath] except KeyError: # we have the module, but no source return None @@ -278,9 +220,11 @@ def load_module(self, fullname): Deprecated since Python 3.10. Use exec_module() instead. """ - msg = ("zipimport.zipimporter.load_module() is deprecated and slated for " - "removal in Python 3.12; use exec_module() instead") - _warnings.warn(msg, DeprecationWarning) + import warnings + warnings._deprecated("zipimport.zipimporter.load_module", + f"{warnings._DEPRECATED_MSG}; " + "use zipimport.zipimporter.exec_module() instead", + remove=(3, 15)) code, ispackage, modpath = _get_module_code(self, fullname) mod = sys.modules.get(fullname) if mod is None or not isinstance(mod, _module_type): @@ -313,28 +257,28 @@ def load_module(self, fullname): def get_resource_reader(self, fullname): - """Return the ResourceReader for a package in a zip file. - - If 'fullname' is a package within the zip file, return the - 'ResourceReader' object for the package. Otherwise return None. - """ - try: - if not self.is_package(fullname): - return None - except ZipImportError: - return None + """Return the ResourceReader for a module in a zip file.""" from importlib.readers import ZipReader + return ZipReader(self, fullname) - def invalidate_caches(self): - """Reload the file data of the archive path.""" + def _get_files(self): + """Return the files within the archive path.""" try: - self._files = _read_directory(self.archive) - _zip_directory_cache[self.archive] = self._files - except ZipImportError: - _zip_directory_cache.pop(self.archive, None) - self._files = {} + files = _zip_directory_cache[self.archive] + except KeyError: + try: + files = _zip_directory_cache[self.archive] = _read_directory(self.archive) + except ZipImportError: + files = {} + + return files + + + def invalidate_caches(self): + """Invalidates the cache of file data of the archive path.""" + _zip_directory_cache.pop(self.archive, None) def __repr__(self): @@ -364,15 +308,15 @@ def _is_dir(self, path): # of a namespace package. We test by seeing if the name, with an # appended path separator, exists. dirpath = path + path_sep - # If dirpath is present in self._files, we have a directory. - return dirpath in self._files + # If dirpath is present in self._get_files(), we have a directory. + return dirpath in self._get_files() # Return some information about a module. def _get_module_info(self, fullname): path = _get_module_path(self, fullname) for suffix, isbytecode, ispackage in _zip_searchorder: fullpath = path + suffix - if fullpath in self._files: + if fullpath in self._get_files(): return ispackage return None @@ -406,16 +350,11 @@ def _read_directory(archive): raise ZipImportError(f"can't open Zip file: {archive!r}", path=archive) with fp: + # GH-87235: On macOS all file descriptors for /dev/fd/N share the same + # file offset, reset the file offset after scanning the zipfile directory + # to not cause problems when some runs 'python3 /dev/fd/9 9<some_script' + start_offset = fp.tell() try: - fp.seek(-END_CENTRAL_DIR_SIZE, 2) - header_position = fp.tell() - buffer = fp.read(END_CENTRAL_DIR_SIZE) - except OSError: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - if len(buffer) != END_CENTRAL_DIR_SIZE: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - if buffer[:4] != STRING_END_ARCHIVE: - # Bad: End of Central Dir signature # Check if there's a comment. try: fp.seek(0, 2) @@ -423,98 +362,209 @@ def _read_directory(archive): except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - max_comment_start = max(file_size - MAX_COMMENT_LEN - - END_CENTRAL_DIR_SIZE, 0) + max_comment_plus_dirs_size = ( + MAX_COMMENT_LEN + END_CENTRAL_DIR_SIZE + + END_CENTRAL_DIR_SIZE_64 + END_CENTRAL_DIR_LOCATOR_SIZE_64) + max_comment_start = max(file_size - max_comment_plus_dirs_size, 0) try: fp.seek(max_comment_start) - data = fp.read() + data = fp.read(max_comment_plus_dirs_size) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) pos = data.rfind(STRING_END_ARCHIVE) - if pos < 0: + pos64 = data.rfind(STRING_END_ZIP_64) + + if (pos64 >= 0 and pos64+END_CENTRAL_DIR_SIZE_64+END_CENTRAL_DIR_LOCATOR_SIZE_64==pos): + # Zip64 at "correct" offset from standard EOCD + buffer = data[pos64:pos64 + END_CENTRAL_DIR_SIZE_64] + if len(buffer) != END_CENTRAL_DIR_SIZE_64: + raise ZipImportError( + f"corrupt Zip64 file: Expected {END_CENTRAL_DIR_SIZE_64} byte " + f"zip64 central directory, but read {len(buffer)} bytes.", + path=archive) + header_position = file_size - len(data) + pos64 + + central_directory_size = _unpack_uint64(buffer[40:48]) + central_directory_position = _unpack_uint64(buffer[48:56]) + num_entries = _unpack_uint64(buffer[24:32]) + elif pos >= 0: + buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] + if len(buffer) != END_CENTRAL_DIR_SIZE: + raise ZipImportError(f"corrupt Zip file: {archive!r}", + path=archive) + + header_position = file_size - len(data) + pos + + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + central_directory_size = _unpack_uint32(buffer[12:16]) + central_directory_position = _unpack_uint32(buffer[16:20]) + num_entries = _unpack_uint16(buffer[8:10]) + + # N.b. if someday you want to prefer the standard (non-zip64) EOCD, + # you need to adjust position by 76 for arc to be 0. + else: raise ZipImportError(f'not a Zip file: {archive!r}', path=archive) - buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] - if len(buffer) != END_CENTRAL_DIR_SIZE: - raise ZipImportError(f"corrupt Zip file: {archive!r}", - path=archive) - header_position = file_size - len(data) + pos - - header_size = _unpack_uint32(buffer[12:16]) - header_offset = _unpack_uint32(buffer[16:20]) - if header_position < header_size: - raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) - if header_position < header_offset: - raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) - header_position -= header_size - arc_offset = header_position - header_offset - if arc_offset < 0: - raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) - - files = {} - # Start of Central Directory - count = 0 - try: - fp.seek(header_position) - except OSError: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - while True: - buffer = fp.read(46) - if len(buffer) < 4: - raise EOFError('EOF read where not expected') - # Start of file header - if buffer[:4] != b'PK\x01\x02': - break # Bad: Central Dir File Header - if len(buffer) != 46: - raise EOFError('EOF read where not expected') - flags = _unpack_uint16(buffer[8:10]) - compress = _unpack_uint16(buffer[10:12]) - time = _unpack_uint16(buffer[12:14]) - date = _unpack_uint16(buffer[14:16]) - crc = _unpack_uint32(buffer[16:20]) - data_size = _unpack_uint32(buffer[20:24]) - file_size = _unpack_uint32(buffer[24:28]) - name_size = _unpack_uint16(buffer[28:30]) - extra_size = _unpack_uint16(buffer[30:32]) - comment_size = _unpack_uint16(buffer[32:34]) - file_offset = _unpack_uint32(buffer[42:46]) - header_size = name_size + extra_size + comment_size - if file_offset > header_offset: - raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) - file_offset += arc_offset + # Buffer now contains a valid EOCD, and header_position gives the + # starting position of it. + # XXX: These are cursory checks but are not as exact or strict as they + # could be. Checking the arc-adjusted value is probably good too. + if header_position < central_directory_size: + raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) + if header_position < central_directory_position: + raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) + header_position -= central_directory_size + # On just-a-zipfile these values are the same and arc_offset is zero; if + # the file has some bytes prepended, `arc_offset` is the number of such + # bytes. This is used for pex as well as self-extracting .exe. + arc_offset = header_position - central_directory_position + if arc_offset < 0: + raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) + + files = {} + # Start of Central Directory + count = 0 try: - name = fp.read(name_size) - except OSError: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - if len(name) != name_size: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - # On Windows, calling fseek to skip over the fields we don't use is - # slower than reading the data because fseek flushes stdio's - # internal buffers. See issue #8745. - try: - if len(fp.read(header_size - name_size)) != header_size - name_size: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + fp.seek(header_position) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + while True: + buffer = fp.read(46) + if len(buffer) < 4: + raise EOFError('EOF read where not expected') + # Start of file header + if buffer[:4] != b'PK\x01\x02': + if count != num_entries: + raise ZipImportError( + f"mismatched num_entries: {count} should be {num_entries} in {archive!r}", + path=archive, + ) + break # Bad: Central Dir File Header + if len(buffer) != 46: + raise EOFError('EOF read where not expected') + flags = _unpack_uint16(buffer[8:10]) + compress = _unpack_uint16(buffer[10:12]) + time = _unpack_uint16(buffer[12:14]) + date = _unpack_uint16(buffer[14:16]) + crc = _unpack_uint32(buffer[16:20]) + data_size = _unpack_uint32(buffer[20:24]) + file_size = _unpack_uint32(buffer[24:28]) + name_size = _unpack_uint16(buffer[28:30]) + extra_size = _unpack_uint16(buffer[30:32]) + comment_size = _unpack_uint16(buffer[32:34]) + file_offset = _unpack_uint32(buffer[42:46]) + header_size = name_size + extra_size + comment_size - if flags & 0x800: - # UTF-8 file names extension - name = name.decode() - else: - # Historical ZIP filename encoding try: - name = name.decode('ascii') - except UnicodeDecodeError: - name = name.decode('latin1').translate(cp437_table) - - name = name.replace('/', path_sep) - path = _bootstrap_external._path_join(archive, name) - t = (path, compress, data_size, file_size, file_offset, time, date, crc) - files[name] = t - count += 1 + name = fp.read(name_size) + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + if len(name) != name_size: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + # On Windows, calling fseek to skip over the fields we don't use is + # slower than reading the data because fseek flushes stdio's + # internal buffers. See issue #8745. + try: + extra_data_len = header_size - name_size + extra_data = memoryview(fp.read(extra_data_len)) + + if len(extra_data) != extra_data_len: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + + if flags & 0x800: + # UTF-8 file names extension + name = name.decode() + else: + # Historical ZIP filename encoding + try: + name = name.decode('ascii') + except UnicodeDecodeError: + name = name.decode('latin1').translate(cp437_table) + + name = name.replace('/', path_sep) + path = _bootstrap_external._path_join(archive, name) + + # Ordering matches unpacking below. + if ( + file_size == MAX_UINT32 or + data_size == MAX_UINT32 or + file_offset == MAX_UINT32 + ): + # need to decode extra_data looking for a zip64 extra (which might not + # be present) + while extra_data: + if len(extra_data) < 4: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + tag = _unpack_uint16(extra_data[:2]) + size = _unpack_uint16(extra_data[2:4]) + if len(extra_data) < 4 + size: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + if tag == ZIP64_EXTRA_TAG: + if (len(extra_data) - 4) % 8 != 0: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + num_extra_values = (len(extra_data) - 4) // 8 + if num_extra_values > 3: + raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) + import struct + values = list(struct.unpack_from(f"<{min(num_extra_values, 3)}Q", + extra_data, offset=4)) + + # N.b. Here be dragons: the ordering of these is different than + # the header fields, and it's really easy to get it wrong since + # naturally-occurring zips that use all 3 are >4GB + if file_size == MAX_UINT32: + file_size = values.pop(0) + if data_size == MAX_UINT32: + data_size = values.pop(0) + if file_offset == MAX_UINT32: + file_offset = values.pop(0) + + break + + # For a typical zip, this bytes-slicing only happens 2-3 times, on + # small data like timestamps and filesizes. + extra_data = extra_data[4+size:] + else: + _bootstrap._verbose_message( + "zipimport: suspected zip64 but no zip64 extra for {!r}", + path, + ) + # XXX These two statements seem swapped because `central_directory_position` + # is a position within the actual file, but `file_offset` (when compared) is + # as encoded in the entry, not adjusted for this file. + # N.b. this must be after we've potentially read the zip64 extra which can + # change `file_offset`. + if file_offset > central_directory_position: + raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) + file_offset += arc_offset + + t = (path, compress, data_size, file_size, file_offset, time, date, crc) + files[name] = t + count += 1 + finally: + fp.seek(start_offset) _bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive) + + # Add implicit directories. + count = 0 + for name in list(files): + while True: + i = name.rstrip(path_sep).rfind(path_sep) + if i < 0: + break + name = name[:i + 1] + if name in files: + break + files[name] = None + count += 1 + if count: + _bootstrap._verbose_message('zipimport: added {} implicit directories in {!r}', + count, archive) return files # During bootstrap, we may need to load the encodings @@ -648,7 +698,7 @@ def _unmarshal_code(self, pathname, fullpath, fullname, data): source_bytes = _get_pyc_source(self, fullpath) if source_bytes is not None: source_hash = _imp.source_hash( - _bootstrap_external._RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) @@ -708,7 +758,7 @@ def _get_mtime_and_size_of_source(self, path): # strip 'c' or 'o' from *.py[co] assert path[-1:] in ('c', 'o') path = path[:-1] - toc_entry = self._files[path] + toc_entry = self._get_files()[path] # fetch the time stamp of the .py file for comparison # with an embedded pyc time stamp time = toc_entry[5] @@ -728,7 +778,7 @@ def _get_pyc_source(self, path): path = path[:-1] try: - toc_entry = self._files[path] + toc_entry = self._get_files()[path] except KeyError: return None else: @@ -744,7 +794,7 @@ def _get_module_code(self, fullname): fullpath = path + suffix _bootstrap._verbose_message('trying {}{}{}', self.archive, path_sep, fullpath, verbosity=2) try: - toc_entry = self._files[fullpath] + toc_entry = self._get_files()[fullpath] except KeyError: pass else: diff --git a/Lib/zoneinfo/__init__.py b/Lib/zoneinfo/__init__.py new file mode 100644 index 00000000000..df2ae909f53 --- /dev/null +++ b/Lib/zoneinfo/__init__.py @@ -0,0 +1,34 @@ +__all__ = [ + "ZoneInfo", + "reset_tzpath", + "available_timezones", + "TZPATH", + "ZoneInfoNotFoundError", + "InvalidTZPathWarning", +] + +from . import _tzpath +from ._common import ZoneInfoNotFoundError + +try: + from _zoneinfo import ZoneInfo +except (ImportError, AttributeError): # pragma: nocover + # AttributeError: module 'datetime' has no attribute 'datetime_CAPI'. + # This happens when the '_datetime' module is not available and the + # pure Python implementation is used instead. + from ._zoneinfo import ZoneInfo + +reset_tzpath = _tzpath.reset_tzpath +available_timezones = _tzpath.available_timezones +InvalidTZPathWarning = _tzpath.InvalidTZPathWarning + + +def __getattr__(name): + if name == "TZPATH": + return _tzpath.TZPATH + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return sorted(list(globals()) + ["TZPATH"]) diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py new file mode 100644 index 00000000000..59f3f0ce853 --- /dev/null +++ b/Lib/zoneinfo/_common.py @@ -0,0 +1,168 @@ +import struct + + +def load_tzdata(key): + from importlib import resources + + components = key.split("/") + package_name = ".".join(["tzdata.zoneinfo"] + components[:-1]) + resource_name = components[-1] + + try: + path = resources.files(package_name).joinpath(resource_name) + # gh-85702: Prevent PermissionError on Windows + if path.is_dir(): + raise IsADirectoryError + return path.open("rb") + except (ImportError, FileNotFoundError, UnicodeEncodeError, IsADirectoryError): + # There are four types of exception that can be raised that all amount + # to "we cannot find this key": + # + # ImportError: If package_name doesn't exist (e.g. if tzdata is not + # installed, or if there's an error in the folder name like + # Amrica/New_York) + # FileNotFoundError: If resource_name doesn't exist in the package + # (e.g. Europe/Krasnoy) + # UnicodeEncodeError: If package_name or resource_name are not UTF-8, + # such as keys containing a surrogate character. + # IsADirectoryError: If package_name without a resource_name specified. + raise ZoneInfoNotFoundError(f"No time zone found with key {key}") + + +def load_data(fobj): + header = _TZifHeader.from_file(fobj) + + if header.version == 1: + time_size = 4 + time_type = "l" + else: + # Version 2+ has 64-bit integer transition times + time_size = 8 + time_type = "q" + + # Version 2+ also starts with a Version 1 header and data, which + # we need to skip now + skip_bytes = ( + header.timecnt * 5 # Transition times and types + + header.typecnt * 6 # Local time type records + + header.charcnt # Time zone designations + + header.leapcnt * 8 # Leap second records + + header.isstdcnt # Standard/wall indicators + + header.isutcnt # UT/local indicators + ) + + fobj.seek(skip_bytes, 1) + + # Now we need to read the second header, which is not the same + # as the first + header = _TZifHeader.from_file(fobj) + + typecnt = header.typecnt + timecnt = header.timecnt + charcnt = header.charcnt + + # The data portion starts with timecnt transitions and indices + if timecnt: + trans_list_utc = struct.unpack( + f">{timecnt}{time_type}", fobj.read(timecnt * time_size) + ) + trans_idx = struct.unpack(f">{timecnt}B", fobj.read(timecnt)) + else: + trans_list_utc = () + trans_idx = () + + # Read the ttinfo struct, (utoff, isdst, abbrind) + if typecnt: + utcoff, isdst, abbrind = zip( + *(struct.unpack(">lbb", fobj.read(6)) for i in range(typecnt)) + ) + else: + utcoff = () + isdst = () + abbrind = () + + # Now read the abbreviations. They are null-terminated strings, indexed + # not by position in the array but by position in the unsplit + # abbreviation string. I suppose this makes more sense in C, which uses + # null to terminate the strings, but it's inconvenient here... + abbr_vals = {} + abbr_chars = fobj.read(charcnt) + + def get_abbr(idx): + # Gets a string starting at idx and running until the next \x00 + # + # We cannot pre-populate abbr_vals by splitting on \x00 because there + # are some zones that use subsets of longer abbreviations, like so: + # + # LMT\x00AHST\x00HDT\x00 + # + # Where the idx to abbr mapping should be: + # + # {0: "LMT", 4: "AHST", 5: "HST", 9: "HDT"} + if idx not in abbr_vals: + span_end = abbr_chars.find(b"\x00", idx) + abbr_vals[idx] = abbr_chars[idx:span_end].decode() + + return abbr_vals[idx] + + abbr = tuple(get_abbr(idx) for idx in abbrind) + + # The remainder of the file consists of leap seconds (currently unused) and + # the standard/wall and ut/local indicators, which are metadata we don't need. + # In version 2 files, we need to skip the unnecessary data to get at the TZ string: + if header.version >= 2: + # Each leap second record has size (time_size + 4) + skip_bytes = header.isutcnt + header.isstdcnt + header.leapcnt * 12 + fobj.seek(skip_bytes, 1) + + c = fobj.read(1) # Should be \n + assert c == b"\n", c + + line = fobj.readline() + if not line.endswith(b"\n"): + raise ValueError("Invalid TZif file: unexpected end of file") + tz_str = line.rstrip(b"\n") + else: + tz_str = None + + return trans_idx, trans_list_utc, utcoff, isdst, abbr, tz_str + + +class _TZifHeader: + __slots__ = [ + "version", + "isutcnt", + "isstdcnt", + "leapcnt", + "timecnt", + "typecnt", + "charcnt", + ] + + def __init__(self, *args): + for attr, val in zip(self.__slots__, args, strict=True): + setattr(self, attr, val) + + @classmethod + def from_file(cls, stream): + # The header starts with a 4-byte "magic" value + if stream.read(4) != b"TZif": + raise ValueError("Invalid TZif file: magic not found") + + _version = stream.read(1) + if _version == b"\x00": + version = 1 + else: + version = int(_version) + stream.read(15) + + args = (version,) + + # Slots are defined in the order that the bytes are arranged + args = args + struct.unpack(">6l", stream.read(24)) + + return cls(*args) + + +class ZoneInfoNotFoundError(KeyError): + """Exception raised when a ZoneInfo key is not found.""" diff --git a/Lib/zoneinfo/_tzpath.py b/Lib/zoneinfo/_tzpath.py new file mode 100644 index 00000000000..177d32c35ef --- /dev/null +++ b/Lib/zoneinfo/_tzpath.py @@ -0,0 +1,189 @@ +import os +import sysconfig + + +def _reset_tzpath(to=None, stacklevel=4): + global TZPATH + + tzpaths = to + if tzpaths is not None: + if isinstance(tzpaths, (str, bytes)): + raise TypeError( + f"tzpaths must be a list or tuple, " + + f"not {type(tzpaths)}: {tzpaths!r}" + ) + + tzpaths = [os.fspath(p) for p in tzpaths] + if not all(isinstance(p, str) for p in tzpaths): + raise TypeError( + "All elements of a tzpath sequence must be strings or " + "os.PathLike objects which convert to strings." + ) + + if not all(map(os.path.isabs, tzpaths)): + raise ValueError(_get_invalid_paths_message(tzpaths)) + base_tzpath = tzpaths + else: + env_var = os.environ.get("PYTHONTZPATH", None) + if env_var is None: + env_var = sysconfig.get_config_var("TZPATH") + base_tzpath = _parse_python_tzpath(env_var, stacklevel) + + TZPATH = tuple(base_tzpath) + + +def reset_tzpath(to=None): + """Reset global TZPATH.""" + # We need `_reset_tzpath` helper function because it produces a warning, + # it is used as both a module-level call and a public API. + # This is how we equalize the stacklevel for both calls. + _reset_tzpath(to) + + +def _parse_python_tzpath(env_var, stacklevel): + if not env_var: + return () + + raw_tzpath = env_var.split(os.pathsep) + new_tzpath = tuple(filter(os.path.isabs, raw_tzpath)) + + # If anything has been filtered out, we will warn about it + if len(new_tzpath) != len(raw_tzpath): + import warnings + + msg = _get_invalid_paths_message(raw_tzpath) + + warnings.warn( + "Invalid paths specified in PYTHONTZPATH environment variable. " + + msg, + InvalidTZPathWarning, + stacklevel=stacklevel, + ) + + return new_tzpath + + +def _get_invalid_paths_message(tzpaths): + invalid_paths = (path for path in tzpaths if not os.path.isabs(path)) + + prefix = "\n " + indented_str = prefix + prefix.join(invalid_paths) + + return ( + "Paths should be absolute but found the following relative paths:" + + indented_str + ) + + +def find_tzfile(key): + """Retrieve the path to a TZif file from a key.""" + _validate_tzfile_path(key) + for search_path in TZPATH: + filepath = os.path.join(search_path, key) + if os.path.isfile(filepath): + return filepath + + return None + + +_TEST_PATH = os.path.normpath(os.path.join("_", "_"))[:-1] + + +def _validate_tzfile_path(path, _base=_TEST_PATH): + if os.path.isabs(path): + raise ValueError( + f"ZoneInfo keys may not be absolute paths, got: {path}" + ) + + # We only care about the kinds of path normalizations that would change the + # length of the key - e.g. a/../b -> a/b, or a/b/ -> a/b. On Windows, + # normpath will also change from a/b to a\b, but that would still preserve + # the length. + new_path = os.path.normpath(path) + if len(new_path) != len(path): + raise ValueError( + f"ZoneInfo keys must be normalized relative paths, got: {path}" + ) + + resolved = os.path.normpath(os.path.join(_base, new_path)) + if not resolved.startswith(_base): + raise ValueError( + f"ZoneInfo keys must refer to subdirectories of TZPATH, got: {path}" + ) + + +del _TEST_PATH + + +def available_timezones(): + """Returns a set containing all available time zones. + + .. caution:: + + This may attempt to open a large number of files, since the best way to + determine if a given file on the time zone search path is to open it + and check for the "magic string" at the beginning. + """ + from importlib import resources + + valid_zones = set() + + # Start with loading from the tzdata package if it exists: this has a + # pre-assembled list of zones that only requires opening one file. + try: + zones_file = resources.files("tzdata").joinpath("zones") + with zones_file.open("r", encoding="utf-8") as f: + for zone in f: + zone = zone.strip() + if zone: + valid_zones.add(zone) + except (ImportError, FileNotFoundError): + pass + + def valid_key(fpath): + try: + with open(fpath, "rb") as f: + return f.read(4) == b"TZif" + except Exception: # pragma: nocover + return False + + for tz_root in TZPATH: + if not os.path.exists(tz_root): + continue + + for root, dirnames, files in os.walk(tz_root): + if root == tz_root: + # right/ and posix/ are special directories and shouldn't be + # included in the output of available zones + if "right" in dirnames: + dirnames.remove("right") + if "posix" in dirnames: + dirnames.remove("posix") + + for file in files: + fpath = os.path.join(root, file) + + key = os.path.relpath(fpath, start=tz_root) + if os.sep != "/": # pragma: nocover + key = key.replace(os.sep, "/") + + if not key or key in valid_zones: + continue + + if valid_key(fpath): + valid_zones.add(key) + + if "posixrules" in valid_zones: + # posixrules is a special symlink-only time zone where it exists, it + # should not be included in the output + valid_zones.remove("posixrules") + + return valid_zones + + +class InvalidTZPathWarning(RuntimeWarning): + """Warning raised if an invalid path is specified in PYTHONTZPATH.""" + + +TZPATH = () +_reset_tzpath(stacklevel=5) diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py new file mode 100644 index 00000000000..3ffdb4c8371 --- /dev/null +++ b/Lib/zoneinfo/_zoneinfo.py @@ -0,0 +1,772 @@ +import bisect +import calendar +import collections +import functools +import re +import weakref +from datetime import datetime, timedelta, tzinfo + +from . import _common, _tzpath + +EPOCH = datetime(1970, 1, 1) +EPOCHORDINAL = datetime(1970, 1, 1).toordinal() + +# It is relatively expensive to construct new timedelta objects, and in most +# cases we're looking at the same deltas, like integer numbers of hours, etc. +# To improve speed and memory use, we'll keep a dictionary with references +# to the ones we've already used so far. +# +# Loading every time zone in the 2020a version of the time zone database +# requires 447 timedeltas, which requires approximately the amount of space +# that ZoneInfo("America/New_York") with 236 transitions takes up, so we will +# set the cache size to 512 so that in the common case we always get cache +# hits, but specifically crafted ZoneInfo objects don't leak arbitrary amounts +# of memory. +@functools.lru_cache(maxsize=512) +def _load_timedelta(seconds): + return timedelta(seconds=seconds) + + +class ZoneInfo(tzinfo): + _strong_cache_size = 8 + _strong_cache = collections.OrderedDict() + _weak_cache = weakref.WeakValueDictionary() + __module__ = "zoneinfo" + + def __init_subclass__(cls): + cls._strong_cache = collections.OrderedDict() + cls._weak_cache = weakref.WeakValueDictionary() + + def __new__(cls, key): + instance = cls._weak_cache.get(key, None) + if instance is None: + instance = cls._weak_cache.setdefault(key, cls._new_instance(key)) + instance._from_cache = True + + # Update the "strong" cache + cls._strong_cache[key] = cls._strong_cache.pop(key, instance) + + if len(cls._strong_cache) > cls._strong_cache_size: + cls._strong_cache.popitem(last=False) + + return instance + + @classmethod + def no_cache(cls, key): + obj = cls._new_instance(key) + obj._from_cache = False + + return obj + + @classmethod + def _new_instance(cls, key): + obj = super().__new__(cls) + obj._key = key + obj._file_path = obj._find_tzfile(key) + + if obj._file_path is not None: + file_obj = open(obj._file_path, "rb") + else: + file_obj = _common.load_tzdata(key) + + with file_obj as f: + obj._load_file(f) + + return obj + + @classmethod + def from_file(cls, file_obj, /, key=None): + obj = super().__new__(cls) + obj._key = key + obj._file_path = None + obj._load_file(file_obj) + obj._file_repr = repr(file_obj) + + # Disable pickling for objects created from files + obj.__reduce__ = obj._file_reduce + + return obj + + @classmethod + def clear_cache(cls, *, only_keys=None): + if only_keys is not None: + for key in only_keys: + cls._weak_cache.pop(key, None) + cls._strong_cache.pop(key, None) + + else: + cls._weak_cache.clear() + cls._strong_cache.clear() + + @property + def key(self): + return self._key + + def utcoffset(self, dt): + return self._find_trans(dt).utcoff + + def dst(self, dt): + return self._find_trans(dt).dstoff + + def tzname(self, dt): + return self._find_trans(dt).tzname + + def fromutc(self, dt): + """Convert from datetime in UTC to datetime in local time""" + + if not isinstance(dt, datetime): + raise TypeError("fromutc() requires a datetime argument") + if dt.tzinfo is not self: + raise ValueError("dt.tzinfo is not self") + + timestamp = self._get_local_timestamp(dt) + num_trans = len(self._trans_utc) + + if num_trans >= 1 and timestamp < self._trans_utc[0]: + tti = self._tti_before + fold = 0 + elif ( + num_trans == 0 or timestamp > self._trans_utc[-1] + ) and not isinstance(self._tz_after, _ttinfo): + tti, fold = self._tz_after.get_trans_info_fromutc( + timestamp, dt.year + ) + elif num_trans == 0: + tti = self._tz_after + fold = 0 + else: + idx = bisect.bisect_right(self._trans_utc, timestamp) + + if num_trans > 1 and timestamp >= self._trans_utc[1]: + tti_prev, tti = self._ttinfos[idx - 2 : idx] + elif timestamp > self._trans_utc[-1]: + tti_prev = self._ttinfos[-1] + tti = self._tz_after + else: + tti_prev = self._tti_before + tti = self._ttinfos[0] + + # Detect fold + shift = tti_prev.utcoff - tti.utcoff + fold = shift.total_seconds() > timestamp - self._trans_utc[idx - 1] + dt += tti.utcoff + if fold: + return dt.replace(fold=1) + else: + return dt + + def _find_trans(self, dt): + if dt is None: + if self._fixed_offset: + return self._tz_after + else: + return _NO_TTINFO + + ts = self._get_local_timestamp(dt) + + lt = self._trans_local[dt.fold] + + num_trans = len(lt) + + if num_trans and ts < lt[0]: + return self._tti_before + elif not num_trans or ts > lt[-1]: + if isinstance(self._tz_after, _TZStr): + return self._tz_after.get_trans_info(ts, dt.year, dt.fold) + else: + return self._tz_after + else: + # idx is the transition that occurs after this timestamp, so we + # subtract off 1 to get the current ttinfo + idx = bisect.bisect_right(lt, ts) - 1 + assert idx >= 0 + return self._ttinfos[idx] + + def _get_local_timestamp(self, dt): + return ( + (dt.toordinal() - EPOCHORDINAL) * 86400 + + dt.hour * 3600 + + dt.minute * 60 + + dt.second + ) + + def __str__(self): + if self._key is not None: + return f"{self._key}" + else: + return repr(self) + + def __repr__(self): + if self._key is not None: + return f"{self.__class__.__name__}(key={self._key!r})" + else: + return f"{self.__class__.__name__}.from_file({self._file_repr})" + + def __reduce__(self): + return (self.__class__._unpickle, (self._key, self._from_cache)) + + def _file_reduce(self): + import pickle + + raise pickle.PicklingError( + "Cannot pickle a ZoneInfo file created from a file stream." + ) + + @classmethod + def _unpickle(cls, key, from_cache, /): + if from_cache: + return cls(key) + else: + return cls.no_cache(key) + + def _find_tzfile(self, key): + return _tzpath.find_tzfile(key) + + def _load_file(self, fobj): + # Retrieve all the data as it exists in the zoneinfo file + trans_idx, trans_utc, utcoff, isdst, abbr, tz_str = _common.load_data( + fobj + ) + + # Infer the DST offsets (needed for .dst()) from the data + dstoff = self._utcoff_to_dstoff(trans_idx, utcoff, isdst) + + # Convert all the transition times (UTC) into "seconds since 1970-01-01 local time" + trans_local = self._ts_to_local(trans_idx, trans_utc, utcoff) + + # Construct `_ttinfo` objects for each transition in the file + _ttinfo_list = [ + _ttinfo( + _load_timedelta(utcoffset), _load_timedelta(dstoffset), tzname + ) + for utcoffset, dstoffset, tzname in zip(utcoff, dstoff, abbr) + ] + + self._trans_utc = trans_utc + self._trans_local = trans_local + self._ttinfos = [_ttinfo_list[idx] for idx in trans_idx] + + # Find the first non-DST transition + for i in range(len(isdst)): + if not isdst[i]: + self._tti_before = _ttinfo_list[i] + break + else: + if self._ttinfos: + self._tti_before = self._ttinfos[0] + else: + self._tti_before = None + + # Set the "fallback" time zone + if tz_str is not None and tz_str != b"": + self._tz_after = _parse_tz_str(tz_str.decode()) + else: + if not self._ttinfos and not _ttinfo_list: + raise ValueError("No time zone information found.") + + if self._ttinfos: + self._tz_after = self._ttinfos[-1] + else: + self._tz_after = _ttinfo_list[-1] + + # Determine if this is a "fixed offset" zone, meaning that the output + # of the utcoffset, dst and tzname functions does not depend on the + # specific datetime passed. + # + # We make three simplifying assumptions here: + # + # 1. If _tz_after is not a _ttinfo, it has transitions that might + # actually occur (it is possible to construct TZ strings that + # specify STD and DST but no transitions ever occur, such as + # AAA0BBB,0/0,J365/25). + # 2. If _ttinfo_list contains more than one _ttinfo object, the objects + # represent different offsets. + # 3. _ttinfo_list contains no unused _ttinfos (in which case an + # otherwise fixed-offset zone with extra _ttinfos defined may + # appear to *not* be a fixed offset zone). + # + # Violations to these assumptions would be fairly exotic, and exotic + # zones should almost certainly not be used with datetime.time (the + # only thing that would be affected by this). + if len(_ttinfo_list) > 1 or not isinstance(self._tz_after, _ttinfo): + self._fixed_offset = False + elif not _ttinfo_list: + self._fixed_offset = True + else: + self._fixed_offset = _ttinfo_list[0] == self._tz_after + + @staticmethod + def _utcoff_to_dstoff(trans_idx, utcoffsets, isdsts): + # Now we must transform our ttis and abbrs into `_ttinfo` objects, + # but there is an issue: .dst() must return a timedelta with the + # difference between utcoffset() and the "standard" offset, but + # the "base offset" and "DST offset" are not encoded in the file; + # we can infer what they are from the isdst flag, but it is not + # sufficient to just look at the last standard offset, because + # occasionally countries will shift both DST offset and base offset. + + typecnt = len(isdsts) + dstoffs = [0] * typecnt # Provisionally assign all to 0. + dst_cnt = sum(isdsts) + dst_found = 0 + + for i in range(1, len(trans_idx)): + if dst_cnt == dst_found: + break + + idx = trans_idx[i] + + dst = isdsts[idx] + + # We're only going to look at daylight saving time + if not dst: + continue + + # Skip any offsets that have already been assigned + if dstoffs[idx] != 0: + continue + + dstoff = 0 + utcoff = utcoffsets[idx] + + comp_idx = trans_idx[i - 1] + + if not isdsts[comp_idx]: + dstoff = utcoff - utcoffsets[comp_idx] + + if not dstoff and idx < (typecnt - 1): + comp_idx = trans_idx[i + 1] + + # If the following transition is also DST and we couldn't + # find the DST offset by this point, we're going to have to + # skip it and hope this transition gets assigned later + if isdsts[comp_idx]: + continue + + dstoff = utcoff - utcoffsets[comp_idx] + + if dstoff: + dst_found += 1 + dstoffs[idx] = dstoff + else: + # If we didn't find a valid value for a given index, we'll end up + # with dstoff = 0 for something where `isdst=1`. This is obviously + # wrong - one hour will be a much better guess than 0 + for idx in range(typecnt): + if not dstoffs[idx] and isdsts[idx]: + dstoffs[idx] = 3600 + + return dstoffs + + @staticmethod + def _ts_to_local(trans_idx, trans_list_utc, utcoffsets): + """Generate number of seconds since 1970 *in the local time*. + + This is necessary to easily find the transition times in local time""" + if not trans_list_utc: + return [[], []] + + # Start with the timestamps and modify in-place + trans_list_wall = [list(trans_list_utc), list(trans_list_utc)] + + if len(utcoffsets) > 1: + offset_0 = utcoffsets[0] + offset_1 = utcoffsets[trans_idx[0]] + if offset_1 > offset_0: + offset_1, offset_0 = offset_0, offset_1 + else: + offset_0 = offset_1 = utcoffsets[0] + + trans_list_wall[0][0] += offset_0 + trans_list_wall[1][0] += offset_1 + + for i in range(1, len(trans_idx)): + offset_0 = utcoffsets[trans_idx[i - 1]] + offset_1 = utcoffsets[trans_idx[i]] + + if offset_1 > offset_0: + offset_1, offset_0 = offset_0, offset_1 + + trans_list_wall[0][i] += offset_0 + trans_list_wall[1][i] += offset_1 + + return trans_list_wall + + +class _ttinfo: + __slots__ = ["utcoff", "dstoff", "tzname"] + + def __init__(self, utcoff, dstoff, tzname): + self.utcoff = utcoff + self.dstoff = dstoff + self.tzname = tzname + + def __eq__(self, other): + return ( + self.utcoff == other.utcoff + and self.dstoff == other.dstoff + and self.tzname == other.tzname + ) + + def __repr__(self): # pragma: nocover + return ( + f"{self.__class__.__name__}" + + f"({self.utcoff}, {self.dstoff}, {self.tzname})" + ) + + +_NO_TTINFO = _ttinfo(None, None, None) + + +class _TZStr: + __slots__ = ( + "std", + "dst", + "start", + "end", + "get_trans_info", + "get_trans_info_fromutc", + "dst_diff", + ) + + def __init__( + self, std_abbr, std_offset, dst_abbr, dst_offset, start=None, end=None + ): + self.dst_diff = dst_offset - std_offset + std_offset = _load_timedelta(std_offset) + self.std = _ttinfo( + utcoff=std_offset, dstoff=_load_timedelta(0), tzname=std_abbr + ) + + self.start = start + self.end = end + + dst_offset = _load_timedelta(dst_offset) + delta = _load_timedelta(self.dst_diff) + self.dst = _ttinfo(utcoff=dst_offset, dstoff=delta, tzname=dst_abbr) + + # These are assertions because the constructor should only be called + # by functions that would fail before passing start or end + assert start is not None, "No transition start specified" + assert end is not None, "No transition end specified" + + self.get_trans_info = self._get_trans_info + self.get_trans_info_fromutc = self._get_trans_info_fromutc + + def transitions(self, year): + start = self.start.year_to_epoch(year) + end = self.end.year_to_epoch(year) + return start, end + + def _get_trans_info(self, ts, year, fold): + """Get the information about the current transition - tti""" + start, end = self.transitions(year) + + # With fold = 0, the period (denominated in local time) with the + # smaller offset starts at the end of the gap and ends at the end of + # the fold; with fold = 1, it runs from the start of the gap to the + # beginning of the fold. + # + # So in order to determine the DST boundaries we need to know both + # the fold and whether DST is positive or negative (rare), and it + # turns out that this boils down to fold XOR is_positive. + if fold == (self.dst_diff >= 0): + end -= self.dst_diff + else: + start += self.dst_diff + + if start < end: + isdst = start <= ts < end + else: + isdst = not (end <= ts < start) + + return self.dst if isdst else self.std + + def _get_trans_info_fromutc(self, ts, year): + start, end = self.transitions(year) + start -= self.std.utcoff.total_seconds() + end -= self.dst.utcoff.total_seconds() + + if start < end: + isdst = start <= ts < end + else: + isdst = not (end <= ts < start) + + # For positive DST, the ambiguous period is one dst_diff after the end + # of DST; for negative DST, the ambiguous period is one dst_diff before + # the start of DST. + if self.dst_diff > 0: + ambig_start = end + ambig_end = end + self.dst_diff + else: + ambig_start = start + ambig_end = start - self.dst_diff + + fold = ambig_start <= ts < ambig_end + + return (self.dst if isdst else self.std, fold) + + +def _post_epoch_days_before_year(year): + """Get the number of days between 1970-01-01 and YEAR-01-01""" + y = year - 1 + return y * 365 + y // 4 - y // 100 + y // 400 - EPOCHORDINAL + + +class _DayOffset: + __slots__ = ["d", "julian", "hour", "minute", "second"] + + def __init__(self, d, julian, hour=2, minute=0, second=0): + min_day = 0 + julian # convert bool to int + if not min_day <= d <= 365: + raise ValueError(f"d must be in [{min_day}, 365], not: {d}") + + self.d = d + self.julian = julian + self.hour = hour + self.minute = minute + self.second = second + + def year_to_epoch(self, year): + days_before_year = _post_epoch_days_before_year(year) + + d = self.d + if self.julian and d >= 59 and calendar.isleap(year): + d += 1 + + epoch = (days_before_year + d) * 86400 + epoch += self.hour * 3600 + self.minute * 60 + self.second + + return epoch + + +class _CalendarOffset: + __slots__ = ["m", "w", "d", "hour", "minute", "second"] + + _DAYS_BEFORE_MONTH = ( + -1, + 0, + 31, + 59, + 90, + 120, + 151, + 181, + 212, + 243, + 273, + 304, + 334, + ) + + def __init__(self, m, w, d, hour=2, minute=0, second=0): + if not 1 <= m <= 12: + raise ValueError("m must be in [1, 12]") + + if not 1 <= w <= 5: + raise ValueError("w must be in [1, 5]") + + if not 0 <= d <= 6: + raise ValueError("d must be in [0, 6]") + + self.m = m + self.w = w + self.d = d + self.hour = hour + self.minute = minute + self.second = second + + @classmethod + def _ymd2ord(cls, year, month, day): + return ( + _post_epoch_days_before_year(year) + + cls._DAYS_BEFORE_MONTH[month] + + (month > 2 and calendar.isleap(year)) + + day + ) + + # TODO: These are not actually epoch dates as they are expressed in local time + def year_to_epoch(self, year): + """Calculates the datetime of the occurrence from the year""" + # We know year and month, we need to convert w, d into day of month + # + # Week 1 is the first week in which day `d` (where 0 = Sunday) appears. + # Week 5 represents the last occurrence of day `d`, so we need to know + # the range of the month. + first_day, days_in_month = calendar.monthrange(year, self.m) + + # This equation seems magical, so I'll break it down: + # 1. calendar says 0 = Monday, POSIX says 0 = Sunday + # so we need first_day + 1 to get 1 = Monday -> 7 = Sunday, + # which is still equivalent because this math is mod 7 + # 2. Get first day - desired day mod 7: -1 % 7 = 6, so we don't need + # to do anything to adjust negative numbers. + # 3. Add 1 because month days are a 1-based index. + month_day = (self.d - (first_day + 1)) % 7 + 1 + + # Now use a 0-based index version of `w` to calculate the w-th + # occurrence of `d` + month_day += (self.w - 1) * 7 + + # month_day will only be > days_in_month if w was 5, and `w` means + # "last occurrence of `d`", so now we just check if we over-shot the + # end of the month and if so knock off 1 week. + if month_day > days_in_month: + month_day -= 7 + + ordinal = self._ymd2ord(year, self.m, month_day) + epoch = ordinal * 86400 + epoch += self.hour * 3600 + self.minute * 60 + self.second + return epoch + + +def _parse_tz_str(tz_str): + # The tz string has the format: + # + # std[offset[dst[offset],start[/time],end[/time]]] + # + # std and dst must be 3 or more characters long and must not contain + # a leading colon, embedded digits, commas, nor a plus or minus signs; + # The spaces between "std" and "offset" are only for display and are + # not actually present in the string. + # + # The format of the offset is ``[+|-]hh[:mm[:ss]]`` + + offset_str, *start_end_str = tz_str.split(",", 1) + + parser_re = re.compile( + r""" + (?P<std>[^<0-9:.+-]+|<[a-zA-Z0-9+-]+>) + (?: + (?P<stdoff>[+-]?\d{1,3}(?::\d{2}(?::\d{2})?)?) + (?: + (?P<dst>[^0-9:.+-]+|<[a-zA-Z0-9+-]+>) + (?P<dstoff>[+-]?\d{1,3}(?::\d{2}(?::\d{2})?)?)? + )? # dst + )? # stdoff + """, + re.ASCII|re.VERBOSE + ) + + m = parser_re.fullmatch(offset_str) + + if m is None: + raise ValueError(f"{tz_str} is not a valid TZ string") + + std_abbr = m.group("std") + dst_abbr = m.group("dst") + dst_offset = None + + std_abbr = std_abbr.strip("<>") + + if dst_abbr: + dst_abbr = dst_abbr.strip("<>") + + if std_offset := m.group("stdoff"): + try: + std_offset = _parse_tz_delta(std_offset) + except ValueError as e: + raise ValueError(f"Invalid STD offset in {tz_str}") from e + else: + std_offset = 0 + + if dst_abbr is not None: + if dst_offset := m.group("dstoff"): + try: + dst_offset = _parse_tz_delta(dst_offset) + except ValueError as e: + raise ValueError(f"Invalid DST offset in {tz_str}") from e + else: + dst_offset = std_offset + 3600 + + if not start_end_str: + raise ValueError(f"Missing transition rules: {tz_str}") + + start_end_strs = start_end_str[0].split(",", 1) + try: + start, end = (_parse_dst_start_end(x) for x in start_end_strs) + except ValueError as e: + raise ValueError(f"Invalid TZ string: {tz_str}") from e + + return _TZStr(std_abbr, std_offset, dst_abbr, dst_offset, start, end) + elif start_end_str: + raise ValueError(f"Transition rule present without DST: {tz_str}") + else: + # This is a static ttinfo, don't return _TZStr + return _ttinfo( + _load_timedelta(std_offset), _load_timedelta(0), std_abbr + ) + + +def _parse_dst_start_end(dststr): + date, *time = dststr.split("/", 1) + type = date[:1] + if type == "M": + n_is_julian = False + m = re.fullmatch(r"M(\d{1,2})\.(\d).(\d)", date, re.ASCII) + if m is None: + raise ValueError(f"Invalid dst start/end date: {dststr}") + date_offset = tuple(map(int, m.groups())) + offset = _CalendarOffset(*date_offset) + else: + if type == "J": + n_is_julian = True + date = date[1:] + else: + n_is_julian = False + + doy = int(date) + offset = _DayOffset(doy, n_is_julian) + + if time: + offset.hour, offset.minute, offset.second = _parse_transition_time(time[0]) + + return offset + + +def _parse_transition_time(time_str): + match = re.fullmatch( + r"(?P<sign>[+-])?(?P<h>\d{1,3})(:(?P<m>\d{2})(:(?P<s>\d{2}))?)?", + time_str, + re.ASCII + ) + if match is None: + raise ValueError(f"Invalid time: {time_str}") + + h, m, s = (int(v or 0) for v in match.group("h", "m", "s")) + + if h > 167: + raise ValueError( + f"Hour must be in [0, 167]: {time_str}" + ) + + if match.group("sign") == "-": + h, m, s = -h, -m, -s + + return h, m, s + + +def _parse_tz_delta(tz_delta): + match = re.fullmatch( + r"(?P<sign>[+-])?(?P<h>\d{1,3})(:(?P<m>\d{2})(:(?P<s>\d{2}))?)?", + tz_delta, + re.ASCII + ) + # Anything passed to this function should already have hit an equivalent + # regular expression to find the section to parse. + assert match is not None, tz_delta + + h, m, s = (int(v or 0) for v in match.group("h", "m", "s")) + + total = h * 3600 + m * 60 + s + + if h > 24: + raise ValueError( + f"Offset hours must be in [0, 24]: {tz_delta}" + ) + + # Yes, +5 maps to an offset of -5h + if match.group("sign") != "-": + total = -total + + return total diff --git a/README.md b/README.md index c4d1fc9a7db..6949c6e66e2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # [RustPython](https://rustpython.github.io/) -A Python-3 (CPython >= 3.12.0) Interpreter written in Rust :snake: :scream: +A Python-3 (CPython >= 3.14.0) Interpreter written in Rust :snake: :scream: :metal:. [![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI) @@ -13,7 +13,6 @@ A Python-3 (CPython >= 3.12.0) Interpreter written in Rust :snake: :scream: [![docs.rs](https://docs.rs/rustpython/badge.svg)](https://docs.rs/rustpython/) [![Crates.io](https://img.shields.io/crates/v/rustpython)](https://crates.io/crates/rustpython) [![dependency status](https://deps.rs/crate/rustpython/0.1.1/status.svg)](https://deps.rs/crate/rustpython/0.1.1) -[![WAPM package](https://wapm.io/package/rustpython/badge.svg?style=flat)](https://wapm.io/package/rustpython) [![Open in Gitpod](https://img.shields.io/static/v1?label=Open%20in&message=Gitpod&color=1aa6e4&logo=gitpod)](https://gitpod.io#https://github.com/RustPython/RustPython) ## Usage @@ -32,6 +31,11 @@ To build RustPython locally, first, clone the source code: git clone https://github.com/RustPython/RustPython ``` +RustPython uses symlinks to manage python libraries in `Lib/`. If on windows, running the following helps: +```bash +git config core.symlinks true +``` + Then you can change into the RustPython directory and run the demo (Note: `--release` is needed to prevent stack overflow on Windows): @@ -56,22 +60,35 @@ NOTE: For windows users, please set `RUSTPYTHONPATH` environment variable as `Li You can also install and run RustPython with the following: ```bash -$ cargo install --git https://github.com/RustPython/RustPython +$ cargo install --git https://github.com/RustPython/RustPython rustpython $ rustpython Welcome to the magnificent Rust Python interpreter >>>>> ``` +### venv + +Because RustPython currently doesn't provide a well-packaged installation, using venv helps to use pip easier. + +```sh +$ rustpython -m venv <your_env_name> +$ . <your_env_name>/bin/activate +$ python # now `python` is the alias of the RustPython for the new env +``` + +### PIP + If you'd like to make https requests, you can enable the `ssl` feature, which also lets you install the `pip` package manager. Note that on Windows, you may need to install OpenSSL, or you can enable the `ssl-vendor` feature instead, which compiles OpenSSL for you but requires a C compiler, perl, and `make`. +OpenSSL version 3 is expected and tested in CI. Older versions may not work. Once you've installed rustpython with SSL support, you can install pip by running: ```bash -cargo install --git https://github.com/RustPython/RustPython --features ssl +cargo install --git https://github.com/RustPython/RustPython rustpython --install-pip ``` @@ -83,6 +100,13 @@ conda install rustpython -c conda-forge rustpython ``` +### SSL provider + +For HTTPS requests, `ssl-rustls` feature is enabled by default. You can replace it with `ssl-openssl` feature if your environment requires OpenSSL. +Note that to use OpenSSL on Windows, you may need to install OpenSSL, or you can enable the `ssl-vendor` feature instead, +which compiles OpenSSL for you but requires a C compiler, perl, and `make`. +OpenSSL version 3 is expected and tested in CI. Older versions may not work. + ### WASI You can compile RustPython to a standalone WebAssembly WASI module so it can run anywhere. @@ -90,13 +114,13 @@ You can compile RustPython to a standalone WebAssembly WASI module so it can run Build ```bash -cargo build --target wasm32-wasi --no-default-features --features freeze-stdlib,stdlib --release +cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release ``` Run by wasmer ```bash -wasmer run --dir . target/wasm32-wasi/release/rustpython.wasm extra_tests/snippets/stdlib_random.py +wasmer run --dir `pwd` -- target/wasm32-wasip1/release/rustpython.wasm `pwd`/extra_tests/snippets/stdlib_random.py ``` Run by wapm @@ -113,10 +137,10 @@ $ wapm run rustpython You can build the WebAssembly WASI file with: ```bash -cargo build --release --target wasm32-wasi --features="freeze-stdlib" +cargo build --release --target wasm32-wasip1 --features="freeze-stdlib" ``` -> Note: we use the `freeze-stdlib` to include the standard library inside the binary. You also have to run once `rustup target add wasm32-wasi`. +> Note: we use the `freeze-stdlib` to include the standard library inside the binary. You also have to run once `rustup target add wasm32-wasip1`. ### JIT (Just in time) compiler @@ -221,7 +245,7 @@ To enhance CPython compatibility, try to increase unittest coverage by checking Another approach is to checkout the source code: builtin functions and object methods are often the simplest and easiest way to contribute. -You can also simply run `./whats_left.py` to assist in finding any unimplemented +You can also simply run `python -I scripts/whats_left.py` to assist in finding any unimplemented method. ## Compiling to WebAssembly diff --git a/architecture/architecture.md b/architecture/architecture.md index 5b1ae9cc680..4b4110ee895 100644 --- a/architecture/architecture.md +++ b/architecture/architecture.md @@ -20,7 +20,43 @@ If, after reading this, you want to contribute to RustPython, take a look at the A high-level overview of the workings of RustPython is visible in the figure below, showing how Python source files are interpreted. -![overview.png](overview.png) +```mermaid +flowchart TB + SourceCode["🐍 Source code"] + + subgraph Interpreter["RustPython Interpreter"] + direction TB + + subgraph Parser["Parser"] + ParserBox["• Tokenize source code<br/>• Validate tokens<br/>• Create AST"] + end + + AST[["AST"]] + + subgraph Compiler["Compiler"] + CompilerBox["• Converts AST to bytecode"] + end + + Bytecode[["Bytecode"]] + + subgraph VM["VM"] + VMBox["• Executes bytecode given input"] + end + + Parser --> AST + AST --> Compiler + Compiler --> Bytecode + Bytecode --> VM + end + + SourceCode -------> Interpreter + + Input[["Input"]] + Output["Code gets executed"] + + Input --> VM + VM --> Output +``` Main architecture of RustPython. @@ -36,9 +72,9 @@ The main entry point of RustPython is located in `src/main.rs` and simply forwar For each of the three components, the entry point is as follows: -- Parser: The Parser is located in a separate project, [RustPython/Parser][10]. See the documentation there for more information. -- Compiler: `compile`, located in [`vm/src/vm/compile.rs`][11], this eventually forwards a call to [`compiler::compile`][12]. -- VM: `run_code_obj`, located in [`vm/src/vm/mod.rs`][13]. This creates a new frame in which the bytecode is executed. +- Parser: The Parser is located in a separate project, [ruff_python_parser][10]. See the documentation there for more information. +- Compiler: `compile`, located in [`crates/vm/src/vm/compile.rs`][11], this eventually forwards a call to [`compiler::compile`][12]. +- VM: `run_code_obj`, located in [`crates/vm/src/vm/mod.rs`][13]. This creates a new frame in which the bytecode is executed. ## Components @@ -46,7 +82,7 @@ Here we give a brief overview of each component and its function. For more detai ### Compiler -This component, implemented as the `rustpython-compiler/` package, is responsible for translating a Python source file into its equivalent bytecode representation. As an example, the following Python file: +This component, implemented as the `rustpython-compiler` package, is responsible for translating a Python source file into its equivalent bytecode representation. As an example, the following Python file: ```python def f(x): @@ -71,11 +107,11 @@ The Parser is the main sub-component of the compiler. All the functionality requ 1. Lexical Analysis 2. Parsing -The functionality for parsing resides in the RustPython/Parser project. See the documentation there for more information. +The functionality for parsing resides in the ruff_python_parser project in the astral-sh/ruff repository. See the documentation there for more information. ### VM -The Virtual Machine (VM) is responsible for executing the bytecode generated by the compiler. It is implemented in the `rustpython-vm/` package. The VM is currently implemented as a stack machine, meaning that it uses a stack to store intermediate results. In the `rustpython-vm/` package, additional sub-components are present, for example: +The Virtual Machine (VM) is responsible for executing the bytecode generated by the compiler. It is implemented in the `rustpython-vm` package. The VM is currently implemented as a stack machine, meaning that it uses a stack to store intermediate results. In the `rustpython-vm` package, additional sub-components are present, for example: - builtins: the built in objects of Python, such as `int` and `str`. - stdlib: parts of the standard library that contains built-in modules needed for the VM to function, such as `sys`. @@ -101,7 +137,7 @@ Part of the Python standard library that's implemented in Rust. The modules that ### Lib -Python side of the standard libary, copied over (with care) from CPython sourcecode. +Python side of the standard library, copied over (with care) from CPython sourcecode. #### Lib/test @@ -122,7 +158,7 @@ The RustPython executable/REPL (Read-Eval-Print-Loop) is implemented here, which Some things to note: - The CLI is defined in the [`run` function of `src/lib.rs`][16]. -- The interface and helper for the REPL are defined in this package, but the actual REPL can be found in `vm/src/readline.rs` +- The interface and helper for the REPL are defined in this package, but the actual REPL can be found in `crates/vm/src/readline.rs` ### WASM @@ -133,18 +169,18 @@ Crate for WebAssembly build, which compiles the RustPython package to a format t Integration and snippets that test for additional edge-cases, implementation specific functionality and bug report snippets. [1]: https://github.com/RustPython/RustPython -[2]: https://2021.desosa.nl/projects/rustpython/posts/vision/ +[2]: https://web.archive.org/web/20240723122357/https://2021.desosa.nl/projects/rustpython/posts/vision/ [3]: https://www.youtube.com/watch?v=nJDY9ASuiLc&t=213s [4]: https://rustpython.github.io/2020/04/02/thing-explainer-parser.html [5]: https://rustpython.github.io/demo/ [6]: https://rustpython.github.io/demo/notebook/ -[7]: https://github.com/RustPython/RustPython/blob/master/DEVELOPMENT.md +[7]: https://github.com/RustPython/RustPython/blob/main/DEVELOPMENT.md [8]: https://rustpython.github.io/guideline/2020/04/04/how-to-contribute-by-cpython-unittest.html -[9]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/src/lib.rs#L63 -[10]: https://github.com/RustPython/Parser -[11]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/vm/src/vm/compile.rs#LL10C17-L10C17 -[12]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/vm/src/vm/compile.rs#L26 -[13]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/vm/src/vm/mod.rs#L356 -[14]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/common/src/float_ops.rs +[9]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/src/lib.rs#L72 +[10]: https://github.com/astral-sh/ruff/tree/2bffef59665ce7d2630dfd72ee99846663660db8/crates/ruff_python_parser +[11]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/crates/vm/src/vm/compile.rs#L10 +[12]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/crates/vm/src/vm/compile.rs#L26 +[13]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/crates/vm/src/vm/mod.rs#L433 +[14]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/crates/common/src/float_ops.rs [15]: https://rustpython.github.io/guideline/2020/04/04/how-to-contribute-by-cpython-unittest.html -[16]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/src/lib.rs#L63 +[16]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/src/lib.rs#L72 diff --git a/architecture/overview.png b/architecture/overview.png deleted file mode 100644 index ce66706f3c8..00000000000 Binary files a/architecture/overview.png and /dev/null differ diff --git a/benches/README.md b/benches/README.md index f6e51296176..4823751fd16 100644 --- a/benches/README.md +++ b/benches/README.md @@ -4,19 +4,26 @@ These are some files to determine performance of rustpython. ## Usage -Running `cargo bench` from the root of the repository will start the benchmarks. Once done there will be a graphical +Running `cargo bench` from the root of the repository will start the benchmarks. Once done there will be a graphical report under `target/criterion/report/index.html` that you can use use to view the results. -To view Python tracebacks during benchmarks, run `RUST_BACKTRACE=1 cargo bench`. You can also bench against a +`cargo bench` supports name matching to run a subset of the benchmarks. To +run only the sorted microbenchmark, you can run: + +```shell +cargo bench sorted +``` + +To view Python tracebacks during benchmarks, run `RUST_BACKTRACE=1 cargo bench`. You can also bench against a specific installed Python version by running: ```shell -$ PYTHON_SYS_EXECUTABLE=python3.7 cargo bench +PYTHON_SYS_EXECUTABLE=python3.13 cargo bench ``` ### Adding a benchmark -Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested +Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested in two ways: 1. The time to parse the file to AST @@ -24,8 +31,9 @@ in two ways: ### Adding a micro benchmark -Micro benchmarks are small snippets of code added under the `microbenchmarks/` directory. A microbenchmark file has +Micro benchmarks are small snippets of code added under the `microbenchmarks/` directory. A microbenchmark file has two sections: + 1. Optional setup code 2. The code to be benchmarked @@ -39,8 +47,8 @@ a_list = [1,2,3] len(a_list) ``` -Only `len(a_list)` will be timed. Setup or benchmarked code can optionally reference a variable called `ITERATIONS`. If -present then the benchmark code will be invoked 5 times with `ITERATIONS` set to a value between 100 and 1,000. For +Only `len(a_list)` will be timed. Setup or benchmarked code can optionally reference a variable called `ITERATIONS`. If +present then the benchmark code will be invoked 5 times with `ITERATIONS` set to a value between 100 and 1,000. For example: ```python @@ -49,7 +57,7 @@ obj = [i for i in range(ITERATIONS)] `ITERATIONS` can appear in both the setup code and the benchmark code. -## MacOS setup +## MacOS setup On MacOS you will need to add the following to a `.cargo/config` file: @@ -63,4 +71,4 @@ rustflags = [ ## Benchmark source -- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html +- <https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html> diff --git a/benches/_data/pypi_org__simple__psutil.json b/benches/_data/pypi_org__simple__psutil.json new file mode 100644 index 00000000000..91e2ff6b39e --- /dev/null +++ b/benches/_data/pypi_org__simple__psutil.json @@ -0,0 +1 @@ +{"alternate-locations":[],"files":[{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.1.1.tar.gz","hashes":{"sha256":"25c6caffbf00d8be77489391a784654e99fcbaf2a5278e80f748be4112ee0188"},"provenance":null,"requires-python":null,"size":44485,"upload-time":"2014-02-06T02:06:57.249874Z","url":"https://files.pythonhosted.org/packages/69/e4/7e36e3e6cbc83b76f1c93a63d4c053a03ca99f1c99b106835cb175b5932a/psutil-0.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.1.2.tar.gz","hashes":{"sha256":"4a13d7f760b043b263346e48823b1dfd4c202e97b23483e481e5ff696e74509e"},"provenance":null,"requires-python":null,"size":61640,"upload-time":"2014-02-06T02:06:51.674389Z","url":"https://files.pythonhosted.org/packages/6e/51/56198d83577106bf89cb23bffcb273f923aea8d5ffe03e3fce55f830c323/psutil-0.1.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.1.3.tar.gz","hashes":{"sha256":"43e327934b4a273da20d3a5797d6abcaab37914f61499d96fcf9e8e1ae75442b"},"provenance":null,"requires-python":null,"size":85749,"upload-time":"2014-02-06T02:06:45.294070Z","url":"https://files.pythonhosted.org/packages/1d/4f/dcfe500fd43e3d6b26d253cb0d7e6e3a7d80224b5059bd50c482aff62eef/psutil-0.1.3.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.tar.gz","hashes":{"sha256":"8de8efa92162c94f623297f522e440e818dc7b832f421f9f490324bc7d5d0d92"},"provenance":null,"requires-python":null,"size":129382,"upload-time":"2014-02-06T02:03:03.376283Z","url":"https://files.pythonhosted.org/packages/58/20/3457e441edc1625c6e1dbfcf780d2b22f2e9caa8606c3fd8ce6c48104e87/psutil-0.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"4ce48cbbe3c915a9b720ce6465da34ce79465ffde63de8254cc9c8235fef824d"},"provenance":null,"requires-python":null,"size":291186,"upload-time":"2014-02-06T16:48:27.727869Z","url":"https://files.pythonhosted.org/packages/e6/1d/9a90eec0aec7e015d16f3922328336b4f8cd783f782e9ab81146b47ffee3/psutil-0.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win-amd64-py3.3.exe","hashes":{"sha256":"eeba1fa1f2455219a05775044f7a76741252ea0bd4c762e7b652d281843c79ff"},"provenance":null,"requires-python":null,"size":289973,"upload-time":"2014-02-06T16:48:36.482056Z","url":"https://files.pythonhosted.org/packages/1d/75/7c67bc2c8304b137a8ff709d21cb2dd7f600bc5ee76ecc88f77ec008e69e/psutil-0.2.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win-amd64-py3.4.exe","hashes":{"sha256":"919dabb5d47dc769a198a5d015c6765ef39b3b920c2ab6d04d05d9aa3ae6bb67"},"provenance":null,"requires-python":null,"size":289890,"upload-time":"2014-02-06T16:48:46.485205Z","url":"https://files.pythonhosted.org/packages/e0/e4/2ec24cecccf111a4dbda892b484bdfd8c00d3da5f99c1a0a79c469c62fb2/psutil-0.2.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py2.5.exe","hashes":{"sha256":"92f51445439f1082dab7733b9150b594b3797f4c96578457768c3e761f86c240"},"provenance":null,"requires-python":null,"size":129555,"upload-time":"2014-02-06T16:47:42.630117Z","url":"https://files.pythonhosted.org/packages/04/4f/478408899102af1d0d877be0b67e302fada1bdd8c8faaa4adc0e19550e53/psutil-0.2.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py2.6.exe","hashes":{"sha256":"c3e3dc863834d03485f6c48a735f76ab9a7cff2911d101c8cbd1e553c03f0fae"},"provenance":null,"requires-python":null,"size":262087,"upload-time":"2014-02-06T16:47:50.995531Z","url":"https://files.pythonhosted.org/packages/d7/f9/984326888f6c519cc813a96640cc9a2900dad85e15b9d368fe1f12a9bd11/psutil-0.2.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py2.7.exe","hashes":{"sha256":"08c6f16ac00babe2c54067dbc305c0587cb896a8eb64383290cdafdba74ea123"},"provenance":null,"requires-python":null,"size":261605,"upload-time":"2014-02-06T16:47:58.286157Z","url":"https://files.pythonhosted.org/packages/69/5c/be56c645a254ad83a9aa3055564aae543fdf23186aa5ec9c495a3937300b/psutil-0.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py3.3.exe","hashes":{"sha256":"e1f7cf0158a94e6b50f9a4b6f4258dec1bd89c9485c65d54b89ba2a09f0d51a3"},"provenance":null,"requires-python":null,"size":256763,"upload-time":"2014-02-06T16:48:06.663656Z","url":"https://files.pythonhosted.org/packages/76/42/d6813ce55af42553b2a3136e5159bd71e544bda5fdaed04c72759514e375/psutil-0.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.0.win32-py3.4.exe","hashes":{"sha256":"e49d818a4a49fd1ced40dac31076af622a40407234f717bc6a5c39b3f2335e81"},"provenance":null,"requires-python":null,"size":256718,"upload-time":"2014-02-06T16:48:16.971617Z","url":"https://files.pythonhosted.org/packages/f5/83/45b721a52001c1ba1f196e6b2ba3b12d99cbc03bf3f508f1fe56b4e25c66/psutil-0.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.tar.gz","hashes":{"sha256":"bd33f5f9e04b7677a932cc90d541ceec0050080d1b053ed39488ef39cb0fb4f4"},"provenance":null,"requires-python":null,"size":144657,"upload-time":"2014-02-06T02:06:37.338565Z","url":"https://files.pythonhosted.org/packages/9b/62/03133a1b4d1439227bc9b27389fc7d1137d111cbb15197094225967e21cf/psutil-0.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"ab9d06c0c2a7baadebf66fbc74a2532520b3b6d4a58ba3c1f04311a175ed9e7f"},"provenance":null,"requires-python":null,"size":294651,"upload-time":"2014-02-06T16:46:46.406386Z","url":"https://files.pythonhosted.org/packages/55/3e/e49b7929b9daf9c3dcf31668a43ba669d22bb8606deca52e6f92cb67324d/psutil-0.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"076c9914d8aadccd459e3efedc49c10f14d16918d0af5735c22a2bdcc3f90cac"},"provenance":null,"requires-python":null,"size":293431,"upload-time":"2014-02-06T16:46:54.986513Z","url":"https://files.pythonhosted.org/packages/34/f5/f471c56be91d10c9b83d22aa9f6f29006437f2196c89156f00f56f6e4661/psutil-0.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win-amd64-py3.4.exe","hashes":{"sha256":"2679f8f9f20e61d85177f3fc6ee10bfbd6c4bd122b3f5774ba36d0035cac4f7a"},"provenance":null,"requires-python":null,"size":293352,"upload-time":"2014-02-06T16:47:04.453988Z","url":"https://files.pythonhosted.org/packages/68/ee/fc85a707394d1b374da61c071dee12299e779d87c09a86c727b6f30ad9a6/psutil-0.2.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py2.5.exe","hashes":{"sha256":"a8345934d7585c75d500098491f2a434dbba9e378fd908d76fbeb0bc20b8a3d3"},"provenance":null,"requires-python":null,"size":132806,"upload-time":"2014-02-06T16:46:02.328109Z","url":"https://files.pythonhosted.org/packages/ca/6e/5c0a87e2c1bac5e32471eb84b349f0808f6d6ed49b9118784af9d0e71baa/psutil-0.2.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py2.6.exe","hashes":{"sha256":"9b6294c3327625839fdbe92dca4093ab55e422fb348790b2ffc5c6cde163fe5b"},"provenance":null,"requires-python":null,"size":265263,"upload-time":"2014-02-06T16:46:11.006069Z","url":"https://files.pythonhosted.org/packages/e5/13/b358b509d4996df82ef77758c14c0999c443df537eefacc7bb24b6339758/psutil-0.2.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py2.7.exe","hashes":{"sha256":"ac4d86abc4c850141cd1e054a64f1fcb6977c4ccbbd65c5bac9bf9c8c5950cb1"},"provenance":null,"requires-python":null,"size":264780,"upload-time":"2014-02-06T16:46:19.079623Z","url":"https://files.pythonhosted.org/packages/a9/1e/8ad399f44f63de6de262fc503b1d3f091f6a9b22feaf3c273ee8bb7a7c38/psutil-0.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py3.3.exe","hashes":{"sha256":"60048c47698aa20936b7593636dd88454bae6c436852aef95fb9774b0551d06c"},"provenance":null,"requires-python":null,"size":259950,"upload-time":"2014-02-06T16:46:28.163437Z","url":"https://files.pythonhosted.org/packages/75/f3/296f43c033c0453ebfdf125dc9b8907193e31ffb92256e4b702d16ef8ac1/psutil-0.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.2.1.win32-py3.4.exe","hashes":{"sha256":"4aad401c1a45848107d395f97fc9cdb783acf120553ca1761dd77a7d52adb585"},"provenance":null,"requires-python":null,"size":259910,"upload-time":"2014-02-06T16:46:37.546918Z","url":"https://files.pythonhosted.org/packages/ae/8c/751f1e2fdcaa0ea817a8529468dca052b5de55033ee8539b996497b5be0e/psutil-0.2.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.tar.gz","hashes":{"sha256":"d70e16e60a575637f9d75a1005a8987d239700c0955134d4c8666a5aefbe16b8"},"provenance":null,"requires-python":null,"size":153990,"upload-time":"2014-02-06T02:06:31.022178Z","url":"https://files.pythonhosted.org/packages/f2/5c/4e74b08905dab9474a53534507e463fe8eec98b2fc0d29964b0c6a7f959d/psutil-0.3.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win-amd64-py2.7.exe","hashes":{"sha256":"a554abcd10534ff32e3cbc4bd80b3c76a73d0bbb50739ee59482f0151703828b"},"provenance":null,"requires-python":null,"size":297305,"upload-time":"2014-02-06T16:45:05.336122Z","url":"https://files.pythonhosted.org/packages/78/91/21944fbddafa1bdf7c0c0ed7d9f4043cd8470e36a1f7a9f6e8b4406a8e39/psutil-0.3.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win-amd64-py3.3.exe","hashes":{"sha256":"b33444731c63361ac316657bb305642e562097e870793bc57203cd9a4e4819a4"},"provenance":null,"requires-python":null,"size":296059,"upload-time":"2014-02-06T16:45:13.056233Z","url":"https://files.pythonhosted.org/packages/88/bd/5e9d566e941e165987b8c9bd61583c58242ef4da12b7d9fbb07494580aff/psutil-0.3.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win-amd64-py3.4.exe","hashes":{"sha256":"89419f0cab4dd3ef4b32933237e34f58799069930fd25ed7f69deb9c64106c85"},"provenance":null,"requires-python":null,"size":296960,"upload-time":"2014-02-06T16:45:22.608830Z","url":"https://files.pythonhosted.org/packages/1f/06/c827177a80996d00f8f46ec99292e9d03fdf230a6e4fcea25ed9d19f3dcf/psutil-0.3.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py2.5.exe","hashes":{"sha256":"515b12e23581730c47e181fa163b457b90b6e848de0637d78b7fa8ca84ec2bc6"},"provenance":null,"requires-python":null,"size":135440,"upload-time":"2014-02-06T16:44:20.300778Z","url":"https://files.pythonhosted.org/packages/e5/55/6a353c646e6d15ee618348ed68cf79800f1275ade127f665779b6be0ed19/psutil-0.3.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py2.6.exe","hashes":{"sha256":"5bc91423ff25ad8dffb3c9979abdafb1ecd2e9a7ef4b9287bb731e76ea2c4b91"},"provenance":null,"requires-python":null,"size":267782,"upload-time":"2014-02-06T16:44:30.319267Z","url":"https://files.pythonhosted.org/packages/61/04/c447b4d468402bfb10d21995b0eae219c8397c00d9e8a2b209fb032ac1aa/psutil-0.3.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py2.7.exe","hashes":{"sha256":"4b7fbaca98b72f69561d593ef18b5254a8f98482b561db3d906aac4dd307bbcc"},"provenance":null,"requires-python":null,"size":267308,"upload-time":"2014-02-06T16:44:39.895580Z","url":"https://files.pythonhosted.org/packages/e4/2f/b060e631686e6455ff483ea5b6333382c6dddb5727edbfc9dcca1a8f024c/psutil-0.3.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py3.3.exe","hashes":{"sha256":"005217c21551618085e3323d653d75f9ad8f7418bff4fd5109850fba11950d6e"},"provenance":null,"requires-python":null,"size":262514,"upload-time":"2014-02-06T16:44:47.866655Z","url":"https://files.pythonhosted.org/packages/50/34/7bfc452cb2cd0069a81064db51b484ef135125cf1d246d0ff82c65653c81/psutil-0.3.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.3.0.win32-py3.4.exe","hashes":{"sha256":"331ac97b50def25e41bae637642d0131d5c4223b52ab55281d002549632ed5e0"},"provenance":null,"requires-python":null,"size":262479,"upload-time":"2014-02-06T16:44:56.381135Z","url":"https://files.pythonhosted.org/packages/b0/08/1944d2281986237cdaeabfa36195f44dc4c847d5c4fda3523b44c6823e19/psutil-0.3.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.tar.gz","hashes":{"sha256":"4b0ecc77d6c503449af3d2f0a41ad4cb8338e173f5d655a8239e41b1a49bc278"},"provenance":null,"requires-python":null,"size":167796,"upload-time":"2014-02-06T02:06:24.041544Z","url":"https://files.pythonhosted.org/packages/88/46/a933ab20c6d9b0ca5704b60307b9e80bdc119b759e89b74a2609b4c10eb6/psutil-0.4.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win-amd64-py2.7.exe","hashes":{"sha256":"9689512e5a32508216bd6cc7cb2fbe938e4a47b483cd345b5bec6dda790a9210"},"provenance":null,"requires-python":null,"size":302844,"upload-time":"2014-02-06T16:43:18.853340Z","url":"https://files.pythonhosted.org/packages/a5/06/3ca1c85b733ceaced144cb211c1fc40a6b3b1c4f7a109bbb8e19fe09eac0/psutil-0.4.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win-amd64-py3.3.exe","hashes":{"sha256":"9ee193dec84c2c91ac8712760eaaae0488dee92db37134dfcddc889ecd4f578d"},"provenance":null,"requires-python":null,"size":301654,"upload-time":"2014-02-06T16:43:28.863005Z","url":"https://files.pythonhosted.org/packages/ac/eb/f6923ea1b46803251f2bb42ad9b8ed90f4b8d04cdb2384c9ea535193e4f7/psutil-0.4.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win-amd64-py3.4.exe","hashes":{"sha256":"94d2d0a24fa62f5655e9e21dd558592bf1d3d515d944eee70e9cd2e2bf2bf244"},"provenance":null,"requires-python":null,"size":302558,"upload-time":"2014-02-06T16:43:38.802980Z","url":"https://files.pythonhosted.org/packages/7c/5b/d9057a158b09c377eab80dae60928f8fe29bdd11635ca725098587981eeb/psutil-0.4.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py2.5.exe","hashes":{"sha256":"fab822fb8968b45695174a2c9b4d40e9ae81126343e24d38b627036579f1ee5e"},"provenance":null,"requires-python":null,"size":140768,"upload-time":"2014-02-06T16:42:35.663297Z","url":"https://files.pythonhosted.org/packages/07/1e/7827d1271248ddef694681747297aeb3f77dafe5c93ff707a625e3947082/psutil-0.4.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py2.6.exe","hashes":{"sha256":"ca511f56db8f1586612851a2eedbf80e38cf29f4a98f133410a74da284822747"},"provenance":null,"requires-python":null,"size":273280,"upload-time":"2014-02-06T16:42:44.697290Z","url":"https://files.pythonhosted.org/packages/b4/56/78f182bf9c81a978067ba5447eca2ea70ef2bc9ac94228adfae5428cd108/psutil-0.4.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py2.7.exe","hashes":{"sha256":"5b6cbf5aaeab55db1d9657fc75e4b54fa94d17d03ca97c94aa150e1bfdc8fb2e"},"provenance":null,"requires-python":null,"size":272807,"upload-time":"2014-02-06T16:42:52.281617Z","url":"https://files.pythonhosted.org/packages/7d/38/b719c8867699a71a98e92e61fd1b0f12bad997a9de60bf8f6215184692e8/psutil-0.4.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py3.3.exe","hashes":{"sha256":"dfe18b8eb9ee6dbbd49e28f245794c2fc22b56d770038668cb6e112935c49927"},"provenance":null,"requires-python":null,"size":268022,"upload-time":"2014-02-06T16:43:01.185161Z","url":"https://files.pythonhosted.org/packages/f4/2b/13be095a72c83fdbe11d519310cb34ff8e442d72ab52691218a83c216ac4/psutil-0.4.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.0.win32-py3.4.exe","hashes":{"sha256":"5b63e0e83dde30d62206497320c06b27c489ffb261e95ac1267b7a1b8d4ce72d"},"provenance":null,"requires-python":null,"size":267989,"upload-time":"2014-02-06T16:43:09.528893Z","url":"https://files.pythonhosted.org/packages/b4/8e/028b849600447c45c20582d7ddae1e39d31b83799289f54f1b79af664295/psutil-0.4.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.tar.gz","hashes":{"sha256":"33002c38f916835c949ae39a84f3f6d09ce01818ed805dfd61f8f3844c395c9d"},"provenance":null,"requires-python":null,"size":171549,"upload-time":"2014-02-06T02:06:17.394021Z","url":"https://files.pythonhosted.org/packages/a0/cd/00550e16a2a0357a9c24946160e60fc947bbe23bc276aff1b4d3e7b90345/psutil-0.4.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win-amd64-py2.7.exe","hashes":{"sha256":"3e408028898b4ad09c207659c1b6cb8e6a74fb39ab3157b761e057a629118fec"},"provenance":null,"requires-python":null,"size":310414,"upload-time":"2014-02-06T16:41:41.640755Z","url":"https://files.pythonhosted.org/packages/06/75/3dc33773b1a1f7055b24245a0c8bec4d5776f1e1125329a13e82449007be/psutil-0.4.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win-amd64-py3.3.exe","hashes":{"sha256":"457cfdecbf63f813bf86e2f736ab3b20447e8d8c2c516dd2a13a7463177f8bc6"},"provenance":null,"requires-python":null,"size":309169,"upload-time":"2014-02-06T16:41:50.926907Z","url":"https://files.pythonhosted.org/packages/6e/9e/6c7f5baf2529d9bba563cdff0b81413b5870b0681739318112adecd99ad0/psutil-0.4.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win-amd64-py3.4.exe","hashes":{"sha256":"fc75e094cf573a7b9977be2a9d3a9d94671bc0ab098d52c391dd63227309cbea"},"provenance":null,"requires-python":null,"size":310087,"upload-time":"2014-02-06T16:42:00.068444Z","url":"https://files.pythonhosted.org/packages/cd/e6/9f963b32ab254b8c2bbedf5ce543cace8b08054924fd26fa233f99863ecc/psutil-0.4.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py2.5.exe","hashes":{"sha256":"ef4ed886b4d2d664fab5c369543690cda04e04a3b7496d72e369ef5240a8af6b"},"provenance":null,"requires-python":null,"size":148474,"upload-time":"2014-02-06T16:40:57.772154Z","url":"https://files.pythonhosted.org/packages/65/ad/e7828797558dd5f209694b02ce079cd3b6beacf6a5175f38c1973c688494/psutil-0.4.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py2.6.exe","hashes":{"sha256":"aafe9e328ffc173b26c78897c168858da335c85d4a02d666ad68fe2fe14601d1"},"provenance":null,"requires-python":null,"size":280897,"upload-time":"2014-02-06T16:41:07.265101Z","url":"https://files.pythonhosted.org/packages/1e/a1/594bf54e7c4056bc5284023be97f67c930175b3329e086a4ed8966cb067a/psutil-0.4.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py2.7.exe","hashes":{"sha256":"cefad502010c78425190498efc54541ae7bbe9eaa73ef4000cf11b032d32a8bb"},"provenance":null,"requires-python":null,"size":280445,"upload-time":"2014-02-06T16:41:15.492817Z","url":"https://files.pythonhosted.org/packages/c6/c4/14807de009a1beab2426b537379a8b05b1d69fef1fde7e23581cc332cdb3/psutil-0.4.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py3.3.exe","hashes":{"sha256":"a35ce124b2dec01d7632627c9370f5202978fce89ae59235cf4d5e05bbd0e02b"},"provenance":null,"requires-python":null,"size":275668,"upload-time":"2014-02-06T16:41:23.498162Z","url":"https://files.pythonhosted.org/packages/38/7d/407f7586bccdeb80f5da82d8ebb98712bfcc7217e3d7c3fc61b3bba893f2/psutil-0.4.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.4.1.win32-py3.4.exe","hashes":{"sha256":"860a039b67ee015a6304e8d05e6dc5c2a447eca331b7d6c9d04f41d7f2fc3a66"},"provenance":null,"requires-python":null,"size":275635,"upload-time":"2014-02-06T16:41:33.515203Z","url":"https://files.pythonhosted.org/packages/c5/b7/d2662ebf114961766c9eb3d9b737cb5541060d35e65b542b0acef470221a/psutil-0.4.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.tar.gz","hashes":{"sha256":"b10d1c19b9d334bea9f025f3b82b5664d642d26ba32ae0e71e7e20c5a6b4164f"},"provenance":null,"requires-python":null,"size":118411,"upload-time":"2014-02-06T02:06:09.471218Z","url":"https://files.pythonhosted.org/packages/3f/eb/e4115c3ecc189fd345b9a50d521e2ff76990bfb12a91921e93ea6398feef/psutil-0.5.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win-amd64-py2.7.exe","hashes":{"sha256":"5bf7a91fd57be3a63c6b9fa78da63c185b9be909ca49032f3ceda662628be74d"},"provenance":null,"requires-python":null,"size":319335,"upload-time":"2014-02-06T15:53:45.976879Z","url":"https://files.pythonhosted.org/packages/c5/cc/9bb29fdd60e347617c3a09f51a595941ddb4777fc1608068d174ced5bfdd/psutil-0.5.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win-amd64-py3.3.exe","hashes":{"sha256":"0cd9872d0d675b0314d2d15513e8ff3693a417e4ebaa158fc1f8780cc611574c"},"provenance":null,"requires-python":null,"size":317930,"upload-time":"2014-02-06T15:53:57.230392Z","url":"https://files.pythonhosted.org/packages/37/d8/276629523e0477639b33a606f93c2fc2e9c4cafba76b3a4cd1ddc42dd307/psutil-0.5.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win-amd64-py3.4.exe","hashes":{"sha256":"13f71acf1d2fe08db072d890280e985e8414080bcd525c13c66e0e8fbc917710"},"provenance":null,"requires-python":null,"size":318790,"upload-time":"2014-02-06T15:54:08.039630Z","url":"https://files.pythonhosted.org/packages/5f/bc/a026bbb10a4df4cf458b5d6feabc72a0017e6a4f472898280442ff3f9393/psutil-0.5.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py2.5.exe","hashes":{"sha256":"d6839532038b97792d90341465430520e58ea3d0956a1df710b3fdd733dde587"},"provenance":null,"requires-python":null,"size":156923,"upload-time":"2014-02-06T16:39:37.158716Z","url":"https://files.pythonhosted.org/packages/26/83/d1aed38bad84576e48b9ca6501edcf829ca3b48631c823736c36fa2b24de/psutil-0.5.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py2.6.exe","hashes":{"sha256":"be79349ecf091470ea4920e45b78914b67f17bec9c23e4bea4ec0ca5fee1feab"},"provenance":null,"requires-python":null,"size":289457,"upload-time":"2014-02-06T15:53:07.782148Z","url":"https://files.pythonhosted.org/packages/4e/b2/bfdeae58283e0fb09e2c6725c01b9429acbd15917c7ead91c96f2df37d05/psutil-0.5.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py2.7.exe","hashes":{"sha256":"e4fa8e965eb58a885e65a360f1a7ffc7ed476e9227a2de3726091d7560c694a8"},"provenance":null,"requires-python":null,"size":289002,"upload-time":"2014-02-06T15:53:16.212429Z","url":"https://files.pythonhosted.org/packages/2d/62/ed3c23fb8648a916460e95306d58f5ba42b3b20c68f2a55d1fc6396190a5/psutil-0.5.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py3.3.exe","hashes":{"sha256":"0380ece3eeb6f170ba156c58eab2fa36508600b1c7e10bc9b7c651f721236f38"},"provenance":null,"requires-python":null,"size":284089,"upload-time":"2014-02-06T15:53:25.963338Z","url":"https://files.pythonhosted.org/packages/59/3d/25912eb67f01af306d8ec035041c07daa7f783aafa5f4e1dd90de91c4849/psutil-0.5.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.0.win32-py3.4.exe","hashes":{"sha256":"d6a9e67de81fc0fc58c790e1355ddfbb8a0760963868a538f217ac2325a81581"},"provenance":null,"requires-python":null,"size":284045,"upload-time":"2014-02-06T15:53:35.675392Z","url":"https://files.pythonhosted.org/packages/38/74/e6c5bfb482f011d19bb1366cecf7d4f0b5c9c9b7664ef31e951cc4d0757b/psutil-0.5.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.tar.gz","hashes":{"sha256":"f660d244a08373f5e89633650970819a59463b37af1c9d205699fb0d0608986d"},"provenance":null,"requires-python":null,"size":118781,"upload-time":"2014-02-06T02:06:02.078066Z","url":"https://files.pythonhosted.org/packages/aa/06/6ebc13a14c0961d7bb8184da530448d8fc198465eb5ecd1ad36c761807d2/psutil-0.5.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win-amd64-py2.7.exe","hashes":{"sha256":"eff5ed677b6bdb5d75f1de925df774d8029582eea17abc3bbd31b9885d3c649d"},"provenance":null,"requires-python":null,"size":319512,"upload-time":"2014-02-06T16:36:10.664265Z","url":"https://files.pythonhosted.org/packages/8a/26/15751b500afdd0aea141905d5ba5319c38be41e0ca55a374c02827d1a79c/psutil-0.5.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win-amd64-py3.3.exe","hashes":{"sha256":"8a12d8e368bfccfe2074616cae2472dfb1e136a2aaaea5b7dbafdb390166b202"},"provenance":null,"requires-python":null,"size":318245,"upload-time":"2014-02-06T16:36:21.268821Z","url":"https://files.pythonhosted.org/packages/13/9d/49f6fb5e1f75b3d6815bbdbbeb1630f7299d6ff96f85836dbd27989780ae/psutil-0.5.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win-amd64-py3.4.exe","hashes":{"sha256":"9d991aae6b993ac58b540f27b2dff71137db62253f5d7acc5ede1ec845fd6b0b"},"provenance":null,"requires-python":null,"size":319083,"upload-time":"2014-02-06T16:36:31.287414Z","url":"https://files.pythonhosted.org/packages/2c/fa/a0469c75acedccc5006b655434e96a8a6a490725a9ae8208faaf39c043f6/psutil-0.5.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py2.5.exe","hashes":{"sha256":"ee32be6667bb69ebefc2fb5e23d115d98e307b5246c01ef50e45cfd8ad824d07"},"provenance":null,"requires-python":null,"size":157149,"upload-time":"2014-02-06T16:35:24.026975Z","url":"https://files.pythonhosted.org/packages/2e/30/9ed6283c7f1a716a3bf76a063b278ace2f89d1a8b2af9d35080dc2a0319b/psutil-0.5.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py2.6.exe","hashes":{"sha256":"f434f546d73366bff6283db907105b2607976ff9be67f2b5a7b35533e5f27219"},"provenance":null,"requires-python":null,"size":289555,"upload-time":"2014-02-06T16:35:32.000842Z","url":"https://files.pythonhosted.org/packages/eb/d9/55f5d02e42dc6c73f66c7cfc99bc3bf97cf10e9f259dbea69e123e0ef459/psutil-0.5.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py2.7.exe","hashes":{"sha256":"45acf090d65f2f47f3b1444b056cafb1b1ca70a8e266ddf10422734fb0ada897"},"provenance":null,"requires-python":null,"size":289095,"upload-time":"2014-02-06T16:35:40.182115Z","url":"https://files.pythonhosted.org/packages/81/25/cbbb80d1957c28dffb9f20c1eaf349d5642215d7ca15ec5b7015b6a510c8/psutil-0.5.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py3.3.exe","hashes":{"sha256":"9a20a99b6727334c8cc3a10ecc556d2f15a1e92efef97670ed022b86e09df36b"},"provenance":null,"requires-python":null,"size":284290,"upload-time":"2014-02-06T16:35:50.441013Z","url":"https://files.pythonhosted.org/packages/73/a2/ab4f2fa7ba6f8fc51d9cc9a0bb5ec269860a34574d391e39d20c2dca7b56/psutil-0.5.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.5.1.win32-py3.4.exe","hashes":{"sha256":"d547e164013a4d78909cbac0c176d65aa51c06de2fb888c63b738906f5ccc105"},"provenance":null,"requires-python":null,"size":284247,"upload-time":"2014-02-06T16:36:00.155604Z","url":"https://files.pythonhosted.org/packages/7f/8b/c9abd21cdc79b70744e169b98be06f0828687dfcac6468ed87455099f88d/psutil-0.5.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.tar.gz","hashes":{"sha256":"e918763243371f69bc98b601769df7337e196029dcdb797224f0ede474e17b94"},"provenance":null,"requires-python":null,"size":130803,"upload-time":"2014-02-06T02:05:55.032752Z","url":"https://files.pythonhosted.org/packages/3c/9f/0de622fc62e74f4c154656440c02c8a24a01a997d605a744272eb0d93742/psutil-0.6.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win-amd64-py2.7.exe","hashes":{"sha256":"1792be8fca4b3f88553e1c5cc197e18932c3da9b969ce7f545b741913597d600"},"provenance":null,"requires-python":null,"size":322153,"upload-time":"2014-02-06T16:34:24.006415Z","url":"https://files.pythonhosted.org/packages/5f/fd/fc06aaf69ad438d1354312c728791a1f571b2c59a49b411128911abf647e/psutil-0.6.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win-amd64-py3.3.exe","hashes":{"sha256":"084a1cbeca8e10f0cbba4b9d3f7d3136cb1d03997a868eea334401f82105207a"},"provenance":null,"requires-python":null,"size":320646,"upload-time":"2014-02-06T16:34:33.535927Z","url":"https://files.pythonhosted.org/packages/70/5e/b1d4a1d5238497d48112bc27e3448d05a0acb6c8fcd537565724d916e7c2/psutil-0.6.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win-amd64-py3.4.exe","hashes":{"sha256":"e7233e45ed7505efe46309978edd3e8a1cb4160400c3535512be2e4114615bb9"},"provenance":null,"requires-python":null,"size":321547,"upload-time":"2014-02-06T16:34:44.543150Z","url":"https://files.pythonhosted.org/packages/47/ad/4139c3eaa2ee1da71d071c9b49e8a04ff23f2c71f6fdecc11850b0e7343e/psutil-0.6.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py2.5.exe","hashes":{"sha256":"469628dea02a7d91837e05777cb66bfcdc2e1639cc8607ed93f19679c4e42cef"},"provenance":null,"requires-python":null,"size":160485,"upload-time":"2014-02-06T16:33:39.286987Z","url":"https://files.pythonhosted.org/packages/10/3a/a0136c299b74224be3b0735812a1544e6962d7c61af41adfcc400791c684/psutil-0.6.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py2.6.exe","hashes":{"sha256":"a628ad11fefc16d6dd144a8d29a03ac379debcf6c2e4b576e6ee2b2677c15836"},"provenance":null,"requires-python":null,"size":292261,"upload-time":"2014-02-06T16:33:47.696744Z","url":"https://files.pythonhosted.org/packages/2b/79/eec1c63d3ea968cc9754871a1b9d17d50bfc24136ecac415f701b1d240ea/psutil-0.6.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py2.7.exe","hashes":{"sha256":"71197523bf8acb49e3e909ea945cc6e77a21693428db2f169e33348988c1bb11"},"provenance":null,"requires-python":null,"size":291811,"upload-time":"2014-02-06T16:33:56.338977Z","url":"https://files.pythonhosted.org/packages/e9/64/fe90d0f1ba2dff8ad2511423720aed3569cfe7b5dbb95ede64af25b77e84/psutil-0.6.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py3.3.exe","hashes":{"sha256":"1d7df98f4532e76685fabbd82e9502c91be9a60e4dfdfd5a19c2152936b7d198"},"provenance":null,"requires-python":null,"size":286963,"upload-time":"2014-02-06T16:34:04.390605Z","url":"https://files.pythonhosted.org/packages/d6/d7/91226f8635d3850917b64e94ca4903910a60739ee7c930ef9678d93638eb/psutil-0.6.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.0.win32-py3.4.exe","hashes":{"sha256":"da568edc3b8ec4be5b8cee54851289eb4c24cbaa8ea165858e9bfc279269bf22"},"provenance":null,"requires-python":null,"size":286905,"upload-time":"2014-02-06T16:34:14.322283Z","url":"https://files.pythonhosted.org/packages/56/67/533833832596e5f8c1fb51d3f94cd110d688579be7af46996cf96ad890e7/psutil-0.6.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.tar.gz","hashes":{"sha256":"52eba795281cdd1079f13ded6a851f6d029551ddf552eadc9a2ee3eb26fe994d"},"provenance":null,"requires-python":null,"size":131473,"upload-time":"2014-02-06T02:05:47.971617Z","url":"https://files.pythonhosted.org/packages/4f/e6/989cb0b2f7f0ebe3ab0e7144b78db17387810f1526e98498be96fb755fa9/psutil-0.6.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win-amd64-py2.7.exe","hashes":{"sha256":"566bcec389a77279e10335eb5d0021eb968445720496b70b46d208dbbf8e49a1"},"provenance":null,"requires-python":null,"size":322415,"upload-time":"2014-02-06T16:32:41.593785Z","url":"https://files.pythonhosted.org/packages/9a/bb/cb19a41fa75a2205ac2d337be45183f879aa77e60bbb19387c872c81d451/psutil-0.6.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win-amd64-py3.3.exe","hashes":{"sha256":"fde5e57f9b51057998cce5a030ad481f826832b26cc2a6490d60317c9024724c"},"provenance":null,"requires-python":null,"size":320911,"upload-time":"2014-02-06T16:32:50.930654Z","url":"https://files.pythonhosted.org/packages/7d/d1/9e75848da16a5d165e3918540418f3052e6409357bcce91a7342b22d674f/psutil-0.6.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win-amd64-py3.4.exe","hashes":{"sha256":"402475d1a93306ac025070da9628a0ca0e0d253b0755bf13131318a39aaeea8e"},"provenance":null,"requires-python":null,"size":321810,"upload-time":"2014-02-06T16:33:01.995635Z","url":"https://files.pythonhosted.org/packages/95/82/1f52f8aaa37e07da54da4468c031f59717d3f40cd82b6faaf8f203d84bd7/psutil-0.6.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py2.5.exe","hashes":{"sha256":"3f5b8afde564c563cd6fa28ac14ed020d91d654ef1efd6c0e1672282fcde8e65"},"provenance":null,"requires-python":null,"size":160754,"upload-time":"2014-02-06T16:31:55.533884Z","url":"https://files.pythonhosted.org/packages/5b/3f/10449a8a3dfb809fbeae3d853453aafb36376d51f33c46e02b5f0723c620/psutil-0.6.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py2.6.exe","hashes":{"sha256":"1c1763ff214042e7be75fd6eb336322a47193342cddfc70b8547f086968e024c"},"provenance":null,"requires-python":null,"size":292530,"upload-time":"2014-02-06T16:32:04.859349Z","url":"https://files.pythonhosted.org/packages/3e/7a/5d53c99ec66ea86548d95acaf98b32f1cbb761cf2e1fc8ee62d0838f8e24/psutil-0.6.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py2.7.exe","hashes":{"sha256":"e411f2374994663ad35bc21afe4ff9003e3675e23b07a2a60b57c760da0c5f3f"},"provenance":null,"requires-python":null,"size":292082,"upload-time":"2014-02-06T16:32:13.267526Z","url":"https://files.pythonhosted.org/packages/4a/40/e43e412c55f66b57aee3a418f47b05e7bb92db83df93c0c93b00f11f1357/psutil-0.6.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py3.3.exe","hashes":{"sha256":"03ea1f851920d6ddc286c1940ebd167ed92c1cb7a870f1789510c21c6fa6b7bc"},"provenance":null,"requires-python":null,"size":287227,"upload-time":"2014-02-06T16:32:22.314264Z","url":"https://files.pythonhosted.org/packages/07/12/1ce6196592e01965a375b307a57f68e90488a49a56a647f17ba027382448/psutil-0.6.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.6.1.win32-py3.4.exe","hashes":{"sha256":"095107d63b176fed2f4a44de3a4d19e8aa6291de0c3ee5e398272c9b72bf2254"},"provenance":null,"requires-python":null,"size":287169,"upload-time":"2014-02-06T16:32:32.819904Z","url":"https://files.pythonhosted.org/packages/f4/88/d20f7eefa6b8cc59f9320c52bf561ba52baea2b9b36568b198bca0b3548d/psutil-0.6.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.tar.gz","hashes":{"sha256":"95089802017ee629b84332deeb367f7f775b42e827ee283d46b7e99d05120d71"},"provenance":null,"requires-python":null,"size":138681,"upload-time":"2014-02-06T02:05:40.485455Z","url":"https://files.pythonhosted.org/packages/0b/8c/aeb6acf5a4610f8d5bb29ade04081d8672c2294c3cefa1f0422c6398bc66/psutil-0.7.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win-amd64-py2.7.exe","hashes":{"sha256":"add60c89c13443617c83116fab34a3c6e4f32ee8b102ff6ffb4cd20ef6f511c3"},"provenance":null,"requires-python":null,"size":325666,"upload-time":"2014-02-06T16:30:51.588803Z","url":"https://files.pythonhosted.org/packages/38/8e/f3023be1a2268b5ee3a20e05ec8997a95c987f6f2b7dadad837674d562e8/psutil-0.7.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win-amd64-py3.3.exe","hashes":{"sha256":"520a43be7fcd21d63c02039cdd7ebfe959b9fa9a865ba9aab6b0194a85184250"},"provenance":null,"requires-python":null,"size":324143,"upload-time":"2014-02-06T16:31:01.226228Z","url":"https://files.pythonhosted.org/packages/7b/03/d6bff7ff8570f3521a1d7949f03741210f11ba5ba807789d37e185a0a780/psutil-0.7.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win-amd64-py3.4.exe","hashes":{"sha256":"0e78c6b441e659d922c5fddd5d5a36a368a61192873a0600ba23171c14dc2a6d"},"provenance":null,"requires-python":null,"size":325001,"upload-time":"2014-02-06T16:31:12.826305Z","url":"https://files.pythonhosted.org/packages/ce/8a/95e7c8b343a92137d82e4635fcb3f5c69c29e2a84ce4d51f2d5d86020b1e/psutil-0.7.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py2.5.exe","hashes":{"sha256":"532b918d8d2df60c20c69c1f764b9b16373ea22c32595fe801e3cc9b1aa5adfb"},"provenance":null,"requires-python":null,"size":163763,"upload-time":"2014-02-06T16:30:02.815062Z","url":"https://files.pythonhosted.org/packages/9e/6f/a946eeb3c6b8848fcf6d21c52884e2639e7d4393b51f330d4555ec177e83/psutil-0.7.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py2.6.exe","hashes":{"sha256":"21e1d7880579ef7eea28547c737923eaeee53389d51fa971d9f70bf89e9759ee"},"provenance":null,"requires-python":null,"size":295425,"upload-time":"2014-02-06T16:30:11.405843Z","url":"https://files.pythonhosted.org/packages/14/e9/ac4865772c55a3c09c85d987ac2a0e679c31e897d536f389485019c29450/psutil-0.7.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py2.7.exe","hashes":{"sha256":"bc36a7350b89e2ada06691583f4c4c2f99988fe6fa710d4f8e4143bed9f9f8bf"},"provenance":null,"requires-python":null,"size":294933,"upload-time":"2014-02-06T16:30:20.866887Z","url":"https://files.pythonhosted.org/packages/12/1c/053e0d5485a39c055e92b845b6311ab54887a091836a3985369cfffa9e53/psutil-0.7.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py3.3.exe","hashes":{"sha256":"9ce0d7a8cedde996eaae1eda3db9b473d441bd6c797b36a166d7b7a70dd4650e"},"provenance":null,"requires-python":null,"size":290006,"upload-time":"2014-02-06T16:30:29.860768Z","url":"https://files.pythonhosted.org/packages/73/19/8009828f3cfcf529570822e52988b95ac1c25c90ba4fb96ae570b5c25e66/psutil-0.7.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.0.win32-py3.4.exe","hashes":{"sha256":"6fdeca414a470d156ebdf1b6e4e9ca0fd9eb1b53e8d9a0cbb7d4a69d31f68bdc"},"provenance":null,"requires-python":null,"size":289986,"upload-time":"2014-02-06T16:30:39.817575Z","url":"https://files.pythonhosted.org/packages/42/7d/8e159acbf98eed98b778c244287b295f06411ac9e1903d2d358ba48b1d36/psutil-0.7.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.tar.gz","hashes":{"sha256":"5236f649318a06dcff8b86947c888d4510abce1783923aa5455b2d62df7204c7"},"provenance":null,"requires-python":null,"size":138525,"upload-time":"2014-02-06T02:05:32.811951Z","url":"https://files.pythonhosted.org/packages/8c/75/1eeb93df943b70c2e18bf412b32d18af3577da831f2bfe8c7d29f8853a67/psutil-0.7.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win-amd64-py2.7.exe","hashes":{"sha256":"47fc6887ec86bd7a422326dfc77e2b43fcbc71e498194f86088bb2268c2372b7"},"provenance":null,"requires-python":null,"size":325663,"upload-time":"2014-02-06T16:29:31.935519Z","url":"https://files.pythonhosted.org/packages/2d/ab/8fde8a7358a21bc2488a5d21a24879ce3cc162d2a99c56f9599c7d759c75/psutil-0.7.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win-amd64-py3.3.exe","hashes":{"sha256":"c00ec6a0e061e31687851c06d542500fd47b0ed415818e0da61c294f2796572b"},"provenance":null,"requires-python":null,"size":324140,"upload-time":"2014-02-06T16:29:41.403650Z","url":"https://files.pythonhosted.org/packages/35/b1/8506f2e78974da833b20b162d70b7e38ac3ab722adf18c03a3e428c9d9c4/psutil-0.7.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win-amd64-py3.4.exe","hashes":{"sha256":"a5238a16cd85d4f3240dfca8ad94cb0bfe8643acb5c21d839a53caa86d8a23ab"},"provenance":null,"requires-python":null,"size":324998,"upload-time":"2014-02-06T16:29:50.929853Z","url":"https://files.pythonhosted.org/packages/cc/29/d00e1011e6db097ffac31c2876fdba51fd1fc251ee472ad6219095739a86/psutil-0.7.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py2.5.exe","hashes":{"sha256":"facf21db78120afb79ec51685043a5f84e897a84a7678db4381c0e377a481364"},"provenance":null,"requires-python":null,"size":163764,"upload-time":"2014-02-06T16:28:44.337888Z","url":"https://files.pythonhosted.org/packages/06/d8/a34c090687fa4c09bee5b5a06a762eb1af134d847b907d3e63c58e5d2cba/psutil-0.7.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py2.6.exe","hashes":{"sha256":"709d2e851c161f1ffad6bd7ed925ee90229dbd2067fbc26126c11d473075d7b7"},"provenance":null,"requires-python":null,"size":295427,"upload-time":"2014-02-06T16:28:53.964081Z","url":"https://files.pythonhosted.org/packages/4d/8a/d51e550aa0928ab02e4f6b2531453b2e63c8877e30543e342c8a35c90d4d/psutil-0.7.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py2.7.exe","hashes":{"sha256":"ee218aa12ee3af563833cf0bf109b15382261a0760a1ce636bf9e26e630f833e"},"provenance":null,"requires-python":null,"size":294930,"upload-time":"2014-02-06T16:29:03.098211Z","url":"https://files.pythonhosted.org/packages/3c/4c/d46a8bc865e58b30dcb160772667abf42f854ec4910ee1100e79961035ce/psutil-0.7.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py3.3.exe","hashes":{"sha256":"fdf22a72f891a45b8a9ebca85b50a055a4ba7a8c05b98f93e1387b8ee88a4562"},"provenance":null,"requires-python":null,"size":290003,"upload-time":"2014-02-06T16:29:12.556491Z","url":"https://files.pythonhosted.org/packages/1a/cf/98fdaf08279cd754414ab6723bbb6bd55d890624b766d64b21c35379d865/psutil-0.7.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-0.7.1.win32-py3.4.exe","hashes":{"sha256":"f675a6346e5bd02469b043afff2db1b8e0b180e8a6d1ca94d44d23f26f60e68f"},"provenance":null,"requires-python":null,"size":289983,"upload-time":"2014-02-06T16:29:21.732021Z","url":"https://files.pythonhosted.org/packages/f4/f9/79ac18809795f53197fb5b6bd452d2b52f62e5bff6a66d9b813077a85eb9/psutil-0.7.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.tar.gz","hashes":{"sha256":"af776ebeaf2420709b1ab741f664c048dcfc2890e3e7d151ae868fbdc3d47920"},"provenance":null,"requires-python":null,"size":156516,"upload-time":"2014-02-06T02:05:25.754044Z","url":"https://files.pythonhosted.org/packages/ed/7e/dd4062bfbe9b735793a5e63a38d83ba37292c4ada3615fe54403c54e3260/psutil-1.0.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"7868ef2ecc1d847620d99b40f8105b6d5303784abbde34da2388b2c18e73a3d2"},"provenance":null,"requires-python":null,"size":327529,"upload-time":"2014-02-06T16:25:06.308578Z","url":"https://files.pythonhosted.org/packages/11/46/8a6581c14d644f747ea66a12c60ffebebd4f8a77af62192aeb20cc16ae17/psutil-1.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"cdced3409a241b9d357b82c72d00ece49602cdc3f694ed988a952761367df560"},"provenance":null,"requires-python":null,"size":326023,"upload-time":"2014-02-06T16:25:15.281576Z","url":"https://files.pythonhosted.org/packages/03/a4/aa6592e676f75705b77df8e319f6aa35cf08a662fb0cfaa34434d273d7c9/psutil-1.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"ef6974ac18407fb15430cd7b81aecc2f3666e75ca83fe3a66848c4c5c086e0a3"},"provenance":null,"requires-python":null,"size":326915,"upload-time":"2014-02-06T16:25:27.390929Z","url":"https://files.pythonhosted.org/packages/5b/c1/003f0071d1e2a4dfe12ac2d66ac14f5a5a93049544c11ac9c78d83d85318/psutil-1.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py2.5.exe","hashes":{"sha256":"d8213feaa4c560c61514cea4e6a7eec83f6df83694bb97b06d009e919f60c48f"},"provenance":null,"requires-python":null,"size":165576,"upload-time":"2014-02-06T16:24:22.454259Z","url":"https://files.pythonhosted.org/packages/d6/28/5753ba13863c26d8a8fef685690b539a4652f1c4929d13d482eb4d691d99/psutil-1.0.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py2.6.exe","hashes":{"sha256":"7571d74d351bba723b28666a13da43c07bb3568543be3f9f7404af2b8c1ecf3a"},"provenance":null,"requires-python":null,"size":297205,"upload-time":"2014-02-06T16:24:31.483662Z","url":"https://files.pythonhosted.org/packages/ad/8c/37599876ea078c7c2db10d746f0e0aa805abcacbab96d1cf5dfb0465c40b/psutil-1.0.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py2.7.exe","hashes":{"sha256":"3f311d4f060b83d7cd0068dbad867ad17e699d3c335e494dc31db5380154fdf4"},"provenance":null,"requires-python":null,"size":296787,"upload-time":"2014-02-06T16:24:39.921427Z","url":"https://files.pythonhosted.org/packages/99/4e/8c1f5db3e90dbdbde891c34fd2496805a93aa279433bac716098efad1b4c/psutil-1.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py3.3.exe","hashes":{"sha256":"acf6cbc2408869b2843d406a802b6895969a358c762b5ab3e6f5afdd0c918faf"},"provenance":null,"requires-python":null,"size":291856,"upload-time":"2014-02-06T16:24:48.963293Z","url":"https://files.pythonhosted.org/packages/5b/f7/0bad3fc8ff5f226fa933a847b5fb3355562421f0456348747cf18f7137f5/psutil-1.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.0.win32-py3.4.exe","hashes":{"sha256":"191c2e6953426c861d7e100d48a5b522307ffc3de7d9e7bc8509aa45fb3de60c"},"provenance":null,"requires-python":null,"size":291810,"upload-time":"2014-02-06T16:24:57.589702Z","url":"https://files.pythonhosted.org/packages/0a/50/18ebd254b6662c98a8310983fff6795c7ac303d25188fd727888ca1c5211/psutil-1.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.tar.gz","hashes":{"sha256":"ba4c81622434836f6645e8d04e221ca5b22a9bd508c29989407f116b917be5b3"},"provenance":null,"requires-python":null,"size":156516,"upload-time":"2014-02-06T02:05:16.665325Z","url":"https://files.pythonhosted.org/packages/94/50/7c9e94cf6cdbf4e4e41d2e318094c2b0d58d3bb9196017fb6e4897adf277/psutil-1.0.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win-amd64-py2.7.exe","hashes":{"sha256":"2d6705cf53a7474b992d0d5beac4697cdd37f2c24098d4e1b5bb35a83b70b27e"},"provenance":null,"requires-python":null,"size":327529,"upload-time":"2014-02-06T16:20:20.120751Z","url":"https://files.pythonhosted.org/packages/a9/28/ac3c9da11fbe1ae3ddefb2cb6b410c8ad701f39d86489931e87c871cc4d1/psutil-1.0.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win-amd64-py3.3.exe","hashes":{"sha256":"6194670fcbddec715846fa9beca4450b4419423b2d466894da2fa1915fc7b712"},"provenance":null,"requires-python":null,"size":326023,"upload-time":"2014-02-06T16:20:43.401705Z","url":"https://files.pythonhosted.org/packages/97/30/094bb08295e15e05a266fcb15c116912f643fbb4dd8090ec64aca801ea72/psutil-1.0.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win-amd64-py3.4.exe","hashes":{"sha256":"5cf4ddea8dd21b99b90656dafa2d28c32fa9c159ab30580782b65b3b615c62a7"},"provenance":null,"requires-python":null,"size":326915,"upload-time":"2014-02-06T16:20:52.673334Z","url":"https://files.pythonhosted.org/packages/e8/86/e72c22699491573508e85b226f1782eab840b713cd6b8d42655c5329324c/psutil-1.0.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py2.5.exe","hashes":{"sha256":"fadf7625c309bc5f6e6d3b9d42836491efabbfda1834db7166775d705db420bf"},"provenance":null,"requires-python":null,"size":165580,"upload-time":"2014-02-06T16:04:23.085098Z","url":"https://files.pythonhosted.org/packages/a5/40/d307abd2ee3015e38cbeeac18c6b0f091cdcac496688f4d2c286145fdc5f/psutil-1.0.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py2.6.exe","hashes":{"sha256":"84a257b4adea431473089ea4b798bea98ce64452587ba7b71856e150d88e9213"},"provenance":null,"requires-python":null,"size":297208,"upload-time":"2014-02-06T15:59:31.669293Z","url":"https://files.pythonhosted.org/packages/75/3f/c10ca801aff50ce6ea84f71ce69a32de1b4a1f0439ada94adeccd36e1afd/psutil-1.0.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py2.7.exe","hashes":{"sha256":"4b5047bb4055d48f718bbb5a457c9b1c1c06776ca6353cb310b471c38d51324d"},"provenance":null,"requires-python":null,"size":296787,"upload-time":"2014-02-06T15:59:41.621679Z","url":"https://files.pythonhosted.org/packages/15/6c/944089b8dd730314a8eec9faa4d7d383f31490130e9e0534c882173e2eb6/psutil-1.0.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py3.3.exe","hashes":{"sha256":"3553a27b4d0e7552d0d863c017e152a88cb689d2090b39e2bca68db7fc27e8b2"},"provenance":null,"requires-python":null,"size":291856,"upload-time":"2014-02-06T15:59:51.241604Z","url":"https://files.pythonhosted.org/packages/4c/51/eb8b12a0124050492faad7fe3e695cb44e4ec5495a64a8570abf6e7cddc3/psutil-1.0.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.0.1.win32-py3.4.exe","hashes":{"sha256":"d3ab1b882647d66099adefa8ff780c4144bf8df1bb97bc31c55cbd999807cae8"},"provenance":null,"requires-python":null,"size":291810,"upload-time":"2014-02-06T16:00:00.814707Z","url":"https://files.pythonhosted.org/packages/4d/2a/33063e4a674e3be38d77fbc6e5e988f0e0323cd000cb7eb60f9300493631/psutil-1.0.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.tar.gz","hashes":{"sha256":"31b4b411d3f6960d26dac1f075709ff6d36f60216bf1f2b47fc4dc62a2d0aa6f"},"provenance":null,"requires-python":null,"size":163785,"upload-time":"2013-09-28T09:48:01.746815Z","url":"https://files.pythonhosted.org/packages/f6/71/1f9049fc7936f7f48f0553abd8bf8394f4ad2d9fb63a881b3a653b67abd6/psutil-1.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"ef250fbe783c0b31d9574d06324abd78ebf23c75981a8d0f9319d080435aa056"},"provenance":null,"requires-python":null,"size":305459,"upload-time":"2013-09-28T09:55:10.368138Z","url":"https://files.pythonhosted.org/packages/28/ff/e0dcc8e0194817191817ff02926a734a51d24a91e1cfc5c47be8c1e8508c/psutil-1.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win-amd64-py3.2.exe","hashes":{"sha256":"a95bf68f2189b7cd3f22558b12a989f368335c905cbd3045ca315514538b3115"},"provenance":null,"requires-python":null,"size":306085,"upload-time":"2013-09-28T09:55:36.198407Z","url":"https://files.pythonhosted.org/packages/a5/be/6f3b8f8ff89eaf480f59bbcd3d18c10ee0206891aa5fb4a25280c031801c/psutil-1.1.0.win-amd64-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py2.4.exe","hashes":{"sha256":"bc53be7f65ccb8a530e0c496a8db3ecd78b57b1faa443e43e228df327cf26899"},"provenance":null,"requires-python":null,"size":142269,"upload-time":"2013-09-28T09:51:43.685077Z","url":"https://files.pythonhosted.org/packages/ec/60/0a9a96611c77866a2c4140159a57c2702523ebc5371eaa24e0353d88ac2c/psutil-1.1.0.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py2.5.exe","hashes":{"sha256":"4a587008d6aae91fd1d3fee21141a27b4a2978fff75c80e57a37678a63a1cfcc"},"provenance":null,"requires-python":null,"size":142249,"upload-time":"2013-09-28T09:52:12.373832Z","url":"https://files.pythonhosted.org/packages/b5/16/dd5a768773cb017c4a66d69ec49abdd33df831215418e4c116639ed41afb/psutil-1.1.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py2.6.exe","hashes":{"sha256":"be853a268f0f3c3336c9bb648253db85ef2024213b6fc199f84b89643b74b130"},"provenance":null,"requires-python":null,"size":276570,"upload-time":"2013-09-28T09:53:51.099422Z","url":"https://files.pythonhosted.org/packages/cc/e0/adb41ece0bb2c27d75a41d98bd8f9802475d5d488df7d4905229596c0a70/psutil-1.1.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py3.2.exe","hashes":{"sha256":"dfdce701cfc7799dc1964debb486907b447fbdb361e06be786a9a5e11a1d5309"},"provenance":null,"requires-python":null,"size":275628,"upload-time":"2013-09-28T09:54:16.430842Z","url":"https://files.pythonhosted.org/packages/8f/c9/231df338b0cef518dbde2f98c1e0bb81142dc7dbec4a39267bb60dde1462/psutil-1.1.0.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.0.win32-py3.3.exe","hashes":{"sha256":"b4106508ea96ad24e1102de2a31a99c557ee608abab15d978bfe8d0f626e518c"},"provenance":null,"requires-python":null,"size":270259,"upload-time":"2013-09-28T09:54:43.666687Z","url":"https://files.pythonhosted.org/packages/a7/03/80f426e186dfdb202128c78b421b3ba34de461c32f6ed2c04041fc548d7b/psutil-1.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.tar.gz","hashes":{"sha256":"a5201e4c2a9b57e9e5d8de92b3a4006d093eedd9b56915b8279f365aaedd0f48"},"provenance":null,"requires-python":null,"size":165467,"upload-time":"2013-10-07T22:38:06.357263Z","url":"https://files.pythonhosted.org/packages/8d/0d/1a4bfc94f9cc783a510b3fc7efd5a2ef39858b3a2b6ab40a094a1ca8a54d/psutil-1.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win-amd64-py2.7.exe","hashes":{"sha256":"b8ccaef0a96d7ef40eda7493ffcaa3c5e2d63f531e090c3567e4198b38ca8e33"},"provenance":null,"requires-python":null,"size":305937,"upload-time":"2013-10-07T22:43:47.570605Z","url":"https://files.pythonhosted.org/packages/5a/b8/66489689c8751acd67bdbc8f37534176a14b63f74e74a229c30b230c8a18/psutil-1.1.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win-amd64-py3.2.exe","hashes":{"sha256":"fdabf11b316b0c406709aabb76f6c3cfdeb448c10ed38193786110f13e5c1248"},"provenance":null,"requires-python":null,"size":306563,"upload-time":"2013-10-07T22:44:09.209012Z","url":"https://files.pythonhosted.org/packages/e3/2d/cd5c620f9e5cb090eea573b2dc16574e746529347b6133b2f0b6e686d917/psutil-1.1.1.win-amd64-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py2.4.exe","hashes":{"sha256":"74bb1ccf0b28a914bb9d303642c08450ea6c5876851eb71e17f24b3d8672ca7d"},"provenance":null,"requires-python":null,"size":142751,"upload-time":"2013-10-07T22:45:21.036212Z","url":"https://files.pythonhosted.org/packages/28/9a/b83f884add09296894a1223c9c404cb57155c1c4317d318abf8c170e07b5/psutil-1.1.1.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py2.5.exe","hashes":{"sha256":"8e52d5b6af64e3eefbda30e0b16c9f29434ef6f79ea10b6fbd4520a6fbdb2481"},"provenance":null,"requires-python":null,"size":142731,"upload-time":"2013-10-07T22:41:32.428090Z","url":"https://files.pythonhosted.org/packages/8b/80/c41382a4f650f47a37300411169b7b248acf0b0925eb92cb22286362c3df/psutil-1.1.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py2.6.exe","hashes":{"sha256":"1e3a17b2a2f2bc138774f9b4b5ff52767f65ffb6349f9e05eb24f243c550ce1b"},"provenance":null,"requires-python":null,"size":277048,"upload-time":"2013-10-07T22:41:54.149373Z","url":"https://files.pythonhosted.org/packages/16/cd/25a3b9af88d130dd1084acab467b30996884219afc0a1e989d2a015ea54b/psutil-1.1.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py2.7.exe","hashes":{"sha256":"2e7691ddbde94b1ec7bb5b4cd986b2f763efc6be5b122dd499e156c9802a195b"},"provenance":null,"requires-python":null,"size":276822,"upload-time":"2013-10-07T22:42:18.407145Z","url":"https://files.pythonhosted.org/packages/42/78/eeacb1210abbe15cf06b9810e84afeabae1f9362abe389e8d5ca2c19df43/psutil-1.1.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py3.2.exe","hashes":{"sha256":"01276bd695b5cebc7bea7c97814713b0f3030861e30be88a60d40c9daf3d529c"},"provenance":null,"requires-python":null,"size":276106,"upload-time":"2013-10-07T22:42:41.551135Z","url":"https://files.pythonhosted.org/packages/92/14/90b9a4690f04ef1aab89a97a7b5407708f56785ccc264d9f9ce372feaea4/psutil-1.1.1.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.1.win32-py3.3.exe","hashes":{"sha256":"86ce0cfa7a95c7c1bc3e2f5358e446b01fc2be5f0c879c8def21e7ede9dc77de"},"provenance":null,"requires-python":null,"size":270733,"upload-time":"2013-10-07T22:43:07.069809Z","url":"https://files.pythonhosted.org/packages/a7/64/ba4601de7df6130c27f42bcec9f11da4ea905eda26d2f5a41efdb481f377/psutil-1.1.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.tar.gz","hashes":{"sha256":"adeb1afcb46327bed6603aa8981dce863f052043a52f003e2742ec7c3739677a"},"provenance":null,"requires-python":null,"size":165709,"upload-time":"2013-10-22T18:13:09.038583Z","url":"https://files.pythonhosted.org/packages/e4/b1/34a4bd75027d08c8db4f6301d6562e333c8d9131dca08b7f76f05aeae00a/psutil-1.1.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win-amd64-py2.7.exe","hashes":{"sha256":"4c9e952a7faf50f11fb8bcd6c12c952b063664d9327957c9f6abd498e6ef3bc8"},"provenance":null,"requires-python":null,"size":306157,"upload-time":"2013-10-22T18:16:38.181306Z","url":"https://files.pythonhosted.org/packages/f9/df/437db01296118d668cf654f097ad2b1c341291ba5dc4b5eb80f0a0a40c52/psutil-1.1.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win-amd64-py3.2.exe","hashes":{"sha256":"6e8ed376b63b15b09a94eff7998dae1f8506bd174d795e509cc735f875fddabe"},"provenance":null,"requires-python":null,"size":306782,"upload-time":"2013-10-22T18:17:05.663884Z","url":"https://files.pythonhosted.org/packages/c9/d0/1e413f0258d02bf77bc1d94002d041b8853584369e4af039cd5cf89e3270/psutil-1.1.2.win-amd64-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py2.4.exe","hashes":{"sha256":"ecc5ab4537259db7254c6f6dc7b7cb5a7f398c322a01612c272a8222696334a8"},"provenance":null,"requires-python":null,"size":142969,"upload-time":"2013-10-22T18:13:39.868851Z","url":"https://files.pythonhosted.org/packages/a7/63/fd5770ec4fe87d30bd836989d314b85662c775a52dbd017747fc69fe8f0e/psutil-1.1.2.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py2.5.exe","hashes":{"sha256":"e05d83d6b53ea24333c8b04d329e28ff11ecad75945f371ff5ce7f785df36aee"},"provenance":null,"requires-python":null,"size":142950,"upload-time":"2013-10-22T18:14:00.511107Z","url":"https://files.pythonhosted.org/packages/8d/e1/22650079452725e44ec790c3e75282f4d341f359b213b2afc7f2ada46930/psutil-1.1.2.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py2.6.exe","hashes":{"sha256":"f06ee6e1508a12afcfed04a4022ded9f872e2a964a62bd86617ece943d89ab01"},"provenance":null,"requires-python":null,"size":277266,"upload-time":"2013-10-22T18:14:33.903116Z","url":"https://files.pythonhosted.org/packages/3e/a4/5177488368f230acd4708a117c0820fb16843521e2a7a492078a2335bb9f/psutil-1.1.2.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py2.7.exe","hashes":{"sha256":"6c5be5538202ed7419911178ded41c65e118104fa634109f528a6d2d3e50a7d0"},"provenance":null,"requires-python":null,"size":277039,"upload-time":"2013-10-22T18:14:56.072979Z","url":"https://files.pythonhosted.org/packages/5e/d6/56f2891f6dd56f950866cc39892e5a56e85331d97c39e2634cfc4014f0df/psutil-1.1.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py3.2.exe","hashes":{"sha256":"5cfa97b52fb48dbb8255c120b567be4f06802afa7d2fe71b8fe7c7c4ee53ad88"},"provenance":null,"requires-python":null,"size":276327,"upload-time":"2013-10-22T18:15:25.393310Z","url":"https://files.pythonhosted.org/packages/dd/be/1aea1e7a1a3fb44f4c8d887a1d55e960de283d86875f15457a284268e197/psutil-1.1.2.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.2.win32-py3.3.exe","hashes":{"sha256":"f7dc7507fb9d4edb42709b356eb2e4b3da356efa54d83900e4cef59f3adebfbf"},"provenance":null,"requires-python":null,"size":270952,"upload-time":"2013-10-22T18:15:51.874969Z","url":"https://files.pythonhosted.org/packages/f5/7c/1a33b78a66a96e740e197ae55719496ba57bb9cee32f710a5a6affa68cc8/psutil-1.1.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.tar.gz","hashes":{"sha256":"5e1164086a7ed3b863ebd12315d35086e22252b328401fce901a0862050ef98c"},"provenance":null,"requires-python":null,"size":165550,"upload-time":"2013-11-07T20:47:17.039714Z","url":"https://files.pythonhosted.org/packages/93/fa/1f70b7fcdff77348f4e79d84cc9b568596874ca34940ede78c63d503a095/psutil-1.1.3.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win-amd64-py2.7.exe","hashes":{"sha256":"43cbf089cbe160d193e1658fe4eec4a719430432e866334b25dc3acef73c3e61"},"provenance":null,"requires-python":null,"size":306145,"upload-time":"2013-11-07T21:05:00.986042Z","url":"https://files.pythonhosted.org/packages/03/ec/f05db404504d67a19397e17e64f0276cc610a9dc450eb51ed70436f37c43/psutil-1.1.3.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py2.4.exe","hashes":{"sha256":"ef7445e1b0449af34ff51a9e1d3f030b8794c9fcd2b99d072b2e815d00e8a783"},"provenance":null,"requires-python":null,"size":142958,"upload-time":"2013-11-07T21:16:11.539460Z","url":"https://files.pythonhosted.org/packages/37/c2/5d40dd0a36f0280c1dea0f651cfde79b5e99a0e6fab92273fa3ac41055f0/psutil-1.1.3.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py2.5.exe","hashes":{"sha256":"b7f4103b1058d2ee597f88902ff50f19421b95c18ac9b82ea9e8a00091786131"},"provenance":null,"requires-python":null,"size":142940,"upload-time":"2013-11-07T21:15:50.256574Z","url":"https://files.pythonhosted.org/packages/ee/cb/7d42052f4057c6233ba3f4b7afade92117f58c5d7544ee6ab16e82c515c7/psutil-1.1.3.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py2.6.exe","hashes":{"sha256":"e21b9c5a26ad3328b6f5b629e30944a9b7e56ea6ac0aa051455c8a352e67fbca"},"provenance":null,"requires-python":null,"size":277254,"upload-time":"2013-11-07T20:58:49.794773Z","url":"https://files.pythonhosted.org/packages/b2/7e/c42d752b333c5846d88a8b56bbab23325b247766c100dc6f68c6bc56019d/psutil-1.1.3.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py2.7.exe","hashes":{"sha256":"fc9732a4dea2a3f73f9320472aa54e1cfae45d324bea24bd207d88d3a0a281b0"},"provenance":null,"requires-python":null,"size":277028,"upload-time":"2013-11-07T21:16:56.712839Z","url":"https://files.pythonhosted.org/packages/2b/76/eec917b6f74ea9bd20a55dd8b4b01e69689278235335dbedc2b9be212815/psutil-1.1.3.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py3.2.exe","hashes":{"sha256":"a141d59c03a28bdd32dfce11a5d51b9a63a8e6c6c4245e416eeffcc1c43f1de9"},"provenance":null,"requires-python":null,"size":276315,"upload-time":"2013-11-07T21:02:04.504719Z","url":"https://files.pythonhosted.org/packages/3a/b9/748bbd0c53c74c682051137830048abd0126ae50e3bf4d5854a0188da143/psutil-1.1.3.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.1.3.win32-py3.3.exe","hashes":{"sha256":"36e855ea7c872beb04f22d968c55f1ef897df0261ea993ecf5ae6e8216939a55"},"provenance":null,"requires-python":null,"size":270942,"upload-time":"2013-11-07T21:04:24.025560Z","url":"https://files.pythonhosted.org/packages/e6/7b/655e08abdf19d06c8a3fd6fba35932867c80fcff05c97cf909b5d364603c/psutil-1.1.3.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.tar.gz","hashes":{"sha256":"c3a2b02e92363837499680760b674b3bf3bd03dd9528a5dc41392625a61b162a"},"provenance":null,"requires-python":null,"size":166747,"upload-time":"2013-11-20T20:02:56.854747Z","url":"https://files.pythonhosted.org/packages/77/89/8ee72ae2b5e3c749e43a9c1e95d61eceff022ab7c929cdde8a3b7539a707/psutil-1.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"2bedcda55618f71bc98d6669bd190e2b51b940e3a3af2512dd77745cb1ebc101"},"provenance":null,"requires-python":null,"size":307021,"upload-time":"2013-11-20T20:08:29.733072Z","url":"https://files.pythonhosted.org/packages/9c/d1/598bcaa9701e06561cb8823a7a04ee44e702b0e327ef0a65bdd97b019613/psutil-1.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py2.4.exe","hashes":{"sha256":"417695f66562a22ac4f92bf7df1e9dda6264579eb308badd2c3e85df88ab9436"},"provenance":null,"requires-python":null,"size":143831,"upload-time":"2013-11-20T20:18:44.725428Z","url":"https://files.pythonhosted.org/packages/a4/ba/4b54baace7b49b73df74b54815337878137d25fda506e518f2d8dd2472fc/psutil-1.2.0.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py2.5.exe","hashes":{"sha256":"40747c59cf92cef26d595a9cefaac9c1a4bc0290abff6004bb092ee2ddec7d7b"},"provenance":null,"requires-python":null,"size":143812,"upload-time":"2013-11-20T20:18:35.170091Z","url":"https://files.pythonhosted.org/packages/88/17/3ba1b45ec63d666c2631f16679adb819564feb775cfd9e774d23e5774a44/psutil-1.2.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py2.6.exe","hashes":{"sha256":"fb0c3942a48fde9c52828dae4074381556eff8eb5651bdc9c4c107b40dc8318e"},"provenance":null,"requires-python":null,"size":278130,"upload-time":"2013-11-20T20:07:34.211552Z","url":"https://files.pythonhosted.org/packages/ae/4a/e62000dbe462270c30f1e2f2dcc913596f70e264d9a656f881bb9f487283/psutil-1.2.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py2.7.exe","hashes":{"sha256":"1024eff34ff14f2899186549ca8f8e461403e5700910e52eb013ffffb078cda3"},"provenance":null,"requires-python":null,"size":277905,"upload-time":"2013-11-20T20:18:07.310116Z","url":"https://files.pythonhosted.org/packages/9d/1a/b8e679a7e47229d07e41439a43fc1be48c0d34774e5e35ad6730398485bd/psutil-1.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py3.2.exe","hashes":{"sha256":"2b588673530d67f45287c31b6dfef02ee4e753aee7e7216448e6dbcb95b482e1"},"provenance":null,"requires-python":null,"size":277189,"upload-time":"2013-11-20T20:05:01.902941Z","url":"https://files.pythonhosted.org/packages/05/10/20a2364e4e69206d66da106288b65bf5bbde0d648aad5bb829f3eb08fabb/psutil-1.2.0.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.0.win32-py3.3.exe","hashes":{"sha256":"3e23d3b1ef5f20a458092887d1df32f55eaf4fc86f2ed49cc68cf164537128d6"},"provenance":null,"requires-python":null,"size":271857,"upload-time":"2013-11-20T20:05:28.007235Z","url":"https://files.pythonhosted.org/packages/24/39/4630dff3d0fa4896db905a118877fce5b72ced6629ca9958e207f7a0b198/psutil-1.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.tar.gz","hashes":{"sha256":"508e4a44c8253a386a0f86d9c9bd4a1b4cbb2f94e88d49a19c1513653ca66c45"},"provenance":null,"requires-python":null,"size":167397,"upload-time":"2013-11-25T20:06:20.566209Z","url":"https://files.pythonhosted.org/packages/8a/45/3b9dbd7a58482018927f756de098388ee252dd230143ddf486b3017117b1/psutil-1.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"f3af7b44925554531dff038e0401976a6b92b089ecca51d50be903420d7a262d"},"provenance":null,"requires-python":null,"size":307074,"upload-time":"2013-11-25T20:09:25.387664Z","url":"https://files.pythonhosted.org/packages/40/47/665755b95ad75e6223af96f2d7c04667f663a53dede0315df9832c38b60d/psutil-1.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"26fd84571ac026861d806a9f64e3bbfd38d619a8195c1edfd31c0a9ee2295b03"},"provenance":null,"requires-python":null,"size":329716,"upload-time":"2014-02-06T00:57:26.931878Z","url":"https://files.pythonhosted.org/packages/d8/65/2e9941492b3d001a87d87b5e5827b1f3cec42e30b7110fa82d24be8c4526/psutil-1.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py2.4.exe","hashes":{"sha256":"84a4e7e2de1ca6f45359cfd7fd60c807f494385a6d97804bd58759a94b9c5e2d"},"provenance":null,"requires-python":null,"size":143888,"upload-time":"2013-11-25T20:07:25.331363Z","url":"https://files.pythonhosted.org/packages/5c/99/d9147b76eea8c185b6cefbb46de73ae880e5ef0ff36d93cb3f6084e50d59/psutil-1.2.1.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py2.5.exe","hashes":{"sha256":"aad7d81607b3ad740fee47b27a9f1434a05fe35dc68abefb1f961f74bae3c3f9"},"provenance":null,"requires-python":null,"size":143868,"upload-time":"2013-11-25T20:07:40.595942Z","url":"https://files.pythonhosted.org/packages/bd/a0/5087d4a5145326a5a07a53ed4f9cd5c09bf5dad4f8d7b9850e6aaa13caa2/psutil-1.2.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py2.6.exe","hashes":{"sha256":"3ad3a40afd859cf0217a2d643c74be3a12ed2f54ebd4a91c40fa7b13084573c6"},"provenance":null,"requires-python":null,"size":278185,"upload-time":"2013-11-25T20:07:59.926444Z","url":"https://files.pythonhosted.org/packages/c6/e0/67810b602a598488d1f2982451655427effe7c7062184fe036c2b5bc928f/psutil-1.2.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py2.7.exe","hashes":{"sha256":"02fb79b9e5336ff179c44ce2017308cf46316e19bea70abb8855afd808db2a0f"},"provenance":null,"requires-python":null,"size":277955,"upload-time":"2013-11-25T20:08:20.641779Z","url":"https://files.pythonhosted.org/packages/5b/7f/9334b57597acabaaf2261c93bb9b1f9f02cdfef5c1b1aa808b262f770adb/psutil-1.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py3.2.exe","hashes":{"sha256":"7e64d065f12e8f941f2dbb2f3df0887b2677fee7b2b4c50ed91e490e094c7273"},"provenance":null,"requires-python":null,"size":277243,"upload-time":"2013-11-25T20:08:40.628203Z","url":"https://files.pythonhosted.org/packages/ae/c5/2842c69c67ae171f219efa8bb11bddc2fcec2cea059721e716fe4d48b50c/psutil-1.2.1.win32-py3.2.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-1.2.1.win32-py3.3.exe","hashes":{"sha256":"1a1c8e6635949a698b6ade3d5d2a8368daff916d8122cf13286c79a52ec8d7a1"},"provenance":null,"requires-python":null,"size":271900,"upload-time":"2013-11-25T20:09:05.464640Z","url":"https://files.pythonhosted.org/packages/86/df/007ca575da6ee7cbb015dc00122028ee0c97fc6a0c9e8bc02333753bfd2f/psutil-1.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.tar.gz","hashes":{"sha256":"38af34b0f40a4f50988a7401b7111ae4468beb5bcce0fbae409504dd3d5f2e8d"},"provenance":null,"requires-python":null,"size":207168,"upload-time":"2014-03-10T11:23:38.510743Z","url":"https://files.pythonhosted.org/packages/9c/2c/d4380234ddc21ecfb03691a982f5f26b03061e165658ac455b61886fe3ff/psutil-2.0.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"32f27f9be7c03f60f61dc27af2464cfb1edd64525d05b81478e80dc12913fe3b"},"provenance":null,"requires-python":null,"size":310569,"upload-time":"2014-03-10T11:16:14.555076Z","url":"https://files.pythonhosted.org/packages/c3/10/c0e1b505d7d2b4a7f3294c1b4b2bc2644a4629462d777fe2cdcd57b1debe/psutil-2.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"219cf2e5832cf68798e522e815d79a7307dea1f6b1d9b2372704f6e7fea085f3"},"provenance":null,"requires-python":null,"size":309024,"upload-time":"2014-03-10T11:16:29.122028Z","url":"https://files.pythonhosted.org/packages/eb/db/ab023b5ce09f314ee58ee4b9e73e85172dd06272501570a34a1afe6115c2/psutil-2.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"12b9c02e108e887a43e0ed44e3a8e9968e65236d6d0b79c45891471ea2b9e14d"},"provenance":null,"requires-python":null,"size":310130,"upload-time":"2014-03-10T11:16:49.562585Z","url":"https://files.pythonhosted.org/packages/67/94/0dded4aab9c4992bddb311d2ae8fd9a638df5f6039d12a4fe66481f3ea1c/psutil-2.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py2.4.exe","hashes":{"sha256":"54d8636623a4f676a9a38b0afe3dfad5f1b90f710f2cb6c55e1a0803813d76a5"},"provenance":null,"requires-python":null,"size":143691,"upload-time":"2014-03-10T11:20:46.107824Z","url":"https://files.pythonhosted.org/packages/76/a3/48c0984c0b65be53a9e3e090df0cd5f3e6bddd767c3f8e62cf286be240e1/psutil-2.0.0.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py2.5.exe","hashes":{"sha256":"f669fe7e7cab107fb738362366cc9e0ecda532269ac3a9815a28930b474edf0b"},"provenance":null,"requires-python":null,"size":147735,"upload-time":"2014-03-10T11:15:02.487295Z","url":"https://files.pythonhosted.org/packages/f8/ba/346cc719249b9a5281dab059cb8796aff6faf487142f50966fc08330ad79/psutil-2.0.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py2.6.exe","hashes":{"sha256":"33b580008e0c65073198472da81d0d0c50d2a30e3e82c269713b1e3bdf14c2c6"},"provenance":null,"requires-python":null,"size":280920,"upload-time":"2014-03-10T11:15:17.132174Z","url":"https://files.pythonhosted.org/packages/2f/6f/7326e900c5333d59aa96a574d13321c94a9357ab56b0dd489b8f24ebab78/psutil-2.0.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py2.7.exe","hashes":{"sha256":"4f9c78cdd57e1c83242096f8343617ae038efd1c23af3864c25992335eabed3f"},"provenance":null,"requires-python":null,"size":280693,"upload-time":"2014-03-10T11:15:31.704968Z","url":"https://files.pythonhosted.org/packages/18/dd/c81485b54894c35fd8b62822563293db7c4dd17a05ea4eade169cf383266/psutil-2.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py3.3.exe","hashes":{"sha256":"380f8ff680ce3c8fdd7b31a3fe42d697b15a0c8677559691ed90b755051a5acf"},"provenance":null,"requires-python":null,"size":275631,"upload-time":"2014-03-10T11:15:45.735326Z","url":"https://files.pythonhosted.org/packages/77/5a/e87efff3e46862421a9c87847e63ebecd3bb332031305b476399918fea4f/psutil-2.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.0.0.win32-py3.4.exe","hashes":{"sha256":"4239b2431c825db2ad98e404a96320f0c78fb1c7d5bdf52a49a00d36bedfa0df"},"provenance":null,"requires-python":null,"size":275638,"upload-time":"2014-03-10T11:16:00.263121Z","url":"https://files.pythonhosted.org/packages/c4/80/35eb7f189482d25e3669871e7dcd295ec38f792dc4670b8635d72b4f949a/psutil-2.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.tar.gz","hashes":{"sha256":"d1e3ce46736d86164b6d72070f2cbaf86dcf9db03066b7f36a7b302e334a8d01"},"provenance":null,"requires-python":null,"size":211640,"upload-time":"2014-04-08T14:59:37.598513Z","url":"https://files.pythonhosted.org/packages/6c/d1/69431c4fab9b5cecaf28f2f2e0abee21805c5783c47143db5f0f7d42dbec/psutil-2.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"9dc35f899946d58baf5b805ebed2f723f75d93f6b6bce212f79581d40f7276db"},"provenance":null,"requires-python":null,"size":312139,"upload-time":"2014-04-08T15:13:59.193442Z","url":"https://files.pythonhosted.org/packages/7a/3c/ce6a447030cdb50cf68a3988337df0c42e52abf45c3adfea3b225760eb70/psutil-2.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win-amd64-py3.3.exe","hashes":{"sha256":"46232b7b0eb48c6ec1f12d17b2cc15ec3cc70ed952f2c98edb50aaa405dafa5d"},"provenance":null,"requires-python":null,"size":310591,"upload-time":"2014-04-08T15:14:05.019441Z","url":"https://files.pythonhosted.org/packages/cd/60/7134a7f812ef1eba9373c86b95ce6254f5f58928baba04af161f0066629f/psutil-2.1.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win-amd64-py3.4.exe","hashes":{"sha256":"fa770a2c9e384924df021aed9aa5a6f92db8062a7f700113ba3943e262028454"},"provenance":null,"requires-python":null,"size":311682,"upload-time":"2014-04-08T15:14:24.701683Z","url":"https://files.pythonhosted.org/packages/45/56/3ac0a63799d54cb9b1914214944aed71e49297fb90de92b8d1fe20de3bd8/psutil-2.1.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py2.4.exe","hashes":{"sha256":"15d0d10bbd60e461690188cdd209d6688a1112648589bf291b17cc84e99cb6e7"},"provenance":null,"requires-python":null,"size":145275,"upload-time":"2014-04-08T15:12:04.809516Z","url":"https://files.pythonhosted.org/packages/09/e8/f6e4209b3d6373ea11fa340c2e6e4640a7ee53ef4697c49d610ffdf86674/psutil-2.1.0.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py2.5.exe","hashes":{"sha256":"381ac1027a270c04cf6decdc011a28a4270105d009f213d62daec4b3116b92b6"},"provenance":null,"requires-python":null,"size":149378,"upload-time":"2014-04-08T15:12:25.638187Z","url":"https://files.pythonhosted.org/packages/6d/05/0e7213e6f0dc71490a523a70e6ab5e7cd5140d87dc93a4447da58c440d6b/psutil-2.1.0.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py2.6.exe","hashes":{"sha256":"7eda949fbdf89548b0e52e7d666e096ea66b30b65bcbe04d305e036a24a76d11"},"provenance":null,"requires-python":null,"size":282509,"upload-time":"2014-04-08T15:12:57.055047Z","url":"https://files.pythonhosted.org/packages/f2/88/a856c5ed36d15b8ad74597f380baa29891ec284e7a1be4cb2f91f8453bd8/psutil-2.1.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py2.7.exe","hashes":{"sha256":"f1ec06db1b0d27681a1bea4c9f0b33705bc5a00035c32f168da0ea193883bb91"},"provenance":null,"requires-python":null,"size":282317,"upload-time":"2014-04-08T15:13:11.798772Z","url":"https://files.pythonhosted.org/packages/d0/17/ec99e9252dae834f4a012e13c804c86907fcb1cb474f7b1bc767562bfa7b/psutil-2.1.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py3.3.exe","hashes":{"sha256":"feccef3ccf09785c6f81e67223cf2d8736a90c8994623dd75f683dd2bf849235"},"provenance":null,"requires-python":null,"size":277268,"upload-time":"2014-04-08T15:13:26.711424Z","url":"https://files.pythonhosted.org/packages/f0/0a/32abfd9b965c9a433b5011574c904372b954330c170c6f92b637d661ecd2/psutil-2.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.0.win32-py3.4.exe","hashes":{"sha256":"4719134be984b0f2ba072ff761f334c2f3dbb8ca6af70ba43d8ca31b7e13c3db"},"provenance":null,"requires-python":null,"size":277279,"upload-time":"2014-04-08T15:13:41.834397Z","url":"https://files.pythonhosted.org/packages/f5/3b/eab6a8d832d805c7a00d0f2398c12a32bea7e8b6eb7d5fbdf869e4bcc9e0/psutil-2.1.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.tar.gz","hashes":{"sha256":"bf812a4aa6a41147d0e96e63d826eb7582fda6b54ad8f22534354b7f8ac45593"},"provenance":null,"requires-python":null,"size":216796,"upload-time":"2014-04-30T14:27:01.651394Z","url":"https://files.pythonhosted.org/packages/64/4b/70601d39b8e445265ed148affc49f7bfbd246940637785be5c80e007fa6e/psutil-2.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win-amd64-py2.7.exe","hashes":{"sha256":"1b5ee64306535a625b77648e32f1b064c31bf58bfc7a37cde6c7bb3fa4abb6bd"},"provenance":null,"requires-python":null,"size":312763,"upload-time":"2014-04-30T14:31:09.411365Z","url":"https://files.pythonhosted.org/packages/e6/50/df05e0cbfcf20f022756d5e2da32b4f4f37d5bca6f5bd6965b4ef0460e8b/psutil-2.1.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win-amd64-py3.3.exe","hashes":{"sha256":"9e28fe35e1185d59ba55313d90efc122d45b7a2e3fa304869024d26b0a809bc5"},"provenance":null,"requires-python":null,"size":311262,"upload-time":"2014-04-30T14:31:28.733500Z","url":"https://files.pythonhosted.org/packages/a3/a9/b3f114141a49244739321f221f35c300ac7f34ec9e3a352ea70c9fae41f8/psutil-2.1.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win-amd64-py3.4.exe","hashes":{"sha256":"3afa327284a218d6a25b2f3520135337cfa4a47eaea030273ad8eb02894d60fe"},"provenance":null,"requires-python":null,"size":312365,"upload-time":"2014-04-30T14:31:47.981590Z","url":"https://files.pythonhosted.org/packages/c7/67/2aae3f66090c2e06fa60c29a2e554fd2a718949aca53bca78d640212cb34/psutil-2.1.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py2.4.exe","hashes":{"sha256":"7c76d99bfaeafbcf66096c69b8fca1f7269603d66cad1b14cd8dd93b14bceeb0"},"provenance":null,"requires-python":null,"size":145848,"upload-time":"2014-04-30T14:37:26.909225Z","url":"https://files.pythonhosted.org/packages/27/4e/c9b4802420d6b5d0a844208c9b8a4e25b3c37305428e40bc1da6f5076891/psutil-2.1.1.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py2.5.exe","hashes":{"sha256":"6aeb358b66cc4367378edadb928c77e82646f36858b2a516298d1917aa6aca25"},"provenance":null,"requires-python":null,"size":149998,"upload-time":"2014-04-30T14:28:49.630619Z","url":"https://files.pythonhosted.org/packages/5d/e9/69dee6454940bb102fbbcaa1e44b46bfefc4c0bf53e5e3835d3de3ebc9ae/psutil-2.1.1.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py2.6.exe","hashes":{"sha256":"a53546550067773920e1f7e6e3f1fad2f2befe45c6ff6c94e22b67f4b54c321a"},"provenance":null,"requires-python":null,"size":283143,"upload-time":"2014-04-30T14:29:06.869445Z","url":"https://files.pythonhosted.org/packages/9f/45/d2dfa6bdd9b64bfd46ad35af774c3819e0d5abf23a99d51adf11984ca658/psutil-2.1.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py2.7.exe","hashes":{"sha256":"37ce747f9042c375f62e861d627b5eb7bace24767303f5d0c4c03d17173a551c"},"provenance":null,"requires-python":null,"size":282941,"upload-time":"2014-04-30T14:29:27.059188Z","url":"https://files.pythonhosted.org/packages/78/47/14db8651f9863d301c0673d25fa22b87d13fde2974f94854502886a21fd1/psutil-2.1.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py3.3.exe","hashes":{"sha256":"8f0f88752e1e9bfeced78daf29d90e0028d17f39b805bb0acf70fefe77ba5ccb"},"provenance":null,"requires-python":null,"size":277944,"upload-time":"2014-04-30T14:29:49.405456Z","url":"https://files.pythonhosted.org/packages/f1/63/2fcaa58b101dce55a12f768508a7f0a0028ccc5a90633d86dd4cc0bcdb52/psutil-2.1.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.1.win32-py3.4.exe","hashes":{"sha256":"bc353df13a6ea40651cba82810a38590e2439c158cf6f130cd0876d0dda53118"},"provenance":null,"requires-python":null,"size":279092,"upload-time":"2014-04-30T14:30:08.449338Z","url":"https://files.pythonhosted.org/packages/5f/26/0be35c7f3dc9e78d405ed6be3aa76a9ce97b51e0076db98408a6f2c288fb/psutil-2.1.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"data-dist-info-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"filename":"psutil-2.1.2-cp26-none-win32.whl","hashes":{"sha256":"ccb5e28357a4b6c572b97e710a070f349e9b45172315eaed6e0b72e86d333b68"},"provenance":null,"requires-python":null,"size":83869,"upload-time":"2014-09-21T13:47:09.164866Z","url":"https://files.pythonhosted.org/packages/c7/22/811ac7c641191e3b65c053c95eb34f6567bbc5155912630464271ab6f3df/psutil-2.1.2-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"data-dist-info-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"filename":"psutil-2.1.2-cp27-none-win32.whl","hashes":{"sha256":"41992126c0281c2f5f279a0e8583382a3b840bd0d48262dfb7bc3cb67bdc6587"},"provenance":null,"requires-python":null,"size":83672,"upload-time":"2014-09-21T13:47:14.034296Z","url":"https://files.pythonhosted.org/packages/79/9d/154b179a73695ae605856c2d77ab5da2a66ef4819c3c4f97e4ab297a2902/psutil-2.1.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"data-dist-info-metadata":{"sha256":"32ea7aa3898a3a77f4d45a211e8c96889ec6e5f933ea2ef9e4249d53fc41e4a1"},"filename":"psutil-2.1.2-cp27-none-win_amd64.whl","hashes":{"sha256":"79cb57bba4cbeebb7e445d19c531107493458a47d0f4c888ce49fc8dec670c32"},"provenance":null,"requires-python":null,"size":85848,"upload-time":"2014-09-21T13:47:30.044802Z","url":"https://files.pythonhosted.org/packages/00/ae/567c30ff44c263cc78dfe50197184e58b62ca9fcfebad144cd235f5d8d2d/psutil-2.1.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"data-dist-info-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"filename":"psutil-2.1.2-cp33-none-win32.whl","hashes":{"sha256":"c5e6424833620f0d0a70ebc1305260bfad4b5c73989b8a2e8bd01bf39b4b600c"},"provenance":null,"requires-python":null,"size":83742,"upload-time":"2014-09-21T13:47:18.481401Z","url":"https://files.pythonhosted.org/packages/62/4d/be87c318274ade9d6589e8545dd5d5bb4bcc42e92334d88d948cf8daa36b/psutil-2.1.2-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"data-dist-info-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"filename":"psutil-2.1.2-cp33-none-win_amd64.whl","hashes":{"sha256":"b4c69627c245025b9209acbffc066ff8ac1237d13e2eddd0461f3149969b7da3"},"provenance":null,"requires-python":null,"size":85822,"upload-time":"2014-09-21T13:47:33.681070Z","url":"https://files.pythonhosted.org/packages/bc/52/ab3c88a0574275ec33d9de0fe7dc9b1a32c573de0e468502876bc1df6f84/psutil-2.1.2-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"data-dist-info-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"filename":"psutil-2.1.2-cp34-none-win32.whl","hashes":{"sha256":"97a1d0ddbb028a20feffaf7fa3d0c451004abc74ef7ea1e492d039f8b7278399"},"provenance":null,"requires-python":null,"size":83748,"upload-time":"2014-09-21T13:47:25.367914Z","url":"https://files.pythonhosted.org/packages/ef/4a/c956675314e4a50b319d9894c2ee2d48ce83df4d639d9c2fc06a99415dec/psutil-2.1.2-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"data-dist-info-metadata":{"sha256":"e9eb9bc1a4851e5f154ad139504894781ffa9f0ee16ea77f8c17f4aee9d5aa50"},"filename":"psutil-2.1.2-cp34-none-win_amd64.whl","hashes":{"sha256":"a78226f9236c674d43b206e7da06478cf2bcf10e5aee647f671878ea2434805f"},"provenance":null,"requires-python":null,"size":85789,"upload-time":"2014-09-21T13:47:38.760178Z","url":"https://files.pythonhosted.org/packages/b9/41/68b5fc38b97e037ef55e6475d618c079fe2b5d148f5e3fda795c21d888a7/psutil-2.1.2-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.tar.gz","hashes":{"sha256":"897e5163e0669001bf8bcb0557362f14703356336519082a93c38d54e5b392e4"},"provenance":null,"requires-python":null,"size":223595,"upload-time":"2014-09-21T13:50:56.650693Z","url":"https://files.pythonhosted.org/packages/53/6a/8051b913b2f94eb00fd045fe9e14a7182b6e7f088b12c308edd7616a559b/psutil-2.1.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win-amd64-py2.7.exe","hashes":{"sha256":"3ed7ef4ab59894e6eb94adae0b656733c91906af169b55443329315322cd63b3"},"provenance":null,"requires-python":null,"size":319335,"upload-time":"2014-09-21T13:48:15.057062Z","url":"https://files.pythonhosted.org/packages/a6/88/cc912d38640ddf3441db1f85f8ff8a87f906056554239a4a211970cf6446/psutil-2.1.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win-amd64-py3.3.exe","hashes":{"sha256":"e20f317187a7186e13dc6fdd6960083c227db86fffbb145fe000e8c167a854a6"},"provenance":null,"requires-python":null,"size":317817,"upload-time":"2014-09-21T13:48:20.665047Z","url":"https://files.pythonhosted.org/packages/d2/b4/887323eb2b0b5becb5a7b77ab04167741346fddffe27bc6ae5deeffcc3c1/psutil-2.1.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win-amd64-py3.4.exe","hashes":{"sha256":"f7f2a61be7a828656dceffdf0d2fa304a144db6ab5ec4e2b033108f3822df6ff"},"provenance":null,"requires-python":null,"size":317804,"upload-time":"2014-09-21T13:48:33.226186Z","url":"https://files.pythonhosted.org/packages/57/e5/c041b08bea32246f53b7cf27b897e882a695f3ea95eb632b384dd35cf41f/psutil-2.1.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py2.4.exe","hashes":{"sha256":"77aac0179fe0e8b39c9b25ea732285d307bb941e715075366dc467b8609ebf09"},"provenance":null,"requires-python":null,"size":155426,"upload-time":"2014-09-21T13:54:03.017765Z","url":"https://files.pythonhosted.org/packages/ce/3f/7c434baec7ca47e65fb1e3bcbc8fe1c9f504d93c750a7fa4f98b16635920/psutil-2.1.2.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py2.5.exe","hashes":{"sha256":"982f4568876d10881f9c90fe992d1d9af353a59b5de7771cc4766d7432aee7ab"},"provenance":null,"requires-python":null,"size":155414,"upload-time":"2014-09-21T13:47:47.970150Z","url":"https://files.pythonhosted.org/packages/46/86/2971b31e2637ddc53adce2d997472cee8d4dec366d5a1e140945b95860d5/psutil-2.1.2.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py2.6.exe","hashes":{"sha256":"f35d91b5fc2b52472c0766fa411ac22f7a5ed8b1ececd96c1486380bdad0bb41"},"provenance":null,"requires-python":null,"size":289714,"upload-time":"2014-09-21T13:47:52.391726Z","url":"https://files.pythonhosted.org/packages/dd/3e/cdc3d4f343e6edc583d0686d1d20d98f9e11de35c51dc7990525ab2ca332/psutil-2.1.2.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py2.7.exe","hashes":{"sha256":"dbd9e4fbe08c0ff6d4a87a4d6dfae16fccd03346f980cffa105941de8801d00b"},"provenance":null,"requires-python":null,"size":289515,"upload-time":"2014-09-21T13:47:57.624310Z","url":"https://files.pythonhosted.org/packages/fb/01/73753147113f6db19734db6e7ac994cee5cce0f0935e12320d7aa1b56a14/psutil-2.1.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py3.3.exe","hashes":{"sha256":"73428e7b695e5889f07f1b37c5ec0cc0f49d3808dc40986050f1b13fd7c4c71e"},"provenance":null,"requires-python":null,"size":284506,"upload-time":"2014-09-21T13:48:02.428235Z","url":"https://files.pythonhosted.org/packages/b0/97/ba13c3915aba7776bb0d23819c04255230c46df1f8582752f7e0382c0b67/psutil-2.1.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.2.win32-py3.4.exe","hashes":{"sha256":"e2326c7b9f64a0f22891d3b362efcd92389e641023b07bc54567280c0c2e160d"},"provenance":null,"requires-python":null,"size":284538,"upload-time":"2014-09-21T13:48:08.358281Z","url":"https://files.pythonhosted.org/packages/88/bd/843fadc578d62f2888d5a7b4e2a418a28140ac239a5a5039c0d3678df647/psutil-2.1.2.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"data-dist-info-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"filename":"psutil-2.1.3-cp26-none-win32.whl","hashes":{"sha256":"331a3b0b6688f95acebe87c02efca53003e613f1c7892a09063a9d3c0af33656"},"provenance":null,"requires-python":null,"size":83927,"upload-time":"2014-09-26T20:26:46.078909Z","url":"https://files.pythonhosted.org/packages/72/4c/f93448dfe2dec286b6eae91a00c32f1fbf65506e089354c51db76f4efdaf/psutil-2.1.3-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"data-dist-info-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"filename":"psutil-2.1.3-cp27-none-win32.whl","hashes":{"sha256":"5ee9e17f6e92aaec7c53f6483c6708eec22124060a52ac339d104effcb4af37f"},"provenance":null,"requires-python":null,"size":83731,"upload-time":"2014-09-26T20:26:51.215134Z","url":"https://files.pythonhosted.org/packages/27/c5/a644b5df545c467569c7b8e768717fad6758eab8e2544b7d412d07c30ffe/psutil-2.1.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"data-dist-info-metadata":{"sha256":"69a75d24381600fdb642efa3827cf0c4e9c1c94273f2866e017a7017816167f8"},"filename":"psutil-2.1.3-cp27-none-win_amd64.whl","hashes":{"sha256":"47ea728de15c16a7cb476eb4f348be57e9daebfa730b9a4a104e9086da2cf6cd"},"provenance":null,"requires-python":null,"size":85906,"upload-time":"2014-09-26T20:27:07.470817Z","url":"https://files.pythonhosted.org/packages/89/82/1070ecb59d83967ddb2ea58b41ba2b9ffa8f3f686a3f49b8d33dc5684964/psutil-2.1.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"data-dist-info-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"filename":"psutil-2.1.3-cp33-none-win32.whl","hashes":{"sha256":"ae0aac37af1d7d6821bb11dcffb12bb885b44773858bd40703e9dcb14325369c"},"provenance":null,"requires-python":null,"size":83804,"upload-time":"2014-09-26T20:26:56.233644Z","url":"https://files.pythonhosted.org/packages/b6/fc/7f2ceac523bb5ce9ea93bd85806ebb2d5327f0e430cbb6e70d32e4306e1d/psutil-2.1.3-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"data-dist-info-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"filename":"psutil-2.1.3-cp33-none-win_amd64.whl","hashes":{"sha256":"45ebaf679050a6d64a32a91d901791868a29574f6961629997dcd4661edb4ece"},"provenance":null,"requires-python":null,"size":85884,"upload-time":"2014-09-26T20:27:12.800416Z","url":"https://files.pythonhosted.org/packages/45/33/a50ed3def836cd8fc53ce04d09e50df67a6816fd24a819e8e9c45b93fc74/psutil-2.1.3-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"data-dist-info-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"filename":"psutil-2.1.3-cp34-none-win32.whl","hashes":{"sha256":"0ae7d503fc458181af7dd9a2bbf43c4b5fc7cd6e6797ef4a6d58bb8046026d76"},"provenance":null,"requires-python":null,"size":83804,"upload-time":"2014-09-26T20:27:02.155242Z","url":"https://files.pythonhosted.org/packages/55/e9/c122a9d2528cc36ff4f2824a714ee6d5da5462e4b6725e6b1c7098142c4a/psutil-2.1.3-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"data-dist-info-metadata":{"sha256":"6ddf7c489626ecf2cd274a5d66e5785193f3eb6b2d596efff52afc15b251e25f"},"filename":"psutil-2.1.3-cp34-none-win_amd64.whl","hashes":{"sha256":"da0961ba9a6d97c137381cd59b7d330894dd4f7deb5a8291bc3251375fd6d6ec"},"provenance":null,"requires-python":null,"size":85840,"upload-time":"2014-09-26T20:27:18.296315Z","url":"https://files.pythonhosted.org/packages/6f/97/5e01561cde882306c28e462b427f67d549add65f5fca324bf0bbdf831d21/psutil-2.1.3-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.tar.gz","hashes":{"sha256":"b434c75f01715777391f10f456002e33d0ca14633f96fdbd9ff9139b42d9452c"},"provenance":null,"requires-python":null,"size":224008,"upload-time":"2014-09-30T18:10:37.283557Z","url":"https://files.pythonhosted.org/packages/fe/a3/7cf43f28bbb52c4d680378f99e900ced201ade5073ee3a7b30e7f09e3c66/psutil-2.1.3.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win-amd64-py2.7.exe","hashes":{"sha256":"449d07df8f8b9700cfae4ee67f0a73e4f96b55697428ae92cab29e33db4c3102"},"provenance":null,"requires-python":null,"size":319620,"upload-time":"2014-09-26T20:24:42.575161Z","url":"https://files.pythonhosted.org/packages/0e/66/bf4346c9ada08acee7e8f87d270fb4d4e6c86a477630adcfa3caa69aa5bb/psutil-2.1.3.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win-amd64-py3.3.exe","hashes":{"sha256":"5b444e5c1f3d3ee7a19a5720c62d6462fe81dd1d1bbb8aa955546ead509b3c4a"},"provenance":null,"requires-python":null,"size":318101,"upload-time":"2014-09-26T20:24:47.973672Z","url":"https://files.pythonhosted.org/packages/bd/a7/888e9fa42c8d475de7b467b9635bb18ed360cb1c261bcf59f5f932d624ea/psutil-2.1.3.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win-amd64-py3.4.exe","hashes":{"sha256":"3d25b4b20aabd360b7eda3dcbf4a14d2b256c2f61a8a569028e1c4b65b4d585a"},"provenance":null,"requires-python":null,"size":318089,"upload-time":"2014-09-26T20:24:55.536742Z","url":"https://files.pythonhosted.org/packages/b1/15/3f657d395213ad90efed423c88f1cb2f3b0a429d12bdea98a28c26de7ff4/psutil-2.1.3.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py2.4.exe","hashes":{"sha256":"36424b4e683d640291a1723f46a3949ebea37e43492f61027c168fb9dfe2055f"},"provenance":null,"requires-python":null,"size":155710,"upload-time":"2014-09-26T20:29:01.901009Z","url":"https://files.pythonhosted.org/packages/31/2e/d5448240fed09e88bbb59de194e1f2d3ba29f936a3231d718e5373736299/psutil-2.1.3.win32-py2.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py2.5.exe","hashes":{"sha256":"baa1803aaa4505fcf48bbd50d038d1f686d9e290defeecde41770d8fe876812b"},"provenance":null,"requires-python":null,"size":155699,"upload-time":"2014-09-26T20:24:13.810499Z","url":"https://files.pythonhosted.org/packages/2a/45/10d7b5057b3f941f803ca8ee690e651d2778e731f15ef3eee83c3f05f82f/psutil-2.1.3.win32-py2.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py2.6.exe","hashes":{"sha256":"55811a125d8a79bed0a65ed855fafc34f7d59cde162542a30cfa518da9e015bc"},"provenance":null,"requires-python":null,"size":289998,"upload-time":"2014-09-26T20:24:18.935804Z","url":"https://files.pythonhosted.org/packages/f3/4d/7f105269ece54ad7344c2a24b42ecb216b2746460f349c0ed1577b5ab8fa/psutil-2.1.3.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py2.7.exe","hashes":{"sha256":"6a1c8f1884f1983f8b74fda5dc89da07a69e3972fc022c3205f4964e1b01d235"},"provenance":null,"requires-python":null,"size":289802,"upload-time":"2014-09-26T20:24:24.112924Z","url":"https://files.pythonhosted.org/packages/93/64/1432ca27dfd7102a6726161b79b15a2997f461d3835867271a6c1e3353f7/psutil-2.1.3.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py3.3.exe","hashes":{"sha256":"8df72fcd30436e78bf8c5c6c796bb6815966511fa0bc8e3065e1aabbe4a2cf3d"},"provenance":null,"requires-python":null,"size":284794,"upload-time":"2014-09-26T20:24:30.110864Z","url":"https://files.pythonhosted.org/packages/0a/bb/d6cc7133624e3532e1d99f6cece35be7bd8d95ff7c82ca28cd502388c225/psutil-2.1.3.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.1.3.win32-py3.4.exe","hashes":{"sha256":"1f353ab0bbe0216e3a1636bfefb70ed366a2b1d95c95689f97e95d603626ef70"},"provenance":null,"requires-python":null,"size":284824,"upload-time":"2014-09-26T20:24:36.842776Z","url":"https://files.pythonhosted.org/packages/4a/66/9b1acf05dba9b9cb012ab3b8ed3d0fd7b01e1620808977fa6f1efe05dee2/psutil-2.1.3.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"data-dist-info-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"filename":"psutil-2.2.0-cp26-none-win32.whl","hashes":{"sha256":"c92d2853ac52d5aaf8c2366435fb74892f7503daf2cf56b785f1ecadbf68e712"},"provenance":null,"requires-python":null,"size":82092,"upload-time":"2015-01-06T15:36:18.243917Z","url":"https://files.pythonhosted.org/packages/3c/65/f5bd2f8b54f14d950596c0c63d7b0f98b6dc779f7c61ac0dd2e9fc7e5e74/psutil-2.2.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"data-dist-info-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"filename":"psutil-2.2.0-cp27-none-win32.whl","hashes":{"sha256":"6ad0e79b95f57a20f0cace08a063b0fc33fd83da1e5e501bc800bc69329a4501"},"provenance":null,"requires-python":null,"size":81891,"upload-time":"2015-01-06T15:36:28.466138Z","url":"https://files.pythonhosted.org/packages/6a/e6/96a4b4976f0eca169715d975b8c669b78b9afca58f9aadf261915694b73e/psutil-2.2.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"data-dist-info-metadata":{"sha256":"30ed17dae0fd30cf429cfd022f7c9e5e8d675d87901927dda94ded45c224eb33"},"filename":"psutil-2.2.0-cp27-none-win_amd64.whl","hashes":{"sha256":"bbe719046986568ed3e84ce66ed617eebc46251ea8193566b3707abdf635afe6"},"provenance":null,"requires-python":null,"size":84072,"upload-time":"2015-01-06T15:36:58.206788Z","url":"https://files.pythonhosted.org/packages/28/28/4e4f94c9778ed16163f6c3dd696ce5e331121ac1600308830a3e98fa979a/psutil-2.2.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"data-dist-info-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"filename":"psutil-2.2.0-cp33-none-win32.whl","hashes":{"sha256":"73013822a953fc4e3fb269256ec01b261752c590e23851e666201d1bfd32a3a9"},"provenance":null,"requires-python":null,"size":81885,"upload-time":"2015-01-06T15:36:38.776238Z","url":"https://files.pythonhosted.org/packages/66/8d/3143623c2f5bc264197727854040fdc02e3175b4ad991490586db7a512ed/psutil-2.2.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"data-dist-info-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"filename":"psutil-2.2.0-cp33-none-win_amd64.whl","hashes":{"sha256":"d664a896feb10ec5bf4c43f4df8b6c7fceeb94677004cc9cf8c9f35b52c0e4fc"},"provenance":null,"requires-python":null,"size":84002,"upload-time":"2015-01-06T15:37:08.775842Z","url":"https://files.pythonhosted.org/packages/bd/08/112807380b8e7b76de8c84f920234a0ebed35e6511271f93f900f691a10c/psutil-2.2.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"data-dist-info-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"filename":"psutil-2.2.0-cp34-none-win32.whl","hashes":{"sha256":"84299c41b251bef2a8b0812d651f4715209b3c4dfebe4a5df0f103bbdec78221"},"provenance":null,"requires-python":null,"size":81886,"upload-time":"2015-01-06T15:36:49.808417Z","url":"https://files.pythonhosted.org/packages/ef/0e/0cf2fea8f6e2e5ef84797eefc2c5ce561123e2417d7b931a6c54f9f8d413/psutil-2.2.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"data-dist-info-metadata":{"sha256":"ac33f92c3e235fbeaf7e70b46e7b2d1503467bd420fac08e570cd9cfc3d3b738"},"filename":"psutil-2.2.0-cp34-none-win_amd64.whl","hashes":{"sha256":"3636879fcbde2b0b63db08abd0e3673c2cc72bb14075e46e15f98774b0c78236"},"provenance":null,"requires-python":null,"size":83954,"upload-time":"2015-01-06T15:37:18.558007Z","url":"https://files.pythonhosted.org/packages/82/ed/ce9c4281fd8a944a725c0f9a5d77f32295777ab22f54b4f706a51d59edd3/psutil-2.2.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.tar.gz","hashes":{"sha256":"b15cc9e7cad0991bd1cb806fa90ea85ba3a95d0f1226625ecef993294ad61521"},"provenance":null,"requires-python":null,"size":223676,"upload-time":"2015-01-06T15:38:45.979998Z","url":"https://files.pythonhosted.org/packages/ba/e4/760f64a8a5a5f1b95f3ce17c0d51134952f930caf1218e6ce21902f6c4ab/psutil-2.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"623baf2a213adc99cbc067e1db04b9e578eb4c38d9535c903e04b6d1ded10cab"},"provenance":null,"requires-python":null,"size":317990,"upload-time":"2015-01-06T15:34:57.187611Z","url":"https://files.pythonhosted.org/packages/30/a8/d1754e9e5492717d1c1afb30970f0677056052859f1af935b48c72c6cd68/psutil-2.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win-amd64-py3.3.exe","hashes":{"sha256":"fc6d348f603ae8465992a7edbc7c62cc16f0493bdd43aa609dcb7437992cac96"},"provenance":null,"requires-python":null,"size":316424,"upload-time":"2015-01-06T15:35:20.860940Z","url":"https://files.pythonhosted.org/packages/32/f1/e98caa1f6be7ba49dfafa0fbb63f50ffa191d316a1b2b3ec431d97ebf494/psutil-2.2.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win-amd64-py3.4.exe","hashes":{"sha256":"5e4508ca6822839f25310bfc14c050e60362dc29ddea6f5eac91de2c7423a471"},"provenance":null,"requires-python":null,"size":316403,"upload-time":"2015-01-06T15:35:42.094623Z","url":"https://files.pythonhosted.org/packages/dd/46/31505418bd950b09d28c6c21be76a4a51d593ec06b412539797080c0aa6b/psutil-2.2.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win32-py2.6.exe","hashes":{"sha256":"1f93ccdf415da40f15a84ab6d9d32ddda61bb1b20079dae602356e087f408d28"},"provenance":null,"requires-python":null,"size":288364,"upload-time":"2015-01-06T15:33:25.830661Z","url":"https://files.pythonhosted.org/packages/9c/0d/f360da29c906dafd825edccc219e70dab73370ad2c287fa5baa9f7fa370b/psutil-2.2.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win32-py2.7.exe","hashes":{"sha256":"83eb1739b7c87a21a65224d77217325e582199bedd0141b29dac81b4e4144c62"},"provenance":null,"requires-python":null,"size":288166,"upload-time":"2015-01-06T15:33:46.994076Z","url":"https://files.pythonhosted.org/packages/7a/30/5bb6644318f2279caf5d334f32df19fe4b2bce11ef418af32124e16f8e98/psutil-2.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win32-py3.3.exe","hashes":{"sha256":"9db585f37d56381c37738e03aa8beb0b501b26adc7c70660ff182e0473f2cb0a"},"provenance":null,"requires-python":null,"size":283089,"upload-time":"2015-01-06T15:34:11.431221Z","url":"https://files.pythonhosted.org/packages/12/c4/cb75d51edb425ff77e2af5bb32e405ed41beb124ad062fb927bbe135c709/psutil-2.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.0.win32-py3.4.exe","hashes":{"sha256":"4a0d8a192d1523a3f02a4028bf4ac296f7da92c935464d302618bd639c03a2c6"},"provenance":null,"requires-python":null,"size":283110,"upload-time":"2015-01-06T15:34:33.472476Z","url":"https://files.pythonhosted.org/packages/17/fb/ae589efdfd076d1961e1f858c969d234a63e26f648a79b3fac0409e95c2f/psutil-2.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"data-dist-info-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"filename":"psutil-2.2.1-cp26-none-win32.whl","hashes":{"sha256":"610f08fc0df646e4997b52f2bf2e40118a560738422d540883297c6f4e38be8c"},"provenance":null,"requires-python":null,"size":82104,"upload-time":"2015-02-02T13:08:03.993201Z","url":"https://files.pythonhosted.org/packages/0a/a7/b6323d30a8dde3f6a139809898e4bdcabcae3129314621aa24eb5ca4468e/psutil-2.2.1-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"data-dist-info-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"filename":"psutil-2.2.1-cp27-none-win32.whl","hashes":{"sha256":"84ec9e77cab1d9ecf2d93bf43eba255a11b26d480966ee542bc846be1e272ea5"},"provenance":null,"requires-python":null,"size":81907,"upload-time":"2015-02-02T13:08:14.963426Z","url":"https://files.pythonhosted.org/packages/40/b6/a94c6d00ac18f779c72973639b78519de0c488e8e8e8d00b65cf4287bec3/psutil-2.2.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"data-dist-info-metadata":{"sha256":"ff83c284ef44f26c82b6ebdf69d521137b95a89dc815e363bc49eedb63a04c98"},"filename":"psutil-2.2.1-cp27-none-win_amd64.whl","hashes":{"sha256":"81a6ab277886f233f230c46073fa6fc97ca2a95a44a14b80c6f6054acb7a644b"},"provenance":null,"requires-python":null,"size":84086,"upload-time":"2015-02-02T13:08:42.917519Z","url":"https://files.pythonhosted.org/packages/1c/ff/6b00405e4eeb3c40d4fc07142ef7bdc3a7a7aa2a1640b65e654bdb7682d1/psutil-2.2.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"data-dist-info-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"filename":"psutil-2.2.1-cp33-none-win32.whl","hashes":{"sha256":"f5e0d247b09c9460b896ff89098015b7687a2934d4ed6165a5c3a662fad7ab6b"},"provenance":null,"requires-python":null,"size":81896,"upload-time":"2015-02-02T13:08:24.437138Z","url":"https://files.pythonhosted.org/packages/a2/fe/40b7d42057c5e68ff4c733b5689e9ae9d8785f13576326ba44371689eff7/psutil-2.2.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"data-dist-info-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"filename":"psutil-2.2.1-cp33-none-win_amd64.whl","hashes":{"sha256":"07f31cbe5fc1183c8efb52bab9a5382ebd001a6e11b5a5df827f69456219d514"},"provenance":null,"requires-python":null,"size":84011,"upload-time":"2015-02-02T13:08:53.420801Z","url":"https://files.pythonhosted.org/packages/fe/8d/ffc94f092a12fc3cb837b9bda93abd88fb12219bcc6684b9bffea5e1f385/psutil-2.2.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"data-dist-info-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"filename":"psutil-2.2.1-cp34-none-win32.whl","hashes":{"sha256":"a98b350251df2658c9be6bf0b6ebbab10fe88ccb513079c2c56fb53330530ee6"},"provenance":null,"requires-python":null,"size":81896,"upload-time":"2015-02-02T13:08:34.116542Z","url":"https://files.pythonhosted.org/packages/7f/3a/4bb24092c3de0a44f2fc1afb7bcaf898f1daba67cba5b9188b0ae6527102/psutil-2.2.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"data-dist-info-metadata":{"sha256":"85bb3fab01138e16635be140adec7d7fd83c5150a93731deb5a4020ec9d569b1"},"filename":"psutil-2.2.1-cp34-none-win_amd64.whl","hashes":{"sha256":"f49d3ca9150f20269eae79d6efac98f0bbe4e5ceac46fe418d881560a04b6d8e"},"provenance":null,"requires-python":null,"size":83963,"upload-time":"2015-02-02T13:09:03.990685Z","url":"https://files.pythonhosted.org/packages/64/f7/c385e9350abbbb082364596fd4b0066fcb0c51ebc103a7ebf6eb9bc837e9/psutil-2.2.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.tar.gz","hashes":{"sha256":"a0e9b96f1946975064724e242ac159f3260db24ffa591c3da0a355361a3a337f"},"provenance":null,"requires-python":null,"size":223688,"upload-time":"2015-02-02T13:10:14.673714Z","url":"https://files.pythonhosted.org/packages/df/47/ee54ef14dd40f8ce831a7581001a5096494dc99fe71586260ca6b531fe86/psutil-2.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"552eaba2dbc9a49af2da64fc00cedf8847c7c9c2559cc620d9c8855583105764"},"provenance":null,"requires-python":null,"size":317995,"upload-time":"2015-02-02T13:06:42.422390Z","url":"https://files.pythonhosted.org/packages/97/27/e64014166df9efc3d00f987a251938d534d3897d62ef21486559a697a770/psutil-2.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"d25746746f2680d6174c9eb0e2793cb73304ca200d5ec7334000fdb575cd6577"},"provenance":null,"requires-python":null,"size":316429,"upload-time":"2015-02-02T13:07:07.458895Z","url":"https://files.pythonhosted.org/packages/c2/dd/5fdd5fd6102ae546452a5f32a6f75ec44c11d6b84c188dd33cfe1e2809e1/psutil-2.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win-amd64-py3.4.exe","hashes":{"sha256":"298ed760a365a846337750c5089adcec2a87014c61132ae39db671982750f35a"},"provenance":null,"requires-python":null,"size":316409,"upload-time":"2015-02-02T13:07:29.047577Z","url":"https://files.pythonhosted.org/packages/d9/59/0f14daf5797c61db2fec25bca49b49aa0a5d57dec57b4e58ae4652dadb95/psutil-2.2.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win32-py2.6.exe","hashes":{"sha256":"6e49f72a1cd57918b275fdf7dd041618696fc39d5705ac722fdfccaf0b784a93"},"provenance":null,"requires-python":null,"size":288372,"upload-time":"2015-02-02T13:05:10.073479Z","url":"https://files.pythonhosted.org/packages/0c/a2/63967c61fbfc3f60c1b17e652a52d1afeb3c56403990c5a8f2fb801e2aa1/psutil-2.2.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win32-py2.7.exe","hashes":{"sha256":"9227949290983df92bb3bb57c5b25605ebbc0e61a3f9baa394f752ed7abc914c"},"provenance":null,"requires-python":null,"size":288171,"upload-time":"2015-02-02T13:05:31.604222Z","url":"https://files.pythonhosted.org/packages/46/f3/d50f059d7d297db7335e340333124eac9441c5897bb29223c85ebaa64e7d/psutil-2.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win32-py3.3.exe","hashes":{"sha256":"f8d94c8228011d1b658c39783596d95244e01950413bfc41fd44e78129ef7075"},"provenance":null,"requires-python":null,"size":283095,"upload-time":"2015-02-02T13:05:54.134208Z","url":"https://files.pythonhosted.org/packages/4a/d6/341767cdc6890adfb19ad60683e66dab3cdde605bc226427c8ec17dbd3f5/psutil-2.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-2.2.1.win32-py3.4.exe","hashes":{"sha256":"1e1df017bfa14c0edc87a124b11ca6e7fd8b6b5c9692eb06c1b72077bcca3845"},"provenance":null,"requires-python":null,"size":283115,"upload-time":"2015-02-02T13:06:12.827127Z","url":"https://files.pythonhosted.org/packages/8c/c1/24179e79ab74bde86ee766c4c0d2bb90b98e4e4d14017efeec024caba268/psutil-2.2.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"e61eb67de1755dd6d2917f81bb9ee7a0ae6dc5cda2948621b91794f7c5348033"},"data-dist-info-metadata":{"sha256":"e61eb67de1755dd6d2917f81bb9ee7a0ae6dc5cda2948621b91794f7c5348033"},"filename":"psutil-3.0.0-cp27-none-win32.whl","hashes":{"sha256":"9641677de91769127c82aa032b70a8d9aa75369d0a9965aba880ac335b02eccb"},"provenance":null,"requires-python":null,"size":85709,"upload-time":"2015-06-13T19:16:33.857911Z","url":"https://files.pythonhosted.org/packages/bb/cf/f893fa2fb0888384b5d668e22b6e3652c9ce187e749604027a64a6bd6646/psutil-3.0.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e61eb67de1755dd6d2917f81bb9ee7a0ae6dc5cda2948621b91794f7c5348033"},"data-dist-info-metadata":{"sha256":"e61eb67de1755dd6d2917f81bb9ee7a0ae6dc5cda2948621b91794f7c5348033"},"filename":"psutil-3.0.0-cp27-none-win_amd64.whl","hashes":{"sha256":"11c6b17b37ab1ea04f6a1365bc4598637490819d6b744f3947fd3b8b425366b3"},"provenance":null,"requires-python":null,"size":88043,"upload-time":"2015-06-13T19:16:52.139563Z","url":"https://files.pythonhosted.org/packages/f1/26/a0904455b550f7fd5baa069ca10889825cc22440147813713998cd0676a0/psutil-3.0.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"data-dist-info-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"filename":"psutil-3.0.0-cp33-none-win32.whl","hashes":{"sha256":"8333d2a496fe5d24171360ab1f78468db7a9691d61fd26bf15ca62336c699bac"},"provenance":null,"requires-python":null,"size":85737,"upload-time":"2015-06-13T19:16:39.871421Z","url":"https://files.pythonhosted.org/packages/ea/45/defb71d5009e178ef13e2474627a1070ae291b2dee64ae00e8f2b1a2aec2/psutil-3.0.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"data-dist-info-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"filename":"psutil-3.0.0-cp33-none-win_amd64.whl","hashes":{"sha256":"44711a1a683369056d99fd08a05789d217205d72b6dbbe75c0b3743c5e1e3768"},"provenance":null,"requires-python":null,"size":87943,"upload-time":"2015-06-13T19:16:58.439865Z","url":"https://files.pythonhosted.org/packages/3c/0b/648fbaef2a037cc0c05388c0416ba3ca9244c22c4156dba73e4a900540a6/psutil-3.0.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"data-dist-info-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"filename":"psutil-3.0.0-cp34-none-win32.whl","hashes":{"sha256":"9036f9c57a8c8a571f36c7d3b2c9a15a22dc9a95137fa60191e2eaa9527209d5"},"provenance":null,"requires-python":null,"size":85695,"upload-time":"2015-06-13T19:16:46.286646Z","url":"https://files.pythonhosted.org/packages/6d/ca/1d5dafae4d7396a825531296b6837b45129b63463b4eb221ac3429eb157d/psutil-3.0.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"data-dist-info-metadata":{"sha256":"adc9c5e259a9c63a9ebd7ef296c4a895baa01701f7d8127d1f989e968fa497aa"},"filename":"psutil-3.0.0-cp34-none-win_amd64.whl","hashes":{"sha256":"18e53256390b299ff03bd58dc134d070bde4871d8c93baf08faea316e87544c8"},"provenance":null,"requires-python":null,"size":87893,"upload-time":"2015-06-13T19:17:05.116909Z","url":"https://files.pythonhosted.org/packages/59/82/51ab7cecaf03c8cbe7eff3b341f6564c3fa80baaa5a37643096cb526cae2/psutil-3.0.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.tar.gz","hashes":{"sha256":"a43cc84c6a2406e8f23785c68a52b792d95eb91e2b43be40f7c814bf80dc5979"},"provenance":null,"requires-python":null,"size":240872,"upload-time":"2015-06-13T19:14:46.063288Z","url":"https://files.pythonhosted.org/packages/ba/18/c5bb52abb67194aabadfd61286b538ee8856c7cb9c0e92dd64c1b132bf5e/psutil-3.0.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"26b2cf4456e039f490f79457100564ff34718f4257b68dee53b42d17493f0187"},"provenance":null,"requires-python":null,"size":323299,"upload-time":"2015-06-13T19:15:53.089846Z","url":"https://files.pythonhosted.org/packages/0c/b1/fa44dd6ed19e8be6f9f05f776e824ce202822f5d6237093c54d6c5102888/psutil-3.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"32808b29473e788487ee28a65caaab1c9fbd44c2d654ce6d563a89f20e106996"},"provenance":null,"requires-python":null,"size":321705,"upload-time":"2015-06-13T19:16:01.370127Z","url":"https://files.pythonhosted.org/packages/62/4f/89c6ac53e424109639f42033d0e87c4de4ddb610060007c6d5f6081a0fba/psutil-3.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"37a9869ef17680e768bf32d7556252a04f426e5b1e6fa51d5af16305c8a701b6"},"provenance":null,"requires-python":null,"size":321688,"upload-time":"2015-06-13T19:16:11.540515Z","url":"https://files.pythonhosted.org/packages/d2/67/9c988331d40c445ebc9d29d95534ce3908f3949f6a84d640a6c125bc56e2/psutil-3.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win32-py2.6.exe","hashes":{"sha256":"f29ee1aafa452cce77667e75f5e0345b678b8c091b288d0118840df30f303bed"},"provenance":null,"requires-python":null,"size":293522,"upload-time":"2015-06-13T19:15:19.130222Z","url":"https://files.pythonhosted.org/packages/10/19/9e2f8a305f5a6c01feee59dfeb38a73dc1ea4f4af0133cef9a36708f4111/psutil-3.0.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win32-py2.7.exe","hashes":{"sha256":"7cbcf866faa23e5229ecfe44ff562f406b4806caae66cb255dbb327247e540aa"},"provenance":null,"requires-python":null,"size":293319,"upload-time":"2015-06-13T19:15:27.353152Z","url":"https://files.pythonhosted.org/packages/fc/0f/92e595cd2f2a80cead241d45b4ce4961e2515deff101644f3812c75e9bc7/psutil-3.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win32-py3.3.exe","hashes":{"sha256":"b968771d3db5eef0c17ef6bed2d7756b8ac3cce40e11d5493824863d195dd8e3"},"provenance":null,"requires-python":null,"size":288278,"upload-time":"2015-06-13T19:15:35.880360Z","url":"https://files.pythonhosted.org/packages/fc/29/387c555f9dc38c6bb084a8df8936e607848de9e2e984dbf7eb2a298c1ceb/psutil-3.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.0.win32-py3.4.exe","hashes":{"sha256":"81ac6f54a27d091c75333f7c90082d028db45a1e78de59d28ca9f38cf6186395"},"provenance":null,"requires-python":null,"size":288253,"upload-time":"2015-06-13T19:15:44.871254Z","url":"https://files.pythonhosted.org/packages/fb/29/25ac80b589c0e56214ac64fdd8216992be162a2b2290f9b88b9a5c517cfd/psutil-3.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"986bdfc454536c2a940c955834882e01a25de324bf458054673b43d6b47a00ee"},"data-dist-info-metadata":{"sha256":"986bdfc454536c2a940c955834882e01a25de324bf458054673b43d6b47a00ee"},"filename":"psutil-3.0.1-cp27-none-win32.whl","hashes":{"sha256":"bcb8d23121848953ed295f7e3c0875b0164ee98d3245060beb3623c59ff2a1bc"},"provenance":null,"requires-python":null,"size":85793,"upload-time":"2015-06-18T02:36:50.974286Z","url":"https://files.pythonhosted.org/packages/17/05/b2c807b470464fbe3d357734b68199451574fcd75431fd3e5c77be24b6e0/psutil-3.0.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"986bdfc454536c2a940c955834882e01a25de324bf458054673b43d6b47a00ee"},"data-dist-info-metadata":{"sha256":"986bdfc454536c2a940c955834882e01a25de324bf458054673b43d6b47a00ee"},"filename":"psutil-3.0.1-cp27-none-win_amd64.whl","hashes":{"sha256":"ba56ec5c052489b7a7015c26ed3f917f2df4ffa9799266e86be41815bc358b80"},"provenance":null,"requires-python":null,"size":88128,"upload-time":"2015-06-18T02:37:08.778852Z","url":"https://files.pythonhosted.org/packages/38/bf/0b743c8a07265f2ecb203f8e60310571dcae33036aa6ba7aa16e2641ac7a/psutil-3.0.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"data-dist-info-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"filename":"psutil-3.0.1-cp33-none-win32.whl","hashes":{"sha256":"23606e9b42760a8fdeada33281d0c3ce88df220949746e0ddf54d5db7974b4c6"},"provenance":null,"requires-python":null,"size":85819,"upload-time":"2015-06-18T02:36:56.905878Z","url":"https://files.pythonhosted.org/packages/b2/9c/b2c4373b9406eaf33654c4be828d9316ee780f4b3c19d0fa56f55eb64d61/psutil-3.0.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"data-dist-info-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"filename":"psutil-3.0.1-cp33-none-win_amd64.whl","hashes":{"sha256":"ae4a7f51f40154d02ab1576e94377171a28cc83fc89c077c152f16ba3dae72f3"},"provenance":null,"requires-python":null,"size":88028,"upload-time":"2015-06-18T02:37:14.406512Z","url":"https://files.pythonhosted.org/packages/7a/59/1a9a10238226dbaed24b8d978e9cc743ac143504b3f4ad8f5e3a169e263a/psutil-3.0.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"data-dist-info-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"filename":"psutil-3.0.1-cp34-none-win32.whl","hashes":{"sha256":"b7520db52c7c4e38cdd50bb11be02a826372d8002bf92bbe955823cac32c7f9d"},"provenance":null,"requires-python":null,"size":85783,"upload-time":"2015-06-18T02:37:03.210308Z","url":"https://files.pythonhosted.org/packages/ed/85/1d4d97ce4c9a8c6dd479f5f717694016651170d3731cdecf740db0e4eae3/psutil-3.0.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"data-dist-info-metadata":{"sha256":"98502550423dd5cf7f5bcfaed4c1d92354cc0353b2b9bf673992e42fa246dce5"},"filename":"psutil-3.0.1-cp34-none-win_amd64.whl","hashes":{"sha256":"1754d4118eaab16f299a37dafaf7d34111e9a8e5ac2a799e2bd9b1a5d9d1122b"},"provenance":null,"requires-python":null,"size":87982,"upload-time":"2015-06-18T02:37:20.490529Z","url":"https://files.pythonhosted.org/packages/1e/e6/0af7e190f74d6f959dfb87f2b56f4711271729952691f843fb91c5e06712/psutil-3.0.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.tar.gz","hashes":{"sha256":"3f213b9ceed3c3068a973e04d7a8b2a29d1076abcb5ef45382517bfc6b808801"},"provenance":null,"requires-python":null,"size":241539,"upload-time":"2015-06-18T02:33:52.119548Z","url":"https://files.pythonhosted.org/packages/aa/5d/cbd3b7227fe7a4c2c77e4031b6c43961563a3ecde2981190e5afe959be51/psutil-3.0.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win-amd64-py2.7.exe","hashes":{"sha256":"b0a2ed567b31f71ae2e893768f0da0d51d51d12714471d4b7431e70ff5e36577"},"provenance":null,"requires-python":null,"size":323471,"upload-time":"2015-06-18T02:36:04.537383Z","url":"https://files.pythonhosted.org/packages/9d/a4/815181cacd33f0ce62a3c3aa188af65bd3945888aa2f6c16a925837ca517/psutil-3.0.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win-amd64-py3.3.exe","hashes":{"sha256":"99c593cb459b54209cdb4aed4a607fa8b2920fbd4f3c5a9219a0ede114975758"},"provenance":null,"requires-python":null,"size":321879,"upload-time":"2015-06-18T02:36:14.693044Z","url":"https://files.pythonhosted.org/packages/08/b2/278ac09b03db15b9e51c3ae7f678b3fbf050e895a2eccab66d05017bdef9/psutil-3.0.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win-amd64-py3.4.exe","hashes":{"sha256":"8f803bfe00a763254581bb6a3d788b3332492fbc67dbf46ad5068b663e44f309"},"provenance":null,"requires-python":null,"size":321861,"upload-time":"2015-06-18T02:36:23.687685Z","url":"https://files.pythonhosted.org/packages/d8/66/d21041db114938ae22d4994ea31f2d8f1353aa61e5f0d0c6c4e185b016a7/psutil-3.0.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win32-py2.6.exe","hashes":{"sha256":"4a19475c1d6071c685b38f85837f6e6daa2c6ccd4d0132ab840123deb8ea2372"},"provenance":null,"requires-python":null,"size":293700,"upload-time":"2015-06-18T02:35:31.083110Z","url":"https://files.pythonhosted.org/packages/99/67/980b4a9257abaa3f53a60a3441f759301a0aca2922d60e18a05d23fb1b0e/psutil-3.0.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win32-py2.7.exe","hashes":{"sha256":"363d3dbd610ce7bcf7f13b0a31133ee231d0990e99315142ed37f6ba2c1a84e6"},"provenance":null,"requires-python":null,"size":293492,"upload-time":"2015-06-18T02:35:39.030138Z","url":"https://files.pythonhosted.org/packages/5f/64/b994ed73ab49d5c847d97c47600b539603ebf0d6cfd8d3575b80db7aefb5/psutil-3.0.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win32-py3.3.exe","hashes":{"sha256":"fc7cfe1d6919cb67f4144947acfcefc099d7d8299dd88bb4d863e62c44d041b4"},"provenance":null,"requires-python":null,"size":288451,"upload-time":"2015-06-18T02:35:47.145538Z","url":"https://files.pythonhosted.org/packages/1f/11/4871e823ff0d5a302a7f8ece60358d20ba7fc1620c4c9e2d94e826e1ff0e/psutil-3.0.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.0.1.win32-py3.4.exe","hashes":{"sha256":"73091a80ed295f990ab377dfd8cd4f7f00e3abfffb5e500192f1ea9fa58de158"},"provenance":null,"requires-python":null,"size":288427,"upload-time":"2015-06-18T02:35:55.768442Z","url":"https://files.pythonhosted.org/packages/08/af/c78ef8dc09ac61a479968b0c912677d0a0cb138c5ea7813b2a196fa32c53/psutil-3.0.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"data-dist-info-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"filename":"psutil-3.1.0-cp26-none-win32.whl","hashes":{"sha256":"399916a016503c9ae99fd6aafbba4628b86ebdee52d67034e8c3f26e88a3504b"},"provenance":null,"requires-python":null,"size":87753,"upload-time":"2015-07-15T00:41:13.293562Z","url":"https://files.pythonhosted.org/packages/4e/0a/ab710541d02ff3ce4169a0922e4412952332991c4c39a0ea9df20e9279b0/psutil-3.1.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"data-dist-info-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"filename":"psutil-3.1.0-cp27-none-win32.whl","hashes":{"sha256":"71af7a30eb6ed694a3624330334bd12da28bffdbe813f4b1080fadeb86f5970f"},"provenance":null,"requires-python":null,"size":87559,"upload-time":"2015-07-15T00:41:27.810047Z","url":"https://files.pythonhosted.org/packages/db/87/34b52811b755db94a7dd123e5c0f1d257885535f08f5f185c17810214d55/psutil-3.1.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"data-dist-info-metadata":{"sha256":"74e797fa9b89ed58d856034a0b901f68c1d317dd41713af1d4e9bd93e2e19b2c"},"filename":"psutil-3.1.0-cp27-none-win_amd64.whl","hashes":{"sha256":"ada1cf324e6aba0affcb23c6fd959dae9f72de6ec135530788cbf17153d4fd3c"},"provenance":null,"requires-python":null,"size":90071,"upload-time":"2015-07-15T00:42:15.297263Z","url":"https://files.pythonhosted.org/packages/0b/7c/90869233a3e4056ddfdd1040d0e7722d3bb023c74b48bf10c09380c26eae/psutil-3.1.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"data-dist-info-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"filename":"psutil-3.1.0-cp33-none-win32.whl","hashes":{"sha256":"0a313ebe14b9e277dfd151f4ad021012fb344dd51248e6de2aa1e7062d678541"},"provenance":null,"requires-python":null,"size":87569,"upload-time":"2015-07-15T00:41:43.765697Z","url":"https://files.pythonhosted.org/packages/f4/7c/56b718693e4c41b32af8bbe39160e8a3ea0ca12d3eece3dbbb8d4c046855/psutil-3.1.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"data-dist-info-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"filename":"psutil-3.1.0-cp33-none-win_amd64.whl","hashes":{"sha256":"92a7f420bc97f899b5abab30392c23ba652304aec18415f2d1167da04dae9913"},"provenance":null,"requires-python":null,"size":89918,"upload-time":"2015-07-15T00:42:29.864336Z","url":"https://files.pythonhosted.org/packages/c7/5a/4046949e207b72b93540f3e19d699813fd35e290ccdf48080f332226b912/psutil-3.1.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"data-dist-info-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"filename":"psutil-3.1.0-cp34-none-win32.whl","hashes":{"sha256":"8f5a0e859ae6dcc349914fb9ea0acc21cfd82a321d1c1b02d3d92c195f523ccd"},"provenance":null,"requires-python":null,"size":87592,"upload-time":"2015-07-15T00:42:00.678342Z","url":"https://files.pythonhosted.org/packages/09/34/09d53d29318a5fea88bd30d629595805064a0e3776e706eca2d79ceaebac/psutil-3.1.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"data-dist-info-metadata":{"sha256":"e5e4b772973543fffe4b2ffb66ad397284be77c3b6886db90b1fca8245b0d0b1"},"filename":"psutil-3.1.0-cp34-none-win_amd64.whl","hashes":{"sha256":"dcb4f208ec28fb72b35d1edf49aa51f2cc116b439aa40c4c415cbfe1fee54078"},"provenance":null,"requires-python":null,"size":89877,"upload-time":"2015-07-15T00:42:45.261251Z","url":"https://files.pythonhosted.org/packages/0b/f6/62592864eb064763989aa5830706034f9ad3c6ae2255fb7cae0c66b336a1/psutil-3.1.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.tar.gz","hashes":{"sha256":"4cdfeb2a328b6f8a2937f9b21f513c8aeda96dc076ecafda424f5c401dbad876"},"provenance":null,"requires-python":null,"size":246767,"upload-time":"2015-07-15T00:40:47.134419Z","url":"https://files.pythonhosted.org/packages/ce/d2/ab7f80718b4eafb2e474b8b410274d2c0d65341b963d730e653be9ed0ec8/psutil-3.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"6570fb3ddde83597e11c062e20ab86210ff84a1fa97e54bc8bda05e4cd34670a"},"provenance":null,"requires-python":null,"size":326179,"upload-time":"2015-07-15T00:42:09.118162Z","url":"https://files.pythonhosted.org/packages/26/f5/c76bf7ef62736913146b0482879705d1d877c9334092a0739f2b3bbae162/psutil-3.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win-amd64-py3.3.exe","hashes":{"sha256":"578a52f4b108857273d1e32de4d9bebf9b8f842f8c53ab3e242252cfc9bde295"},"provenance":null,"requires-python":null,"size":324540,"upload-time":"2015-07-15T00:42:24.223021Z","url":"https://files.pythonhosted.org/packages/00/51/2dc07e5618adb4a3676ab6c9c1759a3f268eda91808d58f45eb4dfd3d2c5/psutil-3.1.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win-amd64-py3.4.exe","hashes":{"sha256":"2aea29ca2a5ea318155fb856f24e6c7563c8741ccfeec862f0fd9af0f2d8ae87"},"provenance":null,"requires-python":null,"size":324490,"upload-time":"2015-07-15T00:42:38.312454Z","url":"https://files.pythonhosted.org/packages/da/76/dcefdcf88fd51becaff6c1ec0ba7566ca654a207fe6d69662723c645e3b0/psutil-3.1.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win32-py2.6.exe","hashes":{"sha256":"271f1b6fcb4861d1b0fc7f612b2abaacb36c0f878fcc2908f1cf673337c3472c"},"provenance":null,"requires-python":null,"size":296215,"upload-time":"2015-07-15T00:41:08.168047Z","url":"https://files.pythonhosted.org/packages/3f/c2/f7ec0a70bc58c1918f814001682cca30f4b168f5b46dce913220e625dee6/psutil-3.1.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win32-py2.7.exe","hashes":{"sha256":"1bde1cea6b7f9bd66202feee289c282b076c00419dc6404db357b05125f4d692"},"provenance":null,"requires-python":null,"size":296024,"upload-time":"2015-07-15T00:41:21.733811Z","url":"https://files.pythonhosted.org/packages/56/2f/c97adcba8f119a23d3580a3a95939c1a37d37d514b304d90912585a85521/psutil-3.1.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win32-py3.3.exe","hashes":{"sha256":"667fa795ca5ccde216b769fc8572398598f1e0e2619f78605df1a4c75a475174"},"provenance":null,"requires-python":null,"size":290958,"upload-time":"2015-07-15T00:41:37.038168Z","url":"https://files.pythonhosted.org/packages/37/89/a949b02d66d600c45230a3a622f0d5f491182dbbc96fff9ffbcd869431bd/psutil-3.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.0.win32-py3.4.exe","hashes":{"sha256":"8e637fe2a23fad4f4ea8ae11402b593e09fc587b86a6ced40b6bc1017be8d978"},"provenance":null,"requires-python":null,"size":290975,"upload-time":"2015-07-15T00:41:53.144204Z","url":"https://files.pythonhosted.org/packages/53/3f/249ff2e418313f30a6dcc4995966725f72b00493848ffddeb72b506e8e50/psutil-3.1.0.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"data-dist-info-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"filename":"psutil-3.1.1-cp26-none-win32.whl","hashes":{"sha256":"13a6377cc8d2859f846058170830127822877e05229c4a43aea893cdcb504d65"},"provenance":null,"requires-python":null,"size":87749,"upload-time":"2015-07-15T12:34:56.810625Z","url":"https://files.pythonhosted.org/packages/e1/9e/721afc99b6fe467b47fa2cad6899acc19b45dee32d30b498dc731b6c09ef/psutil-3.1.1-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"data-dist-info-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"filename":"psutil-3.1.1-cp27-none-win32.whl","hashes":{"sha256":"5b7228cb69fdaea5aeb901704f5ecd21b7846aa60c2c8d408f22573fcbaa7e6f"},"provenance":null,"requires-python":null,"size":87554,"upload-time":"2015-07-15T12:35:15.647007Z","url":"https://files.pythonhosted.org/packages/07/60/c88366202816ba42b3d8e93e793c14d1ac5e71be30dd53c2d0117c106eec/psutil-3.1.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"data-dist-info-metadata":{"sha256":"cb2a855afe0d2280f15e0904acf47929db7a22fcf8be740f65cecf46586d4ca5"},"filename":"psutil-3.1.1-cp27-none-win_amd64.whl","hashes":{"sha256":"da7650e2f3fcf06419d5ad75123e6c68b9bf5ff2a6c91d4c77aaed8e6f444fc4"},"provenance":null,"requires-python":null,"size":90065,"upload-time":"2015-07-15T12:36:14.571710Z","url":"https://files.pythonhosted.org/packages/fa/5b/8834e22cc22b6b0e9c2c68e240ab69754bed7c4c5388fb65abfa716f4a67/psutil-3.1.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"data-dist-info-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"filename":"psutil-3.1.1-cp33-none-win32.whl","hashes":{"sha256":"f3d68eb44ba49e24a18d6f7934463478294a49152f97fea2eefe1e1e1ee957f3"},"provenance":null,"requires-python":null,"size":87562,"upload-time":"2015-07-15T12:35:34.924344Z","url":"https://files.pythonhosted.org/packages/cb/96/0eb8eb289681364d2cda2a22a7d1abeb0196b321ab95694335dd178a5b35/psutil-3.1.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"data-dist-info-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"filename":"psutil-3.1.1-cp33-none-win_amd64.whl","hashes":{"sha256":"f9be0ae975b55a3b5d5a8b769560096d76184b60a56c6e88ff6b7ebecf1bc684"},"provenance":null,"requires-python":null,"size":89916,"upload-time":"2015-07-15T12:36:34.374903Z","url":"https://files.pythonhosted.org/packages/eb/13/a38bc1e0ac6f7c42dddd9c17a206877befb822ba3af9c3b0dab9e85911a6/psutil-3.1.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"data-dist-info-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"filename":"psutil-3.1.1-cp34-none-win32.whl","hashes":{"sha256":"8f25aad572bde88d5ee0b3a11a75ff2ae3c8b0a334c4128d6f8eb4fc95172734"},"provenance":null,"requires-python":null,"size":87574,"upload-time":"2015-07-15T12:35:55.507642Z","url":"https://files.pythonhosted.org/packages/02/10/439ec497e3e38a8c493d0c67c56e23d106b85c8b9f616e8df9ec6ce1e606/psutil-3.1.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"data-dist-info-metadata":{"sha256":"19405a3f0d3e28d54384ce148334aeef185226341d9cc2a81c2d2933432a1de3"},"filename":"psutil-3.1.1-cp34-none-win_amd64.whl","hashes":{"sha256":"4be182c273758dcdbd30827fdeecd889e27cb6a30238798e91bddeebc29cdc4f"},"provenance":null,"requires-python":null,"size":89865,"upload-time":"2015-07-15T12:36:55.016343Z","url":"https://files.pythonhosted.org/packages/ec/2e/8d98579399bc1979904455df182a063dd584b285ee8c141f3c94e7814c47/psutil-3.1.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.tar.gz","hashes":{"sha256":"d3290bd4a027fa0b3a2e2ee87728056fe49d4112640e2b8c2ea4dd94ba0cf057"},"provenance":null,"requires-python":null,"size":247284,"upload-time":"2015-07-15T12:33:47.020532Z","url":"https://files.pythonhosted.org/packages/8d/b3/954de176aa8e3a7782bae52ce938f24726c2c68d0f4c60d159271b6b293d/psutil-3.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win-amd64-py2.7.exe","hashes":{"sha256":"9c4fd3cc19bbc04eaa7ef3c61e3db26a41ac5e056f770977211d4569d0bf0086"},"provenance":null,"requires-python":null,"size":326261,"upload-time":"2015-07-15T12:36:07.064169Z","url":"https://files.pythonhosted.org/packages/e8/ac/7fb95ccc69ced76d0920e411b2fdfd3d38398c4bce53ec1dae92800df88a/psutil-3.1.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win-amd64-py3.3.exe","hashes":{"sha256":"e7cc26f661c9eaa9b32d0543dd7838daea72aad6e9f02fe73715ffd0dcb65170"},"provenance":null,"requires-python":null,"size":324622,"upload-time":"2015-07-15T12:36:27.201153Z","url":"https://files.pythonhosted.org/packages/2f/84/ae41f6bb61d4a93399c621218f99b761171a69a0c9163b9a72db1d53d62a/psutil-3.1.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win-amd64-py3.4.exe","hashes":{"sha256":"46cbfd86d6762e63c7df4ab0df889f6f2fffa9b5781ea3fc0431237f2a408382"},"provenance":null,"requires-python":null,"size":324571,"upload-time":"2015-07-15T12:36:47.165677Z","url":"https://files.pythonhosted.org/packages/88/d4/ca15a913ab43222e308774845317544e765718a9e56bd4efe5b3cedf1fbd/psutil-3.1.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win32-py2.6.exe","hashes":{"sha256":"9efbd578d2f400dfe0ecab123b58d8af105854fdbb6222f841151e010e820b75"},"provenance":null,"requires-python":null,"size":296306,"upload-time":"2015-07-15T12:34:49.442980Z","url":"https://files.pythonhosted.org/packages/0c/f6/e81385c7ec989157eb68688a64a69c5a7477ff93d544893a9e1f251588b1/psutil-3.1.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win32-py2.7.exe","hashes":{"sha256":"e0065e7cade4ac5ac70411674bc32326dee8d11c44469012a2b5164bf6dea97a"},"provenance":null,"requires-python":null,"size":296106,"upload-time":"2015-07-15T12:35:08.277861Z","url":"https://files.pythonhosted.org/packages/6b/fe/51596968f5a6a0970d9424021989f542d5aa715fe21d1a9c6bbbb0e377a9/psutil-3.1.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win32-py3.3.exe","hashes":{"sha256":"c8ab17e07ea4907d2f9129254e82b6765ae08e61f0ce6dc8e2fc1faf145b166c"},"provenance":null,"requires-python":null,"size":291039,"upload-time":"2015-07-15T12:35:27.250487Z","url":"https://files.pythonhosted.org/packages/83/49/ff116fb9981ef04a5aed1c091ace117c214ed752d37be267ea4e2f28efad/psutil-3.1.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.1.1.win32-py3.4.exe","hashes":{"sha256":"3003d8be6e86eb6beb990863a88950f9b9fe53ccaae92edcd8efcd152d7451ea"},"provenance":null,"requires-python":null,"size":291056,"upload-time":"2015-07-15T12:35:47.023807Z","url":"https://files.pythonhosted.org/packages/7b/cd/accf3d7e37006bffe7a569e4fc587eb686d275a19a4e8a37a12930a1e2db/psutil-3.1.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"e8c3ed176a6ecb754b5289ef149667a6343dc0380cb33db608d4d556be0650f4"},"data-dist-info-metadata":{"sha256":"e8c3ed176a6ecb754b5289ef149667a6343dc0380cb33db608d4d556be0650f4"},"filename":"psutil-3.2.0-cp27-none-win32.whl","hashes":{"sha256":"1493041336a591f22c77bcb815a399faf9bdac32f79f4de354eda3507a0d6d6b"},"provenance":null,"requires-python":null,"size":88077,"upload-time":"2015-09-02T11:56:33.437905Z","url":"https://files.pythonhosted.org/packages/5f/fc/5f317fd548909b1bbb111d462c072faf8af3938268f3e7dd3ab2f9181461/psutil-3.2.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e8c3ed176a6ecb754b5289ef149667a6343dc0380cb33db608d4d556be0650f4"},"data-dist-info-metadata":{"sha256":"e8c3ed176a6ecb754b5289ef149667a6343dc0380cb33db608d4d556be0650f4"},"filename":"psutil-3.2.0-cp27-none-win_amd64.whl","hashes":{"sha256":"3744ee760dff697f45731a71e7902514aa043c99800cc8fabeb6bebc9dad973d"},"provenance":null,"requires-python":null,"size":90526,"upload-time":"2015-09-02T11:57:23.745435Z","url":"https://files.pythonhosted.org/packages/84/6c/7efbe64b42748125e7113a90e48c0da9859b7f0363ac85ca5617decbafee/psutil-3.2.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"data-dist-info-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"filename":"psutil-3.2.0-cp33-none-win32.whl","hashes":{"sha256":"8836f77d2c4ae2935431ca66e445435b87b53b4db637fcceb438b78843239210"},"provenance":null,"requires-python":null,"size":88080,"upload-time":"2015-09-02T11:56:48.988091Z","url":"https://files.pythonhosted.org/packages/64/8e/0a06028a1ac093402885febf2aeb18093f1d28ae2110c7eb10b43e7554c1/psutil-3.2.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"data-dist-info-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"filename":"psutil-3.2.0-cp33-none-win_amd64.whl","hashes":{"sha256":"b16eb62d9c21efaa2c9ac8a9f8b23bb7a695cb799b597edf4b1289ce8e6973ac"},"provenance":null,"requires-python":null,"size":90372,"upload-time":"2015-09-02T12:03:01.254324Z","url":"https://files.pythonhosted.org/packages/cd/29/a8383040200a3ebe0e985f54f35691cc078a1deb632abb5340d3deb5b7b7/psutil-3.2.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"data-dist-info-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"filename":"psutil-3.2.0-cp34-none-win32.whl","hashes":{"sha256":"0ac1d68ab3c5a65641cbbb23d19deda466f73226f9d967f91436851995281777"},"provenance":null,"requires-python":null,"size":88096,"upload-time":"2015-09-02T11:57:08.035697Z","url":"https://files.pythonhosted.org/packages/07/c3/76a50982a82c0e9d93d9614a0cd06644c1d3406c9bb80a43f95abdd4ab97/psutil-3.2.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"data-dist-info-metadata":{"sha256":"86ea3a30e605b64b3e8a93353520057387c9224bf456a532eb4f25799e27b050"},"filename":"psutil-3.2.0-cp34-none-win_amd64.whl","hashes":{"sha256":"0b26ef262fe2d10185ab562cd0530af7f6d9a6744c631c44e64be94796f4ba2d"},"provenance":null,"requires-python":null,"size":90354,"upload-time":"2015-09-02T12:07:12.010031Z","url":"https://files.pythonhosted.org/packages/58/2d/8b7abb9b6f8956d9a6dfc3b1dffce27efab8c7c0497a6366e7fee444ae53/psutil-3.2.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.tar.gz","hashes":{"sha256":"06f9d255f8b12a6a04aa2b468ec453c539f54a464d110b3458c32b0152a5c943"},"provenance":null,"requires-python":null,"size":251988,"upload-time":"2015-09-02T11:59:23.849318Z","url":"https://files.pythonhosted.org/packages/1d/3a/d396274e6f086e342dd43401d4012973af98c00b3aabdb5cc4a432df660e/psutil-3.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"71fd8712715f8e6acc5bee5719a83a61a396067cf2bfb15b4d8f1f2955648637"},"provenance":null,"requires-python":null,"size":326895,"upload-time":"2015-09-02T11:57:17.538160Z","url":"https://files.pythonhosted.org/packages/e1/49/990073ab7e010965a7d0df5e48181d07c212edd7fafb890feda664ea9b3c/psutil-3.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.win32-py2.7.exe","hashes":{"sha256":"2a8b5878d4e787d81a1eeddcc09ff28d501a3ceb320c7fffa7e207da5d61d01c"},"provenance":null,"requires-python":null,"size":296802,"upload-time":"2015-09-02T11:56:26.879142Z","url":"https://files.pythonhosted.org/packages/0b/c5/d6ad511c3c17afa9837d08fc26d76e85dc83ebc304c6c7bec3970e74f240/psutil-3.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.win32-py3.3.exe","hashes":{"sha256":"adfc63ceede4e8f6bf21e4bdf6fc91f70f9612ec2b1bf9ad306828909bb71c52"},"provenance":null,"requires-python":null,"size":291738,"upload-time":"2015-09-02T11:56:41.826609Z","url":"https://files.pythonhosted.org/packages/e3/35/81842c6c4366d19c87d1a1fb4ad4e4d22a18aa9facaea8b6f12ccd4c1212/psutil-3.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.0.win32-py3.4.exe","hashes":{"sha256":"4098e0ed7930003ef15feb852e64f73180c17a651c4170fb5573f8c44622d068"},"provenance":null,"requires-python":null,"size":291748,"upload-time":"2015-09-02T11:56:58.479934Z","url":"https://files.pythonhosted.org/packages/e9/e8/5b432a0490328cfff86605b574d2aa31b1fac4e61587dff5a7f76d4cb95e/psutil-3.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"data-dist-info-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"filename":"psutil-3.2.1-cp26-none-win32.whl","hashes":{"sha256":"a77230ecd6f42d0b549f8eb6aa105f14e4bc5908c754d6e10ff979c900934481"},"provenance":null,"requires-python":null,"size":88306,"upload-time":"2015-09-03T15:37:28.878155Z","url":"https://files.pythonhosted.org/packages/37/46/f348f7728dea66436abdfc9fa14ef017e0148c6bca08a822ee4dd7cb6d75/psutil-3.2.1-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"data-dist-info-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"filename":"psutil-3.2.1-cp27-none-win32.whl","hashes":{"sha256":"9453b8ceb249d4d9ddc69153729761be340dfef9c99509390e4fb0f1fcbb3853"},"provenance":null,"requires-python":null,"size":88111,"upload-time":"2015-09-03T15:30:11.278302Z","url":"https://files.pythonhosted.org/packages/ba/27/f55ca7d15af50e731e9bbbff9b22fc31a40b786c02f85d173568e5084152/psutil-3.2.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"data-dist-info-metadata":{"sha256":"c06dad4d23110956ffef0ab97d2845cb62aba9d194019b18cf8e24cbd88c0781"},"filename":"psutil-3.2.1-cp27-none-win_amd64.whl","hashes":{"sha256":"014714beed46a66370834cebe0bbb53799bddc164f7f0149a4a70e2051f7bc1a"},"provenance":null,"requires-python":null,"size":90561,"upload-time":"2015-09-03T15:35:05.710499Z","url":"https://files.pythonhosted.org/packages/6b/ac/5da840018ce300a258925d4535a55a32b75236d5d777a8de6c3de18e71f3/psutil-3.2.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"data-dist-info-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"filename":"psutil-3.2.1-cp33-none-win32.whl","hashes":{"sha256":"5c0daf045fd7d7f105863a5f9508d1698559ebbdfd70d7d8b6fe6fedde575735"},"provenance":null,"requires-python":null,"size":88117,"upload-time":"2015-09-03T15:32:19.746087Z","url":"https://files.pythonhosted.org/packages/c7/96/3ae14e4bf81f18f404bb5285fcda28e50ae6df87e91ad6bf45a9a4f51ac3/psutil-3.2.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"data-dist-info-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"filename":"psutil-3.2.1-cp33-none-win_amd64.whl","hashes":{"sha256":"452622592564cd67f86808c8176720c1443d43e248cfd242d71cff559ed1424c"},"provenance":null,"requires-python":null,"size":90410,"upload-time":"2015-09-03T15:36:13.351055Z","url":"https://files.pythonhosted.org/packages/76/62/fe6f705cb331be5fcc97b268987527dcdb3f3aa104bf830b0ec8bf1e2ad4/psutil-3.2.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"data-dist-info-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"filename":"psutil-3.2.1-cp34-none-win32.whl","hashes":{"sha256":"96379bee09d4c6b4d57d72cb7347dbc51b6847977f2fad01cdfefef3b53e44e3"},"provenance":null,"requires-python":null,"size":88131,"upload-time":"2015-09-03T15:33:36.571683Z","url":"https://files.pythonhosted.org/packages/31/ec/1a54f23c767e27dac09d2372f3522f88ef34f3a0ddd44c16122970259f6f/psutil-3.2.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"data-dist-info-metadata":{"sha256":"51cd3e3b33e3b710f799f247a23955f942e6e11d4882ee5e3f3af539eee619c8"},"filename":"psutil-3.2.1-cp34-none-win_amd64.whl","hashes":{"sha256":"7a7f4f2ed6d2835c48c24a81b251ba4f9b21f6bba2323291f8205c9ecb6f659d"},"provenance":null,"requires-python":null,"size":90387,"upload-time":"2015-09-03T15:36:29.262691Z","url":"https://files.pythonhosted.org/packages/2e/10/f1590ae942a6b8dd2bdeef6088e30e89b30161a264881b14134f3c4a3a0e/psutil-3.2.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.tar.gz","hashes":{"sha256":"7f6bea8bfe2e5cfffd0f411aa316e837daadced1893b44254bb9a38a654340f7"},"provenance":null,"requires-python":null,"size":251653,"upload-time":"2015-09-03T15:30:34.118573Z","url":"https://files.pythonhosted.org/packages/cd/5f/4fae1036903c01929c48ded6800a8705106ee20f9e39e3f2ad5d1824e210/psutil-3.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"e9a8a44f3847a0e20a54d321ed62de4e9cee5bc4e880e25fe88ae20cfa4e32b2"},"provenance":null,"requires-python":null,"size":327122,"upload-time":"2015-09-03T15:34:58.714500Z","url":"https://files.pythonhosted.org/packages/c5/af/4ab069ba93a037a4acf9bb84248daa44204a46687abc6a9f3a82ad8c5ee2/psutil-3.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"5a62c38852de4513f1816b9c431a94f02531619c1edc60a2cc163c8754f51c50"},"provenance":null,"requires-python":null,"size":325481,"upload-time":"2015-09-03T15:36:01.424492Z","url":"https://files.pythonhosted.org/packages/32/87/82e449ff9573dde3c78685b64ac3d17b5d19db11e976eab27c4dc5dca942/psutil-3.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win-amd64-py3.4.exe","hashes":{"sha256":"da898c0708b99b3892bfb7d5caebb447d14d03c7a655c55e484eb5fcc741c3ca"},"provenance":null,"requires-python":null,"size":325465,"upload-time":"2015-09-03T15:36:22.916532Z","url":"https://files.pythonhosted.org/packages/4b/11/f18b29033a0b383e67f664576eca59fbe8552a9fd97f9f22d6d0ff1c4951/psutil-3.2.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win32-py2.6.exe","hashes":{"sha256":"0e5fe3d50f9f8d9a5216cfa23f56890aa0c6a6163869434001f4f2ba463dace5"},"provenance":null,"requires-python":null,"size":297224,"upload-time":"2015-09-03T15:37:22.791161Z","url":"https://files.pythonhosted.org/packages/c0/bb/ed28c191c4d9f27b60d9ea6bd7774b44d78778f0b1fb507ee1e789a490d7/psutil-3.2.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win32-py2.7.exe","hashes":{"sha256":"9fb6f11bdd3fdbe1e611ae02b3ad3dff8f70ef6eaa694d13e8ad0906fd7a7261"},"provenance":null,"requires-python":null,"size":297025,"upload-time":"2015-09-03T15:30:14.327211Z","url":"https://files.pythonhosted.org/packages/9a/c1/075598067efefe25f6a2b0cf1b3eb896322597ee64ba097cc6611b89ada7/psutil-3.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win32-py3.3.exe","hashes":{"sha256":"605cc7dbe2170e89f2f6709cf1577c8a02f89951fe4a0eb48e72530f605141ca"},"provenance":null,"requires-python":null,"size":291964,"upload-time":"2015-09-03T15:32:12.890526Z","url":"https://files.pythonhosted.org/packages/e0/ce/ff1db37cbdf6b3071e89afe4fced6f73d2cdf9c3a87696e7754d207e0ec5/psutil-3.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.1.win32-py3.4.exe","hashes":{"sha256":"0cd127239f527eae6c0f778dd41bb3ced84e6049b919713022c9b72e5f22a1c1"},"provenance":null,"requires-python":null,"size":291976,"upload-time":"2015-09-03T15:33:29.509014Z","url":"https://files.pythonhosted.org/packages/5d/a0/d624d4f5660a476821fe0d920a4c2e995a23151928b15cc3383379228f15/psutil-3.2.1.win32-py3.4.exe","yanked":false},{"core-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"data-dist-info-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"filename":"psutil-3.2.2-cp26-none-win32.whl","hashes":{"sha256":"5a8ce70327c0da578a31ebbf0042671ed9be6f4b6b022c02f03302b690074966"},"provenance":null,"requires-python":null,"size":88377,"upload-time":"2015-10-04T16:38:11.705751Z","url":"https://files.pythonhosted.org/packages/4d/af/5b8c2471ea942a4b6ee85706e9279284ae9dc86ee30b6f97db2d84a95433/psutil-3.2.2-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"data-dist-info-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"filename":"psutil-3.2.2-cp27-none-win32.whl","hashes":{"sha256":"6c5809582d3d165511d2319401bd0f6c0e825d7853e49da59027c1fb8aa8f897"},"provenance":null,"requires-python":null,"size":88183,"upload-time":"2015-10-04T16:38:39.878475Z","url":"https://files.pythonhosted.org/packages/63/1e/a510f3f310b5f530336fbc708fb1456bf3e49e3b3d85c31d151b6e389c4f/psutil-3.2.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"data-dist-info-metadata":{"sha256":"7115f58a06724cf8a31f365d5561fc59f6d68d8e55e20b3d75bf373a45007f7d"},"filename":"psutil-3.2.2-cp27-none-win_amd64.whl","hashes":{"sha256":"37f1cc8fc7586cc930ea3737533d6d79c1f761d577fd1bb1bb5798ccd1543b53"},"provenance":null,"requires-python":null,"size":90637,"upload-time":"2015-10-04T16:40:05.379646Z","url":"https://files.pythonhosted.org/packages/e7/b7/f04d64a692159733ed383b4638abd9d3dc4538d4aacb5e193af02a3840a2/psutil-3.2.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"data-dist-info-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"filename":"psutil-3.2.2-cp33-none-win32.whl","hashes":{"sha256":"f8fb145f8fa9e223696ff2f99924ea42538f3ad6b9738707292d840acbde528f"},"provenance":null,"requires-python":null,"size":88189,"upload-time":"2015-10-04T16:39:00.887284Z","url":"https://files.pythonhosted.org/packages/77/f3/6b3742040b634692393faf3a81e6c0e40366c22bc338ad3fc62ed21b157a/psutil-3.2.2-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"data-dist-info-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"filename":"psutil-3.2.2-cp33-none-win_amd64.whl","hashes":{"sha256":"65c78ba625cf9761d5966603838cc959f396bd03536c480db69f8cf37bdf9994"},"provenance":null,"requires-python":null,"size":90471,"upload-time":"2015-10-04T16:40:27.009383Z","url":"https://files.pythonhosted.org/packages/59/33/3ccdbec4ef1452758ba80f711af46736717f63d73786744d6251afb68624/psutil-3.2.2-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"data-dist-info-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"filename":"psutil-3.2.2-cp34-none-win32.whl","hashes":{"sha256":"3df8d3e32e2b4f7c2ea91014294844670eddb125ba76c24152c0a155a1f73b5b"},"provenance":null,"requires-python":null,"size":88202,"upload-time":"2015-10-04T16:39:39.989654Z","url":"https://files.pythonhosted.org/packages/62/6e/ee3597f32c650f744359e57fd18bcede773dd7465d392dabbb008bc79b48/psutil-3.2.2-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"data-dist-info-metadata":{"sha256":"93db84a195bf6aa1de88b86ddb7fc98d3600b0ecf0ff4ab69e447ca200c3a4b0"},"filename":"psutil-3.2.2-cp34-none-win_amd64.whl","hashes":{"sha256":"e321d3f029268bc8442a7ff214da43fe91041924898f5e23d88bfda7ecb81acc"},"provenance":null,"requires-python":null,"size":90463,"upload-time":"2015-10-04T16:40:56.120875Z","url":"https://files.pythonhosted.org/packages/ed/fe/f31bb708dfdecfbc59b946ecb9ee3379fe7a8183c37ea6c43d6f4da5117d/psutil-3.2.2-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"826e3762fc7145422d64b4a51bb0f8ca6b3a05baed01f7ef3c10b391a0a6c865"},"data-dist-info-metadata":{"sha256":"826e3762fc7145422d64b4a51bb0f8ca6b3a05baed01f7ef3c10b391a0a6c865"},"filename":"psutil-3.2.2-cp35-none-win32.whl","hashes":{"sha256":"76c68c9005a2aa983fce440ef98b66e6f200f740f52064a90fdcc30d11771bc2"},"provenance":null,"requires-python":null,"size":90363,"upload-time":"2015-11-06T10:48:59.454097Z","url":"https://files.pythonhosted.org/packages/cb/79/fcedcf009ab9f8c605f2f345b1797b72134ecc6c9c9f786575e34b3471bc/psutil-3.2.2-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"826e3762fc7145422d64b4a51bb0f8ca6b3a05baed01f7ef3c10b391a0a6c865"},"data-dist-info-metadata":{"sha256":"826e3762fc7145422d64b4a51bb0f8ca6b3a05baed01f7ef3c10b391a0a6c865"},"filename":"psutil-3.2.2-cp35-none-win_amd64.whl","hashes":{"sha256":"d3ac8ad04a509819d7b5d58e453749a3ceb37253267dac6e8856ea7953e22ca0"},"provenance":null,"requires-python":null,"size":93288,"upload-time":"2015-11-06T10:50:14.699381Z","url":"https://files.pythonhosted.org/packages/03/c5/15e44d590afc788228e93cdacf55f98828f326de985242bbf03b3545b129/psutil-3.2.2-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.tar.gz","hashes":{"sha256":"f9d848e5bd475ffe7fa3ab1c20d249807e648568af64bb0058412296ec990a0c"},"provenance":null,"requires-python":null,"size":253502,"upload-time":"2015-10-04T16:39:43.138939Z","url":"https://files.pythonhosted.org/packages/dc/b2/ab65a2209b996c891209b8a7444a0c825125fba850efaec07b95bccb3ff5/psutil-3.2.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win-amd64-py2.7.exe","hashes":{"sha256":"1fca15005063b401cbf94cebe3c01ef6ba3d86ba563730d5d5d6be962a637cf4"},"provenance":null,"requires-python":null,"size":327220,"upload-time":"2015-10-04T16:39:56.198669Z","url":"https://files.pythonhosted.org/packages/01/7c/47b7ac498c9dd6ac9f9b4489d3bd9cea8158e774592661a7c956a815dc78/psutil-3.2.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win-amd64-py3.3.exe","hashes":{"sha256":"01a1f55819019ad13c288c41cb233e17a6ee648baf19591a70f6c2c2295dde6c"},"provenance":null,"requires-python":null,"size":325577,"upload-time":"2015-10-04T16:40:17.960471Z","url":"https://files.pythonhosted.org/packages/5d/89/b3bfca24d038b16af09055912e3a5bc35347ebeb0e6af322959b68dbf237/psutil-3.2.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win-amd64-py3.5.exe","hashes":{"sha256":"9afc68c02717fb4416f91b3c2da4c407756f683804383d1499cdc9e1512e7942"},"provenance":null,"requires-python":null,"size":242452,"upload-time":"2015-11-06T10:49:50.288504Z","url":"https://files.pythonhosted.org/packages/1d/0b/9582cadcba005f4eb0207107baeae7eb2382431fea5e4bc0ea7ce683430f/psutil-3.2.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win32-py2.6.exe","hashes":{"sha256":"c7c516a83d072d1a375ed3a0b5a1b1b9307ad839011e8d30aa16b0f932f6c481"},"provenance":null,"requires-python":null,"size":297317,"upload-time":"2015-10-04T16:38:04.476936Z","url":"https://files.pythonhosted.org/packages/66/e4/bb85391bb46b607be0578e0a091bc064daeb2d1c2e80aa2dab89260dff00/psutil-3.2.2.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win32-py3.3.exe","hashes":{"sha256":"5168f99f065f6116ad1e8529bd5dd5309815198c6250c9180ff6058c6a3641d9"},"provenance":null,"requires-python":null,"size":292060,"upload-time":"2015-10-04T16:38:46.104177Z","url":"https://files.pythonhosted.org/packages/8e/86/8f1e1c0ffc0530dca5e71777f04fb90833c92d3ffc1d075b8b546874eae5/psutil-3.2.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win32-py3.4.exe","hashes":{"sha256":"d5c96aa591ba711dfdadb1ab7a0adf08c5e637644a21437f3a39a9e427aa969b"},"provenance":null,"requires-python":null,"size":292515,"upload-time":"2015-11-06T01:39:51.263901Z","url":"https://files.pythonhosted.org/packages/28/fc/08f1098976de5416cd15967d88e03297569e9a34ca875e7dee38ff9150f0/psutil-3.2.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.2.2.win32-py3.5.exe","hashes":{"sha256":"f0e88f94f9822fd34bcd927aba4bb606bcb0ad0dda1543c9333e8175d5c05822"},"provenance":null,"requires-python":null,"size":232367,"upload-time":"2015-11-06T10:48:42.065794Z","url":"https://files.pythonhosted.org/packages/1d/b8/b725f9bd884f75ce141a9e871a79b2bdd1b5f31e814fc4e396d9ff7c98a2/psutil-3.2.2.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"data-dist-info-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"filename":"psutil-3.3.0-cp26-none-win32.whl","hashes":{"sha256":"584f0b29fcc5d523b433cb8918b2fc74d67e30ee0b44a95baf031528f424619f"},"provenance":null,"requires-python":null,"size":90099,"upload-time":"2015-11-25T18:49:50.211423Z","url":"https://files.pythonhosted.org/packages/91/75/c20c3b9f4d3feb3436d607f498744e46dd28b265b8a72509812322198c7c/psutil-3.3.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"data-dist-info-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"filename":"psutil-3.3.0-cp26-none-win_amd64.whl","hashes":{"sha256":"28ca0b6e9d99aa8dc286e8747a4471362b69812a25291de29b6a8d70a1545a0d"},"provenance":null,"requires-python":null,"size":92645,"upload-time":"2015-11-25T18:50:32.375134Z","url":"https://files.pythonhosted.org/packages/6a/d1/0ce316e4346bcae9dd23911366d894eda65875b88ff447ec8f0402ce556b/psutil-3.3.0-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"data-dist-info-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"filename":"psutil-3.3.0-cp27-none-win32.whl","hashes":{"sha256":"167ad5fff52a672c4ddc1c1a0b25146d6813ebb08a9aab0a3ac45f8a5b669c3b"},"provenance":null,"requires-python":null,"size":90131,"upload-time":"2015-11-25T01:20:43.015358Z","url":"https://files.pythonhosted.org/packages/91/73/1f55b4a19db535759fec5fdbdd0653d7192336557078e3ac9085d7d77cd1/psutil-3.3.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"data-dist-info-metadata":{"sha256":"5977c4a46aa367cb1f96ab1a6c70f8c803d8194d6a6440a31450ac202114431f"},"filename":"psutil-3.3.0-cp27-none-win_amd64.whl","hashes":{"sha256":"e6dea6173a988727bb223d3497349ad5cdef5c0b282eff2d83e5f9065c53f85f"},"provenance":null,"requires-python":null,"size":92586,"upload-time":"2015-11-25T01:22:19.444688Z","url":"https://files.pythonhosted.org/packages/a2/ab/d15a34c6b9090d58601541f8f5564f5b48d01e82f56e07593be969d529e7/psutil-3.3.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp33-none-win32.whl","hashes":{"sha256":"2af5e0a4aad66049955d0734aa4e3dc8caa17a9eaf8b4c1a27a5f1ee6e40f6fc"},"provenance":null,"requires-python":null,"size":90141,"upload-time":"2015-11-25T01:21:10.374023Z","url":"https://files.pythonhosted.org/packages/c5/1f/5038a2567f5853ea1e0fb55f795c30b339a318717573c5b0c85b8814d733/psutil-3.3.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp33-none-win_amd64.whl","hashes":{"sha256":"d9884dc0dc2e55e2448e495778dc9899c1c8bf37aeb2f434c1bea74af93c2683"},"provenance":null,"requires-python":null,"size":92432,"upload-time":"2015-11-25T01:22:39.035467Z","url":"https://files.pythonhosted.org/packages/b1/9c/a9cd75c8cfbac44397a2ca76430229c5496b21e0ab93cba5987d80e3f262/psutil-3.3.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp34-none-win32.whl","hashes":{"sha256":"e27c2fe6dfcc8738be3d2c5a022f785eb72971057e1a9e1e34fba73bce8a71a6"},"provenance":null,"requires-python":null,"size":90151,"upload-time":"2015-11-25T01:21:33.616930Z","url":"https://files.pythonhosted.org/packages/23/57/6a7c3ab4d04d055cada3b5511c40e0e699d8dd5d8217cae6fb68ae61dff6/psutil-3.3.0-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp34-none-win_amd64.whl","hashes":{"sha256":"65afd6fecc8f3aed09ee4be63583bc8eb472f06ceaa4fe24c4d1d5a1a3c0e13f"},"provenance":null,"requires-python":null,"size":92416,"upload-time":"2015-11-25T01:23:04.575877Z","url":"https://files.pythonhosted.org/packages/5d/a8/e62ec8105350c1e615ac84b084c7c8799d09e0d1b4530d3e68291dca8976/psutil-3.3.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp35-none-win32.whl","hashes":{"sha256":"ba1c558fbfcdf94515c2394b1155c1dc56e2bc2a9c17d30349827c9ed8a67e46"},"provenance":null,"requires-python":null,"size":91965,"upload-time":"2015-11-25T01:21:52.452732Z","url":"https://files.pythonhosted.org/packages/6e/6d/cf51e672eef1f1fbf9efce429d5411d4a2f3aa239e079b82531389562cd2/psutil-3.3.0-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"data-dist-info-metadata":{"sha256":"60f0487057f863a2f4cee79fc120f02b60168ae677fac14ba9ccc63127b37514"},"filename":"psutil-3.3.0-cp35-none-win_amd64.whl","hashes":{"sha256":"ba95ea0022dcb64d36f0c1335c0605fae35bdf3e0fea8d92f5d0f6456a35e55b"},"provenance":null,"requires-python":null,"size":94891,"upload-time":"2015-11-25T01:23:25.107121Z","url":"https://files.pythonhosted.org/packages/90/49/3726db12f0fa7ff8f7e5493cc128ee6b40f5720f7397a4ef01db9e28dd7b/psutil-3.3.0-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.tar.gz","hashes":{"sha256":"421b6591d16b509aaa8d8c15821d66bb94cb4a8dc4385cad5c51b85d4a096d85"},"provenance":null,"requires-python":null,"size":261983,"upload-time":"2015-11-25T01:20:55.681846Z","url":"https://files.pythonhosted.org/packages/fe/69/c0d8e9b9f8a58cbf71aa4cf7f27c27ee0ab05abe32d9157ec22e223edef4/psutil-3.3.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py2.6.exe","hashes":{"sha256":"326b305cbdb6f94dafbfe2c26b11da88b0ab07b8a07f8188ab9d75ff0c6e841a"},"provenance":null,"requires-python":null,"size":329214,"upload-time":"2015-11-25T18:50:19.881553Z","url":"https://files.pythonhosted.org/packages/a9/c1/7642d44312cffaa1b3efc6ac5252b7f1ab1c528903b55d56d7bd46805d92/psutil-3.3.0.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py2.7.exe","hashes":{"sha256":"9aede5b2b6fe46b3748ea8e5214443890d1634027bef3d33b7dad16556830278"},"provenance":null,"requires-python":null,"size":329153,"upload-time":"2015-11-25T01:22:03.853435Z","url":"https://files.pythonhosted.org/packages/bb/e0/f8e4e286bf9c075f0e9fb3c0b17cecef04cda5e91f4c54982b91b3baf338/psutil-3.3.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py3.3.exe","hashes":{"sha256":"73bed1db894d1aa9c3c7e611d302cdeab7ae8a0dc0eeaf76727878db1ac5cd87"},"provenance":null,"requires-python":null,"size":327512,"upload-time":"2015-11-25T01:22:31.545622Z","url":"https://files.pythonhosted.org/packages/fb/20/9438b78a3155b1eb480a4ea09dab6370f06e0a003cf43c3975743e0c9e8d/psutil-3.3.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py3.4.exe","hashes":{"sha256":"935b5dd6d558af512f42501a7c08f41d7aff139af1bb3959daa3abb859234d6c"},"provenance":null,"requires-python":null,"size":327497,"upload-time":"2015-11-25T01:22:51.620885Z","url":"https://files.pythonhosted.org/packages/ae/71/c68af8e9b05144de969da58e1bf5ebfe0859b1c83b827e05ae3116178bb1/psutil-3.3.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win-amd64-py3.5.exe","hashes":{"sha256":"4ca0111cf157dcc0f2f69a323c5b5478718d68d45fc9435d84be0ec0f186215b"},"provenance":null,"requires-python":null,"size":243943,"upload-time":"2015-11-25T01:23:16.182734Z","url":"https://files.pythonhosted.org/packages/bd/14/a67db75c827761bf55a50c6ce455cdf0fd7e75d1c7c395b7283359676288/psutil-3.3.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py2.6.exe","hashes":{"sha256":"b6f13c95398a3fcf0226c4dcfa448560ba5865259cd96ec2810658651e932189"},"provenance":null,"requires-python":null,"size":299025,"upload-time":"2015-11-25T18:49:29.675333Z","url":"https://files.pythonhosted.org/packages/ad/ea/d7c41ad9fab6e89263225c66971f9807a0396925dddf7c20901b637b99e2/psutil-3.3.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py2.7.exe","hashes":{"sha256":"ee6be30d1635bbdea4c4325d507dc8a0dbbde7e1c198bd62ddb9f43198b9e214"},"provenance":null,"requires-python":null,"size":299056,"upload-time":"2015-11-25T01:20:34.764583Z","url":"https://files.pythonhosted.org/packages/15/f7/a34370848c11d7d7933c0c107763ee470b54a7e48aa90a301919a3ad6757/psutil-3.3.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py3.3.exe","hashes":{"sha256":"dfa786858c268d7fbbe1b6175e001ec02738d7cfae0a7ce77bf9b651af676729"},"provenance":null,"requires-python":null,"size":293994,"upload-time":"2015-11-25T01:20:57.318743Z","url":"https://files.pythonhosted.org/packages/71/1e/af5675d52b426857441c29ad88d4fccfd55d300867ad02531d77991ab661/psutil-3.3.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py3.4.exe","hashes":{"sha256":"aa77f9de72af9c16cc288cd4a24cf58824388f57d7a81e400c4616457629870e"},"provenance":null,"requires-python":null,"size":294006,"upload-time":"2015-11-25T01:21:23.806352Z","url":"https://files.pythonhosted.org/packages/6d/41/cf5b54535ea052a32a76a8e8e56af817deb95f4ffde49277a52ded29763b/psutil-3.3.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.3.0.win32-py3.5.exe","hashes":{"sha256":"f500093357d04da8140d87932cac2e54ef592a54ca8a743abb2850f60c2c22eb"},"provenance":null,"requires-python":null,"size":233858,"upload-time":"2015-11-25T01:21:44.041656Z","url":"https://files.pythonhosted.org/packages/1e/1d/151535e51338efebe453a28d2f14d4d5b1e1f3ce54ccc63866c96dc7e1bd/psutil-3.3.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"data-dist-info-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"filename":"psutil-3.4.1-cp26-none-win32.whl","hashes":{"sha256":"0b1382db1cf76d53fb1d6e5619b5f3c86126e11a933b200c21ed4fa7fe5037aa"},"provenance":null,"requires-python":null,"size":91763,"upload-time":"2016-01-15T12:34:15.511699Z","url":"https://files.pythonhosted.org/packages/8a/a4/6dfd46e45d06da1a4d42814dbbdcffe3a4fc7f9b655e1c2919ac960512c2/psutil-3.4.1-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"data-dist-info-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"filename":"psutil-3.4.1-cp26-none-win_amd64.whl","hashes":{"sha256":"bcf212a926e8cffd3bec2acaeb584bf59a536e569d404bd8ea306f1752fbfc41"},"provenance":null,"requires-python":null,"size":94311,"upload-time":"2016-01-15T12:34:35.252561Z","url":"https://files.pythonhosted.org/packages/85/38/d9882d4e37f4b791bd949a1f45c620e0f2573bb4048eb16d59d469e97ec6/psutil-3.4.1-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"data-dist-info-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"filename":"psutil-3.4.1-cp27-none-win32.whl","hashes":{"sha256":"1b8424eaa712fef7da41fc7f391b452e8991a641a54e49c4f46eb72ca2585577"},"provenance":null,"requires-python":null,"size":91570,"upload-time":"2016-01-15T12:25:47.323665Z","url":"https://files.pythonhosted.org/packages/cd/2d/760f774b1325037ea4ef85972f45fc9dee417da33ba225b21a0a8e512f5d/psutil-3.4.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"data-dist-info-metadata":{"sha256":"5fd1ef4aa7b3f11f37a72bb6da5e9ae73e666b2a5580c3bf58fe259cac90c4e8"},"filename":"psutil-3.4.1-cp27-none-win_amd64.whl","hashes":{"sha256":"46d7429bae3703a0f2980c0299d4d49ada733c7ebd2cfa4e29fa3e31b5b16014"},"provenance":null,"requires-python":null,"size":94024,"upload-time":"2016-01-15T12:29:47.181880Z","url":"https://files.pythonhosted.org/packages/13/06/0104f224dd52bf9e3fb3ef14f6b6b93e9fac72f562842d54445af041f3f0/psutil-3.4.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp33-none-win32.whl","hashes":{"sha256":"08f4ab9b720310890fa9337321a6e1e8aa525538636526be77e82653588df46b"},"provenance":null,"requires-python":null,"size":91580,"upload-time":"2016-01-15T12:26:59.466185Z","url":"https://files.pythonhosted.org/packages/65/22/f7121341bc75bff65000ecc0c5aad4f2a6d129506c26d5533ade2ca67349/psutil-3.4.1-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp33-none-win_amd64.whl","hashes":{"sha256":"9e52230373076d0ecdb4aec373afd342c576ab52e11c382e058ed0188181a352"},"provenance":null,"requires-python":null,"size":93869,"upload-time":"2016-01-15T12:30:39.415707Z","url":"https://files.pythonhosted.org/packages/b0/84/a9edadc49ef3dbb89298855ae069b18ec534ca9f79a9294de417b8e46571/psutil-3.4.1-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp34-none-win32.whl","hashes":{"sha256":"e658cd0e0ad7a2971b2eeb6ee4b1a0ad14245003ea47425846bc8c3e892fd567"},"provenance":null,"requires-python":null,"size":91584,"upload-time":"2016-01-15T12:28:02.007581Z","url":"https://files.pythonhosted.org/packages/b6/cd/59a87e4f10181ee228c4edc7d4927e3d62f652cff9f25f95a7e7e9ab3df0/psutil-3.4.1-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp34-none-win_amd64.whl","hashes":{"sha256":"9c3e1146003df43aec9274be4741371a06896d70d7d590eb882ad59de2c06120"},"provenance":null,"requires-python":null,"size":93858,"upload-time":"2016-01-15T12:31:25.406165Z","url":"https://files.pythonhosted.org/packages/30/06/cf0559d12ca5ded37e6a32b1671be57fad3bade7f24536b943851aa6393e/psutil-3.4.1-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp35-none-win32.whl","hashes":{"sha256":"3d3b2df184a31646a7e66cc48304f900a82c18ab3dc69d2d5f693ea97fca0572"},"provenance":null,"requires-python":null,"size":93406,"upload-time":"2016-01-15T12:28:48.357070Z","url":"https://files.pythonhosted.org/packages/c9/57/6e65ff27fa567cd9a7bfbc0a435e33293451b80865bc3a3a7b13c9bf7799/psutil-3.4.1-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"data-dist-info-metadata":{"sha256":"ea21e4b91d38833326191bf60eb600b6764ba491e5b44843eced5c91fcf75a03"},"filename":"psutil-3.4.1-cp35-none-win_amd64.whl","hashes":{"sha256":"820ed01d84ffcda1c613be80c09318d7560dd3505299c65bb99f101963bfc3dd"},"provenance":null,"requires-python":null,"size":96333,"upload-time":"2016-01-15T12:31:50.145506Z","url":"https://files.pythonhosted.org/packages/f0/f6/ccf16168a627d10ffbd80120cd2c521c4c9ecdb4545e402b7deca79f93ac/psutil-3.4.1-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.tar.gz","hashes":{"sha256":"c7443659674c87d1f9feecee0dfeea765da02181c58d532e0633337e42180c89"},"provenance":null,"requires-python":null,"size":271657,"upload-time":"2016-01-15T12:22:39.420430Z","url":"https://files.pythonhosted.org/packages/a5/56/c64187a9a6889e622f7ec687254cdb3cc3c706e11bba9244e6ac781ecf38/psutil-3.4.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py2.6.exe","hashes":{"sha256":"e88e43423af984d7f2ecf8babf9d861ff59436794b0fdd2f85e9ea6bf7af6627"},"provenance":null,"requires-python":null,"size":331136,"upload-time":"2016-01-15T12:34:26.229804Z","url":"https://files.pythonhosted.org/packages/f7/90/53adfe2804c9cde062eb5014d88f0d067690fe1457ad2f49a2a553767689/psutil-3.4.1.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py2.7.exe","hashes":{"sha256":"3c08e2c200b222a92a4ecaa8055a48e27e7cfe82d9bf6402b52dd82413a786ed"},"provenance":null,"requires-python":null,"size":330798,"upload-time":"2016-01-15T12:29:26.473954Z","url":"https://files.pythonhosted.org/packages/66/9b/2b58fdab300e5f2a20c3999c485692cfa73cc9d4e50770a19cc871f92743/psutil-3.4.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py3.3.exe","hashes":{"sha256":"d03b8d081a281ebaa2122f259f7c0b3a464b2b98a3c221b9a54bfb0840355a9f"},"provenance":null,"requires-python":null,"size":329156,"upload-time":"2016-01-15T12:30:04.062459Z","url":"https://files.pythonhosted.org/packages/4c/26/695fa5b3578248f424d9a8e5bf2aafc6f706aeb7ec21ee31a5ebc2f79660/psutil-3.4.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py3.4.exe","hashes":{"sha256":"66a243c4b9ad93059be391a18e3f75e015ad70b220df4f7f30f9f578b89f27ad"},"provenance":null,"requires-python":null,"size":329140,"upload-time":"2016-01-15T12:31:00.945171Z","url":"https://files.pythonhosted.org/packages/25/63/54f2ba7cf31bb936b9c2cd7a77fd40a698fb232cd7c95c1ce997295a5954/psutil-3.4.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win-amd64-py3.5.exe","hashes":{"sha256":"4ee8641803d68a2e48952951336f9474a8914854da088fca673d67a91da7f9a4"},"provenance":null,"requires-python":null,"size":245586,"upload-time":"2016-01-15T12:31:39.398299Z","url":"https://files.pythonhosted.org/packages/43/54/f2d3b8845105fe5f55d5f0fde36773ab94fe1e35a0f4a5219adc818d586b/psutil-3.4.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py2.6.exe","hashes":{"sha256":"a4de0daf0dc7aeff6d45c6a1c782ef30d2b4fc6495196acabcb5cde2fb9b5a74"},"provenance":null,"requires-python":null,"size":300946,"upload-time":"2016-01-15T12:33:59.097771Z","url":"https://files.pythonhosted.org/packages/9e/34/53e1e85df97508df1b3eea711ab2809cc8f01b20e3b2341645673eb5d835/psutil-3.4.1.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py2.7.exe","hashes":{"sha256":"0efcecb6fcc21d83e9d4354754c6b8a8deb47a5fa06ec5d09fcf9799719eeac2"},"provenance":null,"requires-python":null,"size":300700,"upload-time":"2016-01-15T12:25:12.391548Z","url":"https://files.pythonhosted.org/packages/2f/4c/a07a53ff938e3bbc2ba73e4a484af8d1e02054b0bfcaf0f6d30117187d9f/psutil-3.4.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py3.3.exe","hashes":{"sha256":"62d0b529e40262293f39d7455db24f7ee297a1a3fe7f0e3e5923ae8168bd865c"},"provenance":null,"requires-python":null,"size":295638,"upload-time":"2016-01-15T12:26:24.412799Z","url":"https://files.pythonhosted.org/packages/39/6c/03ade7ba131b3952d916ed26c277418234bec0c9a5dfad513b9a5bb51046/psutil-3.4.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py3.4.exe","hashes":{"sha256":"6102294d6150f2c072dbc0166348389e8fa5d14a769ad118b697cda5b31c3381"},"provenance":null,"requires-python":null,"size":295651,"upload-time":"2016-01-15T12:27:36.577070Z","url":"https://files.pythonhosted.org/packages/52/d7/c2e9e0cb21482304e39a7681066c32c50e984f109bcda5929a84af926d70/psutil-3.4.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.1.win32-py3.5.exe","hashes":{"sha256":"0ffd99167272bb80c6ecf68f4c3d3176bef0f8c2a68f7e2787cab32413830023"},"provenance":null,"requires-python":null,"size":235502,"upload-time":"2016-01-15T12:28:24.463783Z","url":"https://files.pythonhosted.org/packages/1c/06/4d0ec9a6427db9c3b9885c4d724ca299746519b2ee61b724665d49e352c6/psutil-3.4.1.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"data-dist-info-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"filename":"psutil-3.4.2-cp26-none-win32.whl","hashes":{"sha256":"2ac75c13657ab18eac0014e3f4c80def16978507b30e7719e46042ec93316bb0"},"provenance":null,"requires-python":null,"size":91920,"upload-time":"2016-01-20T16:25:20.303966Z","url":"https://files.pythonhosted.org/packages/be/b7/999dcbee8cc5ae32e64b2d0c9d588a3f5a441a07404772af83e86f3c8bc7/psutil-3.4.2-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"data-dist-info-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"filename":"psutil-3.4.2-cp26-none-win_amd64.whl","hashes":{"sha256":"162f76140ca09490b9d218840bd641cbd1439245dcc2a9dd41f86224ed19490c"},"provenance":null,"requires-python":null,"size":94467,"upload-time":"2016-01-20T16:27:40.461506Z","url":"https://files.pythonhosted.org/packages/60/97/f9ea4fa7a4914350d15347a6a583c8a185643bb6bd5dc76d13e9d7dfc150/psutil-3.4.2-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"data-dist-info-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"filename":"psutil-3.4.2-cp27-none-win32.whl","hashes":{"sha256":"b5f4bfdaa6389552501253b13b6022b7e3d715e4dca4b5cc1808f58cca181359"},"provenance":null,"requires-python":null,"size":91721,"upload-time":"2016-01-20T16:25:38.512050Z","url":"https://files.pythonhosted.org/packages/d4/19/4e5c376587076c969784762de8024bb30168a548b402e1b432221c5b97b1/psutil-3.4.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"data-dist-info-metadata":{"sha256":"13402cacd1df223d4b15886666e72dc0e55f16091f011fc3f0c83e49b8034134"},"filename":"psutil-3.4.2-cp27-none-win_amd64.whl","hashes":{"sha256":"9267e9bccb5c8b1c5ca872eb1caf88ba0ae47e336eb200be138f51d9f75e1113"},"provenance":null,"requires-python":null,"size":94177,"upload-time":"2016-01-20T16:27:58.779061Z","url":"https://files.pythonhosted.org/packages/d3/19/39f42cdfba58ab593d24f49ffc073c07b9b34ff7d5ba079b975018002e51/psutil-3.4.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp33-none-win32.whl","hashes":{"sha256":"413800a94815e6bf3e3227823e4d46b06c63bd22ab9e5af112b9220af9a9c9d8"},"provenance":null,"requires-python":null,"size":91738,"upload-time":"2016-01-20T16:25:59.987045Z","url":"https://files.pythonhosted.org/packages/8e/7a/8ba0c9da039b5733edbe17321f0546f08a7066861ffcdb83c31eb061e8f1/psutil-3.4.2-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp33-none-win_amd64.whl","hashes":{"sha256":"45535c18a0f261f90ff1ebb7d74c5e88d582cfb2006e4588498b9c0c9da5acb3"},"provenance":null,"requires-python":null,"size":94024,"upload-time":"2016-01-20T16:28:42.605657Z","url":"https://files.pythonhosted.org/packages/34/a7/94fc00a023ede02c3a7f525c63997a23c22434cbed65738ee3ef8939e084/psutil-3.4.2-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp34-none-win32.whl","hashes":{"sha256":"c9ef9a08254c251858cf747703e6fd75fe6e9549b1e040bb4a501feaf44a5a75"},"provenance":null,"requires-python":null,"size":91749,"upload-time":"2016-01-20T16:26:31.788083Z","url":"https://files.pythonhosted.org/packages/dc/8b/df5d7dcfe8fc5db0c303e518ce12b7117cb70e1cbb29c0396ea6e36fc7a2/psutil-3.4.2-cp34-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp34-none-win_amd64.whl","hashes":{"sha256":"0bf8925c3d252178c47bd8f29aff99c57a56f94513354b60069b457ca04bc25b"},"provenance":null,"requires-python":null,"size":94004,"upload-time":"2016-01-20T16:29:27.043903Z","url":"https://files.pythonhosted.org/packages/1c/cf/c9ce0014f43f74b1ce72c004c8f2eda68339cbc19117d9f090ee14afce3e/psutil-3.4.2-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp35-none-win32.whl","hashes":{"sha256":"461d1431a14e4da5e687cfdc2a8576b1f0e3bc658694ab9c6ef2fa1e4c1a4871"},"provenance":null,"requires-python":null,"size":93563,"upload-time":"2016-01-20T16:27:20.576920Z","url":"https://files.pythonhosted.org/packages/de/51/d1ab564dfe98d5fcbccfcab0afdedb269f7266192721e94688ab3956b123/psutil-3.4.2-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"data-dist-info-metadata":{"sha256":"23ac4ea4417292cac0684989ffba547b75ac89f3a6ef7965183ec9e389edaa89"},"filename":"psutil-3.4.2-cp35-none-win_amd64.whl","hashes":{"sha256":"23d4ea79fea3de81daf9460662e49ff718555779b2f5e5e3610648c0a8cafecc"},"provenance":null,"requires-python":null,"size":96484,"upload-time":"2016-01-20T16:29:53.151921Z","url":"https://files.pythonhosted.org/packages/89/d8/dc9ce7b8862ab2d86975dd5199791b0ab2b2168fc2389223a216c2da1d45/psutil-3.4.2-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.tar.gz","hashes":{"sha256":"b17fa01aa766daa388362d0eda5c215d77e03a8d37676b68971f37bf3913b725"},"provenance":null,"requires-python":null,"size":274361,"upload-time":"2016-01-20T16:26:46.533423Z","url":"https://files.pythonhosted.org/packages/7b/58/2675697b6831e6ac4b7b7bc4e5dcdb24a2f39f8411186573eb0de16eb6d5/psutil-3.4.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py2.6.exe","hashes":{"sha256":"3716cb36373ecfd033c148c8e8e22d815a9b682c87538c5bde2a3faca1a44705"},"provenance":null,"requires-python":null,"size":331279,"upload-time":"2016-01-20T16:27:30.129391Z","url":"https://files.pythonhosted.org/packages/6f/52/26cefb84d714ecdf39f6d4ea62af49af731cf55d8937a252226de41d3fb0/psutil-3.4.2.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py2.7.exe","hashes":{"sha256":"9eba153441fabd6677f9dec95eedbfdcf4fe832a43b91c18f2c15bfd0a12b6c0"},"provenance":null,"requires-python":null,"size":330991,"upload-time":"2016-01-20T16:27:49.025596Z","url":"https://files.pythonhosted.org/packages/1f/85/817b298f6865d7a140897882015096ab25514e113c98ba3896b2e2c0425c/psutil-3.4.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py3.3.exe","hashes":{"sha256":"2ffc77ec7452675db45174f77e65bfc9abd5780e696c4dd486fff89e18ef104a"},"provenance":null,"requires-python":null,"size":329349,"upload-time":"2016-01-20T16:28:23.729829Z","url":"https://files.pythonhosted.org/packages/2b/05/ccb8e8dc272a6aa126a6066583102b78df8939aeffd910a3ea28a51d48af/psutil-3.4.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py3.4.exe","hashes":{"sha256":"6b576ea8faa312700953de30b92ff49dcd966dcdbf2e039c3655077826b59812"},"provenance":null,"requires-python":null,"size":329334,"upload-time":"2016-01-20T16:29:12.322779Z","url":"https://files.pythonhosted.org/packages/8f/0e/9b3eedad9ea2aa8e51c3ca6aa6485c0ec85bfc73925fbd4d82bbe03ead18/psutil-3.4.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win-amd64-py3.5.exe","hashes":{"sha256":"2a987c57ddb06a1e67f75a4dd34d2962f8675c3d60b2104da2d60fdaa378b50f"},"provenance":null,"requires-python":null,"size":245779,"upload-time":"2016-01-20T16:29:45.537968Z","url":"https://files.pythonhosted.org/packages/2f/bb/5483a7a54dfaedcd5bc6d0f9f8beef21d96785589e10d09e246f7092cfe1/psutil-3.4.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py2.6.exe","hashes":{"sha256":"87b657f1021ab4155669f77baf9557657a015b0762854702d64ee7cfa19d5ae2"},"provenance":null,"requires-python":null,"size":301089,"upload-time":"2016-01-20T16:25:11.034218Z","url":"https://files.pythonhosted.org/packages/2e/9b/2bb0317a5113b4a3d597a9bcb94cafacb52f15a10631b1f0eb781ceb8e7a/psutil-3.4.2.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py2.7.exe","hashes":{"sha256":"d837d654a78fcc6cf7338fc3c3f025e5a43cf646d4d6cf180f0f3573ef255844"},"provenance":null,"requires-python":null,"size":300894,"upload-time":"2016-01-20T16:25:29.293012Z","url":"https://files.pythonhosted.org/packages/8c/3b/bf4d0698153784231768aa79255f1641efde680c4d178ba327546eba69df/psutil-3.4.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py3.3.exe","hashes":{"sha256":"ef756512c86cf24916f47b2209ff5dc69ef4d5ff8b3b0229863aab3537af58a1"},"provenance":null,"requires-python":null,"size":295833,"upload-time":"2016-01-20T16:25:50.417672Z","url":"https://files.pythonhosted.org/packages/50/be/c4911ae27c944e12183d9ba844ff0eee706b5208f92e4929d8120b79448e/psutil-3.4.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py3.4.exe","hashes":{"sha256":"13917c61de33518fcdca94a8f1005c4bb0be1f106af773de8c12d6a5a3f349ae"},"provenance":null,"requires-python":null,"size":295845,"upload-time":"2016-01-20T16:26:17.574774Z","url":"https://files.pythonhosted.org/packages/b5/d9/9a15af2703d8a0d7b685511df811fc4930b12d7b85b96573e843a0ba1067/psutil-3.4.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-3.4.2.win32-py3.5.exe","hashes":{"sha256":"75c4484f0e1038c1a37ae37fc80f5b59b456e363c696ed889027e79a831853ec"},"provenance":null,"requires-python":null,"size":235696,"upload-time":"2016-01-20T16:27:12.935221Z","url":"https://files.pythonhosted.org/packages/2c/74/46cf554a4a8d7d2367f058e6c90ba26a66e52e818812ec6de53a5013bd87/psutil-3.4.2.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp26-none-win32.whl","hashes":{"sha256":"0661261b634f01ec2568136fedf29382f5c94678c34f56b4137b1d019085ca6f"},"provenance":null,"requires-python":null,"size":154479,"upload-time":"2016-02-17T16:42:58.173240Z","url":"https://files.pythonhosted.org/packages/72/e2/24700d1a099dcd824ca7305cf439625a457221459494e677e7649a2f228b/psutil-4.0.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp26-none-win_amd64.whl","hashes":{"sha256":"b613a9fb5c3d2b16c65df3121aa369a28caed83391883bc24918cf16c5de495b"},"provenance":null,"requires-python":null,"size":156749,"upload-time":"2016-02-17T16:45:09.428253Z","url":"https://files.pythonhosted.org/packages/62/c0/1adb11a832d5aab8a984702a4f6fc08e5698fa4bbc8dc6eddac1c711c7ab/psutil-4.0.0-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp27-cp27m-win32.whl","hashes":{"sha256":"3eb6bc7e8d92777deb4288178c25455e21109033bb54ec475485b611e92d3b42"},"provenance":null,"requires-python":null,"size":154302,"upload-time":"2016-02-17T16:43:23.863441Z","url":"https://files.pythonhosted.org/packages/e8/c3/542bc833b743e952cbf99017ecb60add0ad3725a82e942b25aa5de523f8c/psutil-4.0.0-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"c94193f38aa3bc35fd5dbcd24653d1f683c88ec8030997d1d56f92207ba7c523"},"provenance":null,"requires-python":null,"size":156487,"upload-time":"2016-02-24T17:23:02.172281Z","url":"https://files.pythonhosted.org/packages/32/1b/5f3cc96374c4eac441e96bb8698556c6c48eacfdcf843093bebcfd8ce56b/psutil-4.0.0-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"data-dist-info-metadata":{"sha256":"f4174370ba76a8fc971e342367fb6df5d587165b15b69cf652d5870ad0386329"},"filename":"psutil-4.0.0-cp27-none-win_amd64.whl","hashes":{"sha256":"cb969d3c77db8810aba45e8a04e0b2851cd088e338be2430e1ff452f4e06007c"},"provenance":null,"requires-python":null,"size":156486,"upload-time":"2016-02-17T16:45:40.145586Z","url":"https://files.pythonhosted.org/packages/94/37/dc09e24aa80016ddeaff235d2f724d8aac9813b73cc3bf8a7fe3d1878315/psutil-4.0.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp33-cp33m-win32.whl","hashes":{"sha256":"6a372681382b523bc837ee2eff6a84ded0f85b013b7c29ea6211bc928c7cc656"},"provenance":null,"requires-python":null,"size":154244,"upload-time":"2016-02-24T17:21:40.946801Z","url":"https://files.pythonhosted.org/packages/a0/8a/b9352e0daf69b501296715e0fca1b49d861130eb66156ce3b12aeeb039e4/psutil-4.0.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"2bbb75fc2549965b457f313cbdfb98a00624f25fcb36e075322bb8b8912d83b5"},"provenance":null,"requires-python":null,"size":156393,"upload-time":"2016-02-24T17:23:25.807132Z","url":"https://files.pythonhosted.org/packages/4b/77/0fefa732947da69cba7f2580285eff553fe4a416314234f809901bede361/psutil-4.0.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp33-none-win32.whl","hashes":{"sha256":"d6219c89940d745b614716be7c906660f2108a1d84b8ffc720922596b8306e23"},"provenance":null,"requires-python":null,"size":154245,"upload-time":"2016-02-17T16:43:48.027623Z","url":"https://files.pythonhosted.org/packages/1f/d9/34fb4fab5f1bbfeddc76675b1b5ca00b45ef490e63295af33542cedcd26b/psutil-4.0.0-cp33-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp33-none-win_amd64.whl","hashes":{"sha256":"a64bb22e264f91a6d80cf8fdd813bd4fdd349dc367b363d517cf8ae1bc2c5db0"},"provenance":null,"requires-python":null,"size":156399,"upload-time":"2016-02-17T16:46:09.673427Z","url":"https://files.pythonhosted.org/packages/97/03/b9485635cb38dfad854754625422a49f434f53f214bff4885580f0fb21e6/psutil-4.0.0-cp33-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp34-cp34m-win32.whl","hashes":{"sha256":"a521266ac13485772987f00342b53cb230cde98ce91d61154860ba4109fe2ebe"},"provenance":null,"requires-python":null,"size":154269,"upload-time":"2016-02-17T16:44:14.936699Z","url":"https://files.pythonhosted.org/packages/73/32/6399071b097f1251f6fa12770b30a67d5b3c9c0c76e81eacbb6139e1bf6d/psutil-4.0.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"b559b8e8a85cde929e01e94e9635649e8641a88b2d077714933dc7723a967020"},"provenance":null,"requires-python":null,"size":156390,"upload-time":"2016-02-24T17:23:50.194021Z","url":"https://files.pythonhosted.org/packages/71/d7/878b77bad61bd94f4454536e823b6a48cd0af0f23b1506a2c8a49b2578cd/psutil-4.0.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp34-none-win_amd64.whl","hashes":{"sha256":"02d7291f81e78c506ac2b5481aa9dc6d3888195484ac114ac984b37477f60929"},"provenance":null,"requires-python":null,"size":156397,"upload-time":"2016-02-17T16:46:44.626238Z","url":"https://files.pythonhosted.org/packages/ea/22/5f44e6eaa1e82f5a1497f3dfcf045e1998fca36d70de8a370ec96ce0f789/psutil-4.0.0-cp34-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp35-cp35m-win32.whl","hashes":{"sha256":"7906302696960a6a788bb8fe1165b4ccd0156553b8a2f61640fd45a836d39024"},"provenance":null,"requires-python":null,"size":156177,"upload-time":"2016-02-24T17:22:25.142667Z","url":"https://files.pythonhosted.org/packages/e2/fe/a5ec73e62878cc2d0451b7029f4406647435dd8036ab15d6ed2fd42558bf/psutil-4.0.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"f4214bdb2e96374b4c4a3a818bd8c7867f94571d33b91867b6dfd5f9b328c8ac"},"provenance":null,"requires-python":null,"size":158783,"upload-time":"2016-02-24T17:24:26.439061Z","url":"https://files.pythonhosted.org/packages/1d/a7/9300ad3d4071c191894073a94217ed5c0ca9604c782bdbf083bbedfa9cb1/psutil-4.0.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp35-none-win32.whl","hashes":{"sha256":"4a1631cb8c4de2b6c9b4b16f8800d43de23c683805f7b6a5aec1c268a73df270"},"provenance":null,"requires-python":null,"size":156183,"upload-time":"2016-02-17T16:44:42.458015Z","url":"https://files.pythonhosted.org/packages/14/f0/a2436cb642ecfec0bfb6338e5fa26581d4dbcf1a00f7d9fe99380eb6779f/psutil-4.0.0-cp35-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"data-dist-info-metadata":{"sha256":"d2287fe3ce0e3872765b52193af0ec6f19279df57903b85d13961ca84449ba60"},"filename":"psutil-4.0.0-cp35-none-win_amd64.whl","hashes":{"sha256":"994839b6d99acbf90914fddf2e2817aaffb67ceca5d10134319267e3ffe97258"},"provenance":null,"requires-python":null,"size":158787,"upload-time":"2016-02-17T16:47:18.920483Z","url":"https://files.pythonhosted.org/packages/e8/2a/e215824c785d77119af61802bbb4d16dacc26ec0687709274afa3ac039fa/psutil-4.0.0-cp35-none-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.tar.gz","hashes":{"sha256":"1a7c672f9ee79c84ff16b8de6f6040080f0e25002ac47f115f4a54aa88e5cfcd"},"provenance":null,"requires-python":null,"size":293800,"upload-time":"2016-02-17T16:41:45.938066Z","url":"https://files.pythonhosted.org/packages/c4/3b/44bcae6c0fc53362bb7325fde25a73b7fd46541b57c89b7556ca81b08e7e/psutil-4.0.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py2.6.exe","hashes":{"sha256":"61bf81cbe84e679a5b619e65775b0674b2c463885e49ddab73778198608198c5"},"provenance":null,"requires-python":null,"size":394157,"upload-time":"2016-02-17T16:45:00.666953Z","url":"https://files.pythonhosted.org/packages/86/50/6303a28a4ab5c9b6b9ef74eef70b141d5bd743ab096c58d225b6212fa057/psutil-4.0.0.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"23618cbc04e2431d9c4d97f56ab8b4e2e35366c9a9a6e1ef89a3a7287d359864"},"provenance":null,"requires-python":null,"size":393901,"upload-time":"2016-02-17T16:45:21.822412Z","url":"https://files.pythonhosted.org/packages/59/82/93052d6359addea338c528ebd50254806d62bc2b2d1ad1303c49d85162f9/psutil-4.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"b6f16c71be03495eeb4772c1f3f926213e3ea82ea7779bd1143229e6b419760b"},"provenance":null,"requires-python":null,"size":392318,"upload-time":"2016-02-17T16:45:57.189438Z","url":"https://files.pythonhosted.org/packages/e6/14/9ac37705e0753732c7707b000d1e076daac95ee02f35fd43ce906235ea1f/psutil-4.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"24b41e436afbcecb07e485e58a52effdbd7b8065ad8a2e4d555b6d88907f19b7"},"provenance":null,"requires-python":null,"size":392315,"upload-time":"2016-02-17T16:46:29.967466Z","url":"https://files.pythonhosted.org/packages/77/a9/ff7c29d2e244f5bdc7654a626cbfcccd401e78df6e9388713f322c7aa7c7/psutil-4.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win-amd64-py3.5.exe","hashes":{"sha256":"88edc0e7bfa672c245df74b1ac3b59db432cd75e5704beccc268e177ac2ffbbc"},"provenance":null,"requires-python":null,"size":308678,"upload-time":"2016-02-17T16:47:07.280714Z","url":"https://files.pythonhosted.org/packages/cc/1b/863bee07da70fe61cae804333d64242d9001b54288e8ff54e770225bbc0a/psutil-4.0.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py2.6.exe","hashes":{"sha256":"882a8ac29b63f256f76465d8fcd6e9eaeb9c929acdac26af102da97d66b2b619"},"provenance":null,"requires-python":null,"size":364245,"upload-time":"2016-02-17T16:42:49.267819Z","url":"https://files.pythonhosted.org/packages/41/39/6ea85cb0c748aae2943144118f1696004f5a99c54dab1fc635cffbb0d06c/psutil-4.0.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py2.7.exe","hashes":{"sha256":"bf3b2e305ca7408df40156c9aa6261c7baaff831441f0c018d0682bd820286f2"},"provenance":null,"requires-python":null,"size":364072,"upload-time":"2016-02-17T16:43:14.467612Z","url":"https://files.pythonhosted.org/packages/8f/0d/3e9cf8abb62d7241531019d78abaa87a19f3fcc017bfb9c2058ba61e8cf1/psutil-4.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py3.3.exe","hashes":{"sha256":"38d38211bba35c705a007e62b9dcc9be1d222acfcbee812612d4a48f9d8f0230"},"provenance":null,"requires-python":null,"size":358940,"upload-time":"2016-02-17T16:43:38.439349Z","url":"https://files.pythonhosted.org/packages/57/eb/514a71eab624b381473a3df9c3e3a02f5bb15707b12daf02c137271dfd26/psutil-4.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py3.4.exe","hashes":{"sha256":"bc868653a6502c3a01da32b3a598a8575674975f5586ac0bf9181349588925b9"},"provenance":null,"requires-python":null,"size":358966,"upload-time":"2016-02-17T16:44:02.873748Z","url":"https://files.pythonhosted.org/packages/25/f6/ef4b802658c21d5b79a0e038db941f4b08a7cc2de78df1949ad709542682/psutil-4.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.0.0.win32-py3.5.exe","hashes":{"sha256":"38244b0d07d3bece481a6f1c049e6101fdd26f2ee63dadcb63ce993283032fdc"},"provenance":null,"requires-python":null,"size":298915,"upload-time":"2016-02-17T16:44:31.772075Z","url":"https://files.pythonhosted.org/packages/b9/be/b8938c409231dab07d2954ff7b4d1129e725e0e8ab1b016d7f471f3285e9/psutil-4.0.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"data-dist-info-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"filename":"psutil-4.1.0-cp26-none-win32.whl","hashes":{"sha256":"13aed96ad945db5c6b3d5fbe92be65330a3f2f757a300c7d1578a16efa0ece7f"},"provenance":null,"requires-python":null,"size":159498,"upload-time":"2016-03-12T17:13:58.069685Z","url":"https://files.pythonhosted.org/packages/77/04/d5a92cb5c0e79b84294f6c99b9725806921d1d88032e9d056ca8a7ba31c1/psutil-4.1.0-cp26-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"data-dist-info-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"filename":"psutil-4.1.0-cp26-none-win_amd64.whl","hashes":{"sha256":"90b58cf88e80a4af52b79678df474679d231ed22200e6c25605a42ca71708a47"},"provenance":null,"requires-python":null,"size":161924,"upload-time":"2016-03-12T17:17:01.346400Z","url":"https://files.pythonhosted.org/packages/b5/a5/cf96f9f13f9e20bdb4cd2ca1af2ddd74f76fea4bbfb8505c31a5900b38d2/psutil-4.1.0-cp26-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"data-dist-info-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"filename":"psutil-4.1.0-cp27-cp27m-win32.whl","hashes":{"sha256":"ac141a44a5c145e9006bc7081c714b2c317077d158b65fe4624c9cbf2b8ac7bf"},"provenance":null,"requires-python":null,"size":159318,"upload-time":"2016-03-12T17:14:48.199436Z","url":"https://files.pythonhosted.org/packages/8a/31/439614cc2ccd6f3ce1d173c0d7c7a9e45be17cd2bf3ae1f8feaaf0a90cee/psutil-4.1.0-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"data-dist-info-metadata":{"sha256":"9a907b7ac326dc42e1ba74d263d9a38c07d0929a2a9f91960d9961923d796b2d"},"filename":"psutil-4.1.0-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"3605b6b9f23f3e186b157b03a95e0158559eb74bcef5d51920b8ddb48cc3a7e7"},"provenance":null,"requires-python":null,"size":161579,"upload-time":"2016-03-12T17:17:24.047978Z","url":"https://files.pythonhosted.org/packages/90/97/0a34c0e98bb794f0fc19f0eae13d26fbf39583c768a9a6c614c917135c00/psutil-4.1.0-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp33-cp33m-win32.whl","hashes":{"sha256":"5568e21c8eb9de0e56c8a4a38982b725bf42117bca7ac75c7b079e5214aea5c4"},"provenance":null,"requires-python":null,"size":159241,"upload-time":"2016-03-12T17:15:16.355951Z","url":"https://files.pythonhosted.org/packages/bb/de/8e1f8c4ea6035d08e3c87a0cfc8af6f2862da21697c16d1d17311e095117/psutil-4.1.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"3d3f8ae20b04b68e65b46dc2eedf15f32925655dacbb11cb7afe56ac562e112a"},"provenance":null,"requires-python":null,"size":161458,"upload-time":"2016-03-12T17:17:51.825112Z","url":"https://files.pythonhosted.org/packages/ac/cf/6241dd597ef4f995ab8e29746c54890c1acbb322484afed05aa8988118e1/psutil-4.1.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp34-cp34m-win32.whl","hashes":{"sha256":"faafb81bf7717fa8c44bb0f2e826768f561c0311fd0568090c59c9b253b65238"},"provenance":null,"requires-python":null,"size":159266,"upload-time":"2016-03-12T17:15:48.577644Z","url":"https://files.pythonhosted.org/packages/48/2d/46ba91df965d4f0af5fd4252ac249ff408f6cb966fe1208396933275246f/psutil-4.1.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"4ab1ee4152dbb790a37291149b73b1918ab4398c8edb3af7847fa6c884024c93"},"provenance":null,"requires-python":null,"size":161466,"upload-time":"2016-03-12T17:18:23.409579Z","url":"https://files.pythonhosted.org/packages/c0/b1/70868328ddf2cfcde201136bdaf4c9f7fabf868890bc91694fd5fa0fbc19/psutil-4.1.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp35-cp35m-win32.whl","hashes":{"sha256":"2fa06b7ba58e870fdaa1427e82ed427a785493c7a998e059e0806b2c48bdbfaf"},"provenance":null,"requires-python":null,"size":161345,"upload-time":"2016-03-12T17:16:33.062364Z","url":"https://files.pythonhosted.org/packages/dc/f7/5d3f84507c057af85bc70da10b51a827766999f221b42cdf9621ca756e80/psutil-4.1.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"data-dist-info-metadata":{"sha256":"28328dafba00a0d87523ceca7f8bb23231019819caf14ec9eeddc3c4c8104338"},"filename":"psutil-4.1.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"1ca460fea3822d04f332f5dde144accc5ca4610e5bbce53f15d09cd985f30385"},"provenance":null,"requires-python":null,"size":164147,"upload-time":"2016-03-12T17:18:49.483687Z","url":"https://files.pythonhosted.org/packages/f1/65/040624aab6ca646af0c8b68ac54d08e0a33a672feb9405581cd509741367/psutil-4.1.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.tar.gz","hashes":{"sha256":"c6abebec9c8833baaf1c51dd1b0259246d1d50b9b50e9a4aa66f33b1e98b8d17"},"provenance":null,"requires-python":null,"size":301330,"upload-time":"2016-03-12T17:12:53.032151Z","url":"https://files.pythonhosted.org/packages/71/9b/6b6f630ad4262572839033b69905d415ef152d7701ef40aa98941ba75b38/psutil-4.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py2.6.exe","hashes":{"sha256":"30a97a9c3ace92001e419a6bb039b2e899c5ab24cab6ad8bc249506475c84a0c"},"provenance":null,"requires-python":null,"size":399511,"upload-time":"2016-03-12T17:16:48.833953Z","url":"https://files.pythonhosted.org/packages/28/bd/c389af84b684d36010a634834f76932ff60f33505c54413c50eceb720a5d/psutil-4.1.0.win-amd64-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"3ba9236c38fe85088b9d79bd5871e07f253d225b357ce82db9240e5807f147b6"},"provenance":null,"requires-python":null,"size":399168,"upload-time":"2016-03-12T17:17:14.518227Z","url":"https://files.pythonhosted.org/packages/6c/e8/49ff1b33e9fa6f7ea1232d40bbef9a453424fbf0421a7c53d9ecb9a896e7/psutil-4.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py3.3.exe","hashes":{"sha256":"99a902d1bf5beb13cca4d7a3dc82efb6eaf40aafe916a5b632d47393313bfcfd"},"provenance":null,"requires-python":null,"size":397557,"upload-time":"2016-03-12T17:17:40.792655Z","url":"https://files.pythonhosted.org/packages/c5/3e/47adda552a79480e2a5d38a2f90144ab4e7ea34eba2b707523cdd1c65fc6/psutil-4.1.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py3.4.exe","hashes":{"sha256":"ecf8b83e038acdda9102aebadb1525f89231463772cbe89e3e8a2cf5a5c6065d"},"provenance":null,"requires-python":null,"size":397564,"upload-time":"2016-03-12T17:18:11.514028Z","url":"https://files.pythonhosted.org/packages/52/9f/5874f391a300feead5519872395cbb4d4588eace24962150a03cd9b6ffdf/psutil-4.1.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win-amd64-py3.5.exe","hashes":{"sha256":"448db9bf9db5d162f0282af70688e03d67d756b93d5bed94b0790a27a96af75b"},"provenance":null,"requires-python":null,"size":314216,"upload-time":"2016-03-12T17:18:36.745340Z","url":"https://files.pythonhosted.org/packages/32/34/6588580a1775a0741e14946003bf2722c493c7956c7dcc9a40c85dbe19f5/psutil-4.1.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py2.6.exe","hashes":{"sha256":"11b48d7ec00061960ffbef33a42e919f188e10a6a54c4161692d77a3ac37f1e2"},"provenance":null,"requires-python":null,"size":369441,"upload-time":"2016-03-12T17:13:47.639393Z","url":"https://files.pythonhosted.org/packages/5d/cc/7bf8593a60ed54b47def9a49a60b8bc3517d6fef6ee52a229583a5bc9046/psutil-4.1.0.win32-py2.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py2.7.exe","hashes":{"sha256":"c15ddba9f4c278f7d1bbbadc34df89d993171bb328fa1117cbecf68bcc1a01e5"},"provenance":null,"requires-python":null,"size":369263,"upload-time":"2016-03-12T17:14:25.482243Z","url":"https://files.pythonhosted.org/packages/bf/fa/7f0eda490dd5480bb8e6358f9e893fe7f824ec3a5b275b069708272d2260/psutil-4.1.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py3.3.exe","hashes":{"sha256":"8f97a836b89b09718b00aedbfd11d8468bbfcb821feadd65e338bd0b972dac54"},"provenance":null,"requires-python":null,"size":364112,"upload-time":"2016-03-12T17:15:06.252090Z","url":"https://files.pythonhosted.org/packages/e8/1c/e7d17500c0f5899f0fd3fd3b6d93c92254aff6de00cabc9c08de5a3803a2/psutil-4.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py3.4.exe","hashes":{"sha256":"fef37f4e2964043a58b6aa3b3be05ef9b3c7a58feb806d7e49dc91702bac52fa"},"provenance":null,"requires-python":null,"size":364136,"upload-time":"2016-03-12T17:15:29.468205Z","url":"https://files.pythonhosted.org/packages/6d/8e/7acd4079567fc64e1df12be6faeddf0ee19432445210293594444ea980a6/psutil-4.1.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.1.0.win32-py3.5.exe","hashes":{"sha256":"1a9409d2204d397b1ed10528065e45020e0b1c2ee5204bf5a1bc6fdff3f6ab91"},"provenance":null,"requires-python":null,"size":304255,"upload-time":"2016-03-12T17:16:06.024724Z","url":"https://files.pythonhosted.org/packages/5b/30/10f3bef7fa284167a3f9bd3862bcba5cc14fb8509f146a2f696ea5265b93/psutil-4.1.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"5eaefa5c21411ed0414953f28eef77176307e672d668daf95d91e18ad3dc36f7"},"data-dist-info-metadata":{"sha256":"5eaefa5c21411ed0414953f28eef77176307e672d668daf95d91e18ad3dc36f7"},"filename":"psutil-4.2.0-cp27-cp27m-win32.whl","hashes":{"sha256":"19f6c8bd30d7827ce4d4bbcfe23fe7158fea3d72f59505850c5afa12985184bb"},"provenance":null,"requires-python":null,"size":165248,"upload-time":"2016-05-15T06:36:58.089018Z","url":"https://files.pythonhosted.org/packages/58/a5/2ccc9f6180ea769005405381f6b0d01fe1268f20cc85877b02c04c27d306/psutil-4.2.0-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5eaefa5c21411ed0414953f28eef77176307e672d668daf95d91e18ad3dc36f7"},"data-dist-info-metadata":{"sha256":"5eaefa5c21411ed0414953f28eef77176307e672d668daf95d91e18ad3dc36f7"},"filename":"psutil-4.2.0-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"92bc2351bb4bc7672b3d0e251a449ac2234bbe4fac11f708614bdc0a8ebffe3b"},"provenance":null,"requires-python":null,"size":167782,"upload-time":"2016-05-15T06:37:06.221707Z","url":"https://files.pythonhosted.org/packages/c8/e5/5d0a1b2e182e41888fc4e9f4f657f37f126f9fdcd431b592442311c2db98/psutil-4.2.0-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp33-cp33m-win32.whl","hashes":{"sha256":"2e16f792deceb1d33320981aaff7f139561cf6195ee3f1b21256d7f214162517"},"provenance":null,"requires-python":null,"size":165259,"upload-time":"2016-05-15T06:37:12.236981Z","url":"https://files.pythonhosted.org/packages/dd/94/8aeb332d07530b552099eaf207db13d859e09facfa8162892b4f9ef302dd/psutil-4.2.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"3c57a6731b3bd4c4af834b0137493a388b76192f5adc2399825015b777e0b02b"},"provenance":null,"requires-python":null,"size":167667,"upload-time":"2016-05-15T06:37:17.736147Z","url":"https://files.pythonhosted.org/packages/d5/6b/c10a228ef2cdbc077171be3b273cd2f49e4f814bf7dc2deb3a464cc126de/psutil-4.2.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp34-cp34m-win32.whl","hashes":{"sha256":"0cda72a1efacd2b028c9dbf0731111041e6cf9e7be938162811ab32ab3e88254"},"provenance":null,"requires-python":null,"size":165272,"upload-time":"2016-05-15T06:37:23.488252Z","url":"https://files.pythonhosted.org/packages/35/3e/3db756e014fe3e6e22e35c8394057dcf1eef58076d84fdf83ac00a053182/psutil-4.2.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"ce208e1c416e143697a1ee9dd86ae9720c740c11764a1fda88eb28a2ecc0b510"},"provenance":null,"requires-python":null,"size":167627,"upload-time":"2016-05-15T06:37:28.596251Z","url":"https://files.pythonhosted.org/packages/c7/7d/cfb299960cf6923cca782f331b034c09239e9015dedf530dd206177dd6e4/psutil-4.2.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp35-cp35m-win32.whl","hashes":{"sha256":"375b0acad448e49c8bc62e036f948af610b4e0cbe2a9a28eebc06357f20f67ea"},"provenance":null,"requires-python":null,"size":167370,"upload-time":"2016-05-15T06:37:34.104073Z","url":"https://files.pythonhosted.org/packages/4d/bc/f49882e8935f147b8922fc8bb0f430fe0e7b0d3231a601cd12e1c0272f77/psutil-4.2.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"data-dist-info-metadata":{"sha256":"467235151a7ca9506acf3ca52432d5a215f42e2fe44c043d0a0068e133a957dd"},"filename":"psutil-4.2.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"bd4b535996d06728b50bc7cd8777c402bf7294ad05229c843701bd1e63583c2c"},"provenance":null,"requires-python":null,"size":170689,"upload-time":"2016-05-15T06:37:39.395642Z","url":"https://files.pythonhosted.org/packages/7b/e2/2e1078a38189d51409f50af50b598309a2bd84ebe8ca71b79515da915c82/psutil-4.2.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.tar.gz","hashes":{"sha256":"544f013a0aea7199e07e3efe5627f5d4165179a04c66050b234cc3be2eca1ace"},"provenance":null,"requires-python":null,"size":311767,"upload-time":"2016-05-15T06:35:49.367304Z","url":"https://files.pythonhosted.org/packages/a6/bf/5ce23dc9f50de662af3b4bf54812438c298634224924c4e18b7c3b57a2aa/psutil-4.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"1329160e09a86029ef4e07f47dbcc39d511c343257a53acf1af429c537caae57"},"provenance":null,"requires-python":null,"size":406095,"upload-time":"2016-05-15T06:37:44.994723Z","url":"https://files.pythonhosted.org/packages/6b/af/9e43a4a4976f1d1291de8be40c848c591c6e48d7e4053a7b26ad88ba750c/psutil-4.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win-amd64-py3.3.exe","hashes":{"sha256":"12623a1e2e264eac8c899b89d78648e241c12eec754a879453b2e0a4a78b10dd"},"provenance":null,"requires-python":null,"size":404495,"upload-time":"2016-05-15T06:37:50.698273Z","url":"https://files.pythonhosted.org/packages/36/85/64244b1e930aa276205f079ba3e2996e8492bd173af019bbdaee47336a6a/psutil-4.2.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win-amd64-py3.4.exe","hashes":{"sha256":"66a4a7793dc543a3c7413cda3187e3ced45acf302f95c4d596ebcfc663c01b40"},"provenance":null,"requires-python":null,"size":404456,"upload-time":"2016-05-15T06:37:56.103096Z","url":"https://files.pythonhosted.org/packages/30/fa/a734058699f351ef90b757e0fd8d67a6145c8272bbed85c498276acacd2e/psutil-4.2.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win-amd64-py3.5.exe","hashes":{"sha256":"c2b7aa0a99b06967fb76e83e7e9c7153a2d9a5df073986a99a4e9656cfaabe28"},"provenance":null,"requires-python":null,"size":321489,"upload-time":"2016-05-15T06:38:02.113109Z","url":"https://files.pythonhosted.org/packages/a9/1b/c90c802a6db438aeebac412ac3ecf389022f4fd93abef9c0441358f46a71/psutil-4.2.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win32-py2.7.exe","hashes":{"sha256":"856480ce003ecd1601bcb83d97e25bfe79f5b08c430ee9f139a5e768173b06ef"},"provenance":null,"requires-python":null,"size":375917,"upload-time":"2016-05-15T06:38:09.945181Z","url":"https://files.pythonhosted.org/packages/e4/63/267b0977027c8a4a2f98a1ffbc2ecc7c0689d12adabee591a1ac99b4c14e/psutil-4.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win32-py3.3.exe","hashes":{"sha256":"ab83fefffa495813d36300cd3ad3f232cf7c86a5e5a02d8e8ea7ab7dba5a1a90"},"provenance":null,"requires-python":null,"size":370860,"upload-time":"2016-05-15T06:38:16.360218Z","url":"https://files.pythonhosted.org/packages/c0/96/8197557cbebb16be1cfd3c87f1d0972bd2e5b0733b21d0e5d890541634e8/psutil-4.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win32-py3.4.exe","hashes":{"sha256":"44c9f0e26b93c2cc9437eb88c31df32bd4337c394a959e0c31bf006da6e0f073"},"provenance":null,"requires-python":null,"size":370872,"upload-time":"2016-05-15T06:38:22.312293Z","url":"https://files.pythonhosted.org/packages/22/c9/01646a50e3c52dda4b591aae411e85afc83952107f9906ec8a5806c9fcc0/psutil-4.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.2.0.win32-py3.5.exe","hashes":{"sha256":"c0013a6663b794fbe18284e06d4d553a9e2135b5489a2ac6982ad53641966a55"},"provenance":null,"requires-python":null,"size":311007,"upload-time":"2016-05-15T06:38:28.533757Z","url":"https://files.pythonhosted.org/packages/48/6f/2259000fa07a4efcdc843034810eb2f8675ef5b97845912f2265589483f4/psutil-4.2.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"e4d7d684395672838b7bb0e9173a41bda97a9a103de6473eecda8beed8de23f6"},"data-dist-info-metadata":{"sha256":"e4d7d684395672838b7bb0e9173a41bda97a9a103de6473eecda8beed8de23f6"},"filename":"psutil-4.3.0-cp27-none-win32.whl","hashes":{"sha256":"99c2ab6c8f0d60e0c86775f8e5844e266af48cc1d9ecd1be209cd407a3e9c9a1"},"provenance":null,"requires-python":null,"size":167400,"upload-time":"2016-06-18T17:56:45.275403Z","url":"https://files.pythonhosted.org/packages/7a/a5/002caeac2ff88526cf0788315bad93be61e66477acd54209fb01cc874745/psutil-4.3.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e4d7d684395672838b7bb0e9173a41bda97a9a103de6473eecda8beed8de23f6"},"data-dist-info-metadata":{"sha256":"e4d7d684395672838b7bb0e9173a41bda97a9a103de6473eecda8beed8de23f6"},"filename":"psutil-4.3.0-cp27-none-win_amd64.whl","hashes":{"sha256":"a91474d34bf1bc86a0d95e2c198a70723208f9dc9e50258c2060a1bab3796f81"},"provenance":null,"requires-python":null,"size":169895,"upload-time":"2016-06-18T17:56:51.386191Z","url":"https://files.pythonhosted.org/packages/e1/77/fae92fff4ca7092555c7c8fc6e02c5dfc2a9af7e15762b7354d436adeb06/psutil-4.3.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp33-cp33m-win32.whl","hashes":{"sha256":"c987f0691c01cbe81813b0c895208c474240c96e26f7d1e945e8dabee5c85437"},"provenance":null,"requires-python":null,"size":167414,"upload-time":"2016-06-18T17:56:57.419709Z","url":"https://files.pythonhosted.org/packages/73/46/224a6a3c05df4e282240826f6bef0bd51da5ca283ffe34ed53f5601fbca1/psutil-4.3.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"233a943d3e6636d648f05515a4d21b4dda63499d8ca38d6890a57a3f78a9cceb"},"provenance":null,"requires-python":null,"size":169791,"upload-time":"2016-06-18T17:57:03.096187Z","url":"https://files.pythonhosted.org/packages/6b/4d/29b3d73c27cd8f348bb313376dd98a2e97cc8a075862cf483dea4c27e4bf/psutil-4.3.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp34-cp34m-win32.whl","hashes":{"sha256":"1893fe42b0fb5f11bf84ffe770be5b2e27fb7ec959ba8bf620b704552b738c72"},"provenance":null,"requires-python":null,"size":167396,"upload-time":"2016-06-18T17:57:09.859528Z","url":"https://files.pythonhosted.org/packages/dc/f5/34070d328a578c38131d96c4fe539ebbabf1c31128012755e344c030bb34/psutil-4.3.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"1345217075dd5bb4fecbf7cb0fe4c0c170e93ec57d48494756f4b617cd21b449"},"provenance":null,"requires-python":null,"size":169759,"upload-time":"2016-06-18T17:57:15.923623Z","url":"https://files.pythonhosted.org/packages/8d/91/7ae0835ae1a4bc1043b565dec9c6468d56c80a4f199472a7d005ddcd48e1/psutil-4.3.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp35-cp35m-win32.whl","hashes":{"sha256":"c9c4274f5f95a171437c90f65c3e9b71a871753f0a827f930e1b14aa43041eab"},"provenance":null,"requires-python":null,"size":169508,"upload-time":"2016-06-18T17:57:21.072109Z","url":"https://files.pythonhosted.org/packages/e4/40/801cd906da337a5e7a0afaaa1ce5919d9834c50d804d31ee4a1d2120a51c/psutil-4.3.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"data-dist-info-metadata":{"sha256":"21e625db3c2ab1770665e4dd336de5c07e12cfa6636d671163b7e4021a1a4341"},"filename":"psutil-4.3.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"5984ee7b2880abcdaa0819315f69a5f37da963863495c2294392cb3e98141a95"},"provenance":null,"requires-python":null,"size":172803,"upload-time":"2016-06-18T17:57:26.746663Z","url":"https://files.pythonhosted.org/packages/35/54/ddb6e8e583abf2f5a2be52a2223ba4935b382214b696fe334af54fb03dad/psutil-4.3.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.tar.gz","hashes":{"sha256":"86197ae5978f216d33bfff4383d5cc0b80f079d09cf45a2a406d1abb5d0299f0"},"provenance":null,"requires-python":null,"size":316470,"upload-time":"2016-06-18T17:54:55.929749Z","url":"https://files.pythonhosted.org/packages/22/a8/6ab3f0b3b74a36104785808ec874d24203c6a511ffd2732dd215cf32d689/psutil-4.3.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win-amd64-py2.7.exe","hashes":{"sha256":"1ad0075b6c86c0ea5076149ec39dcecf0c692711c34317d43d73a4d8c4d4ec30"},"provenance":null,"requires-python":null,"size":408495,"upload-time":"2016-06-18T17:57:33.655137Z","url":"https://files.pythonhosted.org/packages/d5/1f/638b17eab913d19203ebd721c4e5c726b57bc50def33ab1ec0070fe2ddc2/psutil-4.3.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win-amd64-py3.3.exe","hashes":{"sha256":"02c042e1f6c68c807de5caf45547971cf02977abf4cd92c8961186af9c91c488"},"provenance":null,"requires-python":null,"size":406900,"upload-time":"2016-06-18T17:57:40.580278Z","url":"https://files.pythonhosted.org/packages/a6/9b/94598ec4041ee2834f7ca21c9f1ff67e8f8cef4376ee9d9b9e3ff6950d05/psutil-4.3.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win-amd64-py3.4.exe","hashes":{"sha256":"2bcfeff969f3718bb31effea752d92511f375547d337687db1bd99ccd85b7ad7"},"provenance":null,"requires-python":null,"size":406871,"upload-time":"2016-06-18T17:57:47.827133Z","url":"https://files.pythonhosted.org/packages/79/2a/abe407e5b594b2a7c0432f5605579d33ce21e0049e3dc2e5c4f37402f760/psutil-4.3.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win-amd64-py3.5.exe","hashes":{"sha256":"34736b91c9785ede9d859a79e28129390f609339014f904c276ad46d0a440730"},"provenance":null,"requires-python":null,"size":323883,"upload-time":"2016-06-18T17:57:54.043353Z","url":"https://files.pythonhosted.org/packages/14/5e/072b19d913b3fc1f86b871c2869d19db3b5fa50c2e9f4980ae8646e189e0/psutil-4.3.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win32-py2.7.exe","hashes":{"sha256":"da6ee62fb5ffda188c39aacb0499d401c131046e922ba53fb4b908937e771f94"},"provenance":null,"requires-python":null,"size":378354,"upload-time":"2016-06-18T17:58:00.735354Z","url":"https://files.pythonhosted.org/packages/7e/85/a60111c14eb80aa7a74e1a0086c1a1bbc62282df0c913730e558d16e6a8c/psutil-4.3.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win32-py3.3.exe","hashes":{"sha256":"6f36fa29aa9ca935405a3cebe03cfbc6dde27088f9ad3d9b2d3baa47d4b89914"},"provenance":null,"requires-python":null,"size":373297,"upload-time":"2016-06-18T17:58:08.504160Z","url":"https://files.pythonhosted.org/packages/8e/6e/744b98493947a11625a4c63adb9a148c69c2a6e0cf7ef2afd6212804df1d/psutil-4.3.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win32-py3.4.exe","hashes":{"sha256":"63b75c5374fafdaf3f389229b592f19b88a8c7951d1b973b9113df649ade5cb9"},"provenance":null,"requires-python":null,"size":373279,"upload-time":"2016-06-18T17:58:15.187200Z","url":"https://files.pythonhosted.org/packages/87/ce/a3cfd0e1b7d34fccff488a6a2283e5b947841465f81960bd87eee5f78828/psutil-4.3.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.0.win32-py3.5.exe","hashes":{"sha256":"6d0e1e9bbdab5a7d40b57d8f0841eb9e11c2395ac7ec0ddd396116240d733f46"},"provenance":null,"requires-python":null,"size":313430,"upload-time":"2016-06-18T17:58:22.261628Z","url":"https://files.pythonhosted.org/packages/eb/23/d01e7eccd76a096b0675ab9d07c8fd2de9fd31bc8f4c92f1a150bc612ad4/psutil-4.3.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"f2eb91a4c96ace2af993ac8587ddb20d3ed535289e9c5a0fde7a8c3d67ae0e12"},"data-dist-info-metadata":{"sha256":"f2eb91a4c96ace2af993ac8587ddb20d3ed535289e9c5a0fde7a8c3d67ae0e12"},"filename":"psutil-4.3.1-cp27-none-win32.whl","hashes":{"sha256":"b0c5bf0d2a29a6f18ac22e2d24210730dca458c9f961914289c9e027ccb5ae43"},"provenance":null,"requires-python":null,"size":168476,"upload-time":"2016-09-02T13:58:43.429910Z","url":"https://files.pythonhosted.org/packages/2d/70/0b24c7272efbb1d8cac1be1768aabfb8ddb37bdc9ab8a176f6afc7e52b0d/psutil-4.3.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"f2eb91a4c96ace2af993ac8587ddb20d3ed535289e9c5a0fde7a8c3d67ae0e12"},"data-dist-info-metadata":{"sha256":"f2eb91a4c96ace2af993ac8587ddb20d3ed535289e9c5a0fde7a8c3d67ae0e12"},"filename":"psutil-4.3.1-cp27-none-win_amd64.whl","hashes":{"sha256":"fc78c29075e623b6ea1c4a1620a120a1534ee05370b76c0ec96f6d161d79e7a1"},"provenance":null,"requires-python":null,"size":170725,"upload-time":"2016-09-02T13:58:47.944701Z","url":"https://files.pythonhosted.org/packages/67/d4/0403e6bd1cf78bd597ac960a3a6ad36cea6c12e3b413c0a1d43361128fb5/psutil-4.3.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp33-cp33m-win32.whl","hashes":{"sha256":"aa05f44a77ef83773af39446f99e461aa3b6edb7fdabeefdcf06e913d8884d3a"},"provenance":null,"requires-python":null,"size":168384,"upload-time":"2016-09-02T13:58:52.479199Z","url":"https://files.pythonhosted.org/packages/9c/ec/5f3f06012c54de3b4443a6948fc75fe1e348ca3de408b00815b5976b8877/psutil-4.3.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"6b3882eb16f2f40f1da6208a051800abadb1f82a675d9ef6ca7386e1a208b1ad"},"provenance":null,"requires-python":null,"size":170566,"upload-time":"2016-09-02T13:58:56.870812Z","url":"https://files.pythonhosted.org/packages/de/c6/74f8b4d460d89811b4c8fd426b63530222456bad767fe372ebb4f5f207be/psutil-4.3.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp34-cp34m-win32.whl","hashes":{"sha256":"cf1be0b16b38f0e2081ff0c81a1a4321c206a824ba6bd51903fdd440abb370b6"},"provenance":null,"requires-python":null,"size":168375,"upload-time":"2016-09-02T13:59:01.079445Z","url":"https://files.pythonhosted.org/packages/75/ff/d02c907869d5e4cc260ce72eb253f5007a7cdf0b47326d19693f8f937eb0/psutil-4.3.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"afa94bed972722882264a4df06176f6b6e6acc6bcebcc3f1db5428c7271dacba"},"provenance":null,"requires-python":null,"size":170529,"upload-time":"2016-09-02T13:59:05.507453Z","url":"https://files.pythonhosted.org/packages/c9/72/07da416b1dcf258a0cb0587c823e3611392fe29b1fcae6e078b1c254dce5/psutil-4.3.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp35-cp35m-win32.whl","hashes":{"sha256":"d2254f518624e6b2262f0f878931faa4bdbe8a77d1f8826564bc4576c6a4f85e"},"provenance":null,"requires-python":null,"size":170124,"upload-time":"2016-09-02T13:59:09.837050Z","url":"https://files.pythonhosted.org/packages/cd/b0/07b7083a134c43b58515d59f271734034f8ba06840b1f371eaa6b3ab85b2/psutil-4.3.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"data-dist-info-metadata":{"sha256":"2bf14b2dd8f24b0dedf17492f736d534043ed69d60a0cf1d5a8925a8a351061a"},"filename":"psutil-4.3.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"3b377bc8ba5e62adbc709a90ea07dce2d4addbd6e1cc7acede61ddfa1c66e00a"},"provenance":null,"requires-python":null,"size":173545,"upload-time":"2016-09-02T13:59:14.368908Z","url":"https://files.pythonhosted.org/packages/7e/7e/17c1467158ccac5dc54986a657420fc194686653cfb6feddb6717a60d17f/psutil-4.3.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.tar.gz","hashes":{"sha256":"38f74182fb9e15cafd0cdf0821098a95cc17301807aed25634a18b66537ba51b"},"provenance":null,"requires-python":null,"size":315878,"upload-time":"2016-09-01T20:56:06.777431Z","url":"https://files.pythonhosted.org/packages/78/cc/f267a1371f229bf16db6a4e604428c3b032b823b83155bd33cef45e49a53/psutil-4.3.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win-amd64-py2.7.exe","hashes":{"sha256":"733210f39e95744da26f2256bc36035fc463b0ae88e91496e97486ba21c63cab"},"provenance":null,"requires-python":null,"size":408715,"upload-time":"2016-09-01T21:31:30.452539Z","url":"https://files.pythonhosted.org/packages/57/44/09ec7b6a3fc1216ecc2182a64f8b6c2eaf8dd0d983fb98ecbf9cecbf54b4/psutil-4.3.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win-amd64-py3.3.exe","hashes":{"sha256":"4690f720054beff4fc66551a6a34512faff328588dca8e2dbed94398b6941112"},"provenance":null,"requires-python":null,"size":407063,"upload-time":"2016-09-01T21:31:37.405818Z","url":"https://files.pythonhosted.org/packages/f2/b5/2921abbf7779d2f7bb210aa819dd2d86ecd004430c59aef9cc52c2d4057b/psutil-4.3.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win-amd64-py3.4.exe","hashes":{"sha256":"fd9b66edb9f8943eda6b39e7bb9bff8b14aa8d785f5b417d7a0bfa53d4781a7a"},"provenance":null,"requires-python":null,"size":407028,"upload-time":"2016-09-01T21:31:44.758391Z","url":"https://files.pythonhosted.org/packages/91/ea/d5fb1ef4615c0febab3a347ffbc98d50177878546c22b3291892f41de8f4/psutil-4.3.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win-amd64-py3.5.exe","hashes":{"sha256":"9ab5b62c6571ce545b1c40b9740af81276bd5d94439fd54de07ed59be0ce3f4f"},"provenance":null,"requires-python":null,"size":777648,"upload-time":"2016-09-01T21:31:56.074229Z","url":"https://files.pythonhosted.org/packages/0d/d9/626030b223140d88176b5dda42ea081b843249af11fa71d9e36a465b4b18/psutil-4.3.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win32-py2.7.exe","hashes":{"sha256":"ad8857923e9bc5802d5559ab5d70c1abc1a7be8e74e779adde883c5391e2061c"},"provenance":null,"requires-python":null,"size":378819,"upload-time":"2016-09-01T21:30:59.795815Z","url":"https://files.pythonhosted.org/packages/02/99/2cc2981b30b2b1fb5b393c921ad17e4baaa1f95d7de527fe54ee222e5663/psutil-4.3.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win32-py3.3.exe","hashes":{"sha256":"ae20b76cddb3391ea37de5d2aaa1656d6373161bbc8fd868a0ca055194a46e45"},"provenance":null,"requires-python":null,"size":373653,"upload-time":"2016-09-01T21:31:06.076800Z","url":"https://files.pythonhosted.org/packages/06/2f/bbcae933425945d2c9ca3e30e2f35728827e87cedad113808d3f527fe2ea/psutil-4.3.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win32-py3.4.exe","hashes":{"sha256":"0613437cc28b8721de92c582d5baf742dfa6dd824c84b578f8c49a60077e969a"},"provenance":null,"requires-python":null,"size":373650,"upload-time":"2016-09-01T21:31:12.717399Z","url":"https://files.pythonhosted.org/packages/fb/a6/28bca55c499426202341f8224f66fcd69e6577211201b02605be779f44b1/psutil-4.3.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.3.1.win32-py3.5.exe","hashes":{"sha256":"c2031732cd0fb7536af491bb8d8119c9263020a52450f9999c884fd49d346b26"},"provenance":null,"requires-python":null,"size":644701,"upload-time":"2016-09-01T21:31:22.237380Z","url":"https://files.pythonhosted.org/packages/f3/c7/f5da6e76f69c97ecfabe684f4d9e179ceaf9f7c1a74d6148480b1a6c41a8/psutil-4.3.1.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"73b13e96cdf0d35ce81d30eb0c2e6b9cbfea41ee45b898f54d9e832e8e3ddd92"},"data-dist-info-metadata":{"sha256":"73b13e96cdf0d35ce81d30eb0c2e6b9cbfea41ee45b898f54d9e832e8e3ddd92"},"filename":"psutil-4.4.0-cp27-none-win32.whl","hashes":{"sha256":"4b907a0bed62a76422eae4e1ed8c8eca25fc21e57a31fc080158b8a300e21dad"},"provenance":null,"requires-python":null,"size":172746,"upload-time":"2016-10-23T14:08:37.739179Z","url":"https://files.pythonhosted.org/packages/8b/00/f7203827a3b2576bf8162ea3ba41e39c3307651218ab08e22e1433d8dc36/psutil-4.4.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"73b13e96cdf0d35ce81d30eb0c2e6b9cbfea41ee45b898f54d9e832e8e3ddd92"},"data-dist-info-metadata":{"sha256":"73b13e96cdf0d35ce81d30eb0c2e6b9cbfea41ee45b898f54d9e832e8e3ddd92"},"filename":"psutil-4.4.0-cp27-none-win_amd64.whl","hashes":{"sha256":"f60ab95f5e65c420743d5dd5285bda2a6bba6712e9380fb9a5903ea539507326"},"provenance":null,"requires-python":null,"size":175102,"upload-time":"2016-10-23T14:08:40.758084Z","url":"https://files.pythonhosted.org/packages/61/5c/d1f89100973829813e485e7bfc2f203e0d11937f958109819b62e8995b51/psutil-4.4.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp33-cp33m-win32.whl","hashes":{"sha256":"d45919d8b900a9ae03f3d43c489323842d4051cf7a728169f01a3889f50d24ad"},"provenance":null,"requires-python":null,"size":172612,"upload-time":"2016-10-23T14:08:44.422307Z","url":"https://files.pythonhosted.org/packages/34/16/ecc7366a7d023731b8b051cff64d3bb4bed2efa4947b125a3c9031e3d799/psutil-4.4.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"df706e4a8533b43c1a083c2c94e816c7a605487db49e5d49fc64329e0147c0f5"},"provenance":null,"requires-python":null,"size":174980,"upload-time":"2016-10-23T14:08:48.077823Z","url":"https://files.pythonhosted.org/packages/f7/17/6fc596007c243a12cf8c9e81b150f1e7a49c7c85c0544771398f257d5608/psutil-4.4.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp34-cp34m-win32.whl","hashes":{"sha256":"d119281d253bbcc44b491b7b7e5c38802f0933179af97ab228cfd8d072ad1503"},"provenance":null,"requires-python":null,"size":172623,"upload-time":"2016-10-23T14:08:50.867277Z","url":"https://files.pythonhosted.org/packages/50/8f/e8fcfb1bacd02c743545d58514004978318d8476c8bffeee9fecfc9f1f79/psutil-4.4.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"ff8a22a40bf884f52cf8dd86872b5199e7ed58eef1575e837d1d9b668fb2416a"},"provenance":null,"requires-python":null,"size":174974,"upload-time":"2016-10-23T14:08:53.576968Z","url":"https://files.pythonhosted.org/packages/41/8f/31f1fb4bb639ccaff2192b0b2a760118a05fb108e156d4966c81a246071b/psutil-4.4.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp35-cp35m-win32.whl","hashes":{"sha256":"7b7c850b3afe6c7895cfd3ee630492c4aabe644a6723a583cd56bac0222d7cd8"},"provenance":null,"requires-python":null,"size":174516,"upload-time":"2016-10-23T14:08:56.521588Z","url":"https://files.pythonhosted.org/packages/18/9e/702b1da450a13448f9d96f2d9e2f69eed901c56adb01e173d8307278c883/psutil-4.4.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"data-dist-info-metadata":{"sha256":"ee9b7c8ec564a66dc6543830d64d32a37387ea58517feba35aa4eefeb15ce5de"},"filename":"psutil-4.4.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"321d09e39bb4641c98544e51fba598f894d09355a18d14367468723632cb149e"},"provenance":null,"requires-python":null,"size":178046,"upload-time":"2016-10-23T14:08:59.346105Z","url":"https://files.pythonhosted.org/packages/8a/87/a22b95d91fb1a07591ebf373d1c286a16037bf2e321577a083156d7f6a13/psutil-4.4.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.tar.gz","hashes":{"sha256":"f4da111f473dbf7e813e6610aec1329000536aea5e7d7e73ed20bc42cfda7ecc"},"provenance":null,"requires-python":null,"size":1831734,"upload-time":"2016-10-23T14:09:04.179623Z","url":"https://files.pythonhosted.org/packages/fc/63/af9c6a4f2ab48293f60ec204cc1336f6f17c1cb782ffb0275982ac08d663/psutil-4.4.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win-amd64-py2.7.exe","hashes":{"sha256":"184132916ceb845f12f7cced3a2cf5273097d314f44be8357bdc435a6dee49cd"},"provenance":null,"requires-python":null,"size":413868,"upload-time":"2016-10-23T14:09:10.236158Z","url":"https://files.pythonhosted.org/packages/98/d9/407ee7d17b4e78c45b92639bfbd9b376b6a6022b0c63965946bfd74af6f4/psutil-4.4.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win-amd64-py3.3.exe","hashes":{"sha256":"7bb1c69a062aaff4a1773cca6bbe33b261293318d72246b9519357260b67be1e"},"provenance":null,"requires-python":null,"size":412259,"upload-time":"2016-10-23T14:09:15.072697Z","url":"https://files.pythonhosted.org/packages/3f/cc/b31fbc28247722ae7ece86f8911459e29a08f5075e3d4e794cef20f920b2/psutil-4.4.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win-amd64-py3.4.exe","hashes":{"sha256":"887eca39665d3de362693d66db9c15bf93313cde261de93908452241ee439e56"},"provenance":null,"requires-python":null,"size":412252,"upload-time":"2016-10-23T14:09:19.006281Z","url":"https://files.pythonhosted.org/packages/32/72/88e9e57964144ac3868f817f77e269f987c74fa51e503eb0fe8040523d28/psutil-4.4.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win-amd64-py3.5.exe","hashes":{"sha256":"a289ed4f7be6c535aa4c59c997baf327788d564881c8bf08fee502fab0cdc7cb"},"provenance":null,"requires-python":null,"size":782926,"upload-time":"2016-10-23T14:09:24.493051Z","url":"https://files.pythonhosted.org/packages/45/09/8908f5931a78c90bf2ef7f91a37a752a13d4017347ac2765d0cbb89e82f6/psutil-4.4.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win32-py2.7.exe","hashes":{"sha256":"8a18d0527ac339f5501f0bd5471040d5fb83052453eedf13106feaf99ddef6cb"},"provenance":null,"requires-python":null,"size":383869,"upload-time":"2016-10-23T14:09:28.382095Z","url":"https://files.pythonhosted.org/packages/4a/eb/a131c438621822833e3131c4998741c3b68fb6ff67786d25e9398439a640/psutil-4.4.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win32-py3.3.exe","hashes":{"sha256":"8700a0373fd0c95c79ad79bc08e0a7b76cabc0c463658cbe398600c69d57293d"},"provenance":null,"requires-python":null,"size":378663,"upload-time":"2016-10-23T14:09:32.358639Z","url":"https://files.pythonhosted.org/packages/be/c8/fd76b7ba3e1cc61d586c5779d9b0a45d1ad020e1de677eb344d2583bd0f1/psutil-4.4.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win32-py3.4.exe","hashes":{"sha256":"ddd3f76486a366f1dc7adc8a9e8a285ec24b3b213912b8dce0cbbb629f954b8c"},"provenance":null,"requires-python":null,"size":378674,"upload-time":"2016-10-23T14:09:35.849351Z","url":"https://files.pythonhosted.org/packages/ce/0c/efbe5aefdd136bb3e2fc292b0506761a9b04c215d295be07e11263a90923/psutil-4.4.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.0.win32-py3.5.exe","hashes":{"sha256":"28963853709f6e0958edd8d6d7469152098680800deb98a759a67745d65b164b"},"provenance":null,"requires-python":null,"size":649869,"upload-time":"2016-10-23T14:09:40.045794Z","url":"https://files.pythonhosted.org/packages/fa/b6/b97b6d4dcbc87b91db9fd2753a0c3b999b734595a97155a38003be475a6d/psutil-4.4.0.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"b5aa70030cb34744d7b8b2639183dd13ebd01c60f592c3b0c5e6fd6d8086bcd3"},"data-dist-info-metadata":{"sha256":"b5aa70030cb34744d7b8b2639183dd13ebd01c60f592c3b0c5e6fd6d8086bcd3"},"filename":"psutil-4.4.1-cp27-none-win32.whl","hashes":{"sha256":"7e77ec1a9c75a858781c1fb46fe81c999e1ae0e711198b4aaf59e5f5bd373b11"},"provenance":null,"requires-python":null,"size":172150,"upload-time":"2016-10-25T15:16:10.656877Z","url":"https://files.pythonhosted.org/packages/a9/aa/ff4e5602420cda42e9ff12949eae95abf7d6838fc79d7892b29af416f4c2/psutil-4.4.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"b5aa70030cb34744d7b8b2639183dd13ebd01c60f592c3b0c5e6fd6d8086bcd3"},"data-dist-info-metadata":{"sha256":"b5aa70030cb34744d7b8b2639183dd13ebd01c60f592c3b0c5e6fd6d8086bcd3"},"filename":"psutil-4.4.1-cp27-none-win_amd64.whl","hashes":{"sha256":"0f7f830db35c1baeb0131b2bba458b77f7db98944b2fedafc34922168e467d09"},"provenance":null,"requires-python":null,"size":174507,"upload-time":"2016-10-25T15:16:16.530462Z","url":"https://files.pythonhosted.org/packages/62/e8/c3d3e4161e29bd86d6b06d16456defcb114fd74693b15e5692df9e2b611e/psutil-4.4.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp33-cp33m-win32.whl","hashes":{"sha256":"fa4ad0533adef033bbcbac5e20d06e77f9aadf5d9c1596317d1b668f94b01b99"},"provenance":null,"requires-python":null,"size":172015,"upload-time":"2016-10-25T15:16:19.320410Z","url":"https://files.pythonhosted.org/packages/2c/da/0428f61f2c5eda7c5053ae35a2d57b097e4eb15e48f9d222319df7f4cbd3/psutil-4.4.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"af337d186b07249b86f00a71b4cf6fcfa1964484fe5fb8a7b623f4559c2859c9"},"provenance":null,"requires-python":null,"size":174384,"upload-time":"2016-10-25T15:16:22.634310Z","url":"https://files.pythonhosted.org/packages/da/ce/d36c39da6d387fbfaea68f382319887315e9201e885ef93c418b89209b31/psutil-4.4.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp34-cp34m-win32.whl","hashes":{"sha256":"4e5cb45c9616dd855c07e538f523c838705ce7c24e021e645cdce4c7894e7209"},"provenance":null,"requires-python":null,"size":172026,"upload-time":"2016-10-25T15:16:28.801885Z","url":"https://files.pythonhosted.org/packages/64/63/49aeca2d1a20f5c5203302a25c825be9262f81d8bf74d7d9e0bf0789b189/psutil-4.4.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"0876646748f3db5e1678a94ae3f68dcef3bd51e82b34f06109e6a28bcddc266c"},"provenance":null,"requires-python":null,"size":174374,"upload-time":"2016-10-25T15:16:32.462229Z","url":"https://files.pythonhosted.org/packages/e0/b2/48a62d3204a714b6354105ed540d54d1e272d5b230bc35eddc14a1494dd1/psutil-4.4.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp35-cp35m-win32.whl","hashes":{"sha256":"e4e9033ef5d775ef8a522750688161241e79c7d4669a05784a0a1a8d37dc8c3c"},"provenance":null,"requires-python":null,"size":173920,"upload-time":"2016-10-25T15:16:38.218083Z","url":"https://files.pythonhosted.org/packages/95/e6/d6bad966efeb674bb3357c94f3e89bbd3477a24f2635c3cadc1d0f2aea76/psutil-4.4.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"data-dist-info-metadata":{"sha256":"979b8a1231ad076b4475296c0795b1310188d8e8fda89e2785b4da3efbf4ad52"},"filename":"psutil-4.4.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"d8464100c62932eeeacce2be0a1041f68b3bfcc7be261cf9486c11ee98eaedd2"},"provenance":null,"requires-python":null,"size":177448,"upload-time":"2016-10-25T15:16:42.121191Z","url":"https://files.pythonhosted.org/packages/00/58/6845a2a2f6fbbc56a58f0605744368164bf68889299f134684cc1478baf6/psutil-4.4.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.tar.gz","hashes":{"sha256":"9da43dbf7c08f5c2a8e5e2c8792f5c438f52435677b1334e9653d23ea028f4f7"},"provenance":null,"requires-python":null,"size":1831794,"upload-time":"2016-10-25T15:16:47.508715Z","url":"https://files.pythonhosted.org/packages/e5/f3/b816daefa9a6757f867f81903f40849dcf0887f588793236b476e6a30ded/psutil-4.4.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win-amd64-py2.7.exe","hashes":{"sha256":"23ac3e7b6784751ceeb4338d54b0e683c955cd22b86acbc089696aeb0717ab75"},"provenance":null,"requires-python":null,"size":408863,"upload-time":"2016-10-25T15:16:52.293269Z","url":"https://files.pythonhosted.org/packages/88/93/3c8434bd64a5f89f6b0cc58e1394e88dc3c272f83f1877afac9ede222c63/psutil-4.4.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win-amd64-py3.3.exe","hashes":{"sha256":"8b97ece77d2dce49dd6adbfc0c9bfb7820be4460d00732bb8cf18b77b9ffb07f"},"provenance":null,"requires-python":null,"size":407253,"upload-time":"2016-10-25T15:16:57.006499Z","url":"https://files.pythonhosted.org/packages/50/21/6b462342ab2f091bed70898894322c107493d69e24a834b9f2ab93a1e876/psutil-4.4.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win-amd64-py3.4.exe","hashes":{"sha256":"a0b5976a5b5a781754b6cebe89f4e21413b04b7f4005f5713f65257bee29be5f"},"provenance":null,"requires-python":null,"size":407244,"upload-time":"2016-10-25T15:17:01.612657Z","url":"https://files.pythonhosted.org/packages/09/94/a7c3c875884adef49f52c19faff68bfd2b04160a5eb0489dbffa60b5c2dc/psutil-4.4.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win-amd64-py3.5.exe","hashes":{"sha256":"b0bfe16b9bd095e56b8ec481328ba64094ccd615ee1f789a12a13cfd7bc5e34a"},"provenance":null,"requires-python":null,"size":777920,"upload-time":"2016-10-25T15:17:05.774786Z","url":"https://files.pythonhosted.org/packages/09/bf/3d850244db8e6aa94116db6787edee352692bbb9647e31c056189e6742cc/psutil-4.4.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win32-py2.7.exe","hashes":{"sha256":"392877b5f86977b4a747fb61968e3b44df62c1d6d87536f021c4979b601c68f5"},"provenance":null,"requires-python":null,"size":378863,"upload-time":"2016-10-25T15:17:10.258103Z","url":"https://files.pythonhosted.org/packages/ed/f9/ed4cc19c84086149067d03de7cfb9f3213cf02d58b6f75c2ac16f85e0baf/psutil-4.4.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win32-py3.3.exe","hashes":{"sha256":"a571f179f29215f0aa710ec495573a3522f24e8a8ba0be67d094ad6f593fba05"},"provenance":null,"requires-python":null,"size":373657,"upload-time":"2016-10-25T15:17:17.117641Z","url":"https://files.pythonhosted.org/packages/e6/ad/d8ec058821191334cf17db08c32c0308252d7b3c7490417690061dd12096/psutil-4.4.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win32-py3.4.exe","hashes":{"sha256":"c2614dc6194cafd74c147e4b95febe8778430c2ecb91197ad140bf45e3e3ada7"},"provenance":null,"requires-python":null,"size":373668,"upload-time":"2016-10-25T15:17:20.392896Z","url":"https://files.pythonhosted.org/packages/26/15/6d03a2171262a1be565d06c87d584c10daa51e787c7cdd2ff932a433419e/psutil-4.4.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.1.win32-py3.5.exe","hashes":{"sha256":"5915da802a1648d0baeeee06b0d1a87eb0e4e20654f65b8428a6f4cc8cd32caf"},"provenance":null,"requires-python":null,"size":644863,"upload-time":"2016-10-25T15:17:24.141533Z","url":"https://files.pythonhosted.org/packages/d1/91/89db11eaa91c1de34d77600c47d657fce7b60e847bcf6457dddfc23b387c/psutil-4.4.1.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"5a3e00c7b872d8d4c7c640b7cfec9ede44850d18f5d050b57037614e2ec90c72"},"data-dist-info-metadata":{"sha256":"5a3e00c7b872d8d4c7c640b7cfec9ede44850d18f5d050b57037614e2ec90c72"},"filename":"psutil-4.4.2-cp27-none-win32.whl","hashes":{"sha256":"15aba78f0262d7839702913f5d2ce1e97c89e31456bb26da1a5f9f7d7fe6d336"},"provenance":null,"requires-python":null,"size":172169,"upload-time":"2016-10-26T11:01:25.916580Z","url":"https://files.pythonhosted.org/packages/d7/da/b7895f01868977c9205bf1c8ff6a88290ec443535084e206b7db7f33918f/psutil-4.4.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"5a3e00c7b872d8d4c7c640b7cfec9ede44850d18f5d050b57037614e2ec90c72"},"data-dist-info-metadata":{"sha256":"5a3e00c7b872d8d4c7c640b7cfec9ede44850d18f5d050b57037614e2ec90c72"},"filename":"psutil-4.4.2-cp27-none-win_amd64.whl","hashes":{"sha256":"69e30d789c495b781f7cd47c13ee64452c58abfc7132d6dd1b389af312a78239"},"provenance":null,"requires-python":null,"size":174525,"upload-time":"2016-10-26T11:01:29.001553Z","url":"https://files.pythonhosted.org/packages/0c/76/f50742570195a9a13e26ee3e3e32575a9315df90408056c991af85d23792/psutil-4.4.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp33-cp33m-win32.whl","hashes":{"sha256":"e44d6b758a96539e3e02336430d3f85263d43c470c5bad93572e9b6a86c67f76"},"provenance":null,"requires-python":null,"size":172034,"upload-time":"2016-10-26T11:01:31.931360Z","url":"https://files.pythonhosted.org/packages/f1/58/64db332e3a665d0d0260fcfe39ff8609871a1203ed11afcff0e5991db724/psutil-4.4.2-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"c2b0d8d1d8b5669b9884d0dd49ccb4094d163858d672d3d13a3fa817bc8a3197"},"provenance":null,"requires-python":null,"size":174401,"upload-time":"2016-10-26T11:01:35.879864Z","url":"https://files.pythonhosted.org/packages/39/31/4b5a2ab9d20d14fa6c821b7fbbf69ae93708e1a4ba335d4e2ead549b2f59/psutil-4.4.2-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp34-cp34m-win32.whl","hashes":{"sha256":"10fbb631142a3200623f4ab49f8bf82c32b79b8fe179f6056d01da3dfc589da1"},"provenance":null,"requires-python":null,"size":172044,"upload-time":"2016-10-26T11:01:38.950793Z","url":"https://files.pythonhosted.org/packages/67/f6/f034ad76faa7b9bd24d70a663b5dca8d71b6b7ddb4a3ceecd6a0a63afe92/psutil-4.4.2-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"e423dd9cb12256c742d1d56ec38bc7d2a7fa09287c82c41e475e68b9f932c2af"},"provenance":null,"requires-python":null,"size":174393,"upload-time":"2016-10-26T11:01:41.933150Z","url":"https://files.pythonhosted.org/packages/f8/f2/df2d8de978236067d0fce847e3734c6e78bfd57df9db5a2786cc544ca85f/psutil-4.4.2-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp35-cp35m-win32.whl","hashes":{"sha256":"7481f299ae0e966a10cb8dd93a327efd8f51995d9bdc8810dcc65d3b12d856ee"},"provenance":null,"requires-python":null,"size":173938,"upload-time":"2016-10-26T11:01:44.870925Z","url":"https://files.pythonhosted.org/packages/57/2d/0cdc3fdb9853ec1d815e0948104f1c51894f47c8ee2152c019ba754ce678/psutil-4.4.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"data-dist-info-metadata":{"sha256":"34c5de5faa22e5cdae426206704ec3a7d177ec1d34a20e3ad2d072136ac17b84"},"filename":"psutil-4.4.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"d96d31d83781c7f3d0df8ccb1cc50650ca84d4722c5070b71ce8f1cc112e02e0"},"provenance":null,"requires-python":null,"size":177465,"upload-time":"2016-10-26T11:01:47.925426Z","url":"https://files.pythonhosted.org/packages/cd/50/3c238d67e025701fa18cc37ed445327efcf14ea4d06e357eef92d1a250bf/psutil-4.4.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.tar.gz","hashes":{"sha256":"1c37e6428f7fe3aeea607f9249986d9bb933bb98133c7919837fd9aac4996b07"},"provenance":null,"requires-python":null,"size":1832052,"upload-time":"2016-10-26T11:01:53.029413Z","url":"https://files.pythonhosted.org/packages/6c/49/0f784a247868e167389f6ac76b8699b2f3d6f4e8e85685dfec43e58d1ed1/psutil-4.4.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win-amd64-py2.7.exe","hashes":{"sha256":"11a20c0328206dce68f8da771461aeaef9c44811e639216fd935837e758632dc"},"provenance":null,"requires-python":null,"size":408884,"upload-time":"2016-10-26T11:01:57.830427Z","url":"https://files.pythonhosted.org/packages/58/bc/ac2a052035e945153f0bdc23d7c169e7e64e0729af6ff96f85d960962def/psutil-4.4.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win-amd64-py3.3.exe","hashes":{"sha256":"642194ebefa573de62406883eb33868917bab2cc2e21b68d551248e194dd0b0a"},"provenance":null,"requires-python":null,"size":407275,"upload-time":"2016-10-26T11:02:00.968973Z","url":"https://files.pythonhosted.org/packages/38/b3/c8244a4858de5f9525f176d72cb06584433446a5b9d7272d5eec7a435f81/psutil-4.4.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win-amd64-py3.4.exe","hashes":{"sha256":"c02b9fb5f1f3c857938b26a73b1ca92007e8b0b2fd64693b29300fae0ceaf679"},"provenance":null,"requires-python":null,"size":407267,"upload-time":"2016-10-26T11:02:04.291434Z","url":"https://files.pythonhosted.org/packages/cd/4d/8f447ecba1a3d5cae9f3bdd8395b45e2e6d07834a05fa3ce58c766ffcea6/psutil-4.4.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win-amd64-py3.5.exe","hashes":{"sha256":"6c40dc16b579f645e1804341322364203d0b21045747e62e360fae843d945e20"},"provenance":null,"requires-python":null,"size":777940,"upload-time":"2016-10-26T11:02:10.124058Z","url":"https://files.pythonhosted.org/packages/73/6e/1c58cbdedcf4e2226f8ca182c947e29f29dba6965fdae0aa8e6a361365ce/psutil-4.4.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win32-py2.7.exe","hashes":{"sha256":"c353ecc62e67bf7c7051c087670d49eae9472f1b30bb1623d667b0cd137e8934"},"provenance":null,"requires-python":null,"size":378885,"upload-time":"2016-10-26T11:02:16.586357Z","url":"https://files.pythonhosted.org/packages/37/27/9eba3e29f8291be103a8e5e44c06eefe5357217e9d8d294ff154dc3bff90/psutil-4.4.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win32-py3.3.exe","hashes":{"sha256":"7106cb3722235ccb6fe4b18c51f60a548d4b111ec2d209abdcd3998661f4593a"},"provenance":null,"requires-python":null,"size":373678,"upload-time":"2016-10-26T11:02:20.035049Z","url":"https://files.pythonhosted.org/packages/62/25/189e73856fa3172d5e785404eb9cf4561b01094363cc39ec267399e401c4/psutil-4.4.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win32-py3.4.exe","hashes":{"sha256":"de1f53fe955dfba562f7791f72517935010a2e88f9caad36917e8c5c03de9051"},"provenance":null,"requires-python":null,"size":373689,"upload-time":"2016-10-26T11:02:23.073976Z","url":"https://files.pythonhosted.org/packages/10/4a/e50ca35162538645c4d98816757763e4aaaac1ccaf055b39265acb8739e1/psutil-4.4.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-4.4.2.win32-py3.5.exe","hashes":{"sha256":"2eb123ca86057ed4f31cfc9880e098ee7a8e19c7ec02b068c45e7559ae7539a6"},"provenance":null,"requires-python":null,"size":644886,"upload-time":"2016-10-26T11:02:26.650470Z","url":"https://files.pythonhosted.org/packages/c5/97/018f0580bc3d3d85f1e084e683e904bc33cd58ca96e0ecb5ae6f346d6452/psutil-4.4.2.win32-py3.5.exe","yanked":false},{"core-metadata":{"sha256":"3db8fbdc9c0693e219a15e49b871dc421e85735214638b6e31d68aee5b78c550"},"data-dist-info-metadata":{"sha256":"3db8fbdc9c0693e219a15e49b871dc421e85735214638b6e31d68aee5b78c550"},"filename":"psutil-5.0.0-cp27-none-win32.whl","hashes":{"sha256":"cc2560b527cd88a9bc062ee4bd055c40b9fc107e37db01997422c75a3f94efe9"},"provenance":null,"requires-python":null,"size":175039,"upload-time":"2016-11-06T18:28:50.896397Z","url":"https://files.pythonhosted.org/packages/bb/82/efee0c83eab6ad3dd4a70b2b91013a50776ff6253f9b71d825914e693dfa/psutil-5.0.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"3db8fbdc9c0693e219a15e49b871dc421e85735214638b6e31d68aee5b78c550"},"data-dist-info-metadata":{"sha256":"3db8fbdc9c0693e219a15e49b871dc421e85735214638b6e31d68aee5b78c550"},"filename":"psutil-5.0.0-cp27-none-win_amd64.whl","hashes":{"sha256":"8a6cbc7165a476d08a89ee3078a74d111729cf515fd831db9f635012e56f9759"},"provenance":null,"requires-python":null,"size":177395,"upload-time":"2016-11-06T18:28:54.500906Z","url":"https://files.pythonhosted.org/packages/16/bf/09cb9f5286e5037eba1d5fe347c0f622b325e068757f1072c56f9b130cc3/psutil-5.0.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp33-cp33m-win32.whl","hashes":{"sha256":"9b0f13e325f007a0fd04e9d44cfdb5187c0b3e144f89533324dd9f74c25bd9ec"},"provenance":null,"requires-python":null,"size":174954,"upload-time":"2016-11-06T18:28:57.539707Z","url":"https://files.pythonhosted.org/packages/a2/5d/82b85eb4c24c82064c5c216f52df92f7861ff0a02d3cbcfebfacadc2b28f/psutil-5.0.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"ade8924028b2c23cc9ffe4a0737de38c668d50be5ce19495790154f530ce5389"},"provenance":null,"requires-python":null,"size":177229,"upload-time":"2016-11-06T18:29:00.558809Z","url":"https://files.pythonhosted.org/packages/14/5b/00b74d0ada625090dfaba6244860a9ec4df2e79a6c8440dd490e46c65543/psutil-5.0.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp34-cp34m-win32.whl","hashes":{"sha256":"af01b73fd66f138e06f804508fd33118823fd2abb89f105ae2b99efa4c8fd1a3"},"provenance":null,"requires-python":null,"size":174957,"upload-time":"2016-11-06T18:29:04.527838Z","url":"https://files.pythonhosted.org/packages/fd/e6/5a08cd23cefadd22b8e74d0bd1c525897c9db4dc1d2f44716dee848b8fe3/psutil-5.0.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"846925435e69cc7b802cd7d1a4bd640e180d0db15277c24e196d3a5799bf6760"},"provenance":null,"requires-python":null,"size":177189,"upload-time":"2016-11-06T18:29:08.003063Z","url":"https://files.pythonhosted.org/packages/b4/11/f68bad0d5fc08800006d2355c941e186315987456158a9b06f40e483d4e7/psutil-5.0.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp35-cp35m-win32.whl","hashes":{"sha256":"d7885ff254425c64bcc2dbff256ec1367515c15218bfda0fd3d799092437d908"},"provenance":null,"requires-python":null,"size":176812,"upload-time":"2016-11-06T18:29:11.700563Z","url":"https://files.pythonhosted.org/packages/c4/16/84213b90c78e437eff09285138947d12105ea982cb6f8fda4ab2855014e6/psutil-5.0.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"data-dist-info-metadata":{"sha256":"48627687813fd477d371c4588c61dc9da3e42f22546da21c80f273e30c587f1b"},"filename":"psutil-5.0.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"e35cb38037973ff05bc52dac4f382d17028104d77f0bb51792de93f359046902"},"provenance":null,"requires-python":null,"size":180171,"upload-time":"2016-11-06T18:29:15.915700Z","url":"https://files.pythonhosted.org/packages/98/ef/40582d2d3e39fdcc202a33ba6aab15f6ccb36cdcd04f6756cc9afe30df30/psutil-5.0.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win-amd64-py2.7.exe","hashes":{"sha256":"964e9db2641e3be6a80b5a3135f7a9425f87d8416342b4d967202e4854f3eeba"},"provenance":null,"requires-python":null,"size":411778,"upload-time":"2016-11-06T18:29:32.458135Z","url":"https://files.pythonhosted.org/packages/74/f0/e8964d58e12c7716775157821de2e758fece2581bc9f3b7c333a4de29b90/psutil-5.0.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win-amd64-py3.3.exe","hashes":{"sha256":"d7b933193322523314b0b2d699a43e43a70f43016f73782dfd892bc7ee95ecd1"},"provenance":null,"requires-python":null,"size":410125,"upload-time":"2016-11-06T18:29:37.513499Z","url":"https://files.pythonhosted.org/packages/6e/a9/4438ec3289e3925a638cb079039b748ee4fb7e3ba18a93baed39f4fcd11e/psutil-5.0.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win-amd64-py3.4.exe","hashes":{"sha256":"8709a5057d42d5e55f3940bb1e70370316defb3da03ad869342755b5a2c17c78"},"provenance":null,"requires-python":null,"size":410086,"upload-time":"2016-11-06T18:29:41.067409Z","url":"https://files.pythonhosted.org/packages/db/85/c2e27aab6db3c4d404f0e81762f5a962b7213caed0a789cb5df2990ac489/psutil-5.0.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win-amd64-py3.5.exe","hashes":{"sha256":"c065eaf76a5423341f511e732f1a17e75a55dc4aceee9a321a619a5892aec18f"},"provenance":null,"requires-python":null,"size":780670,"upload-time":"2016-11-06T18:29:45.597507Z","url":"https://files.pythonhosted.org/packages/27/fd/12a4ef4a9940331a5521216e839444dcab0addd3a49fb90c940bdc40e46e/psutil-5.0.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win32-py2.7.exe","hashes":{"sha256":"8eca28511d493209f59fe99cebfb8ecc65b8d6691f8a80fa3ab50dbb4994c81b"},"provenance":null,"requires-python":null,"size":381779,"upload-time":"2016-11-06T18:29:49.728094Z","url":"https://files.pythonhosted.org/packages/bc/77/457ddeac355fe88c8d4ee8062ce66ab5dd142f5c9423d1bfd10a568c7884/psutil-5.0.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win32-py3.3.exe","hashes":{"sha256":"7a8e7654789c468d2a6c32508638563741046f138405fea2a4427a9228ac86f4"},"provenance":null,"requires-python":null,"size":376623,"upload-time":"2016-11-06T18:29:53.256127Z","url":"https://files.pythonhosted.org/packages/8d/1a/2f61de7f89602dfd975f961bf057522ff53ef3bdc48610dbdcf306b542df/psutil-5.0.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win32-py3.4.exe","hashes":{"sha256":"4db1f1d8655a63832c9ab85d77c969ab95b740128ca9053b5c108a1e5efe6a7c"},"provenance":null,"requires-python":null,"size":376624,"upload-time":"2016-11-06T18:29:56.669027Z","url":"https://files.pythonhosted.org/packages/c1/3f/d8921d1a8672545a390fd874b77c9a30444b64ab5f8c48e2e9d971f4e98f/psutil-5.0.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.win32-py3.5.exe","hashes":{"sha256":"10f05841e3bf7b060b3779367b9cd953a6c97303c08a0e2630a9e461ce124412"},"provenance":null,"requires-python":null,"size":647782,"upload-time":"2016-11-06T18:30:00.475783Z","url":"https://files.pythonhosted.org/packages/ec/53/dc7d9e33e77efd26f7c4a9ba8d1e23ba9ce432077e16a55f1f755970e7f7/psutil-5.0.0.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.0.zip","hashes":{"sha256":"5411e22c63168220f4b8cc42fd05ea96f5b5e65e08b93b675ca50653aea482f8"},"provenance":null,"requires-python":null,"size":374074,"upload-time":"2016-11-06T18:41:47.892123Z","url":"https://files.pythonhosted.org/packages/93/7f/347309562d30c688299727e65f4d76ef34180c406dfb6f2c7b6c8d746e13/psutil-5.0.0.zip","yanked":false},{"core-metadata":{"sha256":"3c16b5cea3c7592513b1491a7696e113d3ba33a4a94d74841ee2340fb4081106"},"data-dist-info-metadata":{"sha256":"3c16b5cea3c7592513b1491a7696e113d3ba33a4a94d74841ee2340fb4081106"},"filename":"psutil-5.0.1-cp27-none-win32.whl","hashes":{"sha256":"1f2379809f2182652fc740faefa511f78b5975e6471e5fa419882dd9e082f245"},"provenance":null,"requires-python":null,"size":176737,"upload-time":"2016-12-21T01:35:18.007609Z","url":"https://files.pythonhosted.org/packages/cd/2d/ecb32ce765c52780768e4078d4d2916bdaa790b313c454f52aebe27d559d/psutil-5.0.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"3c16b5cea3c7592513b1491a7696e113d3ba33a4a94d74841ee2340fb4081106"},"data-dist-info-metadata":{"sha256":"3c16b5cea3c7592513b1491a7696e113d3ba33a4a94d74841ee2340fb4081106"},"filename":"psutil-5.0.1-cp27-none-win_amd64.whl","hashes":{"sha256":"86d67da7bfed474b5b0d545b29211f16d622e87c4089de5ea41a3fbcfc4872c7"},"provenance":null,"requires-python":null,"size":179100,"upload-time":"2016-12-21T01:35:20.364405Z","url":"https://files.pythonhosted.org/packages/6b/7e/68835630dcc765452b76b61936f75c1ef71fceae8ebaa7ecb27e42bf3f28/psutil-5.0.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp33-cp33m-win32.whl","hashes":{"sha256":"6a95283fd048810811cf971bec5cec3998e1e62e66237ef7a41a42dd0da29f8c"},"provenance":null,"requires-python":null,"size":176612,"upload-time":"2016-12-21T01:35:22.141278Z","url":"https://files.pythonhosted.org/packages/43/46/3fad73a8c581f63f08fb1f93ef5651e939caa5601a50baca7f2af3c54e2b/psutil-5.0.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"8f492b531c4321c7c43ef82b60421f4bcf5ded4ba4e13f534c064ad6c2d910ed"},"provenance":null,"requires-python":null,"size":178930,"upload-time":"2016-12-21T01:35:23.896324Z","url":"https://files.pythonhosted.org/packages/b0/d0/6d9e39115aaab0d8d52422b909e365938a6dcebe5fa04fac3a6b70398aee/psutil-5.0.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp34-cp34m-win32.whl","hashes":{"sha256":"5cc1b91d4848453b74ad8e63275a19e784ef3acd943c3627134a607b602bc31d"},"provenance":null,"requires-python":null,"size":176623,"upload-time":"2016-12-21T01:35:26.391755Z","url":"https://files.pythonhosted.org/packages/31/21/d836f37c8fde1760e889b0ace67e151c6e56f2ce819b227715a5f291452f/psutil-5.0.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"55874a1814faceaa090b7fa7addbf350603fd7042562adaae13eb6e46d3ec907"},"provenance":null,"requires-python":null,"size":178917,"upload-time":"2016-12-21T01:35:28.280103Z","url":"https://files.pythonhosted.org/packages/a1/9c/cb3f68be56ab366dbf4c901002d281e30c50cf951377ca8155c0ad304394/psutil-5.0.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp35-cp35m-win32.whl","hashes":{"sha256":"7f2d7b97faa524f75736dfde418680eff332f4be66d6217b67d09c630c90d02e"},"provenance":null,"requires-python":null,"size":178514,"upload-time":"2016-12-21T01:35:30.152277Z","url":"https://files.pythonhosted.org/packages/f7/f1/dd823eab436db1eac1ca2c8364918786584db996dc4d2bef5a3b0ebd7619/psutil-5.0.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"bf4072d4d188802505b9229ec00e141083c127bb19a6c5636c62b0daabda4bd5"},"provenance":null,"requires-python":null,"size":181835,"upload-time":"2016-12-21T01:35:32.091445Z","url":"https://files.pythonhosted.org/packages/2a/3a/107a964dc66cb4f22af9e919dde846b73c0b4ff735b10ede680895ed296c/psutil-5.0.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp36-cp36m-win32.whl","hashes":{"sha256":"786bbbeb3ea98d82ff5cedc86b640bad97bff435c819f26bddaa388da58d47da"},"provenance":null,"requires-python":null,"size":178650,"upload-time":"2017-01-15T18:27:34.549244Z","url":"https://files.pythonhosted.org/packages/c0/a1/c5d0aa766323b150dce5e51a5360542ccb75c109723d2d95e70d8cb248b8/psutil-5.0.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"data-dist-info-metadata":{"sha256":"b03798768de1ae02a3387726d3bb1c881c70cdee8bea158f236b08758379017e"},"filename":"psutil-5.0.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"544c0803760995fe42a2b7050cfd6ed32379b09ce6cd7a1eaf89c221d8669cc3"},"provenance":null,"requires-python":null,"size":181970,"upload-time":"2017-01-15T18:27:39.376642Z","url":"https://files.pythonhosted.org/packages/c0/e4/bf1059c5dd55abf65fe2ac92a0b24a92d09644a9bbc17f1ad2b497ae9d7d/psutil-5.0.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.tar.gz","hashes":{"sha256":"9d8b7f8353a2b2eb6eb7271d42ec99d0d264a9338a37be46424d56b4e473b39e"},"provenance":null,"requires-python":null,"size":326693,"upload-time":"2016-12-21T01:35:34.145966Z","url":"https://files.pythonhosted.org/packages/d9/c8/8c7a2ab8ec108ba9ab9a4762c5a0d67c283d41b13b5ce46be81fdcae3656/psutil-5.0.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py2.7.exe","hashes":{"sha256":"0c54e3b7cdc0dbb8a19b58c3eb3845a5f9f48d3be2b06ed9aa1e553db8f9db74"},"provenance":null,"requires-python":null,"size":414521,"upload-time":"2016-12-21T01:35:36.513939Z","url":"https://files.pythonhosted.org/packages/3d/90/61f2860cd2c39a4381e415897b7ff94aab21c9aca38690d0c99c1c83b85f/psutil-5.0.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py3.3.exe","hashes":{"sha256":"0fccb19631d555998fc8c98840c83678244f873486d20d5f24ebb0ac8e19d2f1"},"provenance":null,"requires-python":null,"size":412871,"upload-time":"2016-12-21T01:35:38.959549Z","url":"https://files.pythonhosted.org/packages/05/f7/6ba6efe79620bcaa9af41e0a8501e477815edccc0cad4bd379a8e8bea89c/psutil-5.0.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py3.4.exe","hashes":{"sha256":"77fde4936f26080aa14b89d292b3ebefabb80be69ef407352cbad6d2ff6882d4"},"provenance":null,"requires-python":null,"size":412858,"upload-time":"2016-12-21T01:35:42.062765Z","url":"https://files.pythonhosted.org/packages/c7/b3/3b83e69b5d96ed052b45f1fe88882b3976a3a066c0497597d3ce1407f17f/psutil-5.0.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py3.5.exe","hashes":{"sha256":"a9125b7bc12127174cf7974444ca2b39a3722f59ead1d985053e7358a3d29acd"},"provenance":null,"requires-python":null,"size":783380,"upload-time":"2016-12-21T01:35:44.660601Z","url":"https://files.pythonhosted.org/packages/4e/cf/d1004fa0617f6d6e39adf48d242f76d5d43ecd497f46add2abb513b6ffcd/psutil-5.0.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win-amd64-py3.6.exe","hashes":{"sha256":"9a3f1413b24b6751e97e51284761e1778ec0cd0a456595edad6c2f7c115b3368"},"provenance":null,"requires-python":null,"size":783511,"upload-time":"2017-01-15T18:25:42.785800Z","url":"https://files.pythonhosted.org/packages/a0/94/1f29e03230504dafeb128a4c6e381606b038ce5d4dc9b8731632eb6b0928/psutil-5.0.1.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py2.7.exe","hashes":{"sha256":"671c7b2d3fa8deffb879e9cb6cc00d83d1d990bc81f0c487576b70e811f102bf"},"provenance":null,"requires-python":null,"size":384515,"upload-time":"2016-12-21T01:35:47.208014Z","url":"https://files.pythonhosted.org/packages/2d/7c/85aa8efb36c4fcac4a845b9cfc0744c502620394cdb81db574dd20f7dda0/psutil-5.0.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py3.3.exe","hashes":{"sha256":"603bd19426b59762ad55ba5a0b8237282868addf2d0930e21b4dca79fc188787"},"provenance":null,"requires-python":null,"size":379325,"upload-time":"2016-12-21T01:35:49.642792Z","url":"https://files.pythonhosted.org/packages/1c/06/201f913389b920c24153fbfdc07c7a451653b00688d6c2ba92428b9a2c23/psutil-5.0.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py3.4.exe","hashes":{"sha256":"08e28b4dba54dd6b0d84718c988239c305e58c7f93160496661955de4f6cfe13"},"provenance":null,"requires-python":null,"size":379338,"upload-time":"2016-12-21T01:35:52.592934Z","url":"https://files.pythonhosted.org/packages/58/27/a42b1d12c201880822fd92588321e489c869f4b72eb906f8f79745bee6cb/psutil-5.0.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py3.5.exe","hashes":{"sha256":"1e71dd9aa041a7fa55f3bcc78b578e84f31004e6ce9df97229f05c60529fedb1"},"provenance":null,"requires-python":null,"size":650529,"upload-time":"2016-12-21T01:35:55.051616Z","url":"https://files.pythonhosted.org/packages/fb/c6/c60717f25c8ccfddabd57109d03580d24542eedde8d23a673328876b4137/psutil-5.0.1.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.0.1.win32-py3.6.exe","hashes":{"sha256":"c879310328c0c248331dffeea4adbe691fad7b7095cf9c2ac0a4d78a09cd8a17"},"provenance":null,"requires-python":null,"size":650663,"upload-time":"2017-01-15T18:25:51.768008Z","url":"https://files.pythonhosted.org/packages/00/09/04898f2ce604af89d7ca6866632c05a1fb0fbd7b5ea0c7e84a62570c2678/psutil-5.0.1.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"45f08e36846b408dfbda38b60b36dd8b6430d6fbd955f5945dc985a85eac6068"},"data-dist-info-metadata":{"sha256":"45f08e36846b408dfbda38b60b36dd8b6430d6fbd955f5945dc985a85eac6068"},"filename":"psutil-5.1.0-cp27-cp27m-win32.whl","hashes":{"sha256":"85524e46a1c0c7f5274640809deb96c7cdb5133a0eb805bbed0a1f825c8de77e"},"provenance":null,"requires-python":null,"size":183880,"upload-time":"2017-02-01T18:27:27.314031Z","url":"https://files.pythonhosted.org/packages/d9/bf/eae1c0aa2a5a01456e78d5df4c7cefc512854a24f43103034180ff5ea92f/psutil-5.1.0-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"45f08e36846b408dfbda38b60b36dd8b6430d6fbd955f5945dc985a85eac6068"},"data-dist-info-metadata":{"sha256":"45f08e36846b408dfbda38b60b36dd8b6430d6fbd955f5945dc985a85eac6068"},"filename":"psutil-5.1.0-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"0a64f395295aafe7c22d6cde81076151bf80f26e180e50c00c1cde5857cad224"},"provenance":null,"requires-python":null,"size":186335,"upload-time":"2017-02-01T18:27:35.043588Z","url":"https://files.pythonhosted.org/packages/f6/5d/67120bb4cea2432d333cdf0ab406336ef62122caacb2379e73a88a9cde82/psutil-5.1.0-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp33-cp33m-win32.whl","hashes":{"sha256":"bd432606d09722220f8f492223e8b69343d30f21e7ed193712632e4fdcf87af2"},"provenance":null,"requires-python":null,"size":183793,"upload-time":"2017-02-01T18:27:43.056910Z","url":"https://files.pythonhosted.org/packages/26/35/ec223413c9301d8eeecf3d319d71a117dbd2847b628b079f3757d5ec693d/psutil-5.1.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"f98c37506e5abfeb2c857794ccfbce86b376cf6344210ea55f41b0ced5fd8d98"},"provenance":null,"requires-python":null,"size":186185,"upload-time":"2017-02-01T18:27:49.896261Z","url":"https://files.pythonhosted.org/packages/9f/20/bb95f30c99837f3c41a8a595752d9cc1afe595f160d0612eea6e4a905de9/psutil-5.1.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp34-cp34m-win32.whl","hashes":{"sha256":"8760214ce01f71cd6766527396607cf2b3b41e6f8a34aaa9f716090750fd9925"},"provenance":null,"requires-python":null,"size":183803,"upload-time":"2017-02-01T18:27:56.955786Z","url":"https://files.pythonhosted.org/packages/0b/f2/90a7288b36a18fb522e00116f7f5c657df9dba3a12636d2fc9bcef97f524/psutil-5.1.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"2309cf46f5db0a78f52e56e06cbb258fe76d3d54cf8ab9fa067b4c26cd722541"},"provenance":null,"requires-python":null,"size":186156,"upload-time":"2017-02-01T18:28:03.790958Z","url":"https://files.pythonhosted.org/packages/51/c6/3757b2bea37edc9889fe85e2ca9b8a2c47b3b758d35f65704f2a5ac5c13e/psutil-5.1.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp35-cp35m-win32.whl","hashes":{"sha256":"6d89d5f68433afe6755bd26ae3ea4587f395c7613d7be5bc4f0f97b1e299228a"},"provenance":null,"requires-python":null,"size":185696,"upload-time":"2017-02-01T18:28:10.369149Z","url":"https://files.pythonhosted.org/packages/30/9c/322df29c490ceb97d8cbbc663424fad94e900a1921d03cc9c61e36f17444/psutil-5.1.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"b3c15965b2dfad34ea79494f82007e71b02e1c5352b551d00647fb2be9deabe1"},"provenance":null,"requires-python":null,"size":189130,"upload-time":"2017-02-01T18:28:17.197543Z","url":"https://files.pythonhosted.org/packages/b3/3a/37d6bf7dedf263e1119b03aaee8565182bf2577ed5ecd7975e253e242bd9/psutil-5.1.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp36-cp36m-win32.whl","hashes":{"sha256":"d32551c242c041b0506d13cfd271db56ba57323283b1f952b858505824f3e82a"},"provenance":null,"requires-python":null,"size":185695,"upload-time":"2017-02-01T18:28:25.059542Z","url":"https://files.pythonhosted.org/packages/42/0b/5ee6feda28c2895b0e8cae4d828d356e13e29bf88bc614065e490bfc8b51/psutil-5.1.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"data-dist-info-metadata":{"sha256":"f1cc4b1bb4982cfc6e8259dbc0fd96789e31eff9b65d8e5a44401e11d9926c55"},"filename":"psutil-5.1.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"3b821bb59911afdba5c0c28b1e6e7511cfb0869dd8827c3ab6916ead508c9155"},"provenance":null,"requires-python":null,"size":189127,"upload-time":"2017-02-01T18:28:32.228516Z","url":"https://files.pythonhosted.org/packages/af/6f/ef0799221597f367171c0bb8b3f1b22865e6b7507ee26e1a6a16f408a058/psutil-5.1.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.tar.gz","hashes":{"sha256":"7570e1d82345fab3a0adce24baf993adbca4c87a1be2fa6ee79babdaafa817fb"},"provenance":null,"requires-python":null,"size":339603,"upload-time":"2017-02-01T18:29:04.029583Z","url":"https://files.pythonhosted.org/packages/ba/5f/87b151dd53f8790408adf5096fc81c3061313c36a089d9f7ec9e916da0c1/psutil-5.1.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py2.7.exe","hashes":{"sha256":"07aab3a12c00315a144b2a61d384364e64077c08c0a79966f18b744d867f9727"},"provenance":null,"requires-python":null,"size":422485,"upload-time":"2017-02-01T18:24:47.444024Z","url":"https://files.pythonhosted.org/packages/8c/1c/8bf1c2f6f2b4f012449bd9781d55397e85e30487c2879ed35569c7f616de/psutil-5.1.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py3.3.exe","hashes":{"sha256":"b78de627c157b4889e055fd00a635ad0fbe107ae4b3059f419738edbfb15b3f4"},"provenance":null,"requires-python":null,"size":420852,"upload-time":"2017-02-01T18:25:00.582537Z","url":"https://files.pythonhosted.org/packages/de/5a/3041f7d07d9699ef148291721ee167608761e223bc26bbe33483cbe90517/psutil-5.1.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py3.4.exe","hashes":{"sha256":"48fc3945219f6577b61a76f81fbbaa3ce6bdccf0e87650597c88aac38e63904b"},"provenance":null,"requires-python":null,"size":420825,"upload-time":"2017-02-01T18:25:14.404878Z","url":"https://files.pythonhosted.org/packages/d0/83/4e03c76cd202d4a4ead9b468cf2afa611d689b1ae17e4f152b67c05d2a27/psutil-5.1.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py3.5.exe","hashes":{"sha256":"98180376d2f0a3b0d24dbd8ef64da4aac46f98ccc84d2e3ebc3a486b97a75915"},"provenance":null,"requires-python":null,"size":791399,"upload-time":"2017-02-01T18:25:37.374511Z","url":"https://files.pythonhosted.org/packages/ff/86/328005e07308a87f4d86108a51f940fba1fb1c4f60d4676f80fc66a9dd7a/psutil-5.1.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win-amd64-py3.6.exe","hashes":{"sha256":"a574d3fb56514ae98f2836a218b0dcf6daeb31b9945cec9b7daa9b619ad3005a"},"provenance":null,"requires-python":null,"size":791396,"upload-time":"2017-02-01T18:26:00.310591Z","url":"https://files.pythonhosted.org/packages/83/86/88c20e9ebb1565598401b23ece21367bd059d569cb0ce797509ebdc429be/psutil-5.1.0.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py2.7.exe","hashes":{"sha256":"b014a8df14f00054762a2ef6f9a449ccde18d9c4de48dc8cb1809343f328f83d"},"provenance":null,"requires-python":null,"size":392388,"upload-time":"2017-02-01T18:26:13.878537Z","url":"https://files.pythonhosted.org/packages/a5/e8/1032177e5af2300b39aa50623e0df4d9a91a9b316803aab8f597e6c855e2/psutil-5.1.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py3.3.exe","hashes":{"sha256":"01e68d047f5023d4cb55b3a2653e296d86f53abc7e63e74c8abeefa627d9307f"},"provenance":null,"requires-python":null,"size":387232,"upload-time":"2017-02-01T18:26:25.910876Z","url":"https://files.pythonhosted.org/packages/cc/b8/24bf07e83d063f552132e7808d7029663466ebed945c532ba05068c65ae3/psutil-5.1.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py3.4.exe","hashes":{"sha256":"bfd279124512ee1b744eb7efa269be2096e0ce7075204daccd6f4b73c359316c"},"provenance":null,"requires-python":null,"size":387243,"upload-time":"2017-02-01T18:26:38.999172Z","url":"https://files.pythonhosted.org/packages/2a/ad/40d39c8911ca0783df9684649038241913fa3119c74030b9d1a02eb28da3/psutil-5.1.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py3.5.exe","hashes":{"sha256":"518356f5384a8996eb4056b6fd01a28bb502476e5d0ab9bd112d8e1dc9ce041a"},"provenance":null,"requires-python":null,"size":658437,"upload-time":"2017-02-01T18:26:58.991042Z","url":"https://files.pythonhosted.org/packages/db/b1/023fa6a995356c4a4ae68ba53e1269d693ad66783ee2c0c4ac1d6e556226/psutil-5.1.0.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.0.win32-py3.6.exe","hashes":{"sha256":"23670901cfa4308cc6c442e08efcd93c6a2adb3bfdbbf51d4c120fc2a1595899"},"provenance":null,"requires-python":null,"size":658436,"upload-time":"2017-02-01T18:27:18.934060Z","url":"https://files.pythonhosted.org/packages/f8/ee/be644cdded3a5761f22cd0255534f04ad89a6fabdfb85f58100d25b6bf1a/psutil-5.1.0.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"b880ef4983d10a0a227170484a19bf585b91f8d2cf0238488b8a0e39337acdad"},"data-dist-info-metadata":{"sha256":"b880ef4983d10a0a227170484a19bf585b91f8d2cf0238488b8a0e39337acdad"},"filename":"psutil-5.1.1-cp27-none-win32.whl","hashes":{"sha256":"b9cc631c31794f8150a034a15448ecbde6c65ab078437eb01e0a71103bced297"},"provenance":null,"requires-python":null,"size":184444,"upload-time":"2017-02-03T12:00:02.925645Z","url":"https://files.pythonhosted.org/packages/e3/6f/3546783d70949aff150555cafb1cdd51cfddbb2fea042ed3d5b930232233/psutil-5.1.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"b880ef4983d10a0a227170484a19bf585b91f8d2cf0238488b8a0e39337acdad"},"data-dist-info-metadata":{"sha256":"b880ef4983d10a0a227170484a19bf585b91f8d2cf0238488b8a0e39337acdad"},"filename":"psutil-5.1.1-cp27-none-win_amd64.whl","hashes":{"sha256":"d723be81a8db76ad6d78b04effd33e73813567eb1b1b7669c6926da033cb9774"},"provenance":null,"requires-python":null,"size":186898,"upload-time":"2017-02-03T12:00:10.481992Z","url":"https://files.pythonhosted.org/packages/48/98/9f3277e1d4a6d6fbc501146b9f5e8547c793df038dfa7037816edbf90325/psutil-5.1.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp33-cp33m-win32.whl","hashes":{"sha256":"5fc6d4fe04f014ea68392d1e7ab7103637b9dcbfd7bf3bc6d9d482177bf82777"},"provenance":null,"requires-python":null,"size":184356,"upload-time":"2017-02-03T12:00:18.015691Z","url":"https://files.pythonhosted.org/packages/98/fa/c43ce8992690d3b674259f21c897dab254b3fbbb2758254787e4b0f5aee8/psutil-5.1.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"7d0c6f77ebeb248ee62383340a8bd5a9b067e64618c9056d701eefdccf27f9f4"},"provenance":null,"requires-python":null,"size":186735,"upload-time":"2017-02-03T12:00:25.248665Z","url":"https://files.pythonhosted.org/packages/e1/4f/7d16da2b82d615a7fc57f159e5e7c8776c7b794ca6330a9318ca93eb735b/psutil-5.1.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp34-cp34m-win32.whl","hashes":{"sha256":"d31859dae480bc1a0be48f239bcf3caa26447fae177549a30c4b1a2a2776f299"},"provenance":null,"requires-python":null,"size":184362,"upload-time":"2017-02-03T12:00:31.866390Z","url":"https://files.pythonhosted.org/packages/bc/a8/3f3e69227217d1e7355e6de1980f357f6151ce1543bd520652b47823f93e/psutil-5.1.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"857a3620b12a33ed4169aee959e1640e7323f18dd7726035cc057e1d39639df2"},"provenance":null,"requires-python":null,"size":186740,"upload-time":"2017-02-03T12:00:39.950524Z","url":"https://files.pythonhosted.org/packages/d3/2c/974ced45441b7adf476f1b32ac3f7840f0d7ae295dd4512de21997e2049b/psutil-5.1.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp35-cp35m-win32.whl","hashes":{"sha256":"4ed11e8caff64c452a2eee8c1bea614b717a8e66e97fc88ce272d98a6499cb9a"},"provenance":null,"requires-python":null,"size":186257,"upload-time":"2017-02-03T12:00:46.799733Z","url":"https://files.pythonhosted.org/packages/2f/17/cb34ac2f50ff6499e6b64837a2d34b92696ff1dc14e57933363d94b7301c/psutil-5.1.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"3115d7acbf3cc81345fd7252946a59c3730f7baba546361a535f0a8f00d862c9"},"provenance":null,"requires-python":null,"size":189715,"upload-time":"2017-02-03T12:00:55.165186Z","url":"https://files.pythonhosted.org/packages/09/da/f9f435d53711859d41ee8a0fb87177e8b30b5dbca5bd06dd05b5c4b46ef3/psutil-5.1.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp36-cp36m-win32.whl","hashes":{"sha256":"9f6cb84d0f8e0c993c91d10ef86c637b7f1c1d4d4ca63ec0a73545cc13e9656a"},"provenance":null,"requires-python":null,"size":186256,"upload-time":"2017-02-03T12:01:01.728292Z","url":"https://files.pythonhosted.org/packages/24/19/20a4a33db5d005959458f68816fbb725791ee7843ba4f8a40dfd38e8e840/psutil-5.1.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"data-dist-info-metadata":{"sha256":"9de4440590e9d06325c73aa9a0f85d5e2beb77f570187d8aa12add76025de520"},"filename":"psutil-5.1.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"35311e26f7138276fa3e7af86bcb8ecbaee945c3549e690a481379075de386ba"},"provenance":null,"requires-python":null,"size":189711,"upload-time":"2017-02-03T12:01:08.476075Z","url":"https://files.pythonhosted.org/packages/6e/2b/44aea26dc3a304285e8eb91af628c9722ba4e1f44f7969cc24fded4aa744/psutil-5.1.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.tar.gz","hashes":{"sha256":"ece06401d719050a84cca97764ff5b0e41aafe6b6a2ec8a1d0bb89ca5e206d0f"},"provenance":null,"requires-python":null,"size":341006,"upload-time":"2017-02-03T12:01:19.347630Z","url":"https://files.pythonhosted.org/packages/49/ed/2a0b13f890e798b6f1f3625f0e87e5b712471d2c1c625bdcd396d36c56dc/psutil-5.1.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py2.7.exe","hashes":{"sha256":"e75edc462005475da019f82c8a13b215e2e48db8f284d8be14308b611505d185"},"provenance":null,"requires-python":null,"size":423004,"upload-time":"2017-02-03T12:01:32.247678Z","url":"https://files.pythonhosted.org/packages/99/35/96acf109f463ce31cd29ec327d9b3b1c3eb13afbc5635e54c75e1e22c924/psutil-5.1.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py3.3.exe","hashes":{"sha256":"e18274b77e61b3774bcf11a4aa368032fe3bbc21e4217ca903c77541fbe00eef"},"provenance":null,"requires-python":null,"size":421360,"upload-time":"2017-02-03T12:01:46.775449Z","url":"https://files.pythonhosted.org/packages/b7/56/70fc2173d34f0ee73215a0e88e2702933cf1549e3ff3d8f2d757b9e1351c/psutil-5.1.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py3.4.exe","hashes":{"sha256":"76307f8ac94adc87509df43bba28706c27d6c5e4b7429fb658dd5905adae4dc3"},"provenance":null,"requires-python":null,"size":421366,"upload-time":"2017-02-03T12:02:01.021939Z","url":"https://files.pythonhosted.org/packages/f1/9b/961442950e039c65938665dcbe6e72bff1a2dbbd6c6c08c55e3e10db38ed/psutil-5.1.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py3.5.exe","hashes":{"sha256":"8101a2b83fa3b93fbf5d5edca7169a4289f34ace2ee25d0f758cec5ae553190f"},"provenance":null,"requires-python":null,"size":791943,"upload-time":"2017-02-03T12:02:23.633303Z","url":"https://files.pythonhosted.org/packages/bb/69/95fdd45b2c1cc4735760b81d826a62433d6726ab6e8e6e2e982fb1264c20/psutil-5.1.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win-amd64-py3.6.exe","hashes":{"sha256":"b16b48868a58322edd240cf55a0855e1b6fead3e5f02a41b73503d5c47acf330"},"provenance":null,"requires-python":null,"size":791938,"upload-time":"2017-02-03T12:02:48.386069Z","url":"https://files.pythonhosted.org/packages/15/b4/1ba8944aaae568f8f4acd7fd3ca46a95077f07b005774f212226c99a82d8/psutil-5.1.1.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py2.7.exe","hashes":{"sha256":"12aeeaf5269bc75ed605ffc4979cb95c889989b40d307adf067f04f93f6d3365"},"provenance":null,"requires-python":null,"size":392908,"upload-time":"2017-02-03T12:03:01.259012Z","url":"https://files.pythonhosted.org/packages/d0/38/c274e67564aed7475fba4e0f9eaac3235797226df0b68abce7f3fc96bffa/psutil-5.1.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py3.3.exe","hashes":{"sha256":"2f51c669bd528982fc5397d9c84b8b389a56611c111159b1710570db33fc9750"},"provenance":null,"requires-python":null,"size":387752,"upload-time":"2017-02-03T12:03:13.897990Z","url":"https://files.pythonhosted.org/packages/d9/51/52e109c7884a6fa931882081e94dfd0e1b6adb054d4fc82943e1c9b691d2/psutil-5.1.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py3.4.exe","hashes":{"sha256":"f68ce5d80db909ee655ac8a322cda3abf47f41c134e7cf61b25565f116fce33f"},"provenance":null,"requires-python":null,"size":387758,"upload-time":"2017-02-03T12:03:26.769319Z","url":"https://files.pythonhosted.org/packages/5e/ee/ff6d626da64a799db055bab9f69ff9a50612a645781d14c24078bca418cb/psutil-5.1.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py3.5.exe","hashes":{"sha256":"380aa69aa529e4a4e35579d3e0320617e112473240c95b815d3b421c86e2ab6c"},"provenance":null,"requires-python":null,"size":658955,"upload-time":"2017-02-03T12:03:45.921701Z","url":"https://files.pythonhosted.org/packages/62/9b/847c9c6b1c053a15b44ccf809fce54bbd71e466fd8c1b8eecb0f8bbeb2ce/psutil-5.1.1.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.1.win32-py3.6.exe","hashes":{"sha256":"bd3b881faa071a5f6f999d036cfc0d744eed223390fde05ae2a74f0f514f8bd0"},"provenance":null,"requires-python":null,"size":658953,"upload-time":"2017-02-03T12:04:04.770301Z","url":"https://files.pythonhosted.org/packages/80/ef/c23e653bbc1a523217d676fac3ac4050e9fe4d673c84680498c02ff89d23/psutil-5.1.1.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"40c22031c3c1f82a2758dbd44797eade645ac30207e45f42cb9da4fd1dd3808f"},"data-dist-info-metadata":{"sha256":"40c22031c3c1f82a2758dbd44797eade645ac30207e45f42cb9da4fd1dd3808f"},"filename":"psutil-5.1.2-cp27-none-win32.whl","hashes":{"sha256":"caa870244015bb547eeab7377d9fe41c44319fda6862aa56974108a224c04b1a"},"provenance":null,"requires-python":null,"size":185018,"upload-time":"2017-02-03T19:13:24.939112Z","url":"https://files.pythonhosted.org/packages/57/bc/e0a5e35a9b7f407e229db75b24c46bef012609450d7eac6b5e596a604acb/psutil-5.1.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"40c22031c3c1f82a2758dbd44797eade645ac30207e45f42cb9da4fd1dd3808f"},"data-dist-info-metadata":{"sha256":"40c22031c3c1f82a2758dbd44797eade645ac30207e45f42cb9da4fd1dd3808f"},"filename":"psutil-5.1.2-cp27-none-win_amd64.whl","hashes":{"sha256":"d60d978f5a9b4bc9bb22c5c4bbeb50043db6fb70e3270c08f51e81357f8ca556"},"provenance":null,"requires-python":null,"size":187473,"upload-time":"2017-02-03T19:13:32.487659Z","url":"https://files.pythonhosted.org/packages/84/bd/49681360ca11a1aeba5e48b80b6bd576695a67fe15baca144c6bfcbc9285/psutil-5.1.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp33-cp33m-win32.whl","hashes":{"sha256":"61b9104157df03790fae3b138afb64ad3ecb663669ee3548e609306e0b35ca61"},"provenance":null,"requires-python":null,"size":184928,"upload-time":"2017-02-03T19:13:39.838419Z","url":"https://files.pythonhosted.org/packages/59/c7/7f31631d96ba58e81ad03b4f7cee234a35c67edc44b2fe5936e41e7d1e9e/psutil-5.1.2-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"11bfc49cd680dec42c9a7200f8b0cc4d89a9d5dbad7f3b027cfac3a305343a2a"},"provenance":null,"requires-python":null,"size":187306,"upload-time":"2017-02-03T19:13:47.199966Z","url":"https://files.pythonhosted.org/packages/a3/dc/0c87a42dbff252ab68263f88e3bd8ffaa02d766e56f743cc9074bb28c0e1/psutil-5.1.2-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp34-cp34m-win32.whl","hashes":{"sha256":"75b27bac9e91e9868723d8964e73a157799c26190978029b0ef294ad7727d91d"},"provenance":null,"requires-python":null,"size":184936,"upload-time":"2017-02-03T19:13:54.487932Z","url":"https://files.pythonhosted.org/packages/e6/2c/fa2d3770ea68f7f7577bfcf3fc00a2a68760e7d78b8ccb4a323fb44b27fa/psutil-5.1.2-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"abcfbf43eb47372e961278ee7da9a4757d59999462225b358f49c8e69a393f32"},"provenance":null,"requires-python":null,"size":187315,"upload-time":"2017-02-03T19:14:02.683013Z","url":"https://files.pythonhosted.org/packages/ed/bf/cc8cdbfc7e10518e2e1141e0724623f52bc2f41e7bdd327f8ed63e0edc01/psutil-5.1.2-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp35-cp35m-win32.whl","hashes":{"sha256":"3fd346acebeb84d9d351cabc02ea1bee1536fb7e165e7e5ead0e0912ce40cbb1"},"provenance":null,"requires-python":null,"size":186829,"upload-time":"2017-02-03T19:14:10.158362Z","url":"https://files.pythonhosted.org/packages/bf/60/e78e3455085393ab4902ef9b1c82e52d89f7b212582c58b76407a7078ad6/psutil-5.1.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"69644ed20c08bd257039733c71a47d871f5bdd481d63f8408e28f03f491e2a03"},"provenance":null,"requires-python":null,"size":190288,"upload-time":"2017-02-03T19:14:17.352480Z","url":"https://files.pythonhosted.org/packages/55/0a/46eba44208248da5aaeadeeab57aac3ee172df8a7291ad5d62efbed566ee/psutil-5.1.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp36-cp36m-win32.whl","hashes":{"sha256":"006ea083c66aa2a2be18bce84e35d0f3301d4ee3c869cb9d475fecce68470a71"},"provenance":null,"requires-python":null,"size":186826,"upload-time":"2017-02-03T19:14:24.235507Z","url":"https://files.pythonhosted.org/packages/26/e4/f5316795a709397e911eca478d547a3bb51b8dca447b7ed62d328cafa570/psutil-5.1.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"data-dist-info-metadata":{"sha256":"0f6bf0464a251593159b71879d6e05ca05190e53d209a32229b793365d8de056"},"filename":"psutil-5.1.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"b64fa3a1ec7e74689b093ee6b3a487979157f81915677b9145585a2babe1b9f5"},"provenance":null,"requires-python":null,"size":190282,"upload-time":"2017-02-03T19:14:31.621854Z","url":"https://files.pythonhosted.org/packages/c0/e2/2c758b825b48eb9ae30676499a862b8475efdf683952efbb63f531413489/psutil-5.1.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.tar.gz","hashes":{"sha256":"43f32b0a392c80cff0f480bd0792763333e46d7062285dd1226b70473c55e8ac"},"provenance":null,"requires-python":null,"size":341325,"upload-time":"2017-02-03T19:14:42.976980Z","url":"https://files.pythonhosted.org/packages/19/2c/41c601cdd5586f601663d6985ff2cf1c5322f1ffd32d67d3001035d9f81d/psutil-5.1.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py2.7.exe","hashes":{"sha256":"a44f1735f8464b5cde862d76a78843869da02e1454278a38b1026c9cfa172daf"},"provenance":null,"requires-python":null,"size":423677,"upload-time":"2017-02-03T19:14:56.817144Z","url":"https://files.pythonhosted.org/packages/f2/0e/f26bd5b5e0293f715e18b1de0e46eb3378c8b2a2f54ce67bb48ee47dff44/psutil-5.1.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py3.3.exe","hashes":{"sha256":"e8bb29ba0e1526de8932025e07c13f6e26ab43a4cc2861b849ab2daf83ef4c3a"},"provenance":null,"requires-python":null,"size":422033,"upload-time":"2017-02-03T19:15:12.278688Z","url":"https://files.pythonhosted.org/packages/82/d1/e86d5892bfeb5b8439a96b39b2acc892aa54b9df9feb75ef7384673fe883/psutil-5.1.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py3.4.exe","hashes":{"sha256":"71cd26331eb0c26ba19d5acb67716741666a581f90bce35cc5cb733eb6bbb087"},"provenance":null,"requires-python":null,"size":422041,"upload-time":"2017-02-03T19:15:27.020838Z","url":"https://files.pythonhosted.org/packages/9f/75/f9232734ab6cdf3d656a4ef65fbe5440948c38db910002a4570b01e233d2/psutil-5.1.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py3.5.exe","hashes":{"sha256":"e3ea19ac2c6e1d54cb3de5919040018af2b5a0d846f7f9dc0cc4e2a125725015"},"provenance":null,"requires-python":null,"size":792618,"upload-time":"2017-02-03T19:15:52.787383Z","url":"https://files.pythonhosted.org/packages/e9/3d/cb2444851956cc4c1fe62bc4b266881e07f16b4be3cdc7b5c1ead5d99b76/psutil-5.1.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win-amd64-py3.6.exe","hashes":{"sha256":"18ad3e6d3b46dc56eda32674e9da77527bb4ac98503e45c89837fba641a2cc16"},"provenance":null,"requires-python":null,"size":792613,"upload-time":"2017-02-03T19:16:17.350579Z","url":"https://files.pythonhosted.org/packages/36/a5/803a7fdb45924ee69221259184bb2766a1a2819d91e806912f53c6dccc6d/psutil-5.1.2.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py2.7.exe","hashes":{"sha256":"1f8e54923b5d80b880d0dbc2ec5bcf51cbf1db7d54e4d2acdeeb02f42a21735a"},"provenance":null,"requires-python":null,"size":393580,"upload-time":"2017-02-03T19:16:30.238524Z","url":"https://files.pythonhosted.org/packages/b6/89/191e13e1ba569fd143c66fe340e820b909fef8599f78237cc505dd59552b/psutil-5.1.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py3.3.exe","hashes":{"sha256":"99a77884670999cf6c589539f1af3af66d8f59d9b8a9697b60398434933f56a8"},"provenance":null,"requires-python":null,"size":388425,"upload-time":"2017-02-03T19:16:43.231178Z","url":"https://files.pythonhosted.org/packages/45/ba/764a926681d90302a54494d59665ea906676b974c2ae0af7b44d5d6d9f24/psutil-5.1.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py3.4.exe","hashes":{"sha256":"2a7874d2d2718a4648cfaa4ab731f7de4867230bdcd543bf1c1fda05bc34e068"},"provenance":null,"requires-python":null,"size":388433,"upload-time":"2017-02-03T19:16:58.697184Z","url":"https://files.pythonhosted.org/packages/eb/91/98584bf6f6934c5d02b116634d6e00eddb2ec6b057556d77fdda2fe72ab0/psutil-5.1.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py3.5.exe","hashes":{"sha256":"42cbf7db0ce76431676da30e792b80e1228857e50afe859518b999125b5da673"},"provenance":null,"requires-python":null,"size":659629,"upload-time":"2017-02-03T19:17:19.107384Z","url":"https://files.pythonhosted.org/packages/80/b9/b17a77e77be75640594fba23837490a1dc425c9d6a9fc0b4997f5bfce984/psutil-5.1.2.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.2.win32-py3.6.exe","hashes":{"sha256":"cfd6bddd0db750b606454791432023a4260184204f4462b2a4206b7802546ead"},"provenance":null,"requires-python":null,"size":659626,"upload-time":"2017-02-03T19:17:39.500599Z","url":"https://files.pythonhosted.org/packages/8b/bd/af0e5d889818caa35fc8dc5157363c38bdb6fcb7fa1389c5daec24857a63/psutil-5.1.2.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"f3276ad95ad14e59ea78a4da9482d49f8096583e96beedc6b0261259c3d8e39e"},"data-dist-info-metadata":{"sha256":"f3276ad95ad14e59ea78a4da9482d49f8096583e96beedc6b0261259c3d8e39e"},"filename":"psutil-5.1.3-cp27-none-win32.whl","hashes":{"sha256":"359a66879068ce609f8c034b3c575e357a92c033357f398490fc77cf8af46bf7"},"provenance":null,"requires-python":null,"size":185714,"upload-time":"2017-02-07T21:33:17.748014Z","url":"https://files.pythonhosted.org/packages/6f/c0/82c15d73633fdee59b5bd064396038a63a8920359c86460cd01d6ddcedfc/psutil-5.1.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"f3276ad95ad14e59ea78a4da9482d49f8096583e96beedc6b0261259c3d8e39e"},"data-dist-info-metadata":{"sha256":"f3276ad95ad14e59ea78a4da9482d49f8096583e96beedc6b0261259c3d8e39e"},"filename":"psutil-5.1.3-cp27-none-win_amd64.whl","hashes":{"sha256":"a0becbbe09bed44f8f5dc3909c7eb383315f932faeb0029abe8d5c737e8dcc7e"},"provenance":null,"requires-python":null,"size":188166,"upload-time":"2017-02-07T21:33:23.089530Z","url":"https://files.pythonhosted.org/packages/80/92/c5136bbade8ba85d0aa2f5d5cbe8af80bac6ea1ef77c5445aa625a8caefb/psutil-5.1.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp33-cp33m-win32.whl","hashes":{"sha256":"5b2cc379287ded7f9a22521318bf010429234c2864b4146fe518f11729821771"},"provenance":null,"requires-python":null,"size":185622,"upload-time":"2017-02-07T21:33:27.925426Z","url":"https://files.pythonhosted.org/packages/b2/fb/8ad6ff1a9169b35a001ea089ab8657156df1b9b9903e218f1839fba0aae3/psutil-5.1.3-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"d0e88d2e8ac9ede745f589049a74ac1e3e614c4e5eed69e507d58bda8fa3c958"},"provenance":null,"requires-python":null,"size":188003,"upload-time":"2017-02-07T21:33:33.107156Z","url":"https://files.pythonhosted.org/packages/a7/8c/8a8bca010487f008ca026e69e76c53a025905a7bd1759567ec70d85d89a0/psutil-5.1.3-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp34-cp34m-win32.whl","hashes":{"sha256":"72b67b988c0a42825a8ca76000fc385dde85652310278cca807db7dfbcba5e7e"},"provenance":null,"requires-python":null,"size":185627,"upload-time":"2017-02-07T21:33:37.199567Z","url":"https://files.pythonhosted.org/packages/81/69/0ea7d353a82df8d8251842c71131f24b8a5cfa6982f7b89e809c37819c0b/psutil-5.1.3-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"474ab9a6abc05fcd7bb5c32cb828f3f9fc54a2cd349d63c94dff0af3b3ba7e64"},"provenance":null,"requires-python":null,"size":188011,"upload-time":"2017-02-07T21:33:41.528046Z","url":"https://files.pythonhosted.org/packages/93/82/2172e0a319e4c1387898b588e4dc9b8f2283fd9eda0a486ebe03ece2bff7/psutil-5.1.3-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp35-cp35m-win32.whl","hashes":{"sha256":"92bfc1f1929593ab7793ddce512295336e3e788b86a1bbf32701aa67c5ce27f4"},"provenance":null,"requires-python":null,"size":187522,"upload-time":"2017-02-07T21:33:45.934959Z","url":"https://files.pythonhosted.org/packages/43/74/08fc07b34eeb8e357dbe6bca02b844cf76cef15b36098611a50720bf7786/psutil-5.1.3-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"7be50561ed0060c86385c2ef4dd8a383298f29728eb6e30955ae2ebbd4554e1a"},"provenance":null,"requires-python":null,"size":190983,"upload-time":"2017-02-07T21:33:51.074513Z","url":"https://files.pythonhosted.org/packages/fa/42/42e547764bf65617077b696c4c49bed6109b00696882a196008de5f8917b/psutil-5.1.3-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp36-cp36m-win32.whl","hashes":{"sha256":"4de5566d9d8c3695726f9ec3324cd56d3eb363365508ea39854d2ebe5d57b945"},"provenance":null,"requires-python":null,"size":187516,"upload-time":"2017-02-07T21:33:56.040728Z","url":"https://files.pythonhosted.org/packages/c5/5f/71b89c9dede1da356dbbe321ae410786b94e0b516791191c008864844733/psutil-5.1.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"data-dist-info-metadata":{"sha256":"ca962e552341e254e428a5583893983bb026161e7253c100c751493866519e08"},"filename":"psutil-5.1.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"678ef7b4e38281ff16dbdac98fc1d0679d46fed3fadd5d4648096fbb6d6b1b95"},"provenance":null,"requires-python":null,"size":190979,"upload-time":"2017-02-07T21:34:01.058372Z","url":"https://files.pythonhosted.org/packages/bc/21/1823e2349b1f6ec526a55d21497e5d627ac26a6c5a3d49a07b4afad45547/psutil-5.1.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.tar.gz","hashes":{"sha256":"959bd58bdc8152b0a143cb3bd822d4a1b8f7230617b0e3eb2ff6e63812120f2b"},"provenance":null,"requires-python":null,"size":341980,"upload-time":"2017-02-07T21:34:07.636785Z","url":"https://files.pythonhosted.org/packages/78/0a/aa90434c6337dd50d182a81fe4ae4822c953e166a163d1bf5f06abb1ac0b/psutil-5.1.3.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py2.7.exe","hashes":{"sha256":"6c288b7a639f341391ba474d4c8fb495a19220015284b46e7b23f626afafc810"},"provenance":null,"requires-python":null,"size":424369,"upload-time":"2017-02-07T21:34:15.354723Z","url":"https://files.pythonhosted.org/packages/07/bb/aac12b9c56722cf8b6ed0c89eccf1e3db75795576b7e3575001248802c0d/psutil-5.1.3.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py3.3.exe","hashes":{"sha256":"c4b53e0630b83c784f807170ae2d12f1cf1e45e3913f35f9784e5556ba4a0786"},"provenance":null,"requires-python":null,"size":422726,"upload-time":"2017-02-07T21:34:24.113585Z","url":"https://files.pythonhosted.org/packages/1e/5d/4804b1d23e0e8c442876c02438c4f89b3512806b034dea5807ca1ddd6535/psutil-5.1.3.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py3.4.exe","hashes":{"sha256":"57a4e51a0f2fd8f361fcf545eeff54932a29b716ad01e60247d1abaffbc1b954"},"provenance":null,"requires-python":null,"size":422733,"upload-time":"2017-02-07T21:34:32.211085Z","url":"https://files.pythonhosted.org/packages/51/70/34ea430cf5c21540e30b805f0740d81f75c5766c2f68af9207ae18e147dc/psutil-5.1.3.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py3.5.exe","hashes":{"sha256":"4b26f56f09ad206d9fb8b2fa29926a696419b26e2c5d461afe477481cec1105c"},"provenance":null,"requires-python":null,"size":793310,"upload-time":"2017-02-07T21:34:46.367507Z","url":"https://files.pythonhosted.org/packages/a2/8c/004fabd6406879fd1caa1ed4ea7ab850afad6e27e3c5b8e8a4d0d134de3e/psutil-5.1.3.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win-amd64-py3.6.exe","hashes":{"sha256":"94ed102897b8c7103ff51e2b2953caf56bb80c3343523fd3013db3ec91bd8c4b"},"provenance":null,"requires-python":null,"size":793304,"upload-time":"2017-02-07T21:35:00.329688Z","url":"https://files.pythonhosted.org/packages/55/18/6a48a9b9dad56c54236260c1e0e2313497b3af176b053a110ea134b9bb9f/psutil-5.1.3.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py2.7.exe","hashes":{"sha256":"9fcac25e01c0f9f1b6d86c860c6d4da627e458f277f24415f15b1b29cce35f60"},"provenance":null,"requires-python":null,"size":394272,"upload-time":"2017-02-07T21:35:07.791551Z","url":"https://files.pythonhosted.org/packages/33/97/442e6eefe2a12cd00d09721fb24ddf726dd62c1073579a860682919cc640/psutil-5.1.3.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py3.3.exe","hashes":{"sha256":"8349494ee9405a31f4f9d9d3564663c870fed5dd62efd2edfdf64c5841bb838f"},"provenance":null,"requires-python":null,"size":389117,"upload-time":"2017-02-07T21:35:16.679190Z","url":"https://files.pythonhosted.org/packages/ca/46/6d9a5c657298d1363cb37ab0f84eb1fd54639fa4b2729523a68cd6a1b043/psutil-5.1.3.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py3.4.exe","hashes":{"sha256":"c8dc71de8ba61604a5cae5dee5330229dc71538c82ef13458cee838b6c0f6435"},"provenance":null,"requires-python":null,"size":389124,"upload-time":"2017-02-07T21:35:24.590876Z","url":"https://files.pythonhosted.org/packages/5b/c4/7056b6c602ff5be0095fe403617cded940a75a80db49bb51846bc235a0bb/psutil-5.1.3.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py3.5.exe","hashes":{"sha256":"38c1e88f3a8a548d9caa7f56db1cc7d508eda48eb2c4aa484a908bc5d06f87bd"},"provenance":null,"requires-python":null,"size":660318,"upload-time":"2017-02-07T21:35:36.604103Z","url":"https://files.pythonhosted.org/packages/5f/b3/966c2979172a46f9fe42f34ce7321a59102054e26cdf9b26e3d604807953/psutil-5.1.3.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.1.3.win32-py3.6.exe","hashes":{"sha256":"0961ebc2ba4b1c811ef164612d0d963532ad0a9af1755e022a99648a9027b065"},"provenance":null,"requires-python":null,"size":660315,"upload-time":"2017-02-07T21:35:48.440270Z","url":"https://files.pythonhosted.org/packages/be/43/8f0099425146c01c2d77e3ac90b28a7f42d69ccb2af6e161f059db132d99/psutil-5.1.3.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"1b02bec8364ab9ba69161e6031bf88699daaa9da96dcfa6527b452f562d8d646"},"data-dist-info-metadata":{"sha256":"1b02bec8364ab9ba69161e6031bf88699daaa9da96dcfa6527b452f562d8d646"},"filename":"psutil-5.2.0-cp27-none-win32.whl","hashes":{"sha256":"6eb2f6fb976152f320ee48a90ab732d694b2ae0c835260ce4f5af3907584448a"},"provenance":null,"requires-python":null,"size":187506,"upload-time":"2017-03-05T04:50:46.045903Z","url":"https://files.pythonhosted.org/packages/d2/56/56a15e285c7cf0104ed9fc569b2c75a24f97e7ab5c34567956b266d23ba3/psutil-5.2.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"1b02bec8364ab9ba69161e6031bf88699daaa9da96dcfa6527b452f562d8d646"},"data-dist-info-metadata":{"sha256":"1b02bec8364ab9ba69161e6031bf88699daaa9da96dcfa6527b452f562d8d646"},"filename":"psutil-5.2.0-cp27-none-win_amd64.whl","hashes":{"sha256":"35898b80a3f393a7ace8ad5da9a26800676b7fc40628a3a334902b9d0e444c8d"},"provenance":null,"requires-python":null,"size":189973,"upload-time":"2017-03-05T04:50:49.198757Z","url":"https://files.pythonhosted.org/packages/a5/a5/1039829542b856ca4d3d40bb4978fbb679b7f0bb684ece6340ce655aedc9/psutil-5.2.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp33-cp33m-win32.whl","hashes":{"sha256":"5626533fc459ce1ac4bd017f7a38b99947c039d79175a10a2a6b6246e3a82fc8"},"provenance":null,"requires-python":null,"size":187442,"upload-time":"2017-03-05T04:50:52.517268Z","url":"https://files.pythonhosted.org/packages/8c/1e/7e6ac521b3c393b2f312f1c3795d702f3267dca23d603827d673b8170920/psutil-5.2.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"d34cc4d48245873492e4befc5c58a146f0f6c98038ffa2430e191a6752717c61"},"provenance":null,"requires-python":null,"size":189849,"upload-time":"2017-03-05T04:50:55.188058Z","url":"https://files.pythonhosted.org/packages/b6/d0/6edd271e3ca150104c818ec0f4b2affc447fe79ec1504506cecb2900d391/psutil-5.2.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp34-cp34m-win32.whl","hashes":{"sha256":"5834168071a92037736142616b33691ec4786f8806e28355e74b2e1a037cad4c"},"provenance":null,"requires-python":null,"size":187450,"upload-time":"2017-03-05T04:50:58.524346Z","url":"https://files.pythonhosted.org/packages/89/88/8fb4ce470a2022c33ab3cd16b3f2152f544e264c9db0f2f7159a93e0d2a3/psutil-5.2.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"d29c24bc7c14ecb4e64b3b748814ebe0e3ac049802ea7f129edbfcb068e75c16"},"provenance":null,"requires-python":null,"size":189841,"upload-time":"2017-03-05T04:51:03.287652Z","url":"https://files.pythonhosted.org/packages/43/9b/35cae8c56d3ee2e9a02599fba6a2e1f3fcf3553fe55f70c0ea723f9a9522/psutil-5.2.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp35-cp35m-win32.whl","hashes":{"sha256":"8353692da46bc6024b4001a9ed8849beb863fbb1d022553dd4ed8348745540bb"},"provenance":null,"requires-python":null,"size":189373,"upload-time":"2017-03-05T04:51:07.095988Z","url":"https://files.pythonhosted.org/packages/65/35/fff62f84dc6c165e8a9f7646e2c106bd223a3967a0a3f471979b38b5a5c0/psutil-5.2.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"1e00f5684fb335dacfa750e5e01f83bb79d521eb5f0805b798de0a29a1fb25d4"},"provenance":null,"requires-python":null,"size":192813,"upload-time":"2017-03-05T04:51:10.595180Z","url":"https://files.pythonhosted.org/packages/94/d2/f78b5a0ded0993f4c5127bf17427e4bc10b183dc102a5e469d7f6725ecb9/psutil-5.2.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp36-cp36m-win32.whl","hashes":{"sha256":"55d546333f1423ad219a0798867a9bbf9a90e1912c3336ad275476473624c071"},"provenance":null,"requires-python":null,"size":189376,"upload-time":"2017-03-05T04:51:15.088897Z","url":"https://files.pythonhosted.org/packages/91/8e/bd4f794b9f092d82a5b63b17da95ebd864f544ff62fb70bb1bce0687b013/psutil-5.2.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"data-dist-info-metadata":{"sha256":"49ff4351990c237dc58c60a958925eef7156ce86008b0f5cfef7af0e197da80e"},"filename":"psutil-5.2.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"548f14e3e21225884904e3ab228a769a73f886a3394399c591ec5f31fedc48ac"},"provenance":null,"requires-python":null,"size":192808,"upload-time":"2017-03-05T04:51:18.018177Z","url":"https://files.pythonhosted.org/packages/75/65/8499f256dc203b94f8a439f52b092742247668365dcb0997aee7349d530d/psutil-5.2.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.tar.gz","hashes":{"sha256":"2fc91d068faa5613c093335f0e758673ef8c722ad4bfa4aded64c13ae69089eb"},"provenance":null,"requires-python":null,"size":345519,"upload-time":"2017-03-05T04:51:23.230758Z","url":"https://files.pythonhosted.org/packages/3c/2f/f3ab91349c666f009077157b12057e613a3152a46a6c3be883777546b6de/psutil-5.2.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py2.7.exe","hashes":{"sha256":"7cd5dd38e08d74112c68a8f4e9b9e12fac6c6f6270792604c79a9bbd574053fa"},"provenance":null,"requires-python":null,"size":426460,"upload-time":"2017-03-05T04:51:28.726871Z","url":"https://files.pythonhosted.org/packages/ca/6f/6289db524b6aae542fa36d539524e74f25d7f9296aabadb3b5a9f17746e8/psutil-5.2.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py3.3.exe","hashes":{"sha256":"e5f1688d9bfd9e122edd35adcd8a0050430397094d08d95c380bd9c7dae48da3"},"provenance":null,"requires-python":null,"size":424854,"upload-time":"2017-03-05T04:51:34.189840Z","url":"https://files.pythonhosted.org/packages/93/7c/e92a80f5803be3febbcb40807d5d2bfe66dfe20b256c07616599c14ba2aa/psutil-5.2.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py3.4.exe","hashes":{"sha256":"11e684bf163821bd73843ebf9a27b7cb6f8a8325b943954b49b1f622264b6e80"},"provenance":null,"requires-python":null,"size":424846,"upload-time":"2017-03-05T04:51:38.965015Z","url":"https://files.pythonhosted.org/packages/4a/82/5a78b9d40c17dc4d01a06345596ac1e3f7ba590f7329d83e80b817d47f9b/psutil-5.2.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py3.5.exe","hashes":{"sha256":"9eeae9bef0875432b9aea9504ed4f7c72f8ee3e8d7ded63e484a453ee82d4a98"},"provenance":null,"requires-python":null,"size":793372,"upload-time":"2017-03-05T04:51:43.517651Z","url":"https://files.pythonhosted.org/packages/dd/f1/4bf05d2b34198954b5fd6455ebe06dd08cae5357354f6f142ef4321e41ab/psutil-5.2.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win-amd64-py3.6.exe","hashes":{"sha256":"461445afc35f98d4b7781ea2f67d0ca4ae6cf36c065ee0f4be61ace59045a2a5"},"provenance":null,"requires-python":null,"size":795416,"upload-time":"2017-03-05T04:51:53.165366Z","url":"https://files.pythonhosted.org/packages/94/59/c54e7f853586561ae42c9bcf1aa0644e8c0298479e4654b4ca36fa9eafe6/psutil-5.2.0.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py2.7.exe","hashes":{"sha256":"8bbbd02eb474045d201f6617d16dc8ee1d9903d5cec94f7f39cce610fc1e924b"},"provenance":null,"requires-python":null,"size":396349,"upload-time":"2017-03-05T04:51:59.453206Z","url":"https://files.pythonhosted.org/packages/21/79/40ea4e11ef6ca2b044d0aeb28d829a716a565b34c7786f779e99c005b80a/psutil-5.2.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py3.3.exe","hashes":{"sha256":"63d4320b0f3498da3551028a6ab9ee1c5aebabe0d23a7c38600c35333953ef6c"},"provenance":null,"requires-python":null,"size":391219,"upload-time":"2017-03-05T04:52:03.289639Z","url":"https://files.pythonhosted.org/packages/2e/9d/def6a3fb8150adfd71889f3e3d48160a1ba6210911baf12ce3ebd294307c/psutil-5.2.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py3.4.exe","hashes":{"sha256":"3c06b0162192db85e04846674a55915fca80f728cf626228a6b31684fc6930da"},"provenance":null,"requires-python":null,"size":391229,"upload-time":"2017-03-05T04:52:10.103932Z","url":"https://files.pythonhosted.org/packages/2c/ae/8616ac1eb00a7770d837b15ebb9ae759c43623c182f32fd43d2e6fed8649/psutil-5.2.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py3.5.exe","hashes":{"sha256":"0ba082468d6b45fb15cc1c4488aaf3ffcf0616a674c46393bf04eccc8d7c2196"},"provenance":null,"requires-python":null,"size":660405,"upload-time":"2017-03-05T04:52:18.130601Z","url":"https://files.pythonhosted.org/packages/f7/b6/c8cb94fd6696414a66021aa2229747d71612551eade262e9ab52eeb54ee2/psutil-5.2.0.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.0.win32-py3.6.exe","hashes":{"sha256":"6ed1cb1c9339493e1f3c379de0155c543a4c8de18224bda894190843f9509cad"},"provenance":null,"requires-python":null,"size":662455,"upload-time":"2017-03-05T04:52:27.540851Z","url":"https://files.pythonhosted.org/packages/5d/9d/8b552e9d4c2a5c3baa00d1baa1468f2a8128acd3eba79ef39e59c182676a/psutil-5.2.0.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"84b8c9735984f276c69b5871ce5635af6381a960dd516077459ef1ded0e0580a"},"data-dist-info-metadata":{"sha256":"84b8c9735984f276c69b5871ce5635af6381a960dd516077459ef1ded0e0580a"},"filename":"psutil-5.2.1-cp27-none-win32.whl","hashes":{"sha256":"4e236c4ec6b0b20171c2477ded7a5b4402e4a877530640f814df839af0a40e30"},"provenance":null,"requires-python":null,"size":187855,"upload-time":"2017-03-24T15:42:20.835636Z","url":"https://files.pythonhosted.org/packages/3d/14/1242a70873873e92732dc35162317df448503a7a32e29c8bdbe30d4fa175/psutil-5.2.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"84b8c9735984f276c69b5871ce5635af6381a960dd516077459ef1ded0e0580a"},"data-dist-info-metadata":{"sha256":"84b8c9735984f276c69b5871ce5635af6381a960dd516077459ef1ded0e0580a"},"filename":"psutil-5.2.1-cp27-none-win_amd64.whl","hashes":{"sha256":"e88fe0d0ca5a9623f0d8d6be05a82e33984f27b067f08806bf8a548ba4361b40"},"provenance":null,"requires-python":null,"size":190301,"upload-time":"2017-03-24T15:42:26.623783Z","url":"https://files.pythonhosted.org/packages/4c/03/fffda9f6e1ca56ce989362969b709bf7a7ade16abf7d82661bbec96580f5/psutil-5.2.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp33-cp33m-win32.whl","hashes":{"sha256":"54275bdbfbd20909d37ed7a2570cf9dd373ac702a89bac4814249cbc10503c03"},"provenance":null,"requires-python":null,"size":187792,"upload-time":"2017-03-24T15:42:30.952153Z","url":"https://files.pythonhosted.org/packages/d7/e0/4fde7667fad4271c06ed5e533a156bd600cdad1b69d8e6f278fe425452d2/psutil-5.2.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"316c3e334b046dc12b4f0a3dafa1d1c394e38106ac519003694fc8aeb672eafd"},"provenance":null,"requires-python":null,"size":190190,"upload-time":"2017-03-24T15:42:35.956721Z","url":"https://files.pythonhosted.org/packages/a8/c5/63453c20ac576ccb58ee56f88388434380f5e2a729aa08885d2655eb83b7/psutil-5.2.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp34-cp34m-win32.whl","hashes":{"sha256":"2249c687088145dcce87ecb90221258f9c0e7b7cea830886656cf07351e50e1b"},"provenance":null,"requires-python":null,"size":187785,"upload-time":"2017-03-24T15:42:40.746327Z","url":"https://files.pythonhosted.org/packages/59/8b/8ebb86ae5c0ba81e95bae8263de81038d3d7ee8a050f31b2b58f1a330198/psutil-5.2.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"f74532c2037fac87b76737798c74102e17f8594ea9de07aa3cb19027a630bdb0"},"provenance":null,"requires-python":null,"size":190213,"upload-time":"2017-03-24T15:42:45.196850Z","url":"https://files.pythonhosted.org/packages/e7/81/c4dd47453864984d1bd5ad0c387efc11aa6791b5abb5b369ebe2e81f7ada/psutil-5.2.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp35-cp35m-win32.whl","hashes":{"sha256":"c7c8ed864a9ef04d4736a998273e3ba0f95f22300f1e082c13a7c824b514f411"},"provenance":null,"requires-python":null,"size":189737,"upload-time":"2017-03-24T15:42:50.097781Z","url":"https://files.pythonhosted.org/packages/76/3b/2e6b3306dd2927fef9c81fdc29bc450beeb6f4bfe4cddec80260ab042900/psutil-5.2.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"7a21b9d908a3bf381cc160c157a06bfcea3c6402362b26a2489566914cea9cc5"},"provenance":null,"requires-python":null,"size":193175,"upload-time":"2017-03-24T15:42:55.344665Z","url":"https://files.pythonhosted.org/packages/de/ee/cf9ecf7cea0a984a360bc889bb0bf11335755d5b7d2be9d8399fe5dc01fb/psutil-5.2.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp36-cp36m-win32.whl","hashes":{"sha256":"d1efbfc743555e7fd366956d8fe39690a3ae87e8e9e9ac06cc80bd7e2ca3059b"},"provenance":null,"requires-python":null,"size":189738,"upload-time":"2017-03-24T15:43:00.448274Z","url":"https://files.pythonhosted.org/packages/15/18/e6b1b4288d885218c845f9a340e236f03352358fc83675b9b8ef96e26227/psutil-5.2.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"data-dist-info-metadata":{"sha256":"187f675b1b9bc62e2ffab1ba395e87ae25e1679d8867bd305462e360a04fd386"},"filename":"psutil-5.2.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"cf40e944f47000375320ce0e712585321ec624a0ef67e8259f522e51bcb35a35"},"provenance":null,"requires-python":null,"size":193175,"upload-time":"2017-03-24T15:43:06.043078Z","url":"https://files.pythonhosted.org/packages/e8/7c/240fd3dfcec8d839a9a48dd2f88ba5f6e687263adc8b2452ed973b66b862/psutil-5.2.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.tar.gz","hashes":{"sha256":"fe0ea53b302f68fca1c2a3bac289e11344456786141b73391ed4022b412d5455"},"provenance":null,"requires-python":null,"size":347241,"upload-time":"2017-03-24T15:43:12.551784Z","url":"https://files.pythonhosted.org/packages/b8/47/c85fbcd23f40892db6ecc88782beb6ee66d22008c2f9821d777cb1984240/psutil-5.2.1.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py2.7.exe","hashes":{"sha256":"60e9bd558d640eaf9c7a4fbb0627b423b1e58fce95b41b8a24fda9145b753471"},"provenance":null,"requires-python":null,"size":426778,"upload-time":"2017-03-24T15:43:20.982480Z","url":"https://files.pythonhosted.org/packages/88/e8/40e20ea582157c81e55e1765139a5f6e969d8c01e47c016d90946b495531/psutil-5.2.1.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py3.3.exe","hashes":{"sha256":"318cf7bf546a23564fe4f049eae0bf205895a0524120bd549de3e46599a7f265"},"provenance":null,"requires-python":null,"size":425186,"upload-time":"2017-03-24T15:43:29.554448Z","url":"https://files.pythonhosted.org/packages/dd/55/2a74e973eb217fa5006c910a24abbd720efb7720beae9659be14fe96a413/psutil-5.2.1.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py3.4.exe","hashes":{"sha256":"3bcfbe8b8141c8145f1d54c3f9c2c86597508bb7cc2552e333de770a3c9b9368"},"provenance":null,"requires-python":null,"size":425211,"upload-time":"2017-03-24T15:43:39.017569Z","url":"https://files.pythonhosted.org/packages/9d/12/92575d652d33d28e6f8b0f858f3db326db5ffc4c8d55b09ac411b021d86d/psutil-5.2.1.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py3.5.exe","hashes":{"sha256":"979a5804366b47acd0ebf28923ee645e9fc29f4c54cbc44c41d112a1cd36e9ba"},"provenance":null,"requires-python":null,"size":793725,"upload-time":"2017-03-24T15:43:54.421574Z","url":"https://files.pythonhosted.org/packages/7f/58/de0b10442e2f277de4de0ecfae277576a6bfac1a3137fe547a4085dafa32/psutil-5.2.1.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win-amd64-py3.6.exe","hashes":{"sha256":"bf7d2cff21e3262d2b3e33a4b9dc27bbae81e851d694667d68dc7405c67ff31f"},"provenance":null,"requires-python":null,"size":795772,"upload-time":"2017-03-24T15:44:09.051326Z","url":"https://files.pythonhosted.org/packages/33/c0/7094de6644330b8dcdfefb0bae0a00379238588a6cf6cc9cd71c69e0cdce/psutil-5.2.1.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py2.7.exe","hashes":{"sha256":"ad8b603d7cc6d070cf07d39276869683474dace4da51d8050f29893ac2e22baf"},"provenance":null,"requires-python":null,"size":396688,"upload-time":"2017-03-24T15:44:17.148742Z","url":"https://files.pythonhosted.org/packages/a9/7a/5d19102362c28b6a478f9a7f3262f3ca301f8c5fed12e8d0af9e9e82e6a2/psutil-5.2.1.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py3.3.exe","hashes":{"sha256":"71fbaa3649aa8fa92edb1ad2b45de1e9caa7ffc63f448be951d43d6b5c6263b1"},"provenance":null,"requires-python":null,"size":391560,"upload-time":"2017-03-24T15:44:25.069190Z","url":"https://files.pythonhosted.org/packages/0b/01/f2963d84b439b0802c2354d0f777b5ed4bd0c2c11161ba81e7057a0d0523/psutil-5.2.1.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py3.4.exe","hashes":{"sha256":"cab6e8cfab49511f34e7ae40885792d7e655bb107f6f3c89440d5061cb19ad2f"},"provenance":null,"requires-python":null,"size":391554,"upload-time":"2017-03-24T15:44:33.671369Z","url":"https://files.pythonhosted.org/packages/d5/46/b36ff70ba0ba3b92bb5088be595fdb5641ffd982bac8e206e7c4936b2dc5/psutil-5.2.1.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py3.5.exe","hashes":{"sha256":"0e9e3d74f6ee1a6cac503c0bba08563dc3954e723b8392a4c74ce36f46e119ea"},"provenance":null,"requires-python":null,"size":660759,"upload-time":"2017-03-24T15:44:45.335659Z","url":"https://files.pythonhosted.org/packages/62/81/e7431ad75f9d9ae1524ee886c1aff25ec3714058de6568d305de2e0c8373/psutil-5.2.1.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.1.win32-py3.6.exe","hashes":{"sha256":"03e419618c3c715489ca5073cbdac6a0b12da41def69d3e4ee83f18fbb5798e5"},"provenance":null,"requires-python":null,"size":662807,"upload-time":"2017-03-24T15:44:56.446939Z","url":"https://files.pythonhosted.org/packages/77/c8/e256a28a63d06fe028f8837b860b7f6440c6ef9a475fb8c4490e1e08498b/psutil-5.2.1.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"0414f56fbdc05bdf6b0f2659c3f90672e666ae73ba46b3281e07291af1d58219"},"data-dist-info-metadata":{"sha256":"0414f56fbdc05bdf6b0f2659c3f90672e666ae73ba46b3281e07291af1d58219"},"filename":"psutil-5.2.2-cp27-none-win32.whl","hashes":{"sha256":"db473f0d45a56d422502043f3755385fcfd83f5bb0947bc807fcad689230f37f"},"provenance":null,"requires-python":null,"size":187988,"upload-time":"2017-04-10T17:19:35.132698Z","url":"https://files.pythonhosted.org/packages/9c/31/c651e4c475a4d0df9609024a86fcb358a21b7a01872f7c69c7cf501a2896/psutil-5.2.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"0414f56fbdc05bdf6b0f2659c3f90672e666ae73ba46b3281e07291af1d58219"},"data-dist-info-metadata":{"sha256":"0414f56fbdc05bdf6b0f2659c3f90672e666ae73ba46b3281e07291af1d58219"},"filename":"psutil-5.2.2-cp27-none-win_amd64.whl","hashes":{"sha256":"dcd9d3131f83480648da40d2c39403657c63a81e56e4e8d8e905bf65c133d59c"},"provenance":null,"requires-python":null,"size":190432,"upload-time":"2017-04-10T17:19:39.867178Z","url":"https://files.pythonhosted.org/packages/6e/5c/15f41041a321ffd4058ae67aade067924489acba6d277e10571b59b3127c/psutil-5.2.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp33-cp33m-win32.whl","hashes":{"sha256":"3f79a044db0aae96592ef42be459e37095d0c2cebcae4fd7baf486d37a85a8cd"},"provenance":null,"requires-python":null,"size":187924,"upload-time":"2017-04-10T17:19:44.464637Z","url":"https://files.pythonhosted.org/packages/04/a5/a027a8584208fa2cb6a88e6337b06b11388edf7d39feb0a897c9c2024639/psutil-5.2.2-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"838c66c123cb024bf8c8d2fec902b38c51f75b27988f4487d81383d1d3d8a8ce"},"provenance":null,"requires-python":null,"size":190325,"upload-time":"2017-04-10T17:19:49.563017Z","url":"https://files.pythonhosted.org/packages/bc/95/385e0f7e0299295401d41dd4cb6e568bf50c884af336b92a69d16981f71c/psutil-5.2.2-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp34-cp34m-win32.whl","hashes":{"sha256":"a155875d2fedb614c2cd687fe47953d03a47f76eb39bd5756931b288b685655f"},"provenance":null,"requires-python":null,"size":187918,"upload-time":"2017-04-10T17:19:54.252736Z","url":"https://files.pythonhosted.org/packages/12/ad/aca0f4f146b25fb2b7e9e0735287ba3ebcc02eb2bf84d49916aef730d860/psutil-5.2.2-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"a989876ac0cc7942ef9481b96d3bfc02777dc798d4a7a1b4e8f0f284228f3434"},"provenance":null,"requires-python":null,"size":190345,"upload-time":"2017-04-10T17:19:59.043821Z","url":"https://files.pythonhosted.org/packages/f4/45/6cbf2b7a55375f6aafc68f33581aa143f86ae1be9112546f04d8e9ee34da/psutil-5.2.2-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp35-cp35m-win32.whl","hashes":{"sha256":"32616c5736f1de446e77865305e7f56905c718991f820c8286436adea8192f32"},"provenance":null,"requires-python":null,"size":189869,"upload-time":"2017-04-10T17:20:03.832890Z","url":"https://files.pythonhosted.org/packages/7b/1d/8cef4ee6c1a49b1204dcdca1231ac773e27f2ed0abbbf42deb14aaf2b5cc/psutil-5.2.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"50c8ddc3a6d1cda1de6d7aaf1af10896832c6d686fc7d0fe3d01c1eb51e6f521"},"provenance":null,"requires-python":null,"size":193304,"upload-time":"2017-04-10T17:20:09.444679Z","url":"https://files.pythonhosted.org/packages/82/f4/9d4cb35c5e1c84f93718d1851adf0b4147b253111cb89f1996a08d14dba5/psutil-5.2.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp36-cp36m-win32.whl","hashes":{"sha256":"e8b65a80e978af9bf10be423442155032c589b7042b4a26edc410dc36819d65e"},"provenance":null,"requires-python":null,"size":189868,"upload-time":"2017-04-10T17:20:16.876191Z","url":"https://files.pythonhosted.org/packages/72/9f/5ff6e45db392bc9dad642dffcca44eeee552289595c087a4f1d245fdb4f9/psutil-5.2.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"data-dist-info-metadata":{"sha256":"4c73be3d0927068fd96000b5591a3206eb03f1decaf76929c5e5cd06d004e214"},"filename":"psutil-5.2.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"7a5c0973bd4c1de98d9b225bd4303a0718d31e31d6e2342e825c3e656f7056df"},"provenance":null,"requires-python":null,"size":193305,"upload-time":"2017-04-10T17:20:21.983057Z","url":"https://files.pythonhosted.org/packages/eb/c6/29c695be774c52cca9bb68b94ae4dc866a42ddf29dcd19b7ab4c0d97bdda/psutil-5.2.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.tar.gz","hashes":{"sha256":"44746540c0fab5b95401520d29eb9ffe84b3b4a235bd1d1971cbe36e1f38dd13"},"provenance":null,"requires-python":null,"size":348413,"upload-time":"2017-04-10T17:20:29.132011Z","url":"https://files.pythonhosted.org/packages/57/93/47a2e3befaf194ccc3d05ffbcba2cdcdd22a231100ef7e4cf63f085c900b/psutil-5.2.2.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py2.7.exe","hashes":{"sha256":"70732850abd11f4d9fa46f0e110af21030e0a6088204f332d335921b36e66305"},"provenance":null,"requires-python":null,"size":427159,"upload-time":"2017-04-10T17:20:36.896101Z","url":"https://files.pythonhosted.org/packages/56/bb/d03fa2260839abbcdd4d323e83c2e91ffaedcb1975b88d5d0bc71c95c1fb/psutil-5.2.2.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py3.3.exe","hashes":{"sha256":"5d2f076788d71d2e1c7276f1e5a1bc255f29c2e80eb8879a9ffc633c5bf69481"},"provenance":null,"requires-python":null,"size":425568,"upload-time":"2017-04-10T17:20:45.676662Z","url":"https://files.pythonhosted.org/packages/79/fd/3d2626e6a9fd4d99859fd8eac7d52ff9850d5e4ea62611a1e3ffe6f3d257/psutil-5.2.2.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py3.4.exe","hashes":{"sha256":"fecda42b274dc618278bd9139e8493c9459d2174376f82b65ba929557f10e880"},"provenance":null,"requires-python":null,"size":425592,"upload-time":"2017-04-10T17:20:54.334109Z","url":"https://files.pythonhosted.org/packages/24/56/637ef0dfac83cd3e51096436faf7ea030f780aff3da98a79b7e7ac98a8bb/psutil-5.2.2.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py3.5.exe","hashes":{"sha256":"92e3500dfaf7a5502ebaf4a7472e2afb9ff0cb36b4e5dc1977b3c774f58332db"},"provenance":null,"requires-python":null,"size":794103,"upload-time":"2017-04-10T17:21:08.107023Z","url":"https://files.pythonhosted.org/packages/09/4d/9cf34797696c0a75fd76606c362ddfbbc0f87d2c19c95ae26e61120241ad/psutil-5.2.2.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win-amd64-py3.6.exe","hashes":{"sha256":"ed09521d49ee177f1205ed9791ad62263feacd2fe1cc20d1d33cf37923f240ea"},"provenance":null,"requires-python":null,"size":796151,"upload-time":"2017-04-10T17:21:22.493104Z","url":"https://files.pythonhosted.org/packages/04/74/5ea4412f31c9652335c03537ff4608ff6a12895fb3a34a187578d693b865/psutil-5.2.2.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py2.7.exe","hashes":{"sha256":"147093b75b8874e55e6b26c540544d40e98845bc4ee74dc6054c881fd2a3eed9"},"provenance":null,"requires-python":null,"size":397069,"upload-time":"2017-04-10T17:21:30.834225Z","url":"https://files.pythonhosted.org/packages/b9/aa/b779310ee8a120b5bb90880d22e7b3869f98f7a30381a71188c6fb7ff4a6/psutil-5.2.2.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py3.3.exe","hashes":{"sha256":"3d3c5c117e55c486a53ef796cc715035bf4f56419cc32dbd124fe26e9289ad1e"},"provenance":null,"requires-python":null,"size":391941,"upload-time":"2017-04-10T17:21:38.680258Z","url":"https://files.pythonhosted.org/packages/e0/3f/fd40d4edbfac1d7996b6ecb3bfa5f923662e21686727eb070490ab6dcca8/psutil-5.2.2.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py3.4.exe","hashes":{"sha256":"0c74c6a494b650966b88da256cab4e507f483c53e85b9b10d3ff9c38f059330b"},"provenance":null,"requires-python":null,"size":391936,"upload-time":"2017-04-10T17:21:46.576288Z","url":"https://files.pythonhosted.org/packages/c7/59/f7aa53e3f72d6dcfce7c60d80e74853a1444a8a4d7fef788441435c645bf/psutil-5.2.2.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py3.5.exe","hashes":{"sha256":"b5583d1c2c858056d39bd148ed25839c4f1b76fec8fb2cb9b564c82997a21266"},"provenance":null,"requires-python":null,"size":661141,"upload-time":"2017-04-10T17:21:57.551743Z","url":"https://files.pythonhosted.org/packages/f2/62/0fc0c5459bcc04171ea3bf5603622db6815ddd92a472db06bcd69612177d/psutil-5.2.2.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.2.2.win32-py3.6.exe","hashes":{"sha256":"1da0aa70d66612588d77daed7784e623aac1fd038681c3acd0e1c76b2b2f0819"},"provenance":null,"requires-python":null,"size":663188,"upload-time":"2017-04-10T17:22:09.360293Z","url":"https://files.pythonhosted.org/packages/a0/7f/494b600e45a8a25a76658a45747f232d38538d1c177f5b80123902d2b8da/psutil-5.2.2.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"d331b301ac1684996cf5629df9922a8709a7bf9243a73151f9aa72cf620ac815"},"data-dist-info-metadata":{"sha256":"d331b301ac1684996cf5629df9922a8709a7bf9243a73151f9aa72cf620ac815"},"filename":"psutil-5.3.0-cp27-none-win32.whl","hashes":{"sha256":"6f8f858cdb79397509ee067ae9d25bee8f4b4902453ac8d155fa1629f03aa39d"},"provenance":null,"requires-python":null,"size":210071,"upload-time":"2017-09-01T10:50:37.295614Z","url":"https://files.pythonhosted.org/packages/db/36/71afb537f3718a00a9e63b75e8534bf14fee38b67fb7cb72eb60a3378162/psutil-5.3.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"d331b301ac1684996cf5629df9922a8709a7bf9243a73151f9aa72cf620ac815"},"data-dist-info-metadata":{"sha256":"d331b301ac1684996cf5629df9922a8709a7bf9243a73151f9aa72cf620ac815"},"filename":"psutil-5.3.0-cp27-none-win_amd64.whl","hashes":{"sha256":"b31d6d19e445b56559abaa21703a6bc4b162aaf9ab99867b6f2bbbdb2c7fce66"},"provenance":null,"requires-python":null,"size":212901,"upload-time":"2017-09-01T10:50:41.892260Z","url":"https://files.pythonhosted.org/packages/c8/2b/9bc89bb0d8a2ac49ab29954017e65a9c28b83b13453418c8166938281458/psutil-5.3.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"47a95c843a21d16ad147a795cf6dc0395740dd91d643d06e100a63f2bb9f4d08"},"data-dist-info-metadata":{"sha256":"47a95c843a21d16ad147a795cf6dc0395740dd91d643d06e100a63f2bb9f4d08"},"filename":"psutil-5.3.0-cp33-cp33m-win32.whl","hashes":{"sha256":"7f1ba5011095e39b3f543e9c87008409dd8a57a3e48ea1022c348244b5af77bf"},"provenance":null,"requires-python":null,"size":209934,"upload-time":"2017-09-01T10:50:46.901965Z","url":"https://files.pythonhosted.org/packages/3c/5b/b020c5f5b6fbe69bfe77072a6d35a6c6d68f7ec7fe8b148223d9365bf8b4/psutil-5.3.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"47a95c843a21d16ad147a795cf6dc0395740dd91d643d06e100a63f2bb9f4d08"},"data-dist-info-metadata":{"sha256":"47a95c843a21d16ad147a795cf6dc0395740dd91d643d06e100a63f2bb9f4d08"},"filename":"psutil-5.3.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"853f68a85cec0137acf0504d8ca6d40d899e48ecbe931130f593a072a35b812e"},"provenance":null,"requires-python":null,"size":212722,"upload-time":"2017-09-01T10:50:51.125180Z","url":"https://files.pythonhosted.org/packages/ad/f8/4d7f713241f1786097faf48afb51e6cb9f7966d2fc36656098e182056704/psutil-5.3.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp34-cp34m-win32.whl","hashes":{"sha256":"01d9cb9473eee0e7e88319f9a5205a69e6e160b3ab2bd430a05b93bfae1528c2"},"provenance":null,"requires-python":null,"size":209879,"upload-time":"2017-09-01T10:50:55.651560Z","url":"https://files.pythonhosted.org/packages/d6/77/01a752e6d05061decf570acc800cd490766ea4534eccbe8c523b84fe5cc1/psutil-5.3.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"91d37262095c1a0f97a78f5034e10e0108e3fa326c85baa17f8cdd63fa5f81b9"},"provenance":null,"requires-python":null,"size":212614,"upload-time":"2017-09-01T10:50:59.871475Z","url":"https://files.pythonhosted.org/packages/0f/cf/fd1d752d428c5845fed4904e7bcdbb89ea3327aa063247fddcdca319c615/psutil-5.3.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp35-cp35m-win32.whl","hashes":{"sha256":"bd1776dc14b197388d728db72c103c0ebec834690ef1ce138035abf0123e2268"},"provenance":null,"requires-python":null,"size":212062,"upload-time":"2017-09-01T10:51:04.687879Z","url":"https://files.pythonhosted.org/packages/68/05/6d097706fd9cb43eda36a02a5feaee085aeac21f5bc6ea0b557109bc2eca/psutil-5.3.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"7fadb1b1357ef58821b3f1fc2afb6e1601609b0daa3b55c2fabf765e0ea98901"},"provenance":null,"requires-python":null,"size":215734,"upload-time":"2017-09-01T10:51:09.110127Z","url":"https://files.pythonhosted.org/packages/6e/5a/7c688b472ff5b6cb5413acfa5a178b5e8140ffbdf0011b6d0469e97af3b1/psutil-5.3.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp36-cp36m-win32.whl","hashes":{"sha256":"d5f4634a19e7d4692f37d8d67f8418f85f2bc1e2129914ec0e4208bf7838bf63"},"provenance":null,"requires-python":null,"size":212060,"upload-time":"2017-09-01T10:51:14.595884Z","url":"https://files.pythonhosted.org/packages/28/30/4ab277d7e37cd5ee1c47a89d21465c3eec3435b973ca86cd986efdd0aeac/psutil-5.3.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"data-dist-info-metadata":{"sha256":"70af37a6bd4674f8be60cec24e58ebdb4493432961ecc67d471a1739f1bf5d12"},"filename":"psutil-5.3.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"31505ee459913ef63fa4c1c0d9a11a4da60b5c5ec6a92d6d7f5d12b9653fc61b"},"provenance":null,"requires-python":null,"size":215731,"upload-time":"2017-09-01T10:51:18.736362Z","url":"https://files.pythonhosted.org/packages/c3/e6/98fd6259d8ed834d9a81567d4f41dd3645e12a9aea9d38563efaf245610a/psutil-5.3.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.tar.gz","hashes":{"sha256":"a3940e06e92c84ab6e82b95dad056241beea93c3c9b1d07ddf96485079855185"},"provenance":null,"requires-python":null,"size":397265,"upload-time":"2017-09-01T12:31:14.428985Z","url":"https://files.pythonhosted.org/packages/1c/da/555e3ad3cad30f30bcf0d539cdeae5c8e7ef9e2a6078af645c70aa81e418/psutil-5.3.0.tar.gz","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py2.7.exe","hashes":{"sha256":"ba94f021942d6cc27e18dcdccd2c1a0976f0596765ef412316ecb887d4fd3db2"},"provenance":null,"requires-python":null,"size":450426,"upload-time":"2017-09-01T10:51:26.068723Z","url":"https://files.pythonhosted.org/packages/e3/d6/238a22e898d0a3703d5fd486487108bf57d8fab1137bf085d3602d04894a/psutil-5.3.0.win-amd64-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py3.3.exe","hashes":{"sha256":"0f2fccf98bc25e8d6d61e24b2cc6350b8dfe8fa7f5251c817e977d8c61146e5d"},"provenance":null,"requires-python":null,"size":448781,"upload-time":"2017-09-01T10:51:33.193333Z","url":"https://files.pythonhosted.org/packages/91/d6/4ccff87a9f93f4837c97c3eb105c4cd1ccd544091658ba1938feca366b59/psutil-5.3.0.win-amd64-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py3.4.exe","hashes":{"sha256":"d06f02c53260d16fb445e426410263b2d271cea19136b1bb715cf10b76960359"},"provenance":null,"requires-python":null,"size":448549,"upload-time":"2017-09-01T10:51:40.339200Z","url":"https://files.pythonhosted.org/packages/74/29/91f18e7798a150c922c7da5c2153edecd7af0292332ef4f46e8ebd7184d3/psutil-5.3.0.win-amd64-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py3.5.exe","hashes":{"sha256":"724439fb20d083c943a2c62db1aa240fa15fe23644c4d4a1e9f573ffaf0bbddd"},"provenance":null,"requires-python":null,"size":817222,"upload-time":"2017-09-01T10:51:52.173626Z","url":"https://files.pythonhosted.org/packages/41/fe/5081186ce35c0def7db2f8531ccac908b83edf735c2b8b78241633853b34/psutil-5.3.0.win-amd64-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win-amd64-py3.6.exe","hashes":{"sha256":"a58708f3f6f74897450babb012cd8067f8911e7c8a1f2991643ec9937a8f6c15"},"provenance":null,"requires-python":null,"size":817218,"upload-time":"2017-09-01T10:52:03.695382Z","url":"https://files.pythonhosted.org/packages/aa/49/2076044f5f8554232eb5f8fb69b4a59c8f96da6d107d6f6a35aea2fb0344/psutil-5.3.0.win-amd64-py3.6.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py2.7.exe","hashes":{"sha256":"108dae5ecb68f6e6212bf0553be055a2a0eec210227d8e14c3a26368b118624a"},"provenance":null,"requires-python":null,"size":419952,"upload-time":"2017-09-01T10:52:12.411324Z","url":"https://files.pythonhosted.org/packages/d1/18/7e1a418aff3f65c3072eb765b0e51445df6309be3da25c5a4cb9f1d1d18b/psutil-5.3.0.win32-py2.7.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py3.3.exe","hashes":{"sha256":"9832124af1e9ec0f298f17ab11c3bb91164f8068ec9429c39a7f7a0eae637a94"},"provenance":null,"requires-python":null,"size":414764,"upload-time":"2017-09-01T10:52:32.286415Z","url":"https://files.pythonhosted.org/packages/de/46/0ec33721564e235fab9112fecb836eb084d0475ab97d5a6d5c462656e715/psutil-5.3.0.win32-py3.3.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py3.4.exe","hashes":{"sha256":"7b8d10e7d72862d1e97caba546b60ce263b3fcecd6176e4c94efebef87ee68d3"},"provenance":null,"requires-python":null,"size":414582,"upload-time":"2017-09-01T10:52:41.243220Z","url":"https://files.pythonhosted.org/packages/86/82/254439b29eea5670633a947ec3f67c5ec33b88322a1a443f121d21dc2714/psutil-5.3.0.win32-py3.4.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py3.5.exe","hashes":{"sha256":"ed1f7cbbbf778a6ed98e25d48fdbdc098e66b360427661712610d72c1b4cf5f5"},"provenance":null,"requires-python":null,"size":684021,"upload-time":"2017-09-01T10:52:51.603711Z","url":"https://files.pythonhosted.org/packages/20/9c/2d84e2926e1a89c4d1ea8fc315e05145e6b10af79459727ebb688b22dab8/psutil-5.3.0.win32-py3.5.exe","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.0.win32-py3.6.exe","hashes":{"sha256":"3d8d62f3da0b38dbfaf4756a32e18c866530b9066c298da3fc293cfefae22f0a"},"provenance":null,"requires-python":null,"size":684018,"upload-time":"2017-09-01T10:53:02.878374Z","url":"https://files.pythonhosted.org/packages/61/c7/bbcc29ba03d59f1add008245edf0d56d45434b137dde0c0d6b8441ae3be6/psutil-5.3.0.win32-py3.6.exe","yanked":false},{"core-metadata":{"sha256":"efaa364a2342964849b527ba97bfad9c165b3524ee431b8e5ff9d3b6bd3b4047"},"data-dist-info-metadata":{"sha256":"efaa364a2342964849b527ba97bfad9c165b3524ee431b8e5ff9d3b6bd3b4047"},"filename":"psutil-5.3.1-cp27-none-win32.whl","hashes":{"sha256":"7a669b1897b8cdce1cea79defdf3a10fd6e4f0a8e42ac2a971dfe74bc1ce5679"},"provenance":null,"requires-python":null,"size":210168,"upload-time":"2017-09-10T05:26:42.273181Z","url":"https://files.pythonhosted.org/packages/34/90/145ff234428b4bd519c20d460d6d51db7820e8120879ca9cdc88602c57f4/psutil-5.3.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"efaa364a2342964849b527ba97bfad9c165b3524ee431b8e5ff9d3b6bd3b4047"},"data-dist-info-metadata":{"sha256":"efaa364a2342964849b527ba97bfad9c165b3524ee431b8e5ff9d3b6bd3b4047"},"filename":"psutil-5.3.1-cp27-none-win_amd64.whl","hashes":{"sha256":"57be53c045f2085e28d5371eedfce804f5e49e7b35fa79bcf63e271046058002"},"provenance":null,"requires-python":null,"size":212998,"upload-time":"2017-09-10T05:27:13.646528Z","url":"https://files.pythonhosted.org/packages/c8/c9/d8cbfc3844e1a3e8b648fcca317ad8589283a7cbbc232c2c5d29cae88352/psutil-5.3.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"411e80cfca140c76586e1f0d89c1c6d7b1ff79f31751e6780aee2632e60c3996"},"data-dist-info-metadata":{"sha256":"411e80cfca140c76586e1f0d89c1c6d7b1ff79f31751e6780aee2632e60c3996"},"filename":"psutil-5.3.1-cp33-cp33m-win32.whl","hashes":{"sha256":"27d4c5ff3ab97389a9372d246e1aa27e5f02e4709fede48a0599f89d2873ca88"},"provenance":null,"requires-python":null,"size":210025,"upload-time":"2017-09-10T05:27:19.363106Z","url":"https://files.pythonhosted.org/packages/c4/82/7e412884fcf9ae538b1d96e31688b804377803e34f4e3e86bba869eaa9f7/psutil-5.3.1-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"411e80cfca140c76586e1f0d89c1c6d7b1ff79f31751e6780aee2632e60c3996"},"data-dist-info-metadata":{"sha256":"411e80cfca140c76586e1f0d89c1c6d7b1ff79f31751e6780aee2632e60c3996"},"filename":"psutil-5.3.1-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"72ba7e4c82879b3781ccced1eeb901f07725a36fab66270e7555e484a460760d"},"provenance":null,"requires-python":null,"size":212814,"upload-time":"2017-09-10T05:27:24.067035Z","url":"https://files.pythonhosted.org/packages/8a/1b/e185f211c9a959739c6789872458b78f6424466819afc5c420af09b222af/psutil-5.3.1-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp34-cp34m-win32.whl","hashes":{"sha256":"fc11c3a52990ec44064cbe026338dedcfff0e0027ca7516416eaa7d4f206c5af"},"provenance":null,"requires-python":null,"size":209976,"upload-time":"2017-09-10T05:27:29.519738Z","url":"https://files.pythonhosted.org/packages/ae/a9/ba609de04d2350878c6c3d641997dd37fa362775bf79aca3e6d542aae89e/psutil-5.3.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"7b1f9856c2fc9503a8a687db85e4f419ad1a10bfcab92ba786a7d43a6aa8cea0"},"provenance":null,"requires-python":null,"size":212710,"upload-time":"2017-09-10T05:27:34.832459Z","url":"https://files.pythonhosted.org/packages/9e/4e/e35f4e9b3f5dfb8eb88be75ccbc6e6a6428443afa4d641ff5e9e29a8991f/psutil-5.3.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp35-cp35m-win32.whl","hashes":{"sha256":"54781e463d9b9aa8c143033ee0d6a3149f9f143e6cc63099a95d4078f433dd56"},"provenance":null,"requires-python":null,"size":212159,"upload-time":"2017-09-10T05:27:39.270103Z","url":"https://files.pythonhosted.org/packages/79/4b/4531d21a7e428f3f25dc1b05be7e2024d9c2d45845ba005193dd9420e6b7/psutil-5.3.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"e9ef8d265298268cad784dfece103ab06bd726512d57fc6ed9f94b55452e4571"},"provenance":null,"requires-python":null,"size":215831,"upload-time":"2017-09-10T05:27:43.869238Z","url":"https://files.pythonhosted.org/packages/aa/a7/1bc0baaea0798c1b29bfb6cec05f18f6a4cbaaf96646818429d998feb2f5/psutil-5.3.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp36-cp36m-win32.whl","hashes":{"sha256":"f5d55618cd5b9270355fb52c0430ff30c4c84c5caf5b1254eec27f80d48e7a12"},"provenance":null,"requires-python":null,"size":212160,"upload-time":"2017-09-10T05:27:51.405015Z","url":"https://files.pythonhosted.org/packages/ba/03/1946dc720fec2083148b1d7b579e789bde85cebad67b22e05d5189871114/psutil-5.3.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"data-dist-info-metadata":{"sha256":"c340877ca68a4b72ce8f93ad0b6c48c15a2eeb72742c08998e478251efec5a92"},"filename":"psutil-5.3.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"773ba33fe365cb8b0998eedcbe494dc92ce7428998f07dca652a1360a9e2bce8"},"provenance":null,"requires-python":null,"size":215831,"upload-time":"2017-09-10T05:27:56.438641Z","url":"https://files.pythonhosted.org/packages/9b/de/5bd7038f8bb68516eedafc8bba9daf6740011db8afc1bb25cdcc8d654771/psutil-5.3.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.3.1.tar.gz","hashes":{"sha256":"12dd9c8abbad15f055e9579130035b38617020ce176f4a498b7870e6321ffa67"},"provenance":null,"requires-python":null,"size":397075,"upload-time":"2017-09-10T05:28:34.436466Z","url":"https://files.pythonhosted.org/packages/d3/0a/74dcbb162554909b208e5dbe9f4e7278d78cc27470993e05177005e627d0/psutil-5.3.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"dc4ac37cc269cb56f6fd4292649312b61d28d4970d9dce30eebae6899076846a"},"data-dist-info-metadata":{"sha256":"dc4ac37cc269cb56f6fd4292649312b61d28d4970d9dce30eebae6899076846a"},"filename":"psutil-5.4.0-cp27-none-win32.whl","hashes":{"sha256":"8121039d2280275ac82f99a0a48110450cbf5b356a11c842c8f5cdeafdf105e1"},"provenance":null,"requires-python":null,"size":218177,"upload-time":"2017-10-12T07:26:59.429237Z","url":"https://files.pythonhosted.org/packages/4c/89/08a536124b4ee1fd850982f59ab7268a359e4160a6bcc1d473f6971fbdd4/psutil-5.4.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"dc4ac37cc269cb56f6fd4292649312b61d28d4970d9dce30eebae6899076846a"},"data-dist-info-metadata":{"sha256":"dc4ac37cc269cb56f6fd4292649312b61d28d4970d9dce30eebae6899076846a"},"filename":"psutil-5.4.0-cp27-none-win_amd64.whl","hashes":{"sha256":"fcd93acb2602d01b86e0cfa4c2db689a81badae98d9c572348c94f1b2ea4b30d"},"provenance":null,"requires-python":null,"size":220988,"upload-time":"2017-10-12T07:27:04.404602Z","url":"https://files.pythonhosted.org/packages/c9/3d/0cd95044d1245166ddf24144e1b3e7a6b6a81933de3ff48c1851664109fc/psutil-5.4.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b165c036e28aeff5aef41800231918509ab9366f5817be37e7123de4bd7ce3a2"},"data-dist-info-metadata":{"sha256":"b165c036e28aeff5aef41800231918509ab9366f5817be37e7123de4bd7ce3a2"},"filename":"psutil-5.4.0-cp33-cp33m-win32.whl","hashes":{"sha256":"60a58bfdda1fc6e86ecf95c6eef71252d9049694df9aa0a16c2841a425fc9deb"},"provenance":null,"requires-python":null,"size":218040,"upload-time":"2017-10-12T07:27:13.087638Z","url":"https://files.pythonhosted.org/packages/e5/1a/0f27898ad585d924e768b0c7029c7d7ac429994a3a032419c51f1c1f3e41/psutil-5.4.0-cp33-cp33m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b165c036e28aeff5aef41800231918509ab9366f5817be37e7123de4bd7ce3a2"},"data-dist-info-metadata":{"sha256":"b165c036e28aeff5aef41800231918509ab9366f5817be37e7123de4bd7ce3a2"},"filename":"psutil-5.4.0-cp33-cp33m-win_amd64.whl","hashes":{"sha256":"9956b370243005d5561a94efa44b0cddb826d1f14d21958003925b008d3b9eb1"},"provenance":null,"requires-python":null,"size":220796,"upload-time":"2017-10-12T07:27:19.356245Z","url":"https://files.pythonhosted.org/packages/19/8e/b42236e04fbd03ffa2a08a393c8800fd7d51b11823c0e18723f0780d9b6a/psutil-5.4.0-cp33-cp33m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp34-cp34m-win32.whl","hashes":{"sha256":"104fec73d9ed573351f3efbf1b7ee19eb3b4097e2b3d9ff26b1ac5bca52b6f9e"},"provenance":null,"requires-python":null,"size":217969,"upload-time":"2017-10-12T07:27:24.486998Z","url":"https://files.pythonhosted.org/packages/11/42/0711b59b7f2f2f7de7912d30bc599950011d25401bda8a3330f878ff1e56/psutil-5.4.0-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"64b2814b30452d854d5f7a7c9c0d77423388b44eb2a8bcab3b84feeceaba8ffb"},"provenance":null,"requires-python":null,"size":220696,"upload-time":"2017-10-12T07:27:29.623184Z","url":"https://files.pythonhosted.org/packages/e6/67/e19ccc78646810cbc432c429e2993315cf50a126379b83b0f8b3eb9172b9/psutil-5.4.0-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp35-cp35m-win32.whl","hashes":{"sha256":"1a89ba967d4b9a3d5f19ea2c63b09e5ffb3a81de3116ead7bfb67b9c308e8dba"},"provenance":null,"requires-python":null,"size":220150,"upload-time":"2017-10-12T07:27:34.351564Z","url":"https://files.pythonhosted.org/packages/cc/c1/47edb3fccbb1354499297702ca55aec41ff7aab67b4df69aba9db4e52a7c/psutil-5.4.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"c8362902d4d94640c61960c18d8aa1af074822a549c3d4f137be3aa62c17f4b9"},"provenance":null,"requires-python":null,"size":223827,"upload-time":"2017-10-12T07:27:46.430105Z","url":"https://files.pythonhosted.org/packages/a9/e4/d399d060bbf40c1480ce2bfbc429edbd42769b048c4c7fdc51a49d74c9cf/psutil-5.4.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp36-cp36m-win32.whl","hashes":{"sha256":"04ed7548dfe61ab2561a94ac848a5b79239bb23e9015596ffd0644efd22461ba"},"provenance":null,"requires-python":null,"size":220151,"upload-time":"2017-10-12T07:27:51.153048Z","url":"https://files.pythonhosted.org/packages/2b/46/88fdd6206f01b04547bb89a6d2ff7eec380e5c59bda56ee03a0759ed397c/psutil-5.4.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"data-dist-info-metadata":{"sha256":"eed08ae034f087bf660376ef37f75441a8f3d124c69a4be6089c1e755219a000"},"filename":"psutil-5.4.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"346555603ea903a5524bb37a49b0e0f90960c6c9973ebf794ae0802a4aa875eb"},"provenance":null,"requires-python":null,"size":223825,"upload-time":"2017-10-12T07:27:57.513929Z","url":"https://files.pythonhosted.org/packages/34/e2/8421ca5b99209fa43e48d2f76a21d5d86d7346838053abf7a4d2aa37255e/psutil-5.4.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.0.tar.gz","hashes":{"sha256":"8e6397ec24a2ec09751447d9f169486b68b37ac7a8d794dca003ace4efaafc6a"},"provenance":null,"requires-python":null,"size":406945,"upload-time":"2017-10-12T07:22:51.321681Z","url":"https://files.pythonhosted.org/packages/8d/96/1fc6468be91521192861966c40bd73fdf8b065eae6d82dd0f870b9825a65/psutil-5.4.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"e0b722d181bbe7aba5016d4d14c510f34b0ceccccc534c1324af4b5bb619046f"},"data-dist-info-metadata":{"sha256":"e0b722d181bbe7aba5016d4d14c510f34b0ceccccc534c1324af4b5bb619046f"},"filename":"psutil-5.4.1-cp27-none-win32.whl","hashes":{"sha256":"7ef26ebe728ac821de17df23820e6ffcfd37c409fc865380e4d5ae1388f274a1"},"provenance":null,"requires-python":null,"size":218627,"upload-time":"2017-11-08T13:50:32.260924Z","url":"https://files.pythonhosted.org/packages/dd/dd/1811a99faefd2b5947b4e68bd70767b525fdbad65481a4bd2ee7e6408749/psutil-5.4.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e0b722d181bbe7aba5016d4d14c510f34b0ceccccc534c1324af4b5bb619046f"},"data-dist-info-metadata":{"sha256":"e0b722d181bbe7aba5016d4d14c510f34b0ceccccc534c1324af4b5bb619046f"},"filename":"psutil-5.4.1-cp27-none-win_amd64.whl","hashes":{"sha256":"692dc72817d157aae522231dd334ea2524c6b07d844db0e7a2d6897820083427"},"provenance":null,"requires-python":null,"size":221450,"upload-time":"2017-11-08T13:50:37.542045Z","url":"https://files.pythonhosted.org/packages/51/c1/eec6a42a9f5fcd564c3fe3c6435c2c00e4e951a37f3ea3d324b04503ca6f/psutil-5.4.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp34-cp34m-win32.whl","hashes":{"sha256":"92342777d46e4630cf17d437412dc7fce0a8561217e074d36a35eb911ffd570e"},"provenance":null,"requires-python":null,"size":218421,"upload-time":"2017-11-08T13:50:42.728570Z","url":"https://files.pythonhosted.org/packages/e1/bd/d34935cd39f893d6e9ed46df5245aa71b29d2408b98f23755f234d517f80/psutil-5.4.1-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"f8f2f47a987c32ed3ca2068f3dfa9060dc9ff6cbed023d627d3f27060f4e59c4"},"provenance":null,"requires-python":null,"size":221144,"upload-time":"2017-11-08T13:50:47.928399Z","url":"https://files.pythonhosted.org/packages/24/ae/dcb7394e75b23b71f46742ffdab21d864a7ae74124b7930e5ea4f47b9049/psutil-5.4.1-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp35-cp35m-win32.whl","hashes":{"sha256":"1fce45549618d1930afefe322834ba91758331725bfdaec73ba6abcc83f6dc11"},"provenance":null,"requires-python":null,"size":220611,"upload-time":"2017-11-08T13:50:52.754745Z","url":"https://files.pythonhosted.org/packages/e8/58/c60fbf66c58d1e4b18902d601a294cb6ee993f3d051b44fdf397b6166852/psutil-5.4.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"f8a88553b2b5916f3bd814a91942215822a1dabae6db033cbb019095d6a24bc2"},"provenance":null,"requires-python":null,"size":224288,"upload-time":"2017-11-08T13:50:58.405900Z","url":"https://files.pythonhosted.org/packages/4f/f8/0e4e80114bc58267199c932f9227d09a00dea952b37400f76aa2a3bb9492/psutil-5.4.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp36-cp36m-win32.whl","hashes":{"sha256":"4139f76baa59142b907dd581d7ff3506a5163cb8ef69e8e92060df330bbf5788"},"provenance":null,"requires-python":null,"size":220610,"upload-time":"2017-11-08T13:51:03.361290Z","url":"https://files.pythonhosted.org/packages/8e/16/02eb53ea087776d9f219973e7b52c7d729929a2727c15894842f9b3629e6/psutil-5.4.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"data-dist-info-metadata":{"sha256":"bea16c7d13fd463723ac0b38f421db313ae19443becf2ea91b0a77f6f3d8d8f2"},"filename":"psutil-5.4.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"d61bc04401ce938576e4c6ec201e812ed4114bfb9712202b87003619116c90c6"},"provenance":null,"requires-python":null,"size":224287,"upload-time":"2017-11-08T13:51:07.991086Z","url":"https://files.pythonhosted.org/packages/4a/89/0610a20ab3d1546fbe288001fb79ec4818ef6de29f89259c39daea85984f/psutil-5.4.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.1.tar.gz","hashes":{"sha256":"42e2de159e3c987435cb3b47d6f37035db190a1499f3af714ba7af5c379b6ba2"},"provenance":null,"requires-python":null,"size":408489,"upload-time":"2017-11-08T13:51:15.716367Z","url":"https://files.pythonhosted.org/packages/fe/17/0f0bf5792b2dfe6003efc5175c76225f7d3426f88e2bf8d360cfab870cd8/psutil-5.4.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"3e4ef4cfed824c06d6acaff8f28af76203e0c1238133c71c6de75c654e9c89b1"},"data-dist-info-metadata":{"sha256":"3e4ef4cfed824c06d6acaff8f28af76203e0c1238133c71c6de75c654e9c89b1"},"filename":"psutil-5.4.2-cp27-none-win32.whl","hashes":{"sha256":"2fbbc7dce43c5240b9dc6d56302d57412f1c5a0d665d1f04eb05a6b7279f4e9b"},"provenance":null,"requires-python":null,"size":221114,"upload-time":"2017-12-07T12:03:17.419322Z","url":"https://files.pythonhosted.org/packages/e6/e2/09d55a6e899cf1a7b6a22d0cc2a75d45553df5b63d7c9f2eb2553c7207bc/psutil-5.4.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"3e4ef4cfed824c06d6acaff8f28af76203e0c1238133c71c6de75c654e9c89b1"},"data-dist-info-metadata":{"sha256":"3e4ef4cfed824c06d6acaff8f28af76203e0c1238133c71c6de75c654e9c89b1"},"filename":"psutil-5.4.2-cp27-none-win_amd64.whl","hashes":{"sha256":"259ec8578d19643179eb2377348c63b650b51ba40f58f2620a3d9732b8a0b557"},"provenance":null,"requires-python":null,"size":223996,"upload-time":"2017-12-07T12:03:22.758210Z","url":"https://files.pythonhosted.org/packages/47/fc/e2199322f422e4bb6e25808a335132235fb0f3fabb0ea71ec3442719fdaf/psutil-5.4.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp34-cp34m-win32.whl","hashes":{"sha256":"d3808be8241433db17fa955566c3b8be61dac8ba8f221dcbb202a9daba918db5"},"provenance":null,"requires-python":null,"size":220935,"upload-time":"2017-12-07T12:03:27.058880Z","url":"https://files.pythonhosted.org/packages/d5/50/044ad4b47bf0e992a11ae7cb060e8c7dd52eb6983c2c4b6fdd5314fcc3b2/psutil-5.4.2-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"449747f638c221f8ce6ca3548aefef13339aa05b453cc1f233f4d6c31c206198"},"provenance":null,"requires-python":null,"size":223749,"upload-time":"2017-12-07T12:03:31.671187Z","url":"https://files.pythonhosted.org/packages/b5/77/6f8b4c7c6e9e8fa60b0e0627853ba7182f1a2459ad0b557fe3257bbe014b/psutil-5.4.2-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp35-cp35m-win32.whl","hashes":{"sha256":"f6c2d54abd59ed8691882de7fd6b248f5808a567885f20f50b3b4b9eedaebb1f"},"provenance":null,"requires-python":null,"size":223260,"upload-time":"2017-12-07T12:03:36.139696Z","url":"https://files.pythonhosted.org/packages/ad/fa/ac5b6bc2b25817509fc2e44f910f39b3385d1575262f7802b9c113ab786a/psutil-5.4.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"e3d00d8fc3d4217f05d07af45390f072c04cb7c7dddd70b86b728e5fbe485c81"},"provenance":null,"requires-python":null,"size":226950,"upload-time":"2017-12-07T12:03:40.889125Z","url":"https://files.pythonhosted.org/packages/0e/5c/67b33328d4f307608006c3630838b272719125b36e5456a4dd8f4e76eca9/psutil-5.4.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp36-cp36m-win32.whl","hashes":{"sha256":"3473d6abad9d6ec7b8a97f4dc55f0b3483ecf470d85f08f5e23c1c07592b914f"},"provenance":null,"requires-python":null,"size":223259,"upload-time":"2017-12-07T12:03:45.502272Z","url":"https://files.pythonhosted.org/packages/59/75/97862069d3fa20f1f2897fecd5b6822a3c06ee2af1161f1beb320cc4f5f8/psutil-5.4.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"data-dist-info-metadata":{"sha256":"aee7fd8368b69d2c7b666317dd0c8e73e7af18ef863dabadfb44e8f46ffef9d5"},"filename":"psutil-5.4.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"7dc6c3bbb5d28487f791f195d6abfdef295d34c44ce6cb5f2d178613fb3338ab"},"provenance":null,"requires-python":null,"size":226952,"upload-time":"2017-12-07T12:03:51.257182Z","url":"https://files.pythonhosted.org/packages/8f/9a/5dea138e49addd68d8e65d08103dd28428f1ea0b8d4e8beef9b24f069a16/psutil-5.4.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.2.tar.gz","hashes":{"sha256":"00a1f9ff8d1e035fba7bfdd6977fa8ea7937afdb4477339e5df3dba78194fe11"},"provenance":null,"requires-python":null,"size":411888,"upload-time":"2017-12-07T12:03:58.583601Z","url":"https://files.pythonhosted.org/packages/54/24/aa854703715fa161110daa001afce75d21d1840e9ab5eb28708d6a5058b0/psutil-5.4.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"9de2e14fb34c7d6a6d1b3692c6e7830d8a950ce4740bd70dbc4f2900ad780b67"},"data-dist-info-metadata":{"sha256":"9de2e14fb34c7d6a6d1b3692c6e7830d8a950ce4740bd70dbc4f2900ad780b67"},"filename":"psutil-5.4.3-cp27-none-win32.whl","hashes":{"sha256":"82a06785db8eeb637b349006cc28a92e40cd190fefae9875246d18d0de7ccac8"},"provenance":null,"requires-python":null,"size":220984,"upload-time":"2018-01-01T20:33:16.515515Z","url":"https://files.pythonhosted.org/packages/e5/cc/6dd427e738a8db6d0b66525856da43d2ef12c4c19269863927f7cf0e2aaf/psutil-5.4.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"9de2e14fb34c7d6a6d1b3692c6e7830d8a950ce4740bd70dbc4f2900ad780b67"},"data-dist-info-metadata":{"sha256":"9de2e14fb34c7d6a6d1b3692c6e7830d8a950ce4740bd70dbc4f2900ad780b67"},"filename":"psutil-5.4.3-cp27-none-win_amd64.whl","hashes":{"sha256":"4152ae231709e3e8b80e26b6da20dc965a1a589959c48af1ed024eca6473f60d"},"provenance":null,"requires-python":null,"size":223871,"upload-time":"2018-01-01T20:33:24.887595Z","url":"https://files.pythonhosted.org/packages/b9/e4/6867765edcab8d12a52c84c9b0af492ecb99f8cc565ad552341bcf73ebd9/psutil-5.4.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp34-cp34m-win32.whl","hashes":{"sha256":"230eeb3aeb077814f3a2cd036ddb6e0f571960d327298cc914c02385c3e02a63"},"provenance":null,"requires-python":null,"size":220823,"upload-time":"2018-01-01T20:33:31.019909Z","url":"https://files.pythonhosted.org/packages/5b/fc/745a864190a4221cdb984e666f4218e98d9a53a64b4dcf2eb7a71c1bf693/psutil-5.4.3-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"a3286556d4d2f341108db65d8e20d0cd3fcb9a91741cb5eb496832d7daf2a97c"},"provenance":null,"requires-python":null,"size":223605,"upload-time":"2018-01-01T20:33:38.392793Z","url":"https://files.pythonhosted.org/packages/b0/25/414738d5e8e75418e560a36651d1e1b09c9df05440a2a808d999a5548b1e/psutil-5.4.3-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp35-cp35m-win32.whl","hashes":{"sha256":"94d4e63189f2593960e73acaaf96be235dd8a455fe2bcb37d8ad6f0e87f61556"},"provenance":null,"requires-python":null,"size":223167,"upload-time":"2018-01-01T20:33:43.796884Z","url":"https://files.pythonhosted.org/packages/e9/80/8da216f42050220f37f7133d2accfcd001a1bd0f31d7cdb8660acb46b8fe/psutil-5.4.3-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"c91eee73eea00df5e62c741b380b7e5b6fdd553891bee5669817a3a38d036f13"},"provenance":null,"requires-python":null,"size":226810,"upload-time":"2018-01-01T20:33:50.884798Z","url":"https://files.pythonhosted.org/packages/23/34/b3de39502c2c34899f9e7ae3c8d1050c9317997ab1fe6c647e7a789571a8/psutil-5.4.3-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp36-cp36m-win32.whl","hashes":{"sha256":"779ec7e7621758ca11a8d99a1064996454b3570154277cc21342a01148a49c28"},"provenance":null,"requires-python":null,"size":223168,"upload-time":"2018-01-01T20:33:56.458784Z","url":"https://files.pythonhosted.org/packages/da/c1/caadba7c64f72118b02f019c60ad85a5668ddf0a32836230b71692b0cbfa/psutil-5.4.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"data-dist-info-metadata":{"sha256":"7a6c999b9c5b0661b0db22bb84f2ef4f1cf396a3f1ed6311aad005f3bee574e8"},"filename":"psutil-5.4.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"8a15d773203a1277e57b1d11a7ccdf70804744ef4a9518a87ab8436995c31a4b"},"provenance":null,"requires-python":null,"size":226804,"upload-time":"2018-01-01T20:34:03.130620Z","url":"https://files.pythonhosted.org/packages/71/80/90799d3dc6e33e650ee03f96fa18157faed885593eabea3a6560ebff7de0/psutil-5.4.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.3.tar.gz","hashes":{"sha256":"e2467e9312c2fa191687b89ff4bc2ad8843be4af6fb4dc95a7cc5f7d7a327b18"},"provenance":null,"requires-python":null,"size":412550,"upload-time":"2018-01-01T20:34:13.285899Z","url":"https://files.pythonhosted.org/packages/e2/e1/600326635f97fee89bf8426fef14c5c29f4849c79f68fd79f433d8c1bd96/psutil-5.4.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"485b43e92c234e987a6b5a9be22665d46ab26940b08a2edad8583aa4743c0439"},"data-dist-info-metadata":{"sha256":"485b43e92c234e987a6b5a9be22665d46ab26940b08a2edad8583aa4743c0439"},"filename":"psutil-5.4.4-cp27-none-win32.whl","hashes":{"sha256":"8f208867d41eb3b6de416df098a9a28d08d40b432467d821b8ef5bb589a394ce"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216647,"upload-time":"2018-04-13T09:10:24.220147Z","url":"https://files.pythonhosted.org/packages/e8/cd/dbf537e32de1c9f06a0069bf0ef13c8707f653e9af2b0ea0ed7040b73083/psutil-5.4.4-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"485b43e92c234e987a6b5a9be22665d46ab26940b08a2edad8583aa4743c0439"},"data-dist-info-metadata":{"sha256":"485b43e92c234e987a6b5a9be22665d46ab26940b08a2edad8583aa4743c0439"},"filename":"psutil-5.4.4-cp27-none-win_amd64.whl","hashes":{"sha256":"77b5e310de17085346ef2c4c21b64d5e39616ab4559b8ef6fea9f6f2ab0de66f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219668,"upload-time":"2018-04-13T09:10:29.328807Z","url":"https://files.pythonhosted.org/packages/7f/37/538bb4275d8a26ec369944878a681000e827e72cab9b4f27f4b1b5932446/psutil-5.4.4-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp34-cp34m-win32.whl","hashes":{"sha256":"fec0e59dacbe91db7e063f038301f49da7e9361732fc31d28338ecaa4719520e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216642,"upload-time":"2018-04-13T09:10:33.842941Z","url":"https://files.pythonhosted.org/packages/92/e9/c9c4ec1a0ac55ee1514c1a249d017dd2f7a89e727d236ed6862b493de154/psutil-5.4.4-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"1268fb6959cd8d761c30e13e79908ae73ba5a69c3c3a5d09a7a27278446f9800"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219513,"upload-time":"2018-04-13T09:10:38.041575Z","url":"https://files.pythonhosted.org/packages/83/49/c903f446d28bfb6e92fa08b710f68cd5d17cc2ccfc4a13fe607f8b20f6dd/psutil-5.4.4-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp35-cp35m-win32.whl","hashes":{"sha256":"7eb2d80ef79d90474a03eead13b32e541d1fdeb47468cf04c881f0a7392ddbc5"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219220,"upload-time":"2018-04-13T09:10:42.685382Z","url":"https://files.pythonhosted.org/packages/bb/63/c9b3e9ff8d23409896a7e9e7356a730fdfb5a45a1edc0e6d4ca5ce655f29/psutil-5.4.4-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"69f1db4d13f362ce11a6246b20c752c31b87a6fd77452170fd03c26a8a20a4f2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222757,"upload-time":"2018-04-13T09:10:47.950981Z","url":"https://files.pythonhosted.org/packages/72/22/a5ce34af1285679e02d7fd701ff6389f579a17e623dd89236ea1873ce12b/psutil-5.4.4-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp36-cp36m-win32.whl","hashes":{"sha256":"6eb59bcfd48eade8889bae67a16e0d8c7b18af0732ba64dead61206fd7cb4e45"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219224,"upload-time":"2018-04-13T09:10:52.291473Z","url":"https://files.pythonhosted.org/packages/3c/6d/5e9a3683d4532997525aa20d1d9ce0ca1201271d30aad9e5f18c34459478/psutil-5.4.4-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"data-dist-info-metadata":{"sha256":"893713cd1f34926704c2e5f844b554a79ad42ce1c20046651dad28613eb2d0bc"},"filename":"psutil-5.4.4-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"a4af5d4fcf6022886a30fb3b4fff71ff25f645865a68506680d43a3e634764af"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222759,"upload-time":"2018-04-13T09:10:56.584621Z","url":"https://files.pythonhosted.org/packages/b6/61/eeeab30fa737b8b95b790d3eb8f49ebedeb783e43aef2d8d851687592d6c/psutil-5.4.4-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.4.tar.gz","hashes":{"sha256":"5959e33e0fc69742dd22e88bfc7789a1f2e1fc2297794b543119e10cdac8dfb1"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":417890,"upload-time":"2018-04-13T09:11:03.199870Z","url":"https://files.pythonhosted.org/packages/35/35/7da482448cd9ee3555faa9c5e541e37b18a849fb961e55d6fda6ca936ddb/psutil-5.4.4.tar.gz","yanked":false},{"core-metadata":{"sha256":"c8af77aa4cab9fad5f3310c13c633fb6bb0f5a44421d0e8662a9f16b38ca9aec"},"data-dist-info-metadata":{"sha256":"c8af77aa4cab9fad5f3310c13c633fb6bb0f5a44421d0e8662a9f16b38ca9aec"},"filename":"psutil-5.4.5-cp27-none-win32.whl","hashes":{"sha256":"33384065f0014351fa70187548e3e95952c4df4bc5c38648bd0e647d21eaaf01"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216630,"upload-time":"2018-04-13T17:59:43.195295Z","url":"https://files.pythonhosted.org/packages/63/9f/529a599db3057602114a30aa5e3641e78bce6c6e195adb75309c9286cb88/psutil-5.4.5-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"c8af77aa4cab9fad5f3310c13c633fb6bb0f5a44421d0e8662a9f16b38ca9aec"},"data-dist-info-metadata":{"sha256":"c8af77aa4cab9fad5f3310c13c633fb6bb0f5a44421d0e8662a9f16b38ca9aec"},"filename":"psutil-5.4.5-cp27-none-win_amd64.whl","hashes":{"sha256":"f24cd52bafa06917935fe1b68c5a45593abe1f3097dc35b2dfc4718236795890"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219650,"upload-time":"2018-04-13T17:59:53.389911Z","url":"https://files.pythonhosted.org/packages/b6/ca/2d23b37e9b30908174d2cb596f60f06b3858856a2e595c931f7d4d640c03/psutil-5.4.5-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp34-cp34m-win32.whl","hashes":{"sha256":"99029b6af386b22882f0b6d537ffed5a9c3d5ff31782974aeaa1d683262d8543"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216589,"upload-time":"2018-04-13T18:00:04.166075Z","url":"https://files.pythonhosted.org/packages/bf/bc/f687dfa4679aad782fb78c43fed2626cb0157567a5b06790997e5aa0f166/psutil-5.4.5-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"51e12aa74509832443862373a2655052b20c83cad7322f49d217452500b9a405"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219462,"upload-time":"2018-04-13T18:00:14.637120Z","url":"https://files.pythonhosted.org/packages/32/b5/545953316dd9cb053e4c7d4f70d88aba5362dbbe58422ca6bbec1bbf8956/psutil-5.4.5-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp35-cp35m-win32.whl","hashes":{"sha256":"325c334596ad2d8a178d0e7b4eecc91748096a87489b3701ee16986173000aaa"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219172,"upload-time":"2018-04-13T18:00:28.230677Z","url":"https://files.pythonhosted.org/packages/00/ed/fdf2930c41e76e3a8bc59bf998062ee5ad0c393170a7d2c273dd3b259794/psutil-5.4.5-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"52a91ba928a5e86e0249b4932d6e36972a72d1ad8dcc5b7f753a2ae14825a4ba"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222706,"upload-time":"2018-04-13T18:00:38.180085Z","url":"https://files.pythonhosted.org/packages/c6/bf/09b13c17f54f0004ccb43cc1c2d36bab2eb75f471564b7856749dcaf62c3/psutil-5.4.5-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp36-cp36m-win32.whl","hashes":{"sha256":"b10703a109cc9225cd588c207f7f93480a420ade35c13515ea8f20063b42a392"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219171,"upload-time":"2018-04-13T18:00:52.610707Z","url":"https://files.pythonhosted.org/packages/3c/ae/34952007b4d64f88a03510866b9cd90207e391f6b2b59b6301ad96fa0fb5/psutil-5.4.5-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"data-dist-info-metadata":{"sha256":"4fb804f1abb5eb53785449da8e1201c9af34b09face243251213fa79878f9142"},"filename":"psutil-5.4.5-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"ddba952ed256151844d82fb13c8fb1019fe11ecaeacbd659d67ba5661ae73d0d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222710,"upload-time":"2018-04-13T18:01:02.457641Z","url":"https://files.pythonhosted.org/packages/4c/bb/303f15f4a47b96ff0ae5025d89b330e2be314085c418c0b726877476e937/psutil-5.4.5-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.5.tar.gz","hashes":{"sha256":"ebe293be36bb24b95cdefc5131635496e88b17fabbcf1e4bc9b5c01f5e489cfe"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":418003,"upload-time":"2018-04-13T18:01:19.381491Z","url":"https://files.pythonhosted.org/packages/14/a2/8ac7dda36eac03950ec2668ab1b466314403031c83a95c5efc81d2acf163/psutil-5.4.5.tar.gz","yanked":false},{"core-metadata":{"sha256":"aa807ba097a31687e416722680ba225449cb087946d4db7e62db8c1659a43a5d"},"data-dist-info-metadata":{"sha256":"aa807ba097a31687e416722680ba225449cb087946d4db7e62db8c1659a43a5d"},"filename":"psutil-5.4.6-cp27-none-win32.whl","hashes":{"sha256":"319e12f6bae4d4d988fbff3bed792953fa3b44c791f085b0a1a230f755671ef7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216468,"upload-time":"2018-06-07T15:39:23.967375Z","url":"https://files.pythonhosted.org/packages/b4/00/82c9fb4ffca22f2f6d0d883469584cd0cff71a604a19809015045b1fbab6/psutil-5.4.6-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"aa807ba097a31687e416722680ba225449cb087946d4db7e62db8c1659a43a5d"},"data-dist-info-metadata":{"sha256":"aa807ba097a31687e416722680ba225449cb087946d4db7e62db8c1659a43a5d"},"filename":"psutil-5.4.6-cp27-none-win_amd64.whl","hashes":{"sha256":"7789885a72aa3075d28d028236eb3f2b84d908f81d38ad41769a6ddc2fd81b7c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219532,"upload-time":"2018-06-07T15:39:26.297791Z","url":"https://files.pythonhosted.org/packages/c2/23/22df1d36dc8ae002e9f646f9ed06b4f6bfbc7a22b67804c3a497be21d002/psutil-5.4.6-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp34-cp34m-win32.whl","hashes":{"sha256":"0ff2b16e9045d01edb1dd10d7fbcc184012e37f6cd38029e959f2be9c6223f50"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":216464,"upload-time":"2018-06-07T15:39:28.563215Z","url":"https://files.pythonhosted.org/packages/f9/fa/966988e350306e1a1a9024e77ad5f118cbfe11318e8bdfc258c3d5a1c68b/psutil-5.4.6-cp34-cp34m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp34-cp34m-win_amd64.whl","hashes":{"sha256":"dc85fad15ef98103ecc047a0d81b55bbf5fe1b03313b96e883acc2e2fa87ed5c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219331,"upload-time":"2018-06-07T15:39:31.250304Z","url":"https://files.pythonhosted.org/packages/a5/0d/40b552c2c089523df1f7ab5a0249fbf90cb2e80e89177b0189e41e367adc/psutil-5.4.6-cp34-cp34m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp35-cp35m-win32.whl","hashes":{"sha256":"7f4616bcb44a6afda930cfc40215e5e9fa7c6896e683b287c771c937712fbe2f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219052,"upload-time":"2018-06-07T15:39:33.732713Z","url":"https://files.pythonhosted.org/packages/00/4d/194ec701de80c704f679bf78495c054994cc403884ffec816787813c4fde/psutil-5.4.6-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"529ae235896efb99a6f77653a7138273ab701ec9f0343a1f5030945108dee3c4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222533,"upload-time":"2018-06-07T15:39:36.224840Z","url":"https://files.pythonhosted.org/packages/e3/21/f82f270326e098f211bcc36cbb2ae7100732dcad03bd324e6af8c9d7e407/psutil-5.4.6-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp36-cp36m-win32.whl","hashes":{"sha256":"254adb6a27c888f141d2a6032ae231d8ed4fc5f7583b4c825e5f7d7c78d26d2e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219048,"upload-time":"2018-06-07T15:39:38.273913Z","url":"https://files.pythonhosted.org/packages/d6/e0/0f1b4f61246c4e2b540898b1ca0fa51ee2f52f0366956974f1039e00ed67/psutil-5.4.6-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"a9b85b335b40a528a8e2a6b549592138de8429c6296e7361892958956e6a73cf"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222534,"upload-time":"2018-06-07T15:39:40.675791Z","url":"https://files.pythonhosted.org/packages/36/4b/80d9eb5d39ec4b4d8aec8b098b5097a7291de20bbbe6c2ab233b9d8fe245/psutil-5.4.6-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp37-cp37m-win32.whl","hashes":{"sha256":"6d981b4d863b20c8ceed98b8ac3d1ca7f96d28707a80845d360fa69c8fc2c44b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":219860,"upload-time":"2018-06-28T22:47:57.979619Z","url":"https://files.pythonhosted.org/packages/7a/62/28923c44954b6cf8aee637f3a2f30e0e1ff39ec0f74a4f98069d37f00751/psutil-5.4.6-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"data-dist-info-metadata":{"sha256":"b7bbbf54e7767a4dcfada805607a533828d7e74794bf64ea1ae76700fc24d495"},"filename":"psutil-5.4.6-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"7fdb3d02bfd68f508e6745021311a4a4dbfec53fca03721474e985f310e249ba"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224296,"upload-time":"2018-06-28T22:48:01.565801Z","url":"https://files.pythonhosted.org/packages/b1/be/78f9d786bddc190c4b394a01531741a11b95f1522cf2759958f13b46407f/psutil-5.4.6-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.6.tar.gz","hashes":{"sha256":"686e5a35fe4c0acc25f3466c32e716f2d498aaae7b7edc03e2305b682226bcf6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":418059,"upload-time":"2018-06-07T15:39:42.364107Z","url":"https://files.pythonhosted.org/packages/51/9e/0f8f5423ce28c9109807024f7bdde776ed0b1161de20b408875de7e030c3/psutil-5.4.6.tar.gz","yanked":false},{"core-metadata":{"sha256":"94ef062f36609f64cc686f1930bf277790b85ced443769e242d41eb7a2588bec"},"data-dist-info-metadata":{"sha256":"94ef062f36609f64cc686f1930bf277790b85ced443769e242d41eb7a2588bec"},"filename":"psutil-5.4.7-cp27-none-win32.whl","hashes":{"sha256":"b34611280a2d0697f1c499e15e936d88109170194b390599c98bab8072a71f05"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":217777,"upload-time":"2018-08-14T21:01:10.586697Z","url":"https://files.pythonhosted.org/packages/b7/e9/bedbdfecef9d708489cfcd8b9aeada8d8f014fc14644c7129c7177e80d32/psutil-5.4.7-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"94ef062f36609f64cc686f1930bf277790b85ced443769e242d41eb7a2588bec"},"data-dist-info-metadata":{"sha256":"94ef062f36609f64cc686f1930bf277790b85ced443769e242d41eb7a2588bec"},"filename":"psutil-5.4.7-cp27-none-win_amd64.whl","hashes":{"sha256":"a890c3e490493f21da2817ffc92822693bc0d6bcac9999caa04ffce8dd4e7132"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220989,"upload-time":"2018-08-14T21:01:13.273922Z","url":"https://files.pythonhosted.org/packages/50/6a/34525bc4e6e153bf6e849a4c4e936742b365f6819c0462cebfa4f082a3c4/psutil-5.4.7-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp35-cp35m-win32.whl","hashes":{"sha256":"1914bacbd2fc2af8f795daa44b9d2e0649a147460cfd21b1a70a124472f66d40"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220344,"upload-time":"2018-08-14T21:01:15.659359Z","url":"https://files.pythonhosted.org/packages/36/32/5b10d7c3940c64fe92620c368ede8a10016d51aa36079a5cd69944da5a74/psutil-5.4.7-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"d081707ef0081920533db30200a2d30d5c0ea9cf6afa7cf8881ae4516cc69c48"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224081,"upload-time":"2018-08-14T21:01:18.052521Z","url":"https://files.pythonhosted.org/packages/ef/fc/8dc7731df7de2f4c65378a7147ecc977221093eee90d9777ca501c2790c5/psutil-5.4.7-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp36-cp36m-win32.whl","hashes":{"sha256":"0d8da7333549a998556c18eb2af3ce902c28d66ceb947505c008f91e9f988abd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220342,"upload-time":"2018-08-14T21:01:20.870660Z","url":"https://files.pythonhosted.org/packages/20/6e/a9a0f84bc3efe970b4c8688b7e7f14ee8342497de8a88cffd35bb485cdcc/psutil-5.4.7-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"cea2557ee6a9faa2c100947637ded68414e12b851633c4ce26e0311b2a2ed539"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224085,"upload-time":"2018-08-14T21:01:22.891149Z","url":"https://files.pythonhosted.org/packages/97/9e/c056abafcf0fc7ca5bddbc21ad1bb7c67889e16c088b9759f00b95fefcb4/psutil-5.4.7-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp37-cp37m-win32.whl","hashes":{"sha256":"215d61a901e67b1a35e14c6aedef317f7fa7e6075a20c150fd11bd2c906d2c83"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220347,"upload-time":"2018-08-14T21:01:25.157286Z","url":"https://files.pythonhosted.org/packages/2e/33/5cef36162d94cf0adce428729adeb18b8548ff060781854f3aca71e6b0f0/psutil-5.4.7-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"data-dist-info-metadata":{"sha256":"77861491100106f950300f1104d6fcb309d103d34e379a69c8780e9edefa6710"},"filename":"psutil-5.4.7-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"51057c03aea251ad6667c2bba259bc7ed3210222d3a74152c84e3ab06e1da0ba"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224087,"upload-time":"2018-08-14T21:01:27.563142Z","url":"https://files.pythonhosted.org/packages/bb/15/aa3d11ae8bf04b7683224f7d3b8f2dd4d3f8a918dcce59bb1f987fca9c6e/psutil-5.4.7-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.7.tar.gz","hashes":{"sha256":"5b6322b167a5ba0c5463b4d30dfd379cd4ce245a1162ebf8fc7ab5c5ffae4f3b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":420300,"upload-time":"2018-08-14T21:01:30.063470Z","url":"https://files.pythonhosted.org/packages/7d/9a/1e93d41708f8ed2b564395edfa3389f0fd6d567597401c2e5e2775118d8b/psutil-5.4.7.tar.gz","yanked":false},{"core-metadata":{"sha256":"000e5f1178f3b76020f7866397e41f07c1f4564da730a45c24894e3aebebb7cb"},"data-dist-info-metadata":{"sha256":"000e5f1178f3b76020f7866397e41f07c1f4564da730a45c24894e3aebebb7cb"},"filename":"psutil-5.4.8-cp27-none-win32.whl","hashes":{"sha256":"809c9cef0402e3e48b5a1dddc390a8a6ff58b15362ea5714494073fa46c3d293"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":220106,"upload-time":"2018-10-30T09:57:34.685390Z","url":"https://files.pythonhosted.org/packages/5a/3f/3f0920df352dae7f824e0e612ff02591378f78405d6c7663dcac023005c4/psutil-5.4.8-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"000e5f1178f3b76020f7866397e41f07c1f4564da730a45c24894e3aebebb7cb"},"data-dist-info-metadata":{"sha256":"000e5f1178f3b76020f7866397e41f07c1f4564da730a45c24894e3aebebb7cb"},"filename":"psutil-5.4.8-cp27-none-win_amd64.whl","hashes":{"sha256":"3b7a4daf4223dae171a67a89314ac5ca0738e94064a78d99cfd751c55d05f315"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":223347,"upload-time":"2018-10-30T09:57:37.065973Z","url":"https://files.pythonhosted.org/packages/0f/fb/6aecd2c8c9d0ac83d789eaf9f9ec052dd61dd5aea2b47ffa4704175d7a2a/psutil-5.4.8-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp35-cp35m-win32.whl","hashes":{"sha256":"bbffac64cfd01c6bcf90eb1bedc6c80501c4dae8aef4ad6d6dd49f8f05f6fc5a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222713,"upload-time":"2018-10-30T09:57:39.214229Z","url":"https://files.pythonhosted.org/packages/46/2e/ce4ec4b60decc23e0e4d148b6f44c7ddd06ba0ab207dfaee21958bd669df/psutil-5.4.8-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"b4d1b735bf5b120813f4c89db8ac22d89162c558cbd7fdd298866125fe906219"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226441,"upload-time":"2018-10-30T09:57:41.411069Z","url":"https://files.pythonhosted.org/packages/7f/28/5ccdb98eff12e7741cc2a6d9dcfdd5d9e06f6d363c2c019d5bfa0e0c1282/psutil-5.4.8-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp36-cp36m-win32.whl","hashes":{"sha256":"3e19be3441134445347af3767fa7770137d472a484070840eee6653b94ac5576"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222711,"upload-time":"2018-10-30T09:57:43.901154Z","url":"https://files.pythonhosted.org/packages/b5/31/8ac896ca77a6aa75ee900698f96ddce46e96bb2484a92457c359a4e4bae6/psutil-5.4.8-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"1c19957883e0b93d081d41687089ad630e370e26dc49fd9df6951d6c891c4736"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226440,"upload-time":"2018-10-30T09:57:46.331527Z","url":"https://files.pythonhosted.org/packages/3b/15/62d1eeb4c015e20295e0197f7de0202bd9e5bcb5529b9503932decde2505/psutil-5.4.8-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp37-cp37m-win32.whl","hashes":{"sha256":"bfcea4f189177b2d2ce4a34b03c4ac32c5b4c22e21f5b093d9d315e6e253cd81"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222709,"upload-time":"2018-10-30T09:57:48.860424Z","url":"https://files.pythonhosted.org/packages/21/1e/fe6731e5f03ddf2e57d5b307f25bba294262bc88e27a0fbefdb3515d1727/psutil-5.4.8-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"data-dist-info-metadata":{"sha256":"edcd0deabcf9a4ae02acef42bf31c36db6ad085b36a3cc549cfded47de1f1e8f"},"filename":"psutil-5.4.8-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"1c71b9716790e202a00ab0931a6d1e25db1aa1198bcacaea2f5329f75d257fff"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226447,"upload-time":"2018-10-30T09:57:50.676641Z","url":"https://files.pythonhosted.org/packages/50/00/ae52663b879333aa5c65fc9a87ddc24169f8fdd1831762a1ba9c9be7740d/psutil-5.4.8-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.4.8.tar.gz","hashes":{"sha256":"6e265c8f3da00b015d24b842bfeb111f856b13d24f2c57036582568dc650d6c3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":422742,"upload-time":"2018-10-30T09:57:52.770639Z","url":"https://files.pythonhosted.org/packages/e3/58/0eae6e4466e5abf779d7e2b71fac7fba5f59e00ea36ddb3ed690419ccb0f/psutil-5.4.8.tar.gz","yanked":false},{"core-metadata":{"sha256":"1d8c3b2819c1de19ccc23f8de068eab529322b6df33673da0ec45a41f785bdcc"},"data-dist-info-metadata":{"sha256":"1d8c3b2819c1de19ccc23f8de068eab529322b6df33673da0ec45a41f785bdcc"},"filename":"psutil-5.5.0-cp27-none-win32.whl","hashes":{"sha256":"96f3fdb4ef7467854d46ad5a7e28eb4c6dc6d455d751ddf9640cd6d52bdb03d7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":221314,"upload-time":"2019-01-23T18:23:33.526811Z","url":"https://files.pythonhosted.org/packages/fa/53/53f8c4f1af6f81b169ce76bfd0f56698bc1705da498a47d2ce701f7d7fe3/psutil-5.5.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"1d8c3b2819c1de19ccc23f8de068eab529322b6df33673da0ec45a41f785bdcc"},"data-dist-info-metadata":{"sha256":"1d8c3b2819c1de19ccc23f8de068eab529322b6df33673da0ec45a41f785bdcc"},"filename":"psutil-5.5.0-cp27-none-win_amd64.whl","hashes":{"sha256":"d23f7025bac9b3e38adc6bd032cdaac648ac0074d18e36950a04af35458342e8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224546,"upload-time":"2019-01-23T18:23:36.443393Z","url":"https://files.pythonhosted.org/packages/86/96/a7bfcc3aebedd7112ff353204901db6a1a0c1f3555b2788c68842bb78005/psutil-5.5.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp35-cp35m-win32.whl","hashes":{"sha256":"04d2071100aaad59f9bcbb801be2125d53b2e03b1517d9fed90b45eea51d297e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224084,"upload-time":"2019-01-23T18:23:39.132334Z","url":"https://files.pythonhosted.org/packages/c6/ca/f5d3841ca35e3e3607ed64fe61d2c392054692f05f35e807335299a7952b/psutil-5.5.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"d0c4230d60376aee0757d934020b14899f6020cd70ef8d2cb4f228b6ffc43e8f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":227536,"upload-time":"2019-01-23T18:23:42.027389Z","url":"https://files.pythonhosted.org/packages/86/0d/a13c15ddccd8e2ccabe63f6d4f139f5a94150b758026e030310e87dded80/psutil-5.5.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp36-cp36m-win32.whl","hashes":{"sha256":"3ac48568f5b85fee44cd8002a15a7733deca056a191d313dbf24c11519c0c4a8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224085,"upload-time":"2019-01-23T18:23:44.450649Z","url":"https://files.pythonhosted.org/packages/45/00/7cdd50ded02e18e50667e2f76ceb645ecdfce59deb39422485f198c7be37/psutil-5.5.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"f0fcb7d3006dd4d9ccf3ccd0595d44c6abbfd433ec31b6ca177300ee3f19e54e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":227537,"upload-time":"2019-01-23T18:23:46.937101Z","url":"https://files.pythonhosted.org/packages/48/d1/c9105512328c7f9800c51992b912df6f945eac696dfcd850f719541f67f3/psutil-5.5.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp37-cp37m-win32.whl","hashes":{"sha256":"c8ee08ad1b716911c86f12dc753eb1879006224fd51509f077987bb6493be615"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224078,"upload-time":"2019-01-23T18:23:49.451606Z","url":"https://files.pythonhosted.org/packages/3f/14/5adcad73f22ae0c8fba8b054c0bb7c33c906121b588cc6cbdd686f098947/psutil-5.5.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"data-dist-info-metadata":{"sha256":"053bd08cfadac2a889dd4c4d2e10bd70c7d11e50fa66c23c145303639bb537b1"},"filename":"psutil-5.5.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"b755be689d6fc8ebc401e1d5ce5bac867e35788f10229e166338484eead51b12"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":227542,"upload-time":"2019-01-23T18:23:52.035332Z","url":"https://files.pythonhosted.org/packages/38/f1/a822d2b3d973c1ddd9d8a81d269e36987bab20e7bb28ecaa55aef66e8df5/psutil-5.5.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.5.0.tar.gz","hashes":{"sha256":"1aba93430050270750d046a179c5f3d6e1f5f8b96c20399ba38c596b28fc4d37"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":425058,"upload-time":"2019-01-23T18:23:54.951599Z","url":"https://files.pythonhosted.org/packages/6e/a0/833bcbcede5141cc5615e50c7cc5b960ce93d9c9b885fbe3b7d36e48a2d4/psutil-5.5.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"0fa640ac67268845bdf0ad87325deaf6caae5372bf3703f282c5340f9af1f8c2"},"data-dist-info-metadata":{"sha256":"0fa640ac67268845bdf0ad87325deaf6caae5372bf3703f282c5340f9af1f8c2"},"filename":"psutil-5.5.1-cp27-none-win32.whl","hashes":{"sha256":"77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":221914,"upload-time":"2019-02-15T19:34:04.528250Z","url":"https://files.pythonhosted.org/packages/cc/cd/64aaf20c945662260026a128a08e46b93a49953224c0dccfdc6f37495d45/psutil-5.5.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"0fa640ac67268845bdf0ad87325deaf6caae5372bf3703f282c5340f9af1f8c2"},"data-dist-info-metadata":{"sha256":"0fa640ac67268845bdf0ad87325deaf6caae5372bf3703f282c5340f9af1f8c2"},"filename":"psutil-5.5.1-cp27-none-win_amd64.whl","hashes":{"sha256":"5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":225325,"upload-time":"2019-02-15T19:34:07.193210Z","url":"https://files.pythonhosted.org/packages/08/92/97b011d665ade1caf05dd02a3af4ede751c7b80f34812bc81479ec867d85/psutil-5.5.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp35-cp35m-win32.whl","hashes":{"sha256":"a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224822,"upload-time":"2019-02-15T19:34:09.528347Z","url":"https://files.pythonhosted.org/packages/6f/05/70f033e35cd34bc23a08793eaf713347c615674585ccfc7628b40eac4094/psutil-5.5.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228328,"upload-time":"2019-02-15T19:34:12.035704Z","url":"https://files.pythonhosted.org/packages/8e/8d/1854ed30b9f69f4b2cc04ef4364ae8a52ad2988a3223bf6314d2d47f0f04/psutil-5.5.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp36-cp36m-win32.whl","hashes":{"sha256":"ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224831,"upload-time":"2019-02-15T19:34:15.031823Z","url":"https://files.pythonhosted.org/packages/e8/c1/32fe16cb90192a9413f3ba303021047c158cbdde5aabd7e26ace8b54f69e/psutil-5.5.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228328,"upload-time":"2019-02-15T19:34:17.494390Z","url":"https://files.pythonhosted.org/packages/3e/6e/c0af4900f18811f09b93064588e53f3997abc051ae43f717d1ba610de3b7/psutil-5.5.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp37-cp37m-win32.whl","hashes":{"sha256":"c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":224832,"upload-time":"2019-02-15T19:34:19.685175Z","url":"https://files.pythonhosted.org/packages/00/e6/561fed27453add44af41a52e13e1dfca4d1e35705d698769edea6292339a/psutil-5.5.1-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"data-dist-info-metadata":{"sha256":"2aea42bc7128a735bd3dd530e3776b5c46cfb57362fa41b02ed50e6275148b66"},"filename":"psutil-5.5.1-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228327,"upload-time":"2019-02-15T19:34:22.218113Z","url":"https://files.pythonhosted.org/packages/3d/22/ed4fa46c5bfd95b4dc57d6544c3fe6568abe398aef3990f6011777f1a3f3/psutil-5.5.1-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.5.1.tar.gz","hashes":{"sha256":"72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":426750,"upload-time":"2019-02-15T19:34:24.841527Z","url":"https://files.pythonhosted.org/packages/c7/01/7c30b247cdc5ba29623faa5c8cf1f1bbf7e041783c340414b0ed7e067c64/psutil-5.5.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"254b7bff05e4525511dae40a6fde55a8c45855ad1f6f2e1bc7dc4a50d7c0caf6"},"data-dist-info-metadata":{"sha256":"254b7bff05e4525511dae40a6fde55a8c45855ad1f6f2e1bc7dc4a50d7c0caf6"},"filename":"psutil-5.6.0-cp27-none-win32.whl","hashes":{"sha256":"1020a37214c4138e34962881372b40f390582b5c8245680c04349c2afb785a25"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":222534,"upload-time":"2019-03-05T12:00:59.141291Z","url":"https://files.pythonhosted.org/packages/04/f5/11b1c93a8882615fdaf6222aaf9d3197f250ab3036d7ecf6b6c8594ddf61/psutil-5.6.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"254b7bff05e4525511dae40a6fde55a8c45855ad1f6f2e1bc7dc4a50d7c0caf6"},"data-dist-info-metadata":{"sha256":"254b7bff05e4525511dae40a6fde55a8c45855ad1f6f2e1bc7dc4a50d7c0caf6"},"filename":"psutil-5.6.0-cp27-none-win_amd64.whl","hashes":{"sha256":"d9cdc2e82aeb82200fff3640f375fac39d88b1bed27ce08377cd7fb0e3621cb7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":225803,"upload-time":"2019-03-05T12:01:01.831664Z","url":"https://files.pythonhosted.org/packages/11/88/ed94a7c091fb6ad8fbf545f0f20e140c47286712d6d85dd8cfc40b34fe72/psutil-5.6.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp35-cp35m-win32.whl","hashes":{"sha256":"c4a2f42abee709ed97b4498c21aa608ac31fc1f7cc8aa60ebdcd3c80757a038d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226153,"upload-time":"2019-03-05T12:01:04.695139Z","url":"https://files.pythonhosted.org/packages/ab/5c/dacbb4b6623dc390939e1785a818e6e816eb42c8c96c1f2d30a2fc22a773/psutil-5.6.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"722dc0dcce5272f3c5c41609fdc2c8f0ee3f976550c2d2f2057e26ba760be9c0"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230078,"upload-time":"2019-03-05T12:01:07.429607Z","url":"https://files.pythonhosted.org/packages/41/27/9e6d39ae822387ced9a77da904e24e8796c1fa55c5f637cd35221b170980/psutil-5.6.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp36-cp36m-win32.whl","hashes":{"sha256":"1c8e6444ca1cee9a60a1a35913b8409722f7474616e0e21004e4ffadba59964b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226154,"upload-time":"2019-03-05T12:01:10.226359Z","url":"https://files.pythonhosted.org/packages/fb/d0/14b83939ed41c5a80dca6fa072b1a99cd576e33810f691e73a08a7c045b4/psutil-5.6.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"151c9858c268a1523e16fab33e3bc3bae8a0e57b57cf7fcad85fb409cbac6baf"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230083,"upload-time":"2019-03-05T12:01:12.681476Z","url":"https://files.pythonhosted.org/packages/ce/3a/ff53c0ee59a864c3614fcaea45f4246e670934ea0d30b632c6e3905533c9/psutil-5.6.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp37-cp37m-win32.whl","hashes":{"sha256":"86f61a1438c026c980a4c3e2dd88a5774a3a0f00d6d0954d6c5cf8d1921b804e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226154,"upload-time":"2019-03-05T12:01:15.084497Z","url":"https://files.pythonhosted.org/packages/17/15/bf444a07aae2200f7b4e786c73bfe959201cb9ab63f737be12d21b5f252e/psutil-5.6.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"data-dist-info-metadata":{"sha256":"4587b1b69c3d6666cbea0b1ac5ab97e537ce98c5c4986439eec64421d1b2baeb"},"filename":"psutil-5.6.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"da6676a484adec2fdd3e1ce1b70799881ffcb958e40208dd4c5beba0011f3589"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230081,"upload-time":"2019-03-05T12:01:17.936041Z","url":"https://files.pythonhosted.org/packages/88/fd/a32491e77b37ffc00faf79c864975cfd720ae0ac867d93c90640d44adf43/psutil-5.6.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.0.tar.gz","hashes":{"sha256":"dca71c08335fbfc6929438fe3a502f169ba96dd20e50b3544053d6be5cb19d82"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":426596,"upload-time":"2019-03-05T12:01:20.833448Z","url":"https://files.pythonhosted.org/packages/79/e6/a4e3c92fe19d386dcc6149dbf0b76f1c93c5491ae9d9ecf866f6769b45a4/psutil-5.6.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"36d6733a6cb331fb0a7acb6d901557574835b038d157bc2fc11404d520fe4b92"},"data-dist-info-metadata":{"sha256":"36d6733a6cb331fb0a7acb6d901557574835b038d157bc2fc11404d520fe4b92"},"filename":"psutil-5.6.1-cp27-none-win32.whl","hashes":{"sha256":"23e9cd90db94fbced5151eaaf9033ae9667c033dffe9e709da761c20138d25b6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":223000,"upload-time":"2019-03-11T17:24:48.368476Z","url":"https://files.pythonhosted.org/packages/9d/f2/a84e650af7f04940709466384e94a0894cfe736e7cf43f48fb3bfb01be1b/psutil-5.6.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"36d6733a6cb331fb0a7acb6d901557574835b038d157bc2fc11404d520fe4b92"},"data-dist-info-metadata":{"sha256":"36d6733a6cb331fb0a7acb6d901557574835b038d157bc2fc11404d520fe4b92"},"filename":"psutil-5.6.1-cp27-none-win_amd64.whl","hashes":{"sha256":"e1494d20ffe7891d07d8cb9a8b306c1a38d48b13575265d090fc08910c56d474"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226302,"upload-time":"2019-03-11T17:24:51.027456Z","url":"https://files.pythonhosted.org/packages/46/51/4007e6188b6d6b68c8b4195d6755bd76585cffd4d286d9f1815ff9f1af01/psutil-5.6.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp35-cp35m-win32.whl","hashes":{"sha256":"ec4b4b638b84d42fc48139f9352f6c6587ee1018d55253542ee28db7480cc653"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226675,"upload-time":"2019-03-11T17:24:53.625833Z","url":"https://files.pythonhosted.org/packages/55/f4/2dc147b66111116f5cb6a85fe42519b2cbfdbf6138562f8b0427bc754fc8/psutil-5.6.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"c1fd45931889dc1812ba61a517630d126f6185f688eac1693171c6524901b7de"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230613,"upload-time":"2019-03-11T17:24:56.387303Z","url":"https://files.pythonhosted.org/packages/af/dd/f3bdafb37af7bf417f984db5381a5a27899f93fd2f87d5fd10fb6f3f4087/psutil-5.6.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp36-cp36m-win32.whl","hashes":{"sha256":"d463a142298112426ebd57351b45c39adb41341b91f033aa903fa4c6f76abecc"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226672,"upload-time":"2019-03-11T17:24:58.629155Z","url":"https://files.pythonhosted.org/packages/d5/cc/e3fe388fe3c987973e929cb17cceb33bcd2ff7f8b754cd354826fbb7dfe7/psutil-5.6.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"9c3a768486194b4592c7ae9374faa55b37b9877fd9746fb4028cb0ac38fd4c60"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230621,"upload-time":"2019-03-11T17:25:01.330880Z","url":"https://files.pythonhosted.org/packages/16/6a/cc5ba8d7e3ada0d4621d493dbdcb43ed38f3549642916a14c9e070add21a/psutil-5.6.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp37-cp37m-win32.whl","hashes":{"sha256":"27858d688a58cbfdd4434e1c40f6c79eb5014b709e725c180488ccdf2f721729"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226673,"upload-time":"2019-03-11T17:25:03.865746Z","url":"https://files.pythonhosted.org/packages/a8/8e/3e04eefe955ed94f93c3cde5dc1f1ccd99e2b9a56697fa804ea9f54f7baa/psutil-5.6.1-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"data-dist-info-metadata":{"sha256":"ea349a2dffe50e05b14983a2cf8238d799954fa7f79adc264520315279fa3b54"},"filename":"psutil-5.6.1-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"354601a1d1a1322ae5920ba397c58d06c29728a15113598d1a8158647aaa5385"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230621,"upload-time":"2019-03-11T17:25:06.996226Z","url":"https://files.pythonhosted.org/packages/6a/48/dbcda6d136da319e8bee8196e6c52ff7febf56bd241435cf6a516341a4b1/psutil-5.6.1-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.1.tar.gz","hashes":{"sha256":"fa0a570e0a30b9dd618bffbece590ae15726b47f9f1eaf7518dfb35f4d7dcd21"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":427472,"upload-time":"2019-03-11T17:25:09.802408Z","url":"https://files.pythonhosted.org/packages/2f/b8/11ec5006d2ec2998cb68349b8d1317c24c284cf918ecd6729739388e4c56/psutil-5.6.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"34922aac8bd27ca8268d8103c8b443cb013543dbfb0b3b7cf9f18d802b4fc760"},"data-dist-info-metadata":{"sha256":"34922aac8bd27ca8268d8103c8b443cb013543dbfb0b3b7cf9f18d802b4fc760"},"filename":"psutil-5.6.2-cp27-none-win32.whl","hashes":{"sha256":"76fb0956d6d50e68e3f22e7cc983acf4e243dc0fcc32fd693d398cb21c928802"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226692,"upload-time":"2019-04-26T02:43:03.614779Z","url":"https://files.pythonhosted.org/packages/56/8f/1bfb7e563e413f110ee06ac0cf12bb4b27f78e9cf277892d97ba08de4eac/psutil-5.6.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"34922aac8bd27ca8268d8103c8b443cb013543dbfb0b3b7cf9f18d802b4fc760"},"data-dist-info-metadata":{"sha256":"34922aac8bd27ca8268d8103c8b443cb013543dbfb0b3b7cf9f18d802b4fc760"},"filename":"psutil-5.6.2-cp27-none-win_amd64.whl","hashes":{"sha256":"753c5988edc07da00dafd6d3d279d41f98c62cd4d3a548c4d05741a023b0c2e7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230225,"upload-time":"2019-04-26T02:43:08.318806Z","url":"https://files.pythonhosted.org/packages/91/3f/2ae9cd04b2ccc5340838383ba638839a498d2613936b7830079f77de2bf1/psutil-5.6.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp35-cp35m-win32.whl","hashes":{"sha256":"a4c62319ec6bf2b3570487dd72d471307ae5495ce3802c1be81b8a22e438b4bc"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230751,"upload-time":"2019-04-26T02:43:12.014098Z","url":"https://files.pythonhosted.org/packages/ec/39/33a7d6c4e347ffff7784ec4967c04d6319886c5a77c208d29a0980045bc8/psutil-5.6.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"ef342cb7d9b60e6100364f50c57fa3a77d02ff8665d5b956746ac01901247ac4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234724,"upload-time":"2019-04-26T02:43:15.850786Z","url":"https://files.pythonhosted.org/packages/a9/43/87e610adacc3f8e51d61c27a9d48db2c0e7fbac783764fefca4d4ec71dbe/psutil-5.6.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp36-cp36m-win32.whl","hashes":{"sha256":"acba1df9da3983ec3c9c963adaaf530fcb4be0cd400a8294f1ecc2db56499ddd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230749,"upload-time":"2019-04-26T02:43:19.804035Z","url":"https://files.pythonhosted.org/packages/23/2a/0f14e964507adc1732ade6eea3abd92afe34305614515ab856cbccca489a/psutil-5.6.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"206eb909aa8878101d0eca07f4b31889c748f34ed6820a12eb3168c7aa17478e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234728,"upload-time":"2019-04-26T02:43:23.354778Z","url":"https://files.pythonhosted.org/packages/62/b0/54effe77128bdd8b62ca10edf38c22dbe5594ad8b34ce31836011949ac0a/psutil-5.6.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp37-cp37m-win32.whl","hashes":{"sha256":"649f7ffc02114dced8fbd08afcd021af75f5f5b2311bc0e69e53e8f100fe296f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230750,"upload-time":"2019-04-26T02:43:26.831308Z","url":"https://files.pythonhosted.org/packages/86/90/ea3d046bb26aebcbf25fba32cf9ec7d0954ea6b8e4e9d9c87a31633dd96b/psutil-5.6.2-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"data-dist-info-metadata":{"sha256":"11c017c8a57096625072309b7badf4a1f677690026dbfc728de5f3dfd4ae2936"},"filename":"psutil-5.6.2-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"6ebf2b9c996bb8c7198b385bade468ac8068ad8b78c54a58ff288cd5f61992c7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234731,"upload-time":"2019-04-26T02:43:30.698781Z","url":"https://files.pythonhosted.org/packages/98/3c/65f28f78848730c18dfa3ff3a107e3911b6d51d1442cfce8db53356179c3/psutil-5.6.2-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.2.tar.gz","hashes":{"sha256":"828e1c3ca6756c54ac00f1427fdac8b12e21b8a068c3bb9b631a1734cada25ed"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":432907,"upload-time":"2019-04-26T02:43:34.455685Z","url":"https://files.pythonhosted.org/packages/c6/c1/beed5e4eaa1345901b595048fab1c85aee647ea0fc02d9e8bf9aceb81078/psutil-5.6.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"b5acb81452f4830e88ff6014732a59e7fc4839457f7950deab90c8eed0d4e6b8"},"data-dist-info-metadata":{"sha256":"b5acb81452f4830e88ff6014732a59e7fc4839457f7950deab90c8eed0d4e6b8"},"filename":"psutil-5.6.3-cp27-none-win32.whl","hashes":{"sha256":"d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":226847,"upload-time":"2019-06-11T04:24:07.412816Z","url":"https://files.pythonhosted.org/packages/da/76/7e445566f2c4363691a98f16df0072c6ba92c7c29b4410ef2df6514c9861/psutil-5.6.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"b5acb81452f4830e88ff6014732a59e7fc4839457f7950deab90c8eed0d4e6b8"},"data-dist-info-metadata":{"sha256":"b5acb81452f4830e88ff6014732a59e7fc4839457f7950deab90c8eed0d4e6b8"},"filename":"psutil-5.6.3-cp27-none-win_amd64.whl","hashes":{"sha256":"b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230406,"upload-time":"2019-06-11T04:24:10.852056Z","url":"https://files.pythonhosted.org/packages/72/75/43047d7df3ea2af2bcd072e63420b8fa240b729c052295f8c4b964335d36/psutil-5.6.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp35-cp35m-win32.whl","hashes":{"sha256":"cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230912,"upload-time":"2019-06-11T04:24:14.533883Z","url":"https://files.pythonhosted.org/packages/c2/f2/d99eaeefb2c0b1f7f9aca679db17f4072d4cc362f40f809157d4e2d273dd/psutil-5.6.3-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234942,"upload-time":"2019-06-11T04:24:18.128893Z","url":"https://files.pythonhosted.org/packages/90/86/d5ae0eb79cab6acc00d3640a45243e3e0602dc2f7abca29fc2fe6b4819ca/psutil-5.6.3-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp36-cp36m-win32.whl","hashes":{"sha256":"eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230914,"upload-time":"2019-06-11T04:24:21.735106Z","url":"https://files.pythonhosted.org/packages/28/0c/e41fd3020662487cf92f0c47b09285d7e53c42ee56cdc3ddb03a147cfa5d/psutil-5.6.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234945,"upload-time":"2019-06-11T04:24:26.649579Z","url":"https://files.pythonhosted.org/packages/86/91/f15a3aae2af13f008ed95e02292d1a2e84615ff42b7203357c1c0bbe0651/psutil-5.6.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp37-cp37m-win32.whl","hashes":{"sha256":"503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230914,"upload-time":"2019-06-11T04:24:30.881936Z","url":"https://files.pythonhosted.org/packages/3b/92/2a7fb18054ac12483fb72f281c285d21642ca3d29fc6a06f0e44d4b36d83/psutil-5.6.3-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"data-dist-info-metadata":{"sha256":"2df22107b0d2ddaa4d080521f6dd4f4139500bf928eeee6cbbfd64b2abb197fb"},"filename":"psutil-5.6.3-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234949,"upload-time":"2019-06-11T04:24:34.722512Z","url":"https://files.pythonhosted.org/packages/7c/58/f5d68ddca37480d8557b8566a20bf6108d7e1c6c9b9208ee0786e0cd012b/psutil-5.6.3-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"991ea9c126c59fb12521c83a1676786ad9a7ea2af4f3985b1e8086af3eda4156"},"data-dist-info-metadata":{"sha256":"991ea9c126c59fb12521c83a1676786ad9a7ea2af4f3985b1e8086af3eda4156"},"filename":"psutil-5.6.3-cp38-cp38-win_amd64.whl","hashes":{"sha256":"12542c3642909f4cd1928a2fba59e16fa27e47cbeea60928ebb62a8cbd1ce123"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238041,"upload-time":"2019-10-24T10:11:31.818649Z","url":"https://files.pythonhosted.org/packages/a7/7f/0761489b5467af4e97ae3c5a25f24f8662d69a25692d85490c8ea5d52e45/psutil-5.6.3-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.3.tar.gz","hashes":{"sha256":"863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":435374,"upload-time":"2019-06-11T04:24:39.293855Z","url":"https://files.pythonhosted.org/packages/1c/ca/5b8c1fe032a458c2c4bcbe509d1401dca9dda35c7fc46b36bb81c2834740/psutil-5.6.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"2625785be98a15c8fa27587a95cae17a6673efa1e3bc2154856b9af05aaac32a"},"data-dist-info-metadata":{"sha256":"2625785be98a15c8fa27587a95cae17a6673efa1e3bc2154856b9af05aaac32a"},"filename":"psutil-5.6.4-cp27-none-win32.whl","hashes":{"sha256":"75d50d1138b2476a11dca33ab1ad2b78707d428418b581966ccedac768358f72"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228290,"upload-time":"2019-11-04T08:38:28.711265Z","url":"https://files.pythonhosted.org/packages/8a/63/0273163f0197ff2d8f026ae0f9c7cac371f8819f6451e6cfd2e1e7132b90/psutil-5.6.4-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"2625785be98a15c8fa27587a95cae17a6673efa1e3bc2154856b9af05aaac32a"},"data-dist-info-metadata":{"sha256":"2625785be98a15c8fa27587a95cae17a6673efa1e3bc2154856b9af05aaac32a"},"filename":"psutil-5.6.4-cp27-none-win_amd64.whl","hashes":{"sha256":"0ff1f630ee0df7c048ef53e50196437d2c9cebab8ccca0e3078d9300c4b7da47"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231797,"upload-time":"2019-11-04T08:38:33.155488Z","url":"https://files.pythonhosted.org/packages/db/d0/17a47a1876cf408d0dd679f78ba18a206228fdb95928137dd85538a51298/psutil-5.6.4-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp35-cp35m-win32.whl","hashes":{"sha256":"10175ea15b7e4a1bf1a0863da7e17042862b3ea3e7d24285c96fa4cc65ab9788"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232325,"upload-time":"2019-11-04T08:38:37.308918Z","url":"https://files.pythonhosted.org/packages/cb/22/e2ce6539342fb0ebd65fb9e2a0a168585eb5cae9bd5dca7977735f8d430e/psutil-5.6.4-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"f6b66a5663700b71bac3d8ecf6533a1550a679823e63b2c92dc4c3c8c244c52e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236342,"upload-time":"2019-11-04T08:38:41.583480Z","url":"https://files.pythonhosted.org/packages/32/9a/beaf5df1f8b38d3ec7ec737f873515813586aa26926738896957c3af0a9d/psutil-5.6.4-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp36-cp36m-win32.whl","hashes":{"sha256":"4f637dd25d3bce4879d0b4032d13f4120ba18ed2d028e85d911d429f447c251c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232330,"upload-time":"2019-11-04T08:38:46.049782Z","url":"https://files.pythonhosted.org/packages/74/3c/61d46eccf9fca27fc11f30d27af11c3886609d5eb52475926a1ad12d6ebd/psutil-5.6.4-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"43f0d7536a98c20a538242ce2bd8c64dbc1f6c396e97f2bdceb496d7583b9b80"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236347,"upload-time":"2019-11-04T08:38:50.265170Z","url":"https://files.pythonhosted.org/packages/2c/59/beae1392ad5188b419709d3e04641cbe93e36184b7b8686af825a2232b2b/psutil-5.6.4-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp37-cp37m-win32.whl","hashes":{"sha256":"f0ec1a3ea56503f4facc1dca364cf3dd66dc39169c4603000d3d34270e05fbb3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232327,"upload-time":"2019-11-04T08:38:54.615037Z","url":"https://files.pythonhosted.org/packages/e6/2e/e0c14d0fa46afb15b1467daa792f143e675034f8c544d03a2b7365d4926c/psutil-5.6.4-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"512e77ac987105e2d7aa2386d9f260434ad8b71e41484f8d84bfecd4ae3764ca"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236347,"upload-time":"2019-11-04T08:38:58.188759Z","url":"https://files.pythonhosted.org/packages/a3/59/fe32a3ec677990a5ed6b1a44dc2e372c9ee40693890814e8b6d96c290c4d/psutil-5.6.4-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp38-cp38-win32.whl","hashes":{"sha256":"41d645f100c6b4c995ff342ef7d79a936f3f48e9a816d7d655c69b352460341d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234629,"upload-time":"2019-11-04T08:46:40.809419Z","url":"https://files.pythonhosted.org/packages/50/0c/7b2fa0baedd36147beb820f662844b1079e168c06e9fe62d78a4d2666375/psutil-5.6.4-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"data-dist-info-metadata":{"sha256":"f16d89990a0e49264d1393eb1ee021563470e55f511d8518008f85c41d8ff3b0"},"filename":"psutil-5.6.4-cp38-cp38-win_amd64.whl","hashes":{"sha256":"fb58e87c29ec0fb99937b95c5d473bb786d263aaa767d017a6bd4ad52d694e79"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239057,"upload-time":"2019-11-04T08:46:45.192939Z","url":"https://files.pythonhosted.org/packages/e3/28/fa3a597ed798a5a2cb9c9e53c2d8b6bc1310c4a147c2aaa65c5d3afdb6ed/psutil-5.6.4-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.4.tar.gz","hashes":{"sha256":"512e854d68f8b42f79b2c7864d997b39125baff9bcff00028ce43543867de7c4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":447564,"upload-time":"2019-11-04T08:39:02.419203Z","url":"https://files.pythonhosted.org/packages/47/ea/d3b6d6fd0b4a6c12984df652525f394e68c8678d2b05075219144eb3a1cf/psutil-5.6.4.tar.gz","yanked":false},{"core-metadata":{"sha256":"3ccf04696d9a6c2c81c586788345860eb9fc45feada6313ab8d40865c7a9ec78"},"data-dist-info-metadata":{"sha256":"3ccf04696d9a6c2c81c586788345860eb9fc45feada6313ab8d40865c7a9ec78"},"filename":"psutil-5.6.5-cp27-none-win32.whl","hashes":{"sha256":"145e0f3ab9138165f9e156c307100905fd5d9b7227504b8a9d3417351052dc3d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228289,"upload-time":"2019-11-06T10:07:27.672551Z","url":"https://files.pythonhosted.org/packages/97/52/9a44db00d4400814384d574f3a2c2b588259c212e457f542c2589dbdac58/psutil-5.6.5-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"3ccf04696d9a6c2c81c586788345860eb9fc45feada6313ab8d40865c7a9ec78"},"data-dist-info-metadata":{"sha256":"3ccf04696d9a6c2c81c586788345860eb9fc45feada6313ab8d40865c7a9ec78"},"filename":"psutil-5.6.5-cp27-none-win_amd64.whl","hashes":{"sha256":"3feea46fbd634a93437b718518d15b5dd49599dfb59a30c739e201cc79bb759d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231796,"upload-time":"2019-11-06T10:07:31.973662Z","url":"https://files.pythonhosted.org/packages/c4/35/4422a966d304faa401d55dc45caf342722a14c0a9b56d57ecf208f9bb6a3/psutil-5.6.5-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp35-cp35m-win32.whl","hashes":{"sha256":"348ad4179938c965a27d29cbda4a81a1b2c778ecd330a221aadc7bd33681afbd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232325,"upload-time":"2019-11-06T10:07:36.463607Z","url":"https://files.pythonhosted.org/packages/4c/2f/93ac7f065f8c4c361e422dffff21dea1803cb40f2449e6ea52800caddf9f/psutil-5.6.5-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"474e10a92eeb4100c276d4cc67687adeb9d280bbca01031a3e41fb35dfc1d131"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236340,"upload-time":"2019-11-06T10:07:41.723247Z","url":"https://files.pythonhosted.org/packages/9f/49/fddf3b6138a5d47668e2daf98ac9a6eeea4c28e3eb68b56580f18e8fe6af/psutil-5.6.5-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp36-cp36m-win32.whl","hashes":{"sha256":"e3f5f9278867e95970854e92d0f5fe53af742a7fc4f2eba986943345bcaed05d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232327,"upload-time":"2019-11-06T10:07:45.647021Z","url":"https://files.pythonhosted.org/packages/ae/5d/11bb7fb7cc004bdf1325c0b40827e67e479ababe47c553ee871494353acf/psutil-5.6.5-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"dfb8c5c78579c226841908b539c2374da54da648ee5a837a731aa6a105a54c00"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236344,"upload-time":"2019-11-06T10:07:49.683064Z","url":"https://files.pythonhosted.org/packages/c6/18/221e8a5084585c6a2550894fb0617dc4691b5216f4aa6bb82c330aa5d99c/psutil-5.6.5-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp37-cp37m-win32.whl","hashes":{"sha256":"021d361439586a0fd8e64f8392eb7da27135db980f249329f1a347b9de99c695"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232327,"upload-time":"2019-11-06T10:07:53.427002Z","url":"https://files.pythonhosted.org/packages/31/f8/d612f18fed1422a016bb641d3ce1922904a2aa0cc3472ce4abf2716cf542/psutil-5.6.5-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"e9649bb8fc5cea1f7723af53e4212056a6f984ee31784c10632607f472dec5ee"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236346,"upload-time":"2019-11-06T10:07:57.218779Z","url":"https://files.pythonhosted.org/packages/97/c7/9b18c3b429c987796d74647c61030c7029518ac223d1a664951417781fe4/psutil-5.6.5-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp38-cp38-win32.whl","hashes":{"sha256":"47aeb4280e80f27878caae4b572b29f0ec7967554b701ba33cd3720b17ba1b07"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234627,"upload-time":"2019-11-06T10:08:50.811061Z","url":"https://files.pythonhosted.org/packages/03/94/e4ee514cfbc4cca176fcc6b4b1118a724848b570941e90f0b98a9bd234e1/psutil-5.6.5-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"data-dist-info-metadata":{"sha256":"98b3da0b296c70a30e4e371a72ba5d3205dc455f09fe824464a824ad9b4fd6ad"},"filename":"psutil-5.6.5-cp38-cp38-win_amd64.whl","hashes":{"sha256":"73a7e002781bc42fd014dfebb3fc0e45f8d92a4fb9da18baea6fb279fbc1d966"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239056,"upload-time":"2019-11-06T10:08:55.334494Z","url":"https://files.pythonhosted.org/packages/cd/3b/de8a1f1692de2f16716a108a63366aa66692e5a087b6e0458eef9739e652/psutil-5.6.5-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.5.tar.gz","hashes":{"sha256":"d051532ac944f1be0179e0506f6889833cf96e466262523e57a871de65a15147"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":447489,"upload-time":"2019-11-06T10:08:01.639000Z","url":"https://files.pythonhosted.org/packages/03/9a/95c4b3d0424426e5fd94b5302ff74cea44d5d4f53466e1228ac8e73e14b4/psutil-5.6.5.tar.gz","yanked":false},{"core-metadata":{"sha256":"0ee5221d5d9af774f9490843a768567a29b9f7ace20754cf0822d80eb298fdd1"},"data-dist-info-metadata":{"sha256":"0ee5221d5d9af774f9490843a768567a29b9f7ace20754cf0822d80eb298fdd1"},"filename":"psutil-5.6.6-cp27-none-win32.whl","hashes":{"sha256":"06660136ab88762309775fd47290d7da14094422d915f0466e0adf8e4b22214e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228564,"upload-time":"2019-11-25T12:30:48.823010Z","url":"https://files.pythonhosted.org/packages/c9/c2/4d703fcb5aacd4cb8d472d3c1d0e7d8c21e17f37b515016a7ee34ff3f4a0/psutil-5.6.6-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"0ee5221d5d9af774f9490843a768567a29b9f7ace20754cf0822d80eb298fdd1"},"data-dist-info-metadata":{"sha256":"0ee5221d5d9af774f9490843a768567a29b9f7ace20754cf0822d80eb298fdd1"},"filename":"psutil-5.6.6-cp27-none-win_amd64.whl","hashes":{"sha256":"f21a7bb4b207e4e7c60b3c40ffa89d790997619f04bbecec9db8e3696122bc78"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232093,"upload-time":"2019-11-25T12:30:53.306595Z","url":"https://files.pythonhosted.org/packages/76/4d/6452c4791f9d95b48cca084b7cc6aa8a72b2f4c8d3d8bd38e7f3abfaf364/psutil-5.6.6-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp35-cp35m-win32.whl","hashes":{"sha256":"5e8dbf31871b0072bcba8d1f2861c0ec6c84c78f13c723bb6e981bce51b58f12"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232818,"upload-time":"2019-11-25T12:30:58.414546Z","url":"https://files.pythonhosted.org/packages/f3/b3/e585b9b0c5a40e6a778e32e8d3040ab2363433990417202bf6cc0261ed77/psutil-5.6.6-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"724390895cff80add7a1c4e7e0a04d9c94f3ee61423a2dcafd83784fabbd1ee9"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236751,"upload-time":"2019-11-25T12:31:03.059024Z","url":"https://files.pythonhosted.org/packages/73/38/a8ebf6dc6ada2257591284be45a52dcd19479163b8d3575186333a79a18e/psutil-5.6.6-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp36-cp36m-win32.whl","hashes":{"sha256":"6d81b9714791ef9a3a00b2ca846ee547fc5e53d259e2a6258c3d2054928039ff"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232821,"upload-time":"2019-11-25T12:31:08.161481Z","url":"https://files.pythonhosted.org/packages/5b/04/2223b4fe61d3e5962c08ce5062b09633fcfdd8c3bb08c31b76306f748431/psutil-5.6.6-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"3004361c6b93dbad71330d992c1ae409cb8314a6041a0b67507cc882357f583e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236759,"upload-time":"2019-11-25T12:31:13.077581Z","url":"https://files.pythonhosted.org/packages/33/6c/6eb959bca82064d42e725dddd3aeeb39d9bed34eed7b513880bcfd8a3d59/psutil-5.6.6-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp37-cp37m-win32.whl","hashes":{"sha256":"0fc7a5619b47f74331add476fbc6022d7ca801c22865c7069ec0867920858963"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232820,"upload-time":"2019-11-25T12:31:17.411233Z","url":"https://files.pythonhosted.org/packages/76/46/3a8dc20eb9d7d2c44178e71c2412dcc2c6476ab71c05a2cf2f77247c6e53/psutil-5.6.6-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"f60042bef7dc50a78c06334ca8e25580455948ba2fa98f240d034a4fed9141a5"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236757,"upload-time":"2019-11-25T12:31:21.816790Z","url":"https://files.pythonhosted.org/packages/40/48/5debf9783077ac71f0d715c1171fde9ef287909f61672ed9a3e5fdca63cc/psutil-5.6.6-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp38-cp38-win32.whl","hashes":{"sha256":"0c11adde31011a286197630ba2671e34651f004cc418d30ae06d2033a43c9e20"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233147,"upload-time":"2019-11-25T12:31:26.294340Z","url":"https://files.pythonhosted.org/packages/c6/10/035c432a15bec90d51f3625d4b70b7d12f1b363062d0d8815213229f69ca/psutil-5.6.6-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"data-dist-info-metadata":{"sha256":"29b50729d8b20274ad25b9f22deb812c81cdc56e61cba32bbf7c5423b9c1ef5d"},"filename":"psutil-5.6.6-cp38-cp38-win_amd64.whl","hashes":{"sha256":"0c211eec4185725847cb6c28409646c7cfa56fdb531014b35f97b5dc7fe04ff9"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":237165,"upload-time":"2019-11-25T12:31:30.835876Z","url":"https://files.pythonhosted.org/packages/95/81/fb02ea8de73eca26cfa347f5ce81a7963b9dee6e038e0a7389ccbc971093/psutil-5.6.6-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.6.tar.gz","hashes":{"sha256":"ad21281f7bd6c57578dd53913d2d44218e9e29fd25128d10ff7819ef16fa46e7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":447805,"upload-time":"2019-11-25T12:31:36.347718Z","url":"https://files.pythonhosted.org/packages/5f/dc/edf6758183afc7591a16bd4b8a44d8eea80aca1327ea60161dd3bad9ad22/psutil-5.6.6.tar.gz","yanked":false},{"core-metadata":{"sha256":"943b5895bc5955d1ab23e61bd0a4a6c4a4be1be625fd6bb204033b5d93574bf6"},"data-dist-info-metadata":{"sha256":"943b5895bc5955d1ab23e61bd0a4a6c4a4be1be625fd6bb204033b5d93574bf6"},"filename":"psutil-5.6.7-cp27-none-win32.whl","hashes":{"sha256":"1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":228449,"upload-time":"2019-11-26T07:25:43.864472Z","url":"https://files.pythonhosted.org/packages/13/a7/626f257d22168c954fd3ad69760c02bdec27c0648a62f6ea5060c4d40672/psutil-5.6.7-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"943b5895bc5955d1ab23e61bd0a4a6c4a4be1be625fd6bb204033b5d93574bf6"},"data-dist-info-metadata":{"sha256":"943b5895bc5955d1ab23e61bd0a4a6c4a4be1be625fd6bb204033b5d93574bf6"},"filename":"psutil-5.6.7-cp27-none-win_amd64.whl","hashes":{"sha256":"28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231969,"upload-time":"2019-11-26T07:25:50.426501Z","url":"https://files.pythonhosted.org/packages/52/44/e1e1954da522ea8640e035c8b101c116a9f8a0e94e04e108e56911064de5/psutil-5.6.7-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp35-cp35m-win32.whl","hashes":{"sha256":"21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232644,"upload-time":"2019-11-26T07:25:56.230501Z","url":"https://files.pythonhosted.org/packages/d8/39/bf74da6282d9521fe3987b2d67f581b3464e635e6cb56660d0315d1bf1ed/psutil-5.6.7-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236538,"upload-time":"2019-11-26T07:26:01.396655Z","url":"https://files.pythonhosted.org/packages/b3/84/0a899c6fac13aedfb4734413a2357a504c4f21cbf8acd7ad4caa6712d8cf/psutil-5.6.7-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp36-cp36m-win32.whl","hashes":{"sha256":"e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232649,"upload-time":"2019-11-26T07:26:06.727306Z","url":"https://files.pythonhosted.org/packages/7c/d7/be2b607abfab4a98f04dd2155d6a7a40a666618d69c079897f09ce776a34/psutil-5.6.7-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236541,"upload-time":"2019-11-26T07:26:11.229459Z","url":"https://files.pythonhosted.org/packages/78/e6/ce8a91afd605f254342f1294790f2a77c76202386d6927eb5ff0e36e4449/psutil-5.6.7-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp37-cp37m-win32.whl","hashes":{"sha256":"094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232645,"upload-time":"2019-11-26T07:26:15.814275Z","url":"https://files.pythonhosted.org/packages/c4/e1/80c7840db569ad5b1b60987893e066a5536779c0d2402363cbf1230613a2/psutil-5.6.7-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236539,"upload-time":"2019-11-26T07:26:20.217732Z","url":"https://files.pythonhosted.org/packages/9d/84/0a2006cc263e9f5b6dfbb2301fbcce5558f0d6d17d0c11c7c6749a45c79e/psutil-5.6.7-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp38-cp38-win32.whl","hashes":{"sha256":"70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232983,"upload-time":"2019-11-26T07:26:24.615122Z","url":"https://files.pythonhosted.org/packages/55/9d/9a6df5f730a1e2a3938fad0ccf541b30fad34706128b43ed3f965eaf7550/psutil-5.6.7-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"data-dist-info-metadata":{"sha256":"15f7352602bc6ef07ec1f8cc5816d4cee0171e750a5d38106a7277f703882211"},"filename":"psutil-5.6.7-cp38-cp38-win_amd64.whl","hashes":{"sha256":"10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236965,"upload-time":"2019-11-26T07:26:28.827408Z","url":"https://files.pythonhosted.org/packages/8a/fa/b573850e912d6ffdad4aef3f5f705f94a64d098a83eec15d1cd3e1223f5e/psutil-5.6.7-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.6.7.tar.gz","hashes":{"sha256":"ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":448321,"upload-time":"2019-11-26T07:26:34.515073Z","url":"https://files.pythonhosted.org/packages/73/93/4f8213fbe66fc20cb904f35e6e04e20b47b85bee39845cc66a0bcf5ccdcb/psutil-5.6.7.tar.gz","yanked":false},{"core-metadata":{"sha256":"4665f6ffe5781ebc50d69efc5363ace07e50d1916f179a1cd2e51352df7eb11a"},"data-dist-info-metadata":{"sha256":"4665f6ffe5781ebc50d69efc5363ace07e50d1916f179a1cd2e51352df7eb11a"},"filename":"psutil-5.7.0-cp27-none-win32.whl","hashes":{"sha256":"298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":227641,"upload-time":"2020-02-18T18:02:31.085536Z","url":"https://files.pythonhosted.org/packages/0b/6b/f613593812c5f379c6d609bf5eca36a409812f508e13c704acd25712a73e/psutil-5.7.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"4665f6ffe5781ebc50d69efc5363ace07e50d1916f179a1cd2e51352df7eb11a"},"data-dist-info-metadata":{"sha256":"4665f6ffe5781ebc50d69efc5363ace07e50d1916f179a1cd2e51352df7eb11a"},"filename":"psutil-5.7.0-cp27-none-win_amd64.whl","hashes":{"sha256":"75e22717d4dbc7ca529ec5063000b2b294fc9a367f9c9ede1f65846c7955fd38"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":230741,"upload-time":"2020-02-18T18:02:35.453910Z","url":"https://files.pythonhosted.org/packages/79/b1/377fa0f28630d855cb6b5bfb2ee4c1bf0df3bc2603c691ceefce59a95181/psutil-5.7.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp35-cp35m-win32.whl","hashes":{"sha256":"f344ca230dd8e8d5eee16827596f1c22ec0876127c28e800d7ae20ed44c4b310"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231151,"upload-time":"2020-02-18T18:02:39.410543Z","url":"https://files.pythonhosted.org/packages/74/e6/4a0ef10b1a4ca43954cd8fd9eac02cc8606f9d2a5a66859a283f5f95452b/psutil-5.7.0-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"e2d0c5b07c6fe5a87fa27b7855017edb0d52ee73b71e6ee368fae268605cc3f5"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235435,"upload-time":"2020-02-18T18:02:43.014979Z","url":"https://files.pythonhosted.org/packages/65/c2/0aeb9f0cc7e4be2807aa052b3fd017e59439ed6d830b461f8ecb35b2f367/psutil-5.7.0-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp36-cp36m-win32.whl","hashes":{"sha256":"a02f4ac50d4a23253b68233b07e7cdb567bd025b982d5cf0ee78296990c22d9e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231156,"upload-time":"2020-02-18T18:02:46.432233Z","url":"https://files.pythonhosted.org/packages/c9/37/b94930ae428b2d67d505aecc5ba84c53a0b75479a8a87cd35cc9a2c6eb7e/psutil-5.7.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235439,"upload-time":"2020-02-18T18:02:49.890797Z","url":"https://files.pythonhosted.org/packages/4f/3c/205850b172a14a8b9fdc9b1e84a2c055d6b9aea226431da7685bea644f04/psutil-5.7.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp37-cp37m-win32.whl","hashes":{"sha256":"d008ddc00c6906ec80040d26dc2d3e3962109e40ad07fd8a12d0284ce5e0e4f8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231153,"upload-time":"2020-02-18T18:02:53.653943Z","url":"https://files.pythonhosted.org/packages/54/25/7825fefd62635f7ca556c8e0d44369ce4674aa2ca0eca50b8ae4ff49954b/psutil-5.7.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"73f35ab66c6c7a9ce82ba44b1e9b1050be2a80cd4dcc3352cc108656b115c74f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235439,"upload-time":"2020-02-18T18:02:57.139440Z","url":"https://files.pythonhosted.org/packages/86/f7/385040b90dd190edc28908c4a26af99b00ae37564ee5f5c4526dc1d80c27/psutil-5.7.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp38-cp38-win32.whl","hashes":{"sha256":"60b86f327c198561f101a92be1995f9ae0399736b6eced8f24af41ec64fb88d4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":231789,"upload-time":"2020-02-18T18:03:00.808478Z","url":"https://files.pythonhosted.org/packages/63/d5/f34a9433a0299d944605fb5a970306a89e076f5412164179dc59ebf70fa9/psutil-5.7.0-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"data-dist-info-metadata":{"sha256":"9743d37444bb906e26ad0d76922bd72314c8385e19a68cb8a763a3e70859b867"},"filename":"psutil-5.7.0-cp38-cp38-win_amd64.whl","hashes":{"sha256":"d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235952,"upload-time":"2020-02-18T18:03:04.130414Z","url":"https://files.pythonhosted.org/packages/86/fe/9f1d1f8c1c8138d42fc0e7c06ca5004e01f38e86e61342374d8e0fa919e4/psutil-5.7.0-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.7.0.tar.gz","hashes":{"sha256":"685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":449628,"upload-time":"2020-02-18T18:03:07.566242Z","url":"https://files.pythonhosted.org/packages/c4/b8/3512f0e93e0db23a71d82485ba256071ebef99b227351f0f5540f744af41/psutil-5.7.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"2dfb7b5638ffaa33602a86b39cca60cded2324dabbe2617b1b5e65250e448769"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233646,"upload-time":"2020-07-15T11:14:11.635539Z","url":"https://files.pythonhosted.org/packages/79/e4/cbaa3ecc458c2dd8da64073de983473543b8b6ef4ca21159cea9069d53dd/psutil-5.7.1-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp35-cp35m-macosx_10_9_x86_64.whl","hashes":{"sha256":"36c5e6882caf3d385c6c3a0d2f3b302b4cc337c808ea589d9a8c563b545beb8b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233862,"upload-time":"2020-07-15T11:14:27.607855Z","url":"https://files.pythonhosted.org/packages/4d/d9/48c3d16c1dfbbf528bd69254b5a604c9f6860f12169b1b73a5005723c6bf/psutil-5.7.1-cp35-cp35m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp35-cp35m-win32.whl","hashes":{"sha256":"3c5ffd00bc1ee809350dca97613985d387a7e13dff61d62fc1bdf4dc10892ddd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238370,"upload-time":"2020-07-15T11:14:34.746888Z","url":"https://files.pythonhosted.org/packages/d0/e2/d4cdadda6a9fba79026ab628fc2b4da5e2e48dcfc6beada0a39363732ba1/psutil-5.7.1-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"4975c33aebe7de191d745ee3c545e907edd14d65c850a0b185c05024aa77cbcd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242862,"upload-time":"2020-07-15T11:14:37.084757Z","url":"https://files.pythonhosted.org/packages/a0/6b/cdb41805a6bb62c051cfbb1b65a9cb40767e0144b3d40fdd7082d8271701/psutil-5.7.1-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"436a6e99098eba14b54a149f921c9d4e1df729f02645876af0c828396d36c46a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233861,"upload-time":"2020-07-15T11:14:39.242780Z","url":"https://files.pythonhosted.org/packages/1f/fb/097aeed40c361225cb69d6d04202421f2c172d7e42753130d1b619f68956/psutil-5.7.1-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp36-cp36m-win32.whl","hashes":{"sha256":"630ceda48c16b24ffd981fe06ae1a43684af1a3a837d6a3496a1be3dd3c7d332"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238378,"upload-time":"2020-07-15T11:14:45.606142Z","url":"https://files.pythonhosted.org/packages/ae/49/cba9353fd9946eac95031c85763daaf7904d3c3e8b0b4f2801199586b413/psutil-5.7.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"d3bb7f65199595a72a3ec53e4d05c159857ab832fadaae9d85e68db467d2d191"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242864,"upload-time":"2020-07-15T11:14:47.861971Z","url":"https://files.pythonhosted.org/packages/7b/38/f27fc6a30f81be1ee657bd4c355c2dc03a5fbb49f304a37c79c0bed05821/psutil-5.7.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"66d085317599684f70d995dd4a770894f518fb34d027d7f742b579bf47732858"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":233863,"upload-time":"2020-07-15T11:14:49.991128Z","url":"https://files.pythonhosted.org/packages/23/cb/410a516385c8cd69f090f98c8014636c51d124c96e4d6ab51e1bb2d04232/psutil-5.7.1-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp37-cp37m-win32.whl","hashes":{"sha256":"fb442b912fe28d80e0f966adcc3df4e394fbb7ef7575ae21fd171aeb06c8b0df"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238369,"upload-time":"2020-07-15T11:14:57.383010Z","url":"https://files.pythonhosted.org/packages/59/44/e9cfa470dd2790b5475ceb590949842a5f2feb52445e898576b721033f04/psutil-5.7.1-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"425d6c95ca3ece7ff4da7e67af2954b8eb56b0f15743b237dc84ad975f51c2a4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242865,"upload-time":"2020-07-15T11:14:59.839217Z","url":"https://files.pythonhosted.org/packages/06/76/b4607e0eaf36369ad86f7ac73bde19aeaf32c82fb22675cb8f8dd975c692/psutil-5.7.1-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"f2817a763c33c19fdefbb832c790bc85b3de90b51fb69dae43097a9885be0332"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234129,"upload-time":"2020-07-15T11:15:01.982258Z","url":"https://files.pythonhosted.org/packages/ba/f7/64bf7fd7a12a40c50408b7d90cdf3addc28071e5463af1dbb7f3884a32d2/psutil-5.7.1-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp38-cp38-win32.whl","hashes":{"sha256":"3cf43d2265ee03fcf70f0f574487ed19435c92a330e15a3e773144811c1275f0"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239004,"upload-time":"2020-07-15T11:15:08.125783Z","url":"https://files.pythonhosted.org/packages/7e/64/c3bd24d53f6056ded095e8d147c0ca269bb6d858aea903561bd660d67035/psutil-5.7.1-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"data-dist-info-metadata":{"sha256":"5cfd93088b3a9182d5543595d138ec1f3a9f0fee62e4e1dcbc3332900a0871ea"},"filename":"psutil-5.7.1-cp38-cp38-win_amd64.whl","hashes":{"sha256":"006b720a67881037c8b02b1de012a39a2f007bd2b1b244b58fabef8eff0ad6d2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243418,"upload-time":"2020-07-15T11:15:10.044651Z","url":"https://files.pythonhosted.org/packages/ff/5a/1e990cf86f47721225143ed4a903a226685fa1ba0b43500f52d71500b1be/psutil-5.7.1-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"621e2f3b573aa563717ee39c3f052b4cd3cda5e1f25e5973fa77e727433c6ce1"},"data-dist-info-metadata":{"sha256":"621e2f3b573aa563717ee39c3f052b4cd3cda5e1f25e5973fa77e727433c6ce1"},"filename":"psutil-5.7.1-pp27-pypy_73-macosx_10_9_x86_64.whl","hashes":{"sha256":"0c9187ec0c314a128362c3409afea2b80c6d6d2c2cb1d661fe20631a2ff8ad77"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":232076,"upload-time":"2020-07-15T11:15:12.408818Z","url":"https://files.pythonhosted.org/packages/80/fd/d91ff7582513d093097678eedd141a4879698da26e6163fbae16905aa75b/psutil-5.7.1-pp27-pypy_73-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.7.1.tar.gz","hashes":{"sha256":"4ef6845b35e152e6937d4f28388c2440ca89a0089ced0a30a116fa3ceefdfa3a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":460148,"upload-time":"2020-07-15T11:15:21.985314Z","url":"https://files.pythonhosted.org/packages/2c/5c/cb95a715fb635e1ca858ffb8c50a523a16e2dc06aa3e207ab73cb93516af/psutil-5.7.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"91f2a99bfe758ed3b22b8fc235d7bfa5b885bb5cdd7bd1a38cccff1e9756d183"},"data-dist-info-metadata":{"sha256":"91f2a99bfe758ed3b22b8fc235d7bfa5b885bb5cdd7bd1a38cccff1e9756d183"},"filename":"psutil-5.7.2-cp27-none-win32.whl","hashes":{"sha256":"f2018461733b23f308c298653c8903d32aaad7873d25e1d228765e91ae42c3f2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":234782,"upload-time":"2020-07-15T13:19:05.321836Z","url":"https://files.pythonhosted.org/packages/d0/da/d7da0365f690e7555f6dda34bcb5bde10266379c9a23ee6a0735c3a7fdfd/psutil-5.7.2-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"91f2a99bfe758ed3b22b8fc235d7bfa5b885bb5cdd7bd1a38cccff1e9756d183"},"data-dist-info-metadata":{"sha256":"91f2a99bfe758ed3b22b8fc235d7bfa5b885bb5cdd7bd1a38cccff1e9756d183"},"filename":"psutil-5.7.2-cp27-none-win_amd64.whl","hashes":{"sha256":"66c18ca7680a31bf16ee22b1d21b6397869dda8059dbdb57d9f27efa6615f195"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238050,"upload-time":"2020-07-15T13:19:07.922423Z","url":"https://files.pythonhosted.org/packages/7d/9d/30a053a06d598cee4bdbc6ba69df44ced9e6d2ebb16e2de401a2a3bc6d63/psutil-5.7.2-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp35-cp35m-win32.whl","hashes":{"sha256":"5e9d0f26d4194479a13d5f4b3798260c20cecf9ac9a461e718eb59ea520a360c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238373,"upload-time":"2020-07-15T13:19:10.146779Z","url":"https://files.pythonhosted.org/packages/df/27/e5cf14b0894b4f06c23dc4f58288c60b17e71d8bef9af463f0b32ee46773/psutil-5.7.2-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"4080869ed93cce662905b029a1770fe89c98787e543fa7347f075ade761b19d6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242862,"upload-time":"2020-07-15T13:19:12.906781Z","url":"https://files.pythonhosted.org/packages/0a/4c/d31d58992314664e69bda6d575c1fd47b86ed5d67e00e300fc909040a9aa/psutil-5.7.2-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp36-cp36m-win32.whl","hashes":{"sha256":"d8a82162f23c53b8525cf5f14a355f5d1eea86fa8edde27287dd3a98399e4fdf"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238380,"upload-time":"2020-07-15T13:19:15.247084Z","url":"https://files.pythonhosted.org/packages/57/c5/0aa3b1513b914a417db7ee149b60579a139111f81f79f5f1d38ae440cebf/psutil-5.7.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"0ee3c36428f160d2d8fce3c583a0353e848abb7de9732c50cf3356dd49ad63f8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242866,"upload-time":"2020-07-15T13:19:17.782781Z","url":"https://files.pythonhosted.org/packages/22/f8/7be159475303a508347efc82c0d5858b1786fe73fc2a6b21d82891791920/psutil-5.7.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp37-cp37m-win32.whl","hashes":{"sha256":"ff1977ba1a5f71f89166d5145c3da1cea89a0fdb044075a12c720ee9123ec818"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238375,"upload-time":"2020-07-15T13:19:20.591814Z","url":"https://files.pythonhosted.org/packages/56/de/6f07749f275d0ba7f9b985cd6a4526e2fa47ad63b5179948c6650117f7d9/psutil-5.7.2-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"a5b120bb3c0c71dfe27551f9da2f3209a8257a178ed6c628a819037a8df487f1"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242863,"upload-time":"2020-07-15T13:19:23.022907Z","url":"https://files.pythonhosted.org/packages/f8/9b/1d7df5e1747e047abef4ec877d895b642f3a796ab8bd2e0f682516740dfe/psutil-5.7.2-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp38-cp38-win32.whl","hashes":{"sha256":"10512b46c95b02842c225f58fa00385c08fa00c68bac7da2d9a58ebe2c517498"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239003,"upload-time":"2020-07-15T13:19:25.770960Z","url":"https://files.pythonhosted.org/packages/da/d6/f66bbdbc8831a5cc78ba0e9bf69d924e68eac1a7b4191de93cf4e3643c54/psutil-5.7.2-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"data-dist-info-metadata":{"sha256":"4ece4267ea2940759bedb71b133199d6c5b33ccc17f4c3060d08d56f5697ca45"},"filename":"psutil-5.7.2-cp38-cp38-win_amd64.whl","hashes":{"sha256":"68d36986ded5dac7c2dcd42f2682af1db80d4bce3faa126a6145c1637e1b559f"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243419,"upload-time":"2020-07-15T13:19:28.240707Z","url":"https://files.pythonhosted.org/packages/6c/e6/f963547a36a96f74244cbe5e4046a02f140e3b7cbc5e5176035b38e2deb2/psutil-5.7.2-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.7.2.tar.gz","hashes":{"sha256":"90990af1c3c67195c44c9a889184f84f5b2320dce3ee3acbd054e3ba0b4a7beb"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":460198,"upload-time":"2020-07-15T13:19:30.438785Z","url":"https://files.pythonhosted.org/packages/aa/3e/d18f2c04cf2b528e18515999b0c8e698c136db78f62df34eee89cee205f1/psutil-5.7.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"6032661358cee4f792940fdac32ebc65bab2fa2463828785b8a8f0fb6c2c210b"},"data-dist-info-metadata":{"sha256":"6032661358cee4f792940fdac32ebc65bab2fa2463828785b8a8f0fb6c2c210b"},"filename":"psutil-5.7.3-cp27-none-win32.whl","hashes":{"sha256":"1cd6a0c9fb35ece2ccf2d1dd733c1e165b342604c67454fd56a4c12e0a106787"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235225,"upload-time":"2020-10-24T14:00:54.139761Z","url":"https://files.pythonhosted.org/packages/1d/a2/b732590561ef9d7dbc078ed0e2635e282115604a478911fef97ddaa3ad43/psutil-5.7.3-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"6032661358cee4f792940fdac32ebc65bab2fa2463828785b8a8f0fb6c2c210b"},"data-dist-info-metadata":{"sha256":"6032661358cee4f792940fdac32ebc65bab2fa2463828785b8a8f0fb6c2c210b"},"filename":"psutil-5.7.3-cp27-none-win_amd64.whl","hashes":{"sha256":"e02c31b2990dcd2431f4524b93491941df39f99619b0d312dfe1d4d530b08b4b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238499,"upload-time":"2020-10-24T14:00:57.724543Z","url":"https://files.pythonhosted.org/packages/b4/4c/c14a9485957b00c20f70e208a03663e81ddc8dafdf5137fee2d50aa1ee5e/psutil-5.7.3-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp35-cp35m-win32.whl","hashes":{"sha256":"56c85120fa173a5d2ad1d15a0c6e0ae62b388bfb956bb036ac231fbdaf9e4c22"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238813,"upload-time":"2020-10-24T15:02:21.678008Z","url":"https://files.pythonhosted.org/packages/e0/ba/a7c1096470e11c449019690ee9e7fd3adca1a4b9cfa6e5a13b60db3187b4/psutil-5.7.3-cp35-cp35m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp35-cp35m-win_amd64.whl","hashes":{"sha256":"fa38ac15dbf161ab1e941ff4ce39abd64b53fec5ddf60c23290daed2bc7d1157"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243298,"upload-time":"2020-10-24T15:02:24.823245Z","url":"https://files.pythonhosted.org/packages/3f/7c/98aada1208462c841788712383f4288b3c31e45504570a818c0c303d78e7/psutil-5.7.3-cp35-cp35m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp36-cp36m-win32.whl","hashes":{"sha256":"01bc82813fbc3ea304914581954979e637bcc7084e59ac904d870d6eb8bb2bc7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238821,"upload-time":"2020-10-24T15:02:27.844759Z","url":"https://files.pythonhosted.org/packages/44/a8/ebfcbb4967e74a27049ea6e13b3027ae05c0cb73d1a2b71c2f0519c6d5f2/psutil-5.7.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"6a3e1fd2800ca45083d976b5478a2402dd62afdfb719b30ca46cd28bb25a2eb4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243300,"upload-time":"2020-10-24T15:02:31.552663Z","url":"https://files.pythonhosted.org/packages/9a/c3/3b0023b46fc038eff02fbb69a0e6e50d15a7dce25e717d8469e8eaa837a7/psutil-5.7.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp37-cp37m-win32.whl","hashes":{"sha256":"fbcac492cb082fa38d88587d75feb90785d05d7e12d4565cbf1ecc727aff71b7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238814,"upload-time":"2020-10-24T15:02:34.703606Z","url":"https://files.pythonhosted.org/packages/e1/f0/d4f58ddf077d970440b82b92e909e8e9b2f50e39a2dc2aa716b1e2fde5ef/psutil-5.7.3-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"5d9106ff5ec2712e2f659ebbd112967f44e7d33f40ba40530c485cc5904360b8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243302,"upload-time":"2020-10-24T15:02:38.016752Z","url":"https://files.pythonhosted.org/packages/f1/a0/094a6e32185bd1288a4681d91ebe362d5b41aa64413bbbd96ed547051f17/psutil-5.7.3-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp38-cp38-win32.whl","hashes":{"sha256":"ade6af32eb80a536eff162d799e31b7ef92ddcda707c27bbd077238065018df4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239426,"upload-time":"2020-10-24T15:02:41.250786Z","url":"https://files.pythonhosted.org/packages/bd/95/394485321a128f5ddb23f0a559f940309280f57bd6117580868fb2d5a246/psutil-5.7.3-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"data-dist-info-metadata":{"sha256":"9f3f41a734ced0a9775d53996e940b0d2411f0f5105b0cdcac5e8e3ffd8da0e9"},"filename":"psutil-5.7.3-cp38-cp38-win_amd64.whl","hashes":{"sha256":"2cb55ef9591b03ef0104bedf67cc4edb38a3edf015cf8cf24007b99cb8497542"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243849,"upload-time":"2020-10-24T15:02:44.162783Z","url":"https://files.pythonhosted.org/packages/df/64/8d7b55ac87e67398ffc260d43a5fb327f1e230b09758b7d8caaecf917dd6/psutil-5.7.3-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.7.3.tar.gz","hashes":{"sha256":"af73f7bcebdc538eda9cc81d19db1db7bf26f103f91081d780bbacfcb620dee2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":465556,"upload-time":"2020-10-24T14:02:15.604830Z","url":"https://files.pythonhosted.org/packages/33/e0/82d459af36bda999f82c7ea86c67610591cf5556168f48fd6509e5fa154d/psutil-5.7.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":235772,"upload-time":"2020-12-19T01:19:27.017908Z","url":"https://files.pythonhosted.org/packages/f5/7f/a2559a514bdeb2a33e4bf3dc3d2bb17d5acded718893869a82536130cfb3/psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":284461,"upload-time":"2020-12-19T01:19:30.158384Z","url":"https://files.pythonhosted.org/packages/19/2c/9f1bad783faee4e9704868f381913e68dbb69f0de3fcdc71ee7071c47847/psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":287768,"upload-time":"2020-12-19T01:19:33.394809Z","url":"https://files.pythonhosted.org/packages/82/0a/eddb9a51ba5055cc7c242da07c1643a6b146070740c5eb5540277a0f01f4/psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":284476,"upload-time":"2020-12-19T01:19:36.479406Z","url":"https://files.pythonhosted.org/packages/15/28/47c28171fd7eeb83df74f78ccac090211f4a49408f376eb8e78a7bb47dc0/psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":287754,"upload-time":"2020-12-19T01:19:39.219693Z","url":"https://files.pythonhosted.org/packages/26/ef/461e9eec56fba7fa66692c4af00cbd6547b788a7ca818d9b8b5f1951f228/psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"91c5c4b49dc8b3365ce65ca99e7cee1cf8d862d4379fcc384b1711d4600dd869"},"data-dist-info-metadata":{"sha256":"91c5c4b49dc8b3365ce65ca99e7cee1cf8d862d4379fcc384b1711d4600dd869"},"filename":"psutil-5.8.0-cp27-none-win32.whl","hashes":{"sha256":"ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236561,"upload-time":"2020-12-19T01:19:41.916204Z","url":"https://files.pythonhosted.org/packages/a8/b3/6a21c5b7e4f600bd6eaaecd4a5e76230fa34876e48cbc87b2cef0ab91c0a/psutil-5.8.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"91c5c4b49dc8b3365ce65ca99e7cee1cf8d862d4379fcc384b1711d4600dd869"},"data-dist-info-metadata":{"sha256":"91c5c4b49dc8b3365ce65ca99e7cee1cf8d862d4379fcc384b1711d4600dd869"},"filename":"psutil-5.8.0-cp27-none-win_amd64.whl","hashes":{"sha256":"5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239900,"upload-time":"2020-12-19T01:19:45.046259Z","url":"https://files.pythonhosted.org/packages/b2/3d/01ef1f4bf71413078bf2ce2aae04d47bc132cfede58738183a9de41aa122/psutil-5.8.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236013,"upload-time":"2020-12-19T01:19:47.734642Z","url":"https://files.pythonhosted.org/packages/30/81/37ebe0ba2840b76681072e786bae3319cade8a6861029d0ae885c274fa0b/psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl","hashes":{"sha256":"74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":289247,"upload-time":"2020-12-19T01:19:50.429849Z","url":"https://files.pythonhosted.org/packages/2e/7c/13a6c3f068aa39ffafd99ae159c1a345521e7dd0074ccadb917e5670dbdc/psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl","hashes":{"sha256":"99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291875,"upload-time":"2020-12-19T01:19:53.450222Z","url":"https://files.pythonhosted.org/packages/da/82/56cd16a4c5f53e3e5dd7b2c30d5c803e124f218ebb644ca9c30bc907eadd/psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-win32.whl","hashes":{"sha256":"36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240337,"upload-time":"2020-12-19T01:19:56.748668Z","url":"https://files.pythonhosted.org/packages/19/29/f7a38ee30083f2caa14cc77a6d34c4d5cfd1a69641e87bf1b3d6ba90d0ba/psutil-5.8.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244835,"upload-time":"2020-12-19T01:19:59.102544Z","url":"https://files.pythonhosted.org/packages/44/ed/49d75a29007727d44937ed4d233f116be346bc4657a83b5a9e2f423bca57/psutil-5.8.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236013,"upload-time":"2020-12-19T01:20:02.260367Z","url":"https://files.pythonhosted.org/packages/fe/19/83ab423a7b69cafe4078dea751acdff7377e4b59c71e3718125ba3c341f9/psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl","hashes":{"sha256":"61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":290299,"upload-time":"2020-12-19T01:20:04.653817Z","url":"https://files.pythonhosted.org/packages/cc/5f/2a1967092086acc647962168d0e6fd1c22e14a973f03e3ffb1e2f0da5de9/psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl","hashes":{"sha256":"0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":296329,"upload-time":"2020-12-19T01:20:07.174230Z","url":"https://files.pythonhosted.org/packages/84/da/f7efdcf012b51506938553dbe302aecc22f3f43abd5cffa8320e8e0588d5/psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-win32.whl","hashes":{"sha256":"1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240337,"upload-time":"2020-12-19T01:20:10.138302Z","url":"https://files.pythonhosted.org/packages/18/c9/1db6aa0d28831f60408a6aab9d108c2edbd5a9ed11e5957a91d9d8023898/psutil-5.8.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244834,"upload-time":"2020-12-19T01:20:12.840200Z","url":"https://files.pythonhosted.org/packages/71/ce/35107e81e7eae55c847313f872d4258a71d2640fa04f57c5520fc81473ce/psutil-5.8.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236297,"upload-time":"2020-12-19T01:20:15.489742Z","url":"https://files.pythonhosted.org/packages/10/d6/c5c19e40bb05e2cb5f053f480dfe47e9543a8322f1a5985d7352bf689611/psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl","hashes":{"sha256":"d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":293809,"upload-time":"2020-12-19T01:20:18.539919Z","url":"https://files.pythonhosted.org/packages/e9/d6/7d0bcf272923f6b3433e22effd31860b63ab580d65fb2d8f5cb443a9e6fc/psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl","hashes":{"sha256":"28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":296040,"upload-time":"2020-12-19T01:20:21.423284Z","url":"https://files.pythonhosted.org/packages/3b/c2/78109a12da9febb2f965abf29da6f81b0a3f2b89a7b59d88b759e68dc6db/psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-win32.whl","hashes":{"sha256":"ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240950,"upload-time":"2020-12-19T01:20:24.359373Z","url":"https://files.pythonhosted.org/packages/87/be/6511e1341c203608fe2553249216c40b92cd8a72d8b35fa3c1decee9a616/psutil-5.8.0-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp38-cp38-win_amd64.whl","hashes":{"sha256":"90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245386,"upload-time":"2020-12-19T01:20:26.799194Z","url":"https://files.pythonhosted.org/packages/8e/5c/c4b32c2024daeac35e126b90a1ff7a0209ef8b32675d1d50e55d58e78c81/psutil-5.8.0-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":236274,"upload-time":"2020-12-19T01:20:29.615503Z","url":"https://files.pythonhosted.org/packages/12/80/8d09c345f19af2b29a309f8f9284e3ba1ae1ebd9438419080c14630f743a/psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl","hashes":{"sha256":"245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291032,"upload-time":"2020-12-19T01:20:32.422979Z","url":"https://files.pythonhosted.org/packages/b6/2f/118e23a8f4e59d2c4ffe03a921cc72f364966e25548dc6c5a3011a334dc5/psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl","hashes":{"sha256":"90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":293491,"upload-time":"2020-12-19T01:20:34.915141Z","url":"https://files.pythonhosted.org/packages/91/4d/033cc02ae3a47197d0ced818814e4bb8d9d29ebed4f1eb55badedec160f7/psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-win32.whl","hashes":{"sha256":"ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241508,"upload-time":"2020-12-19T01:20:37.527554Z","url":"https://files.pythonhosted.org/packages/a7/13/7285b74e061da21dfc4f15c8307eb2da1d2137367502d6598f03f4a5b5e7/psutil-5.8.0-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"data-dist-info-metadata":{"sha256":"069863fda59a3742371c9f60ad268b0b04c760a595c7d1a1de557d590bf205e6"},"filename":"psutil-5.8.0-cp39-cp39-win_amd64.whl","hashes":{"sha256":"f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246139,"upload-time":"2020-12-19T01:20:39.890196Z","url":"https://files.pythonhosted.org/packages/21/71/33cb528381c443df1ee25cbb451da975421bddb5099b11e7f2eb3fc90d6d/psutil-5.8.0-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.8.0.tar.gz","hashes":{"sha256":"0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":470886,"upload-time":"2020-12-19T01:20:42.916847Z","url":"https://files.pythonhosted.org/packages/e1/b0/7276de53321c12981717490516b7e612364f2cb372ee8901bd4a66a000d7/psutil-5.8.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"data-dist-info-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"filename":"psutil-5.9.0-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":285348,"upload-time":"2021-12-29T21:26:19.591062Z","url":"https://files.pythonhosted.org/packages/c9/62/5cfcb69c256d469236d4bddeb7ad4ee6a8b37d604dcfc82b7c938fd8ee37/psutil-5.9.0-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"data-dist-info-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"filename":"psutil-5.9.0-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288125,"upload-time":"2021-12-29T21:26:24.644701Z","url":"https://files.pythonhosted.org/packages/eb/0d/c19872c9121208bbbb4335bb13a4a2f2b95661fd69d24f26e32f94e5a8a1/psutil-5.9.0-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"data-dist-info-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"filename":"psutil-5.9.0-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":285364,"upload-time":"2021-12-29T21:26:28.242468Z","url":"https://files.pythonhosted.org/packages/1a/3e/ff287d01bca130b72cf53a9b20bbc31bf566d503ee63adf8c7dcfd9315e2/psutil-5.9.0-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"data-dist-info-metadata":{"sha256":"146a681ce7fa3f640f9204e5391309605d72b0f4ddaaf9927df9b73319a4490e"},"filename":"psutil-5.9.0-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288110,"upload-time":"2021-12-29T21:26:31.963660Z","url":"https://files.pythonhosted.org/packages/d8/49/fbce284331d482703decdc8dec9bfd910fa00a3acd5b974e8efa8c30104a/psutil-5.9.0-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e84069ba77055def96d865c3845efa434ef6204d61f576071bac882eb98e785c"},"data-dist-info-metadata":{"sha256":"e84069ba77055def96d865c3845efa434ef6204d61f576071bac882eb98e785c"},"filename":"psutil-5.9.0-cp27-none-win32.whl","hashes":{"sha256":"ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239056,"upload-time":"2021-12-29T21:26:35.738857Z","url":"https://files.pythonhosted.org/packages/ab/d7/a8b076603943ebce7872ca7d4e012f6dcdc33e86eabb117921a6fe6e1f8a/psutil-5.9.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e84069ba77055def96d865c3845efa434ef6204d61f576071bac882eb98e785c"},"data-dist-info-metadata":{"sha256":"e84069ba77055def96d865c3845efa434ef6204d61f576071bac882eb98e785c"},"filename":"psutil-5.9.0-cp27-none-win_amd64.whl","hashes":{"sha256":"ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242517,"upload-time":"2021-12-29T21:26:39.603955Z","url":"https://files.pythonhosted.org/packages/2d/7a/ee32fa2c5712fa0bc6a9f376ffe9d2e1dc856e2e011d2bab4e12293dcd88/psutil-5.9.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-macosx_10_9_x86_64.whl","hashes":{"sha256":"90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238624,"upload-time":"2021-12-29T21:26:42.964503Z","url":"https://files.pythonhosted.org/packages/89/48/2c6f566d35a38fb9f882e51d75425a6f1d097cb946e05b6aff98d450a151/psutil-5.9.0-cp310-cp310-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279343,"upload-time":"2021-12-29T21:26:46.859457Z","url":"https://files.pythonhosted.org/packages/11/46/e790221e8281af5163517a17a20c88b10a75a5642d9c5106a868f2879edd/psutil-5.9.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281400,"upload-time":"2021-12-29T21:26:51.801761Z","url":"https://files.pythonhosted.org/packages/6f/8a/d1810472a4950a31df385eafbc9bd20cde971814ff6533021dc565bf14ae/psutil-5.9.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-win32.whl","hashes":{"sha256":"8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241383,"upload-time":"2021-12-29T21:26:55.364799Z","url":"https://files.pythonhosted.org/packages/61/93/4251cfa58e5bbd7f92e1bfb965a0c41376cbcbc83c524a8b60d2678f0edd/psutil-5.9.0-cp310-cp310-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp310-cp310-win_amd64.whl","hashes":{"sha256":"9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245540,"upload-time":"2021-12-29T21:26:59.088538Z","url":"https://files.pythonhosted.org/packages/9f/c9/7fb339d6a04db3b4ab94671536d11e03b23c056d1604e50e564075a96cd8/psutil-5.9.0-cp310-cp310-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238331,"upload-time":"2022-01-07T14:28:07.566595Z","url":"https://files.pythonhosted.org/packages/48/cb/6841d4f39b5711652a93359748879f2977ede55c1020f69d038891073592/psutil-5.9.0-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":277267,"upload-time":"2022-01-07T14:28:12.318473Z","url":"https://files.pythonhosted.org/packages/1f/2d/e6640979580db1b51220d3165e256a1d0a31847944a3e2622800a737fe86/psutil-5.9.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279702,"upload-time":"2022-01-07T14:28:16.706587Z","url":"https://files.pythonhosted.org/packages/64/87/461555057b080e1996427098a6c51c64a8a9025ec18571dabfe5be07eeec/psutil-5.9.0-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-win32.whl","hashes":{"sha256":"e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243193,"upload-time":"2022-01-07T14:28:20.695929Z","url":"https://files.pythonhosted.org/packages/2b/a3/24d36239a7bfa30b0eb4302b045417796e9f2c7c21b296d2405735e8949e/psutil-5.9.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247862,"upload-time":"2022-01-07T14:28:23.774803Z","url":"https://files.pythonhosted.org/packages/14/c9/f0bccd60a25197d63a02688ea8f5c5cd5a8b1baf1a7d6bf493d3291132d2/psutil-5.9.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238334,"upload-time":"2021-12-29T21:27:03.461413Z","url":"https://files.pythonhosted.org/packages/70/40/0a6ca5641f7574b6ea38cdb561c30065659734755a1779db67b56e225f84/psutil-5.9.0-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278077,"upload-time":"2021-12-29T21:27:07.864433Z","url":"https://files.pythonhosted.org/packages/6b/c0/0f233f87e816c20e5489bca749798255a464282cdd5911d62bb8344c4b5a/psutil-5.9.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280736,"upload-time":"2021-12-29T21:27:11.252493Z","url":"https://files.pythonhosted.org/packages/60/f9/b78291ed21146ece2417bd1ba715564c6d3bdf2f1e9297ed67709bb36eeb/psutil-5.9.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-win32.whl","hashes":{"sha256":"df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241810,"upload-time":"2021-12-29T21:27:14.020326Z","url":"https://files.pythonhosted.org/packages/47/3f/0475146306d02270243e55cad8167d5185c8918933953c90eda846d72ff3/psutil-5.9.0-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246380,"upload-time":"2021-12-29T21:27:17.444889Z","url":"https://files.pythonhosted.org/packages/7c/d6/4ade7cebfe04710a89e2dc5638f712f09dc5e402a8fea95c3d16dc7f64bf/psutil-5.9.0-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238620,"upload-time":"2021-12-29T21:27:21.799788Z","url":"https://files.pythonhosted.org/packages/89/8e/2a8814f903bc06471621f6e0cd3fc1a7085868656106f31aacf2f844eea2/psutil-5.9.0-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281246,"upload-time":"2021-12-29T21:27:24.591382Z","url":"https://files.pythonhosted.org/packages/4c/95/3c0858c62ec02106cf5f3e79d74223264a6269a16996f31d5ab43abcec86/psutil-5.9.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":283823,"upload-time":"2021-12-29T21:27:27.809766Z","url":"https://files.pythonhosted.org/packages/0a/66/b2188d8e738ee52206a4ee804907f6eab5bcc9fc0e8486e7ab973a8323b7/psutil-5.9.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-win32.whl","hashes":{"sha256":"76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242160,"upload-time":"2021-12-29T21:27:31.512894Z","url":"https://files.pythonhosted.org/packages/d0/cf/7a86fc08f821d66c528939f155079df7d0945678fc474c6a6455c909f6eb/psutil-5.9.0-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp38-cp38-win_amd64.whl","hashes":{"sha256":"3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246488,"upload-time":"2021-12-29T21:27:36.422809Z","url":"https://files.pythonhosted.org/packages/62/d4/72fc44dfd9939851bd672e94e43d12848a98b1d2c3f6f794d54a220fe4a7/psutil-5.9.0-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238613,"upload-time":"2021-12-29T21:27:39.813718Z","url":"https://files.pythonhosted.org/packages/48/6a/c6e88a5584544033dbb8318c380e7e1e3796e5ac336577eb91dc75bdecd7/psutil-5.9.0-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278536,"upload-time":"2021-12-29T21:27:43.204962Z","url":"https://files.pythonhosted.org/packages/f7/b1/82e95f6368dbde6b7e54ea6b18cf8ac3958223540d0bcbde23ba7be19478/psutil-5.9.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280415,"upload-time":"2021-12-29T21:27:47.616041Z","url":"https://files.pythonhosted.org/packages/c4/35/7cec9647be077784d20913404f914fffd8fe6dfd0673e29f7bd822ac1331/psutil-5.9.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-win32.whl","hashes":{"sha256":"4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241428,"upload-time":"2021-12-29T21:27:52.616449Z","url":"https://files.pythonhosted.org/packages/5a/c6/923aed22f6c9c5197998fa6907c983e884975a0ae3430ccd8514f5fd0d6a/psutil-5.9.0-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"data-dist-info-metadata":{"sha256":"ad269c4c58fadbf100c6116eb3ebed5d77878f0c43b9314d18addc011b2b2b80"},"filename":"psutil-5.9.0-cp39-cp39-win_amd64.whl","hashes":{"sha256":"7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245606,"upload-time":"2021-12-29T21:27:56.202709Z","url":"https://files.pythonhosted.org/packages/9e/9e/3a48f15a1539505e2f3058a709eee56acfb379f2b0ff409d6291099e2a7e/psutil-5.9.0-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.0.tar.gz","hashes":{"sha256":"869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25"},"provenance":null,"requires-python":">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":478322,"upload-time":"2021-12-29T21:27:59.163343Z","url":"https://files.pythonhosted.org/packages/47/b6/ea8a7728f096a597f0032564e8013b705aa992a0990becd773dcc4d7b4a7/psutil-5.9.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"data-dist-info-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"filename":"psutil-5.9.1-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"799759d809c31aab5fe4579e50addf84565e71c1dc9f1c31258f159ff70d3f87"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":286186,"upload-time":"2022-05-20T20:09:47.558524Z","url":"https://files.pythonhosted.org/packages/77/06/f9fd79449440d7217d6bf2c90998d540e125cfeffe39d214a328dadc46f4/psutil-5.9.1-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"data-dist-info-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"filename":"psutil-5.9.1-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"9272167b5f5fbfe16945be3db475b3ce8d792386907e673a209da686176552af"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288858,"upload-time":"2022-05-20T20:09:52.956913Z","url":"https://files.pythonhosted.org/packages/cf/29/ad704a45960bfb52ef8bf0beb9c41c09ce92d61c40333f03e9a03f246c22/psutil-5.9.1-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e6c8d739b32cc31942a1726ba4e33d58fe5d46a1400a2526e858888d58324393"},"data-dist-info-metadata":{"sha256":"e6c8d739b32cc31942a1726ba4e33d58fe5d46a1400a2526e858888d58324393"},"filename":"psutil-5.9.1-cp27-cp27m-win32.whl","hashes":{"sha256":"0904727e0b0a038830b019551cf3204dd48ef5c6868adc776e06e93d615fc5fc"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239479,"upload-time":"2022-05-20T20:09:57.166474Z","url":"https://files.pythonhosted.org/packages/7e/8d/e0a66123fa98e309597815de518b47a7a6c571a8f886fc8d4db2331fd2ab/psutil-5.9.1-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"e6c8d739b32cc31942a1726ba4e33d58fe5d46a1400a2526e858888d58324393"},"data-dist-info-metadata":{"sha256":"e6c8d739b32cc31942a1726ba4e33d58fe5d46a1400a2526e858888d58324393"},"filename":"psutil-5.9.1-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"e7e10454cb1ab62cc6ce776e1c135a64045a11ec4c6d254d3f7689c16eb3efd2"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242812,"upload-time":"2022-05-20T20:10:01.868774Z","url":"https://files.pythonhosted.org/packages/1b/53/8f0772df0a6d593bc2fcdf12f4f790bab5c4f6a77bb61a8ddaad2cbba7f8/psutil-5.9.1-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"data-dist-info-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"filename":"psutil-5.9.1-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"56960b9e8edcca1456f8c86a196f0c3d8e3e361320071c93378d41445ffd28b0"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":286184,"upload-time":"2022-05-20T20:10:05.575313Z","url":"https://files.pythonhosted.org/packages/2d/56/54b4ed8102ce5a2f5367b4e766c1873c18f9c32cde321435d0e0ee2abcc5/psutil-5.9.1-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"data-dist-info-metadata":{"sha256":"ac1d788a80e2f59751f3eb37a186124618aa8bafbbe6cd2ad03985f508e49a67"},"filename":"psutil-5.9.1-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"44d1826150d49ffd62035785a9e2c56afcea66e55b43b8b630d7706276e87f22"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288863,"upload-time":"2022-05-20T20:10:09.032899Z","url":"https://files.pythonhosted.org/packages/2c/9d/dc329b7da284677ea843f3ff4b35b8ab3b96b65a58a544b3c3f86d9d032f/psutil-5.9.1-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp310-cp310-macosx_10_9_x86_64.whl","hashes":{"sha256":"c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239266,"upload-time":"2022-05-20T20:10:12.541628Z","url":"https://files.pythonhosted.org/packages/d1/16/6239e76ab5d990dc7866bc22a80585f73421588d63b42884d607f5f815e2/psutil-5.9.1-cp310-cp310-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"abd9246e4cdd5b554a2ddd97c157e292ac11ef3e7af25ac56b08b455c829dca8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280127,"upload-time":"2022-05-20T20:10:16.449769Z","url":"https://files.pythonhosted.org/packages/14/06/39d7e963a6a8bbf26519de208593cdb0ddfe22918b8989f4b2363d4ab49f/psutil-5.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":282049,"upload-time":"2022-05-20T20:10:19.914788Z","url":"https://files.pythonhosted.org/packages/6d/c6/6a4e46802e8690d50ba6a56c7f79ac283e703fcfa0fdae8e41909c8cef1f/psutil-5.9.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp310-cp310-win32.whl","hashes":{"sha256":"20b27771b077dcaa0de1de3ad52d22538fe101f9946d6dc7869e6f694f079329"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241699,"upload-time":"2022-05-20T20:10:23.278390Z","url":"https://files.pythonhosted.org/packages/26/b4/a58cf15ea649faa92c54f00c627aef1d50b9f1abf207485f10c967a50c95/psutil-5.9.1-cp310-cp310-win32.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp310-cp310-win_amd64.whl","hashes":{"sha256":"58678bbadae12e0db55186dc58f2888839228ac9f41cc7848853539b70490021"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245843,"upload-time":"2022-05-20T20:10:26.856840Z","url":"https://files.pythonhosted.org/packages/c0/5a/2ac88d5265b711c8aa4e786825b38d5d0b1e5ecbdd0ce78e9b04a820d247/psutil-5.9.1-cp310-cp310-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"3a76ad658641172d9c6e593de6fe248ddde825b5866464c3b2ee26c35da9d237"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239007,"upload-time":"2022-05-20T20:10:30.191299Z","url":"https://files.pythonhosted.org/packages/65/1d/6a112f146faee6292a6c3ee2a7f24a8e572697adb7e1c5de3d8508f647cc/psutil-5.9.1-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"a6a11e48cb93a5fa606306493f439b4aa7c56cb03fc9ace7f6bfa21aaf07c453"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278074,"upload-time":"2022-05-20T20:10:33.843171Z","url":"https://files.pythonhosted.org/packages/7e/52/a02dc53e26714a339c8b4972d8e3f268e4db8905f5d1a3a100f1e40b6fa7/psutil-5.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280402,"upload-time":"2022-05-20T20:10:38.665061Z","url":"https://files.pythonhosted.org/packages/6b/76/a8cb69ed3566877dcbccf408f5f9d6055227ad4fed694e88809fa8506b0b/psutil-5.9.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-win32.whl","hashes":{"sha256":"0f15a19a05f39a09327345bc279c1ba4a8cfb0172cc0d3c7f7d16c813b2e7d36"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243501,"upload-time":"2022-05-20T20:10:42.164047Z","url":"https://files.pythonhosted.org/packages/85/4d/78173e3dffb74c5fa87914908f143473d0b8b9183f9d275333679a4e4649/psutil-5.9.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"db417f0865f90bdc07fa30e1aadc69b6f4cad7f86324b02aa842034efe8d8c4d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":248156,"upload-time":"2022-05-20T20:10:45.327961Z","url":"https://files.pythonhosted.org/packages/73/1a/d78f2f2de2aad6628415d2a48917cabc2c7fb0c3a31c7cdf187cffa4eb36/psutil-5.9.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"91c7ff2a40c373d0cc9121d54bc5f31c4fa09c346528e6a08d1845bce5771ffc"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238960,"upload-time":"2022-05-20T20:10:48.678116Z","url":"https://files.pythonhosted.org/packages/d6/ef/fd4dc9085e3879c3af63fe60667dd3b71adf50d030b5549315f4a619271b/psutil-5.9.1-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"fea896b54f3a4ae6f790ac1d017101252c93f6fe075d0e7571543510f11d2676"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278553,"upload-time":"2022-05-20T20:10:52.019471Z","url":"https://files.pythonhosted.org/packages/97/f6/0180e58dd1359da7d6fbc27d04dac6fb500dc758b6f4b65407608bb13170/psutil-5.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"3054e923204b8e9c23a55b23b6df73a8089ae1d075cb0bf711d3e9da1724ded4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281356,"upload-time":"2022-05-20T20:10:55.963864Z","url":"https://files.pythonhosted.org/packages/13/71/c25adbd9b33a2e27edbe1fc84b3111a5ad97611885d7abcbdd8d1f2bb7ca/psutil-5.9.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp37-cp37m-win32.whl","hashes":{"sha256":"d2d006286fbcb60f0b391741f520862e9b69f4019b4d738a2a45728c7e952f1b"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242105,"upload-time":"2022-05-20T20:10:59.100543Z","url":"https://files.pythonhosted.org/packages/2a/32/136cd5bf55728ea64a22b1d817890e35fc17314c46a24ee3268b65f9076f/psutil-5.9.1-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"b14ee12da9338f5e5b3a3ef7ca58b3cba30f5b66f7662159762932e6d0b8f680"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246671,"upload-time":"2022-05-20T20:11:02.513602Z","url":"https://files.pythonhosted.org/packages/df/88/427f3959855fcb3ab04891e00c026a246892feb11b20433db814b7a24405/psutil-5.9.1-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"19f36c16012ba9cfc742604df189f2f28d2720e23ff7d1e81602dbe066be9fd1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239250,"upload-time":"2022-05-20T20:11:06.292977Z","url":"https://files.pythonhosted.org/packages/46/80/1de3a9bac336b5c8e4f7b0ff2e80c85ba237f18f2703be68884ee6798432/psutil-5.9.1-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"944c4b4b82dc4a1b805329c980f270f170fdc9945464223f2ec8e57563139cf4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281678,"upload-time":"2022-05-20T20:11:09.990719Z","url":"https://files.pythonhosted.org/packages/fd/ba/c5a3f46f351ab609cc0be6a563e492900c57e3d5c9bda0b79b84d8c3eae9/psutil-5.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"4b6750a73a9c4a4e689490ccb862d53c7b976a2a35c4e1846d049dcc3f17d83b"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":284665,"upload-time":"2022-05-20T20:11:13.584309Z","url":"https://files.pythonhosted.org/packages/9d/41/d5f2db2ab7f5dff2fa795993a0cd6fa8a8f39ca197c3a86857875333ec10/psutil-5.9.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp38-cp38-win32.whl","hashes":{"sha256":"a8746bfe4e8f659528c5c7e9af5090c5a7d252f32b2e859c584ef7d8efb1e689"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242461,"upload-time":"2022-05-20T20:11:16.942263Z","url":"https://files.pythonhosted.org/packages/41/ec/5fd3e9388d0ed1edfdeae71799df374f4a117932646a63413fa95a121e9f/psutil-5.9.1-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp38-cp38-win_amd64.whl","hashes":{"sha256":"79c9108d9aa7fa6fba6e668b61b82facc067a6b81517cab34d07a84aa89f3df0"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246804,"upload-time":"2022-05-20T20:11:20.322459Z","url":"https://files.pythonhosted.org/packages/b2/ad/65e2b2b97677f98d718388dc11b2a9d7f177ebbae5eef72547a32bc28911/psutil-5.9.1-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"28976df6c64ddd6320d281128817f32c29b539a52bdae5e192537bc338a9ec81"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239248,"upload-time":"2022-05-20T20:11:23.352312Z","url":"https://files.pythonhosted.org/packages/9f/ca/84ce3e48b3ca2f0f74314d89929b3a523220f3f4a8dff395d6ef74dadef3/psutil-5.9.1-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"b88f75005586131276634027f4219d06e0561292be8bd6bc7f2f00bdabd63c4e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279320,"upload-time":"2022-05-20T20:11:27.310813Z","url":"https://files.pythonhosted.org/packages/a9/97/b7e3532d97d527349701d2143c3f868733b94e2db6f531b07811b698f549/psutil-5.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"data-dist-info-metadata":{"sha256":"1f39ef2570a850c4cbdb8430a13b995343050494e85d316030a206bd7da4c63b"},"filename":"psutil-5.9.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"645bd4f7bb5b8633803e0b6746ff1628724668681a434482546887d22c7a9537"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281065,"upload-time":"2022-05-20T20:11:30.617653Z","url":"https://files.pythonhosted.org/packages/62/1f/f14225bda76417ab9bd808ff21d5cd59d5435a9796ca09b34d4cb0edcd88/psutil-5.9.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp39-cp39-win32.whl","hashes":{"sha256":"32c52611756096ae91f5d1499fe6c53b86f4a9ada147ee42db4991ba1520e574"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241748,"upload-time":"2022-05-20T20:11:34.374569Z","url":"https://files.pythonhosted.org/packages/b1/d2/c5374a784567c1e42ee8a589b1b42e2bd6e14c7be3c234d84360ab3a0a39/psutil-5.9.1-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"data-dist-info-metadata":{"sha256":"b48f9beb61d032244f4c239867a736641369484a967d3ac29eeab577ce127229"},"filename":"psutil-5.9.1-cp39-cp39-win_amd64.whl","hashes":{"sha256":"f65f9a46d984b8cd9b3750c2bdb419b2996895b005aefa6cbaba9a143b1ce2c5"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245883,"upload-time":"2022-05-20T20:11:37.840887Z","url":"https://files.pythonhosted.org/packages/e0/ac/fd6f098969d49f046083ac032e6788d9f861903596fb9555a02bf50a1238/psutil-5.9.1-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.1.tar.gz","hashes":{"sha256":"57f1819b5d9e95cdfb0c881a8a5b7d542ed0b7c522d575706a80bedc848c8954"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":479090,"upload-time":"2022-05-20T20:11:41.043143Z","url":"https://files.pythonhosted.org/packages/d6/de/0999ea2562b96d7165812606b18f7169307b60cd378bc29cf3673322c7e9/psutil-5.9.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"data-dist-info-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"filename":"psutil-5.9.2-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"8f024fbb26c8daf5d70287bb3edfafa22283c255287cf523c5d81721e8e5d82c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":285739,"upload-time":"2022-09-04T20:15:28.668835Z","url":"https://files.pythonhosted.org/packages/37/a4/cb10e4c0faa3091de22eb78fa1c332566e60b9b59001bef326a4c1070417/psutil-5.9.2-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"data-dist-info-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"filename":"psutil-5.9.2-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"b2f248ffc346f4f4f0d747ee1947963613216b06688be0be2e393986fe20dbbb"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":289461,"upload-time":"2022-09-04T20:15:32.802519Z","url":"https://files.pythonhosted.org/packages/93/40/58dfcab15435b6fedf5385bc7e88a4c162cc6af0056f5d9d97f5ebfd7fa0/psutil-5.9.2-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"5f8d323cf6b24ee1e72553da21c2f9124f45da854ffd5befab575ceb9733c773"},"data-dist-info-metadata":{"sha256":"5f8d323cf6b24ee1e72553da21c2f9124f45da854ffd5befab575ceb9733c773"},"filename":"psutil-5.9.2-cp27-cp27m-win32.whl","hashes":{"sha256":"b1928b9bf478d31fdffdb57101d18f9b70ed4e9b0e41af751851813547b2a9ab"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240036,"upload-time":"2022-09-04T20:15:36.698093Z","url":"https://files.pythonhosted.org/packages/42/eb/83470960f2c13a026b07051456ad834f5fea0c80e8cb83fc65005f5f18d5/psutil-5.9.2-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"5f8d323cf6b24ee1e72553da21c2f9124f45da854ffd5befab575ceb9733c773"},"data-dist-info-metadata":{"sha256":"5f8d323cf6b24ee1e72553da21c2f9124f45da854ffd5befab575ceb9733c773"},"filename":"psutil-5.9.2-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"404f4816c16a2fcc4eaa36d7eb49a66df2d083e829d3e39ee8759a411dbc9ecf"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243378,"upload-time":"2022-09-04T20:15:40.746927Z","url":"https://files.pythonhosted.org/packages/d1/5b/b9d6ac192d3108e1dc7875ab1579b7f65eb7bf0ef799dadd3f3798d0af2e/psutil-5.9.2-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"data-dist-info-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"filename":"psutil-5.9.2-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"94e621c6a4ddb2573d4d30cba074f6d1aa0186645917df42c811c473dd22b339"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":285740,"upload-time":"2022-09-04T20:15:44.420732Z","url":"https://files.pythonhosted.org/packages/b6/96/ddf877440f2686eb17933531507fe4822ff1ed76d85df4a093a605b91db8/psutil-5.9.2-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"data-dist-info-metadata":{"sha256":"fcf26ce4e0d8482a82643f948df7370a2afab20a048f7563ee3fb420705fcf07"},"filename":"psutil-5.9.2-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"256098b4f6ffea6441eb54ab3eb64db9ecef18f6a80d7ba91549195d55420f84"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":289451,"upload-time":"2022-09-04T20:15:47.573981Z","url":"https://files.pythonhosted.org/packages/d7/df/ff5c766b50350f2a4555d5068127d372bb26201a2a5eeda9efc8dbf570b4/psutil-5.9.2-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-macosx_10_9_x86_64.whl","hashes":{"sha256":"614337922702e9be37a39954d67fdb9e855981624d8011a9927b8f2d3c9625d9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239273,"upload-time":"2022-09-04T20:15:52.215984Z","url":"https://files.pythonhosted.org/packages/04/5d/d52473097582db5d3094bc34acf9874de726327a3166426e22ed0806de6a/psutil-5.9.2-cp310-cp310-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"39ec06dc6c934fb53df10c1672e299145ce609ff0611b569e75a88f313634969"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280571,"upload-time":"2022-09-04T20:15:55.608887Z","url":"https://files.pythonhosted.org/packages/47/2b/bd12c4f2d1bd3024fe7c5d8388f8a5627cc02fbe11d62bd451aff356415d/psutil-5.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"e3ac2c0375ef498e74b9b4ec56df3c88be43fe56cac465627572dbfb21c4be34"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":282777,"upload-time":"2022-09-04T20:15:59.840541Z","url":"https://files.pythonhosted.org/packages/4c/85/7a112fb6a8c598a6f5d079228bbc03ae84c472397be79c075e7514b6ed36/psutil-5.9.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-win32.whl","hashes":{"sha256":"e4c4a7636ffc47b7141864f1c5e7d649f42c54e49da2dd3cceb1c5f5d29bfc85"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241713,"upload-time":"2022-09-05T14:16:44.931542Z","url":"https://files.pythonhosted.org/packages/39/07/5cbcf3322031fcf8dcbfa431b1c145f193c96b18964ef374a88d6a83f2c9/psutil-5.9.2-cp310-cp310-win32.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp310-cp310-win_amd64.whl","hashes":{"sha256":"f4cb67215c10d4657e320037109939b1c1d2fd70ca3d76301992f89fe2edb1f1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245853,"upload-time":"2022-09-05T14:16:48.388734Z","url":"https://files.pythonhosted.org/packages/ae/9c/d29dd82d5fda2c6c6d959d57101c78ddbac8325defe94e1b9f983e7cfff3/psutil-5.9.2-cp310-cp310-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"dc9bda7d5ced744622f157cc8d8bdd51735dafcecff807e928ff26bdb0ff097d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238998,"upload-time":"2022-09-04T20:16:03.484887Z","url":"https://files.pythonhosted.org/packages/df/aa/8268eee572fb9bdf3486d384e3973ad9d635403841c6e7f2af7781e5525b/psutil-5.9.2-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"d75291912b945a7351d45df682f9644540d564d62115d4a20d45fa17dc2d48f8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":278226,"upload-time":"2022-09-04T20:16:07.134877Z","url":"https://files.pythonhosted.org/packages/f0/43/bcb92221f5dd45e155337aae37e412fe02a3e5d99e936156a4dcff89fa55/psutil-5.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"b4018d5f9b6651f9896c7a7c2c9f4652e4eea53f10751c4e7d08a9093ab587ec"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280489,"upload-time":"2022-09-04T20:16:10.463113Z","url":"https://files.pythonhosted.org/packages/a4/eb/d841d5bc526641aad65373b0a4850e98284580df967daff5288779090ea3/psutil-5.9.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-win32.whl","hashes":{"sha256":"f40ba362fefc11d6bea4403f070078d60053ed422255bd838cd86a40674364c9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243511,"upload-time":"2022-09-05T14:16:51.624881Z","url":"https://files.pythonhosted.org/packages/54/5f/3619e7d22ded096fa6dbd329fc057bfcf53e998b1e2c1ecc07a4155175b1/psutil-5.9.2-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"data-dist-info-metadata":{"sha256":"c9979ffe6f0cae3b896ebf06f4e0fa63198d968c57d0df7bb3405936e7b9eb5f"},"filename":"psutil-5.9.2-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"9770c1d25aee91417eba7869139d629d6328a9422ce1cdd112bd56377ca98444"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":248164,"upload-time":"2022-09-05T14:16:55.091982Z","url":"https://files.pythonhosted.org/packages/53/ac/7c4ff994b1ea7d46a84932f0c8d49e28e36a668173975876353f4ea38588/psutil-5.9.2-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"42638876b7f5ef43cef8dcf640d3401b27a51ee3fa137cb2aa2e72e188414c32"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":238970,"upload-time":"2022-09-04T20:16:14.127339Z","url":"https://files.pythonhosted.org/packages/55/c5/fd2c45a0845e7bae07c8112ed67c21163742cc116732ac2702d9139a9a92/psutil-5.9.2-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"91aa0dac0c64688667b4285fa29354acfb3e834e1fd98b535b9986c883c2ce1d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279452,"upload-time":"2022-09-04T20:16:18.609472Z","url":"https://files.pythonhosted.org/packages/89/cf/b228a7554eda5e72fd8c33b89c628a86336e5cdbd62fe8b8d2a61a099b2d/psutil-5.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"4fb54941aac044a61db9d8eb56fc5bee207db3bc58645d657249030e15ba3727"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281281,"upload-time":"2022-09-04T20:16:22.476894Z","url":"https://files.pythonhosted.org/packages/3d/73/d8c87b5612c58d1e6c6d91997c1590771d34e4ee27d9c11eb1e64ecbf365/psutil-5.9.2-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-win32.whl","hashes":{"sha256":"7cbb795dcd8ed8fd238bc9e9f64ab188f3f4096d2e811b5a82da53d164b84c3f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242115,"upload-time":"2022-09-05T14:16:58.770138Z","url":"https://files.pythonhosted.org/packages/98/42/62470fae4e1e9c0f4336acf74af9d4a6d5c6b5788c8435ec387e987a7ebe/psutil-5.9.2-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"5d39e3a2d5c40efa977c9a8dd4f679763c43c6c255b1340a56489955dbca767c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246680,"upload-time":"2022-09-05T14:17:03.836879Z","url":"https://files.pythonhosted.org/packages/5e/a2/4025f29069010f118eba4bcd681167d547525d40d2c45029db2f64606f86/psutil-5.9.2-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"fd331866628d18223a4265371fd255774affd86244fc307ef66eaf00de0633d5"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239258,"upload-time":"2022-09-04T20:16:26.490949Z","url":"https://files.pythonhosted.org/packages/2b/52/c69f5d0acc4bbd3cf44178f025e498666d2eebc216f5f5725d9142244365/psutil-5.9.2-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"b315febaebae813326296872fdb4be92ad3ce10d1d742a6b0c49fb619481ed0b"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":282623,"upload-time":"2022-09-04T20:16:29.707162Z","url":"https://files.pythonhosted.org/packages/c4/02/5fc4419f47f141ec0dd28db36fb8bcf1eb6e9df332690617b052c8bec76d/psutil-5.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"f7929a516125f62399d6e8e026129c8835f6c5a3aab88c3fff1a05ee8feb840d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":284813,"upload-time":"2022-09-04T20:16:33.456488Z","url":"https://files.pythonhosted.org/packages/79/61/a8d6d649996494672d8a86fe8be6c81b2880ee30881709d84435f2505b47/psutil-5.9.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-win32.whl","hashes":{"sha256":"561dec454853846d1dd0247b44c2e66a0a0c490f937086930ec4b8f83bf44f06"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242471,"upload-time":"2022-09-05T14:17:07.677347Z","url":"https://files.pythonhosted.org/packages/6f/8d/41c402ae33b1ce3f8e37a0dec691d753cbe66e6784e7fd26ed0cd16d99ab/psutil-5.9.2-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp38-cp38-win_amd64.whl","hashes":{"sha256":"67b33f27fc0427483b61563a16c90d9f3b547eeb7af0ef1b9fe024cdc9b3a6ea"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246816,"upload-time":"2022-09-05T14:17:11.540699Z","url":"https://files.pythonhosted.org/packages/29/07/a35c4127942cce6899d447cb54f9926d33cf1800a37c09192dd9b5a08744/psutil-5.9.2-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"b3591616fa07b15050b2f87e1cdefd06a554382e72866fcc0ab2be9d116486c8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":239256,"upload-time":"2022-09-04T20:16:37.181485Z","url":"https://files.pythonhosted.org/packages/65/74/0ad485d753b2f0d00ee4ec933da1e169bc4c8f4f58db88132e886efed14b/psutil-5.9.2-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"14b29f581b5edab1f133563272a6011925401804d52d603c5c606936b49c8b97"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279700,"upload-time":"2022-09-04T20:16:40.816901Z","url":"https://files.pythonhosted.org/packages/bb/df/0819b9aed416b0dedf668cc6b3f291899c276cb2b566c4aa0dc212a03d55/psutil-5.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"4642fd93785a29353d6917a23e2ac6177308ef5e8be5cc17008d885cb9f70f12"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":281896,"upload-time":"2022-09-04T20:16:45.773716Z","url":"https://files.pythonhosted.org/packages/b3/61/54822666fbbdd4ae1825f7a0b0cf8925a96fac1f778b4a0d5c9c066cf4b2/psutil-5.9.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-win32.whl","hashes":{"sha256":"ed29ea0b9a372c5188cdb2ad39f937900a10fb5478dc077283bf86eeac678ef1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241759,"upload-time":"2022-09-05T14:17:15.034289Z","url":"https://files.pythonhosted.org/packages/67/cf/f620f740da5bb5895b441248e08b0cd167fb545ecaa3e74ea06f3551975e/psutil-5.9.2-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"data-dist-info-metadata":{"sha256":"9318f57df027edece05c45114c189721cfbe0a56aa2ada92fe2b21340be7af15"},"filename":"psutil-5.9.2-cp39-cp39-win_amd64.whl","hashes":{"sha256":"68b35cbff92d1f7103d8f1db77c977e72f49fcefae3d3d2b91c76b0e7aef48b8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245892,"upload-time":"2022-09-05T14:17:18.376384Z","url":"https://files.pythonhosted.org/packages/10/cf/7595896a7487937c171f53bae2eeb0adcc1690ebeef684ac180a77910639/psutil-5.9.2-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.2.tar.gz","hashes":{"sha256":"feb861a10b6c3bb00701063b37e4afc754f8217f0f09c42280586bd6ac712b5c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":479757,"upload-time":"2022-09-04T20:16:49.093359Z","url":"https://files.pythonhosted.org/packages/8f/57/828ac1f70badc691a716e77bfae258ef5db76bb7830109bf4bcf882de020/psutil-5.9.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"b4a247cd3feaae39bb6085fcebf35b3b8ecd9b022db796d89c8f05067ca28e71"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":241919,"upload-time":"2022-10-18T20:12:47.255096Z","url":"https://files.pythonhosted.org/packages/74/42/6268344958236744962c711664de259598fe2005e5818c7d6bc77ae12690/psutil-5.9.3-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"5fa88e3d5d0b480602553d362c4b33a63e0c40bfea7312a7bf78799e01e0810b"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":293266,"upload-time":"2022-10-18T20:12:51.216028Z","url":"https://files.pythonhosted.org/packages/79/6a/7bb45dddeb348cdb9d91d7bc78e903026870ef7f257c35de250392719cf8/psutil-5.9.3-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"767ef4fa33acda16703725c0473a91e1832d296c37c63896c7153ba81698f1ab"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":297585,"upload-time":"2022-10-18T20:12:54.920684Z","url":"https://files.pythonhosted.org/packages/02/c7/d5a6106cf31cc58f4a8a9d88b1ab8405b645b02c482353dd59f5ef19926f/psutil-5.9.3-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"3a494c9ef6c9e0802bc0a2f0d176ca6b72ed9e8ba229b930c23d38f5f92be462"},"data-dist-info-metadata":{"sha256":"3a494c9ef6c9e0802bc0a2f0d176ca6b72ed9e8ba229b930c23d38f5f92be462"},"filename":"psutil-5.9.3-cp27-cp27m-win32.whl","hashes":{"sha256":"9a4af6ed1094f867834f5f07acd1250605a0874169a5fcadbcec864aec2496a6"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":240677,"upload-time":"2022-10-18T20:12:58.145019Z","url":"https://files.pythonhosted.org/packages/62/0a/27aa8d95995fe97a944939f8fff7183f151814a1052b76d125812bed4800/psutil-5.9.3-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"3a494c9ef6c9e0802bc0a2f0d176ca6b72ed9e8ba229b930c23d38f5f92be462"},"data-dist-info-metadata":{"sha256":"3a494c9ef6c9e0802bc0a2f0d176ca6b72ed9e8ba229b930c23d38f5f92be462"},"filename":"psutil-5.9.3-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"fa5e32c7d9b60b2528108ade2929b115167fe98d59f89555574715054f50fa31"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244019,"upload-time":"2022-10-18T20:13:01.999625Z","url":"https://files.pythonhosted.org/packages/2b/0a/36951d279e1d716ab264b04e8ddb12e0c08cc1c7cbd44f2d22c84dc61e33/psutil-5.9.3-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"fe79b4ad4836e3da6c4650cb85a663b3a51aef22e1a829c384e18fae87e5e727"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":293262,"upload-time":"2022-10-18T20:13:05.579750Z","url":"https://files.pythonhosted.org/packages/5b/9c/5412473100e3213d970c8b9291371816e57f1e4a74296b2e3b8a5c8ebb47/psutil-5.9.3-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"data-dist-info-metadata":{"sha256":"eecee58b65701e8da473f7e6142a96da3fac24d305218d11c887ca28bcd6cd61"},"filename":"psutil-5.9.3-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"db8e62016add2235cc87fb7ea000ede9e4ca0aa1f221b40cef049d02d5d2593d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":297584,"upload-time":"2022-10-18T20:13:09.701506Z","url":"https://files.pythonhosted.org/packages/8f/5a/e9e98bb3ade26bc7847d5722d0e4a6d437621fa8fc02269d9cba78f6f241/psutil-5.9.3-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp310-cp310-macosx_10_9_x86_64.whl","hashes":{"sha256":"941a6c2c591da455d760121b44097781bc970be40e0e43081b9139da485ad5b7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242575,"upload-time":"2022-10-18T20:13:13.343655Z","url":"https://files.pythonhosted.org/packages/95/90/822c926e170e8a5769ff11edb92ac59dd523df505b5d56cad0ef3f15c325/psutil-5.9.3-cp310-cp310-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp310-cp310-macosx_11_0_arm64.whl","hashes":{"sha256":"71b1206e7909792d16933a0d2c1c7f04ae196186c51ba8567abae1d041f06dcb"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243240,"upload-time":"2022-10-18T20:13:17.345095Z","url":"https://files.pythonhosted.org/packages/42/9e/243aa51c3d71355913dafc27c5cb7ffdbe9a42c939a5aace526906bfc721/psutil-5.9.3-cp310-cp310-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"f57d63a2b5beaf797b87024d018772439f9d3103a395627b77d17a8d72009543"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":289285,"upload-time":"2022-10-18T20:13:20.535587Z","url":"https://files.pythonhosted.org/packages/f7/b0/6925fbfac4c342cb2f8bad1571b48e12802ac8031e1d4453a31e9a12b64d/psutil-5.9.3-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"e7507f6c7b0262d3e7b0eeda15045bf5881f4ada70473b87bc7b7c93b992a7d7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":292339,"upload-time":"2022-10-18T20:13:26.635784Z","url":"https://files.pythonhosted.org/packages/ed/2c/483ed7332d74b3fef0f5ba13c192d33f21fe95df5468a7ca040f02bd7af9/psutil-5.9.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp310-cp310-win32.whl","hashes":{"sha256":"1b540599481c73408f6b392cdffef5b01e8ff7a2ac8caae0a91b8222e88e8f1e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242906,"upload-time":"2022-10-18T20:13:30.051727Z","url":"https://files.pythonhosted.org/packages/55/07/94730401200098b1119dc9f5d3a271e3bf865b31bfa64a2b58a0bbd9d222/psutil-5.9.3-cp310-cp310-win32.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp310-cp310-win_amd64.whl","hashes":{"sha256":"547ebb02031fdada635452250ff39942db8310b5c4a8102dfe9384ee5791e650"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247045,"upload-time":"2022-10-18T20:13:34.024950Z","url":"https://files.pythonhosted.org/packages/37/c0/8a102d4ce45dbc5d04932b52327c4385b88023635e57af9d457ca5ea6bb3/psutil-5.9.3-cp310-cp310-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"data-dist-info-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"filename":"psutil-5.9.3-cp36-cp36m-macosx_10_9_x86_64.whl","hashes":{"sha256":"d8c3cc6bb76492133474e130a12351a325336c01c96a24aae731abf5a47fe088"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242147,"upload-time":"2022-10-18T20:13:37.172733Z","url":"https://files.pythonhosted.org/packages/61/f2/74908ddbe57863007e3b3a76f39b509bbab9892d0949f1e9d5a888f8ec60/psutil-5.9.3-cp36-cp36m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"data-dist-info-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"filename":"psutil-5.9.3-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"07d880053c6461c9b89cd5d4808f3b8336665fa3acdefd6777662c5ed73a851a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":286758,"upload-time":"2022-10-18T20:13:40.932526Z","url":"https://files.pythonhosted.org/packages/30/2f/696c4459864385cc5c63a21f30584dfd99d2130c21c8b3084ffbaa0edd82/psutil-5.9.3-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"data-dist-info-metadata":{"sha256":"0815ae21fa21e764d04c42f06a34a9f89e9b720a9d80f565b7943a08cf8e10ad"},"filename":"psutil-5.9.3-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"5e8b50241dd3c2ed498507f87a6602825073c07f3b7e9560c58411c14fe1e1c9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":290363,"upload-time":"2022-10-18T20:13:43.847649Z","url":"https://files.pythonhosted.org/packages/2c/80/2f3072492a7f14faf4f4565dd26fe1baf4b3fd28557f1427b6708064a622/psutil-5.9.3-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2ef63858bce55082c5fb7e0b7dabc2fd37fc333286a4d4e5b64df61afb5235de"},"data-dist-info-metadata":{"sha256":"2ef63858bce55082c5fb7e0b7dabc2fd37fc333286a4d4e5b64df61afb5235de"},"filename":"psutil-5.9.3-cp36-cp36m-win32.whl","hashes":{"sha256":"828c9dc9478b34ab96be75c81942d8df0c2bb49edbb481f597314d92b6441d89"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244701,"upload-time":"2022-10-18T20:13:47.431806Z","url":"https://files.pythonhosted.org/packages/db/e3/10363d747d900f89f7920b8e4060b42cd862b580a69a2b9c9788c4de9035/psutil-5.9.3-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2ef63858bce55082c5fb7e0b7dabc2fd37fc333286a4d4e5b64df61afb5235de"},"data-dist-info-metadata":{"sha256":"2ef63858bce55082c5fb7e0b7dabc2fd37fc333286a4d4e5b64df61afb5235de"},"filename":"psutil-5.9.3-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"ed15edb14f52925869250b1375f0ff58ca5c4fa8adefe4883cfb0737d32f5c02"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":249356,"upload-time":"2022-10-18T20:13:51.794050Z","url":"https://files.pythonhosted.org/packages/ac/cc/092ca7ae0c5f270bb14720cd8ac86a3fafda25fae31d08d2465eed4498b3/psutil-5.9.3-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp37-cp37m-macosx_10_9_x86_64.whl","hashes":{"sha256":"d266cd05bd4a95ca1c2b9b5aac50d249cf7c94a542f47e0b22928ddf8b80d1ef"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242154,"upload-time":"2022-10-18T20:14:12.425822Z","url":"https://files.pythonhosted.org/packages/16/51/d431f7db3a3a44d9c03ec1681835a5de52d2f0bb7e28f29ecd806ccc46ec/psutil-5.9.3-cp37-cp37m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"7e4939ff75149b67aef77980409f156f0082fa36accc475d45c705bb00c6c16a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":287383,"upload-time":"2022-10-18T20:14:15.962130Z","url":"https://files.pythonhosted.org/packages/94/b0/cd3be14dc74a6f262b1de296841a5141a794cc485d4e3af5c1c0ffc9b886/psutil-5.9.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"68fa227c32240c52982cb931801c5707a7f96dd8927f9102d6c7771ea1ff5698"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291396,"upload-time":"2022-10-18T20:14:19.855473Z","url":"https://files.pythonhosted.org/packages/5e/86/856aa554ec7eb843fb006ef125cf4543ee9058cb39ad09d131dd820c71f7/psutil-5.9.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp37-cp37m-win32.whl","hashes":{"sha256":"beb57d8a1ca0ae0eb3d08ccaceb77e1a6d93606f0e1754f0d60a6ebd5c288837"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243306,"upload-time":"2022-10-18T20:14:23.302434Z","url":"https://files.pythonhosted.org/packages/ab/10/547feeec01275dd544a389ba05ecb3c316015d4b402cc7b440ca2d98ebcd/psutil-5.9.3-cp37-cp37m-win32.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp37-cp37m-win_amd64.whl","hashes":{"sha256":"12500d761ac091f2426567f19f95fd3f15a197d96befb44a5c1e3cbe6db5752c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247870,"upload-time":"2022-10-18T20:14:26.971453Z","url":"https://files.pythonhosted.org/packages/0c/f1/50e71c11ef14c592686dfc60e2b42a381fe57af2d22713e66a72c07cf9d1/psutil-5.9.3-cp37-cp37m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp38-cp38-macosx_10_9_x86_64.whl","hashes":{"sha256":"ba38cf9984d5462b506e239cf4bc24e84ead4b1d71a3be35e66dad0d13ded7c1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242611,"upload-time":"2022-10-18T20:14:30.842925Z","url":"https://files.pythonhosted.org/packages/01/d6/9ca99b416dddf4a49855a9ebf4af3a2db9526e94e9693da169fa5ed61788/psutil-5.9.3-cp38-cp38-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp38-cp38-macosx_11_0_arm64.whl","hashes":{"sha256":"46907fa62acaac364fff0b8a9da7b360265d217e4fdeaca0a2397a6883dffba2"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243235,"upload-time":"2022-10-18T20:14:34.363643Z","url":"https://files.pythonhosted.org/packages/2c/ce/daf28e50305fdbba0754ba58ab0346ec6cfa41293110412f4c6bf74738bb/psutil-5.9.3-cp38-cp38-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"a04a1836894c8279e5e0a0127c0db8e198ca133d28be8a2a72b4db16f6cf99c1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291620,"upload-time":"2022-10-18T20:14:38.179697Z","url":"https://files.pythonhosted.org/packages/b9/cf/56278ae450741b6390491aecaa5f6152ff491bf00544799830e98340ff48/psutil-5.9.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"8a4e07611997acf178ad13b842377e3d8e9d0a5bac43ece9bfc22a96735d9a4f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":295598,"upload-time":"2022-10-18T20:14:41.691453Z","url":"https://files.pythonhosted.org/packages/af/5d/9c03a47af929fc12699fcf5174313744eef33a7b9e106e8111f57427b7d7/psutil-5.9.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp38-cp38-win32.whl","hashes":{"sha256":"6ced1ad823ecfa7d3ce26fe8aa4996e2e53fb49b7fed8ad81c80958501ec0619"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243663,"upload-time":"2022-10-18T20:14:45.084651Z","url":"https://files.pythonhosted.org/packages/69/3d/e1a12f505eb0171912b94e4689453639bb0deeb70ab4eddbc7b9266f819e/psutil-5.9.3-cp38-cp38-win32.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp38-cp38-win_amd64.whl","hashes":{"sha256":"35feafe232d1aaf35d51bd42790cbccb882456f9f18cdc411532902370d660df"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":248005,"upload-time":"2022-10-18T20:14:48.240035Z","url":"https://files.pythonhosted.org/packages/69/cf/47a028bbb4589fdc0494bc60f134c73e319ec78c86c37e2dc66fd118e4db/psutil-5.9.3-cp38-cp38-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp39-cp39-macosx_10_9_x86_64.whl","hashes":{"sha256":"538fcf6ae856b5e12d13d7da25ad67f02113c96f5989e6ad44422cb5994ca7fc"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242570,"upload-time":"2022-10-18T20:14:51.709705Z","url":"https://files.pythonhosted.org/packages/e5/64/ced1461fd5ebc944d90f9e471149991893bd7ede05b5a88069c1953738dc/psutil-5.9.3-cp39-cp39-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp39-cp39-macosx_11_0_arm64.whl","hashes":{"sha256":"a3d81165b8474087bb90ec4f333a638ccfd1d69d34a9b4a1a7eaac06648f9fbe"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243242,"upload-time":"2022-10-18T20:14:54.641746Z","url":"https://files.pythonhosted.org/packages/ac/55/c108e74f22905382aeeef56110bd6c4b89b5fc64944d21cb83acb66faa4c/psutil-5.9.3-cp39-cp39-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"3a7826e68b0cf4ce2c1ee385d64eab7d70e3133171376cac53d7c1790357ec8f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":288533,"upload-time":"2022-10-18T20:14:57.902650Z","url":"https://files.pythonhosted.org/packages/db/6f/2441388c48306f9b9d561080c6ba652b4ebd1199faac237069ec8983c8ef/psutil-5.9.3-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"data-dist-info-metadata":{"sha256":"235e00601e80ff0d0cb06ce872f02a457343187ed6fdfb79f6f9f1302b209409"},"filename":"psutil-5.9.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"9ec296f565191f89c48f33d9544d8d82b0d2af7dd7d2d4e6319f27a818f8d1cc"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":291401,"upload-time":"2022-10-18T20:15:01.503982Z","url":"https://files.pythonhosted.org/packages/03/47/15604dd812b1b860e81cabaf8c930474c549773389170cd03a093ecf54b6/psutil-5.9.3-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp39-cp39-win32.whl","hashes":{"sha256":"9ec95df684583b5596c82bb380c53a603bb051cf019d5c849c47e117c5064395"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242954,"upload-time":"2022-10-18T20:15:04.992990Z","url":"https://files.pythonhosted.org/packages/2f/5e/c74dab9858ca67a68a543ad8fefac2aec107383c171019b45ba9ac5223c1/psutil-5.9.3-cp39-cp39-win32.whl","yanked":false},{"core-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"data-dist-info-metadata":{"sha256":"ddacf95a106563420486b913e49d19570b225ee95ad876d657e72d3df9c09536"},"filename":"psutil-5.9.3-cp39-cp39-win_amd64.whl","hashes":{"sha256":"4bd4854f0c83aa84a5a40d3b5d0eb1f3c128f4146371e03baed4589fe4f3c931"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247081,"upload-time":"2022-10-18T20:15:08.131670Z","url":"https://files.pythonhosted.org/packages/34/31/9aa19bf0fb0cecae904c9e1ac400c5704d935252515da605aa08fca2be86/psutil-5.9.3-cp39-cp39-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.3.tar.gz","hashes":{"sha256":"7ccfcdfea4fc4b0a02ca2c31de7fcd186beb9cff8207800e14ab66f79c773af6"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":483579,"upload-time":"2022-10-18T20:15:11.635566Z","url":"https://files.pythonhosted.org/packages/de/eb/1c01a34c86ee3b058c556e407ce5b07cb7d186ebe47b3e69d6f152ca5cc5/psutil-5.9.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242993,"upload-time":"2022-11-07T18:44:23.237667Z","url":"https://files.pythonhosted.org/packages/60/f8/b92fecd5297edcecda825a04dfde7cb0a2ecd178eb976cb5a7956e375c6a/psutil-5.9.4-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":294126,"upload-time":"2022-11-07T18:44:28.809923Z","url":"https://files.pythonhosted.org/packages/8e/6b/9a3a5471b74d92dc85bfd71a7f7a55e013b258d86b4c3826ace9d49f7b8c/psutil-5.9.4-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":298394,"upload-time":"2022-11-07T18:44:34.503099Z","url":"https://files.pythonhosted.org/packages/1d/80/e1502ba4ff65390bd17b4612010762075f64f5a0e7c28e889c4820bd95a9/psutil-5.9.4-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"8fc2f52ee81f3c31e5cf1d7031e3122056d1ec9de55bbc714417091f2e0c5d34"},"data-dist-info-metadata":{"sha256":"8fc2f52ee81f3c31e5cf1d7031e3122056d1ec9de55bbc714417091f2e0c5d34"},"filename":"psutil-5.9.4-cp27-cp27m-win32.whl","hashes":{"sha256":"852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":242001,"upload-time":"2022-11-07T18:44:39.844844Z","url":"https://files.pythonhosted.org/packages/53/ae/536719016fe9399187dbf52cdc65aef942f82b75924495918a2f701bcb77/psutil-5.9.4-cp27-cp27m-win32.whl","yanked":false},{"core-metadata":{"sha256":"8fc2f52ee81f3c31e5cf1d7031e3122056d1ec9de55bbc714417091f2e0c5d34"},"data-dist-info-metadata":{"sha256":"8fc2f52ee81f3c31e5cf1d7031e3122056d1ec9de55bbc714417091f2e0c5d34"},"filename":"psutil-5.9.4-cp27-cp27m-win_amd64.whl","hashes":{"sha256":"9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245334,"upload-time":"2022-11-07T18:44:44.877461Z","url":"https://files.pythonhosted.org/packages/99/9c/7a5761f9d2e79e6f781db5b25eeb9e74c2dc533bc52ee4749cb055a32ce9/psutil-5.9.4-cp27-cp27m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":294138,"upload-time":"2022-11-07T18:44:49.025560Z","url":"https://files.pythonhosted.org/packages/ec/be/b8df2071eda861e65a1b2cec35770bb1f4523737e84a10aa41c53e39e9bc/psutil-5.9.4-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"data-dist-info-metadata":{"sha256":"2123949a577021ace561405ef01228f83e4c6d7ccd4ec38ded6fd271e4243960"},"filename":"psutil-5.9.4-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":298409,"upload-time":"2022-11-07T18:44:56.505271Z","url":"https://files.pythonhosted.org/packages/89/a8/dd2f0866a7e87de751fb5f7c6eca99cbb953c81be76e1814ab3c8c3b0908/psutil-5.9.4-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"data-dist-info-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"filename":"psutil-5.9.4-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243468,"upload-time":"2022-11-07T18:45:00.474371Z","url":"https://files.pythonhosted.org/packages/a5/73/35cea01aad1baf901c915dc95ea33a2f271c8ff8cf2f1c73b7f591f1bdf1/psutil-5.9.4-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"data-dist-info-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"filename":"psutil-5.9.4-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":277515,"upload-time":"2022-11-07T18:45:05.428752Z","url":"https://files.pythonhosted.org/packages/5a/37/ef88eed265d93bc28c681316f68762c5e04167519e5627a0187c8878b409/psutil-5.9.4-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"data-dist-info-metadata":{"sha256":"fa7c88f96167219220d13bcc9fd94ae254e582adb547fdbb756d356aa9a18739"},"filename":"psutil-5.9.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":280218,"upload-time":"2022-11-07T18:45:11.831087Z","url":"https://files.pythonhosted.org/packages/6e/c8/784968329c1c67c28cce91991ef9af8a8913aa5a3399a6a8954b1380572f/psutil-5.9.4-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"bca9c896f97136a7e608898e18955f0413786b5391cd3140eec51917f648c864"},"data-dist-info-metadata":{"sha256":"bca9c896f97136a7e608898e18955f0413786b5391cd3140eec51917f648c864"},"filename":"psutil-5.9.4-cp36-abi3-win32.whl","hashes":{"sha256":"149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247217,"upload-time":"2022-11-08T11:50:03.989781Z","url":"https://files.pythonhosted.org/packages/3e/af/fe14b984e8b0f778d502d387b789d846cb2fcc3989f63be942741266d8c8/psutil-5.9.4-cp36-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"bca9c896f97136a7e608898e18955f0413786b5391cd3140eec51917f648c864"},"data-dist-info-metadata":{"sha256":"bca9c896f97136a7e608898e18955f0413786b5391cd3140eec51917f648c864"},"filename":"psutil-5.9.4-cp36-abi3-win_amd64.whl","hashes":{"sha256":"fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":252462,"upload-time":"2022-11-08T11:50:07.829565Z","url":"https://files.pythonhosted.org/packages/25/6e/ba97809175c90cbdcd33b470e466ebf0854d15d1506e605cc0ddd284d5b6/psutil-5.9.4-cp36-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"3c3c1d22a00986e9736611ca6121b0926aba1c13b75f0b3509aebf5f95b0d409"},"data-dist-info-metadata":{"sha256":"3c3c1d22a00986e9736611ca6121b0926aba1c13b75f0b3509aebf5f95b0d409"},"filename":"psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244234,"upload-time":"2022-11-07T19:53:20.872251Z","url":"https://files.pythonhosted.org/packages/79/26/f026804298b933b11640cc2d15155a545805df732e5ead3a2ad7cf45a38b/psutil-5.9.4-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.4.tar.gz","hashes":{"sha256":"3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":485825,"upload-time":"2022-11-07T19:53:36.245577Z","url":"https://files.pythonhosted.org/packages/3d/7d/d05864a69e452f003c0d77e728e155a89a2a26b09e64860ddd70ad64fb26/psutil-5.9.4.tar.gz","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":244852,"upload-time":"2023-04-17T18:24:26.646150Z","url":"https://files.pythonhosted.org/packages/3b/e4/fee119c206545fd37be1e5fa4eeb0c729a52ec2ade4f728ae1fd1acb2a3a/psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":296014,"upload-time":"2023-04-17T18:24:31.346064Z","url":"https://files.pythonhosted.org/packages/8d/24/ed6b6506f187def39887a91a68e58336eff4cf3e3d5a163ded58bee98624/psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":300274,"upload-time":"2023-04-17T18:24:35.244779Z","url":"https://files.pythonhosted.org/packages/89/fa/ab117fa86195050802207639f5daee857791daaabe9a996935b5b77dbe10/psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":295979,"upload-time":"2023-04-17T18:24:38.850555Z","url":"https://files.pythonhosted.org/packages/99/f5/ec768e107445f18baa907509aaa0562a4d148a602bd97e8114d79bd6c84d/psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"data-dist-info-metadata":{"sha256":"950f52c16a7bc656305be0171e1d567927034839d8c5af80880070db4ba278be"},"filename":"psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":300257,"upload-time":"2023-04-17T18:24:42.928863Z","url":"https://files.pythonhosted.org/packages/5f/da/de9d2342db0b7a96863ef84ab94ef1022eec78ece05aac253cddc494e1a7/psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b8327a43c6d94aba442f903852217084687e619e3d853297d6743f1cd1a4fada"},"data-dist-info-metadata":{"sha256":"b8327a43c6d94aba442f903852217084687e619e3d853297d6743f1cd1a4fada"},"filename":"psutil-5.9.5-cp27-none-win32.whl","hashes":{"sha256":"5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":243956,"upload-time":"2023-04-17T18:24:46.410452Z","url":"https://files.pythonhosted.org/packages/cf/e3/6af6ec0cbe72f63e9a16d8b53590489e40ed0ff0c99b6a6f05d6af3bb80e/psutil-5.9.5-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"b8327a43c6d94aba442f903852217084687e619e3d853297d6743f1cd1a4fada"},"data-dist-info-metadata":{"sha256":"b8327a43c6d94aba442f903852217084687e619e3d853297d6743f1cd1a4fada"},"filename":"psutil-5.9.5-cp27-none-win_amd64.whl","hashes":{"sha256":"8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":247338,"upload-time":"2023-04-17T18:24:49.089552Z","url":"https://files.pythonhosted.org/packages/26/f2/dcd8a3cc9c9b1fcd7576a54e3603ce4d1f85672f2687a44050340f7d47b0/psutil-5.9.5-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":245316,"upload-time":"2023-04-17T18:24:52.864585Z","url":"https://files.pythonhosted.org/packages/9a/76/c0195c3443a725c24b3a479f57636dec89efe53d19d435d1752c5188f7de/psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":279398,"upload-time":"2023-04-17T18:24:56.977087Z","url":"https://files.pythonhosted.org/packages/e5/2e/56db2b45508ad484b3f22888b3e1adaaf09b8766eaa058ed0e4486c1abae/psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":282082,"upload-time":"2023-04-17T18:25:00.863664Z","url":"https://files.pythonhosted.org/packages/af/4d/389441079ecef400e2551a3933224885a7bde6b8a4810091d628cdd75afe/psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-win32.whl","hashes":{"sha256":"104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":249834,"upload-time":"2023-04-17T18:25:05.571829Z","url":"https://files.pythonhosted.org/packages/fa/e0/e91277b1cabf5c3f2995c22314553f1be68b17444260101f365c5a5b6ba1/psutil-5.9.5-cp36-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"data-dist-info-metadata":{"sha256":"612932ce0d3ae556043e5e9a609d33777fe6fb88556dd92d0e07cb53270d1db6"},"filename":"psutil-5.9.5-cp36-abi3-win_amd64.whl","hashes":{"sha256":"b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":255148,"upload-time":"2023-04-17T18:25:09.779955Z","url":"https://files.pythonhosted.org/packages/86/f3/23e4e4e7ec7855d506ed928756b04735c246b14d9f778ed7ffaae18d8043/psutil-5.9.5-cp36-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2ae643bfca9fa3b942bf775226368a8ef859018ea312be94e137a8511ba0da07"},"data-dist-info-metadata":{"sha256":"2ae643bfca9fa3b942bf775226368a8ef859018ea312be94e137a8511ba0da07"},"filename":"psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":246094,"upload-time":"2023-04-17T18:25:14.584295Z","url":"https://files.pythonhosted.org/packages/ed/98/2624954f83489ab13fde2b544baa337d5578c07eee304d320d9ba56e1b1f/psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.5.tar.gz","hashes":{"sha256":"5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*","size":493489,"upload-time":"2023-04-17T18:25:18.787463Z","url":"https://files.pythonhosted.org/packages/d6/0f/96b7309212a926c1448366e9ce69b081ea79d63265bde33f11cc9cfc2c07/psutil-5.9.5.tar.gz","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":245665,"upload-time":"2023-10-15T09:08:50.362285Z","url":"https://files.pythonhosted.org/packages/84/d6/7e23b2b208db3953f630934bc0e9c1736a0a831a781acf8c5891c27b29cf/psutil-5.9.6-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":297167,"upload-time":"2023-10-15T09:08:52.370112Z","url":"https://files.pythonhosted.org/packages/d2/76/f154e5169756f3d18da160359a404f49f476756809ef21a79afdd0d5b552/psutil-5.9.6-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":301338,"upload-time":"2023-10-15T09:08:55.187829Z","url":"https://files.pythonhosted.org/packages/35/e8/5cc0e149ec32a91d459fbe51d0ce3c2dd7f8d67bc1400803ff810247d6dc/psutil-5.9.6-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":297167,"upload-time":"2023-10-15T09:08:57.863175Z","url":"https://files.pythonhosted.org/packages/8d/f7/074071fa91dab747c8d1fe2eb74da439b3712248d6b254ba0136ada8694f/psutil-5.9.6-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"data-dist-info-metadata":{"sha256":"03f47dbf7a696dbe30acaf142fc193530fe197d70ea954e60b2a6279946aca36"},"filename":"psutil-5.9.6-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":301350,"upload-time":"2023-10-15T09:09:00.411072Z","url":"https://files.pythonhosted.org/packages/4a/65/557545149422a7845248641c1c35a0c8ea940c838896320f774072e16523/psutil-5.9.6-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"cbf967d735616b8f7384d1cb095719fdf508b6e794f91eda3c559b9eff000943"},"data-dist-info-metadata":{"sha256":"cbf967d735616b8f7384d1cb095719fdf508b6e794f91eda3c559b9eff000943"},"filename":"psutil-5.9.6-cp27-none-win32.whl","hashes":{"sha256":"70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":244900,"upload-time":"2023-10-15T09:09:03.051464Z","url":"https://files.pythonhosted.org/packages/b8/23/d5d9e20c4ae7374abe1f826c69ecf2ab52f93827ca2b92c2c51f9aeb9226/psutil-5.9.6-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"cbf967d735616b8f7384d1cb095719fdf508b6e794f91eda3c559b9eff000943"},"data-dist-info-metadata":{"sha256":"cbf967d735616b8f7384d1cb095719fdf508b6e794f91eda3c559b9eff000943"},"filename":"psutil-5.9.6-cp27-none-win_amd64.whl","hashes":{"sha256":"51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248205,"upload-time":"2023-10-15T09:09:05.456917Z","url":"https://files.pythonhosted.org/packages/7a/5e/db765b94cb620c04aaea0cb03d8b589905e50ec278130d25646eead8dff0/psutil-5.9.6-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":246101,"upload-time":"2023-10-15T09:09:08.012635Z","url":"https://files.pythonhosted.org/packages/f8/36/35b12441ba1bc6684c9215191f955415196ca57ca85d88e313bec7f2cf8e/psutil-5.9.6-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":280854,"upload-time":"2023-10-15T09:09:09.832401Z","url":"https://files.pythonhosted.org/packages/61/c8/e684dea1912943347922ab5c05efc94b4ff3d7470038e8afbe3941ef9efe/psutil-5.9.6-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":283614,"upload-time":"2023-10-15T09:09:12.314910Z","url":"https://files.pythonhosted.org/packages/19/06/4e3fa3c1b79271e933c5ddbad3a48aa2c3d5f592a0fb7c037f3e0f619f4d/psutil-5.9.6-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-cp36m-win32.whl","hashes":{"sha256":"3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":250445,"upload-time":"2023-10-15T09:09:14.942143Z","url":"https://files.pythonhosted.org/packages/3f/63/d4a8dace1756b9c84b94683aa80ed0ba8fc7a4421904933b472d59268976/psutil-5.9.6-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"data-dist-info-metadata":{"sha256":"2e9d8110f5906489070a50d523dc29d42ccde7cff56677d3b49eae7ac8f23006"},"filename":"psutil-5.9.6-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":255893,"upload-time":"2023-10-15T09:09:17.467925Z","url":"https://files.pythonhosted.org/packages/ad/00/c87d449746f8962eb9203554b46ab7dcf243be236dcf007372902791b374/psutil-5.9.6-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"3eb79e7ee359462d9b616150575b67510adf8a297f3b4c2b93aeb95daef17fb8"},"data-dist-info-metadata":{"sha256":"3eb79e7ee359462d9b616150575b67510adf8a297f3b4c2b93aeb95daef17fb8"},"filename":"psutil-5.9.6-cp37-abi3-win32.whl","hashes":{"sha256":"a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248489,"upload-time":"2023-10-15T09:09:19.912001Z","url":"https://files.pythonhosted.org/packages/06/ac/f31a0faf98267e63fc6ed046ad2aca68bd79521380026e92fd4921c869aa/psutil-5.9.6-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"3eb79e7ee359462d9b616150575b67510adf8a297f3b4c2b93aeb95daef17fb8"},"data-dist-info-metadata":{"sha256":"3eb79e7ee359462d9b616150575b67510adf8a297f3b4c2b93aeb95daef17fb8"},"filename":"psutil-5.9.6-cp37-abi3-win_amd64.whl","hashes":{"sha256":"6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":252327,"upload-time":"2023-10-15T09:09:32.052033Z","url":"https://files.pythonhosted.org/packages/c5/b2/699c50fe0b0402a1ccb64ad71313bcb740e735008dd3ab9abeddbe148e45/psutil-5.9.6-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7298b90db12439da117e2817e35cbbb00312edeb4f885274cd1135a533903d6c"},"data-dist-info-metadata":{"sha256":"7298b90db12439da117e2817e35cbbb00312edeb4f885274cd1135a533903d6c"},"filename":"psutil-5.9.6-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":246859,"upload-time":"2023-10-15T09:09:34.494297Z","url":"https://files.pythonhosted.org/packages/9e/cb/e4b83c27eea66bc255effc967053f6fce7c14906dd9b43a348ead9f0cfea/psutil-5.9.6-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.6.tar.gz","hashes":{"sha256":"e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":496866,"upload-time":"2023-10-15T09:08:46.623978Z","url":"https://files.pythonhosted.org/packages/2d/01/beb7331fc6c8d1c49dd051e3611379bfe379e915c808e1301506027fce9d/psutil-5.9.6.tar.gz","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":245542,"upload-time":"2023-12-17T11:25:25.875219Z","url":"https://files.pythonhosted.org/packages/3e/16/c86fcf73f02bd0a3d49b0dcabc8ebd4020647be2ea40ff668f717587af97/psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":312042,"upload-time":"2023-12-17T11:25:29.439835Z","url":"https://files.pythonhosted.org/packages/93/fc/e45a8e9b2acd54fe80ededa2f7b19de21e776f64e00437417c16c3e139d9/psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":312681,"upload-time":"2023-12-17T11:25:33.024609Z","url":"https://files.pythonhosted.org/packages/d7/43/dd7034a3a3a900e95b9dcf47ee710680cfd11a224ab18b31c34370da36a8/psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":312066,"upload-time":"2023-12-17T11:25:37.010306Z","url":"https://files.pythonhosted.org/packages/ff/ea/a47eecddcd97d65b496ac655c9f9ba8af270c203d5ea1630273cfc5ec740/psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"data-dist-info-metadata":{"sha256":"bee12356aebb79b07f3adec05e4a7e1cad1a4e76f3cbf8f7617b97d7598ed4e8"},"filename":"psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":312684,"upload-time":"2023-12-17T11:25:39.891899Z","url":"https://files.pythonhosted.org/packages/cd/ee/d946d0b758120e724d9cdd9607c304ff1eedb9380bf60597c295dc7def6b/psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e17b0155d6125da908d40f364ab8665d24296d45f194128d7d4fee06adf24366"},"data-dist-info-metadata":{"sha256":"e17b0155d6125da908d40f364ab8665d24296d45f194128d7d4fee06adf24366"},"filename":"psutil-5.9.7-cp27-none-win32.whl","hashes":{"sha256":"1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":244785,"upload-time":"2023-12-17T11:25:42.761256Z","url":"https://files.pythonhosted.org/packages/98/c5/6773a3f1c384ac4863665e167cd4da72433b3020580c0b7c6a7b497e11e2/psutil-5.9.7-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"e17b0155d6125da908d40f364ab8665d24296d45f194128d7d4fee06adf24366"},"data-dist-info-metadata":{"sha256":"e17b0155d6125da908d40f364ab8665d24296d45f194128d7d4fee06adf24366"},"filename":"psutil-5.9.7-cp27-none-win_amd64.whl","hashes":{"sha256":"4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248087,"upload-time":"2023-12-17T11:25:45.316036Z","url":"https://files.pythonhosted.org/packages/2d/91/40ac017db38c9f7f325385dd0dab1be3d4c65e3291100e74d5d7b6a213e8/psutil-5.9.7-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":245972,"upload-time":"2023-12-17T11:25:48.202730Z","url":"https://files.pythonhosted.org/packages/6c/63/86a4ccc640b4ee1193800f57bbd20b766853c0cdbdbb248a27cdfafe6cbf/psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":282514,"upload-time":"2023-12-17T11:25:51.371460Z","url":"https://files.pythonhosted.org/packages/58/80/cc6666b3968646f2d94de66bbc63d701d501f4aa04de43dd7d1f5dc477dd/psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":285469,"upload-time":"2023-12-17T11:25:54.250669Z","url":"https://files.pythonhosted.org/packages/be/fa/f1f626620e3b47e6237dcc64cb8cc1472f139e99422e5b9fa5bbcf457f48/psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-cp36m-win32.whl","hashes":{"sha256":"b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":250357,"upload-time":"2023-12-17T12:38:23.681291Z","url":"https://files.pythonhosted.org/packages/63/16/11dfb52cdccd561da711ee2c127b4c0bd2baf4736d10828c707694f31b90/psutil-5.9.7-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"data-dist-info-metadata":{"sha256":"75f8cc7702054674e6edcb7c6c070053f879e044c73942a152279c8d35fec61c"},"filename":"psutil-5.9.7-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":255808,"upload-time":"2023-12-17T12:38:34.170079Z","url":"https://files.pythonhosted.org/packages/0e/88/9b74b25c63b91ff0403a1b89e258238380b4a88e4116cbae4eaadbb4c17a/psutil-5.9.7-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"8eee389ea25ebe96edb28822645ab1709ede8c32315d59c0c91214e32d607b8e"},"data-dist-info-metadata":{"sha256":"8eee389ea25ebe96edb28822645ab1709ede8c32315d59c0c91214e32d607b8e"},"filename":"psutil-5.9.7-cp37-abi3-win32.whl","hashes":{"sha256":"c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248406,"upload-time":"2023-12-17T12:38:50.326952Z","url":"https://files.pythonhosted.org/packages/7c/b8/dc6ebfc030b47cccc5f5229eeb15e64142b4782796c3ce169ccd60b4d511/psutil-5.9.7-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"8eee389ea25ebe96edb28822645ab1709ede8c32315d59c0c91214e32d607b8e"},"data-dist-info-metadata":{"sha256":"8eee389ea25ebe96edb28822645ab1709ede8c32315d59c0c91214e32d607b8e"},"filename":"psutil-5.9.7-cp37-abi3-win_amd64.whl","hashes":{"sha256":"f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":252245,"upload-time":"2023-12-17T12:39:00.686632Z","url":"https://files.pythonhosted.org/packages/50/28/92b74d95dd991c837813ffac0c79a581a3d129eb0fa7c1dd616d9901e0f3/psutil-5.9.7-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c6272489f71a5b9474dff03296e3f8e100ef6f8ed990f7e4ef444ac6e6a3d6fb"},"data-dist-info-metadata":{"sha256":"c6272489f71a5b9474dff03296e3f8e100ef6f8ed990f7e4ef444ac6e6a3d6fb"},"filename":"psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":246739,"upload-time":"2023-12-17T11:25:57.305436Z","url":"https://files.pythonhosted.org/packages/ba/8a/000d0e80156f0b96c55bda6c60f5ed6543d7b5e893ccab83117e50de1400/psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.7.tar.gz","hashes":{"sha256":"3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":498429,"upload-time":"2023-12-17T11:25:21.220127Z","url":"https://files.pythonhosted.org/packages/a0/d0/c9ae661a302931735237791f04cb7086ac244377f78692ba3b3eae3a9619/psutil-5.9.7.tar.gz","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248274,"upload-time":"2024-01-19T20:47:14.006890Z","url":"https://files.pythonhosted.org/packages/15/9a/c3e2922e2d672bafd37cf3b9681097c350463cdcf0e286e907ddd6cfb014/psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":314796,"upload-time":"2024-01-19T20:47:17.872998Z","url":"https://files.pythonhosted.org/packages/62/e6/6d62285989d53a83def28ea49b46d3e00462d1273c7c47d9678ee28a0a39/psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":315422,"upload-time":"2024-01-19T20:47:21.442877Z","url":"https://files.pythonhosted.org/packages/38/ba/41815f353f79374c1ad82aba998c666c7209793daf12f4799cfaa7302f29/psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":314802,"upload-time":"2024-01-19T20:47:24.219052Z","url":"https://files.pythonhosted.org/packages/a8/2f/ad80cc502c452e1f207307a7d53533505ca47c503ec6e9f7e2c9fbb367e8/psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"data-dist-info-metadata":{"sha256":"8da6995843b9daa47e0867912881271ea684ac8d0bcb3aee77791084ee8c0be4"},"filename":"psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":315420,"upload-time":"2024-01-19T20:47:26.828052Z","url":"https://files.pythonhosted.org/packages/e4/c3/357a292dee683282f7a46b752a76c5d56c78bf8f5d9def0ca0d39073344a/psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"bcd64b1f8b81a5a0ea3cff661abf5d6e3a937f442bb982b9bdcddbfd8608aa9c"},"data-dist-info-metadata":{"sha256":"bcd64b1f8b81a5a0ea3cff661abf5d6e3a937f442bb982b9bdcddbfd8608aa9c"},"filename":"psutil-5.9.8-cp27-none-win32.whl","hashes":{"sha256":"36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248660,"upload-time":"2024-01-19T20:47:29.706532Z","url":"https://files.pythonhosted.org/packages/fe/5f/c26deb822fd3daf8fde4bdb658bf87d9ab1ffd3fca483816e89a9a9a9084/psutil-5.9.8-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"bcd64b1f8b81a5a0ea3cff661abf5d6e3a937f442bb982b9bdcddbfd8608aa9c"},"data-dist-info-metadata":{"sha256":"bcd64b1f8b81a5a0ea3cff661abf5d6e3a937f442bb982b9bdcddbfd8608aa9c"},"filename":"psutil-5.9.8-cp27-none-win_amd64.whl","hashes":{"sha256":"bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":251966,"upload-time":"2024-01-19T20:47:33.134054Z","url":"https://files.pythonhosted.org/packages/32/1d/cf66073d74d6146187e2d0081a7616df4437214afa294ee4f16f80a2f96a/psutil-5.9.8-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":248702,"upload-time":"2024-01-19T20:47:36.303498Z","url":"https://files.pythonhosted.org/packages/e7/e3/07ae864a636d70a8a6f58da27cb1179192f1140d5d1da10886ade9405797/psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":285242,"upload-time":"2024-01-19T20:47:39.650099Z","url":"https://files.pythonhosted.org/packages/b3/bd/28c5f553667116b2598b9cc55908ec435cb7f77a34f2bff3e3ca765b0f78/psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":288191,"upload-time":"2024-01-19T20:47:43.078208Z","url":"https://files.pythonhosted.org/packages/c5/4f/0e22aaa246f96d6ac87fe5ebb9c5a693fbe8877f537a1022527c47ca43c5/psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-cp36m-win32.whl","hashes":{"sha256":"7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":253203,"upload-time":"2024-01-19T20:47:46.133427Z","url":"https://files.pythonhosted.org/packages/dd/9e/85c3bd5b466d96c091bbd6339881e99106adb43d5d60bde32ac181ab6fef/psutil-5.9.8-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"data-dist-info-metadata":{"sha256":"19f679f9f89daead325ce0a0c8de1fba70a554793a47fcdf273fb714ac742a35"},"filename":"psutil-5.9.8-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":258655,"upload-time":"2024-01-19T20:47:48.804624Z","url":"https://files.pythonhosted.org/packages/0b/58/bcffb5ab03ec558e565d2871c01215dde74e11f583fb71e7d2b107200caa/psutil-5.9.8-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"949401d7571a13b3b43062a6c13b80a2a1361c3da4af20751f844e6ee3750021"},"data-dist-info-metadata":{"sha256":"949401d7571a13b3b43062a6c13b80a2a1361c3da4af20751f844e6ee3750021"},"filename":"psutil-5.9.8-cp37-abi3-win32.whl","hashes":{"sha256":"bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":251252,"upload-time":"2024-01-19T20:47:52.880124Z","url":"https://files.pythonhosted.org/packages/6e/f5/2aa3a4acdc1e5940b59d421742356f133185667dd190b166dbcfcf5d7b43/psutil-5.9.8-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"949401d7571a13b3b43062a6c13b80a2a1361c3da4af20751f844e6ee3750021"},"data-dist-info-metadata":{"sha256":"949401d7571a13b3b43062a6c13b80a2a1361c3da4af20751f844e6ee3750021"},"filename":"psutil-5.9.8-cp37-abi3-win_amd64.whl","hashes":{"sha256":"8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":255090,"upload-time":"2024-01-19T20:47:56.019799Z","url":"https://files.pythonhosted.org/packages/93/52/3e39d26feae7df0aa0fd510b14012c3678b36ed068f7d78b8d8784d61f0e/psutil-5.9.8-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"1526b4912f0f04a9c6523b41c128f4d1c6c666fbbdccb62c6705d7e5747c95cc"},"data-dist-info-metadata":{"sha256":"1526b4912f0f04a9c6523b41c128f4d1c6c666fbbdccb62c6705d7e5747c95cc"},"filename":"psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":249898,"upload-time":"2024-01-19T20:47:59.238740Z","url":"https://files.pythonhosted.org/packages/05/33/2d74d588408caedd065c2497bdb5ef83ce6082db01289a1e1147f6639802/psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-5.9.8.tar.gz","hashes":{"sha256":"6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"},"provenance":null,"requires-python":">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*","size":503247,"upload-time":"2024-01-19T20:47:09.517227Z","url":"https://files.pythonhosted.org/packages/90/c7/6dc0a455d111f68ee43f27793971cf03fe29b6ef972042549db29eec39a2/psutil-5.9.8.tar.gz","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"a021da3e881cd935e64a3d0a20983bda0bb4cf80e4f74fa9bfcb1bc5785360c6"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250527,"upload-time":"2024-06-18T21:40:17.061973Z","url":"https://files.pythonhosted.org/packages/13/e5/35ebd7169008752be5561cafdba3f1634be98193b85fe3d22e883f9fe2e1/psutil-6.0.0-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"1287c2b95f1c0a364d23bc6f2ea2365a8d4d9b726a3be7294296ff7ba97c17f0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":316838,"upload-time":"2024-06-18T21:40:32.679396Z","url":"https://files.pythonhosted.org/packages/92/a7/083388ef0964a6d74df51c677b3d761e0866d823d37e3a8823551c0d375d/psutil-6.0.0-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"a9a3dbfb4de4f18174528d87cc352d1f788b7496991cca33c6996f40c9e3c92c"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":317493,"upload-time":"2024-06-18T21:40:41.710402Z","url":"https://files.pythonhosted.org/packages/52/2f/44b7005f306ea8bfd24aa662b5d0ba6ea1daf29dbd0b6c7bbcd3606373ad/psutil-6.0.0-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"6ec7588fb3ddaec7344a825afe298db83fe01bfaaab39155fa84cf1c0d6b13c3"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":316855,"upload-time":"2024-06-18T21:40:47.752750Z","url":"https://files.pythonhosted.org/packages/81/c9/8cb36769b6636d817be3414ebbb27a9ab3fbe6d13835d00f31e77e1fccce/psutil-6.0.0-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"data-dist-info-metadata":{"sha256":"c8301b547ede1d12db293b91c2eda41064442de769d8466cfdb34279587226a9"},"filename":"psutil-6.0.0-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"1e7c870afcb7d91fdea2b37c24aeb08f98b6d67257a5cb0a8bc3ac68d0f1a68c"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":317519,"upload-time":"2024-06-18T21:40:53.708954Z","url":"https://files.pythonhosted.org/packages/14/c0/024ac5369ca160e9ed45ed09247d9d779c460017fbd9aa801fd6eb0f060c/psutil-6.0.0-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"30df484959758f1144d54b7788b34cc3dde563aa4d6853a6e549f2f022b23c5e"},"data-dist-info-metadata":{"sha256":"30df484959758f1144d54b7788b34cc3dde563aa4d6853a6e549f2f022b23c5e"},"filename":"psutil-6.0.0-cp27-none-win32.whl","hashes":{"sha256":"02b69001f44cc73c1c5279d02b30a817e339ceb258ad75997325e0e6169d8b35"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":249766,"upload-time":"2024-06-18T21:40:58.381272Z","url":"https://files.pythonhosted.org/packages/c5/66/78c9c3020f573c58101dc43a44f6855d01bbbd747e24da2f0c4491200ea3/psutil-6.0.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"30df484959758f1144d54b7788b34cc3dde563aa4d6853a6e549f2f022b23c5e"},"data-dist-info-metadata":{"sha256":"30df484959758f1144d54b7788b34cc3dde563aa4d6853a6e549f2f022b23c5e"},"filename":"psutil-6.0.0-cp27-none-win_amd64.whl","hashes":{"sha256":"21f1fb635deccd510f69f485b87433460a603919b45e2a324ad65b0cc74f8fb1"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":253024,"upload-time":"2024-06-18T21:41:04.548455Z","url":"https://files.pythonhosted.org/packages/e1/3f/2403aa9558bea4d3854b0e5e567bc3dd8e9fbc1fc4453c0aa9aafeb75467/psutil-6.0.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250961,"upload-time":"2024-06-18T21:41:11.662513Z","url":"https://files.pythonhosted.org/packages/0b/37/f8da2fbd29690b3557cca414c1949f92162981920699cd62095a984983bf/psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"6ed2440ada7ef7d0d608f20ad89a04ec47d2d3ab7190896cd62ca5fc4fe08bf0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":287478,"upload-time":"2024-06-18T21:41:16.180526Z","url":"https://files.pythonhosted.org/packages/35/56/72f86175e81c656a01c4401cd3b1c923f891b31fbcebe98985894176d7c9/psutil-6.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":290455,"upload-time":"2024-06-18T21:41:29.048203Z","url":"https://files.pythonhosted.org/packages/19/74/f59e7e0d392bc1070e9a70e2f9190d652487ac115bb16e2eff6b22ad1d24/psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":292046,"upload-time":"2024-06-18T21:41:33.530555Z","url":"https://files.pythonhosted.org/packages/cd/5f/60038e277ff0a9cc8f0c9ea3d0c5eb6ee1d2470ea3f9389d776432888e47/psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-cp36m-win32.whl","hashes":{"sha256":"fc8c9510cde0146432bbdb433322861ee8c3efbf8589865c8bf8d21cb30c4d14"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":255537,"upload-time":"2024-06-18T21:41:38.034852Z","url":"https://files.pythonhosted.org/packages/cd/ff/39c38910cdb8f02fc9965afb520967a1e9307d53d14879dddd0a4f41f6f8/psutil-6.0.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"data-dist-info-metadata":{"sha256":"cce2895e87556b6152233398a38cd33b9b0712906393b6834b1448aec0a505ca"},"filename":"psutil-6.0.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"34859b8d8f423b86e4385ff3665d3f4d94be3cdf48221fbe476e883514fdb71c"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":260973,"upload-time":"2024-06-18T21:41:41.566213Z","url":"https://files.pythonhosted.org/packages/08/88/16dd53af4a84e719e27a5ad7db040231415d8caeb48f019bacafbb4d0002/psutil-6.0.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"9e9a9bbc68eaa4d9f9716cef6b0d5f90d4d57007c2008e7301c292c7714a6abc"},"data-dist-info-metadata":{"sha256":"9e9a9bbc68eaa4d9f9716cef6b0d5f90d4d57007c2008e7301c292c7714a6abc"},"filename":"psutil-6.0.0-cp37-abi3-win32.whl","hashes":{"sha256":"a495580d6bae27291324fe60cea0b5a7c23fa36a7cd35035a16d93bdcf076b9d"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":253560,"upload-time":"2024-06-18T21:41:46.067057Z","url":"https://files.pythonhosted.org/packages/8b/20/2ff69ad9c35c3df1858ac4e094f20bd2374d33c8643cf41da8fd7cdcb78b/psutil-6.0.0-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"9e9a9bbc68eaa4d9f9716cef6b0d5f90d4d57007c2008e7301c292c7714a6abc"},"data-dist-info-metadata":{"sha256":"9e9a9bbc68eaa4d9f9716cef6b0d5f90d4d57007c2008e7301c292c7714a6abc"},"filename":"psutil-6.0.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":257399,"upload-time":"2024-06-18T21:41:52.100137Z","url":"https://files.pythonhosted.org/packages/73/44/561092313ae925f3acfaace6f9ddc4f6a9c748704317bad9c8c8f8a36a79/psutil-6.0.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"c0710bf31d8e9160c90747acceb7692596b32d632efd784e22990fc01fad42dd"},"data-dist-info-metadata":{"sha256":"c0710bf31d8e9160c90747acceb7692596b32d632efd784e22990fc01fad42dd"},"filename":"psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":251988,"upload-time":"2024-06-18T21:41:57.337231Z","url":"https://files.pythonhosted.org/packages/7c/06/63872a64c312a24fb9b4af123ee7007a306617da63ff13bcc1432386ead7/psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-6.0.0.tar.gz","hashes":{"sha256":"8faae4f310b6d969fa26ca0545338b21f73c6b15db7c4a8d934a5482faa818f2"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":508067,"upload-time":"2024-06-18T21:40:10.559591Z","url":"https://files.pythonhosted.org/packages/18/c7/8c6872f7372eb6a6b2e4708b88419fb46b857f7a2e1892966b851cc79fc9/psutil-6.0.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":247385,"upload-time":"2024-10-17T21:31:49.162372Z","url":"https://files.pythonhosted.org/packages/cd/8e/87b51bedb52f0fa02a6c9399702912a5059b24c7242fa8ea4fd027cb5238/psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312067,"upload-time":"2024-10-17T21:31:51.572046Z","url":"https://files.pythonhosted.org/packages/2c/56/99304ecbf1f25a2aa336c66e43a8f9462de70d089d3fbb487991dfd96b37/psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312345,"upload-time":"2024-10-17T21:31:53.950865Z","url":"https://files.pythonhosted.org/packages/24/87/7c1eeb2fd86a8eb792b15438a3d25eda05c970924df3457669b50e0c022b/psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312058,"upload-time":"2024-10-17T21:31:55.843636Z","url":"https://files.pythonhosted.org/packages/aa/fe/c94a914040c74b2bbe2ddb2c82b3f9a74d8a40401bb1239b0e949331c957/psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"data-dist-info-metadata":{"sha256":"c1b2df70b2192969ac8cec0847acb78d7ddd669a788780aad80d1133c89c4a43"},"filename":"psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312330,"upload-time":"2024-10-17T21:31:57.551704Z","url":"https://files.pythonhosted.org/packages/44/8c/624823d5a5a9ec8635d63b273c3ab1554a4fcc3513f4d0236ff9706f1025/psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"6449e5b027e55d1dc345bb3b5089a60a968abd4f30577107504c292168ff86ea"},"data-dist-info-metadata":{"sha256":"6449e5b027e55d1dc345bb3b5089a60a968abd4f30577107504c292168ff86ea"},"filename":"psutil-6.1.0-cp27-none-win32.whl","hashes":{"sha256":"9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":246648,"upload-time":"2024-10-17T21:31:59.369185Z","url":"https://files.pythonhosted.org/packages/da/2b/f4dea5d993d9cd22ad958eea828a41d5d225556123d372f02547c29c4f97/psutil-6.1.0-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"6449e5b027e55d1dc345bb3b5089a60a968abd4f30577107504c292168ff86ea"},"data-dist-info-metadata":{"sha256":"6449e5b027e55d1dc345bb3b5089a60a968abd4f30577107504c292168ff86ea"},"filename":"psutil-6.1.0-cp27-none-win_amd64.whl","hashes":{"sha256":"a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":249905,"upload-time":"2024-10-17T21:32:01.974050Z","url":"https://files.pythonhosted.org/packages/9f/14/4aa97a7f2e0ac33a050d990ab31686d651ae4ef8c86661fef067f00437b9/psutil-6.1.0-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"data-dist-info-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"filename":"psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":247762,"upload-time":"2024-10-17T21:32:05.991637Z","url":"https://files.pythonhosted.org/packages/01/9e/8be43078a171381953cfee33c07c0d628594b5dbfc5157847b85022c2c1b/psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"31aa42573bd48a390acf1e517f08ce9f05a4954542dc8afb9a8805655df7aa18"},"data-dist-info-metadata":{"sha256":"31aa42573bd48a390acf1e517f08ce9f05a4954542dc8afb9a8805655df7aa18"},"filename":"psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":248777,"upload-time":"2024-10-17T21:32:07.872442Z","url":"https://files.pythonhosted.org/packages/1d/cb/313e80644ea407f04f6602a9e23096540d9dc1878755f3952ea8d3d104be/psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"data-dist-info-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"filename":"psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":284259,"upload-time":"2024-10-17T21:32:10.177301Z","url":"https://files.pythonhosted.org/packages/65/8e/bcbe2025c587b5d703369b6a75b65d41d1367553da6e3f788aff91eaf5bd/psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"data-dist-info-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"filename":"psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":287255,"upload-time":"2024-10-17T21:32:11.964687Z","url":"https://files.pythonhosted.org/packages/58/4d/8245e6f76a93c98aab285a43ea71ff1b171bcd90c9d238bf81f7021fb233/psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"data-dist-info-metadata":{"sha256":"f6ba356ca54ff4137597ca1c9c55cca6c6ccfd672a0c976cc370711d0a2652e8"},"filename":"psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":288804,"upload-time":"2024-10-17T21:32:13.785068Z","url":"https://files.pythonhosted.org/packages/27/c2/d034856ac47e3b3cdfa9720d0e113902e615f4190d5d1bdb8df4b2015fb2/psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"4a33e7577c0dd3577291121b5beb777de2767238e7d66ad068c92fd37d3f3d6a"},"data-dist-info-metadata":{"sha256":"4a33e7577c0dd3577291121b5beb777de2767238e7d66ad068c92fd37d3f3d6a"},"filename":"psutil-6.1.0-cp36-cp36m-win32.whl","hashes":{"sha256":"6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":252360,"upload-time":"2024-10-17T21:32:16.434334Z","url":"https://files.pythonhosted.org/packages/43/39/414d7b67f4df35bb9c373d0fb9a75dd40b223d9bd6d02ebdc7658fd461a3/psutil-6.1.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4a33e7577c0dd3577291121b5beb777de2767238e7d66ad068c92fd37d3f3d6a"},"data-dist-info-metadata":{"sha256":"4a33e7577c0dd3577291121b5beb777de2767238e7d66ad068c92fd37d3f3d6a"},"filename":"psutil-6.1.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":257797,"upload-time":"2024-10-17T21:32:18.946615Z","url":"https://files.pythonhosted.org/packages/ca/da/ef86c99e33be4aa888570e79350caca8c4819b62f84a6d9274c88c40e331/psutil-6.1.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"7fdb0cc933f96b13edc5fc4b2255851b3a0222e29777763dd064aaeacaed9cb6"},"data-dist-info-metadata":{"sha256":"7fdb0cc933f96b13edc5fc4b2255851b3a0222e29777763dd064aaeacaed9cb6"},"filename":"psutil-6.1.0-cp37-abi3-win32.whl","hashes":{"sha256":"1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250386,"upload-time":"2024-10-17T21:32:21.399329Z","url":"https://files.pythonhosted.org/packages/ea/55/5389ed243c878725feffc0d6a3bc5ef6764312b6fc7c081faaa2cfa7ef37/psutil-6.1.0-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"7fdb0cc933f96b13edc5fc4b2255851b3a0222e29777763dd064aaeacaed9cb6"},"data-dist-info-metadata":{"sha256":"7fdb0cc933f96b13edc5fc4b2255851b3a0222e29777763dd064aaeacaed9cb6"},"filename":"psutil-6.1.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":254228,"upload-time":"2024-10-17T21:32:23.880601Z","url":"https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-6.1.0.tar.gz","hashes":{"sha256":"353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":508565,"upload-time":"2024-10-17T21:31:45.680545Z","url":"https://files.pythonhosted.org/packages/26/10/2a30b13c61e7cf937f4adf90710776b7918ed0a9c434e2c38224732af310/psutil-6.1.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl","hashes":{"sha256":"9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":247226,"upload-time":"2024-12-19T18:21:25.276122Z","url":"https://files.pythonhosted.org/packages/09/ea/f8844afff4c8c11d1d0586b737d8d579fd7cb13f1fa3eea599c71877b526/psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl","hashes":{"sha256":"ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312292,"upload-time":"2024-12-19T18:21:30.930117Z","url":"https://files.pythonhosted.org/packages/51/f8/e376f9410beb915bbf64cb4ae8ce5cf2d03e9a661a2519ebc6a63045a1ca/psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl","hashes":{"sha256":"8df0178ba8a9e5bc84fed9cfa61d54601b371fbec5c8eebad27575f1e105c0d4"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312542,"upload-time":"2024-12-19T18:21:34.735400Z","url":"https://files.pythonhosted.org/packages/a7/3a/069d6c1e4a7af3cdb162c9ba0737ff9baed1d05cbab6f082f49e3b9ab0a5/psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl","hashes":{"sha256":"1924e659d6c19c647e763e78670a05dbb7feaf44a0e9c94bf9e14dfc6ba50468"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312279,"upload-time":"2024-12-19T18:21:37.897094Z","url":"https://files.pythonhosted.org/packages/81/d5/ee5de2cb8d0c938bb07dcccd4ff7e950359bd6ddbd2fe3118552f863bb52/psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl","yanked":false},{"core-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"data-dist-info-metadata":{"sha256":"b63844c7465e7a965b315aaeaab5c82b781f31476c21c62e1c9a135bceff2115"},"filename":"psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl","hashes":{"sha256":"018aeae2af92d943fdf1da6b58665124897cfc94faa2ca92098838f83e1b1bca"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":312521,"upload-time":"2024-12-19T18:21:40.651860Z","url":"https://files.pythonhosted.org/packages/37/98/443eff82762b3f2c6a4bd0cdf3bc5c9f62245376c5486b39ee194e920794/psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"55f120e7e066d74e328c2e0d559aa667c611e9e63f5a1bc5cae3d034db0be644"},"data-dist-info-metadata":{"sha256":"55f120e7e066d74e328c2e0d559aa667c611e9e63f5a1bc5cae3d034db0be644"},"filename":"psutil-6.1.1-cp27-none-win32.whl","hashes":{"sha256":"6d4281f5bbca041e2292be3380ec56a9413b790579b8e593b1784499d0005dac"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":246855,"upload-time":"2024-12-19T18:54:12.657947Z","url":"https://files.pythonhosted.org/packages/d2/d4/8095b53c4950f44dc99b8d983b796f405ae1f58d80978fcc0421491b4201/psutil-6.1.1-cp27-none-win32.whl","yanked":false},{"core-metadata":{"sha256":"55f120e7e066d74e328c2e0d559aa667c611e9e63f5a1bc5cae3d034db0be644"},"data-dist-info-metadata":{"sha256":"55f120e7e066d74e328c2e0d559aa667c611e9e63f5a1bc5cae3d034db0be644"},"filename":"psutil-6.1.1-cp27-none-win_amd64.whl","hashes":{"sha256":"c777eb75bb33c47377c9af68f30e9f11bc78e0f07fbf907be4a5d70b2fe5f030"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250110,"upload-time":"2024-12-19T18:54:16.635901Z","url":"https://files.pythonhosted.org/packages/b1/63/0b6425ea4f2375988209a9934c90d6079cc7537847ed58a28fbe30f4277e/psutil-6.1.1-cp27-none-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"data-dist-info-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"filename":"psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":247511,"upload-time":"2024-12-19T18:21:45.163741Z","url":"https://files.pythonhosted.org/packages/61/99/ca79d302be46f7bdd8321089762dd4476ee725fce16fc2b2e1dbba8cac17/psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"f53aa914fe0e63af2de701b5d7b1c28a494b1852f4eddac6bec1c1d8eecc00d7"},"data-dist-info-metadata":{"sha256":"f53aa914fe0e63af2de701b5d7b1c28a494b1852f4eddac6bec1c1d8eecc00d7"},"filename":"psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":248985,"upload-time":"2024-12-19T18:21:49.254078Z","url":"https://files.pythonhosted.org/packages/0b/6b/73dbde0dd38f3782905d4587049b9be64d76671042fdcaf60e2430c6796d/psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"data-dist-info-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"filename":"psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":284488,"upload-time":"2024-12-19T18:21:51.638630Z","url":"https://files.pythonhosted.org/packages/17/38/c319d31a1d3f88c5b79c68b3116c129e5133f1822157dd6da34043e32ed6/psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"data-dist-info-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"filename":"psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":287477,"upload-time":"2024-12-19T18:21:55.306984Z","url":"https://files.pythonhosted.org/packages/9c/39/0f88a830a1c8a3aba27fededc642da37613c57cbff143412e3536f89784f/psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"data-dist-info-metadata":{"sha256":"06627f791c3970710891bde5c145df9bac4b643a1529e21d25a8c1c679aca404"},"filename":"psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":289017,"upload-time":"2024-12-19T18:21:57.875754Z","url":"https://files.pythonhosted.org/packages/47/da/99f4345d4ddf2845cb5b5bd0d93d554e84542d116934fde07a0c50bd4e9f/psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"4899522d2eb9cfd3ec5a1a377c43fb70f9e3963a8989dd7f84c20d6d00083470"},"data-dist-info-metadata":{"sha256":"4899522d2eb9cfd3ec5a1a377c43fb70f9e3963a8989dd7f84c20d6d00083470"},"filename":"psutil-6.1.1-cp36-cp36m-win32.whl","hashes":{"sha256":"384636b1a64b47814437d1173be1427a7c83681b17a450bfc309a1953e329603"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":252576,"upload-time":"2024-12-19T18:22:01.852822Z","url":"https://files.pythonhosted.org/packages/8e/1f/1aebe4dd5914ccba6f7d6cc6d11fb79f6f23f95b858a7f631446bdc5d67f/psutil-6.1.1-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"4899522d2eb9cfd3ec5a1a377c43fb70f9e3963a8989dd7f84c20d6d00083470"},"data-dist-info-metadata":{"sha256":"4899522d2eb9cfd3ec5a1a377c43fb70f9e3963a8989dd7f84c20d6d00083470"},"filename":"psutil-6.1.1-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"8be07491f6ebe1a693f17d4f11e69d0dc1811fa082736500f649f79df7735303"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":258012,"upload-time":"2024-12-19T18:22:04.204308Z","url":"https://files.pythonhosted.org/packages/f4/de/fb4561e59611c19a2d7377c2b2534d11274b8a7df9bb7b7e7f1de5be3641/psutil-6.1.1-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"492e1282d630b4482b89e0c91cd388cea83f8efa851c6c2ffc46a6875c62b52d"},"data-dist-info-metadata":{"sha256":"492e1282d630b4482b89e0c91cd388cea83f8efa851c6c2ffc46a6875c62b52d"},"filename":"psutil-6.1.1-cp37-abi3-win32.whl","hashes":{"sha256":"eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":250602,"upload-time":"2024-12-19T18:22:08.808295Z","url":"https://files.pythonhosted.org/packages/38/53/bd755c2896f4461fd4f36fa6a6dcb66a88a9e4b9fd4e5b66a77cf9d4a584/psutil-6.1.1-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"492e1282d630b4482b89e0c91cd388cea83f8efa851c6c2ffc46a6875c62b52d"},"data-dist-info-metadata":{"sha256":"492e1282d630b4482b89e0c91cd388cea83f8efa851c6c2ffc46a6875c62b52d"},"filename":"psutil-6.1.1-cp37-abi3-win_amd64.whl","hashes":{"sha256":"f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":254444,"upload-time":"2024-12-19T18:22:11.335598Z","url":"https://files.pythonhosted.org/packages/7b/d7/7831438e6c3ebbfa6e01a927127a6cb42ad3ab844247f3c5b96bea25d73d/psutil-6.1.1-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-6.1.1.tar.gz","hashes":{"sha256":"cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5"},"provenance":null,"requires-python":"!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7","size":508502,"upload-time":"2024-12-19T18:21:20.568966Z","url":"https://files.pythonhosted.org/packages/1f/5a/07871137bb752428aa4b659f910b399ba6f291156bdea939be3e96cae7cb/psutil-6.1.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"data-dist-info-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"filename":"psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25"},"provenance":null,"requires-python":">=3.6","size":238051,"upload-time":"2025-02-13T21:54:12.360451Z","url":"https://files.pythonhosted.org/packages/ed/e6/2d26234410f8b8abdbf891c9da62bee396583f713fb9f3325a4760875d22/psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"65f64bcd26d7284f4f07568c343ced6641bf8235c8c5a09a4986dd2ec5f68de7"},"data-dist-info-metadata":{"sha256":"65f64bcd26d7284f4f07568c343ced6641bf8235c8c5a09a4986dd2ec5f68de7"},"filename":"psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da"},"provenance":null,"requires-python":">=3.6","size":239535,"upload-time":"2025-02-13T21:54:16.070769Z","url":"https://files.pythonhosted.org/packages/04/8b/30f930733afe425e3cbfc0e1468a30a18942350c1a8816acfade80c005c4/psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"data-dist-info-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"filename":"psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91"},"provenance":null,"requires-python":">=3.6","size":275004,"upload-time":"2025-02-13T21:54:18.662603Z","url":"https://files.pythonhosted.org/packages/2a/ed/d362e84620dd22876b55389248e522338ed1bf134a5edd3b8231d7207f6d/psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"data-dist-info-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"filename":"psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34"},"provenance":null,"requires-python":">=3.6","size":277986,"upload-time":"2025-02-13T21:54:21.811145Z","url":"https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"data-dist-info-metadata":{"sha256":"0453e4553a618524d8001e2f64a83b1c24419826cab87bf6b734e6723d383c89"},"filename":"psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993"},"provenance":null,"requires-python":">=3.6","size":279544,"upload-time":"2025-02-13T21:54:24.680762Z","url":"https://files.pythonhosted.org/packages/eb/a2/709e0fe2f093556c17fbafda93ac032257242cabcc7ff3369e2cb76a97aa/psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"1326531887a042b91e9db0d6e74bd66dde0a4fd27151ed4b8baeca1fad635542"},"data-dist-info-metadata":{"sha256":"1326531887a042b91e9db0d6e74bd66dde0a4fd27151ed4b8baeca1fad635542"},"filename":"psutil-7.0.0-cp36-cp36m-win32.whl","hashes":{"sha256":"84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17"},"provenance":null,"requires-python":">=3.6","size":243024,"upload-time":"2025-02-13T21:54:27.767214Z","url":"https://files.pythonhosted.org/packages/98/04/9e7b8afdad85824dec17de92c121d0fb1907ded624f486b86cd5e8189ebe/psutil-7.0.0-cp36-cp36m-win32.whl","yanked":false},{"core-metadata":{"sha256":"1326531887a042b91e9db0d6e74bd66dde0a4fd27151ed4b8baeca1fad635542"},"data-dist-info-metadata":{"sha256":"1326531887a042b91e9db0d6e74bd66dde0a4fd27151ed4b8baeca1fad635542"},"filename":"psutil-7.0.0-cp36-cp36m-win_amd64.whl","hashes":{"sha256":"1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e"},"provenance":null,"requires-python":">=3.6","size":248462,"upload-time":"2025-02-13T21:54:31.148496Z","url":"https://files.pythonhosted.org/packages/25/9b/43f2c5f7794a3eba3fc0bb47020d1da44d43ff41c95637c5d760c3ef33eb/psutil-7.0.0-cp36-cp36m-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"8c4198dfca297dfee074ee46394207f317965c25dc1af8ad38862b954795a7c1"},"data-dist-info-metadata":{"sha256":"8c4198dfca297dfee074ee46394207f317965c25dc1af8ad38862b954795a7c1"},"filename":"psutil-7.0.0-cp37-abi3-win32.whl","hashes":{"sha256":"ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99"},"provenance":null,"requires-python":">=3.6","size":241053,"upload-time":"2025-02-13T21:54:34.310916Z","url":"https://files.pythonhosted.org/packages/50/e6/eecf58810b9d12e6427369784efe814a1eec0f492084ce8eb8f4d89d6d61/psutil-7.0.0-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"8c4198dfca297dfee074ee46394207f317965c25dc1af8ad38862b954795a7c1"},"data-dist-info-metadata":{"sha256":"8c4198dfca297dfee074ee46394207f317965c25dc1af8ad38862b954795a7c1"},"filename":"psutil-7.0.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553"},"provenance":null,"requires-python":">=3.6","size":244885,"upload-time":"2025-02-13T21:54:37.486453Z","url":"https://files.pythonhosted.org/packages/50/1b/6921afe68c74868b4c9fa424dad3be35b095e16687989ebbb50ce4fceb7c/psutil-7.0.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.0.0.tar.gz","hashes":{"sha256":"7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456"},"provenance":null,"requires-python":">=3.6","size":497003,"upload-time":"2025-02-13T21:54:07.946974Z","url":"https://files.pythonhosted.org/packages/2a/80/336820c1ad9286a4ded7e845b2eccfcb27851ab8ac6abece774a6ff4d3de/psutil-7.0.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"76168cef4397494250e9f4e73eb3752b146de1dd950040b29186d0cce1d5ca13"},"provenance":null,"requires-python":">=3.6","size":245242,"upload-time":"2025-09-17T20:14:56.126572Z","url":"https://files.pythonhosted.org/packages/46/62/ce4051019ee20ce0ed74432dd73a5bb087a6704284a470bb8adff69a0932/psutil-7.1.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"5d007560c8c372efdff9e4579c2846d71de737e4605f611437255e81efcca2c5"},"provenance":null,"requires-python":">=3.6","size":246682,"upload-time":"2025-09-17T20:14:58.250040Z","url":"https://files.pythonhosted.org/packages/38/61/f76959fba841bf5b61123fbf4b650886dc4094c6858008b5bf73d9057216/psutil-7.1.0-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"22e4454970b32472ce7deaa45d045b34d3648ce478e26a04c7e858a0a6e75ff3"},"provenance":null,"requires-python":">=3.6","size":287994,"upload-time":"2025-09-17T20:14:59.901485Z","url":"https://files.pythonhosted.org/packages/88/7a/37c99d2e77ec30d63398ffa6a660450b8a62517cabe44b3e9bae97696e8d/psutil-7.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"8c70e113920d51e89f212dd7be06219a9b88014e63a4cec69b684c327bc474e3"},"provenance":null,"requires-python":">=3.6","size":291163,"upload-time":"2025-09-17T20:15:01.481447Z","url":"https://files.pythonhosted.org/packages/9d/de/04c8c61232f7244aa0a4b9a9fbd63a89d5aeaf94b2fc9d1d16e2faa5cbb0/psutil-7.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"data-dist-info-metadata":{"sha256":"ac00a0bcd2510cbd355745e1742a41ef2eadd59546c28788d6122a57d329c6b6"},"filename":"psutil-7.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"7d4a113425c037300de3ac8b331637293da9be9713855c4fc9d2d97436d7259d"},"provenance":null,"requires-python":">=3.6","size":293625,"upload-time":"2025-09-17T20:15:04.492789Z","url":"https://files.pythonhosted.org/packages/f4/58/c4f976234bf6d4737bc8c02a81192f045c307b72cf39c9e5c5a2d78927f6/psutil-7.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"24dfa33010cf11743d32bccef7c6bf186df383e24c75c8f904a21dc21061932a"},"data-dist-info-metadata":{"sha256":"24dfa33010cf11743d32bccef7c6bf186df383e24c75c8f904a21dc21061932a"},"filename":"psutil-7.1.0-cp37-abi3-win32.whl","hashes":{"sha256":"09ad740870c8d219ed8daae0ad3b726d3bf9a028a198e7f3080f6a1888b99bca"},"provenance":null,"requires-python":">=3.6","size":244812,"upload-time":"2025-09-17T20:15:07.462276Z","url":"https://files.pythonhosted.org/packages/79/87/157c8e7959ec39ced1b11cc93c730c4fb7f9d408569a6c59dbd92ceb35db/psutil-7.1.0-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"24dfa33010cf11743d32bccef7c6bf186df383e24c75c8f904a21dc21061932a"},"data-dist-info-metadata":{"sha256":"24dfa33010cf11743d32bccef7c6bf186df383e24c75c8f904a21dc21061932a"},"filename":"psutil-7.1.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"57f5e987c36d3146c0dd2528cd42151cf96cd359b9d67cfff836995cc5df9a3d"},"provenance":null,"requires-python":">=3.6","size":247965,"upload-time":"2025-09-17T20:15:09.673366Z","url":"https://files.pythonhosted.org/packages/bf/e9/b44c4f697276a7a95b8e94d0e320a7bf7f3318521b23de69035540b39838/psutil-7.1.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"24a51389a00af9d96fada7f4a79aed176d3a74f0ae399a5305d12ce9784f2635"},"data-dist-info-metadata":{"sha256":"24a51389a00af9d96fada7f4a79aed176d3a74f0ae399a5305d12ce9784f2635"},"filename":"psutil-7.1.0-cp37-abi3-win_arm64.whl","hashes":{"sha256":"6937cb68133e7c97b6cc9649a570c9a18ba0efebed46d8c5dae4c07fa1b67a07"},"provenance":null,"requires-python":">=3.6","size":244971,"upload-time":"2025-09-17T20:15:12.262753Z","url":"https://files.pythonhosted.org/packages/26/65/1070a6e3c036f39142c2820c4b52e9243246fcfc3f96239ac84472ba361e/psutil-7.1.0-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.1.0.tar.gz","hashes":{"sha256":"655708b3c069387c8b77b072fc429a57d0e214221d01c0a772df7dfedcb3bcd2"},"provenance":null,"requires-python":">=3.6","size":497660,"upload-time":"2025-09-17T20:14:52.902036Z","url":"https://files.pythonhosted.org/packages/b3/31/4723d756b59344b643542936e37a31d1d3204bcdc42a7daa8ee9eb06fb50/psutil-7.1.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"8fa59d7b1f01f0337f12cd10dbd76e4312a4d3c730a4fedcbdd4e5447a8b8460"},"provenance":null,"requires-python":">=3.6","size":244221,"upload-time":"2025-10-19T15:44:03.145914Z","url":"https://files.pythonhosted.org/packages/51/30/f97f8fb1f9ecfbeae4b5ca738dcae66ab28323b5cfbc96cb5565f3754056/psutil-7.1.1-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"2a95104eae85d088891716db676f780c1404fc15d47fde48a46a5d61e8f5ad2c"},"provenance":null,"requires-python":">=3.6","size":245660,"upload-time":"2025-10-19T15:44:05.657308Z","url":"https://files.pythonhosted.org/packages/7b/98/b8d1f61ebf35f4dbdbaabadf9208282d8adc820562f0257e5e6e79e67bf2/psutil-7.1.1-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","hashes":{"sha256":"98629cd8567acefcc45afe2f4ba1e9290f579eacf490a917967decce4b74ee9b"},"provenance":null,"requires-python":">=3.6","size":286963,"upload-time":"2025-10-19T15:44:08.877299Z","url":"https://files.pythonhosted.org/packages/f0/4a/b8015d7357fefdfe34bc4a3db48a107bae4bad0b94fb6eb0613f09a08ada/psutil-7.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","hashes":{"sha256":"92ebc58030fb054fa0f26c3206ef01c31c29d67aee1367e3483c16665c25c8d2"},"provenance":null,"requires-python":">=3.6","size":290118,"upload-time":"2025-10-19T15:44:11.897889Z","url":"https://files.pythonhosted.org/packages/3d/3c/b56076bb35303d0733fc47b110a1c9cce081a05ae2e886575a3587c1ee76/psutil-7.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"data-dist-info-metadata":{"sha256":"d94db19e208334cc22bb81d2d0102f837a1101c98a0d350fa3d257181ae1ef09"},"filename":"psutil-7.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","hashes":{"sha256":"146a704f224fb2ded2be3da5ac67fc32b9ea90c45b51676f9114a6ac45616967"},"provenance":null,"requires-python":">=3.6","size":292587,"upload-time":"2025-10-19T15:44:14.670833Z","url":"https://files.pythonhosted.org/packages/dc/af/c13d360c0adc6f6218bf9e2873480393d0f729c8dd0507d171f53061c0d3/psutil-7.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"b46a0b2c74ca7acef22d45a6a1b88ac5c5a55793782e1b66e08e8ac56f33ddb1"},"data-dist-info-metadata":{"sha256":"b46a0b2c74ca7acef22d45a6a1b88ac5c5a55793782e1b66e08e8ac56f33ddb1"},"filename":"psutil-7.1.1-cp37-abi3-win32.whl","hashes":{"sha256":"295c4025b5cd880f7445e4379e6826f7307e3d488947bf9834e865e7847dc5f7"},"provenance":null,"requires-python":">=3.6","size":243772,"upload-time":"2025-10-19T15:44:16.938205Z","url":"https://files.pythonhosted.org/packages/90/2d/c933e7071ba60c7862813f2c7108ec4cf8304f1c79660efeefd0de982258/psutil-7.1.1-cp37-abi3-win32.whl","yanked":false},{"core-metadata":{"sha256":"b46a0b2c74ca7acef22d45a6a1b88ac5c5a55793782e1b66e08e8ac56f33ddb1"},"data-dist-info-metadata":{"sha256":"b46a0b2c74ca7acef22d45a6a1b88ac5c5a55793782e1b66e08e8ac56f33ddb1"},"filename":"psutil-7.1.1-cp37-abi3-win_amd64.whl","hashes":{"sha256":"9b4f17c5f65e44f69bd3a3406071a47b79df45cf2236d1f717970afcb526bcd3"},"provenance":null,"requires-python":">=3.6","size":246936,"upload-time":"2025-10-19T15:44:18.663465Z","url":"https://files.pythonhosted.org/packages/be/f3/11fd213fff15427bc2853552138760c720fd65032d99edfb161910d04127/psutil-7.1.1-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"41f2736f936aaaa81486aad04717acc54669b33ae5ad93df72d7a7b26aa50fa7"},"data-dist-info-metadata":{"sha256":"41f2736f936aaaa81486aad04717acc54669b33ae5ad93df72d7a7b26aa50fa7"},"filename":"psutil-7.1.1-cp37-abi3-win_arm64.whl","hashes":{"sha256":"5457cf741ca13da54624126cd5d333871b454ab133999a9a103fb097a7d7d21a"},"provenance":null,"requires-python":">=3.6","size":243944,"upload-time":"2025-10-19T15:44:20.666512Z","url":"https://files.pythonhosted.org/packages/0a/8d/8a9a45c8b655851f216c1d44f68e3533dc8d2c752ccd0f61f1aa73be4893/psutil-7.1.1-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.1.1.tar.gz","hashes":{"sha256":"092b6350145007389c1cfe5716050f02030a05219d90057ea867d18fe8d372fc"},"provenance":null,"requires-python":">=3.6","size":487067,"upload-time":"2025-10-19T15:43:59.373160Z","url":"https://files.pythonhosted.org/packages/89/fc/889242351a932d6183eec5df1fc6539b6f36b6a88444f1e63f18668253aa/psutil-7.1.1.tar.gz","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp313-cp313t-macosx_10_13_x86_64.whl","hashes":{"sha256":"0cc5c6889b9871f231ed5455a9a02149e388fffcb30b607fb7a8896a6d95f22e"},"provenance":null,"requires-python":">=3.6","size":238575,"upload-time":"2025-10-25T10:46:38.728747Z","url":"https://files.pythonhosted.org/packages/b8/d9/b56cc9f883140ac10021a8c9b0f4e16eed1ba675c22513cdcbce3ba64014/psutil-7.1.2-cp313-cp313t-macosx_10_13_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp313-cp313t-macosx_11_0_arm64.whl","hashes":{"sha256":"8e9e77a977208d84aa363a4a12e0f72189d58bbf4e46b49aae29a2c6e93ef206"},"provenance":null,"requires-python":">=3.6","size":239297,"upload-time":"2025-10-25T10:46:41.347184Z","url":"https://files.pythonhosted.org/packages/36/eb/28d22de383888deb252c818622196e709da98816e296ef95afda33f1c0a2/psutil-7.1.2-cp313-cp313t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"7d9623a5e4164d2220ecceb071f4b333b3c78866141e8887c072129185f41278"},"provenance":null,"requires-python":">=3.6","size":280420,"upload-time":"2025-10-25T10:46:44.122205Z","url":"https://files.pythonhosted.org/packages/89/5d/220039e2f28cc129626e54d63892ab05c0d56a29818bfe7268dcb5008932/psutil-7.1.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"364b1c10fe4ed59c89ec49e5f1a70da353b27986fa8233b4b999df4742a5ee2f"},"provenance":null,"requires-python":">=3.6","size":283049,"upload-time":"2025-10-25T10:46:47.095973Z","url":"https://files.pythonhosted.org/packages/ba/7a/286f0e1c167445b2ef4a6cbdfc8c59fdb45a5a493788950cf8467201dc73/psutil-7.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp313-cp313t-win_amd64.whl","hashes":{"sha256":"f101ef84de7e05d41310e3ccbdd65a6dd1d9eed85e8aaf0758405d022308e204"},"provenance":null,"requires-python":">=3.6","size":248713,"upload-time":"2025-10-25T10:46:49.573269Z","url":"https://files.pythonhosted.org/packages/aa/cc/7eb93260794a42e39b976f3a4dde89725800b9f573b014fac142002a5c98/psutil-7.1.2-cp313-cp313t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp313-cp313t-win_arm64.whl","hashes":{"sha256":"20c00824048a95de67f00afedc7b08b282aa08638585b0206a9fb51f28f1a165"},"provenance":null,"requires-python":">=3.6","size":244644,"upload-time":"2025-10-25T10:46:51.924062Z","url":"https://files.pythonhosted.org/packages/ab/1a/0681a92b53366e01f0a099f5237d0c8a2f79d322ac589cccde5e30c8a4e2/psutil-7.1.2-cp313-cp313t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp314-cp314t-macosx_10_15_x86_64.whl","hashes":{"sha256":"e09cfe92aa8e22b1ec5e2d394820cf86c5dff6367ac3242366485dfa874d43bc"},"provenance":null,"requires-python":">=3.6","size":238640,"upload-time":"2025-10-25T10:46:54.089898Z","url":"https://files.pythonhosted.org/packages/56/9e/f1c5c746b4ed5320952acd3002d3962fe36f30524c00ea79fdf954cc6779/psutil-7.1.2-cp314-cp314t-macosx_10_15_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp314-cp314t-macosx_11_0_arm64.whl","hashes":{"sha256":"fa6342cf859c48b19df3e4aa170e4cfb64aadc50b11e06bb569c6c777b089c9e"},"provenance":null,"requires-python":">=3.6","size":239303,"upload-time":"2025-10-25T10:46:56.932071Z","url":"https://files.pythonhosted.org/packages/32/ee/fd26216a735395cc25c3899634e34aeb41fb1f3dbb44acc67d9e594be562/psutil-7.1.2-cp314-cp314t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"625977443498ee7d6c1e63e93bacca893fd759a66c5f635d05e05811d23fb5ee"},"provenance":null,"requires-python":">=3.6","size":281717,"upload-time":"2025-10-25T10:46:59.116890Z","url":"https://files.pythonhosted.org/packages/3c/cd/7d96eaec4ef7742b845a9ce2759a2769ecce4ab7a99133da24abacbc9e41/psutil-7.1.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"data-dist-info-metadata":{"sha256":"e7b235bb0ba92037ddba6bcb227d0db55f64a7d54b2302a64ee7d8ba78b81c44"},"filename":"psutil-7.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"4a24bcd7b7f2918d934af0fb91859f621b873d6aa81267575e3655cd387572a7"},"provenance":null,"requires-python":">=3.6","size":284575,"upload-time":"2025-10-25T10:47:00.944625Z","url":"https://files.pythonhosted.org/packages/bc/1a/7f0b84bdb067d35fe7fade5fff888408688caf989806ce2d6dae08c72dd5/psutil-7.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp314-cp314t-win_amd64.whl","hashes":{"sha256":"329f05610da6380982e6078b9d0881d9ab1e9a7eb7c02d833bfb7340aa634e31"},"provenance":null,"requires-python":">=3.6","size":249491,"upload-time":"2025-10-25T10:47:03.174087Z","url":"https://files.pythonhosted.org/packages/de/05/7820ef8f7b275268917e0c750eada5834581206d9024ca88edce93c4b762/psutil-7.1.2-cp314-cp314t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp314-cp314t-win_arm64.whl","hashes":{"sha256":"7b04c29e3c0c888e83ed4762b70f31e65c42673ea956cefa8ced0e31e185f582"},"provenance":null,"requires-python":">=3.6","size":244880,"upload-time":"2025-10-25T10:47:05.228789Z","url":"https://files.pythonhosted.org/packages/db/9a/58de399c7cb58489f08498459ff096cd76b3f1ddc4f224ec2c5ef729c7d0/psutil-7.1.2-cp314-cp314t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"data-dist-info-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"filename":"psutil-7.1.2-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"c9ba5c19f2d46203ee8c152c7b01df6eec87d883cfd8ee1af2ef2727f6b0f814"},"provenance":null,"requires-python":">=3.6","size":237244,"upload-time":"2025-10-25T10:47:07.086631Z","url":"https://files.pythonhosted.org/packages/ae/89/b9f8d47ddbc52d7301fc868e8224e5f44ed3c7f55e6d0f54ecaf5dd9ff5e/psutil-7.1.2-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"data-dist-info-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"filename":"psutil-7.1.2-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"2a486030d2fe81bec023f703d3d155f4823a10a47c36784c84f1cc7f8d39bedb"},"provenance":null,"requires-python":">=3.6","size":238101,"upload-time":"2025-10-25T10:47:09.523680Z","url":"https://files.pythonhosted.org/packages/c8/7a/8628c2f6b240680a67d73d8742bb9ff39b1820a693740e43096d5dcb01e5/psutil-7.1.2-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"data-dist-info-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"filename":"psutil-7.1.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"3efd8fc791492e7808a51cb2b94889db7578bfaea22df931424f874468e389e3"},"provenance":null,"requires-python":">=3.6","size":258675,"upload-time":"2025-10-25T10:47:11.082823Z","url":"https://files.pythonhosted.org/packages/30/28/5e27f4d5a0e347f8e3cc16cd7d35533dbce086c95807f1f0e9cd77e26c10/psutil-7.1.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"data-dist-info-metadata":{"sha256":"9060d8b9832fb047375c7eeac0d3ca41668a9f56cea96f12e30ce72cc4acea8b"},"filename":"psutil-7.1.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"e2aeb9b64f481b8eabfc633bd39e0016d4d8bbcd590d984af764d80bf0851b8a"},"provenance":null,"requires-python":">=3.6","size":260203,"upload-time":"2025-10-25T10:47:13.226564Z","url":"https://files.pythonhosted.org/packages/e5/5c/79cf60c9acf36d087f0db0f82066fca4a780e97e5b3a2e4c38209c03d170/psutil-7.1.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"4e0958cd1551f981a3d0063faf8314b741cd97cb43d1e58934800cb4da537f64"},"data-dist-info-metadata":{"sha256":"4e0958cd1551f981a3d0063faf8314b741cd97cb43d1e58934800cb4da537f64"},"filename":"psutil-7.1.2-cp37-abi3-win_amd64.whl","hashes":{"sha256":"8e17852114c4e7996fe9da4745c2bdef001ebbf2f260dec406290e66628bdb91"},"provenance":null,"requires-python":">=3.6","size":246714,"upload-time":"2025-10-25T10:47:15.093571Z","url":"https://files.pythonhosted.org/packages/f7/03/0a464404c51685dcb9329fdd660b1721e076ccd7b3d97dee066bcc9ffb15/psutil-7.1.2-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"data-dist-info-metadata":{"sha256":"b27a63f31491a49efc9ff48050f5fdcca4606a487fac91e6cc46853031038b01"},"filename":"psutil-7.1.2-cp37-abi3-win_arm64.whl","hashes":{"sha256":"3e988455e61c240cc879cb62a008c2699231bf3e3d061d7fce4234463fd2abb4"},"provenance":null,"requires-python":">=3.6","size":243742,"upload-time":"2025-10-25T10:47:17.302139Z","url":"https://files.pythonhosted.org/packages/6a/32/97ca2090f2f1b45b01b6aa7ae161cfe50671de097311975ca6eea3e7aabc/psutil-7.1.2-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.1.2.tar.gz","hashes":{"sha256":"aa225cdde1335ff9684708ee8c72650f6598d5ed2114b9a7c5802030b1785018"},"provenance":null,"requires-python":">=3.6","size":487424,"upload-time":"2025-10-25T10:46:34.931002Z","url":"https://files.pythonhosted.org/packages/cd/ec/7b8e6b9b1d22708138630ef34c53ab2b61032c04f16adfdbb96791c8c70c/psutil-7.1.2.tar.gz","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp313-cp313t-macosx_10_13_x86_64.whl","hashes":{"sha256":"0005da714eee687b4b8decd3d6cc7c6db36215c9e74e5ad2264b90c3df7d92dc"},"provenance":null,"requires-python":">=3.6","size":239751,"upload-time":"2025-11-02T12:25:58.161404Z","url":"https://files.pythonhosted.org/packages/bd/93/0c49e776b8734fef56ec9c5c57f923922f2cf0497d62e0f419465f28f3d0/psutil-7.1.3-cp313-cp313t-macosx_10_13_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp313-cp313t-macosx_11_0_arm64.whl","hashes":{"sha256":"19644c85dcb987e35eeeaefdc3915d059dac7bd1167cdcdbf27e0ce2df0c08c0"},"provenance":null,"requires-python":">=3.6","size":240368,"upload-time":"2025-11-02T12:26:00.491685Z","url":"https://files.pythonhosted.org/packages/6f/8d/b31e39c769e70780f007969815195a55c81a63efebdd4dbe9e7a113adb2f/psutil-7.1.3-cp313-cp313t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"95ef04cf2e5ba0ab9eaafc4a11eaae91b44f4ef5541acd2ee91d9108d00d59a7"},"provenance":null,"requires-python":">=3.6","size":287134,"upload-time":"2025-11-02T12:26:02.613574Z","url":"https://files.pythonhosted.org/packages/62/61/23fd4acc3c9eebbf6b6c78bcd89e5d020cfde4acf0a9233e9d4e3fa698b4/psutil-7.1.3-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"1068c303be3a72f8e18e412c5b2a8f6d31750fb152f9cb106b54090296c9d251"},"provenance":null,"requires-python":">=3.6","size":289904,"upload-time":"2025-11-02T12:26:05.207933Z","url":"https://files.pythonhosted.org/packages/30/1c/f921a009ea9ceb51aa355cb0cc118f68d354db36eae18174bab63affb3e6/psutil-7.1.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp313-cp313t-win_amd64.whl","hashes":{"sha256":"18349c5c24b06ac5612c0428ec2a0331c26443d259e2a0144a9b24b4395b58fa"},"provenance":null,"requires-python":">=3.6","size":249642,"upload-time":"2025-11-02T12:26:07.447774Z","url":"https://files.pythonhosted.org/packages/a6/82/62d68066e13e46a5116df187d319d1724b3f437ddd0f958756fc052677f4/psutil-7.1.3-cp313-cp313t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp313-cp313t-win_arm64.whl","hashes":{"sha256":"c525ffa774fe4496282fb0b1187725793de3e7c6b29e41562733cae9ada151ee"},"provenance":null,"requires-python":">=3.6","size":245518,"upload-time":"2025-11-02T12:26:09.719155Z","url":"https://files.pythonhosted.org/packages/df/ad/c1cd5fe965c14a0392112f68362cfceb5230819dbb5b1888950d18a11d9f/psutil-7.1.3-cp313-cp313t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp314-cp314t-macosx_10_15_x86_64.whl","hashes":{"sha256":"b403da1df4d6d43973dc004d19cee3b848e998ae3154cc8097d139b77156c353"},"provenance":null,"requires-python":">=3.6","size":239843,"upload-time":"2025-11-02T12:26:11.968073Z","url":"https://files.pythonhosted.org/packages/2e/bb/6670bded3e3236eb4287c7bcdc167e9fae6e1e9286e437f7111caed2f909/psutil-7.1.3-cp314-cp314t-macosx_10_15_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp314-cp314t-macosx_11_0_arm64.whl","hashes":{"sha256":"ad81425efc5e75da3f39b3e636293360ad8d0b49bed7df824c79764fb4ba9b8b"},"provenance":null,"requires-python":">=3.6","size":240369,"upload-time":"2025-11-02T12:26:14.358801Z","url":"https://files.pythonhosted.org/packages/b8/66/853d50e75a38c9a7370ddbeefabdd3d3116b9c31ef94dc92c6729bc36bec/psutil-7.1.3-cp314-cp314t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"8f33a3702e167783a9213db10ad29650ebf383946e91bc77f28a5eb083496bc9"},"provenance":null,"requires-python":">=3.6","size":288210,"upload-time":"2025-11-02T12:26:16.699739Z","url":"https://files.pythonhosted.org/packages/41/bd/313aba97cb5bfb26916dc29cf0646cbe4dd6a89ca69e8c6edce654876d39/psutil-7.1.3-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"data-dist-info-metadata":{"sha256":"fee2408ed83d5d231935e659a072f72e41c592bc880496074a506e9d6b3eada7"},"filename":"psutil-7.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"fac9cd332c67f4422504297889da5ab7e05fd11e3c4392140f7370f4208ded1f"},"provenance":null,"requires-python":">=3.6","size":291182,"upload-time":"2025-11-02T12:26:18.848963Z","url":"https://files.pythonhosted.org/packages/c2/fa/76e3c06e760927a0cfb5705eb38164254de34e9bd86db656d4dbaa228b04/psutil-7.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp314-cp314t-win_amd64.whl","hashes":{"sha256":"3792983e23b69843aea49c8f5b8f115572c5ab64c153bada5270086a2123c7e7"},"provenance":null,"requires-python":">=3.6","size":250466,"upload-time":"2025-11-02T12:26:21.183069Z","url":"https://files.pythonhosted.org/packages/0f/1d/5774a91607035ee5078b8fd747686ebec28a962f178712de100d00b78a32/psutil-7.1.3-cp314-cp314t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp314-cp314t-win_arm64.whl","hashes":{"sha256":"31d77fcedb7529f27bb3a0472bea9334349f9a04160e8e6e5020f22c59893264"},"provenance":null,"requires-python":">=3.6","size":245756,"upload-time":"2025-11-02T12:26:23.148427Z","url":"https://files.pythonhosted.org/packages/00/ca/e426584bacb43a5cb1ac91fae1937f478cd8fbe5e4ff96574e698a2c77cd/psutil-7.1.3-cp314-cp314t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"data-dist-info-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"filename":"psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"2bdbcd0e58ca14996a42adf3621a6244f1bb2e2e528886959c72cf1e326677ab"},"provenance":null,"requires-python":">=3.6","size":238359,"upload-time":"2025-11-02T12:26:25.284599Z","url":"https://files.pythonhosted.org/packages/ef/94/46b9154a800253e7ecff5aaacdf8ebf43db99de4a2dfa18575b02548654e/psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"data-dist-info-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"filename":"psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"bc31fa00f1fbc3c3802141eede66f3a2d51d89716a194bf2cd6fc68310a19880"},"provenance":null,"requires-python":">=3.6","size":239171,"upload-time":"2025-11-02T12:26:27.230751Z","url":"https://files.pythonhosted.org/packages/68/3a/9f93cff5c025029a36d9a92fef47220ab4692ee7f2be0fba9f92813d0cb8/psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"data-dist-info-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"filename":"psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"3bb428f9f05c1225a558f53e30ccbad9930b11c3fc206836242de1091d3e7dd3"},"provenance":null,"requires-python":">=3.6","size":263261,"upload-time":"2025-11-02T12:26:29.480632Z","url":"https://files.pythonhosted.org/packages/ce/b1/5f49af514f76431ba4eea935b8ad3725cdeb397e9245ab919dbc1d1dc20f/psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"data-dist-info-metadata":{"sha256":"3578dc4ee5f43e542d29839741b9e3d8760c3c0e5d8c388ec2b0b980bd8c66a1"},"filename":"psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"56d974e02ca2c8eb4812c3f76c30e28836fffc311d55d979f1465c1feeb2b68b"},"provenance":null,"requires-python":">=3.6","size":264635,"upload-time":"2025-11-02T12:26:31.740762Z","url":"https://files.pythonhosted.org/packages/e0/95/992c8816a74016eb095e73585d747e0a8ea21a061ed3689474fabb29a395/psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"89db7acafbe66cd8b6a060ae993afda9c8f50aa4342c34c92a284be2c8c8534f"},"data-dist-info-metadata":{"sha256":"89db7acafbe66cd8b6a060ae993afda9c8f50aa4342c34c92a284be2c8c8534f"},"filename":"psutil-7.1.3-cp37-abi3-win_amd64.whl","hashes":{"sha256":"f39c2c19fe824b47484b96f9692932248a54c43799a84282cfe58d05a6449efd"},"provenance":null,"requires-python":">=3.6","size":247633,"upload-time":"2025-11-02T12:26:33.887174Z","url":"https://files.pythonhosted.org/packages/55/4c/c3ed1a622b6ae2fd3c945a366e64eb35247a31e4db16cf5095e269e8eb3c/psutil-7.1.3-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"data-dist-info-metadata":{"sha256":"a690b24adf2968fc63d76d10c2e8052107d0af31da1588424d365c1ff08c9fd1"},"filename":"psutil-7.1.3-cp37-abi3-win_arm64.whl","hashes":{"sha256":"bd0d69cee829226a761e92f28140bec9a5ee9d5b4fb4b0cc589068dbfff559b1"},"provenance":null,"requires-python":">=3.6","size":244608,"upload-time":"2025-11-02T12:26:36.136434Z","url":"https://files.pythonhosted.org/packages/c9/ad/33b2ccec09bf96c2b2ef3f9a6f66baac8253d7565d8839e024a6b905d45d/psutil-7.1.3-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.1.3.tar.gz","hashes":{"sha256":"6c86281738d77335af7aec228328e944b30930899ea760ecf33a4dba66be5e74"},"provenance":null,"requires-python":">=3.6","size":489059,"upload-time":"2025-11-02T12:25:54.619294Z","url":"https://files.pythonhosted.org/packages/e1/88/bdd0a41e5857d5d703287598cbf08dad90aed56774ea52ae071bae9071b6/psutil-7.1.3.tar.gz","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp313-cp313t-macosx_10_13_x86_64.whl","hashes":{"sha256":"c31e927555539132a00380c971816ea43d089bf4bd5f3e918ed8c16776d68474"},"provenance":null,"requires-python":">=3.6","size":129593,"upload-time":"2025-12-23T20:26:28.019569Z","url":"https://files.pythonhosted.org/packages/a8/8e/b35aae6ed19bc4e2286cac4832e4d522fcf00571867b0a85a3f77ef96a80/psutil-7.2.0-cp313-cp313t-macosx_10_13_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp313-cp313t-macosx_11_0_arm64.whl","hashes":{"sha256":"db8e44e766cef86dea47d9a1fa535d38dc76449e5878a92f33683b7dba5bfcb2"},"provenance":null,"requires-python":">=3.6","size":130104,"upload-time":"2025-12-23T20:26:30.270800Z","url":"https://files.pythonhosted.org/packages/61/a2/773d17d74e122bbffe08b97f73f2d4a01ef53fb03b98e61b8e4f64a9c6b9/psutil-7.2.0-cp313-cp313t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"85ef849ac92169dedc59a7ac2fb565f47b3468fbe1524bf748746bc21afb94c7"},"provenance":null,"requires-python":">=3.6","size":180579,"upload-time":"2025-12-23T20:26:32.628357Z","url":"https://files.pythonhosted.org/packages/0d/e3/d3a9b3f4bd231abbd70a988beb2e3edd15306051bccbfc4472bd34a56e01/psutil-7.2.0-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"26782bdbae2f5c14ce9ebe8ad2411dc2ca870495e0cd90f8910ede7fa5e27117"},"provenance":null,"requires-python":">=3.6","size":183171,"upload-time":"2025-12-23T20:26:34.972726Z","url":"https://files.pythonhosted.org/packages/66/f8/6c73044424aabe1b7824d4d4504029d406648286d8fe7ba8c4682e0d3042/psutil-7.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp313-cp313t-win_amd64.whl","hashes":{"sha256":"b7665f612d3b38a583391b95969667a53aaf6c5706dc27a602c9a4874fbf09e4"},"provenance":null,"requires-python":">=3.6","size":139055,"upload-time":"2025-12-23T20:26:36.848832Z","url":"https://files.pythonhosted.org/packages/48/7d/76d7a863340885d41826562225a566683e653ee6c9ba03c9f3856afa7d80/psutil-7.2.0-cp313-cp313t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp313-cp313t-win_arm64.whl","hashes":{"sha256":"4413373c174520ae28a24a8974ad8ce6b21f060d27dde94e25f8c73a7effe57a"},"provenance":null,"requires-python":">=3.6","size":134737,"upload-time":"2025-12-23T20:26:38.784066Z","url":"https://files.pythonhosted.org/packages/a0/48/200054ada0ae4872c8a71db54f3eb6a9af4101680ee6830d373b7fda526b/psutil-7.2.0-cp313-cp313t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp314-cp314t-macosx_10_15_x86_64.whl","hashes":{"sha256":"2f2f53fd114e7946dfba3afb98c9b7c7f376009447360ca15bfb73f2066f84c7"},"provenance":null,"requires-python":">=3.6","size":129692,"upload-time":"2025-12-23T20:26:40.623176Z","url":"https://files.pythonhosted.org/packages/44/86/98da45dff471b93ef5ce5bcaefa00e3038295a7880a77cf74018243d37fb/psutil-7.2.0-cp314-cp314t-macosx_10_15_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp314-cp314t-macosx_11_0_arm64.whl","hashes":{"sha256":"e65c41d7e60068f60ce43b31a3a7fc90deb0dfd34ffc824a2574c2e5279b377e"},"provenance":null,"requires-python":">=3.6","size":130110,"upload-time":"2025-12-23T20:26:42.569228Z","url":"https://files.pythonhosted.org/packages/50/ee/10eae91ba4ad071c92db3c178ba861f30406342de9f0ddbe6d51fd741236/psutil-7.2.0-cp314-cp314t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"cc66d21366850a4261412ce994ae9976bba9852dafb4f2fa60db68ed17ff5281"},"provenance":null,"requires-python":">=3.6","size":181487,"upload-time":"2025-12-23T20:26:44.633020Z","url":"https://files.pythonhosted.org/packages/87/3a/2b2897443d56fedbbc34ac68a0dc7d55faa05d555372a2f989109052f86d/psutil-7.2.0-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"data-dist-info-metadata":{"sha256":"2170dd4480c057291a293d121c10bf5228540dd58ab43945d29672ec02a0a23a"},"filename":"psutil-7.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"e025d67b42b8f22b096d5d20f5171de0e0fefb2f0ce983a13c5a1b5ed9872706"},"provenance":null,"requires-python":">=3.6","size":184320,"upload-time":"2025-12-23T20:26:46.830615Z","url":"https://files.pythonhosted.org/packages/11/66/44308428f7333db42c5ea7390c52af1b38f59b80b80c437291f58b5dfdad/psutil-7.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp314-cp314t-win_amd64.whl","hashes":{"sha256":"45f6b91f7ad63414d6454fd609e5e3556d0e1038d5d9c75a1368513bdf763f57"},"provenance":null,"requires-python":">=3.6","size":140372,"upload-time":"2025-12-23T20:26:49.334377Z","url":"https://files.pythonhosted.org/packages/18/28/d2feadc7f18e501c5ce687c377db7dca924585418fd694272b8e488ea99f/psutil-7.2.0-cp314-cp314t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp314-cp314t-win_arm64.whl","hashes":{"sha256":"87b18a19574139d60a546e88b5f5b9cbad598e26cdc790d204ab95d7024f03ee"},"provenance":null,"requires-python":">=3.6","size":135400,"upload-time":"2025-12-23T20:26:51.585604Z","url":"https://files.pythonhosted.org/packages/b2/1d/48381f5fd0425aa054c4ee3de24f50de3d6c347019f3aec75f357377d447/psutil-7.2.0-cp314-cp314t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"977a2fcd132d15cb05b32b2d85b98d087cad039b0ce435731670ba74da9e6133"},"provenance":null,"requires-python":">=3.6","size":128116,"upload-time":"2025-12-23T20:26:53.516520Z","url":"https://files.pythonhosted.org/packages/40/c5/a49160bf3e165b7b93a60579a353cf5d939d7f878fe5fd369110f1d18043/psutil-7.2.0-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"24151011c21fadd94214d7139d7c6c54569290d7e553989bdf0eab73b13beb8c"},"provenance":null,"requires-python":">=3.6","size":128925,"upload-time":"2025-12-23T20:26:55.573324Z","url":"https://files.pythonhosted.org/packages/10/a1/c75feb480f60cd768fb6ed00ac362a16a33e5076ec8475a22d8162fb2659/psutil-7.2.0-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"91f211ba9279e7c61d9d8f84b713cfc38fa161cb0597d5cb3f1ca742f6848254"},"provenance":null,"requires-python":">=3.6","size":154666,"upload-time":"2025-12-23T20:26:57.312776Z","url":"https://files.pythonhosted.org/packages/12/ff/e93136587c00a543f4bc768b157fac2c47cd77b180d4f4e5c6efb6ea53a2/psutil-7.2.0-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"f37415188b7ea98faf90fed51131181646c59098b077550246e2e092e127418b"},"provenance":null,"requires-python":">=3.6","size":156109,"upload-time":"2025-12-23T20:26:58.851353Z","url":"https://files.pythonhosted.org/packages/b8/dd/4c2de9c3827c892599d277a69d2224136800870a8a88a80981de905de28d/psutil-7.2.0-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-musllinux_1_2_aarch64.whl","hashes":{"sha256":"0d12c7ce6ed1128cd81fd54606afa054ac7dbb9773469ebb58cf2f171c49f2ac"},"provenance":null,"requires-python":">=3.6","size":148081,"upload-time":"2025-12-23T20:27:01.318270Z","url":"https://files.pythonhosted.org/packages/81/3f/090943c682d3629968dd0b04826ddcbc760ee1379021dbe316e2ddfcd01b/psutil-7.2.0-cp36-abi3-musllinux_1_2_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"data-dist-info-metadata":{"sha256":"78cbedc39a964b930e33a3c9bd2f10aaa72c827cfcce30c4074dffd544c24a58"},"filename":"psutil-7.2.0-cp36-abi3-musllinux_1_2_x86_64.whl","hashes":{"sha256":"ca0faef7976530940dcd39bc5382d0d0d5eb023b186a4901ca341bd8d8684151"},"provenance":null,"requires-python":">=3.6","size":147376,"upload-time":"2025-12-23T20:27:03.347816Z","url":"https://files.pythonhosted.org/packages/c4/88/c39648ebb8ec182d0364af53cdefe6eddb5f3872ba718b5855a8ff65d6d4/psutil-7.2.0-cp36-abi3-musllinux_1_2_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"758019d458979edff64acde45968217765d5828ba125593a164a985cdc0754bd"},"data-dist-info-metadata":{"sha256":"758019d458979edff64acde45968217765d5828ba125593a164a985cdc0754bd"},"filename":"psutil-7.2.0-cp37-abi3-win_amd64.whl","hashes":{"sha256":"abdb74137ca232d20250e9ad471f58d500e7743bc8253ba0bfbf26e570c0e437"},"provenance":null,"requires-python":">=3.6","size":136910,"upload-time":"2025-12-23T20:27:05.289344Z","url":"https://files.pythonhosted.org/packages/01/a2/5b39e08bd9b27476bc7cce7e21c71a481ad60b81ffac49baf02687a50d7f/psutil-7.2.0-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"data-dist-info-metadata":{"sha256":"795be0019027992887b63578f61759602599118117e81710fb28c5b7fb5023d6"},"filename":"psutil-7.2.0-cp37-abi3-win_arm64.whl","hashes":{"sha256":"284e71038b3139e7ab3834b63b3eb5aa5565fcd61a681ec746ef9a0a8c457fd2"},"provenance":null,"requires-python":">=3.6","size":133807,"upload-time":"2025-12-23T20:27:06.825130Z","url":"https://files.pythonhosted.org/packages/59/54/53839db1258c1eaeb4ded57ff202144ebc75b23facc05a74fd98d338b0c6/psutil-7.2.0-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.2.0.tar.gz","hashes":{"sha256":"2e4f8e1552f77d14dc96fb0f6240c5b34a37081c0889f0853b3b29a496e5ef64"},"provenance":null,"requires-python":">=3.6","size":489863,"upload-time":"2025-12-23T20:26:24.616214Z","url":"https://files.pythonhosted.org/packages/be/7c/31d1c3ceb1260301f87565f50689dc6da3db427ece1e1e012af22abca54e/psutil-7.2.0.tar.gz","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp313-cp313t-macosx_10_13_x86_64.whl","hashes":{"sha256":"ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d"},"provenance":null,"requires-python":">=3.6","size":129624,"upload-time":"2025-12-29T08:26:04.255517Z","url":"https://files.pythonhosted.org/packages/77/8e/f0c242053a368c2aa89584ecd1b054a18683f13d6e5a318fc9ec36582c94/psutil-7.2.1-cp313-cp313t-macosx_10_13_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp313-cp313t-macosx_11_0_arm64.whl","hashes":{"sha256":"81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49"},"provenance":null,"requires-python":">=3.6","size":130132,"upload-time":"2025-12-29T08:26:06.228150Z","url":"https://files.pythonhosted.org/packages/26/97/a58a4968f8990617decee234258a2b4fc7cd9e35668387646c1963e69f26/psutil-7.2.1-cp313-cp313t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc"},"provenance":null,"requires-python":">=3.6","size":180612,"upload-time":"2025-12-29T08:26:08.276538Z","url":"https://files.pythonhosted.org/packages/db/6d/ed44901e830739af5f72a85fa7ec5ff1edea7f81bfbf4875e409007149bd/psutil-7.2.1-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf"},"provenance":null,"requires-python":">=3.6","size":183201,"upload-time":"2025-12-29T08:26:10.622093Z","url":"https://files.pythonhosted.org/packages/c7/65/b628f8459bca4efbfae50d4bf3feaab803de9a160b9d5f3bd9295a33f0c2/psutil-7.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp313-cp313t-win_amd64.whl","hashes":{"sha256":"923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f"},"provenance":null,"requires-python":">=3.6","size":139081,"upload-time":"2025-12-29T08:26:12.483257Z","url":"https://files.pythonhosted.org/packages/fb/23/851cadc9764edcc18f0effe7d0bf69f727d4cf2442deb4a9f78d4e4f30f2/psutil-7.2.1-cp313-cp313t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp313-cp313t-win_arm64.whl","hashes":{"sha256":"cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672"},"provenance":null,"requires-python":">=3.6","size":134767,"upload-time":"2025-12-29T08:26:14.528248Z","url":"https://files.pythonhosted.org/packages/59/82/d63e8494ec5758029f31c6cb06d7d161175d8281e91d011a4a441c8a43b5/psutil-7.2.1-cp313-cp313t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp314-cp314t-macosx_10_15_x86_64.whl","hashes":{"sha256":"494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679"},"provenance":null,"requires-python":">=3.6","size":129716,"upload-time":"2025-12-29T08:26:16.017500Z","url":"https://files.pythonhosted.org/packages/05/c2/5fb764bd61e40e1fe756a44bd4c21827228394c17414ade348e28f83cd79/psutil-7.2.1-cp314-cp314t-macosx_10_15_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp314-cp314t-macosx_11_0_arm64.whl","hashes":{"sha256":"3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f"},"provenance":null,"requires-python":">=3.6","size":130133,"upload-time":"2025-12-29T08:26:18.009017Z","url":"https://files.pythonhosted.org/packages/c9/d2/935039c20e06f615d9ca6ca0ab756cf8408a19d298ffaa08666bc18dc805/psutil-7.2.1-cp314-cp314t-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129"},"provenance":null,"requires-python":">=3.6","size":181518,"upload-time":"2025-12-29T08:26:20.241501Z","url":"https://files.pythonhosted.org/packages/77/69/19f1eb0e01d24c2b3eacbc2f78d3b5add8a89bf0bb69465bc8d563cc33de/psutil-7.2.1-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"data-dist-info-metadata":{"sha256":"23740d4f8fcd640f5d191b7b232f6a393e1df6bf023302f619c701f995dff02d"},"filename":"psutil-7.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a"},"provenance":null,"requires-python":">=3.6","size":184348,"upload-time":"2025-12-29T08:26:22.215886Z","url":"https://files.pythonhosted.org/packages/e1/6d/7e18b1b4fa13ad370787626c95887b027656ad4829c156bb6569d02f3262/psutil-7.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp314-cp314t-win_amd64.whl","hashes":{"sha256":"2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79"},"provenance":null,"requires-python":">=3.6","size":140400,"upload-time":"2025-12-29T08:26:23.993885Z","url":"https://files.pythonhosted.org/packages/98/60/1672114392dd879586d60dd97896325df47d9a130ac7401318005aab28ec/psutil-7.2.1-cp314-cp314t-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp314-cp314t-win_arm64.whl","hashes":{"sha256":"08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266"},"provenance":null,"requires-python":">=3.6","size":135430,"upload-time":"2025-12-29T08:26:25.999852Z","url":"https://files.pythonhosted.org/packages/fb/7b/d0e9d4513c46e46897b46bcfc410d51fc65735837ea57a25170f298326e6/psutil-7.2.1-cp314-cp314t-win_arm64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-macosx_10_9_x86_64.whl","hashes":{"sha256":"b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42"},"provenance":null,"requires-python":">=3.6","size":128137,"upload-time":"2025-12-29T08:26:27.759659Z","url":"https://files.pythonhosted.org/packages/c5/cf/5180eb8c8bdf6a503c6919f1da28328bd1e6b3b1b5b9d5b01ae64f019616/psutil-7.2.1-cp36-abi3-macosx_10_9_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl","hashes":{"sha256":"05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1"},"provenance":null,"requires-python":">=3.6","size":128947,"upload-time":"2025-12-29T08:26:29.548034Z","url":"https://files.pythonhosted.org/packages/c5/2c/78e4a789306a92ade5000da4f5de3255202c534acdadc3aac7b5458fadef/psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","hashes":{"sha256":"5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8"},"provenance":null,"requires-python":">=3.6","size":154694,"upload-time":"2025-12-29T08:26:32.147774Z","url":"https://files.pythonhosted.org/packages/29/f8/40e01c350ad9a2b3cb4e6adbcc8a83b17ee50dd5792102b6142385937db5/psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","hashes":{"sha256":"ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6"},"provenance":null,"requires-python":">=3.6","size":156136,"upload-time":"2025-12-29T08:26:34.079911Z","url":"https://files.pythonhosted.org/packages/06/e4/b751cdf839c011a9714a783f120e6a86b7494eb70044d7d81a25a5cd295f/psutil-7.2.1-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-musllinux_1_2_aarch64.whl","hashes":{"sha256":"f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8"},"provenance":null,"requires-python":">=3.6","size":148108,"upload-time":"2025-12-29T08:26:36.225796Z","url":"https://files.pythonhosted.org/packages/44/ad/bbf6595a8134ee1e94a4487af3f132cef7fce43aef4a93b49912a48c3af7/psutil-7.2.1-cp36-abi3-musllinux_1_2_aarch64.whl","yanked":false},{"core-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"data-dist-info-metadata":{"sha256":"c340b0dbe2145d45dac650326d0f34f3ee664206d3bcd32785393b3476525944"},"filename":"psutil-7.2.1-cp36-abi3-musllinux_1_2_x86_64.whl","hashes":{"sha256":"99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67"},"provenance":null,"requires-python":">=3.6","size":147402,"upload-time":"2025-12-29T08:26:39.210698Z","url":"https://files.pythonhosted.org/packages/1c/15/dd6fd869753ce82ff64dcbc18356093471a5a5adf4f77ed1f805d473d859/psutil-7.2.1-cp36-abi3-musllinux_1_2_x86_64.whl","yanked":false},{"core-metadata":{"sha256":"54955c981ab8a8b2b87efc900b6916c908a315660b0c3c5d205c910565bb3261"},"data-dist-info-metadata":{"sha256":"54955c981ab8a8b2b87efc900b6916c908a315660b0c3c5d205c910565bb3261"},"filename":"psutil-7.2.1-cp37-abi3-win_amd64.whl","hashes":{"sha256":"b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17"},"provenance":null,"requires-python":">=3.6","size":136938,"upload-time":"2025-12-29T08:26:41.036253Z","url":"https://files.pythonhosted.org/packages/34/68/d9317542e3f2b180c4306e3f45d3c922d7e86d8ce39f941bb9e2e9d8599e/psutil-7.2.1-cp37-abi3-win_amd64.whl","yanked":false},{"core-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"data-dist-info-metadata":{"sha256":"08f1da19e657e59de5e9a94e4295fa686697c5c4e76447bd048138c2227e8293"},"filename":"psutil-7.2.1-cp37-abi3-win_arm64.whl","hashes":{"sha256":"0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442"},"provenance":null,"requires-python":">=3.6","size":133836,"upload-time":"2025-12-29T08:26:43.086974Z","url":"https://files.pythonhosted.org/packages/3e/73/2ce007f4198c80fcf2cb24c169884f833fe93fbc03d55d302627b094ee91/psutil-7.2.1-cp37-abi3-win_arm64.whl","yanked":false},{"core-metadata":false,"data-dist-info-metadata":false,"filename":"psutil-7.2.1.tar.gz","hashes":{"sha256":"f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3"},"provenance":null,"requires-python":">=3.6","size":490253,"upload-time":"2025-12-29T08:26:00.169622Z","url":"https://files.pythonhosted.org/packages/73/cb/09e5184fb5fc0358d110fc3ca7f6b1d033800734d34cac10f4136cfac10e/psutil-7.2.1.tar.gz","yanked":false}],"meta":{"_last-serial":33241970,"api-version":"1.4"},"name":"psutil","project-status":{"status":"active"},"versions":["0.1.1","0.1.2","0.1.3","0.2.0","0.2.1","0.3.0","0.4.0","0.4.1","0.5.0","0.5.1","0.6.0","0.6.1","0.7.0","0.7.1","1.0.0","1.0.1","1.1.0","1.1.1","1.1.2","1.1.3","1.2.0","1.2.1","2.0.0","2.1.0","2.1.1","2.1.2","2.1.3","2.2.0","2.2.1","3.0.0","3.0.1","3.1.0","3.1.1","3.2.0","3.2.1","3.2.2","3.3.0","3.4.1","3.4.2","4.0.0","4.1.0","4.2.0","4.3.0","4.3.1","4.4.0","4.4.1","4.4.2","5.0.0","5.0.1","5.1.0","5.1.1","5.1.2","5.1.3","5.2.0","5.2.1","5.2.2","5.3.0","5.3.1","5.4.0","5.4.1","5.4.2","5.4.3","5.4.4","5.4.5","5.4.6","5.4.7","5.4.8","5.5.0","5.5.1","5.6.0","5.6.1","5.6.2","5.6.3","5.6.4","5.6.5","5.6.6","5.6.7","5.7.0","5.7.1","5.7.2","5.7.3","5.8.0","5.9.0","5.9.1","5.9.2","5.9.3","5.9.4","5.9.5","5.9.6","5.9.7","5.9.8","6.0.0","6.1.0","6.1.1","7.0.0","7.1.0","7.1.1","7.1.2","7.1.3","7.2.0","7.2.1"]} diff --git a/benches/benchmarks/json_loads.py b/benches/benchmarks/json_loads.py new file mode 100644 index 00000000000..f67f14d2d9e --- /dev/null +++ b/benches/benchmarks/json_loads.py @@ -0,0 +1,7 @@ +import json + +with open('benches/_data/pypi_org__simple__psutil.json') as f: + data = f.read() + + +loaded = json.loads(data) diff --git a/benches/benchmarks/pystone.py b/benches/benchmarks/pystone.py index 3faf675ae73..755b4ba85ca 100644 --- a/benches/benchmarks/pystone.py +++ b/benches/benchmarks/pystone.py @@ -16,7 +16,7 @@ Version History: - Inofficial version 1.1.1 by Chris Arndt: + Unofficial version 1.1.1 by Chris Arndt: - Make it run under Python 2 and 3 by using "from __future__ import print_function". diff --git a/benches/execution.rs b/benches/execution.rs index 14fadfc2a58..2816f0e8201 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -1,30 +1,32 @@ -use criterion::measurement::WallTime; use criterion::{ - criterion_group, criterion_main, Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, + Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, + measurement::WallTime, }; use rustpython_compiler::Mode; -use rustpython_parser::ast; -use rustpython_parser::Parse; -use rustpython_vm::{Interpreter, PyResult}; -use std::collections::HashMap; -use std::path::Path; +use rustpython_vm::{Interpreter, PyResult, Settings}; +use std::{collections::HashMap, hint::black_box, path::Path}; fn bench_cpython_code(b: &mut Bencher, source: &str) { - let gil = cpython::Python::acquire_gil(); - let python = gil.python(); - - b.iter(|| { - let res: cpython::PyResult<()> = python.run(source, None, None); - if let Err(e) = res { - e.print(python); - panic!("Error running source") - } - }); + let c_str_source_head = std::ffi::CString::new(source).unwrap(); + let c_str_source = c_str_source_head.as_c_str(); + pyo3::Python::attach(|py| { + b.iter(|| { + let module = pyo3::types::PyModule::from_code(py, c_str_source, c"", c"") + .expect("Error running source"); + black_box(module); + }) + }) } -fn bench_rustpy_code(b: &mut Bencher, name: &str, source: &str) { +fn bench_rustpython_code(b: &mut Bencher, name: &str, source: &str) { // NOTE: Take long time. - Interpreter::without_stdlib(Default::default()).enter(|vm| { + let mut settings = Settings::default(); + settings.path_list.push("Lib/".to_string()); + settings.write_bytecode = false; + settings.user_site_directory = false; + let builder = Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + builder.add_native_modules(&defs).build().enter(|vm| { // Note: bench_cpython is both compiling and executing the code. // As such we compile the code in the benchmark loop as well. b.iter(|| { @@ -36,67 +38,40 @@ fn bench_rustpy_code(b: &mut Bencher, name: &str, source: &str) { }) } -pub fn benchmark_file_execution( - group: &mut BenchmarkGroup<WallTime>, - name: &str, - contents: &String, -) { +pub fn benchmark_file_execution(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &str) { group.bench_function(BenchmarkId::new(name, "cpython"), |b| { - bench_cpython_code(b, &contents) + bench_cpython_code(b, contents) }); group.bench_function(BenchmarkId::new(name, "rustpython"), |b| { - bench_rustpy_code(b, name, &contents) + bench_rustpython_code(b, name, contents) }); } pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &str) { group.throughput(Throughput::Bytes(contents.len() as u64)); group.bench_function(BenchmarkId::new("rustpython", name), |b| { - b.iter(|| ast::Suite::parse(contents, name).unwrap()) + b.iter(|| ruff_python_parser::parse_module(contents).unwrap()) }); group.bench_function(BenchmarkId::new("cpython", name), |b| { - let gil = cpython::Python::acquire_gil(); - let py = gil.python(); - - let code = std::ffi::CString::new(contents).unwrap(); - let fname = cpython::PyString::new(py, name); - - b.iter(|| parse_program_cpython(py, &code, &fname)) + use pyo3::types::PyAnyMethods; + pyo3::Python::attach(|py| { + let builtins = + pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); + let compile = builtins.getattr("compile").expect("no compile in builtins"); + b.iter(|| { + let x = compile + .call1((contents, name, "exec")) + .expect("Failed to parse code"); + black_box(x); + }) + }) }); } -fn parse_program_cpython( - py: cpython::Python<'_>, - code: &std::ffi::CStr, - fname: &cpython::PyString, -) { - extern "C" { - fn PyArena_New() -> *mut python3_sys::PyArena; - fn PyArena_Free(arena: *mut python3_sys::PyArena); - } - use cpython::PythonObject; - let fname = fname.as_object(); - unsafe { - let arena = PyArena_New(); - assert!(!arena.is_null()); - let ret = python3_sys::PyParser_ASTFromStringObject( - code.as_ptr() as _, - fname.as_ptr(), - python3_sys::Py_file_input, - std::ptr::null_mut(), - arena, - ); - if ret.is_null() { - cpython::PyErr::fetch(py).print(py); - } - PyArena_Free(arena); - } -} - pub fn benchmark_pystone(group: &mut BenchmarkGroup<WallTime>, contents: String) { // Default is 50_000. This takes a while, so reduce it to 30k. for idx in (10_000..=30_000).step_by(10_000) { - let code_with_loops = format!("LOOPS = {}\n{}", idx, contents); + let code_with_loops = format!("LOOPS = {idx}\n{contents}"); let code_str = code_with_loops.as_str(); group.throughput(Throughput::Elements(idx as u64)); @@ -104,7 +79,7 @@ pub fn benchmark_pystone(group: &mut BenchmarkGroup<WallTime>, contents: String) bench_cpython_code(b, code_str) }); group.bench_function(BenchmarkId::new("rustpython", idx), |b| { - bench_rustpy_code(b, "pystone", code_str) + bench_rustpython_code(b, "pystone", code_str) }); } } diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index c30d86722af..3fef2f8dd7e 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -1,11 +1,12 @@ use criterion::{ - criterion_group, criterion_main, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, - Criterion, Throughput, + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, + measurement::WallTime, }; +use pyo3::types::PyAnyMethods; use rustpython_compiler::Mode; use rustpython_vm::{AsObject, Interpreter, PyResult, Settings}; use std::{ - ffi, fs, io, + fs, io, path::{Path, PathBuf}, }; @@ -36,109 +37,86 @@ pub struct MicroBenchmark { } fn bench_cpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchmark) { - let gil = cpython::Python::acquire_gil(); - let py = gil.python(); - - let setup_code = ffi::CString::new(&*bench.setup).unwrap(); - let setup_name = ffi::CString::new(format!("{}_setup", bench.name)).unwrap(); - let setup_code = cpy_compile_code(py, &setup_code, &setup_name).unwrap(); - - let code = ffi::CString::new(&*bench.code).unwrap(); - let name = ffi::CString::new(&*bench.name).unwrap(); - let code = cpy_compile_code(py, &code, &name).unwrap(); - - let bench_func = |(globals, locals): &mut (cpython::PyDict, cpython::PyDict)| { - let res = cpy_run_code(py, &code, globals, locals); - if let Err(e) = res { - e.print(py); - panic!("Error running microbenchmark") - } - }; - - let bench_setup = |iterations| { - let globals = cpython::PyDict::new(py); - // setup the __builtins__ attribute - no other way to do this (other than manually) as far - // as I can tell - let _ = py.run("", Some(&globals), None); - let locals = cpython::PyDict::new(py); - if let Some(idx) = iterations { - globals.set_item(py, "ITERATIONS", idx).unwrap(); - } + pyo3::Python::attach(|py| { + let setup_name = format!("{}_setup", bench.name); + let setup_code = cpy_compile_code(py, &bench.setup, &setup_name).unwrap(); + + let code = cpy_compile_code(py, &bench.code, &bench.name).unwrap(); + + // Grab the exec function in advance so we don't have lookups in the hot code + let builtins = + pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); + let exec = builtins.getattr("exec").expect("no exec in builtins"); + + let bench_func = |(globals, locals): &mut ( + pyo3::Bound<pyo3::types::PyDict>, + pyo3::Bound<pyo3::types::PyDict>, + )| { + let res = exec.call((&code, &*globals, &*locals), None); + if let Err(e) = res { + e.print(py); + panic!("Error running microbenchmark") + } + }; - let res = cpy_run_code(py, &setup_code, &globals, &locals); - if let Err(e) = res { - e.print(py); - panic!("Error running microbenchmark setup code") - } - (globals, locals) - }; - - if bench.iterate { - for idx in (100..=1_000).step_by(200) { - group.throughput(Throughput::Elements(idx as u64)); - group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| { - b.iter_batched_ref( - || bench_setup(Some(*idx)), - bench_func, - BatchSize::LargeInput, - ); - }); - } - } else { - group.bench_function(BenchmarkId::new("cpython", &bench.name), move |b| { - b.iter_batched_ref(|| bench_setup(None), bench_func, BatchSize::LargeInput); - }); - } -} + let bench_setup = |iterations| { + let globals = pyo3::types::PyDict::new(py); + let locals = pyo3::types::PyDict::new(py); + if let Some(idx) = iterations { + globals.set_item("ITERATIONS", idx).unwrap(); + } -unsafe fn cpy_res( - py: cpython::Python<'_>, - x: *mut python3_sys::PyObject, -) -> cpython::PyResult<cpython::PyObject> { - cpython::PyObject::from_owned_ptr_opt(py, x).ok_or_else(|| cpython::PyErr::fetch(py)) -} + let res = exec.call((&setup_code, &globals, &locals), None); + if let Err(e) = res { + e.print(py); + panic!("Error running microbenchmark setup code") + } + (globals, locals) + }; -fn cpy_compile_code( - py: cpython::Python<'_>, - s: &ffi::CStr, - fname: &ffi::CStr, -) -> cpython::PyResult<cpython::PyObject> { - unsafe { - let res = - python3_sys::Py_CompileString(s.as_ptr(), fname.as_ptr(), python3_sys::Py_file_input); - cpy_res(py, res) - } + if bench.iterate { + for idx in (100..=1_000).step_by(200) { + group.throughput(Throughput::Elements(idx as u64)); + group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| { + b.iter_batched_ref( + || bench_setup(Some(*idx)), + bench_func, + BatchSize::LargeInput, + ); + }); + } + } else { + group.bench_function(BenchmarkId::new("cpython", &bench.name), move |b| { + b.iter_batched_ref(|| bench_setup(None), bench_func, BatchSize::LargeInput); + }); + } + }) } -fn cpy_run_code( - py: cpython::Python<'_>, - code: &cpython::PyObject, - locals: &cpython::PyDict, - globals: &cpython::PyDict, -) -> cpython::PyResult<cpython::PyObject> { - use cpython::PythonObject; - unsafe { - let res = python3_sys::PyEval_EvalCode( - code.as_ptr(), - locals.as_object().as_ptr(), - globals.as_object().as_ptr(), - ); - cpy_res(py, res) - } +fn cpy_compile_code<'a>( + py: pyo3::Python<'a>, + code: &str, + name: &str, +) -> pyo3::PyResult<pyo3::Bound<'a, pyo3::types::PyCode>> { + let builtins = + pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); + let compile = builtins.getattr("compile").expect("no compile in builtins"); + Ok(compile + .call1((code, name, "exec"))? + .cast_into() + .expect("compile() should return a code object")) } -fn bench_rustpy_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchmark) { +fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchmark) { let mut settings = Settings::default(); settings.path_list.push("Lib/".to_string()); - settings.dont_write_bytecode = true; - settings.no_user_site = true; + settings.write_bytecode = false; + settings.user_site_directory = false; - Interpreter::with_init(settings, |vm| { - for (name, init) in rustpython_stdlib::get_module_inits() { - vm.add_native_module(name, init); - } - }) - .enter(|vm| { + let builder = Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); + interp.enter(|vm| { let setup_code = vm .compile(&bench.setup, Mode::Exec, bench.name.to_owned()) .expect("Error compiling setup code"); @@ -156,6 +134,8 @@ fn bench_rustpy_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchmar if let Some(idx) = iterations { scope .locals + .as_ref() + .expect("new_scope_with_builtins always provides locals") .as_object() .set_item("ITERATIONS", vm.new_pyobj(idx), vm) .expect("Error adding ITERATIONS local variable"); @@ -192,7 +172,7 @@ pub fn run_micro_benchmark(c: &mut Criterion, benchmark: MicroBenchmark) { let mut group = c.benchmark_group("microbenchmarks"); bench_cpython_code(&mut group, &benchmark); - bench_rustpy_code(&mut group, &benchmark); + bench_rustpython_code(&mut group, &benchmark); group.finish(); } diff --git a/benches/microbenchmarks/frozenset.py b/benches/microbenchmarks/frozenset.py new file mode 100644 index 00000000000..74bfb9ddb42 --- /dev/null +++ b/benches/microbenchmarks/frozenset.py @@ -0,0 +1,5 @@ +fs = frozenset(range(0, ITERATIONS)) + +# --- + +hash(fs) diff --git a/benches/microbenchmarks/sorted.py b/benches/microbenchmarks/sorted.py new file mode 100644 index 00000000000..8e5e0beeb19 --- /dev/null +++ b/benches/microbenchmarks/sorted.py @@ -0,0 +1,9 @@ +from random import random, seed +seed(0) + +unsorted_list = [random() for _ in range(5 * ITERATIONS)] + +# --- + +# Setup code only runs once so do not modify in-place +sorted(unsorted_list) diff --git a/build.rs b/build.rs new file mode 100644 index 00000000000..adebd659ade --- /dev/null +++ b/build.rs @@ -0,0 +1,17 @@ +fn main() { + if std::env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows" { + println!("cargo:rerun-if-changed=logo.ico"); + let mut res = winresource::WindowsResource::new(); + if std::path::Path::new("logo.ico").exists() { + res.set_icon("logo.ico"); + } else { + println!("cargo:warning=logo.ico not found, skipping icon embedding"); + return; + } + res.compile() + .map_err(|e| { + println!("cargo:warning=Failed to compile Windows resources: {e}"); + }) + .ok(); + } +} diff --git a/common/Cargo.toml b/common/Cargo.toml deleted file mode 100644 index 397897f491f..00000000000 --- a/common/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -name = "rustpython-common" -version = "0.3.0" -description = "General python functions and algorithms for use in RustPython" -authors = ["RustPython Team"] -edition = "2021" -repository = "https://github.com/RustPython/RustPython" -license = "MIT" - -[features] -threading = ["parking_lot"] - -[dependencies] -rustpython-format = { workspace = true } - -ascii = { workspace = true } -bitflags = { workspace = true } -bstr = { workspace = true } -cfg-if = { workspace = true } -itertools = { workspace = true } -libc = { workspace = true } -malachite-bigint = { workspace = true } -malachite-q = { workspace = true } -malachite-base = { workspace = true } -num-complex = { workspace = true } -num-traits = { workspace = true } -once_cell = { workspace = true } -parking_lot = { workspace = true, optional = true } -rand = { workspace = true } - -lock_api = "0.4" -radium = "0.7" -siphasher = "0.3" -volatile = "0.3" - -[target.'cfg(windows)'.dependencies] -widestring = { workspace = true } diff --git a/common/src/cmp.rs b/common/src/cmp.rs deleted file mode 100644 index d182340a984..00000000000 --- a/common/src/cmp.rs +++ /dev/null @@ -1,48 +0,0 @@ -use volatile::Volatile; - -/// Compare 2 byte slices in a way that ensures that the timing of the operation can't be used to -/// glean any information about the data. -#[inline(never)] -#[cold] -pub fn timing_safe_cmp(a: &[u8], b: &[u8]) -> bool { - // we use raw pointers here to keep faithful to the C implementation and - // to try to avoid any optimizations rustc might do with slices - let len_a = a.len(); - let a = a.as_ptr(); - let len_b = b.len(); - let b = b.as_ptr(); - /* The volatile type declarations make sure that the compiler has no - * chance to optimize and fold the code in any way that may change - * the timing. - */ - let mut result: u8 = 0; - /* loop count depends on length of b */ - let length: Volatile<usize> = Volatile::new(len_b); - let mut left: Volatile<*const u8> = Volatile::new(std::ptr::null()); - let mut right: Volatile<*const u8> = Volatile::new(b); - - /* don't use else here to keep the amount of CPU instructions constant, - * volatile forces re-evaluation - * */ - if len_a == length.read() { - left.write(Volatile::new(a).read()); - result = 0; - } - if len_a != length.read() { - left.write(b); - result = 1; - } - - for _ in 0..length.read() { - let l = left.read(); - left.write(l.wrapping_add(1)); - let r = right.read(); - right.write(r.wrapping_add(1)); - // safety: the 0..length range will always be either: - // * as long as the length of both a and b, if len_a and len_b are equal - // * as long as b, and both `left` and `right` are b - result |= unsafe { l.read_volatile() ^ r.read_volatile() }; - } - - result == 0 -} diff --git a/common/src/crt_fd.rs b/common/src/crt_fd.rs deleted file mode 100644 index 8eef8d4df08..00000000000 --- a/common/src/crt_fd.rs +++ /dev/null @@ -1,131 +0,0 @@ -//! A module implementing an io type backed by the C runtime's file descriptors, i.e. what's -//! returned from libc::open, even on windows. - -use std::{cmp, ffi, io}; - -#[cfg(windows)] -use libc::commit as fsync; -#[cfg(windows)] -extern "C" { - #[link_name = "_chsize_s"] - fn ftruncate(fd: i32, len: i64) -> i32; -} -#[cfg(not(windows))] -use libc::{fsync, ftruncate}; - -// this is basically what CPython has for Py_off_t; windows uses long long -// for offsets, other platforms just use off_t -#[cfg(not(windows))] -pub type Offset = libc::off_t; -#[cfg(windows)] -pub type Offset = libc::c_longlong; - -#[inline] -fn cvt<T, I: num_traits::PrimInt>(ret: I, f: impl FnOnce(I) -> T) -> io::Result<T> { - if ret < I::zero() { - Err(crate::os::errno()) - } else { - Ok(f(ret)) - } -} - -const MAX_RW: usize = if cfg!(any(windows, target_vendor = "apple")) { - i32::MAX as usize -} else { - isize::MAX as usize -}; - -#[derive(Copy, Clone, PartialEq, Eq)] -#[repr(transparent)] -pub struct Fd(pub i32); - -impl Fd { - pub fn open(path: &ffi::CStr, flags: i32, mode: i32) -> io::Result<Self> { - cvt(unsafe { libc::open(path.as_ptr(), flags, mode) }, Fd) - } - - #[cfg(windows)] - pub fn wopen(path: &widestring::WideCStr, flags: i32, mode: i32) -> io::Result<Self> { - cvt( - unsafe { suppress_iph!(libc::wopen(path.as_ptr(), flags, mode)) }, - Fd, - ) - } - - #[cfg(all(any(unix, target_os = "wasi"), not(target_os = "redox")))] - pub fn openat(&self, path: &ffi::CStr, flags: i32, mode: i32) -> io::Result<Self> { - cvt( - unsafe { libc::openat(self.0, path.as_ptr(), flags, mode) }, - Fd, - ) - } - - pub fn fsync(&self) -> io::Result<()> { - cvt(unsafe { suppress_iph!(fsync(self.0)) }, drop) - } - - pub fn close(&self) -> io::Result<()> { - cvt(unsafe { suppress_iph!(libc::close(self.0)) }, drop) - } - - pub fn ftruncate(&self, len: Offset) -> io::Result<()> { - cvt(unsafe { suppress_iph!(ftruncate(self.0, len)) }, drop) - } - - #[cfg(windows)] - pub fn to_raw_handle(&self) -> io::Result<std::os::windows::io::RawHandle> { - extern "C" { - fn _get_osfhandle(fd: i32) -> libc::intptr_t; - } - let handle = unsafe { suppress_iph!(_get_osfhandle(self.0)) }; - if handle == -1 { - Err(io::Error::last_os_error()) - } else { - Ok(handle as _) - } - } -} - -impl io::Write for &Fd { - fn write(&mut self, buf: &[u8]) -> io::Result<usize> { - let count = cmp::min(buf.len(), MAX_RW); - cvt( - unsafe { suppress_iph!(libc::write(self.0, buf.as_ptr() as _, count as _)) }, - |i| i as usize, - ) - } - - #[inline] - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl io::Write for Fd { - #[inline] - fn write(&mut self, buf: &[u8]) -> io::Result<usize> { - (&*self).write(buf) - } - - #[inline] - fn flush(&mut self) -> io::Result<()> { - (&*self).flush() - } -} - -impl io::Read for &Fd { - fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { - let count = cmp::min(buf.len(), MAX_RW); - cvt( - unsafe { suppress_iph!(libc::read(self.0, buf.as_mut_ptr() as _, count as _)) }, - |i| i as usize, - ) - } -} - -impl io::Read for Fd { - #[inline] - fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { - (&*self).read(buf) - } -} diff --git a/common/src/encodings.rs b/common/src/encodings.rs deleted file mode 100644 index 4e0c1de56a6..00000000000 --- a/common/src/encodings.rs +++ /dev/null @@ -1,323 +0,0 @@ -use std::ops::Range; - -pub type EncodeErrorResult<S, B, E> = Result<(EncodeReplace<S, B>, usize), E>; - -pub type DecodeErrorResult<S, B, E> = Result<(S, Option<B>, usize), E>; - -pub trait StrBuffer: AsRef<str> { - fn is_ascii(&self) -> bool { - self.as_ref().is_ascii() - } -} - -pub trait ErrorHandler { - type Error; - type StrBuf: StrBuffer; - type BytesBuf: AsRef<[u8]>; - fn handle_encode_error( - &self, - data: &str, - char_range: Range<usize>, - reason: &str, - ) -> EncodeErrorResult<Self::StrBuf, Self::BytesBuf, Self::Error>; - fn handle_decode_error( - &self, - data: &[u8], - byte_range: Range<usize>, - reason: &str, - ) -> DecodeErrorResult<Self::StrBuf, Self::BytesBuf, Self::Error>; - fn error_oob_restart(&self, i: usize) -> Self::Error; - fn error_encoding(&self, data: &str, char_range: Range<usize>, reason: &str) -> Self::Error; -} -pub enum EncodeReplace<S, B> { - Str(S), - Bytes(B), -} - -struct DecodeError<'a> { - valid_prefix: &'a str, - rest: &'a [u8], - err_len: Option<usize>, -} -/// # Safety -/// `v[..valid_up_to]` must be valid utf8 -unsafe fn make_decode_err(v: &[u8], valid_up_to: usize, err_len: Option<usize>) -> DecodeError<'_> { - let valid_prefix = core::str::from_utf8_unchecked(v.get_unchecked(..valid_up_to)); - let rest = v.get_unchecked(valid_up_to..); - DecodeError { - valid_prefix, - rest, - err_len, - } -} - -enum HandleResult<'a> { - Done, - Error { - err_len: Option<usize>, - reason: &'a str, - }, -} -fn decode_utf8_compatible<E: ErrorHandler, DecodeF, ErrF>( - data: &[u8], - errors: &E, - decode: DecodeF, - handle_error: ErrF, -) -> Result<(String, usize), E::Error> -where - DecodeF: Fn(&[u8]) -> Result<&str, DecodeError<'_>>, - ErrF: Fn(&[u8], Option<usize>) -> HandleResult<'_>, -{ - if data.is_empty() { - return Ok((String::new(), 0)); - } - // we need to coerce the lifetime to that of the function body rather than the - // anonymous input lifetime, so that we can assign it data borrowed from data_from_err - let mut data = data; - let mut data_from_err: E::BytesBuf; - let mut out = String::with_capacity(data.len()); - let mut remaining_index = 0; - let mut remaining_data = data; - loop { - match decode(remaining_data) { - Ok(decoded) => { - out.push_str(decoded); - remaining_index += decoded.len(); - break; - } - Err(e) => { - out.push_str(e.valid_prefix); - match handle_error(e.rest, e.err_len) { - HandleResult::Done => { - remaining_index += e.valid_prefix.len(); - break; - } - HandleResult::Error { err_len, reason } => { - let err_idx = remaining_index + e.valid_prefix.len(); - let err_range = - err_idx..err_len.map_or_else(|| data.len(), |len| err_idx + len); - let (replace, new_data, restart) = - errors.handle_decode_error(data, err_range, reason)?; - out.push_str(replace.as_ref()); - if let Some(new_data) = new_data { - data_from_err = new_data; - data = data_from_err.as_ref(); - } - remaining_data = data - .get(restart..) - .ok_or_else(|| errors.error_oob_restart(restart))?; - remaining_index = restart; - continue; - } - } - } - } - } - Ok((out, remaining_index)) -} - -pub mod utf8 { - use super::*; - - pub const ENCODING_NAME: &str = "utf-8"; - - #[inline] - pub fn encode<E: ErrorHandler>(s: &str, _errors: &E) -> Result<Vec<u8>, E::Error> { - Ok(s.as_bytes().to_vec()) - } - - pub fn decode<E: ErrorHandler>( - data: &[u8], - errors: &E, - final_decode: bool, - ) -> Result<(String, usize), E::Error> { - decode_utf8_compatible( - data, - errors, - |v| { - core::str::from_utf8(v).map_err(|e| { - // SAFETY: as specified in valid_up_to's documentation, input[..e.valid_up_to()] - // is valid utf8 - unsafe { make_decode_err(v, e.valid_up_to(), e.error_len()) } - }) - }, - |rest, err_len| { - let first_err = rest[0]; - if matches!(first_err, 0x80..=0xc1 | 0xf5..=0xff) { - HandleResult::Error { - err_len: Some(1), - reason: "invalid start byte", - } - } else if err_len.is_none() { - // error_len() == None means unexpected eof - if final_decode { - HandleResult::Error { - err_len, - reason: "unexpected end of data", - } - } else { - HandleResult::Done - } - } else if !final_decode && matches!(rest, [0xed, 0xa0..=0xbf]) { - // truncated surrogate - HandleResult::Done - } else { - HandleResult::Error { - err_len, - reason: "invalid continuation byte", - } - } - }, - ) - } -} - -pub mod latin_1 { - use super::*; - - pub const ENCODING_NAME: &str = "latin-1"; - - const ERR_REASON: &str = "ordinal not in range(256)"; - - #[inline] - pub fn encode<E: ErrorHandler>(s: &str, errors: &E) -> Result<Vec<u8>, E::Error> { - let full_data = s; - let mut data = s; - let mut char_data_index = 0; - let mut out = Vec::<u8>::new(); - loop { - match data - .char_indices() - .enumerate() - .find(|(_, (_, c))| !c.is_ascii()) - { - None => { - out.extend_from_slice(data.as_bytes()); - break; - } - Some((char_i, (byte_i, ch))) => { - out.extend_from_slice(&data.as_bytes()[..byte_i]); - let char_start = char_data_index + char_i; - if (ch as u32) <= 255 { - out.push(ch as u8); - let char_restart = char_start + 1; - data = crate::str::try_get_chars(full_data, char_restart..) - .ok_or_else(|| errors.error_oob_restart(char_restart))?; - char_data_index = char_restart; - } else { - // number of non-latin_1 chars between the first non-latin_1 char and the next latin_1 char - let non_latin_1_run_length = data[byte_i..] - .chars() - .take_while(|c| (*c as u32) > 255) - .count(); - let char_range = char_start..char_start + non_latin_1_run_length; - let (replace, char_restart) = errors.handle_encode_error( - full_data, - char_range.clone(), - ERR_REASON, - )?; - match replace { - EncodeReplace::Str(s) => { - if s.as_ref().chars().any(|c| (c as u32) > 255) { - return Err( - errors.error_encoding(full_data, char_range, ERR_REASON) - ); - } - out.extend_from_slice(s.as_ref().as_bytes()); - } - EncodeReplace::Bytes(b) => { - out.extend_from_slice(b.as_ref()); - } - } - data = crate::str::try_get_chars(full_data, char_restart..) - .ok_or_else(|| errors.error_oob_restart(char_restart))?; - char_data_index = char_restart; - } - continue; - } - } - } - Ok(out) - } - - pub fn decode<E: ErrorHandler>(data: &[u8], _errors: &E) -> Result<(String, usize), E::Error> { - let out: String = data.iter().map(|c| *c as char).collect(); - let out_len = out.len(); - Ok((out, out_len)) - } -} - -pub mod ascii { - use super::*; - use ::ascii::AsciiStr; - - pub const ENCODING_NAME: &str = "ascii"; - - const ERR_REASON: &str = "ordinal not in range(128)"; - - #[inline] - pub fn encode<E: ErrorHandler>(s: &str, errors: &E) -> Result<Vec<u8>, E::Error> { - let full_data = s; - let mut data = s; - let mut char_data_index = 0; - let mut out = Vec::<u8>::new(); - loop { - match data - .char_indices() - .enumerate() - .find(|(_, (_, c))| !c.is_ascii()) - { - None => { - out.extend_from_slice(data.as_bytes()); - break; - } - Some((char_i, (byte_i, _))) => { - out.extend_from_slice(&data.as_bytes()[..byte_i]); - let char_start = char_data_index + char_i; - // number of non-ascii chars between the first non-ascii char and the next ascii char - let non_ascii_run_length = - data[byte_i..].chars().take_while(|c| !c.is_ascii()).count(); - let char_range = char_start..char_start + non_ascii_run_length; - let (replace, char_restart) = - errors.handle_encode_error(full_data, char_range.clone(), ERR_REASON)?; - match replace { - EncodeReplace::Str(s) => { - if !s.is_ascii() { - return Err( - errors.error_encoding(full_data, char_range, ERR_REASON) - ); - } - out.extend_from_slice(s.as_ref().as_bytes()); - } - EncodeReplace::Bytes(b) => { - out.extend_from_slice(b.as_ref()); - } - } - data = crate::str::try_get_chars(full_data, char_restart..) - .ok_or_else(|| errors.error_oob_restart(char_restart))?; - char_data_index = char_restart; - continue; - } - } - } - Ok(out) - } - - pub fn decode<E: ErrorHandler>(data: &[u8], errors: &E) -> Result<(String, usize), E::Error> { - decode_utf8_compatible( - data, - errors, - |v| { - AsciiStr::from_ascii(v).map(|s| s.as_str()).map_err(|e| { - // SAFETY: as specified in valid_up_to's documentation, input[..e.valid_up_to()] - // is valid ascii & therefore valid utf8 - unsafe { make_decode_err(v, e.valid_up_to(), Some(1)) } - }) - }, - |_rest, err_len| HandleResult::Error { - err_len, - reason: ERR_REASON, - }, - ) - } -} diff --git a/common/src/hash.rs b/common/src/hash.rs deleted file mode 100644 index f514dac3264..00000000000 --- a/common/src/hash.rs +++ /dev/null @@ -1,200 +0,0 @@ -use malachite_bigint::BigInt; -use num_traits::ToPrimitive; -use siphasher::sip::SipHasher24; -use std::hash::{BuildHasher, Hash, Hasher}; - -pub type PyHash = i64; -pub type PyUHash = u64; - -/// A PyHash value used to represent a missing hash value, e.g. means "not yet computed" for -/// `str`'s hash cache -pub const SENTINEL: PyHash = -1; - -/// Prime multiplier used in string and various other hashes. -pub const MULTIPLIER: PyHash = 1_000_003; // 0xf4243 -/// Numeric hashes are based on reduction modulo the prime 2**_BITS - 1 -pub const BITS: usize = 61; -pub const MODULUS: PyUHash = (1 << BITS) - 1; -pub const INF: PyHash = 314_159; -pub const NAN: PyHash = 0; -pub const IMAG: PyHash = MULTIPLIER; -pub const ALGO: &str = "siphash24"; -pub const HASH_BITS: usize = std::mem::size_of::<PyHash>() * 8; -// SipHasher24 takes 2 u64s as a seed -pub const SEED_BITS: usize = std::mem::size_of::<u64>() * 2 * 8; - -// pub const CUTOFF: usize = 7; - -pub struct HashSecret { - k0: u64, - k1: u64, -} - -impl BuildHasher for HashSecret { - type Hasher = SipHasher24; - fn build_hasher(&self) -> Self::Hasher { - SipHasher24::new_with_keys(self.k0, self.k1) - } -} - -impl rand::distributions::Distribution<HashSecret> for rand::distributions::Standard { - fn sample<R: rand::Rng + ?Sized>(&self, rng: &mut R) -> HashSecret { - HashSecret { - k0: rng.gen(), - k1: rng.gen(), - } - } -} - -impl HashSecret { - pub fn new(seed: u32) -> Self { - let mut buf = [0u8; 16]; - lcg_urandom(seed, &mut buf); - let (left, right) = buf.split_at(8); - let k0 = u64::from_le_bytes(left.try_into().unwrap()); - let k1 = u64::from_le_bytes(right.try_into().unwrap()); - Self { k0, k1 } - } -} - -impl HashSecret { - pub fn hash_value<T: Hash + ?Sized>(&self, data: &T) -> PyHash { - fix_sentinel(mod_int(self.hash_one(data) as _)) - } - - pub fn hash_iter<'a, T: 'a, I, F, E>(&self, iter: I, hashf: F) -> Result<PyHash, E> - where - I: IntoIterator<Item = &'a T>, - F: Fn(&'a T) -> Result<PyHash, E>, - { - let mut hasher = self.build_hasher(); - for element in iter { - let item_hash = hashf(element)?; - item_hash.hash(&mut hasher); - } - Ok(fix_sentinel(mod_int(hasher.finish() as PyHash))) - } - - pub fn hash_bytes(&self, value: &[u8]) -> PyHash { - if value.is_empty() { - 0 - } else { - self.hash_value(value) - } - } - - pub fn hash_str(&self, value: &str) -> PyHash { - self.hash_bytes(value.as_bytes()) - } -} - -#[inline] -pub fn hash_float(value: f64) -> Option<PyHash> { - // cpython _Py_HashDouble - if !value.is_finite() { - return if value.is_infinite() { - Some(if value > 0.0 { INF } else { -INF }) - } else { - None - }; - } - - let frexp = super::float_ops::ufrexp(value); - - // process 28 bits at a time; this should work well both for binary - // and hexadecimal floating point. - let mut m = frexp.0; - let mut e = frexp.1; - let mut x: PyUHash = 0; - while m != 0.0 { - x = ((x << 28) & MODULUS) | x >> (BITS - 28); - m *= 268_435_456.0; // 2**28 - e -= 28; - let y = m as PyUHash; // pull out integer part - m -= y as f64; - x += y; - if x >= MODULUS { - x -= MODULUS; - } - } - - // adjust for the exponent; first reduce it modulo BITS - const BITS32: i32 = BITS as i32; - e = if e >= 0 { - e % BITS32 - } else { - BITS32 - 1 - ((-1 - e) % BITS32) - }; - x = ((x << e) & MODULUS) | x >> (BITS32 - e); - - Some(fix_sentinel(x as PyHash * value.signum() as PyHash)) -} - -pub fn hash_iter_unordered<'a, T: 'a, I, F, E>(iter: I, hashf: F) -> Result<PyHash, E> -where - I: IntoIterator<Item = &'a T>, - F: Fn(&'a T) -> Result<PyHash, E>, -{ - let mut hash: PyHash = 0; - for element in iter { - let item_hash = hashf(element)?; - // xor is commutative and hash should be independent of order - hash ^= item_hash; - } - Ok(fix_sentinel(mod_int(hash))) -} - -pub fn hash_bigint(value: &BigInt) -> PyHash { - let ret = match value.to_i64() { - Some(i) => mod_int(i), - None => (value % MODULUS).to_i64().unwrap_or_else(|| unsafe { - // SAFETY: MODULUS < i64::MAX, so value % MODULUS is guaranteed to be in the range of i64 - std::hint::unreachable_unchecked() - }), - }; - fix_sentinel(ret) -} - -#[inline(always)] -pub fn fix_sentinel(x: PyHash) -> PyHash { - if x == SENTINEL { - -2 - } else { - x - } -} - -#[inline] -pub fn mod_int(value: i64) -> PyHash { - value % MODULUS as i64 -} - -pub fn lcg_urandom(mut x: u32, buf: &mut [u8]) { - for b in buf { - x = x.wrapping_mul(214013); - x = x.wrapping_add(2531011); - *b = ((x >> 16) & 0xff) as u8; - } -} - -#[inline] -pub fn hash_object_id_raw(p: usize) -> PyHash { - // TODO: Use commented logic when below issue resolved. - // Ref: https://github.com/RustPython/RustPython/pull/3951#issuecomment-1193108966 - - /* bottom 3 or 4 bits are likely to be 0; rotate y by 4 to avoid - excessive hash collisions for dicts and sets */ - // p.rotate_right(4) as PyHash - p as PyHash -} - -#[inline] -pub fn hash_object_id(p: usize) -> PyHash { - fix_sentinel(hash_object_id_raw(p)) -} - -pub fn keyed_hash(key: u64, buf: &[u8]) -> u64 { - let mut hasher = SipHasher24::new_with_keys(key, 0); - buf.hash(&mut hasher); - hasher.finish() -} diff --git a/common/src/int.rs b/common/src/int.rs deleted file mode 100644 index 2de993b59b8..00000000000 --- a/common/src/int.rs +++ /dev/null @@ -1,159 +0,0 @@ -use bstr::ByteSlice; -use malachite_base::{num::conversion::traits::RoundingInto, rounding_modes::RoundingMode}; -use malachite_bigint::{BigInt, BigUint, Sign}; -use malachite_q::Rational; -use num_traits::{One, ToPrimitive, Zero}; - -pub fn true_div(numerator: &BigInt, denominator: &BigInt) -> f64 { - let rational = Rational::from_integers_ref(numerator.into(), denominator.into()); - match rational.rounding_into(RoundingMode::Nearest) { - // returned value is $t::MAX but still less than the original - (val, std::cmp::Ordering::Less) if val == f64::MAX => f64::INFINITY, - // returned value is $t::MIN but still greater than the original - (val, std::cmp::Ordering::Greater) if val == f64::MIN => f64::NEG_INFINITY, - (val, _) => val, - } -} - -pub fn float_to_ratio(value: f64) -> Option<(BigInt, BigInt)> { - let sign = match std::cmp::PartialOrd::partial_cmp(&value, &0.0)? { - std::cmp::Ordering::Less => Sign::Minus, - std::cmp::Ordering::Equal => return Some((BigInt::zero(), BigInt::one())), - std::cmp::Ordering::Greater => Sign::Plus, - }; - Rational::try_from(value).ok().map(|x| { - let (numer, denom) = x.into_numerator_and_denominator(); - ( - BigInt::from_biguint(sign, numer.into()), - BigUint::from(denom).into(), - ) - }) -} - -pub fn bytes_to_int(lit: &[u8], mut base: u32) -> Option<BigInt> { - // split sign - let mut lit = lit.trim(); - let sign = match lit.first()? { - b'+' => Some(Sign::Plus), - b'-' => Some(Sign::Minus), - _ => None, - }; - if sign.is_some() { - lit = &lit[1..]; - } - - // split radix - let first = *lit.first()?; - let has_radix = if first == b'0' { - match base { - 0 => { - if let Some(parsed) = lit.get(1).and_then(detect_base) { - base = parsed; - true - } else { - if let [_first, ref others @ .., last] = lit { - let is_zero = - others.iter().all(|&c| c == b'0' || c == b'_') && *last == b'0'; - if !is_zero { - return None; - } - } - return Some(BigInt::zero()); - } - } - 16 => lit.get(1).map_or(false, |&b| matches!(b, b'x' | b'X')), - 2 => lit.get(1).map_or(false, |&b| matches!(b, b'b' | b'B')), - 8 => lit.get(1).map_or(false, |&b| matches!(b, b'o' | b'O')), - _ => false, - } - } else { - if base == 0 { - base = 10; - } - false - }; - if has_radix { - lit = &lit[2..]; - if lit.first()? == &b'_' { - lit = &lit[1..]; - } - } - - // remove zeroes - let mut last = *lit.first()?; - if last == b'0' { - let mut count = 0; - for &cur in &lit[1..] { - if cur == b'_' { - if last == b'_' { - return None; - } - } else if cur != b'0' { - break; - }; - count += 1; - last = cur; - } - let prefix_last = lit[count]; - lit = &lit[count + 1..]; - if lit.is_empty() && prefix_last == b'_' { - return None; - } - } - - // validate - for c in lit { - let c = *c; - if !(c.is_ascii_alphanumeric() || c == b'_') { - return None; - } - - if c == b'_' && last == b'_' { - return None; - } - - last = c; - } - if last == b'_' { - return None; - } - - // parse - let number = if lit.is_empty() { - BigInt::zero() - } else { - let uint = BigUint::parse_bytes(lit, base)?; - BigInt::from_biguint(sign.unwrap_or(Sign::Plus), uint) - }; - Some(number) -} - -#[inline] -pub fn detect_base(c: &u8) -> Option<u32> { - let base = match c { - b'x' | b'X' => 16, - b'b' | b'B' => 2, - b'o' | b'O' => 8, - _ => return None, - }; - Some(base) -} - -// num-bigint now returns Some(inf) for to_f64() in some cases, so just keep that the same for now -#[inline(always)] -pub fn bigint_to_finite_float(int: &BigInt) -> Option<f64> { - int.to_f64().filter(|f| f.is_finite()) -} - -#[test] -fn test_bytes_to_int() { - assert_eq!(bytes_to_int(&b"0b101"[..], 2).unwrap(), BigInt::from(5)); - assert_eq!(bytes_to_int(&b"0x_10"[..], 16).unwrap(), BigInt::from(16)); - assert_eq!(bytes_to_int(&b"0b"[..], 16).unwrap(), BigInt::from(11)); - assert_eq!(bytes_to_int(&b"+0b101"[..], 2).unwrap(), BigInt::from(5)); - assert_eq!(bytes_to_int(&b"0_0_0"[..], 10).unwrap(), BigInt::from(0)); - assert_eq!(bytes_to_int(&b"09_99"[..], 0), None); - assert_eq!(bytes_to_int(&b"000"[..], 0).unwrap(), BigInt::from(0)); - assert_eq!(bytes_to_int(&b"0_"[..], 0), None); - assert_eq!(bytes_to_int(&b"0_100"[..], 10).unwrap(), BigInt::from(100)); -} diff --git a/common/src/lib.rs b/common/src/lib.rs deleted file mode 100644 index ffd027a0e67..00000000000 --- a/common/src/lib.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! A crate to hold types and functions common to all rustpython components. - -#[macro_use] -mod macros; -pub use macros::*; - -pub mod atomic; -pub mod borrow; -pub mod boxvec; -pub mod cmp; -#[cfg(any(unix, windows, target_os = "wasi"))] -pub mod crt_fd; -pub mod encodings; -pub mod float_ops; -pub mod hash; -pub mod int; -pub mod linked_list; -pub mod lock; -pub mod os; -pub mod rc; -pub mod refcount; -pub mod static_cell; -pub mod str; -#[cfg(windows)] -pub mod windows; - -pub mod vendored { - pub use ascii; -} diff --git a/common/src/lock.rs b/common/src/lock.rs deleted file mode 100644 index 811c4611121..00000000000 --- a/common/src/lock.rs +++ /dev/null @@ -1,42 +0,0 @@ -//! A module containing [`lock_api`]-based lock types that are or are not `Send + Sync` -//! depending on whether the `threading` feature of this module is enabled. - -use lock_api::{ - MappedMutexGuard, MappedRwLockReadGuard, MappedRwLockWriteGuard, Mutex, MutexGuard, RwLock, - RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard, -}; - -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - pub use parking_lot::{RawMutex, RawRwLock, RawThreadId}; - - pub use once_cell::sync::{Lazy, OnceCell}; - } else { - mod cell_lock; - pub use cell_lock::{RawCellMutex as RawMutex, RawCellRwLock as RawRwLock, SingleThreadId as RawThreadId}; - - pub use once_cell::unsync::{Lazy, OnceCell}; - } -} - -mod immutable_mutex; -pub use immutable_mutex::*; -mod thread_mutex; -pub use thread_mutex::*; - -pub type PyMutex<T> = Mutex<RawMutex, T>; -pub type PyMutexGuard<'a, T> = MutexGuard<'a, RawMutex, T>; -pub type PyMappedMutexGuard<'a, T> = MappedMutexGuard<'a, RawMutex, T>; -pub type PyImmutableMappedMutexGuard<'a, T> = ImmutableMappedMutexGuard<'a, RawMutex, T>; -pub type PyThreadMutex<T> = ThreadMutex<RawMutex, RawThreadId, T>; -pub type PyThreadMutexGuard<'a, T> = ThreadMutexGuard<'a, RawMutex, RawThreadId, T>; -pub type PyMappedThreadMutexGuard<'a, T> = MappedThreadMutexGuard<'a, RawMutex, RawThreadId, T>; - -pub type PyRwLock<T> = RwLock<RawRwLock, T>; -pub type PyRwLockUpgradableReadGuard<'a, T> = RwLockUpgradableReadGuard<'a, RawRwLock, T>; -pub type PyRwLockReadGuard<'a, T> = RwLockReadGuard<'a, RawRwLock, T>; -pub type PyMappedRwLockReadGuard<'a, T> = MappedRwLockReadGuard<'a, RawRwLock, T>; -pub type PyRwLockWriteGuard<'a, T> = RwLockWriteGuard<'a, RawRwLock, T>; -pub type PyMappedRwLockWriteGuard<'a, T> = MappedRwLockWriteGuard<'a, RawRwLock, T>; - -// can add fn const_{mutex,rwlock}() if necessary, but we probably won't need to diff --git a/common/src/lock/thread_mutex.rs b/common/src/lock/thread_mutex.rs deleted file mode 100644 index 1a505426452..00000000000 --- a/common/src/lock/thread_mutex.rs +++ /dev/null @@ -1,291 +0,0 @@ -use lock_api::{GetThreadId, GuardNoSend, RawMutex}; -use std::{ - cell::UnsafeCell, - fmt, - marker::PhantomData, - ops::{Deref, DerefMut}, - ptr::NonNull, - sync::atomic::{AtomicUsize, Ordering}, -}; - -// based off ReentrantMutex from lock_api - -/// A mutex type that knows when it would deadlock -pub struct RawThreadMutex<R: RawMutex, G: GetThreadId> { - owner: AtomicUsize, - mutex: R, - get_thread_id: G, -} - -impl<R: RawMutex, G: GetThreadId> RawThreadMutex<R, G> { - #[allow(clippy::declare_interior_mutable_const)] - pub const INIT: Self = RawThreadMutex { - owner: AtomicUsize::new(0), - mutex: R::INIT, - get_thread_id: G::INIT, - }; - - #[inline] - fn lock_internal<F: FnOnce() -> bool>(&self, try_lock: F) -> Option<bool> { - let id = self.get_thread_id.nonzero_thread_id().get(); - if self.owner.load(Ordering::Relaxed) == id { - return None; - } else { - if !try_lock() { - return Some(false); - } - self.owner.store(id, Ordering::Relaxed); - } - Some(true) - } - - /// Blocks for the mutex to be available, and returns true if the mutex isn't already - /// locked on the current thread. - pub fn lock(&self) -> bool { - self.lock_internal(|| { - self.mutex.lock(); - true - }) - .is_some() - } - - /// Returns `Some(true)` if able to successfully lock without blocking, `Some(false)` - /// otherwise, and `None` when the mutex is already locked on the current thread. - pub fn try_lock(&self) -> Option<bool> { - self.lock_internal(|| self.mutex.try_lock()) - } - - /// Unlocks this mutex. The inner mutex may not be unlocked if - /// this mutex was acquired previously in the current thread. - /// - /// # Safety - /// - /// This method may only be called if the mutex is held by the current thread. - pub unsafe fn unlock(&self) { - self.owner.store(0, Ordering::Relaxed); - self.mutex.unlock(); - } -} - -unsafe impl<R: RawMutex + Send, G: GetThreadId + Send> Send for RawThreadMutex<R, G> {} -unsafe impl<R: RawMutex + Sync, G: GetThreadId + Sync> Sync for RawThreadMutex<R, G> {} - -pub struct ThreadMutex<R: RawMutex, G: GetThreadId, T: ?Sized> { - raw: RawThreadMutex<R, G>, - data: UnsafeCell<T>, -} - -impl<R: RawMutex, G: GetThreadId, T> ThreadMutex<R, G, T> { - pub fn new(val: T) -> Self { - ThreadMutex { - raw: RawThreadMutex::INIT, - data: UnsafeCell::new(val), - } - } - - pub fn into_inner(self) -> T { - self.data.into_inner() - } -} -impl<R: RawMutex, G: GetThreadId, T: Default> Default for ThreadMutex<R, G, T> { - fn default() -> Self { - Self::new(T::default()) - } -} -impl<R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutex<R, G, T> { - pub fn lock(&self) -> Option<ThreadMutexGuard<R, G, T>> { - if self.raw.lock() { - Some(ThreadMutexGuard { - mu: self, - marker: PhantomData, - }) - } else { - None - } - } - pub fn try_lock(&self) -> Result<ThreadMutexGuard<R, G, T>, TryLockThreadError> { - match self.raw.try_lock() { - Some(true) => Ok(ThreadMutexGuard { - mu: self, - marker: PhantomData, - }), - Some(false) => Err(TryLockThreadError::Other), - None => Err(TryLockThreadError::Current), - } - } -} -// Whether ThreadMutex::try_lock failed because the mutex was already locked on another thread or -// on the current thread -pub enum TryLockThreadError { - Other, - Current, -} - -struct LockedPlaceholder(&'static str); -impl fmt::Debug for LockedPlaceholder { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.0) - } -} -impl<R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Debug> fmt::Debug for ThreadMutex<R, G, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.try_lock() { - Ok(guard) => f - .debug_struct("ThreadMutex") - .field("data", &&*guard) - .finish(), - Err(e) => { - let msg = match e { - TryLockThreadError::Other => "<locked on other thread>", - TryLockThreadError::Current => "<locked on current thread>", - }; - f.debug_struct("ThreadMutex") - .field("data", &LockedPlaceholder(msg)) - .finish() - } - } - } -} - -unsafe impl<R: RawMutex + Send, G: GetThreadId + Send, T: ?Sized + Send> Send - for ThreadMutex<R, G, T> -{ -} -unsafe impl<R: RawMutex + Sync, G: GetThreadId + Sync, T: ?Sized + Send> Sync - for ThreadMutex<R, G, T> -{ -} - -pub struct ThreadMutexGuard<'a, R: RawMutex, G: GetThreadId, T: ?Sized> { - mu: &'a ThreadMutex<R, G, T>, - marker: PhantomData<(&'a mut T, GuardNoSend)>, -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutexGuard<'a, R, G, T> { - pub fn map<U, F: FnOnce(&mut T) -> &mut U>( - mut s: Self, - f: F, - ) -> MappedThreadMutexGuard<'a, R, G, U> { - let data = f(&mut s).into(); - let mu = &s.mu.raw; - std::mem::forget(s); - MappedThreadMutexGuard { - mu, - data, - marker: PhantomData, - } - } - pub fn try_map<U, F: FnOnce(&mut T) -> Option<&mut U>>( - mut s: Self, - f: F, - ) -> Result<MappedThreadMutexGuard<'a, R, G, U>, Self> { - if let Some(data) = f(&mut s) { - let data = data.into(); - let mu = &s.mu.raw; - std::mem::forget(s); - Ok(MappedThreadMutexGuard { - mu, - data, - marker: PhantomData, - }) - } else { - Err(s) - } - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> Deref for ThreadMutexGuard<'a, R, G, T> { - type Target = T; - fn deref(&self) -> &T { - unsafe { &*self.mu.data.get() } - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> DerefMut for ThreadMutexGuard<'a, R, G, T> { - fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.mu.data.get() } - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> Drop for ThreadMutexGuard<'a, R, G, T> { - fn drop(&mut self) { - unsafe { self.mu.raw.unlock() } - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Display> fmt::Display - for ThreadMutexGuard<'a, R, G, T> -{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&**self, f) - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Debug> fmt::Debug - for ThreadMutexGuard<'a, R, G, T> -{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&**self, f) - } -} -pub struct MappedThreadMutexGuard<'a, R: RawMutex, G: GetThreadId, T: ?Sized> { - mu: &'a RawThreadMutex<R, G>, - data: NonNull<T>, - marker: PhantomData<(&'a mut T, GuardNoSend)>, -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> MappedThreadMutexGuard<'a, R, G, T> { - pub fn map<U, F: FnOnce(&mut T) -> &mut U>( - mut s: Self, - f: F, - ) -> MappedThreadMutexGuard<'a, R, G, U> { - let data = f(&mut s).into(); - let mu = s.mu; - std::mem::forget(s); - MappedThreadMutexGuard { - mu, - data, - marker: PhantomData, - } - } - pub fn try_map<U, F: FnOnce(&mut T) -> Option<&mut U>>( - mut s: Self, - f: F, - ) -> Result<MappedThreadMutexGuard<'a, R, G, U>, Self> { - if let Some(data) = f(&mut s) { - let data = data.into(); - let mu = s.mu; - std::mem::forget(s); - Ok(MappedThreadMutexGuard { - mu, - data, - marker: PhantomData, - }) - } else { - Err(s) - } - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> Deref for MappedThreadMutexGuard<'a, R, G, T> { - type Target = T; - fn deref(&self) -> &T { - unsafe { self.data.as_ref() } - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> DerefMut for MappedThreadMutexGuard<'a, R, G, T> { - fn deref_mut(&mut self) -> &mut T { - unsafe { self.data.as_mut() } - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> Drop for MappedThreadMutexGuard<'a, R, G, T> { - fn drop(&mut self) { - unsafe { self.mu.unlock() } - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Display> fmt::Display - for MappedThreadMutexGuard<'a, R, G, T> -{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&**self, f) - } -} -impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Debug> fmt::Debug - for MappedThreadMutexGuard<'a, R, G, T> -{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&**self, f) - } -} diff --git a/common/src/macros.rs b/common/src/macros.rs deleted file mode 100644 index 318ab06986e..00000000000 --- a/common/src/macros.rs +++ /dev/null @@ -1,59 +0,0 @@ -/// Suppress the MSVC invalid parameter handler, which by default crashes the process. Does nothing -/// on non-MSVC targets. -#[macro_export] -macro_rules! suppress_iph { - ($e:expr) => { - $crate::__suppress_iph_impl!($e) - }; -} - -#[macro_export] -#[doc(hidden)] -#[cfg(all(windows, target_env = "msvc"))] -macro_rules! __suppress_iph_impl { - ($e:expr) => {{ - let old = $crate::__macro_private::_set_thread_local_invalid_parameter_handler( - $crate::__macro_private::silent_iph_handler, - ); - let ret = $e; - $crate::__macro_private::_set_thread_local_invalid_parameter_handler(old); - ret - }}; -} - -#[cfg(not(all(windows, target_env = "msvc")))] -#[macro_export] -#[doc(hidden)] -macro_rules! __suppress_iph_impl { - ($e:expr) => { - $e - }; -} - -#[doc(hidden)] -pub mod __macro_private { - #[cfg(target_env = "msvc")] - type InvalidParamHandler = extern "C" fn( - *const libc::wchar_t, - *const libc::wchar_t, - *const libc::wchar_t, - libc::c_uint, - libc::uintptr_t, - ); - #[cfg(target_env = "msvc")] - extern "C" { - pub fn _set_thread_local_invalid_parameter_handler( - pNew: InvalidParamHandler, - ) -> InvalidParamHandler; - } - - #[cfg(target_env = "msvc")] - pub extern "C" fn silent_iph_handler( - _: *const libc::wchar_t, - _: *const libc::wchar_t, - _: *const libc::wchar_t, - _: libc::c_uint, - _: libc::uintptr_t, - ) { - } -} diff --git a/common/src/os.rs b/common/src/os.rs deleted file mode 100644 index c583647ce77..00000000000 --- a/common/src/os.rs +++ /dev/null @@ -1,39 +0,0 @@ -// TODO: we can move more os-specific bindings/interfaces from stdlib::{os, posix, nt} to here - -use std::{io, str::Utf8Error}; - -#[cfg(windows)] -pub fn errno() -> io::Error { - let err = io::Error::last_os_error(); - // FIXME: probably not ideal, we need a bigger dichotomy between GetLastError and errno - if err.raw_os_error() == Some(0) { - extern "C" { - fn _get_errno(pValue: *mut i32) -> i32; - } - let mut e = 0; - unsafe { suppress_iph!(_get_errno(&mut e)) }; - io::Error::from_raw_os_error(e) - } else { - err - } -} -#[cfg(not(windows))] -pub fn errno() -> io::Error { - io::Error::last_os_error() -} - -#[cfg(unix)] -pub fn bytes_as_osstr(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { - use std::os::unix::ffi::OsStrExt; - Ok(std::ffi::OsStr::from_bytes(b)) -} - -#[cfg(not(unix))] -pub fn bytes_as_osstr(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { - Ok(std::str::from_utf8(b)?.as_ref()) -} - -#[cfg(unix)] -pub use std::os::unix::ffi; -#[cfg(target_os = "wasi")] -pub use std::os::wasi::ffi; diff --git a/common/src/rc.rs b/common/src/rc.rs deleted file mode 100644 index 81207e840c0..00000000000 --- a/common/src/rc.rs +++ /dev/null @@ -1,12 +0,0 @@ -#[cfg(not(feature = "threading"))] -use std::rc::Rc; -#[cfg(feature = "threading")] -use std::sync::Arc; - -// type aliases instead of newtypes because you can't do `fn method(self: PyRc<Self>)` with a -// newtype; requires the arbitrary_self_types unstable feature - -#[cfg(feature = "threading")] -pub type PyRc<T> = Arc<T>; -#[cfg(not(feature = "threading"))] -pub type PyRc<T> = Rc<T>; diff --git a/common/src/refcount.rs b/common/src/refcount.rs deleted file mode 100644 index 910eef436a9..00000000000 --- a/common/src/refcount.rs +++ /dev/null @@ -1,78 +0,0 @@ -use crate::atomic::{Ordering::*, PyAtomic, Radium}; - -/// from alloc::sync -/// A soft limit on the amount of references that may be made to an `Arc`. -/// -/// Going above this limit will abort your program (although not -/// necessarily) at _exactly_ `MAX_REFCOUNT + 1` references. -const MAX_REFCOUNT: usize = isize::MAX as usize; - -pub struct RefCount { - strong: PyAtomic<usize>, -} - -impl Default for RefCount { - fn default() -> Self { - Self::new() - } -} - -impl RefCount { - const MASK: usize = MAX_REFCOUNT; - - pub fn new() -> Self { - RefCount { - strong: Radium::new(1), - } - } - - #[inline] - pub fn get(&self) -> usize { - self.strong.load(SeqCst) - } - - #[inline] - pub fn inc(&self) { - let old_size = self.strong.fetch_add(1, Relaxed); - - if old_size & Self::MASK == Self::MASK { - std::process::abort(); - } - } - - /// Returns true if successful - #[inline] - pub fn safe_inc(&self) -> bool { - self.strong - .fetch_update(AcqRel, Acquire, |prev| (prev != 0).then_some(prev + 1)) - .is_ok() - } - - /// Decrement the reference count. Returns true when the refcount drops to 0. - #[inline] - pub fn dec(&self) -> bool { - if self.strong.fetch_sub(1, Release) != 1 { - return false; - } - - PyAtomic::<usize>::fence(Acquire); - - true - } -} - -impl RefCount { - // move these functions out and give separated type once type range is stabilized - - pub fn leak(&self) { - debug_assert!(!self.is_leaked()); - const BIT_MARKER: usize = (std::isize::MAX as usize) + 1; - debug_assert_eq!(BIT_MARKER.count_ones(), 1); - debug_assert_eq!(BIT_MARKER.leading_zeros(), 0); - self.strong.fetch_add(BIT_MARKER, Relaxed); - } - - pub fn is_leaked(&self) -> bool { - (self.strong.load(Acquire) as isize) < 0 - } -} diff --git a/common/src/static_cell.rs b/common/src/static_cell.rs deleted file mode 100644 index 01a54db29c5..00000000000 --- a/common/src/static_cell.rs +++ /dev/null @@ -1,116 +0,0 @@ -#[cfg(not(feature = "threading"))] -mod non_threading { - use crate::lock::OnceCell; - use std::thread::LocalKey; - - pub struct StaticCell<T: 'static> { - inner: &'static LocalKey<OnceCell<&'static T>>, - } - - fn leak<T>(x: T) -> &'static T { - Box::leak(Box::new(x)) - } - - impl<T> StaticCell<T> { - #[doc(hidden)] - pub const fn _from_localkey(inner: &'static LocalKey<OnceCell<&'static T>>) -> Self { - Self { inner } - } - - pub fn get(&'static self) -> Option<&'static T> { - self.inner.with(|x| x.get().copied()) - } - - pub fn set(&'static self, value: T) -> Result<(), T> { - // thread-safe because it's a unsync::OnceCell - self.inner.with(|x| { - if x.get().is_some() { - Err(value) - } else { - // will never fail - let _ = x.set(leak(value)); - Ok(()) - } - }) - } - - pub fn get_or_init<F>(&'static self, f: F) -> &'static T - where - F: FnOnce() -> T, - { - self.inner.with(|x| *x.get_or_init(|| leak(f()))) - } - - pub fn get_or_try_init<F, E>(&'static self, f: F) -> Result<&'static T, E> - where - F: FnOnce() -> Result<T, E>, - { - self.inner - .with(|x| x.get_or_try_init(|| f().map(leak)).map(|&x| x)) - } - } - - #[macro_export] - macro_rules! static_cell { - ($($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty;)+) => { - $($(#[$attr])* - $vis static $name: $crate::static_cell::StaticCell<$t> = { - ::std::thread_local! { - $vis static $name: $crate::lock::OnceCell<&'static $t> = $crate::lock::OnceCell::new(); - } - $crate::static_cell::StaticCell::_from_localkey(&$name) - };)+ - }; - } -} -#[cfg(not(feature = "threading"))] -pub use non_threading::*; - -#[cfg(feature = "threading")] -mod threading { - use crate::lock::OnceCell; - - pub struct StaticCell<T: 'static> { - inner: OnceCell<T>, - } - - impl<T> StaticCell<T> { - #[doc(hidden)] - pub const fn _from_oncecell(inner: OnceCell<T>) -> Self { - Self { inner } - } - - pub fn get(&'static self) -> Option<&'static T> { - self.inner.get() - } - - pub fn set(&'static self, value: T) -> Result<(), T> { - self.inner.set(value) - } - - pub fn get_or_init<F>(&'static self, f: F) -> &'static T - where - F: FnOnce() -> T, - { - self.inner.get_or_init(f) - } - - pub fn get_or_try_init<F, E>(&'static self, f: F) -> Result<&'static T, E> - where - F: FnOnce() -> Result<T, E>, - { - self.inner.get_or_try_init(f) - } - } - - #[macro_export] - macro_rules! static_cell { - ($($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty;)+) => { - $($(#[$attr])* - $vis static $name: $crate::static_cell::StaticCell<$t> = - $crate::static_cell::StaticCell::_from_oncecell($crate::lock::OnceCell::new());)+ - }; - } -} -#[cfg(feature = "threading")] -pub use threading::*; diff --git a/common/src/str.rs b/common/src/str.rs deleted file mode 100644 index cdee03f14fe..00000000000 --- a/common/src/str.rs +++ /dev/null @@ -1,360 +0,0 @@ -use crate::{ - atomic::{PyAtomic, Radium}, - hash::PyHash, -}; -use ascii::AsciiString; -use rustpython_format::CharLen; -use std::ops::{Bound, RangeBounds}; - -#[cfg(not(target_arch = "wasm32"))] -#[allow(non_camel_case_types)] -pub type wchar_t = libc::wchar_t; -#[cfg(target_arch = "wasm32")] -#[allow(non_camel_case_types)] -pub type wchar_t = u32; - -/// Utf8 + state.ascii (+ PyUnicode_Kind in future) -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum PyStrKind { - Ascii, - Utf8, -} - -impl std::ops::BitOr for PyStrKind { - type Output = Self; - fn bitor(self, other: Self) -> Self { - match (self, other) { - (Self::Ascii, Self::Ascii) => Self::Ascii, - _ => Self::Utf8, - } - } -} - -impl PyStrKind { - #[inline] - pub fn new_data(self) -> PyStrKindData { - match self { - PyStrKind::Ascii => PyStrKindData::Ascii, - PyStrKind::Utf8 => PyStrKindData::Utf8(Radium::new(usize::MAX)), - } - } -} - -#[derive(Debug)] -pub enum PyStrKindData { - Ascii, - // uses usize::MAX as a sentinel for "uncomputed" - Utf8(PyAtomic<usize>), -} - -impl PyStrKindData { - #[inline] - pub fn kind(&self) -> PyStrKind { - match self { - PyStrKindData::Ascii => PyStrKind::Ascii, - PyStrKindData::Utf8(_) => PyStrKind::Utf8, - } - } -} - -pub struct BorrowedStr<'a> { - bytes: &'a [u8], - kind: PyStrKindData, - #[allow(dead_code)] - hash: PyAtomic<PyHash>, -} - -impl<'a> BorrowedStr<'a> { - /// # Safety - /// `s` have to be an ascii string - #[inline] - pub unsafe fn from_ascii_unchecked(s: &'a [u8]) -> Self { - debug_assert!(s.is_ascii()); - Self { - bytes: s, - kind: PyStrKind::Ascii.new_data(), - hash: PyAtomic::<PyHash>::new(0), - } - } - - #[inline] - pub fn from_bytes(s: &'a [u8]) -> Self { - let k = if s.is_ascii() { - PyStrKind::Ascii.new_data() - } else { - PyStrKind::Utf8.new_data() - }; - Self { - bytes: s, - kind: k, - hash: PyAtomic::<PyHash>::new(0), - } - } - - #[inline] - pub fn as_str(&self) -> &str { - unsafe { - // SAFETY: Both PyStrKind::{Ascii, Utf8} are valid utf8 string - std::str::from_utf8_unchecked(self.bytes) - } - } - - #[inline] - pub fn char_len(&self) -> usize { - match self.kind { - PyStrKindData::Ascii => self.bytes.len(), - PyStrKindData::Utf8(ref len) => match len.load(core::sync::atomic::Ordering::Relaxed) { - usize::MAX => self._compute_char_len(), - len => len, - }, - } - } - - #[cold] - fn _compute_char_len(&self) -> usize { - match self.kind { - PyStrKindData::Utf8(ref char_len) => { - let len = self.as_str().chars().count(); - // len cannot be usize::MAX, since vec.capacity() < sys.maxsize - char_len.store(len, core::sync::atomic::Ordering::Relaxed); - len - } - _ => unsafe { - debug_assert!(false); // invalid for non-utf8 strings - std::hint::unreachable_unchecked() - }, - } - } -} - -impl std::ops::Deref for BorrowedStr<'_> { - type Target = str; - fn deref(&self) -> &str { - self.as_str() - } -} - -impl std::fmt::Display for BorrowedStr<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.as_str().fmt(f) - } -} - -impl CharLen for BorrowedStr<'_> { - fn char_len(&self) -> usize { - self.char_len() - } -} - -pub fn try_get_chars(s: &str, range: impl RangeBounds<usize>) -> Option<&str> { - let mut chars = s.chars(); - let start = match range.start_bound() { - Bound::Included(&i) => i, - Bound::Excluded(&i) => i + 1, - Bound::Unbounded => 0, - }; - for _ in 0..start { - chars.next()?; - } - let s = chars.as_str(); - let range_len = match range.end_bound() { - Bound::Included(&i) => i + 1 - start, - Bound::Excluded(&i) => i - start, - Bound::Unbounded => return Some(s), - }; - char_range_end(s, range_len).map(|end| &s[..end]) -} - -pub fn get_chars(s: &str, range: impl RangeBounds<usize>) -> &str { - try_get_chars(s, range).unwrap() -} - -#[inline] -pub fn char_range_end(s: &str, nchars: usize) -> Option<usize> { - let i = match nchars.checked_sub(1) { - Some(last_char_index) => { - let (index, c) = s.char_indices().nth(last_char_index)?; - index + c.len_utf8() - } - None => 0, - }; - Some(i) -} - -pub fn zfill(bytes: &[u8], width: usize) -> Vec<u8> { - if width <= bytes.len() { - bytes.to_vec() - } else { - let (sign, s) = match bytes.first() { - Some(_sign @ b'+') | Some(_sign @ b'-') => { - (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]) - } - _ => (&b""[..], bytes), - }; - let mut filled = Vec::new(); - filled.extend_from_slice(sign); - filled.extend(std::iter::repeat(b'0').take(width - bytes.len())); - filled.extend_from_slice(s); - filled - } -} - -/// Convert a string to ascii compatible, escaping unicodes into escape -/// sequences. -pub fn to_ascii(value: &str) -> AsciiString { - let mut ascii = Vec::new(); - for c in value.chars() { - if c.is_ascii() { - ascii.push(c as u8); - } else { - let c = c as i64; - let hex = if c < 0x100 { - format!("\\x{c:02x}") - } else if c < 0x10000 { - format!("\\u{c:04x}") - } else { - format!("\\U{c:08x}") - }; - ascii.append(&mut hex.into_bytes()); - } - } - unsafe { AsciiString::from_ascii_unchecked(ascii) } -} - -pub mod levenshtein { - use std::{cell::RefCell, thread_local}; - - pub const MOVE_COST: usize = 2; - const CASE_COST: usize = 1; - const MAX_STRING_SIZE: usize = 40; - - fn substitution_cost(mut a: u8, mut b: u8) -> usize { - if (a & 31) != (b & 31) { - return MOVE_COST; - } - if a == b { - return 0; - } - if a.is_ascii_uppercase() { - a += b'a' - b'A'; - } - if b.is_ascii_uppercase() { - b += b'a' - b'A'; - } - if a == b { - CASE_COST - } else { - MOVE_COST - } - } - - pub fn levenshtein_distance(a: &str, b: &str, max_cost: usize) -> usize { - thread_local! { - static BUFFER: RefCell<[usize; MAX_STRING_SIZE]> = RefCell::new([0usize; MAX_STRING_SIZE]); - } - - if a == b { - return 0; - } - - let (mut a_bytes, mut b_bytes) = (a.as_bytes(), b.as_bytes()); - let (mut a_begin, mut a_end) = (0usize, a.len()); - let (mut b_begin, mut b_end) = (0usize, b.len()); - - while a_end > 0 && b_end > 0 && (a_bytes[a_begin] == b_bytes[b_begin]) { - a_begin += 1; - b_begin += 1; - a_end -= 1; - b_end -= 1; - } - while a_end > 0 - && b_end > 0 - && (a_bytes[a_begin + a_end - 1] == b_bytes[b_begin + b_end - 1]) - { - a_end -= 1; - b_end -= 1; - } - if a_end == 0 || b_end == 0 { - return (a_end + b_end) * MOVE_COST; - } - if a_end > MAX_STRING_SIZE || b_end > MAX_STRING_SIZE { - return max_cost + 1; - } - - if b_end < a_end { - std::mem::swap(&mut a_bytes, &mut b_bytes); - std::mem::swap(&mut a_begin, &mut b_begin); - std::mem::swap(&mut a_end, &mut b_end); - } - - if (b_end - a_end) * MOVE_COST > max_cost { - return max_cost + 1; - } - - BUFFER.with(|buffer| { - let mut buffer = buffer.borrow_mut(); - for i in 0..a_end { - buffer[i] = (i + 1) * MOVE_COST; - } - - let mut result = 0usize; - for (b_index, b_code) in b_bytes[b_begin..(b_begin + b_end)].iter().enumerate() { - result = b_index * MOVE_COST; - let mut distance = result; - let mut minimum = usize::MAX; - for (a_index, a_code) in a_bytes[a_begin..(a_begin + a_end)].iter().enumerate() { - let substitute = distance + substitution_cost(*b_code, *a_code); - distance = buffer[a_index]; - let insert_delete = usize::min(result, distance) + MOVE_COST; - result = usize::min(insert_delete, substitute); - - buffer[a_index] = result; - if result < minimum { - minimum = result; - } - } - if minimum > max_cost { - return max_cost + 1; - } - } - result - }) - } -} - -/// Creates an [`AsciiStr`][ascii::AsciiStr] from a string literal, throwing a compile error if the -/// literal isn't actually ascii. -/// -/// ```compile_fail -/// # use rustpython_common::str::ascii; -/// ascii!("I ❤️ Rust & Python"); -/// ``` -#[macro_export] -macro_rules! ascii { - ($x:literal) => {{ - const STR: &str = $x; - const _: () = if !STR.is_ascii() { - panic!("ascii!() argument is not an ascii string"); - }; - unsafe { $crate::vendored::ascii::AsciiStr::from_ascii_unchecked(STR.as_bytes()) } - }}; -} -pub use ascii; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_get_chars() { - let s = "0123456789"; - assert_eq!(get_chars(s, 3..7), "3456"); - assert_eq!(get_chars(s, 3..7), &s[3..7]); - - let s = "0유니코드 문자열9"; - assert_eq!(get_chars(s, 3..7), "코드 문"); - - let s = "0😀😃😄😁😆😅😂🤣9"; - assert_eq!(get_chars(s, 3..7), "😄😁😆😅"); - } -} diff --git a/common/src/windows.rs b/common/src/windows.rs deleted file mode 100644 index e1f296c941d..00000000000 --- a/common/src/windows.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::{ - ffi::{OsStr, OsString}, - os::windows::ffi::{OsStrExt, OsStringExt}, -}; - -pub trait ToWideString { - fn to_wide(&self) -> Vec<u16>; - fn to_wides_with_nul(&self) -> Vec<u16>; -} -impl<T> ToWideString for T -where - T: AsRef<OsStr>, -{ - fn to_wide(&self) -> Vec<u16> { - self.as_ref().encode_wide().collect() - } - fn to_wides_with_nul(&self) -> Vec<u16> { - self.as_ref().encode_wide().chain(Some(0)).collect() - } -} - -pub trait FromWideString -where - Self: Sized, -{ - fn from_wides_until_nul(wide: &[u16]) -> Self; -} -impl FromWideString for OsString { - fn from_wides_until_nul(wide: &[u16]) -> OsString { - let len = wide.iter().take_while(|&&c| c != 0).count(); - OsString::from_wide(&wide[..len]) - } -} diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml deleted file mode 100644 index 8dec426b2b6..00000000000 --- a/compiler/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "rustpython-compiler" -version = "0.3.0" -description = "A usability wrapper around rustpython-parser and rustpython-compiler-core" -authors = ["RustPython Team"] -repository = "https://github.com/RustPython/RustPython" -license = "MIT" -edition = "2021" - -[dependencies] -rustpython-codegen = { workspace = true } -rustpython-compiler-core = { workspace = true } -rustpython-parser = { workspace = true } diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml deleted file mode 100644 index 8cfd485b82a..00000000000 --- a/compiler/codegen/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "rustpython-codegen" -version = "0.3.0" -description = "Compiler for python code into bytecode for the rustpython VM." -authors = ["RustPython Team"] -repository = "https://github.com/RustPython/RustPython" -license = "MIT" -edition = "2021" - -[dependencies] -rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] } -rustpython-parser-core = { workspace = true } -rustpython-compiler-core = { workspace = true } - -ahash = { workspace = true } -bitflags = { workspace = true } -indexmap = { workspace = true } -itertools = { workspace = true } -log = { workspace = true } -num-complex = { workspace = true } -num-traits = { workspace = true } - -[dev-dependencies] -rustpython-parser = { workspace = true } - -insta = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs deleted file mode 100644 index e5976047a3a..00000000000 --- a/compiler/codegen/src/compile.rs +++ /dev/null @@ -1,3018 +0,0 @@ -//! -//! Take an AST and transform it into bytecode -//! -//! Inspirational code: -//! <https://github.com/python/cpython/blob/main/Python/compile.c> -//! <https://github.com/micropython/micropython/blob/master/py/compile.c> - -#![deny(clippy::cast_possible_truncation)] - -use crate::{ - error::{CodegenError, CodegenErrorType}, - ir, - symboltable::{self, SymbolFlags, SymbolScope, SymbolTable}, - IndexSet, -}; -use itertools::Itertools; -use num_complex::Complex64; -use num_traits::ToPrimitive; -use rustpython_ast::located::{self as located_ast, Located}; -use rustpython_compiler_core::{ - bytecode::{self, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, OpArg, OpArgType}, - Mode, -}; -use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; -use std::borrow::Cow; - -type CompileResult<T> = Result<T, CodegenError>; - -#[derive(PartialEq, Eq, Clone, Copy)] -enum NameUsage { - Load, - Store, - Delete, -} - -enum CallType { - Positional { nargs: u32 }, - Keyword { nargs: u32 }, - Ex { has_kwargs: bool }, -} - -fn is_forbidden_name(name: &str) -> bool { - // See https://docs.python.org/3/library/constants.html#built-in-constants - const BUILTIN_CONSTANTS: &[&str] = &["__debug__"]; - - BUILTIN_CONSTANTS.contains(&name) -} - -/// Main structure holding the state of compilation. -struct Compiler { - code_stack: Vec<ir::CodeInfo>, - symbol_table_stack: Vec<SymbolTable>, - source_path: String, - current_source_location: SourceLocation, - qualified_path: Vec<String>, - done_with_future_stmts: bool, - future_annotations: bool, - ctx: CompileContext, - class_name: Option<String>, - opts: CompileOpts, -} - -#[derive(Debug, Clone, Default)] -pub struct CompileOpts { - /// How optimized the bytecode output should be; any optimize > 0 does - /// not emit assert statements - pub optimize: u8, -} - -#[derive(Debug, Clone, Copy)] -struct CompileContext { - loop_data: Option<(ir::BlockIdx, ir::BlockIdx)>, - in_class: bool, - func: FunctionContext, -} - -#[derive(Debug, Clone, Copy, PartialEq)] -enum FunctionContext { - NoFunction, - Function, - AsyncFunction, -} - -impl CompileContext { - fn in_func(self) -> bool { - self.func != FunctionContext::NoFunction - } -} - -/// Compile an located_ast::Mod produced from rustpython_parser::parse() -pub fn compile_top( - ast: &located_ast::Mod, - source_path: String, - mode: Mode, - opts: CompileOpts, -) -> CompileResult<CodeObject> { - match ast { - located_ast::Mod::Module(located_ast::ModModule { body, .. }) => { - compile_program(body, source_path, opts) - } - located_ast::Mod::Interactive(located_ast::ModInteractive { body, .. }) => match mode { - Mode::Single => compile_program_single(body, source_path, opts), - Mode::BlockExpr => compile_block_expression(body, source_path, opts), - _ => unreachable!("only Single and BlockExpr parsed to Interactive"), - }, - located_ast::Mod::Expression(located_ast::ModExpression { body, .. }) => { - compile_expression(body, source_path, opts) - } - located_ast::Mod::FunctionType(_) => panic!("can't compile a FunctionType"), - } -} - -/// A helper function for the shared code of the different compile functions -fn compile_impl<Ast: ?Sized>( - ast: &Ast, - source_path: String, - opts: CompileOpts, - make_symbol_table: impl FnOnce(&Ast) -> Result<SymbolTable, symboltable::SymbolTableError>, - compile: impl FnOnce(&mut Compiler, &Ast, SymbolTable) -> CompileResult<()>, -) -> CompileResult<CodeObject> { - let symbol_table = match make_symbol_table(ast) { - Ok(x) => x, - Err(e) => return Err(e.into_codegen_error(source_path)), - }; - - let mut compiler = Compiler::new(opts, source_path, "<module>".to_owned()); - compile(&mut compiler, ast, symbol_table)?; - let code = compiler.pop_code_object(); - trace!("Compilation completed: {:?}", code); - Ok(code) -} - -/// Compile a standard Python program to bytecode -pub fn compile_program( - ast: &[located_ast::Stmt], - source_path: String, - opts: CompileOpts, -) -> CompileResult<CodeObject> { - compile_impl( - ast, - source_path, - opts, - SymbolTable::scan_program, - Compiler::compile_program, - ) -} - -/// Compile a Python program to bytecode for the context of a REPL -pub fn compile_program_single( - ast: &[located_ast::Stmt], - source_path: String, - opts: CompileOpts, -) -> CompileResult<CodeObject> { - compile_impl( - ast, - source_path, - opts, - SymbolTable::scan_program, - Compiler::compile_program_single, - ) -} - -pub fn compile_block_expression( - ast: &[located_ast::Stmt], - source_path: String, - opts: CompileOpts, -) -> CompileResult<CodeObject> { - compile_impl( - ast, - source_path, - opts, - SymbolTable::scan_program, - Compiler::compile_block_expr, - ) -} - -pub fn compile_expression( - ast: &located_ast::Expr, - source_path: String, - opts: CompileOpts, -) -> CompileResult<CodeObject> { - compile_impl( - ast, - source_path, - opts, - SymbolTable::scan_expr, - Compiler::compile_eval, - ) -} - -macro_rules! emit { - ($c:expr, Instruction::$op:ident { $arg:ident$(,)? }$(,)?) => { - $c.emit_arg($arg, |x| Instruction::$op { $arg: x }) - }; - ($c:expr, Instruction::$op:ident { $arg:ident : $arg_val:expr $(,)? }$(,)?) => { - $c.emit_arg($arg_val, |x| Instruction::$op { $arg: x }) - }; - ($c:expr, Instruction::$op:ident( $arg_val:expr $(,)? )$(,)?) => { - $c.emit_arg($arg_val, Instruction::$op) - }; - ($c:expr, Instruction::$op:ident$(,)?) => { - $c.emit_no_arg(Instruction::$op) - }; -} - -impl Compiler { - fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self { - let module_code = ir::CodeInfo { - flags: bytecode::CodeFlags::NEW_LOCALS, - posonlyarg_count: 0, - arg_count: 0, - kwonlyarg_count: 0, - source_path: source_path.clone(), - first_line_number: LineNumber::MIN, - obj_name: code_name, - - blocks: vec![ir::Block::default()], - current_block: ir::BlockIdx(0), - constants: IndexSet::default(), - name_cache: IndexSet::default(), - varname_cache: IndexSet::default(), - cellvar_cache: IndexSet::default(), - freevar_cache: IndexSet::default(), - }; - Compiler { - code_stack: vec![module_code], - symbol_table_stack: Vec::new(), - source_path, - current_source_location: SourceLocation::default(), - qualified_path: Vec::new(), - done_with_future_stmts: false, - future_annotations: false, - ctx: CompileContext { - loop_data: None, - in_class: false, - func: FunctionContext::NoFunction, - }, - class_name: None, - opts, - } - } - - fn error(&mut self, error: CodegenErrorType) -> CodegenError { - self.error_loc(error, self.current_source_location) - } - fn error_loc(&mut self, error: CodegenErrorType, location: SourceLocation) -> CodegenError { - CodegenError { - error, - location: Some(location), - source_path: self.source_path.clone(), - } - } - - fn push_output( - &mut self, - flags: bytecode::CodeFlags, - posonlyarg_count: u32, - arg_count: u32, - kwonlyarg_count: u32, - obj_name: String, - ) { - let source_path = self.source_path.clone(); - let first_line_number = self.get_source_line_number(); - - let table = self - .symbol_table_stack - .last_mut() - .unwrap() - .sub_tables - .remove(0); - - let cellvar_cache = table - .symbols - .iter() - .filter(|(_, s)| s.scope == SymbolScope::Cell) - .map(|(var, _)| var.clone()) - .collect(); - let freevar_cache = table - .symbols - .iter() - .filter(|(_, s)| { - s.scope == SymbolScope::Free || s.flags.contains(SymbolFlags::FREE_CLASS) - }) - .map(|(var, _)| var.clone()) - .collect(); - - self.symbol_table_stack.push(table); - - let info = ir::CodeInfo { - flags, - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path, - first_line_number, - obj_name, - - blocks: vec![ir::Block::default()], - current_block: ir::BlockIdx(0), - constants: IndexSet::default(), - name_cache: IndexSet::default(), - varname_cache: IndexSet::default(), - cellvar_cache, - freevar_cache, - }; - self.code_stack.push(info); - } - - fn pop_code_object(&mut self) -> CodeObject { - let table = self.symbol_table_stack.pop().unwrap(); - assert!(table.sub_tables.is_empty()); - self.code_stack - .pop() - .unwrap() - .finalize_code(self.opts.optimize) - } - - // could take impl Into<Cow<str>>, but everything is borrowed from ast structs; we never - // actually have a `String` to pass - fn name(&mut self, name: &str) -> bytecode::NameIdx { - self._name_inner(name, |i| &mut i.name_cache) - } - fn varname(&mut self, name: &str) -> CompileResult<bytecode::NameIdx> { - if Compiler::is_forbidden_arg_name(name) { - return Err(self.error(CodegenErrorType::SyntaxError(format!( - "cannot assign to {name}", - )))); - } - Ok(self._name_inner(name, |i| &mut i.varname_cache)) - } - fn _name_inner( - &mut self, - name: &str, - cache: impl FnOnce(&mut ir::CodeInfo) -> &mut IndexSet<String>, - ) -> bytecode::NameIdx { - let name = self.mangle(name); - let cache = cache(self.current_code_info()); - cache - .get_index_of(name.as_ref()) - .unwrap_or_else(|| cache.insert_full(name.into_owned()).0) - .to_u32() - } - - fn compile_program( - &mut self, - body: &[located_ast::Stmt], - symbol_table: SymbolTable, - ) -> CompileResult<()> { - let size_before = self.code_stack.len(); - self.symbol_table_stack.push(symbol_table); - - let (doc, statements) = split_doc(body); - if let Some(value) = doc { - self.emit_constant(ConstantData::Str { value }); - let doc = self.name("__doc__"); - emit!(self, Instruction::StoreGlobal(doc)) - } - - if Self::find_ann(statements) { - emit!(self, Instruction::SetupAnnotation); - } - - self.compile_statements(statements)?; - - assert_eq!(self.code_stack.len(), size_before); - - // Emit None at end: - self.emit_constant(ConstantData::None); - emit!(self, Instruction::ReturnValue); - Ok(()) - } - - fn compile_program_single( - &mut self, - body: &[located_ast::Stmt], - symbol_table: SymbolTable, - ) -> CompileResult<()> { - self.symbol_table_stack.push(symbol_table); - - if let Some((last, body)) = body.split_last() { - for statement in body { - if let located_ast::Stmt::Expr(located_ast::StmtExpr { value, .. }) = &statement { - self.compile_expression(value)?; - emit!(self, Instruction::PrintExpr); - } else { - self.compile_statement(statement)?; - } - } - - if let located_ast::Stmt::Expr(located_ast::StmtExpr { value, .. }) = &last { - self.compile_expression(value)?; - emit!(self, Instruction::Duplicate); - emit!(self, Instruction::PrintExpr); - } else { - self.compile_statement(last)?; - self.emit_constant(ConstantData::None); - } - } else { - self.emit_constant(ConstantData::None); - }; - - emit!(self, Instruction::ReturnValue); - Ok(()) - } - - fn compile_block_expr( - &mut self, - body: &[located_ast::Stmt], - symbol_table: SymbolTable, - ) -> CompileResult<()> { - self.symbol_table_stack.push(symbol_table); - - self.compile_statements(body)?; - - if let Some(last_statement) = body.last() { - match last_statement { - located_ast::Stmt::Expr(_) => { - self.current_block().instructions.pop(); // pop Instruction::Pop - } - located_ast::Stmt::FunctionDef(_) - | located_ast::Stmt::AsyncFunctionDef(_) - | located_ast::Stmt::ClassDef(_) => { - let store_inst = self.current_block().instructions.pop().unwrap(); // pop Instruction::Store - emit!(self, Instruction::Duplicate); - self.current_block().instructions.push(store_inst); - } - _ => self.emit_constant(ConstantData::None), - } - } - emit!(self, Instruction::ReturnValue); - - Ok(()) - } - - // Compile statement in eval mode: - fn compile_eval( - &mut self, - expression: &located_ast::Expr, - symbol_table: SymbolTable, - ) -> CompileResult<()> { - self.symbol_table_stack.push(symbol_table); - self.compile_expression(expression)?; - emit!(self, Instruction::ReturnValue); - Ok(()) - } - - fn compile_statements(&mut self, statements: &[located_ast::Stmt]) -> CompileResult<()> { - for statement in statements { - self.compile_statement(statement)? - } - Ok(()) - } - - fn load_name(&mut self, name: &str) -> CompileResult<()> { - self.compile_name(name, NameUsage::Load) - } - - fn store_name(&mut self, name: &str) -> CompileResult<()> { - self.compile_name(name, NameUsage::Store) - } - - fn mangle<'a>(&self, name: &'a str) -> Cow<'a, str> { - symboltable::mangle_name(self.class_name.as_deref(), name) - } - - fn check_forbidden_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> { - let msg = match usage { - NameUsage::Store if is_forbidden_name(name) => "cannot assign to", - NameUsage::Delete if is_forbidden_name(name) => "cannot delete", - _ => return Ok(()), - }; - Err(self.error(CodegenErrorType::SyntaxError(format!("{msg} {name}")))) - } - - fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> { - let name = self.mangle(name); - - self.check_forbidden_name(&name, usage)?; - - let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = symbol_table.lookup(name.as_ref()).expect( - "The symbol must be present in the symbol table, even when it is undefined in python.", - ); - let info = self.code_stack.last_mut().unwrap(); - let mut cache = &mut info.name_cache; - enum NameOpType { - Fast, - Global, - Deref, - Local, - } - let op_typ = match symbol.scope { - SymbolScope::Local if self.ctx.in_func() => { - cache = &mut info.varname_cache; - NameOpType::Fast - } - SymbolScope::GlobalExplicit => NameOpType::Global, - SymbolScope::GlobalImplicit | SymbolScope::Unknown if self.ctx.in_func() => { - NameOpType::Global - } - SymbolScope::GlobalImplicit | SymbolScope::Unknown => NameOpType::Local, - SymbolScope::Local => NameOpType::Local, - SymbolScope::Free => { - cache = &mut info.freevar_cache; - NameOpType::Deref - } - SymbolScope::Cell => { - cache = &mut info.cellvar_cache; - NameOpType::Deref - } // TODO: is this right? - // SymbolScope::Unknown => NameOpType::Global, - }; - - if NameUsage::Load == usage && name == "__debug__" { - self.emit_constant(ConstantData::Boolean { - value: self.opts.optimize == 0, - }); - return Ok(()); - } - - let mut idx = cache - .get_index_of(name.as_ref()) - .unwrap_or_else(|| cache.insert_full(name.into_owned()).0); - if let SymbolScope::Free = symbol.scope { - idx += info.cellvar_cache.len(); - } - let op = match op_typ { - NameOpType::Fast => match usage { - NameUsage::Load => Instruction::LoadFast, - NameUsage::Store => Instruction::StoreFast, - NameUsage::Delete => Instruction::DeleteFast, - }, - NameOpType::Global => match usage { - NameUsage::Load => Instruction::LoadGlobal, - NameUsage::Store => Instruction::StoreGlobal, - NameUsage::Delete => Instruction::DeleteGlobal, - }, - NameOpType::Deref => match usage { - NameUsage::Load if !self.ctx.in_func() && self.ctx.in_class => { - Instruction::LoadClassDeref - } - NameUsage::Load => Instruction::LoadDeref, - NameUsage::Store => Instruction::StoreDeref, - NameUsage::Delete => Instruction::DeleteDeref, - }, - NameOpType::Local => match usage { - NameUsage::Load => Instruction::LoadNameAny, - NameUsage::Store => Instruction::StoreLocal, - NameUsage::Delete => Instruction::DeleteLocal, - }, - }; - self.emit_arg(idx.to_u32(), op); - - Ok(()) - } - - fn compile_statement(&mut self, statement: &located_ast::Stmt) -> CompileResult<()> { - use located_ast::*; - - trace!("Compiling {:?}", statement); - self.set_source_location(statement.location()); - - match &statement { - // we do this here because `from __future__` still executes that `from` statement at runtime, - // we still need to compile the ImportFrom down below - Stmt::ImportFrom(located_ast::StmtImportFrom { module, names, .. }) - if module.as_ref().map(|id| id.as_str()) == Some("__future__") => - { - self.compile_future_features(names)? - } - // if we find any other statement, stop accepting future statements - _ => self.done_with_future_stmts = true, - } - - match &statement { - Stmt::Import(StmtImport { names, .. }) => { - // import a, b, c as d - for name in names { - let name = &name; - self.emit_constant(ConstantData::Integer { - value: num_traits::Zero::zero(), - }); - self.emit_constant(ConstantData::None); - let idx = self.name(&name.name); - emit!(self, Instruction::ImportName { idx }); - if let Some(alias) = &name.asname { - for part in name.name.split('.').skip(1) { - let idx = self.name(part); - emit!(self, Instruction::LoadAttr { idx }); - } - self.store_name(alias.as_str())? - } else { - self.store_name(name.name.split('.').next().unwrap())? - } - } - } - Stmt::ImportFrom(StmtImportFrom { - level, - module, - names, - .. - }) => { - let import_star = names.iter().any(|n| &n.name == "*"); - - let from_list = if import_star { - if self.ctx.in_func() { - return Err(self.error_loc( - CodegenErrorType::FunctionImportStar, - statement.location(), - )); - } - vec![ConstantData::Str { - value: "*".to_owned(), - }] - } else { - names - .iter() - .map(|n| ConstantData::Str { - value: n.name.to_string(), - }) - .collect() - }; - - let module_idx = module.as_ref().map(|s| self.name(s.as_str())); - - // from .... import (*fromlist) - self.emit_constant(ConstantData::Integer { - value: level.as_ref().map_or(0, |level| level.to_u32()).into(), - }); - self.emit_constant(ConstantData::Tuple { - elements: from_list, - }); - if let Some(idx) = module_idx { - emit!(self, Instruction::ImportName { idx }); - } else { - emit!(self, Instruction::ImportNameless); - } - - if import_star { - // from .... import * - emit!(self, Instruction::ImportStar); - } else { - // from mod import a, b as c - - for name in names { - let name = &name; - let idx = self.name(name.name.as_str()); - // import symbol from module: - emit!(self, Instruction::ImportFrom { idx }); - - // Store module under proper name: - if let Some(alias) = &name.asname { - self.store_name(alias.as_str())? - } else { - self.store_name(name.name.as_str())? - } - } - - // Pop module from stack: - emit!(self, Instruction::Pop); - } - } - Stmt::Expr(StmtExpr { value, .. }) => { - self.compile_expression(value)?; - - // Pop result of stack, since we not use it: - emit!(self, Instruction::Pop); - } - Stmt::Global(_) | Stmt::Nonlocal(_) => { - // Handled during symbol table construction. - } - Stmt::If(StmtIf { - test, body, orelse, .. - }) => { - let after_block = self.new_block(); - if orelse.is_empty() { - // Only if: - self.compile_jump_if(test, false, after_block)?; - self.compile_statements(body)?; - } else { - // if - else: - let else_block = self.new_block(); - self.compile_jump_if(test, false, else_block)?; - self.compile_statements(body)?; - emit!( - self, - Instruction::Jump { - target: after_block, - } - ); - - // else: - self.switch_to_block(else_block); - self.compile_statements(orelse)?; - } - self.switch_to_block(after_block); - } - Stmt::While(StmtWhile { - test, body, orelse, .. - }) => self.compile_while(test, body, orelse)?, - Stmt::With(StmtWith { items, body, .. }) => self.compile_with(items, body, false)?, - Stmt::AsyncWith(StmtAsyncWith { items, body, .. }) => { - self.compile_with(items, body, true)? - } - Stmt::For(StmtFor { - target, - iter, - body, - orelse, - .. - }) => self.compile_for(target, iter, body, orelse, false)?, - Stmt::AsyncFor(StmtAsyncFor { - target, - iter, - body, - orelse, - .. - }) => self.compile_for(target, iter, body, orelse, true)?, - Stmt::Match(StmtMatch { subject, cases, .. }) => self.compile_match(subject, cases)?, - Stmt::Raise(StmtRaise { exc, cause, .. }) => { - let kind = match exc { - Some(value) => { - self.compile_expression(value)?; - match cause { - Some(cause) => { - self.compile_expression(cause)?; - bytecode::RaiseKind::RaiseCause - } - None => bytecode::RaiseKind::Raise, - } - } - None => bytecode::RaiseKind::Reraise, - }; - emit!(self, Instruction::Raise { kind }); - } - Stmt::Try(StmtTry { - body, - handlers, - orelse, - finalbody, - .. - }) => self.compile_try_statement(body, handlers, orelse, finalbody)?, - Stmt::TryStar(StmtTryStar { - body, - handlers, - orelse, - finalbody, - .. - }) => self.compile_try_star_statement(body, handlers, orelse, finalbody)?, - Stmt::FunctionDef(StmtFunctionDef { - name, - args, - body, - decorator_list, - returns, - .. - }) => self.compile_function_def( - name.as_str(), - args, - body, - decorator_list, - returns.as_deref(), - false, - )?, - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { - name, - args, - body, - decorator_list, - returns, - .. - }) => self.compile_function_def( - name.as_str(), - args, - body, - decorator_list, - returns.as_deref(), - true, - )?, - Stmt::ClassDef(StmtClassDef { - name, - body, - bases, - keywords, - decorator_list, - .. - }) => self.compile_class_def(name.as_str(), body, bases, keywords, decorator_list)?, - Stmt::Assert(StmtAssert { test, msg, .. }) => { - // if some flag, ignore all assert statements! - if self.opts.optimize == 0 { - let after_block = self.new_block(); - self.compile_jump_if(test, true, after_block)?; - - let assertion_error = self.name("AssertionError"); - emit!(self, Instruction::LoadGlobal(assertion_error)); - match msg { - Some(e) => { - self.compile_expression(e)?; - emit!(self, Instruction::CallFunctionPositional { nargs: 1 }); - } - None => { - emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); - } - } - emit!( - self, - Instruction::Raise { - kind: bytecode::RaiseKind::Raise, - } - ); - - self.switch_to_block(after_block); - } - } - Stmt::Break(_) => match self.ctx.loop_data { - Some((_, end)) => { - emit!(self, Instruction::Break { target: end }); - } - None => { - return Err( - self.error_loc(CodegenErrorType::InvalidBreak, statement.location()) - ); - } - }, - Stmt::Continue(_) => match self.ctx.loop_data { - Some((start, _)) => { - emit!(self, Instruction::Continue { target: start }); - } - None => { - return Err( - self.error_loc(CodegenErrorType::InvalidContinue, statement.location()) - ); - } - }, - Stmt::Return(StmtReturn { value, .. }) => { - if !self.ctx.in_func() { - return Err( - self.error_loc(CodegenErrorType::InvalidReturn, statement.location()) - ); - } - match value { - Some(v) => { - if self.ctx.func == FunctionContext::AsyncFunction - && self - .current_code_info() - .flags - .contains(bytecode::CodeFlags::IS_GENERATOR) - { - return Err(self.error_loc( - CodegenErrorType::AsyncReturnValue, - statement.location(), - )); - } - self.compile_expression(v)?; - } - None => { - self.emit_constant(ConstantData::None); - } - } - - emit!(self, Instruction::ReturnValue); - } - Stmt::Assign(StmtAssign { targets, value, .. }) => { - self.compile_expression(value)?; - - for (i, target) in targets.iter().enumerate() { - if i + 1 != targets.len() { - emit!(self, Instruction::Duplicate); - } - self.compile_store(target)?; - } - } - Stmt::AugAssign(StmtAugAssign { - target, op, value, .. - }) => self.compile_augassign(target, op, value)?, - Stmt::AnnAssign(StmtAnnAssign { - target, - annotation, - value, - .. - }) => self.compile_annotated_assign(target, annotation, value.as_deref())?, - Stmt::Delete(StmtDelete { targets, .. }) => { - for target in targets { - self.compile_delete(target)?; - } - } - Stmt::Pass(_) => { - // No need to emit any code here :) - } - Stmt::TypeAlias(_) => {} - } - Ok(()) - } - - fn compile_delete(&mut self, expression: &located_ast::Expr) -> CompileResult<()> { - match &expression { - located_ast::Expr::Name(located_ast::ExprName { id, .. }) => { - self.compile_name(id.as_str(), NameUsage::Delete)? - } - located_ast::Expr::Attribute(located_ast::ExprAttribute { value, attr, .. }) => { - self.check_forbidden_name(attr.as_str(), NameUsage::Delete)?; - self.compile_expression(value)?; - let idx = self.name(attr.as_str()); - emit!(self, Instruction::DeleteAttr { idx }); - } - located_ast::Expr::Subscript(located_ast::ExprSubscript { value, slice, .. }) => { - self.compile_expression(value)?; - self.compile_expression(slice)?; - emit!(self, Instruction::DeleteSubscript); - } - located_ast::Expr::Tuple(located_ast::ExprTuple { elts, .. }) - | located_ast::Expr::List(located_ast::ExprList { elts, .. }) => { - for element in elts { - self.compile_delete(element)?; - } - } - located_ast::Expr::BinOp(_) | located_ast::Expr::UnaryOp(_) => { - return Err(self.error(CodegenErrorType::Delete("expression"))) - } - _ => return Err(self.error(CodegenErrorType::Delete(expression.python_name()))), - } - Ok(()) - } - - fn enter_function( - &mut self, - name: &str, - args: &located_ast::Arguments, - ) -> CompileResult<bytecode::MakeFunctionFlags> { - let defaults: Vec<_> = args.defaults().collect(); - let have_defaults = !defaults.is_empty(); - if have_defaults { - // Construct a tuple: - let size = defaults.len().to_u32(); - for element in &defaults { - self.compile_expression(element)?; - } - emit!(self, Instruction::BuildTuple { size }); - } - - let (kw_without_defaults, kw_with_defaults) = args.split_kwonlyargs(); - if !kw_with_defaults.is_empty() { - let default_kw_count = kw_with_defaults.len(); - for (arg, default) in kw_with_defaults.iter() { - self.emit_constant(ConstantData::Str { - value: arg.arg.to_string(), - }); - self.compile_expression(default)?; - } - emit!( - self, - Instruction::BuildMap { - size: default_kw_count.to_u32(), - } - ); - } - - let mut func_flags = bytecode::MakeFunctionFlags::empty(); - if have_defaults { - func_flags |= bytecode::MakeFunctionFlags::DEFAULTS; - } - if !kw_with_defaults.is_empty() { - func_flags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS; - } - - self.push_output( - bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, - args.posonlyargs.len().to_u32(), - (args.posonlyargs.len() + args.args.len()).to_u32(), - args.kwonlyargs.len().to_u32(), - name.to_owned(), - ); - - let args_iter = std::iter::empty() - .chain(&args.posonlyargs) - .chain(&args.args) - .map(|arg| arg.as_arg()) - .chain(kw_without_defaults) - .chain(kw_with_defaults.into_iter().map(|(arg, _)| arg)); - for name in args_iter { - self.varname(name.arg.as_str())?; - } - - if let Some(name) = args.vararg.as_deref() { - self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARARGS; - self.varname(name.arg.as_str())?; - } - if let Some(name) = args.kwarg.as_deref() { - self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARKEYWORDS; - self.varname(name.arg.as_str())?; - } - - Ok(func_flags) - } - - fn prepare_decorators(&mut self, decorator_list: &[located_ast::Expr]) -> CompileResult<()> { - for decorator in decorator_list { - self.compile_expression(decorator)?; - } - Ok(()) - } - - fn apply_decorators(&mut self, decorator_list: &[located_ast::Expr]) { - // Apply decorators: - for _ in decorator_list { - emit!(self, Instruction::CallFunctionPositional { nargs: 1 }); - } - } - - fn compile_try_statement( - &mut self, - body: &[located_ast::Stmt], - handlers: &[located_ast::ExceptHandler], - orelse: &[located_ast::Stmt], - finalbody: &[located_ast::Stmt], - ) -> CompileResult<()> { - let handler_block = self.new_block(); - let finally_block = self.new_block(); - - // Setup a finally block if we have a finally statement. - if !finalbody.is_empty() { - emit!( - self, - Instruction::SetupFinally { - handler: finally_block, - } - ); - } - - let else_block = self.new_block(); - - // try: - emit!( - self, - Instruction::SetupExcept { - handler: handler_block, - } - ); - self.compile_statements(body)?; - emit!(self, Instruction::PopBlock); - emit!(self, Instruction::Jump { target: else_block }); - - // except handlers: - self.switch_to_block(handler_block); - // Exception is on top of stack now - for handler in handlers { - let located_ast::ExceptHandler::ExceptHandler( - located_ast::ExceptHandlerExceptHandler { - type_, name, body, .. - }, - ) = &handler; - let next_handler = self.new_block(); - - // If we gave a typ, - // check if this handler can handle the exception: - if let Some(exc_type) = type_ { - // Duplicate exception for test: - emit!(self, Instruction::Duplicate); - - // Check exception type: - self.compile_expression(exc_type)?; - emit!( - self, - Instruction::TestOperation { - op: bytecode::TestOperator::ExceptionMatch, - } - ); - - // We cannot handle this exception type: - emit!( - self, - Instruction::JumpIfFalse { - target: next_handler, - } - ); - - // We have a match, store in name (except x as y) - if let Some(alias) = name { - self.store_name(alias.as_str())? - } else { - // Drop exception from top of stack: - emit!(self, Instruction::Pop); - } - } else { - // Catch all! - // Drop exception from top of stack: - emit!(self, Instruction::Pop); - } - - // Handler code: - self.compile_statements(body)?; - emit!(self, Instruction::PopException); - - if !finalbody.is_empty() { - emit!(self, Instruction::PopBlock); // pop excepthandler block - // We enter the finally block, without exception. - emit!(self, Instruction::EnterFinally); - } - - emit!( - self, - Instruction::Jump { - target: finally_block, - } - ); - - // Emit a new label for the next handler - self.switch_to_block(next_handler); - } - - // If code flows here, we have an unhandled exception, - // raise the exception again! - emit!( - self, - Instruction::Raise { - kind: bytecode::RaiseKind::Reraise, - } - ); - - // We successfully ran the try block: - // else: - self.switch_to_block(else_block); - self.compile_statements(orelse)?; - - if !finalbody.is_empty() { - emit!(self, Instruction::PopBlock); // pop finally block - - // We enter the finallyhandler block, without return / exception. - emit!(self, Instruction::EnterFinally); - } - - // finally: - self.switch_to_block(finally_block); - if !finalbody.is_empty() { - self.compile_statements(finalbody)?; - emit!(self, Instruction::EndFinally); - } - - Ok(()) - } - - fn compile_try_star_statement( - &mut self, - _body: &[located_ast::Stmt], - _handlers: &[located_ast::ExceptHandler], - _orelse: &[located_ast::Stmt], - _finalbody: &[located_ast::Stmt], - ) -> CompileResult<()> { - Err(self.error(CodegenErrorType::NotImplementedYet)) - } - - fn is_forbidden_arg_name(name: &str) -> bool { - is_forbidden_name(name) - } - - fn compile_function_def( - &mut self, - name: &str, - args: &located_ast::Arguments, - body: &[located_ast::Stmt], - decorator_list: &[located_ast::Expr], - returns: Option<&located_ast::Expr>, // TODO: use type hint somehow.. - is_async: bool, - ) -> CompileResult<()> { - // Create bytecode for this function: - - self.prepare_decorators(decorator_list)?; - let mut func_flags = self.enter_function(name, args)?; - self.current_code_info() - .flags - .set(bytecode::CodeFlags::IS_COROUTINE, is_async); - - // remember to restore self.ctx.in_loop to the original after the function is compiled - let prev_ctx = self.ctx; - - self.ctx = CompileContext { - loop_data: None, - in_class: prev_ctx.in_class, - func: if is_async { - FunctionContext::AsyncFunction - } else { - FunctionContext::Function - }, - }; - - self.push_qualified_path(name); - let qualified_name = self.qualified_path.join("."); - self.push_qualified_path("<locals>"); - - let (doc_str, body) = split_doc(body); - - self.current_code_info() - .constants - .insert_full(ConstantData::None); - - self.compile_statements(body)?; - - // Emit None at end: - match body.last() { - Some(located_ast::Stmt::Return(_)) => { - // the last instruction is a ReturnValue already, we don't need to emit it - } - _ => { - self.emit_constant(ConstantData::None); - emit!(self, Instruction::ReturnValue); - } - } - - let code = self.pop_code_object(); - self.qualified_path.pop(); - self.qualified_path.pop(); - self.ctx = prev_ctx; - - // Prepare type annotations: - let mut num_annotations = 0; - - // Return annotation: - if let Some(annotation) = returns { - // key: - self.emit_constant(ConstantData::Str { - value: "return".to_owned(), - }); - // value: - self.compile_annotation(annotation)?; - num_annotations += 1; - } - - let args_iter = std::iter::empty() - .chain(&args.posonlyargs) - .chain(&args.args) - .chain(&args.kwonlyargs) - .map(|arg| arg.as_arg()) - .chain(args.vararg.as_deref()) - .chain(args.kwarg.as_deref()); - for arg in args_iter { - if let Some(annotation) = &arg.annotation { - self.emit_constant(ConstantData::Str { - value: self.mangle(arg.arg.as_str()).into_owned(), - }); - self.compile_annotation(annotation)?; - num_annotations += 1; - } - } - - if num_annotations > 0 { - func_flags |= bytecode::MakeFunctionFlags::ANNOTATIONS; - emit!( - self, - Instruction::BuildMap { - size: num_annotations, - } - ); - } - - if self.build_closure(&code) { - func_flags |= bytecode::MakeFunctionFlags::CLOSURE; - } - - self.emit_constant(ConstantData::Code { - code: Box::new(code), - }); - self.emit_constant(ConstantData::Str { - value: qualified_name, - }); - - // Turn code object into function object: - emit!(self, Instruction::MakeFunction(func_flags)); - - emit!(self, Instruction::Duplicate); - self.load_docstring(doc_str); - emit!(self, Instruction::Rotate2); - let doc = self.name("__doc__"); - emit!(self, Instruction::StoreAttr { idx: doc }); - - self.apply_decorators(decorator_list); - - self.store_name(name) - } - - fn build_closure(&mut self, code: &CodeObject) -> bool { - if code.freevars.is_empty() { - return false; - } - for var in &*code.freevars { - let table = self.symbol_table_stack.last().unwrap(); - let symbol = table.lookup(var).unwrap_or_else(|| { - panic!( - "couldn't look up var {} in {} in {}", - var, code.obj_name, self.source_path - ) - }); - let parent_code = self.code_stack.last().unwrap(); - let vars = match symbol.scope { - SymbolScope::Free => &parent_code.freevar_cache, - SymbolScope::Cell => &parent_code.cellvar_cache, - _ if symbol.flags.contains(SymbolFlags::FREE_CLASS) => &parent_code.freevar_cache, - x => unreachable!( - "var {} in a {:?} should be free or cell but it's {:?}", - var, table.typ, x - ), - }; - let mut idx = vars.get_index_of(var).unwrap(); - if let SymbolScope::Free = symbol.scope { - idx += parent_code.cellvar_cache.len(); - } - emit!(self, Instruction::LoadClosure(idx.to_u32())) - } - emit!( - self, - Instruction::BuildTuple { - size: code.freevars.len().to_u32(), - } - ); - true - } - - // Python/compile.c find_ann - fn find_ann(body: &[located_ast::Stmt]) -> bool { - use located_ast::*; - - for statement in body { - let res = match &statement { - Stmt::AnnAssign(_) => true, - Stmt::For(StmtFor { body, orelse, .. }) => { - Self::find_ann(body) || Self::find_ann(orelse) - } - Stmt::If(StmtIf { body, orelse, .. }) => { - Self::find_ann(body) || Self::find_ann(orelse) - } - Stmt::While(StmtWhile { body, orelse, .. }) => { - Self::find_ann(body) || Self::find_ann(orelse) - } - Stmt::With(StmtWith { body, .. }) => Self::find_ann(body), - Stmt::Try(StmtTry { - body, - orelse, - finalbody, - .. - }) => Self::find_ann(body) || Self::find_ann(orelse) || Self::find_ann(finalbody), - _ => false, - }; - if res { - return true; - } - } - false - } - - fn compile_class_def( - &mut self, - name: &str, - body: &[located_ast::Stmt], - bases: &[located_ast::Expr], - keywords: &[located_ast::Keyword], - decorator_list: &[located_ast::Expr], - ) -> CompileResult<()> { - self.prepare_decorators(decorator_list)?; - - emit!(self, Instruction::LoadBuildClass); - - let prev_ctx = self.ctx; - self.ctx = CompileContext { - func: FunctionContext::NoFunction, - in_class: true, - loop_data: None, - }; - - let prev_class_name = std::mem::replace(&mut self.class_name, Some(name.to_owned())); - - // Check if the class is declared global - let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = symbol_table.lookup(name.as_ref()).expect( - "The symbol must be present in the symbol table, even when it is undefined in python.", - ); - let mut global_path_prefix = Vec::new(); - if symbol.scope == SymbolScope::GlobalExplicit { - global_path_prefix.append(&mut self.qualified_path); - } - self.push_qualified_path(name); - let qualified_name = self.qualified_path.join("."); - - self.push_output(bytecode::CodeFlags::empty(), 0, 0, 0, name.to_owned()); - - let (doc_str, body) = split_doc(body); - - let dunder_name = self.name("__name__"); - emit!(self, Instruction::LoadGlobal(dunder_name)); - let dunder_module = self.name("__module__"); - emit!(self, Instruction::StoreLocal(dunder_module)); - self.emit_constant(ConstantData::Str { - value: qualified_name, - }); - let qualname = self.name("__qualname__"); - emit!(self, Instruction::StoreLocal(qualname)); - self.load_docstring(doc_str); - let doc = self.name("__doc__"); - emit!(self, Instruction::StoreLocal(doc)); - // setup annotations - if Self::find_ann(body) { - emit!(self, Instruction::SetupAnnotation); - } - self.compile_statements(body)?; - - let classcell_idx = self - .code_stack - .last_mut() - .unwrap() - .cellvar_cache - .iter() - .position(|var| *var == "__class__"); - - if let Some(classcell_idx) = classcell_idx { - emit!(self, Instruction::LoadClosure(classcell_idx.to_u32())); - emit!(self, Instruction::Duplicate); - let classcell = self.name("__classcell__"); - emit!(self, Instruction::StoreLocal(classcell)); - } else { - self.emit_constant(ConstantData::None); - } - - emit!(self, Instruction::ReturnValue); - - let code = self.pop_code_object(); - - self.class_name = prev_class_name; - self.qualified_path.pop(); - self.qualified_path.append(global_path_prefix.as_mut()); - self.ctx = prev_ctx; - - let mut func_flags = bytecode::MakeFunctionFlags::empty(); - - if self.build_closure(&code) { - func_flags |= bytecode::MakeFunctionFlags::CLOSURE; - } - - self.emit_constant(ConstantData::Code { - code: Box::new(code), - }); - self.emit_constant(ConstantData::Str { - value: name.to_owned(), - }); - - // Turn code object into function object: - emit!(self, Instruction::MakeFunction(func_flags)); - - self.emit_constant(ConstantData::Str { - value: name.to_owned(), - }); - - let call = self.compile_call_inner(2, bases, keywords)?; - self.compile_normal_call(call); - - self.apply_decorators(decorator_list); - - self.store_name(name) - } - - fn load_docstring(&mut self, doc_str: Option<String>) { - // TODO: __doc__ must be default None and no bytecode unless it is Some - // Duplicate top of stack (the function or class object) - - // Doc string value: - self.emit_constant(match doc_str { - Some(doc) => ConstantData::Str { value: doc }, - None => ConstantData::None, // set docstring None if not declared - }); - } - - fn compile_while( - &mut self, - test: &located_ast::Expr, - body: &[located_ast::Stmt], - orelse: &[located_ast::Stmt], - ) -> CompileResult<()> { - let while_block = self.new_block(); - let else_block = self.new_block(); - let after_block = self.new_block(); - - emit!(self, Instruction::SetupLoop); - self.switch_to_block(while_block); - - self.compile_jump_if(test, false, else_block)?; - - let was_in_loop = self.ctx.loop_data.replace((while_block, after_block)); - self.compile_statements(body)?; - self.ctx.loop_data = was_in_loop; - emit!( - self, - Instruction::Jump { - target: while_block, - } - ); - self.switch_to_block(else_block); - emit!(self, Instruction::PopBlock); - self.compile_statements(orelse)?; - self.switch_to_block(after_block); - Ok(()) - } - - fn compile_with( - &mut self, - items: &[located_ast::WithItem], - body: &[located_ast::Stmt], - is_async: bool, - ) -> CompileResult<()> { - let with_location = self.current_source_location; - - let Some((item, items)) = items.split_first() else { - return Err(self.error(CodegenErrorType::EmptyWithItems)); - }; - - let final_block = { - let final_block = self.new_block(); - self.compile_expression(&item.context_expr)?; - - self.set_source_location(with_location); - if is_async { - emit!(self, Instruction::BeforeAsyncWith); - emit!(self, Instruction::GetAwaitable); - self.emit_constant(ConstantData::None); - emit!(self, Instruction::YieldFrom); - emit!(self, Instruction::SetupAsyncWith { end: final_block }); - } else { - emit!(self, Instruction::SetupWith { end: final_block }); - } - - match &item.optional_vars { - Some(var) => { - self.set_source_location(var.location()); - self.compile_store(var)?; - } - None => { - emit!(self, Instruction::Pop); - } - } - final_block - }; - - if items.is_empty() { - if body.is_empty() { - return Err(self.error(CodegenErrorType::EmptyWithBody)); - } - self.compile_statements(body)?; - } else { - self.set_source_location(with_location); - self.compile_with(items, body, is_async)?; - } - - // sort of "stack up" the layers of with blocks: - // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a) - self.set_source_location(with_location); - emit!(self, Instruction::PopBlock); - - emit!(self, Instruction::EnterFinally); - - self.switch_to_block(final_block); - emit!(self, Instruction::WithCleanupStart); - - if is_async { - emit!(self, Instruction::GetAwaitable); - self.emit_constant(ConstantData::None); - emit!(self, Instruction::YieldFrom); - } - - emit!(self, Instruction::WithCleanupFinish); - - Ok(()) - } - - fn compile_for( - &mut self, - target: &located_ast::Expr, - iter: &located_ast::Expr, - body: &[located_ast::Stmt], - orelse: &[located_ast::Stmt], - is_async: bool, - ) -> CompileResult<()> { - // Start loop - let for_block = self.new_block(); - let else_block = self.new_block(); - let after_block = self.new_block(); - - emit!(self, Instruction::SetupLoop); - - // The thing iterated: - self.compile_expression(iter)?; - - if is_async { - emit!(self, Instruction::GetAIter); - - self.switch_to_block(for_block); - emit!( - self, - Instruction::SetupExcept { - handler: else_block, - } - ); - emit!(self, Instruction::GetANext); - self.emit_constant(ConstantData::None); - emit!(self, Instruction::YieldFrom); - self.compile_store(target)?; - emit!(self, Instruction::PopBlock); - } else { - // Retrieve Iterator - emit!(self, Instruction::GetIter); - - self.switch_to_block(for_block); - emit!(self, Instruction::ForIter { target: else_block }); - - // Start of loop iteration, set targets: - self.compile_store(target)?; - }; - - let was_in_loop = self.ctx.loop_data.replace((for_block, after_block)); - self.compile_statements(body)?; - self.ctx.loop_data = was_in_loop; - emit!(self, Instruction::Jump { target: for_block }); - - self.switch_to_block(else_block); - if is_async { - emit!(self, Instruction::EndAsyncFor); - } - emit!(self, Instruction::PopBlock); - self.compile_statements(orelse)?; - - self.switch_to_block(after_block); - - Ok(()) - } - - fn compile_match( - &mut self, - subject: &located_ast::Expr, - cases: &[located_ast::MatchCase], - ) -> CompileResult<()> { - eprintln!("match subject: {subject:?}"); - eprintln!("match cases: {cases:?}"); - Err(self.error(CodegenErrorType::NotImplementedYet)) - } - - fn compile_chained_comparison( - &mut self, - left: &located_ast::Expr, - ops: &[located_ast::CmpOp], - exprs: &[located_ast::Expr], - ) -> CompileResult<()> { - assert!(!ops.is_empty()); - assert_eq!(exprs.len(), ops.len()); - let (last_op, mid_ops) = ops.split_last().unwrap(); - let (last_val, mid_exprs) = exprs.split_last().unwrap(); - - use bytecode::ComparisonOperator::*; - use bytecode::TestOperator::*; - let compile_cmpop = |c: &mut Self, op: &located_ast::CmpOp| match op { - located_ast::CmpOp::Eq => emit!(c, Instruction::CompareOperation { op: Equal }), - located_ast::CmpOp::NotEq => emit!(c, Instruction::CompareOperation { op: NotEqual }), - located_ast::CmpOp::Lt => emit!(c, Instruction::CompareOperation { op: Less }), - located_ast::CmpOp::LtE => emit!(c, Instruction::CompareOperation { op: LessOrEqual }), - located_ast::CmpOp::Gt => emit!(c, Instruction::CompareOperation { op: Greater }), - located_ast::CmpOp::GtE => { - emit!(c, Instruction::CompareOperation { op: GreaterOrEqual }) - } - located_ast::CmpOp::In => emit!(c, Instruction::TestOperation { op: In }), - located_ast::CmpOp::NotIn => emit!(c, Instruction::TestOperation { op: NotIn }), - located_ast::CmpOp::Is => emit!(c, Instruction::TestOperation { op: Is }), - located_ast::CmpOp::IsNot => emit!(c, Instruction::TestOperation { op: IsNot }), - }; - - // a == b == c == d - // compile into (pseudo code): - // result = a == b - // if result: - // result = b == c - // if result: - // result = c == d - - // initialize lhs outside of loop - self.compile_expression(left)?; - - let end_blocks = if mid_exprs.is_empty() { - None - } else { - let break_block = self.new_block(); - let after_block = self.new_block(); - Some((break_block, after_block)) - }; - - // for all comparisons except the last (as the last one doesn't need a conditional jump) - for (op, val) in mid_ops.iter().zip(mid_exprs) { - self.compile_expression(val)?; - // store rhs for the next comparison in chain - emit!(self, Instruction::Duplicate); - emit!(self, Instruction::Rotate3); - - compile_cmpop(self, op); - - // if comparison result is false, we break with this value; if true, try the next one. - if let Some((break_block, _)) = end_blocks { - emit!( - self, - Instruction::JumpIfFalseOrPop { - target: break_block, - } - ); - } - } - - // handle the last comparison - self.compile_expression(last_val)?; - compile_cmpop(self, last_op); - - if let Some((break_block, after_block)) = end_blocks { - emit!( - self, - Instruction::Jump { - target: after_block, - } - ); - - // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. - self.switch_to_block(break_block); - emit!(self, Instruction::Rotate2); - emit!(self, Instruction::Pop); - - self.switch_to_block(after_block); - } - - Ok(()) - } - - fn compile_annotation(&mut self, annotation: &located_ast::Expr) -> CompileResult<()> { - if self.future_annotations { - self.emit_constant(ConstantData::Str { - value: annotation.to_string(), - }); - } else { - self.compile_expression(annotation)?; - } - Ok(()) - } - - fn compile_annotated_assign( - &mut self, - target: &located_ast::Expr, - annotation: &located_ast::Expr, - value: Option<&located_ast::Expr>, - ) -> CompileResult<()> { - if let Some(value) = value { - self.compile_expression(value)?; - self.compile_store(target)?; - } - - // Annotations are only evaluated in a module or class. - if self.ctx.in_func() { - return Ok(()); - } - - // Compile annotation: - self.compile_annotation(annotation)?; - - if let located_ast::Expr::Name(located_ast::ExprName { id, .. }) = &target { - // Store as dict entry in __annotations__ dict: - let annotations = self.name("__annotations__"); - emit!(self, Instruction::LoadNameAny(annotations)); - self.emit_constant(ConstantData::Str { - value: self.mangle(id.as_str()).into_owned(), - }); - emit!(self, Instruction::StoreSubscript); - } else { - // Drop annotation if not assigned to simple identifier. - emit!(self, Instruction::Pop); - } - - Ok(()) - } - - fn compile_store(&mut self, target: &located_ast::Expr) -> CompileResult<()> { - match &target { - located_ast::Expr::Name(located_ast::ExprName { id, .. }) => { - self.store_name(id.as_str())? - } - located_ast::Expr::Subscript(located_ast::ExprSubscript { value, slice, .. }) => { - self.compile_expression(value)?; - self.compile_expression(slice)?; - emit!(self, Instruction::StoreSubscript); - } - located_ast::Expr::Attribute(located_ast::ExprAttribute { value, attr, .. }) => { - self.check_forbidden_name(attr.as_str(), NameUsage::Store)?; - self.compile_expression(value)?; - let idx = self.name(attr.as_str()); - emit!(self, Instruction::StoreAttr { idx }); - } - located_ast::Expr::List(located_ast::ExprList { elts, .. }) - | located_ast::Expr::Tuple(located_ast::ExprTuple { elts, .. }) => { - let mut seen_star = false; - - // Scan for star args: - for (i, element) in elts.iter().enumerate() { - if let located_ast::Expr::Starred(_) = &element { - if seen_star { - return Err(self.error(CodegenErrorType::MultipleStarArgs)); - } else { - seen_star = true; - let before = i; - let after = elts.len() - i - 1; - let (before, after) = (|| Some((before.to_u8()?, after.to_u8()?)))() - .ok_or_else(|| { - self.error_loc( - CodegenErrorType::TooManyStarUnpack, - target.location(), - ) - })?; - let args = bytecode::UnpackExArgs { before, after }; - emit!(self, Instruction::UnpackEx { args }); - } - } - } - - if !seen_star { - emit!( - self, - Instruction::UnpackSequence { - size: elts.len().to_u32(), - } - ); - } - - for element in elts { - if let located_ast::Expr::Starred(located_ast::ExprStarred { value, .. }) = - &element - { - self.compile_store(value)?; - } else { - self.compile_store(element)?; - } - } - } - _ => { - return Err(self.error(match target { - located_ast::Expr::Starred(_) => CodegenErrorType::SyntaxError( - "starred assignment target must be in a list or tuple".to_owned(), - ), - _ => CodegenErrorType::Assign(target.python_name()), - })); - } - } - - Ok(()) - } - - fn compile_augassign( - &mut self, - target: &located_ast::Expr, - op: &located_ast::Operator, - value: &located_ast::Expr, - ) -> CompileResult<()> { - enum AugAssignKind<'a> { - Name { id: &'a str }, - Subscript, - Attr { idx: bytecode::NameIdx }, - } - - let kind = match &target { - located_ast::Expr::Name(located_ast::ExprName { id, .. }) => { - let id = id.as_str(); - self.compile_name(id, NameUsage::Load)?; - AugAssignKind::Name { id } - } - located_ast::Expr::Subscript(located_ast::ExprSubscript { value, slice, .. }) => { - self.compile_expression(value)?; - self.compile_expression(slice)?; - emit!(self, Instruction::Duplicate2); - emit!(self, Instruction::Subscript); - AugAssignKind::Subscript - } - located_ast::Expr::Attribute(located_ast::ExprAttribute { value, attr, .. }) => { - let attr = attr.as_str(); - self.check_forbidden_name(attr, NameUsage::Store)?; - self.compile_expression(value)?; - emit!(self, Instruction::Duplicate); - let idx = self.name(attr); - emit!(self, Instruction::LoadAttr { idx }); - AugAssignKind::Attr { idx } - } - _ => { - return Err(self.error(CodegenErrorType::Assign(target.python_name()))); - } - }; - - self.compile_expression(value)?; - self.compile_op(op, true); - - match kind { - AugAssignKind::Name { id } => { - // stack: RESULT - self.compile_name(id, NameUsage::Store)?; - } - AugAssignKind::Subscript => { - // stack: CONTAINER SLICE RESULT - emit!(self, Instruction::Rotate3); - emit!(self, Instruction::StoreSubscript); - } - AugAssignKind::Attr { idx } => { - // stack: CONTAINER RESULT - emit!(self, Instruction::Rotate2); - emit!(self, Instruction::StoreAttr { idx }); - } - } - - Ok(()) - } - - fn compile_op(&mut self, op: &located_ast::Operator, inplace: bool) { - let op = match op { - located_ast::Operator::Add => bytecode::BinaryOperator::Add, - located_ast::Operator::Sub => bytecode::BinaryOperator::Subtract, - located_ast::Operator::Mult => bytecode::BinaryOperator::Multiply, - located_ast::Operator::MatMult => bytecode::BinaryOperator::MatrixMultiply, - located_ast::Operator::Div => bytecode::BinaryOperator::Divide, - located_ast::Operator::FloorDiv => bytecode::BinaryOperator::FloorDivide, - located_ast::Operator::Mod => bytecode::BinaryOperator::Modulo, - located_ast::Operator::Pow => bytecode::BinaryOperator::Power, - located_ast::Operator::LShift => bytecode::BinaryOperator::Lshift, - located_ast::Operator::RShift => bytecode::BinaryOperator::Rshift, - located_ast::Operator::BitOr => bytecode::BinaryOperator::Or, - located_ast::Operator::BitXor => bytecode::BinaryOperator::Xor, - located_ast::Operator::BitAnd => bytecode::BinaryOperator::And, - }; - if inplace { - emit!(self, Instruction::BinaryOperationInplace { op }) - } else { - emit!(self, Instruction::BinaryOperation { op }) - } - } - - /// Implement boolean short circuit evaluation logic. - /// https://en.wikipedia.org/wiki/Short-circuit_evaluation - /// - /// This means, in a boolean statement 'x and y' the variable y will - /// not be evaluated when x is false. - /// - /// The idea is to jump to a label if the expression is either true or false - /// (indicated by the condition parameter). - fn compile_jump_if( - &mut self, - expression: &located_ast::Expr, - condition: bool, - target_block: ir::BlockIdx, - ) -> CompileResult<()> { - // Compile expression for test, and jump to label if false - match &expression { - located_ast::Expr::BoolOp(located_ast::ExprBoolOp { op, values, .. }) => { - match op { - located_ast::BoolOp::And => { - if condition { - // If all values are true. - let end_block = self.new_block(); - let (last_value, values) = values.split_last().unwrap(); - - // If any of the values is false, we can short-circuit. - for value in values { - self.compile_jump_if(value, false, end_block)?; - } - - // It depends upon the last value now: will it be true? - self.compile_jump_if(last_value, true, target_block)?; - self.switch_to_block(end_block); - } else { - // If any value is false, the whole condition is false. - for value in values { - self.compile_jump_if(value, false, target_block)?; - } - } - } - located_ast::BoolOp::Or => { - if condition { - // If any of the values is true. - for value in values { - self.compile_jump_if(value, true, target_block)?; - } - } else { - // If all of the values are false. - let end_block = self.new_block(); - let (last_value, values) = values.split_last().unwrap(); - - // If any value is true, we can short-circuit: - for value in values { - self.compile_jump_if(value, true, end_block)?; - } - - // It all depends upon the last value now! - self.compile_jump_if(last_value, false, target_block)?; - self.switch_to_block(end_block); - } - } - } - } - located_ast::Expr::UnaryOp(located_ast::ExprUnaryOp { - op: located_ast::UnaryOp::Not, - operand, - .. - }) => { - self.compile_jump_if(operand, !condition, target_block)?; - } - _ => { - // Fall back case which always will work! - self.compile_expression(expression)?; - if condition { - emit!( - self, - Instruction::JumpIfTrue { - target: target_block, - } - ); - } else { - emit!( - self, - Instruction::JumpIfFalse { - target: target_block, - } - ); - } - } - } - Ok(()) - } - - /// Compile a boolean operation as an expression. - /// This means, that the last value remains on the stack. - fn compile_bool_op( - &mut self, - op: &located_ast::BoolOp, - values: &[located_ast::Expr], - ) -> CompileResult<()> { - let after_block = self.new_block(); - - let (last_value, values) = values.split_last().unwrap(); - for value in values { - self.compile_expression(value)?; - - match op { - located_ast::BoolOp::And => { - emit!( - self, - Instruction::JumpIfFalseOrPop { - target: after_block, - } - ); - } - located_ast::BoolOp::Or => { - emit!( - self, - Instruction::JumpIfTrueOrPop { - target: after_block, - } - ); - } - } - } - - // If all values did not qualify, take the value of the last value: - self.compile_expression(last_value)?; - self.switch_to_block(after_block); - Ok(()) - } - - fn compile_dict( - &mut self, - keys: &[Option<located_ast::Expr>], - values: &[located_ast::Expr], - ) -> CompileResult<()> { - let mut size = 0; - let (packed, unpacked): (Vec<_>, Vec<_>) = keys - .iter() - .zip(values.iter()) - .partition(|(k, _)| k.is_some()); - for (key, value) in packed { - self.compile_expression(key.as_ref().unwrap())?; - self.compile_expression(value)?; - size += 1; - } - emit!(self, Instruction::BuildMap { size }); - - for (_, value) in unpacked { - self.compile_expression(value)?; - emit!(self, Instruction::DictUpdate); - } - - Ok(()) - } - - fn compile_expression(&mut self, expression: &located_ast::Expr) -> CompileResult<()> { - use located_ast::*; - trace!("Compiling {:?}", expression); - let location = expression.location(); - self.set_source_location(location); - - match &expression { - Expr::Call(ExprCall { - func, - args, - keywords, - .. - }) => self.compile_call(func, args, keywords)?, - Expr::BoolOp(ExprBoolOp { op, values, .. }) => self.compile_bool_op(op, values)?, - Expr::BinOp(ExprBinOp { - left, op, right, .. - }) => { - self.compile_expression(left)?; - self.compile_expression(right)?; - - // Perform operation: - self.compile_op(op, false); - } - Expr::Subscript(ExprSubscript { value, slice, .. }) => { - self.compile_expression(value)?; - self.compile_expression(slice)?; - emit!(self, Instruction::Subscript); - } - Expr::UnaryOp(ExprUnaryOp { op, operand, .. }) => { - self.compile_expression(operand)?; - - // Perform operation: - let op = match op { - UnaryOp::UAdd => bytecode::UnaryOperator::Plus, - UnaryOp::USub => bytecode::UnaryOperator::Minus, - UnaryOp::Not => bytecode::UnaryOperator::Not, - UnaryOp::Invert => bytecode::UnaryOperator::Invert, - }; - emit!(self, Instruction::UnaryOperation { op }); - } - Expr::Attribute(ExprAttribute { value, attr, .. }) => { - self.compile_expression(value)?; - let idx = self.name(attr.as_str()); - emit!(self, Instruction::LoadAttr { idx }); - } - Expr::Compare(ExprCompare { - left, - ops, - comparators, - .. - }) => { - self.compile_chained_comparison(left, ops, comparators)?; - } - Expr::Constant(ExprConstant { value, .. }) => { - self.emit_constant(compile_constant(value)); - } - Expr::List(ExprList { elts, .. }) => { - let (size, unpack) = self.gather_elements(0, elts)?; - if unpack { - emit!(self, Instruction::BuildListUnpack { size }); - } else { - emit!(self, Instruction::BuildList { size }); - } - } - Expr::Tuple(ExprTuple { elts, .. }) => { - let (size, unpack) = self.gather_elements(0, elts)?; - if unpack { - emit!(self, Instruction::BuildTupleUnpack { size }); - } else { - emit!(self, Instruction::BuildTuple { size }); - } - } - Expr::Set(ExprSet { elts, .. }) => { - let (size, unpack) = self.gather_elements(0, elts)?; - if unpack { - emit!(self, Instruction::BuildSetUnpack { size }); - } else { - emit!(self, Instruction::BuildSet { size }); - } - } - Expr::Dict(ExprDict { keys, values, .. }) => { - self.compile_dict(keys, values)?; - } - Expr::Slice(ExprSlice { - lower, upper, step, .. - }) => { - let mut compile_bound = |bound: Option<&located_ast::Expr>| match bound { - Some(exp) => self.compile_expression(exp), - None => { - self.emit_constant(ConstantData::None); - Ok(()) - } - }; - compile_bound(lower.as_deref())?; - compile_bound(upper.as_deref())?; - if let Some(step) = step { - self.compile_expression(step)?; - } - let step = step.is_some(); - emit!(self, Instruction::BuildSlice { step }); - } - Expr::Yield(ExprYield { value, .. }) => { - if !self.ctx.in_func() { - return Err(self.error(CodegenErrorType::InvalidYield)); - } - self.mark_generator(); - match value { - Some(expression) => self.compile_expression(expression)?, - Option::None => self.emit_constant(ConstantData::None), - }; - emit!(self, Instruction::YieldValue); - } - Expr::Await(ExprAwait { value, .. }) => { - if self.ctx.func != FunctionContext::AsyncFunction { - return Err(self.error(CodegenErrorType::InvalidAwait)); - } - self.compile_expression(value)?; - emit!(self, Instruction::GetAwaitable); - self.emit_constant(ConstantData::None); - emit!(self, Instruction::YieldFrom); - } - Expr::YieldFrom(ExprYieldFrom { value, .. }) => { - match self.ctx.func { - FunctionContext::NoFunction => { - return Err(self.error(CodegenErrorType::InvalidYieldFrom)); - } - FunctionContext::AsyncFunction => { - return Err(self.error(CodegenErrorType::AsyncYieldFrom)); - } - FunctionContext::Function => {} - } - self.mark_generator(); - self.compile_expression(value)?; - emit!(self, Instruction::GetIter); - self.emit_constant(ConstantData::None); - emit!(self, Instruction::YieldFrom); - } - Expr::JoinedStr(ExprJoinedStr { values, .. }) => { - if let Some(value) = try_get_constant_string(values) { - self.emit_constant(ConstantData::Str { value }) - } else { - for value in values { - self.compile_expression(value)?; - } - emit!( - self, - Instruction::BuildString { - size: values.len().to_u32(), - } - ) - } - } - Expr::FormattedValue(ExprFormattedValue { - value, - conversion, - format_spec, - .. - }) => { - match format_spec { - Some(spec) => self.compile_expression(spec)?, - None => self.emit_constant(ConstantData::Str { - value: String::new(), - }), - }; - self.compile_expression(value)?; - emit!( - self, - Instruction::FormatValue { - conversion: *conversion, - }, - ); - } - Expr::Name(located_ast::ExprName { id, .. }) => self.load_name(id.as_str())?, - Expr::Lambda(located_ast::ExprLambda { args, body, .. }) => { - let prev_ctx = self.ctx; - - let name = "<lambda>".to_owned(); - let mut func_flags = self.enter_function(&name, args)?; - - self.ctx = CompileContext { - loop_data: Option::None, - in_class: prev_ctx.in_class, - func: FunctionContext::Function, - }; - - self.current_code_info() - .constants - .insert_full(ConstantData::None); - - self.compile_expression(body)?; - emit!(self, Instruction::ReturnValue); - let code = self.pop_code_object(); - if self.build_closure(&code) { - func_flags |= bytecode::MakeFunctionFlags::CLOSURE; - } - self.emit_constant(ConstantData::Code { - code: Box::new(code), - }); - self.emit_constant(ConstantData::Str { value: name }); - // Turn code object into function object: - emit!(self, Instruction::MakeFunction(func_flags)); - - self.ctx = prev_ctx; - } - Expr::ListComp(located_ast::ExprListComp { - elt, generators, .. - }) => { - self.compile_comprehension( - "<listcomp>", - Some(Instruction::BuildList { - size: OpArgMarker::marker(), - }), - generators, - &|compiler| { - compiler.compile_comprehension_element(elt)?; - emit!( - compiler, - Instruction::ListAppend { - i: generators.len().to_u32(), - } - ); - Ok(()) - }, - )?; - } - Expr::SetComp(located_ast::ExprSetComp { - elt, generators, .. - }) => { - self.compile_comprehension( - "<setcomp>", - Some(Instruction::BuildSet { - size: OpArgMarker::marker(), - }), - generators, - &|compiler| { - compiler.compile_comprehension_element(elt)?; - emit!( - compiler, - Instruction::SetAdd { - i: generators.len().to_u32(), - } - ); - Ok(()) - }, - )?; - } - Expr::DictComp(located_ast::ExprDictComp { - key, - value, - generators, - .. - }) => { - self.compile_comprehension( - "<dictcomp>", - Some(Instruction::BuildMap { - size: OpArgMarker::marker(), - }), - generators, - &|compiler| { - // changed evaluation order for Py38 named expression PEP 572 - compiler.compile_expression(key)?; - compiler.compile_expression(value)?; - - emit!( - compiler, - Instruction::MapAdd { - i: generators.len().to_u32(), - } - ); - - Ok(()) - }, - )?; - } - Expr::GeneratorExp(located_ast::ExprGeneratorExp { - elt, generators, .. - }) => { - self.compile_comprehension("<genexpr>", None, generators, &|compiler| { - compiler.compile_comprehension_element(elt)?; - compiler.mark_generator(); - emit!(compiler, Instruction::YieldValue); - emit!(compiler, Instruction::Pop); - - Ok(()) - })?; - } - Expr::Starred(_) => { - return Err(self.error(CodegenErrorType::InvalidStarExpr)); - } - Expr::IfExp(located_ast::ExprIfExp { - test, body, orelse, .. - }) => { - let else_block = self.new_block(); - let after_block = self.new_block(); - self.compile_jump_if(test, false, else_block)?; - - // True case - self.compile_expression(body)?; - emit!( - self, - Instruction::Jump { - target: after_block, - } - ); - - // False case - self.switch_to_block(else_block); - self.compile_expression(orelse)?; - - // End - self.switch_to_block(after_block); - } - - Expr::NamedExpr(located_ast::ExprNamedExpr { - target, - value, - range: _, - }) => { - self.compile_expression(value)?; - emit!(self, Instruction::Duplicate); - self.compile_store(target)?; - } - } - Ok(()) - } - - fn compile_keywords(&mut self, keywords: &[located_ast::Keyword]) -> CompileResult<()> { - let mut size = 0; - let groupby = keywords.iter().group_by(|e| e.arg.is_none()); - for (is_unpacking, sub_keywords) in &groupby { - if is_unpacking { - for keyword in sub_keywords { - self.compile_expression(&keyword.value)?; - size += 1; - } - } else { - let mut sub_size = 0; - for keyword in sub_keywords { - if let Some(name) = &keyword.arg { - self.emit_constant(ConstantData::Str { - value: name.to_string(), - }); - self.compile_expression(&keyword.value)?; - sub_size += 1; - } - } - emit!(self, Instruction::BuildMap { size: sub_size }); - size += 1; - } - } - if size > 1 { - emit!(self, Instruction::BuildMapForCall { size }); - } - Ok(()) - } - - fn compile_call( - &mut self, - func: &located_ast::Expr, - args: &[located_ast::Expr], - keywords: &[located_ast::Keyword], - ) -> CompileResult<()> { - let method = - if let located_ast::Expr::Attribute(located_ast::ExprAttribute { - value, attr, .. - }) = &func - { - self.compile_expression(value)?; - let idx = self.name(attr.as_str()); - emit!(self, Instruction::LoadMethod { idx }); - true - } else { - self.compile_expression(func)?; - false - }; - let call = self.compile_call_inner(0, args, keywords)?; - if method { - self.compile_method_call(call) - } else { - self.compile_normal_call(call) - } - Ok(()) - } - - fn compile_normal_call(&mut self, ty: CallType) { - match ty { - CallType::Positional { nargs } => { - emit!(self, Instruction::CallFunctionPositional { nargs }) - } - CallType::Keyword { nargs } => emit!(self, Instruction::CallFunctionKeyword { nargs }), - CallType::Ex { has_kwargs } => emit!(self, Instruction::CallFunctionEx { has_kwargs }), - } - } - fn compile_method_call(&mut self, ty: CallType) { - match ty { - CallType::Positional { nargs } => { - emit!(self, Instruction::CallMethodPositional { nargs }) - } - CallType::Keyword { nargs } => emit!(self, Instruction::CallMethodKeyword { nargs }), - CallType::Ex { has_kwargs } => emit!(self, Instruction::CallMethodEx { has_kwargs }), - } - } - - fn compile_call_inner( - &mut self, - additional_positional: u32, - args: &[located_ast::Expr], - keywords: &[located_ast::Keyword], - ) -> CompileResult<CallType> { - let count = (args.len() + keywords.len()).to_u32() + additional_positional; - - // Normal arguments: - let (size, unpack) = self.gather_elements(additional_positional, args)?; - let has_double_star = keywords.iter().any(|k| k.arg.is_none()); - - for keyword in keywords { - if let Some(name) = &keyword.arg { - self.check_forbidden_name(name.as_str(), NameUsage::Store)?; - } - } - - let call = if unpack || has_double_star { - // Create a tuple with positional args: - if unpack { - emit!(self, Instruction::BuildTupleUnpack { size }); - } else { - emit!(self, Instruction::BuildTuple { size }); - } - - // Create an optional map with kw-args: - let has_kwargs = !keywords.is_empty(); - if has_kwargs { - self.compile_keywords(keywords)?; - } - CallType::Ex { has_kwargs } - } else if !keywords.is_empty() { - let mut kwarg_names = vec![]; - for keyword in keywords { - if let Some(name) = &keyword.arg { - kwarg_names.push(ConstantData::Str { - value: name.to_string(), - }); - } else { - // This means **kwargs! - panic!("name must be set"); - } - self.compile_expression(&keyword.value)?; - } - - self.emit_constant(ConstantData::Tuple { - elements: kwarg_names, - }); - CallType::Keyword { nargs: count } - } else { - CallType::Positional { nargs: count } - }; - - Ok(call) - } - - // Given a vector of expr / star expr generate code which gives either - // a list of expressions on the stack, or a list of tuples. - fn gather_elements( - &mut self, - before: u32, - elements: &[located_ast::Expr], - ) -> CompileResult<(u32, bool)> { - // First determine if we have starred elements: - let has_stars = elements - .iter() - .any(|e| matches!(e, located_ast::Expr::Starred(_))); - - let size = if has_stars { - let mut size = 0; - - if before > 0 { - emit!(self, Instruction::BuildTuple { size: before }); - size += 1; - } - - let groups = elements - .iter() - .map(|element| { - if let located_ast::Expr::Starred(located_ast::ExprStarred { value, .. }) = - &element - { - (true, value.as_ref()) - } else { - (false, element) - } - }) - .group_by(|(starred, _)| *starred); - - for (starred, run) in &groups { - let mut run_size = 0; - for (_, value) in run { - self.compile_expression(value)?; - run_size += 1 - } - if starred { - size += run_size - } else { - emit!(self, Instruction::BuildTuple { size: run_size }); - size += 1 - } - } - - size - } else { - for element in elements { - self.compile_expression(element)?; - } - before + elements.len().to_u32() - }; - - Ok((size, has_stars)) - } - - fn compile_comprehension_element(&mut self, element: &located_ast::Expr) -> CompileResult<()> { - self.compile_expression(element).map_err(|e| { - if let CodegenErrorType::InvalidStarExpr = e.error { - self.error(CodegenErrorType::SyntaxError( - "iterable unpacking cannot be used in comprehension".to_owned(), - )) - } else { - e - } - }) - } - - fn compile_comprehension( - &mut self, - name: &str, - init_collection: Option<Instruction>, - generators: &[located_ast::Comprehension], - compile_element: &dyn Fn(&mut Self) -> CompileResult<()>, - ) -> CompileResult<()> { - let prev_ctx = self.ctx; - - self.ctx = CompileContext { - loop_data: None, - in_class: prev_ctx.in_class, - func: FunctionContext::Function, - }; - - // We must have at least one generator: - assert!(!generators.is_empty()); - - // Create magnificent function <listcomp>: - self.push_output( - bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, - 1, - 1, - 0, - name.to_owned(), - ); - let arg0 = self.varname(".0")?; - - let return_none = init_collection.is_none(); - // Create empty object of proper type: - if let Some(init_collection) = init_collection { - self._emit(init_collection, OpArg(0), ir::BlockIdx::NULL) - } - - let mut loop_labels = vec![]; - for generator in generators { - if generator.is_async { - unimplemented!("async for comprehensions"); - } - - let loop_block = self.new_block(); - let after_block = self.new_block(); - - if loop_labels.is_empty() { - // Load iterator onto stack (passed as first argument): - emit!(self, Instruction::LoadFast(arg0)); - } else { - // Evaluate iterated item: - self.compile_expression(&generator.iter)?; - - // Get iterator / turn item into an iterator - emit!(self, Instruction::GetIter); - } - - loop_labels.push((loop_block, after_block)); - - self.switch_to_block(loop_block); - emit!( - self, - Instruction::ForIter { - target: after_block, - } - ); - - self.compile_store(&generator.target)?; - - // Now evaluate the ifs: - for if_condition in &generator.ifs { - self.compile_jump_if(if_condition, false, loop_block)? - } - } - - compile_element(self)?; - - for (loop_block, after_block) in loop_labels.iter().rev().copied() { - // Repeat: - emit!(self, Instruction::Jump { target: loop_block }); - - // End of for loop: - self.switch_to_block(after_block); - } - - if return_none { - self.emit_constant(ConstantData::None) - } - - // Return freshly filled list: - emit!(self, Instruction::ReturnValue); - - // Fetch code for listcomp function: - let code = self.pop_code_object(); - - self.ctx = prev_ctx; - - let mut func_flags = bytecode::MakeFunctionFlags::empty(); - if self.build_closure(&code) { - func_flags |= bytecode::MakeFunctionFlags::CLOSURE; - } - - // List comprehension code: - self.emit_constant(ConstantData::Code { - code: Box::new(code), - }); - - // List comprehension function name: - self.emit_constant(ConstantData::Str { - value: name.to_owned(), - }); - - // Turn code object into function object: - emit!(self, Instruction::MakeFunction(func_flags)); - - // Evaluate iterated item: - self.compile_expression(&generators[0].iter)?; - - // Get iterator / turn item into an iterator - emit!(self, Instruction::GetIter); - - // Call just created <listcomp> function: - emit!(self, Instruction::CallFunctionPositional { nargs: 1 }); - Ok(()) - } - - fn compile_future_features( - &mut self, - features: &[located_ast::Alias], - ) -> Result<(), CodegenError> { - if self.done_with_future_stmts { - return Err(self.error(CodegenErrorType::InvalidFuturePlacement)); - } - for feature in features { - match feature.name.as_str() { - // Python 3 features; we've already implemented them by default - "nested_scopes" | "generators" | "division" | "absolute_import" - | "with_statement" | "print_function" | "unicode_literals" => {} - // "generator_stop" => {} - "annotations" => self.future_annotations = true, - other => { - return Err(self.error(CodegenErrorType::InvalidFutureFeature(other.to_owned()))) - } - } - } - Ok(()) - } - - // Low level helper functions: - fn _emit(&mut self, instr: Instruction, arg: OpArg, target: ir::BlockIdx) { - let location = self.current_source_location; - // TODO: insert source filename - self.current_block().instructions.push(ir::InstructionInfo { - instr, - arg, - target, - location, - }); - } - - fn emit_no_arg(&mut self, ins: Instruction) { - self._emit(ins, OpArg::null(), ir::BlockIdx::NULL) - } - - fn emit_arg<A: OpArgType, T: EmitArg<A>>( - &mut self, - arg: T, - f: impl FnOnce(OpArgMarker<A>) -> Instruction, - ) { - let (op, arg, target) = arg.emit(f); - self._emit(op, arg, target) - } - - // fn block_done() - - fn emit_constant(&mut self, constant: ConstantData) { - let info = self.current_code_info(); - let idx = info.constants.insert_full(constant).0.to_u32(); - self.emit_arg(idx, |idx| Instruction::LoadConst { idx }) - } - - fn current_code_info(&mut self) -> &mut ir::CodeInfo { - self.code_stack.last_mut().expect("no code on stack") - } - - fn current_block(&mut self) -> &mut ir::Block { - let info = self.current_code_info(); - &mut info.blocks[info.current_block] - } - - fn new_block(&mut self) -> ir::BlockIdx { - let code = self.current_code_info(); - let idx = ir::BlockIdx(code.blocks.len().to_u32()); - code.blocks.push(ir::Block::default()); - idx - } - - fn switch_to_block(&mut self, block: ir::BlockIdx) { - let code = self.current_code_info(); - let prev = code.current_block; - assert_eq!( - code.blocks[block].next, - ir::BlockIdx::NULL, - "switching to completed block" - ); - let prev_block = &mut code.blocks[prev.0 as usize]; - assert_eq!( - prev_block.next.0, - u32::MAX, - "switching from block that's already got a next" - ); - prev_block.next = block; - code.current_block = block; - } - - fn set_source_location(&mut self, location: SourceLocation) { - self.current_source_location = location; - } - - fn get_source_line_number(&mut self) -> LineNumber { - let location = self.current_source_location; - location.row - } - - fn push_qualified_path(&mut self, name: &str) { - self.qualified_path.push(name.to_owned()); - } - - fn mark_generator(&mut self) { - self.current_code_info().flags |= bytecode::CodeFlags::IS_GENERATOR - } -} - -trait EmitArg<Arg: OpArgType> { - fn emit( - self, - f: impl FnOnce(OpArgMarker<Arg>) -> Instruction, - ) -> (Instruction, OpArg, ir::BlockIdx); -} -impl<T: OpArgType> EmitArg<T> for T { - fn emit( - self, - f: impl FnOnce(OpArgMarker<T>) -> Instruction, - ) -> (Instruction, OpArg, ir::BlockIdx) { - let (marker, arg) = OpArgMarker::new(self); - (f(marker), arg, ir::BlockIdx::NULL) - } -} -impl EmitArg<bytecode::Label> for ir::BlockIdx { - fn emit( - self, - f: impl FnOnce(OpArgMarker<bytecode::Label>) -> Instruction, - ) -> (Instruction, OpArg, ir::BlockIdx) { - (f(OpArgMarker::marker()), OpArg::null(), self) - } -} - -fn split_doc(body: &[located_ast::Stmt]) -> (Option<String>, &[located_ast::Stmt]) { - if let Some((located_ast::Stmt::Expr(expr), body_rest)) = body.split_first() { - if let Some(doc) = try_get_constant_string(std::slice::from_ref(&expr.value)) { - return (Some(doc), body_rest); - } - } - (None, body) -} - -fn try_get_constant_string(values: &[located_ast::Expr]) -> Option<String> { - fn get_constant_string_inner(out_string: &mut String, value: &located_ast::Expr) -> bool { - match value { - located_ast::Expr::Constant(located_ast::ExprConstant { - value: located_ast::Constant::Str(s), - .. - }) => { - out_string.push_str(s); - true - } - located_ast::Expr::JoinedStr(located_ast::ExprJoinedStr { values, .. }) => values - .iter() - .all(|value| get_constant_string_inner(out_string, value)), - _ => false, - } - } - let mut out_string = String::new(); - if values - .iter() - .all(|v| get_constant_string_inner(&mut out_string, v)) - { - Some(out_string) - } else { - None - } -} - -fn compile_constant(value: &located_ast::Constant) -> ConstantData { - match value { - located_ast::Constant::None => ConstantData::None, - located_ast::Constant::Bool(b) => ConstantData::Boolean { value: *b }, - located_ast::Constant::Str(s) => ConstantData::Str { value: s.clone() }, - located_ast::Constant::Bytes(b) => ConstantData::Bytes { value: b.clone() }, - located_ast::Constant::Int(i) => ConstantData::Integer { value: i.clone() }, - located_ast::Constant::Tuple(t) => ConstantData::Tuple { - elements: t.iter().map(compile_constant).collect(), - }, - located_ast::Constant::Float(f) => ConstantData::Float { value: *f }, - located_ast::Constant::Complex { real, imag } => ConstantData::Complex { - value: Complex64::new(*real, *imag), - }, - located_ast::Constant::Ellipsis => ConstantData::Ellipsis, - } -} - -// Note: Not a good practice in general. Keep this trait private only for compiler -trait ToU32 { - fn to_u32(self) -> u32; -} - -impl ToU32 for usize { - fn to_u32(self) -> u32 { - self.try_into().unwrap() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use rustpython_parser as parser; - use rustpython_parser_core::source_code::LinearLocator; - - fn compile_exec(source: &str) -> CodeObject { - let mut locator: LinearLocator = LinearLocator::new(source); - use rustpython_parser::ast::fold::Fold; - let mut compiler: Compiler = Compiler::new( - CompileOpts::default(), - "source_path".to_owned(), - "<module>".to_owned(), - ); - let ast = parser::parse_program(source, "<test>").unwrap(); - let ast = locator.fold(ast).unwrap(); - let symbol_scope = SymbolTable::scan_program(&ast).unwrap(); - compiler.compile_program(&ast, symbol_scope).unwrap(); - compiler.pop_code_object() - } - - macro_rules! assert_dis_snapshot { - ($value:expr) => { - insta::assert_snapshot!( - insta::internals::AutoName, - $value.display_expand_code_objects().to_string(), - stringify!($value) - ) - }; - } - - #[test] - fn test_if_ors() { - assert_dis_snapshot!(compile_exec( - "\ -if True or False or False: - pass -" - )); - } - - #[test] - fn test_if_ands() { - assert_dis_snapshot!(compile_exec( - "\ -if True and False and False: - pass -" - )); - } - - #[test] - fn test_if_mixed() { - assert_dis_snapshot!(compile_exec( - "\ -if (True and False) or (False and True): - pass -" - )); - } - - #[test] - fn test_nested_double_async_with() { - assert_dis_snapshot!(compile_exec( - "\ -for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): - with self.subTest(type=type(stop_exc)): - try: - async with egg(): - raise stop_exc - except Exception as ex: - self.assertIs(ex, stop_exc) - else: - self.fail(f'{stop_exc} was suppressed') -" - )); - } -} diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs deleted file mode 100644 index 017f735105a..00000000000 --- a/compiler/codegen/src/error.rs +++ /dev/null @@ -1,83 +0,0 @@ -use std::fmt; - -pub type CodegenError = rustpython_parser_core::source_code::LocatedError<CodegenErrorType>; - -#[derive(Debug)] -#[non_exhaustive] -pub enum CodegenErrorType { - /// Invalid assignment, cannot store value in target. - Assign(&'static str), - /// Invalid delete - Delete(&'static str), - SyntaxError(String), - /// Multiple `*` detected - MultipleStarArgs, - /// Misplaced `*` expression - InvalidStarExpr, - /// Break statement outside of loop. - InvalidBreak, - /// Continue statement outside of loop. - InvalidContinue, - InvalidReturn, - InvalidYield, - InvalidYieldFrom, - InvalidAwait, - AsyncYieldFrom, - AsyncReturnValue, - InvalidFuturePlacement, - InvalidFutureFeature(String), - FunctionImportStar, - TooManyStarUnpack, - EmptyWithItems, - EmptyWithBody, - NotImplementedYet, // RustPython marker for unimplemented features -} - -impl std::error::Error for CodegenErrorType {} - -impl fmt::Display for CodegenErrorType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use CodegenErrorType::*; - match self { - Assign(target) => write!(f, "cannot assign to {target}"), - Delete(target) => write!(f, "cannot delete {target}"), - SyntaxError(err) => write!(f, "{}", err.as_str()), - MultipleStarArgs => { - write!(f, "two starred expressions in assignment") - } - InvalidStarExpr => write!(f, "cannot use starred expression here"), - InvalidBreak => write!(f, "'break' outside loop"), - InvalidContinue => write!(f, "'continue' outside loop"), - InvalidReturn => write!(f, "'return' outside function"), - InvalidYield => write!(f, "'yield' outside function"), - InvalidYieldFrom => write!(f, "'yield from' outside function"), - InvalidAwait => write!(f, "'await' outside async function"), - AsyncYieldFrom => write!(f, "'yield from' inside async function"), - AsyncReturnValue => { - write!(f, "'return' with value inside async generator") - } - InvalidFuturePlacement => write!( - f, - "from __future__ imports must occur at the beginning of the file" - ), - InvalidFutureFeature(feat) => { - write!(f, "future feature {feat} is not defined") - } - FunctionImportStar => { - write!(f, "import * only allowed at module level") - } - TooManyStarUnpack => { - write!(f, "too many expressions in star-unpacking assignment") - } - EmptyWithItems => { - write!(f, "empty items on With") - } - EmptyWithBody => { - write!(f, "empty body on With") - } - NotImplementedYet => { - write!(f, "RustPython does not implement this feature yet") - } - } - } -} diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs deleted file mode 100644 index de7055e0d77..00000000000 --- a/compiler/codegen/src/ir.rs +++ /dev/null @@ -1,328 +0,0 @@ -use std::ops; - -use crate::IndexSet; -use rustpython_compiler_core::bytecode::{ - CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, OpArg, -}; -use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; - -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub struct BlockIdx(pub u32); -impl BlockIdx { - pub const NULL: BlockIdx = BlockIdx(u32::MAX); - const fn idx(self) -> usize { - self.0 as usize - } -} -impl ops::Index<BlockIdx> for [Block] { - type Output = Block; - fn index(&self, idx: BlockIdx) -> &Block { - &self[idx.idx()] - } -} -impl ops::IndexMut<BlockIdx> for [Block] { - fn index_mut(&mut self, idx: BlockIdx) -> &mut Block { - &mut self[idx.idx()] - } -} -impl ops::Index<BlockIdx> for Vec<Block> { - type Output = Block; - fn index(&self, idx: BlockIdx) -> &Block { - &self[idx.idx()] - } -} -impl ops::IndexMut<BlockIdx> for Vec<Block> { - fn index_mut(&mut self, idx: BlockIdx) -> &mut Block { - &mut self[idx.idx()] - } -} - -#[derive(Debug, Copy, Clone)] -pub struct InstructionInfo { - pub instr: Instruction, - pub arg: OpArg, - pub target: BlockIdx, - pub location: SourceLocation, -} - -// spell-checker:ignore petgraph -// TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it -// might enable more analysis/optimizations -#[derive(Debug)] -pub struct Block { - pub instructions: Vec<InstructionInfo>, - pub next: BlockIdx, -} -impl Default for Block { - fn default() -> Self { - Block { - instructions: Vec::new(), - next: BlockIdx::NULL, - } - } -} - -pub struct CodeInfo { - pub flags: CodeFlags, - pub posonlyarg_count: u32, // Number of positional-only arguments - pub arg_count: u32, - pub kwonlyarg_count: u32, - pub source_path: String, - pub first_line_number: LineNumber, - pub obj_name: String, // Name of the object that created this code object - - pub blocks: Vec<Block>, - pub current_block: BlockIdx, - pub constants: IndexSet<ConstantData>, - pub name_cache: IndexSet<String>, - pub varname_cache: IndexSet<String>, - pub cellvar_cache: IndexSet<String>, - pub freevar_cache: IndexSet<String>, -} -impl CodeInfo { - pub fn finalize_code(mut self, optimize: u8) -> CodeObject { - if optimize > 0 { - self.dce(); - } - - let max_stackdepth = self.max_stackdepth(); - let cell2arg = self.cell2arg(); - - let CodeInfo { - flags, - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path, - first_line_number, - obj_name, - - mut blocks, - current_block: _, - constants, - name_cache, - varname_cache, - cellvar_cache, - freevar_cache, - } = self; - - let mut instructions = Vec::new(); - let mut locations = Vec::new(); - - let mut block_to_offset = vec![Label(0); blocks.len()]; - loop { - let mut num_instructions = 0; - for (idx, block) in iter_blocks(&blocks) { - block_to_offset[idx.idx()] = Label(num_instructions as u32); - for instr in &block.instructions { - num_instructions += instr.arg.instr_size() - } - } - - instructions.reserve_exact(num_instructions); - locations.reserve_exact(num_instructions); - - let mut recompile_extended_arg = false; - let mut next_block = BlockIdx(0); - while next_block != BlockIdx::NULL { - let block = &mut blocks[next_block]; - for info in &mut block.instructions { - let (op, arg, target) = (info.instr, &mut info.arg, info.target); - if target != BlockIdx::NULL { - let new_arg = OpArg(block_to_offset[target.idx()].0); - recompile_extended_arg |= new_arg.instr_size() != arg.instr_size(); - *arg = new_arg; - } - let (extras, lo_arg) = arg.split(); - locations.extend(std::iter::repeat(info.location).take(arg.instr_size())); - instructions.extend( - extras - .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) - .chain([CodeUnit { op, arg: lo_arg }]), - ); - } - next_block = block.next; - } - - if !recompile_extended_arg { - break; - } - - instructions.clear(); - locations.clear() - } - - CodeObject { - flags, - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path, - first_line_number: Some(first_line_number), - obj_name, - - max_stackdepth, - instructions: instructions.into_boxed_slice(), - locations: locations.into_boxed_slice(), - constants: constants.into_iter().collect(), - names: name_cache.into_iter().collect(), - varnames: varname_cache.into_iter().collect(), - cellvars: cellvar_cache.into_iter().collect(), - freevars: freevar_cache.into_iter().collect(), - cell2arg, - } - } - - fn cell2arg(&self) -> Option<Box<[i32]>> { - if self.cellvar_cache.is_empty() { - return None; - } - - let total_args = self.arg_count - + self.kwonlyarg_count - + self.flags.contains(CodeFlags::HAS_VARARGS) as u32 - + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as u32; - - let mut found_cellarg = false; - let cell2arg = self - .cellvar_cache - .iter() - .map(|var| { - self.varname_cache - .get_index_of(var) - // check that it's actually an arg - .filter(|i| *i < total_args as usize) - .map_or(-1, |i| { - found_cellarg = true; - i as i32 - }) - }) - .collect::<Box<[_]>>(); - - if found_cellarg { - Some(cell2arg) - } else { - None - } - } - - fn dce(&mut self) { - for block in &mut self.blocks { - let mut last_instr = None; - for (i, ins) in block.instructions.iter().enumerate() { - if ins.instr.unconditional_branch() { - last_instr = Some(i); - break; - } - } - if let Some(i) = last_instr { - block.instructions.truncate(i + 1); - } - } - } - - fn max_stackdepth(&self) -> u32 { - let mut maxdepth = 0u32; - let mut stack = Vec::with_capacity(self.blocks.len()); - let mut start_depths = vec![u32::MAX; self.blocks.len()]; - start_depths[0] = 0; - stack.push(BlockIdx(0)); - const DEBUG: bool = false; - 'process_blocks: while let Some(block) = stack.pop() { - let mut depth = start_depths[block.idx()]; - if DEBUG { - eprintln!("===BLOCK {}===", block.0); - } - let block = &self.blocks[block]; - for i in &block.instructions { - let instr = &i.instr; - let effect = instr.stack_effect(i.arg, false); - if DEBUG { - let display_arg = if i.target == BlockIdx::NULL { - i.arg - } else { - OpArg(i.target.0) - }; - let instr_display = instr.display(display_arg, self); - eprint!("{instr_display}: {depth} {effect:+} => "); - } - let new_depth = depth.checked_add_signed(effect).unwrap(); - if DEBUG { - eprintln!("{new_depth}"); - } - if new_depth > maxdepth { - maxdepth = new_depth - } - // we don't want to worry about Break/Continue, they use unwinding to jump to - // their targets and as such the stack size is taken care of in frame.rs by setting - // it back to the level it was at when SetupLoop was run - if i.target != BlockIdx::NULL - && !matches!( - instr, - Instruction::Continue { .. } | Instruction::Break { .. } - ) - { - let effect = instr.stack_effect(i.arg, true); - let target_depth = depth.checked_add_signed(effect).unwrap(); - if target_depth > maxdepth { - maxdepth = target_depth - } - stackdepth_push(&mut stack, &mut start_depths, i.target, target_depth); - } - depth = new_depth; - if instr.unconditional_branch() { - continue 'process_blocks; - } - } - stackdepth_push(&mut stack, &mut start_depths, block.next, depth); - } - if DEBUG { - eprintln!("DONE: {maxdepth}"); - } - maxdepth - } -} - -impl InstrDisplayContext for CodeInfo { - type Constant = ConstantData; - fn get_constant(&self, i: usize) -> &ConstantData { - &self.constants[i] - } - fn get_name(&self, i: usize) -> &str { - self.name_cache[i].as_ref() - } - fn get_varname(&self, i: usize) -> &str { - self.varname_cache[i].as_ref() - } - fn get_cell_name(&self, i: usize) -> &str { - self.cellvar_cache - .get_index(i) - .unwrap_or_else(|| &self.freevar_cache[i - self.cellvar_cache.len()]) - .as_ref() - } -} - -fn stackdepth_push( - stack: &mut Vec<BlockIdx>, - start_depths: &mut [u32], - target: BlockIdx, - depth: u32, -) { - let block_depth = &mut start_depths[target.idx()]; - if *block_depth == u32::MAX || depth > *block_depth { - *block_depth = depth; - stack.push(target); - } -} - -fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + '_ { - let mut next = BlockIdx(0); - std::iter::from_fn(move || { - if next == BlockIdx::NULL { - return None; - } - let (idx, b) = (next, &blocks[next]); - next = b.next; - Some((idx, b)) - }) -} diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs deleted file mode 100644 index 910d015a397..00000000000 --- a/compiler/codegen/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Compile a Python AST or source code into bytecode consumable by RustPython. -#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] -#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")] - -#[macro_use] -extern crate log; - -type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>; -type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>; - -pub mod compile; -pub mod error; -pub mod ir; -pub mod symboltable; - -pub use compile::CompileOpts; diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap deleted file mode 100644 index 93eb0ea6ee4..00000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap +++ /dev/null @@ -1,14 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfFalse (6) - 2 LoadConst (False) - 3 JumpIfFalse (6) - 4 LoadConst (False) - 5 JumpIfFalse (6) - - 2 >> 6 LoadConst (None) - 7 ReturnValue - diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap deleted file mode 100644 index 9594af69ee7..00000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap +++ /dev/null @@ -1,16 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfFalse (4) - 2 LoadConst (False) - 3 JumpIfTrue (8) - >> 4 LoadConst (False) - 5 JumpIfFalse (8) - 6 LoadConst (True) - 7 JumpIfFalse (8) - - 2 >> 8 LoadConst (None) - 9 ReturnValue - diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap deleted file mode 100644 index dd582b7d7c1..00000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap +++ /dev/null @@ -1,14 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfTrue (6) - 2 LoadConst (False) - 3 JumpIfTrue (6) - 4 LoadConst (False) - 5 JumpIfFalse (6) - - 2 >> 6 LoadConst (None) - 7 ReturnValue - diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap deleted file mode 100644 index dcc6f4c2c50..00000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap +++ /dev/null @@ -1,85 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" ---- - 1 0 SetupLoop - 1 LoadNameAny (0, StopIteration) - 2 LoadConst ("spam") - 3 CallFunctionPositional(1) - 4 LoadNameAny (1, StopAsyncIteration) - 5 LoadConst ("ham") - 6 CallFunctionPositional(1) - 7 BuildTuple (2) - 8 GetIter - >> 9 ForIter (68) - 10 StoreLocal (2, stop_exc) - - 2 11 LoadNameAny (3, self) - 12 LoadMethod (4, subTest) - 13 LoadNameAny (5, type) - 14 LoadNameAny (2, stop_exc) - 15 CallFunctionPositional(1) - 16 LoadConst (("type")) - 17 CallMethodKeyword (1) - 18 SetupWith (65) - 19 Pop - - 3 20 SetupExcept (40) - - 4 21 LoadNameAny (6, egg) - 22 CallFunctionPositional(0) - 23 BeforeAsyncWith - 24 GetAwaitable - 25 LoadConst (None) - 26 YieldFrom - 27 SetupAsyncWith (33) - 28 Pop - - 5 29 LoadNameAny (2, stop_exc) - 30 Raise (Raise) - - 4 31 PopBlock - 32 EnterFinally - >> 33 WithCleanupStart - 34 GetAwaitable - 35 LoadConst (None) - 36 YieldFrom - 37 WithCleanupFinish - 38 PopBlock - 39 Jump (54) - >> 40 Duplicate - - 6 41 LoadNameAny (7, Exception) - 42 TestOperation (ExceptionMatch) - 43 JumpIfFalse (53) - 44 StoreLocal (8, ex) - - 7 45 LoadNameAny (3, self) - 46 LoadMethod (9, assertIs) - 47 LoadNameAny (8, ex) - 48 LoadNameAny (2, stop_exc) - 49 CallMethodPositional (2) - 50 Pop - 51 PopException - 52 Jump (63) - >> 53 Raise (Reraise) - - 9 >> 54 LoadNameAny (3, self) - 55 LoadMethod (10, fail) - 56 LoadConst ("") - 57 LoadNameAny (2, stop_exc) - 58 FormatValue (None) - 59 LoadConst (" was suppressed") - 60 BuildString (2) - 61 CallMethodPositional (1) - 62 Pop - - 2 >> 63 PopBlock - 64 EnterFinally - >> 65 WithCleanupStart - 66 WithCleanupFinish - 67 Jump (9) - >> 68 PopBlock - 69 LoadConst (None) - 70 ReturnValue - diff --git a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap b/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap deleted file mode 100644 index d80f10dfee5..00000000000 --- a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap +++ /dev/null @@ -1,14 +0,0 @@ ---- -source: compiler/src/compile.rs -expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfFalse (6) - 2 LoadConst (False) - 3 JumpIfFalse (6) - 4 LoadConst (False) - 5 JumpIfFalse (6) - - 2 >> 6 LoadConst (None) - 7 ReturnValue - diff --git a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap b/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap deleted file mode 100644 index 0a9175bb12e..00000000000 --- a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap +++ /dev/null @@ -1,16 +0,0 @@ ---- -source: compiler/src/compile.rs -expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfFalse (4) - 2 LoadConst (False) - 3 JumpIfTrue (8) - >> 4 LoadConst (False) - 5 JumpIfFalse (8) - 6 LoadConst (True) - 7 JumpIfFalse (8) - - 2 >> 8 LoadConst (None) - 9 ReturnValue - diff --git a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap b/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap deleted file mode 100644 index 4b812639b2c..00000000000 --- a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap +++ /dev/null @@ -1,14 +0,0 @@ ---- -source: compiler/src/compile.rs -expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfTrue (6) - 2 LoadConst (False) - 3 JumpIfTrue (6) - 4 LoadConst (False) - 5 JumpIfFalse (6) - - 2 >> 6 LoadConst (None) - 7 ReturnValue - diff --git a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap deleted file mode 100644 index 79a1a86ad1b..00000000000 --- a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap +++ /dev/null @@ -1,87 +0,0 @@ ---- -source: compiler/src/compile.rs -expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with woohoo():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" ---- - 1 0 SetupLoop (69) - 1 LoadNameAny (0, StopIteration) - 2 LoadConst ("spam") - 3 CallFunctionPositional (1) - 4 LoadNameAny (1, StopAsyncIteration) - 5 LoadConst ("ham") - 6 CallFunctionPositional (1) - 7 BuildTuple (2, false) - 8 GetIter - >> 9 ForIter (68) - 10 StoreLocal (2, stop_exc) - - 2 11 LoadNameAny (3, self) - 12 LoadMethod (subTest) - 13 LoadNameAny (5, type) - 14 LoadNameAny (2, stop_exc) - 15 CallFunctionPositional (1) - 16 LoadConst (("type")) - 17 CallMethodKeyword (1) - 18 SetupWith (65) - 19 Pop - - 3 20 SetupExcept (40) - - 4 21 LoadNameAny (6, woohoo) - 22 CallFunctionPositional (0) - 23 BeforeAsyncWith - 24 GetAwaitable - 25 LoadConst (None) - 26 YieldFrom - 27 SetupAsyncWith (33) - 28 Pop - - 5 29 LoadNameAny (2, stop_exc) - 30 Raise (Raise) - - 4 31 PopBlock - 32 EnterFinally - >> 33 WithCleanupStart - 34 GetAwaitable - 35 LoadConst (None) - 36 YieldFrom - 37 WithCleanupFinish - 38 PopBlock - 39 Jump (54) - >> 40 Duplicate - - 6 41 LoadNameAny (7, Exception) - 42 TestOperation (ExceptionMatch) - 43 JumpIfFalse (53) - 44 StoreLocal (8, ex) - - 7 45 LoadNameAny (3, self) - 46 LoadMethod (assertIs) - 47 LoadNameAny (8, ex) - 48 LoadNameAny (2, stop_exc) - 49 CallMethodPositional (2) - 50 Pop - 51 PopException - 52 Jump (63) - >> 53 Raise (Reraise) - - 9 >> 54 LoadNameAny (3, self) - 55 LoadMethod (fail) - 56 LoadConst ("") - - 1 57 LoadNameAny (2, stop_exc) - 58 FormatValue (None) - - 9 59 LoadConst (" was suppressed") - 60 BuildString (2) - 61 CallMethodPositional (1) - 62 Pop - - 2 >> 63 PopBlock - 64 EnterFinally - >> 65 WithCleanupStart - 66 WithCleanupFinish - 67 Jump (9) - >> 68 PopBlock - >> 69 LoadConst (None) - 70 ReturnValue - diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs deleted file mode 100644 index 75e327fe346..00000000000 --- a/compiler/codegen/src/symboltable.rs +++ /dev/null @@ -1,1402 +0,0 @@ -/* Python code is pre-scanned for symbols in the ast. - -This ensures that global and nonlocal keywords are picked up. -Then the compiler can use the symbol table to generate proper -load and store instructions for names. - -Inspirational file: https://github.com/python/cpython/blob/main/Python/symtable.c -*/ - -use crate::{ - error::{CodegenError, CodegenErrorType}, - IndexMap, -}; -use bitflags::bitflags; -use rustpython_ast::{self as ast, located::Located}; -use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; -use std::{borrow::Cow, fmt}; - -/// Captures all symbols in the current scope, and has a list of sub-scopes in this scope. -#[derive(Clone)] -pub struct SymbolTable { - /// The name of this symbol table. Often the name of the class or function. - pub name: String, - - /// The type of symbol table - pub typ: SymbolTableType, - - /// The line number in the source code where this symboltable begins. - pub line_number: u32, - - // Return True if the block is a nested class or function - pub is_nested: bool, - - /// A set of symbols present on this scope level. - pub symbols: IndexMap<String, Symbol>, - - /// A list of sub-scopes in the order as found in the - /// AST nodes. - pub sub_tables: Vec<SymbolTable>, -} - -impl SymbolTable { - fn new(name: String, typ: SymbolTableType, line_number: u32, is_nested: bool) -> Self { - SymbolTable { - name, - typ, - line_number, - is_nested, - symbols: IndexMap::default(), - sub_tables: vec![], - } - } - - pub fn scan_program(program: &[ast::located::Stmt]) -> SymbolTableResult<Self> { - let mut builder = SymbolTableBuilder::new(); - builder.scan_statements(program)?; - builder.finish() - } - - pub fn scan_expr(expr: &ast::located::Expr) -> SymbolTableResult<Self> { - let mut builder = SymbolTableBuilder::new(); - builder.scan_expression(expr, ExpressionContext::Load)?; - builder.finish() - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SymbolTableType { - Module, - Class, - Function, - Comprehension, -} - -impl fmt::Display for SymbolTableType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - SymbolTableType::Module => write!(f, "module"), - SymbolTableType::Class => write!(f, "class"), - SymbolTableType::Function => write!(f, "function"), - SymbolTableType::Comprehension => write!(f, "comprehension"), - } - } -} - -/// Indicator for a single symbol what the scope of this symbol is. -/// The scope can be unknown, which is unfortunate, but not impossible. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SymbolScope { - Unknown, - Local, - GlobalExplicit, - GlobalImplicit, - Free, - Cell, -} - -bitflags! { - #[derive(Copy, Clone, Debug, PartialEq)] - pub struct SymbolFlags: u16 { - const REFERENCED = 0x001; - const ASSIGNED = 0x002; - const PARAMETER = 0x004; - const ANNOTATED = 0x008; - const IMPORTED = 0x010; - const NONLOCAL = 0x020; - // indicates if the symbol gets a value assigned by a named expression in a comprehension - // this is required to correct the scope in the analysis. - const ASSIGNED_IN_COMPREHENSION = 0x040; - // indicates that the symbol is used a bound iterator variable. We distinguish this case - // from normal assignment to detect disallowed re-assignment to iterator variables. - const ITER = 0x080; - /// indicates that the symbol is a free variable in a class method from the scope that the - /// class is defined in, e.g.: - /// ```python - /// def foo(x): - /// class A: - /// def method(self): - /// return x // is_free_class - /// ``` - const FREE_CLASS = 0x100; - const BOUND = Self::ASSIGNED.bits() | Self::PARAMETER.bits() | Self::IMPORTED.bits() | Self::ITER.bits(); - } -} - -/// A single symbol in a table. Has various properties such as the scope -/// of the symbol, and also the various uses of the symbol. -#[derive(Debug, Clone)] -pub struct Symbol { - pub name: String, - pub scope: SymbolScope, - pub flags: SymbolFlags, -} - -impl Symbol { - fn new(name: &str) -> Self { - Symbol { - name: name.to_owned(), - // table, - scope: SymbolScope::Unknown, - flags: SymbolFlags::empty(), - } - } - - pub fn is_global(&self) -> bool { - matches!( - self.scope, - SymbolScope::GlobalExplicit | SymbolScope::GlobalImplicit - ) - } - - pub fn is_local(&self) -> bool { - matches!(self.scope, SymbolScope::Local | SymbolScope::Cell) - } - - pub fn is_bound(&self) -> bool { - self.flags.intersects(SymbolFlags::BOUND) - } -} - -#[derive(Debug)] -pub struct SymbolTableError { - error: String, - location: Option<SourceLocation>, -} - -impl SymbolTableError { - pub fn into_codegen_error(self, source_path: String) -> CodegenError { - CodegenError { - error: CodegenErrorType::SyntaxError(self.error), - location: self.location.map(|l| SourceLocation { - row: l.row, - column: l.column, - }), - source_path, - } - } -} - -type SymbolTableResult<T = ()> = Result<T, SymbolTableError>; - -impl SymbolTable { - pub fn lookup(&self, name: &str) -> Option<&Symbol> { - self.symbols.get(name) - } -} - -impl std::fmt::Debug for SymbolTable { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "SymbolTable({:?} symbols, {:?} sub scopes)", - self.symbols.len(), - self.sub_tables.len() - ) - } -} - -/* Perform some sort of analysis on nonlocals, globals etc.. - See also: https://github.com/python/cpython/blob/main/Python/symtable.c#L410 -*/ -fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult { - let mut analyzer = SymbolTableAnalyzer::default(); - analyzer.analyze_symbol_table(symbol_table) -} - -type SymbolMap = IndexMap<String, Symbol>; - -mod stack { - use std::panic; - use std::ptr::NonNull; - pub struct StackStack<T> { - v: Vec<NonNull<T>>, - } - impl<T> Default for StackStack<T> { - fn default() -> Self { - Self { v: Vec::new() } - } - } - impl<T> StackStack<T> { - /// Appends a reference to this stack for the duration of the function `f`. When `f` - /// returns, the reference will be popped off the stack. - pub fn with_append<F, R>(&mut self, x: &mut T, f: F) -> R - where - F: FnOnce(&mut Self) -> R, - { - self.v.push(x.into()); - let res = panic::catch_unwind(panic::AssertUnwindSafe(|| f(self))); - self.v.pop(); - res.unwrap_or_else(|x| panic::resume_unwind(x)) - } - - pub fn iter(&self) -> impl DoubleEndedIterator<Item = &T> + '_ { - self.as_ref().iter().copied() - } - pub fn iter_mut(&mut self) -> impl DoubleEndedIterator<Item = &mut T> + '_ { - self.as_mut().iter_mut().map(|x| &mut **x) - } - // pub fn top(&self) -> Option<&T> { - // self.as_ref().last().copied() - // } - // pub fn top_mut(&mut self) -> Option<&mut T> { - // self.as_mut().last_mut().map(|x| &mut **x) - // } - pub fn len(&self) -> usize { - self.v.len() - } - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn as_ref(&self) -> &[&T] { - unsafe { &*(self.v.as_slice() as *const [NonNull<T>] as *const [&T]) } - } - - pub fn as_mut(&mut self) -> &mut [&mut T] { - unsafe { &mut *(self.v.as_mut_slice() as *mut [NonNull<T>] as *mut [&mut T]) } - } - } -} -use stack::StackStack; - -/// Symbol table analysis. Can be used to analyze a fully -/// build symbol table structure. It will mark variables -/// as local variables for example. -#[derive(Default)] -#[repr(transparent)] -struct SymbolTableAnalyzer { - tables: StackStack<(SymbolMap, SymbolTableType)>, -} - -impl SymbolTableAnalyzer { - fn analyze_symbol_table(&mut self, symbol_table: &mut SymbolTable) -> SymbolTableResult { - let symbols = std::mem::take(&mut symbol_table.symbols); - let sub_tables = &mut *symbol_table.sub_tables; - - let mut info = (symbols, symbol_table.typ); - self.tables.with_append(&mut info, |list| { - let inner_scope = unsafe { &mut *(list as *mut _ as *mut SymbolTableAnalyzer) }; - // Analyze sub scopes: - for sub_table in sub_tables.iter_mut() { - inner_scope.analyze_symbol_table(sub_table)?; - } - Ok(()) - })?; - - symbol_table.symbols = info.0; - - // Analyze symbols: - for symbol in symbol_table.symbols.values_mut() { - self.analyze_symbol(symbol, symbol_table.typ, sub_tables)?; - } - Ok(()) - } - - fn analyze_symbol( - &mut self, - symbol: &mut Symbol, - st_typ: SymbolTableType, - sub_tables: &[SymbolTable], - ) -> SymbolTableResult { - if symbol - .flags - .contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) - && st_typ == SymbolTableType::Comprehension - { - // propagate symbol to next higher level that can hold it, - // i.e., function or module. Comprehension is skipped and - // Class is not allowed and detected as error. - //symbol.scope = SymbolScope::Nonlocal; - self.analyze_symbol_comprehension(symbol, 0)? - } else { - match symbol.scope { - SymbolScope::Free => { - if !self.tables.as_ref().is_empty() { - let scope_depth = self.tables.as_ref().len(); - // check if the name is already defined in any outer scope - // therefore - if scope_depth < 2 - || self.found_in_outer_scope(&symbol.name) != Some(SymbolScope::Free) - { - return Err(SymbolTableError { - error: format!("no binding for nonlocal '{}' found", symbol.name), - // TODO: accurate location info, somehow - location: None, - }); - } - } else { - return Err(SymbolTableError { - error: format!( - "nonlocal {} defined at place without an enclosing scope", - symbol.name - ), - // TODO: accurate location info, somehow - location: None, - }); - } - } - SymbolScope::GlobalExplicit | SymbolScope::GlobalImplicit => { - // TODO: add more checks for globals? - } - SymbolScope::Local | SymbolScope::Cell => { - // all is well - } - SymbolScope::Unknown => { - // Try hard to figure out what the scope of this symbol is. - let scope = if symbol.is_bound() { - self.found_in_inner_scope(sub_tables, &symbol.name, st_typ) - .unwrap_or(SymbolScope::Local) - } else if let Some(scope) = self.found_in_outer_scope(&symbol.name) { - scope - } else if self.tables.is_empty() { - // Don't make assumptions when we don't know. - SymbolScope::Unknown - } else { - // If there are scopes above we assume global. - SymbolScope::GlobalImplicit - }; - symbol.scope = scope; - } - } - } - Ok(()) - } - - fn found_in_outer_scope(&mut self, name: &str) -> Option<SymbolScope> { - let mut decl_depth = None; - for (i, (symbols, typ)) in self.tables.iter().rev().enumerate() { - if matches!(typ, SymbolTableType::Module) - || matches!(typ, SymbolTableType::Class if name != "__class__") - { - continue; - } - if let Some(sym) = symbols.get(name) { - match sym.scope { - SymbolScope::GlobalExplicit => return Some(SymbolScope::GlobalExplicit), - SymbolScope::GlobalImplicit => {} - _ => { - if sym.is_bound() { - decl_depth = Some(i); - break; - } - } - } - } - } - - if let Some(decl_depth) = decl_depth { - // decl_depth is the number of tables between the current one and - // the one that declared the cell var - for (table, typ) in self.tables.iter_mut().rev().take(decl_depth) { - if let SymbolTableType::Class = typ { - if let Some(free_class) = table.get_mut(name) { - free_class.flags.insert(SymbolFlags::FREE_CLASS) - } else { - let mut symbol = Symbol::new(name); - symbol.flags.insert(SymbolFlags::FREE_CLASS); - symbol.scope = SymbolScope::Free; - table.insert(name.to_owned(), symbol); - } - } else if !table.contains_key(name) { - let mut symbol = Symbol::new(name); - symbol.scope = SymbolScope::Free; - // symbol.is_referenced = true; - table.insert(name.to_owned(), symbol); - } - } - } - - decl_depth.map(|_| SymbolScope::Free) - } - - fn found_in_inner_scope( - &self, - sub_tables: &[SymbolTable], - name: &str, - st_typ: SymbolTableType, - ) -> Option<SymbolScope> { - sub_tables.iter().find_map(|st| { - let sym = st.symbols.get(name)?; - if sym.scope == SymbolScope::Free || sym.flags.contains(SymbolFlags::FREE_CLASS) { - if st_typ == SymbolTableType::Class && name != "__class__" { - None - } else { - Some(SymbolScope::Cell) - } - } else if sym.scope == SymbolScope::GlobalExplicit && self.tables.is_empty() { - // the symbol is defined on the module level, and an inner scope declares - // a global that points to it - Some(SymbolScope::GlobalExplicit) - } else { - None - } - }) - } - - // Implements the symbol analysis and scope extension for names - // assigned by a named expression in a comprehension. See: - // https://github.com/python/cpython/blob/7b78e7f9fd77bb3280ee39fb74b86772a7d46a70/Python/symtable.c#L1435 - fn analyze_symbol_comprehension( - &mut self, - symbol: &mut Symbol, - parent_offset: usize, - ) -> SymbolTableResult { - // when this is called, we expect to be in the direct parent scope of the scope that contains 'symbol' - let last = self.tables.iter_mut().rev().nth(parent_offset).unwrap(); - let symbols = &mut last.0; - let table_type = last.1; - - // it is not allowed to use an iterator variable as assignee in a named expression - if symbol.flags.contains(SymbolFlags::ITER) { - return Err(SymbolTableError { - error: format!( - "assignment expression cannot rebind comprehension iteration variable {}", - symbol.name - ), - // TODO: accurate location info, somehow - location: None, - }); - } - - match table_type { - SymbolTableType::Module => { - symbol.scope = SymbolScope::GlobalImplicit; - } - SymbolTableType::Class => { - // named expressions are forbidden in comprehensions on class scope - return Err(SymbolTableError { - error: "assignment expression within a comprehension cannot be used in a class body".to_string(), - // TODO: accurate location info, somehow - location: None, - }); - } - SymbolTableType::Function => { - if let Some(parent_symbol) = symbols.get_mut(&symbol.name) { - if let SymbolScope::Unknown = parent_symbol.scope { - // this information is new, as the assignment is done in inner scope - parent_symbol.flags.insert(SymbolFlags::ASSIGNED); - } - - symbol.scope = if parent_symbol.is_global() { - parent_symbol.scope - } else { - SymbolScope::Free - }; - } else { - let mut cloned_sym = symbol.clone(); - cloned_sym.scope = SymbolScope::Cell; - last.0.insert(cloned_sym.name.to_owned(), cloned_sym); - } - } - SymbolTableType::Comprehension => { - // TODO check for conflicts - requires more context information about variables - match symbols.get_mut(&symbol.name) { - Some(parent_symbol) => { - // check if assignee is an iterator in top scope - if parent_symbol.flags.contains(SymbolFlags::ITER) { - return Err(SymbolTableError { - error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name), - location: None, - }); - } - - // we synthesize the assignment to the symbol from inner scope - parent_symbol.flags.insert(SymbolFlags::ASSIGNED); // more checks are required - } - None => { - // extend the scope of the inner symbol - // as we are in a nested comprehension, we expect that the symbol is needed - // outside, too, and set it therefore to non-local scope. I.e., we expect to - // find a definition on a higher level - let mut cloned_sym = symbol.clone(); - cloned_sym.scope = SymbolScope::Free; - last.0.insert(cloned_sym.name.to_owned(), cloned_sym); - } - } - - self.analyze_symbol_comprehension(symbol, parent_offset + 1)?; - } - } - Ok(()) - } -} - -#[derive(Debug, Clone)] -enum SymbolUsage { - Global, - Nonlocal, - Used, - Assigned, - Imported, - AnnotationAssigned, - Parameter, - AnnotationParameter, - AssignedNamedExprInComprehension, - Iter, -} - -struct SymbolTableBuilder { - class_name: Option<String>, - // Scope stack. - tables: Vec<SymbolTable>, - future_annotations: bool, -} - -/// Enum to indicate in what mode an expression -/// was used. -/// In cpython this is stored in the AST, but I think this -/// is not logical, since it is not context free. -#[derive(Copy, Clone, PartialEq)] -enum ExpressionContext { - Load, - Store, - Delete, - Iter, - IterDefinitionExp, -} - -impl SymbolTableBuilder { - fn new() -> Self { - let mut this = Self { - class_name: None, - tables: vec![], - future_annotations: false, - }; - this.enter_scope("top", SymbolTableType::Module, 0); - this - } -} - -impl SymbolTableBuilder { - fn finish(mut self) -> Result<SymbolTable, SymbolTableError> { - assert_eq!(self.tables.len(), 1); - let mut symbol_table = self.tables.pop().unwrap(); - analyze_symbol_table(&mut symbol_table)?; - Ok(symbol_table) - } - - fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: u32) { - let is_nested = self - .tables - .last() - .map(|table| table.is_nested || table.typ == SymbolTableType::Function) - .unwrap_or(false); - let table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested); - self.tables.push(table); - } - - /// Pop symbol table and add to sub table of parent table. - fn leave_scope(&mut self) { - let table = self.tables.pop().unwrap(); - self.tables.last_mut().unwrap().sub_tables.push(table); - } - - fn scan_statements(&mut self, statements: &[ast::located::Stmt]) -> SymbolTableResult { - for statement in statements { - self.scan_statement(statement)?; - } - Ok(()) - } - - fn scan_parameters( - &mut self, - parameters: &[ast::located::ArgWithDefault], - ) -> SymbolTableResult { - for parameter in parameters { - let usage = if parameter.def.annotation.is_some() { - SymbolUsage::AnnotationParameter - } else { - SymbolUsage::Parameter - }; - self.register_name(parameter.def.arg.as_str(), usage, parameter.def.location())?; - } - Ok(()) - } - - fn scan_parameter(&mut self, parameter: &ast::located::Arg) -> SymbolTableResult { - let usage = if parameter.annotation.is_some() { - SymbolUsage::AnnotationParameter - } else { - SymbolUsage::Parameter - }; - self.register_name(parameter.arg.as_str(), usage, parameter.location()) - } - - fn scan_annotation(&mut self, annotation: &ast::located::Expr) -> SymbolTableResult { - if self.future_annotations { - Ok(()) - } else { - self.scan_expression(annotation, ExpressionContext::Load) - } - } - - fn scan_statement(&mut self, statement: &ast::located::Stmt) -> SymbolTableResult { - use ast::located::*; - if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement { - if module.as_ref().map(|id| id.as_str()) == Some("__future__") { - for feature in names { - if &feature.name == "annotations" { - self.future_annotations = true; - } - } - } - } - match &statement { - Stmt::Global(StmtGlobal { names, range }) => { - for name in names { - self.register_name(name.as_str(), SymbolUsage::Global, range.start)?; - } - } - Stmt::Nonlocal(StmtNonlocal { names, range }) => { - for name in names { - self.register_name(name.as_str(), SymbolUsage::Nonlocal, range.start)?; - } - } - Stmt::FunctionDef(StmtFunctionDef { - name, - body, - args, - decorator_list, - returns, - range, - .. - }) - | Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { - name, - body, - args, - decorator_list, - returns, - range, - .. - }) => { - self.scan_expressions(decorator_list, ExpressionContext::Load)?; - self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?; - if let Some(expression) = returns { - self.scan_annotation(expression)?; - } - self.enter_function(name.as_str(), args, range.start.row)?; - self.scan_statements(body)?; - self.leave_scope(); - } - Stmt::ClassDef(StmtClassDef { - name, - body, - bases, - keywords, - decorator_list, - type_params: _, - range, - }) => { - self.enter_scope(name.as_str(), SymbolTableType::Class, range.start.row.get()); - let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_string())); - self.register_name("__module__", SymbolUsage::Assigned, range.start)?; - self.register_name("__qualname__", SymbolUsage::Assigned, range.start)?; - self.register_name("__doc__", SymbolUsage::Assigned, range.start)?; - self.register_name("__class__", SymbolUsage::Assigned, range.start)?; - self.scan_statements(body)?; - self.leave_scope(); - self.class_name = prev_class; - self.scan_expressions(bases, ExpressionContext::Load)?; - for keyword in keywords { - self.scan_expression(&keyword.value, ExpressionContext::Load)?; - } - self.scan_expressions(decorator_list, ExpressionContext::Load)?; - self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?; - } - Stmt::Expr(StmtExpr { value, .. }) => { - self.scan_expression(value, ExpressionContext::Load)? - } - Stmt::If(StmtIf { - test, body, orelse, .. - }) => { - self.scan_expression(test, ExpressionContext::Load)?; - self.scan_statements(body)?; - self.scan_statements(orelse)?; - } - Stmt::For(StmtFor { - target, - iter, - body, - orelse, - .. - }) - | Stmt::AsyncFor(StmtAsyncFor { - target, - iter, - body, - orelse, - .. - }) => { - self.scan_expression(target, ExpressionContext::Store)?; - self.scan_expression(iter, ExpressionContext::Load)?; - self.scan_statements(body)?; - self.scan_statements(orelse)?; - } - Stmt::While(StmtWhile { - test, body, orelse, .. - }) => { - self.scan_expression(test, ExpressionContext::Load)?; - self.scan_statements(body)?; - self.scan_statements(orelse)?; - } - Stmt::Break(_) | Stmt::Continue(_) | Stmt::Pass(_) => { - // No symbols here. - } - Stmt::Import(StmtImport { names, range }) - | Stmt::ImportFrom(StmtImportFrom { names, range, .. }) => { - for name in names { - if let Some(alias) = &name.asname { - // `import my_module as my_alias` - self.register_name(alias.as_str(), SymbolUsage::Imported, range.start)?; - } else { - // `import module` - self.register_name( - name.name.split('.').next().unwrap(), - SymbolUsage::Imported, - range.start, - )?; - } - } - } - Stmt::Return(StmtReturn { value, .. }) => { - if let Some(expression) = value { - self.scan_expression(expression, ExpressionContext::Load)?; - } - } - Stmt::Assert(StmtAssert { test, msg, .. }) => { - self.scan_expression(test, ExpressionContext::Load)?; - if let Some(expression) = msg { - self.scan_expression(expression, ExpressionContext::Load)?; - } - } - Stmt::Delete(StmtDelete { targets, .. }) => { - self.scan_expressions(targets, ExpressionContext::Delete)?; - } - Stmt::Assign(StmtAssign { targets, value, .. }) => { - self.scan_expressions(targets, ExpressionContext::Store)?; - self.scan_expression(value, ExpressionContext::Load)?; - } - Stmt::AugAssign(StmtAugAssign { target, value, .. }) => { - self.scan_expression(target, ExpressionContext::Store)?; - self.scan_expression(value, ExpressionContext::Load)?; - } - Stmt::AnnAssign(StmtAnnAssign { - target, - annotation, - value, - simple, - range, - }) => { - // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233 - match &**target { - Expr::Name(ast::ExprName { id, .. }) if *simple => { - self.register_name( - id.as_str(), - SymbolUsage::AnnotationAssigned, - range.start, - )?; - } - _ => { - self.scan_expression(target, ExpressionContext::Store)?; - } - } - self.scan_annotation(annotation)?; - if let Some(value) = value { - self.scan_expression(value, ExpressionContext::Load)?; - } - } - Stmt::With(StmtWith { items, body, .. }) - | Stmt::AsyncWith(StmtAsyncWith { items, body, .. }) => { - for item in items { - self.scan_expression(&item.context_expr, ExpressionContext::Load)?; - if let Some(expression) = &item.optional_vars { - self.scan_expression(expression, ExpressionContext::Store)?; - } - } - self.scan_statements(body)?; - } - Stmt::Try(StmtTry { - body, - handlers, - orelse, - finalbody, - range, - }) - | Stmt::TryStar(StmtTryStar { - body, - handlers, - orelse, - finalbody, - range, - }) => { - self.scan_statements(body)?; - for handler in handlers { - let ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { - type_, - name, - body, - .. - }) = &handler; - if let Some(expression) = type_ { - self.scan_expression(expression, ExpressionContext::Load)?; - } - if let Some(name) = name { - self.register_name(name.as_str(), SymbolUsage::Assigned, range.start)?; - } - self.scan_statements(body)?; - } - self.scan_statements(orelse)?; - self.scan_statements(finalbody)?; - } - Stmt::Match(StmtMatch { subject, .. }) => { - return Err(SymbolTableError { - error: "match expression is not implemented yet".to_owned(), - location: Some(subject.location()), - }); - } - Stmt::Raise(StmtRaise { exc, cause, .. }) => { - if let Some(expression) = exc { - self.scan_expression(expression, ExpressionContext::Load)?; - } - if let Some(expression) = cause { - self.scan_expression(expression, ExpressionContext::Load)?; - } - } - Stmt::TypeAlias(StmtTypeAlias { .. }) => {} - } - Ok(()) - } - - fn scan_expressions( - &mut self, - expressions: &[ast::located::Expr], - context: ExpressionContext, - ) -> SymbolTableResult { - for expression in expressions { - self.scan_expression(expression, context)?; - } - Ok(()) - } - - fn scan_expression( - &mut self, - expression: &ast::located::Expr, - context: ExpressionContext, - ) -> SymbolTableResult { - use ast::located::*; - match expression { - Expr::BinOp(ExprBinOp { - left, - right, - range: _, - .. - }) => { - self.scan_expression(left, context)?; - self.scan_expression(right, context)?; - } - Expr::BoolOp(ExprBoolOp { - values, range: _, .. - }) => { - self.scan_expressions(values, context)?; - } - Expr::Compare(ExprCompare { - left, - comparators, - range: _, - .. - }) => { - self.scan_expression(left, context)?; - self.scan_expressions(comparators, context)?; - } - Expr::Subscript(ExprSubscript { - value, - slice, - range: _, - .. - }) => { - self.scan_expression(value, ExpressionContext::Load)?; - self.scan_expression(slice, ExpressionContext::Load)?; - } - Expr::Attribute(ExprAttribute { - value, range: _, .. - }) => { - self.scan_expression(value, ExpressionContext::Load)?; - } - Expr::Dict(ExprDict { - keys, - values, - range: _, - }) => { - for (key, value) in keys.iter().zip(values.iter()) { - if let Some(key) = key { - self.scan_expression(key, context)?; - } - self.scan_expression(value, context)?; - } - } - Expr::Await(ExprAwait { value, range: _ }) => { - self.scan_expression(value, context)?; - } - Expr::Yield(ExprYield { value, range: _ }) => { - if let Some(expression) = value { - self.scan_expression(expression, context)?; - } - } - Expr::YieldFrom(ExprYieldFrom { value, range: _ }) => { - self.scan_expression(value, context)?; - } - Expr::UnaryOp(ExprUnaryOp { - operand, range: _, .. - }) => { - self.scan_expression(operand, context)?; - } - Expr::Constant(ExprConstant { range: _, .. }) => {} - Expr::Starred(ExprStarred { - value, range: _, .. - }) => { - self.scan_expression(value, context)?; - } - Expr::Tuple(ExprTuple { elts, range: _, .. }) - | Expr::Set(ExprSet { elts, range: _, .. }) - | Expr::List(ExprList { elts, range: _, .. }) => { - self.scan_expressions(elts, context)?; - } - Expr::Slice(ExprSlice { - lower, - upper, - step, - range: _, - }) => { - if let Some(lower) = lower { - self.scan_expression(lower, context)?; - } - if let Some(upper) = upper { - self.scan_expression(upper, context)?; - } - if let Some(step) = step { - self.scan_expression(step, context)?; - } - } - Expr::GeneratorExp(ExprGeneratorExp { - elt, - generators, - range, - }) => { - self.scan_comprehension("genexpr", elt, None, generators, range.start)?; - } - Expr::ListComp(ExprListComp { - elt, - generators, - range, - }) => { - self.scan_comprehension("genexpr", elt, None, generators, range.start)?; - } - Expr::SetComp(ExprSetComp { - elt, - generators, - range, - }) => { - self.scan_comprehension("genexpr", elt, None, generators, range.start)?; - } - Expr::DictComp(ExprDictComp { - key, - value, - generators, - range, - }) => { - self.scan_comprehension("genexpr", key, Some(value), generators, range.start)?; - } - Expr::Call(ExprCall { - func, - args, - keywords, - range: _, - }) => { - match context { - ExpressionContext::IterDefinitionExp => { - self.scan_expression(func, ExpressionContext::IterDefinitionExp)?; - } - _ => { - self.scan_expression(func, ExpressionContext::Load)?; - } - } - - self.scan_expressions(args, ExpressionContext::Load)?; - for keyword in keywords { - self.scan_expression(&keyword.value, ExpressionContext::Load)?; - } - } - Expr::FormattedValue(ExprFormattedValue { - value, - format_spec, - range: _, - .. - }) => { - self.scan_expression(value, ExpressionContext::Load)?; - if let Some(spec) = format_spec { - self.scan_expression(spec, ExpressionContext::Load)?; - } - } - Expr::JoinedStr(ExprJoinedStr { values, range: _ }) => { - for value in values { - self.scan_expression(value, ExpressionContext::Load)?; - } - } - Expr::Name(ExprName { id, range, .. }) => { - let id = id.as_str(); - // Determine the contextual usage of this symbol: - match context { - ExpressionContext::Delete => { - self.register_name(id, SymbolUsage::Assigned, range.start)?; - self.register_name(id, SymbolUsage::Used, range.start)?; - } - ExpressionContext::Load | ExpressionContext::IterDefinitionExp => { - self.register_name(id, SymbolUsage::Used, range.start)?; - } - ExpressionContext::Store => { - self.register_name(id, SymbolUsage::Assigned, range.start)?; - } - ExpressionContext::Iter => { - self.register_name(id, SymbolUsage::Iter, range.start)?; - } - } - // Interesting stuff about the __class__ variable: - // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object - if context == ExpressionContext::Load - && self.tables.last().unwrap().typ == SymbolTableType::Function - && id == "super" - { - self.register_name("__class__", SymbolUsage::Used, range.start)?; - } - } - Expr::Lambda(ExprLambda { - args, - body, - range: _, - }) => { - self.enter_function("lambda", args, expression.location().row)?; - match context { - ExpressionContext::IterDefinitionExp => { - self.scan_expression(body, ExpressionContext::IterDefinitionExp)?; - } - _ => { - self.scan_expression(body, ExpressionContext::Load)?; - } - } - self.leave_scope(); - } - Expr::IfExp(ExprIfExp { - test, - body, - orelse, - range: _, - }) => { - self.scan_expression(test, ExpressionContext::Load)?; - self.scan_expression(body, ExpressionContext::Load)?; - self.scan_expression(orelse, ExpressionContext::Load)?; - } - - Expr::NamedExpr(ExprNamedExpr { - target, - value, - range, - }) => { - // named expressions are not allowed in the definition of - // comprehension iterator definitions - if let ExpressionContext::IterDefinitionExp = context { - return Err(SymbolTableError { - error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), - location: Some(target.location()), - }); - } - - self.scan_expression(value, ExpressionContext::Load)?; - - // special handling for assigned identifier in named expressions - // that are used in comprehensions. This required to correctly - // propagate the scope of the named assigned named and not to - // propagate inner names. - if let Expr::Name(ExprName { id, .. }) = &**target { - let id = id.as_str(); - let table = self.tables.last().unwrap(); - if table.typ == SymbolTableType::Comprehension { - self.register_name( - id, - SymbolUsage::AssignedNamedExprInComprehension, - range.start, - )?; - } else { - // omit one recursion. When the handling of an store changes for - // Identifiers this needs adapted - more forward safe would be - // calling scan_expression directly. - self.register_name(id, SymbolUsage::Assigned, range.start)?; - } - } else { - self.scan_expression(target, ExpressionContext::Store)?; - } - } - } - Ok(()) - } - - fn scan_comprehension( - &mut self, - scope_name: &str, - elt1: &ast::located::Expr, - elt2: Option<&ast::located::Expr>, - generators: &[ast::located::Comprehension], - location: SourceLocation, - ) -> SymbolTableResult { - // Comprehensions are compiled as functions, so create a scope for them: - self.enter_scope( - scope_name, - SymbolTableType::Comprehension, - location.row.get(), - ); - - // Register the passed argument to the generator function as the name ".0" - self.register_name(".0", SymbolUsage::Parameter, location)?; - - self.scan_expression(elt1, ExpressionContext::Load)?; - if let Some(elt2) = elt2 { - self.scan_expression(elt2, ExpressionContext::Load)?; - } - - let mut is_first_generator = true; - for generator in generators { - self.scan_expression(&generator.target, ExpressionContext::Iter)?; - if is_first_generator { - is_first_generator = false; - } else { - self.scan_expression(&generator.iter, ExpressionContext::IterDefinitionExp)?; - } - - for if_expr in &generator.ifs { - self.scan_expression(if_expr, ExpressionContext::Load)?; - } - } - - self.leave_scope(); - - // The first iterable is passed as an argument into the created function: - assert!(!generators.is_empty()); - self.scan_expression(&generators[0].iter, ExpressionContext::IterDefinitionExp)?; - - Ok(()) - } - - fn enter_function( - &mut self, - name: &str, - args: &ast::located::Arguments, - line_number: LineNumber, - ) -> SymbolTableResult { - // Evaluate eventual default parameters: - for default in args - .posonlyargs - .iter() - .chain(args.args.iter()) - .chain(args.kwonlyargs.iter()) - .filter_map(|arg| arg.default.as_ref()) - { - self.scan_expression(default, ExpressionContext::Load)?; // not ExprContext? - } - - // Annotations are scanned in outer scope: - for annotation in args - .posonlyargs - .iter() - .chain(args.args.iter()) - .chain(args.kwonlyargs.iter()) - .filter_map(|arg| arg.def.annotation.as_ref()) - { - self.scan_annotation(annotation)?; - } - if let Some(annotation) = args.vararg.as_ref().and_then(|arg| arg.annotation.as_ref()) { - self.scan_annotation(annotation)?; - } - if let Some(annotation) = args.kwarg.as_ref().and_then(|arg| arg.annotation.as_ref()) { - self.scan_annotation(annotation)?; - } - - self.enter_scope(name, SymbolTableType::Function, line_number.get()); - - // Fill scope with parameter names: - self.scan_parameters(&args.posonlyargs)?; - self.scan_parameters(&args.args)?; - self.scan_parameters(&args.kwonlyargs)?; - if let Some(name) = &args.vararg { - self.scan_parameter(name)?; - } - if let Some(name) = &args.kwarg { - self.scan_parameter(name)?; - } - Ok(()) - } - - fn register_name( - &mut self, - name: &str, - role: SymbolUsage, - location: SourceLocation, - ) -> SymbolTableResult { - let location = Some(location); - let scope_depth = self.tables.len(); - let table = self.tables.last_mut().unwrap(); - - let name = mangle_name(self.class_name.as_deref(), name); - // Some checks for the symbol that present on this scope level: - let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) { - let flags = &symbol.flags; - // Role already set.. - match role { - SymbolUsage::Global if !symbol.is_global() => { - if flags.contains(SymbolFlags::PARAMETER) { - return Err(SymbolTableError { - error: format!("name '{name}' is parameter and global"), - location, - }); - } - if flags.contains(SymbolFlags::REFERENCED) { - return Err(SymbolTableError { - error: format!("name '{name}' is used prior to global declaration"), - location, - }); - } - if flags.contains(SymbolFlags::ANNOTATED) { - return Err(SymbolTableError { - error: format!("annotated name '{name}' can't be global"), - location, - }); - } - if flags.contains(SymbolFlags::ASSIGNED) { - return Err(SymbolTableError { - error: format!( - "name '{name}' is assigned to before global declaration" - ), - location, - }); - } - } - SymbolUsage::Nonlocal => { - if flags.contains(SymbolFlags::PARAMETER) { - return Err(SymbolTableError { - error: format!("name '{name}' is parameter and nonlocal"), - location, - }); - } - if flags.contains(SymbolFlags::REFERENCED) { - return Err(SymbolTableError { - error: format!("name '{name}' is used prior to nonlocal declaration"), - location, - }); - } - if flags.contains(SymbolFlags::ANNOTATED) { - return Err(SymbolTableError { - error: format!("annotated name '{name}' can't be nonlocal"), - location, - }); - } - if flags.contains(SymbolFlags::ASSIGNED) { - return Err(SymbolTableError { - error: format!( - "name '{name}' is assigned to before nonlocal declaration" - ), - location, - }); - } - } - _ => { - // Ok? - } - } - symbol - } else { - // The symbol does not present on this scope level. - // Some checks to insert new symbol into symbol table: - match role { - SymbolUsage::Nonlocal if scope_depth < 2 => { - return Err(SymbolTableError { - error: format!("cannot define nonlocal '{name}' at top level."), - location, - }) - } - _ => { - // Ok! - } - } - // Insert symbol when required: - let symbol = Symbol::new(name.as_ref()); - table.symbols.entry(name.into_owned()).or_insert(symbol) - }; - - // Set proper scope and flags on symbol: - let flags = &mut symbol.flags; - match role { - SymbolUsage::Nonlocal => { - symbol.scope = SymbolScope::Free; - flags.insert(SymbolFlags::NONLOCAL); - } - SymbolUsage::Imported => { - flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::IMPORTED); - } - SymbolUsage::Parameter => { - flags.insert(SymbolFlags::PARAMETER); - } - SymbolUsage::AnnotationParameter => { - flags.insert(SymbolFlags::PARAMETER | SymbolFlags::ANNOTATED); - } - SymbolUsage::AnnotationAssigned => { - flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ANNOTATED); - } - SymbolUsage::Assigned => { - flags.insert(SymbolFlags::ASSIGNED); - } - SymbolUsage::AssignedNamedExprInComprehension => { - flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ASSIGNED_IN_COMPREHENSION); - } - SymbolUsage::Global => { - symbol.scope = SymbolScope::GlobalExplicit; - } - SymbolUsage::Used => { - flags.insert(SymbolFlags::REFERENCED); - } - SymbolUsage::Iter => { - flags.insert(SymbolFlags::ITER); - } - } - - // and even more checking - // it is not allowed to assign to iterator variables (by named expressions) - if flags.contains(SymbolFlags::ITER | SymbolFlags::ASSIGNED) - /*&& symbol.is_assign_named_expr_in_comprehension*/ - { - return Err(SymbolTableError { - error: - "assignment expression cannot be used in a comprehension iterable expression" - .to_string(), - location, - }); - } - Ok(()) - } -} - -pub(crate) fn mangle_name<'a>(class_name: Option<&str>, name: &'a str) -> Cow<'a, str> { - let class_name = match class_name { - Some(n) => n, - None => return name.into(), - }; - if !name.starts_with("__") || name.ends_with("__") || name.contains('.') { - return name.into(); - } - // strip leading underscore - let class_name = class_name.strip_prefix(|c| c == '_').unwrap_or(class_name); - let mut ret = String::with_capacity(1 + class_name.len() + name.len()); - ret.push('_'); - ret.push_str(class_name); - ret.push_str(name); - ret.into() -} diff --git a/compiler/core/Cargo.toml b/compiler/core/Cargo.toml deleted file mode 100644 index 12413b7b9e6..00000000000 --- a/compiler/core/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "rustpython-compiler-core" -description = "RustPython specific bytecode." -version = "0.3.0" -authors = ["RustPython Team"] -edition = "2021" -repository = "https://github.com/RustPython/RustPython" -license = "MIT" - -[dependencies] -rustpython-parser-core = { workspace = true, features=["location"] } - -bitflags = { workspace = true } -itertools = { workspace = true } -malachite-bigint = { workspace = true } -num-complex = { workspace = true } -serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] } - -lz4_flex = "0.11" diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs deleted file mode 100644 index 8ada60a9e6e..00000000000 --- a/compiler/core/src/bytecode.rs +++ /dev/null @@ -1,1481 +0,0 @@ -//! Implement python as a virtual machine with bytecode. This module -//! implements bytecode structure. - -use bitflags::bitflags; -use itertools::Itertools; -use malachite_bigint::BigInt; -use num_complex::Complex64; -use rustpython_parser_core::source_code::{OneIndexed, SourceLocation}; -use std::marker::PhantomData; -use std::{collections::BTreeSet, fmt, hash, mem}; - -pub use rustpython_parser_core::ConversionFlag; - -pub trait Constant: Sized { - type Name: AsRef<str>; - - /// Transforms the given Constant to a BorrowedConstant - fn borrow_constant(&self) -> BorrowedConstant<Self>; -} - -impl Constant for ConstantData { - type Name = String; - fn borrow_constant(&self) -> BorrowedConstant<Self> { - use BorrowedConstant::*; - match self { - ConstantData::Integer { value } => Integer { value }, - ConstantData::Float { value } => Float { value: *value }, - ConstantData::Complex { value } => Complex { value: *value }, - ConstantData::Boolean { value } => Boolean { value: *value }, - ConstantData::Str { value } => Str { value }, - ConstantData::Bytes { value } => Bytes { value }, - ConstantData::Code { code } => Code { code }, - ConstantData::Tuple { elements } => Tuple { elements }, - ConstantData::None => None, - ConstantData::Ellipsis => Ellipsis, - } - } -} - -/// A Constant Bag -pub trait ConstantBag: Sized + Copy { - type Constant: Constant; - fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant; - fn make_int(&self, value: BigInt) -> Self::Constant; - fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant; - fn make_code(&self, code: CodeObject<Self::Constant>) -> Self::Constant; - fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name; -} - -pub trait AsBag { - type Bag: ConstantBag; - #[allow(clippy::wrong_self_convention)] - fn as_bag(self) -> Self::Bag; -} - -impl<Bag: ConstantBag> AsBag for Bag { - type Bag = Self; - fn as_bag(self) -> Self { - self - } -} - -#[derive(Clone, Copy)] -pub struct BasicBag; - -impl ConstantBag for BasicBag { - type Constant = ConstantData; - fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant { - constant.to_owned() - } - fn make_int(&self, value: BigInt) -> Self::Constant { - ConstantData::Integer { value } - } - fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant { - ConstantData::Tuple { - elements: elements.collect(), - } - } - fn make_code(&self, code: CodeObject<Self::Constant>) -> Self::Constant { - ConstantData::Code { - code: Box::new(code), - } - } - fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name { - name.to_owned() - } -} - -/// Primary container of a single code object. Each python function has -/// a code object. Also a module has a code object. -#[derive(Clone)] -pub struct CodeObject<C: Constant = ConstantData> { - pub instructions: Box<[CodeUnit]>, - pub locations: Box<[SourceLocation]>, - pub flags: CodeFlags, - pub posonlyarg_count: u32, - // Number of positional-only arguments - pub arg_count: u32, - pub kwonlyarg_count: u32, - pub source_path: C::Name, - pub first_line_number: Option<OneIndexed>, - pub max_stackdepth: u32, - pub obj_name: C::Name, - // Name of the object that created this code object - pub cell2arg: Option<Box<[i32]>>, - pub constants: Box<[C]>, - pub names: Box<[C::Name]>, - pub varnames: Box<[C::Name]>, - pub cellvars: Box<[C::Name]>, - pub freevars: Box<[C::Name]>, -} - -bitflags! { - #[derive(Copy, Clone, Debug, PartialEq)] - pub struct CodeFlags: u16 { - const NEW_LOCALS = 0x01; - const IS_GENERATOR = 0x02; - const IS_COROUTINE = 0x04; - const HAS_VARARGS = 0x08; - const HAS_VARKEYWORDS = 0x10; - const IS_OPTIMIZED = 0x20; - } -} - -impl CodeFlags { - pub const NAME_MAPPING: &'static [(&'static str, CodeFlags)] = &[ - ("GENERATOR", CodeFlags::IS_GENERATOR), - ("COROUTINE", CodeFlags::IS_COROUTINE), - ( - "ASYNC_GENERATOR", - Self::from_bits_truncate(Self::IS_GENERATOR.bits() | Self::IS_COROUTINE.bits()), - ), - ("VARARGS", CodeFlags::HAS_VARARGS), - ("VARKEYWORDS", CodeFlags::HAS_VARKEYWORDS), - ]; -} - -/// an opcode argument that may be extended by a prior ExtendedArg -#[derive(Copy, Clone, PartialEq, Eq)] -#[repr(transparent)] -pub struct OpArgByte(pub u8); -impl OpArgByte { - pub const fn null() -> Self { - OpArgByte(0) - } -} -impl fmt::Debug for OpArgByte { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -/// a full 32-bit op_arg, including any possible ExtendedArg extension -#[derive(Copy, Clone, Debug)] -#[repr(transparent)] -pub struct OpArg(pub u32); -impl OpArg { - pub const fn null() -> Self { - OpArg(0) - } - - /// Returns how many CodeUnits a instruction with this op_arg will be encoded as - #[inline] - pub fn instr_size(self) -> usize { - (self.0 > 0xff) as usize + (self.0 > 0xff_ff) as usize + (self.0 > 0xff_ff_ff) as usize + 1 - } - - /// returns the arg split into any necessary ExtendedArg components (in big-endian order) and - /// the arg for the real opcode itself - #[inline(always)] - pub fn split(self) -> (impl ExactSizeIterator<Item = OpArgByte>, OpArgByte) { - let mut it = self - .0 - .to_le_bytes() - .map(OpArgByte) - .into_iter() - .take(self.instr_size()); - let lo = it.next().unwrap(); - (it.rev(), lo) - } -} - -#[derive(Default, Copy, Clone)] -#[repr(transparent)] -pub struct OpArgState { - state: u32, -} - -impl OpArgState { - #[inline(always)] - pub fn get(&mut self, ins: CodeUnit) -> (Instruction, OpArg) { - let arg = self.extend(ins.arg); - if ins.op != Instruction::ExtendedArg { - self.reset(); - } - (ins.op, arg) - } - #[inline(always)] - pub fn extend(&mut self, arg: OpArgByte) -> OpArg { - self.state = self.state << 8 | u32::from(arg.0); - OpArg(self.state) - } - #[inline(always)] - pub fn reset(&mut self) { - self.state = 0 - } -} - -pub trait OpArgType: Copy { - fn from_op_arg(x: u32) -> Option<Self>; - fn to_op_arg(self) -> u32; -} - -impl OpArgType for u32 { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - Some(x) - } - #[inline(always)] - fn to_op_arg(self) -> u32 { - self - } -} - -impl OpArgType for bool { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - Some(x != 0) - } - #[inline(always)] - fn to_op_arg(self) -> u32 { - self as u32 - } -} - -macro_rules! op_arg_enum_impl { - (enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $value:literal,)* }) => { - impl OpArgType for $name { - fn to_op_arg(self) -> u32 { - self as u32 - } - fn from_op_arg(x: u32) -> Option<Self> { - Some(match u8::try_from(x).ok()? { - $($value => Self::$var,)* - _ => return None, - }) - } - } - }; -} - -macro_rules! op_arg_enum { - ($(#[$attr:meta])* $vis:vis enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $value:literal,)* }) => { - $(#[$attr])* - $vis enum $name { - $($(#[$var_attr])* $var = $value,)* - } - - op_arg_enum_impl!(enum $name { - $($(#[$var_attr])* $var = $value,)* - }); - }; -} - -#[derive(Copy, Clone)] -pub struct Arg<T: OpArgType>(PhantomData<T>); - -impl<T: OpArgType> Arg<T> { - #[inline] - pub fn marker() -> Self { - Arg(PhantomData) - } - #[inline] - pub fn new(arg: T) -> (Self, OpArg) { - (Self(PhantomData), OpArg(arg.to_op_arg())) - } - #[inline] - pub fn new_single(arg: T) -> (Self, OpArgByte) - where - T: Into<u8>, - { - (Self(PhantomData), OpArgByte(arg.into())) - } - #[inline(always)] - pub fn get(self, arg: OpArg) -> T { - self.try_get(arg).unwrap() - } - #[inline(always)] - pub fn try_get(self, arg: OpArg) -> Option<T> { - T::from_op_arg(arg.0) - } - #[inline(always)] - /// # Safety - /// T::from_op_arg(self) must succeed - pub unsafe fn get_unchecked(self, arg: OpArg) -> T { - match T::from_op_arg(arg.0) { - Some(t) => t, - None => std::hint::unreachable_unchecked(), - } - } -} - -impl<T: OpArgType> PartialEq for Arg<T> { - fn eq(&self, _: &Self) -> bool { - true - } -} -impl<T: OpArgType> Eq for Arg<T> {} - -impl<T: OpArgType> fmt::Debug for Arg<T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Arg<{}>", std::any::type_name::<T>()) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] -#[repr(transparent)] -// XXX: if you add a new instruction that stores a Label, make sure to add it in -// Instruction::label_arg -pub struct Label(pub u32); - -impl OpArgType for Label { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - Some(Label(x)) - } - #[inline(always)] - fn to_op_arg(self) -> u32 { - self.0 - } -} - -impl fmt::Display for Label { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.fmt(f) - } -} - -impl OpArgType for ConversionFlag { - #[inline] - fn from_op_arg(x: u32) -> Option<Self> { - match x as u8 { - b's' => Some(ConversionFlag::Str), - b'a' => Some(ConversionFlag::Ascii), - b'r' => Some(ConversionFlag::Repr), - std::u8::MAX => Some(ConversionFlag::None), - _ => None, - } - } - #[inline] - fn to_op_arg(self) -> u32 { - self as i8 as u8 as u32 - } -} - -op_arg_enum!( - /// The kind of Raise that occurred. - #[derive(Copy, Clone, Debug, PartialEq, Eq)] - #[repr(u8)] - pub enum RaiseKind { - Reraise = 0, - Raise = 1, - RaiseCause = 2, - } -); - -pub type NameIdx = u32; - -/// A Single bytecode instruction. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(u8)] -pub enum Instruction { - /// Importing by name - ImportName { - idx: Arg<NameIdx>, - }, - /// Importing without name - ImportNameless, - /// Import * - ImportStar, - /// from ... import ... - ImportFrom { - idx: Arg<NameIdx>, - }, - LoadFast(Arg<NameIdx>), - LoadNameAny(Arg<NameIdx>), - LoadGlobal(Arg<NameIdx>), - LoadDeref(Arg<NameIdx>), - LoadClassDeref(Arg<NameIdx>), - StoreFast(Arg<NameIdx>), - StoreLocal(Arg<NameIdx>), - StoreGlobal(Arg<NameIdx>), - StoreDeref(Arg<NameIdx>), - DeleteFast(Arg<NameIdx>), - DeleteLocal(Arg<NameIdx>), - DeleteGlobal(Arg<NameIdx>), - DeleteDeref(Arg<NameIdx>), - LoadClosure(Arg<NameIdx>), - Subscript, - StoreSubscript, - DeleteSubscript, - StoreAttr { - idx: Arg<NameIdx>, - }, - DeleteAttr { - idx: Arg<NameIdx>, - }, - LoadConst { - /// index into constants vec - idx: Arg<u32>, - }, - UnaryOperation { - op: Arg<UnaryOperator>, - }, - BinaryOperation { - op: Arg<BinaryOperator>, - }, - BinaryOperationInplace { - op: Arg<BinaryOperator>, - }, - LoadAttr { - idx: Arg<NameIdx>, - }, - TestOperation { - op: Arg<TestOperator>, - }, - CompareOperation { - op: Arg<ComparisonOperator>, - }, - Pop, - Rotate2, - Rotate3, - Duplicate, - Duplicate2, - GetIter, - Continue { - target: Arg<Label>, - }, - Break { - target: Arg<Label>, - }, - Jump { - target: Arg<Label>, - }, - /// Pop the top of the stack, and jump if this value is true. - JumpIfTrue { - target: Arg<Label>, - }, - /// Pop the top of the stack, and jump if this value is false. - JumpIfFalse { - target: Arg<Label>, - }, - /// Peek at the top of the stack, and jump if this value is true. - /// Otherwise, pop top of stack. - JumpIfTrueOrPop { - target: Arg<Label>, - }, - /// Peek at the top of the stack, and jump if this value is false. - /// Otherwise, pop top of stack. - JumpIfFalseOrPop { - target: Arg<Label>, - }, - MakeFunction(Arg<MakeFunctionFlags>), - CallFunctionPositional { - nargs: Arg<u32>, - }, - CallFunctionKeyword { - nargs: Arg<u32>, - }, - CallFunctionEx { - has_kwargs: Arg<bool>, - }, - LoadMethod { - idx: Arg<NameIdx>, - }, - CallMethodPositional { - nargs: Arg<u32>, - }, - CallMethodKeyword { - nargs: Arg<u32>, - }, - CallMethodEx { - has_kwargs: Arg<bool>, - }, - ForIter { - target: Arg<Label>, - }, - ReturnValue, - YieldValue, - YieldFrom, - SetupAnnotation, - SetupLoop, - - /// Setup a finally handler, which will be called whenever one of this events occurs: - /// - the block is popped - /// - the function returns - /// - an exception is returned - SetupFinally { - handler: Arg<Label>, - }, - - /// Enter a finally block, without returning, excepting, just because we are there. - EnterFinally, - - /// Marker bytecode for the end of a finally sequence. - /// When this bytecode is executed, the eval loop does one of those things: - /// - Continue at a certain bytecode position - /// - Propagate the exception - /// - Return from a function - /// - Do nothing at all, just continue - EndFinally, - - SetupExcept { - handler: Arg<Label>, - }, - SetupWith { - end: Arg<Label>, - }, - WithCleanupStart, - WithCleanupFinish, - PopBlock, - Raise { - kind: Arg<RaiseKind>, - }, - BuildString { - size: Arg<u32>, - }, - BuildTuple { - size: Arg<u32>, - }, - BuildTupleUnpack { - size: Arg<u32>, - }, - BuildList { - size: Arg<u32>, - }, - BuildListUnpack { - size: Arg<u32>, - }, - BuildSet { - size: Arg<u32>, - }, - BuildSetUnpack { - size: Arg<u32>, - }, - BuildMap { - size: Arg<u32>, - }, - BuildMapForCall { - size: Arg<u32>, - }, - DictUpdate, - BuildSlice { - /// whether build a slice with a third step argument - step: Arg<bool>, - }, - ListAppend { - i: Arg<u32>, - }, - SetAdd { - i: Arg<u32>, - }, - MapAdd { - i: Arg<u32>, - }, - - PrintExpr, - LoadBuildClass, - UnpackSequence { - size: Arg<u32>, - }, - UnpackEx { - args: Arg<UnpackExArgs>, - }, - FormatValue { - conversion: Arg<ConversionFlag>, - }, - PopException, - Reverse { - amount: Arg<u32>, - }, - GetAwaitable, - BeforeAsyncWith, - SetupAsyncWith { - end: Arg<Label>, - }, - GetAIter, - GetANext, - EndAsyncFor, - ExtendedArg, -} -const _: () = assert!(mem::size_of::<Instruction>() == 1); - -impl From<Instruction> for u8 { - #[inline] - fn from(ins: Instruction) -> u8 { - // SAFETY: there's no padding bits - unsafe { std::mem::transmute::<Instruction, u8>(ins) } - } -} - -impl TryFrom<u8> for Instruction { - type Error = crate::marshal::MarshalError; - - #[inline] - fn try_from(value: u8) -> Result<Self, crate::marshal::MarshalError> { - if value <= u8::from(Instruction::ExtendedArg) { - Ok(unsafe { std::mem::transmute::<u8, Instruction>(value) }) - } else { - Err(crate::marshal::MarshalError::InvalidBytecode) - } - } -} - -#[derive(Copy, Clone)] -#[repr(C)] -pub struct CodeUnit { - pub op: Instruction, - pub arg: OpArgByte, -} - -const _: () = assert!(mem::size_of::<CodeUnit>() == 2); - -impl CodeUnit { - pub fn new(op: Instruction, arg: OpArgByte) -> Self { - Self { op, arg } - } -} - -use self::Instruction::*; - -bitflags! { - #[derive(Copy, Clone, Debug, PartialEq)] - pub struct MakeFunctionFlags: u8 { - const CLOSURE = 0x01; - const ANNOTATIONS = 0x02; - const KW_ONLY_DEFAULTS = 0x04; - const DEFAULTS = 0x08; - } -} -impl OpArgType for MakeFunctionFlags { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - MakeFunctionFlags::from_bits(x as u8) - } - #[inline(always)] - fn to_op_arg(self) -> u32 { - self.bits().into() - } -} - -/// A Constant (which usually encapsulates data within it) -/// -/// # Examples -/// ``` -/// use rustpython_compiler_core::bytecode::ConstantData; -/// let a = ConstantData::Float {value: 120f64}; -/// let b = ConstantData::Boolean {value: false}; -/// assert_ne!(a, b); -/// ``` -#[derive(Debug, Clone)] -pub enum ConstantData { - Tuple { elements: Vec<ConstantData> }, - Integer { value: BigInt }, - Float { value: f64 }, - Complex { value: Complex64 }, - Boolean { value: bool }, - Str { value: String }, - Bytes { value: Vec<u8> }, - Code { code: Box<CodeObject> }, - None, - Ellipsis, -} - -impl PartialEq for ConstantData { - fn eq(&self, other: &Self) -> bool { - use ConstantData::*; - match (self, other) { - (Integer { value: a }, Integer { value: b }) => a == b, - // we want to compare floats *by actual value* - if we have the *exact same* float - // already in a constant cache, we want to use that - (Float { value: a }, Float { value: b }) => a.to_bits() == b.to_bits(), - (Complex { value: a }, Complex { value: b }) => { - a.re.to_bits() == b.re.to_bits() && a.im.to_bits() == b.im.to_bits() - } - (Boolean { value: a }, Boolean { value: b }) => a == b, - (Str { value: a }, Str { value: b }) => a == b, - (Bytes { value: a }, Bytes { value: b }) => a == b, - (Code { code: a }, Code { code: b }) => std::ptr::eq(a.as_ref(), b.as_ref()), - (Tuple { elements: a }, Tuple { elements: b }) => a == b, - (None, None) => true, - (Ellipsis, Ellipsis) => true, - _ => false, - } - } -} - -impl Eq for ConstantData {} - -impl hash::Hash for ConstantData { - fn hash<H: hash::Hasher>(&self, state: &mut H) { - use ConstantData::*; - mem::discriminant(self).hash(state); - match self { - Integer { value } => value.hash(state), - Float { value } => value.to_bits().hash(state), - Complex { value } => { - value.re.to_bits().hash(state); - value.im.to_bits().hash(state); - } - Boolean { value } => value.hash(state), - Str { value } => value.hash(state), - Bytes { value } => value.hash(state), - Code { code } => std::ptr::hash(code.as_ref(), state), - Tuple { elements } => elements.hash(state), - None => {} - Ellipsis => {} - } - } -} - -/// A borrowed Constant -pub enum BorrowedConstant<'a, C: Constant> { - Integer { value: &'a BigInt }, - Float { value: f64 }, - Complex { value: Complex64 }, - Boolean { value: bool }, - Str { value: &'a str }, - Bytes { value: &'a [u8] }, - Code { code: &'a CodeObject<C> }, - Tuple { elements: &'a [C] }, - None, - Ellipsis, -} - -impl<C: Constant> Copy for BorrowedConstant<'_, C> {} -impl<C: Constant> Clone for BorrowedConstant<'_, C> { - fn clone(&self) -> Self { - *self - } -} - -impl<C: Constant> BorrowedConstant<'_, C> { - pub fn fmt_display(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - BorrowedConstant::Integer { value } => write!(f, "{value}"), - BorrowedConstant::Float { value } => write!(f, "{value}"), - BorrowedConstant::Complex { value } => write!(f, "{value}"), - BorrowedConstant::Boolean { value } => { - write!(f, "{}", if *value { "True" } else { "False" }) - } - BorrowedConstant::Str { value } => write!(f, "{value:?}"), - BorrowedConstant::Bytes { value } => write!(f, "b\"{}\"", value.escape_ascii()), - BorrowedConstant::Code { code } => write!(f, "{code:?}"), - BorrowedConstant::Tuple { elements } => { - write!(f, "(")?; - let mut first = true; - for c in *elements { - if first { - first = false - } else { - write!(f, ", ")?; - } - c.borrow_constant().fmt_display(f)?; - } - write!(f, ")") - } - BorrowedConstant::None => write!(f, "None"), - BorrowedConstant::Ellipsis => write!(f, "..."), - } - } - pub fn to_owned(self) -> ConstantData { - use ConstantData::*; - match self { - BorrowedConstant::Integer { value } => Integer { - value: value.clone(), - }, - BorrowedConstant::Float { value } => Float { value }, - BorrowedConstant::Complex { value } => Complex { value }, - BorrowedConstant::Boolean { value } => Boolean { value }, - BorrowedConstant::Str { value } => Str { - value: value.to_owned(), - }, - BorrowedConstant::Bytes { value } => Bytes { - value: value.to_owned(), - }, - BorrowedConstant::Code { code } => Code { - code: Box::new(code.map_clone_bag(&BasicBag)), - }, - BorrowedConstant::Tuple { elements } => Tuple { - elements: elements - .iter() - .map(|c| c.borrow_constant().to_owned()) - .collect(), - }, - BorrowedConstant::None => None, - BorrowedConstant::Ellipsis => Ellipsis, - } - } -} - -op_arg_enum!( - /// The possible comparison operators - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - #[repr(u8)] - pub enum ComparisonOperator { - // be intentional with bits so that we can do eval_ord with just a bitwise and - // bits: | Equal | Greater | Less | - Less = 0b001, - Greater = 0b010, - NotEqual = 0b011, - Equal = 0b100, - LessOrEqual = 0b101, - GreaterOrEqual = 0b110, - } -); - -op_arg_enum!( - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - #[repr(u8)] - pub enum TestOperator { - In = 0, - NotIn = 1, - Is = 2, - IsNot = 3, - /// two exceptions that match? - ExceptionMatch = 4, - } -); - -op_arg_enum!( - /// The possible Binary operators - /// # Examples - /// - /// ```ignore - /// use rustpython_compiler_core::Instruction::BinaryOperation; - /// use rustpython_compiler_core::BinaryOperator::Add; - /// let op = BinaryOperation {op: Add}; - /// ``` - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - #[repr(u8)] - pub enum BinaryOperator { - Power = 0, - Multiply = 1, - MatrixMultiply = 2, - Divide = 3, - FloorDivide = 4, - Modulo = 5, - Add = 6, - Subtract = 7, - Lshift = 8, - Rshift = 9, - And = 10, - Xor = 11, - Or = 12, - } -); - -op_arg_enum!( - /// The possible unary operators - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - #[repr(u8)] - pub enum UnaryOperator { - Not = 0, - Invert = 1, - Minus = 2, - Plus = 3, - } -); - -#[derive(Copy, Clone)] -pub struct UnpackExArgs { - pub before: u8, - pub after: u8, -} - -impl OpArgType for UnpackExArgs { - #[inline(always)] - fn from_op_arg(x: u32) -> Option<Self> { - let [before, after, ..] = x.to_le_bytes(); - Some(Self { before, after }) - } - #[inline(always)] - fn to_op_arg(self) -> u32 { - u32::from_le_bytes([self.before, self.after, 0, 0]) - } -} -impl fmt::Display for UnpackExArgs { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "before: {}, after: {}", self.before, self.after) - } -} - -/* -Maintain a stack of blocks on the VM. -pub enum BlockType { - Loop, - Except, -} -*/ - -/// Argument structure -pub struct Arguments<'a, N: AsRef<str>> { - pub posonlyargs: &'a [N], - pub args: &'a [N], - pub vararg: Option<&'a N>, - pub kwonlyargs: &'a [N], - pub varkwarg: Option<&'a N>, -} - -impl<N: AsRef<str>> fmt::Debug for Arguments<'_, N> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - macro_rules! fmt_slice { - ($x:expr) => { - format_args!("[{}]", $x.iter().map(AsRef::as_ref).format(", ")) - }; - } - f.debug_struct("Arguments") - .field("posonlyargs", &fmt_slice!(self.posonlyargs)) - .field("args", &fmt_slice!(self.posonlyargs)) - .field("vararg", &self.vararg.map(N::as_ref)) - .field("kwonlyargs", &fmt_slice!(self.kwonlyargs)) - .field("varkwarg", &self.varkwarg.map(N::as_ref)) - .finish() - } -} - -impl<C: Constant> CodeObject<C> { - /// Get all arguments of the code object - /// like inspect.getargs - pub fn arg_names(&self) -> Arguments<C::Name> { - let nargs = self.arg_count as usize; - let nkwargs = self.kwonlyarg_count as usize; - let mut varargs_pos = nargs + nkwargs; - let posonlyargs = &self.varnames[..self.posonlyarg_count as usize]; - let args = &self.varnames[..nargs]; - let kwonlyargs = &self.varnames[nargs..varargs_pos]; - - let vararg = if self.flags.contains(CodeFlags::HAS_VARARGS) { - let vararg = &self.varnames[varargs_pos]; - varargs_pos += 1; - Some(vararg) - } else { - None - }; - let varkwarg = if self.flags.contains(CodeFlags::HAS_VARKEYWORDS) { - Some(&self.varnames[varargs_pos]) - } else { - None - }; - - Arguments { - posonlyargs, - args, - vararg, - kwonlyargs, - varkwarg, - } - } - - /// Return the labels targeted by the instructions of this CodeObject - pub fn label_targets(&self) -> BTreeSet<Label> { - let mut label_targets = BTreeSet::new(); - let mut arg_state = OpArgState::default(); - for instruction in &*self.instructions { - let (instruction, arg) = arg_state.get(*instruction); - if let Some(l) = instruction.label_arg() { - label_targets.insert(l.get(arg)); - } - } - label_targets - } - - fn display_inner( - &self, - f: &mut fmt::Formatter, - expand_code_objects: bool, - level: usize, - ) -> fmt::Result { - let label_targets = self.label_targets(); - let line_digits = (3).max(self.locations.last().unwrap().row.to_string().len()); - let offset_digits = (4).max(self.instructions.len().to_string().len()); - let mut last_line = OneIndexed::MAX; - let mut arg_state = OpArgState::default(); - for (offset, &instruction) in self.instructions.iter().enumerate() { - let (instruction, arg) = arg_state.get(instruction); - // optional line number - let line = self.locations[offset].row; - if line != last_line { - if last_line != OneIndexed::MAX { - writeln!(f)?; - } - last_line = line; - write!(f, "{line:line_digits$}")?; - } else { - for _ in 0..line_digits { - write!(f, " ")?; - } - } - write!(f, " ")?; - - // level indent - for _ in 0..level { - write!(f, " ")?; - } - - // arrow and offset - let arrow = if label_targets.contains(&Label(offset as u32)) { - ">>" - } else { - " " - }; - write!(f, "{arrow} {offset:offset_digits$} ")?; - - // instruction - instruction.fmt_dis(arg, f, self, expand_code_objects, 21, level)?; - writeln!(f)?; - } - Ok(()) - } - - /// Recursively display this CodeObject - pub fn display_expand_code_objects(&self) -> impl fmt::Display + '_ { - struct Display<'a, C: Constant>(&'a CodeObject<C>); - impl<C: Constant> fmt::Display for Display<'_, C> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.0.display_inner(f, true, 1) - } - } - Display(self) - } - - /// Map this CodeObject to one that holds a Bag::Constant - pub fn map_bag<Bag: ConstantBag>(self, bag: Bag) -> CodeObject<Bag::Constant> { - let map_names = |names: Box<[C::Name]>| { - names - .into_vec() - .into_iter() - .map(|x| bag.make_name(x.as_ref())) - .collect::<Box<[_]>>() - }; - CodeObject { - constants: self - .constants - .into_vec() - .into_iter() - .map(|x| bag.make_constant(x.borrow_constant())) - .collect(), - names: map_names(self.names), - varnames: map_names(self.varnames), - cellvars: map_names(self.cellvars), - freevars: map_names(self.freevars), - source_path: bag.make_name(self.source_path.as_ref()), - obj_name: bag.make_name(self.obj_name.as_ref()), - - instructions: self.instructions, - locations: self.locations, - flags: self.flags, - posonlyarg_count: self.posonlyarg_count, - arg_count: self.arg_count, - kwonlyarg_count: self.kwonlyarg_count, - first_line_number: self.first_line_number, - max_stackdepth: self.max_stackdepth, - cell2arg: self.cell2arg, - } - } - - /// Same as `map_bag` but clones `self` - pub fn map_clone_bag<Bag: ConstantBag>(&self, bag: &Bag) -> CodeObject<Bag::Constant> { - let map_names = - |names: &[C::Name]| names.iter().map(|x| bag.make_name(x.as_ref())).collect(); - CodeObject { - constants: self - .constants - .iter() - .map(|x| bag.make_constant(x.borrow_constant())) - .collect(), - names: map_names(&self.names), - varnames: map_names(&self.varnames), - cellvars: map_names(&self.cellvars), - freevars: map_names(&self.freevars), - source_path: bag.make_name(self.source_path.as_ref()), - obj_name: bag.make_name(self.obj_name.as_ref()), - - instructions: self.instructions.clone(), - locations: self.locations.clone(), - flags: self.flags, - posonlyarg_count: self.posonlyarg_count, - arg_count: self.arg_count, - kwonlyarg_count: self.kwonlyarg_count, - first_line_number: self.first_line_number, - max_stackdepth: self.max_stackdepth, - cell2arg: self.cell2arg.clone(), - } - } -} - -impl<C: Constant> fmt::Display for CodeObject<C> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.display_inner(f, false, 1)?; - for constant in &*self.constants { - if let BorrowedConstant::Code { code } = constant.borrow_constant() { - writeln!(f, "\nDisassembly of {code:?}")?; - code.fmt(f)?; - } - } - Ok(()) - } -} - -impl Instruction { - /// Gets the label stored inside this instruction, if it exists - #[inline] - pub fn label_arg(&self) -> Option<Arg<Label>> { - match self { - Jump { target: l } - | JumpIfTrue { target: l } - | JumpIfFalse { target: l } - | JumpIfTrueOrPop { target: l } - | JumpIfFalseOrPop { target: l } - | ForIter { target: l } - | SetupFinally { handler: l } - | SetupExcept { handler: l } - | SetupWith { end: l } - | SetupAsyncWith { end: l } - | Break { target: l } - | Continue { target: l } => Some(*l), - _ => None, - } - } - - /// Whether this is an unconditional branching - /// - /// # Examples - /// - /// ``` - /// use rustpython_compiler_core::bytecode::{Arg, Instruction}; - /// let jump_inst = Instruction::Jump { target: Arg::marker() }; - /// assert!(jump_inst.unconditional_branch()) - /// ``` - pub fn unconditional_branch(&self) -> bool { - matches!( - self, - Jump { .. } | Continue { .. } | Break { .. } | ReturnValue | Raise { .. } - ) - } - - /// What effect this instruction has on the stack - /// - /// # Examples - /// - /// ``` - /// use rustpython_compiler_core::bytecode::{Arg, Instruction, Label, UnaryOperator}; - /// let (target, jump_arg) = Arg::new(Label(0xF)); - /// let jump_instruction = Instruction::Jump { target }; - /// let (op, invert_arg) = Arg::new(UnaryOperator::Invert); - /// let invert_instruction = Instruction::UnaryOperation { op }; - /// assert_eq!(jump_instruction.stack_effect(jump_arg, true), 0); - /// assert_eq!(invert_instruction.stack_effect(invert_arg, false), 0); - /// ``` - /// - pub fn stack_effect(&self, arg: OpArg, jump: bool) -> i32 { - match self { - ImportName { .. } | ImportNameless => -1, - ImportStar => -1, - ImportFrom { .. } => 1, - LoadFast(_) | LoadNameAny(_) | LoadGlobal(_) | LoadDeref(_) | LoadClassDeref(_) => 1, - StoreFast(_) | StoreLocal(_) | StoreGlobal(_) | StoreDeref(_) => -1, - DeleteFast(_) | DeleteLocal(_) | DeleteGlobal(_) | DeleteDeref(_) => 0, - LoadClosure(_) => 1, - Subscript => -1, - StoreSubscript => -3, - DeleteSubscript => -2, - LoadAttr { .. } => 0, - StoreAttr { .. } => -2, - DeleteAttr { .. } => -1, - LoadConst { .. } => 1, - UnaryOperation { .. } => 0, - BinaryOperation { .. } - | BinaryOperationInplace { .. } - | TestOperation { .. } - | CompareOperation { .. } => -1, - Pop => -1, - Rotate2 | Rotate3 => 0, - Duplicate => 1, - Duplicate2 => 2, - GetIter => 0, - Continue { .. } => 0, - Break { .. } => 0, - Jump { .. } => 0, - JumpIfTrue { .. } | JumpIfFalse { .. } => -1, - JumpIfTrueOrPop { .. } | JumpIfFalseOrPop { .. } => { - if jump { - 0 - } else { - -1 - } - } - MakeFunction(flags) => { - let flags = flags.get(arg); - -2 - flags.contains(MakeFunctionFlags::CLOSURE) as i32 - - flags.contains(MakeFunctionFlags::ANNOTATIONS) as i32 - - flags.contains(MakeFunctionFlags::KW_ONLY_DEFAULTS) as i32 - - flags.contains(MakeFunctionFlags::DEFAULTS) as i32 - + 1 - } - CallFunctionPositional { nargs } => -(nargs.get(arg) as i32) - 1 + 1, - CallMethodPositional { nargs } => -(nargs.get(arg) as i32) - 3 + 1, - CallFunctionKeyword { nargs } => -1 - (nargs.get(arg) as i32) - 1 + 1, - CallMethodKeyword { nargs } => -1 - (nargs.get(arg) as i32) - 3 + 1, - CallFunctionEx { has_kwargs } => -1 - (has_kwargs.get(arg) as i32) - 1 + 1, - CallMethodEx { has_kwargs } => -1 - (has_kwargs.get(arg) as i32) - 3 + 1, - LoadMethod { .. } => -1 + 3, - ForIter { .. } => { - if jump { - -1 - } else { - 1 - } - } - ReturnValue => -1, - YieldValue => 0, - YieldFrom => -1, - SetupAnnotation | SetupLoop | SetupFinally { .. } | EnterFinally | EndFinally => 0, - SetupExcept { .. } => jump as i32, - SetupWith { .. } => (!jump) as i32, - WithCleanupStart => 0, - WithCleanupFinish => -1, - PopBlock => 0, - Raise { kind } => -(kind.get(arg) as u8 as i32), - BuildString { size } - | BuildTuple { size, .. } - | BuildTupleUnpack { size, .. } - | BuildList { size, .. } - | BuildListUnpack { size, .. } - | BuildSet { size, .. } - | BuildSetUnpack { size, .. } => -(size.get(arg) as i32) + 1, - BuildMap { size } => { - let nargs = size.get(arg) * 2; - -(nargs as i32) + 1 - } - BuildMapForCall { size } => { - let nargs = size.get(arg); - -(nargs as i32) + 1 - } - DictUpdate => -1, - BuildSlice { step } => -2 - (step.get(arg) as i32) + 1, - ListAppend { .. } | SetAdd { .. } => -1, - MapAdd { .. } => -2, - PrintExpr => -1, - LoadBuildClass => 1, - UnpackSequence { size } => -1 + size.get(arg) as i32, - UnpackEx { args } => { - let UnpackExArgs { before, after } = args.get(arg); - -1 + before as i32 + 1 + after as i32 - } - FormatValue { .. } => -1, - PopException => 0, - Reverse { .. } => 0, - GetAwaitable => 0, - BeforeAsyncWith => 1, - SetupAsyncWith { .. } => { - if jump { - -1 - } else { - 0 - } - } - GetAIter => 0, - GetANext => 1, - EndAsyncFor => -2, - ExtendedArg => 0, - } - } - - pub fn display<'a>( - &'a self, - arg: OpArg, - ctx: &'a impl InstrDisplayContext, - ) -> impl fmt::Display + 'a { - struct FmtFn<F>(F); - impl<F: Fn(&mut fmt::Formatter) -> fmt::Result> fmt::Display for FmtFn<F> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - (self.0)(f) - } - } - FmtFn(move |f: &mut fmt::Formatter| self.fmt_dis(arg, f, ctx, false, 0, 0)) - } - - #[allow(clippy::too_many_arguments)] - fn fmt_dis( - &self, - arg: OpArg, - f: &mut fmt::Formatter, - ctx: &impl InstrDisplayContext, - expand_code_objects: bool, - pad: usize, - level: usize, - ) -> fmt::Result { - macro_rules! w { - ($variant:ident) => { - write!(f, stringify!($variant)) - }; - ($variant:ident, $map:ident = $arg_marker:expr) => {{ - let arg = $arg_marker.get(arg); - write!(f, "{:pad$}({}, {})", stringify!($variant), arg, $map(arg)) - }}; - ($variant:ident, $arg_marker:expr) => { - write!(f, "{:pad$}({})", stringify!($variant), $arg_marker.get(arg)) - }; - ($variant:ident, ?$arg_marker:expr) => { - write!( - f, - "{:pad$}({:?})", - stringify!($variant), - $arg_marker.get(arg) - ) - }; - } - - let varname = |i: u32| ctx.get_varname(i as usize); - let name = |i: u32| ctx.get_name(i as usize); - let cell_name = |i: u32| ctx.get_cell_name(i as usize); - - match self { - ImportName { idx } => w!(ImportName, name = idx), - ImportNameless => w!(ImportNameless), - ImportStar => w!(ImportStar), - ImportFrom { idx } => w!(ImportFrom, name = idx), - LoadFast(idx) => w!(LoadFast, varname = idx), - LoadNameAny(idx) => w!(LoadNameAny, name = idx), - LoadGlobal(idx) => w!(LoadGlobal, name = idx), - LoadDeref(idx) => w!(LoadDeref, cell_name = idx), - LoadClassDeref(idx) => w!(LoadClassDeref, cell_name = idx), - StoreFast(idx) => w!(StoreFast, varname = idx), - StoreLocal(idx) => w!(StoreLocal, name = idx), - StoreGlobal(idx) => w!(StoreGlobal, name = idx), - StoreDeref(idx) => w!(StoreDeref, cell_name = idx), - DeleteFast(idx) => w!(DeleteFast, varname = idx), - DeleteLocal(idx) => w!(DeleteLocal, name = idx), - DeleteGlobal(idx) => w!(DeleteGlobal, name = idx), - DeleteDeref(idx) => w!(DeleteDeref, cell_name = idx), - LoadClosure(i) => w!(LoadClosure, cell_name = i), - Subscript => w!(Subscript), - StoreSubscript => w!(StoreSubscript), - DeleteSubscript => w!(DeleteSubscript), - StoreAttr { idx } => w!(StoreAttr, name = idx), - DeleteAttr { idx } => w!(DeleteAttr, name = idx), - LoadConst { idx } => { - let value = ctx.get_constant(idx.get(arg) as usize); - match value.borrow_constant() { - BorrowedConstant::Code { code } if expand_code_objects => { - write!(f, "{:pad$}({:?}):", "LoadConst", code)?; - code.display_inner(f, true, level + 1)?; - Ok(()) - } - c => { - write!(f, "{:pad$}(", "LoadConst")?; - c.fmt_display(f)?; - write!(f, ")") - } - } - } - UnaryOperation { op } => w!(UnaryOperation, ?op), - BinaryOperation { op } => w!(BinaryOperation, ?op), - BinaryOperationInplace { op } => w!(BinaryOperationInplace, ?op), - LoadAttr { idx } => w!(LoadAttr, name = idx), - TestOperation { op } => w!(TestOperation, ?op), - CompareOperation { op } => w!(CompareOperation, ?op), - Pop => w!(Pop), - Rotate2 => w!(Rotate2), - Rotate3 => w!(Rotate3), - Duplicate => w!(Duplicate), - Duplicate2 => w!(Duplicate2), - GetIter => w!(GetIter), - Continue { target } => w!(Continue, target), - Break { target } => w!(Break, target), - Jump { target } => w!(Jump, target), - JumpIfTrue { target } => w!(JumpIfTrue, target), - JumpIfFalse { target } => w!(JumpIfFalse, target), - JumpIfTrueOrPop { target } => w!(JumpIfTrueOrPop, target), - JumpIfFalseOrPop { target } => w!(JumpIfFalseOrPop, target), - MakeFunction(flags) => w!(MakeFunction, ?flags), - CallFunctionPositional { nargs } => w!(CallFunctionPositional, nargs), - CallFunctionKeyword { nargs } => w!(CallFunctionKeyword, nargs), - CallFunctionEx { has_kwargs } => w!(CallFunctionEx, has_kwargs), - LoadMethod { idx } => w!(LoadMethod, name = idx), - CallMethodPositional { nargs } => w!(CallMethodPositional, nargs), - CallMethodKeyword { nargs } => w!(CallMethodKeyword, nargs), - CallMethodEx { has_kwargs } => w!(CallMethodEx, has_kwargs), - ForIter { target } => w!(ForIter, target), - ReturnValue => w!(ReturnValue), - YieldValue => w!(YieldValue), - YieldFrom => w!(YieldFrom), - SetupAnnotation => w!(SetupAnnotation), - SetupLoop => w!(SetupLoop), - SetupExcept { handler } => w!(SetupExcept, handler), - SetupFinally { handler } => w!(SetupFinally, handler), - EnterFinally => w!(EnterFinally), - EndFinally => w!(EndFinally), - SetupWith { end } => w!(SetupWith, end), - WithCleanupStart => w!(WithCleanupStart), - WithCleanupFinish => w!(WithCleanupFinish), - BeforeAsyncWith => w!(BeforeAsyncWith), - SetupAsyncWith { end } => w!(SetupAsyncWith, end), - PopBlock => w!(PopBlock), - Raise { kind } => w!(Raise, ?kind), - BuildString { size } => w!(BuildString, size), - BuildTuple { size } => w!(BuildTuple, size), - BuildTupleUnpack { size } => w!(BuildTupleUnpack, size), - BuildList { size } => w!(BuildList, size), - BuildListUnpack { size } => w!(BuildListUnpack, size), - BuildSet { size } => w!(BuildSet, size), - BuildSetUnpack { size } => w!(BuildSetUnpack, size), - BuildMap { size } => w!(BuildMap, size), - BuildMapForCall { size } => w!(BuildMap, size), - DictUpdate => w!(DictUpdate), - BuildSlice { step } => w!(BuildSlice, step), - ListAppend { i } => w!(ListAppend, i), - SetAdd { i } => w!(SetAdd, i), - MapAdd { i } => w!(MapAdd, i), - PrintExpr => w!(PrintExpr), - LoadBuildClass => w!(LoadBuildClass), - UnpackSequence { size } => w!(UnpackSequence, size), - UnpackEx { args } => w!(UnpackEx, args), - FormatValue { conversion } => w!(FormatValue, ?conversion), - PopException => w!(PopException), - Reverse { amount } => w!(Reverse, amount), - GetAwaitable => w!(GetAwaitable), - GetAIter => w!(GetAIter), - GetANext => w!(GetANext), - EndAsyncFor => w!(EndAsyncFor), - ExtendedArg => w!(ExtendedArg, Arg::<u32>::marker()), - } - } -} - -pub trait InstrDisplayContext { - type Constant: Constant; - fn get_constant(&self, i: usize) -> &Self::Constant; - fn get_name(&self, i: usize) -> &str; - fn get_varname(&self, i: usize) -> &str; - fn get_cell_name(&self, i: usize) -> &str; -} - -impl<C: Constant> InstrDisplayContext for CodeObject<C> { - type Constant = C; - fn get_constant(&self, i: usize) -> &C { - &self.constants[i] - } - fn get_name(&self, i: usize) -> &str { - self.names[i].as_ref() - } - fn get_varname(&self, i: usize) -> &str { - self.varnames[i].as_ref() - } - fn get_cell_name(&self, i: usize) -> &str { - self.cellvars - .get(i) - .unwrap_or_else(|| &self.freevars[i - self.cellvars.len()]) - .as_ref() - } -} - -impl fmt::Display for ConstantData { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.borrow_constant().fmt_display(f) - } -} - -impl<C: Constant> fmt::Debug for CodeObject<C> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "<code object {} at ??? file {:?}, line {}>", - self.obj_name.as_ref(), - self.source_path.as_ref(), - self.first_line_number.map_or(-1, |x| x.get() as i32) - ) - } -} diff --git a/compiler/core/src/lib.rs b/compiler/core/src/lib.rs deleted file mode 100644 index 75fde66684e..00000000000 --- a/compiler/core/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] -#![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")] - -pub mod bytecode; -pub mod frozen; -pub mod marshal; -mod mode; - -pub use mode::Mode; diff --git a/compiler/core/src/marshal.rs b/compiler/core/src/marshal.rs deleted file mode 100644 index 9f3bb8e22e5..00000000000 --- a/compiler/core/src/marshal.rs +++ /dev/null @@ -1,631 +0,0 @@ -use crate::bytecode::*; -use malachite_bigint::{BigInt, Sign}; -use num_complex::Complex64; -use rustpython_parser_core::source_code::{OneIndexed, SourceLocation}; -use std::convert::Infallible; - -pub const FORMAT_VERSION: u32 = 4; - -#[derive(Debug)] -pub enum MarshalError { - /// Unexpected End Of File - Eof, - /// Invalid Bytecode - InvalidBytecode, - /// Invalid utf8 in string - InvalidUtf8, - /// Invalid source location - InvalidLocation, - /// Bad type marker - BadType, -} - -impl std::fmt::Display for MarshalError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Eof => f.write_str("unexpected end of data"), - Self::InvalidBytecode => f.write_str("invalid bytecode"), - Self::InvalidUtf8 => f.write_str("invalid utf8"), - Self::InvalidLocation => f.write_str("invalid source location"), - Self::BadType => f.write_str("bad type marker"), - } - } -} - -impl From<std::str::Utf8Error> for MarshalError { - fn from(_: std::str::Utf8Error) -> Self { - Self::InvalidUtf8 - } -} - -impl std::error::Error for MarshalError {} - -type Result<T, E = MarshalError> = std::result::Result<T, E>; - -#[repr(u8)] -enum Type { - // Null = b'0', - None = b'N', - False = b'F', - True = b'T', - StopIter = b'S', - Ellipsis = b'.', - Int = b'i', - Float = b'g', - Complex = b'y', - // Long = b'l', // i32 - Bytes = b's', // = TYPE_STRING - // Interned = b't', - // Ref = b'r', - Tuple = b'(', - List = b'[', - Dict = b'{', - Code = b'c', - Unicode = b'u', - // Unknown = b'?', - Set = b'<', - FrozenSet = b'>', - Ascii = b'a', - // AsciiInterned = b'A', - // SmallTuple = b')', - // ShortAscii = b'z', - // ShortAsciiInterned = b'Z', -} -// const FLAG_REF: u8 = b'\x80'; - -impl TryFrom<u8> for Type { - type Error = MarshalError; - fn try_from(value: u8) -> Result<Self> { - use Type::*; - Ok(match value { - // b'0' => Null, - b'N' => None, - b'F' => False, - b'T' => True, - b'S' => StopIter, - b'.' => Ellipsis, - b'i' => Int, - b'g' => Float, - b'y' => Complex, - // b'l' => Long, - b's' => Bytes, - // b't' => Interned, - // b'r' => Ref, - b'(' => Tuple, - b'[' => List, - b'{' => Dict, - b'c' => Code, - b'u' => Unicode, - // b'?' => Unknown, - b'<' => Set, - b'>' => FrozenSet, - b'a' => Ascii, - // b'A' => AsciiInterned, - // b')' => SmallTuple, - // b'z' => ShortAscii, - // b'Z' => ShortAsciiInterned, - _ => return Err(MarshalError::BadType), - }) - } -} - -pub trait Read { - fn read_slice(&mut self, n: u32) -> Result<&[u8]>; - fn read_array<const N: usize>(&mut self) -> Result<&[u8; N]> { - self.read_slice(N as u32).map(|s| s.try_into().unwrap()) - } - fn read_str(&mut self, len: u32) -> Result<&str> { - Ok(std::str::from_utf8(self.read_slice(len)?)?) - } - fn read_u8(&mut self) -> Result<u8> { - Ok(u8::from_le_bytes(*self.read_array()?)) - } - fn read_u16(&mut self) -> Result<u16> { - Ok(u16::from_le_bytes(*self.read_array()?)) - } - fn read_u32(&mut self) -> Result<u32> { - Ok(u32::from_le_bytes(*self.read_array()?)) - } - fn read_u64(&mut self) -> Result<u64> { - Ok(u64::from_le_bytes(*self.read_array()?)) - } -} - -pub(crate) trait ReadBorrowed<'a>: Read { - fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]>; - fn read_str_borrow(&mut self, len: u32) -> Result<&'a str> { - Ok(std::str::from_utf8(self.read_slice_borrow(len)?)?) - } -} - -impl Read for &[u8] { - fn read_slice(&mut self, n: u32) -> Result<&[u8]> { - self.read_slice_borrow(n) - } -} - -impl<'a> ReadBorrowed<'a> for &'a [u8] { - fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]> { - let data = self.get(..n as usize).ok_or(MarshalError::Eof)?; - *self = &self[n as usize..]; - Ok(data) - } -} - -pub struct Cursor<B> { - pub data: B, - pub position: usize, -} - -impl<B: AsRef<[u8]>> Read for Cursor<B> { - fn read_slice(&mut self, n: u32) -> Result<&[u8]> { - let data = &self.data.as_ref()[self.position..]; - let slice = data.get(..n as usize).ok_or(MarshalError::Eof)?; - self.position += n as usize; - Ok(slice) - } -} - -pub fn deserialize_code<R: Read, Bag: ConstantBag>( - rdr: &mut R, - bag: Bag, -) -> Result<CodeObject<Bag::Constant>> { - let len = rdr.read_u32()?; - let instructions = rdr.read_slice(len * 2)?; - let instructions = instructions - .chunks_exact(2) - .map(|cu| { - let op = Instruction::try_from(cu[0])?; - let arg = OpArgByte(cu[1]); - Ok(CodeUnit { op, arg }) - }) - .collect::<Result<Box<[CodeUnit]>>>()?; - - let len = rdr.read_u32()?; - let locations = (0..len) - .map(|_| { - Ok(SourceLocation { - row: OneIndexed::new(rdr.read_u32()?).ok_or(MarshalError::InvalidLocation)?, - column: OneIndexed::from_zero_indexed(rdr.read_u32()?), - }) - }) - .collect::<Result<Box<[SourceLocation]>>>()?; - - let flags = CodeFlags::from_bits_truncate(rdr.read_u16()?); - - let posonlyarg_count = rdr.read_u32()?; - let arg_count = rdr.read_u32()?; - let kwonlyarg_count = rdr.read_u32()?; - - let len = rdr.read_u32()?; - let source_path = bag.make_name(rdr.read_str(len)?); - - let first_line_number = OneIndexed::new(rdr.read_u32()?); - let max_stackdepth = rdr.read_u32()?; - - let len = rdr.read_u32()?; - let obj_name = bag.make_name(rdr.read_str(len)?); - - let len = rdr.read_u32()?; - let cell2arg = (len != 0) - .then(|| { - (0..len) - .map(|_| Ok(rdr.read_u32()? as i32)) - .collect::<Result<Box<[i32]>>>() - }) - .transpose()?; - - let len = rdr.read_u32()?; - let constants = (0..len) - .map(|_| deserialize_value(rdr, bag)) - .collect::<Result<Box<[_]>>>()?; - - let mut read_names = || { - let len = rdr.read_u32()?; - (0..len) - .map(|_| { - let len = rdr.read_u32()?; - Ok(bag.make_name(rdr.read_str(len)?)) - }) - .collect::<Result<Box<[_]>>>() - }; - - let names = read_names()?; - let varnames = read_names()?; - let cellvars = read_names()?; - let freevars = read_names()?; - - Ok(CodeObject { - instructions, - locations, - flags, - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path, - first_line_number, - max_stackdepth, - obj_name, - cell2arg, - constants, - names, - varnames, - cellvars, - freevars, - }) -} - -pub trait MarshalBag: Copy { - type Value; - fn make_bool(&self, value: bool) -> Self::Value; - fn make_none(&self) -> Self::Value; - fn make_ellipsis(&self) -> Self::Value; - fn make_float(&self, value: f64) -> Self::Value; - fn make_complex(&self, value: Complex64) -> Self::Value; - fn make_str(&self, value: &str) -> Self::Value; - fn make_bytes(&self, value: &[u8]) -> Self::Value; - fn make_int(&self, value: BigInt) -> Self::Value; - fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value; - fn make_code( - &self, - code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>, - ) -> Self::Value; - fn make_stop_iter(&self) -> Result<Self::Value>; - fn make_list(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>; - fn make_set(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>; - fn make_frozenset(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>; - fn make_dict( - &self, - it: impl Iterator<Item = (Self::Value, Self::Value)>, - ) -> Result<Self::Value>; - type ConstantBag: ConstantBag; - fn constant_bag(self) -> Self::ConstantBag; -} - -impl<Bag: ConstantBag> MarshalBag for Bag { - type Value = Bag::Constant; - fn make_bool(&self, value: bool) -> Self::Value { - self.make_constant::<Bag::Constant>(BorrowedConstant::Boolean { value }) - } - fn make_none(&self) -> Self::Value { - self.make_constant::<Bag::Constant>(BorrowedConstant::None) - } - fn make_ellipsis(&self) -> Self::Value { - self.make_constant::<Bag::Constant>(BorrowedConstant::Ellipsis) - } - fn make_float(&self, value: f64) -> Self::Value { - self.make_constant::<Bag::Constant>(BorrowedConstant::Float { value }) - } - fn make_complex(&self, value: Complex64) -> Self::Value { - self.make_constant::<Bag::Constant>(BorrowedConstant::Complex { value }) - } - fn make_str(&self, value: &str) -> Self::Value { - self.make_constant::<Bag::Constant>(BorrowedConstant::Str { value }) - } - fn make_bytes(&self, value: &[u8]) -> Self::Value { - self.make_constant::<Bag::Constant>(BorrowedConstant::Bytes { value }) - } - fn make_int(&self, value: BigInt) -> Self::Value { - self.make_int(value) - } - fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value { - self.make_tuple(elements) - } - fn make_code( - &self, - code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>, - ) -> Self::Value { - self.make_code(code) - } - fn make_stop_iter(&self) -> Result<Self::Value> { - Err(MarshalError::BadType) - } - fn make_list(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> { - Err(MarshalError::BadType) - } - fn make_set(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> { - Err(MarshalError::BadType) - } - fn make_frozenset(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> { - Err(MarshalError::BadType) - } - fn make_dict( - &self, - _: impl Iterator<Item = (Self::Value, Self::Value)>, - ) -> Result<Self::Value> { - Err(MarshalError::BadType) - } - type ConstantBag = Self; - fn constant_bag(self) -> Self::ConstantBag { - self - } -} - -pub fn deserialize_value<R: Read, Bag: MarshalBag>(rdr: &mut R, bag: Bag) -> Result<Bag::Value> { - let typ = Type::try_from(rdr.read_u8()?)?; - let value = match typ { - Type::True => bag.make_bool(true), - Type::False => bag.make_bool(false), - Type::None => bag.make_none(), - Type::StopIter => bag.make_stop_iter()?, - Type::Ellipsis => bag.make_ellipsis(), - Type::Int => { - let len = rdr.read_u32()? as i32; - let sign = if len < 0 { Sign::Minus } else { Sign::Plus }; - let bytes = rdr.read_slice(len.unsigned_abs())?; - let int = BigInt::from_bytes_le(sign, bytes); - bag.make_int(int) - } - Type::Float => { - let value = f64::from_bits(rdr.read_u64()?); - bag.make_float(value) - } - Type::Complex => { - let re = f64::from_bits(rdr.read_u64()?); - let im = f64::from_bits(rdr.read_u64()?); - let value = Complex64 { re, im }; - bag.make_complex(value) - } - Type::Ascii | Type::Unicode => { - let len = rdr.read_u32()?; - let value = rdr.read_str(len)?; - bag.make_str(value) - } - Type::Tuple => { - let len = rdr.read_u32()?; - let it = (0..len).map(|_| deserialize_value(rdr, bag)); - itertools::process_results(it, |it| bag.make_tuple(it))? - } - Type::List => { - let len = rdr.read_u32()?; - let it = (0..len).map(|_| deserialize_value(rdr, bag)); - itertools::process_results(it, |it| bag.make_list(it))?? - } - Type::Set => { - let len = rdr.read_u32()?; - let it = (0..len).map(|_| deserialize_value(rdr, bag)); - itertools::process_results(it, |it| bag.make_set(it))?? - } - Type::FrozenSet => { - let len = rdr.read_u32()?; - let it = (0..len).map(|_| deserialize_value(rdr, bag)); - itertools::process_results(it, |it| bag.make_frozenset(it))?? - } - Type::Dict => { - let len = rdr.read_u32()?; - let it = (0..len).map(|_| { - let k = deserialize_value(rdr, bag)?; - let v = deserialize_value(rdr, bag)?; - Ok::<_, MarshalError>((k, v)) - }); - itertools::process_results(it, |it| bag.make_dict(it))?? - } - Type::Bytes => { - // Following CPython, after marshaling, byte arrays are converted into bytes. - let len = rdr.read_u32()?; - let value = rdr.read_slice(len)?; - bag.make_bytes(value) - } - Type::Code => bag.make_code(deserialize_code(rdr, bag.constant_bag())?), - }; - Ok(value) -} - -pub trait Dumpable: Sized { - type Error; - type Constant: Constant; - fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error>; -} - -pub enum DumpableValue<'a, D: Dumpable> { - Integer(&'a BigInt), - Float(f64), - Complex(Complex64), - Boolean(bool), - Str(&'a str), - Bytes(&'a [u8]), - Code(&'a CodeObject<D::Constant>), - Tuple(&'a [D]), - None, - Ellipsis, - StopIter, - List(&'a [D]), - Set(&'a [D]), - Frozenset(&'a [D]), - Dict(&'a [(D, D)]), -} - -impl<'a, C: Constant> From<BorrowedConstant<'a, C>> for DumpableValue<'a, C> { - fn from(c: BorrowedConstant<'a, C>) -> Self { - match c { - BorrowedConstant::Integer { value } => Self::Integer(value), - BorrowedConstant::Float { value } => Self::Float(value), - BorrowedConstant::Complex { value } => Self::Complex(value), - BorrowedConstant::Boolean { value } => Self::Boolean(value), - BorrowedConstant::Str { value } => Self::Str(value), - BorrowedConstant::Bytes { value } => Self::Bytes(value), - BorrowedConstant::Code { code } => Self::Code(code), - BorrowedConstant::Tuple { elements } => Self::Tuple(elements), - BorrowedConstant::None => Self::None, - BorrowedConstant::Ellipsis => Self::Ellipsis, - } - } -} - -impl<C: Constant> Dumpable for C { - type Error = Infallible; - type Constant = Self; - #[inline(always)] - fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error> { - Ok(f(self.borrow_constant().into())) - } -} - -pub trait Write { - fn write_slice(&mut self, slice: &[u8]); - fn write_u8(&mut self, v: u8) { - self.write_slice(&v.to_le_bytes()) - } - fn write_u16(&mut self, v: u16) { - self.write_slice(&v.to_le_bytes()) - } - fn write_u32(&mut self, v: u32) { - self.write_slice(&v.to_le_bytes()) - } - fn write_u64(&mut self, v: u64) { - self.write_slice(&v.to_le_bytes()) - } -} - -impl Write for Vec<u8> { - fn write_slice(&mut self, slice: &[u8]) { - self.extend_from_slice(slice) - } -} - -pub(crate) fn write_len<W: Write>(buf: &mut W, len: usize) { - let Ok(len) = len.try_into() else { - panic!("too long to serialize") - }; - buf.write_u32(len); -} - -pub(crate) fn write_vec<W: Write>(buf: &mut W, slice: &[u8]) { - write_len(buf, slice.len()); - buf.write_slice(slice); -} - -pub fn serialize_value<W: Write, D: Dumpable>( - buf: &mut W, - constant: DumpableValue<'_, D>, -) -> Result<(), D::Error> { - match constant { - DumpableValue::Integer(int) => { - buf.write_u8(Type::Int as u8); - let (sign, bytes) = int.to_bytes_le(); - let len: i32 = bytes.len().try_into().expect("too long to serialize"); - let len = if sign == Sign::Minus { -len } else { len }; - buf.write_u32(len as u32); - buf.write_slice(&bytes); - } - DumpableValue::Float(f) => { - buf.write_u8(Type::Float as u8); - buf.write_u64(f.to_bits()); - } - DumpableValue::Complex(c) => { - buf.write_u8(Type::Complex as u8); - buf.write_u64(c.re.to_bits()); - buf.write_u64(c.im.to_bits()); - } - DumpableValue::Boolean(b) => { - buf.write_u8(if b { Type::True } else { Type::False } as u8); - } - DumpableValue::Str(s) => { - buf.write_u8(Type::Unicode as u8); - write_vec(buf, s.as_bytes()); - } - DumpableValue::Bytes(b) => { - buf.write_u8(Type::Bytes as u8); - write_vec(buf, b); - } - DumpableValue::Code(c) => { - buf.write_u8(Type::Code as u8); - serialize_code(buf, c); - } - DumpableValue::Tuple(tup) => { - buf.write_u8(Type::Tuple as u8); - write_len(buf, tup.len()); - for val in tup { - val.with_dump(|val| serialize_value(buf, val))?? - } - } - DumpableValue::None => { - buf.write_u8(Type::None as u8); - } - DumpableValue::Ellipsis => { - buf.write_u8(Type::Ellipsis as u8); - } - DumpableValue::StopIter => { - buf.write_u8(Type::StopIter as u8); - } - DumpableValue::List(l) => { - buf.write_u8(Type::List as u8); - write_len(buf, l.len()); - for val in l { - val.with_dump(|val| serialize_value(buf, val))?? - } - } - DumpableValue::Set(set) => { - buf.write_u8(Type::Set as u8); - write_len(buf, set.len()); - for val in set { - val.with_dump(|val| serialize_value(buf, val))?? - } - } - DumpableValue::Frozenset(set) => { - buf.write_u8(Type::FrozenSet as u8); - write_len(buf, set.len()); - for val in set { - val.with_dump(|val| serialize_value(buf, val))?? - } - } - DumpableValue::Dict(d) => { - buf.write_u8(Type::Dict as u8); - write_len(buf, d.len()); - for (k, v) in d { - k.with_dump(|val| serialize_value(buf, val))??; - v.with_dump(|val| serialize_value(buf, val))??; - } - } - } - Ok(()) -} - -pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) { - write_len(buf, code.instructions.len()); - // SAFETY: it's ok to transmute CodeUnit to [u8; 2] - let (_, instructions_bytes, _) = unsafe { code.instructions.align_to() }; - buf.write_slice(instructions_bytes); - - write_len(buf, code.locations.len()); - for loc in &*code.locations { - buf.write_u32(loc.row.get()); - buf.write_u32(loc.column.to_zero_indexed()); - } - - buf.write_u16(code.flags.bits()); - - buf.write_u32(code.posonlyarg_count); - buf.write_u32(code.arg_count); - buf.write_u32(code.kwonlyarg_count); - - write_vec(buf, code.source_path.as_ref().as_bytes()); - - buf.write_u32(code.first_line_number.map_or(0, |x| x.get())); - buf.write_u32(code.max_stackdepth); - - write_vec(buf, code.obj_name.as_ref().as_bytes()); - - let cell2arg = code.cell2arg.as_deref().unwrap_or(&[]); - write_len(buf, cell2arg.len()); - for &i in cell2arg { - buf.write_u32(i as u32) - } - - write_len(buf, code.constants.len()); - for constant in &*code.constants { - serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {}) - } - - let mut write_names = |names: &[C::Name]| { - write_len(buf, names.len()); - for name in names { - write_vec(buf, name.as_ref().as_bytes()); - } - }; - - write_names(&code.names); - write_names(&code.varnames); - write_names(&code.cellvars); - write_names(&code.freevars); -} diff --git a/compiler/core/src/mode.rs b/compiler/core/src/mode.rs deleted file mode 100644 index 6682540c0df..00000000000 --- a/compiler/core/src/mode.rs +++ /dev/null @@ -1,33 +0,0 @@ -pub use rustpython_parser_core::mode::ModeParseError; - -#[derive(Clone, Copy)] -pub enum Mode { - Exec, - Eval, - Single, - BlockExpr, -} - -impl std::str::FromStr for Mode { - type Err = ModeParseError; - - // To support `builtins.compile()` `mode` argument - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "exec" => Ok(Mode::Exec), - "eval" => Ok(Mode::Eval), - "single" => Ok(Mode::Single), - _ => Err(ModeParseError), - } - } -} - -impl From<Mode> for rustpython_parser_core::Mode { - fn from(mode: Mode) -> Self { - match mode { - Mode::Exec => Self::Module, - Mode::Eval => Self::Expression, - Mode::Single | Mode::BlockExpr => Self::Interactive, - } - } -} diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs deleted file mode 100644 index c2810585555..00000000000 --- a/compiler/src/lib.rs +++ /dev/null @@ -1,90 +0,0 @@ -use rustpython_codegen::{compile, symboltable}; -use rustpython_parser::ast::{self as ast, fold::Fold, ConstantOptimizer}; - -pub use rustpython_codegen::compile::CompileOpts; -pub use rustpython_compiler_core::{bytecode::CodeObject, Mode}; -pub use rustpython_parser::{source_code::LinearLocator, Parse}; - -// these modules are out of repository. re-exporting them here for convenience. -pub use rustpython_codegen as codegen; -pub use rustpython_compiler_core as core; -pub use rustpython_parser as parser; - -#[derive(Debug)] -pub enum CompileErrorType { - Codegen(rustpython_codegen::error::CodegenErrorType), - Parse(parser::ParseErrorType), -} - -impl std::error::Error for CompileErrorType { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - CompileErrorType::Codegen(e) => e.source(), - CompileErrorType::Parse(e) => e.source(), - } - } -} -impl std::fmt::Display for CompileErrorType { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - CompileErrorType::Codegen(e) => e.fmt(f), - CompileErrorType::Parse(e) => e.fmt(f), - } - } -} -impl From<rustpython_codegen::error::CodegenErrorType> for CompileErrorType { - fn from(source: rustpython_codegen::error::CodegenErrorType) -> Self { - CompileErrorType::Codegen(source) - } -} -impl From<parser::ParseErrorType> for CompileErrorType { - fn from(source: parser::ParseErrorType) -> Self { - CompileErrorType::Parse(source) - } -} - -pub type CompileError = rustpython_parser::source_code::LocatedError<CompileErrorType>; - -/// Compile a given source code into a bytecode object. -pub fn compile( - source: &str, - mode: Mode, - source_path: String, - opts: CompileOpts, -) -> Result<CodeObject, CompileError> { - let mut locator = LinearLocator::new(source); - let mut ast = match parser::parse(source, mode.into(), &source_path) { - Ok(x) => x, - Err(e) => return Err(locator.locate_error(e)), - }; - if opts.optimize > 0 { - ast = ConstantOptimizer::new() - .fold_mod(ast) - .unwrap_or_else(|e| match e {}); - } - let ast = locator.fold_mod(ast).unwrap_or_else(|e| match e {}); - compile::compile_top(&ast, source_path, mode, opts).map_err(|e| e.into()) -} - -pub fn compile_symtable( - source: &str, - mode: Mode, - source_path: &str, -) -> Result<symboltable::SymbolTable, CompileError> { - let mut locator = LinearLocator::new(source); - let res = match mode { - Mode::Exec | Mode::Single | Mode::BlockExpr => { - let ast = - ast::Suite::parse(source, source_path).map_err(|e| locator.locate_error(e))?; - let ast = locator.fold(ast).unwrap(); - symboltable::SymbolTable::scan_program(&ast) - } - Mode::Eval => { - let expr = - ast::Expr::parse(source, source_path).map_err(|e| locator.locate_error(e))?; - let expr = locator.fold(expr).unwrap(); - symboltable::SymbolTable::scan_expr(&expr) - } - }; - res.map_err(|e| e.into_codegen_error(source_path.to_owned()).into()) -} diff --git a/crates/codegen/Cargo.toml b/crates/codegen/Cargo.toml new file mode 100644 index 00000000000..78065962fff --- /dev/null +++ b/crates/codegen/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "rustpython-codegen" +description = "Compiler for python code into bytecode for the rustpython VM." +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[features] +default = ["std"] +std = ["thiserror/std", "itertools/use_std"] + +[dependencies] +rustpython-compiler-core = { workspace = true } +rustpython-literal = {workspace = true } +rustpython-wtf8 = { workspace = true } +ruff_python_ast = { workspace = true } +ruff_text_size = { workspace = true } + +ahash = { workspace = true } +bitflags = { workspace = true } +indexmap = { workspace = true } +itertools = { workspace = true } +log = { workspace = true } +num-complex = { workspace = true } +num-traits = { workspace = true } +thiserror = { workspace = true } +malachite-bigint = { workspace = true } +memchr = { workspace = true } +unicode_names2 = { workspace = true } + +[dev-dependencies] +ruff_python_parser = { workspace = true } +insta = { workspace = true } + +[lints] +workspace = true diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs new file mode 100644 index 00000000000..f7f5b944a0f --- /dev/null +++ b/crates/codegen/src/compile.rs @@ -0,0 +1,9322 @@ +//! +//! Take an AST and transform it into bytecode +//! +//! Inspirational code: +//! <https://github.com/python/cpython/blob/main/Python/compile.c> +//! <https://github.com/micropython/micropython/blob/master/py/compile.c> + +// spell-checker:ignore starunpack subscripter + +#![deny(clippy::cast_possible_truncation)] + +use crate::{ + IndexMap, IndexSet, ToPythonName, + error::{CodegenError, CodegenErrorType, InternalError, PatternUnreachableReason}, + ir::{self, BlockIdx}, + symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable}, + unparse::UnparseExpr, +}; +use alloc::borrow::Cow; +use itertools::Itertools; +use malachite_bigint::BigInt; +use num_complex::Complex; +use num_traits::{Num, ToPrimitive}; +use ruff_python_ast as ast; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustpython_compiler_core::{ + Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, + bytecode::{ + self, AnyInstruction, Arg as OpArgMarker, BinaryOperator, BuildSliceArgCount, CodeObject, + ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, IntrinsicFunction1, + Invert, LoadAttr, LoadSuperAttr, OpArg, OpArgType, PseudoInstruction, SpecialMethod, + UnpackExArgs, oparg, + }, +}; +use rustpython_wtf8::Wtf8Buf; + +/// Extension trait for `ast::Expr` to add constant checking methods +trait ExprExt { + /// Check if an expression is a constant literal + fn is_constant(&self) -> bool; + + /// Check if a slice expression has all constant elements + fn is_constant_slice(&self) -> bool; + + /// Check if we should use BINARY_SLICE/STORE_SLICE optimization + fn should_use_slice_optimization(&self) -> bool; +} + +impl ExprExt for ast::Expr { + fn is_constant(&self) -> bool { + matches!( + self, + ast::Expr::NumberLiteral(_) + | ast::Expr::StringLiteral(_) + | ast::Expr::BytesLiteral(_) + | ast::Expr::NoneLiteral(_) + | ast::Expr::BooleanLiteral(_) + | ast::Expr::EllipsisLiteral(_) + ) + } + + fn is_constant_slice(&self) -> bool { + match self { + ast::Expr::Slice(s) => { + let lower_const = + s.lower.is_none() || s.lower.as_deref().is_some_and(|e| e.is_constant()); + let upper_const = + s.upper.is_none() || s.upper.as_deref().is_some_and(|e| e.is_constant()); + let step_const = + s.step.is_none() || s.step.as_deref().is_some_and(|e| e.is_constant()); + lower_const && upper_const && step_const + } + _ => false, + } + } + + fn should_use_slice_optimization(&self) -> bool { + !self.is_constant_slice() && matches!(self, ast::Expr::Slice(s) if s.step.is_none()) + } +} + +const MAXBLOCKS: usize = 20; + +#[derive(Debug, Clone, Copy)] +pub enum FBlockType { + WhileLoop, + ForLoop, + TryExcept, + FinallyTry, + FinallyEnd, + With, + AsyncWith, + HandlerCleanup, + PopValue, + ExceptionHandler, + ExceptionGroupHandler, + AsyncComprehensionGenerator, + StopIteration, +} + +/// Stores additional data for fblock unwinding +// fb_datum +#[derive(Debug, Clone)] +pub enum FBlockDatum { + None, + /// For FinallyTry: stores the finally body statements to compile during unwind + FinallyBody(Vec<ast::Stmt>), + /// For HandlerCleanup: stores the exception variable name (e.g., "e" in "except X as e") + ExceptionName(String), +} + +/// Type of super() call optimization detected by can_optimize_super_call() +#[derive(Debug, Clone)] +enum SuperCallType<'a> { + /// super(class, self) - explicit 2-argument form + TwoArg { + class_arg: &'a ast::Expr, + self_arg: &'a ast::Expr, + }, + /// super() - implicit 0-argument form (uses __class__ cell) + ZeroArg, +} + +#[derive(Debug, Clone)] +pub struct FBlockInfo { + pub fb_type: FBlockType, + pub fb_block: BlockIdx, + pub fb_exit: BlockIdx, + // additional data for fblock unwinding + pub fb_datum: FBlockDatum, +} + +pub(crate) type InternalResult<T> = Result<T, InternalError>; +type CompileResult<T> = Result<T, CodegenError>; + +#[derive(PartialEq, Eq, Clone, Copy)] +enum NameUsage { + Load, + Store, + Delete, +} +/// Main structure holding the state of compilation. +struct Compiler { + code_stack: Vec<ir::CodeInfo>, + symbol_table_stack: Vec<SymbolTable>, + source_file: SourceFile, + // current_source_location: SourceLocation, + current_source_range: TextRange, + done_with_future_stmts: DoneWithFuture, + future_annotations: bool, + ctx: CompileContext, + opts: CompileOpts, + in_annotation: bool, + /// True when compiling in "single" (interactive) mode. + /// Expression statements at module scope emit CALL_INTRINSIC_1(Print). + interactive: bool, +} + +#[derive(Clone, Copy)] +enum DoneWithFuture { + No, + DoneWithDoc, + Yes, +} + +#[derive(Clone, Copy, Debug)] +pub struct CompileOpts { + /// How optimized the bytecode output should be; any optimize > 0 does + /// not emit assert statements + pub optimize: u8, + /// Include column info in bytecode (-X no_debug_ranges disables) + pub debug_ranges: bool, +} + +impl Default for CompileOpts { + fn default() -> Self { + Self { + optimize: 0, + debug_ranges: true, + } + } +} + +#[derive(Debug, Clone, Copy)] +struct CompileContext { + loop_data: Option<(BlockIdx, BlockIdx)>, + in_class: bool, + func: FunctionContext, + /// True if we're anywhere inside an async function (even inside nested comprehensions) + in_async_scope: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +enum FunctionContext { + NoFunction, + Function, + AsyncFunction, +} + +impl CompileContext { + fn in_func(self) -> bool { + self.func != FunctionContext::NoFunction + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +enum ComprehensionType { + Generator, + List, + Set, + Dict, +} + +fn validate_duplicate_params(params: &ast::Parameters) -> Result<(), CodegenErrorType> { + let mut seen_params = IndexSet::default(); + for param in params { + let param_name = param.name().as_str(); + if !seen_params.insert(param_name) { + return Err(CodegenErrorType::SyntaxError(format!( + r#"Duplicate parameter "{param_name}""# + ))); + } + } + + Ok(()) +} + +/// Compile an Mod produced from ruff parser +pub fn compile_top( + ast: ruff_python_ast::Mod, + source_file: SourceFile, + mode: Mode, + opts: CompileOpts, +) -> CompileResult<CodeObject> { + match ast { + ruff_python_ast::Mod::Module(module) => match mode { + Mode::Exec | Mode::Eval => compile_program(&module, source_file, opts), + Mode::Single => compile_program_single(&module, source_file, opts), + Mode::BlockExpr => compile_block_expression(&module, source_file, opts), + }, + ruff_python_ast::Mod::Expression(expr) => compile_expression(&expr, source_file, opts), + } +} + +/// Compile a standard Python program to bytecode +pub fn compile_program( + ast: &ast::ModModule, + source_file: SourceFile, + opts: CompileOpts, +) -> CompileResult<CodeObject> { + let symbol_table = SymbolTable::scan_program(ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned()))?; + let mut compiler = Compiler::new(opts, source_file, "<module>".to_owned()); + compiler.compile_program(ast, symbol_table)?; + let code = compiler.exit_scope(); + trace!("Compilation completed: {code:?}"); + Ok(code) +} + +/// Compile a Python program to bytecode for the context of a REPL +pub fn compile_program_single( + ast: &ast::ModModule, + source_file: SourceFile, + opts: CompileOpts, +) -> CompileResult<CodeObject> { + let symbol_table = SymbolTable::scan_program(ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned()))?; + let mut compiler = Compiler::new(opts, source_file, "<module>".to_owned()); + compiler.compile_program_single(&ast.body, symbol_table)?; + let code = compiler.exit_scope(); + trace!("Compilation completed: {code:?}"); + Ok(code) +} + +pub fn compile_block_expression( + ast: &ast::ModModule, + source_file: SourceFile, + opts: CompileOpts, +) -> CompileResult<CodeObject> { + let symbol_table = SymbolTable::scan_program(ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned()))?; + let mut compiler = Compiler::new(opts, source_file, "<module>".to_owned()); + compiler.compile_block_expr(&ast.body, symbol_table)?; + let code = compiler.exit_scope(); + trace!("Compilation completed: {code:?}"); + Ok(code) +} + +pub fn compile_expression( + ast: &ast::ModExpression, + source_file: SourceFile, + opts: CompileOpts, +) -> CompileResult<CodeObject> { + let symbol_table = SymbolTable::scan_expr(ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned()))?; + let mut compiler = Compiler::new(opts, source_file, "<module>".to_owned()); + compiler.compile_eval(ast, symbol_table)?; + let code = compiler.exit_scope(); + Ok(code) +} + +macro_rules! emit { + // Struct variant with single identifier (e.g., Foo::A { arg }) + ($c:expr, $enum:ident :: $op:ident { $arg:ident $(,)? } $(,)?) => { + $c.emit_arg($arg, |x| $enum::$op { $arg: x }) + }; + + // Struct variant with explicit value (e.g., Foo::A { arg: 42 }) + ($c:expr, $enum:ident :: $op:ident { $arg:ident : $arg_val:expr $(,)? } $(,)?) => { + $c.emit_arg($arg_val, |x| $enum::$op { $arg: x }) + }; + + // Tuple variant (e.g., Foo::B(42)). Should never be reached, here for validation. + ($c:expr, $enum:ident :: $op:ident($arg_val:expr $(,)? ) $(,)?) => { + panic!("No instruction should be defined as `Instruction::Foo(value)` use `Instruction::Foo { x: value }` instead") + }; + + // No-arg variant (e.g., Foo::C) + ($c:expr, $enum:ident :: $op:ident $(,)?) => { + $c.emit_no_arg($enum::$op) + }; +} + +fn eprint_location(zelf: &Compiler) { + let start = zelf + .source_file + .to_source_code() + .source_location(zelf.current_source_range.start(), PositionEncoding::Utf8); + let end = zelf + .source_file + .to_source_code() + .source_location(zelf.current_source_range.end(), PositionEncoding::Utf8); + eprintln!( + "LOCATION: {} from {}:{} to {}:{}", + zelf.source_file.name(), + start.line, + start.character_offset, + end.line, + end.character_offset + ); +} + +/// Better traceback for internal error +#[track_caller] +fn unwrap_internal<T>(zelf: &Compiler, r: InternalResult<T>) -> T { + if let Err(ref r_err) = r { + eprintln!("=== CODEGEN PANIC INFO ==="); + eprintln!("This IS an internal error: {r_err}"); + eprint_location(zelf); + eprintln!("=== END PANIC INFO ==="); + } + r.unwrap() +} + +fn compiler_unwrap_option<T>(zelf: &Compiler, o: Option<T>) -> T { + if o.is_none() { + eprintln!("=== CODEGEN PANIC INFO ==="); + eprintln!("This IS an internal error, an option was unwrapped during codegen"); + eprint_location(zelf); + eprintln!("=== END PANIC INFO ==="); + } + o.unwrap() +} + +// fn compiler_result_unwrap<T, E: core::fmt::Debug>(zelf: &Compiler, result: Result<T, E>) -> T { +// if result.is_err() { +// eprintln!("=== CODEGEN PANIC INFO ==="); +// eprintln!("This IS an internal error, an result was unwrapped during codegen"); +// eprint_location(zelf); +// eprintln!("=== END PANIC INFO ==="); +// } +// result.unwrap() +// } + +/// The pattern context holds information about captured names and jump targets. +#[derive(Clone)] +pub struct PatternContext { + /// A list of names captured by the pattern. + pub stores: Vec<String>, + /// If false, then any name captures against our subject will raise. + pub allow_irrefutable: bool, + /// A list of jump target labels used on pattern failure. + pub fail_pop: Vec<BlockIdx>, + /// The number of items on top of the stack that should remain. + pub on_top: usize, +} + +impl Default for PatternContext { + fn default() -> Self { + Self::new() + } +} + +impl PatternContext { + pub const fn new() -> Self { + Self { + stores: Vec::new(), + allow_irrefutable: false, + fail_pop: Vec::new(), + on_top: 0, + } + } + + pub fn fail_pop_size(&self) -> usize { + self.fail_pop.len() + } +} + +enum JumpOp { + Jump, + PopJumpIfFalse, +} + +/// Type of collection to build in starunpack_helper +#[derive(Debug, Clone, Copy, PartialEq)] +enum CollectionType { + Tuple, + List, + Set, +} + +impl Compiler { + fn new(opts: CompileOpts, source_file: SourceFile, code_name: String) -> Self { + let module_code = ir::CodeInfo { + flags: bytecode::CodeFlags::NEWLOCALS, + source_path: source_file.name().to_owned(), + private: None, + blocks: vec![ir::Block::default()], + current_block: BlockIdx::new(0), + metadata: ir::CodeUnitMetadata { + name: code_name.clone(), + qualname: Some(code_name), + consts: IndexSet::default(), + names: IndexSet::default(), + varnames: IndexSet::default(), + cellvars: IndexSet::default(), + freevars: IndexSet::default(), + fast_hidden: IndexMap::default(), + argcount: 0, + posonlyargcount: 0, + kwonlyargcount: 0, + firstlineno: OneIndexed::MIN, + }, + static_attributes: None, + in_inlined_comp: false, + fblock: Vec::with_capacity(MAXBLOCKS), + symbol_table_index: 0, // Module is always the first symbol table + in_conditional_block: 0, + next_conditional_annotation_index: 0, + }; + Self { + code_stack: vec![module_code], + symbol_table_stack: Vec::new(), + source_file, + // current_source_location: SourceLocation::default(), + current_source_range: TextRange::default(), + done_with_future_stmts: DoneWithFuture::No, + future_annotations: false, + ctx: CompileContext { + loop_data: None, + in_class: false, + func: FunctionContext::NoFunction, + in_async_scope: false, + }, + opts, + in_annotation: false, + interactive: false, + } + } + + /// Compile just start and stop of a slice (for BINARY_SLICE/STORE_SLICE) + // = codegen_slice_two_parts + fn compile_slice_two_parts(&mut self, s: &ast::ExprSlice) -> CompileResult<()> { + // Compile lower (or None) + if let Some(lower) = &s.lower { + self.compile_expression(lower)?; + } else { + self.emit_load_const(ConstantData::None); + } + + // Compile upper (or None) + if let Some(upper) = &s.upper { + self.compile_expression(upper)?; + } else { + self.emit_load_const(ConstantData::None); + } + + Ok(()) + } + /// Compile a subscript expression + // = compiler_subscript + fn compile_subscript( + &mut self, + value: &ast::Expr, + slice: &ast::Expr, + ctx: ast::ExprContext, + ) -> CompileResult<()> { + // Save full subscript expression range (set by compile_expression before this call) + let subscript_range = self.current_source_range; + + // VISIT(c, expr, e->v.Subscript.value) + self.compile_expression(value)?; + + // Handle two-element non-constant slice with BINARY_SLICE/STORE_SLICE + let use_slice_opt = matches!(ctx, ast::ExprContext::Load | ast::ExprContext::Store) + && slice.should_use_slice_optimization(); + if use_slice_opt { + match slice { + ast::Expr::Slice(s) => self.compile_slice_two_parts(s)?, + _ => unreachable!( + "should_use_slice_optimization should only return true for ast::Expr::Slice" + ), + }; + } else { + // VISIT(c, expr, e->v.Subscript.slice) + self.compile_expression(slice)?; + } + + // Restore full subscript expression range before emitting + self.set_source_range(subscript_range); + + match (use_slice_opt, ctx) { + (true, ast::ExprContext::Load) => emit!(self, Instruction::BinarySlice), + (true, ast::ExprContext::Store) => emit!(self, Instruction::StoreSlice), + (true, _) => unreachable!(), + (false, ast::ExprContext::Load) => emit!( + self, + Instruction::BinaryOp { + op: BinaryOperator::Subscr + } + ), + (false, ast::ExprContext::Store) => emit!(self, Instruction::StoreSubscr), + (false, ast::ExprContext::Del) => emit!(self, Instruction::DeleteSubscr), + (false, ast::ExprContext::Invalid) => { + return Err(self.error(CodegenErrorType::SyntaxError( + "Invalid expression context".to_owned(), + ))); + } + } + + Ok(()) + } + + /// Helper function for compiling tuples/lists/sets with starred expressions + /// + /// ast::Parameters: + /// - elts: The elements to compile + /// - pushed: Number of items already on the stack + /// - collection_type: What type of collection to build (tuple, list, set) + /// + // = starunpack_helper in compile.c + fn starunpack_helper( + &mut self, + elts: &[ast::Expr], + pushed: u32, + collection_type: CollectionType, + ) -> CompileResult<()> { + let n = elts.len().to_u32(); + let seen_star = elts.iter().any(|e| matches!(e, ast::Expr::Starred(_))); + + // Determine collection size threshold for optimization + let big = match collection_type { + CollectionType::Set => n > 8, + _ => n > 4, + }; + + // If no stars and not too big, compile all elements and build once + if !seen_star && !big { + for elt in elts { + self.compile_expression(elt)?; + } + let total_size = n + pushed; + match collection_type { + CollectionType::List => { + emit!(self, Instruction::BuildList { count: total_size }); + } + CollectionType::Set => { + emit!(self, Instruction::BuildSet { count: total_size }); + } + CollectionType::Tuple => { + emit!(self, Instruction::BuildTuple { count: total_size }); + } + } + return Ok(()); + } + + // Has stars or too big: use streaming approach + let mut sequence_built = false; + let mut i = 0u32; + + for elt in elts.iter() { + if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = elt { + // When we hit first star, build sequence with elements so far + if !sequence_built { + match collection_type { + CollectionType::List => { + emit!(self, Instruction::BuildList { count: i + pushed }); + } + CollectionType::Set => { + emit!(self, Instruction::BuildSet { count: i + pushed }); + } + CollectionType::Tuple => { + emit!(self, Instruction::BuildList { count: i + pushed }); + } + } + sequence_built = true; + } + + // Compile the starred expression and extend + self.compile_expression(value)?; + match collection_type { + CollectionType::List => { + emit!(self, Instruction::ListExtend { i: 0 }); + } + CollectionType::Set => { + emit!(self, Instruction::SetUpdate { i: 0 }); + } + CollectionType::Tuple => { + emit!(self, Instruction::ListExtend { i: 0 }); + } + } + } else { + // Non-starred element + self.compile_expression(elt)?; + + if sequence_built { + // Sequence already exists, append to it + match collection_type { + CollectionType::List => { + emit!(self, Instruction::ListAppend { i: 0 }); + } + CollectionType::Set => { + emit!(self, Instruction::SetAdd { i: 0 }); + } + CollectionType::Tuple => { + emit!(self, Instruction::ListAppend { i: 0 }); + } + } + } else { + // Still collecting elements before first star + i += 1; + } + } + } + + // If we never built sequence (all non-starred), build it now + if !sequence_built { + match collection_type { + CollectionType::List => { + emit!(self, Instruction::BuildList { count: i + pushed }); + } + CollectionType::Set => { + emit!(self, Instruction::BuildSet { count: i + pushed }); + } + CollectionType::Tuple => { + emit!(self, Instruction::BuildTuple { count: i + pushed }); + } + } + } else if collection_type == CollectionType::Tuple { + // For tuples, convert the list to tuple + emit!( + self, + Instruction::CallIntrinsic1 { + func: IntrinsicFunction1::ListToTuple + } + ); + } + + Ok(()) + } + + fn error(&mut self, error: CodegenErrorType) -> CodegenError { + self.error_ranged(error, self.current_source_range) + } + + fn error_ranged(&mut self, error: CodegenErrorType, range: TextRange) -> CodegenError { + let location = self + .source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8); + CodegenError { + error, + location: Some(location), + source_path: self.source_file.name().to_owned(), + } + } + + /// Get the SymbolTable for the current scope. + fn current_symbol_table(&self) -> &SymbolTable { + self.symbol_table_stack + .last() + .expect("symbol_table_stack is empty! This is a compiler bug.") + } + + /// Get the index of a free variable. + fn get_free_var_index(&mut self, name: &str) -> CompileResult<u32> { + let info = self.code_stack.last_mut().unwrap(); + let idx = info + .metadata + .freevars + .get_index_of(name) + .unwrap_or_else(|| info.metadata.freevars.insert_full(name.to_owned()).0); + Ok((idx + info.metadata.cellvars.len()).to_u32()) + } + + /// Get the index of a cell variable. + fn get_cell_var_index(&mut self, name: &str) -> CompileResult<u32> { + let info = self.code_stack.last_mut().unwrap(); + let idx = info + .metadata + .cellvars + .get_index_of(name) + .unwrap_or_else(|| info.metadata.cellvars.insert_full(name.to_owned()).0); + Ok(idx.to_u32()) + } + + /// Get the index of a local variable. + fn get_local_var_index(&mut self, name: &str) -> CompileResult<oparg::VarNum> { + let info = self.code_stack.last_mut().unwrap(); + let idx = info + .metadata + .varnames + .get_index_of(name) + .unwrap_or_else(|| info.metadata.varnames.insert_full(name.to_owned()).0); + Ok(idx.to_u32().into()) + } + + /// Get the index of a global name. + fn get_global_name_index(&mut self, name: &str) -> u32 { + let info = self.code_stack.last_mut().unwrap(); + let idx = info + .metadata + .names + .get_index_of(name) + .unwrap_or_else(|| info.metadata.names.insert_full(name.to_owned()).0); + idx.to_u32() + } + + /// Push the next symbol table on to the stack + fn push_symbol_table(&mut self) -> CompileResult<&SymbolTable> { + // Look up the next table contained in the scope of the current table + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + if current_table.next_sub_table >= current_table.sub_tables.len() { + let name = current_table.name.clone(); + let typ = current_table.typ; + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "no symbol table available in {} (type: {:?})", + name, typ + )))); + } + + let idx = current_table.next_sub_table; + current_table.next_sub_table += 1; + let table = current_table.sub_tables[idx].clone(); + + // Push the next table onto the stack + self.symbol_table_stack.push(table); + Ok(self.current_symbol_table()) + } + + /// Push the annotation symbol table from the next sub_table's annotation_block + /// The annotation_block is stored in the function's scope, which is the next sub_table + /// Returns true if annotation_block exists, false otherwise + fn push_annotation_symbol_table(&mut self) -> bool { + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // The annotation_block is in the next sub_table (function scope) + let next_idx = current_table.next_sub_table; + if next_idx >= current_table.sub_tables.len() { + return false; + } + + let next_table = &mut current_table.sub_tables[next_idx]; + if let Some(annotation_block) = next_table.annotation_block.take() { + self.symbol_table_stack.push(*annotation_block); + true + } else { + false + } + } + + /// Push the annotation symbol table for module/class level annotations + /// This takes annotation_block from the current symbol table (not sub_tables) + fn push_current_annotation_symbol_table(&mut self) -> bool { + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // For modules/classes, annotation_block is directly in the current table + if let Some(annotation_block) = current_table.annotation_block.take() { + self.symbol_table_stack.push(*annotation_block); + true + } else { + false + } + } + + /// Pop the annotation symbol table and restore it to the function scope's annotation_block + fn pop_annotation_symbol_table(&mut self) { + let annotation_table = self.symbol_table_stack.pop().expect("compiler bug"); + let current_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table"); + + // Restore to the next sub_table (function scope) where it came from + let next_idx = current_table.next_sub_table; + if next_idx < current_table.sub_tables.len() { + current_table.sub_tables[next_idx].annotation_block = Some(Box::new(annotation_table)); + } + } + + /// Pop the current symbol table off the stack + fn pop_symbol_table(&mut self) -> SymbolTable { + self.symbol_table_stack.pop().expect("compiler bug") + } + + /// Check if a super() call can be optimized + /// Returns Some(SuperCallType) if optimization is possible, None otherwise + fn can_optimize_super_call<'a>( + &self, + value: &'a ast::Expr, + attr: &str, + ) -> Option<SuperCallType<'a>> { + // 1. value must be a Call expression + let ast::Expr::Call(ast::ExprCall { + func, arguments, .. + }) = value + else { + return None; + }; + + // 2. func must be Name("super") + let ast::Expr::Name(ast::ExprName { id, .. }) = func.as_ref() else { + return None; + }; + if id.as_str() != "super" { + return None; + } + + // 3. attr must not be "__class__" + if attr == "__class__" { + return None; + } + + // 4. No keyword arguments + if !arguments.keywords.is_empty() { + return None; + } + + // 5. Must be inside a function (not at module level or class body) + if !self.ctx.in_func() { + return None; + } + + // 6. "super" must be GlobalImplicit (not redefined locally or at module level) + let table = self.current_symbol_table(); + if let Some(symbol) = table.lookup("super") + && symbol.scope != SymbolScope::GlobalImplicit + { + return None; + } + // Also check top-level scope to detect module-level shadowing. + // Only block if super is actually *bound* at module level (not just used). + if let Some(top_table) = self.symbol_table_stack.first() + && let Some(sym) = top_table.lookup("super") + && sym.scope != SymbolScope::GlobalImplicit + { + return None; + } + + // 7. Check argument pattern + let args = &arguments.args; + + // No starred expressions allowed + if args.iter().any(|arg| matches!(arg, ast::Expr::Starred(_))) { + return None; + } + + match args.len() { + 2 => { + // 2-arg: super(class, self) + Some(SuperCallType::TwoArg { + class_arg: &args[0], + self_arg: &args[1], + }) + } + 0 => { + // 0-arg: super() - need __class__ cell and first parameter + // Enclosing function should have at least one positional argument + let info = self.code_stack.last()?; + if info.metadata.argcount == 0 && info.metadata.posonlyargcount == 0 { + return None; + } + + // Check if __class__ is available as a cell/free variable + // The scope must be Free (from enclosing class) or have FREE_CLASS flag + if let Some(symbol) = table.lookup("__class__") { + if symbol.scope != SymbolScope::Free + && !symbol.flags.contains(SymbolFlags::FREE_CLASS) + { + return None; + } + } else { + // __class__ not in symbol table, optimization not possible + return None; + } + + Some(SuperCallType::ZeroArg) + } + _ => None, // 1 or 3+ args - not optimizable + } + } + + /// Load arguments for super() optimization onto the stack + /// Stack result: [global_super, class, self] + fn load_args_for_super(&mut self, super_type: &SuperCallType<'_>) -> CompileResult<()> { + // 1. Load global super + self.compile_name("super", NameUsage::Load)?; + + match super_type { + SuperCallType::TwoArg { + class_arg, + self_arg, + } => { + // 2-arg: load provided arguments + self.compile_expression(class_arg)?; + self.compile_expression(self_arg)?; + } + SuperCallType::ZeroArg => { + // 0-arg: load __class__ cell and first parameter + // Load __class__ from cell/free variable + let scope = self.get_ref_type("__class__").map_err(|e| self.error(e))?; + let idx = match scope { + SymbolScope::Cell => self.get_cell_var_index("__class__")?, + SymbolScope::Free => self.get_free_var_index("__class__")?, + _ => { + return Err(self.error(CodegenErrorType::SyntaxError( + "super(): __class__ cell not found".to_owned(), + ))); + } + }; + emit!(self, Instruction::LoadDeref { i: idx }); + + // Load first parameter (typically 'self'). + // Safety: can_optimize_super_call() ensures argcount > 0, and + // parameters are always added to varnames first (see symboltable.rs). + let first_param = { + let info = self.code_stack.last().unwrap(); + info.metadata.varnames.first().cloned() + }; + let first_param = first_param.ok_or_else(|| { + self.error(CodegenErrorType::SyntaxError( + "super(): no arguments and no first parameter".to_owned(), + )) + })?; + self.compile_name(&first_param, NameUsage::Load)?; + } + } + Ok(()) + } + + /// Check if this is an inlined comprehension context (PEP 709) + /// Currently disabled - always returns false to avoid stack issues + fn is_inlined_comprehension_context(&self, _comprehension_type: ComprehensionType) -> bool { + // TODO: Implement PEP 709 inlined comprehensions properly + // For now, disabled to avoid stack underflow issues + false + } + + /// Enter a new scope + // = compiler_enter_scope + fn enter_scope( + &mut self, + name: &str, + scope_type: CompilerScope, + key: usize, // In RustPython, we use the index in symbol_table_stack as key + lineno: u32, + ) -> CompileResult<()> { + // Allocate a new compiler unit + + // In Rust, we'll create the structure directly + let source_path = self.source_file.name().to_owned(); + + // Lookup symbol table entry using key (_PySymtable_Lookup) + let ste = match self.symbol_table_stack.get(key) { + Some(v) => v, + None => { + return Err(self.error(CodegenErrorType::SyntaxError( + "unknown symbol table entry".to_owned(), + ))); + } + }; + + // Use varnames from symbol table (already collected in definition order) + let varname_cache: IndexSet<String> = ste.varnames.iter().cloned().collect(); + + // Build cellvars using dictbytype (CELL scope, sorted) + let mut cellvar_cache = IndexSet::default(); + let mut cell_names: Vec<_> = ste + .symbols + .iter() + .filter(|(_, s)| s.scope == SymbolScope::Cell) + .map(|(name, _)| name.clone()) + .collect(); + cell_names.sort(); + for name in cell_names { + cellvar_cache.insert(name); + } + + // Handle implicit __class__ cell if needed + if ste.needs_class_closure { + // Cook up an implicit __class__ cell + debug_assert_eq!(scope_type, CompilerScope::Class); + cellvar_cache.insert("__class__".to_string()); + } + + // Handle implicit __classdict__ cell if needed + if ste.needs_classdict { + // Cook up an implicit __classdict__ cell + debug_assert_eq!(scope_type, CompilerScope::Class); + cellvar_cache.insert("__classdict__".to_string()); + } + + // Handle implicit __conditional_annotations__ cell if needed + // Only for class scope - module scope uses NAME operations, not DEREF + if ste.has_conditional_annotations && scope_type == CompilerScope::Class { + cellvar_cache.insert("__conditional_annotations__".to_string()); + } + + // Build freevars using dictbytype (FREE scope, offset by cellvars size) + let mut freevar_cache = IndexSet::default(); + let mut free_names: Vec<_> = ste + .symbols + .iter() + .filter(|(_, s)| { + s.scope == SymbolScope::Free || s.flags.contains(SymbolFlags::FREE_CLASS) + }) + .map(|(name, _)| name.clone()) + .collect(); + free_names.sort(); + for name in free_names { + freevar_cache.insert(name); + } + + // Initialize u_metadata fields + let (flags, posonlyarg_count, arg_count, kwonlyarg_count) = match scope_type { + CompilerScope::Module => (bytecode::CodeFlags::empty(), 0, 0, 0), + CompilerScope::Class => (bytecode::CodeFlags::empty(), 0, 0, 0), + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda => ( + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + 0, // Will be set later in enter_function + 0, // Will be set later in enter_function + 0, // Will be set later in enter_function + ), + CompilerScope::Comprehension => ( + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + 0, + 1, // comprehensions take one argument (.0) + 0, + ), + CompilerScope::TypeParams => ( + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + 0, + 0, + 0, + ), + CompilerScope::Annotation => ( + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + 1, // format is positional-only + 1, // annotation scope takes one argument (format) + 0, + ), + }; + + // Get private name from parent scope + let private = if !self.code_stack.is_empty() { + self.code_stack.last().unwrap().private.clone() + } else { + None + }; + + // Create the new compilation unit + let code_info = ir::CodeInfo { + flags, + source_path: source_path.clone(), + private, + blocks: vec![ir::Block::default()], + current_block: BlockIdx::new(0), + metadata: ir::CodeUnitMetadata { + name: name.to_owned(), + qualname: None, // Will be set below + consts: IndexSet::default(), + names: IndexSet::default(), + varnames: varname_cache, + cellvars: cellvar_cache, + freevars: freevar_cache, + fast_hidden: IndexMap::default(), + argcount: arg_count, + posonlyargcount: posonlyarg_count, + kwonlyargcount: kwonlyarg_count, + firstlineno: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), + }, + static_attributes: if scope_type == CompilerScope::Class { + Some(IndexSet::default()) + } else { + None + }, + in_inlined_comp: false, + fblock: Vec::with_capacity(MAXBLOCKS), + symbol_table_index: key, + in_conditional_block: 0, + next_conditional_annotation_index: 0, + }; + + // Push the old compiler unit on the stack (like PyCapsule) + // This happens before setting qualname + self.code_stack.push(code_info); + + // Set qualname after pushing (uses compiler_set_qualname logic) + if scope_type != CompilerScope::Module { + self.set_qualname(); + } + + // Emit RESUME (handles async preamble and module lineno 0) + // CPython: LOCATION(lineno, lineno, 0, 0), then loc.lineno = 0 for module + self.emit_resume_for_scope(scope_type, lineno); + + Ok(()) + } + + /// Emit RESUME instruction with proper handling for async preamble and module lineno. + /// codegen_enter_scope equivalent for RESUME emission. + fn emit_resume_for_scope(&mut self, scope_type: CompilerScope, lineno: u32) { + // For async functions/coroutines, emit RETURN_GENERATOR + POP_TOP before RESUME + if scope_type == CompilerScope::AsyncFunction { + emit!(self, Instruction::ReturnGenerator); + emit!(self, Instruction::PopTop); + } + + // CPython: LOCATION(lineno, lineno, 0, 0) + // Module scope: loc.lineno = 0 (before the first line) + let lineno_override = if scope_type == CompilerScope::Module { + Some(0) + } else { + None + }; + + // Use lineno for location (col = 0 as in CPython) + let location = SourceLocation { + line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), + character_offset: OneIndexed::MIN, // col = 0 + }; + let end_location = location; // end_lineno = lineno, end_col = 0 + let except_handler = None; + + self.current_block().instructions.push(ir::InstructionInfo { + instr: Instruction::Resume { + context: OpArgMarker::marker(), + } + .into(), + arg: OpArg::new(u32::from(bytecode::ResumeType::AtFuncStart)), + target: BlockIdx::NULL, + location, + end_location, + except_handler, + lineno_override, + cache_entries: 0, + }); + } + + fn push_output( + &mut self, + flags: bytecode::CodeFlags, + posonlyarg_count: u32, + arg_count: u32, + kwonlyarg_count: u32, + obj_name: String, + ) -> CompileResult<()> { + // First push the symbol table + let table = self.push_symbol_table()?; + let scope_type = table.typ; + + // The key is the current position in the symbol table stack + let key = self.symbol_table_stack.len() - 1; + + // Get the line number + let lineno = self.get_source_line_number().get(); + + // Call enter_scope which does most of the work + self.enter_scope(&obj_name, scope_type, key, lineno.to_u32())?; + + // Override the values that push_output sets explicitly + // enter_scope sets default values based on scope_type, but push_output + // allows callers to specify exact values + if let Some(info) = self.code_stack.last_mut() { + info.flags = flags; + info.metadata.argcount = arg_count; + info.metadata.posonlyargcount = posonlyarg_count; + info.metadata.kwonlyargcount = kwonlyarg_count; + } + Ok(()) + } + + // compiler_exit_scope + fn exit_scope(&mut self) -> CodeObject { + let _table = self.pop_symbol_table(); + + // Various scopes can have sub_tables: + // - ast::TypeParams scope can have sub_tables (the function body's symbol table) + // - Module scope can have sub_tables (for TypeAlias scopes, nested functions, classes) + // - Function scope can have sub_tables (for nested functions, classes) + // - Class scope can have sub_tables (for nested classes, methods) + + let pop = self.code_stack.pop(); + let stack_top = compiler_unwrap_option(self, pop); + // No parent scope stack to maintain + unwrap_internal(self, stack_top.finalize_code(&self.opts)) + } + + /// Exit annotation scope - similar to exit_scope but restores annotation_block to parent + fn exit_annotation_scope(&mut self, saved_ctx: CompileContext) -> CodeObject { + self.pop_annotation_symbol_table(); + self.ctx = saved_ctx; + + let pop = self.code_stack.pop(); + let stack_top = compiler_unwrap_option(self, pop); + unwrap_internal(self, stack_top.finalize_code(&self.opts)) + } + + /// Enter annotation scope using the symbol table's annotation_block. + /// Returns None if no annotation_block exists. + /// On success, returns the saved CompileContext to pass to exit_annotation_scope. + fn enter_annotation_scope( + &mut self, + _func_name: &str, + ) -> CompileResult<Option<CompileContext>> { + if !self.push_annotation_symbol_table() { + return Ok(None); + } + + // Annotation scopes are never async (even inside async functions) + let saved_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get(); + self.enter_scope( + "__annotate__", + CompilerScope::Annotation, + key, + lineno.to_u32(), + )?; + + // Override arg_count since enter_scope sets it to 1 but we need the varnames + // setup to be correct too + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + // VALUE_WITH_FAKE_GLOBALS = 2 (from annotationlib.Format) + self.emit_format_validation()?; + + Ok(Some(saved_ctx)) + } + + /// Emit format parameter validation for annotation scope + /// if format > VALUE_WITH_FAKE_GLOBALS (2): raise NotImplementedError + fn emit_format_validation(&mut self) -> CompileResult<()> { + // Load format parameter (first local variable, index 0) + emit!( + self, + Instruction::LoadFast { + var_num: oparg::VarNum::from_u32(0) + } + ); + + // Load VALUE_WITH_FAKE_GLOBALS constant (2) + self.emit_load_const(ConstantData::Integer { value: 2.into() }); + + // Compare: format > 2 + emit!( + self, + Instruction::CompareOp { + opname: ComparisonOperator::Greater + } + ); + + // Jump to body if format <= 2 (comparison is false) + let body_block = self.new_block(); + emit!(self, Instruction::PopJumpIfFalse { delta: body_block }); + + // Raise NotImplementedError + emit!( + self, + Instruction::LoadCommonConstant { + idx: bytecode::CommonConstant::NotImplementedError + } + ); + emit!( + self, + Instruction::RaiseVarargs { + argc: bytecode::RaiseKind::Raise + } + ); + + // Body label - continue with annotation evaluation + self.switch_to_block(body_block); + + Ok(()) + } + + /// Push a new fblock + // = compiler_push_fblock + fn push_fblock( + &mut self, + fb_type: FBlockType, + fb_block: BlockIdx, + fb_exit: BlockIdx, + ) -> CompileResult<()> { + self.push_fblock_full(fb_type, fb_block, fb_exit, FBlockDatum::None) + } + + /// Push an fblock with all parameters including fb_datum + fn push_fblock_full( + &mut self, + fb_type: FBlockType, + fb_block: BlockIdx, + fb_exit: BlockIdx, + fb_datum: FBlockDatum, + ) -> CompileResult<()> { + let code = self.current_code_info(); + if code.fblock.len() >= MAXBLOCKS { + return Err(self.error(CodegenErrorType::SyntaxError( + "too many statically nested blocks".to_owned(), + ))); + } + code.fblock.push(FBlockInfo { + fb_type, + fb_block, + fb_exit, + fb_datum, + }); + Ok(()) + } + + /// Pop an fblock + // = compiler_pop_fblock + fn pop_fblock(&mut self, _expected_type: FBlockType) -> FBlockInfo { + let code = self.current_code_info(); + // TODO: Add assertion to check expected type matches + // assert!(matches!(fblock.fb_type, expected_type)); + code.fblock.pop().expect("fblock stack underflow") + } + + /// Unwind a single fblock, emitting cleanup code + /// preserve_tos: if true, preserve the top of stack (e.g., return value) + fn unwind_fblock(&mut self, info: &FBlockInfo, preserve_tos: bool) -> CompileResult<()> { + match info.fb_type { + FBlockType::WhileLoop + | FBlockType::ExceptionHandler + | FBlockType::ExceptionGroupHandler + | FBlockType::AsyncComprehensionGenerator + | FBlockType::StopIteration => { + // No cleanup needed + } + + FBlockType::ForLoop => { + // Pop the iterator + if preserve_tos { + emit!(self, Instruction::Swap { i: 2 }); + } + emit!(self, Instruction::PopIter); + } + + FBlockType::TryExcept => { + emit!(self, PseudoInstruction::PopBlock); + } + + FBlockType::FinallyTry => { + // FinallyTry is now handled specially in unwind_fblock_stack + // to avoid infinite recursion when the finally body contains return/break/continue. + // This branch should not be reached. + unreachable!("FinallyTry should be handled by unwind_fblock_stack"); + } + + FBlockType::FinallyEnd => { + // codegen_unwind_fblock(FINALLY_END) + if preserve_tos { + emit!(self, Instruction::Swap { i: 2 }); + } + emit!(self, Instruction::PopTop); // exc_value + if preserve_tos { + emit!(self, Instruction::Swap { i: 2 }); + } + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::PopExcept); + } + + FBlockType::With | FBlockType::AsyncWith => { + // Stack when entering: [..., __exit__, return_value (if preserve_tos)] + // Need to call __exit__(None, None, None) + + emit!(self, PseudoInstruction::PopBlock); + + // If preserving return value, swap it below __exit__ + if preserve_tos { + emit!(self, Instruction::Swap { i: 2 }); + } + // Stack after swap: [..., return_value, __exit__] or [..., __exit__] + + // Call __exit__(None, None, None) + // Call protocol: [callable, self_or_null, arg1, arg2, arg3] + emit!(self, Instruction::PushNull); + // Stack: [..., __exit__, NULL] + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + // Stack: [..., __exit__, NULL, None, None, None] + emit!(self, Instruction::Call { argc: 3 }); + + // For async with, await the result + if matches!(info.fb_type, FBlockType::AsyncWith) { + emit!(self, Instruction::GetAwaitable { r#where: 2 }); + self.emit_load_const(ConstantData::None); + let _ = self.compile_yield_from_sequence(true)?; + } + + // Pop the __exit__ result + emit!(self, Instruction::PopTop); + } + + FBlockType::HandlerCleanup => { + // codegen_unwind_fblock(HANDLER_CLEANUP) + if let FBlockDatum::ExceptionName(_) = info.fb_datum { + // Named handler: PopBlock for inner SETUP_CLEANUP + emit!(self, PseudoInstruction::PopBlock); + } + if preserve_tos { + emit!(self, Instruction::Swap { i: 2 }); + } + // PopBlock for outer SETUP_CLEANUP (ExceptionHandler) + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::PopExcept); + + // If there's an exception name, clean it up + if let FBlockDatum::ExceptionName(ref name) = info.fb_datum { + self.emit_load_const(ConstantData::None); + self.store_name(name)?; + self.compile_name(name, NameUsage::Delete)?; + } + } + + FBlockType::PopValue => { + if preserve_tos { + emit!(self, Instruction::Swap { i: 2 }); + } + emit!(self, Instruction::PopTop); + } + } + Ok(()) + } + + /// Unwind the fblock stack, emitting cleanup code for each block + /// preserve_tos: if true, preserve the top of stack (e.g., return value) + /// stop_at_loop: if true, stop when encountering a loop (for break/continue) + fn unwind_fblock_stack(&mut self, preserve_tos: bool, stop_at_loop: bool) -> CompileResult<()> { + // Collect the info we need, with indices for FinallyTry blocks + #[derive(Clone)] + enum UnwindInfo { + Normal(FBlockInfo), + FinallyTry { + body: Vec<ruff_python_ast::Stmt>, + fblock_idx: usize, + }, + } + let mut unwind_infos = Vec::new(); + + { + let code = self.current_code_info(); + for i in (0..code.fblock.len()).rev() { + // Check for exception group handler (forbidden) + if matches!(code.fblock[i].fb_type, FBlockType::ExceptionGroupHandler) { + return Err(self.error(CodegenErrorType::BreakContinueReturnInExceptStar)); + } + + // Stop at loop if requested + if stop_at_loop + && matches!( + code.fblock[i].fb_type, + FBlockType::WhileLoop | FBlockType::ForLoop + ) + { + break; + } + + if matches!(code.fblock[i].fb_type, FBlockType::FinallyTry) { + if let FBlockDatum::FinallyBody(ref body) = code.fblock[i].fb_datum { + unwind_infos.push(UnwindInfo::FinallyTry { + body: body.clone(), + fblock_idx: i, + }); + } + } else { + unwind_infos.push(UnwindInfo::Normal(code.fblock[i].clone())); + } + } + } + + // Process each fblock + for info in unwind_infos { + match info { + UnwindInfo::Normal(fblock_info) => { + self.unwind_fblock(&fblock_info, preserve_tos)?; + } + UnwindInfo::FinallyTry { body, fblock_idx } => { + // codegen_unwind_fblock(FINALLY_TRY) + emit!(self, PseudoInstruction::PopBlock); + + // Temporarily remove the FinallyTry fblock so nested return/break/continue + // in the finally body won't see it again + let code = self.current_code_info(); + let saved_fblock = code.fblock.remove(fblock_idx); + + // Push PopValue fblock if preserving tos + if preserve_tos { + self.push_fblock( + FBlockType::PopValue, + saved_fblock.fb_block, + saved_fblock.fb_block, + )?; + } + + self.compile_statements(&body)?; + + if preserve_tos { + self.pop_fblock(FBlockType::PopValue); + } + + // Restore the fblock + let code = self.current_code_info(); + code.fblock.insert(fblock_idx, saved_fblock); + } + } + } + + Ok(()) + } + + // could take impl Into<Cow<str>>, but everything is borrowed from ast structs; we never + // actually have a `String` to pass + fn name(&mut self, name: &str) -> bytecode::NameIdx { + self._name_inner(name, |i| &mut i.metadata.names) + } + + fn varname(&mut self, name: &str) -> CompileResult<oparg::VarNum> { + // Note: __debug__ checks are now handled in symboltable phase + Ok(oparg::VarNum::from_u32( + self._name_inner(name, |i| &mut i.metadata.varnames), + )) + } + + fn _name_inner( + &mut self, + name: &str, + cache: impl FnOnce(&mut ir::CodeInfo) -> &mut IndexSet<String>, + ) -> u32 { + let name = self.mangle(name); + let cache = cache(self.current_code_info()); + cache + .get_index_of(name.as_ref()) + .unwrap_or_else(|| cache.insert_full(name.into_owned()).0) + .to_u32() + } + + /// Set the qualified name for the current code object + // = compiler_set_qualname + fn set_qualname(&mut self) -> String { + let qualname = self.make_qualname(); + self.current_code_info().metadata.qualname = Some(qualname.clone()); + qualname + } + fn make_qualname(&mut self) -> String { + let stack_size = self.code_stack.len(); + assert!(stack_size >= 1); + + let current_obj_name = self.current_code_info().metadata.name.clone(); + + // If we're at the module level (stack_size == 1), qualname is just the name + if stack_size <= 1 { + return current_obj_name; + } + + // Check parent scope + let mut parent_idx = stack_size - 2; + let mut parent = &self.code_stack[parent_idx]; + + // If parent is ast::TypeParams scope, look at grandparent + // Check if parent is a type params scope by name pattern + if parent.metadata.name.starts_with("<generic parameters of ") { + if stack_size == 2 { + // If we're immediately within the module, qualname is just the name + return current_obj_name; + } + // Use grandparent + parent_idx = stack_size - 3; + parent = &self.code_stack[parent_idx]; + } + + // Check if this is a global class/function + let mut force_global = false; + if stack_size > self.symbol_table_stack.len() { + // We might be in a situation where symbol table isn't pushed yet + // In this case, check the parent symbol table + if let Some(parent_table) = self.symbol_table_stack.last() + && let Some(symbol) = parent_table.lookup(&current_obj_name) + && symbol.scope == SymbolScope::GlobalExplicit + { + force_global = true; + } + } else if let Some(_current_table) = self.symbol_table_stack.last() { + // Mangle the name if necessary (for private names in classes) + let mangled_name = self.mangle(&current_obj_name); + + // Look up in parent symbol table to check scope + if self.symbol_table_stack.len() >= 2 { + let parent_table = &self.symbol_table_stack[self.symbol_table_stack.len() - 2]; + if let Some(symbol) = parent_table.lookup(&mangled_name) + && symbol.scope == SymbolScope::GlobalExplicit + { + force_global = true; + } + } + } + + // Build the qualified name + if force_global { + // For global symbols, qualname is just the name + current_obj_name + } else { + // Check parent scope type + let parent_obj_name = &parent.metadata.name; + + // Determine if parent is a function-like scope + let is_function_parent = parent.flags.contains(bytecode::CodeFlags::OPTIMIZED) + && !parent_obj_name.starts_with("<") // Not a special scope like <lambda>, <listcomp>, etc. + && parent_obj_name != "<module>"; // Not the module scope + + if is_function_parent { + // For functions, append .<locals> to parent qualname + // Use parent's qualname if available, otherwise use parent_obj_name + let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); + format!("{parent_qualname}.<locals>.{current_obj_name}") + } else { + // For classes and other scopes, use parent's qualname directly + // Use parent's qualname if available, otherwise use parent_obj_name + let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); + if parent_qualname == "<module>" { + // Module level, just use the name + current_obj_name + } else { + // Concatenate parent qualname with current name + format!("{parent_qualname}.{current_obj_name}") + } + } + } + } + + fn compile_program( + &mut self, + body: &ast::ModModule, + symbol_table: SymbolTable, + ) -> CompileResult<()> { + let size_before = self.code_stack.len(); + // Set future_annotations from symbol table (detected during symbol table scan) + self.future_annotations = symbol_table.future_annotations; + self.symbol_table_stack.push(symbol_table); + + self.emit_resume_for_scope(CompilerScope::Module, 1); + + let (doc, statements) = split_doc(&body.body, &self.opts); + if let Some(value) = doc { + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); + let doc = self.name("__doc__"); + emit!(self, Instruction::StoreGlobal { namei: doc }) + } + + // Handle annotations based on future_annotations flag + if Self::find_ann(statements) { + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Generate __annotate__ function FIRST (before statements) + self.compile_module_annotate(statements)?; + + // PEP 649: Initialize __conditional_annotations__ set after __annotate__ + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { count: 0 }); + self.store_name("__conditional_annotations__")?; + } + } + } + + // Compile all statements + self.compile_statements(statements)?; + + assert_eq!(self.code_stack.len(), size_before); + + // Emit None at end: + self.emit_return_const(ConstantData::None); + Ok(()) + } + + fn compile_program_single( + &mut self, + body: &[ast::Stmt], + symbol_table: SymbolTable, + ) -> CompileResult<()> { + self.interactive = true; + // Set future_annotations from symbol table (detected during symbol table scan) + self.future_annotations = symbol_table.future_annotations; + self.symbol_table_stack.push(symbol_table); + + self.emit_resume_for_scope(CompilerScope::Module, 1); + + // Handle annotations based on future_annotations flag + if Self::find_ann(body) { + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Generate __annotate__ function FIRST (before statements) + self.compile_module_annotate(body)?; + + // PEP 649: Initialize __conditional_annotations__ set after __annotate__ + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { count: 0 }); + self.store_name("__conditional_annotations__")?; + } + } + } + + if let Some((last, body)) = body.split_last() { + for statement in body { + if let ast::Stmt::Expr(ast::StmtExpr { value, .. }) = &statement { + self.compile_expression(value)?; + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::Print + } + ); + + emit!(self, Instruction::PopTop); + } else { + self.compile_statement(statement)?; + } + } + + if let ast::Stmt::Expr(ast::StmtExpr { value, .. }) = &last { + self.compile_expression(value)?; + emit!(self, Instruction::Copy { i: 1 }); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::Print + } + ); + + emit!(self, Instruction::PopTop); + } else { + self.compile_statement(last)?; + self.emit_load_const(ConstantData::None); + } + } else { + self.emit_load_const(ConstantData::None); + }; + + self.emit_return_value(); + Ok(()) + } + + fn compile_block_expr( + &mut self, + body: &[ast::Stmt], + symbol_table: SymbolTable, + ) -> CompileResult<()> { + self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); + + self.compile_statements(body)?; + + if let Some(last_statement) = body.last() { + match last_statement { + ast::Stmt::Expr(_) => { + self.current_block().instructions.pop(); // pop Instruction::PopTop + } + ast::Stmt::FunctionDef(_) | ast::Stmt::ClassDef(_) => { + let pop_instructions = self.current_block().instructions.pop(); + let store_inst = compiler_unwrap_option(self, pop_instructions); // pop Instruction::Store + emit!(self, Instruction::Copy { i: 1 }); + self.current_block().instructions.push(store_inst); + } + _ => self.emit_load_const(ConstantData::None), + } + } + self.emit_return_value(); + + Ok(()) + } + + // Compile statement in eval mode: + fn compile_eval( + &mut self, + expression: &ast::ModExpression, + symbol_table: SymbolTable, + ) -> CompileResult<()> { + self.symbol_table_stack.push(symbol_table); + self.emit_resume_for_scope(CompilerScope::Module, 1); + + self.compile_expression(&expression.body)?; + self.emit_return_value(); + Ok(()) + } + + fn compile_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> { + for statement in statements { + self.compile_statement(statement)? + } + Ok(()) + } + + fn load_name(&mut self, name: &str) -> CompileResult<()> { + self.compile_name(name, NameUsage::Load) + } + + fn store_name(&mut self, name: &str) -> CompileResult<()> { + self.compile_name(name, NameUsage::Store) + } + + fn mangle<'a>(&self, name: &'a str) -> Cow<'a, str> { + // Use private from current code unit for name mangling + let private = self + .code_stack + .last() + .and_then(|info| info.private.as_deref()); + let mangled_names = self.current_symbol_table().mangled_names.as_ref(); + symboltable::maybe_mangle_name(private, mangled_names, name) + } + + // = compiler_nameop + fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> { + enum NameOp { + Fast, + Global, + Deref, + Name, + DictOrGlobals, // PEP 649: can_see_class_scope + } + + let name = self.mangle(name); + + // Special handling for __debug__ + if NameUsage::Load == usage && name == "__debug__" { + self.emit_load_const(ConstantData::Boolean { + value: self.opts.optimize == 0, + }); + return Ok(()); + } + + // Determine the operation type based on symbol scope + let is_function_like = self.ctx.in_func(); + + // Look up the symbol, handling ast::TypeParams and Annotation scopes specially + let (symbol_scope, can_see_class_scope) = { + let current_table = self.current_symbol_table(); + let is_typeparams = current_table.typ == CompilerScope::TypeParams; + let is_annotation = current_table.typ == CompilerScope::Annotation; + let can_see_class = current_table.can_see_class_scope; + + // First try to find in current table + let symbol = current_table.lookup(name.as_ref()); + + // If not found and we're in ast::TypeParams or Annotation scope, try parent scope + let symbol = if symbol.is_none() && (is_typeparams || is_annotation) { + self.symbol_table_stack + .get(self.symbol_table_stack.len() - 2) // Try to get parent index + .expect("Symbol has no parent! This is a compiler bug.") + .lookup(name.as_ref()) + } else { + symbol + }; + + (symbol.map(|s| s.scope), can_see_class) + }; + + // Special handling for class scope implicit cell variables + // These are treated as Cell even if not explicitly marked in symbol table + // __class__ and __classdict__: only LOAD uses Cell (stores go to class namespace) + // __conditional_annotations__: both LOAD and STORE use Cell (it's a mutable set + // that the annotation scope accesses through the closure) + let symbol_scope = { + let current_table = self.current_symbol_table(); + if current_table.typ == CompilerScope::Class + && ((usage == NameUsage::Load + && (name == "__class__" + || name == "__classdict__" + || name == "__conditional_annotations__")) + || (name == "__conditional_annotations__" && usage == NameUsage::Store)) + { + Some(SymbolScope::Cell) + } else { + symbol_scope + } + }; + + // In annotation or type params scope, missing symbols are treated as global implicit + // This allows referencing global names like Union, Optional, etc. that are imported + // at module level but not explicitly bound in the function scope + let actual_scope = match symbol_scope { + Some(scope) => scope, + None => { + let current_table = self.current_symbol_table(); + if matches!( + current_table.typ, + CompilerScope::Annotation | CompilerScope::TypeParams + ) { + SymbolScope::GlobalImplicit + } else { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "the symbol '{name}' must be present in the symbol table" + )))); + } + } + }; + + // Determine operation type based on scope + let op_type = match actual_scope { + SymbolScope::Free => NameOp::Deref, + SymbolScope::Cell => NameOp::Deref, + SymbolScope::Local => { + if is_function_like { + NameOp::Fast + } else { + NameOp::Name + } + } + SymbolScope::GlobalImplicit => { + // PEP 649: In annotation scope with class visibility, use DictOrGlobals + // to check classdict first before globals + if can_see_class_scope { + NameOp::DictOrGlobals + } else if is_function_like { + NameOp::Global + } else { + NameOp::Name + } + } + SymbolScope::GlobalExplicit => NameOp::Global, + SymbolScope::Unknown => NameOp::Name, + }; + + // Generate appropriate instructions based on operation type + match op_type { + NameOp::Deref => { + let i = match actual_scope { + SymbolScope::Free => self.get_free_var_index(&name)?, + SymbolScope::Cell => self.get_cell_var_index(&name)?, + _ => unreachable!("Invalid scope for Deref operation"), + }; + + match usage { + NameUsage::Load => { + // ClassBlock (not inlined comp): LOAD_LOCALS first, then LOAD_FROM_DICT_OR_DEREF + if self.ctx.in_class && !self.ctx.in_func() { + emit!(self, Instruction::LoadLocals); + emit!(self, Instruction::LoadFromDictOrDeref { i }); + // can_see_class_scope: LOAD_DEREF(__classdict__) first + } else if can_see_class_scope { + let classdict_idx = self.get_free_var_index("__classdict__")?; + emit!(self, Instruction::LoadDeref { i: classdict_idx }); + emit!(self, Instruction::LoadFromDictOrDeref { i }); + } else { + emit!(self, Instruction::LoadDeref { i }); + } + } + NameUsage::Store => emit!(self, Instruction::StoreDeref { i }), + NameUsage::Delete => emit!(self, Instruction::DeleteDeref { i }), + }; + } + NameOp::Fast => { + let var_num = self.get_local_var_index(&name)?; + match usage { + NameUsage::Load => emit!(self, Instruction::LoadFast { var_num }), + NameUsage::Store => emit!(self, Instruction::StoreFast { var_num }), + NameUsage::Delete => emit!(self, Instruction::DeleteFast { var_num }), + }; + } + NameOp::Global => { + let namei = self.get_global_name_index(&name); + match usage { + NameUsage::Load => { + self.emit_load_global(namei, false); + return Ok(()); + } + NameUsage::Store => emit!(self, Instruction::StoreGlobal { namei }), + NameUsage::Delete => emit!(self, Instruction::DeleteGlobal { namei }), + }; + } + NameOp::Name => { + let namei = self.get_global_name_index(&name); + match usage { + NameUsage::Load => emit!(self, Instruction::LoadName { namei }), + NameUsage::Store => emit!(self, Instruction::StoreName { namei }), + NameUsage::Delete => emit!(self, Instruction::DeleteName { namei }), + }; + } + NameOp::DictOrGlobals => { + // PEP 649: First check classdict (from __classdict__ freevar), then globals + let idx = self.get_global_name_index(&name); + match usage { + NameUsage::Load => { + // Load __classdict__ first (it's a free variable in annotation scope) + let classdict_idx = self.get_free_var_index("__classdict__")?; + emit!(self, Instruction::LoadDeref { i: classdict_idx }); + emit!(self, Instruction::LoadFromDictOrGlobals { i: idx }); + } + // Store/Delete in annotation scope should use Name ops + NameUsage::Store => { + emit!(self, Instruction::StoreName { namei: idx }); + } + NameUsage::Delete => { + emit!(self, Instruction::DeleteName { namei: idx }); + } + } + } + } + + Ok(()) + } + + fn compile_statement(&mut self, statement: &ast::Stmt) -> CompileResult<()> { + trace!("Compiling {statement:?}"); + let prev_source_range = self.current_source_range; + self.set_source_range(statement.range()); + + match &statement { + // we do this here because `from __future__` still executes that `from` statement at runtime, + // we still need to compile the ImportFrom down below + ast::Stmt::ImportFrom(ast::StmtImportFrom { module, names, .. }) + if module.as_ref().map(|id| id.as_str()) == Some("__future__") => + { + self.compile_future_features(names)? + } + // ignore module-level doc comments + ast::Stmt::Expr(ast::StmtExpr { value, .. }) + if matches!(&**value, ast::Expr::StringLiteral(..)) + && matches!(self.done_with_future_stmts, DoneWithFuture::No) => + { + self.done_with_future_stmts = DoneWithFuture::DoneWithDoc + } + // if we find any other statement, stop accepting future statements + _ => self.done_with_future_stmts = DoneWithFuture::Yes, + } + + match &statement { + ast::Stmt::Import(ast::StmtImport { names, .. }) => { + // import a, b, c as d + for name in names { + let name = &name; + self.emit_load_const(ConstantData::Integer { + value: num_traits::Zero::zero(), + }); + self.emit_load_const(ConstantData::None); + let namei = self.name(&name.name); + emit!(self, Instruction::ImportName { namei }); + if let Some(alias) = &name.asname { + let parts: Vec<&str> = name.name.split('.').skip(1).collect(); + for (i, part) in parts.iter().enumerate() { + let namei = self.name(part); + emit!(self, Instruction::ImportFrom { namei }); + if i < parts.len() - 1 { + emit!(self, Instruction::Swap { i: 2 }); + emit!(self, Instruction::PopTop); + } + } + self.store_name(alias.as_str())?; + if !parts.is_empty() { + emit!(self, Instruction::PopTop); + } + } else { + self.store_name(name.name.split('.').next().unwrap())? + } + } + } + ast::Stmt::ImportFrom(ast::StmtImportFrom { + level, + module, + names, + .. + }) => { + let import_star = names.iter().any(|n| &n.name == "*"); + + let from_list = if import_star { + if self.ctx.in_func() { + return Err(self.error_ranged( + CodegenErrorType::FunctionImportStar, + statement.range(), + )); + } + vec![ConstantData::Str { value: "*".into() }] + } else { + names + .iter() + .map(|n| ConstantData::Str { + value: n.name.as_str().into(), + }) + .collect() + }; + + // from .... import (*fromlist) + self.emit_load_const(ConstantData::Integer { + value: (*level).into(), + }); + self.emit_load_const(ConstantData::Tuple { + elements: from_list, + }); + + let module_name = module.as_ref().map_or("", |s| s.as_str()); + let module_idx = self.name(module_name); + emit!(self, Instruction::ImportName { namei: module_idx }); + + if import_star { + // from .... import * + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::ImportStar + } + ); + emit!(self, Instruction::PopTop); + } else { + // from mod import a, b as c + + for name in names { + let name = &name; + let idx = self.name(name.name.as_str()); + // import symbol from module: + emit!(self, Instruction::ImportFrom { namei: idx }); + + // Store module under proper name: + if let Some(alias) = &name.asname { + self.store_name(alias.as_str())? + } else { + self.store_name(name.name.as_str())? + } + } + + // Pop module from stack: + emit!(self, Instruction::PopTop); + } + } + ast::Stmt::Expr(ast::StmtExpr { value, .. }) => { + self.compile_expression(value)?; + + if self.interactive && !self.ctx.in_func() && !self.ctx.in_class { + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::Print + } + ); + } + + emit!(self, Instruction::PopTop); + } + ast::Stmt::Global(_) | ast::Stmt::Nonlocal(_) => { + // Handled during symbol table construction. + } + ast::Stmt::If(ast::StmtIf { + test, + body, + elif_else_clauses, + .. + }) => { + self.enter_conditional_block(); + match elif_else_clauses.as_slice() { + // Only if + [] => { + let after_block = self.new_block(); + self.compile_jump_if(test, false, after_block)?; + self.compile_statements(body)?; + self.switch_to_block(after_block); + } + // If, elif*, elif/else + [rest @ .., tail] => { + let after_block = self.new_block(); + let mut next_block = self.new_block(); + + self.compile_jump_if(test, false, next_block)?; + self.compile_statements(body)?; + emit!(self, PseudoInstruction::Jump { delta: after_block }); + + for clause in rest { + self.switch_to_block(next_block); + next_block = self.new_block(); + if let Some(test) = &clause.test { + self.compile_jump_if(test, false, next_block)?; + } else { + unreachable!() // must be elif + } + self.compile_statements(&clause.body)?; + emit!(self, PseudoInstruction::Jump { delta: after_block }); + } + + self.switch_to_block(next_block); + if let Some(test) = &tail.test { + self.compile_jump_if(test, false, after_block)?; + } + self.compile_statements(&tail.body)?; + self.switch_to_block(after_block); + } + } + self.leave_conditional_block(); + } + ast::Stmt::While(ast::StmtWhile { + test, body, orelse, .. + }) => self.compile_while(test, body, orelse)?, + ast::Stmt::With(ast::StmtWith { + items, + body, + is_async, + .. + }) => self.compile_with(items, body, *is_async)?, + ast::Stmt::For(ast::StmtFor { + target, + iter, + body, + orelse, + is_async, + .. + }) => self.compile_for(target, iter, body, orelse, *is_async)?, + ast::Stmt::Match(ast::StmtMatch { subject, cases, .. }) => { + self.compile_match(subject, cases)? + } + ast::Stmt::Raise(ast::StmtRaise { + exc, cause, range, .. + }) => { + let kind = match exc { + Some(value) => { + self.compile_expression(value)?; + match cause { + Some(cause) => { + self.compile_expression(cause)?; + bytecode::RaiseKind::RaiseCause + } + None => bytecode::RaiseKind::Raise, + } + } + None => bytecode::RaiseKind::BareRaise, + }; + self.set_source_range(*range); + emit!(self, Instruction::RaiseVarargs { argc: kind }); + // Start a new block so dead code after raise doesn't + // corrupt the except stack in label_exception_targets + let dead = self.new_block(); + self.switch_to_block(dead); + } + ast::Stmt::Try(ast::StmtTry { + body, + handlers, + orelse, + finalbody, + is_star, + .. + }) => { + self.enter_conditional_block(); + if *is_star { + self.compile_try_star_except(body, handlers, orelse, finalbody)? + } else { + self.compile_try_statement(body, handlers, orelse, finalbody)? + } + self.leave_conditional_block(); + } + ast::Stmt::FunctionDef(ast::StmtFunctionDef { + name, + parameters, + body, + decorator_list, + returns, + type_params, + is_async, + .. + }) => { + validate_duplicate_params(parameters).map_err(|e| self.error(e))?; + + self.compile_function_def( + name.as_str(), + parameters, + body, + decorator_list, + returns.as_deref(), + *is_async, + type_params.as_deref(), + )? + } + ast::Stmt::ClassDef(ast::StmtClassDef { + name, + body, + decorator_list, + type_params, + arguments, + .. + }) => self.compile_class_def( + name.as_str(), + body, + decorator_list, + type_params.as_deref(), + arguments.as_deref(), + )?, + ast::Stmt::Assert(ast::StmtAssert { test, msg, .. }) => { + // if some flag, ignore all assert statements! + if self.opts.optimize == 0 { + let after_block = self.new_block(); + self.compile_jump_if(test, true, after_block)?; + + emit!( + self, + Instruction::LoadCommonConstant { + idx: bytecode::CommonConstant::AssertionError + } + ); + emit!(self, Instruction::PushNull); + match msg { + Some(e) => { + self.compile_expression(e)?; + emit!(self, Instruction::Call { argc: 1 }); + } + None => { + emit!(self, Instruction::Call { argc: 0 }); + } + } + emit!( + self, + Instruction::RaiseVarargs { + argc: bytecode::RaiseKind::Raise, + } + ); + + self.switch_to_block(after_block); + } else { + // Optimized-out asserts still need to consume any nested + // scope symbol tables they contain so later nested scopes + // stay aligned with AST traversal order. + self.consume_skipped_nested_scopes_in_expr(test)?; + if let Some(expr) = msg { + self.consume_skipped_nested_scopes_in_expr(expr)?; + } + } + } + ast::Stmt::Break(_) => { + emit!(self, Instruction::Nop); // NOP for line tracing + // Unwind fblock stack until we find a loop, emitting cleanup for each fblock + self.compile_break_continue(statement.range(), true)?; + let dead = self.new_block(); + self.switch_to_block(dead); + } + ast::Stmt::Continue(_) => { + emit!(self, Instruction::Nop); // NOP for line tracing + // Unwind fblock stack until we find a loop, emitting cleanup for each fblock + self.compile_break_continue(statement.range(), false)?; + let dead = self.new_block(); + self.switch_to_block(dead); + } + ast::Stmt::Return(ast::StmtReturn { value, .. }) => { + if !self.ctx.in_func() { + return Err( + self.error_ranged(CodegenErrorType::InvalidReturn, statement.range()) + ); + } + + match value { + Some(v) => { + if self.ctx.func == FunctionContext::AsyncFunction + && self + .current_code_info() + .flags + .contains(bytecode::CodeFlags::GENERATOR) + { + return Err(self.error_ranged( + CodegenErrorType::AsyncReturnValue, + statement.range(), + )); + } + self.compile_expression(v)?; + // Unwind fblock stack with preserve_tos=true (preserve return value) + self.unwind_fblock_stack(true, false)?; + self.emit_return_value(); + } + None => { + // Unwind fblock stack with preserve_tos=false (no value to preserve) + self.unwind_fblock_stack(false, false)?; + self.emit_return_const(ConstantData::None); + } + } + let dead = self.new_block(); + self.switch_to_block(dead); + } + ast::Stmt::Assign(ast::StmtAssign { targets, value, .. }) => { + self.compile_expression(value)?; + + for (i, target) in targets.iter().enumerate() { + if i + 1 != targets.len() { + emit!(self, Instruction::Copy { i: 1 }); + } + self.compile_store(target)?; + } + } + ast::Stmt::AugAssign(ast::StmtAugAssign { + target, op, value, .. + }) => self.compile_augassign(target, op, value)?, + ast::Stmt::AnnAssign(ast::StmtAnnAssign { + target, + annotation, + value, + simple, + .. + }) => { + self.compile_annotated_assign(target, annotation, value.as_deref(), *simple)?; + // Bare annotations in function scope emit no code; restore + // source range so subsequent instructions keep the correct line. + if value.is_none() && self.ctx.in_func() { + self.set_source_range(prev_source_range); + } + } + ast::Stmt::Delete(ast::StmtDelete { targets, .. }) => { + for target in targets { + self.compile_delete(target)?; + } + } + ast::Stmt::Pass(_) => { + emit!(self, Instruction::Nop); // NOP for line tracing + } + ast::Stmt::TypeAlias(ast::StmtTypeAlias { + name, + type_params, + value, + .. + }) => { + // let name_string = name.to_string(); + let Some(name) = name.as_name_expr() else { + // FIXME: is error here? + return Err(self.error(CodegenErrorType::SyntaxError( + "type alias expect name".to_owned(), + ))); + }; + let name_string = name.id.to_string(); + + // For PEP 695 syntax, we need to compile type_params first + // so that they're available when compiling the value expression + // Push name first + self.emit_load_const(ConstantData::Str { + value: name_string.clone().into(), + }); + + if let Some(type_params) = type_params { + // Outer scope for TypeParams + self.push_symbol_table()?; + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get().to_u32(); + let scope_name = format!("<generic parameters of {name_string}>"); + self.enter_scope(&scope_name, CompilerScope::TypeParams, key, lineno)?; + + // TypeParams scope is function-like + let prev_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: prev_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + // Compile type params inside the scope + self.compile_type_params(type_params)?; + // Stack: [type_params_tuple] + + // Inner closure for lazy value evaluation + self.push_symbol_table()?; + let inner_key = self.symbol_table_stack.len() - 1; + self.enter_scope("TypeAlias", CompilerScope::TypeParams, inner_key, lineno)?; + // Evaluator takes a positional-only format parameter + self.current_code_info().metadata.argcount = 1; + self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + self.emit_format_validation()?; + self.compile_expression(value)?; + emit!(self, Instruction::ReturnValue); + let value_code = self.exit_scope(); + self.make_closure(value_code, bytecode::MakeFunctionFlags::new())?; + // Stack: [type_params_tuple, value_closure] + + // Swap so unpack_sequence reverse gives correct order + emit!(self, Instruction::Swap { i: 2 }); + // Stack: [value_closure, type_params_tuple] + + // Build tuple and return from TypeParams scope + emit!(self, Instruction::BuildTuple { count: 2 }); + emit!(self, Instruction::ReturnValue); + + let code = self.exit_scope(); + self.ctx = prev_ctx; + self.make_closure(code, bytecode::MakeFunctionFlags::new())?; + emit!(self, Instruction::PushNull); + emit!(self, Instruction::Call { argc: 0 }); + + // Unpack: (value_closure, type_params_tuple) + // UnpackSequence reverses → stack: [name, type_params_tuple, value_closure] + emit!(self, Instruction::UnpackSequence { count: 2 }); + } else { + // Push None for type_params + self.emit_load_const(ConstantData::None); + // Stack: [name, None] + + // Create a closure for lazy evaluation of the value + self.push_symbol_table()?; + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get().to_u32(); + self.enter_scope("TypeAlias", CompilerScope::TypeParams, key, lineno)?; + // Evaluator takes a positional-only format parameter + self.current_code_info().metadata.argcount = 1; + self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + self.emit_format_validation()?; + + let prev_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: prev_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + self.compile_expression(value)?; + emit!(self, Instruction::ReturnValue); + + let code = self.exit_scope(); + self.ctx = prev_ctx; + self.make_closure(code, bytecode::MakeFunctionFlags::new())?; + // Stack: [name, None, closure] + } + + // Build tuple of 3 elements and call intrinsic + emit!(self, Instruction::BuildTuple { count: 3 }); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::TypeAlias + } + ); + self.store_name(&name_string)?; + } + ast::Stmt::IpyEscapeCommand(_) => todo!(), + } + Ok(()) + } + + fn compile_delete(&mut self, expression: &ast::Expr) -> CompileResult<()> { + match &expression { + ast::Expr::Name(ast::ExprName { id, .. }) => { + self.compile_name(id.as_str(), NameUsage::Delete)? + } + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { + self.compile_expression(value)?; + let namei = self.name(attr.as_str()); + emit!(self, Instruction::DeleteAttr { namei }); + } + ast::Expr::Subscript(ast::ExprSubscript { + value, slice, ctx, .. + }) => { + self.compile_subscript(value, slice, *ctx)?; + } + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) + | ast::Expr::List(ast::ExprList { elts, .. }) => { + for element in elts { + self.compile_delete(element)?; + } + } + ast::Expr::BinOp(_) | ast::Expr::UnaryOp(_) => { + return Err(self.error(CodegenErrorType::Delete("expression"))); + } + _ => return Err(self.error(CodegenErrorType::Delete(expression.python_name()))), + } + Ok(()) + } + + fn enter_function(&mut self, name: &str, parameters: &ast::Parameters) -> CompileResult<()> { + // TODO: partition_in_place + let mut kw_without_defaults = vec![]; + let mut kw_with_defaults = vec![]; + for kwonlyarg in &parameters.kwonlyargs { + if let Some(default) = &kwonlyarg.default { + kw_with_defaults.push((&kwonlyarg.parameter, default)); + } else { + kw_without_defaults.push(&kwonlyarg.parameter); + } + } + + self.push_output( + bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED, + parameters.posonlyargs.len().to_u32(), + (parameters.posonlyargs.len() + parameters.args.len()).to_u32(), + parameters.kwonlyargs.len().to_u32(), + name.to_owned(), + )?; + + let args_iter = core::iter::empty() + .chain(&parameters.posonlyargs) + .chain(&parameters.args) + .map(|arg| &arg.parameter) + .chain(kw_without_defaults) + .chain(kw_with_defaults.into_iter().map(|(arg, _)| arg)); + for name in args_iter { + self.varname(name.name.as_str())?; + } + + if let Some(name) = parameters.vararg.as_deref() { + self.current_code_info().flags |= bytecode::CodeFlags::VARARGS; + self.varname(name.name.as_str())?; + } + if let Some(name) = parameters.kwarg.as_deref() { + self.current_code_info().flags |= bytecode::CodeFlags::VARKEYWORDS; + self.varname(name.name.as_str())?; + } + + Ok(()) + } + + /// Push decorators onto the stack in source order. + /// For @dec1 @dec2 def foo(): stack becomes [dec1, NULL, dec2, NULL] + fn prepare_decorators(&mut self, decorator_list: &[ast::Decorator]) -> CompileResult<()> { + for decorator in decorator_list { + self.compile_expression(&decorator.expression)?; + emit!(self, Instruction::PushNull); + } + Ok(()) + } + + /// Apply decorators in reverse order (LIFO from stack). + /// Stack [dec1, NULL, dec2, NULL, func] -> dec2(func) -> dec1(dec2(func)) + /// The forward loop works because each Call pops from TOS, naturally + /// applying decorators bottom-up (innermost first). + fn apply_decorators(&mut self, decorator_list: &[ast::Decorator]) { + for _ in decorator_list { + emit!(self, Instruction::Call { argc: 1 }); + } + } + + /// Compile type parameter bound or default in a separate scope and return closure + fn compile_type_param_bound_or_default( + &mut self, + expr: &ast::Expr, + name: &str, + allow_starred: bool, + ) -> CompileResult<()> { + // Push the next symbol table onto the stack + self.push_symbol_table()?; + + // Get the current symbol table + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get().to_u32(); + + // Enter scope with the type parameter name + self.enter_scope(name, CompilerScope::TypeParams, key, lineno)?; + + // Evaluator takes a positional-only format parameter + self.current_code_info().metadata.argcount = 1; + self.current_code_info().metadata.posonlyargcount = 1; + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + self.emit_format_validation()?; + + // TypeParams scope is function-like + let prev_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: prev_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + // Compile the expression + if allow_starred && matches!(expr, ast::Expr::Starred(_)) { + if let ast::Expr::Starred(starred) = expr { + self.compile_expression(&starred.value)?; + emit!(self, Instruction::UnpackSequence { count: 1 }); + } + } else { + self.compile_expression(expr)?; + } + + // Return value + emit!(self, Instruction::ReturnValue); + + // Exit scope and create closure + let code = self.exit_scope(); + self.ctx = prev_ctx; + + // Create closure for lazy evaluation + self.make_closure(code, bytecode::MakeFunctionFlags::new())?; + + Ok(()) + } + + /// Store each type parameter so it is accessible to the current scope, and leave a tuple of + /// all the type parameters on the stack. Handles default values per PEP 695. + fn compile_type_params(&mut self, type_params: &ast::TypeParams) -> CompileResult<()> { + // First, compile each type parameter and store it + for type_param in &type_params.type_params { + match type_param { + ast::TypeParam::TypeVar(ast::TypeParamTypeVar { + name, + bound, + default, + .. + }) => { + self.emit_load_const(ConstantData::Str { + value: name.as_str().into(), + }); + + if let Some(expr) = &bound { + let scope_name = if expr.is_tuple_expr() { + format!("<TypeVar constraint of {name}>") + } else { + format!("<TypeVar bound of {name}>") + }; + self.compile_type_param_bound_or_default(expr, &scope_name, false)?; + + let intrinsic = if expr.is_tuple_expr() { + bytecode::IntrinsicFunction2::TypeVarWithConstraint + } else { + bytecode::IntrinsicFunction2::TypeVarWithBound + }; + emit!(self, Instruction::CallIntrinsic2 { func: intrinsic }); + } else { + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::TypeVar + } + ); + } + + if let Some(default_expr) = default { + let scope_name = format!("<TypeVar default of {name}>"); + self.compile_type_param_bound_or_default(default_expr, &scope_name, false)?; + emit!( + self, + Instruction::CallIntrinsic2 { + func: bytecode::IntrinsicFunction2::SetTypeparamDefault + } + ); + } + + emit!(self, Instruction::Copy { i: 1 }); + self.store_name(name.as_ref())?; + } + ast::TypeParam::ParamSpec(ast::TypeParamParamSpec { name, default, .. }) => { + self.emit_load_const(ConstantData::Str { + value: name.as_str().into(), + }); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::ParamSpec + } + ); + + if let Some(default_expr) = default { + let scope_name = format!("<ParamSpec default of {name}>"); + self.compile_type_param_bound_or_default(default_expr, &scope_name, false)?; + emit!( + self, + Instruction::CallIntrinsic2 { + func: bytecode::IntrinsicFunction2::SetTypeparamDefault + } + ); + } + + emit!(self, Instruction::Copy { i: 1 }); + self.store_name(name.as_ref())?; + } + ast::TypeParam::TypeVarTuple(ast::TypeParamTypeVarTuple { + name, default, .. + }) => { + self.emit_load_const(ConstantData::Str { + value: name.as_str().into(), + }); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::TypeVarTuple + } + ); + + if let Some(default_expr) = default { + // TypeVarTuple allows starred expressions + let scope_name = format!("<TypeVarTuple default of {name}>"); + self.compile_type_param_bound_or_default(default_expr, &scope_name, true)?; + emit!( + self, + Instruction::CallIntrinsic2 { + func: bytecode::IntrinsicFunction2::SetTypeparamDefault + } + ); + } + + emit!(self, Instruction::Copy { i: 1 }); + self.store_name(name.as_ref())?; + } + }; + } + emit!( + self, + Instruction::BuildTuple { + count: u32::try_from(type_params.len()).unwrap(), + } + ); + Ok(()) + } + + fn compile_try_statement( + &mut self, + body: &[ast::Stmt], + handlers: &[ast::ExceptHandler], + orelse: &[ast::Stmt], + finalbody: &[ast::Stmt], + ) -> CompileResult<()> { + let handler_block = self.new_block(); + let finally_block = self.new_block(); + + // finally needs TWO blocks: + // - finally_block: normal path (no exception active) + // - finally_except_block: exception path (PUSH_EXC_INFO -> body -> RERAISE) + let finally_except_block = if !finalbody.is_empty() { + Some(self.new_block()) + } else { + None + }; + let finally_cleanup_block = if finally_except_block.is_some() { + Some(self.new_block()) + } else { + None + }; + // End block - continuation point after try-finally + // Normal path jumps here to skip exception path blocks + let end_block = self.new_block(); + + // Emit NOP at the try: line so LINE events fire for it + emit!(self, Instruction::Nop); + + // Setup a finally block if we have a finally statement. + // Push fblock with handler info for exception table generation + // IMPORTANT: handler goes to finally_except_block (exception path), not finally_block + if !finalbody.is_empty() { + // SETUP_FINALLY doesn't push lasti for try body handler + // Exception table: L1 to L2 -> L4 [1] (no lasti) + let setup_target = finally_except_block.unwrap_or(finally_block); + emit!( + self, + PseudoInstruction::SetupFinally { + delta: setup_target + } + ); + // Store finally body in fb_datum for unwind_fblock to compile inline + self.push_fblock_full( + FBlockType::FinallyTry, + finally_block, + finally_block, + FBlockDatum::FinallyBody(finalbody.to_vec()), // Clone finally body for unwind + )?; + } + + let else_block = self.new_block(); + + // if handlers is empty, compile body directly + // without wrapping in TryExcept (only FinallyTry is needed) + if handlers.is_empty() { + // Just compile body with FinallyTry fblock active (if finalbody exists) + self.compile_statements(body)?; + + // Pop FinallyTry fblock BEFORE compiling orelse/finally (normal path) + // This prevents exception table from covering the normal path + if !finalbody.is_empty() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyTry); + } + + // Compile orelse (usually empty for try-finally without except) + self.compile_statements(orelse)?; + + // Snapshot sub_tables before first finally compilation + // This allows us to restore them for the second compilation (exception path) + let sub_table_cursor = if !finalbody.is_empty() && finally_except_block.is_some() { + self.symbol_table_stack.last().map(|t| t.next_sub_table) + } else { + None + }; + + // Compile finally body inline for normal path + if !finalbody.is_empty() { + self.compile_statements(finalbody)?; + } + + // Jump to end (skip exception path blocks) + emit!(self, PseudoInstruction::Jump { delta: end_block }); + + if let Some(finally_except) = finally_except_block { + // Restore sub_tables for exception path compilation + if let Some(cursor) = sub_table_cursor + && let Some(current_table) = self.symbol_table_stack.last_mut() + { + current_table.next_sub_table = cursor; + } + + self.switch_to_block(finally_except); + // SETUP_CLEANUP before PUSH_EXC_INFO + if let Some(cleanup) = finally_cleanup_block { + emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup }); + } + emit!(self, Instruction::PushExcInfo); + if let Some(cleanup) = finally_cleanup_block { + self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; + } + self.compile_statements(finalbody)?; + + // Pop FinallyEnd fblock BEFORE emitting RERAISE + // This ensures RERAISE routes to outer exception handler, not cleanup block + // Cleanup block is only for new exceptions raised during finally body execution + if finally_cleanup_block.is_some() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyEnd); + } + + // Restore prev_exc as current exception before RERAISE + // Stack: [prev_exc, exc] -> COPY 2 -> [prev_exc, exc, prev_exc] + // POP_EXCEPT pops prev_exc and sets exc_info->exc_value = prev_exc + // Stack after POP_EXCEPT: [prev_exc, exc] + emit!(self, Instruction::Copy { i: 2 }); + emit!(self, Instruction::PopExcept); + + // RERAISE 0: re-raise the original exception to outer handler + emit!(self, Instruction::Reraise { depth: 0 }); + } + + if let Some(cleanup) = finally_cleanup_block { + self.switch_to_block(cleanup); + emit!(self, Instruction::Copy { i: 3 }); + emit!(self, Instruction::PopExcept); + emit!(self, Instruction::Reraise { depth: 1 }); + } + + self.switch_to_block(end_block); + return Ok(()); + } + + // try: + emit!( + self, + PseudoInstruction::SetupFinally { + delta: handler_block + } + ); + self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; + self.compile_statements(body)?; + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::TryExcept); + emit!(self, PseudoInstruction::Jump { delta: else_block }); + + // except handlers: + self.switch_to_block(handler_block); + + // SETUP_CLEANUP(cleanup) for except block + // This handles exceptions during exception matching + // Exception table: L2 to L3 -> L5 [1] lasti + // After PUSH_EXC_INFO, stack is [prev_exc, exc] + // depth=1 means keep prev_exc on stack when routing to cleanup + let cleanup_block = self.new_block(); + emit!( + self, + PseudoInstruction::SetupCleanup { + delta: cleanup_block + } + ); + self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + + // Exception is on top of stack now, pushed by unwind_blocks + // PUSH_EXC_INFO transforms [exc] -> [prev_exc, exc] for PopExcept + emit!(self, Instruction::PushExcInfo); + for handler in handlers { + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { + type_, + name, + body, + range: handler_range, + .. + }) = &handler; + self.set_source_range(*handler_range); + let next_handler = self.new_block(); + + // If we gave a typ, + // check if this handler can handle the exception: + if let Some(exc_type) = type_ { + // Check exception type: + // Stack: [prev_exc, exc] + self.compile_expression(exc_type)?; + // Stack: [prev_exc, exc, type] + emit!(self, Instruction::CheckExcMatch); + // Stack: [prev_exc, exc, bool] + emit!( + self, + Instruction::PopJumpIfFalse { + delta: next_handler + } + ); + // Stack: [prev_exc, exc] + + // We have a match, store in name (except x as y) + if let Some(alias) = name { + self.store_name(alias.as_str())? + } else { + // Drop exception from top of stack: + emit!(self, Instruction::PopTop); + } + } else { + // Catch all! + // Drop exception from top of stack: + emit!(self, Instruction::PopTop); + } + + // If name is bound, we need a cleanup handler for RERAISE + let handler_cleanup_block = if name.is_some() { + // SETUP_CLEANUP(cleanup_end) for named handler + let cleanup_end = self.new_block(); + emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup_end }); + self.push_fblock_full( + FBlockType::HandlerCleanup, + cleanup_end, + cleanup_end, + FBlockDatum::ExceptionName(name.as_ref().unwrap().as_str().to_owned()), + )?; + Some(cleanup_end) + } else { + // no SETUP_CLEANUP for unnamed handler + self.push_fblock(FBlockType::HandlerCleanup, finally_block, finally_block)?; + None + }; + + // Handler code: + self.compile_statements(body)?; + + self.pop_fblock(FBlockType::HandlerCleanup); + // PopBlock for inner SETUP_CLEANUP (named handler only) + if handler_cleanup_block.is_some() { + emit!(self, PseudoInstruction::PopBlock); + } + + // Create a block for normal path continuation (after handler body succeeds) + let handler_normal_exit = self.new_block(); + emit!( + self, + PseudoInstruction::JumpNoInterrupt { + delta: handler_normal_exit, + } + ); + + // cleanup_end block for named handler + // IMPORTANT: In CPython, cleanup_end is within outer SETUP_CLEANUP scope. + // so when RERAISE is executed, it goes to the cleanup block which does POP_EXCEPT. + // We MUST compile cleanup_end BEFORE popping ExceptionHandler so RERAISE routes to cleanup_block. + if let Some(cleanup_end) = handler_cleanup_block { + self.switch_to_block(cleanup_end); + if let Some(alias) = name { + // name = None; del name; before RERAISE + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + // RERAISE 1 (with lasti) - exception is on stack from exception table routing + // Stack at entry: [prev_exc (at handler_depth), lasti, exc] + // This RERAISE is within ExceptionHandler scope, so it routes to cleanup_block + // which does COPY 3; POP_EXCEPT; RERAISE + emit!(self, Instruction::Reraise { depth: 1 }); + } + + // Switch to normal exit block - this is where handler body success continues + self.switch_to_block(handler_normal_exit); + + // PopBlock for outer SETUP_CLEANUP (ExceptionHandler) + emit!(self, PseudoInstruction::PopBlock); + // Now pop ExceptionHandler - the normal path continues from here + self.pop_fblock(FBlockType::ExceptionHandler); + emit!(self, Instruction::PopExcept); + + // Delete the exception variable if it was bound (normal path) + if let Some(alias) = name { + // Set the variable to None before deleting + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + + // Pop FinallyTry block before jumping to finally body. + // The else_block path also pops this; both paths must agree + // on the except stack when entering finally_block. + if !finalbody.is_empty() { + emit!(self, PseudoInstruction::PopBlock); + } + + // Jump to finally block + emit!( + self, + PseudoInstruction::JumpNoInterrupt { + delta: finally_block, + } + ); + + // Re-push ExceptionHandler for next handler in the loop + // This will be popped at the end of handlers loop or when matched + self.push_fblock(FBlockType::ExceptionHandler, cleanup_block, cleanup_block)?; + + // Emit a new label for the next handler + self.switch_to_block(next_handler); + } + + // If code flows here, we have an unhandled exception, + // raise the exception again! + // RERAISE 0 + // Stack: [prev_exc, exc] - exception is on stack from PUSH_EXC_INFO + // NOTE: We emit RERAISE 0 BEFORE popping fblock so it is within cleanup handler scope + emit!(self, Instruction::Reraise { depth: 0 }); + + // Pop EXCEPTION_HANDLER fblock + // Pop after RERAISE so the instruction has the correct exception handler + self.pop_fblock(FBlockType::ExceptionHandler); + + // cleanup block (POP_EXCEPT_AND_RERAISE) + // Stack at entry: [prev_exc, lasti, exc] (depth=1 + lasti + exc pushed) + // COPY 3: copy prev_exc to top -> [prev_exc, lasti, exc, prev_exc] + // POP_EXCEPT: pop prev_exc from stack and restore -> [prev_exc, lasti, exc] + // RERAISE 1: reraise with lasti + self.switch_to_block(cleanup_block); + emit!(self, Instruction::Copy { i: 3 }); + emit!(self, Instruction::PopExcept); + emit!(self, Instruction::Reraise { depth: 1 }); + + // We successfully ran the try block: + // else: + self.switch_to_block(else_block); + self.compile_statements(orelse)?; + + // Pop the FinallyTry fblock before jumping to finally + if !finalbody.is_empty() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyTry); + } + + // Snapshot sub_tables before first finally compilation (for double compilation issue) + let sub_table_cursor = if !finalbody.is_empty() && finally_except_block.is_some() { + self.symbol_table_stack.last().map(|t| t.next_sub_table) + } else { + None + }; + + // finally (normal path): + self.switch_to_block(finally_block); + if !finalbody.is_empty() { + self.compile_statements(finalbody)?; + // Jump to end_block to skip exception path blocks + // This prevents fall-through to finally_except_block + emit!(self, PseudoInstruction::Jump { delta: end_block }); + } + + // finally (exception path) + // This is where exceptions go to run finally before reraise + // Stack at entry: [lasti, exc] (from exception table with preserve_lasti=true) + if let Some(finally_except) = finally_except_block { + // Restore sub_tables for exception path compilation + if let Some(cursor) = sub_table_cursor + && let Some(current_table) = self.symbol_table_stack.last_mut() + { + current_table.next_sub_table = cursor; + } + + self.switch_to_block(finally_except); + + // SETUP_CLEANUP for finally body + // Exceptions during finally body need to go to cleanup block + if let Some(cleanup) = finally_cleanup_block { + emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup }); + } + emit!(self, Instruction::PushExcInfo); + if let Some(cleanup) = finally_cleanup_block { + self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; + } + + // Run finally body + self.compile_statements(finalbody)?; + + // Pop FinallyEnd fblock BEFORE emitting RERAISE + // This ensures RERAISE routes to outer exception handler, not cleanup block + // Cleanup block is only for new exceptions raised during finally body execution + if finally_cleanup_block.is_some() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyEnd); + } + + // Restore prev_exc as current exception before RERAISE + // Stack: [lasti, prev_exc, exc] -> COPY 2 -> [lasti, prev_exc, exc, prev_exc] + // POP_EXCEPT pops prev_exc and sets exc_info->exc_value = prev_exc + // Stack after POP_EXCEPT: [lasti, prev_exc, exc] + emit!(self, Instruction::Copy { i: 2 }); + emit!(self, Instruction::PopExcept); + + // RERAISE 0: re-raise the original exception to outer handler + // Stack: [lasti, prev_exc, exc] - exception is on top + emit!(self, Instruction::Reraise { depth: 0 }); + } + + // finally cleanup block + // This handles exceptions that occur during the finally body itself + // Stack at entry: [lasti, prev_exc, lasti2, exc2] after exception table routing + if let Some(cleanup) = finally_cleanup_block { + self.switch_to_block(cleanup); + // COPY 3: copy the exception from position 3 + emit!(self, Instruction::Copy { i: 3 }); + // POP_EXCEPT: restore prev_exc as current exception + emit!(self, Instruction::PopExcept); + // RERAISE 1: reraise with lasti from stack + emit!(self, Instruction::Reraise { depth: 1 }); + } + + // End block - continuation point after try-finally + // Normal execution continues here after the finally block + self.switch_to_block(end_block); + + Ok(()) + } + + fn compile_try_star_except( + &mut self, + body: &[ast::Stmt], + handlers: &[ast::ExceptHandler], + orelse: &[ast::Stmt], + finalbody: &[ast::Stmt], + ) -> CompileResult<()> { + // compiler_try_star_except + // Stack layout during handler processing: [prev_exc, orig, list, rest] + let handler_block = self.new_block(); + let finally_block = self.new_block(); + let else_block = self.new_block(); + let end_block = self.new_block(); + let reraise_star_block = self.new_block(); + let reraise_block = self.new_block(); + let finally_cleanup_block = if !finalbody.is_empty() { + Some(self.new_block()) + } else { + None + }; + let exit_block = self.new_block(); + + // Emit NOP at the try: line so LINE events fire for it + emit!(self, Instruction::Nop); + + // Push fblock with handler info for exception table generation + if !finalbody.is_empty() { + emit!( + self, + PseudoInstruction::SetupFinally { + delta: finally_block + } + ); + self.push_fblock_full( + FBlockType::FinallyTry, + finally_block, + finally_block, + FBlockDatum::FinallyBody(finalbody.to_vec()), + )?; + } + + // SETUP_FINALLY for try body + emit!( + self, + PseudoInstruction::SetupFinally { + delta: handler_block + } + ); + self.push_fblock(FBlockType::TryExcept, handler_block, handler_block)?; + self.compile_statements(body)?; + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::TryExcept); + emit!(self, PseudoInstruction::Jump { delta: else_block }); + + // Exception handler entry + self.switch_to_block(handler_block); + // Stack: [exc] (from exception table) + + // PUSH_EXC_INFO + emit!(self, Instruction::PushExcInfo); + // Stack: [prev_exc, exc] + + // Push EXCEPTION_GROUP_HANDLER fblock + let eg_dummy1 = self.new_block(); + let eg_dummy2 = self.new_block(); + self.push_fblock(FBlockType::ExceptionGroupHandler, eg_dummy1, eg_dummy2)?; + + // Initialize handler stack before the loop + // BUILD_LIST 0 + COPY 2 to set up [prev_exc, orig, list, rest] + emit!(self, Instruction::BuildList { count: 0 }); + // Stack: [prev_exc, exc, []] + emit!(self, Instruction::Copy { i: 2 }); + // Stack: [prev_exc, orig, list, rest] + + let n = handlers.len(); + if n == 0 { + // Empty handlers (invalid AST) - append rest to list and proceed + // Stack: [prev_exc, orig, list, rest] + emit!(self, Instruction::ListAppend { i: 0 }); + // Stack: [prev_exc, orig, list] + emit!( + self, + PseudoInstruction::Jump { + delta: reraise_star_block + } + ); + } + for (i, handler) in handlers.iter().enumerate() { + let ast::ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { + type_, + name, + body, + .. + }) = handler; + + let no_match_block = self.new_block(); + let next_block = self.new_block(); + + // Compile exception type + if let Some(exc_type) = type_ { + // Check for unparenthesized tuple + if let ast::Expr::Tuple(ast::ExprTuple { elts, range, .. }) = exc_type.as_ref() + && let Some(first) = elts.first() + && range.start().to_u32() == first.range().start().to_u32() + { + return Err(self.error(CodegenErrorType::SyntaxError( + "multiple exception types must be parenthesized".to_owned(), + ))); + } + self.compile_expression(exc_type)?; + } else { + return Err(self.error(CodegenErrorType::SyntaxError( + "except* must specify an exception type".to_owned(), + ))); + } + // Stack: [prev_exc, orig, list, rest, type] + + // ADDOP(c, loc, CHECK_EG_MATCH); + emit!(self, Instruction::CheckEgMatch); + // Stack: [prev_exc, orig, list, new_rest, match] + + // ADDOP_I(c, loc, COPY, 1); + // ADDOP_JUMP(c, loc, POP_JUMP_IF_NONE, no_match); + emit!(self, Instruction::Copy { i: 1 }); + emit!( + self, + Instruction::PopJumpIfNone { + delta: no_match_block + } + ); + + // Handler matched + // Stack: [prev_exc, orig, list, new_rest, match] + // Note: CheckEgMatch already sets the matched exception as current exception + let handler_except_block = self.new_block(); + + // Store match to name or pop + if let Some(alias) = name { + self.store_name(alias.as_str())?; + } else { + emit!(self, Instruction::PopTop); // pop match + } + // Stack: [prev_exc, orig, list, new_rest] + + // HANDLER_CLEANUP fblock for handler body + emit!( + self, + PseudoInstruction::SetupCleanup { + delta: handler_except_block + } + ); + self.push_fblock_full( + FBlockType::HandlerCleanup, + next_block, + end_block, + if let Some(alias) = name { + FBlockDatum::ExceptionName(alias.as_str().to_owned()) + } else { + FBlockDatum::None + }, + )?; + + // Execute handler body + self.compile_statements(body)?; + + // Handler body completed normally + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::HandlerCleanup); + + // Cleanup name binding + if let Some(alias) = name { + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + + // Jump to next handler + emit!(self, PseudoInstruction::Jump { delta: next_block }); + + // Handler raised an exception (cleanup_end label) + self.switch_to_block(handler_except_block); + // Stack: [prev_exc, orig, list, new_rest, lasti, raised_exc] + // (lasti is pushed because push_lasti=true in HANDLER_CLEANUP fblock) + + // Cleanup name binding + if let Some(alias) = name { + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + + // LIST_APPEND(3) - append raised_exc to list + // Stack: [prev_exc, orig, list, new_rest, lasti, raised_exc] + // After pop: [prev_exc, orig, list, new_rest, lasti] (len=5) + // nth_value(i) = stack[len - i - 1], we need stack[2] = list + // stack[5 - i - 1] = 2 -> i = 2 + emit!(self, Instruction::ListAppend { i: 2 }); + // Stack: [prev_exc, orig, list, new_rest, lasti] + + // POP_TOP - pop lasti + emit!(self, Instruction::PopTop); + // Stack: [prev_exc, orig, list, new_rest] + + // JUMP except_with_error + // We directly JUMP to next_block since no_match_block falls through to it + emit!(self, PseudoInstruction::Jump { delta: next_block }); + + // No match - pop match (None) + self.switch_to_block(no_match_block); + emit!(self, Instruction::PopTop); // pop match (None) + // Stack: [prev_exc, orig, list, new_rest] + // Falls through to next_block + + // except_with_error label + // All paths merge here at next_block + self.switch_to_block(next_block); + // Stack: [prev_exc, orig, list, rest] + + // After last handler, append rest to list + if i == n - 1 { + // Stack: [prev_exc, orig, list, rest] + // ADDOP_I(c, NO_LOCATION, LIST_APPEND, 1); + // PEEK(1) = stack[len-1] after pop + // RustPython nth_value(i) = stack[len-i-1] after pop + // For LIST_APPEND 1: stack[len-1] = stack[len-i-1] -> i = 0 + emit!(self, Instruction::ListAppend { i: 0 }); + // Stack: [prev_exc, orig, list] + emit!( + self, + PseudoInstruction::Jump { + delta: reraise_star_block + } + ); + } + } + + // Pop EXCEPTION_GROUP_HANDLER fblock + self.pop_fblock(FBlockType::ExceptionGroupHandler); + + // Reraise star block + self.switch_to_block(reraise_star_block); + // Stack: [prev_exc, orig, list] + + // CALL_INTRINSIC_2 PREP_RERAISE_STAR + // Takes 2 args (orig, list) and produces result + emit!( + self, + Instruction::CallIntrinsic2 { + func: bytecode::IntrinsicFunction2::PrepReraiseStar + } + ); + // Stack: [prev_exc, result] + + // COPY 1 + emit!(self, Instruction::Copy { i: 1 }); + // Stack: [prev_exc, result, result] + + // POP_JUMP_IF_NOT_NONE reraise + emit!( + self, + Instruction::PopJumpIfNotNone { + delta: reraise_block + } + ); + // Stack: [prev_exc, result] + + // Nothing to reraise + // POP_TOP - pop result (None) + emit!(self, Instruction::PopTop); + // Stack: [prev_exc] + + // POP_BLOCK - no-op for us with exception tables (fblocks handle this) + // POP_EXCEPT - restore previous exception context + emit!(self, Instruction::PopExcept); + // Stack: [] + + if !finalbody.is_empty() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyTry); + } + + emit!(self, PseudoInstruction::Jump { delta: end_block }); + + // Reraise the result + self.switch_to_block(reraise_block); + // Stack: [prev_exc, result] + + // POP_BLOCK - no-op for us + // SWAP 2 + emit!(self, Instruction::Swap { i: 2 }); + // Stack: [result, prev_exc] + + // POP_EXCEPT + emit!(self, Instruction::PopExcept); + // Stack: [result] + + // RERAISE 0 + emit!(self, Instruction::Reraise { depth: 0 }); + + // try-else path + // NOTE: When we reach here in compilation, the nothing-to-reraise path above + // has already popped FinallyTry. But else_block is a different execution path + // that branches from try body success (where FinallyTry is still active). + // We need to re-push FinallyTry to reflect the correct fblock state for else path. + if !finalbody.is_empty() { + emit!( + self, + PseudoInstruction::SetupFinally { + delta: finally_block + } + ); + self.push_fblock_full( + FBlockType::FinallyTry, + finally_block, + finally_block, + FBlockDatum::FinallyBody(finalbody.to_vec()), + )?; + } + self.switch_to_block(else_block); + self.compile_statements(orelse)?; + + if !finalbody.is_empty() { + // Pop the FinallyTry fblock we just pushed for the else path + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyTry); + } + + emit!(self, PseudoInstruction::Jump { delta: end_block }); + + self.switch_to_block(end_block); + if !finalbody.is_empty() { + // Snapshot sub_tables before first finally compilation + let sub_table_cursor = self.symbol_table_stack.last().map(|t| t.next_sub_table); + + // Compile finally body inline for normal path + self.compile_statements(finalbody)?; + emit!(self, PseudoInstruction::Jump { delta: exit_block }); + + // Restore sub_tables for exception path compilation + if let Some(cursor) = sub_table_cursor + && let Some(current_table) = self.symbol_table_stack.last_mut() + { + current_table.next_sub_table = cursor; + } + + // Exception handler path + self.switch_to_block(finally_block); + emit!(self, Instruction::PushExcInfo); + + if let Some(cleanup) = finally_cleanup_block { + emit!(self, PseudoInstruction::SetupCleanup { delta: cleanup }); + self.push_fblock(FBlockType::FinallyEnd, cleanup, cleanup)?; + } + + self.compile_statements(finalbody)?; + + if finally_cleanup_block.is_some() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::FinallyEnd); + } + + emit!(self, Instruction::Copy { i: 2 }); + emit!(self, Instruction::PopExcept); + emit!(self, Instruction::Reraise { depth: 0 }); + + if let Some(cleanup) = finally_cleanup_block { + self.switch_to_block(cleanup); + emit!(self, Instruction::Copy { i: 3 }); + emit!(self, Instruction::PopExcept); + emit!(self, Instruction::Reraise { depth: 1 }); + } + } + + self.switch_to_block(exit_block); + + Ok(()) + } + + /// Compile default arguments + // = compiler_default_arguments + fn compile_default_arguments( + &mut self, + parameters: &ast::Parameters, + ) -> CompileResult<bytecode::MakeFunctionFlags> { + let mut funcflags = bytecode::MakeFunctionFlags::new(); + + // Handle positional defaults + let defaults: Vec<_> = core::iter::empty() + .chain(&parameters.posonlyargs) + .chain(&parameters.args) + .filter_map(|x| x.default.as_deref()) + .collect(); + + if !defaults.is_empty() { + // Compile defaults and build tuple + for default in &defaults { + self.compile_expression(default)?; + } + emit!( + self, + Instruction::BuildTuple { + count: defaults.len().to_u32() + } + ); + funcflags.insert(bytecode::MakeFunctionFlag::Defaults); + } + + // Handle keyword-only defaults + let mut kw_with_defaults = vec![]; + for kwonlyarg in &parameters.kwonlyargs { + if let Some(default) = &kwonlyarg.default { + kw_with_defaults.push((&kwonlyarg.parameter, default)); + } + } + + if !kw_with_defaults.is_empty() { + // Compile kwdefaults and build dict + for (arg, default) in &kw_with_defaults { + self.emit_load_const(ConstantData::Str { + value: self.mangle(arg.name.as_str()).into_owned().into(), + }); + self.compile_expression(default)?; + } + emit!( + self, + Instruction::BuildMap { + count: kw_with_defaults.len().to_u32(), + } + ); + funcflags.insert(bytecode::MakeFunctionFlag::KwOnlyDefaults); + } + + Ok(funcflags) + } + + /// Compile function body and create function object + // = compiler_function_body + fn compile_function_body( + &mut self, + name: &str, + parameters: &ast::Parameters, + body: &[ast::Stmt], + is_async: bool, + funcflags: bytecode::MakeFunctionFlags, + ) -> CompileResult<()> { + // Save source range so MAKE_FUNCTION gets the `def` line, not the body's last line + let saved_range = self.current_source_range; + + // Always enter function scope + self.enter_function(name, parameters)?; + self.current_code_info() + .flags + .set(bytecode::CodeFlags::COROUTINE, is_async); + + // Set up context + let prev_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: prev_ctx.in_class, + func: if is_async { + FunctionContext::AsyncFunction + } else { + FunctionContext::Function + }, + // A function starts a new async scope only if it's async + in_async_scope: is_async, + }; + + // Set qualname + self.set_qualname(); + + // Handle docstring - store in co_consts[0] if present + let (doc_str, body) = split_doc(body, &self.opts); + if let Some(doc) = &doc_str { + // Docstring present: store in co_consts[0] and set HAS_DOCSTRING flag + self.current_code_info() + .metadata + .consts + .insert_full(ConstantData::Str { + value: doc.to_string().into(), + }); + self.current_code_info().flags |= bytecode::CodeFlags::HAS_DOCSTRING; + } + // If no docstring, don't add None to co_consts + // Note: RETURN_GENERATOR + POP_TOP for async functions is emitted in enter_scope() + + // Compile body statements + self.compile_statements(body)?; + + // Emit None at end if needed + match body.last() { + Some(ast::Stmt::Return(_)) => {} + _ => { + self.emit_return_const(ConstantData::None); + } + } + + // Exit scope and create function object + let code = self.exit_scope(); + self.ctx = prev_ctx; + + self.set_source_range(saved_range); + + // Create function object with closure + self.make_closure(code, funcflags)?; + + // Note: docstring is now retrieved from co_consts[0] by the VM + // when HAS_DOCSTRING flag is set, so no runtime __doc__ assignment needed + + Ok(()) + } + + /// Compile function annotations as a closure (PEP 649) + /// Returns true if an __annotate__ closure was created + /// Uses symbol table's annotation_block for proper scoping. + fn compile_annotations_closure( + &mut self, + func_name: &str, + parameters: &ast::Parameters, + returns: Option<&ast::Expr>, + ) -> CompileResult<bool> { + // Try to enter annotation scope - returns None if no annotation_block exists + let Some(saved_ctx) = self.enter_annotation_scope(func_name)? else { + return Ok(false); + }; + + // Count annotations + let parameters_iter = core::iter::empty() + .chain(&parameters.posonlyargs) + .chain(&parameters.args) + .chain(&parameters.kwonlyargs) + .map(|x| &x.parameter) + .chain(parameters.vararg.as_deref()) + .chain(parameters.kwarg.as_deref()); + + let num_annotations: u32 = + u32::try_from(parameters_iter.filter(|p| p.annotation.is_some()).count()) + .expect("too many annotations") + + if returns.is_some() { 1 } else { 0 }; + + // Compile annotations inside the annotation scope + let parameters_iter = core::iter::empty() + .chain(&parameters.posonlyargs) + .chain(&parameters.args) + .chain(&parameters.kwonlyargs) + .map(|x| &x.parameter) + .chain(parameters.vararg.as_deref()) + .chain(parameters.kwarg.as_deref()); + + for param in parameters_iter { + if let Some(annotation) = &param.annotation { + self.emit_load_const(ConstantData::Str { + value: self.mangle(param.name.as_str()).into_owned().into(), + }); + self.compile_annotation(annotation)?; + } + } + + // Handle return annotation + if let Some(annotation) = returns { + self.emit_load_const(ConstantData::Str { + value: "return".into(), + }); + self.compile_annotation(annotation)?; + } + + // Build the map and return it + emit!( + self, + Instruction::BuildMap { + count: num_annotations, + } + ); + emit!(self, Instruction::ReturnValue); + + // Exit the annotation scope and get the code object + let annotate_code = self.exit_annotation_scope(saved_ctx); + + // Make a closure from the code object + self.make_closure(annotate_code, bytecode::MakeFunctionFlags::new())?; + + Ok(true) + } + + /// Collect simple annotations from module body in AST order (including nested blocks) + /// Returns list of (name, annotation_expr) pairs + /// This must match the order that annotations are compiled to ensure + /// conditional_annotation_index stays in sync with __annotate__ enumeration. + fn collect_simple_annotations(body: &[ast::Stmt]) -> Vec<(&str, &ast::Expr)> { + fn walk<'a>(stmts: &'a [ast::Stmt], out: &mut Vec<(&'a str, &'a ast::Expr)>) { + for stmt in stmts { + match stmt { + ast::Stmt::AnnAssign(ast::StmtAnnAssign { + target, + annotation, + simple, + .. + }) if *simple && matches!(target.as_ref(), ast::Expr::Name(_)) => { + if let ast::Expr::Name(ast::ExprName { id, .. }) = target.as_ref() { + out.push((id.as_str(), annotation.as_ref())); + } + } + ast::Stmt::If(ast::StmtIf { + body, + elif_else_clauses, + .. + }) => { + walk(body, out); + for clause in elif_else_clauses { + walk(&clause.body, out); + } + } + ast::Stmt::For(ast::StmtFor { body, orelse, .. }) + | ast::Stmt::While(ast::StmtWhile { body, orelse, .. }) => { + walk(body, out); + walk(orelse, out); + } + ast::Stmt::With(ast::StmtWith { body, .. }) => walk(body, out), + ast::Stmt::Try(ast::StmtTry { + body, + handlers, + orelse, + finalbody, + .. + }) => { + walk(body, out); + for handler in handlers { + let ast::ExceptHandler::ExceptHandler( + ast::ExceptHandlerExceptHandler { body, .. }, + ) = handler; + walk(body, out); + } + walk(orelse, out); + walk(finalbody, out); + } + ast::Stmt::Match(ast::StmtMatch { cases, .. }) => { + for case in cases { + walk(&case.body, out); + } + } + _ => {} + } + } + } + let mut annotations = Vec::new(); + walk(body, &mut annotations); + annotations + } + + /// Compile module-level __annotate__ function (PEP 649) + /// Returns true if __annotate__ was created and stored + fn compile_module_annotate(&mut self, body: &[ast::Stmt]) -> CompileResult<bool> { + // Collect simple annotations from module body first + let annotations = Self::collect_simple_annotations(body); + + if annotations.is_empty() { + return Ok(false); + } + + // Check if we have conditional annotations + let has_conditional = self.current_symbol_table().has_conditional_annotations; + + // Get parent scope type BEFORE pushing annotation symbol table + let parent_scope_type = self.current_symbol_table().typ; + // Try to push annotation symbol table from current scope + if !self.push_current_annotation_symbol_table() { + return Ok(false); + } + + // Annotation scopes are never async (even inside async functions) + let saved_ctx = self.ctx; + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + // Enter annotation scope for code generation + let key = self.symbol_table_stack.len() - 1; + let lineno = self.get_source_line_number().get(); + self.enter_scope( + "__annotate__", + CompilerScope::Annotation, + key, + lineno.to_u32(), + )?; + + // Add 'format' parameter to varnames + self.current_code_info() + .metadata + .varnames + .insert("format".to_owned()); + + // Emit format validation: if format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError + self.emit_format_validation()?; + + if has_conditional { + // PEP 649: Build dict incrementally, checking conditional annotations + // Start with empty dict + emit!(self, Instruction::BuildMap { count: 0 }); + + // Process each annotation + for (idx, (name, annotation)) in annotations.iter().enumerate() { + // Check if index is in __conditional_annotations__ + let not_set_block = self.new_block(); + + // LOAD_CONST index + self.emit_load_const(ConstantData::Integer { value: idx.into() }); + // Load __conditional_annotations__ from appropriate scope + // Class scope: LoadDeref (freevars), Module scope: LoadGlobal + if parent_scope_type == CompilerScope::Class { + let idx = self.get_free_var_index("__conditional_annotations__")?; + emit!(self, Instruction::LoadDeref { i: idx }); + } else { + let cond_annotations_name = self.name("__conditional_annotations__"); + self.emit_load_global(cond_annotations_name, false); + } + // CONTAINS_OP (in) + emit!( + self, + Instruction::ContainsOp { + invert: bytecode::Invert::No + } + ); + // POP_JUMP_IF_FALSE not_set + emit!( + self, + Instruction::PopJumpIfFalse { + delta: not_set_block + } + ); + + // Annotation value + self.compile_annotation(annotation)?; + // COPY dict to TOS + emit!(self, Instruction::Copy { i: 2 }); + // LOAD_CONST name + self.emit_load_const(ConstantData::Str { + value: self.mangle(name).into_owned().into(), + }); + // STORE_SUBSCR - dict[name] = value + emit!(self, Instruction::StoreSubscr); + + // not_set label + self.switch_to_block(not_set_block); + } + + // Return the dict + emit!(self, Instruction::ReturnValue); + } else { + // No conditional annotations - use simple BuildMap + let num_annotations = u32::try_from(annotations.len()).expect("too many annotations"); + + // Compile annotations inside the annotation scope + for (name, annotation) in annotations { + self.emit_load_const(ConstantData::Str { + value: self.mangle(name).into_owned().into(), + }); + self.compile_annotation(annotation)?; + } + + // Build the map and return it + emit!( + self, + Instruction::BuildMap { + count: num_annotations, + } + ); + emit!(self, Instruction::ReturnValue); + } + + // Exit annotation scope - pop symbol table, restore to parent's annotation_block, and get code + let annotation_table = self.pop_symbol_table(); + // Restore annotation_block to module's symbol table + self.symbol_table_stack + .last_mut() + .expect("no module symbol table") + .annotation_block = Some(Box::new(annotation_table)); + // Restore context + self.ctx = saved_ctx; + // Exit code scope + let pop = self.code_stack.pop(); + let annotate_code = unwrap_internal( + self, + compiler_unwrap_option(self, pop).finalize_code(&self.opts), + ); + + // Make a closure from the code object + self.make_closure(annotate_code, bytecode::MakeFunctionFlags::new())?; + + // Store as __annotate_func__ for classes, __annotate__ for modules + let name = if parent_scope_type == CompilerScope::Class { + "__annotate_func__" + } else { + "__annotate__" + }; + self.store_name(name)?; + + Ok(true) + } + + // = compiler_function + #[allow(clippy::too_many_arguments)] + fn compile_function_def( + &mut self, + name: &str, + parameters: &ast::Parameters, + body: &[ast::Stmt], + decorator_list: &[ast::Decorator], + returns: Option<&ast::Expr>, // TODO: use type hint somehow.. + is_async: bool, + type_params: Option<&ast::TypeParams>, + ) -> CompileResult<()> { + self.prepare_decorators(decorator_list)?; + + // compile defaults and return funcflags + let funcflags = self.compile_default_arguments(parameters)?; + + let is_generic = type_params.is_some(); + let mut num_typeparam_args = 0; + + // Save context before entering TypeParams scope + let saved_ctx = self.ctx; + + if is_generic { + // Count args to pass to type params scope + if funcflags.contains(&bytecode::MakeFunctionFlag::Defaults) { + num_typeparam_args += 1; + } + if funcflags.contains(&bytecode::MakeFunctionFlag::KwOnlyDefaults) { + num_typeparam_args += 1; + } + + // Enter type params scope + let type_params_name = format!("<generic parameters of {name}>"); + self.push_output( + bytecode::CodeFlags::OPTIMIZED | bytecode::CodeFlags::NEWLOCALS, + 0, + num_typeparam_args as u32, + 0, + type_params_name, + )?; + + // TypeParams scope is function-like + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + // Add parameter names to varnames for the type params scope + // These will be passed as arguments when the closure is called + let current_info = self.current_code_info(); + if funcflags.contains(&bytecode::MakeFunctionFlag::Defaults) { + current_info + .metadata + .varnames + .insert(".defaults".to_owned()); + } + if funcflags.contains(&bytecode::MakeFunctionFlag::KwOnlyDefaults) { + current_info + .metadata + .varnames + .insert(".kwdefaults".to_owned()); + } + + // Compile type parameters + self.compile_type_params(type_params.unwrap())?; + + // Load defaults/kwdefaults with LOAD_FAST + for i in 0..num_typeparam_args { + let var_num = oparg::VarNum::from(i as u32); + emit!(self, Instruction::LoadFast { var_num }); + } + } + + // Compile annotations as closure (PEP 649) + let mut annotations_flag = bytecode::MakeFunctionFlags::new(); + if self.compile_annotations_closure(name, parameters, returns)? { + annotations_flag.insert(bytecode::MakeFunctionFlag::Annotate); + } + + // Compile function body + let final_funcflags = funcflags | annotations_flag; + self.compile_function_body(name, parameters, body, is_async, final_funcflags)?; + + // Handle type params if present + if is_generic { + // SWAP to get function on top + // Stack: [type_params_tuple, function] -> [function, type_params_tuple] + emit!(self, Instruction::Swap { i: 2 }); + + // Call INTRINSIC_SET_FUNCTION_TYPE_PARAMS + emit!( + self, + Instruction::CallIntrinsic2 { + func: bytecode::IntrinsicFunction2::SetFunctionTypeParams, + } + ); + + // Return the function object from type params scope + emit!(self, Instruction::ReturnValue); + + // Set argcount for type params scope + self.current_code_info().metadata.argcount = num_typeparam_args as u32; + + // Exit type params scope and create closure + let type_params_code = self.exit_scope(); + self.ctx = saved_ctx; + + // Make closure for type params code + self.make_closure(type_params_code, bytecode::MakeFunctionFlags::new())?; + + // Call the type params closure with defaults/kwdefaults as arguments. + // Call protocol: [callable, self_or_null, arg1, ..., argN] + // We need to reorder: [args..., closure] -> [closure, NULL, args...] + // Using Swap operations to move closure down and insert NULL. + // Note: num_typeparam_args is at most 2 (defaults tuple, kwdefaults dict). + if num_typeparam_args > 0 { + match num_typeparam_args { + 1 => { + // Stack: [arg1, closure] + emit!(self, Instruction::Swap { i: 2 }); // [closure, arg1] + emit!(self, Instruction::PushNull); // [closure, arg1, NULL] + emit!(self, Instruction::Swap { i: 2 }); // [closure, NULL, arg1] + } + 2 => { + // Stack: [arg1, arg2, closure] + emit!(self, Instruction::Swap { i: 3 }); // [closure, arg2, arg1] + emit!(self, Instruction::Swap { i: 2 }); // [closure, arg1, arg2] + emit!(self, Instruction::PushNull); // [closure, arg1, arg2, NULL] + emit!(self, Instruction::Swap { i: 3 }); // [closure, NULL, arg2, arg1] + emit!(self, Instruction::Swap { i: 2 }); // [closure, NULL, arg1, arg2] + } + _ => unreachable!("only defaults and kwdefaults are supported"), + } + emit!( + self, + Instruction::Call { + argc: num_typeparam_args as u32 + } + ); + } else { + // Stack: [closure] + emit!(self, Instruction::PushNull); + // Stack: [closure, NULL] + emit!(self, Instruction::Call { argc: 0 }); + } + } + + // Apply decorators + self.apply_decorators(decorator_list); + + // Store the function + self.store_name(name)?; + + Ok(()) + } + + /// Determines if a variable should be CELL or FREE type + // = get_ref_type + fn get_ref_type(&self, name: &str) -> Result<SymbolScope, CodegenErrorType> { + let table = self.symbol_table_stack.last().unwrap(); + + // Special handling for __class__, __classdict__, and __conditional_annotations__ in class scope + // This should only apply when we're actually IN a class body, + // not when we're in a method nested inside a class. + if table.typ == CompilerScope::Class + && (name == "__class__" + || name == "__classdict__" + || name == "__conditional_annotations__") + { + return Ok(SymbolScope::Cell); + } + match table.lookup(name) { + Some(symbol) => match symbol.scope { + SymbolScope::Cell => Ok(SymbolScope::Cell), + SymbolScope::Free => Ok(SymbolScope::Free), + _ if symbol.flags.contains(SymbolFlags::FREE_CLASS) => Ok(SymbolScope::Free), + _ => Err(CodegenErrorType::SyntaxError(format!( + "get_ref_type: invalid scope for '{name}'" + ))), + }, + None => Err(CodegenErrorType::SyntaxError(format!( + "get_ref_type: cannot find symbol '{name}'" + ))), + } + } + + /// Loads closure variables if needed and creates a function object + // = compiler_make_closure + fn make_closure( + &mut self, + code: CodeObject, + flags: bytecode::MakeFunctionFlags, + ) -> CompileResult<()> { + // Handle free variables (closure) + let has_freevars = !code.freevars.is_empty(); + if has_freevars { + // Build closure tuple by loading free variables + + for var in &code.freevars { + // Special case: If a class contains a method with a + // free variable that has the same name as a method, + // the name will be considered free *and* local in the + // class. It should be handled by the closure, as + // well as by the normal name lookup logic. + + // Get reference type using our get_ref_type function + let ref_type = self.get_ref_type(var).map_err(|e| self.error(e))?; + + // Get parent code info + let parent_code = self.code_stack.last().unwrap(); + let cellvars_len = parent_code.metadata.cellvars.len(); + + // Look up the variable index based on reference type + let idx = match ref_type { + SymbolScope::Cell => parent_code + .metadata + .cellvars + .get_index_of(var) + .or_else(|| { + parent_code + .metadata + .freevars + .get_index_of(var) + .map(|i| i + cellvars_len) + }) + .ok_or_else(|| { + self.error(CodegenErrorType::SyntaxError(format!( + "compiler_make_closure: cannot find '{var}' in parent vars", + ))) + })?, + SymbolScope::Free => parent_code + .metadata + .freevars + .get_index_of(var) + .map(|i| i + cellvars_len) + .or_else(|| parent_code.metadata.cellvars.get_index_of(var)) + .ok_or_else(|| { + self.error(CodegenErrorType::SyntaxError(format!( + "compiler_make_closure: cannot find '{var}' in parent vars", + ))) + })?, + _ => { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "compiler_make_closure: unexpected ref_type {ref_type:?} for '{var}'", + )))); + } + }; + + emit!(self, PseudoInstruction::LoadClosure { i: idx.to_u32() }); + } + + // Build tuple of closure variables + emit!( + self, + Instruction::BuildTuple { + count: code.freevars.len().to_u32(), + } + ); + } + + // load code object and create function + self.emit_load_const(ConstantData::Code { + code: Box::new(code), + }); + + // Create function with no flags + emit!(self, Instruction::MakeFunction); + + // Now set attributes one by one using SET_FUNCTION_ATTRIBUTE + // Note: The order matters! Values must be on stack before calling SET_FUNCTION_ATTRIBUTE + + // Set closure if needed + if has_freevars { + emit!( + self, + Instruction::SetFunctionAttribute { + flag: bytecode::MakeFunctionFlag::Closure + } + ); + } + + // Set annotations if present + if flags.contains(&bytecode::MakeFunctionFlag::Annotations) { + emit!( + self, + Instruction::SetFunctionAttribute { + flag: bytecode::MakeFunctionFlag::Annotations + } + ); + } + + // Set __annotate__ closure if present (PEP 649) + if flags.contains(&bytecode::MakeFunctionFlag::Annotate) { + emit!( + self, + Instruction::SetFunctionAttribute { + flag: bytecode::MakeFunctionFlag::Annotate + } + ); + } + + // Set kwdefaults if present + if flags.contains(&bytecode::MakeFunctionFlag::KwOnlyDefaults) { + emit!( + self, + Instruction::SetFunctionAttribute { + flag: bytecode::MakeFunctionFlag::KwOnlyDefaults + } + ); + } + + // Set defaults if present + if flags.contains(&bytecode::MakeFunctionFlag::Defaults) { + emit!( + self, + Instruction::SetFunctionAttribute { + flag: bytecode::MakeFunctionFlag::Defaults + } + ); + } + + // Set type_params if present + if flags.contains(&bytecode::MakeFunctionFlag::TypeParams) { + emit!( + self, + Instruction::SetFunctionAttribute { + flag: bytecode::MakeFunctionFlag::TypeParams + } + ); + } + + Ok(()) + } + + // Python/compile.c find_ann + fn find_ann(body: &[ast::Stmt]) -> bool { + for statement in body { + let res = match &statement { + ast::Stmt::AnnAssign(_) => true, + ast::Stmt::For(ast::StmtFor { body, orelse, .. }) => { + Self::find_ann(body) || Self::find_ann(orelse) + } + ast::Stmt::If(ast::StmtIf { + body, + elif_else_clauses, + .. + }) => { + Self::find_ann(body) + || elif_else_clauses.iter().any(|x| Self::find_ann(&x.body)) + } + ast::Stmt::While(ast::StmtWhile { body, orelse, .. }) => { + Self::find_ann(body) || Self::find_ann(orelse) + } + ast::Stmt::With(ast::StmtWith { body, .. }) => Self::find_ann(body), + ast::Stmt::Match(ast::StmtMatch { cases, .. }) => { + cases.iter().any(|case| Self::find_ann(&case.body)) + } + ast::Stmt::Try(ast::StmtTry { + body, + handlers, + orelse, + finalbody, + .. + }) => { + Self::find_ann(body) + || handlers.iter().any(|h| { + let ast::ExceptHandler::ExceptHandler( + ast::ExceptHandlerExceptHandler { body, .. }, + ) = h; + Self::find_ann(body) + }) + || Self::find_ann(orelse) + || Self::find_ann(finalbody) + } + _ => false, + }; + if res { + return true; + } + } + false + } + + /// Compile the class body into a code object + // = compiler_class_body + fn compile_class_body( + &mut self, + name: &str, + body: &[ast::Stmt], + type_params: Option<&ast::TypeParams>, + firstlineno: u32, + ) -> CompileResult<CodeObject> { + // 1. Enter class scope + let key = self.symbol_table_stack.len(); + self.push_symbol_table()?; + self.enter_scope(name, CompilerScope::Class, key, firstlineno)?; + + // Set qualname using the new method + let qualname = self.set_qualname(); + + // For class scopes, set u_private to the class name for name mangling + self.code_stack.last_mut().unwrap().private = Some(name.to_owned()); + + // 2. Set up class namespace + let (doc_str, body) = split_doc(body, &self.opts); + + // Load (global) __name__ and store as __module__ + let dunder_name = self.name("__name__"); + self.emit_load_global(dunder_name, false); + let dunder_module = self.name("__module__"); + emit!( + self, + Instruction::StoreName { + namei: dunder_module + } + ); + + // Store __qualname__ + self.emit_load_const(ConstantData::Str { + value: qualname.into(), + }); + let qualname_name = self.name("__qualname__"); + emit!( + self, + Instruction::StoreName { + namei: qualname_name + } + ); + + // Store __doc__ only if there's an explicit docstring + if let Some(doc) = doc_str { + self.emit_load_const(ConstantData::Str { value: doc.into() }); + let doc_name = self.name("__doc__"); + emit!(self, Instruction::StoreName { namei: doc_name }); + } + + // Store __firstlineno__ (new in Python 3.12+) + self.emit_load_const(ConstantData::Integer { + value: BigInt::from(firstlineno), + }); + let firstlineno_name = self.name("__firstlineno__"); + emit!( + self, + Instruction::StoreName { + namei: firstlineno_name + } + ); + + // Set __type_params__ if we have type parameters + if type_params.is_some() { + // Load .type_params from enclosing scope + let dot_type_params = self.name(".type_params"); + emit!( + self, + Instruction::LoadName { + namei: dot_type_params + } + ); + + // Store as __type_params__ + let dunder_type_params = self.name("__type_params__"); + emit!( + self, + Instruction::StoreName { + namei: dunder_type_params + } + ); + } + + // PEP 649: Initialize __classdict__ cell for class annotation scope + if self.current_symbol_table().needs_classdict { + emit!(self, Instruction::LoadLocals); + let classdict_idx = self.get_cell_var_index("__classdict__")?; + emit!(self, Instruction::StoreDeref { i: classdict_idx }); + } + + // Handle class annotations based on future_annotations flag + if Self::find_ann(body) { + if self.future_annotations { + // PEP 563: Initialize __annotations__ dict for class + emit!(self, Instruction::SetupAnnotations); + } else { + // PEP 649: Initialize __conditional_annotations__ set if needed for class + if self.current_symbol_table().has_conditional_annotations { + emit!(self, Instruction::BuildSet { count: 0 }); + self.store_name("__conditional_annotations__")?; + } + + // PEP 649: Generate __annotate__ function for class annotations + self.compile_module_annotate(body)?; + } + } + + // 3. Compile the class body + self.compile_statements(body)?; + + // 4. Handle __classcell__ if needed + let classcell_idx = self + .code_stack + .last_mut() + .unwrap() + .metadata + .cellvars + .iter() + .position(|var| *var == "__class__"); + + if let Some(classcell_idx) = classcell_idx { + emit!( + self, + PseudoInstruction::LoadClosure { + i: classcell_idx.to_u32() + } + ); + emit!(self, Instruction::Copy { i: 1 }); + let classcell = self.name("__classcell__"); + emit!(self, Instruction::StoreName { namei: classcell }); + } else { + self.emit_load_const(ConstantData::None); + } + + // Return the class namespace + self.emit_return_value(); + + // Exit scope and return the code object + Ok(self.exit_scope()) + } + + fn compile_class_def( + &mut self, + name: &str, + body: &[ast::Stmt], + decorator_list: &[ast::Decorator], + type_params: Option<&ast::TypeParams>, + arguments: Option<&ast::Arguments>, + ) -> CompileResult<()> { + self.prepare_decorators(decorator_list)?; + + let is_generic = type_params.is_some(); + let firstlineno = self.get_source_line_number().get().to_u32(); + + // Save context before entering any scopes + let saved_ctx = self.ctx; + + // Step 1: If generic, enter type params scope and compile type params + if is_generic { + let type_params_name = format!("<generic parameters of {name}>"); + self.push_output( + bytecode::CodeFlags::OPTIMIZED | bytecode::CodeFlags::NEWLOCALS, + 0, + 0, + 0, + type_params_name, + )?; + + // Set private name for name mangling + self.code_stack.last_mut().unwrap().private = Some(name.to_owned()); + + // TypeParams scope is function-like + self.ctx = CompileContext { + loop_data: None, + in_class: saved_ctx.in_class, + func: FunctionContext::Function, + in_async_scope: false, + }; + + // Compile type parameters and store as .type_params + self.compile_type_params(type_params.unwrap())?; + let dot_type_params = self.name(".type_params"); + emit!( + self, + Instruction::StoreName { + namei: dot_type_params + } + ); + } + + // Step 2: Compile class body (always done, whether generic or not) + let prev_ctx = self.ctx; + self.ctx = CompileContext { + func: FunctionContext::NoFunction, + in_class: true, + loop_data: None, + in_async_scope: false, + }; + let class_code = self.compile_class_body(name, body, type_params, firstlineno)?; + self.ctx = prev_ctx; + + // Step 3: Generate the rest of the code for the call + if is_generic { + // Still in type params scope + let dot_type_params = self.name(".type_params"); + let dot_generic_base = self.name(".generic_base"); + + // Create .generic_base + emit!( + self, + Instruction::LoadName { + namei: dot_type_params + } + ); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::SubscriptGeneric + } + ); + emit!( + self, + Instruction::StoreName { + namei: dot_generic_base + } + ); + + // Generate class creation code + emit!(self, Instruction::LoadBuildClass); + emit!(self, Instruction::PushNull); + + // Set up the class function with type params + let mut func_flags = bytecode::MakeFunctionFlags::new(); + emit!( + self, + Instruction::LoadName { + namei: dot_type_params + } + ); + func_flags.insert(bytecode::MakeFunctionFlag::TypeParams); + + // Create class function with closure + self.make_closure(class_code, func_flags)?; + self.emit_load_const(ConstantData::Str { value: name.into() }); + + // Compile bases and call __build_class__ + // Check for starred bases or **kwargs + let has_starred = arguments.is_some_and(|args| { + args.args + .iter() + .any(|arg| matches!(arg, ast::Expr::Starred(_))) + }); + let has_double_star = + arguments.is_some_and(|args| args.keywords.iter().any(|kw| kw.arg.is_none())); + + if has_starred || has_double_star { + // Use CallFunctionEx for *bases or **kwargs + // Stack has: [__build_class__, NULL, class_func, name] + // Need to build: args tuple = (class_func, name, *bases, .generic_base) + + // Build a list starting with class_func and name (2 elements already on stack) + emit!(self, Instruction::BuildList { count: 2 }); + + // Add bases to the list + if let Some(arguments) = arguments { + for arg in &arguments.args { + if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = arg { + // Starred: compile and extend + self.compile_expression(value)?; + emit!(self, Instruction::ListExtend { i: 0 }); + } else { + // Non-starred: compile and append + self.compile_expression(arg)?; + emit!(self, Instruction::ListAppend { i: 0 }); + } + } + } + + // Add .generic_base as final element + emit!( + self, + Instruction::LoadName { + namei: dot_generic_base + } + ); + emit!(self, Instruction::ListAppend { i: 0 }); + + // Convert list to tuple + emit!( + self, + Instruction::CallIntrinsic1 { + func: IntrinsicFunction1::ListToTuple + } + ); + + // Build kwargs if needed + if arguments.is_some_and(|args| !args.keywords.is_empty()) { + self.compile_keywords(&arguments.unwrap().keywords)?; + } else { + emit!(self, Instruction::PushNull); + } + emit!(self, Instruction::CallFunctionEx); + } else { + // Simple case: no starred bases, no **kwargs + // Compile bases normally + let base_count = if let Some(arguments) = arguments { + for arg in &arguments.args { + self.compile_expression(arg)?; + } + arguments.args.len() + } else { + 0 + }; + + // Load .generic_base as the last base + emit!( + self, + Instruction::LoadName { + namei: dot_generic_base + } + ); + + let nargs = 2 + u32::try_from(base_count).expect("too many base classes") + 1; + + // Handle keyword arguments (no **kwargs here) + if let Some(arguments) = arguments + && !arguments.keywords.is_empty() + { + let mut kwarg_names = vec![]; + for keyword in &arguments.keywords { + let name = keyword.arg.as_ref().expect( + "keyword argument name must be set (no **kwargs in this branch)", + ); + kwarg_names.push(ConstantData::Str { + value: name.as_str().into(), + }); + self.compile_expression(&keyword.value)?; + } + self.emit_load_const(ConstantData::Tuple { + elements: kwarg_names, + }); + emit!( + self, + Instruction::CallKw { + argc: nargs + + u32::try_from(arguments.keywords.len()) + .expect("too many keyword arguments") + } + ); + } else { + emit!(self, Instruction::Call { argc: nargs }); + } + } + + // Return the created class + self.emit_return_value(); + + // Exit type params scope and wrap in function + let type_params_code = self.exit_scope(); + self.ctx = saved_ctx; + + // Execute the type params function + self.make_closure(type_params_code, bytecode::MakeFunctionFlags::new())?; + emit!(self, Instruction::PushNull); + emit!(self, Instruction::Call { argc: 0 }); + } else { + // Non-generic class: standard path + emit!(self, Instruction::LoadBuildClass); + emit!(self, Instruction::PushNull); + + // Create class function with closure + self.make_closure(class_code, bytecode::MakeFunctionFlags::new())?; + self.emit_load_const(ConstantData::Str { value: name.into() }); + + if let Some(arguments) = arguments { + self.codegen_call_helper(2, arguments, self.current_source_range)?; + } else { + emit!(self, Instruction::Call { argc: 2 }); + } + } + + // Step 4: Apply decorators and store (common to both paths) + self.apply_decorators(decorator_list); + self.store_name(name) + } + + fn compile_while( + &mut self, + test: &ast::Expr, + body: &[ast::Stmt], + orelse: &[ast::Stmt], + ) -> CompileResult<()> { + self.enter_conditional_block(); + + let while_block = self.new_block(); + let else_block = self.new_block(); + let after_block = self.new_block(); + + // Note: SetupLoop is no longer emitted (break/continue use direct jumps) + self.switch_to_block(while_block); + + // Push fblock for while loop + self.push_fblock(FBlockType::WhileLoop, while_block, after_block)?; + + self.compile_jump_if(test, false, else_block)?; + + let was_in_loop = self.ctx.loop_data.replace((while_block, after_block)); + self.compile_statements(body)?; + self.ctx.loop_data = was_in_loop; + emit!(self, PseudoInstruction::Jump { delta: while_block }); + self.switch_to_block(else_block); + + // Pop fblock + self.pop_fblock(FBlockType::WhileLoop); + // Note: PopBlock is no longer emitted for loops + self.compile_statements(orelse)?; + self.switch_to_block(after_block); + + self.leave_conditional_block(); + Ok(()) + } + + fn compile_with( + &mut self, + items: &[ast::WithItem], + body: &[ast::Stmt], + is_async: bool, + ) -> CompileResult<()> { + self.enter_conditional_block(); + + // Python 3.12+ style with statement: + // + // BEFORE_WITH # TOS: ctx_mgr -> [__exit__, __enter__ result] + // L1: STORE_NAME f # exception table: L1 to L2 -> L3 [1] lasti + // L2: ... body ... + // LOAD_CONST None # normal exit + // LOAD_CONST None + // LOAD_CONST None + // CALL 2 # __exit__(None, None, None) + // POP_TOP + // JUMP after + // L3: PUSH_EXC_INFO # exception handler + // WITH_EXCEPT_START # call __exit__(type, value, tb), push result + // TO_BOOL + // POP_JUMP_IF_TRUE suppress + // RERAISE 2 + // suppress: + // POP_TOP # pop exit result + // L5: POP_EXCEPT + // POP_TOP # pop __exit__ + // POP_TOP # pop prev_exc (or lasti depending on layout) + // JUMP after + // L6: COPY 3 # cleanup handler for reraise + // POP_EXCEPT + // RERAISE 1 + // after: ... + + let with_range = self.current_source_range; + + let Some((item, items)) = items.split_first() else { + return Err(self.error(CodegenErrorType::EmptyWithItems)); + }; + + let exc_handler_block = self.new_block(); + let after_block = self.new_block(); + + // Compile context expression and load __enter__/__exit__ methods + self.compile_expression(&item.context_expr)?; + self.set_source_range(with_range); + + // Stack: [cm] + emit!(self, Instruction::Copy { i: 1 }); // [cm, cm] + + if is_async { + if self.ctx.func != FunctionContext::AsyncFunction { + return Err(self.error(CodegenErrorType::InvalidAsyncWith)); + } + // Load __aexit__ and __aenter__, then call __aenter__ + emit!( + self, + Instruction::LoadSpecial { + method: SpecialMethod::AExit + } + ); // [cm, bound_aexit] + emit!(self, Instruction::Swap { i: 2 }); // [bound_aexit, cm] + emit!( + self, + Instruction::LoadSpecial { + method: SpecialMethod::AEnter + } + ); // [bound_aexit, bound_aenter] + // bound_aenter is already bound, call with NULL self_or_null + emit!(self, Instruction::PushNull); // [bound_aexit, bound_aenter, NULL] + emit!(self, Instruction::Call { argc: 0 }); // [bound_aexit, awaitable] + emit!(self, Instruction::GetAwaitable { r#where: 1 }); + self.emit_load_const(ConstantData::None); + let _ = self.compile_yield_from_sequence(true)?; + } else { + // Load __exit__ and __enter__, then call __enter__ + emit!( + self, + Instruction::LoadSpecial { + method: SpecialMethod::Exit + } + ); // [cm, bound_exit] + emit!(self, Instruction::Swap { i: 2 }); // [bound_exit, cm] + emit!( + self, + Instruction::LoadSpecial { + method: SpecialMethod::Enter + } + ); // [bound_exit, bound_enter] + // bound_enter is already bound, call with NULL self_or_null + emit!(self, Instruction::PushNull); // [bound_exit, bound_enter, NULL] + emit!(self, Instruction::Call { argc: 0 }); // [bound_exit, enter_result] + } + + // Stack: [..., __exit__, enter_result] + // Push fblock for exception table - handler goes to exc_handler_block + // preserve_lasti=true for with statements + emit!( + self, + PseudoInstruction::SetupWith { + delta: exc_handler_block + } + ); + self.push_fblock( + if is_async { + FBlockType::AsyncWith + } else { + FBlockType::With + }, + exc_handler_block, // block start (will become exit target after store) + after_block, + )?; + + // Store or pop the enter result + match &item.optional_vars { + Some(var) => { + self.set_source_range(var.range()); + self.compile_store(var)?; + } + None => { + emit!(self, Instruction::PopTop); + } + } + // Stack: [..., __exit__] + + // Compile body or nested with + if items.is_empty() { + if body.is_empty() { + return Err(self.error(CodegenErrorType::EmptyWithBody)); + } + self.compile_statements(body)?; + } else { + self.set_source_range(with_range); + self.compile_with(items, body, is_async)?; + } + + // Pop fblock before normal exit + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(if is_async { + FBlockType::AsyncWith + } else { + FBlockType::With + }); + + // ===== Normal exit path ===== + // Stack: [..., __exit__] + // Call __exit__(None, None, None) + self.set_source_range(with_range); + emit!(self, Instruction::PushNull); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + emit!(self, Instruction::Call { argc: 3 }); + if is_async { + emit!(self, Instruction::GetAwaitable { r#where: 2 }); + self.emit_load_const(ConstantData::None); + let _ = self.compile_yield_from_sequence(true)?; + } + emit!(self, Instruction::PopTop); // Pop __exit__ result + emit!(self, PseudoInstruction::Jump { delta: after_block }); + + // ===== Exception handler path ===== + // Stack at entry (after unwind): [..., __exit__, lasti, exc] + // PUSH_EXC_INFO -> [..., __exit__, lasti, prev_exc, exc] + self.switch_to_block(exc_handler_block); + + // Create blocks for exception handling + let cleanup_block = self.new_block(); + let suppress_block = self.new_block(); + + emit!( + self, + PseudoInstruction::SetupCleanup { + delta: cleanup_block + } + ); + self.push_fblock(FBlockType::ExceptionHandler, exc_handler_block, after_block)?; + + // PUSH_EXC_INFO: [exc] -> [prev_exc, exc] + emit!(self, Instruction::PushExcInfo); + + // WITH_EXCEPT_START: call __exit__(type, value, tb) + // Stack: [..., __exit__, lasti, prev_exc, exc] + // __exit__ is at TOS-3, call with exception info + emit!(self, Instruction::WithExceptStart); + + if is_async { + emit!(self, Instruction::GetAwaitable { r#where: 2 }); + self.emit_load_const(ConstantData::None); + let _ = self.compile_yield_from_sequence(true)?; + } + + // TO_BOOL + POP_JUMP_IF_TRUE: check if exception is suppressed + emit!(self, Instruction::ToBool); + emit!( + self, + Instruction::PopJumpIfTrue { + delta: suppress_block + } + ); + + // Pop the nested fblock BEFORE RERAISE so that RERAISE's exception + // handler points to the outer handler (try-except), not cleanup_block. + // This is critical: when RERAISE propagates the exception, the exception + // table should route it to the outer try-except, not back to cleanup. + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::ExceptionHandler); + + // Not suppressed: RERAISE 2 + emit!(self, Instruction::Reraise { depth: 2 }); + + // ===== Suppress block ===== + // Exception was suppressed, clean up stack + // Stack: [..., __exit__, lasti, prev_exc, exc, True] + // Need to pop: True, exc, prev_exc, __exit__ + self.switch_to_block(suppress_block); + emit!(self, Instruction::PopTop); // pop True (TO_BOOL result) + emit!(self, Instruction::PopExcept); // pop exc and restore prev_exc + emit!(self, Instruction::PopTop); // pop __exit__ + emit!(self, Instruction::PopTop); // pop lasti + emit!(self, PseudoInstruction::Jump { delta: after_block }); + + // ===== Cleanup block (for nested exception during __exit__) ===== + // Stack: [..., __exit__, lasti, prev_exc, lasti2, exc2] + // COPY 3: copy prev_exc to TOS + // POP_EXCEPT: restore exception state + // RERAISE 1: re-raise with lasti + // + // NOTE: We DON'T clear the fblock stack here because we want + // outer exception handlers (e.g., try-except wrapping this with statement) + // to be in the exception table for these instructions. + // If we cleared fblock, exceptions here would propagate uncaught. + self.switch_to_block(cleanup_block); + emit!(self, Instruction::Copy { i: 3 }); + emit!(self, Instruction::PopExcept); + emit!(self, Instruction::Reraise { depth: 1 }); + + // ===== After block ===== + self.switch_to_block(after_block); + + self.leave_conditional_block(); + Ok(()) + } + + fn compile_for( + &mut self, + target: &ast::Expr, + iter: &ast::Expr, + body: &[ast::Stmt], + orelse: &[ast::Stmt], + is_async: bool, + ) -> CompileResult<()> { + self.enter_conditional_block(); + + // Start loop + let for_block = self.new_block(); + let else_block = self.new_block(); + let after_block = self.new_block(); + let mut end_async_for_target = BlockIdx::NULL; + + // The thing iterated: + self.compile_expression(iter)?; + + if is_async { + if self.ctx.func != FunctionContext::AsyncFunction { + return Err(self.error(CodegenErrorType::InvalidAsyncFor)); + } + emit!(self, Instruction::GetAIter); + + self.switch_to_block(for_block); + + // codegen_async_for: push fblock BEFORE SETUP_FINALLY + self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; + + // SETUP_FINALLY to guard the __anext__ call + emit!(self, PseudoInstruction::SetupFinally { delta: else_block }); + emit!(self, Instruction::GetANext); + self.emit_load_const(ConstantData::None); + end_async_for_target = self.compile_yield_from_sequence(true)?; + // POP_BLOCK for SETUP_FINALLY - only GetANext/yield_from are protected + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::NotTaken); + + // Success block for __anext__ + self.compile_store(target)?; + } else { + // Retrieve Iterator + emit!(self, Instruction::GetIter); + + self.switch_to_block(for_block); + + // Push fblock for for loop + self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; + + emit!(self, Instruction::ForIter { delta: else_block }); + + // Start of loop iteration, set targets: + self.compile_store(target)?; + }; + + let was_in_loop = self.ctx.loop_data.replace((for_block, after_block)); + self.compile_statements(body)?; + self.ctx.loop_data = was_in_loop; + emit!(self, PseudoInstruction::Jump { delta: for_block }); + + self.switch_to_block(else_block); + + // Except block for __anext__ / end of sync for + // No PopBlock here - for async, POP_BLOCK is already in for_block + self.pop_fblock(FBlockType::ForLoop); + + // End-of-loop instructions are on the `for` line, not the body's last line + let saved_range = self.current_source_range; + self.set_source_range(iter.range()); + if is_async { + self.emit_end_async_for(end_async_for_target); + } else { + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); + } + self.set_source_range(saved_range); + self.compile_statements(orelse)?; + + self.switch_to_block(after_block); + + // Implicit return after for-loop should be attributed to the `for` line + self.set_source_range(iter.range()); + + self.leave_conditional_block(); + Ok(()) + } + + fn forbidden_name(&mut self, name: &str, ctx: NameUsage) -> CompileResult<bool> { + if ctx == NameUsage::Store && name == "__debug__" { + return Err(self.error(CodegenErrorType::Assign("__debug__"))); + // return Ok(true); + } + if ctx == NameUsage::Delete && name == "__debug__" { + return Err(self.error(CodegenErrorType::Delete("__debug__"))); + // return Ok(true); + } + Ok(false) + } + + fn compile_error_forbidden_name(&mut self, name: &str) -> CodegenError { + self.error(CodegenErrorType::SyntaxError(format!( + "cannot use forbidden name '{name}' in pattern" + ))) + } + + /// Ensures that `pc.fail_pop` has at least `n + 1` entries. + /// If not, new labels are generated and pushed until the required size is reached. + fn ensure_fail_pop(&mut self, pc: &mut PatternContext, n: usize) -> CompileResult<()> { + let required_size = n + 1; + if required_size <= pc.fail_pop.len() { + return Ok(()); + } + while pc.fail_pop.len() < required_size { + let new_block = self.new_block(); + pc.fail_pop.push(new_block); + } + Ok(()) + } + + fn jump_to_fail_pop(&mut self, pc: &mut PatternContext, op: JumpOp) -> CompileResult<()> { + // Compute the total number of items to pop: + // items on top plus the captured objects. + let pops = pc.on_top + pc.stores.len(); + // Ensure that the fail_pop vector has at least `pops + 1` elements. + self.ensure_fail_pop(pc, pops)?; + // Emit a jump using the jump target stored at index `pops`. + match op { + JumpOp::Jump => { + emit!( + self, + PseudoInstruction::Jump { + delta: pc.fail_pop[pops] + } + ); + } + JumpOp::PopJumpIfFalse => { + emit!( + self, + Instruction::PopJumpIfFalse { + delta: pc.fail_pop[pops] + } + ); + } + } + Ok(()) + } + + /// Emits the necessary POP instructions for all failure targets in the pattern context, + /// then resets the fail_pop vector. + fn emit_and_reset_fail_pop(&mut self, pc: &mut PatternContext) -> CompileResult<()> { + // If the fail_pop vector is empty, nothing needs to be done. + if pc.fail_pop.is_empty() { + debug_assert!(pc.fail_pop.is_empty()); + return Ok(()); + } + // Iterate over the fail_pop vector in reverse order, skipping the first label. + for &label in pc.fail_pop.iter().skip(1).rev() { + self.switch_to_block(label); + // Emit the POP instruction. + emit!(self, Instruction::PopTop); + } + // Finally, use the first label. + self.switch_to_block(pc.fail_pop[0]); + pc.fail_pop.clear(); + // Free the memory used by the vector. + pc.fail_pop.shrink_to_fit(); + Ok(()) + } + + /// Duplicate the effect of Python 3.10's ROT_* instructions using SWAPs. + fn pattern_helper_rotate(&mut self, mut count: usize) -> CompileResult<()> { + // Rotate TOS (top of stack) to position `count` down + // This is done by a series of swaps + // For count=1, no rotation needed (already at top) + // For count=2, swap TOS with item 1 position down + // For count=3, swap TOS with item 2 positions down, then with item 1 position down + while count > 1 { + // Emit a SWAP instruction with the current count. + emit!( + self, + Instruction::Swap { + i: u32::try_from(count).unwrap() + } + ); + count -= 1; + } + Ok(()) + } + + /// Helper to store a captured name for a star pattern. + /// + /// If `n` is `None`, it emits a POP_TOP instruction. Otherwise, it first + /// checks that the name is allowed and not already stored. Then it rotates + /// the object on the stack beneath any preserved items and appends the name + /// to the list of captured names. + fn pattern_helper_store_name( + &mut self, + n: Option<&ast::Identifier>, + pc: &mut PatternContext, + ) -> CompileResult<()> { + match n { + // If no name is provided, simply pop the top of the stack. + None => { + emit!(self, Instruction::PopTop); + Ok(()) + } + Some(name) => { + // Check if the name is forbidden for storing. + if self.forbidden_name(name.as_str(), NameUsage::Store)? { + return Err(self.compile_error_forbidden_name(name.as_str())); + } + + // Ensure we don't store the same name twice. + // TODO: maybe pc.stores should be a set? + if pc.stores.contains(&name.to_string()) { + return Err( + self.error(CodegenErrorType::DuplicateStore(name.as_str().to_string())) + ); + } + + // Calculate how many items to rotate: + let rotations = pc.on_top + pc.stores.len() + 1; + self.pattern_helper_rotate(rotations)?; + + // Append the name to the captured stores. + pc.stores.push(name.to_string()); + Ok(()) + } + } + } + + fn pattern_unpack_helper(&mut self, elts: &[ast::Pattern]) -> CompileResult<()> { + let n = elts.len(); + let mut seen_star = false; + for (i, elt) in elts.iter().enumerate() { + if elt.is_match_star() { + if !seen_star { + if i >= (1 << 8) || (n - i - 1) >= ((i32::MAX as usize) >> 8) { + todo!(); + // return self.compiler_error(loc, "too many expressions in star-unpacking sequence pattern"); + } + let counts = UnpackExArgs { + before: u8::try_from(i).unwrap(), + after: u8::try_from(n - i - 1).unwrap(), + }; + emit!(self, Instruction::UnpackEx { counts }); + seen_star = true; + } else { + // TODO: Fix error msg + return Err(self.error(CodegenErrorType::MultipleStarArgs)); + // return self.compiler_error(loc, "multiple starred expressions in sequence pattern"); + } + } + } + if !seen_star { + emit!( + self, + Instruction::UnpackSequence { + count: u32::try_from(n).unwrap() + } + ); + } + Ok(()) + } + + fn pattern_helper_sequence_unpack( + &mut self, + patterns: &[ast::Pattern], + _star: Option<usize>, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Unpack the sequence into individual subjects. + self.pattern_unpack_helper(patterns)?; + let size = patterns.len(); + // Increase the on_top counter for the newly unpacked subjects. + pc.on_top += size; + // For each unpacked subject, compile its subpattern. + for pattern in patterns { + // Decrement on_top for each subject as it is consumed. + pc.on_top -= 1; + self.compile_pattern_subpattern(pattern, pc)?; + } + Ok(()) + } + + fn pattern_helper_sequence_subscr( + &mut self, + patterns: &[ast::Pattern], + star: usize, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Keep the subject around for extracting elements. + pc.on_top += 1; + for (i, pattern) in patterns.iter().enumerate() { + // if pattern.is_wildcard() { + // continue; + // } + if i == star { + // This must be a starred wildcard. + // assert!(pattern.is_star_wildcard()); + continue; + } + // Duplicate the subject. + emit!(self, Instruction::Copy { i: 1 }); + if i < star { + // For indices before the star, use a nonnegative index equal to i. + self.emit_load_const(ConstantData::Integer { value: i.into() }); + } else { + // For indices after the star, compute a nonnegative index: + // index = len(subject) - (size - i) + emit!(self, Instruction::GetLen); + self.emit_load_const(ConstantData::Integer { + value: (patterns.len() - i).into(), + }); + // Subtract to compute the correct index. + emit!( + self, + Instruction::BinaryOp { + op: BinaryOperator::Subtract + } + ); + } + // Use BINARY_OP/NB_SUBSCR to extract the element. + emit!( + self, + Instruction::BinaryOp { + op: BinaryOperator::Subscr + } + ); + // Compile the subpattern in irrefutable mode. + self.compile_pattern_subpattern(pattern, pc)?; + } + // Pop the subject off the stack. + pc.on_top -= 1; + emit!(self, Instruction::PopTop); + Ok(()) + } + + fn compile_pattern_subpattern( + &mut self, + p: &ast::Pattern, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Save the current allow_irrefutable state. + let old_allow_irrefutable = pc.allow_irrefutable; + // Temporarily allow irrefutable patterns. + pc.allow_irrefutable = true; + // Compile the pattern. + self.compile_pattern(p, pc)?; + // Restore the original state. + pc.allow_irrefutable = old_allow_irrefutable; + Ok(()) + } + + fn compile_pattern_as( + &mut self, + p: &ast::PatternMatchAs, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // If there is no sub-pattern, then it's an irrefutable match. + if p.pattern.is_none() { + if !pc.allow_irrefutable { + if let Some(_name) = p.name.as_ref() { + // TODO: This error message does not match cpython exactly + // A name capture makes subsequent patterns unreachable. + return Err(self.error(CodegenErrorType::UnreachablePattern( + PatternUnreachableReason::NameCapture, + ))); + } else { + // A wildcard makes remaining patterns unreachable. + return Err(self.error(CodegenErrorType::UnreachablePattern( + PatternUnreachableReason::Wildcard, + ))); + } + } + // If irrefutable matches are allowed, store the name (if any). + return self.pattern_helper_store_name(p.name.as_ref(), pc); + } + + // Otherwise, there is a sub-pattern. Duplicate the object on top of the stack. + pc.on_top += 1; + emit!(self, Instruction::Copy { i: 1 }); + // Compile the sub-pattern. + self.compile_pattern(p.pattern.as_ref().unwrap(), pc)?; + // After success, decrement the on_top counter. + pc.on_top -= 1; + // Store the captured name (if any). + self.pattern_helper_store_name(p.name.as_ref(), pc)?; + Ok(()) + } + + fn compile_pattern_star( + &mut self, + p: &ast::PatternMatchStar, + pc: &mut PatternContext, + ) -> CompileResult<()> { + self.pattern_helper_store_name(p.name.as_ref(), pc)?; + Ok(()) + } + + /// Validates that keyword attributes in a class pattern are allowed + /// and not duplicated. + fn validate_kwd_attrs( + &mut self, + attrs: &[ast::Identifier], + _patterns: &[ast::Pattern], + ) -> CompileResult<()> { + let n_attrs = attrs.len(); + for i in 0..n_attrs { + let attr = attrs[i].as_str(); + // Check if the attribute name is forbidden in a Store context. + if self.forbidden_name(attr, NameUsage::Store)? { + // Return an error if the name is forbidden. + return Err(self.compile_error_forbidden_name(attr)); + } + // Check for duplicates: compare with every subsequent attribute. + for ident in attrs.iter().take(n_attrs).skip(i + 1) { + let other = ident.as_str(); + if attr == other { + return Err(self.error(CodegenErrorType::RepeatedAttributePattern)); + } + } + } + Ok(()) + } + + fn compile_pattern_class( + &mut self, + p: &ast::PatternMatchClass, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Extract components from the MatchClass pattern. + let match_class = p; + let patterns = &match_class.arguments.patterns; + + // Extract keyword attributes and patterns. + // Capacity is pre-allocated based on the number of keyword arguments. + let mut kwd_attrs = Vec::with_capacity(match_class.arguments.keywords.len()); + let mut kwd_patterns = Vec::with_capacity(match_class.arguments.keywords.len()); + for kwd in &match_class.arguments.keywords { + kwd_attrs.push(kwd.attr.clone()); + kwd_patterns.push(kwd.pattern.clone()); + } + + let nargs = patterns.len(); + let n_attrs = kwd_attrs.len(); + + // Check for too many sub-patterns. + if nargs > u32::MAX as usize || (nargs + n_attrs).saturating_sub(1) > i32::MAX as usize { + return Err(self.error(CodegenErrorType::SyntaxError( + "too many sub-patterns in class pattern".to_owned(), + ))); + } + + // Validate keyword attributes if any. + if n_attrs != 0 { + self.validate_kwd_attrs(&kwd_attrs, &kwd_patterns)?; + } + + // Compile the class expression. + self.compile_expression(&match_class.cls)?; + + // Create a new tuple of attribute names. + let mut attr_names = vec![]; + for name in &kwd_attrs { + // Py_NewRef(name) is emulated by cloning the name into a PyObject. + attr_names.push(ConstantData::Str { + value: name.as_str().to_string().into(), + }); + } + + // Emit instructions: + // 1. Load the new tuple of attribute names. + self.emit_load_const(ConstantData::Tuple { + elements: attr_names, + }); + // 2. Emit MATCH_CLASS with nargs. + emit!( + self, + Instruction::MatchClass { + count: u32::try_from(nargs).unwrap() + } + ); + // 3. Duplicate the top of the stack. + emit!(self, Instruction::Copy { i: 1 }); + // 4. Load None. + self.emit_load_const(ConstantData::None); + // 5. Compare with IS_OP 1. + emit!( + self, + Instruction::IsOp { + invert: Invert::Yes + } + ); + + // At this point the TOS is a tuple of (nargs + n_attrs) attributes (or None). + pc.on_top += 1; + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + + // Unpack the tuple into (nargs + n_attrs) items. + let total = nargs + n_attrs; + emit!( + self, + Instruction::UnpackSequence { + count: u32::try_from(total).unwrap() + } + ); + pc.on_top += total; + pc.on_top -= 1; + + // Process each sub-pattern. + for subpattern in patterns.iter().chain(kwd_patterns.iter()) { + // Check if this is a true wildcard (underscore pattern without name binding) + let is_true_wildcard = match subpattern { + ast::Pattern::MatchAs(match_as) => { + // Only consider it wildcard if both pattern and name are None (i.e., "_") + match_as.pattern.is_none() && match_as.name.is_none() + } + _ => subpattern.is_wildcard(), + }; + + // Decrement the on_top counter for each sub-pattern + pc.on_top -= 1; + + if is_true_wildcard { + emit!(self, Instruction::PopTop); + continue; // Don't compile wildcard patterns + } + + // Compile the subpattern without irrefutability checks. + self.compile_pattern_subpattern(subpattern, pc)?; + } + Ok(()) + } + + fn compile_pattern_mapping( + &mut self, + p: &ast::PatternMatchMapping, + pc: &mut PatternContext, + ) -> CompileResult<()> { + let mapping = p; + let keys = &mapping.keys; + let patterns = &mapping.patterns; + let size = keys.len(); + let star_target = &mapping.rest; + + // Validate pattern count matches key count + if keys.len() != patterns.len() { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "keys ({}) / patterns ({}) length mismatch in mapping pattern", + keys.len(), + patterns.len() + )))); + } + + // Validate rest pattern: '_' cannot be used as a rest target + if let Some(rest) = star_target + && rest.as_str() == "_" + { + return Err(self.error(CodegenErrorType::SyntaxError("invalid syntax".to_string()))); + } + + // Step 1: Check if subject is a mapping + // Stack: [subject] + pc.on_top += 1; + + emit!(self, Instruction::MatchMapping); + // Stack: [subject, is_mapping] + + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + // Stack: [subject] + + // Special case: empty pattern {} with no rest + if size == 0 && star_target.is_none() { + // If the pattern is just "{}", we're done! Pop the subject + pc.on_top -= 1; + emit!(self, Instruction::PopTop); + return Ok(()); + } + + // Length check for patterns with keys + if size > 0 { + // Check if the mapping has at least 'size' keys + emit!(self, Instruction::GetLen); + self.emit_load_const(ConstantData::Integer { value: size.into() }); + // Stack: [subject, len, size] + emit!( + self, + Instruction::CompareOp { + opname: ComparisonOperator::GreaterOrEqual + } + ); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + // Stack: [subject] + } + + // Check for overflow (INT_MAX < size - 1) + if size > (i32::MAX as usize + 1) { + return Err(self.error(CodegenErrorType::SyntaxError( + "too many sub-patterns in mapping pattern".to_string(), + ))); + } + #[allow(clippy::cast_possible_truncation, reason = "checked right before")] + let size = size as u32; + + // Step 2: If we have keys to match + if size > 0 { + // Validate and compile keys + let mut seen = IndexSet::default(); + for key in keys { + let is_attribute = matches!(key, ast::Expr::Attribute(_)); + let is_literal = matches!( + key, + ast::Expr::NumberLiteral(_) + | ast::Expr::StringLiteral(_) + | ast::Expr::BytesLiteral(_) + | ast::Expr::BooleanLiteral(_) + | ast::Expr::NoneLiteral(_) + ); + let key_repr = if is_literal { + UnparseExpr::new(key, &self.source_file).to_string() + } else if is_attribute { + String::new() + } else { + return Err(self.error(CodegenErrorType::SyntaxError( + "mapping pattern keys may only match literals and attribute lookups" + .to_string(), + ))); + }; + + if !key_repr.is_empty() && seen.contains(&key_repr) { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "mapping pattern checks duplicate key ({key_repr})" + )))); + } + if !key_repr.is_empty() { + seen.insert(key_repr); + } + + self.compile_expression(key)?; + } + } + // Stack: [subject, key1, key2, ..., key_n] + + // Build tuple of keys (empty tuple if size==0) + emit!(self, Instruction::BuildTuple { count: size }); + // Stack: [subject, keys_tuple] + + // Match keys + emit!(self, Instruction::MatchKeys); + // Stack: [subject, keys_tuple, values_or_none] + pc.on_top += 2; // subject and keys_tuple are underneath + + // Check if match succeeded + emit!(self, Instruction::Copy { i: 1 }); + // Stack: [subject, keys_tuple, values_tuple, values_tuple_copy] + + // Check if copy is None (consumes the copy like POP_JUMP_IF_NONE) + self.emit_load_const(ConstantData::None); + emit!( + self, + Instruction::IsOp { + invert: Invert::Yes + } + ); + + // Stack: [subject, keys_tuple, values_tuple, bool] + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + // Stack: [subject, keys_tuple, values_tuple] + + // Unpack values (the original values_tuple) + emit!(self, Instruction::UnpackSequence { count: size }); + // Stack after unpack: [subject, keys_tuple, ...unpacked values...] + pc.on_top += size as usize; // Unpacked size values, tuple replaced by values + pc.on_top -= 1; + + // Step 3: Process matched values + for i in 0..size { + pc.on_top -= 1; + self.compile_pattern_subpattern(&patterns[i as usize], pc)?; + } + + // After processing subpatterns, adjust on_top + // "Whatever happens next should consume the tuple of keys and the subject" + // Stack currently: [subject, keys_tuple, ...any captured values...] + pc.on_top -= 2; + + // Step 4: Handle rest pattern or cleanup + if let Some(rest_name) = star_target { + // Build rest dict for **rest pattern + // Stack: [subject, keys_tuple] + + // Build rest dict exactly + emit!(self, Instruction::BuildMap { count: 0 }); + // Stack: [subject, keys_tuple, {}] + emit!(self, Instruction::Swap { i: 3 }); + // Stack: [{}, keys_tuple, subject] + emit!(self, Instruction::DictUpdate { i: 2 }); + // Stack after DICT_UPDATE: [rest_dict, keys_tuple] + // DICT_UPDATE consumes source (subject) and leaves dict in place + + // Unpack keys and delete from rest_dict + emit!(self, Instruction::UnpackSequence { count: size }); + // Stack: [rest_dict, k1, k2, ..., kn] (if size==0, nothing pushed) + + // Delete each key from rest_dict (skipped when size==0) + // while (size) { COPY(1 + size--); SWAP(2); DELETE_SUBSCR } + let mut remaining = size; + while remaining > 0 { + // Copy rest_dict which is at position (1 + remaining) from TOS + emit!(self, Instruction::Copy { i: 1 + remaining }); + // Stack: [rest_dict, k1, ..., kn, rest_dict] + emit!(self, Instruction::Swap { i: 2 }); + // Stack: [rest_dict, k1, ..., kn-1, rest_dict, kn] + emit!(self, Instruction::DeleteSubscr); + // Stack: [rest_dict, k1, ..., kn-1] (removed kn from rest_dict) + remaining -= 1; + } + // Stack: [rest_dict] (plus any previously stored values) + // pattern_helper_store_name will handle the rotation correctly + + // Store the rest dict + self.pattern_helper_store_name(Some(rest_name), pc)?; + + // After storing all values, pc.on_top should be 0 + // The values are rotated to the bottom for later storage + pc.on_top = 0; + } else { + // Non-rest pattern: just clean up the stack + + // Pop them as we're not using them + emit!(self, Instruction::PopTop); // Pop keys_tuple + emit!(self, Instruction::PopTop); // Pop subject + } + + Ok(()) + } + + fn compile_pattern_or( + &mut self, + p: &ast::PatternMatchOr, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Ensure the pattern is a MatchOr. + let end = self.new_block(); // Create a new jump target label. + let size = p.patterns.len(); + if size <= 1 { + return Err(self.error(CodegenErrorType::SyntaxError( + "MatchOr requires at least 2 patterns".to_owned(), + ))); + } + + // Save the current pattern context. + let old_pc = pc.clone(); + // Simulate Py_INCREF on pc.stores by cloning it. + pc.stores = pc.stores.clone(); + let mut control: Option<Vec<String>> = None; // Will hold the capture list of the first alternative. + + // Process each alternative. + for (i, alt) in p.patterns.iter().enumerate() { + // Create a fresh empty store for this alternative. + pc.stores = Vec::new(); + // An irrefutable subpattern must be last (if allowed). + pc.allow_irrefutable = (i == size - 1) && old_pc.allow_irrefutable; + // Reset failure targets and the on_top counter. + pc.fail_pop.clear(); + pc.on_top = 0; + // Emit a COPY(1) instruction before compiling the alternative. + emit!(self, Instruction::Copy { i: 1 }); + self.compile_pattern(alt, pc)?; + + let n_stores = pc.stores.len(); + if i == 0 { + // Save the captured names from the first alternative. + control = Some(pc.stores.clone()); + } else { + let control_vec = control.as_ref().unwrap(); + if n_stores != control_vec.len() { + return Err(self.error(CodegenErrorType::ConflictingNameBindPattern)); + } else if n_stores > 0 { + // Check that the names occur in the same order. + for i_control in (0..n_stores).rev() { + let name = &control_vec[i_control]; + // Find the index of `name` in the current stores. + let i_stores = + pc.stores.iter().position(|n| n == name).ok_or_else(|| { + self.error(CodegenErrorType::ConflictingNameBindPattern) + })?; + if i_control != i_stores { + // The orders differ; we must reorder. + assert!(i_stores < i_control, "expected i_stores < i_control"); + let rotations = i_stores + 1; + // Rotate pc.stores: take a slice of the first `rotations` items... + let rotated = pc.stores[0..rotations].to_vec(); + // Remove those elements. + for _ in 0..rotations { + pc.stores.remove(0); + } + // Insert the rotated slice at the appropriate index. + let insert_pos = i_control - i_stores; + for (j, elem) in rotated.into_iter().enumerate() { + pc.stores.insert(insert_pos + j, elem); + } + // Also perform the same rotation on the evaluation stack. + for _ in 0..=i_stores { + self.pattern_helper_rotate(i_control + 1)?; + } + } + } + } + } + // Emit a jump to the common end label and reset any failure jump targets. + emit!(self, PseudoInstruction::Jump { delta: end }); + self.emit_and_reset_fail_pop(pc)?; + } + + // Restore the original pattern context. + *pc = old_pc.clone(); + // Simulate Py_INCREF on pc.stores. + pc.stores = pc.stores.clone(); + // In C, old_pc.fail_pop is set to NULL to avoid freeing it later. + // In Rust, old_pc is a local clone, so we need not worry about that. + + // No alternative matched: pop the subject and fail. + emit!(self, Instruction::PopTop); + self.jump_to_fail_pop(pc, JumpOp::Jump)?; + + // Use the label "end". + self.switch_to_block(end); + + // Adjust the final captures. + let n_stores = control.as_ref().unwrap().len(); + let n_rots = n_stores + 1 + pc.on_top + pc.stores.len(); + for i in 0..n_stores { + // Rotate the capture to its proper place. + self.pattern_helper_rotate(n_rots)?; + let name = &control.as_ref().unwrap()[i]; + // Check for duplicate binding. + if pc.stores.contains(name) { + return Err(self.error(CodegenErrorType::DuplicateStore(name.to_string()))); + } + pc.stores.push(name.clone()); + } + + // Old context and control will be dropped automatically. + // Finally, pop the copy of the subject. + emit!(self, Instruction::PopTop); + Ok(()) + } + + fn compile_pattern_sequence( + &mut self, + p: &ast::PatternMatchSequence, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Ensure the pattern is a MatchSequence. + let patterns = &p.patterns; // a slice of ast::Pattern + let size = patterns.len(); + let mut star: Option<usize> = None; + let mut only_wildcard = true; + let mut star_wildcard = false; + + // Find a starred pattern, if it exists. There may be at most one. + for (i, pattern) in patterns.iter().enumerate() { + if pattern.is_match_star() { + if star.is_some() { + // TODO: Fix error msg + return Err(self.error(CodegenErrorType::MultipleStarArgs)); + } + // star wildcard check + star_wildcard = pattern + .as_match_star() + .map(|m| m.name.is_none()) + .unwrap_or(false); + only_wildcard &= star_wildcard; + star = Some(i); + continue; + } + // wildcard check + only_wildcard &= pattern + .as_match_as() + .map(|m| m.name.is_none()) + .unwrap_or(false); + } + + // Keep the subject on top during the sequence and length checks. + pc.on_top += 1; + emit!(self, Instruction::MatchSequence); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + + if star.is_none() { + // No star: len(subject) == size + emit!(self, Instruction::GetLen); + self.emit_load_const(ConstantData::Integer { value: size.into() }); + emit!( + self, + Instruction::CompareOp { + opname: ComparisonOperator::Equal + } + ); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + } else if size > 1 { + // Star exists: len(subject) >= size - 1 + emit!(self, Instruction::GetLen); + self.emit_load_const(ConstantData::Integer { + value: (size - 1).into(), + }); + emit!( + self, + Instruction::CompareOp { + opname: ComparisonOperator::GreaterOrEqual + } + ); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + } + + // Whatever comes next should consume the subject. + pc.on_top -= 1; + if only_wildcard { + // ast::Patterns like: [] / [_] / [_, _] / [*_] / [_, *_] / [_, _, *_] / etc. + emit!(self, Instruction::PopTop); + } else if star_wildcard { + self.pattern_helper_sequence_subscr(patterns, star.unwrap(), pc)?; + } else { + self.pattern_helper_sequence_unpack(patterns, star, pc)?; + } + Ok(()) + } + + fn compile_pattern_value( + &mut self, + p: &ast::PatternMatchValue, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // TODO: ensure literal or attribute lookup + self.compile_expression(&p.value)?; + emit!( + self, + Instruction::CompareOp { + opname: bytecode::ComparisonOperator::Equal + } + ); + // emit!(self, Instruction::ToBool); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + Ok(()) + } + + fn compile_pattern_singleton( + &mut self, + p: &ast::PatternMatchSingleton, + pc: &mut PatternContext, + ) -> CompileResult<()> { + // Load the singleton constant value. + self.emit_load_const(match p.value { + ast::Singleton::None => ConstantData::None, + ast::Singleton::False => ConstantData::Boolean { value: false }, + ast::Singleton::True => ConstantData::Boolean { value: true }, + }); + // Compare using the "Is" operator. + emit!(self, Instruction::IsOp { invert: Invert::No }); + // Jump to the failure label if the comparison is false. + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + Ok(()) + } + + fn compile_pattern( + &mut self, + pattern_type: &ast::Pattern, + pattern_context: &mut PatternContext, + ) -> CompileResult<()> { + match &pattern_type { + ast::Pattern::MatchValue(pattern_type) => { + self.compile_pattern_value(pattern_type, pattern_context) + } + ast::Pattern::MatchSingleton(pattern_type) => { + self.compile_pattern_singleton(pattern_type, pattern_context) + } + ast::Pattern::MatchSequence(pattern_type) => { + self.compile_pattern_sequence(pattern_type, pattern_context) + } + ast::Pattern::MatchMapping(pattern_type) => { + self.compile_pattern_mapping(pattern_type, pattern_context) + } + ast::Pattern::MatchClass(pattern_type) => { + self.compile_pattern_class(pattern_type, pattern_context) + } + ast::Pattern::MatchStar(pattern_type) => { + self.compile_pattern_star(pattern_type, pattern_context) + } + ast::Pattern::MatchAs(pattern_type) => { + self.compile_pattern_as(pattern_type, pattern_context) + } + ast::Pattern::MatchOr(pattern_type) => { + self.compile_pattern_or(pattern_type, pattern_context) + } + } + } + + fn compile_match_inner( + &mut self, + subject: &ast::Expr, + cases: &[ast::MatchCase], + pattern_context: &mut PatternContext, + ) -> CompileResult<()> { + self.compile_expression(subject)?; + let end = self.new_block(); + + let num_cases = cases.len(); + assert!(num_cases > 0); + let has_default = cases.iter().last().unwrap().pattern.is_match_star() && num_cases > 1; + + let case_count = num_cases - if has_default { 1 } else { 0 }; + for (i, m) in cases.iter().enumerate().take(case_count) { + // Only copy the subject if not on the last case + if i != case_count - 1 { + emit!(self, Instruction::Copy { i: 1 }); + } + + pattern_context.stores = Vec::with_capacity(1); + pattern_context.allow_irrefutable = m.guard.is_some() || i == case_count - 1; + pattern_context.fail_pop.clear(); + pattern_context.on_top = 0; + + self.compile_pattern(&m.pattern, pattern_context)?; + assert_eq!(pattern_context.on_top, 0); + + for name in &pattern_context.stores { + self.compile_name(name, NameUsage::Store)?; + } + + if let Some(ref guard) = m.guard { + self.ensure_fail_pop(pattern_context, 0)?; + // Compile the guard expression + self.compile_expression(guard)?; + emit!(self, Instruction::ToBool); + emit!( + self, + Instruction::PopJumpIfFalse { + delta: pattern_context.fail_pop[0] + } + ); + } + + if i != case_count - 1 { + emit!(self, Instruction::PopTop); + } + + self.compile_statements(&m.body)?; + emit!(self, PseudoInstruction::Jump { delta: end }); + self.emit_and_reset_fail_pop(pattern_context)?; + } + + if has_default { + let m = &cases[num_cases - 1]; + if num_cases == 1 { + emit!(self, Instruction::PopTop); + } else { + emit!(self, Instruction::Nop); + } + if let Some(ref guard) = m.guard { + // Compile guard and jump to end if false + self.compile_expression(guard)?; + emit!(self, Instruction::Copy { i: 1 }); + emit!(self, Instruction::PopJumpIfFalse { delta: end }); + emit!(self, Instruction::PopTop); + } + self.compile_statements(&m.body)?; + } + self.switch_to_block(end); + Ok(()) + } + + fn compile_match( + &mut self, + subject: &ast::Expr, + cases: &[ast::MatchCase], + ) -> CompileResult<()> { + self.enter_conditional_block(); + let mut pattern_context = PatternContext::new(); + self.compile_match_inner(subject, cases, &mut pattern_context)?; + self.leave_conditional_block(); + Ok(()) + } + + /// [CPython `compiler_addcompare`](https://github.com/python/cpython/blob/627894459a84be3488a1789919679c997056a03c/Python/compile.c#L2880-L2924) + fn compile_addcompare(&mut self, op: &ast::CmpOp) { + use bytecode::ComparisonOperator::*; + match op { + ast::CmpOp::Eq => emit!(self, Instruction::CompareOp { opname: Equal }), + ast::CmpOp::NotEq => emit!(self, Instruction::CompareOp { opname: NotEqual }), + ast::CmpOp::Lt => emit!(self, Instruction::CompareOp { opname: Less }), + ast::CmpOp::LtE => emit!( + self, + Instruction::CompareOp { + opname: LessOrEqual + } + ), + ast::CmpOp::Gt => emit!(self, Instruction::CompareOp { opname: Greater }), + ast::CmpOp::GtE => { + emit!( + self, + Instruction::CompareOp { + opname: GreaterOrEqual + } + ) + } + ast::CmpOp::In => emit!(self, Instruction::ContainsOp { invert: Invert::No }), + ast::CmpOp::NotIn => emit!( + self, + Instruction::ContainsOp { + invert: Invert::Yes + } + ), + ast::CmpOp::Is => emit!(self, Instruction::IsOp { invert: Invert::No }), + ast::CmpOp::IsNot => emit!( + self, + Instruction::IsOp { + invert: Invert::Yes + } + ), + } + } + + /// Compile a chained comparison. + /// + /// ```py + /// a == b == c == d + /// ``` + /// + /// Will compile into (pseudo code): + /// + /// ```py + /// result = a == b + /// if result: + /// result = b == c + /// if result: + /// result = c == d + /// ``` + /// + /// # See Also + /// - [CPython `compiler_compare`](https://github.com/python/cpython/blob/627894459a84be3488a1789919679c997056a03c/Python/compile.c#L4678-L4717) + fn compile_compare( + &mut self, + left: &ast::Expr, + ops: &[ast::CmpOp], + comparators: &[ast::Expr], + ) -> CompileResult<()> { + // Save the full Compare expression range for COMPARE_OP positions + let compare_range = self.current_source_range; + let (last_op, mid_ops) = ops.split_last().unwrap(); + let (last_comparator, mid_comparators) = comparators.split_last().unwrap(); + + // initialize lhs outside of loop + self.compile_expression(left)?; + + if mid_comparators.is_empty() { + self.compile_expression(last_comparator)?; + self.set_source_range(compare_range); + self.compile_addcompare(last_op); + + return Ok(()); + } + + let cleanup = self.new_block(); + + // for all comparisons except the last (as the last one doesn't need a conditional jump) + for (op, comparator) in mid_ops.iter().zip(mid_comparators) { + self.compile_expression(comparator)?; + + // store rhs for the next comparison in chain + self.set_source_range(compare_range); + emit!(self, Instruction::Swap { i: 2 }); + emit!(self, Instruction::Copy { i: 2 }); + + self.compile_addcompare(op); + + // if comparison result is false, we break with this value; if true, try the next one. + emit!(self, Instruction::Copy { i: 1 }); + emit!(self, Instruction::PopJumpIfFalse { delta: cleanup }); + emit!(self, Instruction::PopTop); + } + + self.compile_expression(last_comparator)?; + self.set_source_range(compare_range); + self.compile_addcompare(last_op); + + let end = self.new_block(); + emit!(self, PseudoInstruction::Jump { delta: end }); + + // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. + self.switch_to_block(cleanup); + emit!(self, Instruction::Swap { i: 2 }); + emit!(self, Instruction::PopTop); + + self.switch_to_block(end); + Ok(()) + } + + fn compile_annotation(&mut self, annotation: &ast::Expr) -> CompileResult<()> { + if self.future_annotations { + self.emit_load_const(ConstantData::Str { + value: UnparseExpr::new(annotation, &self.source_file) + .to_string() + .into(), + }); + } else { + let was_in_annotation = self.in_annotation; + self.in_annotation = true; + + // Special handling for starred annotations (*Ts -> Unpack[Ts]) + let result = match annotation { + ast::Expr::Starred(ast::ExprStarred { value, .. }) => { + // *args: *Ts (where Ts is a TypeVarTuple). + // Do [annotation_value] = [*Ts]. + self.compile_expression(value)?; + emit!(self, Instruction::UnpackSequence { count: 1 }); + Ok(()) + } + _ => self.compile_expression(annotation), + }; + + self.in_annotation = was_in_annotation; + result?; + } + Ok(()) + } + + fn compile_annotated_assign( + &mut self, + target: &ast::Expr, + annotation: &ast::Expr, + value: Option<&ast::Expr>, + simple: bool, + ) -> CompileResult<()> { + // Perform the actual assignment first + if let Some(value) = value { + self.compile_expression(value)?; + self.compile_store(target)?; + } + + // If we have a simple name in module or class scope, store annotation + if simple + && !self.ctx.in_func() + && let ast::Expr::Name(ast::ExprName { id, .. }) = target + { + if self.future_annotations { + // PEP 563: Store stringified annotation directly to __annotations__ + // Compile annotation as string + self.compile_annotation(annotation)?; + // Load __annotations__ + let annotations_name = self.name("__annotations__"); + emit!( + self, + Instruction::LoadName { + namei: annotations_name + } + ); + // Load the variable name + self.emit_load_const(ConstantData::Str { + value: self.mangle(id.as_str()).into_owned().into(), + }); + // Store: __annotations__[name] = annotation + emit!(self, Instruction::StoreSubscr); + } else { + // PEP 649: Handle conditional annotations + if self.current_symbol_table().has_conditional_annotations { + // Allocate an index for every annotation when has_conditional_annotations + // This keeps indices aligned with compile_module_annotate's enumeration + let code_info = self.current_code_info(); + let annotation_index = code_info.next_conditional_annotation_index; + code_info.next_conditional_annotation_index += 1; + + // Determine if this annotation is conditional + // Module and Class scopes both need all annotations tracked + let scope_type = self.current_symbol_table().typ; + let in_conditional_block = self.current_code_info().in_conditional_block > 0; + let is_conditional = + matches!(scope_type, CompilerScope::Module | CompilerScope::Class) + || in_conditional_block; + + // Only add to __conditional_annotations__ set if actually conditional + if is_conditional { + self.load_name("__conditional_annotations__")?; + self.emit_load_const(ConstantData::Integer { + value: annotation_index.into(), + }); + emit!(self, Instruction::SetAdd { i: 0 }); + emit!(self, Instruction::PopTop); + } + } + } + } + + Ok(()) + } + + fn compile_store(&mut self, target: &ast::Expr) -> CompileResult<()> { + match &target { + ast::Expr::Name(ast::ExprName { id, .. }) => self.store_name(id.as_str())?, + ast::Expr::Subscript(ast::ExprSubscript { + value, slice, ctx, .. + }) => { + self.compile_subscript(value, slice, *ctx)?; + } + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { + self.compile_expression(value)?; + let namei = self.name(attr.as_str()); + emit!(self, Instruction::StoreAttr { namei }); + } + ast::Expr::List(ast::ExprList { elts, .. }) + | ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { + let mut seen_star = false; + + // Scan for star args: + for (i, element) in elts.iter().enumerate() { + if let ast::Expr::Starred(_) = &element { + if seen_star { + return Err(self.error(CodegenErrorType::MultipleStarArgs)); + } else { + seen_star = true; + let before = i; + let after = elts.len() - i - 1; + let (before, after) = (|| Some((before.to_u8()?, after.to_u8()?)))() + .ok_or_else(|| { + self.error_ranged( + CodegenErrorType::TooManyStarUnpack, + target.range(), + ) + })?; + let counts = bytecode::UnpackExArgs { before, after }; + emit!(self, Instruction::UnpackEx { counts }); + } + } + } + + if !seen_star { + emit!( + self, + Instruction::UnpackSequence { + count: elts.len().to_u32(), + } + ); + } + + for element in elts { + if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = &element { + self.compile_store(value)?; + } else { + self.compile_store(element)?; + } + } + } + _ => { + return Err(self.error(match target { + ast::Expr::Starred(_) => CodegenErrorType::SyntaxError( + "starred assignment target must be in a list or tuple".to_owned(), + ), + _ => CodegenErrorType::Assign(target.python_name()), + })); + } + } + + Ok(()) + } + + fn compile_augassign( + &mut self, + target: &ast::Expr, + op: &ast::Operator, + value: &ast::Expr, + ) -> CompileResult<()> { + enum AugAssignKind<'a> { + Name { id: &'a str }, + Subscript, + Attr { idx: bytecode::NameIdx }, + } + + let kind = match &target { + ast::Expr::Name(ast::ExprName { id, .. }) => { + let id = id.as_str(); + self.compile_name(id, NameUsage::Load)?; + AugAssignKind::Name { id } + } + ast::Expr::Subscript(ast::ExprSubscript { + value, + slice, + ctx: _, + .. + }) => { + // For augmented assignment, we need to load the value first + // But we can't use compile_subscript directly because we need DUP_TOP2 + self.compile_expression(value)?; + self.compile_expression(slice)?; + emit!(self, Instruction::Copy { i: 2 }); + emit!(self, Instruction::Copy { i: 2 }); + emit!( + self, + Instruction::BinaryOp { + op: BinaryOperator::Subscr + } + ); + AugAssignKind::Subscript + } + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { + let attr = attr.as_str(); + self.compile_expression(value)?; + emit!(self, Instruction::Copy { i: 1 }); + let idx = self.name(attr); + self.emit_load_attr(idx); + AugAssignKind::Attr { idx } + } + _ => { + return Err(self.error(CodegenErrorType::Assign(target.python_name()))); + } + }; + + self.compile_expression(value)?; + self.compile_op(op, true); + + match kind { + AugAssignKind::Name { id } => { + // stack: RESULT + self.compile_name(id, NameUsage::Store)?; + } + AugAssignKind::Subscript => { + // stack: CONTAINER SLICE RESULT + emit!(self, Instruction::Swap { i: 3 }); + emit!(self, Instruction::Swap { i: 2 }); + emit!(self, Instruction::StoreSubscr); + } + AugAssignKind::Attr { idx } => { + // stack: CONTAINER RESULT + emit!(self, Instruction::Swap { i: 2 }); + emit!(self, Instruction::StoreAttr { namei: idx }); + } + } + + Ok(()) + } + + fn compile_op(&mut self, op: &ast::Operator, inplace: bool) { + let bin_op = match op { + ast::Operator::Add => BinaryOperator::Add, + ast::Operator::Sub => BinaryOperator::Subtract, + ast::Operator::Mult => BinaryOperator::Multiply, + ast::Operator::MatMult => BinaryOperator::MatrixMultiply, + ast::Operator::Div => BinaryOperator::TrueDivide, + ast::Operator::FloorDiv => BinaryOperator::FloorDivide, + ast::Operator::Mod => BinaryOperator::Remainder, + ast::Operator::Pow => BinaryOperator::Power, + ast::Operator::LShift => BinaryOperator::Lshift, + ast::Operator::RShift => BinaryOperator::Rshift, + ast::Operator::BitOr => BinaryOperator::Or, + ast::Operator::BitXor => BinaryOperator::Xor, + ast::Operator::BitAnd => BinaryOperator::And, + }; + + let op = if inplace { bin_op.as_inplace() } else { bin_op }; + emit!(self, Instruction::BinaryOp { op }) + } + + /// Implement boolean short circuit evaluation logic. + /// https://en.wikipedia.org/wiki/Short-circuit_evaluation + /// + /// This means, in a boolean statement 'x and y' the variable y will + /// not be evaluated when x is false. + /// + /// The idea is to jump to a label if the expression is either true or false + /// (indicated by the condition parameter). + fn compile_jump_if( + &mut self, + expression: &ast::Expr, + condition: bool, + target_block: BlockIdx, + ) -> CompileResult<()> { + // Compile expression for test, and jump to label if false + match &expression { + ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) => { + match op { + ast::BoolOp::And => { + if condition { + // If all values are true. + let end_block = self.new_block(); + let (last_value, values) = values.split_last().unwrap(); + + // If any of the values is false, we can short-circuit. + for value in values { + self.compile_jump_if(value, false, end_block)?; + } + + // It depends upon the last value now: will it be true? + self.compile_jump_if(last_value, true, target_block)?; + self.switch_to_block(end_block); + } else { + // If any value is false, the whole condition is false. + for value in values { + self.compile_jump_if(value, false, target_block)?; + } + } + } + ast::BoolOp::Or => { + if condition { + // If any of the values is true. + for value in values { + self.compile_jump_if(value, true, target_block)?; + } + } else { + // If all of the values are false. + let end_block = self.new_block(); + let (last_value, values) = values.split_last().unwrap(); + + // If any value is true, we can short-circuit: + for value in values { + self.compile_jump_if(value, true, end_block)?; + } + + // It all depends upon the last value now! + self.compile_jump_if(last_value, false, target_block)?; + self.switch_to_block(end_block); + } + } + } + } + ast::Expr::UnaryOp(ast::ExprUnaryOp { + op: ast::UnaryOp::Not, + operand, + .. + }) => { + self.compile_jump_if(operand, !condition, target_block)?; + } + _ => { + // Fall back case which always will work! + self.compile_expression(expression)?; + if condition { + emit!( + self, + Instruction::PopJumpIfTrue { + delta: target_block, + } + ); + } else { + emit!( + self, + Instruction::PopJumpIfFalse { + delta: target_block, + } + ); + } + } + } + Ok(()) + } + + /// Compile a boolean operation as an expression. + /// This means, that the last value remains on the stack. + fn compile_bool_op(&mut self, op: &ast::BoolOp, values: &[ast::Expr]) -> CompileResult<()> { + self.compile_bool_op_with_target(op, values, None) + } + + /// Compile a boolean operation as an expression, with an optional + /// short-circuit target override. When `short_circuit_target` is `Some`, + /// the short-circuit jumps go to that block instead of the default + /// `after_block`, enabling jump threading to avoid redundant `__bool__` calls. + fn compile_bool_op_with_target( + &mut self, + op: &ast::BoolOp, + values: &[ast::Expr], + short_circuit_target: Option<BlockIdx>, + ) -> CompileResult<()> { + let after_block = self.new_block(); + let (last_value, values) = values.split_last().unwrap(); + let jump_target = short_circuit_target.unwrap_or(after_block); + + for value in values { + // Optimization: when a non-last value is a BoolOp with the opposite + // operator, redirect its short-circuit exits to skip the outer's + // redundant __bool__ test (jump threading). + if short_circuit_target.is_none() + && let ast::Expr::BoolOp(ast::ExprBoolOp { + op: inner_op, + values: inner_values, + .. + }) = value + && inner_op != op + { + let pop_block = self.new_block(); + self.compile_bool_op_with_target(inner_op, inner_values, Some(pop_block))?; + self.emit_short_circuit_test(op, after_block); + self.switch_to_block(pop_block); + emit!(self, Instruction::PopTop); + continue; + } + + self.compile_expression(value)?; + self.emit_short_circuit_test(op, jump_target); + emit!(self, Instruction::PopTop); + } + + // If all values did not qualify, take the value of the last value: + self.compile_expression(last_value)?; + self.switch_to_block(after_block); + Ok(()) + } + + /// Emit `Copy 1` + conditional jump for short-circuit evaluation. + /// For `And`, emits `PopJumpIfFalse`; for `Or`, emits `PopJumpIfTrue`. + fn emit_short_circuit_test(&mut self, op: &ast::BoolOp, target: BlockIdx) { + emit!(self, Instruction::Copy { i: 1 }); + match op { + ast::BoolOp::And => { + emit!(self, Instruction::PopJumpIfFalse { delta: target }); + } + ast::BoolOp::Or => { + emit!(self, Instruction::PopJumpIfTrue { delta: target }); + } + } + } + + fn compile_dict(&mut self, items: &[ast::DictItem]) -> CompileResult<()> { + let has_unpacking = items.iter().any(|item| item.key.is_none()); + + if !has_unpacking { + // Simple case: no ** unpacking, build all pairs directly + for item in items { + self.compile_expression(item.key.as_ref().unwrap())?; + self.compile_expression(&item.value)?; + } + emit!( + self, + Instruction::BuildMap { + count: u32::try_from(items.len()).expect("too many dict items"), + } + ); + return Ok(()); + } + + // Complex case with ** unpacking: preserve insertion order. + // Collect runs of regular k:v pairs and emit BUILD_MAP + DICT_UPDATE + // for each run, and DICT_UPDATE for each ** entry. + let mut have_dict = false; + let mut elements: u32 = 0; + + // Flush pending regular pairs as a BUILD_MAP, merging into the + // accumulator dict via DICT_UPDATE when one already exists. + macro_rules! flush_pending { + () => { + #[allow(unused_assignments)] + if elements > 0 { + emit!(self, Instruction::BuildMap { count: elements }); + if have_dict { + emit!(self, Instruction::DictUpdate { i: 1 }); + } else { + have_dict = true; + } + elements = 0; + } + }; + } + + for item in items { + if let Some(key) = &item.key { + // Regular key: value pair + self.compile_expression(key)?; + self.compile_expression(&item.value)?; + elements += 1; + } else { + // ** unpacking entry + flush_pending!(); + if !have_dict { + emit!(self, Instruction::BuildMap { count: 0 }); + have_dict = true; + } + self.compile_expression(&item.value)?; + emit!(self, Instruction::DictUpdate { i: 1 }); + } + } + + flush_pending!(); + if !have_dict { + emit!(self, Instruction::BuildMap { count: 0 }); + } + + Ok(()) + } + + /// Compile the yield-from/await sequence using SEND/END_SEND/CLEANUP_THROW. + /// compiler_add_yield_from + /// This generates: + /// send: + /// SEND exit + /// SETUP_FINALLY fail (via exception table) + /// YIELD_VALUE 1 + /// POP_BLOCK (implicit) + /// RESUME + /// JUMP send + /// fail: + /// CLEANUP_THROW + /// exit: + /// END_SEND + fn compile_yield_from_sequence(&mut self, is_await: bool) -> CompileResult<BlockIdx> { + let send_block = self.new_block(); + let fail_block = self.new_block(); + let exit_block = self.new_block(); + + // send: + self.switch_to_block(send_block); + emit!(self, Instruction::Send { delta: exit_block }); + + // SETUP_FINALLY fail - set up exception handler for YIELD_VALUE + emit!(self, PseudoInstruction::SetupFinally { delta: fail_block }); + self.push_fblock( + FBlockType::TryExcept, // Use TryExcept for exception handler + send_block, + exit_block, + )?; + + // YIELD_VALUE with arg=1 (yield-from/await mode - not wrapped for async gen) + emit!(self, Instruction::YieldValue { arg: 1 }); + + // POP_BLOCK before RESUME + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::TryExcept); + + // RESUME + emit!( + self, + Instruction::Resume { + context: if is_await { + u32::from(bytecode::ResumeType::AfterAwait) + } else { + u32::from(bytecode::ResumeType::AfterYieldFrom) + } + } + ); + + // JUMP_BACKWARD_NO_INTERRUPT send + emit!( + self, + PseudoInstruction::JumpNoInterrupt { delta: send_block } + ); + + // fail: CLEANUP_THROW + // Stack when exception: [receiver, yielded_value, exc] + // CLEANUP_THROW: [sub_iter, last_sent_val, exc] -> [None, value] + // After: stack is [None, value], fall through to exit + self.switch_to_block(fail_block); + emit!(self, Instruction::CleanupThrow); + // Fall through to exit block + + // exit: END_SEND + // Stack: [receiver, value] (from SEND) or [None, value] (from CLEANUP_THROW) + // END_SEND: [receiver/None, value] -> [value] + self.switch_to_block(exit_block); + emit!(self, Instruction::EndSend); + + Ok(send_block) + } + + fn compile_expression(&mut self, expression: &ast::Expr) -> CompileResult<()> { + trace!("Compiling {expression:?}"); + let range = expression.range(); + self.set_source_range(range); + + match &expression { + ast::Expr::Call(ast::ExprCall { + func, arguments, .. + }) => self.compile_call(func, arguments)?, + ast::Expr::BoolOp(ast::ExprBoolOp { op, values, .. }) => { + self.compile_bool_op(op, values)? + } + ast::Expr::BinOp(ast::ExprBinOp { + left, op, right, .. + }) => { + self.compile_expression(left)?; + self.compile_expression(right)?; + + // Restore full expression range before emitting the operation + self.set_source_range(range); + self.compile_op(op, false); + } + ast::Expr::Subscript(ast::ExprSubscript { + value, slice, ctx, .. + }) => { + self.compile_subscript(value, slice, *ctx)?; + } + ast::Expr::UnaryOp(ast::ExprUnaryOp { op, operand, .. }) => { + self.compile_expression(operand)?; + + // Restore full expression range before emitting the operation + self.set_source_range(range); + match op { + ast::UnaryOp::UAdd => emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::UnaryPositive + } + ), + ast::UnaryOp::USub => emit!(self, Instruction::UnaryNegative), + ast::UnaryOp::Not => { + emit!(self, Instruction::ToBool); + emit!(self, Instruction::UnaryNot); + } + ast::UnaryOp::Invert => emit!(self, Instruction::UnaryInvert), + }; + } + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { + // Check for super() attribute access optimization + if let Some(super_type) = self.can_optimize_super_call(value, attr.as_str()) { + // super().attr or super(cls, self).attr optimization + // Stack: [global_super, class, self] → LOAD_SUPER_ATTR → [attr] + // Set source range to super() call for arg-loading instructions + let super_range = value.range(); + self.set_source_range(super_range); + self.load_args_for_super(&super_type)?; + self.set_source_range(super_range); + let idx = self.name(attr.as_str()); + match super_type { + SuperCallType::TwoArg { .. } => { + self.emit_load_super_attr(idx); + } + SuperCallType::ZeroArg => { + self.emit_load_zero_super_attr(idx); + } + } + } else { + // Normal attribute access + self.compile_expression(value)?; + let idx = self.name(attr.as_str()); + self.emit_load_attr(idx); + } + } + ast::Expr::Compare(ast::ExprCompare { + left, + ops, + comparators, + .. + }) => { + self.compile_compare(left, ops, comparators)?; + } + // ast::Expr::Constant(ExprConstant { value, .. }) => { + // self.emit_load_const(compile_constant(value)); + // } + ast::Expr::List(ast::ExprList { elts, .. }) => { + self.starunpack_helper(elts, 0, CollectionType::List)?; + } + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { + self.starunpack_helper(elts, 0, CollectionType::Tuple)?; + } + ast::Expr::Set(ast::ExprSet { elts, .. }) => { + self.starunpack_helper(elts, 0, CollectionType::Set)?; + } + ast::Expr::Dict(ast::ExprDict { items, .. }) => { + self.compile_dict(items)?; + } + ast::Expr::Slice(ast::ExprSlice { + lower, upper, step, .. + }) => { + let mut compile_bound = |bound: Option<&ast::Expr>| match bound { + Some(exp) => self.compile_expression(exp), + None => { + self.emit_load_const(ConstantData::None); + Ok(()) + } + }; + compile_bound(lower.as_deref())?; + compile_bound(upper.as_deref())?; + if let Some(step) = step { + self.compile_expression(step)?; + } + let argc = match step { + Some(_) => BuildSliceArgCount::Three, + None => BuildSliceArgCount::Two, + }; + emit!(self, Instruction::BuildSlice { argc }); + } + ast::Expr::Yield(ast::ExprYield { value, .. }) => { + if !self.ctx.in_func() { + return Err(self.error(CodegenErrorType::InvalidYield)); + } + self.mark_generator(); + match value { + Some(expression) => self.compile_expression(expression)?, + Option::None => self.emit_load_const(ConstantData::None), + }; + // arg=0: direct yield (wrapped for async generators) + emit!(self, Instruction::YieldValue { arg: 0 }); + emit!( + self, + Instruction::Resume { + context: u32::from(bytecode::ResumeType::AfterYield) + } + ); + } + ast::Expr::Await(ast::ExprAwait { value, .. }) => { + if self.ctx.func != FunctionContext::AsyncFunction { + return Err(self.error(CodegenErrorType::InvalidAwait)); + } + self.compile_expression(value)?; + emit!(self, Instruction::GetAwaitable { r#where: 0 }); + self.emit_load_const(ConstantData::None); + let _ = self.compile_yield_from_sequence(true)?; + } + ast::Expr::YieldFrom(ast::ExprYieldFrom { value, .. }) => { + match self.ctx.func { + FunctionContext::NoFunction => { + return Err(self.error(CodegenErrorType::InvalidYieldFrom)); + } + FunctionContext::AsyncFunction => { + return Err(self.error(CodegenErrorType::AsyncYieldFrom)); + } + FunctionContext::Function => {} + } + self.mark_generator(); + self.compile_expression(value)?; + emit!(self, Instruction::GetYieldFromIter); + self.emit_load_const(ConstantData::None); + let _ = self.compile_yield_from_sequence(false)?; + } + ast::Expr::Name(ast::ExprName { id, .. }) => self.load_name(id.as_str())?, + ast::Expr::Lambda(ast::ExprLambda { + parameters, body, .. + }) => { + let default_params = ast::Parameters::default(); + let params = parameters.as_deref().unwrap_or(&default_params); + validate_duplicate_params(params).map_err(|e| self.error(e))?; + + let prev_ctx = self.ctx; + let name = "<lambda>".to_owned(); + + // Prepare defaults before entering function + let defaults: Vec<_> = core::iter::empty() + .chain(&params.posonlyargs) + .chain(&params.args) + .filter_map(|x| x.default.as_deref()) + .collect(); + let have_defaults = !defaults.is_empty(); + + if have_defaults { + let size = defaults.len().to_u32(); + for element in &defaults { + self.compile_expression(element)?; + } + emit!(self, Instruction::BuildTuple { count: size }); + } + + // Prepare keyword-only defaults + let mut kw_with_defaults = vec![]; + for kwonlyarg in &params.kwonlyargs { + if let Some(default) = &kwonlyarg.default { + kw_with_defaults.push((&kwonlyarg.parameter, default)); + } + } + + let have_kwdefaults = !kw_with_defaults.is_empty(); + if have_kwdefaults { + let default_kw_count = kw_with_defaults.len(); + for (arg, default) in &kw_with_defaults { + self.emit_load_const(ConstantData::Str { + value: self.mangle(arg.name.as_str()).into_owned().into(), + }); + self.compile_expression(default)?; + } + emit!( + self, + Instruction::BuildMap { + count: default_kw_count.to_u32(), + } + ); + } + + self.enter_function(&name, params)?; + let mut func_flags = bytecode::MakeFunctionFlags::new(); + if have_defaults { + func_flags.insert(bytecode::MakeFunctionFlag::Defaults); + } + if have_kwdefaults { + func_flags.insert(bytecode::MakeFunctionFlag::KwOnlyDefaults); + } + + // Set qualname for lambda + self.set_qualname(); + + self.ctx = CompileContext { + loop_data: Option::None, + in_class: prev_ctx.in_class, + func: FunctionContext::Function, + // Lambda is never async, so new scope is not async + in_async_scope: false, + }; + + // Lambda cannot have docstrings, so no None is added to co_consts + + self.compile_expression(body)?; + self.emit_return_value(); + let code = self.exit_scope(); + + // Create lambda function with closure + self.make_closure(code, func_flags)?; + + self.ctx = prev_ctx; + } + ast::Expr::ListComp(ast::ExprListComp { + elt, generators, .. + }) => { + self.compile_comprehension( + "<listcomp>", + Some( + Instruction::BuildList { + count: OpArgMarker::marker(), + } + .into(), + ), + generators, + &|compiler| { + compiler.compile_comprehension_element(elt)?; + emit!( + compiler, + Instruction::ListAppend { + i: generators.len().to_u32(), + } + ); + Ok(()) + }, + ComprehensionType::List, + Self::contains_await(elt) || Self::generators_contain_await(generators), + )?; + } + ast::Expr::SetComp(ast::ExprSetComp { + elt, generators, .. + }) => { + self.compile_comprehension( + "<setcomp>", + Some( + Instruction::BuildSet { + count: OpArgMarker::marker(), + } + .into(), + ), + generators, + &|compiler| { + compiler.compile_comprehension_element(elt)?; + emit!( + compiler, + Instruction::SetAdd { + i: generators.len().to_u32(), + } + ); + Ok(()) + }, + ComprehensionType::Set, + Self::contains_await(elt) || Self::generators_contain_await(generators), + )?; + } + ast::Expr::DictComp(ast::ExprDictComp { + key, + value, + generators, + .. + }) => { + self.compile_comprehension( + "<dictcomp>", + Some( + Instruction::BuildMap { + count: OpArgMarker::marker(), + } + .into(), + ), + generators, + &|compiler| { + // changed evaluation order for Py38 named expression PEP 572 + compiler.compile_expression(key)?; + compiler.compile_expression(value)?; + + emit!( + compiler, + Instruction::MapAdd { + i: generators.len().to_u32(), + } + ); + + Ok(()) + }, + ComprehensionType::Dict, + Self::contains_await(key) + || Self::contains_await(value) + || Self::generators_contain_await(generators), + )?; + } + ast::Expr::Generator(ast::ExprGenerator { + elt, generators, .. + }) => { + // Check if element or generators contain async content + // This makes the generator expression into an async generator + let element_contains_await = + Self::contains_await(elt) || Self::generators_contain_await(generators); + self.compile_comprehension( + "<genexpr>", + None, + generators, + &|compiler| { + // Compile the element expression + // Note: if element is an async comprehension, compile_expression + // already handles awaiting it, so we don't need to await again here + compiler.compile_comprehension_element(elt)?; + + compiler.mark_generator(); + // arg=0: direct yield (wrapped for async generators) + emit!(compiler, Instruction::YieldValue { arg: 0 }); + emit!( + compiler, + Instruction::Resume { + context: u32::from(bytecode::ResumeType::AfterYield) + } + ); + emit!(compiler, Instruction::PopTop); + + Ok(()) + }, + ComprehensionType::Generator, + element_contains_await, + )?; + } + ast::Expr::Starred(ast::ExprStarred { value, .. }) => { + if self.in_annotation { + // In annotation context, starred expressions are allowed (PEP 646) + // For now, just compile the inner value without wrapping with Unpack + // This is a temporary solution until we figure out how to properly import typing + self.compile_expression(value)?; + } else { + return Err(self.error(CodegenErrorType::InvalidStarExpr)); + } + } + ast::Expr::If(ast::ExprIf { + test, body, orelse, .. + }) => { + let else_block = self.new_block(); + let after_block = self.new_block(); + self.compile_jump_if(test, false, else_block)?; + + // True case + self.compile_expression(body)?; + emit!(self, PseudoInstruction::Jump { delta: after_block }); + + // False case + self.switch_to_block(else_block); + self.compile_expression(orelse)?; + + // End + self.switch_to_block(after_block); + } + + ast::Expr::Named(ast::ExprNamed { + target, + value, + node_index: _, + range: _, + }) => { + self.compile_expression(value)?; + emit!(self, Instruction::Copy { i: 1 }); + self.compile_store(target)?; + } + ast::Expr::FString(fstring) => { + self.compile_expr_fstring(fstring)?; + } + ast::Expr::TString(tstring) => { + self.compile_expr_tstring(tstring)?; + } + ast::Expr::StringLiteral(string) => { + let value = string.value.to_str(); + if value.contains(char::REPLACEMENT_CHARACTER) { + let value = string + .value + .iter() + .map(|lit| { + let source = self.source_file.slice(lit.range); + crate::string_parser::parse_string_literal(source, lit.flags.into()) + }) + .collect(); + // might have a surrogate literal; should reparse to be sure + self.emit_load_const(ConstantData::Str { value }); + } else { + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); + } + } + ast::Expr::BytesLiteral(bytes) => { + let iter = bytes.value.iter().flat_map(|x| x.iter().copied()); + let v: Vec<u8> = iter.collect(); + self.emit_load_const(ConstantData::Bytes { value: v }); + } + ast::Expr::NumberLiteral(number) => match &number.value { + ast::Number::Int(int) => { + let value = ruff_int_to_bigint(int).map_err(|e| self.error(e))?; + self.emit_load_const(ConstantData::Integer { value }); + } + ast::Number::Float(float) => { + self.emit_load_const(ConstantData::Float { value: *float }); + } + ast::Number::Complex { real, imag } => { + self.emit_load_const(ConstantData::Complex { + value: Complex::new(*real, *imag), + }); + } + }, + ast::Expr::BooleanLiteral(b) => { + self.emit_load_const(ConstantData::Boolean { value: b.value }); + } + ast::Expr::NoneLiteral(_) => { + self.emit_load_const(ConstantData::None); + } + ast::Expr::EllipsisLiteral(_) => { + self.emit_load_const(ConstantData::Ellipsis); + } + ast::Expr::IpyEscapeCommand(_) => { + panic!("unexpected ipy escape command"); + } + } + Ok(()) + } + + fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> CompileResult<()> { + let mut size = 0; + let groupby = keywords.iter().chunk_by(|e| e.arg.is_none()); + for (is_unpacking, sub_keywords) in &groupby { + if is_unpacking { + for keyword in sub_keywords { + self.compile_expression(&keyword.value)?; + size += 1; + } + } else { + let mut sub_size = 0; + for keyword in sub_keywords { + if let Some(name) = &keyword.arg { + self.emit_load_const(ConstantData::Str { + value: name.as_str().into(), + }); + self.compile_expression(&keyword.value)?; + sub_size += 1; + } + } + emit!(self, Instruction::BuildMap { count: sub_size }); + size += 1; + } + } + if size > 1 { + // Merge all dicts: first dict is accumulator, merge rest into it + for _ in 1..size { + emit!(self, Instruction::DictMerge { i: 1 }); + } + } + Ok(()) + } + + fn compile_call(&mut self, func: &ast::Expr, args: &ast::Arguments) -> CompileResult<()> { + // Save the call expression's source range so CALL instructions use the + // call start line, not the last argument's line. + let call_range = self.current_source_range; + + // Method call: obj → LOAD_ATTR_METHOD → [method, self_or_null] → args → CALL + // Regular call: func → PUSH_NULL → args → CALL + if let ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) = &func { + // Check for super() method call optimization + if let Some(super_type) = self.can_optimize_super_call(value, attr.as_str()) { + // super().method() or super(cls, self).method() optimization + // Stack: [global_super, class, self] → LOAD_SUPER_METHOD → [method, self] + // Set source range to the super() call for LOAD_GLOBAL/LOAD_DEREF/etc. + let super_range = value.range(); + self.set_source_range(super_range); + self.load_args_for_super(&super_type)?; + self.set_source_range(super_range); + let idx = self.name(attr.as_str()); + match super_type { + SuperCallType::TwoArg { .. } => { + self.emit_load_super_method(idx); + } + SuperCallType::ZeroArg => { + self.emit_load_zero_super_method(idx); + } + } + // NOP for line tracking at .method( line + self.set_source_range(attr.range()); + emit!(self, Instruction::Nop); + // CALL at .method( line (not the full expression line) + self.codegen_call_helper(0, args, attr.range())?; + } else { + // Normal method call: compile object, then LOAD_ATTR with method flag + // LOAD_ATTR(method=1) pushes [method, self_or_null] on stack + self.compile_expression(value)?; + let idx = self.name(attr.as_str()); + self.emit_load_attr_method(idx); + self.codegen_call_helper(0, args, call_range)?; + } + } else { + // Regular call: push func, then NULL for self_or_null slot + // Stack layout: [func, NULL, args...] - same as method call [func, self, args...] + self.compile_expression(func)?; + emit!(self, Instruction::PushNull); + self.codegen_call_helper(0, args, call_range)?; + } + Ok(()) + } + + /// Compile subkwargs: emit key-value pairs for BUILD_MAP + fn codegen_subkwargs( + &mut self, + keywords: &[ast::Keyword], + begin: usize, + end: usize, + ) -> CompileResult<()> { + let n = end - begin; + assert!(n > 0); + + // For large kwargs, use BUILD_MAP(0) + MAP_ADD to avoid stack overflow + let big = n * 2 > 8; // STACK_USE_GUIDELINE approximation + + if big { + emit!(self, Instruction::BuildMap { count: 0 }); + } + + for kw in &keywords[begin..end] { + // Key first, then value - this is critical! + self.emit_load_const(ConstantData::Str { + value: kw.arg.as_ref().unwrap().as_str().into(), + }); + self.compile_expression(&kw.value)?; + + if big { + emit!(self, Instruction::MapAdd { i: 0 }); + } + } + + if !big { + emit!(self, Instruction::BuildMap { count: n.to_u32() }); + } + + Ok(()) + } + + /// Compile call arguments and emit the appropriate CALL instruction. + /// `call_range` is the source range of the call expression, used to set + /// the correct line number on the CALL instruction. + fn codegen_call_helper( + &mut self, + additional_positional: u32, + arguments: &ast::Arguments, + call_range: TextRange, + ) -> CompileResult<()> { + let nelts = arguments.args.len(); + let nkwelts = arguments.keywords.len(); + + // Check if we have starred args or **kwargs + let has_starred = arguments + .args + .iter() + .any(|arg| matches!(arg, ast::Expr::Starred(_))); + let has_double_star = arguments.keywords.iter().any(|k| k.arg.is_none()); + + // Check if exceeds stack guideline + let too_big = nelts + nkwelts * 2 > 8; + + if !has_starred && !has_double_star && !too_big { + // Simple call path: no * or ** args + for arg in &arguments.args { + self.compile_expression(arg)?; + } + + if nkwelts > 0 { + // Compile keyword values and build kwnames tuple + let mut kwarg_names = Vec::with_capacity(nkwelts); + for keyword in &arguments.keywords { + kwarg_names.push(ConstantData::Str { + value: keyword.arg.as_ref().unwrap().as_str().into(), + }); + self.compile_expression(&keyword.value)?; + } + + // Restore call expression range for kwnames and CALL_KW + self.set_source_range(call_range); + self.emit_load_const(ConstantData::Tuple { + elements: kwarg_names, + }); + + let argc = additional_positional + nelts.to_u32() + nkwelts.to_u32(); + emit!(self, Instruction::CallKw { argc }); + } else { + self.set_source_range(call_range); + let argc = additional_positional + nelts.to_u32(); + emit!(self, Instruction::Call { argc }); + } + } else { + // ex_call path: has * or ** args + + // Compile positional arguments + if additional_positional == 0 + && nelts == 1 + && matches!(arguments.args[0], ast::Expr::Starred(_)) + { + // Single starred arg: pass value directly to CallFunctionEx. + // Runtime will convert to tuple and validate with function name. + if let ast::Expr::Starred(ast::ExprStarred { value, .. }) = &arguments.args[0] { + self.compile_expression(value)?; + } + } else { + // Use starunpack_helper to build a list, then convert to tuple + self.starunpack_helper( + &arguments.args, + additional_positional, + CollectionType::List, + )?; + emit!( + self, + Instruction::CallIntrinsic1 { + func: IntrinsicFunction1::ListToTuple + } + ); + } + + // Compile keyword arguments + if nkwelts > 0 { + let mut have_dict = false; + let mut nseen = 0usize; + + for (i, keyword) in arguments.keywords.iter().enumerate() { + if keyword.arg.is_none() { + // **kwargs unpacking + if nseen > 0 { + // Pack up preceding keywords using codegen_subkwargs + self.codegen_subkwargs(&arguments.keywords, i - nseen, i)?; + if have_dict { + emit!(self, Instruction::DictMerge { i: 1 }); + } + have_dict = true; + nseen = 0; + } + + if !have_dict { + emit!(self, Instruction::BuildMap { count: 0 }); + have_dict = true; + } + + self.compile_expression(&keyword.value)?; + emit!(self, Instruction::DictMerge { i: 1 }); + } else { + nseen += 1; + } + } + + // Pack up any trailing keyword arguments + if nseen > 0 { + self.codegen_subkwargs(&arguments.keywords, nkwelts - nseen, nkwelts)?; + if have_dict { + emit!(self, Instruction::DictMerge { i: 1 }); + } + have_dict = true; + } + + assert!(have_dict); + } else { + emit!(self, Instruction::PushNull); + } + + self.set_source_range(call_range); + emit!(self, Instruction::CallFunctionEx); + } + + Ok(()) + } + + fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> { + self.compile_expression(element).map_err(|e| { + if let CodegenErrorType::InvalidStarExpr = e.error { + self.error(CodegenErrorType::SyntaxError( + "iterable unpacking cannot be used in comprehension".to_owned(), + )) + } else { + e + } + }) + } + + fn consume_next_sub_table(&mut self) -> CompileResult<()> { + { + let _ = self.push_symbol_table()?; + } + let _ = self.pop_symbol_table(); + Ok(()) + } + + fn consume_skipped_nested_scopes_in_expr( + &mut self, + expression: &ast::Expr, + ) -> CompileResult<()> { + use ast::visitor::Visitor; + + struct SkippedScopeVisitor<'a> { + compiler: &'a mut Compiler, + error: Option<CodegenError>, + } + + impl SkippedScopeVisitor<'_> { + fn consume_scope(&mut self) { + if self.error.is_none() { + self.error = self.compiler.consume_next_sub_table().err(); + } + } + } + + impl ast::visitor::Visitor<'_> for SkippedScopeVisitor<'_> { + fn visit_expr(&mut self, expr: &ast::Expr) { + if self.error.is_some() { + return; + } + + match expr { + ast::Expr::Lambda(ast::ExprLambda { parameters, .. }) => { + // Defaults are scanned before enter_scope in the + // symbol table builder, so their nested scopes + // precede the lambda scope in sub_tables. + if let Some(params) = parameters.as_deref() { + for default in params + .posonlyargs + .iter() + .chain(&params.args) + .chain(&params.kwonlyargs) + .filter_map(|p| p.default.as_deref()) + { + self.visit_expr(default); + } + } + self.consume_scope(); + } + ast::Expr::ListComp(ast::ExprListComp { generators, .. }) + | ast::Expr::SetComp(ast::ExprSetComp { generators, .. }) + | ast::Expr::Generator(ast::ExprGenerator { generators, .. }) => { + // leave_scope runs before the first iterator is + // scanned, so the comprehension scope comes first + // in sub_tables, then any nested scopes from the + // first iterator. + self.consume_scope(); + if let Some(first) = generators.first() { + self.visit_expr(&first.iter); + } + } + ast::Expr::DictComp(ast::ExprDictComp { generators, .. }) => { + self.consume_scope(); + if let Some(first) = generators.first() { + self.visit_expr(&first.iter); + } + } + _ => ast::visitor::walk_expr(self, expr), + } + } + } + + let mut visitor = SkippedScopeVisitor { + compiler: self, + error: None, + }; + visitor.visit_expr(expression); + if let Some(err) = visitor.error { + Err(err) + } else { + Ok(()) + } + } + + fn compile_comprehension( + &mut self, + name: &str, + init_collection: Option<AnyInstruction>, + generators: &[ast::Comprehension], + compile_element: &dyn Fn(&mut Self) -> CompileResult<()>, + comprehension_type: ComprehensionType, + element_contains_await: bool, + ) -> CompileResult<()> { + let prev_ctx = self.ctx; + let has_an_async_gen = generators.iter().any(|g| g.is_async); + + // Check for async comprehension outside async function (list/set/dict only, not generator expressions) + // Use in_async_scope to allow nested async comprehensions inside an async function + if comprehension_type != ComprehensionType::Generator + && (has_an_async_gen || element_contains_await) + && !prev_ctx.in_async_scope + { + return Err(self.error(CodegenErrorType::InvalidAsyncComprehension)); + } + + // Check if this comprehension should be inlined (PEP 709) + let is_inlined = self.is_inlined_comprehension_context(comprehension_type); + + // async comprehensions are allowed in various contexts: + // - list/set/dict comprehensions in async functions (or nested within) + // - always for generator expressions + let is_async_list_set_dict_comprehension = comprehension_type + != ComprehensionType::Generator + && (has_an_async_gen || element_contains_await) + && prev_ctx.in_async_scope; + + let is_async_generator_comprehension = comprehension_type == ComprehensionType::Generator + && (has_an_async_gen || element_contains_await); + + debug_assert!(!(is_async_list_set_dict_comprehension && is_async_generator_comprehension)); + + let is_async = is_async_list_set_dict_comprehension || is_async_generator_comprehension; + + // We must have at least one generator: + assert!(!generators.is_empty()); + + if is_inlined { + // PEP 709: Inlined comprehension - compile inline without new scope + return self.compile_inlined_comprehension( + init_collection, + generators, + compile_element, + has_an_async_gen, + ); + } + + // Non-inlined path: create a new code object (generator expressions, etc.) + self.ctx = CompileContext { + loop_data: None, + in_class: prev_ctx.in_class, + func: if is_async { + FunctionContext::AsyncFunction + } else { + FunctionContext::Function + }, + // Inherit in_async_scope from parent - nested async comprehensions are allowed + // if we're anywhere inside an async function + in_async_scope: prev_ctx.in_async_scope || is_async, + }; + + let flags = bytecode::CodeFlags::NEWLOCALS | bytecode::CodeFlags::OPTIMIZED; + let flags = if is_async { + flags | bytecode::CodeFlags::COROUTINE + } else { + flags + }; + + // Create magnificent function <listcomp>: + self.push_output(flags, 1, 1, 0, name.to_owned())?; + + // Mark that we're in an inlined comprehension + self.current_code_info().in_inlined_comp = true; + + // Set qualname for comprehension + self.set_qualname(); + + let arg0 = self.varname(".0")?; + + let return_none = init_collection.is_none(); + // Create empty object of proper type: + if let Some(init_collection) = init_collection { + self._emit(init_collection, OpArg::new(0), BlockIdx::NULL) + } + + let mut loop_labels = vec![]; + for generator in generators { + let loop_block = self.new_block(); + let if_cleanup_block = self.new_block(); + let after_block = self.new_block(); + + if loop_labels.is_empty() { + // Load iterator onto stack (passed as first argument): + emit!(self, Instruction::LoadFast { var_num: arg0 }); + } else { + // Evaluate iterated item: + self.compile_expression(&generator.iter)?; + + // Get iterator / turn item into an iterator + if generator.is_async { + emit!(self, Instruction::GetAIter); + } else { + emit!(self, Instruction::GetIter); + } + } + + self.switch_to_block(loop_block); + let mut end_async_for_target = BlockIdx::NULL; + if generator.is_async { + emit!(self, PseudoInstruction::SetupFinally { delta: after_block }); + emit!(self, Instruction::GetANext); + self.push_fblock( + FBlockType::AsyncComprehensionGenerator, + loop_block, + after_block, + )?; + self.emit_load_const(ConstantData::None); + end_async_for_target = self.compile_yield_from_sequence(true)?; + // POP_BLOCK before store: only __anext__/yield_from are + // protected by SetupFinally targeting END_ASYNC_FOR. + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::AsyncComprehensionGenerator); + self.compile_store(&generator.target)?; + } else { + emit!(self, Instruction::ForIter { delta: after_block }); + self.compile_store(&generator.target)?; + } + loop_labels.push(( + loop_block, + if_cleanup_block, + after_block, + generator.is_async, + end_async_for_target, + )); + + // Now evaluate the ifs: + for if_condition in &generator.ifs { + self.compile_jump_if(if_condition, false, if_cleanup_block)? + } + } + + compile_element(self)?; + + for (loop_block, if_cleanup_block, after_block, is_async, end_async_for_target) in + loop_labels.iter().rev().copied() + { + emit!(self, PseudoInstruction::Jump { delta: loop_block }); + + self.switch_to_block(if_cleanup_block); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); + + self.switch_to_block(after_block); + if is_async { + // EndAsyncFor pops both the exception and the aiter + // (handler depth is before GetANext, so aiter is at handler depth) + self.emit_end_async_for(end_async_for_target); + } else { + // END_FOR + POP_ITER pattern (CPython 3.14) + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); + } + } + + if return_none { + self.emit_load_const(ConstantData::None) + } + + self.emit_return_value(); + + let code = self.exit_scope(); + + self.ctx = prev_ctx; + + // Create comprehension function with closure + self.make_closure(code, bytecode::MakeFunctionFlags::new())?; + emit!(self, Instruction::PushNull); + + // Evaluate iterated item: + self.compile_expression(&generators[0].iter)?; + + // Get iterator / turn item into an iterator + // Use is_async from the first generator, not has_an_async_gen which covers ALL generators + if generators[0].is_async { + emit!(self, Instruction::GetAIter); + } else { + emit!(self, Instruction::GetIter); + }; + + // Call just created <listcomp> function: + emit!(self, Instruction::Call { argc: 1 }); + if is_async_list_set_dict_comprehension { + emit!(self, Instruction::GetAwaitable { r#where: 0 }); + self.emit_load_const(ConstantData::None); + let _ = self.compile_yield_from_sequence(true)?; + } + + Ok(()) + } + + /// Collect variable names from an assignment target expression + fn collect_target_names(&self, target: &ast::Expr, names: &mut Vec<String>) { + match target { + ast::Expr::Name(name) => { + let name_str = name.id.to_string(); + if !names.contains(&name_str) { + names.push(name_str); + } + } + ast::Expr::Tuple(tuple) => { + for elt in &tuple.elts { + self.collect_target_names(elt, names); + } + } + ast::Expr::List(list) => { + for elt in &list.elts { + self.collect_target_names(elt, names); + } + } + ast::Expr::Starred(starred) => { + self.collect_target_names(&starred.value, names); + } + _ => { + // Other targets (attribute, subscript) don't bind local names + } + } + } + + /// Compile an inlined comprehension (PEP 709) + /// This generates bytecode inline without creating a new code object + fn compile_inlined_comprehension( + &mut self, + init_collection: Option<AnyInstruction>, + generators: &[ast::Comprehension], + compile_element: &dyn Fn(&mut Self) -> CompileResult<()>, + _has_an_async_gen: bool, + ) -> CompileResult<()> { + // PEP 709: Consume the comprehension's sub_table (but we won't use it as a separate scope) + // We need to consume it to keep sub_tables in sync with AST traversal order. + // The symbols are already merged into parent scope by analyze_symbol_table. + let _comp_table = self + .symbol_table_stack + .last_mut() + .expect("no current symbol table") + .sub_tables + .remove(0); + + // Collect local variables that need to be saved/restored + // These are variables bound in the comprehension (iteration vars from targets) + let mut pushed_locals: Vec<String> = Vec::new(); + for generator in generators { + self.collect_target_names(&generator.target, &mut pushed_locals); + } + + // Step 1: Compile the outermost iterator + self.compile_expression(&generators[0].iter)?; + // Use is_async from the first generator, not has_an_async_gen which covers ALL generators + if generators[0].is_async { + emit!(self, Instruction::GetAIter); + } else { + emit!(self, Instruction::GetIter); + } + + // Step 2: Save local variables that will be shadowed by the comprehension + for name in &pushed_locals { + let var_num = self.varname(name)?; + emit!(self, Instruction::LoadFastAndClear { var_num }); + } + + // Step 3: SWAP iterator to TOS (above saved locals) + if !pushed_locals.is_empty() { + emit!( + self, + Instruction::Swap { + i: u32::try_from(pushed_locals.len() + 1).unwrap() + } + ); + } + + // Step 4: Create the collection (list/set/dict) + // For generator expressions, init_collection is None + if let Some(init_collection) = init_collection { + self._emit(init_collection, OpArg::new(0), BlockIdx::NULL); + // SWAP to get iterator on top + emit!(self, Instruction::Swap { i: 2 }); + } + + // Set up exception handler for cleanup on exception + let cleanup_block = self.new_block(); + let end_block = self.new_block(); + + if !pushed_locals.is_empty() { + emit!( + self, + PseudoInstruction::SetupFinally { + delta: cleanup_block + } + ); + self.push_fblock(FBlockType::TryExcept, cleanup_block, end_block)?; + } + + // Step 5: Compile the comprehension loop(s) + let mut loop_labels = vec![]; + for (i, generator) in generators.iter().enumerate() { + let loop_block = self.new_block(); + let if_cleanup_block = self.new_block(); + let after_block = self.new_block(); + + if i > 0 { + // For nested loops, compile the iterator expression + self.compile_expression(&generator.iter)?; + if generator.is_async { + emit!(self, Instruction::GetAIter); + } else { + emit!(self, Instruction::GetIter); + } + } + + self.switch_to_block(loop_block); + let mut end_async_for_target = BlockIdx::NULL; + + if generator.is_async { + emit!(self, Instruction::GetANext); + self.emit_load_const(ConstantData::None); + end_async_for_target = self.compile_yield_from_sequence(true)?; + self.compile_store(&generator.target)?; + } else { + emit!(self, Instruction::ForIter { delta: after_block }); + self.compile_store(&generator.target)?; + } + loop_labels.push(( + loop_block, + if_cleanup_block, + after_block, + generator.is_async, + end_async_for_target, + )); + + // Evaluate the if conditions + for if_condition in &generator.ifs { + self.compile_jump_if(if_condition, false, if_cleanup_block)?; + } + } + + // Step 6: Compile the element expression and append to collection + compile_element(self)?; + + // Step 7: Close all loops + for (loop_block, if_cleanup_block, after_block, is_async, end_async_for_target) in + loop_labels.iter().rev().copied() + { + emit!(self, PseudoInstruction::Jump { delta: loop_block }); + + self.switch_to_block(if_cleanup_block); + emit!(self, PseudoInstruction::Jump { delta: loop_block }); + + self.switch_to_block(after_block); + if is_async { + self.emit_end_async_for(end_async_for_target); + // Pop the iterator + emit!(self, Instruction::PopTop); + } else { + // END_FOR + POP_ITER pattern (CPython 3.14) + emit!(self, Instruction::EndFor); + emit!(self, Instruction::PopIter); + } + } + + // Step 8: Clean up - restore saved locals + if !pushed_locals.is_empty() { + emit!(self, PseudoInstruction::PopBlock); + self.pop_fblock(FBlockType::TryExcept); + + // Normal path: jump past cleanup + emit!(self, PseudoInstruction::Jump { delta: end_block }); + + // Exception cleanup path + self.switch_to_block(cleanup_block); + // Stack: [saved_locals..., collection, exception] + // Swap to get collection out from under exception + emit!(self, Instruction::Swap { i: 2 }); + emit!(self, Instruction::PopTop); // Pop incomplete collection + + // Restore locals + emit!( + self, + Instruction::Swap { + i: u32::try_from(pushed_locals.len() + 1).unwrap() + } + ); + for name in pushed_locals.iter().rev() { + let var_num = self.varname(name)?; + emit!(self, Instruction::StoreFast { var_num }); + } + // Re-raise the exception + emit!(self, Instruction::Reraise { depth: 0 }); + + // Normal end path + self.switch_to_block(end_block); + } + + // SWAP result to TOS (above saved locals) + if !pushed_locals.is_empty() { + emit!( + self, + Instruction::Swap { + i: u32::try_from(pushed_locals.len() + 1).unwrap() + } + ); + } + + // Restore saved locals + for name in pushed_locals.iter().rev() { + let var_num = self.varname(name)?; + emit!(self, Instruction::StoreFast { var_num }); + } + + Ok(()) + } + + fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CodegenError> { + if let DoneWithFuture::Yes = self.done_with_future_stmts { + return Err(self.error(CodegenErrorType::InvalidFuturePlacement)); + } + self.done_with_future_stmts = DoneWithFuture::DoneWithDoc; + for feature in features { + match feature.name.as_str() { + // Python 3 features; we've already implemented them by default + "nested_scopes" | "generators" | "division" | "absolute_import" + | "with_statement" | "print_function" | "unicode_literals" | "generator_stop" => {} + "annotations" => self.future_annotations = true, + other => { + return Err( + self.error(CodegenErrorType::InvalidFutureFeature(other.to_owned())) + ); + } + } + } + Ok(()) + } + + // Low level helper functions: + fn _emit<I: Into<AnyInstruction>>(&mut self, instr: I, arg: OpArg, target: BlockIdx) { + let range = self.current_source_range; + let source = self.source_file.to_source_code(); + let location = source.source_location(range.start(), PositionEncoding::Utf8); + let end_location = source.source_location(range.end(), PositionEncoding::Utf8); + let except_handler = None; + self.current_block().instructions.push(ir::InstructionInfo { + instr: instr.into(), + arg, + target, + location, + end_location, + except_handler, + lineno_override: None, + cache_entries: 0, + }); + } + + fn emit_no_arg<I: Into<AnyInstruction>>(&mut self, ins: I) { + self._emit(ins, OpArg::NULL, BlockIdx::NULL) + } + + fn emit_arg<A: OpArgType, T: EmitArg<A>, I: Into<AnyInstruction>>( + &mut self, + arg: T, + f: impl FnOnce(OpArgMarker<A>) -> I, + ) { + let (op, arg, target) = arg.emit(f); + self._emit(op, arg, target) + } + + // fn block_done() + + fn arg_constant(&mut self, constant: ConstantData) -> oparg::ConstIdx { + let info = self.current_code_info(); + info.metadata.consts.insert_full(constant).0.to_u32().into() + } + + fn emit_load_const(&mut self, constant: ConstantData) { + let idx = self.arg_constant(constant); + self.emit_arg(idx, |consti| Instruction::LoadConst { consti }) + } + + fn emit_return_const(&mut self, constant: ConstantData) { + self.emit_load_const(constant); + emit!(self, Instruction::ReturnValue) + } + + fn emit_end_async_for(&mut self, send_target: BlockIdx) { + self._emit(Instruction::EndAsyncFor, OpArg::NULL, send_target); + } + + /// Emit LOAD_ATTR for attribute access (method=false). + /// Encodes: (name_idx << 1) | 0 + fn emit_load_attr(&mut self, name_idx: u32) { + let encoded = LoadAttr::builder() + .name_idx(name_idx) + .is_method(false) + .build(); + self.emit_arg(encoded, |namei| Instruction::LoadAttr { namei }) + } + + /// Emit LOAD_ATTR with method flag set (for method calls). + /// Encodes: (name_idx << 1) | 1 + fn emit_load_attr_method(&mut self, name_idx: u32) { + let encoded = LoadAttr::builder() + .name_idx(name_idx) + .is_method(true) + .build(); + self.emit_arg(encoded, |namei| Instruction::LoadAttr { namei }) + } + + /// Emit LOAD_GLOBAL. + /// Encodes: (name_idx << 1) | push_null_bit + fn emit_load_global(&mut self, name_idx: u32, push_null: bool) { + let encoded = (name_idx << 1) | u32::from(push_null); + self.emit_arg(encoded, |namei| Instruction::LoadGlobal { namei }); + } + + /// Emit LOAD_SUPER_ATTR for 2-arg super().attr access. + /// Encodes: (name_idx << 2) | 0b10 (method=0, class=1) + fn emit_load_super_attr(&mut self, name_idx: u32) { + let encoded = LoadSuperAttr::builder() + .name_idx(name_idx) + .is_load_method(false) + .has_class(true) + .build(); + self.emit_arg(encoded, |namei| Instruction::LoadSuperAttr { namei }) + } + + /// Emit LOAD_SUPER_ATTR for 2-arg super().method() call. + /// Encodes: (name_idx << 2) | 0b11 (method=1, class=1) + fn emit_load_super_method(&mut self, name_idx: u32) { + let encoded = LoadSuperAttr::builder() + .name_idx(name_idx) + .is_load_method(true) + .has_class(true) + .build(); + self.emit_arg(encoded, |namei| Instruction::LoadSuperAttr { namei }) + } + + /// Emit LOAD_SUPER_ATTR for 0-arg super().attr access. + /// Encodes: (name_idx << 2) | 0b00 (method=0, class=0) + fn emit_load_zero_super_attr(&mut self, name_idx: u32) { + let encoded = LoadSuperAttr::builder() + .name_idx(name_idx) + .is_load_method(false) + .has_class(false) + .build(); + self.emit_arg(encoded, |namei| Instruction::LoadSuperAttr { namei }) + } + + /// Emit LOAD_SUPER_ATTR for 0-arg super().method() call. + /// Encodes: (name_idx << 2) | 0b01 (method=1, class=0) + fn emit_load_zero_super_method(&mut self, name_idx: u32) { + let encoded = LoadSuperAttr::builder() + .name_idx(name_idx) + .is_load_method(true) + .has_class(false) + .build(); + self.emit_arg(encoded, |namei| Instruction::LoadSuperAttr { namei }) + } + + fn emit_return_value(&mut self) { + emit!(self, Instruction::ReturnValue) + } + + fn current_code_info(&mut self) -> &mut ir::CodeInfo { + self.code_stack.last_mut().expect("no code on stack") + } + + /// Enter a conditional block (if/for/while/match/try/with) + /// PEP 649: Track conditional annotation context + fn enter_conditional_block(&mut self) { + self.current_code_info().in_conditional_block += 1; + } + + /// Leave a conditional block + fn leave_conditional_block(&mut self) { + let code_info = self.current_code_info(); + debug_assert!(code_info.in_conditional_block > 0); + code_info.in_conditional_block -= 1; + } + + /// Compile break or continue statement with proper fblock cleanup. + /// compiler_break, compiler_continue + /// This handles unwinding through With blocks and exception handlers. + fn compile_break_continue( + &mut self, + range: ruff_text_size::TextRange, + is_break: bool, + ) -> CompileResult<()> { + // unwind_fblock_stack + // We need to unwind fblocks and compile cleanup code. For FinallyTry blocks, + // we need to compile the finally body inline, but we must temporarily pop + // the fblock so that nested break/continue in the finally body don't see it. + + // First, find the loop + let code = self.current_code_info(); + let mut loop_idx = None; + let mut is_for_loop = false; + + for i in (0..code.fblock.len()).rev() { + match code.fblock[i].fb_type { + FBlockType::WhileLoop => { + loop_idx = Some(i); + is_for_loop = false; + break; + } + FBlockType::ForLoop => { + loop_idx = Some(i); + is_for_loop = true; + break; + } + FBlockType::ExceptionGroupHandler => { + return Err( + self.error_ranged(CodegenErrorType::BreakContinueReturnInExceptStar, range) + ); + } + _ => {} + } + } + + let Some(loop_idx) = loop_idx else { + if is_break { + return Err(self.error_ranged(CodegenErrorType::InvalidBreak, range)); + } else { + return Err(self.error_ranged(CodegenErrorType::InvalidContinue, range)); + } + }; + + let loop_block = code.fblock[loop_idx].fb_block; + let exit_block = code.fblock[loop_idx].fb_exit; + + // Collect the fblocks we need to unwind through, from top down to (but not including) the loop + #[derive(Clone)] + enum UnwindAction { + With { + is_async: bool, + }, + HandlerCleanup { + name: Option<String>, + }, + TryExcept, + FinallyTry { + body: Vec<ruff_python_ast::Stmt>, + fblock_idx: usize, + }, + FinallyEnd, + PopValue, // Pop return value when continue/break cancels a return + } + let mut unwind_actions = Vec::new(); + + { + let code = self.current_code_info(); + for i in (loop_idx + 1..code.fblock.len()).rev() { + match code.fblock[i].fb_type { + FBlockType::With => { + unwind_actions.push(UnwindAction::With { is_async: false }); + } + FBlockType::AsyncWith => { + unwind_actions.push(UnwindAction::With { is_async: true }); + } + FBlockType::HandlerCleanup => { + let name = match &code.fblock[i].fb_datum { + FBlockDatum::ExceptionName(name) => Some(name.clone()), + _ => None, + }; + unwind_actions.push(UnwindAction::HandlerCleanup { name }); + } + FBlockType::TryExcept => { + unwind_actions.push(UnwindAction::TryExcept); + } + FBlockType::FinallyTry => { + // Need to execute finally body before break/continue + if let FBlockDatum::FinallyBody(ref body) = code.fblock[i].fb_datum { + unwind_actions.push(UnwindAction::FinallyTry { + body: body.clone(), + fblock_idx: i, + }); + } + } + FBlockType::FinallyEnd => { + // Inside finally block reached via exception - need to pop exception + unwind_actions.push(UnwindAction::FinallyEnd); + } + FBlockType::PopValue => { + // Pop the return value that was saved on stack + unwind_actions.push(UnwindAction::PopValue); + } + _ => {} + } + } + } + + // Emit cleanup for each fblock + for action in unwind_actions { + match action { + UnwindAction::With { is_async } => { + // codegen_unwind_fblock(WITH/ASYNC_WITH) + emit!(self, PseudoInstruction::PopBlock); + // compiler_call_exit_with_nones + emit!(self, Instruction::PushNull); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + self.emit_load_const(ConstantData::None); + emit!(self, Instruction::Call { argc: 3 }); + + if is_async { + emit!(self, Instruction::GetAwaitable { r#where: 2 }); + self.emit_load_const(ConstantData::None); + let _ = self.compile_yield_from_sequence(true)?; + } + + emit!(self, Instruction::PopTop); + } + UnwindAction::HandlerCleanup { ref name } => { + // codegen_unwind_fblock(HANDLER_CLEANUP) + if name.is_some() { + // Named handler: PopBlock for inner SETUP_CLEANUP + emit!(self, PseudoInstruction::PopBlock); + } + // PopBlock for outer SETUP_CLEANUP (ExceptionHandler) + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::PopExcept); + if let Some(name) = name { + self.emit_load_const(ConstantData::None); + self.store_name(name)?; + self.compile_name(name, NameUsage::Delete)?; + } + } + UnwindAction::TryExcept => { + // codegen_unwind_fblock(TRY_EXCEPT) + emit!(self, PseudoInstruction::PopBlock); + } + UnwindAction::FinallyTry { body, fblock_idx } => { + // codegen_unwind_fblock(FINALLY_TRY) + emit!(self, PseudoInstruction::PopBlock); + + // compile finally body inline + // Temporarily pop the FinallyTry fblock so nested break/continue + // in the finally body won't see it again. + let code = self.current_code_info(); + let saved_fblock = code.fblock.remove(fblock_idx); + + self.compile_statements(&body)?; + + // Restore the fblock (though this break/continue will jump away, + // this keeps the fblock stack consistent for error checking) + let code = self.current_code_info(); + code.fblock.insert(fblock_idx, saved_fblock); + } + UnwindAction::FinallyEnd => { + // codegen_unwind_fblock(FINALLY_END) + emit!(self, Instruction::PopTop); // exc_value + emit!(self, PseudoInstruction::PopBlock); + emit!(self, Instruction::PopExcept); + } + UnwindAction::PopValue => { + // Pop the return value - continue/break cancels the pending return + emit!(self, Instruction::PopTop); + } + } + } + + // For break in a for loop, pop the iterator + if is_break && is_for_loop { + emit!(self, Instruction::PopIter); + } + + // Jump to target + let target = if is_break { exit_block } else { loop_block }; + emit!(self, PseudoInstruction::Jump { delta: target }); + + Ok(()) + } + + fn current_block(&mut self) -> &mut ir::Block { + let info = self.current_code_info(); + &mut info.blocks[info.current_block] + } + + fn new_block(&mut self) -> BlockIdx { + let code = self.current_code_info(); + let idx = BlockIdx::new(code.blocks.len().to_u32()); + code.blocks.push(ir::Block::default()); + idx + } + + fn switch_to_block(&mut self, block: BlockIdx) { + let code = self.current_code_info(); + let prev = code.current_block; + assert_ne!(prev, block, "recursive switching {prev:?} -> {block:?}"); + assert_eq!( + code.blocks[block].next, + BlockIdx::NULL, + "switching {prev:?} -> {block:?} to completed block" + ); + let prev_block = &mut code.blocks[prev.idx()]; + assert_eq!( + u32::from(prev_block.next), + u32::MAX, + "switching {prev:?} -> {block:?} from block that's already got a next" + ); + prev_block.next = block; + code.current_block = block; + } + + const fn set_source_range(&mut self, range: TextRange) { + self.current_source_range = range; + } + + fn get_source_line_number(&mut self) -> OneIndexed { + self.source_file + .to_source_code() + .line_index(self.current_source_range.start()) + } + + fn mark_generator(&mut self) { + self.current_code_info().flags |= bytecode::CodeFlags::GENERATOR + } + + /// Whether the expression contains an await expression and + /// thus requires the function to be async. + /// + /// Both: + /// ```py + /// async with: ... + /// async for: ... + /// ``` + /// are statements, so we won't check for them here + fn contains_await(expression: &ast::Expr) -> bool { + use ast::visitor::Visitor; + + #[derive(Default)] + struct AwaitVisitor { + found: bool, + } + + impl ast::visitor::Visitor<'_> for AwaitVisitor { + fn visit_expr(&mut self, expr: &ast::Expr) { + if self.found { + return; + } + + match expr { + ast::Expr::Await(_) => self.found = true, + // Note: We do NOT check for async comprehensions here. + // Async list/set/dict comprehensions are handled by compile_comprehension + // which already awaits the result. A generator expression containing + // an async comprehension as its element does NOT become an async generator, + // because the async comprehension is awaited when evaluating the element. + _ => ast::visitor::walk_expr(self, expr), + } + } + } + + let mut visitor = AwaitVisitor::default(); + visitor.visit_expr(expression); + visitor.found + } + + /// Check if any of the generators (except the first one's iter) contains an await expression. + /// The first generator's iter is evaluated outside the comprehension scope. + fn generators_contain_await(generators: &[ast::Comprehension]) -> bool { + for (i, generator) in generators.iter().enumerate() { + // First generator's iter is evaluated outside the comprehension + if i > 0 && Self::contains_await(&generator.iter) { + return true; + } + // Check ifs in all generators + for if_expr in &generator.ifs { + if Self::contains_await(if_expr) { + return true; + } + } + } + false + } + + fn compile_expr_fstring(&mut self, fstring: &ast::ExprFString) -> CompileResult<()> { + let fstring = &fstring.value; + for part in fstring { + self.compile_fstring_part(part)?; + } + let part_count: u32 = fstring + .iter() + .len() + .try_into() + .expect("BuildString size overflowed"); + if part_count > 1 { + emit!(self, Instruction::BuildString { count: part_count }); + } + + Ok(()) + } + + fn compile_fstring_part(&mut self, part: &ast::FStringPart) -> CompileResult<()> { + match part { + ast::FStringPart::Literal(string) => { + if string.value.contains(char::REPLACEMENT_CHARACTER) { + // might have a surrogate literal; should reparse to be sure + let source = self.source_file.slice(string.range); + let value = + crate::string_parser::parse_string_literal(source, string.flags.into()); + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); + } else { + self.emit_load_const(ConstantData::Str { + value: string.value.to_string().into(), + }); + } + Ok(()) + } + ast::FStringPart::FString(fstring) => self.compile_fstring(fstring), + } + } + + fn compile_fstring(&mut self, fstring: &ast::FString) -> CompileResult<()> { + self.compile_fstring_elements(fstring.flags, &fstring.elements) + } + + fn compile_fstring_elements( + &mut self, + flags: ast::FStringFlags, + fstring_elements: &ast::InterpolatedStringElements, + ) -> CompileResult<()> { + let mut element_count = 0; + for element in fstring_elements { + element_count += 1; + match element { + ast::InterpolatedStringElement::Literal(string) => { + if string.value.contains(char::REPLACEMENT_CHARACTER) { + // might have a surrogate literal; should reparse to be sure + let source = self.source_file.slice(string.range); + let value = crate::string_parser::parse_fstring_literal_element( + source.into(), + flags.into(), + ); + self.emit_load_const(ConstantData::Str { + value: value.into(), + }); + } else { + self.emit_load_const(ConstantData::Str { + value: string.value.to_string().into(), + }); + } + } + ast::InterpolatedStringElement::Interpolation(fstring_expr) => { + let mut conversion = match fstring_expr.conversion { + ast::ConversionFlag::None => ConvertValueOparg::None, + ast::ConversionFlag::Str => ConvertValueOparg::Str, + ast::ConversionFlag::Repr => ConvertValueOparg::Repr, + ast::ConversionFlag::Ascii => ConvertValueOparg::Ascii, + }; + + if let Some(ast::DebugText { leading, trailing }) = &fstring_expr.debug_text { + let range = fstring_expr.expression.range(); + let source = self.source_file.slice(range); + let text = [ + strip_fstring_debug_comments(leading).as_str(), + source, + strip_fstring_debug_comments(trailing).as_str(), + ] + .concat(); + + self.emit_load_const(ConstantData::Str { value: text.into() }); + element_count += 1; + + // If debug text is present, apply repr conversion when no `format_spec` specified. + // See action_helpers.c: fstring_find_expr_replacement + if matches!( + (conversion, &fstring_expr.format_spec), + (ConvertValueOparg::None, None) + ) { + conversion = ConvertValueOparg::Repr; + } + } + + self.compile_expression(&fstring_expr.expression)?; + + match conversion { + ConvertValueOparg::None => {} + ConvertValueOparg::Str + | ConvertValueOparg::Repr + | ConvertValueOparg::Ascii => { + emit!(self, Instruction::ConvertValue { oparg: conversion }) + } + } + + match &fstring_expr.format_spec { + Some(format_spec) => { + self.compile_fstring_elements(flags, &format_spec.elements)?; + + emit!(self, Instruction::FormatWithSpec); + } + None => { + emit!(self, Instruction::FormatSimple); + } + } + } + } + } + + if element_count == 0 { + // ensure to put an empty string on the stack if there aren't any fstring elements + self.emit_load_const(ConstantData::Str { + value: Wtf8Buf::new(), + }); + } else if element_count > 1 { + emit!( + self, + Instruction::BuildString { + count: element_count + } + ); + } + + Ok(()) + } + + fn compile_expr_tstring(&mut self, expr_tstring: &ast::ExprTString) -> CompileResult<()> { + // ast::TStringValue can contain multiple ast::TString parts (implicit concatenation) + // Each ast::TString part should be compiled and the results merged into a single Template + let tstring_value = &expr_tstring.value; + + // Collect all strings and compile all interpolations + let mut all_strings: Vec<Wtf8Buf> = Vec::new(); + let mut current_string = Wtf8Buf::new(); + let mut interp_count: u32 = 0; + + for tstring in tstring_value.iter() { + self.compile_tstring_into( + tstring, + &mut all_strings, + &mut current_string, + &mut interp_count, + )?; + } + + // Add trailing string + all_strings.push(core::mem::take(&mut current_string)); + + // Now build the Template: + // Stack currently has all interpolations from compile_tstring_into calls + + // 1. Build interpolations tuple from the interpolations on the stack + emit!( + self, + Instruction::BuildTuple { + count: interp_count + } + ); + + // 2. Load all string parts + let string_count: u32 = all_strings + .len() + .try_into() + .expect("t-string string count overflowed"); + for s in &all_strings { + self.emit_load_const(ConstantData::Str { value: s.clone() }); + } + + // 3. Build strings tuple + emit!( + self, + Instruction::BuildTuple { + count: string_count + } + ); + + // 4. Swap so strings is below interpolations: [interps, strings] -> [strings, interps] + emit!(self, Instruction::Swap { i: 2 }); + + // 5. Build the Template + emit!(self, Instruction::BuildTemplate); + + Ok(()) + } + + fn compile_tstring_into( + &mut self, + tstring: &ast::TString, + strings: &mut Vec<Wtf8Buf>, + current_string: &mut Wtf8Buf, + interp_count: &mut u32, + ) -> CompileResult<()> { + for element in &tstring.elements { + match element { + ast::InterpolatedStringElement::Literal(lit) => { + // Accumulate literal parts into current_string + current_string.push_str(&lit.value); + } + ast::InterpolatedStringElement::Interpolation(interp) => { + // Finish current string segment + strings.push(core::mem::take(current_string)); + + // Compile the interpolation value + self.compile_expression(&interp.expression)?; + + // Load the expression source string, including any + // whitespace between '{' and the expression start + let expr_range = interp.expression.range(); + let expr_source = if interp.range.start() < expr_range.start() + && interp.range.end() >= expr_range.end() + { + let after_brace = interp.range.start() + TextSize::new(1); + self.source_file + .slice(TextRange::new(after_brace, expr_range.end())) + } else { + // Fallback for programmatically constructed ASTs with dummy ranges + self.source_file.slice(expr_range) + }; + self.emit_load_const(ConstantData::Str { + value: expr_source.to_string().into(), + }); + + // Determine conversion code + let conversion: u32 = match interp.conversion { + ast::ConversionFlag::None => 0, + ast::ConversionFlag::Str => 1, + ast::ConversionFlag::Repr => 2, + ast::ConversionFlag::Ascii => 3, + }; + + // Handle format_spec + let has_format_spec = interp.format_spec.is_some(); + if let Some(format_spec) = &interp.format_spec { + // Compile format_spec as a string using fstring element compilation + // Use default ast::FStringFlags since format_spec syntax is independent of t-string flags + self.compile_fstring_elements( + ast::FStringFlags::empty(), + &format_spec.elements, + )?; + } + + // Emit BUILD_INTERPOLATION + // oparg encoding: (conversion << 2) | has_format_spec + let format = (conversion << 2) | u32::from(has_format_spec); + emit!(self, Instruction::BuildInterpolation { format }); + + *interp_count += 1; + } + } + } + + Ok(()) + } +} + +trait EmitArg<Arg: OpArgType> { + fn emit<I: Into<AnyInstruction>>( + self, + f: impl FnOnce(OpArgMarker<Arg>) -> I, + ) -> (AnyInstruction, OpArg, BlockIdx); +} + +impl<T: OpArgType> EmitArg<T> for T { + fn emit<I: Into<AnyInstruction>>( + self, + f: impl FnOnce(OpArgMarker<T>) -> I, + ) -> (AnyInstruction, OpArg, BlockIdx) { + let (marker, arg) = OpArgMarker::new(self); + (f(marker).into(), arg, BlockIdx::NULL) + } +} + +impl EmitArg<bytecode::Label> for BlockIdx { + fn emit<I: Into<AnyInstruction>>( + self, + f: impl FnOnce(OpArgMarker<bytecode::Label>) -> I, + ) -> (AnyInstruction, OpArg, BlockIdx) { + (f(OpArgMarker::marker()).into(), OpArg::NULL, self) + } +} + +/// Strips leading whitespace from a docstring. +/// +/// `inspect.cleandoc` is a good reference, but has a few incompatibilities. +// = _PyCompile_CleanDoc +fn clean_doc(doc: &str) -> String { + let doc = expandtabs(doc, 8); + // First pass: find minimum indentation of any non-blank lines + // after first line. + let margin = doc + .lines() + // Find the non-blank lines + .filter(|line| !line.trim().is_empty()) + // get the one with the least indentation + .map(|line| line.chars().take_while(|c| c == &' ').count()) + .min(); + if let Some(margin) = margin { + let mut cleaned = String::with_capacity(doc.len()); + // copy first line without leading whitespace + if let Some(first_line) = doc.lines().next() { + cleaned.push_str(first_line.trim_start()); + } + // copy subsequent lines without margin. + for line in doc.split('\n').skip(1) { + cleaned.push('\n'); + let cleaned_line = line.chars().skip(margin).collect::<String>(); + cleaned.push_str(&cleaned_line); + } + + cleaned + } else { + doc.to_owned() + } +} + +// copied from rustpython_common::str, so we don't have to depend on it just for this function +fn expandtabs(input: &str, tab_size: usize) -> String { + let tab_stop = tab_size; + let mut expanded_str = String::with_capacity(input.len()); + let mut tab_size = tab_stop; + let mut col_count = 0usize; + for ch in input.chars() { + match ch { + '\t' => { + let num_spaces = tab_size - col_count; + col_count += num_spaces; + let expand = " ".repeat(num_spaces); + expanded_str.push_str(&expand); + } + '\r' | '\n' => { + expanded_str.push(ch); + col_count = 0; + tab_size = 0; + } + _ => { + expanded_str.push(ch); + col_count += 1; + } + } + if col_count >= tab_size { + tab_size += tab_stop; + } + } + expanded_str +} + +fn split_doc<'a>(body: &'a [ast::Stmt], opts: &CompileOpts) -> (Option<String>, &'a [ast::Stmt]) { + if let Some((ast::Stmt::Expr(expr), body_rest)) = body.split_first() { + let doc_comment = match &*expr.value { + ast::Expr::StringLiteral(value) => Some(&value.value), + // f-strings are not allowed in Python doc comments. + ast::Expr::FString(_) => None, + _ => None, + }; + if let Some(doc) = doc_comment { + return if opts.optimize < 2 { + (Some(clean_doc(doc.to_str())), body_rest) + } else { + (None, body_rest) + }; + } + } + (None, body) +} + +pub fn ruff_int_to_bigint(int: &ast::Int) -> Result<BigInt, CodegenErrorType> { + if let Some(small) = int.as_u64() { + Ok(BigInt::from(small)) + } else { + parse_big_integer(int) + } +} + +/// Converts a `ruff` ast integer into a `BigInt`. +/// Unlike small integers, big integers may be stored in one of four possible radix representations. +fn parse_big_integer(int: &ast::Int) -> Result<BigInt, CodegenErrorType> { + // TODO: Improve ruff API + // Can we avoid this copy? + let s = format!("{int}"); + let mut s = s.as_str(); + // See: https://peps.python.org/pep-0515/#literal-grammar + let radix = match s.get(0..2) { + Some("0b" | "0B") => { + s = s.get(2..).unwrap_or(s); + 2 + } + Some("0o" | "0O") => { + s = s.get(2..).unwrap_or(s); + 8 + } + Some("0x" | "0X") => { + s = s.get(2..).unwrap_or(s); + 16 + } + _ => 10, + }; + + BigInt::from_str_radix(s, radix).map_err(|e| { + CodegenErrorType::SyntaxError(format!( + "unparsed integer literal (radix {radix}): {s} ({e})" + )) + }) +} + +// Note: Not a good practice in general. Keep this trait private only for compiler +trait ToU32 { + fn to_u32(self) -> u32; +} + +impl ToU32 for usize { + fn to_u32(self) -> u32 { + self.try_into().unwrap() + } +} + +/// Strip Python comments from f-string debug text (leading/trailing around `=`). +/// A comment starts with `#` and extends to the end of the line. +/// The newline character itself is preserved. +fn strip_fstring_debug_comments(text: &str) -> String { + let mut result = String::with_capacity(text.len()); + let mut in_comment = false; + for ch in text.chars() { + if in_comment { + if ch == '\n' { + in_comment = false; + result.push(ch); + } + } else if ch == '#' { + in_comment = true; + } else { + result.push(ch); + } + } + result +} + +#[cfg(test)] +mod ruff_tests { + use super::*; + use ast::name::Name; + + /// Test if the compiler can correctly identify fstrings containing an `await` expression. + #[test] + fn test_fstring_contains_await() { + let range = TextRange::default(); + let flags = ast::FStringFlags::empty(); + + // f'{x}' + let expr_x = ast::Expr::Name(ast::ExprName { + node_index: ast::AtomicNodeIndex::NONE, + range, + id: Name::new("x"), + ctx: ast::ExprContext::Load, + }); + let not_present = &ast::Expr::FString(ast::ExprFString { + node_index: ast::AtomicNodeIndex::NONE, + range, + value: ast::FStringValue::single(ast::FString { + node_index: ast::AtomicNodeIndex::NONE, + range, + elements: vec![ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + node_index: ast::AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_x), + debug_text: None, + conversion: ast::ConversionFlag::None, + format_spec: None, + }, + )] + .into(), + flags, + }), + }); + assert!(!Compiler::contains_await(not_present)); + + // f'{await x}' + let expr_await_x = ast::Expr::Await(ast::ExprAwait { + node_index: ast::AtomicNodeIndex::NONE, + range, + value: Box::new(ast::Expr::Name(ast::ExprName { + node_index: ast::AtomicNodeIndex::NONE, + range, + id: Name::new("x"), + ctx: ast::ExprContext::Load, + })), + }); + let present = &ast::Expr::FString(ast::ExprFString { + node_index: ast::AtomicNodeIndex::NONE, + range, + value: ast::FStringValue::single(ast::FString { + node_index: ast::AtomicNodeIndex::NONE, + range, + elements: vec![ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + node_index: ast::AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_await_x), + debug_text: None, + conversion: ast::ConversionFlag::None, + format_spec: None, + }, + )] + .into(), + flags, + }), + }); + assert!(Compiler::contains_await(present)); + + // f'{x:{await y}}' + let expr_x = ast::Expr::Name(ast::ExprName { + node_index: ast::AtomicNodeIndex::NONE, + range, + id: Name::new("x"), + ctx: ast::ExprContext::Load, + }); + let expr_await_y = ast::Expr::Await(ast::ExprAwait { + node_index: ast::AtomicNodeIndex::NONE, + range, + value: Box::new(ast::Expr::Name(ast::ExprName { + node_index: ast::AtomicNodeIndex::NONE, + range, + id: Name::new("y"), + ctx: ast::ExprContext::Load, + })), + }); + let present = &ast::Expr::FString(ast::ExprFString { + node_index: ast::AtomicNodeIndex::NONE, + range, + value: ast::FStringValue::single(ast::FString { + node_index: ast::AtomicNodeIndex::NONE, + range, + elements: vec![ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + node_index: ast::AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_x), + debug_text: None, + conversion: ast::ConversionFlag::None, + format_spec: Some(Box::new(ast::InterpolatedStringFormatSpec { + node_index: ast::AtomicNodeIndex::NONE, + range, + elements: vec![ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + node_index: ast::AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_await_y), + debug_text: None, + conversion: ast::ConversionFlag::None, + format_spec: None, + }, + )] + .into(), + })), + }, + )] + .into(), + flags, + }), + }); + assert!(Compiler::contains_await(present)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustpython_compiler_core::SourceFileBuilder; + + fn compile_exec(source: &str) -> CodeObject { + let opts = CompileOpts::default(); + compile_exec_with_options(source, opts) + } + + fn compile_exec_optimized(source: &str) -> CodeObject { + let opts = CompileOpts { + optimize: 1, + ..CompileOpts::default() + }; + compile_exec_with_options(source, opts) + } + + fn compile_exec_with_options(source: &str, opts: CompileOpts) -> CodeObject { + let source_file = SourceFileBuilder::new("source_path", source).finish(); + let parsed = ruff_python_parser::parse( + source_file.source_text(), + ruff_python_parser::Mode::Module.into(), + ) + .unwrap(); + let ast = parsed.into_syntax(); + let ast = match ast { + ruff_python_ast::Mod::Module(stmts) => stmts, + _ => unreachable!(), + }; + let symbol_table = SymbolTable::scan_program(&ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned())) + .unwrap(); + let mut compiler = Compiler::new(opts, source_file, "<module>".to_owned()); + compiler.compile_program(&ast, symbol_table).unwrap(); + compiler.exit_scope() + } + + macro_rules! assert_dis_snapshot { + ($value:expr) => { + insta::assert_snapshot!( + insta::internals::AutoName, + $value.display_expand_code_objects().to_string(), + stringify!($value) + ) + }; + } + + #[test] + fn test_if_ors() { + assert_dis_snapshot!(compile_exec( + "\ +if True or False or False: + pass +" + )); + } + + #[test] + fn test_if_ands() { + assert_dis_snapshot!(compile_exec( + "\ +if True and False and False: + pass +" + )); + } + + #[test] + fn test_if_mixed() { + assert_dis_snapshot!(compile_exec( + "\ +if (True and False) or (False and True): + pass +" + )); + } + + #[test] + fn test_nested_bool_op() { + assert_dis_snapshot!(compile_exec( + "\ +x = Test() and False or False +" + )); + } + + #[test] + fn test_const_bool_not_op() { + assert_dis_snapshot!(compile_exec_optimized( + "\ +x = not True +" + )); + } + + #[test] + fn test_nested_double_async_with() { + assert_dis_snapshot!(compile_exec( + "\ +async def test(): + for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): + with self.subTest(type=type(stop_exc)): + try: + async with egg(): + raise stop_exc + except Exception as ex: + self.assertIs(ex, stop_exc) + else: + self.fail(f'{stop_exc} was suppressed') +" + )); + } + + #[test] + fn test_optimized_assert_preserves_nested_scope_order() { + compile_exec_optimized( + "\ +class S: + def f(self, sequence): + _formats = [self._types_mapping[type(item)] for item in sequence] + _list_len = len(_formats) + assert sum(len(fmt) <= 8 for fmt in _formats) == _list_len + _recreation_codes = [self._extract_recreation_code(item) for item in sequence] +", + ); + } + + #[test] + fn test_optimized_assert_with_nested_scope_in_first_iter() { + // First iterator of a comprehension is evaluated in the enclosing + // scope, so nested scopes inside it (the generator here) must also + // be consumed when the assert is optimized away. + compile_exec_optimized( + "\ +def f(items): + assert [x for x in (y for y in items)] + return [x for x in items] +", + ); + } + + #[test] + fn test_optimized_assert_with_lambda_defaults() { + // Lambda default values are evaluated in the enclosing scope, + // so nested scopes inside defaults must be consumed. + compile_exec_optimized( + "\ +def f(items): + assert (lambda x=[i for i in items]: x)() + return [x for x in items] +", + ); + } +} diff --git a/crates/codegen/src/error.rs b/crates/codegen/src/error.rs new file mode 100644 index 00000000000..086f9dfd739 --- /dev/null +++ b/crates/codegen/src/error.rs @@ -0,0 +1,176 @@ +use alloc::fmt; +use core::fmt::Display; +use rustpython_compiler_core::SourceLocation; +use thiserror::Error; + +#[derive(Clone, Copy, Debug)] +pub enum PatternUnreachableReason { + NameCapture, + Wildcard, +} + +impl Display for PatternUnreachableReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NameCapture => write!(f, "name capture"), + Self::Wildcard => write!(f, "wildcard"), + } + } +} + +// pub type CodegenError = rustpython_parser_core::source_code::LocatedError<CodegenErrorType>; + +#[derive(Error, Debug)] +pub struct CodegenError { + pub location: Option<SourceLocation>, + #[source] + pub error: CodegenErrorType, + pub source_path: String, +} + +impl fmt::Display for CodegenError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO: + self.error.fmt(f) + } +} + +#[derive(Debug)] +#[non_exhaustive] +pub enum InternalError { + StackOverflow, + StackUnderflow, + MissingSymbol(String), +} + +impl Display for InternalError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::StackOverflow => write!(f, "stack overflow"), + Self::StackUnderflow => write!(f, "stack underflow"), + Self::MissingSymbol(s) => write!( + f, + "The symbol '{s}' must be present in the symbol table, even when it is undefined in python." + ), + } + } +} + +#[derive(Debug)] +#[non_exhaustive] +pub enum CodegenErrorType { + /// Invalid assignment, cannot store value in target. + Assign(&'static str), + /// Invalid delete + Delete(&'static str), + SyntaxError(String), + /// Multiple `*` detected + MultipleStarArgs, + /// Misplaced `*` expression + InvalidStarExpr, + /// Break statement outside of loop. + InvalidBreak, + /// Continue statement outside of loop. + InvalidContinue, + InvalidReturn, + InvalidYield, + InvalidYieldFrom, + InvalidAwait, + InvalidAsyncFor, + InvalidAsyncWith, + InvalidAsyncComprehension, + AsyncYieldFrom, + AsyncReturnValue, + InvalidFuturePlacement, + InvalidFutureFeature(String), + FunctionImportStar, + TooManyStarUnpack, + EmptyWithItems, + EmptyWithBody, + ForbiddenName, + DuplicateStore(String), + UnreachablePattern(PatternUnreachableReason), + RepeatedAttributePattern, + ConflictingNameBindPattern, + /// break/continue/return inside except* block + BreakContinueReturnInExceptStar, + NotImplementedYet, // RustPython marker for unimplemented features +} + +impl core::error::Error for CodegenErrorType {} + +impl fmt::Display for CodegenErrorType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use CodegenErrorType::*; + match self { + Assign(target) => write!(f, "cannot assign to {target}"), + Delete(target) => write!(f, "cannot delete {target}"), + SyntaxError(err) => write!(f, "{}", err.as_str()), + MultipleStarArgs => { + write!(f, "multiple starred expressions in assignment") + } + InvalidStarExpr => write!(f, "can't use starred expression here"), + InvalidBreak => write!(f, "'break' outside loop"), + InvalidContinue => write!(f, "'continue' outside loop"), + InvalidReturn => write!(f, "'return' outside function"), + InvalidYield => write!(f, "'yield' outside function"), + InvalidYieldFrom => write!(f, "'yield from' outside function"), + InvalidAwait => write!(f, "'await' outside async function"), + InvalidAsyncFor => write!(f, "'async for' outside async function"), + InvalidAsyncWith => write!(f, "'async with' outside async function"), + InvalidAsyncComprehension => { + write!( + f, + "asynchronous comprehension outside of an asynchronous function" + ) + } + AsyncYieldFrom => write!(f, "'yield from' inside async function"), + AsyncReturnValue => { + write!(f, "'return' with value inside async generator") + } + InvalidFuturePlacement => write!( + f, + "from __future__ imports must occur at the beginning of the file" + ), + InvalidFutureFeature(feat) => { + write!(f, "future feature {feat} is not defined") + } + FunctionImportStar => { + write!(f, "import * only allowed at module level") + } + TooManyStarUnpack => { + write!(f, "too many expressions in star-unpacking assignment") + } + EmptyWithItems => { + write!(f, "empty items on With") + } + EmptyWithBody => { + write!(f, "empty body on With") + } + ForbiddenName => { + write!(f, "forbidden attribute name") + } + DuplicateStore(s) => { + write!(f, "duplicate store {s}") + } + UnreachablePattern(reason) => { + write!(f, "{reason} makes remaining patterns unreachable") + } + RepeatedAttributePattern => { + write!(f, "attribute name repeated in class pattern") + } + ConflictingNameBindPattern => { + write!(f, "alternative patterns bind different names") + } + BreakContinueReturnInExceptStar => { + write!( + f, + "'break', 'continue' and 'return' cannot appear in an except* block" + ) + } + NotImplementedYet => { + write!(f, "RustPython does not implement this feature yet") + } + } + } +} diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs new file mode 100644 index 00000000000..67c60dd561d --- /dev/null +++ b/crates/codegen/src/ir.rs @@ -0,0 +1,1816 @@ +use alloc::collections::VecDeque; +use core::ops; + +use crate::{IndexMap, IndexSet, error::InternalError}; +use malachite_bigint::BigInt; +use num_traits::ToPrimitive; + +use rustpython_compiler_core::{ + OneIndexed, SourceLocation, + bytecode::{ + AnyInstruction, Arg, CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, + ExceptionTableEntry, InstrDisplayContext, Instruction, InstructionMetadata, Label, OpArg, + PseudoInstruction, PyCodeLocationInfoKind, encode_exception_table, oparg, + }, + varint::{write_signed_varint, write_varint}, +}; + +/// Location info for linetable generation (allows line 0 for RESUME) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct LineTableLocation { + line: i32, + end_line: i32, + col: i32, + end_col: i32, +} + +/// Metadata for a code unit +// = _PyCompile_CodeUnitMetadata +#[derive(Clone, Debug)] +pub struct CodeUnitMetadata { + pub name: String, // u_name (obj_name) + pub qualname: Option<String>, // u_qualname + pub consts: IndexSet<ConstantData>, // u_consts + pub names: IndexSet<String>, // u_names + pub varnames: IndexSet<String>, // u_varnames + pub cellvars: IndexSet<String>, // u_cellvars + pub freevars: IndexSet<String>, // u_freevars + pub fast_hidden: IndexMap<String, bool>, // u_fast_hidden + pub argcount: u32, // u_argcount + pub posonlyargcount: u32, // u_posonlyargcount + pub kwonlyargcount: u32, // u_kwonlyargcount + pub firstlineno: OneIndexed, // u_firstlineno +} +// use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct BlockIdx(u32); + +impl BlockIdx { + pub const NULL: Self = Self::new(u32::MAX); + + /// Creates a new instance of [`BlockIdx`] from a [`u32`]. + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value) + } + + /// Returns the inner value as a [`usize`]. + #[must_use] + pub const fn idx(self) -> usize { + self.0 as usize + } +} + +impl From<BlockIdx> for u32 { + fn from(block_idx: BlockIdx) -> Self { + block_idx.0 + } +} + +impl ops::Index<BlockIdx> for [Block] { + type Output = Block; + + fn index(&self, idx: BlockIdx) -> &Block { + &self[idx.idx()] + } +} + +impl ops::IndexMut<BlockIdx> for [Block] { + fn index_mut(&mut self, idx: BlockIdx) -> &mut Block { + &mut self[idx.idx()] + } +} + +impl ops::Index<BlockIdx> for Vec<Block> { + type Output = Block; + + fn index(&self, idx: BlockIdx) -> &Block { + &self[idx.idx()] + } +} + +impl ops::IndexMut<BlockIdx> for Vec<Block> { + fn index_mut(&mut self, idx: BlockIdx) -> &mut Block { + &mut self[idx.idx()] + } +} + +#[derive(Clone, Copy, Debug)] +pub struct InstructionInfo { + pub instr: AnyInstruction, + pub arg: OpArg, + pub target: BlockIdx, + pub location: SourceLocation, + pub end_location: SourceLocation, + pub except_handler: Option<ExceptHandlerInfo>, + /// Override line number for linetable (e.g., line 0 for module RESUME) + pub lineno_override: Option<i32>, + /// Number of CACHE code units emitted after this instruction + pub cache_entries: u32, +} + +/// Exception handler information for an instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ExceptHandlerInfo { + /// Block to jump to when exception occurs + pub handler_block: BlockIdx, + /// Stack depth at handler entry + pub stack_depth: u32, + /// Whether to push lasti before exception + pub preserve_lasti: bool, +} + +// spell-checker:ignore petgraph +// TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it +// might enable more analysis/optimizations +#[derive(Debug)] +pub struct Block { + pub instructions: Vec<InstructionInfo>, + pub next: BlockIdx, + // Post-codegen analysis fields (set by label_exception_targets) + /// Whether this block is an exception handler target (b_except_handler) + pub except_handler: bool, + /// Whether to preserve lasti for this handler block (b_preserve_lasti) + pub preserve_lasti: bool, + /// Stack depth at block entry, set by stack depth analysis + pub start_depth: Option<u32>, + /// Whether this block is only reachable via exception table (b_cold) + pub cold: bool, +} + +impl Default for Block { + fn default() -> Self { + Self { + instructions: Vec::new(), + next: BlockIdx::NULL, + except_handler: false, + preserve_lasti: false, + start_depth: None, + cold: false, + } + } +} + +pub struct CodeInfo { + pub flags: CodeFlags, + pub source_path: String, + pub private: Option<String>, // For private name mangling, mostly for class + + pub blocks: Vec<Block>, + pub current_block: BlockIdx, + + pub metadata: CodeUnitMetadata, + + // For class scopes: attributes accessed via self.X + pub static_attributes: Option<IndexSet<String>>, + + // True if compiling an inlined comprehension + pub in_inlined_comp: bool, + + // Block stack for tracking nested control structures + pub fblock: Vec<crate::compile::FBlockInfo>, + + // Reference to the symbol table for this scope + pub symbol_table_index: usize, + + // PEP 649: Track nesting depth inside conditional blocks (if/for/while/etc.) + // u_in_conditional_block + pub in_conditional_block: u32, + + // PEP 649: Next index for conditional annotation tracking + // u_next_conditional_annotation_index + pub next_conditional_annotation_index: u32, +} + +impl CodeInfo { + pub fn finalize_code( + mut self, + opts: &crate::compile::CompileOpts, + ) -> crate::InternalResult<CodeObject> { + // Always fold tuple constants + self.fold_tuple_constants(); + // Python only applies LOAD_SMALL_INT conversion to module-level code + // (not inside functions). Module code lacks OPTIMIZED flag. + // Note: RustPython incorrectly sets NEWLOCALS on modules, so only check OPTIMIZED + let is_module_level = !self.flags.contains(CodeFlags::OPTIMIZED); + if is_module_level { + self.convert_to_load_small_int(); + } + self.remove_unused_consts(); + self.remove_nops(); + + if opts.optimize > 0 { + self.dce(); + self.peephole_optimize(); + } + + // Always apply LOAD_FAST_BORROW optimization + self.optimize_load_fast_borrow(); + + // Post-codegen CFG analysis passes (flowgraph.c pipeline) + mark_except_handlers(&mut self.blocks); + label_exception_targets(&mut self.blocks); + push_cold_blocks_to_end(&mut self.blocks); + normalize_jumps(&mut self.blocks); + self.optimize_load_global_push_null(); + + let max_stackdepth = self.max_stackdepth()?; + let cell2arg = self.cell2arg(); + + let Self { + flags, + source_path, + private: _, // private is only used during compilation + + mut blocks, + current_block: _, + metadata, + static_attributes: _, + in_inlined_comp: _, + fblock: _, + symbol_table_index: _, + in_conditional_block: _, + next_conditional_annotation_index: _, + } = self; + + let CodeUnitMetadata { + name: obj_name, + qualname, + consts: constants, + names: name_cache, + varnames: varname_cache, + cellvars: cellvar_cache, + freevars: freevar_cache, + fast_hidden: _, + argcount: arg_count, + posonlyargcount: posonlyarg_count, + kwonlyargcount: kwonlyarg_count, + firstlineno: first_line_number, + } = metadata; + + let mut instructions = Vec::new(); + let mut locations = Vec::new(); + let mut linetable_locations: Vec<LineTableLocation> = Vec::new(); + + // Convert pseudo ops and remove resulting NOPs (keep line-marker NOPs) + convert_pseudo_ops(&mut blocks, varname_cache.len() as u32); + // Remove redundant NOPs, keeping line-marker NOPs only when + // they are needed to preserve tracing. + let mut block_order = Vec::new(); + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + block_order.push(current); + current = blocks[current.idx()].next; + } + for block_idx in block_order { + let bi = block_idx.idx(); + let mut src_instructions = core::mem::take(&mut blocks[bi].instructions); + let mut kept = Vec::with_capacity(src_instructions.len()); + let mut prev_lineno = -1i32; + + for src in 0..src_instructions.len() { + let instr = src_instructions[src]; + let lineno = instr + .lineno_override + .unwrap_or_else(|| instr.location.line.get() as i32); + let mut remove = false; + + if matches!(instr.instr.real(), Some(Instruction::Nop)) { + // Remove location-less NOPs. + if lineno < 0 || prev_lineno == lineno { + remove = true; + } + // Remove if the next instruction has same line or no line. + else if src < src_instructions.len() - 1 { + let next_lineno = + src_instructions[src + 1] + .lineno_override + .unwrap_or_else(|| { + src_instructions[src + 1].location.line.get() as i32 + }); + if next_lineno == lineno { + remove = true; + } else if next_lineno < 0 { + src_instructions[src + 1].lineno_override = Some(lineno); + remove = true; + } + } + // Last instruction in block: compare with first real location + // in the next non-empty block. + else { + let mut next = blocks[bi].next; + while next != BlockIdx::NULL && blocks[next.idx()].instructions.is_empty() { + next = blocks[next.idx()].next; + } + if next != BlockIdx::NULL { + let mut next_lineno = None; + for next_instr in &blocks[next.idx()].instructions { + let line = next_instr + .lineno_override + .unwrap_or_else(|| next_instr.location.line.get() as i32); + if matches!(next_instr.instr.real(), Some(Instruction::Nop)) + && line < 0 + { + continue; + } + next_lineno = Some(line); + break; + } + if next_lineno.is_some_and(|line| line == lineno) { + remove = true; + } + } + } + } + + if !remove { + kept.push(instr); + prev_lineno = lineno; + } + } + + blocks[bi].instructions = kept; + } + + // Pre-compute cache_entries for real (non-pseudo) instructions + for block in blocks.iter_mut() { + for instr in &mut block.instructions { + if let AnyInstruction::Real(op) = instr.instr { + instr.cache_entries = op.cache_entries() as u32; + } + } + } + + let mut block_to_offset = vec![Label::new(0); blocks.len()]; + // block_to_index: maps block idx to instruction index (for exception table) + // This is the index into the final instructions array, including EXTENDED_ARG and CACHE + let mut block_to_index = vec![0u32; blocks.len()]; + // The offset (in code units) of END_SEND from SEND in the yield-from sequence. + const END_SEND_OFFSET: u32 = 5; + loop { + let mut num_instructions = 0; + for (idx, block) in iter_blocks(&blocks) { + block_to_offset[idx.idx()] = Label::new(num_instructions as u32); + // block_to_index uses the same value as block_to_offset but as u32 + // because lasti in frame.rs is the index into instructions array + // and instructions array index == byte offset (each instruction is 1 CodeUnit) + block_to_index[idx.idx()] = num_instructions as u32; + for instr in &block.instructions { + num_instructions += instr.arg.instr_size() + instr.cache_entries as usize; + } + } + + instructions.reserve_exact(num_instructions); + locations.reserve_exact(num_instructions); + + let mut recompile = false; + let mut next_block = BlockIdx(0); + while next_block != BlockIdx::NULL { + let block = &mut blocks[next_block]; + // Track current instruction offset for jump direction resolution + let mut current_offset = block_to_offset[next_block.idx()].as_u32(); + for info in &mut block.instructions { + let target = info.target; + let mut op = info.instr.expect_real(); + let old_arg_size = info.arg.instr_size(); + let old_cache_entries = info.cache_entries; + // Keep offsets fixed within this pass: changes in jump + // arg/cache sizes only take effect in the next iteration. + let offset_after = current_offset + old_arg_size as u32 + old_cache_entries; + + if target != BlockIdx::NULL { + let target_offset = block_to_offset[target.idx()].as_u32(); + // Direction must be based on concrete instruction offsets. + // Empty blocks can share offsets, so block-order-based resolution + // may classify some jumps incorrectly. + op = match op { + Instruction::JumpForward { .. } if target_offset <= current_offset => { + Instruction::JumpBackward { + delta: Arg::marker(), + } + } + Instruction::JumpBackward { .. } if target_offset > current_offset => { + Instruction::JumpForward { + delta: Arg::marker(), + } + } + Instruction::JumpBackwardNoInterrupt { .. } + if target_offset > current_offset => + { + Instruction::JumpForward { + delta: Arg::marker(), + } + } + _ => op, + }; + info.instr = op.into(); + let updated_cache = op.cache_entries() as u32; + recompile |= updated_cache != old_cache_entries; + info.cache_entries = updated_cache; + let new_arg = if matches!(op, Instruction::EndAsyncFor) { + let arg = offset_after + .checked_sub(target_offset + END_SEND_OFFSET) + .expect("END_ASYNC_FOR target must be before instruction"); + OpArg::new(arg) + } else if matches!( + op, + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + ) { + let arg = offset_after + .checked_sub(target_offset) + .expect("backward jump target must be before instruction"); + OpArg::new(arg) + } else { + let arg = target_offset + .checked_sub(offset_after) + .expect("forward jump target must be after instruction"); + OpArg::new(arg) + }; + recompile |= new_arg.instr_size() != old_arg_size; + info.arg = new_arg; + } + + let cache_count = info.cache_entries as usize; + let (extras, lo_arg) = info.arg.split(); + locations.extend(core::iter::repeat_n( + (info.location, info.end_location), + info.arg.instr_size() + cache_count, + )); + // Collect linetable locations with lineno_override support + let lt_loc = LineTableLocation { + line: info + .lineno_override + .unwrap_or_else(|| info.location.line.get() as i32), + end_line: info.end_location.line.get() as i32, + col: info.location.character_offset.to_zero_indexed() as i32, + end_col: info.end_location.character_offset.to_zero_indexed() as i32, + }; + linetable_locations.extend(core::iter::repeat_n(lt_loc, info.arg.instr_size())); + // CACHE entries inherit parent instruction's location + if cache_count > 0 { + linetable_locations.extend(core::iter::repeat_n(lt_loc, cache_count)); + } + instructions.extend( + extras + .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) + .chain([CodeUnit { op, arg: lo_arg }]), + ); + // Emit CACHE code units after the instruction (all zeroed) + if cache_count > 0 { + instructions.extend(core::iter::repeat_n( + CodeUnit::new(Instruction::Cache, 0.into()), + cache_count, + )); + } + current_offset = offset_after; + } + next_block = block.next; + } + + if !recompile { + break; + } + + instructions.clear(); + locations.clear(); + linetable_locations.clear(); + } + + // Generate linetable from linetable_locations (supports line 0 for RESUME) + let linetable = generate_linetable( + &linetable_locations, + first_line_number.get() as i32, + opts.debug_ranges, + ); + + // Generate exception table before moving source_path + let exceptiontable = generate_exception_table(&blocks, &block_to_index); + + Ok(CodeObject { + flags, + posonlyarg_count, + arg_count, + kwonlyarg_count, + source_path, + first_line_number: Some(first_line_number), + obj_name: obj_name.clone(), + qualname: qualname.unwrap_or(obj_name), + + max_stackdepth, + instructions: CodeUnits::from(instructions), + locations: locations.into_boxed_slice(), + constants: constants.into_iter().collect(), + names: name_cache.into_iter().collect(), + varnames: varname_cache.into_iter().collect(), + cellvars: cellvar_cache.into_iter().collect(), + freevars: freevar_cache.into_iter().collect(), + cell2arg, + linetable, + exceptiontable, + }) + } + + fn cell2arg(&self) -> Option<Box<[i32]>> { + if self.metadata.cellvars.is_empty() { + return None; + } + + let total_args = self.metadata.argcount + + self.metadata.kwonlyargcount + + self.flags.contains(CodeFlags::VARARGS) as u32 + + self.flags.contains(CodeFlags::VARKEYWORDS) as u32; + + let mut found_cellarg = false; + let cell2arg = self + .metadata + .cellvars + .iter() + .map(|var| { + self.metadata + .varnames + .get_index_of(var) + // check that it's actually an arg + .filter(|i| *i < total_args as usize) + .map_or(-1, |i| { + found_cellarg = true; + i as i32 + }) + }) + .collect::<Box<[_]>>(); + + if found_cellarg { Some(cell2arg) } else { None } + } + + fn dce(&mut self) { + for block in &mut self.blocks { + let mut last_instr = None; + for (i, ins) in block.instructions.iter().enumerate() { + if ins.instr.is_scope_exit() || ins.instr.is_unconditional_jump() { + last_instr = Some(i); + break; + } + } + if let Some(i) = last_instr { + block.instructions.truncate(i + 1); + } + } + } + + /// Constant folding: fold LOAD_CONST/LOAD_SMALL_INT + BUILD_TUPLE into LOAD_CONST tuple + /// fold_tuple_of_constants + fn fold_tuple_constants(&mut self) { + for block in &mut self.blocks { + let mut i = 0; + while i < block.instructions.len() { + let instr = &block.instructions[i]; + // Look for BUILD_TUPLE + let Some(Instruction::BuildTuple { .. }) = instr.instr.real() else { + i += 1; + continue; + }; + + let tuple_size = u32::from(instr.arg) as usize; + if tuple_size == 0 || i < tuple_size { + i += 1; + continue; + } + + // Check if all preceding instructions are constant-loading + let start_idx = i - tuple_size; + let mut elements = Vec::with_capacity(tuple_size); + let mut all_const = true; + + for j in start_idx..i { + let load_instr = &block.instructions[j]; + match load_instr.instr.real() { + Some(Instruction::LoadConst { .. }) => { + let const_idx = u32::from(load_instr.arg) as usize; + if let Some(constant) = + self.metadata.consts.get_index(const_idx).cloned() + { + elements.push(constant); + } else { + all_const = false; + break; + } + } + Some(Instruction::LoadSmallInt { .. }) => { + // arg is the i32 value stored as u32 (two's complement) + let value = u32::from(load_instr.arg) as i32; + elements.push(ConstantData::Integer { + value: BigInt::from(value), + }); + } + _ => { + all_const = false; + break; + } + } + } + + if !all_const { + i += 1; + continue; + } + + // Note: The first small int is added to co_consts during compilation + // (in compile_default_arguments). + // We don't need to add it here again. + + // Create tuple constant and add to consts + let tuple_const = ConstantData::Tuple { elements }; + let (const_idx, _) = self.metadata.consts.insert_full(tuple_const); + + // Replace preceding LOAD instructions with NOP at the + // BUILD_TUPLE location so remove_nops() can eliminate them. + let folded_loc = block.instructions[i].location; + for j in start_idx..i { + block.instructions[j].instr = Instruction::Nop.into(); + block.instructions[j].location = folded_loc; + } + + // Replace BUILD_TUPLE with LOAD_CONST + block.instructions[i].instr = Instruction::LoadConst { + consti: Arg::marker(), + } + .into(); + block.instructions[i].arg = OpArg::new(const_idx as u32); + + i += 1; + } + } + } + + /// Peephole optimization: combine consecutive instructions into super-instructions + fn peephole_optimize(&mut self) { + for block in &mut self.blocks { + let mut i = 0; + while i + 1 < block.instructions.len() { + let combined = { + let curr = &block.instructions[i]; + let next = &block.instructions[i + 1]; + + // Only combine if both are real instructions (not pseudo) + let (Some(curr_instr), Some(next_instr)) = + (curr.instr.real(), next.instr.real()) + else { + i += 1; + continue; + }; + + match (curr_instr, next_instr) { + // LoadFast + LoadFast -> LoadFastLoadFast (if both indices < 16) + (Instruction::LoadFast { .. }, Instruction::LoadFast { .. }) => { + let idx1 = u32::from(curr.arg); + let idx2 = u32::from(next.arg); + if idx1 < 16 && idx2 < 16 { + let packed = (idx1 << 4) | idx2; + Some(( + Instruction::LoadFastLoadFast { + var_nums: Arg::marker(), + }, + OpArg::new(packed), + )) + } else { + None + } + } + // StoreFast + StoreFast -> StoreFastStoreFast (if both indices < 16) + (Instruction::StoreFast { .. }, Instruction::StoreFast { .. }) => { + let idx1 = u32::from(curr.arg); + let idx2 = u32::from(next.arg); + if idx1 < 16 && idx2 < 16 { + let packed = (idx1 << 4) | idx2; + Some(( + Instruction::StoreFastStoreFast { + var_nums: Arg::marker(), + }, + OpArg::new(packed), + )) + } else { + None + } + } + (Instruction::LoadConst { consti }, Instruction::ToBool) => { + let consti = consti.get(curr.arg); + let constant = &self.metadata.consts[consti.as_usize()]; + if let ConstantData::Boolean { .. } = constant { + Some((curr_instr, OpArg::from(consti.as_u32()))) + } else { + None + } + } + (Instruction::LoadConst { consti }, Instruction::UnaryNot) => { + let constant = &self.metadata.consts[consti.get(curr.arg).as_usize()]; + match constant { + ConstantData::Boolean { value } => { + let (const_idx, _) = self + .metadata + .consts + .insert_full(ConstantData::Boolean { value: !value }); + Some(( + (Instruction::LoadConst { + consti: Arg::marker(), + }), + OpArg::new(const_idx as u32), + )) + } + _ => None, + } + } + _ => None, + } + }; + + if let Some((new_instr, new_arg)) = combined { + // Combine: keep first instruction's location, replace with combined instruction + block.instructions[i].instr = new_instr.into(); + block.instructions[i].arg = new_arg; + // Remove the second instruction + block.instructions.remove(i + 1); + // Don't increment i - check if we can combine again with the next instruction + } else { + i += 1; + } + } + } + } + + /// LOAD_GLOBAL <even> + PUSH_NULL -> LOAD_GLOBAL <odd>, NOP + fn optimize_load_global_push_null(&mut self) { + for block in &mut self.blocks { + let mut i = 0; + while i + 1 < block.instructions.len() { + let curr = &block.instructions[i]; + let next = &block.instructions[i + 1]; + + let (Some(Instruction::LoadGlobal { .. }), Some(Instruction::PushNull)) = + (curr.instr.real(), next.instr.real()) + else { + i += 1; + continue; + }; + + let oparg = u32::from(block.instructions[i].arg); + if (oparg & 1) != 0 { + i += 1; + continue; + } + + block.instructions[i].arg = OpArg::new(oparg | 1); + block.instructions.remove(i + 1); + } + } + } + + /// Convert LOAD_CONST for small integers to LOAD_SMALL_INT + /// maybe_instr_make_load_smallint + fn convert_to_load_small_int(&mut self) { + for block in &mut self.blocks { + for instr in &mut block.instructions { + // Check if it's a LOAD_CONST instruction + let Some(Instruction::LoadConst { .. }) = instr.instr.real() else { + continue; + }; + + // Get the constant value + let const_idx = u32::from(instr.arg) as usize; + let Some(constant) = self.metadata.consts.get_index(const_idx) else { + continue; + }; + + // Check if it's a small integer + let ConstantData::Integer { value } = constant else { + continue; + }; + + // Check if it's in small int range: -5 to 256 (_PY_IS_SMALL_INT) + if let Some(small) = value.to_i32().filter(|v| (-5..=256).contains(v)) { + // Convert LOAD_CONST to LOAD_SMALL_INT + instr.instr = Instruction::LoadSmallInt { i: Arg::marker() }.into(); + // The arg is the i32 value stored as u32 (two's complement) + instr.arg = OpArg::new(small as u32); + } + } + } + } + + /// Remove constants that are no longer referenced by LOAD_CONST instructions. + /// remove_unused_consts + fn remove_unused_consts(&mut self) { + let nconsts = self.metadata.consts.len(); + if nconsts == 0 { + return; + } + + // Mark used constants + // The first constant (index 0) is always kept (may be docstring) + let mut used = vec![false; nconsts]; + used[0] = true; + + for block in &self.blocks { + for instr in &block.instructions { + if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { + let idx = u32::from(instr.arg) as usize; + if idx < nconsts { + used[idx] = true; + } + } + } + } + + // Check if any constants can be removed + let n_used: usize = used.iter().filter(|&&u| u).count(); + if n_used == nconsts { + return; // Nothing to remove + } + + // Build old_to_new index mapping + let mut old_to_new = vec![0usize; nconsts]; + let mut new_idx = 0usize; + for (old_idx, &is_used) in used.iter().enumerate() { + if is_used { + old_to_new[old_idx] = new_idx; + new_idx += 1; + } + } + + // Build new consts list + let old_consts: Vec<_> = self.metadata.consts.iter().cloned().collect(); + self.metadata.consts.clear(); + for (old_idx, constant) in old_consts.into_iter().enumerate() { + if used[old_idx] { + self.metadata.consts.insert(constant); + } + } + + // Update LOAD_CONST instruction arguments + for block in &mut self.blocks { + for instr in &mut block.instructions { + if let Some(Instruction::LoadConst { .. }) = instr.instr.real() { + let old_idx = u32::from(instr.arg) as usize; + if old_idx < nconsts { + instr.arg = OpArg::new(old_to_new[old_idx] as u32); + } + } + } + } + } + + /// Remove NOP instructions from all blocks, but keep NOPs that introduce + /// a new source line (they serve as line markers for monitoring LINE events). + fn remove_nops(&mut self) { + for block in &mut self.blocks { + let mut prev_line = None; + block.instructions.retain(|ins| { + if matches!(ins.instr.real(), Some(Instruction::Nop)) { + let line = ins.location.line; + if prev_line == Some(line) { + return false; + } + } + prev_line = Some(ins.location.line); + true + }); + } + } + + /// Optimize LOAD_FAST to LOAD_FAST_BORROW where safe. + /// + /// A LOAD_FAST can be converted to LOAD_FAST_BORROW if its value is + /// consumed within the same basic block (not passed to another block). + /// This is a reference counting optimization in CPython; in RustPython + /// we implement it for bytecode compatibility. + fn optimize_load_fast_borrow(&mut self) { + // NOT_LOCAL marker: instruction didn't come from a LOAD_FAST + const NOT_LOCAL: usize = usize::MAX; + + for block in &mut self.blocks { + if block.instructions.is_empty() { + continue; + } + + // Track which instructions' outputs are still on stack at block end + // For each instruction, we track if its pushed value(s) are unconsumed + let mut unconsumed = vec![false; block.instructions.len()]; + + // Simulate stack: each entry is the instruction index that pushed it + // (or NOT_LOCAL if not from LOAD_FAST/LOAD_FAST_LOAD_FAST). + // + // CPython (flowgraph.c optimize_load_fast) pre-fills the stack with + // dummy refs for values inherited from predecessor blocks. We take + // the simpler approach of aborting the optimisation for the whole + // block on stack underflow. + let mut stack: Vec<usize> = Vec::new(); + let mut underflow = false; + + for (i, info) in block.instructions.iter().enumerate() { + let Some(instr) = info.instr.real() else { + continue; + }; + + let stack_effect_info = instr.stack_effect_info(info.arg.into()); + let (pushes, pops) = (stack_effect_info.pushed(), stack_effect_info.popped()); + + // Pop values from stack + for _ in 0..pops { + if stack.pop().is_none() { + // Stack underflow — block receives values from a predecessor. + // Abort optimisation for the entire block. + underflow = true; + break; + } + } + if underflow { + break; + } + + // Push values to stack with source instruction index + let source = match instr { + Instruction::LoadFast { .. } | Instruction::LoadFastLoadFast { .. } => i, + _ => NOT_LOCAL, + }; + for _ in 0..pushes { + stack.push(source); + } + } + + if underflow { + continue; + } + + // Mark instructions whose values remain on stack at block end + for &src in &stack { + if src != NOT_LOCAL { + unconsumed[src] = true; + } + } + + // Convert LOAD_FAST to LOAD_FAST_BORROW where value is fully consumed + for (i, info) in block.instructions.iter_mut().enumerate() { + if unconsumed[i] { + continue; + } + let Some(instr) = info.instr.real() else { + continue; + }; + match instr { + Instruction::LoadFast { .. } => { + info.instr = Instruction::LoadFastBorrow { + var_num: Arg::marker(), + } + .into(); + } + Instruction::LoadFastLoadFast { .. } => { + info.instr = Instruction::LoadFastBorrowLoadFastBorrow { + var_nums: Arg::marker(), + } + .into(); + } + _ => {} + } + } + } + } + + fn max_stackdepth(&mut self) -> crate::InternalResult<u32> { + let mut maxdepth = 0u32; + let mut stack = Vec::with_capacity(self.blocks.len()); + let mut start_depths = vec![u32::MAX; self.blocks.len()]; + start_depths[0] = 0; + stack.push(BlockIdx(0)); + const DEBUG: bool = false; + // Global iteration limit as safety guard + // The algorithm is monotonic (depths only increase), so it should converge quickly. + // Max iterations = blocks * max_possible_depth_increases per block + let max_iterations = self.blocks.len() * 100; + let mut iterations = 0usize; + 'process_blocks: while let Some(block_idx) = stack.pop() { + iterations += 1; + if iterations > max_iterations { + // Safety guard: should never happen in valid code + // Return error instead of silently breaking to avoid underestimated stack depth + return Err(InternalError::StackOverflow); + } + let idx = block_idx.idx(); + let mut depth = start_depths[idx]; + if DEBUG { + eprintln!("===BLOCK {}===", block_idx.0); + } + let block = &self.blocks[block_idx]; + for ins in &block.instructions { + let instr = &ins.instr; + let effect = instr.stack_effect(ins.arg.into()); + if DEBUG { + let display_arg = if ins.target == BlockIdx::NULL { + ins.arg + } else { + OpArg::new(ins.target.0) + }; + let instr_display = instr.display(display_arg, self); + eprint!("{instr_display}: {depth} {effect:+} => "); + } + let new_depth = depth.checked_add_signed(effect).ok_or({ + if effect < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow + } + })?; + if DEBUG { + eprintln!("{new_depth}"); + } + if new_depth > maxdepth { + maxdepth = new_depth + } + // Process target blocks for branching instructions + if ins.target != BlockIdx::NULL { + if instr.is_block_push() { + // SETUP_* pseudo ops: target is a handler block. + // Handler entry depth uses the jump-path stack effect: + // SETUP_FINALLY: +1 (pushes exc) + // SETUP_CLEANUP: +2 (pushes lasti + exc) + // SETUP_WITH: +1 (pops __enter__ result, pushes lasti + exc) + let handler_effect: u32 = match instr.pseudo() { + Some(PseudoInstruction::SetupCleanup { .. }) => 2, + _ => 1, // SetupFinally and SetupWith + }; + let handler_depth = depth + handler_effect; + if handler_depth > maxdepth { + maxdepth = handler_depth; + } + stackdepth_push(&mut stack, &mut start_depths, ins.target, handler_depth); + } else { + // SEND jumps to END_SEND with receiver still on stack. + // END_SEND performs the receiver pop. + let jump_effect = match instr.real() { + Some(Instruction::Send { .. }) => 0i32, + _ => effect, + }; + let target_depth = depth.checked_add_signed(jump_effect).ok_or({ + if jump_effect < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow + } + })?; + if target_depth > maxdepth { + maxdepth = target_depth + } + stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth); + } + } + depth = new_depth; + if instr.is_scope_exit() || instr.is_unconditional_jump() { + continue 'process_blocks; + } + } + // Only push next block if it's not NULL + if block.next != BlockIdx::NULL { + stackdepth_push(&mut stack, &mut start_depths, block.next, depth); + } + } + if DEBUG { + eprintln!("DONE: {maxdepth}"); + } + + // Fix up handler stack_depth in ExceptHandlerInfo using start_depths + // computed above: depth = start_depth - 1 - preserve_lasti + for block in self.blocks.iter_mut() { + for ins in &mut block.instructions { + if let Some(ref mut handler) = ins.except_handler { + let h_start = start_depths[handler.handler_block.idx()]; + if h_start != u32::MAX { + let adjustment = 1 + handler.preserve_lasti as u32; + debug_assert!( + h_start >= adjustment, + "handler start depth {h_start} too shallow for adjustment {adjustment}" + ); + handler.stack_depth = h_start.saturating_sub(adjustment); + } + } + } + } + + Ok(maxdepth) + } +} + +impl InstrDisplayContext for CodeInfo { + type Constant = ConstantData; + + fn get_constant(&self, consti: oparg::ConstIdx) -> &ConstantData { + &self.metadata.consts[consti.as_usize()] + } + + fn get_name(&self, i: usize) -> &str { + self.metadata.names[i].as_ref() + } + + fn get_varname(&self, var_num: oparg::VarNum) -> &str { + self.metadata.varnames[var_num.as_usize()].as_ref() + } + + fn get_cell_name(&self, i: usize) -> &str { + self.metadata + .cellvars + .get_index(i) + .unwrap_or_else(|| &self.metadata.freevars[i - self.metadata.cellvars.len()]) + .as_ref() + } +} + +fn stackdepth_push( + stack: &mut Vec<BlockIdx>, + start_depths: &mut [u32], + target: BlockIdx, + depth: u32, +) { + let idx = target.idx(); + let block_depth = &mut start_depths[idx]; + if depth > *block_depth || *block_depth == u32::MAX { + // Found a path with higher depth (or first visit): update max and queue + *block_depth = depth; + stack.push(target); + } +} + +fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + '_ { + let mut next = BlockIdx(0); + core::iter::from_fn(move || { + if next == BlockIdx::NULL { + return None; + } + let (idx, b) = (next, &blocks[next]); + next = b.next; + Some((idx, b)) + }) +} + +/// Generate Python 3.11+ format linetable from source locations +fn generate_linetable( + locations: &[LineTableLocation], + first_line: i32, + debug_ranges: bool, +) -> Box<[u8]> { + if locations.is_empty() { + return Box::new([]); + } + + let mut linetable = Vec::new(); + // Initialize prev_line to first_line + // The first entry's delta is relative to co_firstlineno + let mut prev_line = first_line; + let mut i = 0; + + while i < locations.len() { + let loc = &locations[i]; + + // Count consecutive instructions with the same location + let mut length = 1; + while i + length < locations.len() && locations[i + length] == locations[i] { + length += 1; + } + + // Process in chunks of up to 8 instructions + while length > 0 { + let entry_length = length.min(8); + + // Get line information + let line = loc.line; + + // NO_LOCATION: emit PyCodeLocationInfoKind::None entries (CACHE, etc.) + if line == -1 { + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::None as u8) << 3) | ((entry_length - 1) as u8), + ); + // Do NOT update prev_line + length -= entry_length; + i += entry_length; + continue; + } + + let end_line = loc.end_line; + let line_delta = line - prev_line; + let end_line_delta = end_line - line; + + // When debug_ranges is disabled, only emit line info (NoColumns format) + if !debug_ranges { + // NoColumns format (code 13): line info only, no column data + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::NoColumns as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + + prev_line = line; + length -= entry_length; + i += entry_length; + continue; + } + + // Get column information (only when debug_ranges is enabled) + let col = loc.col; + let end_col = loc.end_col; + + // Choose the appropriate encoding based on line delta and column info + if line_delta == 0 && end_line_delta == 0 { + if col < 80 && end_col - col < 16 && end_col >= col { + // Short form (codes 0-9) for common cases + let code = (col / 8).min(9) as u8; // Short0 to Short9 + linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); + let col_byte = (((col % 8) as u8) << 4) | ((end_col - col) as u8 & 0xf); + linetable.push(col_byte); + } else if col < 128 && end_col < 128 { + // One-line form (code 10) for same line + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::OneLine0 as u8) << 3) + | ((entry_length - 1) as u8), + ); + linetable.push(col as u8); + linetable.push(end_col as u8); + } else { + // Long form for columns >= 128 + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, 0); // line_delta = 0 + write_varint(&mut linetable, 0); // end_line delta = 0 + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); + } + } else if line_delta > 0 && line_delta < 3 && end_line_delta == 0 { + // One-line form (codes 11-12) for line deltas 1-2 + if col < 128 && end_col < 128 { + let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); + linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); + linetable.push(col as u8); + linetable.push(end_col as u8); + } else { + // Long form for columns >= 128 + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + write_varint(&mut linetable, 0); // end_line delta = 0 + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); + } + } else { + // Long form (code 14) for all other cases + // Handles: line_delta < 0, line_delta >= 3, multi-line spans, or columns >= 128 + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + write_varint(&mut linetable, end_line_delta as u32); + write_varint(&mut linetable, (col as u32) + 1); + write_varint(&mut linetable, (end_col as u32) + 1); + } + + prev_line = line; + length -= entry_length; + i += entry_length; + } + } + + linetable.into_boxed_slice() +} + +/// Generate Python 3.11+ exception table from instruction handler info +fn generate_exception_table(blocks: &[Block], block_to_index: &[u32]) -> Box<[u8]> { + let mut entries: Vec<ExceptionTableEntry> = Vec::new(); + let mut current_entry: Option<(ExceptHandlerInfo, u32)> = None; // (handler_info, start_index) + let mut instr_index = 0u32; + + // Iterate through all instructions in block order + // instr_index is the index into the final instructions array (including EXTENDED_ARG) + // This matches how frame.rs uses lasti + for (_, block) in iter_blocks(blocks) { + for instr in &block.instructions { + // instr_size includes EXTENDED_ARG and CACHE entries + let instr_size = instr.arg.instr_size() as u32 + instr.cache_entries; + + match (&current_entry, instr.except_handler) { + // No current entry, no handler - nothing to do + (None, None) => {} + + // No current entry, handler starts - begin new entry + (None, Some(handler)) => { + current_entry = Some((handler, instr_index)); + } + + // Current entry exists, same handler - continue + (Some((curr_handler, _)), Some(handler)) + if curr_handler.handler_block == handler.handler_block + && curr_handler.stack_depth == handler.stack_depth + && curr_handler.preserve_lasti == handler.preserve_lasti => {} + + // Current entry exists, different handler - finish current, start new + (Some((curr_handler, start)), Some(handler)) => { + let target_index = block_to_index[curr_handler.handler_block.idx()]; + entries.push(ExceptionTableEntry::new( + *start, + instr_index, + target_index, + curr_handler.stack_depth as u16, + curr_handler.preserve_lasti, + )); + current_entry = Some((handler, instr_index)); + } + + // Current entry exists, no handler - finish current entry + (Some((curr_handler, start)), None) => { + let target_index = block_to_index[curr_handler.handler_block.idx()]; + entries.push(ExceptionTableEntry::new( + *start, + instr_index, + target_index, + curr_handler.stack_depth as u16, + curr_handler.preserve_lasti, + )); + current_entry = None; + } + } + + instr_index += instr_size; // Account for EXTENDED_ARG instructions + } + } + + // Finish any remaining entry + if let Some((curr_handler, start)) = current_entry { + let target_index = block_to_index[curr_handler.handler_block.idx()]; + entries.push(ExceptionTableEntry::new( + start, + instr_index, + target_index, + curr_handler.stack_depth as u16, + curr_handler.preserve_lasti, + )); + } + + encode_exception_table(&entries) +} + +/// Mark exception handler target blocks. +/// flowgraph.c mark_except_handlers +pub(crate) fn mark_except_handlers(blocks: &mut [Block]) { + // Reset handler flags + for block in blocks.iter_mut() { + block.except_handler = false; + block.preserve_lasti = false; + } + // Mark target blocks of SETUP_* as except handlers + let targets: Vec<usize> = blocks + .iter() + .flat_map(|b| b.instructions.iter()) + .filter(|i| i.instr.is_block_push() && i.target != BlockIdx::NULL) + .map(|i| i.target.idx()) + .collect(); + for idx in targets { + blocks[idx].except_handler = true; + } +} + +/// flowgraph.c mark_cold +fn mark_cold(blocks: &mut [Block]) { + let n = blocks.len(); + let mut warm = vec![false; n]; + let mut queue = VecDeque::new(); + + warm[0] = true; + queue.push_back(BlockIdx(0)); + + while let Some(block_idx) = queue.pop_front() { + let block = &blocks[block_idx.idx()]; + + let has_fallthrough = block + .instructions + .last() + .map(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()) + .unwrap_or(true); + if has_fallthrough && block.next != BlockIdx::NULL { + let next_idx = block.next.idx(); + if !blocks[next_idx].except_handler && !warm[next_idx] { + warm[next_idx] = true; + queue.push_back(block.next); + } + } + + for instr in &block.instructions { + if instr.target != BlockIdx::NULL { + let target_idx = instr.target.idx(); + if !blocks[target_idx].except_handler && !warm[target_idx] { + warm[target_idx] = true; + queue.push_back(instr.target); + } + } + } + } + + for (i, block) in blocks.iter_mut().enumerate() { + block.cold = !warm[i]; + } +} + +/// flowgraph.c push_cold_blocks_to_end +fn push_cold_blocks_to_end(blocks: &mut Vec<Block>) { + if blocks.len() <= 1 { + return; + } + + mark_cold(blocks); + + // If a cold block falls through to a warm block, add an explicit jump + let fixups: Vec<(BlockIdx, BlockIdx)> = iter_blocks(blocks) + .filter(|(_, block)| { + block.cold + && block.next != BlockIdx::NULL + && !blocks[block.next.idx()].cold + && block + .instructions + .last() + .map(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()) + .unwrap_or(true) + }) + .map(|(idx, block)| (idx, block.next)) + .collect(); + + for (cold_idx, warm_next) in fixups { + let jump_block_idx = BlockIdx(blocks.len() as u32); + let loc = blocks[cold_idx.idx()] + .instructions + .last() + .map(|i| i.location) + .unwrap_or_default(); + let end_loc = blocks[cold_idx.idx()] + .instructions + .last() + .map(|i| i.end_location) + .unwrap_or_default(); + let mut jump_block = Block { + cold: true, + ..Block::default() + }; + jump_block.instructions.push(InstructionInfo { + instr: PseudoInstruction::JumpNoInterrupt { + delta: Arg::marker(), + } + .into(), + arg: OpArg::new(0), + target: warm_next, + location: loc, + end_location: end_loc, + except_handler: None, + lineno_override: None, + cache_entries: 0, + }); + jump_block.next = blocks[cold_idx.idx()].next; + blocks[cold_idx.idx()].next = jump_block_idx; + blocks.push(jump_block); + } + + // Extract cold block streaks and append at the end + let mut cold_head: BlockIdx = BlockIdx::NULL; + let mut cold_tail: BlockIdx = BlockIdx::NULL; + let mut current = BlockIdx(0); + assert!(!blocks[0].cold); + + while current != BlockIdx::NULL { + let next = blocks[current.idx()].next; + if next == BlockIdx::NULL { + break; + } + + if blocks[next.idx()].cold { + let cold_start = next; + let mut cold_end = next; + while blocks[cold_end.idx()].next != BlockIdx::NULL + && blocks[blocks[cold_end.idx()].next.idx()].cold + { + cold_end = blocks[cold_end.idx()].next; + } + + let after_cold = blocks[cold_end.idx()].next; + blocks[current.idx()].next = after_cold; + blocks[cold_end.idx()].next = BlockIdx::NULL; + + if cold_head == BlockIdx::NULL { + cold_head = cold_start; + } else { + blocks[cold_tail.idx()].next = cold_start; + } + cold_tail = cold_end; + } else { + current = next; + } + } + + if cold_head != BlockIdx::NULL { + let mut last = current; + while blocks[last.idx()].next != BlockIdx::NULL { + last = blocks[last.idx()].next; + } + blocks[last.idx()].next = cold_head; + } +} + +fn is_conditional_jump(instr: &AnyInstruction) -> bool { + matches!( + instr.real(), + Some( + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + ) + ) +} + +/// flowgraph.c normalize_jumps + remove_redundant_jumps +fn normalize_jumps(blocks: &mut [Block]) { + let mut visit_order = Vec::new(); + let mut visited = vec![false; blocks.len()]; + let mut current = BlockIdx(0); + while current != BlockIdx::NULL { + visit_order.push(current); + visited[current.idx()] = true; + current = blocks[current.idx()].next; + } + + visited.fill(false); + + for &block_idx in &visit_order { + let idx = block_idx.idx(); + visited[idx] = true; + + // Remove redundant unconditional jump to next block + let next = blocks[idx].next; + if next != BlockIdx::NULL { + let last = blocks[idx].instructions.last(); + let is_jump_to_next = last.is_some_and(|ins| { + ins.instr.is_unconditional_jump() + && ins.target != BlockIdx::NULL + && ins.target == next + }); + if is_jump_to_next && let Some(last_instr) = blocks[idx].instructions.last_mut() { + last_instr.instr = Instruction::Nop.into(); + last_instr.target = BlockIdx::NULL; + } + } + + // Insert NOT_TAKEN after forward conditional jumps + let mut insert_positions: Vec<(usize, InstructionInfo)> = Vec::new(); + for (i, ins) in blocks[idx].instructions.iter().enumerate() { + if is_conditional_jump(&ins.instr) + && ins.target != BlockIdx::NULL + && !visited[ins.target.idx()] + { + insert_positions.push(( + i + 1, + InstructionInfo { + instr: Instruction::NotTaken.into(), + arg: OpArg::new(0), + target: BlockIdx::NULL, + location: ins.location, + end_location: ins.end_location, + except_handler: ins.except_handler, + lineno_override: None, + cache_entries: 0, + }, + )); + } + } + + for (pos, info) in insert_positions.into_iter().rev() { + blocks[idx].instructions.insert(pos, info); + } + } + + // Resolve JUMP/JUMP_NO_INTERRUPT pseudo instructions before offset fixpoint. + let mut block_order = vec![0u32; blocks.len()]; + for (pos, &block_idx) in visit_order.iter().enumerate() { + block_order[block_idx.idx()] = pos as u32; + } + + for &block_idx in &visit_order { + let source_pos = block_order[block_idx.idx()]; + for info in &mut blocks[block_idx.idx()].instructions { + let target = info.target; + if target == BlockIdx::NULL { + continue; + } + let target_pos = block_order[target.idx()]; + info.instr = match info.instr { + AnyInstruction::Pseudo(PseudoInstruction::Jump { .. }) => { + if target_pos > source_pos { + Instruction::JumpForward { + delta: Arg::marker(), + } + .into() + } else { + Instruction::JumpBackward { + delta: Arg::marker(), + } + .into() + } + } + AnyInstruction::Pseudo(PseudoInstruction::JumpNoInterrupt { .. }) => { + if target_pos > source_pos { + Instruction::JumpForward { + delta: Arg::marker(), + } + .into() + } else { + Instruction::JumpBackwardNoInterrupt { + delta: Arg::marker(), + } + .into() + } + } + other => other, + }; + } + } +} + +/// Label exception targets: walk CFG with except stack, set per-instruction +/// handler info and block preserve_lasti flag. Converts POP_BLOCK to NOP. +/// flowgraph.c label_exception_targets + push_except_block +pub(crate) fn label_exception_targets(blocks: &mut [Block]) { + #[derive(Clone)] + struct ExceptEntry { + handler_block: BlockIdx, + preserve_lasti: bool, + } + + let num_blocks = blocks.len(); + if num_blocks == 0 { + return; + } + + let mut visited = vec![false; num_blocks]; + let mut block_stacks: Vec<Option<Vec<ExceptEntry>>> = vec![None; num_blocks]; + + // Entry block + visited[0] = true; + block_stacks[0] = Some(Vec::new()); + + let mut todo = vec![BlockIdx(0)]; + + while let Some(block_idx) = todo.pop() { + let bi = block_idx.idx(); + let mut stack = block_stacks[bi].take().unwrap_or_default(); + let mut last_yield_except_depth: i32 = -1; + + let instr_count = blocks[bi].instructions.len(); + for i in 0..instr_count { + // Read all needed fields (each temporary borrow ends immediately) + let target = blocks[bi].instructions[i].target; + let arg = blocks[bi].instructions[i].arg; + let is_push = blocks[bi].instructions[i].instr.is_block_push(); + let is_pop = blocks[bi].instructions[i].instr.is_pop_block(); + + if is_push { + // Determine preserve_lasti from instruction type (push_except_block) + let preserve_lasti = matches!( + blocks[bi].instructions[i].instr.pseudo(), + Some( + PseudoInstruction::SetupWith { .. } + | PseudoInstruction::SetupCleanup { .. } + ) + ); + + // Set preserve_lasti on handler block + if preserve_lasti && target != BlockIdx::NULL { + blocks[target.idx()].preserve_lasti = true; + } + + // Propagate except stack to handler block if not visited + if target != BlockIdx::NULL && !visited[target.idx()] { + visited[target.idx()] = true; + block_stacks[target.idx()] = Some(stack.clone()); + todo.push(target); + } + + // Push handler onto except stack + stack.push(ExceptEntry { + handler_block: target, + preserve_lasti, + }); + } else if is_pop { + debug_assert!( + !stack.is_empty(), + "POP_BLOCK with empty except stack at block {bi} instruction {i}" + ); + stack.pop(); + // POP_BLOCK → NOP + blocks[bi].instructions[i].instr = Instruction::Nop.into(); + } else { + // Set except_handler for this instruction from except stack top + // stack_depth placeholder: filled by fixup_handler_depths + let handler_info = stack.last().map(|e| ExceptHandlerInfo { + handler_block: e.handler_block, + stack_depth: 0, + preserve_lasti: e.preserve_lasti, + }); + blocks[bi].instructions[i].except_handler = handler_info; + + // Track YIELD_VALUE except stack depth + if matches!( + blocks[bi].instructions[i].instr.real(), + Some(Instruction::YieldValue { .. }) + ) { + last_yield_except_depth = stack.len() as i32; + } + + // Set RESUME DEPTH1 flag based on last yield's except depth + if matches!( + blocks[bi].instructions[i].instr.real(), + Some(Instruction::Resume { .. }) + ) { + const RESUME_AT_FUNC_START: u32 = 0; + const RESUME_OPARG_LOCATION_MASK: u32 = 0x3; + const RESUME_OPARG_DEPTH1_MASK: u32 = 0x4; + + if (u32::from(arg) & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START { + if last_yield_except_depth == 1 { + blocks[bi].instructions[i].arg = + OpArg::new(u32::from(arg) | RESUME_OPARG_DEPTH1_MASK); + } + last_yield_except_depth = -1; + } + } + + // For jump instructions, propagate except stack to target + if target != BlockIdx::NULL && !visited[target.idx()] { + visited[target.idx()] = true; + block_stacks[target.idx()] = Some(stack.clone()); + todo.push(target); + } + } + } + + // Propagate to fallthrough block (block.next) + let next = blocks[bi].next; + if next != BlockIdx::NULL && !visited[next.idx()] { + let has_fallthrough = blocks[bi] + .instructions + .last() + .map(|ins| !ins.instr.is_scope_exit() && !ins.instr.is_unconditional_jump()) + .unwrap_or(true); // Empty block falls through + if has_fallthrough { + visited[next.idx()] = true; + block_stacks[next.idx()] = Some(stack); + todo.push(next); + } + } + } +} + +/// Convert remaining pseudo ops to real instructions or NOP. +/// flowgraph.c convert_pseudo_ops +pub(crate) fn convert_pseudo_ops(blocks: &mut [Block], varnames_len: u32) { + for block in blocks.iter_mut() { + for info in &mut block.instructions { + let Some(pseudo) = info.instr.pseudo() else { + continue; + }; + match pseudo { + // Block push pseudo ops → NOP + PseudoInstruction::SetupCleanup { .. } + | PseudoInstruction::SetupFinally { .. } + | PseudoInstruction::SetupWith { .. } => { + info.instr = Instruction::Nop.into(); + } + // PopBlock in reachable blocks is converted to NOP by + // label_exception_targets. Dead blocks may still have them. + PseudoInstruction::PopBlock => { + info.instr = Instruction::Nop.into(); + } + // LOAD_CLOSURE → LOAD_FAST (with varnames offset) + PseudoInstruction::LoadClosure { i } => { + let new_idx = varnames_len + i.get(info.arg); + info.arg = OpArg::new(new_idx); + info.instr = Instruction::LoadFast { + var_num: Arg::marker(), + } + .into(); + } + // Jump pseudo ops are resolved during block linearization + PseudoInstruction::Jump { .. } | PseudoInstruction::JumpNoInterrupt { .. } => {} + // These should have been resolved earlier + PseudoInstruction::AnnotationsPlaceholder + | PseudoInstruction::JumpIfFalse { .. } + | PseudoInstruction::JumpIfTrue { .. } + | PseudoInstruction::StoreFastMaybeNull { .. } => { + unreachable!("Unexpected pseudo instruction in convert_pseudo_ops: {pseudo:?}") + } + } + } + } +} diff --git a/crates/codegen/src/lib.rs b/crates/codegen/src/lib.rs new file mode 100644 index 00000000000..c1a318c19cd --- /dev/null +++ b/crates/codegen/src/lib.rs @@ -0,0 +1,70 @@ +//! Compile a Python AST or source code into bytecode consumable by RustPython. +#![cfg_attr(not(feature = "std"), no_std)] +#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] +#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")] + +#[macro_use] +extern crate log; + +extern crate alloc; + +type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>; +type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>; + +pub mod compile; +pub mod error; +pub mod ir; +mod string_parser; +pub mod symboltable; +mod unparse; + +pub use compile::CompileOpts; +use ruff_python_ast as ast; + +pub(crate) use compile::InternalResult; + +pub trait ToPythonName { + /// Returns a short name for the node suitable for use in error messages. + fn python_name(&self) -> &'static str; +} + +impl ToPythonName for ast::Expr { + fn python_name(&self) -> &'static str { + match self { + Self::BoolOp { .. } | Self::BinOp { .. } | Self::UnaryOp { .. } => "operator", + Self::Subscript { .. } => "subscript", + Self::Await { .. } => "await expression", + Self::Yield { .. } | Self::YieldFrom { .. } => "yield expression", + Self::Compare { .. } => "comparison", + Self::Attribute { .. } => "attribute", + Self::Call { .. } => "function call", + Self::BooleanLiteral(b) => { + if b.value { + "True" + } else { + "False" + } + } + Self::EllipsisLiteral(_) => "ellipsis", + Self::NoneLiteral(_) => "None", + Self::NumberLiteral(_) | Self::BytesLiteral(_) | Self::StringLiteral(_) => "literal", + Self::Tuple(_) => "tuple", + Self::List { .. } => "list", + Self::Dict { .. } => "dict display", + Self::Set { .. } => "set display", + Self::ListComp { .. } => "list comprehension", + Self::DictComp { .. } => "dict comprehension", + Self::SetComp { .. } => "set comprehension", + Self::Generator { .. } => "generator expression", + Self::Starred { .. } => "starred", + Self::Slice { .. } => "slice", + Self::FString { .. } => "f-string expression", + Self::TString { .. } => "t-string expression", + Self::Name { .. } => "name", + Self::Lambda { .. } => "lambda", + Self::If { .. } => "conditional expression", + Self::Named { .. } => "named expression", + Self::IpyEscapeCommand(_) => todo!(), + } + } +} diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__const_bool_not_op.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__const_bool_not_op.snap new file mode 100644 index 00000000000..f9a74c2055c --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__const_bool_not_op.snap @@ -0,0 +1,9 @@ +--- +source: crates/codegen/src/compile.rs +expression: "compile_exec_optimized(\"\\\nx = not True\n\")" +--- + 1 0 RESUME (0) + 1 LOAD_CONST (False) + 2 STORE_NAME (0, x) + 3 LOAD_CONST (None) + 4 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap new file mode 100644 index 00000000000..9dd78c6b7b2 --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap @@ -0,0 +1,21 @@ +--- +source: crates/codegen/src/compile.rs +assertion_line: 9100 +expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" +--- + 1 0 RESUME (0) + >> 1 LOAD_CONST (True) + 2 POP_JUMP_IF_FALSE (9) + 3 CACHE + 4 NOT_TAKEN + >> 5 LOAD_CONST (False) + 6 POP_JUMP_IF_FALSE (5) + 7 CACHE + 8 NOT_TAKEN + >> 9 LOAD_CONST (False) + 10 POP_JUMP_IF_FALSE (1) + 11 CACHE + 12 NOT_TAKEN + + 2 13 LOAD_CONST (None) + 14 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap new file mode 100644 index 00000000000..e9c3ad8a3c6 --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap @@ -0,0 +1,25 @@ +--- +source: crates/codegen/src/compile.rs +assertion_line: 9110 +expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" +--- + 1 0 RESUME (0) + >> 1 LOAD_CONST (True) + 2 POP_JUMP_IF_FALSE (5) + 3 CACHE + 4 NOT_TAKEN + >> 5 LOAD_CONST (False) + 6 POP_JUMP_IF_TRUE (9) + 7 CACHE + 8 NOT_TAKEN + >> 9 LOAD_CONST (False) + 10 POP_JUMP_IF_FALSE (5) + 11 CACHE + 12 NOT_TAKEN + 13 LOAD_CONST (True) + 14 POP_JUMP_IF_FALSE (1) + 15 CACHE + 16 NOT_TAKEN + + 2 17 LOAD_CONST (None) + 18 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap new file mode 100644 index 00000000000..83212144b99 --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap @@ -0,0 +1,21 @@ +--- +source: crates/codegen/src/compile.rs +assertion_line: 9090 +expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" +--- + 1 0 RESUME (0) + >> 1 LOAD_CONST (True) + 2 POP_JUMP_IF_TRUE (9) + 3 CACHE + 4 NOT_TAKEN + >> 5 LOAD_CONST (False) + 6 POP_JUMP_IF_TRUE (5) + 7 CACHE + 8 NOT_TAKEN + >> 9 LOAD_CONST (False) + 10 POP_JUMP_IF_FALSE (1) + 11 CACHE + 12 NOT_TAKEN + + 2 13 LOAD_CONST (None) + 14 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_bool_op.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_bool_op.snap new file mode 100644 index 00000000000..a6db9ca4bdb --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_bool_op.snap @@ -0,0 +1,26 @@ +--- +source: crates/codegen/src/compile.rs +expression: "compile_exec(\"\\\nx = Test() and False or False\n\")" +--- + 1 0 RESUME (0) + 1 LOAD_NAME (0, Test) + 2 PUSH_NULL + >> 3 CALL (0) + 4 CACHE + 5 CACHE + 6 CACHE + >> 7 COPY (1) + 8 POP_JUMP_IF_FALSE (7) + 9 CACHE + 10 NOT_TAKEN + 11 POP_TOP + 12 LOAD_CONST (False) + 13 COPY (1) + 14 POP_JUMP_IF_TRUE (3) + 15 CACHE + 16 NOT_TAKEN + 17 POP_TOP + 18 LOAD_CONST (False) + 19 STORE_NAME (1, x) + 20 LOAD_CONST (None) + 21 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap new file mode 100644 index 00000000000..7a1db8e7b8c --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap @@ -0,0 +1,278 @@ +--- +source: crates/codegen/src/compile.rs +assertion_line: 9089 +expression: "compile_exec(\"\\\nasync def test():\n for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" +--- + 1 0 RESUME (0) + 1 LOAD_CONST (<code object test at ??? file "source_path", line 1>): 1 0 RETURN_GENERATOR + 1 POP_TOP + >> 2 RESUME (0) + + 2 >> 3 LOAD_GLOBAL (1, NULL + StopIteration) + >> 4 CACHE + >> 5 CACHE + 6 CACHE + 7 CACHE + 8 LOAD_CONST ("spam") + 9 CALL (1) + >> 10 CACHE + 11 CACHE + 12 CACHE + 13 LOAD_GLOBAL (3, NULL + StopAsyncIteration) + 14 CACHE + 15 CACHE + 16 CACHE + 17 CACHE + >> 18 LOAD_CONST ("ham") + 19 CALL (1) + 20 CACHE + 21 CACHE + 22 CACHE + 23 BUILD_TUPLE (2) + 24 GET_ITER + 25 FOR_ITER (71) + 26 CACHE + >> 27 STORE_FAST (0, stop_exc) + + 3 >> 28 LOAD_GLOBAL (4, self) + 29 CACHE + 30 CACHE + 31 CACHE + 32 CACHE + 33 LOAD_ATTR (7, subTest, method=true) + >> 34 CACHE + 35 CACHE + 36 CACHE + 37 CACHE + 38 CACHE + 39 CACHE + 40 CACHE + 41 CACHE + 42 CACHE + 43 LOAD_GLOBAL (9, NULL + type) + 44 CACHE + 45 CACHE + 46 CACHE + >> 47 CACHE + 48 LOAD_FAST (0, stop_exc) + 49 CALL (1) + 50 CACHE + 51 CACHE + 52 CACHE + 53 LOAD_CONST (("type")) + 54 CALL_KW (1) + 55 CACHE + 56 CACHE + 57 CACHE + 58 COPY (1) + 59 LOAD_SPECIAL (__exit__) + 60 SWAP (2) + 61 LOAD_SPECIAL (__enter__) + 62 PUSH_NULL + 63 CALL (0) + 64 CACHE + 65 CACHE + 66 CACHE + 67 POP_TOP + + 4 68 NOP + + 5 69 LOAD_GLOBAL (11, NULL + egg) + 70 CACHE + >> 71 CACHE + 72 CACHE + 73 CACHE + 74 CALL (0) + 75 CACHE + 76 CACHE + 77 CACHE + 78 COPY (1) + 79 LOAD_SPECIAL (__aexit__) + 80 SWAP (2) + 81 LOAD_SPECIAL (__aenter__) + 82 PUSH_NULL + 83 CALL (0) + 84 CACHE + 85 CACHE + 86 CACHE + 87 GET_AWAITABLE (1) + 88 LOAD_CONST (None) + 89 SEND (3) + 90 CACHE + 91 YIELD_VALUE (1) + 92 RESUME (3) + 93 JUMP_BACKWARD_NO_INTERRUPT(5) + 94 END_SEND + 95 POP_TOP + + 6 96 LOAD_FAST (0, stop_exc) + 97 RAISE_VARARGS (Raise) + + 2 98 END_FOR + 99 POP_ITER + 100 LOAD_CONST (None) + 101 RETURN_VALUE + + 5 102 CLEANUP_THROW + 103 JUMP_BACKWARD_NO_INTERRUPT(10) + + 6 104 NOP + + 5 105 PUSH_NULL + 106 LOAD_CONST (None) + 107 LOAD_CONST (None) + 108 LOAD_CONST (None) + 109 CALL (3) + 110 CACHE + 111 CACHE + 112 CACHE + 113 GET_AWAITABLE (2) + 114 LOAD_CONST (None) + 115 SEND (4) + 116 CACHE + 117 YIELD_VALUE (1) + 118 RESUME (3) + 119 JUMP_BACKWARD_NO_INTERRUPT(5) + 120 CLEANUP_THROW + 121 END_SEND + 122 POP_TOP + 123 JUMP_FORWARD (27) + 124 PUSH_EXC_INFO + 125 WITH_EXCEPT_START + 126 GET_AWAITABLE (2) + 127 LOAD_CONST (None) + 128 SEND (4) + 129 CACHE + 130 YIELD_VALUE (1) + 131 RESUME (3) + 132 JUMP_BACKWARD_NO_INTERRUPT(5) + 133 CLEANUP_THROW + 134 END_SEND + 135 TO_BOOL + 136 CACHE + 137 CACHE + 138 CACHE + 139 POP_JUMP_IF_TRUE (2) + 140 CACHE + 141 NOT_TAKEN + 142 RERAISE (2) + 143 POP_TOP + 144 POP_EXCEPT + 145 POP_TOP + 146 POP_TOP + 147 JUMP_FORWARD (3) + 148 COPY (3) + 149 POP_EXCEPT + 150 RERAISE (1) + 151 JUMP_FORWARD (47) + 152 PUSH_EXC_INFO + + 7 153 LOAD_GLOBAL (12, Exception) + 154 CACHE + 155 CACHE + 156 CACHE + 157 CACHE + 158 CHECK_EXC_MATCH + 159 POP_JUMP_IF_FALSE (34) + 160 CACHE + 161 NOT_TAKEN + 162 STORE_FAST (1, ex) + + 8 163 LOAD_GLOBAL (4, self) + 164 CACHE + 165 CACHE + 166 CACHE + 167 CACHE + 168 LOAD_ATTR (15, assertIs, method=true) + 169 CACHE + 170 CACHE + 171 CACHE + 172 CACHE + 173 CACHE + 174 CACHE + 175 CACHE + 176 CACHE + 177 CACHE + 178 LOAD_FAST (1, ex) + 179 LOAD_FAST (0, stop_exc) + 180 CALL (2) + 181 CACHE + 182 CACHE + 183 CACHE + 184 POP_TOP + 185 JUMP_FORWARD (4) + 186 LOAD_CONST (None) + 187 STORE_FAST (1, ex) + 188 DELETE_FAST (1, ex) + 189 RERAISE (1) + 190 POP_EXCEPT + 191 LOAD_CONST (None) + 192 STORE_FAST (1, ex) + 193 DELETE_FAST (1, ex) + 194 JUMP_FORWARD (28) + 195 RERAISE (0) + 196 COPY (3) + 197 POP_EXCEPT + 198 RERAISE (1) + + 10 199 LOAD_GLOBAL (4, self) + 200 CACHE + 201 CACHE + 202 CACHE + 203 CACHE + 204 LOAD_ATTR (17, fail, method=true) + 205 CACHE + 206 CACHE + 207 CACHE + 208 CACHE + 209 CACHE + 210 CACHE + 211 CACHE + 212 CACHE + 213 CACHE + 214 LOAD_FAST_BORROW (0, stop_exc) + 215 FORMAT_SIMPLE + 216 LOAD_CONST (" was suppressed") + 217 BUILD_STRING (2) + 218 CALL (1) + 219 CACHE + 220 CACHE + 221 CACHE + 222 POP_TOP + 223 NOP + + 3 224 PUSH_NULL + 225 LOAD_CONST (None) + 226 LOAD_CONST (None) + 227 LOAD_CONST (None) + 228 CALL (3) + >> 229 CACHE + 230 CACHE + 231 CACHE + 232 POP_TOP + 233 JUMP_FORWARD (18) + 234 PUSH_EXC_INFO + 235 WITH_EXCEPT_START + 236 TO_BOOL + 237 CACHE + 238 CACHE + 239 CACHE + 240 POP_JUMP_IF_TRUE (2) + 241 CACHE + 242 NOT_TAKEN + 243 RERAISE (2) + 244 POP_TOP + 245 POP_EXCEPT + 246 POP_TOP + 247 POP_TOP + 248 JUMP_FORWARD (3) + 249 COPY (3) + 250 POP_EXCEPT + 251 RERAISE (1) + 252 JUMP_BACKWARD (229) + 253 CACHE + + 2 MAKE_FUNCTION + 3 STORE_NAME (0, test) + 4 LOAD_CONST (None) + 5 RETURN_VALUE diff --git a/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap new file mode 100644 index 00000000000..bc88cf2349c --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap @@ -0,0 +1,14 @@ +--- +source: compiler/src/compile.rs +expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" +--- + 1 0 LoadConst (True) + 1 PopJumpIfFalse (6) + 2 LoadConst (False) + 3 PopJumpIfFalse (6) + 4 LoadConst (False) + 5 PopJumpIfFalse (6) + + 2 >> 6 LoadConst (None) + 7 ReturnValue + diff --git a/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap new file mode 100644 index 00000000000..b19cbb119d9 --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap @@ -0,0 +1,16 @@ +--- +source: compiler/src/compile.rs +expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" +--- + 1 0 LoadConst (True) + 1 PopJumpIfFalse (4) + 2 LoadConst (False) + 3 PopJumpIfTrue (8) + >> 4 LoadConst (False) + 5 PopJumpIfFalse (8) + 6 LoadConst (True) + 7 PopJumpIfFalse (8) + + 2 >> 8 LoadConst (None) + 9 ReturnValue + diff --git a/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap new file mode 100644 index 00000000000..3d1f5a1d6f0 --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap @@ -0,0 +1,14 @@ +--- +source: compiler/src/compile.rs +expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" +--- + 1 0 LoadConst (True) + 1 PopJumpIfTrue (6) + 2 LoadConst (False) + 3 PopJumpIfTrue (6) + 4 LoadConst (False) + 5 PopJumpIfFalse (6) + + 2 >> 6 LoadConst (None) + 7 ReturnValue + diff --git a/crates/codegen/src/string_parser.rs b/crates/codegen/src/string_parser.rs new file mode 100644 index 00000000000..a7ad8c35a46 --- /dev/null +++ b/crates/codegen/src/string_parser.rs @@ -0,0 +1,297 @@ +//! A stripped-down version of ruff's string literal parser, modified to +//! handle surrogates in string literals and output WTF-8. +//! +//! Any `unreachable!()` statements in this file are because we only get here +//! after ruff has already successfully parsed the string literal, meaning +//! we don't need to do any validation or error handling. + +use core::convert::Infallible; + +use ruff_python_ast::{self as ast, StringFlags as _}; +use rustpython_wtf8::{CodePoint, Wtf8, Wtf8Buf}; + +// use ruff_python_parser::{LexicalError, LexicalErrorType}; +type LexicalError = Infallible; + +enum EscapedChar { + Literal(CodePoint), + Escape(char), +} + +struct StringParser { + /// The raw content of the string e.g., the `foo` part in `"foo"`. + source: Box<str>, + /// Current position of the parser in the source. + cursor: usize, + /// Flags that can be used to query information about the string. + flags: ast::AnyStringFlags, +} + +impl StringParser { + const fn new(source: Box<str>, flags: ast::AnyStringFlags) -> Self { + Self { + source, + cursor: 0, + flags, + } + } + + #[inline] + fn skip_bytes(&mut self, bytes: usize) -> &str { + let skipped_str = &self.source[self.cursor..self.cursor + bytes]; + self.cursor += bytes; + skipped_str + } + + /// Returns the next byte in the string, if there is one. + /// + /// # Panics + /// + /// When the next byte is a part of a multi-byte character. + #[inline] + fn next_byte(&mut self) -> Option<u8> { + self.source[self.cursor..].as_bytes().first().map(|&byte| { + self.cursor += 1; + byte + }) + } + + #[inline] + fn next_char(&mut self) -> Option<char> { + self.source[self.cursor..].chars().next().inspect(|c| { + self.cursor += c.len_utf8(); + }) + } + + #[inline] + fn peek_byte(&self) -> Option<u8> { + self.source[self.cursor..].as_bytes().first().copied() + } + + fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<CodePoint, LexicalError> { + let mut p: u32 = 0u32; + for i in 1..=literal_number { + match self.next_char() { + Some(c) => match c.to_digit(16) { + Some(d) => p += d << ((literal_number - i) * 4), + None => unreachable!(), + }, + None => unreachable!(), + } + } + Ok(CodePoint::from_u32(p).unwrap()) + } + + fn parse_octet(&mut self, o: u8) -> char { + let mut radix_bytes = [o, 0, 0]; + let mut len = 1; + + while len < 3 { + let Some(b'0'..=b'7') = self.peek_byte() else { + break; + }; + + radix_bytes[len] = self.next_byte().unwrap(); + len += 1; + } + + // OK because radix_bytes is always going to be in the ASCII range. + let radix_str = core::str::from_utf8(&radix_bytes[..len]).expect("ASCII bytes"); + let value = u32::from_str_radix(radix_str, 8).unwrap(); + char::from_u32(value).unwrap() + } + + fn parse_unicode_name(&mut self) -> Result<char, LexicalError> { + let Some('{') = self.next_char() else { + unreachable!() + }; + + let Some(close_idx) = self.source[self.cursor..].find('}') else { + unreachable!() + }; + + let name_and_ending = self.skip_bytes(close_idx + 1); + let name = &name_and_ending[..name_and_ending.len() - 1]; + + unicode_names2::character(name).ok_or_else(|| unreachable!()) + } + + /// Parse an escaped character, returning the new character. + fn parse_escaped_char(&mut self) -> Result<Option<EscapedChar>, LexicalError> { + let Some(first_char) = self.next_char() else { + unreachable!() + }; + + let new_char = match first_char { + '\\' => '\\'.into(), + '\'' => '\''.into(), + '\"' => '"'.into(), + 'a' => '\x07'.into(), + 'b' => '\x08'.into(), + 'f' => '\x0c'.into(), + 'n' => '\n'.into(), + 'r' => '\r'.into(), + 't' => '\t'.into(), + 'v' => '\x0b'.into(), + o @ '0'..='7' => self.parse_octet(o as u8).into(), + 'x' => self.parse_unicode_literal(2)?, + 'u' if !self.flags.is_byte_string() => self.parse_unicode_literal(4)?, + 'U' if !self.flags.is_byte_string() => self.parse_unicode_literal(8)?, + 'N' if !self.flags.is_byte_string() => self.parse_unicode_name()?.into(), + // Special cases where the escape sequence is not a single character + '\n' => return Ok(None), + '\r' => { + if self.peek_byte() == Some(b'\n') { + self.next_byte(); + } + + return Ok(None); + } + _ => return Ok(Some(EscapedChar::Escape(first_char))), + }; + + Ok(Some(EscapedChar::Literal(new_char))) + } + + fn parse_fstring_middle(mut self) -> Result<Box<Wtf8>, LexicalError> { + // Fast-path: if the f-string doesn't contain any escape sequences, return the literal. + let Some(mut index) = memchr::memchr3(b'{', b'}', b'\\', self.source.as_bytes()) else { + return Ok(self.source.into()); + }; + + let mut value = Wtf8Buf::with_capacity(self.source.len()); + loop { + // Add the characters before the escape sequence (or curly brace) to the string. + let before_with_slash_or_brace = self.skip_bytes(index + 1); + let before = &before_with_slash_or_brace[..before_with_slash_or_brace.len() - 1]; + value.push_str(before); + + // Add the escaped character to the string. + match &self.source.as_bytes()[self.cursor - 1] { + // If there are any curly braces inside a `FStringMiddle` token, + // then they were escaped (i.e. `{{` or `}}`). This means that + // we need increase the location by 2 instead of 1. + b'{' => value.push_char('{'), + b'}' => value.push_char('}'), + // We can encounter a `\` as the last character in a `FStringMiddle` + // token which is valid in this context. For example, + // + // ```python + // f"\{foo} \{bar:\}" + // # ^ ^^ ^ + // ``` + // + // Here, the `FStringMiddle` token content will be "\" and " \" + // which is invalid if we look at the content in isolation: + // + // ```python + // "\" + // ``` + // + // However, the content is syntactically valid in the context of + // the f-string because it's a substring of the entire f-string. + // This is still an invalid escape sequence, but we don't want to + // raise a syntax error as is done by the CPython parser. It might + // be supported in the future, refer to point 3: https://peps.python.org/pep-0701/#rejected-ideas + b'\\' => { + if !self.flags.is_raw_string() && self.peek_byte().is_some() { + match self.parse_escaped_char()? { + None => {} + Some(EscapedChar::Literal(c)) => value.push(c), + Some(EscapedChar::Escape(c)) => { + value.push_char('\\'); + value.push_char(c); + } + } + } else { + value.push_char('\\'); + } + } + ch => { + unreachable!("Expected '{{', '}}', or '\\' but got {:?}", ch); + } + } + + let Some(next_index) = + memchr::memchr3(b'{', b'}', b'\\', self.source[self.cursor..].as_bytes()) + else { + // Add the rest of the string to the value. + let rest = &self.source[self.cursor..]; + value.push_str(rest); + break; + }; + + index = next_index; + } + + Ok(value.into()) + } + + fn parse_string(mut self) -> Result<Box<Wtf8>, LexicalError> { + if self.flags.is_raw_string() { + // For raw strings, no escaping is necessary. + return Ok(self.source.into()); + } + + let Some(mut escape) = memchr::memchr(b'\\', self.source.as_bytes()) else { + // If the string doesn't contain any escape sequences, return the owned string. + return Ok(self.source.into()); + }; + + // If the string contains escape sequences, we need to parse them. + let mut value = Wtf8Buf::with_capacity(self.source.len()); + + loop { + // Add the characters before the escape sequence to the string. + let before_with_slash = self.skip_bytes(escape + 1); + let before = &before_with_slash[..before_with_slash.len() - 1]; + value.push_str(before); + + // Add the escaped character to the string. + match self.parse_escaped_char()? { + None => {} + Some(EscapedChar::Literal(c)) => value.push(c), + Some(EscapedChar::Escape(c)) => { + value.push_char('\\'); + value.push_char(c); + } + } + + let Some(next_escape) = self.source[self.cursor..].find('\\') else { + // Add the rest of the string to the value. + let rest = &self.source[self.cursor..]; + value.push_str(rest); + break; + }; + + // Update the position of the next escape sequence. + escape = next_escape; + } + + Ok(value.into()) + } +} + +pub(crate) fn parse_string_literal(source: &str, flags: ast::AnyStringFlags) -> Box<Wtf8> { + let opener_len = flags.opener_len().to_usize(); + let quote_len = flags.quote_len().to_usize(); + if source.len() < opener_len + quote_len { + // Source unavailable (e.g., compiling from an AST object with no + // backing source text). Return the raw source as-is. + return Box::<Wtf8>::from(source); + } + let source = &source[opener_len..]; + let source = &source[..source.len() - quote_len]; + StringParser::new(source.into(), flags) + .parse_string() + .unwrap_or_else(|x| match x {}) +} + +pub(crate) fn parse_fstring_literal_element( + source: Box<str>, + flags: ast::AnyStringFlags, +) -> Box<Wtf8> { + StringParser::new(source, flags) + .parse_fstring_middle() + .unwrap_or_else(|x| match x {}) +} diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs new file mode 100644 index 00000000000..fdbdac2b2a7 --- /dev/null +++ b/crates/codegen/src/symboltable.rs @@ -0,0 +1,2679 @@ +/* Python code is pre-scanned for symbols in the ast. + +This ensures that global and nonlocal keywords are picked up. +Then the compiler can use the symbol table to generate proper +load and store instructions for names. + +Inspirational file: https://github.com/python/cpython/blob/main/Python/symtable.c +*/ + +use crate::{ + IndexMap, IndexSet, + error::{CodegenError, CodegenErrorType}, +}; +use alloc::{borrow::Cow, fmt}; +use bitflags::bitflags; +use ruff_python_ast as ast; +use ruff_text_size::{Ranged, TextRange}; +use rustpython_compiler_core::{PositionEncoding, SourceFile, SourceLocation}; + +/// Captures all symbols in the current scope, and has a list of sub-scopes in this scope. +#[derive(Clone)] +pub struct SymbolTable { + /// The name of this symbol table. Often the name of the class or function. + pub name: String, + + /// The type of symbol table + pub typ: CompilerScope, + + /// The line number in the source code where this symboltable begins. + pub line_number: u32, + + // Return True if the block is a nested class or function + pub is_nested: bool, + + /// A set of symbols present on this scope level. + pub symbols: IndexMap<String, Symbol>, + + /// A list of sub-scopes in the order as found in the + /// AST nodes. + pub sub_tables: Vec<SymbolTable>, + + /// Cursor pointing to the next sub-table to consume during compilation. + pub next_sub_table: usize, + + /// Variable names in definition order (parameters first, then locals) + pub varnames: Vec<String>, + + /// Whether this class scope needs an implicit __class__ cell + pub needs_class_closure: bool, + + /// Whether this class scope needs an implicit __classdict__ cell + pub needs_classdict: bool, + + /// Whether this type param scope can see the parent class scope + pub can_see_class_scope: bool, + + /// Whether this comprehension scope should be inlined (PEP 709) + /// True for list/set/dict comprehensions in non-generator expressions + pub comp_inlined: bool, + + /// PEP 649: Reference to annotation scope for this block + /// Annotations are compiled as a separate `__annotate__` function + pub annotation_block: Option<Box<SymbolTable>>, + + /// PEP 649: Whether this scope has conditional annotations + /// (annotations inside if/for/while/etc. blocks or at module level) + pub has_conditional_annotations: bool, + + /// Whether `from __future__ import annotations` is active + pub future_annotations: bool, + + /// Names of type parameters that should still be mangled in type param scopes. + /// When Some, only names in this set are mangled; other names are left unmangled. + /// Set on type param blocks for generic classes; inherited by non-class child scopes. + pub mangled_names: Option<IndexSet<String>>, +} + +impl SymbolTable { + fn new(name: String, typ: CompilerScope, line_number: u32, is_nested: bool) -> Self { + Self { + name, + typ, + line_number, + is_nested, + symbols: IndexMap::default(), + sub_tables: vec![], + next_sub_table: 0, + varnames: Vec::new(), + needs_class_closure: false, + needs_classdict: false, + can_see_class_scope: false, + comp_inlined: false, + annotation_block: None, + has_conditional_annotations: false, + future_annotations: false, + mangled_names: None, + } + } + + pub fn scan_program( + program: &ast::ModModule, + source_file: SourceFile, + ) -> SymbolTableResult<Self> { + let mut builder = SymbolTableBuilder::new(source_file); + builder.scan_statements(program.body.as_ref())?; + builder.finish() + } + + pub fn scan_expr( + expr: &ast::ModExpression, + source_file: SourceFile, + ) -> SymbolTableResult<Self> { + let mut builder = SymbolTableBuilder::new(source_file); + builder.scan_expression(expr.body.as_ref(), ExpressionContext::Load)?; + builder.finish() + } + + pub fn lookup(&self, name: &str) -> Option<&Symbol> { + self.symbols.get(name) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CompilerScope { + Module, + Class, + Function, + AsyncFunction, + Lambda, + Comprehension, + TypeParams, + /// PEP 649: Annotation scope for deferred evaluation + Annotation, +} + +impl fmt::Display for CompilerScope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Module => write!(f, "module"), + Self::Class => write!(f, "class"), + Self::Function => write!(f, "function"), + Self::AsyncFunction => write!(f, "async function"), + Self::Lambda => write!(f, "lambda"), + Self::Comprehension => write!(f, "comprehension"), + Self::TypeParams => write!(f, "type parameter"), + Self::Annotation => write!(f, "annotation"), + // TODO missing types from the C implementation + // if self._table.type == _symtable.TYPE_TYPE_VAR_BOUND: + // return "TypeVar bound" + // if self._table.type == _symtable.TYPE_TYPE_ALIAS: + // return "type alias" + } + } +} + +/// Indicator for a single symbol what the scope of this symbol is. +/// The scope can be unknown, which is unfortunate, but not impossible. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SymbolScope { + Unknown, + Local, + GlobalExplicit, + GlobalImplicit, + Free, + Cell, +} + +bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct SymbolFlags: u16 { + const REFERENCED = 0x001; // USE + const ASSIGNED = 0x002; // DEF_LOCAL + const PARAMETER = 0x004; // DEF_PARAM + const ANNOTATED = 0x008; // DEF_ANNOT + const IMPORTED = 0x010; // DEF_IMPORT + const NONLOCAL = 0x020; // DEF_NONLOCAL + // indicates if the symbol gets a value assigned by a named expression in a comprehension + // this is required to correct the scope in the analysis. + const ASSIGNED_IN_COMPREHENSION = 0x040; + // indicates that the symbol is used a bound iterator variable. We distinguish this case + // from normal assignment to detect disallowed re-assignment to iterator variables. + const ITER = 0x080; + /// indicates that the symbol is a free variable in a class method from the scope that the + /// class is defined in, e.g.: + /// ```python + /// def foo(x): + /// class A: + /// def method(self): + /// return x // is_free_class + /// ``` + const FREE_CLASS = 0x100; // DEF_FREE_CLASS + const GLOBAL = 0x200; // DEF_GLOBAL + const COMP_ITER = 0x400; // DEF_COMP_ITER + const COMP_CELL = 0x800; // DEF_COMP_CELL + const TYPE_PARAM = 0x1000; // DEF_TYPE_PARAM + const BOUND = Self::ASSIGNED.bits() | Self::PARAMETER.bits() | Self::IMPORTED.bits() | Self::ITER.bits() | Self::TYPE_PARAM.bits(); + } +} + +/// A single symbol in a table. Has various properties such as the scope +/// of the symbol, and also the various uses of the symbol. +#[derive(Debug, Clone)] +pub struct Symbol { + pub name: String, + pub scope: SymbolScope, + pub flags: SymbolFlags, +} + +impl Symbol { + fn new(name: &str) -> Self { + Self { + name: name.to_owned(), + // table, + scope: SymbolScope::Unknown, + flags: SymbolFlags::empty(), + } + } + + pub const fn is_global(&self) -> bool { + matches!( + self.scope, + SymbolScope::GlobalExplicit | SymbolScope::GlobalImplicit + ) + } + + pub const fn is_local(&self) -> bool { + matches!(self.scope, SymbolScope::Local | SymbolScope::Cell) + } + + pub const fn is_bound(&self) -> bool { + self.flags.intersects(SymbolFlags::BOUND) + } +} + +#[derive(Debug)] +pub struct SymbolTableError { + error: String, + location: Option<SourceLocation>, +} + +impl SymbolTableError { + pub fn into_codegen_error(self, source_path: String) -> CodegenError { + CodegenError { + location: self.location, + error: CodegenErrorType::SyntaxError(self.error), + source_path, + } + } +} + +type SymbolTableResult<T = ()> = Result<T, SymbolTableError>; + +impl core::fmt::Debug for SymbolTable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "SymbolTable({:?} symbols, {:?} sub scopes)", + self.symbols.len(), + self.sub_tables.len() + ) + } +} + +/* Perform some sort of analysis on nonlocals, globals etc.. + See also: https://github.com/python/cpython/blob/main/Python/symtable.c#L410 +*/ +fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult { + let mut analyzer = SymbolTableAnalyzer::default(); + // Discard the newfree set at the top level - it's only needed for propagation + // Pass None for class_entry at top level + let _newfree = analyzer.analyze_symbol_table(symbol_table, None)?; + Ok(()) +} + +/* Drop __class__ and __classdict__ from free variables in class scope + and set the appropriate flags. Equivalent to CPython's drop_class_free(). + See: https://github.com/python/cpython/blob/main/Python/symtable.c#L884 + + This function removes __class__ and __classdict__ from the + `newfree` set (which contains free variables collected from all child scopes) + and sets the corresponding flags on the class's symbol table entry. +*/ +fn drop_class_free(symbol_table: &mut SymbolTable, newfree: &mut IndexSet<String>) { + // Check if __class__ is in the free variables collected from children + // If found, it means a child scope (method) references __class__ + if newfree.shift_remove("__class__") { + symbol_table.needs_class_closure = true; + } + + // Check if __classdict__ is in the free variables collected from children + if newfree.shift_remove("__classdict__") { + symbol_table.needs_classdict = true; + } + + // Check if __conditional_annotations__ is in the free variables collected from children + // Remove it from free set - it's handled specially in class scope + if newfree.shift_remove("__conditional_annotations__") { + symbol_table.has_conditional_annotations = true; + } +} + +type SymbolMap = IndexMap<String, Symbol>; + +mod stack { + use alloc::vec::Vec; + use core::ptr::NonNull; + pub struct StackStack<T> { + v: Vec<NonNull<T>>, + } + impl<T> Default for StackStack<T> { + fn default() -> Self { + Self { v: Vec::new() } + } + } + impl<T> StackStack<T> { + /// Appends a reference to this stack for the duration of the function `f`. When `f` + /// returns, the reference will be popped off the stack. + #[cfg(feature = "std")] + pub fn with_append<F, R>(&mut self, x: &mut T, f: F) -> R + where + F: FnOnce(&mut Self) -> R, + { + self.v.push(x.into()); + let res = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| f(self))); + self.v.pop(); + res.unwrap_or_else(|x| std::panic::resume_unwind(x)) + } + + /// Appends a reference to this stack for the duration of the function `f`. When `f` + /// returns, the reference will be popped off the stack. + /// + /// Without std, panic cleanup is not guaranteed (no catch_unwind). + #[cfg(not(feature = "std"))] + pub fn with_append<F, R>(&mut self, x: &mut T, f: F) -> R + where + F: FnOnce(&mut Self) -> R, + { + self.v.push(x.into()); + let result = f(self); + self.v.pop(); + result + } + + pub fn iter(&self) -> impl DoubleEndedIterator<Item = &T> + '_ { + self.as_ref().iter().copied() + } + pub fn iter_mut(&mut self) -> impl DoubleEndedIterator<Item = &mut T> + '_ { + self.as_mut().iter_mut().map(|x| &mut **x) + } + // pub fn top(&self) -> Option<&T> { + // self.as_ref().last().copied() + // } + // pub fn top_mut(&mut self) -> Option<&mut T> { + // self.as_mut().last_mut().map(|x| &mut **x) + // } + pub fn len(&self) -> usize { + self.v.len() + } + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn as_ref(&self) -> &[&T] { + unsafe { &*(self.v.as_slice() as *const [NonNull<T>] as *const [&T]) } + } + + pub fn as_mut(&mut self) -> &mut [&mut T] { + unsafe { &mut *(self.v.as_mut_slice() as *mut [NonNull<T>] as *mut [&mut T]) } + } + } +} +use stack::StackStack; + +/// Symbol table analysis. Can be used to analyze a fully +/// build symbol table structure. It will mark variables +/// as local variables for example. +#[derive(Default)] +#[repr(transparent)] +struct SymbolTableAnalyzer { + tables: StackStack<(SymbolMap, CompilerScope)>, +} + +impl SymbolTableAnalyzer { + /// Analyze a symbol table and return the set of free variables. + /// See symtable.c analyze_block(). + /// class_entry: PEP 649 - enclosing class symbols for annotation scopes + fn analyze_symbol_table( + &mut self, + symbol_table: &mut SymbolTable, + class_entry: Option<&SymbolMap>, + ) -> SymbolTableResult<IndexSet<String>> { + let symbols = core::mem::take(&mut symbol_table.symbols); + let sub_tables = &mut *symbol_table.sub_tables; + + // Collect free variables from all child scopes + let mut newfree = IndexSet::default(); + + let annotation_block = &mut symbol_table.annotation_block; + + // PEP 649: Determine class_entry to pass to children + // If current scope is a class with annotation block that can_see_class_scope, + // we need to pass class symbols to the annotation scope + let is_class = symbol_table.typ == CompilerScope::Class; + + // Clone class symbols if needed for child scopes with can_see_class_scope + let needs_class_symbols = (is_class + && (sub_tables.iter().any(|st| st.can_see_class_scope) + || annotation_block + .as_ref() + .is_some_and(|b| b.can_see_class_scope))) + || (!is_class + && class_entry.is_some() + && sub_tables.iter().any(|st| st.can_see_class_scope)); + + let class_symbols_clone = if is_class && needs_class_symbols { + Some(symbols.clone()) + } else { + None + }; + + let mut info = (symbols, symbol_table.typ); + self.tables.with_append(&mut info, |list| { + let inner_scope = unsafe { &mut *(list as *mut _ as *mut Self) }; + // Analyze sub scopes and collect their free variables + for sub_table in sub_tables.iter_mut() { + // Pass class_entry to sub-scopes that can see the class scope + let child_class_entry = if sub_table.can_see_class_scope { + if is_class { + class_symbols_clone.as_ref() + } else { + class_entry + } + } else { + None + }; + let child_free = inner_scope.analyze_symbol_table(sub_table, child_class_entry)?; + // Propagate child's free variables to this scope + newfree.extend(child_free); + } + // PEP 649: Analyze annotation block if present + if let Some(annotation_table) = annotation_block { + // Pass class symbols to annotation scope if can_see_class_scope + let ann_class_entry = if annotation_table.can_see_class_scope { + if is_class { + class_symbols_clone.as_ref() + } else { + class_entry + } + } else { + None + }; + let child_free = + inner_scope.analyze_symbol_table(annotation_table, ann_class_entry)?; + // Propagate annotation's free variables to this scope + newfree.extend(child_free); + } + Ok(()) + })?; + + symbol_table.symbols = info.0; + + // PEP 709: Merge symbols from inlined comprehensions into parent scope + // Only merge symbols that are actually bound in the comprehension, + // not references to outer scope variables (Free symbols). + const BOUND_FLAGS: SymbolFlags = SymbolFlags::ASSIGNED + .union(SymbolFlags::PARAMETER) + .union(SymbolFlags::ITER) + .union(SymbolFlags::ASSIGNED_IN_COMPREHENSION); + + for sub_table in sub_tables.iter() { + if sub_table.comp_inlined { + for (name, sub_symbol) in &sub_table.symbols { + // Skip the .0 parameter - it's internal to the comprehension + if name == ".0" { + continue; + } + // Only merge symbols that are bound in the comprehension + // Skip Free references to outer scope variables + if !sub_symbol.flags.intersects(BOUND_FLAGS) { + continue; + } + // If the symbol doesn't exist in parent, add it + if !symbol_table.symbols.contains_key(name) { + let mut symbol = sub_symbol.clone(); + // Mark as local in parent scope + symbol.scope = SymbolScope::Local; + symbol_table.symbols.insert(name.clone(), symbol); + } + } + } + } + + // Analyze symbols in current scope + for symbol in symbol_table.symbols.values_mut() { + self.analyze_symbol(symbol, symbol_table.typ, sub_tables, class_entry)?; + + // Collect free variables from this scope + // These will be propagated to the parent scope + if symbol.scope == SymbolScope::Free || symbol.flags.contains(SymbolFlags::FREE_CLASS) { + newfree.insert(symbol.name.clone()); + } + } + + // Handle class-specific implicit cells + // This removes __class__ and __classdict__ from newfree if present + // and sets the corresponding flags on the symbol table + if symbol_table.typ == CompilerScope::Class { + drop_class_free(symbol_table, &mut newfree); + } + + Ok(newfree) + } + + fn analyze_symbol( + &mut self, + symbol: &mut Symbol, + st_typ: CompilerScope, + sub_tables: &[SymbolTable], + class_entry: Option<&SymbolMap>, + ) -> SymbolTableResult { + if symbol + .flags + .contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) + && st_typ == CompilerScope::Comprehension + { + // propagate symbol to next higher level that can hold it, + // i.e., function or module. Comprehension is skipped and + // Class is not allowed and detected as error. + //symbol.scope = SymbolScope::Nonlocal; + self.analyze_symbol_comprehension(symbol, 0)? + } else { + match symbol.scope { + SymbolScope::Free => { + if !self.tables.as_ref().is_empty() { + let scope_depth = self.tables.as_ref().len(); + // check if the name is already defined in any outer scope + if scope_depth < 2 + || self.found_in_outer_scope(&symbol.name, st_typ) + != Some(SymbolScope::Free) + { + return Err(SymbolTableError { + error: format!("no binding for nonlocal '{}' found", symbol.name), + // TODO: accurate location info, somehow + location: None, + }); + } + // Check if the nonlocal binding refers to a type parameter + if symbol.flags.contains(SymbolFlags::NONLOCAL) { + for (symbols, _typ) in self.tables.iter().rev() { + if let Some(sym) = symbols.get(&symbol.name) { + if sym.flags.contains(SymbolFlags::TYPE_PARAM) { + return Err(SymbolTableError { + error: format!( + "nonlocal binding not allowed for type parameter '{}'", + symbol.name + ), + location: None, + }); + } + if sym.is_bound() { + break; + } + } + } + } + } else { + return Err(SymbolTableError { + error: format!( + "nonlocal {} defined at place without an enclosing scope", + symbol.name + ), + // TODO: accurate location info, somehow + location: None, + }); + } + } + SymbolScope::GlobalExplicit | SymbolScope::GlobalImplicit => { + // TODO: add more checks for globals? + } + SymbolScope::Local | SymbolScope::Cell => { + // all is well + } + SymbolScope::Unknown => { + // Try hard to figure out what the scope of this symbol is. + let scope = if symbol.is_bound() { + self.found_in_inner_scope(sub_tables, &symbol.name, st_typ) + .unwrap_or(SymbolScope::Local) + } else if let Some(scope) = self.found_in_outer_scope(&symbol.name, st_typ) { + // If found in enclosing scope (function/TypeParams), use that + scope + } else if let Some(class_symbols) = class_entry + && let Some(class_sym) = class_symbols.get(&symbol.name) + && class_sym.is_bound() + && class_sym.scope != SymbolScope::Free + { + // If name is bound in enclosing class, use GlobalImplicit + // so it can be accessed via __classdict__ + SymbolScope::GlobalImplicit + } else if self.tables.is_empty() { + // Don't make assumptions when we don't know. + SymbolScope::Unknown + } else { + // If there are scopes above we assume global. + SymbolScope::GlobalImplicit + }; + symbol.scope = scope; + } + } + } + Ok(()) + } + + fn found_in_outer_scope(&mut self, name: &str, st_typ: CompilerScope) -> Option<SymbolScope> { + let mut decl_depth = None; + for (i, (symbols, typ)) in self.tables.iter().rev().enumerate() { + if matches!(typ, CompilerScope::Module) + || matches!(typ, CompilerScope::Class if name != "__class__" && name != "__classdict__" && name != "__conditional_annotations__") + { + continue; + } + + // PEP 649: Annotation scope is conceptually a sibling of the function, + // not a child. Skip the immediate parent function scope when looking + // for outer variables from annotation scope. + if st_typ == CompilerScope::Annotation + && i == 0 + && matches!( + typ, + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda + ) + { + continue; + } + + // __class__ and __classdict__ are implicitly declared in class scope + // This handles the case where nested scopes reference them + if (name == "__class__" || name == "__classdict__") + && matches!(typ, CompilerScope::Class) + { + decl_depth = Some(i); + break; + } + + // __conditional_annotations__ is implicitly declared in class scope + // for classes with conditional annotations + if name == "__conditional_annotations__" && matches!(typ, CompilerScope::Class) { + decl_depth = Some(i); + break; + } + + if let Some(sym) = symbols.get(name) { + match sym.scope { + SymbolScope::GlobalExplicit => return Some(SymbolScope::GlobalExplicit), + SymbolScope::GlobalImplicit => {} + _ => { + if sym.is_bound() { + decl_depth = Some(i); + break; + } + } + } + } + } + + if let Some(decl_depth) = decl_depth { + // decl_depth is the number of tables between the current one and + // the one that declared the cell var + for (table, typ) in self.tables.iter_mut().rev().take(decl_depth) { + if let CompilerScope::Class = typ { + if let Some(free_class) = table.get_mut(name) { + free_class.flags.insert(SymbolFlags::FREE_CLASS) + } else { + let mut symbol = Symbol::new(name); + symbol.flags.insert(SymbolFlags::FREE_CLASS); + symbol.scope = SymbolScope::Free; + table.insert(name.to_owned(), symbol); + } + } else if !table.contains_key(name) { + let mut symbol = Symbol::new(name); + symbol.scope = SymbolScope::Free; + // symbol.is_referenced = true; + table.insert(name.to_owned(), symbol); + } + } + } + + decl_depth.map(|_| SymbolScope::Free) + } + + fn found_in_inner_scope( + &self, + sub_tables: &[SymbolTable], + name: &str, + st_typ: CompilerScope, + ) -> Option<SymbolScope> { + sub_tables.iter().find_map(|st| { + let sym = st.symbols.get(name)?; + if sym.scope == SymbolScope::Free || sym.flags.contains(SymbolFlags::FREE_CLASS) { + if st_typ == CompilerScope::Class && name != "__class__" { + None + } else { + Some(SymbolScope::Cell) + } + } else if sym.scope == SymbolScope::GlobalExplicit && self.tables.is_empty() { + // the symbol is defined on the module level, and an inner scope declares + // a global that points to it + Some(SymbolScope::GlobalExplicit) + } else { + None + } + }) + } + + // Implements the symbol analysis and scope extension for names + // assigned by a named expression in a comprehension. See: + // https://github.com/python/cpython/blob/7b78e7f9fd77bb3280ee39fb74b86772a7d46a70/Python/symtable.c#L1435 + fn analyze_symbol_comprehension( + &mut self, + symbol: &mut Symbol, + parent_offset: usize, + ) -> SymbolTableResult { + // when this is called, we expect to be in the direct parent scope of the scope that contains 'symbol' + let last = self.tables.iter_mut().rev().nth(parent_offset).unwrap(); + let symbols = &mut last.0; + let table_type = last.1; + + // it is not allowed to use an iterator variable as assignee in a named expression + if symbol.flags.contains(SymbolFlags::ITER) { + return Err(SymbolTableError { + error: format!( + "assignment expression cannot rebind comprehension iteration variable {}", + symbol.name + ), + // TODO: accurate location info, somehow + location: None, + }); + } + + match table_type { + CompilerScope::Module => { + symbol.scope = SymbolScope::GlobalImplicit; + } + CompilerScope::Class => { + // named expressions are forbidden in comprehensions on class scope + return Err(SymbolTableError { + error: "assignment expression within a comprehension cannot be used in a class body".to_string(), + // TODO: accurate location info, somehow + location: None, + }); + } + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda => { + if let Some(parent_symbol) = symbols.get_mut(&symbol.name) { + if let SymbolScope::Unknown = parent_symbol.scope { + // this information is new, as the assignment is done in inner scope + parent_symbol.flags.insert(SymbolFlags::ASSIGNED); + } + + symbol.scope = if parent_symbol.is_global() { + parent_symbol.scope + } else { + SymbolScope::Free + }; + } else { + let mut cloned_sym = symbol.clone(); + cloned_sym.scope = SymbolScope::Cell; + last.0.insert(cloned_sym.name.to_owned(), cloned_sym); + } + } + CompilerScope::Comprehension => { + // TODO check for conflicts - requires more context information about variables + match symbols.get_mut(&symbol.name) { + Some(parent_symbol) => { + // check if assignee is an iterator in top scope + if parent_symbol.flags.contains(SymbolFlags::ITER) { + return Err(SymbolTableError { + error: format!( + "assignment expression cannot rebind comprehension iteration variable {}", + symbol.name + ), + location: None, + }); + } + + // we synthesize the assignment to the symbol from inner scope + parent_symbol.flags.insert(SymbolFlags::ASSIGNED); // more checks are required + } + None => { + // extend the scope of the inner symbol + // as we are in a nested comprehension, we expect that the symbol is needed + // outside, too, and set it therefore to non-local scope. I.e., we expect to + // find a definition on a higher level + let mut cloned_sym = symbol.clone(); + cloned_sym.scope = SymbolScope::Free; + last.0.insert(cloned_sym.name.to_owned(), cloned_sym); + } + } + + self.analyze_symbol_comprehension(symbol, parent_offset + 1)?; + } + CompilerScope::TypeParams => { + // Named expression in comprehension cannot be used in type params + return Err(SymbolTableError { + error: "assignment expression within a comprehension cannot be used within the definition of a generic".to_string(), + location: None, + }); + } + CompilerScope::Annotation => { + // Named expression is not allowed in annotation scope + return Err(SymbolTableError { + error: "named expression cannot be used within an annotation".to_string(), + location: None, + }); + } + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug)] +enum SymbolUsage { + Global, + Nonlocal, + Used, + Assigned, + Imported, + AnnotationAssigned, + Parameter, + AnnotationParameter, + AssignedNamedExprInComprehension, + Iter, + TypeParam, +} + +struct SymbolTableBuilder { + class_name: Option<String>, + // Scope stack. + tables: Vec<SymbolTable>, + future_annotations: bool, + source_file: SourceFile, + // Current scope's varnames being collected (temporary storage) + current_varnames: Vec<String>, + // Stack to preserve parent varnames when entering nested scopes + varnames_stack: Vec<Vec<String>>, + // Track if we're inside an iterable definition expression (for nested comprehensions) + in_iter_def_exp: bool, + // Track if we're inside an annotation (yield/await/named expr not allowed) + in_annotation: bool, + // Track if we're inside a type alias (yield/await/named expr not allowed) + in_type_alias: bool, + // Track if we're scanning an inner loop iteration target (not the first generator) + in_comp_inner_loop_target: bool, + // Scope info for error messages (e.g., "a TypeVar bound") + scope_info: Option<&'static str>, + // PEP 649: Track if we're inside a conditional block (if/for/while/etc.) + in_conditional_block: bool, +} + +/// Enum to indicate in what mode an expression +/// was used. +/// In cpython this is stored in the AST, but I think this +/// is not logical, since it is not context free. +#[derive(Copy, Clone, PartialEq)] +enum ExpressionContext { + Load, + Store, + Delete, + Iter, + IterDefinitionExp, +} + +impl SymbolTableBuilder { + fn new(source_file: SourceFile) -> Self { + let mut this = Self { + class_name: None, + tables: vec![], + future_annotations: false, + source_file, + current_varnames: Vec::new(), + varnames_stack: Vec::new(), + in_iter_def_exp: false, + in_annotation: false, + in_type_alias: false, + in_comp_inner_loop_target: false, + scope_info: None, + in_conditional_block: false, + }; + this.enter_scope("top", CompilerScope::Module, 0); + this + } + + fn finish(mut self) -> Result<SymbolTable, SymbolTableError> { + assert_eq!(self.tables.len(), 1); + let mut symbol_table = self.tables.pop().unwrap(); + // Save varnames for the top-level module scope + symbol_table.varnames = self.current_varnames; + // Propagate future_annotations to the symbol table + symbol_table.future_annotations = self.future_annotations; + analyze_symbol_table(&mut symbol_table)?; + Ok(symbol_table) + } + + fn enter_scope(&mut self, name: &str, typ: CompilerScope, line_number: u32) { + let is_nested = self + .tables + .last() + .map(|table| { + table.is_nested + || matches!( + table.typ, + CompilerScope::Function | CompilerScope::AsyncFunction + ) + }) + .unwrap_or(false); + // Inherit mangled_names from parent for non-class scopes + let inherited_mangled_names = self + .tables + .last() + .and_then(|t| t.mangled_names.clone()) + .filter(|_| typ != CompilerScope::Class); + let mut table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested); + table.mangled_names = inherited_mangled_names; + self.tables.push(table); + // Save parent's varnames and start fresh for the new scope + self.varnames_stack + .push(core::mem::take(&mut self.current_varnames)); + } + + fn enter_type_param_block( + &mut self, + name: &str, + line_number: u32, + for_class: bool, + ) -> SymbolTableResult { + // Check if we're in a class scope + let in_class = self + .tables + .last() + .is_some_and(|t| t.typ == CompilerScope::Class); + + self.enter_scope(name, CompilerScope::TypeParams, line_number); + + // Set properties on the newly created type param scope + if let Some(table) = self.tables.last_mut() { + table.can_see_class_scope = in_class; + // For generic classes, create mangled_names set so that only + // type parameter names get mangled (not bases or other expressions) + if for_class { + table.mangled_names = Some(IndexSet::default()); + } + } + + // Add __classdict__ as a USE symbol in type param scope if in class + if in_class { + self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + } + + // Register .type_params as a SET symbol (it will be converted to cell variable later) + self.register_name(".type_params", SymbolUsage::Assigned, TextRange::default())?; + + Ok(()) + } + + /// Pop symbol table and add to sub table of parent table. + fn leave_scope(&mut self) { + let mut table = self.tables.pop().unwrap(); + // Save the collected varnames to the symbol table + table.varnames = core::mem::take(&mut self.current_varnames); + self.tables.last_mut().unwrap().sub_tables.push(table); + // Restore parent's varnames + self.current_varnames = self.varnames_stack.pop().unwrap_or_default(); + } + + /// Enter annotation scope (PEP 649) + /// Creates or reuses the annotation block for the current scope + fn enter_annotation_scope(&mut self, line_number: u32) { + let current = self.tables.last_mut().unwrap(); + let can_see_class_scope = + current.typ == CompilerScope::Class || current.can_see_class_scope; + let has_conditional = current.has_conditional_annotations; + + // Create annotation block if not exists + if current.annotation_block.is_none() { + let mut annotation_table = SymbolTable::new( + "__annotate__".to_owned(), + CompilerScope::Annotation, + line_number, + true, // is_nested + ); + // Annotation scope in class can see class scope + annotation_table.can_see_class_scope = can_see_class_scope; + // Add 'format' parameter + annotation_table.varnames.push("format".to_owned()); + current.annotation_block = Some(Box::new(annotation_table)); + } + + // Take the annotation block and push to stack for processing + let annotation_table = current.annotation_block.take().unwrap(); + self.tables.push(*annotation_table); + // Save parent's varnames and seed with existing annotation varnames (e.g., "format") + self.varnames_stack + .push(core::mem::take(&mut self.current_varnames)); + self.current_varnames = self.tables.last().unwrap().varnames.clone(); + + if can_see_class_scope && !self.future_annotations { + self.add_classdict_freevar(); + // Also add __conditional_annotations__ as free var if parent has conditional annotations + if has_conditional { + self.add_conditional_annotations_freevar(); + } + } + } + + /// Leave annotation scope (PEP 649) + /// Stores the annotation block back to parent instead of sub_tables + fn leave_annotation_scope(&mut self) { + let mut table = self.tables.pop().unwrap(); + // Save the collected varnames to the symbol table + table.varnames = core::mem::take(&mut self.current_varnames); + // Store back to parent's annotation_block (not sub_tables) + let parent = self.tables.last_mut().unwrap(); + parent.annotation_block = Some(Box::new(table)); + // Restore parent's varnames + self.current_varnames = self.varnames_stack.pop().unwrap_or_default(); + } + + fn add_classdict_freevar(&mut self) { + let table = self.tables.last_mut().unwrap(); + let name = "__classdict__"; + let symbol = table + .symbols + .entry(name.to_owned()) + .or_insert_with(|| Symbol::new(name)); + symbol.scope = SymbolScope::Free; + symbol + .flags + .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); + } + + fn add_conditional_annotations_freevar(&mut self) { + let table = self.tables.last_mut().unwrap(); + let name = "__conditional_annotations__"; + let symbol = table + .symbols + .entry(name.to_owned()) + .or_insert_with(|| Symbol::new(name)); + symbol.scope = SymbolScope::Free; + symbol + .flags + .insert(SymbolFlags::REFERENCED | SymbolFlags::FREE_CLASS); + } + + /// Walk up the scope chain to determine if we're inside an async function. + /// Annotation and TypeParams scopes act as async barriers (always non-async). + /// Comprehension scopes are transparent (inherit parent's async context). + fn is_in_async_context(&self) -> bool { + // Annotations are evaluated in a non-async scope even when + // the enclosing function is async. + if self.in_annotation { + return false; + } + for table in self.tables.iter().rev() { + match table.typ { + CompilerScope::AsyncFunction => return true, + CompilerScope::Function + | CompilerScope::Lambda + | CompilerScope::Class + | CompilerScope::Module + | CompilerScope::Annotation + | CompilerScope::TypeParams => return false, + // Comprehension inherits parent's async context + CompilerScope::Comprehension => continue, + } + } + false + } + + fn line_index_start(&self, range: TextRange) -> u32 { + self.source_file + .to_source_code() + .line_index(range.start()) + .get() as _ + } + + fn scan_statements(&mut self, statements: &[ast::Stmt]) -> SymbolTableResult { + for statement in statements { + self.scan_statement(statement)?; + } + Ok(()) + } + + fn scan_parameters(&mut self, parameters: &[ast::ParameterWithDefault]) -> SymbolTableResult { + for parameter in parameters { + self.scan_parameter(&parameter.parameter)?; + } + Ok(()) + } + + fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + self.check_name( + parameter.name.as_str(), + ExpressionContext::Store, + parameter.name.range, + )?; + + let usage = if parameter.annotation.is_some() { + SymbolUsage::AnnotationParameter + } else { + SymbolUsage::Parameter + }; + + // Check for duplicate parameter names + let table = self.tables.last().unwrap(); + if table.symbols.contains_key(parameter.name.as_str()) { + return Err(SymbolTableError { + error: format!( + "duplicate argument '{}' in function definition", + parameter.name + ), + location: Some( + self.source_file + .to_source_code() + .source_location(parameter.name.range.start(), PositionEncoding::Utf8), + ), + }); + } + + self.register_ident(&parameter.name, usage) + } + + fn scan_annotation(&mut self, annotation: &ast::Expr) -> SymbolTableResult { + let current_scope = self.tables.last().map(|t| t.typ); + + // PEP 649: Check if this is a conditional annotation + // Module-level: always conditional (module may be partially executed) + // Class-level: conditional only when inside if/for/while/etc. + if !self.future_annotations { + let is_conditional = matches!(current_scope, Some(CompilerScope::Module)) + || (matches!(current_scope, Some(CompilerScope::Class)) + && self.in_conditional_block); + + if is_conditional && !self.tables.last().unwrap().has_conditional_annotations { + self.tables.last_mut().unwrap().has_conditional_annotations = true; + // Register __conditional_annotations__ as both Assigned and Used so that + // it becomes a Cell variable in class scope (children reference it as Free) + self.register_name( + "__conditional_annotations__", + SymbolUsage::Assigned, + annotation.range(), + )?; + self.register_name( + "__conditional_annotations__", + SymbolUsage::Used, + annotation.range(), + )?; + } + } + + // Create annotation scope for deferred evaluation + let line_number = self.line_index_start(annotation.range()); + self.enter_annotation_scope(line_number); + + if self.future_annotations { + // PEP 563: annotations are stringified at compile time + // Don't scan expression - symbols would fail to resolve + // Just create the annotation_block structure + self.leave_annotation_scope(); + return Ok(()); + } + + // PEP 649: scan expression for symbol references + // Class annotations are evaluated in class locals (not module globals) + let was_in_annotation = self.in_annotation; + self.in_annotation = true; + let result = self.scan_expression(annotation, ExpressionContext::Load); + self.in_annotation = was_in_annotation; + + self.leave_annotation_scope(); + + result + } + + fn scan_statement(&mut self, statement: &ast::Stmt) -> SymbolTableResult { + use ast::*; + if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement + && module.as_ref().map(|id| id.as_str()) == Some("__future__") + { + self.future_annotations = + self.future_annotations || names.iter().any(|future| &future.name == "annotations"); + } + + match &statement { + Stmt::Global(StmtGlobal { names, .. }) => { + for name in names { + self.register_ident(name, SymbolUsage::Global)?; + } + } + Stmt::Nonlocal(StmtNonlocal { names, .. }) => { + for name in names { + self.register_ident(name, SymbolUsage::Nonlocal)?; + } + } + Stmt::FunctionDef(StmtFunctionDef { + name, + body, + parameters, + decorator_list, + type_params, + returns, + range, + is_async, + .. + }) => { + self.scan_decorators(decorator_list, ExpressionContext::Load)?; + self.register_ident(name, SymbolUsage::Assigned)?; + + // Save the parent's annotation_block before scanning function annotations, + // so function annotations don't interfere with parent scope annotations. + // This applies to both class scope (methods) and module scope (top-level functions). + let parent_scope_typ = self.tables.last().map(|t| t.typ); + let should_save_annotation_block = matches!( + parent_scope_typ, + Some(CompilerScope::Class) + | Some(CompilerScope::Module) + | Some(CompilerScope::Function) + | Some(CompilerScope::AsyncFunction) + ); + let saved_annotation_block = if should_save_annotation_block { + self.tables.last_mut().unwrap().annotation_block.take() + } else { + None + }; + + // For generic functions, scan defaults before entering type_param_block + // (defaults are evaluated in the enclosing scope, not the type param scope) + let has_type_params = type_params.is_some(); + if has_type_params { + self.scan_parameter_defaults(parameters)?; + } + + // For generic functions, enter type_param block FIRST so that + // annotation scopes are nested inside and can see type parameters. + if let Some(type_params) = type_params { + self.enter_type_param_block( + &format!("<generic parameters of {}>", name.as_str()), + self.line_index_start(type_params.range), + false, + )?; + self.scan_type_params(type_params)?; + } + let has_return_annotation = if let Some(expression) = returns { + self.scan_annotation(expression)?; + true + } else { + false + }; + self.enter_scope_with_parameters( + name.as_str(), + parameters, + self.line_index_start(*range), + has_return_annotation, + *is_async, + has_type_params, // skip_defaults: already scanned above + )?; + self.scan_statements(body)?; + self.leave_scope(); + if type_params.is_some() { + self.leave_scope(); + } + + // Restore parent's annotation_block after processing the function + if let Some(block) = saved_annotation_block { + self.tables.last_mut().unwrap().annotation_block = Some(block); + } + } + Stmt::ClassDef(StmtClassDef { + name, + body, + arguments, + decorator_list, + type_params, + range, + node_index: _, + }) => { + // Save class_name for the entire ClassDef processing + let prev_class = self.class_name.take(); + if let Some(type_params) = type_params { + self.enter_type_param_block( + &format!("<generic parameters of {}>", name.as_str()), + self.line_index_start(type_params.range), + true, // for_class: enable selective mangling + )?; + // Set class_name for mangling in type param scope + self.class_name = Some(name.to_string()); + self.scan_type_params(type_params)?; + } + self.enter_scope( + name.as_str(), + CompilerScope::Class, + self.line_index_start(*range), + ); + // Reset in_conditional_block for new class scope + let saved_in_conditional = self.in_conditional_block; + self.in_conditional_block = false; + self.class_name = Some(name.to_string()); + self.register_name("__module__", SymbolUsage::Assigned, *range)?; + self.register_name("__qualname__", SymbolUsage::Assigned, *range)?; + self.register_name("__doc__", SymbolUsage::Assigned, *range)?; + self.register_name("__class__", SymbolUsage::Assigned, *range)?; + self.scan_statements(body)?; + self.leave_scope(); + self.in_conditional_block = saved_in_conditional; + // For non-generic classes, restore class_name before base scanning. + // Bases are evaluated in the enclosing scope, not the class scope. + // For generic classes, bases are scanned within the type_param scope + // where class_name is already correctly set. + if type_params.is_none() { + self.class_name = prev_class.clone(); + } + if let Some(arguments) = arguments { + self.scan_expressions(&arguments.args, ExpressionContext::Load)?; + for keyword in &arguments.keywords { + self.scan_expression(&keyword.value, ExpressionContext::Load)?; + } + } + if type_params.is_some() { + self.leave_scope(); + } + // Restore class_name after all ClassDef processing + self.class_name = prev_class; + self.scan_decorators(decorator_list, ExpressionContext::Load)?; + self.register_ident(name, SymbolUsage::Assigned)?; + } + Stmt::Expr(StmtExpr { value, .. }) => { + self.scan_expression(value, ExpressionContext::Load)? + } + Stmt::If(StmtIf { + test, + body, + elif_else_clauses, + .. + }) => { + self.scan_expression(test, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; + self.scan_statements(body)?; + for elif in elif_else_clauses { + if let Some(test) = &elif.test { + self.scan_expression(test, ExpressionContext::Load)?; + } + self.scan_statements(&elif.body)?; + } + self.in_conditional_block = saved_in_conditional_block; + } + Stmt::For(StmtFor { + target, + iter, + body, + orelse, + .. + }) => { + self.scan_expression(target, ExpressionContext::Store)?; + self.scan_expression(iter, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; + self.scan_statements(body)?; + self.scan_statements(orelse)?; + self.in_conditional_block = saved_in_conditional_block; + } + Stmt::While(StmtWhile { + test, body, orelse, .. + }) => { + self.scan_expression(test, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; + self.scan_statements(body)?; + self.scan_statements(orelse)?; + self.in_conditional_block = saved_in_conditional_block; + } + Stmt::Break(_) | Stmt::Continue(_) | Stmt::Pass(_) => { + // No symbols here. + } + Stmt::Import(StmtImport { names, .. }) + | Stmt::ImportFrom(StmtImportFrom { names, .. }) => { + for name in names { + if let Some(alias) = &name.asname { + // `import my_module as my_alias` + self.check_name(alias.as_str(), ExpressionContext::Store, alias.range)?; + self.register_ident(alias, SymbolUsage::Imported)?; + } else if name.name.as_str() == "*" { + // Star imports are only allowed at module level + if self.tables.last().unwrap().typ != CompilerScope::Module { + return Err(SymbolTableError { + error: "'import *' only allowed at module level".to_string(), + location: Some(self.source_file.to_source_code().source_location( + name.name.range.start(), + PositionEncoding::Utf8, + )), + }); + } + // Don't register star imports as symbols + } else { + // `import module` or `from x import name` + let imported_name = name.name.split('.').next().unwrap(); + self.check_name(imported_name, ExpressionContext::Store, name.name.range)?; + self.register_name(imported_name, SymbolUsage::Imported, name.name.range)?; + } + } + } + Stmt::Return(StmtReturn { value, .. }) => { + if let Some(expression) = value { + self.scan_expression(expression, ExpressionContext::Load)?; + } + } + Stmt::Assert(StmtAssert { test, msg, .. }) => { + self.scan_expression(test, ExpressionContext::Load)?; + if let Some(expression) = msg { + self.scan_expression(expression, ExpressionContext::Load)?; + } + } + Stmt::Delete(StmtDelete { targets, .. }) => { + self.scan_expressions(targets, ExpressionContext::Delete)?; + } + Stmt::Assign(StmtAssign { targets, value, .. }) => { + self.scan_expressions(targets, ExpressionContext::Store)?; + self.scan_expression(value, ExpressionContext::Load)?; + } + Stmt::AugAssign(StmtAugAssign { target, value, .. }) => { + self.scan_expression(target, ExpressionContext::Store)?; + self.scan_expression(value, ExpressionContext::Load)?; + } + Stmt::AnnAssign(StmtAnnAssign { + target, + annotation, + value, + simple, + range, + node_index: _, + }) => { + // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233 + match &**target { + Expr::Name(ast::ExprName { id, .. }) if *simple => { + let id_str = id.as_str(); + + self.check_name(id_str, ExpressionContext::Store, *range)?; + + self.register_name(id_str, SymbolUsage::AnnotationAssigned, *range)?; + // PEP 649: Register annotate function in module/class scope + let current_scope = self.tables.last().map(|t| t.typ); + match current_scope { + Some(CompilerScope::Module) => { + self.register_name("__annotate__", SymbolUsage::Assigned, *range)?; + } + Some(CompilerScope::Class) => { + self.register_name( + "__annotate_func__", + SymbolUsage::Assigned, + *range, + )?; + } + _ => {} + } + } + _ => { + self.scan_expression(target, ExpressionContext::Store)?; + } + } + // Only scan annotation in annotation scope for simple name targets. + // Non-simple annotations (subscript, attribute, parenthesized) are + // never compiled into __annotate__, so scanning them would create + // sub_tables that cause mismatch in the annotation scope's sub_table index. + let is_simple_name = *simple && matches!(&**target, Expr::Name(_)); + if is_simple_name { + self.scan_annotation(annotation)?; + } else { + // Still validate annotation for forbidden expressions + // (yield, await, named) even for non-simple targets. + let was_in_annotation = self.in_annotation; + self.in_annotation = true; + let result = self.scan_expression(annotation, ExpressionContext::Load); + self.in_annotation = was_in_annotation; + result?; + } + if let Some(value) = value { + self.scan_expression(value, ExpressionContext::Load)?; + } + } + Stmt::With(StmtWith { items, body, .. }) => { + for item in items { + self.scan_expression(&item.context_expr, ExpressionContext::Load)?; + if let Some(expression) = &item.optional_vars { + self.scan_expression(expression, ExpressionContext::Store)?; + } + } + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; + self.scan_statements(body)?; + self.in_conditional_block = saved_in_conditional_block; + } + Stmt::Try(StmtTry { + body, + handlers, + orelse, + finalbody, + .. + }) => { + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; + self.scan_statements(body)?; + for handler in handlers { + let ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { + type_, + name, + body, + .. + }) = &handler; + if let Some(expression) = type_ { + self.scan_expression(expression, ExpressionContext::Load)?; + } + if let Some(name) = name { + self.register_ident(name, SymbolUsage::Assigned)?; + } + self.scan_statements(body)?; + } + self.scan_statements(orelse)?; + self.scan_statements(finalbody)?; + self.in_conditional_block = saved_in_conditional_block; + } + Stmt::Match(StmtMatch { subject, cases, .. }) => { + self.scan_expression(subject, ExpressionContext::Load)?; + // PEP 649: Track conditional block for annotations + let saved_in_conditional_block = self.in_conditional_block; + self.in_conditional_block = true; + for case in cases { + self.scan_pattern(&case.pattern)?; + if let Some(guard) = &case.guard { + self.scan_expression(guard, ExpressionContext::Load)?; + } + self.scan_statements(&case.body)?; + } + self.in_conditional_block = saved_in_conditional_block; + } + Stmt::Raise(StmtRaise { exc, cause, .. }) => { + if let Some(expression) = exc { + self.scan_expression(expression, ExpressionContext::Load)?; + } + if let Some(expression) = cause { + self.scan_expression(expression, ExpressionContext::Load)?; + } + } + Stmt::TypeAlias(StmtTypeAlias { + name, + value, + type_params, + .. + }) => { + let was_in_type_alias = self.in_type_alias; + self.in_type_alias = true; + // Check before entering any sub-scopes + let in_class = self + .tables + .last() + .is_some_and(|t| t.typ == CompilerScope::Class); + let is_generic = type_params.is_some(); + if let Some(type_params) = type_params { + self.enter_type_param_block( + "TypeAlias", + self.line_index_start(type_params.range), + false, + )?; + self.scan_type_params(type_params)?; + } + // Value scope for lazy evaluation + self.enter_scope( + "TypeAlias", + CompilerScope::TypeParams, + self.line_index_start(value.range()), + ); + // Evaluator takes a format parameter + self.register_name("format", SymbolUsage::Parameter, TextRange::default())?; + if in_class { + if let Some(table) = self.tables.last_mut() { + table.can_see_class_scope = true; + } + self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + } + self.scan_expression(value, ExpressionContext::Load)?; + self.leave_scope(); + if is_generic { + self.leave_scope(); + } + self.in_type_alias = was_in_type_alias; + self.scan_expression(name, ExpressionContext::Store)?; + } + Stmt::IpyEscapeCommand(_) => todo!(), + } + Ok(()) + } + + fn scan_decorators( + &mut self, + decorators: &[ast::Decorator], + context: ExpressionContext, + ) -> SymbolTableResult { + for decorator in decorators { + self.scan_expression(&decorator.expression, context)?; + } + Ok(()) + } + + fn scan_expressions( + &mut self, + expressions: &[ast::Expr], + context: ExpressionContext, + ) -> SymbolTableResult { + for expression in expressions { + self.scan_expression(expression, context)?; + } + Ok(()) + } + + fn scan_expression( + &mut self, + expression: &ast::Expr, + context: ExpressionContext, + ) -> SymbolTableResult { + use ast::*; + + // Check for expressions not allowed in certain contexts + // (type parameters, annotations, type aliases, TypeVar bounds/defaults) + if let Some(keyword) = match expression { + Expr::Yield(_) | Expr::YieldFrom(_) => Some("yield"), + Expr::Await(_) => Some("await"), + Expr::Named(_) => Some("named"), + _ => None, + } { + // Determine the context name for the error message + // scope_info takes precedence (e.g., "a TypeVar bound") + let context_name = if let Some(scope_info) = self.scope_info { + Some(scope_info) + } else if let Some(table) = self.tables.last() + && table.typ == CompilerScope::TypeParams + { + Some("a type parameter") + } else if self.in_annotation { + Some("an annotation") + } else if self.in_type_alias { + Some("a type alias") + } else { + None + }; + + if let Some(context_name) = context_name { + return Err(SymbolTableError { + error: format!("{keyword} expression cannot be used within {context_name}"), + location: Some( + self.source_file + .to_source_code() + .source_location(expression.range().start(), PositionEncoding::Utf8), + ), + }); + } + } + + match expression { + Expr::BinOp(ExprBinOp { + left, + right, + range: _, + .. + }) => { + self.scan_expression(left, context)?; + self.scan_expression(right, context)?; + } + Expr::BoolOp(ExprBoolOp { + values, range: _, .. + }) => { + self.scan_expressions(values, context)?; + } + Expr::Compare(ExprCompare { + left, + comparators, + range: _, + .. + }) => { + self.scan_expression(left, context)?; + self.scan_expressions(comparators, context)?; + } + Expr::Subscript(ExprSubscript { + value, + slice, + range: _, + .. + }) => { + self.scan_expression(value, ExpressionContext::Load)?; + self.scan_expression(slice, ExpressionContext::Load)?; + } + Expr::Attribute(ExprAttribute { + value, attr, range, .. + }) => { + self.check_name(attr.as_str(), context, *range)?; + self.scan_expression(value, ExpressionContext::Load)?; + } + Expr::Dict(ExprDict { + items, + node_index: _, + range: _, + }) => { + for item in items { + if let Some(key) = &item.key { + self.scan_expression(key, context)?; + } + self.scan_expression(&item.value, context)?; + } + } + Expr::Await(ExprAwait { + value, + node_index: _, + range: _, + }) => { + self.scan_expression(value, context)?; + } + Expr::Yield(ExprYield { + value, + node_index: _, + range: _, + }) => { + if let Some(expression) = value { + self.scan_expression(expression, context)?; + } + } + Expr::YieldFrom(ExprYieldFrom { + value, + node_index: _, + range: _, + }) => { + self.scan_expression(value, context)?; + } + Expr::UnaryOp(ExprUnaryOp { + operand, range: _, .. + }) => { + self.scan_expression(operand, context)?; + } + Expr::Starred(ExprStarred { + value, range: _, .. + }) => { + self.scan_expression(value, context)?; + } + Expr::Tuple(ExprTuple { elts, range: _, .. }) + | Expr::Set(ExprSet { elts, range: _, .. }) + | Expr::List(ExprList { elts, range: _, .. }) => { + self.scan_expressions(elts, context)?; + } + Expr::Slice(ExprSlice { + lower, + upper, + step, + node_index: _, + range: _, + }) => { + if let Some(lower) = lower { + self.scan_expression(lower, context)?; + } + if let Some(upper) = upper { + self.scan_expression(upper, context)?; + } + if let Some(step) = step { + self.scan_expression(step, context)?; + } + } + Expr::Generator(ExprGenerator { + elt, + generators, + range, + .. + }) => { + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } + // Generator expression - is_generator = true + self.scan_comprehension("<genexpr>", elt, None, generators, *range, true)?; + self.in_iter_def_exp = was_in_iter_def_exp; + } + Expr::ListComp(ExprListComp { + elt, + generators, + range, + node_index: _, + }) => { + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } + // List comprehension - is_generator = false (can be inlined) + self.scan_comprehension("<listcomp>", elt, None, generators, *range, false)?; + self.in_iter_def_exp = was_in_iter_def_exp; + } + Expr::SetComp(ExprSetComp { + elt, + generators, + range, + node_index: _, + }) => { + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } + // Set comprehension - is_generator = false (can be inlined) + self.scan_comprehension("<setcomp>", elt, None, generators, *range, false)?; + self.in_iter_def_exp = was_in_iter_def_exp; + } + Expr::DictComp(ExprDictComp { + key, + value, + generators, + range, + node_index: _, + }) => { + let was_in_iter_def_exp = self.in_iter_def_exp; + if context == ExpressionContext::IterDefinitionExp { + self.in_iter_def_exp = true; + } + // Dict comprehension - is_generator = false (can be inlined) + self.scan_comprehension("<dictcomp>", key, Some(value), generators, *range, false)?; + self.in_iter_def_exp = was_in_iter_def_exp; + } + Expr::Call(ExprCall { + func, + arguments, + node_index: _, + range: _, + }) => { + match context { + ExpressionContext::IterDefinitionExp => { + self.scan_expression(func, ExpressionContext::IterDefinitionExp)?; + } + _ => { + self.scan_expression(func, ExpressionContext::Load)?; + } + } + + self.scan_expressions(&arguments.args, ExpressionContext::Load)?; + for keyword in &arguments.keywords { + if let Some(arg) = &keyword.arg { + self.check_name(arg.as_str(), ExpressionContext::Store, keyword.range)?; + } + self.scan_expression(&keyword.value, ExpressionContext::Load)?; + } + } + Expr::Name(ExprName { id, range, .. }) => { + let id = id.as_str(); + + self.check_name(id, context, *range)?; + + // Determine the contextual usage of this symbol: + match context { + ExpressionContext::Delete => { + self.register_name(id, SymbolUsage::Assigned, *range)?; + self.register_name(id, SymbolUsage::Used, *range)?; + } + ExpressionContext::Load | ExpressionContext::IterDefinitionExp => { + self.register_name(id, SymbolUsage::Used, *range)?; + } + ExpressionContext::Store => { + self.register_name(id, SymbolUsage::Assigned, *range)?; + } + ExpressionContext::Iter => { + self.register_name(id, SymbolUsage::Iter, *range)?; + } + } + // Interesting stuff about the __class__ variable: + // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object + if context == ExpressionContext::Load + && matches!( + self.tables.last().unwrap().typ, + CompilerScope::Function | CompilerScope::AsyncFunction + ) + && id == "super" + { + self.register_name("__class__", SymbolUsage::Used, *range)?; + } + } + Expr::Lambda(ExprLambda { + body, + parameters, + node_index: _, + range: _, + }) => { + if let Some(parameters) = parameters { + self.enter_scope_with_parameters( + "lambda", + parameters, + self.line_index_start(expression.range()), + false, // lambdas have no return annotation + false, // lambdas are never async + false, // don't skip defaults + )?; + } else { + self.enter_scope( + "lambda", + CompilerScope::Lambda, + self.line_index_start(expression.range()), + ); + } + match context { + ExpressionContext::IterDefinitionExp => { + self.scan_expression(body, ExpressionContext::IterDefinitionExp)?; + } + _ => { + self.scan_expression(body, ExpressionContext::Load)?; + } + } + self.leave_scope(); + } + Expr::FString(ExprFString { value, .. }) => { + for expr in value.elements().filter_map(|x| x.as_interpolation()) { + self.scan_expression(&expr.expression, ExpressionContext::Load)?; + if let Some(format_spec) = &expr.format_spec { + for element in format_spec.elements.interpolations() { + self.scan_expression(&element.expression, ExpressionContext::Load)? + } + } + } + } + Expr::TString(tstring) => { + // Scan t-string interpolation expressions (similar to f-strings) + for expr in tstring + .value + .elements() + .filter_map(|x| x.as_interpolation()) + { + self.scan_expression(&expr.expression, ExpressionContext::Load)?; + if let Some(format_spec) = &expr.format_spec { + for element in format_spec.elements.interpolations() { + self.scan_expression(&element.expression, ExpressionContext::Load)? + } + } + } + } + // Constants + Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::NumberLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + | Expr::EllipsisLiteral(_) => {} + Expr::IpyEscapeCommand(_) => todo!(), + Expr::If(ExprIf { + test, + body, + orelse, + node_index: _, + range: _, + }) => { + self.scan_expression(test, ExpressionContext::Load)?; + self.scan_expression(body, ExpressionContext::Load)?; + self.scan_expression(orelse, ExpressionContext::Load)?; + } + + Expr::Named(ExprNamed { + target, + value, + range, + node_index: _, + }) => { + // named expressions are not allowed in the definition of + // comprehension iterator definitions (including nested comprehensions) + if context == ExpressionContext::IterDefinitionExp || self.in_iter_def_exp { + return Err(SymbolTableError { + error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), + location: Some(self.source_file.to_source_code().source_location(target.range().start(), PositionEncoding::Utf8)), + }); + } + + self.scan_expression(value, ExpressionContext::Load)?; + + // special handling for assigned identifier in named expressions + // that are used in comprehensions. This required to correctly + // propagate the scope of the named assigned named and not to + // propagate inner names. + if let Expr::Name(ExprName { id, .. }) = &**target { + let id = id.as_str(); + self.check_name(id, ExpressionContext::Store, *range)?; + let table = self.tables.last().unwrap(); + if table.typ == CompilerScope::Comprehension { + self.register_name( + id, + SymbolUsage::AssignedNamedExprInComprehension, + *range, + )?; + } else { + // omit one recursion. When the handling of an store changes for + // Identifiers this needs adapted - more forward safe would be + // calling scan_expression directly. + self.register_name(id, SymbolUsage::Assigned, *range)?; + } + } else { + self.scan_expression(target, ExpressionContext::Store)?; + } + } + } + Ok(()) + } + + fn scan_comprehension( + &mut self, + scope_name: &str, + elt1: &ast::Expr, + elt2: Option<&ast::Expr>, + generators: &[ast::Comprehension], + range: TextRange, + is_generator: bool, + ) -> SymbolTableResult { + // Check for async comprehension outside async function + // (list/set/dict comprehensions only, not generator expressions) + let has_async_gen = generators.iter().any(|g| g.is_async); + if has_async_gen && !is_generator && !self.is_in_async_context() { + return Err(SymbolTableError { + error: "asynchronous comprehension outside of an asynchronous function".to_owned(), + location: Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ), + }); + } + + // Comprehensions are compiled as functions, so create a scope for them: + self.enter_scope( + scope_name, + CompilerScope::Comprehension, + self.line_index_start(range), + ); + + // PEP 709: inlined comprehensions are not yet implemented in the + // compiler (is_inlined_comprehension_context always returns false), + // so do NOT mark comp_inlined here. Setting it would cause the + // symbol-table analyzer to merge comprehension-local symbols into + // the parent scope, while the compiler still emits a separate code + // object — leading to the merged symbols being missing from the + // comprehension's own symbol table lookup. + + // Register the passed argument to the generator function as the name ".0" + self.register_name(".0", SymbolUsage::Parameter, range)?; + + self.scan_expression(elt1, ExpressionContext::Load)?; + if let Some(elt2) = elt2 { + self.scan_expression(elt2, ExpressionContext::Load)?; + } + + let mut is_first_generator = true; + for generator in generators { + // Set flag for INNER_LOOP_CONFLICT check (only for inner loops, not the first) + if !is_first_generator { + self.in_comp_inner_loop_target = true; + } + self.scan_expression(&generator.target, ExpressionContext::Iter)?; + self.in_comp_inner_loop_target = false; + + if is_first_generator { + is_first_generator = false; + } else { + self.scan_expression(&generator.iter, ExpressionContext::IterDefinitionExp)?; + } + + for if_expr in &generator.ifs { + self.scan_expression(if_expr, ExpressionContext::Load)?; + } + } + + self.leave_scope(); + + // The first iterable is passed as an argument into the created function: + assert!(!generators.is_empty()); + self.scan_expression(&generators[0].iter, ExpressionContext::IterDefinitionExp)?; + + Ok(()) + } + + /// Scan type parameter bound or default in a separate scope + // = symtable_visit_type_param_bound_or_default + fn scan_type_param_bound_or_default( + &mut self, + expr: &ast::Expr, + scope_name: &str, + scope_info: &'static str, + ) -> SymbolTableResult { + // Enter a new TypeParams scope for the bound/default expression + // This allows the expression to access outer scope symbols + let in_class = self.tables.last().is_some_and(|t| t.can_see_class_scope); + let line_number = self.line_index_start(expr.range()); + self.enter_scope(scope_name, CompilerScope::TypeParams, line_number); + // Evaluator takes a format parameter + self.register_name("format", SymbolUsage::Parameter, TextRange::default())?; + + if in_class { + if let Some(table) = self.tables.last_mut() { + table.can_see_class_scope = true; + } + self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + } + + // Set scope_info for better error messages + let old_scope_info = self.scope_info; + self.scope_info = Some(scope_info); + + // Scan the expression in this new scope + let result = self.scan_expression(expr, ExpressionContext::Load); + + // Restore scope_info and exit the scope + self.scope_info = old_scope_info; + self.leave_scope(); + + result + } + + fn scan_type_params(&mut self, type_params: &ast::TypeParams) -> SymbolTableResult { + // Check for duplicate type parameter names + let mut seen_names: IndexSet<&str> = IndexSet::default(); + // Check for non-default type parameter after default type parameter + let mut default_seen = false; + for type_param in &type_params.type_params { + let (name, range, has_default) = match type_param { + ast::TypeParam::TypeVar(tv) => (tv.name.as_str(), tv.range, tv.default.is_some()), + ast::TypeParam::ParamSpec(ps) => (ps.name.as_str(), ps.range, ps.default.is_some()), + ast::TypeParam::TypeVarTuple(tvt) => { + (tvt.name.as_str(), tvt.range, tvt.default.is_some()) + } + }; + if !seen_names.insert(name) { + return Err(SymbolTableError { + error: format!("duplicate type parameter '{}'", name), + location: Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ), + }); + } + if has_default { + default_seen = true; + } else if default_seen { + return Err(SymbolTableError { + error: format!( + "non-default type parameter '{}' follows default type parameter", + name + ), + location: Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ), + }); + } + } + + // Register .type_params as a type parameter (automatically becomes cell variable) + self.register_name(".type_params", SymbolUsage::TypeParam, type_params.range)?; + + // First register all type parameters + for type_param in &type_params.type_params { + match type_param { + ast::TypeParam::TypeVar(ast::TypeParamTypeVar { + name, + bound, + range: type_var_range, + default, + node_index: _, + }) => { + self.register_name(name.as_str(), SymbolUsage::TypeParam, *type_var_range)?; + + // Process bound in a separate scope + if let Some(binding) = bound { + let (scope_name, scope_info) = if binding.is_tuple_expr() { + ( + format!("<TypeVar constraint of {name}>"), + "a TypeVar constraint", + ) + } else { + (format!("<TypeVar bound of {name}>"), "a TypeVar bound") + }; + self.scan_type_param_bound_or_default(binding, &scope_name, scope_info)?; + } + + // Process default in a separate scope + if let Some(default_value) = default { + let scope_name = format!("<TypeVar default of {name}>"); + self.scan_type_param_bound_or_default( + default_value, + &scope_name, + "a TypeVar default", + )?; + } + } + ast::TypeParam::ParamSpec(ast::TypeParamParamSpec { + name, + range: param_spec_range, + default, + node_index: _, + }) => { + self.register_name(name, SymbolUsage::TypeParam, *param_spec_range)?; + + // Process default in a separate scope + if let Some(default_value) = default { + let scope_name = format!("<ParamSpec default of {name}>"); + self.scan_type_param_bound_or_default( + default_value, + &scope_name, + "a ParamSpec default", + )?; + } + } + ast::TypeParam::TypeVarTuple(ast::TypeParamTypeVarTuple { + name, + range: type_var_tuple_range, + default, + node_index: _, + }) => { + self.register_name(name, SymbolUsage::TypeParam, *type_var_tuple_range)?; + + // Process default in a separate scope + if let Some(default_value) = default { + let scope_name = format!("<TypeVarTuple default of {name}>"); + self.scan_type_param_bound_or_default( + default_value, + &scope_name, + "a TypeVarTuple default", + )?; + } + } + } + } + Ok(()) + } + + fn scan_patterns(&mut self, patterns: &[ast::Pattern]) -> SymbolTableResult { + for pattern in patterns { + self.scan_pattern(pattern)?; + } + Ok(()) + } + + fn scan_pattern(&mut self, pattern: &ast::Pattern) -> SymbolTableResult { + use ast::Pattern::*; + match pattern { + MatchValue(ast::PatternMatchValue { value, .. }) => { + self.scan_expression(value, ExpressionContext::Load)? + } + MatchSingleton(_) => {} + MatchSequence(ast::PatternMatchSequence { patterns, .. }) => { + self.scan_patterns(patterns)? + } + MatchMapping(ast::PatternMatchMapping { + keys, + patterns, + rest, + .. + }) => { + self.scan_expressions(keys, ExpressionContext::Load)?; + self.scan_patterns(patterns)?; + if let Some(rest) = rest { + self.register_ident(rest, SymbolUsage::Assigned)?; + } + } + MatchClass(ast::PatternMatchClass { cls, arguments, .. }) => { + self.scan_expression(cls, ExpressionContext::Load)?; + self.scan_patterns(&arguments.patterns)?; + for kw in &arguments.keywords { + self.scan_pattern(&kw.pattern)?; + } + } + MatchStar(ast::PatternMatchStar { name, .. }) => { + if let Some(name) = name { + self.register_ident(name, SymbolUsage::Assigned)?; + } + } + MatchAs(ast::PatternMatchAs { pattern, name, .. }) => { + if let Some(pattern) = pattern { + self.scan_pattern(pattern)?; + } + if let Some(name) = name { + self.register_ident(name, SymbolUsage::Assigned)?; + } + } + MatchOr(ast::PatternMatchOr { patterns, .. }) => self.scan_patterns(patterns)?, + } + Ok(()) + } + + /// Scan default parameter values (evaluated in the enclosing scope) + fn scan_parameter_defaults(&mut self, parameters: &ast::Parameters) -> SymbolTableResult { + for default in parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .filter_map(|arg| arg.default.as_ref()) + { + self.scan_expression(default, ExpressionContext::Load)?; + } + Ok(()) + } + + fn enter_scope_with_parameters( + &mut self, + name: &str, + parameters: &ast::Parameters, + line_number: u32, + has_return_annotation: bool, + is_async: bool, + skip_defaults: bool, + ) -> SymbolTableResult { + // Evaluate eventual default parameters (unless already scanned before type_param_block): + if !skip_defaults { + self.scan_parameter_defaults(parameters)?; + } + + // Annotations are scanned in outer scope: + for annotation in parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .filter_map(|arg| arg.parameter.annotation.as_ref()) + { + self.scan_annotation(annotation)?; + } + if let Some(annotation) = parameters + .vararg + .as_ref() + .and_then(|arg| arg.annotation.as_ref()) + { + self.scan_annotation(annotation)?; + } + if let Some(annotation) = parameters + .kwarg + .as_ref() + .and_then(|arg| arg.annotation.as_ref()) + { + self.scan_annotation(annotation)?; + } + + // Check if this function has any annotations (parameter or return) + let has_param_annotations = parameters + .posonlyargs + .iter() + .chain(parameters.args.iter()) + .chain(parameters.kwonlyargs.iter()) + .any(|p| p.parameter.annotation.is_some()) + || parameters + .vararg + .as_ref() + .is_some_and(|p| p.annotation.is_some()) + || parameters + .kwarg + .as_ref() + .is_some_and(|p| p.annotation.is_some()); + + let has_any_annotations = has_param_annotations || has_return_annotation; + + // Take annotation_block if this function has any annotations. + // When in class scope, the class's annotation_block was saved before scanning + // function annotations, so the current annotation_block belongs to this function. + let annotation_block = if has_any_annotations { + self.tables.last_mut().unwrap().annotation_block.take() + } else { + None + }; + + let scope_type = if is_async { + CompilerScope::AsyncFunction + } else { + CompilerScope::Function + }; + self.enter_scope(name, scope_type, line_number); + + // Move annotation_block to function scope only if we have one + if let Some(block) = annotation_block { + self.tables.last_mut().unwrap().annotation_block = Some(block); + } + + // Fill scope with parameter names: + self.scan_parameters(&parameters.posonlyargs)?; + self.scan_parameters(&parameters.args)?; + self.scan_parameters(&parameters.kwonlyargs)?; + if let Some(name) = &parameters.vararg { + self.scan_parameter(name)?; + } + if let Some(name) = &parameters.kwarg { + self.scan_parameter(name)?; + } + Ok(()) + } + + fn register_ident(&mut self, ident: &ast::Identifier, role: SymbolUsage) -> SymbolTableResult { + self.register_name(ident.as_str(), role, ident.range) + } + + fn check_name( + &self, + name: &str, + context: ExpressionContext, + range: TextRange, + ) -> SymbolTableResult { + if name == "__debug__" { + let location = Some( + self.source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8), + ); + match context { + ExpressionContext::Store | ExpressionContext::Iter => { + return Err(SymbolTableError { + error: "cannot assign to __debug__".to_owned(), + location, + }); + } + ExpressionContext::Delete => { + return Err(SymbolTableError { + error: "cannot delete __debug__".to_owned(), + location, + }); + } + _ => {} + } + } + Ok(()) + } + + fn register_name( + &mut self, + name: &str, + role: SymbolUsage, + range: TextRange, + ) -> SymbolTableResult { + let location = self + .source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8); + let location = Some(location); + + // Note: __debug__ checks are handled by check_name function, so no check needed here. + + let scope_depth = self.tables.len(); + let table = self.tables.last_mut().unwrap(); + + // Add type param names to mangled_names set for selective mangling + if matches!(role, SymbolUsage::TypeParam) + && let Some(ref mut set) = table.mangled_names + { + set.insert(name.to_owned()); + } + + let name = maybe_mangle_name( + self.class_name.as_deref(), + table.mangled_names.as_ref(), + name, + ); + // Some checks for the symbol that present on this scope level: + let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) { + let flags = &symbol.flags; + + // INNER_LOOP_CONFLICT: comprehension inner loop cannot rebind + // a variable that was used as a named expression target + // Example: [i for i in range(5) if (j := 0) for j in range(5)] + // Here 'j' is used in named expr first, then as inner loop iter target + if self.in_comp_inner_loop_target + && flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) + { + return Err(SymbolTableError { + error: format!( + "comprehension inner loop cannot rebind assignment expression target '{}'", + name + ), + location, + }); + } + + // Role already set.. + match role { + SymbolUsage::Global if !symbol.is_global() => { + if flags.contains(SymbolFlags::PARAMETER) { + return Err(SymbolTableError { + error: format!("name '{name}' is parameter and global"), + location, + }); + } + if flags.contains(SymbolFlags::REFERENCED) { + return Err(SymbolTableError { + error: format!("name '{name}' is used prior to global declaration"), + location, + }); + } + if flags.contains(SymbolFlags::ANNOTATED) { + return Err(SymbolTableError { + error: format!("annotated name '{name}' can't be global"), + location, + }); + } + if flags.contains(SymbolFlags::ASSIGNED) { + return Err(SymbolTableError { + error: format!( + "name '{name}' is assigned to before global declaration" + ), + location, + }); + } + } + SymbolUsage::Nonlocal => { + if flags.contains(SymbolFlags::PARAMETER) { + return Err(SymbolTableError { + error: format!("name '{name}' is parameter and nonlocal"), + location, + }); + } + if flags.contains(SymbolFlags::REFERENCED) { + return Err(SymbolTableError { + error: format!("name '{name}' is used prior to nonlocal declaration"), + location, + }); + } + if flags.contains(SymbolFlags::ANNOTATED) { + return Err(SymbolTableError { + error: format!("annotated name '{name}' can't be nonlocal"), + location, + }); + } + if flags.contains(SymbolFlags::ASSIGNED) { + return Err(SymbolTableError { + error: format!( + "name '{name}' is assigned to before nonlocal declaration" + ), + location, + }); + } + } + _ => { + // Ok? + } + } + symbol + } else { + // The symbol does not present on this scope level. + // Some checks to insert new symbol into symbol table: + match role { + SymbolUsage::Nonlocal if scope_depth < 2 => { + return Err(SymbolTableError { + error: format!("cannot define nonlocal '{name}' at top level."), + location, + }); + } + _ => { + // Ok! + } + } + // Insert symbol when required: + let symbol = Symbol::new(name.as_ref()); + table.symbols.entry(name.into_owned()).or_insert(symbol) + }; + + // Set proper scope and flags on symbol: + let flags = &mut symbol.flags; + match role { + SymbolUsage::Nonlocal => { + symbol.scope = SymbolScope::Free; + flags.insert(SymbolFlags::NONLOCAL); + } + SymbolUsage::Imported => { + flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::IMPORTED); + } + SymbolUsage::Parameter => { + flags.insert(SymbolFlags::PARAMETER); + // Parameters are always added to varnames first + let name_str = symbol.name.clone(); + if !self.current_varnames.contains(&name_str) { + self.current_varnames.push(name_str); + } + } + SymbolUsage::AnnotationParameter => { + flags.insert(SymbolFlags::PARAMETER | SymbolFlags::ANNOTATED); + // Annotated parameters are also added to varnames + let name_str = symbol.name.clone(); + if !self.current_varnames.contains(&name_str) { + self.current_varnames.push(name_str); + } + } + SymbolUsage::AnnotationAssigned => { + flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ANNOTATED); + } + SymbolUsage::Assigned => { + flags.insert(SymbolFlags::ASSIGNED); + // Local variables (assigned) are added to varnames if they are local scope + // and not already in varnames + if symbol.scope == SymbolScope::Local { + let name_str = symbol.name.clone(); + if !self.current_varnames.contains(&name_str) { + self.current_varnames.push(name_str); + } + } + } + SymbolUsage::AssignedNamedExprInComprehension => { + flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ASSIGNED_IN_COMPREHENSION); + // Named expressions in comprehensions might also be locals + if symbol.scope == SymbolScope::Local { + let name_str = symbol.name.clone(); + if !self.current_varnames.contains(&name_str) { + self.current_varnames.push(name_str); + } + } + } + SymbolUsage::Global => { + symbol.scope = SymbolScope::GlobalExplicit; + flags.insert(SymbolFlags::GLOBAL); + } + SymbolUsage::Used => { + flags.insert(SymbolFlags::REFERENCED); + } + SymbolUsage::Iter => { + flags.insert(SymbolFlags::ITER); + } + SymbolUsage::TypeParam => { + flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::TYPE_PARAM); + } + } + + // and even more checking + // it is not allowed to assign to iterator variables (by named expressions) + if flags.contains(SymbolFlags::ITER) + && flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) + { + return Err(SymbolTableError { + error: format!( + "assignment expression cannot rebind comprehension iteration variable '{}'", + symbol.name + ), + location, + }); + } + Ok(()) + } +} + +pub(crate) fn mangle_name<'a>(class_name: Option<&str>, name: &'a str) -> Cow<'a, str> { + let class_name = match class_name { + Some(n) => n, + None => return name.into(), + }; + if !name.starts_with("__") || name.ends_with("__") || name.contains('.') { + return name.into(); + } + // Strip leading underscores from class name + let class_name = class_name.trim_start_matches('_'); + let mut ret = String::with_capacity(1 + class_name.len() + name.len()); + ret.push('_'); + ret.push_str(class_name); + ret.push_str(name); + ret.into() +} + +/// Selective mangling for type parameter scopes around generic classes. +/// If `mangled_names` is Some, only mangle names that are in the set; +/// other names are left unmangled. +pub(crate) fn maybe_mangle_name<'a>( + class_name: Option<&str>, + mangled_names: Option<&IndexSet<String>>, + name: &'a str, +) -> Cow<'a, str> { + if let Some(set) = mangled_names + && !set.contains(name) + { + return name.into(); + } + mangle_name(class_name, name) +} diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs new file mode 100644 index 00000000000..a590323cb78 --- /dev/null +++ b/crates/codegen/src/unparse.rs @@ -0,0 +1,671 @@ +use alloc::fmt; +use core::fmt::Display as _; +use ruff_python_ast as ast; +use ruff_text_size::Ranged; +use rustpython_compiler_core::SourceFile; +use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; + +mod precedence { + macro_rules! precedence { + ($($op:ident,)*) => { + precedence!(@0, $($op,)*); + }; + (@$i:expr, $op1:ident, $($op:ident,)*) => { + pub const $op1: u8 = $i; + precedence!(@$i + 1, $($op,)*); + }; + (@$i:expr,) => {}; + } + precedence!( + TUPLE, TEST, OR, AND, NOT, CMP, // "EXPR" = + BOR, BXOR, BAND, SHIFT, ARITH, TERM, FACTOR, POWER, AWAIT, ATOM, + ); + pub const EXPR: u8 = BOR; +} + +struct Unparser<'a, 'b, 'c> { + f: &'b mut fmt::Formatter<'a>, + source: &'c SourceFile, +} + +impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { + const fn new(f: &'b mut fmt::Formatter<'a>, source: &'c SourceFile) -> Self { + Self { f, source } + } + + fn p(&mut self, s: &str) -> fmt::Result { + self.f.write_str(s) + } + + fn p_id(&mut self, s: &ast::Identifier) -> fmt::Result { + self.f.write_str(s.as_str()) + } + + fn p_if(&mut self, cond: bool, s: &str) -> fmt::Result { + if cond { + self.f.write_str(s)?; + } + Ok(()) + } + + fn p_delim(&mut self, first: &mut bool, s: &str) -> fmt::Result { + self.p_if(!core::mem::take(first), s) + } + + fn write_fmt(&mut self, f: fmt::Arguments<'_>) -> fmt::Result { + self.f.write_fmt(f) + } + + fn unparse_expr(&mut self, ast: &ast::Expr, level: u8) -> fmt::Result { + macro_rules! op_prec { + ($op_ty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => { + match $x { + $(<$enu>::$var => (op_prec!(@space $op_ty, $op), precedence::$prec),)* + } + }; + (@space bin, $op:literal) => { + concat!(" ", $op, " ") + }; + (@space un, $op:literal) => { + $op + }; + } + macro_rules! group_if { + ($lvl:expr, $body:block) => {{ + let group = level > $lvl; + self.p_if(group, "(")?; + let ret = $body; + self.p_if(group, ")")?; + ret + }}; + } + match &ast { + ast::Expr::BoolOp(ast::ExprBoolOp { + op, + values, + node_index: _, + range: _range, + }) => { + let (op, prec) = op_prec!(bin, op, ast::BoolOp, And("and", AND), Or("or", OR)); + group_if!(prec, { + let mut first = true; + for val in values { + self.p_delim(&mut first, op)?; + self.unparse_expr(val, prec + 1)?; + } + }) + } + ast::Expr::Named(ast::ExprNamed { + target, + value, + node_index: _, + range: _range, + }) => { + group_if!(precedence::TUPLE, { + self.unparse_expr(target, precedence::ATOM)?; + self.p(" := ")?; + self.unparse_expr(value, precedence::ATOM)?; + }) + } + ast::Expr::BinOp(ast::ExprBinOp { + left, + op, + right, + node_index: _, + range: _range, + }) => { + let right_associative = matches!(op, ast::Operator::Pow); + let (op, prec) = op_prec!( + bin, + op, + ast::Operator, + Add("+", ARITH), + Sub("-", ARITH), + Mult("*", TERM), + MatMult("@", TERM), + Div("/", TERM), + Mod("%", TERM), + Pow("**", POWER), + LShift("<<", SHIFT), + RShift(">>", SHIFT), + BitOr("|", BOR), + BitXor("^", BXOR), + BitAnd("&", BAND), + FloorDiv("//", TERM), + ); + group_if!(prec, { + self.unparse_expr(left, prec + right_associative as u8)?; + self.p(op)?; + self.unparse_expr(right, prec + !right_associative as u8)?; + }) + } + ast::Expr::UnaryOp(ast::ExprUnaryOp { + op, + operand, + node_index: _, + range: _range, + }) => { + let (op, prec) = op_prec!( + un, + op, + ast::UnaryOp, + Invert("~", FACTOR), + Not("not ", NOT), + UAdd("+", FACTOR), + USub("-", FACTOR) + ); + group_if!(prec, { + self.p(op)?; + self.unparse_expr(operand, prec)?; + }) + } + ast::Expr::Lambda(ast::ExprLambda { + parameters, + body, + node_index: _, + range: _range, + }) => { + group_if!(precedence::TEST, { + if let Some(parameters) = parameters { + self.p("lambda ")?; + self.unparse_arguments(parameters)?; + } else { + self.p("lambda")?; + } + write!(self, ": {}", UnparseExpr::new(body, self.source))?; + }) + } + ast::Expr::If(ast::ExprIf { + test, + body, + orelse, + node_index: _, + range: _range, + }) => { + group_if!(precedence::TEST, { + self.unparse_expr(body, precedence::TEST + 1)?; + self.p(" if ")?; + self.unparse_expr(test, precedence::TEST + 1)?; + self.p(" else ")?; + self.unparse_expr(orelse, precedence::TEST)?; + }) + } + ast::Expr::Dict(ast::ExprDict { + items, + node_index: _, + range: _range, + }) => { + self.p("{")?; + let mut first = true; + for item in items { + self.p_delim(&mut first, ", ")?; + if let Some(k) = &item.key { + write!(self, "{}: ", UnparseExpr::new(k, self.source))?; + } else { + self.p("**")?; + } + self.unparse_expr(&item.value, level)?; + } + self.p("}")?; + } + ast::Expr::Set(ast::ExprSet { + elts, + node_index: _, + range: _range, + }) => { + self.p("{")?; + let mut first = true; + for v in elts { + self.p_delim(&mut first, ", ")?; + self.unparse_expr(v, precedence::TEST)?; + } + self.p("}")?; + } + ast::Expr::ListComp(ast::ExprListComp { + elt, + generators, + node_index: _, + range: _range, + }) => { + self.p("[")?; + self.unparse_expr(elt, precedence::TEST)?; + self.unparse_comp(generators)?; + self.p("]")?; + } + ast::Expr::SetComp(ast::ExprSetComp { + elt, + generators, + node_index: _, + range: _range, + }) => { + self.p("{")?; + self.unparse_expr(elt, precedence::TEST)?; + self.unparse_comp(generators)?; + self.p("}")?; + } + ast::Expr::DictComp(ast::ExprDictComp { + key, + value, + generators, + node_index: _, + range: _range, + }) => { + self.p("{")?; + self.unparse_expr(key, precedence::TEST)?; + self.p(": ")?; + self.unparse_expr(value, precedence::TEST)?; + self.unparse_comp(generators)?; + self.p("}")?; + } + ast::Expr::Generator(ast::ExprGenerator { + parenthesized: _, + elt, + generators, + node_index: _, + range: _range, + }) => { + self.p("(")?; + self.unparse_expr(elt, precedence::TEST)?; + self.unparse_comp(generators)?; + self.p(")")?; + } + ast::Expr::Await(ast::ExprAwait { + value, + node_index: _, + range: _range, + }) => { + group_if!(precedence::AWAIT, { + self.p("await ")?; + self.unparse_expr(value, precedence::ATOM)?; + }) + } + ast::Expr::Yield(ast::ExprYield { + value, + node_index: _, + range: _range, + }) => { + if let Some(value) = value { + write!(self, "(yield {})", UnparseExpr::new(value, self.source))?; + } else { + self.p("(yield)")?; + } + } + ast::Expr::YieldFrom(ast::ExprYieldFrom { + value, + node_index: _, + range: _range, + }) => { + write!( + self, + "(yield from {})", + UnparseExpr::new(value, self.source) + )?; + } + ast::Expr::Compare(ast::ExprCompare { + left, + ops, + comparators, + node_index: _, + range: _range, + }) => { + group_if!(precedence::CMP, { + let new_lvl = precedence::CMP + 1; + self.unparse_expr(left, new_lvl)?; + for (op, cmp) in ops.iter().zip(comparators) { + self.p(" ")?; + self.p(op.as_str())?; + self.p(" ")?; + self.unparse_expr(cmp, new_lvl)?; + } + }) + } + ast::Expr::Call(ast::ExprCall { + func, + arguments: ast::Arguments { args, keywords, .. }, + node_index: _, + range: _range, + }) => { + self.unparse_expr(func, precedence::ATOM)?; + self.p("(")?; + if let ( + [ + ast::Expr::Generator(ast::ExprGenerator { + elt, + generators, + node_index: _, + range: _range, + .. + }), + ], + [], + ) = (&**args, &**keywords) + { + // make sure a single genexpr doesn't get double parens + self.unparse_expr(elt, precedence::TEST)?; + self.unparse_comp(generators)?; + } else { + let mut first = true; + for arg in args { + self.p_delim(&mut first, ", ")?; + self.unparse_expr(arg, precedence::TEST)?; + } + for kw in keywords { + self.p_delim(&mut first, ", ")?; + if let Some(arg) = &kw.arg { + self.p_id(arg)?; + self.p("=")?; + } else { + self.p("**")?; + } + self.unparse_expr(&kw.value, precedence::TEST)?; + } + } + self.p(")")?; + } + ast::Expr::FString(ast::ExprFString { value, .. }) => self.unparse_fstring(value)?, + ast::Expr::TString(ast::ExprTString { value, .. }) => self.unparse_tstring(value)?, + ast::Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) => { + if value.is_unicode() { + self.p("u")? + } + UnicodeEscape::new_repr(value.to_str().as_ref()) + .str_repr() + .fmt(self.f)? + } + ast::Expr::BytesLiteral(ast::ExprBytesLiteral { value, .. }) => { + AsciiEscape::new_repr(&value.bytes().collect::<Vec<_>>()) + .bytes_repr() + .fmt(self.f)? + } + ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value, .. }) => { + #[allow(clippy::correctness, clippy::assertions_on_constants)] + const { + assert!(f64::MAX_10_EXP == 308) + }; + + let inf_str = "1e309"; + match value { + ast::Number::Int(int) => int.fmt(self.f)?, + &ast::Number::Float(fp) => { + if fp.is_infinite() { + self.p(inf_str)? + } else { + self.p(&rustpython_literal::float::to_string(fp))? + } + } + &ast::Number::Complex { real, imag } => self + .p(&rustpython_literal::complex::to_string(real, imag) + .replace("inf", inf_str))?, + } + } + ast::Expr::BooleanLiteral(ast::ExprBooleanLiteral { value, .. }) => { + self.p(if *value { "True" } else { "False" })? + } + ast::Expr::NoneLiteral(ast::ExprNoneLiteral { .. }) => self.p("None")?, + ast::Expr::EllipsisLiteral(ast::ExprEllipsisLiteral { .. }) => self.p("...")?, + ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) => { + self.unparse_expr(value, precedence::ATOM)?; + let period = if let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { + value: ast::Number::Int(_), + .. + }) = value.as_ref() + { + " ." + } else { + "." + }; + self.p(period)?; + self.p_id(attr)?; + } + ast::Expr::Subscript(ast::ExprSubscript { value, slice, .. }) => { + self.unparse_expr(value, precedence::ATOM)?; + let lvl = precedence::TUPLE; + self.p("[")?; + self.unparse_expr(slice, lvl)?; + self.p("]")?; + } + ast::Expr::Starred(ast::ExprStarred { value, .. }) => { + self.p("*")?; + self.unparse_expr(value, precedence::EXPR)?; + } + ast::Expr::Name(ast::ExprName { id, .. }) => self.p(id.as_str())?, + ast::Expr::List(ast::ExprList { elts, .. }) => { + self.p("[")?; + let mut first = true; + for elt in elts { + self.p_delim(&mut first, ", ")?; + self.unparse_expr(elt, precedence::TEST)?; + } + self.p("]")?; + } + ast::Expr::Tuple(ast::ExprTuple { elts, .. }) => { + if elts.is_empty() { + self.p("()")?; + } else { + group_if!(precedence::TUPLE, { + let mut first = true; + for elt in elts { + self.p_delim(&mut first, ", ")?; + self.unparse_expr(elt, precedence::TEST)?; + } + self.p_if(elts.len() == 1, ",")?; + }) + } + } + ast::Expr::Slice(ast::ExprSlice { + lower, + upper, + step, + node_index: _, + range: _range, + }) => { + if let Some(lower) = lower { + self.unparse_expr(lower, precedence::TEST)?; + } + self.p(":")?; + if let Some(upper) = upper { + self.unparse_expr(upper, precedence::TEST)?; + } + if let Some(step) = step { + self.p(":")?; + self.unparse_expr(step, precedence::TEST)?; + } + } + ast::Expr::IpyEscapeCommand(_) => {} + } + Ok(()) + } + + fn unparse_arguments(&mut self, args: &ast::Parameters) -> fmt::Result { + let mut first = true; + for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() { + self.p_delim(&mut first, ", ")?; + self.unparse_function_arg(arg)?; + self.p_if(i + 1 == args.posonlyargs.len(), ", /")?; + } + if args.vararg.is_some() || !args.kwonlyargs.is_empty() { + self.p_delim(&mut first, ", ")?; + self.p("*")?; + } + if let Some(vararg) = &args.vararg { + self.unparse_arg(vararg)?; + } + for kwarg in &args.kwonlyargs { + self.p_delim(&mut first, ", ")?; + self.unparse_function_arg(kwarg)?; + } + if let Some(kwarg) = &args.kwarg { + self.p_delim(&mut first, ", ")?; + self.p("**")?; + self.unparse_arg(kwarg)?; + } + Ok(()) + } + fn unparse_function_arg(&mut self, arg: &ast::ParameterWithDefault) -> fmt::Result { + self.unparse_arg(&arg.parameter)?; + if let Some(default) = &arg.default { + write!(self, "={}", UnparseExpr::new(default, self.source))?; + } + Ok(()) + } + + fn unparse_arg(&mut self, arg: &ast::Parameter) -> fmt::Result { + self.p_id(&arg.name)?; + if let Some(ann) = &arg.annotation { + write!(self, ": {}", UnparseExpr::new(ann, self.source))?; + } + Ok(()) + } + + fn unparse_comp(&mut self, generators: &[ast::Comprehension]) -> fmt::Result { + for comp in generators { + self.p(if comp.is_async { + " async for " + } else { + " for " + })?; + self.unparse_expr(&comp.target, precedence::TUPLE)?; + self.p(" in ")?; + self.unparse_expr(&comp.iter, precedence::TEST + 1)?; + for cond in &comp.ifs { + self.p(" if ")?; + self.unparse_expr(cond, precedence::TEST + 1)?; + } + } + Ok(()) + } + + fn unparse_fstring_body(&mut self, elements: &[ast::InterpolatedStringElement]) -> fmt::Result { + for elem in elements { + self.unparse_fstring_elem(elem)?; + } + Ok(()) + } + + fn unparse_formatted( + &mut self, + val: &ast::Expr, + debug_text: Option<&ast::DebugText>, + conversion: ast::ConversionFlag, + spec: Option<&ast::InterpolatedStringFormatSpec>, + ) -> fmt::Result { + let buffered = + fmt::from_fn(|f| Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1)) + .to_string(); + if let Some(ast::DebugText { leading, trailing }) = debug_text { + self.p(leading)?; + self.p(self.source.slice(val.range()))?; + self.p(trailing)?; + } + let brace = if buffered.starts_with('{') { + // put a space to avoid escaping the bracket + "{ " + } else { + // Preserve leading whitespace between '{' and the expression + let source_text = self.source.source_text(); + let start = val.range().start().to_usize(); + if start > 0 + && source_text + .as_bytes() + .get(start - 1) + .is_some_and(|b| b.is_ascii_whitespace()) + { + "{ " + } else { + "{" + } + }; + self.p(brace)?; + self.p(&buffered)?; + drop(buffered); + + if conversion != ast::ConversionFlag::None { + self.p("!")?; + let buf = &[conversion as u8]; + let c = core::str::from_utf8(buf).unwrap(); + self.p(c)?; + } + + if let Some(spec) = spec { + self.p(":")?; + self.unparse_fstring_body(&spec.elements)?; + } + + self.p("}")?; + + Ok(()) + } + + fn unparse_fstring_elem(&mut self, elem: &ast::InterpolatedStringElement) -> fmt::Result { + match elem { + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { + expression, + debug_text, + conversion, + format_spec, + .. + }) => self.unparse_formatted( + expression, + debug_text.as_ref(), + *conversion, + format_spec.as_deref(), + ), + ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { + value, + .. + }) => self.unparse_fstring_str(value), + } + } + + fn unparse_fstring_str(&mut self, s: &str) -> fmt::Result { + let s = s.replace('{', "{{").replace('}', "}}"); + self.p(&s) + } + + fn unparse_fstring(&mut self, value: &ast::FStringValue) -> fmt::Result { + self.p("f")?; + let body = fmt::from_fn(|f| { + value.iter().try_for_each(|part| match part { + ast::FStringPart::Literal(lit) => f.write_str(lit), + ast::FStringPart::FString(ast::FString { elements, .. }) => { + Unparser::new(f, self.source).unparse_fstring_body(elements) + } + }) + }) + .to_string(); + // .unparse_fstring_body(elements)); + UnicodeEscape::new_repr(body.as_str().as_ref()) + .str_repr() + .write(self.f) + } + + fn unparse_tstring(&mut self, value: &ast::TStringValue) -> fmt::Result { + self.p("t")?; + let body = fmt::from_fn(|f| { + value.iter().try_for_each(|tstring| { + Unparser::new(f, self.source).unparse_fstring_body(&tstring.elements) + }) + }) + .to_string(); + UnicodeEscape::new_repr(body.as_str().as_ref()) + .str_repr() + .write(self.f) + } +} + +pub struct UnparseExpr<'a> { + expr: &'a ast::Expr, + source: &'a SourceFile, +} + +impl<'a> UnparseExpr<'a> { + pub const fn new(expr: &'a ast::Expr, source: &'a SourceFile) -> Self { + Self { expr, source } + } +} + +impl fmt::Display for UnparseExpr<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + Unparser::new(f, self.source).unparse_expr(self.expr, precedence::TEST) + } +} diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml new file mode 100644 index 00000000000..555336f059a --- /dev/null +++ b/crates/common/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "rustpython-common" +description = "General python functions and algorithms for use in RustPython" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[features] +default = ["std"] +std = [] +threading = ["parking_lot", "std"] +wasm_js = ["getrandom/wasm_js"] + +[dependencies] +rustpython-literal = { workspace = true } +rustpython-wtf8 = { workspace = true } + +ascii = { workspace = true } +bitflags = { workspace = true } +cfg-if = { workspace = true } +getrandom = { workspace = true } +itertools = { workspace = true } +libc = { workspace = true } +malachite-bigint = { workspace = true } +malachite-q = { workspace = true } +malachite-base = { workspace = true } +num-traits = { workspace = true } +parking_lot = { workspace = true, optional = true } +unicode_names2 = { workspace = true } +radium = { workspace = true } + +lock_api = "0.4" +siphasher = "1" +num-complex.workspace = true + +[target.'cfg(unix)'.dependencies] +nix = { workspace = true } + +[target.'cfg(windows)'.dependencies] +widestring = { workspace = true } +windows-sys = { workspace = true, features = [ + "Win32_Foundation", + "Win32_Networking_WinSock", + "Win32_Storage_FileSystem", + "Win32_System_Ioctl", + "Win32_System_LibraryLoader", + "Win32_System_SystemServices", +] } + +[lints] +workspace = true diff --git a/common/src/atomic.rs b/crates/common/src/atomic.rs similarity index 99% rename from common/src/atomic.rs rename to crates/common/src/atomic.rs index afe4afb444b..ef7f41074e8 100644 --- a/common/src/atomic.rs +++ b/crates/common/src/atomic.rs @@ -65,7 +65,7 @@ impl<T> Default for OncePtr<T> { impl<T> OncePtr<T> { #[inline] pub fn new() -> Self { - OncePtr { + Self { inner: Radium::new(ptr::null_mut()), } } diff --git a/common/src/borrow.rs b/crates/common/src/borrow.rs similarity index 97% rename from common/src/borrow.rs rename to crates/common/src/borrow.rs index 0e4b9d6be30..d8389479b33 100644 --- a/common/src/borrow.rs +++ b/crates/common/src/borrow.rs @@ -2,10 +2,8 @@ use crate::lock::{ MapImmutable, PyImmutableMappedMutexGuard, PyMappedMutexGuard, PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutexGuard, PyRwLockReadGuard, PyRwLockWriteGuard, }; -use std::{ - fmt, - ops::{Deref, DerefMut}, -}; +use alloc::fmt; +use core::ops::{Deref, DerefMut}; macro_rules! impl_from { ($lt:lifetime, $gen:ident, $t:ty, $($var:ident($from:ty),)*) => { @@ -56,6 +54,7 @@ impl<'a, T: ?Sized> BorrowedValue<'a, T> { impl<T: ?Sized> Deref for BorrowedValue<'_, T> { type Target = T; + fn deref(&self) -> &T { match self { Self::Ref(r) => r, @@ -68,7 +67,7 @@ impl<T: ?Sized> Deref for BorrowedValue<'_, T> { } impl fmt::Display for BorrowedValue<'_, str> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(self.deref(), f) } } @@ -81,6 +80,7 @@ pub enum BorrowedValueMut<'a, T: ?Sized> { WriteLock(PyRwLockWriteGuard<'a, T>), MappedWriteLock(PyMappedRwLockWriteGuard<'a, T>), } + impl_from!('a, T, BorrowedValueMut<'a, T>, RefMut(&'a mut T), MuLock(PyMutexGuard<'a, T>), @@ -108,6 +108,7 @@ impl<'a, T: ?Sized> BorrowedValueMut<'a, T> { impl<T: ?Sized> Deref for BorrowedValueMut<'_, T> { type Target = T; + fn deref(&self) -> &T { match self { Self::RefMut(r) => r, diff --git a/common/src/boxvec.rs b/crates/common/src/boxvec.rs similarity index 90% rename from common/src/boxvec.rs rename to crates/common/src/boxvec.rs index ca81b751078..3260e76ca87 100644 --- a/common/src/boxvec.rs +++ b/crates/common/src/boxvec.rs @@ -1,12 +1,14 @@ +// spell-checker:disable //! An unresizable vector backed by a `Box<[T]>` -use std::{ - alloc, +#![allow(clippy::needless_lifetimes)] +use alloc::{fmt, slice}; +use core::{ borrow::{Borrow, BorrowMut}, - cmp, fmt, + cmp, mem::{self, MaybeUninit}, ops::{Bound, Deref, DerefMut, RangeBounds}, - ptr, slice, + ptr, }; pub struct BoxVec<T> { @@ -35,52 +37,34 @@ macro_rules! panic_oob { }; } -fn capacity_overflow() -> ! { - panic!("capacity overflow") -} - impl<T> BoxVec<T> { - pub fn new(n: usize) -> BoxVec<T> { - unsafe { - let layout = match alloc::Layout::array::<T>(n) { - Ok(l) => l, - Err(_) => capacity_overflow(), - }; - let ptr = if mem::size_of::<T>() == 0 { - ptr::NonNull::<MaybeUninit<T>>::dangling().as_ptr() - } else { - let ptr = alloc::alloc(layout); - if ptr.is_null() { - alloc::handle_alloc_error(layout) - } - ptr as *mut MaybeUninit<T> - }; - let ptr = ptr::slice_from_raw_parts_mut(ptr, n); - let xs = Box::from_raw(ptr); - BoxVec { xs, len: 0 } + pub fn new(n: usize) -> Self { + Self { + xs: Box::new_uninit_slice(n), + len: 0, } } #[inline] - pub fn len(&self) -> usize { + pub const fn len(&self) -> usize { self.len } #[inline] - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.len() == 0 } #[inline] - pub fn capacity(&self) -> usize { + pub const fn capacity(&self) -> usize { self.xs.len() } - pub fn is_full(&self) -> bool { + pub const fn is_full(&self) -> bool { self.len() == self.capacity() } - pub fn remaining_capacity(&self) -> usize { + pub const fn remaining_capacity(&self) -> usize { self.capacity() - self.len() } @@ -104,13 +88,16 @@ impl<T> BoxVec<T> { pub unsafe fn push_unchecked(&mut self, element: T) { let len = self.len(); debug_assert!(len < self.capacity()); - ptr::write(self.get_unchecked_ptr(len), element); - self.set_len(len + 1); + // SAFETY: len < capacity + unsafe { + ptr::write(self.get_unchecked_ptr(len), element); + self.set_len(len + 1); + } } /// Get pointer to where element at `index` would be unsafe fn get_unchecked_ptr(&mut self, index: usize) -> *mut T { - self.xs.as_mut_ptr().add(index).cast() + unsafe { self.xs.as_mut_ptr().add(index).cast() } } pub fn insert(&mut self, index: usize, element: T) { @@ -178,7 +165,7 @@ impl<T> BoxVec<T> { if index >= self.len() { None } else { - self.drain(index..index + 1).next() + self.drain(index..=index).next() } } @@ -276,7 +263,7 @@ impl<T> BoxVec<T> { /// /// **Panics** if the starting point is greater than the end point or if /// the end point is greater than the length of the vector. - pub fn drain<R>(&mut self, range: R) -> Drain<T> + pub fn drain<R>(&mut self, range: R) -> Drain<'_, T> where R: RangeBounds<usize>, { @@ -304,7 +291,7 @@ impl<T> BoxVec<T> { self.drain_range(start, end) } - fn drain_range(&mut self, start: usize, end: usize) -> Drain<T> { + fn drain_range(&mut self, start: usize, end: usize) -> Drain<'_, T> { let len = self.len(); // bounds check happens here (before length is changed!) @@ -349,6 +336,7 @@ impl<T> BoxVec<T> { impl<T> Deref for BoxVec<T> { type Target = [T]; + #[inline] fn deref(&self) -> &[T] { unsafe { slice::from_raw_parts(self.as_ptr(), self.len()) } @@ -367,6 +355,7 @@ impl<T> DerefMut for BoxVec<T> { impl<'a, T> IntoIterator for &'a BoxVec<T> { type Item = &'a T; type IntoIter = slice::Iter<'a, T>; + fn into_iter(self) -> Self::IntoIter { self.iter() } @@ -376,6 +365,7 @@ impl<'a, T> IntoIterator for &'a BoxVec<T> { impl<'a, T> IntoIterator for &'a mut BoxVec<T> { type Item = &'a mut T; type IntoIter = slice::IterMut<'a, T>; + fn into_iter(self) -> Self::IntoIter { self.iter_mut() } @@ -387,6 +377,7 @@ impl<'a, T> IntoIterator for &'a mut BoxVec<T> { impl<T> IntoIterator for BoxVec<T> { type Item = T; type IntoIter = IntoIter<T>; + fn into_iter(self) -> IntoIter<T> { IntoIter { index: 0, v: self } } @@ -452,7 +443,7 @@ impl<T> fmt::Debug for IntoIter<T> where T: fmt::Debug, { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(&self.v[self.index..]).finish() } } @@ -468,8 +459,8 @@ pub struct Drain<'a, T> { vec: ptr::NonNull<BoxVec<T>>, } -unsafe impl<'a, T: Sync> Sync for Drain<'a, T> {} -unsafe impl<'a, T: Sync> Send for Drain<'a, T> {} +unsafe impl<T: Sync> Sync for Drain<'_, T> {} +unsafe impl<T: Sync> Send for Drain<'_, T> {} impl<T> Iterator for Drain<'_, T> { type Item = T; @@ -564,7 +555,7 @@ impl<T> Extend<T> for BoxVec<T> { }; let mut iter = iter.into_iter(); loop { - if ptr == end_ptr { + if core::ptr::eq(ptr, end_ptr) { break; } if let Some(elt) = iter.next() { @@ -585,7 +576,7 @@ unsafe fn raw_ptr_add<T>(ptr: *mut T, offset: usize) -> *mut T { // Special case for ZST (ptr as usize).wrapping_add(offset) as _ } else { - ptr.add(offset) + unsafe { ptr.add(offset) } } } @@ -593,7 +584,7 @@ unsafe fn raw_ptr_write<T>(ptr: *mut T, value: T) { if mem::size_of::<T>() == 0 { /* nothing */ } else { - ptr::write(ptr, value) + unsafe { ptr::write(ptr, value) } } } @@ -602,7 +593,7 @@ where T: Clone, { fn clone(&self) -> Self { - let mut new = BoxVec::new(self.capacity()); + let mut new = Self::new(self.capacity()); new.extend(self.iter().cloned()); new } @@ -672,7 +663,7 @@ impl<T> fmt::Debug for BoxVec<T> where T: fmt::Debug, { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } } @@ -685,8 +676,8 @@ pub struct CapacityError<T = ()> { impl<T> CapacityError<T> { /// Create a new `CapacityError` from `element`. - pub fn new(element: T) -> CapacityError<T> { - CapacityError { element } + pub const fn new(element: T) -> Self { + Self { element } } /// Extract the overflowing element @@ -702,16 +693,16 @@ impl<T> CapacityError<T> { const CAPERROR: &str = "insufficient capacity"; -impl<T> std::error::Error for CapacityError<T> {} +impl<T> core::error::Error for CapacityError<T> {} impl<T> fmt::Display for CapacityError<T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{CAPERROR}") } } impl<T> fmt::Debug for CapacityError<T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "capacity error: {CAPERROR}") } } diff --git a/crates/common/src/cformat.rs b/crates/common/src/cformat.rs new file mode 100644 index 00000000000..7b9609e90ae --- /dev/null +++ b/crates/common/src/cformat.rs @@ -0,0 +1,1164 @@ +//! Implementation of Printf-Style string formatting +//! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). +use alloc::fmt; +use bitflags::bitflags; +use core::{ + cmp, + iter::{Enumerate, Peekable}, + str::FromStr, +}; +use itertools::Itertools; +use malachite_bigint::{BigInt, Sign}; +use num_traits::Signed; +use rustpython_literal::{float, format::Case}; + +use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum CFormatErrorType { + UnmatchedKeyParentheses, + MissingModuloSign, + UnsupportedFormatChar(CodePoint), + IncompleteFormat, + IntTooBig, + // Unimplemented, +} + +// also contains how many chars the parsing function consumed +pub type ParsingError = (CFormatErrorType, usize); + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CFormatError { + pub typ: CFormatErrorType, // FIXME + pub index: usize, +} + +impl fmt::Display for CFormatError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use CFormatErrorType::*; + match self.typ { + UnmatchedKeyParentheses => write!(f, "incomplete format key"), + IncompleteFormat => write!(f, "incomplete format"), + UnsupportedFormatChar(c) => write!( + f, + "unsupported format character '{}' ({:#x}) at index {}", + c, + c.to_u32(), + self.index + ), + IntTooBig => write!(f, "width/precision too big"), + _ => write!(f, "unexpected error parsing format string"), + } + } +} + +pub type CFormatConversion = super::format::FormatConversion; + +#[derive(Debug, PartialEq, Clone, Copy)] +#[repr(u8)] +pub enum CNumberType { + DecimalD = b'd', + DecimalI = b'i', + DecimalU = b'u', + Octal = b'o', + HexLower = b'x', + HexUpper = b'X', +} + +#[derive(Debug, PartialEq, Clone, Copy)] +#[repr(u8)] +pub enum CFloatType { + ExponentLower = b'e', + ExponentUpper = b'E', + PointDecimalLower = b'f', + PointDecimalUpper = b'F', + GeneralLower = b'g', + GeneralUpper = b'G', +} + +impl CFloatType { + const fn case(self) -> Case { + use CFloatType::*; + + match self { + ExponentLower | PointDecimalLower | GeneralLower => Case::Lower, + ExponentUpper | PointDecimalUpper | GeneralUpper => Case::Upper, + } + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +#[repr(u8)] +pub enum CCharacterType { + Character = b'c', +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum CFormatType { + Number(CNumberType), + Float(CFloatType), + Character(CCharacterType), + String(CFormatConversion), +} + +impl CFormatType { + pub const fn to_char(self) -> char { + match self { + Self::Number(x) => x as u8 as char, + Self::Float(x) => x as u8 as char, + Self::Character(x) => x as u8 as char, + Self::String(x) => x as u8 as char, + } + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum CFormatPrecision { + Quantity(CFormatQuantity), + Dot, +} + +impl From<CFormatQuantity> for CFormatPrecision { + fn from(quantity: CFormatQuantity) -> Self { + Self::Quantity(quantity) + } +} + +bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct CConversionFlags: u32 { + const ALTERNATE_FORM = 0b0000_0001; + const ZERO_PAD = 0b0000_0010; + const LEFT_ADJUST = 0b0000_0100; + const BLANK_SIGN = 0b0000_1000; + const SIGN_CHAR = 0b0001_0000; + } +} + +impl CConversionFlags { + #[inline] + pub const fn sign_string(&self) -> &'static str { + if self.contains(Self::SIGN_CHAR) { + "+" + } else if self.contains(Self::BLANK_SIGN) { + " " + } else { + "" + } + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum CFormatQuantity { + Amount(usize), + FromValuesTuple, +} + +pub trait FormatBuf: + Extend<Self::Char> + Default + FromIterator<Self::Char> + From<String> +{ + type Char: FormatChar; + fn chars(&self) -> impl Iterator<Item = Self::Char>; + fn len(&self) -> usize; + fn is_empty(&self) -> bool { + self.len() == 0 + } + fn concat(self, other: Self) -> Self; +} + +pub trait FormatChar: Copy + Into<CodePoint> + From<u8> { + fn to_char_lossy(self) -> char; + fn eq_char(self, c: char) -> bool; +} + +impl FormatBuf for String { + type Char = char; + + fn chars(&self) -> impl Iterator<Item = Self::Char> { + (**self).chars() + } + + fn len(&self) -> usize { + self.len() + } + + fn concat(mut self, other: Self) -> Self { + self.extend([other]); + self + } +} + +impl FormatChar for char { + fn to_char_lossy(self) -> char { + self + } + + fn eq_char(self, c: char) -> bool { + self == c + } +} + +impl FormatBuf for Wtf8Buf { + type Char = CodePoint; + + fn chars(&self) -> impl Iterator<Item = Self::Char> { + self.code_points() + } + + fn len(&self) -> usize { + (**self).len() + } + + fn concat(mut self, other: Self) -> Self { + self.extend([other]); + self + } +} + +impl FormatChar for CodePoint { + fn to_char_lossy(self) -> char { + self.to_char_lossy() + } + + fn eq_char(self, c: char) -> bool { + self == c + } +} + +impl FormatBuf for Vec<u8> { + type Char = u8; + + fn chars(&self) -> impl Iterator<Item = Self::Char> { + self.iter().copied() + } + + fn len(&self) -> usize { + self.len() + } + + fn concat(mut self, other: Self) -> Self { + self.extend(other); + self + } +} + +impl FormatChar for u8 { + fn to_char_lossy(self) -> char { + self.into() + } + + fn eq_char(self, c: char) -> bool { + char::from(self) == c + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub struct CFormatSpec { + pub flags: CConversionFlags, + pub min_field_width: Option<CFormatQuantity>, + pub precision: Option<CFormatPrecision>, + pub format_type: CFormatType, + // chars_consumed: usize, +} + +#[derive(Debug, PartialEq)] +pub struct CFormatSpecKeyed<T> { + pub mapping_key: Option<T>, + pub spec: CFormatSpec, +} + +#[cfg(test)] +impl FromStr for CFormatSpec { + type Err = ParsingError; + + fn from_str(text: &str) -> Result<Self, Self::Err> { + text.parse::<CFormatSpecKeyed<String>>() + .map(|CFormatSpecKeyed { mapping_key, spec }| { + assert!(mapping_key.is_none()); + spec + }) + } +} + +impl FromStr for CFormatSpecKeyed<String> { + type Err = ParsingError; + + fn from_str(text: &str) -> Result<Self, Self::Err> { + let mut chars = text.chars().enumerate().peekable(); + if chars.next().map(|x| x.1) != Some('%') { + return Err((CFormatErrorType::MissingModuloSign, 1)); + } + + Self::parse(&mut chars) + } +} + +pub type ParseIter<I> = Peekable<Enumerate<I>>; + +impl<T: FormatBuf> CFormatSpecKeyed<T> { + pub fn parse<I>(iter: &mut ParseIter<I>) -> Result<Self, ParsingError> + where + I: Iterator<Item = T::Char>, + { + let mapping_key = parse_spec_mapping_key(iter)?; + let flags = parse_flags(iter); + let min_field_width = parse_quantity(iter)?; + let precision = parse_precision(iter)?; + consume_length(iter); + let format_type = parse_format_type(iter)?; + + let spec = CFormatSpec { + flags, + min_field_width, + precision, + format_type, + }; + Ok(Self { mapping_key, spec }) + } +} + +impl CFormatSpec { + fn compute_fill_string<T: FormatBuf>(fill_char: T::Char, fill_chars_needed: usize) -> T { + (0..fill_chars_needed).map(|_| fill_char).collect() + } + + fn fill_string<T: FormatBuf>( + &self, + string: T, + fill_char: T::Char, + num_prefix_chars: Option<usize>, + ) -> T { + let mut num_chars = string.chars().count(); + if let Some(num_prefix_chars) = num_prefix_chars { + num_chars += num_prefix_chars; + } + let num_chars = num_chars; + + let width = match &self.min_field_width { + Some(CFormatQuantity::Amount(width)) => cmp::max(width, &num_chars), + _ => &num_chars, + }; + let fill_chars_needed = width.saturating_sub(num_chars); + let fill_string: T = Self::compute_fill_string(fill_char, fill_chars_needed); + + if !fill_string.is_empty() { + if self.flags.contains(CConversionFlags::LEFT_ADJUST) { + string.concat(fill_string) + } else { + fill_string.concat(string) + } + } else { + string + } + } + + fn fill_string_with_precision<T: FormatBuf>(&self, string: T, fill_char: T::Char) -> T { + let num_chars = string.chars().count(); + + let width = match &self.precision { + Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(width))) => { + cmp::max(width, &num_chars) + } + _ => &num_chars, + }; + let fill_chars_needed = width.saturating_sub(num_chars); + let fill_string: T = Self::compute_fill_string(fill_char, fill_chars_needed); + + if !fill_string.is_empty() { + // Don't left-adjust if precision-filling: that will always be prepending 0s to %d + // arguments, the LEFT_ADJUST flag will be used by a later call to fill_string with + // the 0-filled string as the string param. + fill_string.concat(string) + } else { + string + } + } + + fn format_string_with_precision<T: FormatBuf>( + &self, + string: T, + precision: Option<&CFormatPrecision>, + ) -> T { + // truncate if needed + let string = match precision { + Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) + if string.chars().count() > *precision => + { + string.chars().take(*precision).collect::<T>() + } + Some(CFormatPrecision::Dot) => { + // truncate to 0 + T::default() + } + _ => string, + }; + self.fill_string(string, b' '.into(), None) + } + + #[inline] + pub fn format_string<T: FormatBuf>(&self, string: T) -> T { + self.format_string_with_precision(string, self.precision.as_ref()) + } + + #[inline] + pub fn format_char<T: FormatBuf>(&self, ch: T::Char) -> T { + self.format_string_with_precision( + T::from_iter([ch]), + Some(&(CFormatQuantity::Amount(1).into())), + ) + } + + pub fn format_bytes(&self, bytes: &[u8]) -> Vec<u8> { + let bytes = if let Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) = + self.precision + { + &bytes[..cmp::min(bytes.len(), precision)] + } else { + bytes + }; + if let Some(CFormatQuantity::Amount(width)) = self.min_field_width { + let fill = cmp::max(0, width - bytes.len()); + let mut v = Vec::with_capacity(bytes.len() + fill); + if self.flags.contains(CConversionFlags::LEFT_ADJUST) { + v.extend_from_slice(bytes); + v.append(&mut vec![b' '; fill]); + } else { + v.append(&mut vec![b' '; fill]); + v.extend_from_slice(bytes); + } + v + } else { + bytes.to_vec() + } + } + + pub fn format_number(&self, num: &BigInt) -> String { + use CNumberType::*; + let CFormatType::Number(format_type) = self.format_type else { + unreachable!() + }; + let magnitude = num.abs(); + let prefix = if self.flags.contains(CConversionFlags::ALTERNATE_FORM) { + match format_type { + Octal => "0o", + HexLower => "0x", + HexUpper => "0X", + _ => "", + } + } else { + "" + }; + + let magnitude_string: String = match format_type { + DecimalD | DecimalI | DecimalU => magnitude.to_str_radix(10), + Octal => magnitude.to_str_radix(8), + HexLower => magnitude.to_str_radix(16), + HexUpper => { + let mut result = magnitude.to_str_radix(16); + result.make_ascii_uppercase(); + result + } + }; + + let sign_string = match num.sign() { + Sign::Minus => "-", + _ => self.flags.sign_string(), + }; + + let padded_magnitude_string = self.fill_string_with_precision(magnitude_string, '0'); + + if self.flags.contains(CConversionFlags::ZERO_PAD) { + let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { + '0' + } else { + ' ' // '-' overrides the '0' conversion if both are given + }; + let signed_prefix = format!("{sign_string}{prefix}"); + format!( + "{}{}", + signed_prefix, + self.fill_string( + padded_magnitude_string, + fill_char, + Some(signed_prefix.chars().count()), + ), + ) + } else { + self.fill_string( + format!("{sign_string}{prefix}{padded_magnitude_string}"), + ' ', + None, + ) + } + } + + pub fn format_float(&self, num: f64) -> String { + let sign_string = if num.is_sign_negative() && !num.is_nan() { + "-" + } else { + self.flags.sign_string() + }; + + let precision = match &self.precision { + Some(CFormatPrecision::Quantity(quantity)) => match quantity { + CFormatQuantity::Amount(amount) => *amount, + CFormatQuantity::FromValuesTuple => 6, + }, + Some(CFormatPrecision::Dot) => 0, + None => 6, + }; + + let CFormatType::Float(format_type) = self.format_type else { + unreachable!() + }; + + let magnitude = num.abs(); + let case = format_type.case(); + + let magnitude_string = match format_type { + CFloatType::PointDecimalLower | CFloatType::PointDecimalUpper => float::format_fixed( + precision, + magnitude, + case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + ), + CFloatType::ExponentLower | CFloatType::ExponentUpper => float::format_exponent( + precision, + magnitude, + case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + ), + CFloatType::GeneralLower | CFloatType::GeneralUpper => { + let precision = if precision == 0 { 1 } else { precision }; + float::format_general( + precision, + magnitude, + case, + self.flags.contains(CConversionFlags::ALTERNATE_FORM), + false, + ) + } + }; + + if self.flags.contains(CConversionFlags::ZERO_PAD) { + let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { + '0' + } else { + ' ' + }; + format!( + "{}{}", + sign_string, + self.fill_string( + magnitude_string, + fill_char, + Some(sign_string.chars().count()), + ) + ) + } else { + self.fill_string(format!("{sign_string}{magnitude_string}"), ' ', None) + } + } +} + +fn parse_spec_mapping_key<T, I>(iter: &mut ParseIter<I>) -> Result<Option<T>, ParsingError> +where + T: FormatBuf, + I: Iterator<Item = T::Char>, +{ + if let Some((index, _)) = iter.next_if(|(_, c)| c.eq_char('(')) { + return match parse_text_inside_parentheses(iter) { + Some(key) => Ok(Some(key)), + None => Err((CFormatErrorType::UnmatchedKeyParentheses, index)), + }; + } + Ok(None) +} + +fn parse_flags<C, I>(iter: &mut ParseIter<I>) -> CConversionFlags +where + C: FormatChar, + I: Iterator<Item = C>, +{ + let mut flags = CConversionFlags::empty(); + iter.peeking_take_while(|(_, c)| { + let flag = match c.to_char_lossy() { + '#' => CConversionFlags::ALTERNATE_FORM, + '0' => CConversionFlags::ZERO_PAD, + '-' => CConversionFlags::LEFT_ADJUST, + ' ' => CConversionFlags::BLANK_SIGN, + '+' => CConversionFlags::SIGN_CHAR, + _ => return false, + }; + flags |= flag; + true + }) + .for_each(drop); + flags +} + +fn consume_length<C, I>(iter: &mut ParseIter<I>) +where + C: FormatChar, + I: Iterator<Item = C>, +{ + iter.next_if(|(_, c)| matches!(c.to_char_lossy(), 'h' | 'l' | 'L')); +} + +fn parse_format_type<C, I>(iter: &mut ParseIter<I>) -> Result<CFormatType, ParsingError> +where + C: FormatChar, + I: Iterator<Item = C>, +{ + use CFloatType::*; + use CNumberType::*; + let (index, c) = iter.next().ok_or_else(|| { + ( + CFormatErrorType::IncompleteFormat, + iter.peek().map(|x| x.0).unwrap_or(0), + ) + })?; + let format_type = match c.to_char_lossy() { + 'd' => CFormatType::Number(DecimalD), + 'i' => CFormatType::Number(DecimalI), + 'u' => CFormatType::Number(DecimalU), + 'o' => CFormatType::Number(Octal), + 'x' => CFormatType::Number(HexLower), + 'X' => CFormatType::Number(HexUpper), + 'e' => CFormatType::Float(ExponentLower), + 'E' => CFormatType::Float(ExponentUpper), + 'f' => CFormatType::Float(PointDecimalLower), + 'F' => CFormatType::Float(PointDecimalUpper), + 'g' => CFormatType::Float(GeneralLower), + 'G' => CFormatType::Float(GeneralUpper), + 'c' => CFormatType::Character(CCharacterType::Character), + 'r' => CFormatType::String(CFormatConversion::Repr), + 's' => CFormatType::String(CFormatConversion::Str), + 'b' => CFormatType::String(CFormatConversion::Bytes), + 'a' => CFormatType::String(CFormatConversion::Ascii), + _ => return Err((CFormatErrorType::UnsupportedFormatChar(c.into()), index)), + }; + Ok(format_type) +} + +fn parse_quantity<C, I>(iter: &mut ParseIter<I>) -> Result<Option<CFormatQuantity>, ParsingError> +where + C: FormatChar, + I: Iterator<Item = C>, +{ + if let Some(&(_, c)) = iter.peek() { + if c.eq_char('*') { + iter.next().unwrap(); + return Ok(Some(CFormatQuantity::FromValuesTuple)); + } + if let Some(i) = c.to_char_lossy().to_digit(10) { + let mut num = i as i32; + iter.next().unwrap(); + while let Some(&(index, c)) = iter.peek() { + if let Some(i) = c.to_char_lossy().to_digit(10) { + num = num + .checked_mul(10) + .and_then(|num| num.checked_add(i as i32)) + .ok_or((CFormatErrorType::IntTooBig, index))?; + iter.next().unwrap(); + } else { + break; + } + } + return Ok(Some(CFormatQuantity::Amount(num.unsigned_abs() as usize))); + } + } + Ok(None) +} + +fn parse_precision<C, I>(iter: &mut ParseIter<I>) -> Result<Option<CFormatPrecision>, ParsingError> +where + C: FormatChar, + I: Iterator<Item = C>, +{ + if iter.next_if(|(_, c)| c.eq_char('.')).is_some() { + let quantity = parse_quantity(iter)?; + let precision = quantity.map_or(CFormatPrecision::Dot, CFormatPrecision::Quantity); + return Ok(Some(precision)); + } + Ok(None) +} + +fn parse_text_inside_parentheses<T, I>(iter: &mut ParseIter<I>) -> Option<T> +where + T: FormatBuf, + I: Iterator<Item = T::Char>, +{ + let mut counter: i32 = 1; + let mut contained_text = T::default(); + loop { + let (_, c) = iter.next()?; + match c.to_char_lossy() { + '(' => { + counter += 1; + } + ')' => { + counter -= 1; + } + _ => (), + } + + if counter > 0 { + contained_text.extend([c]); + } else { + break; + } + } + + Some(contained_text) +} + +#[derive(Debug, PartialEq)] +pub enum CFormatPart<T> { + Literal(T), + Spec(CFormatSpecKeyed<T>), +} + +impl<T> CFormatPart<T> { + #[inline] + pub const fn is_specifier(&self) -> bool { + matches!(self, Self::Spec { .. }) + } + + #[inline] + pub const fn has_key(&self) -> bool { + match self { + Self::Spec(s) => s.mapping_key.is_some(), + _ => false, + } + } +} + +#[derive(Debug, PartialEq)] +pub struct CFormatStrOrBytes<S> { + parts: Vec<(usize, CFormatPart<S>)>, +} + +impl<S> CFormatStrOrBytes<S> { + pub fn check_specifiers(&self) -> Option<(usize, bool)> { + let mut count = 0; + let mut mapping_required = false; + for (_, part) in &self.parts { + if part.is_specifier() { + let has_key = part.has_key(); + if count == 0 { + mapping_required = has_key; + } else if mapping_required != has_key { + return None; + } + count += 1; + } + } + Some((count, mapping_required)) + } + + #[inline] + pub fn iter(&self) -> impl Iterator<Item = &(usize, CFormatPart<S>)> { + self.parts.iter() + } + + #[inline] + pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut (usize, CFormatPart<S>)> { + self.parts.iter_mut() + } + + pub fn parse<I>(iter: &mut ParseIter<I>) -> Result<Self, CFormatError> + where + S: FormatBuf, + I: Iterator<Item = S::Char>, + { + let mut parts = vec![]; + let mut literal = S::default(); + let mut part_index = 0; + while let Some((index, c)) = iter.next() { + if c.eq_char('%') { + if let Some(&(_, second)) = iter.peek() { + if second.eq_char('%') { + iter.next().unwrap(); + literal.extend([second]); + continue; + } else { + if !literal.is_empty() { + parts.push(( + part_index, + CFormatPart::Literal(core::mem::take(&mut literal)), + )); + } + let spec = CFormatSpecKeyed::parse(iter).map_err(|err| CFormatError { + typ: err.0, + index: err.1, + })?; + parts.push((index, CFormatPart::Spec(spec))); + if let Some(&(index, _)) = iter.peek() { + part_index = index; + } + } + } else { + return Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: index + 1, + }); + } + } else { + literal.extend([c]); + } + } + if !literal.is_empty() { + parts.push((part_index, CFormatPart::Literal(literal))); + } + Ok(Self { parts }) + } +} + +impl<S> IntoIterator for CFormatStrOrBytes<S> { + type Item = (usize, CFormatPart<S>); + type IntoIter = alloc::vec::IntoIter<Self::Item>; + + fn into_iter(self) -> Self::IntoIter { + self.parts.into_iter() + } +} + +pub type CFormatBytes = CFormatStrOrBytes<Vec<u8>>; + +impl CFormatBytes { + pub fn parse_from_bytes(bytes: &[u8]) -> Result<Self, CFormatError> { + let mut iter = bytes.iter().cloned().enumerate().peekable(); + Self::parse(&mut iter) + } +} + +pub type CFormatString = CFormatStrOrBytes<String>; + +impl FromStr for CFormatString { + type Err = CFormatError; + + fn from_str(text: &str) -> Result<Self, Self::Err> { + let mut iter = text.chars().enumerate().peekable(); + Self::parse(&mut iter) + } +} + +pub type CFormatWtf8 = CFormatStrOrBytes<Wtf8Buf>; + +impl CFormatWtf8 { + pub fn parse_from_wtf8(s: &Wtf8) -> Result<Self, CFormatError> { + let mut iter = s.code_points().enumerate().peekable(); + Self::parse(&mut iter) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fill_and_align() { + assert_eq!( + "%10s" + .parse::<CFormatSpec>() + .unwrap() + .format_string("test".to_owned()), + " test".to_owned() + ); + assert_eq!( + "%-10s" + .parse::<CFormatSpec>() + .unwrap() + .format_string("test".to_owned()), + "test ".to_owned() + ); + assert_eq!( + "%#10x" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(0x1337)), + " 0x1337".to_owned() + ); + assert_eq!( + "%-#10x" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x1337 ".to_owned() + ); + } + + #[test] + fn test_parse_key() { + let expected = Ok(CFormatSpecKeyed { + mapping_key: Some("amount".to_owned()), + spec: CFormatSpec { + format_type: CFormatType::Number(CNumberType::DecimalD), + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }, + }); + assert_eq!("%(amount)d".parse::<CFormatSpecKeyed<String>>(), expected); + + let expected = Ok(CFormatSpecKeyed { + mapping_key: Some("m((u(((l((((ti))))p)))l))e".to_owned()), + spec: CFormatSpec { + format_type: CFormatType::Number(CNumberType::DecimalD), + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }, + }); + assert_eq!( + "%(m((u(((l((((ti))))p)))l))e)d".parse::<CFormatSpecKeyed<String>>(), + expected + ); + } + + #[test] + fn test_format_parse_key_fail() { + assert_eq!( + "%(aged".parse::<CFormatString>(), + Err(CFormatError { + typ: CFormatErrorType::UnmatchedKeyParentheses, + index: 1 + }) + ); + } + + #[test] + fn test_format_parse_type_fail() { + assert_eq!( + "Hello %n".parse::<CFormatString>(), + Err(CFormatError { + typ: CFormatErrorType::UnsupportedFormatChar('n'.into()), + index: 7 + }) + ); + } + + #[test] + fn test_incomplete_format_fail() { + assert_eq!( + "Hello %".parse::<CFormatString>(), + Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: 7 + }) + ); + } + + #[test] + fn test_parse_flags() { + let expected = Ok(CFormatSpec { + format_type: CFormatType::Number(CNumberType::DecimalD), + min_field_width: Some(CFormatQuantity::Amount(10)), + precision: None, + flags: CConversionFlags::all(), + }); + let parsed = "% 0 -+++###10d".parse::<CFormatSpec>(); + assert_eq!(parsed, expected); + assert_eq!( + parsed.unwrap().format_number(&BigInt::from(12)), + "+12 ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_string() { + assert_eq!( + "%5.4s" + .parse::<CFormatSpec>() + .unwrap() + .format_string("Hello, World!".to_owned()), + " Hell".to_owned() + ); + assert_eq!( + "%-5.4s" + .parse::<CFormatSpec>() + .unwrap() + .format_string("Hello, World!".to_owned()), + "Hell ".to_owned() + ); + assert_eq!( + "%.s" + .parse::<CFormatSpec>() + .unwrap() + .format_string("Hello, World!".to_owned()), + "".to_owned() + ); + assert_eq!( + "%5.s" + .parse::<CFormatSpec>() + .unwrap() + .format_string("Hello, World!".to_owned()), + " ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_unicode_string() { + assert_eq!( + "%.2s" + .parse::<CFormatSpec>() + .unwrap() + .format_string("❤❤❤❤❤❤❤❤".to_owned()), + "❤❤".to_owned() + ); + } + + #[test] + fn test_parse_and_format_number() { + assert_eq!( + "%5d" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(27)), + " 27".to_owned() + ); + assert_eq!( + "%05d" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(27)), + "00027".to_owned() + ); + assert_eq!( + "%.5d" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(27)), + "00027".to_owned() + ); + assert_eq!( + "%+05d" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(27)), + "+0027".to_owned() + ); + assert_eq!( + "%-d" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(-27)), + "-27".to_owned() + ); + assert_eq!( + "% d" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(27)), + " 27".to_owned() + ); + assert_eq!( + "% d" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(-27)), + "-27".to_owned() + ); + assert_eq!( + "%08x" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "00001337".to_owned() + ); + assert_eq!( + "%#010x" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x00001337".to_owned() + ); + assert_eq!( + "%-#010x" + .parse::<CFormatSpec>() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x1337 ".to_owned() + ); + } + + #[test] + fn test_parse_and_format_float() { + assert_eq!( + "%f".parse::<CFormatSpec>().unwrap().format_float(1.2345), + "1.234500" + ); + assert_eq!( + "%.2f".parse::<CFormatSpec>().unwrap().format_float(1.2345), + "1.23" + ); + assert_eq!( + "%.f".parse::<CFormatSpec>().unwrap().format_float(1.2345), + "1" + ); + assert_eq!( + "%+.f".parse::<CFormatSpec>().unwrap().format_float(1.2345), + "+1" + ); + assert_eq!( + "%+f".parse::<CFormatSpec>().unwrap().format_float(1.2345), + "+1.234500" + ); + assert_eq!( + "% f".parse::<CFormatSpec>().unwrap().format_float(1.2345), + " 1.234500" + ); + assert_eq!( + "%f".parse::<CFormatSpec>().unwrap().format_float(-1.2345), + "-1.234500" + ); + assert_eq!( + "%f".parse::<CFormatSpec>() + .unwrap() + .format_float(1.2345678901), + "1.234568" + ); + } + + #[test] + fn test_format_parse() { + let fmt = "Hello, my name is %s and I'm %d years old"; + let expected = Ok(CFormatString { + parts: vec![ + (0, CFormatPart::Literal("Hello, my name is ".to_owned())), + ( + 18, + CFormatPart::Spec(CFormatSpecKeyed { + mapping_key: None, + spec: CFormatSpec { + format_type: CFormatType::String(CFormatConversion::Str), + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }, + }), + ), + (20, CFormatPart::Literal(" and I'm ".to_owned())), + ( + 29, + CFormatPart::Spec(CFormatSpecKeyed { + mapping_key: None, + spec: CFormatSpec { + format_type: CFormatType::Number(CNumberType::DecimalD), + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }, + }), + ), + (31, CFormatPart::Literal(" years old".to_owned())), + ], + }); + let result = fmt.parse::<CFormatString>(); + assert_eq!( + result, expected, + "left = {result:#?} \n\n\n right = {expected:#?}" + ); + } +} diff --git a/crates/common/src/crt_fd.rs b/crates/common/src/crt_fd.rs new file mode 100644 index 00000000000..ab7c94f8b3b --- /dev/null +++ b/crates/common/src/crt_fd.rs @@ -0,0 +1,416 @@ +//! A module implementing an io type backed by the C runtime's file descriptors, i.e. what's +//! returned from libc::open, even on windows. + +use alloc::fmt; +use core::cmp; +use std::{ffi, io}; + +#[cfg(unix)] +use std::os::fd::AsFd; +#[cfg(not(windows))] +use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +#[cfg(windows)] +use std::os::windows::io::BorrowedHandle; + +mod c { + pub(super) use libc::*; + + #[cfg(windows)] + pub(super) use libc::commit as fsync; + #[cfg(windows)] + unsafe extern "C" { + #[link_name = "_chsize_s"] + pub(super) fn ftruncate(fd: i32, len: i64) -> i32; + } +} + +// this is basically what CPython has for Py_off_t; windows uses long long +// for offsets, other platforms just use off_t +#[cfg(not(windows))] +pub type Offset = c::off_t; +#[cfg(windows)] +pub type Offset = c::c_longlong; + +#[cfg(not(windows))] +pub type Raw = RawFd; +#[cfg(windows)] +pub type Raw = i32; + +#[inline] +fn cvt<I: num_traits::PrimInt>(ret: I) -> io::Result<I> { + if ret < I::zero() { + // CRT functions set errno, not GetLastError(), so use errno_io_error + Err(crate::os::errno_io_error()) + } else { + Ok(ret) + } +} + +fn cvt_fd(ret: Raw) -> io::Result<Owned> { + cvt(ret).map(|fd| unsafe { Owned::from_raw(fd) }) +} + +const MAX_RW: usize = if cfg!(any(windows, target_vendor = "apple")) { + i32::MAX as usize +} else { + isize::MAX as usize +}; + +#[cfg(not(windows))] +type OwnedInner = OwnedFd; +#[cfg(not(windows))] +type BorrowedInner<'fd> = BorrowedFd<'fd>; + +#[cfg(windows)] +mod win { + use super::*; + use core::marker::PhantomData; + use core::mem::ManuallyDrop; + + #[repr(transparent)] + pub(super) struct OwnedInner(i32); + + impl OwnedInner { + #[inline] + pub unsafe fn from_raw_fd(fd: Raw) -> Self { + Self(fd) + } + #[inline] + pub fn as_raw_fd(&self) -> Raw { + self.0 + } + #[inline] + pub fn into_raw_fd(self) -> Raw { + let me = ManuallyDrop::new(self); + me.0 + } + } + + impl Drop for OwnedInner { + #[inline] + fn drop(&mut self) { + let _ = _close(self.0); + } + } + + #[derive(Copy, Clone)] + #[repr(transparent)] + pub(super) struct BorrowedInner<'fd> { + fd: Raw, + _marker: PhantomData<&'fd Owned>, + } + + impl BorrowedInner<'_> { + #[inline] + pub const unsafe fn borrow_raw(fd: Raw) -> Self { + Self { + fd, + _marker: PhantomData, + } + } + #[inline] + pub fn as_raw_fd(&self) -> Raw { + self.fd + } + } +} + +#[cfg(windows)] +use self::win::{BorrowedInner, OwnedInner}; + +#[repr(transparent)] +pub struct Owned { + inner: OwnedInner, +} + +impl fmt::Debug for Owned { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("crt_fd::Owned") + .field(&self.as_raw()) + .finish() + } +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Borrowed<'fd> { + inner: BorrowedInner<'fd>, +} + +impl<'fd> PartialEq for Borrowed<'fd> { + fn eq(&self, other: &Self) -> bool { + self.as_raw() == other.as_raw() + } +} +impl<'fd> Eq for Borrowed<'fd> {} + +impl fmt::Debug for Borrowed<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("crt_fd::Borrowed") + .field(&self.as_raw()) + .finish() + } +} + +impl Owned { + /// Create a `crt_fd::Owned` from a raw file descriptor. + /// + /// # Safety + /// + /// `fd` must be a valid file descriptor. + #[inline] + pub unsafe fn from_raw(fd: Raw) -> Self { + let inner = unsafe { OwnedInner::from_raw_fd(fd) }; + Self { inner } + } + + /// Create a `crt_fd::Owned` from a raw file descriptor. + /// + /// Returns an error if `fd` is -1. + /// + /// # Safety + /// + /// `fd` must be a valid file descriptor. + #[inline] + pub unsafe fn try_from_raw(fd: Raw) -> io::Result<Self> { + if fd == -1 { + Err(ebadf()) + } else { + Ok(unsafe { Self::from_raw(fd) }) + } + } + + #[inline] + pub fn borrow(&self) -> Borrowed<'_> { + unsafe { Borrowed::borrow_raw(self.as_raw()) } + } + + #[inline] + pub fn as_raw(&self) -> Raw { + self.inner.as_raw_fd() + } + + #[inline] + pub fn into_raw(self) -> Raw { + self.inner.into_raw_fd() + } + + pub fn leak<'fd>(self) -> Borrowed<'fd> { + unsafe { Borrowed::borrow_raw(self.into_raw()) } + } +} + +#[cfg(unix)] +impl From<Owned> for OwnedFd { + fn from(fd: Owned) -> Self { + fd.inner + } +} + +#[cfg(unix)] +impl From<OwnedFd> for Owned { + fn from(fd: OwnedFd) -> Self { + Self { inner: fd } + } +} + +#[cfg(unix)] +impl AsFd for Owned { + fn as_fd(&self) -> BorrowedFd<'_> { + self.inner.as_fd() + } +} + +#[cfg(unix)] +impl AsRawFd for Owned { + fn as_raw_fd(&self) -> RawFd { + self.as_raw() + } +} + +#[cfg(unix)] +impl FromRawFd for Owned { + unsafe fn from_raw_fd(fd: RawFd) -> Self { + unsafe { Self::from_raw(fd) } + } +} + +#[cfg(unix)] +impl IntoRawFd for Owned { + fn into_raw_fd(self) -> RawFd { + self.into_raw() + } +} + +impl<'fd> Borrowed<'fd> { + /// Create a `crt_fd::Borrowed` from a raw file descriptor. + /// + /// # Safety + /// + /// `fd` must be a valid file descriptor. + #[inline] + pub const unsafe fn borrow_raw(fd: Raw) -> Self { + let inner = unsafe { BorrowedInner::borrow_raw(fd) }; + Self { inner } + } + + /// Create a `crt_fd::Borrowed` from a raw file descriptor. + /// + /// Returns an error if `fd` is -1. + /// + /// # Safety + /// + /// `fd` must be a valid file descriptor. + #[inline] + pub unsafe fn try_borrow_raw(fd: Raw) -> io::Result<Self> { + if fd == -1 { + Err(ebadf()) + } else { + Ok(unsafe { Self::borrow_raw(fd) }) + } + } + + #[inline] + pub fn as_raw(self) -> Raw { + self.inner.as_raw_fd() + } +} + +#[cfg(unix)] +impl<'fd> From<Borrowed<'fd>> for BorrowedFd<'fd> { + fn from(fd: Borrowed<'fd>) -> Self { + fd.inner + } +} + +#[cfg(unix)] +impl<'fd> From<BorrowedFd<'fd>> for Borrowed<'fd> { + fn from(fd: BorrowedFd<'fd>) -> Self { + Self { inner: fd } + } +} + +#[cfg(unix)] +impl AsFd for Borrowed<'_> { + fn as_fd(&self) -> BorrowedFd<'_> { + self.inner.as_fd() + } +} + +#[cfg(unix)] +impl AsRawFd for Borrowed<'_> { + fn as_raw_fd(&self) -> RawFd { + self.as_raw() + } +} + +#[inline] +fn ebadf() -> io::Error { + io::Error::from_raw_os_error(c::EBADF) +} + +pub fn open(path: &ffi::CStr, flags: i32, mode: i32) -> io::Result<Owned> { + cvt_fd(unsafe { c::open(path.as_ptr(), flags, mode) }) +} + +#[cfg(windows)] +pub fn wopen(path: &widestring::WideCStr, flags: i32, mode: i32) -> io::Result<Owned> { + cvt_fd(unsafe { suppress_iph!(c::wopen(path.as_ptr(), flags, mode)) }) +} + +#[cfg(all(any(unix, target_os = "wasi"), not(target_os = "redox")))] +pub fn openat(dir: Borrowed<'_>, path: &ffi::CStr, flags: i32, mode: i32) -> io::Result<Owned> { + cvt_fd(unsafe { c::openat(dir.as_raw(), path.as_ptr(), flags, mode) }) +} + +pub fn fsync(fd: Borrowed<'_>) -> io::Result<()> { + cvt(unsafe { suppress_iph!(c::fsync(fd.as_raw())) })?; + Ok(()) +} + +fn _close(fd: Raw) -> io::Result<()> { + cvt(unsafe { suppress_iph!(c::close(fd)) })?; + Ok(()) +} + +pub fn close(fd: Owned) -> io::Result<()> { + _close(fd.into_raw()) +} + +pub fn ftruncate(fd: Borrowed<'_>, len: Offset) -> io::Result<()> { + let ret = unsafe { suppress_iph!(c::ftruncate(fd.as_raw(), len)) }; + // On Windows, _chsize_s returns 0 on success, or a positive error code (errno value) on failure. + // On other platforms, ftruncate returns 0 on success, or -1 on failure with errno set. + #[cfg(windows)] + { + if ret != 0 { + // _chsize_s returns errno directly, convert to Windows error code + let winerror = crate::os::errno_to_winerror(ret); + return Err(io::Error::from_raw_os_error(winerror)); + } + } + #[cfg(not(windows))] + { + cvt(ret)?; + } + Ok(()) +} + +#[cfg(windows)] +pub fn as_handle(fd: Borrowed<'_>) -> io::Result<BorrowedHandle<'_>> { + use windows_sys::Win32::Foundation::{HANDLE, INVALID_HANDLE_VALUE}; + unsafe extern "C" { + fn _get_osfhandle(fd: Borrowed<'_>) -> c::intptr_t; + } + let handle = unsafe { suppress_iph!(_get_osfhandle(fd)) }; + if handle as HANDLE == INVALID_HANDLE_VALUE { + // _get_osfhandle is a CRT function that sets errno, not GetLastError() + Err(crate::os::errno_io_error()) + } else { + Ok(unsafe { BorrowedHandle::borrow_raw(handle as _) }) + } +} + +fn _write(fd: Raw, buf: &[u8]) -> io::Result<usize> { + let count = cmp::min(buf.len(), MAX_RW); + let n = cvt(unsafe { suppress_iph!(c::write(fd, buf.as_ptr() as _, count as _)) })?; + Ok(n as usize) +} + +fn _read(fd: Raw, buf: &mut [u8]) -> io::Result<usize> { + let count = cmp::min(buf.len(), MAX_RW); + let n = cvt(unsafe { suppress_iph!(libc::read(fd, buf.as_mut_ptr() as _, count as _)) })?; + Ok(n as usize) +} + +pub fn write(fd: Borrowed<'_>, buf: &[u8]) -> io::Result<usize> { + _write(fd.as_raw(), buf) +} + +pub fn read(fd: Borrowed<'_>, buf: &mut [u8]) -> io::Result<usize> { + _read(fd.as_raw(), buf) +} + +macro_rules! impl_rw { + ($t:ty) => { + impl io::Write for $t { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + _write(self.as_raw(), buf) + } + + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + impl io::Read for $t { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + _read(self.as_raw(), buf) + } + } + }; +} + +impl_rw!(Owned); +impl_rw!(Borrowed<'_>); diff --git a/crates/common/src/encodings.rs b/crates/common/src/encodings.rs new file mode 100644 index 00000000000..913f0521e16 --- /dev/null +++ b/crates/common/src/encodings.rs @@ -0,0 +1,651 @@ +use core::ops::{self, Range}; + +use num_traits::ToPrimitive; + +use crate::str::StrKind; +use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + +pub trait StrBuffer: AsRef<Wtf8> { + fn is_compatible_with(&self, kind: StrKind) -> bool { + let s = self.as_ref(); + match kind { + StrKind::Ascii => s.is_ascii(), + StrKind::Utf8 => s.is_utf8(), + StrKind::Wtf8 => true, + } + } +} + +pub trait CodecContext: Sized { + type Error; + type StrBuf: StrBuffer; + type BytesBuf: AsRef<[u8]>; + + fn string(&self, s: Wtf8Buf) -> Self::StrBuf; + fn bytes(&self, b: Vec<u8>) -> Self::BytesBuf; +} + +pub trait EncodeContext: CodecContext { + fn full_data(&self) -> &Wtf8; + fn data_len(&self) -> StrSize; + + fn remaining_data(&self) -> &Wtf8; + fn position(&self) -> StrSize; + + fn restart_from(&mut self, pos: StrSize) -> Result<(), Self::Error>; + + fn error_encoding(&self, range: Range<StrSize>, reason: Option<&str>) -> Self::Error; + + fn handle_error<E>( + &mut self, + errors: &E, + range: Range<StrSize>, + reason: Option<&str>, + ) -> Result<EncodeReplace<Self>, Self::Error> + where + E: EncodeErrorHandler<Self>, + { + let (replace, restart) = errors.handle_encode_error(self, range, reason)?; + self.restart_from(restart)?; + Ok(replace) + } +} + +pub trait DecodeContext: CodecContext { + fn full_data(&self) -> &[u8]; + + fn remaining_data(&self) -> &[u8]; + fn position(&self) -> usize; + + fn advance(&mut self, by: usize); + + fn restart_from(&mut self, pos: usize) -> Result<(), Self::Error>; + + fn error_decoding(&self, byte_range: Range<usize>, reason: Option<&str>) -> Self::Error; + + fn handle_error<E>( + &mut self, + errors: &E, + byte_range: Range<usize>, + reason: Option<&str>, + ) -> Result<Self::StrBuf, Self::Error> + where + E: DecodeErrorHandler<Self>, + { + let (replace, restart) = errors.handle_decode_error(self, byte_range, reason)?; + self.restart_from(restart)?; + Ok(replace) + } +} + +pub trait EncodeErrorHandler<Ctx: EncodeContext> { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range<StrSize>, + reason: Option<&str>, + ) -> Result<(EncodeReplace<Ctx>, StrSize), Ctx::Error>; +} +pub trait DecodeErrorHandler<Ctx: DecodeContext> { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range<usize>, + reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error>; +} + +pub enum EncodeReplace<Ctx: CodecContext> { + Str(Ctx::StrBuf), + Bytes(Ctx::BytesBuf), +} + +#[derive(Copy, Clone, Default, Debug)] +pub struct StrSize { + pub bytes: usize, + pub chars: usize, +} + +fn iter_code_points(w: &Wtf8) -> impl Iterator<Item = (StrSize, CodePoint)> { + w.code_point_indices() + .enumerate() + .map(|(chars, (bytes, c))| (StrSize { bytes, chars }, c)) +} + +impl ops::Add for StrSize { + type Output = Self; + fn add(self, rhs: Self) -> Self::Output { + Self { + bytes: self.bytes + rhs.bytes, + chars: self.chars + rhs.chars, + } + } +} + +impl ops::AddAssign for StrSize { + fn add_assign(&mut self, rhs: Self) { + self.bytes += rhs.bytes; + self.chars += rhs.chars; + } +} + +struct DecodeError<'a> { + valid_prefix: &'a str, + rest: &'a [u8], + err_len: Option<usize>, +} + +/// # Safety +/// `v[..valid_up_to]` must be valid utf8 +const unsafe fn make_decode_err( + v: &[u8], + valid_up_to: usize, + err_len: Option<usize>, +) -> DecodeError<'_> { + let (valid_prefix, rest) = unsafe { v.split_at_unchecked(valid_up_to) }; + let valid_prefix = unsafe { core::str::from_utf8_unchecked(valid_prefix) }; + DecodeError { + valid_prefix, + rest, + err_len, + } +} + +enum HandleResult<'a> { + Done, + Error { + err_len: Option<usize>, + reason: &'a str, + }, +} + +fn decode_utf8_compatible<Ctx, E, DecodeF, ErrF>( + mut ctx: Ctx, + errors: &E, + decode: DecodeF, + handle_error: ErrF, +) -> Result<(Wtf8Buf, usize), Ctx::Error> +where + Ctx: DecodeContext, + E: DecodeErrorHandler<Ctx>, + DecodeF: Fn(&[u8]) -> Result<&str, DecodeError<'_>>, + ErrF: Fn(&[u8], Option<usize>) -> HandleResult<'static>, +{ + if ctx.remaining_data().is_empty() { + return Ok((Wtf8Buf::new(), 0)); + } + let mut out = Wtf8Buf::with_capacity(ctx.remaining_data().len()); + loop { + match decode(ctx.remaining_data()) { + Ok(decoded) => { + out.push_str(decoded); + ctx.advance(decoded.len()); + break; + } + Err(e) => { + out.push_str(e.valid_prefix); + match handle_error(e.rest, e.err_len) { + HandleResult::Done => { + ctx.advance(e.valid_prefix.len()); + break; + } + HandleResult::Error { err_len, reason } => { + let err_start = ctx.position() + e.valid_prefix.len(); + let err_end = match err_len { + Some(len) => err_start + len, + None => ctx.full_data().len(), + }; + let err_range = err_start..err_end; + let replace = ctx.handle_error(errors, err_range, Some(reason))?; + out.push_wtf8(replace.as_ref()); + continue; + } + } + } + } + } + Ok((out, ctx.position())) +} + +#[inline] +fn encode_utf8_compatible<Ctx, E>( + mut ctx: Ctx, + errors: &E, + err_reason: &str, + target_kind: StrKind, +) -> Result<Vec<u8>, Ctx::Error> +where + Ctx: EncodeContext, + E: EncodeErrorHandler<Ctx>, +{ + // let mut data = s.as_ref(); + // let mut char_data_index = 0; + let mut out = Vec::<u8>::with_capacity(ctx.remaining_data().len()); + loop { + let data = ctx.remaining_data(); + let mut iter = iter_code_points(data); + let Some((i, _)) = iter.find(|(_, c)| !target_kind.can_encode(*c)) else { + break; + }; + + out.extend_from_slice(&ctx.remaining_data().as_bytes()[..i.bytes]); + + let err_start = ctx.position() + i; + // number of non-compatible chars between the first non-compatible char and the next compatible char + let err_end = match { iter }.find(|(_, c)| target_kind.can_encode(*c)) { + Some((i, _)) => ctx.position() + i, + None => ctx.data_len(), + }; + + let range = err_start..err_end; + let replace = ctx.handle_error(errors, range.clone(), Some(err_reason))?; + match replace { + EncodeReplace::Str(s) => { + if s.is_compatible_with(target_kind) { + out.extend_from_slice(s.as_ref().as_bytes()); + } else { + return Err(ctx.error_encoding(range, Some(err_reason))); + } + } + EncodeReplace::Bytes(b) => { + out.extend_from_slice(b.as_ref()); + } + } + } + out.extend_from_slice(ctx.remaining_data().as_bytes()); + Ok(out) +} + +pub mod errors { + use crate::str::UnicodeEscapeCodepoint; + + use super::*; + use core::fmt::Write; + + #[derive(Clone, Copy)] + pub struct Strict; + + impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for Strict { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range<StrSize>, + reason: Option<&str>, + ) -> Result<(EncodeReplace<Ctx>, StrSize), Ctx::Error> { + Err(ctx.error_encoding(range, reason)) + } + } + + impl<Ctx: DecodeContext> DecodeErrorHandler<Ctx> for Strict { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range<usize>, + reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + Err(ctx.error_decoding(byte_range, reason)) + } + } + + #[derive(Clone, Copy)] + pub struct Ignore; + + impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for Ignore { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range<StrSize>, + _reason: Option<&str>, + ) -> Result<(EncodeReplace<Ctx>, StrSize), Ctx::Error> { + Ok((EncodeReplace::Bytes(ctx.bytes(b"".into())), range.end)) + } + } + + impl<Ctx: DecodeContext> DecodeErrorHandler<Ctx> for Ignore { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range<usize>, + _reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + Ok((ctx.string("".into()), byte_range.end)) + } + } + + #[derive(Clone, Copy)] + pub struct Replace; + + impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for Replace { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range<StrSize>, + _reason: Option<&str>, + ) -> Result<(EncodeReplace<Ctx>, StrSize), Ctx::Error> { + let replace = "?".repeat(range.end.chars - range.start.chars); + Ok((EncodeReplace::Str(ctx.string(replace.into())), range.end)) + } + } + + impl<Ctx: DecodeContext> DecodeErrorHandler<Ctx> for Replace { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range<usize>, + _reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + Ok(( + ctx.string(char::REPLACEMENT_CHARACTER.to_string().into()), + byte_range.end, + )) + } + } + + #[derive(Clone, Copy)] + pub struct XmlCharRefReplace; + + impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for XmlCharRefReplace { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range<StrSize>, + _reason: Option<&str>, + ) -> Result<(EncodeReplace<Ctx>, StrSize), Ctx::Error> { + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + // capacity rough guess; assuming that the codepoints are 3 digits in decimal + the &#; + let mut out = String::with_capacity(num_chars * 6); + for c in err_str.code_points() { + write!(out, "&#{};", c.to_u32()).unwrap() + } + Ok((EncodeReplace::Str(ctx.string(out.into())), range.end)) + } + } + + #[derive(Clone, Copy)] + pub struct BackslashReplace; + + impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for BackslashReplace { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range<StrSize>, + _reason: Option<&str>, + ) -> Result<(EncodeReplace<Ctx>, StrSize), Ctx::Error> { + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + // minimum 4 output bytes per char: \xNN + let mut out = String::with_capacity(num_chars * 4); + for c in err_str.code_points() { + write!(out, "{}", UnicodeEscapeCodepoint(c)).unwrap(); + } + Ok((EncodeReplace::Str(ctx.string(out.into())), range.end)) + } + } + + impl<Ctx: DecodeContext> DecodeErrorHandler<Ctx> for BackslashReplace { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range<usize>, + _reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + let err_bytes = &ctx.full_data()[byte_range.clone()]; + let mut replace = String::with_capacity(4 * err_bytes.len()); + for &c in err_bytes { + write!(replace, "\\x{c:02x}").unwrap(); + } + Ok((ctx.string(replace.into()), byte_range.end)) + } + } + + #[derive(Clone, Copy)] + pub struct NameReplace; + + impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for NameReplace { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range<StrSize>, + _reason: Option<&str>, + ) -> Result<(EncodeReplace<Ctx>, StrSize), Ctx::Error> { + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + let mut out = String::with_capacity(num_chars * 4); + for c in err_str.code_points() { + let c_u32 = c.to_u32(); + if let Some(c_name) = c.to_char().and_then(unicode_names2::name) { + write!(out, "\\N{{{c_name}}}").unwrap(); + } else if c_u32 >= 0x10000 { + write!(out, "\\U{c_u32:08x}").unwrap(); + } else if c_u32 >= 0x100 { + write!(out, "\\u{c_u32:04x}").unwrap(); + } else { + write!(out, "\\x{c_u32:02x}").unwrap(); + } + } + Ok((EncodeReplace::Str(ctx.string(out.into())), range.end)) + } + } + + #[derive(Clone, Copy)] + pub struct SurrogateEscape; + + impl<Ctx: EncodeContext> EncodeErrorHandler<Ctx> for SurrogateEscape { + fn handle_encode_error( + &self, + ctx: &mut Ctx, + range: Range<StrSize>, + reason: Option<&str>, + ) -> Result<(EncodeReplace<Ctx>, StrSize), Ctx::Error> { + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + let mut out = Vec::with_capacity(num_chars); + let mut pos = range.start; + for ch in err_str.code_points() { + let ch_u32 = ch.to_u32(); + if !(0xdc80..=0xdcff).contains(&ch_u32) { + if out.is_empty() { + // Can't handle even the first character + return Err(ctx.error_encoding(range, reason)); + } + // Return partial result, restart from this character + return Ok((EncodeReplace::Bytes(ctx.bytes(out)), pos)); + } + out.push((ch_u32 - 0xdc00) as u8); + pos += StrSize { + bytes: ch.len_wtf8(), + chars: 1, + }; + } + Ok((EncodeReplace::Bytes(ctx.bytes(out)), range.end)) + } + } + + impl<Ctx: DecodeContext> DecodeErrorHandler<Ctx> for SurrogateEscape { + fn handle_decode_error( + &self, + ctx: &mut Ctx, + byte_range: Range<usize>, + reason: Option<&str>, + ) -> Result<(Ctx::StrBuf, usize), Ctx::Error> { + let err_bytes = &ctx.full_data()[byte_range.clone()]; + let mut consumed = 0; + let mut replace = Wtf8Buf::with_capacity(4 * byte_range.len()); + while consumed < 4 && consumed < byte_range.len() { + let c = err_bytes[consumed] as u16; + // Refuse to escape ASCII bytes + if c < 128 { + break; + } + replace.push(CodePoint::from(0xdc00 + c)); + consumed += 1; + } + if consumed == 0 { + return Err(ctx.error_decoding(byte_range, reason)); + } + Ok((ctx.string(replace), byte_range.start + consumed)) + } + } +} + +pub mod utf8 { + use super::*; + + pub const ENCODING_NAME: &str = "utf-8"; + + #[inline] + pub fn encode<Ctx, E>(ctx: Ctx, errors: &E) -> Result<Vec<u8>, Ctx::Error> + where + Ctx: EncodeContext, + E: EncodeErrorHandler<Ctx>, + { + encode_utf8_compatible(ctx, errors, "surrogates not allowed", StrKind::Utf8) + } + + pub fn decode<Ctx: DecodeContext, E: DecodeErrorHandler<Ctx>>( + ctx: Ctx, + errors: &E, + final_decode: bool, + ) -> Result<(Wtf8Buf, usize), Ctx::Error> { + decode_utf8_compatible( + ctx, + errors, + |v| { + core::str::from_utf8(v).map_err(|e| { + // SAFETY: as specified in valid_up_to's documentation, input[..e.valid_up_to()] + // is valid utf8 + unsafe { make_decode_err(v, e.valid_up_to(), e.error_len()) } + }) + }, + |rest, err_len| { + let first_err = rest[0]; + if matches!(first_err, 0x80..=0xc1 | 0xf5..=0xff) { + HandleResult::Error { + err_len: Some(1), + reason: "invalid start byte", + } + } else if err_len.is_none() { + // error_len() == None means unexpected eof + if final_decode { + HandleResult::Error { + err_len, + reason: "unexpected end of data", + } + } else { + HandleResult::Done + } + } else if !final_decode && matches!(rest, [0xed, 0xa0..=0xbf]) { + // truncated surrogate + HandleResult::Done + } else { + HandleResult::Error { + err_len, + reason: "invalid continuation byte", + } + } + }, + ) + } +} + +pub mod latin_1 { + use super::*; + + pub const ENCODING_NAME: &str = "latin-1"; + + const ERR_REASON: &str = "ordinal not in range(256)"; + + #[inline] + pub fn encode<Ctx, E>(mut ctx: Ctx, errors: &E) -> Result<Vec<u8>, Ctx::Error> + where + Ctx: EncodeContext, + E: EncodeErrorHandler<Ctx>, + { + let mut out = Vec::<u8>::new(); + loop { + let data = ctx.remaining_data(); + let mut iter = iter_code_points(ctx.remaining_data()); + let Some((i, ch)) = iter.find(|(_, c)| !c.is_ascii()) else { + break; + }; + out.extend_from_slice(&data.as_bytes()[..i.bytes]); + let err_start = ctx.position() + i; + if let Some(byte) = ch.to_u32().to_u8() { + drop(iter); + out.push(byte); + // if the codepoint is between 128..=255, it's utf8-length is 2 + ctx.restart_from(err_start + StrSize { bytes: 2, chars: 1 })?; + } else { + // number of non-latin_1 chars between the first non-latin_1 char and the next latin_1 char + let err_end = match { iter }.find(|(_, c)| c.to_u32() <= 255) { + Some((i, _)) => ctx.position() + i, + None => ctx.data_len(), + }; + let err_range = err_start..err_end; + let replace = ctx.handle_error(errors, err_range.clone(), Some(ERR_REASON))?; + match replace { + EncodeReplace::Str(s) => { + if s.as_ref().code_points().any(|c| c.to_u32() > 255) { + return Err(ctx.error_encoding(err_range, Some(ERR_REASON))); + } + out.extend(s.as_ref().code_points().map(|c| c.to_u32() as u8)); + } + EncodeReplace::Bytes(b) => { + out.extend_from_slice(b.as_ref()); + } + } + } + } + out.extend_from_slice(ctx.remaining_data().as_bytes()); + Ok(out) + } + + pub fn decode<Ctx: DecodeContext, E: DecodeErrorHandler<Ctx>>( + ctx: Ctx, + _errors: &E, + ) -> Result<(Wtf8Buf, usize), Ctx::Error> { + let out: String = ctx.remaining_data().iter().map(|c| *c as char).collect(); + let out_len = out.len(); + Ok((out.into(), out_len)) + } +} + +pub mod ascii { + use super::*; + use ::ascii::AsciiStr; + + pub const ENCODING_NAME: &str = "ascii"; + + const ERR_REASON: &str = "ordinal not in range(128)"; + + #[inline] + pub fn encode<Ctx, E>(ctx: Ctx, errors: &E) -> Result<Vec<u8>, Ctx::Error> + where + Ctx: EncodeContext, + E: EncodeErrorHandler<Ctx>, + { + encode_utf8_compatible(ctx, errors, ERR_REASON, StrKind::Ascii) + } + + pub fn decode<Ctx: DecodeContext, E: DecodeErrorHandler<Ctx>>( + ctx: Ctx, + errors: &E, + ) -> Result<(Wtf8Buf, usize), Ctx::Error> { + decode_utf8_compatible( + ctx, + errors, + |v| { + AsciiStr::from_ascii(v).map(|s| s.as_str()).map_err(|e| { + // SAFETY: as specified in valid_up_to's documentation, input[..e.valid_up_to()] + // is valid ascii & therefore valid utf8 + unsafe { make_decode_err(v, e.valid_up_to(), Some(1)) } + }) + }, + |_rest, err_len| HandleResult::Error { + err_len, + reason: ERR_REASON, + }, + ) + } +} diff --git a/crates/common/src/fileutils.rs b/crates/common/src/fileutils.rs new file mode 100644 index 00000000000..a20140a6e04 --- /dev/null +++ b/crates/common/src/fileutils.rs @@ -0,0 +1,532 @@ +// Python/fileutils.c in CPython +#![allow(non_snake_case)] + +#[cfg(not(windows))] +pub use libc::stat as StatStruct; + +#[cfg(windows)] +pub use windows::{StatStruct, fstat}; + +#[cfg(not(windows))] +pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result<StatStruct> { + let mut stat = core::mem::MaybeUninit::uninit(); + unsafe { + let ret = libc::fstat(fd.as_raw(), stat.as_mut_ptr()); + if ret == -1 { + Err(crate::os::errno_io_error()) + } else { + Ok(stat.assume_init()) + } + } +} + +#[cfg(windows)] +pub mod windows { + use crate::crt_fd; + use crate::windows::ToWideString; + use alloc::ffi::CString; + use libc::{S_IFCHR, S_IFDIR, S_IFMT}; + use std::ffi::{OsStr, OsString}; + use std::os::windows::io::AsRawHandle; + use std::sync::OnceLock; + use windows_sys::Win32::Foundation::{ + ERROR_INVALID_HANDLE, ERROR_NOT_SUPPORTED, FILETIME, FreeLibrary, SetLastError, + }; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_READONLY, + FILE_ATTRIBUTE_REPARSE_POINT, FILE_BASIC_INFO, FILE_ID_INFO, FILE_TYPE_CHAR, + FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_UNKNOWN, FileBasicInfo, FileIdInfo, + GetFileInformationByHandle, GetFileInformationByHandleEx, GetFileType, + }; + use windows_sys::Win32::System::LibraryLoader::{GetProcAddress, LoadLibraryW}; + use windows_sys::Win32::System::SystemServices::IO_REPARSE_TAG_SYMLINK; + use windows_sys::core::PCWSTR; + + pub const S_IFIFO: libc::c_int = 0o010000; + pub const S_IFLNK: libc::c_int = 0o120000; + + pub const SECS_BETWEEN_EPOCHS: i64 = 11644473600; // Seconds between 1.1.1601 and 1.1.1970 + + #[derive(Clone, Copy, Default)] + pub struct StatStruct { + pub st_dev: libc::c_ulong, + pub st_ino: u64, + pub st_mode: libc::c_ushort, + pub st_nlink: i32, + pub st_uid: i32, + pub st_gid: i32, + pub st_rdev: libc::c_ulong, + pub st_size: u64, + pub st_atime: libc::time_t, + pub st_atime_nsec: i32, + pub st_mtime: libc::time_t, + pub st_mtime_nsec: i32, + pub st_ctime: libc::time_t, + pub st_ctime_nsec: i32, + pub st_birthtime: libc::time_t, + pub st_birthtime_nsec: i32, + pub st_file_attributes: libc::c_ulong, + pub st_reparse_tag: u32, + pub st_ino_high: u64, + } + + impl StatStruct { + // update_st_mode_from_path in cpython + pub fn update_st_mode_from_path(&mut self, path: &OsStr, attr: u32) { + if attr & FILE_ATTRIBUTE_DIRECTORY == 0 { + let file_extension = path + .to_wide() + .split(|&c| c == '.' as u16) + .next_back() + .and_then(|s| String::from_utf16(s).ok()); + + if let Some(file_extension) = file_extension + && (file_extension.eq_ignore_ascii_case("exe") + || file_extension.eq_ignore_ascii_case("bat") + || file_extension.eq_ignore_ascii_case("cmd") + || file_extension.eq_ignore_ascii_case("com")) + { + self.st_mode |= 0o111; + } + } + } + } + + // _Py_fstat_noraise in cpython + pub fn fstat(fd: crt_fd::Borrowed<'_>) -> std::io::Result<StatStruct> { + let h = crt_fd::as_handle(fd); + if h.is_err() { + unsafe { SetLastError(ERROR_INVALID_HANDLE) }; + } + let h = h?; + let h = h.as_raw_handle(); + // reset stat? + + let file_type = unsafe { GetFileType(h as _) }; + if file_type == FILE_TYPE_UNKNOWN { + return Err(std::io::Error::last_os_error()); + } + if file_type != FILE_TYPE_DISK { + let st_mode = if file_type == FILE_TYPE_CHAR { + S_IFCHR + } else if file_type == FILE_TYPE_PIPE { + S_IFIFO + } else { + 0 + } as u16; + return Ok(StatStruct { + st_mode, + ..Default::default() + }); + } + + let mut info = unsafe { core::mem::zeroed() }; + let mut basic_info: FILE_BASIC_INFO = unsafe { core::mem::zeroed() }; + let mut id_info: FILE_ID_INFO = unsafe { core::mem::zeroed() }; + + if unsafe { GetFileInformationByHandle(h as _, &mut info) } == 0 + || unsafe { + GetFileInformationByHandleEx( + h as _, + FileBasicInfo, + &mut basic_info as *mut _ as *mut _, + core::mem::size_of_val(&basic_info) as u32, + ) + } == 0 + { + return Err(std::io::Error::last_os_error()); + } + + let p_id_info = if unsafe { + GetFileInformationByHandleEx( + h as _, + FileIdInfo, + &mut id_info as *mut _ as *mut _, + core::mem::size_of_val(&id_info) as u32, + ) + } == 0 + { + None + } else { + Some(&id_info) + }; + + Ok(attribute_data_to_stat( + &info, + 0, + Some(&basic_info), + p_id_info, + )) + } + + fn large_integer_to_time_t_nsec(input: i64) -> (libc::time_t, libc::c_int) { + let nsec_out = (input % 10_000_000) * 100; // FILETIME is in units of 100 nsec. + let time_out = ((input / 10_000_000) - SECS_BETWEEN_EPOCHS) as libc::time_t; + (time_out, nsec_out as _) + } + + fn file_time_to_time_t_nsec(in_ptr: &FILETIME) -> (libc::time_t, libc::c_int) { + let in_val: i64 = unsafe { core::mem::transmute_copy(in_ptr) }; + let nsec_out = (in_val % 10_000_000) * 100; // FILETIME is in units of 100 nsec. + let time_out = (in_val / 10_000_000) - SECS_BETWEEN_EPOCHS; + (time_out, nsec_out as _) + } + + fn attribute_data_to_stat( + info: &BY_HANDLE_FILE_INFORMATION, + reparse_tag: u32, + basic_info: Option<&FILE_BASIC_INFO>, + id_info: Option<&FILE_ID_INFO>, + ) -> StatStruct { + let mut st_mode = attributes_to_mode(info.dwFileAttributes); + let st_size = ((info.nFileSizeHigh as u64) << 32) + info.nFileSizeLow as u64; + let st_dev: libc::c_ulong = if let Some(id_info) = id_info { + id_info.VolumeSerialNumber as _ + } else { + info.dwVolumeSerialNumber + }; + let st_rdev = 0; + + let (st_birthtime, st_ctime, st_mtime, st_atime) = if let Some(basic_info) = basic_info { + ( + large_integer_to_time_t_nsec(basic_info.CreationTime), + large_integer_to_time_t_nsec(basic_info.ChangeTime), + large_integer_to_time_t_nsec(basic_info.LastWriteTime), + large_integer_to_time_t_nsec(basic_info.LastAccessTime), + ) + } else { + ( + file_time_to_time_t_nsec(&info.ftCreationTime), + (0, 0), + file_time_to_time_t_nsec(&info.ftLastWriteTime), + file_time_to_time_t_nsec(&info.ftLastAccessTime), + ) + }; + let st_nlink = info.nNumberOfLinks as i32; + + let st_ino = if let Some(id_info) = id_info { + let file_id: [u64; 2] = unsafe { core::mem::transmute_copy(&id_info.FileId) }; + file_id + } else { + let ino = ((info.nFileIndexHigh as u64) << 32) + info.nFileIndexLow as u64; + [ino, 0] + }; + + if info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + && reparse_tag == IO_REPARSE_TAG_SYMLINK + { + st_mode = (st_mode & !(S_IFMT as u16)) | (S_IFLNK as u16); + } + let st_file_attributes = info.dwFileAttributes; + + StatStruct { + st_dev, + st_ino: st_ino[0], + st_mode, + st_nlink, + st_uid: 0, + st_gid: 0, + st_rdev, + st_size, + st_atime: st_atime.0, + st_atime_nsec: st_atime.1, + st_mtime: st_mtime.0, + st_mtime_nsec: st_mtime.1, + st_ctime: st_ctime.0, + st_ctime_nsec: st_ctime.1, + st_birthtime: st_birthtime.0, + st_birthtime_nsec: st_birthtime.1, + st_file_attributes, + st_reparse_tag: reparse_tag, + st_ino_high: st_ino[1], + } + } + + const fn attributes_to_mode(attr: u32) -> u16 { + let mut m = 0; + if attr & FILE_ATTRIBUTE_DIRECTORY != 0 { + m |= libc::S_IFDIR | 0o111; // IFEXEC for user,group,other + } else { + m |= libc::S_IFREG; + } + if attr & FILE_ATTRIBUTE_READONLY != 0 { + m |= 0o444; + } else { + m |= 0o666; + } + m as _ + } + + #[derive(Clone, Copy)] + #[repr(C)] + pub struct FILE_STAT_BASIC_INFORMATION { + pub FileId: i64, + pub CreationTime: i64, + pub LastAccessTime: i64, + pub LastWriteTime: i64, + pub ChangeTime: i64, + pub AllocationSize: i64, + pub EndOfFile: i64, + pub FileAttributes: u32, + pub ReparseTag: u32, + pub NumberOfLinks: u32, + pub DeviceType: u32, + pub DeviceCharacteristics: u32, + pub Reserved: u32, + pub VolumeSerialNumber: i64, + pub FileId128: [u64; 2], + } + + #[allow(dead_code)] + #[derive(Clone, Copy)] + #[repr(C)] + pub enum FILE_INFO_BY_NAME_CLASS { + FileStatByNameInfo, + FileStatLxByNameInfo, + FileCaseSensitiveByNameInfo, + FileStatBasicByNameInfo, + MaximumFileInfoByNameClass, + } + + // _Py_GetFileInformationByName in cpython + pub fn get_file_information_by_name( + file_name: &OsStr, + file_information_class: FILE_INFO_BY_NAME_CLASS, + ) -> std::io::Result<FILE_STAT_BASIC_INFORMATION> { + static GET_FILE_INFORMATION_BY_NAME: OnceLock< + Option< + unsafe extern "system" fn( + PCWSTR, + FILE_INFO_BY_NAME_CLASS, + *mut libc::c_void, + u32, + ) -> i32, + >, + > = OnceLock::new(); + + let GetFileInformationByName = GET_FILE_INFORMATION_BY_NAME + .get_or_init(|| { + let library_name = + OsString::from("api-ms-win-core-file-l2-1-4.dll").to_wide_with_nul(); + let module = unsafe { LoadLibraryW(library_name.as_ptr()) }; + if module.is_null() { + return None; + } + let name = CString::new("GetFileInformationByName").unwrap(); + if let Some(proc) = + unsafe { GetProcAddress(module, name.as_bytes_with_nul().as_ptr()) } + { + Some(unsafe { + core::mem::transmute::< + unsafe extern "system" fn() -> isize, + unsafe extern "system" fn( + *const u16, + FILE_INFO_BY_NAME_CLASS, + *mut libc::c_void, + u32, + ) -> i32, + >(proc) + }) + } else { + unsafe { FreeLibrary(module) }; + None + } + }) + .ok_or_else(|| std::io::Error::from_raw_os_error(ERROR_NOT_SUPPORTED as _))?; + + let file_name = file_name.to_wide_with_nul(); + let file_info_buffer_size = core::mem::size_of::<FILE_STAT_BASIC_INFORMATION>() as u32; + let mut file_info_buffer = core::mem::MaybeUninit::<FILE_STAT_BASIC_INFORMATION>::uninit(); + unsafe { + if GetFileInformationByName( + file_name.as_ptr(), + file_information_class as _, + file_info_buffer.as_mut_ptr() as _, + file_info_buffer_size, + ) == 0 + { + Err(std::io::Error::last_os_error()) + } else { + Ok(file_info_buffer.assume_init()) + } + } + } + + pub fn stat_basic_info_to_stat(info: &FILE_STAT_BASIC_INFORMATION) -> StatStruct { + use windows_sys::Win32::Storage::FileSystem; + use windows_sys::Win32::System::Ioctl; + + const S_IFMT: u16 = self::S_IFMT as _; + const S_IFDIR: u16 = self::S_IFDIR as _; + const S_IFCHR: u16 = self::S_IFCHR as _; + const S_IFIFO: u16 = self::S_IFIFO as _; + const S_IFLNK: u16 = self::S_IFLNK as _; + + let mut st_mode = attributes_to_mode(info.FileAttributes); + let st_size = info.EndOfFile as u64; + let st_birthtime = large_integer_to_time_t_nsec(info.CreationTime); + let st_ctime = large_integer_to_time_t_nsec(info.ChangeTime); + let st_mtime = large_integer_to_time_t_nsec(info.LastWriteTime); + let st_atime = large_integer_to_time_t_nsec(info.LastAccessTime); + let st_nlink = info.NumberOfLinks as _; + let st_dev = info.VolumeSerialNumber as u32; + // File systems with less than 128-bits zero pad into this field + let st_ino = info.FileId128; + // bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will + // open other name surrogate reparse points without traversing them. To + // detect/handle these, check st_file_attributes and st_reparse_tag. + let st_reparse_tag = info.ReparseTag; + if info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + && info.ReparseTag == IO_REPARSE_TAG_SYMLINK + { + // set the bits that make this a symlink + st_mode = (st_mode & !S_IFMT) | S_IFLNK; + } + let st_file_attributes = info.FileAttributes; + match info.DeviceType { + FileSystem::FILE_DEVICE_DISK + | Ioctl::FILE_DEVICE_VIRTUAL_DISK + | Ioctl::FILE_DEVICE_DFS + | FileSystem::FILE_DEVICE_CD_ROM + | Ioctl::FILE_DEVICE_CONTROLLER + | Ioctl::FILE_DEVICE_DATALINK => {} + Ioctl::FILE_DEVICE_DISK_FILE_SYSTEM + | Ioctl::FILE_DEVICE_CD_ROM_FILE_SYSTEM + | Ioctl::FILE_DEVICE_NETWORK_FILE_SYSTEM => { + st_mode = (st_mode & !S_IFMT) | 0x6000; // _S_IFBLK + } + Ioctl::FILE_DEVICE_CONSOLE + | Ioctl::FILE_DEVICE_NULL + | Ioctl::FILE_DEVICE_KEYBOARD + | Ioctl::FILE_DEVICE_MODEM + | Ioctl::FILE_DEVICE_MOUSE + | Ioctl::FILE_DEVICE_PARALLEL_PORT + | Ioctl::FILE_DEVICE_PRINTER + | Ioctl::FILE_DEVICE_SCREEN + | Ioctl::FILE_DEVICE_SERIAL_PORT + | Ioctl::FILE_DEVICE_SOUND => { + st_mode = (st_mode & !S_IFMT) | S_IFCHR; + } + Ioctl::FILE_DEVICE_NAMED_PIPE => { + st_mode = (st_mode & !S_IFMT) | S_IFIFO; + } + _ => { + if info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY != 0 { + st_mode = (st_mode & !S_IFMT) | S_IFDIR; + } + } + } + + StatStruct { + st_dev, + st_ino: st_ino[0], + st_mode, + st_nlink, + st_uid: 0, + st_gid: 0, + st_rdev: 0, + st_size, + st_atime: st_atime.0, + st_atime_nsec: st_atime.1, + st_mtime: st_mtime.0, + st_mtime_nsec: st_mtime.1, + st_ctime: st_ctime.0, + st_ctime_nsec: st_ctime.1, + st_birthtime: st_birthtime.0, + st_birthtime_nsec: st_birthtime.1, + st_file_attributes, + st_reparse_tag, + st_ino_high: st_ino[1], + } + } +} + +// _Py_fopen_obj in cpython (Python/fileutils.c:1757-1835) +// Open a file using std::fs::File and convert to FILE* +// Automatically handles path encoding and EINTR retries +pub fn fopen(path: &std::path::Path, mode: &str) -> std::io::Result<*mut libc::FILE> { + use alloc::ffi::CString; + use std::fs::File; + + // Currently only supports read mode + // Can be extended to support "wb", "w+b", etc. if needed + if mode != "rb" { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("unsupported mode: {}", mode), + )); + } + + // Open file using std::fs::File (handles path encoding and EINTR automatically) + let file = File::open(path)?; + + #[cfg(windows)] + { + use std::os::windows::io::IntoRawHandle; + + // Convert File handle to CRT file descriptor + let handle = file.into_raw_handle(); + let fd = unsafe { libc::open_osfhandle(handle as isize, libc::O_RDONLY) }; + if fd == -1 { + return Err(std::io::Error::last_os_error()); + } + + // Convert fd to FILE* + let mode_cstr = CString::new(mode).unwrap(); + let fp = unsafe { libc::fdopen(fd, mode_cstr.as_ptr()) }; + if fp.is_null() { + unsafe { libc::close(fd) }; + return Err(std::io::Error::last_os_error()); + } + + // Set non-inheritable (Windows needs this explicitly) + if let Err(e) = set_inheritable(fd, false) { + unsafe { libc::fclose(fp) }; + return Err(e); + } + + Ok(fp) + } + + #[cfg(not(windows))] + { + use std::os::fd::IntoRawFd; + + // Convert File to raw fd + let fd = file.into_raw_fd(); + + // Convert fd to FILE* + let mode_cstr = CString::new(mode).unwrap(); + let fp = unsafe { libc::fdopen(fd, mode_cstr.as_ptr()) }; + if fp.is_null() { + unsafe { libc::close(fd) }; + return Err(std::io::Error::last_os_error()); + } + + // Unix: O_CLOEXEC is already set by File::open, so non-inheritable is automatic + Ok(fp) + } +} + +// set_inheritable in cpython (Python/fileutils.c:1443-1570) +// Set the inheritable flag of the specified file descriptor +// Only used on Windows; Unix automatically sets O_CLOEXEC +#[cfg(windows)] +fn set_inheritable(fd: libc::c_int, inheritable: bool) -> std::io::Result<()> { + use windows_sys::Win32::Foundation::{ + HANDLE, HANDLE_FLAG_INHERIT, INVALID_HANDLE_VALUE, SetHandleInformation, + }; + + let handle = unsafe { libc::get_osfhandle(fd) }; + if handle == INVALID_HANDLE_VALUE as isize { + return Err(std::io::Error::last_os_error()); + } + + let flags = if inheritable { HANDLE_FLAG_INHERIT } else { 0 }; + let result = unsafe { SetHandleInformation(handle as HANDLE, HANDLE_FLAG_INHERIT, flags) }; + if result == 0 { + return Err(std::io::Error::last_os_error()); + } + + Ok(()) +} diff --git a/common/src/float_ops.rs b/crates/common/src/float_ops.rs similarity index 78% rename from common/src/float_ops.rs rename to crates/common/src/float_ops.rs index 69ae8833a22..c6b7c71e494 100644 --- a/common/src/float_ops.rs +++ b/crates/common/src/float_ops.rs @@ -1,8 +1,8 @@ +use core::f64; use malachite_bigint::{BigInt, ToBigInt}; use num_traits::{Float, Signed, ToPrimitive, Zero}; -use std::f64; -pub fn ufrexp(value: f64) -> (f64, i32) { +pub const fn decompose_float(value: f64) -> (f64, i32) { if 0.0 == value { (0.0, 0i32) } else { @@ -63,12 +63,8 @@ pub fn gt_int(value: f64, other_int: &BigInt) -> bool { } } -pub fn div(v1: f64, v2: f64) -> Option<f64> { - if v2 != 0.0 { - Some(v1 / v2) - } else { - None - } +pub const fn div(v1: f64, v2: f64) -> Option<f64> { + if v2 != 0.0 { Some(v1 / v2) } else { None } } pub fn mod_(v1: f64, v2: f64) -> Option<f64> { @@ -125,10 +121,69 @@ pub fn nextafter(x: f64, y: f64) -> f64 { let b = x.to_bits(); let bits = if (y > x) == (x > 0.0) { b + 1 } else { b - 1 }; let ret = f64::from_bits(bits); - if ret == 0.0 { - ret.copysign(x) + if ret == 0.0 { ret.copysign(x) } else { ret } + } +} + +#[allow(clippy::float_cmp)] +pub fn nextafter_with_steps(x: f64, y: f64, steps: u64) -> f64 { + if x == y { + y + } else if x.is_nan() || y.is_nan() { + f64::NAN + } else if x >= f64::INFINITY { + f64::MAX + } else if x <= f64::NEG_INFINITY { + f64::MIN + } else if x == 0.0 { + f64::from_bits(1).copysign(y) + } else { + if steps == 0 { + return x; + } + + if x.is_nan() { + return x; + } + + if y.is_nan() { + return y; + } + + let sign_bit: u64 = 1 << 63; + + let mut ux = x.to_bits(); + let uy = y.to_bits(); + + let ax = ux & !sign_bit; + let ay = uy & !sign_bit; + + // If signs are different + if ((ux ^ uy) & sign_bit) != 0 { + return if ax + ay <= steps { + f64::from_bits(uy) + } else if ax < steps { + let result = (uy & sign_bit) | (steps - ax); + f64::from_bits(result) + } else { + ux -= steps; + f64::from_bits(ux) + }; + } + + // If signs are the same + if ax > ay { + if ax - ay >= steps { + ux -= steps; + f64::from_bits(ux) + } else { + f64::from_bits(uy) + } + } else if ay - ax >= steps { + ux += steps; + f64::from_bits(ux) } else { - ret + f64::from_bits(uy) } } } diff --git a/crates/common/src/format.rs b/crates/common/src/format.rs new file mode 100644 index 00000000000..930c764acf3 --- /dev/null +++ b/crates/common/src/format.rs @@ -0,0 +1,1732 @@ +// spell-checker:ignore ddfe +use core::ops::Deref; +use core::{cmp, str::FromStr}; +use itertools::{Itertools, PeekingNext}; +use malachite_base::num::basic::floats::PrimitiveFloat; +use malachite_bigint::{BigInt, Sign}; +use num_complex::Complex64; +use num_traits::FromPrimitive; +use num_traits::{Signed, cast::ToPrimitive}; +use rustpython_literal::float; +use rustpython_literal::format::Case; + +use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + +/// Locale information for 'n' format specifier. +/// Contains thousands separator, decimal point, and grouping pattern +/// from the C library's `localeconv()`. +#[derive(Clone, Debug)] +pub struct LocaleInfo { + pub thousands_sep: String, + pub decimal_point: String, + /// Grouping pattern from `lconv.grouping`. + /// Each element is a group size. The last non-zero element repeats. + /// e.g. `[3, 0]` means groups of 3 repeating forever. + pub grouping: Vec<u8>, +} + +trait FormatParse { + fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) + where + Self: Sized; +} + +#[derive(Debug, Copy, Clone, PartialEq)] +#[repr(u8)] +pub enum FormatConversion { + Str = b's', + Repr = b'r', + Ascii = b'b', + Bytes = b'a', +} + +impl FormatParse for FormatConversion { + fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) { + let Some(conversion) = Self::from_string(text) else { + return (None, text); + }; + let mut chars = text.code_points(); + chars.next(); // Consume the bang + chars.next(); // Consume one r,s,a char + (Some(conversion), chars.as_wtf8()) + } +} + +impl FormatConversion { + pub fn from_char(c: CodePoint) -> Option<Self> { + match c.to_char_lossy() { + 's' => Some(Self::Str), + 'r' => Some(Self::Repr), + 'a' => Some(Self::Ascii), + 'b' => Some(Self::Bytes), + _ => None, + } + } + + fn from_string(text: &Wtf8) -> Option<Self> { + let mut chars = text.code_points(); + if chars.next()? != '!' { + return None; + } + + Self::from_char(chars.next()?) + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatAlign { + Left, + Right, + AfterSign, + Center, +} + +impl FormatAlign { + fn from_char(c: CodePoint) -> Option<Self> { + match c.to_char_lossy() { + '<' => Some(Self::Left), + '>' => Some(Self::Right), + '=' => Some(Self::AfterSign), + '^' => Some(Self::Center), + _ => None, + } + } +} + +impl FormatParse for FormatAlign { + fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) { + let mut chars = text.code_points(); + if let Some(maybe_align) = chars.next().and_then(Self::from_char) { + (Some(maybe_align), chars.as_wtf8()) + } else { + (None, text) + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatSign { + Plus, + Minus, + MinusOrSpace, +} + +impl FormatParse for FormatSign { + fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('-') => (Some(Self::Minus), chars.as_wtf8()), + Some('+') => (Some(Self::Plus), chars.as_wtf8()), + Some(' ') => (Some(Self::MinusOrSpace), chars.as_wtf8()), + _ => (None, text), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FormatGrouping { + Comma, + Underscore, +} + +impl FormatParse for FormatGrouping { + fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('_') => (Some(Self::Underscore), chars.as_wtf8()), + Some(',') => (Some(Self::Comma), chars.as_wtf8()), + _ => (None, text), + } + } +} + +impl From<&FormatGrouping> for char { + fn from(fg: &FormatGrouping) -> Self { + match fg { + FormatGrouping::Comma => ',', + FormatGrouping::Underscore => '_', + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FormatType { + String, + Binary, + Character, + Decimal, + Octal, + Number(Case), + Hex(Case), + Exponent(Case), + GeneralFormat(Case), + FixedPoint(Case), + Percentage, + Unknown(char), +} + +impl From<&FormatType> for char { + fn from(from: &FormatType) -> Self { + match from { + FormatType::String => 's', + FormatType::Binary => 'b', + FormatType::Character => 'c', + FormatType::Decimal => 'd', + FormatType::Octal => 'o', + FormatType::Number(Case::Lower) => 'n', + FormatType::Number(Case::Upper) => 'N', + FormatType::Hex(Case::Lower) => 'x', + FormatType::Hex(Case::Upper) => 'X', + FormatType::Exponent(Case::Lower) => 'e', + FormatType::Exponent(Case::Upper) => 'E', + FormatType::GeneralFormat(Case::Lower) => 'g', + FormatType::GeneralFormat(Case::Upper) => 'G', + FormatType::FixedPoint(Case::Lower) => 'f', + FormatType::FixedPoint(Case::Upper) => 'F', + FormatType::Percentage => '%', + FormatType::Unknown(c) => *c, + } + } +} + +impl FormatParse for FormatType { + fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('s') => (Some(Self::String), chars.as_wtf8()), + Some('b') => (Some(Self::Binary), chars.as_wtf8()), + Some('c') => (Some(Self::Character), chars.as_wtf8()), + Some('d') => (Some(Self::Decimal), chars.as_wtf8()), + Some('o') => (Some(Self::Octal), chars.as_wtf8()), + Some('n') => (Some(Self::Number(Case::Lower)), chars.as_wtf8()), + Some('N') => (Some(Self::Number(Case::Upper)), chars.as_wtf8()), + Some('x') => (Some(Self::Hex(Case::Lower)), chars.as_wtf8()), + Some('X') => (Some(Self::Hex(Case::Upper)), chars.as_wtf8()), + Some('e') => (Some(Self::Exponent(Case::Lower)), chars.as_wtf8()), + Some('E') => (Some(Self::Exponent(Case::Upper)), chars.as_wtf8()), + Some('f') => (Some(Self::FixedPoint(Case::Lower)), chars.as_wtf8()), + Some('F') => (Some(Self::FixedPoint(Case::Upper)), chars.as_wtf8()), + Some('g') => (Some(Self::GeneralFormat(Case::Lower)), chars.as_wtf8()), + Some('G') => (Some(Self::GeneralFormat(Case::Upper)), chars.as_wtf8()), + Some('%') => (Some(Self::Percentage), chars.as_wtf8()), + Some(c) => (Some(Self::Unknown(c)), chars.as_wtf8()), + _ => (None, text), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FormatSpec { + conversion: Option<FormatConversion>, + fill: Option<CodePoint>, + align: Option<FormatAlign>, + sign: Option<FormatSign>, + alternate_form: bool, + width: Option<usize>, + grouping_option: Option<FormatGrouping>, + precision: Option<usize>, + format_type: Option<FormatType>, +} + +fn get_num_digits(text: &Wtf8) -> usize { + for (index, character) in text.code_point_indices() { + if !character.is_char_and(|c| c.is_ascii_digit()) { + return index; + } + } + text.len() +} + +fn parse_fill_and_align(text: &Wtf8) -> (Option<CodePoint>, Option<FormatAlign>, &Wtf8) { + let char_indices: Vec<(usize, CodePoint)> = text.code_point_indices().take(3).collect(); + if char_indices.is_empty() { + (None, None, text) + } else if char_indices.len() == 1 { + let (maybe_align, remaining) = FormatAlign::parse(text); + (None, maybe_align, remaining) + } else { + let (maybe_align, remaining) = FormatAlign::parse(&text[char_indices[1].0..]); + if maybe_align.is_some() { + (Some(char_indices[0].1), maybe_align, remaining) + } else { + let (only_align, only_align_remaining) = FormatAlign::parse(text); + (None, only_align, only_align_remaining) + } + } +} + +fn parse_number(text: &Wtf8) -> Result<(Option<usize>, &Wtf8), FormatSpecError> { + let num_digits: usize = get_num_digits(text); + if num_digits == 0 { + return Ok((None, text)); + } + if let Some(num) = parse_usize(&text[..num_digits]) { + Ok((Some(num), &text[num_digits..])) + } else { + // NOTE: this condition is different from CPython + Err(FormatSpecError::DecimalDigitsTooMany) + } +} + +fn parse_alternate_form(text: &Wtf8) -> (bool, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('#') => (true, chars.as_wtf8()), + _ => (false, text), + } +} + +fn parse_zero(text: &Wtf8) -> (bool, &Wtf8) { + let mut chars = text.code_points(); + match chars.next().and_then(CodePoint::to_char) { + Some('0') => (true, chars.as_wtf8()), + _ => (false, text), + } +} + +fn parse_precision(text: &Wtf8) -> Result<(Option<usize>, &Wtf8), FormatSpecError> { + let mut chars = text.code_points(); + Ok(match chars.next().and_then(CodePoint::to_char) { + Some('.') => { + let (size, remaining) = parse_number(chars.as_wtf8())?; + if let Some(size) = size { + if size > i32::MAX as usize { + return Err(FormatSpecError::PrecisionTooBig); + } + (Some(size), remaining) + } else { + (None, text) + } + } + _ => (None, text), + }) +} + +impl FormatSpec { + pub fn parse(text: impl AsRef<Wtf8>) -> Result<Self, FormatSpecError> { + Self::_parse(text.as_ref()) + } + + fn _parse(text: &Wtf8) -> Result<Self, FormatSpecError> { + // get_integer in CPython + let (conversion, text) = FormatConversion::parse(text); + let (mut fill, mut align, text) = parse_fill_and_align(text); + let (sign, text) = FormatSign::parse(text); + let (alternate_form, text) = parse_alternate_form(text); + let (zero, text) = parse_zero(text); + let (width, text) = parse_number(text)?; + let (grouping_option, text) = FormatGrouping::parse(text); + if let Some(grouping) = &grouping_option { + Self::validate_separator(grouping, text)?; + } + let (precision, text) = parse_precision(text)?; + let (format_type, text) = FormatType::parse(text); + if !text.is_empty() { + return Err(FormatSpecError::InvalidFormatSpecifier); + } + + if zero && fill.is_none() { + fill.replace('0'.into()); + align = align.or(Some(FormatAlign::AfterSign)); + } + + Ok(Self { + conversion, + fill, + align, + sign, + alternate_form, + width, + grouping_option, + precision, + format_type, + }) + } + + fn validate_separator(grouping: &FormatGrouping, text: &Wtf8) -> Result<(), FormatSpecError> { + let mut chars = text.code_points().peekable(); + match chars.peek().and_then(|cp| CodePoint::to_char(*cp)) { + Some(c) if c == ',' || c == '_' => { + if c == char::from(grouping) { + Err(FormatSpecError::UnspecifiedFormat(c, c)) + } else { + Err(FormatSpecError::ExclusiveFormat(',', '_')) + } + } + _ => Ok(()), + } + } + + fn compute_fill_string(fill_char: CodePoint, fill_chars_needed: i32) -> Wtf8Buf { + (0..fill_chars_needed).map(|_| fill_char).collect() + } + + fn add_magnitude_separators_for_char( + magnitude_str: String, + inter: i32, + sep: char, + disp_digit_cnt: i32, + ) -> String { + // Don't add separators to the floating decimal point of numbers + let mut parts = magnitude_str.splitn(2, '.'); + let magnitude_int_str = parts.next().unwrap().to_string(); + let dec_digit_cnt = magnitude_str.len() as i32 - magnitude_int_str.len() as i32; + let int_digit_cnt = disp_digit_cnt - dec_digit_cnt; + let mut result = Self::separate_integer(magnitude_int_str, inter, sep, int_digit_cnt); + if let Some(part) = parts.next() { + result.push_str(&format!(".{part}")) + } + result + } + + fn separate_integer( + magnitude_str: String, + inter: i32, + sep: char, + disp_digit_cnt: i32, + ) -> String { + let magnitude_len = magnitude_str.len() as i32; + let offset = (disp_digit_cnt % (inter + 1) == 0) as i32; + let disp_digit_cnt = disp_digit_cnt + offset; + let pad_cnt = disp_digit_cnt - magnitude_len; + let sep_cnt = disp_digit_cnt / (inter + 1); + let diff = pad_cnt - sep_cnt; + if pad_cnt > 0 && diff > 0 { + // separate with 0 padding + let padding = "0".repeat(diff as usize); + let padded_num = format!("{padding}{magnitude_str}"); + Self::insert_separator(padded_num, inter, sep, sep_cnt) + } else { + // separate without padding + let sep_cnt = (magnitude_len - 1) / inter; + Self::insert_separator(magnitude_str, inter, sep, sep_cnt) + } + } + + fn insert_separator(mut magnitude_str: String, inter: i32, sep: char, sep_cnt: i32) -> String { + let magnitude_len = magnitude_str.len() as i32; + for i in 1..=sep_cnt { + magnitude_str.insert((magnitude_len - inter * i) as usize, sep); + } + magnitude_str + } + + fn validate_format(&self, default_format_type: FormatType) -> Result<(), FormatSpecError> { + let format_type = self.format_type.as_ref().unwrap_or(&default_format_type); + match (&self.grouping_option, format_type) { + ( + Some(FormatGrouping::Comma), + FormatType::String + | FormatType::Character + | FormatType::Binary + | FormatType::Octal + | FormatType::Hex(_) + | FormatType::Number(_), + ) => { + let ch = char::from(format_type); + Err(FormatSpecError::UnspecifiedFormat(',', ch)) + } + ( + Some(FormatGrouping::Underscore), + FormatType::String | FormatType::Character | FormatType::Number(_), + ) => { + let ch = char::from(format_type); + Err(FormatSpecError::UnspecifiedFormat('_', ch)) + } + _ => Ok(()), + } + } + + const fn get_separator_interval(&self) -> usize { + match self.format_type { + Some(FormatType::Binary | FormatType::Octal | FormatType::Hex(_)) => 4, + Some( + FormatType::Decimal + | FormatType::FixedPoint(_) + | FormatType::GeneralFormat(_) + | FormatType::Exponent(_) + | FormatType::Percentage + | FormatType::Number(_), + ) => 3, + None => 3, + _ => panic!("Separators only valid for numbers!"), + } + } + + fn add_magnitude_separators(&self, magnitude_str: String, prefix: &str) -> String { + match &self.grouping_option { + Some(fg) => { + let sep = char::from(fg); + let inter = self.get_separator_interval().try_into().unwrap(); + let magnitude_len = magnitude_str.len(); + let disp_digit_cnt = if self.fill == Some('0'.into()) + && self.align == Some(FormatAlign::AfterSign) + { + let width = self.width.unwrap_or(magnitude_len) as i32 - prefix.len() as i32; + cmp::max(width, magnitude_len as i32) + } else { + magnitude_len as i32 + }; + Self::add_magnitude_separators_for_char(magnitude_str, inter, sep, disp_digit_cnt) + } + None => magnitude_str, + } + } + + /// Returns true if this format spec uses the locale-aware 'n' format type. + pub fn has_locale_format(&self) -> bool { + matches!(self.format_type, Some(FormatType::Number(Case::Lower))) + } + + /// Insert locale-aware thousands separators into an integer string. + /// Follows CPython's GroupGenerator logic for variable-width grouping. + fn insert_locale_grouping(int_part: &str, locale: &LocaleInfo) -> String { + if locale.grouping.is_empty() || locale.thousands_sep.is_empty() || int_part.len() <= 1 { + return int_part.to_string(); + } + + let mut group_idx = 0; + let mut group_size = locale.grouping[0] as usize; + + if group_size == 0 { + return int_part.to_string(); + } + + // Collect groups of digits from right to left + let len = int_part.len(); + let mut groups: Vec<&str> = Vec::new(); + let mut pos = len; + + loop { + if pos <= group_size { + groups.push(&int_part[..pos]); + break; + } + + groups.push(&int_part[pos - group_size..pos]); + pos -= group_size; + + // Advance to next group size + if group_idx + 1 < locale.grouping.len() { + let next = locale.grouping[group_idx + 1] as usize; + if next != 0 { + group_size = next; + group_idx += 1; + } + // 0 means repeat previous group size forever + } + } + + // Groups were collected right-to-left, reverse to get left-to-right + groups.reverse(); + groups.join(&locale.thousands_sep) + } + + /// Apply locale-aware grouping and decimal point replacement to a formatted number. + fn apply_locale_formatting(magnitude_str: String, locale: &LocaleInfo) -> String { + let mut parts = magnitude_str.splitn(2, '.'); + let int_part = parts.next().unwrap(); + let grouped = Self::insert_locale_grouping(int_part, locale); + + if let Some(frac_part) = parts.next() { + format!("{grouped}{}{frac_part}", locale.decimal_point) + } else { + grouped + } + } + + /// Format an integer with locale-aware 'n' format. + pub fn format_int_locale( + &self, + num: &BigInt, + locale: &LocaleInfo, + ) -> Result<String, FormatSpecError> { + self.validate_format(FormatType::Decimal)?; + let magnitude = num.abs(); + + let raw_magnitude_str = match self.format_type { + Some(FormatType::Number(Case::Lower)) => self.format_int_radix(magnitude, 10), + _ => return self.format_int(num), + }?; + + let magnitude_str = Self::apply_locale_formatting(raw_magnitude_str, locale); + + let format_sign = self.sign.unwrap_or(FormatSign::Minus); + let sign_str = match num.sign() { + Sign::Minus => "-", + _ => match format_sign { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + }, + }; + + self.format_sign_and_align(&AsciiStr::new(&magnitude_str), sign_str, FormatAlign::Right) + } + + /// Format a float with locale-aware 'n' format. + pub fn format_float_locale( + &self, + num: f64, + locale: &LocaleInfo, + ) -> Result<String, FormatSpecError> { + self.validate_format(FormatType::FixedPoint(Case::Lower))?; + let precision = self.precision.unwrap_or(6); + let magnitude = num.abs(); + + let raw_magnitude_str = match &self.format_type { + Some(FormatType::Number(case)) => { + let precision = if precision == 0 { 1 } else { precision }; + Ok(float::format_general( + precision, + magnitude, + *case, + self.alternate_form, + false, + )) + } + _ => return self.format_float(num), + }?; + + let magnitude_str = Self::apply_locale_formatting(raw_magnitude_str, locale); + + let format_sign = self.sign.unwrap_or(FormatSign::Minus); + let sign_str = if num.is_sign_negative() && !num.is_nan() { + "-" + } else { + match format_sign { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + } + }; + + self.format_sign_and_align(&AsciiStr::new(&magnitude_str), sign_str, FormatAlign::Right) + } + + /// Format a complex number with locale-aware 'n' format. + pub fn format_complex_locale( + &self, + num: &Complex64, + locale: &LocaleInfo, + ) -> Result<String, FormatSpecError> { + // Reuse format_complex_re_im with 'g' type to get the base formatted parts, + // then apply locale grouping. This matches CPython's format_complex_internal: + // 'n' → 'g', add_parens=0, skip_re=0. + let locale_spec = FormatSpec { + format_type: Some(FormatType::GeneralFormat(Case::Lower)), + ..*self + }; + let (formatted_re, formatted_im) = locale_spec.format_complex_re_im(num)?; + + // Apply locale grouping to both parts + let grouped_re = if formatted_re.is_empty() { + formatted_re + } else { + // Split sign from magnitude, apply grouping, recombine + let (sign, mag) = if formatted_re.starts_with('-') + || formatted_re.starts_with('+') + || formatted_re.starts_with(' ') + { + formatted_re.split_at(1) + } else { + ("", formatted_re.as_str()) + }; + format!( + "{sign}{}", + Self::apply_locale_formatting(mag.to_string(), locale) + ) + }; + + // formatted_im is like "+1234j" or "-1234j" or "1234j" + // Split sign, magnitude, and 'j' suffix + let im_str = &formatted_im; + let (im_sign, im_rest) = if im_str.starts_with('+') || im_str.starts_with('-') { + im_str.split_at(1) + } else { + ("", im_str.as_str()) + }; + let im_mag = im_rest.strip_suffix('j').unwrap_or(im_rest); + let im_grouped = Self::apply_locale_formatting(im_mag.to_string(), locale); + let grouped_im = format!("{im_sign}{im_grouped}j"); + + // No parentheses for 'n' format (CPython: add_parens=0) + let magnitude_str = format!("{grouped_re}{grouped_im}"); + + self.format_sign_and_align(&AsciiStr::new(&magnitude_str), "", FormatAlign::Right) + } + + pub fn format_bool(&self, input: bool) -> Result<String, FormatSpecError> { + let x = u8::from(input); + match &self.format_type { + Some( + FormatType::Binary + | FormatType::Decimal + | FormatType::Octal + | FormatType::Number(Case::Lower) + | FormatType::Hex(_) + | FormatType::GeneralFormat(_) + | FormatType::Character, + ) => self.format_int(&BigInt::from_u8(x).unwrap()), + Some(FormatType::Exponent(_) | FormatType::FixedPoint(_) | FormatType::Percentage) => { + self.format_float(x as f64) + } + None => { + let first_letter = (input.to_string().as_bytes()[0] as char).to_uppercase(); + Ok(first_letter.collect::<String>() + &input.to_string()[1..]) + } + Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(*c, "int")), + _ => Err(FormatSpecError::InvalidFormatSpecifier), + } + } + + pub fn format_float(&self, num: f64) -> Result<String, FormatSpecError> { + self.validate_format(FormatType::FixedPoint(Case::Lower))?; + let precision = self.precision.unwrap_or(6); + let magnitude = num.abs(); + let raw_magnitude_str: Result<String, FormatSpecError> = match &self.format_type { + Some(FormatType::FixedPoint(case)) => Ok(float::format_fixed( + precision, + magnitude, + *case, + self.alternate_form, + )), + Some(FormatType::Decimal) + | Some(FormatType::Binary) + | Some(FormatType::Octal) + | Some(FormatType::Hex(_)) + | Some(FormatType::String) + | Some(FormatType::Character) + | Some(FormatType::Number(Case::Upper)) + | Some(FormatType::Unknown(_)) => { + let ch = char::from(self.format_type.as_ref().unwrap()); + Err(FormatSpecError::UnknownFormatCode(ch, "float")) + } + Some(FormatType::GeneralFormat(case)) | Some(FormatType::Number(case)) => { + let precision = if precision == 0 { 1 } else { precision }; + Ok(float::format_general( + precision, + magnitude, + *case, + self.alternate_form, + false, + )) + } + Some(FormatType::Exponent(case)) => Ok(float::format_exponent( + precision, + magnitude, + *case, + self.alternate_form, + )), + Some(FormatType::Percentage) => match magnitude { + magnitude if magnitude.is_nan() => Ok("nan%".to_owned()), + magnitude if magnitude.is_infinite() => Ok("inf%".to_owned()), + _ => { + let result = format!("{:.*}", precision, magnitude * 100.0); + let point = float::decimal_point_or_empty(precision, self.alternate_form); + Ok(format!("{result}{point}%")) + } + }, + None => match magnitude { + magnitude if magnitude.is_nan() => Ok("nan".to_owned()), + magnitude if magnitude.is_infinite() => Ok("inf".to_owned()), + _ => match self.precision { + Some(precision) => Ok(float::format_general( + precision, + magnitude, + Case::Lower, + self.alternate_form, + true, + )), + None => Ok(float::to_string(magnitude)), + }, + }, + }; + let format_sign = self.sign.unwrap_or(FormatSign::Minus); + let sign_str = if num.is_sign_negative() && !num.is_nan() { + "-" + } else { + match format_sign { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + } + }; + let magnitude_str = self.add_magnitude_separators(raw_magnitude_str?, sign_str); + self.format_sign_and_align(&AsciiStr::new(&magnitude_str), sign_str, FormatAlign::Right) + } + + #[inline] + fn format_int_radix(&self, magnitude: BigInt, radix: u32) -> Result<String, FormatSpecError> { + match self.precision { + Some(_) => Err(FormatSpecError::PrecisionNotAllowed), + None => Ok(magnitude.to_str_radix(radix)), + } + } + + pub fn format_int(&self, num: &BigInt) -> Result<String, FormatSpecError> { + self.validate_format(FormatType::Decimal)?; + let magnitude = num.abs(); + let prefix = if self.alternate_form { + match self.format_type { + Some(FormatType::Binary) => "0b", + Some(FormatType::Octal) => "0o", + Some(FormatType::Hex(Case::Lower)) => "0x", + Some(FormatType::Hex(Case::Upper)) => "0X", + _ => "", + } + } else { + "" + }; + let raw_magnitude_str = match self.format_type { + Some(FormatType::Binary) => self.format_int_radix(magnitude, 2), + Some(FormatType::Decimal) => self.format_int_radix(magnitude, 10), + Some(FormatType::Octal) => self.format_int_radix(magnitude, 8), + Some(FormatType::Hex(Case::Lower)) => self.format_int_radix(magnitude, 16), + Some(FormatType::Hex(Case::Upper)) => match self.precision { + Some(_) => Err(FormatSpecError::PrecisionNotAllowed), + None => { + let mut result = magnitude.to_str_radix(16); + result.make_ascii_uppercase(); + Ok(result) + } + }, + Some(FormatType::Number(Case::Lower)) => self.format_int_radix(magnitude, 10), + Some(FormatType::Number(Case::Upper)) => { + Err(FormatSpecError::UnknownFormatCode('N', "int")) + } + Some(FormatType::String) => Err(FormatSpecError::UnknownFormatCode('s', "int")), + Some(FormatType::Character) => match (self.sign, self.alternate_form) { + (Some(_), _) => Err(FormatSpecError::NotAllowed("Sign")), + (_, true) => Err(FormatSpecError::NotAllowed("Alternate form (#)")), + (_, _) => match num.to_u32() { + Some(n) if n <= 0x10ffff => Ok(core::char::from_u32(n).unwrap().to_string()), + Some(_) | None => Err(FormatSpecError::CodeNotInRange), + }, + }, + Some(FormatType::GeneralFormat(_)) + | Some(FormatType::FixedPoint(_)) + | Some(FormatType::Exponent(_)) + | Some(FormatType::Percentage) => match num.to_f64() { + Some(float) => return self.format_float(float), + _ => Err(FormatSpecError::UnableToConvert), + }, + Some(FormatType::Unknown(c)) => Err(FormatSpecError::UnknownFormatCode(c, "int")), + None => self.format_int_radix(magnitude, 10), + }?; + let format_sign = self.sign.unwrap_or(FormatSign::Minus); + let sign_str = match num.sign() { + Sign::Minus => "-", + _ => match format_sign { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + }, + }; + let sign_prefix = format!("{sign_str}{prefix}"); + let magnitude_str = self.add_magnitude_separators(raw_magnitude_str, &sign_prefix); + self.format_sign_and_align( + &AsciiStr::new(&magnitude_str), + &sign_prefix, + FormatAlign::Right, + ) + } + + pub fn format_string<T>(&self, s: &T) -> Result<String, FormatSpecError> + where + T: CharLen + Deref<Target = str>, + { + self.validate_format(FormatType::String)?; + match self.format_type { + Some(FormatType::String) | None => self + .format_sign_and_align(s, "", FormatAlign::Left) + .map(|mut value| { + if let Some(precision) = self.precision { + value.truncate(precision); + } + value + }), + _ => { + let ch = char::from(self.format_type.as_ref().unwrap()); + Err(FormatSpecError::UnknownFormatCode(ch, "str")) + } + } + } + + pub fn format_complex(&self, num: &Complex64) -> Result<String, FormatSpecError> { + let (formatted_re, formatted_im) = self.format_complex_re_im(num)?; + // Enclose in parentheses if there is no format type and formatted_re is not empty + let magnitude_str = if self.format_type.is_none() && !formatted_re.is_empty() { + format!("({formatted_re}{formatted_im})") + } else { + format!("{formatted_re}{formatted_im}") + }; + if let Some(FormatAlign::AfterSign) = &self.align { + return Err(FormatSpecError::AlignmentFlag); + } + match &self.fill.unwrap_or(' '.into()).to_char() { + Some('0') => Err(FormatSpecError::ZeroPadding), + _ => self.format_sign_and_align(&AsciiStr::new(&magnitude_str), "", FormatAlign::Right), + } + } + + fn format_complex_re_im(&self, num: &Complex64) -> Result<(String, String), FormatSpecError> { + // Format real part + let mut formatted_re = String::new(); + if num.re != 0.0 || num.re.is_negative_zero() || self.format_type.is_some() { + let sign_re = if num.re.is_sign_negative() && !num.is_nan() { + "-" + } else { + match self.sign.unwrap_or(FormatSign::Minus) { + FormatSign::Plus => "+", + FormatSign::Minus => "", + FormatSign::MinusOrSpace => " ", + } + }; + let re = self.format_complex_float(num.re)?; + formatted_re = format!("{sign_re}{re}"); + } + // Format imaginary part + let sign_im = if num.im.is_sign_negative() && !num.im.is_nan() { + "-" + } else if formatted_re.is_empty() { + "" + } else { + "+" + }; + let im = self.format_complex_float(num.im)?; + Ok((formatted_re, format!("{sign_im}{im}j"))) + } + + fn format_complex_float(&self, num: f64) -> Result<String, FormatSpecError> { + self.validate_format(FormatType::FixedPoint(Case::Lower))?; + let precision = self.precision.unwrap_or(6); + let magnitude = num.abs(); + let magnitude_str = match &self.format_type { + Some(FormatType::Decimal) + | Some(FormatType::Binary) + | Some(FormatType::Octal) + | Some(FormatType::Hex(_)) + | Some(FormatType::String) + | Some(FormatType::Character) + | Some(FormatType::Number(Case::Upper)) + | Some(FormatType::Percentage) + | Some(FormatType::Unknown(_)) => { + let ch = char::from(self.format_type.as_ref().unwrap()); + Err(FormatSpecError::UnknownFormatCode(ch, "complex")) + } + Some(FormatType::FixedPoint(case)) => Ok(float::format_fixed( + precision, + magnitude, + *case, + self.alternate_form, + )), + Some(FormatType::GeneralFormat(case)) | Some(FormatType::Number(case)) => { + let precision = if precision == 0 { 1 } else { precision }; + Ok(float::format_general( + precision, + magnitude, + *case, + self.alternate_form, + false, + )) + } + Some(FormatType::Exponent(case)) => Ok(float::format_exponent( + precision, + magnitude, + *case, + self.alternate_form, + )), + None => match magnitude { + magnitude if magnitude.is_nan() => Ok("nan".to_owned()), + magnitude if magnitude.is_infinite() => Ok("inf".to_owned()), + _ => match self.precision { + Some(precision) => Ok(float::format_general( + precision, + magnitude, + Case::Lower, + self.alternate_form, + true, + )), + None => { + if magnitude.fract() == 0.0 { + Ok(magnitude.trunc().to_string()) + } else { + Ok(magnitude.to_string()) + } + } + }, + }, + }?; + match &self.grouping_option { + Some(fg) => { + let sep = char::from(fg); + let inter = self.get_separator_interval().try_into().unwrap(); + let len = magnitude_str.len() as i32; + let separated_magnitude = + Self::add_magnitude_separators_for_char(magnitude_str, inter, sep, len); + Ok(separated_magnitude) + } + None => Ok(magnitude_str), + } + } + + fn format_sign_and_align<T>( + &self, + magnitude_str: &T, + sign_str: &str, + default_align: FormatAlign, + ) -> Result<String, FormatSpecError> + where + T: CharLen + Deref<Target = str>, + { + let align = self.align.unwrap_or(default_align); + + let num_chars = magnitude_str.char_len(); + let fill_char = self.fill.unwrap_or(' '.into()); + let fill_chars_needed: i32 = self.width.map_or(0, |w| { + cmp::max(0, (w as i32) - (num_chars as i32) - (sign_str.len() as i32)) + }); + + let magnitude_str = magnitude_str.deref(); + Ok(match align { + FormatAlign::Left => format!( + "{}{}{}", + sign_str, + magnitude_str, + Self::compute_fill_string(fill_char, fill_chars_needed) + ), + FormatAlign::Right => format!( + "{}{}{}", + Self::compute_fill_string(fill_char, fill_chars_needed), + sign_str, + magnitude_str + ), + FormatAlign::AfterSign => format!( + "{}{}{}", + sign_str, + Self::compute_fill_string(fill_char, fill_chars_needed), + magnitude_str + ), + FormatAlign::Center => { + let left_fill_chars_needed = fill_chars_needed / 2; + let right_fill_chars_needed = fill_chars_needed - left_fill_chars_needed; + let left_fill_string = Self::compute_fill_string(fill_char, left_fill_chars_needed); + let right_fill_string = + Self::compute_fill_string(fill_char, right_fill_chars_needed); + format!("{left_fill_string}{sign_str}{magnitude_str}{right_fill_string}") + } + }) + } +} + +pub trait CharLen { + /// Returns the number of characters in the text + fn char_len(&self) -> usize; +} + +struct AsciiStr<'a> { + inner: &'a str, +} + +impl<'a> AsciiStr<'a> { + const fn new(inner: &'a str) -> Self { + Self { inner } + } +} + +impl CharLen for AsciiStr<'_> { + fn char_len(&self) -> usize { + self.inner.len() + } +} + +impl Deref for AsciiStr<'_> { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.inner + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FormatSpecError { + DecimalDigitsTooMany, + PrecisionTooBig, + InvalidFormatSpecifier, + UnspecifiedFormat(char, char), + ExclusiveFormat(char, char), + UnknownFormatCode(char, &'static str), + PrecisionNotAllowed, + NotAllowed(&'static str), + UnableToConvert, + CodeNotInRange, + ZeroPadding, + AlignmentFlag, + NotImplemented(char, &'static str), +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FormatParseError { + UnmatchedBracket, + MissingStartBracket, + UnescapedStartBracketInLiteral, + InvalidFormatSpecifier, + UnknownConversion, + EmptyAttribute, + MissingRightBracket, + InvalidCharacterAfterRightBracket, +} + +impl FromStr for FormatSpec { + type Err = FormatSpecError; + fn from_str(s: &str) -> Result<Self, Self::Err> { + Self::parse(s) + } +} + +#[derive(Debug, PartialEq)] +pub enum FieldNamePart { + Attribute(Wtf8Buf), + Index(usize), + StringIndex(Wtf8Buf), +} + +impl FieldNamePart { + fn parse_part( + chars: &mut impl PeekingNext<Item = CodePoint>, + ) -> Result<Option<Self>, FormatParseError> { + chars + .next() + .map(|ch| match ch.to_char_lossy() { + '.' => { + let mut attribute = Wtf8Buf::new(); + for ch in chars.peeking_take_while(|ch| *ch != '.' && *ch != '[') { + attribute.push(ch); + } + if attribute.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else { + Ok(Self::Attribute(attribute)) + } + } + '[' => { + let mut index = Wtf8Buf::new(); + for ch in chars { + if ch == ']' { + return if index.is_empty() { + Err(FormatParseError::EmptyAttribute) + } else if let Some(index) = parse_usize(&index) { + Ok(Self::Index(index)) + } else { + Ok(Self::StringIndex(index)) + }; + } + index.push(ch); + } + Err(FormatParseError::MissingRightBracket) + } + _ => Err(FormatParseError::InvalidCharacterAfterRightBracket), + }) + .transpose() + } +} + +#[derive(Debug, PartialEq)] +pub enum FieldType { + Auto, + Index(usize), + Keyword(Wtf8Buf), +} + +#[derive(Debug, PartialEq)] +pub struct FieldName { + pub field_type: FieldType, + pub parts: Vec<FieldNamePart>, +} + +fn parse_usize(s: &Wtf8) -> Option<usize> { + s.as_str().ok().and_then(|s| s.parse().ok()) +} + +impl FieldName { + pub fn parse(text: &Wtf8) -> Result<Self, FormatParseError> { + let mut chars = text.code_points().peekable(); + let first: Wtf8Buf = chars + .peeking_take_while(|ch| *ch != '.' && *ch != '[') + .collect(); + + let field_type = if first.is_empty() { + FieldType::Auto + } else if let Some(index) = parse_usize(&first) { + FieldType::Index(index) + } else { + FieldType::Keyword(first) + }; + + let mut parts = Vec::new(); + while let Some(part) = FieldNamePart::parse_part(&mut chars)? { + parts.push(part) + } + + Ok(Self { field_type, parts }) + } +} + +#[derive(Debug, PartialEq)] +pub enum FormatPart { + Field { + field_name: Wtf8Buf, + conversion_spec: Option<CodePoint>, + format_spec: Wtf8Buf, + }, + Literal(Wtf8Buf), +} + +#[derive(Debug, PartialEq)] +pub struct FormatString { + pub format_parts: Vec<FormatPart>, +} + +impl FormatString { + fn parse_literal_single(text: &Wtf8) -> Result<(CodePoint, &Wtf8), FormatParseError> { + let mut chars = text.code_points(); + // This should never be called with an empty str + let first_char = chars.next().unwrap(); + // isn't this detectable only with bytes operation? + if first_char == '{' || first_char == '}' { + let maybe_next_char = chars.next(); + // if we see a bracket, it has to be escaped by doubling up to be in a literal + return if maybe_next_char.is_none() || maybe_next_char.unwrap() != first_char { + Err(FormatParseError::UnescapedStartBracketInLiteral) + } else { + Ok((first_char, chars.as_wtf8())) + }; + } + Ok((first_char, chars.as_wtf8())) + } + + fn parse_literal(text: &Wtf8) -> Result<(FormatPart, &Wtf8), FormatParseError> { + let mut cur_text = text; + let mut result_string = Wtf8Buf::new(); + while !cur_text.is_empty() { + match Self::parse_literal_single(cur_text) { + Ok((next_char, remaining)) => { + result_string.push(next_char); + cur_text = remaining; + } + Err(err) => { + return if !result_string.is_empty() { + Ok((FormatPart::Literal(result_string), cur_text)) + } else { + Err(err) + }; + } + } + } + Ok((FormatPart::Literal(result_string), "".as_ref())) + } + + fn parse_part_in_brackets(text: &Wtf8) -> Result<FormatPart, FormatParseError> { + let mut chars = text.code_points().peekable(); + + let mut left = Wtf8Buf::new(); + let mut right = Wtf8Buf::new(); + + let mut split = false; + let mut selected = &mut left; + let mut inside_brackets = false; + + while let Some(char) = chars.next() { + if char == '[' { + inside_brackets = true; + + selected.push(char); + + while let Some(next_char) = chars.next() { + selected.push(next_char); + + if next_char == ']' { + inside_brackets = false; + break; + } + if chars.peek().is_none() { + return Err(FormatParseError::MissingRightBracket); + } + } + } else if char == ':' && !split && !inside_brackets { + split = true; + selected = &mut right; + } else { + selected.push(char); + } + } + + // before the comma is a keyword or arg index, after the comma is maybe a spec. + let arg_part: &Wtf8 = &left; + + let format_spec = if split { right } else { Wtf8Buf::new() }; + + // left can still be the conversion (!r, !s, !a) + let parts: Vec<&Wtf8> = arg_part.splitn(2, "!".as_ref()).collect(); + // before the bang is a keyword or arg index, after the comma is maybe a conversion spec. + let arg_part = parts[0]; + + let conversion_spec = parts + .get(1) + .map(|conversion| { + // conversions are only every one character + conversion + .code_points() + .exactly_one() + .map_err(|_| FormatParseError::UnknownConversion) + }) + .transpose()?; + + Ok(FormatPart::Field { + field_name: arg_part.to_owned(), + conversion_spec, + format_spec, + }) + } + + fn parse_spec(text: &Wtf8) -> Result<(FormatPart, &Wtf8), FormatParseError> { + let mut nested = false; + let mut end_bracket_pos = None; + let mut left = Wtf8Buf::new(); + + // There may be one layer nesting brackets in spec + for (idx, c) in text.code_point_indices() { + if idx == 0 { + if c != '{' { + return Err(FormatParseError::MissingStartBracket); + } + } else if c == '{' { + if nested { + return Err(FormatParseError::InvalidFormatSpecifier); + } else { + nested = true; + left.push(c); + continue; + } + } else if c == '}' { + if nested { + nested = false; + left.push(c); + continue; + } else { + end_bracket_pos = Some(idx); + break; + } + } else { + left.push(c); + } + } + if let Some(pos) = end_bracket_pos { + let right = &text[pos..]; + let format_part = Self::parse_part_in_brackets(&left)?; + Ok((format_part, &right[1..])) + } else { + Err(FormatParseError::UnmatchedBracket) + } + } +} + +pub trait FromTemplate<'a>: Sized { + type Err; + fn from_str(s: &'a Wtf8) -> Result<Self, Self::Err>; +} + +impl<'a> FromTemplate<'a> for FormatString { + type Err = FormatParseError; + + fn from_str(text: &'a Wtf8) -> Result<Self, Self::Err> { + let mut cur_text: &Wtf8 = text; + let mut parts: Vec<FormatPart> = Vec::new(); + while !cur_text.is_empty() { + // Try to parse both literals and bracketed format parts until we + // run out of text + cur_text = Self::parse_literal(cur_text) + .or_else(|_| Self::parse_spec(cur_text)) + .map(|(part, new_text)| { + parts.push(part); + new_text + })?; + } + Ok(Self { + format_parts: parts, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fill_and_align() { + let parse_fill_and_align = |text| { + let (fill, align, rest) = parse_fill_and_align(str::as_ref(text)); + ( + fill.and_then(CodePoint::to_char), + align, + rest.as_str().unwrap(), + ) + }; + assert_eq!( + parse_fill_and_align(" <"), + (Some(' '), Some(FormatAlign::Left), "") + ); + assert_eq!( + parse_fill_and_align(" <22"), + (Some(' '), Some(FormatAlign::Left), "22") + ); + assert_eq!( + parse_fill_and_align("<22"), + (None, Some(FormatAlign::Left), "22") + ); + assert_eq!( + parse_fill_and_align(" ^^"), + (Some(' '), Some(FormatAlign::Center), "^") + ); + assert_eq!( + parse_fill_and_align("==="), + (Some('='), Some(FormatAlign::AfterSign), "=") + ); + } + + #[test] + fn test_width_only() { + let expected = Ok(FormatSpec { + conversion: None, + fill: None, + align: None, + sign: None, + alternate_form: false, + width: Some(33), + grouping_option: None, + precision: None, + format_type: None, + }); + assert_eq!(FormatSpec::parse("33"), expected); + } + + #[test] + fn test_fill_and_width() { + let expected = Ok(FormatSpec { + conversion: None, + fill: Some('<'.into()), + align: Some(FormatAlign::Right), + sign: None, + alternate_form: false, + width: Some(33), + grouping_option: None, + precision: None, + format_type: None, + }); + assert_eq!(FormatSpec::parse("<>33"), expected); + } + + #[test] + fn test_all() { + let expected = Ok(FormatSpec { + conversion: None, + fill: Some('<'.into()), + align: Some(FormatAlign::Right), + sign: Some(FormatSign::Minus), + alternate_form: true, + width: Some(23), + grouping_option: Some(FormatGrouping::Comma), + precision: Some(11), + format_type: Some(FormatType::Binary), + }); + assert_eq!(FormatSpec::parse("<>-#23,.11b"), expected); + } + + fn format_bool(text: &str, value: bool) -> Result<String, FormatSpecError> { + FormatSpec::parse(text).and_then(|spec| spec.format_bool(value)) + } + + #[test] + fn test_format_bool() { + assert_eq!(format_bool("b", true), Ok("1".to_owned())); + assert_eq!(format_bool("b", false), Ok("0".to_owned())); + assert_eq!(format_bool("d", true), Ok("1".to_owned())); + assert_eq!(format_bool("d", false), Ok("0".to_owned())); + assert_eq!(format_bool("o", true), Ok("1".to_owned())); + assert_eq!(format_bool("o", false), Ok("0".to_owned())); + assert_eq!(format_bool("n", true), Ok("1".to_owned())); + assert_eq!(format_bool("n", false), Ok("0".to_owned())); + assert_eq!(format_bool("x", true), Ok("1".to_owned())); + assert_eq!(format_bool("x", false), Ok("0".to_owned())); + assert_eq!(format_bool("X", true), Ok("1".to_owned())); + assert_eq!(format_bool("X", false), Ok("0".to_owned())); + assert_eq!(format_bool("g", true), Ok("1".to_owned())); + assert_eq!(format_bool("g", false), Ok("0".to_owned())); + assert_eq!(format_bool("G", true), Ok("1".to_owned())); + assert_eq!(format_bool("G", false), Ok("0".to_owned())); + assert_eq!(format_bool("c", true), Ok("\x01".to_owned())); + assert_eq!(format_bool("c", false), Ok("\x00".to_owned())); + assert_eq!(format_bool("e", true), Ok("1.000000e+00".to_owned())); + assert_eq!(format_bool("e", false), Ok("0.000000e+00".to_owned())); + assert_eq!(format_bool("E", true), Ok("1.000000E+00".to_owned())); + assert_eq!(format_bool("E", false), Ok("0.000000E+00".to_owned())); + assert_eq!(format_bool("f", true), Ok("1.000000".to_owned())); + assert_eq!(format_bool("f", false), Ok("0.000000".to_owned())); + assert_eq!(format_bool("F", true), Ok("1.000000".to_owned())); + assert_eq!(format_bool("F", false), Ok("0.000000".to_owned())); + assert_eq!(format_bool("%", true), Ok("100.000000%".to_owned())); + assert_eq!(format_bool("%", false), Ok("0.000000%".to_owned())); + } + + #[test] + fn test_format_int() { + assert_eq!( + FormatSpec::parse("d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("16".to_owned()) + ); + assert_eq!( + FormatSpec::parse("x") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("10".to_owned()) + ); + assert_eq!( + FormatSpec::parse("b") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("10000".to_owned()) + ); + assert_eq!( + FormatSpec::parse("o") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("20".to_owned()) + ); + assert_eq!( + FormatSpec::parse("+d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("+16".to_owned()) + ); + assert_eq!( + FormatSpec::parse("^ 5d") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Minus, b"\x10")), + Ok(" -16 ".to_owned()) + ); + assert_eq!( + FormatSpec::parse("0>+#10x") + .unwrap() + .format_int(&BigInt::from_bytes_be(Sign::Plus, b"\x10")), + Ok("00000+0x10".to_owned()) + ); + } + + #[test] + fn test_format_int_sep() { + let spec = FormatSpec::parse(",").expect(""); + assert_eq!(spec.grouping_option, Some(FormatGrouping::Comma)); + assert_eq!( + spec.format_int(&BigInt::from_str("1234567890123456789012345678").unwrap()), + Ok("1,234,567,890,123,456,789,012,345,678".to_owned()) + ); + } + + #[test] + fn test_format_int_width_and_grouping() { + // issue #5922: width + comma grouping should pad left, not inside the number + let spec = FormatSpec::parse("10,").unwrap(); + let result = spec.format_int(&BigInt::from(1234)).unwrap(); + assert_eq!(result, " 1,234"); // CPython 3.13.5 + } + + #[test] + fn test_format_int_padding_with_grouping() { + // CPython behavior: f'{1234:010,}' results in "00,001,234" + let spec1 = FormatSpec::parse("010,").unwrap(); + let result1 = spec1.format_int(&BigInt::from(1234)).unwrap(); + assert_eq!(result1, "00,001,234"); + + // CPython behavior: f'{-1234:010,}' results in "-0,001,234" + let spec2 = FormatSpec::parse("010,").unwrap(); + let result2 = spec2.format_int(&BigInt::from(-1234)).unwrap(); + assert_eq!(result2, "-0,001,234"); + + // CPython behavior: f'{-1234:=10,}' results in "- 1,234" + let spec3 = FormatSpec::parse("=10,").unwrap(); + let result3 = spec3.format_int(&BigInt::from(-1234)).unwrap(); + assert_eq!(result3, "- 1,234"); + + // CPython behavior: f'{1234:=10,}' results in " 1,234" (same as right-align for positive numbers) + let spec4 = FormatSpec::parse("=10,").unwrap(); + let result4 = spec4.format_int(&BigInt::from(1234)).unwrap(); + assert_eq!(result4, " 1,234"); + } + + #[test] + fn test_format_int_non_aftersign_zero_padding() { + // CPython behavior: f'{1234:0>10,}' results in "000001,234" + let spec = FormatSpec::parse("0>10,").unwrap(); + let result = spec.format_int(&BigInt::from(1234)).unwrap(); + assert_eq!(result, "000001,234"); + } + + #[test] + fn test_format_parse() { + let expected = Ok(FormatString { + format_parts: vec![ + FormatPart::Literal("abcd".into()), + FormatPart::Field { + field_name: "1".into(), + conversion_spec: None, + format_spec: "".into(), + }, + FormatPart::Literal(":".into()), + FormatPart::Field { + field_name: "key".into(), + conversion_spec: None, + format_spec: "".into(), + }, + ], + }); + + assert_eq!(FormatString::from_str("abcd{1}:{key}".as_ref()), expected); + } + + #[test] + fn test_format_parse_multi_byte_char() { + assert!(FormatString::from_str("{a:%ЫйЯЧ}".as_ref()).is_ok()); + } + + #[test] + fn test_format_parse_fail() { + assert_eq!( + FormatString::from_str("{s".as_ref()), + Err(FormatParseError::UnmatchedBracket) + ); + } + + #[test] + fn test_square_brackets_inside_format() { + assert_eq!( + FormatString::from_str("{[:123]}".as_ref()), + Ok(FormatString { + format_parts: vec![FormatPart::Field { + field_name: "[:123]".into(), + conversion_spec: None, + format_spec: "".into(), + }], + }), + ); + + assert_eq!(FormatString::from_str("{asdf[:123]asdf}".as_ref()), { + Ok(FormatString { + format_parts: vec![FormatPart::Field { + field_name: "asdf[:123]asdf".into(), + conversion_spec: None, + format_spec: "".into(), + }], + }) + }); + + assert_eq!(FormatString::from_str("{[1234}".as_ref()), { + Err(FormatParseError::MissingRightBracket) + }); + } + + #[test] + fn test_format_parse_escape() { + let expected = Ok(FormatString { + format_parts: vec![ + FormatPart::Literal("{".into()), + FormatPart::Field { + field_name: "key".into(), + conversion_spec: None, + format_spec: "".into(), + }, + FormatPart::Literal("}ddfe".into()), + ], + }); + + assert_eq!(FormatString::from_str("{{{key}}}ddfe".as_ref()), expected); + } + + #[test] + fn test_format_invalid_specification() { + assert_eq!( + FormatSpec::parse("%3"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse(".2fa"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("ds"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("x+"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("b4"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("o!"), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + assert_eq!( + FormatSpec::parse("d "), + Err(FormatSpecError::InvalidFormatSpecifier) + ); + } + + #[test] + fn test_parse_field_name() { + let parse = |s: &str| FieldName::parse(s.as_ref()); + assert_eq!( + parse(""), + Ok(FieldName { + field_type: FieldType::Auto, + parts: Vec::new(), + }) + ); + assert_eq!( + parse("0"), + Ok(FieldName { + field_type: FieldType::Index(0), + parts: Vec::new(), + }) + ); + assert_eq!( + parse("key"), + Ok(FieldName { + field_type: FieldType::Keyword("key".into()), + parts: Vec::new(), + }) + ); + assert_eq!( + parse("key.attr[0][string]"), + Ok(FieldName { + field_type: FieldType::Keyword("key".into()), + parts: vec![ + FieldNamePart::Attribute("attr".into()), + FieldNamePart::Index(0), + FieldNamePart::StringIndex("string".into()) + ], + }) + ); + assert_eq!(parse("key.."), Err(FormatParseError::EmptyAttribute)); + assert_eq!(parse("key[]"), Err(FormatParseError::EmptyAttribute)); + assert_eq!(parse("key["), Err(FormatParseError::MissingRightBracket)); + assert_eq!( + parse("key[0]after"), + Err(FormatParseError::InvalidCharacterAfterRightBracket) + ); + } +} diff --git a/crates/common/src/hash.rs b/crates/common/src/hash.rs new file mode 100644 index 00000000000..f8f3783d224 --- /dev/null +++ b/crates/common/src/hash.rs @@ -0,0 +1,187 @@ +use core::hash::{BuildHasher, Hash, Hasher}; +use malachite_bigint::BigInt; +use num_traits::ToPrimitive; +use siphasher::sip::SipHasher24; + +pub type PyHash = i64; +pub type PyUHash = u64; + +/// A PyHash value used to represent a missing hash value, e.g. means "not yet computed" for +/// `str`'s hash cache +pub const SENTINEL: PyHash = -1; + +/// Prime multiplier used in string and various other hashes. +pub const MULTIPLIER: PyHash = 1_000_003; // 0xf4243 +/// Numeric hashes are based on reduction modulo the prime 2**_BITS - 1 +pub const BITS: usize = 61; +pub const MODULUS: PyUHash = (1 << BITS) - 1; +pub const INF: PyHash = 314_159; +pub const NAN: PyHash = 0; +pub const IMAG: PyHash = MULTIPLIER; +pub const ALGO: &str = "siphash24"; +pub const HASH_BITS: usize = core::mem::size_of::<PyHash>() * 8; +// SipHasher24 takes 2 u64s as a seed +pub const SEED_BITS: usize = core::mem::size_of::<u64>() * 2 * 8; + +// pub const CUTOFF: usize = 7; + +#[derive(Clone, Copy)] +pub struct HashSecret { + k0: u64, + k1: u64, +} + +impl BuildHasher for HashSecret { + type Hasher = SipHasher24; + + fn build_hasher(&self) -> Self::Hasher { + SipHasher24::new_with_keys(self.k0, self.k1) + } +} + +impl HashSecret { + pub fn new(seed: u32) -> Self { + let mut buf = [0u8; 16]; + lcg_urandom(seed, &mut buf); + let (left, right) = buf.split_at(8); + let k0 = u64::from_le_bytes(left.try_into().unwrap()); + let k1 = u64::from_le_bytes(right.try_into().unwrap()); + Self { k0, k1 } + } +} + +impl HashSecret { + pub fn hash_value<T: Hash + ?Sized>(&self, data: &T) -> PyHash { + fix_sentinel(mod_int(self.hash_one(data) as _)) + } + + pub fn hash_iter<'a, T: 'a, I, F, E>(&self, iter: I, hash_func: F) -> Result<PyHash, E> + where + I: IntoIterator<Item = &'a T>, + F: Fn(&'a T) -> Result<PyHash, E>, + { + let mut hasher = self.build_hasher(); + for element in iter { + let item_hash = hash_func(element)?; + item_hash.hash(&mut hasher); + } + Ok(fix_sentinel(mod_int(hasher.finish() as PyHash))) + } + + pub fn hash_bytes(&self, value: &[u8]) -> PyHash { + if value.is_empty() { + 0 + } else { + self.hash_value(value) + } + } + + pub fn hash_str(&self, value: &str) -> PyHash { + self.hash_bytes(value.as_bytes()) + } +} + +#[inline] +pub const fn hash_pointer(value: usize) -> PyHash { + // TODO: 32bit? + let hash = (value >> 4) | value; + hash as _ +} + +#[inline] +pub fn hash_float(value: f64) -> Option<PyHash> { + // cpython _Py_HashDouble + if !value.is_finite() { + return if value.is_infinite() { + Some(if value > 0.0 { INF } else { -INF }) + } else { + None + }; + } + + let frexp = super::float_ops::decompose_float(value); + + // process 28 bits at a time; this should work well both for binary + // and hexadecimal floating point. + let mut m = frexp.0; + let mut e = frexp.1; + let mut x: PyUHash = 0; + while m != 0.0 { + x = ((x << 28) & MODULUS) | (x >> (BITS - 28)); + m *= 268_435_456.0; // 2**28 + e -= 28; + let y = m as PyUHash; // pull out integer part + m -= y as f64; + x += y; + if x >= MODULUS { + x -= MODULUS; + } + } + + // adjust for the exponent; first reduce it modulo BITS + const BITS32: i32 = BITS as i32; + e = if e >= 0 { + e % BITS32 + } else { + BITS32 - 1 - ((-1 - e) % BITS32) + }; + x = ((x << e) & MODULUS) | (x >> (BITS32 - e)); + + Some(fix_sentinel(x as PyHash * value.signum() as PyHash)) +} + +pub fn hash_bigint(value: &BigInt) -> PyHash { + let ret = match value.to_i64() { + Some(i) => mod_int(i), + None => (value % MODULUS).to_i64().unwrap_or_else(|| unsafe { + // SAFETY: MODULUS < i64::MAX, so value % MODULUS is guaranteed to be in the range of i64 + core::hint::unreachable_unchecked() + }), + }; + fix_sentinel(ret) +} + +#[inline] +pub const fn hash_usize(data: usize) -> PyHash { + fix_sentinel(mod_int(data as i64)) +} + +#[inline(always)] +pub const fn fix_sentinel(x: PyHash) -> PyHash { + if x == SENTINEL { -2 } else { x } +} + +#[inline] +pub const fn mod_int(value: i64) -> PyHash { + value % MODULUS as i64 +} + +pub fn lcg_urandom(mut x: u32, buf: &mut [u8]) { + for b in buf { + x = x.wrapping_mul(214013); + x = x.wrapping_add(2531011); + *b = ((x >> 16) & 0xff) as u8; + } +} + +#[inline] +pub const fn hash_object_id_raw(p: usize) -> PyHash { + // TODO: Use commented logic when below issue resolved. + // Ref: https://github.com/RustPython/RustPython/pull/3951#issuecomment-1193108966 + + /* bottom 3 or 4 bits are likely to be 0; rotate y by 4 to avoid + excessive hash collisions for dicts and sets */ + // p.rotate_right(4) as PyHash + p as PyHash +} + +#[inline] +pub const fn hash_object_id(p: usize) -> PyHash { + fix_sentinel(hash_object_id_raw(p)) +} + +pub fn keyed_hash(key: u64, buf: &[u8]) -> u64 { + let mut hasher = SipHasher24::new_with_keys(key, 0); + buf.hash(&mut hasher); + hasher.finish() +} diff --git a/crates/common/src/int.rs b/crates/common/src/int.rs new file mode 100644 index 00000000000..9cfe2e0d738 --- /dev/null +++ b/crates/common/src/int.rs @@ -0,0 +1,208 @@ +use malachite_base::{num::conversion::traits::RoundingInto, rounding_modes::RoundingMode}; +use malachite_bigint::{BigInt, BigUint, Sign}; +use malachite_q::Rational; +use num_traits::{One, ToPrimitive, Zero}; + +pub fn true_div(numerator: &BigInt, denominator: &BigInt) -> f64 { + let rational = Rational::from_integers_ref(numerator.into(), denominator.into()); + match rational.rounding_into(RoundingMode::Nearest) { + // returned value is $t::MAX but still less than the original + (val, core::cmp::Ordering::Less) if val == f64::MAX => f64::INFINITY, + // returned value is $t::MIN but still greater than the original + (val, core::cmp::Ordering::Greater) if val == f64::MIN => f64::NEG_INFINITY, + (val, _) => val, + } +} + +pub fn float_to_ratio(value: f64) -> Option<(BigInt, BigInt)> { + let sign = match core::cmp::PartialOrd::partial_cmp(&value, &0.0)? { + core::cmp::Ordering::Less => Sign::Minus, + core::cmp::Ordering::Equal => return Some((BigInt::zero(), BigInt::one())), + core::cmp::Ordering::Greater => Sign::Plus, + }; + Rational::try_from(value).ok().map(|x| { + let (numer, denom) = x.into_numerator_and_denominator(); + ( + BigInt::from_biguint(sign, numer.into()), + BigUint::from(denom).into(), + ) + }) +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum BytesToIntError { + InvalidLiteral { base: u32 }, + InvalidBase, + DigitLimit { got: usize, limit: usize }, +} + +// https://github.com/python/cpython/blob/4e665351082c50018fb31d80db25b4693057393e/Objects/longobject.c#L2977 +// https://github.com/python/cpython/blob/4e665351082c50018fb31d80db25b4693057393e/Objects/longobject.c#L2884 +pub fn bytes_to_int( + buf: &[u8], + mut base: u32, + digit_limit: usize, +) -> Result<BigInt, BytesToIntError> { + if base != 0 && !(2..=36).contains(&base) { + return Err(BytesToIntError::InvalidBase); + } + + let mut buf = buf.trim_ascii(); + + // split sign + let sign = match buf.first() { + Some(b'+') => Some(Sign::Plus), + Some(b'-') => Some(Sign::Minus), + None => return Err(BytesToIntError::InvalidLiteral { base }), + _ => None, + }; + + if sign.is_some() { + buf = &buf[1..]; + } + + let mut error_if_nonzero = false; + if base == 0 { + match (buf.first(), buf.get(1)) { + (Some(v), _) if *v != b'0' => base = 10, + (_, Some(b'x' | b'X')) => base = 16, + (_, Some(b'o' | b'O')) => base = 8, + (_, Some(b'b' | b'B')) => base = 2, + (_, _) => { + // "old" (C-style) octal literal, now invalid. it might still be zero though + base = 10; + error_if_nonzero = true; + } + } + } + + if error_if_nonzero { + if let [_first, others @ .., last] = buf { + let is_zero = *last == b'0' && others.iter().all(|&c| c == b'0' || c == b'_'); + if !is_zero { + return Err(BytesToIntError::InvalidLiteral { base }); + } + } + return Ok(BigInt::zero()); + } + + if buf.first().is_some_and(|&v| v == b'0') + && buf.get(1).is_some_and(|&v| { + (base == 16 && (v == b'x' || v == b'X')) + || (base == 8 && (v == b'o' || v == b'O')) + || (base == 2 && (v == b'b' || v == b'B')) + }) + { + buf = &buf[2..]; + + // One underscore allowed here + if buf.first().is_some_and(|&v| v == b'_') { + buf = &buf[1..]; + } + } + + // Reject empty strings + let mut prev = *buf + .first() + .ok_or(BytesToIntError::InvalidLiteral { base })?; + + // Leading underscore not allowed + if prev == b'_' || !prev.is_ascii_alphanumeric() { + return Err(BytesToIntError::InvalidLiteral { base }); + } + + // Verify all characters are digits and underscores + let mut digits = 1; + for &cur in buf.iter().skip(1) { + if cur == b'_' { + // Double underscore not allowed + if prev == b'_' { + return Err(BytesToIntError::InvalidLiteral { base }); + } + } else if cur.is_ascii_alphanumeric() { + digits += 1; + } else { + return Err(BytesToIntError::InvalidLiteral { base }); + } + + prev = cur; + } + + // Trailing underscore not allowed + if prev == b'_' { + return Err(BytesToIntError::InvalidLiteral { base }); + } + + if digit_limit > 0 && !base.is_power_of_two() && digits > digit_limit { + return Err(BytesToIntError::DigitLimit { + got: digits, + limit: digit_limit, + }); + } + + let uint = BigUint::parse_bytes(buf, base).ok_or(BytesToIntError::InvalidLiteral { base })?; + Ok(BigInt::from_biguint(sign.unwrap_or(Sign::Plus), uint)) +} + +// num-bigint now returns Some(inf) for to_f64() in some cases, so just keep that the same for now +#[inline(always)] +pub fn bigint_to_finite_float(int: &BigInt) -> Option<f64> { + int.to_f64().filter(|f| f.is_finite()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const DIGIT_LIMIT: usize = 4300; // Default of Cpython + + #[test] + fn bytes_to_int_valid() { + for ((buf, base), expected) in [ + (("0b101", 2), BigInt::from(5)), + (("0x_10", 16), BigInt::from(16)), + (("0b", 16), BigInt::from(11)), + (("+0b101", 2), BigInt::from(5)), + (("0_0_0", 10), BigInt::from(0)), + (("000", 0), BigInt::from(0)), + (("0_100", 10), BigInt::from(100)), + ] { + assert_eq!( + bytes_to_int(buf.as_bytes(), base, DIGIT_LIMIT), + Ok(expected) + ); + } + } + + #[test] + fn bytes_to_int_invalid_literal() { + for ((buf, base), expected) in [ + (("09_99", 0), BytesToIntError::InvalidLiteral { base: 10 }), + (("0_", 0), BytesToIntError::InvalidLiteral { base: 10 }), + (("0_", 2), BytesToIntError::InvalidLiteral { base: 2 }), + ] { + assert_eq!( + bytes_to_int(buf.as_bytes(), base, DIGIT_LIMIT), + Err(expected) + ) + } + } + + #[test] + fn bytes_to_int_invalid_base() { + for base in [1, 37] { + assert_eq!( + bytes_to_int("012345".as_bytes(), base, DIGIT_LIMIT), + Err(BytesToIntError::InvalidBase) + ) + } + } + + #[test] + fn bytes_to_int_digit_limit() { + assert_eq!( + bytes_to_int("012345".as_bytes(), 10, 5), + Err(BytesToIntError::DigitLimit { got: 6, limit: 5 }) + ); + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs new file mode 100644 index 00000000000..e514c17541f --- /dev/null +++ b/crates/common/src/lib.rs @@ -0,0 +1,40 @@ +//! A crate to hold types and functions common to all rustpython components. + +#![cfg_attr(not(feature = "std"), no_std)] + +extern crate alloc; + +#[macro_use] +mod macros; +pub use macros::*; + +pub mod atomic; +pub mod borrow; +pub mod boxvec; +pub mod cformat; +#[cfg(all(feature = "std", any(unix, windows, target_os = "wasi")))] +pub mod crt_fd; +pub mod encodings; +#[cfg(all(feature = "std", any(not(target_arch = "wasm32"), target_os = "wasi")))] +pub mod fileutils; +pub mod float_ops; +pub mod format; +pub mod hash; +pub mod int; +pub mod linked_list; +pub mod lock; +#[cfg(feature = "std")] +pub mod os; +pub mod rand; +pub mod rc; +pub mod refcount; +pub mod static_cell; +pub mod str; +#[cfg(all(feature = "std", windows))] +pub mod windows; + +pub use rustpython_wtf8 as wtf8; + +pub mod vendored { + pub use ascii; +} diff --git a/common/src/linked_list.rs b/crates/common/src/linked_list.rs similarity index 76% rename from common/src/linked_list.rs rename to crates/common/src/linked_list.rs index 3941d2c830f..223d9ce0eb1 100644 --- a/common/src/linked_list.rs +++ b/crates/common/src/linked_list.rs @@ -1,4 +1,6 @@ -//! This module is modified from tokio::util::linked_list: https://github.com/tokio-rs/tokio/blob/master/tokio/src/util/linked_list.rs +// spell-checker:disable + +//! This module is modified from tokio::util::linked_list: <https://github.com/tokio-rs/tokio/blob/master/tokio/src/util/linked_list.rs> //! Tokio is licensed under the MIT license: //! //! Copyright (c) 2021 Tokio Contributors @@ -138,8 +140,8 @@ unsafe impl<T: Sync> Sync for Pointers<T> {} impl<L, T> LinkedList<L, T> { /// Creates an empty linked list. - pub const fn new() -> LinkedList<L, T> { - LinkedList { + pub const fn new() -> Self { + Self { head: None, // tail: None, _marker: PhantomData, @@ -155,6 +157,15 @@ impl<L: Link> LinkedList<L, L::Target> { let ptr = L::as_raw(&val); assert_ne!(self.head, Some(ptr)); unsafe { + // Verify the node is not already in a list (pointers must be clean) + debug_assert!( + L::pointers(ptr).as_ref().get_prev().is_none(), + "push_front: node already has prev pointer (double-insert?)" + ); + debug_assert!( + L::pointers(ptr).as_ref().get_next().is_none(), + "push_front: node already has next pointer (double-insert?)" + ); L::pointers(ptr).as_mut().set_next(self.head); L::pointers(ptr).as_mut().set_prev(None); @@ -190,8 +201,22 @@ impl<L: Link> LinkedList<L, L::Target> { // } // } + /// Removes the first element from the list and returns it, or None if empty. + pub fn pop_front(&mut self) -> Option<L::Handle> { + let head = self.head?; + unsafe { + self.head = L::pointers(head).as_ref().get_next(); + if let Some(new_head) = self.head { + L::pointers(new_head).as_mut().set_prev(None); + } + L::pointers(head).as_mut().set_next(None); + L::pointers(head).as_mut().set_prev(None); + Some(L::from_raw(head)) + } + } + /// Returns whether the linked list does not contain any node - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.head.is_none() // if self.head.is_some() { // return false; @@ -208,37 +233,47 @@ impl<L: Link> LinkedList<L, L::Target> { /// The caller **must** ensure that `node` is currently contained by /// `self` or not contained by any other list. pub unsafe fn remove(&mut self, node: NonNull<L::Target>) -> Option<L::Handle> { - if let Some(prev) = L::pointers(node).as_ref().get_prev() { - debug_assert_eq!(L::pointers(prev).as_ref().get_next(), Some(node)); - L::pointers(prev) - .as_mut() - .set_next(L::pointers(node).as_ref().get_next()); - } else { - if self.head != Some(node) { - return None; + unsafe { + if let Some(prev) = L::pointers(node).as_ref().get_prev() { + debug_assert_eq!( + L::pointers(prev).as_ref().get_next(), + Some(node), + "linked list corruption: prev->next != node (prev={prev:p}, node={node:p})" + ); + L::pointers(prev) + .as_mut() + .set_next(L::pointers(node).as_ref().get_next()); + } else { + if self.head != Some(node) { + return None; + } + + self.head = L::pointers(node).as_ref().get_next(); } - self.head = L::pointers(node).as_ref().get_next(); - } + if let Some(next) = L::pointers(node).as_ref().get_next() { + debug_assert_eq!( + L::pointers(next).as_ref().get_prev(), + Some(node), + "linked list corruption: next->prev != node (next={next:p}, node={node:p})" + ); + L::pointers(next) + .as_mut() + .set_prev(L::pointers(node).as_ref().get_prev()); + } else { + // // This might be the last item in the list + // if self.tail != Some(node) { + // return None; + // } + + // self.tail = L::pointers(node).as_ref().get_prev(); + } - if let Some(next) = L::pointers(node).as_ref().get_next() { - debug_assert_eq!(L::pointers(next).as_ref().get_prev(), Some(node)); - L::pointers(next) - .as_mut() - .set_prev(L::pointers(node).as_ref().get_prev()); - } else { - // // This might be the last item in the list - // if self.tail != Some(node) { - // return None; - // } + L::pointers(node).as_mut().set_next(None); + L::pointers(node).as_mut().set_prev(None); - // self.tail = L::pointers(node).as_ref().get_prev(); + Some(L::from_raw(node)) } - - L::pointers(node).as_mut().set_next(None); - L::pointers(node).as_mut().set_prev(None); - - Some(L::from_raw(node)) } // pub fn last(&self) -> Option<&L::Target> { @@ -249,7 +284,7 @@ impl<L: Link> LinkedList<L, L::Target> { // === rustpython additions === pub fn iter(&self) -> impl Iterator<Item = &L::Target> { - std::iter::successors(self.head, |node| unsafe { + core::iter::successors(self.head, |node| unsafe { L::pointers(*node).as_ref().get_next() }) .map(|ptr| unsafe { ptr.as_ref() }) @@ -280,7 +315,7 @@ pub struct DrainFilter<'a, T: Link, F> { } impl<T: Link> LinkedList<T, T::Target> { - pub fn drain_filter<F>(&mut self, filter: F) -> DrainFilter<'_, T, F> + pub const fn drain_filter<F>(&mut self, filter: F) -> DrainFilter<'_, T, F> where F: FnMut(&mut T::Target) -> bool, { @@ -293,7 +328,7 @@ impl<T: Link> LinkedList<T, T::Target> { } } -impl<'a, T, F> Iterator for DrainFilter<'a, T, F> +impl<T, F> Iterator for DrainFilter<'_, T, F> where T: Link, F: FnMut(&mut T::Target) -> bool, @@ -319,8 +354,8 @@ where impl<T> Pointers<T> { /// Create a new set of empty pointers - pub fn new() -> Pointers<T> { - Pointers { + pub const fn new() -> Self { + Self { inner: UnsafeCell::new(PointersInner { prev: None, next: None, @@ -329,7 +364,7 @@ impl<T> Pointers<T> { } } - fn get_prev(&self) -> Option<NonNull<T>> { + pub const fn get_prev(&self) -> Option<NonNull<T>> { // SAFETY: prev is the first field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); @@ -337,7 +372,7 @@ impl<T> Pointers<T> { ptr::read(prev) } } - fn get_next(&self) -> Option<NonNull<T>> { + pub const fn get_next(&self) -> Option<NonNull<T>> { // SAFETY: next is the second field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); @@ -347,7 +382,7 @@ impl<T> Pointers<T> { } } - fn set_prev(&mut self, value: Option<NonNull<T>>) { + pub const fn set_prev(&mut self, value: Option<NonNull<T>>) { // SAFETY: prev is the first field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); @@ -355,7 +390,7 @@ impl<T> Pointers<T> { ptr::write(prev, value); } } - fn set_next(&mut self, value: Option<NonNull<T>>) { + pub const fn set_next(&mut self, value: Option<NonNull<T>>) { // SAFETY: next is the second field in PointersInner, which is #[repr(C)]. unsafe { let inner = self.inner.get(); diff --git a/crates/common/src/lock.rs b/crates/common/src/lock.rs new file mode 100644 index 00000000000..cd7df512d83 --- /dev/null +++ b/crates/common/src/lock.rs @@ -0,0 +1,125 @@ +//! A module containing [`lock_api`]-based lock types that are or are not `Send + Sync` +//! depending on whether the `threading` feature of this module is enabled. + +use lock_api::{ + MappedMutexGuard, MappedRwLockReadGuard, MappedRwLockWriteGuard, Mutex, MutexGuard, RwLock, + RwLockReadGuard, RwLockUpgradableReadGuard, RwLockWriteGuard, +}; + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + pub use parking_lot::{RawMutex, RawRwLock, RawThreadId}; + + pub use std::sync::OnceLock as OnceCell; + pub use core::cell::LazyCell; + } else { + mod cell_lock; + pub use cell_lock::{RawCellMutex as RawMutex, RawCellRwLock as RawRwLock, SingleThreadId as RawThreadId}; + + pub use core::cell::{LazyCell, OnceCell}; + } +} + +// LazyLock: uses std::sync::LazyLock when std is available (even without +// threading, because Rust test runner uses parallel threads). +// Without std, uses a LazyCell wrapper (truly single-threaded only). +cfg_if::cfg_if! { + if #[cfg(any(feature = "threading", feature = "std"))] { + pub use std::sync::LazyLock; + } else { + pub struct LazyLock<T, F = fn() -> T>(core::cell::LazyCell<T, F>); + // SAFETY: This branch is only active when both "std" and "threading" + // features are absent — i.e., truly single-threaded no_std environments + // (e.g., embedded or bare-metal WASM). Without std, the Rust runtime + // cannot spawn threads, so Sync is trivially satisfied. + unsafe impl<T, F> Sync for LazyLock<T, F> {} + + impl<T, F: FnOnce() -> T> LazyLock<T, F> { + pub const fn new(f: F) -> Self { Self(core::cell::LazyCell::new(f)) } + pub fn force(this: &Self) -> &T { core::cell::LazyCell::force(&this.0) } + } + + impl<T, F: FnOnce() -> T> core::ops::Deref for LazyLock<T, F> { + type Target = T; + fn deref(&self) -> &T { &self.0 } + } + } +} + +mod immutable_mutex; +pub use immutable_mutex::*; +mod thread_mutex; +pub use thread_mutex::*; + +pub type PyMutex<T> = Mutex<RawMutex, T>; +pub type PyMutexGuard<'a, T> = MutexGuard<'a, RawMutex, T>; +pub type PyMappedMutexGuard<'a, T> = MappedMutexGuard<'a, RawMutex, T>; +pub type PyImmutableMappedMutexGuard<'a, T> = ImmutableMappedMutexGuard<'a, RawMutex, T>; +pub type PyThreadMutex<T> = ThreadMutex<RawMutex, RawThreadId, T>; +pub type PyThreadMutexGuard<'a, T> = ThreadMutexGuard<'a, RawMutex, RawThreadId, T>; +pub type PyMappedThreadMutexGuard<'a, T> = MappedThreadMutexGuard<'a, RawMutex, RawThreadId, T>; + +pub type PyRwLock<T> = RwLock<RawRwLock, T>; +pub type PyRwLockUpgradableReadGuard<'a, T> = RwLockUpgradableReadGuard<'a, RawRwLock, T>; +pub type PyRwLockReadGuard<'a, T> = RwLockReadGuard<'a, RawRwLock, T>; +pub type PyMappedRwLockReadGuard<'a, T> = MappedRwLockReadGuard<'a, RawRwLock, T>; +pub type PyRwLockWriteGuard<'a, T> = RwLockWriteGuard<'a, RawRwLock, T>; +pub type PyMappedRwLockWriteGuard<'a, T> = MappedRwLockWriteGuard<'a, RawRwLock, T>; + +// can add fn const_{mutex,rw_lock}() if necessary, but we probably won't need to + +/// Reset a lock to its initial (unlocked) state by zeroing its bytes. +/// +/// After `fork()`, any lock held by a now-dead thread would remain +/// permanently locked. We zero the raw bytes (the unlocked state for all +/// `parking_lot` raw lock types) instead of using the normal unlock path, +/// which would interact with stale waiter queues. +/// +/// # Safety +/// +/// Must only be called from the single-threaded child process immediately +/// after `fork()`, before any other thread is created. +/// The type `T` must represent the unlocked state as all-zero bytes +/// (true for `parking_lot::RawMutex`, `RawRwLock`, `RawReentrantMutex`, etc.). +pub unsafe fn zero_reinit_after_fork<T>(lock: *const T) { + unsafe { + core::ptr::write_bytes(lock as *mut u8, 0, core::mem::size_of::<T>()); + } +} + +/// Reset a `PyMutex` after `fork()`. See [`zero_reinit_after_fork`]. +/// +/// # Safety +/// +/// Must only be called from the single-threaded child process immediately +/// after `fork()`, before any other thread is created. +#[cfg(unix)] +pub unsafe fn reinit_mutex_after_fork<T: ?Sized>(mutex: &PyMutex<T>) { + unsafe { zero_reinit_after_fork(mutex.raw()) } +} + +/// Reset a `PyRwLock` after `fork()`. See [`zero_reinit_after_fork`]. +/// +/// # Safety +/// +/// Must only be called from the single-threaded child process immediately +/// after `fork()`, before any other thread is created. +#[cfg(unix)] +pub unsafe fn reinit_rwlock_after_fork<T: ?Sized>(rwlock: &PyRwLock<T>) { + unsafe { zero_reinit_after_fork(rwlock.raw()) } +} + +/// Reset a `PyThreadMutex` to its initial (unlocked, unowned) state after `fork()`. +/// +/// `PyThreadMutex` is used by buffered IO objects (`BufferedReader`, +/// `BufferedWriter`, `TextIOWrapper`). If a dead parent thread held one of +/// these locks during `fork()`, the child would deadlock on any IO operation. +/// +/// # Safety +/// +/// Must only be called from the single-threaded child process immediately +/// after `fork()`, before any other thread is created. +#[cfg(unix)] +pub unsafe fn reinit_thread_mutex_after_fork<T: ?Sized>(mutex: &PyThreadMutex<T>) { + unsafe { mutex.raw().reinit_after_fork() } +} diff --git a/common/src/lock/cell_lock.rs b/crates/common/src/lock/cell_lock.rs similarity index 84% rename from common/src/lock/cell_lock.rs rename to crates/common/src/lock/cell_lock.rs index 5c31fcdbb70..ab9cfb08c84 100644 --- a/common/src/lock/cell_lock.rs +++ b/crates/common/src/lock/cell_lock.rs @@ -1,16 +1,20 @@ +// spell-checker:ignore upgradably sharedly +use core::{cell::Cell, num::NonZero}; use lock_api::{ GetThreadId, RawMutex, RawRwLock, RawRwLockDowngrade, RawRwLockRecursive, RawRwLockUpgrade, RawRwLockUpgradeDowngrade, }; -use std::{cell::Cell, num::NonZeroUsize}; pub struct RawCellMutex { locked: Cell<bool>, } unsafe impl RawMutex for RawCellMutex { - #[allow(clippy::declare_interior_mutable_const)] - const INIT: Self = RawCellMutex { + #[allow( + clippy::declare_interior_mutable_const, + reason = "const lock initializer intentionally uses interior mutability" + )] + const INIT: Self = Self { locked: Cell::new(false), }; @@ -59,8 +63,11 @@ impl RawCellRwLock { } unsafe impl RawRwLock for RawCellRwLock { - #[allow(clippy::declare_interior_mutable_const)] - const INIT: Self = RawCellRwLock { + #[allow( + clippy::declare_interior_mutable_const, + reason = "const rwlock initializer intentionally uses interior mutability" + )] + const INIT: Self = Self { state: Cell::new(0), }; @@ -88,7 +95,7 @@ unsafe impl RawRwLock for RawCellRwLock { #[inline] unsafe fn unlock_shared(&self) { - self.state.set(self.state.get() - ONE_READER) + self.state.update(|x| x - ONE_READER) } #[inline] @@ -140,12 +147,12 @@ unsafe impl RawRwLockUpgrade for RawCellRwLock { #[inline] unsafe fn unlock_upgradable(&self) { - self.unlock_shared() + unsafe { self.unlock_shared() } } #[inline] unsafe fn upgrade(&self) { - if !self.try_upgrade() { + if !unsafe { self.try_upgrade() } { deadlock("upgrade ", "RwLock") } } @@ -200,10 +207,14 @@ fn deadlock(lock_kind: &str, ty: &str) -> ! { panic!("deadlock: tried to {lock_kind}lock a Cell{ty} twice") } +#[derive(Clone, Copy)] pub struct SingleThreadId(()); + unsafe impl GetThreadId for SingleThreadId { - const INIT: Self = SingleThreadId(()); - fn nonzero_thread_id(&self) -> NonZeroUsize { - NonZeroUsize::new(1).unwrap() + const INIT: Self = Self(()); + + fn nonzero_thread_id(&self) -> NonZero<usize> { + // Safety: This is constant. + unsafe { NonZero::new_unchecked(1) } } } diff --git a/common/src/lock/immutable_mutex.rs b/crates/common/src/lock/immutable_mutex.rs similarity index 80% rename from common/src/lock/immutable_mutex.rs rename to crates/common/src/lock/immutable_mutex.rs index 3f5eba78787..2013cf1c60d 100644 --- a/common/src/lock/immutable_mutex.rs +++ b/crates/common/src/lock/immutable_mutex.rs @@ -1,5 +1,8 @@ +#![allow(clippy::needless_lifetimes)] + +use alloc::fmt; +use core::{marker::PhantomData, ops::Deref}; use lock_api::{MutexGuard, RawMutex}; -use std::{fmt, marker::PhantomData, ops::Deref}; /// A mutex guard that has an exclusive lock, but only an immutable reference; useful if you /// need to map a mutex guard with a function that returns an `&T`. Construct using the @@ -20,7 +23,7 @@ impl<'a, R: RawMutex, T: ?Sized> MapImmutable<'a, R, T> for MutexGuard<'a, R, T> { let raw = unsafe { MutexGuard::mutex(&s).raw() }; let data = f(&s) as *const U; - std::mem::forget(s); + core::mem::forget(s); ImmutableMappedMutexGuard { raw, data, @@ -36,7 +39,7 @@ impl<'a, R: RawMutex, T: ?Sized> ImmutableMappedMutexGuard<'a, R, T> { { let raw = s.raw; let data = f(&s) as *const U; - std::mem::forget(s); + core::mem::forget(s); ImmutableMappedMutexGuard { raw, data, @@ -45,7 +48,7 @@ impl<'a, R: RawMutex, T: ?Sized> ImmutableMappedMutexGuard<'a, R, T> { } } -impl<'a, R: RawMutex, T: ?Sized> Deref for ImmutableMappedMutexGuard<'a, R, T> { +impl<R: RawMutex, T: ?Sized> Deref for ImmutableMappedMutexGuard<'_, R, T> { type Target = T; fn deref(&self) -> &Self::Target { // SAFETY: self.data is valid for the lifetime of the guard @@ -53,22 +56,20 @@ impl<'a, R: RawMutex, T: ?Sized> Deref for ImmutableMappedMutexGuard<'a, R, T> { } } -impl<'a, R: RawMutex, T: ?Sized> Drop for ImmutableMappedMutexGuard<'a, R, T> { +impl<R: RawMutex, T: ?Sized> Drop for ImmutableMappedMutexGuard<'_, R, T> { fn drop(&mut self) { // SAFETY: An ImmutableMappedMutexGuard always holds the lock unsafe { self.raw.unlock() } } } -impl<'a, R: RawMutex, T: fmt::Debug + ?Sized> fmt::Debug for ImmutableMappedMutexGuard<'a, R, T> { +impl<R: RawMutex, T: fmt::Debug + ?Sized> fmt::Debug for ImmutableMappedMutexGuard<'_, R, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&**self, f) } } -impl<'a, R: RawMutex, T: fmt::Display + ?Sized> fmt::Display - for ImmutableMappedMutexGuard<'a, R, T> -{ +impl<R: RawMutex, T: fmt::Display + ?Sized> fmt::Display for ImmutableMappedMutexGuard<'_, R, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } diff --git a/crates/common/src/lock/thread_mutex.rs b/crates/common/src/lock/thread_mutex.rs new file mode 100644 index 00000000000..884556c4476 --- /dev/null +++ b/crates/common/src/lock/thread_mutex.rs @@ -0,0 +1,356 @@ +#![allow(clippy::needless_lifetimes)] + +use alloc::fmt; +use core::{ + cell::UnsafeCell, + marker::PhantomData, + ops::{Deref, DerefMut}, + ptr::NonNull, + sync::atomic::{AtomicUsize, Ordering}, +}; +use lock_api::{GetThreadId, GuardNoSend, RawMutex}; + +// based off ReentrantMutex from lock_api + +/// A mutex type that knows when it would deadlock +pub struct RawThreadMutex<R: RawMutex, G: GetThreadId> { + owner: AtomicUsize, + mutex: R, + get_thread_id: G, +} + +impl<R: RawMutex, G: GetThreadId> RawThreadMutex<R, G> { + #[allow( + clippy::declare_interior_mutable_const, + reason = "const initializer for lock primitive contains atomics by design" + )] + pub const INIT: Self = Self { + owner: AtomicUsize::new(0), + mutex: R::INIT, + get_thread_id: G::INIT, + }; + + #[inline] + fn lock_internal<F: FnOnce() -> bool>(&self, try_lock: F) -> Option<bool> { + let id = self.get_thread_id.nonzero_thread_id().get(); + if self.owner.load(Ordering::Relaxed) == id { + return None; + } else { + if !try_lock() { + return Some(false); + } + self.owner.store(id, Ordering::Relaxed); + } + Some(true) + } + + /// Blocks for the mutex to be available, and returns true if the mutex isn't already + /// locked on the current thread. + pub fn lock(&self) -> bool { + self.lock_internal(|| { + self.mutex.lock(); + true + }) + .is_some() + } + + /// Like `lock()` but wraps the blocking wait in `wrap_fn`. + /// The caller can use this to detach thread state while waiting. + pub fn lock_wrapped<F: FnOnce(&dyn Fn())>(&self, wrap_fn: F) -> bool { + let id = self.get_thread_id.nonzero_thread_id().get(); + if self.owner.load(Ordering::Relaxed) == id { + return false; + } + wrap_fn(&|| self.mutex.lock()); + self.owner.store(id, Ordering::Relaxed); + true + } + + /// Returns `Some(true)` if able to successfully lock without blocking, `Some(false)` + /// otherwise, and `None` when the mutex is already locked on the current thread. + pub fn try_lock(&self) -> Option<bool> { + self.lock_internal(|| self.mutex.try_lock()) + } + + /// Unlocks this mutex. The inner mutex may not be unlocked if + /// this mutex was acquired previously in the current thread. + /// + /// # Safety + /// + /// This method may only be called if the mutex is held by the current thread. + pub unsafe fn unlock(&self) { + self.owner.store(0, Ordering::Relaxed); + unsafe { self.mutex.unlock() }; + } +} + +impl<R: RawMutex, G: GetThreadId> RawThreadMutex<R, G> { + /// Reset this mutex to its initial (unlocked, unowned) state after `fork()`. + /// + /// # Safety + /// + /// Must only be called from the single-threaded child process immediately + /// after `fork()`, before any other thread is created. + #[cfg(unix)] + pub unsafe fn reinit_after_fork(&self) { + self.owner.store(0, Ordering::Relaxed); + unsafe { + let mutex_ptr = &self.mutex as *const R as *mut u8; + core::ptr::write_bytes(mutex_ptr, 0, core::mem::size_of::<R>()); + } + } +} + +unsafe impl<R: RawMutex + Send, G: GetThreadId + Send> Send for RawThreadMutex<R, G> {} +unsafe impl<R: RawMutex + Sync, G: GetThreadId + Sync> Sync for RawThreadMutex<R, G> {} + +pub struct ThreadMutex<R: RawMutex, G: GetThreadId, T: ?Sized> { + raw: RawThreadMutex<R, G>, + data: UnsafeCell<T>, +} + +impl<R: RawMutex, G: GetThreadId, T> ThreadMutex<R, G, T> { + pub const fn new(val: T) -> Self { + Self { + raw: RawThreadMutex::INIT, + data: UnsafeCell::new(val), + } + } + + pub fn into_inner(self) -> T { + self.data.into_inner() + } +} +impl<R: RawMutex, G: GetThreadId, T: Default> Default for ThreadMutex<R, G, T> { + fn default() -> Self { + Self::new(T::default()) + } +} +impl<R: RawMutex, G: GetThreadId, T> From<T> for ThreadMutex<R, G, T> { + fn from(val: T) -> Self { + Self::new(val) + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutex<R, G, T> { + /// Access the underlying raw thread mutex. + pub fn raw(&self) -> &RawThreadMutex<R, G> { + &self.raw + } + + pub fn lock(&self) -> Option<ThreadMutexGuard<'_, R, G, T>> { + if self.raw.lock() { + Some(ThreadMutexGuard { + mu: self, + marker: PhantomData, + }) + } else { + None + } + } + + /// Like `lock()` but wraps the blocking wait in `wrap_fn`. + /// The caller can use this to detach thread state while waiting. + pub fn lock_wrapped<F: FnOnce(&dyn Fn())>( + &self, + wrap_fn: F, + ) -> Option<ThreadMutexGuard<'_, R, G, T>> { + if self.raw.lock_wrapped(wrap_fn) { + Some(ThreadMutexGuard { + mu: self, + marker: PhantomData, + }) + } else { + None + } + } + + pub fn try_lock(&self) -> Result<ThreadMutexGuard<'_, R, G, T>, TryLockThreadError> { + match self.raw.try_lock() { + Some(true) => Ok(ThreadMutexGuard { + mu: self, + marker: PhantomData, + }), + Some(false) => Err(TryLockThreadError::Other), + None => Err(TryLockThreadError::Current), + } + } +} + +#[derive(Clone, Copy)] +pub enum TryLockThreadError { + /// Failed to lock because mutex was already locked on another thread. + Other, + /// Failed to lock because mutex was already locked on current thread. + Current, +} + +struct LockedPlaceholder(&'static str); + +impl fmt::Debug for LockedPlaceholder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.0) + } +} + +impl<R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Debug> fmt::Debug for ThreadMutex<R, G, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.try_lock() { + Ok(guard) => f + .debug_struct("ThreadMutex") + .field("data", &&*guard) + .finish(), + Err(e) => { + let msg = match e { + TryLockThreadError::Other => "<locked on other thread>", + TryLockThreadError::Current => "<locked on current thread>", + }; + f.debug_struct("ThreadMutex") + .field("data", &LockedPlaceholder(msg)) + .finish() + } + } + } +} + +unsafe impl<R: RawMutex + Send, G: GetThreadId + Send, T: ?Sized + Send> Send + for ThreadMutex<R, G, T> +{ +} +unsafe impl<R: RawMutex + Sync, G: GetThreadId + Sync, T: ?Sized + Send> Sync + for ThreadMutex<R, G, T> +{ +} + +pub struct ThreadMutexGuard<'a, R: RawMutex, G: GetThreadId, T: ?Sized> { + mu: &'a ThreadMutex<R, G, T>, + marker: PhantomData<(&'a mut T, GuardNoSend)>, +} +impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutexGuard<'a, R, G, T> { + pub fn map<U, F: FnOnce(&mut T) -> &mut U>( + mut s: Self, + f: F, + ) -> MappedThreadMutexGuard<'a, R, G, U> { + let data = f(&mut s).into(); + let mu = &s.mu.raw; + core::mem::forget(s); + MappedThreadMutexGuard { + mu, + data, + marker: PhantomData, + } + } + pub fn try_map<U, F: FnOnce(&mut T) -> Option<&mut U>>( + mut s: Self, + f: F, + ) -> Result<MappedThreadMutexGuard<'a, R, G, U>, Self> { + if let Some(data) = f(&mut s) { + let data = data.into(); + let mu = &s.mu.raw; + core::mem::forget(s); + Ok(MappedThreadMutexGuard { + mu, + data, + marker: PhantomData, + }) + } else { + Err(s) + } + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized> Deref for ThreadMutexGuard<'_, R, G, T> { + type Target = T; + fn deref(&self) -> &T { + unsafe { &*self.mu.data.get() } + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized> DerefMut for ThreadMutexGuard<'_, R, G, T> { + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.mu.data.get() } + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized> Drop for ThreadMutexGuard<'_, R, G, T> { + fn drop(&mut self) { + unsafe { self.mu.raw.unlock() } + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Display> fmt::Display + for ThreadMutexGuard<'_, R, G, T> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Debug> fmt::Debug + for ThreadMutexGuard<'_, R, G, T> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} +pub struct MappedThreadMutexGuard<'a, R: RawMutex, G: GetThreadId, T: ?Sized> { + mu: &'a RawThreadMutex<R, G>, + data: NonNull<T>, + marker: PhantomData<(&'a mut T, GuardNoSend)>, +} +impl<'a, R: RawMutex, G: GetThreadId, T: ?Sized> MappedThreadMutexGuard<'a, R, G, T> { + pub fn map<U, F: FnOnce(&mut T) -> &mut U>( + mut s: Self, + f: F, + ) -> MappedThreadMutexGuard<'a, R, G, U> { + let data = f(&mut s).into(); + let mu = s.mu; + core::mem::forget(s); + MappedThreadMutexGuard { + mu, + data, + marker: PhantomData, + } + } + pub fn try_map<U, F: FnOnce(&mut T) -> Option<&mut U>>( + mut s: Self, + f: F, + ) -> Result<MappedThreadMutexGuard<'a, R, G, U>, Self> { + if let Some(data) = f(&mut s) { + let data = data.into(); + let mu = s.mu; + core::mem::forget(s); + Ok(MappedThreadMutexGuard { + mu, + data, + marker: PhantomData, + }) + } else { + Err(s) + } + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized> Deref for MappedThreadMutexGuard<'_, R, G, T> { + type Target = T; + fn deref(&self) -> &T { + unsafe { self.data.as_ref() } + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized> DerefMut for MappedThreadMutexGuard<'_, R, G, T> { + fn deref_mut(&mut self) -> &mut T { + unsafe { self.data.as_mut() } + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized> Drop for MappedThreadMutexGuard<'_, R, G, T> { + fn drop(&mut self) { + unsafe { self.mu.unlock() } + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Display> fmt::Display + for MappedThreadMutexGuard<'_, R, G, T> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} +impl<R: RawMutex, G: GetThreadId, T: ?Sized + fmt::Debug> fmt::Debug + for MappedThreadMutexGuard<'_, R, G, T> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} diff --git a/crates/common/src/macros.rs b/crates/common/src/macros.rs new file mode 100644 index 00000000000..08d00e592d9 --- /dev/null +++ b/crates/common/src/macros.rs @@ -0,0 +1,59 @@ +/// Suppress the MSVC invalid parameter handler, which by default crashes the process. Does nothing +/// on non-MSVC targets. +#[macro_export] +macro_rules! suppress_iph { + ($e:expr) => { + $crate::__suppress_iph_impl!($e) + }; +} + +#[macro_export] +#[doc(hidden)] +#[cfg(all(windows, target_env = "msvc"))] +macro_rules! __suppress_iph_impl { + ($e:expr) => {{ + let old = $crate::__macro_private::_set_thread_local_invalid_parameter_handler( + $crate::__macro_private::silent_iph_handler, + ); + let ret = $e; + $crate::__macro_private::_set_thread_local_invalid_parameter_handler(old); + ret + }}; +} + +#[cfg(not(all(windows, target_env = "msvc")))] +#[macro_export] +#[doc(hidden)] +macro_rules! __suppress_iph_impl { + ($e:expr) => { + $e + }; +} + +#[doc(hidden)] +pub mod __macro_private { + #[cfg(target_env = "msvc")] + type InvalidParamHandler = extern "C" fn( + *const libc::wchar_t, + *const libc::wchar_t, + *const libc::wchar_t, + libc::c_uint, + libc::uintptr_t, + ); + #[cfg(target_env = "msvc")] + unsafe extern "C" { + pub fn _set_thread_local_invalid_parameter_handler( + pNew: InvalidParamHandler, + ) -> InvalidParamHandler; + } + + #[cfg(target_env = "msvc")] + pub extern "C" fn silent_iph_handler( + _: *const libc::wchar_t, + _: *const libc::wchar_t, + _: *const libc::wchar_t, + _: libc::c_uint, + _: libc::uintptr_t, + ) { + } +} diff --git a/crates/common/src/os.rs b/crates/common/src/os.rs new file mode 100644 index 00000000000..2a318f477e5 --- /dev/null +++ b/crates/common/src/os.rs @@ -0,0 +1,288 @@ +// spell-checker:disable +// TODO: we can move more os-specific bindings/interfaces from stdlib::{os, posix, nt} to here + +use core::str::Utf8Error; +use std::{io, process::ExitCode}; + +/// Convert exit code to std::process::ExitCode +/// +/// On Windows, this supports the full u32 range including STATUS_CONTROL_C_EXIT (0xC000013A). +/// On other platforms, only the lower 8 bits are used. +pub fn exit_code(code: u32) -> ExitCode { + #[cfg(windows)] + { + // For large exit codes like STATUS_CONTROL_C_EXIT (0xC000013A), + // we need to call std::process::exit() directly since ExitCode::from(u8) + // would truncate the value, and ExitCode::from_raw() is still unstable. + // FIXME: side effect in exit_code is not ideal. + if code > u8::MAX as u32 { + std::process::exit(code as i32) + } + } + ExitCode::from(code as u8) +} + +pub trait ErrorExt { + fn posix_errno(&self) -> i32; +} + +impl ErrorExt for io::Error { + #[cfg(not(windows))] + fn posix_errno(&self) -> i32 { + self.raw_os_error().unwrap_or(0) + } + #[cfg(windows)] + fn posix_errno(&self) -> i32 { + let winerror = self.raw_os_error().unwrap_or(0); + winerror_to_errno(winerror) + } +} + +/// Get the last error from C runtime library functions (like _dup, _dup2, _fstat, etc.) +/// CRT functions set errno, not GetLastError(), so we need to read errno directly. +#[cfg(windows)] +pub fn errno_io_error() -> io::Error { + let errno: i32 = get_errno(); + let winerror = errno_to_winerror(errno); + io::Error::from_raw_os_error(winerror) +} + +#[cfg(not(windows))] +pub fn errno_io_error() -> io::Error { + std::io::Error::last_os_error() +} + +#[cfg(windows)] +pub fn get_errno() -> i32 { + unsafe extern "C" { + fn _get_errno(pValue: *mut i32) -> i32; + } + let mut errno = 0; + unsafe { suppress_iph!(_get_errno(&mut errno)) }; + errno +} + +#[cfg(not(windows))] +pub fn get_errno() -> i32 { + std::io::Error::last_os_error().posix_errno() +} + +/// Set errno to the specified value. +#[cfg(windows)] +pub fn set_errno(value: i32) { + unsafe extern "C" { + fn _set_errno(value: i32) -> i32; + } + unsafe { suppress_iph!(_set_errno(value)) }; +} + +#[cfg(unix)] +pub fn set_errno(value: i32) { + nix::errno::Errno::from_raw(value).set(); +} + +#[cfg(unix)] +pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { + use std::os::unix::ffi::OsStrExt; + Ok(std::ffi::OsStr::from_bytes(b)) +} + +#[cfg(not(unix))] +pub fn bytes_as_os_str(b: &[u8]) -> Result<&std::ffi::OsStr, Utf8Error> { + Ok(core::str::from_utf8(b)?.as_ref()) +} + +#[cfg(unix)] +pub use std::os::unix::ffi; + +// WASIp1 uses stable std::os::wasi::ffi +#[cfg(all(target_os = "wasi", not(target_env = "p2")))] +pub use std::os::wasi::ffi; + +// WASIp2: std::os::wasip2::ffi is unstable, so we provide a stable implementation +// leveraging WASI's UTF-8 string guarantee +#[cfg(all(target_os = "wasi", target_env = "p2"))] +pub mod ffi { + use std::ffi::{OsStr, OsString}; + + pub trait OsStrExt: sealed::Sealed { + fn as_bytes(&self) -> &[u8]; + fn from_bytes(slice: &[u8]) -> &Self; + } + + impl OsStrExt for OsStr { + fn as_bytes(&self) -> &[u8] { + // WASI strings are guaranteed to be UTF-8 + self.to_str().expect("wasip2 strings are UTF-8").as_bytes() + } + + fn from_bytes(slice: &[u8]) -> &OsStr { + // WASI strings are guaranteed to be UTF-8 + OsStr::new(core::str::from_utf8(slice).expect("wasip2 strings are UTF-8")) + } + } + + pub trait OsStringExt: sealed::Sealed { + fn from_vec(vec: Vec<u8>) -> Self; + fn into_vec(self) -> Vec<u8>; + } + + impl OsStringExt for OsString { + fn from_vec(vec: Vec<u8>) -> OsString { + // WASI strings are guaranteed to be UTF-8 + OsString::from(String::from_utf8(vec).expect("wasip2 strings are UTF-8")) + } + + fn into_vec(self) -> Vec<u8> { + // WASI strings are guaranteed to be UTF-8 + self.to_str() + .expect("wasip2 strings are UTF-8") + .as_bytes() + .to_vec() + } + } + + mod sealed { + pub trait Sealed {} + impl Sealed for std::ffi::OsStr {} + impl Sealed for std::ffi::OsString {} + } +} + +#[cfg(windows)] +pub fn errno_to_winerror(errno: i32) -> i32 { + use libc::*; + use windows_sys::Win32::Foundation::*; + let winerror = match errno { + ENOENT => ERROR_FILE_NOT_FOUND, + E2BIG => ERROR_BAD_ENVIRONMENT, + ENOEXEC => ERROR_BAD_FORMAT, + EBADF => ERROR_INVALID_HANDLE, + ECHILD => ERROR_WAIT_NO_CHILDREN, + EAGAIN => ERROR_NO_PROC_SLOTS, + ENOMEM => ERROR_NOT_ENOUGH_MEMORY, + EACCES => ERROR_ACCESS_DENIED, + EEXIST => ERROR_FILE_EXISTS, + EXDEV => ERROR_NOT_SAME_DEVICE, + ENOTDIR => ERROR_DIRECTORY, + EMFILE => ERROR_TOO_MANY_OPEN_FILES, + ENOSPC => ERROR_DISK_FULL, + EPIPE => ERROR_BROKEN_PIPE, + ENOTEMPTY => ERROR_DIR_NOT_EMPTY, + EILSEQ => ERROR_NO_UNICODE_TRANSLATION, + EINVAL => ERROR_INVALID_FUNCTION, + _ => ERROR_INVALID_FUNCTION, + }; + winerror as _ +} + +// winerror: https://learn.microsoft.com/windows/win32/debug/system-error-codes--0-499- +// errno: https://learn.microsoft.com/cpp/c-runtime-library/errno-constants?view=msvc-170 +#[cfg(windows)] +pub fn winerror_to_errno(winerror: i32) -> i32 { + use libc::*; + use windows_sys::Win32::{ + Foundation::*, + Networking::WinSock::{ + WSAEACCES, WSAEBADF, WSAECONNABORTED, WSAECONNREFUSED, WSAECONNRESET, WSAEFAULT, + WSAEINTR, WSAEINVAL, WSAEMFILE, + }, + }; + // Unwrap FACILITY_WIN32 HRESULT errors. + // if ((winerror & 0xFFFF0000) == 0x80070000) { + // winerror &= 0x0000FFFF; + // } + + // Winsock error codes (10000-11999) are errno values. + if (10000..12000).contains(&winerror) { + match winerror { + WSAEINTR | WSAEBADF | WSAEACCES | WSAEFAULT | WSAEINVAL | WSAEMFILE => { + // Winsock definitions of errno values. See WinSock2.h + return winerror - 10000; + } + _ => return winerror as _, + } + } + + #[allow(non_upper_case_globals)] + match winerror as u32 { + ERROR_FILE_NOT_FOUND + | ERROR_PATH_NOT_FOUND + | ERROR_INVALID_DRIVE + | ERROR_NO_MORE_FILES + | ERROR_BAD_NETPATH + | ERROR_BAD_NET_NAME + | ERROR_BAD_PATHNAME + | ERROR_FILENAME_EXCED_RANGE => ENOENT, + ERROR_BAD_ENVIRONMENT => E2BIG, + ERROR_BAD_FORMAT + | ERROR_INVALID_STARTING_CODESEG + | ERROR_INVALID_STACKSEG + | ERROR_INVALID_MODULETYPE + | ERROR_INVALID_EXE_SIGNATURE + | ERROR_EXE_MARKED_INVALID + | ERROR_BAD_EXE_FORMAT + | ERROR_ITERATED_DATA_EXCEEDS_64k + | ERROR_INVALID_MINALLOCSIZE + | ERROR_DYNLINK_FROM_INVALID_RING + | ERROR_IOPL_NOT_ENABLED + | ERROR_INVALID_SEGDPL + | ERROR_AUTODATASEG_EXCEEDS_64k + | ERROR_RING2SEG_MUST_BE_MOVABLE + | ERROR_RELOC_CHAIN_XEEDS_SEGLIM + | ERROR_INFLOOP_IN_RELOC_CHAIN => ENOEXEC, + ERROR_INVALID_HANDLE | ERROR_INVALID_TARGET_HANDLE | ERROR_DIRECT_ACCESS_HANDLE => EBADF, + ERROR_WAIT_NO_CHILDREN | ERROR_CHILD_NOT_COMPLETE => ECHILD, + ERROR_NO_PROC_SLOTS | ERROR_MAX_THRDS_REACHED | ERROR_NESTING_NOT_ALLOWED => EAGAIN, + ERROR_ARENA_TRASHED + | ERROR_NOT_ENOUGH_MEMORY + | ERROR_INVALID_BLOCK + | ERROR_NOT_ENOUGH_QUOTA => ENOMEM, + ERROR_ACCESS_DENIED + | ERROR_CURRENT_DIRECTORY + | ERROR_WRITE_PROTECT + | ERROR_BAD_UNIT + | ERROR_NOT_READY + | ERROR_BAD_COMMAND + | ERROR_CRC + | ERROR_BAD_LENGTH + | ERROR_SEEK + | ERROR_NOT_DOS_DISK + | ERROR_SECTOR_NOT_FOUND + | ERROR_OUT_OF_PAPER + | ERROR_WRITE_FAULT + | ERROR_READ_FAULT + | ERROR_GEN_FAILURE + | ERROR_SHARING_VIOLATION + | ERROR_LOCK_VIOLATION + | ERROR_WRONG_DISK + | ERROR_SHARING_BUFFER_EXCEEDED + | ERROR_NETWORK_ACCESS_DENIED + | ERROR_CANNOT_MAKE + | ERROR_FAIL_I24 + | ERROR_DRIVE_LOCKED + | ERROR_SEEK_ON_DEVICE + | ERROR_NOT_LOCKED + | ERROR_LOCK_FAILED + | 35 => EACCES, + ERROR_FILE_EXISTS | ERROR_ALREADY_EXISTS => EEXIST, + ERROR_NOT_SAME_DEVICE => EXDEV, + ERROR_DIRECTORY => ENOTDIR, + ERROR_TOO_MANY_OPEN_FILES => EMFILE, + ERROR_DISK_FULL => ENOSPC, + ERROR_BROKEN_PIPE | ERROR_NO_DATA => EPIPE, + ERROR_DIR_NOT_EMPTY => ENOTEMPTY, + ERROR_NO_UNICODE_TRANSLATION => EILSEQ, + // Connection-related Windows error codes - map to Winsock error codes + // which Python uses on Windows (errno.ECONNREFUSED = 10061, etc.) + ERROR_CONNECTION_REFUSED => WSAECONNREFUSED, + ERROR_CONNECTION_ABORTED => WSAECONNABORTED, + ERROR_NETNAME_DELETED => WSAECONNRESET, + ERROR_INVALID_FUNCTION + | ERROR_INVALID_ACCESS + | ERROR_INVALID_DATA + | ERROR_INVALID_PARAMETER + | ERROR_NEGATIVE_SEEK => EINVAL, + _ => EINVAL, + } +} diff --git a/crates/common/src/rand.rs b/crates/common/src/rand.rs new file mode 100644 index 00000000000..334505ac945 --- /dev/null +++ b/crates/common/src/rand.rs @@ -0,0 +1,13 @@ +/// Get `N` bytes of random data. +/// +/// This function is mildly expensive to call, as it fetches random data +/// directly from the OS entropy source. +/// +/// # Panics +/// +/// Panics if the OS entropy source returns an error. +pub fn os_random<const N: usize>() -> [u8; N] { + let mut buf = [0u8; N]; + getrandom::fill(&mut buf).unwrap(); + buf +} diff --git a/crates/common/src/rc.rs b/crates/common/src/rc.rs new file mode 100644 index 00000000000..9e4cca228fd --- /dev/null +++ b/crates/common/src/rc.rs @@ -0,0 +1,12 @@ +#[cfg(not(feature = "threading"))] +use alloc::rc::Rc; +#[cfg(feature = "threading")] +use alloc::sync::Arc; + +// type aliases instead of new-types because you can't do `fn method(self: PyRc<Self>)` with a +// newtype; requires the arbitrary_self_types unstable feature + +#[cfg(feature = "threading")] +pub type PyRc<T> = Arc<T>; +#[cfg(not(feature = "threading"))] +pub type PyRc<T> = Rc<T>; diff --git a/crates/common/src/refcount.rs b/crates/common/src/refcount.rs new file mode 100644 index 00000000000..4d871a67a0d --- /dev/null +++ b/crates/common/src/refcount.rs @@ -0,0 +1,276 @@ +use crate::atomic::{Ordering, PyAtomic, Radium}; + +// State layout (usize): +// [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [N bits: weak_count] [M bits: strong_count] +// 64-bit: N=30, M=31. 32-bit: N=14, M=15. +const FLAG_BITS: u32 = 3; +const DESTRUCTED: usize = 1 << (usize::BITS - 1); +const LEAKED: usize = 1 << (usize::BITS - 3); +const TOTAL_COUNT_WIDTH: u32 = usize::BITS - FLAG_BITS; +const WEAK_WIDTH: u32 = TOTAL_COUNT_WIDTH / 2; +const STRONG_WIDTH: u32 = TOTAL_COUNT_WIDTH - WEAK_WIDTH; +const STRONG: usize = (1 << STRONG_WIDTH) - 1; +const COUNT: usize = 1; +const WEAK_COUNT: usize = 1 << STRONG_WIDTH; + +#[inline(never)] +#[cold] +fn refcount_overflow() -> ! { + #[cfg(feature = "std")] + std::process::abort(); + #[cfg(not(feature = "std"))] + core::panic!("refcount overflow"); +} + +/// State wraps reference count + flags in a single word (platform usize) +#[derive(Clone, Copy)] +struct State { + inner: usize, +} + +impl State { + #[inline] + fn from_raw(inner: usize) -> Self { + Self { inner } + } + + #[inline] + fn as_raw(self) -> usize { + self.inner + } + + #[inline] + fn strong(self) -> u32 { + ((self.inner & STRONG) / COUNT) as u32 + } + + #[inline] + fn destructed(self) -> bool { + (self.inner & DESTRUCTED) != 0 + } + + #[inline] + fn leaked(self) -> bool { + (self.inner & LEAKED) != 0 + } + + #[inline] + fn add_strong(self, val: u32) -> Self { + Self::from_raw(self.inner + (val as usize) * COUNT) + } + + #[inline] + fn with_leaked(self, leaked: bool) -> Self { + Self::from_raw((self.inner & !LEAKED) | if leaked { LEAKED } else { 0 }) + } +} + +/// Reference count using state layout with LEAKED support. +/// +/// State layout (usize): +/// 64-bit: [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [30 bits: weak_count] [31 bits: strong_count] +/// 32-bit: [1 bit: destructed] [1 bit: reserved] [1 bit: leaked] [14 bits: weak_count] [15 bits: strong_count] +pub struct RefCount { + state: PyAtomic<usize>, +} + +impl Default for RefCount { + fn default() -> Self { + Self::new() + } +} + +impl RefCount { + /// Create a new RefCount with strong count = 1 + pub fn new() -> Self { + // Initial state: strong=1, weak=1 (implicit weak for strong refs) + Self { + state: Radium::new(COUNT + WEAK_COUNT), + } + } + + /// Get current strong count + #[inline] + pub fn get(&self) -> usize { + State::from_raw(self.state.load(Ordering::Relaxed)).strong() as usize + } + + /// Increment strong count + #[inline] + pub fn inc(&self) { + let val = State::from_raw(self.state.fetch_add(COUNT, Ordering::Relaxed)); + if val.destructed() || (val.strong() as usize) > STRONG - 1 { + refcount_overflow(); + } + if val.strong() == 0 { + // The previous fetch_add created a permission to run decrement again + self.state.fetch_add(COUNT, Ordering::Relaxed); + } + } + + #[inline] + pub fn inc_by(&self, n: usize) { + debug_assert!(n <= STRONG); + let val = State::from_raw(self.state.fetch_add(n * COUNT, Ordering::Relaxed)); + if val.destructed() || (val.strong() as usize) > STRONG - n { + refcount_overflow(); + } + } + + /// Returns true if successful + #[inline] + #[must_use] + pub fn safe_inc(&self) -> bool { + let mut old = State::from_raw(self.state.load(Ordering::Relaxed)); + loop { + if old.destructed() || old.strong() == 0 { + return false; + } + if (old.strong() as usize) >= STRONG { + refcount_overflow(); + } + let new_state = old.add_strong(1); + match self.state.compare_exchange_weak( + old.as_raw(), + new_state.as_raw(), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(curr) => old = State::from_raw(curr), + } + } + } + + /// Decrement strong count. Returns true when count drops to 0. + #[inline] + #[must_use] + pub fn dec(&self) -> bool { + let old = State::from_raw(self.state.fetch_sub(COUNT, Ordering::Release)); + + // LEAKED objects never reach 0 + if old.leaked() { + return false; + } + + if old.strong() == 1 { + core::sync::atomic::fence(Ordering::Acquire); + return true; + } + false + } + + /// Mark this object as leaked (interned). It will never be deallocated. + pub fn leak(&self) { + debug_assert!(!self.is_leaked()); + let mut old = State::from_raw(self.state.load(Ordering::Relaxed)); + loop { + let new_state = old.with_leaked(true); + match self.state.compare_exchange_weak( + old.as_raw(), + new_state.as_raw(), + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => return, + Err(curr) => old = State::from_raw(curr), + } + } + } + + /// Check if this object is leaked (interned). + pub fn is_leaked(&self) -> bool { + State::from_raw(self.state.load(Ordering::Acquire)).leaked() + } +} + +// Deferred Drop Infrastructure +// +// This mechanism allows untrack_object() calls to be deferred until after +// the GC collection phase completes, preventing deadlocks that occur when +// clear (pop_edges) triggers object destruction while holding the tracked_objects lock. + +#[cfg(feature = "std")] +use core::cell::{Cell, RefCell}; + +#[cfg(feature = "std")] +thread_local! { + /// Flag indicating if we're inside a deferred drop context. + /// When true, drop operations should defer untrack calls. + static IN_DEFERRED_CONTEXT: Cell<bool> = const { Cell::new(false) }; + + /// Queue of deferred untrack operations. + /// No Send bound needed - this is thread-local and only accessed from the same thread. + static DEFERRED_QUEUE: RefCell<Vec<Box<dyn FnOnce()>>> = const { RefCell::new(Vec::new()) }; +} + +#[cfg(feature = "std")] +struct DeferredDropGuard { + was_in_context: bool, +} + +#[cfg(feature = "std")] +impl Drop for DeferredDropGuard { + fn drop(&mut self) { + IN_DEFERRED_CONTEXT.with(|in_ctx| { + in_ctx.set(self.was_in_context); + }); + // Only flush if we're the outermost context and not already panicking + // (flushing during unwinding risks double-panic → process abort). + if !self.was_in_context && !std::thread::panicking() { + flush_deferred_drops(); + } + } +} + +/// Execute a function within a deferred drop context. +/// Any calls to `try_defer_drop` within this context will be queued +/// and executed when the context exits (even on panic). +#[cfg(feature = "std")] +#[inline] +pub fn with_deferred_drops<F, R>(f: F) -> R +where + F: FnOnce() -> R, +{ + let _guard = IN_DEFERRED_CONTEXT.with(|in_ctx| { + let was_in_context = in_ctx.get(); + in_ctx.set(true); + DeferredDropGuard { was_in_context } + }); + f() +} + +/// Try to defer a drop-related operation. +/// If inside a deferred context, the operation is queued. +/// Otherwise, it executes immediately. +#[cfg(feature = "std")] +#[inline] +pub fn try_defer_drop<F>(f: F) +where + F: FnOnce() + 'static, +{ + let should_defer = IN_DEFERRED_CONTEXT.with(|in_ctx| in_ctx.get()); + + if should_defer { + DEFERRED_QUEUE.with(|q| { + q.borrow_mut().push(Box::new(f)); + }); + } else { + f(); + } +} + +/// Flush all deferred drop operations. +/// This is automatically called when exiting a deferred context. +#[cfg(feature = "std")] +#[inline] +pub fn flush_deferred_drops() { + DEFERRED_QUEUE.with(|q| { + // Take all queued operations + let ops: Vec<_> = q.borrow_mut().drain(..).collect(); + // Execute them outside the borrow + for op in ops { + op(); + } + }); +} diff --git a/crates/common/src/static_cell.rs b/crates/common/src/static_cell.rs new file mode 100644 index 00000000000..bf277e60ea7 --- /dev/null +++ b/crates/common/src/static_cell.rs @@ -0,0 +1,190 @@ +#[cfg(feature = "threading")] +mod threading { + use crate::lock::OnceCell; + + pub struct StaticCell<T: 'static> { + inner: OnceCell<T>, + } + + impl<T> StaticCell<T> { + #[doc(hidden)] + pub const fn _from_once_cell(inner: OnceCell<T>) -> Self { + Self { inner } + } + + pub fn get(&'static self) -> Option<&'static T> { + self.inner.get() + } + + pub fn set(&'static self, value: T) -> Result<(), T> { + self.inner.set(value) + } + + pub fn get_or_init<F>(&'static self, f: F) -> &'static T + where + F: FnOnce() -> T, + { + self.inner.get_or_init(f) + } + + pub fn get_or_try_init<F, E>(&'static self, f: F) -> Result<&'static T, E> + where + F: FnOnce() -> Result<T, E>, + { + if let Some(val) = self.inner.get() { + return Ok(val); + } + let val = f()?; + let _ = self.inner.set(val); + Ok(self.inner.get().unwrap()) + } + } + + #[macro_export] + macro_rules! static_cell { + ($($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty;)+) => { + $($(#[$attr])* + $vis static $name: $crate::static_cell::StaticCell<$t> = + $crate::static_cell::StaticCell::_from_once_cell($crate::lock::OnceCell::new());)+ + }; + } +} +#[cfg(feature = "threading")] +pub use threading::*; + +#[cfg(all(not(feature = "threading"), feature = "std"))] +mod non_threading { + use crate::lock::OnceCell; + use std::thread::LocalKey; + + pub struct StaticCell<T: 'static> { + inner: &'static LocalKey<OnceCell<&'static T>>, + } + + fn leak<T>(x: T) -> &'static T { + Box::leak(Box::new(x)) + } + + impl<T> StaticCell<T> { + #[doc(hidden)] + pub const fn _from_local_key(inner: &'static LocalKey<OnceCell<&'static T>>) -> Self { + Self { inner } + } + + pub fn get(&'static self) -> Option<&'static T> { + self.inner.with(|x| x.get().copied()) + } + + pub fn set(&'static self, value: T) -> Result<(), T> { + self.inner.with(|x| { + if x.get().is_some() { + Err(value) + } else { + let _ = x.set(leak(value)); + Ok(()) + } + }) + } + + pub fn get_or_init<F>(&'static self, f: F) -> &'static T + where + F: FnOnce() -> T, + { + self.inner.with(|x| *x.get_or_init(|| leak(f()))) + } + + pub fn get_or_try_init<F, E>(&'static self, f: F) -> Result<&'static T, E> + where + F: FnOnce() -> Result<T, E>, + { + self.inner.with(|x| { + if let Some(val) = x.get() { + Ok(*val) + } else { + let val = leak(f()?); + let _ = x.set(val); + Ok(val) + } + }) + } + } + + #[macro_export] + macro_rules! static_cell { + ($($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty;)+) => { + $($(#[$attr])* + $vis static $name: $crate::static_cell::StaticCell<$t> = { + ::std::thread_local! { + $vis static $name: $crate::lock::OnceCell<&'static $t> = const { + $crate::lock::OnceCell::new() + }; + } + $crate::static_cell::StaticCell::_from_local_key(&$name) + };)+ + }; + } +} +#[cfg(all(not(feature = "threading"), feature = "std"))] +pub use non_threading::*; + +// Same as `threading` variant, but wraps unsync::OnceCell with Sync. +#[cfg(all(not(feature = "threading"), not(feature = "std")))] +mod no_std { + use crate::lock::OnceCell; + + // unsync::OnceCell is !Sync, but without std there can be no threads. + struct SyncOnceCell<T>(OnceCell<T>); + // SAFETY: Without std, threading is impossible. + unsafe impl<T> Sync for SyncOnceCell<T> {} + + pub struct StaticCell<T: 'static> { + inner: SyncOnceCell<T>, + } + + impl<T> StaticCell<T> { + #[doc(hidden)] + pub const fn _from_once_cell(inner: OnceCell<T>) -> Self { + Self { + inner: SyncOnceCell(inner), + } + } + + pub fn get(&'static self) -> Option<&'static T> { + self.inner.0.get() + } + + pub fn set(&'static self, value: T) -> Result<(), T> { + self.inner.0.set(value) + } + + pub fn get_or_init<F>(&'static self, f: F) -> &'static T + where + F: FnOnce() -> T, + { + self.inner.0.get_or_init(f) + } + + pub fn get_or_try_init<F, E>(&'static self, f: F) -> Result<&'static T, E> + where + F: FnOnce() -> Result<T, E>, + { + if let Some(val) = self.inner.0.get() { + return Ok(val); + } + let val = f()?; + let _ = self.inner.0.set(val); + Ok(self.inner.0.get().unwrap()) + } + } + + #[macro_export] + macro_rules! static_cell { + ($($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty;)+) => { + $($(#[$attr])* + $vis static $name: $crate::static_cell::StaticCell<$t> = + $crate::static_cell::StaticCell::_from_once_cell($crate::lock::OnceCell::new());)+ + }; + } +} +#[cfg(all(not(feature = "threading"), not(feature = "std")))] +pub use no_std::*; diff --git a/crates/common/src/str.rs b/crates/common/src/str.rs new file mode 100644 index 00000000000..38e73a683f2 --- /dev/null +++ b/crates/common/src/str.rs @@ -0,0 +1,669 @@ +// spell-checker:ignore uncomputed +use crate::atomic::{PyAtomic, Radium}; +use crate::format::CharLen; +use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; +use ascii::{AsciiChar, AsciiStr, AsciiString}; +use core::fmt; +use core::ops::{Bound, RangeBounds}; +use core::sync::atomic::Ordering::Relaxed; + +#[cfg(not(target_arch = "wasm32"))] +#[allow(non_camel_case_types)] +pub type wchar_t = libc::wchar_t; +#[cfg(target_arch = "wasm32")] +#[allow(non_camel_case_types)] +pub type wchar_t = u32; + +/// Utf8 + state.ascii (+ PyUnicode_Kind in future) +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum StrKind { + Ascii, + Utf8, + Wtf8, +} + +impl core::ops::BitOr for StrKind { + type Output = Self; + + fn bitor(self, other: Self) -> Self { + use StrKind::*; + match (self, other) { + (Wtf8, _) | (_, Wtf8) => Wtf8, + (Utf8, _) | (_, Utf8) => Utf8, + (Ascii, Ascii) => Ascii, + } + } +} + +impl StrKind { + pub const fn is_ascii(&self) -> bool { + matches!(self, Self::Ascii) + } + + pub const fn is_utf8(&self) -> bool { + matches!(self, Self::Ascii | Self::Utf8) + } + + #[inline(always)] + pub fn can_encode(&self, code: CodePoint) -> bool { + match self { + Self::Ascii => code.is_ascii(), + Self::Utf8 => code.to_char().is_some(), + Self::Wtf8 => true, + } + } +} + +pub trait DeduceStrKind { + fn str_kind(&self) -> StrKind; +} + +impl DeduceStrKind for str { + fn str_kind(&self) -> StrKind { + if self.is_ascii() { + StrKind::Ascii + } else { + StrKind::Utf8 + } + } +} + +impl DeduceStrKind for Wtf8 { + fn str_kind(&self) -> StrKind { + if self.is_ascii() { + StrKind::Ascii + } else if self.is_utf8() { + StrKind::Utf8 + } else { + StrKind::Wtf8 + } + } +} + +impl DeduceStrKind for String { + fn str_kind(&self) -> StrKind { + (**self).str_kind() + } +} + +impl DeduceStrKind for Wtf8Buf { + fn str_kind(&self) -> StrKind { + (**self).str_kind() + } +} + +impl<T: DeduceStrKind + ?Sized> DeduceStrKind for &T { + fn str_kind(&self) -> StrKind { + (**self).str_kind() + } +} + +impl<T: DeduceStrKind + ?Sized> DeduceStrKind for Box<T> { + fn str_kind(&self) -> StrKind { + (**self).str_kind() + } +} + +#[derive(Debug)] +pub enum PyKindStr<'a> { + Ascii(&'a AsciiStr), + Utf8(&'a str), + Wtf8(&'a Wtf8), +} + +#[derive(Debug, Clone)] +pub struct StrData { + data: Box<Wtf8>, + kind: StrKind, + len: StrLen, +} + +struct StrLen(PyAtomic<usize>); + +impl From<usize> for StrLen { + #[inline(always)] + fn from(value: usize) -> Self { + Self(Radium::new(value)) + } +} + +impl fmt::Debug for StrLen { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let len = self.0.load(Relaxed); + if len == usize::MAX { + f.write_str("<uncomputed>") + } else { + len.fmt(f) + } + } +} + +impl StrLen { + #[inline(always)] + fn zero() -> Self { + 0usize.into() + } + + #[inline(always)] + fn uncomputed() -> Self { + usize::MAX.into() + } +} + +impl Clone for StrLen { + fn clone(&self) -> Self { + Self(self.0.load(Relaxed).into()) + } +} + +impl Default for StrData { + fn default() -> Self { + Self { + data: <Box<Wtf8>>::default(), + kind: StrKind::Ascii, + len: StrLen::zero(), + } + } +} + +impl From<Box<Wtf8>> for StrData { + fn from(value: Box<Wtf8>) -> Self { + // doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for + // non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532 + let kind = value.str_kind(); + unsafe { Self::new_str_unchecked(value, kind) } + } +} + +impl From<Box<str>> for StrData { + #[inline] + fn from(value: Box<str>) -> Self { + // doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for + // non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532 + let kind = value.str_kind(); + unsafe { Self::new_str_unchecked(value.into(), kind) } + } +} + +impl From<Box<AsciiStr>> for StrData { + #[inline] + fn from(value: Box<AsciiStr>) -> Self { + Self { + len: value.len().into(), + data: value.into(), + kind: StrKind::Ascii, + } + } +} + +impl From<AsciiChar> for StrData { + fn from(ch: AsciiChar) -> Self { + AsciiString::from(ch).into_boxed_ascii_str().into() + } +} + +impl From<char> for StrData { + fn from(ch: char) -> Self { + if let Ok(ch) = ascii::AsciiChar::from_ascii(ch) { + ch.into() + } else { + Self { + data: ch.to_string().into(), + kind: StrKind::Utf8, + len: 1.into(), + } + } + } +} + +impl From<CodePoint> for StrData { + fn from(ch: CodePoint) -> Self { + if let Some(ch) = ch.to_char() { + ch.into() + } else { + Self { + data: Wtf8Buf::from(ch).into(), + kind: StrKind::Wtf8, + len: 1.into(), + } + } + } +} + +impl StrData { + /// # Safety + /// + /// Given `bytes` must be valid data for given `kind` + pub unsafe fn new_str_unchecked(data: Box<Wtf8>, kind: StrKind) -> Self { + let len = match kind { + StrKind::Ascii => data.len().into(), + _ => StrLen::uncomputed(), + }; + Self { data, kind, len } + } + + /// # Safety + /// + /// `char_len` must be accurate. + pub unsafe fn new_with_char_len(data: Box<Wtf8>, kind: StrKind, char_len: usize) -> Self { + Self { + data, + kind, + len: char_len.into(), + } + } + + #[inline] + pub const fn as_wtf8(&self) -> &Wtf8 { + &self.data + } + + // TODO: rename to to_str + #[inline] + pub fn as_str(&self) -> Option<&str> { + self.kind + .is_utf8() + .then(|| unsafe { core::str::from_utf8_unchecked(self.data.as_bytes()) }) + } + + pub fn as_ascii(&self) -> Option<&AsciiStr> { + self.kind + .is_ascii() + .then(|| unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) }) + } + + pub const fn kind(&self) -> StrKind { + self.kind + } + + #[inline] + pub fn as_str_kind(&self) -> PyKindStr<'_> { + match self.kind { + StrKind::Ascii => { + PyKindStr::Ascii(unsafe { AsciiStr::from_ascii_unchecked(self.data.as_bytes()) }) + } + StrKind::Utf8 => { + PyKindStr::Utf8(unsafe { core::str::from_utf8_unchecked(self.data.as_bytes()) }) + } + StrKind::Wtf8 => PyKindStr::Wtf8(&self.data), + } + } + + #[inline] + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + + #[inline] + pub fn char_len(&self) -> usize { + match self.len.0.load(Relaxed) { + usize::MAX => self._compute_char_len(), + len => len, + } + } + + #[cold] + fn _compute_char_len(&self) -> usize { + let len = if let Some(s) = self.as_str() { + // utf8 chars().count() is optimized + s.chars().count() + } else { + self.data.code_points().count() + }; + // len cannot be usize::MAX, since vec.capacity() < sys.maxsize + self.len.0.store(len, Relaxed); + len + } + + pub fn nth_char(&self, index: usize) -> CodePoint { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s[index].into(), + PyKindStr::Utf8(s) => s.chars().nth(index).unwrap().into(), + PyKindStr::Wtf8(w) => w.code_points().nth(index).unwrap(), + } + } +} + +impl core::fmt::Display for StrData { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.data.fmt(f) + } +} + +impl CharLen for StrData { + fn char_len(&self) -> usize { + self.char_len() + } +} + +pub fn try_get_chars(s: &str, range: impl RangeBounds<usize>) -> Option<&str> { + let mut chars = s.chars(); + let start = match range.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => i + 1, + Bound::Unbounded => 0, + }; + for _ in 0..start { + chars.next()?; + } + let s = chars.as_str(); + let range_len = match range.end_bound() { + Bound::Included(&i) => i + 1 - start, + Bound::Excluded(&i) => i - start, + Bound::Unbounded => return Some(s), + }; + char_range_end(s, range_len).map(|end| &s[..end]) +} + +pub fn get_chars(s: &str, range: impl RangeBounds<usize>) -> &str { + try_get_chars(s, range).unwrap() +} + +#[inline] +pub fn char_range_end(s: &str, n_chars: usize) -> Option<usize> { + let i = match n_chars.checked_sub(1) { + Some(last_char_index) => { + let (index, c) = s.char_indices().nth(last_char_index)?; + index + c.len_utf8() + } + None => 0, + }; + Some(i) +} + +pub fn try_get_codepoints(w: &Wtf8, range: impl RangeBounds<usize>) -> Option<&Wtf8> { + let mut chars = w.code_points(); + let start = match range.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => i + 1, + Bound::Unbounded => 0, + }; + for _ in 0..start { + chars.next()?; + } + let s = chars.as_wtf8(); + let range_len = match range.end_bound() { + Bound::Included(&i) => i + 1 - start, + Bound::Excluded(&i) => i - start, + Bound::Unbounded => return Some(s), + }; + codepoint_range_end(s, range_len).map(|end| &s[..end]) +} + +pub fn get_codepoints(w: &Wtf8, range: impl RangeBounds<usize>) -> &Wtf8 { + try_get_codepoints(w, range).unwrap() +} + +#[inline] +pub fn codepoint_range_end(s: &Wtf8, n_chars: usize) -> Option<usize> { + let i = match n_chars.checked_sub(1) { + Some(last_char_index) => { + let (index, c) = s.code_point_indices().nth(last_char_index)?; + index + c.len_wtf8() + } + None => 0, + }; + Some(i) +} + +pub fn zfill(bytes: &[u8], width: usize) -> Vec<u8> { + if width <= bytes.len() { + bytes.to_vec() + } else { + let (sign, s) = match bytes.first() { + Some(_sign @ b'+') | Some(_sign @ b'-') => { + (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]) + } + _ => (&b""[..], bytes), + }; + let mut filled = Vec::new(); + filled.extend_from_slice(sign); + filled.extend(core::iter::repeat_n(b'0', width - bytes.len())); + filled.extend_from_slice(s); + filled + } +} + +/// Convert a string to ascii compatible, escaping unicode-s into escape +/// sequences. +pub fn to_ascii(value: &Wtf8) -> AsciiString { + let mut ascii = Vec::new(); + for cp in value.code_points() { + if cp.is_ascii() { + ascii.push(cp.to_u32() as u8); + } else { + let c = cp.to_u32(); + let hex = if c < 0x100 { + format!("\\x{c:02x}") + } else if c < 0x10000 { + format!("\\u{c:04x}") + } else { + format!("\\U{c:08x}") + }; + ascii.append(&mut hex.into_bytes()); + } + } + unsafe { AsciiString::from_ascii_unchecked(ascii) } +} + +#[derive(Clone, Copy)] +pub struct UnicodeEscapeCodepoint(pub CodePoint); + +impl fmt::Display for UnicodeEscapeCodepoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let c = self.0.to_u32(); + if c >= 0x10000 { + write!(f, "\\U{c:08x}") + } else if c >= 0x100 { + write!(f, "\\u{c:04x}") + } else { + write!(f, "\\x{c:02x}") + } + } +} + +pub mod levenshtein { + pub const MOVE_COST: usize = 2; + const CASE_COST: usize = 1; + const MAX_STRING_SIZE: usize = 40; + + const fn substitution_cost(mut a: u8, mut b: u8) -> usize { + if (a & 31) != (b & 31) { + return MOVE_COST; + } + if a == b { + return 0; + } + if a.is_ascii_uppercase() { + a += b'a' - b'A'; + } + if b.is_ascii_uppercase() { + b += b'a' - b'A'; + } + if a == b { CASE_COST } else { MOVE_COST } + } + + pub fn levenshtein_distance(a: &[u8], b: &[u8], max_cost: usize) -> usize { + if a == b { + return 0; + } + + let (mut a_bytes, mut b_bytes) = (a, b); + let (mut a_begin, mut a_end) = (0usize, a.len()); + let (mut b_begin, mut b_end) = (0usize, b.len()); + + while a_end > 0 && b_end > 0 && (a_bytes[a_begin] == b_bytes[b_begin]) { + a_begin += 1; + b_begin += 1; + a_end -= 1; + b_end -= 1; + } + while a_end > 0 + && b_end > 0 + && (a_bytes[a_begin + a_end - 1] == b_bytes[b_begin + b_end - 1]) + { + a_end -= 1; + b_end -= 1; + } + if a_end == 0 || b_end == 0 { + return (a_end + b_end) * MOVE_COST; + } + if a_end > MAX_STRING_SIZE || b_end > MAX_STRING_SIZE { + return max_cost + 1; + } + + if b_end < a_end { + core::mem::swap(&mut a_bytes, &mut b_bytes); + core::mem::swap(&mut a_begin, &mut b_begin); + core::mem::swap(&mut a_end, &mut b_end); + } + + if (b_end - a_end) * MOVE_COST > max_cost { + return max_cost + 1; + } + + let mut buffer = [0usize; MAX_STRING_SIZE]; + + for (i, x) in buffer.iter_mut().take(a_end).enumerate() { + *x = (i + 1) * MOVE_COST; + } + + let mut result = 0usize; + for (b_index, b_code) in b_bytes[b_begin..(b_begin + b_end)].iter().enumerate() { + result = b_index * MOVE_COST; + let mut distance = result; + let mut minimum = usize::MAX; + for (a_index, a_code) in a_bytes[a_begin..(a_begin + a_end)].iter().enumerate() { + let substitute = distance + substitution_cost(*b_code, *a_code); + distance = buffer[a_index]; + let insert_delete = usize::min(result, distance) + MOVE_COST; + result = usize::min(insert_delete, substitute); + + buffer[a_index] = result; + if result < minimum { + minimum = result; + } + } + if minimum > max_cost { + return max_cost + 1; + } + } + result + } +} + +/// Replace all tabs in a string with spaces, using the given tab size. +pub fn expandtabs(input: &str, tab_size: usize) -> String { + let tab_stop = tab_size; + let mut expanded_str = String::with_capacity(input.len()); + let mut tab_size = tab_stop; + let mut col_count = 0usize; + for ch in input.chars() { + match ch { + '\t' => { + let num_spaces = tab_size - col_count; + col_count += num_spaces; + let expand = " ".repeat(num_spaces); + expanded_str.push_str(&expand); + } + '\r' | '\n' => { + expanded_str.push(ch); + col_count = 0; + tab_size = 0; + } + _ => { + expanded_str.push(ch); + col_count += 1; + } + } + if col_count >= tab_size { + tab_size += tab_stop; + } + } + expanded_str +} + +/// Creates an [`AsciiStr`][ascii::AsciiStr] from a string literal, throwing a compile error if the +/// literal isn't actually ascii. +/// +/// ```compile_fail +/// # use rustpython_common::str::ascii; +/// ascii!("I ❤️ Rust & Python"); +/// ``` +#[macro_export] +macro_rules! ascii { + ($x:expr $(,)?) => {{ + let s = const { + let s: &str = $x; + assert!(s.is_ascii(), "ascii!() argument is not an ascii string"); + s + }; + unsafe { $crate::vendored::ascii::AsciiStr::from_ascii_unchecked(s.as_bytes()) } + }}; +} +pub use ascii; + +// TODO: this should probably live in a crate like unic or unicode-properties +const UNICODE_DECIMAL_VALUES: &[char] = &[ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '٠', '١', '٢', '٣', '٤', '٥', '٦', '٧', '٨', + '٩', '۰', '۱', '۲', '۳', '۴', '۵', '۶', '۷', '۸', '۹', '߀', '߁', '߂', '߃', '߄', '߅', '߆', '߇', + '߈', '߉', '०', '१', '२', '३', '४', '५', '६', '७', '८', '९', '০', '১', '২', '৩', '৪', '৫', '৬', + '৭', '৮', '৯', '੦', '੧', '੨', '੩', '੪', '੫', '੬', '੭', '੮', '੯', '૦', '૧', '૨', '૩', '૪', '૫', + '૬', '૭', '૮', '૯', '୦', '୧', '୨', '୩', '୪', '୫', '୬', '୭', '୮', '୯', '௦', '௧', '௨', '௩', '௪', + '௫', '௬', '௭', '௮', '௯', '౦', '౧', '౨', '౩', '౪', '౫', '౬', '౭', '౮', '౯', '೦', '೧', '೨', '೩', + '೪', '೫', '೬', '೭', '೮', '೯', '൦', '൧', '൨', '൩', '൪', '൫', '൬', '൭', '൮', '൯', '෦', '෧', '෨', + '෩', '෪', '෫', '෬', '෭', '෮', '෯', '๐', '๑', '๒', '๓', '๔', '๕', '๖', '๗', '๘', '๙', '໐', '໑', + '໒', '໓', '໔', '໕', '໖', '໗', '໘', '໙', '༠', '༡', '༢', '༣', '༤', '༥', '༦', '༧', '༨', '༩', '၀', + '၁', '၂', '၃', '၄', '၅', '၆', '၇', '၈', '၉', '႐', '႑', '႒', '႓', '႔', '႕', '႖', '႗', '႘', '႙', + '០', '១', '២', '៣', '៤', '៥', '៦', '៧', '៨', '៩', '᠐', '᠑', '᠒', '᠓', '᠔', '᠕', '᠖', '᠗', '᠘', + '᠙', '᥆', '᥇', '᥈', '᥉', '᥊', '᥋', '᥌', '᥍', '᥎', '᥏', '᧐', '᧑', '᧒', '᧓', '᧔', '᧕', '᧖', '᧗', + '᧘', '᧙', '᪀', '᪁', '᪂', '᪃', '᪄', '᪅', '᪆', '᪇', '᪈', '᪉', '᪐', '᪑', '᪒', '᪓', '᪔', '᪕', '᪖', + '᪗', '᪘', '᪙', '᭐', '᭑', '᭒', '᭓', '᭔', '᭕', '᭖', '᭗', '᭘', '᭙', '᮰', '᮱', '᮲', '᮳', '᮴', '᮵', + '᮶', '᮷', '᮸', '᮹', '᱀', '᱁', '᱂', '᱃', '᱄', '᱅', '᱆', '᱇', '᱈', '᱉', '᱐', '᱑', '᱒', '᱓', '᱔', + '᱕', '᱖', '᱗', '᱘', '᱙', '꘠', '꘡', '꘢', '꘣', '꘤', '꘥', '꘦', '꘧', '꘨', '꘩', '꣐', '꣑', '꣒', '꣓', + '꣔', '꣕', '꣖', '꣗', '꣘', '꣙', '꤀', '꤁', '꤂', '꤃', '꤄', '꤅', '꤆', '꤇', '꤈', '꤉', '꧐', '꧑', '꧒', + '꧓', '꧔', '꧕', '꧖', '꧗', '꧘', '꧙', '꧰', '꧱', '꧲', '꧳', '꧴', '꧵', '꧶', '꧷', '꧸', '꧹', '꩐', '꩑', + '꩒', '꩓', '꩔', '꩕', '꩖', '꩗', '꩘', '꩙', '꯰', '꯱', '꯲', '꯳', '꯴', '꯵', '꯶', '꯷', '꯸', '꯹', '０', + '１', '２', '３', '４', '５', '６', '７', '８', '９', '𐒠', '𐒡', '𐒢', '𐒣', '𐒤', '𐒥', '𐒦', '𐒧', + '𐒨', '𐒩', '𑁦', '𑁧', '𑁨', '𑁩', '𑁪', '𑁫', '𑁬', '𑁭', '𑁮', '𑁯', '𑃰', '𑃱', '𑃲', '𑃳', '𑃴', '𑃵', '𑃶', + '𑃷', '𑃸', '𑃹', '𑄶', '𑄷', '𑄸', '𑄹', '𑄺', '𑄻', '𑄼', '𑄽', '𑄾', '𑄿', '𑇐', '𑇑', '𑇒', '𑇓', '𑇔', '𑇕', + '𑇖', '𑇗', '𑇘', '𑇙', '𑋰', '𑋱', '𑋲', '𑋳', '𑋴', '𑋵', '𑋶', '𑋷', '𑋸', '𑋹', '𑑐', '𑑑', '𑑒', '𑑓', '𑑔', + '𑑕', '𑑖', '𑑗', '𑑘', '𑑙', '𑓐', '𑓑', '𑓒', '𑓓', '𑓔', '𑓕', '𑓖', '𑓗', '𑓘', '𑓙', '𑙐', '𑙑', '𑙒', '𑙓', + '𑙔', '𑙕', '𑙖', '𑙗', '𑙘', '𑙙', '𑛀', '𑛁', '𑛂', '𑛃', '𑛄', '𑛅', '𑛆', '𑛇', '𑛈', '𑛉', '𑜰', '𑜱', '𑜲', + '𑜳', '𑜴', '𑜵', '𑜶', '𑜷', '𑜸', '𑜹', '𑣠', '𑣡', '𑣢', '𑣣', '𑣤', '𑣥', '𑣦', '𑣧', '𑣨', '𑣩', '𑱐', '𑱑', + '𑱒', '𑱓', '𑱔', '𑱕', '𑱖', '𑱗', '𑱘', '𑱙', '𑵐', '𑵑', '𑵒', '𑵓', '𑵔', '𑵕', '𑵖', '𑵗', '𑵘', '𑵙', '𖩠', + '𖩡', '𖩢', '𖩣', '𖩤', '𖩥', '𖩦', '𖩧', '𖩨', '𖩩', '𖭐', '𖭑', '𖭒', '𖭓', '𖭔', '𖭕', '𖭖', '𖭗', '𖭘', '𖭙', + '𝟎', '𝟏', '𝟐', '𝟑', '𝟒', '𝟓', '𝟔', '𝟕', '𝟖', '𝟗', '𝟘', '𝟙', '𝟚', '𝟛', '𝟜', '𝟝', '𝟞', '𝟟', '𝟠', + '𝟡', '𝟢', '𝟣', '𝟤', '𝟥', '𝟦', '𝟧', '𝟨', '𝟩', '𝟪', '𝟫', '𝟬', '𝟭', '𝟮', '𝟯', '𝟰', '𝟱', '𝟲', '𝟳', + '𝟴', '𝟵', '𝟶', '𝟷', '𝟸', '𝟹', '𝟺', '𝟻', '𝟼', '𝟽', '𝟾', '𝟿', '𞥐', '𞥑', '𞥒', '𞥓', '𞥔', '𞥕', '𞥖', + '𞥗', '𞥘', '𞥙', +]; + +pub fn char_to_decimal(ch: char) -> Option<u8> { + UNICODE_DECIMAL_VALUES + .binary_search(&ch) + .ok() + .map(|i| (i % 10) as u8) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_chars() { + let s = "0123456789"; + assert_eq!(get_chars(s, 3..7), "3456"); + assert_eq!(get_chars(s, 3..7), &s[3..7]); + + let s = "0유니코드 문자열9"; + assert_eq!(get_chars(s, 3..7), "코드 문"); + + let s = "0😀😃😄😁😆😅😂🤣9"; + assert_eq!(get_chars(s, 3..7), "😄😁😆😅"); + } +} diff --git a/crates/common/src/windows.rs b/crates/common/src/windows.rs new file mode 100644 index 00000000000..a934a118ecf --- /dev/null +++ b/crates/common/src/windows.rs @@ -0,0 +1,56 @@ +use rustpython_wtf8::Wtf8; +use std::{ + ffi::{OsStr, OsString}, + os::windows::ffi::{OsStrExt, OsStringExt}, +}; + +/// _MAX_ENV from Windows CRT stdlib.h - maximum environment variable size +pub const _MAX_ENV: usize = 32767; + +pub trait ToWideString { + fn to_wide(&self) -> Vec<u16>; + fn to_wide_with_nul(&self) -> Vec<u16>; +} + +impl<T> ToWideString for T +where + T: AsRef<OsStr>, +{ + fn to_wide(&self) -> Vec<u16> { + self.as_ref().encode_wide().collect() + } + fn to_wide_with_nul(&self) -> Vec<u16> { + self.as_ref().encode_wide().chain(Some(0)).collect() + } +} + +impl ToWideString for OsStr { + fn to_wide(&self) -> Vec<u16> { + self.encode_wide().collect() + } + fn to_wide_with_nul(&self) -> Vec<u16> { + self.encode_wide().chain(Some(0)).collect() + } +} + +impl ToWideString for Wtf8 { + fn to_wide(&self) -> Vec<u16> { + self.encode_wide().collect() + } + fn to_wide_with_nul(&self) -> Vec<u16> { + self.encode_wide().chain(Some(0)).collect() + } +} + +pub trait FromWideString +where + Self: Sized, +{ + fn from_wides_until_nul(wide: &[u16]) -> Self; +} +impl FromWideString for OsString { + fn from_wides_until_nul(wide: &[u16]) -> OsString { + let len = wide.iter().take_while(|&&c| c != 0).count(); + OsString::from_wide(&wide[..len]) + } +} diff --git a/crates/compiler-core/Cargo.toml b/crates/compiler-core/Cargo.toml new file mode 100644 index 00000000000..7be58432cdf --- /dev/null +++ b/crates/compiler-core/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "rustpython-compiler-core" +description = "RustPython specific bytecode." +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +# rustpython-parser-core = { workspace = true, features=["location"] } +ruff_source_file = { workspace = true } +rustpython-wtf8 = { workspace = true } + +bitflags = { workspace = true } +bitflagset = { workspace = true } +itertools = { workspace = true } +malachite-bigint = { workspace = true } +num-complex = { workspace = true } + +lz4_flex = "0.12" + +[lints] +workspace = true diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs new file mode 100644 index 00000000000..5120d371a39 --- /dev/null +++ b/crates/compiler-core/src/bytecode.rs @@ -0,0 +1,1257 @@ +//! Implement python as a virtual machine with bytecode. This module +//! implements bytecode structure. + +use crate::{ + marshal::MarshalError, + varint::{read_varint, read_varint_with_start, write_varint, write_varint_with_start}, + {OneIndexed, SourceLocation}, +}; +use alloc::{borrow::ToOwned, boxed::Box, collections::BTreeSet, fmt, string::String, vec::Vec}; +use bitflags::bitflags; +use core::{ + cell::UnsafeCell, + hash, mem, + ops::{Deref, Index, IndexMut}, + sync::atomic::{AtomicU8, AtomicU16, AtomicUsize, Ordering}, +}; +use itertools::Itertools; +use malachite_bigint::BigInt; +use num_complex::Complex64; +use rustpython_wtf8::{Wtf8, Wtf8Buf}; + +pub use crate::bytecode::{ + instruction::{ + AnyInstruction, Arg, Instruction, InstructionMetadata, PseudoInstruction, StackEffect, + }, + oparg::{ + BinaryOperator, BuildSliceArgCount, CommonConstant, ComparisonOperator, ConvertValueOparg, + IntrinsicFunction1, IntrinsicFunction2, Invert, Label, LoadAttr, LoadSuperAttr, + MakeFunctionFlag, MakeFunctionFlags, NameIdx, OpArg, OpArgByte, OpArgState, OpArgType, + RaiseKind, ResumeType, SpecialMethod, UnpackExArgs, + }, +}; + +mod instruction; +pub mod oparg; + +/// Exception table entry for zero-cost exception handling +/// Format: (start, size, target, depth<<1|lasti) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ExceptionTableEntry { + /// Start instruction offset (inclusive) + pub start: u32, + /// End instruction offset (exclusive) + pub end: u32, + /// Handler target offset + pub target: u32, + /// Stack depth at handler entry + pub depth: u16, + /// Whether to push lasti before exception + pub push_lasti: bool, +} + +impl ExceptionTableEntry { + pub const fn new(start: u32, end: u32, target: u32, depth: u16, push_lasti: bool) -> Self { + Self { + start, + end, + target, + depth, + push_lasti, + } + } +} + +/// Encode exception table entries. +/// Uses 6-bit varint encoding with start marker (MSB) and continuation bit. +pub fn encode_exception_table(entries: &[ExceptionTableEntry]) -> alloc::boxed::Box<[u8]> { + let mut data = Vec::new(); + for entry in entries { + let size = entry.end.saturating_sub(entry.start); + let depth_lasti = ((entry.depth as u32) << 1) | (entry.push_lasti as u32); + + write_varint_with_start(&mut data, entry.start); + write_varint(&mut data, size); + write_varint(&mut data, entry.target); + write_varint(&mut data, depth_lasti); + } + data.into_boxed_slice() +} + +/// Find exception handler for given instruction offset. +pub fn find_exception_handler(table: &[u8], offset: u32) -> Option<ExceptionTableEntry> { + let mut pos = 0; + while pos < table.len() { + let start = read_varint_with_start(table, &mut pos)?; + let size = read_varint(table, &mut pos)?; + let target = read_varint(table, &mut pos)?; + let depth_lasti = read_varint(table, &mut pos)?; + + let end = start + size; + let depth = (depth_lasti >> 1) as u16; + let push_lasti = (depth_lasti & 1) != 0; + + if offset >= start && offset < end { + return Some(ExceptionTableEntry { + start, + end, + target, + depth, + push_lasti, + }); + } + } + None +} + +/// Decode all exception table entries. +pub fn decode_exception_table(table: &[u8]) -> Vec<ExceptionTableEntry> { + let mut entries = Vec::new(); + let mut pos = 0; + while pos < table.len() { + let Some(start) = read_varint_with_start(table, &mut pos) else { + break; + }; + let Some(size) = read_varint(table, &mut pos) else { + break; + }; + let Some(target) = read_varint(table, &mut pos) else { + break; + }; + let Some(depth_lasti) = read_varint(table, &mut pos) else { + break; + }; + let Some(end) = start.checked_add(size) else { + break; + }; + entries.push(ExceptionTableEntry { + start, + end, + target, + depth: (depth_lasti >> 1) as u16, + push_lasti: (depth_lasti & 1) != 0, + }); + } + entries +} + +/// CPython 3.11+ linetable location info codes +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum PyCodeLocationInfoKind { + // Short forms are 0 to 9 + Short0 = 0, + Short1 = 1, + Short2 = 2, + Short3 = 3, + Short4 = 4, + Short5 = 5, + Short6 = 6, + Short7 = 7, + Short8 = 8, + Short9 = 9, + // One line forms are 10 to 12 + OneLine0 = 10, + OneLine1 = 11, + OneLine2 = 12, + NoColumns = 13, + Long = 14, + None = 15, +} + +impl PyCodeLocationInfoKind { + pub fn from_code(code: u8) -> Option<Self> { + match code { + 0 => Some(Self::Short0), + 1 => Some(Self::Short1), + 2 => Some(Self::Short2), + 3 => Some(Self::Short3), + 4 => Some(Self::Short4), + 5 => Some(Self::Short5), + 6 => Some(Self::Short6), + 7 => Some(Self::Short7), + 8 => Some(Self::Short8), + 9 => Some(Self::Short9), + 10 => Some(Self::OneLine0), + 11 => Some(Self::OneLine1), + 12 => Some(Self::OneLine2), + 13 => Some(Self::NoColumns), + 14 => Some(Self::Long), + 15 => Some(Self::None), + _ => Option::None, + } + } + + pub fn is_short(&self) -> bool { + (*self as u8) <= 9 + } + + pub fn short_column_group(&self) -> Option<u8> { + if self.is_short() { + Some(*self as u8) + } else { + Option::None + } + } + + pub fn one_line_delta(&self) -> Option<i32> { + match self { + Self::OneLine0 => Some(0), + Self::OneLine1 => Some(1), + Self::OneLine2 => Some(2), + _ => Option::None, + } + } +} + +pub trait Constant: Sized { + type Name: AsRef<str>; + + /// Transforms the given Constant to a BorrowedConstant + fn borrow_constant(&self) -> BorrowedConstant<'_, Self>; +} + +impl Constant for ConstantData { + type Name = String; + + fn borrow_constant(&self) -> BorrowedConstant<'_, Self> { + use BorrowedConstant::*; + + match self { + Self::Integer { value } => Integer { value }, + Self::Float { value } => Float { value: *value }, + Self::Complex { value } => Complex { value: *value }, + Self::Boolean { value } => Boolean { value: *value }, + Self::Str { value } => Str { value }, + Self::Bytes { value } => Bytes { value }, + Self::Code { code } => Code { code }, + Self::Tuple { elements } => Tuple { elements }, + Self::None => None, + Self::Ellipsis => Ellipsis, + } + } +} + +/// A Constant Bag +pub trait ConstantBag: Sized + Copy { + type Constant: Constant; + + fn make_constant<C: Constant>(&self, constant: BorrowedConstant<'_, C>) -> Self::Constant; + + fn make_int(&self, value: BigInt) -> Self::Constant; + + fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant; + + fn make_code(&self, code: CodeObject<Self::Constant>) -> Self::Constant; + + fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name; +} + +pub trait AsBag { + type Bag: ConstantBag; + + #[allow(clippy::wrong_self_convention)] + fn as_bag(self) -> Self::Bag; +} + +impl<Bag: ConstantBag> AsBag for Bag { + type Bag = Self; + + fn as_bag(self) -> Self { + self + } +} + +#[derive(Clone, Copy)] +pub struct BasicBag; + +impl ConstantBag for BasicBag { + type Constant = ConstantData; + + fn make_constant<C: Constant>(&self, constant: BorrowedConstant<'_, C>) -> Self::Constant { + constant.to_owned() + } + + fn make_int(&self, value: BigInt) -> Self::Constant { + ConstantData::Integer { value } + } + + fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant { + ConstantData::Tuple { + elements: elements.collect(), + } + } + + fn make_code(&self, code: CodeObject<Self::Constant>) -> Self::Constant { + ConstantData::Code { + code: Box::new(code), + } + } + + fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name { + name.to_owned() + } +} + +#[derive(Clone)] +pub struct Constants<C: Constant>(Box<[C]>); + +impl<C: Constant> Deref for Constants<C> { + type Target = [C]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<C: Constant> Index<oparg::ConstIdx> for Constants<C> { + type Output = C; + + fn index(&self, consti: oparg::ConstIdx) -> &Self::Output { + &self.0[consti.as_usize()] + } +} + +impl<C: Constant> FromIterator<C> for Constants<C> { + fn from_iter<T: IntoIterator<Item = C>>(iter: T) -> Self { + Self(iter.into_iter().collect()) + } +} + +// TODO: Newtype "CodeObject.varnames". Make sure only `oparg:VarNum` can be used as index +impl<T> Index<oparg::VarNum> for [T] { + type Output = T; + + fn index(&self, var_num: oparg::VarNum) -> &Self::Output { + &self[var_num.as_usize()] + } +} + +// TODO: Newtype "CodeObject.varnames". Make sure only `oparg:VarNum` can be used as index +impl<T> IndexMut<oparg::VarNum> for [T] { + fn index_mut(&mut self, var_num: oparg::VarNum) -> &mut Self::Output { + &mut self[var_num.as_usize()] + } +} + +/// Primary container of a single code object. Each python function has +/// a code object. Also a module has a code object. +#[derive(Clone)] +pub struct CodeObject<C: Constant = ConstantData> { + pub instructions: CodeUnits, + pub locations: Box<[(SourceLocation, SourceLocation)]>, + pub flags: CodeFlags, + /// Number of positional-only arguments + pub posonlyarg_count: u32, + pub arg_count: u32, + pub kwonlyarg_count: u32, + pub source_path: C::Name, + pub first_line_number: Option<OneIndexed>, + pub max_stackdepth: u32, + /// Name of the object that created this code object + pub obj_name: C::Name, + /// Qualified name of the object (like CPython's co_qualname) + pub qualname: C::Name, + pub cell2arg: Option<Box<[i32]>>, + pub constants: Constants<C>, + pub names: Box<[C::Name]>, + pub varnames: Box<[C::Name]>, + pub cellvars: Box<[C::Name]>, + pub freevars: Box<[C::Name]>, + /// Line number table (CPython 3.11+ format) + pub linetable: Box<[u8]>, + /// Exception handling table + pub exceptiontable: Box<[u8]>, +} + +bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct CodeFlags: u32 { + const OPTIMIZED = 0x0001; + const NEWLOCALS = 0x0002; + const VARARGS = 0x0004; + const VARKEYWORDS = 0x0008; + const GENERATOR = 0x0020; + const COROUTINE = 0x0080; + const ITERABLE_COROUTINE = 0x0100; + /// If a code object represents a function and has a docstring, + /// this bit is set and the first item in co_consts is the docstring. + const HAS_DOCSTRING = 0x4000000; + } +} + +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub struct CodeUnit { + pub op: Instruction, + pub arg: OpArgByte, +} + +const _: () = assert!(mem::size_of::<CodeUnit>() == 2); + +/// Adaptive specialization: number of executions before attempting specialization. +/// +/// Matches CPython's `_Py_BackoffCounter` encoding. +pub const ADAPTIVE_WARMUP_VALUE: u16 = adaptive_counter_bits(1, 1); +/// Adaptive specialization: cooldown counter after a successful specialization. +/// +/// Value/backoff = (52, 0), matching CPython's ADAPTIVE_COOLDOWN bits. +pub const ADAPTIVE_COOLDOWN_VALUE: u16 = adaptive_counter_bits(52, 0); +/// Initial JUMP_BACKWARD counter bits (value/backoff = 4095/12). +pub const JUMP_BACKWARD_INITIAL_VALUE: u16 = adaptive_counter_bits(4095, 12); + +const BACKOFF_BITS: u16 = 4; +const MAX_BACKOFF: u16 = 12; +const UNREACHABLE_BACKOFF: u16 = 15; + +/// Encode an adaptive counter as `(value << 4) | backoff`. +pub const fn adaptive_counter_bits(value: u16, backoff: u16) -> u16 { + (value << BACKOFF_BITS) | backoff +} + +/// True when the adaptive counter should trigger specialization. +#[inline] +pub const fn adaptive_counter_triggers(counter: u16) -> bool { + counter < UNREACHABLE_BACKOFF +} + +/// Decrement adaptive counter by one countdown step. +#[inline] +pub const fn advance_adaptive_counter(counter: u16) -> u16 { + counter.wrapping_sub(1 << BACKOFF_BITS) +} + +/// Reset adaptive counter with exponential backoff. +#[inline] +pub const fn adaptive_counter_backoff(counter: u16) -> u16 { + let backoff = counter & ((1 << BACKOFF_BITS) - 1); + if backoff < MAX_BACKOFF { + adaptive_counter_bits((1 << (backoff + 1)) - 1, backoff + 1) + } else { + adaptive_counter_bits((1 << MAX_BACKOFF) - 1, MAX_BACKOFF) + } +} + +impl CodeUnit { + pub const fn new(op: Instruction, arg: OpArgByte) -> Self { + Self { op, arg } + } +} + +impl TryFrom<&[u8]> for CodeUnit { + type Error = MarshalError; + + fn try_from(value: &[u8]) -> Result<Self, Self::Error> { + match value.len() { + 2 => Ok(Self::new(value[0].try_into()?, value[1].into())), + _ => Err(Self::Error::InvalidBytecode), + } + } +} + +pub struct CodeUnits { + units: UnsafeCell<Box<[CodeUnit]>>, + adaptive_counters: Box<[AtomicU16]>, + /// Pointer-sized cache entries for descriptor pointers. + /// Single atomic load/store prevents torn reads when multiple threads + /// specialize the same instruction concurrently. + pointer_cache: Box<[AtomicUsize]>, +} + +// SAFETY: All cache operations use atomic read/write instructions. +// - replace_op / compare_exchange_op: AtomicU8 store/CAS (Release) +// - cache read/write: AtomicU16 load/store (Relaxed) +// - adaptive counter: AtomicU16 load/store (Relaxed) +// Ordering is established by: +// - replace_op (Release) ↔ dispatch loop read_op (Acquire) for cache data visibility +// - tp_version_tag (Acquire) for descriptor pointer validity +unsafe impl Sync for CodeUnits {} + +impl Clone for CodeUnits { + fn clone(&self) -> Self { + // SAFETY: No concurrent mutation during clone — cloning is only done + // during code object construction or marshaling, not while instrumented. + let units = unsafe { &*self.units.get() }.clone(); + let adaptive_counters = self + .adaptive_counters + .iter() + .map(|c| AtomicU16::new(c.load(Ordering::Relaxed))) + .collect(); + let pointer_cache = self + .pointer_cache + .iter() + .map(|c| AtomicUsize::new(c.load(Ordering::Relaxed))) + .collect(); + Self { + units: UnsafeCell::new(units), + adaptive_counters, + pointer_cache, + } + } +} + +impl fmt::Debug for CodeUnits { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: Debug formatting doesn't race with replace_op + let inner = unsafe { &*self.units.get() }; + f.debug_tuple("CodeUnits").field(inner).finish() + } +} + +impl TryFrom<&[u8]> for CodeUnits { + type Error = MarshalError; + + fn try_from(value: &[u8]) -> Result<Self, Self::Error> { + if !value.len().is_multiple_of(2) { + return Err(Self::Error::InvalidBytecode); + } + + let units = value + .chunks_exact(2) + .map(CodeUnit::try_from) + .collect::<Result<Vec<_>, _>>()?; + Ok(units.into()) + } +} + +impl<const N: usize> From<[CodeUnit; N]> for CodeUnits { + fn from(value: [CodeUnit; N]) -> Self { + Self::from(Vec::from(value)) + } +} + +impl From<Vec<CodeUnit>> for CodeUnits { + fn from(value: Vec<CodeUnit>) -> Self { + let units = value.into_boxed_slice(); + let len = units.len(); + let adaptive_counters = (0..len) + .map(|_| AtomicU16::new(0)) + .collect::<Vec<_>>() + .into_boxed_slice(); + let pointer_cache = (0..len) + .map(|_| AtomicUsize::new(0)) + .collect::<Vec<_>>() + .into_boxed_slice(); + Self { + units: UnsafeCell::new(units), + adaptive_counters, + pointer_cache, + } + } +} + +impl FromIterator<CodeUnit> for CodeUnits { + fn from_iter<T: IntoIterator<Item = CodeUnit>>(iter: T) -> Self { + Self::from(iter.into_iter().collect::<Vec<_>>()) + } +} + +impl Deref for CodeUnits { + type Target = [CodeUnit]; + + fn deref(&self) -> &Self::Target { + // SAFETY: Shared references to the slice are valid even while replace_op + // may update individual opcode bytes — readers tolerate stale opcodes + // (they will re-read on the next iteration). + unsafe { &*self.units.get() } + } +} + +impl CodeUnits { + /// Replace the opcode at `index` in-place without changing the arg byte. + /// Uses atomic Release store to ensure prior cache writes are visible + /// to threads that subsequently read the new opcode with Acquire. + /// + /// # Safety + /// - `index` must be in bounds. + /// - `new_op` must have the same arg semantics as the original opcode. + pub unsafe fn replace_op(&self, index: usize, new_op: Instruction) { + let units = unsafe { &*self.units.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU8; + unsafe { &*ptr }.store(new_op.into(), Ordering::Release); + } + + /// Atomically replace opcode only if it still matches `expected`. + /// Returns true on success. Uses Release ordering on success. + /// + /// # Safety + /// - `index` must be in bounds. + pub unsafe fn compare_exchange_op( + &self, + index: usize, + expected: Instruction, + new_op: Instruction, + ) -> bool { + let units = unsafe { &*self.units.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU8; + unsafe { &*ptr } + .compare_exchange( + expected.into(), + new_op.into(), + Ordering::Release, + Ordering::Relaxed, + ) + .is_ok() + } + + /// Atomically read the opcode at `index` with Acquire ordering. + /// Pairs with `replace_op` (Release) to ensure cache data visibility. + pub fn read_op(&self, index: usize) -> Instruction { + let units = unsafe { &*self.units.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU8; + let byte = unsafe { &*ptr }.load(Ordering::Acquire); + // SAFETY: Only valid Instruction values are stored via replace_op/compare_exchange_op. + unsafe { mem::transmute::<u8, Instruction>(byte) } + } + + /// Atomically read the arg byte at `index` with Relaxed ordering. + pub fn read_arg(&self, index: usize) -> OpArgByte { + let units = unsafe { &*self.units.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const u8; + let arg_ptr = unsafe { ptr.add(1) } as *const AtomicU8; + OpArgByte::from(unsafe { &*arg_ptr }.load(Ordering::Relaxed)) + } + + /// Write a u16 value into a CACHE code unit at `index`. + /// Each CodeUnit is 2 bytes (#[repr(C)]: op u8 + arg u8), so one u16 fits exactly. + /// Uses Relaxed atomic store; ordering is provided by replace_op (Release). + /// + /// # Safety + /// - `index` must be in bounds and point to a CACHE entry. + pub unsafe fn write_cache_u16(&self, index: usize, value: u16) { + let units = unsafe { &*self.units.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU16; + unsafe { &*ptr }.store(value, Ordering::Relaxed); + } + + /// Read a u16 value from a CACHE code unit at `index`. + /// Uses Relaxed atomic load; ordering is provided by read_op (Acquire). + /// + /// # Panics + /// Panics if `index` is out of bounds. + pub fn read_cache_u16(&self, index: usize) -> u16 { + let units = unsafe { &*self.units.get() }; + assert!(index < units.len(), "read_cache_u16: index out of bounds"); + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU16; + unsafe { &*ptr }.load(Ordering::Relaxed) + } + + /// Write a u32 value across two consecutive CACHE code units starting at `index`. + /// + /// # Safety + /// Same requirements as `write_cache_u16`. + pub unsafe fn write_cache_u32(&self, index: usize, value: u32) { + unsafe { + self.write_cache_u16(index, value as u16); + self.write_cache_u16(index + 1, (value >> 16) as u16); + } + } + + /// Read a u32 value from two consecutive CACHE code units starting at `index`. + /// + /// # Panics + /// Panics if `index + 1` is out of bounds. + pub fn read_cache_u32(&self, index: usize) -> u32 { + let lo = self.read_cache_u16(index) as u32; + let hi = self.read_cache_u16(index + 1) as u32; + lo | (hi << 16) + } + + /// Store a pointer-sized value atomically in the pointer cache at `index`. + /// + /// Uses a single `AtomicUsize` store to prevent torn writes when + /// multiple threads specialize the same instruction concurrently. + /// + /// # Safety + /// - `index` must be in bounds. + /// - `value` must be `0` or a valid `*const PyObject` encoded as `usize`. + /// - Callers must follow the cache invalidation/upgrade protocol: + /// invalidate the version guard before writing and publish the new + /// version after writing. + pub unsafe fn write_cache_ptr(&self, index: usize, value: usize) { + self.pointer_cache[index].store(value, Ordering::Relaxed); + } + + /// Load a pointer-sized value atomically from the pointer cache at `index`. + /// + /// Uses a single `AtomicUsize` load to prevent torn reads. + /// + /// # Panics + /// Panics if `index` is out of bounds. + pub fn read_cache_ptr(&self, index: usize) -> usize { + self.pointer_cache[index].load(Ordering::Relaxed) + } + + /// Read adaptive counter bits for instruction at `index`. + /// Uses Relaxed atomic load. + pub fn read_adaptive_counter(&self, index: usize) -> u16 { + self.adaptive_counters[index].load(Ordering::Relaxed) + } + + /// Write adaptive counter bits for instruction at `index`. + /// Uses Relaxed atomic store. + /// + /// # Safety + /// - `index` must be in bounds. + pub unsafe fn write_adaptive_counter(&self, index: usize, value: u16) { + self.adaptive_counters[index].store(value, Ordering::Relaxed); + } + + /// Produce a clean copy of the bytecode suitable for serialization + /// (marshal) and `co_code`. Specialized opcodes are mapped back to their + /// base variants via `deoptimize()` and all CACHE entries are zeroed. + pub fn original_bytes(&self) -> Vec<u8> { + let len = self.len(); + let mut out = Vec::with_capacity(len * 2); + let mut i = 0; + while i < len { + let op = self.read_op(i).deoptimize(); + let arg = self.read_arg(i); + let caches = op.cache_entries(); + out.push(u8::from(op)); + out.push(u8::from(arg)); + // Zero-fill all CACHE entries (counter + cached data) + for _ in 0..caches { + i += 1; + out.push(0); // op = Cache = 0 + out.push(0); // arg = 0 + } + i += 1; + } + out + } + + /// Initialize adaptive warmup counters for all cacheable instructions. + /// Called lazily at RESUME (first execution of a code object). + /// Counters are stored out-of-line to preserve `op = Instruction::Cache`. + /// All writes are atomic (Relaxed) to avoid data races with concurrent readers. + pub fn quicken(&self) { + let len = self.len(); + let mut i = 0; + while i < len { + let op = self.read_op(i); + let caches = op.cache_entries(); + if caches > 0 { + // Don't write adaptive counter for instrumented opcodes; + // specialization is skipped while monitoring is active. + if !op.is_instrumented() { + let cache_base = i + 1; + if cache_base < len { + let initial_counter = if matches!(op, Instruction::JumpBackward { .. }) { + JUMP_BACKWARD_INITIAL_VALUE + } else { + ADAPTIVE_WARMUP_VALUE + }; + unsafe { + self.write_adaptive_counter(cache_base, initial_counter); + } + } + } + i += 1 + caches; + } else { + i += 1; + } + } + } +} + +/// A Constant (which usually encapsulates data within it) +/// +/// # Examples +/// ``` +/// use rustpython_compiler_core::bytecode::ConstantData; +/// let a = ConstantData::Float {value: 120f64}; +/// let b = ConstantData::Boolean {value: false}; +/// assert_ne!(a, b); +/// ``` +#[derive(Debug, Clone)] +pub enum ConstantData { + Tuple { elements: Vec<ConstantData> }, + Integer { value: BigInt }, + Float { value: f64 }, + Complex { value: Complex64 }, + Boolean { value: bool }, + Str { value: Wtf8Buf }, + Bytes { value: Vec<u8> }, + Code { code: Box<CodeObject> }, + None, + Ellipsis, +} + +impl PartialEq for ConstantData { + fn eq(&self, other: &Self) -> bool { + use ConstantData::*; + + match (self, other) { + (Integer { value: a }, Integer { value: b }) => a == b, + // we want to compare floats *by actual value* - if we have the *exact same* float + // already in a constant cache, we want to use that + (Float { value: a }, Float { value: b }) => a.to_bits() == b.to_bits(), + (Complex { value: a }, Complex { value: b }) => { + a.re.to_bits() == b.re.to_bits() && a.im.to_bits() == b.im.to_bits() + } + (Boolean { value: a }, Boolean { value: b }) => a == b, + (Str { value: a }, Str { value: b }) => a == b, + (Bytes { value: a }, Bytes { value: b }) => a == b, + (Code { code: a }, Code { code: b }) => core::ptr::eq(a.as_ref(), b.as_ref()), + (Tuple { elements: a }, Tuple { elements: b }) => a == b, + (None, None) => true, + (Ellipsis, Ellipsis) => true, + _ => false, + } + } +} + +impl Eq for ConstantData {} + +impl hash::Hash for ConstantData { + fn hash<H: hash::Hasher>(&self, state: &mut H) { + use ConstantData::*; + + mem::discriminant(self).hash(state); + match self { + Integer { value } => value.hash(state), + Float { value } => value.to_bits().hash(state), + Complex { value } => { + value.re.to_bits().hash(state); + value.im.to_bits().hash(state); + } + Boolean { value } => value.hash(state), + Str { value } => value.hash(state), + Bytes { value } => value.hash(state), + Code { code } => core::ptr::hash(code.as_ref(), state), + Tuple { elements } => elements.hash(state), + None => {} + Ellipsis => {} + } + } +} + +/// A borrowed Constant +pub enum BorrowedConstant<'a, C: Constant> { + Integer { value: &'a BigInt }, + Float { value: f64 }, + Complex { value: Complex64 }, + Boolean { value: bool }, + Str { value: &'a Wtf8 }, + Bytes { value: &'a [u8] }, + Code { code: &'a CodeObject<C> }, + Tuple { elements: &'a [C] }, + None, + Ellipsis, +} + +impl<C: Constant> Copy for BorrowedConstant<'_, C> {} + +impl<C: Constant> Clone for BorrowedConstant<'_, C> { + fn clone(&self) -> Self { + *self + } +} + +impl<C: Constant> BorrowedConstant<'_, C> { + pub fn fmt_display(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BorrowedConstant::Integer { value } => write!(f, "{value}"), + BorrowedConstant::Float { value } => write!(f, "{value}"), + BorrowedConstant::Complex { value } => write!(f, "{value}"), + BorrowedConstant::Boolean { value } => { + write!(f, "{}", if *value { "True" } else { "False" }) + } + BorrowedConstant::Str { value } => write!(f, "{value:?}"), + BorrowedConstant::Bytes { value } => write!(f, r#"b"{}""#, value.escape_ascii()), + BorrowedConstant::Code { code } => write!(f, "{code:?}"), + BorrowedConstant::Tuple { elements } => { + write!(f, "(")?; + let mut first = true; + for c in *elements { + if first { + first = false + } else { + write!(f, ", ")?; + } + c.borrow_constant().fmt_display(f)?; + } + write!(f, ")") + } + BorrowedConstant::None => write!(f, "None"), + BorrowedConstant::Ellipsis => write!(f, "..."), + } + } + + pub fn to_owned(self) -> ConstantData { + use ConstantData::*; + + match self { + BorrowedConstant::Integer { value } => Integer { + value: value.clone(), + }, + BorrowedConstant::Float { value } => Float { value }, + BorrowedConstant::Complex { value } => Complex { value }, + BorrowedConstant::Boolean { value } => Boolean { value }, + BorrowedConstant::Str { value } => Str { + value: value.to_owned(), + }, + BorrowedConstant::Bytes { value } => Bytes { + value: value.to_owned(), + }, + BorrowedConstant::Code { code } => Code { + code: Box::new(code.map_clone_bag(&BasicBag)), + }, + BorrowedConstant::Tuple { elements } => Tuple { + elements: elements + .iter() + .map(|c| c.borrow_constant().to_owned()) + .collect(), + }, + BorrowedConstant::None => None, + BorrowedConstant::Ellipsis => Ellipsis, + } + } +} + +/* +Maintain a stack of blocks on the VM. +pub enum BlockType { + Loop, + Except, +} +*/ + +/// Argument structure +pub struct Arguments<'a, N: AsRef<str>> { + pub posonlyargs: &'a [N], + pub args: &'a [N], + pub vararg: Option<&'a N>, + pub kwonlyargs: &'a [N], + pub varkwarg: Option<&'a N>, +} + +impl<N: AsRef<str>> fmt::Debug for Arguments<'_, N> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + macro_rules! fmt_slice { + ($x:expr) => { + format_args!("[{}]", $x.iter().map(AsRef::as_ref).format(", ")) + }; + } + f.debug_struct("Arguments") + .field("posonlyargs", &fmt_slice!(self.posonlyargs)) + .field("args", &fmt_slice!(self.posonlyargs)) + .field("vararg", &self.vararg.map(N::as_ref)) + .field("kwonlyargs", &fmt_slice!(self.kwonlyargs)) + .field("varkwarg", &self.varkwarg.map(N::as_ref)) + .finish() + } +} + +impl<C: Constant> CodeObject<C> { + /// Get all arguments of the code object + /// like inspect.getargs + pub fn arg_names(&self) -> Arguments<'_, C::Name> { + let nargs = self.arg_count as usize; + let nkwargs = self.kwonlyarg_count as usize; + let mut varargs_pos = nargs + nkwargs; + let posonlyargs = &self.varnames[..self.posonlyarg_count as usize]; + let args = &self.varnames[..nargs]; + let kwonlyargs = &self.varnames[nargs..varargs_pos]; + + let vararg = if self.flags.contains(CodeFlags::VARARGS) { + let vararg = &self.varnames[varargs_pos]; + varargs_pos += 1; + Some(vararg) + } else { + None + }; + let varkwarg = if self.flags.contains(CodeFlags::VARKEYWORDS) { + Some(&self.varnames[varargs_pos]) + } else { + None + }; + + Arguments { + posonlyargs, + args, + vararg, + kwonlyargs, + varkwarg, + } + } + + /// Return the labels targeted by the instructions of this CodeObject + pub fn label_targets(&self) -> BTreeSet<Label> { + let mut label_targets = BTreeSet::new(); + let mut arg_state = OpArgState::default(); + for instruction in &*self.instructions { + let (instruction, arg) = arg_state.get(*instruction); + if let Some(l) = instruction.label_arg() { + label_targets.insert(l.get(arg)); + } + } + label_targets + } + + fn display_inner( + &self, + f: &mut fmt::Formatter<'_>, + expand_code_objects: bool, + level: usize, + ) -> fmt::Result { + let label_targets = self.label_targets(); + let line_digits = (3).max(self.locations.last().unwrap().0.line.digits().get()); + let offset_digits = (4).max(1 + self.instructions.len().ilog10() as usize); + let mut last_line = OneIndexed::MAX; + let mut arg_state = OpArgState::default(); + for (offset, &instruction) in self.instructions.iter().enumerate() { + let (instruction, arg) = arg_state.get(instruction); + // optional line number + let line = self.locations[offset].0.line; + if line != last_line { + if last_line != OneIndexed::MAX { + writeln!(f)?; + } + last_line = line; + write!(f, "{line:line_digits$}")?; + } else { + for _ in 0..line_digits { + write!(f, " ")?; + } + } + write!(f, " ")?; + + // level indent + for _ in 0..level { + write!(f, " ")?; + } + + // arrow and offset + let arrow = if label_targets.contains(&Label::new(offset as u32)) { + ">>" + } else { + " " + }; + write!(f, "{arrow} {offset:offset_digits$} ")?; + + // instruction + instruction.fmt_dis(arg, f, self, expand_code_objects, 21, level)?; + writeln!(f)?; + } + Ok(()) + } + + /// Recursively display this CodeObject + pub fn display_expand_code_objects(&self) -> impl fmt::Display + '_ { + struct Display<'a, C: Constant>(&'a CodeObject<C>); + impl<C: Constant> fmt::Display for Display<'_, C> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.display_inner(f, true, 1) + } + } + Display(self) + } + + /// Map this CodeObject to one that holds a Bag::Constant + pub fn map_bag<Bag: ConstantBag>(self, bag: Bag) -> CodeObject<Bag::Constant> { + let map_names = |names: Box<[C::Name]>| { + names + .iter() + .map(|x| bag.make_name(x.as_ref())) + .collect::<Box<[_]>>() + }; + CodeObject { + constants: self + .constants + .iter() + .map(|x| bag.make_constant(x.borrow_constant())) + .collect(), + names: map_names(self.names), + varnames: map_names(self.varnames), + cellvars: map_names(self.cellvars), + freevars: map_names(self.freevars), + source_path: bag.make_name(self.source_path.as_ref()), + obj_name: bag.make_name(self.obj_name.as_ref()), + qualname: bag.make_name(self.qualname.as_ref()), + + instructions: self.instructions, + locations: self.locations, + flags: self.flags, + posonlyarg_count: self.posonlyarg_count, + arg_count: self.arg_count, + kwonlyarg_count: self.kwonlyarg_count, + first_line_number: self.first_line_number, + max_stackdepth: self.max_stackdepth, + cell2arg: self.cell2arg, + linetable: self.linetable, + exceptiontable: self.exceptiontable, + } + } + + /// Same as `map_bag` but clones `self` + pub fn map_clone_bag<Bag: ConstantBag>(&self, bag: &Bag) -> CodeObject<Bag::Constant> { + let map_names = + |names: &[C::Name]| names.iter().map(|x| bag.make_name(x.as_ref())).collect(); + CodeObject { + constants: self + .constants + .iter() + .map(|x| bag.make_constant(x.borrow_constant())) + .collect(), + names: map_names(&self.names), + varnames: map_names(&self.varnames), + cellvars: map_names(&self.cellvars), + freevars: map_names(&self.freevars), + source_path: bag.make_name(self.source_path.as_ref()), + obj_name: bag.make_name(self.obj_name.as_ref()), + qualname: bag.make_name(self.qualname.as_ref()), + + instructions: self.instructions.clone(), + locations: self.locations.clone(), + flags: self.flags, + posonlyarg_count: self.posonlyarg_count, + arg_count: self.arg_count, + kwonlyarg_count: self.kwonlyarg_count, + first_line_number: self.first_line_number, + max_stackdepth: self.max_stackdepth, + cell2arg: self.cell2arg.clone(), + linetable: self.linetable.clone(), + exceptiontable: self.exceptiontable.clone(), + } + } +} + +impl<C: Constant> fmt::Display for CodeObject<C> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.display_inner(f, false, 1)?; + for constant in &*self.constants { + if let BorrowedConstant::Code { code } = constant.borrow_constant() { + writeln!(f, "\nDisassembly of {code:?}")?; + code.fmt(f)?; + } + } + Ok(()) + } +} + +pub trait InstrDisplayContext { + type Constant: Constant; + + fn get_constant(&self, consti: oparg::ConstIdx) -> &Self::Constant; + + fn get_name(&self, i: usize) -> &str; + + fn get_varname(&self, var_num: oparg::VarNum) -> &str; + + fn get_cell_name(&self, i: usize) -> &str; +} + +impl<C: Constant> InstrDisplayContext for CodeObject<C> { + type Constant = C; + + fn get_constant(&self, consti: oparg::ConstIdx) -> &C { + &self.constants[consti] + } + + fn get_name(&self, i: usize) -> &str { + self.names[i].as_ref() + } + + fn get_varname(&self, var_num: oparg::VarNum) -> &str { + self.varnames[var_num].as_ref() + } + + fn get_cell_name(&self, i: usize) -> &str { + self.cellvars + .get(i) + .unwrap_or_else(|| &self.freevars[i - self.cellvars.len()]) + .as_ref() + } +} + +impl fmt::Display for ConstantData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.borrow_constant().fmt_display(f) + } +} + +impl<C: Constant> fmt::Debug for CodeObject<C> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "<code object {} at ??? file {:?}, line {}>", + self.obj_name.as_ref(), + self.source_path.as_ref(), + self.first_line_number.map_or(-1, |x| x.get() as i32) + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::{vec, vec::Vec}; + + #[test] + fn test_exception_table_encode_decode() { + let entries = vec![ + ExceptionTableEntry::new(0, 10, 20, 2, false), + ExceptionTableEntry::new(15, 25, 30, 1, true), + ]; + + let encoded = encode_exception_table(&entries); + + // Find handler at offset 5 (in range [0, 10)) + let handler = find_exception_handler(&encoded, 5); + assert!(handler.is_some()); + let handler = handler.unwrap(); + assert_eq!(handler.start, 0); + assert_eq!(handler.end, 10); + assert_eq!(handler.target, 20); + assert_eq!(handler.depth, 2); + assert!(!handler.push_lasti); + + // Find handler at offset 20 (in range [15, 25)) + let handler = find_exception_handler(&encoded, 20); + assert!(handler.is_some()); + let handler = handler.unwrap(); + assert_eq!(handler.start, 15); + assert_eq!(handler.end, 25); + assert_eq!(handler.target, 30); + assert_eq!(handler.depth, 1); + assert!(handler.push_lasti); + + // No handler at offset 12 (not in any range) + let handler = find_exception_handler(&encoded, 12); + assert!(handler.is_none()); + + // No handler at offset 30 (past all ranges) + let handler = find_exception_handler(&encoded, 30); + assert!(handler.is_none()); + } + + #[test] + fn test_exception_table_empty() { + let entries: Vec<ExceptionTableEntry> = vec![]; + let encoded = encode_exception_table(&entries); + assert!(encoded.is_empty()); + assert!(find_exception_handler(&encoded, 0).is_none()); + } + + #[test] + fn test_exception_table_single_entry() { + let entries = vec![ExceptionTableEntry::new(5, 15, 100, 3, true)]; + let encoded = encode_exception_table(&entries); + + // Inside range + let handler = find_exception_handler(&encoded, 10); + assert!(handler.is_some()); + let handler = handler.unwrap(); + assert_eq!(handler.target, 100); + assert_eq!(handler.depth, 3); + assert!(handler.push_lasti); + + // At start boundary (inclusive) + assert!(find_exception_handler(&encoded, 5).is_some()); + + // At end boundary (exclusive) + assert!(find_exception_handler(&encoded, 15).is_none()); + } +} diff --git a/crates/compiler-core/src/bytecode/instruction.rs b/crates/compiler-core/src/bytecode/instruction.rs new file mode 100644 index 00000000000..ea5fe181861 --- /dev/null +++ b/crates/compiler-core/src/bytecode/instruction.rs @@ -0,0 +1,1751 @@ +use core::{fmt, marker::PhantomData, mem}; + +use crate::{ + bytecode::{ + BorrowedConstant, Constant, InstrDisplayContext, + oparg::{ + self, BinaryOperator, BuildSliceArgCount, CommonConstant, ComparisonOperator, + ConvertValueOparg, IntrinsicFunction1, IntrinsicFunction2, Invert, Label, LoadAttr, + LoadSuperAttr, MakeFunctionFlag, NameIdx, OpArg, OpArgByte, OpArgType, RaiseKind, + SpecialMethod, UnpackExArgs, + }, + }, + marshal::MarshalError, +}; + +/// A Single bytecode instruction that are executed by the VM. +/// +/// Currently aligned with CPython 3.14. +/// +/// ## See also +/// - [CPython opcode IDs](https://github.com/python/cpython/blob/v3.14.2/Include/opcode_ids.h) +#[derive(Clone, Copy, Debug)] +#[repr(u8)] +pub enum Instruction { + // No-argument instructions (opcode < HAVE_ARGUMENT=44) + Cache = 0, + BinarySlice = 1, + BuildTemplate = 2, + BinaryOpInplaceAddUnicode = 3, + CallFunctionEx = 4, + CheckEgMatch = 5, + CheckExcMatch = 6, + CleanupThrow = 7, + DeleteSubscr = 8, + EndFor = 9, + EndSend = 10, + ExitInitCheck = 11, // Placeholder + FormatSimple = 12, + FormatWithSpec = 13, + GetAIter = 14, + GetANext = 15, + GetIter = 16, + Reserved = 17, + GetLen = 18, + GetYieldFromIter = 19, + InterpreterExit = 20, // Placeholder + LoadBuildClass = 21, + LoadLocals = 22, + MakeFunction = 23, + MatchKeys = 24, + MatchMapping = 25, + MatchSequence = 26, + Nop = 27, + NotTaken = 28, + PopExcept = 29, + PopIter = 30, + PopTop = 31, + PushExcInfo = 32, + PushNull = 33, + ReturnGenerator = 34, + ReturnValue = 35, + SetupAnnotations = 36, + StoreSlice = 37, + StoreSubscr = 38, + ToBool = 39, + UnaryInvert = 40, + UnaryNegative = 41, + UnaryNot = 42, + WithExceptStart = 43, + // CPython 3.14 opcodes with arguments (44-120) + BinaryOp { + op: Arg<BinaryOperator>, + } = 44, + /// Build an Interpolation from value, expression string, and optional format_spec on stack. + /// + /// oparg encoding: (conversion << 2) | has_format_spec + /// - has_format_spec (bit 0): if 1, format_spec is on stack + /// - conversion (bits 2+): 0=None, 1=Str, 2=Repr, 3=Ascii + /// + /// Stack: [value, expression_str, format_spec?] -> [interpolation] + BuildInterpolation { + format: Arg<u32>, + } = 45, + BuildList { + count: Arg<u32>, + } = 46, + BuildMap { + count: Arg<u32>, + } = 47, + BuildSet { + count: Arg<u32>, + } = 48, + BuildSlice { + argc: Arg<BuildSliceArgCount>, + } = 49, + BuildString { + count: Arg<u32>, + } = 50, + BuildTuple { + count: Arg<u32>, + } = 51, + Call { + argc: Arg<u32>, + } = 52, + CallIntrinsic1 { + func: Arg<IntrinsicFunction1>, + } = 53, + CallIntrinsic2 { + func: Arg<IntrinsicFunction2>, + } = 54, + CallKw { + argc: Arg<u32>, + } = 55, + CompareOp { + opname: Arg<ComparisonOperator>, + } = 56, + ContainsOp { + invert: Arg<Invert>, + } = 57, + ConvertValue { + oparg: Arg<ConvertValueOparg>, + } = 58, + Copy { + i: Arg<u32>, + } = 59, + CopyFreeVars { + n: Arg<u32>, + } = 60, + DeleteAttr { + namei: Arg<NameIdx>, + } = 61, + DeleteDeref { + i: Arg<NameIdx>, + } = 62, + DeleteFast { + var_num: Arg<oparg::VarNum>, + } = 63, + DeleteGlobal { + namei: Arg<NameIdx>, + } = 64, + DeleteName { + namei: Arg<NameIdx>, + } = 65, + DictMerge { + i: Arg<u32>, + } = 66, + DictUpdate { + i: Arg<u32>, + } = 67, + EndAsyncFor = 68, + ExtendedArg = 69, + ForIter { + delta: Arg<Label>, + } = 70, + GetAwaitable { + r#where: Arg<u32>, + } = 71, + ImportFrom { + namei: Arg<NameIdx>, + } = 72, + ImportName { + namei: Arg<NameIdx>, + } = 73, + IsOp { + invert: Arg<Invert>, + } = 74, + JumpBackward { + delta: Arg<Label>, + } = 75, + JumpBackwardNoInterrupt { + delta: Arg<Label>, + } = 76, // Placeholder + JumpForward { + delta: Arg<Label>, + } = 77, + ListAppend { + i: Arg<u32>, + } = 78, + ListExtend { + i: Arg<u32>, + } = 79, + LoadAttr { + namei: Arg<LoadAttr>, + } = 80, + LoadCommonConstant { + idx: Arg<CommonConstant>, + } = 81, + LoadConst { + consti: Arg<oparg::ConstIdx>, + } = 82, + LoadDeref { + i: Arg<NameIdx>, + } = 83, + LoadFast { + var_num: Arg<oparg::VarNum>, + } = 84, + LoadFastAndClear { + var_num: Arg<oparg::VarNum>, + } = 85, + LoadFastBorrow { + var_num: Arg<oparg::VarNum>, + } = 86, + LoadFastBorrowLoadFastBorrow { + var_nums: Arg<oparg::VarNums>, + } = 87, + LoadFastCheck { + var_num: Arg<oparg::VarNum>, + } = 88, + LoadFastLoadFast { + var_nums: Arg<oparg::VarNums>, + } = 89, + LoadFromDictOrDeref { + i: Arg<NameIdx>, + } = 90, + LoadFromDictOrGlobals { + i: Arg<NameIdx>, + } = 91, + LoadGlobal { + namei: Arg<NameIdx>, + } = 92, + LoadName { + namei: Arg<NameIdx>, + } = 93, + LoadSmallInt { + i: Arg<u32>, + } = 94, + LoadSpecial { + method: Arg<SpecialMethod>, + } = 95, + LoadSuperAttr { + namei: Arg<LoadSuperAttr>, + } = 96, + MakeCell { + i: Arg<NameIdx>, + } = 97, + MapAdd { + i: Arg<u32>, + } = 98, + MatchClass { + count: Arg<u32>, + } = 99, + PopJumpIfFalse { + delta: Arg<Label>, + } = 100, + PopJumpIfNone { + delta: Arg<Label>, + } = 101, + PopJumpIfNotNone { + delta: Arg<Label>, + } = 102, + PopJumpIfTrue { + delta: Arg<Label>, + } = 103, + RaiseVarargs { + argc: Arg<RaiseKind>, + } = 104, + Reraise { + depth: Arg<u32>, + } = 105, + Send { + delta: Arg<Label>, + } = 106, + SetAdd { + i: Arg<u32>, + } = 107, + SetFunctionAttribute { + flag: Arg<MakeFunctionFlag>, + } = 108, + SetUpdate { + i: Arg<u32>, + } = 109, + StoreAttr { + namei: Arg<NameIdx>, + } = 110, + StoreDeref { + i: Arg<NameIdx>, + } = 111, + StoreFast { + var_num: Arg<oparg::VarNum>, + } = 112, + StoreFastLoadFast { + var_nums: Arg<oparg::VarNums>, + } = 113, + StoreFastStoreFast { + var_nums: Arg<oparg::VarNums>, + } = 114, + StoreGlobal { + namei: Arg<NameIdx>, + } = 115, + StoreName { + namei: Arg<NameIdx>, + } = 116, + Swap { + i: Arg<u32>, + } = 117, + UnpackEx { + counts: Arg<UnpackExArgs>, + } = 118, + UnpackSequence { + count: Arg<u32>, + } = 119, + YieldValue { + arg: Arg<u32>, + } = 120, + // CPython 3.14 RESUME (128) + Resume { + context: Arg<u32>, + } = 128, + // CPython 3.14 specialized opcodes (129-211) + BinaryOpAddFloat = 129, // Placeholder + BinaryOpAddInt = 130, // Placeholder + BinaryOpAddUnicode = 131, // Placeholder + BinaryOpExtend = 132, // Placeholder + BinaryOpMultiplyFloat = 133, // Placeholder + BinaryOpMultiplyInt = 134, // Placeholder + BinaryOpSubscrDict = 135, // Placeholder + BinaryOpSubscrGetitem = 136, // Placeholder + BinaryOpSubscrListInt = 137, // Placeholder + BinaryOpSubscrListSlice = 138, // Placeholder + BinaryOpSubscrStrInt = 139, // Placeholder + BinaryOpSubscrTupleInt = 140, // Placeholder + BinaryOpSubtractFloat = 141, // Placeholder + BinaryOpSubtractInt = 142, // Placeholder + CallAllocAndEnterInit = 143, // Placeholder + CallBoundMethodExactArgs = 144, // Placeholder + CallBoundMethodGeneral = 145, // Placeholder + CallBuiltinClass = 146, // Placeholder + CallBuiltinFast = 147, // Placeholder + CallBuiltinFastWithKeywords = 148, // Placeholder + CallBuiltinO = 149, // Placeholder + CallIsinstance = 150, // Placeholder + CallKwBoundMethod = 151, // Placeholder + CallKwNonPy = 152, // Placeholder + CallKwPy = 153, // Placeholder + CallLen = 154, // Placeholder + CallListAppend = 155, // Placeholder + CallMethodDescriptorFast = 156, // Placeholder + CallMethodDescriptorFastWithKeywords = 157, // Placeholder + CallMethodDescriptorNoargs = 158, // Placeholder + CallMethodDescriptorO = 159, // Placeholder + CallNonPyGeneral = 160, // Placeholder + CallPyExactArgs = 161, // Placeholder + CallPyGeneral = 162, // Placeholder + CallStr1 = 163, // Placeholder + CallTuple1 = 164, // Placeholder + CallType1 = 165, // Placeholder + CompareOpFloat = 166, // Placeholder + CompareOpInt = 167, // Placeholder + CompareOpStr = 168, // Placeholder + ContainsOpDict = 169, // Placeholder + ContainsOpSet = 170, // Placeholder + ForIterGen = 171, // Placeholder + ForIterList = 172, // Placeholder + ForIterRange = 173, // Placeholder + ForIterTuple = 174, // Placeholder + JumpBackwardJit = 175, // Placeholder + JumpBackwardNoJit = 176, // Placeholder + LoadAttrClass = 177, // Placeholder + LoadAttrClassWithMetaclassCheck = 178, // Placeholder + LoadAttrGetattributeOverridden = 179, // Placeholder + LoadAttrInstanceValue = 180, // Placeholder + LoadAttrMethodLazyDict = 181, // Placeholder + LoadAttrMethodNoDict = 182, // Placeholder + LoadAttrMethodWithValues = 183, // Placeholder + LoadAttrModule = 184, // Placeholder + LoadAttrNondescriptorNoDict = 185, // Placeholder + LoadAttrNondescriptorWithValues = 186, // Placeholder + LoadAttrProperty = 187, // Placeholder + LoadAttrSlot = 188, // Placeholder + LoadAttrWithHint = 189, // Placeholder + LoadConstImmortal = 190, // Placeholder + LoadConstMortal = 191, // Placeholder + LoadGlobalBuiltin = 192, // Placeholder + LoadGlobalModule = 193, // Placeholder + LoadSuperAttrAttr = 194, // Placeholder + LoadSuperAttrMethod = 195, // Placeholder + ResumeCheck = 196, // Placeholder + SendGen = 197, // Placeholder + StoreAttrInstanceValue = 198, // Placeholder + StoreAttrSlot = 199, // Placeholder + StoreAttrWithHint = 200, // Placeholder + StoreSubscrDict = 201, // Placeholder + StoreSubscrListInt = 202, // Placeholder + ToBoolAlwaysTrue = 203, // Placeholder + ToBoolBool = 204, // Placeholder + ToBoolInt = 205, // Placeholder + ToBoolList = 206, // Placeholder + ToBoolNone = 207, // Placeholder + ToBoolStr = 208, // Placeholder + UnpackSequenceList = 209, // Placeholder + UnpackSequenceTuple = 210, // Placeholder + UnpackSequenceTwoTuple = 211, // Placeholder + // CPython 3.14 instrumented opcodes (234-254) + InstrumentedEndFor = 234, + InstrumentedPopIter = 235, + InstrumentedEndSend = 236, + InstrumentedForIter = 237, + InstrumentedInstruction = 238, + InstrumentedJumpForward = 239, + InstrumentedNotTaken = 240, + InstrumentedPopJumpIfTrue = 241, + InstrumentedPopJumpIfFalse = 242, + InstrumentedPopJumpIfNone = 243, + InstrumentedPopJumpIfNotNone = 244, + InstrumentedResume = 245, + InstrumentedReturnValue = 246, + InstrumentedYieldValue = 247, + InstrumentedEndAsyncFor = 248, + InstrumentedLoadSuperAttr = 249, + InstrumentedCall = 250, + InstrumentedCallKw = 251, + InstrumentedCallFunctionEx = 252, + InstrumentedJumpBackward = 253, + InstrumentedLine = 254, + EnterExecutor = 255, // Placeholder +} + +const _: () = assert!(mem::size_of::<Instruction>() == 1); + +impl From<Instruction> for u8 { + #[inline] + fn from(ins: Instruction) -> Self { + // SAFETY: there's no padding bits + unsafe { mem::transmute::<Instruction, Self>(ins) } + } +} + +impl TryFrom<u8> for Instruction { + type Error = MarshalError; + + #[inline] + fn try_from(value: u8) -> Result<Self, MarshalError> { + // CPython-compatible opcodes (0-120) + let cpython_start = u8::from(Self::Cache); + let cpython_end = u8::from(Self::YieldValue { arg: Arg::marker() }); + + // Resume has a non-contiguous opcode (128) + let resume_id = u8::from(Self::Resume { + context: Arg::marker(), + }); + let enter_executor_id = u8::from(Self::EnterExecutor); + + let specialized_start = u8::from(Self::BinaryOpAddFloat); + let specialized_end = u8::from(Self::UnpackSequenceTwoTuple); + + let instrumented_start = u8::from(Self::InstrumentedEndFor); + let instrumented_end = u8::from(Self::InstrumentedLine); + + // No RustPython-only opcodes anymore - all opcodes match CPython 3.14 + let custom_ops: &[u8] = &[]; + + if (cpython_start..=cpython_end).contains(&value) + || value == resume_id + || value == enter_executor_id + || custom_ops.contains(&value) + || (specialized_start..=specialized_end).contains(&value) + || (instrumented_start..=instrumented_end).contains(&value) + { + Ok(unsafe { mem::transmute::<u8, Self>(value) }) + } else { + Err(Self::Error::InvalidBytecode) + } + } +} + +impl Instruction { + /// Returns `true` if this is any instrumented opcode + /// (regular INSTRUMENTED_*, INSTRUMENTED_LINE, or INSTRUMENTED_INSTRUCTION). + pub fn is_instrumented(self) -> bool { + self.to_base().is_some() + || matches!(self, Self::InstrumentedLine | Self::InstrumentedInstruction) + } + + /// Map a base opcode to its INSTRUMENTED_* variant. + /// Returns `None` if this opcode has no instrumented counterpart. + /// + /// # Panics (debug) + /// Panics if called on an already-instrumented opcode. + pub fn to_instrumented(self) -> Option<Self> { + debug_assert!( + !self.is_instrumented(), + "to_instrumented called on already-instrumented opcode {self:?}" + ); + Some(match self { + Self::Resume { .. } => Self::InstrumentedResume, + Self::ReturnValue => Self::InstrumentedReturnValue, + Self::YieldValue { .. } => Self::InstrumentedYieldValue, + Self::Call { .. } => Self::InstrumentedCall, + Self::CallKw { .. } => Self::InstrumentedCallKw, + Self::CallFunctionEx => Self::InstrumentedCallFunctionEx, + Self::LoadSuperAttr { .. } => Self::InstrumentedLoadSuperAttr, + Self::JumpForward { .. } => Self::InstrumentedJumpForward, + Self::JumpBackward { .. } => Self::InstrumentedJumpBackward, + Self::ForIter { .. } => Self::InstrumentedForIter, + Self::EndFor => Self::InstrumentedEndFor, + Self::EndSend => Self::InstrumentedEndSend, + Self::PopJumpIfTrue { .. } => Self::InstrumentedPopJumpIfTrue, + Self::PopJumpIfFalse { .. } => Self::InstrumentedPopJumpIfFalse, + Self::PopJumpIfNone { .. } => Self::InstrumentedPopJumpIfNone, + Self::PopJumpIfNotNone { .. } => Self::InstrumentedPopJumpIfNotNone, + Self::NotTaken => Self::InstrumentedNotTaken, + Self::PopIter => Self::InstrumentedPopIter, + Self::EndAsyncFor => Self::InstrumentedEndAsyncFor, + _ => return None, + }) + } + + /// Map an INSTRUMENTED_* opcode back to its base variant. + /// Returns `None` for non-instrumented opcodes, and also for + /// `InstrumentedLine` / `InstrumentedInstruction` which are event-layer + /// placeholders without a fixed base opcode (the real opcode is stored in + /// `CoMonitoringData`). + /// + /// The returned base opcode uses `Arg::marker()` for typed fields — + /// only the opcode byte matters since `replace_op` preserves the arg byte. + pub fn to_base(self) -> Option<Self> { + Some(match self { + Self::InstrumentedResume => Self::Resume { + context: Arg::marker(), + }, + Self::InstrumentedReturnValue => Self::ReturnValue, + Self::InstrumentedYieldValue => Self::YieldValue { arg: Arg::marker() }, + Self::InstrumentedCall => Self::Call { + argc: Arg::marker(), + }, + Self::InstrumentedCallKw => Self::CallKw { + argc: Arg::marker(), + }, + Self::InstrumentedCallFunctionEx => Self::CallFunctionEx, + Self::InstrumentedLoadSuperAttr => Self::LoadSuperAttr { + namei: Arg::marker(), + }, + Self::InstrumentedJumpForward => Self::JumpForward { + delta: Arg::marker(), + }, + Self::InstrumentedJumpBackward => Self::JumpBackward { + delta: Arg::marker(), + }, + Self::InstrumentedForIter => Self::ForIter { + delta: Arg::marker(), + }, + Self::InstrumentedEndFor => Self::EndFor, + Self::InstrumentedEndSend => Self::EndSend, + Self::InstrumentedPopJumpIfTrue => Self::PopJumpIfTrue { + delta: Arg::marker(), + }, + Self::InstrumentedPopJumpIfFalse => Self::PopJumpIfFalse { + delta: Arg::marker(), + }, + Self::InstrumentedPopJumpIfNone => Self::PopJumpIfNone { + delta: Arg::marker(), + }, + Self::InstrumentedPopJumpIfNotNone => Self::PopJumpIfNotNone { + delta: Arg::marker(), + }, + Self::InstrumentedNotTaken => Self::NotTaken, + Self::InstrumentedPopIter => Self::PopIter, + Self::InstrumentedEndAsyncFor => Self::EndAsyncFor, + _ => return None, + }) + } + + /// Map a specialized opcode back to its adaptive (base) variant. + /// `_PyOpcode_Deopt` + pub fn deoptimize(self) -> Self { + match self { + // LOAD_ATTR specializations + Self::LoadAttrClass + | Self::LoadAttrClassWithMetaclassCheck + | Self::LoadAttrGetattributeOverridden + | Self::LoadAttrInstanceValue + | Self::LoadAttrMethodLazyDict + | Self::LoadAttrMethodNoDict + | Self::LoadAttrMethodWithValues + | Self::LoadAttrModule + | Self::LoadAttrNondescriptorNoDict + | Self::LoadAttrNondescriptorWithValues + | Self::LoadAttrProperty + | Self::LoadAttrSlot + | Self::LoadAttrWithHint => Self::LoadAttr { + namei: Arg::marker(), + }, + // BINARY_OP specializations + Self::BinaryOpAddFloat + | Self::BinaryOpAddInt + | Self::BinaryOpAddUnicode + | Self::BinaryOpExtend + | Self::BinaryOpInplaceAddUnicode + | Self::BinaryOpMultiplyFloat + | Self::BinaryOpMultiplyInt + | Self::BinaryOpSubscrDict + | Self::BinaryOpSubscrGetitem + | Self::BinaryOpSubscrListInt + | Self::BinaryOpSubscrListSlice + | Self::BinaryOpSubscrStrInt + | Self::BinaryOpSubscrTupleInt + | Self::BinaryOpSubtractFloat + | Self::BinaryOpSubtractInt => Self::BinaryOp { op: Arg::marker() }, + // CALL specializations + Self::CallAllocAndEnterInit + | Self::CallBoundMethodExactArgs + | Self::CallBoundMethodGeneral + | Self::CallBuiltinClass + | Self::CallBuiltinFast + | Self::CallBuiltinFastWithKeywords + | Self::CallBuiltinO + | Self::CallIsinstance + | Self::CallLen + | Self::CallListAppend + | Self::CallMethodDescriptorFast + | Self::CallMethodDescriptorFastWithKeywords + | Self::CallMethodDescriptorNoargs + | Self::CallMethodDescriptorO + | Self::CallNonPyGeneral + | Self::CallPyExactArgs + | Self::CallPyGeneral + | Self::CallStr1 + | Self::CallTuple1 + | Self::CallType1 => Self::Call { + argc: Arg::marker(), + }, + // CALL_KW specializations + Self::CallKwBoundMethod | Self::CallKwNonPy | Self::CallKwPy => Self::CallKw { + argc: Arg::marker(), + }, + // TO_BOOL specializations + Self::ToBoolAlwaysTrue + | Self::ToBoolBool + | Self::ToBoolInt + | Self::ToBoolList + | Self::ToBoolNone + | Self::ToBoolStr => Self::ToBool, + // COMPARE_OP specializations + Self::CompareOpFloat | Self::CompareOpInt | Self::CompareOpStr => Self::CompareOp { + opname: Arg::marker(), + }, + // CONTAINS_OP specializations + Self::ContainsOpDict | Self::ContainsOpSet => Self::ContainsOp { + invert: Arg::marker(), + }, + // FOR_ITER specializations + Self::ForIterGen | Self::ForIterList | Self::ForIterRange | Self::ForIterTuple => { + Self::ForIter { + delta: Arg::marker(), + } + } + // LOAD_GLOBAL specializations + Self::LoadGlobalBuiltin | Self::LoadGlobalModule => Self::LoadGlobal { + namei: Arg::marker(), + }, + // STORE_ATTR specializations + Self::StoreAttrInstanceValue | Self::StoreAttrSlot | Self::StoreAttrWithHint => { + Self::StoreAttr { + namei: Arg::marker(), + } + } + // LOAD_SUPER_ATTR specializations + Self::LoadSuperAttrAttr | Self::LoadSuperAttrMethod => Self::LoadSuperAttr { + namei: Arg::marker(), + }, + // STORE_SUBSCR specializations + Self::StoreSubscrDict | Self::StoreSubscrListInt => Self::StoreSubscr, + // UNPACK_SEQUENCE specializations + Self::UnpackSequenceList | Self::UnpackSequenceTuple | Self::UnpackSequenceTwoTuple => { + Self::UnpackSequence { + count: Arg::marker(), + } + } + // SEND specializations + Self::SendGen => Self::Send { + delta: Arg::marker(), + }, + // LOAD_CONST specializations + Self::LoadConstImmortal | Self::LoadConstMortal => Self::LoadConst { + consti: Arg::marker(), + }, + // RESUME specializations + Self::ResumeCheck => Self::Resume { + context: Arg::marker(), + }, + // JUMP_BACKWARD specializations + Self::JumpBackwardJit | Self::JumpBackwardNoJit => Self::JumpBackward { + delta: Arg::marker(), + }, + // Instrumented opcodes map back to their base + _ => match self.to_base() { + Some(base) => base, + None => self, + }, + } + } + + /// Number of CACHE code units that follow this instruction. + /// _PyOpcode_Caches + pub fn cache_entries(self) -> usize { + match self { + // LOAD_ATTR: 9 cache entries + Self::LoadAttr { .. } + | Self::LoadAttrClass + | Self::LoadAttrClassWithMetaclassCheck + | Self::LoadAttrGetattributeOverridden + | Self::LoadAttrInstanceValue + | Self::LoadAttrMethodLazyDict + | Self::LoadAttrMethodNoDict + | Self::LoadAttrMethodWithValues + | Self::LoadAttrModule + | Self::LoadAttrNondescriptorNoDict + | Self::LoadAttrNondescriptorWithValues + | Self::LoadAttrProperty + | Self::LoadAttrSlot + | Self::LoadAttrWithHint => 9, + + // BINARY_OP: 5 cache entries + Self::BinaryOp { .. } + | Self::BinaryOpAddFloat + | Self::BinaryOpAddInt + | Self::BinaryOpAddUnicode + | Self::BinaryOpExtend + | Self::BinaryOpInplaceAddUnicode + | Self::BinaryOpMultiplyFloat + | Self::BinaryOpMultiplyInt + | Self::BinaryOpSubscrDict + | Self::BinaryOpSubscrGetitem + | Self::BinaryOpSubscrListInt + | Self::BinaryOpSubscrListSlice + | Self::BinaryOpSubscrStrInt + | Self::BinaryOpSubscrTupleInt + | Self::BinaryOpSubtractFloat + | Self::BinaryOpSubtractInt => 5, + + // LOAD_GLOBAL / STORE_ATTR: 4 cache entries + Self::LoadGlobal { .. } + | Self::LoadGlobalBuiltin + | Self::LoadGlobalModule + | Self::StoreAttr { .. } + | Self::StoreAttrInstanceValue + | Self::StoreAttrSlot + | Self::StoreAttrWithHint => 4, + + // CALL / CALL_KW / TO_BOOL: 3 cache entries + Self::Call { .. } + | Self::CallAllocAndEnterInit + | Self::CallBoundMethodExactArgs + | Self::CallBoundMethodGeneral + | Self::CallBuiltinClass + | Self::CallBuiltinFast + | Self::CallBuiltinFastWithKeywords + | Self::CallBuiltinO + | Self::CallIsinstance + | Self::CallLen + | Self::CallListAppend + | Self::CallMethodDescriptorFast + | Self::CallMethodDescriptorFastWithKeywords + | Self::CallMethodDescriptorNoargs + | Self::CallMethodDescriptorO + | Self::CallNonPyGeneral + | Self::CallPyExactArgs + | Self::CallPyGeneral + | Self::CallStr1 + | Self::CallTuple1 + | Self::CallType1 + | Self::CallKw { .. } + | Self::CallKwBoundMethod + | Self::CallKwNonPy + | Self::CallKwPy + | Self::ToBool + | Self::ToBoolAlwaysTrue + | Self::ToBoolBool + | Self::ToBoolInt + | Self::ToBoolList + | Self::ToBoolNone + | Self::ToBoolStr => 3, + + // 1 cache entry + Self::CompareOp { .. } + | Self::CompareOpFloat + | Self::CompareOpInt + | Self::CompareOpStr + | Self::ContainsOp { .. } + | Self::ContainsOpDict + | Self::ContainsOpSet + | Self::ForIter { .. } + | Self::ForIterGen + | Self::ForIterList + | Self::ForIterRange + | Self::ForIterTuple + | Self::JumpBackward { .. } + | Self::JumpBackwardJit + | Self::JumpBackwardNoJit + | Self::LoadSuperAttr { .. } + | Self::LoadSuperAttrAttr + | Self::LoadSuperAttrMethod + | Self::PopJumpIfTrue { .. } + | Self::PopJumpIfFalse { .. } + | Self::PopJumpIfNone { .. } + | Self::PopJumpIfNotNone { .. } + | Self::Send { .. } + | Self::SendGen + | Self::StoreSubscr + | Self::StoreSubscrDict + | Self::StoreSubscrListInt + | Self::UnpackSequence { .. } + | Self::UnpackSequenceList + | Self::UnpackSequenceTuple + | Self::UnpackSequenceTwoTuple => 1, + + // Instrumented opcodes have the same cache entries as their base + _ => match self.to_base() { + Some(base) => base.cache_entries(), + None => 0, + }, + } + } +} + +impl InstructionMetadata for Instruction { + #[inline] + fn label_arg(&self) -> Option<Arg<Label>> { + match self { + Self::JumpBackward { delta: l } + | Self::JumpBackwardNoInterrupt { delta: l } + | Self::JumpForward { delta: l } + | Self::PopJumpIfTrue { delta: l } + | Self::PopJumpIfFalse { delta: l } + | Self::PopJumpIfNone { delta: l } + | Self::PopJumpIfNotNone { delta: l } + | Self::ForIter { delta: l } + | Self::Send { delta: l } => Some(*l), + _ => None, + } + } + + fn is_unconditional_jump(&self) -> bool { + matches!( + self, + Self::JumpForward { .. } + | Self::JumpBackward { .. } + | Self::JumpBackwardNoInterrupt { .. } + ) + } + + fn is_scope_exit(&self) -> bool { + matches!( + self, + Self::ReturnValue | Self::RaiseVarargs { .. } | Self::Reraise { .. } + ) + } + + fn stack_effect_info(&self, oparg: u32) -> StackEffect { + // Reason for converting oparg to i32 is because of expressions like `1 + (oparg -1)` + // that causes underflow errors. + let oparg = i32::try_from(oparg).expect("oparg does not fit in an `i32`"); + + // NOTE: Please don't "simplify" expressions here (i.e. `1 + (oparg - 1)`) + // as it will be harder to see diff with what CPython auto-generates + let (pushed, popped) = match self { + Self::BinaryOp { .. } => (1, 2), + Self::BinaryOpAddFloat => (1, 2), + Self::BinaryOpAddInt => (1, 2), + Self::BinaryOpAddUnicode => (1, 2), + Self::BinaryOpExtend => (1, 2), + Self::BinaryOpInplaceAddUnicode => (0, 2), + Self::BinaryOpMultiplyFloat => (1, 2), + Self::BinaryOpMultiplyInt => (1, 2), + Self::BinaryOpSubscrDict => (1, 2), + Self::BinaryOpSubscrGetitem => (0, 2), + Self::BinaryOpSubscrListInt => (1, 2), + Self::BinaryOpSubscrListSlice => (1, 2), + Self::BinaryOpSubscrStrInt => (1, 2), + Self::BinaryOpSubscrTupleInt => (1, 2), + Self::BinaryOpSubtractFloat => (1, 2), + Self::BinaryOpSubtractInt => (1, 2), + Self::BinarySlice { .. } => (1, 3), + Self::BuildInterpolation { .. } => (1, 2 + (oparg & 1)), + Self::BuildList { .. } => (1, oparg), + Self::BuildMap { .. } => (1, oparg * 2), + Self::BuildSet { .. } => (1, oparg), + Self::BuildSlice { .. } => (1, oparg), + Self::BuildString { .. } => (1, oparg), + Self::BuildTemplate { .. } => (1, 2), + Self::BuildTuple { .. } => (1, oparg), + Self::Cache => (0, 0), + Self::Call { .. } => (1, 2 + oparg), + Self::CallAllocAndEnterInit => (0, 2 + oparg), + Self::CallBoundMethodExactArgs => (0, 2 + oparg), + Self::CallBoundMethodGeneral => (0, 2 + oparg), + Self::CallBuiltinClass => (1, 2 + oparg), + Self::CallBuiltinFast => (1, 2 + oparg), + Self::CallBuiltinFastWithKeywords => (1, 2 + oparg), + Self::CallBuiltinO => (1, 2 + oparg), + Self::CallFunctionEx => (1, 4), + Self::CallIntrinsic1 { .. } => (1, 1), + Self::CallIntrinsic2 { .. } => (1, 2), + Self::CallIsinstance => (1, 2 + oparg), + Self::CallKw { .. } => (1, 3 + oparg), + Self::CallKwBoundMethod => (0, 3 + oparg), + Self::CallKwNonPy => (1, 3 + oparg), + Self::CallKwPy => (0, 3 + oparg), + Self::CallLen => (1, 3), + Self::CallListAppend => (0, 3), + Self::CallMethodDescriptorFast => (1, 2 + oparg), + Self::CallMethodDescriptorFastWithKeywords => (1, 2 + oparg), + Self::CallMethodDescriptorNoargs => (1, 2 + oparg), + Self::CallMethodDescriptorO => (1, 2 + oparg), + Self::CallNonPyGeneral => (1, 2 + oparg), + Self::CallPyExactArgs => (0, 2 + oparg), + Self::CallPyGeneral => (0, 2 + oparg), + Self::CallStr1 => (1, 3), + Self::CallTuple1 => (1, 3), + Self::CallType1 => (1, 3), + Self::CheckEgMatch => (2, 2), + Self::CheckExcMatch => (2, 2), + Self::CleanupThrow => (2, 3), + Self::CompareOp { .. } => (1, 2), + Self::CompareOpFloat => (1, 2), + Self::CompareOpInt => (1, 2), + Self::CompareOpStr => (1, 2), + Self::ContainsOp { .. } => (1, 2), + Self::ContainsOpDict => (1, 2), + Self::ContainsOpSet => (1, 2), + Self::ConvertValue { .. } => (1, 1), + Self::Copy { .. } => (2 + (oparg - 1), 1 + (oparg - 1)), + Self::CopyFreeVars { .. } => (0, 0), + Self::DeleteAttr { .. } => (0, 1), + Self::DeleteDeref { .. } => (0, 0), + Self::DeleteFast { .. } => (0, 0), + Self::DeleteGlobal { .. } => (0, 0), + Self::DeleteName { .. } => (0, 0), + Self::DeleteSubscr => (0, 2), + Self::DictMerge { .. } => (4 + (oparg - 1), 5 + (oparg - 1)), + Self::DictUpdate { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::EndAsyncFor => (0, 2), + Self::EndFor => (0, 1), + Self::EndSend => (1, 2), + Self::EnterExecutor => (0, 0), + Self::ExitInitCheck => (0, 1), + Self::ExtendedArg => (0, 0), + Self::ForIter { .. } => (2, 1), + Self::ForIterGen => (1, 1), + Self::ForIterList => (2, 1), + Self::ForIterRange => (2, 1), + Self::ForIterTuple => (2, 1), + Self::FormatSimple => (1, 1), + Self::FormatWithSpec => (1, 2), + Self::GetAIter => (1, 1), + Self::GetANext => (2, 1), + Self::GetAwaitable { .. } => (1, 1), + Self::GetIter => (1, 1), + Self::GetLen => (2, 1), + Self::GetYieldFromIter => (1, 1), + Self::ImportFrom { .. } => (2, 1), + Self::ImportName { .. } => (1, 2), + Self::InstrumentedCall => (1, 2 + oparg), + Self::InstrumentedCallFunctionEx => (1, 4), + Self::InstrumentedCallKw => (1, 3 + oparg), + Self::InstrumentedEndAsyncFor => (0, 2), + Self::InstrumentedEndFor => (1, 2), + Self::InstrumentedEndSend => (1, 2), + Self::InstrumentedForIter => (2, 1), + Self::InstrumentedInstruction => (0, 0), + Self::InstrumentedJumpBackward => (0, 0), + Self::InstrumentedJumpForward => (0, 0), + Self::InstrumentedLine => (0, 0), + Self::InstrumentedLoadSuperAttr => (1 + (oparg & 1), 3), + Self::InstrumentedNotTaken => (0, 0), + Self::InstrumentedPopIter => (0, 1), + Self::InstrumentedPopJumpIfFalse => (0, 1), + Self::InstrumentedPopJumpIfNone => (0, 1), + Self::InstrumentedPopJumpIfNotNone => (0, 1), + Self::InstrumentedPopJumpIfTrue => (0, 1), + Self::InstrumentedResume => (0, 0), + Self::InstrumentedReturnValue => (1, 1), + Self::InstrumentedYieldValue => (1, 1), + Self::InterpreterExit => (0, 1), + Self::IsOp { .. } => (1, 2), + Self::JumpBackward { .. } => (0, 0), + Self::JumpBackwardJit => (0, 0), + Self::JumpBackwardNoInterrupt { .. } => (0, 0), + Self::JumpBackwardNoJit => (0, 0), + Self::JumpForward { .. } => (0, 0), + Self::ListAppend { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::ListExtend { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::LoadAttr { .. } => (1 + (oparg & 1), 1), + Self::LoadAttrClass => (1 + (oparg & 1), 1), + Self::LoadAttrClassWithMetaclassCheck => (1 + (oparg & 1), 1), + Self::LoadAttrGetattributeOverridden => (1, 1), + Self::LoadAttrInstanceValue => (1 + (oparg & 1), 1), + Self::LoadAttrMethodLazyDict => (2, 1), + Self::LoadAttrMethodNoDict => (2, 1), + Self::LoadAttrMethodWithValues => (2, 1), + Self::LoadAttrModule => (1 + (oparg & 1), 1), + Self::LoadAttrNondescriptorNoDict => (1, 1), + Self::LoadAttrNondescriptorWithValues => (1, 1), + Self::LoadAttrProperty => (0, 1), + Self::LoadAttrSlot => (1 + (oparg & 1), 1), + Self::LoadAttrWithHint => (1 + (oparg & 1), 1), + Self::LoadBuildClass => (1, 0), + Self::LoadCommonConstant { .. } => (1, 0), + Self::LoadConst { .. } => (1, 0), + Self::LoadConstImmortal => (1, 0), + Self::LoadConstMortal => (1, 0), + Self::LoadDeref { .. } => (1, 0), + Self::LoadFast { .. } => (1, 0), + Self::LoadFastAndClear { .. } => (1, 0), + Self::LoadFastBorrow { .. } => (1, 0), + Self::LoadFastBorrowLoadFastBorrow { .. } => (2, 0), + Self::LoadFastCheck { .. } => (1, 0), + Self::LoadFastLoadFast { .. } => (2, 0), + Self::LoadFromDictOrDeref { .. } => (1, 1), + Self::LoadFromDictOrGlobals { .. } => (1, 1), + Self::LoadGlobal { .. } => (1 + (oparg & 1), 0), + Self::LoadGlobalBuiltin => (1 + (oparg & 1), 0), + Self::LoadGlobalModule => (1 + (oparg & 1), 0), + Self::LoadLocals => (1, 0), + Self::LoadName { .. } => (1, 0), + Self::LoadSmallInt { .. } => (1, 0), + Self::LoadSpecial { .. } => (1, 1), + Self::LoadSuperAttr { .. } => (1 + (oparg & 1), 3), + Self::LoadSuperAttrAttr => (1, 3), + Self::LoadSuperAttrMethod => (2, 3), + Self::MakeCell { .. } => (0, 0), + Self::MakeFunction { .. } => (1, 1), + Self::MapAdd { .. } => (1 + (oparg - 1), 3 + (oparg - 1)), + Self::MatchClass { .. } => (1, 3), + Self::MatchKeys { .. } => (3, 2), + Self::MatchMapping => (2, 1), + Self::MatchSequence => (2, 1), + Self::Nop => (0, 0), + Self::NotTaken => (0, 0), + Self::PopExcept => (0, 1), + Self::PopIter => (0, 1), + Self::PopJumpIfFalse { .. } => (0, 1), + Self::PopJumpIfNone { .. } => (0, 1), + Self::PopJumpIfNotNone { .. } => (0, 1), + Self::PopJumpIfTrue { .. } => (0, 1), + Self::PopTop => (0, 1), + Self::PushExcInfo => (2, 1), + Self::PushNull => (1, 0), + Self::RaiseVarargs { .. } => (0, oparg), + Self::Reraise { .. } => (oparg, 1 + oparg), + Self::Reserved => (0, 0), + Self::Resume { .. } => (0, 0), + Self::ResumeCheck => (0, 0), + Self::ReturnGenerator => (1, 0), + Self::ReturnValue => (1, 1), + Self::Send { .. } => (2, 2), + Self::SendGen => (1, 2), + Self::SetAdd { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::SetFunctionAttribute { .. } => (1, 2), + Self::SetUpdate { .. } => (1 + (oparg - 1), 2 + (oparg - 1)), + Self::SetupAnnotations => (0, 0), + Self::StoreAttr { .. } => (0, 2), + Self::StoreAttrInstanceValue => (0, 2), + Self::StoreAttrSlot => (0, 2), + Self::StoreAttrWithHint => (0, 2), + Self::StoreDeref { .. } => (0, 1), + Self::StoreFast { .. } => (0, 1), + Self::StoreFastLoadFast { .. } => (1, 1), + Self::StoreFastStoreFast { .. } => (0, 2), + Self::StoreGlobal { .. } => (0, 1), + Self::StoreName { .. } => (0, 1), + Self::StoreSlice => (0, 4), + Self::StoreSubscr => (0, 3), + Self::StoreSubscrDict => (0, 3), + Self::StoreSubscrListInt => (0, 3), + Self::Swap { .. } => (2 + (oparg - 2), 2 + (oparg - 2)), + Self::ToBool => (1, 1), + Self::ToBoolAlwaysTrue => (1, 1), + Self::ToBoolBool => (1, 1), + Self::ToBoolInt => (1, 1), + Self::ToBoolList => (1, 1), + Self::ToBoolNone => (1, 1), + Self::ToBoolStr => (1, 1), + Self::UnaryInvert => (1, 1), + Self::UnaryNegative => (1, 1), + Self::UnaryNot => (1, 1), + Self::UnpackEx { .. } => (1 + (oparg & 0xFF) + (oparg >> 8), 1), + Self::UnpackSequence { .. } => (oparg, 1), + Self::UnpackSequenceList => (oparg, 1), + Self::UnpackSequenceTuple => (oparg, 1), + Self::UnpackSequenceTwoTuple => (2, 1), + Self::WithExceptStart => (6, 5), + Self::YieldValue { .. } => (1, 1), + }; + + debug_assert!((0..=i32::MAX).contains(&pushed)); + debug_assert!((0..=i32::MAX).contains(&popped)); + + StackEffect::new(pushed as u32, popped as u32) + } + + #[allow(clippy::too_many_arguments)] + fn fmt_dis( + &self, + arg: OpArg, + f: &mut fmt::Formatter<'_>, + ctx: &impl InstrDisplayContext, + expand_code_objects: bool, + pad: usize, + level: usize, + ) -> fmt::Result { + macro_rules! w { + ($variant:ident) => { + write!(f, stringify!($variant)) + }; + ($variant:ident, $map:ident = $arg_marker:expr) => {{ + let arg = $arg_marker.get(arg); + write!(f, "{:pad$}({}, {})", stringify!($variant), arg, $map(arg)) + }}; + ($variant:ident, $arg_marker:expr) => { + write!(f, "{:pad$}({})", stringify!($variant), $arg_marker.get(arg)) + }; + ($variant:ident, ?$arg_marker:expr) => { + write!( + f, + "{:pad$}({:?})", + stringify!($variant), + $arg_marker.get(arg) + ) + }; + } + + let varname = |var_num: oparg::VarNum| ctx.get_varname(var_num); + let name = |i: u32| ctx.get_name(i as usize); + let cell_name = |i: u32| ctx.get_cell_name(i as usize); + + let fmt_const = |op: &str, + arg: OpArg, + f: &mut fmt::Formatter<'_>, + consti: &Arg<oparg::ConstIdx>| + -> fmt::Result { + let value = ctx.get_constant(consti.get(arg)); + match value.borrow_constant() { + BorrowedConstant::Code { code } if expand_code_objects => { + write!(f, "{op:pad$}({code:?}):")?; + code.display_inner(f, true, level + 1)?; + Ok(()) + } + c => { + write!(f, "{op:pad$}(")?; + c.fmt_display(f)?; + write!(f, ")") + } + } + }; + + match self { + Self::BinarySlice => w!(BINARY_SLICE), + Self::BinaryOp { op } => write!(f, "{:pad$}({})", "BINARY_OP", op.get(arg)), + Self::BinaryOpInplaceAddUnicode => w!(BINARY_OP_INPLACE_ADD_UNICODE), + Self::BuildList { count } => w!(BUILD_LIST, count), + Self::BuildMap { count } => w!(BUILD_MAP, count), + Self::BuildSet { count } => w!(BUILD_SET, count), + Self::BuildSlice { argc } => w!(BUILD_SLICE, ?argc), + Self::BuildString { count } => w!(BUILD_STRING, count), + Self::BuildTuple { count } => w!(BUILD_TUPLE, count), + Self::Call { argc } => w!(CALL, argc), + Self::CallFunctionEx => w!(CALL_FUNCTION_EX), + Self::CallKw { argc } => w!(CALL_KW, argc), + Self::CallIntrinsic1 { func } => w!(CALL_INTRINSIC_1, ?func), + Self::CallIntrinsic2 { func } => w!(CALL_INTRINSIC_2, ?func), + Self::Cache => w!(CACHE), + Self::CheckEgMatch => w!(CHECK_EG_MATCH), + Self::CheckExcMatch => w!(CHECK_EXC_MATCH), + Self::CleanupThrow => w!(CLEANUP_THROW), + Self::CompareOp { opname } => w!(COMPARE_OP, ?opname), + Self::ContainsOp { invert } => w!(CONTAINS_OP, ?invert), + Self::ConvertValue { oparg } => write!(f, "{:pad$}{}", "CONVERT_VALUE", oparg.get(arg)), + Self::Copy { i } => w!(COPY, i), + Self::CopyFreeVars { n } => w!(COPY_FREE_VARS, n), + Self::DeleteAttr { namei } => w!(DELETE_ATTR, name = namei), + Self::DeleteDeref { i } => w!(DELETE_DEREF, cell_name = i), + Self::DeleteFast { var_num } => w!(DELETE_FAST, varname = var_num), + Self::DeleteGlobal { namei } => w!(DELETE_GLOBAL, name = namei), + Self::DeleteName { namei } => w!(DELETE_NAME, name = namei), + Self::DeleteSubscr => w!(DELETE_SUBSCR), + Self::DictMerge { i } => w!(DICT_MERGE, i), + Self::DictUpdate { i } => w!(DICT_UPDATE, i), + Self::EndAsyncFor => w!(END_ASYNC_FOR), + Self::EndSend => w!(END_SEND), + Self::ExtendedArg => w!(EXTENDED_ARG, Arg::<u32>::marker()), + Self::ExitInitCheck => w!(EXIT_INIT_CHECK), + Self::ForIter { delta } => w!(FOR_ITER, delta), + Self::FormatSimple => w!(FORMAT_SIMPLE), + Self::FormatWithSpec => w!(FORMAT_WITH_SPEC), + Self::GetAIter => w!(GET_AITER), + Self::GetANext => w!(GET_ANEXT), + Self::GetAwaitable { r#where } => w!(GET_AWAITABLE, r#where), + Self::Reserved => w!(RESERVED), + Self::GetIter => w!(GET_ITER), + Self::GetLen => w!(GET_LEN), + Self::ImportFrom { namei } => w!(IMPORT_FROM, name = namei), + Self::ImportName { namei } => w!(IMPORT_NAME, name = namei), + Self::InterpreterExit => w!(INTERPRETER_EXIT), + Self::IsOp { invert } => w!(IS_OP, ?invert), + Self::JumpBackward { delta } => w!(JUMP_BACKWARD, delta), + Self::JumpBackwardNoInterrupt { delta } => w!(JUMP_BACKWARD_NO_INTERRUPT, delta), + Self::JumpForward { delta } => w!(JUMP_FORWARD, delta), + Self::ListAppend { i } => w!(LIST_APPEND, i), + Self::ListExtend { i } => w!(LIST_EXTEND, i), + Self::LoadAttr { namei } => { + let oparg = namei.get(arg); + let oparg_u32 = u32::from(oparg); + let attr_name = name(oparg.name_idx()); + if oparg.is_method() { + write!( + f, + "{:pad$}({}, {}, method=true)", + "LOAD_ATTR", oparg_u32, attr_name + ) + } else { + write!(f, "{:pad$}({}, {})", "LOAD_ATTR", oparg_u32, attr_name) + } + } + Self::LoadBuildClass => w!(LOAD_BUILD_CLASS), + Self::LoadCommonConstant { idx } => w!(LOAD_COMMON_CONSTANT, ?idx), + Self::LoadFromDictOrDeref { i } => w!(LOAD_FROM_DICT_OR_DEREF, cell_name = i), + Self::LoadConst { consti } => fmt_const("LOAD_CONST", arg, f, consti), + Self::LoadSmallInt { i } => w!(LOAD_SMALL_INT, i), + Self::LoadDeref { i } => w!(LOAD_DEREF, cell_name = i), + Self::LoadFast { var_num } => w!(LOAD_FAST, varname = var_num), + Self::LoadFastAndClear { var_num } => w!(LOAD_FAST_AND_CLEAR, varname = var_num), + Self::LoadFastBorrow { var_num } => w!(LOAD_FAST_BORROW, varname = var_num), + Self::LoadFastCheck { var_num } => w!(LOAD_FAST_CHECK, varname = var_num), + Self::LoadFastLoadFast { var_nums } => { + let oparg = var_nums.get(arg); + let (idx1, idx2) = oparg.indexes(); + let name1 = varname(idx1); + let name2 = varname(idx2); + write!(f, "{:pad$}({}, {})", "LOAD_FAST_LOAD_FAST", name1, name2) + } + Self::LoadFastBorrowLoadFastBorrow { var_nums } => { + let oparg = var_nums.get(arg); + let (idx1, idx2) = oparg.indexes(); + let name1 = varname(idx1); + let name2 = varname(idx2); + write!( + f, + "{:pad$}({}, {})", + "LOAD_FAST_BORROW_LOAD_FAST_BORROW", name1, name2 + ) + } + Self::LoadFromDictOrGlobals { i } => w!(LOAD_FROM_DICT_OR_GLOBALS, name = i), + Self::LoadGlobal { namei } => { + let oparg = namei.get(arg); + let name_idx = oparg >> 1; + if (oparg & 1) != 0 { + write!( + f, + "{:pad$}({}, NULL + {})", + "LOAD_GLOBAL", + oparg, + name(name_idx) + ) + } else { + write!(f, "{:pad$}({}, {})", "LOAD_GLOBAL", oparg, name(name_idx)) + } + } + Self::LoadGlobalBuiltin => { + let oparg = u32::from(arg); + let name_idx = oparg >> 1; + if (oparg & 1) != 0 { + write!( + f, + "{:pad$}({}, NULL + {})", + "LOAD_GLOBAL_BUILTIN", + oparg, + name(name_idx) + ) + } else { + write!( + f, + "{:pad$}({}, {})", + "LOAD_GLOBAL_BUILTIN", + oparg, + name(name_idx) + ) + } + } + Self::LoadGlobalModule => { + let oparg = u32::from(arg); + let name_idx = oparg >> 1; + if (oparg & 1) != 0 { + write!( + f, + "{:pad$}({}, NULL + {})", + "LOAD_GLOBAL_MODULE", + oparg, + name(name_idx) + ) + } else { + write!( + f, + "{:pad$}({}, {})", + "LOAD_GLOBAL_MODULE", + oparg, + name(name_idx) + ) + } + } + Self::LoadLocals => w!(LOAD_LOCALS), + Self::LoadName { namei } => w!(LOAD_NAME, name = namei), + Self::LoadSpecial { method } => w!(LOAD_SPECIAL, method), + Self::LoadSuperAttr { namei } => { + let oparg = namei.get(arg); + write!( + f, + "{:pad$}({}, {}, method={}, class={})", + "LOAD_SUPER_ATTR", + u32::from(oparg), + name(oparg.name_idx()), + oparg.is_load_method(), + oparg.has_class() + ) + } + Self::MakeCell { i } => w!(MAKE_CELL, cell_name = i), + Self::MakeFunction => w!(MAKE_FUNCTION), + Self::MapAdd { i } => w!(MAP_ADD, i), + Self::MatchClass { count } => w!(MATCH_CLASS, count), + Self::MatchKeys => w!(MATCH_KEYS), + Self::MatchMapping => w!(MATCH_MAPPING), + Self::MatchSequence => w!(MATCH_SEQUENCE), + Self::Nop => w!(NOP), + Self::NotTaken => w!(NOT_TAKEN), + Self::PopExcept => w!(POP_EXCEPT), + Self::PopJumpIfFalse { delta } => w!(POP_JUMP_IF_FALSE, delta), + Self::PopJumpIfNone { delta } => w!(POP_JUMP_IF_NONE, delta), + Self::PopJumpIfNotNone { delta } => w!(POP_JUMP_IF_NOT_NONE, delta), + Self::PopJumpIfTrue { delta } => w!(POP_JUMP_IF_TRUE, delta), + Self::PopTop => w!(POP_TOP), + Self::EndFor => w!(END_FOR), + Self::PopIter => w!(POP_ITER), + Self::PushExcInfo => w!(PUSH_EXC_INFO), + Self::PushNull => w!(PUSH_NULL), + Self::RaiseVarargs { argc } => w!(RAISE_VARARGS, ?argc), + Self::Reraise { depth } => w!(RERAISE, depth), + Self::Resume { context } => w!(RESUME, context), + Self::ReturnValue => w!(RETURN_VALUE), + Self::ReturnGenerator => w!(RETURN_GENERATOR), + Self::Send { delta } => w!(SEND, delta), + Self::SetAdd { i } => w!(SET_ADD, i), + Self::SetFunctionAttribute { flag } => w!(SET_FUNCTION_ATTRIBUTE, ?flag), + Self::SetupAnnotations => w!(SETUP_ANNOTATIONS), + Self::SetUpdate { i } => w!(SET_UPDATE, i), + Self::StoreAttr { namei } => w!(STORE_ATTR, name = namei), + Self::StoreDeref { i } => w!(STORE_DEREF, cell_name = i), + Self::StoreFast { var_num } => w!(STORE_FAST, varname = var_num), + Self::StoreFastLoadFast { var_nums } => { + let oparg = var_nums.get(arg); + let (store_idx, load_idx) = oparg.indexes(); + write!(f, "STORE_FAST_LOAD_FAST")?; + write!(f, " ({}, {})", store_idx, load_idx) + } + Self::StoreFastStoreFast { var_nums } => { + let oparg = var_nums.get(arg); + let (idx1, idx2) = oparg.indexes(); + write!( + f, + "{:pad$}({}, {})", + "STORE_FAST_STORE_FAST", + varname(idx1), + varname(idx2) + ) + } + Self::StoreGlobal { namei } => w!(STORE_GLOBAL, name = namei), + Self::StoreName { namei } => w!(STORE_NAME, name = namei), + Self::StoreSlice => w!(STORE_SLICE), + Self::StoreSubscr => w!(STORE_SUBSCR), + Self::Swap { i } => w!(SWAP, i), + Self::ToBool => w!(TO_BOOL), + Self::UnpackEx { counts } => w!(UNPACK_EX, counts), + Self::UnpackSequence { count } => w!(UNPACK_SEQUENCE, count), + Self::WithExceptStart => w!(WITH_EXCEPT_START), + Self::UnaryInvert => w!(UNARY_INVERT), + Self::UnaryNegative => w!(UNARY_NEGATIVE), + Self::UnaryNot => w!(UNARY_NOT), + Self::YieldValue { arg } => w!(YIELD_VALUE, arg), + Self::GetYieldFromIter => w!(GET_YIELD_FROM_ITER), + Self::BuildTemplate => w!(BUILD_TEMPLATE), + Self::BuildInterpolation { format } => w!(BUILD_INTERPOLATION, format), + _ => w!(RUSTPYTHON_PLACEHOLDER), + } + } +} + +/// Instructions used by the compiler. They are not executed by the VM. +/// +/// CPython 3.14.2 aligned (256-266). +#[derive(Clone, Copy, Debug)] +#[repr(u16)] +pub enum PseudoInstruction { + // CPython 3.14.2 pseudo instructions (256-266) + AnnotationsPlaceholder = 256, + Jump { delta: Arg<Label> } = 257, + JumpIfFalse { delta: Arg<Label> } = 258, + JumpIfTrue { delta: Arg<Label> } = 259, + JumpNoInterrupt { delta: Arg<Label> } = 260, + LoadClosure { i: Arg<NameIdx> } = 261, + PopBlock = 262, + SetupCleanup { delta: Arg<Label> } = 263, + SetupFinally { delta: Arg<Label> } = 264, + SetupWith { delta: Arg<Label> } = 265, + StoreFastMaybeNull { var_num: Arg<NameIdx> } = 266, +} + +const _: () = assert!(mem::size_of::<PseudoInstruction>() == 2); + +impl From<PseudoInstruction> for u16 { + #[inline] + fn from(ins: PseudoInstruction) -> Self { + // SAFETY: there's no padding bits + unsafe { mem::transmute::<PseudoInstruction, Self>(ins) } + } +} + +impl TryFrom<u16> for PseudoInstruction { + type Error = MarshalError; + + #[inline] + fn try_from(value: u16) -> Result<Self, MarshalError> { + let start = u16::from(Self::AnnotationsPlaceholder); + let end = u16::from(Self::StoreFastMaybeNull { + var_num: Arg::marker(), + }); + + if (start..=end).contains(&value) { + Ok(unsafe { mem::transmute::<u16, Self>(value) }) + } else { + Err(Self::Error::InvalidBytecode) + } + } +} + +impl PseudoInstruction { + /// Returns true if this is a block push pseudo instruction + /// (SETUP_FINALLY, SETUP_CLEANUP, or SETUP_WITH). + pub fn is_block_push(&self) -> bool { + matches!( + self, + Self::SetupCleanup { .. } | Self::SetupFinally { .. } | Self::SetupWith { .. } + ) + } +} + +impl InstructionMetadata for PseudoInstruction { + fn label_arg(&self) -> Option<Arg<Label>> { + match self { + Self::Jump { delta: l } + | Self::JumpIfFalse { delta: l } + | Self::JumpIfTrue { delta: l } + | Self::JumpNoInterrupt { delta: l } + | Self::SetupCleanup { delta: l } + | Self::SetupFinally { delta: l } + | Self::SetupWith { delta: l } => Some(*l), + _ => None, + } + } + + fn is_scope_exit(&self) -> bool { + false + } + + fn stack_effect_info(&self, _oparg: u32) -> StackEffect { + // Reason for converting oparg to i32 is because of expressions like `1 + (oparg -1)` + // that causes underflow errors. + let _oparg = i32::try_from(_oparg).expect("oparg does not fit in an `i32`"); + + // NOTE: Please don't "simplify" expressions here (i.e. `1 + (oparg - 1)`) + // as it will be harder to see diff with what CPython auto-generates + let (pushed, popped) = match self { + Self::AnnotationsPlaceholder => (0, 0), + Self::Jump { .. } => (0, 0), + Self::JumpIfFalse { .. } => (1, 1), + Self::JumpIfTrue { .. } => (1, 1), + Self::JumpNoInterrupt { .. } => (0, 0), + Self::LoadClosure { .. } => (1, 0), + Self::PopBlock => (0, 0), + // Normal path effect is 0 (these are NOPs on fall-through). + // Handler entry effects are computed directly in max_stackdepth(). + Self::SetupCleanup { .. } => (0, 0), + Self::SetupFinally { .. } => (0, 0), + Self::SetupWith { .. } => (0, 0), + Self::StoreFastMaybeNull { .. } => (0, 1), + }; + + debug_assert!((0..=i32::MAX).contains(&pushed)); + debug_assert!((0..=i32::MAX).contains(&popped)); + + StackEffect::new(pushed as u32, popped as u32) + } + + fn is_unconditional_jump(&self) -> bool { + matches!(self, Self::Jump { .. } | Self::JumpNoInterrupt { .. }) + } + + fn fmt_dis( + &self, + _arg: OpArg, + _f: &mut fmt::Formatter<'_>, + _ctx: &impl InstrDisplayContext, + _expand_code_objects: bool, + _pad: usize, + _level: usize, + ) -> fmt::Result { + unimplemented!() + } +} + +#[derive(Clone, Copy, Debug)] +pub enum AnyInstruction { + Real(Instruction), + Pseudo(PseudoInstruction), +} + +impl From<Instruction> for AnyInstruction { + fn from(value: Instruction) -> Self { + Self::Real(value) + } +} + +impl From<PseudoInstruction> for AnyInstruction { + fn from(value: PseudoInstruction) -> Self { + Self::Pseudo(value) + } +} + +impl TryFrom<u8> for AnyInstruction { + type Error = MarshalError; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + Ok(Instruction::try_from(value)?.into()) + } +} + +impl TryFrom<u16> for AnyInstruction { + type Error = MarshalError; + + fn try_from(value: u16) -> Result<Self, Self::Error> { + match u8::try_from(value) { + Ok(v) => v.try_into(), + Err(_) => Ok(PseudoInstruction::try_from(value)?.into()), + } + } +} + +macro_rules! inst_either { + (fn $name:ident ( &self $(, $arg:ident : $arg_ty:ty )* ) -> $ret:ty ) => { + fn $name(&self $(, $arg : $arg_ty )* ) -> $ret { + match self { + Self::Real(op) => op.$name($($arg),*), + Self::Pseudo(op) => op.$name($($arg),*), + } + } + }; +} + +impl InstructionMetadata for AnyInstruction { + inst_either!(fn label_arg(&self) -> Option<Arg<Label>>); + + inst_either!(fn is_unconditional_jump(&self) -> bool); + + inst_either!(fn is_scope_exit(&self) -> bool); + + inst_either!(fn stack_effect(&self, oparg: u32) -> i32); + + inst_either!(fn stack_effect_info(&self, oparg: u32) -> StackEffect); + + inst_either!(fn fmt_dis( + &self, + arg: OpArg, + f: &mut fmt::Formatter<'_>, + ctx: &impl InstrDisplayContext, + expand_code_objects: bool, + pad: usize, + level: usize + ) -> fmt::Result); +} + +impl AnyInstruction { + /// Gets the inner value of [`Self::Real`]. + pub const fn real(self) -> Option<Instruction> { + match self { + Self::Real(ins) => Some(ins), + _ => None, + } + } + + /// Gets the inner value of [`Self::Pseudo`]. + pub const fn pseudo(self) -> Option<PseudoInstruction> { + match self { + Self::Pseudo(ins) => Some(ins), + _ => None, + } + } + + /// Same as [`Self::real`] but panics if wasn't called on [`Self::Real`]. + /// + /// # Panics + /// + /// If was called on something else other than [`Self::Real`]. + pub const fn expect_real(self) -> Instruction { + self.real() + .expect("Expected Instruction::Real, found Instruction::Pseudo") + } + + /// Same as [`Self::pseudo`] but panics if wasn't called on [`Self::Pseudo`]. + /// + /// # Panics + /// + /// If was called on something else other than [`Self::Pseudo`]. + pub const fn expect_pseudo(self) -> PseudoInstruction { + self.pseudo() + .expect("Expected Instruction::Pseudo, found Instruction::Real") + } + + /// Returns true if this is a block push pseudo instruction + /// (SETUP_FINALLY, SETUP_CLEANUP, or SETUP_WITH). + pub fn is_block_push(&self) -> bool { + matches!(self, Self::Pseudo(p) if p.is_block_push()) + } + + /// Returns true if this is a POP_BLOCK pseudo instruction. + pub fn is_pop_block(&self) -> bool { + matches!(self, Self::Pseudo(PseudoInstruction::PopBlock)) + } +} + +/// What effect the instruction has on the stack. +#[derive(Clone, Copy)] +pub struct StackEffect { + /// How many items the instruction is pushing on the stack. + pushed: u32, + /// How many items the instruction is popping from the stack. + popped: u32, +} + +impl StackEffect { + /// Creates a new [`Self`]. + pub const fn new(pushed: u32, popped: u32) -> Self { + Self { pushed, popped } + } + + /// Get the calculated stack effect as [`i32`]. + pub fn effect(self) -> i32 { + self.into() + } + + /// Get the pushed count. + pub const fn pushed(self) -> u32 { + self.pushed + } + + /// Get the popped count. + pub const fn popped(self) -> u32 { + self.popped + } +} + +impl From<StackEffect> for i32 { + fn from(effect: StackEffect) -> Self { + (effect.pushed() as i32) - (effect.popped() as i32) + } +} + +pub trait InstructionMetadata { + /// Gets the label stored inside this instruction, if it exists. + fn label_arg(&self) -> Option<Arg<Label>>; + + fn is_scope_exit(&self) -> bool; + + fn is_unconditional_jump(&self) -> bool; + + /// Stack effect info for how many items are pushed/popped from the stack, + /// for this instruction. + fn stack_effect_info(&self, oparg: u32) -> StackEffect; + + /// Stack effect of [`Self::stack_effect_info`]. + fn stack_effect(&self, oparg: u32) -> i32 { + self.stack_effect_info(oparg).effect() + } + + #[allow(clippy::too_many_arguments)] + fn fmt_dis( + &self, + arg: OpArg, + f: &mut fmt::Formatter<'_>, + ctx: &impl InstrDisplayContext, + expand_code_objects: bool, + pad: usize, + level: usize, + ) -> fmt::Result; + + fn display(&self, arg: OpArg, ctx: &impl InstrDisplayContext) -> impl fmt::Display { + fmt::from_fn(move |f| self.fmt_dis(arg, f, ctx, false, 0, 0)) + } +} + +#[derive(Copy, Clone)] +pub struct Arg<T: OpArgType>(PhantomData<T>); + +impl<T: OpArgType> Arg<T> { + #[inline] + pub const fn marker() -> Self { + Self(PhantomData) + } + + #[inline] + pub fn new(arg: T) -> (Self, OpArg) { + (Self(PhantomData), OpArg::new(arg.into())) + } + + #[inline] + pub fn new_single(arg: T) -> (Self, OpArgByte) + where + T: Into<u8>, + { + (Self(PhantomData), OpArgByte::new(arg.into())) + } + + #[inline(always)] + pub fn get(self, arg: OpArg) -> T { + self.try_get(arg).unwrap() + } + + #[inline(always)] + pub fn try_get(self, arg: OpArg) -> Result<T, MarshalError> { + T::try_from(u32::from(arg)).map_err(|_| MarshalError::InvalidBytecode) + } + + /// # Safety + /// T::from_op_arg(self) must succeed + #[inline(always)] + pub unsafe fn get_unchecked(self, arg: OpArg) -> T { + // SAFETY: requirements forwarded from caller + unsafe { T::try_from(u32::from(arg)).unwrap_unchecked() } + } +} + +impl<T: OpArgType> PartialEq for Arg<T> { + fn eq(&self, _: &Self) -> bool { + true + } +} + +impl<T: OpArgType> Eq for Arg<T> {} + +impl<T: OpArgType> fmt::Debug for Arg<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Arg<{}>", core::any::type_name::<T>()) + } +} diff --git a/crates/compiler-core/src/bytecode/oparg.rs b/crates/compiler-core/src/bytecode/oparg.rs new file mode 100644 index 00000000000..2fa40ce2271 --- /dev/null +++ b/crates/compiler-core/src/bytecode/oparg.rs @@ -0,0 +1,877 @@ +use core::fmt; + +use crate::{ + bytecode::{CodeUnit, instruction::Instruction}, + marshal::MarshalError, +}; + +pub trait OpArgType: Copy + Into<u32> + TryFrom<u32> {} + +/// Opcode argument that may be extended by a prior ExtendedArg. +#[derive(Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct OpArgByte(u8); + +impl OpArgByte { + pub const NULL: Self = Self::new(0); + + #[must_use] + pub const fn new(value: u8) -> Self { + Self(value) + } +} + +impl From<u8> for OpArgByte { + fn from(raw: u8) -> Self { + Self::new(raw) + } +} + +impl From<OpArgByte> for u8 { + fn from(value: OpArgByte) -> Self { + value.0 + } +} + +impl fmt::Debug for OpArgByte { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// Full 32-bit op_arg, including any possible ExtendedArg extension. +#[derive(Copy, Clone, Debug)] +#[repr(transparent)] +pub struct OpArg(u32); + +impl OpArg { + pub const NULL: Self = Self::new(0); + + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value) + } + + /// Returns how many CodeUnits a instruction with this op_arg will be encoded as + #[inline] + pub const fn instr_size(self) -> usize { + (self.0 > 0xff) as usize + (self.0 > 0xff_ff) as usize + (self.0 > 0xff_ff_ff) as usize + 1 + } + + /// returns the arg split into any necessary ExtendedArg components (in big-endian order) and + /// the arg for the real opcode itself + #[inline(always)] + pub fn split(self) -> (impl ExactSizeIterator<Item = OpArgByte>, OpArgByte) { + let mut it = self + .0 + .to_le_bytes() + .map(OpArgByte) + .into_iter() + .take(self.instr_size()); + let lo = it.next().unwrap(); + (it.rev(), lo) + } +} + +impl From<u32> for OpArg { + fn from(raw: u32) -> Self { + Self::new(raw) + } +} + +impl From<OpArg> for u32 { + fn from(value: OpArg) -> Self { + value.0 + } +} + +#[derive(Default, Copy, Clone)] +#[repr(transparent)] +pub struct OpArgState { + state: u32, +} + +impl OpArgState { + #[inline(always)] + pub fn get(&mut self, ins: CodeUnit) -> (Instruction, OpArg) { + let arg = self.extend(ins.arg); + if !matches!(ins.op, Instruction::ExtendedArg) { + self.reset(); + } + (ins.op, arg) + } + + #[inline(always)] + pub fn extend(&mut self, arg: OpArgByte) -> OpArg { + self.state = (self.state << 8) | u32::from(arg.0); + self.state.into() + } + + #[inline(always)] + pub const fn reset(&mut self) { + self.state = 0 + } +} + +/// Helper macro for defining oparg enums in an optimal way. +/// +/// Will generate the following: +/// +/// - Enum which variant's aren't assigned any value (for optimizations). +/// - impl [`TryFrom<u8>`] +/// - impl [`TryFrom<u32>`] +/// - impl [`Into<u8>`] +/// - impl [`Into<u32>`] +/// - impl [`OpArgType`] +/// +/// # Note +/// If an enum variant has "alternative" values (i.e. `Foo = 0 | 1`), the first value will be the +/// result of converting to a number. +/// +/// # Examples +/// +/// ```ignore +/// oparg_enum!( +/// /// Oparg for the `X` opcode. +/// #[derive(Clone, Copy)] +/// pub enum MyOpArg { +/// /// Some doc. +/// Foo = 4, +/// Bar = 8, +/// Baz = 15 | 16, +/// Qux = 23 | 42 +/// } +/// ); +/// ``` +macro_rules! oparg_enum { + ( + $(#[$enum_meta:meta])* + $vis:vis enum $name:ident { + $( + $(#[$variant_meta:meta])* + $variant:ident = $value:literal $(| $alternatives:expr)* + ),* $(,)? + } + ) => { + $(#[$enum_meta])* + $vis enum $name { + $( + $(#[$variant_meta])* + $variant, // Do assign value to variant. + )* + } + + impl_oparg_enum!( + enum $name { + $( + $variant = $value $(| $alternatives)*, + )* + } + ); + }; +} + +macro_rules! impl_oparg_enum { + ( + enum $name:ident { + $( + $variant:ident = $value:literal $(| $alternatives:expr)* + ),* $(,)? + } + ) => { + impl TryFrom<u8> for $name { + type Error = $crate::marshal::MarshalError; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + Ok(match value { + $( + $value $(| $alternatives)* => Self::$variant, + )* + _ => return Err(Self::Error::InvalidBytecode), + }) + } + } + + impl TryFrom<u32> for $name { + type Error = $crate::marshal::MarshalError; + + fn try_from(value: u32) -> Result<Self, Self::Error> { + u8::try_from(value) + .map_err(|_| Self::Error::InvalidBytecode) + .map(TryInto::try_into)? + } + } + + impl From<$name> for u8 { + fn from(value: $name) -> Self { + match value { + $( + $name::$variant => $value, + )* + } + } + } + + impl From<$name> for u32 { + fn from(value: $name) -> Self { + Self::from(u8::from(value)) + } + } + + impl OpArgType for $name {} + }; +} + +oparg_enum!( + /// Oparg values for [`Instruction::ConvertValue`]. + /// + /// ## See also + /// + /// - [CPython FVC_* flags](https://github.com/python/cpython/blob/8183fa5e3f78ca6ab862de7fb8b14f3d929421e0/Include/ceval.h#L129-L132) + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] + pub enum ConvertValueOparg { + /// No conversion. + /// + /// ```python + /// f"{x}" + /// f"{x:4}" + /// ``` + // Ruff `ConversionFlag::None` is `-1i8`, when its converted to `u8` its value is `u8::MAX`. + None = 0 | 255, + /// Converts by calling `str(<value>)`. + /// + /// ```python + /// f"{x!s}" + /// f"{x!s:2}" + /// ``` + Str = 1, + /// Converts by calling `repr(<value>)`. + /// + /// ```python + /// f"{x!r}" + /// f"{x!r:2}" + /// ``` + Repr = 2, + /// Converts by calling `ascii(<value>)`. + /// + /// ```python + /// f"{x!a}" + /// f"{x!a:2}" + /// ``` + Ascii = 3, + } +); + +impl fmt::Display for ConvertValueOparg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let out = match self { + Self::Str => "1 (str)", + Self::Repr => "2 (repr)", + Self::Ascii => "3 (ascii)", + // We should never reach this. `FVC_NONE` are being handled by `Instruction::FormatSimple` + Self::None => "", + }; + + write!(f, "{out}") + } +} + +oparg_enum!( + /// Resume type for the RESUME instruction + #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] + pub enum ResumeType { + AtFuncStart = 0, + AfterYield = 1, + AfterYieldFrom = 2, + AfterAwait = 3, + } +); + +pub type NameIdx = u32; + +impl OpArgType for u32 {} + +oparg_enum!( + /// The kind of Raise that occurred. + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub enum RaiseKind { + /// Bare `raise` statement with no arguments. + /// Gets the current exception from VM state (topmost_exception). + /// Maps to RAISE_VARARGS with oparg=0. + BareRaise = 0, + /// `raise exc` - exception is on the stack. + /// Maps to RAISE_VARARGS with oparg=1. + Raise = 1, + /// `raise exc from cause` - exception and cause are on the stack. + /// Maps to RAISE_VARARGS with oparg=2. + RaiseCause = 2, + /// Reraise exception from the stack top. + /// Used in exception handler cleanup blocks (finally, except). + /// Gets exception from stack, not from VM state. + /// Maps to the RERAISE opcode. + ReraiseFromStack = 3, + } +); + +oparg_enum!( + /// Intrinsic function for CALL_INTRINSIC_1 + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub enum IntrinsicFunction1 { + // Invalid = 0, + Print = 1, + /// Import * operation + ImportStar = 2, + /// Convert StopIteration to RuntimeError in async context + StopIterationError = 3, + AsyncGenWrap = 4, + UnaryPositive = 5, + /// Convert list to tuple + ListToTuple = 6, + /// Type parameter related + TypeVar = 7, + ParamSpec = 8, + TypeVarTuple = 9, + /// Generic subscript for PEP 695 + SubscriptGeneric = 10, + TypeAlias = 11, + } +); + +oparg_enum!( + /// Intrinsic function for CALL_INTRINSIC_2 + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub enum IntrinsicFunction2 { + PrepReraiseStar = 1, + TypeVarWithBound = 2, + TypeVarWithConstraint = 3, + SetFunctionTypeParams = 4, + /// Set default value for type parameter (PEP 695) + SetTypeparamDefault = 5, + } +); + +bitflagset::bitflag! { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + #[repr(u8)] + pub enum MakeFunctionFlag { + Closure = 0, + Annotations = 1, + KwOnlyDefaults = 2, + Defaults = 3, + TypeParams = 4, + /// PEP 649: __annotate__ function closure (instead of __annotations__ dict) + Annotate = 5, + } +} + +bitflagset::bitflagset! { + #[derive(Copy, Clone, PartialEq, Eq)] + pub struct MakeFunctionFlags(u8): MakeFunctionFlag +} + +impl TryFrom<u32> for MakeFunctionFlag { + type Error = MarshalError; + + fn try_from(value: u32) -> Result<Self, Self::Error> { + Self::try_from(value as u8).map_err(|_| MarshalError::InvalidBytecode) + } +} + +impl From<MakeFunctionFlag> for u32 { + fn from(flag: MakeFunctionFlag) -> Self { + flag as u32 + } +} + +impl OpArgType for MakeFunctionFlag {} + +oparg_enum!( + /// The possible comparison operators. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum ComparisonOperator { + // be intentional with bits so that we can do eval_ord with just a bitwise and + // bits: | Equal | Greater | Less | + Less = 0b001, + Greater = 0b010, + NotEqual = 0b011, + Equal = 0b100, + LessOrEqual = 0b101, + GreaterOrEqual = 0b110, + } +); + +oparg_enum!( + /// The possible Binary operators + /// + /// # Examples + /// + /// ```rust + /// use rustpython_compiler_core::bytecode::{Arg, BinaryOperator, Instruction}; + /// let (op, _) = Arg::new(BinaryOperator::Add); + /// let instruction = Instruction::BinaryOp { op }; + /// ``` + /// + /// See also: + /// - [_PyEval_BinaryOps](https://github.com/python/cpython/blob/8183fa5e3f78ca6ab862de7fb8b14f3d929421e0/Python/ceval.c#L316-L343) + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub enum BinaryOperator { + /// `+` + Add = 0, + /// `&` + And = 1, + /// `//` + FloorDivide = 2, + /// `<<` + Lshift = 3, + /// `@` + MatrixMultiply = 4, + /// `*` + Multiply = 5, + /// `%` + Remainder = 6, + /// `|` + Or = 7, + /// `**` + Power = 8, + /// `>>` + Rshift = 9, + /// `-` + Subtract = 10, + /// `/` + TrueDivide = 11, + /// `^` + Xor = 12, + /// `+=` + InplaceAdd = 13, + /// `&=` + InplaceAnd = 14, + /// `//=` + InplaceFloorDivide = 15, + /// `<<=` + InplaceLshift = 16, + /// `@=` + InplaceMatrixMultiply = 17, + /// `*=` + InplaceMultiply = 18, + /// `%=` + InplaceRemainder = 19, + /// `|=` + InplaceOr = 20, + /// `**=` + InplacePower = 21, + /// `>>=` + InplaceRshift = 22, + /// `-=` + InplaceSubtract = 23, + /// `/=` + InplaceTrueDivide = 24, + /// `^=` + InplaceXor = 25, + /// `[]` subscript + Subscr = 26, + } +); + +impl BinaryOperator { + /// Get the "inplace" version of the operator. + /// This has no effect if `self` is already an "inplace" operator. + /// + /// # Example + /// ```rust + /// use rustpython_compiler_core::bytecode::BinaryOperator; + /// + /// assert_eq!(BinaryOperator::Power.as_inplace(), BinaryOperator::InplacePower); + /// + /// assert_eq!(BinaryOperator::InplaceSubtract.as_inplace(), BinaryOperator::InplaceSubtract); + /// ``` + #[must_use] + pub const fn as_inplace(self) -> Self { + match self { + Self::Add => Self::InplaceAdd, + Self::And => Self::InplaceAnd, + Self::FloorDivide => Self::InplaceFloorDivide, + Self::Lshift => Self::InplaceLshift, + Self::MatrixMultiply => Self::InplaceMatrixMultiply, + Self::Multiply => Self::InplaceMultiply, + Self::Remainder => Self::InplaceRemainder, + Self::Or => Self::InplaceOr, + Self::Power => Self::InplacePower, + Self::Rshift => Self::InplaceRshift, + Self::Subtract => Self::InplaceSubtract, + Self::TrueDivide => Self::InplaceTrueDivide, + Self::Xor => Self::InplaceXor, + _ => self, + } + } +} + +impl fmt::Display for BinaryOperator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let op = match self { + Self::Add => "+", + Self::And => "&", + Self::FloorDivide => "//", + Self::Lshift => "<<", + Self::MatrixMultiply => "@", + Self::Multiply => "*", + Self::Remainder => "%", + Self::Or => "|", + Self::Power => "**", + Self::Rshift => ">>", + Self::Subtract => "-", + Self::TrueDivide => "/", + Self::Xor => "^", + Self::InplaceAdd => "+=", + Self::InplaceAnd => "&=", + Self::InplaceFloorDivide => "//=", + Self::InplaceLshift => "<<=", + Self::InplaceMatrixMultiply => "@=", + Self::InplaceMultiply => "*=", + Self::InplaceRemainder => "%=", + Self::InplaceOr => "|=", + Self::InplacePower => "**=", + Self::InplaceRshift => ">>=", + Self::InplaceSubtract => "-=", + Self::InplaceTrueDivide => "/=", + Self::InplaceXor => "^=", + Self::Subscr => "[]", + }; + write!(f, "{op}") + } +} + +oparg_enum!( + /// Whether or not to invert the operation. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum Invert { + /// ```py + /// foo is bar + /// x in lst + /// ``` + No = 0, + /// ```py + /// foo is not bar + /// x not in lst + /// ``` + Yes = 1, + } +); + +oparg_enum!( + /// Special method for LOAD_SPECIAL opcode (context managers). + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum SpecialMethod { + /// `__enter__` for sync context manager + Enter = 0, + /// `__exit__` for sync context manager + Exit = 1, + /// `__aenter__` for async context manager + AEnter = 2, + /// `__aexit__` for async context manager + AExit = 3, + } +); + +impl fmt::Display for SpecialMethod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let method_name = match self { + Self::Enter => "__enter__", + Self::Exit => "__exit__", + Self::AEnter => "__aenter__", + Self::AExit => "__aexit__", + }; + write!(f, "{method_name}") + } +} + +oparg_enum!( + /// Common constants for LOAD_COMMON_CONSTANT opcode. + /// pycore_opcode_utils.h CONSTANT_* + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub enum CommonConstant { + /// `AssertionError` exception type + AssertionError = 0, + /// `NotImplementedError` exception type + NotImplementedError = 1, + /// Built-in `tuple` type + BuiltinTuple = 2, + /// Built-in `all` function + BuiltinAll = 3, + /// Built-in `any` function + BuiltinAny = 4, + } +); + +impl fmt::Display for CommonConstant { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + Self::AssertionError => "AssertionError", + Self::NotImplementedError => "NotImplementedError", + Self::BuiltinTuple => "tuple", + Self::BuiltinAll => "all", + Self::BuiltinAny => "any", + }; + write!(f, "{name}") + } +} + +oparg_enum!( + /// Specifies if a slice is built with either 2 or 3 arguments. + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub enum BuildSliceArgCount { + /// ```py + /// x[5:10] + /// ``` + Two = 2, + /// ```py + /// x[5:10:2] + /// ``` + Three = 3, + } +); + +#[derive(Copy, Clone)] +pub struct UnpackExArgs { + pub before: u8, + pub after: u8, +} + +impl From<u32> for UnpackExArgs { + fn from(value: u32) -> Self { + let [before, after, ..] = value.to_le_bytes(); + Self { before, after } + } +} + +impl From<UnpackExArgs> for u32 { + fn from(value: UnpackExArgs) -> Self { + Self::from_le_bytes([value.before, value.after, 0, 0]) + } +} + +impl OpArgType for UnpackExArgs {} + +impl fmt::Display for UnpackExArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "before: {}, after: {}", self.before, self.after) + } +} + +macro_rules! newtype_oparg { + ( + $(#[$oparg_meta:meta])* + $vis:vis struct $name:ident(u32) + ) => { + $(#[$oparg_meta])* + $vis struct $name(u32); + + impl $name { + /// Creates a new [`$name`] instance. + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value) + } + + /// Alias to [`$name::new`]. + #[must_use] + pub const fn from_u32(value: u32) -> Self { + Self::new(value) + } + + /// Returns the oparg as a `u32` value. + #[must_use] + pub const fn as_u32(self) -> u32 { + self.0 + } + + /// Returns the oparg as a `usize` value. + #[must_use] + pub const fn as_usize(self) -> usize { + self.0 as usize + } + } + + impl From<u32> for $name { + fn from(value: u32) -> Self { + Self::from_u32(value) + } + } + + impl From<$name> for u32 { + fn from(value: $name) -> Self { + value.as_u32() + } + } + + impl From<$name> for usize { + fn from(value: $name) -> Self { + value.as_usize() + } + } + + impl ::core::fmt::Display for $name { + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + self.0.fmt(f) + } + } + impl OpArgType for $name {} + } +} + +newtype_oparg!( + #[derive(Clone, Copy)] + #[repr(transparent)] + pub struct ConstIdx(u32) +); + +newtype_oparg!( + #[derive(Clone, Copy)] + #[repr(transparent)] + pub struct VarNum(u32) +); + +newtype_oparg!( + #[derive(Clone, Copy)] + #[repr(transparent)] + pub struct VarNums(u32) +); + +newtype_oparg!( + #[derive(Clone, Copy)] + #[repr(transparent)] + pub struct LoadAttr(u32) +); + +newtype_oparg!( + #[derive(Clone, Copy)] + #[repr(transparent)] + pub struct LoadSuperAttr(u32) +); + +newtype_oparg!( + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] + #[repr(transparent)] + pub struct Label(u32) +); + +impl VarNums { + #[must_use] + pub const fn idx_1(self) -> VarNum { + VarNum::new(self.0 >> 4) + } + + #[must_use] + pub const fn idx_2(self) -> VarNum { + VarNum::new(self.0 & 15) + } + + #[must_use] + pub const fn indexes(self) -> (VarNum, VarNum) { + (self.idx_1(), self.idx_2()) + } +} + +impl LoadAttr { + #[must_use] + pub fn builder() -> LoadAttrBuilder { + LoadAttrBuilder::default() + } + + #[must_use] + pub const fn name_idx(self) -> u32 { + self.0 >> 1 + } + + #[must_use] + pub const fn is_method(self) -> bool { + (self.0 & 1) == 1 + } +} + +#[derive(Clone, Copy, Default)] +pub struct LoadAttrBuilder { + name_idx: u32, + is_method: bool, +} + +impl LoadAttrBuilder { + #[must_use] + pub const fn build(self) -> LoadAttr { + let value = (self.name_idx << 1) | (self.is_method as u32); + LoadAttr::new(value) + } + + #[must_use] + pub const fn name_idx(mut self, value: u32) -> Self { + self.name_idx = value; + self + } + + #[must_use] + pub const fn is_method(mut self, value: bool) -> Self { + self.is_method = value; + self + } +} + +impl LoadSuperAttr { + #[must_use] + pub fn builder() -> LoadSuperAttrBuilder { + LoadSuperAttrBuilder::default() + } + + #[must_use] + pub const fn name_idx(self) -> u32 { + self.0 >> 2 + } + + #[must_use] + pub const fn is_load_method(self) -> bool { + (self.0 & 1) == 1 + } + + #[must_use] + pub const fn has_class(self) -> bool { + (self.0 & 2) == 2 + } +} + +#[derive(Clone, Copy, Default)] +pub struct LoadSuperAttrBuilder { + name_idx: u32, + is_load_method: bool, + has_class: bool, +} + +impl LoadSuperAttrBuilder { + #[must_use] + pub const fn build(self) -> LoadSuperAttr { + let value = + (self.name_idx << 2) | ((self.has_class as u32) << 1) | (self.is_load_method as u32); + LoadSuperAttr::new(value) + } + + #[must_use] + pub const fn name_idx(mut self, value: u32) -> Self { + self.name_idx = value; + self + } + + #[must_use] + pub const fn is_load_method(mut self, value: bool) -> Self { + self.is_load_method = value; + self + } + + #[must_use] + pub const fn has_class(mut self, value: bool) -> Self { + self.has_class = value; + self + } +} + +impl From<LoadSuperAttrBuilder> for LoadSuperAttr { + fn from(builder: LoadSuperAttrBuilder) -> Self { + builder.build() + } +} diff --git a/compiler/core/src/frozen.rs b/crates/compiler-core/src/frozen.rs similarity index 91% rename from compiler/core/src/frozen.rs rename to crates/compiler-core/src/frozen.rs index 4565eac8aae..81530610d0e 100644 --- a/compiler/core/src/frozen.rs +++ b/crates/compiler-core/src/frozen.rs @@ -1,5 +1,6 @@ use crate::bytecode::*; use crate::marshal::{self, Read, ReadBorrowed, Write}; +use alloc::vec::Vec; /// A frozen module. Holds a frozen code object and whether it is part of a package #[derive(Copy, Clone)] @@ -32,7 +33,7 @@ impl FrozenCodeObject<Vec<u8>> { let mut data = Vec::new(); marshal::serialize_code(&mut data, code); let bytes = lz4_flex::compress_prepend_size(&data); - FrozenCodeObject { bytes } + Self { bytes } } } @@ -42,8 +43,8 @@ pub struct FrozenLib<B: ?Sized = [u8]> { } impl<B: AsRef<[u8]> + ?Sized> FrozenLib<B> { - pub const fn from_ref(b: &B) -> &FrozenLib<B> { - unsafe { &*(b as *const B as *const FrozenLib<B>) } + pub const fn from_ref(b: &B) -> &Self { + unsafe { &*(b as *const B as *const Self) } } /// Decode a library to a iterable of frozen modules @@ -57,6 +58,7 @@ impl<B: AsRef<[u8]> + ?Sized> FrozenLib<B> { impl<'a, B: AsRef<[u8]> + ?Sized> IntoIterator for &'a FrozenLib<B> { type Item = (&'a str, FrozenModule<&'a [u8]>); type IntoIter = FrozenModulesIter<'a>; + fn into_iter(self) -> Self::IntoIter { self.decode() } @@ -84,6 +86,7 @@ impl<'a> Iterator for FrozenModulesIter<'a> { (self.remaining as usize, Some(self.remaining as usize)) } } + impl ExactSizeIterator for FrozenModulesIter<'_> {} fn read_entry<'a>( @@ -100,10 +103,9 @@ fn read_entry<'a>( impl FrozenLib<Vec<u8>> { /// Encode the given iterator of frozen modules into a compressed vector of bytes - pub fn encode<'a, I, B: AsRef<[u8]>>(lib: I) -> FrozenLib<Vec<u8>> + pub fn encode<'a, I, B: AsRef<[u8]>>(lib: I) -> Self where - I: IntoIterator<Item = (&'a str, FrozenModule<B>)>, - I::IntoIter: ExactSizeIterator + Clone, + I: IntoIterator<Item = (&'a str, FrozenModule<B>), IntoIter: ExactSizeIterator + Clone>, { let iter = lib.into_iter(); let mut bytes = Vec::new(); diff --git a/crates/compiler-core/src/lib.rs b/crates/compiler-core/src/lib.rs new file mode 100644 index 00000000000..245713d1a14 --- /dev/null +++ b/crates/compiler-core/src/lib.rs @@ -0,0 +1,17 @@ +#![no_std] +#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] +#![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")] + +extern crate alloc; + +pub mod bytecode; +pub mod frozen; +pub mod marshal; +mod mode; +pub mod varint; + +pub use mode::Mode; + +pub use ruff_source_file::{ + LineIndex, OneIndexed, PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation, +}; diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs new file mode 100644 index 00000000000..ba3cf7a35c3 --- /dev/null +++ b/crates/compiler-core/src/marshal.rs @@ -0,0 +1,716 @@ +use crate::{OneIndexed, SourceLocation, bytecode::*}; +use alloc::{boxed::Box, vec::Vec}; +use core::convert::Infallible; +use malachite_bigint::{BigInt, Sign}; +use num_complex::Complex64; +use rustpython_wtf8::Wtf8; + +pub const FORMAT_VERSION: u32 = 5; + +#[derive(Clone, Copy, Debug)] +pub enum MarshalError { + /// Unexpected End Of File + Eof, + /// Invalid Bytecode + InvalidBytecode, + /// Invalid utf8 in string + InvalidUtf8, + /// Invalid source location + InvalidLocation, + /// Bad type marker + BadType, +} + +impl core::fmt::Display for MarshalError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Eof => f.write_str("unexpected end of data"), + Self::InvalidBytecode => f.write_str("invalid bytecode"), + Self::InvalidUtf8 => f.write_str("invalid utf8"), + Self::InvalidLocation => f.write_str("invalid source location"), + Self::BadType => f.write_str("bad type marker"), + } + } +} + +impl From<core::str::Utf8Error> for MarshalError { + fn from(_: core::str::Utf8Error) -> Self { + Self::InvalidUtf8 + } +} + +impl core::error::Error for MarshalError {} + +type Result<T, E = MarshalError> = core::result::Result<T, E>; + +#[derive(Clone, Copy)] +#[repr(u8)] +enum Type { + // Null = b'0', + None = b'N', + False = b'F', + True = b'T', + StopIter = b'S', + Ellipsis = b'.', + Int = b'i', + Float = b'g', + Complex = b'y', + // Long = b'l', // i32 + Bytes = b's', // = TYPE_STRING + // Interned = b't', + // Ref = b'r', + Tuple = b'(', + List = b'[', + Dict = b'{', + Code = b'c', + Unicode = b'u', + // Unknown = b'?', + Set = b'<', + FrozenSet = b'>', + Slice = b':', // Added in version 5 + Ascii = b'a', + // AsciiInterned = b'A', + // SmallTuple = b')', + // ShortAscii = b'z', + // ShortAsciiInterned = b'Z', +} +// const FLAG_REF: u8 = b'\x80'; + +impl TryFrom<u8> for Type { + type Error = MarshalError; + + fn try_from(value: u8) -> Result<Self> { + use Type::*; + + Ok(match value { + // b'0' => Null, + b'N' => None, + b'F' => False, + b'T' => True, + b'S' => StopIter, + b'.' => Ellipsis, + b'i' => Int, + b'g' => Float, + b'y' => Complex, + // b'l' => Long, + b's' => Bytes, + // b't' => Interned, + // b'r' => Ref, + b'(' => Tuple, + b'[' => List, + b'{' => Dict, + b'c' => Code, + b'u' => Unicode, + // b'?' => Unknown, + b'<' => Set, + b'>' => FrozenSet, + b':' => Slice, + b'a' => Ascii, + // b'A' => AsciiInterned, + // b')' => SmallTuple, + // b'z' => ShortAscii, + // b'Z' => ShortAsciiInterned, + _ => return Err(MarshalError::BadType), + }) + } +} + +pub trait Read { + fn read_slice(&mut self, n: u32) -> Result<&[u8]>; + + fn read_array<const N: usize>(&mut self) -> Result<&[u8; N]> { + self.read_slice(N as u32).map(|s| s.try_into().unwrap()) + } + + fn read_str(&mut self, len: u32) -> Result<&str> { + Ok(core::str::from_utf8(self.read_slice(len)?)?) + } + + fn read_wtf8(&mut self, len: u32) -> Result<&Wtf8> { + Wtf8::from_bytes(self.read_slice(len)?).ok_or(MarshalError::InvalidUtf8) + } + + fn read_u8(&mut self) -> Result<u8> { + Ok(u8::from_le_bytes(*self.read_array()?)) + } + + fn read_u16(&mut self) -> Result<u16> { + Ok(u16::from_le_bytes(*self.read_array()?)) + } + + fn read_u32(&mut self) -> Result<u32> { + Ok(u32::from_le_bytes(*self.read_array()?)) + } + + fn read_u64(&mut self) -> Result<u64> { + Ok(u64::from_le_bytes(*self.read_array()?)) + } +} + +pub(crate) trait ReadBorrowed<'a>: Read { + fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]>; + + fn read_str_borrow(&mut self, len: u32) -> Result<&'a str> { + Ok(core::str::from_utf8(self.read_slice_borrow(len)?)?) + } +} + +impl Read for &[u8] { + fn read_slice(&mut self, n: u32) -> Result<&[u8]> { + self.read_slice_borrow(n) + } + + fn read_array<const N: usize>(&mut self) -> Result<&[u8; N]> { + let (chunk, rest) = self.split_first_chunk::<N>().ok_or(MarshalError::Eof)?; + *self = rest; + Ok(chunk) + } +} + +impl<'a> ReadBorrowed<'a> for &'a [u8] { + fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]> { + self.split_off(..n as usize).ok_or(MarshalError::Eof) + } +} + +pub struct Cursor<B> { + pub data: B, + pub position: usize, +} + +impl<B: AsRef<[u8]>> Read for Cursor<B> { + fn read_slice(&mut self, n: u32) -> Result<&[u8]> { + let data = &self.data.as_ref()[self.position..]; + let slice = data.get(..n as usize).ok_or(MarshalError::Eof)?; + self.position += n as usize; + Ok(slice) + } +} + +pub fn deserialize_code<R: Read, Bag: ConstantBag>( + rdr: &mut R, + bag: Bag, +) -> Result<CodeObject<Bag::Constant>> { + let len = rdr.read_u32()?; + let raw_instructions = rdr.read_slice(len * 2)?; + let instructions = CodeUnits::try_from(raw_instructions)?; + + let len = rdr.read_u32()?; + let locations = (0..len) + .map(|_| { + let start = SourceLocation { + line: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, + character_offset: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), + }; + let end = SourceLocation { + line: OneIndexed::new(rdr.read_u32()? as _).ok_or(MarshalError::InvalidLocation)?, + character_offset: OneIndexed::from_zero_indexed(rdr.read_u32()? as _), + }; + Ok((start, end)) + }) + .collect::<Result<Box<[(SourceLocation, SourceLocation)]>>>()?; + + let flags = CodeFlags::from_bits_truncate(rdr.read_u32()?); + + let posonlyarg_count = rdr.read_u32()?; + let arg_count = rdr.read_u32()?; + let kwonlyarg_count = rdr.read_u32()?; + + let len = rdr.read_u32()?; + let source_path = bag.make_name(rdr.read_str(len)?); + + let first_line_number = OneIndexed::new(rdr.read_u32()? as _); + let max_stackdepth = rdr.read_u32()?; + + let len = rdr.read_u32()?; + let obj_name = bag.make_name(rdr.read_str(len)?); + + let len = rdr.read_u32()?; + let qualname = bag.make_name(rdr.read_str(len)?); + + let len = rdr.read_u32()?; + let cell2arg = (len != 0) + .then(|| { + (0..len) + .map(|_| Ok(rdr.read_u32()? as i32)) + .collect::<Result<Box<[i32]>>>() + }) + .transpose()?; + + let len = rdr.read_u32()?; + let constants = (0..len) + .map(|_| deserialize_value(rdr, bag)) + .collect::<Result<_>>()?; + + let mut read_names = || { + let len = rdr.read_u32()?; + (0..len) + .map(|_| { + let len = rdr.read_u32()?; + Ok(bag.make_name(rdr.read_str(len)?)) + }) + .collect::<Result<Box<[_]>>>() + }; + + let names = read_names()?; + let varnames = read_names()?; + let cellvars = read_names()?; + let freevars = read_names()?; + + // Read linetable and exceptiontable + let linetable_len = rdr.read_u32()?; + let linetable = rdr.read_slice(linetable_len)?.to_vec().into_boxed_slice(); + + let exceptiontable_len = rdr.read_u32()?; + let exceptiontable = rdr + .read_slice(exceptiontable_len)? + .to_vec() + .into_boxed_slice(); + + Ok(CodeObject { + instructions, + locations, + flags, + posonlyarg_count, + arg_count, + kwonlyarg_count, + source_path, + first_line_number, + max_stackdepth, + obj_name, + qualname, + cell2arg, + constants, + names, + varnames, + cellvars, + freevars, + linetable, + exceptiontable, + }) +} + +pub trait MarshalBag: Copy { + type Value; + type ConstantBag: ConstantBag; + + fn make_bool(&self, value: bool) -> Self::Value; + + fn make_none(&self) -> Self::Value; + + fn make_ellipsis(&self) -> Self::Value; + + fn make_float(&self, value: f64) -> Self::Value; + + fn make_complex(&self, value: Complex64) -> Self::Value; + + fn make_str(&self, value: &Wtf8) -> Self::Value; + + fn make_bytes(&self, value: &[u8]) -> Self::Value; + + fn make_int(&self, value: BigInt) -> Self::Value; + + fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value; + + fn make_code( + &self, + code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>, + ) -> Self::Value; + + fn make_stop_iter(&self) -> Result<Self::Value>; + + fn make_list(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>; + + fn make_set(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>; + + fn make_frozenset(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>; + + fn make_dict( + &self, + it: impl Iterator<Item = (Self::Value, Self::Value)>, + ) -> Result<Self::Value>; + + fn constant_bag(self) -> Self::ConstantBag; +} + +impl<Bag: ConstantBag> MarshalBag for Bag { + type Value = Bag::Constant; + type ConstantBag = Self; + + fn make_bool(&self, value: bool) -> Self::Value { + self.make_constant::<Bag::Constant>(BorrowedConstant::Boolean { value }) + } + + fn make_none(&self) -> Self::Value { + self.make_constant::<Bag::Constant>(BorrowedConstant::None) + } + + fn make_ellipsis(&self) -> Self::Value { + self.make_constant::<Bag::Constant>(BorrowedConstant::Ellipsis) + } + + fn make_float(&self, value: f64) -> Self::Value { + self.make_constant::<Bag::Constant>(BorrowedConstant::Float { value }) + } + + fn make_complex(&self, value: Complex64) -> Self::Value { + self.make_constant::<Bag::Constant>(BorrowedConstant::Complex { value }) + } + + fn make_str(&self, value: &Wtf8) -> Self::Value { + self.make_constant::<Bag::Constant>(BorrowedConstant::Str { value }) + } + + fn make_bytes(&self, value: &[u8]) -> Self::Value { + self.make_constant::<Bag::Constant>(BorrowedConstant::Bytes { value }) + } + + fn make_int(&self, value: BigInt) -> Self::Value { + self.make_int(value) + } + + fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value { + self.make_tuple(elements) + } + + fn make_code( + &self, + code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>, + ) -> Self::Value { + self.make_code(code) + } + + fn make_stop_iter(&self) -> Result<Self::Value> { + Err(MarshalError::BadType) + } + + fn make_list(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> { + Err(MarshalError::BadType) + } + + fn make_set(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> { + Err(MarshalError::BadType) + } + + fn make_frozenset(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> { + Err(MarshalError::BadType) + } + + fn make_dict( + &self, + _: impl Iterator<Item = (Self::Value, Self::Value)>, + ) -> Result<Self::Value> { + Err(MarshalError::BadType) + } + + fn constant_bag(self) -> Self::ConstantBag { + self + } +} + +pub fn deserialize_value<R: Read, Bag: MarshalBag>(rdr: &mut R, bag: Bag) -> Result<Bag::Value> { + let typ = Type::try_from(rdr.read_u8()?)?; + let value = match typ { + Type::True => bag.make_bool(true), + Type::False => bag.make_bool(false), + Type::None => bag.make_none(), + Type::StopIter => bag.make_stop_iter()?, + Type::Ellipsis => bag.make_ellipsis(), + Type::Int => { + let len = rdr.read_u32()? as i32; + let sign = if len < 0 { Sign::Minus } else { Sign::Plus }; + let bytes = rdr.read_slice(len.unsigned_abs())?; + let int = BigInt::from_bytes_le(sign, bytes); + bag.make_int(int) + } + Type::Float => { + let value = f64::from_bits(rdr.read_u64()?); + bag.make_float(value) + } + Type::Complex => { + let re = f64::from_bits(rdr.read_u64()?); + let im = f64::from_bits(rdr.read_u64()?); + let value = Complex64 { re, im }; + bag.make_complex(value) + } + Type::Ascii | Type::Unicode => { + let len = rdr.read_u32()?; + let value = rdr.read_wtf8(len)?; + bag.make_str(value) + } + Type::Tuple => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| deserialize_value(rdr, bag)); + itertools::process_results(it, |it| bag.make_tuple(it))? + } + Type::List => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| deserialize_value(rdr, bag)); + itertools::process_results(it, |it| bag.make_list(it))?? + } + Type::Set => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| deserialize_value(rdr, bag)); + itertools::process_results(it, |it| bag.make_set(it))?? + } + Type::FrozenSet => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| deserialize_value(rdr, bag)); + itertools::process_results(it, |it| bag.make_frozenset(it))?? + } + Type::Dict => { + let len = rdr.read_u32()?; + let it = (0..len).map(|_| { + let k = deserialize_value(rdr, bag)?; + let v = deserialize_value(rdr, bag)?; + Ok::<_, MarshalError>((k, v)) + }); + itertools::process_results(it, |it| bag.make_dict(it))?? + } + Type::Bytes => { + // Following CPython, after marshaling, byte arrays are converted into bytes. + let len = rdr.read_u32()?; + let value = rdr.read_slice(len)?; + bag.make_bytes(value) + } + Type::Code => bag.make_code(deserialize_code(rdr, bag.constant_bag())?), + Type::Slice => { + // Slice constants are not yet supported in RustPython + // This would require adding a Slice variant to ConstantData enum + // For now, return an error if we encounter a slice in marshal data + return Err(MarshalError::BadType); + } + }; + Ok(value) +} + +pub trait Dumpable: Sized { + type Error; + type Constant: Constant; + + fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error>; +} + +pub enum DumpableValue<'a, D: Dumpable> { + Integer(&'a BigInt), + Float(f64), + Complex(Complex64), + Boolean(bool), + Str(&'a Wtf8), + Bytes(&'a [u8]), + Code(&'a CodeObject<D::Constant>), + Tuple(&'a [D]), + None, + Ellipsis, + StopIter, + List(&'a [D]), + Set(&'a [D]), + Frozenset(&'a [D]), + Dict(&'a [(D, D)]), +} + +impl<'a, C: Constant> From<BorrowedConstant<'a, C>> for DumpableValue<'a, C> { + fn from(c: BorrowedConstant<'a, C>) -> Self { + match c { + BorrowedConstant::Integer { value } => Self::Integer(value), + BorrowedConstant::Float { value } => Self::Float(value), + BorrowedConstant::Complex { value } => Self::Complex(value), + BorrowedConstant::Boolean { value } => Self::Boolean(value), + BorrowedConstant::Str { value } => Self::Str(value), + BorrowedConstant::Bytes { value } => Self::Bytes(value), + BorrowedConstant::Code { code } => Self::Code(code), + BorrowedConstant::Tuple { elements } => Self::Tuple(elements), + BorrowedConstant::None => Self::None, + BorrowedConstant::Ellipsis => Self::Ellipsis, + } + } +} + +impl<C: Constant> Dumpable for C { + type Error = Infallible; + type Constant = Self; + + #[inline(always)] + fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error> { + Ok(f(self.borrow_constant().into())) + } +} + +pub trait Write { + fn write_slice(&mut self, slice: &[u8]); + + fn write_u8(&mut self, v: u8) { + self.write_slice(&v.to_le_bytes()) + } + + fn write_u16(&mut self, v: u16) { + self.write_slice(&v.to_le_bytes()) + } + + fn write_u32(&mut self, v: u32) { + self.write_slice(&v.to_le_bytes()) + } + + fn write_u64(&mut self, v: u64) { + self.write_slice(&v.to_le_bytes()) + } +} + +impl Write for Vec<u8> { + fn write_slice(&mut self, slice: &[u8]) { + self.extend_from_slice(slice) + } +} + +pub(crate) fn write_len<W: Write>(buf: &mut W, len: usize) { + let Ok(len) = len.try_into() else { + panic!("too long to serialize") + }; + buf.write_u32(len); +} + +pub(crate) fn write_vec<W: Write>(buf: &mut W, slice: &[u8]) { + write_len(buf, slice.len()); + buf.write_slice(slice); +} + +pub fn serialize_value<W: Write, D: Dumpable>( + buf: &mut W, + constant: DumpableValue<'_, D>, +) -> Result<(), D::Error> { + match constant { + DumpableValue::Integer(int) => { + buf.write_u8(Type::Int as u8); + let (sign, bytes) = int.to_bytes_le(); + let len: i32 = bytes.len().try_into().expect("too long to serialize"); + let len = if sign == Sign::Minus { -len } else { len }; + buf.write_u32(len as u32); + buf.write_slice(&bytes); + } + DumpableValue::Float(f) => { + buf.write_u8(Type::Float as u8); + buf.write_u64(f.to_bits()); + } + DumpableValue::Complex(c) => { + buf.write_u8(Type::Complex as u8); + buf.write_u64(c.re.to_bits()); + buf.write_u64(c.im.to_bits()); + } + DumpableValue::Boolean(b) => { + buf.write_u8(if b { Type::True } else { Type::False } as u8); + } + DumpableValue::Str(s) => { + buf.write_u8(Type::Unicode as u8); + write_vec(buf, s.as_bytes()); + } + DumpableValue::Bytes(b) => { + buf.write_u8(Type::Bytes as u8); + write_vec(buf, b); + } + DumpableValue::Code(c) => { + buf.write_u8(Type::Code as u8); + serialize_code(buf, c); + } + DumpableValue::Tuple(tup) => { + buf.write_u8(Type::Tuple as u8); + write_len(buf, tup.len()); + for val in tup { + val.with_dump(|val| serialize_value(buf, val))?? + } + } + DumpableValue::None => { + buf.write_u8(Type::None as u8); + } + DumpableValue::Ellipsis => { + buf.write_u8(Type::Ellipsis as u8); + } + DumpableValue::StopIter => { + buf.write_u8(Type::StopIter as u8); + } + DumpableValue::List(l) => { + buf.write_u8(Type::List as u8); + write_len(buf, l.len()); + for val in l { + val.with_dump(|val| serialize_value(buf, val))?? + } + } + DumpableValue::Set(set) => { + buf.write_u8(Type::Set as u8); + write_len(buf, set.len()); + for val in set { + val.with_dump(|val| serialize_value(buf, val))?? + } + } + DumpableValue::Frozenset(set) => { + buf.write_u8(Type::FrozenSet as u8); + write_len(buf, set.len()); + for val in set { + val.with_dump(|val| serialize_value(buf, val))?? + } + } + DumpableValue::Dict(d) => { + buf.write_u8(Type::Dict as u8); + write_len(buf, d.len()); + for (k, v) in d { + k.with_dump(|val| serialize_value(buf, val))??; + v.with_dump(|val| serialize_value(buf, val))??; + } + } + } + Ok(()) +} + +pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) { + write_len(buf, code.instructions.len()); + let original = code.instructions.original_bytes(); + buf.write_slice(&original); + + write_len(buf, code.locations.len()); + for (start, end) in &*code.locations { + buf.write_u32(start.line.get() as _); + buf.write_u32(start.character_offset.to_zero_indexed() as _); + buf.write_u32(end.line.get() as _); + buf.write_u32(end.character_offset.to_zero_indexed() as _); + } + + buf.write_u32(code.flags.bits()); + + buf.write_u32(code.posonlyarg_count); + buf.write_u32(code.arg_count); + buf.write_u32(code.kwonlyarg_count); + + write_vec(buf, code.source_path.as_ref().as_bytes()); + + buf.write_u32(code.first_line_number.map_or(0, |x| x.get() as _)); + buf.write_u32(code.max_stackdepth); + + write_vec(buf, code.obj_name.as_ref().as_bytes()); + write_vec(buf, code.qualname.as_ref().as_bytes()); + + let cell2arg = code.cell2arg.as_deref().unwrap_or(&[]); + write_len(buf, cell2arg.len()); + for &i in cell2arg { + buf.write_u32(i as u32) + } + + write_len(buf, code.constants.len()); + for constant in &*code.constants { + serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {}) + } + + let mut write_names = |names: &[C::Name]| { + write_len(buf, names.len()); + for name in names { + write_vec(buf, name.as_ref().as_bytes()); + } + }; + + write_names(&code.names); + write_names(&code.varnames); + write_names(&code.cellvars); + write_names(&code.freevars); + + // Serialize linetable and exceptiontable + write_vec(buf, &code.linetable); + write_vec(buf, &code.exceptiontable); +} diff --git a/crates/compiler-core/src/mode.rs b/crates/compiler-core/src/mode.rs new file mode 100644 index 00000000000..181ea4fdfe7 --- /dev/null +++ b/crates/compiler-core/src/mode.rs @@ -0,0 +1,32 @@ +#[derive(Clone, Copy)] +pub enum Mode { + Exec, + Eval, + Single, + /// Returns the value of the last statement in the statement list. + BlockExpr, +} + +impl core::str::FromStr for Mode { + type Err = ModeParseError; + + // To support `builtins.compile()` `mode` argument + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "exec" => Ok(Self::Exec), + "eval" => Ok(Self::Eval), + "single" => Ok(Self::Single), + _ => Err(ModeParseError), + } + } +} + +/// Returned when a given mode is not valid. +#[derive(Clone, Copy, Debug)] +pub struct ModeParseError; + +impl core::fmt::Display for ModeParseError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, r#"mode must be "exec", "eval", or "single""#) + } +} diff --git a/crates/compiler-core/src/varint.rs b/crates/compiler-core/src/varint.rs new file mode 100644 index 00000000000..f1ea6b17ec0 --- /dev/null +++ b/crates/compiler-core/src/varint.rs @@ -0,0 +1,140 @@ +//! Variable-length integer encoding utilities. +//! +//! Uses 6-bit chunks with a continuation bit (0x40) to encode integers. +//! Used for exception tables and line number tables. + +use alloc::vec::Vec; + +/// Write a variable-length unsigned integer using 6-bit chunks. +/// Returns the number of bytes written. +#[inline] +pub fn write_varint(buf: &mut Vec<u8>, mut val: u32) -> usize { + let start_len = buf.len(); + while val >= 64 { + buf.push(0x40 | (val & 0x3f) as u8); + val >>= 6; + } + buf.push(val as u8); + buf.len() - start_len +} + +/// Write a variable-length signed integer. +/// Returns the number of bytes written. +#[inline] +pub fn write_signed_varint(buf: &mut Vec<u8>, val: i32) -> usize { + let uval = if val < 0 { + // (0 - val as u32) handles INT_MIN correctly + ((0u32.wrapping_sub(val as u32)) << 1) | 1 + } else { + (val as u32) << 1 + }; + write_varint(buf, uval) +} + +/// Write a variable-length unsigned integer with a start marker (0x80 bit). +/// Used for exception table entries where each entry starts with the marker. +pub fn write_varint_with_start(data: &mut Vec<u8>, val: u32) { + let start_pos = data.len(); + write_varint(data, val); + // Set start bit on first byte + if let Some(first) = data.get_mut(start_pos) { + *first |= 0x80; + } +} + +/// Read a variable-length unsigned integer that starts with a start marker (0x80 bit). +/// Returns None if not at a valid start byte or end of data. +pub fn read_varint_with_start(data: &[u8], pos: &mut usize) -> Option<u32> { + if *pos >= data.len() { + return None; + } + let first = data[*pos]; + if first & 0x80 == 0 { + return None; // Not a start byte + } + *pos += 1; + + let mut val = (first & 0x3f) as u32; + let mut shift = 6; + let mut has_continuation = first & 0x40 != 0; + + while has_continuation && *pos < data.len() { + let byte = data[*pos]; + if byte & 0x80 != 0 { + break; // Next entry start + } + *pos += 1; + val |= ((byte & 0x3f) as u32) << shift; + shift += 6; + has_continuation = byte & 0x40 != 0; + } + Some(val) +} + +/// Read a variable-length unsigned integer. +/// Returns None if end of data or malformed. +pub fn read_varint(data: &[u8], pos: &mut usize) -> Option<u32> { + if *pos >= data.len() { + return None; + } + + let mut val = 0u32; + let mut shift = 0; + + loop { + if *pos >= data.len() { + return None; + } + let byte = data[*pos]; + if byte & 0x80 != 0 && shift > 0 { + break; // Next entry start + } + *pos += 1; + val |= ((byte & 0x3f) as u32) << shift; + shift += 6; + if byte & 0x40 == 0 { + break; + } + } + Some(val) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_write_read_varint() { + let mut buf = Vec::new(); + write_varint(&mut buf, 0); + write_varint(&mut buf, 63); + write_varint(&mut buf, 64); + write_varint(&mut buf, 4095); + + // Values: 0, 63, 64, 4095 + assert_eq!(buf.len(), 1 + 1 + 2 + 2); + } + + #[test] + fn test_write_read_signed_varint() { + let mut buf = Vec::new(); + write_signed_varint(&mut buf, 0); + write_signed_varint(&mut buf, 1); + write_signed_varint(&mut buf, -1); + write_signed_varint(&mut buf, i32::MIN); + + assert!(!buf.is_empty()); + } + + #[test] + fn test_varint_with_start() { + let mut buf = Vec::new(); + write_varint_with_start(&mut buf, 42); + write_varint_with_start(&mut buf, 100); + + let mut pos = 0; + assert_eq!(read_varint_with_start(&buf, &mut pos), Some(42)); + assert_eq!(read_varint_with_start(&buf, &mut pos), Some(100)); + assert_eq!(read_varint_with_start(&buf, &mut pos), None); + } +} diff --git a/crates/compiler-source/Cargo.toml b/crates/compiler-source/Cargo.toml new file mode 100644 index 00000000000..068d31e87ed --- /dev/null +++ b/crates/compiler-source/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "rustpython-compiler-source" +description = "(DEPRECATED) RustPython Source and Index" +version = "0.5.0+deprecated" +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +ruff_source_file = { workspace = true } +ruff_text_size = { workspace = true } + +[lints] +workspace = true diff --git a/crates/compiler-source/src/lib.rs b/crates/compiler-source/src/lib.rs new file mode 100644 index 00000000000..e3fb55ef603 --- /dev/null +++ b/crates/compiler-source/src/lib.rs @@ -0,0 +1,43 @@ +pub use ruff_source_file::{LineIndex, OneIndexed as LineNumber, PositionEncoding, SourceLocation}; +use ruff_text_size::TextRange; +pub use ruff_text_size::TextSize; + +#[derive(Clone)] +pub struct SourceCode<'src> { + pub path: &'src str, + pub text: &'src str, + pub index: LineIndex, +} + +impl<'src> SourceCode<'src> { + pub fn new(path: &'src str, text: &'src str) -> Self { + let index = LineIndex::from_source_text(text); + Self { path, text, index } + } + + pub fn line_index(&self, offset: TextSize) -> LineNumber { + self.index.line_index(offset) + } + + pub fn source_location(&self, offset: TextSize) -> SourceLocation { + self.index + .source_location(offset, self.text, PositionEncoding::Utf8) + } + + pub fn get_range(&'src self, range: TextRange) -> &'src str { + &self.text[range.start().to_usize()..range.end().to_usize()] + } +} + +pub struct SourceCodeOwned { + pub path: String, + pub text: String, + pub index: LineIndex, +} + +impl SourceCodeOwned { + pub fn new(path: String, text: String) -> Self { + let index = LineIndex::from_source_text(&text); + Self { path, text, index } + } +} diff --git a/crates/compiler/Cargo.toml b/crates/compiler/Cargo.toml new file mode 100644 index 00000000000..5c793241110 --- /dev/null +++ b/crates/compiler/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "rustpython-compiler" +description = "A usability wrapper around rustpython-parser and rustpython-compiler-core" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +rustpython-codegen = { workspace = true } +rustpython-compiler-core = { workspace = true } +ruff_python_parser = { workspace = true } +ruff_python_ast = { workspace = true } +ruff_source_file = { workspace = true } +ruff_text_size = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] + +[lints] +workspace = true diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs new file mode 100644 index 00000000000..815d45f2cd8 --- /dev/null +++ b/crates/compiler/src/lib.rs @@ -0,0 +1,476 @@ +use ruff_source_file::{PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation}; +use rustpython_codegen::{compile, symboltable}; + +pub use rustpython_codegen::compile::CompileOpts; +pub use rustpython_compiler_core::{Mode, bytecode::CodeObject}; + +// these modules are out of repository. re-exporting them here for convenience. +pub use ruff_python_ast as ast; +pub use ruff_python_parser as parser; +pub use rustpython_codegen as codegen; +pub use rustpython_compiler_core as core; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CompileErrorType { + #[error(transparent)] + Codegen(#[from] codegen::error::CodegenErrorType), + #[error(transparent)] + Parse(#[from] parser::ParseErrorType), +} + +#[derive(Error, Debug)] +pub struct ParseError { + #[source] + pub error: parser::ParseErrorType, + pub raw_location: ruff_text_size::TextRange, + pub location: SourceLocation, + pub end_location: SourceLocation, + pub source_path: String, + /// Set when the error is an unclosed bracket (converted from EOF). + pub is_unclosed_bracket: bool, +} + +impl ::core::fmt::Display for ParseError { + fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + self.error.fmt(f) + } +} + +#[derive(Error, Debug)] +pub enum CompileError { + #[error(transparent)] + Codegen(#[from] codegen::error::CodegenError), + #[error(transparent)] + Parse(#[from] ParseError), +} + +impl CompileError { + pub fn from_ruff_parse_error(error: parser::ParseError, source_file: &SourceFile) -> Self { + let source_code = source_file.to_source_code(); + let source_text = source_file.source_text(); + + // For EOF errors (unclosed brackets), find the unclosed bracket position + // and adjust both the error location and message + let mut is_unclosed_bracket = false; + let (error_type, location, end_location) = if matches!( + &error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::Eof) + ) { + if let Some((bracket_char, bracket_offset)) = find_unclosed_bracket(source_text) { + let bracket_text_size = ruff_text_size::TextSize::new(bracket_offset as u32); + let loc = source_code.source_location(bracket_text_size, PositionEncoding::Utf8); + let end_loc = SourceLocation { + line: loc.line, + character_offset: loc.character_offset.saturating_add(1), + }; + let msg = format!("'{}' was never closed", bracket_char); + is_unclosed_bracket = true; + (parser::ParseErrorType::OtherError(msg), loc, end_loc) + } else { + let loc = + source_code.source_location(error.location.start(), PositionEncoding::Utf8); + let end_loc = + source_code.source_location(error.location.end(), PositionEncoding::Utf8); + (error.error, loc, end_loc) + } + } else if matches!( + &error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::IndentationError) + ) { + // For IndentationError, point the offset to the end of the line content + // instead of the beginning + let loc = source_code.source_location(error.location.start(), PositionEncoding::Utf8); + let line_idx = loc.line.to_zero_indexed(); + let line = source_text.split('\n').nth(line_idx).unwrap_or(""); + let line_end_col = line.chars().count() + 1; // 1-indexed, past last char + let end_loc = SourceLocation { + line: loc.line, + character_offset: ruff_source_file::OneIndexed::new(line_end_col) + .unwrap_or(loc.character_offset), + }; + (error.error, end_loc, end_loc) + } else { + let loc = source_code.source_location(error.location.start(), PositionEncoding::Utf8); + let mut end_loc = + source_code.source_location(error.location.end(), PositionEncoding::Utf8); + + // If the error range ends at the start of a new line (column 1), + // adjust it to the end of the previous line + if end_loc.character_offset.get() == 1 && end_loc.line > loc.line { + let prev_line_end = error.location.end() - ruff_text_size::TextSize::from(1); + end_loc = source_code.source_location(prev_line_end, PositionEncoding::Utf8); + end_loc.character_offset = end_loc.character_offset.saturating_add(1); + } + + (error.error, loc, end_loc) + }; + + Self::Parse(ParseError { + error: error_type, + raw_location: error.location, + location, + end_location, + source_path: source_file.name().to_owned(), + is_unclosed_bracket, + }) + } + + pub const fn location(&self) -> Option<SourceLocation> { + match self { + Self::Codegen(codegen_error) => codegen_error.location, + Self::Parse(parse_error) => Some(parse_error.location), + } + } + + pub const fn python_location(&self) -> (usize, usize) { + if let Some(location) = self.location() { + (location.line.get(), location.character_offset.get()) + } else { + (0, 0) + } + } + + pub fn python_end_location(&self) -> Option<(usize, usize)> { + match self { + CompileError::Codegen(_) => None, + CompileError::Parse(parse_error) => Some(( + parse_error.end_location.line.get(), + parse_error.end_location.character_offset.get(), + )), + } + } + + pub fn source_path(&self) -> &str { + match self { + Self::Codegen(codegen_error) => &codegen_error.source_path, + Self::Parse(parse_error) => &parse_error.source_path, + } + } +} + +/// Find the last unclosed opening bracket in source code. +/// Returns the bracket character and its byte offset, or None if all brackets are balanced. +fn find_unclosed_bracket(source: &str) -> Option<(char, usize)> { + let mut stack: Vec<(char, usize)> = Vec::new(); + let mut in_string = false; + let mut string_quote = '\0'; + let mut triple_quote = false; + let mut escape_next = false; + let mut is_raw_string = false; + + let chars: Vec<(usize, char)> = source.char_indices().collect(); + let mut i = 0; + + while i < chars.len() { + let (byte_offset, ch) = chars[i]; + + if escape_next { + escape_next = false; + i += 1; + continue; + } + + if in_string { + if ch == '\\' && !is_raw_string { + escape_next = true; + } else if triple_quote { + if ch == string_quote + && i + 2 < chars.len() + && chars[i + 1].1 == string_quote + && chars[i + 2].1 == string_quote + { + in_string = false; + i += 3; + continue; + } + } else if ch == string_quote { + in_string = false; + } + i += 1; + continue; + } + + // Check for comments + if ch == '#' { + // Skip to end of line + while i < chars.len() && chars[i].1 != '\n' { + i += 1; + } + continue; + } + + // Check for string start (with optional prefix like r, b, f, u, rb, br, etc.) + if ch == '\'' || ch == '"' { + // Check up to 2 characters before the quote for string prefix + is_raw_string = false; + for look_back in 1..=2.min(i) { + let prev = chars[i - look_back].1; + if matches!(prev, 'r' | 'R') { + is_raw_string = true; + break; + } + if !matches!(prev, 'b' | 'B' | 'f' | 'F' | 'u' | 'U') { + break; + } + } + string_quote = ch; + if i + 2 < chars.len() && chars[i + 1].1 == ch && chars[i + 2].1 == ch { + triple_quote = true; + in_string = true; + i += 3; + continue; + } + triple_quote = false; + in_string = true; + i += 1; + continue; + } + + match ch { + '(' | '[' | '{' => stack.push((ch, byte_offset)), + ')' | ']' | '}' => { + let expected = match ch { + ')' => '(', + ']' => '[', + '}' => '{', + _ => unreachable!(), + }; + if stack.last().is_some_and(|&(open, _)| open == expected) { + stack.pop(); + } + } + _ => {} + } + + i += 1; + } + + stack.last().copied() +} + +/// Compile a given source code into a bytecode object. +pub fn compile( + source: &str, + mode: Mode, + source_path: &str, + opts: CompileOpts, +) -> Result<CodeObject, CompileError> { + // TODO: do this less hacky; ruff's parser should translate a CRLF line + // break in a multiline string into just an LF in the parsed value + #[cfg(windows)] + let source = source.replace("\r\n", "\n"); + #[cfg(windows)] + let source = source.as_str(); + + let source_file = SourceFileBuilder::new(source_path, source).finish(); + _compile(source_file, mode, opts) + // let index = LineIndex::from_source_text(source); + // let source_code = SourceCode::new(source, &index); + // let mut locator = LinearLocator::new(source); + // let mut ast = match parser::parse(source, mode.into(), &source_path) { + // Ok(x) => x, + // Err(e) => return Err(locator.locate_error(e)), + // }; + + // TODO: + // if opts.optimize > 0 { + // ast = ConstantOptimizer::new() + // .fold_mod(ast) + // .unwrap_or_else(|e| match e {}); + // } + // let ast = locator.fold_mod(ast).unwrap_or_else(|e| match e {}); +} + +fn _compile( + source_file: SourceFile, + mode: Mode, + opts: CompileOpts, +) -> Result<CodeObject, CompileError> { + let parser_mode = match mode { + Mode::Exec => parser::Mode::Module, + Mode::Eval => parser::Mode::Expression, + // ruff does not have an interactive mode, which is fine, + // since these are only different in terms of compilation + Mode::Single | Mode::BlockExpr => parser::Mode::Module, + }; + let parsed = parser::parse(source_file.source_text(), parser_mode.into()) + .map_err(|err| CompileError::from_ruff_parse_error(err, &source_file))?; + let ast = parsed.into_syntax(); + compile::compile_top(ast, source_file, mode, opts).map_err(|e| e.into()) +} + +pub fn compile_symtable( + source: &str, + mode: Mode, + source_path: &str, +) -> Result<symboltable::SymbolTable, CompileError> { + let source_file = SourceFileBuilder::new(source_path, source).finish(); + _compile_symtable(source_file, mode) +} + +pub fn _compile_symtable( + source_file: SourceFile, + mode: Mode, +) -> Result<symboltable::SymbolTable, CompileError> { + let res = match mode { + Mode::Exec | Mode::Single | Mode::BlockExpr => { + let ast = ruff_python_parser::parse_module(source_file.source_text()) + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?; + symboltable::SymbolTable::scan_program(&ast.into_syntax(), source_file.clone()) + } + Mode::Eval => { + let ast = ruff_python_parser::parse( + source_file.source_text(), + parser::Mode::Expression.into(), + ) + .map_err(|e| CompileError::from_ruff_parse_error(e, &source_file))?; + symboltable::SymbolTable::scan_expr( + &ast.into_syntax().expect_expression(), + source_file.clone(), + ) + } + }; + res.map_err(|e| e.into_codegen_error(source_file.name().to_owned()).into()) +} + +#[test] +fn test_compile() { + let code = "x = 'abc'"; + let compiled = compile(code, Mode::Single, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_phello() { + let code = r#" +initialized = True +def main(): + print("Hello world!") +if __name__ == '__main__': + main() +"#; + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_if_elif_else() { + let code = r#" +if False: + pass +elif False: + pass +elif False: + pass +else: + pass +"#; + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_lambda() { + let code = r#" +lambda: 'a' +"#; + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_lambda2() { + let code = r#" +(lambda x: f'hello, {x}')('world}') +"#; + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_lambda3() { + let code = r#" +def g(): + pass +def f(): + if False: + return lambda x: g(x) + elif False: + return g + else: + return g +"#; + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_int() { + let code = r#" +a = 0xFF +"#; + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_bigint() { + let code = r#" +a = 0xFFFFFFFFFFFFFFFFFFFFFFFF +"#; + let compiled = compile(code, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_compile_fstring() { + let code1 = r#" +assert f"1" == '1' + "#; + let compiled = compile(code1, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + + let code2 = r#" +assert f"{1}" == '1' + "#; + let compiled = compile(code2, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + let code3 = r#" +assert f"{1+1}" == '2' + "#; + let compiled = compile(code3, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + + let code4 = r#" +assert f"{{{(lambda: f'{1}')}" == '{1' + "#; + let compiled = compile(code4, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + + let code5 = r#" +assert f"a{1}" == 'a1' + "#; + let compiled = compile(code5, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); + + let code6 = r#" +assert f"{{{(lambda x: f'hello, {x}')('world}')}" == '{hello, world}' + "#; + let compiled = compile(code6, Mode::Exec, "<>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} + +#[test] +fn test_simple_enum() { + let code = r#" +import enum +@enum._simple_enum(enum.IntFlag, boundary=enum.KEEP) +class RegexFlag: + NOFLAG = 0 + DEBUG = 1 +print(RegexFlag.NOFLAG & RegexFlag.DEBUG) +"#; + let compiled = compile(code, Mode::Exec, "<string>", CompileOpts::default()); + dbg!(compiled.expect("compile error")); +} diff --git a/crates/derive-impl/Cargo.toml b/crates/derive-impl/Cargo.toml new file mode 100644 index 00000000000..a772cab1d38 --- /dev/null +++ b/crates/derive-impl/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "rustpython-derive-impl" +description = "Rust language extensions and macros specific to rustpython." +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +rustpython-compiler-core = { workspace = true } +# rustpython-parser-core = { workspace = true } +rustpython-doc = { workspace = true } + +itertools = { workspace = true } +syn = { workspace = true, features = ["full", "extra-traits"] } + +maplit = "1.0.2" +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn-ext = { version = "0.5.0", features = ["full"] } +textwrap = { version = "0.16.1", default-features = false } + +[lints] +workspace = true diff --git a/crates/derive-impl/src/compile_bytecode.rs b/crates/derive-impl/src/compile_bytecode.rs new file mode 100644 index 00000000000..16984139fcd --- /dev/null +++ b/crates/derive-impl/src/compile_bytecode.rs @@ -0,0 +1,382 @@ +//! Parsing and processing for this form: +//! ```ignore +//! py_compile!( +//! // either: +//! source = "python_source_code", +//! // or +//! file = "file/path/relative/to/this/file", +//! +//! // the mode to compile the code in +//! mode = "exec", // or "eval" or "single" +//! // the path put into the CodeObject, defaults to "frozen" +//! module_name = "frozen", +//! ) +//! ``` + +use crate::Diagnostic; +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use rustpython_compiler_core::{Mode, bytecode::CodeObject, frozen}; +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, +}; +use syn::{ + self, LitByteStr, LitStr, Macro, + parse::{ParseStream, Parser, Result as ParseResult}, + spanned::Spanned, +}; + +enum CompilationSourceKind { + /// Source is a File (Path) + File { base: PathBuf, rel_path: PathBuf }, + /// Direct Raw source code + SourceCode(String), + /// Source is a directory + Dir { base: PathBuf, rel_path: PathBuf }, +} + +struct CompiledModule { + code: CodeObject, + package: bool, +} + +struct CompilationSource { + kind: CompilationSourceKind, + span: (Span, Span), +} + +pub trait Compiler { + fn compile( + &self, + source: &str, + mode: Mode, + module_name: String, + ) -> Result<CodeObject, Box<dyn core::error::Error>>; +} + +impl CompilationSource { + fn compile_string<D: core::fmt::Display, F: FnOnce() -> D>( + &self, + source: &str, + mode: Mode, + module_name: String, + compiler: &dyn Compiler, + origin: F, + ) -> Result<CodeObject, Diagnostic> { + compiler.compile(source, mode, module_name).map_err(|err| { + Diagnostic::spans_error( + self.span, + format!("Python compile error from {}: {}", origin(), err), + ) + }) + } + + fn compile( + &self, + mode: Mode, + module_name: String, + compiler: &dyn Compiler, + ) -> Result<HashMap<String, CompiledModule>, Diagnostic> { + match &self.kind { + CompilationSourceKind::Dir { base, rel_path } => { + self.compile_dir(base, &base.join(rel_path), String::new(), mode, compiler) + } + _ => Ok(hashmap! { + module_name.clone() => CompiledModule { + code: self.compile_single(mode, module_name, compiler)?, + package: false, + }, + }), + } + } + + fn compile_single( + &self, + mode: Mode, + module_name: String, + compiler: &dyn Compiler, + ) -> Result<CodeObject, Diagnostic> { + match &self.kind { + CompilationSourceKind::File { base, rel_path } => { + let path = base.join(rel_path); + let source = fs::read_to_string(&path).map_err(|err| { + Diagnostic::spans_error( + self.span, + format!("Error reading file {path:?}: {err}"), + ) + })?; + self.compile_string(&source, mode, module_name, compiler, || rel_path.display()) + } + CompilationSourceKind::SourceCode(code) => self.compile_string( + &textwrap::dedent(code), + mode, + module_name, + compiler, + || "string literal", + ), + CompilationSourceKind::Dir { .. } => { + unreachable!("Can't use compile_single with directory source") + } + } + } + + fn compile_dir( + &self, + base: &Path, + path: &Path, + parent: String, + mode: Mode, + compiler: &dyn Compiler, + ) -> Result<HashMap<String, CompiledModule>, Diagnostic> { + let mut code_map = HashMap::new(); + let paths = fs::read_dir(path) + .or_else(|e| { + if cfg!(windows) + && let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) + { + return fs::read_dir(real_path.trim()); + } + Err(e) + }) + .map_err(|err| { + Diagnostic::spans_error(self.span, format!("Error listing dir {path:?}: {err}")) + })?; + for path in paths { + let path = path.map_err(|err| { + Diagnostic::spans_error(self.span, format!("Failed to list file: {err}")) + })?; + let path = path.path(); + let file_name = path.file_name().unwrap().to_str().ok_or_else(|| { + Diagnostic::spans_error(self.span, format!("Invalid UTF-8 in file name {path:?}")) + })?; + if path.is_dir() { + code_map.extend(self.compile_dir( + base, + &path, + if parent.is_empty() { + file_name.to_string() + } else { + format!("{parent}.{file_name}") + }, + mode, + compiler, + )?); + } else if file_name.ends_with(".py") { + let stem = path.file_stem().unwrap().to_str().unwrap(); + let is_init = stem == "__init__"; + let module_name = if is_init { + parent.clone() + } else if parent.is_empty() { + stem.to_owned() + } else { + format!("{parent}.{stem}") + }; + + let compile_path = |src_path: &Path| { + let source = fs::read_to_string(src_path).map_err(|err| { + Diagnostic::spans_error( + self.span, + format!("Error reading file {path:?}: {err}"), + ) + })?; + self.compile_string(&source, mode, module_name.clone(), compiler, || { + path.strip_prefix(base).ok().unwrap_or(&path).display() + }) + }; + let code = compile_path(&path).or_else(|e| { + if cfg!(windows) + && let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) + { + let joined = path.parent().unwrap().join(real_path.trim()); + if joined.exists() { + return compile_path(&joined); + } else { + return Err(e); + } + } + Err(e) + }); + + let code = match code { + Ok(code) => code, + Err(_) + if stem.starts_with("badsyntax_") + | parent.ends_with(".encoded_modules") => + { + // TODO: handle with macro arg rather than hard-coded path + continue; + } + Err(e) => return Err(e), + }; + + code_map.insert( + module_name, + CompiledModule { + code, + package: is_init, + }, + ); + } + } + Ok(code_map) + } +} + +impl PyCompileArgs { + fn parse(input: TokenStream, allow_dir: bool) -> Result<Self, Diagnostic> { + let mut module_name = None; + let mut mode = None; + let mut source: Option<CompilationSource> = None; + let mut crate_name = None; + + fn assert_source_empty(source: &Option<CompilationSource>) -> Result<(), syn::Error> { + if let Some(source) = source { + Err(syn::Error::new( + source.span.0, + "Cannot have more than one source", + )) + } else { + Ok(()) + } + } + + syn::meta::parser(|meta| { + let ident = meta + .path + .get_ident() + .ok_or_else(|| meta.error("unknown arg"))?; + let check_str = || meta.value()?.call(parse_str); + let str_path = || { + let s = check_str()?; + let mut base_path = s + .span() + .unwrap() + .local_file() + .ok_or_else(|| err_span!(s, "filepath literal has no span information"))?; + base_path.pop(); + Ok::<_, syn::Error>((base_path, PathBuf::from(s.value()))) + }; + if ident == "mode" { + let s = check_str()?; + match s.value().parse() { + Ok(mode_val) => mode = Some(mode_val), + Err(e) => bail_span!(s, "{}", e), + } + } else if ident == "module_name" { + module_name = Some(check_str()?.value()) + } else if ident == "source" { + assert_source_empty(&source)?; + let code = check_str()?.value(); + source = Some(CompilationSource { + kind: CompilationSourceKind::SourceCode(code), + span: (ident.span(), meta.input.cursor().span()), + }); + } else if ident == "file" { + assert_source_empty(&source)?; + let (base, rel_path) = str_path()?; + source = Some(CompilationSource { + kind: CompilationSourceKind::File { base, rel_path }, + span: (ident.span(), meta.input.cursor().span()), + }); + } else if ident == "dir" { + if !allow_dir { + bail_span!(ident, "py_compile doesn't accept dir") + } + + assert_source_empty(&source)?; + let (base, rel_path) = str_path()?; + source = Some(CompilationSource { + kind: CompilationSourceKind::Dir { base, rel_path }, + span: (ident.span(), meta.input.cursor().span()), + }); + } else if ident == "crate_name" { + let name = check_str()?.parse()?; + crate_name = Some(name); + } else { + return Err(meta.error("unknown attr")); + } + Ok(()) + }) + .parse2(input)?; + + let source = source.ok_or_else(|| { + syn::Error::new( + Span::call_site(), + "Must have either file or source in py_compile!()/py_freeze!()", + ) + })?; + + Ok(Self { + source, + mode: mode.unwrap_or(Mode::Exec), + module_name: module_name.unwrap_or_else(|| "frozen".to_owned()), + crate_name: crate_name.unwrap_or_else(|| syn::parse_quote!(::rustpython_vm)), + }) + } +} + +fn parse_str(input: ParseStream<'_>) -> ParseResult<LitStr> { + let span = input.span(); + if input.peek(LitStr) { + input.parse() + } else if let Ok(mac) = input.parse::<Macro>() { + Ok(LitStr::new(&mac.tokens.to_string(), mac.span())) + } else { + Err(syn::Error::new(span, "Expected string or stringify macro")) + } +} + +struct PyCompileArgs { + source: CompilationSource, + mode: Mode, + module_name: String, + crate_name: syn::Path, +} + +pub fn impl_py_compile( + input: TokenStream, + compiler: &dyn Compiler, +) -> Result<TokenStream, Diagnostic> { + let args = PyCompileArgs::parse(input, false)?; + + let crate_name = args.crate_name; + let code = args + .source + .compile_single(args.mode, args.module_name, compiler)?; + + let frozen = frozen::FrozenCodeObject::encode(&code); + let bytes = LitByteStr::new(&frozen.bytes, Span::call_site()); + + let output = quote! { + #crate_name::frozen::FrozenCodeObject { bytes: &#bytes[..] } + }; + + Ok(output) +} + +pub fn impl_py_freeze( + input: TokenStream, + compiler: &dyn Compiler, +) -> Result<TokenStream, Diagnostic> { + let args = PyCompileArgs::parse(input, true)?; + + let crate_name = args.crate_name; + let code_map = args.source.compile(args.mode, args.module_name, compiler)?; + + let data = frozen::FrozenLib::encode(code_map.iter().map(|(k, v)| { + let v = frozen::FrozenModule { + code: frozen::FrozenCodeObject::encode(&v.code), + package: v.package, + }; + (&**k, v) + })); + let bytes = LitByteStr::new(&data.bytes, Span::call_site()); + + let output = quote! { + #crate_name::frozen::FrozenLib::from_ref(#bytes) + }; + + Ok(output) +} diff --git a/crates/derive-impl/src/error.rs b/crates/derive-impl/src/error.rs new file mode 100644 index 00000000000..2fdf7a4b4a3 --- /dev/null +++ b/crates/derive-impl/src/error.rs @@ -0,0 +1,156 @@ +// Taken from https://github.com/rustwasm/wasm-bindgen/blob/master/crates/backend/src/error.rs +// +// Copyright (c) 2014 Alex Crichton +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +#![allow(dead_code)] + +use proc_macro2::*; +use quote::{ToTokens, TokenStreamExt}; +use syn::parse::Error; + +macro_rules! err_span { + ($span:expr, $($msg:tt)*) => ( + syn::Error::new_spanned(&$span, format_args!($($msg)*)) + ) +} + +macro_rules! bail_span { + ($($t:tt)*) => ( + return Err(err_span!($($t)*).into()) + ) +} + +// macro_rules! push_err_span { +// ($diagnostics:expr, $($t:tt)*) => { +// $diagnostics.push(err_span!($($t)*)) +// }; +// } + +// macro_rules! push_diag_result { +// ($diags:expr, $x:expr $(,)?) => { +// if let Err(e) = $x { +// $diags.push(e); +// } +// }; +// } + +#[derive(Debug)] +pub struct Diagnostic { + inner: Repr, +} + +#[derive(Debug)] +enum Repr { + Single { + text: String, + span: Option<(Span, Span)>, + }, + SynError(Error), + Multi { + diagnostics: Vec<Diagnostic>, + }, +} + +impl Diagnostic { + pub fn error<T: Into<String>>(text: T) -> Self { + Self { + inner: Repr::Single { + text: text.into(), + span: None, + }, + } + } + + pub(crate) fn spans_error<T: Into<String>>(spans: (Span, Span), text: T) -> Self { + Self { + inner: Repr::Single { + text: text.into(), + span: Some(spans), + }, + } + } + + pub fn from_vec(diagnostics: Vec<Self>) -> Result<(), Self> { + if diagnostics.is_empty() { + Ok(()) + } else { + Err(Self { + inner: Repr::Multi { diagnostics }, + }) + } + } + + pub fn panic(&self) -> ! { + match &self.inner { + Repr::Single { text, .. } => panic!("{}", text), + Repr::SynError(error) => panic!("{}", error), + Repr::Multi { diagnostics } => diagnostics[0].panic(), + } + } +} + +impl From<Error> for Diagnostic { + fn from(err: Error) -> Self { + Self { + inner: Repr::SynError(err), + } + } +} + +pub fn extract_spans(node: &dyn ToTokens) -> Option<(Span, Span)> { + let mut t = TokenStream::new(); + node.to_tokens(&mut t); + let mut tokens = t.into_iter(); + let start = tokens.next().map(|t| t.span()); + let end = tokens.last().map(|t| t.span()); + start.map(|start| (start, end.unwrap_or(start))) +} + +impl ToTokens for Diagnostic { + fn to_tokens(&self, dst: &mut TokenStream) { + match &self.inner { + Repr::Single { text, span } => { + let cs2 = (Span::call_site(), Span::call_site()); + let (start, end) = span.unwrap_or(cs2); + dst.append(Ident::new("compile_error", start)); + dst.append(Punct::new('!', Spacing::Alone)); + let mut message = TokenStream::new(); + message.append(Literal::string(text)); + let mut group = Group::new(Delimiter::Brace, message); + group.set_span(end); + dst.append(group); + } + Repr::Multi { diagnostics } => { + for diagnostic in diagnostics { + diagnostic.to_tokens(dst); + } + } + Repr::SynError(err) => { + err.to_compile_error().to_tokens(dst); + } + } + } +} diff --git a/crates/derive-impl/src/from_args.rs b/crates/derive-impl/src/from_args.rs new file mode 100644 index 00000000000..9f2d0460fb0 --- /dev/null +++ b/crates/derive-impl/src/from_args.rs @@ -0,0 +1,258 @@ +use proc_macro2::TokenStream; +use quote::{ToTokens, quote}; +use syn::ext::IdentExt; +use syn::meta::ParseNestedMeta; +use syn::{Attribute, Data, DeriveInput, Expr, Field, Ident, Result, Token, parse_quote}; + +/// The kind of the python parameter, this corresponds to the value of Parameter.kind +/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) +#[derive(Default)] +enum ParameterKind { + PositionalOnly, + #[default] + PositionalOrKeyword, + KeywordOnly, + Flatten, +} + +impl TryFrom<&Ident> for ParameterKind { + type Error = (); + + fn try_from(ident: &Ident) -> core::result::Result<Self, Self::Error> { + Ok(match ident.to_string().as_str() { + "positional" => Self::PositionalOnly, + "any" => Self::PositionalOrKeyword, + "named" => Self::KeywordOnly, + "flatten" => Self::Flatten, + _ => return Err(()), + }) + } +} + +// None == quote!(Default::default()) +type DefaultValue = Option<Expr>; + +#[derive(Default)] +struct ArgAttribute { + name: Option<String>, + kind: ParameterKind, + default: Option<DefaultValue>, + error_msg: Option<String>, +} + +impl ArgAttribute { + fn from_attribute(attr: &Attribute) -> Option<Result<Self>> { + if !attr.path().is_ident("pyarg") { + return None; + } + + let inner = move || { + let mut arg_attr = None; + attr.parse_nested_meta(|meta| { + let Some(arg_attr) = &mut arg_attr else { + let kind = meta + .path + .get_ident() + .and_then(|ident| ParameterKind::try_from(ident).ok()) + .ok_or_else(|| { + meta.error( + "The first argument to #[pyarg()] must be the parameter type, \ + either 'positional', 'any', 'named', or 'flatten'.", + ) + })?; + arg_attr = Some(Self { + name: None, + kind, + default: None, + error_msg: None, + }); + return Ok(()); + }; + arg_attr.parse_argument(meta) + })?; + arg_attr + .ok_or_else(|| err_span!(attr, "There must be at least one argument to #[pyarg()]")) + }; + Some(inner()) + } + + fn parse_argument(&mut self, meta: ParseNestedMeta<'_>) -> Result<()> { + if let ParameterKind::Flatten = self.kind { + return Err(meta.error("can't put additional arguments on a flatten arg")); + } + if meta.path.is_ident("default") && meta.input.peek(Token![=]) { + if matches!(self.default, Some(Some(_))) { + return Err(meta.error("Default already set")); + } + let val = meta.value()?; + self.default = Some(Some(val.parse()?)) + } else if meta.path.is_ident("default") || meta.path.is_ident("optional") { + if self.default.is_none() { + self.default = Some(None); + } + } else if meta.path.is_ident("name") { + if self.name.is_some() { + return Err(meta.error("already have a name")); + } + let val = meta.value()?.parse::<syn::LitStr>()?; + self.name = Some(val.value()) + } else if meta.path.is_ident("error_msg") { + if self.error_msg.is_some() { + return Err(meta.error("already have an error_msg")); + } + let val = meta.value()?.parse::<syn::LitStr>()?; + self.error_msg = Some(val.value()) + } else { + return Err(meta.error("Unrecognized pyarg attribute")); + } + + Ok(()) + } +} + +impl TryFrom<&Field> for ArgAttribute { + type Error = syn::Error; + + fn try_from(field: &Field) -> core::result::Result<Self, Self::Error> { + let mut pyarg_attrs = field + .attrs + .iter() + .filter_map(Self::from_attribute) + .collect::<core::result::Result<Vec<_>, _>>()?; + + if pyarg_attrs.len() >= 2 { + bail_span!(field, "Multiple pyarg attributes on field") + }; + + Ok(pyarg_attrs.pop().unwrap_or_default()) + } +} + +fn generate_field((i, field): (usize, &Field)) -> Result<TokenStream> { + let attr = ArgAttribute::try_from(field)?; + let name = field.ident.as_ref(); + let name_string = name.map(|ident| ident.unraw().to_string()); + if matches!(&name_string, Some(s) if s.starts_with("_phantom")) { + return Ok(quote! { + #name: ::std::marker::PhantomData, + }); + } + + let field_name = match name { + Some(id) => id.to_token_stream(), + None => syn::Index::from(i).into_token_stream(), + }; + + if let ParameterKind::Flatten = attr.kind { + return Ok(quote! { + #field_name: ::rustpython_vm::function::FromArgs::from_args(vm, args)?, + }); + } + + let pyname = attr + .name + .or(name_string) + .ok_or_else(|| err_span!(field, "field in tuple struct must have name attribute"))?; + + let middle = if let Some(error_msg) = &attr.error_msg { + quote! { + .map(|x| ::rustpython_vm::convert::TryFromObject::try_from_object(vm, x) + .map_err(|_| vm.new_type_error(#error_msg))).transpose()? + } + } else { + quote! { + .map(|x| ::rustpython_vm::convert::TryFromObject::try_from_object(vm, x)).transpose()? + } + }; + + let ending = if let Some(default) = attr.default { + let ty = &field.ty; + let default = default.unwrap_or_else(|| parse_quote!(::std::default::Default::default())); + quote! { + .map(<#ty as ::rustpython_vm::function::FromArgOptional>::from_inner) + .unwrap_or_else(|| #default) + } + } else { + let err = match attr.kind { + ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { + ::rustpython_vm::function::ArgumentError::TooFewArgs + }, + ParameterKind::KeywordOnly => quote! { + ::rustpython_vm::function::ArgumentError::RequiredKeywordArgument(#pyname.to_owned()) + }, + ParameterKind::Flatten => unreachable!(), + }; + quote! { + .ok_or_else(|| #err)? + } + }; + + let file_output = match attr.kind { + ParameterKind::PositionalOnly => quote! { + #field_name: args.take_positional()#middle #ending, + }, + ParameterKind::PositionalOrKeyword => quote! { + #field_name: args.take_positional_keyword(#pyname)#middle #ending, + }, + ParameterKind::KeywordOnly => quote! { + #field_name: args.take_keyword(#pyname)#middle #ending, + }, + ParameterKind::Flatten => unreachable!(), + }; + + Ok(file_output) +} + +fn compute_arity_bounds(field_attrs: &[ArgAttribute]) -> (usize, usize) { + let positional_fields = field_attrs.iter().filter(|attr| { + matches!( + attr.kind, + ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword + ) + }); + + let min_arity = positional_fields + .clone() + .filter(|attr| attr.default.is_none()) + .count(); + let max_arity = positional_fields.count(); + + (min_arity, max_arity) +} + +pub fn impl_from_args(input: DeriveInput) -> Result<TokenStream> { + let (fields, field_attrs) = match input.data { + Data::Struct(syn::DataStruct { fields, .. }) => ( + fields + .iter() + .enumerate() + .map(generate_field) + .collect::<Result<TokenStream>>()?, + fields + .iter() + .filter_map(|field| field.try_into().ok()) + .collect::<Vec<ArgAttribute>>(), + ), + _ => bail_span!(input, "FromArgs input must be a struct"), + }; + + let (min_arity, max_arity) = compute_arity_bounds(&field_attrs); + + let name = input.ident; + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + let output = quote! { + impl #impl_generics ::rustpython_vm::function::FromArgs for #name #ty_generics #where_clause { + fn arity() -> ::std::ops::RangeInclusive<usize> { + #min_arity..=#max_arity + } + + fn from_args( + vm: &::rustpython_vm::VirtualMachine, + args: &mut ::rustpython_vm::function::FuncArgs + ) -> ::core::result::Result<Self, ::rustpython_vm::function::ArgumentError> { + Ok(Self { #fields }) + } + } + }; + Ok(output) +} diff --git a/crates/derive-impl/src/lib.rs b/crates/derive-impl/src/lib.rs new file mode 100644 index 00000000000..c00299794de --- /dev/null +++ b/crates/derive-impl/src/lib.rs @@ -0,0 +1,85 @@ +#![recursion_limit = "128"] +#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] +#![doc(html_root_url = "https://docs.rs/rustpython-derive/")] + +extern crate proc_macro; + +#[macro_use] +extern crate maplit; + +#[macro_use] +mod error; +#[macro_use] +mod util; + +mod compile_bytecode; +mod from_args; +mod pyclass; +mod pymodule; +mod pypayload; +mod pystructseq; +mod pytraverse; + +use error::Diagnostic; +use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::{DeriveInput, Item}; +use syn_ext::types::PunctuatedNestedMeta; + +pub use pymodule::PyModuleArgs; + +pub use compile_bytecode::Compiler; + +fn result_to_tokens(result: Result<TokenStream, impl Into<Diagnostic>>) -> TokenStream { + result + .map_err(|e| e.into()) + .unwrap_or_else(ToTokens::into_token_stream) +} + +pub fn derive_from_args(input: DeriveInput) -> TokenStream { + result_to_tokens(from_args::impl_from_args(input)) +} + +pub fn pyclass(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { + if matches!(item, syn::Item::Impl(_) | syn::Item::Trait(_)) { + result_to_tokens(pyclass::impl_pyclass_impl(attr, item)) + } else { + result_to_tokens(pyclass::impl_pyclass(attr, item)) + } +} + +pub fn pyexception(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { + if matches!(item, syn::Item::Impl(_)) { + result_to_tokens(pyclass::impl_pyexception_impl(attr, item)) + } else { + result_to_tokens(pyclass::impl_pyexception(attr, item)) + } +} + +pub fn pymodule(attr: PyModuleArgs, item: Item) -> TokenStream { + result_to_tokens(pymodule::impl_pymodule(attr, item)) +} + +pub fn pystruct_sequence(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { + result_to_tokens(pystructseq::impl_pystruct_sequence(attr, item)) +} + +pub fn pystruct_sequence_data(attr: PunctuatedNestedMeta, item: Item) -> TokenStream { + result_to_tokens(pystructseq::impl_pystruct_sequence_data(attr, item)) +} + +pub fn py_compile(input: TokenStream, compiler: &dyn Compiler) -> TokenStream { + result_to_tokens(compile_bytecode::impl_py_compile(input, compiler)) +} + +pub fn py_freeze(input: TokenStream, compiler: &dyn Compiler) -> TokenStream { + result_to_tokens(compile_bytecode::impl_py_freeze(input, compiler)) +} + +pub fn pypayload(input: DeriveInput) -> TokenStream { + result_to_tokens(pypayload::impl_pypayload(input)) +} + +pub fn pytraverse(item: DeriveInput) -> TokenStream { + result_to_tokens(pytraverse::impl_pytraverse(item)) +} diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs new file mode 100644 index 00000000000..d4590c6c4d3 --- /dev/null +++ b/crates/derive-impl/src/pyclass.rs @@ -0,0 +1,1988 @@ +use super::Diagnostic; +use crate::util::{ + ALL_ALLOWED_NAMES, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ExceptionItemMeta, + ItemMeta, ItemMetaInner, ItemNursery, SimpleItemMeta, format_doc, infer_native_call_flags, + pyclass_ident_and_attrs, pyexception_ident_and_attrs, text_signature, +}; +use core::str::FromStr; +use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree}; +use quote::{ToTokens, quote, quote_spanned}; +use rustpython_doc::DB; +use std::collections::{HashMap, HashSet}; +use syn::{Attribute, Ident, Item, Result, parse_quote, spanned::Spanned}; +use syn_ext::ext::*; +use syn_ext::types::*; + +#[derive(Copy, Clone, Debug)] +enum AttrName { + Method, + ClassMethod, + StaticMethod, + GetSet, + Slot, + Attr, + ExtendClass, + Member, +} + +impl core::fmt::Display for AttrName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let s = match self { + Self::Method => "pymethod", + Self::ClassMethod => "pyclassmethod", + Self::StaticMethod => "pystaticmethod", + Self::GetSet => "pygetset", + Self::Slot => "pyslot", + Self::Attr => "pyattr", + Self::ExtendClass => "extend_class", + Self::Member => "pymember", + }; + s.fmt(f) + } +} + +impl FromStr for AttrName { + type Err = String; + + fn from_str(s: &str) -> core::result::Result<Self, Self::Err> { + Ok(match s { + "pymethod" => Self::Method, + "pyclassmethod" => Self::ClassMethod, + "pystaticmethod" => Self::StaticMethod, + "pygetset" => Self::GetSet, + "pyslot" => Self::Slot, + "pyattr" => Self::Attr, + "extend_class" => Self::ExtendClass, + "pymember" => Self::Member, + s => { + return Err(s.to_owned()); + } + }) + } +} + +#[derive(Default)] +struct ImplContext { + is_trait: bool, + attribute_items: ItemNursery, + method_items: MethodNursery, + getset_items: GetSetNursery, + member_items: MemberNursery, + extend_slots_items: ItemNursery, + class_extensions: Vec<TokenStream>, + errors: Vec<syn::Error>, +} + +fn extract_items_into_context<'a, Item>( + context: &mut ImplContext, + items: impl Iterator<Item = &'a mut Item>, +) where + Item: ItemLike + ToTokens + GetIdent + syn_ext::ext::ItemAttrExt + 'a, +{ + for item in items { + let r = item.try_split_attr_mut(|attrs, item| { + let (py_items, cfgs) = attrs_to_content_items(attrs, impl_item_new::<Item>)?; + for py_item in py_items.iter().rev() { + let r = py_item.gen_impl_item(ImplItemArgs::<Item> { + item, + attrs, + context, + cfgs: cfgs.as_slice(), + }); + context.errors.ok_or_push(r); + } + Ok(()) + }); + context.errors.ok_or_push(r); + } + context.errors.ok_or_push(context.method_items.validate()); + context.errors.ok_or_push(context.getset_items.validate()); + context.errors.ok_or_push(context.member_items.validate()); +} + +pub(crate) fn impl_pyclass_impl(attr: PunctuatedNestedMeta, item: Item) -> Result<TokenStream> { + let mut context = ImplContext::default(); + let mut tokens = match item { + Item::Impl(mut imp) => { + extract_items_into_context(&mut context, imp.items.iter_mut()); + + let (impl_ty, payload_guess) = match imp.self_ty.as_ref() { + syn::Type::Path(syn::TypePath { + path: syn::Path { segments, .. }, + .. + }) if segments.len() == 1 => { + let segment = &segments[0]; + let payload_ty = if segment.ident == "Py" || segment.ident == "PyRef" { + match &segment.arguments { + syn::PathArguments::AngleBracketed( + syn::AngleBracketedGenericArguments { args, .. }, + ) if args.len() == 1 => { + let arg = &args[0]; + match arg { + syn::GenericArgument::Type(syn::Type::Path( + syn::TypePath { + path: syn::Path { segments, .. }, + .. + }, + )) if segments.len() == 1 => segments[0].ident.clone(), + _ => { + return Err(syn::Error::new_spanned( + segment, + "Py{Ref}<T> is expected but Py{Ref}<?> is found", + )); + } + } + } + _ => { + return Err(syn::Error::new_spanned( + segment, + "Py{Ref}<T> is expected but Py{Ref}? is found", + )); + } + } + } else { + if !matches!(segment.arguments, syn::PathArguments::None) { + return Err(syn::Error::new_spanned( + segment, + "PyImpl can only be implemented for Py{Ref}<T> or T", + )); + } + segment.ident.clone() + }; + (segment.ident.clone(), payload_ty) + } + _ => { + return Err(syn::Error::new_spanned( + imp.self_ty, + "PyImpl can only be implemented for Py{Ref}<T> or T", + )); + } + }; + + let ExtractedImplAttrs { + payload: attr_payload, + flags, + with_impl, + with_method_defs, + with_slots, + itemsize, + } = extract_impl_attrs(attr, &impl_ty)?; + let payload_ty = attr_payload.unwrap_or(payload_guess); + let method_def = &context.method_items; + let getset_impl = &context.getset_items; + let member_impl = &context.member_items; + let extend_impl = context.attribute_items.validate()?; + let slots_impl = context.extend_slots_items.validate()?; + let class_extensions = &context.class_extensions; + + let extra_methods = [ + parse_quote! { + const __OWN_METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_def; + }, + parse_quote! { + fn __extend_py_class( + ctx: &'static ::rustpython_vm::Context, + class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + ) { + #getset_impl + #member_impl + #extend_impl + #(#class_extensions)* + } + }, + { + let itemsize_impl = itemsize.as_ref().map(|size| { + quote! { + slots.itemsize = #size; + } + }); + parse_quote! { + fn __extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { + #itemsize_impl + #slots_impl + } + } + }, + ]; + imp.items.extend(extra_methods); + let is_main_impl = impl_ty == payload_ty; + if is_main_impl { + let method_defs = if with_method_defs.is_empty() { + quote!(#impl_ty::__OWN_METHOD_DEFS) + } else { + quote!( + rustpython_vm::function::PyMethodDef::__const_concat_arrays::< + { #impl_ty::__OWN_METHOD_DEFS.len() #(+ #with_method_defs.len())* }, + >(&[#impl_ty::__OWN_METHOD_DEFS, #(#with_method_defs,)*]) + ) + }; + quote! { + #imp + impl ::rustpython_vm::class::PyClassImpl for #payload_ty { + const TP_FLAGS: ::rustpython_vm::types::PyTypeFlags = #flags; + + fn impl_extend_class( + ctx: &'static ::rustpython_vm::Context, + class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + ) { + #impl_ty::__extend_py_class(ctx, class); + #with_impl + } + + const METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_defs; + + fn extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { + #with_slots + #impl_ty::__extend_slots(slots); + } + } + } + } else { + imp.into_token_stream() + } + } + Item::Trait(mut trai) => { + let mut context = ImplContext { + is_trait: true, + ..Default::default() + }; + let mut has_extend_slots = false; + for item in &trai.items { + let has = match item { + syn::TraitItem::Fn(item) => item.sig.ident == "extend_slots", + _ => false, + }; + if has { + has_extend_slots = has; + break; + } + } + extract_items_into_context(&mut context, trai.items.iter_mut()); + + let ExtractedImplAttrs { + with_impl, + with_slots, + .. + } = extract_impl_attrs(attr, &trai.ident)?; + + let method_def = &context.method_items; + let getset_impl = &context.getset_items; + let member_impl = &context.member_items; + let extend_impl = &context.attribute_items.validate()?; + let slots_impl = &context.extend_slots_items.validate()?; + let class_extensions = &context.class_extensions; + let call_extend_slots = if has_extend_slots { + quote! { + Self::extend_slots(slots); + } + } else { + quote! {} + }; + let extra_methods = [ + parse_quote! { + const __OWN_METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_def; + }, + parse_quote! { + fn __extend_py_class( + ctx: &'static ::rustpython_vm::Context, + class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + ) { + #getset_impl + #member_impl + #extend_impl + #with_impl + #(#class_extensions)* + } + }, + parse_quote! { + fn __extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { + #with_slots + #slots_impl + #call_extend_slots + } + }, + ]; + trai.items.extend(extra_methods); + + trai.into_token_stream() + } + item => item.into_token_stream(), + }; + if let Some(error) = context.errors.into_error() { + let error = Diagnostic::from(error); + tokens = quote! { + #tokens + #error + } + } + Ok(tokens) +} + +/// Validates that when a base class is specified, the struct has the base type as its first field. +/// This ensures proper memory layout for subclassing (required for #[repr(transparent)] to work correctly). +fn validate_base_field(item: &Item, base_path: &syn::Path) -> Result<()> { + let Item::Struct(item_struct) = item else { + // Only validate structs - enums with base are already an error elsewhere + return Ok(()); + }; + + // Get the base type name for error messages + let base_name = base_path + .segments + .last() + .map(|s| s.ident.to_string()) + .unwrap_or_else(|| quote!(#base_path).to_string()); + + match &item_struct.fields { + syn::Fields::Named(fields) => { + let Some(first_field) = fields.named.first() else { + bail_span!( + item_struct, + "#[pyclass] with base = {base_name} requires the first field to be of type {base_name}, but the struct has no fields" + ); + }; + if !type_matches_path(&first_field.ty, base_path) { + bail_span!( + first_field, + "#[pyclass] with base = {base_name} requires the first field to be of type {base_name}" + ); + } + } + syn::Fields::Unnamed(fields) => { + let Some(first_field) = fields.unnamed.first() else { + bail_span!( + item_struct, + "#[pyclass] with base = {base_name} requires the first field to be of type {base_name}, but the struct has no fields" + ); + }; + if !type_matches_path(&first_field.ty, base_path) { + bail_span!( + first_field, + "#[pyclass] with base = {base_name} requires the first field to be of type {base_name}" + ); + } + } + syn::Fields::Unit => { + bail_span!( + item_struct, + "#[pyclass] with base = {base_name} requires the first field to be of type {base_name}, but the struct is a unit struct" + ); + } + } + + Ok(()) +} + +/// Check if a type matches a given path (handles simple cases like `Foo` or `path::to::Foo`) +fn type_matches_path(ty: &syn::Type, path: &syn::Path) -> bool { + // Compare by converting both to string representation for macro hygiene + let ty_str = quote!(#ty).to_string().replace(' ', ""); + let path_str = quote!(#path).to_string().replace(' ', ""); + + // Check if both are the same or if the type ends with the path's last segment + if ty_str == path_str { + return true; + } + + // Also match if just the last segment matches (e.g., foo::Bar matches Bar) + let syn::Type::Path(type_path) = ty else { + return false; + }; + let Some(type_last) = type_path.path.segments.last() else { + return false; + }; + let Some(path_last) = path.segments.last() else { + return false; + }; + type_last.ident == path_last.ident +} + +fn generate_class_def( + ident: &Ident, + name: &str, + module_name: Option<&str>, + base: Option<syn::Path>, + metaclass: Option<String>, + unhashable: bool, + attrs: &[Attribute], +) -> Result<TokenStream> { + let doc = attrs.doc().or_else(|| { + let module_name = module_name.unwrap_or("builtins"); + DB.get(&format!("{module_name}.{name}")) + .copied() + .map(str::to_owned) + }); + let doc = if let Some(doc) = doc { + quote!(Some(#doc)) + } else { + quote!(None) + }; + let module_class_name = if let Some(module_name) = module_name { + format!("{module_name}.{name}") + } else { + name.to_owned() + }; + let module_name = match module_name { + Some(v) => quote!(Some(#v) ), + None => quote!(None), + }; + let unhashable = if unhashable { + quote!(true) + } else { + quote!(false) + }; + let is_pystruct = attrs.iter().any(|attr| { + attr.path().is_ident("derive") + && if let Ok(Meta::List(l)) = attr.parse_meta() { + l.nested + .into_iter() + .any(|n| n.get_ident().is_some_and(|p| p == "PyStructSequence")) + } else { + false + } + }); + // Check if the type has #[repr(transparent)] - only then we can safely + // generate PySubclass impl (requires same memory layout as base type) + let is_repr_transparent = attrs.iter().any(|attr| { + attr.path().is_ident("repr") + && if let Ok(Meta::List(l)) = attr.parse_meta() { + l.nested + .into_iter() + .any(|n| n.get_ident().is_some_and(|p| p == "transparent")) + } else { + false + } + }); + // If repr(transparent) with a base, the type has the same memory layout as base, + // so basicsize should be 0 (no additional space beyond the base type) + // Otherwise, basicsize = sizeof(payload). The header size is added in __basicsize__ getter. + let basicsize = if is_repr_transparent && base.is_some() { + quote!(0) + } else { + quote!(std::mem::size_of::<#ident>()) + }; + if base.is_some() && is_pystruct { + bail_span!(ident, "PyStructSequence cannot have `base` class attr",); + } + let base_class = if is_pystruct { + Some(quote! { rustpython_vm::builtins::PyTuple }) + } else { + base.as_ref().map(|typ| { + quote_spanned! { ident.span() => #typ } + }) + } + .map(|typ| { + quote! { + fn static_baseclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + use rustpython_vm::class::StaticType; + #typ::static_type() + } + } + }); + + let meta_class = metaclass.map(|typ| { + let typ = Ident::new(&typ, ident.span()); + quote! { + fn static_metaclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + use rustpython_vm::class::StaticType; + #typ::static_type() + } + } + }); + + let base_or_object = if let Some(ref base) = base { + quote! { #base } + } else { + quote! { ::rustpython_vm::builtins::PyBaseObject } + }; + + // Generate PySubclass impl for #[repr(transparent)] types with base class + // (tuple struct assumed, so &self.0 works) + let subclass_impl = if !is_pystruct && is_repr_transparent { + base.as_ref().map(|typ| { + quote! { + impl ::rustpython_vm::class::PySubclass for #ident { + type Base = #typ; + + #[inline] + fn as_base(&self) -> &Self::Base { + &self.0 + } + } + } + }) + } else { + None + }; + + let tokens = quote! { + impl ::rustpython_vm::class::PyClassDef for #ident { + const NAME: &'static str = #name; + const MODULE_NAME: Option<&'static str> = #module_name; + const TP_NAME: &'static str = #module_class_name; + const DOC: Option<&'static str> = #doc; + const BASICSIZE: usize = #basicsize; + const UNHASHABLE: bool = #unhashable; + + type Base = #base_or_object; + } + + impl ::rustpython_vm::class::StaticType for #ident { + fn static_cell() -> &'static ::rustpython_vm::common::static_cell::StaticCell<::rustpython_vm::builtins::PyTypeRef> { + ::rustpython_vm::common::static_cell! { + static CELL: ::rustpython_vm::builtins::PyTypeRef; + } + &CELL + } + + #meta_class + + #base_class + } + + #subclass_impl + }; + Ok(tokens) +} + +pub(crate) fn impl_pyclass(attr: PunctuatedNestedMeta, item: Item) -> Result<TokenStream> { + if matches!(item, syn::Item::Use(_)) { + return Ok(quote!(#item)); + } + let (ident, attrs) = pyclass_ident_and_attrs(&item)?; + let fake_ident = Ident::new("pyclass", item.span()); + let class_meta = ClassItemMeta::from_nested(ident.clone(), fake_ident, attr.into_iter())?; + let class_name = class_meta.class_name()?; + let module_name = class_meta.module()?; + let base = class_meta.base()?; + let metaclass = class_meta.metaclass()?; + let unhashable = class_meta.unhashable()?; + + // Validate that if base is specified, the first field must be of the base type + if let Some(ref base_path) = base { + validate_base_field(&item, base_path)?; + } + + let class_def = generate_class_def( + ident, + &class_name, + module_name.as_deref(), + base.clone(), + metaclass, + unhashable, + attrs, + )?; + + const ALLOWED_TRAVERSE_OPTS: &[&str] = &["manual"]; + // Generate MaybeTraverse impl with both traverse and clear support + // + // For traverse: + // 1. no `traverse` at all: HAS_TRAVERSE = false, try_traverse does nothing + // 2. `traverse = "manual"`: HAS_TRAVERSE = true, but no #[derive(Traverse)] + // 3. `traverse`: HAS_TRAVERSE = true, and #[derive(Traverse)] + // + // For clear (tp_clear): + // 1. no `clear`: HAS_CLEAR = HAS_TRAVERSE (default: same as traverse) + // 2. `clear` or `clear = true`: HAS_CLEAR = true, try_clear calls Traverse::clear + // 3. `clear = false`: HAS_CLEAR = false (rare: traverse without clear) + let has_traverse = class_meta.inner()._has_key("traverse")?; + let has_clear = if class_meta.inner()._has_key("clear")? { + // If clear attribute is present, use its value + class_meta.inner()._bool("clear")? + } else { + // If clear attribute is absent, default to same as traverse + has_traverse + }; + + let derive_trace = if has_traverse { + // _optional_str returns Err when key exists without value (e.g., `traverse` vs `traverse = "manual"`) + // We want to derive Traverse in that case, so we handle Err as Ok(None) + let value = class_meta.inner()._optional_str("traverse").ok().flatten(); + if let Some(s) = value { + if !ALLOWED_TRAVERSE_OPTS.contains(&s.as_str()) { + bail_span!( + item, + "traverse attribute only accept {ALLOWED_TRAVERSE_OPTS:?} as value or no value at all", + ); + } + assert_eq!(s, "manual"); + quote! {} + } else { + quote! {#[derive(Traverse)]} + } + } else { + quote! {} + }; + + let maybe_traverse_code = { + let try_traverse_body = if has_traverse { + quote! { + ::rustpython_vm::object::Traverse::traverse(self, tracer_fn); + } + } else { + quote! { + // do nothing + } + }; + + let try_clear_body = if has_clear { + quote! { + ::rustpython_vm::object::Traverse::clear(self, out); + } + } else { + quote! { + // do nothing + } + }; + + quote! { + impl ::rustpython_vm::object::MaybeTraverse for #ident { + const HAS_TRAVERSE: bool = #has_traverse; + const HAS_CLEAR: bool = #has_clear; + + fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { + #try_traverse_body + } + + fn try_clear(&mut self, out: &mut ::std::vec::Vec<::rustpython_vm::PyObjectRef>) { + #try_clear_body + } + } + } + }; + + // Generate PyPayload impl based on whether base exists + #[allow(clippy::collapsible_else_if)] + let impl_payload = if let Some(base_type) = &base { + let class_fn = if let Some(ctx_type_name) = class_meta.ctx_name()? { + let ctx_type_ident = Ident::new(&ctx_type_name, ident.span()); + quote! { ctx.types.#ctx_type_ident } + } else { + quote! { <Self as ::rustpython_vm::class::StaticType>::static_type() } + }; + + quote! { + // static_assertions::const_assert!(std::mem::size_of::<#base_type>() <= std::mem::size_of::<#ident>()); + impl ::rustpython_vm::PyPayload for #ident { + const PAYLOAD_TYPE_ID: ::core::any::TypeId = <#base_type as ::rustpython_vm::PyPayload>::PAYLOAD_TYPE_ID; + + #[inline] + unsafe fn validate_downcastable_from(obj: &::rustpython_vm::PyObject) -> bool { + <Self as ::rustpython_vm::class::PyClassDef>::BASICSIZE <= obj.class().slots.basicsize && obj.class().fast_issubclass(<Self as ::rustpython_vm::class::StaticType>::static_type()) + } + + fn class(ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + #class_fn + } + } + } + } else { + if let Some(ctx_type_name) = class_meta.ctx_name()? { + let ctx_type_ident = Ident::new(&ctx_type_name, ident.span()); + quote! { + impl ::rustpython_vm::PyPayload for #ident { + fn class(ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + ctx.types.#ctx_type_ident + } + } + } + } else { + quote! {} + } + }; + + let empty_impl = if let Some(attrs) = class_meta.impl_attrs()? { + let attrs: Meta = parse_quote! (#attrs); + quote! { + #[pyclass(#attrs)] + impl #ident {} + } + } else { + quote! {} + }; + + let ret = quote! { + #derive_trace + #item + #maybe_traverse_code + #class_def + #impl_payload + #empty_impl + }; + Ok(ret) +} + +/// Special macro to create exception types. +/// +/// Why do we need it and why can't we just use `pyclass` macro instead? +/// We generate exception types with a `macro_rules`, +/// similar to how CPython does it. +/// But, inside `macro_rules` we don't have an opportunity +/// to add non-literal attributes to `pyclass`. +/// That's why we have to use this proxy. +pub(crate) fn impl_pyexception(attr: PunctuatedNestedMeta, item: Item) -> Result<TokenStream> { + let (ident, _attrs) = pyexception_ident_and_attrs(&item)?; + let fake_ident = Ident::new("pyclass", item.span()); + let class_meta = ExceptionItemMeta::from_nested(ident.clone(), fake_ident, attr.into_iter())?; + let class_name = class_meta.class_name()?; + + let base_class_name = class_meta.base()?; + let impl_pyclass = if class_meta.has_impl()? { + quote! { + #[pyexception] + impl #ident {} + } + } else { + quote! {} + }; + + let ret = quote! { + #[pyclass(module = false, name = #class_name, base = #base_class_name)] + #item + #impl_pyclass + }; + Ok(ret) +} + +pub(crate) fn impl_pyexception_impl(attr: PunctuatedNestedMeta, item: Item) -> Result<TokenStream> { + let Item::Impl(imp) = item else { + return Ok(item.into_token_stream()); + }; + + // Check if with(Constructor) is specified. If Constructor trait is used, don't generate slot_new + let mut extra_attrs = Vec::new(); + let mut with_items = vec![]; + for nested in &attr { + if let NestedMeta::Meta(Meta::List(MetaList { path, nested, .. })) = nested { + // If we already found the constructor trait, no need to keep looking for it + if path.is_ident("with") { + for meta in nested { + with_items.push(meta.get_ident().expect("with() has non-ident item").clone()); + } + continue; + } + extra_attrs.push(NestedMeta::Meta(Meta::List(MetaList { + path: path.clone(), + paren_token: Default::default(), + nested: nested.clone(), + }))); + } + } + + let with_contains = |with_items: &[Ident], s: &str| { + // Check if Constructor is in the list + with_items.iter().any(|ident| ident == s) + }; + + let syn::ItemImpl { + generics, + self_ty, + items, + .. + } = &imp; + + let slot_new = if with_contains(&with_items, "Constructor") { + quote!() + } else { + with_items.push(Ident::new("Constructor", Span::call_site())); + quote! { + impl ::rustpython_vm::types::Constructor for #self_ty { + type Args = ::rustpython_vm::function::FuncArgs; + + fn slot_new( + cls: ::rustpython_vm::builtins::PyTypeRef, + args: ::rustpython_vm::function::FuncArgs, + vm: &::rustpython_vm::VirtualMachine, + ) -> ::rustpython_vm::PyResult { + <Self as ::rustpython_vm::class::PyClassDef>::Base::slot_new(cls, args, vm) + } + fn py_new( + _cls: &::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, + _args: Self::Args, + _vm: &::rustpython_vm::VirtualMachine + ) -> ::rustpython_vm::PyResult<Self> { + unreachable!("slot_new is defined") + } + } + } + }; + + // SimpleExtendsException: inherits BaseException_init from the base class via MRO. + // Only exceptions that explicitly specify `with(Initializer)` will have + // their own __init__ in __dict__. + let slot_init = quote!(); + + let extra_attrs_tokens = if extra_attrs.is_empty() { + quote!() + } else { + quote!(, #(#extra_attrs),*) + }; + + Ok(quote! { + #[pyclass(flags(BASETYPE, HAS_DICT), with(#(#with_items),*) #extra_attrs_tokens)] + impl #generics #self_ty { + #(#items)* + } + + #slot_new + #slot_init + }) +} + +macro_rules! define_content_item { + ( + $(#[$meta:meta])* + $vis:vis struct $name:ident + ) => { + $(#[$meta])* + $vis struct $name { + inner: ContentItemInner<AttrName>, + } + + impl ContentItem for $name { + type AttrName = AttrName; + + fn inner(&self) -> &ContentItemInner<AttrName> { + &self.inner + } + } + }; +} + +define_content_item!( + /// #[pymethod] and #[pyclassmethod] + struct MethodItem +); + +define_content_item!( + /// #[pygetset] + struct GetSetItem +); + +define_content_item!( + /// #[pyslot] + struct SlotItem +); + +define_content_item!( + /// #[pyattr] + struct AttributeItem +); + +define_content_item!( + /// #[extend_class] + struct ExtendClassItem +); + +define_content_item!( + /// #[pymember] + struct MemberItem +); + +struct ImplItemArgs<'a, Item: ItemLike> { + item: &'a Item, + attrs: &'a mut Vec<Attribute>, + context: &'a mut ImplContext, + cfgs: &'a [Attribute], +} + +trait ImplItem<Item>: ContentItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()>; +} + +impl<Item> ImplItem<Item> for MethodItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { + let func = args + .item + .function_or_method() + .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))?; + let ident = &func.sig().ident; + + let item_attr = args.attrs.remove(self.index()); + let item_meta = MethodItemMeta::from_attr(ident.clone(), &item_attr)?; + + let py_name = item_meta.method_name()?; + + // Disallow slot methods - they should be defined via trait implementations + // These are exposed as wrapper_descriptor via add_operators from SLOT_DEFS + if !args.context.is_trait { + const FORBIDDEN_SLOT_METHODS: &[(&str, &str)] = &[ + // Constructor/Initializer traits + ("__new__", "Constructor"), + ("__init__", "Initializer"), + // Representable trait + ("__repr__", "Representable"), + // ("__str__", "???"), // allow __str__ + // Hashable trait + ("__hash__", "Hashable"), + // Callable trait + ("__call__", "Callable"), + // GetAttr/SetAttr traits + // NOTE: __getattribute__, __setattr__, __delattr__ are intentionally NOT forbidden + // because they need pymethod for subclass override mechanism to work properly. + // GetDescriptor/SetDescriptor traits + ("__get__", "GetDescriptor"), + ("__set__", "SetDescriptor"), + ("__delete__", "SetDescriptor"), + // AsNumber trait + ("__add__", "AsNumber"), + ("__radd__", "AsNumber"), + ("__iadd__", "AsNumber"), + ("__sub__", "AsNumber"), + ("__rsub__", "AsNumber"), + ("__isub__", "AsNumber"), + ("__mul__", "AsNumber"), + ("__rmul__", "AsNumber"), + ("__imul__", "AsNumber"), + ("__truediv__", "AsNumber"), + ("__rtruediv__", "AsNumber"), + ("__itruediv__", "AsNumber"), + ("__floordiv__", "AsNumber"), + ("__rfloordiv__", "AsNumber"), + ("__ifloordiv__", "AsNumber"), + ("__mod__", "AsNumber"), + ("__rmod__", "AsNumber"), + ("__imod__", "AsNumber"), + ("__pow__", "AsNumber"), + ("__rpow__", "AsNumber"), + ("__ipow__", "AsNumber"), + ("__divmod__", "AsNumber"), + ("__rdivmod__", "AsNumber"), + ("__matmul__", "AsNumber"), + ("__rmatmul__", "AsNumber"), + ("__imatmul__", "AsNumber"), + ("__lshift__", "AsNumber"), + ("__rlshift__", "AsNumber"), + ("__ilshift__", "AsNumber"), + ("__rshift__", "AsNumber"), + ("__rrshift__", "AsNumber"), + ("__irshift__", "AsNumber"), + ("__and__", "AsNumber"), + ("__rand__", "AsNumber"), + ("__iand__", "AsNumber"), + ("__or__", "AsNumber"), + ("__ror__", "AsNumber"), + ("__ior__", "AsNumber"), + ("__xor__", "AsNumber"), + ("__rxor__", "AsNumber"), + ("__ixor__", "AsNumber"), + ("__neg__", "AsNumber"), + ("__pos__", "AsNumber"), + ("__abs__", "AsNumber"), + ("__invert__", "AsNumber"), + ("__int__", "AsNumber"), + ("__float__", "AsNumber"), + ("__index__", "AsNumber"), + ("__bool__", "AsNumber"), + // AsSequence trait + ("__len__", "AsSequence (or AsMapping)"), + ("__contains__", "AsSequence"), + // AsMapping trait + ("__getitem__", "AsMapping (or AsSequence)"), + ("__setitem__", "AsMapping (or AsSequence)"), + ("__delitem__", "AsMapping (or AsSequence)"), + // Iterable/IterNext traits + ("__iter__", "Iterable"), + ("__next__", "IterNext"), + // Comparable trait + ("__eq__", "Comparable"), + ("__ne__", "Comparable"), + ("__lt__", "Comparable"), + ("__le__", "Comparable"), + ("__gt__", "Comparable"), + ("__ge__", "Comparable"), + ]; + + if let Some((_, trait_name)) = FORBIDDEN_SLOT_METHODS + .iter() + .find(|(method, _)| *method == py_name.as_str()) + { + return Err(syn::Error::new( + ident.span(), + format!( + "#[pymethod] cannot define '{py_name}'. Use `impl {trait_name} for ...` instead. \ + Slot methods are exposed as wrapper_descriptor automatically.", + ), + )); + } + } + + let raw = item_meta.raw()?; + let sig_doc = text_signature(func.sig(), &py_name); + let has_receiver = func + .sig() + .inputs + .iter() + .any(|arg| matches!(arg, syn::FnArg::Receiver(_))); + let drop_first_typed = match self.inner.attr_name { + AttrName::Method | AttrName::ClassMethod if !has_receiver && !raw => 1, + _ => 0, + }; + let call_flags = infer_native_call_flags(func.sig(), drop_first_typed); + + // Add #[allow(non_snake_case)] for setter methods like set___name__ + let method_name = ident.to_string(); + if method_name.starts_with("set_") && method_name.contains("__") { + let allow_attr: Attribute = parse_quote!(#[allow(non_snake_case)]); + args.attrs.push(allow_attr); + } + + let doc = args.attrs.doc().map(|doc| format_doc(&sig_doc, &doc)); + args.context.method_items.add_item(MethodNurseryItem { + py_name, + cfgs: args.cfgs.to_vec(), + ident: ident.to_owned(), + doc, + raw, + attr_name: self.inner.attr_name, + call_flags, + }); + Ok(()) + } +} + +impl<Item> ImplItem<Item> for GetSetItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { + let func = args + .item + .function_or_method() + .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))?; + let ident = &func.sig().ident; + + let item_attr = args.attrs.remove(self.index()); + let item_meta = GetSetItemMeta::from_attr(ident.clone(), &item_attr)?; + + let (py_name, kind) = item_meta.getset_name()?; + + // Add #[allow(non_snake_case)] for setter methods + if matches!(kind, GetSetItemKind::Set) { + let allow_attr: Attribute = parse_quote!(#[allow(non_snake_case)]); + args.attrs.push(allow_attr); + } + + args.context + .getset_items + .add_item(py_name, args.cfgs.to_vec(), kind, ident.clone())?; + Ok(()) + } +} + +impl<Item> ImplItem<Item> for SlotItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { + let (ident, span) = if let Ok(c) = args.item.constant() { + (c.ident(), c.span()) + } else if let Ok(f) = args.item.function_or_method() { + (&f.sig().ident, f.span()) + } else { + return Err(self.new_syn_error( + args.item.span(), + "can only be on a method or const function pointer", + )); + }; + + let item_attr = args.attrs.remove(self.index()); + let item_meta = SlotItemMeta::from_attr(ident.clone(), &item_attr)?; + + let slot_ident = item_meta.slot_name()?; + let slot_ident = Ident::new(&slot_ident.to_string().to_lowercase(), slot_ident.span()); + let slot_name = slot_ident.to_string(); + let tokens = { + const NON_ATOMIC_SLOTS: &[&str] = &["as_buffer"]; + const POINTER_SLOTS: &[&str] = &["as_sequence", "as_mapping"]; + const STATIC_GEN_SLOTS: &[&str] = &["as_number"]; + + if NON_ATOMIC_SLOTS.contains(&slot_name.as_str()) { + quote_spanned! { span => + slots.#slot_ident = Some(Self::#ident as _); + } + } else if POINTER_SLOTS.contains(&slot_name.as_str()) { + quote_spanned! { span => + slots.#slot_ident.store(Some(PointerSlot::from(Self::#ident()))); + } + } else if STATIC_GEN_SLOTS.contains(&slot_name.as_str()) { + quote_spanned! { span => + slots.#slot_ident = Self::#ident().into(); + } + } else { + quote_spanned! { span => + slots.#slot_ident.store(Some(Self::#ident as _)); + } + } + }; + + let pyname = format!("(slot {slot_name})"); + args.context.extend_slots_items.add_item( + ident.clone(), + vec![pyname], + args.cfgs.to_vec(), + tokens, + 2, + )?; + + Ok(()) + } +} + +impl<Item> ImplItem<Item> for AttributeItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { + let cfgs = args.cfgs.to_vec(); + let attr = args.attrs.remove(self.index()); + + let get_py_name = |attr: &Attribute, ident: &Ident| -> Result<_> { + let item_meta = SimpleItemMeta::from_attr(ident.clone(), attr)?; + let py_name = item_meta.simple_name()?; + Ok(py_name) + }; + let (ident, py_name, tokens) = + if args.item.function_or_method().is_ok() || args.item.constant().is_ok() { + let ident = args.item.get_ident().unwrap(); + let py_name = get_py_name(&attr, ident)?; + + let value = if args.item.constant().is_ok() { + // TODO: ctx.new_value + quote_spanned!(ident.span() => ctx.new_int(Self::#ident).into()) + } else { + quote_spanned!(ident.span() => Self::#ident(ctx)) + }; + ( + ident, + py_name.clone(), + quote! { + class.set_str_attr(#py_name, #value, ctx); + }, + ) + } else { + return Err(self.new_syn_error( + args.item.span(), + "can only be on a const or an associated method without argument", + )); + }; + + args.context + .attribute_items + .add_item(ident.clone(), vec![py_name], cfgs, tokens, 1)?; + + Ok(()) + } +} + +impl<Item> ImplItem<Item> for ExtendClassItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { + args.attrs.remove(self.index()); + + let ident = &args + .item + .function_or_method() + .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))? + .sig() + .ident; + + args.context.class_extensions.push(quote! { + Self::#ident(ctx, class); + }); + + Ok(()) + } +} + +impl<Item> ImplItem<Item> for MemberItem +where + Item: ItemLike + ToTokens + GetIdent, +{ + fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { + let func = args + .item + .function_or_method() + .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))?; + let ident = &func.sig().ident; + + let item_attr = args.attrs.remove(self.index()); + let item_meta = MemberItemMeta::from_attr(ident.clone(), &item_attr)?; + + let (py_name, member_item_kind) = item_meta.member_name()?; + let member_kind = item_meta.member_kind()?; + if let Some(ref s) = member_kind { + match s.as_str() { + "bool" | "object" => {} + other => { + return Err(self.new_syn_error( + args.item.span(), + &format!("unknown member type '{other}'"), + )); + } + } + } + + // Add #[allow(non_snake_case)] for setter methods + if matches!(member_item_kind, MemberItemKind::Set) { + let allow_attr: Attribute = parse_quote!(#[allow(non_snake_case)]); + args.attrs.push(allow_attr); + } + + args.context.member_items.add_item( + py_name, + member_item_kind, + member_kind, + ident.clone(), + )?; + Ok(()) + } +} + +#[derive(Default)] +struct MethodNursery { + items: Vec<MethodNurseryItem>, +} + +struct MethodNurseryItem { + py_name: String, + cfgs: Vec<Attribute>, + ident: Ident, + raw: bool, + doc: Option<String>, + attr_name: AttrName, + call_flags: TokenStream, +} + +impl MethodNursery { + fn add_item(&mut self, item: MethodNurseryItem) { + self.items.push(item); + } + + fn validate(&mut self) -> Result<()> { + let mut name_set = HashSet::new(); + for item in &self.items { + if !name_set.insert((&item.py_name, &item.cfgs)) { + bail_span!(item.ident, "duplicate method name `{}`", item.py_name); + } + } + Ok(()) + } +} + +impl ToTokens for MethodNursery { + fn to_tokens(&self, tokens: &mut TokenStream) { + let mut inner_tokens = TokenStream::new(); + for item in &self.items { + let py_name = &item.py_name; + let ident = &item.ident; + let cfgs = &item.cfgs; + let doc = if let Some(doc) = item.doc.as_ref() { + quote! { Some(#doc) } + } else { + quote! { None } + }; + let binding_flags = match &item.attr_name { + AttrName::Method => { + quote! { rustpython_vm::function::PyMethodFlags::METHOD } + } + AttrName::ClassMethod => { + quote! { rustpython_vm::function::PyMethodFlags::CLASS } + } + AttrName::StaticMethod => { + quote! { rustpython_vm::function::PyMethodFlags::STATIC } + } + _ => unreachable!(), + }; + let call_flags = &item.call_flags; + let flags = quote! { + rustpython_vm::function::PyMethodFlags::from_bits_retain( + (#binding_flags).bits() | (#call_flags).bits() + ) + }; + // TODO: intern + // let py_name = if py_name.starts_with("__") && py_name.ends_with("__") { + // let name_ident = Ident::new(&py_name, ident.span()); + // quote_spanned! { ident.span() => ctx.names.#name_ident } + // } else { + // quote_spanned! { ident.span() => #py_name } + // }; + let method_new = if item.raw { + quote!(new_raw_const) + } else { + quote!(new_const) + }; + inner_tokens.extend(quote! [ + #(#cfgs)* + rustpython_vm::function::PyMethodDef::#method_new( + #py_name, + Self::#ident, + #flags, + #doc, + ), + ]); + } + let array: TokenTree = Group::new(Delimiter::Bracket, inner_tokens).into(); + tokens.extend([array]); + } +} + +#[derive(Default)] +#[allow(clippy::type_complexity)] +struct GetSetNursery { + map: HashMap<(String, Vec<Attribute>), (Option<Ident>, Option<Ident>)>, + validated: bool, +} + +enum GetSetItemKind { + Get, + Set, +} + +impl GetSetNursery { + fn add_item( + &mut self, + name: String, + cfgs: Vec<Attribute>, + kind: GetSetItemKind, + item_ident: Ident, + ) -> Result<()> { + assert!(!self.validated, "new item is not allowed after validation"); + // Note: Both getter and setter can have #[cfg], but they must have matching cfgs + // since the map key is (name, cfgs). This ensures getter and setter are paired correctly. + let entry = self.map.entry((name.clone(), cfgs)).or_default(); + let func = match kind { + GetSetItemKind::Get => &mut entry.0, + GetSetItemKind::Set => &mut entry.1, + }; + if func.is_some() { + bail_span!( + item_ident, + "Multiple property accessors with name '{}'", + name + ); + } + *func = Some(item_ident); + Ok(()) + } + + fn validate(&mut self) -> Result<()> { + let mut errors = Vec::new(); + for ((name, _cfgs), (getter, setter)) in &self.map { + if getter.is_none() { + errors.push(err_span!( + setter.as_ref().unwrap(), + "GetSet '{}' is missing a getter", + name + )); + }; + } + errors.into_result()?; + self.validated = true; + Ok(()) + } +} + +impl ToTokens for GetSetNursery { + fn to_tokens(&self, tokens: &mut TokenStream) { + assert!(self.validated, "Call `validate()` before token generation"); + let properties = self.map.iter().map(|((name, cfgs), (getter, setter))| { + let setter = match setter { + Some(setter) => quote_spanned! { setter.span() => .with_set(Self::#setter)}, + None => quote! {}, + }; + quote_spanned! { getter.span() => + #( #cfgs )* + class.set_str_attr( + #name, + ::rustpython_vm::PyRef::new_ref( + ::rustpython_vm::builtins::PyGetSet::new(#name.into(), class) + .with_get(Self::#getter) + #setter, + ctx.types.getset_type.to_owned(), None), + ctx + ); + } + }); + tokens.extend(properties); + } +} + +/// Member kind as string, matching `rustpython_vm::builtins::descriptor::MemberKind` variants. +/// None means ObjectEx (default). Valid values: "bool", "object". +type MemberKindStr = Option<String>; + +#[derive(Default)] +struct MemberNursery { + map: HashMap<String, MemberNurseryEntry>, + validated: bool, +} + +struct MemberNurseryEntry { + kind: MemberKindStr, + getter: Option<Ident>, + setter: Option<Ident>, +} + +enum MemberItemKind { + Get, + Set, +} + +impl MemberNursery { + fn add_item( + &mut self, + name: String, + kind: MemberItemKind, + member_kind: MemberKindStr, + item_ident: Ident, + ) -> Result<()> { + assert!(!self.validated, "new item is not allowed after validation"); + let entry = self + .map + .entry(name.clone()) + .or_insert_with(|| MemberNurseryEntry { + kind: member_kind, + getter: None, + setter: None, + }); + let func = match kind { + MemberItemKind::Get => &mut entry.getter, + MemberItemKind::Set => &mut entry.setter, + }; + if func.is_some() { + bail_span!(item_ident, "Multiple member accessors with name '{}'", name); + } + *func = Some(item_ident); + Ok(()) + } + + fn validate(&mut self) -> Result<()> { + let mut errors = Vec::new(); + for (name, entry) in &self.map { + if entry.getter.is_none() { + errors.push(err_span!( + entry.setter.as_ref().unwrap(), + "Member '{}' is missing a getter", + name + )); + }; + } + errors.into_result()?; + self.validated = true; + Ok(()) + } +} + +impl ToTokens for MemberNursery { + fn to_tokens(&self, tokens: &mut TokenStream) { + assert!(self.validated, "Call `validate()` before token generation"); + let properties = self.map.iter().map(|(name, entry)| { + let setter = match &entry.setter { + Some(setter) => quote_spanned! { setter.span() => Some(Self::#setter)}, + None => quote! { None }, + }; + let member_kind = match entry.kind.as_deref() { + Some("bool") => { + quote!(::rustpython_vm::builtins::descriptor::MemberKind::Bool) + } + Some("object") => { + quote!(::rustpython_vm::builtins::descriptor::MemberKind::Object) + } + _ => { + quote!(::rustpython_vm::builtins::descriptor::MemberKind::ObjectEx) + } + }; + let getter = entry.getter.as_ref().unwrap(); + quote_spanned! { getter.span() => + class.set_str_attr( + #name, + ctx.new_member(#name, #member_kind, Self::#getter, #setter, class), + ctx, + ); + } + }); + tokens.extend(properties); + } +} + +struct MethodItemMeta(ItemMetaInner); + +impl ItemMeta for MethodItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name", "raw"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl MethodItemMeta { + fn raw(&self) -> Result<bool> { + self.inner()._bool("raw") + } + + fn method_name(&self) -> Result<String> { + let inner = self.inner(); + let name = inner._optional_str("name")?; + Ok(if let Some(name) = name { + name + } else { + inner.item_name() + }) + } +} + +struct GetSetItemMeta(ItemMetaInner); + +impl ItemMeta for GetSetItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name", "setter"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl GetSetItemMeta { + fn getset_name(&self) -> Result<(String, GetSetItemKind)> { + let inner = self.inner(); + let kind = if inner._bool("setter")? { + GetSetItemKind::Set + } else { + GetSetItemKind::Get + }; + let name = inner._optional_str("name")?; + let py_name = if let Some(name) = name { + name + } else { + let sig_name = inner.item_name(); + let extract_prefix_name = |prefix, item_typ| { + if let Some(name) = sig_name.strip_prefix(prefix) { + if name.is_empty() { + Err(err_span!( + inner.meta_ident, + r#"A #[{}({typ})] fn with a {prefix}* name must \ + have something after "{prefix}""#, + inner.meta_name(), + typ = item_typ, + prefix = prefix + )) + } else { + Ok(name.to_owned()) + } + } else { + Err(err_span!( + inner.meta_ident, + r#"A #[{}(setter)] fn must either have a `name` \ + parameter or a fn name along the lines of "set_*""#, + inner.meta_name() + )) + } + }; + match kind { + GetSetItemKind::Get => sig_name, + GetSetItemKind::Set => extract_prefix_name("set_", "setter")?, + } + }; + Ok((py_name, kind)) + } +} + +struct SlotItemMeta(ItemMetaInner); + +impl ItemMeta for SlotItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &[]; // not used + + fn from_nested<I>(item_ident: Ident, meta_ident: Ident, mut nested: I) -> Result<Self> + where + I: core::iter::Iterator<Item = NestedMeta>, + { + let meta_map = if let Some(nested_meta) = nested.next() { + match nested_meta { + NestedMeta::Meta(meta) => { + Some([("name".to_owned(), (0, meta))].iter().cloned().collect()) + } + _ => None, + } + } else { + Some(HashMap::default()) + }; + let (Some(meta_map), None) = (meta_map, nested.next()) else { + bail_span!( + meta_ident, + "#[pyslot] must be of the form #[pyslot] or #[pyslot(slot_name)]" + ) + }; + Ok(Self::from_inner(ItemMetaInner { + item_ident, + meta_ident, + meta_map, + })) + } + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl SlotItemMeta { + fn slot_name(&self) -> Result<Ident> { + let inner = self.inner(); + let method_name = if let Some((_, meta)) = inner.meta_map.get("name") { + match meta { + Meta::Path(path) => path.get_ident().cloned(), + _ => None, + } + } else { + let ident_str = self.inner().item_name(); + // Convert to lowercase to handle both SLOT_NEW and slot_new + let ident_lower = ident_str.to_lowercase(); + let name = if let Some(stripped) = ident_lower.strip_prefix("slot_") { + proc_macro2::Ident::new(stripped, inner.item_ident.span()) + } else { + inner.item_ident.clone() + }; + Some(name) + }; + let method_name = method_name.ok_or_else(|| { + err_span!( + inner.meta_ident, + "#[pyslot] must be of the form #[pyslot] or #[pyslot(slot_name)]", + ) + })?; + + // Strip double underscores from slot names like __init__ -> init + let method_name_str = method_name.to_string(); + let slot_name = if method_name_str.starts_with("__") + && method_name_str.ends_with("__") + && method_name_str.len() > 4 + { + &method_name_str[2..method_name_str.len() - 2] + } else { + &method_name_str + }; + + Ok(proc_macro2::Ident::new(slot_name, slot_name.span())) + } +} + +struct MemberItemMeta(ItemMetaInner); + +impl ItemMeta for MemberItemMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["type", "setter"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self(inner) + } + + fn inner(&self) -> &ItemMetaInner { + &self.0 + } +} + +impl MemberItemMeta { + fn member_name(&self) -> Result<(String, MemberItemKind)> { + let inner = self.inner(); + let sig_name = inner.item_name(); + let extract_prefix_name = |prefix, item_typ| { + if let Some(name) = sig_name.strip_prefix(prefix) { + if name.is_empty() { + Err(err_span!( + inner.meta_ident, + r#"A #[{}({typ})] fn with a {prefix}* name must \ + have something after "{prefix}""#, + inner.meta_name(), + typ = item_typ, + prefix = prefix + )) + } else { + Ok(name.to_owned()) + } + } else { + Err(err_span!( + inner.meta_ident, + r#"A #[{}(setter)] fn must either have a `name` \ + parameter or a fn name along the lines of "set_*""#, + inner.meta_name() + )) + } + }; + let kind = if inner._bool("setter")? { + MemberItemKind::Set + } else { + MemberItemKind::Get + }; + let name = match kind { + MemberItemKind::Get => sig_name, + MemberItemKind::Set => extract_prefix_name("set_", "setter")?, + }; + Ok((name, kind)) + } + + fn member_kind(&self) -> Result<Option<String>> { + let inner = self.inner(); + inner._optional_str("type") + } +} + +struct ExtractedImplAttrs { + payload: Option<Ident>, + flags: TokenStream, + with_impl: TokenStream, + with_method_defs: Vec<TokenStream>, + with_slots: TokenStream, + itemsize: Option<syn::Expr>, +} + +fn extract_impl_attrs(attr: PunctuatedNestedMeta, item: &Ident) -> Result<ExtractedImplAttrs> { + let mut withs = Vec::new(); + let mut with_method_defs = Vec::new(); + let mut with_slots = Vec::new(); + let mut flags = vec![quote! { + { + #[cfg(not(debug_assertions))] { + ::rustpython_vm::types::PyTypeFlags::DEFAULT + } + #[cfg(debug_assertions)] { + ::rustpython_vm::types::PyTypeFlags::DEFAULT + .union(::rustpython_vm::types::PyTypeFlags::_CREATED_WITH_FLAGS) + } + } + }]; + let mut payload = None; + let mut itemsize = None; + + for attr in attr { + match attr { + NestedMeta::Meta(Meta::List(MetaList { path, nested, .. })) => { + if path.is_ident("with") { + for meta in nested { + let NestedMeta::Meta(Meta::Path(path)) = &meta else { + bail_span!(meta, "#[pyclass(with(...))] arguments must be paths") + }; + let (extend_class, method_defs, extend_slots) = if path.is_ident("PyRef") + || path.is_ident("Py") + { + // special handling for PyRef + ( + quote!(#path::<Self>::__extend_py_class), + quote!(#path::<Self>::__OWN_METHOD_DEFS), + quote!(#path::<Self>::__extend_slots), + ) + } else { + if path.is_ident("DefaultConstructor") { + bail_span!( + meta, + "Try `#[pyclass(with(Constructor, ...))]` instead of `#[pyclass(with(DefaultConstructor, ...))]`. DefaultConstructor implicitly implements Constructor." + ) + } + ( + quote!(<Self as #path>::__extend_py_class), + quote!(<Self as #path>::__OWN_METHOD_DEFS), + quote!(<Self as #path>::__extend_slots), + ) + }; + let item_span = item.span().resolved_at(Span::call_site()); + withs.push(quote_spanned! { path.span() => + #extend_class(ctx, class); + }); + with_method_defs.push(method_defs); + // For Initializer and Constructor traits, directly set the slot + // instead of calling __extend_slots. This ensures that the trait + // impl's override (e.g., slot_init in impl Initializer) is used, + // not the trait's default implementation. + let slot_code = if path.is_ident("Initializer") { + quote_spanned! { item_span => + slots.init.store(Some(<Self as ::rustpython_vm::types::Initializer>::slot_init as _)); + } + } else if path.is_ident("Constructor") { + quote_spanned! { item_span => + slots.new.store(Some(<Self as ::rustpython_vm::types::Constructor>::slot_new as _)); + } + } else { + quote_spanned! { item_span => + #extend_slots(slots); + } + }; + with_slots.push(slot_code); + } + } else if path.is_ident("flags") { + for meta in nested { + let NestedMeta::Meta(Meta::Path(path)) = meta else { + bail_span!(meta, "#[pyclass(flags(...))] arguments should be ident") + }; + let ident = path.get_ident().ok_or_else(|| { + err_span!(path, "#[pyclass(flags(...))] arguments should be ident") + })?; + flags.push(quote_spanned! { ident.span() => + .union(::rustpython_vm::types::PyTypeFlags::#ident) + }); + } + flags.push(quote! { + .union(::rustpython_vm::types::PyTypeFlags::IMMUTABLETYPE) + }); + } else { + bail_span!(path, "Unknown pyimpl attribute") + } + } + NestedMeta::Meta(Meta::NameValue(syn::MetaNameValue { path, value, .. })) => { + if path.is_ident("payload") { + if let syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }) = value + { + payload = Some(Ident::new(&lit.value(), lit.span())); + } else { + bail_span!(value, "payload must be a string literal") + } + } else if path.is_ident("itemsize") { + itemsize = Some(value); + } else { + bail_span!(path, "Unknown pyimpl attribute") + } + } + attr => bail_span!(attr, "Unknown pyimpl attribute"), + } + } + + Ok(ExtractedImplAttrs { + payload, + flags: quote! { + #(#flags)* + }, + with_impl: quote! { + #(#withs)* + }, + with_method_defs, + with_slots: quote! { + #(#with_slots)* + }, + itemsize, + }) +} + +fn impl_item_new<Item>( + index: usize, + attr_name: AttrName, +) -> Result<Box<dyn ImplItem<Item, AttrName = AttrName>>> +where + Item: ItemLike + ToTokens + GetIdent, +{ + use AttrName::*; + Ok(match attr_name { + attr_name @ Method | attr_name @ ClassMethod | attr_name @ StaticMethod => { + Box::new(MethodItem { + inner: ContentItemInner { index, attr_name }, + }) + } + GetSet => Box::new(GetSetItem { + inner: ContentItemInner { index, attr_name }, + }), + Slot => Box::new(SlotItem { + inner: ContentItemInner { index, attr_name }, + }), + Attr => Box::new(AttributeItem { + inner: ContentItemInner { index, attr_name }, + }), + ExtendClass => Box::new(ExtendClassItem { + inner: ContentItemInner { index, attr_name }, + }), + Member => Box::new(MemberItem { + inner: ContentItemInner { index, attr_name }, + }), + }) +} + +fn attrs_to_content_items<F, R>( + attrs: &[Attribute], + item_new: F, +) -> Result<(Vec<R>, Vec<Attribute>)> +where + F: Fn(usize, AttrName) -> Result<R>, +{ + let mut cfgs: Vec<Attribute> = Vec::new(); + let mut result = Vec::new(); + + let mut iter = attrs.iter().enumerate().peekable(); + while let Some((_, attr)) = iter.peek() { + // take all cfgs but no py items + let attr = *attr; + let attr_name = if let Some(ident) = attr.get_ident() { + ident.to_string() + } else { + continue; + }; + if attr_name == "cfg" { + cfgs.push(attr.clone()); + } else if ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { + break; + } + iter.next(); + } + + for (i, attr) in iter { + // take py items but no cfgs + let attr_name = if let Some(ident) = attr.get_ident() { + ident.to_string() + } else { + continue; + }; + if attr_name == "cfg" { + bail_span!(attr, "#[py*] items must be placed under `cfgs`",); + } + let attr_name = match AttrName::from_str(attr_name.as_str()) { + Ok(name) => name, + Err(wrong_name) => { + if ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { + bail_span!(attr, "#[pyclass] doesn't accept #[{}]", wrong_name) + } else { + continue; + } + } + }; + + result.push(item_new(i, attr_name)?); + } + Ok((result, cfgs)) +} + +#[allow(dead_code)] +fn parse_vec_ident( + attr: &[NestedMeta], + item: &Item, + index: usize, + message: &str, +) -> Result<String> { + Ok(attr + .get(index) + .ok_or_else(|| err_span!(item, "We require {} argument to be set", message))? + .get_ident() + .ok_or_else(|| { + err_span!( + item, + "We require {} argument to be ident or string", + message + ) + })? + .to_string()) +} diff --git a/crates/derive-impl/src/pymodule.rs b/crates/derive-impl/src/pymodule.rs new file mode 100644 index 00000000000..b4b5535200c --- /dev/null +++ b/crates/derive-impl/src/pymodule.rs @@ -0,0 +1,1061 @@ +use crate::error::Diagnostic; +use crate::pystructseq::PyStructSequenceMeta; +use crate::util::{ + ALL_ALLOWED_NAMES, AttrItemMeta, AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, + ErrorVec, ItemMeta, ItemNursery, ModuleItemMeta, SimpleItemMeta, format_doc, + infer_native_call_flags, iter_use_idents, pyclass_ident_and_attrs, text_signature, +}; +use core::str::FromStr; +use proc_macro2::{Delimiter, Group, TokenStream, TokenTree}; +use quote::{ToTokens, format_ident, quote, quote_spanned}; +use rustpython_doc::DB; +use std::collections::HashSet; +use syn::{Attribute, Ident, Item, Result, parse_quote, spanned::Spanned}; +use syn_ext::ext::*; +use syn_ext::types::NestedMeta; + +/// A `with(...)` item that may be gated by `#[cfg(...)]` attributes. +pub struct WithItem { + pub cfg_attrs: Vec<Attribute>, + pub path: syn::Path, +} + +impl syn::parse::Parse for WithItem { + fn parse(input: syn::parse::ParseStream<'_>) -> Result<Self> { + let cfg_attrs = Attribute::parse_outer(input)?; + for attr in &cfg_attrs { + if !attr.path().is_ident("cfg") { + return Err(syn::Error::new_spanned( + attr, + "only #[cfg(...)] is supported in with()", + )); + } + } + let path = input.parse()?; + Ok(WithItem { cfg_attrs, path }) + } +} + +/// Parsed arguments for `#[pymodule(...)]`, supporting `#[cfg]` inside `with(...)`. +pub struct PyModuleArgs { + pub metas: Vec<NestedMeta>, + pub with_items: Vec<WithItem>, +} + +impl syn::parse::Parse for PyModuleArgs { + fn parse(input: syn::parse::ParseStream<'_>) -> Result<Self> { + let mut metas = Vec::new(); + let mut with_items = Vec::new(); + + while !input.is_empty() { + // Detect `with(...)` — an ident "with" followed by a paren group + if input.peek(Ident) && input.peek2(syn::token::Paren) { + let fork = input.fork(); + let ident: Ident = fork.parse()?; + if ident == "with" { + // Advance past "with" + let _: Ident = input.parse()?; + let content; + syn::parenthesized!(content in input); + let items = + syn::punctuated::Punctuated::<WithItem, syn::Token![,]>::parse_terminated( + &content, + )?; + with_items.extend(items); + if !input.is_empty() { + input.parse::<syn::Token![,]>()?; + } + continue; + } + } + metas.push(input.parse::<NestedMeta>()?); + if input.is_empty() { + break; + } + input.parse::<syn::Token![,]>()?; + } + + Ok(PyModuleArgs { metas, with_items }) + } +} + +/// Generate `#[cfg(not(...))]` attributes that negate the given `#[cfg(...)]` attributes. +fn negate_cfg_attrs(cfg_attrs: &[Attribute]) -> Vec<Attribute> { + if cfg_attrs.is_empty() { + return vec![]; + } + let predicates: Vec<_> = cfg_attrs + .iter() + .map(|attr| match &attr.meta { + syn::Meta::List(list) => list.tokens.clone(), + _ => unreachable!("only #[cfg(...)] should be here"), + }) + .collect(); + if predicates.len() == 1 { + let predicate = &predicates[0]; + vec![parse_quote!(#[cfg(not(#predicate))])] + } else { + vec![parse_quote!(#[cfg(not(all(#(#predicates),*)))])] + } +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum AttrName { + Function, + Attr, + Class, + Exception, + StructSequence, +} + +impl core::fmt::Display for AttrName { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let s = match self { + Self::Function => "pyfunction", + Self::Attr => "pyattr", + Self::Class => "pyclass", + Self::Exception => "pyexception", + Self::StructSequence => "pystruct_sequence", + }; + s.fmt(f) + } +} + +impl FromStr for AttrName { + type Err = String; + + fn from_str(s: &str) -> core::result::Result<Self, Self::Err> { + Ok(match s { + "pyfunction" => Self::Function, + "pyattr" => Self::Attr, + "pyclass" => Self::Class, + "pyexception" => Self::Exception, + "pystruct_sequence" => Self::StructSequence, + s => { + return Err(s.to_owned()); + } + }) + } +} + +#[derive(Default)] +struct ModuleContext { + name: String, + function_items: FunctionNursery, + attribute_items: ItemNursery, + has_module_exec: bool, + errors: Vec<syn::Error>, +} + +pub fn impl_pymodule(args: PyModuleArgs, module_item: Item) -> Result<TokenStream> { + let PyModuleArgs { metas, with_items } = args; + let (doc, mut module_item) = match module_item { + Item::Mod(m) => (m.attrs.doc(), m), + other => bail_span!(other, "#[pymodule] can only be on a full module"), + }; + let fake_ident = Ident::new("pymodule", module_item.span()); + let module_meta = + ModuleItemMeta::from_nested(module_item.ident.clone(), fake_ident, metas.into_iter())?; + + // generation resources + let mut context = ModuleContext { + name: module_meta.simple_name()?, + ..Default::default() + }; + let items = module_item.items_mut().ok_or_else(|| { + module_meta.new_meta_error("requires actual module, not a module declaration") + })?; + + // collect to context + for item in items.iter_mut() { + // Check if module_exec function is already defined + if let Item::Fn(func) = item + && func.sig.ident == "module_exec" + { + context.has_module_exec = true; + } + if matches!(item, Item::Impl(_) | Item::Trait(_)) { + // #[pyclass] implementations + continue; + } + let r = item.try_split_attr_mut(|attrs, item| { + let (py_items, cfgs) = attrs_to_module_items(attrs, module_item_new)?; + for py_item in py_items.iter().rev() { + let r = py_item.gen_module_item(ModuleItemArgs { + item, + attrs, + context: &mut context, + cfgs: cfgs.as_slice(), + }); + context.errors.ok_or_push(r); + } + Ok(()) + }); + context.errors.ok_or_push(r); + } + + // Detect nested #[pymodule] items (non-sub) and generate submodule init code + let mut submodule_inits: Vec<TokenStream> = Vec::new(); + for item in items.iter() { + if let Item::Mod(item_mod) = item { + let r = (|| -> Result<()> { + let attr = match item_mod + .attrs + .iter() + .find(|a| a.path().is_ident("pymodule")) + { + Some(attr) => attr, + None => return Ok(()), + }; + let args_tokens = match &attr.meta { + syn::Meta::Path(_) => TokenStream::new(), + syn::Meta::List(list) => list.tokens.clone(), + _ => return Ok(()), + }; + let mod_args: PyModuleArgs = syn::parse2(args_tokens)?; + let fake_ident = Ident::new("pymodule", attr.span()); + let mod_meta = ModuleItemMeta::from_nested( + item_mod.ident.clone(), + fake_ident, + mod_args.metas.into_iter(), + )?; + if mod_meta.sub()? { + return Ok(()); + } + let py_name = mod_meta.simple_name()?; + let mod_ident = &item_mod.ident; + let cfgs: Vec<_> = item_mod + .attrs + .iter() + .filter(|a| a.path().is_ident("cfg")) + .cloned() + .collect(); + submodule_inits.push(quote! { + #(#cfgs)* + { + let child_def = #mod_ident::module_def(ctx); + let child = child_def.create_module(vm).expect("submodule create_module failed"); + child.__init_methods(vm).expect("submodule __init_methods failed"); + #mod_ident::module_exec(vm, &child).expect("submodule module_exec failed"); + let child: ::rustpython_vm::PyObjectRef = child.into(); + vm.__module_set_attr(module, ctx.intern_str(#py_name), child).expect("module set_attr submodule failed"); + } + }); + Ok(()) + })(); + context.errors.ok_or_push(r); + } + } + + // append additional items + let module_name = context.name.as_str(); + let function_items = context.function_items.validate()?; + let attribute_items = context.attribute_items.validate()?; + let doc = doc.or_else(|| DB.get(module_name).copied().map(str::to_owned)); + let doc = if let Some(doc) = doc { + quote!(Some(#doc)) + } else { + quote!(None) + }; + let is_submodule = module_meta.sub()?; + if !is_submodule { + items.extend([ + parse_quote! { + pub(crate) const MODULE_NAME: &'static str = #module_name; + }, + parse_quote! { + pub(crate) const DOC: Option<&'static str> = #doc; + }, + parse_quote! { + pub(crate) fn module_def( + ctx: &::rustpython_vm::Context, + ) -> &'static ::rustpython_vm::builtins::PyModuleDef { + DEF.get_or_init(|| { + let mut def = ::rustpython_vm::builtins::PyModuleDef { + name: ctx.intern_str(MODULE_NAME), + doc: DOC.map(|doc| ctx.intern_str(doc)), + methods: METHOD_DEFS, + slots: Default::default(), + }; + def.slots.exec = Some(module_exec); + def + }) + } + }, + ]); + } + if !is_submodule && !context.has_module_exec { + items.push(parse_quote! { + pub(crate) fn module_exec(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> { + __module_exec(vm, module); + Ok(()) + } + }); + } + // Split with_items into unconditional and cfg-gated groups + let (uncond_withs, cond_withs): (Vec<_>, Vec<_>) = + with_items.iter().partition(|w| w.cfg_attrs.is_empty()); + let uncond_paths: Vec<_> = uncond_withs.iter().map(|w| &w.path).collect(); + + let method_defs = if with_items.is_empty() { + quote!(#function_items) + } else { + // For cfg-gated with items, generate conditional const declarations + // so the total array size adapts to the cfg at compile time + let cond_const_names: Vec<_> = cond_withs + .iter() + .enumerate() + .map(|(i, _)| format_ident!("__WITH_METHODS_{}", i)) + .collect(); + let cond_const_decls: Vec<_> = cond_withs + .iter() + .zip(&cond_const_names) + .map(|(w, name)| { + let cfg_attrs = &w.cfg_attrs; + let neg_attrs = negate_cfg_attrs(&w.cfg_attrs); + let path = &w.path; + quote! { + #(#cfg_attrs)* + const #name: &'static [::rustpython_vm::function::PyMethodDef] = super::#path::METHOD_DEFS; + #(#neg_attrs)* + const #name: &'static [::rustpython_vm::function::PyMethodDef] = &[]; + } + }) + .collect(); + + quote!({ + const OWN_METHODS: &'static [::rustpython_vm::function::PyMethodDef] = &#function_items; + #(#cond_const_decls)* + rustpython_vm::function::PyMethodDef::__const_concat_arrays::< + { OWN_METHODS.len() + #(+ super::#uncond_paths::METHOD_DEFS.len())* + #(+ #cond_const_names.len())* + }, + >(&[ + #(super::#uncond_paths::METHOD_DEFS,)* + #(#cond_const_names,)* + OWN_METHODS + ]) + }) + }; + + // Generate __init_attributes calls, wrapping cfg-gated items + let init_with_calls: Vec<_> = with_items + .iter() + .map(|w| { + let cfg_attrs = &w.cfg_attrs; + let path = &w.path; + quote! { + #(#cfg_attrs)* + super::#path::__init_attributes(vm, module); + } + }) + .collect(); + + items.extend([ + parse_quote! { + ::rustpython_vm::common::static_cell! { + pub(crate) static DEF: ::rustpython_vm::builtins::PyModuleDef; + } + }, + parse_quote! { + pub(crate) const METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_defs; + }, + parse_quote! { + pub(crate) fn __init_attributes( + vm: &::rustpython_vm::VirtualMachine, + module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>, + ) { + #(#init_with_calls)* + let ctx = &vm.ctx; + #attribute_items + #(#submodule_inits)* + } + }, + parse_quote! { + pub(crate) fn __module_exec( + vm: &::rustpython_vm::VirtualMachine, + module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>, + ) { + __init_attributes(vm, module); + } + }, + parse_quote! { + pub(crate) fn __init_dict( + vm: &::rustpython_vm::VirtualMachine, + module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>, + ) { + ::rustpython_vm::builtins::PyModule::__init_dict_from_def(vm, module); + } + }, + ]); + + Ok(if let Some(error) = context.errors.into_error() { + let error = Diagnostic::from(error); + quote! { + #module_item + #error + } + } else { + module_item.into_token_stream() + }) +} + +fn module_item_new( + index: usize, + attr_name: AttrName, + py_attrs: Vec<usize>, +) -> Box<dyn ModuleItem<AttrName = AttrName>> { + match attr_name { + AttrName::Function => Box::new(FunctionItem { + inner: ContentItemInner { index, attr_name }, + py_attrs, + }), + AttrName::Attr => Box::new(AttributeItem { + inner: ContentItemInner { index, attr_name }, + py_attrs, + }), + // pyexception is treated like pyclass - both define types + AttrName::Class | AttrName::Exception => Box::new(ClassItem { + inner: ContentItemInner { index, attr_name }, + py_attrs, + }), + AttrName::StructSequence => Box::new(StructSequenceItem { + inner: ContentItemInner { index, attr_name }, + py_attrs, + }), + } +} + +fn attrs_to_module_items<F, R>(attrs: &[Attribute], item_new: F) -> Result<(Vec<R>, Vec<Attribute>)> +where + F: Fn(usize, AttrName, Vec<usize>) -> R, +{ + let mut cfgs: Vec<Attribute> = Vec::new(); + let mut result = Vec::new(); + + let mut iter = attrs.iter().enumerate().peekable(); + while let Some((_, attr)) = iter.peek() { + // take all cfgs but no py items + let attr = *attr; + if let Some(ident) = attr.get_ident() { + let attr_name = ident.to_string(); + if attr_name == "cfg" { + cfgs.push(attr.clone()); + } else if ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { + break; + } + } + iter.next(); + } + + let mut closed = false; + let mut py_attrs = Vec::new(); + for (i, attr) in iter { + // take py items but no cfgs + let attr_name = if let Some(ident) = attr.get_ident() { + ident.to_string() + } else { + continue; + }; + if attr_name == "cfg" { + bail_span!(attr, "#[py*] items must be placed under `cfgs`") + } + + let attr_name = match AttrName::from_str(attr_name.as_str()) { + Ok(name) => name, + Err(wrong_name) => { + if !ALL_ALLOWED_NAMES.contains(&wrong_name.as_str()) { + continue; + } else if closed { + bail_span!(attr, "Only one #[pyattr] annotated #[py*] item can exist") + } else { + bail_span!(attr, "#[pymodule] doesn't accept #[{}]", wrong_name) + } + } + }; + + if attr_name == AttrName::Attr { + if !result.is_empty() { + bail_span!( + attr, + "#[pyattr] must be placed on top of other #[py*] items", + ) + } + py_attrs.push(i); + continue; + } + + if py_attrs.is_empty() { + result.push(item_new(i, attr_name, Vec::new())); + } else { + match attr_name { + AttrName::Class + | AttrName::Function + | AttrName::Exception + | AttrName::StructSequence => { + result.push(item_new(i, attr_name, py_attrs.clone())); + } + _ => { + bail_span!( + attr, + "#[pyclass], #[pyfunction], #[pyexception], or #[pystruct_sequence] can follow #[pyattr]", + ) + } + } + py_attrs.clear(); + closed = true; + } + } + + if let Some(last) = py_attrs.pop() { + assert!(!closed); + result.push(item_new(last, AttrName::Attr, py_attrs)); + } + Ok((result, cfgs)) +} + +#[derive(Default)] +struct FunctionNursery { + items: Vec<FunctionNurseryItem>, +} + +struct FunctionNurseryItem { + py_names: Vec<String>, + cfgs: Vec<Attribute>, + ident: Ident, + doc: String, + call_flags: TokenStream, +} + +impl FunctionNursery { + fn add_item(&mut self, item: FunctionNurseryItem) { + self.items.push(item); + } + + fn validate(self) -> Result<ValidatedFunctionNursery> { + let mut name_set = HashSet::new(); + for item in &self.items { + for py_name in &item.py_names { + if !name_set.insert((py_name.to_owned(), &item.cfgs)) { + bail_span!(item.ident, "duplicate method name `{}`", py_name); + } + } + } + Ok(ValidatedFunctionNursery(self)) + } +} + +struct ValidatedFunctionNursery(FunctionNursery); + +impl ToTokens for ValidatedFunctionNursery { + fn to_tokens(&self, tokens: &mut TokenStream) { + let mut inner_tokens = TokenStream::new(); + for item in &self.0.items { + let ident = &item.ident; + let cfgs = &item.cfgs; + let cfgs = quote!(#(#cfgs)*); + let py_names = &item.py_names; + let doc = &item.doc; + let doc = quote!(Some(#doc)); + let flags = &item.call_flags; + + inner_tokens.extend(quote![ + #( + #cfgs + rustpython_vm::function::PyMethodDef::new_const( + #py_names, + #ident, + #flags, + #doc, + ), + )* + ]); + } + let array: TokenTree = Group::new(Delimiter::Bracket, inner_tokens).into(); + tokens.extend([array]); + } +} + +/// #[pyfunction] +struct FunctionItem { + inner: ContentItemInner<AttrName>, + py_attrs: Vec<usize>, +} + +/// #[pyclass] +struct ClassItem { + inner: ContentItemInner<AttrName>, + py_attrs: Vec<usize>, +} + +/// #[pyattr] +struct AttributeItem { + inner: ContentItemInner<AttrName>, + py_attrs: Vec<usize>, +} + +/// #[pystruct_sequence] +struct StructSequenceItem { + inner: ContentItemInner<AttrName>, + py_attrs: Vec<usize>, +} + +impl ContentItem for FunctionItem { + type AttrName = AttrName; + + fn inner(&self) -> &ContentItemInner<AttrName> { + &self.inner + } +} + +impl ContentItem for ClassItem { + type AttrName = AttrName; + + fn inner(&self) -> &ContentItemInner<AttrName> { + &self.inner + } +} + +impl ContentItem for AttributeItem { + type AttrName = AttrName; + + fn inner(&self) -> &ContentItemInner<AttrName> { + &self.inner + } +} + +impl ContentItem for StructSequenceItem { + type AttrName = AttrName; + + fn inner(&self) -> &ContentItemInner<AttrName> { + &self.inner + } +} + +struct ModuleItemArgs<'a> { + item: &'a mut Item, + attrs: &'a mut Vec<Attribute>, + context: &'a mut ModuleContext, + cfgs: &'a [Attribute], +} + +impl<'a> ModuleItemArgs<'a> { + fn module_name(&'a self) -> &'a str { + self.context.name.as_str() + } +} + +trait ModuleItem: ContentItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()>; +} + +impl ModuleItem for FunctionItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { + let func = args + .item + .function_or_method() + .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a function"))?; + let ident = &func.sig().ident; + + let item_attr = args.attrs.remove(self.index()); + let item_meta = SimpleItemMeta::from_attr(ident.clone(), &item_attr)?; + + let py_name = item_meta.simple_name()?; + let sig_doc = text_signature(func.sig(), &py_name); + + let module = args.module_name(); + // TODO: doc must exist at least one of code or CPython + let doc = args.attrs.doc().or_else(|| { + DB.get(&format!("{module}.{py_name}")) + .copied() + .map(str::to_owned) + }); + let doc = if let Some(doc) = doc { + format_doc(&sig_doc, &doc) + } else { + sig_doc + }; + + let py_names = { + if self.py_attrs.is_empty() { + vec![py_name] + } else { + let mut py_names = HashSet::new(); + py_names.insert(py_name); + for attr_index in self.py_attrs.iter().rev() { + let mut loop_unit = || { + let attr_attr = args.attrs.remove(*attr_index); + let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr_attr)?; + + let py_name = item_meta.simple_name()?; + let inserted = py_names.insert(py_name.clone()); + if !inserted { + return Err(self.new_syn_error( + ident.span(), + &format!( + "`{py_name}` is duplicated name for multiple py* attribute" + ), + )); + } + Ok(()) + }; + let r = loop_unit(); + args.context.errors.ok_or_push(r); + } + let py_names: Vec<_> = py_names.into_iter().collect(); + py_names + } + }; + let call_flags = infer_native_call_flags(func.sig(), 0); + + args.context.function_items.add_item(FunctionNurseryItem { + ident: ident.to_owned(), + py_names, + cfgs: args.cfgs.to_vec(), + doc, + call_flags, + }); + Ok(()) + } +} + +impl ModuleItem for ClassItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { + let (ident, _) = pyclass_ident_and_attrs(args.item)?; + let (class_name, class_new) = { + let class_attr = &mut args.attrs[self.inner.index]; + let no_attr = class_attr.try_remove_name("no_attr")?; + if self.py_attrs.is_empty() { + // check no_attr before ClassItemMeta::from_attr + if no_attr.is_none() { + bail_span!( + ident, + "#[{name}] requires #[pyattr] to be a module attribute. \ + To keep it free type, try #[{name}(no_attr)]", + name = self.attr_name() + ) + } + } + let no_attr = no_attr.is_some(); + let is_use = matches!(&args.item, syn::Item::Use(_)); + + let class_meta = ClassItemMeta::from_attr(ident.clone(), class_attr)?; + let module_name = args.context.name.clone(); + let module_name = if let Some(class_module_name) = class_meta.module().ok().flatten() { + class_module_name + } else { + class_attr.fill_nested_meta("module", || { + parse_quote! {module = #module_name} + })?; + module_name + }; + let class_name = if no_attr && is_use { + "<NO ATTR>".to_owned() + } else { + class_meta.class_name()? + }; + let class_new = quote_spanned!(ident.span() => + let new_class = <#ident as ::rustpython_vm::class::PyClassImpl>::make_static_type(); + // Only set __module__ string if the class doesn't already have a + // getset descriptor for __module__ (which provides instance-level + // module resolution, e.g. TypeAliasType) + { + let module_key = rustpython_vm::identifier!(ctx, __module__); + let has_module_getset = new_class.attributes.read() + .get(module_key) + .is_some_and(|v| v.downcastable::<rustpython_vm::builtins::PyGetSet>()); + if !has_module_getset { + new_class.set_attr(module_key, vm.new_pyobj(#module_name)); + } + } + ); + (class_name, class_new) + }; + + let mut py_names = Vec::new(); + for attr_index in self.py_attrs.iter().rev() { + let mut loop_unit = || { + let attr_attr = args.attrs.remove(*attr_index); + let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr_attr)?; + + let py_name = item_meta + .optional_name() + .unwrap_or_else(|| class_name.clone()); + py_names.push(py_name); + + Ok(()) + }; + let r = loop_unit(); + args.context.errors.ok_or_push(r); + } + + let set_attr = match py_names.len() { + 0 => quote! { + let _ = new_class; // suppress warning + let _ = vm.ctx.intern_str(#class_name); + }, + 1 => { + let py_name = &py_names[0]; + quote! { + vm.__module_set_attr(&module, vm.ctx.intern_str(#py_name), new_class).unwrap(); + } + } + _ => quote! { + for name in [#(#py_names,)*] { + vm.__module_set_attr(&module, vm.ctx.intern_str(name), new_class.clone()).unwrap(); + } + }, + }; + + args.context.attribute_items.add_item( + ident.clone(), + py_names, + args.cfgs.to_vec(), + quote_spanned! { ident.span() => + #class_new + #set_attr + }, + 0, + )?; + Ok(()) + } +} + +impl ModuleItem for StructSequenceItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { + // Get the struct identifier (this IS the Python type, e.g., PyStructTime) + let pytype_ident = match args.item { + Item::Struct(s) => s.ident.clone(), + other => bail_span!(other, "#[pystruct_sequence] can only be on a struct"), + }; + + // Parse the #[pystruct_sequence(name = "...", module = "...", no_attr)] attribute + let structseq_attr = &args.attrs[self.inner.index]; + let meta = PyStructSequenceMeta::from_attr(pytype_ident.clone(), structseq_attr)?; + + let class_name = meta.class_name()?.ok_or_else(|| { + syn::Error::new_spanned( + structseq_attr, + "#[pystruct_sequence] requires name parameter", + ) + })?; + let module_opt = meta.module()?; + let has_module = module_opt.is_some(); + let module_name = module_opt.unwrap_or_else(|| args.context.name.clone()); + if !has_module { + let structseq_attr = &mut args.attrs[self.inner.index]; + structseq_attr.fill_nested_meta("module", || { + parse_quote! {module = #module_name} + })?; + } + let no_attr = meta.no_attr()?; + + // Generate the class creation code + let class_new = quote_spanned!(pytype_ident.span() => + let new_class = <#pytype_ident as ::rustpython_vm::class::PyClassImpl>::make_static_type(); + { + let module_key = rustpython_vm::identifier!(ctx, __module__); + let has_module_getset = new_class.attributes.read() + .get(module_key) + .is_some_and(|v| v.downcastable::<rustpython_vm::builtins::PyGetSet>()); + if !has_module_getset { + new_class.set_attr(module_key, vm.new_pyobj(#module_name)); + } + } + ); + + // Handle py_attrs for custom names, or use class_name as default + let mut py_names = Vec::new(); + for attr_index in self.py_attrs.iter().rev() { + let attr_attr = args.attrs.remove(*attr_index); + let item_meta = SimpleItemMeta::from_attr(pytype_ident.clone(), &attr_attr)?; + let py_name = item_meta + .optional_name() + .unwrap_or_else(|| class_name.clone()); + py_names.push(py_name); + } + + // Require explicit #[pyattr] or no_attr, like #[pyclass] + if self.py_attrs.is_empty() && !no_attr { + bail_span!( + pytype_ident, + "#[pystruct_sequence] requires #[pyattr] to be a module attribute. \ + To keep it free type, try #[pystruct_sequence(..., no_attr)]" + ) + } + + let set_attr = match py_names.len() { + 0 => quote! { + let _ = new_class; // suppress warning + }, + 1 => { + let py_name = &py_names[0]; + quote! { + vm.__module_set_attr(&module, vm.ctx.intern_str(#py_name), new_class).unwrap(); + } + } + _ => quote! { + for name in [#(#py_names,)*] { + vm.__module_set_attr(&module, vm.ctx.intern_str(name), new_class.clone()).unwrap(); + } + }, + }; + + args.context.attribute_items.add_item( + pytype_ident.clone(), + py_names, + args.cfgs.to_vec(), + quote_spanned! { pytype_ident.span() => + #class_new + #set_attr + }, + 0, + )?; + Ok(()) + } +} + +impl ModuleItem for AttributeItem { + fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { + let cfgs = args.cfgs.to_vec(); + let attr = args.attrs.remove(self.index()); + let (ident, py_name, let_obj) = match args.item { + Item::Fn(syn::ItemFn { sig, block, .. }) => { + let ident = &sig.ident; + // If `once` keyword is in #[pyattr], + // wrapping it with static_cell for preventing it from using it as function + let attr_meta = AttrItemMeta::from_attr(ident.clone(), &attr)?; + if attr_meta.inner()._bool("once")? { + let stmts = &block.stmts; + let return_type = match &sig.output { + syn::ReturnType::Default => { + unreachable!("#[pyattr] attached function must have return type.") + } + syn::ReturnType::Type(_, ty) => ty, + }; + let stmt: syn::Stmt = parse_quote! { + { + rustpython_common::static_cell! { + static ERROR: #return_type; + } + ERROR + .get_or_init(|| { + #(#stmts)* + }) + .clone() + } + }; + block.stmts = vec![stmt]; + } + + let py_name = attr_meta.simple_name()?; + ( + ident.clone(), + py_name, + quote_spanned! { ident.span() => + let obj = vm.new_pyobj(#ident(vm)); + }, + ) + } + Item::Const(syn::ItemConst { ident, .. }) => { + let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr)?; + let py_name = item_meta.simple_name()?; + ( + ident.clone(), + py_name, + quote_spanned! { ident.span() => + let obj = vm.new_pyobj(#ident); + }, + ) + } + Item::Use(item) => { + if !self.py_attrs.is_empty() { + return Err(self + .new_syn_error(item.span(), "Only single #[pyattr] is allowed for `use`")); + } + let _ = iter_use_idents(item, |ident, is_unique| { + let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr)?; + let py_name = if is_unique { + item_meta.simple_name()? + } else if item_meta.optional_name().is_some() { + // this check actually doesn't need to be placed in loop + return Err(self.new_syn_error( + ident.span(), + "`name` attribute is not allowed for multiple use items", + )); + } else { + ident.to_string() + }; + let tokens = quote_spanned! { ident.span() => + vm.__module_set_attr(module, vm.ctx.intern_str(#py_name), vm.new_pyobj(#ident)).unwrap(); + }; + args.context.attribute_items.add_item( + ident.clone(), + vec![py_name], + cfgs.clone(), + tokens, + 1, + )?; + Ok(()) + })?; + return Ok(()); + } + other => { + return Err( + self.new_syn_error(other.span(), "can only be on a function, const and use") + ); + } + }; + + let (tokens, py_names) = if self.py_attrs.is_empty() { + ( + quote_spanned! { ident.span() => { + #let_obj + vm.__module_set_attr(module, vm.ctx.intern_str(#py_name), obj).unwrap(); + }}, + vec![py_name], + ) + } else { + let mut names = vec![py_name]; + for attr_index in self.py_attrs.iter().rev() { + let mut loop_unit = || { + let attr_attr = args.attrs.remove(*attr_index); + let item_meta = AttrItemMeta::from_attr(ident.clone(), &attr_attr)?; + if item_meta.inner()._bool("once")? { + return Err(self.new_syn_error( + ident.span(), + "#[pyattr(once)] is only allowed for the bottom-most item", + )); + } + + let py_name = item_meta.optional_name().ok_or_else(|| { + self.new_syn_error( + ident.span(), + "#[pyattr(name = ...)] is mandatory except for the bottom-most item", + ) + })?; + names.push(py_name); + Ok(()) + }; + let r = loop_unit(); + args.context.errors.ok_or_push(r); + } + ( + quote_spanned! { ident.span() => { + #let_obj + for name in [#(#names),*] { + vm.__module_set_attr(module, vm.ctx.intern_str(name), obj.clone()).unwrap(); + } + }}, + names, + ) + }; + + args.context + .attribute_items + .add_item(ident, py_names, cfgs, tokens, 1)?; + + Ok(()) + } +} diff --git a/derive-impl/src/pypayload.rs b/crates/derive-impl/src/pypayload.rs similarity index 95% rename from derive-impl/src/pypayload.rs rename to crates/derive-impl/src/pypayload.rs index a72c36148ad..eeae27c3fa4 100644 --- a/derive-impl/src/pypayload.rs +++ b/crates/derive-impl/src/pypayload.rs @@ -7,6 +7,7 @@ pub(crate) fn impl_pypayload(input: DeriveInput) -> Result<TokenStream> { let ret = quote! { impl ::rustpython_vm::PyPayload for #ty { + #[inline] fn class(_ctx: &::rustpython_vm::vm::Context) -> &'static rustpython_vm::Py<::rustpython_vm::builtins::PyType> { <Self as ::rustpython_vm::class::StaticType>::static_type() } diff --git a/crates/derive-impl/src/pystructseq.rs b/crates/derive-impl/src/pystructseq.rs new file mode 100644 index 00000000000..c59a1df2e31 --- /dev/null +++ b/crates/derive-impl/src/pystructseq.rs @@ -0,0 +1,646 @@ +use crate::util::{ItemMeta, ItemMetaInner}; +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; +use syn::{DeriveInput, Ident, Item, Result}; +use syn_ext::ext::{AttributeExt, GetIdent}; +use syn_ext::types::{Meta, PunctuatedNestedMeta}; + +// #[pystruct_sequence_data] - For Data structs + +/// Field kind for struct sequence +#[derive(Clone, Copy, PartialEq, Eq)] +enum FieldKind { + /// Named visible field (has getter, shown in repr) + Named, + /// Unnamed visible field (index-only, no getter) + Unnamed, + /// Hidden/skipped field (stored in tuple, but hidden from repr/len/index) + Skipped, +} + +/// Parsed field with its kind +struct ParsedField { + ident: Ident, + kind: FieldKind, + /// Optional cfg attributes for conditional compilation + cfg_attrs: Vec<syn::Attribute>, +} + +/// Parsed field info from struct +struct FieldInfo { + /// All fields in order with their kinds + fields: Vec<ParsedField>, +} + +impl FieldInfo { + fn named_fields(&self) -> Vec<&ParsedField> { + self.fields + .iter() + .filter(|f| f.kind == FieldKind::Named) + .collect() + } + + fn visible_fields(&self) -> Vec<&ParsedField> { + self.fields + .iter() + .filter(|f| f.kind != FieldKind::Skipped) + .collect() + } + + fn skipped_fields(&self) -> Vec<&ParsedField> { + self.fields + .iter() + .filter(|f| f.kind == FieldKind::Skipped) + .collect() + } + + fn n_unnamed_fields(&self) -> usize { + self.fields + .iter() + .filter(|f| f.kind == FieldKind::Unnamed) + .count() + } +} + +/// Parse field info from struct +fn parse_fields(input: &mut DeriveInput) -> Result<FieldInfo> { + let syn::Data::Struct(struc) = &mut input.data else { + bail_span!(input, "#[pystruct_sequence_data] can only be on a struct") + }; + + let syn::Fields::Named(fields) = &mut struc.fields else { + bail_span!( + input, + "#[pystruct_sequence_data] can only be on a struct with named fields" + ); + }; + + let mut parsed_fields = Vec::with_capacity(fields.named.len()); + + for field in &mut fields.named { + let mut skip = false; + let mut unnamed = false; + let mut attrs_to_remove = Vec::new(); + let mut cfg_attrs = Vec::new(); + + for (i, attr) in field.attrs.iter().enumerate() { + // Collect cfg attributes for conditional compilation + if attr.path().is_ident("cfg") { + cfg_attrs.push(attr.clone()); + continue; + } + + if !attr.path().is_ident("pystruct_sequence") { + continue; + } + + let Ok(meta) = attr.parse_meta() else { + continue; + }; + + let Meta::List(l) = meta else { + bail_span!(input, "Only #[pystruct_sequence(...)] form is allowed"); + }; + + let idents: Vec<_> = l + .nested + .iter() + .filter_map(|n| n.get_ident()) + .cloned() + .collect(); + + for ident in idents { + match ident.to_string().as_str() { + "skip" => { + skip = true; + } + "unnamed" => { + unnamed = true; + } + _ => { + bail_span!(ident, "Unknown item for #[pystruct_sequence(...)]") + } + } + } + + attrs_to_remove.push(i); + } + + // Remove attributes in reverse order + attrs_to_remove.sort_unstable_by(|a, b| b.cmp(a)); + for index in attrs_to_remove { + field.attrs.remove(index); + } + + let ident = field.ident.clone().unwrap(); + let kind = if skip { + FieldKind::Skipped + } else if unnamed { + FieldKind::Unnamed + } else { + FieldKind::Named + }; + + parsed_fields.push(ParsedField { + ident, + kind, + cfg_attrs, + }); + } + + Ok(FieldInfo { + fields: parsed_fields, + }) +} + +/// Check if `try_from_object` is present in attribute arguments +fn has_try_from_object(attr: &PunctuatedNestedMeta) -> bool { + attr.iter().any(|nested| { + nested + .get_ident() + .is_some_and(|ident| ident == "try_from_object") + }) +} + +/// Attribute macro for Data structs: #[pystruct_sequence_data(...)] +/// +/// Generates: +/// - `REQUIRED_FIELD_NAMES` constant (named visible fields) +/// - `OPTIONAL_FIELD_NAMES` constant (hidden/skipped fields) +/// - `UNNAMED_FIELDS_LEN` constant +/// - `into_tuple()` method +/// - Field index constants (e.g., `TM_YEAR_INDEX`) +/// +/// Options: +/// - `try_from_object`: Generate `try_from_elements()` method and `TryFromObject` impl +pub(crate) fn impl_pystruct_sequence_data( + attr: PunctuatedNestedMeta, + item: Item, +) -> Result<TokenStream> { + let Item::Struct(item_struct) = item else { + bail_span!( + item, + "#[pystruct_sequence_data] can only be applied to structs" + ); + }; + + let try_from_object = has_try_from_object(&attr); + let mut input: DeriveInput = DeriveInput { + attrs: item_struct.attrs.clone(), + vis: item_struct.vis.clone(), + ident: item_struct.ident.clone(), + generics: item_struct.generics.clone(), + data: syn::Data::Struct(syn::DataStruct { + struct_token: item_struct.struct_token, + fields: item_struct.fields.clone(), + semi_token: item_struct.semi_token, + }), + }; + let field_info = parse_fields(&mut input)?; + let data_ident = &input.ident; + + let named_fields = field_info.named_fields(); + let visible_fields = field_info.visible_fields(); + let skipped_fields = field_info.skipped_fields(); + let n_unnamed_fields = field_info.n_unnamed_fields(); + + // Generate field index constants for visible fields (with cfg guards) + let field_indices: Vec<_> = visible_fields + .iter() + .enumerate() + .map(|(i, field)| { + let const_name = format_ident!("{}_INDEX", field.ident.to_string().to_uppercase()); + let cfg_attrs = &field.cfg_attrs; + quote! { + #(#cfg_attrs)* + pub const #const_name: usize = #i; + } + }) + .collect(); + + // Generate field name entries with cfg guards for named fields + let named_field_names: Vec<_> = named_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { stringify!(#ident), } + } else { + quote! { + #(#cfg_attrs)* + { stringify!(#ident) }, + } + } + }) + .collect(); + + // Generate field name entries with cfg guards for skipped fields + let skipped_field_names: Vec<_> = skipped_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { stringify!(#ident), } + } else { + quote! { + #(#cfg_attrs)* + { stringify!(#ident) }, + } + } + }) + .collect(); + + // Generate into_tuple items with cfg guards + let visible_tuple_items: Vec<_> = visible_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { + ::rustpython_vm::convert::ToPyObject::to_pyobject(self.#ident, vm), + } + } else { + quote! { + #(#cfg_attrs)* + { ::rustpython_vm::convert::ToPyObject::to_pyobject(self.#ident, vm) }, + } + } + }) + .collect(); + + let skipped_tuple_items: Vec<_> = skipped_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { + ::rustpython_vm::convert::ToPyObject::to_pyobject(self.#ident, vm), + } + } else { + quote! { + #(#cfg_attrs)* + { ::rustpython_vm::convert::ToPyObject::to_pyobject(self.#ident, vm) }, + } + } + }) + .collect(); + + // Generate TryFromObject impl only when try_from_object=true + let try_from_object_impl = if try_from_object { + let n_required = visible_fields.len(); + quote! { + impl ::rustpython_vm::TryFromObject for #data_ident { + fn try_from_object( + vm: &::rustpython_vm::VirtualMachine, + obj: ::rustpython_vm::PyObjectRef, + ) -> ::rustpython_vm::PyResult<Self> { + let seq: Vec<::rustpython_vm::PyObjectRef> = obj.try_into_value(vm)?; + if seq.len() != #n_required { + return Err(vm.new_type_error(format!( + "{} requires a {}-sequence ({}-sequence given)", + stringify!(#data_ident), + #n_required, + seq.len() + ))); + } + <Self as ::rustpython_vm::types::PyStructSequenceData>::try_from_elements(seq, vm) + } + } + } + } else { + quote! {} + }; + + // Generate try_from_elements trait override only when try_from_object=true + let try_from_elements_trait_override = if try_from_object { + let visible_field_inits: Vec<_> = visible_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { #ident: iter.next().unwrap().clone().try_into_value(vm)?, } + } else { + quote! { + #(#cfg_attrs)* + #ident: iter.next().unwrap().clone().try_into_value(vm)?, + } + } + }) + .collect(); + let skipped_field_inits: Vec<_> = skipped_fields + .iter() + .map(|f| { + let ident = &f.ident; + let cfg_attrs = &f.cfg_attrs; + if cfg_attrs.is_empty() { + quote! { + #ident: match iter.next() { + Some(v) => v.clone().try_into_value(vm)?, + None => vm.ctx.none(), + }, + } + } else { + quote! { + #(#cfg_attrs)* + #ident: match iter.next() { + Some(v) => v.clone().try_into_value(vm)?, + None => vm.ctx.none(), + }, + } + } + }) + .collect(); + quote! { + fn try_from_elements( + elements: Vec<::rustpython_vm::PyObjectRef>, + vm: &::rustpython_vm::VirtualMachine, + ) -> ::rustpython_vm::PyResult<Self> { + let mut iter = elements.into_iter(); + Ok(Self { + #(#visible_field_inits)* + #(#skipped_field_inits)* + }) + } + } + } else { + quote! {} + }; + + let output = quote! { + impl #data_ident { + #(#field_indices)* + } + + // PyStructSequenceData trait impl + impl ::rustpython_vm::types::PyStructSequenceData for #data_ident { + const REQUIRED_FIELD_NAMES: &'static [&'static str] = &[#(#named_field_names)*]; + const OPTIONAL_FIELD_NAMES: &'static [&'static str] = &[#(#skipped_field_names)*]; + const UNNAMED_FIELDS_LEN: usize = #n_unnamed_fields; + + fn into_tuple(self, vm: &::rustpython_vm::VirtualMachine) -> ::rustpython_vm::builtins::PyTuple { + let items = vec![ + #(#visible_tuple_items)* + #(#skipped_tuple_items)* + ]; + ::rustpython_vm::builtins::PyTuple::new_unchecked(items.into_boxed_slice()) + } + + #try_from_elements_trait_override + } + + #try_from_object_impl + }; + + // For attribute macro, we need to output the original struct as well + // But first, strip #[pystruct_sequence] attributes from fields + let mut clean_struct = item_struct.clone(); + if let syn::Fields::Named(ref mut fields) = clean_struct.fields { + for field in &mut fields.named { + field + .attrs + .retain(|attr| !attr.path().is_ident("pystruct_sequence")); + } + } + + Ok(quote! { + #clean_struct + #output + }) +} + +// #[pystruct_sequence(...)] - For Python type structs + +/// Meta parser for #[pystruct_sequence(...)] +pub(crate) struct PyStructSequenceMeta { + inner: ItemMetaInner, +} + +impl ItemMeta for PyStructSequenceMeta { + const ALLOWED_NAMES: &'static [&'static str] = &["name", "module", "data", "no_attr"]; + + fn from_inner(inner: ItemMetaInner) -> Self { + Self { inner } + } + fn inner(&self) -> &ItemMetaInner { + &self.inner + } +} + +impl PyStructSequenceMeta { + pub fn class_name(&self) -> Result<Option<String>> { + const KEY: &str = "name"; + let inner = self.inner(); + if let Some((_, meta)) = inner.meta_map.get(KEY) { + if let Meta::NameValue(syn::MetaNameValue { + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), + .. + }) = meta + { + return Ok(Some(lit.value())); + } + bail_span!( + inner.meta_ident, + "#[pystruct_sequence({KEY}=value)] expects a string value" + ) + } else { + Ok(None) + } + } + + pub fn module(&self) -> Result<Option<String>> { + const KEY: &str = "module"; + let inner = self.inner(); + if let Some((_, meta)) = inner.meta_map.get(KEY) { + if let Meta::NameValue(syn::MetaNameValue { + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), + .. + }) = meta + { + return Ok(Some(lit.value())); + } + bail_span!( + inner.meta_ident, + "#[pystruct_sequence({KEY}=value)] expects a string value" + ) + } else { + Ok(None) + } + } + + fn data_type(&self) -> Result<Ident> { + const KEY: &str = "data"; + let inner = self.inner(); + if let Some((_, meta)) = inner.meta_map.get(KEY) { + if let Meta::NameValue(syn::MetaNameValue { + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), + .. + }) = meta + { + return Ok(format_ident!("{}", lit.value())); + } + bail_span!( + inner.meta_ident, + "#[pystruct_sequence({KEY}=value)] expects a string value" + ) + } else { + bail_span!( + inner.meta_ident, + "#[pystruct_sequence] requires data parameter (e.g., data = \"DataStructName\")" + ) + } + } + + pub fn no_attr(&self) -> Result<bool> { + self.inner()._bool("no_attr") + } +} + +/// Attribute macro for struct sequences. +/// +/// Usage: +/// ```ignore +/// #[pystruct_sequence_data] +/// struct StructTimeData { ... } +/// +/// #[pystruct_sequence(name = "struct_time", module = "time", data = "StructTimeData")] +/// struct PyStructTime; +/// ``` +pub(crate) fn impl_pystruct_sequence( + attr: PunctuatedNestedMeta, + item: Item, +) -> Result<TokenStream> { + let Item::Struct(struct_item) = item else { + bail_span!(item, "#[pystruct_sequence] can only be applied to a struct"); + }; + + let ident = struct_item.ident.clone(); + let fake_ident = Ident::new("pystruct_sequence", ident.span()); + let meta = PyStructSequenceMeta::from_nested(ident, fake_ident, attr.into_iter())?; + + let pytype_ident = struct_item.ident.clone(); + let pytype_vis = struct_item.vis.clone(); + let data_ident = meta.data_type()?; + + let class_name = meta.class_name()?.ok_or_else(|| { + syn::Error::new_spanned( + &struct_item.ident, + "#[pystruct_sequence] requires name parameter", + ) + })?; + let module_name = meta.module()?; + + // Module name handling + let module_name_tokens = match &module_name { + Some(m) => quote!(Some(#m)), + None => quote!(None), + }; + + let module_class_name = if let Some(ref m) = module_name { + format!("{}.{}", m, class_name) + } else { + class_name.clone() + }; + + let output = quote! { + // The Python type struct - newtype wrapping PyTuple + #[derive(Debug)] + #[repr(transparent)] + #pytype_vis struct #pytype_ident(pub ::rustpython_vm::builtins::PyTuple); + + // PyClassDef for Python type + impl ::rustpython_vm::class::PyClassDef for #pytype_ident { + const NAME: &'static str = #class_name; + const MODULE_NAME: Option<&'static str> = #module_name_tokens; + const TP_NAME: &'static str = #module_class_name; + const DOC: Option<&'static str> = None; + const BASICSIZE: usize = 0; + const UNHASHABLE: bool = false; + + type Base = ::rustpython_vm::builtins::PyTuple; + } + + // StaticType for Python type + impl ::rustpython_vm::class::StaticType for #pytype_ident { + fn static_cell() -> &'static ::rustpython_vm::common::static_cell::StaticCell<::rustpython_vm::builtins::PyTypeRef> { + ::rustpython_vm::common::static_cell! { + static CELL: ::rustpython_vm::builtins::PyTypeRef; + } + &CELL + } + + fn static_baseclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + use ::rustpython_vm::class::StaticType; + ::rustpython_vm::builtins::PyTuple::static_type() + } + } + + // Subtype uses base type's payload_type_id + impl ::rustpython_vm::PyPayload for #pytype_ident { + const PAYLOAD_TYPE_ID: ::core::any::TypeId = <::rustpython_vm::builtins::PyTuple as ::rustpython_vm::PyPayload>::PAYLOAD_TYPE_ID; + + #[inline] + unsafe fn validate_downcastable_from(obj: &::rustpython_vm::PyObject) -> bool { + obj.class().fast_issubclass(<Self as ::rustpython_vm::class::StaticType>::static_type()) + } + + fn class(_ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { + <Self as ::rustpython_vm::class::StaticType>::static_type() + } + } + + // MaybeTraverse - delegate to inner PyTuple + impl ::rustpython_vm::object::MaybeTraverse for #pytype_ident { + const HAS_TRAVERSE: bool = true; + const HAS_CLEAR: bool = true; + + fn try_traverse(&self, traverse_fn: &mut ::rustpython_vm::object::TraverseFn<'_>) { + self.0.try_traverse(traverse_fn) + } + + fn try_clear(&mut self, out: &mut ::std::vec::Vec<::rustpython_vm::PyObjectRef>) { + self.0.try_clear(out) + } + } + + // PySubclass for proper inheritance + impl ::rustpython_vm::class::PySubclass for #pytype_ident { + type Base = ::rustpython_vm::builtins::PyTuple; + + #[inline] + fn as_base(&self) -> &Self::Base { + &self.0 + } + } + + // PyStructSequence trait for Python type + impl ::rustpython_vm::types::PyStructSequence for #pytype_ident { + type Data = #data_ident; + } + + // ToPyObject for Data struct - uses PyStructSequence::from_data + impl ::rustpython_vm::convert::ToPyObject for #data_ident { + fn to_pyobject(self, vm: &::rustpython_vm::VirtualMachine) -> ::rustpython_vm::PyObjectRef { + <#pytype_ident as ::rustpython_vm::types::PyStructSequence>::from_data(self, vm).into() + } + } + }; + + Ok(output) +} diff --git a/crates/derive-impl/src/pytraverse.rs b/crates/derive-impl/src/pytraverse.rs new file mode 100644 index 00000000000..c4ec3823298 --- /dev/null +++ b/crates/derive-impl/src/pytraverse.rs @@ -0,0 +1,127 @@ +use proc_macro2::TokenStream; +use quote::quote; +use syn::{Attribute, DeriveInput, Field, Result}; + +struct TraverseAttr { + /// set to `true` if the attribute is `#[pytraverse(skip)]` + skip: bool, +} + +const ATTR_TRAVERSE: &str = "pytraverse"; + +/// only accept `#[pytraverse(skip)]` for now +fn pytraverse_arg(attr: &Attribute) -> Option<Result<TraverseAttr>> { + if !attr.path().is_ident(ATTR_TRAVERSE) { + return None; + } + let ret = || { + let mut skip = false; + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("skip") { + if skip { + return Err(meta.error("already specified skip")); + } + skip = true; + } else { + return Err(meta.error("unknown attr")); + } + Ok(()) + })?; + Ok(TraverseAttr { skip }) + }; + Some(ret()) +} + +fn field_to_traverse_code(field: &Field) -> Result<TokenStream> { + let pytraverse_attrs = field + .attrs + .iter() + .filter_map(pytraverse_arg) + .collect::<core::result::Result<Vec<_>, _>>()?; + let do_trace = if pytraverse_attrs.len() > 1 { + bail_span!( + field, + "found multiple #[pytraverse] attributes on the same field, expect at most one" + ) + } else if pytraverse_attrs.is_empty() { + // default to always traverse every field + true + } else { + !pytraverse_attrs[0].skip + }; + let name = field.ident.as_ref().ok_or_else(|| { + syn::Error::new_spanned( + field.clone(), + "Field should have a name in non-tuple struct", + ) + })?; + if do_trace { + Ok(quote!( + ::rustpython_vm::object::Traverse::traverse(&self.#name, tracer_fn); + )) + } else { + Ok(quote!()) + } +} + +/// not trace corresponding field +fn gen_trace_code(item: &mut DeriveInput) -> Result<TokenStream> { + match &mut item.data { + syn::Data::Struct(s) => { + let fields = &mut s.fields; + match fields { + syn::Fields::Named(fields) => { + let res: Vec<TokenStream> = fields + .named + .iter_mut() + .map(|f| -> Result<TokenStream> { field_to_traverse_code(f) }) + .collect::<Result<_>>()?; + let res = res.into_iter().collect::<TokenStream>(); + Ok(res) + } + syn::Fields::Unnamed(fields) => { + let res: TokenStream = (0..fields.unnamed.len()) + .map(|i| { + let i = syn::Index::from(i); + quote!( + ::rustpython_vm::object::Traverse::traverse(&self.#i, tracer_fn); + ) + }) + .collect(); + Ok(res) + } + _ => Err(syn::Error::new_spanned( + fields, + "Only named and unnamed fields are supported", + )), + } + } + _ => Err(syn::Error::new_spanned(item, "Only structs are supported")), + } +} + +pub(crate) fn impl_pytraverse(mut item: DeriveInput) -> Result<TokenStream> { + let trace_code = gen_trace_code(&mut item)?; + + let ty = &item.ident; + + // Add Traverse bound to all type parameters + for param in &mut item.generics.params { + if let syn::GenericParam::Type(type_param) = param { + type_param + .bounds + .push(syn::parse_quote!(::rustpython_vm::object::Traverse)); + } + } + + let (impl_generics, ty_generics, where_clause) = item.generics.split_for_impl(); + + let ret = quote! { + unsafe impl #impl_generics ::rustpython_vm::object::Traverse for #ty #ty_generics #where_clause { + fn traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { + #trace_code + } + } + }; + Ok(ret) +} diff --git a/derive-impl/src/util.rs b/crates/derive-impl/src/util.rs similarity index 77% rename from derive-impl/src/util.rs rename to crates/derive-impl/src/util.rs index 916b19db064..068bde9bccd 100644 --- a/derive-impl/src/util.rs +++ b/crates/derive-impl/src/util.rs @@ -1,13 +1,11 @@ use itertools::Itertools; use proc_macro2::{Span, TokenStream}; -use quote::{quote, ToTokens}; +use quote::{ToTokens, quote}; use std::collections::{HashMap, HashSet}; -use syn::{ - spanned::Spanned, Attribute, Ident, Meta, MetaList, NestedMeta, Result, Signature, UseTree, -}; +use syn::{Attribute, Ident, Result, Signature, UseTree, spanned::Spanned}; use syn_ext::{ ext::{AttributeExt as SynAttributeExt, *}, - types::PunctuatedNestedMeta, + types::*, }; pub(crate) const ALL_ALLOWED_NAMES: &[&str] = &[ @@ -77,8 +75,8 @@ impl ItemNursery { impl ToTokens for ValidatedItemNursery { fn to_tokens(&self, tokens: &mut TokenStream) { - let mut sorted = self.0 .0.clone(); - sorted.sort_by(|a, b| a.sort_order.cmp(&b.sort_order)); + let mut sorted = self.0.0.clone(); + sorted.sort_by_key(|a| a.sort_order); tokens.extend(sorted.iter().map(|item| { let cfgs = &item.cfgs; let tokens = &item.tokens; @@ -99,7 +97,7 @@ pub(crate) struct ContentItemInner<T> { } pub(crate) trait ContentItem { - type AttrName: std::str::FromStr + std::fmt::Display; + type AttrName: core::str::FromStr + core::fmt::Display; fn inner(&self) -> &ContentItemInner<Self::AttrName>; fn index(&self) -> usize { @@ -127,7 +125,7 @@ impl ItemMetaInner { allowed_names: &[&'static str], ) -> Result<Self> where - I: std::iter::Iterator<Item = NestedMeta>, + I: core::iter::Iterator<Item = NestedMeta>, { let (meta_map, lits) = nested.into_unique_map_and_lits(|path| { if let Some(ident) = path.get_ident() { @@ -167,7 +165,11 @@ impl ItemMetaInner { pub fn _optional_str(&self, key: &str) -> Result<Option<String>> { let value = if let Some((_, meta)) = self.meta_map.get(key) { let Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Str(lit), + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), .. }) = meta else { @@ -185,6 +187,35 @@ impl ItemMetaInner { Ok(value) } + pub fn _optional_path(&self, key: &str) -> Result<Option<syn::Path>> { + let value = if let Some((_, meta)) = self.meta_map.get(key) { + let Meta::NameValue(syn::MetaNameValue { value, .. }) = meta else { + bail_span!( + meta, + "#[{}({} = ...)] must be a name-value pair", + self.meta_name(), + key + ) + }; + + // Try to parse as a Path (identifier or path like Foo or foo::Bar) + match syn::parse2::<syn::Path>(value.to_token_stream()) { + Ok(path) => Some(path), + Err(_) => { + bail_span!( + value, + "#[{}({} = ...)] must be a valid type path (e.g., PyBaseException)", + self.meta_name(), + key + ) + } + } + } else { + None + }; + Ok(value) + } + pub fn _has_key(&self, key: &str) -> Result<bool> { Ok(matches!(self.meta_map.get(key), Some((_, _)))) } @@ -193,7 +224,11 @@ impl ItemMetaInner { let value = if let Some((_, meta)) = self.meta_map.get(key) { match meta { Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Bool(lit), + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Bool(lit), + .. + }), .. }) => lit.value, Meta::Path(_) => true, @@ -208,9 +243,9 @@ impl ItemMetaInner { pub fn _optional_list( &self, key: &str, - ) -> Result<Option<impl std::iter::Iterator<Item = &'_ NestedMeta>>> { + ) -> Result<Option<impl core::iter::Iterator<Item = &'_ NestedMeta>>> { let value = if let Some((_, meta)) = self.meta_map.get(key) { - let Meta::List(syn::MetaList { + let Meta::List(MetaList { path: _, nested, .. }) = meta else { @@ -234,7 +269,7 @@ pub(crate) trait ItemMeta: Sized { fn from_nested<I>(item_ident: Ident, meta_ident: Ident, nested: I) -> Result<Self> where - I: std::iter::Iterator<Item = NestedMeta>, + I: core::iter::Iterator<Item = NestedMeta>, { Ok(Self::from_inner(ItemMetaInner::from_nested( item_ident, @@ -271,6 +306,7 @@ impl ItemMeta for SimpleItemMeta { fn from_inner(inner: ItemMetaInner) -> Self { Self(inner) } + fn inner(&self) -> &ItemMetaInner { &self.0 } @@ -279,11 +315,12 @@ impl ItemMeta for SimpleItemMeta { pub(crate) struct ModuleItemMeta(pub ItemMetaInner); impl ItemMeta for ModuleItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["name", "with", "sub"]; + const ALLOWED_NAMES: &'static [&'static str] = &["name", "sub"]; fn from_inner(inner: ItemMetaInner) -> Self { Self(inner) } + fn inner(&self) -> &ItemMetaInner { &self.0 } @@ -293,19 +330,6 @@ impl ModuleItemMeta { pub fn sub(&self) -> Result<bool> { self.inner()._bool("sub") } - pub fn with(&self) -> Result<Vec<&syn::Path>> { - let mut withs = Vec::new(); - let Some(nested) = self.inner()._optional_list("with")? else { - return Ok(withs); - }; - for meta in nested { - let NestedMeta::Meta(Meta::Path(path)) = meta else { - bail_span!(meta, "#[pymodule(with(...))] arguments should be paths") - }; - withs.push(path); - } - Ok(withs) - } } pub(crate) struct AttrItemMeta(pub ItemMetaInner); @@ -316,6 +340,7 @@ impl ItemMeta for AttrItemMeta { fn from_inner(inner: ItemMetaInner) -> Self { Self(inner) } + fn inner(&self) -> &ItemMetaInner { &self.0 } @@ -333,11 +358,13 @@ impl ItemMeta for ClassItemMeta { "ctx", "impl", "traverse", + "clear", // tp_clear ]; fn from_inner(inner: ItemMetaInner) -> Self { Self(inner) } + fn inner(&self) -> &ItemMetaInner { &self.0 } @@ -350,7 +377,11 @@ impl ClassItemMeta { if let Some((_, meta)) = inner.meta_map.get(KEY) { match meta { Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Str(lit), + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), .. }) => return Ok(lit.value()), Meta::Path(_) => return Ok(inner.item_name()), @@ -369,8 +400,8 @@ impl ClassItemMeta { self.inner()._optional_str("ctx") } - pub fn base(&self) -> Result<Option<String>> { - self.inner()._optional_str("base") + pub fn base(&self) -> Result<Option<syn::Path>> { + self.inner()._optional_path("base") } pub fn unhashable(&self) -> Result<bool> { @@ -387,11 +418,11 @@ impl ClassItemMeta { let value = if let Some((_, meta)) = inner.meta_map.get(KEY) { match meta { Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Str(lit), + value: syn::Expr::Lit(syn::ExprLit{lit:syn::Lit::Str(lit),..}), .. }) => Ok(Some(lit.value())), Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Bool(lit), + value: syn::Expr::Lit(syn::ExprLit{lit:syn::Lit::Bool(lit),..}), .. }) => if lit.value { Err(lit.span()) @@ -431,13 +462,15 @@ impl ClassItemMeta { pub(crate) struct ExceptionItemMeta(ClassItemMeta); impl ItemMeta for ExceptionItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["name", "base", "unhashable", "ctx", "impl"]; + const ALLOWED_NAMES: &'static [&'static str] = + &["module", "name", "base", "unhashable", "ctx", "impl"]; fn from_inner(inner: ItemMetaInner) -> Self { Self(ClassItemMeta(inner)) } + fn inner(&self) -> &ItemMetaInner { - &self.0 .0 + &self.0.0 } } @@ -448,7 +481,11 @@ impl ExceptionItemMeta { if let Some((_, meta)) = inner.meta_map.get(KEY) { match meta { Meta::NameValue(syn::MetaNameValue { - lit: syn::Lit::Str(lit), + value: + syn::Expr::Lit(syn::ExprLit { + lit: syn::Lit::Str(lit), + .. + }), .. }) => return Ok(lit.value()), Meta::Path(_) => { @@ -456,12 +493,12 @@ impl ExceptionItemMeta { let type_name = inner.item_name(); let Some(py_name) = type_name.as_str().strip_prefix("Py") else { bail_span!( - inner.item_ident, - "#[pyexception] expects its underlying type to be named `Py` prefixed" - ) + inner.item_ident, + "#[pyexception] expects its underlying type to be named `Py` prefixed" + ) }; py_name.to_string() - }) + }); } _ => {} } @@ -479,7 +516,7 @@ impl ExceptionItemMeta { } } -impl std::ops::Deref for ExceptionItemMeta { +impl core::ops::Deref for ExceptionItemMeta { type Target = ClassItemMeta; fn deref(&self) -> &Self::Target { &self.0 @@ -489,7 +526,7 @@ impl std::ops::Deref for ExceptionItemMeta { pub(crate) trait AttributeExt: SynAttributeExt { fn promoted_nested(&self) -> Result<PunctuatedNestedMeta>; fn ident_and_promoted_nested(&self) -> Result<(&Ident, PunctuatedNestedMeta)>; - fn try_remove_name(&mut self, name: &str) -> Result<Option<syn::NestedMeta>>; + fn try_remove_name(&mut self, name: &str) -> Result<Option<NestedMeta>>; fn fill_nested_meta<F>(&mut self, name: &str, new_item: F) -> Result<()> where F: Fn() -> NestedMeta; @@ -501,8 +538,8 @@ impl AttributeExt for Attribute { let name = self.get_ident().unwrap().to_string(); e.combine(err_span!( self, - "#[{name} = \"...\"] cannot be a name/value, you probably meant \ - #[{name}(name = \"...\")]", + r##"#[{name} = "..."] cannot be a name/value, you probably meant \ + #[{name}(name = "...")]"##, )); e })?; @@ -512,10 +549,10 @@ impl AttributeExt for Attribute { Ok((self.get_ident().unwrap(), self.promoted_nested()?)) } - fn try_remove_name(&mut self, item_name: &str) -> Result<Option<syn::NestedMeta>> { + fn try_remove_name(&mut self, item_name: &str) -> Result<Option<NestedMeta>> { self.try_meta_mut(|meta| { let nested = match meta { - Meta::List(MetaList { ref mut nested, .. }) => Ok(nested), + Meta::List(MetaList { nested, .. }) => Ok(nested), other => Err(syn::Error::new( other.span(), format!( @@ -558,7 +595,7 @@ impl AttributeExt for Attribute { let has_name = list .nested .iter() - .any(|nested_meta| nested_meta.get_path().map_or(false, |p| p.is_ident(name))); + .any(|nested_meta| nested_meta.get_path().is_some_and(|p| p.is_ident(name))); if !has_name { list.nested.push(new_item()) } @@ -606,6 +643,7 @@ pub(crate) fn pyexception_ident_and_attrs(item: &syn::Item) -> Result<(&Ident, & pub(crate) trait ErrorVec: Sized { fn into_error(self) -> Option<syn::Error>; + fn into_result(self) -> Result<()> { if let Some(error) = self.into_error() { Err(error) @@ -613,6 +651,7 @@ pub(crate) trait ErrorVec: Sized { Ok(()) } } + fn ok_or_push<T>(&mut self, r: Result<T>) -> Option<T>; } @@ -628,6 +667,7 @@ impl ErrorVec for Vec<syn::Error> { None } } + fn ok_or_push<T>(&mut self, r: Result<T>) -> Option<T> { match r { Ok(v) => Some(v), @@ -692,6 +732,77 @@ pub(crate) fn text_signature(sig: &Signature, name: &str) -> String { } } +pub(crate) fn infer_native_call_flags(sig: &Signature, drop_first_typed: usize) -> TokenStream { + // Best-effort mapping of Rust function signatures to CPython-style + // METH_* calling convention flags used by CALL specialization. + let mut typed_args = Vec::new(); + for arg in &sig.inputs { + let syn::FnArg::Typed(typed) = arg else { + continue; + }; + let ty_tokens = &typed.ty; + let ty = quote!(#ty_tokens).to_string().replace(' ', ""); + // `vm: &VirtualMachine` is not a Python-level argument. + if ty.starts_with('&') && ty.ends_with("VirtualMachine") { + continue; + } + typed_args.push(ty); + } + + let mut user_args = typed_args.into_iter(); + for _ in 0..drop_first_typed { + if user_args.next().is_none() { + break; + } + } + + let mut has_keywords = false; + let mut variable_arity = false; + let mut fixed_positional = 0usize; + + for ty in user_args { + let is_named = |name: &str| { + ty == name + || ty.starts_with(&format!("{name}<")) + || ty.contains(&format!("::{name}<")) + || ty.ends_with(&format!("::{name}")) + }; + + if is_named("FuncArgs") { + has_keywords = true; + variable_arity = true; + continue; + } + if is_named("KwArgs") { + has_keywords = true; + variable_arity = true; + continue; + } + if is_named("PosArgs") || is_named("OptionalArg") || is_named("OptionalOption") { + variable_arity = true; + continue; + } + fixed_positional += 1; + } + + if has_keywords { + quote! { + rustpython_vm::function::PyMethodFlags::from_bits_retain( + rustpython_vm::function::PyMethodFlags::FASTCALL.bits() + | rustpython_vm::function::PyMethodFlags::KEYWORDS.bits() + ) + } + } else if variable_arity { + quote! { rustpython_vm::function::PyMethodFlags::FASTCALL } + } else { + match fixed_positional { + 0 => quote! { rustpython_vm::function::PyMethodFlags::NOARGS }, + 1 => quote! { rustpython_vm::function::PyMethodFlags::O }, + _ => quote! { rustpython_vm::function::PyMethodFlags::FASTCALL }, + } + } +} + fn func_sig(sig: &Signature) -> String { sig.inputs .iter() @@ -718,7 +829,7 @@ fn func_sig(sig: &Signature) -> String { return Some("$self".to_owned()); } if ident == "vm" { - unreachable!("type &VirtualMachine(`{}`) must be filtered already", ty); + unreachable!("type &VirtualMachine(`{ty}`) must be filtered already"); } Some(ident) }) diff --git a/crates/derive/Cargo.toml b/crates/derive/Cargo.toml new file mode 100644 index 00000000000..5ecd490cf94 --- /dev/null +++ b/crates/derive/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rustpython-derive" +description = "Rust language extensions and macros specific to rustpython." +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[lib] +proc-macro = true + +[dependencies] +rustpython-compiler = { workspace = true } +rustpython-derive-impl = { workspace = true } +syn = { workspace = true } + +[lints] +workspace = true diff --git a/crates/derive/src/lib.rs b/crates/derive/src/lib.rs new file mode 100644 index 00000000000..224aad4ea3c --- /dev/null +++ b/crates/derive/src/lib.rs @@ -0,0 +1,321 @@ +#![recursion_limit = "128"] +#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] +#![doc(html_root_url = "https://docs.rs/rustpython-derive/")] + +use proc_macro::TokenStream; +use rustpython_derive_impl as derive_impl; +use syn::parse_macro_input; +use syn::punctuated::Punctuated; + +#[proc_macro_derive(FromArgs, attributes(pyarg))] +pub fn derive_from_args(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + derive_impl::derive_from_args(input).into() +} + +/// The attribute can be applied either to a struct, trait, or impl. +/// # Struct +/// This implements `MaybeTraverse`, `PyClassDef`, and `StaticType` for the struct. +/// Consider deriving `Traverse` to implement it. +/// ## Arguments +/// - `module`: the module which contains the class -- can be omitted if in a `#[pymodule]`. +/// - `name`: the name of the Python class, by default it is the name of the struct. +/// - `base`: the base class of the Python class. +/// This does not cause inheritance of functions or attributes that must be done by a separate trait. +/// # Impl +/// This part implements `PyClassImpl` for the struct. +/// This includes methods, getters/setters, etc.; only annotated methods will be included. +/// Common functions and abilities like instantiation and `__call__` are often implemented by +/// traits rather than in the `impl` itself; see `Constructor` and `Callable` respectively for those. +/// ## Arguments +/// - `name`: the name of the Python class, when no name is provided the struct name is used. +/// - `flags`: the flags of the class, see `PyTypeFlags`. +/// - `BASETYPE`: allows the class to be inheritable. +/// - `IMMUTABLETYPE`: class attributes are immutable. +/// - `with`: which trait implementations are to be included in the python class. +/// ```rust, ignore +/// #[pyclass(module = "my_module", name = "MyClass", base = BaseClass)] +/// struct MyStruct { +/// x: i32, +/// } +/// +/// impl Constructor for MyStruct { +/// ... +/// } +/// +/// #[pyclass(with(Constructor))] +/// impl MyStruct { +/// ... +/// } +/// ``` +/// ## Inner markers +/// ### pymethod/pyclassmethod/pystaticmethod +/// `pymethod` is used to mark a method of the Python class. +/// `pyclassmethod` is used to mark a class method. +/// `pystaticmethod` is used to mark a static method. +/// #### Method signature +/// The first parameter can be either `&self` or `<var>: PyRef<Self>` for `pymethod`. +/// The first parameter can be `cls: PyTypeRef` for `pyclassmethod`. +/// There is no mandatory parameter for `pystaticmethod`. +/// Both are valid and essentially the same, but the latter can yield more control. +/// The last parameter can optionally be of the type `&VirtualMachine` to access the VM. +/// All other values must implement `IntoPyResult`. +/// Numeric types, `String`, `bool`, and `PyObjectRef` implement this trait, +/// but so does any object that implements `PyValue`. +/// Consider using `OptionalArg` for optional arguments. +/// #### Arguments +/// - `name`: the name of the method in Python, +/// by default it is the same as the Rust method, or surrounded by double underscores if magic is present. +/// This overrides `magic` and the default name and cannot be used with `magic` to prevent ambiguity. +/// ### pygetset +/// This is used to mark a getter/setter pair. +/// #### Arguments +/// - `setter`: marks the method as a setter, it acts as a getter by default. +/// Setter method names should be prefixed with `set_`. +/// - `name`: the name of the attribute in Python, by default it is the same as the Rust method. +/// - `magic`: marks the method as a magic method: the method name is surrounded with double underscores. +/// This cannot be used with `name` to prevent ambiguity. +/// +/// Ensure both the getter and setter are marked with `name` and `magic` in the same manner. +/// #### Examples +/// ```rust, ignore +/// #[pyclass] +/// impl MyStruct { +/// #[pygetset] +/// fn x(&self) -> PyResult<i32> { +/// Ok(self.x.lock()) +/// } +/// #[pygetset(setter)] +/// fn set_x(&mut self, value: i32) -> PyResult<()> { +/// self.x.set(value); +/// Ok(()) +/// } +/// } +/// ``` +/// ### pyslot +/// This is used to mark a slot method it should be marked by prefixing the method in rust with `slot_`. +/// #### Arguments +/// - name: the name of the slot method. +/// ### pyattr +/// ### extend_class +/// This helps inherit attributes from a parent class. +/// The method this is applied on should be called `extend_class_with_fields`. +/// #### Examples +/// ```rust, ignore +/// #[extend_class] +/// fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { +/// class.set_attr( +/// identifier!(ctx, _fields), +/// ctx.new_tuple(vec![ +/// ctx.new_str(ascii!("body")).into(), +/// ctx.new_str(ascii!("type_ignores")).into(), +/// ]) +/// .into(), +/// ); +/// class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); +/// } +/// ``` +/// ### pymember +/// # Trait +/// `#[pyclass]` on traits functions a lot like `#[pyclass]` on `impl` blocks. +/// Note that associated functions that are annotated with `#[pymethod]` or similar **must** +/// have a body, abstract functions should be wrapped before applying an annotation. +#[proc_macro_attribute] +pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr with Punctuated::parse_terminated); + let item = parse_macro_input!(item); + derive_impl::pyclass(attr, item).into() +} + +/// Helper macro to define `Exception` types. +/// More-or-less is an alias to `pyclass` macro. +/// +/// This macro serves a goal of generating multiple +/// `BaseException` / `Exception` +/// subtypes in a uniform and convenient manner. +/// It looks like `SimpleExtendsException` in `CPython`. +/// <https://github.com/python/cpython/blob/main/Objects/exceptions.c> +#[proc_macro_attribute] +pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr with Punctuated::parse_terminated); + let item = parse_macro_input!(item); + derive_impl::pyexception(attr, item).into() +} + +/// This attribute must be applied to an inline module. +/// It defines a Python module in the form of a `module_def` function in the module; +/// this has to be used in a `add_native_module` to properly register the module. +/// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module. +/// # Arguments +/// - `name`: the name of the python module, +/// by default, it is the name of the module, but this can be configured. +/// ```rust, ignore +/// // This will create a module named `my_module` +/// #[pymodule(name = "my_module")] +/// mod module { +/// } +/// ``` +/// - `sub`: declare the module as a submodule of another module. +/// ```rust, ignore +/// #[pymodule(sub)] +/// mod submodule { +/// } +/// +/// #[pymodule(with(submodule))] +/// mod my_module { +/// } +/// ``` +/// - `with`: declare the list of submodules that this module contains (see `sub` for example). +/// ## Inner markers +/// ### pyattr +/// `pyattr` is a multipurpose marker that can be used in a pymodule. +/// The most common use is to mark a function or class as a part of the module. +/// This can be done by applying it to a function or struct prior to the `#[pyfunction]` or `#[pyclass]` macro. +/// If applied to a constant, it will be added to the module as an attribute. +/// If applied to a function not marked with `pyfunction`, +/// it will also be added to the module as an attribute but the value is the result of the function. +/// If `#[pyattr(once)]` is used in this case, the function will be called once +/// and the result will be stored using a `static_cell`. +/// #### Examples +/// ```rust, ignore +/// #[pymodule] +/// mod my_module { +/// #[pyattr] +/// const MY_CONSTANT: i32 = 42; +/// #[pyattr] +/// fn another_constant() -> PyResult<i32> { +/// Ok(42) +/// } +/// #[pyattr(once)] +/// fn once() -> PyResult<i32> { +/// // This will only be called once and the result will be stored. +/// Ok(2 ** 24) +/// } +/// +/// #[pyattr] +/// #[pyfunction] +/// fn my_function(vm: &VirtualMachine) -> PyResult<()> { +/// ... +/// } +/// } +/// ``` +/// ### pyfunction +/// This is used to create a python function. +/// #### Function signature +/// The last argument can optionally be of the type `&VirtualMachine` to access the VM. +/// Refer to the `pymethod` documentation (located in the `pyclass` macro documentation) +/// for more information on what regular argument types are permitted. +/// #### Arguments +/// - `name`: the name of the function in Python, by default it is the same as the associated Rust function. +#[proc_macro_attribute] +pub fn pymodule(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as derive_impl::PyModuleArgs); + let item = parse_macro_input!(item); + derive_impl::pymodule(attr, item).into() +} + +/// Attribute macro for defining Python struct sequence types. +/// +/// This macro is applied to an empty struct to create a Python type +/// that wraps a Data struct. +/// +/// # Example +/// ```ignore +/// #[pystruct_sequence_data] +/// struct StructTimeData { +/// pub tm_year: PyObjectRef, +/// #[pystruct_sequence(skip)] +/// pub tm_gmtoff: PyObjectRef, +/// } +/// +/// #[pystruct_sequence(name = "struct_time", module = "time", data = "StructTimeData")] +/// struct PyStructTime; +/// ``` +#[proc_macro_attribute] +pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr with Punctuated::parse_terminated); + let item = parse_macro_input!(item); + derive_impl::pystruct_sequence(attr, item).into() +} + +/// Attribute macro for struct sequence Data structs. +/// +/// Generates field name constants, index constants, and `into_tuple()` method. +/// +/// # Example +/// ```ignore +/// #[pystruct_sequence_data] +/// struct StructTimeData { +/// pub tm_year: PyObjectRef, +/// pub tm_mon: PyObjectRef, +/// #[pystruct_sequence(skip)] // optional field, not included in tuple +/// pub tm_gmtoff: PyObjectRef, +/// } +/// ``` +/// +/// # Options +/// - `try_from_object`: Generate `try_from_elements()` method and `TryFromObject` impl +/// +/// ```ignore +/// #[pystruct_sequence_data(try_from_object)] +/// struct StructTimeData { ... } +/// ``` +#[proc_macro_attribute] +pub fn pystruct_sequence_data(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr with Punctuated::parse_terminated); + let item = parse_macro_input!(item); + derive_impl::pystruct_sequence_data(attr, item).into() +} + +struct Compiler; +impl derive_impl::Compiler for Compiler { + fn compile( + &self, + source: &str, + mode: rustpython_compiler::Mode, + module_name: String, + ) -> Result<rustpython_compiler::CodeObject, Box<dyn core::error::Error>> { + use rustpython_compiler::{CompileOpts, compile}; + Ok(compile(source, mode, &module_name, CompileOpts::default())?) + } +} + +#[proc_macro] +pub fn py_compile(input: TokenStream) -> TokenStream { + derive_impl::py_compile(input.into(), &Compiler).into() +} + +#[proc_macro] +pub fn py_freeze(input: TokenStream) -> TokenStream { + derive_impl::py_freeze(input.into(), &Compiler).into() +} + +#[proc_macro_derive(PyPayload)] +pub fn pypayload(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + derive_impl::pypayload(input).into() +} + +/// use on struct with named fields like `struct A{x:PyRef<B>, y:PyRef<C>}` to impl `Traverse` for datatype. +/// +/// use `#[pytraverse(skip)]` on fields you wish not to trace +/// +/// add `trace` attr to `#[pyclass]` to make it impl `MaybeTraverse` that will call `Traverse`'s `traverse` method so make it +/// traceable(Even from type-erased PyObject)(i.e. write `#[pyclass(trace)]`). +/// # Example +/// ```rust, ignore +/// #[pyclass(module = false, traverse)] +/// #[derive(Default, Traverse)] +/// pub struct PyList { +/// elements: PyRwLock<Vec<PyObjectRef>>, +/// #[pytraverse(skip)] +/// len: AtomicCell<usize>, +/// } +/// ``` +/// This create both `MaybeTraverse` that call `Traverse`'s `traverse` method and `Traverse` that impl `Traverse` +/// for `PyList` which call elements' `traverse` method and ignore `len` field. +#[proc_macro_derive(Traverse, attributes(pytraverse))] +pub fn pytraverse(item: proc_macro::TokenStream) -> proc_macro::TokenStream { + let item = parse_macro_input!(item); + derive_impl::pytraverse(item).into() +} diff --git a/crates/doc/.gitignore b/crates/doc/.gitignore new file mode 100644 index 00000000000..9ab870da897 --- /dev/null +++ b/crates/doc/.gitignore @@ -0,0 +1 @@ +generated/ diff --git a/crates/doc/Cargo.toml b/crates/doc/Cargo.toml new file mode 100644 index 00000000000..3435fabc8b5 --- /dev/null +++ b/crates/doc/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "rustpython-doc" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license-file = "LICENSE" + +[dependencies] +phf = { workspace = true } + +[lib] +doctest = false # Crashes when true + +[lints] +workspace = true diff --git a/crates/doc/LICENSE b/crates/doc/LICENSE new file mode 100644 index 00000000000..20cf39097c6 --- /dev/null +++ b/crates/doc/LICENSE @@ -0,0 +1,277 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see https://opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +Python software and documentation are licensed under the +Python Software Foundation License Version 2. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Version 2 +and the Zero-Clause BSD license. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. + + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001 Python Software Foundation; All Rights Reserved" +are retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION +---------------------------------------------------------------------- + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/crates/doc/generate.py b/crates/doc/generate.py new file mode 100644 index 00000000000..189e69705e1 --- /dev/null +++ b/crates/doc/generate.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python +import argparse +import inspect +import json +import os +import pathlib +import platform +import pydoc +import re +import sys +import types +import typing +import warnings +from importlib.machinery import EXTENSION_SUFFIXES, ExtensionFileLoader + +if typing.TYPE_CHECKING: + from collections.abc import Iterable + +OUTPUT_FILE = pathlib.Path(__file__).parent / "generated" / f"{sys.platform}.json" +OUTPUT_FILE.parent.mkdir(exist_ok=True) + +UNICODE_ESCAPE = re.compile(r"\\u([0-9]+)") + +IGNORED_MODULES = {"this", "antigravity"} +IGNORED_ATTRS = { + "__annotations__", + "__class__", + "__dict__", + "__dir__", + "__doc__", + "__file__", + "__name__", + "__qualname__", +} + + +type Parts = tuple[str, ...] + + +class DocEntry(typing.NamedTuple): + parts: Parts + raw_doc: str | None + + @property + def key(self) -> str: + return ".".join(self.parts) + + @property + def doc(self) -> str: + assert self.raw_doc is not None + + return re.sub(UNICODE_ESCAPE, r"\\u{\1}", self.raw_doc.strip()) + + +def is_c_extension(module: types.ModuleType) -> bool: + """ + Check whether a module was written in C. + + Returns + ------- + bool + + Notes + ----- + Adapted from: https://stackoverflow.com/a/39304199 + """ + loader = getattr(module, "__loader__", None) + if isinstance(loader, ExtensionFileLoader): + return True + + try: + inspect.getsource(module) + except (OSError, TypeError): + return True + + try: + module_filename = inspect.getfile(module) + except TypeError: + return True + + module_filetype = os.path.splitext(module_filename)[1] + return module_filetype in EXTENSION_SUFFIXES + + +def is_child_of(obj: typing.Any, module: types.ModuleType) -> bool: + """ + Whether or not an object is a child of a module. + + Returns + ------- + bool + """ + if inspect.getmodule(obj) is module: + return True + # Some C modules (e.g. _ast) set __module__ to a different name (e.g. "ast"), + # causing inspect.getmodule() to return a different module object. + # Fall back to checking the module's namespace directly. + obj_name = getattr(obj, "__name__", None) + if obj_name is not None: + return module.__dict__.get(obj_name) is obj + return False + + +def iter_modules() -> "Iterable[types.ModuleType]": + """ + Yields + ------ + :class:`types.Module` + Python modules. + """ + for module_name in sys.stdlib_module_names - IGNORED_MODULES: + try: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + module = __import__(module_name) + except ImportError: + warnings.warn(f"Could not import {module_name}", category=ImportWarning) + continue + + yield module + + +def iter_c_modules() -> "Iterable[types.ModuleType]": + """ + Yields + ------ + :class:`types.Module` + Modules that are written in C. (not pure python) + """ + yield from filter(is_c_extension, iter_modules()) + + +def traverse( + obj: typing.Any, module: types.ModuleType, parts: Parts = () +) -> "typing.Iterable[DocEntry]": + if inspect.ismodule(obj): + parts += (obj.__name__,) + + if any(f(obj) for f in (inspect.ismodule, inspect.isclass, inspect.isbuiltin)): + yield DocEntry(parts, pydoc._getowndoc(obj)) + + for name, attr in inspect.getmembers(obj): + if name in IGNORED_ATTRS: + continue + + if attr == obj: + continue + + if (module is obj) and (not is_child_of(attr, module)): + continue + + # Don't recurse into modules imported by our module. i.e. `ipaddress.py` imports `re` don't traverse `re` + if (not inspect.ismodule(obj)) and inspect.ismodule(attr): + continue + + new_parts = parts + (name,) + + attr_typ = type(attr) + is_type_or_builtin = any(attr_typ is x for x in (type, type(__builtins__))) + + if is_type_or_builtin: + yield from traverse(attr, module, new_parts) + continue + + is_callable = ( + callable(attr) + or not issubclass(attr_typ, type) + or attr_typ.__name__ in ("getset_descriptor", "member_descriptor") + ) + + is_func = any( + f(attr) + for f in (inspect.isfunction, inspect.ismethod, inspect.ismethoddescriptor) + ) + + if is_callable or is_func: + yield DocEntry(new_parts, pydoc._getowndoc(attr)) + + +def find_doc_entries() -> "Iterable[DocEntry]": + yield from ( + doc_entry + for module in iter_c_modules() + for doc_entry in traverse(module, module) + ) + yield from (doc_entry for doc_entry in traverse(__builtins__, __builtins__)) + + builtin_types = [ + type(None), + type(bytearray().__iter__()), + type(bytes().__iter__()), + type(dict().__iter__()), + type(dict().items()), + type(dict().items().__iter__()), + type(dict().values()), + type(dict().values().__iter__()), + type(lambda: ...), + type(list().__iter__()), + type(memoryview(b"").__iter__()), + type(range(0).__iter__()), + type(set().__iter__()), + type(str().__iter__()), + type(tuple().__iter__()), + ] + + # Add types from the types module (e.g., ModuleType, FunctionType, etc.) + for name in dir(types): + if name.startswith("_"): + continue + obj = getattr(types, name) + if isinstance(obj, type): + builtin_types.append(obj) + + for typ in builtin_types: + parts = ("builtins", typ.__name__) + yield DocEntry(parts, pydoc._getowndoc(typ)) + yield from traverse(typ, __builtins__, parts) + + +def main(): + docs = { + entry.key: entry.doc + for entry in find_doc_entries() + if entry.raw_doc is not None and isinstance(entry.raw_doc, str) + } + dumped = json.dumps(docs, sort_keys=True, indent=4) + OUTPUT_FILE.write_text(dumped) + + +if __name__ == "__main__": + main() diff --git a/crates/doc/src/data.inc.rs b/crates/doc/src/data.inc.rs new file mode 100644 index 00000000000..d347962ae59 --- /dev/null +++ b/crates/doc/src/data.inc.rs @@ -0,0 +1,15717 @@ +// This file was auto-generated by `.github/workflows/update-doc-db.yml`. +// CPython version: 3.14.3 +// spell-checker: disable + +pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! { + "_abc" => "Module contains faster C implementation of abc.ABCMeta", + "_abc._abc_init" => "Internal ABC helper for class set-up. Should be never used outside abc module.", + "_abc._abc_instancecheck" => "Internal ABC helper for instance checks. Should be never used outside abc module.", + "_abc._abc_register" => "Internal ABC helper for subclasss registration. Should be never used outside abc module.", + "_abc._abc_subclasscheck" => "Internal ABC helper for subclasss checks. Should be never used outside abc module.", + "_abc._get_dump" => "Internal ABC helper for cache and registry debugging.\n\nReturn shallow copies of registry, of both caches, and\nnegative cache version. Don't call this function directly,\ninstead use ABC._dump_registry() for a nice repr.", + "_abc._reset_caches" => "Internal ABC helper to reset both caches of a given class.\n\nShould be only used by refleak.py", + "_abc._reset_registry" => "Internal ABC helper to reset registry of a given class.\n\nShould be only used by refleak.py", + "_abc.get_cache_token" => "Returns the current ABC cache token.\n\nThe token is an opaque object (supporting equality testing) identifying the\ncurrent version of the ABC cache for virtual subclasses. The token changes\nwith every call to register() on any ABC.", + "_ast.AST.__delattr__" => "Implement delattr(self, name).", + "_ast.AST.__eq__" => "Return self==value.", + "_ast.AST.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AST.__ge__" => "Return self>=value.", + "_ast.AST.__getattribute__" => "Return getattr(self, name).", + "_ast.AST.__getstate__" => "Helper for pickle.", + "_ast.AST.__gt__" => "Return self>value.", + "_ast.AST.__hash__" => "Return hash(self).", + "_ast.AST.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AST.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AST.__le__" => "Return self<=value.", + "_ast.AST.__lt__" => "Return self<value.", + "_ast.AST.__ne__" => "Return self!=value.", + "_ast.AST.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AST.__reduce_ex__" => "Helper for pickle.", + "_ast.AST.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AST.__repr__" => "Return repr(self).", + "_ast.AST.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AST.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AST.__str__" => "Return str(self).", + "_ast.AST.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Add" => "Add", + "_ast.Add.__delattr__" => "Implement delattr(self, name).", + "_ast.Add.__eq__" => "Return self==value.", + "_ast.Add.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Add.__ge__" => "Return self>=value.", + "_ast.Add.__getattribute__" => "Return getattr(self, name).", + "_ast.Add.__getstate__" => "Helper for pickle.", + "_ast.Add.__gt__" => "Return self>value.", + "_ast.Add.__hash__" => "Return hash(self).", + "_ast.Add.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Add.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Add.__le__" => "Return self<=value.", + "_ast.Add.__lt__" => "Return self<value.", + "_ast.Add.__ne__" => "Return self!=value.", + "_ast.Add.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Add.__reduce_ex__" => "Helper for pickle.", + "_ast.Add.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Add.__repr__" => "Return repr(self).", + "_ast.Add.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Add.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Add.__str__" => "Return str(self).", + "_ast.Add.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Add.__weakref__" => "list of weak references to the object", + "_ast.And" => "And", + "_ast.And.__delattr__" => "Implement delattr(self, name).", + "_ast.And.__eq__" => "Return self==value.", + "_ast.And.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.And.__ge__" => "Return self>=value.", + "_ast.And.__getattribute__" => "Return getattr(self, name).", + "_ast.And.__getstate__" => "Helper for pickle.", + "_ast.And.__gt__" => "Return self>value.", + "_ast.And.__hash__" => "Return hash(self).", + "_ast.And.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.And.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.And.__le__" => "Return self<=value.", + "_ast.And.__lt__" => "Return self<value.", + "_ast.And.__ne__" => "Return self!=value.", + "_ast.And.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.And.__reduce_ex__" => "Helper for pickle.", + "_ast.And.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.And.__repr__" => "Return repr(self).", + "_ast.And.__setattr__" => "Implement setattr(self, name, value).", + "_ast.And.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.And.__str__" => "Return str(self).", + "_ast.And.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.And.__weakref__" => "list of weak references to the object", + "_ast.AnnAssign" => "AnnAssign(expr target, expr annotation, expr? value, int simple)", + "_ast.AnnAssign.__delattr__" => "Implement delattr(self, name).", + "_ast.AnnAssign.__eq__" => "Return self==value.", + "_ast.AnnAssign.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AnnAssign.__ge__" => "Return self>=value.", + "_ast.AnnAssign.__getattribute__" => "Return getattr(self, name).", + "_ast.AnnAssign.__getstate__" => "Helper for pickle.", + "_ast.AnnAssign.__gt__" => "Return self>value.", + "_ast.AnnAssign.__hash__" => "Return hash(self).", + "_ast.AnnAssign.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AnnAssign.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AnnAssign.__le__" => "Return self<=value.", + "_ast.AnnAssign.__lt__" => "Return self<value.", + "_ast.AnnAssign.__ne__" => "Return self!=value.", + "_ast.AnnAssign.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AnnAssign.__reduce_ex__" => "Helper for pickle.", + "_ast.AnnAssign.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AnnAssign.__repr__" => "Return repr(self).", + "_ast.AnnAssign.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AnnAssign.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AnnAssign.__str__" => "Return str(self).", + "_ast.AnnAssign.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AnnAssign.__weakref__" => "list of weak references to the object", + "_ast.Assert" => "Assert(expr test, expr? msg)", + "_ast.Assert.__delattr__" => "Implement delattr(self, name).", + "_ast.Assert.__eq__" => "Return self==value.", + "_ast.Assert.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Assert.__ge__" => "Return self>=value.", + "_ast.Assert.__getattribute__" => "Return getattr(self, name).", + "_ast.Assert.__getstate__" => "Helper for pickle.", + "_ast.Assert.__gt__" => "Return self>value.", + "_ast.Assert.__hash__" => "Return hash(self).", + "_ast.Assert.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Assert.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Assert.__le__" => "Return self<=value.", + "_ast.Assert.__lt__" => "Return self<value.", + "_ast.Assert.__ne__" => "Return self!=value.", + "_ast.Assert.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Assert.__reduce_ex__" => "Helper for pickle.", + "_ast.Assert.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Assert.__repr__" => "Return repr(self).", + "_ast.Assert.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Assert.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Assert.__str__" => "Return str(self).", + "_ast.Assert.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Assert.__weakref__" => "list of weak references to the object", + "_ast.Assign" => "Assign(expr* targets, expr value, string? type_comment)", + "_ast.Assign.__delattr__" => "Implement delattr(self, name).", + "_ast.Assign.__eq__" => "Return self==value.", + "_ast.Assign.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Assign.__ge__" => "Return self>=value.", + "_ast.Assign.__getattribute__" => "Return getattr(self, name).", + "_ast.Assign.__getstate__" => "Helper for pickle.", + "_ast.Assign.__gt__" => "Return self>value.", + "_ast.Assign.__hash__" => "Return hash(self).", + "_ast.Assign.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Assign.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Assign.__le__" => "Return self<=value.", + "_ast.Assign.__lt__" => "Return self<value.", + "_ast.Assign.__ne__" => "Return self!=value.", + "_ast.Assign.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Assign.__reduce_ex__" => "Helper for pickle.", + "_ast.Assign.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Assign.__repr__" => "Return repr(self).", + "_ast.Assign.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Assign.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Assign.__str__" => "Return str(self).", + "_ast.Assign.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Assign.__weakref__" => "list of weak references to the object", + "_ast.AsyncFor" => "AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)", + "_ast.AsyncFor.__delattr__" => "Implement delattr(self, name).", + "_ast.AsyncFor.__eq__" => "Return self==value.", + "_ast.AsyncFor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AsyncFor.__ge__" => "Return self>=value.", + "_ast.AsyncFor.__getattribute__" => "Return getattr(self, name).", + "_ast.AsyncFor.__getstate__" => "Helper for pickle.", + "_ast.AsyncFor.__gt__" => "Return self>value.", + "_ast.AsyncFor.__hash__" => "Return hash(self).", + "_ast.AsyncFor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AsyncFor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AsyncFor.__le__" => "Return self<=value.", + "_ast.AsyncFor.__lt__" => "Return self<value.", + "_ast.AsyncFor.__ne__" => "Return self!=value.", + "_ast.AsyncFor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AsyncFor.__reduce_ex__" => "Helper for pickle.", + "_ast.AsyncFor.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AsyncFor.__repr__" => "Return repr(self).", + "_ast.AsyncFor.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AsyncFor.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AsyncFor.__str__" => "Return str(self).", + "_ast.AsyncFor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AsyncFor.__weakref__" => "list of weak references to the object", + "_ast.AsyncFunctionDef" => "AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)", + "_ast.AsyncFunctionDef.__delattr__" => "Implement delattr(self, name).", + "_ast.AsyncFunctionDef.__eq__" => "Return self==value.", + "_ast.AsyncFunctionDef.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AsyncFunctionDef.__ge__" => "Return self>=value.", + "_ast.AsyncFunctionDef.__getattribute__" => "Return getattr(self, name).", + "_ast.AsyncFunctionDef.__getstate__" => "Helper for pickle.", + "_ast.AsyncFunctionDef.__gt__" => "Return self>value.", + "_ast.AsyncFunctionDef.__hash__" => "Return hash(self).", + "_ast.AsyncFunctionDef.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AsyncFunctionDef.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AsyncFunctionDef.__le__" => "Return self<=value.", + "_ast.AsyncFunctionDef.__lt__" => "Return self<value.", + "_ast.AsyncFunctionDef.__ne__" => "Return self!=value.", + "_ast.AsyncFunctionDef.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AsyncFunctionDef.__reduce_ex__" => "Helper for pickle.", + "_ast.AsyncFunctionDef.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AsyncFunctionDef.__repr__" => "Return repr(self).", + "_ast.AsyncFunctionDef.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AsyncFunctionDef.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AsyncFunctionDef.__str__" => "Return str(self).", + "_ast.AsyncFunctionDef.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AsyncFunctionDef.__weakref__" => "list of weak references to the object", + "_ast.AsyncWith" => "AsyncWith(withitem* items, stmt* body, string? type_comment)", + "_ast.AsyncWith.__delattr__" => "Implement delattr(self, name).", + "_ast.AsyncWith.__eq__" => "Return self==value.", + "_ast.AsyncWith.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AsyncWith.__ge__" => "Return self>=value.", + "_ast.AsyncWith.__getattribute__" => "Return getattr(self, name).", + "_ast.AsyncWith.__getstate__" => "Helper for pickle.", + "_ast.AsyncWith.__gt__" => "Return self>value.", + "_ast.AsyncWith.__hash__" => "Return hash(self).", + "_ast.AsyncWith.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AsyncWith.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AsyncWith.__le__" => "Return self<=value.", + "_ast.AsyncWith.__lt__" => "Return self<value.", + "_ast.AsyncWith.__ne__" => "Return self!=value.", + "_ast.AsyncWith.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AsyncWith.__reduce_ex__" => "Helper for pickle.", + "_ast.AsyncWith.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AsyncWith.__repr__" => "Return repr(self).", + "_ast.AsyncWith.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AsyncWith.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AsyncWith.__str__" => "Return str(self).", + "_ast.AsyncWith.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AsyncWith.__weakref__" => "list of weak references to the object", + "_ast.Attribute" => "Attribute(expr value, identifier attr, expr_context ctx)", + "_ast.Attribute.__delattr__" => "Implement delattr(self, name).", + "_ast.Attribute.__eq__" => "Return self==value.", + "_ast.Attribute.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Attribute.__ge__" => "Return self>=value.", + "_ast.Attribute.__getattribute__" => "Return getattr(self, name).", + "_ast.Attribute.__getstate__" => "Helper for pickle.", + "_ast.Attribute.__gt__" => "Return self>value.", + "_ast.Attribute.__hash__" => "Return hash(self).", + "_ast.Attribute.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Attribute.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Attribute.__le__" => "Return self<=value.", + "_ast.Attribute.__lt__" => "Return self<value.", + "_ast.Attribute.__ne__" => "Return self!=value.", + "_ast.Attribute.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Attribute.__reduce_ex__" => "Helper for pickle.", + "_ast.Attribute.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Attribute.__repr__" => "Return repr(self).", + "_ast.Attribute.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Attribute.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Attribute.__str__" => "Return str(self).", + "_ast.Attribute.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Attribute.__weakref__" => "list of weak references to the object", + "_ast.AugAssign" => "AugAssign(expr target, operator op, expr value)", + "_ast.AugAssign.__delattr__" => "Implement delattr(self, name).", + "_ast.AugAssign.__eq__" => "Return self==value.", + "_ast.AugAssign.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.AugAssign.__ge__" => "Return self>=value.", + "_ast.AugAssign.__getattribute__" => "Return getattr(self, name).", + "_ast.AugAssign.__getstate__" => "Helper for pickle.", + "_ast.AugAssign.__gt__" => "Return self>value.", + "_ast.AugAssign.__hash__" => "Return hash(self).", + "_ast.AugAssign.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.AugAssign.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.AugAssign.__le__" => "Return self<=value.", + "_ast.AugAssign.__lt__" => "Return self<value.", + "_ast.AugAssign.__ne__" => "Return self!=value.", + "_ast.AugAssign.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.AugAssign.__reduce_ex__" => "Helper for pickle.", + "_ast.AugAssign.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.AugAssign.__repr__" => "Return repr(self).", + "_ast.AugAssign.__setattr__" => "Implement setattr(self, name, value).", + "_ast.AugAssign.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.AugAssign.__str__" => "Return str(self).", + "_ast.AugAssign.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.AugAssign.__weakref__" => "list of weak references to the object", + "_ast.Await" => "Await(expr value)", + "_ast.Await.__delattr__" => "Implement delattr(self, name).", + "_ast.Await.__eq__" => "Return self==value.", + "_ast.Await.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Await.__ge__" => "Return self>=value.", + "_ast.Await.__getattribute__" => "Return getattr(self, name).", + "_ast.Await.__getstate__" => "Helper for pickle.", + "_ast.Await.__gt__" => "Return self>value.", + "_ast.Await.__hash__" => "Return hash(self).", + "_ast.Await.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Await.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Await.__le__" => "Return self<=value.", + "_ast.Await.__lt__" => "Return self<value.", + "_ast.Await.__ne__" => "Return self!=value.", + "_ast.Await.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Await.__reduce_ex__" => "Helper for pickle.", + "_ast.Await.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Await.__repr__" => "Return repr(self).", + "_ast.Await.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Await.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Await.__str__" => "Return str(self).", + "_ast.Await.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Await.__weakref__" => "list of weak references to the object", + "_ast.BinOp" => "BinOp(expr left, operator op, expr right)", + "_ast.BinOp.__delattr__" => "Implement delattr(self, name).", + "_ast.BinOp.__eq__" => "Return self==value.", + "_ast.BinOp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BinOp.__ge__" => "Return self>=value.", + "_ast.BinOp.__getattribute__" => "Return getattr(self, name).", + "_ast.BinOp.__getstate__" => "Helper for pickle.", + "_ast.BinOp.__gt__" => "Return self>value.", + "_ast.BinOp.__hash__" => "Return hash(self).", + "_ast.BinOp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BinOp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BinOp.__le__" => "Return self<=value.", + "_ast.BinOp.__lt__" => "Return self<value.", + "_ast.BinOp.__ne__" => "Return self!=value.", + "_ast.BinOp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BinOp.__reduce_ex__" => "Helper for pickle.", + "_ast.BinOp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BinOp.__repr__" => "Return repr(self).", + "_ast.BinOp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BinOp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BinOp.__str__" => "Return str(self).", + "_ast.BinOp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BinOp.__weakref__" => "list of weak references to the object", + "_ast.BitAnd" => "BitAnd", + "_ast.BitAnd.__delattr__" => "Implement delattr(self, name).", + "_ast.BitAnd.__eq__" => "Return self==value.", + "_ast.BitAnd.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BitAnd.__ge__" => "Return self>=value.", + "_ast.BitAnd.__getattribute__" => "Return getattr(self, name).", + "_ast.BitAnd.__getstate__" => "Helper for pickle.", + "_ast.BitAnd.__gt__" => "Return self>value.", + "_ast.BitAnd.__hash__" => "Return hash(self).", + "_ast.BitAnd.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BitAnd.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BitAnd.__le__" => "Return self<=value.", + "_ast.BitAnd.__lt__" => "Return self<value.", + "_ast.BitAnd.__ne__" => "Return self!=value.", + "_ast.BitAnd.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BitAnd.__reduce_ex__" => "Helper for pickle.", + "_ast.BitAnd.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BitAnd.__repr__" => "Return repr(self).", + "_ast.BitAnd.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BitAnd.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BitAnd.__str__" => "Return str(self).", + "_ast.BitAnd.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BitAnd.__weakref__" => "list of weak references to the object", + "_ast.BitOr" => "BitOr", + "_ast.BitOr.__delattr__" => "Implement delattr(self, name).", + "_ast.BitOr.__eq__" => "Return self==value.", + "_ast.BitOr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BitOr.__ge__" => "Return self>=value.", + "_ast.BitOr.__getattribute__" => "Return getattr(self, name).", + "_ast.BitOr.__getstate__" => "Helper for pickle.", + "_ast.BitOr.__gt__" => "Return self>value.", + "_ast.BitOr.__hash__" => "Return hash(self).", + "_ast.BitOr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BitOr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BitOr.__le__" => "Return self<=value.", + "_ast.BitOr.__lt__" => "Return self<value.", + "_ast.BitOr.__ne__" => "Return self!=value.", + "_ast.BitOr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BitOr.__reduce_ex__" => "Helper for pickle.", + "_ast.BitOr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BitOr.__repr__" => "Return repr(self).", + "_ast.BitOr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BitOr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BitOr.__str__" => "Return str(self).", + "_ast.BitOr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BitOr.__weakref__" => "list of weak references to the object", + "_ast.BitXor" => "BitXor", + "_ast.BitXor.__delattr__" => "Implement delattr(self, name).", + "_ast.BitXor.__eq__" => "Return self==value.", + "_ast.BitXor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BitXor.__ge__" => "Return self>=value.", + "_ast.BitXor.__getattribute__" => "Return getattr(self, name).", + "_ast.BitXor.__getstate__" => "Helper for pickle.", + "_ast.BitXor.__gt__" => "Return self>value.", + "_ast.BitXor.__hash__" => "Return hash(self).", + "_ast.BitXor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BitXor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BitXor.__le__" => "Return self<=value.", + "_ast.BitXor.__lt__" => "Return self<value.", + "_ast.BitXor.__ne__" => "Return self!=value.", + "_ast.BitXor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BitXor.__reduce_ex__" => "Helper for pickle.", + "_ast.BitXor.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BitXor.__repr__" => "Return repr(self).", + "_ast.BitXor.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BitXor.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BitXor.__str__" => "Return str(self).", + "_ast.BitXor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BitXor.__weakref__" => "list of weak references to the object", + "_ast.BoolOp" => "BoolOp(boolop op, expr* values)", + "_ast.BoolOp.__delattr__" => "Implement delattr(self, name).", + "_ast.BoolOp.__eq__" => "Return self==value.", + "_ast.BoolOp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.BoolOp.__ge__" => "Return self>=value.", + "_ast.BoolOp.__getattribute__" => "Return getattr(self, name).", + "_ast.BoolOp.__getstate__" => "Helper for pickle.", + "_ast.BoolOp.__gt__" => "Return self>value.", + "_ast.BoolOp.__hash__" => "Return hash(self).", + "_ast.BoolOp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.BoolOp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.BoolOp.__le__" => "Return self<=value.", + "_ast.BoolOp.__lt__" => "Return self<value.", + "_ast.BoolOp.__ne__" => "Return self!=value.", + "_ast.BoolOp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.BoolOp.__reduce_ex__" => "Helper for pickle.", + "_ast.BoolOp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.BoolOp.__repr__" => "Return repr(self).", + "_ast.BoolOp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.BoolOp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.BoolOp.__str__" => "Return str(self).", + "_ast.BoolOp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.BoolOp.__weakref__" => "list of weak references to the object", + "_ast.Break" => "Break", + "_ast.Break.__delattr__" => "Implement delattr(self, name).", + "_ast.Break.__eq__" => "Return self==value.", + "_ast.Break.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Break.__ge__" => "Return self>=value.", + "_ast.Break.__getattribute__" => "Return getattr(self, name).", + "_ast.Break.__getstate__" => "Helper for pickle.", + "_ast.Break.__gt__" => "Return self>value.", + "_ast.Break.__hash__" => "Return hash(self).", + "_ast.Break.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Break.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Break.__le__" => "Return self<=value.", + "_ast.Break.__lt__" => "Return self<value.", + "_ast.Break.__ne__" => "Return self!=value.", + "_ast.Break.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Break.__reduce_ex__" => "Helper for pickle.", + "_ast.Break.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Break.__repr__" => "Return repr(self).", + "_ast.Break.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Break.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Break.__str__" => "Return str(self).", + "_ast.Break.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Break.__weakref__" => "list of weak references to the object", + "_ast.Call" => "Call(expr func, expr* args, keyword* keywords)", + "_ast.Call.__delattr__" => "Implement delattr(self, name).", + "_ast.Call.__eq__" => "Return self==value.", + "_ast.Call.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Call.__ge__" => "Return self>=value.", + "_ast.Call.__getattribute__" => "Return getattr(self, name).", + "_ast.Call.__getstate__" => "Helper for pickle.", + "_ast.Call.__gt__" => "Return self>value.", + "_ast.Call.__hash__" => "Return hash(self).", + "_ast.Call.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Call.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Call.__le__" => "Return self<=value.", + "_ast.Call.__lt__" => "Return self<value.", + "_ast.Call.__ne__" => "Return self!=value.", + "_ast.Call.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Call.__reduce_ex__" => "Helper for pickle.", + "_ast.Call.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Call.__repr__" => "Return repr(self).", + "_ast.Call.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Call.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Call.__str__" => "Return str(self).", + "_ast.Call.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Call.__weakref__" => "list of weak references to the object", + "_ast.ClassDef" => "ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list, type_param* type_params)", + "_ast.ClassDef.__delattr__" => "Implement delattr(self, name).", + "_ast.ClassDef.__eq__" => "Return self==value.", + "_ast.ClassDef.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ClassDef.__ge__" => "Return self>=value.", + "_ast.ClassDef.__getattribute__" => "Return getattr(self, name).", + "_ast.ClassDef.__getstate__" => "Helper for pickle.", + "_ast.ClassDef.__gt__" => "Return self>value.", + "_ast.ClassDef.__hash__" => "Return hash(self).", + "_ast.ClassDef.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ClassDef.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ClassDef.__le__" => "Return self<=value.", + "_ast.ClassDef.__lt__" => "Return self<value.", + "_ast.ClassDef.__ne__" => "Return self!=value.", + "_ast.ClassDef.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ClassDef.__reduce_ex__" => "Helper for pickle.", + "_ast.ClassDef.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ClassDef.__repr__" => "Return repr(self).", + "_ast.ClassDef.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ClassDef.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ClassDef.__str__" => "Return str(self).", + "_ast.ClassDef.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ClassDef.__weakref__" => "list of weak references to the object", + "_ast.Compare" => "Compare(expr left, cmpop* ops, expr* comparators)", + "_ast.Compare.__delattr__" => "Implement delattr(self, name).", + "_ast.Compare.__eq__" => "Return self==value.", + "_ast.Compare.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Compare.__ge__" => "Return self>=value.", + "_ast.Compare.__getattribute__" => "Return getattr(self, name).", + "_ast.Compare.__getstate__" => "Helper for pickle.", + "_ast.Compare.__gt__" => "Return self>value.", + "_ast.Compare.__hash__" => "Return hash(self).", + "_ast.Compare.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Compare.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Compare.__le__" => "Return self<=value.", + "_ast.Compare.__lt__" => "Return self<value.", + "_ast.Compare.__ne__" => "Return self!=value.", + "_ast.Compare.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Compare.__reduce_ex__" => "Helper for pickle.", + "_ast.Compare.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Compare.__repr__" => "Return repr(self).", + "_ast.Compare.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Compare.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Compare.__str__" => "Return str(self).", + "_ast.Compare.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Compare.__weakref__" => "list of weak references to the object", + "_ast.Constant" => "Constant(constant value, string? kind)", + "_ast.Constant.__delattr__" => "Implement delattr(self, name).", + "_ast.Constant.__eq__" => "Return self==value.", + "_ast.Constant.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Constant.__ge__" => "Return self>=value.", + "_ast.Constant.__getattribute__" => "Return getattr(self, name).", + "_ast.Constant.__getstate__" => "Helper for pickle.", + "_ast.Constant.__gt__" => "Return self>value.", + "_ast.Constant.__hash__" => "Return hash(self).", + "_ast.Constant.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Constant.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Constant.__le__" => "Return self<=value.", + "_ast.Constant.__lt__" => "Return self<value.", + "_ast.Constant.__ne__" => "Return self!=value.", + "_ast.Constant.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Constant.__reduce_ex__" => "Helper for pickle.", + "_ast.Constant.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Constant.__repr__" => "Return repr(self).", + "_ast.Constant.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Constant.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Constant.__str__" => "Return str(self).", + "_ast.Constant.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Constant.__weakref__" => "list of weak references to the object", + "_ast.Continue" => "Continue", + "_ast.Continue.__delattr__" => "Implement delattr(self, name).", + "_ast.Continue.__eq__" => "Return self==value.", + "_ast.Continue.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Continue.__ge__" => "Return self>=value.", + "_ast.Continue.__getattribute__" => "Return getattr(self, name).", + "_ast.Continue.__getstate__" => "Helper for pickle.", + "_ast.Continue.__gt__" => "Return self>value.", + "_ast.Continue.__hash__" => "Return hash(self).", + "_ast.Continue.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Continue.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Continue.__le__" => "Return self<=value.", + "_ast.Continue.__lt__" => "Return self<value.", + "_ast.Continue.__ne__" => "Return self!=value.", + "_ast.Continue.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Continue.__reduce_ex__" => "Helper for pickle.", + "_ast.Continue.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Continue.__repr__" => "Return repr(self).", + "_ast.Continue.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Continue.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Continue.__str__" => "Return str(self).", + "_ast.Continue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Continue.__weakref__" => "list of weak references to the object", + "_ast.Del" => "Del", + "_ast.Del.__delattr__" => "Implement delattr(self, name).", + "_ast.Del.__eq__" => "Return self==value.", + "_ast.Del.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Del.__ge__" => "Return self>=value.", + "_ast.Del.__getattribute__" => "Return getattr(self, name).", + "_ast.Del.__getstate__" => "Helper for pickle.", + "_ast.Del.__gt__" => "Return self>value.", + "_ast.Del.__hash__" => "Return hash(self).", + "_ast.Del.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Del.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Del.__le__" => "Return self<=value.", + "_ast.Del.__lt__" => "Return self<value.", + "_ast.Del.__ne__" => "Return self!=value.", + "_ast.Del.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Del.__reduce_ex__" => "Helper for pickle.", + "_ast.Del.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Del.__repr__" => "Return repr(self).", + "_ast.Del.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Del.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Del.__str__" => "Return str(self).", + "_ast.Del.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Del.__weakref__" => "list of weak references to the object", + "_ast.Delete" => "Delete(expr* targets)", + "_ast.Delete.__delattr__" => "Implement delattr(self, name).", + "_ast.Delete.__eq__" => "Return self==value.", + "_ast.Delete.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Delete.__ge__" => "Return self>=value.", + "_ast.Delete.__getattribute__" => "Return getattr(self, name).", + "_ast.Delete.__getstate__" => "Helper for pickle.", + "_ast.Delete.__gt__" => "Return self>value.", + "_ast.Delete.__hash__" => "Return hash(self).", + "_ast.Delete.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Delete.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Delete.__le__" => "Return self<=value.", + "_ast.Delete.__lt__" => "Return self<value.", + "_ast.Delete.__ne__" => "Return self!=value.", + "_ast.Delete.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Delete.__reduce_ex__" => "Helper for pickle.", + "_ast.Delete.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Delete.__repr__" => "Return repr(self).", + "_ast.Delete.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Delete.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Delete.__str__" => "Return str(self).", + "_ast.Delete.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Delete.__weakref__" => "list of weak references to the object", + "_ast.Dict" => "Dict(expr?* keys, expr* values)", + "_ast.Dict.__delattr__" => "Implement delattr(self, name).", + "_ast.Dict.__eq__" => "Return self==value.", + "_ast.Dict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Dict.__ge__" => "Return self>=value.", + "_ast.Dict.__getattribute__" => "Return getattr(self, name).", + "_ast.Dict.__getstate__" => "Helper for pickle.", + "_ast.Dict.__gt__" => "Return self>value.", + "_ast.Dict.__hash__" => "Return hash(self).", + "_ast.Dict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Dict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Dict.__le__" => "Return self<=value.", + "_ast.Dict.__lt__" => "Return self<value.", + "_ast.Dict.__ne__" => "Return self!=value.", + "_ast.Dict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Dict.__reduce_ex__" => "Helper for pickle.", + "_ast.Dict.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Dict.__repr__" => "Return repr(self).", + "_ast.Dict.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Dict.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Dict.__str__" => "Return str(self).", + "_ast.Dict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Dict.__weakref__" => "list of weak references to the object", + "_ast.DictComp" => "DictComp(expr key, expr value, comprehension* generators)", + "_ast.DictComp.__delattr__" => "Implement delattr(self, name).", + "_ast.DictComp.__eq__" => "Return self==value.", + "_ast.DictComp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.DictComp.__ge__" => "Return self>=value.", + "_ast.DictComp.__getattribute__" => "Return getattr(self, name).", + "_ast.DictComp.__getstate__" => "Helper for pickle.", + "_ast.DictComp.__gt__" => "Return self>value.", + "_ast.DictComp.__hash__" => "Return hash(self).", + "_ast.DictComp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.DictComp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.DictComp.__le__" => "Return self<=value.", + "_ast.DictComp.__lt__" => "Return self<value.", + "_ast.DictComp.__ne__" => "Return self!=value.", + "_ast.DictComp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.DictComp.__reduce_ex__" => "Helper for pickle.", + "_ast.DictComp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.DictComp.__repr__" => "Return repr(self).", + "_ast.DictComp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.DictComp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.DictComp.__str__" => "Return str(self).", + "_ast.DictComp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.DictComp.__weakref__" => "list of weak references to the object", + "_ast.Div" => "Div", + "_ast.Div.__delattr__" => "Implement delattr(self, name).", + "_ast.Div.__eq__" => "Return self==value.", + "_ast.Div.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Div.__ge__" => "Return self>=value.", + "_ast.Div.__getattribute__" => "Return getattr(self, name).", + "_ast.Div.__getstate__" => "Helper for pickle.", + "_ast.Div.__gt__" => "Return self>value.", + "_ast.Div.__hash__" => "Return hash(self).", + "_ast.Div.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Div.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Div.__le__" => "Return self<=value.", + "_ast.Div.__lt__" => "Return self<value.", + "_ast.Div.__ne__" => "Return self!=value.", + "_ast.Div.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Div.__reduce_ex__" => "Helper for pickle.", + "_ast.Div.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Div.__repr__" => "Return repr(self).", + "_ast.Div.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Div.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Div.__str__" => "Return str(self).", + "_ast.Div.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Div.__weakref__" => "list of weak references to the object", + "_ast.Eq" => "Eq", + "_ast.Eq.__delattr__" => "Implement delattr(self, name).", + "_ast.Eq.__eq__" => "Return self==value.", + "_ast.Eq.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Eq.__ge__" => "Return self>=value.", + "_ast.Eq.__getattribute__" => "Return getattr(self, name).", + "_ast.Eq.__getstate__" => "Helper for pickle.", + "_ast.Eq.__gt__" => "Return self>value.", + "_ast.Eq.__hash__" => "Return hash(self).", + "_ast.Eq.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Eq.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Eq.__le__" => "Return self<=value.", + "_ast.Eq.__lt__" => "Return self<value.", + "_ast.Eq.__ne__" => "Return self!=value.", + "_ast.Eq.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Eq.__reduce_ex__" => "Helper for pickle.", + "_ast.Eq.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Eq.__repr__" => "Return repr(self).", + "_ast.Eq.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Eq.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Eq.__str__" => "Return str(self).", + "_ast.Eq.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Eq.__weakref__" => "list of weak references to the object", + "_ast.ExceptHandler" => "ExceptHandler(expr? type, identifier? name, stmt* body)", + "_ast.ExceptHandler.__delattr__" => "Implement delattr(self, name).", + "_ast.ExceptHandler.__eq__" => "Return self==value.", + "_ast.ExceptHandler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ExceptHandler.__ge__" => "Return self>=value.", + "_ast.ExceptHandler.__getattribute__" => "Return getattr(self, name).", + "_ast.ExceptHandler.__getstate__" => "Helper for pickle.", + "_ast.ExceptHandler.__gt__" => "Return self>value.", + "_ast.ExceptHandler.__hash__" => "Return hash(self).", + "_ast.ExceptHandler.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ExceptHandler.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ExceptHandler.__le__" => "Return self<=value.", + "_ast.ExceptHandler.__lt__" => "Return self<value.", + "_ast.ExceptHandler.__ne__" => "Return self!=value.", + "_ast.ExceptHandler.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ExceptHandler.__reduce_ex__" => "Helper for pickle.", + "_ast.ExceptHandler.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ExceptHandler.__repr__" => "Return repr(self).", + "_ast.ExceptHandler.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ExceptHandler.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ExceptHandler.__str__" => "Return str(self).", + "_ast.ExceptHandler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ExceptHandler.__weakref__" => "list of weak references to the object", + "_ast.Expr" => "Expr(expr value)", + "_ast.Expr.__delattr__" => "Implement delattr(self, name).", + "_ast.Expr.__eq__" => "Return self==value.", + "_ast.Expr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Expr.__ge__" => "Return self>=value.", + "_ast.Expr.__getattribute__" => "Return getattr(self, name).", + "_ast.Expr.__getstate__" => "Helper for pickle.", + "_ast.Expr.__gt__" => "Return self>value.", + "_ast.Expr.__hash__" => "Return hash(self).", + "_ast.Expr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Expr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Expr.__le__" => "Return self<=value.", + "_ast.Expr.__lt__" => "Return self<value.", + "_ast.Expr.__ne__" => "Return self!=value.", + "_ast.Expr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Expr.__reduce_ex__" => "Helper for pickle.", + "_ast.Expr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Expr.__repr__" => "Return repr(self).", + "_ast.Expr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Expr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Expr.__str__" => "Return str(self).", + "_ast.Expr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Expr.__weakref__" => "list of weak references to the object", + "_ast.Expression" => "Expression(expr body)", + "_ast.Expression.__delattr__" => "Implement delattr(self, name).", + "_ast.Expression.__eq__" => "Return self==value.", + "_ast.Expression.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Expression.__ge__" => "Return self>=value.", + "_ast.Expression.__getattribute__" => "Return getattr(self, name).", + "_ast.Expression.__getstate__" => "Helper for pickle.", + "_ast.Expression.__gt__" => "Return self>value.", + "_ast.Expression.__hash__" => "Return hash(self).", + "_ast.Expression.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Expression.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Expression.__le__" => "Return self<=value.", + "_ast.Expression.__lt__" => "Return self<value.", + "_ast.Expression.__ne__" => "Return self!=value.", + "_ast.Expression.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Expression.__reduce_ex__" => "Helper for pickle.", + "_ast.Expression.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Expression.__repr__" => "Return repr(self).", + "_ast.Expression.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Expression.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Expression.__str__" => "Return str(self).", + "_ast.Expression.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Expression.__weakref__" => "list of weak references to the object", + "_ast.FloorDiv" => "FloorDiv", + "_ast.FloorDiv.__delattr__" => "Implement delattr(self, name).", + "_ast.FloorDiv.__eq__" => "Return self==value.", + "_ast.FloorDiv.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.FloorDiv.__ge__" => "Return self>=value.", + "_ast.FloorDiv.__getattribute__" => "Return getattr(self, name).", + "_ast.FloorDiv.__getstate__" => "Helper for pickle.", + "_ast.FloorDiv.__gt__" => "Return self>value.", + "_ast.FloorDiv.__hash__" => "Return hash(self).", + "_ast.FloorDiv.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.FloorDiv.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.FloorDiv.__le__" => "Return self<=value.", + "_ast.FloorDiv.__lt__" => "Return self<value.", + "_ast.FloorDiv.__ne__" => "Return self!=value.", + "_ast.FloorDiv.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.FloorDiv.__reduce_ex__" => "Helper for pickle.", + "_ast.FloorDiv.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.FloorDiv.__repr__" => "Return repr(self).", + "_ast.FloorDiv.__setattr__" => "Implement setattr(self, name, value).", + "_ast.FloorDiv.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.FloorDiv.__str__" => "Return str(self).", + "_ast.FloorDiv.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.FloorDiv.__weakref__" => "list of weak references to the object", + "_ast.For" => "For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)", + "_ast.For.__delattr__" => "Implement delattr(self, name).", + "_ast.For.__eq__" => "Return self==value.", + "_ast.For.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.For.__ge__" => "Return self>=value.", + "_ast.For.__getattribute__" => "Return getattr(self, name).", + "_ast.For.__getstate__" => "Helper for pickle.", + "_ast.For.__gt__" => "Return self>value.", + "_ast.For.__hash__" => "Return hash(self).", + "_ast.For.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.For.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.For.__le__" => "Return self<=value.", + "_ast.For.__lt__" => "Return self<value.", + "_ast.For.__ne__" => "Return self!=value.", + "_ast.For.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.For.__reduce_ex__" => "Helper for pickle.", + "_ast.For.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.For.__repr__" => "Return repr(self).", + "_ast.For.__setattr__" => "Implement setattr(self, name, value).", + "_ast.For.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.For.__str__" => "Return str(self).", + "_ast.For.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.For.__weakref__" => "list of weak references to the object", + "_ast.FormattedValue" => "FormattedValue(expr value, int conversion, expr? format_spec)", + "_ast.FormattedValue.__delattr__" => "Implement delattr(self, name).", + "_ast.FormattedValue.__eq__" => "Return self==value.", + "_ast.FormattedValue.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.FormattedValue.__ge__" => "Return self>=value.", + "_ast.FormattedValue.__getattribute__" => "Return getattr(self, name).", + "_ast.FormattedValue.__getstate__" => "Helper for pickle.", + "_ast.FormattedValue.__gt__" => "Return self>value.", + "_ast.FormattedValue.__hash__" => "Return hash(self).", + "_ast.FormattedValue.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.FormattedValue.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.FormattedValue.__le__" => "Return self<=value.", + "_ast.FormattedValue.__lt__" => "Return self<value.", + "_ast.FormattedValue.__ne__" => "Return self!=value.", + "_ast.FormattedValue.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.FormattedValue.__reduce_ex__" => "Helper for pickle.", + "_ast.FormattedValue.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.FormattedValue.__repr__" => "Return repr(self).", + "_ast.FormattedValue.__setattr__" => "Implement setattr(self, name, value).", + "_ast.FormattedValue.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.FormattedValue.__str__" => "Return str(self).", + "_ast.FormattedValue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.FormattedValue.__weakref__" => "list of weak references to the object", + "_ast.FunctionDef" => "FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)", + "_ast.FunctionDef.__delattr__" => "Implement delattr(self, name).", + "_ast.FunctionDef.__eq__" => "Return self==value.", + "_ast.FunctionDef.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.FunctionDef.__ge__" => "Return self>=value.", + "_ast.FunctionDef.__getattribute__" => "Return getattr(self, name).", + "_ast.FunctionDef.__getstate__" => "Helper for pickle.", + "_ast.FunctionDef.__gt__" => "Return self>value.", + "_ast.FunctionDef.__hash__" => "Return hash(self).", + "_ast.FunctionDef.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.FunctionDef.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.FunctionDef.__le__" => "Return self<=value.", + "_ast.FunctionDef.__lt__" => "Return self<value.", + "_ast.FunctionDef.__ne__" => "Return self!=value.", + "_ast.FunctionDef.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.FunctionDef.__reduce_ex__" => "Helper for pickle.", + "_ast.FunctionDef.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.FunctionDef.__repr__" => "Return repr(self).", + "_ast.FunctionDef.__setattr__" => "Implement setattr(self, name, value).", + "_ast.FunctionDef.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.FunctionDef.__str__" => "Return str(self).", + "_ast.FunctionDef.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.FunctionDef.__weakref__" => "list of weak references to the object", + "_ast.FunctionType" => "FunctionType(expr* argtypes, expr returns)", + "_ast.FunctionType.__delattr__" => "Implement delattr(self, name).", + "_ast.FunctionType.__eq__" => "Return self==value.", + "_ast.FunctionType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.FunctionType.__ge__" => "Return self>=value.", + "_ast.FunctionType.__getattribute__" => "Return getattr(self, name).", + "_ast.FunctionType.__getstate__" => "Helper for pickle.", + "_ast.FunctionType.__gt__" => "Return self>value.", + "_ast.FunctionType.__hash__" => "Return hash(self).", + "_ast.FunctionType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.FunctionType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.FunctionType.__le__" => "Return self<=value.", + "_ast.FunctionType.__lt__" => "Return self<value.", + "_ast.FunctionType.__ne__" => "Return self!=value.", + "_ast.FunctionType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.FunctionType.__reduce_ex__" => "Helper for pickle.", + "_ast.FunctionType.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.FunctionType.__repr__" => "Return repr(self).", + "_ast.FunctionType.__setattr__" => "Implement setattr(self, name, value).", + "_ast.FunctionType.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.FunctionType.__str__" => "Return str(self).", + "_ast.FunctionType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.FunctionType.__weakref__" => "list of weak references to the object", + "_ast.GeneratorExp" => "GeneratorExp(expr elt, comprehension* generators)", + "_ast.GeneratorExp.__delattr__" => "Implement delattr(self, name).", + "_ast.GeneratorExp.__eq__" => "Return self==value.", + "_ast.GeneratorExp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.GeneratorExp.__ge__" => "Return self>=value.", + "_ast.GeneratorExp.__getattribute__" => "Return getattr(self, name).", + "_ast.GeneratorExp.__getstate__" => "Helper for pickle.", + "_ast.GeneratorExp.__gt__" => "Return self>value.", + "_ast.GeneratorExp.__hash__" => "Return hash(self).", + "_ast.GeneratorExp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.GeneratorExp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.GeneratorExp.__le__" => "Return self<=value.", + "_ast.GeneratorExp.__lt__" => "Return self<value.", + "_ast.GeneratorExp.__ne__" => "Return self!=value.", + "_ast.GeneratorExp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.GeneratorExp.__reduce_ex__" => "Helper for pickle.", + "_ast.GeneratorExp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.GeneratorExp.__repr__" => "Return repr(self).", + "_ast.GeneratorExp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.GeneratorExp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.GeneratorExp.__str__" => "Return str(self).", + "_ast.GeneratorExp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.GeneratorExp.__weakref__" => "list of weak references to the object", + "_ast.Global" => "Global(identifier* names)", + "_ast.Global.__delattr__" => "Implement delattr(self, name).", + "_ast.Global.__eq__" => "Return self==value.", + "_ast.Global.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Global.__ge__" => "Return self>=value.", + "_ast.Global.__getattribute__" => "Return getattr(self, name).", + "_ast.Global.__getstate__" => "Helper for pickle.", + "_ast.Global.__gt__" => "Return self>value.", + "_ast.Global.__hash__" => "Return hash(self).", + "_ast.Global.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Global.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Global.__le__" => "Return self<=value.", + "_ast.Global.__lt__" => "Return self<value.", + "_ast.Global.__ne__" => "Return self!=value.", + "_ast.Global.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Global.__reduce_ex__" => "Helper for pickle.", + "_ast.Global.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Global.__repr__" => "Return repr(self).", + "_ast.Global.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Global.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Global.__str__" => "Return str(self).", + "_ast.Global.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Global.__weakref__" => "list of weak references to the object", + "_ast.Gt" => "Gt", + "_ast.Gt.__delattr__" => "Implement delattr(self, name).", + "_ast.Gt.__eq__" => "Return self==value.", + "_ast.Gt.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Gt.__ge__" => "Return self>=value.", + "_ast.Gt.__getattribute__" => "Return getattr(self, name).", + "_ast.Gt.__getstate__" => "Helper for pickle.", + "_ast.Gt.__gt__" => "Return self>value.", + "_ast.Gt.__hash__" => "Return hash(self).", + "_ast.Gt.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Gt.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Gt.__le__" => "Return self<=value.", + "_ast.Gt.__lt__" => "Return self<value.", + "_ast.Gt.__ne__" => "Return self!=value.", + "_ast.Gt.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Gt.__reduce_ex__" => "Helper for pickle.", + "_ast.Gt.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Gt.__repr__" => "Return repr(self).", + "_ast.Gt.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Gt.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Gt.__str__" => "Return str(self).", + "_ast.Gt.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Gt.__weakref__" => "list of weak references to the object", + "_ast.GtE" => "GtE", + "_ast.GtE.__delattr__" => "Implement delattr(self, name).", + "_ast.GtE.__eq__" => "Return self==value.", + "_ast.GtE.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.GtE.__ge__" => "Return self>=value.", + "_ast.GtE.__getattribute__" => "Return getattr(self, name).", + "_ast.GtE.__getstate__" => "Helper for pickle.", + "_ast.GtE.__gt__" => "Return self>value.", + "_ast.GtE.__hash__" => "Return hash(self).", + "_ast.GtE.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.GtE.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.GtE.__le__" => "Return self<=value.", + "_ast.GtE.__lt__" => "Return self<value.", + "_ast.GtE.__ne__" => "Return self!=value.", + "_ast.GtE.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.GtE.__reduce_ex__" => "Helper for pickle.", + "_ast.GtE.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.GtE.__repr__" => "Return repr(self).", + "_ast.GtE.__setattr__" => "Implement setattr(self, name, value).", + "_ast.GtE.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.GtE.__str__" => "Return str(self).", + "_ast.GtE.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.GtE.__weakref__" => "list of weak references to the object", + "_ast.If" => "If(expr test, stmt* body, stmt* orelse)", + "_ast.If.__delattr__" => "Implement delattr(self, name).", + "_ast.If.__eq__" => "Return self==value.", + "_ast.If.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.If.__ge__" => "Return self>=value.", + "_ast.If.__getattribute__" => "Return getattr(self, name).", + "_ast.If.__getstate__" => "Helper for pickle.", + "_ast.If.__gt__" => "Return self>value.", + "_ast.If.__hash__" => "Return hash(self).", + "_ast.If.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.If.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.If.__le__" => "Return self<=value.", + "_ast.If.__lt__" => "Return self<value.", + "_ast.If.__ne__" => "Return self!=value.", + "_ast.If.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.If.__reduce_ex__" => "Helper for pickle.", + "_ast.If.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.If.__repr__" => "Return repr(self).", + "_ast.If.__setattr__" => "Implement setattr(self, name, value).", + "_ast.If.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.If.__str__" => "Return str(self).", + "_ast.If.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.If.__weakref__" => "list of weak references to the object", + "_ast.IfExp" => "IfExp(expr test, expr body, expr orelse)", + "_ast.IfExp.__delattr__" => "Implement delattr(self, name).", + "_ast.IfExp.__eq__" => "Return self==value.", + "_ast.IfExp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.IfExp.__ge__" => "Return self>=value.", + "_ast.IfExp.__getattribute__" => "Return getattr(self, name).", + "_ast.IfExp.__getstate__" => "Helper for pickle.", + "_ast.IfExp.__gt__" => "Return self>value.", + "_ast.IfExp.__hash__" => "Return hash(self).", + "_ast.IfExp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.IfExp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.IfExp.__le__" => "Return self<=value.", + "_ast.IfExp.__lt__" => "Return self<value.", + "_ast.IfExp.__ne__" => "Return self!=value.", + "_ast.IfExp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.IfExp.__reduce_ex__" => "Helper for pickle.", + "_ast.IfExp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.IfExp.__repr__" => "Return repr(self).", + "_ast.IfExp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.IfExp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.IfExp.__str__" => "Return str(self).", + "_ast.IfExp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.IfExp.__weakref__" => "list of weak references to the object", + "_ast.Import" => "Import(alias* names)", + "_ast.Import.__delattr__" => "Implement delattr(self, name).", + "_ast.Import.__eq__" => "Return self==value.", + "_ast.Import.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Import.__ge__" => "Return self>=value.", + "_ast.Import.__getattribute__" => "Return getattr(self, name).", + "_ast.Import.__getstate__" => "Helper for pickle.", + "_ast.Import.__gt__" => "Return self>value.", + "_ast.Import.__hash__" => "Return hash(self).", + "_ast.Import.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Import.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Import.__le__" => "Return self<=value.", + "_ast.Import.__lt__" => "Return self<value.", + "_ast.Import.__ne__" => "Return self!=value.", + "_ast.Import.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Import.__reduce_ex__" => "Helper for pickle.", + "_ast.Import.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Import.__repr__" => "Return repr(self).", + "_ast.Import.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Import.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Import.__str__" => "Return str(self).", + "_ast.Import.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Import.__weakref__" => "list of weak references to the object", + "_ast.ImportFrom" => "ImportFrom(identifier? module, alias* names, int? level)", + "_ast.ImportFrom.__delattr__" => "Implement delattr(self, name).", + "_ast.ImportFrom.__eq__" => "Return self==value.", + "_ast.ImportFrom.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ImportFrom.__ge__" => "Return self>=value.", + "_ast.ImportFrom.__getattribute__" => "Return getattr(self, name).", + "_ast.ImportFrom.__getstate__" => "Helper for pickle.", + "_ast.ImportFrom.__gt__" => "Return self>value.", + "_ast.ImportFrom.__hash__" => "Return hash(self).", + "_ast.ImportFrom.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ImportFrom.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ImportFrom.__le__" => "Return self<=value.", + "_ast.ImportFrom.__lt__" => "Return self<value.", + "_ast.ImportFrom.__ne__" => "Return self!=value.", + "_ast.ImportFrom.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ImportFrom.__reduce_ex__" => "Helper for pickle.", + "_ast.ImportFrom.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ImportFrom.__repr__" => "Return repr(self).", + "_ast.ImportFrom.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ImportFrom.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ImportFrom.__str__" => "Return str(self).", + "_ast.ImportFrom.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ImportFrom.__weakref__" => "list of weak references to the object", + "_ast.In" => "In", + "_ast.In.__delattr__" => "Implement delattr(self, name).", + "_ast.In.__eq__" => "Return self==value.", + "_ast.In.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.In.__ge__" => "Return self>=value.", + "_ast.In.__getattribute__" => "Return getattr(self, name).", + "_ast.In.__getstate__" => "Helper for pickle.", + "_ast.In.__gt__" => "Return self>value.", + "_ast.In.__hash__" => "Return hash(self).", + "_ast.In.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.In.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.In.__le__" => "Return self<=value.", + "_ast.In.__lt__" => "Return self<value.", + "_ast.In.__ne__" => "Return self!=value.", + "_ast.In.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.In.__reduce_ex__" => "Helper for pickle.", + "_ast.In.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.In.__repr__" => "Return repr(self).", + "_ast.In.__setattr__" => "Implement setattr(self, name, value).", + "_ast.In.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.In.__str__" => "Return str(self).", + "_ast.In.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.In.__weakref__" => "list of weak references to the object", + "_ast.Interactive" => "Interactive(stmt* body)", + "_ast.Interactive.__delattr__" => "Implement delattr(self, name).", + "_ast.Interactive.__eq__" => "Return self==value.", + "_ast.Interactive.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Interactive.__ge__" => "Return self>=value.", + "_ast.Interactive.__getattribute__" => "Return getattr(self, name).", + "_ast.Interactive.__getstate__" => "Helper for pickle.", + "_ast.Interactive.__gt__" => "Return self>value.", + "_ast.Interactive.__hash__" => "Return hash(self).", + "_ast.Interactive.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Interactive.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Interactive.__le__" => "Return self<=value.", + "_ast.Interactive.__lt__" => "Return self<value.", + "_ast.Interactive.__ne__" => "Return self!=value.", + "_ast.Interactive.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Interactive.__reduce_ex__" => "Helper for pickle.", + "_ast.Interactive.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Interactive.__repr__" => "Return repr(self).", + "_ast.Interactive.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Interactive.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Interactive.__str__" => "Return str(self).", + "_ast.Interactive.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Interactive.__weakref__" => "list of weak references to the object", + "_ast.Interpolation" => "Interpolation(expr value, constant str, int conversion, expr? format_spec)", + "_ast.Interpolation.__delattr__" => "Implement delattr(self, name).", + "_ast.Interpolation.__eq__" => "Return self==value.", + "_ast.Interpolation.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Interpolation.__ge__" => "Return self>=value.", + "_ast.Interpolation.__getattribute__" => "Return getattr(self, name).", + "_ast.Interpolation.__getstate__" => "Helper for pickle.", + "_ast.Interpolation.__gt__" => "Return self>value.", + "_ast.Interpolation.__hash__" => "Return hash(self).", + "_ast.Interpolation.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Interpolation.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Interpolation.__le__" => "Return self<=value.", + "_ast.Interpolation.__lt__" => "Return self<value.", + "_ast.Interpolation.__ne__" => "Return self!=value.", + "_ast.Interpolation.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Interpolation.__reduce_ex__" => "Helper for pickle.", + "_ast.Interpolation.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Interpolation.__repr__" => "Return repr(self).", + "_ast.Interpolation.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Interpolation.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Interpolation.__str__" => "Return str(self).", + "_ast.Interpolation.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Interpolation.__weakref__" => "list of weak references to the object", + "_ast.Invert" => "Invert", + "_ast.Invert.__delattr__" => "Implement delattr(self, name).", + "_ast.Invert.__eq__" => "Return self==value.", + "_ast.Invert.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Invert.__ge__" => "Return self>=value.", + "_ast.Invert.__getattribute__" => "Return getattr(self, name).", + "_ast.Invert.__getstate__" => "Helper for pickle.", + "_ast.Invert.__gt__" => "Return self>value.", + "_ast.Invert.__hash__" => "Return hash(self).", + "_ast.Invert.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Invert.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Invert.__le__" => "Return self<=value.", + "_ast.Invert.__lt__" => "Return self<value.", + "_ast.Invert.__ne__" => "Return self!=value.", + "_ast.Invert.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Invert.__reduce_ex__" => "Helper for pickle.", + "_ast.Invert.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Invert.__repr__" => "Return repr(self).", + "_ast.Invert.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Invert.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Invert.__str__" => "Return str(self).", + "_ast.Invert.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Invert.__weakref__" => "list of weak references to the object", + "_ast.Is" => "Is", + "_ast.Is.__delattr__" => "Implement delattr(self, name).", + "_ast.Is.__eq__" => "Return self==value.", + "_ast.Is.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Is.__ge__" => "Return self>=value.", + "_ast.Is.__getattribute__" => "Return getattr(self, name).", + "_ast.Is.__getstate__" => "Helper for pickle.", + "_ast.Is.__gt__" => "Return self>value.", + "_ast.Is.__hash__" => "Return hash(self).", + "_ast.Is.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Is.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Is.__le__" => "Return self<=value.", + "_ast.Is.__lt__" => "Return self<value.", + "_ast.Is.__ne__" => "Return self!=value.", + "_ast.Is.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Is.__reduce_ex__" => "Helper for pickle.", + "_ast.Is.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Is.__repr__" => "Return repr(self).", + "_ast.Is.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Is.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Is.__str__" => "Return str(self).", + "_ast.Is.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Is.__weakref__" => "list of weak references to the object", + "_ast.IsNot" => "IsNot", + "_ast.IsNot.__delattr__" => "Implement delattr(self, name).", + "_ast.IsNot.__eq__" => "Return self==value.", + "_ast.IsNot.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.IsNot.__ge__" => "Return self>=value.", + "_ast.IsNot.__getattribute__" => "Return getattr(self, name).", + "_ast.IsNot.__getstate__" => "Helper for pickle.", + "_ast.IsNot.__gt__" => "Return self>value.", + "_ast.IsNot.__hash__" => "Return hash(self).", + "_ast.IsNot.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.IsNot.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.IsNot.__le__" => "Return self<=value.", + "_ast.IsNot.__lt__" => "Return self<value.", + "_ast.IsNot.__ne__" => "Return self!=value.", + "_ast.IsNot.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.IsNot.__reduce_ex__" => "Helper for pickle.", + "_ast.IsNot.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.IsNot.__repr__" => "Return repr(self).", + "_ast.IsNot.__setattr__" => "Implement setattr(self, name, value).", + "_ast.IsNot.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.IsNot.__str__" => "Return str(self).", + "_ast.IsNot.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.IsNot.__weakref__" => "list of weak references to the object", + "_ast.JoinedStr" => "JoinedStr(expr* values)", + "_ast.JoinedStr.__delattr__" => "Implement delattr(self, name).", + "_ast.JoinedStr.__eq__" => "Return self==value.", + "_ast.JoinedStr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.JoinedStr.__ge__" => "Return self>=value.", + "_ast.JoinedStr.__getattribute__" => "Return getattr(self, name).", + "_ast.JoinedStr.__getstate__" => "Helper for pickle.", + "_ast.JoinedStr.__gt__" => "Return self>value.", + "_ast.JoinedStr.__hash__" => "Return hash(self).", + "_ast.JoinedStr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.JoinedStr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.JoinedStr.__le__" => "Return self<=value.", + "_ast.JoinedStr.__lt__" => "Return self<value.", + "_ast.JoinedStr.__ne__" => "Return self!=value.", + "_ast.JoinedStr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.JoinedStr.__reduce_ex__" => "Helper for pickle.", + "_ast.JoinedStr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.JoinedStr.__repr__" => "Return repr(self).", + "_ast.JoinedStr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.JoinedStr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.JoinedStr.__str__" => "Return str(self).", + "_ast.JoinedStr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.JoinedStr.__weakref__" => "list of weak references to the object", + "_ast.LShift" => "LShift", + "_ast.LShift.__delattr__" => "Implement delattr(self, name).", + "_ast.LShift.__eq__" => "Return self==value.", + "_ast.LShift.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.LShift.__ge__" => "Return self>=value.", + "_ast.LShift.__getattribute__" => "Return getattr(self, name).", + "_ast.LShift.__getstate__" => "Helper for pickle.", + "_ast.LShift.__gt__" => "Return self>value.", + "_ast.LShift.__hash__" => "Return hash(self).", + "_ast.LShift.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.LShift.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.LShift.__le__" => "Return self<=value.", + "_ast.LShift.__lt__" => "Return self<value.", + "_ast.LShift.__ne__" => "Return self!=value.", + "_ast.LShift.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.LShift.__reduce_ex__" => "Helper for pickle.", + "_ast.LShift.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.LShift.__repr__" => "Return repr(self).", + "_ast.LShift.__setattr__" => "Implement setattr(self, name, value).", + "_ast.LShift.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.LShift.__str__" => "Return str(self).", + "_ast.LShift.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.LShift.__weakref__" => "list of weak references to the object", + "_ast.Lambda" => "Lambda(arguments args, expr body)", + "_ast.Lambda.__delattr__" => "Implement delattr(self, name).", + "_ast.Lambda.__eq__" => "Return self==value.", + "_ast.Lambda.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Lambda.__ge__" => "Return self>=value.", + "_ast.Lambda.__getattribute__" => "Return getattr(self, name).", + "_ast.Lambda.__getstate__" => "Helper for pickle.", + "_ast.Lambda.__gt__" => "Return self>value.", + "_ast.Lambda.__hash__" => "Return hash(self).", + "_ast.Lambda.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Lambda.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Lambda.__le__" => "Return self<=value.", + "_ast.Lambda.__lt__" => "Return self<value.", + "_ast.Lambda.__ne__" => "Return self!=value.", + "_ast.Lambda.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Lambda.__reduce_ex__" => "Helper for pickle.", + "_ast.Lambda.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Lambda.__repr__" => "Return repr(self).", + "_ast.Lambda.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Lambda.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Lambda.__str__" => "Return str(self).", + "_ast.Lambda.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Lambda.__weakref__" => "list of weak references to the object", + "_ast.List" => "List(expr* elts, expr_context ctx)", + "_ast.List.__delattr__" => "Implement delattr(self, name).", + "_ast.List.__eq__" => "Return self==value.", + "_ast.List.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.List.__ge__" => "Return self>=value.", + "_ast.List.__getattribute__" => "Return getattr(self, name).", + "_ast.List.__getstate__" => "Helper for pickle.", + "_ast.List.__gt__" => "Return self>value.", + "_ast.List.__hash__" => "Return hash(self).", + "_ast.List.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.List.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.List.__le__" => "Return self<=value.", + "_ast.List.__lt__" => "Return self<value.", + "_ast.List.__ne__" => "Return self!=value.", + "_ast.List.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.List.__reduce_ex__" => "Helper for pickle.", + "_ast.List.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.List.__repr__" => "Return repr(self).", + "_ast.List.__setattr__" => "Implement setattr(self, name, value).", + "_ast.List.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.List.__str__" => "Return str(self).", + "_ast.List.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.List.__weakref__" => "list of weak references to the object", + "_ast.ListComp" => "ListComp(expr elt, comprehension* generators)", + "_ast.ListComp.__delattr__" => "Implement delattr(self, name).", + "_ast.ListComp.__eq__" => "Return self==value.", + "_ast.ListComp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ListComp.__ge__" => "Return self>=value.", + "_ast.ListComp.__getattribute__" => "Return getattr(self, name).", + "_ast.ListComp.__getstate__" => "Helper for pickle.", + "_ast.ListComp.__gt__" => "Return self>value.", + "_ast.ListComp.__hash__" => "Return hash(self).", + "_ast.ListComp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ListComp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ListComp.__le__" => "Return self<=value.", + "_ast.ListComp.__lt__" => "Return self<value.", + "_ast.ListComp.__ne__" => "Return self!=value.", + "_ast.ListComp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ListComp.__reduce_ex__" => "Helper for pickle.", + "_ast.ListComp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ListComp.__repr__" => "Return repr(self).", + "_ast.ListComp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ListComp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ListComp.__str__" => "Return str(self).", + "_ast.ListComp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ListComp.__weakref__" => "list of weak references to the object", + "_ast.Load" => "Load", + "_ast.Load.__delattr__" => "Implement delattr(self, name).", + "_ast.Load.__eq__" => "Return self==value.", + "_ast.Load.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Load.__ge__" => "Return self>=value.", + "_ast.Load.__getattribute__" => "Return getattr(self, name).", + "_ast.Load.__getstate__" => "Helper for pickle.", + "_ast.Load.__gt__" => "Return self>value.", + "_ast.Load.__hash__" => "Return hash(self).", + "_ast.Load.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Load.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Load.__le__" => "Return self<=value.", + "_ast.Load.__lt__" => "Return self<value.", + "_ast.Load.__ne__" => "Return self!=value.", + "_ast.Load.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Load.__reduce_ex__" => "Helper for pickle.", + "_ast.Load.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Load.__repr__" => "Return repr(self).", + "_ast.Load.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Load.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Load.__str__" => "Return str(self).", + "_ast.Load.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Load.__weakref__" => "list of weak references to the object", + "_ast.Lt" => "Lt", + "_ast.Lt.__delattr__" => "Implement delattr(self, name).", + "_ast.Lt.__eq__" => "Return self==value.", + "_ast.Lt.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Lt.__ge__" => "Return self>=value.", + "_ast.Lt.__getattribute__" => "Return getattr(self, name).", + "_ast.Lt.__getstate__" => "Helper for pickle.", + "_ast.Lt.__gt__" => "Return self>value.", + "_ast.Lt.__hash__" => "Return hash(self).", + "_ast.Lt.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Lt.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Lt.__le__" => "Return self<=value.", + "_ast.Lt.__lt__" => "Return self<value.", + "_ast.Lt.__ne__" => "Return self!=value.", + "_ast.Lt.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Lt.__reduce_ex__" => "Helper for pickle.", + "_ast.Lt.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Lt.__repr__" => "Return repr(self).", + "_ast.Lt.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Lt.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Lt.__str__" => "Return str(self).", + "_ast.Lt.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Lt.__weakref__" => "list of weak references to the object", + "_ast.LtE" => "LtE", + "_ast.LtE.__delattr__" => "Implement delattr(self, name).", + "_ast.LtE.__eq__" => "Return self==value.", + "_ast.LtE.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.LtE.__ge__" => "Return self>=value.", + "_ast.LtE.__getattribute__" => "Return getattr(self, name).", + "_ast.LtE.__getstate__" => "Helper for pickle.", + "_ast.LtE.__gt__" => "Return self>value.", + "_ast.LtE.__hash__" => "Return hash(self).", + "_ast.LtE.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.LtE.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.LtE.__le__" => "Return self<=value.", + "_ast.LtE.__lt__" => "Return self<value.", + "_ast.LtE.__ne__" => "Return self!=value.", + "_ast.LtE.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.LtE.__reduce_ex__" => "Helper for pickle.", + "_ast.LtE.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.LtE.__repr__" => "Return repr(self).", + "_ast.LtE.__setattr__" => "Implement setattr(self, name, value).", + "_ast.LtE.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.LtE.__str__" => "Return str(self).", + "_ast.LtE.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.LtE.__weakref__" => "list of weak references to the object", + "_ast.MatMult" => "MatMult", + "_ast.MatMult.__delattr__" => "Implement delattr(self, name).", + "_ast.MatMult.__eq__" => "Return self==value.", + "_ast.MatMult.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatMult.__ge__" => "Return self>=value.", + "_ast.MatMult.__getattribute__" => "Return getattr(self, name).", + "_ast.MatMult.__getstate__" => "Helper for pickle.", + "_ast.MatMult.__gt__" => "Return self>value.", + "_ast.MatMult.__hash__" => "Return hash(self).", + "_ast.MatMult.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatMult.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatMult.__le__" => "Return self<=value.", + "_ast.MatMult.__lt__" => "Return self<value.", + "_ast.MatMult.__ne__" => "Return self!=value.", + "_ast.MatMult.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatMult.__reduce_ex__" => "Helper for pickle.", + "_ast.MatMult.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatMult.__repr__" => "Return repr(self).", + "_ast.MatMult.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatMult.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatMult.__str__" => "Return str(self).", + "_ast.MatMult.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatMult.__weakref__" => "list of weak references to the object", + "_ast.Match" => "Match(expr subject, match_case* cases)", + "_ast.Match.__delattr__" => "Implement delattr(self, name).", + "_ast.Match.__eq__" => "Return self==value.", + "_ast.Match.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Match.__ge__" => "Return self>=value.", + "_ast.Match.__getattribute__" => "Return getattr(self, name).", + "_ast.Match.__getstate__" => "Helper for pickle.", + "_ast.Match.__gt__" => "Return self>value.", + "_ast.Match.__hash__" => "Return hash(self).", + "_ast.Match.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Match.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Match.__le__" => "Return self<=value.", + "_ast.Match.__lt__" => "Return self<value.", + "_ast.Match.__ne__" => "Return self!=value.", + "_ast.Match.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Match.__reduce_ex__" => "Helper for pickle.", + "_ast.Match.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Match.__repr__" => "Return repr(self).", + "_ast.Match.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Match.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Match.__str__" => "Return str(self).", + "_ast.Match.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Match.__weakref__" => "list of weak references to the object", + "_ast.MatchAs" => "MatchAs(pattern? pattern, identifier? name)", + "_ast.MatchAs.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchAs.__eq__" => "Return self==value.", + "_ast.MatchAs.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchAs.__ge__" => "Return self>=value.", + "_ast.MatchAs.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchAs.__getstate__" => "Helper for pickle.", + "_ast.MatchAs.__gt__" => "Return self>value.", + "_ast.MatchAs.__hash__" => "Return hash(self).", + "_ast.MatchAs.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchAs.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchAs.__le__" => "Return self<=value.", + "_ast.MatchAs.__lt__" => "Return self<value.", + "_ast.MatchAs.__ne__" => "Return self!=value.", + "_ast.MatchAs.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchAs.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchAs.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchAs.__repr__" => "Return repr(self).", + "_ast.MatchAs.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchAs.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchAs.__str__" => "Return str(self).", + "_ast.MatchAs.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchAs.__weakref__" => "list of weak references to the object", + "_ast.MatchClass" => "MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)", + "_ast.MatchClass.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchClass.__eq__" => "Return self==value.", + "_ast.MatchClass.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchClass.__ge__" => "Return self>=value.", + "_ast.MatchClass.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchClass.__getstate__" => "Helper for pickle.", + "_ast.MatchClass.__gt__" => "Return self>value.", + "_ast.MatchClass.__hash__" => "Return hash(self).", + "_ast.MatchClass.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchClass.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchClass.__le__" => "Return self<=value.", + "_ast.MatchClass.__lt__" => "Return self<value.", + "_ast.MatchClass.__ne__" => "Return self!=value.", + "_ast.MatchClass.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchClass.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchClass.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchClass.__repr__" => "Return repr(self).", + "_ast.MatchClass.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchClass.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchClass.__str__" => "Return str(self).", + "_ast.MatchClass.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchClass.__weakref__" => "list of weak references to the object", + "_ast.MatchMapping" => "MatchMapping(expr* keys, pattern* patterns, identifier? rest)", + "_ast.MatchMapping.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchMapping.__eq__" => "Return self==value.", + "_ast.MatchMapping.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchMapping.__ge__" => "Return self>=value.", + "_ast.MatchMapping.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchMapping.__getstate__" => "Helper for pickle.", + "_ast.MatchMapping.__gt__" => "Return self>value.", + "_ast.MatchMapping.__hash__" => "Return hash(self).", + "_ast.MatchMapping.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchMapping.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchMapping.__le__" => "Return self<=value.", + "_ast.MatchMapping.__lt__" => "Return self<value.", + "_ast.MatchMapping.__ne__" => "Return self!=value.", + "_ast.MatchMapping.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchMapping.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchMapping.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchMapping.__repr__" => "Return repr(self).", + "_ast.MatchMapping.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchMapping.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchMapping.__str__" => "Return str(self).", + "_ast.MatchMapping.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchMapping.__weakref__" => "list of weak references to the object", + "_ast.MatchOr" => "MatchOr(pattern* patterns)", + "_ast.MatchOr.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchOr.__eq__" => "Return self==value.", + "_ast.MatchOr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchOr.__ge__" => "Return self>=value.", + "_ast.MatchOr.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchOr.__getstate__" => "Helper for pickle.", + "_ast.MatchOr.__gt__" => "Return self>value.", + "_ast.MatchOr.__hash__" => "Return hash(self).", + "_ast.MatchOr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchOr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchOr.__le__" => "Return self<=value.", + "_ast.MatchOr.__lt__" => "Return self<value.", + "_ast.MatchOr.__ne__" => "Return self!=value.", + "_ast.MatchOr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchOr.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchOr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchOr.__repr__" => "Return repr(self).", + "_ast.MatchOr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchOr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchOr.__str__" => "Return str(self).", + "_ast.MatchOr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchOr.__weakref__" => "list of weak references to the object", + "_ast.MatchSequence" => "MatchSequence(pattern* patterns)", + "_ast.MatchSequence.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchSequence.__eq__" => "Return self==value.", + "_ast.MatchSequence.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchSequence.__ge__" => "Return self>=value.", + "_ast.MatchSequence.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchSequence.__getstate__" => "Helper for pickle.", + "_ast.MatchSequence.__gt__" => "Return self>value.", + "_ast.MatchSequence.__hash__" => "Return hash(self).", + "_ast.MatchSequence.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchSequence.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchSequence.__le__" => "Return self<=value.", + "_ast.MatchSequence.__lt__" => "Return self<value.", + "_ast.MatchSequence.__ne__" => "Return self!=value.", + "_ast.MatchSequence.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchSequence.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchSequence.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchSequence.__repr__" => "Return repr(self).", + "_ast.MatchSequence.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchSequence.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchSequence.__str__" => "Return str(self).", + "_ast.MatchSequence.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchSequence.__weakref__" => "list of weak references to the object", + "_ast.MatchSingleton" => "MatchSingleton(constant value)", + "_ast.MatchSingleton.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchSingleton.__eq__" => "Return self==value.", + "_ast.MatchSingleton.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchSingleton.__ge__" => "Return self>=value.", + "_ast.MatchSingleton.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchSingleton.__getstate__" => "Helper for pickle.", + "_ast.MatchSingleton.__gt__" => "Return self>value.", + "_ast.MatchSingleton.__hash__" => "Return hash(self).", + "_ast.MatchSingleton.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchSingleton.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchSingleton.__le__" => "Return self<=value.", + "_ast.MatchSingleton.__lt__" => "Return self<value.", + "_ast.MatchSingleton.__ne__" => "Return self!=value.", + "_ast.MatchSingleton.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchSingleton.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchSingleton.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchSingleton.__repr__" => "Return repr(self).", + "_ast.MatchSingleton.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchSingleton.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchSingleton.__str__" => "Return str(self).", + "_ast.MatchSingleton.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchSingleton.__weakref__" => "list of weak references to the object", + "_ast.MatchStar" => "MatchStar(identifier? name)", + "_ast.MatchStar.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchStar.__eq__" => "Return self==value.", + "_ast.MatchStar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchStar.__ge__" => "Return self>=value.", + "_ast.MatchStar.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchStar.__getstate__" => "Helper for pickle.", + "_ast.MatchStar.__gt__" => "Return self>value.", + "_ast.MatchStar.__hash__" => "Return hash(self).", + "_ast.MatchStar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchStar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchStar.__le__" => "Return self<=value.", + "_ast.MatchStar.__lt__" => "Return self<value.", + "_ast.MatchStar.__ne__" => "Return self!=value.", + "_ast.MatchStar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchStar.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchStar.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchStar.__repr__" => "Return repr(self).", + "_ast.MatchStar.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchStar.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchStar.__str__" => "Return str(self).", + "_ast.MatchStar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchStar.__weakref__" => "list of weak references to the object", + "_ast.MatchValue" => "MatchValue(expr value)", + "_ast.MatchValue.__delattr__" => "Implement delattr(self, name).", + "_ast.MatchValue.__eq__" => "Return self==value.", + "_ast.MatchValue.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.MatchValue.__ge__" => "Return self>=value.", + "_ast.MatchValue.__getattribute__" => "Return getattr(self, name).", + "_ast.MatchValue.__getstate__" => "Helper for pickle.", + "_ast.MatchValue.__gt__" => "Return self>value.", + "_ast.MatchValue.__hash__" => "Return hash(self).", + "_ast.MatchValue.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.MatchValue.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.MatchValue.__le__" => "Return self<=value.", + "_ast.MatchValue.__lt__" => "Return self<value.", + "_ast.MatchValue.__ne__" => "Return self!=value.", + "_ast.MatchValue.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.MatchValue.__reduce_ex__" => "Helper for pickle.", + "_ast.MatchValue.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.MatchValue.__repr__" => "Return repr(self).", + "_ast.MatchValue.__setattr__" => "Implement setattr(self, name, value).", + "_ast.MatchValue.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.MatchValue.__str__" => "Return str(self).", + "_ast.MatchValue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.MatchValue.__weakref__" => "list of weak references to the object", + "_ast.Mod" => "Mod", + "_ast.Mod.__delattr__" => "Implement delattr(self, name).", + "_ast.Mod.__eq__" => "Return self==value.", + "_ast.Mod.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Mod.__ge__" => "Return self>=value.", + "_ast.Mod.__getattribute__" => "Return getattr(self, name).", + "_ast.Mod.__getstate__" => "Helper for pickle.", + "_ast.Mod.__gt__" => "Return self>value.", + "_ast.Mod.__hash__" => "Return hash(self).", + "_ast.Mod.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Mod.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Mod.__le__" => "Return self<=value.", + "_ast.Mod.__lt__" => "Return self<value.", + "_ast.Mod.__ne__" => "Return self!=value.", + "_ast.Mod.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Mod.__reduce_ex__" => "Helper for pickle.", + "_ast.Mod.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Mod.__repr__" => "Return repr(self).", + "_ast.Mod.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Mod.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Mod.__str__" => "Return str(self).", + "_ast.Mod.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Mod.__weakref__" => "list of weak references to the object", + "_ast.Module" => "Module(stmt* body, type_ignore* type_ignores)", + "_ast.Module.__delattr__" => "Implement delattr(self, name).", + "_ast.Module.__eq__" => "Return self==value.", + "_ast.Module.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Module.__ge__" => "Return self>=value.", + "_ast.Module.__getattribute__" => "Return getattr(self, name).", + "_ast.Module.__getstate__" => "Helper for pickle.", + "_ast.Module.__gt__" => "Return self>value.", + "_ast.Module.__hash__" => "Return hash(self).", + "_ast.Module.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Module.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Module.__le__" => "Return self<=value.", + "_ast.Module.__lt__" => "Return self<value.", + "_ast.Module.__ne__" => "Return self!=value.", + "_ast.Module.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Module.__reduce_ex__" => "Helper for pickle.", + "_ast.Module.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Module.__repr__" => "Return repr(self).", + "_ast.Module.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Module.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Module.__str__" => "Return str(self).", + "_ast.Module.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Module.__weakref__" => "list of weak references to the object", + "_ast.Mult" => "Mult", + "_ast.Mult.__delattr__" => "Implement delattr(self, name).", + "_ast.Mult.__eq__" => "Return self==value.", + "_ast.Mult.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Mult.__ge__" => "Return self>=value.", + "_ast.Mult.__getattribute__" => "Return getattr(self, name).", + "_ast.Mult.__getstate__" => "Helper for pickle.", + "_ast.Mult.__gt__" => "Return self>value.", + "_ast.Mult.__hash__" => "Return hash(self).", + "_ast.Mult.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Mult.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Mult.__le__" => "Return self<=value.", + "_ast.Mult.__lt__" => "Return self<value.", + "_ast.Mult.__ne__" => "Return self!=value.", + "_ast.Mult.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Mult.__reduce_ex__" => "Helper for pickle.", + "_ast.Mult.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Mult.__repr__" => "Return repr(self).", + "_ast.Mult.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Mult.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Mult.__str__" => "Return str(self).", + "_ast.Mult.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Mult.__weakref__" => "list of weak references to the object", + "_ast.Name" => "Name(identifier id, expr_context ctx)", + "_ast.Name.__delattr__" => "Implement delattr(self, name).", + "_ast.Name.__eq__" => "Return self==value.", + "_ast.Name.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Name.__ge__" => "Return self>=value.", + "_ast.Name.__getattribute__" => "Return getattr(self, name).", + "_ast.Name.__getstate__" => "Helper for pickle.", + "_ast.Name.__gt__" => "Return self>value.", + "_ast.Name.__hash__" => "Return hash(self).", + "_ast.Name.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Name.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Name.__le__" => "Return self<=value.", + "_ast.Name.__lt__" => "Return self<value.", + "_ast.Name.__ne__" => "Return self!=value.", + "_ast.Name.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Name.__reduce_ex__" => "Helper for pickle.", + "_ast.Name.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Name.__repr__" => "Return repr(self).", + "_ast.Name.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Name.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Name.__str__" => "Return str(self).", + "_ast.Name.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Name.__weakref__" => "list of weak references to the object", + "_ast.NamedExpr" => "NamedExpr(expr target, expr value)", + "_ast.NamedExpr.__delattr__" => "Implement delattr(self, name).", + "_ast.NamedExpr.__eq__" => "Return self==value.", + "_ast.NamedExpr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.NamedExpr.__ge__" => "Return self>=value.", + "_ast.NamedExpr.__getattribute__" => "Return getattr(self, name).", + "_ast.NamedExpr.__getstate__" => "Helper for pickle.", + "_ast.NamedExpr.__gt__" => "Return self>value.", + "_ast.NamedExpr.__hash__" => "Return hash(self).", + "_ast.NamedExpr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.NamedExpr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.NamedExpr.__le__" => "Return self<=value.", + "_ast.NamedExpr.__lt__" => "Return self<value.", + "_ast.NamedExpr.__ne__" => "Return self!=value.", + "_ast.NamedExpr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.NamedExpr.__reduce_ex__" => "Helper for pickle.", + "_ast.NamedExpr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.NamedExpr.__repr__" => "Return repr(self).", + "_ast.NamedExpr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.NamedExpr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.NamedExpr.__str__" => "Return str(self).", + "_ast.NamedExpr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.NamedExpr.__weakref__" => "list of weak references to the object", + "_ast.Nonlocal" => "Nonlocal(identifier* names)", + "_ast.Nonlocal.__delattr__" => "Implement delattr(self, name).", + "_ast.Nonlocal.__eq__" => "Return self==value.", + "_ast.Nonlocal.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Nonlocal.__ge__" => "Return self>=value.", + "_ast.Nonlocal.__getattribute__" => "Return getattr(self, name).", + "_ast.Nonlocal.__getstate__" => "Helper for pickle.", + "_ast.Nonlocal.__gt__" => "Return self>value.", + "_ast.Nonlocal.__hash__" => "Return hash(self).", + "_ast.Nonlocal.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Nonlocal.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Nonlocal.__le__" => "Return self<=value.", + "_ast.Nonlocal.__lt__" => "Return self<value.", + "_ast.Nonlocal.__ne__" => "Return self!=value.", + "_ast.Nonlocal.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Nonlocal.__reduce_ex__" => "Helper for pickle.", + "_ast.Nonlocal.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Nonlocal.__repr__" => "Return repr(self).", + "_ast.Nonlocal.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Nonlocal.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Nonlocal.__str__" => "Return str(self).", + "_ast.Nonlocal.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Nonlocal.__weakref__" => "list of weak references to the object", + "_ast.Not" => "Not", + "_ast.Not.__delattr__" => "Implement delattr(self, name).", + "_ast.Not.__eq__" => "Return self==value.", + "_ast.Not.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Not.__ge__" => "Return self>=value.", + "_ast.Not.__getattribute__" => "Return getattr(self, name).", + "_ast.Not.__getstate__" => "Helper for pickle.", + "_ast.Not.__gt__" => "Return self>value.", + "_ast.Not.__hash__" => "Return hash(self).", + "_ast.Not.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Not.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Not.__le__" => "Return self<=value.", + "_ast.Not.__lt__" => "Return self<value.", + "_ast.Not.__ne__" => "Return self!=value.", + "_ast.Not.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Not.__reduce_ex__" => "Helper for pickle.", + "_ast.Not.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Not.__repr__" => "Return repr(self).", + "_ast.Not.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Not.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Not.__str__" => "Return str(self).", + "_ast.Not.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Not.__weakref__" => "list of weak references to the object", + "_ast.NotEq" => "NotEq", + "_ast.NotEq.__delattr__" => "Implement delattr(self, name).", + "_ast.NotEq.__eq__" => "Return self==value.", + "_ast.NotEq.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.NotEq.__ge__" => "Return self>=value.", + "_ast.NotEq.__getattribute__" => "Return getattr(self, name).", + "_ast.NotEq.__getstate__" => "Helper for pickle.", + "_ast.NotEq.__gt__" => "Return self>value.", + "_ast.NotEq.__hash__" => "Return hash(self).", + "_ast.NotEq.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.NotEq.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.NotEq.__le__" => "Return self<=value.", + "_ast.NotEq.__lt__" => "Return self<value.", + "_ast.NotEq.__ne__" => "Return self!=value.", + "_ast.NotEq.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.NotEq.__reduce_ex__" => "Helper for pickle.", + "_ast.NotEq.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.NotEq.__repr__" => "Return repr(self).", + "_ast.NotEq.__setattr__" => "Implement setattr(self, name, value).", + "_ast.NotEq.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.NotEq.__str__" => "Return str(self).", + "_ast.NotEq.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.NotEq.__weakref__" => "list of weak references to the object", + "_ast.NotIn" => "NotIn", + "_ast.NotIn.__delattr__" => "Implement delattr(self, name).", + "_ast.NotIn.__eq__" => "Return self==value.", + "_ast.NotIn.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.NotIn.__ge__" => "Return self>=value.", + "_ast.NotIn.__getattribute__" => "Return getattr(self, name).", + "_ast.NotIn.__getstate__" => "Helper for pickle.", + "_ast.NotIn.__gt__" => "Return self>value.", + "_ast.NotIn.__hash__" => "Return hash(self).", + "_ast.NotIn.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.NotIn.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.NotIn.__le__" => "Return self<=value.", + "_ast.NotIn.__lt__" => "Return self<value.", + "_ast.NotIn.__ne__" => "Return self!=value.", + "_ast.NotIn.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.NotIn.__reduce_ex__" => "Helper for pickle.", + "_ast.NotIn.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.NotIn.__repr__" => "Return repr(self).", + "_ast.NotIn.__setattr__" => "Implement setattr(self, name, value).", + "_ast.NotIn.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.NotIn.__str__" => "Return str(self).", + "_ast.NotIn.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.NotIn.__weakref__" => "list of weak references to the object", + "_ast.Or" => "Or", + "_ast.Or.__delattr__" => "Implement delattr(self, name).", + "_ast.Or.__eq__" => "Return self==value.", + "_ast.Or.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Or.__ge__" => "Return self>=value.", + "_ast.Or.__getattribute__" => "Return getattr(self, name).", + "_ast.Or.__getstate__" => "Helper for pickle.", + "_ast.Or.__gt__" => "Return self>value.", + "_ast.Or.__hash__" => "Return hash(self).", + "_ast.Or.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Or.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Or.__le__" => "Return self<=value.", + "_ast.Or.__lt__" => "Return self<value.", + "_ast.Or.__ne__" => "Return self!=value.", + "_ast.Or.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Or.__reduce_ex__" => "Helper for pickle.", + "_ast.Or.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Or.__repr__" => "Return repr(self).", + "_ast.Or.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Or.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Or.__str__" => "Return str(self).", + "_ast.Or.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Or.__weakref__" => "list of weak references to the object", + "_ast.ParamSpec" => "ParamSpec(identifier name, expr? default_value)", + "_ast.ParamSpec.__delattr__" => "Implement delattr(self, name).", + "_ast.ParamSpec.__eq__" => "Return self==value.", + "_ast.ParamSpec.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.ParamSpec.__ge__" => "Return self>=value.", + "_ast.ParamSpec.__getattribute__" => "Return getattr(self, name).", + "_ast.ParamSpec.__getstate__" => "Helper for pickle.", + "_ast.ParamSpec.__gt__" => "Return self>value.", + "_ast.ParamSpec.__hash__" => "Return hash(self).", + "_ast.ParamSpec.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.ParamSpec.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.ParamSpec.__le__" => "Return self<=value.", + "_ast.ParamSpec.__lt__" => "Return self<value.", + "_ast.ParamSpec.__ne__" => "Return self!=value.", + "_ast.ParamSpec.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.ParamSpec.__reduce_ex__" => "Helper for pickle.", + "_ast.ParamSpec.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.ParamSpec.__repr__" => "Return repr(self).", + "_ast.ParamSpec.__setattr__" => "Implement setattr(self, name, value).", + "_ast.ParamSpec.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.ParamSpec.__str__" => "Return str(self).", + "_ast.ParamSpec.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.ParamSpec.__weakref__" => "list of weak references to the object", + "_ast.Pass" => "Pass", + "_ast.Pass.__delattr__" => "Implement delattr(self, name).", + "_ast.Pass.__eq__" => "Return self==value.", + "_ast.Pass.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Pass.__ge__" => "Return self>=value.", + "_ast.Pass.__getattribute__" => "Return getattr(self, name).", + "_ast.Pass.__getstate__" => "Helper for pickle.", + "_ast.Pass.__gt__" => "Return self>value.", + "_ast.Pass.__hash__" => "Return hash(self).", + "_ast.Pass.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Pass.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Pass.__le__" => "Return self<=value.", + "_ast.Pass.__lt__" => "Return self<value.", + "_ast.Pass.__ne__" => "Return self!=value.", + "_ast.Pass.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Pass.__reduce_ex__" => "Helper for pickle.", + "_ast.Pass.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Pass.__repr__" => "Return repr(self).", + "_ast.Pass.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Pass.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Pass.__str__" => "Return str(self).", + "_ast.Pass.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Pass.__weakref__" => "list of weak references to the object", + "_ast.Pow" => "Pow", + "_ast.Pow.__delattr__" => "Implement delattr(self, name).", + "_ast.Pow.__eq__" => "Return self==value.", + "_ast.Pow.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Pow.__ge__" => "Return self>=value.", + "_ast.Pow.__getattribute__" => "Return getattr(self, name).", + "_ast.Pow.__getstate__" => "Helper for pickle.", + "_ast.Pow.__gt__" => "Return self>value.", + "_ast.Pow.__hash__" => "Return hash(self).", + "_ast.Pow.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Pow.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Pow.__le__" => "Return self<=value.", + "_ast.Pow.__lt__" => "Return self<value.", + "_ast.Pow.__ne__" => "Return self!=value.", + "_ast.Pow.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Pow.__reduce_ex__" => "Helper for pickle.", + "_ast.Pow.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Pow.__repr__" => "Return repr(self).", + "_ast.Pow.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Pow.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Pow.__str__" => "Return str(self).", + "_ast.Pow.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Pow.__weakref__" => "list of weak references to the object", + "_ast.RShift" => "RShift", + "_ast.RShift.__delattr__" => "Implement delattr(self, name).", + "_ast.RShift.__eq__" => "Return self==value.", + "_ast.RShift.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.RShift.__ge__" => "Return self>=value.", + "_ast.RShift.__getattribute__" => "Return getattr(self, name).", + "_ast.RShift.__getstate__" => "Helper for pickle.", + "_ast.RShift.__gt__" => "Return self>value.", + "_ast.RShift.__hash__" => "Return hash(self).", + "_ast.RShift.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.RShift.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.RShift.__le__" => "Return self<=value.", + "_ast.RShift.__lt__" => "Return self<value.", + "_ast.RShift.__ne__" => "Return self!=value.", + "_ast.RShift.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.RShift.__reduce_ex__" => "Helper for pickle.", + "_ast.RShift.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.RShift.__repr__" => "Return repr(self).", + "_ast.RShift.__setattr__" => "Implement setattr(self, name, value).", + "_ast.RShift.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.RShift.__str__" => "Return str(self).", + "_ast.RShift.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.RShift.__weakref__" => "list of weak references to the object", + "_ast.Raise" => "Raise(expr? exc, expr? cause)", + "_ast.Raise.__delattr__" => "Implement delattr(self, name).", + "_ast.Raise.__eq__" => "Return self==value.", + "_ast.Raise.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Raise.__ge__" => "Return self>=value.", + "_ast.Raise.__getattribute__" => "Return getattr(self, name).", + "_ast.Raise.__getstate__" => "Helper for pickle.", + "_ast.Raise.__gt__" => "Return self>value.", + "_ast.Raise.__hash__" => "Return hash(self).", + "_ast.Raise.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Raise.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Raise.__le__" => "Return self<=value.", + "_ast.Raise.__lt__" => "Return self<value.", + "_ast.Raise.__ne__" => "Return self!=value.", + "_ast.Raise.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Raise.__reduce_ex__" => "Helper for pickle.", + "_ast.Raise.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Raise.__repr__" => "Return repr(self).", + "_ast.Raise.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Raise.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Raise.__str__" => "Return str(self).", + "_ast.Raise.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Raise.__weakref__" => "list of weak references to the object", + "_ast.Return" => "Return(expr? value)", + "_ast.Return.__delattr__" => "Implement delattr(self, name).", + "_ast.Return.__eq__" => "Return self==value.", + "_ast.Return.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Return.__ge__" => "Return self>=value.", + "_ast.Return.__getattribute__" => "Return getattr(self, name).", + "_ast.Return.__getstate__" => "Helper for pickle.", + "_ast.Return.__gt__" => "Return self>value.", + "_ast.Return.__hash__" => "Return hash(self).", + "_ast.Return.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Return.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Return.__le__" => "Return self<=value.", + "_ast.Return.__lt__" => "Return self<value.", + "_ast.Return.__ne__" => "Return self!=value.", + "_ast.Return.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Return.__reduce_ex__" => "Helper for pickle.", + "_ast.Return.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Return.__repr__" => "Return repr(self).", + "_ast.Return.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Return.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Return.__str__" => "Return str(self).", + "_ast.Return.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Return.__weakref__" => "list of weak references to the object", + "_ast.Set" => "Set(expr* elts)", + "_ast.Set.__delattr__" => "Implement delattr(self, name).", + "_ast.Set.__eq__" => "Return self==value.", + "_ast.Set.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Set.__ge__" => "Return self>=value.", + "_ast.Set.__getattribute__" => "Return getattr(self, name).", + "_ast.Set.__getstate__" => "Helper for pickle.", + "_ast.Set.__gt__" => "Return self>value.", + "_ast.Set.__hash__" => "Return hash(self).", + "_ast.Set.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Set.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Set.__le__" => "Return self<=value.", + "_ast.Set.__lt__" => "Return self<value.", + "_ast.Set.__ne__" => "Return self!=value.", + "_ast.Set.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Set.__reduce_ex__" => "Helper for pickle.", + "_ast.Set.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Set.__repr__" => "Return repr(self).", + "_ast.Set.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Set.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Set.__str__" => "Return str(self).", + "_ast.Set.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Set.__weakref__" => "list of weak references to the object", + "_ast.SetComp" => "SetComp(expr elt, comprehension* generators)", + "_ast.SetComp.__delattr__" => "Implement delattr(self, name).", + "_ast.SetComp.__eq__" => "Return self==value.", + "_ast.SetComp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.SetComp.__ge__" => "Return self>=value.", + "_ast.SetComp.__getattribute__" => "Return getattr(self, name).", + "_ast.SetComp.__getstate__" => "Helper for pickle.", + "_ast.SetComp.__gt__" => "Return self>value.", + "_ast.SetComp.__hash__" => "Return hash(self).", + "_ast.SetComp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.SetComp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.SetComp.__le__" => "Return self<=value.", + "_ast.SetComp.__lt__" => "Return self<value.", + "_ast.SetComp.__ne__" => "Return self!=value.", + "_ast.SetComp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.SetComp.__reduce_ex__" => "Helper for pickle.", + "_ast.SetComp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.SetComp.__repr__" => "Return repr(self).", + "_ast.SetComp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.SetComp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.SetComp.__str__" => "Return str(self).", + "_ast.SetComp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.SetComp.__weakref__" => "list of weak references to the object", + "_ast.Slice" => "Slice(expr? lower, expr? upper, expr? step)", + "_ast.Slice.__delattr__" => "Implement delattr(self, name).", + "_ast.Slice.__eq__" => "Return self==value.", + "_ast.Slice.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Slice.__ge__" => "Return self>=value.", + "_ast.Slice.__getattribute__" => "Return getattr(self, name).", + "_ast.Slice.__getstate__" => "Helper for pickle.", + "_ast.Slice.__gt__" => "Return self>value.", + "_ast.Slice.__hash__" => "Return hash(self).", + "_ast.Slice.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Slice.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Slice.__le__" => "Return self<=value.", + "_ast.Slice.__lt__" => "Return self<value.", + "_ast.Slice.__ne__" => "Return self!=value.", + "_ast.Slice.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Slice.__reduce_ex__" => "Helper for pickle.", + "_ast.Slice.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Slice.__repr__" => "Return repr(self).", + "_ast.Slice.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Slice.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Slice.__str__" => "Return str(self).", + "_ast.Slice.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Slice.__weakref__" => "list of weak references to the object", + "_ast.Starred" => "Starred(expr value, expr_context ctx)", + "_ast.Starred.__delattr__" => "Implement delattr(self, name).", + "_ast.Starred.__eq__" => "Return self==value.", + "_ast.Starred.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Starred.__ge__" => "Return self>=value.", + "_ast.Starred.__getattribute__" => "Return getattr(self, name).", + "_ast.Starred.__getstate__" => "Helper for pickle.", + "_ast.Starred.__gt__" => "Return self>value.", + "_ast.Starred.__hash__" => "Return hash(self).", + "_ast.Starred.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Starred.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Starred.__le__" => "Return self<=value.", + "_ast.Starred.__lt__" => "Return self<value.", + "_ast.Starred.__ne__" => "Return self!=value.", + "_ast.Starred.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Starred.__reduce_ex__" => "Helper for pickle.", + "_ast.Starred.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Starred.__repr__" => "Return repr(self).", + "_ast.Starred.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Starred.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Starred.__str__" => "Return str(self).", + "_ast.Starred.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Starred.__weakref__" => "list of weak references to the object", + "_ast.Store" => "Store", + "_ast.Store.__delattr__" => "Implement delattr(self, name).", + "_ast.Store.__eq__" => "Return self==value.", + "_ast.Store.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Store.__ge__" => "Return self>=value.", + "_ast.Store.__getattribute__" => "Return getattr(self, name).", + "_ast.Store.__getstate__" => "Helper for pickle.", + "_ast.Store.__gt__" => "Return self>value.", + "_ast.Store.__hash__" => "Return hash(self).", + "_ast.Store.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Store.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Store.__le__" => "Return self<=value.", + "_ast.Store.__lt__" => "Return self<value.", + "_ast.Store.__ne__" => "Return self!=value.", + "_ast.Store.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Store.__reduce_ex__" => "Helper for pickle.", + "_ast.Store.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Store.__repr__" => "Return repr(self).", + "_ast.Store.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Store.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Store.__str__" => "Return str(self).", + "_ast.Store.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Store.__weakref__" => "list of weak references to the object", + "_ast.Sub" => "Sub", + "_ast.Sub.__delattr__" => "Implement delattr(self, name).", + "_ast.Sub.__eq__" => "Return self==value.", + "_ast.Sub.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Sub.__ge__" => "Return self>=value.", + "_ast.Sub.__getattribute__" => "Return getattr(self, name).", + "_ast.Sub.__getstate__" => "Helper for pickle.", + "_ast.Sub.__gt__" => "Return self>value.", + "_ast.Sub.__hash__" => "Return hash(self).", + "_ast.Sub.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Sub.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Sub.__le__" => "Return self<=value.", + "_ast.Sub.__lt__" => "Return self<value.", + "_ast.Sub.__ne__" => "Return self!=value.", + "_ast.Sub.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Sub.__reduce_ex__" => "Helper for pickle.", + "_ast.Sub.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Sub.__repr__" => "Return repr(self).", + "_ast.Sub.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Sub.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Sub.__str__" => "Return str(self).", + "_ast.Sub.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Sub.__weakref__" => "list of weak references to the object", + "_ast.Subscript" => "Subscript(expr value, expr slice, expr_context ctx)", + "_ast.Subscript.__delattr__" => "Implement delattr(self, name).", + "_ast.Subscript.__eq__" => "Return self==value.", + "_ast.Subscript.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Subscript.__ge__" => "Return self>=value.", + "_ast.Subscript.__getattribute__" => "Return getattr(self, name).", + "_ast.Subscript.__getstate__" => "Helper for pickle.", + "_ast.Subscript.__gt__" => "Return self>value.", + "_ast.Subscript.__hash__" => "Return hash(self).", + "_ast.Subscript.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Subscript.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Subscript.__le__" => "Return self<=value.", + "_ast.Subscript.__lt__" => "Return self<value.", + "_ast.Subscript.__ne__" => "Return self!=value.", + "_ast.Subscript.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Subscript.__reduce_ex__" => "Helper for pickle.", + "_ast.Subscript.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Subscript.__repr__" => "Return repr(self).", + "_ast.Subscript.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Subscript.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Subscript.__str__" => "Return str(self).", + "_ast.Subscript.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Subscript.__weakref__" => "list of weak references to the object", + "_ast.TemplateStr" => "TemplateStr(expr* values)", + "_ast.TemplateStr.__delattr__" => "Implement delattr(self, name).", + "_ast.TemplateStr.__eq__" => "Return self==value.", + "_ast.TemplateStr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TemplateStr.__ge__" => "Return self>=value.", + "_ast.TemplateStr.__getattribute__" => "Return getattr(self, name).", + "_ast.TemplateStr.__getstate__" => "Helper for pickle.", + "_ast.TemplateStr.__gt__" => "Return self>value.", + "_ast.TemplateStr.__hash__" => "Return hash(self).", + "_ast.TemplateStr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TemplateStr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TemplateStr.__le__" => "Return self<=value.", + "_ast.TemplateStr.__lt__" => "Return self<value.", + "_ast.TemplateStr.__ne__" => "Return self!=value.", + "_ast.TemplateStr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TemplateStr.__reduce_ex__" => "Helper for pickle.", + "_ast.TemplateStr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TemplateStr.__repr__" => "Return repr(self).", + "_ast.TemplateStr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TemplateStr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TemplateStr.__str__" => "Return str(self).", + "_ast.TemplateStr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TemplateStr.__weakref__" => "list of weak references to the object", + "_ast.Try" => "Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)", + "_ast.Try.__delattr__" => "Implement delattr(self, name).", + "_ast.Try.__eq__" => "Return self==value.", + "_ast.Try.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Try.__ge__" => "Return self>=value.", + "_ast.Try.__getattribute__" => "Return getattr(self, name).", + "_ast.Try.__getstate__" => "Helper for pickle.", + "_ast.Try.__gt__" => "Return self>value.", + "_ast.Try.__hash__" => "Return hash(self).", + "_ast.Try.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Try.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Try.__le__" => "Return self<=value.", + "_ast.Try.__lt__" => "Return self<value.", + "_ast.Try.__ne__" => "Return self!=value.", + "_ast.Try.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Try.__reduce_ex__" => "Helper for pickle.", + "_ast.Try.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Try.__repr__" => "Return repr(self).", + "_ast.Try.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Try.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Try.__str__" => "Return str(self).", + "_ast.Try.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Try.__weakref__" => "list of weak references to the object", + "_ast.TryStar" => "TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)", + "_ast.TryStar.__delattr__" => "Implement delattr(self, name).", + "_ast.TryStar.__eq__" => "Return self==value.", + "_ast.TryStar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TryStar.__ge__" => "Return self>=value.", + "_ast.TryStar.__getattribute__" => "Return getattr(self, name).", + "_ast.TryStar.__getstate__" => "Helper for pickle.", + "_ast.TryStar.__gt__" => "Return self>value.", + "_ast.TryStar.__hash__" => "Return hash(self).", + "_ast.TryStar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TryStar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TryStar.__le__" => "Return self<=value.", + "_ast.TryStar.__lt__" => "Return self<value.", + "_ast.TryStar.__ne__" => "Return self!=value.", + "_ast.TryStar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TryStar.__reduce_ex__" => "Helper for pickle.", + "_ast.TryStar.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TryStar.__repr__" => "Return repr(self).", + "_ast.TryStar.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TryStar.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TryStar.__str__" => "Return str(self).", + "_ast.TryStar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TryStar.__weakref__" => "list of weak references to the object", + "_ast.Tuple" => "Tuple(expr* elts, expr_context ctx)", + "_ast.Tuple.__delattr__" => "Implement delattr(self, name).", + "_ast.Tuple.__eq__" => "Return self==value.", + "_ast.Tuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Tuple.__ge__" => "Return self>=value.", + "_ast.Tuple.__getattribute__" => "Return getattr(self, name).", + "_ast.Tuple.__getstate__" => "Helper for pickle.", + "_ast.Tuple.__gt__" => "Return self>value.", + "_ast.Tuple.__hash__" => "Return hash(self).", + "_ast.Tuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Tuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Tuple.__le__" => "Return self<=value.", + "_ast.Tuple.__lt__" => "Return self<value.", + "_ast.Tuple.__ne__" => "Return self!=value.", + "_ast.Tuple.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Tuple.__reduce_ex__" => "Helper for pickle.", + "_ast.Tuple.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Tuple.__repr__" => "Return repr(self).", + "_ast.Tuple.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Tuple.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Tuple.__str__" => "Return str(self).", + "_ast.Tuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Tuple.__weakref__" => "list of weak references to the object", + "_ast.Tuple.dims" => "Deprecated. Use elts instead.", + "_ast.TypeAlias" => "TypeAlias(expr name, type_param* type_params, expr value)", + "_ast.TypeAlias.__delattr__" => "Implement delattr(self, name).", + "_ast.TypeAlias.__eq__" => "Return self==value.", + "_ast.TypeAlias.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TypeAlias.__ge__" => "Return self>=value.", + "_ast.TypeAlias.__getattribute__" => "Return getattr(self, name).", + "_ast.TypeAlias.__getstate__" => "Helper for pickle.", + "_ast.TypeAlias.__gt__" => "Return self>value.", + "_ast.TypeAlias.__hash__" => "Return hash(self).", + "_ast.TypeAlias.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TypeAlias.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TypeAlias.__le__" => "Return self<=value.", + "_ast.TypeAlias.__lt__" => "Return self<value.", + "_ast.TypeAlias.__ne__" => "Return self!=value.", + "_ast.TypeAlias.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TypeAlias.__reduce_ex__" => "Helper for pickle.", + "_ast.TypeAlias.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TypeAlias.__repr__" => "Return repr(self).", + "_ast.TypeAlias.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TypeAlias.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TypeAlias.__str__" => "Return str(self).", + "_ast.TypeAlias.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TypeAlias.__weakref__" => "list of weak references to the object", + "_ast.TypeIgnore" => "TypeIgnore(int lineno, string tag)", + "_ast.TypeIgnore.__delattr__" => "Implement delattr(self, name).", + "_ast.TypeIgnore.__eq__" => "Return self==value.", + "_ast.TypeIgnore.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TypeIgnore.__ge__" => "Return self>=value.", + "_ast.TypeIgnore.__getattribute__" => "Return getattr(self, name).", + "_ast.TypeIgnore.__getstate__" => "Helper for pickle.", + "_ast.TypeIgnore.__gt__" => "Return self>value.", + "_ast.TypeIgnore.__hash__" => "Return hash(self).", + "_ast.TypeIgnore.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TypeIgnore.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TypeIgnore.__le__" => "Return self<=value.", + "_ast.TypeIgnore.__lt__" => "Return self<value.", + "_ast.TypeIgnore.__ne__" => "Return self!=value.", + "_ast.TypeIgnore.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TypeIgnore.__reduce_ex__" => "Helper for pickle.", + "_ast.TypeIgnore.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TypeIgnore.__repr__" => "Return repr(self).", + "_ast.TypeIgnore.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TypeIgnore.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TypeIgnore.__str__" => "Return str(self).", + "_ast.TypeIgnore.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TypeIgnore.__weakref__" => "list of weak references to the object", + "_ast.TypeVar" => "TypeVar(identifier name, expr? bound, expr? default_value)", + "_ast.TypeVar.__delattr__" => "Implement delattr(self, name).", + "_ast.TypeVar.__eq__" => "Return self==value.", + "_ast.TypeVar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TypeVar.__ge__" => "Return self>=value.", + "_ast.TypeVar.__getattribute__" => "Return getattr(self, name).", + "_ast.TypeVar.__getstate__" => "Helper for pickle.", + "_ast.TypeVar.__gt__" => "Return self>value.", + "_ast.TypeVar.__hash__" => "Return hash(self).", + "_ast.TypeVar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TypeVar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TypeVar.__le__" => "Return self<=value.", + "_ast.TypeVar.__lt__" => "Return self<value.", + "_ast.TypeVar.__ne__" => "Return self!=value.", + "_ast.TypeVar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TypeVar.__reduce_ex__" => "Helper for pickle.", + "_ast.TypeVar.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TypeVar.__repr__" => "Return repr(self).", + "_ast.TypeVar.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TypeVar.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TypeVar.__str__" => "Return str(self).", + "_ast.TypeVar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TypeVar.__weakref__" => "list of weak references to the object", + "_ast.TypeVarTuple" => "TypeVarTuple(identifier name, expr? default_value)", + "_ast.TypeVarTuple.__delattr__" => "Implement delattr(self, name).", + "_ast.TypeVarTuple.__eq__" => "Return self==value.", + "_ast.TypeVarTuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.TypeVarTuple.__ge__" => "Return self>=value.", + "_ast.TypeVarTuple.__getattribute__" => "Return getattr(self, name).", + "_ast.TypeVarTuple.__getstate__" => "Helper for pickle.", + "_ast.TypeVarTuple.__gt__" => "Return self>value.", + "_ast.TypeVarTuple.__hash__" => "Return hash(self).", + "_ast.TypeVarTuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.TypeVarTuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.TypeVarTuple.__le__" => "Return self<=value.", + "_ast.TypeVarTuple.__lt__" => "Return self<value.", + "_ast.TypeVarTuple.__ne__" => "Return self!=value.", + "_ast.TypeVarTuple.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.TypeVarTuple.__reduce_ex__" => "Helper for pickle.", + "_ast.TypeVarTuple.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.TypeVarTuple.__repr__" => "Return repr(self).", + "_ast.TypeVarTuple.__setattr__" => "Implement setattr(self, name, value).", + "_ast.TypeVarTuple.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.TypeVarTuple.__str__" => "Return str(self).", + "_ast.TypeVarTuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.TypeVarTuple.__weakref__" => "list of weak references to the object", + "_ast.UAdd" => "UAdd", + "_ast.UAdd.__delattr__" => "Implement delattr(self, name).", + "_ast.UAdd.__eq__" => "Return self==value.", + "_ast.UAdd.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.UAdd.__ge__" => "Return self>=value.", + "_ast.UAdd.__getattribute__" => "Return getattr(self, name).", + "_ast.UAdd.__getstate__" => "Helper for pickle.", + "_ast.UAdd.__gt__" => "Return self>value.", + "_ast.UAdd.__hash__" => "Return hash(self).", + "_ast.UAdd.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.UAdd.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.UAdd.__le__" => "Return self<=value.", + "_ast.UAdd.__lt__" => "Return self<value.", + "_ast.UAdd.__ne__" => "Return self!=value.", + "_ast.UAdd.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.UAdd.__reduce_ex__" => "Helper for pickle.", + "_ast.UAdd.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.UAdd.__repr__" => "Return repr(self).", + "_ast.UAdd.__setattr__" => "Implement setattr(self, name, value).", + "_ast.UAdd.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.UAdd.__str__" => "Return str(self).", + "_ast.UAdd.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.UAdd.__weakref__" => "list of weak references to the object", + "_ast.USub" => "USub", + "_ast.USub.__delattr__" => "Implement delattr(self, name).", + "_ast.USub.__eq__" => "Return self==value.", + "_ast.USub.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.USub.__ge__" => "Return self>=value.", + "_ast.USub.__getattribute__" => "Return getattr(self, name).", + "_ast.USub.__getstate__" => "Helper for pickle.", + "_ast.USub.__gt__" => "Return self>value.", + "_ast.USub.__hash__" => "Return hash(self).", + "_ast.USub.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.USub.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.USub.__le__" => "Return self<=value.", + "_ast.USub.__lt__" => "Return self<value.", + "_ast.USub.__ne__" => "Return self!=value.", + "_ast.USub.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.USub.__reduce_ex__" => "Helper for pickle.", + "_ast.USub.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.USub.__repr__" => "Return repr(self).", + "_ast.USub.__setattr__" => "Implement setattr(self, name, value).", + "_ast.USub.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.USub.__str__" => "Return str(self).", + "_ast.USub.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.USub.__weakref__" => "list of weak references to the object", + "_ast.UnaryOp" => "UnaryOp(unaryop op, expr operand)", + "_ast.UnaryOp.__delattr__" => "Implement delattr(self, name).", + "_ast.UnaryOp.__eq__" => "Return self==value.", + "_ast.UnaryOp.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.UnaryOp.__ge__" => "Return self>=value.", + "_ast.UnaryOp.__getattribute__" => "Return getattr(self, name).", + "_ast.UnaryOp.__getstate__" => "Helper for pickle.", + "_ast.UnaryOp.__gt__" => "Return self>value.", + "_ast.UnaryOp.__hash__" => "Return hash(self).", + "_ast.UnaryOp.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.UnaryOp.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.UnaryOp.__le__" => "Return self<=value.", + "_ast.UnaryOp.__lt__" => "Return self<value.", + "_ast.UnaryOp.__ne__" => "Return self!=value.", + "_ast.UnaryOp.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.UnaryOp.__reduce_ex__" => "Helper for pickle.", + "_ast.UnaryOp.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.UnaryOp.__repr__" => "Return repr(self).", + "_ast.UnaryOp.__setattr__" => "Implement setattr(self, name, value).", + "_ast.UnaryOp.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.UnaryOp.__str__" => "Return str(self).", + "_ast.UnaryOp.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.UnaryOp.__weakref__" => "list of weak references to the object", + "_ast.While" => "While(expr test, stmt* body, stmt* orelse)", + "_ast.While.__delattr__" => "Implement delattr(self, name).", + "_ast.While.__eq__" => "Return self==value.", + "_ast.While.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.While.__ge__" => "Return self>=value.", + "_ast.While.__getattribute__" => "Return getattr(self, name).", + "_ast.While.__getstate__" => "Helper for pickle.", + "_ast.While.__gt__" => "Return self>value.", + "_ast.While.__hash__" => "Return hash(self).", + "_ast.While.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.While.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.While.__le__" => "Return self<=value.", + "_ast.While.__lt__" => "Return self<value.", + "_ast.While.__ne__" => "Return self!=value.", + "_ast.While.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.While.__reduce_ex__" => "Helper for pickle.", + "_ast.While.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.While.__repr__" => "Return repr(self).", + "_ast.While.__setattr__" => "Implement setattr(self, name, value).", + "_ast.While.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.While.__str__" => "Return str(self).", + "_ast.While.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.While.__weakref__" => "list of weak references to the object", + "_ast.With" => "With(withitem* items, stmt* body, string? type_comment)", + "_ast.With.__delattr__" => "Implement delattr(self, name).", + "_ast.With.__eq__" => "Return self==value.", + "_ast.With.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.With.__ge__" => "Return self>=value.", + "_ast.With.__getattribute__" => "Return getattr(self, name).", + "_ast.With.__getstate__" => "Helper for pickle.", + "_ast.With.__gt__" => "Return self>value.", + "_ast.With.__hash__" => "Return hash(self).", + "_ast.With.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.With.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.With.__le__" => "Return self<=value.", + "_ast.With.__lt__" => "Return self<value.", + "_ast.With.__ne__" => "Return self!=value.", + "_ast.With.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.With.__reduce_ex__" => "Helper for pickle.", + "_ast.With.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.With.__repr__" => "Return repr(self).", + "_ast.With.__setattr__" => "Implement setattr(self, name, value).", + "_ast.With.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.With.__str__" => "Return str(self).", + "_ast.With.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.With.__weakref__" => "list of weak references to the object", + "_ast.Yield" => "Yield(expr? value)", + "_ast.Yield.__delattr__" => "Implement delattr(self, name).", + "_ast.Yield.__eq__" => "Return self==value.", + "_ast.Yield.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.Yield.__ge__" => "Return self>=value.", + "_ast.Yield.__getattribute__" => "Return getattr(self, name).", + "_ast.Yield.__getstate__" => "Helper for pickle.", + "_ast.Yield.__gt__" => "Return self>value.", + "_ast.Yield.__hash__" => "Return hash(self).", + "_ast.Yield.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.Yield.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.Yield.__le__" => "Return self<=value.", + "_ast.Yield.__lt__" => "Return self<value.", + "_ast.Yield.__ne__" => "Return self!=value.", + "_ast.Yield.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.Yield.__reduce_ex__" => "Helper for pickle.", + "_ast.Yield.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.Yield.__repr__" => "Return repr(self).", + "_ast.Yield.__setattr__" => "Implement setattr(self, name, value).", + "_ast.Yield.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.Yield.__str__" => "Return str(self).", + "_ast.Yield.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.Yield.__weakref__" => "list of weak references to the object", + "_ast.YieldFrom" => "YieldFrom(expr value)", + "_ast.YieldFrom.__delattr__" => "Implement delattr(self, name).", + "_ast.YieldFrom.__eq__" => "Return self==value.", + "_ast.YieldFrom.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.YieldFrom.__ge__" => "Return self>=value.", + "_ast.YieldFrom.__getattribute__" => "Return getattr(self, name).", + "_ast.YieldFrom.__getstate__" => "Helper for pickle.", + "_ast.YieldFrom.__gt__" => "Return self>value.", + "_ast.YieldFrom.__hash__" => "Return hash(self).", + "_ast.YieldFrom.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.YieldFrom.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.YieldFrom.__le__" => "Return self<=value.", + "_ast.YieldFrom.__lt__" => "Return self<value.", + "_ast.YieldFrom.__ne__" => "Return self!=value.", + "_ast.YieldFrom.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.YieldFrom.__reduce_ex__" => "Helper for pickle.", + "_ast.YieldFrom.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.YieldFrom.__repr__" => "Return repr(self).", + "_ast.YieldFrom.__setattr__" => "Implement setattr(self, name, value).", + "_ast.YieldFrom.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.YieldFrom.__str__" => "Return str(self).", + "_ast.YieldFrom.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.YieldFrom.__weakref__" => "list of weak references to the object", + "_ast.alias" => "alias(identifier name, identifier? asname)", + "_ast.alias.__delattr__" => "Implement delattr(self, name).", + "_ast.alias.__eq__" => "Return self==value.", + "_ast.alias.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.alias.__ge__" => "Return self>=value.", + "_ast.alias.__getattribute__" => "Return getattr(self, name).", + "_ast.alias.__getstate__" => "Helper for pickle.", + "_ast.alias.__gt__" => "Return self>value.", + "_ast.alias.__hash__" => "Return hash(self).", + "_ast.alias.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.alias.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.alias.__le__" => "Return self<=value.", + "_ast.alias.__lt__" => "Return self<value.", + "_ast.alias.__ne__" => "Return self!=value.", + "_ast.alias.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.alias.__reduce_ex__" => "Helper for pickle.", + "_ast.alias.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.alias.__repr__" => "Return repr(self).", + "_ast.alias.__setattr__" => "Implement setattr(self, name, value).", + "_ast.alias.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.alias.__str__" => "Return str(self).", + "_ast.alias.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.alias.__weakref__" => "list of weak references to the object", + "_ast.arg" => "arg(identifier arg, expr? annotation, string? type_comment)", + "_ast.arg.__delattr__" => "Implement delattr(self, name).", + "_ast.arg.__eq__" => "Return self==value.", + "_ast.arg.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.arg.__ge__" => "Return self>=value.", + "_ast.arg.__getattribute__" => "Return getattr(self, name).", + "_ast.arg.__getstate__" => "Helper for pickle.", + "_ast.arg.__gt__" => "Return self>value.", + "_ast.arg.__hash__" => "Return hash(self).", + "_ast.arg.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.arg.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.arg.__le__" => "Return self<=value.", + "_ast.arg.__lt__" => "Return self<value.", + "_ast.arg.__ne__" => "Return self!=value.", + "_ast.arg.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.arg.__reduce_ex__" => "Helper for pickle.", + "_ast.arg.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.arg.__repr__" => "Return repr(self).", + "_ast.arg.__setattr__" => "Implement setattr(self, name, value).", + "_ast.arg.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.arg.__str__" => "Return str(self).", + "_ast.arg.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.arg.__weakref__" => "list of weak references to the object", + "_ast.arguments" => "arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr?* kw_defaults, arg? kwarg, expr* defaults)", + "_ast.arguments.__delattr__" => "Implement delattr(self, name).", + "_ast.arguments.__eq__" => "Return self==value.", + "_ast.arguments.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.arguments.__ge__" => "Return self>=value.", + "_ast.arguments.__getattribute__" => "Return getattr(self, name).", + "_ast.arguments.__getstate__" => "Helper for pickle.", + "_ast.arguments.__gt__" => "Return self>value.", + "_ast.arguments.__hash__" => "Return hash(self).", + "_ast.arguments.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.arguments.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.arguments.__le__" => "Return self<=value.", + "_ast.arguments.__lt__" => "Return self<value.", + "_ast.arguments.__ne__" => "Return self!=value.", + "_ast.arguments.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.arguments.__reduce_ex__" => "Helper for pickle.", + "_ast.arguments.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.arguments.__repr__" => "Return repr(self).", + "_ast.arguments.__setattr__" => "Implement setattr(self, name, value).", + "_ast.arguments.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.arguments.__str__" => "Return str(self).", + "_ast.arguments.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.arguments.__weakref__" => "list of weak references to the object", + "_ast.boolop" => "boolop = And | Or", + "_ast.boolop.__delattr__" => "Implement delattr(self, name).", + "_ast.boolop.__eq__" => "Return self==value.", + "_ast.boolop.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.boolop.__ge__" => "Return self>=value.", + "_ast.boolop.__getattribute__" => "Return getattr(self, name).", + "_ast.boolop.__getstate__" => "Helper for pickle.", + "_ast.boolop.__gt__" => "Return self>value.", + "_ast.boolop.__hash__" => "Return hash(self).", + "_ast.boolop.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.boolop.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.boolop.__le__" => "Return self<=value.", + "_ast.boolop.__lt__" => "Return self<value.", + "_ast.boolop.__ne__" => "Return self!=value.", + "_ast.boolop.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.boolop.__reduce_ex__" => "Helper for pickle.", + "_ast.boolop.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.boolop.__repr__" => "Return repr(self).", + "_ast.boolop.__setattr__" => "Implement setattr(self, name, value).", + "_ast.boolop.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.boolop.__str__" => "Return str(self).", + "_ast.boolop.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.boolop.__weakref__" => "list of weak references to the object", + "_ast.cmpop" => "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn", + "_ast.cmpop.__delattr__" => "Implement delattr(self, name).", + "_ast.cmpop.__eq__" => "Return self==value.", + "_ast.cmpop.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.cmpop.__ge__" => "Return self>=value.", + "_ast.cmpop.__getattribute__" => "Return getattr(self, name).", + "_ast.cmpop.__getstate__" => "Helper for pickle.", + "_ast.cmpop.__gt__" => "Return self>value.", + "_ast.cmpop.__hash__" => "Return hash(self).", + "_ast.cmpop.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.cmpop.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.cmpop.__le__" => "Return self<=value.", + "_ast.cmpop.__lt__" => "Return self<value.", + "_ast.cmpop.__ne__" => "Return self!=value.", + "_ast.cmpop.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.cmpop.__reduce_ex__" => "Helper for pickle.", + "_ast.cmpop.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.cmpop.__repr__" => "Return repr(self).", + "_ast.cmpop.__setattr__" => "Implement setattr(self, name, value).", + "_ast.cmpop.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.cmpop.__str__" => "Return str(self).", + "_ast.cmpop.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.cmpop.__weakref__" => "list of weak references to the object", + "_ast.comprehension" => "comprehension(expr target, expr iter, expr* ifs, int is_async)", + "_ast.comprehension.__delattr__" => "Implement delattr(self, name).", + "_ast.comprehension.__eq__" => "Return self==value.", + "_ast.comprehension.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.comprehension.__ge__" => "Return self>=value.", + "_ast.comprehension.__getattribute__" => "Return getattr(self, name).", + "_ast.comprehension.__getstate__" => "Helper for pickle.", + "_ast.comprehension.__gt__" => "Return self>value.", + "_ast.comprehension.__hash__" => "Return hash(self).", + "_ast.comprehension.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.comprehension.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.comprehension.__le__" => "Return self<=value.", + "_ast.comprehension.__lt__" => "Return self<value.", + "_ast.comprehension.__ne__" => "Return self!=value.", + "_ast.comprehension.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.comprehension.__reduce_ex__" => "Helper for pickle.", + "_ast.comprehension.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.comprehension.__repr__" => "Return repr(self).", + "_ast.comprehension.__setattr__" => "Implement setattr(self, name, value).", + "_ast.comprehension.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.comprehension.__str__" => "Return str(self).", + "_ast.comprehension.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.comprehension.__weakref__" => "list of weak references to the object", + "_ast.excepthandler" => "excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)", + "_ast.excepthandler.__delattr__" => "Implement delattr(self, name).", + "_ast.excepthandler.__eq__" => "Return self==value.", + "_ast.excepthandler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.excepthandler.__ge__" => "Return self>=value.", + "_ast.excepthandler.__getattribute__" => "Return getattr(self, name).", + "_ast.excepthandler.__getstate__" => "Helper for pickle.", + "_ast.excepthandler.__gt__" => "Return self>value.", + "_ast.excepthandler.__hash__" => "Return hash(self).", + "_ast.excepthandler.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.excepthandler.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.excepthandler.__le__" => "Return self<=value.", + "_ast.excepthandler.__lt__" => "Return self<value.", + "_ast.excepthandler.__ne__" => "Return self!=value.", + "_ast.excepthandler.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.excepthandler.__reduce_ex__" => "Helper for pickle.", + "_ast.excepthandler.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.excepthandler.__repr__" => "Return repr(self).", + "_ast.excepthandler.__setattr__" => "Implement setattr(self, name, value).", + "_ast.excepthandler.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.excepthandler.__str__" => "Return str(self).", + "_ast.excepthandler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.excepthandler.__weakref__" => "list of weak references to the object", + "_ast.expr" => "expr = BoolOp(boolop op, expr* values)\n | NamedExpr(expr target, expr value)\n | BinOp(expr left, operator op, expr right)\n | UnaryOp(unaryop op, expr operand)\n | Lambda(arguments args, expr body)\n | IfExp(expr test, expr body, expr orelse)\n | Dict(expr?* keys, expr* values)\n | Set(expr* elts)\n | ListComp(expr elt, comprehension* generators)\n | SetComp(expr elt, comprehension* generators)\n | DictComp(expr key, expr value, comprehension* generators)\n | GeneratorExp(expr elt, comprehension* generators)\n | Await(expr value)\n | Yield(expr? value)\n | YieldFrom(expr value)\n | Compare(expr left, cmpop* ops, expr* comparators)\n | Call(expr func, expr* args, keyword* keywords)\n | FormattedValue(expr value, int conversion, expr? format_spec)\n | Interpolation(expr value, constant str, int conversion, expr? format_spec)\n | JoinedStr(expr* values)\n | TemplateStr(expr* values)\n | Constant(constant value, string? kind)\n | Attribute(expr value, identifier attr, expr_context ctx)\n | Subscript(expr value, expr slice, expr_context ctx)\n | Starred(expr value, expr_context ctx)\n | Name(identifier id, expr_context ctx)\n | List(expr* elts, expr_context ctx)\n | Tuple(expr* elts, expr_context ctx)\n | Slice(expr? lower, expr? upper, expr? step)", + "_ast.expr.__delattr__" => "Implement delattr(self, name).", + "_ast.expr.__eq__" => "Return self==value.", + "_ast.expr.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.expr.__ge__" => "Return self>=value.", + "_ast.expr.__getattribute__" => "Return getattr(self, name).", + "_ast.expr.__getstate__" => "Helper for pickle.", + "_ast.expr.__gt__" => "Return self>value.", + "_ast.expr.__hash__" => "Return hash(self).", + "_ast.expr.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.expr.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.expr.__le__" => "Return self<=value.", + "_ast.expr.__lt__" => "Return self<value.", + "_ast.expr.__ne__" => "Return self!=value.", + "_ast.expr.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.expr.__reduce_ex__" => "Helper for pickle.", + "_ast.expr.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.expr.__repr__" => "Return repr(self).", + "_ast.expr.__setattr__" => "Implement setattr(self, name, value).", + "_ast.expr.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.expr.__str__" => "Return str(self).", + "_ast.expr.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.expr.__weakref__" => "list of weak references to the object", + "_ast.expr_context" => "expr_context = Load | Store | Del", + "_ast.expr_context.__delattr__" => "Implement delattr(self, name).", + "_ast.expr_context.__eq__" => "Return self==value.", + "_ast.expr_context.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.expr_context.__ge__" => "Return self>=value.", + "_ast.expr_context.__getattribute__" => "Return getattr(self, name).", + "_ast.expr_context.__getstate__" => "Helper for pickle.", + "_ast.expr_context.__gt__" => "Return self>value.", + "_ast.expr_context.__hash__" => "Return hash(self).", + "_ast.expr_context.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.expr_context.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.expr_context.__le__" => "Return self<=value.", + "_ast.expr_context.__lt__" => "Return self<value.", + "_ast.expr_context.__ne__" => "Return self!=value.", + "_ast.expr_context.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.expr_context.__reduce_ex__" => "Helper for pickle.", + "_ast.expr_context.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.expr_context.__repr__" => "Return repr(self).", + "_ast.expr_context.__setattr__" => "Implement setattr(self, name, value).", + "_ast.expr_context.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.expr_context.__str__" => "Return str(self).", + "_ast.expr_context.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.expr_context.__weakref__" => "list of weak references to the object", + "_ast.keyword" => "keyword(identifier? arg, expr value)", + "_ast.keyword.__delattr__" => "Implement delattr(self, name).", + "_ast.keyword.__eq__" => "Return self==value.", + "_ast.keyword.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.keyword.__ge__" => "Return self>=value.", + "_ast.keyword.__getattribute__" => "Return getattr(self, name).", + "_ast.keyword.__getstate__" => "Helper for pickle.", + "_ast.keyword.__gt__" => "Return self>value.", + "_ast.keyword.__hash__" => "Return hash(self).", + "_ast.keyword.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.keyword.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.keyword.__le__" => "Return self<=value.", + "_ast.keyword.__lt__" => "Return self<value.", + "_ast.keyword.__ne__" => "Return self!=value.", + "_ast.keyword.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.keyword.__reduce_ex__" => "Helper for pickle.", + "_ast.keyword.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.keyword.__repr__" => "Return repr(self).", + "_ast.keyword.__setattr__" => "Implement setattr(self, name, value).", + "_ast.keyword.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.keyword.__str__" => "Return str(self).", + "_ast.keyword.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.keyword.__weakref__" => "list of weak references to the object", + "_ast.match_case" => "match_case(pattern pattern, expr? guard, stmt* body)", + "_ast.match_case.__delattr__" => "Implement delattr(self, name).", + "_ast.match_case.__eq__" => "Return self==value.", + "_ast.match_case.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.match_case.__ge__" => "Return self>=value.", + "_ast.match_case.__getattribute__" => "Return getattr(self, name).", + "_ast.match_case.__getstate__" => "Helper for pickle.", + "_ast.match_case.__gt__" => "Return self>value.", + "_ast.match_case.__hash__" => "Return hash(self).", + "_ast.match_case.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.match_case.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.match_case.__le__" => "Return self<=value.", + "_ast.match_case.__lt__" => "Return self<value.", + "_ast.match_case.__ne__" => "Return self!=value.", + "_ast.match_case.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.match_case.__reduce_ex__" => "Helper for pickle.", + "_ast.match_case.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.match_case.__repr__" => "Return repr(self).", + "_ast.match_case.__setattr__" => "Implement setattr(self, name, value).", + "_ast.match_case.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.match_case.__str__" => "Return str(self).", + "_ast.match_case.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.match_case.__weakref__" => "list of weak references to the object", + "_ast.mod" => "mod = Module(stmt* body, type_ignore* type_ignores)\n | Interactive(stmt* body)\n | Expression(expr body)\n | FunctionType(expr* argtypes, expr returns)", + "_ast.mod.__delattr__" => "Implement delattr(self, name).", + "_ast.mod.__eq__" => "Return self==value.", + "_ast.mod.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.mod.__ge__" => "Return self>=value.", + "_ast.mod.__getattribute__" => "Return getattr(self, name).", + "_ast.mod.__getstate__" => "Helper for pickle.", + "_ast.mod.__gt__" => "Return self>value.", + "_ast.mod.__hash__" => "Return hash(self).", + "_ast.mod.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.mod.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.mod.__le__" => "Return self<=value.", + "_ast.mod.__lt__" => "Return self<value.", + "_ast.mod.__ne__" => "Return self!=value.", + "_ast.mod.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.mod.__reduce_ex__" => "Helper for pickle.", + "_ast.mod.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.mod.__repr__" => "Return repr(self).", + "_ast.mod.__setattr__" => "Implement setattr(self, name, value).", + "_ast.mod.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.mod.__str__" => "Return str(self).", + "_ast.mod.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.mod.__weakref__" => "list of weak references to the object", + "_ast.operator" => "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv", + "_ast.operator.__delattr__" => "Implement delattr(self, name).", + "_ast.operator.__eq__" => "Return self==value.", + "_ast.operator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.operator.__ge__" => "Return self>=value.", + "_ast.operator.__getattribute__" => "Return getattr(self, name).", + "_ast.operator.__getstate__" => "Helper for pickle.", + "_ast.operator.__gt__" => "Return self>value.", + "_ast.operator.__hash__" => "Return hash(self).", + "_ast.operator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.operator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.operator.__le__" => "Return self<=value.", + "_ast.operator.__lt__" => "Return self<value.", + "_ast.operator.__ne__" => "Return self!=value.", + "_ast.operator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.operator.__reduce_ex__" => "Helper for pickle.", + "_ast.operator.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.operator.__repr__" => "Return repr(self).", + "_ast.operator.__setattr__" => "Implement setattr(self, name, value).", + "_ast.operator.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.operator.__str__" => "Return str(self).", + "_ast.operator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.operator.__weakref__" => "list of weak references to the object", + "_ast.pattern" => "pattern = MatchValue(expr value)\n | MatchSingleton(constant value)\n | MatchSequence(pattern* patterns)\n | MatchMapping(expr* keys, pattern* patterns, identifier? rest)\n | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)\n | MatchStar(identifier? name)\n | MatchAs(pattern? pattern, identifier? name)\n | MatchOr(pattern* patterns)", + "_ast.pattern.__delattr__" => "Implement delattr(self, name).", + "_ast.pattern.__eq__" => "Return self==value.", + "_ast.pattern.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.pattern.__ge__" => "Return self>=value.", + "_ast.pattern.__getattribute__" => "Return getattr(self, name).", + "_ast.pattern.__getstate__" => "Helper for pickle.", + "_ast.pattern.__gt__" => "Return self>value.", + "_ast.pattern.__hash__" => "Return hash(self).", + "_ast.pattern.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.pattern.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.pattern.__le__" => "Return self<=value.", + "_ast.pattern.__lt__" => "Return self<value.", + "_ast.pattern.__ne__" => "Return self!=value.", + "_ast.pattern.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.pattern.__reduce_ex__" => "Helper for pickle.", + "_ast.pattern.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.pattern.__repr__" => "Return repr(self).", + "_ast.pattern.__setattr__" => "Implement setattr(self, name, value).", + "_ast.pattern.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.pattern.__str__" => "Return str(self).", + "_ast.pattern.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.pattern.__weakref__" => "list of weak references to the object", + "_ast.stmt" => "stmt = FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)\n | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment, type_param* type_params)\n | ClassDef(identifier name, expr* bases, keyword* keywords, stmt* body, expr* decorator_list, type_param* type_params)\n | Return(expr? value)\n | Delete(expr* targets)\n | Assign(expr* targets, expr value, string? type_comment)\n | TypeAlias(expr name, type_param* type_params, expr value)\n | AugAssign(expr target, operator op, expr value)\n | AnnAssign(expr target, expr annotation, expr? value, int simple)\n | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)\n | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)\n | While(expr test, stmt* body, stmt* orelse)\n | If(expr test, stmt* body, stmt* orelse)\n | With(withitem* items, stmt* body, string? type_comment)\n | AsyncWith(withitem* items, stmt* body, string? type_comment)\n | Match(expr subject, match_case* cases)\n | Raise(expr? exc, expr? cause)\n | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)\n | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)\n | Assert(expr test, expr? msg)\n | Import(alias* names)\n | ImportFrom(identifier? module, alias* names, int? level)\n | Global(identifier* names)\n | Nonlocal(identifier* names)\n | Expr(expr value)\n | Pass\n | Break\n | Continue", + "_ast.stmt.__delattr__" => "Implement delattr(self, name).", + "_ast.stmt.__eq__" => "Return self==value.", + "_ast.stmt.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.stmt.__ge__" => "Return self>=value.", + "_ast.stmt.__getattribute__" => "Return getattr(self, name).", + "_ast.stmt.__getstate__" => "Helper for pickle.", + "_ast.stmt.__gt__" => "Return self>value.", + "_ast.stmt.__hash__" => "Return hash(self).", + "_ast.stmt.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.stmt.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.stmt.__le__" => "Return self<=value.", + "_ast.stmt.__lt__" => "Return self<value.", + "_ast.stmt.__ne__" => "Return self!=value.", + "_ast.stmt.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.stmt.__reduce_ex__" => "Helper for pickle.", + "_ast.stmt.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.stmt.__repr__" => "Return repr(self).", + "_ast.stmt.__setattr__" => "Implement setattr(self, name, value).", + "_ast.stmt.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.stmt.__str__" => "Return str(self).", + "_ast.stmt.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.stmt.__weakref__" => "list of weak references to the object", + "_ast.type_ignore" => "type_ignore = TypeIgnore(int lineno, string tag)", + "_ast.type_ignore.__delattr__" => "Implement delattr(self, name).", + "_ast.type_ignore.__eq__" => "Return self==value.", + "_ast.type_ignore.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.type_ignore.__ge__" => "Return self>=value.", + "_ast.type_ignore.__getattribute__" => "Return getattr(self, name).", + "_ast.type_ignore.__getstate__" => "Helper for pickle.", + "_ast.type_ignore.__gt__" => "Return self>value.", + "_ast.type_ignore.__hash__" => "Return hash(self).", + "_ast.type_ignore.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.type_ignore.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.type_ignore.__le__" => "Return self<=value.", + "_ast.type_ignore.__lt__" => "Return self<value.", + "_ast.type_ignore.__ne__" => "Return self!=value.", + "_ast.type_ignore.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.type_ignore.__reduce_ex__" => "Helper for pickle.", + "_ast.type_ignore.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.type_ignore.__repr__" => "Return repr(self).", + "_ast.type_ignore.__setattr__" => "Implement setattr(self, name, value).", + "_ast.type_ignore.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.type_ignore.__str__" => "Return str(self).", + "_ast.type_ignore.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.type_ignore.__weakref__" => "list of weak references to the object", + "_ast.type_param" => "type_param = TypeVar(identifier name, expr? bound, expr? default_value)\n | ParamSpec(identifier name, expr? default_value)\n | TypeVarTuple(identifier name, expr? default_value)", + "_ast.type_param.__delattr__" => "Implement delattr(self, name).", + "_ast.type_param.__eq__" => "Return self==value.", + "_ast.type_param.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.type_param.__ge__" => "Return self>=value.", + "_ast.type_param.__getattribute__" => "Return getattr(self, name).", + "_ast.type_param.__getstate__" => "Helper for pickle.", + "_ast.type_param.__gt__" => "Return self>value.", + "_ast.type_param.__hash__" => "Return hash(self).", + "_ast.type_param.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.type_param.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.type_param.__le__" => "Return self<=value.", + "_ast.type_param.__lt__" => "Return self<value.", + "_ast.type_param.__ne__" => "Return self!=value.", + "_ast.type_param.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.type_param.__reduce_ex__" => "Helper for pickle.", + "_ast.type_param.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.type_param.__repr__" => "Return repr(self).", + "_ast.type_param.__setattr__" => "Implement setattr(self, name, value).", + "_ast.type_param.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.type_param.__str__" => "Return str(self).", + "_ast.type_param.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.type_param.__weakref__" => "list of weak references to the object", + "_ast.unaryop" => "unaryop = Invert | Not | UAdd | USub", + "_ast.unaryop.__delattr__" => "Implement delattr(self, name).", + "_ast.unaryop.__eq__" => "Return self==value.", + "_ast.unaryop.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.unaryop.__ge__" => "Return self>=value.", + "_ast.unaryop.__getattribute__" => "Return getattr(self, name).", + "_ast.unaryop.__getstate__" => "Helper for pickle.", + "_ast.unaryop.__gt__" => "Return self>value.", + "_ast.unaryop.__hash__" => "Return hash(self).", + "_ast.unaryop.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.unaryop.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.unaryop.__le__" => "Return self<=value.", + "_ast.unaryop.__lt__" => "Return self<value.", + "_ast.unaryop.__ne__" => "Return self!=value.", + "_ast.unaryop.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.unaryop.__reduce_ex__" => "Helper for pickle.", + "_ast.unaryop.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.unaryop.__repr__" => "Return repr(self).", + "_ast.unaryop.__setattr__" => "Implement setattr(self, name, value).", + "_ast.unaryop.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.unaryop.__str__" => "Return str(self).", + "_ast.unaryop.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.unaryop.__weakref__" => "list of weak references to the object", + "_ast.withitem" => "withitem(expr context_expr, expr? optional_vars)", + "_ast.withitem.__delattr__" => "Implement delattr(self, name).", + "_ast.withitem.__eq__" => "Return self==value.", + "_ast.withitem.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ast.withitem.__ge__" => "Return self>=value.", + "_ast.withitem.__getattribute__" => "Return getattr(self, name).", + "_ast.withitem.__getstate__" => "Helper for pickle.", + "_ast.withitem.__gt__" => "Return self>value.", + "_ast.withitem.__hash__" => "Return hash(self).", + "_ast.withitem.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ast.withitem.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ast.withitem.__le__" => "Return self<=value.", + "_ast.withitem.__lt__" => "Return self<value.", + "_ast.withitem.__ne__" => "Return self!=value.", + "_ast.withitem.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ast.withitem.__reduce_ex__" => "Helper for pickle.", + "_ast.withitem.__replace__" => "Return a copy of the AST node with new values for the specified fields.", + "_ast.withitem.__repr__" => "Return repr(self).", + "_ast.withitem.__setattr__" => "Implement setattr(self, name, value).", + "_ast.withitem.__sizeof__" => "Size of object in memory, in bytes.", + "_ast.withitem.__str__" => "Return str(self).", + "_ast.withitem.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ast.withitem.__weakref__" => "list of weak references to the object", + "_asyncio" => "Accelerator module for asyncio", + "_asyncio.Future" => "This class is *almost* compatible with concurrent.futures.Future.\n\n Differences:\n\n - result() and exception() do not take a timeout argument and\n raise an exception when the future isn't done yet.\n\n - Callbacks registered with add_done_callback() are always called\n via the event loop's call_soon_threadsafe().\n\n - This class is not compatible with the wait() and as_completed()\n methods in the concurrent.futures package.", + "_asyncio.Future.__await__" => "Return an iterator to be used in await expression.", + "_asyncio.Future.__class_getitem__" => "See PEP 585", + "_asyncio.Future.__del__" => "Called when the instance is about to be destroyed.", + "_asyncio.Future.__delattr__" => "Implement delattr(self, name).", + "_asyncio.Future.__eq__" => "Return self==value.", + "_asyncio.Future.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_asyncio.Future.__ge__" => "Return self>=value.", + "_asyncio.Future.__getattribute__" => "Return getattr(self, name).", + "_asyncio.Future.__getstate__" => "Helper for pickle.", + "_asyncio.Future.__gt__" => "Return self>value.", + "_asyncio.Future.__hash__" => "Return hash(self).", + "_asyncio.Future.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_asyncio.Future.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_asyncio.Future.__iter__" => "Implement iter(self).", + "_asyncio.Future.__le__" => "Return self<=value.", + "_asyncio.Future.__lt__" => "Return self<value.", + "_asyncio.Future.__ne__" => "Return self!=value.", + "_asyncio.Future.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_asyncio.Future.__reduce__" => "Helper for pickle.", + "_asyncio.Future.__reduce_ex__" => "Helper for pickle.", + "_asyncio.Future.__repr__" => "Return repr(self).", + "_asyncio.Future.__setattr__" => "Implement setattr(self, name, value).", + "_asyncio.Future.__sizeof__" => "Size of object in memory, in bytes.", + "_asyncio.Future.__str__" => "Return str(self).", + "_asyncio.Future.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_asyncio.Future._make_cancelled_error" => "Create the CancelledError to raise if the Future is cancelled.\n\nThis should only be called once when handling a cancellation since\nit erases the context exception value.", + "_asyncio.Future.add_done_callback" => "Add a callback to be run when the future becomes done.\n\nThe callback is called with a single argument - the future object. If\nthe future is already done when this is called, the callback is\nscheduled with call_soon.", + "_asyncio.Future.cancel" => "Cancel the future and schedule callbacks.\n\nIf the future is already done or cancelled, return False. Otherwise,\nchange the future's state to cancelled, schedule the callbacks and\nreturn True.", + "_asyncio.Future.cancelled" => "Return True if the future was cancelled.", + "_asyncio.Future.done" => "Return True if the future is done.\n\nDone means either that a result / exception are available, or that the\nfuture was cancelled.", + "_asyncio.Future.exception" => "Return the exception that was set on this future.\n\nThe exception (or None if no exception was set) is returned only if\nthe future is done. If the future has been cancelled, raises\nCancelledError. If the future isn't done yet, raises\nInvalidStateError.", + "_asyncio.Future.get_loop" => "Return the event loop the Future is bound to.", + "_asyncio.Future.remove_done_callback" => "Remove all instances of a callback from the \"call when done\" list.\n\nReturns the number of callbacks removed.", + "_asyncio.Future.result" => "Return the result this future represents.\n\nIf the future has been cancelled, raises CancelledError. If the\nfuture's result isn't yet available, raises InvalidStateError. If\nthe future is done and has an exception set, this exception is raised.", + "_asyncio.Future.set_exception" => "Mark the future done and set an exception.\n\nIf the future is already done when this method is called, raises\nInvalidStateError.", + "_asyncio.Future.set_result" => "Mark the future done and set its result.\n\nIf the future is already done when this method is called, raises\nInvalidStateError.", + "_asyncio.Task" => "A coroutine wrapped in a Future.", + "_asyncio.Task.__await__" => "Return an iterator to be used in await expression.", + "_asyncio.Task.__class_getitem__" => "See PEP 585", + "_asyncio.Task.__del__" => "Called when the instance is about to be destroyed.", + "_asyncio.Task.__delattr__" => "Implement delattr(self, name).", + "_asyncio.Task.__eq__" => "Return self==value.", + "_asyncio.Task.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_asyncio.Task.__ge__" => "Return self>=value.", + "_asyncio.Task.__getattribute__" => "Return getattr(self, name).", + "_asyncio.Task.__getstate__" => "Helper for pickle.", + "_asyncio.Task.__gt__" => "Return self>value.", + "_asyncio.Task.__hash__" => "Return hash(self).", + "_asyncio.Task.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_asyncio.Task.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_asyncio.Task.__iter__" => "Implement iter(self).", + "_asyncio.Task.__le__" => "Return self<=value.", + "_asyncio.Task.__lt__" => "Return self<value.", + "_asyncio.Task.__ne__" => "Return self!=value.", + "_asyncio.Task.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_asyncio.Task.__reduce__" => "Helper for pickle.", + "_asyncio.Task.__reduce_ex__" => "Helper for pickle.", + "_asyncio.Task.__repr__" => "Return repr(self).", + "_asyncio.Task.__setattr__" => "Implement setattr(self, name, value).", + "_asyncio.Task.__sizeof__" => "Size of object in memory, in bytes.", + "_asyncio.Task.__str__" => "Return str(self).", + "_asyncio.Task.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_asyncio.Task._make_cancelled_error" => "Create the CancelledError to raise if the Task is cancelled.\n\nThis should only be called once when handling a cancellation since\nit erases the context exception value.", + "_asyncio.Task.add_done_callback" => "Add a callback to be run when the future becomes done.\n\nThe callback is called with a single argument - the future object. If\nthe future is already done when this is called, the callback is\nscheduled with call_soon.", + "_asyncio.Task.cancel" => "Request that this task cancel itself.\n\nThis arranges for a CancelledError to be thrown into the\nwrapped coroutine on the next cycle through the event loop.\nThe coroutine then has a chance to clean up or even deny\nthe request using try/except/finally.\n\nUnlike Future.cancel, this does not guarantee that the\ntask will be cancelled: the exception might be caught and\nacted upon, delaying cancellation of the task or preventing\ncancellation completely. The task may also return a value or\nraise a different exception.\n\nImmediately after this method is called, Task.cancelled() will\nnot return True (unless the task was already cancelled). A\ntask will be marked as cancelled when the wrapped coroutine\nterminates with a CancelledError exception (even if cancel()\nwas not called).\n\nThis also increases the task's count of cancellation requests.", + "_asyncio.Task.cancelled" => "Return True if the future was cancelled.", + "_asyncio.Task.cancelling" => "Return the count of the task's cancellation requests.\n\nThis count is incremented when .cancel() is called\nand may be decremented using .uncancel().", + "_asyncio.Task.done" => "Return True if the future is done.\n\nDone means either that a result / exception are available, or that the\nfuture was cancelled.", + "_asyncio.Task.exception" => "Return the exception that was set on this future.\n\nThe exception (or None if no exception was set) is returned only if\nthe future is done. If the future has been cancelled, raises\nCancelledError. If the future isn't done yet, raises\nInvalidStateError.", + "_asyncio.Task.get_loop" => "Return the event loop the Future is bound to.", + "_asyncio.Task.get_stack" => "Return the list of stack frames for this task's coroutine.\n\nIf the coroutine is not done, this returns the stack where it is\nsuspended. If the coroutine has completed successfully or was\ncancelled, this returns an empty list. If the coroutine was\nterminated by an exception, this returns the list of traceback\nframes.\n\nThe frames are always ordered from oldest to newest.\n\nThe optional limit gives the maximum number of frames to\nreturn; by default all available frames are returned. Its\nmeaning differs depending on whether a stack or a traceback is\nreturned: the newest frames of a stack are returned, but the\noldest frames of a traceback are returned. (This matches the\nbehavior of the traceback module.)\n\nFor reasons beyond our control, only one stack frame is\nreturned for a suspended coroutine.", + "_asyncio.Task.print_stack" => "Print the stack or traceback for this task's coroutine.\n\nThis produces output similar to that of the traceback module,\nfor the frames retrieved by get_stack(). The limit argument\nis passed to get_stack(). The file argument is an I/O stream\nto which the output is written; by default output is written\nto sys.stderr.", + "_asyncio.Task.remove_done_callback" => "Remove all instances of a callback from the \"call when done\" list.\n\nReturns the number of callbacks removed.", + "_asyncio.Task.result" => "Return the result this future represents.\n\nIf the future has been cancelled, raises CancelledError. If the\nfuture's result isn't yet available, raises InvalidStateError. If\nthe future is done and has an exception set, this exception is raised.", + "_asyncio.Task.uncancel" => "Decrement the task's count of cancellation requests.\n\nThis should be used by tasks that catch CancelledError\nand wish to continue indefinitely until they are cancelled again.\n\nReturns the remaining number of cancellation requests.", + "_asyncio._enter_task" => "Enter into task execution or resume suspended task.\n\nTask belongs to loop.\n\nReturns None.", + "_asyncio._get_running_loop" => "Return the running event loop or None.\n\nThis is a low-level function intended to be used by event loops.\nThis function is thread-specific.", + "_asyncio._leave_task" => "Leave task execution or suspend a task.\n\nTask belongs to loop.\n\nReturns None.", + "_asyncio._register_eager_task" => "Register a new task in asyncio as executed by loop.\n\nReturns None.", + "_asyncio._register_task" => "Register a new task in asyncio as executed by loop.\n\nReturns None.", + "_asyncio._set_running_loop" => "Set the running event loop.\n\nThis is a low-level function intended to be used by event loops.\nThis function is thread-specific.", + "_asyncio._swap_current_task" => "Temporarily swap in the supplied task and return the original one (or None).\n\nThis is intended for use during eager coroutine execution.", + "_asyncio._unregister_eager_task" => "Unregister a task.\n\nReturns None.", + "_asyncio._unregister_task" => "Unregister a task.\n\nReturns None.", + "_asyncio.all_tasks" => "Return a set of all tasks for the loop.", + "_asyncio.current_task" => "Return a currently executed task.", + "_asyncio.future_add_to_awaited_by" => "Record that `fut` is awaited on by `waiter`.", + "_asyncio.get_event_loop" => "Return an asyncio event loop.\n\nWhen called from a coroutine or a callback (e.g. scheduled with\ncall_soon or similar API), this function will always return the\nrunning event loop.\n\nIf there is no running event loop set, the function will return\nthe result of `get_event_loop_policy().get_event_loop()` call.", + "_asyncio.get_running_loop" => "Return the running event loop. Raise a RuntimeError if there is none.\n\nThis function is thread-specific.", + "_bisect" => "Bisection algorithms.\n\nThis module provides support for maintaining a list in sorted order without\nhaving to sort the list after each insertion. For long lists of items with\nexpensive comparison operations, this can be an improvement over the more\ncommon approach.", + "_bisect.bisect_left" => "Return the index where to insert item x in list a, assuming a is sorted.\n\nThe return value i is such that all e in a[:i] have e < x, and all e in\na[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will\ninsert just before the leftmost x already there.\n\nOptional args lo (default 0) and hi (default len(a)) bound the\nslice of a to be searched.\n\nA custom key function can be supplied to customize the sort order.", + "_bisect.bisect_right" => "Return the index where to insert item x in list a, assuming a is sorted.\n\nThe return value i is such that all e in a[:i] have e <= x, and all e in\na[i:] have e > x. So if x already appears in the list, a.insert(i, x) will\ninsert just after the rightmost x already there.\n\nOptional args lo (default 0) and hi (default len(a)) bound the\nslice of a to be searched.\n\nA custom key function can be supplied to customize the sort order.", + "_bisect.insort_left" => "Insert item x in list a, and keep it sorted assuming a is sorted.\n\nIf x is already in a, insert it to the left of the leftmost x.\n\nOptional args lo (default 0) and hi (default len(a)) bound the\nslice of a to be searched.\n\nA custom key function can be supplied to customize the sort order.", + "_bisect.insort_right" => "Insert item x in list a, and keep it sorted assuming a is sorted.\n\nIf x is already in a, insert it to the right of the rightmost x.\n\nOptional args lo (default 0) and hi (default len(a)) bound the\nslice of a to be searched.\n\nA custom key function can be supplied to customize the sort order.", + "_blake2" => "_blake2b provides BLAKE2b for hashlib", + "_blake2.blake2b" => "Return a new BLAKE2b hash object.", + "_blake2.blake2b.__delattr__" => "Implement delattr(self, name).", + "_blake2.blake2b.__eq__" => "Return self==value.", + "_blake2.blake2b.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_blake2.blake2b.__ge__" => "Return self>=value.", + "_blake2.blake2b.__getattribute__" => "Return getattr(self, name).", + "_blake2.blake2b.__getstate__" => "Helper for pickle.", + "_blake2.blake2b.__gt__" => "Return self>value.", + "_blake2.blake2b.__hash__" => "Return hash(self).", + "_blake2.blake2b.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_blake2.blake2b.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_blake2.blake2b.__le__" => "Return self<=value.", + "_blake2.blake2b.__lt__" => "Return self<value.", + "_blake2.blake2b.__ne__" => "Return self!=value.", + "_blake2.blake2b.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_blake2.blake2b.__reduce__" => "Helper for pickle.", + "_blake2.blake2b.__reduce_ex__" => "Helper for pickle.", + "_blake2.blake2b.__repr__" => "Return repr(self).", + "_blake2.blake2b.__setattr__" => "Implement setattr(self, name, value).", + "_blake2.blake2b.__sizeof__" => "Size of object in memory, in bytes.", + "_blake2.blake2b.__str__" => "Return str(self).", + "_blake2.blake2b.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_blake2.blake2b.copy" => "Return a copy of the hash object.", + "_blake2.blake2b.digest" => "Return the digest value as a bytes object.", + "_blake2.blake2b.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_blake2.blake2b.update" => "Update this hash object's state with the provided bytes-like object.", + "_blake2.blake2s" => "Return a new BLAKE2s hash object.", + "_blake2.blake2s.__delattr__" => "Implement delattr(self, name).", + "_blake2.blake2s.__eq__" => "Return self==value.", + "_blake2.blake2s.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_blake2.blake2s.__ge__" => "Return self>=value.", + "_blake2.blake2s.__getattribute__" => "Return getattr(self, name).", + "_blake2.blake2s.__getstate__" => "Helper for pickle.", + "_blake2.blake2s.__gt__" => "Return self>value.", + "_blake2.blake2s.__hash__" => "Return hash(self).", + "_blake2.blake2s.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_blake2.blake2s.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_blake2.blake2s.__le__" => "Return self<=value.", + "_blake2.blake2s.__lt__" => "Return self<value.", + "_blake2.blake2s.__ne__" => "Return self!=value.", + "_blake2.blake2s.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_blake2.blake2s.__reduce__" => "Helper for pickle.", + "_blake2.blake2s.__reduce_ex__" => "Helper for pickle.", + "_blake2.blake2s.__repr__" => "Return repr(self).", + "_blake2.blake2s.__setattr__" => "Implement setattr(self, name, value).", + "_blake2.blake2s.__sizeof__" => "Size of object in memory, in bytes.", + "_blake2.blake2s.__str__" => "Return str(self).", + "_blake2.blake2s.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_blake2.blake2s.copy" => "Return a copy of the hash object.", + "_blake2.blake2s.digest" => "Return the digest value as a bytes object.", + "_blake2.blake2s.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_blake2.blake2s.update" => "Update this hash object's state with the provided bytes-like object.", + "_bz2.BZ2Compressor" => "Create a compressor object for compressing data incrementally.\n\n compresslevel\n Compression level, as a number between 1 and 9.\n\nFor one-shot compression, use the compress() function instead.", + "_bz2.BZ2Compressor.__delattr__" => "Implement delattr(self, name).", + "_bz2.BZ2Compressor.__eq__" => "Return self==value.", + "_bz2.BZ2Compressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_bz2.BZ2Compressor.__ge__" => "Return self>=value.", + "_bz2.BZ2Compressor.__getattribute__" => "Return getattr(self, name).", + "_bz2.BZ2Compressor.__getstate__" => "Helper for pickle.", + "_bz2.BZ2Compressor.__gt__" => "Return self>value.", + "_bz2.BZ2Compressor.__hash__" => "Return hash(self).", + "_bz2.BZ2Compressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_bz2.BZ2Compressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_bz2.BZ2Compressor.__le__" => "Return self<=value.", + "_bz2.BZ2Compressor.__lt__" => "Return self<value.", + "_bz2.BZ2Compressor.__ne__" => "Return self!=value.", + "_bz2.BZ2Compressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_bz2.BZ2Compressor.__reduce__" => "Helper for pickle.", + "_bz2.BZ2Compressor.__reduce_ex__" => "Helper for pickle.", + "_bz2.BZ2Compressor.__repr__" => "Return repr(self).", + "_bz2.BZ2Compressor.__setattr__" => "Implement setattr(self, name, value).", + "_bz2.BZ2Compressor.__sizeof__" => "Size of object in memory, in bytes.", + "_bz2.BZ2Compressor.__str__" => "Return str(self).", + "_bz2.BZ2Compressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_bz2.BZ2Compressor.compress" => "Provide data to the compressor object.\n\nReturns a chunk of compressed data if possible, or b'' otherwise.\n\nWhen you have finished providing data to the compressor, call the\nflush() method to finish the compression process.", + "_bz2.BZ2Compressor.flush" => "Finish the compression process.\n\nReturns the compressed data left in internal buffers.\n\nThe compressor object may not be used after this method is called.", + "_bz2.BZ2Decompressor" => "Create a decompressor object for decompressing data incrementally.\n\nFor one-shot decompression, use the decompress() function instead.", + "_bz2.BZ2Decompressor.__delattr__" => "Implement delattr(self, name).", + "_bz2.BZ2Decompressor.__eq__" => "Return self==value.", + "_bz2.BZ2Decompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_bz2.BZ2Decompressor.__ge__" => "Return self>=value.", + "_bz2.BZ2Decompressor.__getattribute__" => "Return getattr(self, name).", + "_bz2.BZ2Decompressor.__getstate__" => "Helper for pickle.", + "_bz2.BZ2Decompressor.__gt__" => "Return self>value.", + "_bz2.BZ2Decompressor.__hash__" => "Return hash(self).", + "_bz2.BZ2Decompressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_bz2.BZ2Decompressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_bz2.BZ2Decompressor.__le__" => "Return self<=value.", + "_bz2.BZ2Decompressor.__lt__" => "Return self<value.", + "_bz2.BZ2Decompressor.__ne__" => "Return self!=value.", + "_bz2.BZ2Decompressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_bz2.BZ2Decompressor.__reduce__" => "Helper for pickle.", + "_bz2.BZ2Decompressor.__reduce_ex__" => "Helper for pickle.", + "_bz2.BZ2Decompressor.__repr__" => "Return repr(self).", + "_bz2.BZ2Decompressor.__setattr__" => "Implement setattr(self, name, value).", + "_bz2.BZ2Decompressor.__sizeof__" => "Size of object in memory, in bytes.", + "_bz2.BZ2Decompressor.__str__" => "Return str(self).", + "_bz2.BZ2Decompressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_bz2.BZ2Decompressor.decompress" => "Decompress *data*, returning uncompressed data as bytes.\n\nIf *max_length* is nonnegative, returns at most *max_length* bytes of\ndecompressed data. If this limit is reached and further output can be\nproduced, *self.needs_input* will be set to ``False``. In this case, the next\ncall to *decompress()* may provide *data* as b'' to obtain more of the output.\n\nIf all of the input data was decompressed and returned (either because this\nwas less than *max_length* bytes, or because *max_length* was negative),\n*self.needs_input* will be set to True.\n\nAttempting to decompress data after the end of stream is reached raises an\nEOFError. Any data found after the end of the stream is ignored and saved in\nthe unused_data attribute.", + "_bz2.BZ2Decompressor.eof" => "True if the end-of-stream marker has been reached.", + "_bz2.BZ2Decompressor.needs_input" => "True if more input is needed before more decompressed data can be produced.", + "_bz2.BZ2Decompressor.unused_data" => "Data found after the end of the compressed stream.", + "_codecs._unregister_error" => "Un-register the specified error handler for the error handling `errors'.\n\nOnly custom error handlers can be un-registered. An exception is raised\nif the error handling is a built-in one (e.g., 'strict'), or if an error\noccurs.\n\nOtherwise, this returns True if a custom handler has been successfully\nun-registered, and False if no custom handler for the specified error\nhandling exists.", + "_codecs.decode" => "Decodes obj using the codec registered for encoding.\n\nDefault encoding is 'utf-8'. errors may be given to set a\ndifferent error handling scheme. Default is 'strict' meaning that encoding\nerrors raise a ValueError. Other possible values are 'ignore', 'replace'\nand 'backslashreplace' as well as any other name registered with\ncodecs.register_error that can handle ValueErrors.", + "_codecs.encode" => "Encodes obj using the codec registered for encoding.\n\nThe default encoding is 'utf-8'. errors may be given to set a\ndifferent error handling scheme. Default is 'strict' meaning that encoding\nerrors raise a ValueError. Other possible values are 'ignore', 'replace'\nand 'backslashreplace' as well as any other name registered with\ncodecs.register_error that can handle ValueErrors.", + "_codecs.lookup" => "Looks up a codec tuple in the Python codec registry and returns a CodecInfo object.", + "_codecs.lookup_error" => "lookup_error(errors) -> handler\n\nReturn the error handler for the specified error handling name or raise a\nLookupError, if no handler exists under this name.", + "_codecs.register" => "Register a codec search function.\n\nSearch functions are expected to take one argument, the encoding name in\nall lower case letters, and either return None, or a tuple of functions\n(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object).", + "_codecs.register_error" => "Register the specified error handler under the name errors.\n\nhandler must be a callable object, that will be called with an exception\ninstance containing information about the location of the encoding/decoding\nerror and must return a (replacement, new position) tuple.", + "_codecs.unregister" => "Unregister a codec search function and clear the registry's cache.\n\nIf the search function is not registered, do nothing.", + "_collections" => "High performance data structures.\n- deque: ordered collection accessible from endpoints only\n- defaultdict: dict subclass with a default value factory", + "_collections.OrderedDict" => "Dictionary that remembers insertion order", + "_collections.OrderedDict.__class_getitem__" => "See PEP 585", + "_collections.OrderedDict.__contains__" => "True if the dictionary has the specified key, else False.", + "_collections.OrderedDict.__delattr__" => "Implement delattr(self, name).", + "_collections.OrderedDict.__delitem__" => "Delete self[key].", + "_collections.OrderedDict.__eq__" => "Return self==value.", + "_collections.OrderedDict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections.OrderedDict.__ge__" => "Return self>=value.", + "_collections.OrderedDict.__getattribute__" => "Return getattr(self, name).", + "_collections.OrderedDict.__getitem__" => "Return self[key].", + "_collections.OrderedDict.__getstate__" => "Helper for pickle.", + "_collections.OrderedDict.__gt__" => "Return self>value.", + "_collections.OrderedDict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections.OrderedDict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections.OrderedDict.__ior__" => "Return self|=value.", + "_collections.OrderedDict.__iter__" => "Implement iter(self).", + "_collections.OrderedDict.__le__" => "Return self<=value.", + "_collections.OrderedDict.__len__" => "Return len(self).", + "_collections.OrderedDict.__lt__" => "Return self<value.", + "_collections.OrderedDict.__ne__" => "Return self!=value.", + "_collections.OrderedDict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections.OrderedDict.__or__" => "Return self|value.", + "_collections.OrderedDict.__reduce__" => "Return state information for pickling", + "_collections.OrderedDict.__reduce_ex__" => "Helper for pickle.", + "_collections.OrderedDict.__repr__" => "Return repr(self).", + "_collections.OrderedDict.__reversed__" => "od.__reversed__() <==> reversed(od)", + "_collections.OrderedDict.__ror__" => "Return value|self.", + "_collections.OrderedDict.__setattr__" => "Implement setattr(self, name, value).", + "_collections.OrderedDict.__setitem__" => "Set self[key] to value.", + "_collections.OrderedDict.__str__" => "Return str(self).", + "_collections.OrderedDict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections.OrderedDict.clear" => "Remove all items from ordered dict.", + "_collections.OrderedDict.copy" => "A shallow copy of ordered dict.", + "_collections.OrderedDict.fromkeys" => "Create a new ordered dictionary with keys from iterable and values set to value.", + "_collections.OrderedDict.get" => "Return the value for key if key is in the dictionary, else default.", + "_collections.OrderedDict.move_to_end" => "Move an existing element to the end (or beginning if last is false).\n\nRaise KeyError if the element does not exist.", + "_collections.OrderedDict.pop" => "od.pop(key[,default]) -> v, remove specified key and return the corresponding value.\n\nIf the key is not found, return the default if given; otherwise,\nraise a KeyError.", + "_collections.OrderedDict.popitem" => "Remove and return a (key, value) pair from the dictionary.\n\nPairs are returned in LIFO order if last is true or FIFO order if false.", + "_collections.OrderedDict.setdefault" => "Insert key with a value of default if key is not in the dictionary.\n\nReturn the value for key if key is in the dictionary, else default.", + "_collections._count_elements" => "Count elements in the iterable, updating the mapping", + "_collections._deque_iterator.__delattr__" => "Implement delattr(self, name).", + "_collections._deque_iterator.__eq__" => "Return self==value.", + "_collections._deque_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections._deque_iterator.__ge__" => "Return self>=value.", + "_collections._deque_iterator.__getattribute__" => "Return getattr(self, name).", + "_collections._deque_iterator.__getstate__" => "Helper for pickle.", + "_collections._deque_iterator.__gt__" => "Return self>value.", + "_collections._deque_iterator.__hash__" => "Return hash(self).", + "_collections._deque_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections._deque_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections._deque_iterator.__iter__" => "Implement iter(self).", + "_collections._deque_iterator.__le__" => "Return self<=value.", + "_collections._deque_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "_collections._deque_iterator.__lt__" => "Return self<value.", + "_collections._deque_iterator.__ne__" => "Return self!=value.", + "_collections._deque_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections._deque_iterator.__next__" => "Implement next(self).", + "_collections._deque_iterator.__reduce__" => "Return state information for pickling.", + "_collections._deque_iterator.__reduce_ex__" => "Helper for pickle.", + "_collections._deque_iterator.__repr__" => "Return repr(self).", + "_collections._deque_iterator.__setattr__" => "Implement setattr(self, name, value).", + "_collections._deque_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "_collections._deque_iterator.__str__" => "Return str(self).", + "_collections._deque_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections._deque_reverse_iterator.__delattr__" => "Implement delattr(self, name).", + "_collections._deque_reverse_iterator.__eq__" => "Return self==value.", + "_collections._deque_reverse_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections._deque_reverse_iterator.__ge__" => "Return self>=value.", + "_collections._deque_reverse_iterator.__getattribute__" => "Return getattr(self, name).", + "_collections._deque_reverse_iterator.__getstate__" => "Helper for pickle.", + "_collections._deque_reverse_iterator.__gt__" => "Return self>value.", + "_collections._deque_reverse_iterator.__hash__" => "Return hash(self).", + "_collections._deque_reverse_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections._deque_reverse_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections._deque_reverse_iterator.__iter__" => "Implement iter(self).", + "_collections._deque_reverse_iterator.__le__" => "Return self<=value.", + "_collections._deque_reverse_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "_collections._deque_reverse_iterator.__lt__" => "Return self<value.", + "_collections._deque_reverse_iterator.__ne__" => "Return self!=value.", + "_collections._deque_reverse_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections._deque_reverse_iterator.__next__" => "Implement next(self).", + "_collections._deque_reverse_iterator.__reduce__" => "Return state information for pickling.", + "_collections._deque_reverse_iterator.__reduce_ex__" => "Helper for pickle.", + "_collections._deque_reverse_iterator.__repr__" => "Return repr(self).", + "_collections._deque_reverse_iterator.__setattr__" => "Implement setattr(self, name, value).", + "_collections._deque_reverse_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "_collections._deque_reverse_iterator.__str__" => "Return str(self).", + "_collections._deque_reverse_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections._tuplegetter.__delattr__" => "Implement delattr(self, name).", + "_collections._tuplegetter.__delete__" => "Delete an attribute of instance.", + "_collections._tuplegetter.__eq__" => "Return self==value.", + "_collections._tuplegetter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections._tuplegetter.__ge__" => "Return self>=value.", + "_collections._tuplegetter.__get__" => "Return an attribute of instance, which is of type owner.", + "_collections._tuplegetter.__getattribute__" => "Return getattr(self, name).", + "_collections._tuplegetter.__getstate__" => "Helper for pickle.", + "_collections._tuplegetter.__gt__" => "Return self>value.", + "_collections._tuplegetter.__hash__" => "Return hash(self).", + "_collections._tuplegetter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections._tuplegetter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections._tuplegetter.__le__" => "Return self<=value.", + "_collections._tuplegetter.__lt__" => "Return self<value.", + "_collections._tuplegetter.__ne__" => "Return self!=value.", + "_collections._tuplegetter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections._tuplegetter.__reduce_ex__" => "Helper for pickle.", + "_collections._tuplegetter.__repr__" => "Return repr(self).", + "_collections._tuplegetter.__set__" => "Set an attribute of instance to value.", + "_collections._tuplegetter.__setattr__" => "Implement setattr(self, name, value).", + "_collections._tuplegetter.__sizeof__" => "Size of object in memory, in bytes.", + "_collections._tuplegetter.__str__" => "Return str(self).", + "_collections._tuplegetter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections.defaultdict" => "defaultdict(default_factory=None, /, [...]) --> dict with default factory\n\nThe default factory is called without arguments to produce\na new value when a key is not present, in __getitem__ only.\nA defaultdict compares equal to a dict with the same items.\nAll remaining arguments are treated the same as if they were\npassed to the dict constructor, including keyword arguments.", + "_collections.defaultdict.__class_getitem__" => "See PEP 585", + "_collections.defaultdict.__contains__" => "True if the dictionary has the specified key, else False.", + "_collections.defaultdict.__copy__" => "D.copy() -> a shallow copy of D.", + "_collections.defaultdict.__delattr__" => "Implement delattr(self, name).", + "_collections.defaultdict.__delitem__" => "Delete self[key].", + "_collections.defaultdict.__eq__" => "Return self==value.", + "_collections.defaultdict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections.defaultdict.__ge__" => "Return self>=value.", + "_collections.defaultdict.__getattribute__" => "Return getattr(self, name).", + "_collections.defaultdict.__getitem__" => "Return self[key].", + "_collections.defaultdict.__getstate__" => "Helper for pickle.", + "_collections.defaultdict.__gt__" => "Return self>value.", + "_collections.defaultdict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections.defaultdict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections.defaultdict.__ior__" => "Return self|=value.", + "_collections.defaultdict.__iter__" => "Implement iter(self).", + "_collections.defaultdict.__le__" => "Return self<=value.", + "_collections.defaultdict.__len__" => "Return len(self).", + "_collections.defaultdict.__lt__" => "Return self<value.", + "_collections.defaultdict.__missing__" => "__missing__(key) # Called by __getitem__ for missing key; pseudo-code:\n if self.default_factory is None: raise KeyError((key,))\n self[key] = value = self.default_factory()\n return value", + "_collections.defaultdict.__ne__" => "Return self!=value.", + "_collections.defaultdict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections.defaultdict.__or__" => "Return self|value.", + "_collections.defaultdict.__reduce__" => "Return state information for pickling.", + "_collections.defaultdict.__reduce_ex__" => "Helper for pickle.", + "_collections.defaultdict.__repr__" => "Return repr(self).", + "_collections.defaultdict.__reversed__" => "Return a reverse iterator over the dict keys.", + "_collections.defaultdict.__ror__" => "Return value|self.", + "_collections.defaultdict.__setattr__" => "Implement setattr(self, name, value).", + "_collections.defaultdict.__setitem__" => "Set self[key] to value.", + "_collections.defaultdict.__sizeof__" => "Return the size of the dict in memory, in bytes.", + "_collections.defaultdict.__str__" => "Return str(self).", + "_collections.defaultdict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections.defaultdict.clear" => "Remove all items from the dict.", + "_collections.defaultdict.copy" => "D.copy() -> a shallow copy of D.", + "_collections.defaultdict.default_factory" => "Factory for default value called by __missing__().", + "_collections.defaultdict.fromkeys" => "Create a new dictionary with keys from iterable and values set to value.", + "_collections.defaultdict.get" => "Return the value for key if key is in the dictionary, else default.", + "_collections.defaultdict.items" => "Return a set-like object providing a view on the dict's items.", + "_collections.defaultdict.keys" => "Return a set-like object providing a view on the dict's keys.", + "_collections.defaultdict.pop" => "D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n\nIf the key is not found, return the default if given; otherwise,\nraise a KeyError.", + "_collections.defaultdict.popitem" => "Remove and return a (key, value) pair as a 2-tuple.\n\nPairs are returned in LIFO (last-in, first-out) order.\nRaises KeyError if the dict is empty.", + "_collections.defaultdict.setdefault" => "Insert key with a value of default if key is not in the dictionary.\n\nReturn the value for key if key is in the dictionary, else default.", + "_collections.defaultdict.update" => "D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\nIf E is present and has a .keys() method, then does: for k in E.keys(): D[k] = E[k]\nIf E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\nIn either case, this is followed by: for k in F: D[k] = F[k]", + "_collections.defaultdict.values" => "Return an object providing a view on the dict's values.", + "_collections.deque" => "A list-like sequence optimized for data accesses near its endpoints.", + "_collections.deque.__add__" => "Return self+value.", + "_collections.deque.__class_getitem__" => "See PEP 585", + "_collections.deque.__contains__" => "Return bool(key in self).", + "_collections.deque.__copy__" => "Return a shallow copy of a deque.", + "_collections.deque.__delattr__" => "Implement delattr(self, name).", + "_collections.deque.__delitem__" => "Delete self[key].", + "_collections.deque.__eq__" => "Return self==value.", + "_collections.deque.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_collections.deque.__ge__" => "Return self>=value.", + "_collections.deque.__getattribute__" => "Return getattr(self, name).", + "_collections.deque.__getitem__" => "Return self[key].", + "_collections.deque.__getstate__" => "Helper for pickle.", + "_collections.deque.__gt__" => "Return self>value.", + "_collections.deque.__iadd__" => "Implement self+=value.", + "_collections.deque.__imul__" => "Implement self*=value.", + "_collections.deque.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_collections.deque.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_collections.deque.__iter__" => "Implement iter(self).", + "_collections.deque.__le__" => "Return self<=value.", + "_collections.deque.__len__" => "Return len(self).", + "_collections.deque.__lt__" => "Return self<value.", + "_collections.deque.__mul__" => "Return self*value.", + "_collections.deque.__ne__" => "Return self!=value.", + "_collections.deque.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_collections.deque.__reduce__" => "Return state information for pickling.", + "_collections.deque.__reduce_ex__" => "Helper for pickle.", + "_collections.deque.__repr__" => "Return repr(self).", + "_collections.deque.__reversed__" => "Return a reverse iterator over the deque.", + "_collections.deque.__rmul__" => "Return value*self.", + "_collections.deque.__setattr__" => "Implement setattr(self, name, value).", + "_collections.deque.__setitem__" => "Set self[key] to value.", + "_collections.deque.__sizeof__" => "Return the size of the deque in memory, in bytes.", + "_collections.deque.__str__" => "Return str(self).", + "_collections.deque.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_collections.deque.append" => "Add an element to the right side of the deque.", + "_collections.deque.appendleft" => "Add an element to the left side of the deque.", + "_collections.deque.clear" => "Remove all elements from the deque.", + "_collections.deque.copy" => "Return a shallow copy of a deque.", + "_collections.deque.count" => "Return number of occurrences of value.", + "_collections.deque.extend" => "Extend the right side of the deque with elements from the iterable.", + "_collections.deque.extendleft" => "Extend the left side of the deque with elements from the iterable.", + "_collections.deque.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_collections.deque.insert" => "Insert value before index.", + "_collections.deque.maxlen" => "maximum size of a deque or None if unbounded", + "_collections.deque.pop" => "Remove and return the rightmost element.", + "_collections.deque.popleft" => "Remove and return the leftmost element.", + "_collections.deque.remove" => "Remove first occurrence of value.", + "_collections.deque.reverse" => "Reverse *IN PLACE*.", + "_collections.deque.rotate" => "Rotate the deque n steps to the right. If n is negative, rotates left.", + "_contextvars" => "Context Variables", + "_contextvars.Context.__contains__" => "Return bool(key in self).", + "_contextvars.Context.__delattr__" => "Implement delattr(self, name).", + "_contextvars.Context.__eq__" => "Return self==value.", + "_contextvars.Context.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_contextvars.Context.__ge__" => "Return self>=value.", + "_contextvars.Context.__getattribute__" => "Return getattr(self, name).", + "_contextvars.Context.__getitem__" => "Return self[key].", + "_contextvars.Context.__getstate__" => "Helper for pickle.", + "_contextvars.Context.__gt__" => "Return self>value.", + "_contextvars.Context.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_contextvars.Context.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_contextvars.Context.__iter__" => "Implement iter(self).", + "_contextvars.Context.__le__" => "Return self<=value.", + "_contextvars.Context.__len__" => "Return len(self).", + "_contextvars.Context.__lt__" => "Return self<value.", + "_contextvars.Context.__ne__" => "Return self!=value.", + "_contextvars.Context.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_contextvars.Context.__reduce__" => "Helper for pickle.", + "_contextvars.Context.__reduce_ex__" => "Helper for pickle.", + "_contextvars.Context.__repr__" => "Return repr(self).", + "_contextvars.Context.__setattr__" => "Implement setattr(self, name, value).", + "_contextvars.Context.__sizeof__" => "Size of object in memory, in bytes.", + "_contextvars.Context.__str__" => "Return str(self).", + "_contextvars.Context.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_contextvars.Context.copy" => "Return a shallow copy of the context object.", + "_contextvars.Context.get" => "Return the value for `key` if `key` has the value in the context object.\n\nIf `key` does not exist, return `default`. If `default` is not given,\nreturn None.", + "_contextvars.Context.items" => "Return all variables and their values in the context object.\n\nThe result is returned as a list of 2-tuples (variable, value).", + "_contextvars.Context.keys" => "Return a list of all variables in the context object.", + "_contextvars.Context.values" => "Return a list of all variables' values in the context object.", + "_contextvars.ContextVar.__class_getitem__" => "See PEP 585", + "_contextvars.ContextVar.__delattr__" => "Implement delattr(self, name).", + "_contextvars.ContextVar.__eq__" => "Return self==value.", + "_contextvars.ContextVar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_contextvars.ContextVar.__ge__" => "Return self>=value.", + "_contextvars.ContextVar.__getattribute__" => "Return getattr(self, name).", + "_contextvars.ContextVar.__getstate__" => "Helper for pickle.", + "_contextvars.ContextVar.__gt__" => "Return self>value.", + "_contextvars.ContextVar.__hash__" => "Return hash(self).", + "_contextvars.ContextVar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_contextvars.ContextVar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_contextvars.ContextVar.__le__" => "Return self<=value.", + "_contextvars.ContextVar.__lt__" => "Return self<value.", + "_contextvars.ContextVar.__ne__" => "Return self!=value.", + "_contextvars.ContextVar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_contextvars.ContextVar.__reduce__" => "Helper for pickle.", + "_contextvars.ContextVar.__reduce_ex__" => "Helper for pickle.", + "_contextvars.ContextVar.__repr__" => "Return repr(self).", + "_contextvars.ContextVar.__setattr__" => "Implement setattr(self, name, value).", + "_contextvars.ContextVar.__sizeof__" => "Size of object in memory, in bytes.", + "_contextvars.ContextVar.__str__" => "Return str(self).", + "_contextvars.ContextVar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_contextvars.ContextVar.get" => "Return a value for the context variable for the current context.\n\nIf there is no value for the variable in the current context, the method will:\n * return the value of the default argument of the method, if provided; or\n * return the default value for the context variable, if it was created\n with one; or\n * raise a LookupError.", + "_contextvars.ContextVar.reset" => "Reset the context variable.\n\nThe variable is reset to the value it had before the `ContextVar.set()` that\ncreated the token was used.", + "_contextvars.ContextVar.set" => "Call to set a new value for the context variable in the current context.\n\nThe required value argument is the new value for the context variable.\n\nReturns a Token object that can be used to restore the variable to its previous\nvalue via the `ContextVar.reset()` method.", + "_contextvars.Token.__class_getitem__" => "See PEP 585", + "_contextvars.Token.__delattr__" => "Implement delattr(self, name).", + "_contextvars.Token.__enter__" => "Enter into Token context manager.", + "_contextvars.Token.__eq__" => "Return self==value.", + "_contextvars.Token.__exit__" => "Exit from Token context manager, restore the linked ContextVar.", + "_contextvars.Token.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_contextvars.Token.__ge__" => "Return self>=value.", + "_contextvars.Token.__getattribute__" => "Return getattr(self, name).", + "_contextvars.Token.__getstate__" => "Helper for pickle.", + "_contextvars.Token.__gt__" => "Return self>value.", + "_contextvars.Token.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_contextvars.Token.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_contextvars.Token.__le__" => "Return self<=value.", + "_contextvars.Token.__lt__" => "Return self<value.", + "_contextvars.Token.__ne__" => "Return self!=value.", + "_contextvars.Token.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_contextvars.Token.__reduce__" => "Helper for pickle.", + "_contextvars.Token.__reduce_ex__" => "Helper for pickle.", + "_contextvars.Token.__repr__" => "Return repr(self).", + "_contextvars.Token.__setattr__" => "Implement setattr(self, name, value).", + "_contextvars.Token.__sizeof__" => "Size of object in memory, in bytes.", + "_contextvars.Token.__str__" => "Return str(self).", + "_contextvars.Token.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_csv" => "CSV parsing and writing.", + "_csv.Dialect" => "CSV dialect\n\nThe Dialect type records CSV parsing and generation options.", + "_csv.Dialect.__delattr__" => "Implement delattr(self, name).", + "_csv.Dialect.__eq__" => "Return self==value.", + "_csv.Dialect.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_csv.Dialect.__ge__" => "Return self>=value.", + "_csv.Dialect.__getattribute__" => "Return getattr(self, name).", + "_csv.Dialect.__getstate__" => "Helper for pickle.", + "_csv.Dialect.__gt__" => "Return self>value.", + "_csv.Dialect.__hash__" => "Return hash(self).", + "_csv.Dialect.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_csv.Dialect.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_csv.Dialect.__le__" => "Return self<=value.", + "_csv.Dialect.__lt__" => "Return self<value.", + "_csv.Dialect.__ne__" => "Return self!=value.", + "_csv.Dialect.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_csv.Dialect.__reduce__" => "raises an exception to avoid pickling", + "_csv.Dialect.__reduce_ex__" => "raises an exception to avoid pickling", + "_csv.Dialect.__repr__" => "Return repr(self).", + "_csv.Dialect.__setattr__" => "Implement setattr(self, name, value).", + "_csv.Dialect.__sizeof__" => "Size of object in memory, in bytes.", + "_csv.Dialect.__str__" => "Return str(self).", + "_csv.Dialect.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_csv.Error.__delattr__" => "Implement delattr(self, name).", + "_csv.Error.__eq__" => "Return self==value.", + "_csv.Error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_csv.Error.__ge__" => "Return self>=value.", + "_csv.Error.__getattribute__" => "Return getattr(self, name).", + "_csv.Error.__getstate__" => "Helper for pickle.", + "_csv.Error.__gt__" => "Return self>value.", + "_csv.Error.__hash__" => "Return hash(self).", + "_csv.Error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_csv.Error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_csv.Error.__le__" => "Return self<=value.", + "_csv.Error.__lt__" => "Return self<value.", + "_csv.Error.__ne__" => "Return self!=value.", + "_csv.Error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_csv.Error.__reduce_ex__" => "Helper for pickle.", + "_csv.Error.__repr__" => "Return repr(self).", + "_csv.Error.__setattr__" => "Implement setattr(self, name, value).", + "_csv.Error.__sizeof__" => "Size of object in memory, in bytes.", + "_csv.Error.__str__" => "Return str(self).", + "_csv.Error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_csv.Error.add_note" => "Add a note to the exception", + "_csv.Error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_csv.Reader" => "CSV reader\n\nReader objects are responsible for reading and parsing tabular data\nin CSV format.", + "_csv.Reader.__delattr__" => "Implement delattr(self, name).", + "_csv.Reader.__eq__" => "Return self==value.", + "_csv.Reader.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_csv.Reader.__ge__" => "Return self>=value.", + "_csv.Reader.__getattribute__" => "Return getattr(self, name).", + "_csv.Reader.__getstate__" => "Helper for pickle.", + "_csv.Reader.__gt__" => "Return self>value.", + "_csv.Reader.__hash__" => "Return hash(self).", + "_csv.Reader.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_csv.Reader.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_csv.Reader.__iter__" => "Implement iter(self).", + "_csv.Reader.__le__" => "Return self<=value.", + "_csv.Reader.__lt__" => "Return self<value.", + "_csv.Reader.__ne__" => "Return self!=value.", + "_csv.Reader.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_csv.Reader.__next__" => "Implement next(self).", + "_csv.Reader.__reduce__" => "Helper for pickle.", + "_csv.Reader.__reduce_ex__" => "Helper for pickle.", + "_csv.Reader.__repr__" => "Return repr(self).", + "_csv.Reader.__setattr__" => "Implement setattr(self, name, value).", + "_csv.Reader.__sizeof__" => "Size of object in memory, in bytes.", + "_csv.Reader.__str__" => "Return str(self).", + "_csv.Reader.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_csv.Writer" => "CSV writer\n\nWriter objects are responsible for generating tabular data\nin CSV format from sequence input.", + "_csv.Writer.__delattr__" => "Implement delattr(self, name).", + "_csv.Writer.__eq__" => "Return self==value.", + "_csv.Writer.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_csv.Writer.__ge__" => "Return self>=value.", + "_csv.Writer.__getattribute__" => "Return getattr(self, name).", + "_csv.Writer.__getstate__" => "Helper for pickle.", + "_csv.Writer.__gt__" => "Return self>value.", + "_csv.Writer.__hash__" => "Return hash(self).", + "_csv.Writer.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_csv.Writer.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_csv.Writer.__le__" => "Return self<=value.", + "_csv.Writer.__lt__" => "Return self<value.", + "_csv.Writer.__ne__" => "Return self!=value.", + "_csv.Writer.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_csv.Writer.__reduce__" => "Helper for pickle.", + "_csv.Writer.__reduce_ex__" => "Helper for pickle.", + "_csv.Writer.__repr__" => "Return repr(self).", + "_csv.Writer.__setattr__" => "Implement setattr(self, name, value).", + "_csv.Writer.__sizeof__" => "Size of object in memory, in bytes.", + "_csv.Writer.__str__" => "Return str(self).", + "_csv.Writer.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_csv.Writer.writerow" => "Construct and write a CSV record from an iterable of fields.\n\nNon-string elements will be converted to string.", + "_csv.Writer.writerows" => "Construct and write a series of iterables to a csv file.\n\nNon-string elements will be converted to string.", + "_csv.field_size_limit" => "Sets an upper limit on parsed fields.\n\nReturns old limit. If limit is not given, no new limit is set and\nthe old limit is returned", + "_csv.get_dialect" => "Return the dialect instance associated with name.", + "_csv.list_dialects" => "Return a list of all known dialect names.", + "_csv.reader" => "Return a reader object that will process lines from the given iterable.\n\nThe \"iterable\" argument can be any object that returns a line\nof input for each iteration, such as a file object or a list. The\noptional \"dialect\" argument defines a CSV dialect. The function\nalso accepts optional keyword arguments which override settings\nprovided by the dialect.\n\nThe returned object is an iterator. Each iteration returns a row\nof the CSV file (which can span multiple input lines).", + "_csv.register_dialect" => "Create a mapping from a string name to a CVS dialect.\n\nThe optional \"dialect\" argument specifies the base dialect instance\nor the name of the registered dialect. The function also accepts\noptional keyword arguments which override settings provided by the\ndialect.", + "_csv.unregister_dialect" => "Delete the name/dialect mapping associated with a string name.", + "_csv.writer" => "Return a writer object that will write user data on the given file object.\n\nThe \"fileobj\" argument can be any object that supports the file API.\nThe optional \"dialect\" argument defines a CSV dialect. The function\nalso accepts optional keyword arguments which override settings\nprovided by the dialect.", + "_ctypes" => "Create and manipulate C compatible data types in Python.", + "_ctypes.ArgumentError.__delattr__" => "Implement delattr(self, name).", + "_ctypes.ArgumentError.__eq__" => "Return self==value.", + "_ctypes.ArgumentError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ctypes.ArgumentError.__ge__" => "Return self>=value.", + "_ctypes.ArgumentError.__getattribute__" => "Return getattr(self, name).", + "_ctypes.ArgumentError.__getstate__" => "Helper for pickle.", + "_ctypes.ArgumentError.__gt__" => "Return self>value.", + "_ctypes.ArgumentError.__hash__" => "Return hash(self).", + "_ctypes.ArgumentError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ctypes.ArgumentError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ctypes.ArgumentError.__le__" => "Return self<=value.", + "_ctypes.ArgumentError.__lt__" => "Return self<value.", + "_ctypes.ArgumentError.__ne__" => "Return self!=value.", + "_ctypes.ArgumentError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ctypes.ArgumentError.__reduce_ex__" => "Helper for pickle.", + "_ctypes.ArgumentError.__repr__" => "Return repr(self).", + "_ctypes.ArgumentError.__setattr__" => "Implement setattr(self, name, value).", + "_ctypes.ArgumentError.__sizeof__" => "Size of object in memory, in bytes.", + "_ctypes.ArgumentError.__str__" => "Return str(self).", + "_ctypes.ArgumentError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ctypes.ArgumentError.__weakref__" => "list of weak references to the object", + "_ctypes.ArgumentError.add_note" => "Add a note to the exception", + "_ctypes.ArgumentError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ctypes.Array" => "Abstract base class for arrays.\n\nThe recommended way to create concrete array types is by multiplying any\nctypes data type with a non-negative integer. Alternatively, you can subclass\nthis type and define _length_ and _type_ class variables. Array elements can\nbe read and written using standard subscript and slice accesses for slice\nreads, the resulting object is not itself an Array.", + "_ctypes.CField" => "Structure/Union member", + "_ctypes.CField.__delattr__" => "Implement delattr(self, name).", + "_ctypes.CField.__delete__" => "Delete an attribute of instance.", + "_ctypes.CField.__eq__" => "Return self==value.", + "_ctypes.CField.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ctypes.CField.__ge__" => "Return self>=value.", + "_ctypes.CField.__get__" => "Return an attribute of instance, which is of type owner.", + "_ctypes.CField.__getattribute__" => "Return getattr(self, name).", + "_ctypes.CField.__getstate__" => "Helper for pickle.", + "_ctypes.CField.__gt__" => "Return self>value.", + "_ctypes.CField.__hash__" => "Return hash(self).", + "_ctypes.CField.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ctypes.CField.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ctypes.CField.__le__" => "Return self<=value.", + "_ctypes.CField.__lt__" => "Return self<value.", + "_ctypes.CField.__ne__" => "Return self!=value.", + "_ctypes.CField.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ctypes.CField.__reduce__" => "Helper for pickle.", + "_ctypes.CField.__reduce_ex__" => "Helper for pickle.", + "_ctypes.CField.__repr__" => "Return repr(self).", + "_ctypes.CField.__set__" => "Set an attribute of instance to value.", + "_ctypes.CField.__setattr__" => "Implement setattr(self, name, value).", + "_ctypes.CField.__sizeof__" => "Size of object in memory, in bytes.", + "_ctypes.CField.__str__" => "Return str(self).", + "_ctypes.CField.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ctypes.CField.bit_offset" => "additional offset in bits (relative to byte_offset); zero for non-bitfields", + "_ctypes.CField.bit_size" => "size of this field in bits", + "_ctypes.CField.byte_offset" => "offset in bytes of this field. For bitfields: excludes bit_offset.", + "_ctypes.CField.byte_size" => "size of this field in bytes", + "_ctypes.CField.is_anonymous" => "true if this field is anonymous", + "_ctypes.CField.is_bitfield" => "true if this is a bitfield", + "_ctypes.CField.name" => "name of this field", + "_ctypes.CField.offset" => "offset in bytes of this field (same as byte_offset)", + "_ctypes.CField.size" => "size in bytes of this field. For bitfields, this is a legacy packed value; use byte_size instead", + "_ctypes.CField.type" => "type of this field", + "_ctypes.CFuncPtr" => "Function Pointer", + "_ctypes.COMError" => "Raised when a COM method call failed.", + "_ctypes.COMError.__delattr__" => "Implement delattr(self, name).", + "_ctypes.COMError.__eq__" => "Return self==value.", + "_ctypes.COMError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ctypes.COMError.__ge__" => "Return self>=value.", + "_ctypes.COMError.__getattribute__" => "Return getattr(self, name).", + "_ctypes.COMError.__getstate__" => "Helper for pickle.", + "_ctypes.COMError.__gt__" => "Return self>value.", + "_ctypes.COMError.__hash__" => "Return hash(self).", + "_ctypes.COMError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ctypes.COMError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ctypes.COMError.__le__" => "Return self<=value.", + "_ctypes.COMError.__lt__" => "Return self<value.", + "_ctypes.COMError.__ne__" => "Return self!=value.", + "_ctypes.COMError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ctypes.COMError.__reduce_ex__" => "Helper for pickle.", + "_ctypes.COMError.__repr__" => "Return repr(self).", + "_ctypes.COMError.__setattr__" => "Implement setattr(self, name, value).", + "_ctypes.COMError.__sizeof__" => "Size of object in memory, in bytes.", + "_ctypes.COMError.__str__" => "Return str(self).", + "_ctypes.COMError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ctypes.COMError.add_note" => "Add a note to the exception", + "_ctypes.COMError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ctypes.CopyComPointer" => "CopyComPointer(src, dst) -> HRESULT value", + "_ctypes.FormatError" => "FormatError([integer]) -> string\n\nConvert a win32 error code into a string. If the error code is not\ngiven, the return value of a call to GetLastError() is used.", + "_ctypes.FreeLibrary" => "FreeLibrary(handle) -> void\n\nFree the handle of an executable previously loaded by LoadLibrary.", + "_ctypes.LoadLibrary" => "LoadLibrary(name, load_flags) -> handle\n\nLoad an executable (usually a DLL), and return a handle to it.\nThe handle may be used to locate exported functions in this\nmodule. load_flags are as defined for LoadLibraryEx in the\nWindows API.", + "_ctypes.Structure" => "Structure base class", + "_ctypes.Union" => "Union base class", + "_ctypes._Pointer" => "XXX to be provided", + "_ctypes._SimpleCData" => "XXX to be provided", + "_ctypes._dyld_shared_cache_contains_path" => "check if path is in the shared cache", + "_ctypes.addressof" => "Return the address of the C instance internal buffer", + "_ctypes.alignment" => "alignment(C type) -> integer\nalignment(C instance) -> integer\nReturn the alignment requirements of a C instance", + "_ctypes.buffer_info" => "Return buffer interface information", + "_ctypes.byref" => "Return a pointer lookalike to a C instance, only usable as function argument.", + "_ctypes.dlclose" => "dlclose a library", + "_ctypes.dlopen" => "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library", + "_ctypes.dlsym" => "find symbol in shared library", + "_ctypes.sizeof" => "Return the size in bytes of a C instance.", + "_curses.assume_default_colors" => "Allow use of default values for colors on terminals supporting this feature.\n\nAssign terminal default foreground/background colors to color number -1.\nChange the definition of the color-pair 0 to (fg, bg).\n\nUse this to support transparency in your application.", + "_curses.baudrate" => "Return the output speed of the terminal in bits per second.", + "_curses.beep" => "Emit a short attention sound.", + "_curses.can_change_color" => "Return True if the programmer can change the colors displayed by the terminal.", + "_curses.cbreak" => "Enter cbreak mode.\n\n flag\n If false, the effect is the same as calling nocbreak().\n\nIn cbreak mode (sometimes called \"rare\" mode) normal tty line buffering is\nturned off and characters are available to be read one by one. However,\nunlike raw mode, special characters (interrupt, quit, suspend, and flow\ncontrol) retain their effects on the tty driver and calling program.\nCalling first raw() then cbreak() leaves the terminal in cbreak mode.", + "_curses.color_content" => "Return the red, green, and blue (RGB) components of the specified color.\n\n color_number\n The number of the color (0 - (COLORS-1)).\n\nA 3-tuple is returned, containing the R, G, B values for the given color,\nwhich will be between 0 (no component) and 1000 (maximum amount of component).", + "_curses.color_pair" => "Return the attribute value for displaying text in the specified color.\n\n pair_number\n The number of the color pair.\n\nThis attribute value can be combined with A_STANDOUT, A_REVERSE, and the\nother A_* attributes. pair_number() is the counterpart to this function.", + "_curses.curs_set" => "Set the cursor state.\n\n visibility\n 0 for invisible, 1 for normal visible, or 2 for very visible.\n\nIf the terminal supports the visibility requested, the previous cursor\nstate is returned; otherwise, an exception is raised. On many terminals,\nthe \"visible\" mode is an underline cursor and the \"very visible\" mode is\na block cursor.", + "_curses.def_prog_mode" => "Save the current terminal mode as the \"program\" mode.\n\nThe \"program\" mode is the mode when the running program is using curses.\n\nSubsequent calls to reset_prog_mode() will restore this mode.", + "_curses.def_shell_mode" => "Save the current terminal mode as the \"shell\" mode.\n\nThe \"shell\" mode is the mode when the running program is not using curses.\n\nSubsequent calls to reset_shell_mode() will restore this mode.", + "_curses.delay_output" => "Insert a pause in output.\n\n ms\n Duration in milliseconds.", + "_curses.doupdate" => "Update the physical screen to match the virtual screen.", + "_curses.echo" => "Enter echo mode.\n\n flag\n If false, the effect is the same as calling noecho().\n\nIn echo mode, each character input is echoed to the screen as it is entered.", + "_curses.endwin" => "De-initialize the library, and return terminal to normal status.", + "_curses.erasechar" => "Return the user's current erase character.", + "_curses.error.__delattr__" => "Implement delattr(self, name).", + "_curses.error.__eq__" => "Return self==value.", + "_curses.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_curses.error.__ge__" => "Return self>=value.", + "_curses.error.__getattribute__" => "Return getattr(self, name).", + "_curses.error.__getstate__" => "Helper for pickle.", + "_curses.error.__gt__" => "Return self>value.", + "_curses.error.__hash__" => "Return hash(self).", + "_curses.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_curses.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_curses.error.__le__" => "Return self<=value.", + "_curses.error.__lt__" => "Return self<value.", + "_curses.error.__ne__" => "Return self!=value.", + "_curses.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_curses.error.__reduce_ex__" => "Helper for pickle.", + "_curses.error.__repr__" => "Return repr(self).", + "_curses.error.__setattr__" => "Implement setattr(self, name, value).", + "_curses.error.__sizeof__" => "Size of object in memory, in bytes.", + "_curses.error.__str__" => "Return str(self).", + "_curses.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_curses.error.__weakref__" => "list of weak references to the object", + "_curses.error.add_note" => "Add a note to the exception", + "_curses.error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_curses.flash" => "Flash the screen.\n\nThat is, change it to reverse-video and then change it back in a short interval.", + "_curses.flushinp" => "Flush all input buffers.\n\nThis throws away any typeahead that has been typed by the user and has not\nyet been processed by the program.", + "_curses.get_escdelay" => "Gets the curses ESCDELAY setting.\n\nGets the number of milliseconds to wait after reading an escape character,\nto distinguish between an individual escape character entered on the\nkeyboard from escape sequences sent by cursor and function keys.", + "_curses.get_tabsize" => "Gets the curses TABSIZE setting.\n\nGets the number of columns used by the curses library when converting a tab\ncharacter to spaces as it adds the tab to a window.", + "_curses.getmouse" => "Retrieve the queued mouse event.\n\nAfter getch() returns KEY_MOUSE to signal a mouse event, this function\nreturns a 5-tuple (id, x, y, z, bstate).", + "_curses.getsyx" => "Return the current coordinates of the virtual screen cursor.\n\nReturn a (y, x) tuple. If leaveok is currently true, return (-1, -1).", + "_curses.getwin" => "Read window related data stored in the file by an earlier putwin() call.\n\nThe routine then creates and initializes a new window using that data,\nreturning the new window object.", + "_curses.halfdelay" => "Enter half-delay mode.\n\n tenths\n Maximal blocking delay in tenths of seconds (1 - 255).\n\nUse nocbreak() to leave half-delay mode.", + "_curses.has_colors" => "Return True if the terminal can display colors; otherwise, return False.", + "_curses.has_extended_color_support" => "Return True if the module supports extended colors; otherwise, return False.\n\nExtended color support allows more than 256 color-pairs for terminals\nthat support more than 16 colors (e.g. xterm-256color).", + "_curses.has_ic" => "Return True if the terminal has insert- and delete-character capabilities.", + "_curses.has_il" => "Return True if the terminal has insert- and delete-line capabilities.", + "_curses.has_key" => "Return True if the current terminal type recognizes a key with that value.\n\n key\n Key number.", + "_curses.init_color" => "Change the definition of a color.\n\n color_number\n The number of the color to be changed (0 - (COLORS-1)).\n r\n Red component (0 - 1000).\n g\n Green component (0 - 1000).\n b\n Blue component (0 - 1000).\n\nWhen init_color() is used, all occurrences of that color on the screen\nimmediately change to the new definition. This function is a no-op on\nmost terminals; it is active only if can_change_color() returns true.", + "_curses.init_pair" => "Change the definition of a color-pair.\n\n pair_number\n The number of the color-pair to be changed (1 - (COLOR_PAIRS-1)).\n fg\n Foreground color number (-1 - (COLORS-1)).\n bg\n Background color number (-1 - (COLORS-1)).\n\nIf the color-pair was previously initialized, the screen is refreshed and\nall occurrences of that color-pair are changed to the new definition.", + "_curses.initscr" => "Initialize the library.\n\nReturn a WindowObject which represents the whole screen.", + "_curses.is_term_resized" => "Return True if resize_term() would modify the window structure, False otherwise.\n\n nlines\n Height.\n ncols\n Width.", + "_curses.isendwin" => "Return True if endwin() has been called.", + "_curses.keyname" => "Return the name of specified key.\n\n key\n Key number.", + "_curses.killchar" => "Return the user's current line kill character.", + "_curses.longname" => "Return the terminfo long name field describing the current terminal.\n\nThe maximum length of a verbose description is 128 characters. It is defined\nonly after the call to initscr().", + "_curses.meta" => "Enable/disable meta keys.\n\nIf yes is True, allow 8-bit characters to be input. If yes is False,\nallow only 7-bit characters.", + "_curses.mouseinterval" => "Set and retrieve the maximum time between press and release in a click.\n\n interval\n Time in milliseconds.\n\nSet the maximum time that can elapse between press and release events in\norder for them to be recognized as a click, and return the previous interval\nvalue.", + "_curses.mousemask" => "Set the mouse events to be reported, and return a tuple (availmask, oldmask).\n\nReturn a tuple (availmask, oldmask). availmask indicates which of the\nspecified mouse events can be reported; on complete failure it returns 0.\noldmask is the previous value of the given window's mouse event mask.\nIf this function is never called, no mouse events are ever reported.", + "_curses.napms" => "Sleep for specified time.\n\n ms\n Duration in milliseconds.", + "_curses.newpad" => "Create and return a pointer to a new pad data structure.\n\n nlines\n Height.\n ncols\n Width.", + "_curses.newwin" => "newwin(nlines, ncols, [begin_y=0, begin_x=0])\nReturn a new window.\n\n nlines\n Height.\n ncols\n Width.\n begin_y\n Top side y-coordinate.\n begin_x\n Left side x-coordinate.\n\nBy default, the window will extend from the specified position to the lower\nright corner of the screen.", + "_curses.nl" => "Enter newline mode.\n\n flag\n If false, the effect is the same as calling nonl().\n\nThis mode translates the return key into newline on input, and translates\nnewline into return and line-feed on output. Newline mode is initially on.", + "_curses.nocbreak" => "Leave cbreak mode.\n\nReturn to normal \"cooked\" mode with line buffering.", + "_curses.noecho" => "Leave echo mode.\n\nEchoing of input characters is turned off.", + "_curses.nonl" => "Leave newline mode.\n\nDisable translation of return into newline on input, and disable low-level\ntranslation of newline into newline/return on output.", + "_curses.noqiflush" => "Disable queue flushing.\n\nWhen queue flushing is disabled, normal flush of input and output queues\nassociated with the INTR, QUIT and SUSP characters will not be done.", + "_curses.noraw" => "Leave raw mode.\n\nReturn to normal \"cooked\" mode with line buffering.", + "_curses.pair_content" => "Return a tuple (fg, bg) containing the colors for the requested color pair.\n\n pair_number\n The number of the color pair (0 - (COLOR_PAIRS-1)).", + "_curses.pair_number" => "Return the number of the color-pair set by the specified attribute value.\n\ncolor_pair() is the counterpart to this function.", + "_curses.putp" => "Emit the value of a specified terminfo capability for the current terminal.\n\nNote that the output of putp() always goes to standard output.", + "_curses.qiflush" => "Enable queue flushing.\n\n flag\n If false, the effect is the same as calling noqiflush().\n\nIf queue flushing is enabled, all output in the display driver queue\nwill be flushed when the INTR, QUIT and SUSP characters are read.", + "_curses.raw" => "Enter raw mode.\n\n flag\n If false, the effect is the same as calling noraw().\n\nIn raw mode, normal line buffering and processing of interrupt, quit,\nsuspend, and flow control keys are turned off; characters are presented to\ncurses input functions one by one.", + "_curses.reset_prog_mode" => "Restore the terminal to \"program\" mode, as previously saved by def_prog_mode().", + "_curses.reset_shell_mode" => "Restore the terminal to \"shell\" mode, as previously saved by def_shell_mode().", + "_curses.resetty" => "Restore terminal mode.", + "_curses.resize_term" => "Backend function used by resizeterm(), performing most of the work.\n\n nlines\n Height.\n ncols\n Width.\n\nWhen resizing the windows, resize_term() blank-fills the areas that are\nextended. The calling application should fill in these areas with appropriate\ndata. The resize_term() function attempts to resize all windows. However,\ndue to the calling convention of pads, it is not possible to resize these\nwithout additional interaction with the application.", + "_curses.resizeterm" => "Resize the standard and current windows to the specified dimensions.\n\n nlines\n Height.\n ncols\n Width.\n\nAdjusts other bookkeeping data used by the curses library that record the\nwindow dimensions (in particular the SIGWINCH handler).", + "_curses.savetty" => "Save terminal mode.", + "_curses.set_escdelay" => "Sets the curses ESCDELAY setting.\n\n ms\n length of the delay in milliseconds.\n\nSets the number of milliseconds to wait after reading an escape character,\nto distinguish between an individual escape character entered on the\nkeyboard from escape sequences sent by cursor and function keys.", + "_curses.set_tabsize" => "Sets the curses TABSIZE setting.\n\n size\n rendered cell width of a tab character.\n\nSets the number of columns used by the curses library when converting a tab\ncharacter to spaces as it adds the tab to a window.", + "_curses.setsyx" => "Set the virtual screen cursor.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nIf y and x are both -1, then leaveok is set.", + "_curses.setupterm" => "Initialize the terminal.\n\n term\n Terminal name.\n If omitted, the value of the TERM environment variable will be used.\n fd\n File descriptor to which any initialization sequences will be sent.\n If not supplied, the file descriptor for sys.stdout will be used.", + "_curses.start_color" => "Initializes eight basic colors and global variables COLORS and COLOR_PAIRS.\n\nMust be called if the programmer wants to use colors, and before any other\ncolor manipulation routine is called. It is good practice to call this\nroutine right after initscr().\n\nIt also restores the colors on the terminal to the values they had when the\nterminal was just turned on.", + "_curses.termattrs" => "Return a logical OR of all video attributes supported by the terminal.", + "_curses.termname" => "Return the value of the environment variable TERM, truncated to 14 characters.", + "_curses.tigetflag" => "Return the value of the Boolean capability.\n\n capname\n The terminfo capability name.\n\nThe value -1 is returned if capname is not a Boolean capability, or 0 if\nit is canceled or absent from the terminal description.", + "_curses.tigetnum" => "Return the value of the numeric capability.\n\n capname\n The terminfo capability name.\n\nThe value -2 is returned if capname is not a numeric capability, or -1 if\nit is canceled or absent from the terminal description.", + "_curses.tigetstr" => "Return the value of the string capability.\n\n capname\n The terminfo capability name.\n\nNone is returned if capname is not a string capability, or is canceled or\nabsent from the terminal description.", + "_curses.tparm" => "Instantiate the specified byte string with the supplied parameters.\n\n str\n Parameterized byte string obtained from the terminfo database.", + "_curses.typeahead" => "Specify that the file descriptor fd be used for typeahead checking.\n\n fd\n File descriptor.\n\nIf fd is -1, then no typeahead checking is done.", + "_curses.unctrl" => "Return a string which is a printable representation of the character ch.\n\nControl characters are displayed as a caret followed by the character,\nfor example as ^C. Printing characters are left as they are.", + "_curses.unget_wch" => "Push ch so the next get_wch() will return it.", + "_curses.ungetch" => "Push ch so the next getch() will return it.", + "_curses.ungetmouse" => "Push a KEY_MOUSE event onto the input queue.\n\nThe following getmouse() will return the given state data.", + "_curses.use_default_colors" => "Equivalent to assume_default_colors(-1, -1).", + "_curses.use_env" => "Use environment variables LINES and COLUMNS.\n\nIf used, this function should be called before initscr() or newterm() are\ncalled.\n\nWhen flag is False, the values of lines and columns specified in the terminfo\ndatabase will be used, even if environment variables LINES and COLUMNS (used\nby default) are set, or if curses is running in a window (in which case\ndefault behavior would be to use the window size if LINES and COLUMNS are\nnot set).", + "_curses.window.__delattr__" => "Implement delattr(self, name).", + "_curses.window.__eq__" => "Return self==value.", + "_curses.window.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_curses.window.__ge__" => "Return self>=value.", + "_curses.window.__getattribute__" => "Return getattr(self, name).", + "_curses.window.__getstate__" => "Helper for pickle.", + "_curses.window.__gt__" => "Return self>value.", + "_curses.window.__hash__" => "Return hash(self).", + "_curses.window.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_curses.window.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_curses.window.__le__" => "Return self<=value.", + "_curses.window.__lt__" => "Return self<value.", + "_curses.window.__ne__" => "Return self!=value.", + "_curses.window.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_curses.window.__reduce__" => "Helper for pickle.", + "_curses.window.__reduce_ex__" => "Helper for pickle.", + "_curses.window.__repr__" => "Return repr(self).", + "_curses.window.__setattr__" => "Implement setattr(self, name, value).", + "_curses.window.__sizeof__" => "Size of object in memory, in bytes.", + "_curses.window.__str__" => "Return str(self).", + "_curses.window.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_curses.window.addch" => "addch([y, x,] ch, [attr=_curses.A_NORMAL])\nPaint the character.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n ch\n Character to add.\n attr\n Attributes for the character.\n\nPaint character ch at (y, x) with attributes attr,\noverwriting any character previously painted at that location.\nBy default, the character position and attributes are the\ncurrent settings for the window object.", + "_curses.window.addnstr" => "addnstr([y, x,] str, n, [attr])\nPaint at most n characters of the string.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n str\n String to add.\n n\n Maximal number of characters.\n attr\n Attributes for characters.\n\nPaint at most n characters of the string str at (y, x) with\nattributes attr, overwriting anything previously on the display.\nBy default, the character position and attributes are the\ncurrent settings for the window object.", + "_curses.window.addstr" => "addstr([y, x,] str, [attr])\nPaint the string.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n str\n String to add.\n attr\n Attributes for characters.\n\nPaint the string str at (y, x) with attributes attr,\noverwriting anything previously on the display.\nBy default, the character position and attributes are the\ncurrent settings for the window object.", + "_curses.window.attroff" => "Remove attribute attr from the \"background\" set.", + "_curses.window.attron" => "Add attribute attr to the \"background\" set.", + "_curses.window.attrset" => "Set the \"background\" set of attributes.", + "_curses.window.bkgd" => "Set the background property of the window.\n\n ch\n Background character.\n attr\n Background attributes.", + "_curses.window.bkgdset" => "Set the window's background.\n\n ch\n Background character.\n attr\n Background attributes.", + "_curses.window.border" => "Draw a border around the edges of the window.\n\n ls\n Left side.\n rs\n Right side.\n ts\n Top side.\n bs\n Bottom side.\n tl\n Upper-left corner.\n tr\n Upper-right corner.\n bl\n Bottom-left corner.\n br\n Bottom-right corner.\n\nEach parameter specifies the character to use for a specific part of the\nborder. The characters can be specified as integers or as one-character\nstrings. A 0 value for any parameter will cause the default character to be\nused for that parameter.", + "_curses.window.box" => "box([verch=0, horch=0])\nDraw a border around the edges of the window.\n\n verch\n Left and right side.\n horch\n Top and bottom side.\n\nSimilar to border(), but both ls and rs are verch and both ts and bs are\nhorch. The default corner characters are always used by this function.", + "_curses.window.chgat" => "chgat([y, x,] [n=-1,] attr)\nSet the attributes of characters.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n n\n Number of characters.\n attr\n Attributes for characters.\n\nSet the attributes of num characters at the current cursor position, or at\nposition (y, x) if supplied. If no value of num is given or num = -1, the\nattribute will be set on all the characters to the end of the line. This\nfunction does not move the cursor. The changed line will be touched using\nthe touchline() method so that the contents will be redisplayed by the next\nwindow refresh.", + "_curses.window.delch" => "delch([y, x])\nDelete any character at (y, x).\n\n y\n Y-coordinate.\n x\n X-coordinate.", + "_curses.window.derwin" => "derwin([nlines=0, ncols=0,] begin_y, begin_x)\nCreate a sub-window (window-relative coordinates).\n\n nlines\n Height.\n ncols\n Width.\n begin_y\n Top side y-coordinate.\n begin_x\n Left side x-coordinate.\n\nderwin() is the same as calling subwin(), except that begin_y and begin_x\nare relative to the origin of the window, rather than relative to the entire\nscreen.", + "_curses.window.echochar" => "Add character ch with attribute attr, and refresh.\n\n ch\n Character to add.\n attr\n Attributes for the character.", + "_curses.window.enclose" => "Return True if the screen-relative coordinates are enclosed by the window.\n\n y\n Y-coordinate.\n x\n X-coordinate.", + "_curses.window.encoding" => "the typecode character used to create the array", + "_curses.window.get_wch" => "get_wch([y, x])\nGet a wide character from terminal keyboard.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nReturn a character for most keys, or an integer for function keys,\nkeypad keys, and other special keys.", + "_curses.window.getbkgd" => "Return the window's current background character/attribute pair.", + "_curses.window.getch" => "getch([y, x])\nGet a character code from terminal keyboard.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nThe integer returned does not have to be in ASCII range: function keys,\nkeypad keys and so on return numbers higher than 256. In no-delay mode, -1\nis returned if there is no input, else getch() waits until a key is pressed.", + "_curses.window.getkey" => "getkey([y, x])\nGet a character (string) from terminal keyboard.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nReturning a string instead of an integer, as getch() does. Function keys,\nkeypad keys and other special keys return a multibyte string containing the\nkey name. In no-delay mode, an exception is raised if there is no input.", + "_curses.window.getstr" => "getstr([[y, x,] n=2047])\nRead a string from the user, with primitive line editing capacity.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n n\n Maximal number of characters.", + "_curses.window.hline" => "hline([y, x,] ch, n, [attr=_curses.A_NORMAL])\nDisplay a horizontal line.\n\n y\n Starting Y-coordinate.\n x\n Starting X-coordinate.\n ch\n Character to draw.\n n\n Line length.\n attr\n Attributes for the characters.", + "_curses.window.inch" => "inch([y, x])\nReturn the character at the given position in the window.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n\nThe bottom 8 bits are the character proper, and upper bits are the attributes.", + "_curses.window.insch" => "insch([y, x,] ch, [attr=_curses.A_NORMAL])\nInsert a character before the current or specified position.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n ch\n Character to insert.\n attr\n Attributes for the character.\n\nAll characters to the right of the cursor are shifted one position right, with\nthe rightmost characters on the line being lost.", + "_curses.window.insnstr" => "insnstr([y, x,] str, n, [attr])\nInsert at most n characters of the string.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n str\n String to insert.\n n\n Maximal number of characters.\n attr\n Attributes for characters.\n\nInsert a character string (as many characters as will fit on the line)\nbefore the character under the cursor, up to n characters. If n is zero\nor negative, the entire string is inserted. All characters to the right\nof the cursor are shifted right, with the rightmost characters on the line\nbeing lost. The cursor position does not change (after moving to y, x, if\nspecified).", + "_curses.window.insstr" => "insstr([y, x,] str, [attr])\nInsert the string before the current or specified position.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n str\n String to insert.\n attr\n Attributes for characters.\n\nInsert a character string (as many characters as will fit on the line)\nbefore the character under the cursor. All characters to the right of\nthe cursor are shifted right, with the rightmost characters on the line\nbeing lost. The cursor position does not change (after moving to y, x,\nif specified).", + "_curses.window.instr" => "instr([y, x,] n=2047)\nReturn a string of characters, extracted from the window.\n\n y\n Y-coordinate.\n x\n X-coordinate.\n n\n Maximal number of characters.\n\nReturn a string of characters, extracted from the window starting at the\ncurrent cursor position, or at y, x if specified. Attributes are stripped\nfrom the characters. If n is specified, instr() returns a string at most\nn characters long (exclusive of the trailing NUL).", + "_curses.window.is_linetouched" => "Return True if the specified line was modified, otherwise return False.\n\n line\n Line number.\n\nRaise a curses.error exception if line is not valid for the given window.", + "_curses.window.noutrefresh" => "noutrefresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol])\nMark for refresh but wait.\n\nThis function updates the data structure representing the desired state of the\nwindow, but does not force an update of the physical screen. To accomplish\nthat, call doupdate().", + "_curses.window.overlay" => "overlay(destwin, [sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol])\nOverlay the window on top of destwin.\n\nThe windows need not be the same size, only the overlapping region is copied.\nThis copy is non-destructive, which means that the current background\ncharacter does not overwrite the old contents of destwin.\n\nTo get fine-grained control over the copied region, the second form of\noverlay() can be used. sminrow and smincol are the upper-left coordinates\nof the source window, and the other variables mark a rectangle in the\ndestination window.", + "_curses.window.overwrite" => "overwrite(destwin, [sminrow, smincol, dminrow, dmincol, dmaxrow,\n dmaxcol])\nOverwrite the window on top of destwin.\n\nThe windows need not be the same size, in which case only the overlapping\nregion is copied. This copy is destructive, which means that the current\nbackground character overwrites the old contents of destwin.\n\nTo get fine-grained control over the copied region, the second form of\noverwrite() can be used. sminrow and smincol are the upper-left coordinates\nof the source window, the other variables mark a rectangle in the destination\nwindow.", + "_curses.window.putwin" => "Write all data associated with the window into the provided file object.\n\nThis information can be later retrieved using the getwin() function.", + "_curses.window.redrawln" => "Mark the specified lines corrupted.\n\n beg\n Starting line number.\n num\n The number of lines.\n\nThey should be completely redrawn on the next refresh() call.", + "_curses.window.refresh" => "refresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol])\nUpdate the display immediately.\n\nSynchronize actual screen with previous drawing/deleting methods.\nThe 6 optional arguments can only be specified when the window is a pad\ncreated with newpad(). The additional parameters are needed to indicate\nwhat part of the pad and screen are involved. pminrow and pmincol specify\nthe upper left-hand corner of the rectangle to be displayed in the pad.\nsminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to\nbe displayed on the screen. The lower right-hand corner of the rectangle to\nbe displayed in the pad is calculated from the screen coordinates, since the\nrectangles must be the same size. Both rectangles must be entirely contained\nwithin their respective structures. Negative values of pminrow, pmincol,\nsminrow, or smincol are treated as if they were zero.", + "_curses.window.scroll" => "scroll([lines=1])\nScroll the screen or scrolling region.\n\n lines\n Number of lines to scroll.\n\nScroll upward if the argument is positive and downward if it is negative.", + "_curses.window.setscrreg" => "Define a software scrolling region.\n\n top\n First line number.\n bottom\n Last line number.\n\nAll scrolling actions will take place in this region.", + "_curses.window.subpad" => "subwin([nlines=0, ncols=0,] begin_y, begin_x)\nCreate a sub-window (screen-relative coordinates).\n\n nlines\n Height.\n ncols\n Width.\n begin_y\n Top side y-coordinate.\n begin_x\n Left side x-coordinate.\n\nBy default, the sub-window will extend from the specified position to the\nlower right corner of the window.", + "_curses.window.subwin" => "subwin([nlines=0, ncols=0,] begin_y, begin_x)\nCreate a sub-window (screen-relative coordinates).\n\n nlines\n Height.\n ncols\n Width.\n begin_y\n Top side y-coordinate.\n begin_x\n Left side x-coordinate.\n\nBy default, the sub-window will extend from the specified position to the\nlower right corner of the window.", + "_curses.window.touchline" => "touchline(start, count, [changed=True])\nPretend count lines have been changed, starting with line start.\n\nIf changed is supplied, it specifies whether the affected lines are marked\nas having been changed (changed=True) or unchanged (changed=False).", + "_curses.window.vline" => "vline([y, x,] ch, n, [attr=_curses.A_NORMAL])\nDisplay a vertical line.\n\n y\n Starting Y-coordinate.\n x\n Starting X-coordinate.\n ch\n Character to draw.\n n\n Line length.\n attr\n Attributes for the character.", + "_curses_panel.bottom_panel" => "Return the bottom panel in the panel stack.", + "_curses_panel.error.__delattr__" => "Implement delattr(self, name).", + "_curses_panel.error.__eq__" => "Return self==value.", + "_curses_panel.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_curses_panel.error.__ge__" => "Return self>=value.", + "_curses_panel.error.__getattribute__" => "Return getattr(self, name).", + "_curses_panel.error.__getstate__" => "Helper for pickle.", + "_curses_panel.error.__gt__" => "Return self>value.", + "_curses_panel.error.__hash__" => "Return hash(self).", + "_curses_panel.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_curses_panel.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_curses_panel.error.__le__" => "Return self<=value.", + "_curses_panel.error.__lt__" => "Return self<value.", + "_curses_panel.error.__ne__" => "Return self!=value.", + "_curses_panel.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_curses_panel.error.__reduce_ex__" => "Helper for pickle.", + "_curses_panel.error.__repr__" => "Return repr(self).", + "_curses_panel.error.__setattr__" => "Implement setattr(self, name, value).", + "_curses_panel.error.__sizeof__" => "Size of object in memory, in bytes.", + "_curses_panel.error.__str__" => "Return str(self).", + "_curses_panel.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_curses_panel.error.__weakref__" => "list of weak references to the object", + "_curses_panel.error.add_note" => "Add a note to the exception", + "_curses_panel.error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_curses_panel.new_panel" => "Return a panel object, associating it with the given window win.", + "_curses_panel.panel.__delattr__" => "Implement delattr(self, name).", + "_curses_panel.panel.__eq__" => "Return self==value.", + "_curses_panel.panel.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_curses_panel.panel.__ge__" => "Return self>=value.", + "_curses_panel.panel.__getattribute__" => "Return getattr(self, name).", + "_curses_panel.panel.__getstate__" => "Helper for pickle.", + "_curses_panel.panel.__gt__" => "Return self>value.", + "_curses_panel.panel.__hash__" => "Return hash(self).", + "_curses_panel.panel.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_curses_panel.panel.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_curses_panel.panel.__le__" => "Return self<=value.", + "_curses_panel.panel.__lt__" => "Return self<value.", + "_curses_panel.panel.__ne__" => "Return self!=value.", + "_curses_panel.panel.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_curses_panel.panel.__reduce__" => "Helper for pickle.", + "_curses_panel.panel.__reduce_ex__" => "Helper for pickle.", + "_curses_panel.panel.__repr__" => "Return repr(self).", + "_curses_panel.panel.__setattr__" => "Implement setattr(self, name, value).", + "_curses_panel.panel.__sizeof__" => "Size of object in memory, in bytes.", + "_curses_panel.panel.__str__" => "Return str(self).", + "_curses_panel.panel.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_curses_panel.panel.above" => "Return the panel above the current panel.", + "_curses_panel.panel.below" => "Return the panel below the current panel.", + "_curses_panel.panel.bottom" => "Push the panel to the bottom of the stack.", + "_curses_panel.panel.hidden" => "Return True if the panel is hidden (not visible), False otherwise.", + "_curses_panel.panel.hide" => "Hide the panel.\n\nThis does not delete the object, it just makes the window on screen invisible.", + "_curses_panel.panel.move" => "Move the panel to the screen coordinates (y, x).", + "_curses_panel.panel.replace" => "Change the window associated with the panel to the window win.", + "_curses_panel.panel.set_userptr" => "Set the panel's user pointer to obj.", + "_curses_panel.panel.show" => "Display the panel (which might have been hidden).", + "_curses_panel.panel.top" => "Push panel to the top of the stack.", + "_curses_panel.panel.userptr" => "Return the user pointer for the panel.", + "_curses_panel.panel.window" => "Return the window object associated with the panel.", + "_curses_panel.top_panel" => "Return the top panel in the panel stack.", + "_curses_panel.update_panels" => "Updates the virtual screen after changes in the panel stack.\n\nThis does not call curses.doupdate(), so you'll have to do this yourself.", + "_datetime" => "Fast implementation of the datetime module.", + "_datetime.date" => "date(year, month, day) --> date object", + "_datetime.date.__add__" => "Return self+value.", + "_datetime.date.__delattr__" => "Implement delattr(self, name).", + "_datetime.date.__eq__" => "Return self==value.", + "_datetime.date.__format__" => "Formats self with strftime.", + "_datetime.date.__ge__" => "Return self>=value.", + "_datetime.date.__getattribute__" => "Return getattr(self, name).", + "_datetime.date.__getstate__" => "Helper for pickle.", + "_datetime.date.__gt__" => "Return self>value.", + "_datetime.date.__hash__" => "Return hash(self).", + "_datetime.date.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.date.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.date.__le__" => "Return self<=value.", + "_datetime.date.__lt__" => "Return self<value.", + "_datetime.date.__ne__" => "Return self!=value.", + "_datetime.date.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.date.__radd__" => "Return value+self.", + "_datetime.date.__reduce__" => "__reduce__() -> (cls, state)", + "_datetime.date.__reduce_ex__" => "Helper for pickle.", + "_datetime.date.__replace__" => "The same as replace().", + "_datetime.date.__repr__" => "Return repr(self).", + "_datetime.date.__rsub__" => "Return value-self.", + "_datetime.date.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.date.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.date.__str__" => "Return str(self).", + "_datetime.date.__sub__" => "Return self-value.", + "_datetime.date.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.date.ctime" => "Return ctime() style string.", + "_datetime.date.fromisocalendar" => "int, int, int -> Construct a date from the ISO year, week number and weekday.\n\nThis is the inverse of the date.isocalendar() function", + "_datetime.date.fromisoformat" => "str -> Construct a date from a string in ISO 8601 format.", + "_datetime.date.fromordinal" => "int -> date corresponding to a proleptic Gregorian ordinal.", + "_datetime.date.fromtimestamp" => "Create a date from a POSIX timestamp.\n\nThe timestamp is a number, e.g. created via time.time(), that is interpreted\nas local time.", + "_datetime.date.isocalendar" => "Return a named tuple containing ISO year, week number, and weekday.", + "_datetime.date.isoformat" => "Return string in ISO 8601 format, YYYY-MM-DD.", + "_datetime.date.isoweekday" => "Return the day of the week represented by the date.\nMonday == 1 ... Sunday == 7", + "_datetime.date.replace" => "Return date with new specified fields.", + "_datetime.date.strftime" => "format -> strftime() style string.", + "_datetime.date.strptime" => "string, format -> new date parsed from a string (like time.strptime()).", + "_datetime.date.timetuple" => "Return time tuple, compatible with time.localtime().", + "_datetime.date.today" => "Current date or datetime: same as self.__class__.fromtimestamp(time.time()).", + "_datetime.date.toordinal" => "Return proleptic Gregorian ordinal. January 1 of year 1 is day 1.", + "_datetime.date.weekday" => "Return the day of the week represented by the date.\nMonday == 0 ... Sunday == 6", + "_datetime.datetime" => "datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])\n\nThe year, month and day arguments are required. tzinfo may be None, or an\ninstance of a tzinfo subclass. The remaining arguments may be ints.", + "_datetime.datetime.__add__" => "Return self+value.", + "_datetime.datetime.__delattr__" => "Implement delattr(self, name).", + "_datetime.datetime.__eq__" => "Return self==value.", + "_datetime.datetime.__format__" => "Formats self with strftime.", + "_datetime.datetime.__ge__" => "Return self>=value.", + "_datetime.datetime.__getattribute__" => "Return getattr(self, name).", + "_datetime.datetime.__getstate__" => "Helper for pickle.", + "_datetime.datetime.__gt__" => "Return self>value.", + "_datetime.datetime.__hash__" => "Return hash(self).", + "_datetime.datetime.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.datetime.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.datetime.__le__" => "Return self<=value.", + "_datetime.datetime.__lt__" => "Return self<value.", + "_datetime.datetime.__ne__" => "Return self!=value.", + "_datetime.datetime.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.datetime.__radd__" => "Return value+self.", + "_datetime.datetime.__reduce__" => "__reduce__() -> (cls, state)", + "_datetime.datetime.__reduce_ex__" => "__reduce_ex__(proto) -> (cls, state)", + "_datetime.datetime.__replace__" => "The same as replace().", + "_datetime.datetime.__repr__" => "Return repr(self).", + "_datetime.datetime.__rsub__" => "Return value-self.", + "_datetime.datetime.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.datetime.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.datetime.__str__" => "Return str(self).", + "_datetime.datetime.__sub__" => "Return self-value.", + "_datetime.datetime.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.datetime.astimezone" => "tz -> convert to local time in new timezone tz", + "_datetime.datetime.combine" => "date, time -> datetime with same date and time fields", + "_datetime.datetime.ctime" => "Return ctime() style string.", + "_datetime.datetime.date" => "Return date object with same year, month and day.", + "_datetime.datetime.dst" => "Return self.tzinfo.dst(self).", + "_datetime.datetime.fromisocalendar" => "int, int, int -> Construct a date from the ISO year, week number and weekday.\n\nThis is the inverse of the date.isocalendar() function", + "_datetime.datetime.fromisoformat" => "string -> datetime from a string in most ISO 8601 formats", + "_datetime.datetime.fromordinal" => "int -> date corresponding to a proleptic Gregorian ordinal.", + "_datetime.datetime.fromtimestamp" => "timestamp[, tz] -> tz's local time from POSIX timestamp.", + "_datetime.datetime.isocalendar" => "Return a named tuple containing ISO year, week number, and weekday.", + "_datetime.datetime.isoformat" => "[sep] -> string in ISO 8601 format, YYYY-MM-DDT[HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\nsep is used to separate the year from the time, and defaults to 'T'.\nThe optional argument timespec specifies the number of additional terms\nof the time to include. Valid options are 'auto', 'hours', 'minutes',\n'seconds', 'milliseconds' and 'microseconds'.", + "_datetime.datetime.isoweekday" => "Return the day of the week represented by the date.\nMonday == 1 ... Sunday == 7", + "_datetime.datetime.now" => "Returns new datetime object representing current time local to tz.\n\n tz\n Timezone object.\n\nIf no tz is specified, uses local timezone.", + "_datetime.datetime.replace" => "Return datetime with new specified fields.", + "_datetime.datetime.strftime" => "format -> strftime() style string.", + "_datetime.datetime.strptime" => "string, format -> new datetime parsed from a string (like time.strptime()).", + "_datetime.datetime.time" => "Return time object with same time but with tzinfo=None.", + "_datetime.datetime.timestamp" => "Return POSIX timestamp as float.", + "_datetime.datetime.timetuple" => "Return time tuple, compatible with time.localtime().", + "_datetime.datetime.timetz" => "Return time object with same time and tzinfo.", + "_datetime.datetime.today" => "Current date or datetime: same as self.__class__.fromtimestamp(time.time()).", + "_datetime.datetime.toordinal" => "Return proleptic Gregorian ordinal. January 1 of year 1 is day 1.", + "_datetime.datetime.tzname" => "Return self.tzinfo.tzname(self).", + "_datetime.datetime.utcfromtimestamp" => "Construct a naive UTC datetime from a POSIX timestamp.", + "_datetime.datetime.utcnow" => "Return a new datetime representing UTC day and time.", + "_datetime.datetime.utcoffset" => "Return self.tzinfo.utcoffset(self).", + "_datetime.datetime.utctimetuple" => "Return UTC time tuple, compatible with time.localtime().", + "_datetime.datetime.weekday" => "Return the day of the week represented by the date.\nMonday == 0 ... Sunday == 6", + "_datetime.time" => "time([hour[, minute[, second[, microsecond[, tzinfo]]]]]) --> a time object\n\nAll arguments are optional. tzinfo may be None, or an instance of\na tzinfo subclass. The remaining arguments may be ints.", + "_datetime.time.__delattr__" => "Implement delattr(self, name).", + "_datetime.time.__eq__" => "Return self==value.", + "_datetime.time.__format__" => "Formats self with strftime.", + "_datetime.time.__ge__" => "Return self>=value.", + "_datetime.time.__getattribute__" => "Return getattr(self, name).", + "_datetime.time.__getstate__" => "Helper for pickle.", + "_datetime.time.__gt__" => "Return self>value.", + "_datetime.time.__hash__" => "Return hash(self).", + "_datetime.time.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.time.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.time.__le__" => "Return self<=value.", + "_datetime.time.__lt__" => "Return self<value.", + "_datetime.time.__ne__" => "Return self!=value.", + "_datetime.time.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.time.__reduce__" => "__reduce__() -> (cls, state)", + "_datetime.time.__reduce_ex__" => "__reduce_ex__(proto) -> (cls, state)", + "_datetime.time.__replace__" => "The same as replace().", + "_datetime.time.__repr__" => "Return repr(self).", + "_datetime.time.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.time.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.time.__str__" => "Return str(self).", + "_datetime.time.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.time.dst" => "Return self.tzinfo.dst(self).", + "_datetime.time.fromisoformat" => "string -> time from a string in ISO 8601 format", + "_datetime.time.isoformat" => "Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\n\nThe optional argument timespec specifies the number of additional terms\nof the time to include. Valid options are 'auto', 'hours', 'minutes',\n'seconds', 'milliseconds' and 'microseconds'.", + "_datetime.time.replace" => "Return time with new specified fields.", + "_datetime.time.strftime" => "format -> strftime() style string.", + "_datetime.time.strptime" => "string, format -> new time parsed from a string (like time.strptime()).", + "_datetime.time.tzname" => "Return self.tzinfo.tzname(self).", + "_datetime.time.utcoffset" => "Return self.tzinfo.utcoffset(self).", + "_datetime.timedelta" => "Difference between two datetime values.\n\ntimedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)\n\nAll arguments are optional and default to 0.\nArguments may be integers or floats, and may be positive or negative.", + "_datetime.timedelta.__abs__" => "abs(self)", + "_datetime.timedelta.__add__" => "Return self+value.", + "_datetime.timedelta.__bool__" => "True if self else False", + "_datetime.timedelta.__delattr__" => "Implement delattr(self, name).", + "_datetime.timedelta.__divmod__" => "Return divmod(self, value).", + "_datetime.timedelta.__eq__" => "Return self==value.", + "_datetime.timedelta.__floordiv__" => "Return self//value.", + "_datetime.timedelta.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_datetime.timedelta.__ge__" => "Return self>=value.", + "_datetime.timedelta.__getattribute__" => "Return getattr(self, name).", + "_datetime.timedelta.__getstate__" => "Helper for pickle.", + "_datetime.timedelta.__gt__" => "Return self>value.", + "_datetime.timedelta.__hash__" => "Return hash(self).", + "_datetime.timedelta.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.timedelta.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.timedelta.__le__" => "Return self<=value.", + "_datetime.timedelta.__lt__" => "Return self<value.", + "_datetime.timedelta.__mod__" => "Return self%value.", + "_datetime.timedelta.__mul__" => "Return self*value.", + "_datetime.timedelta.__ne__" => "Return self!=value.", + "_datetime.timedelta.__neg__" => "-self", + "_datetime.timedelta.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.timedelta.__pos__" => "+self", + "_datetime.timedelta.__radd__" => "Return value+self.", + "_datetime.timedelta.__rdivmod__" => "Return divmod(value, self).", + "_datetime.timedelta.__reduce__" => "__reduce__() -> (cls, state)", + "_datetime.timedelta.__reduce_ex__" => "Helper for pickle.", + "_datetime.timedelta.__repr__" => "Return repr(self).", + "_datetime.timedelta.__rfloordiv__" => "Return value//self.", + "_datetime.timedelta.__rmod__" => "Return value%self.", + "_datetime.timedelta.__rmul__" => "Return value*self.", + "_datetime.timedelta.__rsub__" => "Return value-self.", + "_datetime.timedelta.__rtruediv__" => "Return value/self.", + "_datetime.timedelta.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.timedelta.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.timedelta.__str__" => "Return str(self).", + "_datetime.timedelta.__sub__" => "Return self-value.", + "_datetime.timedelta.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.timedelta.__truediv__" => "Return self/value.", + "_datetime.timedelta.days" => "Number of days.", + "_datetime.timedelta.microseconds" => "Number of microseconds (>= 0 and less than 1 second).", + "_datetime.timedelta.seconds" => "Number of seconds (>= 0 and less than 1 day).", + "_datetime.timedelta.total_seconds" => "Total seconds in the duration.", + "_datetime.timezone" => "Fixed offset from UTC implementation of tzinfo.", + "_datetime.timezone.__delattr__" => "Implement delattr(self, name).", + "_datetime.timezone.__eq__" => "Return self==value.", + "_datetime.timezone.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_datetime.timezone.__ge__" => "Return self>=value.", + "_datetime.timezone.__getattribute__" => "Return getattr(self, name).", + "_datetime.timezone.__getinitargs__" => "pickle support", + "_datetime.timezone.__getstate__" => "Helper for pickle.", + "_datetime.timezone.__gt__" => "Return self>value.", + "_datetime.timezone.__hash__" => "Return hash(self).", + "_datetime.timezone.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.timezone.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.timezone.__le__" => "Return self<=value.", + "_datetime.timezone.__lt__" => "Return self<value.", + "_datetime.timezone.__ne__" => "Return self!=value.", + "_datetime.timezone.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.timezone.__reduce__" => "-> (cls, state)", + "_datetime.timezone.__reduce_ex__" => "Helper for pickle.", + "_datetime.timezone.__repr__" => "Return repr(self).", + "_datetime.timezone.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.timezone.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.timezone.__str__" => "Return str(self).", + "_datetime.timezone.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.timezone.dst" => "Return None.", + "_datetime.timezone.fromutc" => "datetime in UTC -> datetime in local time.", + "_datetime.timezone.tzname" => "If name is specified when timezone is created, returns the name. Otherwise returns offset as 'UTC(+|-)HH:MM'.", + "_datetime.timezone.utcoffset" => "Return fixed offset.", + "_datetime.tzinfo" => "Abstract base class for time zone info objects.", + "_datetime.tzinfo.__delattr__" => "Implement delattr(self, name).", + "_datetime.tzinfo.__eq__" => "Return self==value.", + "_datetime.tzinfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_datetime.tzinfo.__ge__" => "Return self>=value.", + "_datetime.tzinfo.__getattribute__" => "Return getattr(self, name).", + "_datetime.tzinfo.__getstate__" => "Helper for pickle.", + "_datetime.tzinfo.__gt__" => "Return self>value.", + "_datetime.tzinfo.__hash__" => "Return hash(self).", + "_datetime.tzinfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_datetime.tzinfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_datetime.tzinfo.__le__" => "Return self<=value.", + "_datetime.tzinfo.__lt__" => "Return self<value.", + "_datetime.tzinfo.__ne__" => "Return self!=value.", + "_datetime.tzinfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_datetime.tzinfo.__reduce__" => "-> (cls, state)", + "_datetime.tzinfo.__reduce_ex__" => "Helper for pickle.", + "_datetime.tzinfo.__repr__" => "Return repr(self).", + "_datetime.tzinfo.__setattr__" => "Implement setattr(self, name, value).", + "_datetime.tzinfo.__sizeof__" => "Size of object in memory, in bytes.", + "_datetime.tzinfo.__str__" => "Return str(self).", + "_datetime.tzinfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_datetime.tzinfo.dst" => "datetime -> DST offset as timedelta positive east of UTC.", + "_datetime.tzinfo.fromutc" => "datetime in UTC -> datetime in local time.", + "_datetime.tzinfo.tzname" => "datetime -> string name of time zone.", + "_datetime.tzinfo.utcoffset" => "datetime -> timedelta showing offset from UTC, negative values indicating West of UTC", + "_dbm.error.__delattr__" => "Implement delattr(self, name).", + "_dbm.error.__eq__" => "Return self==value.", + "_dbm.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_dbm.error.__ge__" => "Return self>=value.", + "_dbm.error.__getattribute__" => "Return getattr(self, name).", + "_dbm.error.__getstate__" => "Helper for pickle.", + "_dbm.error.__gt__" => "Return self>value.", + "_dbm.error.__hash__" => "Return hash(self).", + "_dbm.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_dbm.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_dbm.error.__le__" => "Return self<=value.", + "_dbm.error.__lt__" => "Return self<value.", + "_dbm.error.__ne__" => "Return self!=value.", + "_dbm.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_dbm.error.__reduce_ex__" => "Helper for pickle.", + "_dbm.error.__repr__" => "Return repr(self).", + "_dbm.error.__setattr__" => "Implement setattr(self, name, value).", + "_dbm.error.__sizeof__" => "Size of object in memory, in bytes.", + "_dbm.error.__str__" => "Return str(self).", + "_dbm.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_dbm.error.__weakref__" => "list of weak references to the object", + "_dbm.error.add_note" => "Add a note to the exception", + "_dbm.error.errno" => "POSIX exception code", + "_dbm.error.filename" => "exception filename", + "_dbm.error.filename2" => "second exception filename", + "_dbm.error.strerror" => "exception strerror", + "_dbm.error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_dbm.open" => "Return a database object.\n\n filename\n The filename to open.\n flags\n How to open the file. \"r\" for reading, \"w\" for writing, etc.\n mode\n If creating a new file, the mode bits for the new file\n (e.g. os.O_RDWR).", + "_decimal" => "C decimal arithmetic module", + "_decimal.Clamped.__delattr__" => "Implement delattr(self, name).", + "_decimal.Clamped.__eq__" => "Return self==value.", + "_decimal.Clamped.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Clamped.__ge__" => "Return self>=value.", + "_decimal.Clamped.__getattribute__" => "Return getattr(self, name).", + "_decimal.Clamped.__getstate__" => "Helper for pickle.", + "_decimal.Clamped.__gt__" => "Return self>value.", + "_decimal.Clamped.__hash__" => "Return hash(self).", + "_decimal.Clamped.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Clamped.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Clamped.__le__" => "Return self<=value.", + "_decimal.Clamped.__lt__" => "Return self<value.", + "_decimal.Clamped.__ne__" => "Return self!=value.", + "_decimal.Clamped.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Clamped.__reduce_ex__" => "Helper for pickle.", + "_decimal.Clamped.__repr__" => "Return repr(self).", + "_decimal.Clamped.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Clamped.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Clamped.__str__" => "Return str(self).", + "_decimal.Clamped.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Clamped.__weakref__" => "list of weak references to the object", + "_decimal.Clamped.add_note" => "Add a note to the exception", + "_decimal.Clamped.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Context" => "The context affects almost all operations and controls rounding,\nOver/Underflow, raising of exceptions and much more. A new context\ncan be constructed as follows:\n\n >>> c = Context(prec=28, Emin=-425000000, Emax=425000000,\n ... rounding=ROUND_HALF_EVEN, capitals=1, clamp=1,\n ... traps=[InvalidOperation, DivisionByZero, Overflow],\n ... flags=[])\n >>>", + "_decimal.Context.Etiny" => "Return a value equal to Emin - prec + 1, which is the minimum exponent value\nfor subnormal results. When underflow occurs, the exponent is set to Etiny.", + "_decimal.Context.Etop" => "Return a value equal to Emax - prec + 1. This is the maximum exponent\nif the _clamp field of the context is set to 1 (IEEE clamp mode). Etop()\nmust not be negative.", + "_decimal.Context.__delattr__" => "Implement delattr(self, name).", + "_decimal.Context.__eq__" => "Return self==value.", + "_decimal.Context.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Context.__ge__" => "Return self>=value.", + "_decimal.Context.__getattribute__" => "Return getattr(self, name).", + "_decimal.Context.__getstate__" => "Helper for pickle.", + "_decimal.Context.__gt__" => "Return self>value.", + "_decimal.Context.__hash__" => "Return hash(self).", + "_decimal.Context.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Context.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Context.__le__" => "Return self<=value.", + "_decimal.Context.__lt__" => "Return self<value.", + "_decimal.Context.__ne__" => "Return self!=value.", + "_decimal.Context.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Context.__reduce_ex__" => "Helper for pickle.", + "_decimal.Context.__repr__" => "Return repr(self).", + "_decimal.Context.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Context.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Context.__str__" => "Return str(self).", + "_decimal.Context.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Context.abs" => "Return the absolute value of x.", + "_decimal.Context.add" => "Return the sum of x and y.", + "_decimal.Context.canonical" => "Return a new instance of x.", + "_decimal.Context.clear_flags" => "Reset all flags to False.", + "_decimal.Context.clear_traps" => "Set all traps to False.", + "_decimal.Context.compare" => "Compare x and y numerically.", + "_decimal.Context.compare_signal" => "Compare x and y numerically. All NaNs signal.", + "_decimal.Context.compare_total" => "Compare x and y using their abstract representation.", + "_decimal.Context.compare_total_mag" => "Compare x and y using their abstract representation, ignoring sign.", + "_decimal.Context.copy" => "Return a duplicate of the context with all flags cleared.", + "_decimal.Context.copy_abs" => "Return a copy of x with the sign set to 0.", + "_decimal.Context.copy_decimal" => "Return a copy of Decimal x.", + "_decimal.Context.copy_negate" => "Return a copy of x with the sign inverted.", + "_decimal.Context.copy_sign" => "Copy the sign from y to x.", + "_decimal.Context.create_decimal" => "Create a new Decimal instance from num, using self as the context. Unlike the\nDecimal constructor, this function observes the context limits.", + "_decimal.Context.create_decimal_from_float" => "Create a new Decimal instance from float f. Unlike the Decimal.from_float()\nclass method, this function observes the context limits.", + "_decimal.Context.divide" => "Return x divided by y.", + "_decimal.Context.divide_int" => "Return x divided by y, truncated to an integer.", + "_decimal.Context.divmod" => "Return quotient and remainder of the division x / y.", + "_decimal.Context.exp" => "Return e ** x.", + "_decimal.Context.fma" => "Return x multiplied by y, plus z.", + "_decimal.Context.is_canonical" => "Return True if x is canonical, False otherwise.", + "_decimal.Context.is_finite" => "Return True if x is finite, False otherwise.", + "_decimal.Context.is_infinite" => "Return True if x is infinite, False otherwise.", + "_decimal.Context.is_nan" => "Return True if x is a qNaN or sNaN, False otherwise.", + "_decimal.Context.is_normal" => "Return True if x is a normal number, False otherwise.", + "_decimal.Context.is_qnan" => "Return True if x is a quiet NaN, False otherwise.", + "_decimal.Context.is_signed" => "Return True if x is negative, False otherwise.", + "_decimal.Context.is_snan" => "Return True if x is a signaling NaN, False otherwise.", + "_decimal.Context.is_subnormal" => "Return True if x is subnormal, False otherwise.", + "_decimal.Context.is_zero" => "Return True if x is a zero, False otherwise.", + "_decimal.Context.ln" => "Return the natural (base e) logarithm of x.", + "_decimal.Context.log10" => "Return the base 10 logarithm of x.", + "_decimal.Context.logb" => "Return the exponent of the magnitude of the operand's MSD.", + "_decimal.Context.logical_and" => "Applies the logical operation 'and' between each operand's digits.\n\nThe operands must be both logical numbers.\n\n >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0'))\n Decimal('0')\n >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1'))\n Decimal('0')\n >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0'))\n Decimal('0')\n >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1'))\n Decimal('1')\n >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010'))\n Decimal('1000')\n >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10'))\n Decimal('10')\n >>> ExtendedContext.logical_and(110, 1101)\n Decimal('100')\n >>> ExtendedContext.logical_and(Decimal(110), 1101)\n Decimal('100')\n >>> ExtendedContext.logical_and(110, Decimal(1101))\n Decimal('100')", + "_decimal.Context.logical_invert" => "Invert all the digits in the operand.\n\nThe operand must be a logical number.\n\n >>> ExtendedContext.logical_invert(Decimal('0'))\n Decimal('111111111')\n >>> ExtendedContext.logical_invert(Decimal('1'))\n Decimal('111111110')\n >>> ExtendedContext.logical_invert(Decimal('111111111'))\n Decimal('0')\n >>> ExtendedContext.logical_invert(Decimal('101010101'))\n Decimal('10101010')\n >>> ExtendedContext.logical_invert(1101)\n Decimal('111110010')", + "_decimal.Context.logical_or" => "Applies the logical operation 'or' between each operand's digits.\n\nThe operands must be both logical numbers.\n\n >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0'))\n Decimal('0')\n >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1'))\n Decimal('1')\n >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0'))\n Decimal('1')\n >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1'))\n Decimal('1')\n >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010'))\n Decimal('1110')\n >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10'))\n Decimal('1110')\n >>> ExtendedContext.logical_or(110, 1101)\n Decimal('1111')\n >>> ExtendedContext.logical_or(Decimal(110), 1101)\n Decimal('1111')\n >>> ExtendedContext.logical_or(110, Decimal(1101))\n Decimal('1111')", + "_decimal.Context.logical_xor" => "Applies the logical operation 'xor' between each operand's digits.\n\nThe operands must be both logical numbers.\n\n >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0'))\n Decimal('0')\n >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1'))\n Decimal('1')\n >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0'))\n Decimal('1')\n >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1'))\n Decimal('0')\n >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010'))\n Decimal('110')\n >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10'))\n Decimal('1101')\n >>> ExtendedContext.logical_xor(110, 1101)\n Decimal('1011')\n >>> ExtendedContext.logical_xor(Decimal(110), 1101)\n Decimal('1011')\n >>> ExtendedContext.logical_xor(110, Decimal(1101))\n Decimal('1011')", + "_decimal.Context.max" => "Compare the values numerically and return the maximum.", + "_decimal.Context.max_mag" => "Compare the values numerically with their sign ignored.", + "_decimal.Context.min" => "Compare the values numerically and return the minimum.", + "_decimal.Context.min_mag" => "Compare the values numerically with their sign ignored.", + "_decimal.Context.minus" => "Minus corresponds to the unary prefix minus operator in Python, but applies\nthe context to the result.", + "_decimal.Context.multiply" => "Return the product of x and y.", + "_decimal.Context.next_minus" => "Return the largest representable number smaller than x.", + "_decimal.Context.next_plus" => "Return the smallest representable number larger than x.", + "_decimal.Context.next_toward" => "Return the number closest to x, in the direction towards y.", + "_decimal.Context.normalize" => "Reduce x to its simplest form. Alias for reduce(x).", + "_decimal.Context.number_class" => "Return an indication of the class of x.", + "_decimal.Context.plus" => "Plus corresponds to the unary prefix plus operator in Python, but applies\nthe context to the result.", + "_decimal.Context.power" => "Compute a**b. If 'a' is negative, then 'b' must be integral. The result\nwill be inexact unless 'a' is integral and the result is finite and can\nbe expressed exactly in 'precision' digits. In the Python version the\nresult is always correctly rounded, in the C version the result is almost\nalways correctly rounded.\n\nIf modulo is given, compute (a**b) % modulo. The following restrictions\nhold:\n\n * all three arguments must be integral\n * 'b' must be nonnegative\n * at least one of 'a' or 'b' must be nonzero\n * modulo must be nonzero and less than 10**prec in absolute value", + "_decimal.Context.quantize" => "Return a value equal to x (rounded), having the exponent of y.", + "_decimal.Context.radix" => "Return 10.", + "_decimal.Context.remainder" => "Return the remainder from integer division. The sign of the result,\nif non-zero, is the same as that of the original dividend.", + "_decimal.Context.remainder_near" => "Return x - y * n, where n is the integer nearest the exact value of x / y\n(if the result is 0 then its sign will be the sign of x).", + "_decimal.Context.rotate" => "Return a copy of x, rotated by y places.", + "_decimal.Context.same_quantum" => "Return True if the two operands have the same exponent.", + "_decimal.Context.scaleb" => "Return the first operand after adding the second value to its exp.", + "_decimal.Context.shift" => "Return a copy of x, shifted by y places.", + "_decimal.Context.sqrt" => "Square root of a non-negative number to context precision.", + "_decimal.Context.subtract" => "Return the difference between x and y.", + "_decimal.Context.to_eng_string" => "Convert a number to a string, using engineering notation.", + "_decimal.Context.to_integral" => "Identical to to_integral_value(x).", + "_decimal.Context.to_integral_exact" => "Round to an integer. Signal if the result is rounded or inexact.", + "_decimal.Context.to_integral_value" => "Round to an integer.", + "_decimal.Context.to_sci_string" => "Convert a number to a string using scientific notation.", + "_decimal.ConversionSyntax.__delattr__" => "Implement delattr(self, name).", + "_decimal.ConversionSyntax.__eq__" => "Return self==value.", + "_decimal.ConversionSyntax.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.ConversionSyntax.__ge__" => "Return self>=value.", + "_decimal.ConversionSyntax.__getattribute__" => "Return getattr(self, name).", + "_decimal.ConversionSyntax.__getstate__" => "Helper for pickle.", + "_decimal.ConversionSyntax.__gt__" => "Return self>value.", + "_decimal.ConversionSyntax.__hash__" => "Return hash(self).", + "_decimal.ConversionSyntax.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.ConversionSyntax.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.ConversionSyntax.__le__" => "Return self<=value.", + "_decimal.ConversionSyntax.__lt__" => "Return self<value.", + "_decimal.ConversionSyntax.__ne__" => "Return self!=value.", + "_decimal.ConversionSyntax.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.ConversionSyntax.__reduce_ex__" => "Helper for pickle.", + "_decimal.ConversionSyntax.__repr__" => "Return repr(self).", + "_decimal.ConversionSyntax.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.ConversionSyntax.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.ConversionSyntax.__str__" => "Return str(self).", + "_decimal.ConversionSyntax.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.ConversionSyntax.__weakref__" => "list of weak references to the object", + "_decimal.ConversionSyntax.add_note" => "Add a note to the exception", + "_decimal.ConversionSyntax.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Decimal" => "Construct a new Decimal object. 'value' can be an integer, string, tuple,\nor another Decimal object. If no value is given, return Decimal('0'). The\ncontext does not affect the conversion and is only passed to determine if\nthe InvalidOperation trap is active.", + "_decimal.Decimal.__abs__" => "abs(self)", + "_decimal.Decimal.__add__" => "Return self+value.", + "_decimal.Decimal.__bool__" => "True if self else False", + "_decimal.Decimal.__delattr__" => "Implement delattr(self, name).", + "_decimal.Decimal.__divmod__" => "Return divmod(self, value).", + "_decimal.Decimal.__eq__" => "Return self==value.", + "_decimal.Decimal.__float__" => "float(self)", + "_decimal.Decimal.__floordiv__" => "Return self//value.", + "_decimal.Decimal.__ge__" => "Return self>=value.", + "_decimal.Decimal.__getattribute__" => "Return getattr(self, name).", + "_decimal.Decimal.__getstate__" => "Helper for pickle.", + "_decimal.Decimal.__gt__" => "Return self>value.", + "_decimal.Decimal.__hash__" => "Return hash(self).", + "_decimal.Decimal.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Decimal.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Decimal.__int__" => "int(self)", + "_decimal.Decimal.__le__" => "Return self<=value.", + "_decimal.Decimal.__lt__" => "Return self<value.", + "_decimal.Decimal.__mod__" => "Return self%value.", + "_decimal.Decimal.__mul__" => "Return self*value.", + "_decimal.Decimal.__ne__" => "Return self!=value.", + "_decimal.Decimal.__neg__" => "-self", + "_decimal.Decimal.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Decimal.__pos__" => "+self", + "_decimal.Decimal.__pow__" => "Return pow(self, value, mod).", + "_decimal.Decimal.__radd__" => "Return value+self.", + "_decimal.Decimal.__rdivmod__" => "Return divmod(value, self).", + "_decimal.Decimal.__reduce_ex__" => "Helper for pickle.", + "_decimal.Decimal.__repr__" => "Return repr(self).", + "_decimal.Decimal.__rfloordiv__" => "Return value//self.", + "_decimal.Decimal.__rmod__" => "Return value%self.", + "_decimal.Decimal.__rmul__" => "Return value*self.", + "_decimal.Decimal.__rpow__" => "Return pow(value, self, mod).", + "_decimal.Decimal.__rsub__" => "Return value-self.", + "_decimal.Decimal.__rtruediv__" => "Return value/self.", + "_decimal.Decimal.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Decimal.__str__" => "Return str(self).", + "_decimal.Decimal.__sub__" => "Return self-value.", + "_decimal.Decimal.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Decimal.__truediv__" => "Return self/value.", + "_decimal.Decimal.adjusted" => "Return the adjusted exponent of the number. Defined as exp + digits - 1.", + "_decimal.Decimal.as_integer_ratio" => "Decimal.as_integer_ratio() -> (int, int)\n\nReturn a pair of integers, whose ratio is exactly equal to the original\nDecimal and with a positive denominator. The ratio is in lowest terms.\nRaise OverflowError on infinities and a ValueError on NaNs.", + "_decimal.Decimal.as_tuple" => "Return a tuple representation of the number.", + "_decimal.Decimal.canonical" => "Return the canonical encoding of the argument. Currently, the encoding\nof a Decimal instance is always canonical, so this operation returns its\nargument unchanged.", + "_decimal.Decimal.compare" => "Compare self to other. Return a decimal value:\n\n a or b is a NaN ==> Decimal('NaN')\n a < b ==> Decimal('-1')\n a == b ==> Decimal('0')\n a > b ==> Decimal('1')", + "_decimal.Decimal.compare_signal" => "Identical to compare, except that all NaNs signal.", + "_decimal.Decimal.compare_total" => "Compare two operands using their abstract representation rather than\ntheir numerical value. Similar to the compare() method, but the result\ngives a total ordering on Decimal instances. Two Decimal instances with\nthe same numeric value but different representations compare unequal\nin this ordering:\n\n >>> Decimal('12.0').compare_total(Decimal('12'))\n Decimal('-1')\n\nQuiet and signaling NaNs are also included in the total ordering. The result\nof this function is Decimal('0') if both operands have the same representation,\nDecimal('-1') if the first operand is lower in the total order than the second,\nand Decimal('1') if the first operand is higher in the total order than the\nsecond operand. See the specification for details of the total order.\n\nThis operation is unaffected by context and is quiet: no flags are changed\nand no rounding is performed. As an exception, the C version may raise\nInvalidOperation if the second operand cannot be converted exactly.", + "_decimal.Decimal.compare_total_mag" => "Compare two operands using their abstract representation rather than their\nvalue as in compare_total(), but ignoring the sign of each operand.\n\nx.compare_total_mag(y) is equivalent to x.copy_abs().compare_total(y.copy_abs()).\n\nThis operation is unaffected by context and is quiet: no flags are changed\nand no rounding is performed. As an exception, the C version may raise\nInvalidOperation if the second operand cannot be converted exactly.", + "_decimal.Decimal.conjugate" => "Return self.", + "_decimal.Decimal.copy_abs" => "Return the absolute value of the argument. This operation is unaffected by\ncontext and is quiet: no flags are changed and no rounding is performed.", + "_decimal.Decimal.copy_negate" => "Return the negation of the argument. This operation is unaffected by context\nand is quiet: no flags are changed and no rounding is performed.", + "_decimal.Decimal.copy_sign" => "Return a copy of the first operand with the sign set to be the same as the\nsign of the second operand. For example:\n\n >>> Decimal('2.3').copy_sign(Decimal('-1.5'))\n Decimal('-2.3')\n\nThis operation is unaffected by context and is quiet: no flags are changed\nand no rounding is performed. As an exception, the C version may raise\nInvalidOperation if the second operand cannot be converted exactly.", + "_decimal.Decimal.exp" => "Return the value of the (natural) exponential function e**x at the given\nnumber. The function always uses the ROUND_HALF_EVEN mode and the result\nis correctly rounded.", + "_decimal.Decimal.fma" => "Fused multiply-add. Return self*other+third with no rounding of the\nintermediate product self*other.\n\n >>> Decimal(2).fma(3, 5)\n Decimal('11')", + "_decimal.Decimal.from_float" => "Class method that converts a float to a decimal number, exactly.\nSince 0.1 is not exactly representable in binary floating point,\nDecimal.from_float(0.1) is not the same as Decimal('0.1').\n\n >>> Decimal.from_float(0.1)\n Decimal('0.1000000000000000055511151231257827021181583404541015625')\n >>> Decimal.from_float(float('nan'))\n Decimal('NaN')\n >>> Decimal.from_float(float('inf'))\n Decimal('Infinity')\n >>> Decimal.from_float(float('-inf'))\n Decimal('-Infinity')", + "_decimal.Decimal.from_number" => "Class method that converts a real number to a decimal number, exactly.\n\n >>> Decimal.from_number(314) # int\n Decimal('314')\n >>> Decimal.from_number(0.1) # float\n Decimal('0.1000000000000000055511151231257827021181583404541015625')\n >>> Decimal.from_number(Decimal('3.14')) # another decimal instance\n Decimal('3.14')", + "_decimal.Decimal.is_canonical" => "Return True if the argument is canonical and False otherwise. Currently,\na Decimal instance is always canonical, so this operation always returns\nTrue.", + "_decimal.Decimal.is_finite" => "Return True if the argument is a finite number, and False if the argument\nis infinite or a NaN.", + "_decimal.Decimal.is_infinite" => "Return True if the argument is either positive or negative infinity and\nFalse otherwise.", + "_decimal.Decimal.is_nan" => "Return True if the argument is a (quiet or signaling) NaN and False\notherwise.", + "_decimal.Decimal.is_normal" => "Return True if the argument is a normal finite non-zero number with an\nadjusted exponent greater than or equal to Emin. Return False if the\nargument is zero, subnormal, infinite or a NaN.", + "_decimal.Decimal.is_qnan" => "Return True if the argument is a quiet NaN, and False otherwise.", + "_decimal.Decimal.is_signed" => "Return True if the argument has a negative sign and False otherwise.\nNote that both zeros and NaNs can carry signs.", + "_decimal.Decimal.is_snan" => "Return True if the argument is a signaling NaN and False otherwise.", + "_decimal.Decimal.is_subnormal" => "Return True if the argument is subnormal, and False otherwise. A number is\nsubnormal if it is non-zero, finite, and has an adjusted exponent less\nthan Emin.", + "_decimal.Decimal.is_zero" => "Return True if the argument is a (positive or negative) zero and False\notherwise.", + "_decimal.Decimal.ln" => "Return the natural (base e) logarithm of the operand. The function always\nuses the ROUND_HALF_EVEN mode and the result is correctly rounded.", + "_decimal.Decimal.log10" => "Return the base ten logarithm of the operand. The function always uses the\nROUND_HALF_EVEN mode and the result is correctly rounded.", + "_decimal.Decimal.logb" => "For a non-zero number, return the adjusted exponent of the operand as a\nDecimal instance. If the operand is a zero, then Decimal('-Infinity') is\nreturned and the DivisionByZero condition is raised. If the operand is\nan infinity then Decimal('Infinity') is returned.", + "_decimal.Decimal.logical_and" => "Applies an 'and' operation between self and other's digits.\n\nBoth self and other must be logical numbers.", + "_decimal.Decimal.logical_invert" => "Invert all its digits.\n\nThe self must be logical number.", + "_decimal.Decimal.logical_or" => "Applies an 'or' operation between self and other's digits.\n\nBoth self and other must be logical numbers.", + "_decimal.Decimal.logical_xor" => "Applies an 'xor' operation between self and other's digits.\n\nBoth self and other must be logical numbers.", + "_decimal.Decimal.max" => "Maximum of self and other. If one operand is a quiet NaN and the other is\nnumeric, the numeric operand is returned.", + "_decimal.Decimal.max_mag" => "Similar to the max() method, but the comparison is done using the absolute\nvalues of the operands.", + "_decimal.Decimal.min" => "Minimum of self and other. If one operand is a quiet NaN and the other is\nnumeric, the numeric operand is returned.", + "_decimal.Decimal.min_mag" => "Similar to the min() method, but the comparison is done using the absolute\nvalues of the operands.", + "_decimal.Decimal.next_minus" => "Return the largest number representable in the given context (or in the\ncurrent default context if no context is given) that is smaller than the\ngiven operand.", + "_decimal.Decimal.next_plus" => "Return the smallest number representable in the given context (or in the\ncurrent default context if no context is given) that is larger than the\ngiven operand.", + "_decimal.Decimal.next_toward" => "If the two operands are unequal, return the number closest to the first\noperand in the direction of the second operand. If both operands are\nnumerically equal, return a copy of the first operand with the sign set\nto be the same as the sign of the second operand.", + "_decimal.Decimal.normalize" => "Normalize the number by stripping the rightmost trailing zeros and\nconverting any result equal to Decimal('0') to Decimal('0e0'). Used\nfor producing canonical values for members of an equivalence class.\nFor example, Decimal('32.100') and Decimal('0.321000e+2') both normalize\nto the equivalent value Decimal('32.1').", + "_decimal.Decimal.number_class" => "Return a string describing the class of the operand. The returned value\nis one of the following ten strings:\n\n * '-Infinity', indicating that the operand is negative infinity.\n * '-Normal', indicating that the operand is a negative normal number.\n * '-Subnormal', indicating that the operand is negative and subnormal.\n * '-Zero', indicating that the operand is a negative zero.\n * '+Zero', indicating that the operand is a positive zero.\n * '+Subnormal', indicating that the operand is positive and subnormal.\n * '+Normal', indicating that the operand is a positive normal number.\n * '+Infinity', indicating that the operand is positive infinity.\n * 'NaN', indicating that the operand is a quiet NaN (Not a Number).\n * 'sNaN', indicating that the operand is a signaling NaN.", + "_decimal.Decimal.quantize" => "Return a value equal to the first operand after rounding and having the\nexponent of the second operand.\n\n >>> Decimal('1.41421356').quantize(Decimal('1.000'))\n Decimal('1.414')\n\nUnlike other operations, if the length of the coefficient after the quantize\noperation would be greater than precision, then an InvalidOperation is signaled.\nThis guarantees that, unless there is an error condition, the quantized exponent\nis always equal to that of the right-hand operand.\n\nAlso unlike other operations, quantize never signals Underflow, even if the\nresult is subnormal and inexact.\n\nIf the exponent of the second operand is larger than that of the first, then\nrounding may be necessary. In this case, the rounding mode is determined by the\nrounding argument if given, else by the given context argument; if neither\nargument is given, the rounding mode of the current thread's context is used.", + "_decimal.Decimal.radix" => "Return Decimal(10), the radix (base) in which the Decimal class does\nall its arithmetic. Included for compatibility with the specification.", + "_decimal.Decimal.remainder_near" => "Return the remainder from dividing self by other. This differs from\nself % other in that the sign of the remainder is chosen so as to minimize\nits absolute value. More precisely, the return value is self - n * other\nwhere n is the integer nearest to the exact value of self / other, and\nif two integers are equally near then the even one is chosen.\n\nIf the result is zero then its sign will be the sign of self.", + "_decimal.Decimal.rotate" => "Return the result of rotating the digits of the first operand by an amount\nspecified by the second operand. The second operand must be an integer in\nthe range -precision through precision. The absolute value of the second\noperand gives the number of places to rotate. If the second operand is\npositive then rotation is to the left; otherwise rotation is to the right.\nThe coefficient of the first operand is padded on the left with zeros to\nlength precision if necessary. The sign and exponent of the first operand are\nunchanged.", + "_decimal.Decimal.same_quantum" => "Test whether self and other have the same exponent or whether both are NaN.\n\nThis operation is unaffected by context and is quiet: no flags are changed\nand no rounding is performed. As an exception, the C version may raise\nInvalidOperation if the second operand cannot be converted exactly.", + "_decimal.Decimal.scaleb" => "Return the first operand with the exponent adjusted the second. Equivalently,\nreturn the first operand multiplied by 10**other. The second operand must be\nan integer.", + "_decimal.Decimal.shift" => "Return the result of shifting the digits of the first operand by an amount\nspecified by the second operand. The second operand must be an integer in\nthe range -precision through precision. The absolute value of the second\noperand gives the number of places to shift. If the second operand is\npositive, then the shift is to the left; otherwise the shift is to the\nright. Digits shifted into the coefficient are zeros. The sign and exponent\nof the first operand are unchanged.", + "_decimal.Decimal.sqrt" => "Return the square root of the argument to full precision. The result is\ncorrectly rounded using the ROUND_HALF_EVEN rounding mode.", + "_decimal.Decimal.to_eng_string" => "Convert to an engineering-type string. Engineering notation has an exponent\nwhich is a multiple of 3, so there are up to 3 digits left of the decimal\nplace. For example, Decimal('123E+1') is converted to Decimal('1.23E+3').\n\nThe value of context.capitals determines whether the exponent sign is lower\nor upper case. Otherwise, the context does not affect the operation.", + "_decimal.Decimal.to_integral" => "Identical to the to_integral_value() method. The to_integral() name has been\nkept for compatibility with older versions.", + "_decimal.Decimal.to_integral_exact" => "Round to the nearest integer, signaling Inexact or Rounded as appropriate if\nrounding occurs. The rounding mode is determined by the rounding parameter\nif given, else by the given context. If neither parameter is given, then the\nrounding mode of the current default context is used.", + "_decimal.Decimal.to_integral_value" => "Round to the nearest integer without signaling Inexact or Rounded. The\nrounding mode is determined by the rounding parameter if given, else by\nthe given context. If neither parameter is given, then the rounding mode\nof the current default context is used.", + "_decimal.DecimalException.__delattr__" => "Implement delattr(self, name).", + "_decimal.DecimalException.__eq__" => "Return self==value.", + "_decimal.DecimalException.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DecimalException.__ge__" => "Return self>=value.", + "_decimal.DecimalException.__getattribute__" => "Return getattr(self, name).", + "_decimal.DecimalException.__getstate__" => "Helper for pickle.", + "_decimal.DecimalException.__gt__" => "Return self>value.", + "_decimal.DecimalException.__hash__" => "Return hash(self).", + "_decimal.DecimalException.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DecimalException.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DecimalException.__le__" => "Return self<=value.", + "_decimal.DecimalException.__lt__" => "Return self<value.", + "_decimal.DecimalException.__ne__" => "Return self!=value.", + "_decimal.DecimalException.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.DecimalException.__reduce_ex__" => "Helper for pickle.", + "_decimal.DecimalException.__repr__" => "Return repr(self).", + "_decimal.DecimalException.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DecimalException.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DecimalException.__str__" => "Return str(self).", + "_decimal.DecimalException.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DecimalException.__weakref__" => "list of weak references to the object", + "_decimal.DecimalException.add_note" => "Add a note to the exception", + "_decimal.DecimalException.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.DecimalTuple" => "DecimalTuple(sign, digits, exponent)", + "_decimal.DecimalTuple.__add__" => "Return self+value.", + "_decimal.DecimalTuple.__class_getitem__" => "See PEP 585", + "_decimal.DecimalTuple.__contains__" => "Return bool(key in self).", + "_decimal.DecimalTuple.__delattr__" => "Implement delattr(self, name).", + "_decimal.DecimalTuple.__eq__" => "Return self==value.", + "_decimal.DecimalTuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DecimalTuple.__ge__" => "Return self>=value.", + "_decimal.DecimalTuple.__getattribute__" => "Return getattr(self, name).", + "_decimal.DecimalTuple.__getitem__" => "Return self[key].", + "_decimal.DecimalTuple.__getnewargs__" => "Return self as a plain tuple. Used by copy and pickle.", + "_decimal.DecimalTuple.__getstate__" => "Helper for pickle.", + "_decimal.DecimalTuple.__gt__" => "Return self>value.", + "_decimal.DecimalTuple.__hash__" => "Return hash(self).", + "_decimal.DecimalTuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DecimalTuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DecimalTuple.__iter__" => "Implement iter(self).", + "_decimal.DecimalTuple.__le__" => "Return self<=value.", + "_decimal.DecimalTuple.__len__" => "Return len(self).", + "_decimal.DecimalTuple.__lt__" => "Return self<value.", + "_decimal.DecimalTuple.__mul__" => "Return self*value.", + "_decimal.DecimalTuple.__ne__" => "Return self!=value.", + "_decimal.DecimalTuple.__new__" => "Create new instance of DecimalTuple(sign, digits, exponent)", + "_decimal.DecimalTuple.__reduce__" => "Helper for pickle.", + "_decimal.DecimalTuple.__reduce_ex__" => "Helper for pickle.", + "_decimal.DecimalTuple.__replace__" => "Return a new DecimalTuple object replacing specified fields with new values", + "_decimal.DecimalTuple.__repr__" => "Return a nicely formatted representation string", + "_decimal.DecimalTuple.__rmul__" => "Return value*self.", + "_decimal.DecimalTuple.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DecimalTuple.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DecimalTuple.__str__" => "Return str(self).", + "_decimal.DecimalTuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DecimalTuple._asdict" => "Return a new dict which maps field names to their values.", + "_decimal.DecimalTuple._make" => "Make a new DecimalTuple object from a sequence or iterable", + "_decimal.DecimalTuple._replace" => "Return a new DecimalTuple object replacing specified fields with new values", + "_decimal.DecimalTuple.count" => "Return number of occurrences of value.", + "_decimal.DecimalTuple.digits" => "Alias for field number 1", + "_decimal.DecimalTuple.exponent" => "Alias for field number 2", + "_decimal.DecimalTuple.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_decimal.DecimalTuple.sign" => "Alias for field number 0", + "_decimal.DivisionByZero.__delattr__" => "Implement delattr(self, name).", + "_decimal.DivisionByZero.__eq__" => "Return self==value.", + "_decimal.DivisionByZero.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DivisionByZero.__ge__" => "Return self>=value.", + "_decimal.DivisionByZero.__getattribute__" => "Return getattr(self, name).", + "_decimal.DivisionByZero.__getstate__" => "Helper for pickle.", + "_decimal.DivisionByZero.__gt__" => "Return self>value.", + "_decimal.DivisionByZero.__hash__" => "Return hash(self).", + "_decimal.DivisionByZero.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DivisionByZero.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DivisionByZero.__le__" => "Return self<=value.", + "_decimal.DivisionByZero.__lt__" => "Return self<value.", + "_decimal.DivisionByZero.__ne__" => "Return self!=value.", + "_decimal.DivisionByZero.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.DivisionByZero.__reduce_ex__" => "Helper for pickle.", + "_decimal.DivisionByZero.__repr__" => "Return repr(self).", + "_decimal.DivisionByZero.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DivisionByZero.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DivisionByZero.__str__" => "Return str(self).", + "_decimal.DivisionByZero.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DivisionByZero.__weakref__" => "list of weak references to the object", + "_decimal.DivisionByZero.add_note" => "Add a note to the exception", + "_decimal.DivisionByZero.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.DivisionImpossible.__delattr__" => "Implement delattr(self, name).", + "_decimal.DivisionImpossible.__eq__" => "Return self==value.", + "_decimal.DivisionImpossible.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DivisionImpossible.__ge__" => "Return self>=value.", + "_decimal.DivisionImpossible.__getattribute__" => "Return getattr(self, name).", + "_decimal.DivisionImpossible.__getstate__" => "Helper for pickle.", + "_decimal.DivisionImpossible.__gt__" => "Return self>value.", + "_decimal.DivisionImpossible.__hash__" => "Return hash(self).", + "_decimal.DivisionImpossible.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DivisionImpossible.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DivisionImpossible.__le__" => "Return self<=value.", + "_decimal.DivisionImpossible.__lt__" => "Return self<value.", + "_decimal.DivisionImpossible.__ne__" => "Return self!=value.", + "_decimal.DivisionImpossible.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.DivisionImpossible.__reduce_ex__" => "Helper for pickle.", + "_decimal.DivisionImpossible.__repr__" => "Return repr(self).", + "_decimal.DivisionImpossible.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DivisionImpossible.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DivisionImpossible.__str__" => "Return str(self).", + "_decimal.DivisionImpossible.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DivisionImpossible.__weakref__" => "list of weak references to the object", + "_decimal.DivisionImpossible.add_note" => "Add a note to the exception", + "_decimal.DivisionImpossible.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.DivisionUndefined.__delattr__" => "Implement delattr(self, name).", + "_decimal.DivisionUndefined.__eq__" => "Return self==value.", + "_decimal.DivisionUndefined.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.DivisionUndefined.__ge__" => "Return self>=value.", + "_decimal.DivisionUndefined.__getattribute__" => "Return getattr(self, name).", + "_decimal.DivisionUndefined.__getstate__" => "Helper for pickle.", + "_decimal.DivisionUndefined.__gt__" => "Return self>value.", + "_decimal.DivisionUndefined.__hash__" => "Return hash(self).", + "_decimal.DivisionUndefined.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.DivisionUndefined.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.DivisionUndefined.__le__" => "Return self<=value.", + "_decimal.DivisionUndefined.__lt__" => "Return self<value.", + "_decimal.DivisionUndefined.__ne__" => "Return self!=value.", + "_decimal.DivisionUndefined.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.DivisionUndefined.__reduce_ex__" => "Helper for pickle.", + "_decimal.DivisionUndefined.__repr__" => "Return repr(self).", + "_decimal.DivisionUndefined.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.DivisionUndefined.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.DivisionUndefined.__str__" => "Return str(self).", + "_decimal.DivisionUndefined.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.DivisionUndefined.__weakref__" => "list of weak references to the object", + "_decimal.DivisionUndefined.add_note" => "Add a note to the exception", + "_decimal.DivisionUndefined.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.FloatOperation.__delattr__" => "Implement delattr(self, name).", + "_decimal.FloatOperation.__eq__" => "Return self==value.", + "_decimal.FloatOperation.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.FloatOperation.__ge__" => "Return self>=value.", + "_decimal.FloatOperation.__getattribute__" => "Return getattr(self, name).", + "_decimal.FloatOperation.__getstate__" => "Helper for pickle.", + "_decimal.FloatOperation.__gt__" => "Return self>value.", + "_decimal.FloatOperation.__hash__" => "Return hash(self).", + "_decimal.FloatOperation.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.FloatOperation.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.FloatOperation.__le__" => "Return self<=value.", + "_decimal.FloatOperation.__lt__" => "Return self<value.", + "_decimal.FloatOperation.__ne__" => "Return self!=value.", + "_decimal.FloatOperation.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.FloatOperation.__reduce_ex__" => "Helper for pickle.", + "_decimal.FloatOperation.__repr__" => "Return repr(self).", + "_decimal.FloatOperation.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.FloatOperation.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.FloatOperation.__str__" => "Return str(self).", + "_decimal.FloatOperation.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.FloatOperation.__weakref__" => "list of weak references to the object", + "_decimal.FloatOperation.add_note" => "Add a note to the exception", + "_decimal.FloatOperation.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.IEEEContext" => "Return a context object initialized to the proper values for one of the\nIEEE interchange formats. The argument must be a multiple of 32 and less\nthan IEEE_CONTEXT_MAX_BITS.", + "_decimal.Inexact.__delattr__" => "Implement delattr(self, name).", + "_decimal.Inexact.__eq__" => "Return self==value.", + "_decimal.Inexact.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Inexact.__ge__" => "Return self>=value.", + "_decimal.Inexact.__getattribute__" => "Return getattr(self, name).", + "_decimal.Inexact.__getstate__" => "Helper for pickle.", + "_decimal.Inexact.__gt__" => "Return self>value.", + "_decimal.Inexact.__hash__" => "Return hash(self).", + "_decimal.Inexact.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Inexact.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Inexact.__le__" => "Return self<=value.", + "_decimal.Inexact.__lt__" => "Return self<value.", + "_decimal.Inexact.__ne__" => "Return self!=value.", + "_decimal.Inexact.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Inexact.__reduce_ex__" => "Helper for pickle.", + "_decimal.Inexact.__repr__" => "Return repr(self).", + "_decimal.Inexact.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Inexact.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Inexact.__str__" => "Return str(self).", + "_decimal.Inexact.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Inexact.__weakref__" => "list of weak references to the object", + "_decimal.Inexact.add_note" => "Add a note to the exception", + "_decimal.Inexact.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.InvalidContext.__delattr__" => "Implement delattr(self, name).", + "_decimal.InvalidContext.__eq__" => "Return self==value.", + "_decimal.InvalidContext.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.InvalidContext.__ge__" => "Return self>=value.", + "_decimal.InvalidContext.__getattribute__" => "Return getattr(self, name).", + "_decimal.InvalidContext.__getstate__" => "Helper for pickle.", + "_decimal.InvalidContext.__gt__" => "Return self>value.", + "_decimal.InvalidContext.__hash__" => "Return hash(self).", + "_decimal.InvalidContext.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.InvalidContext.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.InvalidContext.__le__" => "Return self<=value.", + "_decimal.InvalidContext.__lt__" => "Return self<value.", + "_decimal.InvalidContext.__ne__" => "Return self!=value.", + "_decimal.InvalidContext.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.InvalidContext.__reduce_ex__" => "Helper for pickle.", + "_decimal.InvalidContext.__repr__" => "Return repr(self).", + "_decimal.InvalidContext.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.InvalidContext.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.InvalidContext.__str__" => "Return str(self).", + "_decimal.InvalidContext.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.InvalidContext.__weakref__" => "list of weak references to the object", + "_decimal.InvalidContext.add_note" => "Add a note to the exception", + "_decimal.InvalidContext.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.InvalidOperation.__delattr__" => "Implement delattr(self, name).", + "_decimal.InvalidOperation.__eq__" => "Return self==value.", + "_decimal.InvalidOperation.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.InvalidOperation.__ge__" => "Return self>=value.", + "_decimal.InvalidOperation.__getattribute__" => "Return getattr(self, name).", + "_decimal.InvalidOperation.__getstate__" => "Helper for pickle.", + "_decimal.InvalidOperation.__gt__" => "Return self>value.", + "_decimal.InvalidOperation.__hash__" => "Return hash(self).", + "_decimal.InvalidOperation.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.InvalidOperation.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.InvalidOperation.__le__" => "Return self<=value.", + "_decimal.InvalidOperation.__lt__" => "Return self<value.", + "_decimal.InvalidOperation.__ne__" => "Return self!=value.", + "_decimal.InvalidOperation.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.InvalidOperation.__reduce_ex__" => "Helper for pickle.", + "_decimal.InvalidOperation.__repr__" => "Return repr(self).", + "_decimal.InvalidOperation.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.InvalidOperation.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.InvalidOperation.__str__" => "Return str(self).", + "_decimal.InvalidOperation.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.InvalidOperation.__weakref__" => "list of weak references to the object", + "_decimal.InvalidOperation.add_note" => "Add a note to the exception", + "_decimal.InvalidOperation.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Overflow.__delattr__" => "Implement delattr(self, name).", + "_decimal.Overflow.__eq__" => "Return self==value.", + "_decimal.Overflow.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Overflow.__ge__" => "Return self>=value.", + "_decimal.Overflow.__getattribute__" => "Return getattr(self, name).", + "_decimal.Overflow.__getstate__" => "Helper for pickle.", + "_decimal.Overflow.__gt__" => "Return self>value.", + "_decimal.Overflow.__hash__" => "Return hash(self).", + "_decimal.Overflow.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Overflow.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Overflow.__le__" => "Return self<=value.", + "_decimal.Overflow.__lt__" => "Return self<value.", + "_decimal.Overflow.__ne__" => "Return self!=value.", + "_decimal.Overflow.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Overflow.__reduce_ex__" => "Helper for pickle.", + "_decimal.Overflow.__repr__" => "Return repr(self).", + "_decimal.Overflow.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Overflow.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Overflow.__str__" => "Return str(self).", + "_decimal.Overflow.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Overflow.__weakref__" => "list of weak references to the object", + "_decimal.Overflow.add_note" => "Add a note to the exception", + "_decimal.Overflow.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Rounded.__delattr__" => "Implement delattr(self, name).", + "_decimal.Rounded.__eq__" => "Return self==value.", + "_decimal.Rounded.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Rounded.__ge__" => "Return self>=value.", + "_decimal.Rounded.__getattribute__" => "Return getattr(self, name).", + "_decimal.Rounded.__getstate__" => "Helper for pickle.", + "_decimal.Rounded.__gt__" => "Return self>value.", + "_decimal.Rounded.__hash__" => "Return hash(self).", + "_decimal.Rounded.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Rounded.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Rounded.__le__" => "Return self<=value.", + "_decimal.Rounded.__lt__" => "Return self<value.", + "_decimal.Rounded.__ne__" => "Return self!=value.", + "_decimal.Rounded.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Rounded.__reduce_ex__" => "Helper for pickle.", + "_decimal.Rounded.__repr__" => "Return repr(self).", + "_decimal.Rounded.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Rounded.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Rounded.__str__" => "Return str(self).", + "_decimal.Rounded.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Rounded.__weakref__" => "list of weak references to the object", + "_decimal.Rounded.add_note" => "Add a note to the exception", + "_decimal.Rounded.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Subnormal.__delattr__" => "Implement delattr(self, name).", + "_decimal.Subnormal.__eq__" => "Return self==value.", + "_decimal.Subnormal.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Subnormal.__ge__" => "Return self>=value.", + "_decimal.Subnormal.__getattribute__" => "Return getattr(self, name).", + "_decimal.Subnormal.__getstate__" => "Helper for pickle.", + "_decimal.Subnormal.__gt__" => "Return self>value.", + "_decimal.Subnormal.__hash__" => "Return hash(self).", + "_decimal.Subnormal.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Subnormal.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Subnormal.__le__" => "Return self<=value.", + "_decimal.Subnormal.__lt__" => "Return self<value.", + "_decimal.Subnormal.__ne__" => "Return self!=value.", + "_decimal.Subnormal.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Subnormal.__reduce_ex__" => "Helper for pickle.", + "_decimal.Subnormal.__repr__" => "Return repr(self).", + "_decimal.Subnormal.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Subnormal.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Subnormal.__str__" => "Return str(self).", + "_decimal.Subnormal.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Subnormal.__weakref__" => "list of weak references to the object", + "_decimal.Subnormal.add_note" => "Add a note to the exception", + "_decimal.Subnormal.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.Underflow.__delattr__" => "Implement delattr(self, name).", + "_decimal.Underflow.__eq__" => "Return self==value.", + "_decimal.Underflow.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_decimal.Underflow.__ge__" => "Return self>=value.", + "_decimal.Underflow.__getattribute__" => "Return getattr(self, name).", + "_decimal.Underflow.__getstate__" => "Helper for pickle.", + "_decimal.Underflow.__gt__" => "Return self>value.", + "_decimal.Underflow.__hash__" => "Return hash(self).", + "_decimal.Underflow.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_decimal.Underflow.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_decimal.Underflow.__le__" => "Return self<=value.", + "_decimal.Underflow.__lt__" => "Return self<value.", + "_decimal.Underflow.__ne__" => "Return self!=value.", + "_decimal.Underflow.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_decimal.Underflow.__reduce_ex__" => "Helper for pickle.", + "_decimal.Underflow.__repr__" => "Return repr(self).", + "_decimal.Underflow.__setattr__" => "Implement setattr(self, name, value).", + "_decimal.Underflow.__sizeof__" => "Size of object in memory, in bytes.", + "_decimal.Underflow.__str__" => "Return str(self).", + "_decimal.Underflow.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_decimal.Underflow.__weakref__" => "list of weak references to the object", + "_decimal.Underflow.add_note" => "Add a note to the exception", + "_decimal.Underflow.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_decimal.getcontext" => "Get the current default context.", + "_decimal.localcontext" => "Return a context manager that will set the default context to a copy of ctx\non entry to the with-statement and restore the previous default context when\nexiting the with-statement. If no context is specified, a copy of the current\ndefault context is used.", + "_decimal.setcontext" => "Set a new default context.", + "_elementtree.Element.__bool__" => "True if self else False", + "_elementtree.Element.__delattr__" => "Implement delattr(self, name).", + "_elementtree.Element.__delitem__" => "Delete self[key].", + "_elementtree.Element.__eq__" => "Return self==value.", + "_elementtree.Element.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_elementtree.Element.__ge__" => "Return self>=value.", + "_elementtree.Element.__getattribute__" => "Return getattr(self, name).", + "_elementtree.Element.__getitem__" => "Return self[key].", + "_elementtree.Element.__gt__" => "Return self>value.", + "_elementtree.Element.__hash__" => "Return hash(self).", + "_elementtree.Element.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_elementtree.Element.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_elementtree.Element.__le__" => "Return self<=value.", + "_elementtree.Element.__len__" => "Return len(self).", + "_elementtree.Element.__lt__" => "Return self<value.", + "_elementtree.Element.__ne__" => "Return self!=value.", + "_elementtree.Element.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_elementtree.Element.__reduce__" => "Helper for pickle.", + "_elementtree.Element.__reduce_ex__" => "Helper for pickle.", + "_elementtree.Element.__repr__" => "Return repr(self).", + "_elementtree.Element.__setattr__" => "Implement setattr(self, name, value).", + "_elementtree.Element.__setitem__" => "Set self[key] to value.", + "_elementtree.Element.__str__" => "Return str(self).", + "_elementtree.Element.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_elementtree.Element.attrib" => "A dictionary containing the element's attributes", + "_elementtree.Element.tag" => "A string identifying what kind of data this element represents", + "_elementtree.Element.tail" => "A string of text directly after the end tag, or None", + "_elementtree.Element.text" => "A string of text directly after the start tag, or None", + "_elementtree.ParseError.__delattr__" => "Implement delattr(self, name).", + "_elementtree.ParseError.__eq__" => "Return self==value.", + "_elementtree.ParseError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_elementtree.ParseError.__ge__" => "Return self>=value.", + "_elementtree.ParseError.__getattribute__" => "Return getattr(self, name).", + "_elementtree.ParseError.__getstate__" => "Helper for pickle.", + "_elementtree.ParseError.__gt__" => "Return self>value.", + "_elementtree.ParseError.__hash__" => "Return hash(self).", + "_elementtree.ParseError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_elementtree.ParseError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_elementtree.ParseError.__le__" => "Return self<=value.", + "_elementtree.ParseError.__lt__" => "Return self<value.", + "_elementtree.ParseError.__ne__" => "Return self!=value.", + "_elementtree.ParseError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_elementtree.ParseError.__reduce_ex__" => "Helper for pickle.", + "_elementtree.ParseError.__repr__" => "Return repr(self).", + "_elementtree.ParseError.__setattr__" => "Implement setattr(self, name, value).", + "_elementtree.ParseError.__sizeof__" => "Size of object in memory, in bytes.", + "_elementtree.ParseError.__str__" => "Return str(self).", + "_elementtree.ParseError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_elementtree.ParseError.__weakref__" => "list of weak references to the object", + "_elementtree.ParseError._metadata" => "exception private metadata", + "_elementtree.ParseError.add_note" => "Add a note to the exception", + "_elementtree.ParseError.end_lineno" => "exception end lineno", + "_elementtree.ParseError.end_offset" => "exception end offset", + "_elementtree.ParseError.filename" => "exception filename", + "_elementtree.ParseError.lineno" => "exception lineno", + "_elementtree.ParseError.msg" => "exception msg", + "_elementtree.ParseError.offset" => "exception offset", + "_elementtree.ParseError.print_file_and_line" => "exception print_file_and_line", + "_elementtree.ParseError.text" => "exception text", + "_elementtree.ParseError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_elementtree.TreeBuilder.__delattr__" => "Implement delattr(self, name).", + "_elementtree.TreeBuilder.__eq__" => "Return self==value.", + "_elementtree.TreeBuilder.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_elementtree.TreeBuilder.__ge__" => "Return self>=value.", + "_elementtree.TreeBuilder.__getattribute__" => "Return getattr(self, name).", + "_elementtree.TreeBuilder.__getstate__" => "Helper for pickle.", + "_elementtree.TreeBuilder.__gt__" => "Return self>value.", + "_elementtree.TreeBuilder.__hash__" => "Return hash(self).", + "_elementtree.TreeBuilder.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_elementtree.TreeBuilder.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_elementtree.TreeBuilder.__le__" => "Return self<=value.", + "_elementtree.TreeBuilder.__lt__" => "Return self<value.", + "_elementtree.TreeBuilder.__ne__" => "Return self!=value.", + "_elementtree.TreeBuilder.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_elementtree.TreeBuilder.__reduce__" => "Helper for pickle.", + "_elementtree.TreeBuilder.__reduce_ex__" => "Helper for pickle.", + "_elementtree.TreeBuilder.__repr__" => "Return repr(self).", + "_elementtree.TreeBuilder.__setattr__" => "Implement setattr(self, name, value).", + "_elementtree.TreeBuilder.__sizeof__" => "Size of object in memory, in bytes.", + "_elementtree.TreeBuilder.__str__" => "Return str(self).", + "_elementtree.TreeBuilder.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_elementtree.XMLParser.__delattr__" => "Implement delattr(self, name).", + "_elementtree.XMLParser.__eq__" => "Return self==value.", + "_elementtree.XMLParser.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_elementtree.XMLParser.__ge__" => "Return self>=value.", + "_elementtree.XMLParser.__getattribute__" => "Return getattr(self, name).", + "_elementtree.XMLParser.__getstate__" => "Helper for pickle.", + "_elementtree.XMLParser.__gt__" => "Return self>value.", + "_elementtree.XMLParser.__hash__" => "Return hash(self).", + "_elementtree.XMLParser.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_elementtree.XMLParser.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_elementtree.XMLParser.__le__" => "Return self<=value.", + "_elementtree.XMLParser.__lt__" => "Return self<value.", + "_elementtree.XMLParser.__ne__" => "Return self!=value.", + "_elementtree.XMLParser.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_elementtree.XMLParser.__reduce__" => "Helper for pickle.", + "_elementtree.XMLParser.__reduce_ex__" => "Helper for pickle.", + "_elementtree.XMLParser.__repr__" => "Return repr(self).", + "_elementtree.XMLParser.__setattr__" => "Implement setattr(self, name, value).", + "_elementtree.XMLParser.__sizeof__" => "Size of object in memory, in bytes.", + "_elementtree.XMLParser.__str__" => "Return str(self).", + "_elementtree.XMLParser.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_elementtree._set_factories" => "Change the factories used to create comments and processing instructions.\n\nFor internal use only.", + "_functools" => "Tools that operate on functions.", + "_functools._PlaceholderType" => "The type of the Placeholder singleton.\n\nUsed as a placeholder for partial arguments.", + "_functools._PlaceholderType.__delattr__" => "Implement delattr(self, name).", + "_functools._PlaceholderType.__eq__" => "Return self==value.", + "_functools._PlaceholderType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_functools._PlaceholderType.__ge__" => "Return self>=value.", + "_functools._PlaceholderType.__getattribute__" => "Return getattr(self, name).", + "_functools._PlaceholderType.__getstate__" => "Helper for pickle.", + "_functools._PlaceholderType.__gt__" => "Return self>value.", + "_functools._PlaceholderType.__hash__" => "Return hash(self).", + "_functools._PlaceholderType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_functools._PlaceholderType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_functools._PlaceholderType.__le__" => "Return self<=value.", + "_functools._PlaceholderType.__lt__" => "Return self<value.", + "_functools._PlaceholderType.__ne__" => "Return self!=value.", + "_functools._PlaceholderType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_functools._PlaceholderType.__reduce_ex__" => "Helper for pickle.", + "_functools._PlaceholderType.__repr__" => "Return repr(self).", + "_functools._PlaceholderType.__setattr__" => "Implement setattr(self, name, value).", + "_functools._PlaceholderType.__sizeof__" => "Size of object in memory, in bytes.", + "_functools._PlaceholderType.__str__" => "Return str(self).", + "_functools._PlaceholderType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_functools._lru_cache_wrapper" => "Create a cached callable that wraps another function.\n\nuser_function: the function being cached\n\nmaxsize: 0 for no caching\n None for unlimited cache size\n n for a bounded cache\n\ntyped: False cache f(3) and f(3.0) as identical calls\n True cache f(3) and f(3.0) as distinct calls\n\ncache_info_type: namedtuple class with the fields:\n hits misses currsize maxsize", + "_functools._lru_cache_wrapper.__call__" => "Call self as a function.", + "_functools._lru_cache_wrapper.__delattr__" => "Implement delattr(self, name).", + "_functools._lru_cache_wrapper.__eq__" => "Return self==value.", + "_functools._lru_cache_wrapper.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_functools._lru_cache_wrapper.__ge__" => "Return self>=value.", + "_functools._lru_cache_wrapper.__get__" => "Return an attribute of instance, which is of type owner.", + "_functools._lru_cache_wrapper.__getattribute__" => "Return getattr(self, name).", + "_functools._lru_cache_wrapper.__getstate__" => "Helper for pickle.", + "_functools._lru_cache_wrapper.__gt__" => "Return self>value.", + "_functools._lru_cache_wrapper.__hash__" => "Return hash(self).", + "_functools._lru_cache_wrapper.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_functools._lru_cache_wrapper.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_functools._lru_cache_wrapper.__le__" => "Return self<=value.", + "_functools._lru_cache_wrapper.__lt__" => "Return self<value.", + "_functools._lru_cache_wrapper.__ne__" => "Return self!=value.", + "_functools._lru_cache_wrapper.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_functools._lru_cache_wrapper.__reduce_ex__" => "Helper for pickle.", + "_functools._lru_cache_wrapper.__repr__" => "Return repr(self).", + "_functools._lru_cache_wrapper.__setattr__" => "Implement setattr(self, name, value).", + "_functools._lru_cache_wrapper.__sizeof__" => "Size of object in memory, in bytes.", + "_functools._lru_cache_wrapper.__str__" => "Return str(self).", + "_functools._lru_cache_wrapper.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_functools._lru_cache_wrapper.cache_clear" => "Clear the cache and cache statistics", + "_functools._lru_cache_wrapper.cache_info" => "Report cache statistics", + "_functools.cmp_to_key" => "Convert a cmp= function into a key= function.\n\n mycmp\n Function that compares two objects.", + "_functools.partial" => "Create a new function with partial application of the given arguments\nand keywords.", + "_functools.partial.__call__" => "Call self as a function.", + "_functools.partial.__class_getitem__" => "See PEP 585", + "_functools.partial.__delattr__" => "Implement delattr(self, name).", + "_functools.partial.__eq__" => "Return self==value.", + "_functools.partial.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_functools.partial.__ge__" => "Return self>=value.", + "_functools.partial.__get__" => "Return an attribute of instance, which is of type owner.", + "_functools.partial.__getattribute__" => "Return getattr(self, name).", + "_functools.partial.__getstate__" => "Helper for pickle.", + "_functools.partial.__gt__" => "Return self>value.", + "_functools.partial.__hash__" => "Return hash(self).", + "_functools.partial.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_functools.partial.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_functools.partial.__le__" => "Return self<=value.", + "_functools.partial.__lt__" => "Return self<value.", + "_functools.partial.__ne__" => "Return self!=value.", + "_functools.partial.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_functools.partial.__reduce_ex__" => "Helper for pickle.", + "_functools.partial.__repr__" => "Return repr(self).", + "_functools.partial.__setattr__" => "Implement setattr(self, name, value).", + "_functools.partial.__sizeof__" => "Size of object in memory, in bytes.", + "_functools.partial.__str__" => "Return str(self).", + "_functools.partial.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_functools.partial.args" => "tuple of arguments to future partial calls", + "_functools.partial.func" => "function object to use in future partial calls", + "_functools.partial.keywords" => "dictionary of keyword arguments to future partial calls", + "_functools.reduce" => "Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n\nThis effectively reduces the iterable to a single value. If initial is present,\nit is placed before the items of the iterable in the calculation, and serves as\na default when the iterable is empty.\n\nFor example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\ncalculates ((((1 + 2) + 3) + 4) + 5).", + "_gdbm" => "This module provides an interface to the GNU DBM (GDBM) library.\n\nThis module is quite similar to the dbm module, but uses GDBM instead to\nprovide some additional functionality. Please note that the file formats\ncreated by GDBM and dbm are incompatible.\n\nGDBM objects behave like mappings (dictionaries), except that keys and\nvalues are always immutable bytes-like objects or strings. Printing\na GDBM object doesn't print the keys and values, and the items() and\nvalues() methods are not supported.", + "_gdbm.error.__delattr__" => "Implement delattr(self, name).", + "_gdbm.error.__eq__" => "Return self==value.", + "_gdbm.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_gdbm.error.__ge__" => "Return self>=value.", + "_gdbm.error.__getattribute__" => "Return getattr(self, name).", + "_gdbm.error.__getstate__" => "Helper for pickle.", + "_gdbm.error.__gt__" => "Return self>value.", + "_gdbm.error.__hash__" => "Return hash(self).", + "_gdbm.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_gdbm.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_gdbm.error.__le__" => "Return self<=value.", + "_gdbm.error.__lt__" => "Return self<value.", + "_gdbm.error.__ne__" => "Return self!=value.", + "_gdbm.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_gdbm.error.__reduce_ex__" => "Helper for pickle.", + "_gdbm.error.__repr__" => "Return repr(self).", + "_gdbm.error.__setattr__" => "Implement setattr(self, name, value).", + "_gdbm.error.__sizeof__" => "Size of object in memory, in bytes.", + "_gdbm.error.__str__" => "Return str(self).", + "_gdbm.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_gdbm.error.__weakref__" => "list of weak references to the object", + "_gdbm.error.add_note" => "Add a note to the exception", + "_gdbm.error.errno" => "POSIX exception code", + "_gdbm.error.filename" => "exception filename", + "_gdbm.error.filename2" => "second exception filename", + "_gdbm.error.strerror" => "exception strerror", + "_gdbm.error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_gdbm.open" => "Open a dbm database and return a dbm object.\n\nThe filename argument is the name of the database file.\n\nThe optional flags argument can be 'r' (to open an existing database\nfor reading only -- default), 'w' (to open an existing database for\nreading and writing), 'c' (which creates the database if it doesn't\nexist), or 'n' (which always creates a new empty database).\n\nSome versions of gdbm support additional flags which must be\nappended to one of the flags described above. The module constant\n'open_flags' is a string of valid additional flags. The 'f' flag\nopens the database in fast mode; altered data will not automatically\nbe written to the disk after every change. This results in faster\nwrites to the database, but may result in an inconsistent database\nif the program crashes while the database is still open. Use the\nsync() method to force any unwritten data to be written to the disk.\nThe 's' flag causes all database operations to be synchronized to\ndisk. The 'u' flag disables locking of the database file.\n\nThe optional mode argument is the Unix mode of the file, used only\nwhen the database has to be created. It defaults to octal 0o666.", + "_hashlib" => "OpenSSL interface for hashlib module", + "_hashlib.HASH" => "A hash is an object used to calculate a checksum of a string of information.\n\nMethods:\n\nupdate() -- updates the current digest with an additional string\ndigest() -- return the current digest value\nhexdigest() -- return the current digest as a string of hexadecimal digits\ncopy() -- return a copy of the current hash object\n\nAttributes:\n\nname -- the hash algorithm being used by this object\ndigest_size -- number of bytes in this hashes output", + "_hashlib.HASH.__delattr__" => "Implement delattr(self, name).", + "_hashlib.HASH.__eq__" => "Return self==value.", + "_hashlib.HASH.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_hashlib.HASH.__ge__" => "Return self>=value.", + "_hashlib.HASH.__getattribute__" => "Return getattr(self, name).", + "_hashlib.HASH.__getstate__" => "Helper for pickle.", + "_hashlib.HASH.__gt__" => "Return self>value.", + "_hashlib.HASH.__hash__" => "Return hash(self).", + "_hashlib.HASH.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_hashlib.HASH.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_hashlib.HASH.__le__" => "Return self<=value.", + "_hashlib.HASH.__lt__" => "Return self<value.", + "_hashlib.HASH.__ne__" => "Return self!=value.", + "_hashlib.HASH.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_hashlib.HASH.__reduce__" => "Helper for pickle.", + "_hashlib.HASH.__reduce_ex__" => "Helper for pickle.", + "_hashlib.HASH.__repr__" => "Return repr(self).", + "_hashlib.HASH.__setattr__" => "Implement setattr(self, name, value).", + "_hashlib.HASH.__sizeof__" => "Size of object in memory, in bytes.", + "_hashlib.HASH.__str__" => "Return str(self).", + "_hashlib.HASH.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_hashlib.HASH.copy" => "Return a copy of the hash object.", + "_hashlib.HASH.digest" => "Return the digest value as a bytes object.", + "_hashlib.HASH.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_hashlib.HASH.update" => "Update this hash object's state with the provided string.", + "_hashlib.HASHXOF" => "A hash is an object used to calculate a checksum of a string of information.\n\nMethods:\n\nupdate() -- updates the current digest with an additional string\ndigest(length) -- return the current digest value\nhexdigest(length) -- return the current digest as a string of hexadecimal digits\ncopy() -- return a copy of the current hash object\n\nAttributes:\n\nname -- the hash algorithm being used by this object\ndigest_size -- number of bytes in this hashes output", + "_hashlib.HASHXOF.__delattr__" => "Implement delattr(self, name).", + "_hashlib.HASHXOF.__eq__" => "Return self==value.", + "_hashlib.HASHXOF.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_hashlib.HASHXOF.__ge__" => "Return self>=value.", + "_hashlib.HASHXOF.__getattribute__" => "Return getattr(self, name).", + "_hashlib.HASHXOF.__getstate__" => "Helper for pickle.", + "_hashlib.HASHXOF.__gt__" => "Return self>value.", + "_hashlib.HASHXOF.__hash__" => "Return hash(self).", + "_hashlib.HASHXOF.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_hashlib.HASHXOF.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_hashlib.HASHXOF.__le__" => "Return self<=value.", + "_hashlib.HASHXOF.__lt__" => "Return self<value.", + "_hashlib.HASHXOF.__ne__" => "Return self!=value.", + "_hashlib.HASHXOF.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_hashlib.HASHXOF.__reduce__" => "Helper for pickle.", + "_hashlib.HASHXOF.__reduce_ex__" => "Helper for pickle.", + "_hashlib.HASHXOF.__repr__" => "Return repr(self).", + "_hashlib.HASHXOF.__setattr__" => "Implement setattr(self, name, value).", + "_hashlib.HASHXOF.__sizeof__" => "Size of object in memory, in bytes.", + "_hashlib.HASHXOF.__str__" => "Return str(self).", + "_hashlib.HASHXOF.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_hashlib.HASHXOF.copy" => "Return a copy of the hash object.", + "_hashlib.HASHXOF.digest" => "Return the digest value as a bytes object.", + "_hashlib.HASHXOF.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_hashlib.HASHXOF.update" => "Update this hash object's state with the provided string.", + "_hashlib.HMAC" => "The object used to calculate HMAC of a message.\n\nMethods:\n\nupdate() -- updates the current digest with an additional string\ndigest() -- return the current digest value\nhexdigest() -- return the current digest as a string of hexadecimal digits\ncopy() -- return a copy of the current hash object\n\nAttributes:\n\nname -- the name, including the hash algorithm used by this object\ndigest_size -- number of bytes in digest() output", + "_hashlib.HMAC.__delattr__" => "Implement delattr(self, name).", + "_hashlib.HMAC.__eq__" => "Return self==value.", + "_hashlib.HMAC.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_hashlib.HMAC.__ge__" => "Return self>=value.", + "_hashlib.HMAC.__getattribute__" => "Return getattr(self, name).", + "_hashlib.HMAC.__getstate__" => "Helper for pickle.", + "_hashlib.HMAC.__gt__" => "Return self>value.", + "_hashlib.HMAC.__hash__" => "Return hash(self).", + "_hashlib.HMAC.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_hashlib.HMAC.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_hashlib.HMAC.__le__" => "Return self<=value.", + "_hashlib.HMAC.__lt__" => "Return self<value.", + "_hashlib.HMAC.__ne__" => "Return self!=value.", + "_hashlib.HMAC.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_hashlib.HMAC.__reduce__" => "Helper for pickle.", + "_hashlib.HMAC.__reduce_ex__" => "Helper for pickle.", + "_hashlib.HMAC.__repr__" => "Return repr(self).", + "_hashlib.HMAC.__setattr__" => "Implement setattr(self, name, value).", + "_hashlib.HMAC.__sizeof__" => "Size of object in memory, in bytes.", + "_hashlib.HMAC.__str__" => "Return str(self).", + "_hashlib.HMAC.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_hashlib.HMAC.copy" => "Return a copy (\"clone\") of the HMAC object.", + "_hashlib.HMAC.digest" => "Return the digest of the bytes passed to the update() method so far.", + "_hashlib.HMAC.hexdigest" => "Return hexadecimal digest of the bytes passed to the update() method so far.\n\nThis may be used to exchange the value safely in email or other non-binary\nenvironments.", + "_hashlib.HMAC.update" => "Update the HMAC object with msg.", + "_hashlib.UnsupportedDigestmodError.__delattr__" => "Implement delattr(self, name).", + "_hashlib.UnsupportedDigestmodError.__eq__" => "Return self==value.", + "_hashlib.UnsupportedDigestmodError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_hashlib.UnsupportedDigestmodError.__ge__" => "Return self>=value.", + "_hashlib.UnsupportedDigestmodError.__getattribute__" => "Return getattr(self, name).", + "_hashlib.UnsupportedDigestmodError.__getstate__" => "Helper for pickle.", + "_hashlib.UnsupportedDigestmodError.__gt__" => "Return self>value.", + "_hashlib.UnsupportedDigestmodError.__hash__" => "Return hash(self).", + "_hashlib.UnsupportedDigestmodError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_hashlib.UnsupportedDigestmodError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_hashlib.UnsupportedDigestmodError.__le__" => "Return self<=value.", + "_hashlib.UnsupportedDigestmodError.__lt__" => "Return self<value.", + "_hashlib.UnsupportedDigestmodError.__ne__" => "Return self!=value.", + "_hashlib.UnsupportedDigestmodError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_hashlib.UnsupportedDigestmodError.__reduce_ex__" => "Helper for pickle.", + "_hashlib.UnsupportedDigestmodError.__repr__" => "Return repr(self).", + "_hashlib.UnsupportedDigestmodError.__setattr__" => "Implement setattr(self, name, value).", + "_hashlib.UnsupportedDigestmodError.__sizeof__" => "Size of object in memory, in bytes.", + "_hashlib.UnsupportedDigestmodError.__str__" => "Return str(self).", + "_hashlib.UnsupportedDigestmodError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_hashlib.UnsupportedDigestmodError.__weakref__" => "list of weak references to the object", + "_hashlib.UnsupportedDigestmodError.add_note" => "Add a note to the exception", + "_hashlib.UnsupportedDigestmodError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_hashlib.compare_digest" => "Return 'a == b'.\n\nThis function uses an approach designed to prevent\ntiming analysis, making it appropriate for cryptography.\n\na and b must both be of the same type: either str (ASCII only),\nor any bytes-like object.\n\nNote: If a and b are of different lengths, or if an error occurs,\na timing attack could theoretically reveal information about the\ntypes and lengths of a and b--but not their values.", + "_hashlib.get_fips_mode" => "Determine the OpenSSL FIPS mode of operation.\n\nFor OpenSSL 3.0.0 and newer it returns the state of the default provider\nin the default OSSL context. It's not quite the same as FIPS_mode() but good\nenough for unittests.\n\nEffectively any non-zero return value indicates FIPS mode;\nvalues other than 1 may have additional significance.", + "_hashlib.hmac_digest" => "Single-shot HMAC.", + "_hashlib.hmac_new" => "Return a new hmac object.", + "_hashlib.new" => "Return a new hash object using the named algorithm.\n\nAn optional string argument may be provided and will be\nautomatically hashed.\n\nThe MD5 and SHA1 algorithms are always supported.", + "_hashlib.openssl_md5" => "Returns a md5 hash object; optionally initialized with a string", + "_hashlib.openssl_sha1" => "Returns a sha1 hash object; optionally initialized with a string", + "_hashlib.openssl_sha224" => "Returns a sha224 hash object; optionally initialized with a string", + "_hashlib.openssl_sha256" => "Returns a sha256 hash object; optionally initialized with a string", + "_hashlib.openssl_sha384" => "Returns a sha384 hash object; optionally initialized with a string", + "_hashlib.openssl_sha3_224" => "Returns a sha3-224 hash object; optionally initialized with a string", + "_hashlib.openssl_sha3_256" => "Returns a sha3-256 hash object; optionally initialized with a string", + "_hashlib.openssl_sha3_384" => "Returns a sha3-384 hash object; optionally initialized with a string", + "_hashlib.openssl_sha3_512" => "Returns a sha3-512 hash object; optionally initialized with a string", + "_hashlib.openssl_sha512" => "Returns a sha512 hash object; optionally initialized with a string", + "_hashlib.openssl_shake_128" => "Returns a shake-128 variable hash object; optionally initialized with a string", + "_hashlib.openssl_shake_256" => "Returns a shake-256 variable hash object; optionally initialized with a string", + "_hashlib.pbkdf2_hmac" => "Password based key derivation function 2 (PKCS #5 v2.0) with HMAC as pseudorandom function.", + "_hashlib.scrypt" => "scrypt password-based key derivation function.", + "_heapq" => "Heap queue algorithm (a.k.a. priority queue).\n\nHeaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for\nall k, counting elements from 0. For the sake of comparison,\nnon-existing elements are considered to be infinite. The interesting\nproperty of a heap is that a[0] is always its smallest element.\n\nUsage:\n\nheap = [] # creates an empty heap\nheappush(heap, item) # pushes a new item on the heap\nitem = heappop(heap) # pops the smallest item from the heap\nitem = heap[0] # smallest item on the heap without popping it\nheapify(x) # transforms list into a heap, in-place, in linear time\nitem = heapreplace(heap, item) # pops and returns smallest item, and adds\n # new item; the heap size is unchanged\n\nOur API differs from textbook heap algorithms as follows:\n\n- We use 0-based indexing. This makes the relationship between the\n index for a node and the indexes for its children slightly less\n obvious, but is more suitable since Python uses 0-based indexing.\n\n- Our heappop() method returns the smallest item, not the largest.\n\nThese two make it possible to view the heap as a regular Python list\nwithout surprises: heap[0] is the smallest item, and heap.sort()\nmaintains the heap invariant!", + "_heapq.heapify" => "Transform list into a heap, in-place, in O(len(heap)) time.", + "_heapq.heapify_max" => "Maxheap variant of heapify.", + "_heapq.heappop" => "Pop the smallest item off the heap, maintaining the heap invariant.", + "_heapq.heappop_max" => "Maxheap variant of heappop.", + "_heapq.heappush" => "Push item onto heap, maintaining the heap invariant.", + "_heapq.heappush_max" => "Push item onto max heap, maintaining the heap invariant.", + "_heapq.heappushpop" => "Push item on the heap, then pop and return the smallest item from the heap.\n\nThe combined action runs more efficiently than heappush() followed by\na separate call to heappop().", + "_heapq.heappushpop_max" => "Maxheap variant of heappushpop.\n\nThe combined action runs more efficiently than heappush_max() followed by\na separate call to heappop_max().", + "_heapq.heapreplace" => "Pop and return the current smallest value, and add the new item.\n\nThis is more efficient than heappop() followed by heappush(), and can be\nmore appropriate when using a fixed-size heap. Note that the value\nreturned may be larger than item! That constrains reasonable uses of\nthis routine unless written as part of a conditional replacement:\n\n if item > heap[0]:\n item = heapreplace(heap, item)", + "_heapq.heapreplace_max" => "Maxheap variant of heapreplace.", + "_hmac.HMAC.__delattr__" => "Implement delattr(self, name).", + "_hmac.HMAC.__eq__" => "Return self==value.", + "_hmac.HMAC.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_hmac.HMAC.__ge__" => "Return self>=value.", + "_hmac.HMAC.__getattribute__" => "Return getattr(self, name).", + "_hmac.HMAC.__getstate__" => "Helper for pickle.", + "_hmac.HMAC.__gt__" => "Return self>value.", + "_hmac.HMAC.__hash__" => "Return hash(self).", + "_hmac.HMAC.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_hmac.HMAC.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_hmac.HMAC.__le__" => "Return self<=value.", + "_hmac.HMAC.__lt__" => "Return self<value.", + "_hmac.HMAC.__ne__" => "Return self!=value.", + "_hmac.HMAC.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_hmac.HMAC.__reduce__" => "Helper for pickle.", + "_hmac.HMAC.__reduce_ex__" => "Helper for pickle.", + "_hmac.HMAC.__repr__" => "Return repr(self).", + "_hmac.HMAC.__setattr__" => "Implement setattr(self, name, value).", + "_hmac.HMAC.__sizeof__" => "Size of object in memory, in bytes.", + "_hmac.HMAC.__str__" => "Return str(self).", + "_hmac.HMAC.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_hmac.HMAC.copy" => "Return a copy (\"clone\") of the HMAC object.", + "_hmac.HMAC.digest" => "Return the digest of the bytes passed to the update() method so far.\n\nThis method may raise a MemoryError.", + "_hmac.HMAC.hexdigest" => "Return hexadecimal digest of the bytes passed to the update() method so far.\n\nThis may be used to exchange the value safely in email or other non-binary\nenvironments.\n\nThis method may raise a MemoryError.", + "_hmac.HMAC.update" => "Update the HMAC object with the given message.", + "_hmac.UnknownHashError.__delattr__" => "Implement delattr(self, name).", + "_hmac.UnknownHashError.__eq__" => "Return self==value.", + "_hmac.UnknownHashError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_hmac.UnknownHashError.__ge__" => "Return self>=value.", + "_hmac.UnknownHashError.__getattribute__" => "Return getattr(self, name).", + "_hmac.UnknownHashError.__getstate__" => "Helper for pickle.", + "_hmac.UnknownHashError.__gt__" => "Return self>value.", + "_hmac.UnknownHashError.__hash__" => "Return hash(self).", + "_hmac.UnknownHashError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_hmac.UnknownHashError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_hmac.UnknownHashError.__le__" => "Return self<=value.", + "_hmac.UnknownHashError.__lt__" => "Return self<value.", + "_hmac.UnknownHashError.__ne__" => "Return self!=value.", + "_hmac.UnknownHashError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_hmac.UnknownHashError.__reduce_ex__" => "Helper for pickle.", + "_hmac.UnknownHashError.__repr__" => "Return repr(self).", + "_hmac.UnknownHashError.__setattr__" => "Implement setattr(self, name, value).", + "_hmac.UnknownHashError.__sizeof__" => "Size of object in memory, in bytes.", + "_hmac.UnknownHashError.__str__" => "Return str(self).", + "_hmac.UnknownHashError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_hmac.UnknownHashError.__weakref__" => "list of weak references to the object", + "_hmac.UnknownHashError.add_note" => "Add a note to the exception", + "_hmac.UnknownHashError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_hmac.new" => "Return a new HMAC object.", + "_imp" => "(Extremely) low-level import machinery bits as used by importlib.", + "_imp._fix_co_filename" => "Changes code.co_filename to specify the passed-in file path.\n\n code\n Code object to change.\n path\n File path to use.", + "_imp._frozen_module_names" => "Returns the list of available frozen modules.", + "_imp._override_frozen_modules_for_tests" => "(internal-only) Override PyConfig.use_frozen_modules.\n\n(-1: \"off\", 1: \"on\", 0: no override)\nSee frozen_modules() in Lib/test/support/import_helper.py.", + "_imp._override_multi_interp_extensions_check" => "(internal-only) Override PyInterpreterConfig.check_multi_interp_extensions.\n\n(-1: \"never\", 1: \"always\", 0: no override)", + "_imp.acquire_lock" => "Acquires the interpreter's import lock for the current thread.\n\nThis lock should be used by import hooks to ensure thread-safety when importing\nmodules. On platforms without threads, this function does nothing.", + "_imp.create_builtin" => "Create an extension module.", + "_imp.create_dynamic" => "Create an extension module.", + "_imp.exec_builtin" => "Initialize a built-in module.", + "_imp.exec_dynamic" => "Initialize an extension module.", + "_imp.extension_suffixes" => "Returns the list of file suffixes used to identify extension modules.", + "_imp.find_frozen" => "Return info about the corresponding frozen module (if there is one) or None.\n\nThe returned info (a 2-tuple):\n\n * data the raw marshalled bytes\n * is_package whether or not it is a package\n * origname the originally frozen module's name, or None if not\n a stdlib module (this will usually be the same as\n the module's current name)", + "_imp.get_frozen_object" => "Create a code object for a frozen module.", + "_imp.init_frozen" => "Initializes a frozen module.", + "_imp.is_builtin" => "Returns True if the module name corresponds to a built-in module.", + "_imp.is_frozen" => "Returns True if the module name corresponds to a frozen module.", + "_imp.is_frozen_package" => "Returns True if the module name is of a frozen package.", + "_imp.lock_held" => "Return True if the import lock is currently held, else False.\n\nOn platforms without threads, return False.", + "_imp.release_lock" => "Release the interpreter's import lock.\n\nOn platforms without threads, this function does nothing.", + "_interpchannels" => "This module provides primitive operations to manage Python interpreters.\nThe 'interpreters' module provides a more convenient interface.", + "_interpchannels.ChannelClosedError.__delattr__" => "Implement delattr(self, name).", + "_interpchannels.ChannelClosedError.__eq__" => "Return self==value.", + "_interpchannels.ChannelClosedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpchannels.ChannelClosedError.__ge__" => "Return self>=value.", + "_interpchannels.ChannelClosedError.__getattribute__" => "Return getattr(self, name).", + "_interpchannels.ChannelClosedError.__getstate__" => "Helper for pickle.", + "_interpchannels.ChannelClosedError.__gt__" => "Return self>value.", + "_interpchannels.ChannelClosedError.__hash__" => "Return hash(self).", + "_interpchannels.ChannelClosedError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpchannels.ChannelClosedError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpchannels.ChannelClosedError.__le__" => "Return self<=value.", + "_interpchannels.ChannelClosedError.__lt__" => "Return self<value.", + "_interpchannels.ChannelClosedError.__ne__" => "Return self!=value.", + "_interpchannels.ChannelClosedError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpchannels.ChannelClosedError.__reduce_ex__" => "Helper for pickle.", + "_interpchannels.ChannelClosedError.__repr__" => "Return repr(self).", + "_interpchannels.ChannelClosedError.__setattr__" => "Implement setattr(self, name, value).", + "_interpchannels.ChannelClosedError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpchannels.ChannelClosedError.__str__" => "Return str(self).", + "_interpchannels.ChannelClosedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpchannels.ChannelClosedError.__weakref__" => "list of weak references to the object", + "_interpchannels.ChannelClosedError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelClosedError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpchannels.ChannelEmptyError.__delattr__" => "Implement delattr(self, name).", + "_interpchannels.ChannelEmptyError.__eq__" => "Return self==value.", + "_interpchannels.ChannelEmptyError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpchannels.ChannelEmptyError.__ge__" => "Return self>=value.", + "_interpchannels.ChannelEmptyError.__getattribute__" => "Return getattr(self, name).", + "_interpchannels.ChannelEmptyError.__getstate__" => "Helper for pickle.", + "_interpchannels.ChannelEmptyError.__gt__" => "Return self>value.", + "_interpchannels.ChannelEmptyError.__hash__" => "Return hash(self).", + "_interpchannels.ChannelEmptyError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpchannels.ChannelEmptyError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpchannels.ChannelEmptyError.__le__" => "Return self<=value.", + "_interpchannels.ChannelEmptyError.__lt__" => "Return self<value.", + "_interpchannels.ChannelEmptyError.__ne__" => "Return self!=value.", + "_interpchannels.ChannelEmptyError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpchannels.ChannelEmptyError.__reduce_ex__" => "Helper for pickle.", + "_interpchannels.ChannelEmptyError.__repr__" => "Return repr(self).", + "_interpchannels.ChannelEmptyError.__setattr__" => "Implement setattr(self, name, value).", + "_interpchannels.ChannelEmptyError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpchannels.ChannelEmptyError.__str__" => "Return str(self).", + "_interpchannels.ChannelEmptyError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpchannels.ChannelEmptyError.__weakref__" => "list of weak references to the object", + "_interpchannels.ChannelEmptyError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelEmptyError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpchannels.ChannelError.__delattr__" => "Implement delattr(self, name).", + "_interpchannels.ChannelError.__eq__" => "Return self==value.", + "_interpchannels.ChannelError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpchannels.ChannelError.__ge__" => "Return self>=value.", + "_interpchannels.ChannelError.__getattribute__" => "Return getattr(self, name).", + "_interpchannels.ChannelError.__getstate__" => "Helper for pickle.", + "_interpchannels.ChannelError.__gt__" => "Return self>value.", + "_interpchannels.ChannelError.__hash__" => "Return hash(self).", + "_interpchannels.ChannelError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpchannels.ChannelError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpchannels.ChannelError.__le__" => "Return self<=value.", + "_interpchannels.ChannelError.__lt__" => "Return self<value.", + "_interpchannels.ChannelError.__ne__" => "Return self!=value.", + "_interpchannels.ChannelError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpchannels.ChannelError.__reduce_ex__" => "Helper for pickle.", + "_interpchannels.ChannelError.__repr__" => "Return repr(self).", + "_interpchannels.ChannelError.__setattr__" => "Implement setattr(self, name, value).", + "_interpchannels.ChannelError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpchannels.ChannelError.__str__" => "Return str(self).", + "_interpchannels.ChannelError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpchannels.ChannelError.__weakref__" => "list of weak references to the object", + "_interpchannels.ChannelError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpchannels.ChannelID" => "A channel ID identifies a channel and may be used as an int.", + "_interpchannels.ChannelID.__delattr__" => "Implement delattr(self, name).", + "_interpchannels.ChannelID.__eq__" => "Return self==value.", + "_interpchannels.ChannelID.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpchannels.ChannelID.__ge__" => "Return self>=value.", + "_interpchannels.ChannelID.__getattribute__" => "Return getattr(self, name).", + "_interpchannels.ChannelID.__getstate__" => "Helper for pickle.", + "_interpchannels.ChannelID.__gt__" => "Return self>value.", + "_interpchannels.ChannelID.__hash__" => "Return hash(self).", + "_interpchannels.ChannelID.__index__" => "Return self converted to an integer, if self is suitable for use as an index into a list.", + "_interpchannels.ChannelID.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpchannels.ChannelID.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpchannels.ChannelID.__int__" => "int(self)", + "_interpchannels.ChannelID.__le__" => "Return self<=value.", + "_interpchannels.ChannelID.__lt__" => "Return self<value.", + "_interpchannels.ChannelID.__ne__" => "Return self!=value.", + "_interpchannels.ChannelID.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpchannels.ChannelID.__reduce__" => "Helper for pickle.", + "_interpchannels.ChannelID.__reduce_ex__" => "Helper for pickle.", + "_interpchannels.ChannelID.__repr__" => "Return repr(self).", + "_interpchannels.ChannelID.__setattr__" => "Implement setattr(self, name, value).", + "_interpchannels.ChannelID.__sizeof__" => "Size of object in memory, in bytes.", + "_interpchannels.ChannelID.__str__" => "Return str(self).", + "_interpchannels.ChannelID.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpchannels.ChannelID.end" => "'send', 'recv', or 'both'", + "_interpchannels.ChannelID.recv" => "the 'recv' end of the channel", + "_interpchannels.ChannelID.send" => "the 'send' end of the channel", + "_interpchannels.ChannelInfo" => "ChannelInfo\n\nA named tuple of a channel's state.", + "_interpchannels.ChannelInfo.__add__" => "Return self+value.", + "_interpchannels.ChannelInfo.__class_getitem__" => "See PEP 585", + "_interpchannels.ChannelInfo.__contains__" => "Return bool(key in self).", + "_interpchannels.ChannelInfo.__delattr__" => "Implement delattr(self, name).", + "_interpchannels.ChannelInfo.__eq__" => "Return self==value.", + "_interpchannels.ChannelInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpchannels.ChannelInfo.__ge__" => "Return self>=value.", + "_interpchannels.ChannelInfo.__getattribute__" => "Return getattr(self, name).", + "_interpchannels.ChannelInfo.__getitem__" => "Return self[key].", + "_interpchannels.ChannelInfo.__getstate__" => "Helper for pickle.", + "_interpchannels.ChannelInfo.__gt__" => "Return self>value.", + "_interpchannels.ChannelInfo.__hash__" => "Return hash(self).", + "_interpchannels.ChannelInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpchannels.ChannelInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpchannels.ChannelInfo.__iter__" => "Implement iter(self).", + "_interpchannels.ChannelInfo.__le__" => "Return self<=value.", + "_interpchannels.ChannelInfo.__len__" => "Return len(self).", + "_interpchannels.ChannelInfo.__lt__" => "Return self<value.", + "_interpchannels.ChannelInfo.__mul__" => "Return self*value.", + "_interpchannels.ChannelInfo.__ne__" => "Return self!=value.", + "_interpchannels.ChannelInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpchannels.ChannelInfo.__reduce_ex__" => "Helper for pickle.", + "_interpchannels.ChannelInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_interpchannels.ChannelInfo.__repr__" => "Return repr(self).", + "_interpchannels.ChannelInfo.__rmul__" => "Return value*self.", + "_interpchannels.ChannelInfo.__setattr__" => "Implement setattr(self, name, value).", + "_interpchannels.ChannelInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_interpchannels.ChannelInfo.__str__" => "Return str(self).", + "_interpchannels.ChannelInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpchannels.ChannelInfo.closed" => "both ends are closed", + "_interpchannels.ChannelInfo.closing" => "send is closed, recv is non-empty", + "_interpchannels.ChannelInfo.count" => "queued objects", + "_interpchannels.ChannelInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_interpchannels.ChannelInfo.num_interp_both" => "interpreters bound to both ends", + "_interpchannels.ChannelInfo.num_interp_both_recv_released" => "interpreters bound to both ends and released_from_the recv end", + "_interpchannels.ChannelInfo.num_interp_both_released" => "interpreters bound to both ends and released_from_both", + "_interpchannels.ChannelInfo.num_interp_both_send_released" => "interpreters bound to both ends and released_from_the send end", + "_interpchannels.ChannelInfo.num_interp_recv" => "interpreters bound to the send end", + "_interpchannels.ChannelInfo.num_interp_recv_released" => "interpreters bound to the send end and released", + "_interpchannels.ChannelInfo.num_interp_send" => "interpreters bound to the send end", + "_interpchannels.ChannelInfo.num_interp_send_released" => "interpreters bound to the send end and released", + "_interpchannels.ChannelInfo.open" => "both ends are open", + "_interpchannels.ChannelInfo.recv_associated" => "current interpreter is bound to the recv end", + "_interpchannels.ChannelInfo.recv_released" => "current interpreter *was* bound to the recv end", + "_interpchannels.ChannelInfo.send_associated" => "current interpreter is bound to the send end", + "_interpchannels.ChannelInfo.send_released" => "current interpreter *was* bound to the send end", + "_interpchannels.ChannelNotEmptyError.__delattr__" => "Implement delattr(self, name).", + "_interpchannels.ChannelNotEmptyError.__eq__" => "Return self==value.", + "_interpchannels.ChannelNotEmptyError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpchannels.ChannelNotEmptyError.__ge__" => "Return self>=value.", + "_interpchannels.ChannelNotEmptyError.__getattribute__" => "Return getattr(self, name).", + "_interpchannels.ChannelNotEmptyError.__getstate__" => "Helper for pickle.", + "_interpchannels.ChannelNotEmptyError.__gt__" => "Return self>value.", + "_interpchannels.ChannelNotEmptyError.__hash__" => "Return hash(self).", + "_interpchannels.ChannelNotEmptyError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpchannels.ChannelNotEmptyError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpchannels.ChannelNotEmptyError.__le__" => "Return self<=value.", + "_interpchannels.ChannelNotEmptyError.__lt__" => "Return self<value.", + "_interpchannels.ChannelNotEmptyError.__ne__" => "Return self!=value.", + "_interpchannels.ChannelNotEmptyError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpchannels.ChannelNotEmptyError.__reduce_ex__" => "Helper for pickle.", + "_interpchannels.ChannelNotEmptyError.__repr__" => "Return repr(self).", + "_interpchannels.ChannelNotEmptyError.__setattr__" => "Implement setattr(self, name, value).", + "_interpchannels.ChannelNotEmptyError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpchannels.ChannelNotEmptyError.__str__" => "Return str(self).", + "_interpchannels.ChannelNotEmptyError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpchannels.ChannelNotEmptyError.__weakref__" => "list of weak references to the object", + "_interpchannels.ChannelNotEmptyError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelNotEmptyError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpchannels.ChannelNotFoundError.__delattr__" => "Implement delattr(self, name).", + "_interpchannels.ChannelNotFoundError.__eq__" => "Return self==value.", + "_interpchannels.ChannelNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpchannels.ChannelNotFoundError.__ge__" => "Return self>=value.", + "_interpchannels.ChannelNotFoundError.__getattribute__" => "Return getattr(self, name).", + "_interpchannels.ChannelNotFoundError.__getstate__" => "Helper for pickle.", + "_interpchannels.ChannelNotFoundError.__gt__" => "Return self>value.", + "_interpchannels.ChannelNotFoundError.__hash__" => "Return hash(self).", + "_interpchannels.ChannelNotFoundError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpchannels.ChannelNotFoundError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpchannels.ChannelNotFoundError.__le__" => "Return self<=value.", + "_interpchannels.ChannelNotFoundError.__lt__" => "Return self<value.", + "_interpchannels.ChannelNotFoundError.__ne__" => "Return self!=value.", + "_interpchannels.ChannelNotFoundError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpchannels.ChannelNotFoundError.__reduce_ex__" => "Helper for pickle.", + "_interpchannels.ChannelNotFoundError.__repr__" => "Return repr(self).", + "_interpchannels.ChannelNotFoundError.__setattr__" => "Implement setattr(self, name, value).", + "_interpchannels.ChannelNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpchannels.ChannelNotFoundError.__str__" => "Return str(self).", + "_interpchannels.ChannelNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpchannels.ChannelNotFoundError.__weakref__" => "list of weak references to the object", + "_interpchannels.ChannelNotFoundError.add_note" => "Add a note to the exception", + "_interpchannels.ChannelNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpchannels.close" => "channel_close(cid, *, send=None, recv=None, force=False)\n\nClose the channel for all interpreters.\n\nIf the channel is empty then the keyword args are ignored and both\nends are immediately closed. Otherwise, if 'force' is True then\nall queued items are released and both ends are immediately\nclosed.\n\nIf the channel is not empty *and* 'force' is False then following\nhappens:\n\n * recv is True (regardless of send):\n - raise ChannelNotEmptyError\n * recv is None and send is None:\n - raise ChannelNotEmptyError\n * send is True and recv is not True:\n - fully close the 'send' end\n - close the 'recv' end to interpreters not already receiving\n - fully close it once empty\n\nClosing an already closed channel results in a ChannelClosedError.\n\nOnce the channel's ID has no more ref counts in any interpreter\nthe channel will be destroyed.", + "_interpchannels.create" => "channel_create(unboundop) -> cid\n\nCreate a new cross-interpreter channel and return a unique generated ID.", + "_interpchannels.destroy" => "channel_destroy(cid)\n\nClose and finalize the channel. Afterward attempts to use the channel\nwill behave as though it never existed.", + "_interpchannels.get_channel_defaults" => "get_channel_defaults(cid)\n\nReturn the channel's default values, set when it was created.", + "_interpchannels.get_count" => "get_count(cid)\n\nReturn the number of items in the channel.", + "_interpchannels.get_info" => "get_info(cid)\n\nReturn details about the channel.", + "_interpchannels.list_all" => "channel_list_all() -> [cid]\n\nReturn the list of all IDs for active channels.", + "_interpchannels.list_interpreters" => "channel_list_interpreters(cid, *, send) -> [id]\n\nReturn the list of all interpreter IDs associated with an end of the channel.\n\nThe 'send' argument should be a boolean indicating whether to use the send or\nreceive end.", + "_interpchannels.recv" => "channel_recv(cid, [default]) -> (obj, unboundop)\n\nReturn a new object from the data at the front of the channel's queue.\n\nIf there is nothing to receive then raise ChannelEmptyError, unless\na default value is provided. In that case return it.", + "_interpchannels.release" => "channel_release(cid, *, send=None, recv=None, force=True)\n\nClose the channel for the current interpreter. 'send' and 'recv'\n(bool) may be used to indicate the ends to close. By default both\nends are closed. Closing an already closed end is a noop.", + "_interpchannels.send" => "channel_send(cid, obj, *, blocking=True, timeout=None)\n\nAdd the object's data to the channel's queue.\nBy default this waits for the object to be received.", + "_interpchannels.send_buffer" => "channel_send_buffer(cid, obj, *, blocking=True, timeout=None)\n\nAdd the object's buffer to the channel's queue.\nBy default this waits for the object to be received.", + "_interpqueues" => "This module provides primitive operations to manage Python interpreters.\nThe 'interpreters' module provides a more convenient interface.", + "_interpqueues.QueueError" => "Indicates that a queue-related error happened.", + "_interpqueues.QueueError.__delattr__" => "Implement delattr(self, name).", + "_interpqueues.QueueError.__eq__" => "Return self==value.", + "_interpqueues.QueueError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpqueues.QueueError.__ge__" => "Return self>=value.", + "_interpqueues.QueueError.__getattribute__" => "Return getattr(self, name).", + "_interpqueues.QueueError.__getstate__" => "Helper for pickle.", + "_interpqueues.QueueError.__gt__" => "Return self>value.", + "_interpqueues.QueueError.__hash__" => "Return hash(self).", + "_interpqueues.QueueError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpqueues.QueueError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpqueues.QueueError.__le__" => "Return self<=value.", + "_interpqueues.QueueError.__lt__" => "Return self<value.", + "_interpqueues.QueueError.__ne__" => "Return self!=value.", + "_interpqueues.QueueError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpqueues.QueueError.__reduce_ex__" => "Helper for pickle.", + "_interpqueues.QueueError.__repr__" => "Return repr(self).", + "_interpqueues.QueueError.__setattr__" => "Implement setattr(self, name, value).", + "_interpqueues.QueueError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpqueues.QueueError.__str__" => "Return str(self).", + "_interpqueues.QueueError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpqueues.QueueError.__weakref__" => "list of weak references to the object", + "_interpqueues.QueueError.add_note" => "Add a note to the exception", + "_interpqueues.QueueError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpqueues.QueueNotFoundError.__delattr__" => "Implement delattr(self, name).", + "_interpqueues.QueueNotFoundError.__eq__" => "Return self==value.", + "_interpqueues.QueueNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpqueues.QueueNotFoundError.__ge__" => "Return self>=value.", + "_interpqueues.QueueNotFoundError.__getattribute__" => "Return getattr(self, name).", + "_interpqueues.QueueNotFoundError.__getstate__" => "Helper for pickle.", + "_interpqueues.QueueNotFoundError.__gt__" => "Return self>value.", + "_interpqueues.QueueNotFoundError.__hash__" => "Return hash(self).", + "_interpqueues.QueueNotFoundError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpqueues.QueueNotFoundError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpqueues.QueueNotFoundError.__le__" => "Return self<=value.", + "_interpqueues.QueueNotFoundError.__lt__" => "Return self<value.", + "_interpqueues.QueueNotFoundError.__ne__" => "Return self!=value.", + "_interpqueues.QueueNotFoundError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpqueues.QueueNotFoundError.__reduce_ex__" => "Helper for pickle.", + "_interpqueues.QueueNotFoundError.__repr__" => "Return repr(self).", + "_interpqueues.QueueNotFoundError.__setattr__" => "Implement setattr(self, name, value).", + "_interpqueues.QueueNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpqueues.QueueNotFoundError.__str__" => "Return str(self).", + "_interpqueues.QueueNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpqueues.QueueNotFoundError.__weakref__" => "list of weak references to the object", + "_interpqueues.QueueNotFoundError.add_note" => "Add a note to the exception", + "_interpqueues.QueueNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpqueues.bind" => "bind(qid)\n\nTake a reference to the identified queue.\nThe queue is not destroyed until there are no references left.", + "_interpqueues.create" => "create(maxsize, unboundop, fallback) -> qid\n\nCreate a new cross-interpreter queue and return its unique generated ID.\nIt is a new reference as though bind() had been called on the queue.\n\nThe caller is responsible for calling destroy() for the new queue\nbefore the runtime is finalized.", + "_interpqueues.destroy" => "destroy(qid)\n\nClear and destroy the queue. Afterward attempts to use the queue\nwill behave as though it never existed.", + "_interpqueues.get" => "get(qid) -> (obj, unboundop)\n\nReturn a new object from the data at the front of the queue.\nThe unbound op is also returned.\n\nIf there is nothing to receive then raise QueueEmpty.", + "_interpqueues.get_count" => "get_count(qid)\n\nReturn the number of items in the queue.", + "_interpqueues.get_maxsize" => "get_maxsize(qid)\n\nReturn the maximum number of items in the queue.", + "_interpqueues.get_queue_defaults" => "get_queue_defaults(qid)\n\nReturn the queue's default values, set when it was created.", + "_interpqueues.is_full" => "is_full(qid)\n\nReturn true if the queue has a maxsize and has reached it.", + "_interpqueues.list_all" => "list_all() -> [(qid, unboundop, fallback)]\n\nReturn the list of IDs for all queues.\nEach corresponding default unbound op and fallback is also included.", + "_interpqueues.put" => "put(qid, obj)\n\nAdd the object's data to the queue.", + "_interpqueues.release" => "release(qid)\n\nRelease a reference to the queue.\nThe queue is destroyed once there are no references left.", + "_interpreters" => "This module provides primitive operations to manage Python interpreters.\nThe 'interpreters' module provides a more convenient interface.", + "_interpreters.CrossInterpreterBufferView.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "_interpreters.CrossInterpreterBufferView.__delattr__" => "Implement delattr(self, name).", + "_interpreters.CrossInterpreterBufferView.__eq__" => "Return self==value.", + "_interpreters.CrossInterpreterBufferView.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpreters.CrossInterpreterBufferView.__ge__" => "Return self>=value.", + "_interpreters.CrossInterpreterBufferView.__getattribute__" => "Return getattr(self, name).", + "_interpreters.CrossInterpreterBufferView.__getstate__" => "Helper for pickle.", + "_interpreters.CrossInterpreterBufferView.__gt__" => "Return self>value.", + "_interpreters.CrossInterpreterBufferView.__hash__" => "Return hash(self).", + "_interpreters.CrossInterpreterBufferView.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpreters.CrossInterpreterBufferView.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpreters.CrossInterpreterBufferView.__le__" => "Return self<=value.", + "_interpreters.CrossInterpreterBufferView.__lt__" => "Return self<value.", + "_interpreters.CrossInterpreterBufferView.__ne__" => "Return self!=value.", + "_interpreters.CrossInterpreterBufferView.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpreters.CrossInterpreterBufferView.__reduce__" => "Helper for pickle.", + "_interpreters.CrossInterpreterBufferView.__reduce_ex__" => "Helper for pickle.", + "_interpreters.CrossInterpreterBufferView.__repr__" => "Return repr(self).", + "_interpreters.CrossInterpreterBufferView.__setattr__" => "Implement setattr(self, name, value).", + "_interpreters.CrossInterpreterBufferView.__sizeof__" => "Size of object in memory, in bytes.", + "_interpreters.CrossInterpreterBufferView.__str__" => "Return str(self).", + "_interpreters.CrossInterpreterBufferView.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpreters.InterpreterError" => "A cross-interpreter operation failed", + "_interpreters.InterpreterError.__delattr__" => "Implement delattr(self, name).", + "_interpreters.InterpreterError.__eq__" => "Return self==value.", + "_interpreters.InterpreterError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpreters.InterpreterError.__ge__" => "Return self>=value.", + "_interpreters.InterpreterError.__getattribute__" => "Return getattr(self, name).", + "_interpreters.InterpreterError.__getstate__" => "Helper for pickle.", + "_interpreters.InterpreterError.__gt__" => "Return self>value.", + "_interpreters.InterpreterError.__hash__" => "Return hash(self).", + "_interpreters.InterpreterError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpreters.InterpreterError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpreters.InterpreterError.__le__" => "Return self<=value.", + "_interpreters.InterpreterError.__lt__" => "Return self<value.", + "_interpreters.InterpreterError.__ne__" => "Return self!=value.", + "_interpreters.InterpreterError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpreters.InterpreterError.__reduce_ex__" => "Helper for pickle.", + "_interpreters.InterpreterError.__repr__" => "Return repr(self).", + "_interpreters.InterpreterError.__setattr__" => "Implement setattr(self, name, value).", + "_interpreters.InterpreterError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpreters.InterpreterError.__str__" => "Return str(self).", + "_interpreters.InterpreterError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpreters.InterpreterError.add_note" => "Add a note to the exception", + "_interpreters.InterpreterError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpreters.InterpreterNotFoundError" => "An interpreter was not found", + "_interpreters.InterpreterNotFoundError.__delattr__" => "Implement delattr(self, name).", + "_interpreters.InterpreterNotFoundError.__eq__" => "Return self==value.", + "_interpreters.InterpreterNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpreters.InterpreterNotFoundError.__ge__" => "Return self>=value.", + "_interpreters.InterpreterNotFoundError.__getattribute__" => "Return getattr(self, name).", + "_interpreters.InterpreterNotFoundError.__getstate__" => "Helper for pickle.", + "_interpreters.InterpreterNotFoundError.__gt__" => "Return self>value.", + "_interpreters.InterpreterNotFoundError.__hash__" => "Return hash(self).", + "_interpreters.InterpreterNotFoundError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpreters.InterpreterNotFoundError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpreters.InterpreterNotFoundError.__le__" => "Return self<=value.", + "_interpreters.InterpreterNotFoundError.__lt__" => "Return self<value.", + "_interpreters.InterpreterNotFoundError.__ne__" => "Return self!=value.", + "_interpreters.InterpreterNotFoundError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpreters.InterpreterNotFoundError.__reduce_ex__" => "Helper for pickle.", + "_interpreters.InterpreterNotFoundError.__repr__" => "Return repr(self).", + "_interpreters.InterpreterNotFoundError.__setattr__" => "Implement setattr(self, name, value).", + "_interpreters.InterpreterNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpreters.InterpreterNotFoundError.__str__" => "Return str(self).", + "_interpreters.InterpreterNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpreters.InterpreterNotFoundError.add_note" => "Add a note to the exception", + "_interpreters.InterpreterNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpreters.NotShareableError.__delattr__" => "Implement delattr(self, name).", + "_interpreters.NotShareableError.__eq__" => "Return self==value.", + "_interpreters.NotShareableError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_interpreters.NotShareableError.__ge__" => "Return self>=value.", + "_interpreters.NotShareableError.__getattribute__" => "Return getattr(self, name).", + "_interpreters.NotShareableError.__getstate__" => "Helper for pickle.", + "_interpreters.NotShareableError.__gt__" => "Return self>value.", + "_interpreters.NotShareableError.__hash__" => "Return hash(self).", + "_interpreters.NotShareableError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_interpreters.NotShareableError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_interpreters.NotShareableError.__le__" => "Return self<=value.", + "_interpreters.NotShareableError.__lt__" => "Return self<value.", + "_interpreters.NotShareableError.__ne__" => "Return self!=value.", + "_interpreters.NotShareableError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_interpreters.NotShareableError.__reduce_ex__" => "Helper for pickle.", + "_interpreters.NotShareableError.__repr__" => "Return repr(self).", + "_interpreters.NotShareableError.__setattr__" => "Implement setattr(self, name, value).", + "_interpreters.NotShareableError.__sizeof__" => "Size of object in memory, in bytes.", + "_interpreters.NotShareableError.__str__" => "Return str(self).", + "_interpreters.NotShareableError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_interpreters.NotShareableError.__weakref__" => "list of weak references to the object", + "_interpreters.NotShareableError.add_note" => "Add a note to the exception", + "_interpreters.NotShareableError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_interpreters.call" => "call(id, callable, args=None, kwargs=None, *, restrict=False)\n\nCall the provided object in the identified interpreter.\nPass the given args and kwargs, if possible.", + "_interpreters.capture_exception" => "capture_exception(exc=None) -> types.SimpleNamespace\n\nReturn a snapshot of an exception. If \"exc\" is None\nthen the current exception, if any, is used (but not cleared).\n\nThe returned snapshot is the same as what _interpreters.exec() returns.", + "_interpreters.create" => "create([config], *, reqrefs=False) -> ID\n\nCreate a new interpreter and return a unique generated ID.\n\nThe caller is responsible for destroying the interpreter before exiting,\ntypically by using _interpreters.destroy(). This can be managed \nautomatically by passing \"reqrefs=True\" and then using _incref() and\n_decref() appropriately.\n\n\"config\" must be a valid interpreter config or the name of a\npredefined config (\"isolated\" or \"legacy\"). The default\nis \"isolated\".", + "_interpreters.destroy" => "destroy(id, *, restrict=False)\n\nDestroy the identified interpreter.\n\nAttempting to destroy the current interpreter raises InterpreterError.\nSo does an unrecognized ID.", + "_interpreters.exec" => "exec(id, code, shared=None, *, restrict=False)\n\nExecute the provided code in the identified interpreter.\nThis is equivalent to running the builtin exec() under the target\ninterpreter, using the __dict__ of its __main__ module as both\nglobals and locals.\n\n\"code\" may be a string containing the text of a Python script.\n\nFunctions (and code objects) are also supported, with some restrictions.\nThe code/function must not take any arguments or be a closure\n(i.e. have cell vars). Methods and other callables are not supported.\n\nIf a function is provided, its code object is used and all its state\nis ignored, including its __globals__ dict.", + "_interpreters.get_config" => "get_config(id, *, restrict=False) -> types.SimpleNamespace\n\nReturn a representation of the config used to initialize the interpreter.", + "_interpreters.get_current" => "get_current() -> (ID, whence)\n\nReturn the ID of current interpreter.", + "_interpreters.get_main" => "get_main() -> (ID, whence)\n\nReturn the ID of main interpreter.", + "_interpreters.is_running" => "is_running(id, *, restrict=False) -> bool\n\nReturn whether or not the identified interpreter is running.", + "_interpreters.is_shareable" => "is_shareable(obj) -> bool\n\nReturn True if the object's data may be shared between interpreters and\nFalse otherwise.", + "_interpreters.list_all" => "list_all() -> [(ID, whence)]\n\nReturn a list containing the ID of every existing interpreter.", + "_interpreters.new_config" => "new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\nReturn a representation of a new PyInterpreterConfig.\n\nThe name determines the initial values of the config. Supported named\nconfigs are: default, isolated, legacy, and empty.\n\nAny keyword arguments are set on the corresponding config fields,\noverriding the initial values.", + "_interpreters.run_func" => "run_func(id, func, shared=None, *, restrict=False)\n\nExecute the body of the provided function in the identified interpreter.\nCode objects are also supported. In both cases, closures and args\nare not supported. Methods and other callables are not supported either.\n\n(See _interpreters.exec().", + "_interpreters.run_string" => "run_string(id, script, shared=None, *, restrict=False)\n\nExecute the provided string in the identified interpreter.\n\n(See _interpreters.exec().", + "_interpreters.set___main___attrs" => "set___main___attrs(id, ns, *, restrict=False)\n\nBind the given attributes in the interpreter's __main__ module.", + "_interpreters.whence" => "whence(id) -> int\n\nReturn an identifier for where the interpreter was created.", + "_io" => "The io module provides the Python interfaces to stream handling. The\nbuiltin open function is defined in this module.\n\nAt the top of the I/O hierarchy is the abstract base class IOBase. It\ndefines the basic interface to a stream. Note, however, that there is no\nseparation between reading and writing to streams; implementations are\nallowed to raise an OSError if they do not support a given operation.\n\nExtending IOBase is RawIOBase which deals simply with the reading and\nwriting of raw bytes to a stream. FileIO subclasses RawIOBase to provide\nan interface to OS files.\n\nBufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its\nsubclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer\nstreams that are readable, writable, and both respectively.\nBufferedRandom provides a buffered interface to random access\nstreams. BytesIO is a simple stream of in-memory bytes.\n\nAnother IOBase subclass, TextIOBase, deals with the encoding and decoding\nof streams into text. TextIOWrapper, which extends it, is a buffered text\ninterface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO\nis an in-memory stream for text.\n\nArgument names are not part of the specification, and only the arguments\nof open() are intended to be used as keyword arguments.\n\ndata:\n\nDEFAULT_BUFFER_SIZE\n\n An int containing the default buffer size used by the module's buffered\n I/O classes.", + "_io.BlockingIOError" => "I/O operation would block.", + "_io.BlockingIOError.__delattr__" => "Implement delattr(self, name).", + "_io.BlockingIOError.__eq__" => "Return self==value.", + "_io.BlockingIOError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.BlockingIOError.__ge__" => "Return self>=value.", + "_io.BlockingIOError.__getattribute__" => "Return getattr(self, name).", + "_io.BlockingIOError.__getstate__" => "Helper for pickle.", + "_io.BlockingIOError.__gt__" => "Return self>value.", + "_io.BlockingIOError.__hash__" => "Return hash(self).", + "_io.BlockingIOError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.BlockingIOError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.BlockingIOError.__le__" => "Return self<=value.", + "_io.BlockingIOError.__lt__" => "Return self<value.", + "_io.BlockingIOError.__ne__" => "Return self!=value.", + "_io.BlockingIOError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.BlockingIOError.__reduce_ex__" => "Helper for pickle.", + "_io.BlockingIOError.__repr__" => "Return repr(self).", + "_io.BlockingIOError.__setattr__" => "Implement setattr(self, name, value).", + "_io.BlockingIOError.__sizeof__" => "Size of object in memory, in bytes.", + "_io.BlockingIOError.__str__" => "Return str(self).", + "_io.BlockingIOError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.BlockingIOError.add_note" => "Add a note to the exception", + "_io.BlockingIOError.errno" => "POSIX exception code", + "_io.BlockingIOError.filename" => "exception filename", + "_io.BlockingIOError.filename2" => "second exception filename", + "_io.BlockingIOError.strerror" => "exception strerror", + "_io.BlockingIOError.winerror" => "Win32 exception code", + "_io.BlockingIOError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_io.BufferedRWPair" => "A buffered reader and writer object together.\n\nA buffered reader object and buffered writer object put together to\nform a sequential IO object that can read and write. This is typically\nused with a socket or two-way pipe.\n\nreader and writer are RawIOBase objects that are readable and\nwriteable respectively. If the buffer_size is omitted it defaults to\nDEFAULT_BUFFER_SIZE.", + "_io.BufferedRWPair.__del__" => "Called when the instance is about to be destroyed.", + "_io.BufferedRWPair.__delattr__" => "Implement delattr(self, name).", + "_io.BufferedRWPair.__eq__" => "Return self==value.", + "_io.BufferedRWPair.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.BufferedRWPair.__ge__" => "Return self>=value.", + "_io.BufferedRWPair.__getattribute__" => "Return getattr(self, name).", + "_io.BufferedRWPair.__getstate__" => "Helper for pickle.", + "_io.BufferedRWPair.__gt__" => "Return self>value.", + "_io.BufferedRWPair.__hash__" => "Return hash(self).", + "_io.BufferedRWPair.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.BufferedRWPair.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.BufferedRWPair.__iter__" => "Implement iter(self).", + "_io.BufferedRWPair.__le__" => "Return self<=value.", + "_io.BufferedRWPair.__lt__" => "Return self<value.", + "_io.BufferedRWPair.__ne__" => "Return self!=value.", + "_io.BufferedRWPair.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.BufferedRWPair.__next__" => "Implement next(self).", + "_io.BufferedRWPair.__reduce__" => "Helper for pickle.", + "_io.BufferedRWPair.__reduce_ex__" => "Helper for pickle.", + "_io.BufferedRWPair.__repr__" => "Return repr(self).", + "_io.BufferedRWPair.__setattr__" => "Implement setattr(self, name, value).", + "_io.BufferedRWPair.__sizeof__" => "Size of object in memory, in bytes.", + "_io.BufferedRWPair.__str__" => "Return str(self).", + "_io.BufferedRWPair.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.BufferedRWPair.detach" => "Disconnect this buffer from its underlying raw stream and return it.\n\nAfter the raw stream has been detached, the buffer is in an unusable\nstate.", + "_io.BufferedRWPair.fileno" => "Return underlying file descriptor if one exists.\n\nRaise OSError if the IO object does not use a file descriptor.", + "_io.BufferedRWPair.readline" => "Read and return a line from the stream.\n\nIf size is specified, at most size bytes will be read.\n\nThe line terminator is always b'\\n' for binary files; for text\nfiles, the newlines argument to open can be used to select the line\nterminator(s) recognized.", + "_io.BufferedRWPair.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io.BufferedRWPair.seek" => "Change the stream position to the given byte offset.\n\n offset\n The stream position, relative to 'whence'.\n whence\n The relative position to seek from.\n\nThe offset is interpreted relative to the position indicated by whence.\nValues for whence are:\n\n* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n* os.SEEK_END or 2 -- end of stream; offset is usually negative\n\nReturn the new absolute position.", + "_io.BufferedRWPair.seekable" => "Return whether object supports random access.\n\nIf False, seek(), tell() and truncate() will raise OSError.\nThis method may need to do a test seek().", + "_io.BufferedRWPair.tell" => "Return current stream position.", + "_io.BufferedRWPair.truncate" => "Truncate file to size bytes.\n\nFile pointer is left unchanged. Size defaults to the current IO position\nas reported by tell(). Return the new size.", + "_io.BufferedRWPair.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.BufferedRandom" => "A buffered interface to random access streams.\n\nThe constructor creates a reader and writer for a seekable stream,\nraw, given in the first argument. If the buffer_size is omitted it\ndefaults to DEFAULT_BUFFER_SIZE.", + "_io.BufferedRandom.__del__" => "Called when the instance is about to be destroyed.", + "_io.BufferedRandom.__delattr__" => "Implement delattr(self, name).", + "_io.BufferedRandom.__eq__" => "Return self==value.", + "_io.BufferedRandom.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.BufferedRandom.__ge__" => "Return self>=value.", + "_io.BufferedRandom.__getattribute__" => "Return getattr(self, name).", + "_io.BufferedRandom.__gt__" => "Return self>value.", + "_io.BufferedRandom.__hash__" => "Return hash(self).", + "_io.BufferedRandom.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.BufferedRandom.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.BufferedRandom.__iter__" => "Implement iter(self).", + "_io.BufferedRandom.__le__" => "Return self<=value.", + "_io.BufferedRandom.__lt__" => "Return self<value.", + "_io.BufferedRandom.__ne__" => "Return self!=value.", + "_io.BufferedRandom.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.BufferedRandom.__next__" => "Implement next(self).", + "_io.BufferedRandom.__reduce__" => "Helper for pickle.", + "_io.BufferedRandom.__reduce_ex__" => "Helper for pickle.", + "_io.BufferedRandom.__repr__" => "Return repr(self).", + "_io.BufferedRandom.__setattr__" => "Implement setattr(self, name, value).", + "_io.BufferedRandom.__str__" => "Return str(self).", + "_io.BufferedRandom.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.BufferedRandom.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io.BufferedRandom.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.BufferedReader" => "Create a new buffered reader using the given readable raw IO object.", + "_io.BufferedReader.__del__" => "Called when the instance is about to be destroyed.", + "_io.BufferedReader.__delattr__" => "Implement delattr(self, name).", + "_io.BufferedReader.__eq__" => "Return self==value.", + "_io.BufferedReader.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.BufferedReader.__ge__" => "Return self>=value.", + "_io.BufferedReader.__getattribute__" => "Return getattr(self, name).", + "_io.BufferedReader.__gt__" => "Return self>value.", + "_io.BufferedReader.__hash__" => "Return hash(self).", + "_io.BufferedReader.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.BufferedReader.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.BufferedReader.__iter__" => "Implement iter(self).", + "_io.BufferedReader.__le__" => "Return self<=value.", + "_io.BufferedReader.__lt__" => "Return self<value.", + "_io.BufferedReader.__ne__" => "Return self!=value.", + "_io.BufferedReader.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.BufferedReader.__next__" => "Implement next(self).", + "_io.BufferedReader.__reduce__" => "Helper for pickle.", + "_io.BufferedReader.__reduce_ex__" => "Helper for pickle.", + "_io.BufferedReader.__repr__" => "Return repr(self).", + "_io.BufferedReader.__setattr__" => "Implement setattr(self, name, value).", + "_io.BufferedReader.__str__" => "Return str(self).", + "_io.BufferedReader.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.BufferedReader.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io.BufferedReader.writable" => "Return whether object was opened for writing.\n\nIf False, write() will raise OSError.", + "_io.BufferedReader.write" => "Write buffer b to the IO stream.\n\nReturn the number of bytes written, which is always\nthe length of b in bytes.\n\nRaise BlockingIOError if the buffer is full and the\nunderlying raw stream cannot accept more data at the moment.", + "_io.BufferedReader.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.BufferedWriter" => "A buffer for a writeable sequential RawIO object.\n\nThe constructor creates a BufferedWriter for the given writeable raw\nstream. If the buffer_size is not given, it defaults to\nDEFAULT_BUFFER_SIZE.", + "_io.BufferedWriter.__del__" => "Called when the instance is about to be destroyed.", + "_io.BufferedWriter.__delattr__" => "Implement delattr(self, name).", + "_io.BufferedWriter.__eq__" => "Return self==value.", + "_io.BufferedWriter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.BufferedWriter.__ge__" => "Return self>=value.", + "_io.BufferedWriter.__getattribute__" => "Return getattr(self, name).", + "_io.BufferedWriter.__gt__" => "Return self>value.", + "_io.BufferedWriter.__hash__" => "Return hash(self).", + "_io.BufferedWriter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.BufferedWriter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.BufferedWriter.__iter__" => "Implement iter(self).", + "_io.BufferedWriter.__le__" => "Return self<=value.", + "_io.BufferedWriter.__lt__" => "Return self<value.", + "_io.BufferedWriter.__ne__" => "Return self!=value.", + "_io.BufferedWriter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.BufferedWriter.__next__" => "Implement next(self).", + "_io.BufferedWriter.__reduce__" => "Helper for pickle.", + "_io.BufferedWriter.__reduce_ex__" => "Helper for pickle.", + "_io.BufferedWriter.__repr__" => "Return repr(self).", + "_io.BufferedWriter.__setattr__" => "Implement setattr(self, name, value).", + "_io.BufferedWriter.__str__" => "Return str(self).", + "_io.BufferedWriter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.BufferedWriter.read" => "Read and return up to n bytes.\n\nIf the size argument is omitted, None, or negative, read and\nreturn all data until EOF.\n\nIf the size argument is positive, and the underlying raw stream is\nnot 'interactive', multiple raw reads may be issued to satisfy\nthe byte count (unless EOF is reached first).\nHowever, for interactive raw streams (as well as sockets and pipes),\nat most one raw read will be issued, and a short result does not\nimply that EOF is imminent.\n\nReturn an empty bytes object on EOF.\n\nReturn None if the underlying raw stream was open in non-blocking\nmode and no data is available at the moment.", + "_io.BufferedWriter.read1" => "Read and return up to size bytes, with at most one read() call to the underlying raw stream.\n\nReturn an empty bytes object on EOF.\nA short result does not imply that EOF is imminent.", + "_io.BufferedWriter.readable" => "Return whether object was opened for reading.\n\nIf False, read() will raise OSError.", + "_io.BufferedWriter.readline" => "Read and return a line from the stream.\n\nIf size is specified, at most size bytes will be read.\n\nThe line terminator is always b'\\n' for binary files; for text\nfiles, the newlines argument to open can be used to select the line\nterminator(s) recognized.", + "_io.BufferedWriter.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io.BufferedWriter.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.BytesIO" => "Buffered I/O implementation using an in-memory bytes buffer.", + "_io.BytesIO.__del__" => "Called when the instance is about to be destroyed.", + "_io.BytesIO.__delattr__" => "Implement delattr(self, name).", + "_io.BytesIO.__eq__" => "Return self==value.", + "_io.BytesIO.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.BytesIO.__ge__" => "Return self>=value.", + "_io.BytesIO.__getattribute__" => "Return getattr(self, name).", + "_io.BytesIO.__gt__" => "Return self>value.", + "_io.BytesIO.__hash__" => "Return hash(self).", + "_io.BytesIO.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.BytesIO.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.BytesIO.__iter__" => "Implement iter(self).", + "_io.BytesIO.__le__" => "Return self<=value.", + "_io.BytesIO.__lt__" => "Return self<value.", + "_io.BytesIO.__ne__" => "Return self!=value.", + "_io.BytesIO.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.BytesIO.__next__" => "Implement next(self).", + "_io.BytesIO.__reduce__" => "Helper for pickle.", + "_io.BytesIO.__reduce_ex__" => "Helper for pickle.", + "_io.BytesIO.__repr__" => "Return repr(self).", + "_io.BytesIO.__setattr__" => "Implement setattr(self, name, value).", + "_io.BytesIO.__str__" => "Return str(self).", + "_io.BytesIO.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.BytesIO.close" => "Disable all I/O operations.", + "_io.BytesIO.closed" => "True if the file is closed.", + "_io.BytesIO.detach" => "Disconnect this buffer from its underlying raw stream and return it.\n\nAfter the raw stream has been detached, the buffer is in an unusable\nstate.", + "_io.BytesIO.fileno" => "Return underlying file descriptor if one exists.\n\nRaise OSError if the IO object does not use a file descriptor.", + "_io.BytesIO.flush" => "Does nothing.", + "_io.BytesIO.getbuffer" => "Get a read-write view over the contents of the BytesIO object.", + "_io.BytesIO.getvalue" => "Retrieve the entire contents of the BytesIO object.", + "_io.BytesIO.isatty" => "Always returns False.\n\nBytesIO objects are not connected to a TTY-like device.", + "_io.BytesIO.read" => "Read at most size bytes, returned as a bytes object.\n\nIf the size argument is negative, read until EOF is reached.\nReturn an empty bytes object at EOF.", + "_io.BytesIO.read1" => "Read at most size bytes, returned as a bytes object.\n\nIf the size argument is negative or omitted, read until EOF is reached.\nReturn an empty bytes object at EOF.", + "_io.BytesIO.readable" => "Returns True if the IO object can be read.", + "_io.BytesIO.readinto" => "Read bytes into buffer.\n\nReturns number of bytes read (0 for EOF), or None if the object\nis set not to block and has no data to read.", + "_io.BytesIO.readline" => "Next line from the file, as a bytes object.\n\nRetain newline. A non-negative size argument limits the maximum\nnumber of bytes to return (an incomplete line may be returned then).\nReturn an empty bytes object at EOF.", + "_io.BytesIO.readlines" => "List of bytes objects, each a line from the file.\n\nCall readline() repeatedly and return a list of the lines so read.\nThe optional size argument, if given, is an approximate bound on the\ntotal number of bytes in the lines returned.", + "_io.BytesIO.seek" => "Change stream position.\n\nSeek to byte offset pos relative to position indicated by whence:\n 0 Start of stream (the default). pos should be >= 0;\n 1 Current position - pos may be negative;\n 2 End of stream - pos usually negative.\nReturns the new absolute position.", + "_io.BytesIO.seekable" => "Returns True if the IO object can be seeked.", + "_io.BytesIO.tell" => "Current file position, an integer.", + "_io.BytesIO.truncate" => "Truncate the file to at most size bytes.\n\nSize defaults to the current file position, as returned by tell().\nThe current file position is unchanged. Returns the new size.", + "_io.BytesIO.writable" => "Returns True if the IO object can be written.", + "_io.BytesIO.write" => "Write bytes to file.\n\nReturn the number of bytes written.", + "_io.BytesIO.writelines" => "Write lines to the file.\n\nNote that newlines are not added. lines can be any iterable object\nproducing bytes-like objects. This is equivalent to calling write() for\neach element.", + "_io.FileIO" => "Open a file.\n\nThe mode can be 'r' (default), 'w', 'x' or 'a' for reading,\nwriting, exclusive creation or appending. The file will be created if it\ndoesn't exist when opened for writing or appending; it will be truncated\nwhen opened for writing. A FileExistsError will be raised if it already\nexists when opened for creating. Opening a file for creating implies\nwriting so this mode behaves in a similar way to 'w'.Add a '+' to the mode\nto allow simultaneous reading and writing. A custom opener can be used by\npassing a callable as *opener*. The underlying file descriptor for the file\nobject is then obtained by calling opener with (*name*, *flags*).\n*opener* must return an open file descriptor (passing os.open as *opener*\nresults in functionality similar to passing None).", + "_io.FileIO.__del__" => "Called when the instance is about to be destroyed.", + "_io.FileIO.__delattr__" => "Implement delattr(self, name).", + "_io.FileIO.__eq__" => "Return self==value.", + "_io.FileIO.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.FileIO.__ge__" => "Return self>=value.", + "_io.FileIO.__getattribute__" => "Return getattr(self, name).", + "_io.FileIO.__gt__" => "Return self>value.", + "_io.FileIO.__hash__" => "Return hash(self).", + "_io.FileIO.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.FileIO.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.FileIO.__iter__" => "Implement iter(self).", + "_io.FileIO.__le__" => "Return self<=value.", + "_io.FileIO.__lt__" => "Return self<value.", + "_io.FileIO.__ne__" => "Return self!=value.", + "_io.FileIO.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.FileIO.__next__" => "Implement next(self).", + "_io.FileIO.__reduce__" => "Helper for pickle.", + "_io.FileIO.__reduce_ex__" => "Helper for pickle.", + "_io.FileIO.__repr__" => "Return repr(self).", + "_io.FileIO.__setattr__" => "Implement setattr(self, name, value).", + "_io.FileIO.__sizeof__" => "Size of object in memory, in bytes.", + "_io.FileIO.__str__" => "Return str(self).", + "_io.FileIO.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.FileIO._blksize" => "Stat st_blksize if available", + "_io.FileIO.close" => "Close the file.\n\nA closed file cannot be used for further I/O operations. close() may be\ncalled more than once without error.", + "_io.FileIO.closed" => "True if the file is closed", + "_io.FileIO.closefd" => "True if the file descriptor will be closed by close().", + "_io.FileIO.fileno" => "Return the underlying file descriptor (an integer).", + "_io.FileIO.flush" => "Flush write buffers, if applicable.\n\nThis is not implemented for read-only and non-blocking streams.", + "_io.FileIO.isatty" => "True if the file is connected to a TTY device.", + "_io.FileIO.mode" => "String giving the file mode", + "_io.FileIO.read" => "Read at most size bytes, returned as bytes.\n\nIf size is less than 0, read all bytes in the file making multiple read calls.\nSee ``FileIO.readall``.\n\nAttempts to make only one system call, retrying only per PEP 475 (EINTR). This\nmeans less data may be returned than requested.\n\nIn non-blocking mode, returns None if no data is available. Return an empty\nbytes object at EOF.", + "_io.FileIO.readable" => "True if file was opened in a read mode.", + "_io.FileIO.readall" => "Read all data from the file, returned as bytes.\n\nReads until either there is an error or read() returns size 0 (indicates EOF).\nIf the file is already at EOF, returns an empty bytes object.\n\nIn non-blocking mode, returns as much data as could be read before EAGAIN. If no\ndata is available (EAGAIN is returned before bytes are read) returns None.", + "_io.FileIO.readinto" => "Same as RawIOBase.readinto().", + "_io.FileIO.readline" => "Read and return a line from the stream.\n\nIf size is specified, at most size bytes will be read.\n\nThe line terminator is always b'\\n' for binary files; for text\nfiles, the newlines argument to open can be used to select the line\nterminator(s) recognized.", + "_io.FileIO.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io.FileIO.seek" => "Move to new file position and return the file position.\n\nArgument offset is a byte count. Optional argument whence defaults to\nSEEK_SET or 0 (offset from start of file, offset should be >= 0); other values\nare SEEK_CUR or 1 (move relative to current position, positive or negative),\nand SEEK_END or 2 (move relative to end of file, usually negative, although\nmany platforms allow seeking beyond the end of a file).\n\nNote that not all file objects are seekable.", + "_io.FileIO.seekable" => "True if file supports random-access.", + "_io.FileIO.tell" => "Current file position.\n\nCan raise OSError for non seekable files.", + "_io.FileIO.truncate" => "Truncate the file to at most size bytes and return the truncated size.\n\nSize defaults to the current file position, as returned by tell().\nThe current file position is changed to the value of size.", + "_io.FileIO.writable" => "True if file was opened in a write mode.", + "_io.FileIO.write" => "Write buffer b to file, return number of bytes written.\n\nOnly makes one system call, so not all of the data may be written.\nThe number of bytes actually written is returned. In non-blocking mode,\nreturns None if the write would block.", + "_io.FileIO.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.IncrementalNewlineDecoder" => "Codec used when reading a file in universal newlines mode.\n\nIt wraps another incremental decoder, translating \\r\\n and \\r into \\n.\nIt also records the types of newlines encountered. When used with\ntranslate=False, it ensures that the newline sequence is returned in\none piece. When used with decoder=None, it expects unicode strings as\ndecode input and translates newlines without first invoking an external\ndecoder.", + "_io.IncrementalNewlineDecoder.__delattr__" => "Implement delattr(self, name).", + "_io.IncrementalNewlineDecoder.__eq__" => "Return self==value.", + "_io.IncrementalNewlineDecoder.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.IncrementalNewlineDecoder.__ge__" => "Return self>=value.", + "_io.IncrementalNewlineDecoder.__getattribute__" => "Return getattr(self, name).", + "_io.IncrementalNewlineDecoder.__getstate__" => "Helper for pickle.", + "_io.IncrementalNewlineDecoder.__gt__" => "Return self>value.", + "_io.IncrementalNewlineDecoder.__hash__" => "Return hash(self).", + "_io.IncrementalNewlineDecoder.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.IncrementalNewlineDecoder.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.IncrementalNewlineDecoder.__le__" => "Return self<=value.", + "_io.IncrementalNewlineDecoder.__lt__" => "Return self<value.", + "_io.IncrementalNewlineDecoder.__ne__" => "Return self!=value.", + "_io.IncrementalNewlineDecoder.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.IncrementalNewlineDecoder.__reduce__" => "Helper for pickle.", + "_io.IncrementalNewlineDecoder.__reduce_ex__" => "Helper for pickle.", + "_io.IncrementalNewlineDecoder.__repr__" => "Return repr(self).", + "_io.IncrementalNewlineDecoder.__setattr__" => "Implement setattr(self, name, value).", + "_io.IncrementalNewlineDecoder.__sizeof__" => "Size of object in memory, in bytes.", + "_io.IncrementalNewlineDecoder.__str__" => "Return str(self).", + "_io.IncrementalNewlineDecoder.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.StringIO" => "Text I/O implementation using an in-memory buffer.\n\nThe initial_value argument sets the value of object. The newline\nargument is like the one of TextIOWrapper's constructor.", + "_io.StringIO.__del__" => "Called when the instance is about to be destroyed.", + "_io.StringIO.__delattr__" => "Implement delattr(self, name).", + "_io.StringIO.__eq__" => "Return self==value.", + "_io.StringIO.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.StringIO.__ge__" => "Return self>=value.", + "_io.StringIO.__getattribute__" => "Return getattr(self, name).", + "_io.StringIO.__gt__" => "Return self>value.", + "_io.StringIO.__hash__" => "Return hash(self).", + "_io.StringIO.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.StringIO.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.StringIO.__iter__" => "Implement iter(self).", + "_io.StringIO.__le__" => "Return self<=value.", + "_io.StringIO.__lt__" => "Return self<value.", + "_io.StringIO.__ne__" => "Return self!=value.", + "_io.StringIO.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.StringIO.__next__" => "Implement next(self).", + "_io.StringIO.__reduce__" => "Helper for pickle.", + "_io.StringIO.__reduce_ex__" => "Helper for pickle.", + "_io.StringIO.__repr__" => "Return repr(self).", + "_io.StringIO.__setattr__" => "Implement setattr(self, name, value).", + "_io.StringIO.__sizeof__" => "Size of object in memory, in bytes.", + "_io.StringIO.__str__" => "Return str(self).", + "_io.StringIO.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.StringIO.close" => "Close the IO object.\n\nAttempting any further operation after the object is closed\nwill raise a ValueError.\n\nThis method has no effect if the file is already closed.", + "_io.StringIO.detach" => "Separate the underlying buffer from the TextIOBase and return it.\n\nAfter the underlying buffer has been detached, the TextIO is in an unusable state.", + "_io.StringIO.encoding" => "Encoding of the text stream.\n\nSubclasses should override.", + "_io.StringIO.errors" => "The error setting of the decoder or encoder.\n\nSubclasses should override.", + "_io.StringIO.fileno" => "Return underlying file descriptor if one exists.\n\nRaise OSError if the IO object does not use a file descriptor.", + "_io.StringIO.flush" => "Flush write buffers, if applicable.\n\nThis is not implemented for read-only and non-blocking streams.", + "_io.StringIO.getvalue" => "Retrieve the entire contents of the object.", + "_io.StringIO.isatty" => "Return whether this is an 'interactive' stream.\n\nReturn False if it can't be determined.", + "_io.StringIO.read" => "Read at most size characters, returned as a string.\n\nIf the argument is negative or omitted, read until EOF\nis reached. Return an empty string at EOF.", + "_io.StringIO.readable" => "Returns True if the IO object can be read.", + "_io.StringIO.readline" => "Read until newline or EOF.\n\nReturns an empty string if EOF is hit immediately.", + "_io.StringIO.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io.StringIO.seek" => "Change stream position.\n\nSeek to character offset pos relative to position indicated by whence:\n 0 Start of stream (the default). pos should be >= 0;\n 1 Current position - pos must be 0;\n 2 End of stream - pos must be 0.\nReturns the new absolute position.", + "_io.StringIO.seekable" => "Returns True if the IO object can be seeked.", + "_io.StringIO.tell" => "Tell the current file position.", + "_io.StringIO.truncate" => "Truncate size to pos.\n\nThe pos argument defaults to the current file position, as\nreturned by tell(). The current file position is unchanged.\nReturns the new absolute position.", + "_io.StringIO.writable" => "Returns True if the IO object can be written.", + "_io.StringIO.write" => "Write string to file.\n\nReturns the number of characters written, which is always equal to\nthe length of the string.", + "_io.StringIO.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.TextIOWrapper" => "Character and line based layer over a BufferedIOBase object, buffer.\n\nencoding gives the name of the encoding that the stream will be\ndecoded or encoded with. It defaults to locale.getencoding().\n\nerrors determines the strictness of encoding and decoding (see\nhelp(codecs.Codec) or the documentation for codecs.register) and\ndefaults to \"strict\".\n\nnewline controls how line endings are handled. It can be None, '',\n'\\n', '\\r', and '\\r\\n'. It works as follows:\n\n* On input, if newline is None, universal newlines mode is\n enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n these are translated into '\\n' before being returned to the\n caller. If it is '', universal newline mode is enabled, but line\n endings are returned to the caller untranslated. If it has any of\n the other legal values, input lines are only terminated by the given\n string, and the line ending is returned to the caller untranslated.\n\n* On output, if newline is None, any '\\n' characters written are\n translated to the system default line separator, os.linesep. If\n newline is '' or '\\n', no translation takes place. If newline is any\n of the other legal values, any '\\n' characters written are translated\n to the given string.\n\nIf line_buffering is True, a call to flush is implied when a call to\nwrite contains a newline character.", + "_io.TextIOWrapper.__del__" => "Called when the instance is about to be destroyed.", + "_io.TextIOWrapper.__delattr__" => "Implement delattr(self, name).", + "_io.TextIOWrapper.__eq__" => "Return self==value.", + "_io.TextIOWrapper.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.TextIOWrapper.__ge__" => "Return self>=value.", + "_io.TextIOWrapper.__getattribute__" => "Return getattr(self, name).", + "_io.TextIOWrapper.__gt__" => "Return self>value.", + "_io.TextIOWrapper.__hash__" => "Return hash(self).", + "_io.TextIOWrapper.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.TextIOWrapper.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.TextIOWrapper.__iter__" => "Implement iter(self).", + "_io.TextIOWrapper.__le__" => "Return self<=value.", + "_io.TextIOWrapper.__lt__" => "Return self<value.", + "_io.TextIOWrapper.__ne__" => "Return self!=value.", + "_io.TextIOWrapper.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.TextIOWrapper.__next__" => "Implement next(self).", + "_io.TextIOWrapper.__reduce__" => "Helper for pickle.", + "_io.TextIOWrapper.__reduce_ex__" => "Helper for pickle.", + "_io.TextIOWrapper.__repr__" => "Return repr(self).", + "_io.TextIOWrapper.__setattr__" => "Implement setattr(self, name, value).", + "_io.TextIOWrapper.__sizeof__" => "Size of object in memory, in bytes.", + "_io.TextIOWrapper.__str__" => "Return str(self).", + "_io.TextIOWrapper.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.TextIOWrapper.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io.TextIOWrapper.reconfigure" => "Reconfigure the text stream with new parameters.\n\nThis also does an implicit stream flush.", + "_io.TextIOWrapper.seek" => "Set the stream position, and return the new stream position.\n\n cookie\n Zero or an opaque number returned by tell().\n whence\n The relative position to seek from.\n\nFour operations are supported, given by the following argument\ncombinations:\n\n- seek(0, SEEK_SET): Rewind to the start of the stream.\n- seek(cookie, SEEK_SET): Restore a previous position;\n 'cookie' must be a number returned by tell().\n- seek(0, SEEK_END): Fast-forward to the end of the stream.\n- seek(0, SEEK_CUR): Leave the current stream position unchanged.\n\nAny other argument combinations are invalid,\nand may raise exceptions.", + "_io.TextIOWrapper.tell" => "Return the stream position as an opaque number.\n\nThe return value of tell() can be given as input to seek(), to restore a\nprevious stream position.", + "_io.TextIOWrapper.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.UnsupportedOperation.__delattr__" => "Implement delattr(self, name).", + "_io.UnsupportedOperation.__eq__" => "Return self==value.", + "_io.UnsupportedOperation.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io.UnsupportedOperation.__ge__" => "Return self>=value.", + "_io.UnsupportedOperation.__getattribute__" => "Return getattr(self, name).", + "_io.UnsupportedOperation.__getstate__" => "Helper for pickle.", + "_io.UnsupportedOperation.__gt__" => "Return self>value.", + "_io.UnsupportedOperation.__hash__" => "Return hash(self).", + "_io.UnsupportedOperation.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io.UnsupportedOperation.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io.UnsupportedOperation.__le__" => "Return self<=value.", + "_io.UnsupportedOperation.__lt__" => "Return self<value.", + "_io.UnsupportedOperation.__ne__" => "Return self!=value.", + "_io.UnsupportedOperation.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io.UnsupportedOperation.__reduce_ex__" => "Helper for pickle.", + "_io.UnsupportedOperation.__repr__" => "Return repr(self).", + "_io.UnsupportedOperation.__setattr__" => "Implement setattr(self, name, value).", + "_io.UnsupportedOperation.__sizeof__" => "Size of object in memory, in bytes.", + "_io.UnsupportedOperation.__str__" => "Return str(self).", + "_io.UnsupportedOperation.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io.UnsupportedOperation.__weakref__" => "list of weak references to the object", + "_io.UnsupportedOperation.add_note" => "Add a note to the exception", + "_io.UnsupportedOperation.errno" => "POSIX exception code", + "_io.UnsupportedOperation.filename" => "exception filename", + "_io.UnsupportedOperation.filename2" => "second exception filename", + "_io.UnsupportedOperation.strerror" => "exception strerror", + "_io.UnsupportedOperation.winerror" => "Win32 exception code", + "_io.UnsupportedOperation.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_io._BufferedIOBase" => "Base class for buffered IO objects.\n\nThe main difference with RawIOBase is that the read() method\nsupports omitting the size argument, and does not have a default\nimplementation that defers to readinto().\n\nIn addition, read(), readinto() and write() may raise\nBlockingIOError if the underlying raw stream is in non-blocking\nmode and not ready; unlike their raw counterparts, they will never\nreturn None.\n\nA typical implementation should not inherit from a RawIOBase\nimplementation, but wrap one.", + "_io._BufferedIOBase.__del__" => "Called when the instance is about to be destroyed.", + "_io._BufferedIOBase.__delattr__" => "Implement delattr(self, name).", + "_io._BufferedIOBase.__eq__" => "Return self==value.", + "_io._BufferedIOBase.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io._BufferedIOBase.__ge__" => "Return self>=value.", + "_io._BufferedIOBase.__getattribute__" => "Return getattr(self, name).", + "_io._BufferedIOBase.__getstate__" => "Helper for pickle.", + "_io._BufferedIOBase.__gt__" => "Return self>value.", + "_io._BufferedIOBase.__hash__" => "Return hash(self).", + "_io._BufferedIOBase.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io._BufferedIOBase.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io._BufferedIOBase.__iter__" => "Implement iter(self).", + "_io._BufferedIOBase.__le__" => "Return self<=value.", + "_io._BufferedIOBase.__lt__" => "Return self<value.", + "_io._BufferedIOBase.__ne__" => "Return self!=value.", + "_io._BufferedIOBase.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io._BufferedIOBase.__next__" => "Implement next(self).", + "_io._BufferedIOBase.__reduce__" => "Helper for pickle.", + "_io._BufferedIOBase.__reduce_ex__" => "Helper for pickle.", + "_io._BufferedIOBase.__repr__" => "Return repr(self).", + "_io._BufferedIOBase.__setattr__" => "Implement setattr(self, name, value).", + "_io._BufferedIOBase.__sizeof__" => "Size of object in memory, in bytes.", + "_io._BufferedIOBase.__str__" => "Return str(self).", + "_io._BufferedIOBase.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io._BufferedIOBase.close" => "Flush and close the IO object.\n\nThis method has no effect if the file is already closed.", + "_io._BufferedIOBase.detach" => "Disconnect this buffer from its underlying raw stream and return it.\n\nAfter the raw stream has been detached, the buffer is in an unusable\nstate.", + "_io._BufferedIOBase.fileno" => "Return underlying file descriptor if one exists.\n\nRaise OSError if the IO object does not use a file descriptor.", + "_io._BufferedIOBase.flush" => "Flush write buffers, if applicable.\n\nThis is not implemented for read-only and non-blocking streams.", + "_io._BufferedIOBase.isatty" => "Return whether this is an 'interactive' stream.\n\nReturn False if it can't be determined.", + "_io._BufferedIOBase.read" => "Read and return up to n bytes.\n\nIf the size argument is omitted, None, or negative, read and\nreturn all data until EOF.\n\nIf the size argument is positive, and the underlying raw stream is\nnot 'interactive', multiple raw reads may be issued to satisfy\nthe byte count (unless EOF is reached first).\nHowever, for interactive raw streams (as well as sockets and pipes),\nat most one raw read will be issued, and a short result does not\nimply that EOF is imminent.\n\nReturn an empty bytes object on EOF.\n\nReturn None if the underlying raw stream was open in non-blocking\nmode and no data is available at the moment.", + "_io._BufferedIOBase.read1" => "Read and return up to size bytes, with at most one read() call to the underlying raw stream.\n\nReturn an empty bytes object on EOF.\nA short result does not imply that EOF is imminent.", + "_io._BufferedIOBase.readable" => "Return whether object was opened for reading.\n\nIf False, read() will raise OSError.", + "_io._BufferedIOBase.readline" => "Read and return a line from the stream.\n\nIf size is specified, at most size bytes will be read.\n\nThe line terminator is always b'\\n' for binary files; for text\nfiles, the newlines argument to open can be used to select the line\nterminator(s) recognized.", + "_io._BufferedIOBase.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io._BufferedIOBase.seek" => "Change the stream position to the given byte offset.\n\n offset\n The stream position, relative to 'whence'.\n whence\n The relative position to seek from.\n\nThe offset is interpreted relative to the position indicated by whence.\nValues for whence are:\n\n* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n* os.SEEK_END or 2 -- end of stream; offset is usually negative\n\nReturn the new absolute position.", + "_io._BufferedIOBase.seekable" => "Return whether object supports random access.\n\nIf False, seek(), tell() and truncate() will raise OSError.\nThis method may need to do a test seek().", + "_io._BufferedIOBase.tell" => "Return current stream position.", + "_io._BufferedIOBase.truncate" => "Truncate file to size bytes.\n\nFile pointer is left unchanged. Size defaults to the current IO position\nas reported by tell(). Return the new size.", + "_io._BufferedIOBase.writable" => "Return whether object was opened for writing.\n\nIf False, write() will raise OSError.", + "_io._BufferedIOBase.write" => "Write buffer b to the IO stream.\n\nReturn the number of bytes written, which is always\nthe length of b in bytes.\n\nRaise BlockingIOError if the buffer is full and the\nunderlying raw stream cannot accept more data at the moment.", + "_io._BufferedIOBase.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io._BytesIOBuffer.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "_io._BytesIOBuffer.__delattr__" => "Implement delattr(self, name).", + "_io._BytesIOBuffer.__eq__" => "Return self==value.", + "_io._BytesIOBuffer.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io._BytesIOBuffer.__ge__" => "Return self>=value.", + "_io._BytesIOBuffer.__getattribute__" => "Return getattr(self, name).", + "_io._BytesIOBuffer.__getstate__" => "Helper for pickle.", + "_io._BytesIOBuffer.__gt__" => "Return self>value.", + "_io._BytesIOBuffer.__hash__" => "Return hash(self).", + "_io._BytesIOBuffer.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io._BytesIOBuffer.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io._BytesIOBuffer.__le__" => "Return self<=value.", + "_io._BytesIOBuffer.__lt__" => "Return self<value.", + "_io._BytesIOBuffer.__ne__" => "Return self!=value.", + "_io._BytesIOBuffer.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io._BytesIOBuffer.__reduce__" => "Helper for pickle.", + "_io._BytesIOBuffer.__reduce_ex__" => "Helper for pickle.", + "_io._BytesIOBuffer.__release_buffer__" => "Release the buffer object that exposes the underlying memory of the object.", + "_io._BytesIOBuffer.__repr__" => "Return repr(self).", + "_io._BytesIOBuffer.__setattr__" => "Implement setattr(self, name, value).", + "_io._BytesIOBuffer.__sizeof__" => "Size of object in memory, in bytes.", + "_io._BytesIOBuffer.__str__" => "Return str(self).", + "_io._BytesIOBuffer.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io._IOBase" => "The abstract base class for all I/O classes.\n\nThis class provides dummy implementations for many methods that\nderived classes can override selectively; the default implementations\nrepresent a file that cannot be read, written or seeked.\n\nEven though IOBase does not declare read, readinto, or write because\ntheir signatures will vary, implementations and clients should\nconsider those methods part of the interface. Also, implementations\nmay raise UnsupportedOperation when operations they do not support are\ncalled.\n\nThe basic type used for binary data read from or written to a file is\nbytes. Other bytes-like objects are accepted as method arguments too.\nIn some cases (such as readinto), a writable object is required. Text\nI/O classes work with str data.\n\nNote that calling any method (except additional calls to close(),\nwhich are ignored) on a closed stream should raise a ValueError.\n\nIOBase (and its subclasses) support the iterator protocol, meaning\nthat an IOBase object can be iterated over yielding the lines in a\nstream.\n\nIOBase also supports the :keyword:`with` statement. In this example,\nfp is closed after the suite of the with statement is complete:\n\nwith open('spam.txt', 'r') as fp:\n fp.write('Spam and eggs!')", + "_io._IOBase.__del__" => "Called when the instance is about to be destroyed.", + "_io._IOBase.__delattr__" => "Implement delattr(self, name).", + "_io._IOBase.__eq__" => "Return self==value.", + "_io._IOBase.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io._IOBase.__ge__" => "Return self>=value.", + "_io._IOBase.__getattribute__" => "Return getattr(self, name).", + "_io._IOBase.__getstate__" => "Helper for pickle.", + "_io._IOBase.__gt__" => "Return self>value.", + "_io._IOBase.__hash__" => "Return hash(self).", + "_io._IOBase.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io._IOBase.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io._IOBase.__iter__" => "Implement iter(self).", + "_io._IOBase.__le__" => "Return self<=value.", + "_io._IOBase.__lt__" => "Return self<value.", + "_io._IOBase.__ne__" => "Return self!=value.", + "_io._IOBase.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io._IOBase.__next__" => "Implement next(self).", + "_io._IOBase.__reduce__" => "Helper for pickle.", + "_io._IOBase.__reduce_ex__" => "Helper for pickle.", + "_io._IOBase.__repr__" => "Return repr(self).", + "_io._IOBase.__setattr__" => "Implement setattr(self, name, value).", + "_io._IOBase.__sizeof__" => "Size of object in memory, in bytes.", + "_io._IOBase.__str__" => "Return str(self).", + "_io._IOBase.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io._IOBase.close" => "Flush and close the IO object.\n\nThis method has no effect if the file is already closed.", + "_io._IOBase.fileno" => "Return underlying file descriptor if one exists.\n\nRaise OSError if the IO object does not use a file descriptor.", + "_io._IOBase.flush" => "Flush write buffers, if applicable.\n\nThis is not implemented for read-only and non-blocking streams.", + "_io._IOBase.isatty" => "Return whether this is an 'interactive' stream.\n\nReturn False if it can't be determined.", + "_io._IOBase.readable" => "Return whether object was opened for reading.\n\nIf False, read() will raise OSError.", + "_io._IOBase.readline" => "Read and return a line from the stream.\n\nIf size is specified, at most size bytes will be read.\n\nThe line terminator is always b'\\n' for binary files; for text\nfiles, the newlines argument to open can be used to select the line\nterminator(s) recognized.", + "_io._IOBase.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io._IOBase.seek" => "Change the stream position to the given byte offset.\n\n offset\n The stream position, relative to 'whence'.\n whence\n The relative position to seek from.\n\nThe offset is interpreted relative to the position indicated by whence.\nValues for whence are:\n\n* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n* os.SEEK_END or 2 -- end of stream; offset is usually negative\n\nReturn the new absolute position.", + "_io._IOBase.seekable" => "Return whether object supports random access.\n\nIf False, seek(), tell() and truncate() will raise OSError.\nThis method may need to do a test seek().", + "_io._IOBase.tell" => "Return current stream position.", + "_io._IOBase.truncate" => "Truncate file to size bytes.\n\nFile pointer is left unchanged. Size defaults to the current IO position\nas reported by tell(). Return the new size.", + "_io._IOBase.writable" => "Return whether object was opened for writing.\n\nIf False, write() will raise OSError.", + "_io._IOBase.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io._RawIOBase" => "Base class for raw binary I/O.", + "_io._RawIOBase.__del__" => "Called when the instance is about to be destroyed.", + "_io._RawIOBase.__delattr__" => "Implement delattr(self, name).", + "_io._RawIOBase.__eq__" => "Return self==value.", + "_io._RawIOBase.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io._RawIOBase.__ge__" => "Return self>=value.", + "_io._RawIOBase.__getattribute__" => "Return getattr(self, name).", + "_io._RawIOBase.__getstate__" => "Helper for pickle.", + "_io._RawIOBase.__gt__" => "Return self>value.", + "_io._RawIOBase.__hash__" => "Return hash(self).", + "_io._RawIOBase.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io._RawIOBase.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io._RawIOBase.__iter__" => "Implement iter(self).", + "_io._RawIOBase.__le__" => "Return self<=value.", + "_io._RawIOBase.__lt__" => "Return self<value.", + "_io._RawIOBase.__ne__" => "Return self!=value.", + "_io._RawIOBase.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io._RawIOBase.__next__" => "Implement next(self).", + "_io._RawIOBase.__reduce__" => "Helper for pickle.", + "_io._RawIOBase.__reduce_ex__" => "Helper for pickle.", + "_io._RawIOBase.__repr__" => "Return repr(self).", + "_io._RawIOBase.__setattr__" => "Implement setattr(self, name, value).", + "_io._RawIOBase.__sizeof__" => "Size of object in memory, in bytes.", + "_io._RawIOBase.__str__" => "Return str(self).", + "_io._RawIOBase.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io._RawIOBase.close" => "Flush and close the IO object.\n\nThis method has no effect if the file is already closed.", + "_io._RawIOBase.fileno" => "Return underlying file descriptor if one exists.\n\nRaise OSError if the IO object does not use a file descriptor.", + "_io._RawIOBase.flush" => "Flush write buffers, if applicable.\n\nThis is not implemented for read-only and non-blocking streams.", + "_io._RawIOBase.isatty" => "Return whether this is an 'interactive' stream.\n\nReturn False if it can't be determined.", + "_io._RawIOBase.readable" => "Return whether object was opened for reading.\n\nIf False, read() will raise OSError.", + "_io._RawIOBase.readall" => "Read until EOF, using multiple read() call.", + "_io._RawIOBase.readline" => "Read and return a line from the stream.\n\nIf size is specified, at most size bytes will be read.\n\nThe line terminator is always b'\\n' for binary files; for text\nfiles, the newlines argument to open can be used to select the line\nterminator(s) recognized.", + "_io._RawIOBase.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io._RawIOBase.seek" => "Change the stream position to the given byte offset.\n\n offset\n The stream position, relative to 'whence'.\n whence\n The relative position to seek from.\n\nThe offset is interpreted relative to the position indicated by whence.\nValues for whence are:\n\n* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n* os.SEEK_END or 2 -- end of stream; offset is usually negative\n\nReturn the new absolute position.", + "_io._RawIOBase.seekable" => "Return whether object supports random access.\n\nIf False, seek(), tell() and truncate() will raise OSError.\nThis method may need to do a test seek().", + "_io._RawIOBase.tell" => "Return current stream position.", + "_io._RawIOBase.truncate" => "Truncate file to size bytes.\n\nFile pointer is left unchanged. Size defaults to the current IO position\nas reported by tell(). Return the new size.", + "_io._RawIOBase.writable" => "Return whether object was opened for writing.\n\nIf False, write() will raise OSError.", + "_io._RawIOBase.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io._TextIOBase" => "Base class for text I/O.\n\nThis class provides a character and line based interface to stream\nI/O. There is no readinto method because Python's character strings\nare immutable.", + "_io._TextIOBase.__del__" => "Called when the instance is about to be destroyed.", + "_io._TextIOBase.__delattr__" => "Implement delattr(self, name).", + "_io._TextIOBase.__eq__" => "Return self==value.", + "_io._TextIOBase.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io._TextIOBase.__ge__" => "Return self>=value.", + "_io._TextIOBase.__getattribute__" => "Return getattr(self, name).", + "_io._TextIOBase.__getstate__" => "Helper for pickle.", + "_io._TextIOBase.__gt__" => "Return self>value.", + "_io._TextIOBase.__hash__" => "Return hash(self).", + "_io._TextIOBase.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io._TextIOBase.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io._TextIOBase.__iter__" => "Implement iter(self).", + "_io._TextIOBase.__le__" => "Return self<=value.", + "_io._TextIOBase.__lt__" => "Return self<value.", + "_io._TextIOBase.__ne__" => "Return self!=value.", + "_io._TextIOBase.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io._TextIOBase.__next__" => "Implement next(self).", + "_io._TextIOBase.__reduce__" => "Helper for pickle.", + "_io._TextIOBase.__reduce_ex__" => "Helper for pickle.", + "_io._TextIOBase.__repr__" => "Return repr(self).", + "_io._TextIOBase.__setattr__" => "Implement setattr(self, name, value).", + "_io._TextIOBase.__sizeof__" => "Size of object in memory, in bytes.", + "_io._TextIOBase.__str__" => "Return str(self).", + "_io._TextIOBase.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io._TextIOBase.close" => "Flush and close the IO object.\n\nThis method has no effect if the file is already closed.", + "_io._TextIOBase.detach" => "Separate the underlying buffer from the TextIOBase and return it.\n\nAfter the underlying buffer has been detached, the TextIO is in an unusable state.", + "_io._TextIOBase.encoding" => "Encoding of the text stream.\n\nSubclasses should override.", + "_io._TextIOBase.errors" => "The error setting of the decoder or encoder.\n\nSubclasses should override.", + "_io._TextIOBase.fileno" => "Return underlying file descriptor if one exists.\n\nRaise OSError if the IO object does not use a file descriptor.", + "_io._TextIOBase.flush" => "Flush write buffers, if applicable.\n\nThis is not implemented for read-only and non-blocking streams.", + "_io._TextIOBase.isatty" => "Return whether this is an 'interactive' stream.\n\nReturn False if it can't be determined.", + "_io._TextIOBase.newlines" => "Line endings translated so far.\n\nOnly line endings translated during reading are considered.\n\nSubclasses should override.", + "_io._TextIOBase.read" => "Read at most size characters from stream.\n\nRead from underlying buffer until we have size characters or we hit EOF.\nIf size is negative or omitted, read until EOF.", + "_io._TextIOBase.readable" => "Return whether object was opened for reading.\n\nIf False, read() will raise OSError.", + "_io._TextIOBase.readline" => "Read until newline or EOF.\n\nReturn an empty string if EOF is hit immediately.\nIf size is specified, at most size characters will be read.", + "_io._TextIOBase.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io._TextIOBase.seek" => "Change the stream position to the given byte offset.\n\n offset\n The stream position, relative to 'whence'.\n whence\n The relative position to seek from.\n\nThe offset is interpreted relative to the position indicated by whence.\nValues for whence are:\n\n* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n* os.SEEK_END or 2 -- end of stream; offset is usually negative\n\nReturn the new absolute position.", + "_io._TextIOBase.seekable" => "Return whether object supports random access.\n\nIf False, seek(), tell() and truncate() will raise OSError.\nThis method may need to do a test seek().", + "_io._TextIOBase.tell" => "Return current stream position.", + "_io._TextIOBase.truncate" => "Truncate file to size bytes.\n\nFile pointer is left unchanged. Size defaults to the current IO position\nas reported by tell(). Return the new size.", + "_io._TextIOBase.writable" => "Return whether object was opened for writing.\n\nIf False, write() will raise OSError.", + "_io._TextIOBase.write" => "Write string s to stream.\n\nReturn the number of characters written\n(which is always equal to the length of the string).", + "_io._TextIOBase.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io._WindowsConsoleIO" => "Open a console buffer by file descriptor.\n\nThe mode can be 'rb' (default), or 'wb' for reading or writing bytes. All\nother mode characters will be ignored. Mode 'b' will be assumed if it is\nomitted. The *opener* parameter is always ignored.", + "_io._WindowsConsoleIO.__del__" => "Called when the instance is about to be destroyed.", + "_io._WindowsConsoleIO.__delattr__" => "Implement delattr(self, name).", + "_io._WindowsConsoleIO.__eq__" => "Return self==value.", + "_io._WindowsConsoleIO.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_io._WindowsConsoleIO.__ge__" => "Return self>=value.", + "_io._WindowsConsoleIO.__getattribute__" => "Return getattr(self, name).", + "_io._WindowsConsoleIO.__getstate__" => "Helper for pickle.", + "_io._WindowsConsoleIO.__gt__" => "Return self>value.", + "_io._WindowsConsoleIO.__hash__" => "Return hash(self).", + "_io._WindowsConsoleIO.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_io._WindowsConsoleIO.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_io._WindowsConsoleIO.__iter__" => "Implement iter(self).", + "_io._WindowsConsoleIO.__le__" => "Return self<=value.", + "_io._WindowsConsoleIO.__lt__" => "Return self<value.", + "_io._WindowsConsoleIO.__ne__" => "Return self!=value.", + "_io._WindowsConsoleIO.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_io._WindowsConsoleIO.__next__" => "Implement next(self).", + "_io._WindowsConsoleIO.__reduce__" => "Helper for pickle.", + "_io._WindowsConsoleIO.__reduce_ex__" => "Helper for pickle.", + "_io._WindowsConsoleIO.__repr__" => "Return repr(self).", + "_io._WindowsConsoleIO.__setattr__" => "Implement setattr(self, name, value).", + "_io._WindowsConsoleIO.__sizeof__" => "Size of object in memory, in bytes.", + "_io._WindowsConsoleIO.__str__" => "Return str(self).", + "_io._WindowsConsoleIO.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_io._WindowsConsoleIO.close" => "Close the console object.\n\nA closed console object cannot be used for further I/O operations.\nclose() may be called more than once without error.", + "_io._WindowsConsoleIO.closed" => "True if the file is closed", + "_io._WindowsConsoleIO.closefd" => "True if the file descriptor will be closed by close().", + "_io._WindowsConsoleIO.fileno" => "Return the underlying file descriptor (an integer).", + "_io._WindowsConsoleIO.flush" => "Flush write buffers, if applicable.\n\nThis is not implemented for read-only and non-blocking streams.", + "_io._WindowsConsoleIO.isatty" => "Always True.", + "_io._WindowsConsoleIO.mode" => "String giving the file mode", + "_io._WindowsConsoleIO.read" => "Read at most size bytes, returned as bytes.\n\nOnly makes one system call when size is a positive integer,\nso less data may be returned than requested.\nReturn an empty bytes object at EOF.", + "_io._WindowsConsoleIO.readable" => "True if console is an input buffer.", + "_io._WindowsConsoleIO.readall" => "Read all data from the console, returned as bytes.\n\nReturn an empty bytes object at EOF.", + "_io._WindowsConsoleIO.readinto" => "Same as RawIOBase.readinto().", + "_io._WindowsConsoleIO.readline" => "Read and return a line from the stream.\n\nIf size is specified, at most size bytes will be read.\n\nThe line terminator is always b'\\n' for binary files; for text\nfiles, the newlines argument to open can be used to select the line\nterminator(s) recognized.", + "_io._WindowsConsoleIO.readlines" => "Return a list of lines from the stream.\n\nhint can be specified to control the number of lines read: no more\nlines will be read if the total size (in bytes/characters) of all\nlines so far exceeds hint.", + "_io._WindowsConsoleIO.seek" => "Change the stream position to the given byte offset.\n\n offset\n The stream position, relative to 'whence'.\n whence\n The relative position to seek from.\n\nThe offset is interpreted relative to the position indicated by whence.\nValues for whence are:\n\n* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n* os.SEEK_END or 2 -- end of stream; offset is usually negative\n\nReturn the new absolute position.", + "_io._WindowsConsoleIO.seekable" => "Return whether object supports random access.\n\nIf False, seek(), tell() and truncate() will raise OSError.\nThis method may need to do a test seek().", + "_io._WindowsConsoleIO.tell" => "Return current stream position.", + "_io._WindowsConsoleIO.truncate" => "Truncate file to size bytes.\n\nFile pointer is left unchanged. Size defaults to the current IO position\nas reported by tell(). Return the new size.", + "_io._WindowsConsoleIO.writable" => "True if console is an output buffer.", + "_io._WindowsConsoleIO.write" => "Write buffer b to file, return number of bytes written.\n\nOnly makes one system call, so not all of the data may be written.\nThe number of bytes actually written is returned.", + "_io._WindowsConsoleIO.writelines" => "Write a list of lines to stream.\n\nLine separators are not added, so it is usual for each of the\nlines provided to have a line separator at the end.", + "_io.open" => "Open file and return a stream. Raise OSError upon failure.\n\nfile is either a text or byte string giving the name (and the path\nif the file isn't in the current working directory) of the file to\nbe opened or an integer file descriptor of the file to be\nwrapped. (If a file descriptor is given, it is closed when the\nreturned I/O object is closed, unless closefd is set to False.)\n\nmode is an optional string that specifies the mode in which the file\nis opened. It defaults to 'r' which means open for reading in text\nmode. Other common values are 'w' for writing (truncating the file if\nit already exists), 'x' for creating and writing to a new file, and\n'a' for appending (which on some Unix systems, means that all writes\nappend to the end of the file regardless of the current seek position).\nIn text mode, if encoding is not specified the encoding used is platform\ndependent: locale.getencoding() is called to get the current locale encoding.\n(For reading and writing raw bytes use binary mode and leave encoding\nunspecified.) The available modes are:\n\n========= ===============================================================\nCharacter Meaning\n--------- ---------------------------------------------------------------\n'r' open for reading (default)\n'w' open for writing, truncating the file first\n'x' create a new file and open it for writing\n'a' open for writing, appending to the end of the file if it exists\n'b' binary mode\n't' text mode (default)\n'+' open a disk file for updating (reading and writing)\n========= ===============================================================\n\nThe default mode is 'rt' (open for reading text). For binary random\naccess, the mode 'w+b' opens and truncates the file to 0 bytes, while\n'r+b' opens the file without truncation. The 'x' mode implies 'w' and\nraises an `FileExistsError` if the file already exists.\n\nPython distinguishes between files opened in binary and text modes,\neven when the underlying operating system doesn't. Files opened in\nbinary mode (appending 'b' to the mode argument) return contents as\nbytes objects without any decoding. In text mode (the default, or when\n't' is appended to the mode argument), the contents of the file are\nreturned as strings, the bytes having been first decoded using a\nplatform-dependent encoding or using the specified encoding if given.\n\nbuffering is an optional integer used to set the buffering policy.\nPass 0 to switch buffering off (only allowed in binary mode), 1 to select\nline buffering (only usable in text mode), and an integer > 1 to indicate\nthe size of a fixed-size chunk buffer. When no buffering argument is\ngiven, the default buffering policy works as follows:\n\n* Binary files are buffered in fixed-size chunks; the size of the buffer\n is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)\n when the device block size is available.\n On most systems, the buffer will typically be 128 kilobytes long.\n\n* \"Interactive\" text files (files for which isatty() returns True)\n use line buffering. Other text files use the policy described above\n for binary files.\n\nencoding is the name of the encoding used to decode or encode the\nfile. This should only be used in text mode. The default encoding is\nplatform dependent, but any encoding supported by Python can be\npassed. See the codecs module for the list of supported encodings.\n\nerrors is an optional string that specifies how encoding errors are to\nbe handled---this argument should not be used in binary mode. Pass\n'strict' to raise a ValueError exception if there is an encoding error\n(the default of None has the same effect), or pass 'ignore' to ignore\nerrors. (Note that ignoring encoding errors can lead to data loss.)\nSee the documentation for codecs.register or run 'help(codecs.Codec)'\nfor a list of the permitted encoding error strings.\n\nnewline controls how universal newlines works (it only applies to text\nmode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\nfollows:\n\n* On input, if newline is None, universal newlines mode is\n enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n these are translated into '\\n' before being returned to the\n caller. If it is '', universal newline mode is enabled, but line\n endings are returned to the caller untranslated. If it has any of\n the other legal values, input lines are only terminated by the given\n string, and the line ending is returned to the caller untranslated.\n\n* On output, if newline is None, any '\\n' characters written are\n translated to the system default line separator, os.linesep. If\n newline is '' or '\\n', no translation takes place. If newline is any\n of the other legal values, any '\\n' characters written are translated\n to the given string.\n\nIf closefd is False, the underlying file descriptor will be kept open\nwhen the file is closed. This does not work when a file name is given\nand must be True in that case.\n\nA custom opener can be used by passing a callable as *opener*. The\nunderlying file descriptor for the file object is then obtained by\ncalling *opener* with (*file*, *flags*). *opener* must return an open\nfile descriptor (passing os.open as *opener* results in functionality\nsimilar to passing None).\n\nopen() returns a file object whose type depends on the mode, and\nthrough which the standard file operations such as reading and writing\nare performed. When open() is used to open a file in a text mode ('w',\n'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\na file in a binary mode, the returned class varies: in read binary\nmode, it returns a BufferedReader; in write binary and append binary\nmodes, it returns a BufferedWriter, and in read/write mode, it returns\na BufferedRandom.\n\nIt is also possible to use a string or bytearray as a file for both\nreading and writing. For strings StringIO can be used like a file\nopened in a text mode, and for bytes a BytesIO can be used like a file\nopened in a binary mode.", + "_io.open_code" => "Opens the provided file with the intent to import the contents.\n\nThis may perform extra validation beyond open(), but is otherwise interchangeable\nwith calling open(path, 'rb').", + "_io.text_encoding" => "A helper function to choose the text encoding.\n\nWhen encoding is not None, this function returns it.\nOtherwise, this function returns the default text encoding\n(i.e. \"locale\" or \"utf-8\" depends on UTF-8 mode).\n\nThis function emits an EncodingWarning if encoding is None and\nsys.flags.warn_default_encoding is true.\n\nThis can be used in APIs with an encoding=None parameter.\nHowever, please consider using encoding=\"utf-8\" for new APIs.", + "_json" => "json speedups", + "_json.encode_basestring" => "encode_basestring(string) -> string\n\nReturn a JSON representation of a Python string", + "_json.encode_basestring_ascii" => "encode_basestring_ascii(string) -> string\n\nReturn an ASCII-only JSON representation of a Python string", + "_json.make_encoder" => "Encoder(markers, default, encoder, indent, key_separator, item_separator, sort_keys, skipkeys, allow_nan)", + "_json.make_encoder.__call__" => "Call self as a function.", + "_json.make_encoder.__delattr__" => "Implement delattr(self, name).", + "_json.make_encoder.__eq__" => "Return self==value.", + "_json.make_encoder.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_json.make_encoder.__ge__" => "Return self>=value.", + "_json.make_encoder.__getattribute__" => "Return getattr(self, name).", + "_json.make_encoder.__getstate__" => "Helper for pickle.", + "_json.make_encoder.__gt__" => "Return self>value.", + "_json.make_encoder.__hash__" => "Return hash(self).", + "_json.make_encoder.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_json.make_encoder.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_json.make_encoder.__le__" => "Return self<=value.", + "_json.make_encoder.__lt__" => "Return self<value.", + "_json.make_encoder.__ne__" => "Return self!=value.", + "_json.make_encoder.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_json.make_encoder.__reduce__" => "Helper for pickle.", + "_json.make_encoder.__reduce_ex__" => "Helper for pickle.", + "_json.make_encoder.__repr__" => "Return repr(self).", + "_json.make_encoder.__setattr__" => "Implement setattr(self, name, value).", + "_json.make_encoder.__sizeof__" => "Size of object in memory, in bytes.", + "_json.make_encoder.__str__" => "Return str(self).", + "_json.make_encoder.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_json.make_encoder.default" => "default", + "_json.make_encoder.encoder" => "encoder", + "_json.make_encoder.indent" => "indent", + "_json.make_encoder.item_separator" => "item_separator", + "_json.make_encoder.key_separator" => "key_separator", + "_json.make_encoder.markers" => "markers", + "_json.make_encoder.skipkeys" => "skipkeys", + "_json.make_encoder.sort_keys" => "sort_keys", + "_json.make_scanner" => "JSON scanner object", + "_json.make_scanner.__call__" => "Call self as a function.", + "_json.make_scanner.__delattr__" => "Implement delattr(self, name).", + "_json.make_scanner.__eq__" => "Return self==value.", + "_json.make_scanner.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_json.make_scanner.__ge__" => "Return self>=value.", + "_json.make_scanner.__getattribute__" => "Return getattr(self, name).", + "_json.make_scanner.__getstate__" => "Helper for pickle.", + "_json.make_scanner.__gt__" => "Return self>value.", + "_json.make_scanner.__hash__" => "Return hash(self).", + "_json.make_scanner.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_json.make_scanner.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_json.make_scanner.__le__" => "Return self<=value.", + "_json.make_scanner.__lt__" => "Return self<value.", + "_json.make_scanner.__ne__" => "Return self!=value.", + "_json.make_scanner.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_json.make_scanner.__reduce__" => "Helper for pickle.", + "_json.make_scanner.__reduce_ex__" => "Helper for pickle.", + "_json.make_scanner.__repr__" => "Return repr(self).", + "_json.make_scanner.__setattr__" => "Implement setattr(self, name, value).", + "_json.make_scanner.__sizeof__" => "Size of object in memory, in bytes.", + "_json.make_scanner.__str__" => "Return str(self).", + "_json.make_scanner.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_json.make_scanner.object_hook" => "object_hook", + "_json.make_scanner.parse_constant" => "parse_constant", + "_json.make_scanner.parse_float" => "parse_float", + "_json.make_scanner.parse_int" => "parse_int", + "_json.make_scanner.strict" => "strict", + "_json.scanstring" => "scanstring(string, end, strict=True) -> (string, end)\n\nScan the string s for a JSON string. End is the index of the\ncharacter in s after the quote that started the JSON string.\nUnescapes all valid JSON string escape sequences and raises ValueError\non attempt to decode an invalid string. If strict is False then literal\ncontrol characters are allowed in the string.\n\nReturns a tuple of the decoded string and the index of the character in s\nafter the end quote.", + "_locale" => "Support for POSIX locales.", + "_locale.Error.__delattr__" => "Implement delattr(self, name).", + "_locale.Error.__eq__" => "Return self==value.", + "_locale.Error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_locale.Error.__ge__" => "Return self>=value.", + "_locale.Error.__getattribute__" => "Return getattr(self, name).", + "_locale.Error.__getstate__" => "Helper for pickle.", + "_locale.Error.__gt__" => "Return self>value.", + "_locale.Error.__hash__" => "Return hash(self).", + "_locale.Error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_locale.Error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_locale.Error.__le__" => "Return self<=value.", + "_locale.Error.__lt__" => "Return self<value.", + "_locale.Error.__ne__" => "Return self!=value.", + "_locale.Error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_locale.Error.__reduce_ex__" => "Helper for pickle.", + "_locale.Error.__repr__" => "Return repr(self).", + "_locale.Error.__setattr__" => "Implement setattr(self, name, value).", + "_locale.Error.__sizeof__" => "Size of object in memory, in bytes.", + "_locale.Error.__str__" => "Return str(self).", + "_locale.Error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_locale.Error.__weakref__" => "list of weak references to the object", + "_locale.Error.add_note" => "Add a note to the exception", + "_locale.Error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_locale.bind_textdomain_codeset" => "Bind the C library's domain to codeset.", + "_locale.bindtextdomain" => "Bind the C library's domain to dir.", + "_locale.dcgettext" => "Return translation of msg in domain and category.", + "_locale.dgettext" => "dgettext(domain, msg) -> string\n\nReturn translation of msg in domain.", + "_locale.getencoding" => "Get the current locale encoding.", + "_locale.gettext" => "gettext(msg) -> string\n\nReturn translation of msg.", + "_locale.localeconv" => "Returns numeric and monetary locale-specific parameters.", + "_locale.nl_langinfo" => "Return the value for the locale information associated with key.", + "_locale.setlocale" => "Activates/queries locale processing.", + "_locale.strcoll" => "Compares two strings according to the locale.", + "_locale.strxfrm" => "Return a string that can be used as a key for locale-aware comparisons.", + "_locale.textdomain" => "Set the C library's textdmain to domain, returning the new domain.", + "_lsprof" => "Fast profiler", + "_lsprof.Profiler" => "Build a profiler object using the specified timer function.\n\nThe default timer is a fast built-in one based on real time.\nFor custom timer functions returning integers, 'timeunit' can\nbe a float specifying a scale (that is, how long each integer unit\nis, in seconds).", + "_lsprof.Profiler.__delattr__" => "Implement delattr(self, name).", + "_lsprof.Profiler.__eq__" => "Return self==value.", + "_lsprof.Profiler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_lsprof.Profiler.__ge__" => "Return self>=value.", + "_lsprof.Profiler.__getattribute__" => "Return getattr(self, name).", + "_lsprof.Profiler.__getstate__" => "Helper for pickle.", + "_lsprof.Profiler.__gt__" => "Return self>value.", + "_lsprof.Profiler.__hash__" => "Return hash(self).", + "_lsprof.Profiler.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_lsprof.Profiler.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_lsprof.Profiler.__le__" => "Return self<=value.", + "_lsprof.Profiler.__lt__" => "Return self<value.", + "_lsprof.Profiler.__ne__" => "Return self!=value.", + "_lsprof.Profiler.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_lsprof.Profiler.__reduce__" => "Helper for pickle.", + "_lsprof.Profiler.__reduce_ex__" => "Helper for pickle.", + "_lsprof.Profiler.__repr__" => "Return repr(self).", + "_lsprof.Profiler.__setattr__" => "Implement setattr(self, name, value).", + "_lsprof.Profiler.__sizeof__" => "Size of object in memory, in bytes.", + "_lsprof.Profiler.__str__" => "Return str(self).", + "_lsprof.Profiler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_lsprof.Profiler.clear" => "Clear all profiling information collected so far.", + "_lsprof.Profiler.disable" => "Stop collecting profiling information.", + "_lsprof.Profiler.enable" => "Start collecting profiling information.\n\n subcalls\n If True, also records for each function\n statistics separated according to its current caller.\n builtins\n If True, records the time spent in\n built-in functions separately from their caller.", + "_lsprof.Profiler.getstats" => "list of profiler_entry objects.\n\ngetstats() -> list of profiler_entry objects\n\nReturn all information collected by the profiler.\nEach profiler_entry is a tuple-like object with the\nfollowing attributes:\n\n code code object\n callcount how many times this was called\n reccallcount how many times called recursively\n totaltime total time in this entry\n inlinetime inline time in this entry (not in subcalls)\n calls details of the calls\n\nThe calls attribute is either None or a list of\nprofiler_subentry objects:\n\n code called code object\n callcount how many times this is called\n reccallcount how many times this is called recursively\n totaltime total time spent in this call\n inlinetime inline time (not in further subcalls)", + "_lsprof.profiler_entry.__add__" => "Return self+value.", + "_lsprof.profiler_entry.__class_getitem__" => "See PEP 585", + "_lsprof.profiler_entry.__contains__" => "Return bool(key in self).", + "_lsprof.profiler_entry.__delattr__" => "Implement delattr(self, name).", + "_lsprof.profiler_entry.__eq__" => "Return self==value.", + "_lsprof.profiler_entry.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_lsprof.profiler_entry.__ge__" => "Return self>=value.", + "_lsprof.profiler_entry.__getattribute__" => "Return getattr(self, name).", + "_lsprof.profiler_entry.__getitem__" => "Return self[key].", + "_lsprof.profiler_entry.__getstate__" => "Helper for pickle.", + "_lsprof.profiler_entry.__gt__" => "Return self>value.", + "_lsprof.profiler_entry.__hash__" => "Return hash(self).", + "_lsprof.profiler_entry.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_lsprof.profiler_entry.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_lsprof.profiler_entry.__iter__" => "Implement iter(self).", + "_lsprof.profiler_entry.__le__" => "Return self<=value.", + "_lsprof.profiler_entry.__len__" => "Return len(self).", + "_lsprof.profiler_entry.__lt__" => "Return self<value.", + "_lsprof.profiler_entry.__mul__" => "Return self*value.", + "_lsprof.profiler_entry.__ne__" => "Return self!=value.", + "_lsprof.profiler_entry.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_lsprof.profiler_entry.__reduce_ex__" => "Helper for pickle.", + "_lsprof.profiler_entry.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_lsprof.profiler_entry.__repr__" => "Return repr(self).", + "_lsprof.profiler_entry.__rmul__" => "Return value*self.", + "_lsprof.profiler_entry.__setattr__" => "Implement setattr(self, name, value).", + "_lsprof.profiler_entry.__sizeof__" => "Size of object in memory, in bytes.", + "_lsprof.profiler_entry.__str__" => "Return str(self).", + "_lsprof.profiler_entry.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_lsprof.profiler_entry.callcount" => "how many times this was called", + "_lsprof.profiler_entry.calls" => "details of the calls", + "_lsprof.profiler_entry.code" => "code object or built-in function name", + "_lsprof.profiler_entry.count" => "Return number of occurrences of value.", + "_lsprof.profiler_entry.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_lsprof.profiler_entry.inlinetime" => "inline time in this entry (not in subcalls)", + "_lsprof.profiler_entry.reccallcount" => "how many times called recursively", + "_lsprof.profiler_entry.totaltime" => "total time in this entry", + "_lsprof.profiler_subentry.__add__" => "Return self+value.", + "_lsprof.profiler_subentry.__class_getitem__" => "See PEP 585", + "_lsprof.profiler_subentry.__contains__" => "Return bool(key in self).", + "_lsprof.profiler_subentry.__delattr__" => "Implement delattr(self, name).", + "_lsprof.profiler_subentry.__eq__" => "Return self==value.", + "_lsprof.profiler_subentry.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_lsprof.profiler_subentry.__ge__" => "Return self>=value.", + "_lsprof.profiler_subentry.__getattribute__" => "Return getattr(self, name).", + "_lsprof.profiler_subentry.__getitem__" => "Return self[key].", + "_lsprof.profiler_subentry.__getstate__" => "Helper for pickle.", + "_lsprof.profiler_subentry.__gt__" => "Return self>value.", + "_lsprof.profiler_subentry.__hash__" => "Return hash(self).", + "_lsprof.profiler_subentry.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_lsprof.profiler_subentry.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_lsprof.profiler_subentry.__iter__" => "Implement iter(self).", + "_lsprof.profiler_subentry.__le__" => "Return self<=value.", + "_lsprof.profiler_subentry.__len__" => "Return len(self).", + "_lsprof.profiler_subentry.__lt__" => "Return self<value.", + "_lsprof.profiler_subentry.__mul__" => "Return self*value.", + "_lsprof.profiler_subentry.__ne__" => "Return self!=value.", + "_lsprof.profiler_subentry.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_lsprof.profiler_subentry.__reduce_ex__" => "Helper for pickle.", + "_lsprof.profiler_subentry.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_lsprof.profiler_subentry.__repr__" => "Return repr(self).", + "_lsprof.profiler_subentry.__rmul__" => "Return value*self.", + "_lsprof.profiler_subentry.__setattr__" => "Implement setattr(self, name, value).", + "_lsprof.profiler_subentry.__sizeof__" => "Size of object in memory, in bytes.", + "_lsprof.profiler_subentry.__str__" => "Return str(self).", + "_lsprof.profiler_subentry.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_lsprof.profiler_subentry.callcount" => "how many times this is called", + "_lsprof.profiler_subentry.code" => "called code object or built-in function name", + "_lsprof.profiler_subentry.count" => "Return number of occurrences of value.", + "_lsprof.profiler_subentry.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_lsprof.profiler_subentry.inlinetime" => "inline time (not in further subcalls)", + "_lsprof.profiler_subentry.reccallcount" => "how many times this is called recursively", + "_lsprof.profiler_subentry.totaltime" => "total time spent in this call", + "_lzma.LZMACompressor" => "LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)\n\nCreate a compressor object for compressing data incrementally.\n\nformat specifies the container format to use for the output. This can\nbe FORMAT_XZ (default), FORMAT_ALONE, or FORMAT_RAW.\n\ncheck specifies the integrity check to use. For FORMAT_XZ, the default\nis CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not support integrity\nchecks; for these formats, check must be omitted, or be CHECK_NONE.\n\nThe settings used by the compressor can be specified either as a\npreset compression level (with the 'preset' argument), or in detail\nas a custom filter chain (with the 'filters' argument). For FORMAT_XZ\nand FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset\nlevel. For FORMAT_RAW, the caller must always specify a filter chain;\nthe raw compressor does not support preset compression levels.\n\npreset (if provided) should be an integer in the range 0-9, optionally\nOR-ed with the constant PRESET_EXTREME.\n\nfilters (if provided) should be a sequence of dicts. Each dict should\nhave an entry for \"id\" indicating the ID of the filter, plus\nadditional entries for options to the filter.\n\nFor one-shot compression, use the compress() function instead.", + "_lzma.LZMACompressor.__delattr__" => "Implement delattr(self, name).", + "_lzma.LZMACompressor.__eq__" => "Return self==value.", + "_lzma.LZMACompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_lzma.LZMACompressor.__ge__" => "Return self>=value.", + "_lzma.LZMACompressor.__getattribute__" => "Return getattr(self, name).", + "_lzma.LZMACompressor.__getstate__" => "Helper for pickle.", + "_lzma.LZMACompressor.__gt__" => "Return self>value.", + "_lzma.LZMACompressor.__hash__" => "Return hash(self).", + "_lzma.LZMACompressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_lzma.LZMACompressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_lzma.LZMACompressor.__le__" => "Return self<=value.", + "_lzma.LZMACompressor.__lt__" => "Return self<value.", + "_lzma.LZMACompressor.__ne__" => "Return self!=value.", + "_lzma.LZMACompressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_lzma.LZMACompressor.__reduce__" => "Helper for pickle.", + "_lzma.LZMACompressor.__reduce_ex__" => "Helper for pickle.", + "_lzma.LZMACompressor.__repr__" => "Return repr(self).", + "_lzma.LZMACompressor.__setattr__" => "Implement setattr(self, name, value).", + "_lzma.LZMACompressor.__sizeof__" => "Size of object in memory, in bytes.", + "_lzma.LZMACompressor.__str__" => "Return str(self).", + "_lzma.LZMACompressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_lzma.LZMACompressor.compress" => "Provide data to the compressor object.\n\nReturns a chunk of compressed data if possible, or b'' otherwise.\n\nWhen you have finished providing data to the compressor, call the\nflush() method to finish the compression process.", + "_lzma.LZMACompressor.flush" => "Finish the compression process.\n\nReturns the compressed data left in internal buffers.\n\nThe compressor object may not be used after this method is called.", + "_lzma.LZMADecompressor" => "Create a decompressor object for decompressing data incrementally.\n\n format\n Specifies the container format of the input stream. If this is\n FORMAT_AUTO (the default), the decompressor will automatically detect\n whether the input is FORMAT_XZ or FORMAT_ALONE. Streams created with\n FORMAT_RAW cannot be autodetected.\n memlimit\n Limit the amount of memory used by the decompressor. This will cause\n decompression to fail if the input cannot be decompressed within the\n given limit.\n filters\n A custom filter chain. This argument is required for FORMAT_RAW, and\n not accepted with any other format. When provided, this should be a\n sequence of dicts, each indicating the ID and options for a single\n filter.\n\nFor one-shot decompression, use the decompress() function instead.", + "_lzma.LZMADecompressor.__delattr__" => "Implement delattr(self, name).", + "_lzma.LZMADecompressor.__eq__" => "Return self==value.", + "_lzma.LZMADecompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_lzma.LZMADecompressor.__ge__" => "Return self>=value.", + "_lzma.LZMADecompressor.__getattribute__" => "Return getattr(self, name).", + "_lzma.LZMADecompressor.__getstate__" => "Helper for pickle.", + "_lzma.LZMADecompressor.__gt__" => "Return self>value.", + "_lzma.LZMADecompressor.__hash__" => "Return hash(self).", + "_lzma.LZMADecompressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_lzma.LZMADecompressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_lzma.LZMADecompressor.__le__" => "Return self<=value.", + "_lzma.LZMADecompressor.__lt__" => "Return self<value.", + "_lzma.LZMADecompressor.__ne__" => "Return self!=value.", + "_lzma.LZMADecompressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_lzma.LZMADecompressor.__reduce__" => "Helper for pickle.", + "_lzma.LZMADecompressor.__reduce_ex__" => "Helper for pickle.", + "_lzma.LZMADecompressor.__repr__" => "Return repr(self).", + "_lzma.LZMADecompressor.__setattr__" => "Implement setattr(self, name, value).", + "_lzma.LZMADecompressor.__sizeof__" => "Size of object in memory, in bytes.", + "_lzma.LZMADecompressor.__str__" => "Return str(self).", + "_lzma.LZMADecompressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_lzma.LZMADecompressor.check" => "ID of the integrity check used by the input stream.", + "_lzma.LZMADecompressor.decompress" => "Decompress *data*, returning uncompressed data as bytes.\n\nIf *max_length* is nonnegative, returns at most *max_length* bytes of\ndecompressed data. If this limit is reached and further output can be\nproduced, *self.needs_input* will be set to ``False``. In this case, the next\ncall to *decompress()* may provide *data* as b'' to obtain more of the output.\n\nIf all of the input data was decompressed and returned (either because this\nwas less than *max_length* bytes, or because *max_length* was negative),\n*self.needs_input* will be set to True.\n\nAttempting to decompress data after the end of stream is reached raises an\nEOFError. Any data found after the end of the stream is ignored and saved in\nthe unused_data attribute.", + "_lzma.LZMADecompressor.eof" => "True if the end-of-stream marker has been reached.", + "_lzma.LZMADecompressor.needs_input" => "True if more input is needed before more decompressed data can be produced.", + "_lzma.LZMADecompressor.unused_data" => "Data found after the end of the compressed stream.", + "_lzma.LZMAError" => "Call to liblzma failed.", + "_lzma.LZMAError.__delattr__" => "Implement delattr(self, name).", + "_lzma.LZMAError.__eq__" => "Return self==value.", + "_lzma.LZMAError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_lzma.LZMAError.__ge__" => "Return self>=value.", + "_lzma.LZMAError.__getattribute__" => "Return getattr(self, name).", + "_lzma.LZMAError.__getstate__" => "Helper for pickle.", + "_lzma.LZMAError.__gt__" => "Return self>value.", + "_lzma.LZMAError.__hash__" => "Return hash(self).", + "_lzma.LZMAError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_lzma.LZMAError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_lzma.LZMAError.__le__" => "Return self<=value.", + "_lzma.LZMAError.__lt__" => "Return self<value.", + "_lzma.LZMAError.__ne__" => "Return self!=value.", + "_lzma.LZMAError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_lzma.LZMAError.__reduce_ex__" => "Helper for pickle.", + "_lzma.LZMAError.__repr__" => "Return repr(self).", + "_lzma.LZMAError.__setattr__" => "Implement setattr(self, name, value).", + "_lzma.LZMAError.__sizeof__" => "Size of object in memory, in bytes.", + "_lzma.LZMAError.__str__" => "Return str(self).", + "_lzma.LZMAError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_lzma.LZMAError.__weakref__" => "list of weak references to the object", + "_lzma.LZMAError.add_note" => "Add a note to the exception", + "_lzma.LZMAError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_lzma._decode_filter_properties" => "Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).\n\nThe result does not include the filter ID itself, only the options.", + "_lzma._encode_filter_properties" => "Return a bytes object encoding the options (properties) of the filter specified by *filter* (a dict).\n\nThe result does not include the filter ID itself, only the options.", + "_lzma.is_check_supported" => "Test whether the given integrity check is supported.\n\nAlways returns True for CHECK_NONE and CHECK_CRC32.", + "_md5.MD5Type.__delattr__" => "Implement delattr(self, name).", + "_md5.MD5Type.__eq__" => "Return self==value.", + "_md5.MD5Type.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_md5.MD5Type.__ge__" => "Return self>=value.", + "_md5.MD5Type.__getattribute__" => "Return getattr(self, name).", + "_md5.MD5Type.__getstate__" => "Helper for pickle.", + "_md5.MD5Type.__gt__" => "Return self>value.", + "_md5.MD5Type.__hash__" => "Return hash(self).", + "_md5.MD5Type.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_md5.MD5Type.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_md5.MD5Type.__le__" => "Return self<=value.", + "_md5.MD5Type.__lt__" => "Return self<value.", + "_md5.MD5Type.__ne__" => "Return self!=value.", + "_md5.MD5Type.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_md5.MD5Type.__reduce__" => "Helper for pickle.", + "_md5.MD5Type.__reduce_ex__" => "Helper for pickle.", + "_md5.MD5Type.__repr__" => "Return repr(self).", + "_md5.MD5Type.__setattr__" => "Implement setattr(self, name, value).", + "_md5.MD5Type.__sizeof__" => "Size of object in memory, in bytes.", + "_md5.MD5Type.__str__" => "Return str(self).", + "_md5.MD5Type.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_md5.MD5Type.copy" => "Return a copy of the hash object.", + "_md5.MD5Type.digest" => "Return the digest value as a bytes object.", + "_md5.MD5Type.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_md5.MD5Type.update" => "Update this hash object's state with the provided string.", + "_md5.md5" => "Return a new MD5 hash object; optionally initialized with a string.", + "_multibytecodec.MultibyteIncrementalDecoder.__delattr__" => "Implement delattr(self, name).", + "_multibytecodec.MultibyteIncrementalDecoder.__eq__" => "Return self==value.", + "_multibytecodec.MultibyteIncrementalDecoder.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_multibytecodec.MultibyteIncrementalDecoder.__ge__" => "Return self>=value.", + "_multibytecodec.MultibyteIncrementalDecoder.__getattribute__" => "Return getattr(self, name).", + "_multibytecodec.MultibyteIncrementalDecoder.__getstate__" => "Helper for pickle.", + "_multibytecodec.MultibyteIncrementalDecoder.__gt__" => "Return self>value.", + "_multibytecodec.MultibyteIncrementalDecoder.__hash__" => "Return hash(self).", + "_multibytecodec.MultibyteIncrementalDecoder.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_multibytecodec.MultibyteIncrementalDecoder.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_multibytecodec.MultibyteIncrementalDecoder.__le__" => "Return self<=value.", + "_multibytecodec.MultibyteIncrementalDecoder.__lt__" => "Return self<value.", + "_multibytecodec.MultibyteIncrementalDecoder.__ne__" => "Return self!=value.", + "_multibytecodec.MultibyteIncrementalDecoder.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_multibytecodec.MultibyteIncrementalDecoder.__reduce__" => "Helper for pickle.", + "_multibytecodec.MultibyteIncrementalDecoder.__reduce_ex__" => "Helper for pickle.", + "_multibytecodec.MultibyteIncrementalDecoder.__repr__" => "Return repr(self).", + "_multibytecodec.MultibyteIncrementalDecoder.__setattr__" => "Implement setattr(self, name, value).", + "_multibytecodec.MultibyteIncrementalDecoder.__sizeof__" => "Size of object in memory, in bytes.", + "_multibytecodec.MultibyteIncrementalDecoder.__str__" => "Return str(self).", + "_multibytecodec.MultibyteIncrementalDecoder.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_multibytecodec.MultibyteIncrementalDecoder.errors" => "how to treat errors", + "_multibytecodec.MultibyteIncrementalEncoder.__delattr__" => "Implement delattr(self, name).", + "_multibytecodec.MultibyteIncrementalEncoder.__eq__" => "Return self==value.", + "_multibytecodec.MultibyteIncrementalEncoder.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_multibytecodec.MultibyteIncrementalEncoder.__ge__" => "Return self>=value.", + "_multibytecodec.MultibyteIncrementalEncoder.__getattribute__" => "Return getattr(self, name).", + "_multibytecodec.MultibyteIncrementalEncoder.__getstate__" => "Helper for pickle.", + "_multibytecodec.MultibyteIncrementalEncoder.__gt__" => "Return self>value.", + "_multibytecodec.MultibyteIncrementalEncoder.__hash__" => "Return hash(self).", + "_multibytecodec.MultibyteIncrementalEncoder.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_multibytecodec.MultibyteIncrementalEncoder.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_multibytecodec.MultibyteIncrementalEncoder.__le__" => "Return self<=value.", + "_multibytecodec.MultibyteIncrementalEncoder.__lt__" => "Return self<value.", + "_multibytecodec.MultibyteIncrementalEncoder.__ne__" => "Return self!=value.", + "_multibytecodec.MultibyteIncrementalEncoder.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_multibytecodec.MultibyteIncrementalEncoder.__reduce__" => "Helper for pickle.", + "_multibytecodec.MultibyteIncrementalEncoder.__reduce_ex__" => "Helper for pickle.", + "_multibytecodec.MultibyteIncrementalEncoder.__repr__" => "Return repr(self).", + "_multibytecodec.MultibyteIncrementalEncoder.__setattr__" => "Implement setattr(self, name, value).", + "_multibytecodec.MultibyteIncrementalEncoder.__sizeof__" => "Size of object in memory, in bytes.", + "_multibytecodec.MultibyteIncrementalEncoder.__str__" => "Return str(self).", + "_multibytecodec.MultibyteIncrementalEncoder.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_multibytecodec.MultibyteIncrementalEncoder.errors" => "how to treat errors", + "_multibytecodec.MultibyteStreamReader.__delattr__" => "Implement delattr(self, name).", + "_multibytecodec.MultibyteStreamReader.__eq__" => "Return self==value.", + "_multibytecodec.MultibyteStreamReader.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_multibytecodec.MultibyteStreamReader.__ge__" => "Return self>=value.", + "_multibytecodec.MultibyteStreamReader.__getattribute__" => "Return getattr(self, name).", + "_multibytecodec.MultibyteStreamReader.__getstate__" => "Helper for pickle.", + "_multibytecodec.MultibyteStreamReader.__gt__" => "Return self>value.", + "_multibytecodec.MultibyteStreamReader.__hash__" => "Return hash(self).", + "_multibytecodec.MultibyteStreamReader.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_multibytecodec.MultibyteStreamReader.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_multibytecodec.MultibyteStreamReader.__le__" => "Return self<=value.", + "_multibytecodec.MultibyteStreamReader.__lt__" => "Return self<value.", + "_multibytecodec.MultibyteStreamReader.__ne__" => "Return self!=value.", + "_multibytecodec.MultibyteStreamReader.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_multibytecodec.MultibyteStreamReader.__reduce__" => "Helper for pickle.", + "_multibytecodec.MultibyteStreamReader.__reduce_ex__" => "Helper for pickle.", + "_multibytecodec.MultibyteStreamReader.__repr__" => "Return repr(self).", + "_multibytecodec.MultibyteStreamReader.__setattr__" => "Implement setattr(self, name, value).", + "_multibytecodec.MultibyteStreamReader.__sizeof__" => "Size of object in memory, in bytes.", + "_multibytecodec.MultibyteStreamReader.__str__" => "Return str(self).", + "_multibytecodec.MultibyteStreamReader.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_multibytecodec.MultibyteStreamReader.errors" => "how to treat errors", + "_multibytecodec.MultibyteStreamWriter.__delattr__" => "Implement delattr(self, name).", + "_multibytecodec.MultibyteStreamWriter.__eq__" => "Return self==value.", + "_multibytecodec.MultibyteStreamWriter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_multibytecodec.MultibyteStreamWriter.__ge__" => "Return self>=value.", + "_multibytecodec.MultibyteStreamWriter.__getattribute__" => "Return getattr(self, name).", + "_multibytecodec.MultibyteStreamWriter.__getstate__" => "Helper for pickle.", + "_multibytecodec.MultibyteStreamWriter.__gt__" => "Return self>value.", + "_multibytecodec.MultibyteStreamWriter.__hash__" => "Return hash(self).", + "_multibytecodec.MultibyteStreamWriter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_multibytecodec.MultibyteStreamWriter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_multibytecodec.MultibyteStreamWriter.__le__" => "Return self<=value.", + "_multibytecodec.MultibyteStreamWriter.__lt__" => "Return self<value.", + "_multibytecodec.MultibyteStreamWriter.__ne__" => "Return self!=value.", + "_multibytecodec.MultibyteStreamWriter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_multibytecodec.MultibyteStreamWriter.__reduce__" => "Helper for pickle.", + "_multibytecodec.MultibyteStreamWriter.__reduce_ex__" => "Helper for pickle.", + "_multibytecodec.MultibyteStreamWriter.__repr__" => "Return repr(self).", + "_multibytecodec.MultibyteStreamWriter.__setattr__" => "Implement setattr(self, name, value).", + "_multibytecodec.MultibyteStreamWriter.__sizeof__" => "Size of object in memory, in bytes.", + "_multibytecodec.MultibyteStreamWriter.__str__" => "Return str(self).", + "_multibytecodec.MultibyteStreamWriter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_multibytecodec.MultibyteStreamWriter.errors" => "how to treat errors", + "_multiprocessing.SemLock" => "Semaphore/Mutex type", + "_multiprocessing.SemLock.__delattr__" => "Implement delattr(self, name).", + "_multiprocessing.SemLock.__enter__" => "Enter the semaphore/lock.", + "_multiprocessing.SemLock.__eq__" => "Return self==value.", + "_multiprocessing.SemLock.__exit__" => "Exit the semaphore/lock.", + "_multiprocessing.SemLock.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_multiprocessing.SemLock.__ge__" => "Return self>=value.", + "_multiprocessing.SemLock.__getattribute__" => "Return getattr(self, name).", + "_multiprocessing.SemLock.__getstate__" => "Helper for pickle.", + "_multiprocessing.SemLock.__gt__" => "Return self>value.", + "_multiprocessing.SemLock.__hash__" => "Return hash(self).", + "_multiprocessing.SemLock.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_multiprocessing.SemLock.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_multiprocessing.SemLock.__le__" => "Return self<=value.", + "_multiprocessing.SemLock.__lt__" => "Return self<value.", + "_multiprocessing.SemLock.__ne__" => "Return self!=value.", + "_multiprocessing.SemLock.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_multiprocessing.SemLock.__reduce__" => "Helper for pickle.", + "_multiprocessing.SemLock.__reduce_ex__" => "Helper for pickle.", + "_multiprocessing.SemLock.__repr__" => "Return repr(self).", + "_multiprocessing.SemLock.__setattr__" => "Implement setattr(self, name, value).", + "_multiprocessing.SemLock.__sizeof__" => "Size of object in memory, in bytes.", + "_multiprocessing.SemLock.__str__" => "Return str(self).", + "_multiprocessing.SemLock.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_multiprocessing.SemLock._after_fork" => "Rezero the net acquisition count after fork().", + "_multiprocessing.SemLock._count" => "Num of `acquire()`s minus num of `release()`s for this process.", + "_multiprocessing.SemLock._get_value" => "Get the value of the semaphore.", + "_multiprocessing.SemLock._is_mine" => "Whether the lock is owned by this thread.", + "_multiprocessing.SemLock._is_zero" => "Return whether semaphore has value zero.", + "_multiprocessing.SemLock.acquire" => "Acquire the semaphore/lock.", + "_multiprocessing.SemLock.handle" => "", + "_multiprocessing.SemLock.kind" => "", + "_multiprocessing.SemLock.maxvalue" => "", + "_multiprocessing.SemLock.name" => "", + "_multiprocessing.SemLock.release" => "Release the semaphore/lock.", + "_opcode" => "Opcode support module.", + "_opcode.get_executor" => "Return the executor object at offset in code if exists, None otherwise.", + "_opcode.get_intrinsic1_descs" => "Return a list of names of the unary intrinsics.", + "_opcode.get_intrinsic2_descs" => "Return a list of names of the binary intrinsics.", + "_opcode.get_nb_ops" => "Return array of symbols of binary ops.\n\nIndexed by the BINARY_OP oparg value.", + "_opcode.get_special_method_names" => "Return a list of special method names.", + "_opcode.get_specialization_stats" => "Return the specialization stats", + "_opcode.has_arg" => "Return True if the opcode uses its oparg, False otherwise.", + "_opcode.has_const" => "Return True if the opcode accesses a constant, False otherwise.", + "_opcode.has_exc" => "Return True if the opcode sets an exception handler, False otherwise.", + "_opcode.has_free" => "Return True if the opcode accesses a free variable, False otherwise.\n\nNote that 'free' in this context refers to names in the current scope\nthat are referenced by inner scopes or names in outer scopes that are\nreferenced from this scope. It does not include references to global\nor builtin scopes.", + "_opcode.has_jump" => "Return True if the opcode has a jump target, False otherwise.", + "_opcode.has_local" => "Return True if the opcode accesses a local variable, False otherwise.", + "_opcode.has_name" => "Return True if the opcode accesses an attribute by name, False otherwise.", + "_opcode.is_valid" => "Return True if opcode is valid, False otherwise.", + "_opcode.stack_effect" => "Compute the stack effect of the opcode.", + "_operator" => "Operator interface.\n\nThis module exports a set of functions implemented in C corresponding\nto the intrinsic operators of Python. For example, operator.add(x, y)\nis equivalent to the expression x+y. The function names are those\nused for special methods; variants without leading and trailing\n'__' are also provided for convenience.", + "_operator._compare_digest" => "Return 'a == b'.\n\nThis function uses an approach designed to prevent\ntiming analysis, making it appropriate for cryptography.\n\na and b must both be of the same type: either str (ASCII only),\nor any bytes-like object.\n\nNote: If a and b are of different lengths, or if an error occurs,\na timing attack could theoretically reveal information about the\ntypes and lengths of a and b--but not their values.", + "_operator.abs" => "Same as abs(a).", + "_operator.add" => "Same as a + b.", + "_operator.and_" => "Same as a & b.", + "_operator.attrgetter" => "Return a callable object that fetches the given attribute(s) from its operand.\nAfter f = attrgetter('name'), the call f(r) returns r.name.\nAfter g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).\nAfter h = attrgetter('name.first', 'name.last'), the call h(r) returns\n(r.name.first, r.name.last).", + "_operator.attrgetter.__call__" => "Call self as a function.", + "_operator.attrgetter.__delattr__" => "Implement delattr(self, name).", + "_operator.attrgetter.__eq__" => "Return self==value.", + "_operator.attrgetter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_operator.attrgetter.__ge__" => "Return self>=value.", + "_operator.attrgetter.__getattribute__" => "Return getattr(self, name).", + "_operator.attrgetter.__getstate__" => "Helper for pickle.", + "_operator.attrgetter.__gt__" => "Return self>value.", + "_operator.attrgetter.__hash__" => "Return hash(self).", + "_operator.attrgetter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_operator.attrgetter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_operator.attrgetter.__le__" => "Return self<=value.", + "_operator.attrgetter.__lt__" => "Return self<value.", + "_operator.attrgetter.__ne__" => "Return self!=value.", + "_operator.attrgetter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_operator.attrgetter.__reduce__" => "Return state information for pickling", + "_operator.attrgetter.__reduce_ex__" => "Helper for pickle.", + "_operator.attrgetter.__repr__" => "Return repr(self).", + "_operator.attrgetter.__setattr__" => "Implement setattr(self, name, value).", + "_operator.attrgetter.__sizeof__" => "Size of object in memory, in bytes.", + "_operator.attrgetter.__str__" => "Return str(self).", + "_operator.attrgetter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_operator.call" => "Same as obj(*args, **kwargs).", + "_operator.concat" => "Same as a + b, for a and b sequences.", + "_operator.contains" => "Same as b in a (note reversed operands).", + "_operator.countOf" => "Return the number of items in a which are, or which equal, b.", + "_operator.delitem" => "Same as del a[b].", + "_operator.eq" => "Same as a == b.", + "_operator.floordiv" => "Same as a // b.", + "_operator.ge" => "Same as a >= b.", + "_operator.getitem" => "Same as a[b].", + "_operator.gt" => "Same as a > b.", + "_operator.iadd" => "Same as a += b.", + "_operator.iand" => "Same as a &= b.", + "_operator.iconcat" => "Same as a += b, for a and b sequences.", + "_operator.ifloordiv" => "Same as a //= b.", + "_operator.ilshift" => "Same as a <<= b.", + "_operator.imatmul" => "Same as a @= b.", + "_operator.imod" => "Same as a %= b.", + "_operator.imul" => "Same as a *= b.", + "_operator.index" => "Same as a.__index__()", + "_operator.indexOf" => "Return the first index of b in a.", + "_operator.inv" => "Same as ~a.", + "_operator.invert" => "Same as ~a.", + "_operator.ior" => "Same as a |= b.", + "_operator.ipow" => "Same as a **= b.", + "_operator.irshift" => "Same as a >>= b.", + "_operator.is_" => "Same as a is b.", + "_operator.is_none" => "Same as a is None.", + "_operator.is_not" => "Same as a is not b.", + "_operator.is_not_none" => "Same as a is not None.", + "_operator.isub" => "Same as a -= b.", + "_operator.itemgetter" => "Return a callable object that fetches the given item(s) from its operand.\nAfter f = itemgetter(2), the call f(r) returns r[2].\nAfter g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3])", + "_operator.itemgetter.__call__" => "Call self as a function.", + "_operator.itemgetter.__delattr__" => "Implement delattr(self, name).", + "_operator.itemgetter.__eq__" => "Return self==value.", + "_operator.itemgetter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_operator.itemgetter.__ge__" => "Return self>=value.", + "_operator.itemgetter.__getattribute__" => "Return getattr(self, name).", + "_operator.itemgetter.__getstate__" => "Helper for pickle.", + "_operator.itemgetter.__gt__" => "Return self>value.", + "_operator.itemgetter.__hash__" => "Return hash(self).", + "_operator.itemgetter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_operator.itemgetter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_operator.itemgetter.__le__" => "Return self<=value.", + "_operator.itemgetter.__lt__" => "Return self<value.", + "_operator.itemgetter.__ne__" => "Return self!=value.", + "_operator.itemgetter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_operator.itemgetter.__reduce__" => "Return state information for pickling", + "_operator.itemgetter.__reduce_ex__" => "Helper for pickle.", + "_operator.itemgetter.__repr__" => "Return repr(self).", + "_operator.itemgetter.__setattr__" => "Implement setattr(self, name, value).", + "_operator.itemgetter.__sizeof__" => "Size of object in memory, in bytes.", + "_operator.itemgetter.__str__" => "Return str(self).", + "_operator.itemgetter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_operator.itruediv" => "Same as a /= b.", + "_operator.ixor" => "Same as a ^= b.", + "_operator.le" => "Same as a <= b.", + "_operator.length_hint" => "Return an estimate of the number of items in obj.\n\nThis is useful for presizing containers when building from an iterable.\n\nIf the object supports len(), the result will be exact.\nOtherwise, it may over- or under-estimate by an arbitrary amount.\nThe result will be an integer >= 0.", + "_operator.lshift" => "Same as a << b.", + "_operator.lt" => "Same as a < b.", + "_operator.matmul" => "Same as a @ b.", + "_operator.methodcaller" => "Return a callable object that calls the given method on its operand.\nAfter f = methodcaller('name'), the call f(r) returns r.name().\nAfter g = methodcaller('name', 'date', foo=1), the call g(r) returns\nr.name('date', foo=1).", + "_operator.methodcaller.__call__" => "Call self as a function.", + "_operator.methodcaller.__delattr__" => "Implement delattr(self, name).", + "_operator.methodcaller.__eq__" => "Return self==value.", + "_operator.methodcaller.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_operator.methodcaller.__ge__" => "Return self>=value.", + "_operator.methodcaller.__getattribute__" => "Return getattr(self, name).", + "_operator.methodcaller.__getstate__" => "Helper for pickle.", + "_operator.methodcaller.__gt__" => "Return self>value.", + "_operator.methodcaller.__hash__" => "Return hash(self).", + "_operator.methodcaller.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_operator.methodcaller.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_operator.methodcaller.__le__" => "Return self<=value.", + "_operator.methodcaller.__lt__" => "Return self<value.", + "_operator.methodcaller.__ne__" => "Return self!=value.", + "_operator.methodcaller.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_operator.methodcaller.__reduce__" => "Return state information for pickling", + "_operator.methodcaller.__reduce_ex__" => "Helper for pickle.", + "_operator.methodcaller.__repr__" => "Return repr(self).", + "_operator.methodcaller.__setattr__" => "Implement setattr(self, name, value).", + "_operator.methodcaller.__sizeof__" => "Size of object in memory, in bytes.", + "_operator.methodcaller.__str__" => "Return str(self).", + "_operator.methodcaller.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_operator.mod" => "Same as a % b.", + "_operator.mul" => "Same as a * b.", + "_operator.ne" => "Same as a != b.", + "_operator.neg" => "Same as -a.", + "_operator.not_" => "Same as not a.", + "_operator.or_" => "Same as a | b.", + "_operator.pos" => "Same as +a.", + "_operator.pow" => "Same as a ** b.", + "_operator.rshift" => "Same as a >> b.", + "_operator.setitem" => "Same as a[b] = c.", + "_operator.sub" => "Same as a - b.", + "_operator.truediv" => "Same as a / b.", + "_operator.truth" => "Return True if a is true, False otherwise.", + "_operator.xor" => "Same as a ^ b.", + "_overlapped.BindLocal" => "Bind a socket handle to an arbitrary local port.\n\nfamily should be AF_INET or AF_INET6.", + "_overlapped.ConnectPipe" => "Connect to the pipe for asynchronous I/O (overlapped).", + "_overlapped.CreateEvent" => "Create an event.\n\nEventAttributes must be None.", + "_overlapped.CreateIoCompletionPort" => "Create a completion port or register a handle with a port.", + "_overlapped.FormatMessage" => "Return error message for an error code.", + "_overlapped.GetQueuedCompletionStatus" => "Get a message from completion port.\n\nWait for up to msecs milliseconds.", + "_overlapped.Overlapped" => "OVERLAPPED structure wrapper.", + "_overlapped.Overlapped.AcceptEx" => "Start overlapped wait for client to connect.", + "_overlapped.Overlapped.ConnectEx" => "Start overlapped connect.\n\nclient_handle should be unbound.", + "_overlapped.Overlapped.ConnectNamedPipe" => "Start overlapped wait for a client to connect.", + "_overlapped.Overlapped.ReadFile" => "Start overlapped read.", + "_overlapped.Overlapped.ReadFileInto" => "Start overlapped receive.", + "_overlapped.Overlapped.TransmitFile" => "Transmit file data over a connected socket.", + "_overlapped.Overlapped.WSARecv" => "Start overlapped receive.", + "_overlapped.Overlapped.WSARecvFrom" => "Start overlapped receive.", + "_overlapped.Overlapped.WSARecvFromInto" => "Start overlapped receive.", + "_overlapped.Overlapped.WSARecvInto" => "Start overlapped receive.", + "_overlapped.Overlapped.WSASend" => "Start overlapped send.", + "_overlapped.Overlapped.WSASendTo" => "Start overlapped sendto over a connectionless (UDP) socket.", + "_overlapped.Overlapped.WriteFile" => "Start overlapped write.", + "_overlapped.Overlapped.__delattr__" => "Implement delattr(self, name).", + "_overlapped.Overlapped.__eq__" => "Return self==value.", + "_overlapped.Overlapped.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_overlapped.Overlapped.__ge__" => "Return self>=value.", + "_overlapped.Overlapped.__getattribute__" => "Return getattr(self, name).", + "_overlapped.Overlapped.__getstate__" => "Helper for pickle.", + "_overlapped.Overlapped.__gt__" => "Return self>value.", + "_overlapped.Overlapped.__hash__" => "Return hash(self).", + "_overlapped.Overlapped.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_overlapped.Overlapped.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_overlapped.Overlapped.__le__" => "Return self<=value.", + "_overlapped.Overlapped.__lt__" => "Return self<value.", + "_overlapped.Overlapped.__ne__" => "Return self!=value.", + "_overlapped.Overlapped.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_overlapped.Overlapped.__reduce__" => "Helper for pickle.", + "_overlapped.Overlapped.__reduce_ex__" => "Helper for pickle.", + "_overlapped.Overlapped.__repr__" => "Return repr(self).", + "_overlapped.Overlapped.__setattr__" => "Implement setattr(self, name, value).", + "_overlapped.Overlapped.__sizeof__" => "Size of object in memory, in bytes.", + "_overlapped.Overlapped.__str__" => "Return str(self).", + "_overlapped.Overlapped.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_overlapped.Overlapped.address" => "Address of overlapped structure", + "_overlapped.Overlapped.cancel" => "Cancel overlapped operation.", + "_overlapped.Overlapped.error" => "Error from last operation", + "_overlapped.Overlapped.event" => "Overlapped event handle", + "_overlapped.Overlapped.getresult" => "Retrieve result of operation.\n\nIf wait is true then it blocks until the operation is finished. If wait\nis false and the operation is still pending then an error is raised.", + "_overlapped.Overlapped.pending" => "Whether the operation is pending", + "_overlapped.PostQueuedCompletionStatus" => "Post a message to completion port.", + "_overlapped.RegisterWaitWithQueue" => "Register wait for Object; when complete CompletionPort is notified.", + "_overlapped.ResetEvent" => "Reset event.", + "_overlapped.SetEvent" => "Set event.", + "_overlapped.UnregisterWait" => "Unregister wait handle.", + "_overlapped.UnregisterWaitEx" => "Unregister wait handle.", + "_overlapped.WSAConnect" => "Bind a remote address to a connectionless (UDP) socket.", + "_pickle" => "Optimized C implementation for the Python pickle module.", + "_pickle.PickleBuffer" => "Wrapper for potentially out-of-band buffers", + "_pickle.PickleBuffer.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "_pickle.PickleBuffer.__delattr__" => "Implement delattr(self, name).", + "_pickle.PickleBuffer.__eq__" => "Return self==value.", + "_pickle.PickleBuffer.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_pickle.PickleBuffer.__ge__" => "Return self>=value.", + "_pickle.PickleBuffer.__getattribute__" => "Return getattr(self, name).", + "_pickle.PickleBuffer.__getstate__" => "Helper for pickle.", + "_pickle.PickleBuffer.__gt__" => "Return self>value.", + "_pickle.PickleBuffer.__hash__" => "Return hash(self).", + "_pickle.PickleBuffer.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_pickle.PickleBuffer.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_pickle.PickleBuffer.__le__" => "Return self<=value.", + "_pickle.PickleBuffer.__lt__" => "Return self<value.", + "_pickle.PickleBuffer.__ne__" => "Return self!=value.", + "_pickle.PickleBuffer.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_pickle.PickleBuffer.__reduce__" => "Helper for pickle.", + "_pickle.PickleBuffer.__reduce_ex__" => "Helper for pickle.", + "_pickle.PickleBuffer.__release_buffer__" => "Release the buffer object that exposes the underlying memory of the object.", + "_pickle.PickleBuffer.__repr__" => "Return repr(self).", + "_pickle.PickleBuffer.__setattr__" => "Implement setattr(self, name, value).", + "_pickle.PickleBuffer.__sizeof__" => "Size of object in memory, in bytes.", + "_pickle.PickleBuffer.__str__" => "Return str(self).", + "_pickle.PickleBuffer.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_pickle.PickleBuffer.raw" => "Return a memoryview of the raw memory underlying this buffer.\nWill raise BufferError is the buffer isn't contiguous.", + "_pickle.PickleBuffer.release" => "Release the underlying buffer exposed by the PickleBuffer object.", + "_pickle.PickleError.__delattr__" => "Implement delattr(self, name).", + "_pickle.PickleError.__eq__" => "Return self==value.", + "_pickle.PickleError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_pickle.PickleError.__ge__" => "Return self>=value.", + "_pickle.PickleError.__getattribute__" => "Return getattr(self, name).", + "_pickle.PickleError.__getstate__" => "Helper for pickle.", + "_pickle.PickleError.__gt__" => "Return self>value.", + "_pickle.PickleError.__hash__" => "Return hash(self).", + "_pickle.PickleError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_pickle.PickleError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_pickle.PickleError.__le__" => "Return self<=value.", + "_pickle.PickleError.__lt__" => "Return self<value.", + "_pickle.PickleError.__ne__" => "Return self!=value.", + "_pickle.PickleError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_pickle.PickleError.__reduce_ex__" => "Helper for pickle.", + "_pickle.PickleError.__repr__" => "Return repr(self).", + "_pickle.PickleError.__setattr__" => "Implement setattr(self, name, value).", + "_pickle.PickleError.__sizeof__" => "Size of object in memory, in bytes.", + "_pickle.PickleError.__str__" => "Return str(self).", + "_pickle.PickleError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_pickle.PickleError.__weakref__" => "list of weak references to the object", + "_pickle.PickleError.add_note" => "Add a note to the exception", + "_pickle.PickleError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_pickle.Pickler" => "This takes a binary file for writing a pickle data stream.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 5. It was introduced in Python 3.8, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nThe *file* argument must have a write() method that accepts a single\nbytes argument. It can thus be a file object opened for binary\nwriting, an io.BytesIO instance, or any other custom object that meets\nthis interface.\n\nIf *fix_imports* is True and protocol is less than 3, pickle will try\nto map the new Python 3 names to the old module names used in Python\n2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are\nserialized into *file* as part of the pickle stream.\n\nIf *buffer_callback* is not None, then it can be called any number\nof times with a buffer view. If the callback returns a false value\n(such as None), the given buffer is out-of-band; otherwise the\nbuffer is serialized in-band, i.e. inside the pickle stream.\n\nIt is an error if *buffer_callback* is not None and *protocol*\nis None or smaller than 5.", + "_pickle.Pickler.__delattr__" => "Implement delattr(self, name).", + "_pickle.Pickler.__eq__" => "Return self==value.", + "_pickle.Pickler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_pickle.Pickler.__ge__" => "Return self>=value.", + "_pickle.Pickler.__getattribute__" => "Return getattr(self, name).", + "_pickle.Pickler.__getstate__" => "Helper for pickle.", + "_pickle.Pickler.__gt__" => "Return self>value.", + "_pickle.Pickler.__hash__" => "Return hash(self).", + "_pickle.Pickler.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_pickle.Pickler.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_pickle.Pickler.__le__" => "Return self<=value.", + "_pickle.Pickler.__lt__" => "Return self<value.", + "_pickle.Pickler.__ne__" => "Return self!=value.", + "_pickle.Pickler.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_pickle.Pickler.__reduce__" => "Helper for pickle.", + "_pickle.Pickler.__reduce_ex__" => "Helper for pickle.", + "_pickle.Pickler.__repr__" => "Return repr(self).", + "_pickle.Pickler.__setattr__" => "Implement setattr(self, name, value).", + "_pickle.Pickler.__sizeof__" => "Returns size in memory, in bytes.", + "_pickle.Pickler.__str__" => "Return str(self).", + "_pickle.Pickler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_pickle.Pickler.clear_memo" => "Clears the pickler's \"memo\".\n\nThe memo is the data structure that remembers which objects the\npickler has already seen, so that shared or recursive objects are\npickled by reference and not by value. This method is useful when\nre-using picklers.", + "_pickle.Pickler.dump" => "Write a pickled representation of the given object to the open file.", + "_pickle.PicklingError.__delattr__" => "Implement delattr(self, name).", + "_pickle.PicklingError.__eq__" => "Return self==value.", + "_pickle.PicklingError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_pickle.PicklingError.__ge__" => "Return self>=value.", + "_pickle.PicklingError.__getattribute__" => "Return getattr(self, name).", + "_pickle.PicklingError.__getstate__" => "Helper for pickle.", + "_pickle.PicklingError.__gt__" => "Return self>value.", + "_pickle.PicklingError.__hash__" => "Return hash(self).", + "_pickle.PicklingError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_pickle.PicklingError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_pickle.PicklingError.__le__" => "Return self<=value.", + "_pickle.PicklingError.__lt__" => "Return self<value.", + "_pickle.PicklingError.__ne__" => "Return self!=value.", + "_pickle.PicklingError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_pickle.PicklingError.__reduce_ex__" => "Helper for pickle.", + "_pickle.PicklingError.__repr__" => "Return repr(self).", + "_pickle.PicklingError.__setattr__" => "Implement setattr(self, name, value).", + "_pickle.PicklingError.__sizeof__" => "Size of object in memory, in bytes.", + "_pickle.PicklingError.__str__" => "Return str(self).", + "_pickle.PicklingError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_pickle.PicklingError.__weakref__" => "list of weak references to the object", + "_pickle.PicklingError.add_note" => "Add a note to the exception", + "_pickle.PicklingError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_pickle.Unpickler" => "This takes a binary file for reading a pickle data stream.\n\nThe protocol version of the pickle is detected automatically, so no\nprotocol argument is needed. Bytes past the pickled object's\nrepresentation are ignored.\n\nThe argument *file* must have two methods, a read() method that takes\nan integer argument, and a readline() method that requires no\narguments. Both methods should return bytes. Thus *file* can be a\nbinary file object opened for reading, an io.BytesIO object, or any\nother custom object that meets this interface.\n\nOptional keyword arguments are *fix_imports*, *encoding* and *errors*,\nwhich are used to control compatibility support for pickle stream\ngenerated by Python 2. If *fix_imports* is True, pickle will try to\nmap the old Python 2 names to the new names used in Python 3. The\n*encoding* and *errors* tell pickle how to decode 8-bit string\ninstances pickled by Python 2; these default to 'ASCII' and 'strict',\nrespectively. The *encoding* can be 'bytes' to read these 8-bit\nstring instances as bytes objects.", + "_pickle.Unpickler.__delattr__" => "Implement delattr(self, name).", + "_pickle.Unpickler.__eq__" => "Return self==value.", + "_pickle.Unpickler.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_pickle.Unpickler.__ge__" => "Return self>=value.", + "_pickle.Unpickler.__getattribute__" => "Return getattr(self, name).", + "_pickle.Unpickler.__getstate__" => "Helper for pickle.", + "_pickle.Unpickler.__gt__" => "Return self>value.", + "_pickle.Unpickler.__hash__" => "Return hash(self).", + "_pickle.Unpickler.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_pickle.Unpickler.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_pickle.Unpickler.__le__" => "Return self<=value.", + "_pickle.Unpickler.__lt__" => "Return self<value.", + "_pickle.Unpickler.__ne__" => "Return self!=value.", + "_pickle.Unpickler.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_pickle.Unpickler.__reduce__" => "Helper for pickle.", + "_pickle.Unpickler.__reduce_ex__" => "Helper for pickle.", + "_pickle.Unpickler.__repr__" => "Return repr(self).", + "_pickle.Unpickler.__setattr__" => "Implement setattr(self, name, value).", + "_pickle.Unpickler.__sizeof__" => "Returns size in memory, in bytes.", + "_pickle.Unpickler.__str__" => "Return str(self).", + "_pickle.Unpickler.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_pickle.Unpickler.find_class" => "Return an object from a specified module.\n\nIf necessary, the module will be imported. Subclasses may override\nthis method (e.g. to restrict unpickling of arbitrary classes and\nfunctions).\n\nThis method is called whenever a class or a function object is\nneeded. Both arguments passed are str objects.", + "_pickle.Unpickler.load" => "Load a pickle.\n\nRead a pickled object representation from the open file object given\nin the constructor, and return the reconstituted object hierarchy\nspecified therein.", + "_pickle.UnpicklingError.__delattr__" => "Implement delattr(self, name).", + "_pickle.UnpicklingError.__eq__" => "Return self==value.", + "_pickle.UnpicklingError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_pickle.UnpicklingError.__ge__" => "Return self>=value.", + "_pickle.UnpicklingError.__getattribute__" => "Return getattr(self, name).", + "_pickle.UnpicklingError.__getstate__" => "Helper for pickle.", + "_pickle.UnpicklingError.__gt__" => "Return self>value.", + "_pickle.UnpicklingError.__hash__" => "Return hash(self).", + "_pickle.UnpicklingError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_pickle.UnpicklingError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_pickle.UnpicklingError.__le__" => "Return self<=value.", + "_pickle.UnpicklingError.__lt__" => "Return self<value.", + "_pickle.UnpicklingError.__ne__" => "Return self!=value.", + "_pickle.UnpicklingError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_pickle.UnpicklingError.__reduce_ex__" => "Helper for pickle.", + "_pickle.UnpicklingError.__repr__" => "Return repr(self).", + "_pickle.UnpicklingError.__setattr__" => "Implement setattr(self, name, value).", + "_pickle.UnpicklingError.__sizeof__" => "Size of object in memory, in bytes.", + "_pickle.UnpicklingError.__str__" => "Return str(self).", + "_pickle.UnpicklingError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_pickle.UnpicklingError.__weakref__" => "list of weak references to the object", + "_pickle.UnpicklingError.add_note" => "Add a note to the exception", + "_pickle.UnpicklingError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_pickle.dump" => "Write a pickled representation of obj to the open file object file.\n\nThis is equivalent to ``Pickler(file, protocol).dump(obj)``, but may\nbe more efficient.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 5. It was introduced in Python 3.8, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nThe *file* argument must have a write() method that accepts a single\nbytes argument. It can thus be a file object opened for binary\nwriting, an io.BytesIO instance, or any other custom object that meets\nthis interface.\n\nIf *fix_imports* is True and protocol is less than 3, pickle will try\nto map the new Python 3 names to the old module names used in Python\n2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are serialized\ninto *file* as part of the pickle stream. It is an error if\n*buffer_callback* is not None and *protocol* is None or smaller than 5.", + "_pickle.dumps" => "Return the pickled representation of the object as a bytes object.\n\nThe optional *protocol* argument tells the pickler to use the given\nprotocol; supported protocols are 0, 1, 2, 3, 4 and 5. The default\nprotocol is 5. It was introduced in Python 3.8, and is incompatible\nwith previous versions.\n\nSpecifying a negative protocol version selects the highest protocol\nversion supported. The higher the protocol used, the more recent the\nversion of Python needed to read the pickle produced.\n\nIf *fix_imports* is True and *protocol* is less than 3, pickle will\ntry to map the new Python 3 names to the old module names used in\nPython 2, so that the pickle data stream is readable with Python 2.\n\nIf *buffer_callback* is None (the default), buffer views are serialized\ninto *file* as part of the pickle stream. It is an error if\n*buffer_callback* is not None and *protocol* is None or smaller than 5.", + "_pickle.load" => "Read and return an object from the pickle data stored in a file.\n\nThis is equivalent to ``Unpickler(file).load()``, but may be more\nefficient.\n\nThe protocol version of the pickle is detected automatically, so no\nprotocol argument is needed. Bytes past the pickled object's\nrepresentation are ignored.\n\nThe argument *file* must have two methods, a read() method that takes\nan integer argument, and a readline() method that requires no\narguments. Both methods should return bytes. Thus *file* can be a\nbinary file object opened for reading, an io.BytesIO object, or any\nother custom object that meets this interface.\n\nOptional keyword arguments are *fix_imports*, *encoding* and *errors*,\nwhich are used to control compatibility support for pickle stream\ngenerated by Python 2. If *fix_imports* is True, pickle will try to\nmap the old Python 2 names to the new names used in Python 3. The\n*encoding* and *errors* tell pickle how to decode 8-bit string\ninstances pickled by Python 2; these default to 'ASCII' and 'strict',\nrespectively. The *encoding* can be 'bytes' to read these 8-bit\nstring instances as bytes objects.", + "_pickle.loads" => "Read and return an object from the given pickle data.\n\nThe protocol version of the pickle is detected automatically, so no\nprotocol argument is needed. Bytes past the pickled object's\nrepresentation are ignored.\n\nOptional keyword arguments are *fix_imports*, *encoding* and *errors*,\nwhich are used to control compatibility support for pickle stream\ngenerated by Python 2. If *fix_imports* is True, pickle will try to\nmap the old Python 2 names to the new names used in Python 3. The\n*encoding* and *errors* tell pickle how to decode 8-bit string\ninstances pickled by Python 2; these default to 'ASCII' and 'strict',\nrespectively. The *encoding* can be 'bytes' to read these 8-bit\nstring instances as bytes objects.", + "_posixshmem" => "POSIX shared memory module", + "_posixshmem.shm_open" => "Open a shared memory object. Returns a file descriptor (integer).", + "_posixshmem.shm_unlink" => "Remove a shared memory object (similar to unlink()).\n\nRemove a shared memory object name, and, once all processes have unmapped\nthe object, de-allocates and destroys the contents of the associated memory\nregion.", + "_posixsubprocess" => "A POSIX helper for the subprocess module.", + "_posixsubprocess.fork_exec" => "Spawn a fresh new child process.\n\nFork a child process, close parent file descriptors as appropriate in the\nchild and duplicate the few that are needed before calling exec() in the\nchild process.\n\nIf close_fds is True, close file descriptors 3 and higher, except those listed\nin the sorted tuple pass_fds.\n\nThe preexec_fn, if supplied, will be called immediately before closing file\ndescriptors and exec.\n\nWARNING: preexec_fn is NOT SAFE if your application uses threads.\n It may trigger infrequent, difficult to debug deadlocks.\n\nIf an error occurs in the child process before the exec, it is\nserialized and written to the errpipe_write fd per subprocess.py.\n\nReturns: the child process's PID.\n\nRaises: Only on an error in the parent process.", + "_queue" => "C implementation of the Python queue module.\nThis module is an implementation detail, please do not use it directly.", + "_queue.Empty" => "Exception raised by Queue.get(block=0)/get_nowait().", + "_queue.Empty.__delattr__" => "Implement delattr(self, name).", + "_queue.Empty.__eq__" => "Return self==value.", + "_queue.Empty.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_queue.Empty.__ge__" => "Return self>=value.", + "_queue.Empty.__getattribute__" => "Return getattr(self, name).", + "_queue.Empty.__getstate__" => "Helper for pickle.", + "_queue.Empty.__gt__" => "Return self>value.", + "_queue.Empty.__hash__" => "Return hash(self).", + "_queue.Empty.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_queue.Empty.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_queue.Empty.__le__" => "Return self<=value.", + "_queue.Empty.__lt__" => "Return self<value.", + "_queue.Empty.__ne__" => "Return self!=value.", + "_queue.Empty.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_queue.Empty.__reduce_ex__" => "Helper for pickle.", + "_queue.Empty.__repr__" => "Return repr(self).", + "_queue.Empty.__setattr__" => "Implement setattr(self, name, value).", + "_queue.Empty.__sizeof__" => "Size of object in memory, in bytes.", + "_queue.Empty.__str__" => "Return str(self).", + "_queue.Empty.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_queue.Empty.__weakref__" => "list of weak references to the object", + "_queue.Empty.add_note" => "Add a note to the exception", + "_queue.Empty.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_queue.SimpleQueue" => "Simple, unbounded, reentrant FIFO queue.", + "_queue.SimpleQueue.__class_getitem__" => "See PEP 585", + "_queue.SimpleQueue.__delattr__" => "Implement delattr(self, name).", + "_queue.SimpleQueue.__eq__" => "Return self==value.", + "_queue.SimpleQueue.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_queue.SimpleQueue.__ge__" => "Return self>=value.", + "_queue.SimpleQueue.__getattribute__" => "Return getattr(self, name).", + "_queue.SimpleQueue.__getstate__" => "Helper for pickle.", + "_queue.SimpleQueue.__gt__" => "Return self>value.", + "_queue.SimpleQueue.__hash__" => "Return hash(self).", + "_queue.SimpleQueue.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_queue.SimpleQueue.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_queue.SimpleQueue.__le__" => "Return self<=value.", + "_queue.SimpleQueue.__lt__" => "Return self<value.", + "_queue.SimpleQueue.__ne__" => "Return self!=value.", + "_queue.SimpleQueue.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_queue.SimpleQueue.__reduce__" => "Helper for pickle.", + "_queue.SimpleQueue.__reduce_ex__" => "Helper for pickle.", + "_queue.SimpleQueue.__repr__" => "Return repr(self).", + "_queue.SimpleQueue.__setattr__" => "Implement setattr(self, name, value).", + "_queue.SimpleQueue.__sizeof__" => "Size of object in memory, in bytes.", + "_queue.SimpleQueue.__str__" => "Return str(self).", + "_queue.SimpleQueue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_queue.SimpleQueue.empty" => "Return True if the queue is empty, False otherwise (not reliable!).", + "_queue.SimpleQueue.get" => "Remove and return an item from the queue.\n\nIf optional args 'block' is true and 'timeout' is None (the default),\nblock if necessary until an item is available. If 'timeout' is\na non-negative number, it blocks at most 'timeout' seconds and raises\nthe Empty exception if no item was available within that time.\nOtherwise ('block' is false), return an item if one is immediately\navailable, else raise the Empty exception ('timeout' is ignored\nin that case).", + "_queue.SimpleQueue.get_nowait" => "Remove and return an item from the queue without blocking.\n\nOnly get an item if one is immediately available. Otherwise\nraise the Empty exception.", + "_queue.SimpleQueue.put" => "Put the item on the queue.\n\nThe optional 'block' and 'timeout' arguments are ignored, as this method\nnever blocks. They are provided for compatibility with the Queue class.", + "_queue.SimpleQueue.put_nowait" => "Put an item into the queue without blocking.\n\nThis is exactly equivalent to `put(item)` and is only provided\nfor compatibility with the Queue class.", + "_queue.SimpleQueue.qsize" => "Return the approximate size of the queue (not reliable!).", + "_random" => "Module implements the Mersenne Twister random number generator.", + "_random.Random" => "Random() -> create a random number generator with its own internal state.", + "_random.Random.__delattr__" => "Implement delattr(self, name).", + "_random.Random.__eq__" => "Return self==value.", + "_random.Random.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_random.Random.__ge__" => "Return self>=value.", + "_random.Random.__getattribute__" => "Return getattr(self, name).", + "_random.Random.__getstate__" => "Helper for pickle.", + "_random.Random.__gt__" => "Return self>value.", + "_random.Random.__hash__" => "Return hash(self).", + "_random.Random.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_random.Random.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_random.Random.__le__" => "Return self<=value.", + "_random.Random.__lt__" => "Return self<value.", + "_random.Random.__ne__" => "Return self!=value.", + "_random.Random.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_random.Random.__reduce__" => "Helper for pickle.", + "_random.Random.__reduce_ex__" => "Helper for pickle.", + "_random.Random.__repr__" => "Return repr(self).", + "_random.Random.__setattr__" => "Implement setattr(self, name, value).", + "_random.Random.__sizeof__" => "Size of object in memory, in bytes.", + "_random.Random.__str__" => "Return str(self).", + "_random.Random.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_random.Random.getrandbits" => "getrandbits(k) -> x. Generates an int with k random bits.", + "_random.Random.getstate" => "getstate() -> tuple containing the current state.", + "_random.Random.random" => "random() -> x in the interval [0, 1).", + "_random.Random.seed" => "seed([n]) -> None.\n\nDefaults to use urandom and falls back to a combination\nof the current time and the process identifier.", + "_random.Random.setstate" => "setstate(state) -> None. Restores generator state.", + "_remote_debugging.AwaitedInfo" => "Information about what a thread is awaiting", + "_remote_debugging.AwaitedInfo.__add__" => "Return self+value.", + "_remote_debugging.AwaitedInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.AwaitedInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.AwaitedInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.AwaitedInfo.__eq__" => "Return self==value.", + "_remote_debugging.AwaitedInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.AwaitedInfo.__ge__" => "Return self>=value.", + "_remote_debugging.AwaitedInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.AwaitedInfo.__getitem__" => "Return self[key].", + "_remote_debugging.AwaitedInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.AwaitedInfo.__gt__" => "Return self>value.", + "_remote_debugging.AwaitedInfo.__hash__" => "Return hash(self).", + "_remote_debugging.AwaitedInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.AwaitedInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.AwaitedInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.AwaitedInfo.__le__" => "Return self<=value.", + "_remote_debugging.AwaitedInfo.__len__" => "Return len(self).", + "_remote_debugging.AwaitedInfo.__lt__" => "Return self<value.", + "_remote_debugging.AwaitedInfo.__mul__" => "Return self*value.", + "_remote_debugging.AwaitedInfo.__ne__" => "Return self!=value.", + "_remote_debugging.AwaitedInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.AwaitedInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.AwaitedInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.AwaitedInfo.__repr__" => "Return repr(self).", + "_remote_debugging.AwaitedInfo.__rmul__" => "Return value*self.", + "_remote_debugging.AwaitedInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.AwaitedInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.AwaitedInfo.__str__" => "Return str(self).", + "_remote_debugging.AwaitedInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.AwaitedInfo.awaited_by" => "List of tasks awaited by this thread", + "_remote_debugging.AwaitedInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.AwaitedInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.AwaitedInfo.thread_id" => "Thread ID", + "_remote_debugging.CoroInfo" => "Information about a coroutine", + "_remote_debugging.CoroInfo.__add__" => "Return self+value.", + "_remote_debugging.CoroInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.CoroInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.CoroInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.CoroInfo.__eq__" => "Return self==value.", + "_remote_debugging.CoroInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.CoroInfo.__ge__" => "Return self>=value.", + "_remote_debugging.CoroInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.CoroInfo.__getitem__" => "Return self[key].", + "_remote_debugging.CoroInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.CoroInfo.__gt__" => "Return self>value.", + "_remote_debugging.CoroInfo.__hash__" => "Return hash(self).", + "_remote_debugging.CoroInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.CoroInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.CoroInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.CoroInfo.__le__" => "Return self<=value.", + "_remote_debugging.CoroInfo.__len__" => "Return len(self).", + "_remote_debugging.CoroInfo.__lt__" => "Return self<value.", + "_remote_debugging.CoroInfo.__mul__" => "Return self*value.", + "_remote_debugging.CoroInfo.__ne__" => "Return self!=value.", + "_remote_debugging.CoroInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.CoroInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.CoroInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.CoroInfo.__repr__" => "Return repr(self).", + "_remote_debugging.CoroInfo.__rmul__" => "Return value*self.", + "_remote_debugging.CoroInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.CoroInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.CoroInfo.__str__" => "Return str(self).", + "_remote_debugging.CoroInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.CoroInfo.call_stack" => "Coroutine call stack", + "_remote_debugging.CoroInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.CoroInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.CoroInfo.task_name" => "Task name", + "_remote_debugging.FrameInfo" => "Information about a frame", + "_remote_debugging.FrameInfo.__add__" => "Return self+value.", + "_remote_debugging.FrameInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.FrameInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.FrameInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.FrameInfo.__eq__" => "Return self==value.", + "_remote_debugging.FrameInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.FrameInfo.__ge__" => "Return self>=value.", + "_remote_debugging.FrameInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.FrameInfo.__getitem__" => "Return self[key].", + "_remote_debugging.FrameInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.FrameInfo.__gt__" => "Return self>value.", + "_remote_debugging.FrameInfo.__hash__" => "Return hash(self).", + "_remote_debugging.FrameInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.FrameInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.FrameInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.FrameInfo.__le__" => "Return self<=value.", + "_remote_debugging.FrameInfo.__len__" => "Return len(self).", + "_remote_debugging.FrameInfo.__lt__" => "Return self<value.", + "_remote_debugging.FrameInfo.__mul__" => "Return self*value.", + "_remote_debugging.FrameInfo.__ne__" => "Return self!=value.", + "_remote_debugging.FrameInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.FrameInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.FrameInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.FrameInfo.__repr__" => "Return repr(self).", + "_remote_debugging.FrameInfo.__rmul__" => "Return value*self.", + "_remote_debugging.FrameInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.FrameInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.FrameInfo.__str__" => "Return str(self).", + "_remote_debugging.FrameInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.FrameInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.FrameInfo.filename" => "Source code filename", + "_remote_debugging.FrameInfo.funcname" => "Function name", + "_remote_debugging.FrameInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.FrameInfo.lineno" => "Line number", + "_remote_debugging.RemoteUnwinder" => "RemoteUnwinder(pid): Inspect stack of a remote Python process.", + "_remote_debugging.RemoteUnwinder.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.RemoteUnwinder.__eq__" => "Return self==value.", + "_remote_debugging.RemoteUnwinder.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.RemoteUnwinder.__ge__" => "Return self>=value.", + "_remote_debugging.RemoteUnwinder.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.RemoteUnwinder.__getstate__" => "Helper for pickle.", + "_remote_debugging.RemoteUnwinder.__gt__" => "Return self>value.", + "_remote_debugging.RemoteUnwinder.__hash__" => "Return hash(self).", + "_remote_debugging.RemoteUnwinder.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.RemoteUnwinder.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.RemoteUnwinder.__le__" => "Return self<=value.", + "_remote_debugging.RemoteUnwinder.__lt__" => "Return self<value.", + "_remote_debugging.RemoteUnwinder.__ne__" => "Return self!=value.", + "_remote_debugging.RemoteUnwinder.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.RemoteUnwinder.__reduce__" => "Helper for pickle.", + "_remote_debugging.RemoteUnwinder.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.RemoteUnwinder.__repr__" => "Return repr(self).", + "_remote_debugging.RemoteUnwinder.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.RemoteUnwinder.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.RemoteUnwinder.__str__" => "Return str(self).", + "_remote_debugging.RemoteUnwinder.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.RemoteUnwinder.get_all_awaited_by" => "Get all tasks and their awaited_by relationships from the remote process.\n\nThis provides a tree structure showing which tasks are waiting for other tasks.\n\nFor each task, returns:\n1. The call stack frames leading to where the task is currently executing\n2. The name of the task\n3. A list of tasks that this task is waiting for, with their own frames/names/etc\n\nReturns a list of [frames, task_name, subtasks] where:\n- frames: List of (func_name, filename, lineno) showing the call stack\n- task_name: String identifier for the task\n- subtasks: List of tasks being awaited by this task, in same format\n\nRaises:\n RuntimeError: If AsyncioDebug section is not available in the remote process\n MemoryError: If memory allocation fails\n OSError: If reading from the remote process fails\n\nExample output:\n[\n [\n [(\"c5\", \"script.py\", 10), (\"c4\", \"script.py\", 14)],\n \"c2_root\",\n [\n [\n [(\"c1\", \"script.py\", 23)],\n \"sub_main_2\",\n [...]\n ],\n [...]\n ]\n ]\n]", + "_remote_debugging.RemoteUnwinder.get_async_stack_trace" => "Get the currently running async tasks and their dependency graphs from the remote process.\n\nThis returns information about running tasks and all tasks that are waiting for them,\nforming a complete dependency graph for each thread's active task.\n\nFor each thread with a running task, returns the running task plus all tasks that\ntransitively depend on it (tasks waiting for the running task, tasks waiting for\nthose tasks, etc.).\n\nReturns a list of per-thread results, where each thread result contains:\n- Thread ID\n- List of task information for the running task and all its waiters\n\nEach task info contains:\n- Task ID (memory address)\n- Task name\n- Call stack frames: List of (func_name, filename, lineno)\n- List of tasks waiting for this task (recursive structure)\n\nRaises:\n RuntimeError: If AsyncioDebug section is not available in the target process\n MemoryError: If memory allocation fails\n OSError: If reading from the remote process fails\n\nExample output (similar structure to get_all_awaited_by but only for running tasks):\n[\n (140234, [\n (4345585712, 'main_task',\n [(\"run_server\", \"server.py\", 127), (\"main\", \"app.py\", 23)],\n [\n (4345585800, 'worker_1', [...], [...]),\n (4345585900, 'worker_2', [...], [...])\n ])\n ])\n]", + "_remote_debugging.RemoteUnwinder.get_stack_trace" => "Returns a list of stack traces for threads in the target process.\n\nEach element in the returned list is a tuple of (thread_id, frame_list), where:\n- thread_id is the OS thread identifier\n- frame_list is a list of tuples (function_name, filename, line_number) representing\n the Python stack frames for that thread, ordered from most recent to oldest\n\nThe threads returned depend on the initialization parameters:\n- If only_active_thread was True: returns only the thread holding the GIL\n- If all_threads was True: returns all threads\n- Otherwise: returns only the main thread\n\nExample:\n [\n (1234, [\n ('process_data', 'worker.py', 127),\n ('run_worker', 'worker.py', 45),\n ('main', 'app.py', 23)\n ]),\n (1235, [\n ('handle_request', 'server.py', 89),\n ('serve_forever', 'server.py', 52)\n ])\n ]\n\nRaises:\n RuntimeError: If there is an error copying memory from the target process\n OSError: If there is an error accessing the target process\n PermissionError: If access to the target process is denied\n UnicodeDecodeError: If there is an error decoding strings from the target process", + "_remote_debugging.TaskInfo" => "Information about an asyncio task", + "_remote_debugging.TaskInfo.__add__" => "Return self+value.", + "_remote_debugging.TaskInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.TaskInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.TaskInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.TaskInfo.__eq__" => "Return self==value.", + "_remote_debugging.TaskInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.TaskInfo.__ge__" => "Return self>=value.", + "_remote_debugging.TaskInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.TaskInfo.__getitem__" => "Return self[key].", + "_remote_debugging.TaskInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.TaskInfo.__gt__" => "Return self>value.", + "_remote_debugging.TaskInfo.__hash__" => "Return hash(self).", + "_remote_debugging.TaskInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.TaskInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.TaskInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.TaskInfo.__le__" => "Return self<=value.", + "_remote_debugging.TaskInfo.__len__" => "Return len(self).", + "_remote_debugging.TaskInfo.__lt__" => "Return self<value.", + "_remote_debugging.TaskInfo.__mul__" => "Return self*value.", + "_remote_debugging.TaskInfo.__ne__" => "Return self!=value.", + "_remote_debugging.TaskInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.TaskInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.TaskInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.TaskInfo.__repr__" => "Return repr(self).", + "_remote_debugging.TaskInfo.__rmul__" => "Return value*self.", + "_remote_debugging.TaskInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.TaskInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.TaskInfo.__str__" => "Return str(self).", + "_remote_debugging.TaskInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.TaskInfo.awaited_by" => "Tasks awaiting this task", + "_remote_debugging.TaskInfo.coroutine_stack" => "Coroutine call stack", + "_remote_debugging.TaskInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.TaskInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.TaskInfo.task_id" => "Task ID (memory address)", + "_remote_debugging.TaskInfo.task_name" => "Task name", + "_remote_debugging.ThreadInfo" => "Information about a thread", + "_remote_debugging.ThreadInfo.__add__" => "Return self+value.", + "_remote_debugging.ThreadInfo.__class_getitem__" => "See PEP 585", + "_remote_debugging.ThreadInfo.__contains__" => "Return bool(key in self).", + "_remote_debugging.ThreadInfo.__delattr__" => "Implement delattr(self, name).", + "_remote_debugging.ThreadInfo.__eq__" => "Return self==value.", + "_remote_debugging.ThreadInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_remote_debugging.ThreadInfo.__ge__" => "Return self>=value.", + "_remote_debugging.ThreadInfo.__getattribute__" => "Return getattr(self, name).", + "_remote_debugging.ThreadInfo.__getitem__" => "Return self[key].", + "_remote_debugging.ThreadInfo.__getstate__" => "Helper for pickle.", + "_remote_debugging.ThreadInfo.__gt__" => "Return self>value.", + "_remote_debugging.ThreadInfo.__hash__" => "Return hash(self).", + "_remote_debugging.ThreadInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_remote_debugging.ThreadInfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_remote_debugging.ThreadInfo.__iter__" => "Implement iter(self).", + "_remote_debugging.ThreadInfo.__le__" => "Return self<=value.", + "_remote_debugging.ThreadInfo.__len__" => "Return len(self).", + "_remote_debugging.ThreadInfo.__lt__" => "Return self<value.", + "_remote_debugging.ThreadInfo.__mul__" => "Return self*value.", + "_remote_debugging.ThreadInfo.__ne__" => "Return self!=value.", + "_remote_debugging.ThreadInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_remote_debugging.ThreadInfo.__reduce_ex__" => "Helper for pickle.", + "_remote_debugging.ThreadInfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_remote_debugging.ThreadInfo.__repr__" => "Return repr(self).", + "_remote_debugging.ThreadInfo.__rmul__" => "Return value*self.", + "_remote_debugging.ThreadInfo.__setattr__" => "Implement setattr(self, name, value).", + "_remote_debugging.ThreadInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_remote_debugging.ThreadInfo.__str__" => "Return str(self).", + "_remote_debugging.ThreadInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_remote_debugging.ThreadInfo.count" => "Return number of occurrences of value.", + "_remote_debugging.ThreadInfo.frame_info" => "Frame information", + "_remote_debugging.ThreadInfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_remote_debugging.ThreadInfo.thread_id" => "Thread ID", + "_sha1.SHA1Type.__delattr__" => "Implement delattr(self, name).", + "_sha1.SHA1Type.__eq__" => "Return self==value.", + "_sha1.SHA1Type.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha1.SHA1Type.__ge__" => "Return self>=value.", + "_sha1.SHA1Type.__getattribute__" => "Return getattr(self, name).", + "_sha1.SHA1Type.__getstate__" => "Helper for pickle.", + "_sha1.SHA1Type.__gt__" => "Return self>value.", + "_sha1.SHA1Type.__hash__" => "Return hash(self).", + "_sha1.SHA1Type.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha1.SHA1Type.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha1.SHA1Type.__le__" => "Return self<=value.", + "_sha1.SHA1Type.__lt__" => "Return self<value.", + "_sha1.SHA1Type.__ne__" => "Return self!=value.", + "_sha1.SHA1Type.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha1.SHA1Type.__reduce__" => "Helper for pickle.", + "_sha1.SHA1Type.__reduce_ex__" => "Helper for pickle.", + "_sha1.SHA1Type.__repr__" => "Return repr(self).", + "_sha1.SHA1Type.__setattr__" => "Implement setattr(self, name, value).", + "_sha1.SHA1Type.__sizeof__" => "Size of object in memory, in bytes.", + "_sha1.SHA1Type.__str__" => "Return str(self).", + "_sha1.SHA1Type.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha1.SHA1Type.copy" => "Return a copy of the hash object.", + "_sha1.SHA1Type.digest" => "Return the digest value as a bytes object.", + "_sha1.SHA1Type.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha1.SHA1Type.update" => "Update this hash object's state with the provided string.", + "_sha1.sha1" => "Return a new SHA1 hash object; optionally initialized with a string.", + "_sha2.SHA224Type.__delattr__" => "Implement delattr(self, name).", + "_sha2.SHA224Type.__eq__" => "Return self==value.", + "_sha2.SHA224Type.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha2.SHA224Type.__ge__" => "Return self>=value.", + "_sha2.SHA224Type.__getattribute__" => "Return getattr(self, name).", + "_sha2.SHA224Type.__getstate__" => "Helper for pickle.", + "_sha2.SHA224Type.__gt__" => "Return self>value.", + "_sha2.SHA224Type.__hash__" => "Return hash(self).", + "_sha2.SHA224Type.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha2.SHA224Type.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha2.SHA224Type.__le__" => "Return self<=value.", + "_sha2.SHA224Type.__lt__" => "Return self<value.", + "_sha2.SHA224Type.__ne__" => "Return self!=value.", + "_sha2.SHA224Type.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha2.SHA224Type.__reduce__" => "Helper for pickle.", + "_sha2.SHA224Type.__reduce_ex__" => "Helper for pickle.", + "_sha2.SHA224Type.__repr__" => "Return repr(self).", + "_sha2.SHA224Type.__setattr__" => "Implement setattr(self, name, value).", + "_sha2.SHA224Type.__sizeof__" => "Size of object in memory, in bytes.", + "_sha2.SHA224Type.__str__" => "Return str(self).", + "_sha2.SHA224Type.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha2.SHA224Type.copy" => "Return a copy of the hash object.", + "_sha2.SHA224Type.digest" => "Return the digest value as a bytes object.", + "_sha2.SHA224Type.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha2.SHA224Type.update" => "Update this hash object's state with the provided string.", + "_sha2.SHA256Type.__delattr__" => "Implement delattr(self, name).", + "_sha2.SHA256Type.__eq__" => "Return self==value.", + "_sha2.SHA256Type.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha2.SHA256Type.__ge__" => "Return self>=value.", + "_sha2.SHA256Type.__getattribute__" => "Return getattr(self, name).", + "_sha2.SHA256Type.__getstate__" => "Helper for pickle.", + "_sha2.SHA256Type.__gt__" => "Return self>value.", + "_sha2.SHA256Type.__hash__" => "Return hash(self).", + "_sha2.SHA256Type.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha2.SHA256Type.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha2.SHA256Type.__le__" => "Return self<=value.", + "_sha2.SHA256Type.__lt__" => "Return self<value.", + "_sha2.SHA256Type.__ne__" => "Return self!=value.", + "_sha2.SHA256Type.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha2.SHA256Type.__reduce__" => "Helper for pickle.", + "_sha2.SHA256Type.__reduce_ex__" => "Helper for pickle.", + "_sha2.SHA256Type.__repr__" => "Return repr(self).", + "_sha2.SHA256Type.__setattr__" => "Implement setattr(self, name, value).", + "_sha2.SHA256Type.__sizeof__" => "Size of object in memory, in bytes.", + "_sha2.SHA256Type.__str__" => "Return str(self).", + "_sha2.SHA256Type.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha2.SHA256Type.copy" => "Return a copy of the hash object.", + "_sha2.SHA256Type.digest" => "Return the digest value as a bytes object.", + "_sha2.SHA256Type.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha2.SHA256Type.update" => "Update this hash object's state with the provided string.", + "_sha2.SHA384Type.__delattr__" => "Implement delattr(self, name).", + "_sha2.SHA384Type.__eq__" => "Return self==value.", + "_sha2.SHA384Type.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha2.SHA384Type.__ge__" => "Return self>=value.", + "_sha2.SHA384Type.__getattribute__" => "Return getattr(self, name).", + "_sha2.SHA384Type.__getstate__" => "Helper for pickle.", + "_sha2.SHA384Type.__gt__" => "Return self>value.", + "_sha2.SHA384Type.__hash__" => "Return hash(self).", + "_sha2.SHA384Type.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha2.SHA384Type.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha2.SHA384Type.__le__" => "Return self<=value.", + "_sha2.SHA384Type.__lt__" => "Return self<value.", + "_sha2.SHA384Type.__ne__" => "Return self!=value.", + "_sha2.SHA384Type.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha2.SHA384Type.__reduce__" => "Helper for pickle.", + "_sha2.SHA384Type.__reduce_ex__" => "Helper for pickle.", + "_sha2.SHA384Type.__repr__" => "Return repr(self).", + "_sha2.SHA384Type.__setattr__" => "Implement setattr(self, name, value).", + "_sha2.SHA384Type.__sizeof__" => "Size of object in memory, in bytes.", + "_sha2.SHA384Type.__str__" => "Return str(self).", + "_sha2.SHA384Type.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha2.SHA384Type.copy" => "Return a copy of the hash object.", + "_sha2.SHA384Type.digest" => "Return the digest value as a bytes object.", + "_sha2.SHA384Type.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha2.SHA384Type.update" => "Update this hash object's state with the provided string.", + "_sha2.SHA512Type.__delattr__" => "Implement delattr(self, name).", + "_sha2.SHA512Type.__eq__" => "Return self==value.", + "_sha2.SHA512Type.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha2.SHA512Type.__ge__" => "Return self>=value.", + "_sha2.SHA512Type.__getattribute__" => "Return getattr(self, name).", + "_sha2.SHA512Type.__getstate__" => "Helper for pickle.", + "_sha2.SHA512Type.__gt__" => "Return self>value.", + "_sha2.SHA512Type.__hash__" => "Return hash(self).", + "_sha2.SHA512Type.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha2.SHA512Type.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha2.SHA512Type.__le__" => "Return self<=value.", + "_sha2.SHA512Type.__lt__" => "Return self<value.", + "_sha2.SHA512Type.__ne__" => "Return self!=value.", + "_sha2.SHA512Type.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha2.SHA512Type.__reduce__" => "Helper for pickle.", + "_sha2.SHA512Type.__reduce_ex__" => "Helper for pickle.", + "_sha2.SHA512Type.__repr__" => "Return repr(self).", + "_sha2.SHA512Type.__setattr__" => "Implement setattr(self, name, value).", + "_sha2.SHA512Type.__sizeof__" => "Size of object in memory, in bytes.", + "_sha2.SHA512Type.__str__" => "Return str(self).", + "_sha2.SHA512Type.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha2.SHA512Type.copy" => "Return a copy of the hash object.", + "_sha2.SHA512Type.digest" => "Return the digest value as a bytes object.", + "_sha2.SHA512Type.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha2.SHA512Type.update" => "Update this hash object's state with the provided string.", + "_sha2.sha224" => "Return a new SHA-224 hash object; optionally initialized with a string.", + "_sha2.sha256" => "Return a new SHA-256 hash object; optionally initialized with a string.", + "_sha2.sha384" => "Return a new SHA-384 hash object; optionally initialized with a string.", + "_sha2.sha512" => "Return a new SHA-512 hash object; optionally initialized with a string.", + "_sha3.sha3_224" => "sha3_224([data], *, usedforsecurity=True) -> SHA3 object\n\nReturn a new SHA3 hash object with a hashbit length of 28 bytes.", + "_sha3.sha3_224.__delattr__" => "Implement delattr(self, name).", + "_sha3.sha3_224.__eq__" => "Return self==value.", + "_sha3.sha3_224.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha3.sha3_224.__ge__" => "Return self>=value.", + "_sha3.sha3_224.__getattribute__" => "Return getattr(self, name).", + "_sha3.sha3_224.__getstate__" => "Helper for pickle.", + "_sha3.sha3_224.__gt__" => "Return self>value.", + "_sha3.sha3_224.__hash__" => "Return hash(self).", + "_sha3.sha3_224.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha3.sha3_224.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha3.sha3_224.__le__" => "Return self<=value.", + "_sha3.sha3_224.__lt__" => "Return self<value.", + "_sha3.sha3_224.__ne__" => "Return self!=value.", + "_sha3.sha3_224.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha3.sha3_224.__reduce__" => "Helper for pickle.", + "_sha3.sha3_224.__reduce_ex__" => "Helper for pickle.", + "_sha3.sha3_224.__repr__" => "Return repr(self).", + "_sha3.sha3_224.__setattr__" => "Implement setattr(self, name, value).", + "_sha3.sha3_224.__sizeof__" => "Size of object in memory, in bytes.", + "_sha3.sha3_224.__str__" => "Return str(self).", + "_sha3.sha3_224.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha3.sha3_224.copy" => "Return a copy of the hash object.", + "_sha3.sha3_224.digest" => "Return the digest value as a bytes object.", + "_sha3.sha3_224.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha3.sha3_224.update" => "Update this hash object's state with the provided bytes-like object.", + "_sha3.sha3_256" => "sha3_256([data], *, usedforsecurity=True) -> SHA3 object\n\nReturn a new SHA3 hash object with a hashbit length of 32 bytes.", + "_sha3.sha3_256.__delattr__" => "Implement delattr(self, name).", + "_sha3.sha3_256.__eq__" => "Return self==value.", + "_sha3.sha3_256.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha3.sha3_256.__ge__" => "Return self>=value.", + "_sha3.sha3_256.__getattribute__" => "Return getattr(self, name).", + "_sha3.sha3_256.__getstate__" => "Helper for pickle.", + "_sha3.sha3_256.__gt__" => "Return self>value.", + "_sha3.sha3_256.__hash__" => "Return hash(self).", + "_sha3.sha3_256.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha3.sha3_256.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha3.sha3_256.__le__" => "Return self<=value.", + "_sha3.sha3_256.__lt__" => "Return self<value.", + "_sha3.sha3_256.__ne__" => "Return self!=value.", + "_sha3.sha3_256.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha3.sha3_256.__reduce__" => "Helper for pickle.", + "_sha3.sha3_256.__reduce_ex__" => "Helper for pickle.", + "_sha3.sha3_256.__repr__" => "Return repr(self).", + "_sha3.sha3_256.__setattr__" => "Implement setattr(self, name, value).", + "_sha3.sha3_256.__sizeof__" => "Size of object in memory, in bytes.", + "_sha3.sha3_256.__str__" => "Return str(self).", + "_sha3.sha3_256.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha3.sha3_256.copy" => "Return a copy of the hash object.", + "_sha3.sha3_256.digest" => "Return the digest value as a bytes object.", + "_sha3.sha3_256.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha3.sha3_256.update" => "Update this hash object's state with the provided bytes-like object.", + "_sha3.sha3_384" => "sha3_384([data], *, usedforsecurity=True) -> SHA3 object\n\nReturn a new SHA3 hash object with a hashbit length of 48 bytes.", + "_sha3.sha3_384.__delattr__" => "Implement delattr(self, name).", + "_sha3.sha3_384.__eq__" => "Return self==value.", + "_sha3.sha3_384.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha3.sha3_384.__ge__" => "Return self>=value.", + "_sha3.sha3_384.__getattribute__" => "Return getattr(self, name).", + "_sha3.sha3_384.__getstate__" => "Helper for pickle.", + "_sha3.sha3_384.__gt__" => "Return self>value.", + "_sha3.sha3_384.__hash__" => "Return hash(self).", + "_sha3.sha3_384.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha3.sha3_384.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha3.sha3_384.__le__" => "Return self<=value.", + "_sha3.sha3_384.__lt__" => "Return self<value.", + "_sha3.sha3_384.__ne__" => "Return self!=value.", + "_sha3.sha3_384.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha3.sha3_384.__reduce__" => "Helper for pickle.", + "_sha3.sha3_384.__reduce_ex__" => "Helper for pickle.", + "_sha3.sha3_384.__repr__" => "Return repr(self).", + "_sha3.sha3_384.__setattr__" => "Implement setattr(self, name, value).", + "_sha3.sha3_384.__sizeof__" => "Size of object in memory, in bytes.", + "_sha3.sha3_384.__str__" => "Return str(self).", + "_sha3.sha3_384.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha3.sha3_384.copy" => "Return a copy of the hash object.", + "_sha3.sha3_384.digest" => "Return the digest value as a bytes object.", + "_sha3.sha3_384.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha3.sha3_384.update" => "Update this hash object's state with the provided bytes-like object.", + "_sha3.sha3_512" => "sha3_512([data], *, usedforsecurity=True) -> SHA3 object\n\nReturn a new SHA3 hash object with a hashbit length of 64 bytes.", + "_sha3.sha3_512.__delattr__" => "Implement delattr(self, name).", + "_sha3.sha3_512.__eq__" => "Return self==value.", + "_sha3.sha3_512.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha3.sha3_512.__ge__" => "Return self>=value.", + "_sha3.sha3_512.__getattribute__" => "Return getattr(self, name).", + "_sha3.sha3_512.__getstate__" => "Helper for pickle.", + "_sha3.sha3_512.__gt__" => "Return self>value.", + "_sha3.sha3_512.__hash__" => "Return hash(self).", + "_sha3.sha3_512.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha3.sha3_512.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha3.sha3_512.__le__" => "Return self<=value.", + "_sha3.sha3_512.__lt__" => "Return self<value.", + "_sha3.sha3_512.__ne__" => "Return self!=value.", + "_sha3.sha3_512.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha3.sha3_512.__reduce__" => "Helper for pickle.", + "_sha3.sha3_512.__reduce_ex__" => "Helper for pickle.", + "_sha3.sha3_512.__repr__" => "Return repr(self).", + "_sha3.sha3_512.__setattr__" => "Implement setattr(self, name, value).", + "_sha3.sha3_512.__sizeof__" => "Size of object in memory, in bytes.", + "_sha3.sha3_512.__str__" => "Return str(self).", + "_sha3.sha3_512.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha3.sha3_512.copy" => "Return a copy of the hash object.", + "_sha3.sha3_512.digest" => "Return the digest value as a bytes object.", + "_sha3.sha3_512.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha3.sha3_512.update" => "Update this hash object's state with the provided bytes-like object.", + "_sha3.shake_128" => "shake_128([data], *, usedforsecurity=True) -> SHAKE object\n\nReturn a new SHAKE hash object.", + "_sha3.shake_128.__delattr__" => "Implement delattr(self, name).", + "_sha3.shake_128.__eq__" => "Return self==value.", + "_sha3.shake_128.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha3.shake_128.__ge__" => "Return self>=value.", + "_sha3.shake_128.__getattribute__" => "Return getattr(self, name).", + "_sha3.shake_128.__getstate__" => "Helper for pickle.", + "_sha3.shake_128.__gt__" => "Return self>value.", + "_sha3.shake_128.__hash__" => "Return hash(self).", + "_sha3.shake_128.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha3.shake_128.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha3.shake_128.__le__" => "Return self<=value.", + "_sha3.shake_128.__lt__" => "Return self<value.", + "_sha3.shake_128.__ne__" => "Return self!=value.", + "_sha3.shake_128.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha3.shake_128.__reduce__" => "Helper for pickle.", + "_sha3.shake_128.__reduce_ex__" => "Helper for pickle.", + "_sha3.shake_128.__repr__" => "Return repr(self).", + "_sha3.shake_128.__setattr__" => "Implement setattr(self, name, value).", + "_sha3.shake_128.__sizeof__" => "Size of object in memory, in bytes.", + "_sha3.shake_128.__str__" => "Return str(self).", + "_sha3.shake_128.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha3.shake_128.copy" => "Return a copy of the hash object.", + "_sha3.shake_128.digest" => "Return the digest value as a bytes object.", + "_sha3.shake_128.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha3.shake_128.update" => "Update this hash object's state with the provided bytes-like object.", + "_sha3.shake_256" => "shake_256([data], *, usedforsecurity=True) -> SHAKE object\n\nReturn a new SHAKE hash object.", + "_sha3.shake_256.__delattr__" => "Implement delattr(self, name).", + "_sha3.shake_256.__eq__" => "Return self==value.", + "_sha3.shake_256.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sha3.shake_256.__ge__" => "Return self>=value.", + "_sha3.shake_256.__getattribute__" => "Return getattr(self, name).", + "_sha3.shake_256.__getstate__" => "Helper for pickle.", + "_sha3.shake_256.__gt__" => "Return self>value.", + "_sha3.shake_256.__hash__" => "Return hash(self).", + "_sha3.shake_256.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sha3.shake_256.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sha3.shake_256.__le__" => "Return self<=value.", + "_sha3.shake_256.__lt__" => "Return self<value.", + "_sha3.shake_256.__ne__" => "Return self!=value.", + "_sha3.shake_256.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sha3.shake_256.__reduce__" => "Helper for pickle.", + "_sha3.shake_256.__reduce_ex__" => "Helper for pickle.", + "_sha3.shake_256.__repr__" => "Return repr(self).", + "_sha3.shake_256.__setattr__" => "Implement setattr(self, name, value).", + "_sha3.shake_256.__sizeof__" => "Size of object in memory, in bytes.", + "_sha3.shake_256.__str__" => "Return str(self).", + "_sha3.shake_256.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sha3.shake_256.copy" => "Return a copy of the hash object.", + "_sha3.shake_256.digest" => "Return the digest value as a bytes object.", + "_sha3.shake_256.hexdigest" => "Return the digest value as a string of hexadecimal digits.", + "_sha3.shake_256.update" => "Update this hash object's state with the provided bytes-like object.", + "_signal" => "This module provides mechanisms to use signal handlers in Python.\n\nFunctions:\n\nalarm() -- cause SIGALRM after a specified time [Unix only]\nsetitimer() -- cause a signal (described below) after a specified\n float time and the timer may restart then [Unix only]\ngetitimer() -- get current value of timer [Unix only]\nsignal() -- set the action for a given signal\ngetsignal() -- get the signal action for a given signal\npause() -- wait until a signal arrives [Unix only]\ndefault_int_handler() -- default SIGINT handler\n\nsignal constants:\nSIG_DFL -- used to refer to the system default handler\nSIG_IGN -- used to ignore the signal\nNSIG -- number of defined signals\nSIGINT, SIGTERM, etc. -- signal numbers\n\nitimer constants:\nITIMER_REAL -- decrements in real time, and delivers SIGALRM upon\n expiration\nITIMER_VIRTUAL -- decrements only when the process is executing,\n and delivers SIGVTALRM upon expiration\nITIMER_PROF -- decrements both when the process is executing and\n when the system is executing on behalf of the process.\n Coupled with ITIMER_VIRTUAL, this timer is usually\n used to profile the time spent by the application\n in user and kernel space. SIGPROF is delivered upon\n expiration.\n\n\n*** IMPORTANT NOTICE ***\nA signal handler function is called with two arguments:\nthe first is the signal number, the second is the interrupted stack frame.", + "_signal.ItimerError.__delattr__" => "Implement delattr(self, name).", + "_signal.ItimerError.__eq__" => "Return self==value.", + "_signal.ItimerError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_signal.ItimerError.__ge__" => "Return self>=value.", + "_signal.ItimerError.__getattribute__" => "Return getattr(self, name).", + "_signal.ItimerError.__getstate__" => "Helper for pickle.", + "_signal.ItimerError.__gt__" => "Return self>value.", + "_signal.ItimerError.__hash__" => "Return hash(self).", + "_signal.ItimerError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_signal.ItimerError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_signal.ItimerError.__le__" => "Return self<=value.", + "_signal.ItimerError.__lt__" => "Return self<value.", + "_signal.ItimerError.__ne__" => "Return self!=value.", + "_signal.ItimerError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_signal.ItimerError.__reduce_ex__" => "Helper for pickle.", + "_signal.ItimerError.__repr__" => "Return repr(self).", + "_signal.ItimerError.__setattr__" => "Implement setattr(self, name, value).", + "_signal.ItimerError.__sizeof__" => "Size of object in memory, in bytes.", + "_signal.ItimerError.__str__" => "Return str(self).", + "_signal.ItimerError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_signal.ItimerError.__weakref__" => "list of weak references to the object", + "_signal.ItimerError.add_note" => "Add a note to the exception", + "_signal.ItimerError.errno" => "POSIX exception code", + "_signal.ItimerError.filename" => "exception filename", + "_signal.ItimerError.filename2" => "second exception filename", + "_signal.ItimerError.strerror" => "exception strerror", + "_signal.ItimerError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_signal.alarm" => "Arrange for SIGALRM to arrive after the given number of seconds.", + "_signal.default_int_handler" => "The default handler for SIGINT installed by Python.\n\nIt raises KeyboardInterrupt.", + "_signal.getitimer" => "Returns current value of given itimer.", + "_signal.getsignal" => "Return the current action for the given signal.\n\nThe return value can be:\n SIG_IGN -- if the signal is being ignored\n SIG_DFL -- if the default action for the signal is in effect\n None -- if an unknown handler is in effect\n anything else -- the callable Python object used as a handler", + "_signal.pause" => "Wait until a signal arrives.", + "_signal.pidfd_send_signal" => "Send a signal to a process referred to by a pid file descriptor.", + "_signal.pthread_kill" => "Send a signal to a thread.", + "_signal.pthread_sigmask" => "Fetch and/or change the signal mask of the calling thread.", + "_signal.raise_signal" => "Send a signal to the executing process.", + "_signal.set_wakeup_fd" => "Sets the fd to be written to (with the signal number) when a signal comes in.\n\nA library can use this to wakeup select or poll.\nThe previous fd or -1 is returned.\n\nThe fd must be non-blocking.", + "_signal.setitimer" => "Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF).\n\nThe timer will fire after value seconds and after that every interval seconds.\nThe itimer can be cleared by setting seconds to zero.\n\nReturns old values as a tuple: (delay, interval).", + "_signal.siginterrupt" => "Change system call restart behaviour.\n\nIf flag is False, system calls will be restarted when interrupted by\nsignal sig, else system calls will be interrupted.", + "_signal.signal" => "Set the action for the given signal.\n\nThe action can be SIG_DFL, SIG_IGN, or a callable Python object.\nThe previous action is returned. See getsignal() for possible return values.\n\n*** IMPORTANT NOTICE ***\nA signal handler function is called with two arguments:\nthe first is the signal number, the second is the interrupted stack frame.", + "_signal.sigpending" => "Examine pending signals.\n\nReturns a set of signal numbers that are pending for delivery to\nthe calling thread.", + "_signal.sigtimedwait" => "Like sigwaitinfo(), but with a timeout.\n\nThe timeout is specified in seconds, with floating-point numbers allowed.", + "_signal.sigwait" => "Wait for a signal.\n\nSuspend execution of the calling thread until the delivery of one of the\nsignals specified in the signal set sigset. The function accepts the signal\nand returns the signal number.", + "_signal.sigwaitinfo" => "Wait synchronously until one of the signals in *sigset* is delivered.\n\nReturns a struct_siginfo containing information about the signal.", + "_signal.strsignal" => "Return the system description of the given signal.\n\nReturns the description of signal *signalnum*, such as \"Interrupt\"\nfor :const:`SIGINT`. Returns :const:`None` if *signalnum* has no\ndescription. Raises :exc:`ValueError` if *signalnum* is invalid.", + "_signal.struct_siginfo" => "struct_siginfo: Result from sigwaitinfo or sigtimedwait.\n\nThis object may be accessed either as a tuple of\n(si_signo, si_code, si_errno, si_pid, si_uid, si_status, si_band),\nor via the attributes si_signo, si_code, and so on.", + "_signal.struct_siginfo.__add__" => "Return self+value.", + "_signal.struct_siginfo.__class_getitem__" => "See PEP 585", + "_signal.struct_siginfo.__contains__" => "Return bool(key in self).", + "_signal.struct_siginfo.__delattr__" => "Implement delattr(self, name).", + "_signal.struct_siginfo.__eq__" => "Return self==value.", + "_signal.struct_siginfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_signal.struct_siginfo.__ge__" => "Return self>=value.", + "_signal.struct_siginfo.__getattribute__" => "Return getattr(self, name).", + "_signal.struct_siginfo.__getitem__" => "Return self[key].", + "_signal.struct_siginfo.__getstate__" => "Helper for pickle.", + "_signal.struct_siginfo.__gt__" => "Return self>value.", + "_signal.struct_siginfo.__hash__" => "Return hash(self).", + "_signal.struct_siginfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_signal.struct_siginfo.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_signal.struct_siginfo.__iter__" => "Implement iter(self).", + "_signal.struct_siginfo.__le__" => "Return self<=value.", + "_signal.struct_siginfo.__len__" => "Return len(self).", + "_signal.struct_siginfo.__lt__" => "Return self<value.", + "_signal.struct_siginfo.__mul__" => "Return self*value.", + "_signal.struct_siginfo.__ne__" => "Return self!=value.", + "_signal.struct_siginfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_signal.struct_siginfo.__reduce_ex__" => "Helper for pickle.", + "_signal.struct_siginfo.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_signal.struct_siginfo.__repr__" => "Return repr(self).", + "_signal.struct_siginfo.__rmul__" => "Return value*self.", + "_signal.struct_siginfo.__setattr__" => "Implement setattr(self, name, value).", + "_signal.struct_siginfo.__sizeof__" => "Size of object in memory, in bytes.", + "_signal.struct_siginfo.__str__" => "Return str(self).", + "_signal.struct_siginfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_signal.struct_siginfo.count" => "Return number of occurrences of value.", + "_signal.struct_siginfo.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_signal.struct_siginfo.si_band" => "band event for SIGPOLL", + "_signal.struct_siginfo.si_code" => "signal code", + "_signal.struct_siginfo.si_errno" => "errno associated with this signal", + "_signal.struct_siginfo.si_pid" => "sending process ID", + "_signal.struct_siginfo.si_signo" => "signal number", + "_signal.struct_siginfo.si_status" => "exit value or signal", + "_signal.struct_siginfo.si_uid" => "real user ID of sending process", + "_signal.valid_signals" => "Return a set of valid signal numbers on this platform.\n\nThe signal numbers returned by this function can be safely passed to\nfunctions like `pthread_sigmask`.", + "_socket" => "Implementation module for socket operations.\n\nSee the socket module for documentation.", + "_socket.CMSG_LEN" => "CMSG_LEN(length) -> control message length\n\nReturn the total length, without trailing padding, of an ancillary\ndata item with associated data of the given length. This value can\noften be used as the buffer size for recvmsg() to receive a single\nitem of ancillary data, but RFC 3542 requires portable applications to\nuse CMSG_SPACE() and thus include space for padding, even when the\nitem will be the last in the buffer. Raises OverflowError if length\nis outside the permissible range of values.", + "_socket.CMSG_SPACE" => "CMSG_SPACE(length) -> buffer size\n\nReturn the buffer size needed for recvmsg() to receive an ancillary\ndata item with associated data of the given length, along with any\ntrailing padding. The buffer space needed to receive multiple items\nis the sum of the CMSG_SPACE() values for their associated data\nlengths. Raises OverflowError if length is outside the permissible\nrange of values.", + "_socket.SocketType" => "socket(family=AF_INET, type=SOCK_STREAM, proto=0) -> socket object\nsocket(family=-1, type=-1, proto=-1, fileno=None) -> socket object\n\nOpen a socket of the given type. The family argument specifies the\naddress family; it defaults to AF_INET. The type argument specifies\nwhether this is a stream (SOCK_STREAM, this is the default)\nor datagram (SOCK_DGRAM) socket. The protocol argument defaults to 0,\nspecifying the default protocol. Keyword arguments are accepted.\nThe socket is created as non-inheritable.\n\nWhen a fileno is passed in, family, type and proto are auto-detected,\nunless they are explicitly set.\n\nA socket object represents one endpoint of a network connection.\n\nMethods of socket objects (keyword arguments not allowed):\n\n_accept() -- accept connection, returning new socket fd and client address\nbind(addr) -- bind the socket to a local address\nclose() -- close the socket\nconnect(addr) -- connect the socket to a remote address\nconnect_ex(addr) -- connect, return an error code instead of an exception\ndup() -- return a new socket fd duplicated from fileno()\nfileno() -- return underlying file descriptor\ngetpeername() -- return remote address [*]\ngetsockname() -- return local address\ngetsockopt(level, optname[, buflen]) -- get socket options\ngettimeout() -- return timeout or None\nlisten([n]) -- start listening for incoming connections\nrecv(buflen[, flags]) -- receive data\nrecv_into(buffer[, nbytes[, flags]]) -- receive data (into a buffer)\nrecvfrom(buflen[, flags]) -- receive data and sender's address\nrecvfrom_into(buffer[, nbytes, [, flags])\n -- receive data and sender's address (into a buffer)\nsendall(data[, flags]) -- send all data\nsend(data[, flags]) -- send data, may not send all of it\nsendto(data[, flags], addr) -- send data to a given address\nsetblocking(bool) -- set or clear the blocking I/O flag\ngetblocking() -- return True if socket is blocking, False if non-blocking\nsetsockopt(level, optname, value[, optlen]) -- set socket options\nsettimeout(None | float) -- set or clear the timeout\nshutdown(how) -- shut down traffic in one or both directions\n\n [*] not available on all platforms!", + "_socket.SocketType.__del__" => "Called when the instance is about to be destroyed.", + "_socket.SocketType.__delattr__" => "Implement delattr(self, name).", + "_socket.SocketType.__eq__" => "Return self==value.", + "_socket.SocketType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_socket.SocketType.__ge__" => "Return self>=value.", + "_socket.SocketType.__getattribute__" => "Return getattr(self, name).", + "_socket.SocketType.__getstate__" => "Helper for pickle.", + "_socket.SocketType.__gt__" => "Return self>value.", + "_socket.SocketType.__hash__" => "Return hash(self).", + "_socket.SocketType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_socket.SocketType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_socket.SocketType.__le__" => "Return self<=value.", + "_socket.SocketType.__lt__" => "Return self<value.", + "_socket.SocketType.__ne__" => "Return self!=value.", + "_socket.SocketType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_socket.SocketType.__reduce__" => "Helper for pickle.", + "_socket.SocketType.__reduce_ex__" => "Helper for pickle.", + "_socket.SocketType.__repr__" => "Return repr(self).", + "_socket.SocketType.__setattr__" => "Implement setattr(self, name, value).", + "_socket.SocketType.__sizeof__" => "Size of object in memory, in bytes.", + "_socket.SocketType.__str__" => "Return str(self).", + "_socket.SocketType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_socket.SocketType._accept" => "_accept() -> (integer, address info)\n\nWait for an incoming connection. Return a new socket file descriptor\nrepresenting the connection, and the address of the client.\nFor IP sockets, the address info is a pair (hostaddr, port).", + "_socket.SocketType.bind" => "bind(address)\n\nBind the socket to a local address. For IP sockets, the address is a\npair (host, port); the host must refer to the local host. For raw packet\nsockets the address is a tuple (ifname, proto [,pkttype [,hatype [,addr]]])", + "_socket.SocketType.close" => "close()\n\nClose the socket. It cannot be used after this call.", + "_socket.SocketType.connect" => "connect(address)\n\nConnect the socket to a remote address. For IP sockets, the address\nis a pair (host, port).", + "_socket.SocketType.connect_ex" => "connect_ex(address) -> errno\n\nThis is like connect(address), but returns an error code (the errno value)\ninstead of raising an exception when an error occurs.", + "_socket.SocketType.detach" => "detach()\n\nClose the socket object without closing the underlying file descriptor.\nThe object cannot be used after this call, but the file descriptor\ncan be reused for other purposes. The file descriptor is returned.", + "_socket.SocketType.family" => "the socket family", + "_socket.SocketType.fileno" => "fileno() -> integer\n\nReturn the integer file descriptor of the socket.", + "_socket.SocketType.getblocking" => "getblocking()\n\nReturns True if socket is in blocking mode, or False if it\nis in non-blocking mode.", + "_socket.SocketType.getpeername" => "getpeername() -> address info\n\nReturn the address of the remote endpoint. For IP sockets, the address\ninfo is a pair (hostaddr, port).", + "_socket.SocketType.getsockname" => "getsockname() -> address info\n\nReturn the address of the local endpoint. The format depends on the\naddress family. For IPv4 sockets, the address info is a pair\n(hostaddr, port). For IPv6 sockets, the address info is a 4-tuple\n(hostaddr, port, flowinfo, scope_id).", + "_socket.SocketType.getsockopt" => "getsockopt(level, option[, buffersize]) -> value\n\nGet a socket option. See the Unix manual for level and option.\nIf a nonzero buffersize argument is given, the return value is a\nstring of that length; otherwise it is an integer.", + "_socket.SocketType.gettimeout" => "gettimeout() -> timeout\n\nReturns the timeout in seconds (float) associated with socket\noperations. A timeout of None indicates that timeouts on socket\noperations are disabled.", + "_socket.SocketType.ioctl" => "ioctl(cmd, option) -> long\n\nControl the socket with WSAIoctl syscall. Currently supported 'cmd' values are\nSIO_RCVALL: 'option' must be one of the socket.RCVALL_* constants.\nSIO_KEEPALIVE_VALS: 'option' is a tuple of (onoff, timeout, interval).\nSIO_LOOPBACK_FAST_PATH: 'option' is a boolean value, and is disabled by default", + "_socket.SocketType.listen" => "listen([backlog])\n\nEnable a server to accept connections. If backlog is specified, it must be\nat least 0 (if it is lower, it is set to 0); it specifies the number of\nunaccepted connections that the system will allow before refusing new\nconnections. If not specified, a default reasonable value is chosen.", + "_socket.SocketType.proto" => "the socket protocol", + "_socket.SocketType.recv" => "recv(buffersize[, flags]) -> data\n\nReceive up to buffersize bytes from the socket. For the optional flags\nargument, see the Unix manual. When no data is available, block until\nat least one byte is available or until the remote end is closed. When\nthe remote end is closed and all data is read, return the empty string.", + "_socket.SocketType.recv_into" => "recv_into(buffer, [nbytes[, flags]]) -> nbytes_read\n\nA version of recv() that stores its data into a buffer rather than creating\na new string. Receive up to buffersize bytes from the socket. If buffersize\nis not specified (or 0), receive up to the size available in the given buffer.\n\nSee recv() for documentation about the flags.", + "_socket.SocketType.recvfrom" => "recvfrom(buffersize[, flags]) -> (data, address info)\n\nLike recv(buffersize, flags) but also return the sender's address info.", + "_socket.SocketType.recvfrom_into" => "recvfrom_into(buffer[, nbytes[, flags]]) -> (nbytes, address info)\n\nLike recv_into(buffer[, nbytes[, flags]]) but also return the sender's address info.", + "_socket.SocketType.recvmsg" => "recvmsg(bufsize[, ancbufsize[, flags]]) -> (data, ancdata, msg_flags, address)\n\nReceive normal data (up to bufsize bytes) and ancillary data from the\nsocket. The ancbufsize argument sets the size in bytes of the\ninternal buffer used to receive the ancillary data; it defaults to 0,\nmeaning that no ancillary data will be received. Appropriate buffer\nsizes for ancillary data can be calculated using CMSG_SPACE() or\nCMSG_LEN(), and items which do not fit into the buffer might be\ntruncated or discarded. The flags argument defaults to 0 and has the\nsame meaning as for recv().\n\nThe return value is a 4-tuple: (data, ancdata, msg_flags, address).\nThe data item is a bytes object holding the non-ancillary data\nreceived. The ancdata item is a list of zero or more tuples\n(cmsg_level, cmsg_type, cmsg_data) representing the ancillary data\n(control messages) received: cmsg_level and cmsg_type are integers\nspecifying the protocol level and protocol-specific type respectively,\nand cmsg_data is a bytes object holding the associated data. The\nmsg_flags item is the bitwise OR of various flags indicating\nconditions on the received message; see your system documentation for\ndetails. If the receiving socket is unconnected, address is the\naddress of the sending socket, if available; otherwise, its value is\nunspecified.\n\nIf recvmsg() raises an exception after the system call returns, it\nwill first attempt to close any file descriptors received via the\nSCM_RIGHTS mechanism.", + "_socket.SocketType.recvmsg_into" => "recvmsg_into(buffers[, ancbufsize[, flags]]) -> (nbytes, ancdata, msg_flags, address)\n\nReceive normal data and ancillary data from the socket, scattering the\nnon-ancillary data into a series of buffers. The buffers argument\nmust be an iterable of objects that export writable buffers\n(e.g. bytearray objects); these will be filled with successive chunks\nof the non-ancillary data until it has all been written or there are\nno more buffers. The ancbufsize argument sets the size in bytes of\nthe internal buffer used to receive the ancillary data; it defaults to\n0, meaning that no ancillary data will be received. Appropriate\nbuffer sizes for ancillary data can be calculated using CMSG_SPACE()\nor CMSG_LEN(), and items which do not fit into the buffer might be\ntruncated or discarded. The flags argument defaults to 0 and has the\nsame meaning as for recv().\n\nThe return value is a 4-tuple: (nbytes, ancdata, msg_flags, address).\nThe nbytes item is the total number of bytes of non-ancillary data\nwritten into the buffers. The ancdata item is a list of zero or more\ntuples (cmsg_level, cmsg_type, cmsg_data) representing the ancillary\ndata (control messages) received: cmsg_level and cmsg_type are\nintegers specifying the protocol level and protocol-specific type\nrespectively, and cmsg_data is a bytes object holding the associated\ndata. The msg_flags item is the bitwise OR of various flags\nindicating conditions on the received message; see your system\ndocumentation for details. If the receiving socket is unconnected,\naddress is the address of the sending socket, if available; otherwise,\nits value is unspecified.\n\nIf recvmsg_into() raises an exception after the system call returns,\nit will first attempt to close any file descriptors received via the\nSCM_RIGHTS mechanism.", + "_socket.SocketType.send" => "send(data[, flags]) -> count\n\nSend a data string to the socket. For the optional flags\nargument, see the Unix manual. Return the number of bytes\nsent; this may be less than len(data) if the network is busy.", + "_socket.SocketType.sendall" => "sendall(data[, flags])\n\nSend a data string to the socket. For the optional flags\nargument, see the Unix manual. This calls send() repeatedly\nuntil all data is sent. If an error occurs, it's impossible\nto tell how much data has been sent.", + "_socket.SocketType.sendmsg" => "sendmsg(buffers[, ancdata[, flags[, address]]]) -> count\n\nSend normal and ancillary data to the socket, gathering the\nnon-ancillary data from a series of buffers and concatenating it into\na single message. The buffers argument specifies the non-ancillary\ndata as an iterable of bytes-like objects (e.g. bytes objects).\nThe ancdata argument specifies the ancillary data (control messages)\nas an iterable of zero or more tuples (cmsg_level, cmsg_type,\ncmsg_data), where cmsg_level and cmsg_type are integers specifying the\nprotocol level and protocol-specific type respectively, and cmsg_data\nis a bytes-like object holding the associated data. The flags\nargument defaults to 0 and has the same meaning as for send(). If\naddress is supplied and not None, it sets a destination address for\nthe message. The return value is the number of bytes of non-ancillary\ndata sent.", + "_socket.SocketType.sendmsg_afalg" => "sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags=MSG_MORE]]])\n\nSet operation mode, IV and length of associated data for an AF_ALG\noperation socket.", + "_socket.SocketType.sendto" => "sendto(data[, flags], address) -> count\n\nLike send(data, flags) but allows specifying the destination address.\nFor IP sockets, the address is a pair (hostaddr, port).", + "_socket.SocketType.setblocking" => "setblocking(flag)\n\nSet the socket to blocking (flag is true) or non-blocking (false).\nsetblocking(True) is equivalent to settimeout(None);\nsetblocking(False) is equivalent to settimeout(0.0).", + "_socket.SocketType.setsockopt" => "setsockopt(level, option, value: int)\nsetsockopt(level, option, value: buffer)\nsetsockopt(level, option, None, optlen: int)\n\nSet a socket option. See the Unix manual for level and option.\nThe value argument can either be an integer, a string buffer, or\nNone, optlen.", + "_socket.SocketType.settimeout" => "settimeout(timeout)\n\nSet a timeout on socket operations. 'timeout' can be a float,\ngiving in seconds, or None. Setting a timeout of None disables\nthe timeout feature and is equivalent to setblocking(1).\nSetting a timeout of zero is the same as setblocking(0).", + "_socket.SocketType.share" => "share(process_id) -> bytes\n\nShare the socket with another process. The target process id\nmust be provided and the resulting bytes object passed to the target\nprocess. There the shared socket can be instantiated by calling\nsocket.fromshare().", + "_socket.SocketType.shutdown" => "shutdown(flag)\n\nShut down the reading side of the socket (flag == SHUT_RD), the writing side\nof the socket (flag == SHUT_WR), or both ends (flag == SHUT_RDWR).", + "_socket.SocketType.timeout" => "the socket timeout", + "_socket.SocketType.type" => "the socket type", + "_socket.close" => "close(integer) -> None\n\nClose an integer socket file descriptor. This is like os.close(), but for\nsockets; on some platforms os.close() won't work for socket file descriptors.", + "_socket.dup" => "dup(integer) -> integer\n\nDuplicate an integer socket file descriptor. This is like os.dup(), but for\nsockets; on some platforms os.dup() won't work for socket file descriptors.", + "_socket.gaierror.__delattr__" => "Implement delattr(self, name).", + "_socket.gaierror.__eq__" => "Return self==value.", + "_socket.gaierror.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_socket.gaierror.__ge__" => "Return self>=value.", + "_socket.gaierror.__getattribute__" => "Return getattr(self, name).", + "_socket.gaierror.__getstate__" => "Helper for pickle.", + "_socket.gaierror.__gt__" => "Return self>value.", + "_socket.gaierror.__hash__" => "Return hash(self).", + "_socket.gaierror.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_socket.gaierror.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_socket.gaierror.__le__" => "Return self<=value.", + "_socket.gaierror.__lt__" => "Return self<value.", + "_socket.gaierror.__ne__" => "Return self!=value.", + "_socket.gaierror.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_socket.gaierror.__reduce_ex__" => "Helper for pickle.", + "_socket.gaierror.__repr__" => "Return repr(self).", + "_socket.gaierror.__setattr__" => "Implement setattr(self, name, value).", + "_socket.gaierror.__sizeof__" => "Size of object in memory, in bytes.", + "_socket.gaierror.__str__" => "Return str(self).", + "_socket.gaierror.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_socket.gaierror.__weakref__" => "list of weak references to the object", + "_socket.gaierror.add_note" => "Add a note to the exception", + "_socket.gaierror.errno" => "POSIX exception code", + "_socket.gaierror.filename" => "exception filename", + "_socket.gaierror.filename2" => "second exception filename", + "_socket.gaierror.strerror" => "exception strerror", + "_socket.gaierror.winerror" => "Win32 exception code", + "_socket.gaierror.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_socket.getaddrinfo" => "getaddrinfo(host, port [, family, type, proto, flags])\n -> list of (family, type, proto, canonname, sockaddr)\n\nResolve host and port into addrinfo struct.", + "_socket.getdefaulttimeout" => "getdefaulttimeout() -> timeout\n\nReturns the default timeout in seconds (float) for new socket objects.\nA value of None indicates that new socket objects have no timeout.\nWhen the socket module is first imported, the default is None.", + "_socket.gethostbyaddr" => "gethostbyaddr(host) -> (name, aliaslist, addresslist)\n\nReturn the true host name, a list of aliases, and a list of IP addresses,\nfor a host. The host argument is a string giving a host name or IP number.", + "_socket.gethostbyname" => "gethostbyname(host) -> address\n\nReturn the IP address (a string of the form '255.255.255.255') for a host.", + "_socket.gethostbyname_ex" => "gethostbyname_ex(host) -> (name, aliaslist, addresslist)\n\nReturn the true host name, a list of aliases, and a list of IP addresses,\nfor a host. The host argument is a string giving a host name or IP number.", + "_socket.gethostname" => "gethostname() -> string\n\nReturn the current host name.", + "_socket.getnameinfo" => "getnameinfo(sockaddr, flags) --> (host, port)\n\nGet host and port for a sockaddr.", + "_socket.getprotobyname" => "getprotobyname(name) -> integer\n\nReturn the protocol number for the named protocol. (Rarely used.)", + "_socket.getservbyname" => "getservbyname(servicename[, protocolname]) -> integer\n\nReturn a port number from a service name and protocol name.\nThe optional protocol name, if given, should be 'tcp' or 'udp',\notherwise any protocol will match.", + "_socket.getservbyport" => "getservbyport(port[, protocolname]) -> string\n\nReturn the service name from a port number and protocol name.\nThe optional protocol name, if given, should be 'tcp' or 'udp',\notherwise any protocol will match.", + "_socket.herror.__delattr__" => "Implement delattr(self, name).", + "_socket.herror.__eq__" => "Return self==value.", + "_socket.herror.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_socket.herror.__ge__" => "Return self>=value.", + "_socket.herror.__getattribute__" => "Return getattr(self, name).", + "_socket.herror.__getstate__" => "Helper for pickle.", + "_socket.herror.__gt__" => "Return self>value.", + "_socket.herror.__hash__" => "Return hash(self).", + "_socket.herror.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_socket.herror.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_socket.herror.__le__" => "Return self<=value.", + "_socket.herror.__lt__" => "Return self<value.", + "_socket.herror.__ne__" => "Return self!=value.", + "_socket.herror.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_socket.herror.__reduce_ex__" => "Helper for pickle.", + "_socket.herror.__repr__" => "Return repr(self).", + "_socket.herror.__setattr__" => "Implement setattr(self, name, value).", + "_socket.herror.__sizeof__" => "Size of object in memory, in bytes.", + "_socket.herror.__str__" => "Return str(self).", + "_socket.herror.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_socket.herror.__weakref__" => "list of weak references to the object", + "_socket.herror.add_note" => "Add a note to the exception", + "_socket.herror.errno" => "POSIX exception code", + "_socket.herror.filename" => "exception filename", + "_socket.herror.filename2" => "second exception filename", + "_socket.herror.strerror" => "exception strerror", + "_socket.herror.winerror" => "Win32 exception code", + "_socket.herror.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_socket.htonl" => "Convert a 32-bit unsigned integer from host to network byte order.", + "_socket.htons" => "Convert a 16-bit unsigned integer from host to network byte order.", + "_socket.if_indextoname" => "Returns the interface name corresponding to the interface index if_index.", + "_socket.if_nameindex" => "if_nameindex()\n\nReturns a list of network interface information (index, name) tuples.", + "_socket.if_nametoindex" => "Returns the interface index corresponding to the interface name if_name.", + "_socket.inet_aton" => "Convert an IP address in string format (123.45.67.89) to the 32-bit packed binary format used in low-level network functions.", + "_socket.inet_ntoa" => "Convert an IP address from 32-bit packed binary format to string format.", + "_socket.inet_ntop" => "inet_ntop(af, packed_ip) -> string formatted IP address\n\nConvert a packed IP address of the given family to string format.", + "_socket.inet_pton" => "inet_pton(af, ip) -> packed IP address string\n\nConvert an IP address from string format to a packed string suitable\nfor use with low-level network functions.", + "_socket.ntohl" => "Convert a 32-bit unsigned integer from network to host byte order.", + "_socket.ntohs" => "Convert a 16-bit unsigned integer from network to host byte order.", + "_socket.setdefaulttimeout" => "setdefaulttimeout(timeout)\n\nSet the default timeout in seconds (float) for new socket objects.\nA value of None indicates that new socket objects have no timeout.\nWhen the socket module is first imported, the default is None.", + "_socket.sethostname" => "sethostname(name)\n\nSets the hostname to name.", + "_socket.socket" => "socket(family=AF_INET, type=SOCK_STREAM, proto=0) -> socket object\nsocket(family=-1, type=-1, proto=-1, fileno=None) -> socket object\n\nOpen a socket of the given type. The family argument specifies the\naddress family; it defaults to AF_INET. The type argument specifies\nwhether this is a stream (SOCK_STREAM, this is the default)\nor datagram (SOCK_DGRAM) socket. The protocol argument defaults to 0,\nspecifying the default protocol. Keyword arguments are accepted.\nThe socket is created as non-inheritable.\n\nWhen a fileno is passed in, family, type and proto are auto-detected,\nunless they are explicitly set.\n\nA socket object represents one endpoint of a network connection.\n\nMethods of socket objects (keyword arguments not allowed):\n\n_accept() -- accept connection, returning new socket fd and client address\nbind(addr) -- bind the socket to a local address\nclose() -- close the socket\nconnect(addr) -- connect the socket to a remote address\nconnect_ex(addr) -- connect, return an error code instead of an exception\ndup() -- return a new socket fd duplicated from fileno()\nfileno() -- return underlying file descriptor\ngetpeername() -- return remote address [*]\ngetsockname() -- return local address\ngetsockopt(level, optname[, buflen]) -- get socket options\ngettimeout() -- return timeout or None\nlisten([n]) -- start listening for incoming connections\nrecv(buflen[, flags]) -- receive data\nrecv_into(buffer[, nbytes[, flags]]) -- receive data (into a buffer)\nrecvfrom(buflen[, flags]) -- receive data and sender's address\nrecvfrom_into(buffer[, nbytes, [, flags])\n -- receive data and sender's address (into a buffer)\nsendall(data[, flags]) -- send all data\nsend(data[, flags]) -- send data, may not send all of it\nsendto(data[, flags], addr) -- send data to a given address\nsetblocking(bool) -- set or clear the blocking I/O flag\ngetblocking() -- return True if socket is blocking, False if non-blocking\nsetsockopt(level, optname, value[, optlen]) -- set socket options\nsettimeout(None | float) -- set or clear the timeout\nshutdown(how) -- shut down traffic in one or both directions\n\n [*] not available on all platforms!", + "_socket.socket.__del__" => "Called when the instance is about to be destroyed.", + "_socket.socket.__delattr__" => "Implement delattr(self, name).", + "_socket.socket.__eq__" => "Return self==value.", + "_socket.socket.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_socket.socket.__ge__" => "Return self>=value.", + "_socket.socket.__getattribute__" => "Return getattr(self, name).", + "_socket.socket.__getstate__" => "Helper for pickle.", + "_socket.socket.__gt__" => "Return self>value.", + "_socket.socket.__hash__" => "Return hash(self).", + "_socket.socket.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_socket.socket.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_socket.socket.__le__" => "Return self<=value.", + "_socket.socket.__lt__" => "Return self<value.", + "_socket.socket.__ne__" => "Return self!=value.", + "_socket.socket.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_socket.socket.__reduce__" => "Helper for pickle.", + "_socket.socket.__reduce_ex__" => "Helper for pickle.", + "_socket.socket.__repr__" => "Return repr(self).", + "_socket.socket.__setattr__" => "Implement setattr(self, name, value).", + "_socket.socket.__sizeof__" => "Size of object in memory, in bytes.", + "_socket.socket.__str__" => "Return str(self).", + "_socket.socket.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_socket.socket._accept" => "_accept() -> (integer, address info)\n\nWait for an incoming connection. Return a new socket file descriptor\nrepresenting the connection, and the address of the client.\nFor IP sockets, the address info is a pair (hostaddr, port).", + "_socket.socket.bind" => "bind(address)\n\nBind the socket to a local address. For IP sockets, the address is a\npair (host, port); the host must refer to the local host. For raw packet\nsockets the address is a tuple (ifname, proto [,pkttype [,hatype [,addr]]])", + "_socket.socket.close" => "close()\n\nClose the socket. It cannot be used after this call.", + "_socket.socket.connect" => "connect(address)\n\nConnect the socket to a remote address. For IP sockets, the address\nis a pair (host, port).", + "_socket.socket.connect_ex" => "connect_ex(address) -> errno\n\nThis is like connect(address), but returns an error code (the errno value)\ninstead of raising an exception when an error occurs.", + "_socket.socket.detach" => "detach()\n\nClose the socket object without closing the underlying file descriptor.\nThe object cannot be used after this call, but the file descriptor\ncan be reused for other purposes. The file descriptor is returned.", + "_socket.socket.family" => "the socket family", + "_socket.socket.fileno" => "fileno() -> integer\n\nReturn the integer file descriptor of the socket.", + "_socket.socket.getblocking" => "getblocking()\n\nReturns True if socket is in blocking mode, or False if it\nis in non-blocking mode.", + "_socket.socket.getpeername" => "getpeername() -> address info\n\nReturn the address of the remote endpoint. For IP sockets, the address\ninfo is a pair (hostaddr, port).", + "_socket.socket.getsockname" => "getsockname() -> address info\n\nReturn the address of the local endpoint. The format depends on the\naddress family. For IPv4 sockets, the address info is a pair\n(hostaddr, port). For IPv6 sockets, the address info is a 4-tuple\n(hostaddr, port, flowinfo, scope_id).", + "_socket.socket.getsockopt" => "getsockopt(level, option[, buffersize]) -> value\n\nGet a socket option. See the Unix manual for level and option.\nIf a nonzero buffersize argument is given, the return value is a\nstring of that length; otherwise it is an integer.", + "_socket.socket.gettimeout" => "gettimeout() -> timeout\n\nReturns the timeout in seconds (float) associated with socket\noperations. A timeout of None indicates that timeouts on socket\noperations are disabled.", + "_socket.socket.ioctl" => "ioctl(cmd, option) -> long\n\nControl the socket with WSAIoctl syscall. Currently supported 'cmd' values are\nSIO_RCVALL: 'option' must be one of the socket.RCVALL_* constants.\nSIO_KEEPALIVE_VALS: 'option' is a tuple of (onoff, timeout, interval).\nSIO_LOOPBACK_FAST_PATH: 'option' is a boolean value, and is disabled by default", + "_socket.socket.listen" => "listen([backlog])\n\nEnable a server to accept connections. If backlog is specified, it must be\nat least 0 (if it is lower, it is set to 0); it specifies the number of\nunaccepted connections that the system will allow before refusing new\nconnections. If not specified, a default reasonable value is chosen.", + "_socket.socket.proto" => "the socket protocol", + "_socket.socket.recv" => "recv(buffersize[, flags]) -> data\n\nReceive up to buffersize bytes from the socket. For the optional flags\nargument, see the Unix manual. When no data is available, block until\nat least one byte is available or until the remote end is closed. When\nthe remote end is closed and all data is read, return the empty string.", + "_socket.socket.recv_into" => "recv_into(buffer, [nbytes[, flags]]) -> nbytes_read\n\nA version of recv() that stores its data into a buffer rather than creating\na new string. Receive up to buffersize bytes from the socket. If buffersize\nis not specified (or 0), receive up to the size available in the given buffer.\n\nSee recv() for documentation about the flags.", + "_socket.socket.recvfrom" => "recvfrom(buffersize[, flags]) -> (data, address info)\n\nLike recv(buffersize, flags) but also return the sender's address info.", + "_socket.socket.recvfrom_into" => "recvfrom_into(buffer[, nbytes[, flags]]) -> (nbytes, address info)\n\nLike recv_into(buffer[, nbytes[, flags]]) but also return the sender's address info.", + "_socket.socket.recvmsg" => "recvmsg(bufsize[, ancbufsize[, flags]]) -> (data, ancdata, msg_flags, address)\n\nReceive normal data (up to bufsize bytes) and ancillary data from the\nsocket. The ancbufsize argument sets the size in bytes of the\ninternal buffer used to receive the ancillary data; it defaults to 0,\nmeaning that no ancillary data will be received. Appropriate buffer\nsizes for ancillary data can be calculated using CMSG_SPACE() or\nCMSG_LEN(), and items which do not fit into the buffer might be\ntruncated or discarded. The flags argument defaults to 0 and has the\nsame meaning as for recv().\n\nThe return value is a 4-tuple: (data, ancdata, msg_flags, address).\nThe data item is a bytes object holding the non-ancillary data\nreceived. The ancdata item is a list of zero or more tuples\n(cmsg_level, cmsg_type, cmsg_data) representing the ancillary data\n(control messages) received: cmsg_level and cmsg_type are integers\nspecifying the protocol level and protocol-specific type respectively,\nand cmsg_data is a bytes object holding the associated data. The\nmsg_flags item is the bitwise OR of various flags indicating\nconditions on the received message; see your system documentation for\ndetails. If the receiving socket is unconnected, address is the\naddress of the sending socket, if available; otherwise, its value is\nunspecified.\n\nIf recvmsg() raises an exception after the system call returns, it\nwill first attempt to close any file descriptors received via the\nSCM_RIGHTS mechanism.", + "_socket.socket.recvmsg_into" => "recvmsg_into(buffers[, ancbufsize[, flags]]) -> (nbytes, ancdata, msg_flags, address)\n\nReceive normal data and ancillary data from the socket, scattering the\nnon-ancillary data into a series of buffers. The buffers argument\nmust be an iterable of objects that export writable buffers\n(e.g. bytearray objects); these will be filled with successive chunks\nof the non-ancillary data until it has all been written or there are\nno more buffers. The ancbufsize argument sets the size in bytes of\nthe internal buffer used to receive the ancillary data; it defaults to\n0, meaning that no ancillary data will be received. Appropriate\nbuffer sizes for ancillary data can be calculated using CMSG_SPACE()\nor CMSG_LEN(), and items which do not fit into the buffer might be\ntruncated or discarded. The flags argument defaults to 0 and has the\nsame meaning as for recv().\n\nThe return value is a 4-tuple: (nbytes, ancdata, msg_flags, address).\nThe nbytes item is the total number of bytes of non-ancillary data\nwritten into the buffers. The ancdata item is a list of zero or more\ntuples (cmsg_level, cmsg_type, cmsg_data) representing the ancillary\ndata (control messages) received: cmsg_level and cmsg_type are\nintegers specifying the protocol level and protocol-specific type\nrespectively, and cmsg_data is a bytes object holding the associated\ndata. The msg_flags item is the bitwise OR of various flags\nindicating conditions on the received message; see your system\ndocumentation for details. If the receiving socket is unconnected,\naddress is the address of the sending socket, if available; otherwise,\nits value is unspecified.\n\nIf recvmsg_into() raises an exception after the system call returns,\nit will first attempt to close any file descriptors received via the\nSCM_RIGHTS mechanism.", + "_socket.socket.send" => "send(data[, flags]) -> count\n\nSend a data string to the socket. For the optional flags\nargument, see the Unix manual. Return the number of bytes\nsent; this may be less than len(data) if the network is busy.", + "_socket.socket.sendall" => "sendall(data[, flags])\n\nSend a data string to the socket. For the optional flags\nargument, see the Unix manual. This calls send() repeatedly\nuntil all data is sent. If an error occurs, it's impossible\nto tell how much data has been sent.", + "_socket.socket.sendmsg" => "sendmsg(buffers[, ancdata[, flags[, address]]]) -> count\n\nSend normal and ancillary data to the socket, gathering the\nnon-ancillary data from a series of buffers and concatenating it into\na single message. The buffers argument specifies the non-ancillary\ndata as an iterable of bytes-like objects (e.g. bytes objects).\nThe ancdata argument specifies the ancillary data (control messages)\nas an iterable of zero or more tuples (cmsg_level, cmsg_type,\ncmsg_data), where cmsg_level and cmsg_type are integers specifying the\nprotocol level and protocol-specific type respectively, and cmsg_data\nis a bytes-like object holding the associated data. The flags\nargument defaults to 0 and has the same meaning as for send(). If\naddress is supplied and not None, it sets a destination address for\nthe message. The return value is the number of bytes of non-ancillary\ndata sent.", + "_socket.socket.sendmsg_afalg" => "sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags=MSG_MORE]]])\n\nSet operation mode, IV and length of associated data for an AF_ALG\noperation socket.", + "_socket.socket.sendto" => "sendto(data[, flags], address) -> count\n\nLike send(data, flags) but allows specifying the destination address.\nFor IP sockets, the address is a pair (hostaddr, port).", + "_socket.socket.setblocking" => "setblocking(flag)\n\nSet the socket to blocking (flag is true) or non-blocking (false).\nsetblocking(True) is equivalent to settimeout(None);\nsetblocking(False) is equivalent to settimeout(0.0).", + "_socket.socket.setsockopt" => "setsockopt(level, option, value: int)\nsetsockopt(level, option, value: buffer)\nsetsockopt(level, option, None, optlen: int)\n\nSet a socket option. See the Unix manual for level and option.\nThe value argument can either be an integer, a string buffer, or\nNone, optlen.", + "_socket.socket.settimeout" => "settimeout(timeout)\n\nSet a timeout on socket operations. 'timeout' can be a float,\ngiving in seconds, or None. Setting a timeout of None disables\nthe timeout feature and is equivalent to setblocking(1).\nSetting a timeout of zero is the same as setblocking(0).", + "_socket.socket.share" => "share(process_id) -> bytes\n\nShare the socket with another process. The target process id\nmust be provided and the resulting bytes object passed to the target\nprocess. There the shared socket can be instantiated by calling\nsocket.fromshare().", + "_socket.socket.shutdown" => "shutdown(flag)\n\nShut down the reading side of the socket (flag == SHUT_RD), the writing side\nof the socket (flag == SHUT_WR), or both ends (flag == SHUT_RDWR).", + "_socket.socket.timeout" => "the socket timeout", + "_socket.socket.type" => "the socket type", + "_socket.socketpair" => "socketpair([family[, type [, proto]]]) -> (socket object, socket object)\n\nCreate a pair of socket objects from the sockets returned by the platform\nsocketpair() function.\nThe arguments are the same as for socket() except the default family is\nAF_UNIX if defined on the platform; otherwise, the default is AF_INET.", + "_sqlite3.Blob.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Blob.__delitem__" => "Delete self[key].", + "_sqlite3.Blob.__enter__" => "Blob context manager enter.", + "_sqlite3.Blob.__eq__" => "Return self==value.", + "_sqlite3.Blob.__exit__" => "Blob context manager exit.", + "_sqlite3.Blob.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Blob.__ge__" => "Return self>=value.", + "_sqlite3.Blob.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Blob.__getitem__" => "Return self[key].", + "_sqlite3.Blob.__getstate__" => "Helper for pickle.", + "_sqlite3.Blob.__gt__" => "Return self>value.", + "_sqlite3.Blob.__hash__" => "Return hash(self).", + "_sqlite3.Blob.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Blob.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Blob.__le__" => "Return self<=value.", + "_sqlite3.Blob.__len__" => "Return len(self).", + "_sqlite3.Blob.__lt__" => "Return self<value.", + "_sqlite3.Blob.__ne__" => "Return self!=value.", + "_sqlite3.Blob.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Blob.__reduce__" => "Helper for pickle.", + "_sqlite3.Blob.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Blob.__repr__" => "Return repr(self).", + "_sqlite3.Blob.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Blob.__setitem__" => "Set self[key] to value.", + "_sqlite3.Blob.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Blob.__str__" => "Return str(self).", + "_sqlite3.Blob.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Blob.close" => "Close the blob.", + "_sqlite3.Blob.read" => "Read data at the current offset position.\n\n length\n Read length in bytes.\n\nIf the end of the blob is reached, the data up to end of file will be returned.\nWhen length is not specified, or is negative, Blob.read() will read until the\nend of the blob.", + "_sqlite3.Blob.seek" => "Set the current access position to offset.\n\nThe origin argument defaults to os.SEEK_SET (absolute blob positioning).\nOther values for origin are os.SEEK_CUR (seek relative to the current position)\nand os.SEEK_END (seek relative to the blob's end).", + "_sqlite3.Blob.tell" => "Return the current access position for the blob.", + "_sqlite3.Blob.write" => "Write data at the current offset.\n\nThis function cannot change the blob length. Writing beyond the end of the\nblob will result in an exception being raised.", + "_sqlite3.Connection" => "SQLite database connection object.", + "_sqlite3.Connection.__call__" => "Call self as a function.", + "_sqlite3.Connection.__del__" => "Called when the instance is about to be destroyed.", + "_sqlite3.Connection.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Connection.__enter__" => "Called when the connection is used as a context manager.\n\nReturns itself as a convenience to the caller.", + "_sqlite3.Connection.__eq__" => "Return self==value.", + "_sqlite3.Connection.__exit__" => "Called when the connection is used as a context manager.\n\nIf there was any exception, a rollback takes place; otherwise we commit.", + "_sqlite3.Connection.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Connection.__ge__" => "Return self>=value.", + "_sqlite3.Connection.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Connection.__getstate__" => "Helper for pickle.", + "_sqlite3.Connection.__gt__" => "Return self>value.", + "_sqlite3.Connection.__hash__" => "Return hash(self).", + "_sqlite3.Connection.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Connection.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Connection.__le__" => "Return self<=value.", + "_sqlite3.Connection.__lt__" => "Return self<value.", + "_sqlite3.Connection.__ne__" => "Return self!=value.", + "_sqlite3.Connection.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Connection.__reduce__" => "Helper for pickle.", + "_sqlite3.Connection.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Connection.__repr__" => "Return repr(self).", + "_sqlite3.Connection.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Connection.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Connection.__str__" => "Return str(self).", + "_sqlite3.Connection.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Connection.backup" => "Makes a backup of the database.", + "_sqlite3.Connection.blobopen" => "Open and return a BLOB object.\n\n table\n Table name.\n column\n Column name.\n rowid\n Row id.\n readonly\n Open the BLOB without write permissions.\n name\n Database name.", + "_sqlite3.Connection.close" => "Close the database connection.\n\nAny pending transaction is not committed implicitly.", + "_sqlite3.Connection.commit" => "Commit any pending transaction to the database.\n\nIf there is no open transaction, this method is a no-op.", + "_sqlite3.Connection.create_aggregate" => "Creates a new aggregate.\n\nNote: Passing keyword arguments 'name', 'n_arg' and 'aggregate_class'\nto _sqlite3.Connection.create_aggregate() is deprecated. Parameters\n'name', 'n_arg' and 'aggregate_class' will become positional-only in\nPython 3.15.", + "_sqlite3.Connection.create_collation" => "Creates a collation function.", + "_sqlite3.Connection.create_function" => "Creates a new function.\n\nNote: Passing keyword arguments 'name', 'narg' and 'func' to\n_sqlite3.Connection.create_function() is deprecated. Parameters\n'name', 'narg' and 'func' will become positional-only in Python 3.15.", + "_sqlite3.Connection.create_window_function" => "Creates or redefines an aggregate window function. Non-standard.\n\n name\n The name of the SQL aggregate window function to be created or\n redefined.\n num_params\n The number of arguments the step and inverse methods takes.\n aggregate_class\n A class with step(), finalize(), value(), and inverse() methods.\n Set to None to clear the window function.", + "_sqlite3.Connection.cursor" => "Return a cursor for the connection.", + "_sqlite3.Connection.deserialize" => "Load a serialized database.\n\n data\n The serialized database content.\n name\n Which database to reopen with the deserialization.\n\nThe deserialize interface causes the database connection to disconnect from the\ntarget database, and then reopen it as an in-memory database based on the given\nserialized data.\n\nThe deserialize interface will fail with SQLITE_BUSY if the database is\ncurrently in a read transaction or is involved in a backup operation.", + "_sqlite3.Connection.enable_load_extension" => "Enable dynamic loading of SQLite extension modules.", + "_sqlite3.Connection.execute" => "Executes an SQL statement.", + "_sqlite3.Connection.executemany" => "Repeatedly executes an SQL statement.", + "_sqlite3.Connection.executescript" => "Executes multiple SQL statements at once.", + "_sqlite3.Connection.getconfig" => "Query a boolean connection configuration option.\n\n op\n The configuration verb; one of the sqlite3.SQLITE_DBCONFIG codes.", + "_sqlite3.Connection.getlimit" => "Get connection run-time limits.\n\n category\n The limit category to be queried.", + "_sqlite3.Connection.interrupt" => "Abort any pending database operation.", + "_sqlite3.Connection.iterdump" => "Returns iterator to the dump of the database in an SQL text format.\n\n filter\n An optional LIKE pattern for database objects to dump", + "_sqlite3.Connection.load_extension" => "Load SQLite extension module.", + "_sqlite3.Connection.rollback" => "Roll back to the start of any pending transaction.\n\nIf there is no open transaction, this method is a no-op.", + "_sqlite3.Connection.serialize" => "Serialize a database into a byte string.\n\n name\n Which database to serialize.\n\nFor an ordinary on-disk database file, the serialization is just a copy of the\ndisk file. For an in-memory database or a \"temp\" database, the serialization is\nthe same sequence of bytes which would be written to disk if that database\nwere backed up to disk.", + "_sqlite3.Connection.set_authorizer" => "Set authorizer callback.\n\nNote: Passing keyword argument 'authorizer_callback' to\n_sqlite3.Connection.set_authorizer() is deprecated. Parameter\n'authorizer_callback' will become positional-only in Python 3.15.", + "_sqlite3.Connection.set_progress_handler" => "Set progress handler callback.\n\n progress_handler\n A callable that takes no arguments.\n If the callable returns non-zero, the current query is terminated,\n and an exception is raised.\n n\n The number of SQLite virtual machine instructions that are\n executed between invocations of 'progress_handler'.\n\nIf 'progress_handler' is None or 'n' is 0, the progress handler is disabled.\n\nNote: Passing keyword argument 'progress_handler' to\n_sqlite3.Connection.set_progress_handler() is deprecated. Parameter\n'progress_handler' will become positional-only in Python 3.15.", + "_sqlite3.Connection.set_trace_callback" => "Set a trace callback called for each SQL statement (passed as unicode).\n\nNote: Passing keyword argument 'trace_callback' to\n_sqlite3.Connection.set_trace_callback() is deprecated. Parameter\n'trace_callback' will become positional-only in Python 3.15.", + "_sqlite3.Connection.setconfig" => "Set a boolean connection configuration option.\n\n op\n The configuration verb; one of the sqlite3.SQLITE_DBCONFIG codes.", + "_sqlite3.Connection.setlimit" => "Set connection run-time limits.\n\n category\n The limit category to be set.\n limit\n The new limit. If the new limit is a negative number, the limit is\n unchanged.\n\nAttempts to increase a limit above its hard upper bound are silently truncated\nto the hard upper bound. Regardless of whether or not the limit was changed,\nthe prior value of the limit is returned.", + "_sqlite3.Cursor" => "SQLite database cursor class.", + "_sqlite3.Cursor.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Cursor.__eq__" => "Return self==value.", + "_sqlite3.Cursor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Cursor.__ge__" => "Return self>=value.", + "_sqlite3.Cursor.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Cursor.__getstate__" => "Helper for pickle.", + "_sqlite3.Cursor.__gt__" => "Return self>value.", + "_sqlite3.Cursor.__hash__" => "Return hash(self).", + "_sqlite3.Cursor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Cursor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Cursor.__iter__" => "Implement iter(self).", + "_sqlite3.Cursor.__le__" => "Return self<=value.", + "_sqlite3.Cursor.__lt__" => "Return self<value.", + "_sqlite3.Cursor.__ne__" => "Return self!=value.", + "_sqlite3.Cursor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Cursor.__next__" => "Implement next(self).", + "_sqlite3.Cursor.__reduce__" => "Helper for pickle.", + "_sqlite3.Cursor.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Cursor.__repr__" => "Return repr(self).", + "_sqlite3.Cursor.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Cursor.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Cursor.__str__" => "Return str(self).", + "_sqlite3.Cursor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Cursor.close" => "Closes the cursor.", + "_sqlite3.Cursor.execute" => "Executes an SQL statement.", + "_sqlite3.Cursor.executemany" => "Repeatedly executes an SQL statement.", + "_sqlite3.Cursor.executescript" => "Executes multiple SQL statements at once.", + "_sqlite3.Cursor.fetchall" => "Fetches all rows from the resultset.", + "_sqlite3.Cursor.fetchmany" => "Fetches several rows from the resultset.\n\n size\n The default value is set by the Cursor.arraysize attribute.", + "_sqlite3.Cursor.fetchone" => "Fetches one row from the resultset.", + "_sqlite3.Cursor.setinputsizes" => "Required by DB-API. Does nothing in sqlite3.", + "_sqlite3.Cursor.setoutputsize" => "Required by DB-API. Does nothing in sqlite3.", + "_sqlite3.DataError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.DataError.__eq__" => "Return self==value.", + "_sqlite3.DataError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.DataError.__ge__" => "Return self>=value.", + "_sqlite3.DataError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.DataError.__getstate__" => "Helper for pickle.", + "_sqlite3.DataError.__gt__" => "Return self>value.", + "_sqlite3.DataError.__hash__" => "Return hash(self).", + "_sqlite3.DataError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.DataError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.DataError.__le__" => "Return self<=value.", + "_sqlite3.DataError.__lt__" => "Return self<value.", + "_sqlite3.DataError.__ne__" => "Return self!=value.", + "_sqlite3.DataError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.DataError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.DataError.__repr__" => "Return repr(self).", + "_sqlite3.DataError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.DataError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.DataError.__str__" => "Return str(self).", + "_sqlite3.DataError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.DataError.__weakref__" => "list of weak references to the object", + "_sqlite3.DataError.add_note" => "Add a note to the exception", + "_sqlite3.DataError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.DatabaseError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.DatabaseError.__eq__" => "Return self==value.", + "_sqlite3.DatabaseError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.DatabaseError.__ge__" => "Return self>=value.", + "_sqlite3.DatabaseError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.DatabaseError.__getstate__" => "Helper for pickle.", + "_sqlite3.DatabaseError.__gt__" => "Return self>value.", + "_sqlite3.DatabaseError.__hash__" => "Return hash(self).", + "_sqlite3.DatabaseError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.DatabaseError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.DatabaseError.__le__" => "Return self<=value.", + "_sqlite3.DatabaseError.__lt__" => "Return self<value.", + "_sqlite3.DatabaseError.__ne__" => "Return self!=value.", + "_sqlite3.DatabaseError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.DatabaseError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.DatabaseError.__repr__" => "Return repr(self).", + "_sqlite3.DatabaseError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.DatabaseError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.DatabaseError.__str__" => "Return str(self).", + "_sqlite3.DatabaseError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.DatabaseError.__weakref__" => "list of weak references to the object", + "_sqlite3.DatabaseError.add_note" => "Add a note to the exception", + "_sqlite3.DatabaseError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.Error.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Error.__eq__" => "Return self==value.", + "_sqlite3.Error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Error.__ge__" => "Return self>=value.", + "_sqlite3.Error.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Error.__getstate__" => "Helper for pickle.", + "_sqlite3.Error.__gt__" => "Return self>value.", + "_sqlite3.Error.__hash__" => "Return hash(self).", + "_sqlite3.Error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Error.__le__" => "Return self<=value.", + "_sqlite3.Error.__lt__" => "Return self<value.", + "_sqlite3.Error.__ne__" => "Return self!=value.", + "_sqlite3.Error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Error.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Error.__repr__" => "Return repr(self).", + "_sqlite3.Error.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Error.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Error.__str__" => "Return str(self).", + "_sqlite3.Error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Error.__weakref__" => "list of weak references to the object", + "_sqlite3.Error.add_note" => "Add a note to the exception", + "_sqlite3.Error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.IntegrityError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.IntegrityError.__eq__" => "Return self==value.", + "_sqlite3.IntegrityError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.IntegrityError.__ge__" => "Return self>=value.", + "_sqlite3.IntegrityError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.IntegrityError.__getstate__" => "Helper for pickle.", + "_sqlite3.IntegrityError.__gt__" => "Return self>value.", + "_sqlite3.IntegrityError.__hash__" => "Return hash(self).", + "_sqlite3.IntegrityError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.IntegrityError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.IntegrityError.__le__" => "Return self<=value.", + "_sqlite3.IntegrityError.__lt__" => "Return self<value.", + "_sqlite3.IntegrityError.__ne__" => "Return self!=value.", + "_sqlite3.IntegrityError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.IntegrityError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.IntegrityError.__repr__" => "Return repr(self).", + "_sqlite3.IntegrityError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.IntegrityError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.IntegrityError.__str__" => "Return str(self).", + "_sqlite3.IntegrityError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.IntegrityError.__weakref__" => "list of weak references to the object", + "_sqlite3.IntegrityError.add_note" => "Add a note to the exception", + "_sqlite3.IntegrityError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.InterfaceError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.InterfaceError.__eq__" => "Return self==value.", + "_sqlite3.InterfaceError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.InterfaceError.__ge__" => "Return self>=value.", + "_sqlite3.InterfaceError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.InterfaceError.__getstate__" => "Helper for pickle.", + "_sqlite3.InterfaceError.__gt__" => "Return self>value.", + "_sqlite3.InterfaceError.__hash__" => "Return hash(self).", + "_sqlite3.InterfaceError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.InterfaceError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.InterfaceError.__le__" => "Return self<=value.", + "_sqlite3.InterfaceError.__lt__" => "Return self<value.", + "_sqlite3.InterfaceError.__ne__" => "Return self!=value.", + "_sqlite3.InterfaceError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.InterfaceError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.InterfaceError.__repr__" => "Return repr(self).", + "_sqlite3.InterfaceError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.InterfaceError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.InterfaceError.__str__" => "Return str(self).", + "_sqlite3.InterfaceError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.InterfaceError.__weakref__" => "list of weak references to the object", + "_sqlite3.InterfaceError.add_note" => "Add a note to the exception", + "_sqlite3.InterfaceError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.InternalError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.InternalError.__eq__" => "Return self==value.", + "_sqlite3.InternalError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.InternalError.__ge__" => "Return self>=value.", + "_sqlite3.InternalError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.InternalError.__getstate__" => "Helper for pickle.", + "_sqlite3.InternalError.__gt__" => "Return self>value.", + "_sqlite3.InternalError.__hash__" => "Return hash(self).", + "_sqlite3.InternalError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.InternalError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.InternalError.__le__" => "Return self<=value.", + "_sqlite3.InternalError.__lt__" => "Return self<value.", + "_sqlite3.InternalError.__ne__" => "Return self!=value.", + "_sqlite3.InternalError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.InternalError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.InternalError.__repr__" => "Return repr(self).", + "_sqlite3.InternalError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.InternalError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.InternalError.__str__" => "Return str(self).", + "_sqlite3.InternalError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.InternalError.__weakref__" => "list of weak references to the object", + "_sqlite3.InternalError.add_note" => "Add a note to the exception", + "_sqlite3.InternalError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.NotSupportedError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.NotSupportedError.__eq__" => "Return self==value.", + "_sqlite3.NotSupportedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.NotSupportedError.__ge__" => "Return self>=value.", + "_sqlite3.NotSupportedError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.NotSupportedError.__getstate__" => "Helper for pickle.", + "_sqlite3.NotSupportedError.__gt__" => "Return self>value.", + "_sqlite3.NotSupportedError.__hash__" => "Return hash(self).", + "_sqlite3.NotSupportedError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.NotSupportedError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.NotSupportedError.__le__" => "Return self<=value.", + "_sqlite3.NotSupportedError.__lt__" => "Return self<value.", + "_sqlite3.NotSupportedError.__ne__" => "Return self!=value.", + "_sqlite3.NotSupportedError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.NotSupportedError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.NotSupportedError.__repr__" => "Return repr(self).", + "_sqlite3.NotSupportedError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.NotSupportedError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.NotSupportedError.__str__" => "Return str(self).", + "_sqlite3.NotSupportedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.NotSupportedError.__weakref__" => "list of weak references to the object", + "_sqlite3.NotSupportedError.add_note" => "Add a note to the exception", + "_sqlite3.NotSupportedError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.OperationalError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.OperationalError.__eq__" => "Return self==value.", + "_sqlite3.OperationalError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.OperationalError.__ge__" => "Return self>=value.", + "_sqlite3.OperationalError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.OperationalError.__getstate__" => "Helper for pickle.", + "_sqlite3.OperationalError.__gt__" => "Return self>value.", + "_sqlite3.OperationalError.__hash__" => "Return hash(self).", + "_sqlite3.OperationalError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.OperationalError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.OperationalError.__le__" => "Return self<=value.", + "_sqlite3.OperationalError.__lt__" => "Return self<value.", + "_sqlite3.OperationalError.__ne__" => "Return self!=value.", + "_sqlite3.OperationalError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.OperationalError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.OperationalError.__repr__" => "Return repr(self).", + "_sqlite3.OperationalError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.OperationalError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.OperationalError.__str__" => "Return str(self).", + "_sqlite3.OperationalError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.OperationalError.__weakref__" => "list of weak references to the object", + "_sqlite3.OperationalError.add_note" => "Add a note to the exception", + "_sqlite3.OperationalError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.PrepareProtocol" => "PEP 246 style object adaption protocol type.", + "_sqlite3.PrepareProtocol.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.PrepareProtocol.__eq__" => "Return self==value.", + "_sqlite3.PrepareProtocol.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.PrepareProtocol.__ge__" => "Return self>=value.", + "_sqlite3.PrepareProtocol.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.PrepareProtocol.__getstate__" => "Helper for pickle.", + "_sqlite3.PrepareProtocol.__gt__" => "Return self>value.", + "_sqlite3.PrepareProtocol.__hash__" => "Return hash(self).", + "_sqlite3.PrepareProtocol.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.PrepareProtocol.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.PrepareProtocol.__le__" => "Return self<=value.", + "_sqlite3.PrepareProtocol.__lt__" => "Return self<value.", + "_sqlite3.PrepareProtocol.__ne__" => "Return self!=value.", + "_sqlite3.PrepareProtocol.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.PrepareProtocol.__reduce__" => "Helper for pickle.", + "_sqlite3.PrepareProtocol.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.PrepareProtocol.__repr__" => "Return repr(self).", + "_sqlite3.PrepareProtocol.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.PrepareProtocol.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.PrepareProtocol.__str__" => "Return str(self).", + "_sqlite3.PrepareProtocol.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.ProgrammingError.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.ProgrammingError.__eq__" => "Return self==value.", + "_sqlite3.ProgrammingError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.ProgrammingError.__ge__" => "Return self>=value.", + "_sqlite3.ProgrammingError.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.ProgrammingError.__getstate__" => "Helper for pickle.", + "_sqlite3.ProgrammingError.__gt__" => "Return self>value.", + "_sqlite3.ProgrammingError.__hash__" => "Return hash(self).", + "_sqlite3.ProgrammingError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.ProgrammingError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.ProgrammingError.__le__" => "Return self<=value.", + "_sqlite3.ProgrammingError.__lt__" => "Return self<value.", + "_sqlite3.ProgrammingError.__ne__" => "Return self!=value.", + "_sqlite3.ProgrammingError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.ProgrammingError.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.ProgrammingError.__repr__" => "Return repr(self).", + "_sqlite3.ProgrammingError.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.ProgrammingError.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.ProgrammingError.__str__" => "Return str(self).", + "_sqlite3.ProgrammingError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.ProgrammingError.__weakref__" => "list of weak references to the object", + "_sqlite3.ProgrammingError.add_note" => "Add a note to the exception", + "_sqlite3.ProgrammingError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.Row.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Row.__eq__" => "Return self==value.", + "_sqlite3.Row.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Row.__ge__" => "Return self>=value.", + "_sqlite3.Row.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Row.__getitem__" => "Return self[key].", + "_sqlite3.Row.__getstate__" => "Helper for pickle.", + "_sqlite3.Row.__gt__" => "Return self>value.", + "_sqlite3.Row.__hash__" => "Return hash(self).", + "_sqlite3.Row.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Row.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Row.__iter__" => "Implement iter(self).", + "_sqlite3.Row.__le__" => "Return self<=value.", + "_sqlite3.Row.__len__" => "Return len(self).", + "_sqlite3.Row.__lt__" => "Return self<value.", + "_sqlite3.Row.__ne__" => "Return self!=value.", + "_sqlite3.Row.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Row.__reduce__" => "Helper for pickle.", + "_sqlite3.Row.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Row.__repr__" => "Return repr(self).", + "_sqlite3.Row.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Row.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Row.__str__" => "Return str(self).", + "_sqlite3.Row.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Row.keys" => "Returns the keys of the row.", + "_sqlite3.Warning.__delattr__" => "Implement delattr(self, name).", + "_sqlite3.Warning.__eq__" => "Return self==value.", + "_sqlite3.Warning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_sqlite3.Warning.__ge__" => "Return self>=value.", + "_sqlite3.Warning.__getattribute__" => "Return getattr(self, name).", + "_sqlite3.Warning.__getstate__" => "Helper for pickle.", + "_sqlite3.Warning.__gt__" => "Return self>value.", + "_sqlite3.Warning.__hash__" => "Return hash(self).", + "_sqlite3.Warning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_sqlite3.Warning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_sqlite3.Warning.__le__" => "Return self<=value.", + "_sqlite3.Warning.__lt__" => "Return self<value.", + "_sqlite3.Warning.__ne__" => "Return self!=value.", + "_sqlite3.Warning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_sqlite3.Warning.__reduce_ex__" => "Helper for pickle.", + "_sqlite3.Warning.__repr__" => "Return repr(self).", + "_sqlite3.Warning.__setattr__" => "Implement setattr(self, name, value).", + "_sqlite3.Warning.__sizeof__" => "Size of object in memory, in bytes.", + "_sqlite3.Warning.__str__" => "Return str(self).", + "_sqlite3.Warning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_sqlite3.Warning.__weakref__" => "list of weak references to the object", + "_sqlite3.Warning.add_note" => "Add a note to the exception", + "_sqlite3.Warning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_sqlite3.adapt" => "Adapt given object to given protocol.", + "_sqlite3.complete_statement" => "Checks if a string contains a complete SQL statement.", + "_sqlite3.connect" => "Open a connection to the SQLite database file 'database'.\n\nYou can use \":memory:\" to open a database connection to a database that\nresides in RAM instead of on disk.\n\nNote: Passing more than 1 positional argument to _sqlite3.connect() is\ndeprecated. Parameters 'timeout', 'detect_types', 'isolation_level',\n'check_same_thread', 'factory', 'cached_statements' and 'uri' will\nbecome keyword-only parameters in Python 3.15.", + "_sqlite3.enable_callback_tracebacks" => "Enable or disable callback functions throwing errors to stderr.", + "_sqlite3.register_adapter" => "Register a function to adapt Python objects to SQLite values.", + "_sqlite3.register_converter" => "Register a function to convert SQLite values to Python objects.", + "_sre.template" => "template\n A list containing interleaved literal strings (str or bytes) and group\n indices (int), as returned by re._parser.parse_template():\n [literal1, group1, ..., literalN, groupN]", + "_ssl" => "Implementation module for SSL socket operations. See the socket module\nfor documentation.", + "_ssl.Certificate.__delattr__" => "Implement delattr(self, name).", + "_ssl.Certificate.__eq__" => "Return self==value.", + "_ssl.Certificate.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.Certificate.__ge__" => "Return self>=value.", + "_ssl.Certificate.__getattribute__" => "Return getattr(self, name).", + "_ssl.Certificate.__getstate__" => "Helper for pickle.", + "_ssl.Certificate.__gt__" => "Return self>value.", + "_ssl.Certificate.__hash__" => "Return hash(self).", + "_ssl.Certificate.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.Certificate.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.Certificate.__le__" => "Return self<=value.", + "_ssl.Certificate.__lt__" => "Return self<value.", + "_ssl.Certificate.__ne__" => "Return self!=value.", + "_ssl.Certificate.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.Certificate.__reduce__" => "Helper for pickle.", + "_ssl.Certificate.__reduce_ex__" => "Helper for pickle.", + "_ssl.Certificate.__repr__" => "Return repr(self).", + "_ssl.Certificate.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.Certificate.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.Certificate.__str__" => "Return str(self).", + "_ssl.Certificate.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.MemoryBIO.__delattr__" => "Implement delattr(self, name).", + "_ssl.MemoryBIO.__eq__" => "Return self==value.", + "_ssl.MemoryBIO.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.MemoryBIO.__ge__" => "Return self>=value.", + "_ssl.MemoryBIO.__getattribute__" => "Return getattr(self, name).", + "_ssl.MemoryBIO.__getstate__" => "Helper for pickle.", + "_ssl.MemoryBIO.__gt__" => "Return self>value.", + "_ssl.MemoryBIO.__hash__" => "Return hash(self).", + "_ssl.MemoryBIO.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.MemoryBIO.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.MemoryBIO.__le__" => "Return self<=value.", + "_ssl.MemoryBIO.__lt__" => "Return self<value.", + "_ssl.MemoryBIO.__ne__" => "Return self!=value.", + "_ssl.MemoryBIO.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.MemoryBIO.__reduce__" => "Helper for pickle.", + "_ssl.MemoryBIO.__reduce_ex__" => "Helper for pickle.", + "_ssl.MemoryBIO.__repr__" => "Return repr(self).", + "_ssl.MemoryBIO.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.MemoryBIO.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.MemoryBIO.__str__" => "Return str(self).", + "_ssl.MemoryBIO.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.MemoryBIO.eof" => "Whether the memory BIO is at EOF.", + "_ssl.MemoryBIO.pending" => "The number of bytes pending in the memory BIO.", + "_ssl.MemoryBIO.read" => "Read up to size bytes from the memory BIO.\n\nIf size is not specified, read the entire buffer.\nIf the return value is an empty bytes instance, this means either\nEOF or that no data is available. Use the \"eof\" property to\ndistinguish between the two.", + "_ssl.MemoryBIO.write" => "Writes the bytes b into the memory BIO.\n\nReturns the number of bytes written.", + "_ssl.MemoryBIO.write_eof" => "Write an EOF marker to the memory BIO.\n\nWhen all data has been read, the \"eof\" property will be True.", + "_ssl.RAND_add" => "Mix string into the OpenSSL PRNG state.\n\nentropy (a float) is a lower bound on the entropy contained in\nstring. See RFC 4086.", + "_ssl.RAND_bytes" => "Generate n cryptographically strong pseudo-random bytes.", + "_ssl.RAND_status" => "Returns True if the OpenSSL PRNG has been seeded with enough data and False if not.\n\nIt is necessary to seed the PRNG with RAND_add() on some platforms before\nusing the ssl() function.", + "_ssl.SSLCertVerificationError" => "A certificate could not be verified.", + "_ssl.SSLCertVerificationError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLCertVerificationError.__eq__" => "Return self==value.", + "_ssl.SSLCertVerificationError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLCertVerificationError.__ge__" => "Return self>=value.", + "_ssl.SSLCertVerificationError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLCertVerificationError.__getstate__" => "Helper for pickle.", + "_ssl.SSLCertVerificationError.__gt__" => "Return self>value.", + "_ssl.SSLCertVerificationError.__hash__" => "Return hash(self).", + "_ssl.SSLCertVerificationError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLCertVerificationError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLCertVerificationError.__le__" => "Return self<=value.", + "_ssl.SSLCertVerificationError.__lt__" => "Return self<value.", + "_ssl.SSLCertVerificationError.__ne__" => "Return self!=value.", + "_ssl.SSLCertVerificationError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLCertVerificationError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLCertVerificationError.__repr__" => "Return repr(self).", + "_ssl.SSLCertVerificationError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLCertVerificationError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLCertVerificationError.__str__" => "Return str(self).", + "_ssl.SSLCertVerificationError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLCertVerificationError.__weakref__" => "list of weak references to the object", + "_ssl.SSLCertVerificationError.add_note" => "Add a note to the exception", + "_ssl.SSLCertVerificationError.errno" => "POSIX exception code", + "_ssl.SSLCertVerificationError.filename" => "exception filename", + "_ssl.SSLCertVerificationError.filename2" => "second exception filename", + "_ssl.SSLCertVerificationError.strerror" => "exception strerror", + "_ssl.SSLCertVerificationError.winerror" => "Win32 exception code", + "_ssl.SSLCertVerificationError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLEOFError" => "SSL/TLS connection terminated abruptly.", + "_ssl.SSLEOFError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLEOFError.__eq__" => "Return self==value.", + "_ssl.SSLEOFError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLEOFError.__ge__" => "Return self>=value.", + "_ssl.SSLEOFError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLEOFError.__getstate__" => "Helper for pickle.", + "_ssl.SSLEOFError.__gt__" => "Return self>value.", + "_ssl.SSLEOFError.__hash__" => "Return hash(self).", + "_ssl.SSLEOFError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLEOFError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLEOFError.__le__" => "Return self<=value.", + "_ssl.SSLEOFError.__lt__" => "Return self<value.", + "_ssl.SSLEOFError.__ne__" => "Return self!=value.", + "_ssl.SSLEOFError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLEOFError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLEOFError.__repr__" => "Return repr(self).", + "_ssl.SSLEOFError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLEOFError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLEOFError.__str__" => "Return str(self).", + "_ssl.SSLEOFError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLEOFError.__weakref__" => "list of weak references to the object", + "_ssl.SSLEOFError.add_note" => "Add a note to the exception", + "_ssl.SSLEOFError.errno" => "POSIX exception code", + "_ssl.SSLEOFError.filename" => "exception filename", + "_ssl.SSLEOFError.filename2" => "second exception filename", + "_ssl.SSLEOFError.strerror" => "exception strerror", + "_ssl.SSLEOFError.winerror" => "Win32 exception code", + "_ssl.SSLEOFError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLError" => "An error occurred in the SSL implementation.", + "_ssl.SSLError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLError.__eq__" => "Return self==value.", + "_ssl.SSLError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLError.__ge__" => "Return self>=value.", + "_ssl.SSLError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLError.__getstate__" => "Helper for pickle.", + "_ssl.SSLError.__gt__" => "Return self>value.", + "_ssl.SSLError.__hash__" => "Return hash(self).", + "_ssl.SSLError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLError.__le__" => "Return self<=value.", + "_ssl.SSLError.__lt__" => "Return self<value.", + "_ssl.SSLError.__ne__" => "Return self!=value.", + "_ssl.SSLError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLError.__repr__" => "Return repr(self).", + "_ssl.SSLError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLError.__str__" => "Return str(self).", + "_ssl.SSLError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLError.add_note" => "Add a note to the exception", + "_ssl.SSLError.errno" => "POSIX exception code", + "_ssl.SSLError.filename" => "exception filename", + "_ssl.SSLError.filename2" => "second exception filename", + "_ssl.SSLError.strerror" => "exception strerror", + "_ssl.SSLError.winerror" => "Win32 exception code", + "_ssl.SSLError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLSession.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLSession.__eq__" => "Return self==value.", + "_ssl.SSLSession.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLSession.__ge__" => "Return self>=value.", + "_ssl.SSLSession.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLSession.__getstate__" => "Helper for pickle.", + "_ssl.SSLSession.__gt__" => "Return self>value.", + "_ssl.SSLSession.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLSession.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLSession.__le__" => "Return self<=value.", + "_ssl.SSLSession.__lt__" => "Return self<value.", + "_ssl.SSLSession.__ne__" => "Return self!=value.", + "_ssl.SSLSession.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLSession.__reduce__" => "Helper for pickle.", + "_ssl.SSLSession.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLSession.__repr__" => "Return repr(self).", + "_ssl.SSLSession.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLSession.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLSession.__str__" => "Return str(self).", + "_ssl.SSLSession.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLSession.has_ticket" => "Does the session contain a ticket?", + "_ssl.SSLSession.id" => "Session ID.", + "_ssl.SSLSession.ticket_lifetime_hint" => "Ticket life time hint.", + "_ssl.SSLSession.time" => "Session creation time (seconds since epoch).", + "_ssl.SSLSession.timeout" => "Session timeout (delta in seconds).", + "_ssl.SSLSyscallError" => "System error when attempting SSL operation.", + "_ssl.SSLSyscallError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLSyscallError.__eq__" => "Return self==value.", + "_ssl.SSLSyscallError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLSyscallError.__ge__" => "Return self>=value.", + "_ssl.SSLSyscallError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLSyscallError.__getstate__" => "Helper for pickle.", + "_ssl.SSLSyscallError.__gt__" => "Return self>value.", + "_ssl.SSLSyscallError.__hash__" => "Return hash(self).", + "_ssl.SSLSyscallError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLSyscallError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLSyscallError.__le__" => "Return self<=value.", + "_ssl.SSLSyscallError.__lt__" => "Return self<value.", + "_ssl.SSLSyscallError.__ne__" => "Return self!=value.", + "_ssl.SSLSyscallError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLSyscallError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLSyscallError.__repr__" => "Return repr(self).", + "_ssl.SSLSyscallError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLSyscallError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLSyscallError.__str__" => "Return str(self).", + "_ssl.SSLSyscallError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLSyscallError.__weakref__" => "list of weak references to the object", + "_ssl.SSLSyscallError.add_note" => "Add a note to the exception", + "_ssl.SSLSyscallError.errno" => "POSIX exception code", + "_ssl.SSLSyscallError.filename" => "exception filename", + "_ssl.SSLSyscallError.filename2" => "second exception filename", + "_ssl.SSLSyscallError.strerror" => "exception strerror", + "_ssl.SSLSyscallError.winerror" => "Win32 exception code", + "_ssl.SSLSyscallError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLWantReadError" => "Non-blocking SSL socket needs to read more data\nbefore the requested operation can be completed.", + "_ssl.SSLWantReadError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLWantReadError.__eq__" => "Return self==value.", + "_ssl.SSLWantReadError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLWantReadError.__ge__" => "Return self>=value.", + "_ssl.SSLWantReadError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLWantReadError.__getstate__" => "Helper for pickle.", + "_ssl.SSLWantReadError.__gt__" => "Return self>value.", + "_ssl.SSLWantReadError.__hash__" => "Return hash(self).", + "_ssl.SSLWantReadError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLWantReadError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLWantReadError.__le__" => "Return self<=value.", + "_ssl.SSLWantReadError.__lt__" => "Return self<value.", + "_ssl.SSLWantReadError.__ne__" => "Return self!=value.", + "_ssl.SSLWantReadError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLWantReadError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLWantReadError.__repr__" => "Return repr(self).", + "_ssl.SSLWantReadError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLWantReadError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLWantReadError.__str__" => "Return str(self).", + "_ssl.SSLWantReadError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLWantReadError.__weakref__" => "list of weak references to the object", + "_ssl.SSLWantReadError.add_note" => "Add a note to the exception", + "_ssl.SSLWantReadError.errno" => "POSIX exception code", + "_ssl.SSLWantReadError.filename" => "exception filename", + "_ssl.SSLWantReadError.filename2" => "second exception filename", + "_ssl.SSLWantReadError.strerror" => "exception strerror", + "_ssl.SSLWantReadError.winerror" => "Win32 exception code", + "_ssl.SSLWantReadError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLWantWriteError" => "Non-blocking SSL socket needs to write more data\nbefore the requested operation can be completed.", + "_ssl.SSLWantWriteError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLWantWriteError.__eq__" => "Return self==value.", + "_ssl.SSLWantWriteError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLWantWriteError.__ge__" => "Return self>=value.", + "_ssl.SSLWantWriteError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLWantWriteError.__getstate__" => "Helper for pickle.", + "_ssl.SSLWantWriteError.__gt__" => "Return self>value.", + "_ssl.SSLWantWriteError.__hash__" => "Return hash(self).", + "_ssl.SSLWantWriteError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLWantWriteError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLWantWriteError.__le__" => "Return self<=value.", + "_ssl.SSLWantWriteError.__lt__" => "Return self<value.", + "_ssl.SSLWantWriteError.__ne__" => "Return self!=value.", + "_ssl.SSLWantWriteError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLWantWriteError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLWantWriteError.__repr__" => "Return repr(self).", + "_ssl.SSLWantWriteError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLWantWriteError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLWantWriteError.__str__" => "Return str(self).", + "_ssl.SSLWantWriteError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLWantWriteError.__weakref__" => "list of weak references to the object", + "_ssl.SSLWantWriteError.add_note" => "Add a note to the exception", + "_ssl.SSLWantWriteError.errno" => "POSIX exception code", + "_ssl.SSLWantWriteError.filename" => "exception filename", + "_ssl.SSLWantWriteError.filename2" => "second exception filename", + "_ssl.SSLWantWriteError.strerror" => "exception strerror", + "_ssl.SSLWantWriteError.winerror" => "Win32 exception code", + "_ssl.SSLWantWriteError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl.SSLZeroReturnError" => "SSL/TLS session closed cleanly.", + "_ssl.SSLZeroReturnError.__delattr__" => "Implement delattr(self, name).", + "_ssl.SSLZeroReturnError.__eq__" => "Return self==value.", + "_ssl.SSLZeroReturnError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl.SSLZeroReturnError.__ge__" => "Return self>=value.", + "_ssl.SSLZeroReturnError.__getattribute__" => "Return getattr(self, name).", + "_ssl.SSLZeroReturnError.__getstate__" => "Helper for pickle.", + "_ssl.SSLZeroReturnError.__gt__" => "Return self>value.", + "_ssl.SSLZeroReturnError.__hash__" => "Return hash(self).", + "_ssl.SSLZeroReturnError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl.SSLZeroReturnError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl.SSLZeroReturnError.__le__" => "Return self<=value.", + "_ssl.SSLZeroReturnError.__lt__" => "Return self<value.", + "_ssl.SSLZeroReturnError.__ne__" => "Return self!=value.", + "_ssl.SSLZeroReturnError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl.SSLZeroReturnError.__reduce_ex__" => "Helper for pickle.", + "_ssl.SSLZeroReturnError.__repr__" => "Return repr(self).", + "_ssl.SSLZeroReturnError.__setattr__" => "Implement setattr(self, name, value).", + "_ssl.SSLZeroReturnError.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl.SSLZeroReturnError.__str__" => "Return str(self).", + "_ssl.SSLZeroReturnError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl.SSLZeroReturnError.__weakref__" => "list of weak references to the object", + "_ssl.SSLZeroReturnError.add_note" => "Add a note to the exception", + "_ssl.SSLZeroReturnError.errno" => "POSIX exception code", + "_ssl.SSLZeroReturnError.filename" => "exception filename", + "_ssl.SSLZeroReturnError.filename2" => "second exception filename", + "_ssl.SSLZeroReturnError.strerror" => "exception strerror", + "_ssl.SSLZeroReturnError.winerror" => "Win32 exception code", + "_ssl.SSLZeroReturnError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_ssl._SSLContext.__delattr__" => "Implement delattr(self, name).", + "_ssl._SSLContext.__eq__" => "Return self==value.", + "_ssl._SSLContext.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl._SSLContext.__ge__" => "Return self>=value.", + "_ssl._SSLContext.__getattribute__" => "Return getattr(self, name).", + "_ssl._SSLContext.__getstate__" => "Helper for pickle.", + "_ssl._SSLContext.__gt__" => "Return self>value.", + "_ssl._SSLContext.__hash__" => "Return hash(self).", + "_ssl._SSLContext.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl._SSLContext.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl._SSLContext.__le__" => "Return self<=value.", + "_ssl._SSLContext.__lt__" => "Return self<value.", + "_ssl._SSLContext.__ne__" => "Return self!=value.", + "_ssl._SSLContext.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl._SSLContext.__reduce__" => "Helper for pickle.", + "_ssl._SSLContext.__reduce_ex__" => "Helper for pickle.", + "_ssl._SSLContext.__repr__" => "Return repr(self).", + "_ssl._SSLContext.__setattr__" => "Implement setattr(self, name, value).", + "_ssl._SSLContext.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl._SSLContext.__str__" => "Return str(self).", + "_ssl._SSLContext.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl._SSLContext.cert_store_stats" => "Returns quantities of loaded X.509 certificates.\n\nX.509 certificates with a CA extension and certificate revocation lists\ninside the context's cert store.\n\nNOTE: Certificates in a capath directory aren't loaded unless they have\nbeen used at least once.", + "_ssl._SSLContext.get_ca_certs" => "Returns a list of dicts with information of loaded CA certs.\n\nIf the optional argument is True, returns a DER-encoded copy of the CA\ncertificate.\n\nNOTE: Certificates in a capath directory aren't loaded unless they have\nbeen used at least once.", + "_ssl._SSLContext.num_tickets" => "Control the number of TLSv1.3 session tickets.", + "_ssl._SSLContext.security_level" => "The current security level.", + "_ssl._SSLContext.sni_callback" => "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n\nIf the argument is None then the callback is disabled. The method is called\nwith the SSLSocket, the server name as a string, and the SSLContext object.\n\nSee RFC 6066 for details of the SNI extension.", + "_ssl._SSLSocket.__delattr__" => "Implement delattr(self, name).", + "_ssl._SSLSocket.__eq__" => "Return self==value.", + "_ssl._SSLSocket.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_ssl._SSLSocket.__ge__" => "Return self>=value.", + "_ssl._SSLSocket.__getattribute__" => "Return getattr(self, name).", + "_ssl._SSLSocket.__getstate__" => "Helper for pickle.", + "_ssl._SSLSocket.__gt__" => "Return self>value.", + "_ssl._SSLSocket.__hash__" => "Return hash(self).", + "_ssl._SSLSocket.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_ssl._SSLSocket.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_ssl._SSLSocket.__le__" => "Return self<=value.", + "_ssl._SSLSocket.__lt__" => "Return self<value.", + "_ssl._SSLSocket.__ne__" => "Return self!=value.", + "_ssl._SSLSocket.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_ssl._SSLSocket.__reduce__" => "Helper for pickle.", + "_ssl._SSLSocket.__reduce_ex__" => "Helper for pickle.", + "_ssl._SSLSocket.__repr__" => "Return repr(self).", + "_ssl._SSLSocket.__setattr__" => "Implement setattr(self, name, value).", + "_ssl._SSLSocket.__sizeof__" => "Size of object in memory, in bytes.", + "_ssl._SSLSocket.__str__" => "Return str(self).", + "_ssl._SSLSocket.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_ssl._SSLSocket.context" => "This changes the context associated with the SSLSocket.\n\nThis is typically used from within a callback function set by the sni_callback\non the SSLContext to change the certificate information associated with the\nSSLSocket before the cryptographic exchange handshake messages.", + "_ssl._SSLSocket.get_channel_binding" => "Get channel binding data for current connection.\n\nRaise ValueError if the requested `cb_type` is not supported. Return bytes\nof the data or None if the data is not available (e.g. before the handshake).\nOnly 'tls-unique' channel binding data from RFC 5929 is supported.", + "_ssl._SSLSocket.getpeercert" => "Returns the certificate for the peer.\n\nIf no certificate was provided, returns None. If a certificate was\nprovided, but not validated, returns an empty dictionary. Otherwise\nreturns a dict containing information about the peer certificate.\n\nIf the optional argument is True, returns a DER-encoded copy of the\npeer certificate, or None if no certificate was provided. This will\nreturn the certificate even if it wasn't validated.", + "_ssl._SSLSocket.owner" => "The Python-level owner of this object.\n\nPassed as \"self\" in servername callback.", + "_ssl._SSLSocket.pending" => "Returns the number of already decrypted bytes available for read, pending on the connection.", + "_ssl._SSLSocket.read" => "read(size, [buffer])\nRead up to size bytes from the SSL socket.", + "_ssl._SSLSocket.server_hostname" => "The currently set server hostname (for SNI).", + "_ssl._SSLSocket.server_side" => "Whether this is a server-side socket.", + "_ssl._SSLSocket.session" => "The underlying SSLSession object.", + "_ssl._SSLSocket.session_reused" => "Was the client session reused during handshake?", + "_ssl._SSLSocket.shutdown" => "Does the SSL shutdown handshake with the remote end.", + "_ssl._SSLSocket.verify_client_post_handshake" => "Initiate TLS 1.3 post-handshake authentication", + "_ssl._SSLSocket.write" => "Writes the bytes-like object b into the SSL object.\n\nReturns the number of bytes written.", + "_ssl.enum_certificates" => "Retrieve certificates from Windows' cert store.\n\nstore_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide\nmore cert storages, too. The function returns a list of (bytes,\nencoding_type, trust) tuples. The encoding_type flag can be interpreted\nwith X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either\na set of OIDs or the boolean True.", + "_ssl.enum_crls" => "Retrieve CRLs from Windows' cert store.\n\nstore_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide\nmore cert storages, too. The function returns a list of (bytes,\nencoding_type) tuples. The encoding_type flag can be interpreted with\nX509_ASN_ENCODING or PKCS_7_ASN_ENCODING.", + "_ssl.get_default_verify_paths" => "Return search paths and environment vars that are used by SSLContext's set_default_verify_paths() to load default CAs.\n\nThe values are 'cert_file_env', 'cert_file', 'cert_dir_env', 'cert_dir'.", + "_ssl.nid2obj" => "Lookup NID, short name, long name and OID of an ASN1_OBJECT by NID.", + "_ssl.txt2obj" => "Lookup NID, short name, long name and OID of an ASN1_OBJECT.\n\nBy default objects are looked up by OID. With name=True short and\nlong name are also matched.", + "_stat" => "S_IFMT_: file type bits\nS_IFDIR: directory\nS_IFCHR: character device\nS_IFBLK: block device\nS_IFREG: regular file\nS_IFIFO: fifo (named pipe)\nS_IFLNK: symbolic link\nS_IFSOCK: socket file\nS_IFDOOR: door\nS_IFPORT: event port\nS_IFWHT: whiteout\n\nS_ISUID: set UID bit\nS_ISGID: set GID bit\nS_ENFMT: file locking enforcement\nS_ISVTX: sticky bit\nS_IREAD: Unix V7 synonym for S_IRUSR\nS_IWRITE: Unix V7 synonym for S_IWUSR\nS_IEXEC: Unix V7 synonym for S_IXUSR\nS_IRWXU: mask for owner permissions\nS_IRUSR: read by owner\nS_IWUSR: write by owner\nS_IXUSR: execute by owner\nS_IRWXG: mask for group permissions\nS_IRGRP: read by group\nS_IWGRP: write by group\nS_IXGRP: execute by group\nS_IRWXO: mask for others (not in group) permissions\nS_IROTH: read by others\nS_IWOTH: write by others\nS_IXOTH: execute by others\n\nUF_SETTABLE: mask of owner changeable flags\nUF_NODUMP: do not dump file\nUF_IMMUTABLE: file may not be changed\nUF_APPEND: file may only be appended to\nUF_OPAQUE: directory is opaque when viewed through a union stack\nUF_NOUNLINK: file may not be renamed or deleted\nUF_COMPRESSED: macOS: file is hfs-compressed\nUF_TRACKED: used for dealing with document IDs\nUF_DATAVAULT: entitlement required for reading and writing\nUF_HIDDEN: macOS: file should not be displayed\nSF_SETTABLE: mask of super user changeable flags\nSF_ARCHIVED: file may be archived\nSF_IMMUTABLE: file may not be changed\nSF_APPEND: file may only be appended to\nSF_RESTRICTED: entitlement required for writing\nSF_NOUNLINK: file may not be renamed or deleted\nSF_SNAPSHOT: file is a snapshot file\nSF_FIRMLINK: file is a firmlink\nSF_DATALESS: file is a dataless object\n\nOn macOS:\nSF_SUPPORTED: mask of super user supported flags\nSF_SYNTHETIC: mask of read-only synthetic flags\n\nST_MODE\nST_INO\nST_DEV\nST_NLINK\nST_UID\nST_GID\nST_SIZE\nST_ATIME\nST_MTIME\nST_CTIME\n\nFILE_ATTRIBUTE_*: Windows file attribute constants\n (only present on Windows)", + "_stat.S_IFMT" => "Return the portion of the file's mode that describes the file type.", + "_stat.S_IMODE" => "Return the portion of the file's mode that can be set by os.chmod().", + "_stat.S_ISBLK" => "S_ISBLK(mode) -> bool\n\nReturn True if mode is from a block special device file.", + "_stat.S_ISCHR" => "S_ISCHR(mode) -> bool\n\nReturn True if mode is from a character special device file.", + "_stat.S_ISDIR" => "S_ISDIR(mode) -> bool\n\nReturn True if mode is from a directory.", + "_stat.S_ISDOOR" => "S_ISDOOR(mode) -> bool\n\nReturn True if mode is from a door.", + "_stat.S_ISFIFO" => "S_ISFIFO(mode) -> bool\n\nReturn True if mode is from a FIFO (named pipe).", + "_stat.S_ISLNK" => "S_ISLNK(mode) -> bool\n\nReturn True if mode is from a symbolic link.", + "_stat.S_ISPORT" => "S_ISPORT(mode) -> bool\n\nReturn True if mode is from an event port.", + "_stat.S_ISREG" => "S_ISREG(mode) -> bool\n\nReturn True if mode is from a regular file.", + "_stat.S_ISSOCK" => "S_ISSOCK(mode) -> bool\n\nReturn True if mode is from a socket.", + "_stat.S_ISWHT" => "S_ISWHT(mode) -> bool\n\nReturn True if mode is from a whiteout.", + "_stat.filemode" => "Convert a file's mode to a string of the form '-rwxrwxrwx'", + "_statistics" => "Accelerators for the statistics module.", + "_string" => "string helper module", + "_string.formatter_field_name_split" => "split the argument as a field name", + "_string.formatter_parser" => "parse the argument as a format string", + "_struct" => "Functions to convert between Python values and C structs.\nPython bytes objects are used to hold the data representing the C struct\nand also as format strings (explained below) to describe the layout of data\nin the C struct.\n\nThe optional first format char indicates byte order, size and alignment:\n @: native order, size & alignment (default)\n =: native order, std. size & alignment\n <: little-endian, std. size & alignment\n >: big-endian, std. size & alignment\n !: same as >\n\nThe remaining chars indicate types of args and must match exactly;\nthese can be preceded by a decimal repeat count:\n x: pad byte (no data); c:char; b:signed byte; B:unsigned byte;\n ?:_Bool; h:short; H:unsigned short; i:int; I:unsigned int;\n l:long; L:unsigned long; f:float; d:double; e:half-float.\n F:float complex; D:double complex.\nSpecial cases (preceding decimal count indicates length):\n s:string (array of char); p: pascal string (with count byte).\nSpecial cases (only available in native format):\n n:ssize_t; N:size_t;\n P:an integer type that is wide enough to hold a pointer.\nSpecial case (not in native mode unless 'long long' in platform C):\n q:long long; Q:unsigned long long\nWhitespace between formats is ignored.\n\nThe variable struct.error is an exception raised on errors.", + "_struct.Struct" => "Struct(fmt) --> compiled struct object", + "_struct.Struct.__delattr__" => "Implement delattr(self, name).", + "_struct.Struct.__eq__" => "Return self==value.", + "_struct.Struct.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_struct.Struct.__ge__" => "Return self>=value.", + "_struct.Struct.__getattribute__" => "Return getattr(self, name).", + "_struct.Struct.__getstate__" => "Helper for pickle.", + "_struct.Struct.__gt__" => "Return self>value.", + "_struct.Struct.__hash__" => "Return hash(self).", + "_struct.Struct.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_struct.Struct.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_struct.Struct.__le__" => "Return self<=value.", + "_struct.Struct.__lt__" => "Return self<value.", + "_struct.Struct.__ne__" => "Return self!=value.", + "_struct.Struct.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_struct.Struct.__reduce__" => "Helper for pickle.", + "_struct.Struct.__reduce_ex__" => "Helper for pickle.", + "_struct.Struct.__repr__" => "Return repr(self).", + "_struct.Struct.__setattr__" => "Implement setattr(self, name, value).", + "_struct.Struct.__sizeof__" => "S.__sizeof__() -> size of S in memory, in bytes", + "_struct.Struct.__str__" => "Return str(self).", + "_struct.Struct.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_struct.Struct.format" => "struct format string", + "_struct.Struct.iter_unpack" => "Return an iterator yielding tuples.\n\nTuples are unpacked from the given bytes source, like a repeated\ninvocation of unpack_from().\n\nRequires that the bytes length be a multiple of the struct size.", + "_struct.Struct.pack" => "S.pack(v1, v2, ...) -> bytes\n\nReturn a bytes object containing values v1, v2, ... packed according\nto the format string S.format. See help(struct) for more on format\nstrings.", + "_struct.Struct.pack_into" => "S.pack_into(buffer, offset, v1, v2, ...)\n\nPack the values v1, v2, ... according to the format string S.format\nand write the packed bytes into the writable buffer buf starting at\noffset. Note that the offset is a required argument. See\nhelp(struct) for more on format strings.", + "_struct.Struct.size" => "struct size in bytes", + "_struct.Struct.unpack" => "Return a tuple containing unpacked values.\n\nUnpack according to the format string Struct.format. The buffer's size\nin bytes must be Struct.size.\n\nSee help(struct) for more on format strings.", + "_struct.Struct.unpack_from" => "Return a tuple containing unpacked values.\n\nValues are unpacked according to the format string Struct.format.\n\nThe buffer's size in bytes, starting at position offset, must be\nat least Struct.size.\n\nSee help(struct) for more on format strings.", + "_struct._clearcache" => "Clear the internal cache.", + "_struct.calcsize" => "Return size in bytes of the struct described by the format string.", + "_struct.error.__delattr__" => "Implement delattr(self, name).", + "_struct.error.__eq__" => "Return self==value.", + "_struct.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_struct.error.__ge__" => "Return self>=value.", + "_struct.error.__getattribute__" => "Return getattr(self, name).", + "_struct.error.__getstate__" => "Helper for pickle.", + "_struct.error.__gt__" => "Return self>value.", + "_struct.error.__hash__" => "Return hash(self).", + "_struct.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_struct.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_struct.error.__le__" => "Return self<=value.", + "_struct.error.__lt__" => "Return self<value.", + "_struct.error.__ne__" => "Return self!=value.", + "_struct.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_struct.error.__reduce_ex__" => "Helper for pickle.", + "_struct.error.__repr__" => "Return repr(self).", + "_struct.error.__setattr__" => "Implement setattr(self, name, value).", + "_struct.error.__sizeof__" => "Size of object in memory, in bytes.", + "_struct.error.__str__" => "Return str(self).", + "_struct.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_struct.error.__weakref__" => "list of weak references to the object", + "_struct.error.add_note" => "Add a note to the exception", + "_struct.error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_struct.iter_unpack" => "Return an iterator yielding tuples unpacked from the given bytes.\n\nThe bytes are unpacked according to the format string, like\na repeated invocation of unpack_from().\n\nRequires that the bytes length be a multiple of the format struct size.", + "_struct.pack" => "pack(format, v1, v2, ...) -> bytes\n\nReturn a bytes object containing the values v1, v2, ... packed according\nto the format string. See help(struct) for more on format strings.", + "_struct.pack_into" => "pack_into(format, buffer, offset, v1, v2, ...)\n\nPack the values v1, v2, ... according to the format string and write\nthe packed bytes into the writable buffer buf starting at offset. Note\nthat the offset is a required argument. See help(struct) for more\non format strings.", + "_struct.unpack" => "Return a tuple containing values unpacked according to the format string.\n\nThe buffer's size in bytes must be calcsize(format).\n\nSee help(struct) for more on format strings.", + "_struct.unpack_from" => "Return a tuple containing values unpacked according to the format string.\n\nThe buffer's size, minus offset, must be at least calcsize(format).\n\nSee help(struct) for more on format strings.", + "_suggestions._generate_suggestions" => "Returns the candidate in candidates that's closest to item", + "_symtable.symtable" => "Return symbol and scope dictionaries used internally by compiler.", + "_sysconfig" => "A helper for the sysconfig module.", + "_sysconfig.config_vars" => "Returns a dictionary containing build variables intended to be exposed by sysconfig.", + "_thread" => "This module provides primitive operations to write multi-threaded programs.\nThe 'threading' module provides a more convenient interface.", + "_thread.LockType" => "A lock object is a synchronization primitive. To create a lock,\ncall threading.Lock(). Methods are:\n\nacquire() -- lock the lock, possibly blocking until it can be obtained\nrelease() -- unlock of the lock\nlocked() -- test whether the lock is currently locked\n\nA lock is not owned by the thread that locked it; another thread may\nunlock it. A thread attempting to lock a lock that it has already locked\nwill block until another thread unlocks it. Deadlocks may ensue.", + "_thread.LockType.__delattr__" => "Implement delattr(self, name).", + "_thread.LockType.__enter__" => "Lock the lock.", + "_thread.LockType.__eq__" => "Return self==value.", + "_thread.LockType.__exit__" => "Release the lock.", + "_thread.LockType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_thread.LockType.__ge__" => "Return self>=value.", + "_thread.LockType.__getattribute__" => "Return getattr(self, name).", + "_thread.LockType.__getstate__" => "Helper for pickle.", + "_thread.LockType.__gt__" => "Return self>value.", + "_thread.LockType.__hash__" => "Return hash(self).", + "_thread.LockType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_thread.LockType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_thread.LockType.__le__" => "Return self<=value.", + "_thread.LockType.__lt__" => "Return self<value.", + "_thread.LockType.__ne__" => "Return self!=value.", + "_thread.LockType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_thread.LockType.__reduce__" => "Helper for pickle.", + "_thread.LockType.__reduce_ex__" => "Helper for pickle.", + "_thread.LockType.__repr__" => "Return repr(self).", + "_thread.LockType.__setattr__" => "Implement setattr(self, name, value).", + "_thread.LockType.__sizeof__" => "Size of object in memory, in bytes.", + "_thread.LockType.__str__" => "Return str(self).", + "_thread.LockType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_thread.LockType.acquire" => "Lock the lock. Without argument, this blocks if the lock is already\nlocked (even by the same thread), waiting for another thread to release\nthe lock, and return True once the lock is acquired.\nWith an argument, this will only block if the argument is true,\nand the return value reflects whether the lock is acquired.\nThe blocking operation is interruptible.", + "_thread.LockType.acquire_lock" => "An obsolete synonym of acquire().", + "_thread.LockType.locked" => "Return whether the lock is in the locked state.", + "_thread.LockType.locked_lock" => "An obsolete synonym of locked().", + "_thread.LockType.release" => "Release the lock, allowing another thread that is blocked waiting for\nthe lock to acquire the lock. The lock must be in the locked state,\nbut it needn't be locked by the same thread that unlocks it.", + "_thread.LockType.release_lock" => "An obsolete synonym of release().", + "_thread.RLock.__delattr__" => "Implement delattr(self, name).", + "_thread.RLock.__enter__" => "Lock the lock.", + "_thread.RLock.__eq__" => "Return self==value.", + "_thread.RLock.__exit__" => "Release the lock.", + "_thread.RLock.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_thread.RLock.__ge__" => "Return self>=value.", + "_thread.RLock.__getattribute__" => "Return getattr(self, name).", + "_thread.RLock.__getstate__" => "Helper for pickle.", + "_thread.RLock.__gt__" => "Return self>value.", + "_thread.RLock.__hash__" => "Return hash(self).", + "_thread.RLock.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_thread.RLock.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_thread.RLock.__le__" => "Return self<=value.", + "_thread.RLock.__lt__" => "Return self<value.", + "_thread.RLock.__ne__" => "Return self!=value.", + "_thread.RLock.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_thread.RLock.__reduce__" => "Helper for pickle.", + "_thread.RLock.__reduce_ex__" => "Helper for pickle.", + "_thread.RLock.__repr__" => "Return repr(self).", + "_thread.RLock.__setattr__" => "Implement setattr(self, name, value).", + "_thread.RLock.__sizeof__" => "Size of object in memory, in bytes.", + "_thread.RLock.__str__" => "Return str(self).", + "_thread.RLock.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_thread.RLock._acquire_restore" => "For internal use by `threading.Condition`.", + "_thread.RLock._is_owned" => "For internal use by `threading.Condition`.", + "_thread.RLock._recursion_count" => "For internal use by reentrancy checks.", + "_thread.RLock._release_save" => "For internal use by `threading.Condition`.", + "_thread.RLock.acquire" => "Lock the lock. `blocking` indicates whether we should wait\nfor the lock to be available or not. If `blocking` is False\nand another thread holds the lock, the method will return False\nimmediately. If `blocking` is True and another thread holds\nthe lock, the method will wait for the lock to be released,\ntake it and then return True.\n(note: the blocking operation is interruptible.)\n\nIn all other cases, the method will return True immediately.\nPrecisely, if the current thread already holds the lock, its\ninternal counter is simply incremented. If nobody holds the lock,\nthe lock is taken and its internal counter initialized to 1.", + "_thread.RLock.locked" => "locked()\n\nReturn a boolean indicating whether this object is locked right now.", + "_thread.RLock.release" => "Release the lock, allowing another thread that is blocked waiting for\nthe lock to acquire the lock. The lock must be in the locked state,\nand must be locked by the same thread that unlocks it; otherwise a\n`RuntimeError` is raised.\n\nDo note that if the lock was acquire()d several times in a row by the\ncurrent thread, release() needs to be called as many times for the lock\nto be available for other threads.", + "_thread._ExceptHookArgs" => "ExceptHookArgs\n\nType used to pass arguments to threading.excepthook.", + "_thread._ExceptHookArgs.__add__" => "Return self+value.", + "_thread._ExceptHookArgs.__class_getitem__" => "See PEP 585", + "_thread._ExceptHookArgs.__contains__" => "Return bool(key in self).", + "_thread._ExceptHookArgs.__delattr__" => "Implement delattr(self, name).", + "_thread._ExceptHookArgs.__eq__" => "Return self==value.", + "_thread._ExceptHookArgs.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_thread._ExceptHookArgs.__ge__" => "Return self>=value.", + "_thread._ExceptHookArgs.__getattribute__" => "Return getattr(self, name).", + "_thread._ExceptHookArgs.__getitem__" => "Return self[key].", + "_thread._ExceptHookArgs.__getstate__" => "Helper for pickle.", + "_thread._ExceptHookArgs.__gt__" => "Return self>value.", + "_thread._ExceptHookArgs.__hash__" => "Return hash(self).", + "_thread._ExceptHookArgs.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_thread._ExceptHookArgs.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_thread._ExceptHookArgs.__iter__" => "Implement iter(self).", + "_thread._ExceptHookArgs.__le__" => "Return self<=value.", + "_thread._ExceptHookArgs.__len__" => "Return len(self).", + "_thread._ExceptHookArgs.__lt__" => "Return self<value.", + "_thread._ExceptHookArgs.__mul__" => "Return self*value.", + "_thread._ExceptHookArgs.__ne__" => "Return self!=value.", + "_thread._ExceptHookArgs.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_thread._ExceptHookArgs.__reduce_ex__" => "Helper for pickle.", + "_thread._ExceptHookArgs.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "_thread._ExceptHookArgs.__repr__" => "Return repr(self).", + "_thread._ExceptHookArgs.__rmul__" => "Return value*self.", + "_thread._ExceptHookArgs.__setattr__" => "Implement setattr(self, name, value).", + "_thread._ExceptHookArgs.__sizeof__" => "Size of object in memory, in bytes.", + "_thread._ExceptHookArgs.__str__" => "Return str(self).", + "_thread._ExceptHookArgs.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_thread._ExceptHookArgs.count" => "Return number of occurrences of value.", + "_thread._ExceptHookArgs.exc_traceback" => "Exception traceback", + "_thread._ExceptHookArgs.exc_type" => "Exception type", + "_thread._ExceptHookArgs.exc_value" => "Exception value", + "_thread._ExceptHookArgs.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "_thread._ExceptHookArgs.thread" => "Thread", + "_thread._ThreadHandle.__delattr__" => "Implement delattr(self, name).", + "_thread._ThreadHandle.__eq__" => "Return self==value.", + "_thread._ThreadHandle.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_thread._ThreadHandle.__ge__" => "Return self>=value.", + "_thread._ThreadHandle.__getattribute__" => "Return getattr(self, name).", + "_thread._ThreadHandle.__getstate__" => "Helper for pickle.", + "_thread._ThreadHandle.__gt__" => "Return self>value.", + "_thread._ThreadHandle.__hash__" => "Return hash(self).", + "_thread._ThreadHandle.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_thread._ThreadHandle.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_thread._ThreadHandle.__le__" => "Return self<=value.", + "_thread._ThreadHandle.__lt__" => "Return self<value.", + "_thread._ThreadHandle.__ne__" => "Return self!=value.", + "_thread._ThreadHandle.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_thread._ThreadHandle.__reduce__" => "Helper for pickle.", + "_thread._ThreadHandle.__reduce_ex__" => "Helper for pickle.", + "_thread._ThreadHandle.__repr__" => "Return repr(self).", + "_thread._ThreadHandle.__setattr__" => "Implement setattr(self, name, value).", + "_thread._ThreadHandle.__sizeof__" => "Size of object in memory, in bytes.", + "_thread._ThreadHandle.__str__" => "Return str(self).", + "_thread._ThreadHandle.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_thread._count" => "Return the number of currently running Python threads, excluding\nthe main thread. The returned number comprises all threads created\nthrough `start_new_thread()` as well as `threading.Thread`, and not\nyet finished.\n\nThis function is meant for internal and specialized purposes only.\nIn most applications `threading.enumerate()` should be used instead.", + "_thread._excepthook" => "Handle uncaught Thread.run() exception.", + "_thread._get_main_thread_ident" => "Internal only. Return a non-zero integer that uniquely identifies the main thread\nof the main interpreter.", + "_thread._get_name" => "Get the name of the current thread.", + "_thread._is_main_interpreter" => "Return True if the current interpreter is the main Python interpreter.", + "_thread._local" => "Thread-local data", + "_thread._local.__delattr__" => "Implement delattr(self, name).", + "_thread._local.__eq__" => "Return self==value.", + "_thread._local.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_thread._local.__ge__" => "Return self>=value.", + "_thread._local.__getattribute__" => "Return getattr(self, name).", + "_thread._local.__getstate__" => "Helper for pickle.", + "_thread._local.__gt__" => "Return self>value.", + "_thread._local.__hash__" => "Return hash(self).", + "_thread._local.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_thread._local.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_thread._local.__le__" => "Return self<=value.", + "_thread._local.__lt__" => "Return self<value.", + "_thread._local.__ne__" => "Return self!=value.", + "_thread._local.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_thread._local.__reduce__" => "Helper for pickle.", + "_thread._local.__reduce_ex__" => "Helper for pickle.", + "_thread._local.__repr__" => "Return repr(self).", + "_thread._local.__setattr__" => "Implement setattr(self, name, value).", + "_thread._local.__sizeof__" => "Size of object in memory, in bytes.", + "_thread._local.__str__" => "Return str(self).", + "_thread._local.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_thread._make_thread_handle" => "Internal only. Make a thread handle for threads not spawned\nby the _thread or threading module.", + "_thread._shutdown" => "Wait for all non-daemon threads (other than the calling thread) to stop.", + "_thread.allocate" => "An obsolete synonym of allocate_lock().", + "_thread.allocate_lock" => "Create a new lock object. See help(type(threading.Lock())) for\ninformation about locks.", + "_thread.daemon_threads_allowed" => "Return True if daemon threads are allowed in the current interpreter,\nand False otherwise.", + "_thread.exit" => "This is synonymous to ``raise SystemExit''. It will cause the current\nthread to exit silently unless the exception is caught.", + "_thread.exit_thread" => "An obsolete synonym of exit().", + "_thread.get_ident" => "Return a non-zero integer that uniquely identifies the current thread\namongst other threads that exist simultaneously.\nThis may be used to identify per-thread resources.\nEven though on some platforms threads identities may appear to be\nallocated consecutive numbers starting at 1, this behavior should not\nbe relied upon, and the number should be seen purely as a magic cookie.\nA thread's identity may be reused for another thread after it exits.", + "_thread.get_native_id" => "Return a non-negative integer identifying the thread as reported\nby the OS (kernel). This may be used to uniquely identify a\nparticular thread within a system.", + "_thread.interrupt_main" => "Simulate the arrival of the given signal in the main thread,\nwhere the corresponding signal handler will be executed.\nIf *signum* is omitted, SIGINT is assumed.\nA subthread can use this function to interrupt the main thread.\n\nNote: the default signal handler for SIGINT raises ``KeyboardInterrupt``.", + "_thread.lock" => "A lock object is a synchronization primitive. To create a lock,\ncall threading.Lock(). Methods are:\n\nacquire() -- lock the lock, possibly blocking until it can be obtained\nrelease() -- unlock of the lock\nlocked() -- test whether the lock is currently locked\n\nA lock is not owned by the thread that locked it; another thread may\nunlock it. A thread attempting to lock a lock that it has already locked\nwill block until another thread unlocks it. Deadlocks may ensue.", + "_thread.lock.__delattr__" => "Implement delattr(self, name).", + "_thread.lock.__enter__" => "Lock the lock.", + "_thread.lock.__eq__" => "Return self==value.", + "_thread.lock.__exit__" => "Release the lock.", + "_thread.lock.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_thread.lock.__ge__" => "Return self>=value.", + "_thread.lock.__getattribute__" => "Return getattr(self, name).", + "_thread.lock.__getstate__" => "Helper for pickle.", + "_thread.lock.__gt__" => "Return self>value.", + "_thread.lock.__hash__" => "Return hash(self).", + "_thread.lock.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_thread.lock.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_thread.lock.__le__" => "Return self<=value.", + "_thread.lock.__lt__" => "Return self<value.", + "_thread.lock.__ne__" => "Return self!=value.", + "_thread.lock.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_thread.lock.__reduce__" => "Helper for pickle.", + "_thread.lock.__reduce_ex__" => "Helper for pickle.", + "_thread.lock.__repr__" => "Return repr(self).", + "_thread.lock.__setattr__" => "Implement setattr(self, name, value).", + "_thread.lock.__sizeof__" => "Size of object in memory, in bytes.", + "_thread.lock.__str__" => "Return str(self).", + "_thread.lock.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_thread.lock.acquire" => "Lock the lock. Without argument, this blocks if the lock is already\nlocked (even by the same thread), waiting for another thread to release\nthe lock, and return True once the lock is acquired.\nWith an argument, this will only block if the argument is true,\nand the return value reflects whether the lock is acquired.\nThe blocking operation is interruptible.", + "_thread.lock.acquire_lock" => "An obsolete synonym of acquire().", + "_thread.lock.locked" => "Return whether the lock is in the locked state.", + "_thread.lock.locked_lock" => "An obsolete synonym of locked().", + "_thread.lock.release" => "Release the lock, allowing another thread that is blocked waiting for\nthe lock to acquire the lock. The lock must be in the locked state,\nbut it needn't be locked by the same thread that unlocks it.", + "_thread.lock.release_lock" => "An obsolete synonym of release().", + "_thread.set_name" => "Set the name of the current thread.", + "_thread.stack_size" => "Return the thread stack size used when creating new threads. The\noptional size argument specifies the stack size (in bytes) to be used\nfor subsequently created threads, and must be 0 (use platform or\nconfigured default) or a positive integer value of at least 32,768 (32k).\nIf changing the thread stack size is unsupported, a ThreadError\nexception is raised. If the specified size is invalid, a ValueError\nexception is raised, and the stack size is unmodified. 32k bytes\n currently the minimum supported stack size value to guarantee\nsufficient stack space for the interpreter itself.\n\nNote that some platforms may have particular restrictions on values for\nthe stack size, such as requiring a minimum stack size larger than 32 KiB or\nrequiring allocation in multiples of the system memory page size\n- platform documentation should be referred to for more information\n(4 KiB pages are common; using multiples of 4096 for the stack size is\nthe suggested approach in the absence of more specific information).", + "_thread.start_joinable_thread" => "*For internal use only*: start a new thread.\n\nLike start_new_thread(), this starts a new thread calling the given function.\nUnlike start_new_thread(), this returns a handle object with methods to join\nor detach the given thread.\nThis function is not for third-party code, please use the\n`threading` module instead. During finalization the runtime will not wait for\nthe thread to exit if daemon is True. If handle is provided it must be a\nnewly created thread._ThreadHandle instance.", + "_thread.start_new" => "An obsolete synonym of start_new_thread().", + "_thread.start_new_thread" => "Start a new thread and return its identifier.\n\nThe thread will call the function with positional arguments from the\ntuple args and keyword arguments taken from the optional dictionary\nkwargs. The thread exits when the function returns; the return value\nis ignored. The thread will also exit when the function raises an\nunhandled exception; a stack trace will be printed unless the exception\nis SystemExit.", + "_tkinter.TclError.__delattr__" => "Implement delattr(self, name).", + "_tkinter.TclError.__eq__" => "Return self==value.", + "_tkinter.TclError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_tkinter.TclError.__ge__" => "Return self>=value.", + "_tkinter.TclError.__getattribute__" => "Return getattr(self, name).", + "_tkinter.TclError.__getstate__" => "Helper for pickle.", + "_tkinter.TclError.__gt__" => "Return self>value.", + "_tkinter.TclError.__hash__" => "Return hash(self).", + "_tkinter.TclError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_tkinter.TclError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_tkinter.TclError.__le__" => "Return self<=value.", + "_tkinter.TclError.__lt__" => "Return self<value.", + "_tkinter.TclError.__ne__" => "Return self!=value.", + "_tkinter.TclError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_tkinter.TclError.__reduce_ex__" => "Helper for pickle.", + "_tkinter.TclError.__repr__" => "Return repr(self).", + "_tkinter.TclError.__setattr__" => "Implement setattr(self, name, value).", + "_tkinter.TclError.__sizeof__" => "Size of object in memory, in bytes.", + "_tkinter.TclError.__str__" => "Return str(self).", + "_tkinter.TclError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_tkinter.TclError.__weakref__" => "list of weak references to the object", + "_tkinter.TclError.add_note" => "Add a note to the exception", + "_tkinter.TclError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_tkinter.Tcl_Obj.__delattr__" => "Implement delattr(self, name).", + "_tkinter.Tcl_Obj.__eq__" => "Return self==value.", + "_tkinter.Tcl_Obj.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_tkinter.Tcl_Obj.__ge__" => "Return self>=value.", + "_tkinter.Tcl_Obj.__getattribute__" => "Return getattr(self, name).", + "_tkinter.Tcl_Obj.__getstate__" => "Helper for pickle.", + "_tkinter.Tcl_Obj.__gt__" => "Return self>value.", + "_tkinter.Tcl_Obj.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_tkinter.Tcl_Obj.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_tkinter.Tcl_Obj.__le__" => "Return self<=value.", + "_tkinter.Tcl_Obj.__lt__" => "Return self<value.", + "_tkinter.Tcl_Obj.__ne__" => "Return self!=value.", + "_tkinter.Tcl_Obj.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_tkinter.Tcl_Obj.__reduce__" => "Helper for pickle.", + "_tkinter.Tcl_Obj.__reduce_ex__" => "Helper for pickle.", + "_tkinter.Tcl_Obj.__repr__" => "Return repr(self).", + "_tkinter.Tcl_Obj.__setattr__" => "Implement setattr(self, name, value).", + "_tkinter.Tcl_Obj.__sizeof__" => "Size of object in memory, in bytes.", + "_tkinter.Tcl_Obj.__str__" => "Return str(self).", + "_tkinter.Tcl_Obj.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_tkinter.Tcl_Obj.string" => "the string representation of this object, either as str or bytes", + "_tkinter.Tcl_Obj.typename" => "name of the Tcl type", + "_tkinter.TkappType.__delattr__" => "Implement delattr(self, name).", + "_tkinter.TkappType.__eq__" => "Return self==value.", + "_tkinter.TkappType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_tkinter.TkappType.__ge__" => "Return self>=value.", + "_tkinter.TkappType.__getattribute__" => "Return getattr(self, name).", + "_tkinter.TkappType.__getstate__" => "Helper for pickle.", + "_tkinter.TkappType.__gt__" => "Return self>value.", + "_tkinter.TkappType.__hash__" => "Return hash(self).", + "_tkinter.TkappType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_tkinter.TkappType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_tkinter.TkappType.__le__" => "Return self<=value.", + "_tkinter.TkappType.__lt__" => "Return self<value.", + "_tkinter.TkappType.__ne__" => "Return self!=value.", + "_tkinter.TkappType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_tkinter.TkappType.__reduce__" => "Helper for pickle.", + "_tkinter.TkappType.__reduce_ex__" => "Helper for pickle.", + "_tkinter.TkappType.__repr__" => "Return repr(self).", + "_tkinter.TkappType.__setattr__" => "Implement setattr(self, name, value).", + "_tkinter.TkappType.__sizeof__" => "Size of object in memory, in bytes.", + "_tkinter.TkappType.__str__" => "Return str(self).", + "_tkinter.TkappType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_tkinter.TkappType.gettrace" => "Get the tracing function.", + "_tkinter.TkappType.settrace" => "Set the tracing function.", + "_tkinter.TkttType.__delattr__" => "Implement delattr(self, name).", + "_tkinter.TkttType.__eq__" => "Return self==value.", + "_tkinter.TkttType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_tkinter.TkttType.__ge__" => "Return self>=value.", + "_tkinter.TkttType.__getattribute__" => "Return getattr(self, name).", + "_tkinter.TkttType.__getstate__" => "Helper for pickle.", + "_tkinter.TkttType.__gt__" => "Return self>value.", + "_tkinter.TkttType.__hash__" => "Return hash(self).", + "_tkinter.TkttType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_tkinter.TkttType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_tkinter.TkttType.__le__" => "Return self<=value.", + "_tkinter.TkttType.__lt__" => "Return self<value.", + "_tkinter.TkttType.__ne__" => "Return self!=value.", + "_tkinter.TkttType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_tkinter.TkttType.__reduce__" => "Helper for pickle.", + "_tkinter.TkttType.__reduce_ex__" => "Helper for pickle.", + "_tkinter.TkttType.__repr__" => "Return repr(self).", + "_tkinter.TkttType.__setattr__" => "Implement setattr(self, name, value).", + "_tkinter.TkttType.__sizeof__" => "Size of object in memory, in bytes.", + "_tkinter.TkttType.__str__" => "Return str(self).", + "_tkinter.TkttType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_tkinter.create" => "wantTk\n if false, then Tk_Init() doesn't get called\n sync\n if true, then pass -sync to wish\n use\n if not None, then pass -use to wish", + "_tkinter.getbusywaitinterval" => "Return the current busy-wait interval between successive calls to Tcl_DoOneEvent in a threaded Python interpreter.", + "_tkinter.setbusywaitinterval" => "Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter.\n\nIt should be set to a divisor of the maximum time between frames in an animation.", + "_tokenize.TokenizerIter.__delattr__" => "Implement delattr(self, name).", + "_tokenize.TokenizerIter.__eq__" => "Return self==value.", + "_tokenize.TokenizerIter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_tokenize.TokenizerIter.__ge__" => "Return self>=value.", + "_tokenize.TokenizerIter.__getattribute__" => "Return getattr(self, name).", + "_tokenize.TokenizerIter.__getstate__" => "Helper for pickle.", + "_tokenize.TokenizerIter.__gt__" => "Return self>value.", + "_tokenize.TokenizerIter.__hash__" => "Return hash(self).", + "_tokenize.TokenizerIter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_tokenize.TokenizerIter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_tokenize.TokenizerIter.__iter__" => "Implement iter(self).", + "_tokenize.TokenizerIter.__le__" => "Return self<=value.", + "_tokenize.TokenizerIter.__lt__" => "Return self<value.", + "_tokenize.TokenizerIter.__ne__" => "Return self!=value.", + "_tokenize.TokenizerIter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_tokenize.TokenizerIter.__next__" => "Implement next(self).", + "_tokenize.TokenizerIter.__reduce__" => "Helper for pickle.", + "_tokenize.TokenizerIter.__reduce_ex__" => "Helper for pickle.", + "_tokenize.TokenizerIter.__repr__" => "Return repr(self).", + "_tokenize.TokenizerIter.__setattr__" => "Implement setattr(self, name, value).", + "_tokenize.TokenizerIter.__sizeof__" => "Size of object in memory, in bytes.", + "_tokenize.TokenizerIter.__str__" => "Return str(self).", + "_tokenize.TokenizerIter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_tracemalloc" => "Debug module to trace memory blocks allocated by Python.", + "_tracemalloc._get_object_traceback" => "Get the traceback where the Python object obj was allocated.\n\nReturn a tuple of (filename: str, lineno: int) tuples.\nReturn None if the tracemalloc module is disabled or did not\ntrace the allocation of the object.", + "_tracemalloc._get_traces" => "Get traces of all memory blocks allocated by Python.\n\nReturn a list of (size: int, traceback: tuple) tuples.\ntraceback is a tuple of (filename: str, lineno: int) tuples.\n\nReturn an empty list if the tracemalloc module is disabled.", + "_tracemalloc.clear_traces" => "Clear traces of memory blocks allocated by Python.", + "_tracemalloc.get_traceback_limit" => "Get the maximum number of frames stored in the traceback of a trace.\n\nBy default, a trace of an allocated memory block only stores\nthe most recent frame: the limit is 1.", + "_tracemalloc.get_traced_memory" => "Get the current size and peak size of memory blocks traced by tracemalloc.\n\nReturns a tuple: (current: int, peak: int).", + "_tracemalloc.get_tracemalloc_memory" => "Get the memory usage in bytes of the tracemalloc module.\n\nThis memory is used internally to trace memory allocations.", + "_tracemalloc.is_tracing" => "Return True if the tracemalloc module is tracing Python memory allocations.", + "_tracemalloc.reset_peak" => "Set the peak size of memory blocks traced by tracemalloc to the current size.\n\nDo nothing if the tracemalloc module is not tracing memory allocations.", + "_tracemalloc.start" => "Start tracing Python memory allocations.\n\nAlso set the maximum number of frames stored in the traceback of a\ntrace to nframe.", + "_tracemalloc.stop" => "Stop tracing Python memory allocations.\n\nAlso clear traces of memory blocks allocated by Python.", + "_types" => "Define names for built-in types.", + "_types.GenericAlias" => "Represent a PEP 585 generic type\n\nE.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,).", + "_types.GenericAlias.__call__" => "Call self as a function.", + "_types.GenericAlias.__delattr__" => "Implement delattr(self, name).", + "_types.GenericAlias.__eq__" => "Return self==value.", + "_types.GenericAlias.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_types.GenericAlias.__ge__" => "Return self>=value.", + "_types.GenericAlias.__getattribute__" => "Return getattr(self, name).", + "_types.GenericAlias.__getitem__" => "Return self[key].", + "_types.GenericAlias.__getstate__" => "Helper for pickle.", + "_types.GenericAlias.__gt__" => "Return self>value.", + "_types.GenericAlias.__hash__" => "Return hash(self).", + "_types.GenericAlias.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_types.GenericAlias.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_types.GenericAlias.__iter__" => "Implement iter(self).", + "_types.GenericAlias.__le__" => "Return self<=value.", + "_types.GenericAlias.__lt__" => "Return self<value.", + "_types.GenericAlias.__ne__" => "Return self!=value.", + "_types.GenericAlias.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_types.GenericAlias.__or__" => "Return self|value.", + "_types.GenericAlias.__parameters__" => "Type variables in the GenericAlias.", + "_types.GenericAlias.__reduce_ex__" => "Helper for pickle.", + "_types.GenericAlias.__repr__" => "Return repr(self).", + "_types.GenericAlias.__ror__" => "Return value|self.", + "_types.GenericAlias.__setattr__" => "Implement setattr(self, name, value).", + "_types.GenericAlias.__sizeof__" => "Size of object in memory, in bytes.", + "_types.GenericAlias.__str__" => "Return str(self).", + "_types.GenericAlias.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_types.NoneType" => "The type of the None singleton.", + "_types.NoneType.__bool__" => "True if self else False", + "_types.NoneType.__delattr__" => "Implement delattr(self, name).", + "_types.NoneType.__eq__" => "Return self==value.", + "_types.NoneType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_types.NoneType.__ge__" => "Return self>=value.", + "_types.NoneType.__getattribute__" => "Return getattr(self, name).", + "_types.NoneType.__getstate__" => "Helper for pickle.", + "_types.NoneType.__gt__" => "Return self>value.", + "_types.NoneType.__hash__" => "Return hash(self).", + "_types.NoneType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_types.NoneType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_types.NoneType.__le__" => "Return self<=value.", + "_types.NoneType.__lt__" => "Return self<value.", + "_types.NoneType.__ne__" => "Return self!=value.", + "_types.NoneType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_types.NoneType.__reduce__" => "Helper for pickle.", + "_types.NoneType.__reduce_ex__" => "Helper for pickle.", + "_types.NoneType.__repr__" => "Return repr(self).", + "_types.NoneType.__setattr__" => "Implement setattr(self, name, value).", + "_types.NoneType.__sizeof__" => "Size of object in memory, in bytes.", + "_types.NoneType.__str__" => "Return str(self).", + "_types.NoneType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_types.NotImplementedType" => "The type of the NotImplemented singleton.", + "_types.NotImplementedType.__bool__" => "True if self else False", + "_types.NotImplementedType.__delattr__" => "Implement delattr(self, name).", + "_types.NotImplementedType.__eq__" => "Return self==value.", + "_types.NotImplementedType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_types.NotImplementedType.__ge__" => "Return self>=value.", + "_types.NotImplementedType.__getattribute__" => "Return getattr(self, name).", + "_types.NotImplementedType.__getstate__" => "Helper for pickle.", + "_types.NotImplementedType.__gt__" => "Return self>value.", + "_types.NotImplementedType.__hash__" => "Return hash(self).", + "_types.NotImplementedType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_types.NotImplementedType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_types.NotImplementedType.__le__" => "Return self<=value.", + "_types.NotImplementedType.__lt__" => "Return self<value.", + "_types.NotImplementedType.__ne__" => "Return self!=value.", + "_types.NotImplementedType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_types.NotImplementedType.__reduce_ex__" => "Helper for pickle.", + "_types.NotImplementedType.__repr__" => "Return repr(self).", + "_types.NotImplementedType.__setattr__" => "Implement setattr(self, name, value).", + "_types.NotImplementedType.__sizeof__" => "Size of object in memory, in bytes.", + "_types.NotImplementedType.__str__" => "Return str(self).", + "_types.NotImplementedType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_types.SimpleNamespace" => "A simple attribute-based namespace.", + "_types.SimpleNamespace.__delattr__" => "Implement delattr(self, name).", + "_types.SimpleNamespace.__eq__" => "Return self==value.", + "_types.SimpleNamespace.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_types.SimpleNamespace.__ge__" => "Return self>=value.", + "_types.SimpleNamespace.__getattribute__" => "Return getattr(self, name).", + "_types.SimpleNamespace.__getstate__" => "Helper for pickle.", + "_types.SimpleNamespace.__gt__" => "Return self>value.", + "_types.SimpleNamespace.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_types.SimpleNamespace.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_types.SimpleNamespace.__le__" => "Return self<=value.", + "_types.SimpleNamespace.__lt__" => "Return self<value.", + "_types.SimpleNamespace.__ne__" => "Return self!=value.", + "_types.SimpleNamespace.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_types.SimpleNamespace.__reduce__" => "Return state information for pickling", + "_types.SimpleNamespace.__reduce_ex__" => "Helper for pickle.", + "_types.SimpleNamespace.__replace__" => "Return a copy of the namespace object with new values for the specified attributes.", + "_types.SimpleNamespace.__repr__" => "Return repr(self).", + "_types.SimpleNamespace.__setattr__" => "Implement setattr(self, name, value).", + "_types.SimpleNamespace.__sizeof__" => "Size of object in memory, in bytes.", + "_types.SimpleNamespace.__str__" => "Return str(self).", + "_types.SimpleNamespace.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing" => "Primitives and accelerators for the typing module.", + "_typing.Generic" => "Abstract base class for generic types.\n\nOn Python 3.12 and newer, generic classes implicitly inherit from\nGeneric when they declare a parameter list after the class's name::\n\n class Mapping[KT, VT]:\n def __getitem__(self, key: KT) -> VT:\n ...\n # Etc.\n\nOn older versions of Python, however, generic classes have to\nexplicitly inherit from Generic.\n\nAfter a class has been declared to be generic, it can then be used as\nfollows::\n\n def lookup_name[KT, VT](mapping: Mapping[KT, VT], key: KT, default: VT) -> VT:\n try:\n return mapping[key]\n except KeyError:\n return default", + "_typing.Generic.__class_getitem__" => "Parameterizes a generic class.\n\nAt least, parameterizing a generic class is the *main* thing this\nmethod does. For example, for some generic class `Foo`, this is called\nwhen we do `Foo[int]` - there, with `cls=Foo` and `params=int`.\n\nHowever, note that this method is also called when defining generic\nclasses in the first place with `class Foo[T]: ...`.", + "_typing.Generic.__delattr__" => "Implement delattr(self, name).", + "_typing.Generic.__eq__" => "Return self==value.", + "_typing.Generic.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.Generic.__ge__" => "Return self>=value.", + "_typing.Generic.__getattribute__" => "Return getattr(self, name).", + "_typing.Generic.__getstate__" => "Helper for pickle.", + "_typing.Generic.__gt__" => "Return self>value.", + "_typing.Generic.__hash__" => "Return hash(self).", + "_typing.Generic.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.Generic.__init_subclass__" => "Function to initialize subclasses.", + "_typing.Generic.__le__" => "Return self<=value.", + "_typing.Generic.__lt__" => "Return self<value.", + "_typing.Generic.__ne__" => "Return self!=value.", + "_typing.Generic.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.Generic.__reduce__" => "Helper for pickle.", + "_typing.Generic.__reduce_ex__" => "Helper for pickle.", + "_typing.Generic.__repr__" => "Return repr(self).", + "_typing.Generic.__setattr__" => "Implement setattr(self, name, value).", + "_typing.Generic.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.Generic.__str__" => "Return str(self).", + "_typing.Generic.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.ParamSpec" => "Parameter specification variable.\n\nThe preferred way to construct a parameter specification is via the\ndedicated syntax for generic functions, classes, and type aliases,\nwhere the use of '**' creates a parameter specification::\n\n type IntFunc[**P] = Callable[P, int]\n\nThe following syntax creates a parameter specification that defaults\nto a callable accepting two positional-only arguments of types int\nand str:\n\n type IntFuncDefault[**P = [int, str]] = Callable[P, int]\n\nFor compatibility with Python 3.11 and earlier, ParamSpec objects\ncan also be created as follows::\n\n P = ParamSpec('P')\n DefaultP = ParamSpec('DefaultP', default=[int, str])\n\nParameter specification variables exist primarily for the benefit of\nstatic type checkers. They are used to forward the parameter types of\none callable to another callable, a pattern commonly found in\nhigher-order functions and decorators. They are only valid when used\nin ``Concatenate``, or as the first argument to ``Callable``, or as\nparameters for user-defined Generics. See class Generic for more\ninformation on generic types.\n\nAn example for annotating a decorator::\n\n def add_logging[**P, T](f: Callable[P, T]) -> Callable[P, T]:\n '''A type-safe decorator to add logging to a function.'''\n def inner(*args: P.args, **kwargs: P.kwargs) -> T:\n logging.info(f'{f.__name__} was called')\n return f(*args, **kwargs)\n return inner\n\n @add_logging\n def add_two(x: float, y: float) -> float:\n '''Add two numbers together.'''\n return x + y\n\nParameter specification variables can be introspected. e.g.::\n\n >>> P = ParamSpec(\"P\")\n >>> P.__name__\n 'P'\n\nNote that only parameter specification variables defined in the global\nscope can be pickled.", + "_typing.ParamSpec.__default__" => "The default value for this ParamSpec.", + "_typing.ParamSpec.__delattr__" => "Implement delattr(self, name).", + "_typing.ParamSpec.__eq__" => "Return self==value.", + "_typing.ParamSpec.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.ParamSpec.__ge__" => "Return self>=value.", + "_typing.ParamSpec.__getattribute__" => "Return getattr(self, name).", + "_typing.ParamSpec.__getstate__" => "Helper for pickle.", + "_typing.ParamSpec.__gt__" => "Return self>value.", + "_typing.ParamSpec.__hash__" => "Return hash(self).", + "_typing.ParamSpec.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.ParamSpec.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.ParamSpec.__le__" => "Return self<=value.", + "_typing.ParamSpec.__lt__" => "Return self<value.", + "_typing.ParamSpec.__ne__" => "Return self!=value.", + "_typing.ParamSpec.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.ParamSpec.__or__" => "Return self|value.", + "_typing.ParamSpec.__reduce_ex__" => "Helper for pickle.", + "_typing.ParamSpec.__repr__" => "Return repr(self).", + "_typing.ParamSpec.__ror__" => "Return value|self.", + "_typing.ParamSpec.__setattr__" => "Implement setattr(self, name, value).", + "_typing.ParamSpec.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.ParamSpec.__str__" => "Return str(self).", + "_typing.ParamSpec.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.ParamSpec.args" => "Represents positional arguments.", + "_typing.ParamSpec.kwargs" => "Represents keyword arguments.", + "_typing.ParamSpecArgs" => "The args for a ParamSpec object.\n\nGiven a ParamSpec object P, P.args is an instance of ParamSpecArgs.\n\nParamSpecArgs objects have a reference back to their ParamSpec::\n\n >>> P = ParamSpec(\"P\")\n >>> P.args.__origin__ is P\n True\n\nThis type is meant for runtime introspection and has no special meaning\nto static type checkers.", + "_typing.ParamSpecArgs.__delattr__" => "Implement delattr(self, name).", + "_typing.ParamSpecArgs.__eq__" => "Return self==value.", + "_typing.ParamSpecArgs.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.ParamSpecArgs.__ge__" => "Return self>=value.", + "_typing.ParamSpecArgs.__getattribute__" => "Return getattr(self, name).", + "_typing.ParamSpecArgs.__getstate__" => "Helper for pickle.", + "_typing.ParamSpecArgs.__gt__" => "Return self>value.", + "_typing.ParamSpecArgs.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.ParamSpecArgs.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.ParamSpecArgs.__le__" => "Return self<=value.", + "_typing.ParamSpecArgs.__lt__" => "Return self<value.", + "_typing.ParamSpecArgs.__ne__" => "Return self!=value.", + "_typing.ParamSpecArgs.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.ParamSpecArgs.__reduce__" => "Helper for pickle.", + "_typing.ParamSpecArgs.__reduce_ex__" => "Helper for pickle.", + "_typing.ParamSpecArgs.__repr__" => "Return repr(self).", + "_typing.ParamSpecArgs.__setattr__" => "Implement setattr(self, name, value).", + "_typing.ParamSpecArgs.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.ParamSpecArgs.__str__" => "Return str(self).", + "_typing.ParamSpecArgs.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.ParamSpecKwargs" => "The kwargs for a ParamSpec object.\n\nGiven a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs.\n\nParamSpecKwargs objects have a reference back to their ParamSpec::\n\n >>> P = ParamSpec(\"P\")\n >>> P.kwargs.__origin__ is P\n True\n\nThis type is meant for runtime introspection and has no special meaning\nto static type checkers.", + "_typing.ParamSpecKwargs.__delattr__" => "Implement delattr(self, name).", + "_typing.ParamSpecKwargs.__eq__" => "Return self==value.", + "_typing.ParamSpecKwargs.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.ParamSpecKwargs.__ge__" => "Return self>=value.", + "_typing.ParamSpecKwargs.__getattribute__" => "Return getattr(self, name).", + "_typing.ParamSpecKwargs.__getstate__" => "Helper for pickle.", + "_typing.ParamSpecKwargs.__gt__" => "Return self>value.", + "_typing.ParamSpecKwargs.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.ParamSpecKwargs.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.ParamSpecKwargs.__le__" => "Return self<=value.", + "_typing.ParamSpecKwargs.__lt__" => "Return self<value.", + "_typing.ParamSpecKwargs.__ne__" => "Return self!=value.", + "_typing.ParamSpecKwargs.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.ParamSpecKwargs.__reduce__" => "Helper for pickle.", + "_typing.ParamSpecKwargs.__reduce_ex__" => "Helper for pickle.", + "_typing.ParamSpecKwargs.__repr__" => "Return repr(self).", + "_typing.ParamSpecKwargs.__setattr__" => "Implement setattr(self, name, value).", + "_typing.ParamSpecKwargs.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.ParamSpecKwargs.__str__" => "Return str(self).", + "_typing.ParamSpecKwargs.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.TypeAliasType" => "Type alias.\n\nType aliases are created through the type statement::\n\n type Alias = int\n\nIn this example, Alias and int will be treated equivalently by static\ntype checkers.\n\nAt runtime, Alias is an instance of TypeAliasType. The __name__\nattribute holds the name of the type alias. The value of the type alias\nis stored in the __value__ attribute. It is evaluated lazily, so the\nvalue is computed only if the attribute is accessed.\n\nType aliases can also be generic::\n\n type ListOrSet[T] = list[T] | set[T]\n\nIn this case, the type parameters of the alias are stored in the\n__type_params__ attribute.\n\nSee PEP 695 for more information.", + "_typing.TypeAliasType.__delattr__" => "Implement delattr(self, name).", + "_typing.TypeAliasType.__eq__" => "Return self==value.", + "_typing.TypeAliasType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.TypeAliasType.__ge__" => "Return self>=value.", + "_typing.TypeAliasType.__getattribute__" => "Return getattr(self, name).", + "_typing.TypeAliasType.__getitem__" => "Return self[key].", + "_typing.TypeAliasType.__getstate__" => "Helper for pickle.", + "_typing.TypeAliasType.__gt__" => "Return self>value.", + "_typing.TypeAliasType.__hash__" => "Return hash(self).", + "_typing.TypeAliasType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.TypeAliasType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.TypeAliasType.__iter__" => "Implement iter(self).", + "_typing.TypeAliasType.__le__" => "Return self<=value.", + "_typing.TypeAliasType.__lt__" => "Return self<value.", + "_typing.TypeAliasType.__ne__" => "Return self!=value.", + "_typing.TypeAliasType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.TypeAliasType.__or__" => "Return self|value.", + "_typing.TypeAliasType.__reduce_ex__" => "Helper for pickle.", + "_typing.TypeAliasType.__repr__" => "Return repr(self).", + "_typing.TypeAliasType.__ror__" => "Return value|self.", + "_typing.TypeAliasType.__setattr__" => "Implement setattr(self, name, value).", + "_typing.TypeAliasType.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.TypeAliasType.__str__" => "Return str(self).", + "_typing.TypeAliasType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.TypeVar" => "Type variable.\n\nThe preferred way to construct a type variable is via the dedicated\nsyntax for generic functions, classes, and type aliases::\n\n class Sequence[T]: # T is a TypeVar\n ...\n\nThis syntax can also be used to create bound and constrained type\nvariables::\n\n # S is a TypeVar bound to str\n class StrSequence[S: str]:\n ...\n\n # A is a TypeVar constrained to str or bytes\n class StrOrBytesSequence[A: (str, bytes)]:\n ...\n\nType variables can also have defaults:\n\n class IntDefault[T = int]:\n ...\n\nHowever, if desired, reusable type variables can also be constructed\nmanually, like so::\n\n T = TypeVar('T') # Can be anything\n S = TypeVar('S', bound=str) # Can be any subtype of str\n A = TypeVar('A', str, bytes) # Must be exactly str or bytes\n D = TypeVar('D', default=int) # Defaults to int\n\nType variables exist primarily for the benefit of static type\ncheckers. They serve as the parameters for generic types as well\nas for generic function and type alias definitions.\n\nThe variance of type variables is inferred by type checkers when they\nare created through the type parameter syntax and when\n``infer_variance=True`` is passed. Manually created type variables may\nbe explicitly marked covariant or contravariant by passing\n``covariant=True`` or ``contravariant=True``. By default, manually\ncreated type variables are invariant. See PEP 484 and PEP 695 for more\ndetails.", + "_typing.TypeVar.__delattr__" => "Implement delattr(self, name).", + "_typing.TypeVar.__eq__" => "Return self==value.", + "_typing.TypeVar.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.TypeVar.__ge__" => "Return self>=value.", + "_typing.TypeVar.__getattribute__" => "Return getattr(self, name).", + "_typing.TypeVar.__getstate__" => "Helper for pickle.", + "_typing.TypeVar.__gt__" => "Return self>value.", + "_typing.TypeVar.__hash__" => "Return hash(self).", + "_typing.TypeVar.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.TypeVar.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.TypeVar.__le__" => "Return self<=value.", + "_typing.TypeVar.__lt__" => "Return self<value.", + "_typing.TypeVar.__ne__" => "Return self!=value.", + "_typing.TypeVar.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.TypeVar.__or__" => "Return self|value.", + "_typing.TypeVar.__reduce_ex__" => "Helper for pickle.", + "_typing.TypeVar.__repr__" => "Return repr(self).", + "_typing.TypeVar.__ror__" => "Return value|self.", + "_typing.TypeVar.__setattr__" => "Implement setattr(self, name, value).", + "_typing.TypeVar.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.TypeVar.__str__" => "Return str(self).", + "_typing.TypeVar.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.TypeVarTuple" => "Type variable tuple. A specialized form of type variable that enables\nvariadic generics.\n\nThe preferred way to construct a type variable tuple is via the\ndedicated syntax for generic functions, classes, and type aliases,\nwhere a single '*' indicates a type variable tuple::\n\n def move_first_element_to_last[T, *Ts](tup: tuple[T, *Ts]) -> tuple[*Ts, T]:\n return (*tup[1:], tup[0])\n\nType variables tuples can have default values:\n\n type AliasWithDefault[*Ts = (str, int)] = tuple[*Ts]\n\nFor compatibility with Python 3.11 and earlier, TypeVarTuple objects\ncan also be created as follows::\n\n Ts = TypeVarTuple('Ts') # Can be given any name\n DefaultTs = TypeVarTuple('Ts', default=(str, int))\n\nJust as a TypeVar (type variable) is a placeholder for a single type,\na TypeVarTuple is a placeholder for an *arbitrary* number of types. For\nexample, if we define a generic class using a TypeVarTuple::\n\n class C[*Ts]: ...\n\nThen we can parameterize that class with an arbitrary number of type\narguments::\n\n C[int] # Fine\n C[int, str] # Also fine\n C[()] # Even this is fine\n\nFor more details, see PEP 646.\n\nNote that only TypeVarTuples defined in the global scope can be\npickled.", + "_typing.TypeVarTuple.__default__" => "The default value for this TypeVarTuple.", + "_typing.TypeVarTuple.__delattr__" => "Implement delattr(self, name).", + "_typing.TypeVarTuple.__eq__" => "Return self==value.", + "_typing.TypeVarTuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.TypeVarTuple.__ge__" => "Return self>=value.", + "_typing.TypeVarTuple.__getattribute__" => "Return getattr(self, name).", + "_typing.TypeVarTuple.__getstate__" => "Helper for pickle.", + "_typing.TypeVarTuple.__gt__" => "Return self>value.", + "_typing.TypeVarTuple.__hash__" => "Return hash(self).", + "_typing.TypeVarTuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.TypeVarTuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.TypeVarTuple.__iter__" => "Implement iter(self).", + "_typing.TypeVarTuple.__le__" => "Return self<=value.", + "_typing.TypeVarTuple.__lt__" => "Return self<value.", + "_typing.TypeVarTuple.__ne__" => "Return self!=value.", + "_typing.TypeVarTuple.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.TypeVarTuple.__reduce_ex__" => "Helper for pickle.", + "_typing.TypeVarTuple.__repr__" => "Return repr(self).", + "_typing.TypeVarTuple.__setattr__" => "Implement setattr(self, name, value).", + "_typing.TypeVarTuple.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.TypeVarTuple.__str__" => "Return str(self).", + "_typing.TypeVarTuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_typing.Union" => "Represent a union type\n\nE.g. for int | str", + "_typing.Union.__class_getitem__" => "See PEP 585", + "_typing.Union.__delattr__" => "Implement delattr(self, name).", + "_typing.Union.__eq__" => "Return self==value.", + "_typing.Union.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_typing.Union.__ge__" => "Return self>=value.", + "_typing.Union.__getattribute__" => "Return getattr(self, name).", + "_typing.Union.__getitem__" => "Return self[key].", + "_typing.Union.__getstate__" => "Helper for pickle.", + "_typing.Union.__gt__" => "Return self>value.", + "_typing.Union.__hash__" => "Return hash(self).", + "_typing.Union.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_typing.Union.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_typing.Union.__le__" => "Return self<=value.", + "_typing.Union.__lt__" => "Return self<value.", + "_typing.Union.__ne__" => "Return self!=value.", + "_typing.Union.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_typing.Union.__or__" => "Return self|value.", + "_typing.Union.__origin__" => "Always returns the type", + "_typing.Union.__parameters__" => "Type variables in the types.UnionType.", + "_typing.Union.__reduce__" => "Helper for pickle.", + "_typing.Union.__reduce_ex__" => "Helper for pickle.", + "_typing.Union.__repr__" => "Return repr(self).", + "_typing.Union.__ror__" => "Return value|self.", + "_typing.Union.__setattr__" => "Implement setattr(self, name, value).", + "_typing.Union.__sizeof__" => "Size of object in memory, in bytes.", + "_typing.Union.__str__" => "Return str(self).", + "_typing.Union.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_warnings" => "_warnings provides basic warning filtering support.\nIt is a helper module to speed up interpreter start-up.", + "_warnings.warn" => "Issue a warning, or maybe ignore it or raise an exception.\n\n message\n Text of the warning message.\n category\n The Warning category subclass. Defaults to UserWarning.\n stacklevel\n How far up the call stack to make this warning appear. A value of 2 for\n example attributes the warning to the caller of the code calling warn().\n source\n If supplied, the destroyed object which emitted a ResourceWarning\n skip_file_prefixes\n An optional tuple of module filename prefixes indicating frames to skip\n during stacklevel computations for stack frame attribution.", + "_warnings.warn_explicit" => "Issue a warning, or maybe ignore it or raise an exception.", + "_weakref" => "Weak-reference support module.", + "_weakref.CallableProxyType.__abs__" => "abs(self)", + "_weakref.CallableProxyType.__add__" => "Return self+value.", + "_weakref.CallableProxyType.__and__" => "Return self&value.", + "_weakref.CallableProxyType.__bool__" => "True if self else False", + "_weakref.CallableProxyType.__call__" => "Call self as a function.", + "_weakref.CallableProxyType.__contains__" => "Return bool(key in self).", + "_weakref.CallableProxyType.__delattr__" => "Implement delattr(self, name).", + "_weakref.CallableProxyType.__delitem__" => "Delete self[key].", + "_weakref.CallableProxyType.__divmod__" => "Return divmod(self, value).", + "_weakref.CallableProxyType.__eq__" => "Return self==value.", + "_weakref.CallableProxyType.__float__" => "float(self)", + "_weakref.CallableProxyType.__floordiv__" => "Return self//value.", + "_weakref.CallableProxyType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_weakref.CallableProxyType.__ge__" => "Return self>=value.", + "_weakref.CallableProxyType.__getattribute__" => "Return getattr(self, name).", + "_weakref.CallableProxyType.__getitem__" => "Return self[key].", + "_weakref.CallableProxyType.__getstate__" => "Helper for pickle.", + "_weakref.CallableProxyType.__gt__" => "Return self>value.", + "_weakref.CallableProxyType.__iadd__" => "Return self+=value.", + "_weakref.CallableProxyType.__iand__" => "Return self&=value.", + "_weakref.CallableProxyType.__ifloordiv__" => "Return self//=value.", + "_weakref.CallableProxyType.__ilshift__" => "Return self<<=value.", + "_weakref.CallableProxyType.__imatmul__" => "Return self@=value.", + "_weakref.CallableProxyType.__imod__" => "Return self%=value.", + "_weakref.CallableProxyType.__imul__" => "Return self*=value.", + "_weakref.CallableProxyType.__index__" => "Return self converted to an integer, if self is suitable for use as an index into a list.", + "_weakref.CallableProxyType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_weakref.CallableProxyType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_weakref.CallableProxyType.__int__" => "int(self)", + "_weakref.CallableProxyType.__invert__" => "~self", + "_weakref.CallableProxyType.__ior__" => "Return self|=value.", + "_weakref.CallableProxyType.__ipow__" => "Return self**=value.", + "_weakref.CallableProxyType.__irshift__" => "Return self>>=value.", + "_weakref.CallableProxyType.__isub__" => "Return self-=value.", + "_weakref.CallableProxyType.__iter__" => "Implement iter(self).", + "_weakref.CallableProxyType.__itruediv__" => "Return self/=value.", + "_weakref.CallableProxyType.__ixor__" => "Return self^=value.", + "_weakref.CallableProxyType.__le__" => "Return self<=value.", + "_weakref.CallableProxyType.__len__" => "Return len(self).", + "_weakref.CallableProxyType.__lshift__" => "Return self<<value.", + "_weakref.CallableProxyType.__lt__" => "Return self<value.", + "_weakref.CallableProxyType.__matmul__" => "Return self@value.", + "_weakref.CallableProxyType.__mod__" => "Return self%value.", + "_weakref.CallableProxyType.__mul__" => "Return self*value.", + "_weakref.CallableProxyType.__ne__" => "Return self!=value.", + "_weakref.CallableProxyType.__neg__" => "-self", + "_weakref.CallableProxyType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_weakref.CallableProxyType.__next__" => "Implement next(self).", + "_weakref.CallableProxyType.__or__" => "Return self|value.", + "_weakref.CallableProxyType.__pos__" => "+self", + "_weakref.CallableProxyType.__pow__" => "Return pow(self, value, mod).", + "_weakref.CallableProxyType.__radd__" => "Return value+self.", + "_weakref.CallableProxyType.__rand__" => "Return value&self.", + "_weakref.CallableProxyType.__rdivmod__" => "Return divmod(value, self).", + "_weakref.CallableProxyType.__reduce__" => "Helper for pickle.", + "_weakref.CallableProxyType.__reduce_ex__" => "Helper for pickle.", + "_weakref.CallableProxyType.__repr__" => "Return repr(self).", + "_weakref.CallableProxyType.__rfloordiv__" => "Return value//self.", + "_weakref.CallableProxyType.__rlshift__" => "Return value<<self.", + "_weakref.CallableProxyType.__rmatmul__" => "Return value@self.", + "_weakref.CallableProxyType.__rmod__" => "Return value%self.", + "_weakref.CallableProxyType.__rmul__" => "Return value*self.", + "_weakref.CallableProxyType.__ror__" => "Return value|self.", + "_weakref.CallableProxyType.__rpow__" => "Return pow(value, self, mod).", + "_weakref.CallableProxyType.__rrshift__" => "Return value>>self.", + "_weakref.CallableProxyType.__rshift__" => "Return self>>value.", + "_weakref.CallableProxyType.__rsub__" => "Return value-self.", + "_weakref.CallableProxyType.__rtruediv__" => "Return value/self.", + "_weakref.CallableProxyType.__rxor__" => "Return value^self.", + "_weakref.CallableProxyType.__setattr__" => "Implement setattr(self, name, value).", + "_weakref.CallableProxyType.__setitem__" => "Set self[key] to value.", + "_weakref.CallableProxyType.__sizeof__" => "Size of object in memory, in bytes.", + "_weakref.CallableProxyType.__str__" => "Return str(self).", + "_weakref.CallableProxyType.__sub__" => "Return self-value.", + "_weakref.CallableProxyType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_weakref.CallableProxyType.__truediv__" => "Return self/value.", + "_weakref.CallableProxyType.__xor__" => "Return self^value.", + "_weakref.ProxyType.__abs__" => "abs(self)", + "_weakref.ProxyType.__add__" => "Return self+value.", + "_weakref.ProxyType.__and__" => "Return self&value.", + "_weakref.ProxyType.__bool__" => "True if self else False", + "_weakref.ProxyType.__contains__" => "Return bool(key in self).", + "_weakref.ProxyType.__delattr__" => "Implement delattr(self, name).", + "_weakref.ProxyType.__delitem__" => "Delete self[key].", + "_weakref.ProxyType.__divmod__" => "Return divmod(self, value).", + "_weakref.ProxyType.__eq__" => "Return self==value.", + "_weakref.ProxyType.__float__" => "float(self)", + "_weakref.ProxyType.__floordiv__" => "Return self//value.", + "_weakref.ProxyType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_weakref.ProxyType.__ge__" => "Return self>=value.", + "_weakref.ProxyType.__getattribute__" => "Return getattr(self, name).", + "_weakref.ProxyType.__getitem__" => "Return self[key].", + "_weakref.ProxyType.__getstate__" => "Helper for pickle.", + "_weakref.ProxyType.__gt__" => "Return self>value.", + "_weakref.ProxyType.__iadd__" => "Return self+=value.", + "_weakref.ProxyType.__iand__" => "Return self&=value.", + "_weakref.ProxyType.__ifloordiv__" => "Return self//=value.", + "_weakref.ProxyType.__ilshift__" => "Return self<<=value.", + "_weakref.ProxyType.__imatmul__" => "Return self@=value.", + "_weakref.ProxyType.__imod__" => "Return self%=value.", + "_weakref.ProxyType.__imul__" => "Return self*=value.", + "_weakref.ProxyType.__index__" => "Return self converted to an integer, if self is suitable for use as an index into a list.", + "_weakref.ProxyType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_weakref.ProxyType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_weakref.ProxyType.__int__" => "int(self)", + "_weakref.ProxyType.__invert__" => "~self", + "_weakref.ProxyType.__ior__" => "Return self|=value.", + "_weakref.ProxyType.__ipow__" => "Return self**=value.", + "_weakref.ProxyType.__irshift__" => "Return self>>=value.", + "_weakref.ProxyType.__isub__" => "Return self-=value.", + "_weakref.ProxyType.__iter__" => "Implement iter(self).", + "_weakref.ProxyType.__itruediv__" => "Return self/=value.", + "_weakref.ProxyType.__ixor__" => "Return self^=value.", + "_weakref.ProxyType.__le__" => "Return self<=value.", + "_weakref.ProxyType.__len__" => "Return len(self).", + "_weakref.ProxyType.__lshift__" => "Return self<<value.", + "_weakref.ProxyType.__lt__" => "Return self<value.", + "_weakref.ProxyType.__matmul__" => "Return self@value.", + "_weakref.ProxyType.__mod__" => "Return self%value.", + "_weakref.ProxyType.__mul__" => "Return self*value.", + "_weakref.ProxyType.__ne__" => "Return self!=value.", + "_weakref.ProxyType.__neg__" => "-self", + "_weakref.ProxyType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_weakref.ProxyType.__next__" => "Implement next(self).", + "_weakref.ProxyType.__or__" => "Return self|value.", + "_weakref.ProxyType.__pos__" => "+self", + "_weakref.ProxyType.__pow__" => "Return pow(self, value, mod).", + "_weakref.ProxyType.__radd__" => "Return value+self.", + "_weakref.ProxyType.__rand__" => "Return value&self.", + "_weakref.ProxyType.__rdivmod__" => "Return divmod(value, self).", + "_weakref.ProxyType.__reduce__" => "Helper for pickle.", + "_weakref.ProxyType.__reduce_ex__" => "Helper for pickle.", + "_weakref.ProxyType.__repr__" => "Return repr(self).", + "_weakref.ProxyType.__rfloordiv__" => "Return value//self.", + "_weakref.ProxyType.__rlshift__" => "Return value<<self.", + "_weakref.ProxyType.__rmatmul__" => "Return value@self.", + "_weakref.ProxyType.__rmod__" => "Return value%self.", + "_weakref.ProxyType.__rmul__" => "Return value*self.", + "_weakref.ProxyType.__ror__" => "Return value|self.", + "_weakref.ProxyType.__rpow__" => "Return pow(value, self, mod).", + "_weakref.ProxyType.__rrshift__" => "Return value>>self.", + "_weakref.ProxyType.__rshift__" => "Return self>>value.", + "_weakref.ProxyType.__rsub__" => "Return value-self.", + "_weakref.ProxyType.__rtruediv__" => "Return value/self.", + "_weakref.ProxyType.__rxor__" => "Return value^self.", + "_weakref.ProxyType.__setattr__" => "Implement setattr(self, name, value).", + "_weakref.ProxyType.__setitem__" => "Set self[key] to value.", + "_weakref.ProxyType.__sizeof__" => "Size of object in memory, in bytes.", + "_weakref.ProxyType.__str__" => "Return str(self).", + "_weakref.ProxyType.__sub__" => "Return self-value.", + "_weakref.ProxyType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_weakref.ProxyType.__truediv__" => "Return self/value.", + "_weakref.ProxyType.__xor__" => "Return self^value.", + "_weakref.ReferenceType.__call__" => "Call self as a function.", + "_weakref.ReferenceType.__class_getitem__" => "See PEP 585", + "_weakref.ReferenceType.__delattr__" => "Implement delattr(self, name).", + "_weakref.ReferenceType.__eq__" => "Return self==value.", + "_weakref.ReferenceType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_weakref.ReferenceType.__ge__" => "Return self>=value.", + "_weakref.ReferenceType.__getattribute__" => "Return getattr(self, name).", + "_weakref.ReferenceType.__getstate__" => "Helper for pickle.", + "_weakref.ReferenceType.__gt__" => "Return self>value.", + "_weakref.ReferenceType.__hash__" => "Return hash(self).", + "_weakref.ReferenceType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_weakref.ReferenceType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_weakref.ReferenceType.__le__" => "Return self<=value.", + "_weakref.ReferenceType.__lt__" => "Return self<value.", + "_weakref.ReferenceType.__ne__" => "Return self!=value.", + "_weakref.ReferenceType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_weakref.ReferenceType.__reduce__" => "Helper for pickle.", + "_weakref.ReferenceType.__reduce_ex__" => "Helper for pickle.", + "_weakref.ReferenceType.__repr__" => "Return repr(self).", + "_weakref.ReferenceType.__setattr__" => "Implement setattr(self, name, value).", + "_weakref.ReferenceType.__sizeof__" => "Size of object in memory, in bytes.", + "_weakref.ReferenceType.__str__" => "Return str(self).", + "_weakref.ReferenceType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_weakref._remove_dead_weakref" => "Atomically remove key from dict if it points to a dead weakref.", + "_weakref.getweakrefcount" => "Return the number of weak references to 'object'.", + "_weakref.getweakrefs" => "Return a list of all weak reference objects pointing to 'object'.", + "_weakref.proxy" => "Create a proxy object that weakly references 'object'.\n\n'callback', if given, is called with a reference to the\nproxy when 'object' is about to be finalized.", + "_weakref.ref.__call__" => "Call self as a function.", + "_weakref.ref.__class_getitem__" => "See PEP 585", + "_weakref.ref.__delattr__" => "Implement delattr(self, name).", + "_weakref.ref.__eq__" => "Return self==value.", + "_weakref.ref.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_weakref.ref.__ge__" => "Return self>=value.", + "_weakref.ref.__getattribute__" => "Return getattr(self, name).", + "_weakref.ref.__getstate__" => "Helper for pickle.", + "_weakref.ref.__gt__" => "Return self>value.", + "_weakref.ref.__hash__" => "Return hash(self).", + "_weakref.ref.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_weakref.ref.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_weakref.ref.__le__" => "Return self<=value.", + "_weakref.ref.__lt__" => "Return self<value.", + "_weakref.ref.__ne__" => "Return self!=value.", + "_weakref.ref.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_weakref.ref.__reduce__" => "Helper for pickle.", + "_weakref.ref.__reduce_ex__" => "Helper for pickle.", + "_weakref.ref.__repr__" => "Return repr(self).", + "_weakref.ref.__setattr__" => "Implement setattr(self, name, value).", + "_weakref.ref.__sizeof__" => "Size of object in memory, in bytes.", + "_weakref.ref.__str__" => "Return str(self).", + "_weakref.ref.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_winapi.BatchedWaitForMultipleObjects" => "Supports a larger number of handles than WaitForMultipleObjects\n\nNote that the handles may be waited on other threads, which could cause\nissues for objects like mutexes that become associated with the thread\nthat was waiting for them. Objects may also be left signalled, even if\nthe wait fails.\n\nIt is recommended to use WaitForMultipleObjects whenever possible, and\nonly switch to BatchedWaitForMultipleObjects for scenarios where you\ncontrol all the handles involved, such as your own thread pool or\nfiles, and all wait objects are left unmodified by a wait (for example,\nmanual reset events, threads, and files/pipes).\n\nOverlapped handles returned from this module use manual reset events.", + "_winapi.CloseHandle" => "Close handle.", + "_winapi.CopyFile2" => "Copies a file from one name to a new name.\n\nThis is implemented using the CopyFile2 API, which preserves all stat\nand metadata information apart from security attributes.\n\nprogress_routine is reserved for future use, but is currently not\nimplemented. Its value is ignored.", + "_winapi.CreatePipe" => "Create an anonymous pipe.\n\n pipe_attrs\n Ignored internally, can be None.\n\nReturns a 2-tuple of handles, to the read and write ends of the pipe.", + "_winapi.CreateProcess" => "Create a new process and its primary thread.\n\n command_line\n Can be str or None\n proc_attrs\n Ignored internally, can be None.\n thread_attrs\n Ignored internally, can be None.\n\nThe return value is a tuple of the process handle, thread handle,\nprocess ID, and thread ID.", + "_winapi.DuplicateHandle" => "Return a duplicate handle object.\n\nThe duplicate handle refers to the same object as the original\nhandle. Therefore, any changes to the object are reflected\nthrough both handles.", + "_winapi.GetACP" => "Get the current Windows ANSI code page identifier.", + "_winapi.GetCurrentProcess" => "Return a handle object for the current process.", + "_winapi.GetExitCodeProcess" => "Return the termination status of the specified process.", + "_winapi.GetLongPathName" => "Return the long version of the provided path.\n\nIf the path is already in its long form, returns the same value.\n\nThe path must already be a 'str'. If the type is not known, use\nos.fsdecode before calling this function.", + "_winapi.GetModuleFileName" => "Return the fully-qualified path for the file that contains module.\n\nThe module must have been loaded by the current process.\n\nThe module parameter should be a handle to the loaded module\nwhose path is being requested. If this parameter is 0,\nGetModuleFileName retrieves the path of the executable file\nof the current process.", + "_winapi.GetShortPathName" => "Return the short version of the provided path.\n\nIf the path is already in its short form, returns the same value.\n\nThe path must already be a 'str'. If the type is not known, use\nos.fsdecode before calling this function.", + "_winapi.GetStdHandle" => "Return a handle to the specified standard device.\n\n std_handle\n One of STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, or STD_ERROR_HANDLE.\n\nThe integer associated with the handle object is returned.", + "_winapi.GetVersion" => "Return the version number of the current operating system.", + "_winapi.Overlapped" => "OVERLAPPED structure wrapper", + "_winapi.Overlapped.__delattr__" => "Implement delattr(self, name).", + "_winapi.Overlapped.__eq__" => "Return self==value.", + "_winapi.Overlapped.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_winapi.Overlapped.__ge__" => "Return self>=value.", + "_winapi.Overlapped.__getattribute__" => "Return getattr(self, name).", + "_winapi.Overlapped.__getstate__" => "Helper for pickle.", + "_winapi.Overlapped.__gt__" => "Return self>value.", + "_winapi.Overlapped.__hash__" => "Return hash(self).", + "_winapi.Overlapped.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_winapi.Overlapped.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_winapi.Overlapped.__le__" => "Return self<=value.", + "_winapi.Overlapped.__lt__" => "Return self<value.", + "_winapi.Overlapped.__ne__" => "Return self!=value.", + "_winapi.Overlapped.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_winapi.Overlapped.__reduce__" => "Helper for pickle.", + "_winapi.Overlapped.__reduce_ex__" => "Helper for pickle.", + "_winapi.Overlapped.__repr__" => "Return repr(self).", + "_winapi.Overlapped.__setattr__" => "Implement setattr(self, name, value).", + "_winapi.Overlapped.__sizeof__" => "Size of object in memory, in bytes.", + "_winapi.Overlapped.__str__" => "Return str(self).", + "_winapi.Overlapped.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_winapi.Overlapped.event" => "overlapped event handle", + "_winapi.TerminateProcess" => "Terminate the specified process and all of its threads.", + "_winapi.WaitForSingleObject" => "Wait for a single object.\n\nWait until the specified object is in the signaled state or\nthe time-out interval elapses. The timeout value is specified\nin milliseconds.", + "_winapi._mimetypes_read_windows_registry" => "Optimized function for reading all known MIME types from the registry.\n\n*on_type_read* is a callable taking *type* and *ext* arguments, as for\nMimeTypes.add_type.", + "_wmi.exec_query" => "Runs a WMI query against the local machine.\n\nThis returns a single string with 'name=value' pairs in a flat array separated\nby null characters.", + "_zoneinfo" => "C implementation of the zoneinfo module", + "_zoneinfo.ZoneInfo.__delattr__" => "Implement delattr(self, name).", + "_zoneinfo.ZoneInfo.__eq__" => "Return self==value.", + "_zoneinfo.ZoneInfo.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zoneinfo.ZoneInfo.__ge__" => "Return self>=value.", + "_zoneinfo.ZoneInfo.__getattribute__" => "Return getattr(self, name).", + "_zoneinfo.ZoneInfo.__getstate__" => "Helper for pickle.", + "_zoneinfo.ZoneInfo.__gt__" => "Return self>value.", + "_zoneinfo.ZoneInfo.__hash__" => "Return hash(self).", + "_zoneinfo.ZoneInfo.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zoneinfo.ZoneInfo.__init_subclass__" => "Function to initialize subclasses.", + "_zoneinfo.ZoneInfo.__le__" => "Return self<=value.", + "_zoneinfo.ZoneInfo.__lt__" => "Return self<value.", + "_zoneinfo.ZoneInfo.__ne__" => "Return self!=value.", + "_zoneinfo.ZoneInfo.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zoneinfo.ZoneInfo.__reduce__" => "Function for serialization with the pickle protocol.", + "_zoneinfo.ZoneInfo.__reduce_ex__" => "Helper for pickle.", + "_zoneinfo.ZoneInfo.__repr__" => "Return repr(self).", + "_zoneinfo.ZoneInfo.__setattr__" => "Implement setattr(self, name, value).", + "_zoneinfo.ZoneInfo.__sizeof__" => "Size of object in memory, in bytes.", + "_zoneinfo.ZoneInfo.__str__" => "Return str(self).", + "_zoneinfo.ZoneInfo.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zoneinfo.ZoneInfo.dst" => "Retrieve a timedelta representing the amount of DST applied in a zone at the given datetime.", + "_zoneinfo.ZoneInfo.fromutc" => "Given a datetime with local time in UTC, retrieve an adjusted datetime in local time.", + "_zoneinfo.ZoneInfo.tzname" => "Retrieve a string containing the abbreviation for the time zone that applies in a zone at a given datetime.", + "_zoneinfo.ZoneInfo.utcoffset" => "Retrieve a timedelta representing the UTC offset in a zone at the given datetime.", + "_zstd" => "Implementation module for Zstandard compression.", + "_zstd.ZstdCompressor" => "Create a compressor object for compressing data incrementally.\n\n level\n The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n options\n A dict object that contains advanced compression parameters.\n zstd_dict\n A ZstdDict object, a pre-trained Zstandard dictionary.\n\nThread-safe at method level. For one-shot compression, use the compress()\nfunction instead.", + "_zstd.ZstdCompressor.__delattr__" => "Implement delattr(self, name).", + "_zstd.ZstdCompressor.__eq__" => "Return self==value.", + "_zstd.ZstdCompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zstd.ZstdCompressor.__ge__" => "Return self>=value.", + "_zstd.ZstdCompressor.__getattribute__" => "Return getattr(self, name).", + "_zstd.ZstdCompressor.__getstate__" => "Helper for pickle.", + "_zstd.ZstdCompressor.__gt__" => "Return self>value.", + "_zstd.ZstdCompressor.__hash__" => "Return hash(self).", + "_zstd.ZstdCompressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zstd.ZstdCompressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_zstd.ZstdCompressor.__le__" => "Return self<=value.", + "_zstd.ZstdCompressor.__lt__" => "Return self<value.", + "_zstd.ZstdCompressor.__ne__" => "Return self!=value.", + "_zstd.ZstdCompressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zstd.ZstdCompressor.__reduce__" => "Helper for pickle.", + "_zstd.ZstdCompressor.__reduce_ex__" => "Helper for pickle.", + "_zstd.ZstdCompressor.__repr__" => "Return repr(self).", + "_zstd.ZstdCompressor.__setattr__" => "Implement setattr(self, name, value).", + "_zstd.ZstdCompressor.__sizeof__" => "Size of object in memory, in bytes.", + "_zstd.ZstdCompressor.__str__" => "Return str(self).", + "_zstd.ZstdCompressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zstd.ZstdCompressor.compress" => "Provide data to the compressor object.\n\n mode\n Can be these 3 values ZstdCompressor.CONTINUE,\n ZstdCompressor.FLUSH_BLOCK, ZstdCompressor.FLUSH_FRAME\n\nReturn a chunk of compressed data if possible, or b'' otherwise. When you have\nfinished providing data to the compressor, call the flush() method to finish\nthe compression process.", + "_zstd.ZstdCompressor.flush" => "Finish the compression process.\n\n mode\n Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n ZstdCompressor.FLUSH_BLOCK\n\nFlush any remaining data left in internal buffers. Since Zstandard data\nconsists of one or more independent frames, the compressor object can still\nbe used after this method is called.", + "_zstd.ZstdCompressor.last_mode" => "The last mode used to this compressor object, its value can be .CONTINUE,\n.FLUSH_BLOCK, .FLUSH_FRAME. Initialized to .FLUSH_FRAME.\n\nIt can be used to get the current state of a compressor, such as, data\nflushed, or a frame ended.", + "_zstd.ZstdCompressor.set_pledged_input_size" => "Set the uncompressed content size to be written into the frame header.\n\n size\n The size of the uncompressed data to be provided to the compressor.\n\nThis method can be used to ensure the header of the frame about to be written\nincludes the size of the data, unless the CompressionParameter.content_size_flag\nis set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n\nIt is important to ensure that the pledged data size matches the actual data\nsize. If they do not match the compressed output data may be corrupted and the\nfinal chunk written may be lost.", + "_zstd.ZstdDecompressor" => "Create a decompressor object for decompressing data incrementally.\n\n zstd_dict\n A ZstdDict object, a pre-trained Zstandard dictionary.\n options\n A dict object that contains advanced decompression parameters.\n\nThread-safe at method level. For one-shot decompression, use the decompress()\nfunction instead.", + "_zstd.ZstdDecompressor.__delattr__" => "Implement delattr(self, name).", + "_zstd.ZstdDecompressor.__eq__" => "Return self==value.", + "_zstd.ZstdDecompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zstd.ZstdDecompressor.__ge__" => "Return self>=value.", + "_zstd.ZstdDecompressor.__getattribute__" => "Return getattr(self, name).", + "_zstd.ZstdDecompressor.__getstate__" => "Helper for pickle.", + "_zstd.ZstdDecompressor.__gt__" => "Return self>value.", + "_zstd.ZstdDecompressor.__hash__" => "Return hash(self).", + "_zstd.ZstdDecompressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zstd.ZstdDecompressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_zstd.ZstdDecompressor.__le__" => "Return self<=value.", + "_zstd.ZstdDecompressor.__lt__" => "Return self<value.", + "_zstd.ZstdDecompressor.__ne__" => "Return self!=value.", + "_zstd.ZstdDecompressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zstd.ZstdDecompressor.__reduce__" => "Helper for pickle.", + "_zstd.ZstdDecompressor.__reduce_ex__" => "Helper for pickle.", + "_zstd.ZstdDecompressor.__repr__" => "Return repr(self).", + "_zstd.ZstdDecompressor.__setattr__" => "Implement setattr(self, name, value).", + "_zstd.ZstdDecompressor.__sizeof__" => "Size of object in memory, in bytes.", + "_zstd.ZstdDecompressor.__str__" => "Return str(self).", + "_zstd.ZstdDecompressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zstd.ZstdDecompressor.decompress" => "Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise.\n\n data\n A bytes-like object, Zstandard data to be decompressed.\n max_length\n Maximum size of returned data. When it is negative, the size of\n output buffer is unlimited. When it is nonnegative, returns at\n most max_length bytes of decompressed data.\n\nIf *max_length* is nonnegative, returns at most *max_length* bytes of\ndecompressed data. If this limit is reached and further output can be\nproduced, *self.needs_input* will be set to ``False``. In this case, the next\ncall to *decompress()* may provide *data* as b'' to obtain more of the output.\n\nIf all of the input data was decompressed and returned (either because this\nwas less than *max_length* bytes, or because *max_length* was negative),\n*self.needs_input* will be set to True.\n\nAttempting to decompress data after the end of a frame is reached raises an\nEOFError. Any data found after the end of the frame is ignored and saved in\nthe self.unused_data attribute.", + "_zstd.ZstdDecompressor.eof" => "True means the end of the first frame has been reached. If decompress data\nafter that, an EOFError exception will be raised.", + "_zstd.ZstdDecompressor.needs_input" => "If the max_length output limit in .decompress() method has been reached,\nand the decompressor has (or may has) unconsumed input data, it will be set\nto False. In this case, passing b'' to the .decompress() method may output\nfurther data.", + "_zstd.ZstdDecompressor.unused_data" => "A bytes object of un-consumed input data.\n\nWhen ZstdDecompressor object stops after a frame is\ndecompressed, unused input data after the frame. Otherwise this will be b''.", + "_zstd.ZstdDict" => "Represents a Zstandard dictionary.\n\n dict_content\n The content of a Zstandard dictionary as a bytes-like object.\n is_raw\n If true, perform no checks on *dict_content*, useful for some\n advanced cases. Otherwise, check that the content represents\n a Zstandard dictionary created by the zstd library or CLI.\n\nThe dictionary can be used for compression or decompression, and can be shared\nby multiple ZstdCompressor or ZstdDecompressor objects.", + "_zstd.ZstdDict.__delattr__" => "Implement delattr(self, name).", + "_zstd.ZstdDict.__eq__" => "Return self==value.", + "_zstd.ZstdDict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zstd.ZstdDict.__ge__" => "Return self>=value.", + "_zstd.ZstdDict.__getattribute__" => "Return getattr(self, name).", + "_zstd.ZstdDict.__getstate__" => "Helper for pickle.", + "_zstd.ZstdDict.__gt__" => "Return self>value.", + "_zstd.ZstdDict.__hash__" => "Return hash(self).", + "_zstd.ZstdDict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zstd.ZstdDict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_zstd.ZstdDict.__le__" => "Return self<=value.", + "_zstd.ZstdDict.__len__" => "Return len(self).", + "_zstd.ZstdDict.__lt__" => "Return self<value.", + "_zstd.ZstdDict.__ne__" => "Return self!=value.", + "_zstd.ZstdDict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zstd.ZstdDict.__reduce__" => "Helper for pickle.", + "_zstd.ZstdDict.__reduce_ex__" => "Helper for pickle.", + "_zstd.ZstdDict.__repr__" => "Return repr(self).", + "_zstd.ZstdDict.__setattr__" => "Implement setattr(self, name, value).", + "_zstd.ZstdDict.__sizeof__" => "Size of object in memory, in bytes.", + "_zstd.ZstdDict.__str__" => "Return str(self).", + "_zstd.ZstdDict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zstd.ZstdDict.as_digested_dict" => "Load as a digested dictionary to compressor.\n\nPass this attribute as zstd_dict argument:\ncompress(dat, zstd_dict=zd.as_digested_dict)\n\n1. Some advanced compression parameters of compressor may be overridden\n by parameters of digested dictionary.\n2. ZstdDict has a digested dictionaries cache for each compression level.\n It's faster when loading again a digested dictionary with the same\n compression level.\n3. No need to use this for decompression.", + "_zstd.ZstdDict.as_prefix" => "Load as a prefix to compressor/decompressor.\n\nPass this attribute as zstd_dict argument:\ncompress(dat, zstd_dict=zd.as_prefix)\n\n1. Prefix is compatible with long distance matching, while dictionary is not.\n2. It only works for the first frame, then the compressor/decompressor will\n return to no prefix state.\n3. When decompressing, must use the same prefix as when compressing.", + "_zstd.ZstdDict.as_undigested_dict" => "Load as an undigested dictionary to compressor.\n\nPass this attribute as zstd_dict argument:\ncompress(dat, zstd_dict=zd.as_undigested_dict)\n\n1. The advanced compression parameters of compressor will not be overridden.\n2. Loading an undigested dictionary is costly. If load an undigested dictionary\n multiple times, consider reusing a compressor object.\n3. No need to use this for decompression.", + "_zstd.ZstdDict.dict_content" => "The content of a Zstandard dictionary, as a bytes object.", + "_zstd.ZstdDict.dict_id" => "The Zstandard dictionary, an int between 0 and 2**32.\n\nA non-zero value represents an ordinary Zstandard dictionary,\nconforming to the standardised format.\n\nA value of zero indicates a 'raw content' dictionary,\nwithout any restrictions on format or content.", + "_zstd.ZstdError" => "An error occurred in the zstd library.", + "_zstd.ZstdError.__delattr__" => "Implement delattr(self, name).", + "_zstd.ZstdError.__eq__" => "Return self==value.", + "_zstd.ZstdError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "_zstd.ZstdError.__ge__" => "Return self>=value.", + "_zstd.ZstdError.__getattribute__" => "Return getattr(self, name).", + "_zstd.ZstdError.__getstate__" => "Helper for pickle.", + "_zstd.ZstdError.__gt__" => "Return self>value.", + "_zstd.ZstdError.__hash__" => "Return hash(self).", + "_zstd.ZstdError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "_zstd.ZstdError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "_zstd.ZstdError.__le__" => "Return self<=value.", + "_zstd.ZstdError.__lt__" => "Return self<value.", + "_zstd.ZstdError.__ne__" => "Return self!=value.", + "_zstd.ZstdError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "_zstd.ZstdError.__reduce_ex__" => "Helper for pickle.", + "_zstd.ZstdError.__repr__" => "Return repr(self).", + "_zstd.ZstdError.__setattr__" => "Implement setattr(self, name, value).", + "_zstd.ZstdError.__sizeof__" => "Size of object in memory, in bytes.", + "_zstd.ZstdError.__str__" => "Return str(self).", + "_zstd.ZstdError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "_zstd.ZstdError.__weakref__" => "list of weak references to the object", + "_zstd.ZstdError.add_note" => "Add a note to the exception", + "_zstd.ZstdError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "_zstd.finalize_dict" => "Finalize a Zstandard dictionary.\n\n custom_dict_bytes\n Custom dictionary content.\n samples_bytes\n Concatenation of samples.\n samples_sizes\n Tuple of samples' sizes.\n dict_size\n The size of the dictionary.\n compression_level\n Optimize for a specific Zstandard compression level, 0 means default.", + "_zstd.get_frame_info" => "Get Zstandard frame infomation from a frame header.\n\n frame_buffer\n A bytes-like object, containing the header of a Zstandard frame.", + "_zstd.get_frame_size" => "Get the size of a Zstandard frame, including the header and optional checksum.\n\n frame_buffer\n A bytes-like object, it should start from the beginning of a frame,\n and contains at least one complete frame.", + "_zstd.get_param_bounds" => "Get CompressionParameter/DecompressionParameter bounds.\n\n parameter\n The parameter to get bounds.\n is_compress\n True for CompressionParameter, False for DecompressionParameter.", + "_zstd.set_parameter_types" => "Set CompressionParameter and DecompressionParameter types for validity check.\n\n c_parameter_type\n CompressionParameter IntEnum type object\n d_parameter_type\n DecompressionParameter IntEnum type object", + "_zstd.train_dict" => "Train a Zstandard dictionary on sample data.\n\n samples_bytes\n Concatenation of samples.\n samples_sizes\n Tuple of samples' sizes.\n dict_size\n The size of the dictionary.", + "array" => "This module defines an object type which can efficiently represent\nan array of basic values: characters, integers, floating-point\nnumbers. Arrays are sequence types and behave very much like lists,\nexcept that the type of objects stored in them is constrained.", + "array.ArrayType" => "array(typecode [, initializer]) -> array\n\nReturn a new array whose items are restricted by typecode, and\ninitialized from the optional initializer value, which must be a list,\nstring or iterable over elements of the appropriate type.\n\nArrays represent basic values and behave very much like lists, except\nthe type of objects stored in them is constrained. The type is specified\nat object creation time by using a type code, which is a single character.\nThe following type codes are defined:\n\n Type code C Type Minimum size in bytes\n 'b' signed integer 1\n 'B' unsigned integer 1\n 'u' Unicode character 2 (see note)\n 'h' signed integer 2\n 'H' unsigned integer 2\n 'i' signed integer 2\n 'I' unsigned integer 2\n 'l' signed integer 4\n 'L' unsigned integer 4\n 'q' signed integer 8 (see note)\n 'Q' unsigned integer 8 (see note)\n 'f' floating-point 4\n 'd' floating-point 8\n\nNOTE: The 'u' typecode corresponds to Python's unicode character. On\nnarrow builds this is 2-bytes on wide builds this is 4-bytes.\n\nNOTE: The 'q' and 'Q' type codes are only available if the platform\nC compiler used to build Python supports 'long long', or, on Windows,\n'__int64'.\n\nMethods:\n\nappend() -- append a new item to the end of the array\nbuffer_info() -- return information giving the current memory info\nbyteswap() -- byteswap all the items of the array\ncount() -- return number of occurrences of an object\nextend() -- extend array by appending multiple elements from an iterable\nfromfile() -- read items from a file object\nfromlist() -- append items from the list\nfrombytes() -- append items from the string\nindex() -- return index of first occurrence of an object\ninsert() -- insert a new item into the array at a provided position\npop() -- remove and return item (default last)\nremove() -- remove first occurrence of an object\nreverse() -- reverse the order of the items in the array\ntofile() -- write all items to a file object\ntolist() -- return the array converted to an ordinary list\ntobytes() -- return the array converted to a string\n\nAttributes:\n\ntypecode -- the typecode character used to create the array\nitemsize -- the length in bytes of one array item", + "array.ArrayType.__add__" => "Return self+value.", + "array.ArrayType.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "array.ArrayType.__class_getitem__" => "See PEP 585", + "array.ArrayType.__contains__" => "Return bool(key in self).", + "array.ArrayType.__copy__" => "Return a copy of the array.", + "array.ArrayType.__deepcopy__" => "Return a copy of the array.", + "array.ArrayType.__delattr__" => "Implement delattr(self, name).", + "array.ArrayType.__delitem__" => "Delete self[key].", + "array.ArrayType.__eq__" => "Return self==value.", + "array.ArrayType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "array.ArrayType.__ge__" => "Return self>=value.", + "array.ArrayType.__getattribute__" => "Return getattr(self, name).", + "array.ArrayType.__getitem__" => "Return self[key].", + "array.ArrayType.__getstate__" => "Helper for pickle.", + "array.ArrayType.__gt__" => "Return self>value.", + "array.ArrayType.__iadd__" => "Implement self+=value.", + "array.ArrayType.__imul__" => "Implement self*=value.", + "array.ArrayType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "array.ArrayType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "array.ArrayType.__iter__" => "Implement iter(self).", + "array.ArrayType.__le__" => "Return self<=value.", + "array.ArrayType.__len__" => "Return len(self).", + "array.ArrayType.__lt__" => "Return self<value.", + "array.ArrayType.__mul__" => "Return self*value.", + "array.ArrayType.__ne__" => "Return self!=value.", + "array.ArrayType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "array.ArrayType.__reduce__" => "Helper for pickle.", + "array.ArrayType.__reduce_ex__" => "Return state information for pickling.", + "array.ArrayType.__release_buffer__" => "Release the buffer object that exposes the underlying memory of the object.", + "array.ArrayType.__repr__" => "Return repr(self).", + "array.ArrayType.__rmul__" => "Return value*self.", + "array.ArrayType.__setattr__" => "Implement setattr(self, name, value).", + "array.ArrayType.__setitem__" => "Set self[key] to value.", + "array.ArrayType.__sizeof__" => "Size of the array in memory, in bytes.", + "array.ArrayType.__str__" => "Return str(self).", + "array.ArrayType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "array.ArrayType.append" => "Append new value v to the end of the array.", + "array.ArrayType.buffer_info" => "Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents.\n\nThe length should be multiplied by the itemsize attribute to calculate\nthe buffer length in bytes.", + "array.ArrayType.byteswap" => "Byteswap all items of the array.\n\nIf the items in the array are not 1, 2, 4, or 8 bytes in size, RuntimeError is\nraised.", + "array.ArrayType.clear" => "Remove all items from the array.", + "array.ArrayType.count" => "Return number of occurrences of v in the array.", + "array.ArrayType.extend" => "Append items to the end of the array.", + "array.ArrayType.frombytes" => "Appends items from the string, interpreting it as an array of machine values, as if it had been read from a file using the fromfile() method.", + "array.ArrayType.fromfile" => "Read n objects from the file object f and append them to the end of the array.", + "array.ArrayType.fromlist" => "Append items to array from list.", + "array.ArrayType.fromunicode" => "Extends this array with data from the unicode string ustr.\n\nThe array must be a unicode type array; otherwise a ValueError is raised.\nUse array.frombytes(ustr.encode(...)) to append Unicode data to an array of\nsome other type.", + "array.ArrayType.index" => "Return index of first occurrence of v in the array.\n\nRaise ValueError if the value is not present.", + "array.ArrayType.insert" => "Insert a new item v into the array before position i.", + "array.ArrayType.itemsize" => "the size, in bytes, of one array item", + "array.ArrayType.pop" => "Return the i-th element and delete it from the array.\n\ni defaults to -1.", + "array.ArrayType.remove" => "Remove the first occurrence of v in the array.", + "array.ArrayType.reverse" => "Reverse the order of the items in the array.", + "array.ArrayType.tobytes" => "Convert the array to an array of machine values and return the bytes representation.", + "array.ArrayType.tofile" => "Write all items (as machine values) to the file object f.", + "array.ArrayType.tolist" => "Convert array to an ordinary list with the same items.", + "array.ArrayType.tounicode" => "Extends this array with data from the unicode string ustr.\n\nConvert the array to a unicode string. The array must be a unicode type array;\notherwise a ValueError is raised. Use array.tobytes().decode() to obtain a\nunicode string from an array of some other type.", + "array.ArrayType.typecode" => "the typecode character used to create the array", + "array._array_reconstructor" => "Internal. Used for pickling support.", + "array.array" => "array(typecode [, initializer]) -> array\n\nReturn a new array whose items are restricted by typecode, and\ninitialized from the optional initializer value, which must be a list,\nstring or iterable over elements of the appropriate type.\n\nArrays represent basic values and behave very much like lists, except\nthe type of objects stored in them is constrained. The type is specified\nat object creation time by using a type code, which is a single character.\nThe following type codes are defined:\n\n Type code C Type Minimum size in bytes\n 'b' signed integer 1\n 'B' unsigned integer 1\n 'u' Unicode character 2 (see note)\n 'h' signed integer 2\n 'H' unsigned integer 2\n 'i' signed integer 2\n 'I' unsigned integer 2\n 'l' signed integer 4\n 'L' unsigned integer 4\n 'q' signed integer 8 (see note)\n 'Q' unsigned integer 8 (see note)\n 'f' floating-point 4\n 'd' floating-point 8\n\nNOTE: The 'u' typecode corresponds to Python's unicode character. On\nnarrow builds this is 2-bytes on wide builds this is 4-bytes.\n\nNOTE: The 'q' and 'Q' type codes are only available if the platform\nC compiler used to build Python supports 'long long', or, on Windows,\n'__int64'.\n\nMethods:\n\nappend() -- append a new item to the end of the array\nbuffer_info() -- return information giving the current memory info\nbyteswap() -- byteswap all the items of the array\ncount() -- return number of occurrences of an object\nextend() -- extend array by appending multiple elements from an iterable\nfromfile() -- read items from a file object\nfromlist() -- append items from the list\nfrombytes() -- append items from the string\nindex() -- return index of first occurrence of an object\ninsert() -- insert a new item into the array at a provided position\npop() -- remove and return item (default last)\nremove() -- remove first occurrence of an object\nreverse() -- reverse the order of the items in the array\ntofile() -- write all items to a file object\ntolist() -- return the array converted to an ordinary list\ntobytes() -- return the array converted to a string\n\nAttributes:\n\ntypecode -- the typecode character used to create the array\nitemsize -- the length in bytes of one array item", + "array.array.__add__" => "Return self+value.", + "array.array.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "array.array.__class_getitem__" => "See PEP 585", + "array.array.__contains__" => "Return bool(key in self).", + "array.array.__copy__" => "Return a copy of the array.", + "array.array.__deepcopy__" => "Return a copy of the array.", + "array.array.__delattr__" => "Implement delattr(self, name).", + "array.array.__delitem__" => "Delete self[key].", + "array.array.__eq__" => "Return self==value.", + "array.array.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "array.array.__ge__" => "Return self>=value.", + "array.array.__getattribute__" => "Return getattr(self, name).", + "array.array.__getitem__" => "Return self[key].", + "array.array.__getstate__" => "Helper for pickle.", + "array.array.__gt__" => "Return self>value.", + "array.array.__iadd__" => "Implement self+=value.", + "array.array.__imul__" => "Implement self*=value.", + "array.array.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "array.array.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "array.array.__iter__" => "Implement iter(self).", + "array.array.__le__" => "Return self<=value.", + "array.array.__len__" => "Return len(self).", + "array.array.__lt__" => "Return self<value.", + "array.array.__mul__" => "Return self*value.", + "array.array.__ne__" => "Return self!=value.", + "array.array.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "array.array.__reduce__" => "Helper for pickle.", + "array.array.__reduce_ex__" => "Return state information for pickling.", + "array.array.__release_buffer__" => "Release the buffer object that exposes the underlying memory of the object.", + "array.array.__repr__" => "Return repr(self).", + "array.array.__rmul__" => "Return value*self.", + "array.array.__setattr__" => "Implement setattr(self, name, value).", + "array.array.__setitem__" => "Set self[key] to value.", + "array.array.__sizeof__" => "Size of the array in memory, in bytes.", + "array.array.__str__" => "Return str(self).", + "array.array.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "array.array.append" => "Append new value v to the end of the array.", + "array.array.buffer_info" => "Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents.\n\nThe length should be multiplied by the itemsize attribute to calculate\nthe buffer length in bytes.", + "array.array.byteswap" => "Byteswap all items of the array.\n\nIf the items in the array are not 1, 2, 4, or 8 bytes in size, RuntimeError is\nraised.", + "array.array.clear" => "Remove all items from the array.", + "array.array.count" => "Return number of occurrences of v in the array.", + "array.array.extend" => "Append items to the end of the array.", + "array.array.frombytes" => "Appends items from the string, interpreting it as an array of machine values, as if it had been read from a file using the fromfile() method.", + "array.array.fromfile" => "Read n objects from the file object f and append them to the end of the array.", + "array.array.fromlist" => "Append items to array from list.", + "array.array.fromunicode" => "Extends this array with data from the unicode string ustr.\n\nThe array must be a unicode type array; otherwise a ValueError is raised.\nUse array.frombytes(ustr.encode(...)) to append Unicode data to an array of\nsome other type.", + "array.array.index" => "Return index of first occurrence of v in the array.\n\nRaise ValueError if the value is not present.", + "array.array.insert" => "Insert a new item v into the array before position i.", + "array.array.itemsize" => "the size, in bytes, of one array item", + "array.array.pop" => "Return the i-th element and delete it from the array.\n\ni defaults to -1.", + "array.array.remove" => "Remove the first occurrence of v in the array.", + "array.array.reverse" => "Reverse the order of the items in the array.", + "array.array.tobytes" => "Convert the array to an array of machine values and return the bytes representation.", + "array.array.tofile" => "Write all items (as machine values) to the file object f.", + "array.array.tolist" => "Convert array to an ordinary list with the same items.", + "array.array.tounicode" => "Extends this array with data from the unicode string ustr.\n\nConvert the array to a unicode string. The array must be a unicode type array;\notherwise a ValueError is raised. Use array.tobytes().decode() to obtain a\nunicode string from an array of some other type.", + "array.array.typecode" => "the typecode character used to create the array", + "atexit" => "allow programmer to define multiple exit functions to be executed\nupon normal program termination.\n\nTwo public functions, register and unregister, are defined.", + "atexit._clear" => "Clear the list of previously registered exit functions.", + "atexit._ncallbacks" => "Return the number of registered exit functions.", + "atexit._run_exitfuncs" => "Run all registered exit functions.\n\nIf a callback raises an exception, it is logged with sys.unraisablehook.", + "atexit.register" => "Register a function to be executed upon normal program termination\n\n func - function to be called at exit\n args - optional arguments to pass to func\n kwargs - optional keyword arguments to pass to func\n\n func is returned to facilitate usage as a decorator.", + "atexit.unregister" => "Unregister an exit function which was previously registered using\natexit.register\n\n func - function to be unregistered", + "binascii" => "Conversion between binary data and ASCII", + "binascii.Error.__delattr__" => "Implement delattr(self, name).", + "binascii.Error.__eq__" => "Return self==value.", + "binascii.Error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "binascii.Error.__ge__" => "Return self>=value.", + "binascii.Error.__getattribute__" => "Return getattr(self, name).", + "binascii.Error.__getstate__" => "Helper for pickle.", + "binascii.Error.__gt__" => "Return self>value.", + "binascii.Error.__hash__" => "Return hash(self).", + "binascii.Error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "binascii.Error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "binascii.Error.__le__" => "Return self<=value.", + "binascii.Error.__lt__" => "Return self<value.", + "binascii.Error.__ne__" => "Return self!=value.", + "binascii.Error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "binascii.Error.__reduce_ex__" => "Helper for pickle.", + "binascii.Error.__repr__" => "Return repr(self).", + "binascii.Error.__setattr__" => "Implement setattr(self, name, value).", + "binascii.Error.__sizeof__" => "Size of object in memory, in bytes.", + "binascii.Error.__str__" => "Return str(self).", + "binascii.Error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "binascii.Error.__weakref__" => "list of weak references to the object", + "binascii.Error.add_note" => "Add a note to the exception", + "binascii.Error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "binascii.Incomplete.__delattr__" => "Implement delattr(self, name).", + "binascii.Incomplete.__eq__" => "Return self==value.", + "binascii.Incomplete.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "binascii.Incomplete.__ge__" => "Return self>=value.", + "binascii.Incomplete.__getattribute__" => "Return getattr(self, name).", + "binascii.Incomplete.__getstate__" => "Helper for pickle.", + "binascii.Incomplete.__gt__" => "Return self>value.", + "binascii.Incomplete.__hash__" => "Return hash(self).", + "binascii.Incomplete.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "binascii.Incomplete.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "binascii.Incomplete.__le__" => "Return self<=value.", + "binascii.Incomplete.__lt__" => "Return self<value.", + "binascii.Incomplete.__ne__" => "Return self!=value.", + "binascii.Incomplete.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "binascii.Incomplete.__reduce_ex__" => "Helper for pickle.", + "binascii.Incomplete.__repr__" => "Return repr(self).", + "binascii.Incomplete.__setattr__" => "Implement setattr(self, name, value).", + "binascii.Incomplete.__sizeof__" => "Size of object in memory, in bytes.", + "binascii.Incomplete.__str__" => "Return str(self).", + "binascii.Incomplete.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "binascii.Incomplete.__weakref__" => "list of weak references to the object", + "binascii.Incomplete.add_note" => "Add a note to the exception", + "binascii.Incomplete.with_traceback" => "Set self.__traceback__ to tb and return self.", + "binascii.a2b_base64" => "Decode a line of base64 data.\n\n strict_mode\n When set to True, bytes that are not part of the base64 standard are not allowed.\n The same applies to excess data after padding (= / ==).", + "binascii.a2b_hex" => "Binary data of hexadecimal representation.\n\nhexstr must contain an even number of hex digits (upper or lower case).\nThis function is also available as \"unhexlify()\".", + "binascii.a2b_qp" => "Decode a string of qp-encoded data.", + "binascii.a2b_uu" => "Decode a line of uuencoded data.", + "binascii.b2a_base64" => "Base64-code line of data.", + "binascii.b2a_hex" => "Hexadecimal representation of binary data.\n\n sep\n An optional single character or byte to separate hex bytes.\n bytes_per_sep\n How many bytes between separators. Positive values count from the\n right, negative values count from the left.\n\nThe return value is a bytes object. This function is also\navailable as \"hexlify()\".\n\nExample:\n>>> binascii.b2a_hex(b'\\xb9\\x01\\xef')\nb'b901ef'\n>>> binascii.hexlify(b'\\xb9\\x01\\xef', ':')\nb'b9:01:ef'\n>>> binascii.b2a_hex(b'\\xb9\\x01\\xef', b'_', 2)\nb'b9_01ef'", + "binascii.b2a_qp" => "Encode a string using quoted-printable encoding.\n\nOn encoding, when istext is set, newlines are not encoded, and white\nspace at end of lines is. When istext is not set, \\r and \\n (CR/LF)\nare both encoded. When quotetabs is set, space and tabs are encoded.", + "binascii.b2a_uu" => "Uuencode line of data.", + "binascii.crc32" => "Compute CRC-32 incrementally.", + "binascii.crc_hqx" => "Compute CRC-CCITT incrementally.", + "binascii.hexlify" => "Hexadecimal representation of binary data.\n\n sep\n An optional single character or byte to separate hex bytes.\n bytes_per_sep\n How many bytes between separators. Positive values count from the\n right, negative values count from the left.\n\nThe return value is a bytes object. This function is also\navailable as \"b2a_hex()\".", + "binascii.unhexlify" => "Binary data of hexadecimal representation.\n\nhexstr must contain an even number of hex digits (upper or lower case).", + "builtins" => "Built-in functions, types, exceptions, and other objects.\n\nThis module provides direct access to all 'built-in'\nidentifiers of Python; for example, builtins.len is\nthe full name for the built-in function len().\n\nThis module is not normally accessed explicitly by most\napplications, but can be useful in modules that provide\nobjects with the same name as a built-in value, but in\nwhich the built-in of that name is also needed.", + "builtins.ArithmeticError" => "Base class for arithmetic errors.", + "builtins.ArithmeticError.__delattr__" => "Implement delattr(self, name).", + "builtins.ArithmeticError.__eq__" => "Return self==value.", + "builtins.ArithmeticError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ArithmeticError.__ge__" => "Return self>=value.", + "builtins.ArithmeticError.__getattribute__" => "Return getattr(self, name).", + "builtins.ArithmeticError.__getstate__" => "Helper for pickle.", + "builtins.ArithmeticError.__gt__" => "Return self>value.", + "builtins.ArithmeticError.__hash__" => "Return hash(self).", + "builtins.ArithmeticError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ArithmeticError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ArithmeticError.__le__" => "Return self<=value.", + "builtins.ArithmeticError.__lt__" => "Return self<value.", + "builtins.ArithmeticError.__ne__" => "Return self!=value.", + "builtins.ArithmeticError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ArithmeticError.__reduce_ex__" => "Helper for pickle.", + "builtins.ArithmeticError.__repr__" => "Return repr(self).", + "builtins.ArithmeticError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ArithmeticError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ArithmeticError.__str__" => "Return str(self).", + "builtins.ArithmeticError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ArithmeticError.add_note" => "Add a note to the exception", + "builtins.ArithmeticError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.AssertionError" => "Assertion failed.", + "builtins.AssertionError.__delattr__" => "Implement delattr(self, name).", + "builtins.AssertionError.__eq__" => "Return self==value.", + "builtins.AssertionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.AssertionError.__ge__" => "Return self>=value.", + "builtins.AssertionError.__getattribute__" => "Return getattr(self, name).", + "builtins.AssertionError.__getstate__" => "Helper for pickle.", + "builtins.AssertionError.__gt__" => "Return self>value.", + "builtins.AssertionError.__hash__" => "Return hash(self).", + "builtins.AssertionError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.AssertionError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.AssertionError.__le__" => "Return self<=value.", + "builtins.AssertionError.__lt__" => "Return self<value.", + "builtins.AssertionError.__ne__" => "Return self!=value.", + "builtins.AssertionError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.AssertionError.__reduce_ex__" => "Helper for pickle.", + "builtins.AssertionError.__repr__" => "Return repr(self).", + "builtins.AssertionError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.AssertionError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.AssertionError.__str__" => "Return str(self).", + "builtins.AssertionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.AssertionError.add_note" => "Add a note to the exception", + "builtins.AssertionError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.AttributeError" => "Attribute not found.", + "builtins.AttributeError.__delattr__" => "Implement delattr(self, name).", + "builtins.AttributeError.__eq__" => "Return self==value.", + "builtins.AttributeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.AttributeError.__ge__" => "Return self>=value.", + "builtins.AttributeError.__getattribute__" => "Return getattr(self, name).", + "builtins.AttributeError.__gt__" => "Return self>value.", + "builtins.AttributeError.__hash__" => "Return hash(self).", + "builtins.AttributeError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.AttributeError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.AttributeError.__le__" => "Return self<=value.", + "builtins.AttributeError.__lt__" => "Return self<value.", + "builtins.AttributeError.__ne__" => "Return self!=value.", + "builtins.AttributeError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.AttributeError.__reduce_ex__" => "Helper for pickle.", + "builtins.AttributeError.__repr__" => "Return repr(self).", + "builtins.AttributeError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.AttributeError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.AttributeError.__str__" => "Return str(self).", + "builtins.AttributeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.AttributeError.add_note" => "Add a note to the exception", + "builtins.AttributeError.name" => "attribute name", + "builtins.AttributeError.obj" => "object", + "builtins.AttributeError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.BaseException" => "Common base class for all exceptions", + "builtins.BaseException.__delattr__" => "Implement delattr(self, name).", + "builtins.BaseException.__eq__" => "Return self==value.", + "builtins.BaseException.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.BaseException.__ge__" => "Return self>=value.", + "builtins.BaseException.__getattribute__" => "Return getattr(self, name).", + "builtins.BaseException.__getstate__" => "Helper for pickle.", + "builtins.BaseException.__gt__" => "Return self>value.", + "builtins.BaseException.__hash__" => "Return hash(self).", + "builtins.BaseException.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.BaseException.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.BaseException.__le__" => "Return self<=value.", + "builtins.BaseException.__lt__" => "Return self<value.", + "builtins.BaseException.__ne__" => "Return self!=value.", + "builtins.BaseException.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.BaseException.__reduce_ex__" => "Helper for pickle.", + "builtins.BaseException.__repr__" => "Return repr(self).", + "builtins.BaseException.__setattr__" => "Implement setattr(self, name, value).", + "builtins.BaseException.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.BaseException.__str__" => "Return str(self).", + "builtins.BaseException.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.BaseException.add_note" => "Add a note to the exception", + "builtins.BaseException.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.BaseExceptionGroup" => "A combination of multiple unrelated exceptions.", + "builtins.BaseExceptionGroup.__class_getitem__" => "See PEP 585", + "builtins.BaseExceptionGroup.__delattr__" => "Implement delattr(self, name).", + "builtins.BaseExceptionGroup.__eq__" => "Return self==value.", + "builtins.BaseExceptionGroup.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.BaseExceptionGroup.__ge__" => "Return self>=value.", + "builtins.BaseExceptionGroup.__getattribute__" => "Return getattr(self, name).", + "builtins.BaseExceptionGroup.__getstate__" => "Helper for pickle.", + "builtins.BaseExceptionGroup.__gt__" => "Return self>value.", + "builtins.BaseExceptionGroup.__hash__" => "Return hash(self).", + "builtins.BaseExceptionGroup.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.BaseExceptionGroup.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.BaseExceptionGroup.__le__" => "Return self<=value.", + "builtins.BaseExceptionGroup.__lt__" => "Return self<value.", + "builtins.BaseExceptionGroup.__ne__" => "Return self!=value.", + "builtins.BaseExceptionGroup.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.BaseExceptionGroup.__reduce_ex__" => "Helper for pickle.", + "builtins.BaseExceptionGroup.__repr__" => "Return repr(self).", + "builtins.BaseExceptionGroup.__setattr__" => "Implement setattr(self, name, value).", + "builtins.BaseExceptionGroup.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.BaseExceptionGroup.__str__" => "Return str(self).", + "builtins.BaseExceptionGroup.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.BaseExceptionGroup.add_note" => "Add a note to the exception", + "builtins.BaseExceptionGroup.exceptions" => "nested exceptions", + "builtins.BaseExceptionGroup.message" => "exception message", + "builtins.BaseExceptionGroup.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.BlockingIOError" => "I/O operation would block.", + "builtins.BlockingIOError.__delattr__" => "Implement delattr(self, name).", + "builtins.BlockingIOError.__eq__" => "Return self==value.", + "builtins.BlockingIOError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.BlockingIOError.__ge__" => "Return self>=value.", + "builtins.BlockingIOError.__getattribute__" => "Return getattr(self, name).", + "builtins.BlockingIOError.__getstate__" => "Helper for pickle.", + "builtins.BlockingIOError.__gt__" => "Return self>value.", + "builtins.BlockingIOError.__hash__" => "Return hash(self).", + "builtins.BlockingIOError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.BlockingIOError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.BlockingIOError.__le__" => "Return self<=value.", + "builtins.BlockingIOError.__lt__" => "Return self<value.", + "builtins.BlockingIOError.__ne__" => "Return self!=value.", + "builtins.BlockingIOError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.BlockingIOError.__reduce_ex__" => "Helper for pickle.", + "builtins.BlockingIOError.__repr__" => "Return repr(self).", + "builtins.BlockingIOError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.BlockingIOError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.BlockingIOError.__str__" => "Return str(self).", + "builtins.BlockingIOError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.BlockingIOError.add_note" => "Add a note to the exception", + "builtins.BlockingIOError.errno" => "POSIX exception code", + "builtins.BlockingIOError.filename" => "exception filename", + "builtins.BlockingIOError.filename2" => "second exception filename", + "builtins.BlockingIOError.strerror" => "exception strerror", + "builtins.BlockingIOError.winerror" => "Win32 exception code", + "builtins.BlockingIOError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.BrokenPipeError" => "Broken pipe.", + "builtins.BrokenPipeError.__delattr__" => "Implement delattr(self, name).", + "builtins.BrokenPipeError.__eq__" => "Return self==value.", + "builtins.BrokenPipeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.BrokenPipeError.__ge__" => "Return self>=value.", + "builtins.BrokenPipeError.__getattribute__" => "Return getattr(self, name).", + "builtins.BrokenPipeError.__getstate__" => "Helper for pickle.", + "builtins.BrokenPipeError.__gt__" => "Return self>value.", + "builtins.BrokenPipeError.__hash__" => "Return hash(self).", + "builtins.BrokenPipeError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.BrokenPipeError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.BrokenPipeError.__le__" => "Return self<=value.", + "builtins.BrokenPipeError.__lt__" => "Return self<value.", + "builtins.BrokenPipeError.__ne__" => "Return self!=value.", + "builtins.BrokenPipeError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.BrokenPipeError.__reduce_ex__" => "Helper for pickle.", + "builtins.BrokenPipeError.__repr__" => "Return repr(self).", + "builtins.BrokenPipeError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.BrokenPipeError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.BrokenPipeError.__str__" => "Return str(self).", + "builtins.BrokenPipeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.BrokenPipeError.add_note" => "Add a note to the exception", + "builtins.BrokenPipeError.errno" => "POSIX exception code", + "builtins.BrokenPipeError.filename" => "exception filename", + "builtins.BrokenPipeError.filename2" => "second exception filename", + "builtins.BrokenPipeError.strerror" => "exception strerror", + "builtins.BrokenPipeError.winerror" => "Win32 exception code", + "builtins.BrokenPipeError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.BufferError" => "Buffer error.", + "builtins.BufferError.__delattr__" => "Implement delattr(self, name).", + "builtins.BufferError.__eq__" => "Return self==value.", + "builtins.BufferError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.BufferError.__ge__" => "Return self>=value.", + "builtins.BufferError.__getattribute__" => "Return getattr(self, name).", + "builtins.BufferError.__getstate__" => "Helper for pickle.", + "builtins.BufferError.__gt__" => "Return self>value.", + "builtins.BufferError.__hash__" => "Return hash(self).", + "builtins.BufferError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.BufferError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.BufferError.__le__" => "Return self<=value.", + "builtins.BufferError.__lt__" => "Return self<value.", + "builtins.BufferError.__ne__" => "Return self!=value.", + "builtins.BufferError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.BufferError.__reduce_ex__" => "Helper for pickle.", + "builtins.BufferError.__repr__" => "Return repr(self).", + "builtins.BufferError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.BufferError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.BufferError.__str__" => "Return str(self).", + "builtins.BufferError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.BufferError.add_note" => "Add a note to the exception", + "builtins.BufferError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.BytesWarning" => "Base class for warnings about bytes and buffer related problems, mostly\nrelated to conversion from str or comparing to str.", + "builtins.BytesWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.BytesWarning.__eq__" => "Return self==value.", + "builtins.BytesWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.BytesWarning.__ge__" => "Return self>=value.", + "builtins.BytesWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.BytesWarning.__getstate__" => "Helper for pickle.", + "builtins.BytesWarning.__gt__" => "Return self>value.", + "builtins.BytesWarning.__hash__" => "Return hash(self).", + "builtins.BytesWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.BytesWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.BytesWarning.__le__" => "Return self<=value.", + "builtins.BytesWarning.__lt__" => "Return self<value.", + "builtins.BytesWarning.__ne__" => "Return self!=value.", + "builtins.BytesWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.BytesWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.BytesWarning.__repr__" => "Return repr(self).", + "builtins.BytesWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.BytesWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.BytesWarning.__str__" => "Return str(self).", + "builtins.BytesWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.BytesWarning.add_note" => "Add a note to the exception", + "builtins.BytesWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ChildProcessError" => "Child process error.", + "builtins.ChildProcessError.__delattr__" => "Implement delattr(self, name).", + "builtins.ChildProcessError.__eq__" => "Return self==value.", + "builtins.ChildProcessError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ChildProcessError.__ge__" => "Return self>=value.", + "builtins.ChildProcessError.__getattribute__" => "Return getattr(self, name).", + "builtins.ChildProcessError.__getstate__" => "Helper for pickle.", + "builtins.ChildProcessError.__gt__" => "Return self>value.", + "builtins.ChildProcessError.__hash__" => "Return hash(self).", + "builtins.ChildProcessError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ChildProcessError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ChildProcessError.__le__" => "Return self<=value.", + "builtins.ChildProcessError.__lt__" => "Return self<value.", + "builtins.ChildProcessError.__ne__" => "Return self!=value.", + "builtins.ChildProcessError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ChildProcessError.__reduce_ex__" => "Helper for pickle.", + "builtins.ChildProcessError.__repr__" => "Return repr(self).", + "builtins.ChildProcessError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ChildProcessError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ChildProcessError.__str__" => "Return str(self).", + "builtins.ChildProcessError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ChildProcessError.add_note" => "Add a note to the exception", + "builtins.ChildProcessError.errno" => "POSIX exception code", + "builtins.ChildProcessError.filename" => "exception filename", + "builtins.ChildProcessError.filename2" => "second exception filename", + "builtins.ChildProcessError.strerror" => "exception strerror", + "builtins.ChildProcessError.winerror" => "Win32 exception code", + "builtins.ChildProcessError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ConnectionAbortedError" => "Connection aborted.", + "builtins.ConnectionAbortedError.__delattr__" => "Implement delattr(self, name).", + "builtins.ConnectionAbortedError.__eq__" => "Return self==value.", + "builtins.ConnectionAbortedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ConnectionAbortedError.__ge__" => "Return self>=value.", + "builtins.ConnectionAbortedError.__getattribute__" => "Return getattr(self, name).", + "builtins.ConnectionAbortedError.__getstate__" => "Helper for pickle.", + "builtins.ConnectionAbortedError.__gt__" => "Return self>value.", + "builtins.ConnectionAbortedError.__hash__" => "Return hash(self).", + "builtins.ConnectionAbortedError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ConnectionAbortedError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ConnectionAbortedError.__le__" => "Return self<=value.", + "builtins.ConnectionAbortedError.__lt__" => "Return self<value.", + "builtins.ConnectionAbortedError.__ne__" => "Return self!=value.", + "builtins.ConnectionAbortedError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ConnectionAbortedError.__reduce_ex__" => "Helper for pickle.", + "builtins.ConnectionAbortedError.__repr__" => "Return repr(self).", + "builtins.ConnectionAbortedError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ConnectionAbortedError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ConnectionAbortedError.__str__" => "Return str(self).", + "builtins.ConnectionAbortedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ConnectionAbortedError.add_note" => "Add a note to the exception", + "builtins.ConnectionAbortedError.errno" => "POSIX exception code", + "builtins.ConnectionAbortedError.filename" => "exception filename", + "builtins.ConnectionAbortedError.filename2" => "second exception filename", + "builtins.ConnectionAbortedError.strerror" => "exception strerror", + "builtins.ConnectionAbortedError.winerror" => "Win32 exception code", + "builtins.ConnectionAbortedError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ConnectionError" => "Connection error.", + "builtins.ConnectionError.__delattr__" => "Implement delattr(self, name).", + "builtins.ConnectionError.__eq__" => "Return self==value.", + "builtins.ConnectionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ConnectionError.__ge__" => "Return self>=value.", + "builtins.ConnectionError.__getattribute__" => "Return getattr(self, name).", + "builtins.ConnectionError.__getstate__" => "Helper for pickle.", + "builtins.ConnectionError.__gt__" => "Return self>value.", + "builtins.ConnectionError.__hash__" => "Return hash(self).", + "builtins.ConnectionError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ConnectionError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ConnectionError.__le__" => "Return self<=value.", + "builtins.ConnectionError.__lt__" => "Return self<value.", + "builtins.ConnectionError.__ne__" => "Return self!=value.", + "builtins.ConnectionError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ConnectionError.__reduce_ex__" => "Helper for pickle.", + "builtins.ConnectionError.__repr__" => "Return repr(self).", + "builtins.ConnectionError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ConnectionError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ConnectionError.__str__" => "Return str(self).", + "builtins.ConnectionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ConnectionError.add_note" => "Add a note to the exception", + "builtins.ConnectionError.errno" => "POSIX exception code", + "builtins.ConnectionError.filename" => "exception filename", + "builtins.ConnectionError.filename2" => "second exception filename", + "builtins.ConnectionError.strerror" => "exception strerror", + "builtins.ConnectionError.winerror" => "Win32 exception code", + "builtins.ConnectionError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ConnectionRefusedError" => "Connection refused.", + "builtins.ConnectionRefusedError.__delattr__" => "Implement delattr(self, name).", + "builtins.ConnectionRefusedError.__eq__" => "Return self==value.", + "builtins.ConnectionRefusedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ConnectionRefusedError.__ge__" => "Return self>=value.", + "builtins.ConnectionRefusedError.__getattribute__" => "Return getattr(self, name).", + "builtins.ConnectionRefusedError.__getstate__" => "Helper for pickle.", + "builtins.ConnectionRefusedError.__gt__" => "Return self>value.", + "builtins.ConnectionRefusedError.__hash__" => "Return hash(self).", + "builtins.ConnectionRefusedError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ConnectionRefusedError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ConnectionRefusedError.__le__" => "Return self<=value.", + "builtins.ConnectionRefusedError.__lt__" => "Return self<value.", + "builtins.ConnectionRefusedError.__ne__" => "Return self!=value.", + "builtins.ConnectionRefusedError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ConnectionRefusedError.__reduce_ex__" => "Helper for pickle.", + "builtins.ConnectionRefusedError.__repr__" => "Return repr(self).", + "builtins.ConnectionRefusedError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ConnectionRefusedError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ConnectionRefusedError.__str__" => "Return str(self).", + "builtins.ConnectionRefusedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ConnectionRefusedError.add_note" => "Add a note to the exception", + "builtins.ConnectionRefusedError.errno" => "POSIX exception code", + "builtins.ConnectionRefusedError.filename" => "exception filename", + "builtins.ConnectionRefusedError.filename2" => "second exception filename", + "builtins.ConnectionRefusedError.strerror" => "exception strerror", + "builtins.ConnectionRefusedError.winerror" => "Win32 exception code", + "builtins.ConnectionRefusedError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ConnectionResetError" => "Connection reset.", + "builtins.ConnectionResetError.__delattr__" => "Implement delattr(self, name).", + "builtins.ConnectionResetError.__eq__" => "Return self==value.", + "builtins.ConnectionResetError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ConnectionResetError.__ge__" => "Return self>=value.", + "builtins.ConnectionResetError.__getattribute__" => "Return getattr(self, name).", + "builtins.ConnectionResetError.__getstate__" => "Helper for pickle.", + "builtins.ConnectionResetError.__gt__" => "Return self>value.", + "builtins.ConnectionResetError.__hash__" => "Return hash(self).", + "builtins.ConnectionResetError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ConnectionResetError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ConnectionResetError.__le__" => "Return self<=value.", + "builtins.ConnectionResetError.__lt__" => "Return self<value.", + "builtins.ConnectionResetError.__ne__" => "Return self!=value.", + "builtins.ConnectionResetError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ConnectionResetError.__reduce_ex__" => "Helper for pickle.", + "builtins.ConnectionResetError.__repr__" => "Return repr(self).", + "builtins.ConnectionResetError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ConnectionResetError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ConnectionResetError.__str__" => "Return str(self).", + "builtins.ConnectionResetError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ConnectionResetError.add_note" => "Add a note to the exception", + "builtins.ConnectionResetError.errno" => "POSIX exception code", + "builtins.ConnectionResetError.filename" => "exception filename", + "builtins.ConnectionResetError.filename2" => "second exception filename", + "builtins.ConnectionResetError.strerror" => "exception strerror", + "builtins.ConnectionResetError.winerror" => "Win32 exception code", + "builtins.ConnectionResetError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.DeprecationWarning" => "Base class for warnings about deprecated features.", + "builtins.DeprecationWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.DeprecationWarning.__eq__" => "Return self==value.", + "builtins.DeprecationWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.DeprecationWarning.__ge__" => "Return self>=value.", + "builtins.DeprecationWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.DeprecationWarning.__getstate__" => "Helper for pickle.", + "builtins.DeprecationWarning.__gt__" => "Return self>value.", + "builtins.DeprecationWarning.__hash__" => "Return hash(self).", + "builtins.DeprecationWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.DeprecationWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.DeprecationWarning.__le__" => "Return self<=value.", + "builtins.DeprecationWarning.__lt__" => "Return self<value.", + "builtins.DeprecationWarning.__ne__" => "Return self!=value.", + "builtins.DeprecationWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.DeprecationWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.DeprecationWarning.__repr__" => "Return repr(self).", + "builtins.DeprecationWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.DeprecationWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.DeprecationWarning.__str__" => "Return str(self).", + "builtins.DeprecationWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.DeprecationWarning.add_note" => "Add a note to the exception", + "builtins.DeprecationWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.DynamicClassAttribute" => "Route attribute access on a class to __getattr__.\n\nThis is a descriptor, used to define attributes that act differently when\naccessed through an instance and through a class. Instance access remains\nnormal, but access to an attribute through a class will be routed to the\nclass's __getattr__ method; this is done by raising AttributeError.\n\nThis allows one to have properties active on an instance, and have virtual\nattributes on the class with the same name. (Enum used this between Python\nversions 3.4 - 3.9 .)\n\nSubclass from this to use a different method of accessing virtual attributes\nand still be treated properly by the inspect module. (Enum uses this since\nPython 3.10 .)", + "builtins.DynamicClassAttribute.__delattr__" => "Implement delattr(self, name).", + "builtins.DynamicClassAttribute.__eq__" => "Return self==value.", + "builtins.DynamicClassAttribute.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.DynamicClassAttribute.__ge__" => "Return self>=value.", + "builtins.DynamicClassAttribute.__getattribute__" => "Return getattr(self, name).", + "builtins.DynamicClassAttribute.__getstate__" => "Helper for pickle.", + "builtins.DynamicClassAttribute.__gt__" => "Return self>value.", + "builtins.DynamicClassAttribute.__hash__" => "Return hash(self).", + "builtins.DynamicClassAttribute.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.DynamicClassAttribute.__le__" => "Return self<=value.", + "builtins.DynamicClassAttribute.__lt__" => "Return self<value.", + "builtins.DynamicClassAttribute.__ne__" => "Return self!=value.", + "builtins.DynamicClassAttribute.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.DynamicClassAttribute.__reduce__" => "Helper for pickle.", + "builtins.DynamicClassAttribute.__reduce_ex__" => "Helper for pickle.", + "builtins.DynamicClassAttribute.__repr__" => "Return repr(self).", + "builtins.DynamicClassAttribute.__setattr__" => "Implement setattr(self, name, value).", + "builtins.DynamicClassAttribute.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.DynamicClassAttribute.__str__" => "Return str(self).", + "builtins.DynamicClassAttribute.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.DynamicClassAttribute.__weakref__" => "list of weak references to the object", + "builtins.EOFError" => "Read beyond end of file.", + "builtins.EOFError.__delattr__" => "Implement delattr(self, name).", + "builtins.EOFError.__eq__" => "Return self==value.", + "builtins.EOFError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.EOFError.__ge__" => "Return self>=value.", + "builtins.EOFError.__getattribute__" => "Return getattr(self, name).", + "builtins.EOFError.__getstate__" => "Helper for pickle.", + "builtins.EOFError.__gt__" => "Return self>value.", + "builtins.EOFError.__hash__" => "Return hash(self).", + "builtins.EOFError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.EOFError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.EOFError.__le__" => "Return self<=value.", + "builtins.EOFError.__lt__" => "Return self<value.", + "builtins.EOFError.__ne__" => "Return self!=value.", + "builtins.EOFError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.EOFError.__reduce_ex__" => "Helper for pickle.", + "builtins.EOFError.__repr__" => "Return repr(self).", + "builtins.EOFError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.EOFError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.EOFError.__str__" => "Return str(self).", + "builtins.EOFError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.EOFError.add_note" => "Add a note to the exception", + "builtins.EOFError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.EncodingWarning" => "Base class for warnings about encodings.", + "builtins.EncodingWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.EncodingWarning.__eq__" => "Return self==value.", + "builtins.EncodingWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.EncodingWarning.__ge__" => "Return self>=value.", + "builtins.EncodingWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.EncodingWarning.__getstate__" => "Helper for pickle.", + "builtins.EncodingWarning.__gt__" => "Return self>value.", + "builtins.EncodingWarning.__hash__" => "Return hash(self).", + "builtins.EncodingWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.EncodingWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.EncodingWarning.__le__" => "Return self<=value.", + "builtins.EncodingWarning.__lt__" => "Return self<value.", + "builtins.EncodingWarning.__ne__" => "Return self!=value.", + "builtins.EncodingWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.EncodingWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.EncodingWarning.__repr__" => "Return repr(self).", + "builtins.EncodingWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.EncodingWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.EncodingWarning.__str__" => "Return str(self).", + "builtins.EncodingWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.EncodingWarning.add_note" => "Add a note to the exception", + "builtins.EncodingWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.EnvironmentError" => "Base class for I/O related errors.", + "builtins.EnvironmentError.__delattr__" => "Implement delattr(self, name).", + "builtins.EnvironmentError.__eq__" => "Return self==value.", + "builtins.EnvironmentError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.EnvironmentError.__ge__" => "Return self>=value.", + "builtins.EnvironmentError.__getattribute__" => "Return getattr(self, name).", + "builtins.EnvironmentError.__getstate__" => "Helper for pickle.", + "builtins.EnvironmentError.__gt__" => "Return self>value.", + "builtins.EnvironmentError.__hash__" => "Return hash(self).", + "builtins.EnvironmentError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.EnvironmentError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.EnvironmentError.__le__" => "Return self<=value.", + "builtins.EnvironmentError.__lt__" => "Return self<value.", + "builtins.EnvironmentError.__ne__" => "Return self!=value.", + "builtins.EnvironmentError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.EnvironmentError.__reduce_ex__" => "Helper for pickle.", + "builtins.EnvironmentError.__repr__" => "Return repr(self).", + "builtins.EnvironmentError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.EnvironmentError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.EnvironmentError.__str__" => "Return str(self).", + "builtins.EnvironmentError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.EnvironmentError.add_note" => "Add a note to the exception", + "builtins.EnvironmentError.errno" => "POSIX exception code", + "builtins.EnvironmentError.filename" => "exception filename", + "builtins.EnvironmentError.filename2" => "second exception filename", + "builtins.EnvironmentError.strerror" => "exception strerror", + "builtins.EnvironmentError.winerror" => "Win32 exception code", + "builtins.EnvironmentError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.Exception" => "Common base class for all non-exit exceptions.", + "builtins.Exception.__delattr__" => "Implement delattr(self, name).", + "builtins.Exception.__eq__" => "Return self==value.", + "builtins.Exception.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.Exception.__ge__" => "Return self>=value.", + "builtins.Exception.__getattribute__" => "Return getattr(self, name).", + "builtins.Exception.__getstate__" => "Helper for pickle.", + "builtins.Exception.__gt__" => "Return self>value.", + "builtins.Exception.__hash__" => "Return hash(self).", + "builtins.Exception.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.Exception.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.Exception.__le__" => "Return self<=value.", + "builtins.Exception.__lt__" => "Return self<value.", + "builtins.Exception.__ne__" => "Return self!=value.", + "builtins.Exception.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.Exception.__reduce_ex__" => "Helper for pickle.", + "builtins.Exception.__repr__" => "Return repr(self).", + "builtins.Exception.__setattr__" => "Implement setattr(self, name, value).", + "builtins.Exception.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.Exception.__str__" => "Return str(self).", + "builtins.Exception.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.Exception.add_note" => "Add a note to the exception", + "builtins.Exception.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ExceptionGroup.__class_getitem__" => "See PEP 585", + "builtins.ExceptionGroup.__delattr__" => "Implement delattr(self, name).", + "builtins.ExceptionGroup.__eq__" => "Return self==value.", + "builtins.ExceptionGroup.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ExceptionGroup.__ge__" => "Return self>=value.", + "builtins.ExceptionGroup.__getattribute__" => "Return getattr(self, name).", + "builtins.ExceptionGroup.__getstate__" => "Helper for pickle.", + "builtins.ExceptionGroup.__gt__" => "Return self>value.", + "builtins.ExceptionGroup.__hash__" => "Return hash(self).", + "builtins.ExceptionGroup.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ExceptionGroup.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ExceptionGroup.__le__" => "Return self<=value.", + "builtins.ExceptionGroup.__lt__" => "Return self<value.", + "builtins.ExceptionGroup.__ne__" => "Return self!=value.", + "builtins.ExceptionGroup.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ExceptionGroup.__reduce_ex__" => "Helper for pickle.", + "builtins.ExceptionGroup.__repr__" => "Return repr(self).", + "builtins.ExceptionGroup.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ExceptionGroup.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ExceptionGroup.__str__" => "Return str(self).", + "builtins.ExceptionGroup.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ExceptionGroup.__weakref__" => "list of weak references to the object", + "builtins.ExceptionGroup.add_note" => "Add a note to the exception", + "builtins.ExceptionGroup.exceptions" => "nested exceptions", + "builtins.ExceptionGroup.message" => "exception message", + "builtins.ExceptionGroup.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.FileExistsError" => "File already exists.", + "builtins.FileExistsError.__delattr__" => "Implement delattr(self, name).", + "builtins.FileExistsError.__eq__" => "Return self==value.", + "builtins.FileExistsError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.FileExistsError.__ge__" => "Return self>=value.", + "builtins.FileExistsError.__getattribute__" => "Return getattr(self, name).", + "builtins.FileExistsError.__getstate__" => "Helper for pickle.", + "builtins.FileExistsError.__gt__" => "Return self>value.", + "builtins.FileExistsError.__hash__" => "Return hash(self).", + "builtins.FileExistsError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.FileExistsError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.FileExistsError.__le__" => "Return self<=value.", + "builtins.FileExistsError.__lt__" => "Return self<value.", + "builtins.FileExistsError.__ne__" => "Return self!=value.", + "builtins.FileExistsError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.FileExistsError.__reduce_ex__" => "Helper for pickle.", + "builtins.FileExistsError.__repr__" => "Return repr(self).", + "builtins.FileExistsError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.FileExistsError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.FileExistsError.__str__" => "Return str(self).", + "builtins.FileExistsError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.FileExistsError.add_note" => "Add a note to the exception", + "builtins.FileExistsError.errno" => "POSIX exception code", + "builtins.FileExistsError.filename" => "exception filename", + "builtins.FileExistsError.filename2" => "second exception filename", + "builtins.FileExistsError.strerror" => "exception strerror", + "builtins.FileExistsError.winerror" => "Win32 exception code", + "builtins.FileExistsError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.FileNotFoundError" => "File not found.", + "builtins.FileNotFoundError.__delattr__" => "Implement delattr(self, name).", + "builtins.FileNotFoundError.__eq__" => "Return self==value.", + "builtins.FileNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.FileNotFoundError.__ge__" => "Return self>=value.", + "builtins.FileNotFoundError.__getattribute__" => "Return getattr(self, name).", + "builtins.FileNotFoundError.__getstate__" => "Helper for pickle.", + "builtins.FileNotFoundError.__gt__" => "Return self>value.", + "builtins.FileNotFoundError.__hash__" => "Return hash(self).", + "builtins.FileNotFoundError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.FileNotFoundError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.FileNotFoundError.__le__" => "Return self<=value.", + "builtins.FileNotFoundError.__lt__" => "Return self<value.", + "builtins.FileNotFoundError.__ne__" => "Return self!=value.", + "builtins.FileNotFoundError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.FileNotFoundError.__reduce_ex__" => "Helper for pickle.", + "builtins.FileNotFoundError.__repr__" => "Return repr(self).", + "builtins.FileNotFoundError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.FileNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.FileNotFoundError.__str__" => "Return str(self).", + "builtins.FileNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.FileNotFoundError.add_note" => "Add a note to the exception", + "builtins.FileNotFoundError.errno" => "POSIX exception code", + "builtins.FileNotFoundError.filename" => "exception filename", + "builtins.FileNotFoundError.filename2" => "second exception filename", + "builtins.FileNotFoundError.strerror" => "exception strerror", + "builtins.FileNotFoundError.winerror" => "Win32 exception code", + "builtins.FileNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.FloatingPointError" => "Floating-point operation failed.", + "builtins.FloatingPointError.__delattr__" => "Implement delattr(self, name).", + "builtins.FloatingPointError.__eq__" => "Return self==value.", + "builtins.FloatingPointError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.FloatingPointError.__ge__" => "Return self>=value.", + "builtins.FloatingPointError.__getattribute__" => "Return getattr(self, name).", + "builtins.FloatingPointError.__getstate__" => "Helper for pickle.", + "builtins.FloatingPointError.__gt__" => "Return self>value.", + "builtins.FloatingPointError.__hash__" => "Return hash(self).", + "builtins.FloatingPointError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.FloatingPointError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.FloatingPointError.__le__" => "Return self<=value.", + "builtins.FloatingPointError.__lt__" => "Return self<value.", + "builtins.FloatingPointError.__ne__" => "Return self!=value.", + "builtins.FloatingPointError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.FloatingPointError.__reduce_ex__" => "Helper for pickle.", + "builtins.FloatingPointError.__repr__" => "Return repr(self).", + "builtins.FloatingPointError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.FloatingPointError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.FloatingPointError.__str__" => "Return str(self).", + "builtins.FloatingPointError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.FloatingPointError.add_note" => "Add a note to the exception", + "builtins.FloatingPointError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.FutureWarning" => "Base class for warnings about constructs that will change semantically\nin the future.", + "builtins.FutureWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.FutureWarning.__eq__" => "Return self==value.", + "builtins.FutureWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.FutureWarning.__ge__" => "Return self>=value.", + "builtins.FutureWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.FutureWarning.__getstate__" => "Helper for pickle.", + "builtins.FutureWarning.__gt__" => "Return self>value.", + "builtins.FutureWarning.__hash__" => "Return hash(self).", + "builtins.FutureWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.FutureWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.FutureWarning.__le__" => "Return self<=value.", + "builtins.FutureWarning.__lt__" => "Return self<value.", + "builtins.FutureWarning.__ne__" => "Return self!=value.", + "builtins.FutureWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.FutureWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.FutureWarning.__repr__" => "Return repr(self).", + "builtins.FutureWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.FutureWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.FutureWarning.__str__" => "Return str(self).", + "builtins.FutureWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.FutureWarning.add_note" => "Add a note to the exception", + "builtins.FutureWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.GeneratorExit" => "Request that a generator exit.", + "builtins.GeneratorExit.__delattr__" => "Implement delattr(self, name).", + "builtins.GeneratorExit.__eq__" => "Return self==value.", + "builtins.GeneratorExit.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.GeneratorExit.__ge__" => "Return self>=value.", + "builtins.GeneratorExit.__getattribute__" => "Return getattr(self, name).", + "builtins.GeneratorExit.__getstate__" => "Helper for pickle.", + "builtins.GeneratorExit.__gt__" => "Return self>value.", + "builtins.GeneratorExit.__hash__" => "Return hash(self).", + "builtins.GeneratorExit.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.GeneratorExit.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.GeneratorExit.__le__" => "Return self<=value.", + "builtins.GeneratorExit.__lt__" => "Return self<value.", + "builtins.GeneratorExit.__ne__" => "Return self!=value.", + "builtins.GeneratorExit.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.GeneratorExit.__reduce_ex__" => "Helper for pickle.", + "builtins.GeneratorExit.__repr__" => "Return repr(self).", + "builtins.GeneratorExit.__setattr__" => "Implement setattr(self, name, value).", + "builtins.GeneratorExit.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.GeneratorExit.__str__" => "Return str(self).", + "builtins.GeneratorExit.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.GeneratorExit.add_note" => "Add a note to the exception", + "builtins.GeneratorExit.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.GenericAlias" => "Represent a PEP 585 generic type\n\nE.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,).", + "builtins.GenericAlias.__call__" => "Call self as a function.", + "builtins.GenericAlias.__delattr__" => "Implement delattr(self, name).", + "builtins.GenericAlias.__eq__" => "Return self==value.", + "builtins.GenericAlias.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.GenericAlias.__ge__" => "Return self>=value.", + "builtins.GenericAlias.__getattribute__" => "Return getattr(self, name).", + "builtins.GenericAlias.__getitem__" => "Return self[key].", + "builtins.GenericAlias.__getstate__" => "Helper for pickle.", + "builtins.GenericAlias.__gt__" => "Return self>value.", + "builtins.GenericAlias.__hash__" => "Return hash(self).", + "builtins.GenericAlias.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.GenericAlias.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.GenericAlias.__iter__" => "Implement iter(self).", + "builtins.GenericAlias.__le__" => "Return self<=value.", + "builtins.GenericAlias.__lt__" => "Return self<value.", + "builtins.GenericAlias.__ne__" => "Return self!=value.", + "builtins.GenericAlias.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.GenericAlias.__or__" => "Return self|value.", + "builtins.GenericAlias.__parameters__" => "Type variables in the GenericAlias.", + "builtins.GenericAlias.__reduce_ex__" => "Helper for pickle.", + "builtins.GenericAlias.__repr__" => "Return repr(self).", + "builtins.GenericAlias.__ror__" => "Return value|self.", + "builtins.GenericAlias.__setattr__" => "Implement setattr(self, name, value).", + "builtins.GenericAlias.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.GenericAlias.__str__" => "Return str(self).", + "builtins.GenericAlias.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.IOError" => "Base class for I/O related errors.", + "builtins.IOError.__delattr__" => "Implement delattr(self, name).", + "builtins.IOError.__eq__" => "Return self==value.", + "builtins.IOError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.IOError.__ge__" => "Return self>=value.", + "builtins.IOError.__getattribute__" => "Return getattr(self, name).", + "builtins.IOError.__getstate__" => "Helper for pickle.", + "builtins.IOError.__gt__" => "Return self>value.", + "builtins.IOError.__hash__" => "Return hash(self).", + "builtins.IOError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.IOError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.IOError.__le__" => "Return self<=value.", + "builtins.IOError.__lt__" => "Return self<value.", + "builtins.IOError.__ne__" => "Return self!=value.", + "builtins.IOError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.IOError.__reduce_ex__" => "Helper for pickle.", + "builtins.IOError.__repr__" => "Return repr(self).", + "builtins.IOError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.IOError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.IOError.__str__" => "Return str(self).", + "builtins.IOError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.IOError.add_note" => "Add a note to the exception", + "builtins.IOError.errno" => "POSIX exception code", + "builtins.IOError.filename" => "exception filename", + "builtins.IOError.filename2" => "second exception filename", + "builtins.IOError.strerror" => "exception strerror", + "builtins.IOError.winerror" => "Win32 exception code", + "builtins.IOError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ImportError" => "Import can't find module, or can't find name in module.", + "builtins.ImportError.__delattr__" => "Implement delattr(self, name).", + "builtins.ImportError.__eq__" => "Return self==value.", + "builtins.ImportError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ImportError.__ge__" => "Return self>=value.", + "builtins.ImportError.__getattribute__" => "Return getattr(self, name).", + "builtins.ImportError.__getstate__" => "Helper for pickle.", + "builtins.ImportError.__gt__" => "Return self>value.", + "builtins.ImportError.__hash__" => "Return hash(self).", + "builtins.ImportError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ImportError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ImportError.__le__" => "Return self<=value.", + "builtins.ImportError.__lt__" => "Return self<value.", + "builtins.ImportError.__ne__" => "Return self!=value.", + "builtins.ImportError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ImportError.__reduce_ex__" => "Helper for pickle.", + "builtins.ImportError.__repr__" => "Return repr(self).", + "builtins.ImportError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ImportError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ImportError.__str__" => "Return str(self).", + "builtins.ImportError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ImportError.add_note" => "Add a note to the exception", + "builtins.ImportError.msg" => "exception message", + "builtins.ImportError.name" => "module name", + "builtins.ImportError.name_from" => "name imported from module", + "builtins.ImportError.path" => "module path", + "builtins.ImportError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ImportWarning" => "Base class for warnings about probable mistakes in module imports", + "builtins.ImportWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.ImportWarning.__eq__" => "Return self==value.", + "builtins.ImportWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ImportWarning.__ge__" => "Return self>=value.", + "builtins.ImportWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.ImportWarning.__getstate__" => "Helper for pickle.", + "builtins.ImportWarning.__gt__" => "Return self>value.", + "builtins.ImportWarning.__hash__" => "Return hash(self).", + "builtins.ImportWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ImportWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ImportWarning.__le__" => "Return self<=value.", + "builtins.ImportWarning.__lt__" => "Return self<value.", + "builtins.ImportWarning.__ne__" => "Return self!=value.", + "builtins.ImportWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ImportWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.ImportWarning.__repr__" => "Return repr(self).", + "builtins.ImportWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ImportWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ImportWarning.__str__" => "Return str(self).", + "builtins.ImportWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ImportWarning.add_note" => "Add a note to the exception", + "builtins.ImportWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.IndentationError" => "Improper indentation.", + "builtins.IndentationError.__delattr__" => "Implement delattr(self, name).", + "builtins.IndentationError.__eq__" => "Return self==value.", + "builtins.IndentationError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.IndentationError.__ge__" => "Return self>=value.", + "builtins.IndentationError.__getattribute__" => "Return getattr(self, name).", + "builtins.IndentationError.__getstate__" => "Helper for pickle.", + "builtins.IndentationError.__gt__" => "Return self>value.", + "builtins.IndentationError.__hash__" => "Return hash(self).", + "builtins.IndentationError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.IndentationError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.IndentationError.__le__" => "Return self<=value.", + "builtins.IndentationError.__lt__" => "Return self<value.", + "builtins.IndentationError.__ne__" => "Return self!=value.", + "builtins.IndentationError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.IndentationError.__reduce_ex__" => "Helper for pickle.", + "builtins.IndentationError.__repr__" => "Return repr(self).", + "builtins.IndentationError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.IndentationError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.IndentationError.__str__" => "Return str(self).", + "builtins.IndentationError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.IndentationError._metadata" => "exception private metadata", + "builtins.IndentationError.add_note" => "Add a note to the exception", + "builtins.IndentationError.end_lineno" => "exception end lineno", + "builtins.IndentationError.end_offset" => "exception end offset", + "builtins.IndentationError.filename" => "exception filename", + "builtins.IndentationError.lineno" => "exception lineno", + "builtins.IndentationError.msg" => "exception msg", + "builtins.IndentationError.offset" => "exception offset", + "builtins.IndentationError.print_file_and_line" => "exception print_file_and_line", + "builtins.IndentationError.text" => "exception text", + "builtins.IndentationError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.IndexError" => "Sequence index out of range.", + "builtins.IndexError.__delattr__" => "Implement delattr(self, name).", + "builtins.IndexError.__eq__" => "Return self==value.", + "builtins.IndexError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.IndexError.__ge__" => "Return self>=value.", + "builtins.IndexError.__getattribute__" => "Return getattr(self, name).", + "builtins.IndexError.__getstate__" => "Helper for pickle.", + "builtins.IndexError.__gt__" => "Return self>value.", + "builtins.IndexError.__hash__" => "Return hash(self).", + "builtins.IndexError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.IndexError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.IndexError.__le__" => "Return self<=value.", + "builtins.IndexError.__lt__" => "Return self<value.", + "builtins.IndexError.__ne__" => "Return self!=value.", + "builtins.IndexError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.IndexError.__reduce_ex__" => "Helper for pickle.", + "builtins.IndexError.__repr__" => "Return repr(self).", + "builtins.IndexError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.IndexError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.IndexError.__str__" => "Return str(self).", + "builtins.IndexError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.IndexError.add_note" => "Add a note to the exception", + "builtins.IndexError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.InterruptedError" => "Interrupted by signal.", + "builtins.InterruptedError.__delattr__" => "Implement delattr(self, name).", + "builtins.InterruptedError.__eq__" => "Return self==value.", + "builtins.InterruptedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.InterruptedError.__ge__" => "Return self>=value.", + "builtins.InterruptedError.__getattribute__" => "Return getattr(self, name).", + "builtins.InterruptedError.__getstate__" => "Helper for pickle.", + "builtins.InterruptedError.__gt__" => "Return self>value.", + "builtins.InterruptedError.__hash__" => "Return hash(self).", + "builtins.InterruptedError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.InterruptedError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.InterruptedError.__le__" => "Return self<=value.", + "builtins.InterruptedError.__lt__" => "Return self<value.", + "builtins.InterruptedError.__ne__" => "Return self!=value.", + "builtins.InterruptedError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.InterruptedError.__reduce_ex__" => "Helper for pickle.", + "builtins.InterruptedError.__repr__" => "Return repr(self).", + "builtins.InterruptedError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.InterruptedError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.InterruptedError.__str__" => "Return str(self).", + "builtins.InterruptedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.InterruptedError.add_note" => "Add a note to the exception", + "builtins.InterruptedError.errno" => "POSIX exception code", + "builtins.InterruptedError.filename" => "exception filename", + "builtins.InterruptedError.filename2" => "second exception filename", + "builtins.InterruptedError.strerror" => "exception strerror", + "builtins.InterruptedError.winerror" => "Win32 exception code", + "builtins.InterruptedError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.IsADirectoryError" => "Operation doesn't work on directories.", + "builtins.IsADirectoryError.__delattr__" => "Implement delattr(self, name).", + "builtins.IsADirectoryError.__eq__" => "Return self==value.", + "builtins.IsADirectoryError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.IsADirectoryError.__ge__" => "Return self>=value.", + "builtins.IsADirectoryError.__getattribute__" => "Return getattr(self, name).", + "builtins.IsADirectoryError.__getstate__" => "Helper for pickle.", + "builtins.IsADirectoryError.__gt__" => "Return self>value.", + "builtins.IsADirectoryError.__hash__" => "Return hash(self).", + "builtins.IsADirectoryError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.IsADirectoryError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.IsADirectoryError.__le__" => "Return self<=value.", + "builtins.IsADirectoryError.__lt__" => "Return self<value.", + "builtins.IsADirectoryError.__ne__" => "Return self!=value.", + "builtins.IsADirectoryError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.IsADirectoryError.__reduce_ex__" => "Helper for pickle.", + "builtins.IsADirectoryError.__repr__" => "Return repr(self).", + "builtins.IsADirectoryError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.IsADirectoryError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.IsADirectoryError.__str__" => "Return str(self).", + "builtins.IsADirectoryError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.IsADirectoryError.add_note" => "Add a note to the exception", + "builtins.IsADirectoryError.errno" => "POSIX exception code", + "builtins.IsADirectoryError.filename" => "exception filename", + "builtins.IsADirectoryError.filename2" => "second exception filename", + "builtins.IsADirectoryError.strerror" => "exception strerror", + "builtins.IsADirectoryError.winerror" => "Win32 exception code", + "builtins.IsADirectoryError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.KeyError" => "Mapping key not found.", + "builtins.KeyError.__delattr__" => "Implement delattr(self, name).", + "builtins.KeyError.__eq__" => "Return self==value.", + "builtins.KeyError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.KeyError.__ge__" => "Return self>=value.", + "builtins.KeyError.__getattribute__" => "Return getattr(self, name).", + "builtins.KeyError.__getstate__" => "Helper for pickle.", + "builtins.KeyError.__gt__" => "Return self>value.", + "builtins.KeyError.__hash__" => "Return hash(self).", + "builtins.KeyError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.KeyError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.KeyError.__le__" => "Return self<=value.", + "builtins.KeyError.__lt__" => "Return self<value.", + "builtins.KeyError.__ne__" => "Return self!=value.", + "builtins.KeyError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.KeyError.__reduce_ex__" => "Helper for pickle.", + "builtins.KeyError.__repr__" => "Return repr(self).", + "builtins.KeyError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.KeyError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.KeyError.__str__" => "Return str(self).", + "builtins.KeyError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.KeyError.add_note" => "Add a note to the exception", + "builtins.KeyError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.KeyboardInterrupt" => "Program interrupted by user.", + "builtins.KeyboardInterrupt.__delattr__" => "Implement delattr(self, name).", + "builtins.KeyboardInterrupt.__eq__" => "Return self==value.", + "builtins.KeyboardInterrupt.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.KeyboardInterrupt.__ge__" => "Return self>=value.", + "builtins.KeyboardInterrupt.__getattribute__" => "Return getattr(self, name).", + "builtins.KeyboardInterrupt.__getstate__" => "Helper for pickle.", + "builtins.KeyboardInterrupt.__gt__" => "Return self>value.", + "builtins.KeyboardInterrupt.__hash__" => "Return hash(self).", + "builtins.KeyboardInterrupt.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.KeyboardInterrupt.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.KeyboardInterrupt.__le__" => "Return self<=value.", + "builtins.KeyboardInterrupt.__lt__" => "Return self<value.", + "builtins.KeyboardInterrupt.__ne__" => "Return self!=value.", + "builtins.KeyboardInterrupt.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.KeyboardInterrupt.__reduce_ex__" => "Helper for pickle.", + "builtins.KeyboardInterrupt.__repr__" => "Return repr(self).", + "builtins.KeyboardInterrupt.__setattr__" => "Implement setattr(self, name, value).", + "builtins.KeyboardInterrupt.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.KeyboardInterrupt.__str__" => "Return str(self).", + "builtins.KeyboardInterrupt.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.KeyboardInterrupt.add_note" => "Add a note to the exception", + "builtins.KeyboardInterrupt.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.LookupError" => "Base class for lookup errors.", + "builtins.LookupError.__delattr__" => "Implement delattr(self, name).", + "builtins.LookupError.__eq__" => "Return self==value.", + "builtins.LookupError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.LookupError.__ge__" => "Return self>=value.", + "builtins.LookupError.__getattribute__" => "Return getattr(self, name).", + "builtins.LookupError.__getstate__" => "Helper for pickle.", + "builtins.LookupError.__gt__" => "Return self>value.", + "builtins.LookupError.__hash__" => "Return hash(self).", + "builtins.LookupError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.LookupError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.LookupError.__le__" => "Return self<=value.", + "builtins.LookupError.__lt__" => "Return self<value.", + "builtins.LookupError.__ne__" => "Return self!=value.", + "builtins.LookupError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.LookupError.__reduce_ex__" => "Helper for pickle.", + "builtins.LookupError.__repr__" => "Return repr(self).", + "builtins.LookupError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.LookupError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.LookupError.__str__" => "Return str(self).", + "builtins.LookupError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.LookupError.add_note" => "Add a note to the exception", + "builtins.LookupError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.MemoryError" => "Out of memory.", + "builtins.MemoryError.__delattr__" => "Implement delattr(self, name).", + "builtins.MemoryError.__eq__" => "Return self==value.", + "builtins.MemoryError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.MemoryError.__ge__" => "Return self>=value.", + "builtins.MemoryError.__getattribute__" => "Return getattr(self, name).", + "builtins.MemoryError.__getstate__" => "Helper for pickle.", + "builtins.MemoryError.__gt__" => "Return self>value.", + "builtins.MemoryError.__hash__" => "Return hash(self).", + "builtins.MemoryError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.MemoryError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.MemoryError.__le__" => "Return self<=value.", + "builtins.MemoryError.__lt__" => "Return self<value.", + "builtins.MemoryError.__ne__" => "Return self!=value.", + "builtins.MemoryError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.MemoryError.__reduce_ex__" => "Helper for pickle.", + "builtins.MemoryError.__repr__" => "Return repr(self).", + "builtins.MemoryError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.MemoryError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.MemoryError.__str__" => "Return str(self).", + "builtins.MemoryError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.MemoryError.add_note" => "Add a note to the exception", + "builtins.MemoryError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ModuleNotFoundError" => "Module not found.", + "builtins.ModuleNotFoundError.__delattr__" => "Implement delattr(self, name).", + "builtins.ModuleNotFoundError.__eq__" => "Return self==value.", + "builtins.ModuleNotFoundError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ModuleNotFoundError.__ge__" => "Return self>=value.", + "builtins.ModuleNotFoundError.__getattribute__" => "Return getattr(self, name).", + "builtins.ModuleNotFoundError.__getstate__" => "Helper for pickle.", + "builtins.ModuleNotFoundError.__gt__" => "Return self>value.", + "builtins.ModuleNotFoundError.__hash__" => "Return hash(self).", + "builtins.ModuleNotFoundError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ModuleNotFoundError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ModuleNotFoundError.__le__" => "Return self<=value.", + "builtins.ModuleNotFoundError.__lt__" => "Return self<value.", + "builtins.ModuleNotFoundError.__ne__" => "Return self!=value.", + "builtins.ModuleNotFoundError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ModuleNotFoundError.__reduce_ex__" => "Helper for pickle.", + "builtins.ModuleNotFoundError.__repr__" => "Return repr(self).", + "builtins.ModuleNotFoundError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ModuleNotFoundError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ModuleNotFoundError.__str__" => "Return str(self).", + "builtins.ModuleNotFoundError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ModuleNotFoundError.add_note" => "Add a note to the exception", + "builtins.ModuleNotFoundError.msg" => "exception message", + "builtins.ModuleNotFoundError.name" => "module name", + "builtins.ModuleNotFoundError.name_from" => "name imported from module", + "builtins.ModuleNotFoundError.path" => "module path", + "builtins.ModuleNotFoundError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.NameError" => "Name not found globally.", + "builtins.NameError.__delattr__" => "Implement delattr(self, name).", + "builtins.NameError.__eq__" => "Return self==value.", + "builtins.NameError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.NameError.__ge__" => "Return self>=value.", + "builtins.NameError.__getattribute__" => "Return getattr(self, name).", + "builtins.NameError.__getstate__" => "Helper for pickle.", + "builtins.NameError.__gt__" => "Return self>value.", + "builtins.NameError.__hash__" => "Return hash(self).", + "builtins.NameError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.NameError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.NameError.__le__" => "Return self<=value.", + "builtins.NameError.__lt__" => "Return self<value.", + "builtins.NameError.__ne__" => "Return self!=value.", + "builtins.NameError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.NameError.__reduce_ex__" => "Helper for pickle.", + "builtins.NameError.__repr__" => "Return repr(self).", + "builtins.NameError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.NameError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.NameError.__str__" => "Return str(self).", + "builtins.NameError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.NameError.add_note" => "Add a note to the exception", + "builtins.NameError.name" => "name", + "builtins.NameError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.NoneType" => "The type of the None singleton.", + "builtins.NoneType.__bool__" => "True if self else False", + "builtins.NoneType.__delattr__" => "Implement delattr(self, name).", + "builtins.NoneType.__eq__" => "Return self==value.", + "builtins.NoneType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.NoneType.__ge__" => "Return self>=value.", + "builtins.NoneType.__getattribute__" => "Return getattr(self, name).", + "builtins.NoneType.__getstate__" => "Helper for pickle.", + "builtins.NoneType.__gt__" => "Return self>value.", + "builtins.NoneType.__hash__" => "Return hash(self).", + "builtins.NoneType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.NoneType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.NoneType.__le__" => "Return self<=value.", + "builtins.NoneType.__lt__" => "Return self<value.", + "builtins.NoneType.__ne__" => "Return self!=value.", + "builtins.NoneType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.NoneType.__reduce__" => "Helper for pickle.", + "builtins.NoneType.__reduce_ex__" => "Helper for pickle.", + "builtins.NoneType.__repr__" => "Return repr(self).", + "builtins.NoneType.__setattr__" => "Implement setattr(self, name, value).", + "builtins.NoneType.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.NoneType.__str__" => "Return str(self).", + "builtins.NoneType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.NotADirectoryError" => "Operation only works on directories.", + "builtins.NotADirectoryError.__delattr__" => "Implement delattr(self, name).", + "builtins.NotADirectoryError.__eq__" => "Return self==value.", + "builtins.NotADirectoryError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.NotADirectoryError.__ge__" => "Return self>=value.", + "builtins.NotADirectoryError.__getattribute__" => "Return getattr(self, name).", + "builtins.NotADirectoryError.__getstate__" => "Helper for pickle.", + "builtins.NotADirectoryError.__gt__" => "Return self>value.", + "builtins.NotADirectoryError.__hash__" => "Return hash(self).", + "builtins.NotADirectoryError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.NotADirectoryError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.NotADirectoryError.__le__" => "Return self<=value.", + "builtins.NotADirectoryError.__lt__" => "Return self<value.", + "builtins.NotADirectoryError.__ne__" => "Return self!=value.", + "builtins.NotADirectoryError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.NotADirectoryError.__reduce_ex__" => "Helper for pickle.", + "builtins.NotADirectoryError.__repr__" => "Return repr(self).", + "builtins.NotADirectoryError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.NotADirectoryError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.NotADirectoryError.__str__" => "Return str(self).", + "builtins.NotADirectoryError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.NotADirectoryError.add_note" => "Add a note to the exception", + "builtins.NotADirectoryError.errno" => "POSIX exception code", + "builtins.NotADirectoryError.filename" => "exception filename", + "builtins.NotADirectoryError.filename2" => "second exception filename", + "builtins.NotADirectoryError.strerror" => "exception strerror", + "builtins.NotADirectoryError.winerror" => "Win32 exception code", + "builtins.NotADirectoryError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.NotImplementedError" => "Method or function hasn't been implemented yet.", + "builtins.NotImplementedError.__delattr__" => "Implement delattr(self, name).", + "builtins.NotImplementedError.__eq__" => "Return self==value.", + "builtins.NotImplementedError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.NotImplementedError.__ge__" => "Return self>=value.", + "builtins.NotImplementedError.__getattribute__" => "Return getattr(self, name).", + "builtins.NotImplementedError.__getstate__" => "Helper for pickle.", + "builtins.NotImplementedError.__gt__" => "Return self>value.", + "builtins.NotImplementedError.__hash__" => "Return hash(self).", + "builtins.NotImplementedError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.NotImplementedError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.NotImplementedError.__le__" => "Return self<=value.", + "builtins.NotImplementedError.__lt__" => "Return self<value.", + "builtins.NotImplementedError.__ne__" => "Return self!=value.", + "builtins.NotImplementedError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.NotImplementedError.__reduce_ex__" => "Helper for pickle.", + "builtins.NotImplementedError.__repr__" => "Return repr(self).", + "builtins.NotImplementedError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.NotImplementedError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.NotImplementedError.__str__" => "Return str(self).", + "builtins.NotImplementedError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.NotImplementedError.add_note" => "Add a note to the exception", + "builtins.NotImplementedError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.NotImplementedType" => "The type of the NotImplemented singleton.", + "builtins.NotImplementedType.__bool__" => "True if self else False", + "builtins.NotImplementedType.__delattr__" => "Implement delattr(self, name).", + "builtins.NotImplementedType.__eq__" => "Return self==value.", + "builtins.NotImplementedType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.NotImplementedType.__ge__" => "Return self>=value.", + "builtins.NotImplementedType.__getattribute__" => "Return getattr(self, name).", + "builtins.NotImplementedType.__getstate__" => "Helper for pickle.", + "builtins.NotImplementedType.__gt__" => "Return self>value.", + "builtins.NotImplementedType.__hash__" => "Return hash(self).", + "builtins.NotImplementedType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.NotImplementedType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.NotImplementedType.__le__" => "Return self<=value.", + "builtins.NotImplementedType.__lt__" => "Return self<value.", + "builtins.NotImplementedType.__ne__" => "Return self!=value.", + "builtins.NotImplementedType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.NotImplementedType.__reduce_ex__" => "Helper for pickle.", + "builtins.NotImplementedType.__repr__" => "Return repr(self).", + "builtins.NotImplementedType.__setattr__" => "Implement setattr(self, name, value).", + "builtins.NotImplementedType.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.NotImplementedType.__str__" => "Return str(self).", + "builtins.NotImplementedType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.OSError" => "Base class for I/O related errors.", + "builtins.OSError.__delattr__" => "Implement delattr(self, name).", + "builtins.OSError.__eq__" => "Return self==value.", + "builtins.OSError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.OSError.__ge__" => "Return self>=value.", + "builtins.OSError.__getattribute__" => "Return getattr(self, name).", + "builtins.OSError.__getstate__" => "Helper for pickle.", + "builtins.OSError.__gt__" => "Return self>value.", + "builtins.OSError.__hash__" => "Return hash(self).", + "builtins.OSError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.OSError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.OSError.__le__" => "Return self<=value.", + "builtins.OSError.__lt__" => "Return self<value.", + "builtins.OSError.__ne__" => "Return self!=value.", + "builtins.OSError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.OSError.__reduce_ex__" => "Helper for pickle.", + "builtins.OSError.__repr__" => "Return repr(self).", + "builtins.OSError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.OSError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.OSError.__str__" => "Return str(self).", + "builtins.OSError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.OSError.add_note" => "Add a note to the exception", + "builtins.OSError.errno" => "POSIX exception code", + "builtins.OSError.filename" => "exception filename", + "builtins.OSError.filename2" => "second exception filename", + "builtins.OSError.strerror" => "exception strerror", + "builtins.OSError.winerror" => "Win32 exception code", + "builtins.OSError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.OverflowError" => "Result too large to be represented.", + "builtins.OverflowError.__delattr__" => "Implement delattr(self, name).", + "builtins.OverflowError.__eq__" => "Return self==value.", + "builtins.OverflowError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.OverflowError.__ge__" => "Return self>=value.", + "builtins.OverflowError.__getattribute__" => "Return getattr(self, name).", + "builtins.OverflowError.__getstate__" => "Helper for pickle.", + "builtins.OverflowError.__gt__" => "Return self>value.", + "builtins.OverflowError.__hash__" => "Return hash(self).", + "builtins.OverflowError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.OverflowError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.OverflowError.__le__" => "Return self<=value.", + "builtins.OverflowError.__lt__" => "Return self<value.", + "builtins.OverflowError.__ne__" => "Return self!=value.", + "builtins.OverflowError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.OverflowError.__reduce_ex__" => "Helper for pickle.", + "builtins.OverflowError.__repr__" => "Return repr(self).", + "builtins.OverflowError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.OverflowError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.OverflowError.__str__" => "Return str(self).", + "builtins.OverflowError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.OverflowError.add_note" => "Add a note to the exception", + "builtins.OverflowError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.PendingDeprecationWarning" => "Base class for warnings about features which will be deprecated\nin the future.", + "builtins.PendingDeprecationWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.PendingDeprecationWarning.__eq__" => "Return self==value.", + "builtins.PendingDeprecationWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.PendingDeprecationWarning.__ge__" => "Return self>=value.", + "builtins.PendingDeprecationWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.PendingDeprecationWarning.__getstate__" => "Helper for pickle.", + "builtins.PendingDeprecationWarning.__gt__" => "Return self>value.", + "builtins.PendingDeprecationWarning.__hash__" => "Return hash(self).", + "builtins.PendingDeprecationWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.PendingDeprecationWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.PendingDeprecationWarning.__le__" => "Return self<=value.", + "builtins.PendingDeprecationWarning.__lt__" => "Return self<value.", + "builtins.PendingDeprecationWarning.__ne__" => "Return self!=value.", + "builtins.PendingDeprecationWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.PendingDeprecationWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.PendingDeprecationWarning.__repr__" => "Return repr(self).", + "builtins.PendingDeprecationWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.PendingDeprecationWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.PendingDeprecationWarning.__str__" => "Return str(self).", + "builtins.PendingDeprecationWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.PendingDeprecationWarning.add_note" => "Add a note to the exception", + "builtins.PendingDeprecationWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.PermissionError" => "Not enough permissions.", + "builtins.PermissionError.__delattr__" => "Implement delattr(self, name).", + "builtins.PermissionError.__eq__" => "Return self==value.", + "builtins.PermissionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.PermissionError.__ge__" => "Return self>=value.", + "builtins.PermissionError.__getattribute__" => "Return getattr(self, name).", + "builtins.PermissionError.__getstate__" => "Helper for pickle.", + "builtins.PermissionError.__gt__" => "Return self>value.", + "builtins.PermissionError.__hash__" => "Return hash(self).", + "builtins.PermissionError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.PermissionError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.PermissionError.__le__" => "Return self<=value.", + "builtins.PermissionError.__lt__" => "Return self<value.", + "builtins.PermissionError.__ne__" => "Return self!=value.", + "builtins.PermissionError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.PermissionError.__reduce_ex__" => "Helper for pickle.", + "builtins.PermissionError.__repr__" => "Return repr(self).", + "builtins.PermissionError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.PermissionError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.PermissionError.__str__" => "Return str(self).", + "builtins.PermissionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.PermissionError.add_note" => "Add a note to the exception", + "builtins.PermissionError.errno" => "POSIX exception code", + "builtins.PermissionError.filename" => "exception filename", + "builtins.PermissionError.filename2" => "second exception filename", + "builtins.PermissionError.strerror" => "exception strerror", + "builtins.PermissionError.winerror" => "Win32 exception code", + "builtins.PermissionError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ProcessLookupError" => "Process not found.", + "builtins.ProcessLookupError.__delattr__" => "Implement delattr(self, name).", + "builtins.ProcessLookupError.__eq__" => "Return self==value.", + "builtins.ProcessLookupError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ProcessLookupError.__ge__" => "Return self>=value.", + "builtins.ProcessLookupError.__getattribute__" => "Return getattr(self, name).", + "builtins.ProcessLookupError.__getstate__" => "Helper for pickle.", + "builtins.ProcessLookupError.__gt__" => "Return self>value.", + "builtins.ProcessLookupError.__hash__" => "Return hash(self).", + "builtins.ProcessLookupError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ProcessLookupError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ProcessLookupError.__le__" => "Return self<=value.", + "builtins.ProcessLookupError.__lt__" => "Return self<value.", + "builtins.ProcessLookupError.__ne__" => "Return self!=value.", + "builtins.ProcessLookupError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ProcessLookupError.__reduce_ex__" => "Helper for pickle.", + "builtins.ProcessLookupError.__repr__" => "Return repr(self).", + "builtins.ProcessLookupError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ProcessLookupError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ProcessLookupError.__str__" => "Return str(self).", + "builtins.ProcessLookupError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ProcessLookupError.add_note" => "Add a note to the exception", + "builtins.ProcessLookupError.errno" => "POSIX exception code", + "builtins.ProcessLookupError.filename" => "exception filename", + "builtins.ProcessLookupError.filename2" => "second exception filename", + "builtins.ProcessLookupError.strerror" => "exception strerror", + "builtins.ProcessLookupError.winerror" => "Win32 exception code", + "builtins.ProcessLookupError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.PyCapsule" => "Capsule objects let you wrap a C \"void *\" pointer in a Python\nobject. They're a way of passing data through the Python interpreter\nwithout creating your own custom type.\n\nCapsules are used for communication between extension modules.\nThey provide a way for an extension module to export a C interface\nto other extension modules, so that extension modules can use the\nPython import mechanism to link to one another.", + "builtins.PyCapsule.__delattr__" => "Implement delattr(self, name).", + "builtins.PyCapsule.__eq__" => "Return self==value.", + "builtins.PyCapsule.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.PyCapsule.__ge__" => "Return self>=value.", + "builtins.PyCapsule.__getattribute__" => "Return getattr(self, name).", + "builtins.PyCapsule.__getstate__" => "Helper for pickle.", + "builtins.PyCapsule.__gt__" => "Return self>value.", + "builtins.PyCapsule.__hash__" => "Return hash(self).", + "builtins.PyCapsule.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.PyCapsule.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.PyCapsule.__le__" => "Return self<=value.", + "builtins.PyCapsule.__lt__" => "Return self<value.", + "builtins.PyCapsule.__ne__" => "Return self!=value.", + "builtins.PyCapsule.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.PyCapsule.__reduce__" => "Helper for pickle.", + "builtins.PyCapsule.__reduce_ex__" => "Helper for pickle.", + "builtins.PyCapsule.__repr__" => "Return repr(self).", + "builtins.PyCapsule.__setattr__" => "Implement setattr(self, name, value).", + "builtins.PyCapsule.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.PyCapsule.__str__" => "Return str(self).", + "builtins.PyCapsule.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.PythonFinalizationError" => "Operation blocked during Python finalization.", + "builtins.PythonFinalizationError.__delattr__" => "Implement delattr(self, name).", + "builtins.PythonFinalizationError.__eq__" => "Return self==value.", + "builtins.PythonFinalizationError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.PythonFinalizationError.__ge__" => "Return self>=value.", + "builtins.PythonFinalizationError.__getattribute__" => "Return getattr(self, name).", + "builtins.PythonFinalizationError.__getstate__" => "Helper for pickle.", + "builtins.PythonFinalizationError.__gt__" => "Return self>value.", + "builtins.PythonFinalizationError.__hash__" => "Return hash(self).", + "builtins.PythonFinalizationError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.PythonFinalizationError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.PythonFinalizationError.__le__" => "Return self<=value.", + "builtins.PythonFinalizationError.__lt__" => "Return self<value.", + "builtins.PythonFinalizationError.__ne__" => "Return self!=value.", + "builtins.PythonFinalizationError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.PythonFinalizationError.__reduce_ex__" => "Helper for pickle.", + "builtins.PythonFinalizationError.__repr__" => "Return repr(self).", + "builtins.PythonFinalizationError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.PythonFinalizationError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.PythonFinalizationError.__str__" => "Return str(self).", + "builtins.PythonFinalizationError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.PythonFinalizationError.add_note" => "Add a note to the exception", + "builtins.PythonFinalizationError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.RecursionError" => "Recursion limit exceeded.", + "builtins.RecursionError.__delattr__" => "Implement delattr(self, name).", + "builtins.RecursionError.__eq__" => "Return self==value.", + "builtins.RecursionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.RecursionError.__ge__" => "Return self>=value.", + "builtins.RecursionError.__getattribute__" => "Return getattr(self, name).", + "builtins.RecursionError.__getstate__" => "Helper for pickle.", + "builtins.RecursionError.__gt__" => "Return self>value.", + "builtins.RecursionError.__hash__" => "Return hash(self).", + "builtins.RecursionError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.RecursionError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.RecursionError.__le__" => "Return self<=value.", + "builtins.RecursionError.__lt__" => "Return self<value.", + "builtins.RecursionError.__ne__" => "Return self!=value.", + "builtins.RecursionError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.RecursionError.__reduce_ex__" => "Helper for pickle.", + "builtins.RecursionError.__repr__" => "Return repr(self).", + "builtins.RecursionError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.RecursionError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.RecursionError.__str__" => "Return str(self).", + "builtins.RecursionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.RecursionError.add_note" => "Add a note to the exception", + "builtins.RecursionError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ReferenceError" => "Weak ref proxy used after referent went away.", + "builtins.ReferenceError.__delattr__" => "Implement delattr(self, name).", + "builtins.ReferenceError.__eq__" => "Return self==value.", + "builtins.ReferenceError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ReferenceError.__ge__" => "Return self>=value.", + "builtins.ReferenceError.__getattribute__" => "Return getattr(self, name).", + "builtins.ReferenceError.__getstate__" => "Helper for pickle.", + "builtins.ReferenceError.__gt__" => "Return self>value.", + "builtins.ReferenceError.__hash__" => "Return hash(self).", + "builtins.ReferenceError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ReferenceError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ReferenceError.__le__" => "Return self<=value.", + "builtins.ReferenceError.__lt__" => "Return self<value.", + "builtins.ReferenceError.__ne__" => "Return self!=value.", + "builtins.ReferenceError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ReferenceError.__reduce_ex__" => "Helper for pickle.", + "builtins.ReferenceError.__repr__" => "Return repr(self).", + "builtins.ReferenceError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ReferenceError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ReferenceError.__str__" => "Return str(self).", + "builtins.ReferenceError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ReferenceError.add_note" => "Add a note to the exception", + "builtins.ReferenceError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ResourceWarning" => "Base class for warnings about resource usage.", + "builtins.ResourceWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.ResourceWarning.__eq__" => "Return self==value.", + "builtins.ResourceWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ResourceWarning.__ge__" => "Return self>=value.", + "builtins.ResourceWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.ResourceWarning.__getstate__" => "Helper for pickle.", + "builtins.ResourceWarning.__gt__" => "Return self>value.", + "builtins.ResourceWarning.__hash__" => "Return hash(self).", + "builtins.ResourceWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ResourceWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ResourceWarning.__le__" => "Return self<=value.", + "builtins.ResourceWarning.__lt__" => "Return self<value.", + "builtins.ResourceWarning.__ne__" => "Return self!=value.", + "builtins.ResourceWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ResourceWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.ResourceWarning.__repr__" => "Return repr(self).", + "builtins.ResourceWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ResourceWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ResourceWarning.__str__" => "Return str(self).", + "builtins.ResourceWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ResourceWarning.add_note" => "Add a note to the exception", + "builtins.ResourceWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.RuntimeError" => "Unspecified run-time error.", + "builtins.RuntimeError.__delattr__" => "Implement delattr(self, name).", + "builtins.RuntimeError.__eq__" => "Return self==value.", + "builtins.RuntimeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.RuntimeError.__ge__" => "Return self>=value.", + "builtins.RuntimeError.__getattribute__" => "Return getattr(self, name).", + "builtins.RuntimeError.__getstate__" => "Helper for pickle.", + "builtins.RuntimeError.__gt__" => "Return self>value.", + "builtins.RuntimeError.__hash__" => "Return hash(self).", + "builtins.RuntimeError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.RuntimeError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.RuntimeError.__le__" => "Return self<=value.", + "builtins.RuntimeError.__lt__" => "Return self<value.", + "builtins.RuntimeError.__ne__" => "Return self!=value.", + "builtins.RuntimeError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.RuntimeError.__reduce_ex__" => "Helper for pickle.", + "builtins.RuntimeError.__repr__" => "Return repr(self).", + "builtins.RuntimeError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.RuntimeError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.RuntimeError.__str__" => "Return str(self).", + "builtins.RuntimeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.RuntimeError.add_note" => "Add a note to the exception", + "builtins.RuntimeError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.RuntimeWarning" => "Base class for warnings about dubious runtime behavior.", + "builtins.RuntimeWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.RuntimeWarning.__eq__" => "Return self==value.", + "builtins.RuntimeWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.RuntimeWarning.__ge__" => "Return self>=value.", + "builtins.RuntimeWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.RuntimeWarning.__getstate__" => "Helper for pickle.", + "builtins.RuntimeWarning.__gt__" => "Return self>value.", + "builtins.RuntimeWarning.__hash__" => "Return hash(self).", + "builtins.RuntimeWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.RuntimeWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.RuntimeWarning.__le__" => "Return self<=value.", + "builtins.RuntimeWarning.__lt__" => "Return self<value.", + "builtins.RuntimeWarning.__ne__" => "Return self!=value.", + "builtins.RuntimeWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.RuntimeWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.RuntimeWarning.__repr__" => "Return repr(self).", + "builtins.RuntimeWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.RuntimeWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.RuntimeWarning.__str__" => "Return str(self).", + "builtins.RuntimeWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.RuntimeWarning.add_note" => "Add a note to the exception", + "builtins.RuntimeWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.SimpleNamespace" => "A simple attribute-based namespace.", + "builtins.SimpleNamespace.__delattr__" => "Implement delattr(self, name).", + "builtins.SimpleNamespace.__eq__" => "Return self==value.", + "builtins.SimpleNamespace.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.SimpleNamespace.__ge__" => "Return self>=value.", + "builtins.SimpleNamespace.__getattribute__" => "Return getattr(self, name).", + "builtins.SimpleNamespace.__getstate__" => "Helper for pickle.", + "builtins.SimpleNamespace.__gt__" => "Return self>value.", + "builtins.SimpleNamespace.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.SimpleNamespace.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.SimpleNamespace.__le__" => "Return self<=value.", + "builtins.SimpleNamespace.__lt__" => "Return self<value.", + "builtins.SimpleNamespace.__ne__" => "Return self!=value.", + "builtins.SimpleNamespace.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.SimpleNamespace.__reduce__" => "Return state information for pickling", + "builtins.SimpleNamespace.__reduce_ex__" => "Helper for pickle.", + "builtins.SimpleNamespace.__replace__" => "Return a copy of the namespace object with new values for the specified attributes.", + "builtins.SimpleNamespace.__repr__" => "Return repr(self).", + "builtins.SimpleNamespace.__setattr__" => "Implement setattr(self, name, value).", + "builtins.SimpleNamespace.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.SimpleNamespace.__str__" => "Return str(self).", + "builtins.SimpleNamespace.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.StopAsyncIteration" => "Signal the end from iterator.__anext__().", + "builtins.StopAsyncIteration.__delattr__" => "Implement delattr(self, name).", + "builtins.StopAsyncIteration.__eq__" => "Return self==value.", + "builtins.StopAsyncIteration.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.StopAsyncIteration.__ge__" => "Return self>=value.", + "builtins.StopAsyncIteration.__getattribute__" => "Return getattr(self, name).", + "builtins.StopAsyncIteration.__getstate__" => "Helper for pickle.", + "builtins.StopAsyncIteration.__gt__" => "Return self>value.", + "builtins.StopAsyncIteration.__hash__" => "Return hash(self).", + "builtins.StopAsyncIteration.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.StopAsyncIteration.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.StopAsyncIteration.__le__" => "Return self<=value.", + "builtins.StopAsyncIteration.__lt__" => "Return self<value.", + "builtins.StopAsyncIteration.__ne__" => "Return self!=value.", + "builtins.StopAsyncIteration.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.StopAsyncIteration.__reduce_ex__" => "Helper for pickle.", + "builtins.StopAsyncIteration.__repr__" => "Return repr(self).", + "builtins.StopAsyncIteration.__setattr__" => "Implement setattr(self, name, value).", + "builtins.StopAsyncIteration.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.StopAsyncIteration.__str__" => "Return str(self).", + "builtins.StopAsyncIteration.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.StopAsyncIteration.add_note" => "Add a note to the exception", + "builtins.StopAsyncIteration.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.StopIteration" => "Signal the end from iterator.__next__().", + "builtins.StopIteration.__delattr__" => "Implement delattr(self, name).", + "builtins.StopIteration.__eq__" => "Return self==value.", + "builtins.StopIteration.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.StopIteration.__ge__" => "Return self>=value.", + "builtins.StopIteration.__getattribute__" => "Return getattr(self, name).", + "builtins.StopIteration.__getstate__" => "Helper for pickle.", + "builtins.StopIteration.__gt__" => "Return self>value.", + "builtins.StopIteration.__hash__" => "Return hash(self).", + "builtins.StopIteration.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.StopIteration.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.StopIteration.__le__" => "Return self<=value.", + "builtins.StopIteration.__lt__" => "Return self<value.", + "builtins.StopIteration.__ne__" => "Return self!=value.", + "builtins.StopIteration.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.StopIteration.__reduce_ex__" => "Helper for pickle.", + "builtins.StopIteration.__repr__" => "Return repr(self).", + "builtins.StopIteration.__setattr__" => "Implement setattr(self, name, value).", + "builtins.StopIteration.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.StopIteration.__str__" => "Return str(self).", + "builtins.StopIteration.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.StopIteration.add_note" => "Add a note to the exception", + "builtins.StopIteration.value" => "generator return value", + "builtins.StopIteration.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.SyntaxError" => "Invalid syntax.", + "builtins.SyntaxError.__delattr__" => "Implement delattr(self, name).", + "builtins.SyntaxError.__eq__" => "Return self==value.", + "builtins.SyntaxError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.SyntaxError.__ge__" => "Return self>=value.", + "builtins.SyntaxError.__getattribute__" => "Return getattr(self, name).", + "builtins.SyntaxError.__getstate__" => "Helper for pickle.", + "builtins.SyntaxError.__gt__" => "Return self>value.", + "builtins.SyntaxError.__hash__" => "Return hash(self).", + "builtins.SyntaxError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.SyntaxError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.SyntaxError.__le__" => "Return self<=value.", + "builtins.SyntaxError.__lt__" => "Return self<value.", + "builtins.SyntaxError.__ne__" => "Return self!=value.", + "builtins.SyntaxError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.SyntaxError.__reduce_ex__" => "Helper for pickle.", + "builtins.SyntaxError.__repr__" => "Return repr(self).", + "builtins.SyntaxError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.SyntaxError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.SyntaxError.__str__" => "Return str(self).", + "builtins.SyntaxError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.SyntaxError._metadata" => "exception private metadata", + "builtins.SyntaxError.add_note" => "Add a note to the exception", + "builtins.SyntaxError.end_lineno" => "exception end lineno", + "builtins.SyntaxError.end_offset" => "exception end offset", + "builtins.SyntaxError.filename" => "exception filename", + "builtins.SyntaxError.lineno" => "exception lineno", + "builtins.SyntaxError.msg" => "exception msg", + "builtins.SyntaxError.offset" => "exception offset", + "builtins.SyntaxError.print_file_and_line" => "exception print_file_and_line", + "builtins.SyntaxError.text" => "exception text", + "builtins.SyntaxError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.SyntaxWarning" => "Base class for warnings about dubious syntax.", + "builtins.SyntaxWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.SyntaxWarning.__eq__" => "Return self==value.", + "builtins.SyntaxWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.SyntaxWarning.__ge__" => "Return self>=value.", + "builtins.SyntaxWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.SyntaxWarning.__getstate__" => "Helper for pickle.", + "builtins.SyntaxWarning.__gt__" => "Return self>value.", + "builtins.SyntaxWarning.__hash__" => "Return hash(self).", + "builtins.SyntaxWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.SyntaxWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.SyntaxWarning.__le__" => "Return self<=value.", + "builtins.SyntaxWarning.__lt__" => "Return self<value.", + "builtins.SyntaxWarning.__ne__" => "Return self!=value.", + "builtins.SyntaxWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.SyntaxWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.SyntaxWarning.__repr__" => "Return repr(self).", + "builtins.SyntaxWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.SyntaxWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.SyntaxWarning.__str__" => "Return str(self).", + "builtins.SyntaxWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.SyntaxWarning.add_note" => "Add a note to the exception", + "builtins.SyntaxWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.SystemError" => "Internal error in the Python interpreter.\n\nPlease report this to the Python maintainer, along with the traceback,\nthe Python version, and the hardware/OS platform and version.", + "builtins.SystemError.__delattr__" => "Implement delattr(self, name).", + "builtins.SystemError.__eq__" => "Return self==value.", + "builtins.SystemError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.SystemError.__ge__" => "Return self>=value.", + "builtins.SystemError.__getattribute__" => "Return getattr(self, name).", + "builtins.SystemError.__getstate__" => "Helper for pickle.", + "builtins.SystemError.__gt__" => "Return self>value.", + "builtins.SystemError.__hash__" => "Return hash(self).", + "builtins.SystemError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.SystemError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.SystemError.__le__" => "Return self<=value.", + "builtins.SystemError.__lt__" => "Return self<value.", + "builtins.SystemError.__ne__" => "Return self!=value.", + "builtins.SystemError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.SystemError.__reduce_ex__" => "Helper for pickle.", + "builtins.SystemError.__repr__" => "Return repr(self).", + "builtins.SystemError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.SystemError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.SystemError.__str__" => "Return str(self).", + "builtins.SystemError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.SystemError.add_note" => "Add a note to the exception", + "builtins.SystemError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.SystemExit" => "Request to exit from the interpreter.", + "builtins.SystemExit.__delattr__" => "Implement delattr(self, name).", + "builtins.SystemExit.__eq__" => "Return self==value.", + "builtins.SystemExit.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.SystemExit.__ge__" => "Return self>=value.", + "builtins.SystemExit.__getattribute__" => "Return getattr(self, name).", + "builtins.SystemExit.__getstate__" => "Helper for pickle.", + "builtins.SystemExit.__gt__" => "Return self>value.", + "builtins.SystemExit.__hash__" => "Return hash(self).", + "builtins.SystemExit.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.SystemExit.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.SystemExit.__le__" => "Return self<=value.", + "builtins.SystemExit.__lt__" => "Return self<value.", + "builtins.SystemExit.__ne__" => "Return self!=value.", + "builtins.SystemExit.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.SystemExit.__reduce_ex__" => "Helper for pickle.", + "builtins.SystemExit.__repr__" => "Return repr(self).", + "builtins.SystemExit.__setattr__" => "Implement setattr(self, name, value).", + "builtins.SystemExit.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.SystemExit.__str__" => "Return str(self).", + "builtins.SystemExit.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.SystemExit.add_note" => "Add a note to the exception", + "builtins.SystemExit.code" => "exception code", + "builtins.SystemExit.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.TabError" => "Improper mixture of spaces and tabs.", + "builtins.TabError.__delattr__" => "Implement delattr(self, name).", + "builtins.TabError.__eq__" => "Return self==value.", + "builtins.TabError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.TabError.__ge__" => "Return self>=value.", + "builtins.TabError.__getattribute__" => "Return getattr(self, name).", + "builtins.TabError.__getstate__" => "Helper for pickle.", + "builtins.TabError.__gt__" => "Return self>value.", + "builtins.TabError.__hash__" => "Return hash(self).", + "builtins.TabError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.TabError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.TabError.__le__" => "Return self<=value.", + "builtins.TabError.__lt__" => "Return self<value.", + "builtins.TabError.__ne__" => "Return self!=value.", + "builtins.TabError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.TabError.__reduce_ex__" => "Helper for pickle.", + "builtins.TabError.__repr__" => "Return repr(self).", + "builtins.TabError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.TabError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.TabError.__str__" => "Return str(self).", + "builtins.TabError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.TabError._metadata" => "exception private metadata", + "builtins.TabError.add_note" => "Add a note to the exception", + "builtins.TabError.end_lineno" => "exception end lineno", + "builtins.TabError.end_offset" => "exception end offset", + "builtins.TabError.filename" => "exception filename", + "builtins.TabError.lineno" => "exception lineno", + "builtins.TabError.msg" => "exception msg", + "builtins.TabError.offset" => "exception offset", + "builtins.TabError.print_file_and_line" => "exception print_file_and_line", + "builtins.TabError.text" => "exception text", + "builtins.TabError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.TimeoutError" => "Timeout expired.", + "builtins.TimeoutError.__delattr__" => "Implement delattr(self, name).", + "builtins.TimeoutError.__eq__" => "Return self==value.", + "builtins.TimeoutError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.TimeoutError.__ge__" => "Return self>=value.", + "builtins.TimeoutError.__getattribute__" => "Return getattr(self, name).", + "builtins.TimeoutError.__getstate__" => "Helper for pickle.", + "builtins.TimeoutError.__gt__" => "Return self>value.", + "builtins.TimeoutError.__hash__" => "Return hash(self).", + "builtins.TimeoutError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.TimeoutError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.TimeoutError.__le__" => "Return self<=value.", + "builtins.TimeoutError.__lt__" => "Return self<value.", + "builtins.TimeoutError.__ne__" => "Return self!=value.", + "builtins.TimeoutError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.TimeoutError.__reduce_ex__" => "Helper for pickle.", + "builtins.TimeoutError.__repr__" => "Return repr(self).", + "builtins.TimeoutError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.TimeoutError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.TimeoutError.__str__" => "Return str(self).", + "builtins.TimeoutError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.TimeoutError.add_note" => "Add a note to the exception", + "builtins.TimeoutError.errno" => "POSIX exception code", + "builtins.TimeoutError.filename" => "exception filename", + "builtins.TimeoutError.filename2" => "second exception filename", + "builtins.TimeoutError.strerror" => "exception strerror", + "builtins.TimeoutError.winerror" => "Win32 exception code", + "builtins.TimeoutError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.TypeError" => "Inappropriate argument type.", + "builtins.TypeError.__delattr__" => "Implement delattr(self, name).", + "builtins.TypeError.__eq__" => "Return self==value.", + "builtins.TypeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.TypeError.__ge__" => "Return self>=value.", + "builtins.TypeError.__getattribute__" => "Return getattr(self, name).", + "builtins.TypeError.__getstate__" => "Helper for pickle.", + "builtins.TypeError.__gt__" => "Return self>value.", + "builtins.TypeError.__hash__" => "Return hash(self).", + "builtins.TypeError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.TypeError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.TypeError.__le__" => "Return self<=value.", + "builtins.TypeError.__lt__" => "Return self<value.", + "builtins.TypeError.__ne__" => "Return self!=value.", + "builtins.TypeError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.TypeError.__reduce_ex__" => "Helper for pickle.", + "builtins.TypeError.__repr__" => "Return repr(self).", + "builtins.TypeError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.TypeError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.TypeError.__str__" => "Return str(self).", + "builtins.TypeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.TypeError.add_note" => "Add a note to the exception", + "builtins.TypeError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.UnboundLocalError" => "Local name referenced but not bound to a value.", + "builtins.UnboundLocalError.__delattr__" => "Implement delattr(self, name).", + "builtins.UnboundLocalError.__eq__" => "Return self==value.", + "builtins.UnboundLocalError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.UnboundLocalError.__ge__" => "Return self>=value.", + "builtins.UnboundLocalError.__getattribute__" => "Return getattr(self, name).", + "builtins.UnboundLocalError.__getstate__" => "Helper for pickle.", + "builtins.UnboundLocalError.__gt__" => "Return self>value.", + "builtins.UnboundLocalError.__hash__" => "Return hash(self).", + "builtins.UnboundLocalError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.UnboundLocalError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.UnboundLocalError.__le__" => "Return self<=value.", + "builtins.UnboundLocalError.__lt__" => "Return self<value.", + "builtins.UnboundLocalError.__ne__" => "Return self!=value.", + "builtins.UnboundLocalError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.UnboundLocalError.__reduce_ex__" => "Helper for pickle.", + "builtins.UnboundLocalError.__repr__" => "Return repr(self).", + "builtins.UnboundLocalError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.UnboundLocalError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.UnboundLocalError.__str__" => "Return str(self).", + "builtins.UnboundLocalError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UnboundLocalError.add_note" => "Add a note to the exception", + "builtins.UnboundLocalError.name" => "name", + "builtins.UnboundLocalError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.UnicodeDecodeError" => "Unicode decoding error.", + "builtins.UnicodeDecodeError.__delattr__" => "Implement delattr(self, name).", + "builtins.UnicodeDecodeError.__eq__" => "Return self==value.", + "builtins.UnicodeDecodeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.UnicodeDecodeError.__ge__" => "Return self>=value.", + "builtins.UnicodeDecodeError.__getattribute__" => "Return getattr(self, name).", + "builtins.UnicodeDecodeError.__getstate__" => "Helper for pickle.", + "builtins.UnicodeDecodeError.__gt__" => "Return self>value.", + "builtins.UnicodeDecodeError.__hash__" => "Return hash(self).", + "builtins.UnicodeDecodeError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.UnicodeDecodeError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.UnicodeDecodeError.__le__" => "Return self<=value.", + "builtins.UnicodeDecodeError.__lt__" => "Return self<value.", + "builtins.UnicodeDecodeError.__ne__" => "Return self!=value.", + "builtins.UnicodeDecodeError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.UnicodeDecodeError.__reduce_ex__" => "Helper for pickle.", + "builtins.UnicodeDecodeError.__repr__" => "Return repr(self).", + "builtins.UnicodeDecodeError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.UnicodeDecodeError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.UnicodeDecodeError.__str__" => "Return str(self).", + "builtins.UnicodeDecodeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UnicodeDecodeError.add_note" => "Add a note to the exception", + "builtins.UnicodeDecodeError.encoding" => "exception encoding", + "builtins.UnicodeDecodeError.end" => "exception end", + "builtins.UnicodeDecodeError.object" => "exception object", + "builtins.UnicodeDecodeError.reason" => "exception reason", + "builtins.UnicodeDecodeError.start" => "exception start", + "builtins.UnicodeDecodeError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.UnicodeEncodeError" => "Unicode encoding error.", + "builtins.UnicodeEncodeError.__delattr__" => "Implement delattr(self, name).", + "builtins.UnicodeEncodeError.__eq__" => "Return self==value.", + "builtins.UnicodeEncodeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.UnicodeEncodeError.__ge__" => "Return self>=value.", + "builtins.UnicodeEncodeError.__getattribute__" => "Return getattr(self, name).", + "builtins.UnicodeEncodeError.__getstate__" => "Helper for pickle.", + "builtins.UnicodeEncodeError.__gt__" => "Return self>value.", + "builtins.UnicodeEncodeError.__hash__" => "Return hash(self).", + "builtins.UnicodeEncodeError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.UnicodeEncodeError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.UnicodeEncodeError.__le__" => "Return self<=value.", + "builtins.UnicodeEncodeError.__lt__" => "Return self<value.", + "builtins.UnicodeEncodeError.__ne__" => "Return self!=value.", + "builtins.UnicodeEncodeError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.UnicodeEncodeError.__reduce_ex__" => "Helper for pickle.", + "builtins.UnicodeEncodeError.__repr__" => "Return repr(self).", + "builtins.UnicodeEncodeError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.UnicodeEncodeError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.UnicodeEncodeError.__str__" => "Return str(self).", + "builtins.UnicodeEncodeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UnicodeEncodeError.add_note" => "Add a note to the exception", + "builtins.UnicodeEncodeError.encoding" => "exception encoding", + "builtins.UnicodeEncodeError.end" => "exception end", + "builtins.UnicodeEncodeError.object" => "exception object", + "builtins.UnicodeEncodeError.reason" => "exception reason", + "builtins.UnicodeEncodeError.start" => "exception start", + "builtins.UnicodeEncodeError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.UnicodeError" => "Unicode related error.", + "builtins.UnicodeError.__delattr__" => "Implement delattr(self, name).", + "builtins.UnicodeError.__eq__" => "Return self==value.", + "builtins.UnicodeError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.UnicodeError.__ge__" => "Return self>=value.", + "builtins.UnicodeError.__getattribute__" => "Return getattr(self, name).", + "builtins.UnicodeError.__getstate__" => "Helper for pickle.", + "builtins.UnicodeError.__gt__" => "Return self>value.", + "builtins.UnicodeError.__hash__" => "Return hash(self).", + "builtins.UnicodeError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.UnicodeError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.UnicodeError.__le__" => "Return self<=value.", + "builtins.UnicodeError.__lt__" => "Return self<value.", + "builtins.UnicodeError.__ne__" => "Return self!=value.", + "builtins.UnicodeError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.UnicodeError.__reduce_ex__" => "Helper for pickle.", + "builtins.UnicodeError.__repr__" => "Return repr(self).", + "builtins.UnicodeError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.UnicodeError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.UnicodeError.__str__" => "Return str(self).", + "builtins.UnicodeError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UnicodeError.add_note" => "Add a note to the exception", + "builtins.UnicodeError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.UnicodeTranslateError" => "Unicode translation error.", + "builtins.UnicodeTranslateError.__delattr__" => "Implement delattr(self, name).", + "builtins.UnicodeTranslateError.__eq__" => "Return self==value.", + "builtins.UnicodeTranslateError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.UnicodeTranslateError.__ge__" => "Return self>=value.", + "builtins.UnicodeTranslateError.__getattribute__" => "Return getattr(self, name).", + "builtins.UnicodeTranslateError.__getstate__" => "Helper for pickle.", + "builtins.UnicodeTranslateError.__gt__" => "Return self>value.", + "builtins.UnicodeTranslateError.__hash__" => "Return hash(self).", + "builtins.UnicodeTranslateError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.UnicodeTranslateError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.UnicodeTranslateError.__le__" => "Return self<=value.", + "builtins.UnicodeTranslateError.__lt__" => "Return self<value.", + "builtins.UnicodeTranslateError.__ne__" => "Return self!=value.", + "builtins.UnicodeTranslateError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.UnicodeTranslateError.__reduce_ex__" => "Helper for pickle.", + "builtins.UnicodeTranslateError.__repr__" => "Return repr(self).", + "builtins.UnicodeTranslateError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.UnicodeTranslateError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.UnicodeTranslateError.__str__" => "Return str(self).", + "builtins.UnicodeTranslateError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UnicodeTranslateError.add_note" => "Add a note to the exception", + "builtins.UnicodeTranslateError.encoding" => "exception encoding", + "builtins.UnicodeTranslateError.end" => "exception end", + "builtins.UnicodeTranslateError.object" => "exception object", + "builtins.UnicodeTranslateError.reason" => "exception reason", + "builtins.UnicodeTranslateError.start" => "exception start", + "builtins.UnicodeTranslateError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.UnicodeWarning" => "Base class for warnings about Unicode related problems, mostly\nrelated to conversion problems.", + "builtins.UnicodeWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.UnicodeWarning.__eq__" => "Return self==value.", + "builtins.UnicodeWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.UnicodeWarning.__ge__" => "Return self>=value.", + "builtins.UnicodeWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.UnicodeWarning.__getstate__" => "Helper for pickle.", + "builtins.UnicodeWarning.__gt__" => "Return self>value.", + "builtins.UnicodeWarning.__hash__" => "Return hash(self).", + "builtins.UnicodeWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.UnicodeWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.UnicodeWarning.__le__" => "Return self<=value.", + "builtins.UnicodeWarning.__lt__" => "Return self<value.", + "builtins.UnicodeWarning.__ne__" => "Return self!=value.", + "builtins.UnicodeWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.UnicodeWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.UnicodeWarning.__repr__" => "Return repr(self).", + "builtins.UnicodeWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.UnicodeWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.UnicodeWarning.__str__" => "Return str(self).", + "builtins.UnicodeWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UnicodeWarning.add_note" => "Add a note to the exception", + "builtins.UnicodeWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.Union" => "Represent a union type\n\nE.g. for int | str", + "builtins.Union.__class_getitem__" => "See PEP 585", + "builtins.Union.__delattr__" => "Implement delattr(self, name).", + "builtins.Union.__eq__" => "Return self==value.", + "builtins.Union.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.Union.__ge__" => "Return self>=value.", + "builtins.Union.__getattribute__" => "Return getattr(self, name).", + "builtins.Union.__getitem__" => "Return self[key].", + "builtins.Union.__getstate__" => "Helper for pickle.", + "builtins.Union.__gt__" => "Return self>value.", + "builtins.Union.__hash__" => "Return hash(self).", + "builtins.Union.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.Union.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.Union.__le__" => "Return self<=value.", + "builtins.Union.__lt__" => "Return self<value.", + "builtins.Union.__ne__" => "Return self!=value.", + "builtins.Union.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.Union.__or__" => "Return self|value.", + "builtins.Union.__origin__" => "Always returns the type", + "builtins.Union.__parameters__" => "Type variables in the types.UnionType.", + "builtins.Union.__reduce__" => "Helper for pickle.", + "builtins.Union.__reduce_ex__" => "Helper for pickle.", + "builtins.Union.__repr__" => "Return repr(self).", + "builtins.Union.__ror__" => "Return value|self.", + "builtins.Union.__setattr__" => "Implement setattr(self, name, value).", + "builtins.Union.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.Union.__str__" => "Return str(self).", + "builtins.Union.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UserWarning" => "Base class for warnings generated by user code.", + "builtins.UserWarning.__delattr__" => "Implement delattr(self, name).", + "builtins.UserWarning.__eq__" => "Return self==value.", + "builtins.UserWarning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.UserWarning.__ge__" => "Return self>=value.", + "builtins.UserWarning.__getattribute__" => "Return getattr(self, name).", + "builtins.UserWarning.__getstate__" => "Helper for pickle.", + "builtins.UserWarning.__gt__" => "Return self>value.", + "builtins.UserWarning.__hash__" => "Return hash(self).", + "builtins.UserWarning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.UserWarning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.UserWarning.__le__" => "Return self<=value.", + "builtins.UserWarning.__lt__" => "Return self<value.", + "builtins.UserWarning.__ne__" => "Return self!=value.", + "builtins.UserWarning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.UserWarning.__reduce_ex__" => "Helper for pickle.", + "builtins.UserWarning.__repr__" => "Return repr(self).", + "builtins.UserWarning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.UserWarning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.UserWarning.__str__" => "Return str(self).", + "builtins.UserWarning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.UserWarning.add_note" => "Add a note to the exception", + "builtins.UserWarning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ValueError" => "Inappropriate argument value (of correct type).", + "builtins.ValueError.__delattr__" => "Implement delattr(self, name).", + "builtins.ValueError.__eq__" => "Return self==value.", + "builtins.ValueError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ValueError.__ge__" => "Return self>=value.", + "builtins.ValueError.__getattribute__" => "Return getattr(self, name).", + "builtins.ValueError.__getstate__" => "Helper for pickle.", + "builtins.ValueError.__gt__" => "Return self>value.", + "builtins.ValueError.__hash__" => "Return hash(self).", + "builtins.ValueError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ValueError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ValueError.__le__" => "Return self<=value.", + "builtins.ValueError.__lt__" => "Return self<value.", + "builtins.ValueError.__ne__" => "Return self!=value.", + "builtins.ValueError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ValueError.__reduce_ex__" => "Helper for pickle.", + "builtins.ValueError.__repr__" => "Return repr(self).", + "builtins.ValueError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ValueError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ValueError.__str__" => "Return str(self).", + "builtins.ValueError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ValueError.add_note" => "Add a note to the exception", + "builtins.ValueError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.Warning" => "Base class for warning categories.", + "builtins.Warning.__delattr__" => "Implement delattr(self, name).", + "builtins.Warning.__eq__" => "Return self==value.", + "builtins.Warning.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.Warning.__ge__" => "Return self>=value.", + "builtins.Warning.__getattribute__" => "Return getattr(self, name).", + "builtins.Warning.__getstate__" => "Helper for pickle.", + "builtins.Warning.__gt__" => "Return self>value.", + "builtins.Warning.__hash__" => "Return hash(self).", + "builtins.Warning.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.Warning.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.Warning.__le__" => "Return self<=value.", + "builtins.Warning.__lt__" => "Return self<value.", + "builtins.Warning.__ne__" => "Return self!=value.", + "builtins.Warning.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.Warning.__reduce_ex__" => "Helper for pickle.", + "builtins.Warning.__repr__" => "Return repr(self).", + "builtins.Warning.__setattr__" => "Implement setattr(self, name, value).", + "builtins.Warning.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.Warning.__str__" => "Return str(self).", + "builtins.Warning.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.Warning.add_note" => "Add a note to the exception", + "builtins.Warning.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.WindowsError" => "Base class for I/O related errors.", + "builtins.WindowsError.__delattr__" => "Implement delattr(self, name).", + "builtins.WindowsError.__eq__" => "Return self==value.", + "builtins.WindowsError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.WindowsError.__ge__" => "Return self>=value.", + "builtins.WindowsError.__getattribute__" => "Return getattr(self, name).", + "builtins.WindowsError.__getstate__" => "Helper for pickle.", + "builtins.WindowsError.__gt__" => "Return self>value.", + "builtins.WindowsError.__hash__" => "Return hash(self).", + "builtins.WindowsError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.WindowsError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.WindowsError.__le__" => "Return self<=value.", + "builtins.WindowsError.__lt__" => "Return self<value.", + "builtins.WindowsError.__ne__" => "Return self!=value.", + "builtins.WindowsError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.WindowsError.__reduce_ex__" => "Helper for pickle.", + "builtins.WindowsError.__repr__" => "Return repr(self).", + "builtins.WindowsError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.WindowsError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.WindowsError.__str__" => "Return str(self).", + "builtins.WindowsError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.WindowsError.add_note" => "Add a note to the exception", + "builtins.WindowsError.errno" => "POSIX exception code", + "builtins.WindowsError.filename" => "exception filename", + "builtins.WindowsError.filename2" => "second exception filename", + "builtins.WindowsError.strerror" => "exception strerror", + "builtins.WindowsError.winerror" => "Win32 exception code", + "builtins.WindowsError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.ZeroDivisionError" => "Second argument to a division or modulo operation was zero.", + "builtins.ZeroDivisionError.__delattr__" => "Implement delattr(self, name).", + "builtins.ZeroDivisionError.__eq__" => "Return self==value.", + "builtins.ZeroDivisionError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ZeroDivisionError.__ge__" => "Return self>=value.", + "builtins.ZeroDivisionError.__getattribute__" => "Return getattr(self, name).", + "builtins.ZeroDivisionError.__getstate__" => "Helper for pickle.", + "builtins.ZeroDivisionError.__gt__" => "Return self>value.", + "builtins.ZeroDivisionError.__hash__" => "Return hash(self).", + "builtins.ZeroDivisionError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ZeroDivisionError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ZeroDivisionError.__le__" => "Return self<=value.", + "builtins.ZeroDivisionError.__lt__" => "Return self<value.", + "builtins.ZeroDivisionError.__ne__" => "Return self!=value.", + "builtins.ZeroDivisionError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ZeroDivisionError.__reduce_ex__" => "Helper for pickle.", + "builtins.ZeroDivisionError.__repr__" => "Return repr(self).", + "builtins.ZeroDivisionError.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ZeroDivisionError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ZeroDivisionError.__str__" => "Return str(self).", + "builtins.ZeroDivisionError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.ZeroDivisionError.add_note" => "Add a note to the exception", + "builtins.ZeroDivisionError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins._IncompleteInputError" => "incomplete input.", + "builtins._IncompleteInputError.__delattr__" => "Implement delattr(self, name).", + "builtins._IncompleteInputError.__eq__" => "Return self==value.", + "builtins._IncompleteInputError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins._IncompleteInputError.__ge__" => "Return self>=value.", + "builtins._IncompleteInputError.__getattribute__" => "Return getattr(self, name).", + "builtins._IncompleteInputError.__getstate__" => "Helper for pickle.", + "builtins._IncompleteInputError.__gt__" => "Return self>value.", + "builtins._IncompleteInputError.__hash__" => "Return hash(self).", + "builtins._IncompleteInputError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins._IncompleteInputError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins._IncompleteInputError.__le__" => "Return self<=value.", + "builtins._IncompleteInputError.__lt__" => "Return self<value.", + "builtins._IncompleteInputError.__ne__" => "Return self!=value.", + "builtins._IncompleteInputError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins._IncompleteInputError.__reduce_ex__" => "Helper for pickle.", + "builtins._IncompleteInputError.__repr__" => "Return repr(self).", + "builtins._IncompleteInputError.__setattr__" => "Implement setattr(self, name, value).", + "builtins._IncompleteInputError.__sizeof__" => "Size of object in memory, in bytes.", + "builtins._IncompleteInputError.__str__" => "Return str(self).", + "builtins._IncompleteInputError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins._IncompleteInputError._metadata" => "exception private metadata", + "builtins._IncompleteInputError.add_note" => "Add a note to the exception", + "builtins._IncompleteInputError.end_lineno" => "exception end lineno", + "builtins._IncompleteInputError.end_offset" => "exception end offset", + "builtins._IncompleteInputError.filename" => "exception filename", + "builtins._IncompleteInputError.lineno" => "exception lineno", + "builtins._IncompleteInputError.msg" => "exception msg", + "builtins._IncompleteInputError.offset" => "exception offset", + "builtins._IncompleteInputError.print_file_and_line" => "exception print_file_and_line", + "builtins._IncompleteInputError.text" => "exception text", + "builtins._IncompleteInputError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "builtins.__build_class__" => "__build_class__(func, name, /, *bases, [metaclass], **kwds) -> class\n\nInternal helper function used by the class statement.", + "builtins.__import__" => "Import a module.\n\nBecause this function is meant for use by the Python\ninterpreter and not for general use, it is better to use\nimportlib.import_module() to programmatically import a module.\n\nThe globals argument is only used to determine the context;\nthey are not modified. The locals argument is unused. The fromlist\nshould be a list of names to emulate ``from name import ...``, or an\nempty list to emulate ``import name``.\nWhen importing a module from a package, note that __import__('A.B', ...)\nreturns package A when fromlist is empty, but its submodule B when\nfromlist is not empty. The level argument is used to determine whether to\nperform absolute or relative imports: 0 is absolute, while a positive number\nis the number of parent directories to search relative to the current module.", + "builtins.abs" => "Return the absolute value of the argument.", + "builtins.aiter" => "Return an AsyncIterator for an AsyncIterable object.", + "builtins.all" => "Return True if bool(x) is True for all values x in the iterable.\n\nIf the iterable is empty, return True.", + "builtins.anext" => "Return the next item from the async iterator.\n\nIf default is given and the async iterator is exhausted,\nit is returned instead of raising StopAsyncIteration.", + "builtins.any" => "Return True if bool(x) is True for any x in the iterable.\n\nIf the iterable is empty, return False.", + "builtins.ascii" => "Return an ASCII-only representation of an object.\n\nAs repr(), return a string containing a printable representation of an\nobject, but escape the non-ASCII characters in the string returned by\nrepr() using \\\\x, \\\\u or \\\\U escapes. This generates a string similar\nto that returned by repr() in Python 2.", + "builtins.async_generator.__aiter__" => "Return an awaitable, that resolves in asynchronous iterator.", + "builtins.async_generator.__anext__" => "Return a value or raise StopAsyncIteration.", + "builtins.async_generator.__class_getitem__" => "See PEP 585", + "builtins.async_generator.__del__" => "Called when the instance is about to be destroyed.", + "builtins.async_generator.__delattr__" => "Implement delattr(self, name).", + "builtins.async_generator.__eq__" => "Return self==value.", + "builtins.async_generator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.async_generator.__ge__" => "Return self>=value.", + "builtins.async_generator.__getattribute__" => "Return getattr(self, name).", + "builtins.async_generator.__getstate__" => "Helper for pickle.", + "builtins.async_generator.__gt__" => "Return self>value.", + "builtins.async_generator.__hash__" => "Return hash(self).", + "builtins.async_generator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.async_generator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.async_generator.__le__" => "Return self<=value.", + "builtins.async_generator.__lt__" => "Return self<value.", + "builtins.async_generator.__ne__" => "Return self!=value.", + "builtins.async_generator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.async_generator.__reduce__" => "Helper for pickle.", + "builtins.async_generator.__reduce_ex__" => "Helper for pickle.", + "builtins.async_generator.__repr__" => "Return repr(self).", + "builtins.async_generator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.async_generator.__sizeof__" => "gen.__sizeof__() -> size of gen in memory, in bytes", + "builtins.async_generator.__str__" => "Return str(self).", + "builtins.async_generator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.async_generator.aclose" => "aclose() -> raise GeneratorExit inside generator.", + "builtins.async_generator.ag_await" => "object being awaited on, or None", + "builtins.async_generator.asend" => "asend(v) -> send 'v' in generator.", + "builtins.async_generator.athrow" => "athrow(value)\nathrow(type[,value[,tb]])\n\nraise exception in generator.\nthe (type, val, tb) signature is deprecated, \nand may be removed in a future version of Python.", + "builtins.bin" => "Return the binary representation of an integer.\n\n >>> bin(2796202)\n '0b1010101010101010101010'", + "builtins.bool" => "Returns True when the argument is true, False otherwise.\nThe builtins True and False are the only two instances of the class bool.\nThe class bool is a subclass of the class int, and cannot be subclassed.", + "builtins.bool.__abs__" => "abs(self)", + "builtins.bool.__add__" => "Return self+value.", + "builtins.bool.__and__" => "Return self&value.", + "builtins.bool.__bool__" => "True if self else False", + "builtins.bool.__ceil__" => "Ceiling of an Integral returns itself.", + "builtins.bool.__delattr__" => "Implement delattr(self, name).", + "builtins.bool.__divmod__" => "Return divmod(self, value).", + "builtins.bool.__eq__" => "Return self==value.", + "builtins.bool.__float__" => "float(self)", + "builtins.bool.__floor__" => "Flooring an Integral returns itself.", + "builtins.bool.__floordiv__" => "Return self//value.", + "builtins.bool.__format__" => "Convert to a string according to format_spec.", + "builtins.bool.__ge__" => "Return self>=value.", + "builtins.bool.__getattribute__" => "Return getattr(self, name).", + "builtins.bool.__getstate__" => "Helper for pickle.", + "builtins.bool.__gt__" => "Return self>value.", + "builtins.bool.__hash__" => "Return hash(self).", + "builtins.bool.__index__" => "Return self converted to an integer, if self is suitable for use as an index into a list.", + "builtins.bool.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.bool.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.bool.__int__" => "int(self)", + "builtins.bool.__invert__" => "~self", + "builtins.bool.__le__" => "Return self<=value.", + "builtins.bool.__lshift__" => "Return self<<value.", + "builtins.bool.__lt__" => "Return self<value.", + "builtins.bool.__mod__" => "Return self%value.", + "builtins.bool.__mul__" => "Return self*value.", + "builtins.bool.__ne__" => "Return self!=value.", + "builtins.bool.__neg__" => "-self", + "builtins.bool.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.bool.__or__" => "Return self|value.", + "builtins.bool.__pos__" => "+self", + "builtins.bool.__pow__" => "Return pow(self, value, mod).", + "builtins.bool.__radd__" => "Return value+self.", + "builtins.bool.__rand__" => "Return value&self.", + "builtins.bool.__rdivmod__" => "Return divmod(value, self).", + "builtins.bool.__reduce__" => "Helper for pickle.", + "builtins.bool.__reduce_ex__" => "Helper for pickle.", + "builtins.bool.__repr__" => "Return repr(self).", + "builtins.bool.__rfloordiv__" => "Return value//self.", + "builtins.bool.__rlshift__" => "Return value<<self.", + "builtins.bool.__rmod__" => "Return value%self.", + "builtins.bool.__rmul__" => "Return value*self.", + "builtins.bool.__ror__" => "Return value|self.", + "builtins.bool.__round__" => "Rounding an Integral returns itself.\n\nRounding with an ndigits argument also returns an integer.", + "builtins.bool.__rpow__" => "Return pow(value, self, mod).", + "builtins.bool.__rrshift__" => "Return value>>self.", + "builtins.bool.__rshift__" => "Return self>>value.", + "builtins.bool.__rsub__" => "Return value-self.", + "builtins.bool.__rtruediv__" => "Return value/self.", + "builtins.bool.__rxor__" => "Return value^self.", + "builtins.bool.__setattr__" => "Implement setattr(self, name, value).", + "builtins.bool.__sizeof__" => "Returns size in memory, in bytes.", + "builtins.bool.__str__" => "Return str(self).", + "builtins.bool.__sub__" => "Return self-value.", + "builtins.bool.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.bool.__truediv__" => "Return self/value.", + "builtins.bool.__trunc__" => "Truncating an Integral returns itself.", + "builtins.bool.__xor__" => "Return self^value.", + "builtins.bool.as_integer_ratio" => "Return a pair of integers, whose ratio is equal to the original int.\n\nThe ratio is in lowest terms and has a positive denominator.\n\n>>> (10).as_integer_ratio()\n(10, 1)\n>>> (-10).as_integer_ratio()\n(-10, 1)\n>>> (0).as_integer_ratio()\n(0, 1)", + "builtins.bool.bit_count" => "Number of ones in the binary representation of the absolute value of self.\n\nAlso known as the population count.\n\n>>> bin(13)\n'0b1101'\n>>> (13).bit_count()\n3", + "builtins.bool.bit_length" => "Number of bits necessary to represent self in binary.\n\n>>> bin(37)\n'0b100101'\n>>> (37).bit_length()\n6", + "builtins.bool.conjugate" => "Returns self, the complex conjugate of any int.", + "builtins.bool.denominator" => "the denominator of a rational number in lowest terms", + "builtins.bool.from_bytes" => "Return the integer represented by the given array of bytes.\n\n bytes\n Holds the array of bytes to convert. The argument must either\n support the buffer protocol or be an iterable object producing bytes.\n Bytes and bytearray are examples of built-in objects that support the\n buffer protocol.\n byteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\n signed\n Indicates whether two's complement is used to represent the integer.", + "builtins.bool.imag" => "the imaginary part of a complex number", + "builtins.bool.is_integer" => "Returns True. Exists for duck type compatibility with float.is_integer.", + "builtins.bool.numerator" => "the numerator of a rational number in lowest terms", + "builtins.bool.real" => "the real part of a complex number", + "builtins.bool.to_bytes" => "Return an array of bytes representing an integer.\n\n length\n Length of bytes object to use. An OverflowError is raised if the\n integer is not representable with the given number of bytes. Default\n is length 1.\n byteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\n signed\n Determines whether two's complement is used to represent the integer.\n If signed is False and a negative integer is given, an OverflowError\n is raised.", + "builtins.breakpoint" => "Call sys.breakpointhook(*args, **kws). sys.breakpointhook() must accept\nwhatever arguments are passed.\n\nBy default, this drops you into the pdb debugger.", + "builtins.builtin_function_or_method.__call__" => "Call self as a function.", + "builtins.builtin_function_or_method.__delattr__" => "Implement delattr(self, name).", + "builtins.builtin_function_or_method.__eq__" => "Return self==value.", + "builtins.builtin_function_or_method.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.builtin_function_or_method.__ge__" => "Return self>=value.", + "builtins.builtin_function_or_method.__getattribute__" => "Return getattr(self, name).", + "builtins.builtin_function_or_method.__getstate__" => "Helper for pickle.", + "builtins.builtin_function_or_method.__gt__" => "Return self>value.", + "builtins.builtin_function_or_method.__hash__" => "Return hash(self).", + "builtins.builtin_function_or_method.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.builtin_function_or_method.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.builtin_function_or_method.__le__" => "Return self<=value.", + "builtins.builtin_function_or_method.__lt__" => "Return self<value.", + "builtins.builtin_function_or_method.__ne__" => "Return self!=value.", + "builtins.builtin_function_or_method.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.builtin_function_or_method.__reduce_ex__" => "Helper for pickle.", + "builtins.builtin_function_or_method.__repr__" => "Return repr(self).", + "builtins.builtin_function_or_method.__setattr__" => "Implement setattr(self, name, value).", + "builtins.builtin_function_or_method.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.builtin_function_or_method.__str__" => "Return str(self).", + "builtins.builtin_function_or_method.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.bytearray" => "bytearray(iterable_of_ints) -> bytearray\nbytearray(string, encoding[, errors]) -> bytearray\nbytearray(bytes_or_buffer) -> mutable copy of bytes_or_buffer\nbytearray(int) -> bytes array of size given by the parameter initialized with null bytes\nbytearray() -> empty bytes array\n\nConstruct a mutable bytearray object from:\n - an iterable yielding integers in range(256)\n - a text string encoded using the specified encoding\n - a bytes or a buffer object\n - any object implementing the buffer API.\n - an integer", + "builtins.bytearray.__add__" => "Return self+value.", + "builtins.bytearray.__alloc__" => "B.__alloc__() -> int\n\nReturn the number of bytes actually allocated.", + "builtins.bytearray.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "builtins.bytearray.__contains__" => "Return bool(key in self).", + "builtins.bytearray.__delattr__" => "Implement delattr(self, name).", + "builtins.bytearray.__delitem__" => "Delete self[key].", + "builtins.bytearray.__eq__" => "Return self==value.", + "builtins.bytearray.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.bytearray.__ge__" => "Return self>=value.", + "builtins.bytearray.__getattribute__" => "Return getattr(self, name).", + "builtins.bytearray.__getitem__" => "Return self[key].", + "builtins.bytearray.__getstate__" => "Helper for pickle.", + "builtins.bytearray.__gt__" => "Return self>value.", + "builtins.bytearray.__iadd__" => "Implement self+=value.", + "builtins.bytearray.__imul__" => "Implement self*=value.", + "builtins.bytearray.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.bytearray.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.bytearray.__iter__" => "Implement iter(self).", + "builtins.bytearray.__le__" => "Return self<=value.", + "builtins.bytearray.__len__" => "Return len(self).", + "builtins.bytearray.__lt__" => "Return self<value.", + "builtins.bytearray.__mod__" => "Return self%value.", + "builtins.bytearray.__mul__" => "Return self*value.", + "builtins.bytearray.__ne__" => "Return self!=value.", + "builtins.bytearray.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.bytearray.__reduce__" => "Return state information for pickling.", + "builtins.bytearray.__reduce_ex__" => "Return state information for pickling.", + "builtins.bytearray.__release_buffer__" => "Release the buffer object that exposes the underlying memory of the object.", + "builtins.bytearray.__repr__" => "Return repr(self).", + "builtins.bytearray.__rmod__" => "Return value%self.", + "builtins.bytearray.__rmul__" => "Return value*self.", + "builtins.bytearray.__setattr__" => "Implement setattr(self, name, value).", + "builtins.bytearray.__setitem__" => "Set self[key] to value.", + "builtins.bytearray.__sizeof__" => "Returns the size of the bytearray object in memory, in bytes.", + "builtins.bytearray.__str__" => "Return str(self).", + "builtins.bytearray.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.bytearray.append" => "Append a single item to the end of the bytearray.\n\n item\n The item to be appended.", + "builtins.bytearray.capitalize" => "B.capitalize() -> copy of B\n\nReturn a copy of B with only its first character capitalized (ASCII)\nand the rest lower-cased.", + "builtins.bytearray.center" => "Return a centered string of length width.\n\nPadding is done using the specified fill character.", + "builtins.bytearray.clear" => "Remove all items from the bytearray.", + "builtins.bytearray.copy" => "Return a copy of B.", + "builtins.bytearray.count" => "Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.", + "builtins.bytearray.decode" => "Decode the bytearray using the codec registered for encoding.\n\n encoding\n The encoding with which to decode the bytearray.\n errors\n The error handling scheme to use for the handling of decoding errors.\n The default is 'strict' meaning that decoding errors raise a\n UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n as well as any other name registered with codecs.register_error that\n can handle UnicodeDecodeErrors.", + "builtins.bytearray.endswith" => "Return True if the bytearray ends with the specified suffix, False otherwise.\n\n suffix\n A bytes or a tuple of bytes to try.\n start\n Optional start position. Default: start of the bytearray.\n end\n Optional stop position. Default: end of the bytearray.", + "builtins.bytearray.expandtabs" => "Return a copy where all tab characters are expanded using spaces.\n\nIf tabsize is not given, a tab size of 8 characters is assumed.", + "builtins.bytearray.extend" => "Append all the items from the iterator or sequence to the end of the bytearray.\n\n iterable_of_ints\n The iterable of items to append.", + "builtins.bytearray.find" => "Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nReturn -1 on failure.", + "builtins.bytearray.fromhex" => "Create a bytearray object from a string of hexadecimal numbers.\n\nSpaces between two numbers are accepted.\nExample: bytearray.fromhex('B9 01EF') -> bytearray(b'\\\\xb9\\\\x01\\\\xef')", + "builtins.bytearray.hex" => "Create a string of hexadecimal numbers from a bytearray object.\n\n sep\n An optional single character or byte to separate hex bytes.\n bytes_per_sep\n How many bytes between separators. Positive values count from the\n right, negative values count from the left.\n\nExample:\n>>> value = bytearray([0xb9, 0x01, 0xef])\n>>> value.hex()\n'b901ef'\n>>> value.hex(':')\n'b9:01:ef'\n>>> value.hex(':', 2)\n'b9:01ef'\n>>> value.hex(':', -2)\n'b901:ef'", + "builtins.bytearray.index" => "Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nRaise ValueError if the subsection is not found.", + "builtins.bytearray.insert" => "Insert a single item into the bytearray before the given index.\n\n index\n The index where the value is to be inserted.\n item\n The item to be inserted.", + "builtins.bytearray.isalnum" => "B.isalnum() -> bool\n\nReturn True if all characters in B are alphanumeric\nand there is at least one character in B, False otherwise.", + "builtins.bytearray.isalpha" => "B.isalpha() -> bool\n\nReturn True if all characters in B are alphabetic\nand there is at least one character in B, False otherwise.", + "builtins.bytearray.isascii" => "B.isascii() -> bool\n\nReturn True if B is empty or all characters in B are ASCII,\nFalse otherwise.", + "builtins.bytearray.isdigit" => "B.isdigit() -> bool\n\nReturn True if all characters in B are digits\nand there is at least one character in B, False otherwise.", + "builtins.bytearray.islower" => "B.islower() -> bool\n\nReturn True if all cased characters in B are lowercase and there is\nat least one cased character in B, False otherwise.", + "builtins.bytearray.isspace" => "B.isspace() -> bool\n\nReturn True if all characters in B are whitespace\nand there is at least one character in B, False otherwise.", + "builtins.bytearray.istitle" => "B.istitle() -> bool\n\nReturn True if B is a titlecased string and there is at least one\ncharacter in B, i.e. uppercase characters may only follow uncased\ncharacters and lowercase characters only cased ones. Return False\notherwise.", + "builtins.bytearray.isupper" => "B.isupper() -> bool\n\nReturn True if all cased characters in B are uppercase and there is\nat least one cased character in B, False otherwise.", + "builtins.bytearray.join" => "Concatenate any number of bytes/bytearray objects.\n\nThe bytearray whose method is called is inserted in between each pair.\n\nThe result is returned as a new bytearray object.", + "builtins.bytearray.ljust" => "Return a left-justified string of length width.\n\nPadding is done using the specified fill character.", + "builtins.bytearray.lower" => "B.lower() -> copy of B\n\nReturn a copy of B with all ASCII characters converted to lowercase.", + "builtins.bytearray.lstrip" => "Strip leading bytes contained in the argument.\n\nIf the argument is omitted or None, strip leading ASCII whitespace.", + "builtins.bytearray.maketrans" => "Return a translation table usable for the bytes or bytearray translate method.\n\nThe returned table will be one where each byte in frm is mapped to the byte at\nthe same position in to.\n\nThe bytes objects frm and to must be of the same length.", + "builtins.bytearray.partition" => "Partition the bytearray into three parts using the given separator.\n\nThis will search for the separator sep in the bytearray. If the separator is\nfound, returns a 3-tuple containing the part before the separator, the\nseparator itself, and the part after it as new bytearray objects.\n\nIf the separator is not found, returns a 3-tuple containing the copy of the\noriginal bytearray object and two empty bytearray objects.", + "builtins.bytearray.pop" => "Remove and return a single item from B.\n\n index\n The index from where to remove the item.\n -1 (the default value) means remove the last item.\n\nIf no index argument is given, will pop the last item.", + "builtins.bytearray.remove" => "Remove the first occurrence of a value in the bytearray.\n\n value\n The value to remove.", + "builtins.bytearray.removeprefix" => "Return a bytearray with the given prefix string removed if present.\n\nIf the bytearray starts with the prefix string, return\nbytearray[len(prefix):]. Otherwise, return a copy of the original\nbytearray.", + "builtins.bytearray.removesuffix" => "Return a bytearray with the given suffix string removed if present.\n\nIf the bytearray ends with the suffix string and that suffix is not\nempty, return bytearray[:-len(suffix)]. Otherwise, return a copy of\nthe original bytearray.", + "builtins.bytearray.replace" => "Return a copy with all occurrences of substring old replaced by new.\n\n count\n Maximum number of occurrences to replace.\n -1 (the default value) means replace all occurrences.\n\nIf the optional argument count is given, only the first count occurrences are\nreplaced.", + "builtins.bytearray.resize" => "Resize the internal buffer of bytearray to len.\n\n size\n New size to resize to.", + "builtins.bytearray.reverse" => "Reverse the order of the values in B in place.", + "builtins.bytearray.rfind" => "Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nReturn -1 on failure.", + "builtins.bytearray.rindex" => "Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nRaise ValueError if the subsection is not found.", + "builtins.bytearray.rjust" => "Return a right-justified string of length width.\n\nPadding is done using the specified fill character.", + "builtins.bytearray.rpartition" => "Partition the bytearray into three parts using the given separator.\n\nThis will search for the separator sep in the bytearray, starting at the end.\nIf the separator is found, returns a 3-tuple containing the part before the\nseparator, the separator itself, and the part after it as new bytearray\nobjects.\n\nIf the separator is not found, returns a 3-tuple containing two empty bytearray\nobjects and the copy of the original bytearray object.", + "builtins.bytearray.rsplit" => "Return a list of the sections in the bytearray, using sep as the delimiter.\n\n sep\n The delimiter according which to split the bytearray.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\n maxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.\n\nSplitting is done starting at the end of the bytearray and working to the front.", + "builtins.bytearray.rstrip" => "Strip trailing bytes contained in the argument.\n\nIf the argument is omitted or None, strip trailing ASCII whitespace.", + "builtins.bytearray.split" => "Return a list of the sections in the bytearray, using sep as the delimiter.\n\n sep\n The delimiter according which to split the bytearray.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\n maxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.", + "builtins.bytearray.splitlines" => "Return a list of the lines in the bytearray, breaking at line boundaries.\n\nLine breaks are not included in the resulting list unless keepends is given and\ntrue.", + "builtins.bytearray.startswith" => "Return True if the bytearray starts with the specified prefix, False otherwise.\n\n prefix\n A bytes or a tuple of bytes to try.\n start\n Optional start position. Default: start of the bytearray.\n end\n Optional stop position. Default: end of the bytearray.", + "builtins.bytearray.strip" => "Strip leading and trailing bytes contained in the argument.\n\nIf the argument is omitted or None, strip leading and trailing ASCII whitespace.", + "builtins.bytearray.swapcase" => "B.swapcase() -> copy of B\n\nReturn a copy of B with uppercase ASCII characters converted\nto lowercase ASCII and vice versa.", + "builtins.bytearray.title" => "B.title() -> copy of B\n\nReturn a titlecased version of B, i.e. ASCII words start with uppercase\ncharacters, all remaining cased characters have lowercase.", + "builtins.bytearray.translate" => "Return a copy with each character mapped by the given translation table.\n\n table\n Translation table, which must be a bytes object of length 256.\n\nAll characters occurring in the optional argument delete are removed.\nThe remaining characters are mapped through the given translation table.", + "builtins.bytearray.upper" => "B.upper() -> copy of B\n\nReturn a copy of B with all ASCII characters converted to uppercase.", + "builtins.bytearray.zfill" => "Pad a numeric string with zeros on the left, to fill a field of the given width.\n\nThe original string is never truncated.", + "builtins.bytearray_iterator.__delattr__" => "Implement delattr(self, name).", + "builtins.bytearray_iterator.__eq__" => "Return self==value.", + "builtins.bytearray_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.bytearray_iterator.__ge__" => "Return self>=value.", + "builtins.bytearray_iterator.__getattribute__" => "Return getattr(self, name).", + "builtins.bytearray_iterator.__getstate__" => "Helper for pickle.", + "builtins.bytearray_iterator.__gt__" => "Return self>value.", + "builtins.bytearray_iterator.__hash__" => "Return hash(self).", + "builtins.bytearray_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.bytearray_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.bytearray_iterator.__iter__" => "Implement iter(self).", + "builtins.bytearray_iterator.__le__" => "Return self<=value.", + "builtins.bytearray_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.bytearray_iterator.__lt__" => "Return self<value.", + "builtins.bytearray_iterator.__ne__" => "Return self!=value.", + "builtins.bytearray_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.bytearray_iterator.__next__" => "Implement next(self).", + "builtins.bytearray_iterator.__reduce__" => "Return state information for pickling.", + "builtins.bytearray_iterator.__reduce_ex__" => "Helper for pickle.", + "builtins.bytearray_iterator.__repr__" => "Return repr(self).", + "builtins.bytearray_iterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.bytearray_iterator.__setstate__" => "Set state information for unpickling.", + "builtins.bytearray_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.bytearray_iterator.__str__" => "Return str(self).", + "builtins.bytearray_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.bytes" => "bytes(iterable_of_ints) -> bytes\nbytes(string, encoding[, errors]) -> bytes\nbytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\nbytes(int) -> bytes object of size given by the parameter initialized with null bytes\nbytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n - an iterable yielding integers in range(256)\n - a text string encoded using the specified encoding\n - any object implementing the buffer API.\n - an integer", + "builtins.bytes.__add__" => "Return self+value.", + "builtins.bytes.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "builtins.bytes.__bytes__" => "Convert this value to exact type bytes.", + "builtins.bytes.__contains__" => "Return bool(key in self).", + "builtins.bytes.__delattr__" => "Implement delattr(self, name).", + "builtins.bytes.__eq__" => "Return self==value.", + "builtins.bytes.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.bytes.__ge__" => "Return self>=value.", + "builtins.bytes.__getattribute__" => "Return getattr(self, name).", + "builtins.bytes.__getitem__" => "Return self[key].", + "builtins.bytes.__getstate__" => "Helper for pickle.", + "builtins.bytes.__gt__" => "Return self>value.", + "builtins.bytes.__hash__" => "Return hash(self).", + "builtins.bytes.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.bytes.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.bytes.__iter__" => "Implement iter(self).", + "builtins.bytes.__le__" => "Return self<=value.", + "builtins.bytes.__len__" => "Return len(self).", + "builtins.bytes.__lt__" => "Return self<value.", + "builtins.bytes.__mod__" => "Return self%value.", + "builtins.bytes.__mul__" => "Return self*value.", + "builtins.bytes.__ne__" => "Return self!=value.", + "builtins.bytes.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.bytes.__reduce__" => "Helper for pickle.", + "builtins.bytes.__reduce_ex__" => "Helper for pickle.", + "builtins.bytes.__repr__" => "Return repr(self).", + "builtins.bytes.__rmod__" => "Return value%self.", + "builtins.bytes.__rmul__" => "Return value*self.", + "builtins.bytes.__setattr__" => "Implement setattr(self, name, value).", + "builtins.bytes.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.bytes.__str__" => "Return str(self).", + "builtins.bytes.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.bytes.capitalize" => "B.capitalize() -> copy of B\n\nReturn a copy of B with only its first character capitalized (ASCII)\nand the rest lower-cased.", + "builtins.bytes.center" => "Return a centered string of length width.\n\nPadding is done using the specified fill character.", + "builtins.bytes.count" => "Return the number of non-overlapping occurrences of subsection 'sub' in bytes B[start:end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.", + "builtins.bytes.decode" => "Decode the bytes using the codec registered for encoding.\n\n encoding\n The encoding with which to decode the bytes.\n errors\n The error handling scheme to use for the handling of decoding errors.\n The default is 'strict' meaning that decoding errors raise a\n UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n as well as any other name registered with codecs.register_error that\n can handle UnicodeDecodeErrors.", + "builtins.bytes.endswith" => "Return True if the bytes ends with the specified suffix, False otherwise.\n\n suffix\n A bytes or a tuple of bytes to try.\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.", + "builtins.bytes.expandtabs" => "Return a copy where all tab characters are expanded using spaces.\n\nIf tabsize is not given, a tab size of 8 characters is assumed.", + "builtins.bytes.find" => "Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nReturn -1 on failure.", + "builtins.bytes.fromhex" => "Create a bytes object from a string of hexadecimal numbers.\n\nSpaces between two numbers are accepted.\nExample: bytes.fromhex('B9 01EF') -> b'\\\\xb9\\\\x01\\\\xef'.", + "builtins.bytes.hex" => "Create a string of hexadecimal numbers from a bytes object.\n\n sep\n An optional single character or byte to separate hex bytes.\n bytes_per_sep\n How many bytes between separators. Positive values count from the\n right, negative values count from the left.\n\nExample:\n>>> value = b'\\xb9\\x01\\xef'\n>>> value.hex()\n'b901ef'\n>>> value.hex(':')\n'b9:01:ef'\n>>> value.hex(':', 2)\n'b9:01ef'\n>>> value.hex(':', -2)\n'b901:ef'", + "builtins.bytes.index" => "Return the lowest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nRaise ValueError if the subsection is not found.", + "builtins.bytes.isalnum" => "B.isalnum() -> bool\n\nReturn True if all characters in B are alphanumeric\nand there is at least one character in B, False otherwise.", + "builtins.bytes.isalpha" => "B.isalpha() -> bool\n\nReturn True if all characters in B are alphabetic\nand there is at least one character in B, False otherwise.", + "builtins.bytes.isascii" => "B.isascii() -> bool\n\nReturn True if B is empty or all characters in B are ASCII,\nFalse otherwise.", + "builtins.bytes.isdigit" => "B.isdigit() -> bool\n\nReturn True if all characters in B are digits\nand there is at least one character in B, False otherwise.", + "builtins.bytes.islower" => "B.islower() -> bool\n\nReturn True if all cased characters in B are lowercase and there is\nat least one cased character in B, False otherwise.", + "builtins.bytes.isspace" => "B.isspace() -> bool\n\nReturn True if all characters in B are whitespace\nand there is at least one character in B, False otherwise.", + "builtins.bytes.istitle" => "B.istitle() -> bool\n\nReturn True if B is a titlecased string and there is at least one\ncharacter in B, i.e. uppercase characters may only follow uncased\ncharacters and lowercase characters only cased ones. Return False\notherwise.", + "builtins.bytes.isupper" => "B.isupper() -> bool\n\nReturn True if all cased characters in B are uppercase and there is\nat least one cased character in B, False otherwise.", + "builtins.bytes.join" => "Concatenate any number of bytes objects.\n\nThe bytes whose method is called is inserted in between each pair.\n\nThe result is returned as a new bytes object.\n\nExample: b'.'.join([b'ab', b'pq', b'rs']) -> b'ab.pq.rs'.", + "builtins.bytes.ljust" => "Return a left-justified string of length width.\n\nPadding is done using the specified fill character.", + "builtins.bytes.lower" => "B.lower() -> copy of B\n\nReturn a copy of B with all ASCII characters converted to lowercase.", + "builtins.bytes.lstrip" => "Strip leading bytes contained in the argument.\n\nIf the argument is omitted or None, strip leading ASCII whitespace.", + "builtins.bytes.maketrans" => "Return a translation table usable for the bytes or bytearray translate method.\n\nThe returned table will be one where each byte in frm is mapped to the byte at\nthe same position in to.\n\nThe bytes objects frm and to must be of the same length.", + "builtins.bytes.partition" => "Partition the bytes into three parts using the given separator.\n\nThis will search for the separator sep in the bytes. If the separator is found,\nreturns a 3-tuple containing the part before the separator, the separator\nitself, and the part after it.\n\nIf the separator is not found, returns a 3-tuple containing the original bytes\nobject and two empty bytes objects.", + "builtins.bytes.removeprefix" => "Return a bytes object with the given prefix string removed if present.\n\nIf the bytes starts with the prefix string, return bytes[len(prefix):].\nOtherwise, return a copy of the original bytes.", + "builtins.bytes.removesuffix" => "Return a bytes object with the given suffix string removed if present.\n\nIf the bytes ends with the suffix string and that suffix is not empty,\nreturn bytes[:-len(prefix)]. Otherwise, return a copy of the original\nbytes.", + "builtins.bytes.replace" => "Return a copy with all occurrences of substring old replaced by new.\n\n count\n Maximum number of occurrences to replace.\n -1 (the default value) means replace all occurrences.\n\nIf the optional argument count is given, only the first count occurrences are\nreplaced.", + "builtins.bytes.rfind" => "Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nReturn -1 on failure.", + "builtins.bytes.rindex" => "Return the highest index in B where subsection 'sub' is found, such that 'sub' is contained within B[start,end].\n\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.\n\nRaise ValueError if the subsection is not found.", + "builtins.bytes.rjust" => "Return a right-justified string of length width.\n\nPadding is done using the specified fill character.", + "builtins.bytes.rpartition" => "Partition the bytes into three parts using the given separator.\n\nThis will search for the separator sep in the bytes, starting at the end. If\nthe separator is found, returns a 3-tuple containing the part before the\nseparator, the separator itself, and the part after it.\n\nIf the separator is not found, returns a 3-tuple containing two empty bytes\nobjects and the original bytes object.", + "builtins.bytes.rsplit" => "Return a list of the sections in the bytes, using sep as the delimiter.\n\n sep\n The delimiter according which to split the bytes.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\n maxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.\n\nSplitting is done starting at the end of the bytes and working to the front.", + "builtins.bytes.rstrip" => "Strip trailing bytes contained in the argument.\n\nIf the argument is omitted or None, strip trailing ASCII whitespace.", + "builtins.bytes.split" => "Return a list of the sections in the bytes, using sep as the delimiter.\n\n sep\n The delimiter according which to split the bytes.\n None (the default value) means split on ASCII whitespace characters\n (space, tab, return, newline, formfeed, vertical tab).\n maxsplit\n Maximum number of splits to do.\n -1 (the default value) means no limit.", + "builtins.bytes.splitlines" => "Return a list of the lines in the bytes, breaking at line boundaries.\n\nLine breaks are not included in the resulting list unless keepends is given and\ntrue.", + "builtins.bytes.startswith" => "Return True if the bytes starts with the specified prefix, False otherwise.\n\n prefix\n A bytes or a tuple of bytes to try.\n start\n Optional start position. Default: start of the bytes.\n end\n Optional stop position. Default: end of the bytes.", + "builtins.bytes.strip" => "Strip leading and trailing bytes contained in the argument.\n\nIf the argument is omitted or None, strip leading and trailing ASCII whitespace.", + "builtins.bytes.swapcase" => "B.swapcase() -> copy of B\n\nReturn a copy of B with uppercase ASCII characters converted\nto lowercase ASCII and vice versa.", + "builtins.bytes.title" => "B.title() -> copy of B\n\nReturn a titlecased version of B, i.e. ASCII words start with uppercase\ncharacters, all remaining cased characters have lowercase.", + "builtins.bytes.translate" => "Return a copy with each character mapped by the given translation table.\n\n table\n Translation table, which must be a bytes object of length 256.\n\nAll characters occurring in the optional argument delete are removed.\nThe remaining characters are mapped through the given translation table.", + "builtins.bytes.upper" => "B.upper() -> copy of B\n\nReturn a copy of B with all ASCII characters converted to uppercase.", + "builtins.bytes.zfill" => "Pad a numeric string with zeros on the left, to fill a field of the given width.\n\nThe original string is never truncated.", + "builtins.bytes_iterator.__delattr__" => "Implement delattr(self, name).", + "builtins.bytes_iterator.__eq__" => "Return self==value.", + "builtins.bytes_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.bytes_iterator.__ge__" => "Return self>=value.", + "builtins.bytes_iterator.__getattribute__" => "Return getattr(self, name).", + "builtins.bytes_iterator.__getstate__" => "Helper for pickle.", + "builtins.bytes_iterator.__gt__" => "Return self>value.", + "builtins.bytes_iterator.__hash__" => "Return hash(self).", + "builtins.bytes_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.bytes_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.bytes_iterator.__iter__" => "Implement iter(self).", + "builtins.bytes_iterator.__le__" => "Return self<=value.", + "builtins.bytes_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.bytes_iterator.__lt__" => "Return self<value.", + "builtins.bytes_iterator.__ne__" => "Return self!=value.", + "builtins.bytes_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.bytes_iterator.__next__" => "Implement next(self).", + "builtins.bytes_iterator.__reduce__" => "Return state information for pickling.", + "builtins.bytes_iterator.__reduce_ex__" => "Helper for pickle.", + "builtins.bytes_iterator.__repr__" => "Return repr(self).", + "builtins.bytes_iterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.bytes_iterator.__setstate__" => "Set state information for unpickling.", + "builtins.bytes_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.bytes_iterator.__str__" => "Return str(self).", + "builtins.bytes_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.callable" => "Return whether the object is callable (i.e., some kind of function).\n\nNote that classes are callable, as are instances of classes with a\n__call__() method.", + "builtins.cell" => "Create a new cell object.\n\n contents\n the contents of the cell. If not specified, the cell will be empty,\n and \n further attempts to access its cell_contents attribute will\n raise a ValueError.", + "builtins.cell.__delattr__" => "Implement delattr(self, name).", + "builtins.cell.__eq__" => "Return self==value.", + "builtins.cell.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.cell.__ge__" => "Return self>=value.", + "builtins.cell.__getattribute__" => "Return getattr(self, name).", + "builtins.cell.__getstate__" => "Helper for pickle.", + "builtins.cell.__gt__" => "Return self>value.", + "builtins.cell.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.cell.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.cell.__le__" => "Return self<=value.", + "builtins.cell.__lt__" => "Return self<value.", + "builtins.cell.__ne__" => "Return self!=value.", + "builtins.cell.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.cell.__reduce__" => "Helper for pickle.", + "builtins.cell.__reduce_ex__" => "Helper for pickle.", + "builtins.cell.__repr__" => "Return repr(self).", + "builtins.cell.__setattr__" => "Implement setattr(self, name, value).", + "builtins.cell.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.cell.__str__" => "Return str(self).", + "builtins.cell.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.chr" => "Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff.", + "builtins.classmethod" => "Convert a function to be a class method.\n\nA class method receives the class as implicit first argument,\njust like an instance method receives the instance.\nTo declare a class method, use this idiom:\n\n class C:\n @classmethod\n def f(cls, arg1, arg2, argN):\n ...\n\nIt can be called either on the class (e.g. C.f()) or on an instance\n(e.g. C().f()). The instance is ignored except for its class.\nIf a class method is called for a derived class, the derived class\nobject is passed as the implied first argument.\n\nClass methods are different than C++ or Java static methods.\nIf you want those, see the staticmethod builtin.", + "builtins.classmethod.__delattr__" => "Implement delattr(self, name).", + "builtins.classmethod.__eq__" => "Return self==value.", + "builtins.classmethod.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.classmethod.__ge__" => "Return self>=value.", + "builtins.classmethod.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.classmethod.__getattribute__" => "Return getattr(self, name).", + "builtins.classmethod.__getstate__" => "Helper for pickle.", + "builtins.classmethod.__gt__" => "Return self>value.", + "builtins.classmethod.__hash__" => "Return hash(self).", + "builtins.classmethod.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.classmethod.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.classmethod.__le__" => "Return self<=value.", + "builtins.classmethod.__lt__" => "Return self<value.", + "builtins.classmethod.__ne__" => "Return self!=value.", + "builtins.classmethod.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.classmethod.__reduce__" => "Helper for pickle.", + "builtins.classmethod.__reduce_ex__" => "Helper for pickle.", + "builtins.classmethod.__repr__" => "Return repr(self).", + "builtins.classmethod.__setattr__" => "Implement setattr(self, name, value).", + "builtins.classmethod.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.classmethod.__str__" => "Return str(self).", + "builtins.classmethod.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.classmethod_descriptor.__call__" => "Call self as a function.", + "builtins.classmethod_descriptor.__delattr__" => "Implement delattr(self, name).", + "builtins.classmethod_descriptor.__eq__" => "Return self==value.", + "builtins.classmethod_descriptor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.classmethod_descriptor.__ge__" => "Return self>=value.", + "builtins.classmethod_descriptor.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.classmethod_descriptor.__getattribute__" => "Return getattr(self, name).", + "builtins.classmethod_descriptor.__getstate__" => "Helper for pickle.", + "builtins.classmethod_descriptor.__gt__" => "Return self>value.", + "builtins.classmethod_descriptor.__hash__" => "Return hash(self).", + "builtins.classmethod_descriptor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.classmethod_descriptor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.classmethod_descriptor.__le__" => "Return self<=value.", + "builtins.classmethod_descriptor.__lt__" => "Return self<value.", + "builtins.classmethod_descriptor.__ne__" => "Return self!=value.", + "builtins.classmethod_descriptor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.classmethod_descriptor.__reduce__" => "Helper for pickle.", + "builtins.classmethod_descriptor.__reduce_ex__" => "Helper for pickle.", + "builtins.classmethod_descriptor.__repr__" => "Return repr(self).", + "builtins.classmethod_descriptor.__setattr__" => "Implement setattr(self, name, value).", + "builtins.classmethod_descriptor.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.classmethod_descriptor.__str__" => "Return str(self).", + "builtins.classmethod_descriptor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.code" => "Create a code object. Not for the faint of heart.", + "builtins.code.__delattr__" => "Implement delattr(self, name).", + "builtins.code.__eq__" => "Return self==value.", + "builtins.code.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.code.__ge__" => "Return self>=value.", + "builtins.code.__getattribute__" => "Return getattr(self, name).", + "builtins.code.__getstate__" => "Helper for pickle.", + "builtins.code.__gt__" => "Return self>value.", + "builtins.code.__hash__" => "Return hash(self).", + "builtins.code.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.code.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.code.__le__" => "Return self<=value.", + "builtins.code.__lt__" => "Return self<value.", + "builtins.code.__ne__" => "Return self!=value.", + "builtins.code.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.code.__reduce__" => "Helper for pickle.", + "builtins.code.__reduce_ex__" => "Helper for pickle.", + "builtins.code.__replace__" => "The same as replace().", + "builtins.code.__repr__" => "Return repr(self).", + "builtins.code.__setattr__" => "Implement setattr(self, name, value).", + "builtins.code.__str__" => "Return str(self).", + "builtins.code.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.code._varname_from_oparg" => "(internal-only) Return the local variable name for the given oparg.\n\nWARNING: this method is for internal use only and may change or go away.", + "builtins.code.replace" => "Return a copy of the code object with new values for the specified fields.", + "builtins.compile" => "Compile source into a code object that can be executed by exec() or eval().\n\nThe source code may represent a Python module, statement or expression.\nThe filename will be used for run-time error messages.\nThe mode must be 'exec' to compile a module, 'single' to compile a\nsingle (interactive) statement, or 'eval' to compile an expression.\nThe flags argument, if present, controls which future statements influence\nthe compilation of the code.\nThe dont_inherit argument, if true, stops the compilation inheriting\nthe effects of any future statements in effect in the code calling\ncompile; if absent or false these statements do influence the compilation,\nin addition to any features explicitly specified.", + "builtins.complex" => "Create a complex number from a string or numbers.\n\nIf a string is given, parse it as a complex number.\nIf a single number is given, convert it to a complex number.\nIf the 'real' or 'imag' arguments are given, create a complex number\nwith the specified real and imaginary components.", + "builtins.complex.__abs__" => "abs(self)", + "builtins.complex.__add__" => "Return self+value.", + "builtins.complex.__bool__" => "True if self else False", + "builtins.complex.__complex__" => "Convert this value to exact type complex.", + "builtins.complex.__delattr__" => "Implement delattr(self, name).", + "builtins.complex.__eq__" => "Return self==value.", + "builtins.complex.__format__" => "Convert to a string according to format_spec.", + "builtins.complex.__ge__" => "Return self>=value.", + "builtins.complex.__getattribute__" => "Return getattr(self, name).", + "builtins.complex.__getstate__" => "Helper for pickle.", + "builtins.complex.__gt__" => "Return self>value.", + "builtins.complex.__hash__" => "Return hash(self).", + "builtins.complex.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.complex.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.complex.__le__" => "Return self<=value.", + "builtins.complex.__lt__" => "Return self<value.", + "builtins.complex.__mul__" => "Return self*value.", + "builtins.complex.__ne__" => "Return self!=value.", + "builtins.complex.__neg__" => "-self", + "builtins.complex.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.complex.__pos__" => "+self", + "builtins.complex.__pow__" => "Return pow(self, value, mod).", + "builtins.complex.__radd__" => "Return value+self.", + "builtins.complex.__reduce__" => "Helper for pickle.", + "builtins.complex.__reduce_ex__" => "Helper for pickle.", + "builtins.complex.__repr__" => "Return repr(self).", + "builtins.complex.__rmul__" => "Return value*self.", + "builtins.complex.__rpow__" => "Return pow(value, self, mod).", + "builtins.complex.__rsub__" => "Return value-self.", + "builtins.complex.__rtruediv__" => "Return value/self.", + "builtins.complex.__setattr__" => "Implement setattr(self, name, value).", + "builtins.complex.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.complex.__str__" => "Return str(self).", + "builtins.complex.__sub__" => "Return self-value.", + "builtins.complex.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.complex.__truediv__" => "Return self/value.", + "builtins.complex.conjugate" => "Return the complex conjugate of its argument. (3-4j).conjugate() == 3+4j.", + "builtins.complex.from_number" => "Convert number to a complex floating-point number.", + "builtins.complex.imag" => "the imaginary part of a complex number", + "builtins.complex.real" => "the real part of a complex number", + "builtins.coroutine.__await__" => "Return an iterator to be used in await expression.", + "builtins.coroutine.__class_getitem__" => "See PEP 585", + "builtins.coroutine.__del__" => "Called when the instance is about to be destroyed.", + "builtins.coroutine.__delattr__" => "Implement delattr(self, name).", + "builtins.coroutine.__eq__" => "Return self==value.", + "builtins.coroutine.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.coroutine.__ge__" => "Return self>=value.", + "builtins.coroutine.__getattribute__" => "Return getattr(self, name).", + "builtins.coroutine.__getstate__" => "Helper for pickle.", + "builtins.coroutine.__gt__" => "Return self>value.", + "builtins.coroutine.__hash__" => "Return hash(self).", + "builtins.coroutine.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.coroutine.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.coroutine.__le__" => "Return self<=value.", + "builtins.coroutine.__lt__" => "Return self<value.", + "builtins.coroutine.__ne__" => "Return self!=value.", + "builtins.coroutine.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.coroutine.__reduce__" => "Helper for pickle.", + "builtins.coroutine.__reduce_ex__" => "Helper for pickle.", + "builtins.coroutine.__repr__" => "Return repr(self).", + "builtins.coroutine.__setattr__" => "Implement setattr(self, name, value).", + "builtins.coroutine.__sizeof__" => "gen.__sizeof__() -> size of gen in memory, in bytes", + "builtins.coroutine.__str__" => "Return str(self).", + "builtins.coroutine.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.coroutine.close" => "close() -> raise GeneratorExit inside coroutine.", + "builtins.coroutine.cr_await" => "object being awaited on, or None", + "builtins.coroutine.send" => "send(arg) -> send 'arg' into coroutine,\nreturn next iterated value or raise StopIteration.", + "builtins.coroutine.throw" => "throw(value)\nthrow(type[,value[,traceback]])\n\nRaise exception in coroutine, return next iterated value or raise\nStopIteration.\nthe (type, val, tb) signature is deprecated, \nand may be removed in a future version of Python.", + "builtins.delattr" => "Deletes the named attribute from the given object.\n\ndelattr(x, 'y') is equivalent to ``del x.y``", + "builtins.dict" => "dict() -> new empty dictionary\ndict(mapping) -> new dictionary initialized from a mapping object's\n (key, value) pairs\ndict(iterable) -> new dictionary initialized as if via:\n d = {}\n for k, v in iterable:\n d[k] = v\ndict(**kwargs) -> new dictionary initialized with the name=value pairs\n in the keyword argument list. For example: dict(one=1, two=2)", + "builtins.dict.__class_getitem__" => "See PEP 585", + "builtins.dict.__contains__" => "True if the dictionary has the specified key, else False.", + "builtins.dict.__delattr__" => "Implement delattr(self, name).", + "builtins.dict.__delitem__" => "Delete self[key].", + "builtins.dict.__eq__" => "Return self==value.", + "builtins.dict.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.dict.__ge__" => "Return self>=value.", + "builtins.dict.__getattribute__" => "Return getattr(self, name).", + "builtins.dict.__getitem__" => "Return self[key].", + "builtins.dict.__getstate__" => "Helper for pickle.", + "builtins.dict.__gt__" => "Return self>value.", + "builtins.dict.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.dict.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.dict.__ior__" => "Return self|=value.", + "builtins.dict.__iter__" => "Implement iter(self).", + "builtins.dict.__le__" => "Return self<=value.", + "builtins.dict.__len__" => "Return len(self).", + "builtins.dict.__lt__" => "Return self<value.", + "builtins.dict.__ne__" => "Return self!=value.", + "builtins.dict.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.dict.__or__" => "Return self|value.", + "builtins.dict.__reduce__" => "Helper for pickle.", + "builtins.dict.__reduce_ex__" => "Helper for pickle.", + "builtins.dict.__repr__" => "Return repr(self).", + "builtins.dict.__reversed__" => "Return a reverse iterator over the dict keys.", + "builtins.dict.__ror__" => "Return value|self.", + "builtins.dict.__setattr__" => "Implement setattr(self, name, value).", + "builtins.dict.__setitem__" => "Set self[key] to value.", + "builtins.dict.__sizeof__" => "Return the size of the dict in memory, in bytes.", + "builtins.dict.__str__" => "Return str(self).", + "builtins.dict.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.dict.clear" => "Remove all items from the dict.", + "builtins.dict.copy" => "Return a shallow copy of the dict.", + "builtins.dict.fromkeys" => "Create a new dictionary with keys from iterable and values set to value.", + "builtins.dict.get" => "Return the value for key if key is in the dictionary, else default.", + "builtins.dict.items" => "Return a set-like object providing a view on the dict's items.", + "builtins.dict.keys" => "Return a set-like object providing a view on the dict's keys.", + "builtins.dict.pop" => "D.pop(k[,d]) -> v, remove specified key and return the corresponding value.\n\nIf the key is not found, return the default if given; otherwise,\nraise a KeyError.", + "builtins.dict.popitem" => "Remove and return a (key, value) pair as a 2-tuple.\n\nPairs are returned in LIFO (last-in, first-out) order.\nRaises KeyError if the dict is empty.", + "builtins.dict.setdefault" => "Insert key with a value of default if key is not in the dictionary.\n\nReturn the value for key if key is in the dictionary, else default.", + "builtins.dict.update" => "D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\nIf E is present and has a .keys() method, then does: for k in E.keys(): D[k] = E[k]\nIf E is present and lacks a .keys() method, then does: for k, v in E: D[k] = v\nIn either case, this is followed by: for k in F: D[k] = F[k]", + "builtins.dict.values" => "Return an object providing a view on the dict's values.", + "builtins.dict_itemiterator.__delattr__" => "Implement delattr(self, name).", + "builtins.dict_itemiterator.__eq__" => "Return self==value.", + "builtins.dict_itemiterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.dict_itemiterator.__ge__" => "Return self>=value.", + "builtins.dict_itemiterator.__getattribute__" => "Return getattr(self, name).", + "builtins.dict_itemiterator.__getstate__" => "Helper for pickle.", + "builtins.dict_itemiterator.__gt__" => "Return self>value.", + "builtins.dict_itemiterator.__hash__" => "Return hash(self).", + "builtins.dict_itemiterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.dict_itemiterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.dict_itemiterator.__iter__" => "Implement iter(self).", + "builtins.dict_itemiterator.__le__" => "Return self<=value.", + "builtins.dict_itemiterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.dict_itemiterator.__lt__" => "Return self<value.", + "builtins.dict_itemiterator.__ne__" => "Return self!=value.", + "builtins.dict_itemiterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.dict_itemiterator.__next__" => "Implement next(self).", + "builtins.dict_itemiterator.__reduce__" => "Return state information for pickling.", + "builtins.dict_itemiterator.__reduce_ex__" => "Helper for pickle.", + "builtins.dict_itemiterator.__repr__" => "Return repr(self).", + "builtins.dict_itemiterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.dict_itemiterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.dict_itemiterator.__str__" => "Return str(self).", + "builtins.dict_itemiterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.dict_items.__and__" => "Return self&value.", + "builtins.dict_items.__contains__" => "Return bool(key in self).", + "builtins.dict_items.__delattr__" => "Implement delattr(self, name).", + "builtins.dict_items.__eq__" => "Return self==value.", + "builtins.dict_items.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.dict_items.__ge__" => "Return self>=value.", + "builtins.dict_items.__getattribute__" => "Return getattr(self, name).", + "builtins.dict_items.__getstate__" => "Helper for pickle.", + "builtins.dict_items.__gt__" => "Return self>value.", + "builtins.dict_items.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.dict_items.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.dict_items.__iter__" => "Implement iter(self).", + "builtins.dict_items.__le__" => "Return self<=value.", + "builtins.dict_items.__len__" => "Return len(self).", + "builtins.dict_items.__lt__" => "Return self<value.", + "builtins.dict_items.__ne__" => "Return self!=value.", + "builtins.dict_items.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.dict_items.__or__" => "Return self|value.", + "builtins.dict_items.__rand__" => "Return value&self.", + "builtins.dict_items.__reduce__" => "Helper for pickle.", + "builtins.dict_items.__reduce_ex__" => "Helper for pickle.", + "builtins.dict_items.__repr__" => "Return repr(self).", + "builtins.dict_items.__reversed__" => "Return a reverse iterator over the dict items.", + "builtins.dict_items.__ror__" => "Return value|self.", + "builtins.dict_items.__rsub__" => "Return value-self.", + "builtins.dict_items.__rxor__" => "Return value^self.", + "builtins.dict_items.__setattr__" => "Implement setattr(self, name, value).", + "builtins.dict_items.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.dict_items.__str__" => "Return str(self).", + "builtins.dict_items.__sub__" => "Return self-value.", + "builtins.dict_items.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.dict_items.__xor__" => "Return self^value.", + "builtins.dict_items.isdisjoint" => "Return True if the view and the given iterable have a null intersection.", + "builtins.dict_items.mapping" => "dictionary that this view refers to", + "builtins.dict_keyiterator.__delattr__" => "Implement delattr(self, name).", + "builtins.dict_keyiterator.__eq__" => "Return self==value.", + "builtins.dict_keyiterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.dict_keyiterator.__ge__" => "Return self>=value.", + "builtins.dict_keyiterator.__getattribute__" => "Return getattr(self, name).", + "builtins.dict_keyiterator.__getstate__" => "Helper for pickle.", + "builtins.dict_keyiterator.__gt__" => "Return self>value.", + "builtins.dict_keyiterator.__hash__" => "Return hash(self).", + "builtins.dict_keyiterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.dict_keyiterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.dict_keyiterator.__iter__" => "Implement iter(self).", + "builtins.dict_keyiterator.__le__" => "Return self<=value.", + "builtins.dict_keyiterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.dict_keyiterator.__lt__" => "Return self<value.", + "builtins.dict_keyiterator.__ne__" => "Return self!=value.", + "builtins.dict_keyiterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.dict_keyiterator.__next__" => "Implement next(self).", + "builtins.dict_keyiterator.__reduce__" => "Return state information for pickling.", + "builtins.dict_keyiterator.__reduce_ex__" => "Helper for pickle.", + "builtins.dict_keyiterator.__repr__" => "Return repr(self).", + "builtins.dict_keyiterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.dict_keyiterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.dict_keyiterator.__str__" => "Return str(self).", + "builtins.dict_keyiterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.dict_valueiterator.__delattr__" => "Implement delattr(self, name).", + "builtins.dict_valueiterator.__eq__" => "Return self==value.", + "builtins.dict_valueiterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.dict_valueiterator.__ge__" => "Return self>=value.", + "builtins.dict_valueiterator.__getattribute__" => "Return getattr(self, name).", + "builtins.dict_valueiterator.__getstate__" => "Helper for pickle.", + "builtins.dict_valueiterator.__gt__" => "Return self>value.", + "builtins.dict_valueiterator.__hash__" => "Return hash(self).", + "builtins.dict_valueiterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.dict_valueiterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.dict_valueiterator.__iter__" => "Implement iter(self).", + "builtins.dict_valueiterator.__le__" => "Return self<=value.", + "builtins.dict_valueiterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.dict_valueiterator.__lt__" => "Return self<value.", + "builtins.dict_valueiterator.__ne__" => "Return self!=value.", + "builtins.dict_valueiterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.dict_valueiterator.__next__" => "Implement next(self).", + "builtins.dict_valueiterator.__reduce__" => "Return state information for pickling.", + "builtins.dict_valueiterator.__reduce_ex__" => "Helper for pickle.", + "builtins.dict_valueiterator.__repr__" => "Return repr(self).", + "builtins.dict_valueiterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.dict_valueiterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.dict_valueiterator.__str__" => "Return str(self).", + "builtins.dict_valueiterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.dict_values.__delattr__" => "Implement delattr(self, name).", + "builtins.dict_values.__eq__" => "Return self==value.", + "builtins.dict_values.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.dict_values.__ge__" => "Return self>=value.", + "builtins.dict_values.__getattribute__" => "Return getattr(self, name).", + "builtins.dict_values.__getstate__" => "Helper for pickle.", + "builtins.dict_values.__gt__" => "Return self>value.", + "builtins.dict_values.__hash__" => "Return hash(self).", + "builtins.dict_values.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.dict_values.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.dict_values.__iter__" => "Implement iter(self).", + "builtins.dict_values.__le__" => "Return self<=value.", + "builtins.dict_values.__len__" => "Return len(self).", + "builtins.dict_values.__lt__" => "Return self<value.", + "builtins.dict_values.__ne__" => "Return self!=value.", + "builtins.dict_values.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.dict_values.__reduce__" => "Helper for pickle.", + "builtins.dict_values.__reduce_ex__" => "Helper for pickle.", + "builtins.dict_values.__repr__" => "Return repr(self).", + "builtins.dict_values.__reversed__" => "Return a reverse iterator over the dict values.", + "builtins.dict_values.__setattr__" => "Implement setattr(self, name, value).", + "builtins.dict_values.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.dict_values.__str__" => "Return str(self).", + "builtins.dict_values.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.dict_values.mapping" => "dictionary that this view refers to", + "builtins.dir" => "dir([object]) -> list of strings\n\nIf called without an argument, return the names in the current scope.\nElse, return an alphabetized list of names comprising (some of) the attributes\nof the given object, and of attributes reachable from it.\nIf the object supplies a method named __dir__, it will be used; otherwise\nthe default dir() logic is used and returns:\n for a module object: the module's attributes.\n for a class object: its attributes, and recursively the attributes\n of its bases.\n for any other object: its attributes, its class's attributes, and\n recursively the attributes of its class's base classes.", + "builtins.divmod" => "Return the tuple (x//y, x%y). Invariant: div*y + mod == x.", + "builtins.ellipsis" => "The type of the Ellipsis singleton.", + "builtins.ellipsis.__delattr__" => "Implement delattr(self, name).", + "builtins.ellipsis.__eq__" => "Return self==value.", + "builtins.ellipsis.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.ellipsis.__ge__" => "Return self>=value.", + "builtins.ellipsis.__getattribute__" => "Return getattr(self, name).", + "builtins.ellipsis.__getstate__" => "Helper for pickle.", + "builtins.ellipsis.__gt__" => "Return self>value.", + "builtins.ellipsis.__hash__" => "Return hash(self).", + "builtins.ellipsis.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.ellipsis.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.ellipsis.__le__" => "Return self<=value.", + "builtins.ellipsis.__lt__" => "Return self<value.", + "builtins.ellipsis.__ne__" => "Return self!=value.", + "builtins.ellipsis.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.ellipsis.__reduce_ex__" => "Helper for pickle.", + "builtins.ellipsis.__repr__" => "Return repr(self).", + "builtins.ellipsis.__setattr__" => "Implement setattr(self, name, value).", + "builtins.ellipsis.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.ellipsis.__str__" => "Return str(self).", + "builtins.ellipsis.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.enumerate" => "Return an enumerate object.\n\n iterable\n an object supporting iteration\n\nThe enumerate object yields pairs containing a count (from start, which\ndefaults to zero) and a value yielded by the iterable argument.\n\nenumerate is useful for obtaining an indexed list:\n (0, seq[0]), (1, seq[1]), (2, seq[2]), ...", + "builtins.enumerate.__class_getitem__" => "See PEP 585", + "builtins.enumerate.__delattr__" => "Implement delattr(self, name).", + "builtins.enumerate.__eq__" => "Return self==value.", + "builtins.enumerate.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.enumerate.__ge__" => "Return self>=value.", + "builtins.enumerate.__getattribute__" => "Return getattr(self, name).", + "builtins.enumerate.__getstate__" => "Helper for pickle.", + "builtins.enumerate.__gt__" => "Return self>value.", + "builtins.enumerate.__hash__" => "Return hash(self).", + "builtins.enumerate.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.enumerate.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.enumerate.__iter__" => "Implement iter(self).", + "builtins.enumerate.__le__" => "Return self<=value.", + "builtins.enumerate.__lt__" => "Return self<value.", + "builtins.enumerate.__ne__" => "Return self!=value.", + "builtins.enumerate.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.enumerate.__next__" => "Implement next(self).", + "builtins.enumerate.__reduce__" => "Return state information for pickling.", + "builtins.enumerate.__reduce_ex__" => "Helper for pickle.", + "builtins.enumerate.__repr__" => "Return repr(self).", + "builtins.enumerate.__setattr__" => "Implement setattr(self, name, value).", + "builtins.enumerate.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.enumerate.__str__" => "Return str(self).", + "builtins.enumerate.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.eval" => "Evaluate the given source in the context of globals and locals.\n\nThe source may be a string representing a Python expression\nor a code object as returned by compile().\nThe globals must be a dictionary and locals can be any mapping,\ndefaulting to the current globals and locals.\nIf only globals is given, locals defaults to it.", + "builtins.exec" => "Execute the given source in the context of globals and locals.\n\nThe source may be a string representing one or more Python statements\nor a code object as returned by compile().\nThe globals must be a dictionary and locals can be any mapping,\ndefaulting to the current globals and locals.\nIf only globals is given, locals defaults to it.\nThe closure must be a tuple of cellvars, and can only be used\nwhen source is a code object requiring exactly that many cellvars.", + "builtins.filter" => "Return an iterator yielding those items of iterable for which function(item)\nis true. If function is None, return the items that are true.", + "builtins.filter.__delattr__" => "Implement delattr(self, name).", + "builtins.filter.__eq__" => "Return self==value.", + "builtins.filter.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.filter.__ge__" => "Return self>=value.", + "builtins.filter.__getattribute__" => "Return getattr(self, name).", + "builtins.filter.__getstate__" => "Helper for pickle.", + "builtins.filter.__gt__" => "Return self>value.", + "builtins.filter.__hash__" => "Return hash(self).", + "builtins.filter.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.filter.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.filter.__iter__" => "Implement iter(self).", + "builtins.filter.__le__" => "Return self<=value.", + "builtins.filter.__lt__" => "Return self<value.", + "builtins.filter.__ne__" => "Return self!=value.", + "builtins.filter.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.filter.__next__" => "Implement next(self).", + "builtins.filter.__reduce__" => "Return state information for pickling.", + "builtins.filter.__reduce_ex__" => "Helper for pickle.", + "builtins.filter.__repr__" => "Return repr(self).", + "builtins.filter.__setattr__" => "Implement setattr(self, name, value).", + "builtins.filter.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.filter.__str__" => "Return str(self).", + "builtins.filter.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.float" => "Convert a string or number to a floating-point number, if possible.", + "builtins.float.__abs__" => "abs(self)", + "builtins.float.__add__" => "Return self+value.", + "builtins.float.__bool__" => "True if self else False", + "builtins.float.__ceil__" => "Return the ceiling as an Integral.", + "builtins.float.__delattr__" => "Implement delattr(self, name).", + "builtins.float.__divmod__" => "Return divmod(self, value).", + "builtins.float.__eq__" => "Return self==value.", + "builtins.float.__float__" => "float(self)", + "builtins.float.__floor__" => "Return the floor as an Integral.", + "builtins.float.__floordiv__" => "Return self//value.", + "builtins.float.__format__" => "Formats the float according to format_spec.", + "builtins.float.__ge__" => "Return self>=value.", + "builtins.float.__getattribute__" => "Return getattr(self, name).", + "builtins.float.__getformat__" => "You probably don't want to use this function.\n\n typestr\n Must be 'double' or 'float'.\n\nIt exists mainly to be used in Python's test suite.\n\nThis function returns whichever of 'unknown', 'IEEE, big-endian' or 'IEEE,\nlittle-endian' best describes the format of floating-point numbers used by the\nC type named by typestr.", + "builtins.float.__getstate__" => "Helper for pickle.", + "builtins.float.__gt__" => "Return self>value.", + "builtins.float.__hash__" => "Return hash(self).", + "builtins.float.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.float.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.float.__int__" => "int(self)", + "builtins.float.__le__" => "Return self<=value.", + "builtins.float.__lt__" => "Return self<value.", + "builtins.float.__mod__" => "Return self%value.", + "builtins.float.__mul__" => "Return self*value.", + "builtins.float.__ne__" => "Return self!=value.", + "builtins.float.__neg__" => "-self", + "builtins.float.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.float.__pos__" => "+self", + "builtins.float.__pow__" => "Return pow(self, value, mod).", + "builtins.float.__radd__" => "Return value+self.", + "builtins.float.__rdivmod__" => "Return divmod(value, self).", + "builtins.float.__reduce__" => "Helper for pickle.", + "builtins.float.__reduce_ex__" => "Helper for pickle.", + "builtins.float.__repr__" => "Return repr(self).", + "builtins.float.__rfloordiv__" => "Return value//self.", + "builtins.float.__rmod__" => "Return value%self.", + "builtins.float.__rmul__" => "Return value*self.", + "builtins.float.__round__" => "Return the Integral closest to x, rounding half toward even.\n\nWhen an argument is passed, work like built-in round(x, ndigits).", + "builtins.float.__rpow__" => "Return pow(value, self, mod).", + "builtins.float.__rsub__" => "Return value-self.", + "builtins.float.__rtruediv__" => "Return value/self.", + "builtins.float.__setattr__" => "Implement setattr(self, name, value).", + "builtins.float.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.float.__str__" => "Return str(self).", + "builtins.float.__sub__" => "Return self-value.", + "builtins.float.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.float.__truediv__" => "Return self/value.", + "builtins.float.__trunc__" => "Return the Integral closest to x between 0 and x.", + "builtins.float.as_integer_ratio" => "Return a pair of integers, whose ratio is exactly equal to the original float.\n\nThe ratio is in lowest terms and has a positive denominator. Raise\nOverflowError on infinities and a ValueError on NaNs.\n\n>>> (10.0).as_integer_ratio()\n(10, 1)\n>>> (0.0).as_integer_ratio()\n(0, 1)\n>>> (-.25).as_integer_ratio()\n(-1, 4)", + "builtins.float.conjugate" => "Return self, the complex conjugate of any float.", + "builtins.float.from_number" => "Convert real number to a floating-point number.", + "builtins.float.fromhex" => "Create a floating-point number from a hexadecimal string.\n\n>>> float.fromhex('0x1.ffffp10')\n2047.984375\n>>> float.fromhex('-0x1p-1074')\n-5e-324", + "builtins.float.hex" => "Return a hexadecimal representation of a floating-point number.\n\n>>> (-0.1).hex()\n'-0x1.999999999999ap-4'\n>>> 3.14159.hex()\n'0x1.921f9f01b866ep+1'", + "builtins.float.imag" => "the imaginary part of a complex number", + "builtins.float.is_integer" => "Return True if the float is an integer.", + "builtins.float.real" => "the real part of a complex number", + "builtins.format" => "Return type(value).__format__(value, format_spec)\n\nMany built-in types implement format_spec according to the\nFormat Specification Mini-language. See help('FORMATTING').\n\nIf type(value) does not supply a method named __format__\nand format_spec is empty, then str(value) is returned.\nSee also help('SPECIALMETHODS').", + "builtins.frame.__delattr__" => "Implement delattr(self, name).", + "builtins.frame.__eq__" => "Return self==value.", + "builtins.frame.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.frame.__ge__" => "Return self>=value.", + "builtins.frame.__getattribute__" => "Return getattr(self, name).", + "builtins.frame.__getstate__" => "Helper for pickle.", + "builtins.frame.__gt__" => "Return self>value.", + "builtins.frame.__hash__" => "Return hash(self).", + "builtins.frame.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.frame.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.frame.__le__" => "Return self<=value.", + "builtins.frame.__lt__" => "Return self<value.", + "builtins.frame.__ne__" => "Return self!=value.", + "builtins.frame.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.frame.__reduce__" => "Helper for pickle.", + "builtins.frame.__reduce_ex__" => "Helper for pickle.", + "builtins.frame.__repr__" => "Return repr(self).", + "builtins.frame.__setattr__" => "Implement setattr(self, name, value).", + "builtins.frame.__sizeof__" => "Return the size of the frame in memory, in bytes.", + "builtins.frame.__str__" => "Return str(self).", + "builtins.frame.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.frame.clear" => "Clear all references held by the frame.", + "builtins.frame.f_builtins" => "Return the built-in variables in the frame.", + "builtins.frame.f_code" => "Return the code object being executed in this frame.", + "builtins.frame.f_generator" => "Return the generator or coroutine associated with this frame, or None.", + "builtins.frame.f_globals" => "Return the global variables in the frame.", + "builtins.frame.f_lasti" => "Return the index of the last attempted instruction in the frame.", + "builtins.frame.f_lineno" => "Return the current line number in the frame.", + "builtins.frame.f_locals" => "Return the mapping used by the frame to look up local variables.", + "builtins.frame.f_trace" => "Return the trace function for this frame, or None if no trace function is set.", + "builtins.frame.f_trace_opcodes" => "Return True if opcode tracing is enabled, False otherwise.", + "builtins.frozenset" => "Build an immutable unordered collection of unique elements.", + "builtins.frozenset.__and__" => "Return self&value.", + "builtins.frozenset.__class_getitem__" => "See PEP 585", + "builtins.frozenset.__contains__" => "x.__contains__(y) <==> y in x.", + "builtins.frozenset.__delattr__" => "Implement delattr(self, name).", + "builtins.frozenset.__eq__" => "Return self==value.", + "builtins.frozenset.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.frozenset.__ge__" => "Return self>=value.", + "builtins.frozenset.__getattribute__" => "Return getattr(self, name).", + "builtins.frozenset.__getstate__" => "Helper for pickle.", + "builtins.frozenset.__gt__" => "Return self>value.", + "builtins.frozenset.__hash__" => "Return hash(self).", + "builtins.frozenset.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.frozenset.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.frozenset.__iter__" => "Implement iter(self).", + "builtins.frozenset.__le__" => "Return self<=value.", + "builtins.frozenset.__len__" => "Return len(self).", + "builtins.frozenset.__lt__" => "Return self<value.", + "builtins.frozenset.__ne__" => "Return self!=value.", + "builtins.frozenset.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.frozenset.__or__" => "Return self|value.", + "builtins.frozenset.__rand__" => "Return value&self.", + "builtins.frozenset.__reduce__" => "Return state information for pickling.", + "builtins.frozenset.__reduce_ex__" => "Helper for pickle.", + "builtins.frozenset.__repr__" => "Return repr(self).", + "builtins.frozenset.__ror__" => "Return value|self.", + "builtins.frozenset.__rsub__" => "Return value-self.", + "builtins.frozenset.__rxor__" => "Return value^self.", + "builtins.frozenset.__setattr__" => "Implement setattr(self, name, value).", + "builtins.frozenset.__sizeof__" => "S.__sizeof__() -> size of S in memory, in bytes.", + "builtins.frozenset.__str__" => "Return str(self).", + "builtins.frozenset.__sub__" => "Return self-value.", + "builtins.frozenset.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.frozenset.__xor__" => "Return self^value.", + "builtins.frozenset.copy" => "Return a shallow copy of a set.", + "builtins.frozenset.difference" => "Return a new set with elements in the set that are not in the others.", + "builtins.frozenset.intersection" => "Return a new set with elements common to the set and all others.", + "builtins.frozenset.isdisjoint" => "Return True if two sets have a null intersection.", + "builtins.frozenset.issubset" => "Report whether another set contains this set.", + "builtins.frozenset.issuperset" => "Report whether this set contains another set.", + "builtins.frozenset.symmetric_difference" => "Return a new set with elements in either the set or other but not both.", + "builtins.frozenset.union" => "Return a new set with elements from the set and all others.", + "builtins.function" => "Create a function object.\n\n code\n a code object\n globals\n the globals dictionary\n name\n a string that overrides the name from the code object\n argdefs\n a tuple that specifies the default argument values\n closure\n a tuple that supplies the bindings for free variables\n kwdefaults\n a dictionary that specifies the default keyword argument values", + "builtins.function.__annotate__" => "Get the code object for a function.", + "builtins.function.__call__" => "Call self as a function.", + "builtins.function.__delattr__" => "Implement delattr(self, name).", + "builtins.function.__eq__" => "Return self==value.", + "builtins.function.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.function.__ge__" => "Return self>=value.", + "builtins.function.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.function.__getattribute__" => "Return getattr(self, name).", + "builtins.function.__getstate__" => "Helper for pickle.", + "builtins.function.__gt__" => "Return self>value.", + "builtins.function.__hash__" => "Return hash(self).", + "builtins.function.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.function.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.function.__le__" => "Return self<=value.", + "builtins.function.__lt__" => "Return self<value.", + "builtins.function.__ne__" => "Return self!=value.", + "builtins.function.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.function.__reduce__" => "Helper for pickle.", + "builtins.function.__reduce_ex__" => "Helper for pickle.", + "builtins.function.__repr__" => "Return repr(self).", + "builtins.function.__setattr__" => "Implement setattr(self, name, value).", + "builtins.function.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.function.__str__" => "Return str(self).", + "builtins.function.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.function.__type_params__" => "Get the declared type parameters for a function.", + "builtins.generator.__class_getitem__" => "See PEP 585", + "builtins.generator.__del__" => "Called when the instance is about to be destroyed.", + "builtins.generator.__delattr__" => "Implement delattr(self, name).", + "builtins.generator.__eq__" => "Return self==value.", + "builtins.generator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.generator.__ge__" => "Return self>=value.", + "builtins.generator.__getattribute__" => "Return getattr(self, name).", + "builtins.generator.__getstate__" => "Helper for pickle.", + "builtins.generator.__gt__" => "Return self>value.", + "builtins.generator.__hash__" => "Return hash(self).", + "builtins.generator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.generator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.generator.__iter__" => "Implement iter(self).", + "builtins.generator.__le__" => "Return self<=value.", + "builtins.generator.__lt__" => "Return self<value.", + "builtins.generator.__ne__" => "Return self!=value.", + "builtins.generator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.generator.__next__" => "Implement next(self).", + "builtins.generator.__reduce__" => "Helper for pickle.", + "builtins.generator.__reduce_ex__" => "Helper for pickle.", + "builtins.generator.__repr__" => "Return repr(self).", + "builtins.generator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.generator.__sizeof__" => "gen.__sizeof__() -> size of gen in memory, in bytes", + "builtins.generator.__str__" => "Return str(self).", + "builtins.generator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.generator.close" => "close() -> raise GeneratorExit inside generator.", + "builtins.generator.gi_yieldfrom" => "object being iterated by yield from, or None", + "builtins.generator.send" => "send(value) -> send 'value' into generator,\nreturn next yielded value or raise StopIteration.", + "builtins.generator.throw" => "throw(value)\nthrow(type[,value[,tb]])\n\nRaise exception in generator, return next yielded value or raise\nStopIteration.\nthe (type, val, tb) signature is deprecated, \nand may be removed in a future version of Python.", + "builtins.getattr" => "getattr(object, name[, default]) -> value\n\nGet a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\nWhen a default argument is given, it is returned when the attribute doesn't\nexist; without it, an exception is raised in that case.", + "builtins.getset_descriptor.__delattr__" => "Implement delattr(self, name).", + "builtins.getset_descriptor.__delete__" => "Delete an attribute of instance.", + "builtins.getset_descriptor.__eq__" => "Return self==value.", + "builtins.getset_descriptor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.getset_descriptor.__ge__" => "Return self>=value.", + "builtins.getset_descriptor.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.getset_descriptor.__getattribute__" => "Return getattr(self, name).", + "builtins.getset_descriptor.__getstate__" => "Helper for pickle.", + "builtins.getset_descriptor.__gt__" => "Return self>value.", + "builtins.getset_descriptor.__hash__" => "Return hash(self).", + "builtins.getset_descriptor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.getset_descriptor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.getset_descriptor.__le__" => "Return self<=value.", + "builtins.getset_descriptor.__lt__" => "Return self<value.", + "builtins.getset_descriptor.__ne__" => "Return self!=value.", + "builtins.getset_descriptor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.getset_descriptor.__reduce__" => "Helper for pickle.", + "builtins.getset_descriptor.__reduce_ex__" => "Helper for pickle.", + "builtins.getset_descriptor.__repr__" => "Return repr(self).", + "builtins.getset_descriptor.__set__" => "Set an attribute of instance to value.", + "builtins.getset_descriptor.__setattr__" => "Implement setattr(self, name, value).", + "builtins.getset_descriptor.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.getset_descriptor.__str__" => "Return str(self).", + "builtins.getset_descriptor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.globals" => "Return the dictionary containing the current scope's global variables.\n\nNOTE: Updates to this dictionary *will* affect name lookups in the current\nglobal scope and vice-versa.", + "builtins.hasattr" => "Return whether the object has an attribute with the given name.\n\nThis is done by calling getattr(obj, name) and catching AttributeError.", + "builtins.hash" => "Return the hash value for the given object.\n\nTwo objects that compare equal must also have the same hash value, but the\nreverse is not necessarily true.", + "builtins.hex" => "Return the hexadecimal representation of an integer.\n\n >>> hex(12648430)\n '0xc0ffee'", + "builtins.id" => "Return the identity of an object.\n\nThis is guaranteed to be unique among simultaneously existing objects.\n(CPython uses the object's memory address.)", + "builtins.input" => "Read a string from standard input. The trailing newline is stripped.\n\nThe prompt string, if given, is printed to standard output without a\ntrailing newline before reading input.\n\nIf the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError.\nOn *nix systems, readline is used if available.", + "builtins.int" => "int([x]) -> integer\nint(x, base=10) -> integer\n\nConvert a number or string to an integer, or return 0 if no arguments\nare given. If x is a number, return x.__int__(). For floating-point\nnumbers, this truncates towards zero.\n\nIf x is not a number or if base is given, then x must be a string,\nbytes, or bytearray instance representing an integer literal in the\ngiven base. The literal can be preceded by '+' or '-' and be surrounded\nby whitespace. The base defaults to 10. Valid bases are 0 and 2-36.\nBase 0 means to interpret the base from the string as an integer literal.\n>>> int('0b100', base=0)\n4", + "builtins.int.__abs__" => "abs(self)", + "builtins.int.__add__" => "Return self+value.", + "builtins.int.__and__" => "Return self&value.", + "builtins.int.__bool__" => "True if self else False", + "builtins.int.__ceil__" => "Ceiling of an Integral returns itself.", + "builtins.int.__delattr__" => "Implement delattr(self, name).", + "builtins.int.__divmod__" => "Return divmod(self, value).", + "builtins.int.__eq__" => "Return self==value.", + "builtins.int.__float__" => "float(self)", + "builtins.int.__floor__" => "Flooring an Integral returns itself.", + "builtins.int.__floordiv__" => "Return self//value.", + "builtins.int.__format__" => "Convert to a string according to format_spec.", + "builtins.int.__ge__" => "Return self>=value.", + "builtins.int.__getattribute__" => "Return getattr(self, name).", + "builtins.int.__getstate__" => "Helper for pickle.", + "builtins.int.__gt__" => "Return self>value.", + "builtins.int.__hash__" => "Return hash(self).", + "builtins.int.__index__" => "Return self converted to an integer, if self is suitable for use as an index into a list.", + "builtins.int.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.int.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.int.__int__" => "int(self)", + "builtins.int.__invert__" => "~self", + "builtins.int.__le__" => "Return self<=value.", + "builtins.int.__lshift__" => "Return self<<value.", + "builtins.int.__lt__" => "Return self<value.", + "builtins.int.__mod__" => "Return self%value.", + "builtins.int.__mul__" => "Return self*value.", + "builtins.int.__ne__" => "Return self!=value.", + "builtins.int.__neg__" => "-self", + "builtins.int.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.int.__or__" => "Return self|value.", + "builtins.int.__pos__" => "+self", + "builtins.int.__pow__" => "Return pow(self, value, mod).", + "builtins.int.__radd__" => "Return value+self.", + "builtins.int.__rand__" => "Return value&self.", + "builtins.int.__rdivmod__" => "Return divmod(value, self).", + "builtins.int.__reduce__" => "Helper for pickle.", + "builtins.int.__reduce_ex__" => "Helper for pickle.", + "builtins.int.__repr__" => "Return repr(self).", + "builtins.int.__rfloordiv__" => "Return value//self.", + "builtins.int.__rlshift__" => "Return value<<self.", + "builtins.int.__rmod__" => "Return value%self.", + "builtins.int.__rmul__" => "Return value*self.", + "builtins.int.__ror__" => "Return value|self.", + "builtins.int.__round__" => "Rounding an Integral returns itself.\n\nRounding with an ndigits argument also returns an integer.", + "builtins.int.__rpow__" => "Return pow(value, self, mod).", + "builtins.int.__rrshift__" => "Return value>>self.", + "builtins.int.__rshift__" => "Return self>>value.", + "builtins.int.__rsub__" => "Return value-self.", + "builtins.int.__rtruediv__" => "Return value/self.", + "builtins.int.__rxor__" => "Return value^self.", + "builtins.int.__setattr__" => "Implement setattr(self, name, value).", + "builtins.int.__sizeof__" => "Returns size in memory, in bytes.", + "builtins.int.__str__" => "Return str(self).", + "builtins.int.__sub__" => "Return self-value.", + "builtins.int.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.int.__truediv__" => "Return self/value.", + "builtins.int.__trunc__" => "Truncating an Integral returns itself.", + "builtins.int.__xor__" => "Return self^value.", + "builtins.int.as_integer_ratio" => "Return a pair of integers, whose ratio is equal to the original int.\n\nThe ratio is in lowest terms and has a positive denominator.\n\n>>> (10).as_integer_ratio()\n(10, 1)\n>>> (-10).as_integer_ratio()\n(-10, 1)\n>>> (0).as_integer_ratio()\n(0, 1)", + "builtins.int.bit_count" => "Number of ones in the binary representation of the absolute value of self.\n\nAlso known as the population count.\n\n>>> bin(13)\n'0b1101'\n>>> (13).bit_count()\n3", + "builtins.int.bit_length" => "Number of bits necessary to represent self in binary.\n\n>>> bin(37)\n'0b100101'\n>>> (37).bit_length()\n6", + "builtins.int.conjugate" => "Returns self, the complex conjugate of any int.", + "builtins.int.denominator" => "the denominator of a rational number in lowest terms", + "builtins.int.from_bytes" => "Return the integer represented by the given array of bytes.\n\n bytes\n Holds the array of bytes to convert. The argument must either\n support the buffer protocol or be an iterable object producing bytes.\n Bytes and bytearray are examples of built-in objects that support the\n buffer protocol.\n byteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\n signed\n Indicates whether two's complement is used to represent the integer.", + "builtins.int.imag" => "the imaginary part of a complex number", + "builtins.int.is_integer" => "Returns True. Exists for duck type compatibility with float.is_integer.", + "builtins.int.numerator" => "the numerator of a rational number in lowest terms", + "builtins.int.real" => "the real part of a complex number", + "builtins.int.to_bytes" => "Return an array of bytes representing an integer.\n\n length\n Length of bytes object to use. An OverflowError is raised if the\n integer is not representable with the given number of bytes. Default\n is length 1.\n byteorder\n The byte order used to represent the integer. If byteorder is 'big',\n the most significant byte is at the beginning of the byte array. If\n byteorder is 'little', the most significant byte is at the end of the\n byte array. To request the native byte order of the host system, use\n sys.byteorder as the byte order value. Default is to use 'big'.\n signed\n Determines whether two's complement is used to represent the integer.\n If signed is False and a negative integer is given, an OverflowError\n is raised.", + "builtins.isinstance" => "Return whether an object is an instance of a class or of a subclass thereof.\n\nA tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to\ncheck against. This is equivalent to ``isinstance(x, A) or isinstance(x, B)\nor ...`` etc.", + "builtins.issubclass" => "Return whether 'cls' is derived from another class or is the same class.\n\nA tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to\ncheck against. This is equivalent to ``issubclass(x, A) or issubclass(x, B)\nor ...``.", + "builtins.iter" => "iter(iterable) -> iterator\niter(callable, sentinel) -> iterator\n\nGet an iterator from an object. In the first form, the argument must\nsupply its own iterator, or be a sequence.\nIn the second form, the callable is called until it returns the sentinel.", + "builtins.len" => "Return the number of items in a container.", + "builtins.list" => "Built-in mutable sequence.\n\nIf no argument is given, the constructor creates a new empty list.\nThe argument must be an iterable if specified.", + "builtins.list.__add__" => "Return self+value.", + "builtins.list.__class_getitem__" => "See PEP 585", + "builtins.list.__contains__" => "Return bool(key in self).", + "builtins.list.__delattr__" => "Implement delattr(self, name).", + "builtins.list.__delitem__" => "Delete self[key].", + "builtins.list.__eq__" => "Return self==value.", + "builtins.list.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.list.__ge__" => "Return self>=value.", + "builtins.list.__getattribute__" => "Return getattr(self, name).", + "builtins.list.__getitem__" => "Return self[index].", + "builtins.list.__getstate__" => "Helper for pickle.", + "builtins.list.__gt__" => "Return self>value.", + "builtins.list.__iadd__" => "Implement self+=value.", + "builtins.list.__imul__" => "Implement self*=value.", + "builtins.list.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.list.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.list.__iter__" => "Implement iter(self).", + "builtins.list.__le__" => "Return self<=value.", + "builtins.list.__len__" => "Return len(self).", + "builtins.list.__lt__" => "Return self<value.", + "builtins.list.__mul__" => "Return self*value.", + "builtins.list.__ne__" => "Return self!=value.", + "builtins.list.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.list.__reduce__" => "Helper for pickle.", + "builtins.list.__reduce_ex__" => "Helper for pickle.", + "builtins.list.__repr__" => "Return repr(self).", + "builtins.list.__reversed__" => "Return a reverse iterator over the list.", + "builtins.list.__rmul__" => "Return value*self.", + "builtins.list.__setattr__" => "Implement setattr(self, name, value).", + "builtins.list.__setitem__" => "Set self[key] to value.", + "builtins.list.__sizeof__" => "Return the size of the list in memory, in bytes.", + "builtins.list.__str__" => "Return str(self).", + "builtins.list.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.list.append" => "Append object to the end of the list.", + "builtins.list.clear" => "Remove all items from list.", + "builtins.list.copy" => "Return a shallow copy of the list.", + "builtins.list.count" => "Return number of occurrences of value.", + "builtins.list.extend" => "Extend list by appending elements from the iterable.", + "builtins.list.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "builtins.list.insert" => "Insert object before index.", + "builtins.list.pop" => "Remove and return item at index (default last).\n\nRaises IndexError if list is empty or index is out of range.", + "builtins.list.remove" => "Remove first occurrence of value.\n\nRaises ValueError if the value is not present.", + "builtins.list.reverse" => "Reverse *IN PLACE*.", + "builtins.list.sort" => "Sort the list in ascending order and return None.\n\nThe sort is in-place (i.e. the list itself is modified) and stable (i.e. the\norder of two equal elements is maintained).\n\nIf a key function is given, apply it once to each list item and sort them,\nascending or descending, according to their function values.\n\nThe reverse flag can be set to sort in descending order.", + "builtins.list_iterator.__delattr__" => "Implement delattr(self, name).", + "builtins.list_iterator.__eq__" => "Return self==value.", + "builtins.list_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.list_iterator.__ge__" => "Return self>=value.", + "builtins.list_iterator.__getattribute__" => "Return getattr(self, name).", + "builtins.list_iterator.__getstate__" => "Helper for pickle.", + "builtins.list_iterator.__gt__" => "Return self>value.", + "builtins.list_iterator.__hash__" => "Return hash(self).", + "builtins.list_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.list_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.list_iterator.__iter__" => "Implement iter(self).", + "builtins.list_iterator.__le__" => "Return self<=value.", + "builtins.list_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.list_iterator.__lt__" => "Return self<value.", + "builtins.list_iterator.__ne__" => "Return self!=value.", + "builtins.list_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.list_iterator.__next__" => "Implement next(self).", + "builtins.list_iterator.__reduce__" => "Return state information for pickling.", + "builtins.list_iterator.__reduce_ex__" => "Helper for pickle.", + "builtins.list_iterator.__repr__" => "Return repr(self).", + "builtins.list_iterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.list_iterator.__setstate__" => "Set state information for unpickling.", + "builtins.list_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.list_iterator.__str__" => "Return str(self).", + "builtins.list_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.locals" => "Return a dictionary containing the current scope's local variables.\n\nNOTE: Whether or not updates to this dictionary will affect name lookups in\nthe local scope and vice-versa is *implementation dependent* and not\ncovered by any backwards compatibility guarantees.", + "builtins.map" => "Make an iterator that computes the function using arguments from\neach of the iterables. Stops when the shortest iterable is exhausted.\n\nIf strict is true and one of the arguments is exhausted before the others,\nraise a ValueError.", + "builtins.map.__delattr__" => "Implement delattr(self, name).", + "builtins.map.__eq__" => "Return self==value.", + "builtins.map.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.map.__ge__" => "Return self>=value.", + "builtins.map.__getattribute__" => "Return getattr(self, name).", + "builtins.map.__getstate__" => "Helper for pickle.", + "builtins.map.__gt__" => "Return self>value.", + "builtins.map.__hash__" => "Return hash(self).", + "builtins.map.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.map.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.map.__iter__" => "Implement iter(self).", + "builtins.map.__le__" => "Return self<=value.", + "builtins.map.__lt__" => "Return self<value.", + "builtins.map.__ne__" => "Return self!=value.", + "builtins.map.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.map.__next__" => "Implement next(self).", + "builtins.map.__reduce__" => "Return state information for pickling.", + "builtins.map.__reduce_ex__" => "Helper for pickle.", + "builtins.map.__repr__" => "Return repr(self).", + "builtins.map.__setattr__" => "Implement setattr(self, name, value).", + "builtins.map.__setstate__" => "Set state information for unpickling.", + "builtins.map.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.map.__str__" => "Return str(self).", + "builtins.map.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.mappingproxy" => "Read-only proxy of a mapping.", + "builtins.mappingproxy.__class_getitem__" => "See PEP 585", + "builtins.mappingproxy.__contains__" => "Return bool(key in self).", + "builtins.mappingproxy.__delattr__" => "Implement delattr(self, name).", + "builtins.mappingproxy.__eq__" => "Return self==value.", + "builtins.mappingproxy.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.mappingproxy.__ge__" => "Return self>=value.", + "builtins.mappingproxy.__getattribute__" => "Return getattr(self, name).", + "builtins.mappingproxy.__getitem__" => "Return self[key].", + "builtins.mappingproxy.__getstate__" => "Helper for pickle.", + "builtins.mappingproxy.__gt__" => "Return self>value.", + "builtins.mappingproxy.__hash__" => "Return hash(self).", + "builtins.mappingproxy.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.mappingproxy.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.mappingproxy.__ior__" => "Return self|=value.", + "builtins.mappingproxy.__iter__" => "Implement iter(self).", + "builtins.mappingproxy.__le__" => "Return self<=value.", + "builtins.mappingproxy.__len__" => "Return len(self).", + "builtins.mappingproxy.__lt__" => "Return self<value.", + "builtins.mappingproxy.__ne__" => "Return self!=value.", + "builtins.mappingproxy.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.mappingproxy.__or__" => "Return self|value.", + "builtins.mappingproxy.__reduce__" => "Helper for pickle.", + "builtins.mappingproxy.__reduce_ex__" => "Helper for pickle.", + "builtins.mappingproxy.__repr__" => "Return repr(self).", + "builtins.mappingproxy.__reversed__" => "D.__reversed__() -> reverse iterator", + "builtins.mappingproxy.__ror__" => "Return value|self.", + "builtins.mappingproxy.__setattr__" => "Implement setattr(self, name, value).", + "builtins.mappingproxy.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.mappingproxy.__str__" => "Return str(self).", + "builtins.mappingproxy.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.mappingproxy.copy" => "D.copy() -> a shallow copy of D", + "builtins.mappingproxy.get" => "Return the value for key if key is in the mapping, else default.", + "builtins.mappingproxy.items" => "D.items() -> a set-like object providing a view on D's items", + "builtins.mappingproxy.keys" => "D.keys() -> a set-like object providing a view on D's keys", + "builtins.mappingproxy.values" => "D.values() -> an object providing a view on D's values", + "builtins.max" => "max(iterable, *[, default=obj, key=func]) -> value\nmax(arg1, arg2, *args, *[, key=func]) -> value\n\nWith a single iterable argument, return its biggest item. The\ndefault keyword-only argument specifies an object to return if\nthe provided iterable is empty.\nWith two or more positional arguments, return the largest argument.", + "builtins.member_descriptor.__delattr__" => "Implement delattr(self, name).", + "builtins.member_descriptor.__delete__" => "Delete an attribute of instance.", + "builtins.member_descriptor.__eq__" => "Return self==value.", + "builtins.member_descriptor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.member_descriptor.__ge__" => "Return self>=value.", + "builtins.member_descriptor.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.member_descriptor.__getattribute__" => "Return getattr(self, name).", + "builtins.member_descriptor.__getstate__" => "Helper for pickle.", + "builtins.member_descriptor.__gt__" => "Return self>value.", + "builtins.member_descriptor.__hash__" => "Return hash(self).", + "builtins.member_descriptor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.member_descriptor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.member_descriptor.__le__" => "Return self<=value.", + "builtins.member_descriptor.__lt__" => "Return self<value.", + "builtins.member_descriptor.__ne__" => "Return self!=value.", + "builtins.member_descriptor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.member_descriptor.__reduce_ex__" => "Helper for pickle.", + "builtins.member_descriptor.__repr__" => "Return repr(self).", + "builtins.member_descriptor.__set__" => "Set an attribute of instance to value.", + "builtins.member_descriptor.__setattr__" => "Implement setattr(self, name, value).", + "builtins.member_descriptor.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.member_descriptor.__str__" => "Return str(self).", + "builtins.member_descriptor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.memory_iterator.__delattr__" => "Implement delattr(self, name).", + "builtins.memory_iterator.__eq__" => "Return self==value.", + "builtins.memory_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.memory_iterator.__ge__" => "Return self>=value.", + "builtins.memory_iterator.__getattribute__" => "Return getattr(self, name).", + "builtins.memory_iterator.__getstate__" => "Helper for pickle.", + "builtins.memory_iterator.__gt__" => "Return self>value.", + "builtins.memory_iterator.__hash__" => "Return hash(self).", + "builtins.memory_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.memory_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.memory_iterator.__iter__" => "Implement iter(self).", + "builtins.memory_iterator.__le__" => "Return self<=value.", + "builtins.memory_iterator.__lt__" => "Return self<value.", + "builtins.memory_iterator.__ne__" => "Return self!=value.", + "builtins.memory_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.memory_iterator.__next__" => "Implement next(self).", + "builtins.memory_iterator.__reduce__" => "Helper for pickle.", + "builtins.memory_iterator.__reduce_ex__" => "Helper for pickle.", + "builtins.memory_iterator.__repr__" => "Return repr(self).", + "builtins.memory_iterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.memory_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.memory_iterator.__str__" => "Return str(self).", + "builtins.memory_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.memoryview" => "Create a new memoryview object which references the given object.", + "builtins.memoryview.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "builtins.memoryview.__class_getitem__" => "See PEP 585", + "builtins.memoryview.__delattr__" => "Implement delattr(self, name).", + "builtins.memoryview.__delitem__" => "Delete self[key].", + "builtins.memoryview.__eq__" => "Return self==value.", + "builtins.memoryview.__exit__" => "Release the underlying buffer exposed by the memoryview object.", + "builtins.memoryview.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.memoryview.__ge__" => "Return self>=value.", + "builtins.memoryview.__getattribute__" => "Return getattr(self, name).", + "builtins.memoryview.__getitem__" => "Return self[key].", + "builtins.memoryview.__getstate__" => "Helper for pickle.", + "builtins.memoryview.__gt__" => "Return self>value.", + "builtins.memoryview.__hash__" => "Return hash(self).", + "builtins.memoryview.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.memoryview.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.memoryview.__iter__" => "Implement iter(self).", + "builtins.memoryview.__le__" => "Return self<=value.", + "builtins.memoryview.__len__" => "Return len(self).", + "builtins.memoryview.__lt__" => "Return self<value.", + "builtins.memoryview.__ne__" => "Return self!=value.", + "builtins.memoryview.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.memoryview.__reduce__" => "Helper for pickle.", + "builtins.memoryview.__reduce_ex__" => "Helper for pickle.", + "builtins.memoryview.__release_buffer__" => "Release the buffer object that exposes the underlying memory of the object.", + "builtins.memoryview.__repr__" => "Return repr(self).", + "builtins.memoryview.__setattr__" => "Implement setattr(self, name, value).", + "builtins.memoryview.__setitem__" => "Set self[key] to value.", + "builtins.memoryview.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.memoryview.__str__" => "Return str(self).", + "builtins.memoryview.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.memoryview._from_flags" => "Create a new memoryview object which references the given object.", + "builtins.memoryview.c_contiguous" => "A bool indicating whether the memory is C contiguous.", + "builtins.memoryview.cast" => "Cast a memoryview to a new format or shape.", + "builtins.memoryview.contiguous" => "A bool indicating whether the memory is contiguous.", + "builtins.memoryview.count" => "Count the number of occurrences of a value.", + "builtins.memoryview.f_contiguous" => "A bool indicating whether the memory is Fortran contiguous.", + "builtins.memoryview.format" => "A string containing the format (in struct module style)\n for each element in the view.", + "builtins.memoryview.hex" => "Return the data in the buffer as a str of hexadecimal numbers.\n\n sep\n An optional single character or byte to separate hex bytes.\n bytes_per_sep\n How many bytes between separators. Positive values count from the\n right, negative values count from the left.\n\nExample:\n>>> value = memoryview(b'\\xb9\\x01\\xef')\n>>> value.hex()\n'b901ef'\n>>> value.hex(':')\n'b9:01:ef'\n>>> value.hex(':', 2)\n'b9:01ef'\n>>> value.hex(':', -2)\n'b901:ef'", + "builtins.memoryview.index" => "Return the index of the first occurrence of a value.\n\nRaises ValueError if the value is not present.", + "builtins.memoryview.itemsize" => "The size in bytes of each element of the memoryview.", + "builtins.memoryview.nbytes" => "The amount of space in bytes that the array would use in\n a contiguous representation.", + "builtins.memoryview.ndim" => "An integer indicating how many dimensions of a multi-dimensional\n array the memory represents.", + "builtins.memoryview.obj" => "The underlying object of the memoryview.", + "builtins.memoryview.readonly" => "A bool indicating whether the memory is read only.", + "builtins.memoryview.release" => "Release the underlying buffer exposed by the memoryview object.", + "builtins.memoryview.shape" => "A tuple of ndim integers giving the shape of the memory\n as an N-dimensional array.", + "builtins.memoryview.strides" => "A tuple of ndim integers giving the size in bytes to access\n each element for each dimension of the array.", + "builtins.memoryview.suboffsets" => "A tuple of integers used internally for PIL-style arrays.", + "builtins.memoryview.tobytes" => "Return the data in the buffer as a byte string.\n\nOrder can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of the\noriginal array is converted to C or Fortran order. For contiguous views,\n'A' returns an exact copy of the physical memory. In particular, in-memory\nFortran order is preserved. For non-contiguous views, the data is converted\nto C first. order=None is the same as order='C'.", + "builtins.memoryview.tolist" => "Return the data in the buffer as a list of elements.", + "builtins.memoryview.toreadonly" => "Return a readonly version of the memoryview.", + "builtins.method" => "Create a bound instance method object.", + "builtins.method-wrapper.__call__" => "Call self as a function.", + "builtins.method-wrapper.__delattr__" => "Implement delattr(self, name).", + "builtins.method-wrapper.__eq__" => "Return self==value.", + "builtins.method-wrapper.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.method-wrapper.__ge__" => "Return self>=value.", + "builtins.method-wrapper.__getattribute__" => "Return getattr(self, name).", + "builtins.method-wrapper.__getstate__" => "Helper for pickle.", + "builtins.method-wrapper.__gt__" => "Return self>value.", + "builtins.method-wrapper.__hash__" => "Return hash(self).", + "builtins.method-wrapper.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.method-wrapper.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.method-wrapper.__le__" => "Return self<=value.", + "builtins.method-wrapper.__lt__" => "Return self<value.", + "builtins.method-wrapper.__ne__" => "Return self!=value.", + "builtins.method-wrapper.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.method-wrapper.__reduce_ex__" => "Helper for pickle.", + "builtins.method-wrapper.__repr__" => "Return repr(self).", + "builtins.method-wrapper.__setattr__" => "Implement setattr(self, name, value).", + "builtins.method-wrapper.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.method-wrapper.__str__" => "Return str(self).", + "builtins.method-wrapper.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.method.__call__" => "Call self as a function.", + "builtins.method.__delattr__" => "Implement delattr(self, name).", + "builtins.method.__eq__" => "Return self==value.", + "builtins.method.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.method.__func__" => "the function (or other callable) implementing a method", + "builtins.method.__ge__" => "Return self>=value.", + "builtins.method.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.method.__getattribute__" => "Return getattr(self, name).", + "builtins.method.__getstate__" => "Helper for pickle.", + "builtins.method.__gt__" => "Return self>value.", + "builtins.method.__hash__" => "Return hash(self).", + "builtins.method.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.method.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.method.__le__" => "Return self<=value.", + "builtins.method.__lt__" => "Return self<value.", + "builtins.method.__ne__" => "Return self!=value.", + "builtins.method.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.method.__reduce_ex__" => "Helper for pickle.", + "builtins.method.__repr__" => "Return repr(self).", + "builtins.method.__self__" => "the instance to which a method is bound", + "builtins.method.__setattr__" => "Implement setattr(self, name, value).", + "builtins.method.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.method.__str__" => "Return str(self).", + "builtins.method.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.method_descriptor.__call__" => "Call self as a function.", + "builtins.method_descriptor.__delattr__" => "Implement delattr(self, name).", + "builtins.method_descriptor.__eq__" => "Return self==value.", + "builtins.method_descriptor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.method_descriptor.__ge__" => "Return self>=value.", + "builtins.method_descriptor.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.method_descriptor.__getattribute__" => "Return getattr(self, name).", + "builtins.method_descriptor.__getstate__" => "Helper for pickle.", + "builtins.method_descriptor.__gt__" => "Return self>value.", + "builtins.method_descriptor.__hash__" => "Return hash(self).", + "builtins.method_descriptor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.method_descriptor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.method_descriptor.__le__" => "Return self<=value.", + "builtins.method_descriptor.__lt__" => "Return self<value.", + "builtins.method_descriptor.__ne__" => "Return self!=value.", + "builtins.method_descriptor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.method_descriptor.__reduce_ex__" => "Helper for pickle.", + "builtins.method_descriptor.__repr__" => "Return repr(self).", + "builtins.method_descriptor.__setattr__" => "Implement setattr(self, name, value).", + "builtins.method_descriptor.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.method_descriptor.__str__" => "Return str(self).", + "builtins.method_descriptor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.min" => "min(iterable, *[, default=obj, key=func]) -> value\nmin(arg1, arg2, *args, *[, key=func]) -> value\n\nWith a single iterable argument, return its smallest item. The\ndefault keyword-only argument specifies an object to return if\nthe provided iterable is empty.\nWith two or more positional arguments, return the smallest argument.", + "builtins.module" => "Create a module object.\n\nThe name must be a string; the optional doc argument can have any type.", + "builtins.module.__delattr__" => "Implement delattr(self, name).", + "builtins.module.__eq__" => "Return self==value.", + "builtins.module.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.module.__ge__" => "Return self>=value.", + "builtins.module.__getattribute__" => "Return getattr(self, name).", + "builtins.module.__getstate__" => "Helper for pickle.", + "builtins.module.__gt__" => "Return self>value.", + "builtins.module.__hash__" => "Return hash(self).", + "builtins.module.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.module.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.module.__le__" => "Return self<=value.", + "builtins.module.__lt__" => "Return self<value.", + "builtins.module.__ne__" => "Return self!=value.", + "builtins.module.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.module.__reduce__" => "Helper for pickle.", + "builtins.module.__reduce_ex__" => "Helper for pickle.", + "builtins.module.__repr__" => "Return repr(self).", + "builtins.module.__setattr__" => "Implement setattr(self, name, value).", + "builtins.module.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.module.__str__" => "Return str(self).", + "builtins.module.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.next" => "next(iterator[, default])\n\nReturn the next item from the iterator. If default is given and the iterator\nis exhausted, it is returned instead of raising StopIteration.", + "builtins.object" => "The base class of the class hierarchy.\n\nWhen called, it accepts no arguments and returns a new featureless\ninstance that has no instance attributes and cannot be given any.", + "builtins.object.__delattr__" => "Implement delattr(self, name).", + "builtins.object.__eq__" => "Return self==value.", + "builtins.object.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.object.__ge__" => "Return self>=value.", + "builtins.object.__getattribute__" => "Return getattr(self, name).", + "builtins.object.__getstate__" => "Helper for pickle.", + "builtins.object.__gt__" => "Return self>value.", + "builtins.object.__hash__" => "Return hash(self).", + "builtins.object.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.object.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.object.__le__" => "Return self<=value.", + "builtins.object.__lt__" => "Return self<value.", + "builtins.object.__ne__" => "Return self!=value.", + "builtins.object.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.object.__reduce__" => "Helper for pickle.", + "builtins.object.__reduce_ex__" => "Helper for pickle.", + "builtins.object.__repr__" => "Return repr(self).", + "builtins.object.__setattr__" => "Implement setattr(self, name, value).", + "builtins.object.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.object.__str__" => "Return str(self).", + "builtins.object.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.oct" => "Return the octal representation of an integer.\n\n >>> oct(342391)\n '0o1234567'", + "builtins.open" => "Open file and return a stream. Raise OSError upon failure.\n\nfile is either a text or byte string giving the name (and the path\nif the file isn't in the current working directory) of the file to\nbe opened or an integer file descriptor of the file to be\nwrapped. (If a file descriptor is given, it is closed when the\nreturned I/O object is closed, unless closefd is set to False.)\n\nmode is an optional string that specifies the mode in which the file\nis opened. It defaults to 'r' which means open for reading in text\nmode. Other common values are 'w' for writing (truncating the file if\nit already exists), 'x' for creating and writing to a new file, and\n'a' for appending (which on some Unix systems, means that all writes\nappend to the end of the file regardless of the current seek position).\nIn text mode, if encoding is not specified the encoding used is platform\ndependent: locale.getencoding() is called to get the current locale encoding.\n(For reading and writing raw bytes use binary mode and leave encoding\nunspecified.) The available modes are:\n\n========= ===============================================================\nCharacter Meaning\n--------- ---------------------------------------------------------------\n'r' open for reading (default)\n'w' open for writing, truncating the file first\n'x' create a new file and open it for writing\n'a' open for writing, appending to the end of the file if it exists\n'b' binary mode\n't' text mode (default)\n'+' open a disk file for updating (reading and writing)\n========= ===============================================================\n\nThe default mode is 'rt' (open for reading text). For binary random\naccess, the mode 'w+b' opens and truncates the file to 0 bytes, while\n'r+b' opens the file without truncation. The 'x' mode implies 'w' and\nraises an `FileExistsError` if the file already exists.\n\nPython distinguishes between files opened in binary and text modes,\neven when the underlying operating system doesn't. Files opened in\nbinary mode (appending 'b' to the mode argument) return contents as\nbytes objects without any decoding. In text mode (the default, or when\n't' is appended to the mode argument), the contents of the file are\nreturned as strings, the bytes having been first decoded using a\nplatform-dependent encoding or using the specified encoding if given.\n\nbuffering is an optional integer used to set the buffering policy.\nPass 0 to switch buffering off (only allowed in binary mode), 1 to select\nline buffering (only usable in text mode), and an integer > 1 to indicate\nthe size of a fixed-size chunk buffer. When no buffering argument is\ngiven, the default buffering policy works as follows:\n\n* Binary files are buffered in fixed-size chunks; the size of the buffer\n is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)\n when the device block size is available.\n On most systems, the buffer will typically be 128 kilobytes long.\n\n* \"Interactive\" text files (files for which isatty() returns True)\n use line buffering. Other text files use the policy described above\n for binary files.\n\nencoding is the name of the encoding used to decode or encode the\nfile. This should only be used in text mode. The default encoding is\nplatform dependent, but any encoding supported by Python can be\npassed. See the codecs module for the list of supported encodings.\n\nerrors is an optional string that specifies how encoding errors are to\nbe handled---this argument should not be used in binary mode. Pass\n'strict' to raise a ValueError exception if there is an encoding error\n(the default of None has the same effect), or pass 'ignore' to ignore\nerrors. (Note that ignoring encoding errors can lead to data loss.)\nSee the documentation for codecs.register or run 'help(codecs.Codec)'\nfor a list of the permitted encoding error strings.\n\nnewline controls how universal newlines works (it only applies to text\nmode). It can be None, '', '\\n', '\\r', and '\\r\\n'. It works as\nfollows:\n\n* On input, if newline is None, universal newlines mode is\n enabled. Lines in the input can end in '\\n', '\\r', or '\\r\\n', and\n these are translated into '\\n' before being returned to the\n caller. If it is '', universal newline mode is enabled, but line\n endings are returned to the caller untranslated. If it has any of\n the other legal values, input lines are only terminated by the given\n string, and the line ending is returned to the caller untranslated.\n\n* On output, if newline is None, any '\\n' characters written are\n translated to the system default line separator, os.linesep. If\n newline is '' or '\\n', no translation takes place. If newline is any\n of the other legal values, any '\\n' characters written are translated\n to the given string.\n\nIf closefd is False, the underlying file descriptor will be kept open\nwhen the file is closed. This does not work when a file name is given\nand must be True in that case.\n\nA custom opener can be used by passing a callable as *opener*. The\nunderlying file descriptor for the file object is then obtained by\ncalling *opener* with (*file*, *flags*). *opener* must return an open\nfile descriptor (passing os.open as *opener* results in functionality\nsimilar to passing None).\n\nopen() returns a file object whose type depends on the mode, and\nthrough which the standard file operations such as reading and writing\nare performed. When open() is used to open a file in a text mode ('w',\n'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open\na file in a binary mode, the returned class varies: in read binary\nmode, it returns a BufferedReader; in write binary and append binary\nmodes, it returns a BufferedWriter, and in read/write mode, it returns\na BufferedRandom.\n\nIt is also possible to use a string or bytearray as a file for both\nreading and writing. For strings StringIO can be used like a file\nopened in a text mode, and for bytes a BytesIO can be used like a file\nopened in a binary mode.", + "builtins.ord" => "Return the ordinal value of a character.\n\nIf the argument is a one-character string, return the Unicode code\npoint of that character.\n\nIf the argument is a bytes or bytearray object of length 1, return its\nsingle byte value.", + "builtins.pow" => "Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments\n\nSome types, such as ints, are able to use a more efficient algorithm when\ninvoked using the three argument form.", + "builtins.print" => "Prints the values to a stream, or to sys.stdout by default.\n\n sep\n string inserted between values, default a space.\n end\n string appended after the last value, default a newline.\n file\n a file-like object (stream); defaults to the current sys.stdout.\n flush\n whether to forcibly flush the stream.", + "builtins.property" => "Property attribute.\n\n fget\n function to be used for getting an attribute value\n fset\n function to be used for setting an attribute value\n fdel\n function to be used for del'ing an attribute\n doc\n docstring\n\nTypical use is to define a managed attribute x:\n\nclass C(object):\n def getx(self): return self._x\n def setx(self, value): self._x = value\n def delx(self): del self._x\n x = property(getx, setx, delx, \"I'm the 'x' property.\")\n\nDecorators make defining new properties or modifying existing ones easy:\n\nclass C(object):\n @property\n def x(self):\n \"I am the 'x' property.\"\n return self._x\n @x.setter\n def x(self, value):\n self._x = value\n @x.deleter\n def x(self):\n del self._x", + "builtins.property.__delattr__" => "Implement delattr(self, name).", + "builtins.property.__delete__" => "Delete an attribute of instance.", + "builtins.property.__eq__" => "Return self==value.", + "builtins.property.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.property.__ge__" => "Return self>=value.", + "builtins.property.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.property.__getattribute__" => "Return getattr(self, name).", + "builtins.property.__getstate__" => "Helper for pickle.", + "builtins.property.__gt__" => "Return self>value.", + "builtins.property.__hash__" => "Return hash(self).", + "builtins.property.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.property.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.property.__le__" => "Return self<=value.", + "builtins.property.__lt__" => "Return self<value.", + "builtins.property.__ne__" => "Return self!=value.", + "builtins.property.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.property.__reduce__" => "Helper for pickle.", + "builtins.property.__reduce_ex__" => "Helper for pickle.", + "builtins.property.__repr__" => "Return repr(self).", + "builtins.property.__set__" => "Set an attribute of instance to value.", + "builtins.property.__set_name__" => "Method to set name of a property.", + "builtins.property.__setattr__" => "Implement setattr(self, name, value).", + "builtins.property.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.property.__str__" => "Return str(self).", + "builtins.property.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.property.deleter" => "Descriptor to obtain a copy of the property with a different deleter.", + "builtins.property.getter" => "Descriptor to obtain a copy of the property with a different getter.", + "builtins.property.setter" => "Descriptor to obtain a copy of the property with a different setter.", + "builtins.range" => "range(stop) -> range object\nrange(start, stop[, step]) -> range object\n\nReturn an object that produces a sequence of integers from start (inclusive)\nto stop (exclusive) by step. range(i, j) produces i, i+1, i+2, ..., j-1.\nstart defaults to 0, and stop is omitted! range(4) produces 0, 1, 2, 3.\nThese are exactly the valid indices for a list of 4 elements.\nWhen step is given, it specifies the increment (or decrement).", + "builtins.range.__bool__" => "True if self else False", + "builtins.range.__contains__" => "Return bool(key in self).", + "builtins.range.__delattr__" => "Implement delattr(self, name).", + "builtins.range.__eq__" => "Return self==value.", + "builtins.range.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.range.__ge__" => "Return self>=value.", + "builtins.range.__getattribute__" => "Return getattr(self, name).", + "builtins.range.__getitem__" => "Return self[key].", + "builtins.range.__getstate__" => "Helper for pickle.", + "builtins.range.__gt__" => "Return self>value.", + "builtins.range.__hash__" => "Return hash(self).", + "builtins.range.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.range.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.range.__iter__" => "Implement iter(self).", + "builtins.range.__le__" => "Return self<=value.", + "builtins.range.__len__" => "Return len(self).", + "builtins.range.__lt__" => "Return self<value.", + "builtins.range.__ne__" => "Return self!=value.", + "builtins.range.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.range.__reduce_ex__" => "Helper for pickle.", + "builtins.range.__repr__" => "Return repr(self).", + "builtins.range.__reversed__" => "Return a reverse iterator.", + "builtins.range.__setattr__" => "Implement setattr(self, name, value).", + "builtins.range.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.range.__str__" => "Return str(self).", + "builtins.range.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.range.count" => "rangeobject.count(value) -> integer -- return number of occurrences of value", + "builtins.range.index" => "rangeobject.index(value) -> integer -- return index of value.\nRaise ValueError if the value is not present.", + "builtins.range_iterator.__delattr__" => "Implement delattr(self, name).", + "builtins.range_iterator.__eq__" => "Return self==value.", + "builtins.range_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.range_iterator.__ge__" => "Return self>=value.", + "builtins.range_iterator.__getattribute__" => "Return getattr(self, name).", + "builtins.range_iterator.__getstate__" => "Helper for pickle.", + "builtins.range_iterator.__gt__" => "Return self>value.", + "builtins.range_iterator.__hash__" => "Return hash(self).", + "builtins.range_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.range_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.range_iterator.__iter__" => "Implement iter(self).", + "builtins.range_iterator.__le__" => "Return self<=value.", + "builtins.range_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.range_iterator.__lt__" => "Return self<value.", + "builtins.range_iterator.__ne__" => "Return self!=value.", + "builtins.range_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.range_iterator.__next__" => "Implement next(self).", + "builtins.range_iterator.__reduce__" => "Return state information for pickling.", + "builtins.range_iterator.__reduce_ex__" => "Helper for pickle.", + "builtins.range_iterator.__repr__" => "Return repr(self).", + "builtins.range_iterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.range_iterator.__setstate__" => "Set state information for unpickling.", + "builtins.range_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.range_iterator.__str__" => "Return str(self).", + "builtins.range_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.repr" => "Return the canonical string representation of the object.\n\nFor many object types, including most builtins, eval(repr(obj)) == obj.", + "builtins.reversed" => "Return a reverse iterator over the values of the given sequence.", + "builtins.reversed.__delattr__" => "Implement delattr(self, name).", + "builtins.reversed.__eq__" => "Return self==value.", + "builtins.reversed.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.reversed.__ge__" => "Return self>=value.", + "builtins.reversed.__getattribute__" => "Return getattr(self, name).", + "builtins.reversed.__getstate__" => "Helper for pickle.", + "builtins.reversed.__gt__" => "Return self>value.", + "builtins.reversed.__hash__" => "Return hash(self).", + "builtins.reversed.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.reversed.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.reversed.__iter__" => "Implement iter(self).", + "builtins.reversed.__le__" => "Return self<=value.", + "builtins.reversed.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.reversed.__lt__" => "Return self<value.", + "builtins.reversed.__ne__" => "Return self!=value.", + "builtins.reversed.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.reversed.__next__" => "Implement next(self).", + "builtins.reversed.__reduce__" => "Return state information for pickling.", + "builtins.reversed.__reduce_ex__" => "Helper for pickle.", + "builtins.reversed.__repr__" => "Return repr(self).", + "builtins.reversed.__setattr__" => "Implement setattr(self, name, value).", + "builtins.reversed.__setstate__" => "Set state information for unpickling.", + "builtins.reversed.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.reversed.__str__" => "Return str(self).", + "builtins.reversed.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.round" => "Round a number to a given precision in decimal digits.\n\nThe return value is an integer if ndigits is omitted or None. Otherwise\nthe return value has the same type as the number. ndigits may be negative.", + "builtins.set" => "Build an unordered collection of unique elements.", + "builtins.set.__and__" => "Return self&value.", + "builtins.set.__class_getitem__" => "See PEP 585", + "builtins.set.__contains__" => "x.__contains__(y) <==> y in x.", + "builtins.set.__delattr__" => "Implement delattr(self, name).", + "builtins.set.__eq__" => "Return self==value.", + "builtins.set.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.set.__ge__" => "Return self>=value.", + "builtins.set.__getattribute__" => "Return getattr(self, name).", + "builtins.set.__getstate__" => "Helper for pickle.", + "builtins.set.__gt__" => "Return self>value.", + "builtins.set.__iand__" => "Return self&=value.", + "builtins.set.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.set.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.set.__ior__" => "Return self|=value.", + "builtins.set.__isub__" => "Return self-=value.", + "builtins.set.__iter__" => "Implement iter(self).", + "builtins.set.__ixor__" => "Return self^=value.", + "builtins.set.__le__" => "Return self<=value.", + "builtins.set.__len__" => "Return len(self).", + "builtins.set.__lt__" => "Return self<value.", + "builtins.set.__ne__" => "Return self!=value.", + "builtins.set.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.set.__or__" => "Return self|value.", + "builtins.set.__rand__" => "Return value&self.", + "builtins.set.__reduce__" => "Return state information for pickling.", + "builtins.set.__reduce_ex__" => "Helper for pickle.", + "builtins.set.__repr__" => "Return repr(self).", + "builtins.set.__ror__" => "Return value|self.", + "builtins.set.__rsub__" => "Return value-self.", + "builtins.set.__rxor__" => "Return value^self.", + "builtins.set.__setattr__" => "Implement setattr(self, name, value).", + "builtins.set.__sizeof__" => "S.__sizeof__() -> size of S in memory, in bytes.", + "builtins.set.__str__" => "Return str(self).", + "builtins.set.__sub__" => "Return self-value.", + "builtins.set.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.set.__xor__" => "Return self^value.", + "builtins.set.add" => "Add an element to a set.\n\nThis has no effect if the element is already present.", + "builtins.set.clear" => "Remove all elements from this set.", + "builtins.set.copy" => "Return a shallow copy of a set.", + "builtins.set.difference" => "Return a new set with elements in the set that are not in the others.", + "builtins.set.difference_update" => "Update the set, removing elements found in others.", + "builtins.set.discard" => "Remove an element from a set if it is a member.\n\nUnlike set.remove(), the discard() method does not raise\nan exception when an element is missing from the set.", + "builtins.set.intersection" => "Return a new set with elements common to the set and all others.", + "builtins.set.intersection_update" => "Update the set, keeping only elements found in it and all others.", + "builtins.set.isdisjoint" => "Return True if two sets have a null intersection.", + "builtins.set.issubset" => "Report whether another set contains this set.", + "builtins.set.issuperset" => "Report whether this set contains another set.", + "builtins.set.pop" => "Remove and return an arbitrary set element.\n\nRaises KeyError if the set is empty.", + "builtins.set.remove" => "Remove an element from a set; it must be a member.\n\nIf the element is not a member, raise a KeyError.", + "builtins.set.symmetric_difference" => "Return a new set with elements in either the set or other but not both.", + "builtins.set.symmetric_difference_update" => "Update the set, keeping only elements found in either set, but not in both.", + "builtins.set.union" => "Return a new set with elements from the set and all others.", + "builtins.set.update" => "Update the set, adding elements from all others.", + "builtins.set_iterator.__delattr__" => "Implement delattr(self, name).", + "builtins.set_iterator.__eq__" => "Return self==value.", + "builtins.set_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.set_iterator.__ge__" => "Return self>=value.", + "builtins.set_iterator.__getattribute__" => "Return getattr(self, name).", + "builtins.set_iterator.__getstate__" => "Helper for pickle.", + "builtins.set_iterator.__gt__" => "Return self>value.", + "builtins.set_iterator.__hash__" => "Return hash(self).", + "builtins.set_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.set_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.set_iterator.__iter__" => "Implement iter(self).", + "builtins.set_iterator.__le__" => "Return self<=value.", + "builtins.set_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.set_iterator.__lt__" => "Return self<value.", + "builtins.set_iterator.__ne__" => "Return self!=value.", + "builtins.set_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.set_iterator.__next__" => "Implement next(self).", + "builtins.set_iterator.__reduce__" => "Return state information for pickling.", + "builtins.set_iterator.__reduce_ex__" => "Helper for pickle.", + "builtins.set_iterator.__repr__" => "Return repr(self).", + "builtins.set_iterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.set_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.set_iterator.__str__" => "Return str(self).", + "builtins.set_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.setattr" => "Sets the named attribute on the given object to the specified value.\n\nsetattr(x, 'y', v) is equivalent to ``x.y = v``", + "builtins.slice" => "slice(stop)\nslice(start, stop[, step])\n\nCreate a slice object. This is used for extended slicing (e.g. a[0:10:2]).", + "builtins.slice.__delattr__" => "Implement delattr(self, name).", + "builtins.slice.__eq__" => "Return self==value.", + "builtins.slice.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.slice.__ge__" => "Return self>=value.", + "builtins.slice.__getattribute__" => "Return getattr(self, name).", + "builtins.slice.__getstate__" => "Helper for pickle.", + "builtins.slice.__gt__" => "Return self>value.", + "builtins.slice.__hash__" => "Return hash(self).", + "builtins.slice.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.slice.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.slice.__le__" => "Return self<=value.", + "builtins.slice.__lt__" => "Return self<value.", + "builtins.slice.__ne__" => "Return self!=value.", + "builtins.slice.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.slice.__reduce__" => "Return state information for pickling.", + "builtins.slice.__reduce_ex__" => "Helper for pickle.", + "builtins.slice.__repr__" => "Return repr(self).", + "builtins.slice.__setattr__" => "Implement setattr(self, name, value).", + "builtins.slice.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.slice.__str__" => "Return str(self).", + "builtins.slice.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.slice.indices" => "S.indices(len) -> (start, stop, stride)\n\nAssuming a sequence of length len, calculate the start and stop\nindices, and the stride length of the extended slice described by\nS. Out of bounds indices are clipped in a manner consistent with the\nhandling of normal slices.", + "builtins.sorted" => "Return a new list containing all items from the iterable in ascending order.\n\nA custom key function can be supplied to customize the sort order, and the\nreverse flag can be set to request the result in descending order.", + "builtins.staticmethod" => "Convert a function to be a static method.\n\nA static method does not receive an implicit first argument.\nTo declare a static method, use this idiom:\n\n class C:\n @staticmethod\n def f(arg1, arg2, argN):\n ...\n\nIt can be called either on the class (e.g. C.f()) or on an instance\n(e.g. C().f()). Both the class and the instance are ignored, and\nneither is passed implicitly as the first argument to the method.\n\nStatic methods in Python are similar to those found in Java or C++.\nFor a more advanced concept, see the classmethod builtin.", + "builtins.staticmethod.__call__" => "Call self as a function.", + "builtins.staticmethod.__delattr__" => "Implement delattr(self, name).", + "builtins.staticmethod.__eq__" => "Return self==value.", + "builtins.staticmethod.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.staticmethod.__ge__" => "Return self>=value.", + "builtins.staticmethod.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.staticmethod.__getattribute__" => "Return getattr(self, name).", + "builtins.staticmethod.__getstate__" => "Helper for pickle.", + "builtins.staticmethod.__gt__" => "Return self>value.", + "builtins.staticmethod.__hash__" => "Return hash(self).", + "builtins.staticmethod.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.staticmethod.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.staticmethod.__le__" => "Return self<=value.", + "builtins.staticmethod.__lt__" => "Return self<value.", + "builtins.staticmethod.__ne__" => "Return self!=value.", + "builtins.staticmethod.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.staticmethod.__reduce__" => "Helper for pickle.", + "builtins.staticmethod.__reduce_ex__" => "Helper for pickle.", + "builtins.staticmethod.__repr__" => "Return repr(self).", + "builtins.staticmethod.__setattr__" => "Implement setattr(self, name, value).", + "builtins.staticmethod.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.staticmethod.__str__" => "Return str(self).", + "builtins.staticmethod.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.str" => "str(object='') -> str\nstr(bytes_or_buffer[, encoding[, errors]]) -> str\n\nCreate a new string object from the given object. If encoding or\nerrors is specified, then the object must expose a data buffer\nthat will be decoded using the given encoding and error handler.\nOtherwise, returns the result of object.__str__() (if defined)\nor repr(object).\nencoding defaults to 'utf-8'.\nerrors defaults to 'strict'.", + "builtins.str.__add__" => "Return self+value.", + "builtins.str.__contains__" => "Return bool(key in self).", + "builtins.str.__delattr__" => "Implement delattr(self, name).", + "builtins.str.__eq__" => "Return self==value.", + "builtins.str.__format__" => "Return a formatted version of the string as described by format_spec.", + "builtins.str.__ge__" => "Return self>=value.", + "builtins.str.__getattribute__" => "Return getattr(self, name).", + "builtins.str.__getitem__" => "Return self[key].", + "builtins.str.__getstate__" => "Helper for pickle.", + "builtins.str.__gt__" => "Return self>value.", + "builtins.str.__hash__" => "Return hash(self).", + "builtins.str.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.str.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.str.__iter__" => "Implement iter(self).", + "builtins.str.__le__" => "Return self<=value.", + "builtins.str.__len__" => "Return len(self).", + "builtins.str.__lt__" => "Return self<value.", + "builtins.str.__mod__" => "Return self%value.", + "builtins.str.__mul__" => "Return self*value.", + "builtins.str.__ne__" => "Return self!=value.", + "builtins.str.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.str.__reduce__" => "Helper for pickle.", + "builtins.str.__reduce_ex__" => "Helper for pickle.", + "builtins.str.__repr__" => "Return repr(self).", + "builtins.str.__rmod__" => "Return value%self.", + "builtins.str.__rmul__" => "Return value*self.", + "builtins.str.__setattr__" => "Implement setattr(self, name, value).", + "builtins.str.__sizeof__" => "Return the size of the string in memory, in bytes.", + "builtins.str.__str__" => "Return str(self).", + "builtins.str.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.str.capitalize" => "Return a capitalized version of the string.\n\nMore specifically, make the first character have upper case and the rest lower\ncase.", + "builtins.str.casefold" => "Return a version of the string suitable for caseless comparisons.", + "builtins.str.center" => "Return a centered string of length width.\n\nPadding is done using the specified fill character (default is a space).", + "builtins.str.count" => "Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n\nOptional arguments start and end are interpreted as in slice notation.", + "builtins.str.encode" => "Encode the string using the codec registered for encoding.\n\n encoding\n The encoding in which to encode the string.\n errors\n The error handling scheme to use for encoding errors.\n The default is 'strict' meaning that encoding errors raise a\n UnicodeEncodeError. Other possible values are 'ignore', 'replace' and\n 'xmlcharrefreplace' as well as any other name registered with\n codecs.register_error that can handle UnicodeEncodeErrors.", + "builtins.str.endswith" => "Return True if the string ends with the specified suffix, False otherwise.\n\n suffix\n A string or a tuple of strings to try.\n start\n Optional start position. Default: start of the string.\n end\n Optional stop position. Default: end of the string.", + "builtins.str.expandtabs" => "Return a copy where all tab characters are expanded using spaces.\n\nIf tabsize is not given, a tab size of 8 characters is assumed.", + "builtins.str.find" => "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n\nOptional arguments start and end are interpreted as in slice notation.\nReturn -1 on failure.", + "builtins.str.format" => "Return a formatted version of the string, using substitutions from args and kwargs.\nThe substitutions are identified by braces ('{' and '}').", + "builtins.str.format_map" => "Return a formatted version of the string, using substitutions from mapping.\nThe substitutions are identified by braces ('{' and '}').", + "builtins.str.index" => "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n\nOptional arguments start and end are interpreted as in slice notation.\nRaises ValueError when the substring is not found.", + "builtins.str.isalnum" => "Return True if the string is an alpha-numeric string, False otherwise.\n\nA string is alpha-numeric if all characters in the string are alpha-numeric and\nthere is at least one character in the string.", + "builtins.str.isalpha" => "Return True if the string is an alphabetic string, False otherwise.\n\nA string is alphabetic if all characters in the string are alphabetic and there\nis at least one character in the string.", + "builtins.str.isascii" => "Return True if all characters in the string are ASCII, False otherwise.\n\nASCII characters have code points in the range U+0000-U+007F.\nEmpty string is ASCII too.", + "builtins.str.isdecimal" => "Return True if the string is a decimal string, False otherwise.\n\nA string is a decimal string if all characters in the string are decimal and\nthere is at least one character in the string.", + "builtins.str.isdigit" => "Return True if the string is a digit string, False otherwise.\n\nA string is a digit string if all characters in the string are digits and there\nis at least one character in the string.", + "builtins.str.isidentifier" => "Return True if the string is a valid Python identifier, False otherwise.\n\nCall keyword.iskeyword(s) to test whether string s is a reserved identifier,\nsuch as \"def\" or \"class\".", + "builtins.str.islower" => "Return True if the string is a lowercase string, False otherwise.\n\nA string is lowercase if all cased characters in the string are lowercase and\nthere is at least one cased character in the string.", + "builtins.str.isnumeric" => "Return True if the string is a numeric string, False otherwise.\n\nA string is numeric if all characters in the string are numeric and there is at\nleast one character in the string.", + "builtins.str.isprintable" => "Return True if all characters in the string are printable, False otherwise.\n\nA character is printable if repr() may use it in its output.", + "builtins.str.isspace" => "Return True if the string is a whitespace string, False otherwise.\n\nA string is whitespace if all characters in the string are whitespace and there\nis at least one character in the string.", + "builtins.str.istitle" => "Return True if the string is a title-cased string, False otherwise.\n\nIn a title-cased string, upper- and title-case characters may only\nfollow uncased characters and lowercase characters only cased ones.", + "builtins.str.isupper" => "Return True if the string is an uppercase string, False otherwise.\n\nA string is uppercase if all cased characters in the string are uppercase and\nthere is at least one cased character in the string.", + "builtins.str.join" => "Concatenate any number of strings.\n\nThe string whose method is called is inserted in between each given string.\nThe result is returned as a new string.\n\nExample: '.'.join(['ab', 'pq', 'rs']) -> 'ab.pq.rs'", + "builtins.str.ljust" => "Return a left-justified string of length width.\n\nPadding is done using the specified fill character (default is a space).", + "builtins.str.lower" => "Return a copy of the string converted to lowercase.", + "builtins.str.lstrip" => "Return a copy of the string with leading whitespace removed.\n\nIf chars is given and not None, remove characters in chars instead.", + "builtins.str.maketrans" => "Return a translation table usable for str.translate().\n\nIf there is only one argument, it must be a dictionary mapping Unicode\nordinals (integers) or characters to Unicode ordinals, strings or None.\nCharacter keys will be then converted to ordinals.\nIf there are two arguments, they must be strings of equal length, and\nin the resulting dictionary, each character in x will be mapped to the\ncharacter at the same position in y. If there is a third argument, it\nmust be a string, whose characters will be mapped to None in the result.", + "builtins.str.partition" => "Partition the string into three parts using the given separator.\n\nThis will search for the separator in the string. If the separator is found,\nreturns a 3-tuple containing the part before the separator, the separator\nitself, and the part after it.\n\nIf the separator is not found, returns a 3-tuple containing the original string\nand two empty strings.", + "builtins.str.removeprefix" => "Return a str with the given prefix string removed if present.\n\nIf the string starts with the prefix string, return string[len(prefix):].\nOtherwise, return a copy of the original string.", + "builtins.str.removesuffix" => "Return a str with the given suffix string removed if present.\n\nIf the string ends with the suffix string and that suffix is not empty,\nreturn string[:-len(suffix)]. Otherwise, return a copy of the original\nstring.", + "builtins.str.replace" => "Return a copy with all occurrences of substring old replaced by new.\n\n count\n Maximum number of occurrences to replace.\n -1 (the default value) means replace all occurrences.\n\nIf the optional argument count is given, only the first count occurrences are\nreplaced.", + "builtins.str.rfind" => "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n\nOptional arguments start and end are interpreted as in slice notation.\nReturn -1 on failure.", + "builtins.str.rindex" => "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n\nOptional arguments start and end are interpreted as in slice notation.\nRaises ValueError when the substring is not found.", + "builtins.str.rjust" => "Return a right-justified string of length width.\n\nPadding is done using the specified fill character (default is a space).", + "builtins.str.rpartition" => "Partition the string into three parts using the given separator.\n\nThis will search for the separator in the string, starting at the end. If\nthe separator is found, returns a 3-tuple containing the part before the\nseparator, the separator itself, and the part after it.\n\nIf the separator is not found, returns a 3-tuple containing two empty strings\nand the original string.", + "builtins.str.rsplit" => "Return a list of the substrings in the string, using sep as the separator string.\n\n sep\n The separator used to split the string.\n\n When set to None (the default value), will split on any whitespace\n character (including \\n \\r \\t \\f and spaces) and will discard\n empty strings from the result.\n maxsplit\n Maximum number of splits.\n -1 (the default value) means no limit.\n\nSplitting starts at the end of the string and works to the front.", + "builtins.str.rstrip" => "Return a copy of the string with trailing whitespace removed.\n\nIf chars is given and not None, remove characters in chars instead.", + "builtins.str.split" => "Return a list of the substrings in the string, using sep as the separator string.\n\n sep\n The separator used to split the string.\n\n When set to None (the default value), will split on any whitespace\n character (including \\n \\r \\t \\f and spaces) and will discard\n empty strings from the result.\n maxsplit\n Maximum number of splits.\n -1 (the default value) means no limit.\n\nSplitting starts at the front of the string and works to the end.\n\nNote, str.split() is mainly useful for data that has been intentionally\ndelimited. With natural text that includes punctuation, consider using\nthe regular expression module.", + "builtins.str.splitlines" => "Return a list of the lines in the string, breaking at line boundaries.\n\nLine breaks are not included in the resulting list unless keepends is given and\ntrue.", + "builtins.str.startswith" => "Return True if the string starts with the specified prefix, False otherwise.\n\n prefix\n A string or a tuple of strings to try.\n start\n Optional start position. Default: start of the string.\n end\n Optional stop position. Default: end of the string.", + "builtins.str.strip" => "Return a copy of the string with leading and trailing whitespace removed.\n\nIf chars is given and not None, remove characters in chars instead.", + "builtins.str.swapcase" => "Convert uppercase characters to lowercase and lowercase characters to uppercase.", + "builtins.str.title" => "Return a version of the string where each word is titlecased.\n\nMore specifically, words start with uppercased characters and all remaining\ncased characters have lower case.", + "builtins.str.translate" => "Replace each character in the string using the given translation table.\n\n table\n Translation table, which must be a mapping of Unicode ordinals to\n Unicode ordinals, strings, or None.\n\nThe table must implement lookup/indexing via __getitem__, for instance a\ndictionary or list. If this operation raises LookupError, the character is\nleft untouched. Characters mapped to None are deleted.", + "builtins.str.upper" => "Return a copy of the string converted to uppercase.", + "builtins.str.zfill" => "Pad a numeric string with zeros on the left, to fill a field of the given width.\n\nThe string is never truncated.", + "builtins.str_ascii_iterator.__delattr__" => "Implement delattr(self, name).", + "builtins.str_ascii_iterator.__eq__" => "Return self==value.", + "builtins.str_ascii_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.str_ascii_iterator.__ge__" => "Return self>=value.", + "builtins.str_ascii_iterator.__getattribute__" => "Return getattr(self, name).", + "builtins.str_ascii_iterator.__getstate__" => "Helper for pickle.", + "builtins.str_ascii_iterator.__gt__" => "Return self>value.", + "builtins.str_ascii_iterator.__hash__" => "Return hash(self).", + "builtins.str_ascii_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.str_ascii_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.str_ascii_iterator.__iter__" => "Implement iter(self).", + "builtins.str_ascii_iterator.__le__" => "Return self<=value.", + "builtins.str_ascii_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.str_ascii_iterator.__lt__" => "Return self<value.", + "builtins.str_ascii_iterator.__ne__" => "Return self!=value.", + "builtins.str_ascii_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.str_ascii_iterator.__next__" => "Implement next(self).", + "builtins.str_ascii_iterator.__reduce__" => "Return state information for pickling.", + "builtins.str_ascii_iterator.__reduce_ex__" => "Helper for pickle.", + "builtins.str_ascii_iterator.__repr__" => "Return repr(self).", + "builtins.str_ascii_iterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.str_ascii_iterator.__setstate__" => "Set state information for unpickling.", + "builtins.str_ascii_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.str_ascii_iterator.__str__" => "Return str(self).", + "builtins.str_ascii_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.sum" => "Return the sum of a 'start' value (default: 0) plus an iterable of numbers\n\nWhen the iterable is empty, return the start value.\nThis function is intended specifically for use with numeric values and may\nreject non-numeric types.", + "builtins.super" => "super() -> same as super(__class__, <first argument>)\nsuper(type) -> unbound super object\nsuper(type, obj) -> bound super object; requires isinstance(obj, type)\nsuper(type, type2) -> bound super object; requires issubclass(type2, type)\nTypical use to call a cooperative superclass method:\nclass C(B):\n def meth(self, arg):\n super().meth(arg)\nThis works for class methods too:\nclass C(B):\n @classmethod\n def cmeth(cls, arg):\n super().cmeth(arg)", + "builtins.super.__delattr__" => "Implement delattr(self, name).", + "builtins.super.__eq__" => "Return self==value.", + "builtins.super.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.super.__ge__" => "Return self>=value.", + "builtins.super.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.super.__getattribute__" => "Return getattr(self, name).", + "builtins.super.__getstate__" => "Helper for pickle.", + "builtins.super.__gt__" => "Return self>value.", + "builtins.super.__hash__" => "Return hash(self).", + "builtins.super.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.super.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.super.__le__" => "Return self<=value.", + "builtins.super.__lt__" => "Return self<value.", + "builtins.super.__ne__" => "Return self!=value.", + "builtins.super.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.super.__reduce__" => "Helper for pickle.", + "builtins.super.__reduce_ex__" => "Helper for pickle.", + "builtins.super.__repr__" => "Return repr(self).", + "builtins.super.__self__" => "the instance invoking super(); may be None", + "builtins.super.__self_class__" => "the type of the instance invoking super(); may be None", + "builtins.super.__setattr__" => "Implement setattr(self, name, value).", + "builtins.super.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.super.__str__" => "Return str(self).", + "builtins.super.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.super.__thisclass__" => "the class invoking super()", + "builtins.traceback" => "Create a new traceback object.", + "builtins.traceback.__delattr__" => "Implement delattr(self, name).", + "builtins.traceback.__eq__" => "Return self==value.", + "builtins.traceback.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.traceback.__ge__" => "Return self>=value.", + "builtins.traceback.__getattribute__" => "Return getattr(self, name).", + "builtins.traceback.__getstate__" => "Helper for pickle.", + "builtins.traceback.__gt__" => "Return self>value.", + "builtins.traceback.__hash__" => "Return hash(self).", + "builtins.traceback.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.traceback.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.traceback.__le__" => "Return self<=value.", + "builtins.traceback.__lt__" => "Return self<value.", + "builtins.traceback.__ne__" => "Return self!=value.", + "builtins.traceback.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.traceback.__reduce__" => "Helper for pickle.", + "builtins.traceback.__reduce_ex__" => "Helper for pickle.", + "builtins.traceback.__repr__" => "Return repr(self).", + "builtins.traceback.__setattr__" => "Implement setattr(self, name, value).", + "builtins.traceback.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.traceback.__str__" => "Return str(self).", + "builtins.traceback.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.tuple" => "Built-in immutable sequence.\n\nIf no argument is given, the constructor returns an empty tuple.\nIf iterable is specified the tuple is initialized from iterable's items.\n\nIf the argument is a tuple, the return value is the same object.", + "builtins.tuple.__add__" => "Return self+value.", + "builtins.tuple.__class_getitem__" => "See PEP 585", + "builtins.tuple.__contains__" => "Return bool(key in self).", + "builtins.tuple.__delattr__" => "Implement delattr(self, name).", + "builtins.tuple.__eq__" => "Return self==value.", + "builtins.tuple.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.tuple.__ge__" => "Return self>=value.", + "builtins.tuple.__getattribute__" => "Return getattr(self, name).", + "builtins.tuple.__getitem__" => "Return self[key].", + "builtins.tuple.__getstate__" => "Helper for pickle.", + "builtins.tuple.__gt__" => "Return self>value.", + "builtins.tuple.__hash__" => "Return hash(self).", + "builtins.tuple.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.tuple.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.tuple.__iter__" => "Implement iter(self).", + "builtins.tuple.__le__" => "Return self<=value.", + "builtins.tuple.__len__" => "Return len(self).", + "builtins.tuple.__lt__" => "Return self<value.", + "builtins.tuple.__mul__" => "Return self*value.", + "builtins.tuple.__ne__" => "Return self!=value.", + "builtins.tuple.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.tuple.__reduce__" => "Helper for pickle.", + "builtins.tuple.__reduce_ex__" => "Helper for pickle.", + "builtins.tuple.__repr__" => "Return repr(self).", + "builtins.tuple.__rmul__" => "Return value*self.", + "builtins.tuple.__setattr__" => "Implement setattr(self, name, value).", + "builtins.tuple.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.tuple.__str__" => "Return str(self).", + "builtins.tuple.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.tuple.count" => "Return number of occurrences of value.", + "builtins.tuple.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "builtins.tuple_iterator.__delattr__" => "Implement delattr(self, name).", + "builtins.tuple_iterator.__eq__" => "Return self==value.", + "builtins.tuple_iterator.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.tuple_iterator.__ge__" => "Return self>=value.", + "builtins.tuple_iterator.__getattribute__" => "Return getattr(self, name).", + "builtins.tuple_iterator.__getstate__" => "Helper for pickle.", + "builtins.tuple_iterator.__gt__" => "Return self>value.", + "builtins.tuple_iterator.__hash__" => "Return hash(self).", + "builtins.tuple_iterator.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.tuple_iterator.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.tuple_iterator.__iter__" => "Implement iter(self).", + "builtins.tuple_iterator.__le__" => "Return self<=value.", + "builtins.tuple_iterator.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "builtins.tuple_iterator.__lt__" => "Return self<value.", + "builtins.tuple_iterator.__ne__" => "Return self!=value.", + "builtins.tuple_iterator.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.tuple_iterator.__next__" => "Implement next(self).", + "builtins.tuple_iterator.__reduce__" => "Return state information for pickling.", + "builtins.tuple_iterator.__reduce_ex__" => "Helper for pickle.", + "builtins.tuple_iterator.__repr__" => "Return repr(self).", + "builtins.tuple_iterator.__setattr__" => "Implement setattr(self, name, value).", + "builtins.tuple_iterator.__setstate__" => "Set state information for unpickling.", + "builtins.tuple_iterator.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.tuple_iterator.__str__" => "Return str(self).", + "builtins.tuple_iterator.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.type" => "type(object) -> the object's type\ntype(name, bases, dict, **kwds) -> a new type", + "builtins.type.__base__" => "The base class of the class hierarchy.\n\nWhen called, it accepts no arguments and returns a new featureless\ninstance that has no instance attributes and cannot be given any.", + "builtins.type.__base__.__delattr__" => "Implement delattr(self, name).", + "builtins.type.__base__.__eq__" => "Return self==value.", + "builtins.type.__base__.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.type.__base__.__ge__" => "Return self>=value.", + "builtins.type.__base__.__getattribute__" => "Return getattr(self, name).", + "builtins.type.__base__.__getstate__" => "Helper for pickle.", + "builtins.type.__base__.__gt__" => "Return self>value.", + "builtins.type.__base__.__hash__" => "Return hash(self).", + "builtins.type.__base__.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.type.__base__.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.type.__base__.__le__" => "Return self<=value.", + "builtins.type.__base__.__lt__" => "Return self<value.", + "builtins.type.__base__.__ne__" => "Return self!=value.", + "builtins.type.__base__.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.type.__base__.__reduce__" => "Helper for pickle.", + "builtins.type.__base__.__reduce_ex__" => "Helper for pickle.", + "builtins.type.__base__.__repr__" => "Return repr(self).", + "builtins.type.__base__.__setattr__" => "Implement setattr(self, name, value).", + "builtins.type.__base__.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.type.__base__.__str__" => "Return str(self).", + "builtins.type.__base__.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.type.__call__" => "Call self as a function.", + "builtins.type.__delattr__" => "Implement delattr(self, name).", + "builtins.type.__eq__" => "Return self==value.", + "builtins.type.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.type.__ge__" => "Return self>=value.", + "builtins.type.__getattribute__" => "Return getattr(self, name).", + "builtins.type.__getstate__" => "Helper for pickle.", + "builtins.type.__gt__" => "Return self>value.", + "builtins.type.__hash__" => "Return hash(self).", + "builtins.type.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.type.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.type.__instancecheck__" => "Check if an object is an instance.", + "builtins.type.__le__" => "Return self<=value.", + "builtins.type.__lt__" => "Return self<value.", + "builtins.type.__ne__" => "Return self!=value.", + "builtins.type.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.type.__or__" => "Return self|value.", + "builtins.type.__prepare__" => "Create the namespace for the class statement", + "builtins.type.__reduce__" => "Helper for pickle.", + "builtins.type.__reduce_ex__" => "Helper for pickle.", + "builtins.type.__repr__" => "Return repr(self).", + "builtins.type.__ror__" => "Return value|self.", + "builtins.type.__setattr__" => "Implement setattr(self, name, value).", + "builtins.type.__sizeof__" => "Return memory consumption of the type object.", + "builtins.type.__str__" => "Return str(self).", + "builtins.type.__subclasscheck__" => "Check if a class is a subclass.", + "builtins.type.__subclasses__" => "Return a list of immediate subclasses.", + "builtins.type.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.type.mro" => "Return a type's method resolution order.", + "builtins.vars" => "vars([object]) -> dictionary\n\nWithout arguments, equivalent to locals().\nWith an argument, equivalent to object.__dict__.", + "builtins.wrapper_descriptor.__call__" => "Call self as a function.", + "builtins.wrapper_descriptor.__delattr__" => "Implement delattr(self, name).", + "builtins.wrapper_descriptor.__eq__" => "Return self==value.", + "builtins.wrapper_descriptor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.wrapper_descriptor.__ge__" => "Return self>=value.", + "builtins.wrapper_descriptor.__get__" => "Return an attribute of instance, which is of type owner.", + "builtins.wrapper_descriptor.__getattribute__" => "Return getattr(self, name).", + "builtins.wrapper_descriptor.__getstate__" => "Helper for pickle.", + "builtins.wrapper_descriptor.__gt__" => "Return self>value.", + "builtins.wrapper_descriptor.__hash__" => "Return hash(self).", + "builtins.wrapper_descriptor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.wrapper_descriptor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.wrapper_descriptor.__le__" => "Return self<=value.", + "builtins.wrapper_descriptor.__lt__" => "Return self<value.", + "builtins.wrapper_descriptor.__ne__" => "Return self!=value.", + "builtins.wrapper_descriptor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.wrapper_descriptor.__reduce_ex__" => "Helper for pickle.", + "builtins.wrapper_descriptor.__repr__" => "Return repr(self).", + "builtins.wrapper_descriptor.__setattr__" => "Implement setattr(self, name, value).", + "builtins.wrapper_descriptor.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.wrapper_descriptor.__str__" => "Return str(self).", + "builtins.wrapper_descriptor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "builtins.zip" => "The zip object yields n-length tuples, where n is the number of iterables\npassed as positional arguments to zip(). The i-th element in every tuple\ncomes from the i-th iterable argument to zip(). This continues until the\nshortest argument is exhausted.\n\nIf strict is true and one of the arguments is exhausted before the others,\nraise a ValueError.\n\n >>> list(zip('abcdefg', range(3), range(4)))\n [('a', 0, 0), ('b', 1, 1), ('c', 2, 2)]", + "builtins.zip.__delattr__" => "Implement delattr(self, name).", + "builtins.zip.__eq__" => "Return self==value.", + "builtins.zip.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "builtins.zip.__ge__" => "Return self>=value.", + "builtins.zip.__getattribute__" => "Return getattr(self, name).", + "builtins.zip.__getstate__" => "Helper for pickle.", + "builtins.zip.__gt__" => "Return self>value.", + "builtins.zip.__hash__" => "Return hash(self).", + "builtins.zip.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "builtins.zip.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "builtins.zip.__iter__" => "Implement iter(self).", + "builtins.zip.__le__" => "Return self<=value.", + "builtins.zip.__lt__" => "Return self<value.", + "builtins.zip.__ne__" => "Return self!=value.", + "builtins.zip.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "builtins.zip.__next__" => "Implement next(self).", + "builtins.zip.__reduce__" => "Return state information for pickling.", + "builtins.zip.__reduce_ex__" => "Helper for pickle.", + "builtins.zip.__repr__" => "Return repr(self).", + "builtins.zip.__setattr__" => "Implement setattr(self, name, value).", + "builtins.zip.__setstate__" => "Set state information for unpickling.", + "builtins.zip.__sizeof__" => "Size of object in memory, in bytes.", + "builtins.zip.__str__" => "Return str(self).", + "builtins.zip.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "cmath" => "This module provides access to mathematical functions for complex\nnumbers.", + "cmath.acos" => "Return the arc cosine of z.", + "cmath.acosh" => "Return the inverse hyperbolic cosine of z.", + "cmath.asin" => "Return the arc sine of z.", + "cmath.asinh" => "Return the inverse hyperbolic sine of z.", + "cmath.atan" => "Return the arc tangent of z.", + "cmath.atanh" => "Return the inverse hyperbolic tangent of z.", + "cmath.cos" => "Return the cosine of z.", + "cmath.cosh" => "Return the hyperbolic cosine of z.", + "cmath.exp" => "Return the exponential value e**z.", + "cmath.isclose" => "Determine whether two complex numbers are close in value.\n\n rel_tol\n maximum difference for being considered \"close\", relative to the\n magnitude of the input values\n abs_tol\n maximum difference for being considered \"close\", regardless of the\n magnitude of the input values\n\nReturn True if a is close in value to b, and False otherwise.\n\nFor the values to be considered close, the difference between them must be\nsmaller than at least one of the tolerances.\n\n-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is\nnot close to anything, even itself. inf and -inf are only close to themselves.", + "cmath.isfinite" => "Return True if both the real and imaginary parts of z are finite, else False.", + "cmath.isinf" => "Checks if the real or imaginary part of z is infinite.", + "cmath.isnan" => "Checks if the real or imaginary part of z not a number (NaN).", + "cmath.log" => "log(z[, base]) -> the logarithm of z to the given base.\n\nIf the base is not specified, returns the natural logarithm (base e) of z.", + "cmath.log10" => "Return the base-10 logarithm of z.", + "cmath.phase" => "Return argument, also known as the phase angle, of a complex.", + "cmath.polar" => "Convert a complex from rectangular coordinates to polar coordinates.\n\nr is the distance from 0 and phi the phase angle.", + "cmath.rect" => "Convert from polar coordinates to rectangular coordinates.", + "cmath.sin" => "Return the sine of z.", + "cmath.sinh" => "Return the hyperbolic sine of z.", + "cmath.sqrt" => "Return the square root of z.", + "cmath.tan" => "Return the tangent of z.", + "cmath.tanh" => "Return the hyperbolic tangent of z.", + "errno" => "This module makes available standard errno system symbols.\n\nThe value of each symbol is the corresponding integer value,\ne.g., on most systems, errno.ENOENT equals the integer 2.\n\nThe dictionary errno.errorcode maps numeric codes to symbol names,\ne.g., errno.errorcode[2] could be the string 'ENOENT'.\n\nSymbols that are not relevant to the underlying system are not defined.\n\nTo map error codes to error messages, use the function os.strerror(),\ne.g. os.strerror(2) could return 'No such file or directory'.", + "faulthandler" => "faulthandler module.", + "faulthandler._fatal_error_c_thread" => "Call Py_FatalError() in a new C thread.", + "faulthandler._raise_exception" => "Call RaiseException(code, flags).", + "faulthandler._read_null" => "Read from NULL, raise a SIGSEGV or SIGBUS signal depending on the platform.", + "faulthandler._sigabrt" => "Raise a SIGABRT signal.", + "faulthandler._sigfpe" => "Raise a SIGFPE signal.", + "faulthandler._sigsegv" => "Raise a SIGSEGV signal.", + "faulthandler._stack_overflow" => "Recursive call to raise a stack overflow.", + "faulthandler.cancel_dump_traceback_later" => "Cancel the previous call to dump_traceback_later().", + "faulthandler.disable" => "Disable the fault handler.", + "faulthandler.dump_c_stack" => "Dump the C stack of the current thread.", + "faulthandler.dump_traceback" => "Dump the traceback of the current thread, or of all threads if all_threads is True, into file.", + "faulthandler.dump_traceback_later" => "Dump the traceback of all threads in timeout seconds,\nor each timeout seconds if repeat is True. If exit is True, call _exit(1) which is not safe.", + "faulthandler.enable" => "Enable the fault handler.", + "faulthandler.is_enabled" => "Check if the handler is enabled.", + "faulthandler.register" => "Register a handler for the signal 'signum': dump the traceback of the current thread, or of all threads if all_threads is True, into file.", + "faulthandler.unregister" => "Unregister the handler of the signal 'signum' registered by register().", + "fcntl" => "This module performs file control and I/O control on file\ndescriptors. It is an interface to the fcntl() and ioctl() Unix\nroutines. File descriptors can be obtained with the fileno() method of\na file or socket object.", + "fcntl.fcntl" => "Perform the operation cmd on file descriptor fd.\n\nThe values used for cmd are operating system dependent, and are\navailable as constants in the fcntl module, using the same names as used\nin the relevant C header files. The argument arg is optional, and\ndefaults to 0; it may be an integer, a bytes-like object or a string.\nIf arg is given as a string, it will be encoded to binary using the\nUTF-8 encoding.\n\nIf the arg given is an integer or if none is specified, the result value\nis an integer corresponding to the return value of the fcntl() call in\nthe C code.\n\nIf arg is given as a bytes-like object, the return value of fcntl() is a\nbytes object of that length, containing the resulting value put in the\narg buffer by the operating system. The length of the arg buffer is not\nallowed to exceed 1024 bytes.", + "fcntl.flock" => "Perform the lock operation on file descriptor fd.\n\nSee the Unix manual page for flock(2) for details (On some systems, this\nfunction is emulated using fcntl()).", + "fcntl.ioctl" => "Perform the operation request on file descriptor fd.\n\nThe values used for request are operating system dependent, and are\navailable as constants in the fcntl or termios library modules, using\nthe same names as used in the relevant C header files.\n\nThe argument arg is optional, and defaults to 0; it may be an integer, a\nbytes-like object or a string. If arg is given as a string, it will be\nencoded to binary using the UTF-8 encoding.\n\nIf the arg given is an integer or if none is specified, the result value\nis an integer corresponding to the return value of the ioctl() call in\nthe C code.\n\nIf the argument is a mutable buffer (such as a bytearray) and the\nmutate_flag argument is true (default) then the buffer is (in effect)\npassed to the operating system and changes made by the OS will be\nreflected in the contents of the buffer after the call has returned.\nThe return value is the integer returned by the ioctl() system call.\n\nIf the argument is a mutable buffer and the mutable_flag argument is\nfalse, the behavior is as if an immutable buffer had been passed.\n\nIf the argument is an immutable buffer then a copy of the buffer is\npassed to the operating system and the return value is a bytes object of\nthe same length containing whatever the operating system put in the\nbuffer. The length of the arg buffer in this case is not allowed to\nexceed 1024 bytes.", + "fcntl.lockf" => "A wrapper around the fcntl() locking calls.\n\nfd is the file descriptor of the file to lock or unlock, and operation\nis one of the following values:\n\n LOCK_UN - unlock\n LOCK_SH - acquire a shared lock\n LOCK_EX - acquire an exclusive lock\n\nWhen operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\nLOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and\nthe lock cannot be acquired, an OSError will be raised and the exception\nwill have an errno attribute set to EACCES or EAGAIN (depending on the\noperating system -- for portability, check for either value).\n\nlen is the number of bytes to lock, with the default meaning to lock to\nEOF. start is the byte offset, relative to whence, to that the lock\nstarts. whence is as with fileobj.seek(), specifically:\n\n 0 - relative to the start of the file (SEEK_SET)\n 1 - relative to the current buffer position (SEEK_CUR)\n 2 - relative to the end of the file (SEEK_END)", + "gc" => "This module provides access to the garbage collector for reference cycles.\n\nenable() -- Enable automatic garbage collection.\ndisable() -- Disable automatic garbage collection.\nisenabled() -- Returns true if automatic collection is enabled.\ncollect() -- Do a full collection right now.\nget_count() -- Return the current collection counts.\nget_stats() -- Return list of dictionaries containing per-generation stats.\nset_debug() -- Set debugging flags.\nget_debug() -- Get debugging flags.\nset_threshold() -- Set the collection thresholds.\nget_threshold() -- Return the current collection thresholds.\nget_objects() -- Return a list of all objects tracked by the collector.\nis_tracked() -- Returns true if a given object is tracked.\nis_finalized() -- Returns true if a given object has been already finalized.\nget_referrers() -- Return the list of objects that refer to an object.\nget_referents() -- Return the list of objects that an object refers to.\nfreeze() -- Freeze all tracked objects and ignore them for future collections.\nunfreeze() -- Unfreeze all objects in the permanent generation.\nget_freeze_count() -- Return the number of objects in the permanent generation.", + "gc.collect" => "Run the garbage collector.\n\nWith no arguments, run a full collection. The optional argument\nmay be an integer specifying which generation to collect. A ValueError\nis raised if the generation number is invalid.\n\nThe number of unreachable objects is returned.", + "gc.disable" => "Disable automatic garbage collection.", + "gc.enable" => "Enable automatic garbage collection.", + "gc.freeze" => "Freeze all current tracked objects and ignore them for future collections.\n\nThis can be used before a POSIX fork() call to make the gc copy-on-write friendly.\nNote: collection before a POSIX fork() call may free pages for future allocation\nwhich can cause copy-on-write.", + "gc.get_count" => "Return a three-tuple of the current collection counts.", + "gc.get_debug" => "Get the garbage collection debugging flags.", + "gc.get_freeze_count" => "Return the number of objects in the permanent generation.", + "gc.get_objects" => "Return a list of objects tracked by the collector (excluding the list returned).\n\n generation\n Generation to extract the objects from.\n\nIf generation is not None, return only the objects tracked by the collector\nthat are in that generation.", + "gc.get_referents" => "Return the list of objects that are directly referred to by 'objs'.", + "gc.get_referrers" => "Return the list of objects that directly refer to any of 'objs'.", + "gc.get_stats" => "Return a list of dictionaries containing per-generation statistics.", + "gc.get_threshold" => "Return the current collection thresholds.", + "gc.is_finalized" => "Returns true if the object has been already finalized by the GC.", + "gc.is_tracked" => "Returns true if the object is tracked by the garbage collector.\n\nSimple atomic objects will return false.", + "gc.isenabled" => "Returns true if automatic garbage collection is enabled.", + "gc.set_debug" => "Set the garbage collection debugging flags.\n\n flags\n An integer that can have the following bits turned on:\n DEBUG_STATS - Print statistics during collection.\n DEBUG_COLLECTABLE - Print collectable objects found.\n DEBUG_UNCOLLECTABLE - Print unreachable but uncollectable objects\n found.\n DEBUG_SAVEALL - Save objects to gc.garbage rather than freeing them.\n DEBUG_LEAK - Debug leaking programs (everything but STATS).\n\nDebugging information is written to sys.stderr.", + "gc.set_threshold" => "set_threshold(threshold0, [threshold1, [threshold2]])\nSet the collection thresholds (the collection frequency).\n\nSetting 'threshold0' to zero disables collection.", + "gc.unfreeze" => "Unfreeze all objects in the permanent generation.\n\nPut all objects in the permanent generation back into oldest generation.", + "grp" => "Access to the Unix group database.\n\nGroup entries are reported as 4-tuples containing the following fields\nfrom the group database, in order:\n\n gr_name - name of the group\n gr_passwd - group password (encrypted); often empty\n gr_gid - numeric ID of the group\n gr_mem - list of members\n\nThe gid is an integer, name and password are strings. (Note that most\nusers are not explicitly listed as members of the groups they are in\naccording to the password database. Check both databases to get\ncomplete membership information.)", + "grp.getgrall" => "Return a list of all available group entries, in arbitrary order.\n\nAn entry whose name starts with '+' or '-' represents an instruction\nto use YP/NIS and may not be accessible via getgrnam or getgrgid.", + "grp.getgrgid" => "Return the group database entry for the given numeric group ID.\n\nIf id is not valid, raise KeyError.", + "grp.getgrnam" => "Return the group database entry for the given group name.\n\nIf name is not valid, raise KeyError.", + "grp.struct_group" => "grp.struct_group: Results from getgr*() routines.\n\nThis object may be accessed either as a tuple of\n (gr_name,gr_passwd,gr_gid,gr_mem)\nor via the object attributes as named in the above tuple.", + "grp.struct_group.__add__" => "Return self+value.", + "grp.struct_group.__class_getitem__" => "See PEP 585", + "grp.struct_group.__contains__" => "Return bool(key in self).", + "grp.struct_group.__delattr__" => "Implement delattr(self, name).", + "grp.struct_group.__eq__" => "Return self==value.", + "grp.struct_group.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "grp.struct_group.__ge__" => "Return self>=value.", + "grp.struct_group.__getattribute__" => "Return getattr(self, name).", + "grp.struct_group.__getitem__" => "Return self[key].", + "grp.struct_group.__getstate__" => "Helper for pickle.", + "grp.struct_group.__gt__" => "Return self>value.", + "grp.struct_group.__hash__" => "Return hash(self).", + "grp.struct_group.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "grp.struct_group.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "grp.struct_group.__iter__" => "Implement iter(self).", + "grp.struct_group.__le__" => "Return self<=value.", + "grp.struct_group.__len__" => "Return len(self).", + "grp.struct_group.__lt__" => "Return self<value.", + "grp.struct_group.__mul__" => "Return self*value.", + "grp.struct_group.__ne__" => "Return self!=value.", + "grp.struct_group.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "grp.struct_group.__reduce_ex__" => "Helper for pickle.", + "grp.struct_group.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "grp.struct_group.__repr__" => "Return repr(self).", + "grp.struct_group.__rmul__" => "Return value*self.", + "grp.struct_group.__setattr__" => "Implement setattr(self, name, value).", + "grp.struct_group.__sizeof__" => "Size of object in memory, in bytes.", + "grp.struct_group.__str__" => "Return str(self).", + "grp.struct_group.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "grp.struct_group.count" => "Return number of occurrences of value.", + "grp.struct_group.gr_gid" => "group id", + "grp.struct_group.gr_mem" => "group members", + "grp.struct_group.gr_name" => "group name", + "grp.struct_group.gr_passwd" => "password", + "grp.struct_group.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "itertools" => "Functional tools for creating and using iterators.\n\nInfinite iterators:\ncount(start=0, step=1) --> start, start+step, start+2*step, ...\ncycle(p) --> p0, p1, ... plast, p0, p1, ...\nrepeat(elem [,n]) --> elem, elem, elem, ... endlessly or up to n times\n\nIterators terminating on the shortest input sequence:\naccumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\nbatched(p, n) --> [p0, p1, ..., p_n-1], [p_n, p_n+1, ..., p_2n-1], ...\nchain(p, q, ...) --> p0, p1, ... plast, q0, q1, ...\nchain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ...\ncompress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\ndropwhile(predicate, seq) --> seq[n], seq[n+1], starting when predicate fails\ngroupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\nfilterfalse(predicate, seq) --> elements of seq where predicate(elem) is False\nislice(seq, [start,] stop [, step]) --> elements from\n seq[start:stop:step]\npairwise(s) --> (s[0],s[1]), (s[1],s[2]), (s[2], s[3]), ...\nstarmap(fun, seq) --> fun(*seq[0]), fun(*seq[1]), ...\ntee(it, n=2) --> (it1, it2 , ... itn) splits one iterator into n\ntakewhile(predicate, seq) --> seq[0], seq[1], until predicate fails\nzip_longest(p, q, ...) --> (p[0], q[0]), (p[1], q[1]), ...\n\nCombinatoric generators:\nproduct(p, q, ... [repeat=1]) --> cartesian product\npermutations(p[, r])\ncombinations(p, r)\ncombinations_with_replacement(p, r)", + "itertools._grouper.__delattr__" => "Implement delattr(self, name).", + "itertools._grouper.__eq__" => "Return self==value.", + "itertools._grouper.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools._grouper.__ge__" => "Return self>=value.", + "itertools._grouper.__getattribute__" => "Return getattr(self, name).", + "itertools._grouper.__getstate__" => "Helper for pickle.", + "itertools._grouper.__gt__" => "Return self>value.", + "itertools._grouper.__hash__" => "Return hash(self).", + "itertools._grouper.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools._grouper.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools._grouper.__iter__" => "Implement iter(self).", + "itertools._grouper.__le__" => "Return self<=value.", + "itertools._grouper.__lt__" => "Return self<value.", + "itertools._grouper.__ne__" => "Return self!=value.", + "itertools._grouper.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools._grouper.__next__" => "Implement next(self).", + "itertools._grouper.__reduce__" => "Helper for pickle.", + "itertools._grouper.__reduce_ex__" => "Helper for pickle.", + "itertools._grouper.__repr__" => "Return repr(self).", + "itertools._grouper.__setattr__" => "Implement setattr(self, name, value).", + "itertools._grouper.__sizeof__" => "Size of object in memory, in bytes.", + "itertools._grouper.__str__" => "Return str(self).", + "itertools._grouper.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools._tee" => "Iterator wrapped to make it copyable.", + "itertools._tee.__copy__" => "Returns an independent iterator.", + "itertools._tee.__delattr__" => "Implement delattr(self, name).", + "itertools._tee.__eq__" => "Return self==value.", + "itertools._tee.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools._tee.__ge__" => "Return self>=value.", + "itertools._tee.__getattribute__" => "Return getattr(self, name).", + "itertools._tee.__getstate__" => "Helper for pickle.", + "itertools._tee.__gt__" => "Return self>value.", + "itertools._tee.__hash__" => "Return hash(self).", + "itertools._tee.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools._tee.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools._tee.__iter__" => "Implement iter(self).", + "itertools._tee.__le__" => "Return self<=value.", + "itertools._tee.__lt__" => "Return self<value.", + "itertools._tee.__ne__" => "Return self!=value.", + "itertools._tee.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools._tee.__next__" => "Implement next(self).", + "itertools._tee.__reduce__" => "Helper for pickle.", + "itertools._tee.__reduce_ex__" => "Helper for pickle.", + "itertools._tee.__repr__" => "Return repr(self).", + "itertools._tee.__setattr__" => "Implement setattr(self, name, value).", + "itertools._tee.__sizeof__" => "Size of object in memory, in bytes.", + "itertools._tee.__str__" => "Return str(self).", + "itertools._tee.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools._tee_dataobject" => "teedataobject(iterable, values, next, /)\n--\n\nData container common to multiple tee objects.", + "itertools._tee_dataobject.__delattr__" => "Implement delattr(self, name).", + "itertools._tee_dataobject.__eq__" => "Return self==value.", + "itertools._tee_dataobject.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools._tee_dataobject.__ge__" => "Return self>=value.", + "itertools._tee_dataobject.__getattribute__" => "Return getattr(self, name).", + "itertools._tee_dataobject.__getstate__" => "Helper for pickle.", + "itertools._tee_dataobject.__gt__" => "Return self>value.", + "itertools._tee_dataobject.__hash__" => "Return hash(self).", + "itertools._tee_dataobject.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools._tee_dataobject.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools._tee_dataobject.__le__" => "Return self<=value.", + "itertools._tee_dataobject.__lt__" => "Return self<value.", + "itertools._tee_dataobject.__ne__" => "Return self!=value.", + "itertools._tee_dataobject.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools._tee_dataobject.__reduce__" => "Helper for pickle.", + "itertools._tee_dataobject.__reduce_ex__" => "Helper for pickle.", + "itertools._tee_dataobject.__repr__" => "Return repr(self).", + "itertools._tee_dataobject.__setattr__" => "Implement setattr(self, name, value).", + "itertools._tee_dataobject.__sizeof__" => "Size of object in memory, in bytes.", + "itertools._tee_dataobject.__str__" => "Return str(self).", + "itertools._tee_dataobject.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.accumulate" => "Return series of accumulated sums (or other binary function results).", + "itertools.accumulate.__delattr__" => "Implement delattr(self, name).", + "itertools.accumulate.__eq__" => "Return self==value.", + "itertools.accumulate.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.accumulate.__ge__" => "Return self>=value.", + "itertools.accumulate.__getattribute__" => "Return getattr(self, name).", + "itertools.accumulate.__getstate__" => "Helper for pickle.", + "itertools.accumulate.__gt__" => "Return self>value.", + "itertools.accumulate.__hash__" => "Return hash(self).", + "itertools.accumulate.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.accumulate.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.accumulate.__iter__" => "Implement iter(self).", + "itertools.accumulate.__le__" => "Return self<=value.", + "itertools.accumulate.__lt__" => "Return self<value.", + "itertools.accumulate.__ne__" => "Return self!=value.", + "itertools.accumulate.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.accumulate.__next__" => "Implement next(self).", + "itertools.accumulate.__reduce__" => "Helper for pickle.", + "itertools.accumulate.__reduce_ex__" => "Helper for pickle.", + "itertools.accumulate.__repr__" => "Return repr(self).", + "itertools.accumulate.__setattr__" => "Implement setattr(self, name, value).", + "itertools.accumulate.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.accumulate.__str__" => "Return str(self).", + "itertools.accumulate.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.batched" => "Batch data into tuples of length n. The last batch may be shorter than n.\n\nLoops over the input iterable and accumulates data into tuples\nup to size n. The input is consumed lazily, just enough to\nfill a batch. The result is yielded as soon as a batch is full\nor when the input iterable is exhausted.\n\n >>> for batch in batched('ABCDEFG', 3):\n ... print(batch)\n ...\n ('A', 'B', 'C')\n ('D', 'E', 'F')\n ('G',)\n\nIf \"strict\" is True, raises a ValueError if the final batch is shorter\nthan n.", + "itertools.batched.__delattr__" => "Implement delattr(self, name).", + "itertools.batched.__eq__" => "Return self==value.", + "itertools.batched.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.batched.__ge__" => "Return self>=value.", + "itertools.batched.__getattribute__" => "Return getattr(self, name).", + "itertools.batched.__getstate__" => "Helper for pickle.", + "itertools.batched.__gt__" => "Return self>value.", + "itertools.batched.__hash__" => "Return hash(self).", + "itertools.batched.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.batched.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.batched.__iter__" => "Implement iter(self).", + "itertools.batched.__le__" => "Return self<=value.", + "itertools.batched.__lt__" => "Return self<value.", + "itertools.batched.__ne__" => "Return self!=value.", + "itertools.batched.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.batched.__next__" => "Implement next(self).", + "itertools.batched.__reduce__" => "Helper for pickle.", + "itertools.batched.__reduce_ex__" => "Helper for pickle.", + "itertools.batched.__repr__" => "Return repr(self).", + "itertools.batched.__setattr__" => "Implement setattr(self, name, value).", + "itertools.batched.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.batched.__str__" => "Return str(self).", + "itertools.batched.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.chain" => "Return a chain object whose .__next__() method returns elements from the\nfirst iterable until it is exhausted, then elements from the next\niterable, until all of the iterables are exhausted.", + "itertools.chain.__class_getitem__" => "See PEP 585", + "itertools.chain.__delattr__" => "Implement delattr(self, name).", + "itertools.chain.__eq__" => "Return self==value.", + "itertools.chain.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.chain.__ge__" => "Return self>=value.", + "itertools.chain.__getattribute__" => "Return getattr(self, name).", + "itertools.chain.__getstate__" => "Helper for pickle.", + "itertools.chain.__gt__" => "Return self>value.", + "itertools.chain.__hash__" => "Return hash(self).", + "itertools.chain.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.chain.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.chain.__iter__" => "Implement iter(self).", + "itertools.chain.__le__" => "Return self<=value.", + "itertools.chain.__lt__" => "Return self<value.", + "itertools.chain.__ne__" => "Return self!=value.", + "itertools.chain.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.chain.__next__" => "Implement next(self).", + "itertools.chain.__reduce__" => "Helper for pickle.", + "itertools.chain.__reduce_ex__" => "Helper for pickle.", + "itertools.chain.__repr__" => "Return repr(self).", + "itertools.chain.__setattr__" => "Implement setattr(self, name, value).", + "itertools.chain.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.chain.__str__" => "Return str(self).", + "itertools.chain.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.chain.from_iterable" => "Alternative chain() constructor taking a single iterable argument that evaluates lazily.", + "itertools.combinations" => "Return successive r-length combinations of elements in the iterable.\n\ncombinations(range(4), 3) --> (0,1,2), (0,1,3), (0,2,3), (1,2,3)", + "itertools.combinations.__delattr__" => "Implement delattr(self, name).", + "itertools.combinations.__eq__" => "Return self==value.", + "itertools.combinations.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.combinations.__ge__" => "Return self>=value.", + "itertools.combinations.__getattribute__" => "Return getattr(self, name).", + "itertools.combinations.__getstate__" => "Helper for pickle.", + "itertools.combinations.__gt__" => "Return self>value.", + "itertools.combinations.__hash__" => "Return hash(self).", + "itertools.combinations.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.combinations.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.combinations.__iter__" => "Implement iter(self).", + "itertools.combinations.__le__" => "Return self<=value.", + "itertools.combinations.__lt__" => "Return self<value.", + "itertools.combinations.__ne__" => "Return self!=value.", + "itertools.combinations.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.combinations.__next__" => "Implement next(self).", + "itertools.combinations.__reduce__" => "Helper for pickle.", + "itertools.combinations.__reduce_ex__" => "Helper for pickle.", + "itertools.combinations.__repr__" => "Return repr(self).", + "itertools.combinations.__setattr__" => "Implement setattr(self, name, value).", + "itertools.combinations.__sizeof__" => "Returns size in memory, in bytes.", + "itertools.combinations.__str__" => "Return str(self).", + "itertools.combinations.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.combinations_with_replacement" => "Return successive r-length combinations of elements in the iterable allowing individual elements to have successive repeats.\n\ncombinations_with_replacement('ABC', 2) --> ('A','A'), ('A','B'), ('A','C'), ('B','B'), ('B','C'), ('C','C')", + "itertools.combinations_with_replacement.__delattr__" => "Implement delattr(self, name).", + "itertools.combinations_with_replacement.__eq__" => "Return self==value.", + "itertools.combinations_with_replacement.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.combinations_with_replacement.__ge__" => "Return self>=value.", + "itertools.combinations_with_replacement.__getattribute__" => "Return getattr(self, name).", + "itertools.combinations_with_replacement.__getstate__" => "Helper for pickle.", + "itertools.combinations_with_replacement.__gt__" => "Return self>value.", + "itertools.combinations_with_replacement.__hash__" => "Return hash(self).", + "itertools.combinations_with_replacement.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.combinations_with_replacement.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.combinations_with_replacement.__iter__" => "Implement iter(self).", + "itertools.combinations_with_replacement.__le__" => "Return self<=value.", + "itertools.combinations_with_replacement.__lt__" => "Return self<value.", + "itertools.combinations_with_replacement.__ne__" => "Return self!=value.", + "itertools.combinations_with_replacement.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.combinations_with_replacement.__next__" => "Implement next(self).", + "itertools.combinations_with_replacement.__reduce__" => "Helper for pickle.", + "itertools.combinations_with_replacement.__reduce_ex__" => "Helper for pickle.", + "itertools.combinations_with_replacement.__repr__" => "Return repr(self).", + "itertools.combinations_with_replacement.__setattr__" => "Implement setattr(self, name, value).", + "itertools.combinations_with_replacement.__sizeof__" => "Returns size in memory, in bytes.", + "itertools.combinations_with_replacement.__str__" => "Return str(self).", + "itertools.combinations_with_replacement.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.compress" => "Return data elements corresponding to true selector elements.\n\nForms a shorter iterator from selected data elements using the selectors to\nchoose the data elements.", + "itertools.compress.__delattr__" => "Implement delattr(self, name).", + "itertools.compress.__eq__" => "Return self==value.", + "itertools.compress.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.compress.__ge__" => "Return self>=value.", + "itertools.compress.__getattribute__" => "Return getattr(self, name).", + "itertools.compress.__getstate__" => "Helper for pickle.", + "itertools.compress.__gt__" => "Return self>value.", + "itertools.compress.__hash__" => "Return hash(self).", + "itertools.compress.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.compress.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.compress.__iter__" => "Implement iter(self).", + "itertools.compress.__le__" => "Return self<=value.", + "itertools.compress.__lt__" => "Return self<value.", + "itertools.compress.__ne__" => "Return self!=value.", + "itertools.compress.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.compress.__next__" => "Implement next(self).", + "itertools.compress.__reduce__" => "Helper for pickle.", + "itertools.compress.__reduce_ex__" => "Helper for pickle.", + "itertools.compress.__repr__" => "Return repr(self).", + "itertools.compress.__setattr__" => "Implement setattr(self, name, value).", + "itertools.compress.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.compress.__str__" => "Return str(self).", + "itertools.compress.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.count" => "Return a count object whose .__next__() method returns consecutive values.\n\nEquivalent to:\n def count(firstval=0, step=1):\n x = firstval\n while 1:\n yield x\n x += step", + "itertools.count.__delattr__" => "Implement delattr(self, name).", + "itertools.count.__eq__" => "Return self==value.", + "itertools.count.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.count.__ge__" => "Return self>=value.", + "itertools.count.__getattribute__" => "Return getattr(self, name).", + "itertools.count.__getstate__" => "Helper for pickle.", + "itertools.count.__gt__" => "Return self>value.", + "itertools.count.__hash__" => "Return hash(self).", + "itertools.count.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.count.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.count.__iter__" => "Implement iter(self).", + "itertools.count.__le__" => "Return self<=value.", + "itertools.count.__lt__" => "Return self<value.", + "itertools.count.__ne__" => "Return self!=value.", + "itertools.count.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.count.__next__" => "Implement next(self).", + "itertools.count.__reduce__" => "Helper for pickle.", + "itertools.count.__reduce_ex__" => "Helper for pickle.", + "itertools.count.__repr__" => "Return repr(self).", + "itertools.count.__setattr__" => "Implement setattr(self, name, value).", + "itertools.count.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.count.__str__" => "Return str(self).", + "itertools.count.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.cycle" => "Return elements from the iterable until it is exhausted. Then repeat the sequence indefinitely.", + "itertools.cycle.__delattr__" => "Implement delattr(self, name).", + "itertools.cycle.__eq__" => "Return self==value.", + "itertools.cycle.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.cycle.__ge__" => "Return self>=value.", + "itertools.cycle.__getattribute__" => "Return getattr(self, name).", + "itertools.cycle.__getstate__" => "Helper for pickle.", + "itertools.cycle.__gt__" => "Return self>value.", + "itertools.cycle.__hash__" => "Return hash(self).", + "itertools.cycle.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.cycle.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.cycle.__iter__" => "Implement iter(self).", + "itertools.cycle.__le__" => "Return self<=value.", + "itertools.cycle.__lt__" => "Return self<value.", + "itertools.cycle.__ne__" => "Return self!=value.", + "itertools.cycle.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.cycle.__next__" => "Implement next(self).", + "itertools.cycle.__reduce__" => "Helper for pickle.", + "itertools.cycle.__reduce_ex__" => "Helper for pickle.", + "itertools.cycle.__repr__" => "Return repr(self).", + "itertools.cycle.__setattr__" => "Implement setattr(self, name, value).", + "itertools.cycle.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.cycle.__str__" => "Return str(self).", + "itertools.cycle.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.dropwhile" => "Drop items from the iterable while predicate(item) is true.\n\nAfterwards, return every element until the iterable is exhausted.", + "itertools.dropwhile.__delattr__" => "Implement delattr(self, name).", + "itertools.dropwhile.__eq__" => "Return self==value.", + "itertools.dropwhile.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.dropwhile.__ge__" => "Return self>=value.", + "itertools.dropwhile.__getattribute__" => "Return getattr(self, name).", + "itertools.dropwhile.__getstate__" => "Helper for pickle.", + "itertools.dropwhile.__gt__" => "Return self>value.", + "itertools.dropwhile.__hash__" => "Return hash(self).", + "itertools.dropwhile.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.dropwhile.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.dropwhile.__iter__" => "Implement iter(self).", + "itertools.dropwhile.__le__" => "Return self<=value.", + "itertools.dropwhile.__lt__" => "Return self<value.", + "itertools.dropwhile.__ne__" => "Return self!=value.", + "itertools.dropwhile.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.dropwhile.__next__" => "Implement next(self).", + "itertools.dropwhile.__reduce__" => "Helper for pickle.", + "itertools.dropwhile.__reduce_ex__" => "Helper for pickle.", + "itertools.dropwhile.__repr__" => "Return repr(self).", + "itertools.dropwhile.__setattr__" => "Implement setattr(self, name, value).", + "itertools.dropwhile.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.dropwhile.__str__" => "Return str(self).", + "itertools.dropwhile.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.filterfalse" => "Return those items of iterable for which function(item) is false.\n\nIf function is None, return the items that are false.", + "itertools.filterfalse.__delattr__" => "Implement delattr(self, name).", + "itertools.filterfalse.__eq__" => "Return self==value.", + "itertools.filterfalse.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.filterfalse.__ge__" => "Return self>=value.", + "itertools.filterfalse.__getattribute__" => "Return getattr(self, name).", + "itertools.filterfalse.__getstate__" => "Helper for pickle.", + "itertools.filterfalse.__gt__" => "Return self>value.", + "itertools.filterfalse.__hash__" => "Return hash(self).", + "itertools.filterfalse.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.filterfalse.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.filterfalse.__iter__" => "Implement iter(self).", + "itertools.filterfalse.__le__" => "Return self<=value.", + "itertools.filterfalse.__lt__" => "Return self<value.", + "itertools.filterfalse.__ne__" => "Return self!=value.", + "itertools.filterfalse.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.filterfalse.__next__" => "Implement next(self).", + "itertools.filterfalse.__reduce__" => "Helper for pickle.", + "itertools.filterfalse.__reduce_ex__" => "Helper for pickle.", + "itertools.filterfalse.__repr__" => "Return repr(self).", + "itertools.filterfalse.__setattr__" => "Implement setattr(self, name, value).", + "itertools.filterfalse.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.filterfalse.__str__" => "Return str(self).", + "itertools.filterfalse.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.groupby" => "make an iterator that returns consecutive keys and groups from the iterable\n\n iterable\n Elements to divide into groups according to the key function.\n key\n A function for computing the group category for each element.\n If the key function is not specified or is None, the element itself\n is used for grouping.", + "itertools.groupby.__delattr__" => "Implement delattr(self, name).", + "itertools.groupby.__eq__" => "Return self==value.", + "itertools.groupby.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.groupby.__ge__" => "Return self>=value.", + "itertools.groupby.__getattribute__" => "Return getattr(self, name).", + "itertools.groupby.__getstate__" => "Helper for pickle.", + "itertools.groupby.__gt__" => "Return self>value.", + "itertools.groupby.__hash__" => "Return hash(self).", + "itertools.groupby.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.groupby.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.groupby.__iter__" => "Implement iter(self).", + "itertools.groupby.__le__" => "Return self<=value.", + "itertools.groupby.__lt__" => "Return self<value.", + "itertools.groupby.__ne__" => "Return self!=value.", + "itertools.groupby.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.groupby.__next__" => "Implement next(self).", + "itertools.groupby.__reduce__" => "Helper for pickle.", + "itertools.groupby.__reduce_ex__" => "Helper for pickle.", + "itertools.groupby.__repr__" => "Return repr(self).", + "itertools.groupby.__setattr__" => "Implement setattr(self, name, value).", + "itertools.groupby.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.groupby.__str__" => "Return str(self).", + "itertools.groupby.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.islice" => "islice(iterable, stop) --> islice object\nislice(iterable, start, stop[, step]) --> islice object\n\nReturn an iterator whose next() method returns selected values from an\niterable. If start is specified, will skip all preceding elements;\notherwise, start defaults to zero. Step defaults to one. If\nspecified as another value, step determines how many values are\nskipped between successive calls. Works like a slice() on a list\nbut returns an iterator.", + "itertools.islice.__delattr__" => "Implement delattr(self, name).", + "itertools.islice.__eq__" => "Return self==value.", + "itertools.islice.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.islice.__ge__" => "Return self>=value.", + "itertools.islice.__getattribute__" => "Return getattr(self, name).", + "itertools.islice.__getstate__" => "Helper for pickle.", + "itertools.islice.__gt__" => "Return self>value.", + "itertools.islice.__hash__" => "Return hash(self).", + "itertools.islice.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.islice.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.islice.__iter__" => "Implement iter(self).", + "itertools.islice.__le__" => "Return self<=value.", + "itertools.islice.__lt__" => "Return self<value.", + "itertools.islice.__ne__" => "Return self!=value.", + "itertools.islice.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.islice.__next__" => "Implement next(self).", + "itertools.islice.__reduce__" => "Helper for pickle.", + "itertools.islice.__reduce_ex__" => "Helper for pickle.", + "itertools.islice.__repr__" => "Return repr(self).", + "itertools.islice.__setattr__" => "Implement setattr(self, name, value).", + "itertools.islice.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.islice.__str__" => "Return str(self).", + "itertools.islice.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.pairwise" => "Return an iterator of overlapping pairs taken from the input iterator.\n\n s -> (s0,s1), (s1,s2), (s2, s3), ...", + "itertools.pairwise.__delattr__" => "Implement delattr(self, name).", + "itertools.pairwise.__eq__" => "Return self==value.", + "itertools.pairwise.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.pairwise.__ge__" => "Return self>=value.", + "itertools.pairwise.__getattribute__" => "Return getattr(self, name).", + "itertools.pairwise.__getstate__" => "Helper for pickle.", + "itertools.pairwise.__gt__" => "Return self>value.", + "itertools.pairwise.__hash__" => "Return hash(self).", + "itertools.pairwise.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.pairwise.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.pairwise.__iter__" => "Implement iter(self).", + "itertools.pairwise.__le__" => "Return self<=value.", + "itertools.pairwise.__lt__" => "Return self<value.", + "itertools.pairwise.__ne__" => "Return self!=value.", + "itertools.pairwise.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.pairwise.__next__" => "Implement next(self).", + "itertools.pairwise.__reduce__" => "Helper for pickle.", + "itertools.pairwise.__reduce_ex__" => "Helper for pickle.", + "itertools.pairwise.__repr__" => "Return repr(self).", + "itertools.pairwise.__setattr__" => "Implement setattr(self, name, value).", + "itertools.pairwise.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.pairwise.__str__" => "Return str(self).", + "itertools.pairwise.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.permutations" => "Return successive r-length permutations of elements in the iterable.\n\npermutations(range(3), 2) --> (0,1), (0,2), (1,0), (1,2), (2,0), (2,1)", + "itertools.permutations.__delattr__" => "Implement delattr(self, name).", + "itertools.permutations.__eq__" => "Return self==value.", + "itertools.permutations.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.permutations.__ge__" => "Return self>=value.", + "itertools.permutations.__getattribute__" => "Return getattr(self, name).", + "itertools.permutations.__getstate__" => "Helper for pickle.", + "itertools.permutations.__gt__" => "Return self>value.", + "itertools.permutations.__hash__" => "Return hash(self).", + "itertools.permutations.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.permutations.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.permutations.__iter__" => "Implement iter(self).", + "itertools.permutations.__le__" => "Return self<=value.", + "itertools.permutations.__lt__" => "Return self<value.", + "itertools.permutations.__ne__" => "Return self!=value.", + "itertools.permutations.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.permutations.__next__" => "Implement next(self).", + "itertools.permutations.__reduce__" => "Helper for pickle.", + "itertools.permutations.__reduce_ex__" => "Helper for pickle.", + "itertools.permutations.__repr__" => "Return repr(self).", + "itertools.permutations.__setattr__" => "Implement setattr(self, name, value).", + "itertools.permutations.__sizeof__" => "Returns size in memory, in bytes.", + "itertools.permutations.__str__" => "Return str(self).", + "itertools.permutations.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.product" => "Cartesian product of input iterables. Equivalent to nested for-loops.\n\nFor example, product(A, B) returns the same as: ((x,y) for x in A for y in B).\nThe leftmost iterators are in the outermost for-loop, so the output tuples\ncycle in a manner similar to an odometer (with the rightmost element changing\non every iteration).\n\nTo compute the product of an iterable with itself, specify the number\nof repetitions with the optional repeat keyword argument. For example,\nproduct(A, repeat=4) means the same as product(A, A, A, A).\n\nproduct('ab', range(3)) --> ('a',0) ('a',1) ('a',2) ('b',0) ('b',1) ('b',2)\nproduct((0,1), (0,1), (0,1)) --> (0,0,0) (0,0,1) (0,1,0) (0,1,1) (1,0,0) ...", + "itertools.product.__delattr__" => "Implement delattr(self, name).", + "itertools.product.__eq__" => "Return self==value.", + "itertools.product.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.product.__ge__" => "Return self>=value.", + "itertools.product.__getattribute__" => "Return getattr(self, name).", + "itertools.product.__getstate__" => "Helper for pickle.", + "itertools.product.__gt__" => "Return self>value.", + "itertools.product.__hash__" => "Return hash(self).", + "itertools.product.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.product.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.product.__iter__" => "Implement iter(self).", + "itertools.product.__le__" => "Return self<=value.", + "itertools.product.__lt__" => "Return self<value.", + "itertools.product.__ne__" => "Return self!=value.", + "itertools.product.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.product.__next__" => "Implement next(self).", + "itertools.product.__reduce__" => "Helper for pickle.", + "itertools.product.__reduce_ex__" => "Helper for pickle.", + "itertools.product.__repr__" => "Return repr(self).", + "itertools.product.__setattr__" => "Implement setattr(self, name, value).", + "itertools.product.__sizeof__" => "Returns size in memory, in bytes.", + "itertools.product.__str__" => "Return str(self).", + "itertools.product.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.repeat" => "repeat(object [,times]) -> create an iterator which returns the object\nfor the specified number of times. If not specified, returns the object\nendlessly.", + "itertools.repeat.__delattr__" => "Implement delattr(self, name).", + "itertools.repeat.__eq__" => "Return self==value.", + "itertools.repeat.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.repeat.__ge__" => "Return self>=value.", + "itertools.repeat.__getattribute__" => "Return getattr(self, name).", + "itertools.repeat.__getstate__" => "Helper for pickle.", + "itertools.repeat.__gt__" => "Return self>value.", + "itertools.repeat.__hash__" => "Return hash(self).", + "itertools.repeat.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.repeat.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.repeat.__iter__" => "Implement iter(self).", + "itertools.repeat.__le__" => "Return self<=value.", + "itertools.repeat.__length_hint__" => "Private method returning an estimate of len(list(it)).", + "itertools.repeat.__lt__" => "Return self<value.", + "itertools.repeat.__ne__" => "Return self!=value.", + "itertools.repeat.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.repeat.__next__" => "Implement next(self).", + "itertools.repeat.__reduce__" => "Helper for pickle.", + "itertools.repeat.__reduce_ex__" => "Helper for pickle.", + "itertools.repeat.__repr__" => "Return repr(self).", + "itertools.repeat.__setattr__" => "Implement setattr(self, name, value).", + "itertools.repeat.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.repeat.__str__" => "Return str(self).", + "itertools.repeat.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.starmap" => "Return an iterator whose values are returned from the function evaluated with an argument tuple taken from the given sequence.", + "itertools.starmap.__delattr__" => "Implement delattr(self, name).", + "itertools.starmap.__eq__" => "Return self==value.", + "itertools.starmap.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.starmap.__ge__" => "Return self>=value.", + "itertools.starmap.__getattribute__" => "Return getattr(self, name).", + "itertools.starmap.__getstate__" => "Helper for pickle.", + "itertools.starmap.__gt__" => "Return self>value.", + "itertools.starmap.__hash__" => "Return hash(self).", + "itertools.starmap.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.starmap.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.starmap.__iter__" => "Implement iter(self).", + "itertools.starmap.__le__" => "Return self<=value.", + "itertools.starmap.__lt__" => "Return self<value.", + "itertools.starmap.__ne__" => "Return self!=value.", + "itertools.starmap.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.starmap.__next__" => "Implement next(self).", + "itertools.starmap.__reduce__" => "Helper for pickle.", + "itertools.starmap.__reduce_ex__" => "Helper for pickle.", + "itertools.starmap.__repr__" => "Return repr(self).", + "itertools.starmap.__setattr__" => "Implement setattr(self, name, value).", + "itertools.starmap.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.starmap.__str__" => "Return str(self).", + "itertools.starmap.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.takewhile" => "Return successive entries from an iterable as long as the predicate evaluates to true for each entry.", + "itertools.takewhile.__delattr__" => "Implement delattr(self, name).", + "itertools.takewhile.__eq__" => "Return self==value.", + "itertools.takewhile.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.takewhile.__ge__" => "Return self>=value.", + "itertools.takewhile.__getattribute__" => "Return getattr(self, name).", + "itertools.takewhile.__getstate__" => "Helper for pickle.", + "itertools.takewhile.__gt__" => "Return self>value.", + "itertools.takewhile.__hash__" => "Return hash(self).", + "itertools.takewhile.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.takewhile.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.takewhile.__iter__" => "Implement iter(self).", + "itertools.takewhile.__le__" => "Return self<=value.", + "itertools.takewhile.__lt__" => "Return self<value.", + "itertools.takewhile.__ne__" => "Return self!=value.", + "itertools.takewhile.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.takewhile.__next__" => "Implement next(self).", + "itertools.takewhile.__reduce__" => "Helper for pickle.", + "itertools.takewhile.__reduce_ex__" => "Helper for pickle.", + "itertools.takewhile.__repr__" => "Return repr(self).", + "itertools.takewhile.__setattr__" => "Implement setattr(self, name, value).", + "itertools.takewhile.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.takewhile.__str__" => "Return str(self).", + "itertools.takewhile.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "itertools.tee" => "Returns a tuple of n independent iterators.", + "itertools.zip_longest" => "Return a zip_longest object whose .__next__() method returns a tuple where\nthe i-th element comes from the i-th iterable argument. The .__next__()\nmethod continues until the longest iterable in the argument sequence\nis exhausted and then it raises StopIteration. When the shorter iterables\nare exhausted, the fillvalue is substituted in their place. The fillvalue\ndefaults to None or can be specified by a keyword argument.", + "itertools.zip_longest.__delattr__" => "Implement delattr(self, name).", + "itertools.zip_longest.__eq__" => "Return self==value.", + "itertools.zip_longest.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "itertools.zip_longest.__ge__" => "Return self>=value.", + "itertools.zip_longest.__getattribute__" => "Return getattr(self, name).", + "itertools.zip_longest.__getstate__" => "Helper for pickle.", + "itertools.zip_longest.__gt__" => "Return self>value.", + "itertools.zip_longest.__hash__" => "Return hash(self).", + "itertools.zip_longest.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "itertools.zip_longest.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "itertools.zip_longest.__iter__" => "Implement iter(self).", + "itertools.zip_longest.__le__" => "Return self<=value.", + "itertools.zip_longest.__lt__" => "Return self<value.", + "itertools.zip_longest.__ne__" => "Return self!=value.", + "itertools.zip_longest.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "itertools.zip_longest.__next__" => "Implement next(self).", + "itertools.zip_longest.__reduce__" => "Helper for pickle.", + "itertools.zip_longest.__reduce_ex__" => "Helper for pickle.", + "itertools.zip_longest.__repr__" => "Return repr(self).", + "itertools.zip_longest.__setattr__" => "Implement setattr(self, name, value).", + "itertools.zip_longest.__sizeof__" => "Size of object in memory, in bytes.", + "itertools.zip_longest.__str__" => "Return str(self).", + "itertools.zip_longest.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "marshal" => "This module contains functions that can read and write Python values in\na binary format. The format is specific to Python, but independent of\nmachine architecture issues.\n\nNot all Python object types are supported; in general, only objects\nwhose value is independent from a particular invocation of Python can be\nwritten and read by this module. The following types are supported:\nNone, integers, floating-point numbers, strings, bytes, bytearrays,\ntuples, lists, sets, dictionaries, and code objects, where it\nshould be understood that tuples, lists and dictionaries are only\nsupported as long as the values contained therein are themselves\nsupported; and recursive lists and dictionaries should not be written\n(they will cause infinite loops).\n\nVariables:\n\nversion -- indicates the format that the module uses. Version 0 is the\n historical format, version 1 shares interned strings and version 2\n uses a binary format for floating-point numbers.\n Version 3 shares common object references (New in version 3.4).\n\nFunctions:\n\ndump() -- write value to a file\nload() -- read value from a file\ndumps() -- marshal value as a bytes object\nloads() -- read value from a bytes-like object", + "marshal.dump" => "Write the value on the open file.\n\n value\n Must be a supported type.\n file\n Must be a writeable binary file.\n version\n Indicates the data format that dump should use.\n allow_code\n Allow to write code objects.\n\nIf the value has (or contains an object that has) an unsupported type, a\nValueError exception is raised - but garbage data will also be written\nto the file. The object will not be properly read back by load().", + "marshal.dumps" => "Return the bytes object that would be written to a file by dump(value, file).\n\n value\n Must be a supported type.\n version\n Indicates the data format that dumps should use.\n allow_code\n Allow to write code objects.\n\nRaise a ValueError exception if value has (or contains an object that has) an\nunsupported type.", + "marshal.load" => "Read one value from the open file and return it.\n\n file\n Must be readable binary file.\n allow_code\n Allow to load code objects.\n\nIf no valid value is read (e.g. because the data has a different Python\nversion's incompatible marshal format), raise EOFError, ValueError or\nTypeError.\n\nNote: If an object containing an unsupported type was marshalled with\ndump(), load() will substitute None for the unmarshallable type.", + "marshal.loads" => "Convert the bytes-like object to a value.\n\n allow_code\n Allow to load code objects.\n\nIf no valid value is found, raise EOFError, ValueError or TypeError. Extra\nbytes in the input are ignored.", + "math" => "This module provides access to the mathematical functions\ndefined by the C standard.", + "math.acos" => "Return the arc cosine (measured in radians) of x.\n\nThe result is between 0 and pi.", + "math.acosh" => "Return the inverse hyperbolic cosine of x.", + "math.asin" => "Return the arc sine (measured in radians) of x.\n\nThe result is between -pi/2 and pi/2.", + "math.asinh" => "Return the inverse hyperbolic sine of x.", + "math.atan" => "Return the arc tangent (measured in radians) of x.\n\nThe result is between -pi/2 and pi/2.", + "math.atan2" => "Return the arc tangent (measured in radians) of y/x.\n\nUnlike atan(y/x), the signs of both x and y are considered.", + "math.atanh" => "Return the inverse hyperbolic tangent of x.", + "math.cbrt" => "Return the cube root of x.", + "math.ceil" => "Return the ceiling of x as an Integral.\n\nThis is the smallest integer >= x.", + "math.comb" => "Number of ways to choose k items from n items without repetition and without order.\n\nEvaluates to n! / (k! * (n - k)!) when k <= n and evaluates\nto zero when k > n.\n\nAlso called the binomial coefficient because it is equivalent\nto the coefficient of k-th term in polynomial expansion of the\nexpression (1 + x)**n.\n\nRaises TypeError if either of the arguments are not integers.\nRaises ValueError if either of the arguments are negative.", + "math.copysign" => "Return a float with the magnitude (absolute value) of x but the sign of y.\n\nOn platforms that support signed zeros, copysign(1.0, -0.0)\nreturns -1.0.", + "math.cos" => "Return the cosine of x (measured in radians).", + "math.cosh" => "Return the hyperbolic cosine of x.", + "math.degrees" => "Convert angle x from radians to degrees.", + "math.dist" => "Return the Euclidean distance between two points p and q.\n\nThe points should be specified as sequences (or iterables) of\ncoordinates. Both inputs must have the same dimension.\n\nRoughly equivalent to:\n sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q)))", + "math.erf" => "Error function at x.", + "math.erfc" => "Complementary error function at x.", + "math.exp" => "Return e raised to the power of x.", + "math.exp2" => "Return 2 raised to the power of x.", + "math.expm1" => "Return exp(x)-1.\n\nThis function avoids the loss of precision involved in the direct evaluation of exp(x)-1 for small x.", + "math.fabs" => "Return the absolute value of the float x.", + "math.factorial" => "Find n!.", + "math.floor" => "Return the floor of x as an Integral.\n\nThis is the largest integer <= x.", + "math.fma" => "Fused multiply-add operation.\n\nCompute (x * y) + z with a single round.", + "math.fmod" => "Return fmod(x, y), according to platform C.\n\nx % y may differ.", + "math.frexp" => "Return the mantissa and exponent of x, as pair (m, e).\n\nm is a float and e is an int, such that x = m * 2.**e.\nIf x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0.", + "math.fsum" => "Return an accurate floating-point sum of values in the iterable seq.\n\nAssumes IEEE-754 floating-point arithmetic.", + "math.gamma" => "Gamma function at x.", + "math.gcd" => "Greatest Common Divisor.", + "math.hypot" => "Multidimensional Euclidean distance from the origin to a point.\n\nRoughly equivalent to:\n sqrt(sum(x**2 for x in coordinates))\n\nFor a two dimensional point (x, y), gives the hypotenuse\nusing the Pythagorean theorem: sqrt(x*x + y*y).\n\nFor example, the hypotenuse of a 3/4/5 right triangle is:\n\n >>> hypot(3.0, 4.0)\n 5.0", + "math.isclose" => "Determine whether two floating-point numbers are close in value.\n\n rel_tol\n maximum difference for being considered \"close\", relative to the\n magnitude of the input values\n abs_tol\n maximum difference for being considered \"close\", regardless of the\n magnitude of the input values\n\nReturn True if a is close in value to b, and False otherwise.\n\nFor the values to be considered close, the difference between them\nmust be smaller than at least one of the tolerances.\n\n-inf, inf and NaN behave similarly to the IEEE 754 Standard. That\nis, NaN is not close to anything, even itself. inf and -inf are\nonly close to themselves.", + "math.isfinite" => "Return True if x is neither an infinity nor a NaN, and False otherwise.", + "math.isinf" => "Return True if x is a positive or negative infinity, and False otherwise.", + "math.isnan" => "Return True if x is a NaN (not a number), and False otherwise.", + "math.isqrt" => "Return the integer part of the square root of the input.", + "math.lcm" => "Least Common Multiple.", + "math.ldexp" => "Return x * (2**i).\n\nThis is essentially the inverse of frexp().", + "math.lgamma" => "Natural logarithm of absolute value of Gamma function at x.", + "math.log" => "log(x, [base=math.e])\nReturn the logarithm of x to the given base.\n\nIf the base is not specified, returns the natural logarithm (base e) of x.", + "math.log10" => "Return the base 10 logarithm of x.", + "math.log1p" => "Return the natural logarithm of 1+x (base e).\n\nThe result is computed in a way which is accurate for x near zero.", + "math.log2" => "Return the base 2 logarithm of x.", + "math.modf" => "Return the fractional and integer parts of x.\n\nBoth results carry the sign of x and are floats.", + "math.nextafter" => "Return the floating-point value the given number of steps after x towards y.\n\nIf steps is not specified or is None, it defaults to 1.\n\nRaises a TypeError, if x or y is not a double, or if steps is not an integer.\nRaises ValueError if steps is negative.", + "math.perm" => "Number of ways to choose k items from n items without repetition and with order.\n\nEvaluates to n! / (n - k)! when k <= n and evaluates\nto zero when k > n.\n\nIf k is not specified or is None, then k defaults to n\nand the function returns n!.\n\nRaises TypeError if either of the arguments are not integers.\nRaises ValueError if either of the arguments are negative.", + "math.pow" => "Return x**y (x to the power of y).", + "math.prod" => "Calculate the product of all the elements in the input iterable.\n\nThe default start value for the product is 1.\n\nWhen the iterable is empty, return the start value. This function is\nintended specifically for use with numeric values and may reject\nnon-numeric types.", + "math.radians" => "Convert angle x from degrees to radians.", + "math.remainder" => "Difference between x and the closest integer multiple of y.\n\nReturn x - n*y where n*y is the closest integer multiple of y.\nIn the case where x is exactly halfway between two multiples of\ny, the nearest even value of n is used. The result is always exact.", + "math.sin" => "Return the sine of x (measured in radians).", + "math.sinh" => "Return the hyperbolic sine of x.", + "math.sqrt" => "Return the square root of x.", + "math.sumprod" => "Return the sum of products of values from two iterables p and q.\n\nRoughly equivalent to:\n\n sum(map(operator.mul, p, q, strict=True))\n\nFor float and mixed int/float inputs, the intermediate products\nand sums are computed with extended precision.", + "math.tan" => "Return the tangent of x (measured in radians).", + "math.tanh" => "Return the hyperbolic tangent of x.", + "math.trunc" => "Truncates the Real x to the nearest Integral toward 0.\n\nUses the __trunc__ magic method.", + "math.ulp" => "Return the value of the least significant bit of the float x.", + "mmap.mmap" => "Windows: mmap(fileno, length[, tagname[, access[, offset]]])\n\nMaps length bytes from the file specified by the file handle fileno,\nand returns a mmap object. If length is larger than the current size\nof the file, the file is extended to contain length bytes. If length\nis 0, the maximum length of the map is the current size of the file,\nexcept that if the file is empty Windows raises an exception (you cannot\ncreate an empty mapping on Windows).\n\nUnix: mmap(fileno, length[, flags[, prot[, access[, offset[, trackfd]]]]])\n\nMaps length bytes from the file specified by the file descriptor fileno,\nand returns a mmap object. If length is 0, the maximum length of the map\nwill be the current size of the file when mmap is called.\nflags specifies the nature of the mapping. MAP_PRIVATE creates a\nprivate copy-on-write mapping, so changes to the contents of the mmap\nobject will be private to this process, and MAP_SHARED creates a mapping\nthat's shared with all other processes mapping the same areas of the file.\nThe default value is MAP_SHARED.\n\nTo map anonymous memory, pass -1 as the fileno (both versions).", + "mmap.mmap.__buffer__" => "Return a buffer object that exposes the underlying memory of the object.", + "mmap.mmap.__delattr__" => "Implement delattr(self, name).", + "mmap.mmap.__delitem__" => "Delete self[key].", + "mmap.mmap.__eq__" => "Return self==value.", + "mmap.mmap.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "mmap.mmap.__ge__" => "Return self>=value.", + "mmap.mmap.__getattribute__" => "Return getattr(self, name).", + "mmap.mmap.__getitem__" => "Return self[key].", + "mmap.mmap.__getstate__" => "Helper for pickle.", + "mmap.mmap.__gt__" => "Return self>value.", + "mmap.mmap.__hash__" => "Return hash(self).", + "mmap.mmap.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "mmap.mmap.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "mmap.mmap.__le__" => "Return self<=value.", + "mmap.mmap.__len__" => "Return len(self).", + "mmap.mmap.__lt__" => "Return self<value.", + "mmap.mmap.__ne__" => "Return self!=value.", + "mmap.mmap.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "mmap.mmap.__reduce__" => "Helper for pickle.", + "mmap.mmap.__reduce_ex__" => "Helper for pickle.", + "mmap.mmap.__release_buffer__" => "Release the buffer object that exposes the underlying memory of the object.", + "mmap.mmap.__repr__" => "Return repr(self).", + "mmap.mmap.__setattr__" => "Implement setattr(self, name, value).", + "mmap.mmap.__setitem__" => "Set self[key] to value.", + "mmap.mmap.__sizeof__" => "Size of object in memory, in bytes.", + "mmap.mmap.__str__" => "Return str(self).", + "mmap.mmap.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "msvcrt.GetErrorMode" => "Wrapper around GetErrorMode.", + "msvcrt.SetErrorMode" => "Wrapper around SetErrorMode.", + "msvcrt.get_osfhandle" => "Return the file handle for the file descriptor fd.\n\nRaises OSError if fd is not recognized.", + "msvcrt.getch" => "Read a keypress and return the resulting character as a byte string.\n\nNothing is echoed to the console. This call will block if a keypress is\nnot already available, but will not wait for Enter to be pressed. If the\npressed key was a special function key, this will return '\\000' or\n'\\xe0'; the next call will return the keycode. The Control-C keypress\ncannot be read with this function.", + "msvcrt.getche" => "Similar to getch(), but the keypress will be echoed if possible.", + "msvcrt.getwch" => "Wide char variant of getch(), returning a Unicode value.", + "msvcrt.getwche" => "Wide char variant of getche(), returning a Unicode value.", + "msvcrt.heapmin" => "Minimize the malloc() heap.\n\nForce the malloc() heap to clean itself up and return unused blocks\nto the operating system. On failure, this raises OSError.", + "msvcrt.kbhit" => "Returns a nonzero value if a keypress is waiting to be read. Otherwise, return 0.", + "msvcrt.locking" => "Lock part of a file based on file descriptor fd from the C runtime.\n\nRaises OSError on failure. The locked region of the file extends from\nthe current file position for nbytes bytes, and may continue beyond\nthe end of the file. mode must be one of the LK_* constants listed\nbelow. Multiple regions in a file may be locked at the same time, but\nmay not overlap. Adjacent regions are not merged; they must be unlocked\nindividually.", + "msvcrt.open_osfhandle" => "Create a C runtime file descriptor from the file handle handle.\n\nThe flags parameter should be a bitwise OR of os.O_APPEND, os.O_RDONLY,\nand os.O_TEXT. The returned file descriptor may be used as a parameter\nto os.fdopen() to create a file object.", + "msvcrt.putch" => "Print the byte string char to the console without buffering.", + "msvcrt.putwch" => "Wide char variant of putch(), accepting a Unicode value.", + "msvcrt.setmode" => "Set the line-end translation mode for the file descriptor fd.\n\nTo set it to text mode, flags should be os.O_TEXT; for binary, it\nshould be os.O_BINARY.\n\nReturn value is the previous mode.", + "msvcrt.ungetch" => "Opposite of getch.\n\nCause the byte string char to be \"pushed back\" into the\nconsole buffer; it will be the next character read by\ngetch() or getche().", + "msvcrt.ungetwch" => "Wide char variant of ungetch(), accepting a Unicode value.", + "nt" => "This module provides access to operating system functionality that is\nstandardized by the C Standard and the POSIX standard (a thinly\ndisguised Unix interface). Refer to the library manual and\ncorresponding Unix manual entries for more information on calls.", + "nt.DirEntry.__class_getitem__" => "See PEP 585", + "nt.DirEntry.__delattr__" => "Implement delattr(self, name).", + "nt.DirEntry.__eq__" => "Return self==value.", + "nt.DirEntry.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.DirEntry.__fspath__" => "Returns the path for the entry.", + "nt.DirEntry.__ge__" => "Return self>=value.", + "nt.DirEntry.__getattribute__" => "Return getattr(self, name).", + "nt.DirEntry.__getstate__" => "Helper for pickle.", + "nt.DirEntry.__gt__" => "Return self>value.", + "nt.DirEntry.__hash__" => "Return hash(self).", + "nt.DirEntry.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.DirEntry.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.DirEntry.__le__" => "Return self<=value.", + "nt.DirEntry.__lt__" => "Return self<value.", + "nt.DirEntry.__ne__" => "Return self!=value.", + "nt.DirEntry.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.DirEntry.__reduce__" => "Helper for pickle.", + "nt.DirEntry.__reduce_ex__" => "Helper for pickle.", + "nt.DirEntry.__repr__" => "Return repr(self).", + "nt.DirEntry.__setattr__" => "Implement setattr(self, name, value).", + "nt.DirEntry.__sizeof__" => "Size of object in memory, in bytes.", + "nt.DirEntry.__str__" => "Return str(self).", + "nt.DirEntry.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.DirEntry.inode" => "Return inode of the entry; cached per entry.", + "nt.DirEntry.is_dir" => "Return True if the entry is a directory; cached per entry.", + "nt.DirEntry.is_file" => "Return True if the entry is a file; cached per entry.", + "nt.DirEntry.is_junction" => "Return True if the entry is a junction; cached per entry.", + "nt.DirEntry.is_symlink" => "Return True if the entry is a symbolic link; cached per entry.", + "nt.DirEntry.name" => "the entry's base filename, relative to scandir() \"path\" argument", + "nt.DirEntry.path" => "the entry's full path name; equivalent to os.path.join(scandir_path, entry.name)", + "nt.DirEntry.stat" => "Return stat_result object for the entry; cached per entry.", + "nt._add_dll_directory" => "Add a path to the DLL search path.\n\nThis search path is used when resolving dependencies for imported\nextension modules (the module itself is resolved through sys.path),\nand also by ctypes.\n\nReturns an opaque value that may be passed to os.remove_dll_directory\nto remove this directory from the search path.", + "nt._create_environ" => "Create the environment dictionary.", + "nt._exit" => "Exit to the system with specified status, without normal exit processing.", + "nt._findfirstfile" => "A function to get the real file name without accessing the file in Windows.", + "nt._getdiskusage" => "Return disk usage statistics about the given path as a (total, free) tuple.", + "nt._getfinalpathname" => "A helper function for samepath on windows.", + "nt._getvolumepathname" => "A helper function for ismount on Win32.", + "nt._inputhook" => "Calls PyOS_CallInputHook droppong the GIL first", + "nt._is_inputhook_installed" => "Checks if PyOS_CallInputHook is set", + "nt._path_exists" => "Test whether a path exists. Returns False for broken symbolic links.", + "nt._path_isdevdrive" => "Determines whether the specified path is on a Windows Dev Drive.", + "nt._path_isdir" => "Return true if the pathname refers to an existing directory.", + "nt._path_isfile" => "Test whether a path is a regular file", + "nt._path_isjunction" => "Test whether a path is a junction", + "nt._path_islink" => "Test whether a path is a symbolic link", + "nt._path_lexists" => "Test whether a path exists. Returns True for broken symbolic links.", + "nt._path_normpath" => "Normalize path, eliminating double slashes, etc.", + "nt._path_splitroot" => "Removes everything after the root on Win32.", + "nt._path_splitroot_ex" => "Split a pathname into drive, root and tail.\n\nThe tail contains anything after the root.", + "nt._remove_dll_directory" => "Removes a path from the DLL search path.\n\nThe parameter is an opaque value that was returned from\nos.add_dll_directory. You can only remove directories that you added\nyourself.", + "nt._supports_virtual_terminal" => "Checks if virtual terminal is supported in windows", + "nt.abort" => "Abort the interpreter immediately.\n\nThis function 'dumps core' or otherwise fails in the hardest way possible\non the hosting operating system. This function never returns.", + "nt.access" => "Use the real uid/gid to test for access to a path.\n\n path\n Path to be tested; can be string, bytes, or a path-like object.\n mode\n Operating-system mode bitfield. Can be F_OK to test existence,\n or the inclusive-OR of R_OK, W_OK, and X_OK.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that\n directory.\n effective_ids\n If True, access will use the effective uid/gid instead of\n the real uid/gid.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n access will examine the symbolic link itself instead of the file\n the link points to.\n\ndir_fd, effective_ids, and follow_symlinks may not be implemented\n on your platform. If they are unavailable, using them will raise a\n NotImplementedError.\n\nNote that most operations will use the effective uid/gid, therefore this\n routine can be used in a suid/sgid environment to test if the invoking user\n has the specified access to the path.", + "nt.chdir" => "Change the current working directory to the specified path.\n\npath may always be specified as a string.\nOn some platforms, path may also be specified as an open file descriptor.\nIf this functionality is unavailable, using it raises an exception.", + "nt.chmod" => "Change the access permissions of a file.\n\n path\n Path to be modified. May always be specified as a str, bytes, or a path-like object.\n On some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.\n mode\n Operating-system mode bitfield.\n Be careful when using number literals for *mode*. The conventional UNIX notation for\n numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n Python.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that\n directory.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n chmod will modify the symbolic link itself instead of the file\n the link points to.\n\nIt is an error to use dir_fd or follow_symlinks when specifying path as\n an open file descriptor.\ndir_fd and follow_symlinks may not be implemented on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "nt.close" => "Close a file descriptor.", + "nt.closerange" => "Closes all file descriptors in [fd_low, fd_high), ignoring errors.", + "nt.cpu_count" => "Return the number of logical CPUs in the system.\n\nReturn None if indeterminable.", + "nt.device_encoding" => "Return a string describing the encoding of a terminal's file descriptor.\n\nThe file descriptor must be attached to a terminal.\nIf the device is not a terminal, return None.", + "nt.dup" => "Return a duplicate of a file descriptor.", + "nt.dup2" => "Duplicate file descriptor.", + "nt.execv" => "Execute an executable path with arguments, replacing current process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.", + "nt.execve" => "Execute an executable path with arguments, replacing current process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.", + "nt.fchmod" => "Change the access permissions of the file given by file descriptor fd.\n\n fd\n The file descriptor of the file to be modified.\n mode\n Operating-system mode bitfield.\n Be careful when using number literals for *mode*. The conventional UNIX notation for\n numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n Python.\n\nEquivalent to os.chmod(fd, mode).", + "nt.fspath" => "Return the file system path representation of the object.\n\nIf the object is str or bytes, then allow it to pass through as-is. If the\nobject defines __fspath__(), then return the result of that method. All other\ntypes raise a TypeError.", + "nt.fstat" => "Perform a stat system call on the given file descriptor.\n\nLike stat(), but for an open file descriptor.\nEquivalent to os.stat(fd).", + "nt.fsync" => "Force write of fd to disk.", + "nt.ftruncate" => "Truncate a file, specified by file descriptor, to a specific length.", + "nt.get_blocking" => "Get the blocking mode of the file descriptor.\n\nReturn False if the O_NONBLOCK flag is set, True if the flag is cleared.", + "nt.get_handle_inheritable" => "Get the close-on-exe flag of the specified file descriptor.", + "nt.get_inheritable" => "Get the close-on-exe flag of the specified file descriptor.", + "nt.get_terminal_size" => "Return the size of the terminal window as (columns, lines).\n\nThe optional argument fd (default standard output) specifies\nwhich file descriptor should be queried.\n\nIf the file descriptor is not connected to a terminal, an OSError\nis thrown.\n\nThis function will only be defined if an implementation is\navailable for this system.\n\nshutil.get_terminal_size is the high-level function which should\nnormally be used, os.get_terminal_size is the low-level implementation.", + "nt.getcwd" => "Return a unicode string representing the current working directory.", + "nt.getcwdb" => "Return a bytes string representing the current working directory.", + "nt.getlogin" => "Return the actual login name.", + "nt.getpid" => "Return the current process id.", + "nt.getppid" => "Return the parent's process id.\n\nIf the parent process has already exited, Windows machines will still\nreturn its id; others systems will return the id of the 'init' process (1).", + "nt.isatty" => "Return True if the fd is connected to a terminal.\n\nReturn True if the file descriptor is an open file descriptor\nconnected to the slave end of a terminal.", + "nt.kill" => "Kill a process with a signal.", + "nt.lchmod" => "Change the access permissions of a file, without following symbolic links.\n\nIf path is a symlink, this affects the link itself rather than the target.\nEquivalent to chmod(path, mode, follow_symlinks=False).\"", + "nt.link" => "Create a hard link to a file.\n\nIf either src_dir_fd or dst_dir_fd is not None, it should be a file\n descriptor open to a directory, and the respective path string (src or dst)\n should be relative; the path will then be relative to that directory.\nIf follow_symlinks is False, and the last element of src is a symbolic\n link, link will create a link to the symbolic link itself instead of the\n file the link points to.\nsrc_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your\n platform. If they are unavailable, using them will raise a\n NotImplementedError.", + "nt.listdir" => "Return a list containing the names of the files in the directory.\n\npath can be specified as either str, bytes, or a path-like object. If path is bytes,\n the filenames returned will also be bytes; in all other circumstances\n the filenames returned will be str.\nIf path is None, uses the path='.'.\nOn some platforms, path may also be specified as an open file descriptor;\\\n the file descriptor must refer to a directory.\n If this functionality is unavailable, using it raises NotImplementedError.\n\nThe list is in arbitrary order. It does not include the special\nentries '.' and '..' even if they are present in the directory.", + "nt.listdrives" => "Return a list containing the names of drives in the system.\n\nA drive name typically looks like 'C:\\\\'.", + "nt.listmounts" => "Return a list containing mount points for a particular volume.\n\n'volume' should be a GUID path as returned from os.listvolumes.", + "nt.listvolumes" => "Return a list containing the volumes in the system.\n\nVolumes are typically represented as a GUID path.", + "nt.lseek" => "Set the position of a file descriptor. Return the new position.\n\n fd\n An open file descriptor, as returned by os.open().\n position\n Position, interpreted relative to 'whence'.\n whence\n The relative position to seek from. Valid values are:\n - SEEK_SET: seek from the start of the file.\n - SEEK_CUR: seek from the current file position.\n - SEEK_END: seek from the end of the file.\n\nThe return value is the number of bytes relative to the beginning of the file.", + "nt.lstat" => "Perform a stat system call on the given path, without following symbolic links.\n\nLike stat(), but do not follow symbolic links.\nEquivalent to stat(path, follow_symlinks=False).", + "nt.mkdir" => "Create a directory.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.\n\nThe mode argument is ignored on Windows. Where it is used, the current umask\nvalue is first masked out.", + "nt.open" => "Open a file for low level IO. Returns a file descriptor (integer).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "nt.pipe" => "Create a pipe.\n\nReturns a tuple of two file descriptors:\n (read_fd, write_fd)", + "nt.putenv" => "Change or add an environment variable.", + "nt.read" => "Read from a file descriptor. Returns a bytes object.", + "nt.readinto" => "Read into a buffer object from a file descriptor.\n\nThe buffer should be mutable and bytes-like. On success, returns the number of\nbytes read. Less bytes may be read than the size of the buffer. The underlying\nsystem call will be retried when interrupted by a signal, unless the signal\nhandler raises an exception. Other errors will not be retried and an error will\nbe raised.\n\nReturns 0 if *fd* is at end of file or if the provided *buffer* has length 0\n(which can be used to check for errors without reading data). Never returns\nnegative.", + "nt.readlink" => "Return a string representing the path to which the symbolic link points.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\nand path should be relative; path will then be relative to that directory.\n\ndir_fd may not be implemented on your platform. If it is unavailable,\nusing it will raise a NotImplementedError.", + "nt.remove" => "Remove a file (same as unlink()).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "nt.rename" => "Rename a file or directory.\n\nIf either src_dir_fd or dst_dir_fd is not None, it should be a file\n descriptor open to a directory, and the respective path string (src or dst)\n should be relative; the path will then be relative to that directory.\nsrc_dir_fd and dst_dir_fd, may not be implemented on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "nt.replace" => "Rename a file or directory, overwriting the destination.\n\nIf either src_dir_fd or dst_dir_fd is not None, it should be a file\n descriptor open to a directory, and the respective path string (src or dst)\n should be relative; the path will then be relative to that directory.\nsrc_dir_fd and dst_dir_fd, may not be implemented on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "nt.rmdir" => "Remove a directory.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "nt.scandir" => "Return an iterator of DirEntry objects for given path.\n\npath can be specified as either str, bytes, or a path-like object. If path\nis bytes, the names of yielded DirEntry objects will also be bytes; in\nall other circumstances they will be str.\n\nIf path is None, uses the path='.'.", + "nt.set_blocking" => "Set the blocking mode of the specified file descriptor.\n\nSet the O_NONBLOCK flag if blocking is False,\nclear the O_NONBLOCK flag otherwise.", + "nt.set_handle_inheritable" => "Set the inheritable flag of the specified handle.", + "nt.set_inheritable" => "Set the inheritable flag of the specified file descriptor.", + "nt.spawnv" => "Execute the program specified by path in a new process.\n\n mode\n Mode of process creation.\n path\n Path of executable file.\n argv\n Tuple or list of strings.", + "nt.spawnve" => "Execute the program specified by path in a new process.\n\n mode\n Mode of process creation.\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.", + "nt.startfile" => "Start a file with its associated application.\n\nWhen \"operation\" is not specified or \"open\", this acts like\ndouble-clicking the file in Explorer, or giving the file name as an\nargument to the DOS \"start\" command: the file is opened with whatever\napplication (if any) its extension is associated.\nWhen another \"operation\" is given, it specifies what should be done with\nthe file. A typical operation is \"print\".\n\n\"arguments\" is passed to the application, but should be omitted if the\nfile is a document.\n\n\"cwd\" is the working directory for the operation. If \"filepath\" is\nrelative, it will be resolved against this directory. This argument\nshould usually be an absolute path.\n\n\"show_cmd\" can be used to override the recommended visibility option.\nSee the Windows ShellExecute documentation for values.\n\nstartfile returns as soon as the associated application is launched.\nThere is no option to wait for the application to close, and no way\nto retrieve the application's exit status.\n\nThe filepath is relative to the current directory. If you want to use\nan absolute path, make sure the first character is not a slash (\"/\");\nthe underlying Win32 ShellExecute function doesn't work if it is.", + "nt.stat" => "Perform a stat system call on the given path.\n\n path\n Path to be examined; can be string, bytes, a path-like object or\n open-file-descriptor int.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be a relative string; path will then be relative to\n that directory.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n stat will examine the symbolic link itself instead of the file\n the link points to.\n\ndir_fd and follow_symlinks may not be implemented\n on your platform. If they are unavailable, using them will raise a\n NotImplementedError.\n\nIt's an error to use dir_fd or follow_symlinks when specifying path as\n an open file descriptor.", + "nt.stat_result" => "stat_result: Result from stat, fstat, or lstat.\n\nThis object may be accessed either as a tuple of\n (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)\nor via the attributes st_mode, st_ino, st_dev, st_nlink, st_uid, and so on.\n\nPosix/windows: If your platform supports st_blksize, st_blocks, st_rdev,\nor st_flags, they are available as attributes only.\n\nSee os.stat for more information.", + "nt.stat_result.__add__" => "Return self+value.", + "nt.stat_result.__class_getitem__" => "See PEP 585", + "nt.stat_result.__contains__" => "Return bool(key in self).", + "nt.stat_result.__delattr__" => "Implement delattr(self, name).", + "nt.stat_result.__eq__" => "Return self==value.", + "nt.stat_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.stat_result.__ge__" => "Return self>=value.", + "nt.stat_result.__getattribute__" => "Return getattr(self, name).", + "nt.stat_result.__getitem__" => "Return self[key].", + "nt.stat_result.__getstate__" => "Helper for pickle.", + "nt.stat_result.__gt__" => "Return self>value.", + "nt.stat_result.__hash__" => "Return hash(self).", + "nt.stat_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.stat_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.stat_result.__iter__" => "Implement iter(self).", + "nt.stat_result.__le__" => "Return self<=value.", + "nt.stat_result.__len__" => "Return len(self).", + "nt.stat_result.__lt__" => "Return self<value.", + "nt.stat_result.__mul__" => "Return self*value.", + "nt.stat_result.__ne__" => "Return self!=value.", + "nt.stat_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.stat_result.__reduce_ex__" => "Helper for pickle.", + "nt.stat_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "nt.stat_result.__repr__" => "Return repr(self).", + "nt.stat_result.__rmul__" => "Return value*self.", + "nt.stat_result.__setattr__" => "Implement setattr(self, name, value).", + "nt.stat_result.__sizeof__" => "Size of object in memory, in bytes.", + "nt.stat_result.__str__" => "Return str(self).", + "nt.stat_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.stat_result.count" => "Return number of occurrences of value.", + "nt.stat_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "nt.stat_result.st_atime" => "time of last access", + "nt.stat_result.st_atime_ns" => "time of last access in nanoseconds", + "nt.stat_result.st_birthtime" => "time of creation", + "nt.stat_result.st_birthtime_ns" => "time of creation in nanoseconds", + "nt.stat_result.st_ctime" => "time of last change", + "nt.stat_result.st_ctime_ns" => "time of last change in nanoseconds", + "nt.stat_result.st_dev" => "device", + "nt.stat_result.st_file_attributes" => "Windows file attribute bits", + "nt.stat_result.st_gid" => "group ID of owner", + "nt.stat_result.st_ino" => "inode", + "nt.stat_result.st_mode" => "protection bits", + "nt.stat_result.st_mtime" => "time of last modification", + "nt.stat_result.st_mtime_ns" => "time of last modification in nanoseconds", + "nt.stat_result.st_nlink" => "number of hard links", + "nt.stat_result.st_reparse_tag" => "Windows reparse tag", + "nt.stat_result.st_size" => "total size, in bytes", + "nt.stat_result.st_uid" => "user ID of owner", + "nt.statvfs_result" => "statvfs_result: Result from statvfs or fstatvfs.\n\nThis object may be accessed either as a tuple of\n (bsize, frsize, blocks, bfree, bavail, files, ffree, favail, flag, namemax),\nor via the attributes f_bsize, f_frsize, f_blocks, f_bfree, and so on.\n\nSee os.statvfs for more information.", + "nt.statvfs_result.__add__" => "Return self+value.", + "nt.statvfs_result.__class_getitem__" => "See PEP 585", + "nt.statvfs_result.__contains__" => "Return bool(key in self).", + "nt.statvfs_result.__delattr__" => "Implement delattr(self, name).", + "nt.statvfs_result.__eq__" => "Return self==value.", + "nt.statvfs_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.statvfs_result.__ge__" => "Return self>=value.", + "nt.statvfs_result.__getattribute__" => "Return getattr(self, name).", + "nt.statvfs_result.__getitem__" => "Return self[key].", + "nt.statvfs_result.__getstate__" => "Helper for pickle.", + "nt.statvfs_result.__gt__" => "Return self>value.", + "nt.statvfs_result.__hash__" => "Return hash(self).", + "nt.statvfs_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.statvfs_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.statvfs_result.__iter__" => "Implement iter(self).", + "nt.statvfs_result.__le__" => "Return self<=value.", + "nt.statvfs_result.__len__" => "Return len(self).", + "nt.statvfs_result.__lt__" => "Return self<value.", + "nt.statvfs_result.__mul__" => "Return self*value.", + "nt.statvfs_result.__ne__" => "Return self!=value.", + "nt.statvfs_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.statvfs_result.__reduce_ex__" => "Helper for pickle.", + "nt.statvfs_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "nt.statvfs_result.__repr__" => "Return repr(self).", + "nt.statvfs_result.__rmul__" => "Return value*self.", + "nt.statvfs_result.__setattr__" => "Implement setattr(self, name, value).", + "nt.statvfs_result.__sizeof__" => "Size of object in memory, in bytes.", + "nt.statvfs_result.__str__" => "Return str(self).", + "nt.statvfs_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.statvfs_result.count" => "Return number of occurrences of value.", + "nt.statvfs_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "nt.strerror" => "Translate an error code to a message string.", + "nt.symlink" => "Create a symbolic link pointing to src named dst.\n\ntarget_is_directory is required on Windows if the target is to be\n interpreted as a directory. (On Windows, symlink requires\n Windows 6.0 or greater, and raises a NotImplementedError otherwise.)\n target_is_directory is ignored on non-Windows platforms.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "nt.system" => "Execute the command in a subshell.", + "nt.terminal_size" => "A tuple of (columns, lines) for holding terminal window size", + "nt.terminal_size.__add__" => "Return self+value.", + "nt.terminal_size.__class_getitem__" => "See PEP 585", + "nt.terminal_size.__contains__" => "Return bool(key in self).", + "nt.terminal_size.__delattr__" => "Implement delattr(self, name).", + "nt.terminal_size.__eq__" => "Return self==value.", + "nt.terminal_size.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.terminal_size.__ge__" => "Return self>=value.", + "nt.terminal_size.__getattribute__" => "Return getattr(self, name).", + "nt.terminal_size.__getitem__" => "Return self[key].", + "nt.terminal_size.__getstate__" => "Helper for pickle.", + "nt.terminal_size.__gt__" => "Return self>value.", + "nt.terminal_size.__hash__" => "Return hash(self).", + "nt.terminal_size.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.terminal_size.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.terminal_size.__iter__" => "Implement iter(self).", + "nt.terminal_size.__le__" => "Return self<=value.", + "nt.terminal_size.__len__" => "Return len(self).", + "nt.terminal_size.__lt__" => "Return self<value.", + "nt.terminal_size.__mul__" => "Return self*value.", + "nt.terminal_size.__ne__" => "Return self!=value.", + "nt.terminal_size.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.terminal_size.__reduce_ex__" => "Helper for pickle.", + "nt.terminal_size.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "nt.terminal_size.__repr__" => "Return repr(self).", + "nt.terminal_size.__rmul__" => "Return value*self.", + "nt.terminal_size.__setattr__" => "Implement setattr(self, name, value).", + "nt.terminal_size.__sizeof__" => "Size of object in memory, in bytes.", + "nt.terminal_size.__str__" => "Return str(self).", + "nt.terminal_size.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.terminal_size.columns" => "width of the terminal window in characters", + "nt.terminal_size.count" => "Return number of occurrences of value.", + "nt.terminal_size.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "nt.terminal_size.lines" => "height of the terminal window in characters", + "nt.times" => "Return a collection containing process timing information.\n\nThe object returned behaves like a named tuple with these fields:\n (utime, stime, cutime, cstime, elapsed_time)\nAll fields are floating-point numbers.", + "nt.times_result" => "times_result: Result from os.times().\n\nThis object may be accessed either as a tuple of\n (user, system, children_user, children_system, elapsed),\nor via the attributes user, system, children_user, children_system,\nand elapsed.\n\nSee os.times for more information.", + "nt.times_result.__add__" => "Return self+value.", + "nt.times_result.__class_getitem__" => "See PEP 585", + "nt.times_result.__contains__" => "Return bool(key in self).", + "nt.times_result.__delattr__" => "Implement delattr(self, name).", + "nt.times_result.__eq__" => "Return self==value.", + "nt.times_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.times_result.__ge__" => "Return self>=value.", + "nt.times_result.__getattribute__" => "Return getattr(self, name).", + "nt.times_result.__getitem__" => "Return self[key].", + "nt.times_result.__getstate__" => "Helper for pickle.", + "nt.times_result.__gt__" => "Return self>value.", + "nt.times_result.__hash__" => "Return hash(self).", + "nt.times_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.times_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.times_result.__iter__" => "Implement iter(self).", + "nt.times_result.__le__" => "Return self<=value.", + "nt.times_result.__len__" => "Return len(self).", + "nt.times_result.__lt__" => "Return self<value.", + "nt.times_result.__mul__" => "Return self*value.", + "nt.times_result.__ne__" => "Return self!=value.", + "nt.times_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.times_result.__reduce_ex__" => "Helper for pickle.", + "nt.times_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "nt.times_result.__repr__" => "Return repr(self).", + "nt.times_result.__rmul__" => "Return value*self.", + "nt.times_result.__setattr__" => "Implement setattr(self, name, value).", + "nt.times_result.__sizeof__" => "Size of object in memory, in bytes.", + "nt.times_result.__str__" => "Return str(self).", + "nt.times_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.times_result.children_system" => "system time of children", + "nt.times_result.children_user" => "user time of children", + "nt.times_result.count" => "Return number of occurrences of value.", + "nt.times_result.elapsed" => "elapsed time since an arbitrary point in the past", + "nt.times_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "nt.times_result.system" => "system time", + "nt.times_result.user" => "user time", + "nt.truncate" => "Truncate a file, specified by path, to a specific length.\n\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.", + "nt.umask" => "Set the current numeric umask and return the previous umask.", + "nt.uname_result" => "uname_result: Result from os.uname().\n\nThis object may be accessed either as a tuple of\n (sysname, nodename, release, version, machine),\nor via the attributes sysname, nodename, release, version, and machine.\n\nSee os.uname for more information.", + "nt.uname_result.__add__" => "Return self+value.", + "nt.uname_result.__class_getitem__" => "See PEP 585", + "nt.uname_result.__contains__" => "Return bool(key in self).", + "nt.uname_result.__delattr__" => "Implement delattr(self, name).", + "nt.uname_result.__eq__" => "Return self==value.", + "nt.uname_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "nt.uname_result.__ge__" => "Return self>=value.", + "nt.uname_result.__getattribute__" => "Return getattr(self, name).", + "nt.uname_result.__getitem__" => "Return self[key].", + "nt.uname_result.__getstate__" => "Helper for pickle.", + "nt.uname_result.__gt__" => "Return self>value.", + "nt.uname_result.__hash__" => "Return hash(self).", + "nt.uname_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "nt.uname_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "nt.uname_result.__iter__" => "Implement iter(self).", + "nt.uname_result.__le__" => "Return self<=value.", + "nt.uname_result.__len__" => "Return len(self).", + "nt.uname_result.__lt__" => "Return self<value.", + "nt.uname_result.__mul__" => "Return self*value.", + "nt.uname_result.__ne__" => "Return self!=value.", + "nt.uname_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "nt.uname_result.__reduce_ex__" => "Helper for pickle.", + "nt.uname_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "nt.uname_result.__repr__" => "Return repr(self).", + "nt.uname_result.__rmul__" => "Return value*self.", + "nt.uname_result.__setattr__" => "Implement setattr(self, name, value).", + "nt.uname_result.__sizeof__" => "Size of object in memory, in bytes.", + "nt.uname_result.__str__" => "Return str(self).", + "nt.uname_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "nt.uname_result.count" => "Return number of occurrences of value.", + "nt.uname_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "nt.uname_result.machine" => "hardware identifier", + "nt.uname_result.nodename" => "name of machine on network (implementation-defined)", + "nt.uname_result.release" => "operating system release", + "nt.uname_result.sysname" => "operating system name", + "nt.uname_result.version" => "operating system version", + "nt.unlink" => "Remove a file (same as remove()).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "nt.unsetenv" => "Delete an environment variable.", + "nt.urandom" => "Return a bytes object containing random bytes suitable for cryptographic use.", + "nt.utime" => "Set the access and modified time of path.\n\npath may always be specified as a string.\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.\n\nIf times is not None, it must be a tuple (atime, mtime);\n atime and mtime should be expressed as float seconds since the epoch.\nIf ns is specified, it must be a tuple (atime_ns, mtime_ns);\n atime_ns and mtime_ns should be expressed as integer nanoseconds\n since the epoch.\nIf times is None and ns is unspecified, utime uses the current time.\nSpecifying tuples for both times and ns is an error.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, utime will modify the symbolic link itself instead of the file the\n link points to.\nIt is an error to use dir_fd or follow_symlinks when specifying path\n as an open file descriptor.\ndir_fd and follow_symlinks may not be available on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "nt.waitpid" => "Wait for completion of a given process.\n\nReturns a tuple of information regarding the process:\n (pid, status << 8)\n\nThe options argument is ignored on Windows.", + "nt.waitstatus_to_exitcode" => "Convert a wait status to an exit code.\n\nOn Unix:\n\n* If WIFEXITED(status) is true, return WEXITSTATUS(status).\n* If WIFSIGNALED(status) is true, return -WTERMSIG(status).\n* Otherwise, raise a ValueError.\n\nOn Windows, return status shifted right by 8 bits.\n\nOn Unix, if the process is being traced or if waitpid() was called with\nWUNTRACED option, the caller must first check if WIFSTOPPED(status) is true.\nThis function must not be called if WIFSTOPPED(status) is true.", + "nt.write" => "Write a bytes object to a file descriptor.", + "posix" => "This module provides access to operating system functionality that is\nstandardized by the C Standard and the POSIX standard (a thinly\ndisguised Unix interface). Refer to the library manual and\ncorresponding Unix manual entries for more information on calls.", + "posix.DirEntry.__class_getitem__" => "See PEP 585", + "posix.DirEntry.__delattr__" => "Implement delattr(self, name).", + "posix.DirEntry.__eq__" => "Return self==value.", + "posix.DirEntry.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.DirEntry.__fspath__" => "Returns the path for the entry.", + "posix.DirEntry.__ge__" => "Return self>=value.", + "posix.DirEntry.__getattribute__" => "Return getattr(self, name).", + "posix.DirEntry.__getstate__" => "Helper for pickle.", + "posix.DirEntry.__gt__" => "Return self>value.", + "posix.DirEntry.__hash__" => "Return hash(self).", + "posix.DirEntry.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.DirEntry.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.DirEntry.__le__" => "Return self<=value.", + "posix.DirEntry.__lt__" => "Return self<value.", + "posix.DirEntry.__ne__" => "Return self!=value.", + "posix.DirEntry.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.DirEntry.__reduce__" => "Helper for pickle.", + "posix.DirEntry.__reduce_ex__" => "Helper for pickle.", + "posix.DirEntry.__repr__" => "Return repr(self).", + "posix.DirEntry.__setattr__" => "Implement setattr(self, name, value).", + "posix.DirEntry.__sizeof__" => "Size of object in memory, in bytes.", + "posix.DirEntry.__str__" => "Return str(self).", + "posix.DirEntry.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.DirEntry.inode" => "Return inode of the entry; cached per entry.", + "posix.DirEntry.is_dir" => "Return True if the entry is a directory; cached per entry.", + "posix.DirEntry.is_file" => "Return True if the entry is a file; cached per entry.", + "posix.DirEntry.is_junction" => "Return True if the entry is a junction; cached per entry.", + "posix.DirEntry.is_symlink" => "Return True if the entry is a symbolic link; cached per entry.", + "posix.DirEntry.name" => "the entry's base filename, relative to scandir() \"path\" argument", + "posix.DirEntry.path" => "the entry's full path name; equivalent to os.path.join(scandir_path, entry.name)", + "posix.DirEntry.stat" => "Return stat_result object for the entry; cached per entry.", + "posix.WCOREDUMP" => "Return True if the process returning status was dumped to a core file.", + "posix.WEXITSTATUS" => "Return the process return code from status.", + "posix.WIFCONTINUED" => "Return True if a particular process was continued from a job control stop.\n\nReturn True if the process returning status was continued from a\njob control stop.", + "posix.WIFEXITED" => "Return True if the process returning status exited via the exit() system call.", + "posix.WIFSIGNALED" => "Return True if the process returning status was terminated by a signal.", + "posix.WIFSTOPPED" => "Return True if the process returning status was stopped.", + "posix.WSTOPSIG" => "Return the signal that stopped the process that provided the status value.", + "posix.WTERMSIG" => "Return the signal that terminated the process that provided the status value.", + "posix._create_environ" => "Create the environment dictionary.", + "posix._exit" => "Exit to the system with specified status, without normal exit processing.", + "posix._fcopyfile" => "Efficiently copy content or metadata of 2 regular file descriptors (macOS).", + "posix._inputhook" => "Calls PyOS_CallInputHook droppong the GIL first", + "posix._is_inputhook_installed" => "Checks if PyOS_CallInputHook is set", + "posix._path_normpath" => "Normalize path, eliminating double slashes, etc.", + "posix._path_splitroot_ex" => "Split a pathname into drive, root and tail.\n\nThe tail contains anything after the root.", + "posix.abort" => "Abort the interpreter immediately.\n\nThis function 'dumps core' or otherwise fails in the hardest way possible\non the hosting operating system. This function never returns.", + "posix.access" => "Use the real uid/gid to test for access to a path.\n\n path\n Path to be tested; can be string, bytes, or a path-like object.\n mode\n Operating-system mode bitfield. Can be F_OK to test existence,\n or the inclusive-OR of R_OK, W_OK, and X_OK.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that\n directory.\n effective_ids\n If True, access will use the effective uid/gid instead of\n the real uid/gid.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n access will examine the symbolic link itself instead of the file\n the link points to.\n\ndir_fd, effective_ids, and follow_symlinks may not be implemented\n on your platform. If they are unavailable, using them will raise a\n NotImplementedError.\n\nNote that most operations will use the effective uid/gid, therefore this\n routine can be used in a suid/sgid environment to test if the invoking user\n has the specified access to the path.", + "posix.chdir" => "Change the current working directory to the specified path.\n\npath may always be specified as a string.\nOn some platforms, path may also be specified as an open file descriptor.\nIf this functionality is unavailable, using it raises an exception.", + "posix.chflags" => "Set file flags.\n\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, chflags will change flags on the symbolic link itself instead of the\n file the link points to.\nfollow_symlinks may not be implemented on your platform. If it is\nunavailable, using it will raise a NotImplementedError.", + "posix.chmod" => "Change the access permissions of a file.\n\n path\n Path to be modified. May always be specified as a str, bytes, or a path-like object.\n On some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.\n mode\n Operating-system mode bitfield.\n Be careful when using number literals for *mode*. The conventional UNIX notation for\n numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n Python.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that\n directory.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n chmod will modify the symbolic link itself instead of the file\n the link points to.\n\nIt is an error to use dir_fd or follow_symlinks when specifying path as\n an open file descriptor.\ndir_fd and follow_symlinks may not be implemented on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "posix.chown" => "Change the owner and group id of path to the numeric uid and gid.\\\n\n path\n Path to be examined; can be string, bytes, a path-like object, or open-file-descriptor int.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that\n directory.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n stat will examine the symbolic link itself instead of the file\n the link points to.\n\npath may always be specified as a string.\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, chown will modify the symbolic link itself instead of the file the\n link points to.\nIt is an error to use dir_fd or follow_symlinks when specifying path as\n an open file descriptor.\ndir_fd and follow_symlinks may not be implemented on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "posix.chroot" => "Change root directory to path.", + "posix.close" => "Close a file descriptor.", + "posix.closerange" => "Closes all file descriptors in [fd_low, fd_high), ignoring errors.", + "posix.confstr" => "Return a string-valued system configuration variable.", + "posix.copy_file_range" => "Copy count bytes from one file descriptor to another.\n\n src\n Source file descriptor.\n dst\n Destination file descriptor.\n count\n Number of bytes to copy.\n offset_src\n Starting offset in src.\n offset_dst\n Starting offset in dst.\n\nIf offset_src is None, then src is read from the current position;\nrespectively for offset_dst.", + "posix.cpu_count" => "Return the number of logical CPUs in the system.\n\nReturn None if indeterminable.", + "posix.ctermid" => "Return the name of the controlling terminal for this process.", + "posix.device_encoding" => "Return a string describing the encoding of a terminal's file descriptor.\n\nThe file descriptor must be attached to a terminal.\nIf the device is not a terminal, return None.", + "posix.dup" => "Return a duplicate of a file descriptor.", + "posix.dup2" => "Duplicate file descriptor.", + "posix.eventfd" => "Creates and returns an event notification file descriptor.", + "posix.eventfd_read" => "Read eventfd value", + "posix.eventfd_write" => "Write eventfd value.", + "posix.execv" => "Execute an executable path with arguments, replacing current process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.", + "posix.execve" => "Execute an executable path with arguments, replacing current process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.", + "posix.fchdir" => "Change to the directory of the given file descriptor.\n\nfd must be opened on a directory, not a file.\nEquivalent to os.chdir(fd).", + "posix.fchmod" => "Change the access permissions of the file given by file descriptor fd.\n\n fd\n The file descriptor of the file to be modified.\n mode\n Operating-system mode bitfield.\n Be careful when using number literals for *mode*. The conventional UNIX notation for\n numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n Python.\n\nEquivalent to os.chmod(fd, mode).", + "posix.fchown" => "Change the owner and group id of the file specified by file descriptor.\n\nEquivalent to os.chown(fd, uid, gid).", + "posix.fdatasync" => "Force write of fd to disk without forcing update of metadata.", + "posix.fork" => "Fork a child process.\n\nReturn 0 to child process and PID of child to parent process.", + "posix.forkpty" => "Fork a new process with a new pseudo-terminal as controlling tty.\n\nReturns a tuple of (pid, master_fd).\nLike fork(), return pid of 0 to the child process,\nand pid of child to the parent process.\nTo both, return fd of newly opened pseudo-terminal.", + "posix.fpathconf" => "Return the configuration limit name for the file descriptor fd.\n\nIf there is no limit, return -1.", + "posix.fspath" => "Return the file system path representation of the object.\n\nIf the object is str or bytes, then allow it to pass through as-is. If the\nobject defines __fspath__(), then return the result of that method. All other\ntypes raise a TypeError.", + "posix.fstat" => "Perform a stat system call on the given file descriptor.\n\nLike stat(), but for an open file descriptor.\nEquivalent to os.stat(fd).", + "posix.fstatvfs" => "Perform an fstatvfs system call on the given fd.\n\nEquivalent to statvfs(fd).", + "posix.fsync" => "Force write of fd to disk.", + "posix.ftruncate" => "Truncate a file, specified by file descriptor, to a specific length.", + "posix.get_blocking" => "Get the blocking mode of the file descriptor.\n\nReturn False if the O_NONBLOCK flag is set, True if the flag is cleared.", + "posix.get_inheritable" => "Get the close-on-exe flag of the specified file descriptor.", + "posix.get_terminal_size" => "Return the size of the terminal window as (columns, lines).\n\nThe optional argument fd (default standard output) specifies\nwhich file descriptor should be queried.\n\nIf the file descriptor is not connected to a terminal, an OSError\nis thrown.\n\nThis function will only be defined if an implementation is\navailable for this system.\n\nshutil.get_terminal_size is the high-level function which should\nnormally be used, os.get_terminal_size is the low-level implementation.", + "posix.getcwd" => "Return a unicode string representing the current working directory.", + "posix.getcwdb" => "Return a bytes string representing the current working directory.", + "posix.getegid" => "Return the current process's effective group id.", + "posix.geteuid" => "Return the current process's effective user id.", + "posix.getgid" => "Return the current process's group id.", + "posix.getgrouplist" => "Returns a list of groups to which a user belongs.\n\n user\n username to lookup\n group\n base group id of the user", + "posix.getgroups" => "Return list of supplemental group IDs for the process.", + "posix.getloadavg" => "Return average recent system load information.\n\nReturn the number of processes in the system run queue averaged over\nthe last 1, 5, and 15 minutes as a tuple of three floats.\nRaises OSError if the load average was unobtainable.", + "posix.getlogin" => "Return the actual login name.", + "posix.getpgid" => "Call the system call getpgid(), and return the result.", + "posix.getpgrp" => "Return the current process group id.", + "posix.getpid" => "Return the current process id.", + "posix.getppid" => "Return the parent's process id.\n\nIf the parent process has already exited, Windows machines will still\nreturn its id; others systems will return the id of the 'init' process (1).", + "posix.getpriority" => "Return program scheduling priority.", + "posix.getrandom" => "Obtain a series of random bytes.", + "posix.getresgid" => "Return a tuple of the current process's real, effective, and saved group ids.", + "posix.getresuid" => "Return a tuple of the current process's real, effective, and saved user ids.", + "posix.getsid" => "Call the system call getsid(pid) and return the result.", + "posix.getuid" => "Return the current process's user id.", + "posix.getxattr" => "Return the value of extended attribute attribute on path.\n\npath may be either a string, a path-like object, or an open file descriptor.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, getxattr will examine the symbolic link itself instead of the file\n the link points to.", + "posix.grantpt" => "Grant access to the slave pseudo-terminal device.\n\n fd\n File descriptor of a master pseudo-terminal device.\n\nPerforms a grantpt() C function call.", + "posix.initgroups" => "Initialize the group access list.\n\nCall the system initgroups() to initialize the group access list with all of\nthe groups of which the specified username is a member, plus the specified\ngroup id.", + "posix.isatty" => "Return True if the fd is connected to a terminal.\n\nReturn True if the file descriptor is an open file descriptor\nconnected to the slave end of a terminal.", + "posix.kill" => "Kill a process with a signal.", + "posix.killpg" => "Kill a process group with a signal.", + "posix.lchflags" => "Set file flags.\n\nThis function will not follow symbolic links.\nEquivalent to chflags(path, flags, follow_symlinks=False).", + "posix.lchmod" => "Change the access permissions of a file, without following symbolic links.\n\nIf path is a symlink, this affects the link itself rather than the target.\nEquivalent to chmod(path, mode, follow_symlinks=False).\"", + "posix.lchown" => "Change the owner and group id of path to the numeric uid and gid.\n\nThis function will not follow symbolic links.\nEquivalent to os.chown(path, uid, gid, follow_symlinks=False).", + "posix.link" => "Create a hard link to a file.\n\nIf either src_dir_fd or dst_dir_fd is not None, it should be a file\n descriptor open to a directory, and the respective path string (src or dst)\n should be relative; the path will then be relative to that directory.\nIf follow_symlinks is False, and the last element of src is a symbolic\n link, link will create a link to the symbolic link itself instead of the\n file the link points to.\nsrc_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your\n platform. If they are unavailable, using them will raise a\n NotImplementedError.", + "posix.listdir" => "Return a list containing the names of the files in the directory.\n\npath can be specified as either str, bytes, or a path-like object. If path is bytes,\n the filenames returned will also be bytes; in all other circumstances\n the filenames returned will be str.\nIf path is None, uses the path='.'.\nOn some platforms, path may also be specified as an open file descriptor;\\\n the file descriptor must refer to a directory.\n If this functionality is unavailable, using it raises NotImplementedError.\n\nThe list is in arbitrary order. It does not include the special\nentries '.' and '..' even if they are present in the directory.", + "posix.listxattr" => "Return a list of extended attributes on path.\n\npath may be either None, a string, a path-like object, or an open file descriptor.\nif path is None, listxattr will examine the current directory.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, listxattr will examine the symbolic link itself instead of the file\n the link points to.", + "posix.lockf" => "Apply, test or remove a POSIX lock on an open file descriptor.\n\n fd\n An open file descriptor.\n command\n One of F_LOCK, F_TLOCK, F_ULOCK or F_TEST.\n length\n The number of bytes to lock, starting at the current position.", + "posix.login_tty" => "Prepare the tty of which fd is a file descriptor for a new login session.\n\nMake the calling process a session leader; make the tty the\ncontrolling tty, the stdin, the stdout, and the stderr of the\ncalling process; close fd.", + "posix.lseek" => "Set the position of a file descriptor. Return the new position.\n\n fd\n An open file descriptor, as returned by os.open().\n position\n Position, interpreted relative to 'whence'.\n whence\n The relative position to seek from. Valid values are:\n - SEEK_SET: seek from the start of the file.\n - SEEK_CUR: seek from the current file position.\n - SEEK_END: seek from the end of the file.\n\nThe return value is the number of bytes relative to the beginning of the file.", + "posix.lstat" => "Perform a stat system call on the given path, without following symbolic links.\n\nLike stat(), but do not follow symbolic links.\nEquivalent to stat(path, follow_symlinks=False).", + "posix.major" => "Extracts a device major number from a raw device number.", + "posix.makedev" => "Composes a raw device number from the major and minor device numbers.", + "posix.minor" => "Extracts a device minor number from a raw device number.", + "posix.mkdir" => "Create a directory.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.\n\nThe mode argument is ignored on Windows. Where it is used, the current umask\nvalue is first masked out.", + "posix.mkfifo" => "Create a \"fifo\" (a POSIX named pipe).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "posix.mknod" => "Create a node in the file system.\n\nCreate a node in the file system (file, device special file or named pipe)\nat path. mode specifies both the permissions to use and the\ntype of node to be created, being combined (bitwise OR) with one of\nS_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set on mode,\ndevice defines the newly created device special file (probably using\nos.makedev()). Otherwise device is ignored.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "posix.nice" => "Add increment to the priority of process and return the new priority.", + "posix.open" => "Open a file for low level IO. Returns a file descriptor (integer).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "posix.openpty" => "Open a pseudo-terminal.\n\nReturn a tuple of (master_fd, slave_fd) containing open file descriptors\nfor both the master and slave ends.", + "posix.pathconf" => "Return the configuration limit name for the file or directory path.\n\nIf there is no limit, return -1.\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.", + "posix.pidfd_open" => "Return a file descriptor referring to the process *pid*.\n\nThe descriptor can be used to perform process management without races and\nsignals.", + "posix.pipe" => "Create a pipe.\n\nReturns a tuple of two file descriptors:\n (read_fd, write_fd)", + "posix.pipe2" => "Create a pipe with flags set atomically.\n\nReturns a tuple of two file descriptors:\n (read_fd, write_fd)\n\nflags can be constructed by ORing together one or more of these values:\nO_NONBLOCK, O_CLOEXEC.", + "posix.posix_fadvise" => "Announce an intention to access data in a specific pattern.\n\nAnnounce an intention to access data in a specific pattern, thus allowing\nthe kernel to make optimizations.\nThe advice applies to the region of the file specified by fd starting at\noffset and continuing for length bytes.\nadvice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL,\nPOSIX_FADV_RANDOM, POSIX_FADV_NOREUSE, POSIX_FADV_WILLNEED, or\nPOSIX_FADV_DONTNEED.", + "posix.posix_fallocate" => "Ensure a file has allocated at least a particular number of bytes on disk.\n\nEnsure that the file specified by fd encompasses a range of bytes\nstarting at offset bytes from the beginning and continuing for length bytes.", + "posix.posix_openpt" => "Open and return a file descriptor for a master pseudo-terminal device.\n\nPerforms a posix_openpt() C function call. The oflag argument is used to\nset file status flags and file access modes as specified in the manual page\nof posix_openpt() of your system.", + "posix.posix_spawn" => "Execute the program specified by path in a new process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.\n file_actions\n A sequence of file action tuples.\n setpgroup\n The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\n resetids\n If the value is `true` the POSIX_SPAWN_RESETIDS will be activated.\n setsid\n If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n setsigmask\n The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n setsigdef\n The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag.\n scheduler\n A tuple with the scheduler policy (optional) and parameters.", + "posix.posix_spawnp" => "Execute the program specified by path in a new process.\n\n path\n Path of executable file.\n argv\n Tuple or list of strings.\n env\n Dictionary of strings mapping to strings.\n file_actions\n A sequence of file action tuples.\n setpgroup\n The pgroup to use with the POSIX_SPAWN_SETPGROUP flag.\n resetids\n If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n setsid\n If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n setsigmask\n The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n setsigdef\n The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag.\n scheduler\n A tuple with the scheduler policy (optional) and parameters.", + "posix.pread" => "Read a number of bytes from a file descriptor starting at a particular offset.\n\nRead length bytes from file descriptor fd, starting at offset bytes from\nthe beginning of the file. The file offset remains unchanged.", + "posix.preadv" => "Reads from a file descriptor into a number of mutable bytes-like objects.\n\nCombines the functionality of readv() and pread(). As readv(), it will\ntransfer data into each buffer until it is full and then move on to the next\nbuffer in the sequence to hold the rest of the data. Its fourth argument,\nspecifies the file offset at which the input operation is to be performed. It\nwill return the total number of bytes read (which can be less than the total\ncapacity of all the objects).\n\nThe flags argument contains a bitwise OR of zero or more of the following flags:\n\n- RWF_HIPRI\n- RWF_NOWAIT\n\nUsing non-zero flags requires Linux 4.6 or newer.", + "posix.ptsname" => "Return the name of the slave pseudo-terminal device.\n\n fd\n File descriptor of a master pseudo-terminal device.\n\nIf the ptsname_r() C function is available, it is called;\notherwise, performs a ptsname() C function call.", + "posix.putenv" => "Change or add an environment variable.", + "posix.pwrite" => "Write bytes to a file descriptor starting at a particular offset.\n\nWrite buffer to fd, starting at offset bytes from the beginning of\nthe file. Returns the number of bytes written. Does not change the\ncurrent file offset.", + "posix.pwritev" => "Writes the contents of bytes-like objects to a file descriptor at a given offset.\n\nCombines the functionality of writev() and pwrite(). All buffers must be a sequence\nof bytes-like objects. Buffers are processed in array order. Entire contents of first\nbuffer is written before proceeding to second, and so on. The operating system may\nset a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used.\nThis function writes the contents of each object to the file descriptor and returns\nthe total number of bytes written.\n\nThe flags argument contains a bitwise OR of zero or more of the following flags:\n\n- RWF_DSYNC\n- RWF_SYNC\n- RWF_APPEND\n\nUsing non-zero flags requires Linux 4.7 or newer.", + "posix.read" => "Read from a file descriptor. Returns a bytes object.", + "posix.readinto" => "Read into a buffer object from a file descriptor.\n\nThe buffer should be mutable and bytes-like. On success, returns the number of\nbytes read. Less bytes may be read than the size of the buffer. The underlying\nsystem call will be retried when interrupted by a signal, unless the signal\nhandler raises an exception. Other errors will not be retried and an error will\nbe raised.\n\nReturns 0 if *fd* is at end of file or if the provided *buffer* has length 0\n(which can be used to check for errors without reading data). Never returns\nnegative.", + "posix.readlink" => "Return a string representing the path to which the symbolic link points.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\nand path should be relative; path will then be relative to that directory.\n\ndir_fd may not be implemented on your platform. If it is unavailable,\nusing it will raise a NotImplementedError.", + "posix.readv" => "Read from a file descriptor fd into an iterable of buffers.\n\nThe buffers should be mutable buffers accepting bytes.\nreadv will transfer data into each buffer until it is full\nand then move on to the next buffer in the sequence to hold\nthe rest of the data.\n\nreadv returns the total number of bytes read,\nwhich may be less than the total capacity of all the buffers.", + "posix.register_at_fork" => "Register callables to be called when forking a new process.\n\n before\n A callable to be called in the parent before the fork() syscall.\n after_in_child\n A callable to be called in the child after fork().\n after_in_parent\n A callable to be called in the parent after fork().\n\n'before' callbacks are called in reverse order.\n'after_in_child' and 'after_in_parent' callbacks are called in order.", + "posix.remove" => "Remove a file (same as unlink()).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "posix.removexattr" => "Remove extended attribute attribute on path.\n\npath may be either a string, a path-like object, or an open file descriptor.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, removexattr will modify the symbolic link itself instead of the file\n the link points to.", + "posix.rename" => "Rename a file or directory.\n\nIf either src_dir_fd or dst_dir_fd is not None, it should be a file\n descriptor open to a directory, and the respective path string (src or dst)\n should be relative; the path will then be relative to that directory.\nsrc_dir_fd and dst_dir_fd, may not be implemented on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "posix.replace" => "Rename a file or directory, overwriting the destination.\n\nIf either src_dir_fd or dst_dir_fd is not None, it should be a file\n descriptor open to a directory, and the respective path string (src or dst)\n should be relative; the path will then be relative to that directory.\nsrc_dir_fd and dst_dir_fd, may not be implemented on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "posix.rmdir" => "Remove a directory.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "posix.scandir" => "Return an iterator of DirEntry objects for given path.\n\npath can be specified as either str, bytes, or a path-like object. If path\nis bytes, the names of yielded DirEntry objects will also be bytes; in\nall other circumstances they will be str.\n\nIf path is None, uses the path='.'.", + "posix.sched_get_priority_max" => "Get the maximum scheduling priority for policy.", + "posix.sched_get_priority_min" => "Get the minimum scheduling priority for policy.", + "posix.sched_getaffinity" => "Return the affinity of the process identified by pid (or the current process if zero).\n\nThe affinity is returned as a set of CPU identifiers.", + "posix.sched_getparam" => "Returns scheduling parameters for the process identified by pid.\n\nIf pid is 0, returns parameters for the calling process.\nReturn value is an instance of sched_param.", + "posix.sched_getscheduler" => "Get the scheduling policy for the process identified by pid.\n\nPassing 0 for pid returns the scheduling policy for the calling process.", + "posix.sched_param" => "Currently has only one field: sched_priority\n\n sched_priority\n A scheduling parameter.", + "posix.sched_param.__add__" => "Return self+value.", + "posix.sched_param.__class_getitem__" => "See PEP 585", + "posix.sched_param.__contains__" => "Return bool(key in self).", + "posix.sched_param.__delattr__" => "Implement delattr(self, name).", + "posix.sched_param.__eq__" => "Return self==value.", + "posix.sched_param.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.sched_param.__ge__" => "Return self>=value.", + "posix.sched_param.__getattribute__" => "Return getattr(self, name).", + "posix.sched_param.__getitem__" => "Return self[key].", + "posix.sched_param.__getstate__" => "Helper for pickle.", + "posix.sched_param.__gt__" => "Return self>value.", + "posix.sched_param.__hash__" => "Return hash(self).", + "posix.sched_param.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.sched_param.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.sched_param.__iter__" => "Implement iter(self).", + "posix.sched_param.__le__" => "Return self<=value.", + "posix.sched_param.__len__" => "Return len(self).", + "posix.sched_param.__lt__" => "Return self<value.", + "posix.sched_param.__mul__" => "Return self*value.", + "posix.sched_param.__ne__" => "Return self!=value.", + "posix.sched_param.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.sched_param.__reduce_ex__" => "Helper for pickle.", + "posix.sched_param.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.sched_param.__repr__" => "Return repr(self).", + "posix.sched_param.__rmul__" => "Return value*self.", + "posix.sched_param.__setattr__" => "Implement setattr(self, name, value).", + "posix.sched_param.__sizeof__" => "Size of object in memory, in bytes.", + "posix.sched_param.__str__" => "Return str(self).", + "posix.sched_param.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.sched_param.count" => "Return number of occurrences of value.", + "posix.sched_param.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.sched_param.sched_priority" => "the scheduling priority", + "posix.sched_rr_get_interval" => "Return the round-robin quantum for the process identified by pid, in seconds.\n\nValue returned is a float.", + "posix.sched_setaffinity" => "Set the CPU affinity of the process identified by pid to mask.\n\nmask should be an iterable of integers identifying CPUs.", + "posix.sched_setparam" => "Set scheduling parameters for the process identified by pid.\n\nIf pid is 0, sets parameters for the calling process.\nparam should be an instance of sched_param.", + "posix.sched_setscheduler" => "Set the scheduling policy for the process identified by pid.\n\nIf pid is 0, the calling process is changed.\nparam is an instance of sched_param.", + "posix.sched_yield" => "Voluntarily relinquish the CPU.", + "posix.sendfile" => "Copy count bytes from file descriptor in_fd to file descriptor out_fd.", + "posix.set_blocking" => "Set the blocking mode of the specified file descriptor.\n\nSet the O_NONBLOCK flag if blocking is False,\nclear the O_NONBLOCK flag otherwise.", + "posix.set_inheritable" => "Set the inheritable flag of the specified file descriptor.", + "posix.setegid" => "Set the current process's effective group id.", + "posix.seteuid" => "Set the current process's effective user id.", + "posix.setgid" => "Set the current process's group id.", + "posix.setgroups" => "Set the groups of the current process to list.", + "posix.setns" => "Move the calling thread into different namespaces.\n\n fd\n A file descriptor to a namespace.\n nstype\n Type of namespace.", + "posix.setpgid" => "Call the system call setpgid(pid, pgrp).", + "posix.setpgrp" => "Make the current process the leader of its process group.", + "posix.setpriority" => "Set program scheduling priority.", + "posix.setregid" => "Set the current process's real and effective group ids.", + "posix.setresgid" => "Set the current process's real, effective, and saved group ids.", + "posix.setresuid" => "Set the current process's real, effective, and saved user ids.", + "posix.setreuid" => "Set the current process's real and effective user ids.", + "posix.setsid" => "Call the system call setsid().", + "posix.setuid" => "Set the current process's user id.", + "posix.setxattr" => "Set extended attribute attribute on path to value.\n\npath may be either a string, a path-like object, or an open file descriptor.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, setxattr will modify the symbolic link itself instead of the file\n the link points to.", + "posix.splice" => "Transfer count bytes from one pipe to a descriptor or vice versa.\n\n src\n Source file descriptor.\n dst\n Destination file descriptor.\n count\n Number of bytes to copy.\n offset_src\n Starting offset in src.\n offset_dst\n Starting offset in dst.\n flags\n Flags to modify the semantics of the call.\n\nIf offset_src is None, then src is read from the current position;\nrespectively for offset_dst. The offset associated to the file\ndescriptor that refers to a pipe must be None.", + "posix.stat" => "Perform a stat system call on the given path.\n\n path\n Path to be examined; can be string, bytes, a path-like object or\n open-file-descriptor int.\n dir_fd\n If not None, it should be a file descriptor open to a directory,\n and path should be a relative string; path will then be relative to\n that directory.\n follow_symlinks\n If False, and the last element of the path is a symbolic link,\n stat will examine the symbolic link itself instead of the file\n the link points to.\n\ndir_fd and follow_symlinks may not be implemented\n on your platform. If they are unavailable, using them will raise a\n NotImplementedError.\n\nIt's an error to use dir_fd or follow_symlinks when specifying path as\n an open file descriptor.", + "posix.stat_result" => "stat_result: Result from stat, fstat, or lstat.\n\nThis object may be accessed either as a tuple of\n (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)\nor via the attributes st_mode, st_ino, st_dev, st_nlink, st_uid, and so on.\n\nPosix/windows: If your platform supports st_blksize, st_blocks, st_rdev,\nor st_flags, they are available as attributes only.\n\nSee os.stat for more information.", + "posix.stat_result.__add__" => "Return self+value.", + "posix.stat_result.__class_getitem__" => "See PEP 585", + "posix.stat_result.__contains__" => "Return bool(key in self).", + "posix.stat_result.__delattr__" => "Implement delattr(self, name).", + "posix.stat_result.__eq__" => "Return self==value.", + "posix.stat_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.stat_result.__ge__" => "Return self>=value.", + "posix.stat_result.__getattribute__" => "Return getattr(self, name).", + "posix.stat_result.__getitem__" => "Return self[key].", + "posix.stat_result.__getstate__" => "Helper for pickle.", + "posix.stat_result.__gt__" => "Return self>value.", + "posix.stat_result.__hash__" => "Return hash(self).", + "posix.stat_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.stat_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.stat_result.__iter__" => "Implement iter(self).", + "posix.stat_result.__le__" => "Return self<=value.", + "posix.stat_result.__len__" => "Return len(self).", + "posix.stat_result.__lt__" => "Return self<value.", + "posix.stat_result.__mul__" => "Return self*value.", + "posix.stat_result.__ne__" => "Return self!=value.", + "posix.stat_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.stat_result.__reduce_ex__" => "Helper for pickle.", + "posix.stat_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.stat_result.__repr__" => "Return repr(self).", + "posix.stat_result.__rmul__" => "Return value*self.", + "posix.stat_result.__setattr__" => "Implement setattr(self, name, value).", + "posix.stat_result.__sizeof__" => "Size of object in memory, in bytes.", + "posix.stat_result.__str__" => "Return str(self).", + "posix.stat_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.stat_result.count" => "Return number of occurrences of value.", + "posix.stat_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.stat_result.st_atime" => "time of last access", + "posix.stat_result.st_atime_ns" => "time of last access in nanoseconds", + "posix.stat_result.st_birthtime" => "time of creation", + "posix.stat_result.st_blksize" => "blocksize for filesystem I/O", + "posix.stat_result.st_blocks" => "number of blocks allocated", + "posix.stat_result.st_ctime" => "time of last change", + "posix.stat_result.st_ctime_ns" => "time of last change in nanoseconds", + "posix.stat_result.st_dev" => "device", + "posix.stat_result.st_flags" => "user defined flags for file", + "posix.stat_result.st_gen" => "generation number", + "posix.stat_result.st_gid" => "group ID of owner", + "posix.stat_result.st_ino" => "inode", + "posix.stat_result.st_mode" => "protection bits", + "posix.stat_result.st_mtime" => "time of last modification", + "posix.stat_result.st_mtime_ns" => "time of last modification in nanoseconds", + "posix.stat_result.st_nlink" => "number of hard links", + "posix.stat_result.st_rdev" => "device type (if inode device)", + "posix.stat_result.st_size" => "total size, in bytes", + "posix.stat_result.st_uid" => "user ID of owner", + "posix.statvfs" => "Perform a statvfs system call on the given path.\n\npath may always be specified as a string.\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.", + "posix.statvfs_result" => "statvfs_result: Result from statvfs or fstatvfs.\n\nThis object may be accessed either as a tuple of\n (bsize, frsize, blocks, bfree, bavail, files, ffree, favail, flag, namemax),\nor via the attributes f_bsize, f_frsize, f_blocks, f_bfree, and so on.\n\nSee os.statvfs for more information.", + "posix.statvfs_result.__add__" => "Return self+value.", + "posix.statvfs_result.__class_getitem__" => "See PEP 585", + "posix.statvfs_result.__contains__" => "Return bool(key in self).", + "posix.statvfs_result.__delattr__" => "Implement delattr(self, name).", + "posix.statvfs_result.__eq__" => "Return self==value.", + "posix.statvfs_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.statvfs_result.__ge__" => "Return self>=value.", + "posix.statvfs_result.__getattribute__" => "Return getattr(self, name).", + "posix.statvfs_result.__getitem__" => "Return self[key].", + "posix.statvfs_result.__getstate__" => "Helper for pickle.", + "posix.statvfs_result.__gt__" => "Return self>value.", + "posix.statvfs_result.__hash__" => "Return hash(self).", + "posix.statvfs_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.statvfs_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.statvfs_result.__iter__" => "Implement iter(self).", + "posix.statvfs_result.__le__" => "Return self<=value.", + "posix.statvfs_result.__len__" => "Return len(self).", + "posix.statvfs_result.__lt__" => "Return self<value.", + "posix.statvfs_result.__mul__" => "Return self*value.", + "posix.statvfs_result.__ne__" => "Return self!=value.", + "posix.statvfs_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.statvfs_result.__reduce_ex__" => "Helper for pickle.", + "posix.statvfs_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.statvfs_result.__repr__" => "Return repr(self).", + "posix.statvfs_result.__rmul__" => "Return value*self.", + "posix.statvfs_result.__setattr__" => "Implement setattr(self, name, value).", + "posix.statvfs_result.__sizeof__" => "Size of object in memory, in bytes.", + "posix.statvfs_result.__str__" => "Return str(self).", + "posix.statvfs_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.statvfs_result.count" => "Return number of occurrences of value.", + "posix.statvfs_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.strerror" => "Translate an error code to a message string.", + "posix.symlink" => "Create a symbolic link pointing to src named dst.\n\ntarget_is_directory is required on Windows if the target is to be\n interpreted as a directory. (On Windows, symlink requires\n Windows 6.0 or greater, and raises a NotImplementedError otherwise.)\n target_is_directory is ignored on non-Windows platforms.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "posix.sync" => "Force write of everything to disk.", + "posix.sysconf" => "Return an integer-valued system configuration variable.", + "posix.system" => "Execute the command in a subshell.", + "posix.tcgetpgrp" => "Return the process group associated with the terminal specified by fd.", + "posix.tcsetpgrp" => "Set the process group associated with the terminal specified by fd.", + "posix.terminal_size" => "A tuple of (columns, lines) for holding terminal window size", + "posix.terminal_size.__add__" => "Return self+value.", + "posix.terminal_size.__class_getitem__" => "See PEP 585", + "posix.terminal_size.__contains__" => "Return bool(key in self).", + "posix.terminal_size.__delattr__" => "Implement delattr(self, name).", + "posix.terminal_size.__eq__" => "Return self==value.", + "posix.terminal_size.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.terminal_size.__ge__" => "Return self>=value.", + "posix.terminal_size.__getattribute__" => "Return getattr(self, name).", + "posix.terminal_size.__getitem__" => "Return self[key].", + "posix.terminal_size.__getstate__" => "Helper for pickle.", + "posix.terminal_size.__gt__" => "Return self>value.", + "posix.terminal_size.__hash__" => "Return hash(self).", + "posix.terminal_size.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.terminal_size.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.terminal_size.__iter__" => "Implement iter(self).", + "posix.terminal_size.__le__" => "Return self<=value.", + "posix.terminal_size.__len__" => "Return len(self).", + "posix.terminal_size.__lt__" => "Return self<value.", + "posix.terminal_size.__mul__" => "Return self*value.", + "posix.terminal_size.__ne__" => "Return self!=value.", + "posix.terminal_size.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.terminal_size.__reduce_ex__" => "Helper for pickle.", + "posix.terminal_size.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.terminal_size.__repr__" => "Return repr(self).", + "posix.terminal_size.__rmul__" => "Return value*self.", + "posix.terminal_size.__setattr__" => "Implement setattr(self, name, value).", + "posix.terminal_size.__sizeof__" => "Size of object in memory, in bytes.", + "posix.terminal_size.__str__" => "Return str(self).", + "posix.terminal_size.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.terminal_size.columns" => "width of the terminal window in characters", + "posix.terminal_size.count" => "Return number of occurrences of value.", + "posix.terminal_size.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.terminal_size.lines" => "height of the terminal window in characters", + "posix.timerfd_create" => "Create and return a timer file descriptor.\n\n clockid\n A valid clock ID constant as timer file descriptor.\n\n time.CLOCK_REALTIME\n time.CLOCK_MONOTONIC\n time.CLOCK_BOOTTIME\n flags\n 0 or a bit mask of os.TFD_NONBLOCK or os.TFD_CLOEXEC.\n\n os.TFD_NONBLOCK\n If *TFD_NONBLOCK* is set as a flag, read doesn't blocks.\n If *TFD_NONBLOCK* is not set as a flag, read block until the timer fires.\n\n os.TFD_CLOEXEC\n If *TFD_CLOEXEC* is set as a flag, enable the close-on-exec flag", + "posix.timerfd_gettime" => "Return a tuple of a timer file descriptor's (interval, next expiration) in float seconds.\n\n fd\n A timer file descriptor.", + "posix.timerfd_gettime_ns" => "Return a tuple of a timer file descriptor's (interval, next expiration) in nanoseconds.\n\n fd\n A timer file descriptor.", + "posix.timerfd_settime" => "Alter a timer file descriptor's internal timer in seconds.\n\n fd\n A timer file descriptor.\n flags\n 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET.\n initial\n The initial expiration time, in seconds.\n interval\n The timer's interval, in seconds.", + "posix.timerfd_settime_ns" => "Alter a timer file descriptor's internal timer in nanoseconds.\n\n fd\n A timer file descriptor.\n flags\n 0 or a bit mask of TFD_TIMER_ABSTIME or TFD_TIMER_CANCEL_ON_SET.\n initial\n initial expiration timing in seconds.\n interval\n interval for the timer in seconds.", + "posix.times" => "Return a collection containing process timing information.\n\nThe object returned behaves like a named tuple with these fields:\n (utime, stime, cutime, cstime, elapsed_time)\nAll fields are floating-point numbers.", + "posix.times_result" => "times_result: Result from os.times().\n\nThis object may be accessed either as a tuple of\n (user, system, children_user, children_system, elapsed),\nor via the attributes user, system, children_user, children_system,\nand elapsed.\n\nSee os.times for more information.", + "posix.times_result.__add__" => "Return self+value.", + "posix.times_result.__class_getitem__" => "See PEP 585", + "posix.times_result.__contains__" => "Return bool(key in self).", + "posix.times_result.__delattr__" => "Implement delattr(self, name).", + "posix.times_result.__eq__" => "Return self==value.", + "posix.times_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.times_result.__ge__" => "Return self>=value.", + "posix.times_result.__getattribute__" => "Return getattr(self, name).", + "posix.times_result.__getitem__" => "Return self[key].", + "posix.times_result.__getstate__" => "Helper for pickle.", + "posix.times_result.__gt__" => "Return self>value.", + "posix.times_result.__hash__" => "Return hash(self).", + "posix.times_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.times_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.times_result.__iter__" => "Implement iter(self).", + "posix.times_result.__le__" => "Return self<=value.", + "posix.times_result.__len__" => "Return len(self).", + "posix.times_result.__lt__" => "Return self<value.", + "posix.times_result.__mul__" => "Return self*value.", + "posix.times_result.__ne__" => "Return self!=value.", + "posix.times_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.times_result.__reduce_ex__" => "Helper for pickle.", + "posix.times_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.times_result.__repr__" => "Return repr(self).", + "posix.times_result.__rmul__" => "Return value*self.", + "posix.times_result.__setattr__" => "Implement setattr(self, name, value).", + "posix.times_result.__sizeof__" => "Size of object in memory, in bytes.", + "posix.times_result.__str__" => "Return str(self).", + "posix.times_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.times_result.children_system" => "system time of children", + "posix.times_result.children_user" => "user time of children", + "posix.times_result.count" => "Return number of occurrences of value.", + "posix.times_result.elapsed" => "elapsed time since an arbitrary point in the past", + "posix.times_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.times_result.system" => "system time", + "posix.times_result.user" => "user time", + "posix.truncate" => "Truncate a file, specified by path, to a specific length.\n\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.", + "posix.ttyname" => "Return the name of the terminal device connected to 'fd'.\n\n fd\n Integer file descriptor handle.", + "posix.umask" => "Set the current numeric umask and return the previous umask.", + "posix.uname" => "Return an object identifying the current operating system.\n\nThe object behaves like a named tuple with the following fields:\n (sysname, nodename, release, version, machine)", + "posix.uname_result" => "uname_result: Result from os.uname().\n\nThis object may be accessed either as a tuple of\n (sysname, nodename, release, version, machine),\nor via the attributes sysname, nodename, release, version, and machine.\n\nSee os.uname for more information.", + "posix.uname_result.__add__" => "Return self+value.", + "posix.uname_result.__class_getitem__" => "See PEP 585", + "posix.uname_result.__contains__" => "Return bool(key in self).", + "posix.uname_result.__delattr__" => "Implement delattr(self, name).", + "posix.uname_result.__eq__" => "Return self==value.", + "posix.uname_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.uname_result.__ge__" => "Return self>=value.", + "posix.uname_result.__getattribute__" => "Return getattr(self, name).", + "posix.uname_result.__getitem__" => "Return self[key].", + "posix.uname_result.__getstate__" => "Helper for pickle.", + "posix.uname_result.__gt__" => "Return self>value.", + "posix.uname_result.__hash__" => "Return hash(self).", + "posix.uname_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.uname_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.uname_result.__iter__" => "Implement iter(self).", + "posix.uname_result.__le__" => "Return self<=value.", + "posix.uname_result.__len__" => "Return len(self).", + "posix.uname_result.__lt__" => "Return self<value.", + "posix.uname_result.__mul__" => "Return self*value.", + "posix.uname_result.__ne__" => "Return self!=value.", + "posix.uname_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.uname_result.__reduce_ex__" => "Helper for pickle.", + "posix.uname_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.uname_result.__repr__" => "Return repr(self).", + "posix.uname_result.__rmul__" => "Return value*self.", + "posix.uname_result.__setattr__" => "Implement setattr(self, name, value).", + "posix.uname_result.__sizeof__" => "Size of object in memory, in bytes.", + "posix.uname_result.__str__" => "Return str(self).", + "posix.uname_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.uname_result.count" => "Return number of occurrences of value.", + "posix.uname_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.uname_result.machine" => "hardware identifier", + "posix.uname_result.nodename" => "name of machine on network (implementation-defined)", + "posix.uname_result.release" => "operating system release", + "posix.uname_result.sysname" => "operating system name", + "posix.uname_result.version" => "operating system version", + "posix.unlink" => "Remove a file (same as remove()).\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\ndir_fd may not be implemented on your platform.\n If it is unavailable, using it will raise a NotImplementedError.", + "posix.unlockpt" => "Unlock a pseudo-terminal master/slave pair.\n\n fd\n File descriptor of a master pseudo-terminal device.\n\nPerforms an unlockpt() C function call.", + "posix.unsetenv" => "Delete an environment variable.", + "posix.unshare" => "Disassociate parts of a process (or thread) execution context.\n\n flags\n Namespaces to be unshared.", + "posix.urandom" => "Return a bytes object containing random bytes suitable for cryptographic use.", + "posix.utime" => "Set the access and modified time of path.\n\npath may always be specified as a string.\nOn some platforms, path may also be specified as an open file descriptor.\n If this functionality is unavailable, using it raises an exception.\n\nIf times is not None, it must be a tuple (atime, mtime);\n atime and mtime should be expressed as float seconds since the epoch.\nIf ns is specified, it must be a tuple (atime_ns, mtime_ns);\n atime_ns and mtime_ns should be expressed as integer nanoseconds\n since the epoch.\nIf times is None and ns is unspecified, utime uses the current time.\nSpecifying tuples for both times and ns is an error.\n\nIf dir_fd is not None, it should be a file descriptor open to a directory,\n and path should be relative; path will then be relative to that directory.\nIf follow_symlinks is False, and the last element of the path is a symbolic\n link, utime will modify the symbolic link itself instead of the file the\n link points to.\nIt is an error to use dir_fd or follow_symlinks when specifying path\n as an open file descriptor.\ndir_fd and follow_symlinks may not be available on your platform.\n If they are unavailable, using them will raise a NotImplementedError.", + "posix.wait" => "Wait for completion of a child process.\n\nReturns a tuple of information about the child process:\n (pid, status)", + "posix.wait3" => "Wait for completion of a child process.\n\nReturns a tuple of information about the child process:\n (pid, status, rusage)", + "posix.wait4" => "Wait for completion of a specific child process.\n\nReturns a tuple of information about the child process:\n (pid, status, rusage)", + "posix.waitid" => "Returns the result of waiting for a process or processes.\n\n idtype\n Must be one of be P_PID, P_PGID or P_ALL.\n id\n The id to wait on.\n options\n Constructed from the ORing of one or more of WEXITED, WSTOPPED\n or WCONTINUED and additionally may be ORed with WNOHANG or WNOWAIT.\n\nReturns either waitid_result or None if WNOHANG is specified and there are\nno children in a waitable state.", + "posix.waitid_result" => "waitid_result: Result from waitid.\n\nThis object may be accessed either as a tuple of\n (si_pid, si_uid, si_signo, si_status, si_code),\nor via the attributes si_pid, si_uid, and so on.\n\nSee os.waitid for more information.", + "posix.waitid_result.__add__" => "Return self+value.", + "posix.waitid_result.__class_getitem__" => "See PEP 585", + "posix.waitid_result.__contains__" => "Return bool(key in self).", + "posix.waitid_result.__delattr__" => "Implement delattr(self, name).", + "posix.waitid_result.__eq__" => "Return self==value.", + "posix.waitid_result.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "posix.waitid_result.__ge__" => "Return self>=value.", + "posix.waitid_result.__getattribute__" => "Return getattr(self, name).", + "posix.waitid_result.__getitem__" => "Return self[key].", + "posix.waitid_result.__getstate__" => "Helper for pickle.", + "posix.waitid_result.__gt__" => "Return self>value.", + "posix.waitid_result.__hash__" => "Return hash(self).", + "posix.waitid_result.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "posix.waitid_result.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "posix.waitid_result.__iter__" => "Implement iter(self).", + "posix.waitid_result.__le__" => "Return self<=value.", + "posix.waitid_result.__len__" => "Return len(self).", + "posix.waitid_result.__lt__" => "Return self<value.", + "posix.waitid_result.__mul__" => "Return self*value.", + "posix.waitid_result.__ne__" => "Return self!=value.", + "posix.waitid_result.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "posix.waitid_result.__reduce_ex__" => "Helper for pickle.", + "posix.waitid_result.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "posix.waitid_result.__repr__" => "Return repr(self).", + "posix.waitid_result.__rmul__" => "Return value*self.", + "posix.waitid_result.__setattr__" => "Implement setattr(self, name, value).", + "posix.waitid_result.__sizeof__" => "Size of object in memory, in bytes.", + "posix.waitid_result.__str__" => "Return str(self).", + "posix.waitid_result.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "posix.waitid_result.count" => "Return number of occurrences of value.", + "posix.waitid_result.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "posix.waitpid" => "Wait for completion of a given child process.\n\nReturns a tuple of information regarding the child process:\n (pid, status)\n\nThe options argument is ignored on Windows.", + "posix.waitstatus_to_exitcode" => "Convert a wait status to an exit code.\n\nOn Unix:\n\n* If WIFEXITED(status) is true, return WEXITSTATUS(status).\n* If WIFSIGNALED(status) is true, return -WTERMSIG(status).\n* Otherwise, raise a ValueError.\n\nOn Windows, return status shifted right by 8 bits.\n\nOn Unix, if the process is being traced or if waitpid() was called with\nWUNTRACED option, the caller must first check if WIFSTOPPED(status) is true.\nThis function must not be called if WIFSTOPPED(status) is true.", + "posix.write" => "Write a bytes object to a file descriptor.", + "posix.writev" => "Iterate over buffers, and write the contents of each to a file descriptor.\n\nReturns the total number of bytes written.\nbuffers must be a sequence of bytes-like objects.", + "pwd" => "This module provides access to the Unix password database.\nIt is available on all Unix versions.\n\nPassword database entries are reported as 7-tuples containing the following\nitems from the password database (see `<pwd.h>'), in order:\npw_name, pw_passwd, pw_uid, pw_gid, pw_gecos, pw_dir, pw_shell.\nThe uid and gid items are integers, all others are strings. An\nexception is raised if the entry asked for cannot be found.", + "pwd.getpwall" => "Return a list of all available password database entries, in arbitrary order.\n\nSee help(pwd) for more on password database entries.", + "pwd.getpwnam" => "Return the password database entry for the given user name.\n\nSee `help(pwd)` for more on password database entries.", + "pwd.getpwuid" => "Return the password database entry for the given numeric user ID.\n\nSee `help(pwd)` for more on password database entries.", + "pwd.struct_passwd" => "pwd.struct_passwd: Results from getpw*() routines.\n\nThis object may be accessed either as a tuple of\n (pw_name,pw_passwd,pw_uid,pw_gid,pw_gecos,pw_dir,pw_shell)\nor via the object attributes as named in the above tuple.", + "pwd.struct_passwd.__add__" => "Return self+value.", + "pwd.struct_passwd.__class_getitem__" => "See PEP 585", + "pwd.struct_passwd.__contains__" => "Return bool(key in self).", + "pwd.struct_passwd.__delattr__" => "Implement delattr(self, name).", + "pwd.struct_passwd.__eq__" => "Return self==value.", + "pwd.struct_passwd.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "pwd.struct_passwd.__ge__" => "Return self>=value.", + "pwd.struct_passwd.__getattribute__" => "Return getattr(self, name).", + "pwd.struct_passwd.__getitem__" => "Return self[key].", + "pwd.struct_passwd.__getstate__" => "Helper for pickle.", + "pwd.struct_passwd.__gt__" => "Return self>value.", + "pwd.struct_passwd.__hash__" => "Return hash(self).", + "pwd.struct_passwd.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "pwd.struct_passwd.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "pwd.struct_passwd.__iter__" => "Implement iter(self).", + "pwd.struct_passwd.__le__" => "Return self<=value.", + "pwd.struct_passwd.__len__" => "Return len(self).", + "pwd.struct_passwd.__lt__" => "Return self<value.", + "pwd.struct_passwd.__mul__" => "Return self*value.", + "pwd.struct_passwd.__ne__" => "Return self!=value.", + "pwd.struct_passwd.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "pwd.struct_passwd.__reduce_ex__" => "Helper for pickle.", + "pwd.struct_passwd.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "pwd.struct_passwd.__repr__" => "Return repr(self).", + "pwd.struct_passwd.__rmul__" => "Return value*self.", + "pwd.struct_passwd.__setattr__" => "Implement setattr(self, name, value).", + "pwd.struct_passwd.__sizeof__" => "Size of object in memory, in bytes.", + "pwd.struct_passwd.__str__" => "Return str(self).", + "pwd.struct_passwd.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "pwd.struct_passwd.count" => "Return number of occurrences of value.", + "pwd.struct_passwd.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "pwd.struct_passwd.pw_dir" => "home directory", + "pwd.struct_passwd.pw_gecos" => "real name", + "pwd.struct_passwd.pw_gid" => "group id", + "pwd.struct_passwd.pw_name" => "user name", + "pwd.struct_passwd.pw_passwd" => "password", + "pwd.struct_passwd.pw_shell" => "shell program", + "pwd.struct_passwd.pw_uid" => "user id", + "pyexpat" => "Python wrapper for Expat parser.", + "pyexpat.ErrorString" => "Returns string error for given number.", + "pyexpat.ExpatError.__delattr__" => "Implement delattr(self, name).", + "pyexpat.ExpatError.__eq__" => "Return self==value.", + "pyexpat.ExpatError.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "pyexpat.ExpatError.__ge__" => "Return self>=value.", + "pyexpat.ExpatError.__getattribute__" => "Return getattr(self, name).", + "pyexpat.ExpatError.__getstate__" => "Helper for pickle.", + "pyexpat.ExpatError.__gt__" => "Return self>value.", + "pyexpat.ExpatError.__hash__" => "Return hash(self).", + "pyexpat.ExpatError.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "pyexpat.ExpatError.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "pyexpat.ExpatError.__le__" => "Return self<=value.", + "pyexpat.ExpatError.__lt__" => "Return self<value.", + "pyexpat.ExpatError.__ne__" => "Return self!=value.", + "pyexpat.ExpatError.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "pyexpat.ExpatError.__reduce_ex__" => "Helper for pickle.", + "pyexpat.ExpatError.__repr__" => "Return repr(self).", + "pyexpat.ExpatError.__setattr__" => "Implement setattr(self, name, value).", + "pyexpat.ExpatError.__sizeof__" => "Size of object in memory, in bytes.", + "pyexpat.ExpatError.__str__" => "Return str(self).", + "pyexpat.ExpatError.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "pyexpat.ExpatError.__weakref__" => "list of weak references to the object", + "pyexpat.ExpatError.add_note" => "Add a note to the exception", + "pyexpat.ExpatError.with_traceback" => "Set self.__traceback__ to tb and return self.", + "pyexpat.ParserCreate" => "Return a new XML parser object.", + "pyexpat.XMLParserType" => "XML parser", + "pyexpat.XMLParserType.ExternalEntityParserCreate" => "Create a parser for parsing an external entity based on the information passed to the ExternalEntityRefHandler.", + "pyexpat.XMLParserType.GetBase" => "Return base URL string for the parser.", + "pyexpat.XMLParserType.GetInputContext" => "Return the untranslated text of the input that caused the current event.\n\nIf the event was generated by a large amount of text (such as a start tag\nfor an element with many attributes), not all of the text may be available.", + "pyexpat.XMLParserType.GetReparseDeferralEnabled" => "Retrieve reparse deferral enabled status; always returns false with Expat <2.6.0.", + "pyexpat.XMLParserType.Parse" => "Parse XML data.\n\n'isfinal' should be true at end of input.", + "pyexpat.XMLParserType.ParseFile" => "Parse XML data from file-like object.", + "pyexpat.XMLParserType.SetAllocTrackerActivationThreshold" => "Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM.\n\nBy default, parser objects have an allocation activation threshold of 64 MiB.", + "pyexpat.XMLParserType.SetAllocTrackerMaximumAmplification" => "Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated.\n\nThe amplification factor is calculated as \"allocated / direct\" while parsing,\nwhere \"direct\" is the number of bytes read from the primary document in parsing\nand \"allocated\" is the number of bytes of dynamic memory allocated in the parser\nhierarchy.\n\nThe 'max_factor' value must be a non-NaN floating point value greater than\nor equal to 1.0. Amplification factors greater than 100.0 can be observed\nnear the start of parsing even with benign files in practice. In particular,\nthe activation threshold should be carefully chosen to avoid false positives.\n\nBy default, parser objects have a maximum amplification factor of 100.0.", + "pyexpat.XMLParserType.SetBase" => "Set the base URL for the parser.", + "pyexpat.XMLParserType.SetParamEntityParsing" => "Controls parsing of parameter entities (including the external DTD subset).\n\nPossible flag values are XML_PARAM_ENTITY_PARSING_NEVER,\nXML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE and\nXML_PARAM_ENTITY_PARSING_ALWAYS. Returns true if setting the flag\nwas successful.", + "pyexpat.XMLParserType.SetReparseDeferralEnabled" => "Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0.", + "pyexpat.XMLParserType.UseForeignDTD" => "Allows the application to provide an artificial external subset if one is not specified as part of the document instance.\n\nThis readily allows the use of a 'default' document type controlled by the\napplication, while still getting the advantage of providing document type\ninformation to the parser. 'flag' defaults to True if not provided.", + "pyexpat.XMLParserType.__delattr__" => "Implement delattr(self, name).", + "pyexpat.XMLParserType.__eq__" => "Return self==value.", + "pyexpat.XMLParserType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "pyexpat.XMLParserType.__ge__" => "Return self>=value.", + "pyexpat.XMLParserType.__getattribute__" => "Return getattr(self, name).", + "pyexpat.XMLParserType.__getstate__" => "Helper for pickle.", + "pyexpat.XMLParserType.__gt__" => "Return self>value.", + "pyexpat.XMLParserType.__hash__" => "Return hash(self).", + "pyexpat.XMLParserType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "pyexpat.XMLParserType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "pyexpat.XMLParserType.__le__" => "Return self<=value.", + "pyexpat.XMLParserType.__lt__" => "Return self<value.", + "pyexpat.XMLParserType.__ne__" => "Return self!=value.", + "pyexpat.XMLParserType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "pyexpat.XMLParserType.__reduce__" => "Helper for pickle.", + "pyexpat.XMLParserType.__reduce_ex__" => "Helper for pickle.", + "pyexpat.XMLParserType.__repr__" => "Return repr(self).", + "pyexpat.XMLParserType.__setattr__" => "Implement setattr(self, name, value).", + "pyexpat.XMLParserType.__sizeof__" => "Size of object in memory, in bytes.", + "pyexpat.XMLParserType.__str__" => "Return str(self).", + "pyexpat.XMLParserType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "pyexpat.error.__delattr__" => "Implement delattr(self, name).", + "pyexpat.error.__eq__" => "Return self==value.", + "pyexpat.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "pyexpat.error.__ge__" => "Return self>=value.", + "pyexpat.error.__getattribute__" => "Return getattr(self, name).", + "pyexpat.error.__getstate__" => "Helper for pickle.", + "pyexpat.error.__gt__" => "Return self>value.", + "pyexpat.error.__hash__" => "Return hash(self).", + "pyexpat.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "pyexpat.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "pyexpat.error.__le__" => "Return self<=value.", + "pyexpat.error.__lt__" => "Return self<value.", + "pyexpat.error.__ne__" => "Return self!=value.", + "pyexpat.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "pyexpat.error.__reduce_ex__" => "Helper for pickle.", + "pyexpat.error.__repr__" => "Return repr(self).", + "pyexpat.error.__setattr__" => "Implement setattr(self, name, value).", + "pyexpat.error.__sizeof__" => "Size of object in memory, in bytes.", + "pyexpat.error.__str__" => "Return str(self).", + "pyexpat.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "pyexpat.error.__weakref__" => "list of weak references to the object", + "pyexpat.error.add_note" => "Add a note to the exception", + "pyexpat.error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "readline" => "Importing this module enables command line editing using GNU readline.", + "readline.add_history" => "Add an item to the history buffer.", + "readline.append_history_file" => "Append the last nelements items of the history list to file.\n\nThe default filename is ~/.history.", + "readline.clear_history" => "Clear the current readline history.", + "readline.get_begidx" => "Get the beginning index of the completion scope.", + "readline.get_completer" => "Get the current completer function.", + "readline.get_completer_delims" => "Get the word delimiters for completion.", + "readline.get_completion_type" => "Get the type of completion being attempted.", + "readline.get_current_history_length" => "Return the current (not the maximum) length of history.", + "readline.get_endidx" => "Get the ending index of the completion scope.", + "readline.get_history_item" => "Return the current contents of history item at one-based index.", + "readline.get_history_length" => "Return the maximum number of lines that will be written to the history file.", + "readline.get_line_buffer" => "Return the current contents of the line buffer.", + "readline.insert_text" => "Insert text into the line buffer at the cursor position.", + "readline.parse_and_bind" => "Execute the init line provided in the string argument.", + "readline.read_history_file" => "Load a readline history file.\n\nThe default filename is ~/.history.", + "readline.read_init_file" => "Execute a readline initialization file.\n\nThe default filename is the last filename used.", + "readline.redisplay" => "Change what's displayed on the screen to reflect contents of the line buffer.", + "readline.remove_history_item" => "Remove history item given by its zero-based position.", + "readline.replace_history_item" => "Replaces history item given by its position with contents of line.\n\npos is zero-based.", + "readline.set_auto_history" => "Enables or disables automatic history.", + "readline.set_completer" => "Set or remove the completer function.\n\nThe function is called as function(text, state),\nfor state in 0, 1, 2, ..., until it returns a non-string.\nIt should return the next possible completion starting with 'text'.", + "readline.set_completer_delims" => "Set the word delimiters for completion.", + "readline.set_completion_display_matches_hook" => "Set or remove the completion display function.\n\nThe function is called as\n function(substitution, [matches], longest_match_length)\nonce each time matches need to be displayed.", + "readline.set_history_length" => "Set the maximal number of lines which will be written to the history file.\n\nA negative length is used to inhibit history truncation.", + "readline.set_pre_input_hook" => "Set or remove the function invoked by the rl_pre_input_hook callback.\n\nThe function is called with no arguments after the first prompt\nhas been printed and just before readline starts reading input\ncharacters.", + "readline.set_startup_hook" => "Set or remove the function invoked by the rl_startup_hook callback.\n\nThe function is called with no arguments just\nbefore readline prints the first prompt.", + "readline.write_history_file" => "Save a readline history file.\n\nThe default filename is ~/.history.", + "resource.struct_rusage" => "struct_rusage: Result from getrusage.\n\nThis object may be accessed either as a tuple of\n (utime,stime,maxrss,ixrss,idrss,isrss,minflt,majflt,\n nswap,inblock,oublock,msgsnd,msgrcv,nsignals,nvcsw,nivcsw)\nor via the attributes ru_utime, ru_stime, ru_maxrss, and so on.", + "resource.struct_rusage.__add__" => "Return self+value.", + "resource.struct_rusage.__class_getitem__" => "See PEP 585", + "resource.struct_rusage.__contains__" => "Return bool(key in self).", + "resource.struct_rusage.__delattr__" => "Implement delattr(self, name).", + "resource.struct_rusage.__eq__" => "Return self==value.", + "resource.struct_rusage.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "resource.struct_rusage.__ge__" => "Return self>=value.", + "resource.struct_rusage.__getattribute__" => "Return getattr(self, name).", + "resource.struct_rusage.__getitem__" => "Return self[key].", + "resource.struct_rusage.__getstate__" => "Helper for pickle.", + "resource.struct_rusage.__gt__" => "Return self>value.", + "resource.struct_rusage.__hash__" => "Return hash(self).", + "resource.struct_rusage.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "resource.struct_rusage.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "resource.struct_rusage.__iter__" => "Implement iter(self).", + "resource.struct_rusage.__le__" => "Return self<=value.", + "resource.struct_rusage.__len__" => "Return len(self).", + "resource.struct_rusage.__lt__" => "Return self<value.", + "resource.struct_rusage.__mul__" => "Return self*value.", + "resource.struct_rusage.__ne__" => "Return self!=value.", + "resource.struct_rusage.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "resource.struct_rusage.__reduce_ex__" => "Helper for pickle.", + "resource.struct_rusage.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "resource.struct_rusage.__repr__" => "Return repr(self).", + "resource.struct_rusage.__rmul__" => "Return value*self.", + "resource.struct_rusage.__setattr__" => "Implement setattr(self, name, value).", + "resource.struct_rusage.__sizeof__" => "Size of object in memory, in bytes.", + "resource.struct_rusage.__str__" => "Return str(self).", + "resource.struct_rusage.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "resource.struct_rusage.count" => "Return number of occurrences of value.", + "resource.struct_rusage.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "resource.struct_rusage.ru_idrss" => "unshared data size", + "resource.struct_rusage.ru_inblock" => "block input operations", + "resource.struct_rusage.ru_isrss" => "unshared stack size", + "resource.struct_rusage.ru_ixrss" => "shared memory size", + "resource.struct_rusage.ru_majflt" => "page faults requiring I/O", + "resource.struct_rusage.ru_maxrss" => "max. resident set size", + "resource.struct_rusage.ru_minflt" => "page faults not requiring I/O", + "resource.struct_rusage.ru_msgrcv" => "IPC messages received", + "resource.struct_rusage.ru_msgsnd" => "IPC messages sent", + "resource.struct_rusage.ru_nivcsw" => "involuntary context switches", + "resource.struct_rusage.ru_nsignals" => "signals received", + "resource.struct_rusage.ru_nswap" => "number of swap outs", + "resource.struct_rusage.ru_nvcsw" => "voluntary context switches", + "resource.struct_rusage.ru_oublock" => "block output operations", + "resource.struct_rusage.ru_stime" => "system time used", + "resource.struct_rusage.ru_utime" => "user time used", + "select" => "This module supports asynchronous I/O on multiple file descriptors.\n\n*** IMPORTANT NOTICE ***\nOn Windows, only sockets are supported; on Unix, all file descriptors.", + "select.epoll" => "select.epoll(sizehint=-1, flags=0)\n\nReturns an epolling object\n\nsizehint must be a positive integer or -1 for the default size. The\nsizehint is used to optimize internal data structures. It doesn't limit\nthe maximum number of monitored events.", + "select.epoll.__delattr__" => "Implement delattr(self, name).", + "select.epoll.__eq__" => "Return self==value.", + "select.epoll.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "select.epoll.__ge__" => "Return self>=value.", + "select.epoll.__getattribute__" => "Return getattr(self, name).", + "select.epoll.__getstate__" => "Helper for pickle.", + "select.epoll.__gt__" => "Return self>value.", + "select.epoll.__hash__" => "Return hash(self).", + "select.epoll.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "select.epoll.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "select.epoll.__le__" => "Return self<=value.", + "select.epoll.__lt__" => "Return self<value.", + "select.epoll.__ne__" => "Return self!=value.", + "select.epoll.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "select.epoll.__reduce__" => "Helper for pickle.", + "select.epoll.__reduce_ex__" => "Helper for pickle.", + "select.epoll.__repr__" => "Return repr(self).", + "select.epoll.__setattr__" => "Implement setattr(self, name, value).", + "select.epoll.__sizeof__" => "Size of object in memory, in bytes.", + "select.epoll.__str__" => "Return str(self).", + "select.epoll.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "select.epoll.close" => "Close the epoll control file descriptor.\n\nFurther operations on the epoll object will raise an exception.", + "select.epoll.closed" => "True if the epoll handler is closed", + "select.epoll.fileno" => "Return the epoll control file descriptor.", + "select.epoll.fromfd" => "Create an epoll object from a given control fd.", + "select.epoll.modify" => "Modify event mask for a registered file descriptor.\n\n fd\n the target file descriptor of the operation\n eventmask\n a bit set composed of the various EPOLL constants", + "select.epoll.poll" => "Wait for events on the epoll file descriptor.\n\n timeout\n the maximum time to wait in seconds (as float);\n a timeout of None or -1 makes poll wait indefinitely\n maxevents\n the maximum number of events returned; -1 means no limit\n\nReturns a list containing any descriptors that have events to report,\nas a list of (fd, events) 2-tuples.", + "select.epoll.register" => "Registers a new fd or raises an OSError if the fd is already registered.\n\n fd\n the target file descriptor of the operation\n eventmask\n a bit set composed of the various EPOLL constants\n\nThe epoll interface supports all file descriptors that support poll.", + "select.epoll.unregister" => "Remove a registered file descriptor from the epoll object.\n\n fd\n the target file descriptor of the operation", + "select.kevent" => "kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0)\n\nThis object is the equivalent of the struct kevent for the C API.\n\nSee the kqueue manpage for more detailed information about the meaning\nof the arguments.\n\nOne minor note: while you might hope that udata could store a\nreference to a python object, it cannot, because it is impossible to\nkeep a proper reference count of the object once it's passed into the\nkernel. Therefore, I have restricted it to only storing an integer. I\nrecommend ignoring it and simply using the 'ident' field to key off\nof. You could also set up a dictionary on the python side to store a\nudata->object mapping.", + "select.kevent.__delattr__" => "Implement delattr(self, name).", + "select.kevent.__eq__" => "Return self==value.", + "select.kevent.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "select.kevent.__ge__" => "Return self>=value.", + "select.kevent.__getattribute__" => "Return getattr(self, name).", + "select.kevent.__getstate__" => "Helper for pickle.", + "select.kevent.__gt__" => "Return self>value.", + "select.kevent.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "select.kevent.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "select.kevent.__le__" => "Return self<=value.", + "select.kevent.__lt__" => "Return self<value.", + "select.kevent.__ne__" => "Return self!=value.", + "select.kevent.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "select.kevent.__reduce__" => "Helper for pickle.", + "select.kevent.__reduce_ex__" => "Helper for pickle.", + "select.kevent.__repr__" => "Return repr(self).", + "select.kevent.__setattr__" => "Implement setattr(self, name, value).", + "select.kevent.__sizeof__" => "Size of object in memory, in bytes.", + "select.kevent.__str__" => "Return str(self).", + "select.kevent.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "select.kqueue" => "Kqueue syscall wrapper.\n\nFor example, to start watching a socket for input:\n>>> kq = kqueue()\n>>> sock = socket()\n>>> sock.connect((host, port))\n>>> kq.control([kevent(sock, KQ_FILTER_WRITE, KQ_EV_ADD)], 0)\n\nTo wait one second for it to become writeable:\n>>> kq.control(None, 1, 1000)\n\nTo stop listening:\n>>> kq.control([kevent(sock, KQ_FILTER_WRITE, KQ_EV_DELETE)], 0)", + "select.kqueue.__del__" => "Called when the instance is about to be destroyed.", + "select.kqueue.__delattr__" => "Implement delattr(self, name).", + "select.kqueue.__eq__" => "Return self==value.", + "select.kqueue.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "select.kqueue.__ge__" => "Return self>=value.", + "select.kqueue.__getattribute__" => "Return getattr(self, name).", + "select.kqueue.__getstate__" => "Helper for pickle.", + "select.kqueue.__gt__" => "Return self>value.", + "select.kqueue.__hash__" => "Return hash(self).", + "select.kqueue.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "select.kqueue.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "select.kqueue.__le__" => "Return self<=value.", + "select.kqueue.__lt__" => "Return self<value.", + "select.kqueue.__ne__" => "Return self!=value.", + "select.kqueue.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "select.kqueue.__reduce__" => "Helper for pickle.", + "select.kqueue.__reduce_ex__" => "Helper for pickle.", + "select.kqueue.__repr__" => "Return repr(self).", + "select.kqueue.__setattr__" => "Implement setattr(self, name, value).", + "select.kqueue.__sizeof__" => "Size of object in memory, in bytes.", + "select.kqueue.__str__" => "Return str(self).", + "select.kqueue.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "select.kqueue.close" => "Close the kqueue control file descriptor.\n\nFurther operations on the kqueue object will raise an exception.", + "select.kqueue.closed" => "True if the kqueue handler is closed", + "select.kqueue.control" => "Calls the kernel kevent function.\n\n changelist\n Must be an iterable of kevent objects describing the changes to be made\n to the kernel's watch list or None.\n maxevents\n The maximum number of events that the kernel will return.\n timeout\n The maximum time to wait in seconds, or else None to wait forever.\n This accepts floats for smaller timeouts, too.", + "select.kqueue.fileno" => "Return the kqueue control file descriptor.", + "select.kqueue.fromfd" => "Create a kqueue object from a given control fd.", + "select.poll" => "Returns a polling object.\n\nThis object supports registering and unregistering file descriptors, and then\npolling them for I/O events.", + "select.select" => "Wait until one or more file descriptors are ready for some kind of I/O.\n\nThe first three arguments are iterables of file descriptors to be waited for:\nrlist -- wait until ready for reading\nwlist -- wait until ready for writing\nxlist -- wait for an \"exceptional condition\"\nIf only one kind of condition is required, pass [] for the other lists.\n\nA file descriptor is either a socket or file object, or a small integer\ngotten from a fileno() method call on one of those.\n\nThe optional 4th argument specifies a timeout in seconds; it may be\na floating-point number to specify fractions of seconds. If it is absent\nor None, the call will never time out.\n\nThe return value is a tuple of three lists corresponding to the first three\narguments; each contains the subset of the corresponding file descriptors\nthat are ready.\n\n*** IMPORTANT NOTICE ***\nOn Windows, only sockets are supported; on Unix, all file\ndescriptors can be used.", + "sys" => "This module provides access to some objects used or maintained by the\ninterpreter and to functions that interact strongly with the interpreter.\n\nDynamic objects:\n\nargv -- command line arguments; argv[0] is the script pathname if known\npath -- module search path; path[0] is the script directory, else ''\nmodules -- dictionary of loaded modules\n\ndisplayhook -- called to show results in an interactive session\nexcepthook -- called to handle any uncaught exception other than SystemExit\n To customize printing in an interactive session or to install a custom\n top-level exception handler, assign other functions to replace these.\n\nstdin -- standard input file object; used by input()\nstdout -- standard output file object; used by print()\nstderr -- standard error object; used for error messages\n By assigning other file objects (or objects that behave like files)\n to these, it is possible to redirect all of the interpreter's I/O.\n\nlast_exc - the last uncaught exception\n Only available in an interactive session after a\n traceback has been printed.\nlast_type -- type of last uncaught exception\nlast_value -- value of last uncaught exception\nlast_traceback -- traceback of last uncaught exception\n These three are the (deprecated) legacy representation of last_exc.\n\nStatic objects:\n\nbuiltin_module_names -- tuple of module names built into this interpreter\ncopyright -- copyright notice pertaining to this interpreter\nexec_prefix -- prefix used to find the machine-specific Python library\nexecutable -- absolute path of the executable binary of the Python interpreter\nfloat_info -- a named tuple with information about the float implementation.\nfloat_repr_style -- string indicating the style of repr() output for floats\nhash_info -- a named tuple with information about the hash algorithm.\nhexversion -- version information encoded as a single integer\nimplementation -- Python implementation information.\nint_info -- a named tuple with information about the int implementation.\nmaxsize -- the largest supported length of containers.\nmaxunicode -- the value of the largest Unicode code point\nplatform -- platform identifier\nprefix -- prefix used to find the Python library\nthread_info -- a named tuple with information about the thread implementation.\nversion -- the version of this interpreter as a string\nversion_info -- version information as a named tuple\ndllhandle -- [Windows only] integer handle of the Python DLL\nwinver -- [Windows only] version number of the Python DLL\n_enablelegacywindowsfsencoding -- [Windows only]\n__stdin__ -- the original stdin; don't touch!\n__stdout__ -- the original stdout; don't touch!\n__stderr__ -- the original stderr; don't touch!\n__displayhook__ -- the original displayhook; don't touch!\n__excepthook__ -- the original excepthook; don't touch!\n\nFunctions:\n\ndisplayhook() -- print an object to the screen, and save it in builtins._\nexcepthook() -- print an exception and its traceback to sys.stderr\nexception() -- return the current thread's active exception\nexc_info() -- return information about the current thread's active exception\nexit() -- exit the interpreter by raising SystemExit\ngetdlopenflags() -- returns flags to be used for dlopen() calls\ngetprofile() -- get the global profiling function\ngetrefcount() -- return the reference count for an object (plus one :-)\ngetrecursionlimit() -- return the max recursion depth for the interpreter\ngetsizeof() -- return the size of an object in bytes\ngettrace() -- get the global debug tracing function\nsetdlopenflags() -- set the flags to be used for dlopen() calls\nsetprofile() -- set the global profiling function\nsetrecursionlimit() -- set the max recursion depth for the interpreter\nsettrace() -- set the global debug tracing function", + "sys.__breakpointhook__" => "This hook function is called by built-in breakpoint().", + "sys.__displayhook__" => "Print an object to sys.stdout and also save it in builtins._", + "sys.__excepthook__" => "Handle an exception by displaying it with a traceback on sys.stderr.", + "sys.__unraisablehook__" => "Handle an unraisable exception.\n\nThe unraisable argument has the following attributes:\n\n* exc_type: Exception type.\n* exc_value: Exception value, can be None.\n* exc_traceback: Exception traceback, can be None.\n* err_msg: Error message, can be None.\n* object: Object causing the exception, can be None.", + "sys._baserepl" => "Private function for getting the base REPL", + "sys._clear_internal_caches" => "Clear all internal performance-related caches.", + "sys._clear_type_cache" => "Clear the internal type lookup cache.", + "sys._clear_type_descriptors" => "Private function for clearing certain descriptors from a type's dictionary.\n\nSee gh-135228 for context.", + "sys._current_exceptions" => "Return a dict mapping each thread's identifier to its current raised exception.\n\nThis function should be used for specialized purposes only.", + "sys._current_frames" => "Return a dict mapping each thread's thread id to its current stack frame.\n\nThis function should be used for specialized purposes only.", + "sys._debugmallocstats" => "Print summary info to stderr about the state of pymalloc's structures.\n\nIn Py_DEBUG mode, also perform some expensive internal consistency\nchecks.", + "sys._dump_tracelets" => "Dump the graph of tracelets in graphviz format", + "sys._enablelegacywindowsfsencoding" => "Changes the default filesystem encoding to mbcs:replace.\n\nThis is done for consistency with earlier versions of Python. See PEP\n529 for more information.\n\nThis is equivalent to defining the PYTHONLEGACYWINDOWSFSENCODING\nenvironment variable before launching Python.", + "sys._get_cpu_count_config" => "Private function for getting PyConfig.cpu_count", + "sys._getframe" => "Return a frame object from the call stack.\n\nIf optional integer depth is given, return the frame object that many\ncalls below the top of the stack. If that is deeper than the call\nstack, ValueError is raised. The default for depth is zero, returning\nthe frame at the top of the call stack.\n\nThis function should be used for internal and specialized purposes\nonly.", + "sys._getframemodulename" => "Return the name of the module for a calling frame.\n\nThe default depth returns the module containing the call to this API.\nA more typical use in a library will pass a depth of 1 to get the user's\nmodule rather than the library module.\n\nIf no frame, module, or name can be found, returns None.", + "sys._is_gil_enabled" => "Return True if the GIL is currently enabled and False otherwise.", + "sys._is_immortal" => "Return True if the given object is \"immortal\" per PEP 683.\n\nThis function should be used for specialized purposes only.", + "sys._is_interned" => "Return True if the given string is \"interned\".", + "sys._setprofileallthreads" => "Set the profiling function in all running threads belonging to the current interpreter.\n\nIt will be called on each function call and return. See the profiler\nchapter in the library manual.", + "sys._settraceallthreads" => "Set the global debug tracing function in all running threads belonging to the current interpreter.\n\nIt will be called on each function call. See the debugger chapter\nin the library manual.", + "sys.activate_stack_trampoline" => "Activate stack profiler trampoline *backend*.", + "sys.addaudithook" => "Adds a new audit hook callback.", + "sys.audit" => "Passes the event to any audit hooks that are attached.", + "sys.breakpointhook" => "This hook function is called by built-in breakpoint().", + "sys.call_tracing" => "Call func(*args), while tracing is enabled.\n\nThe tracing state is saved, and restored afterwards. This is intended\nto be called from a debugger from a checkpoint, to recursively debug\nsome other code.", + "sys.deactivate_stack_trampoline" => "Deactivate the current stack profiler trampoline backend.\n\nIf no stack profiler is activated, this function has no effect.", + "sys.displayhook" => "Print an object to sys.stdout and also save it in builtins._", + "sys.exc_info" => "Return current exception information: (type, value, traceback).\n\nReturn information about the most recent exception caught by an except\nclause in the current stack frame or in an older stack frame.", + "sys.excepthook" => "Handle an exception by displaying it with a traceback on sys.stderr.", + "sys.exception" => "Return the current exception.\n\nReturn the most recent exception caught by an except clause\nin the current stack frame or in an older stack frame, or None\nif no such exception exists.", + "sys.exit" => "Exit the interpreter by raising SystemExit(status).\n\nIf the status is omitted or None, it defaults to zero (i.e., success).\nIf the status is an integer, it will be used as the system exit status.\nIf it is another kind of object, it will be printed and the system\nexit status will be one (i.e., failure).", + "sys.get_asyncgen_hooks" => "Return the installed asynchronous generators hooks.\n\nThis returns a namedtuple of the form (firstiter, finalizer).", + "sys.get_coroutine_origin_tracking_depth" => "Check status of origin tracking for coroutine objects in this thread.", + "sys.get_int_max_str_digits" => "Return the maximum string digits limit for non-binary int<->str conversions.", + "sys.getallocatedblocks" => "Return the number of memory blocks currently allocated.", + "sys.getdefaultencoding" => "Return the current default encoding used by the Unicode implementation.", + "sys.getdlopenflags" => "Return the current value of the flags that are used for dlopen calls.\n\nThe flag constants are defined in the os module.", + "sys.getfilesystemencodeerrors" => "Return the error mode used Unicode to OS filename conversion.", + "sys.getfilesystemencoding" => "Return the encoding used to convert Unicode filenames to OS filenames.", + "sys.getprofile" => "Return the profiling function set with sys.setprofile.\n\nSee the profiler chapter in the library manual.", + "sys.getrecursionlimit" => "Return the current value of the recursion limit.\n\nThe recursion limit is the maximum depth of the Python interpreter\nstack. This limit prevents infinite recursion from causing an overflow\nof the C stack and crashing Python.", + "sys.getrefcount" => "Return the reference count of object.\n\nThe count returned is generally one higher than you might expect,\nbecause it includes the (temporary) reference as an argument to\ngetrefcount().", + "sys.getsizeof" => "getsizeof(object [, default]) -> int\n\nReturn the size of object in bytes.", + "sys.getswitchinterval" => "Return the current thread switch interval; see sys.setswitchinterval().", + "sys.gettrace" => "Return the global debug tracing function set with sys.settrace.\n\nSee the debugger chapter in the library manual.", + "sys.getunicodeinternedsize" => "Return the number of elements of the unicode interned dictionary", + "sys.getwindowsversion" => "Return info about the running version of Windows as a named tuple.\n\nThe members are named: major, minor, build, platform, service_pack,\nservice_pack_major, service_pack_minor, suite_mask, product_type and\nplatform_version. For backward compatibility, only the first 5 items\nare available by indexing. All elements are numbers, except\nservice_pack and platform_type which are strings, and platform_version\nwhich is a 3-tuple. Platform is always 2. Product_type may be 1 for a\nworkstation, 2 for a domain controller, 3 for a server.\nPlatform_version is a 3-tuple containing a version number that is\nintended for identifying the OS rather than feature detection.", + "sys.intern" => "``Intern'' the given string.\n\nThis enters the string in the (global) table of interned strings whose\npurpose is to speed up dictionary lookups. Return the string itself or\nthe previously interned string object with the same value.", + "sys.is_finalizing" => "Return True if Python is exiting.", + "sys.is_remote_debug_enabled" => "Return True if remote debugging is enabled, False otherwise.", + "sys.is_stack_trampoline_active" => "Return *True* if a stack profiler trampoline is active.", + "sys.remote_exec" => "Executes a file containing Python code in a given remote Python process.\n\nThis function returns immediately, and the code will be executed by the\ntarget process's main thread at the next available opportunity, similarly\nto how signals are handled. There is no interface to determine when the\ncode has been executed. The caller is responsible for making sure that\nthe file still exists whenever the remote process tries to read it and that\nit hasn't been overwritten.\n\nThe remote process must be running a CPython interpreter of the same major\nand minor version as the local process. If either the local or remote\ninterpreter is pre-release (alpha, beta, or release candidate) then the\nlocal and remote interpreters must be the same exact version.\n\nArgs:\n pid (int): The process ID of the target Python process.\n script (str|bytes): The path to a file containing\n the Python code to be executed.", + "sys.set_asyncgen_hooks" => "set_asyncgen_hooks([firstiter] [, finalizer])\n\nSet a finalizer for async generators objects.", + "sys.set_coroutine_origin_tracking_depth" => "Enable or disable origin tracking for coroutine objects in this thread.\n\nCoroutine objects will track 'depth' frames of traceback information\nabout where they came from, available in their cr_origin attribute.\n\nSet a depth of 0 to disable.", + "sys.set_int_max_str_digits" => "Set the maximum string digits limit for non-binary int<->str conversions.", + "sys.setdlopenflags" => "Set the flags used by the interpreter for dlopen calls.\n\nThis is used, for example, when the interpreter loads extension\nmodules. Among other things, this will enable a lazy resolving of\nsymbols when importing a module, if called as sys.setdlopenflags(0).\nTo share symbols across extension modules, call as\nsys.setdlopenflags(os.RTLD_GLOBAL). Symbolic names for the flag\nmodules can be found in the os module (RTLD_xxx constants, e.g.\nos.RTLD_LAZY).", + "sys.setprofile" => "Set the profiling function.\n\nIt will be called on each function call and return. See the profiler\nchapter in the library manual.", + "sys.setrecursionlimit" => "Set the maximum depth of the Python interpreter stack to n.\n\nThis limit prevents infinite recursion from causing an overflow of the C\nstack and crashing Python. The highest possible limit is platform-\ndependent.", + "sys.setswitchinterval" => "Set the ideal thread switching delay inside the Python interpreter.\n\nThe actual frequency of switching threads can be lower if the\ninterpreter executes long sequences of uninterruptible code\n(this is implementation-specific and workload-dependent).\n\nThe parameter must represent the desired switching delay in seconds\nA typical value is 0.005 (5 milliseconds).", + "sys.settrace" => "Set the global debug tracing function.\n\nIt will be called on each function call. See the debugger chapter\nin the library manual.", + "sys.unraisablehook" => "Handle an unraisable exception.\n\nThe unraisable argument has the following attributes:\n\n* exc_type: Exception type.\n* exc_value: Exception value, can be None.\n* exc_traceback: Exception traceback, can be None.\n* err_msg: Error message, can be None.\n* object: Object causing the exception, can be None.", + "syslog.LOG_MASK" => "Calculates the mask for the individual priority pri.", + "syslog.LOG_UPTO" => "Calculates the mask for all priorities up to and including pri.", + "syslog.closelog" => "Reset the syslog module values and call the system library closelog().", + "syslog.openlog" => "Set logging options of subsequent syslog() calls.", + "syslog.setlogmask" => "Set the priority mask to maskpri and return the previous mask value.", + "syslog.syslog" => "syslog([priority=LOG_INFO,] message)\nSend the string message to the system logger.", + "termios" => "This module provides an interface to the Posix calls for tty I/O control.\nFor a complete description of these calls, see the Posix or Unix manual\npages. It is only available for those Unix versions that support Posix\ntermios style tty I/O control.\n\nAll functions in this module take a file descriptor fd as their first\nargument. This can be an integer file descriptor, such as returned by\nsys.stdin.fileno(), or a file object, such as sys.stdin itself.", + "termios.error.__delattr__" => "Implement delattr(self, name).", + "termios.error.__eq__" => "Return self==value.", + "termios.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "termios.error.__ge__" => "Return self>=value.", + "termios.error.__getattribute__" => "Return getattr(self, name).", + "termios.error.__getstate__" => "Helper for pickle.", + "termios.error.__gt__" => "Return self>value.", + "termios.error.__hash__" => "Return hash(self).", + "termios.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "termios.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "termios.error.__le__" => "Return self<=value.", + "termios.error.__lt__" => "Return self<value.", + "termios.error.__ne__" => "Return self!=value.", + "termios.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "termios.error.__reduce_ex__" => "Helper for pickle.", + "termios.error.__repr__" => "Return repr(self).", + "termios.error.__setattr__" => "Implement setattr(self, name, value).", + "termios.error.__sizeof__" => "Size of object in memory, in bytes.", + "termios.error.__str__" => "Return str(self).", + "termios.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "termios.error.__weakref__" => "list of weak references to the object", + "termios.error.add_note" => "Add a note to the exception", + "termios.error.with_traceback" => "Set self.__traceback__ to tb and return self.", + "termios.tcdrain" => "Wait until all output written to file descriptor fd has been transmitted.", + "termios.tcflow" => "Suspend or resume input or output on file descriptor fd.\n\nThe action argument can be termios.TCOOFF to suspend output,\ntermios.TCOON to restart output, termios.TCIOFF to suspend input,\nor termios.TCION to restart input.", + "termios.tcflush" => "Discard queued data on file descriptor fd.\n\nThe queue selector specifies which queue: termios.TCIFLUSH for the input\nqueue, termios.TCOFLUSH for the output queue, or termios.TCIOFLUSH for\nboth queues.", + "termios.tcgetattr" => "Get the tty attributes for file descriptor fd.\n\nReturns a list [iflag, oflag, cflag, lflag, ispeed, ospeed, cc]\nwhere cc is a list of the tty special characters (each a string of\nlength 1, except the items with indices VMIN and VTIME, which are\nintegers when these fields are defined). The interpretation of the\nflags and the speeds as well as the indexing in the cc array must be\ndone using the symbolic constants defined in this module.", + "termios.tcgetwinsize" => "Get the tty winsize for file descriptor fd.\n\nReturns a tuple (ws_row, ws_col).", + "termios.tcsendbreak" => "Send a break on file descriptor fd.\n\nA zero duration sends a break for 0.25-0.5 seconds; a nonzero duration\nhas a system dependent meaning.", + "termios.tcsetattr" => "Set the tty attributes for file descriptor fd.\n\nThe attributes to be set are taken from the attributes argument, which\nis a list like the one returned by tcgetattr(). The when argument\ndetermines when the attributes are changed: termios.TCSANOW to\nchange immediately, termios.TCSADRAIN to change after transmitting all\nqueued output, or termios.TCSAFLUSH to change after transmitting all\nqueued output and discarding all queued input.", + "termios.tcsetwinsize" => "Set the tty winsize for file descriptor fd.\n\nThe winsize to be set is taken from the winsize argument, which\nis a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize().", + "time" => "This module provides various functions to manipulate time values.\n\nThere are two standard representations of time. One is the number\nof seconds since the Epoch, in UTC (a.k.a. GMT). It may be an integer\nor a floating-point number (to represent fractions of seconds).\nThe epoch is the point where the time starts, the return value of time.gmtime(0).\nIt is January 1, 1970, 00:00:00 (UTC) on all platforms.\n\nThe other representation is a tuple of 9 integers giving local time.\nThe tuple items are:\n year (including century, e.g. 1998)\n month (1-12)\n day (1-31)\n hours (0-23)\n minutes (0-59)\n seconds (0-59)\n weekday (0-6, Monday is 0)\n Julian day (day in the year, 1-366)\n DST (Daylight Savings Time) flag (-1, 0 or 1)\nIf the DST flag is 0, the time is given in the regular time zone;\nif it is 1, the time is given in the DST time zone;\nif it is -1, mktime() should guess based on the date and time.", + "time.asctime" => "asctime([tuple]) -> string\n\nConvert a time tuple to a string, e.g. 'Sat Jun 06 16:26:11 1998'.\nWhen the time tuple is not present, current time as returned by localtime()\nis used.", + "time.clock_getres" => "clock_getres(clk_id) -> floating-point number\n\nReturn the resolution (precision) of the specified clock clk_id.", + "time.clock_gettime" => "Return the time of the specified clock clk_id as a float.", + "time.clock_gettime_ns" => "Return the time of the specified clock clk_id as nanoseconds (int).", + "time.clock_settime" => "clock_settime(clk_id, time)\n\nSet the time of the specified clock clk_id.", + "time.clock_settime_ns" => "clock_settime_ns(clk_id, time)\n\nSet the time of the specified clock clk_id with nanoseconds.", + "time.ctime" => "ctime(seconds) -> string\n\nConvert a time in seconds since the Epoch to a string in local time.\nThis is equivalent to asctime(localtime(seconds)). When the time tuple is\nnot present, current time as returned by localtime() is used.", + "time.get_clock_info" => "get_clock_info(name: str) -> dict\n\nGet information of the specified clock.", + "time.gmtime" => "gmtime([seconds]) -> (tm_year, tm_mon, tm_mday, tm_hour, tm_min,\n tm_sec, tm_wday, tm_yday, tm_isdst)\n\nConvert seconds since the Epoch to a time tuple expressing UTC (a.k.a.\nGMT). When 'seconds' is not passed in, convert the current time instead.\n\nIf the platform supports the tm_gmtoff and tm_zone, they are available as\nattributes only.", + "time.localtime" => "localtime([seconds]) -> (tm_year,tm_mon,tm_mday,tm_hour,tm_min,\n tm_sec,tm_wday,tm_yday,tm_isdst)\n\nConvert seconds since the Epoch to a time tuple expressing local time.\nWhen 'seconds' is not passed in, convert the current time instead.", + "time.mktime" => "mktime(tuple) -> floating-point number\n\nConvert a time tuple in local time to seconds since the Epoch.\nNote that mktime(gmtime(0)) will not generally return zero for most\ntime zones; instead the returned value will either be equal to that\nof the timezone or altzone attributes on the time module.", + "time.monotonic" => "monotonic() -> float\n\nMonotonic clock, cannot go backward.", + "time.monotonic_ns" => "monotonic_ns() -> int\n\nMonotonic clock, cannot go backward, as nanoseconds.", + "time.perf_counter" => "perf_counter() -> float\n\nPerformance counter for benchmarking.", + "time.perf_counter_ns" => "perf_counter_ns() -> int\n\nPerformance counter for benchmarking as nanoseconds.", + "time.process_time" => "process_time() -> float\n\nProcess time for profiling: sum of the kernel and user-space CPU time.", + "time.process_time_ns" => "process_time() -> int\n\nProcess time for profiling as nanoseconds:\nsum of the kernel and user-space CPU time.", + "time.pthread_getcpuclockid" => "pthread_getcpuclockid(thread_id) -> int\n\nReturn the clk_id of a thread's CPU time clock.", + "time.sleep" => "sleep(seconds)\n\nDelay execution for a given number of seconds. The argument may be\na floating-point number for subsecond precision.", + "time.strftime" => "strftime(format[, tuple]) -> string\n\nConvert a time tuple to a string according to a format specification.\nSee the library reference manual for formatting codes. When the time tuple\nis not present, current time as returned by localtime() is used.\n\nCommonly used format codes:\n\n%Y Year with century as a decimal number.\n%m Month as a decimal number [01,12].\n%d Day of the month as a decimal number [01,31].\n%H Hour (24-hour clock) as a decimal number [00,23].\n%M Minute as a decimal number [00,59].\n%S Second as a decimal number [00,61].\n%z Time zone offset from UTC.\n%a Locale's abbreviated weekday name.\n%A Locale's full weekday name.\n%b Locale's abbreviated month name.\n%B Locale's full month name.\n%c Locale's appropriate date and time representation.\n%I Hour (12-hour clock) as a decimal number [01,12].\n%p Locale's equivalent of either AM or PM.\n\nOther codes may be available on your platform. See documentation for\nthe C library strftime function.", + "time.strptime" => "strptime(string, format) -> struct_time\n\nParse a string to a time tuple according to a format specification.\nSee the library reference manual for formatting codes (same as\nstrftime()).\n\nCommonly used format codes:\n\n%Y Year with century as a decimal number.\n%m Month as a decimal number [01,12].\n%d Day of the month as a decimal number [01,31].\n%H Hour (24-hour clock) as a decimal number [00,23].\n%M Minute as a decimal number [00,59].\n%S Second as a decimal number [00,61].\n%z Time zone offset from UTC.\n%a Locale's abbreviated weekday name.\n%A Locale's full weekday name.\n%b Locale's abbreviated month name.\n%B Locale's full month name.\n%c Locale's appropriate date and time representation.\n%I Hour (12-hour clock) as a decimal number [01,12].\n%p Locale's equivalent of either AM or PM.\n\nOther codes may be available on your platform. See documentation for\nthe C library strftime function.", + "time.struct_time" => "The time value as returned by gmtime(), localtime(), and strptime(), and\n accepted by asctime(), mktime() and strftime(). May be considered as a\n sequence of 9 integers.\n\n Note that several fields' values are not the same as those defined by\n the C language standard for struct tm. For example, the value of the\n field tm_year is the actual year, not year - 1900. See individual\n fields' descriptions for details.", + "time.struct_time.__add__" => "Return self+value.", + "time.struct_time.__class_getitem__" => "See PEP 585", + "time.struct_time.__contains__" => "Return bool(key in self).", + "time.struct_time.__delattr__" => "Implement delattr(self, name).", + "time.struct_time.__eq__" => "Return self==value.", + "time.struct_time.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "time.struct_time.__ge__" => "Return self>=value.", + "time.struct_time.__getattribute__" => "Return getattr(self, name).", + "time.struct_time.__getitem__" => "Return self[key].", + "time.struct_time.__getstate__" => "Helper for pickle.", + "time.struct_time.__gt__" => "Return self>value.", + "time.struct_time.__hash__" => "Return hash(self).", + "time.struct_time.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "time.struct_time.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "time.struct_time.__iter__" => "Implement iter(self).", + "time.struct_time.__le__" => "Return self<=value.", + "time.struct_time.__len__" => "Return len(self).", + "time.struct_time.__lt__" => "Return self<value.", + "time.struct_time.__mul__" => "Return self*value.", + "time.struct_time.__ne__" => "Return self!=value.", + "time.struct_time.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "time.struct_time.__reduce_ex__" => "Helper for pickle.", + "time.struct_time.__replace__" => "Return a copy of the structure with new values for the specified fields.", + "time.struct_time.__repr__" => "Return repr(self).", + "time.struct_time.__rmul__" => "Return value*self.", + "time.struct_time.__setattr__" => "Implement setattr(self, name, value).", + "time.struct_time.__sizeof__" => "Size of object in memory, in bytes.", + "time.struct_time.__str__" => "Return str(self).", + "time.struct_time.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "time.struct_time.count" => "Return number of occurrences of value.", + "time.struct_time.index" => "Return first index of value.\n\nRaises ValueError if the value is not present.", + "time.struct_time.tm_gmtoff" => "offset from UTC in seconds", + "time.struct_time.tm_hour" => "hours, range [0, 23]", + "time.struct_time.tm_isdst" => "1 if summer time is in effect, 0 if not, and -1 if unknown", + "time.struct_time.tm_mday" => "day of month, range [1, 31]", + "time.struct_time.tm_min" => "minutes, range [0, 59]", + "time.struct_time.tm_mon" => "month of year, range [1, 12]", + "time.struct_time.tm_sec" => "seconds, range [0, 61])", + "time.struct_time.tm_wday" => "day of week, range [0, 6], Monday is 0", + "time.struct_time.tm_yday" => "day of year, range [1, 366]", + "time.struct_time.tm_year" => "year, for example, 1993", + "time.struct_time.tm_zone" => "abbreviation of timezone name", + "time.thread_time" => "thread_time() -> float\n\nThread time for profiling: sum of the kernel and user-space CPU time.", + "time.thread_time_ns" => "thread_time() -> int\n\nThread time for profiling as nanoseconds:\nsum of the kernel and user-space CPU time.", + "time.time" => "time() -> floating-point number\n\nReturn the current time in seconds since the Epoch.\nFractions of a second may be present if the system clock provides them.", + "time.time_ns" => "time_ns() -> int\n\nReturn the current time in nanoseconds since the Epoch.", + "time.tzset" => "tzset()\n\nInitialize, or reinitialize, the local timezone to the value stored in\nos.environ['TZ']. The TZ environment variable should be specified in\nstandard Unix timezone format as documented in the tzset man page\n(eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently\nfall back to UTC. If the TZ environment variable is not set, the local\ntimezone is set to the systems best guess of wallclock time.\nChanging the TZ environment variable without calling tzset *may* change\nthe local timezone used by methods such as localtime, but this behaviour\nshould not be relied on.", + "unicodedata" => "This module provides access to the Unicode Character Database which\ndefines character properties for all Unicode characters. The data in\nthis database is based on the UnicodeData.txt file version\n16.0.0 which is publicly available from ftp://ftp.unicode.org/.\n\nThe module uses the same names and symbols as defined by the\nUnicodeData File Format 16.0.0.", + "unicodedata.UCD.__delattr__" => "Implement delattr(self, name).", + "unicodedata.UCD.__eq__" => "Return self==value.", + "unicodedata.UCD.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "unicodedata.UCD.__ge__" => "Return self>=value.", + "unicodedata.UCD.__getattribute__" => "Return getattr(self, name).", + "unicodedata.UCD.__getstate__" => "Helper for pickle.", + "unicodedata.UCD.__gt__" => "Return self>value.", + "unicodedata.UCD.__hash__" => "Return hash(self).", + "unicodedata.UCD.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "unicodedata.UCD.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "unicodedata.UCD.__le__" => "Return self<=value.", + "unicodedata.UCD.__lt__" => "Return self<value.", + "unicodedata.UCD.__ne__" => "Return self!=value.", + "unicodedata.UCD.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "unicodedata.UCD.__reduce__" => "Helper for pickle.", + "unicodedata.UCD.__reduce_ex__" => "Helper for pickle.", + "unicodedata.UCD.__repr__" => "Return repr(self).", + "unicodedata.UCD.__setattr__" => "Implement setattr(self, name, value).", + "unicodedata.UCD.__sizeof__" => "Size of object in memory, in bytes.", + "unicodedata.UCD.__str__" => "Return str(self).", + "unicodedata.UCD.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "unicodedata.UCD.bidirectional" => "Returns the bidirectional class assigned to the character chr as string.\n\nIf no such value is defined, an empty string is returned.", + "unicodedata.UCD.category" => "Returns the general category assigned to the character chr as string.", + "unicodedata.UCD.combining" => "Returns the canonical combining class assigned to the character chr as integer.\n\nReturns 0 if no combining class is defined.", + "unicodedata.UCD.decimal" => "Converts a Unicode character into its equivalent decimal value.\n\nReturns the decimal value assigned to the character chr as integer.\nIf no such value is defined, default is returned, or, if not given,\nValueError is raised.", + "unicodedata.UCD.decomposition" => "Returns the character decomposition mapping assigned to the character chr as string.\n\nAn empty string is returned in case no such mapping is defined.", + "unicodedata.UCD.digit" => "Converts a Unicode character into its equivalent digit value.\n\nReturns the digit value assigned to the character chr as integer.\nIf no such value is defined, default is returned, or, if not given,\nValueError is raised.", + "unicodedata.UCD.east_asian_width" => "Returns the east asian width assigned to the character chr as string.", + "unicodedata.UCD.is_normalized" => "Return whether the Unicode string unistr is in the normal form 'form'.\n\nValid values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + "unicodedata.UCD.lookup" => "Look up character by name.\n\nIf a character with the given name is found, return the\ncorresponding character. If not found, KeyError is raised.", + "unicodedata.UCD.mirrored" => "Returns the mirrored property assigned to the character chr as integer.\n\nReturns 1 if the character has been identified as a \"mirrored\"\ncharacter in bidirectional text, 0 otherwise.", + "unicodedata.UCD.name" => "Returns the name assigned to the character chr as a string.\n\nIf no name is defined, default is returned, or, if not given,\nValueError is raised.", + "unicodedata.UCD.normalize" => "Return the normal form 'form' for the Unicode string unistr.\n\nValid values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + "unicodedata.UCD.numeric" => "Converts a Unicode character into its equivalent numeric value.\n\nReturns the numeric value assigned to the character chr as float.\nIf no such value is defined, default is returned, or, if not given,\nValueError is raised.", + "unicodedata.bidirectional" => "Returns the bidirectional class assigned to the character chr as string.\n\nIf no such value is defined, an empty string is returned.", + "unicodedata.category" => "Returns the general category assigned to the character chr as string.", + "unicodedata.combining" => "Returns the canonical combining class assigned to the character chr as integer.\n\nReturns 0 if no combining class is defined.", + "unicodedata.decimal" => "Converts a Unicode character into its equivalent decimal value.\n\nReturns the decimal value assigned to the character chr as integer.\nIf no such value is defined, default is returned, or, if not given,\nValueError is raised.", + "unicodedata.decomposition" => "Returns the character decomposition mapping assigned to the character chr as string.\n\nAn empty string is returned in case no such mapping is defined.", + "unicodedata.digit" => "Converts a Unicode character into its equivalent digit value.\n\nReturns the digit value assigned to the character chr as integer.\nIf no such value is defined, default is returned, or, if not given,\nValueError is raised.", + "unicodedata.east_asian_width" => "Returns the east asian width assigned to the character chr as string.", + "unicodedata.is_normalized" => "Return whether the Unicode string unistr is in the normal form 'form'.\n\nValid values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + "unicodedata.lookup" => "Look up character by name.\n\nIf a character with the given name is found, return the\ncorresponding character. If not found, KeyError is raised.", + "unicodedata.mirrored" => "Returns the mirrored property assigned to the character chr as integer.\n\nReturns 1 if the character has been identified as a \"mirrored\"\ncharacter in bidirectional text, 0 otherwise.", + "unicodedata.name" => "Returns the name assigned to the character chr as a string.\n\nIf no name is defined, default is returned, or, if not given,\nValueError is raised.", + "unicodedata.normalize" => "Return the normal form 'form' for the Unicode string unistr.\n\nValid values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'.", + "unicodedata.numeric" => "Converts a Unicode character into its equivalent numeric value.\n\nReturns the numeric value assigned to the character chr as float.\nIf no such value is defined, default is returned, or, if not given,\nValueError is raised.", + "winreg" => "This module provides access to the Windows registry API.\n\nFunctions:\n\nCloseKey() - Closes a registry key.\nConnectRegistry() - Establishes a connection to a predefined registry handle\n on another computer.\nCreateKey() - Creates the specified key, or opens it if it already exists.\nDeleteKey() - Deletes the specified key.\nDeleteValue() - Removes a named value from the specified registry key.\nDeleteTree() - Deletes the specified key and all its subkeys and values recursively.\nEnumKey() - Enumerates subkeys of the specified open registry key.\nEnumValue() - Enumerates values of the specified open registry key.\nExpandEnvironmentStrings() - Expand the env strings in a REG_EXPAND_SZ\n string.\nFlushKey() - Writes all the attributes of the specified key to the registry.\nLoadKey() - Creates a subkey under HKEY_USER or HKEY_LOCAL_MACHINE and\n stores registration information from a specified file into that\n subkey.\nOpenKey() - Opens the specified key.\nOpenKeyEx() - Alias of OpenKey().\nQueryValue() - Retrieves the value associated with the unnamed value for a\n specified key in the registry.\nQueryValueEx() - Retrieves the type and data for a specified value name\n associated with an open registry key.\nQueryInfoKey() - Returns information about the specified key.\nSaveKey() - Saves the specified key, and all its subkeys a file.\nSetValue() - Associates a value with a specified key.\nSetValueEx() - Stores data in the value field of an open registry key.\n\nSpecial objects:\n\nHKEYType -- type object for HKEY objects\nerror -- exception raised for Win32 errors\n\nInteger constants:\nMany constants are defined - see the documentation for each function\nto see what constants are used, and where.", + "winreg.CloseKey" => "Closes a previously opened registry key.\n\n hkey\n A previously opened key.\n\nNote that if the key is not closed using this method, it will be\nclosed when the hkey object is destroyed by Python.", + "winreg.ConnectRegistry" => "Establishes a connection to the registry on another computer.\n\n computer_name\n The name of the remote computer, of the form r\"\\\\computername\". If\n None, the local computer is used.\n key\n The predefined key to connect to.\n\nThe return value is the handle of the opened key.\nIf the function fails, an OSError exception is raised.", + "winreg.CreateKey" => "Creates or opens the specified key.\n\n key\n An already open key, or one of the predefined HKEY_* constants.\n sub_key\n The name of the key this method opens or creates.\n\nIf key is one of the predefined keys, sub_key may be None. In that case,\nthe handle returned is the same key handle passed in to the function.\n\nIf the key already exists, this function opens the existing key.\n\nThe return value is the handle of the opened key.\nIf the function fails, an OSError exception is raised.", + "winreg.CreateKeyEx" => "Creates or opens the specified key.\n\n key\n An already open key, or one of the predefined HKEY_* constants.\n sub_key\n The name of the key this method opens or creates.\n reserved\n A reserved integer, and must be zero. Default is zero.\n access\n An integer that specifies an access mask that describes the\n desired security access for the key. Default is KEY_WRITE.\n\nIf key is one of the predefined keys, sub_key may be None. In that case,\nthe handle returned is the same key handle passed in to the function.\n\nIf the key already exists, this function opens the existing key\n\nThe return value is the handle of the opened key.\nIf the function fails, an OSError exception is raised.", + "winreg.DeleteKey" => "Deletes the specified key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that must be the name of a subkey of the key identified by\n the key parameter. This value must not be None, and the key may not\n have subkeys.\n\nThis method can not delete keys with subkeys.\n\nIf the function succeeds, the entire key, including all of its values,\nis removed. If the function fails, an OSError exception is raised.", + "winreg.DeleteKeyEx" => "Deletes the specified key (intended for 64-bit OS).\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that must be the name of a subkey of the key identified by\n the key parameter. This value must not be None, and the key may not\n have subkeys.\n access\n An integer that specifies an access mask that describes the\n desired security access for the key. Default is KEY_WOW64_64KEY.\n reserved\n A reserved integer, and must be zero. Default is zero.\n\nWhile this function is intended to be used for 64-bit OS, it is also\n available on 32-bit systems.\n\nThis method can not delete keys with subkeys.\n\nIf the function succeeds, the entire key, including all of its values,\nis removed. If the function fails, an OSError exception is raised.\nOn unsupported Windows versions, NotImplementedError is raised.", + "winreg.DeleteValue" => "Removes a named value from a registry key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n value\n A string that identifies the value to remove.", + "winreg.DisableReflectionKey" => "Disables registry reflection for 32bit processes running on a 64bit OS.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n\nWill generally raise NotImplementedError if executed on a 32bit OS.\n\nIf the key is not on the reflection list, the function succeeds but has\nno effect. Disabling reflection for a key does not affect reflection\nof any subkeys.", + "winreg.EnableReflectionKey" => "Restores registry reflection for the specified disabled key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n\nWill generally raise NotImplementedError if executed on a 32bit OS.\nRestoring reflection for a key does not affect reflection of any\nsubkeys.", + "winreg.EnumKey" => "Enumerates subkeys of an open registry key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n index\n An integer that identifies the index of the key to retrieve.\n\nThe function retrieves the name of one subkey each time it is called.\nIt is typically called repeatedly until an OSError exception is\nraised, indicating no more values are available.", + "winreg.EnumValue" => "Enumerates values of an open registry key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n index\n An integer that identifies the index of the value to retrieve.\n\nThe function retrieves the name of one subkey each time it is called.\nIt is typically called repeatedly, until an OSError exception\nis raised, indicating no more values.\n\nThe result is a tuple of 3 items:\n value_name\n A string that identifies the value.\n value_data\n An object that holds the value data, and whose type depends\n on the underlying registry type.\n data_type\n An integer that identifies the type of the value data.", + "winreg.ExpandEnvironmentStrings" => "Expand environment vars.", + "winreg.FlushKey" => "Writes all the attributes of a key to the registry.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n\nIt is not necessary to call FlushKey to change a key. Registry changes\nare flushed to disk by the registry using its lazy flusher. Registry\nchanges are also flushed to disk at system shutdown. Unlike\nCloseKey(), the FlushKey() method returns only when all the data has\nbeen written to the registry.\n\nAn application should only call FlushKey() if it requires absolute\ncertainty that registry changes are on disk. If you don't know whether\na FlushKey() call is required, it probably isn't.", + "winreg.HKEYType" => "PyHKEY Object - A Python object, representing a win32 registry key.\n\nThis object wraps a Windows HKEY object, automatically closing it when\nthe object is destroyed. To guarantee cleanup, you can call either\nthe Close() method on the PyHKEY, or the CloseKey() method.\n\nAll functions which accept a handle object also accept an integer --\nhowever, use of the handle object is encouraged.\n\nFunctions:\nClose() - Closes the underlying handle.\nDetach() - Returns the integer Win32 handle, detaching it from the object\n\nProperties:\nhandle - The integer Win32 handle.\n\nOperations:\n__bool__ - Handles with an open object return true, otherwise false.\n__int__ - Converting a handle to an integer returns the Win32 handle.\n__enter__, __exit__ - Context manager support for 'with' statement,\nautomatically closes handle.", + "winreg.HKEYType.Close" => "Closes the underlying Windows handle.\n\nIf the handle is already closed, no error is raised.", + "winreg.HKEYType.Detach" => "Detaches the Windows handle from the handle object.\n\nThe result is the value of the handle before it is detached. If the\nhandle is already detached, this will return zero.\n\nAfter calling this function, the handle is effectively invalidated,\nbut the handle is not closed. You would call this function when you\nneed the underlying win32 handle to exist beyond the lifetime of the\nhandle object.", + "winreg.HKEYType.__abs__" => "abs(self)", + "winreg.HKEYType.__add__" => "Return self+value.", + "winreg.HKEYType.__and__" => "Return self&value.", + "winreg.HKEYType.__bool__" => "True if self else False", + "winreg.HKEYType.__delattr__" => "Implement delattr(self, name).", + "winreg.HKEYType.__divmod__" => "Return divmod(self, value).", + "winreg.HKEYType.__eq__" => "Return self==value.", + "winreg.HKEYType.__float__" => "float(self)", + "winreg.HKEYType.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "winreg.HKEYType.__ge__" => "Return self>=value.", + "winreg.HKEYType.__getattribute__" => "Return getattr(self, name).", + "winreg.HKEYType.__getstate__" => "Helper for pickle.", + "winreg.HKEYType.__gt__" => "Return self>value.", + "winreg.HKEYType.__hash__" => "Return hash(self).", + "winreg.HKEYType.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "winreg.HKEYType.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "winreg.HKEYType.__int__" => "int(self)", + "winreg.HKEYType.__invert__" => "~self", + "winreg.HKEYType.__le__" => "Return self<=value.", + "winreg.HKEYType.__lshift__" => "Return self<<value.", + "winreg.HKEYType.__lt__" => "Return self<value.", + "winreg.HKEYType.__mod__" => "Return self%value.", + "winreg.HKEYType.__mul__" => "Return self*value.", + "winreg.HKEYType.__ne__" => "Return self!=value.", + "winreg.HKEYType.__neg__" => "-self", + "winreg.HKEYType.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "winreg.HKEYType.__or__" => "Return self|value.", + "winreg.HKEYType.__pos__" => "+self", + "winreg.HKEYType.__pow__" => "Return pow(self, value, mod).", + "winreg.HKEYType.__radd__" => "Return value+self.", + "winreg.HKEYType.__rand__" => "Return value&self.", + "winreg.HKEYType.__rdivmod__" => "Return divmod(value, self).", + "winreg.HKEYType.__reduce__" => "Helper for pickle.", + "winreg.HKEYType.__reduce_ex__" => "Helper for pickle.", + "winreg.HKEYType.__repr__" => "Return repr(self).", + "winreg.HKEYType.__rlshift__" => "Return value<<self.", + "winreg.HKEYType.__rmod__" => "Return value%self.", + "winreg.HKEYType.__rmul__" => "Return value*self.", + "winreg.HKEYType.__ror__" => "Return value|self.", + "winreg.HKEYType.__rpow__" => "Return pow(value, self, mod).", + "winreg.HKEYType.__rrshift__" => "Return value>>self.", + "winreg.HKEYType.__rshift__" => "Return self>>value.", + "winreg.HKEYType.__rsub__" => "Return value-self.", + "winreg.HKEYType.__rxor__" => "Return value^self.", + "winreg.HKEYType.__setattr__" => "Implement setattr(self, name, value).", + "winreg.HKEYType.__sizeof__" => "Size of object in memory, in bytes.", + "winreg.HKEYType.__str__" => "Return str(self).", + "winreg.HKEYType.__sub__" => "Return self-value.", + "winreg.HKEYType.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "winreg.HKEYType.__xor__" => "Return self^value.", + "winreg.LoadKey" => "Insert data into the registry from a file.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that identifies the sub-key to load.\n file_name\n The name of the file to load registry data from. This file must\n have been created with the SaveKey() function. Under the file\n allocation table (FAT) file system, the filename may not have an\n extension.\n\nCreates a subkey under the specified key and stores registration\ninformation from a specified file into that subkey.\n\nA call to LoadKey() fails if the calling process does not have the\nSE_RESTORE_PRIVILEGE privilege.\n\nIf key is a handle returned by ConnectRegistry(), then the path\nspecified in fileName is relative to the remote computer.\n\nThe MSDN docs imply key must be in the HKEY_USER or HKEY_LOCAL_MACHINE\ntree.", + "winreg.OpenKey" => "Opens the specified key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that identifies the sub_key to open.\n reserved\n A reserved integer that must be zero. Default is zero.\n access\n An integer that specifies an access mask that describes the desired\n security access for the key. Default is KEY_READ.\n\nThe result is a new handle to the specified key.\nIf the function fails, an OSError exception is raised.", + "winreg.OpenKeyEx" => "Opens the specified key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that identifies the sub_key to open.\n reserved\n A reserved integer that must be zero. Default is zero.\n access\n An integer that specifies an access mask that describes the desired\n security access for the key. Default is KEY_READ.\n\nThe result is a new handle to the specified key.\nIf the function fails, an OSError exception is raised.", + "winreg.QueryInfoKey" => "Returns information about a key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n\nThe result is a tuple of 3 items:\nAn integer that identifies the number of sub keys this key has.\nAn integer that identifies the number of values this key has.\nAn integer that identifies when the key was last modified (if available)\nas 100's of nanoseconds since Jan 1, 1600.", + "winreg.QueryReflectionKey" => "Returns the reflection state for the specified key as a bool.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n\nWill generally raise NotImplementedError if executed on a 32bit OS.", + "winreg.QueryValue" => "Retrieves the unnamed value for a key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that holds the name of the subkey with which the value\n is associated. If this parameter is None or empty, the function\n retrieves the value set by the SetValue() method for the key\n identified by key.\n\nValues in the registry have name, type, and data components. This method\nretrieves the data for a key's first value that has a NULL name.\nBut since the underlying API call doesn't return the type, you'll\nprobably be happier using QueryValueEx; this function is just here for\ncompleteness.", + "winreg.QueryValueEx" => "Retrieves the type and value of a specified sub-key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n name\n A string indicating the value to query.\n\nBehaves mostly like QueryValue(), but also returns the type of the\nspecified value name associated with the given open registry key.\n\nThe return value is a tuple of the value and the type_id.", + "winreg.SaveKey" => "Saves the specified key, and all its subkeys to the specified file.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n file_name\n The name of the file to save registry data to. This file cannot\n already exist. If this filename includes an extension, it cannot be\n used on file allocation table (FAT) file systems by the LoadKey(),\n ReplaceKey() or RestoreKey() methods.\n\nIf key represents a key on a remote computer, the path described by\nfile_name is relative to the remote computer.\n\nThe caller of this method must possess the SeBackupPrivilege\nsecurity privilege. This function passes NULL for security_attributes\nto the API.", + "winreg.SetValue" => "Associates a value with a specified key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n sub_key\n A string that names the subkey with which the value is associated.\n type\n An integer that specifies the type of the data. Currently this must\n be REG_SZ, meaning only strings are supported.\n value\n A string that specifies the new value.\n\nIf the key specified by the sub_key parameter does not exist, the\nSetValue function creates it.\n\nValue lengths are limited by available memory. Long values (more than\n2048 bytes) should be stored as files with the filenames stored in\nthe configuration registry to help the registry perform efficiently.\n\nThe key identified by the key parameter must have been opened with\nKEY_SET_VALUE access.", + "winreg.SetValueEx" => "Stores data in the value field of an open registry key.\n\n key\n An already open key, or any one of the predefined HKEY_* constants.\n value_name\n A string containing the name of the value to set, or None.\n reserved\n Can be anything - zero is always passed to the API.\n type\n An integer that specifies the type of the data, one of:\n REG_BINARY -- Binary data in any form.\n REG_DWORD -- A 32-bit number.\n REG_DWORD_LITTLE_ENDIAN -- A 32-bit number in little-endian format. Equivalent to REG_DWORD\n REG_DWORD_BIG_ENDIAN -- A 32-bit number in big-endian format.\n REG_EXPAND_SZ -- A null-terminated string that contains unexpanded\n references to environment variables (for example,\n %PATH%).\n REG_LINK -- A Unicode symbolic link.\n REG_MULTI_SZ -- A sequence of null-terminated strings, terminated\n by two null characters. Note that Python handles\n this termination automatically.\n REG_NONE -- No defined value type.\n REG_QWORD -- A 64-bit number.\n REG_QWORD_LITTLE_ENDIAN -- A 64-bit number in little-endian format. Equivalent to REG_QWORD.\n REG_RESOURCE_LIST -- A device-driver resource list.\n REG_SZ -- A null-terminated string.\n value\n A string that specifies the new value.\n\nThis method can also set additional value and type information for the\nspecified key. The key identified by the key parameter must have been\nopened with KEY_SET_VALUE access.\n\nTo open the key, use the CreateKeyEx() or OpenKeyEx() methods.\n\nValue lengths are limited by available memory. Long values (more than\n2048 bytes) should be stored as files with the filenames stored in\nthe configuration registry to help the registry perform efficiently.", + "winsound" => "PlaySound(sound, flags) - play a sound\nSND_FILENAME - sound is a wav file name\nSND_ALIAS - sound is a registry sound association name\nSND_LOOP - Play the sound repeatedly; must also specify SND_ASYNC\nSND_MEMORY - sound is a memory image of a wav file\nSND_PURGE - stop all instances of the specified sound\nSND_ASYNC - PlaySound returns immediately\nSND_NODEFAULT - Do not play a default beep if the sound can not be found\nSND_NOSTOP - Do not interrupt any sounds currently playing\nSND_NOWAIT - Return immediately if the sound driver is busy\nSND_APPLICATION - sound is an application-specific alias in the registry.\nSND_SENTRY - Triggers a SoundSentry event when the sound is played.\nSND_SYNC - Play the sound synchronously, default behavior.\nSND_SYSTEM - Assign sound to the audio session for system notification sounds.\n\nBeep(frequency, duration) - Make a beep through the PC speaker.\nMessageBeep(type) - Call Windows MessageBeep.", + "winsound.Beep" => "A wrapper around the Windows Beep API.\n\n frequency\n Frequency of the sound in hertz.\n Must be in the range 37 through 32,767.\n duration\n How long the sound should play, in milliseconds.", + "winsound.MessageBeep" => "Call Windows MessageBeep(x).\n\nx defaults to MB_OK.", + "winsound.PlaySound" => "A wrapper around the Windows PlaySound API.\n\n sound\n The sound to play; a filename, data, or None.\n flags\n Flag values, ored together. See module documentation.", + "zlib" => "The functions in this module allow compression and decompression using the\nzlib library, which is based on GNU zip.\n\nadler32(string[, start]) -- Compute an Adler-32 checksum.\ncompress(data[, level]) -- Compress data, with compression level 0-9 or -1.\ncompressobj([level[, ...]]) -- Return a compressor object.\ncrc32(string[, start]) -- Compute a CRC-32 checksum.\ndecompress(string,[wbits],[bufsize]) -- Decompresses a compressed string.\ndecompressobj([wbits[, zdict]]) -- Return a decompressor object.\n\n'wbits' is window buffer size and container format.\nCompressor objects support compress() and flush() methods; decompressor\nobjects support decompress() and flush().", + "zlib._ZlibDecompressor" => "Create a decompressor object for decompressing data incrementally.\n\n wbits = 15\n zdict\n The predefined compression dictionary. This is a sequence of bytes\n (such as a bytes object) containing subsequences that are expected\n to occur frequently in the data that is to be compressed. Those\n subsequences that are expected to be most common should come at the\n end of the dictionary. This must be the same dictionary as used by the\n compressor that produced the input data.", + "zlib._ZlibDecompressor.__delattr__" => "Implement delattr(self, name).", + "zlib._ZlibDecompressor.__eq__" => "Return self==value.", + "zlib._ZlibDecompressor.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "zlib._ZlibDecompressor.__ge__" => "Return self>=value.", + "zlib._ZlibDecompressor.__getattribute__" => "Return getattr(self, name).", + "zlib._ZlibDecompressor.__getstate__" => "Helper for pickle.", + "zlib._ZlibDecompressor.__gt__" => "Return self>value.", + "zlib._ZlibDecompressor.__hash__" => "Return hash(self).", + "zlib._ZlibDecompressor.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "zlib._ZlibDecompressor.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "zlib._ZlibDecompressor.__le__" => "Return self<=value.", + "zlib._ZlibDecompressor.__lt__" => "Return self<value.", + "zlib._ZlibDecompressor.__ne__" => "Return self!=value.", + "zlib._ZlibDecompressor.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "zlib._ZlibDecompressor.__reduce__" => "Helper for pickle.", + "zlib._ZlibDecompressor.__reduce_ex__" => "Helper for pickle.", + "zlib._ZlibDecompressor.__repr__" => "Return repr(self).", + "zlib._ZlibDecompressor.__setattr__" => "Implement setattr(self, name, value).", + "zlib._ZlibDecompressor.__sizeof__" => "Size of object in memory, in bytes.", + "zlib._ZlibDecompressor.__str__" => "Return str(self).", + "zlib._ZlibDecompressor.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "zlib._ZlibDecompressor.decompress" => "Decompress *data*, returning uncompressed data as bytes.\n\nIf *max_length* is nonnegative, returns at most *max_length* bytes of\ndecompressed data. If this limit is reached and further output can be\nproduced, *self.needs_input* will be set to ``False``. In this case, the next\ncall to *decompress()* may provide *data* as b'' to obtain more of the output.\n\nIf all of the input data was decompressed and returned (either because this\nwas less than *max_length* bytes, or because *max_length* was negative),\n*self.needs_input* will be set to True.\n\nAttempting to decompress data after the end of stream is reached raises an\nEOFError. Any data found after the end of the stream is ignored and saved in\nthe unused_data attribute.", + "zlib._ZlibDecompressor.eof" => "True if the end-of-stream marker has been reached.", + "zlib._ZlibDecompressor.needs_input" => "True if more input is needed before more decompressed data can be produced.", + "zlib._ZlibDecompressor.unused_data" => "Data found after the end of the compressed stream.", + "zlib.adler32" => "Compute an Adler-32 checksum of data.\n\n value\n Starting value of the checksum.\n\nThe returned checksum is an integer.", + "zlib.compress" => "Returns a bytes object containing compressed data.\n\n data\n Binary data to be compressed.\n level\n Compression level, in 0-9 or -1.\n wbits\n The window buffer size and container format.", + "zlib.compressobj" => "Return a compressor object.\n\n level\n The compression level (an integer in the range 0-9 or -1; default is\n currently equivalent to 6). Higher compression levels are slower,\n but produce smaller results.\n method\n The compression algorithm. If given, this must be DEFLATED.\n wbits\n +9 to +15: The base-two logarithm of the window size. Include a zlib\n container.\n -9 to -15: Generate a raw stream.\n +25 to +31: Include a gzip container.\n memLevel\n Controls the amount of memory used for internal compression state.\n Valid values range from 1 to 9. Higher values result in higher memory\n usage, faster compression, and smaller output.\n strategy\n Used to tune the compression algorithm. Possible values are\n Z_DEFAULT_STRATEGY, Z_FILTERED, and Z_HUFFMAN_ONLY.\n zdict\n The predefined compression dictionary - a sequence of bytes\n containing subsequences that are likely to occur in the input data.", + "zlib.crc32" => "Compute a CRC-32 checksum of data.\n\n value\n Starting value of the checksum.\n\nThe returned checksum is an integer.", + "zlib.decompress" => "Returns a bytes object containing the uncompressed data.\n\n data\n Compressed data.\n wbits\n The window buffer size and container format.\n bufsize\n The initial output buffer size.", + "zlib.decompressobj" => "Return a decompressor object.\n\n wbits\n The window buffer size and container format.\n zdict\n The predefined compression dictionary. This must be the same\n dictionary as used by the compressor that produced the input data.", + "zlib.error.__delattr__" => "Implement delattr(self, name).", + "zlib.error.__eq__" => "Return self==value.", + "zlib.error.__format__" => "Default object formatter.\n\nReturn str(self) if format_spec is empty. Raise TypeError otherwise.", + "zlib.error.__ge__" => "Return self>=value.", + "zlib.error.__getattribute__" => "Return getattr(self, name).", + "zlib.error.__getstate__" => "Helper for pickle.", + "zlib.error.__gt__" => "Return self>value.", + "zlib.error.__hash__" => "Return hash(self).", + "zlib.error.__init__" => "Initialize self. See help(type(self)) for accurate signature.", + "zlib.error.__init_subclass__" => "This method is called when a class is subclassed.\n\nThe default implementation does nothing. It may be\noverridden to extend subclasses.", + "zlib.error.__le__" => "Return self<=value.", + "zlib.error.__lt__" => "Return self<value.", + "zlib.error.__ne__" => "Return self!=value.", + "zlib.error.__new__" => "Create and return a new object. See help(type) for accurate signature.", + "zlib.error.__reduce_ex__" => "Helper for pickle.", + "zlib.error.__repr__" => "Return repr(self).", + "zlib.error.__setattr__" => "Implement setattr(self, name, value).", + "zlib.error.__sizeof__" => "Size of object in memory, in bytes.", + "zlib.error.__str__" => "Return str(self).", + "zlib.error.__subclasshook__" => "Abstract classes can override this to customize issubclass().\n\nThis is invoked early on by abc.ABCMeta.__subclasscheck__().\nIt should return True, False or NotImplemented. If it returns\nNotImplemented, the normal algorithm is used. Otherwise, it\noverrides the normal algorithm (and the outcome is cached).", + "zlib.error.__weakref__" => "list of weak references to the object", + "zlib.error.add_note" => "Add a note to the exception", + "zlib.error.with_traceback" => "Set self.__traceback__ to tb and return self.", +}; diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs new file mode 100644 index 00000000000..8b7f5d8f75b --- /dev/null +++ b/crates/doc/src/lib.rs @@ -0,0 +1,14 @@ +#![no_std] + +include!("./data.inc.rs"); + +#[cfg(test)] +mod test { + use super::DB; + + #[test] + fn test_db() { + let doc = DB.get("array._array_reconstructor"); + assert!(doc.is_some()); + } +} diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml new file mode 100644 index 00000000000..acb1209e7b3 --- /dev/null +++ b/crates/jit/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "rustpython-jit" +description = "Experimental JIT(just in time) compiler for python code." +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +autotests = false + +[dependencies] +rustpython-compiler-core = { workspace = true } + +num-traits = { workspace = true } +thiserror = { workspace = true } +libffi = { workspace = true } + +cranelift = "0.129.1" +cranelift-jit = "0.129.1" +cranelift-module = "0.129.1" + +[dev-dependencies] +rustpython-derive = { workspace = true } +rustpython-wtf8 = { workspace = true } + +approx = "0.5.1" + +[[test]] +name = "integration" +path = "tests/lib.rs" + +[lints] +workspace = true diff --git a/crates/jit/src/instructions.rs b/crates/jit/src/instructions.rs new file mode 100644 index 00000000000..922f665f350 --- /dev/null +++ b/crates/jit/src/instructions.rs @@ -0,0 +1,1458 @@ +// spell-checker: disable +use super::{JitCompileError, JitSig, JitType}; +use alloc::collections::BTreeSet; +use cranelift::codegen::ir::FuncRef; +use cranelift::prelude::*; +use num_traits::cast::ToPrimitive; +use rustpython_compiler_core::bytecode::{ + self, BinaryOperator, BorrowedConstant, CodeObject, ComparisonOperator, Instruction, + IntrinsicFunction1, Label, OpArg, OpArgState, oparg, +}; +use std::collections::HashMap; + +#[repr(u16)] +enum CustomTrapCode { + /// Raised when shifting by a negative number + NegativeShiftCount = 1, +} + +#[derive(Clone)] +struct Local { + var: Variable, + ty: JitType, +} + +#[derive(Debug)] +enum JitValue { + Int(Value), + Float(Value), + Bool(Value), + None, + Null, + Tuple(Vec<JitValue>), + FuncRef(FuncRef), +} + +impl JitValue { + fn from_type_and_value(ty: JitType, val: Value) -> JitValue { + match ty { + JitType::Int => JitValue::Int(val), + JitType::Float => JitValue::Float(val), + JitType::Bool => JitValue::Bool(val), + } + } + + fn to_jit_type(&self) -> Option<JitType> { + match self { + JitValue::Int(_) => Some(JitType::Int), + JitValue::Float(_) => Some(JitType::Float), + JitValue::Bool(_) => Some(JitType::Bool), + JitValue::None | JitValue::Null | JitValue::Tuple(_) | JitValue::FuncRef(_) => None, + } + } + + fn into_value(self) -> Option<Value> { + match self { + JitValue::Int(val) | JitValue::Float(val) | JitValue::Bool(val) => Some(val), + JitValue::None | JitValue::Null | JitValue::Tuple(_) | JitValue::FuncRef(_) => None, + } + } +} + +#[derive(Clone)] +struct DDValue { + hi: Value, + lo: Value, +} + +pub struct FunctionCompiler<'a, 'b> { + builder: &'a mut FunctionBuilder<'b>, + stack: Vec<JitValue>, + variables: Box<[Option<Local>]>, + label_to_block: HashMap<Label, Block>, + pub(crate) sig: JitSig, +} + +impl<'a, 'b> FunctionCompiler<'a, 'b> { + pub fn new( + builder: &'a mut FunctionBuilder<'b>, + num_variables: usize, + arg_types: &[JitType], + ret_type: Option<JitType>, + entry_block: Block, + ) -> FunctionCompiler<'a, 'b> { + let mut compiler = FunctionCompiler { + builder, + stack: Vec::new(), + variables: vec![None; num_variables].into_boxed_slice(), + label_to_block: HashMap::new(), + sig: JitSig { + args: arg_types.to_vec(), + ret: ret_type, + }, + }; + let params = compiler.builder.func.dfg.block_params(entry_block).to_vec(); + for (i, (ty, val)) in arg_types.iter().zip(params).enumerate() { + compiler + .store_variable( + (i as u32).into(), + JitValue::from_type_and_value(ty.clone(), val), + ) + .unwrap(); + } + compiler + } + + fn pop_multiple(&mut self, count: usize) -> Vec<JitValue> { + let stack_len = self.stack.len(); + self.stack.drain(stack_len - count..).collect() + } + + fn store_variable(&mut self, idx: oparg::VarNum, val: JitValue) -> Result<(), JitCompileError> { + let builder = &mut self.builder; + let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; + let local = self.variables[idx].get_or_insert_with(|| { + let var = builder.declare_var(ty.to_cranelift()); + Local { + var, + ty: ty.clone(), + } + }); + if ty != local.ty { + Err(JitCompileError::NotSupported) + } else { + self.builder.def_var(local.var, val.into_value().unwrap()); + Ok(()) + } + } + + fn boolean_val(&mut self, val: JitValue) -> Result<Value, JitCompileError> { + match val { + JitValue::Float(val) => { + let zero = self.builder.ins().f64const(0.0); + let val = self.builder.ins().fcmp(FloatCC::NotEqual, val, zero); + Ok(val) + } + JitValue::Int(val) => { + let zero = self.builder.ins().iconst(types::I64, 0); + let val = self.builder.ins().icmp(IntCC::NotEqual, val, zero); + Ok(val) + } + JitValue::Bool(val) => Ok(val), + JitValue::None => Ok(self.builder.ins().iconst(types::I8, 0)), + JitValue::Null | JitValue::Tuple(_) | JitValue::FuncRef(_) => { + Err(JitCompileError::NotSupported) + } + } + } + + fn get_or_create_block(&mut self, label: Label) -> Block { + let builder = &mut self.builder; + *self + .label_to_block + .entry(label) + .or_insert_with(|| builder.create_block()) + } + + fn jump_target_forward(offset: u32, caches: u32, arg: OpArg) -> Result<Label, JitCompileError> { + let after = offset + .checked_add(1) + .and_then(|i| i.checked_add(caches)) + .ok_or(JitCompileError::BadBytecode)?; + let target = after + .checked_add(u32::from(arg)) + .ok_or(JitCompileError::BadBytecode)?; + Ok(Label::new(target)) + } + + fn jump_target_backward( + offset: u32, + caches: u32, + arg: OpArg, + ) -> Result<Label, JitCompileError> { + let after = offset + .checked_add(1) + .and_then(|i| i.checked_add(caches)) + .ok_or(JitCompileError::BadBytecode)?; + let target = after + .checked_sub(u32::from(arg)) + .ok_or(JitCompileError::BadBytecode)?; + Ok(Label::new(target)) + } + + fn instruction_target( + offset: u32, + instruction: Instruction, + arg: OpArg, + ) -> Result<Option<Label>, JitCompileError> { + let caches = instruction.cache_entries() as u32; + let target = match instruction { + Instruction::JumpForward { .. } => { + Some(Self::jump_target_forward(offset, caches, arg)?) + } + Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } => { + Some(Self::jump_target_backward(offset, caches, arg)?) + } + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + | Instruction::ForIter { .. } + | Instruction::Send { .. } => Some(Self::jump_target_forward(offset, caches, arg)?), + _ => None, + }; + Ok(target) + } + + pub fn compile<C: bytecode::Constant>( + &mut self, + func_ref: FuncRef, + bytecode: &CodeObject<C>, + ) -> Result<(), JitCompileError> { + // JIT should consume a stable instruction stream: de-specialized opcodes + // with zeroed CACHE entries, not runtime-mutated quickened code. + let clean_instructions: bytecode::CodeUnits = bytecode + .instructions + .original_bytes() + .as_slice() + .try_into() + .map_err(|_| JitCompileError::BadBytecode)?; + + let mut label_targets = BTreeSet::new(); + let mut target_arg_state = OpArgState::default(); + for (offset, &raw_instr) in clean_instructions.iter().enumerate() { + let (instruction, arg) = target_arg_state.get(raw_instr); + if let Some(target) = Self::instruction_target(offset as u32, instruction, arg)? { + label_targets.insert(target); + } + } + let mut arg_state = OpArgState::default(); + + // Track whether we have "returned" in the current block + let mut in_unreachable_code = false; + + for (offset, &raw_instr) in clean_instructions.iter().enumerate() { + let label = Label::new(offset as u32); + let (instruction, arg) = arg_state.get(raw_instr); + + // If this is a label that some earlier jump can target, + // treat it as the start of a new reachable block: + if label_targets.contains(&label) { + // Create or get the block for this label: + let target_block = self.get_or_create_block(label); + + // If the current block isn't terminated, add a fallthrough jump + if let Some(cur) = self.builder.current_block() + && cur != target_block + { + // Check if the block needs a terminator by examining the last instruction + let needs_terminator = match self.builder.func.layout.last_inst(cur) { + None => true, // Empty block needs terminator + Some(inst) => { + // Check if the last instruction is a terminator + !self.builder.func.dfg.insts[inst].opcode().is_terminator() + } + }; + if needs_terminator { + self.builder.ins().jump(target_block, &[]); + } + } + // Switch to the target block + if self.builder.current_block() != Some(target_block) { + self.builder.switch_to_block(target_block); + } + + // We are definitely reachable again at this label + in_unreachable_code = false; + } + + // If we're in unreachable code, skip this instruction unless the label re-entered above. + if in_unreachable_code { + continue; + } + + // Actually compile this instruction: + self.add_instruction(func_ref, bytecode, offset as u32, instruction, arg)?; + + // If that was an unconditional branch or return, mark future instructions unreachable + match instruction { + Instruction::ReturnValue + | Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::JumpForward { .. } => { + in_unreachable_code = true; + } + _ => {} + } + } + + // After processing, if the current block is unterminated, insert a trap + if let Some(cur) = self.builder.current_block() { + let needs_terminator = match self.builder.func.layout.last_inst(cur) { + None => true, + Some(inst) => !self.builder.func.dfg.insts[inst].opcode().is_terminator(), + }; + if needs_terminator { + self.builder.ins().trap(TrapCode::user(0).unwrap()); + } + } + Ok(()) + } + + fn prepare_const<C: bytecode::Constant>( + &mut self, + constant: BorrowedConstant<'_, C>, + ) -> Result<JitValue, JitCompileError> { + let value = match constant { + BorrowedConstant::Integer { value } => { + let val = self.builder.ins().iconst( + types::I64, + value.to_i64().ok_or(JitCompileError::NotSupported)?, + ); + JitValue::Int(val) + } + BorrowedConstant::Float { value } => { + let val = self.builder.ins().f64const(value); + JitValue::Float(val) + } + BorrowedConstant::Boolean { value } => { + let val = self.builder.ins().iconst(types::I8, value as i64); + JitValue::Bool(val) + } + BorrowedConstant::None => JitValue::None, + _ => return Err(JitCompileError::NotSupported), + }; + Ok(value) + } + + fn return_value(&mut self, val: JitValue) -> Result<(), JitCompileError> { + if let Some(ref ty) = self.sig.ret { + // If the signature has a return type, enforce it + if val.to_jit_type().as_ref() != Some(ty) { + return Err(JitCompileError::NotSupported); + } + } else { + // First time we see a return, define it in the signature + let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; + self.sig.ret = Some(ty.clone()); + self.builder + .func + .signature + .returns + .push(AbiParam::new(ty.to_cranelift())); + } + + // If this is e.g. an Int, Float, or Bool we have a Cranelift `Value`. + // If we have JitValue::None or .Tuple(...) but can't handle that, error out (or handle differently). + let cr_val = val.into_value().ok_or(JitCompileError::NotSupported)?; + + self.builder.ins().return_(&[cr_val]); + Ok(()) + } + + pub fn add_instruction<C: bytecode::Constant>( + &mut self, + func_ref: FuncRef, + bytecode: &CodeObject<C>, + offset: u32, + instruction: Instruction, + arg: OpArg, + ) -> Result<(), JitCompileError> { + match instruction { + Instruction::BinaryOp { op } => { + let op = op.get(arg); + // the rhs is popped off first + let b = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + let a = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + + let a_type = a.to_jit_type(); + let b_type = b.to_jit_type(); + + let val = match (op, a, b) { + ( + BinaryOperator::Add | BinaryOperator::InplaceAdd, + JitValue::Int(a), + JitValue::Int(b), + ) => { + let (out, carry) = self.builder.ins().sadd_overflow(a, b); + self.builder.ins().trapnz(carry, TrapCode::INTEGER_OVERFLOW); + JitValue::Int(out) + } + ( + BinaryOperator::Subtract | BinaryOperator::InplaceSubtract, + JitValue::Int(a), + JitValue::Int(b), + ) => JitValue::Int(self.compile_sub(a, b)), + ( + BinaryOperator::FloorDivide | BinaryOperator::InplaceFloorDivide, + JitValue::Int(a), + JitValue::Int(b), + ) => JitValue::Int(self.builder.ins().sdiv(a, b)), + ( + BinaryOperator::TrueDivide | BinaryOperator::InplaceTrueDivide, + JitValue::Int(a), + JitValue::Int(b), + ) => { + // Check if b == 0, If so trap with a division by zero error + self.builder + .ins() + .trapz(b, TrapCode::INTEGER_DIVISION_BY_ZERO); + // Else convert to float and divide + let a_float = self.builder.ins().fcvt_from_sint(types::F64, a); + let b_float = self.builder.ins().fcvt_from_sint(types::F64, b); + JitValue::Float(self.builder.ins().fdiv(a_float, b_float)) + } + ( + BinaryOperator::Multiply | BinaryOperator::InplaceMultiply, + JitValue::Int(a), + JitValue::Int(b), + ) => JitValue::Int(self.builder.ins().imul(a, b)), + ( + BinaryOperator::Remainder | BinaryOperator::InplaceRemainder, + JitValue::Int(a), + JitValue::Int(b), + ) => JitValue::Int(self.builder.ins().srem(a, b)), + ( + BinaryOperator::Power | BinaryOperator::InplacePower, + JitValue::Int(a), + JitValue::Int(b), + ) => JitValue::Int(self.compile_ipow(a, b)), + ( + BinaryOperator::Lshift | BinaryOperator::Rshift, + JitValue::Int(a), + JitValue::Int(b), + ) => { + // Shifts throw an exception if we have a negative shift count + // Remove all bits except the sign bit, and trap if its 1 (i.e. negative). + let sign = self.builder.ins().ushr_imm(b, 63); + self.builder.ins().trapnz( + sign, + TrapCode::user(CustomTrapCode::NegativeShiftCount as u8).unwrap(), + ); + + let out = + if matches!(op, BinaryOperator::Lshift | BinaryOperator::InplaceLshift) + { + self.builder.ins().ishl(a, b) + } else { + self.builder.ins().sshr(a, b) + }; + JitValue::Int(out) + } + ( + BinaryOperator::And | BinaryOperator::InplaceAnd, + JitValue::Int(a), + JitValue::Int(b), + ) => JitValue::Int(self.builder.ins().band(a, b)), + ( + BinaryOperator::Or | BinaryOperator::InplaceOr, + JitValue::Int(a), + JitValue::Int(b), + ) => JitValue::Int(self.builder.ins().bor(a, b)), + ( + BinaryOperator::Xor | BinaryOperator::InplaceXor, + JitValue::Int(a), + JitValue::Int(b), + ) => JitValue::Int(self.builder.ins().bxor(a, b)), + + // Floats + ( + BinaryOperator::Add | BinaryOperator::InplaceAdd, + JitValue::Float(a), + JitValue::Float(b), + ) => JitValue::Float(self.builder.ins().fadd(a, b)), + ( + BinaryOperator::Subtract | BinaryOperator::InplaceSubtract, + JitValue::Float(a), + JitValue::Float(b), + ) => JitValue::Float(self.builder.ins().fsub(a, b)), + ( + BinaryOperator::Multiply | BinaryOperator::InplaceMultiply, + JitValue::Float(a), + JitValue::Float(b), + ) => JitValue::Float(self.builder.ins().fmul(a, b)), + ( + BinaryOperator::TrueDivide | BinaryOperator::InplaceTrueDivide, + JitValue::Float(a), + JitValue::Float(b), + ) => JitValue::Float(self.builder.ins().fdiv(a, b)), + ( + BinaryOperator::Power | BinaryOperator::InplacePower, + JitValue::Float(a), + JitValue::Float(b), + ) => JitValue::Float(self.compile_fpow(a, b)), + + // Floats and Integers + (_, JitValue::Int(a), JitValue::Float(b)) + | (_, JitValue::Float(a), JitValue::Int(b)) => { + let operand_one = match a_type.unwrap() { + JitType::Int => self.builder.ins().fcvt_from_sint(types::F64, a), + _ => a, + }; + + let operand_two = match b_type.unwrap() { + JitType::Int => self.builder.ins().fcvt_from_sint(types::F64, b), + _ => b, + }; + + match op { + BinaryOperator::Add | BinaryOperator::InplaceAdd => { + JitValue::Float(self.builder.ins().fadd(operand_one, operand_two)) + } + BinaryOperator::Subtract | BinaryOperator::InplaceSubtract => { + JitValue::Float(self.builder.ins().fsub(operand_one, operand_two)) + } + BinaryOperator::Multiply | BinaryOperator::InplaceMultiply => { + JitValue::Float(self.builder.ins().fmul(operand_one, operand_two)) + } + BinaryOperator::TrueDivide | BinaryOperator::InplaceTrueDivide => { + JitValue::Float(self.builder.ins().fdiv(operand_one, operand_two)) + } + BinaryOperator::Power | BinaryOperator::InplacePower => { + JitValue::Float(self.compile_fpow(operand_one, operand_two)) + } + _ => return Err(JitCompileError::NotSupported), + } + } + _ => return Err(JitCompileError::NotSupported), + }; + self.stack.push(val); + + Ok(()) + } + Instruction::BuildTuple { count } => { + let elements = self.pop_multiple(count.get(arg) as usize); + self.stack.push(JitValue::Tuple(elements)); + Ok(()) + } + Instruction::Call { argc } => { + let nargs = argc.get(arg); + + let mut args = Vec::new(); + for _ in 0..nargs { + let arg = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + args.push(arg.into_value().unwrap()); + } + + // Pop self_or_null (should be Null for JIT-compiled recursive calls) + let self_or_null = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + if !matches!(self_or_null, JitValue::Null) { + return Err(JitCompileError::NotSupported); + } + + match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { + JitValue::FuncRef(reference) => { + let call = self.builder.ins().call(reference, &args); + let returns = self.builder.inst_results(call); + self.stack.push(JitValue::Int(returns[0])); + + Ok(()) + } + _ => Err(JitCompileError::BadBytecode), + } + } + Instruction::PushNull => { + self.stack.push(JitValue::Null); + Ok(()) + } + Instruction::CallIntrinsic1 { func } => { + match func.get(arg) { + IntrinsicFunction1::UnaryPositive => { + match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { + JitValue::Int(val) => { + // Nothing to do + self.stack.push(JitValue::Int(val)); + Ok(()) + } + _ => Err(JitCompileError::NotSupported), + } + } + _ => Err(JitCompileError::NotSupported), + } + } + Instruction::CompareOp { opname } => { + let op = opname.get(arg); + // the rhs is popped off first + let b = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + let a = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + + let a_type: Option<JitType> = a.to_jit_type(); + let b_type: Option<JitType> = b.to_jit_type(); + + match (a, b) { + (JitValue::Int(a), JitValue::Int(b)) + | (JitValue::Bool(a), JitValue::Bool(b)) + | (JitValue::Bool(a), JitValue::Int(b)) + | (JitValue::Int(a), JitValue::Bool(b)) => { + let operand_one = match a_type.unwrap() { + JitType::Bool => self.builder.ins().uextend(types::I64, a), + _ => a, + }; + + let operand_two = match b_type.unwrap() { + JitType::Bool => self.builder.ins().uextend(types::I64, b), + _ => b, + }; + + let cond = match op { + ComparisonOperator::Equal => IntCC::Equal, + ComparisonOperator::NotEqual => IntCC::NotEqual, + ComparisonOperator::Less => IntCC::SignedLessThan, + ComparisonOperator::LessOrEqual => IntCC::SignedLessThanOrEqual, + ComparisonOperator::Greater => IntCC::SignedGreaterThan, + ComparisonOperator::GreaterOrEqual => IntCC::SignedGreaterThanOrEqual, + }; + + let val = self.builder.ins().icmp(cond, operand_one, operand_two); + self.stack.push(JitValue::Bool(val)); + Ok(()) + } + (JitValue::Float(a), JitValue::Float(b)) => { + let cond = match op { + ComparisonOperator::Equal => FloatCC::Equal, + ComparisonOperator::NotEqual => FloatCC::NotEqual, + ComparisonOperator::Less => FloatCC::LessThan, + ComparisonOperator::LessOrEqual => FloatCC::LessThanOrEqual, + ComparisonOperator::Greater => FloatCC::GreaterThan, + ComparisonOperator::GreaterOrEqual => FloatCC::GreaterThanOrEqual, + }; + + let val = self.builder.ins().fcmp(cond, a, b); + self.stack.push(JitValue::Bool(val)); + Ok(()) + } + _ => Err(JitCompileError::NotSupported), + } + } + Instruction::ExtendedArg | Instruction::Cache => Ok(()), + + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } + | Instruction::JumpForward { .. } => { + let target = Self::instruction_target(offset, instruction, arg)? + .ok_or(JitCompileError::BadBytecode)?; + let target_block = self.get_or_create_block(target); + self.builder.ins().jump(target_block, &[]); + Ok(()) + } + Instruction::LoadConst { consti } => { + let val = + self.prepare_const(bytecode.constants[consti.get(arg)].borrow_constant())?; + self.stack.push(val); + Ok(()) + } + Instruction::LoadSmallInt { i } => { + let small_int = i.get(arg) as i64; + let val = self.builder.ins().iconst(types::I64, small_int); + self.stack.push(JitValue::Int(val)); + Ok(()) + } + Instruction::LoadFast { var_num } | Instruction::LoadFastBorrow { var_num } => { + let local = self.variables[var_num.get(arg)] + .as_ref() + .ok_or(JitCompileError::BadBytecode)?; + self.stack.push(JitValue::from_type_and_value( + local.ty.clone(), + self.builder.use_var(local.var), + )); + Ok(()) + } + Instruction::LoadFastLoadFast { var_nums } + | Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let oparg = var_nums.get(arg); + let (idx1, idx2) = oparg.indexes(); + for idx in [idx1, idx2] { + let local = self.variables[idx] + .as_ref() + .ok_or(JitCompileError::BadBytecode)?; + self.stack.push(JitValue::from_type_and_value( + local.ty.clone(), + self.builder.use_var(local.var), + )); + } + Ok(()) + } + Instruction::LoadGlobal { namei } => { + let oparg = namei.get(arg); + let name = &bytecode.names[(oparg >> 1) as usize]; + + if name.as_ref() != bytecode.obj_name.as_ref() { + Err(JitCompileError::NotSupported) + } else { + self.stack.push(JitValue::FuncRef(func_ref)); + if (oparg & 1) != 0 { + self.stack.push(JitValue::Null); + } + Ok(()) + } + } + Instruction::Nop | Instruction::NotTaken => Ok(()), + Instruction::PopJumpIfFalse { .. } => { + let cond = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + let val = self.boolean_val(cond)?; + let then_label = Self::instruction_target(offset, instruction, arg)? + .ok_or(JitCompileError::BadBytecode)?; + let then_block = self.get_or_create_block(then_label); + let else_block = self.builder.create_block(); + + self.builder + .ins() + .brif(val, else_block, &[], then_block, &[]); + self.builder.switch_to_block(else_block); + + Ok(()) + } + Instruction::PopJumpIfTrue { .. } => { + let cond = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + let val = self.boolean_val(cond)?; + let then_label = Self::instruction_target(offset, instruction, arg)? + .ok_or(JitCompileError::BadBytecode)?; + let then_block = self.get_or_create_block(then_label); + let else_block = self.builder.create_block(); + + self.builder + .ins() + .brif(val, then_block, &[], else_block, &[]); + self.builder.switch_to_block(else_block); + + Ok(()) + } + Instruction::PopTop => { + self.stack.pop(); + Ok(()) + } + Instruction::Resume { .. } => { + // TODO: Implement the resume instruction + Ok(()) + } + Instruction::ReturnValue => { + let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + self.return_value(val) + } + Instruction::StoreFast { var_num } => { + let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + self.store_variable(var_num.get(arg), val) + } + Instruction::Swap { i: index } => { + let len = self.stack.len(); + let i = len - 1; + let j = len - 1 - index.get(arg) as usize; + self.stack.swap(i, j); + Ok(()) + } + Instruction::ToBool => { + let a = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + let value = self.boolean_val(a)?; + self.stack.push(JitValue::Bool(value)); + Ok(()) + } + Instruction::UnaryNot => { + let boolean = match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { + JitValue::Bool(val) => val, + _ => return Err(JitCompileError::BadBytecode), + }; + let not_boolean = self.builder.ins().bxor_imm(boolean, 1); + self.stack.push(JitValue::Bool(not_boolean)); + Ok(()) + } + Instruction::UnaryNegative => { + match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { + JitValue::Int(val) => { + // Compile minus as 0 - val. + let zero = self.builder.ins().iconst(types::I64, 0); + let out = self.compile_sub(zero, val); + self.stack.push(JitValue::Int(out)); + Ok(()) + } + _ => Err(JitCompileError::NotSupported), + } + } + Instruction::UnpackSequence { count } => { + let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; + + let elements = match val { + JitValue::Tuple(elements) => elements, + _ => return Err(JitCompileError::NotSupported), + }; + + if elements.len() != count.get(arg) as usize { + return Err(JitCompileError::NotSupported); + } + + self.stack.extend(elements.into_iter().rev()); + Ok(()) + } + _ => Err(JitCompileError::NotSupported), + } + } + + fn compile_sub(&mut self, a: Value, b: Value) -> Value { + let (out, carry) = self.builder.ins().ssub_overflow(a, b); + self.builder.ins().trapnz(carry, TrapCode::INTEGER_OVERFLOW); + out + } + + /// Creates a double–double (DDValue) from a regular f64 constant. + /// The high part is set to x and the low part is set to 0.0. + fn dd_from_f64(&mut self, x: f64) -> DDValue { + DDValue { + hi: self.builder.ins().f64const(x), + lo: self.builder.ins().f64const(0.0), + } + } + + /// Creates a DDValue from a Value (assumed to represent an f64). + /// This function initializes the high part with x and the low part to 0.0. + fn dd_from_value(&mut self, x: Value) -> DDValue { + DDValue { + hi: x, + lo: self.builder.ins().f64const(0.0), + } + } + + /// Creates a DDValue from two f64 parts. + /// The 'hi' parameter sets the high part and 'lo' sets the low part. + fn dd_from_parts(&mut self, hi: f64, lo: f64) -> DDValue { + DDValue { + hi: self.builder.ins().f64const(hi), + lo: self.builder.ins().f64const(lo), + } + } + + /// Converts a DDValue back to a single f64 value by adding the high and low parts. + fn dd_to_f64(&mut self, dd: DDValue) -> Value { + self.builder.ins().fadd(dd.hi, dd.lo) + } + + /// Computes the negation of a DDValue. + /// It subtracts both the high and low parts from zero. + fn dd_neg(&mut self, dd: DDValue) -> DDValue { + let zero = self.builder.ins().f64const(0.0); + DDValue { + hi: self.builder.ins().fsub(zero, dd.hi), + lo: self.builder.ins().fsub(zero, dd.lo), + } + } + + /// Adds two DDValue numbers using error-free transformations to maintain extra precision. + /// It carefully adds the high parts, computes the rounding error, adds the low parts along with the error, + /// and then normalizes the result. + fn dd_add(&mut self, a: DDValue, b: DDValue) -> DDValue { + // Compute the sum of the high parts. + let s = self.builder.ins().fadd(a.hi, b.hi); + // Compute t = s - a.hi to capture part of the rounding error. + let t = self.builder.ins().fsub(s, a.hi); + // Compute the error e from the high part additions. + let s_minus_t = self.builder.ins().fsub(s, t); + let part1 = self.builder.ins().fsub(a.hi, s_minus_t); + let part2 = self.builder.ins().fsub(b.hi, t); + let e = self.builder.ins().fadd(part1, part2); + // Sum the low parts along with the error. + let lo = self.builder.ins().fadd(a.lo, b.lo); + let lo_sum = self.builder.ins().fadd(lo, e); + // Renormalize: add the low sum to s and compute a new low component. + let hi_new = self.builder.ins().fadd(s, lo_sum); + let hi_new_minus_s = self.builder.ins().fsub(hi_new, s); + let lo_new = self.builder.ins().fsub(lo_sum, hi_new_minus_s); + DDValue { + hi: hi_new, + lo: lo_new, + } + } + + /// Subtracts DDValue b from DDValue a by negating b and then using the addition function. + fn dd_sub(&mut self, a: DDValue, b: DDValue) -> DDValue { + let neg_b = self.dd_neg(b); + self.dd_add(a, neg_b) + } + + /// Multiplies two DDValue numbers using double–double arithmetic. + /// It calculates the high product, uses a fused multiply–add (FMA) to capture rounding error, + /// computes the cross products, and then normalizes the result. + fn dd_mul(&mut self, a: DDValue, b: DDValue) -> DDValue { + // p = a.hi * b.hi (primary product) + let p = self.builder.ins().fmul(a.hi, b.hi); + // err = fma(a.hi, b.hi, -p) recovers the rounding error. + let zero = self.builder.ins().f64const(0.0); + let neg_p = self.builder.ins().fsub(zero, p); + let err = self.builder.ins().fma(a.hi, b.hi, neg_p); + // Compute cross terms: a.hi*b.lo + a.lo*b.hi. + let a_hi_b_lo = self.builder.ins().fmul(a.hi, b.lo); + let a_lo_b_hi = self.builder.ins().fmul(a.lo, b.hi); + let cross = self.builder.ins().fadd(a_hi_b_lo, a_lo_b_hi); + // Sum p and the cross terms. + let s = self.builder.ins().fadd(p, cross); + // Isolate rounding error from the addition. + let t = self.builder.ins().fsub(s, p); + let s_minus_t = self.builder.ins().fsub(s, t); + let part1 = self.builder.ins().fsub(p, s_minus_t); + let part2 = self.builder.ins().fsub(cross, t); + let e = self.builder.ins().fadd(part1, part2); + // Include the error from the low parts multiplication. + let a_lo_b_lo = self.builder.ins().fmul(a.lo, b.lo); + let err_plus_e = self.builder.ins().fadd(err, e); + let lo_sum = self.builder.ins().fadd(err_plus_e, a_lo_b_lo); + // Renormalize the sum. + let hi_new = self.builder.ins().fadd(s, lo_sum); + let hi_new_minus_s = self.builder.ins().fsub(hi_new, s); + let lo_new = self.builder.ins().fsub(lo_sum, hi_new_minus_s); + DDValue { + hi: hi_new, + lo: lo_new, + } + } + + /// Multiplies a DDValue by a regular f64 (Value) using similar techniques as dd_mul. + /// It multiplies both the high and low parts by b, computes the rounding error, + /// and then renormalizes the result. + fn dd_mul_f64(&mut self, a: DDValue, b: Value) -> DDValue { + // p = a.hi * b (primary product) + let p = self.builder.ins().fmul(a.hi, b); + // Compute the rounding error using fma. + let zero = self.builder.ins().f64const(0.0); + let neg_p = self.builder.ins().fsub(zero, p); + let err = self.builder.ins().fma(a.hi, b, neg_p); + // Multiply the low part. + let cross = self.builder.ins().fmul(a.lo, b); + // Sum the primary product and the low multiplication. + let s = self.builder.ins().fadd(p, cross); + // Capture rounding error from addition. + let t = self.builder.ins().fsub(s, p); + let s_minus_t = self.builder.ins().fsub(s, t); + let part1 = self.builder.ins().fsub(p, s_minus_t); + let part2 = self.builder.ins().fsub(cross, t); + let e = self.builder.ins().fadd(part1, part2); + // Combine the error components. + let lo_sum = self.builder.ins().fadd(err, e); + // Renormalize to form the final double–double number. + let hi_new = self.builder.ins().fadd(s, lo_sum); + let hi_new_minus_s = self.builder.ins().fsub(hi_new, s); + let lo_new = self.builder.ins().fsub(lo_sum, hi_new_minus_s); + DDValue { + hi: hi_new, + lo: lo_new, + } + } + + /// Scales a DDValue by multiplying both its high and low parts by the given factor. + fn dd_scale(&mut self, dd: DDValue, factor: Value) -> DDValue { + DDValue { + hi: self.builder.ins().fmul(dd.hi, factor), + lo: self.builder.ins().fmul(dd.lo, factor), + } + } + + /// Approximates ln(1+f) using its Taylor series expansion in double–double arithmetic. + /// It computes the series ∑ (-1)^(i-1) * f^i / i from i = 1 to 1000 for high precision. + fn dd_ln_1p_series(&mut self, f: Value) -> DDValue { + // Convert f to a DDValue and initialize the sum and term. + let f_dd = self.dd_from_value(f); + let mut sum = f_dd.clone(); + let mut term = f_dd; + // Alternating sign starts at -1 for the second term. + let mut sign = -1.0_f64; + let range = 1000; + + // Loop over terms from i = 2 to 1000. + for i in 2..=range { + // Compute f^i by multiplying the previous term by f. + term = self.dd_mul_f64(term, f); + // Divide the term by i. + let inv_i = 1.0 / (i as f64); + let c_inv_i = self.builder.ins().f64const(inv_i); + let term_div = self.dd_mul_f64(term.clone(), c_inv_i); + // Multiply by the alternating sign. + let dd_sign = self.dd_from_f64(sign); + let to_add = self.dd_mul(dd_sign, term_div); + // Add the term to the cumulative sum. + sum = self.dd_add(sum, to_add); + // Flip the sign for the next term. + sign = -sign; + } + sum + } + + /// Computes the natural logarithm ln(x) in double–double arithmetic. + /// It first checks for domain errors (x ≤ 0 or NaN), then extracts the exponent + /// and mantissa from the bit-level representation of x. It computes ln(mantissa) using + /// the ln(1+f) series and adds k*ln2 to obtain ln(x). + fn dd_ln(&mut self, x: Value) -> DDValue { + // (A) Prepare a DDValue representing NaN. + let dd_nan = self.dd_from_f64(f64::NAN); + + // Build a zero constant for comparisons. + let zero_f64 = self.builder.ins().f64const(0.0); + + // Check if x is less than or equal to 0 or is NaN. + let cmp_le = self + .builder + .ins() + .fcmp(FloatCC::LessThanOrEqual, x, zero_f64); + let cmp_nan = self.builder.ins().fcmp(FloatCC::Unordered, x, x); + let need_nan = self.builder.ins().bor(cmp_le, cmp_nan); + + // (B) Reinterpret the bits of x as an integer. + let bits = self.builder.ins().bitcast(types::I64, MemFlags::new(), x); + + // (C) Extract the exponent (top 11 bits) from the bit representation. + let shift_52 = self.builder.ins().ushr_imm(bits, 52); + let exponent_mask = self.builder.ins().iconst(types::I64, 0x7FF); + let exponent = self.builder.ins().band(shift_52, exponent_mask); + + // k = exponent - 1023 (unbias the exponent). + let bias = self.builder.ins().iconst(types::I64, 1023); + let k_i64 = self.builder.ins().isub(exponent, bias); + + // (D) Extract the fraction (mantissa) from the lower 52 bits. + let fraction_mask = self.builder.ins().iconst(types::I64, 0x000F_FFFF_FFFF_FFFF); + let fraction_part = self.builder.ins().band(bits, fraction_mask); + + // (E) For normal numbers (exponent ≠ 0), add the implicit leading 1. + let implicit_one = self.builder.ins().iconst(types::I64, 1 << 52); + let zero_exp = self.builder.ins().icmp_imm(IntCC::Equal, exponent, 0); + let frac_one_bor = self.builder.ins().bor(fraction_part, implicit_one); + let fraction_with_leading_one = self.builder.ins().select( + zero_exp, + fraction_part, // For subnormals, do not add the implicit 1. + frac_one_bor, + ); + + // (F) Force the exponent bits to 1023, yielding a mantissa m in [1, 2). + let new_exp = self.builder.ins().iconst(types::I64, 0x3FF0_0000_0000_0000); + let fraction_bits = self.builder.ins().bor(fraction_with_leading_one, new_exp); + let m = self + .builder + .ins() + .bitcast(types::F64, MemFlags::new(), fraction_bits); + + // (G) Compute ln(m) using the series ln(1+f) with f = m - 1. + let one_f64 = self.builder.ins().f64const(1.0); + let f_val = self.builder.ins().fsub(m, one_f64); + let dd_ln_m = self.dd_ln_1p_series(f_val); + + // (H) Compute k*ln2 in double–double arithmetic. + let ln2_dd = self.dd_from_parts( + f64::from_bits(0x3fe62e42fefa39ef), + f64::from_bits(0x3c7abc9e3b39803f), + ); + let k_f64 = self.builder.ins().fcvt_from_sint(types::F64, k_i64); + let dd_ln2_k = self.dd_mul_f64(ln2_dd, k_f64); + + // Add ln(m) and k*ln2 to get the final ln(x). + let normal_result = self.dd_add(dd_ln_m, dd_ln2_k); + + // (I) If x was nonpositive or NaN, return NaN; otherwise, return the computed result. + let final_hi = self + .builder + .ins() + .select(need_nan, dd_nan.hi, normal_result.hi); + let final_lo = self + .builder + .ins() + .select(need_nan, dd_nan.lo, normal_result.lo); + + DDValue { + hi: final_hi, + lo: final_lo, + } + } + + /// Computes the exponential function exp(x) in double–double arithmetic. + /// It uses range reduction to write x = k*ln2 + r, computes exp(r) via a Taylor series, + /// scales the result by 2^k, and handles overflow by checking if k exceeds the maximum. + fn dd_exp(&mut self, dd: DDValue) -> DDValue { + // (A) Range reduction: Convert dd to a single f64 value. + let x = self.dd_to_f64(dd.clone()); + let ln2_f64 = self + .builder + .ins() + .f64const(f64::from_bits(0x3fe62e42fefa39ef)); + let div = self.builder.ins().fdiv(x, ln2_f64); + let half = self.builder.ins().f64const(0.5); + let div_plus_half = self.builder.ins().fadd(div, half); + // Rounding: floor(div + 0.5) gives the nearest integer k. + let k = self.builder.ins().fcvt_to_sint(types::I64, div_plus_half); + + // --- OVERFLOW CHECK --- + // Check if k is greater than the maximum exponent for finite doubles (1023). + let max_k = self.builder.ins().iconst(types::I64, 1023); + let is_overflow = self.builder.ins().icmp(IntCC::SignedGreaterThan, k, max_k); + + // Define infinity and zero for the overflow case. + let inf = self.builder.ins().f64const(f64::INFINITY); + let zero = self.builder.ins().f64const(0.0); + + // (B) Compute exp(x) normally when not overflowing. + // Compute k*ln2 in double–double arithmetic and subtract it from x. + let ln2_dd = self.dd_from_parts( + f64::from_bits(0x3fe62e42fefa39ef), + f64::from_bits(0x3c7abc9e3b39803f), + ); + let k_f64 = self.builder.ins().fcvt_from_sint(types::F64, k); + let k_ln2 = self.dd_mul_f64(ln2_dd, k_f64); + let r = self.dd_sub(dd, k_ln2); + + // Compute exp(r) using a Taylor series. + let mut sum = self.dd_from_f64(1.0); // Initialize sum to 1. + let mut term = self.dd_from_f64(1.0); // Initialize the first term to 1. + let n_terms = 1000; + for i in 1..=n_terms { + term = self.dd_mul(term, r.clone()); + let inv = 1.0 / (i as f64); + let inv_const = self.builder.ins().f64const(inv); + term = self.dd_mul_f64(term, inv_const); + sum = self.dd_add(sum, term.clone()); + } + + // Reconstruct the final result by scaling with 2^k. + let bias = self.builder.ins().iconst(types::I64, 1023); + let k_plus_bias = self.builder.ins().iadd(k, bias); + let shift_count = self.builder.ins().iconst(types::I64, 52); + let shifted = self.builder.ins().ishl(k_plus_bias, shift_count); + let two_to_k = self + .builder + .ins() + .bitcast(types::F64, MemFlags::new(), shifted); + let result = self.dd_scale(sum, two_to_k); + + // (C) If overflow was detected, return infinity; otherwise, return the computed value. + let final_hi = self.builder.ins().select(is_overflow, inf, result.hi); + let final_lo = self.builder.ins().select(is_overflow, zero, result.lo); + DDValue { + hi: final_hi, + lo: final_lo, + } + } + + /// Computes the power function a^b (f_pow) for f64 values using double–double arithmetic for high precision. + /// It handles different cases for the base 'a': + /// - For a > 0: Computes exp(b * ln(a)). + /// - For a == 0: Handles special cases for 0^b, including returning 0, 1, or a domain error. + /// - For a < 0: Allows only an integer exponent b and adjusts the sign if b is odd. + fn compile_fpow(&mut self, a: Value, b: Value) -> Value { + let f64_ty = types::F64; + let i64_ty = types::I64; + let zero_f = self.builder.ins().f64const(0.0); + let one_f = self.builder.ins().f64const(1.0); + let nan_f = self.builder.ins().f64const(f64::NAN); + let inf_f = self.builder.ins().f64const(f64::INFINITY); + let neg_inf_f = self.builder.ins().f64const(f64::NEG_INFINITY); + + // Merge block for final result. + let merge_block = self.builder.create_block(); + self.builder.append_block_param(merge_block, f64_ty); + + // --- Edge Case 1: b == 0.0 → return 1.0 + let cmp_b_zero = self.builder.ins().fcmp(FloatCC::Equal, b, zero_f); + let b_zero_block = self.builder.create_block(); + let continue_block = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_b_zero, b_zero_block, &[], continue_block, &[]); + self.builder.switch_to_block(b_zero_block); + self.builder.ins().jump(merge_block, &[one_f.into()]); + self.builder.switch_to_block(continue_block); + + // --- Edge Case 2: b is NaN → return NaN + let cmp_b_nan = self.builder.ins().fcmp(FloatCC::Unordered, b, b); + let b_nan_block = self.builder.create_block(); + let continue_block2 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_b_nan, b_nan_block, &[], continue_block2, &[]); + self.builder.switch_to_block(b_nan_block); + self.builder.ins().jump(merge_block, &[nan_f.into()]); + self.builder.switch_to_block(continue_block2); + + // --- Edge Case 3: a == 0.0 → return 0.0 + let cmp_a_zero = self.builder.ins().fcmp(FloatCC::Equal, a, zero_f); + let a_zero_block = self.builder.create_block(); + let continue_block3 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_a_zero, a_zero_block, &[], continue_block3, &[]); + self.builder.switch_to_block(a_zero_block); + self.builder.ins().jump(merge_block, &[zero_f.into()]); + self.builder.switch_to_block(continue_block3); + + // --- Edge Case 4: a is NaN → return NaN + let cmp_a_nan = self.builder.ins().fcmp(FloatCC::Unordered, a, a); + let a_nan_block = self.builder.create_block(); + let continue_block4 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_a_nan, a_nan_block, &[], continue_block4, &[]); + self.builder.switch_to_block(a_nan_block); + self.builder.ins().jump(merge_block, &[nan_f.into()]); + self.builder.switch_to_block(continue_block4); + + // --- Edge Case 5: b == +infinity → return +infinity + let cmp_b_inf = self.builder.ins().fcmp(FloatCC::Equal, b, inf_f); + let b_inf_block = self.builder.create_block(); + let continue_block5 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_b_inf, b_inf_block, &[], continue_block5, &[]); + self.builder.switch_to_block(b_inf_block); + self.builder.ins().jump(merge_block, &[inf_f.into()]); + self.builder.switch_to_block(continue_block5); + + // --- Edge Case 6: b == -infinity → return 0.0 + let cmp_b_neg_inf = self.builder.ins().fcmp(FloatCC::Equal, b, neg_inf_f); + let b_neg_inf_block = self.builder.create_block(); + let continue_block6 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_b_neg_inf, b_neg_inf_block, &[], continue_block6, &[]); + self.builder.switch_to_block(b_neg_inf_block); + self.builder.ins().jump(merge_block, &[zero_f.into()]); + self.builder.switch_to_block(continue_block6); + + // --- Edge Case 7: a == +infinity → return +infinity + let cmp_a_inf = self.builder.ins().fcmp(FloatCC::Equal, a, inf_f); + let a_inf_block = self.builder.create_block(); + let continue_block7 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_a_inf, a_inf_block, &[], continue_block7, &[]); + self.builder.switch_to_block(a_inf_block); + self.builder.ins().jump(merge_block, &[inf_f.into()]); + self.builder.switch_to_block(continue_block7); + + // --- Edge Case 8: a == -infinity → check exponent parity + let cmp_a_neg_inf = self.builder.ins().fcmp(FloatCC::Equal, a, neg_inf_f); + let a_neg_inf_block = self.builder.create_block(); + let continue_block8 = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_a_neg_inf, a_neg_inf_block, &[], continue_block8, &[]); + + self.builder.switch_to_block(a_neg_inf_block); + // a is -infinity here. First, ensure that b is an integer. + let b_floor = self.builder.ins().floor(b); + let cmp_int = self.builder.ins().fcmp(FloatCC::Equal, b_floor, b); + let domain_error_blk = self.builder.create_block(); + let continue_neg_inf = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_int, continue_neg_inf, &[], domain_error_blk, &[]); + + self.builder.switch_to_block(domain_error_blk); + self.builder.ins().jump(merge_block, &[nan_f.into()]); + + self.builder.switch_to_block(continue_neg_inf); + // b is an integer here; convert b_floor to an i64. + let b_i64 = self.builder.ins().fcvt_to_sint(i64_ty, b_floor); + let one_i = self.builder.ins().iconst(i64_ty, 1); + let remainder = self.builder.ins().band(b_i64, one_i); + let zero_i = self.builder.ins().iconst(i64_ty, 0); + let is_odd = self.builder.ins().icmp(IntCC::NotEqual, remainder, zero_i); + + // Create separate blocks for odd and even cases. + let odd_block = self.builder.create_block(); + let even_block = self.builder.create_block(); + self.builder.append_block_param(odd_block, f64_ty); + self.builder.append_block_param(even_block, f64_ty); + self.builder.ins().brif( + is_odd, + odd_block, + &[neg_inf_f.into()], + even_block, + &[inf_f.into()], + ); + + self.builder.switch_to_block(odd_block); + let phi_neg_inf = self.builder.block_params(odd_block)[0]; + self.builder.ins().jump(merge_block, &[phi_neg_inf.into()]); + + self.builder.switch_to_block(even_block); + let phi_inf = self.builder.block_params(even_block)[0]; + self.builder.ins().jump(merge_block, &[phi_inf.into()]); + + self.builder.switch_to_block(continue_block8); + + // --- Normal branch: neither a nor b hit the special cases. + // Here we branch based on the sign of a. + let cmp_lt = self.builder.ins().fcmp(FloatCC::LessThan, a, zero_f); + let a_neg_block = self.builder.create_block(); + let a_pos_block = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_lt, a_neg_block, &[], a_pos_block, &[]); + + // ----- Case: a > 0: Compute a^b = exp(b * ln(a)) using double–double arithmetic. + self.builder.switch_to_block(a_pos_block); + let ln_a_dd = self.dd_ln(a); + let b_dd = self.dd_from_value(b); + let product_dd = self.dd_mul(ln_a_dd, b_dd); + let exp_dd = self.dd_exp(product_dd); + let pos_res = self.dd_to_f64(exp_dd); + self.builder.ins().jump(merge_block, &[pos_res.into()]); + + // ----- Case: a < 0: Only allow an integral exponent. + self.builder.switch_to_block(a_neg_block); + let b_floor = self.builder.ins().floor(b); + let cmp_int = self.builder.ins().fcmp(FloatCC::Equal, b_floor, b); + let neg_int_block = self.builder.create_block(); + let domain_error_blk = self.builder.create_block(); + self.builder + .ins() + .brif(cmp_int, neg_int_block, &[], domain_error_blk, &[]); + + // Domain error: non-integer exponent for negative base + self.builder.switch_to_block(domain_error_blk); + self.builder.ins().jump(merge_block, &[nan_f.into()]); + + // For negative base with an integer exponent: + self.builder.switch_to_block(neg_int_block); + let abs_a = self.builder.ins().fabs(a); + let ln_abs_dd = self.dd_ln(abs_a); + let b_dd = self.dd_from_value(b); + let product_dd = self.dd_mul(ln_abs_dd, b_dd); + let exp_dd = self.dd_exp(product_dd); + let mag_val = self.dd_to_f64(exp_dd); + + let b_i64 = self.builder.ins().fcvt_to_sint(i64_ty, b_floor); + let one_i = self.builder.ins().iconst(i64_ty, 1); + let remainder = self.builder.ins().band(b_i64, one_i); + let zero_i = self.builder.ins().iconst(i64_ty, 0); + let is_odd = self.builder.ins().icmp(IntCC::NotEqual, remainder, zero_i); + + let odd_block = self.builder.create_block(); + let even_block = self.builder.create_block(); + // Append block parameters for both branches: + self.builder.append_block_param(odd_block, f64_ty); + self.builder.append_block_param(even_block, f64_ty); + // Pass mag_val to both branches: + self.builder.ins().brif( + is_odd, + odd_block, + &[mag_val.into()], + even_block, + &[mag_val.into()], + ); + + self.builder.switch_to_block(odd_block); + let phi_mag_val = self.builder.block_params(odd_block)[0]; + let neg_val = self.builder.ins().fneg(phi_mag_val); + self.builder.ins().jump(merge_block, &[neg_val.into()]); + + self.builder.switch_to_block(even_block); + let phi_mag_val_even = self.builder.block_params(even_block)[0]; + self.builder + .ins() + .jump(merge_block, &[phi_mag_val_even.into()]); + + // ----- Merge: Return the final result. + self.builder.switch_to_block(merge_block); + self.builder.block_params(merge_block)[0] + } + + fn compile_ipow(&mut self, a: Value, b: Value) -> Value { + let zero = self.builder.ins().iconst(types::I64, 0); + let one_i64 = self.builder.ins().iconst(types::I64, 1); + + // Create required blocks + let check_negative = self.builder.create_block(); + let handle_negative = self.builder.create_block(); + let loop_block = self.builder.create_block(); + let continue_block = self.builder.create_block(); + let exit_block = self.builder.create_block(); + + // Set up block parameters + self.builder.append_block_param(check_negative, types::I64); // exponent + self.builder.append_block_param(check_negative, types::I64); // base + + self.builder.append_block_param(handle_negative, types::I64); // abs(exponent) + self.builder.append_block_param(handle_negative, types::I64); // base + + self.builder.append_block_param(loop_block, types::I64); // exponent + self.builder.append_block_param(loop_block, types::I64); // result + self.builder.append_block_param(loop_block, types::I64); // base + + self.builder.append_block_param(exit_block, types::I64); // final result + + // Set up parameters for continue_block + self.builder.append_block_param(continue_block, types::I64); // exponent + self.builder.append_block_param(continue_block, types::I64); // result + self.builder.append_block_param(continue_block, types::I64); // base + + // Initial jump to check if exponent is negative + self.builder + .ins() + .jump(check_negative, &[b.into(), a.into()]); + + // Check if exponent is negative + self.builder.switch_to_block(check_negative); + let params = self.builder.block_params(check_negative); + let exp_check = params[0]; + let base_check = params[1]; + + let is_negative = self + .builder + .ins() + .icmp(IntCC::SignedLessThan, exp_check, zero); + self.builder.ins().brif( + is_negative, + handle_negative, + &[exp_check.into(), base_check.into()], + loop_block, + &[exp_check.into(), one_i64.into(), base_check.into()], + ); + + // Handle negative exponent (return 0 for integer exponentiation) + self.builder.switch_to_block(handle_negative); + self.builder.ins().jump(exit_block, &[zero.into()]); // Return 0 for negative exponents + + // Loop block logic (square-and-multiply algorithm) + self.builder.switch_to_block(loop_block); + let params = self.builder.block_params(loop_block); + let exp_phi = params[0]; + let result_phi = params[1]; + let base_phi = params[2]; + + // Check if exponent is zero + let is_zero = self.builder.ins().icmp(IntCC::Equal, exp_phi, zero); + self.builder.ins().brif( + is_zero, + exit_block, + &[result_phi.into()], + continue_block, + &[exp_phi.into(), result_phi.into(), base_phi.into()], + ); + + // Continue block for non-zero case + self.builder.switch_to_block(continue_block); + let params = self.builder.block_params(continue_block); + let exp_phi = params[0]; + let result_phi = params[1]; + let base_phi = params[2]; + + // If exponent is odd, multiply result by base + let is_odd = self.builder.ins().band_imm(exp_phi, 1); + let is_odd = self.builder.ins().icmp_imm(IntCC::Equal, is_odd, 1); + let mul_result = self.builder.ins().imul(result_phi, base_phi); + let new_result = self.builder.ins().select(is_odd, mul_result, result_phi); + + // Square the base and divide exponent by 2 + let squared_base = self.builder.ins().imul(base_phi, base_phi); + let new_exp = self.builder.ins().sshr_imm(exp_phi, 1); + self.builder.ins().jump( + loop_block, + &[new_exp.into(), new_result.into(), squared_base.into()], + ); + + // Exit block + self.builder.switch_to_block(exit_block); + let res = self.builder.block_params(exit_block)[0]; + + // Seal all blocks + self.builder.seal_block(check_negative); + self.builder.seal_block(handle_negative); + self.builder.seal_block(loop_block); + self.builder.seal_block(continue_block); + self.builder.seal_block(exit_block); + + res + } +} diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs new file mode 100644 index 00000000000..1e278617661 --- /dev/null +++ b/crates/jit/src/lib.rs @@ -0,0 +1,377 @@ +mod instructions; + +extern crate alloc; + +use alloc::fmt; +use core::mem::ManuallyDrop; +use cranelift::prelude::*; +use cranelift_jit::{JITBuilder, JITModule}; +use cranelift_module::{FuncId, Linkage, Module, ModuleError}; +use instructions::FunctionCompiler; +use rustpython_compiler_core::bytecode; + +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum JitCompileError { + #[error("function can't be jitted")] + NotSupported, + #[error("bad bytecode")] + BadBytecode, + #[error("error while compiling to machine code: {0}")] + CraneliftError(Box<ModuleError>), +} + +impl From<ModuleError> for JitCompileError { + fn from(err: ModuleError) -> Self { + Self::CraneliftError(Box::new(err)) + } +} + +#[derive(Debug, thiserror::Error, Eq, PartialEq)] +#[non_exhaustive] +pub enum JitArgumentError { + #[error("argument is of wrong type")] + ArgumentTypeMismatch, + #[error("wrong number of arguments")] + WrongNumberOfArguments, +} + +struct Jit { + builder_context: FunctionBuilderContext, + ctx: codegen::Context, + module: JITModule, +} + +impl Jit { + fn new() -> Self { + let builder = JITBuilder::new(cranelift_module::default_libcall_names()) + .expect("Failed to build JITBuilder"); + let module = JITModule::new(builder); + Self { + builder_context: FunctionBuilderContext::new(), + ctx: module.make_context(), + module, + } + } + + fn build_function<C: bytecode::Constant>( + &mut self, + bytecode: &bytecode::CodeObject<C>, + args: &[JitType], + ret: Option<JitType>, + ) -> Result<(FuncId, JitSig), JitCompileError> { + for arg in args { + self.ctx + .func + .signature + .params + .push(AbiParam::new(arg.to_cranelift())); + } + + if ret.is_some() { + self.ctx + .func + .signature + .returns + .push(AbiParam::new(ret.clone().unwrap().to_cranelift())); + } + + let id = self.module.declare_function( + &format!("jit_{}", bytecode.obj_name.as_ref()), + Linkage::Export, + &self.ctx.func.signature, + )?; + + let func_ref = self.module.declare_func_in_func(id, &mut self.ctx.func); + + let mut builder = FunctionBuilder::new(&mut self.ctx.func, &mut self.builder_context); + let entry_block = builder.create_block(); + builder.append_block_params_for_function_params(entry_block); + builder.switch_to_block(entry_block); + + let sig = { + let mut compiler = FunctionCompiler::new( + &mut builder, + bytecode.varnames.len(), + args, + ret, + entry_block, + ); + + compiler.compile(func_ref, bytecode)?; + + compiler.sig + }; + + builder.seal_all_blocks(); + builder.finalize(); + + self.module.define_function(id, &mut self.ctx)?; + + self.module.clear_context(&mut self.ctx); + + Ok((id, sig)) + } +} + +pub fn compile<C: bytecode::Constant>( + bytecode: &bytecode::CodeObject<C>, + args: &[JitType], + ret: Option<JitType>, +) -> Result<CompiledCode, JitCompileError> { + let mut jit = Jit::new(); + + let (id, sig) = jit.build_function(bytecode, args, ret)?; + + jit.module.finalize_definitions()?; + + let code = jit.module.get_finalized_function(id); + Ok(CompiledCode { + sig, + code, + module: ManuallyDrop::new(jit.module), + }) +} + +pub struct CompiledCode { + sig: JitSig, + code: *const u8, + module: ManuallyDrop<JITModule>, +} + +impl CompiledCode { + pub fn args_builder(&self) -> ArgsBuilder<'_> { + ArgsBuilder::new(self) + } + + pub fn invoke(&self, args: &[AbiValue]) -> Result<Option<AbiValue>, JitArgumentError> { + if self.sig.args.len() != args.len() { + return Err(JitArgumentError::WrongNumberOfArguments); + } + + let cif_args = self + .sig + .args + .iter() + .zip(args.iter()) + .map(|(ty, val)| type_check(ty, val).map(|_| val)) + .map(|v| v.map(AbiValue::to_libffi_arg)) + .collect::<Result<Vec<_>, _>>()?; + Ok(unsafe { self.invoke_raw(&cif_args) }) + } + + unsafe fn invoke_raw(&self, cif_args: &[libffi::middle::Arg<'_>]) -> Option<AbiValue> { + unsafe { + let cif = self.sig.to_cif(); + let value = cif.call::<UnTypedAbiValue>( + libffi::middle::CodePtr::from_ptr(self.code as *const _), + cif_args, + ); + self.sig.ret.as_ref().map(|ty| value.to_typed(ty)) + } + } +} + +struct JitSig { + args: Vec<JitType>, + ret: Option<JitType>, +} + +impl JitSig { + fn to_cif(&self) -> libffi::middle::Cif { + let ret = match self.ret { + Some(ref ty) => ty.to_libffi(), + None => libffi::middle::Type::void(), + }; + libffi::middle::Cif::new(self.args.iter().map(JitType::to_libffi), ret) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum JitType { + Int, + Float, + Bool, +} + +impl JitType { + fn to_cranelift(&self) -> types::Type { + match self { + Self::Int => types::I64, + Self::Float => types::F64, + Self::Bool => types::I8, + } + } + + fn to_libffi(&self) -> libffi::middle::Type { + match self { + Self::Int => libffi::middle::Type::i64(), + Self::Float => libffi::middle::Type::f64(), + Self::Bool => libffi::middle::Type::u8(), + } + } +} + +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum AbiValue { + Float(f64), + Int(i64), + Bool(bool), +} + +impl AbiValue { + fn to_libffi_arg(&self) -> libffi::middle::Arg<'_> { + match self { + AbiValue::Int(i) => libffi::middle::Arg::new(i), + AbiValue::Float(f) => libffi::middle::Arg::new(f), + AbiValue::Bool(b) => libffi::middle::Arg::new(b), + } + } +} + +impl From<i64> for AbiValue { + fn from(i: i64) -> Self { + AbiValue::Int(i) + } +} + +impl From<f64> for AbiValue { + fn from(f: f64) -> Self { + AbiValue::Float(f) + } +} + +impl From<bool> for AbiValue { + fn from(b: bool) -> Self { + AbiValue::Bool(b) + } +} + +impl TryFrom<AbiValue> for i64 { + type Error = (); + + fn try_from(value: AbiValue) -> Result<Self, Self::Error> { + match value { + AbiValue::Int(i) => Ok(i), + _ => Err(()), + } + } +} + +impl TryFrom<AbiValue> for f64 { + type Error = (); + + fn try_from(value: AbiValue) -> Result<Self, Self::Error> { + match value { + AbiValue::Float(f) => Ok(f), + _ => Err(()), + } + } +} + +impl TryFrom<AbiValue> for bool { + type Error = (); + + fn try_from(value: AbiValue) -> Result<Self, Self::Error> { + match value { + AbiValue::Bool(b) => Ok(b), + _ => Err(()), + } + } +} + +fn type_check(ty: &JitType, val: &AbiValue) -> Result<(), JitArgumentError> { + match (ty, val) { + (JitType::Int, AbiValue::Int(_)) + | (JitType::Float, AbiValue::Float(_)) + | (JitType::Bool, AbiValue::Bool(_)) => Ok(()), + _ => Err(JitArgumentError::ArgumentTypeMismatch), + } +} + +#[derive(Copy, Clone)] +union UnTypedAbiValue { + float: f64, + int: i64, + boolean: u8, + _void: (), +} + +impl UnTypedAbiValue { + unsafe fn to_typed(self, ty: &JitType) -> AbiValue { + unsafe { + match ty { + JitType::Int => AbiValue::Int(self.int), + JitType::Float => AbiValue::Float(self.float), + JitType::Bool => AbiValue::Bool(self.boolean != 0), + } + } + } +} + +// we don't actually ever touch CompiledCode til we drop it, it should be safe. +// TODO: confirm with wasmtime ppl that it's not unsound? +unsafe impl Send for CompiledCode {} +unsafe impl Sync for CompiledCode {} + +impl Drop for CompiledCode { + fn drop(&mut self) { + // SAFETY: The only pointer that this memory will also be dropped now + unsafe { ManuallyDrop::take(&mut self.module).free_memory() } + } +} + +impl fmt::Debug for CompiledCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("[compiled code]") + } +} + +pub struct ArgsBuilder<'a> { + values: Vec<Option<AbiValue>>, + code: &'a CompiledCode, +} + +impl<'a> ArgsBuilder<'a> { + fn new(code: &'a CompiledCode) -> ArgsBuilder<'a> { + ArgsBuilder { + values: vec![None; code.sig.args.len()], + code, + } + } + + pub fn set(&mut self, idx: usize, value: AbiValue) -> Result<(), JitArgumentError> { + type_check(&self.code.sig.args[idx], &value).map(|_| { + self.values[idx] = Some(value); + }) + } + + pub fn is_set(&self, idx: usize) -> bool { + self.values[idx].is_some() + } + + pub fn into_args(self) -> Option<Args<'a>> { + // Ensure all values are set + if self.values.iter().any(|v| v.is_none()) { + return None; + } + Some(Args { + values: self.values.into_iter().map(|v| v.unwrap()).collect(), + code: self.code, + }) + } +} + +pub struct Args<'a> { + values: Vec<AbiValue>, + code: &'a CompiledCode, +} + +impl Args<'_> { + pub fn invoke(&self) -> Option<AbiValue> { + let cif_args: Vec<_> = self.values.iter().map(AbiValue::to_libffi_arg).collect(); + unsafe { self.code.invoke_raw(&cif_args) } + } +} diff --git a/jit/tests/bool_tests.rs b/crates/jit/tests/bool_tests.rs similarity index 100% rename from jit/tests/bool_tests.rs rename to crates/jit/tests/bool_tests.rs diff --git a/crates/jit/tests/common.rs b/crates/jit/tests/common.rs new file mode 100644 index 00000000000..629cdccc7fd --- /dev/null +++ b/crates/jit/tests/common.rs @@ -0,0 +1,342 @@ +use core::ops::ControlFlow; +use rustpython_compiler_core::bytecode::{ + CodeObject, ConstantData, Constants, Instruction, OpArg, OpArgState, +}; +use rustpython_jit::{CompiledCode, JitType}; +use rustpython_wtf8::{Wtf8, Wtf8Buf}; +use std::collections::HashMap; + +#[derive(Debug, Clone)] +pub struct Function { + code: Box<CodeObject>, + annotations: HashMap<Wtf8Buf, StackValue>, +} + +impl Function { + pub fn compile(self) -> CompiledCode { + let mut arg_types = Vec::new(); + for arg in self.code.arg_names().args { + let arg_type = match self.annotations.get(AsRef::<Wtf8>::as_ref(arg.as_str())) { + Some(StackValue::String(annotation)) => match annotation.as_str() { + "int" => JitType::Int, + "float" => JitType::Float, + "bool" => JitType::Bool, + _ => panic!("Unrecognised jit type"), + }, + _ => panic!("Argument have annotation"), + }; + arg_types.push(arg_type); + } + + let ret_type = match self.annotations.get(AsRef::<Wtf8>::as_ref("return")) { + Some(StackValue::String(annotation)) => match annotation.as_str() { + "int" => Some(JitType::Int), + "float" => Some(JitType::Float), + "bool" => Some(JitType::Bool), + _ => panic!("Unrecognised jit type"), + }, + _ => None, + }; + + rustpython_jit::compile(&self.code, &arg_types, ret_type).expect("Compile failure") + } +} + +#[derive(Debug, Clone)] +enum StackValue { + String(String), + None, + Map(HashMap<Wtf8Buf, StackValue>), + Code(Box<CodeObject>), + Function(Function), +} + +impl From<ConstantData> for StackValue { + fn from(value: ConstantData) -> Self { + match value { + ConstantData::Str { value } => { + StackValue::String(value.into_string().expect("surrogate in test code")) + } + ConstantData::None => StackValue::None, + ConstantData::Code { code } => StackValue::Code(code), + c => unimplemented!("constant {:?} isn't yet supported in py_function!", c), + } + } +} + +/// Extract annotations from an annotate function's bytecode. +/// The annotate function uses BUILD_MAP with key-value pairs loaded before it. +/// Keys are parameter names (from LOAD_CONST), values are type names (from LOAD_NAME/LOAD_GLOBAL). +fn extract_annotations_from_annotate_code(code: &CodeObject) -> HashMap<Wtf8Buf, StackValue> { + let mut annotations = HashMap::new(); + let mut stack: Vec<(bool, usize)> = Vec::new(); // (is_const, index) + let mut op_arg_state = OpArgState::default(); + + for &word in code.instructions.iter() { + let (instruction, arg) = op_arg_state.get(word); + + match instruction { + Instruction::LoadConst { consti } => { + stack.push((true, consti.get(arg).as_usize())); + } + Instruction::LoadName { namei } => { + stack.push((false, namei.get(arg) as usize)); + } + Instruction::LoadGlobal { namei } => { + stack.push((false, (namei.get(arg) >> 1) as usize)); + } + Instruction::BuildMap { count } => { + let count = count.get(arg) as usize; + // Stack has key-value pairs in order: k1, v1, k2, v2, ... + // So we need count * 2 items from the stack + let start = stack.len().saturating_sub(count * 2); + let pairs: Vec<_> = stack.drain(start..).collect(); + + for chunk in pairs.chunks(2) { + if chunk.len() == 2 { + let (key_is_const, key_idx) = chunk[0]; + let (val_is_const, val_idx) = chunk[1]; + + // Key should be a const string (parameter name) + if key_is_const + && let ConstantData::Str { value } = + &code.constants[(key_idx as u32).into()] + { + let param_name = value; + // Value can be a name (type ref) or a const string (forward ref) + let type_name = if val_is_const { + match code.constants.get(val_idx) { + Some(ConstantData::Str { value }) => value + .as_str() + .map(|s| s.to_owned()) + .unwrap_or_else(|_| value.to_string_lossy().into_owned()), + Some(other) => panic!( + "Unsupported annotation const for '{:?}' at idx {}: {:?}", + param_name, val_idx, other + ), + None => panic!( + "Annotation const idx out of bounds for '{:?}': {} (len={})", + param_name, + val_idx, + code.constants.len() + ), + } + } else { + match code.names.get(val_idx) { + Some(name) => name.clone(), + None => panic!( + "Annotation name idx out of bounds for '{:?}': {} (len={})", + param_name, + val_idx, + code.names.len() + ), + } + }; + annotations.insert(param_name.clone(), StackValue::String(type_name)); + } + } + } + // Return after processing BUILD_MAP - we got our annotations + return annotations; + } + Instruction::Resume { .. } + | Instruction::LoadFast { .. } + | Instruction::CompareOp { .. } + | Instruction::ExtendedArg + | Instruction::Cache + | Instruction::NotTaken => { + // Ignore these instructions for annotation extraction + } + Instruction::ReturnValue => { + // End of function - return what we have + return annotations; + } + _ => { + // For other instructions, clear the stack tracking as we don't understand the effect + stack.clear(); + } + } + } + + annotations +} + +pub struct StackMachine { + stack: Vec<StackValue>, + locals: HashMap<String, StackValue>, +} + +impl StackMachine { + pub fn new() -> StackMachine { + StackMachine { + stack: Vec::new(), + locals: HashMap::new(), + } + } + + pub fn run(&mut self, code: CodeObject) { + let mut op_arg_state = OpArgState::default(); + let _ = code.instructions.iter().try_for_each(|&word| { + let (instruction, arg) = op_arg_state.get(word); + self.process_instruction(instruction, arg, &code.constants, &code.names) + }); + } + + fn process_instruction( + &mut self, + instruction: Instruction, + arg: OpArg, + constants: &Constants<ConstantData>, + names: &[String], + ) -> ControlFlow<()> { + match instruction { + Instruction::Resume { .. } | Instruction::Cache | Instruction::NotTaken => { + // No-op for JIT tests + } + Instruction::LoadConst { consti } => { + self.stack.push(constants[consti.get(arg)].clone().into()) + } + Instruction::LoadName { namei } => self + .stack + .push(StackValue::String(names[namei.get(arg) as usize].clone())), + Instruction::StoreName { namei } => { + let idx = namei.get(arg); + self.locals + .insert(names[idx as usize].clone(), self.stack.pop().unwrap()); + } + Instruction::StoreAttr { .. } => { + // Do nothing except throw away the stack values + self.stack.pop().unwrap(); + self.stack.pop().unwrap(); + } + Instruction::BuildMap { count } => { + let mut map = HashMap::new(); + for _ in 0..count.get(arg) { + let value = self.stack.pop().unwrap(); + let name = if let Some(StackValue::String(name)) = self.stack.pop() { + Wtf8Buf::from(name) + } else { + unimplemented!("no string keys isn't yet supported in py_function!") + }; + map.insert(name, value); + } + self.stack.push(StackValue::Map(map)); + } + Instruction::MakeFunction => { + let code = if let Some(StackValue::Code(code)) = self.stack.pop() { + code + } else { + panic!("Expected function code") + }; + // Other attributes will be set by SET_FUNCTION_ATTRIBUTE + self.stack.push(StackValue::Function(Function { + code, + annotations: HashMap::new(), // empty annotations, will be set later if needed + })); + } + Instruction::SetFunctionAttribute { flag } => { + // Stack: [..., attr_value, func] -> [..., func] + let func = if let Some(StackValue::Function(func)) = self.stack.pop() { + func + } else { + panic!("Expected function on stack for SET_FUNCTION_ATTRIBUTE") + }; + let attr_value = self.stack.pop().expect("Expected attribute value on stack"); + + let flag_value = flag.get(arg); + + match flag_value { + rustpython_compiler_core::bytecode::MakeFunctionFlag::Annotate => { + // Handle ANNOTATE flag (PEP 649 style - Python 3.14+) + if let StackValue::Function(annotate_func) = attr_value { + let annotate_code = &annotate_func.code; + let annotations = extract_annotations_from_annotate_code(annotate_code); + + let updated_func = Function { + code: func.code, + annotations, + }; + self.stack.push(StackValue::Function(updated_func)); + } else { + panic!("Expected annotate function for ANNOTATE flag"); + } + } + rustpython_compiler_core::bytecode::MakeFunctionFlag::Annotations => { + // Handle old ANNOTATIONS flag (Python 3.12 style) + if let StackValue::Map(annotations) = attr_value { + let updated_func = Function { + code: func.code, + annotations, + }; + self.stack.push(StackValue::Function(updated_func)); + } else { + panic!("Expected annotations to be a map"); + } + } + _ => { + // For other attributes, just push the function back unchanged + self.stack.push(StackValue::Function(func)); + } + } + } + Instruction::ReturnValue => return ControlFlow::Break(()), + Instruction::ExtendedArg => {} + _ => unimplemented!( + "instruction {:?} isn't yet supported in py_function!", + instruction + ), + } + ControlFlow::Continue(()) + } + + pub fn get_function(&self, name: &str) -> Function { + if let Some(StackValue::Function(function)) = self.locals.get(name) { + function.clone() + } else { + panic!("There was no function named {name}") + } + } +} + +macro_rules! jit_function { + ($func_name:ident => $($t:tt)*) => { + { + let code = rustpython_derive::py_compile!( + crate_name = "rustpython_compiler_core", + source = $($t)* + ); + let code = code.decode(rustpython_compiler_core::bytecode::BasicBag); + let mut machine = $crate::common::StackMachine::new(); + machine.run(code); + machine.get_function(stringify!($func_name)).compile() + } + }; + ($func_name:ident($($arg_name:ident:$arg_type:ty),*) -> $ret_type:ty => $($t:tt)*) => { + { + let jit_code = jit_function!($func_name => $($t)*); + + move |$($arg_name:$arg_type),*| -> Result<$ret_type, rustpython_jit::JitArgumentError> { + jit_code + .invoke(&[$($arg_name.into()),*]) + .map(|ret| match ret { + Some(ret) => ret.try_into().expect("jit function returned unexpected type"), + None => panic!("jit function unexpectedly returned None") + }) + } + } + }; + ($func_name:ident($($arg_name:ident:$arg_type:ty),*) => $($t:tt)*) => { + { + let jit_code = jit_function!($func_name => $($t)*); + + move |$($arg_name:$arg_type),*| -> Result<(), rustpython_jit::JitArgumentError> { + jit_code + .invoke(&[$($arg_name.into()),*]) + .map(|ret| match ret { + Some(ret) => panic!("jit function unexpectedly returned a value {:?}", ret), + None => () + }) + } + } + }; +} diff --git a/crates/jit/tests/float_tests.rs b/crates/jit/tests/float_tests.rs new file mode 100644 index 00000000000..b5fcba9fc6a --- /dev/null +++ b/crates/jit/tests/float_tests.rs @@ -0,0 +1,379 @@ +macro_rules! assert_approx_eq { + ($left:expr, $right:expr) => { + match ($left, $right) { + (Ok(lhs), Ok(rhs)) => approx::assert_relative_eq!(lhs, rhs), + (lhs, rhs) => assert_eq!(lhs, rhs), + } + }; +} + +macro_rules! assert_bits_eq { + ($left:expr, $right:expr) => { + match ($left, $right) { + (Ok(lhs), Ok(rhs)) => assert!(lhs.to_bits() == rhs.to_bits()), + (lhs, rhs) => assert_eq!(lhs, rhs), + } + }; +} + +#[test] +fn test_add() { + let add = jit_function! { add(a:f64, b:f64) -> f64 => r##" + def add(a: float, b: float): + return a + b + "## }; + + assert_approx_eq!(add(5.5, 10.2), Ok(15.7)); + assert_approx_eq!(add(-4.5, 7.6), Ok(3.1)); + assert_approx_eq!(add(-5.2, -3.9), Ok(-9.1)); + assert_bits_eq!(add(-5.2, f64::NAN), Ok(f64::NAN)); + assert_eq!(add(2.0, f64::INFINITY), Ok(f64::INFINITY)); + assert_eq!(add(-2.0, f64::NEG_INFINITY), Ok(f64::NEG_INFINITY)); + assert_eq!(add(1.0, f64::NEG_INFINITY), Ok(f64::NEG_INFINITY)); +} + +#[test] +fn test_add_with_integer() { + let add = jit_function! { add(a:f64, b:i64) -> f64 => r##" + def add(a: float, b: int): + return a + b + "## }; + + assert_approx_eq!(add(5.5, 10), Ok(15.5)); + assert_approx_eq!(add(-4.6, 7), Ok(2.4)); + assert_approx_eq!(add(-5.2, -3), Ok(-8.2)); +} + +#[test] +fn test_sub() { + let sub = jit_function! { sub(a:f64, b:f64) -> f64 => r##" + def sub(a: float, b: float): + return a - b + "## }; + + assert_approx_eq!(sub(5.2, 3.6), Ok(1.6)); + assert_approx_eq!(sub(3.4, 4.2), Ok(-0.8)); + assert_approx_eq!(sub(-2.1, 1.3), Ok(-3.4)); + assert_approx_eq!(sub(3.1, -1.3), Ok(4.4)); + assert_bits_eq!(sub(-5.2, f64::NAN), Ok(f64::NAN)); + assert_eq!(sub(f64::INFINITY, 2.0), Ok(f64::INFINITY)); + assert_eq!(sub(-2.0, f64::NEG_INFINITY), Ok(f64::INFINITY)); + assert_eq!(sub(1.0, f64::INFINITY), Ok(f64::NEG_INFINITY)); +} + +#[test] +fn test_sub_with_integer() { + let sub = jit_function! { sub(a:i64, b:f64) -> f64 => r##" + def sub(a: int, b: float): + return a - b + "## }; + + assert_approx_eq!(sub(5, 3.6), Ok(1.4)); + assert_approx_eq!(sub(3, -4.2), Ok(7.2)); + assert_approx_eq!(sub(-2, 1.3), Ok(-3.3)); + assert_approx_eq!(sub(-3, -1.3), Ok(-1.7)); +} + +#[test] +fn test_mul() { + let mul = jit_function! { mul(a:f64, b:f64) -> f64 => r##" + def mul(a: float, b: float): + return a * b + "## }; + + assert_approx_eq!(mul(5.2, 2.0), Ok(10.4)); + assert_approx_eq!(mul(3.4, -1.7), Ok(-5.779999999999999)); + assert_bits_eq!(mul(1.0, 0.0), Ok(0.0f64)); + assert_bits_eq!(mul(1.0, -0.0), Ok(-0.0f64)); + assert_bits_eq!(mul(-1.0, 0.0), Ok(-0.0f64)); + assert_bits_eq!(mul(-1.0, -0.0), Ok(0.0f64)); + assert_bits_eq!(mul(-5.2, f64::NAN), Ok(f64::NAN)); + assert_eq!(mul(1.0, f64::INFINITY), Ok(f64::INFINITY)); + assert_eq!(mul(1.0, f64::NEG_INFINITY), Ok(f64::NEG_INFINITY)); + assert_eq!(mul(-1.0, f64::INFINITY), Ok(f64::NEG_INFINITY)); + assert!(mul(0.0, f64::INFINITY).unwrap().is_nan()); + assert_eq!(mul(f64::NEG_INFINITY, f64::INFINITY), Ok(f64::NEG_INFINITY)); +} + +#[test] +fn test_mul_with_integer() { + let mul = jit_function! { mul(a:f64, b:i64) -> f64 => r##" + def mul(a: float, b: int): + return a * b + "## }; + + assert_approx_eq!(mul(5.2, 2), Ok(10.4)); + assert_approx_eq!(mul(3.4, -1), Ok(-3.4)); + assert_bits_eq!(mul(1.0, 0), Ok(0.0f64)); + assert_bits_eq!(mul(-0.0, 1), Ok(-0.0f64)); + assert_bits_eq!(mul(0.0, -1), Ok(-0.0f64)); + assert_bits_eq!(mul(-0.0, -1), Ok(0.0f64)); +} + +#[test] +fn test_power() { + let pow = jit_function! { pow(a:f64, b:f64) -> f64 => r##" + def pow(a:float, b: float): + return a**b + "##}; + // Test base cases + assert_approx_eq!(pow(0.0, 0.0), Ok(1.0)); + assert_approx_eq!(pow(0.0, 1.0), Ok(0.0)); + assert_approx_eq!(pow(1.0, 0.0), Ok(1.0)); + assert_approx_eq!(pow(1.0, 1.0), Ok(1.0)); + assert_approx_eq!(pow(1.0, -1.0), Ok(1.0)); + assert_approx_eq!(pow(-1.0, 0.0), Ok(1.0)); + assert_approx_eq!(pow(-1.0, 1.0), Ok(-1.0)); + assert_approx_eq!(pow(-1.0, -1.0), Ok(-1.0)); + + // NaN and Infinity cases + assert_approx_eq!(pow(f64::NAN, 0.0), Ok(1.0)); + //assert_approx_eq!(pow(f64::NAN, 1.0), Ok(f64::NAN)); // Return the correct answer but fails compare + //assert_approx_eq!(pow(0.0, f64::NAN), Ok(f64::NAN)); // Return the correct answer but fails compare + assert_approx_eq!(pow(f64::INFINITY, 0.0), Ok(1.0)); + assert_approx_eq!(pow(f64::INFINITY, 1.0), Ok(f64::INFINITY)); + assert_approx_eq!(pow(f64::INFINITY, f64::INFINITY), Ok(f64::INFINITY)); + // Negative infinity cases: + // For any exponent of 0.0, the result is 1.0. + assert_approx_eq!(pow(f64::NEG_INFINITY, 0.0), Ok(1.0)); + // For negative infinity base, when b is an odd integer, result is -infinity; + // when b is even, result is +infinity. + assert_approx_eq!(pow(f64::NEG_INFINITY, 1.0), Ok(f64::NEG_INFINITY)); + assert_approx_eq!(pow(f64::NEG_INFINITY, 2.0), Ok(f64::INFINITY)); + assert_approx_eq!(pow(f64::NEG_INFINITY, 3.0), Ok(f64::NEG_INFINITY)); + // Exponent -infinity gives 0.0. + assert_approx_eq!(pow(f64::NEG_INFINITY, f64::NEG_INFINITY), Ok(0.0)); + + // Test positive float base, positive float exponent + assert_approx_eq!(pow(2.0, 2.0), Ok(4.0)); + assert_approx_eq!(pow(3.0, 3.0), Ok(27.0)); + assert_approx_eq!(pow(4.0, 4.0), Ok(256.0)); + assert_approx_eq!(pow(2.0, 3.0), Ok(8.0)); + assert_approx_eq!(pow(2.0, 4.0), Ok(16.0)); + // Test negative float base, positive float exponent (integral exponents only) + assert_approx_eq!(pow(-2.0, 2.0), Ok(4.0)); + assert_approx_eq!(pow(-3.0, 3.0), Ok(-27.0)); + assert_approx_eq!(pow(-4.0, 4.0), Ok(256.0)); + assert_approx_eq!(pow(-2.0, 3.0), Ok(-8.0)); + assert_approx_eq!(pow(-2.0, 4.0), Ok(16.0)); + // Test positive float base, positive float exponent + assert_approx_eq!(pow(2.5, 2.0), Ok(6.25)); + assert_approx_eq!(pow(3.5, 3.0), Ok(42.875)); + assert_approx_eq!(pow(4.5, 4.0), Ok(410.0625)); + assert_approx_eq!(pow(2.5, 3.0), Ok(15.625)); + assert_approx_eq!(pow(2.5, 4.0), Ok(39.0625)); + // Test negative float base, positive float exponent (integral exponents only) + assert_approx_eq!(pow(-2.5, 2.0), Ok(6.25)); + assert_approx_eq!(pow(-3.5, 3.0), Ok(-42.875)); + assert_approx_eq!(pow(-4.5, 4.0), Ok(410.0625)); + assert_approx_eq!(pow(-2.5, 3.0), Ok(-15.625)); + assert_approx_eq!(pow(-2.5, 4.0), Ok(39.0625)); + // Test positive float base, positive float exponent with non-integral exponents + assert_approx_eq!(pow(2.0, 2.5), Ok(5.656854249492381)); + assert_approx_eq!(pow(3.0, 3.5), Ok(46.76537180435969)); + assert_approx_eq!(pow(4.0, 4.5), Ok(512.0)); + assert_approx_eq!(pow(2.0, 3.5), Ok(11.313708498984761)); + assert_approx_eq!(pow(2.0, 4.5), Ok(22.627416997969522)); + // Test positive float base, negative float exponent + assert_approx_eq!(pow(2.0, -2.5), Ok(0.1767766952966369)); + assert_approx_eq!(pow(3.0, -3.5), Ok(0.021383343303319473)); + assert_approx_eq!(pow(4.0, -4.5), Ok(0.001953125)); + assert_approx_eq!(pow(2.0, -3.5), Ok(0.08838834764831845)); + assert_approx_eq!(pow(2.0, -4.5), Ok(0.04419417382415922)); + // Test negative float base, negative float exponent (integral exponents only) + assert_approx_eq!(pow(-2.0, -2.0), Ok(0.25)); + assert_approx_eq!(pow(-3.0, -3.0), Ok(-0.037037037037037035)); + assert_approx_eq!(pow(-4.0, -4.0), Ok(0.00390625)); + assert_approx_eq!(pow(-2.0, -3.0), Ok(-0.125)); + assert_approx_eq!(pow(-2.0, -4.0), Ok(0.0625)); + + // Currently negative float base with non-integral exponent is not supported: + // assert_approx_eq!(pow(-2.0, 2.5), Ok(5.656854249492381)); + // assert_approx_eq!(pow(-3.0, 3.5), Ok(-46.76537180435969)); + // assert_approx_eq!(pow(-4.0, 4.5), Ok(512.0)); + // assert_approx_eq!(pow(-2.0, -2.5), Ok(0.1767766952966369)); + // assert_approx_eq!(pow(-3.0, -3.5), Ok(0.021383343303319473)); + // assert_approx_eq!(pow(-4.0, -4.5), Ok(0.001953125)); + + // Extra cases **NOTE** these are not all working: + // * If they are commented in then they work + // * If they are commented out with a number that is the current return value it throws vs the expected value + // * If they are commented out with a "fail to run" that means I couldn't get them to work, could add a case for really big or small values + // 1e308^2.0 + assert_approx_eq!(pow(1e308, 2.0), Ok(f64::INFINITY)); + // 1e308^(1e-2) + assert_approx_eq!(pow(1e308, 1e-2), Ok(1202.2644346174131)); + // 1e-308^2.0 + //assert_approx_eq!(pow(1e-308, 2.0), Ok(0.0)); // --8.403311421507407 + // 1e-308^-2.0 + assert_approx_eq!(pow(1e-308, -2.0), Ok(f64::INFINITY)); + // 1e100^(1e50) + //assert_approx_eq!(pow(1e100, 1e50), Ok(1.0000000000000002e+150)); // fail to run (Crashes as "illegal hardware instruction") + // 1e50^(1e-100) + assert_approx_eq!(pow(1e50, 1e-100), Ok(1.0)); + // 1e308^(-1e2) + //assert_approx_eq!(pow(1e308, -1e2), Ok(0.0)); // 2.961801792837933e25 + // 1e-308^(1e2) + //assert_approx_eq!(pow(1e-308, 1e2), Ok(f64::INFINITY)); // 1.6692559244043896e46 + // 1e308^(-1e308) + // assert_approx_eq!(pow(1e308, -1e308), Ok(0.0)); // fail to run (Crashes as "illegal hardware instruction") + // 1e-308^(1e308) + // assert_approx_eq!(pow(1e-308, 1e308), Ok(0.0)); // fail to run (Crashes as "illegal hardware instruction") +} + +#[test] +fn test_div() { + let div = jit_function! { div(a:f64, b:f64) -> f64 => r##" + def div(a: float, b: float): + return a / b + "## }; + + assert_approx_eq!(div(5.2, 2.0), Ok(2.6)); + assert_approx_eq!(div(3.4, -1.7), Ok(-2.0)); + assert_eq!(div(1.0, 0.0), Ok(f64::INFINITY)); + assert_eq!(div(1.0, -0.0), Ok(f64::NEG_INFINITY)); + assert_eq!(div(-1.0, 0.0), Ok(f64::NEG_INFINITY)); + assert_eq!(div(-1.0, -0.0), Ok(f64::INFINITY)); + assert_bits_eq!(div(-5.2, f64::NAN), Ok(f64::NAN)); + assert_eq!(div(f64::INFINITY, 2.0), Ok(f64::INFINITY)); + assert_bits_eq!(div(-2.0, f64::NEG_INFINITY), Ok(0.0f64)); + assert_bits_eq!(div(1.0, f64::INFINITY), Ok(0.0f64)); + assert_bits_eq!(div(2.0, f64::NEG_INFINITY), Ok(-0.0f64)); + assert_bits_eq!(div(-1.0, f64::INFINITY), Ok(-0.0f64)); +} + +#[test] +fn test_div_with_integer() { + let div = jit_function! { div(a:f64, b:i64) -> f64 => r##" + def div(a: float, b: int): + return a / b + "## }; + + assert_approx_eq!(div(5.2, 2), Ok(2.6)); + assert_approx_eq!(div(3.4, -1), Ok(-3.4)); + assert_eq!(div(1.0, 0), Ok(f64::INFINITY)); + assert_eq!(div(1.0, -0), Ok(f64::INFINITY)); + assert_eq!(div(-1.0, 0), Ok(f64::NEG_INFINITY)); + assert_eq!(div(-1.0, -0), Ok(f64::NEG_INFINITY)); + assert_eq!(div(f64::INFINITY, 2), Ok(f64::INFINITY)); + assert_eq!(div(f64::NEG_INFINITY, 3), Ok(f64::NEG_INFINITY)); +} + +#[test] +fn test_if_bool() { + let if_bool = jit_function! { if_bool(a:f64) -> i64 => r##" + def if_bool(a: float): + if a: + return 1 + return 0 + "## }; + + assert_eq!(if_bool(5.2), Ok(1)); + assert_eq!(if_bool(-3.4), Ok(1)); + assert_eq!(if_bool(f64::NAN), Ok(1)); + assert_eq!(if_bool(f64::INFINITY), Ok(1)); + + assert_eq!(if_bool(0.0), Ok(0)); +} + +#[test] +fn test_float_eq() { + let float_eq = jit_function! { float_eq(a: f64, b: f64) -> bool => r##" + def float_eq(a: float, b: float): + return a == b + "## }; + + assert_eq!(float_eq(2.0, 2.0), Ok(true)); + assert_eq!(float_eq(3.4, -1.7), Ok(false)); + assert_eq!(float_eq(0.0, 0.0), Ok(true)); + assert_eq!(float_eq(-0.0, -0.0), Ok(true)); + assert_eq!(float_eq(-0.0, 0.0), Ok(true)); + assert_eq!(float_eq(-5.2, f64::NAN), Ok(false)); + assert_eq!(float_eq(f64::NAN, f64::NAN), Ok(false)); + assert_eq!(float_eq(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); +} + +#[test] +fn test_float_ne() { + let float_ne = jit_function! { float_ne(a: f64, b: f64) -> bool => r##" + def float_ne(a: float, b: float): + return a != b + "## }; + + assert_eq!(float_ne(2.0, 2.0), Ok(false)); + assert_eq!(float_ne(3.4, -1.7), Ok(true)); + assert_eq!(float_ne(0.0, 0.0), Ok(false)); + assert_eq!(float_ne(-0.0, -0.0), Ok(false)); + assert_eq!(float_ne(-0.0, 0.0), Ok(false)); + assert_eq!(float_ne(-5.2, f64::NAN), Ok(true)); + assert_eq!(float_ne(f64::NAN, f64::NAN), Ok(true)); + assert_eq!(float_ne(f64::INFINITY, f64::NEG_INFINITY), Ok(true)); +} + +#[test] +fn test_float_gt() { + let float_gt = jit_function! { float_gt(a: f64, b: f64) -> bool => r##" + def float_gt(a: float, b: float): + return a > b + "## }; + + assert_eq!(float_gt(2.0, 2.0), Ok(false)); + assert_eq!(float_gt(3.4, -1.7), Ok(true)); + assert_eq!(float_gt(0.0, 0.0), Ok(false)); + assert_eq!(float_gt(-0.0, -0.0), Ok(false)); + assert_eq!(float_gt(-0.0, 0.0), Ok(false)); + assert_eq!(float_gt(-5.2, f64::NAN), Ok(false)); + assert_eq!(float_gt(f64::NAN, f64::NAN), Ok(false)); + assert_eq!(float_gt(f64::INFINITY, f64::NEG_INFINITY), Ok(true)); +} + +#[test] +fn test_float_gte() { + let float_gte = jit_function! { float_gte(a: f64, b: f64) -> bool => r##" + def float_gte(a: float, b: float): + return a >= b + "## }; + + assert_eq!(float_gte(2.0, 2.0), Ok(true)); + assert_eq!(float_gte(3.4, -1.7), Ok(true)); + assert_eq!(float_gte(0.0, 0.0), Ok(true)); + assert_eq!(float_gte(-0.0, -0.0), Ok(true)); + assert_eq!(float_gte(-0.0, 0.0), Ok(true)); + assert_eq!(float_gte(-5.2, f64::NAN), Ok(false)); + assert_eq!(float_gte(f64::NAN, f64::NAN), Ok(false)); + assert_eq!(float_gte(f64::INFINITY, f64::NEG_INFINITY), Ok(true)); +} + +#[test] +fn test_float_lt() { + let float_lt = jit_function! { float_lt(a: f64, b: f64) -> bool => r##" + def float_lt(a: float, b: float): + return a < b + "## }; + + assert_eq!(float_lt(2.0, 2.0), Ok(false)); + assert_eq!(float_lt(3.4, -1.7), Ok(false)); + assert_eq!(float_lt(0.0, 0.0), Ok(false)); + assert_eq!(float_lt(-0.0, -0.0), Ok(false)); + assert_eq!(float_lt(-0.0, 0.0), Ok(false)); + assert_eq!(float_lt(-5.2, f64::NAN), Ok(false)); + assert_eq!(float_lt(f64::NAN, f64::NAN), Ok(false)); + assert_eq!(float_lt(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); +} + +#[test] +fn test_float_lte() { + let float_lte = jit_function! { float_lte(a: f64, b: f64) -> bool => r##" + def float_lte(a: float, b: float): + return a <= b + "## }; + + assert_eq!(float_lte(2.0, 2.0), Ok(true)); + assert_eq!(float_lte(3.4, -1.7), Ok(false)); + assert_eq!(float_lte(0.0, 0.0), Ok(true)); + assert_eq!(float_lte(-0.0, -0.0), Ok(true)); + assert_eq!(float_lte(-0.0, 0.0), Ok(true)); + assert_eq!(float_lte(-5.2, f64::NAN), Ok(false)); + assert_eq!(float_lte(f64::NAN, f64::NAN), Ok(false)); + assert_eq!(float_lte(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); +} diff --git a/crates/jit/tests/int_tests.rs b/crates/jit/tests/int_tests.rs new file mode 100644 index 00000000000..5ab2697e075 --- /dev/null +++ b/crates/jit/tests/int_tests.rs @@ -0,0 +1,326 @@ +use core::f64; + +#[test] +fn test_add() { + let add = jit_function! { add(a:i64, b:i64) -> i64 => r##" + def add(a: int, b: int): + return a + b + "## }; + + assert_eq!(add(5, 10), Ok(15)); + assert_eq!(add(-5, 12), Ok(7)); + assert_eq!(add(-5, -3), Ok(-8)); +} + +#[test] +fn test_sub() { + let sub = jit_function! { sub(a:i64, b:i64) -> i64 => r##" + def sub(a: int, b: int): + return a - b + "## }; + + assert_eq!(sub(5, 10), Ok(-5)); + assert_eq!(sub(12, 10), Ok(2)); + assert_eq!(sub(7, 10), Ok(-3)); + assert_eq!(sub(-3, -10), Ok(7)); +} + +#[test] +fn test_mul() { + let mul = jit_function! { mul(a:i64, b:i64) -> i64 => r##" + def mul(a: int, b: int): + return a * b + "## }; + + assert_eq!(mul(5, 10), Ok(50)); + assert_eq!(mul(0, 5), Ok(0)); + assert_eq!(mul(5, 0), Ok(0)); + assert_eq!(mul(0, 0), Ok(0)); + assert_eq!(mul(-5, 10), Ok(-50)); + assert_eq!(mul(5, -10), Ok(-50)); + assert_eq!(mul(-5, -10), Ok(50)); + assert_eq!(mul(999999, 999999), Ok(999998000001)); + assert_eq!(mul(i64::MAX, 1), Ok(i64::MAX)); + assert_eq!(mul(1, i64::MAX), Ok(i64::MAX)); +} + +#[test] +fn test_div() { + let div = jit_function! { div(a:i64, b:i64) -> f64 => r##" + def div(a: int, b: int): + return a / b + "## }; + + assert_eq!(div(0, 1), Ok(0.0)); + assert_eq!(div(5, 1), Ok(5.0)); + assert_eq!(div(5, 10), Ok(0.5)); + assert_eq!(div(5, 2), Ok(2.5)); + assert_eq!(div(12, 10), Ok(1.2)); + assert_eq!(div(7, 10), Ok(0.7)); + assert_eq!(div(-3, -1), Ok(3.0)); + assert_eq!(div(-3, 1), Ok(-3.0)); + assert_eq!(div(1, 1000), Ok(0.001)); + assert_eq!(div(1, 100000), Ok(0.00001)); + assert_eq!(div(2, 3), Ok(0.6666666666666666)); + assert_eq!(div(1, 3), Ok(0.3333333333333333)); + assert_eq!(div(i64::MAX, 2), Ok(4611686018427387904.0)); + assert_eq!(div(i64::MIN, 2), Ok(-4611686018427387904.0)); + assert_eq!(div(i64::MIN, -1), Ok(9223372036854775808.0)); // Overflow case + assert_eq!(div(i64::MIN, i64::MAX), Ok(-1.0)); +} + +#[test] +fn test_floor_div() { + let floor_div = jit_function! { floor_div(a:i64, b:i64) -> i64 => r##" + def floor_div(a: int, b: int): + return a // b + "## }; + + assert_eq!(floor_div(5, 10), Ok(0)); + assert_eq!(floor_div(5, 2), Ok(2)); + assert_eq!(floor_div(12, 10), Ok(1)); + assert_eq!(floor_div(7, 10), Ok(0)); + assert_eq!(floor_div(-3, -1), Ok(3)); +} + +#[test] + +fn test_exp() { + let exp = jit_function! { exp(a: i64, b: i64) -> i64 => r##" + def exp(a: int, b: int): + return a ** b + "## }; + + assert_eq!(exp(2, 3), Ok(8)); + assert_eq!(exp(3, 2), Ok(9)); + assert_eq!(exp(5, 0), Ok(1)); + assert_eq!(exp(0, 0), Ok(1)); + assert_eq!(exp(-5, 0), Ok(1)); + assert_eq!(exp(0, 1), Ok(0)); + assert_eq!(exp(0, 5), Ok(0)); + assert_eq!(exp(-2, 2), Ok(4)); + assert_eq!(exp(-3, 4), Ok(81)); + assert_eq!(exp(-2, 3), Ok(-8)); + assert_eq!(exp(-3, 3), Ok(-27)); + assert_eq!(exp(1000, 2), Ok(1000000)); +} + +#[test] +fn test_mod() { + let modulo = jit_function! { modulo(a:i64, b:i64) -> i64 => r##" + def modulo(a: int, b: int): + return a % b + "## }; + + assert_eq!(modulo(5, 10), Ok(5)); + assert_eq!(modulo(5, 2), Ok(1)); + assert_eq!(modulo(12, 10), Ok(2)); + assert_eq!(modulo(7, 10), Ok(7)); + assert_eq!(modulo(-3, 1), Ok(0)); + assert_eq!(modulo(-5, 10), Ok(-5)); +} + +#[test] +fn test_power() { + let power = jit_function! { power(a:i64, b:i64) -> i64 => r##" + def power(a: int, b: int): + return a ** b + "## + }; + assert_eq!(power(10, 2), Ok(100)); + assert_eq!(power(5, 1), Ok(5)); + assert_eq!(power(1, 0), Ok(1)); +} + +#[test] +fn test_lshift() { + let lshift = jit_function! { lshift(a:i64, b:i64) -> i64 => r##" + def lshift(a: int, b: int): + return a << b + "## }; + + assert_eq!(lshift(5, 10), Ok(5120)); + assert_eq!(lshift(5, 2), Ok(20)); + assert_eq!(lshift(12, 10), Ok(12288)); + assert_eq!(lshift(7, 10), Ok(7168)); + assert_eq!(lshift(-3, 1), Ok(-6)); + assert_eq!(lshift(-10, 2), Ok(-40)); +} + +#[test] +fn test_rshift() { + let rshift = jit_function! { rshift(a:i64, b:i64) -> i64 => r##" + def rshift(a: int, b: int): + return a >> b + "## }; + + assert_eq!(rshift(5120, 10), Ok(5)); + assert_eq!(rshift(20, 2), Ok(5)); + assert_eq!(rshift(12288, 10), Ok(12)); + assert_eq!(rshift(7168, 10), Ok(7)); + assert_eq!(rshift(-3, 1), Ok(-2)); + assert_eq!(rshift(-10, 2), Ok(-3)); +} + +#[test] +fn test_and() { + let bitand = jit_function! { bitand(a:i64, b:i64) -> i64 => r##" + def bitand(a: int, b: int): + return a & b + "## }; + + assert_eq!(bitand(5120, 10), Ok(0)); + assert_eq!(bitand(20, 16), Ok(16)); + assert_eq!(bitand(12488, 4249), Ok(4232)); + assert_eq!(bitand(7168, 2), Ok(0)); + assert_eq!(bitand(-3, 1), Ok(1)); + assert_eq!(bitand(-10, 2), Ok(2)); +} + +#[test] +fn test_or() { + let bitor = jit_function! { bitor(a:i64, b:i64) -> i64 => r##" + def bitor(a: int, b: int): + return a | b + "## }; + + assert_eq!(bitor(5120, 10), Ok(5130)); + assert_eq!(bitor(20, 16), Ok(20)); + assert_eq!(bitor(12488, 4249), Ok(12505)); + assert_eq!(bitor(7168, 2), Ok(7170)); + assert_eq!(bitor(-3, 1), Ok(-3)); + assert_eq!(bitor(-10, 2), Ok(-10)); +} + +#[test] +fn test_xor() { + let bitxor = jit_function! { bitxor(a:i64, b:i64) -> i64 => r##" + def bitxor(a: int, b: int): + return a ^ b + "## }; + + assert_eq!(bitxor(5120, 10), Ok(5130)); + assert_eq!(bitxor(20, 16), Ok(4)); + assert_eq!(bitxor(12488, 4249), Ok(8273)); + assert_eq!(bitxor(7168, 2), Ok(7170)); + assert_eq!(bitxor(-3, 1), Ok(-4)); + assert_eq!(bitxor(-10, 2), Ok(-12)); +} + +#[test] +fn test_eq() { + let eq = jit_function! { eq(a:i64, b:i64) -> i64 => r##" + def eq(a: int, b: int): + if a == b: + return 1 + return 0 + "## }; + + assert_eq!(eq(0, 0), Ok(1)); + assert_eq!(eq(1, 1), Ok(1)); + assert_eq!(eq(0, 1), Ok(0)); + assert_eq!(eq(-200, 200), Ok(0)); +} + +#[test] +fn test_gt() { + let gt = jit_function! { gt(a:i64, b:i64) -> i64 => r##" + def gt(a: int, b: int): + if a > b: + return 1 + return 0 + "## }; + + assert_eq!(gt(5, 2), Ok(1)); + assert_eq!(gt(2, 5), Ok(0)); + assert_eq!(gt(2, 2), Ok(0)); + assert_eq!(gt(5, 5), Ok(0)); + assert_eq!(gt(-1, -10), Ok(1)); + assert_eq!(gt(1, -1), Ok(1)); +} + +#[test] +fn test_lt() { + let lt = jit_function! { lt(a:i64, b:i64) -> i64 => r##" + def lt(a: int, b: int): + if a < b: + return 1 + return 0 + "## }; + + assert_eq!(lt(-1, -5), Ok(0)); + assert_eq!(lt(10, 0), Ok(0)); + assert_eq!(lt(0, 1), Ok(1)); + assert_eq!(lt(-10, -1), Ok(1)); + assert_eq!(lt(100, 100), Ok(0)); +} + +#[test] +fn test_gte() { + let gte = jit_function! { gte(a:i64, b:i64) -> i64 => r##" + def gte(a: int, b: int): + if a >= b: + return 1 + return 0 + "## }; + + assert_eq!(gte(-64, -64), Ok(1)); + assert_eq!(gte(100, -1), Ok(1)); + assert_eq!(gte(1, 2), Ok(0)); + assert_eq!(gte(1, 0), Ok(1)); +} + +#[test] +fn test_lte() { + let lte = jit_function! { lte(a:i64, b:i64) -> i64 => r##" + def lte(a: int, b: int): + if a <= b: + return 1 + return 0 + "## }; + + assert_eq!(lte(-100, -100), Ok(1)); + assert_eq!(lte(-100, 100), Ok(1)); + assert_eq!(lte(10, 1), Ok(0)); + assert_eq!(lte(0, -2), Ok(0)); +} + +#[test] +fn test_minus() { + let minus = jit_function! { minus(a:i64) -> i64 => r##" + def minus(a: int): + return -a + "## }; + + assert_eq!(minus(5), Ok(-5)); + assert_eq!(minus(12), Ok(-12)); + assert_eq!(minus(-7), Ok(7)); + assert_eq!(minus(-3), Ok(3)); + assert_eq!(minus(0), Ok(0)); +} + +#[test] +fn test_plus() { + let plus = jit_function! { plus(a:i64) -> i64 => r##" + def plus(a: int): + return +a + "## }; + + assert_eq!(plus(5), Ok(5)); + assert_eq!(plus(12), Ok(12)); + assert_eq!(plus(-7), Ok(-7)); + assert_eq!(plus(-3), Ok(-3)); + assert_eq!(plus(0), Ok(0)); +} + +#[test] +fn test_not() { + let not_ = jit_function! { not_(a: i64) -> bool => r##" + def not_(a: int): + return not a + "## }; + + assert_eq!(not_(0), Ok(true)); + assert_eq!(not_(1), Ok(false)); + assert_eq!(not_(-1), Ok(false)); +} diff --git a/jit/tests/lib.rs b/crates/jit/tests/lib.rs similarity index 100% rename from jit/tests/lib.rs rename to crates/jit/tests/lib.rs diff --git a/jit/tests/misc_tests.rs b/crates/jit/tests/misc_tests.rs similarity index 92% rename from jit/tests/misc_tests.rs rename to crates/jit/tests/misc_tests.rs index 7c1b6c3afb5..25d66c46c06 100644 --- a/jit/tests/misc_tests.rs +++ b/crates/jit/tests/misc_tests.rs @@ -95,7 +95,6 @@ fn test_while_loop() { a -= 1 return b "## }; - assert_eq!(while_loop(0), Ok(0)); assert_eq!(while_loop(-1), Ok(0)); assert_eq!(while_loop(1), Ok(1)); @@ -113,3 +112,15 @@ fn test_unpack_tuple() { assert_eq!(unpack_tuple(0, 1), Ok(1)); assert_eq!(unpack_tuple(1, 2), Ok(2)); } + +#[test] +fn test_recursive_fib() { + let fib = jit_function! { fib(n: i64) -> i64 => r##" + def fib(n: int) -> int: + if n == 0 or n == 1: + return 1 + return fib(n-1) + fib(n-2) + "## }; + + assert_eq!(fib(10), Ok(89)); +} diff --git a/jit/tests/none_tests.rs b/crates/jit/tests/none_tests.rs similarity index 100% rename from jit/tests/none_tests.rs rename to crates/jit/tests/none_tests.rs diff --git a/crates/literal/Cargo.toml b/crates/literal/Cargo.toml new file mode 100644 index 00000000000..bd6a2699742 --- /dev/null +++ b/crates/literal/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "rustpython-literal" +description = "Common literal handling utilities mostly useful for unparse and repr." +edition = { workspace = true } +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +rustpython-wtf8 = { workspace = true } + +hexf-parse = "0.2.1" +is-macro.workspace = true +lexical-parse-float = { version = "1.0.6", features = ["format"] } +num-traits = { workspace = true } +unic-ucd-category = { workspace = true } + +[dev-dependencies] +rand = { workspace = true } diff --git a/crates/literal/src/char.rs b/crates/literal/src/char.rs new file mode 100644 index 00000000000..cd64f6dfa9e --- /dev/null +++ b/crates/literal/src/char.rs @@ -0,0 +1,15 @@ +use unic_ucd_category::GeneralCategory; + +/// According to python following categories aren't printable: +/// * Cc (Other, Control) +/// * Cf (Other, Format) +/// * Cs (Other, Surrogate) +/// * Co (Other, Private Use) +/// * Cn (Other, Not Assigned) +/// * Zl Separator, Line ('\u2028', LINE SEPARATOR) +/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) +/// * Zs (Separator, Space) other than ASCII space('\x20'). +pub fn is_printable(c: char) -> bool { + let cat = GeneralCategory::of(c); + !(cat.is_other() || cat.is_separator()) +} diff --git a/crates/literal/src/complex.rs b/crates/literal/src/complex.rs new file mode 100644 index 00000000000..c91d1c1439e --- /dev/null +++ b/crates/literal/src/complex.rs @@ -0,0 +1,75 @@ +use crate::float; +use alloc::borrow::ToOwned; +use alloc::string::{String, ToString}; + +/// Convert a complex number to a string. +pub fn to_string(re: f64, im: f64) -> String { + // integer => drop ., fractional => float_ops + let mut im_part = if im.fract() == 0.0 { + im.to_string() + } else { + float::to_string(im) + }; + im_part.push('j'); + + // positive empty => return im_part, integer => drop ., fractional => float_ops + let re_part = if re == 0.0 { + if re.is_sign_positive() { + return im_part; + } else { + "-0".to_owned() + } + } else if re.fract() == 0.0 { + re.to_string() + } else { + float::to_string(re) + }; + let mut result = + String::with_capacity(re_part.len() + im_part.len() + 2 + im.is_sign_positive() as usize); + result.push('('); + result.push_str(&re_part); + if im.is_sign_positive() || im.is_nan() { + result.push('+'); + } + result.push_str(&im_part); + result.push(')'); + result +} + +/// Parse a complex number from a string. +/// +/// Returns `Some((re, im))` on success. +pub fn parse_str(s: &str) -> Option<(f64, f64)> { + let s = s.trim(); + // Handle parentheses + let s = match s.strip_prefix('(') { + None => s, + Some(s) => s.strip_suffix(')')?.trim(), + }; + + let value = match s.strip_suffix(|c| c == 'j' || c == 'J') { + None => (float::parse_str(s)?, 0.0), + Some(mut s) => { + let mut real = 0.0; + // Find the central +/- operator. If it exists, parse the real part. + for (i, w) in s.as_bytes().windows(2).enumerate() { + if (w[1] == b'+' || w[1] == b'-') && !(w[0] == b'e' || w[0] == b'E') { + real = float::parse_str(&s[..=i])?; + s = &s[i + 1..]; + break; + } + } + + let imag = match s { + // "j", "+j" + "" | "+" => 1.0, + // "-j" + "-" => -1.0, + s => float::parse_str(s)?, + }; + + (real, imag) + } + }; + Some(value) +} diff --git a/crates/literal/src/escape.rs b/crates/literal/src/escape.rs new file mode 100644 index 00000000000..1099c0a02bc --- /dev/null +++ b/crates/literal/src/escape.rs @@ -0,0 +1,468 @@ +use alloc::string::String; +use rustpython_wtf8::{CodePoint, Wtf8}; + +#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash, is_macro::Is)] +pub enum Quote { + Single, + Double, +} + +impl Quote { + #[inline] + pub const fn swap(self) -> Self { + match self { + Self::Single => Self::Double, + Self::Double => Self::Single, + } + } + + #[inline] + pub const fn to_byte(&self) -> u8 { + match self { + Self::Single => b'\'', + Self::Double => b'"', + } + } + + #[inline] + pub const fn to_char(&self) -> char { + match self { + Self::Single => '\'', + Self::Double => '"', + } + } +} + +pub struct EscapeLayout { + pub quote: Quote, + pub len: Option<usize>, +} + +/// Represents string types that can be escape-printed. +/// +/// # Safety +/// +/// `source_len` and `layout` must be accurate, and `layout.len` must not be equal +/// to `Some(source_len)` if the string contains non-printable characters. +pub unsafe trait Escape { + fn source_len(&self) -> usize; + fn layout(&self) -> &EscapeLayout; + fn changed(&self) -> bool { + self.layout().len != Some(self.source_len()) + } + + /// Write the body of the string directly to the formatter. + /// + /// # Safety + /// + /// This string must only contain printable characters. + unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result; + fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result; + fn write_body(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { + if self.changed() { + self.write_body_slow(formatter) + } else { + // SAFETY: verified the string contains only printable characters. + unsafe { self.write_source(formatter) } + } + } +} + +/// Returns the outer quotes to use and the number of quotes that need to be +/// escaped. +pub(crate) const fn choose_quote( + single_count: usize, + double_count: usize, + preferred_quote: Quote, +) -> (Quote, usize) { + let (primary_count, secondary_count) = match preferred_quote { + Quote::Single => (single_count, double_count), + Quote::Double => (double_count, single_count), + }; + + // always use primary unless we have primary but no secondary + let use_secondary = primary_count > 0 && secondary_count == 0; + if use_secondary { + (preferred_quote.swap(), secondary_count) + } else { + (preferred_quote, primary_count) + } +} + +pub struct UnicodeEscape<'a> { + source: &'a Wtf8, + layout: EscapeLayout, +} + +impl<'a> UnicodeEscape<'a> { + #[inline] + pub const fn with_forced_quote(source: &'a Wtf8, quote: Quote) -> Self { + let layout = EscapeLayout { quote, len: None }; + Self { source, layout } + } + #[inline] + pub fn with_preferred_quote(source: &'a Wtf8, quote: Quote) -> Self { + let layout = Self::repr_layout(source, quote); + Self { source, layout } + } + #[inline] + pub fn new_repr(source: &'a Wtf8) -> Self { + Self::with_preferred_quote(source, Quote::Single) + } + #[inline] + pub const fn str_repr<'r>(&'a self) -> StrRepr<'r, 'a> { + StrRepr(self) + } +} + +pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>); + +impl StrRepr<'_, '_> { + pub fn write(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { + let quote = self.0.layout().quote.to_char(); + formatter.write_char(quote)?; + self.0.write_body(formatter)?; + formatter.write_char(quote) + } + + pub fn to_string(&self) -> Option<String> { + let mut s = String::with_capacity(self.0.layout().len?); + self.write(&mut s).unwrap(); + Some(s) + } +} + +impl core::fmt::Display for StrRepr<'_, '_> { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.write(formatter) + } +} + +impl UnicodeEscape<'_> { + const REPR_RESERVED_LEN: usize = 2; // for quotes + + pub fn repr_layout(source: &Wtf8, preferred_quote: Quote) -> EscapeLayout { + Self::output_layout_with_checker(source, preferred_quote, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + fn output_layout_with_checker( + source: &Wtf8, + preferred_quote: Quote, + length_add: impl Fn(usize, usize) -> Option<usize>, + ) -> EscapeLayout { + let mut out_len = Self::REPR_RESERVED_LEN; + let mut single_count = 0; + let mut double_count = 0; + + for ch in source.code_points() { + let incr = match ch.to_char() { + Some('\'') => { + single_count += 1; + 1 + } + Some('"') => { + double_count += 1; + 1 + } + _ => Self::escaped_char_len(ch), + }; + let Some(new_len) = length_add(out_len, incr) else { + #[cold] + const fn stop( + single_count: usize, + double_count: usize, + preferred_quote: Quote, + ) -> EscapeLayout { + EscapeLayout { + quote: choose_quote(single_count, double_count, preferred_quote).0, + len: None, + } + } + return stop(single_count, double_count, preferred_quote); + }; + out_len = new_len; + } + + let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote); + // we'll be adding backslashes in front of the existing inner quotes + let Some(out_len) = length_add(out_len, num_escaped_quotes) else { + return EscapeLayout { quote, len: None }; + }; + + EscapeLayout { + quote, + len: Some(out_len - Self::REPR_RESERVED_LEN), + } + } + + fn escaped_char_len(ch: CodePoint) -> usize { + // surrogates are \uHHHH + let Some(ch) = ch.to_char() else { return 6 }; + match ch { + '\\' | '\t' | '\r' | '\n' => 2, + ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH + ch if ch.is_ascii() => 1, + ch if crate::char::is_printable(ch) => { + // max = std::cmp::max(ch, max); + ch.len_utf8() + } + ch if (ch as u32) < 0x100 => 4, // \xHH + ch if (ch as u32) < 0x10000 => 6, // \uHHHH + _ => 10, // \uHHHHHHHH + } + } + + fn write_char( + ch: CodePoint, + quote: Quote, + formatter: &mut impl core::fmt::Write, + ) -> core::fmt::Result { + let Some(ch) = ch.to_char() else { + return write!(formatter, "\\u{:04x}", ch.to_u32()); + }; + match ch { + '\n' => formatter.write_str("\\n"), + '\t' => formatter.write_str("\\t"), + '\r' => formatter.write_str("\\r"), + // these 2 branches *would* be handled below, but we shouldn't have to do a + // unicodedata lookup just for ascii characters + '\x20'..='\x7e' => { + // printable ascii range + if ch == quote.to_char() || ch == '\\' { + formatter.write_char('\\')?; + } + formatter.write_char(ch) + } + ch if ch.is_ascii() => { + write!(formatter, "\\x{:02x}", ch as u8) + } + ch if crate::char::is_printable(ch) => formatter.write_char(ch), + '\0'..='\u{ff}' => { + write!(formatter, "\\x{:02x}", ch as u32) + } + '\0'..='\u{ffff}' => { + write!(formatter, "\\u{:04x}", ch as u32) + } + _ => { + write!(formatter, "\\U{:08x}", ch as u32) + } + } + } +} + +unsafe impl Escape for UnicodeEscape<'_> { + fn source_len(&self) -> usize { + self.source.len() + } + + fn layout(&self) -> &EscapeLayout { + &self.layout + } + + unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { + formatter.write_str(unsafe { + // SAFETY: this function must be called only when source is printable characters (i.e. no surrogates) + core::str::from_utf8_unchecked(self.source.as_bytes()) + }) + } + + #[cold] + fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { + for ch in self.source.code_points() { + Self::write_char(ch, self.layout().quote, formatter)?; + } + Ok(()) + } +} + +pub struct AsciiEscape<'a> { + source: &'a [u8], + layout: EscapeLayout, +} + +impl<'a> AsciiEscape<'a> { + #[inline] + pub const fn new(source: &'a [u8], layout: EscapeLayout) -> Self { + Self { source, layout } + } + #[inline] + pub const fn with_forced_quote(source: &'a [u8], quote: Quote) -> Self { + let layout = EscapeLayout { quote, len: None }; + Self { source, layout } + } + #[inline] + pub fn with_preferred_quote(source: &'a [u8], quote: Quote) -> Self { + let layout = Self::repr_layout(source, quote); + Self { source, layout } + } + #[inline] + pub fn new_repr(source: &'a [u8]) -> Self { + Self::with_preferred_quote(source, Quote::Single) + } + #[inline] + pub const fn bytes_repr<'r>(&'a self) -> BytesRepr<'r, 'a> { + BytesRepr(self) + } +} + +impl AsciiEscape<'_> { + pub fn repr_layout(source: &[u8], preferred_quote: Quote) -> EscapeLayout { + Self::output_layout_with_checker(source, preferred_quote, 3, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + pub fn named_repr_layout(source: &[u8], name: &str) -> EscapeLayout { + Self::output_layout_with_checker(source, Quote::Single, name.len() + 2 + 3, |a, b| { + Some((a as isize).checked_add(b as isize)? as usize) + }) + } + + fn output_layout_with_checker( + source: &[u8], + preferred_quote: Quote, + reserved_len: usize, + length_add: impl Fn(usize, usize) -> Option<usize>, + ) -> EscapeLayout { + let mut out_len = reserved_len; + let mut single_count = 0; + let mut double_count = 0; + + for ch in source { + let incr = match ch { + b'\'' => { + single_count += 1; + 1 + } + b'"' => { + double_count += 1; + 1 + } + c => Self::escaped_char_len(*c), + }; + let Some(new_len) = length_add(out_len, incr) else { + #[cold] + const fn stop( + single_count: usize, + double_count: usize, + preferred_quote: Quote, + ) -> EscapeLayout { + EscapeLayout { + quote: choose_quote(single_count, double_count, preferred_quote).0, + len: None, + } + } + return stop(single_count, double_count, preferred_quote); + }; + out_len = new_len; + } + + let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote); + // we'll be adding backslashes in front of the existing inner quotes + let Some(out_len) = length_add(out_len, num_escaped_quotes) else { + return EscapeLayout { quote, len: None }; + }; + + EscapeLayout { + quote, + len: Some(out_len - reserved_len), + } + } + + const fn escaped_char_len(ch: u8) -> usize { + match ch { + b'\\' | b'\t' | b'\r' | b'\n' => 2, + 0x20..=0x7e => 1, + _ => 4, // \xHH + } + } + + fn write_char( + ch: u8, + quote: Quote, + formatter: &mut impl core::fmt::Write, + ) -> core::fmt::Result { + match ch { + b'\t' => formatter.write_str("\\t"), + b'\n' => formatter.write_str("\\n"), + b'\r' => formatter.write_str("\\r"), + 0x20..=0x7e => { + // printable ascii range + if ch == quote.to_byte() || ch == b'\\' { + formatter.write_char('\\')?; + } + formatter.write_char(ch as char) + } + ch => write!(formatter, "\\x{ch:02x}"), + } + } +} + +unsafe impl Escape for AsciiEscape<'_> { + fn source_len(&self) -> usize { + self.source.len() + } + + fn layout(&self) -> &EscapeLayout { + &self.layout + } + + unsafe fn write_source(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { + formatter.write_str(unsafe { + // SAFETY: this function must be called only when source is printable ascii characters + core::str::from_utf8_unchecked(self.source) + }) + } + + #[cold] + fn write_body_slow(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { + for ch in self.source { + Self::write_char(*ch, self.layout().quote, formatter)?; + } + Ok(()) + } +} + +pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>); + +impl BytesRepr<'_, '_> { + pub fn write(&self, formatter: &mut impl core::fmt::Write) -> core::fmt::Result { + let quote = self.0.layout().quote.to_char(); + formatter.write_char('b')?; + formatter.write_char(quote)?; + self.0.write_body(formatter)?; + formatter.write_char(quote) + } + + pub fn to_string(&self) -> Option<String> { + let mut s = String::with_capacity(self.0.layout().len?); + self.write(&mut s).unwrap(); + Some(s) + } +} + +impl core::fmt::Display for BytesRepr<'_, '_> { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.write(formatter) + } +} + +#[cfg(test)] +mod unicode_escape_tests { + use super::*; + + #[test] + fn changed() { + fn test(s: &str) -> bool { + UnicodeEscape::new_repr(s.as_ref()).changed() + } + assert!(!test("hello")); + assert!(!test("'hello'")); + assert!(!test("\"hello\"")); + + assert!(test("'\"hello")); + assert!(test("hello\n")); + } +} diff --git a/crates/literal/src/float.rs b/crates/literal/src/float.rs new file mode 100644 index 00000000000..0fc51782438 --- /dev/null +++ b/crates/literal/src/float.rs @@ -0,0 +1,301 @@ +use crate::format::Case; +use alloc::borrow::ToOwned; +use alloc::format; +use alloc::string::{String, ToString}; +use core::f64; +use num_traits::{Float, Zero}; + +pub fn parse_str(literal: &str) -> Option<f64> { + parse_inner(literal.trim().as_bytes()) +} + +pub fn parse_bytes(literal: &[u8]) -> Option<f64> { + parse_inner(literal.trim_ascii()) +} + +fn parse_inner(literal: &[u8]) -> Option<f64> { + use lexical_parse_float::{ + FromLexicalWithOptions, NumberFormatBuilder, Options, format::PYTHON3_LITERAL, + }; + + // lexical-core's format::PYTHON_STRING is inaccurate + const PYTHON_STRING: u128 = NumberFormatBuilder::rebuild(PYTHON3_LITERAL) + .no_special(false) + .build_unchecked(); + f64::from_lexical_with_options::<PYTHON_STRING>(literal, &Options::new()).ok() +} + +pub fn is_integer(v: f64) -> bool { + v.is_finite() && v.fract() == 0.0 +} + +fn format_nan(case: Case) -> String { + let nan = match case { + Case::Lower => "nan", + Case::Upper => "NAN", + }; + + nan.to_string() +} + +fn format_inf(case: Case) -> String { + let inf = match case { + Case::Lower => "inf", + Case::Upper => "INF", + }; + + inf.to_string() +} + +pub const fn decimal_point_or_empty(precision: usize, alternate_form: bool) -> &'static str { + match (precision, alternate_form) { + (0, true) => ".", + _ => "", + } +} + +pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form: bool) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let point = decimal_point_or_empty(precision, alternate_form); + let precision = core::cmp::min(precision, u16::MAX as usize); + format!("{magnitude:.precision$}{point}") + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +// Formats floats into Python style exponent notation, by first formatting in Rust style +// exponent notation (`1.0000e0`), then convert to Python style (`1.0000e+00`). +pub fn format_exponent( + precision: usize, + magnitude: f64, + case: Case, + alternate_form: bool, +) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let r_exp = format!("{magnitude:.precision$e}"); + let mut parts = r_exp.splitn(2, 'e'); + let base = parts.next().unwrap(); + let exponent = parts.next().unwrap().parse::<i64>().unwrap(); + let e = match case { + Case::Lower => 'e', + Case::Upper => 'E', + }; + let point = decimal_point_or_empty(precision, alternate_form); + format!("{base}{point}{e}{exponent:+#03}") + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +/// If s represents a floating point value, trailing zeros and a possibly trailing +/// decimal point will be removed. +/// This function does NOT work with decimal commas. +fn maybe_remove_trailing_redundant_chars(s: String, alternate_form: bool) -> String { + if !alternate_form && s.contains('.') { + // only truncate floating point values when not in alternate form + let s = remove_trailing_zeros(s); + remove_trailing_decimal_point(s) + } else { + s + } +} + +fn remove_trailing_zeros(s: String) -> String { + let mut s = s; + while s.ends_with('0') { + s.pop(); + } + s +} + +fn remove_trailing_decimal_point(s: String) -> String { + let mut s = s; + if s.ends_with('.') { + s.pop(); + } + s +} + +pub fn format_general( + precision: usize, + magnitude: f64, + case: Case, + alternate_form: bool, + always_shows_fract: bool, +) -> String { + match magnitude { + magnitude if magnitude.is_finite() => { + let r_exp = format!("{:.*e}", precision.saturating_sub(1), magnitude); + let mut parts = r_exp.splitn(2, 'e'); + let base = parts.next().unwrap(); + let exponent = parts.next().unwrap().parse::<i64>().unwrap(); + if exponent < -4 || exponent + (always_shows_fract as i64) >= (precision as i64) { + let e = match case { + Case::Lower => 'e', + Case::Upper => 'E', + }; + let magnitude = format!("{:.*}", precision + 1, base); + let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form); + let point = decimal_point_or_empty(precision.saturating_sub(1), alternate_form); + format!("{base}{point}{e}{exponent:+#03}") + } else { + let precision = ((precision as i64) - 1 - exponent) as usize; + let magnitude = format!("{magnitude:.precision$}"); + let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form); + let point = decimal_point_or_empty(precision, alternate_form); + format!("{base}{point}") + } + } + magnitude if magnitude.is_nan() => format_nan(case), + magnitude if magnitude.is_infinite() => format_inf(case), + _ => "".to_string(), + } +} + +// TODO: rewrite using format_general +pub fn to_string(value: f64) -> String { + let lit = format!("{value:e}"); + if let Some(position) = lit.find('e') { + let significand = &lit[..position]; + let exponent = &lit[position + 1..]; + let exponent = exponent.parse::<i32>().unwrap(); + if exponent < 16 && exponent > -5 { + if is_integer(value) { + format!("{value:.1?}") + } else { + value.to_string() + } + } else { + format!("{significand}e{exponent:+#03}") + } + } else { + let mut s = value.to_string(); + s.make_ascii_lowercase(); + s + } +} + +pub fn from_hex(s: &str) -> Option<f64> { + if let Ok(f) = hexf_parse::parse_hexf64(s, false) { + return Some(f); + } + match s.to_ascii_lowercase().as_str() { + "nan" | "+nan" | "-nan" => Some(f64::NAN), + "inf" | "infinity" | "+inf" | "+infinity" => Some(f64::INFINITY), + "-inf" | "-infinity" => Some(f64::NEG_INFINITY), + value => { + let mut hex = String::with_capacity(value.len()); + let has_0x = value.contains("0x"); + let has_p = value.contains('p'); + let has_dot = value.contains('.'); + let mut start = 0; + + if !has_0x && value.starts_with('-') { + hex.push_str("-0x"); + start += 1; + } else if !has_0x { + hex.push_str("0x"); + if value.starts_with('+') { + start += 1; + } + } + + for (index, ch) in value.chars().enumerate() { + if ch == 'p' { + if has_dot { + hex.push('p'); + } else { + hex.push_str(".p"); + } + } else if index >= start { + hex.push(ch); + } + } + + if !has_p && has_dot { + hex.push_str("p0"); + } else if !has_p && !has_dot { + hex.push_str(".p0") + } + + hexf_parse::parse_hexf64(hex.as_str(), false).ok() + } + } +} + +pub fn to_hex(value: f64) -> String { + let (mantissa, exponent, sign) = value.integer_decode(); + let sign_fmt = if sign < 0 { "-" } else { "" }; + match value { + value if value.is_zero() => format!("{sign_fmt}0x0.0p+0"), + value if value.is_infinite() => format!("{sign_fmt}inf"), + value if value.is_nan() => "nan".to_owned(), + _ => { + const BITS: i16 = 52; + const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff; + format!( + "{}{:#x}.{:013x}p{:+}", + sign_fmt, + mantissa >> BITS, + mantissa & FRACT_MASK, + exponent + BITS + ) + } + } +} + +#[test] +fn test_to_hex() { + use rand::Rng; + for _ in 0..20000 { + let bytes = rand::rng().random::<u64>(); + let f = f64::from_bits(bytes); + if !f.is_finite() { + continue; + } + let hex = to_hex(f); + // println!("{} -> {}", f, hex); + let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap(); + // println!(" -> {}", roundtrip); + assert!(f == roundtrip, "{f} {hex} {roundtrip}"); + } +} + +#[test] +fn test_remove_trailing_zeros() { + assert!(remove_trailing_zeros(String::from("100")) == *"1"); + assert!(remove_trailing_zeros(String::from("100.00")) == *"100."); + + // leave leading zeros untouched + assert!(remove_trailing_zeros(String::from("001")) == *"001"); + + // leave strings untouched if they don't end with 0 + assert!(remove_trailing_zeros(String::from("101")) == *"101"); +} + +#[test] +fn test_remove_trailing_decimal_point() { + assert!(remove_trailing_decimal_point(String::from("100.")) == *"100"); + assert!(remove_trailing_decimal_point(String::from("1.")) == *"1"); + + // leave leading decimal points untouched + assert!(remove_trailing_decimal_point(String::from(".5")) == *".5"); +} + +#[test] +fn test_maybe_remove_trailing_redundant_chars() { + assert!(maybe_remove_trailing_redundant_chars(String::from("100."), true) == *"100."); + assert!(maybe_remove_trailing_redundant_chars(String::from("100."), false) == *"100"); + assert!(maybe_remove_trailing_redundant_chars(String::from("1."), false) == *"1"); + assert!(maybe_remove_trailing_redundant_chars(String::from("10.0"), false) == *"10"); + + // don't truncate integers + assert!(maybe_remove_trailing_redundant_chars(String::from("1000"), false) == *"1000"); +} diff --git a/crates/literal/src/format.rs b/crates/literal/src/format.rs new file mode 100644 index 00000000000..4ce21ad7815 --- /dev/null +++ b/crates/literal/src/format.rs @@ -0,0 +1,5 @@ +#[derive(Debug, PartialEq, Eq, Clone, Copy, is_macro::Is, Hash)] +pub enum Case { + Lower, + Upper, +} diff --git a/crates/literal/src/lib.rs b/crates/literal/src/lib.rs new file mode 100644 index 00000000000..a863dd87738 --- /dev/null +++ b/crates/literal/src/lib.rs @@ -0,0 +1,9 @@ +#![no_std] + +extern crate alloc; + +pub mod char; +pub mod complex; +pub mod escape; +pub mod float; +pub mod format; diff --git a/crates/pylib/Cargo.toml b/crates/pylib/Cargo.toml new file mode 100644 index 00000000000..dcbb5928599 --- /dev/null +++ b/crates/pylib/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "rustpython-pylib" +description = "A subset of the Python standard library for use with RustPython" +license-file = "Lib/PSF-LICENSE" +include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"] +authors = ["CPython Developers"] +version.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true + +[features] +freeze-stdlib = ["dep:rustpython-compiler-core", "dep:rustpython-derive"] + +[dependencies] +rustpython-compiler-core = { workspace = true, optional = true } +rustpython-derive = { workspace = true, optional = true } + +[build-dependencies] +glob = { workspace = true } + +[lints] +workspace = true \ No newline at end of file diff --git a/crates/pylib/Lib b/crates/pylib/Lib new file mode 120000 index 00000000000..5665252daf7 --- /dev/null +++ b/crates/pylib/Lib @@ -0,0 +1 @@ +../../Lib/ \ No newline at end of file diff --git a/crates/pylib/build.rs b/crates/pylib/build.rs new file mode 100644 index 00000000000..f96ef9b477c --- /dev/null +++ b/crates/pylib/build.rs @@ -0,0 +1,52 @@ +const CRATE_ROOT: &str = "../.."; + +fn main() { + process_python_libs(format!("{CRATE_ROOT}/vm/Lib/python_builtins/*").as_str()); + process_python_libs(format!("{CRATE_ROOT}/vm/Lib/core_modules/*").as_str()); + + #[cfg(feature = "freeze-stdlib")] + if cfg!(windows) { + process_python_libs(format!("{CRATE_ROOT}/Lib/**/*").as_str()); + } else { + process_python_libs("./Lib/**/*"); + } + + if cfg!(windows) { + // On Windows, the Lib entry can be either: + // 1. A text file containing the relative path (git without symlink support) + // 2. A proper symlink (git with symlink support) + // We handle both cases to resolve to the actual Lib directory. + let lib_path = if let Ok(real_path) = std::fs::read_to_string("Lib") { + // Case 1: Text file containing relative path + std::path::PathBuf::from(real_path.trim()) + } else { + // Case 2: Symlink or directory - canonicalize directly + std::path::PathBuf::from("Lib") + }; + + if let Ok(canonicalized_path) = std::fs::canonicalize(&lib_path) { + // Strip the extended path prefix (\\?\) that canonicalize adds on Windows + let path_str = canonicalized_path.to_str().unwrap(); + let path_str = path_str.strip_prefix(r"\\?\").unwrap_or(path_str); + println!("cargo:rustc-env=win_lib_path={path_str}"); + } + } +} + +// remove *.pyc files and add *.py to watch list +fn process_python_libs(pattern: &str) { + let glob = glob::glob(pattern).unwrap_or_else(|e| panic!("failed to glob {pattern:?}: {e}")); + for entry in glob.flatten() { + if entry.is_dir() { + continue; + } + let display = entry.display(); + if display.to_string().ends_with(".pyc") { + if std::fs::remove_file(&entry).is_err() { + println!("cargo:warning=failed to remove {display}") + } + continue; + } + println!("cargo:rerun-if-changed={display}"); + } +} diff --git a/crates/pylib/src/lib.rs b/crates/pylib/src/lib.rs new file mode 100644 index 00000000000..db957f633fc --- /dev/null +++ b/crates/pylib/src/lib.rs @@ -0,0 +1,16 @@ +//! This crate includes the compiled python bytecode of the RustPython standard library. The most +//! common way to use this crate is to just add the `"freeze-stdlib"` feature to `rustpython-vm`, +//! in order to automatically include the python part of the standard library into the binary. + +#![no_std] + +// windows needs to read the symlink out of `Lib` as git turns it into a text file, +// so build.rs sets this env var +pub const LIB_PATH: &str = match option_env!("win_lib_path") { + Some(s) => s, + None => concat!(env!("CARGO_MANIFEST_DIR"), "/Lib"), +}; + +#[cfg(feature = "freeze-stdlib")] +pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = + rustpython_derive::py_freeze!(dir = "../Lib", crate_name = "rustpython_compiler_core"); diff --git a/crates/sre_engine/.gitignore b/crates/sre_engine/.gitignore new file mode 100644 index 00000000000..96ef6c0b944 --- /dev/null +++ b/crates/sre_engine/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/crates/sre_engine/Cargo.toml b/crates/sre_engine/Cargo.toml new file mode 100644 index 00000000000..4f899e6b3e9 --- /dev/null +++ b/crates/sre_engine/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "rustpython-sre_engine" +authors = ["Kangzhi Shi <shikangzhi@gmail.com>", "RustPython Team"] +description = "A low-level implementation of Python's SRE regex engine" +keywords = ["regex"] +include = ["LICENSE", "src/**/*.rs"] +version.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[[bench]] +name = "benches" +harness = false + +[dependencies] +rustpython-wtf8 = { workspace = true } +num_enum = { workspace = true } +bitflags = { workspace = true } +optional = { workspace = true } + +[dev-dependencies] +criterion = { workspace = true } + +[lints] +workspace = true diff --git a/crates/sre_engine/LICENSE b/crates/sre_engine/LICENSE new file mode 100644 index 00000000000..e2aa2ed952e --- /dev/null +++ b/crates/sre_engine/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 RustPython Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/sre_engine/benches/benches.rs b/crates/sre_engine/benches/benches.rs new file mode 100644 index 00000000000..9905a8db70f --- /dev/null +++ b/crates/sre_engine/benches/benches.rs @@ -0,0 +1,116 @@ +use rustpython_sre_engine::{Request, State, StrDrive}; + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; + +struct Pattern { + pattern: &'static str, + code: &'static [u32], +} + +impl Pattern { + fn state<'a, S: StrDrive>(&self, string: S) -> (Request<'a, S>, State) { + self.state_range(string, 0..usize::MAX) + } + + fn state_range<'a, S: StrDrive>( + &self, + string: S, + range: core::ops::Range<usize>, + ) -> (Request<'a, S>, State) { + let req = Request::new(string, range.start, range.end, self.code, false); + let state = State::default(); + (req, state) + } +} + +fn basic(c: &mut Criterion) { + // # test common prefix + // pattern p1 = re.compile('Python|Perl') # , 'Perl'), # Alternation + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p1 = Pattern { pattern: "Python|Perl", code: &[14, 8, 1, 4, 6, 1, 1, 80, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 1] }; + // END GENERATED + // pattern p2 = re.compile('(Python|Perl)') #, 'Perl'), # Grouped alternation + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p2 = Pattern { pattern: "(Python|Perl)", code: &[14, 8, 1, 4, 6, 1, 0, 80, 0, 17, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 17, 1, 1] }; + // END GENERATED + // pattern p3 = re.compile('Python|Perl|Tcl') #, 'Perl'), # Alternation + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p3 = Pattern { pattern: "Python|Perl|Tcl", code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 1] }; + // END GENERATED + // pattern p4 = re.compile('(Python|Perl|Tcl)') #, 'Perl'), # Grouped alternation + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p4 = Pattern { pattern: "(Python|Perl|Tcl)", code: &[14, 9, 4, 3, 6, 16, 80, 16, 84, 0, 17, 0, 7, 15, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 22, 11, 16, 80, 16, 101, 16, 114, 16, 108, 15, 11, 9, 16, 84, 16, 99, 16, 108, 15, 2, 0, 17, 1, 1] }; + // END GENERATED + // pattern p5 = re.compile('(Python)\\1') #, 'PythonPython'), # Backreference + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p5 = Pattern { pattern: "(Python)\\1", code: &[14, 18, 1, 12, 12, 6, 0, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 11, 0, 1] }; + // END GENERATED + // pattern p6 = re.compile('([0a-z][a-z0-9]*,)+') #, 'a5,b7,c9,'), # Disable the fast map optimization + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p6 = Pattern { pattern: "([0a-z][a-z0-9]*,)+", code: &[14, 4, 0, 2, 4294967295, 23, 31, 1, 4294967295, 17, 0, 13, 7, 16, 48, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; + // END GENERATED + // pattern p7 = re.compile('([a-z][a-z0-9]*,)+') #, 'a5,b7,c9,'), # A few sets + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p7 = Pattern { pattern: "([a-z][a-z0-9]*,)+", code: &[14, 4, 0, 2, 4294967295, 23, 29, 1, 4294967295, 17, 0, 13, 5, 22, 97, 122, 0, 24, 13, 0, 4294967295, 13, 8, 22, 97, 122, 22, 48, 57, 0, 1, 16, 44, 17, 1, 18, 1] }; + // END GENERATED + // pattern p8 = re.compile('Python') #, 'Python'), # Simple text literal + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p8 = Pattern { pattern: "Python", code: &[14, 18, 3, 6, 6, 6, 6, 80, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; + // END GENERATED + // pattern p9 = re.compile('.*Python') #, 'Python'), # Bad text literal + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p9 = Pattern { pattern: ".*Python", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 1] }; + // END GENERATED + // pattern p10 = re.compile('.*Python.*') #, 'Python'), # Worse text literal + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p10 = Pattern { pattern: ".*Python.*", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 24, 5, 0, 4294967295, 2, 1, 1] }; + // END GENERATED + // pattern p11 = re.compile('.*(Python)') #, 'Python'), # Bad text literal with grouping + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p11 = Pattern { pattern: ".*(Python)", code: &[14, 4, 0, 6, 4294967295, 24, 5, 0, 4294967295, 2, 1, 17, 0, 16, 80, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 1] }; + // END GENERATED + + let tests = [ + (p1, "Perl"), + (p2, "Perl"), + (p3, "Perl"), + (p4, "Perl"), + (p5, "PythonPython"), + (p6, "a5,b7,c9,"), + (p7, "a5,b7,c9,"), + (p8, "Python"), + (p9, "Python"), + (p10, "Python"), + (p11, "Python"), + ]; + + let mut group = c.benchmark_group("basic"); + + for (p, s) in tests { + group.bench_with_input(BenchmarkId::new(p.pattern, s), s, |b, s| { + b.iter(|| { + let (req, mut state) = p.state(s); + assert!(state.search(req)); + let (req, mut state) = p.state(s); + assert!(state.py_match(&req)); + let (mut req, mut state) = p.state(s); + req.match_all = true; + assert!(state.py_match(&req)); + let s2 = format!("{}{}{}", " ".repeat(10000), s, " ".repeat(10000)); + let (req, mut state) = p.state_range(s2.as_str(), 0..usize::MAX); + assert!(state.search(req)); + let (req, mut state) = p.state_range(s2.as_str(), 10000..usize::MAX); + assert!(state.py_match(&req)); + let (req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); + assert!(state.py_match(&req)); + let (mut req, mut state) = p.state_range(s2.as_str(), 10000..10000 + s.len()); + req.match_all = true; + assert!(state.py_match(&req)); + }); + }); + } +} + +criterion_group!(benches, basic); + +criterion_main!(benches); diff --git a/crates/sre_engine/generate_tests.py b/crates/sre_engine/generate_tests.py new file mode 100644 index 00000000000..542852a2496 --- /dev/null +++ b/crates/sre_engine/generate_tests.py @@ -0,0 +1,75 @@ +import json +import os +import re +from itertools import chain +from pathlib import Path + +m = re.search(r"const SRE_MAGIC: usize = (\d+);", open("src/constants.rs").read()) +sre_engine_magic = int(m.group(1)) +del m + +assert re._constants.MAGIC == sre_engine_magic + + +class CompiledPattern: + @classmethod + def compile(cls, pattern, flags=0): + p = re._parser.parse(pattern) + code = re._compiler._code(p, flags) + self = cls() + self.pattern = pattern + self.code = code + self.flags = re.RegexFlag(flags | p.state.flags) + return self + + +for k, v in re.RegexFlag.__members__.items(): + setattr(CompiledPattern, k, v) + + +class EscapeRustStr: + hardcoded = { + ord("\r"): "\\r", + ord("\t"): "\\t", + ord("\r"): "\\r", + ord("\n"): "\\n", + ord("\\"): "\\\\", + ord("'"): "\\'", + ord('"'): '\\"', + } + + @classmethod + def __class_getitem__(cls, ch): + if (rpl := cls.hardcoded.get(ch)) is not None: + return rpl + if ch in range(0x20, 0x7F): + return ch + return f"\\u{{{ch:x}}}" + + +def rust_str(s): + return '"' + s.translate(EscapeRustStr) + '"' + + +# matches `// pattern {varname} = re.compile(...)` +pattern_pattern = re.compile( + r"^((\s*)\/\/\s*pattern\s+(\w+)\s+=\s+(.+?))$(?:.+?END GENERATED)?", re.M | re.S +) + + +def replace_compiled(m): + line, indent, varname, pattern = m.groups() + pattern = eval(pattern, {"re": CompiledPattern}) + pattern = f"Pattern {{ pattern: {rust_str(pattern.pattern)}, code: &{json.dumps(pattern.code)} }}" + return f"""{line} +{indent}// START GENERATED by generate_tests.py +{indent}#[rustfmt::skip] let {varname} = {pattern}; +{indent}// END GENERATED""" + + +with os.scandir("tests") as t, os.scandir("benches") as b: + for f in chain(t, b): + path = Path(f.path) + if path.suffix == ".rs": + replaced = pattern_pattern.sub(replace_compiled, path.read_text()) + path.write_text(replaced) diff --git a/crates/sre_engine/src/constants.rs b/crates/sre_engine/src/constants.rs new file mode 100644 index 00000000000..b38ecb109b8 --- /dev/null +++ b/crates/sre_engine/src/constants.rs @@ -0,0 +1,130 @@ +/* + * Secret Labs' Regular Expression Engine + * + * regular expression matching engine + * + * Auto-generated by scripts/generate_sre_constants.py from + * Lib/re/_constants.py. + * + * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. + * + * See the sre.c file for information on usage and redistribution. + */ + +use bitflags::bitflags; + +pub const SRE_MAGIC: usize = 20230612; + +#[allow(non_camel_case_types, clippy::upper_case_acronyms)] +#[derive(num_enum::TryFromPrimitive, Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub enum SreOpcode { + FAILURE = 0, + SUCCESS = 1, + ANY = 2, + ANY_ALL = 3, + ASSERT = 4, + ASSERT_NOT = 5, + AT = 6, + BRANCH = 7, + CATEGORY = 8, + CHARSET = 9, + BIGCHARSET = 10, + GROUPREF = 11, + GROUPREF_EXISTS = 12, + IN = 13, + INFO = 14, + JUMP = 15, + LITERAL = 16, + MARK = 17, + MAX_UNTIL = 18, + MIN_UNTIL = 19, + NOT_LITERAL = 20, + NEGATE = 21, + RANGE = 22, + REPEAT = 23, + REPEAT_ONE = 24, + SUBPATTERN = 25, + MIN_REPEAT_ONE = 26, + ATOMIC_GROUP = 27, + POSSESSIVE_REPEAT = 28, + POSSESSIVE_REPEAT_ONE = 29, + GROUPREF_IGNORE = 30, + IN_IGNORE = 31, + LITERAL_IGNORE = 32, + NOT_LITERAL_IGNORE = 33, + GROUPREF_LOC_IGNORE = 34, + IN_LOC_IGNORE = 35, + LITERAL_LOC_IGNORE = 36, + NOT_LITERAL_LOC_IGNORE = 37, + GROUPREF_UNI_IGNORE = 38, + IN_UNI_IGNORE = 39, + LITERAL_UNI_IGNORE = 40, + NOT_LITERAL_UNI_IGNORE = 41, + RANGE_UNI_IGNORE = 42, +} + +#[allow(non_camel_case_types, clippy::upper_case_acronyms)] +#[derive(num_enum::TryFromPrimitive, Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] +pub enum SreAtCode { + BEGINNING = 0, + BEGINNING_LINE = 1, + BEGINNING_STRING = 2, + BOUNDARY = 3, + NON_BOUNDARY = 4, + END = 5, + END_LINE = 6, + END_STRING = 7, + LOC_BOUNDARY = 8, + LOC_NON_BOUNDARY = 9, + UNI_BOUNDARY = 10, + UNI_NON_BOUNDARY = 11, +} + +#[allow(non_camel_case_types, clippy::upper_case_acronyms)] +#[derive(num_enum::TryFromPrimitive, Clone, Copy, Debug)] +#[repr(u32)] +pub enum SreCatCode { + DIGIT = 0, + NOT_DIGIT = 1, + SPACE = 2, + NOT_SPACE = 3, + WORD = 4, + NOT_WORD = 5, + LINEBREAK = 6, + NOT_LINEBREAK = 7, + LOC_WORD = 8, + LOC_NOT_WORD = 9, + UNI_DIGIT = 10, + UNI_NOT_DIGIT = 11, + UNI_SPACE = 12, + UNI_NOT_SPACE = 13, + UNI_WORD = 14, + UNI_NOT_WORD = 15, + UNI_LINEBREAK = 16, + UNI_NOT_LINEBREAK = 17, +} + +bitflags! { +#[derive(Debug, PartialEq, Eq, Clone, Copy)] + pub struct SreFlag: u16 { + const IGNORECASE = 2; + const LOCALE = 4; + const MULTILINE = 8; + const DOTALL = 16; + const UNICODE = 32; + const VERBOSE = 64; + const DEBUG = 128; + const ASCII = 256; + } +} + +bitflags! { + #[derive(Clone, Copy)] + pub struct SreInfo: u32 { + const PREFIX = 1; + const LITERAL = 2; + const CHARSET = 4; + } +} diff --git a/crates/sre_engine/src/engine.rs b/crates/sre_engine/src/engine.rs new file mode 100644 index 00000000000..73e263012fc --- /dev/null +++ b/crates/sre_engine/src/engine.rs @@ -0,0 +1,1421 @@ +// good luck to those that follow; here be dragons + +use crate::string::{ + is_digit, is_linebreak, is_loc_word, is_space, is_uni_digit, is_uni_linebreak, is_uni_space, + is_uni_word, is_word, lower_ascii, lower_locate, lower_unicode, upper_locate, upper_unicode, +}; + +use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor}; +use alloc::{vec, vec::Vec}; +use core::{convert::TryFrom, ptr::null}; +use optional::Optioned; + +#[derive(Debug, Clone, Copy)] +pub struct Request<'a, S> { + pub string: S, + pub start: usize, + pub end: usize, + pub pattern_codes: &'a [u32], + pub match_all: bool, + pub must_advance: bool, +} + +impl<'a, S: StrDrive> Request<'a, S> { + pub fn new( + string: S, + start: usize, + end: usize, + pattern_codes: &'a [u32], + match_all: bool, + ) -> Self { + let end = core::cmp::min(end, string.count()); + let start = core::cmp::min(start, end); + + Self { + string, + start, + end, + pattern_codes, + match_all, + must_advance: false, + } + } +} + +#[derive(Debug)] +pub struct Marks { + last_index: isize, + marks: Vec<Optioned<usize>>, + marks_stack: Vec<(Vec<Optioned<usize>>, isize)>, +} + +impl Default for Marks { + fn default() -> Self { + Self { + last_index: -1, + marks: Vec::new(), + marks_stack: Vec::new(), + } + } +} + +impl Marks { + pub fn get(&self, group_index: usize) -> (Optioned<usize>, Optioned<usize>) { + let marks_index = 2 * group_index; + if marks_index + 1 < self.marks.len() { + (self.marks[marks_index], self.marks[marks_index + 1]) + } else { + (Optioned::none(), Optioned::none()) + } + } + + pub const fn last_index(&self) -> isize { + self.last_index + } + + pub fn raw(&self) -> &[Optioned<usize>] { + self.marks.as_slice() + } + + fn set(&mut self, mark_nr: usize, position: usize) { + if mark_nr & 1 != 0 { + self.last_index = mark_nr as isize / 2 + 1; + } + if mark_nr >= self.marks.len() { + self.marks.resize(mark_nr + 1, Optioned::none()); + } + self.marks[mark_nr] = Optioned::some(position); + } + + fn push(&mut self) { + self.marks_stack.push((self.marks.clone(), self.last_index)); + } + + fn pop(&mut self) { + let (marks, last_index) = self.marks_stack.pop().unwrap(); + self.marks = marks; + self.last_index = last_index; + } + + fn pop_keep(&mut self) { + let (marks, last_index) = self.marks_stack.last().unwrap().clone(); + self.marks = marks; + self.last_index = last_index; + } + + fn pop_discard(&mut self) { + self.marks_stack.pop(); + } + + fn clear(&mut self) { + self.last_index = -1; + self.marks.clear(); + self.marks_stack.clear(); + } +} + +#[derive(Debug, Default)] +pub struct State { + pub start: usize, + pub marks: Marks, + pub cursor: StringCursor, + repeat_stack: Vec<RepeatContext>, +} + +impl State { + pub fn reset<S: StrDrive>(&mut self, req: &Request<'_, S>, start: usize) { + self.marks.clear(); + self.repeat_stack.clear(); + self.start = start; + req.string.adjust_cursor(&mut self.cursor, start); + } + + pub fn py_match<S: StrDrive>(&mut self, req: &Request<'_, S>) -> bool { + self.start = req.start; + req.string.adjust_cursor(&mut self.cursor, self.start); + + let ctx = MatchContext { + cursor: self.cursor, + code_position: 0, + toplevel: true, + jump: Jump::OpCode, + repeat_ctx_id: usize::MAX, + count: -1, + }; + _match(req, self, ctx) + } + + pub fn search<S: StrDrive>(&mut self, mut req: Request<'_, S>) -> bool { + self.start = req.start; + req.string.adjust_cursor(&mut self.cursor, self.start); + + if req.start > req.end { + return false; + } + + let mut end = req.end; + + let mut ctx = MatchContext { + cursor: self.cursor, + code_position: 0, + toplevel: true, + jump: Jump::OpCode, + repeat_ctx_id: usize::MAX, + count: -1, + }; + + if ctx.peek_code(&req, 0) == SreOpcode::INFO as u32 { + /* optimization info block */ + /* <INFO> <1=skip> <2=flags> <3=min> <4=max> <5=prefix info> */ + let min = ctx.peek_code(&req, 3) as usize; + + if ctx.remaining_chars(&req) < min { + return false; + } + + if min > 1 { + /* adjust end point (but make sure we leave at least one + character in there, so literal search will work) */ + // no overflow can happen as remaining chars >= min + end -= min - 1; + + // adjust ctx position + if end < ctx.cursor.position { + let skip = end - self.cursor.position; + S::skip(&mut self.cursor, skip); + } + } + + let flags = SreInfo::from_bits_truncate(ctx.peek_code(&req, 2)); + + if flags.contains(SreInfo::PREFIX) { + if flags.contains(SreInfo::LITERAL) { + return search_info_literal::<true, S>(&mut req, self, ctx); + } else { + return search_info_literal::<false, S>(&mut req, self, ctx); + } + } else if flags.contains(SreInfo::CHARSET) { + return search_info_charset(&mut req, self, ctx); + } + // fallback to general search + // skip OP INFO + ctx.skip_code_from(&req, 1); + } + + if _match(&req, self, ctx) { + return true; + } + + if ctx.try_peek_code_as::<SreOpcode, _>(&req, 0).unwrap() == SreOpcode::AT + && (ctx.try_peek_code_as::<SreAtCode, _>(&req, 1).unwrap() == SreAtCode::BEGINNING + || ctx.try_peek_code_as::<SreAtCode, _>(&req, 1).unwrap() + == SreAtCode::BEGINNING_STRING) + { + self.cursor.position = req.end; + self.cursor.ptr = null(); + // self.reset(&req, req.end); + return false; + } + + req.must_advance = false; + ctx.toplevel = false; + while req.start < end { + req.start += 1; + self.reset(&req, req.start); + ctx.cursor = self.cursor; + + if _match(&req, self, ctx) { + return true; + } + } + false + } +} + +pub struct SearchIter<'a, S: StrDrive> { + pub req: Request<'a, S>, + pub state: State, +} + +impl<S: StrDrive> Iterator for SearchIter<'_, S> { + type Item = (); + + fn next(&mut self) -> Option<Self::Item> { + if self.req.start > self.req.end { + return None; + } + + self.state.reset(&self.req, self.req.start); + if !self.state.search(self.req) { + return None; + } + + self.req.must_advance = self.state.cursor.position == self.state.start; + self.req.start = self.state.cursor.position; + + Some(()) + } +} + +#[derive(Debug, Clone, Copy)] +enum Jump { + OpCode, + Assert1, + AssertNot1, + Branch1, + Branch2, + Repeat1, + UntilBacktrace, + MaxUntil2, + MaxUntil3, + MinUntil1, + RepeatOne1, + RepeatOne2, + MinRepeatOne1, + MinRepeatOne2, + AtomicGroup1, + PossessiveRepeat1, + PossessiveRepeat2, + PossessiveRepeat3, + PossessiveRepeat4, +} + +fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchContext) -> bool { + let mut context_stack = vec![]; + let mut popped_result = false; + + #[allow( + clippy::never_loop, + reason = "'result loop is not an actual loop but break label" + )] + 'coro: loop { + popped_result = 'result: loop { + let yielded = 'context: loop { + match ctx.jump { + Jump::OpCode => {} + Jump::Assert1 => { + if popped_result { + ctx.skip_code_from(req, 1); + } else { + break 'result false; + } + } + Jump::AssertNot1 => { + if popped_result { + break 'result false; + } + state.marks.pop(); + ctx.skip_code_from(req, 1); + } + Jump::Branch1 => { + let branch_offset = ctx.count as usize; + let next_length = ctx.peek_code(req, branch_offset) as isize; + if next_length == 0 { + state.marks.pop_discard(); + break 'result false; + } + state.cursor = ctx.cursor; + let next_ctx = ctx.next_offset(branch_offset + 1, Jump::Branch2); + ctx.count += next_length; + break 'context next_ctx; + } + Jump::Branch2 => { + if popped_result { + break 'result true; + } + state.marks.pop_keep(); + ctx.jump = Jump::Branch1; + continue 'context; + } + Jump::Repeat1 => { + state.repeat_stack.pop(); + break 'result popped_result; + } + Jump::UntilBacktrace => { + if !popped_result { + state.repeat_stack[ctx.repeat_ctx_id].count -= 1; + state.cursor = ctx.cursor; + } + break 'result popped_result; + } + Jump::MaxUntil2 => { + let save_last_position = ctx.count as usize; + let repeat_ctx = &mut state.repeat_stack[ctx.repeat_ctx_id]; + repeat_ctx.last_position = save_last_position; + + if popped_result { + state.marks.pop_discard(); + break 'result true; + } + + state.marks.pop(); + repeat_ctx.count -= 1; + state.cursor = ctx.cursor; + + /* cannot match more repeated items here. make sure the + tail matches */ + let mut next_ctx = ctx.next_offset(1, Jump::MaxUntil3); + next_ctx.repeat_ctx_id = repeat_ctx.prev_id; + break 'context next_ctx; + } + Jump::MaxUntil3 => { + if !popped_result { + state.cursor = ctx.cursor; + } + break 'result popped_result; + } + Jump::MinUntil1 => { + if popped_result { + break 'result true; + } + ctx.repeat_ctx_id = ctx.count as usize; + let repeat_ctx = &mut state.repeat_stack[ctx.repeat_ctx_id]; + state.cursor = ctx.cursor; + state.marks.pop(); + + // match more until tail matches + if repeat_ctx.count as usize >= repeat_ctx.max_count + && repeat_ctx.max_count != MAXREPEAT + || state.cursor.position == repeat_ctx.last_position + { + repeat_ctx.count -= 1; + break 'result false; + } + + /* zero-width match protection */ + repeat_ctx.last_position = state.cursor.position; + + break 'context ctx + .next_at(repeat_ctx.code_position + 4, Jump::UntilBacktrace); + } + Jump::RepeatOne1 => { + let min_count = ctx.peek_code(req, 2) as isize; + let next_code = ctx.peek_code(req, ctx.peek_code(req, 1) as usize + 1); + if next_code == SreOpcode::LITERAL as u32 { + // Special case: Tail starts with a literal. Skip positions where + // the rest of the pattern cannot possibly match. + let c = ctx.peek_code(req, ctx.peek_code(req, 1) as usize + 2); + while ctx.at_end(req) || ctx.peek_char::<S>() != c { + if ctx.count <= min_count { + state.marks.pop_discard(); + break 'result false; + } + ctx.back_advance_char::<S>(); + ctx.count -= 1; + } + } + + state.cursor = ctx.cursor; + // General case: backtracking + break 'context ctx.next_peek_from(1, req, Jump::RepeatOne2); + } + Jump::RepeatOne2 => { + if popped_result { + break 'result true; + } + + let min_count = ctx.peek_code(req, 2) as isize; + if ctx.count <= min_count { + state.marks.pop_discard(); + break 'result false; + } + + ctx.back_advance_char::<S>(); + ctx.count -= 1; + + state.marks.pop_keep(); + ctx.jump = Jump::RepeatOne1; + continue 'context; + } + Jump::MinRepeatOne1 => { + let max_count = ctx.peek_code(req, 3) as usize; + if max_count == MAXREPEAT || ctx.count as usize <= max_count { + state.cursor = ctx.cursor; + break 'context ctx.next_peek_from(1, req, Jump::MinRepeatOne2); + } else { + state.marks.pop_discard(); + break 'result false; + } + } + Jump::MinRepeatOne2 => { + if popped_result { + break 'result true; + } + + state.cursor = ctx.cursor; + + let mut count_ctx = ctx; + count_ctx.skip_code(4); + if _count(req, state, &mut count_ctx, 1) == 0 { + state.marks.pop_discard(); + break 'result false; + } + + ctx.advance_char::<S>(); + ctx.count += 1; + state.marks.pop_keep(); + ctx.jump = Jump::MinRepeatOne1; + continue 'context; + } + Jump::AtomicGroup1 => { + if popped_result { + ctx.skip_code_from(req, 1); + ctx.cursor = state.cursor; + // dispatch opcode + } else { + state.cursor = ctx.cursor; + break 'result false; + } + } + Jump::PossessiveRepeat1 => { + let min_count = ctx.peek_code(req, 2) as isize; + if ctx.count < min_count { + // modified next.toplevel from inherited to false + let mut next = ctx.next_offset(4, Jump::PossessiveRepeat2); + next.toplevel = false; + break 'context next; + } + // zero match protection + ctx.cursor.position = usize::MAX; + ctx.jump = Jump::PossessiveRepeat3; + continue 'context; + } + Jump::PossessiveRepeat2 => { + if popped_result { + ctx.count += 1; + ctx.jump = Jump::PossessiveRepeat1; + continue 'context; + } else { + state.cursor = ctx.cursor; + break 'result false; + } + } + Jump::PossessiveRepeat3 => { + let max_count = ctx.peek_code(req, 3) as usize; + if ((ctx.count as usize) < max_count || max_count == MAXREPEAT) + && ctx.cursor.position != state.cursor.position + { + state.marks.push(); + ctx.cursor = state.cursor; + let mut next = ctx.next_offset(4, Jump::PossessiveRepeat4); + next.toplevel = false; // modified next.toplevel from inherited to false + break 'context next; + } + ctx.cursor = state.cursor; + ctx.skip_code_from(req, 1); + ctx.skip_code(1); + } + Jump::PossessiveRepeat4 => { + if popped_result { + state.marks.pop_discard(); + ctx.count += 1; + ctx.jump = Jump::PossessiveRepeat3; + continue 'context; + } + state.marks.pop(); + state.cursor = ctx.cursor; + ctx.skip_code_from(req, 1); + ctx.skip_code(1); + } + } + ctx.jump = Jump::OpCode; + + loop { + macro_rules! general_op_literal { + ($f:expr) => {{ + #[allow(clippy::redundant_closure_call)] + if ctx.at_end(req) || !$f(ctx.peek_code(req, 1), ctx.peek_char::<S>()) { + break 'result false; + } + ctx.skip_code(2); + ctx.advance_char::<S>(); + }}; + } + + macro_rules! general_op_in { + ($f:expr) => {{ + #[allow(clippy::redundant_closure_call)] + if ctx.at_end(req) || !$f(&ctx.pattern(req)[2..], ctx.peek_char::<S>()) + { + break 'result false; + } + ctx.skip_code_from(req, 1); + ctx.advance_char::<S>(); + }}; + } + + macro_rules! general_op_groupref { + ($f:expr) => {{ + let (group_start, group_end) = + state.marks.get(ctx.peek_code(req, 1) as usize); + let (group_start, group_end) = if group_start.is_some() + && group_end.is_some() + && group_start.unpack() <= group_end.unpack() + { + (group_start.unpack(), group_end.unpack()) + } else { + break 'result false; + }; + + let mut g_ctx = MatchContext { + cursor: req.string.create_cursor(group_start), + ..ctx + }; + + for _ in group_start..group_end { + #[allow(clippy::redundant_closure_call)] + if ctx.at_end(req) + || $f(ctx.peek_char::<S>()) != $f(g_ctx.peek_char::<S>()) + { + break 'result false; + } + ctx.advance_char::<S>(); + g_ctx.advance_char::<S>(); + } + + ctx.skip_code(2); + }}; + } + + if ctx.remaining_codes(req) == 0 { + break 'result false; + } + let opcode = ctx.peek_code(req, 0); + let opcode = SreOpcode::try_from(opcode).unwrap(); + + match opcode { + SreOpcode::FAILURE => break 'result false, + SreOpcode::SUCCESS => { + if ctx.can_success(req) { + state.cursor = ctx.cursor; + break 'result true; + } + break 'result false; + } + SreOpcode::ANY => { + if ctx.at_end(req) || ctx.at_linebreak(req) { + break 'result false; + } + ctx.skip_code(1); + ctx.advance_char::<S>(); + } + SreOpcode::ANY_ALL => { + if ctx.at_end(req) { + break 'result false; + } + ctx.skip_code(1); + ctx.advance_char::<S>(); + } + /* <ASSERT> <skip> <back> <pattern> */ + SreOpcode::ASSERT => { + let back = ctx.peek_code(req, 2) as usize; + if ctx.cursor.position < back { + break 'result false; + } + + let mut next_ctx = ctx.next_offset(3, Jump::Assert1); + next_ctx.toplevel = false; + next_ctx.back_skip_char::<S>(back); + state.cursor = next_ctx.cursor; + break 'context next_ctx; + } + /* <ASSERT_NOT> <skip> <back> <pattern> */ + SreOpcode::ASSERT_NOT => { + let back = ctx.peek_code(req, 2) as usize; + if ctx.cursor.position < back { + ctx.skip_code_from(req, 1); + continue; + } + state.marks.push(); + + let mut next_ctx = ctx.next_offset(3, Jump::AssertNot1); + next_ctx.toplevel = false; + next_ctx.back_skip_char::<S>(back); + state.cursor = next_ctx.cursor; + break 'context next_ctx; + } + SreOpcode::AT => { + let at_code = SreAtCode::try_from(ctx.peek_code(req, 1)).unwrap(); + if at(req, &ctx, at_code) { + ctx.skip_code(2); + } else { + break 'result false; + } + } + // <BRANCH> <0=skip> code <JUMP> ... <NULL> + SreOpcode::BRANCH => { + state.marks.push(); + ctx.count = 1; + ctx.jump = Jump::Branch1; + continue 'context; + } + SreOpcode::CATEGORY => { + let cat_code = SreCatCode::try_from(ctx.peek_code(req, 1)).unwrap(); + if ctx.at_end(req) || !category(cat_code, ctx.peek_char::<S>()) { + break 'result false; + } + ctx.skip_code(2); + ctx.advance_char::<S>(); + } + SreOpcode::IN => general_op_in!(charset), + SreOpcode::IN_IGNORE => { + general_op_in!(|set, c| charset(set, lower_ascii(c))) + } + SreOpcode::IN_UNI_IGNORE => { + general_op_in!(|set, c| charset(set, lower_unicode(c))) + } + SreOpcode::IN_LOC_IGNORE => general_op_in!(charset_loc_ignore), + SreOpcode::MARK => { + state + .marks + .set(ctx.peek_code(req, 1) as usize, ctx.cursor.position); + ctx.skip_code(2); + } + SreOpcode::INFO | SreOpcode::JUMP => ctx.skip_code_from(req, 1), + /* <REPEAT> <skip> <1=min> <2=max> item <UNTIL> tail */ + SreOpcode::REPEAT => { + let repeat_ctx = RepeatContext { + count: -1, + min_count: ctx.peek_code(req, 2) as usize, + max_count: ctx.peek_code(req, 3) as usize, + code_position: ctx.code_position, + last_position: usize::MAX, + prev_id: ctx.repeat_ctx_id, + }; + state.repeat_stack.push(repeat_ctx); + let repeat_ctx_id = state.repeat_stack.len() - 1; + state.cursor = ctx.cursor; + let mut next_ctx = ctx.next_peek_from(1, req, Jump::Repeat1); + next_ctx.repeat_ctx_id = repeat_ctx_id; + break 'context next_ctx; + } + SreOpcode::MAX_UNTIL => { + let repeat_ctx = &mut state.repeat_stack[ctx.repeat_ctx_id]; + state.cursor = ctx.cursor; + repeat_ctx.count += 1; + + if (repeat_ctx.count as usize) < repeat_ctx.min_count { + // not enough matches + break 'context ctx + .next_at(repeat_ctx.code_position + 4, Jump::UntilBacktrace); + } + + if ((repeat_ctx.count as usize) < repeat_ctx.max_count + || repeat_ctx.max_count == MAXREPEAT) + && state.cursor.position != repeat_ctx.last_position + { + /* we may have enough matches, but if we can + match another item, do so */ + state.marks.push(); + ctx.count = repeat_ctx.last_position as isize; + repeat_ctx.last_position = state.cursor.position; + + break 'context ctx + .next_at(repeat_ctx.code_position + 4, Jump::MaxUntil2); + } + + /* cannot match more repeated items here. make sure the + tail matches */ + let mut next_ctx = ctx.next_offset(1, Jump::MaxUntil3); + next_ctx.repeat_ctx_id = repeat_ctx.prev_id; + break 'context next_ctx; + } + SreOpcode::MIN_UNTIL => { + let repeat_ctx = state.repeat_stack.last_mut().unwrap(); + state.cursor = ctx.cursor; + repeat_ctx.count += 1; + + if (repeat_ctx.count as usize) < repeat_ctx.min_count { + // not enough matches + break 'context ctx + .next_at(repeat_ctx.code_position + 4, Jump::UntilBacktrace); + } + + state.marks.push(); + ctx.count = ctx.repeat_ctx_id as isize; + let mut next_ctx = ctx.next_offset(1, Jump::MinUntil1); + next_ctx.repeat_ctx_id = repeat_ctx.prev_id; + break 'context next_ctx; + } + /* <REPEAT_ONE> <skip> <1=min> <2=max> item <SUCCESS> tail */ + SreOpcode::REPEAT_ONE => { + let min_count = ctx.peek_code(req, 2) as usize; + let max_count = ctx.peek_code(req, 3) as usize; + + if ctx.remaining_chars(req) < min_count { + break 'result false; + } + + state.cursor = ctx.cursor; + + let mut count_ctx = ctx; + count_ctx.skip_code(4); + let count = _count(req, state, &mut count_ctx, max_count); + if count < min_count { + break 'result false; + } + ctx.cursor = count_ctx.cursor; + + let next_code = ctx.peek_code(req, ctx.peek_code(req, 1) as usize + 1); + if next_code == SreOpcode::SUCCESS as u32 && ctx.can_success(req) { + // tail is empty. we're finished + state.cursor = ctx.cursor; + break 'result true; + } + + state.marks.push(); + ctx.count = count as isize; + ctx.jump = Jump::RepeatOne1; + continue 'context; + } + /* <MIN_REPEAT_ONE> <skip> <1=min> <2=max> item <SUCCESS> tail */ + SreOpcode::MIN_REPEAT_ONE => { + let min_count = ctx.peek_code(req, 2) as usize; + if ctx.remaining_chars(req) < min_count { + break 'result false; + } + + state.cursor = ctx.cursor; + ctx.count = if min_count == 0 { + 0 + } else { + let mut count_ctx = ctx; + count_ctx.skip_code(4); + let count = _count(req, state, &mut count_ctx, min_count); + if count < min_count { + break 'result false; + } + ctx.cursor = count_ctx.cursor; + count as isize + }; + + let next_code = ctx.peek_code(req, ctx.peek_code(req, 1) as usize + 1); + if next_code == SreOpcode::SUCCESS as u32 && ctx.can_success(req) { + // tail is empty. we're finished + state.cursor = ctx.cursor; + break 'result true; + } + + state.marks.push(); + ctx.jump = Jump::MinRepeatOne1; + continue 'context; + } + SreOpcode::LITERAL => general_op_literal!(|code, c| code == c), + SreOpcode::NOT_LITERAL => general_op_literal!(|code, c| code != c), + SreOpcode::LITERAL_IGNORE => { + general_op_literal!(|code, c| code == lower_ascii(c)) + } + SreOpcode::NOT_LITERAL_IGNORE => { + general_op_literal!(|code, c| code != lower_ascii(c)) + } + SreOpcode::LITERAL_UNI_IGNORE => { + general_op_literal!(|code, c| code == lower_unicode(c)) + } + SreOpcode::NOT_LITERAL_UNI_IGNORE => { + general_op_literal!(|code, c| code != lower_unicode(c)) + } + SreOpcode::LITERAL_LOC_IGNORE => general_op_literal!(char_loc_ignore), + SreOpcode::NOT_LITERAL_LOC_IGNORE => { + general_op_literal!(|code, c| !char_loc_ignore(code, c)) + } + SreOpcode::GROUPREF => general_op_groupref!(|x| x), + SreOpcode::GROUPREF_IGNORE => general_op_groupref!(lower_ascii), + SreOpcode::GROUPREF_LOC_IGNORE => general_op_groupref!(lower_locate), + SreOpcode::GROUPREF_UNI_IGNORE => general_op_groupref!(lower_unicode), + SreOpcode::GROUPREF_EXISTS => { + let (group_start, group_end) = + state.marks.get(ctx.peek_code(req, 1) as usize); + if group_start.is_some() + && group_end.is_some() + && group_start.unpack() <= group_end.unpack() + { + ctx.skip_code(3); + } else { + ctx.skip_code_from(req, 2) + } + } + /* <ATOMIC_GROUP> <skip> pattern <SUCCESS> tail */ + SreOpcode::ATOMIC_GROUP => { + state.cursor = ctx.cursor; + let mut next_ctx = ctx.next_offset(2, Jump::AtomicGroup1); + next_ctx.toplevel = false; // modified next.toplevel from inherited to false + break 'context next_ctx; + } + /* <POSSESSIVE_REPEAT> <skip> <1=min> <2=max> pattern + <SUCCESS> tail */ + SreOpcode::POSSESSIVE_REPEAT => { + state.cursor = ctx.cursor; + ctx.count = 0; + ctx.jump = Jump::PossessiveRepeat1; + continue 'context; + } + /* <POSSESSIVE_REPEAT_ONE> <skip> <1=min> <2=max> item <SUCCESS> + tail */ + SreOpcode::POSSESSIVE_REPEAT_ONE => { + let min_count = ctx.peek_code(req, 2) as usize; + let max_count = ctx.peek_code(req, 3) as usize; + if ctx.remaining_chars(req) < min_count { + break 'result false; + } + state.cursor = ctx.cursor; + let mut count_ctx = ctx; + count_ctx.skip_code(4); + let count = _count(req, state, &mut count_ctx, max_count); + if count < min_count { + break 'result false; + } + ctx.cursor = count_ctx.cursor; + ctx.skip_code_from(req, 1); + } + SreOpcode::CHARSET + | SreOpcode::BIGCHARSET + | SreOpcode::NEGATE + | SreOpcode::RANGE + | SreOpcode::RANGE_UNI_IGNORE + | SreOpcode::SUBPATTERN => { + unreachable!("unexpected opcode on main dispatch") + } + } + } + }; + context_stack.push(ctx); + ctx = yielded; + continue 'coro; + }; + if let Some(popped_ctx) = context_stack.pop() { + ctx = popped_ctx; + } else { + break; + } + } + popped_result +} + +fn search_info_literal<const LITERAL: bool, S: StrDrive>( + req: &mut Request<'_, S>, + state: &mut State, + mut ctx: MatchContext, +) -> bool { + /* pattern starts with a known prefix */ + /* <length> <skip> <prefix data> <overlap data> */ + let len = ctx.peek_code(req, 5) as usize; + let skip = ctx.peek_code(req, 6) as usize; + let prefix = &ctx.pattern(req)[7..7 + len]; + let overlap = &ctx.pattern(req)[7 + len - 1..7 + len * 2]; + + // code_position ready for tail match + ctx.skip_code_from(req, 1); + ctx.skip_code(2 * skip); + + req.must_advance = false; + + if len == 1 { + // pattern starts with a literal character + let c = prefix[0]; + + while !ctx.at_end(req) { + // find the next matched literal + while ctx.peek_char::<S>() != c { + ctx.advance_char::<S>(); + if ctx.at_end(req) { + return false; + } + } + + req.start = ctx.cursor.position; + state.start = req.start; + state.cursor = ctx.cursor; + S::skip(&mut state.cursor, skip); + + // literal only + if LITERAL { + return true; + } + + let mut next_ctx = ctx; + next_ctx.skip_char::<S>(skip); + + if _match(req, state, next_ctx) { + return true; + } + + ctx.advance_char::<S>(); + state.marks.clear(); + } + } else { + while !ctx.at_end(req) { + let c = prefix[0]; + while ctx.peek_char::<S>() != c { + ctx.advance_char::<S>(); + if ctx.at_end(req) { + return false; + } + } + ctx.advance_char::<S>(); + if ctx.at_end(req) { + return false; + } + + let mut i = 1; + loop { + if ctx.peek_char::<S>() == prefix[i] { + i += 1; + if i != len { + ctx.advance_char::<S>(); + if ctx.at_end(req) { + return false; + } + continue; + } + + req.start = ctx.cursor.position - (len - 1); + state.reset(req, req.start); + S::skip(&mut state.cursor, skip); + // state.start = req.start; + // state.cursor = req.string.create_cursor(req.start + skip); + + // literal only + if LITERAL { + return true; + } + + let mut next_ctx = ctx; + if skip != 0 { + next_ctx.advance_char::<S>(); + } else { + next_ctx.cursor = state.cursor; + } + + if _match(req, state, next_ctx) { + return true; + } + + ctx.advance_char::<S>(); + if ctx.at_end(req) { + return false; + } + state.marks.clear(); + } + + i = overlap[i] as usize; + if i == 0 { + break; + } + } + } + } + false +} + +fn search_info_charset<S: StrDrive>( + req: &mut Request<'_, S>, + state: &mut State, + mut ctx: MatchContext, +) -> bool { + let set = &ctx.pattern(req)[5..]; + + ctx.skip_code_from(req, 1); + + req.must_advance = false; + + loop { + while !ctx.at_end(req) && !charset(set, ctx.peek_char::<S>()) { + ctx.advance_char::<S>(); + } + if ctx.at_end(req) { + return false; + } + + req.start = ctx.cursor.position; + state.start = ctx.cursor.position; + state.cursor = ctx.cursor; + + if _match(req, state, ctx) { + return true; + } + + ctx.advance_char::<S>(); + state.marks.clear(); + } +} + +#[derive(Debug, Clone, Copy)] +struct RepeatContext { + count: isize, + min_count: usize, + max_count: usize, + code_position: usize, + last_position: usize, + prev_id: usize, +} + +#[derive(Clone, Copy)] +struct MatchContext { + cursor: StringCursor, + code_position: usize, + toplevel: bool, + jump: Jump, + repeat_ctx_id: usize, + count: isize, +} + +impl MatchContext { + fn pattern<'a, S>(&self, req: &Request<'a, S>) -> &'a [u32] { + &req.pattern_codes[self.code_position..] + } + + const fn remaining_codes<S>(&self, req: &Request<'_, S>) -> usize { + req.pattern_codes.len() - self.code_position + } + + const fn remaining_chars<S>(&self, req: &Request<'_, S>) -> usize { + req.end - self.cursor.position + } + + fn peek_char<S: StrDrive>(&self) -> u32 { + S::peek(&self.cursor) + } + + fn skip_char<S: StrDrive>(&mut self, skip: usize) { + S::skip(&mut self.cursor, skip); + } + + fn advance_char<S: StrDrive>(&mut self) -> u32 { + S::advance(&mut self.cursor) + } + + fn back_peek_char<S: StrDrive>(&self) -> u32 { + S::back_peek(&self.cursor) + } + + fn back_skip_char<S: StrDrive>(&mut self, skip: usize) { + S::back_skip(&mut self.cursor, skip); + } + + fn back_advance_char<S: StrDrive>(&mut self) -> u32 { + S::back_advance(&mut self.cursor) + } + + fn peek_code<S>(&self, req: &Request<'_, S>, peek: usize) -> u32 { + req.pattern_codes[self.code_position + peek] + } + + fn try_peek_code_as<T, S>(&self, req: &Request<'_, S>, peek: usize) -> Result<T, T::Error> + where + T: TryFrom<u32>, + { + self.peek_code(req, peek).try_into() + } + + const fn skip_code(&mut self, skip: usize) { + self.code_position += skip; + } + + fn skip_code_from<S>(&mut self, req: &Request<'_, S>, peek: usize) { + self.skip_code(self.peek_code(req, peek) as usize + 1); + } + + const fn at_beginning(&self) -> bool { + // self.ctx().string_position == self.state().start + self.cursor.position == 0 + } + + const fn at_end<S>(&self, req: &Request<'_, S>) -> bool { + self.cursor.position == req.end + } + + fn at_linebreak<S: StrDrive>(&self, req: &Request<'_, S>) -> bool { + !self.at_end(req) && is_linebreak(self.peek_char::<S>()) + } + + fn at_boundary<S: StrDrive, F: FnMut(u32) -> bool>( + &self, + req: &Request<'_, S>, + mut word_checker: F, + ) -> bool { + if self.at_beginning() && self.at_end(req) { + return false; + } + let that = !self.at_beginning() && word_checker(self.back_peek_char::<S>()); + let this = !self.at_end(req) && word_checker(self.peek_char::<S>()); + this != that + } + + fn at_non_boundary<S: StrDrive, F: FnMut(u32) -> bool>( + &self, + req: &Request<'_, S>, + mut word_checker: F, + ) -> bool { + if self.at_beginning() && self.at_end(req) { + return false; + } + let that = !self.at_beginning() && word_checker(self.back_peek_char::<S>()); + let this = !self.at_end(req) && word_checker(self.peek_char::<S>()); + this == that + } + + const fn can_success<S>(&self, req: &Request<'_, S>) -> bool { + if !self.toplevel { + return true; + } + if req.match_all && !self.at_end(req) { + return false; + } + if req.must_advance && self.cursor.position == req.start { + return false; + } + true + } + + #[must_use] + fn next_peek_from<S>(&mut self, peek: usize, req: &Request<'_, S>, jump: Jump) -> Self { + self.next_offset(self.peek_code(req, peek) as usize + 1, jump) + } + + #[must_use] + const fn next_offset(&mut self, offset: usize, jump: Jump) -> Self { + self.next_at(self.code_position + offset, jump) + } + + #[must_use] + const fn next_at(&mut self, code_position: usize, jump: Jump) -> Self { + self.jump = jump; + Self { + code_position, + jump: Jump::OpCode, + count: -1, + ..*self + } + } +} + +fn at<S: StrDrive>(req: &Request<'_, S>, ctx: &MatchContext, at_code: SreAtCode) -> bool { + match at_code { + SreAtCode::BEGINNING | SreAtCode::BEGINNING_STRING => ctx.at_beginning(), + SreAtCode::BEGINNING_LINE => ctx.at_beginning() || is_linebreak(ctx.back_peek_char::<S>()), + SreAtCode::BOUNDARY => ctx.at_boundary(req, is_word), + SreAtCode::NON_BOUNDARY => ctx.at_non_boundary(req, is_word), + SreAtCode::END => { + (ctx.remaining_chars(req) == 1 && ctx.at_linebreak(req)) || ctx.at_end(req) + } + SreAtCode::END_LINE => ctx.at_linebreak(req) || ctx.at_end(req), + SreAtCode::END_STRING => ctx.at_end(req), + SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, is_loc_word), + SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, is_loc_word), + SreAtCode::UNI_BOUNDARY => ctx.at_boundary(req, is_uni_word), + SreAtCode::UNI_NON_BOUNDARY => ctx.at_non_boundary(req, is_uni_word), + } +} + +fn char_loc_ignore(code: u32, c: u32) -> bool { + code == c || code == lower_locate(c) || code == upper_locate(c) +} + +fn charset_loc_ignore(set: &[u32], c: u32) -> bool { + let lo = lower_locate(c); + if charset(set, c) { + return true; + } + let up = upper_locate(c); + up != lo && charset(set, up) +} + +fn category(cat_code: SreCatCode, c: u32) -> bool { + match cat_code { + SreCatCode::DIGIT => is_digit(c), + SreCatCode::NOT_DIGIT => !is_digit(c), + SreCatCode::SPACE => is_space(c), + SreCatCode::NOT_SPACE => !is_space(c), + SreCatCode::WORD => is_word(c), + SreCatCode::NOT_WORD => !is_word(c), + SreCatCode::LINEBREAK => is_linebreak(c), + SreCatCode::NOT_LINEBREAK => !is_linebreak(c), + SreCatCode::LOC_WORD => is_loc_word(c), + SreCatCode::LOC_NOT_WORD => !is_loc_word(c), + SreCatCode::UNI_DIGIT => is_uni_digit(c), + SreCatCode::UNI_NOT_DIGIT => !is_uni_digit(c), + SreCatCode::UNI_SPACE => is_uni_space(c), + SreCatCode::UNI_NOT_SPACE => !is_uni_space(c), + SreCatCode::UNI_WORD => is_uni_word(c), + SreCatCode::UNI_NOT_WORD => !is_uni_word(c), + SreCatCode::UNI_LINEBREAK => is_uni_linebreak(c), + SreCatCode::UNI_NOT_LINEBREAK => !is_uni_linebreak(c), + } +} + +fn charset(set: &[u32], ch: u32) -> bool { + /* check if character is a member of the given set */ + let mut ok = true; + let mut i = 0; + while i < set.len() { + let opcode = match SreOpcode::try_from(set[i]) { + Ok(code) => code, + Err(_) => { + break; + } + }; + match opcode { + SreOpcode::FAILURE => { + return !ok; + } + SreOpcode::CATEGORY => { + /* <CATEGORY> <code> */ + let cat_code = match SreCatCode::try_from(set[i + 1]) { + Ok(code) => code, + Err(_) => { + break; + } + }; + if category(cat_code, ch) { + return ok; + } + i += 2; + } + SreOpcode::CHARSET => { + /* <CHARSET> <bitmap> */ + let set = &set[i + 1..]; + if ch < 256 && ((set[(ch >> 5) as usize] & (1u32 << (ch & 31))) != 0) { + return ok; + } + i += 1 + 8; + } + SreOpcode::BIGCHARSET => { + /* <BIGCHARSET> <block_count> <256 block_indices> <blocks> */ + let count = set[i + 1] as usize; + if ch < 0x10000 { + let set = &set[i + 2..]; + let block_index = ch >> 8; + let (_, block_indices, _) = unsafe { set.align_to::<u8>() }; + let blocks = &set[64..]; + let block = block_indices[block_index as usize]; + if blocks[((block as u32 * 256 + (ch & 255)) / 32) as usize] + & (1u32 << (ch & 31)) + != 0 + { + return ok; + } + } + i += 2 + 64 + count * 8; + } + SreOpcode::LITERAL => { + /* <LITERAL> <code> */ + if ch == set[i + 1] { + return ok; + } + i += 2; + } + SreOpcode::NEGATE => { + ok = !ok; + i += 1; + } + SreOpcode::RANGE => { + /* <RANGE> <lower> <upper> */ + if set[i + 1] <= ch && ch <= set[i + 2] { + return ok; + } + i += 3; + } + SreOpcode::RANGE_UNI_IGNORE => { + /* <RANGE_UNI_IGNORE> <lower> <upper> */ + if set[i + 1] <= ch && ch <= set[i + 2] { + return ok; + } + let ch = upper_unicode(ch); + if set[i + 1] <= ch && ch <= set[i + 2] { + return ok; + } + i += 3; + } + _ => { + break; + } + } + } + /* internal error -- there's not much we can do about it + here, so let's just pretend it didn't match... */ + false +} + +fn _count<S: StrDrive>( + req: &Request<'_, S>, + state: &mut State, + ctx: &mut MatchContext, + max_count: usize, +) -> usize { + let max_count = core::cmp::min(max_count, ctx.remaining_chars(req)); + let end = ctx.cursor.position + max_count; + let opcode = SreOpcode::try_from(ctx.peek_code(req, 0)).unwrap(); + + match opcode { + SreOpcode::ANY => { + while ctx.cursor.position < end && !ctx.at_linebreak(req) { + ctx.advance_char::<S>(); + } + } + SreOpcode::ANY_ALL => { + ctx.skip_char::<S>(max_count); + } + SreOpcode::IN => { + while ctx.cursor.position < end && charset(&ctx.pattern(req)[2..], ctx.peek_char::<S>()) + { + ctx.advance_char::<S>(); + } + } + SreOpcode::LITERAL => { + general_count_literal(req, ctx, end, |code, c| code == c); + } + SreOpcode::NOT_LITERAL => { + general_count_literal(req, ctx, end, |code, c| code != c); + } + SreOpcode::LITERAL_IGNORE => { + general_count_literal(req, ctx, end, |code, c| code == lower_ascii(c)); + } + SreOpcode::NOT_LITERAL_IGNORE => { + general_count_literal(req, ctx, end, |code, c| code != lower_ascii(c)); + } + SreOpcode::LITERAL_LOC_IGNORE => { + general_count_literal(req, ctx, end, char_loc_ignore); + } + SreOpcode::NOT_LITERAL_LOC_IGNORE => { + general_count_literal(req, ctx, end, |code, c| !char_loc_ignore(code, c)); + } + SreOpcode::LITERAL_UNI_IGNORE => { + general_count_literal(req, ctx, end, |code, c| code == lower_unicode(c)); + } + SreOpcode::NOT_LITERAL_UNI_IGNORE => { + general_count_literal(req, ctx, end, |code, c| code != lower_unicode(c)); + } + _ => { + /* General case */ + ctx.toplevel = false; + ctx.jump = Jump::OpCode; + ctx.repeat_ctx_id = usize::MAX; + ctx.count = -1; + + let mut sub_state = State { + marks: Marks::default(), + repeat_stack: vec![], + ..*state + }; + + while ctx.cursor.position < end && _match(req, &mut sub_state, *ctx) { + ctx.advance_char::<S>(); + } + } + } + + // TODO: return offset + ctx.cursor.position - state.cursor.position +} + +fn general_count_literal<S: StrDrive, F: FnMut(u32, u32) -> bool>( + req: &Request<'_, S>, + ctx: &mut MatchContext, + end: usize, + mut f: F, +) { + let ch = ctx.peek_code(req, 1); + while ctx.cursor.position < end && f(ch, ctx.peek_char::<S>()) { + ctx.advance_char::<S>(); + } +} diff --git a/crates/sre_engine/src/lib.rs b/crates/sre_engine/src/lib.rs new file mode 100644 index 00000000000..df598974ba9 --- /dev/null +++ b/crates/sre_engine/src/lib.rs @@ -0,0 +1,23 @@ +#![no_std] + +extern crate alloc; + +pub mod constants; +pub mod engine; +pub mod string; + +pub use constants::{SRE_MAGIC, SreAtCode, SreCatCode, SreFlag, SreInfo, SreOpcode}; +pub use engine::{Request, SearchIter, State}; +pub use string::{StrDrive, StringCursor}; + +pub const CODESIZE: usize = 4; + +#[cfg(target_pointer_width = "32")] +pub const MAXREPEAT: usize = usize::MAX - 1; +#[cfg(target_pointer_width = "64")] +pub const MAXREPEAT: usize = u32::MAX as usize; + +#[cfg(target_pointer_width = "32")] +pub const MAXGROUPS: usize = MAXREPEAT / 4 / 2; +#[cfg(target_pointer_width = "64")] +pub const MAXGROUPS: usize = MAXREPEAT / 2; diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs new file mode 100644 index 00000000000..489819bfb3e --- /dev/null +++ b/crates/sre_engine/src/string.rs @@ -0,0 +1,466 @@ +use rustpython_wtf8::Wtf8; + +#[derive(Debug, Clone, Copy)] +pub struct StringCursor { + pub(crate) ptr: *const u8, + pub position: usize, +} + +impl Default for StringCursor { + fn default() -> Self { + Self { + ptr: core::ptr::null(), + position: 0, + } + } +} + +pub trait StrDrive: Copy { + fn count(&self) -> usize; + fn create_cursor(&self, n: usize) -> StringCursor; + fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize); + fn advance(cursor: &mut StringCursor) -> u32; + fn peek(cursor: &StringCursor) -> u32; + fn skip(cursor: &mut StringCursor, n: usize); + fn back_advance(cursor: &mut StringCursor) -> u32; + fn back_peek(cursor: &StringCursor) -> u32; + fn back_skip(cursor: &mut StringCursor, n: usize); +} + +impl StrDrive for &[u8] { + #[inline] + fn count(&self) -> usize { + self.len() + } + + #[inline] + fn create_cursor(&self, n: usize) -> StringCursor { + StringCursor { + ptr: self[n..].as_ptr(), + position: n, + } + } + + #[inline] + fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize) { + cursor.position = n; + cursor.ptr = self[n..].as_ptr(); + } + + #[inline] + fn advance(cursor: &mut StringCursor) -> u32 { + cursor.position += 1; + unsafe { cursor.ptr = cursor.ptr.add(1) }; + unsafe { *cursor.ptr as u32 } + } + + #[inline] + fn peek(cursor: &StringCursor) -> u32 { + unsafe { *cursor.ptr as u32 } + } + + #[inline] + fn skip(cursor: &mut StringCursor, n: usize) { + cursor.position += n; + unsafe { cursor.ptr = cursor.ptr.add(n) }; + } + + #[inline] + fn back_advance(cursor: &mut StringCursor) -> u32 { + cursor.position -= 1; + unsafe { cursor.ptr = cursor.ptr.sub(1) }; + unsafe { *cursor.ptr as u32 } + } + + #[inline] + fn back_peek(cursor: &StringCursor) -> u32 { + unsafe { *cursor.ptr.offset(-1) as u32 } + } + + #[inline] + fn back_skip(cursor: &mut StringCursor, n: usize) { + cursor.position -= n; + unsafe { cursor.ptr = cursor.ptr.sub(n) }; + } +} + +impl StrDrive for &str { + #[inline] + fn count(&self) -> usize { + self.chars().count() + } + + #[inline] + fn create_cursor(&self, n: usize) -> StringCursor { + let mut cursor = StringCursor { + ptr: self.as_ptr(), + position: 0, + }; + Self::skip(&mut cursor, n); + cursor + } + + #[inline] + fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize) { + if cursor.ptr.is_null() || cursor.position > n { + *cursor = Self::create_cursor(self, n); + } else if cursor.position < n { + Self::skip(cursor, n - cursor.position); + } + } + + #[inline] + fn advance(cursor: &mut StringCursor) -> u32 { + cursor.position += 1; + unsafe { next_code_point(&mut cursor.ptr) } + } + + #[inline] + fn peek(cursor: &StringCursor) -> u32 { + let mut ptr = cursor.ptr; + unsafe { next_code_point(&mut ptr) } + } + + #[inline] + fn skip(cursor: &mut StringCursor, n: usize) { + cursor.position += n; + for _ in 0..n { + unsafe { next_code_point(&mut cursor.ptr) }; + } + } + + #[inline] + fn back_advance(cursor: &mut StringCursor) -> u32 { + cursor.position -= 1; + unsafe { next_code_point_reverse(&mut cursor.ptr) } + } + + #[inline] + fn back_peek(cursor: &StringCursor) -> u32 { + let mut ptr = cursor.ptr; + unsafe { next_code_point_reverse(&mut ptr) } + } + + #[inline] + fn back_skip(cursor: &mut StringCursor, n: usize) { + cursor.position -= n; + for _ in 0..n { + unsafe { next_code_point_reverse(&mut cursor.ptr) }; + } + } +} + +impl StrDrive for &Wtf8 { + #[inline] + fn count(&self) -> usize { + self.code_points().count() + } + + #[inline] + fn create_cursor(&self, n: usize) -> StringCursor { + let mut cursor = StringCursor { + ptr: self.as_bytes().as_ptr(), + position: 0, + }; + Self::skip(&mut cursor, n); + cursor + } + + #[inline] + fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize) { + if cursor.ptr.is_null() || cursor.position > n { + *cursor = Self::create_cursor(self, n); + } else if cursor.position < n { + Self::skip(cursor, n - cursor.position); + } + } + + #[inline] + fn advance(cursor: &mut StringCursor) -> u32 { + cursor.position += 1; + unsafe { next_code_point(&mut cursor.ptr) } + } + + #[inline] + fn peek(cursor: &StringCursor) -> u32 { + let mut ptr = cursor.ptr; + unsafe { next_code_point(&mut ptr) } + } + + #[inline] + fn skip(cursor: &mut StringCursor, n: usize) { + cursor.position += n; + for _ in 0..n { + unsafe { next_code_point(&mut cursor.ptr) }; + } + } + + #[inline] + fn back_advance(cursor: &mut StringCursor) -> u32 { + cursor.position -= 1; + unsafe { next_code_point_reverse(&mut cursor.ptr) } + } + + #[inline] + fn back_peek(cursor: &StringCursor) -> u32 { + let mut ptr = cursor.ptr; + unsafe { next_code_point_reverse(&mut ptr) } + } + + #[inline] + fn back_skip(cursor: &mut StringCursor, n: usize) { + cursor.position -= n; + for _ in 0..n { + unsafe { next_code_point_reverse(&mut cursor.ptr) }; + } + } +} + +/// Reads the next code point out of a byte iterator (assuming a +/// UTF-8-like encoding). +/// +/// # Safety +/// +/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string +#[inline] +const unsafe fn next_code_point(ptr: &mut *const u8) -> u32 { + // Decode UTF-8 + let x = unsafe { **ptr }; + *ptr = unsafe { ptr.offset(1) }; + + if x < 128 { + return x as u32; + } + + // Multibyte case follows + // Decode from a byte combination out of: [[[x y] z] w] + // NOTE: Performance is sensitive to the exact formulation here + let init = utf8_first_byte(x, 2); + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let y = unsafe { **ptr }; + *ptr = unsafe { ptr.offset(1) }; + let mut ch = utf8_acc_cont_byte(init, y); + if x >= 0xE0 { + // [[x y z] w] case + // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let z = unsafe { **ptr }; + *ptr = unsafe { ptr.offset(1) }; + let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z); + ch = (init << 12) | y_z; + if x >= 0xF0 { + // [x y z w] case + // use only the lower 3 bits of `init` + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let w = unsafe { **ptr }; + *ptr = unsafe { ptr.offset(1) }; + ch = ((init & 7) << 18) | utf8_acc_cont_byte(y_z, w); + } + } + + ch +} + +/// Reads the last code point out of a byte iterator (assuming a +/// UTF-8-like encoding). +/// +/// # Safety +/// +/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string +#[inline] +const unsafe fn next_code_point_reverse(ptr: &mut *const u8) -> u32 { + // Decode UTF-8 + *ptr = unsafe { ptr.offset(-1) }; + let w = match unsafe { **ptr } { + next_byte if next_byte < 128 => return next_byte as u32, + back_byte => back_byte, + }; + + // Multibyte case follows + // Decode from a byte combination out of: [x [y [z w]]] + let mut ch; + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + *ptr = unsafe { ptr.offset(-1) }; + let z = unsafe { **ptr }; + ch = utf8_first_byte(z, 2); + if utf8_is_cont_byte(z) { + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + *ptr = unsafe { ptr.offset(-1) }; + let y = unsafe { **ptr }; + ch = utf8_first_byte(y, 3); + if utf8_is_cont_byte(y) { + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + *ptr = unsafe { ptr.offset(-1) }; + let x = unsafe { **ptr }; + ch = utf8_first_byte(x, 4); + ch = utf8_acc_cont_byte(ch, y); + } + ch = utf8_acc_cont_byte(ch, z); + } + ch = utf8_acc_cont_byte(ch, w); + + ch +} + +/// Returns the initial codepoint accumulator for the first byte. +/// The first byte is special, only want bottom 5 bits for width 2, 4 bits +/// for width 3, and 3 bits for width 4. +#[inline] +const fn utf8_first_byte(byte: u8, width: u32) -> u32 { + (byte & (0x7F >> width)) as u32 +} + +/// Returns the value of `ch` updated with continuation byte `byte`. +#[inline] +const fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 { + (ch << 6) | (byte & CONT_MASK) as u32 +} + +/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the +/// bits `10`). +#[inline] +const fn utf8_is_cont_byte(byte: u8) -> bool { + (byte as i8) < -64 +} + +/// Mask of the value bits of a continuation byte. +const CONT_MASK: u8 = 0b0011_1111; + +const fn is_py_ascii_whitespace(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') +} + +#[inline] +pub(crate) fn is_word(ch: u32) -> bool { + ch == '_' as u32 + || u8::try_from(ch) + .map(|x| x.is_ascii_alphanumeric()) + .unwrap_or(false) +} +#[inline] +pub(crate) fn is_space(ch: u32) -> bool { + u8::try_from(ch) + .map(is_py_ascii_whitespace) + .unwrap_or(false) +} +#[inline] +pub(crate) fn is_digit(ch: u32) -> bool { + u8::try_from(ch) + .map(|x| x.is_ascii_digit()) + .unwrap_or(false) +} +#[inline] +pub(crate) fn is_loc_alnum(ch: u32) -> bool { + // FIXME: Ignore the locales + u8::try_from(ch) + .map(|x| x.is_ascii_alphanumeric()) + .unwrap_or(false) +} +#[inline] +pub(crate) fn is_loc_word(ch: u32) -> bool { + ch == '_' as u32 || is_loc_alnum(ch) +} +#[inline] +pub(crate) const fn is_linebreak(ch: u32) -> bool { + ch == '\n' as u32 +} +#[inline] +pub fn lower_ascii(ch: u32) -> u32 { + u8::try_from(ch) + .map(|x| x.to_ascii_lowercase() as u32) + .unwrap_or(ch) +} +#[inline] +pub(crate) fn lower_locate(ch: u32) -> u32 { + // FIXME: Ignore the locales + lower_ascii(ch) +} +#[inline] +pub(crate) fn upper_locate(ch: u32) -> u32 { + // FIXME: Ignore the locales + u8::try_from(ch) + .map(|x| x.to_ascii_uppercase() as u32) + .unwrap_or(ch) +} +#[inline] +pub(crate) fn is_uni_digit(ch: u32) -> bool { + // TODO: check with cpython + char::try_from(ch) + .map(|x| x.is_ascii_digit()) + .unwrap_or(false) +} +#[inline] +pub(crate) fn is_uni_space(ch: u32) -> bool { + // TODO: check with cpython + is_space(ch) + || matches!( + ch, + 0x0009 + | 0x000A + | 0x000B + | 0x000C + | 0x000D + | 0x001C + | 0x001D + | 0x001E + | 0x001F + | 0x0020 + | 0x0085 + | 0x00A0 + | 0x1680 + | 0x2000 + | 0x2001 + | 0x2002 + | 0x2003 + | 0x2004 + | 0x2005 + | 0x2006 + | 0x2007 + | 0x2008 + | 0x2009 + | 0x200A + | 0x2028 + | 0x2029 + | 0x202F + | 0x205F + | 0x3000 + ) +} +#[inline] +pub(crate) const fn is_uni_linebreak(ch: u32) -> bool { + matches!( + ch, + 0x000A | 0x000B | 0x000C | 0x000D | 0x001C | 0x001D | 0x001E | 0x0085 | 0x2028 | 0x2029 + ) +} +#[inline] +pub(crate) fn is_uni_alnum(ch: u32) -> bool { + // TODO: check with cpython + char::try_from(ch) + .map(|x| x.is_alphanumeric()) + .unwrap_or(false) +} +#[inline] +pub(crate) fn is_uni_word(ch: u32) -> bool { + ch == '_' as u32 || is_uni_alnum(ch) +} +#[inline] +pub fn lower_unicode(ch: u32) -> u32 { + // TODO: check with cpython + char::try_from(ch) + .map(|x| x.to_lowercase().next().unwrap() as u32) + .unwrap_or(ch) +} +#[inline] +pub fn upper_unicode(ch: u32) -> u32 { + // TODO: check with cpython + char::try_from(ch) + .map(|x| x.to_uppercase().next().unwrap() as u32) + .unwrap_or(ch) +} diff --git a/crates/sre_engine/tests/tests.rs b/crates/sre_engine/tests/tests.rs new file mode 100644 index 00000000000..795c9a05d42 --- /dev/null +++ b/crates/sre_engine/tests/tests.rs @@ -0,0 +1,200 @@ +// spell-checker:disable +use rustpython_sre_engine::{Request, State, StrDrive}; + +struct Pattern { + #[allow(unused)] + pattern: &'static str, + code: &'static [u32], +} + +impl Pattern { + fn state<'a, S: StrDrive>(&self, string: S) -> (Request<'a, S>, State) { + let req = Request::new(string, 0, usize::MAX, self.code, false); + let state = State::default(); + (req, state) + } +} + +#[test] +fn test_2427() { + // pattern lookbehind = re.compile(r'(?<!\.)x\b') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let lookbehind = Pattern { pattern: "(?<!\\.)x\\b", code: &[14, 4, 0, 1, 1, 5, 5, 1, 16, 46, 1, 16, 120, 6, 10, 1] }; + // END GENERATED + let (req, mut state) = lookbehind.state("x"); + assert!(state.py_match(&req)); +} + +#[test] +fn test_assert() { + // pattern positive_lookbehind = re.compile(r'(?<=abc)def') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let positive_lookbehind = Pattern { pattern: "(?<=abc)def", code: &[14, 4, 0, 3, 3, 4, 9, 3, 16, 97, 16, 98, 16, 99, 1, 16, 100, 16, 101, 16, 102, 1] }; + // END GENERATED + let (req, mut state) = positive_lookbehind.state("abcdef"); + assert!(state.search(req)); +} + +#[test] +fn test_string_boundaries() { + // pattern big_b = re.compile(r'\B') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let big_b = Pattern { pattern: "\\B", code: &[14, 4, 0, 0, 0, 6, 11, 1] }; + // END GENERATED + let (req, mut state) = big_b.state(""); + assert!(!state.search(req)); +} + +#[test] +fn test_zerowidth() { + // pattern p = re.compile(r'\b|:+') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "\\b|:+", code: &[14, 4, 0, 0, 4294967295, 7, 5, 6, 10, 15, 12, 10, 24, 6, 1, 4294967295, 16, 58, 1, 15, 2, 0, 1] }; + // END GENERATED + let (mut req, mut state) = p.state("a:"); + req.must_advance = true; + assert!(state.search(req)); + assert_eq!(state.cursor.position, 1); +} + +#[test] +fn test_repeat_context_panic() { + use optional::Optioned; + // pattern p = re.compile(r'(?:a*?(xx)??z)*') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "(?:a*?(xx)??z)*", code: &[14, 4, 0, 0, 4294967295, 23, 25, 0, 4294967295, 26, 6, 0, 4294967295, 16, 97, 1, 23, 11, 0, 1, 17, 0, 16, 120, 16, 120, 17, 1, 19, 16, 122, 18, 1] }; + // END GENERATED + let (req, mut state) = p.state("axxzaz"); + assert!(state.py_match(&req)); + assert_eq!( + *state.marks.raw(), + vec![Optioned::some(1), Optioned::some(3)] + ); +} + +#[test] +fn test_double_max_until() { + // pattern p = re.compile(r'((1)?)*') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "((1)?)*", code: &[14, 4, 0, 0, 4294967295, 23, 18, 0, 4294967295, 17, 0, 23, 9, 0, 1, 17, 2, 16, 49, 17, 3, 18, 17, 1, 18, 1] }; + // END GENERATED + let (req, mut state) = p.state("1111"); + assert!(state.py_match(&req)); + assert_eq!(state.cursor.position, 4); +} + +#[test] +fn test_info_single() { + // pattern p = re.compile(r'aa*') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "aa*", code: &[14, 8, 1, 1, 4294967295, 1, 1, 97, 0, 16, 97, 24, 6, 0, 4294967295, 16, 97, 1, 1] }; + // END GENERATED + let (req, mut state) = p.state("baaaa"); + assert!(state.search(req)); + assert_eq!(state.start, 1); + assert_eq!(state.cursor.position, 5); +} + +#[test] +fn test_info_single2() { + // pattern p = re.compile(r'Python|Perl') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "Python|Perl", code: &[14, 8, 1, 4, 6, 1, 1, 80, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 1] }; + // END GENERATED + let (req, mut state) = p.state("Perl"); + assert!(state.search(req)); +} + +#[test] +fn test_info_literal() { + // pattern p = re.compile(r'ababc+') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "ababc+", code: &[14, 14, 1, 5, 4294967295, 4, 4, 97, 98, 97, 98, 0, 0, 1, 2, 16, 97, 16, 98, 16, 97, 16, 98, 24, 6, 1, 4294967295, 16, 99, 1, 1] }; + // END GENERATED + let (req, mut state) = p.state("!ababc"); + assert!(state.search(req)); +} + +#[test] +fn test_info_literal2() { + // pattern p = re.compile(r'(python)\1') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "(python)\\1", code: &[14, 18, 1, 12, 12, 6, 0, 112, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 17, 0, 16, 112, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 11, 0, 1] }; + // END GENERATED + let (req, mut state) = p.state("pythonpython"); + assert!(state.search(req)); +} + +#[test] +fn test_repeat_in_assertions() { + // pattern p = re.compile('^([ab]*?)(?=(b)?)c', re.IGNORECASE) + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "^([ab]*?)(?=(b)?)c", code: &[14, 4, 0, 1, 4294967295, 6, 0, 17, 0, 26, 10, 0, 4294967295, 39, 5, 22, 97, 98, 0, 1, 17, 1, 4, 14, 0, 23, 9, 0, 1, 17, 2, 40, 98, 17, 3, 18, 1, 40, 99, 1] }; + // END GENERATED + let (req, mut state) = p.state("abc"); + assert!(state.search(req)); +} + +#[test] +fn test_possessive_quantifier() { + // pattern p = re.compile('e++a') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "e++a", code: &[14, 4, 0, 2, 4294967295, 29, 6, 1, 4294967295, 16, 101, 1, 16, 97, 1] }; + // END GENERATED + let (req, mut state) = p.state("eeea"); + assert!(state.py_match(&req)); +} + +#[test] +fn test_possessive_repeat_fullmatch() { + // pattern p = re.compile("([0-9]++(?:\.[0-9]+)*+)", re.I ) + // [INFO, 4, 0, 1, 4294967295, MARK, 0, POSSESSIVE_REPEAT_ONE, 10, 1, MAXREPEAT, IN, 5, RANGE, 48, 57, FAILURE, SUCCESS, POSSESSIVE_REPEAT, 16, 0, MAXREPEAT, LITERAL, 46, REPEAT_ONE, 10, 1, MAXREPEAT, IN, 5, RANGE, 48, 57, FAILURE, SUCCESS, SUCCESS, MARK, 1, SUCCESS] + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "([0-9]++(?:\\.[0-9]+)*+)", code: &[14, 4, 0, 1, 4294967295, 17, 0, 29, 10, 1, 4294967295, 13, 5, 22, 48, 57, 0, 1, 28, 16, 0, 4294967295, 16, 46, 24, 10, 1, 4294967295, 13, 5, 22, 48, 57, 0, 1, 1, 17, 1, 1] }; + // END GENERATED + let (mut req, mut state) = p.state("1.25.38"); + req.match_all = true; + assert!(state.py_match(&req), "should match"); +} + +#[test] +fn test_possessive_atomic_group() { + // pattern p = re.compile('(?>x)++x') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "(?>x)++x", code: &[14, 4, 0, 2, 4294967295, 28, 8, 1, 4294967295, 27, 4, 16, 120, 1, 1, 16, 120, 1] }; + // END GENERATED + let (req, mut state) = p.state("xxx"); + assert!(!state.py_match(&req)); +} + +#[test] +fn test_bug_20998() { + // pattern p = re.compile('[a-c]+', re.I) + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "[a-c]+", code: &[14, 4, 0, 1, 4294967295, 24, 10, 1, 4294967295, 39, 5, 22, 97, 99, 0, 1, 1] }; + // END GENERATED + let (mut req, mut state) = p.state("ABC"); + req.match_all = true; + assert!(state.py_match(&req)); + assert_eq!(state.cursor.position, 3); +} + +#[test] +fn test_bigcharset() { + // pattern p = re.compile('[a-z]*', re.I) + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "[a-z]*", code: &[14, 4, 0, 0, 4294967295, 24, 97, 0, 4294967295, 39, 92, 10, 3, 33685760, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 0, 0, 0, 134217726, 0, 0, 0, 0, 0, 131072, 0, 2147483648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] }; + // END GENERATED + let (req, mut state) = p.state("x "); + assert!(state.py_match(&req)); + assert_eq!(state.cursor.position, 1); +} + +#[test] +fn test_search_nonascii() { + #[allow(unused)] + // pattern p = re.compile('\xe0+') + // START GENERATED by generate_tests.py + #[rustfmt::skip] let p = Pattern { pattern: "\u{e0}+", code: &[14, 4, 0, 1, 4294967295, 24, 6, 1, 4294967295, 16, 224, 1, 1] }; + // END GENERATED +} diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml new file mode 100644 index 00000000000..4942467b586 --- /dev/null +++ b/crates/stdlib/Cargo.toml @@ -0,0 +1,177 @@ +[package] +name = "rustpython-stdlib" +description = "RustPython standard libraries in Rust." +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["compiler", "host_env"] +host_env = ["rustpython-vm/host_env"] +compiler = ["rustpython-vm/compiler"] +threading = ["rustpython-common/threading", "rustpython-vm/threading"] +sqlite = ["dep:libsqlite3-sys"] +# SSL backends - default to rustls +ssl = [] +ssl-rustls = ["ssl", "rustls", "rustls-native-certs", "rustls-pemfile", "rustls-platform-verifier", "x509-cert", "x509-parser", "der", "pem-rfc7468", "webpki-roots", "aws-lc-rs", "oid-registry", "pkcs8"] +ssl-rustls-fips = ["ssl-rustls", "aws-lc-rs/fips"] +ssl-openssl = ["ssl", "openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] +ssl-vendor = ["ssl-openssl", "openssl/vendored"] +tkinter = ["dep:tk-sys", "dep:tcl-sys", "dep:widestring"] +flame-it = ["flame"] + +[dependencies] +# rustpython crates +rustpython-derive = { workspace = true } +rustpython-vm = { workspace = true, default-features = false, features = ["compiler"]} +rustpython-common = { workspace = true } + +ruff_python_parser = { workspace = true } +ruff_python_ast = { workspace = true } +ruff_text_size = { workspace = true } +ruff_source_file = { workspace = true } + +ahash = { workspace = true } +ascii = { workspace = true } +cfg-if = { workspace = true } +crossbeam-utils = { workspace = true } +flame = { workspace = true, optional = true } +hex = { workspace = true } +itertools = { workspace = true } +indexmap = { workspace = true } +libc = { workspace = true } +nix = { workspace = true } +num-complex = { workspace = true } +malachite-bigint = { workspace = true } +num-integer = { workspace = true } +num-traits = { workspace = true } +num_enum = { workspace = true } +parking_lot = { workspace = true } +phf = { version = "0.13", features = ["macros"] } + +memchr = { workspace = true } +base64 = "0.22" +csv-core = "0.1.11" +dyn-clone = "1.0.10" +pymath = { workspace = true } +xml = "1.2" + +# random +rand_core = { workspace = true } +mt19937 = "<=3.2" # upgrade it once rand is upgraded + +# Crypto: +digest = "0.10.7" +md-5 = "0.10.1" +sha-1 = "0.10.0" +sha2 = "0.10.2" +sha3 = "0.10.1" +blake2 = "0.10.4" +hmac = "0.12" +pbkdf2 = { version = "0.12", features = ["hmac"] } +constant_time_eq = { workspace = true } + +## unicode stuff +unicode_names2 = { workspace = true } +# TODO: use unic for this; needed for title case: +# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 +unicode-casing = { workspace = true } +# update version all at the same time +unic-char-property = { workspace = true } +unic-normal = { workspace = true } +unic-ucd-bidi = { workspace = true } +unic-ucd-category = { workspace = true } +unic-ucd-age = { workspace = true } +unic-ucd-ident = { workspace = true } +ucd = "0.1.1" +unicode-bidi-mirroring = { workspace = true } + +# compression +adler32 = "1.2.0" +crc32fast = "1.3.2" +flate2 = { version = "1.1.9", default-features = false, features = ["zlib-rs"] } +libz-sys = { package = "libz-rs-sys", version = "0.5" } +bzip2 = "0.6" + +# tkinter +tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } +tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } +widestring = { workspace = true, optional = true } +chrono.workspace = true + +# uuid +[target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies] +mac_address = "1.1.3" +uuid = { version = "1.22.0", features = ["v1"] } + +[target.'cfg(all(unix, not(target_os = "redox"), not(target_os = "ios")))'.dependencies] +termios = "0.3.3" + +[target.'cfg(unix)'.dependencies] +rustix = { workspace = true } + +# mmap + socket dependencies +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +memmap2 = "0.9.10" +page_size = "0.6" +gethostname = "1.0.2" +socket2 = { version = "0.6.3", features = ["all"] } +dns-lookup = "3.0" + +# OpenSSL dependencies (optional, for ssl-openssl feature) +openssl = { version = "0.10.76", optional = true } +openssl-sys = { version = "0.9.110", optional = true } +openssl-probe = { version = "0.2.1", optional = true } +foreign-types-shared = { version = "0.1.1", optional = true } + +# Rustls dependencies (optional, for ssl-rustls feature) +rustls = { version = "0.23.37", default-features = false, features = ["std", "tls12", "aws_lc_rs"], optional = true } +rustls-native-certs = { version = "0.8", optional = true } +rustls-pemfile = { version = "2.2", optional = true } +rustls-platform-verifier = { version = "0.6", optional = true } +x509-cert = { version = "0.2.5", features = ["pem", "builder"], optional = true } +x509-parser = { version = "0.18", optional = true } +der = { version = "0.7", features = ["alloc", "oid"], optional = true } +pem-rfc7468 = { version = "1.0", features = ["alloc"], optional = true } +webpki-roots = { version = "1.0", optional = true } +aws-lc-rs = { version = "1.16.0", optional = true } +oid-registry = { version = "0.8", features = ["x509", "pkcs1", "nist_algs"], optional = true } +pkcs8 = { version = "0.10", features = ["encryption", "pkcs5", "pem"], optional = true } + +[target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies] +libsqlite3-sys = { version = "0.36", features = ["bundled"], optional = true } +liblzma = "0.4" +liblzma-sys = "0.4" + +[target.'cfg(windows)'.dependencies] +paste = { workspace = true } +schannel = { workspace = true } +widestring = { workspace = true } + +[target.'cfg(windows)'.dependencies.windows-sys] +workspace = true +features = [ + "Win32_Foundation", + "Win32_Networking_WinSock", + "Win32_NetworkManagement_IpHelper", + "Win32_NetworkManagement_Ndis", + "Win32_Security_Cryptography", + "Win32_Storage_FileSystem", + "Win32_System_Diagnostics_Debug", + "Win32_System_Environment", + "Win32_System_Console", + "Win32_System_IO", + "Win32_System_Memory", + "Win32_System_Threading" +] + +[target.'cfg(target_os = "macos")'.dependencies] +system-configuration = "0.7.0" + +[lints] +workspace = true diff --git a/crates/stdlib/build.rs b/crates/stdlib/build.rs new file mode 100644 index 00000000000..95c34c4fb3c --- /dev/null +++ b/crates/stdlib/build.rs @@ -0,0 +1,56 @@ +// spell-checker:ignore ossl osslconf + +fn main() { + println!(r#"cargo::rustc-check-cfg=cfg(osslconf, values("OPENSSL_NO_COMP"))"#); + println!(r#"cargo::rustc-check-cfg=cfg(openssl_vendored)"#); + + #[allow( + clippy::unusual_byte_groupings, + reason = "hex groups follow OpenSSL version field boundaries" + )] + let ossl_vers = [ + (0x1_00_01_00_0, "ossl101"), + (0x1_00_02_00_0, "ossl102"), + (0x1_01_00_00_0, "ossl110"), + (0x1_01_00_07_0, "ossl110g"), + (0x1_01_00_08_0, "ossl110h"), + (0x1_01_01_00_0, "ossl111"), + (0x1_01_01_04_0, "ossl111d"), + (0x3_00_00_00_0, "ossl300"), + (0x3_01_00_00_0, "ossl310"), + (0x3_02_00_00_0, "ossl320"), + (0x3_03_00_00_0, "ossl330"), + ]; + + for (_, cfg) in ossl_vers { + println!("cargo::rustc-check-cfg=cfg({cfg})"); + } + + #[cfg(feature = "ssl-openssl")] + { + #[allow( + clippy::unusual_byte_groupings, + reason = "OpenSSL version number is parsed with grouped hex fields" + )] + if let Ok(v) = std::env::var("DEP_OPENSSL_VERSION_NUMBER") { + println!("cargo:rustc-env=OPENSSL_API_VERSION={v}"); + // cfg setup from openssl crate's build script + let version = u64::from_str_radix(&v, 16).unwrap(); + for (ver, cfg) in ossl_vers { + if version >= ver { + println!("cargo:rustc-cfg={cfg}"); + } + } + } + if let Ok(v) = std::env::var("DEP_OPENSSL_CONF") { + for conf in v.split(',') { + println!("cargo:rustc-cfg=osslconf=\"{conf}\""); + } + } + // it's possible for openssl-sys to link against the system openssl under certain conditions, + // so let the ssl module know to only perform a probe if we're actually vendored + if std::env::var("DEP_OPENSSL_VENDORED").is_ok_and(|s| s == "1") { + println!("cargo::rustc-cfg=openssl_vendored") + } + } +} diff --git a/crates/stdlib/src/_asyncio.rs b/crates/stdlib/src/_asyncio.rs new file mode 100644 index 00000000000..afa372cffe6 --- /dev/null +++ b/crates/stdlib/src/_asyncio.rs @@ -0,0 +1,2766 @@ +//! _asyncio module - provides native asyncio support +//! +//! This module provides native implementations of Future and Task classes, + +pub(crate) use _asyncio::module_def; + +#[pymodule] +pub(crate) mod _asyncio { + use crate::common::wtf8::{Wtf8Buf, wtf8_concat}; + use crate::{ + common::lock::PyRwLock, + vm::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{ + PyBaseException, PyBaseExceptionRef, PyDict, PyDictRef, PyGenericAlias, PyList, + PyListRef, PyModule, PySet, PyTuple, PyType, PyTypeRef, + }, + extend_module, + function::{FuncArgs, KwArgs, OptionalArg, OptionalOption, PySetterValue}, + protocol::PyIterReturn, + recursion::ReprGuard, + types::{ + Callable, Constructor, Destructor, Initializer, IterNext, Iterable, Representable, + SelfIter, + }, + warn, + }, + }; + use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering}; + use crossbeam_utils::atomic::AtomicCell; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + // Initialize module-level state + let weakref_module = vm.import("weakref", 0)?; + let weak_set_class = vm + .get_attribute_opt(weakref_module, vm.ctx.intern_str("WeakSet"))? + .ok_or_else(|| vm.new_attribute_error("WeakSet not found"))?; + let scheduled_tasks = weak_set_class.call((), vm)?; + let eager_tasks = PySet::default().into_ref(&vm.ctx); + let current_tasks = PyDict::default().into_ref(&vm.ctx); + + extend_module!(vm, module, { + "_scheduled_tasks" => scheduled_tasks, + "_eager_tasks" => eager_tasks, + "_current_tasks" => current_tasks, + }); + + // Register fork handler to clear task state in child process + #[cfg(unix)] + { + let on_fork = vm + .get_attribute_opt(module.to_owned().into(), vm.ctx.intern_str("_on_fork"))? + .expect("_on_fork not found in _asyncio module"); + vm.state.after_forkers_child.lock().push(on_fork); + } + + Ok(()) + } + + #[derive(FromArgs)] + struct AddDoneCallbackArgs { + #[pyarg(positional)] + func: PyObjectRef, + #[pyarg(named, optional)] + context: OptionalOption<PyObjectRef>, + } + + #[derive(FromArgs)] + struct CancelArgs { + #[pyarg(any, optional)] + msg: OptionalOption<PyObjectRef>, + } + + #[derive(FromArgs)] + struct LoopArg { + #[pyarg(any, name = "loop", optional)] + loop_: OptionalOption<PyObjectRef>, + } + + #[derive(FromArgs)] + struct GetStackArgs { + #[pyarg(named, optional)] + limit: OptionalOption<PyObjectRef>, + } + + #[derive(FromArgs)] + struct PrintStackArgs { + #[pyarg(named, optional)] + limit: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + file: OptionalOption<PyObjectRef>, + } + + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + enum FutureState { + Pending, + Cancelled, + Finished, + } + + impl FutureState { + fn as_str(&self) -> &'static str { + match self { + FutureState::Pending => "PENDING", + FutureState::Cancelled => "CANCELLED", + FutureState::Finished => "FINISHED", + } + } + } + + /// asyncio.Future implementation + #[pyattr] + #[pyclass(name = "Future", module = "_asyncio", traverse)] + #[derive(Debug, PyPayload)] + #[repr(C)] // Required for inheritance - ensures base field is at offset 0 in subclasses + struct PyFuture { + fut_loop: PyRwLock<Option<PyObjectRef>>, + fut_callback0: PyRwLock<Option<PyObjectRef>>, + fut_context0: PyRwLock<Option<PyObjectRef>>, + fut_callbacks: PyRwLock<Option<PyObjectRef>>, + fut_exception: PyRwLock<Option<PyObjectRef>>, + fut_exception_tb: PyRwLock<Option<PyObjectRef>>, + fut_result: PyRwLock<Option<PyObjectRef>>, + fut_source_tb: PyRwLock<Option<PyObjectRef>>, + fut_cancel_msg: PyRwLock<Option<PyObjectRef>>, + fut_cancelled_exc: PyRwLock<Option<PyObjectRef>>, + fut_awaited_by: PyRwLock<Option<PyObjectRef>>, + #[pytraverse(skip)] + fut_state: AtomicCell<FutureState>, + #[pytraverse(skip)] + fut_awaited_by_is_set: AtomicBool, + #[pytraverse(skip)] + fut_log_tb: AtomicBool, + #[pytraverse(skip)] + fut_blocking: AtomicBool, + } + + impl Constructor for PyFuture { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(PyFuture::new_empty()) + } + } + + impl Initializer for PyFuture { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Future does not accept positional arguments + if !args.args.is_empty() { + return Err(vm.new_type_error("Future() takes no positional arguments")); + } + // Extract only 'loop' keyword argument + let loop_ = args.kwargs.get("loop").cloned(); + PyFuture::py_init(&zelf, loop_, vm) + } + } + + #[pyclass( + flags(BASETYPE, HAS_DICT, HAS_WEAKREF), + with(Constructor, Initializer, Destructor, Representable, Iterable) + )] + impl PyFuture { + fn new_empty() -> Self { + Self { + fut_loop: PyRwLock::new(None), + fut_callback0: PyRwLock::new(None), + fut_context0: PyRwLock::new(None), + fut_callbacks: PyRwLock::new(None), + fut_exception: PyRwLock::new(None), + fut_exception_tb: PyRwLock::new(None), + fut_result: PyRwLock::new(None), + fut_source_tb: PyRwLock::new(None), + fut_cancel_msg: PyRwLock::new(None), + fut_cancelled_exc: PyRwLock::new(None), + fut_awaited_by: PyRwLock::new(None), + fut_state: AtomicCell::new(FutureState::Pending), + fut_awaited_by_is_set: AtomicBool::new(false), + fut_log_tb: AtomicBool::new(false), + fut_blocking: AtomicBool::new(false), + } + } + + fn py_init( + zelf: &PyRef<Self>, + loop_: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Get the event loop + let loop_obj = match loop_ { + Some(l) if !vm.is_none(&l) => l, + _ => get_event_loop(vm)?, + }; + *zelf.fut_loop.write() = Some(loop_obj.clone()); + + // Check if loop has get_debug method and call it + if let Ok(Some(get_debug)) = + vm.get_attribute_opt(loop_obj.clone(), vm.ctx.intern_str("get_debug")) + && let Ok(debug) = get_debug.call((), vm) + && debug.try_to_bool(vm).unwrap_or(false) + { + // Get source traceback + if let Ok(tb_module) = vm.import("traceback", 0) + && let Ok(Some(extract_stack)) = + vm.get_attribute_opt(tb_module, vm.ctx.intern_str("extract_stack")) + && let Ok(tb) = extract_stack.call((), vm) + { + *zelf.fut_source_tb.write() = Some(tb); + } + } + + Ok(()) + } + + #[pymethod] + fn result(&self, vm: &VirtualMachine) -> PyResult { + match self.fut_state.load() { + FutureState::Pending => Err(new_invalid_state_error(vm, "Result is not ready.")), + FutureState::Cancelled => { + let exc = self.make_cancelled_error_impl(vm); + Err(exc) + } + FutureState::Finished => { + self.fut_log_tb.store(false, Ordering::Relaxed); + if let Some(exc) = self.fut_exception.read().clone() { + let exc: PyBaseExceptionRef = exc.downcast().unwrap(); + // Restore the original traceback to prevent traceback accumulation + if let Some(tb) = self.fut_exception_tb.read().clone() { + let _ = exc.set___traceback__(tb, vm); + } + Err(exc) + } else { + Ok(self + .fut_result + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none())) + } + } + } + } + + #[pymethod] + fn exception(&self, vm: &VirtualMachine) -> PyResult { + match self.fut_state.load() { + FutureState::Pending => Err(new_invalid_state_error(vm, "Exception is not set.")), + FutureState::Cancelled => { + let exc = self.make_cancelled_error_impl(vm); + Err(exc) + } + FutureState::Finished => { + self.fut_log_tb.store(false, Ordering::Relaxed); + Ok(self + .fut_exception + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none())) + } + } + } + + #[pymethod] + fn set_result(zelf: PyRef<Self>, result: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if zelf.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.")); + } + if zelf.fut_state.load() != FutureState::Pending { + return Err(new_invalid_state_error(vm, "invalid state")); + } + *zelf.fut_result.write() = Some(result); + zelf.fut_state.store(FutureState::Finished); + Self::schedule_callbacks(&zelf, vm)?; + Ok(()) + } + + #[pymethod] + fn set_exception( + zelf: PyRef<Self>, + exception: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if zelf.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.")); + } + if zelf.fut_state.load() != FutureState::Pending { + return Err(new_invalid_state_error(vm, "invalid state")); + } + + // Normalize the exception + let exc = if exception.fast_isinstance(vm.ctx.types.type_type) { + exception.call((), vm)? + } else { + exception + }; + + if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error(format!( + "exception must be a BaseException, not {}", + exc.class().name() + ))); + } + + // Wrap StopIteration in RuntimeError + let exc = if exc.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + let msg = "StopIteration interacts badly with generators and cannot be raised into a Future"; + let runtime_err = vm.new_runtime_error(msg.to_string()); + // Set cause and context to the original StopIteration + let stop_iter: PyRef<PyBaseException> = exc.downcast().unwrap(); + runtime_err.set___cause__(Some(stop_iter.clone())); + runtime_err.set___context__(Some(stop_iter)); + runtime_err.into() + } else { + exc + }; + + // Save the original traceback for later restoration + if let Ok(exc_ref) = exc.clone().downcast::<PyBaseException>() { + let tb = exc_ref.__traceback__().map(|tb| tb.into()); + *zelf.fut_exception_tb.write() = tb; + } + + *zelf.fut_exception.write() = Some(exc); + zelf.fut_state.store(FutureState::Finished); + zelf.fut_log_tb.store(true, Ordering::Relaxed); + Self::schedule_callbacks(&zelf, vm)?; + Ok(()) + } + + #[pymethod] + fn add_done_callback( + zelf: PyRef<Self>, + args: AddDoneCallbackArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if zelf.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.")); + } + let ctx = match args.context.flatten() { + Some(c) => c, + None => get_copy_context(vm)?, + }; + + if zelf.fut_state.load() != FutureState::Pending { + Self::call_soon_with_context(&zelf, args.func, Some(ctx), vm)?; + } else if zelf.fut_callback0.read().is_none() { + *zelf.fut_callback0.write() = Some(args.func); + *zelf.fut_context0.write() = Some(ctx); + } else { + let tuple = vm.ctx.new_tuple(vec![args.func, ctx]); + let mut callbacks = zelf.fut_callbacks.write(); + if callbacks.is_none() { + *callbacks = Some(vm.ctx.new_list(vec![tuple.into()]).into()); + } else { + let list = callbacks.as_ref().unwrap(); + vm.call_method(list, "append", (tuple,))?; + } + } + Ok(()) + } + + #[pymethod] + fn remove_done_callback(&self, func: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + if self.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.")); + } + let mut cleared_callback0 = 0usize; + + // Check fut_callback0 first + // Clone to release lock before comparison (which may run Python code) + let cb0 = self.fut_callback0.read().clone(); + if let Some(cb0) = cb0 { + let cmp = vm.identical_or_equal(&cb0, &func)?; + if cmp { + *self.fut_callback0.write() = None; + *self.fut_context0.write() = None; + cleared_callback0 = 1; + } + } + + // Check if fut_callbacks exists + let callbacks = self.fut_callbacks.read().clone(); + let callbacks = match callbacks { + Some(c) => c, + None => return Ok(cleared_callback0), + }; + + let list: PyListRef = callbacks.downcast().unwrap(); + let len = list.borrow_vec().len(); + + if len == 0 { + *self.fut_callbacks.write() = None; + return Ok(cleared_callback0); + } + + // Special case for single callback + if len == 1 { + let item = list.borrow_vec().first().cloned(); + if let Some(item) = item { + let tuple: &PyTuple = item.downcast_ref().unwrap(); + let cb = tuple.first().unwrap().clone(); + let cmp = vm.identical_or_equal(&cb, &func)?; + if cmp { + *self.fut_callbacks.write() = None; + return Ok(1 + cleared_callback0); + } + } + return Ok(cleared_callback0); + } + + // Multiple callbacks - iterate with index, checking validity each time + // to handle evil comparisons + let mut new_callbacks = Vec::with_capacity(len); + let mut i = 0usize; + let mut removed = 0usize; + + loop { + // Re-check fut_callbacks on each iteration (evil code may have cleared it) + let callbacks = self.fut_callbacks.read().clone(); + let callbacks = match callbacks { + Some(c) => c, + None => break, + }; + let list: PyListRef = callbacks.downcast().unwrap(); + let current_len = list.borrow_vec().len(); + if i >= current_len { + break; + } + + // Get item and release lock before comparison + let item = list.borrow_vec().get(i).cloned(); + let item = match item { + Some(item) => item, + None => break, + }; + + let tuple: &PyTuple = item.downcast_ref().unwrap(); + let cb = tuple.first().unwrap().clone(); + let cmp = vm.identical_or_equal(&cb, &func)?; + + if !cmp { + new_callbacks.push(item); + } else { + removed += 1; + } + i += 1; + } + + // Update fut_callbacks with filtered list + if new_callbacks.is_empty() { + *self.fut_callbacks.write() = None; + } else { + *self.fut_callbacks.write() = Some(vm.ctx.new_list(new_callbacks).into()); + } + + Ok(removed + cleared_callback0) + } + + #[pymethod] + fn cancel(zelf: PyRef<Self>, args: CancelArgs, vm: &VirtualMachine) -> PyResult<bool> { + if zelf.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.")); + } + if zelf.fut_state.load() != FutureState::Pending { + // Clear log_tb even when cancel fails + zelf.fut_log_tb.store(false, Ordering::Relaxed); + return Ok(false); + } + + *zelf.fut_cancel_msg.write() = args.msg.flatten(); + zelf.fut_state.store(FutureState::Cancelled); + Self::schedule_callbacks(&zelf, vm)?; + Ok(true) + } + + #[pymethod] + fn cancelled(&self) -> bool { + self.fut_state.load() == FutureState::Cancelled + } + + #[pymethod] + fn done(&self) -> bool { + self.fut_state.load() != FutureState::Pending + } + + #[pymethod] + fn get_loop(&self, vm: &VirtualMachine) -> PyResult { + self.fut_loop + .read() + .clone() + .ok_or_else(|| vm.new_runtime_error("Future object is not initialized.")) + } + + #[pymethod] + fn _make_cancelled_error(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.make_cancelled_error_impl(vm) + } + + fn make_cancelled_error_impl(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + // If a saved CancelledError exists, take it (clearing the stored reference) + if let Some(exc) = self.fut_cancelled_exc.write().take() + && let Ok(exc) = exc.downcast::<PyBaseException>() + { + return exc; + } + + let msg = self.fut_cancel_msg.read().clone(); + let args = if let Some(m) = msg { vec![m] } else { vec![] }; + + match get_cancelled_error_type(vm) { + Ok(cancelled_error) => vm.new_exception(cancelled_error, args), + Err(_) => vm.new_runtime_error("cancelled"), + } + } + + fn schedule_callbacks(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Collect all callbacks first to avoid holding locks during callback execution + // This prevents deadlock when callbacks access the future's properties + let mut callbacks_to_call: Vec<(PyObjectRef, Option<PyObjectRef>)> = Vec::new(); + + // Take callback0 - release lock before collecting from list + let cb0 = zelf.fut_callback0.write().take(); + let ctx0 = zelf.fut_context0.write().take(); + if let Some(cb) = cb0 { + callbacks_to_call.push((cb, ctx0)); + } + + // Take callbacks list and collect items + let callbacks_list = zelf.fut_callbacks.write().take(); + if let Some(callbacks) = callbacks_list + && let Ok(list) = callbacks.downcast::<PyList>() + { + // Clone the items while holding the list lock, then release + let items: Vec<_> = list.borrow_vec().iter().cloned().collect(); + for item in items { + if let Some(tuple) = item.downcast_ref::<PyTuple>() + && let (Some(cb), Some(ctx)) = (tuple.first(), tuple.get(1)) + { + callbacks_to_call.push((cb.clone(), Some(ctx.clone()))); + } + } + } + + // Now call all callbacks without holding any locks + for (cb, ctx) in callbacks_to_call { + Self::call_soon_with_context(zelf, cb, ctx, vm)?; + } + + Ok(()) + } + + fn call_soon_with_context( + zelf: &PyRef<Self>, + callback: PyObjectRef, + context: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let loop_obj = zelf.fut_loop.read().clone(); + if let Some(loop_obj) = loop_obj { + // call_soon(callback, *args, context=context) + // callback receives the future as its argument + let future_arg: PyObjectRef = zelf.clone().into(); + let args = if let Some(ctx) = context { + FuncArgs::new( + vec![callback, future_arg], + KwArgs::new([("context".to_owned(), ctx)].into_iter().collect()), + ) + } else { + FuncArgs::new(vec![callback, future_arg], KwArgs::default()) + }; + vm.call_method(&loop_obj, "call_soon", args)?; + } + Ok(()) + } + + // Properties + #[pygetset] + fn _state(&self) -> &'static str { + self.fut_state.load().as_str() + } + + #[pygetset] + fn _asyncio_future_blocking(&self) -> bool { + self.fut_blocking.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__asyncio_future_blocking( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + self.fut_blocking.store(v, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => Err(vm.new_attribute_error("cannot delete attribute")), + } + } + + #[pygetset] + fn _loop(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_loop + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _callbacks(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let mut result = Vec::new(); + + if let Some(cb0) = self.fut_callback0.read().clone() { + let ctx0 = self + .fut_context0 + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()); + result.push(vm.ctx.new_tuple(vec![cb0, ctx0]).into()); + } + + if let Some(callbacks) = self.fut_callbacks.read().clone() { + let list: PyListRef = callbacks.downcast().unwrap(); + for item in list.borrow_vec().iter() { + result.push(item.clone()); + } + } + + // Return None if no callbacks + if result.is_empty() { + Ok(vm.ctx.none()) + } else { + Ok(vm.ctx.new_list(result).into()) + } + } + + #[pygetset] + fn _result(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_result + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _exception(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_exception + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _log_traceback(&self) -> bool { + self.fut_log_tb.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__log_traceback( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + if v { + return Err(vm.new_value_error("_log_traceback can only be set to False")); + } + self.fut_log_tb.store(false, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => Err(vm.new_attribute_error("cannot delete attribute")), + } + } + + #[pygetset] + fn _source_traceback(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_source_tb + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _cancel_message(&self, vm: &VirtualMachine) -> PyObjectRef { + self.fut_cancel_msg + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset(setter)] + fn set__cancel_message(&self, value: PySetterValue) { + match value { + PySetterValue::Assign(v) => *self.fut_cancel_msg.write() = Some(v), + PySetterValue::Delete => *self.fut_cancel_msg.write() = None, + } + } + + #[pygetset] + fn _asyncio_awaited_by(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let awaited_by = self.fut_awaited_by.read().clone(); + match awaited_by { + None => Ok(vm.ctx.none()), + Some(obj) => { + if self.fut_awaited_by_is_set.load(Ordering::Relaxed) { + // Already a Set + Ok(obj) + } else { + // Single object - create a Set for the return value + let new_set = PySet::default().into_ref(&vm.ctx); + new_set.add(obj, vm)?; + Ok(new_set.into()) + } + } + } + } + + /// Add waiter to fut_awaited_by with single-object optimization + fn awaited_by_add(&self, waiter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut awaited_by = self.fut_awaited_by.write(); + if awaited_by.is_none() { + // First waiter - store directly + *awaited_by = Some(waiter); + return Ok(()); + } + + if self.fut_awaited_by_is_set.load(Ordering::Relaxed) { + // Already a Set - add to it + let set = awaited_by.as_ref().unwrap(); + vm.call_method(set, "add", (waiter,))?; + } else { + // Single object - convert to Set + let existing = awaited_by.take().unwrap(); + let new_set = PySet::default().into_ref(&vm.ctx); + new_set.add(existing, vm)?; + new_set.add(waiter, vm)?; + *awaited_by = Some(new_set.into()); + self.fut_awaited_by_is_set.store(true, Ordering::Relaxed); + } + Ok(()) + } + + /// Discard waiter from fut_awaited_by with single-object optimization + fn awaited_by_discard(&self, waiter: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + let mut awaited_by = self.fut_awaited_by.write(); + if awaited_by.is_none() { + return Ok(()); + } + + let obj = awaited_by.as_ref().unwrap(); + if !self.fut_awaited_by_is_set.load(Ordering::Relaxed) { + // Single object - check if it matches + if obj.is(waiter) { + *awaited_by = None; + } + } else { + // It's a Set - use discard + vm.call_method(obj, "discard", (waiter.to_owned(),))?; + } + Ok(()) + } + + #[pymethod] + fn __await__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyFutureIter> { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + }) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl Destructor for PyFuture { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Check if we should log the traceback + // Don't log if log_tb is false or if the future was cancelled + if !zelf.fut_log_tb.load(Ordering::Relaxed) { + return Ok(()); + } + + if zelf.fut_state.load() == FutureState::Cancelled { + return Ok(()); + } + + let exc = zelf.fut_exception.read().clone(); + let exc = match exc { + Some(e) => e, + None => return Ok(()), + }; + + let loop_obj = zelf.fut_loop.read().clone(); + let loop_obj = match loop_obj { + Some(l) => l, + None => return Ok(()), + }; + + // Create context dict for call_exception_handler + let context = PyDict::default().into_ref(&vm.ctx); + let class_name = zelf.class().name().to_string(); + let message = format!("{} exception was never retrieved", class_name); + context.set_item( + vm.ctx.intern_str("message"), + vm.ctx.new_str(message).into(), + vm, + )?; + context.set_item(vm.ctx.intern_str("exception"), exc, vm)?; + context.set_item(vm.ctx.intern_str("future"), zelf.to_owned().into(), vm)?; + + if let Some(tb) = zelf.fut_source_tb.read().clone() { + context.set_item(vm.ctx.intern_str("source_traceback"), tb, vm)?; + } + + // Call loop.call_exception_handler(context) + let _ = vm.call_method(&loop_obj, "call_exception_handler", (context,)); + Ok(()) + } + } + + impl Representable for PyFuture { + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let class_name = zelf.class().name().to_string(); + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let info = get_future_repr_info(zelf.as_object(), vm)?; + Ok(format!("<{} {}>", class_name, info)) + } else { + Ok(format!("<{} ...>", class_name)) + } + } + } + + impl Iterable for PyFuture { + fn iter(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + } + .into_pyobject(_vm)) + } + } + + fn get_future_repr_info(future: &PyObject, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + // Try to use asyncio.base_futures._future_repr_info + // Import from sys.modules if available, otherwise try regular import + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + let module = + if let Ok(m) = sys_modules.get_item(&*vm.ctx.new_str("asyncio.base_futures"), vm) { + m + } else { + // vm.import returns the top-level module, get base_futures submodule + match vm + .import("asyncio.base_futures", 0) + .and_then(|asyncio| asyncio.get_attr(vm.ctx.intern_str("base_futures"), vm)) + { + Ok(m) => m, + Err(_) => return get_future_repr_info_fallback(future, vm), + } + }; + + let func = match vm.get_attribute_opt(module, vm.ctx.intern_str("_future_repr_info")) { + Ok(Some(f)) => f, + _ => return get_future_repr_info_fallback(future, vm), + }; + + let info = match func.call((future.to_owned(),), vm) { + Ok(i) => i, + Err(_) => return get_future_repr_info_fallback(future, vm), + }; + + let list: PyListRef = match info.downcast() { + Ok(l) => l, + Err(_) => return get_future_repr_info_fallback(future, vm), + }; + + let mut result = Wtf8Buf::new(); + let parts = list.borrow_vec(); + for (i, x) in parts.iter().enumerate() { + if i > 0 { + result.push_str(" "); + } + if let Ok(s) = x.str(vm) { + result.push_wtf8(s.as_wtf8()); + } + } + Ok(result) + } + + fn get_future_repr_info_fallback(future: &PyObject, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + // Fallback: build repr from properties directly + if let Ok(Some(state)) = + vm.get_attribute_opt(future.to_owned(), vm.ctx.intern_str("_state")) + { + let s = state + .str(vm) + .map(|s| s.as_wtf8().to_lowercase()) + .unwrap_or_else(|_| Wtf8Buf::from("unknown")); + return Ok(s); + } + Ok(Wtf8Buf::from("state=unknown")) + } + + fn get_task_repr_info(task: &PyObject, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + // vm.import returns the top-level module, get base_tasks submodule + match vm + .import("asyncio.base_tasks", 0) + .and_then(|asyncio| asyncio.get_attr(vm.ctx.intern_str("base_tasks"), vm)) + { + Ok(base_tasks) => { + match vm.get_attribute_opt(base_tasks, vm.ctx.intern_str("_task_repr_info")) { + Ok(Some(func)) => { + let info: PyObjectRef = func.call((task.to_owned(),), vm)?; + let list: PyListRef = info.downcast().map_err(|_| { + vm.new_type_error("_task_repr_info should return a list") + })?; + let mut result = Wtf8Buf::new(); + let parts = list.borrow_vec(); + for (i, x) in parts.iter().enumerate() { + if i > 0 { + result.push_str(" "); + } + result.push_wtf8(x.str(vm)?.as_wtf8()); + } + Ok(result) + } + _ => get_future_repr_info(task, vm), + } + } + Err(_) => get_future_repr_info(task, vm), + } + } + + #[pyattr] + #[pyclass(name = "FutureIter", module = "_asyncio", traverse)] + #[derive(Debug, PyPayload)] + struct PyFutureIter { + future: PyRwLock<Option<PyObjectRef>>, + } + + #[pyclass(with(IterNext, Iterable))] + impl PyFutureIter { + #[pymethod] + fn send(&self, _value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let future = self.future.read().clone(); + let future = match future { + Some(f) => f, + None => return Err(vm.new_stop_iteration(None)), + }; + + // Try to get blocking flag (check Task first since it inherits from Future) + let blocking = if let Some(task) = future.downcast_ref::<PyTask>() { + task.base.fut_blocking.load(Ordering::Relaxed) + } else if let Some(fut) = future.downcast_ref::<PyFuture>() { + fut.fut_blocking.load(Ordering::Relaxed) + } else { + // For non-native futures, check the attribute + vm.get_attribute_opt( + future.clone(), + vm.ctx.intern_str("_asyncio_future_blocking"), + )? + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false) + }; + + // Check if future is done + let done = vm.call_method(&future, "done", ())?; + if done.try_to_bool(vm)? { + *self.future.write() = None; + let result = vm.call_method(&future, "result", ())?; + return Err(vm.new_stop_iteration(Some(result))); + } + + // If still pending and blocking is already set, raise RuntimeError + // This means await wasn't used with future + if blocking { + return Err(vm.new_runtime_error("await wasn't used with future")); + } + + // First call: set blocking flag and yield the future (check Task first) + if let Some(task) = future.downcast_ref::<PyTask>() { + task.base.fut_blocking.store(true, Ordering::Relaxed); + } else if let Some(fut) = future.downcast_ref::<PyFuture>() { + fut.fut_blocking.store(true, Ordering::Relaxed); + } else { + future.set_attr( + vm.ctx.intern_str("_asyncio_future_blocking"), + vm.ctx.true_value.clone(), + vm, + )?; + } + Ok(future) + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + // Warn about deprecated (type, val, tb) signature + if exc_val.is_present() || exc_tb.is_present() { + warn::warn( + vm.ctx + .new_str( + "the (type, val, tb) signature of throw() is deprecated, \ + use throw(val) instead", + ) + .into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + + *self.future.write() = None; + + // Validate tb if present + if let OptionalArg::Present(ref tb) = exc_tb + && !vm.is_none(tb) + && !tb.fast_isinstance(vm.ctx.types.traceback_type) + { + return Err(vm.new_type_error(format!( + "throw() third argument must be a traceback object, not '{}'", + tb.class().name() + ))); + } + + let exc = if exc_type.fast_isinstance(vm.ctx.types.type_type) { + // exc_type is a class + let exc_class: PyTypeRef = exc_type.clone().downcast().unwrap(); + // Must be a subclass of BaseException + if !exc_class.fast_issubclass(vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error( + "exceptions must be classes or instances deriving from BaseException, not type" + )); + } + + let val = exc_val.unwrap_or_none(vm); + if vm.is_none(&val) { + exc_type.call((), vm)? + } else if val.fast_isinstance(&exc_class) { + val + } else { + exc_type.call((val,), vm)? + } + } else if exc_type.fast_isinstance(vm.ctx.exceptions.base_exception_type) { + // exc_type is an exception instance + if let OptionalArg::Present(ref val) = exc_val + && !vm.is_none(val) + { + return Err( + vm.new_type_error("instance exception may not have a separate value") + ); + } + exc_type + } else { + // exc_type is neither a class nor an exception instance + return Err(vm.new_type_error(format!( + "exceptions must be classes or instances deriving from BaseException, not {}", + exc_type.class().name() + ))); + }; + + if let OptionalArg::Present(tb) = exc_tb + && !vm.is_none(&tb) + { + exc.set_attr(vm.ctx.intern_str("__traceback__"), tb, vm)?; + } + + Err(exc.downcast().unwrap()) + } + + #[pymethod] + fn close(&self) { + *self.future.write() = None; + } + } + + impl SelfIter for PyFutureIter {} + impl IterNext for PyFutureIter { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + PyIterReturn::from_pyresult(zelf.send(vm.ctx.none(), vm), vm) + } + } + + #[pyattr] + #[pyclass(name = "Task", module = "_asyncio", base = PyFuture, traverse)] + #[derive(Debug)] + #[repr(C)] + struct PyTask { + // Base class (must be first field for inheritance) + base: PyFuture, + // Task-specific fields + task_coro: PyRwLock<Option<PyObjectRef>>, + task_fut_waiter: PyRwLock<Option<PyObjectRef>>, + task_name: PyRwLock<Option<PyObjectRef>>, + task_context: PyRwLock<Option<PyObjectRef>>, + #[pytraverse(skip)] + task_must_cancel: AtomicBool, + #[pytraverse(skip)] + task_num_cancels_requested: AtomicI32, + #[pytraverse(skip)] + task_log_destroy_pending: AtomicBool, + } + + #[derive(FromArgs)] + struct TaskInitArgs { + #[pyarg(positional)] + coro: PyObjectRef, + #[pyarg(named, name = "loop", optional)] + loop_: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + name: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + context: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + eager_start: OptionalOption<bool>, + } + + static TASK_NAME_COUNTER: AtomicU64 = AtomicU64::new(0); + + impl Constructor for PyTask { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + base: PyFuture::new_empty(), + task_coro: PyRwLock::new(None), + task_fut_waiter: PyRwLock::new(None), + task_name: PyRwLock::new(None), + task_context: PyRwLock::new(None), + task_must_cancel: AtomicBool::new(false), + task_num_cancels_requested: AtomicI32::new(0), + task_log_destroy_pending: AtomicBool::new(true), + }) + } + } + + impl Initializer for PyTask { + type Args = TaskInitArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + PyTask::py_init(&zelf, args, vm) + } + } + + #[pyclass( + flags(BASETYPE, HAS_DICT, HAS_WEAKREF), + with(Constructor, Initializer, Destructor, Representable, Iterable) + )] + impl PyTask { + fn py_init(zelf: &PyRef<Self>, args: TaskInitArgs, vm: &VirtualMachine) -> PyResult<()> { + // Validate coroutine + if !is_coroutine(args.coro.clone(), vm)? { + return Err(vm.new_type_error(format!( + "a coroutine was expected, got {}", + args.coro.repr(vm)? + ))); + } + + // Get the event loop + let loop_obj = match args.loop_.flatten() { + Some(l) => l, + None => get_running_loop(vm) + .map_err(|_| vm.new_runtime_error("no current event loop"))?, + }; + *zelf.base.fut_loop.write() = Some(loop_obj.clone()); + + // Check if loop has get_debug method and capture source traceback if enabled + if let Ok(Some(get_debug)) = + vm.get_attribute_opt(loop_obj.clone(), vm.ctx.intern_str("get_debug")) + && let Ok(debug) = get_debug.call((), vm) + && debug.try_to_bool(vm).unwrap_or(false) + { + // Get source traceback + if let Ok(tb_module) = vm.import("traceback", 0) + && let Ok(Some(extract_stack)) = + vm.get_attribute_opt(tb_module, vm.ctx.intern_str("extract_stack")) + && let Ok(tb) = extract_stack.call((), vm) + { + *zelf.base.fut_source_tb.write() = Some(tb); + } + } + + // Get or create context + let context = match args.context.flatten() { + Some(c) => c, + None => get_copy_context(vm)?, + }; + *zelf.task_context.write() = Some(context); + + // Set coroutine + *zelf.task_coro.write() = Some(args.coro); + + // Set task name + let name = match args.name.flatten() { + Some(n) => { + if !n.fast_isinstance(vm.ctx.types.str_type) { + n.str(vm)?.into() + } else { + n + } + } + None => { + let counter = TASK_NAME_COUNTER.fetch_add(1, Ordering::SeqCst); + vm.ctx.new_str(format!("Task-{}", counter + 1)).into() + } + }; + *zelf.task_name.write() = Some(name); + + let eager_start = args.eager_start.flatten().unwrap_or(false); + + // Check if we should do eager start: only if the loop is running + let do_eager_start = if eager_start { + let is_running = vm.call_method(&loop_obj, "is_running", ())?; + is_running.is_true(vm)? + } else { + false + }; + + if do_eager_start { + // Eager start: run first step synchronously (loop is already running) + task_eager_start(zelf, vm)?; + } else { + // Non-eager or loop not running: schedule the first step + _register_task(zelf.clone().into(), vm)?; + let task_obj: PyObjectRef = zelf.clone().into(); + let step_wrapper = TaskStepMethWrapper::new(task_obj).into_ref(&vm.ctx); + vm.call_method(&loop_obj, "call_soon", (step_wrapper,))?; + } + + Ok(()) + } + + // Future methods delegation + #[pymethod] + fn result(&self, vm: &VirtualMachine) -> PyResult { + match self.base.fut_state.load() { + FutureState::Pending => Err(new_invalid_state_error(vm, "Result is not ready.")), + FutureState::Cancelled => Err(self.make_cancelled_error_impl(vm)), + FutureState::Finished => { + self.base.fut_log_tb.store(false, Ordering::Relaxed); + if let Some(exc) = self.base.fut_exception.read().clone() { + let exc: PyBaseExceptionRef = exc.downcast().unwrap(); + // Restore the original traceback to prevent traceback accumulation + if let Some(tb) = self.base.fut_exception_tb.read().clone() { + let _ = exc.set___traceback__(tb, vm); + } + Err(exc) + } else { + Ok(self + .base + .fut_result + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none())) + } + } + } + } + + #[pymethod] + fn exception(&self, vm: &VirtualMachine) -> PyResult { + match self.base.fut_state.load() { + FutureState::Pending => Err(new_invalid_state_error(vm, "Exception is not set.")), + FutureState::Cancelled => Err(self.make_cancelled_error_impl(vm)), + FutureState::Finished => { + self.base.fut_log_tb.store(false, Ordering::Relaxed); + Ok(self + .base + .fut_exception + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none())) + } + } + } + + #[pymethod] + fn set_result( + _zelf: PyObjectRef, + _result: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + Err(vm.new_runtime_error("Task does not support set_result operation")) + } + + #[pymethod] + fn set_exception(&self, _exception: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_runtime_error("Task does not support set_exception operation")) + } + + fn make_cancelled_error_impl(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + // If a saved CancelledError exists, take it (clearing the stored reference) + if let Some(exc) = self.base.fut_cancelled_exc.write().take() + && let Ok(exc) = exc.downcast::<PyBaseException>() + { + return exc; + } + + let msg = self.base.fut_cancel_msg.read().clone(); + let args = if let Some(m) = msg { vec![m] } else { vec![] }; + + match get_cancelled_error_type(vm) { + Ok(cancelled_error) => vm.new_exception(cancelled_error, args), + Err(_) => vm.new_runtime_error("cancelled"), + } + } + + #[pymethod] + fn add_done_callback( + zelf: PyRef<Self>, + args: AddDoneCallbackArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if zelf.base.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.")); + } + let ctx = match args.context.flatten() { + Some(c) => c, + None => get_copy_context(vm)?, + }; + + if zelf.base.fut_state.load() != FutureState::Pending { + Self::call_soon_with_context(&zelf, args.func, Some(ctx), vm)?; + } else if zelf.base.fut_callback0.read().is_none() { + *zelf.base.fut_callback0.write() = Some(args.func); + *zelf.base.fut_context0.write() = Some(ctx); + } else { + let tuple = vm.ctx.new_tuple(vec![args.func, ctx]); + let mut callbacks = zelf.base.fut_callbacks.write(); + if callbacks.is_none() { + *callbacks = Some(vm.ctx.new_list(vec![tuple.into()]).into()); + } else { + let list = callbacks.as_ref().unwrap(); + vm.call_method(list, "append", (tuple,))?; + } + } + Ok(()) + } + + #[pymethod] + fn remove_done_callback(&self, func: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + if self.base.fut_loop.read().is_none() { + return Err(vm.new_runtime_error("Future object is not initialized.")); + } + let mut cleared_callback0 = 0usize; + + // Check fut_callback0 first + // Clone to release lock before comparison (which may run Python code) + let cb0 = self.base.fut_callback0.read().clone(); + if let Some(cb0) = cb0 { + let cmp = vm.identical_or_equal(&cb0, &func)?; + if cmp { + *self.base.fut_callback0.write() = None; + *self.base.fut_context0.write() = None; + cleared_callback0 = 1; + } + } + + // Check if fut_callbacks exists + let callbacks = self.base.fut_callbacks.read().clone(); + let callbacks = match callbacks { + Some(c) => c, + None => return Ok(cleared_callback0), + }; + + let list: PyListRef = callbacks.downcast().unwrap(); + let len = list.borrow_vec().len(); + + if len == 0 { + *self.base.fut_callbacks.write() = None; + return Ok(cleared_callback0); + } + + // Special case for single callback + if len == 1 { + let item = list.borrow_vec().first().cloned(); + if let Some(item) = item { + let tuple: &PyTuple = item.downcast_ref().unwrap(); + let cb = tuple.first().unwrap().clone(); + let cmp = vm.identical_or_equal(&cb, &func)?; + if cmp { + *self.base.fut_callbacks.write() = None; + return Ok(1 + cleared_callback0); + } + } + return Ok(cleared_callback0); + } + + // Multiple callbacks - iterate with index, checking validity each time + // to handle evil comparisons + let mut new_callbacks = Vec::with_capacity(len); + let mut i = 0usize; + let mut removed = 0usize; + + loop { + // Re-check fut_callbacks on each iteration (evil code may have cleared it) + let callbacks = self.base.fut_callbacks.read().clone(); + let callbacks = match callbacks { + Some(c) => c, + None => break, + }; + let list: PyListRef = callbacks.downcast().unwrap(); + let current_len = list.borrow_vec().len(); + if i >= current_len { + break; + } + + // Get item and release lock before comparison + let item = list.borrow_vec().get(i).cloned(); + let item = match item { + Some(item) => item, + None => break, + }; + + let tuple: &PyTuple = item.downcast_ref().unwrap(); + let cb = tuple.first().unwrap().clone(); + let cmp = vm.identical_or_equal(&cb, &func)?; + + if !cmp { + new_callbacks.push(item); + } else { + removed += 1; + } + i += 1; + } + + // Update fut_callbacks with filtered list + if new_callbacks.is_empty() { + *self.base.fut_callbacks.write() = None; + } else { + *self.base.fut_callbacks.write() = Some(vm.ctx.new_list(new_callbacks).into()); + } + + Ok(removed + cleared_callback0) + } + + fn schedule_callbacks(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Collect all callbacks first to avoid holding locks during callback execution + // This prevents deadlock when callbacks access the future's properties + let mut callbacks_to_call: Vec<(PyObjectRef, Option<PyObjectRef>)> = Vec::new(); + + // Take callback0 - release lock before collecting from list + let cb0 = zelf.base.fut_callback0.write().take(); + let ctx0 = zelf.base.fut_context0.write().take(); + if let Some(cb) = cb0 { + callbacks_to_call.push((cb, ctx0)); + } + + // Take callbacks list and collect items + let callbacks_list = zelf.base.fut_callbacks.write().take(); + if let Some(callbacks) = callbacks_list + && let Ok(list) = callbacks.downcast::<PyList>() + { + // Clone the items while holding the list lock, then release + let items: Vec<_> = list.borrow_vec().iter().cloned().collect(); + for item in items { + if let Some(tuple) = item.downcast_ref::<PyTuple>() + && let (Some(cb), Some(ctx)) = (tuple.first(), tuple.get(1)) + { + callbacks_to_call.push((cb.clone(), Some(ctx.clone()))); + } + } + } + + // Now call all callbacks without holding any locks + for (cb, ctx) in callbacks_to_call { + Self::call_soon_with_context(zelf, cb, ctx, vm)?; + } + + Ok(()) + } + + fn call_soon_with_context( + zelf: &PyRef<Self>, + callback: PyObjectRef, + context: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let loop_obj = zelf.base.fut_loop.read().clone(); + if let Some(loop_obj) = loop_obj { + // call_soon(callback, *args, context=context) + // callback receives the task as its argument + let task_arg: PyObjectRef = zelf.clone().into(); + let args = if let Some(ctx) = context { + FuncArgs::new( + vec![callback, task_arg], + KwArgs::new([("context".to_owned(), ctx)].into_iter().collect()), + ) + } else { + FuncArgs::new(vec![callback, task_arg], KwArgs::default()) + }; + vm.call_method(&loop_obj, "call_soon", args)?; + } + Ok(()) + } + + #[pymethod] + fn cancel(&self, args: CancelArgs, vm: &VirtualMachine) -> PyResult<bool> { + if self.base.fut_state.load() != FutureState::Pending { + // Clear log_tb even when cancel fails (task is already done) + self.base.fut_log_tb.store(false, Ordering::Relaxed); + return Ok(false); + } + + self.task_num_cancels_requested + .fetch_add(1, Ordering::SeqCst); + + let msg_value = args.msg.flatten(); + + if let Some(fut_waiter) = self.task_fut_waiter.read().clone() { + // Call cancel with msg=msg keyword argument + let cancel_args = if let Some(ref m) = msg_value { + FuncArgs::new( + vec![], + KwArgs::new([("msg".to_owned(), m.clone())].into_iter().collect()), + ) + } else { + FuncArgs::new(vec![], KwArgs::default()) + }; + let cancel_result = vm.call_method(&fut_waiter, "cancel", cancel_args)?; + if cancel_result.try_to_bool(vm)? { + return Ok(true); + } + } + + self.task_must_cancel.store(true, Ordering::Relaxed); + *self.base.fut_cancel_msg.write() = msg_value; + Ok(true) + } + + #[pymethod] + fn cancelled(&self) -> bool { + self.base.fut_state.load() == FutureState::Cancelled + } + + #[pymethod] + fn done(&self) -> bool { + self.base.fut_state.load() != FutureState::Pending + } + + #[pymethod] + fn cancelling(&self) -> i32 { + self.task_num_cancels_requested.load(Ordering::SeqCst) + } + + #[pymethod] + fn uncancel(&self) -> i32 { + let prev = self + .task_num_cancels_requested + .fetch_sub(1, Ordering::SeqCst); + if prev <= 0 { + self.task_num_cancels_requested.store(0, Ordering::SeqCst); + 0 + } else { + let new_val = prev - 1; + // When cancelling count reaches 0, reset _must_cancel + if new_val == 0 { + self.task_must_cancel.store(false, Ordering::SeqCst); + } + new_val + } + } + + #[pymethod] + fn get_coro(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_coro + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pymethod] + fn get_context(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_context + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pymethod] + fn get_name(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_name + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pymethod] + fn set_name(&self, name: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let name = if !name.fast_isinstance(vm.ctx.types.str_type) { + name.str(vm)?.into() + } else { + name + }; + *self.task_name.write() = Some(name); + Ok(()) + } + + #[pymethod] + fn get_loop(&self, vm: &VirtualMachine) -> PyResult { + self.base + .fut_loop + .read() + .clone() + .ok_or_else(|| vm.new_runtime_error("Task object is not initialized.")) + } + + #[pymethod] + fn get_stack(zelf: PyRef<Self>, args: GetStackArgs, vm: &VirtualMachine) -> PyResult { + let limit = args.limit.flatten().unwrap_or_else(|| vm.ctx.none()); + // vm.import returns the top-level module, get base_tasks submodule + let asyncio = vm.import("asyncio.base_tasks", 0)?; + let base_tasks = asyncio.get_attr(vm.ctx.intern_str("base_tasks"), vm)?; + let get_stack_func = base_tasks.get_attr(vm.ctx.intern_str("_task_get_stack"), vm)?; + get_stack_func.call((zelf, limit), vm) + } + + #[pymethod] + fn print_stack( + zelf: PyRef<Self>, + args: PrintStackArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + let limit = args.limit.flatten().unwrap_or_else(|| vm.ctx.none()); + let file = args.file.flatten().unwrap_or_else(|| vm.ctx.none()); + // vm.import returns the top-level module, get base_tasks submodule + let asyncio = vm.import("asyncio.base_tasks", 0)?; + let base_tasks = asyncio.get_attr(vm.ctx.intern_str("base_tasks"), vm)?; + let print_stack_func = + base_tasks.get_attr(vm.ctx.intern_str("_task_print_stack"), vm)?; + print_stack_func.call((zelf, limit, file), vm)?; + Ok(()) + } + + #[pymethod] + fn _make_cancelled_error(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.make_cancelled_error_impl(vm) + } + + // Properties + #[pygetset] + fn _state(&self) -> &'static str { + self.base.fut_state.load().as_str() + } + + #[pygetset] + fn _asyncio_future_blocking(&self) -> bool { + self.base.fut_blocking.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__asyncio_future_blocking( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + self.base.fut_blocking.store(v, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => Err(vm.new_attribute_error("cannot delete attribute")), + } + } + + #[pygetset] + fn _loop(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_loop + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _log_destroy_pending(&self) -> bool { + self.task_log_destroy_pending.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__log_destroy_pending( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + self.task_log_destroy_pending.store(v, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => { + Err(vm.new_attribute_error("can't delete _log_destroy_pending")) + } + } + } + + #[pygetset] + fn _log_traceback(&self) -> bool { + self.base.fut_log_tb.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set__log_traceback( + &self, + value: PySetterValue<bool>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => { + if v { + return Err(vm.new_value_error("_log_traceback can only be set to False")); + } + self.base.fut_log_tb.store(false, Ordering::Relaxed); + Ok(()) + } + PySetterValue::Delete => Err(vm.new_attribute_error("cannot delete attribute")), + } + } + + #[pygetset] + fn _must_cancel(&self) -> bool { + self.task_must_cancel.load(Ordering::Relaxed) + } + + #[pygetset] + fn _coro(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_coro + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _fut_waiter(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task_fut_waiter + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _source_traceback(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_source_tb + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _result(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_result + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _exception(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_exception + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn _cancel_message(&self, vm: &VirtualMachine) -> PyObjectRef { + self.base + .fut_cancel_msg + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset(setter)] + fn set__cancel_message(&self, value: PySetterValue) { + match value { + PySetterValue::Assign(v) => *self.base.fut_cancel_msg.write() = Some(v), + PySetterValue::Delete => *self.base.fut_cancel_msg.write() = None, + } + } + + #[pygetset] + fn _callbacks(&self, vm: &VirtualMachine) -> PyObjectRef { + let mut result: Vec<PyObjectRef> = Vec::new(); + if let Some(cb) = self.base.fut_callback0.read().clone() { + let ctx = self + .base + .fut_context0 + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()); + result.push(vm.ctx.new_tuple(vec![cb, ctx]).into()); + } + if let Some(callbacks) = self.base.fut_callbacks.read().clone() + && let Ok(list) = callbacks.downcast::<PyList>() + { + for item in list.borrow_vec().iter() { + result.push(item.clone()); + } + } + // Return None if no callbacks + if result.is_empty() { + vm.ctx.none() + } else { + vm.ctx.new_list(result).into() + } + } + + #[pymethod] + fn __await__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyFutureIter> { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + }) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl Destructor for PyTask { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + let loop_obj = zelf.base.fut_loop.read().clone(); + + // Check if task is pending and log_destroy_pending is True + if zelf.base.fut_state.load() == FutureState::Pending + && zelf.task_log_destroy_pending.load(Ordering::Relaxed) + { + if let Some(loop_obj) = loop_obj.clone() { + let context = PyDict::default().into_ref(&vm.ctx); + let task_repr = zelf + .as_object() + .repr(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<Task>")); + let message = + format!("Task was destroyed but it is pending!\ntask: {}", task_repr); + context.set_item( + vm.ctx.intern_str("message"), + vm.ctx.new_str(message).into(), + vm, + )?; + context.set_item(vm.ctx.intern_str("task"), zelf.to_owned().into(), vm)?; + + if let Some(tb) = zelf.base.fut_source_tb.read().clone() { + context.set_item(vm.ctx.intern_str("source_traceback"), tb, vm)?; + } + + let _ = vm.call_method(&loop_obj, "call_exception_handler", (context,)); + } + return Ok(()); + } + + // Check if we should log the traceback for exception + if !zelf.base.fut_log_tb.load(Ordering::Relaxed) { + return Ok(()); + } + + let exc = zelf.base.fut_exception.read().clone(); + let exc = match exc { + Some(e) => e, + None => return Ok(()), + }; + + let loop_obj = match loop_obj { + Some(l) => l, + None => return Ok(()), + }; + + // Create context dict for call_exception_handler + let context = PyDict::default().into_ref(&vm.ctx); + let class_name = zelf.class().name().to_string(); + let message = format!("{} exception was never retrieved", class_name); + context.set_item( + vm.ctx.intern_str("message"), + vm.ctx.new_str(message).into(), + vm, + )?; + context.set_item(vm.ctx.intern_str("exception"), exc, vm)?; + context.set_item(vm.ctx.intern_str("future"), zelf.to_owned().into(), vm)?; + + if let Some(tb) = zelf.base.fut_source_tb.read().clone() { + context.set_item(vm.ctx.intern_str("source_traceback"), tb, vm)?; + } + + // Call loop.call_exception_handler(context) + let _ = vm.call_method(&loop_obj, "call_exception_handler", (context,)); + Ok(()) + } + } + + impl Representable for PyTask { + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let class_name = zelf.class().name().to_string(); + + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + // Try to use _task_repr_info if available + if let Ok(info) = get_task_repr_info(zelf.as_object(), vm) + && info.as_bytes() != b"state=unknown" + { + return Ok(wtf8_concat!("<", class_name, " ", info, ">")); + } + + // Fallback: build repr from task properties directly + let state = zelf.base.fut_state.load().as_str().to_lowercase(); + let name = zelf.task_name.read().as_ref().and_then(|n| n.str(vm).ok()); + let coro_repr = zelf.task_coro.read().as_ref().and_then(|c| c.repr(vm).ok()); + let name = name.as_ref().map_or("?".as_ref(), |s| s.as_wtf8()); + let coro_repr = coro_repr.as_ref().map_or("?".as_ref(), |s| s.as_wtf8()); + Ok(wtf8_concat!( + "<", class_name, " ", state, " name='", name, "' coro=", coro_repr, ">" + )) + } else { + Ok(Wtf8Buf::from(format!("<{class_name} ...>"))) + } + } + } + + impl Iterable for PyTask { + fn iter(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + Ok(PyFutureIter { + future: PyRwLock::new(Some(zelf.into())), + } + .into_pyobject(_vm)) + } + } + + /// Eager start: run first step synchronously + fn task_eager_start(zelf: &PyRef<PyTask>, vm: &VirtualMachine) -> PyResult<()> { + let loop_obj = zelf.base.fut_loop.read().clone(); + let loop_obj = match loop_obj { + Some(l) => l, + None => return Err(vm.new_runtime_error("Task has no loop")), + }; + + // Register task before running step + let task_obj: PyObjectRef = zelf.clone().into(); + _register_task(task_obj.clone(), vm)?; + + // Register as eager task + _register_eager_task(task_obj.clone(), vm)?; + + // Swap current task - save previous task + let prev_task = _swap_current_task(loop_obj.clone(), task_obj.clone(), vm)?; + + // Get coro and context + let coro = zelf.task_coro.read().clone(); + let context = zelf.task_context.read().clone(); + + // Run the first step with context (using context.run(callable, *args)) + let step_result = if let Some(ctx) = context { + // Call context.run(coro.send, None) + let coro_ref = match coro { + Some(c) => c, + None => { + let _ = _swap_current_task(loop_obj.clone(), prev_task, vm); + _unregister_eager_task(task_obj.clone(), vm)?; + return Ok(()); + } + }; + let send_method = coro_ref.get_attr(vm.ctx.intern_str("send"), vm)?; + vm.call_method(&ctx, "run", (send_method, vm.ctx.none())) + } else { + // Run without context + match coro { + Some(c) => vm.call_method(&c, "send", (vm.ctx.none(),)), + None => { + let _ = _swap_current_task(loop_obj.clone(), prev_task, vm); + _unregister_eager_task(task_obj.clone(), vm)?; + return Ok(()); + } + } + }; + + // Restore previous task + let _ = _swap_current_task(loop_obj.clone(), prev_task, vm); + + // Unregister from eager tasks + _unregister_eager_task(task_obj.clone(), vm)?; + + // Handle the result + match step_result { + Ok(result) => { + task_step_handle_result(zelf, result, vm)?; + } + Err(e) => { + task_step_handle_exception(zelf, e, vm)?; + } + } + + // If task is no longer pending, clear the coroutine + if zelf.base.fut_state.load() != FutureState::Pending { + *zelf.task_coro.write() = None; + } + + Ok(()) + } + + /// Task step implementation + fn task_step_impl( + task: &PyObjectRef, + exc: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let task_ref: PyRef<PyTask> = task + .clone() + .downcast() + .map_err(|_| vm.new_type_error("task_step called with non-Task object"))?; + + if task_ref.base.fut_state.load() != FutureState::Pending { + // Task is already done - report InvalidStateError via exception handler + let loop_obj = task_ref.base.fut_loop.read().clone(); + if let Some(loop_obj) = loop_obj { + let exc = new_invalid_state_error(vm, "step(): already done"); + let context = vm.ctx.new_dict(); + context.set_item("message", vm.new_pyobj("step(): already done"), vm)?; + context.set_item("exception", exc.clone().into(), vm)?; + context.set_item("task", task.clone(), vm)?; + let _ = vm.call_method(&loop_obj, "call_exception_handler", (context,)); + } + return Ok(vm.ctx.none()); + } + + *task_ref.task_fut_waiter.write() = None; + + let coro = task_ref.task_coro.read().clone(); + let coro = match coro { + Some(c) => c, + None => return Ok(vm.ctx.none()), + }; + + // Get event loop for enter/leave task + let loop_obj = task_ref.base.fut_loop.read().clone(); + let loop_obj = match loop_obj { + Some(l) => l, + None => return Ok(vm.ctx.none()), + }; + + // Get task context + let context = task_ref.task_context.read().clone(); + + // Enter task - register as current task + _enter_task(loop_obj.clone(), task.clone(), vm)?; + + // Determine the exception to throw (if any) + // If task_must_cancel is set and exc is None or not CancelledError, create CancelledError + let exc_to_throw = if task_ref.task_must_cancel.load(Ordering::Relaxed) { + task_ref.task_must_cancel.store(false, Ordering::Relaxed); + if let Some(ref e) = exc { + if is_cancelled_error_obj(e, vm) { + exc.clone() + } else { + Some(task_ref.make_cancelled_error_impl(vm).into()) + } + } else { + Some(task_ref.make_cancelled_error_impl(vm).into()) + } + } else { + exc + }; + + // Run coroutine step within task's context + let result = if let Some(ctx) = context { + // Use context.run(callable, *args) to run within the task's context + if let Some(ref exc_obj) = exc_to_throw { + let throw_method = coro.get_attr(vm.ctx.intern_str("throw"), vm)?; + vm.call_method(&ctx, "run", (throw_method, exc_obj.clone())) + } else { + let send_method = coro.get_attr(vm.ctx.intern_str("send"), vm)?; + vm.call_method(&ctx, "run", (send_method, vm.ctx.none())) + } + } else { + // Fallback: run without context + if let Some(ref exc_obj) = exc_to_throw { + vm.call_method(&coro, "throw", (exc_obj.clone(),)) + } else { + vm.call_method(&coro, "send", (vm.ctx.none(),)) + } + }; + + // Leave task - unregister as current task (must happen even on error) + let _ = _leave_task(loop_obj, task.clone(), vm); + + match result { + Ok(result) => { + task_step_handle_result(&task_ref, result, vm)?; + } + Err(e) => { + task_step_handle_exception(&task_ref, e, vm)?; + } + } + + Ok(vm.ctx.none()) + } + + fn task_step_handle_result( + task: &PyRef<PyTask>, + result: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if task awaits on itself + let task_obj: PyObjectRef = task.clone().into(); + if result.is(&task_obj) { + let task_repr = task_obj.repr(vm)?; + let msg = format!("Task cannot await on itself: {}", task_repr.as_wtf8()); + task.base.fut_state.store(FutureState::Finished); + *task.base.fut_exception.write() = Some(vm.new_runtime_error(msg).into()); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task_obj, vm)?; + return Ok(()); + } + + let blocking = vm + .get_attribute_opt( + result.clone(), + vm.ctx.intern_str("_asyncio_future_blocking"), + )? + .and_then(|v| v.try_to_bool(vm).ok()) + .unwrap_or(false); + + if blocking { + result.set_attr( + vm.ctx.intern_str("_asyncio_future_blocking"), + vm.ctx.new_bool(false), + vm, + )?; + + // Get the future's loop, similar to get_future_loop: + // 1. If it's our native Future/Task, access fut_loop directly (check Task first) + // 2. Otherwise try get_loop(), falling back to _loop on AttributeError + let fut_loop = if let Ok(task) = result.clone().downcast::<PyTask>() { + task.base + .fut_loop + .read() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } else if let Ok(fut) = result.clone().downcast::<PyFuture>() { + fut.fut_loop.read().clone().unwrap_or_else(|| vm.ctx.none()) + } else { + // Try get_loop(), fall back to _loop on AttributeError + match vm.call_method(&result, "get_loop", ()) { + Ok(loop_obj) => loop_obj, + Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => { + result.get_attr(vm.ctx.intern_str("_loop"), vm)? + } + Err(e) => return Err(e), + } + }; + let task_loop = task.base.fut_loop.read().clone(); + if let Some(task_loop) = task_loop + && !fut_loop.is(&task_loop) + { + let task_repr = task + .as_object() + .repr(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<Task>")); + let result_repr = result + .repr(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<Future>")); + let msg = format!( + "Task {} got Future {} attached to a different loop", + task_repr, result_repr + ); + task.base.fut_state.store(FutureState::Finished); + *task.base.fut_exception.write() = Some(vm.new_runtime_error(msg).into()); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + return Ok(()); + } + + *task.task_fut_waiter.write() = Some(result.clone()); + + let task_obj: PyObjectRef = task.clone().into(); + let wakeup_wrapper = TaskWakeupMethWrapper::new(task_obj.clone()).into_ref(&vm.ctx); + vm.call_method(&result, "add_done_callback", (wakeup_wrapper,))?; + + // Track awaited_by relationship for introspection + future_add_to_awaited_by(result.clone(), task_obj, vm)?; + + // If task_must_cancel is set, cancel the awaited future immediately + // This propagates the cancellation through the future chain + if task.task_must_cancel.load(Ordering::Relaxed) { + let cancel_msg = task.base.fut_cancel_msg.read().clone(); + let cancel_args = if let Some(ref m) = cancel_msg { + FuncArgs::new( + vec![], + KwArgs::new([("msg".to_owned(), m.clone())].into_iter().collect()), + ) + } else { + FuncArgs::new(vec![], KwArgs::default()) + }; + let cancel_result = vm.call_method(&result, "cancel", cancel_args)?; + if cancel_result.try_to_bool(vm).unwrap_or(false) { + task.task_must_cancel.store(false, Ordering::Relaxed); + } + } + } else if vm.is_none(&result) { + let loop_obj = task.base.fut_loop.read().clone(); + if let Some(loop_obj) = loop_obj { + let task_obj: PyObjectRef = task.clone().into(); + let step_wrapper = TaskStepMethWrapper::new(task_obj).into_ref(&vm.ctx); + vm.call_method(&loop_obj, "call_soon", (step_wrapper,))?; + } + } else { + let result_repr = result.repr(vm)?; + let msg = format!("Task got bad yield: {}", result_repr.as_wtf8()); + task.base.fut_state.store(FutureState::Finished); + *task.base.fut_exception.write() = Some(vm.new_runtime_error(msg).into()); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + } + + Ok(()) + } + + fn task_step_handle_exception( + task: &PyRef<PyTask>, + exc: PyBaseExceptionRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check for KeyboardInterrupt or SystemExit - these should be re-raised + let should_reraise = exc.fast_isinstance(vm.ctx.exceptions.keyboard_interrupt) + || exc.fast_isinstance(vm.ctx.exceptions.system_exit); + + if exc.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + // Check if task was cancelled while running + if task.task_must_cancel.load(Ordering::Relaxed) { + // Task was cancelled - treat as cancelled instead of result + task.task_must_cancel.store(false, Ordering::Relaxed); + let cancelled_exc = task.base.make_cancelled_error_impl(vm); + task.base.fut_state.store(FutureState::Cancelled); + *task.base.fut_cancelled_exc.write() = Some(cancelled_exc.into()); + } else { + let result = exc.get_arg(0).unwrap_or_else(|| vm.ctx.none()); + task.base.fut_state.store(FutureState::Finished); + *task.base.fut_result.write() = Some(result); + } + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + } else if is_cancelled_error(&exc, vm) { + task.base.fut_state.store(FutureState::Cancelled); + *task.base.fut_cancelled_exc.write() = Some(exc.clone().into()); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + } else { + task.base.fut_state.store(FutureState::Finished); + // Save the original traceback for later restoration + let tb = exc.__traceback__().map(|tb| tb.into()); + *task.base.fut_exception_tb.write() = tb; + *task.base.fut_exception.write() = Some(exc.clone().into()); + task.base.fut_log_tb.store(true, Ordering::Relaxed); + PyTask::schedule_callbacks(task, vm)?; + _unregister_task(task.clone().into(), vm)?; + } + + // Re-raise KeyboardInterrupt and SystemExit after storing in task + if should_reraise { + return Err(exc); + } + + Ok(()) + } + + fn task_wakeup_impl(task: &PyObjectRef, fut: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + let task_ref: PyRef<PyTask> = task + .clone() + .downcast() + .map_err(|_| vm.new_type_error("task_wakeup called with non-Task object"))?; + + // Remove awaited_by relationship before resuming + future_discard_from_awaited_by(fut.clone(), task.clone(), vm)?; + + *task_ref.task_fut_waiter.write() = None; + + // Call result() on the awaited future to get either result or exception + // If result() raises an exception (like CancelledError), pass it to task_step + let exc = match vm.call_method(fut, "result", ()) { + Ok(_) => None, + Err(e) => Some(e.into()), + }; + + // Call task_step directly instead of using call_soon + // This allows the awaiting task to continue in the same event loop iteration + task_step_impl(task, exc, vm) + } + + // Module Functions + + fn get_all_tasks_set(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Use the module-level _scheduled_tasks WeakSet + let asyncio_module = vm.import("_asyncio", 0)?; + vm.get_attribute_opt(asyncio_module, vm.ctx.intern_str("_scheduled_tasks"))? + .ok_or_else(|| vm.new_attribute_error("_scheduled_tasks not found")) + } + + fn get_eager_tasks_set(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Use the module-level _eager_tasks Set + let asyncio_module = vm.import("_asyncio", 0)?; + vm.get_attribute_opt(asyncio_module, vm.ctx.intern_str("_eager_tasks"))? + .ok_or_else(|| vm.new_attribute_error("_eager_tasks not found")) + } + + fn get_current_tasks_dict(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Use the module-level _current_tasks Dict + let asyncio_module = vm.import("_asyncio", 0)?; + vm.get_attribute_opt(asyncio_module, vm.ctx.intern_str("_current_tasks"))? + .ok_or_else(|| vm.new_attribute_error("_current_tasks not found")) + } + + #[pyfunction] + fn _get_running_loop(vm: &VirtualMachine) -> PyObjectRef { + vm.asyncio_running_loop + .borrow() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pyfunction] + fn _set_running_loop(loop_: OptionalOption<PyObjectRef>, vm: &VirtualMachine) { + *vm.asyncio_running_loop.borrow_mut() = loop_.flatten(); + } + + #[pyfunction] + fn get_running_loop(vm: &VirtualMachine) -> PyResult { + vm.asyncio_running_loop + .borrow() + .clone() + .ok_or_else(|| vm.new_runtime_error("no running event loop")) + } + + #[pyfunction] + fn get_event_loop(vm: &VirtualMachine) -> PyResult { + if let Some(loop_) = vm.asyncio_running_loop.borrow().clone() { + return Ok(loop_); + } + + let asyncio_events = vm.import("asyncio.events", 0)?; + let get_event_loop_policy = vm + .get_attribute_opt(asyncio_events, vm.ctx.intern_str("get_event_loop_policy"))? + .ok_or_else(|| vm.new_attribute_error("get_event_loop_policy"))?; + let policy = get_event_loop_policy.call((), vm)?; + let get_event_loop = vm + .get_attribute_opt(policy, vm.ctx.intern_str("get_event_loop"))? + .ok_or_else(|| vm.new_attribute_error("get_event_loop"))?; + get_event_loop.call((), vm) + } + + #[pyfunction] + fn current_task(args: LoopArg, vm: &VirtualMachine) -> PyResult { + let loop_obj = match args.loop_.flatten() { + Some(l) if !vm.is_none(&l) => l, + _ => { + // When loop is None or not provided, use the running loop + match vm.asyncio_running_loop.borrow().clone() { + Some(l) => l, + None => return Err(vm.new_runtime_error("no running event loop")), + } + } + }; + + // Fast path: if the loop is the current thread's running loop, + // return the per-thread running task directly + let is_current_loop = vm + .asyncio_running_loop + .borrow() + .as_ref() + .is_some_and(|rl| rl.is(&loop_obj)); + + if is_current_loop { + return Ok(vm + .asyncio_running_task + .borrow() + .clone() + .unwrap_or_else(|| vm.ctx.none())); + } + + // Slow path: look up in the module-level dict for cross-thread queries + let current_tasks = get_current_tasks_dict(vm)?; + let dict: PyDictRef = current_tasks.downcast().unwrap(); + + match dict.get_item(&*loop_obj, vm) { + Ok(task) => Ok(task), + Err(_) => Ok(vm.ctx.none()), + } + } + + #[pyfunction] + fn all_tasks(args: LoopArg, vm: &VirtualMachine) -> PyResult { + let loop_obj = match args.loop_.flatten() { + Some(l) if !vm.is_none(&l) => l, + _ => get_running_loop(vm)?, + }; + + let all_tasks_set = get_all_tasks_set(vm)?; + let result_set = PySet::default().into_ref(&vm.ctx); + + let iter = vm.call_method(&all_tasks_set, "__iter__", ())?; + loop { + match vm.call_method(&iter, "__next__", ()) { + Ok(task) => { + // Try get_loop() method first, fallback to _loop property + let task_loop = if let Ok(l) = vm.call_method(&task, "get_loop", ()) { + Some(l) + } else if let Ok(Some(l)) = + vm.get_attribute_opt(task.clone(), vm.ctx.intern_str("_loop")) + { + Some(l) + } else { + None + }; + + if let Some(task_loop) = task_loop + && task_loop.is(&loop_obj) + && let Ok(done) = vm.call_method(&task, "done", ()) + && !done.try_to_bool(vm).unwrap_or(true) + { + result_set.add(task, vm)?; + } + } + Err(e) if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) => break, + Err(e) => return Err(e), + } + } + + Ok(result_set.into()) + } + + #[pyfunction] + fn _register_task(task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let all_tasks_set = get_all_tasks_set(vm)?; + vm.call_method(&all_tasks_set, "add", (task,))?; + Ok(()) + } + + #[pyfunction] + fn _unregister_task(task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let all_tasks_set = get_all_tasks_set(vm)?; + vm.call_method(&all_tasks_set, "discard", (task,))?; + Ok(()) + } + + #[pyfunction] + fn _register_eager_task(task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let eager_tasks_set = get_eager_tasks_set(vm)?; + vm.call_method(&eager_tasks_set, "add", (task,))?; + Ok(()) + } + + #[pyfunction] + fn _unregister_eager_task(task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let eager_tasks_set = get_eager_tasks_set(vm)?; + vm.call_method(&eager_tasks_set, "discard", (task,))?; + Ok(()) + } + + #[pyfunction] + fn _enter_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Per-thread check, matching CPython's ts->asyncio_running_task + { + let running_task = vm.asyncio_running_task.borrow(); + if running_task.is_some() { + return Err(vm.new_runtime_error(format!( + "Cannot enter into task {:?} while another task {:?} is being executed.", + task, + running_task.as_ref().unwrap() + ))); + } + } + + *vm.asyncio_running_task.borrow_mut() = Some(task.clone()); + + // Also update the module-level dict for cross-thread queries + if let Ok(current_tasks) = get_current_tasks_dict(vm) + && let Ok(dict) = current_tasks.downcast::<rustpython_vm::builtins::PyDict>() + { + let _ = dict.set_item(&*loop_, task, vm); + } + Ok(()) + } + + #[pyfunction] + fn _leave_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Per-thread check, matching CPython's ts->asyncio_running_task + { + let running_task = vm.asyncio_running_task.borrow(); + match running_task.as_ref() { + None => { + return Err(vm.new_runtime_error("_leave_task: task is not the current task")); + } + Some(current) if !current.is(&task) => { + return Err(vm.new_runtime_error("_leave_task: task is not the current task")); + } + _ => {} + } + } + + *vm.asyncio_running_task.borrow_mut() = None; + + // Also update the module-level dict + if let Ok(current_tasks) = get_current_tasks_dict(vm) + && let Ok(dict) = current_tasks.downcast::<rustpython_vm::builtins::PyDict>() + { + let _ = dict.del_item(&*loop_, vm); + } + Ok(()) + } + + #[pyfunction] + fn _swap_current_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Per-thread swap, matching CPython's swap_current_task + let prev = vm + .asyncio_running_task + .borrow() + .clone() + .unwrap_or_else(|| vm.ctx.none()); + + if vm.is_none(&task) { + *vm.asyncio_running_task.borrow_mut() = None; + } else { + *vm.asyncio_running_task.borrow_mut() = Some(task.clone()); + } + + // Also update the module-level dict for cross-thread queries + if let Ok(current_tasks) = get_current_tasks_dict(vm) + && let Ok(dict) = current_tasks.downcast::<rustpython_vm::builtins::PyDict>() + { + if vm.is_none(&task) { + let _ = dict.del_item(&*loop_, vm); + } else { + let _ = dict.set_item(&*loop_, task, vm); + } + } + + Ok(prev) + } + + /// Reset task state after fork in child process. + #[pyfunction] + fn _on_fork(vm: &VirtualMachine) -> PyResult<()> { + // Clear current_tasks dict so child process doesn't inherit parent's tasks + if let Ok(current_tasks) = get_current_tasks_dict(vm) { + vm.call_method(&current_tasks, "clear", ())?; + } + // Clear the running loop and task + *vm.asyncio_running_loop.borrow_mut() = None; + *vm.asyncio_running_task.borrow_mut() = None; + Ok(()) + } + + #[pyfunction] + fn future_add_to_awaited_by( + fut: PyObjectRef, + waiter: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Only operate on native Future/Task objects (including subclasses). + // Non-native objects are silently ignored. + if let Some(task) = fut.downcast_ref::<PyTask>() { + return task.base.awaited_by_add(waiter, vm); + } + if let Some(future) = fut.downcast_ref::<PyFuture>() { + return future.awaited_by_add(waiter, vm); + } + Ok(()) + } + + #[pyfunction] + fn future_discard_from_awaited_by( + fut: PyObjectRef, + waiter: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Only operate on native Future/Task objects (including subclasses). + // Non-native objects are silently ignored. + if let Some(task) = fut.downcast_ref::<PyTask>() { + return task.base.awaited_by_discard(&waiter, vm); + } + if let Some(future) = fut.downcast_ref::<PyFuture>() { + return future.awaited_by_discard(&waiter, vm); + } + Ok(()) + } + + // TaskStepMethWrapper - wrapper for task step callback with proper repr + + #[pyattr] + #[pyclass(name, traverse)] + #[derive(Debug, PyPayload)] + struct TaskStepMethWrapper { + task: PyRwLock<Option<PyObjectRef>>, + } + + #[pyclass(with(Callable, Representable))] + impl TaskStepMethWrapper { + fn new(task: PyObjectRef) -> Self { + Self { + task: PyRwLock::new(Some(task)), + } + } + + // __self__ property returns the task, used by _format_handle in base_events.py + #[pygetset] + fn __self__(&self, vm: &VirtualMachine) -> PyObjectRef { + self.task.read().clone().unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + match self.task.read().as_ref() { + Some(t) => vm.get_attribute_opt(t.clone(), vm.ctx.intern_str("__qualname__")), + None => Ok(None), + } + } + } + + impl Callable for TaskStepMethWrapper { + type Args = (); + fn call(zelf: &Py<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult { + let task = zelf.task.read().clone(); + match task { + Some(t) => task_step_impl(&t, None, vm), + None => Ok(vm.ctx.none()), + } + } + } + + impl Representable for TaskStepMethWrapper { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} object at {:#x}>", + zelf.class().name(), + zelf.get_id() + )) + } + } + + /// TaskWakeupMethWrapper - wrapper for task wakeup callback with proper repr + #[pyattr] + #[pyclass(name, traverse)] + #[derive(Debug, PyPayload)] + struct TaskWakeupMethWrapper { + task: PyRwLock<Option<PyObjectRef>>, + } + + #[pyclass(with(Callable, Representable))] + impl TaskWakeupMethWrapper { + fn new(task: PyObjectRef) -> Self { + Self { + task: PyRwLock::new(Some(task)), + } + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + match self.task.read().as_ref() { + Some(t) => vm.get_attribute_opt(t.clone(), vm.ctx.intern_str("__qualname__")), + None => Ok(None), + } + } + } + + impl Callable for TaskWakeupMethWrapper { + type Args = (PyObjectRef,); + fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let task = zelf.task.read().clone(); + match task { + Some(t) => task_wakeup_impl(&t, &args.0, vm), + None => Ok(vm.ctx.none()), + } + } + } + + impl Representable for TaskWakeupMethWrapper { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} object at {:#x}>", + zelf.class().name(), + zelf.get_id() + )) + } + } + + fn is_coroutine(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + if obj.class().is(vm.ctx.types.coroutine_type) { + return Ok(true); + } + + let asyncio_coroutines = vm.import("asyncio.coroutines", 0)?; + if let Some(iscoroutine) = + vm.get_attribute_opt(asyncio_coroutines, vm.ctx.intern_str("iscoroutine"))? + { + let result = iscoroutine.call((obj,), vm)?; + result.try_to_bool(vm) + } else { + Ok(false) + } + } + + fn new_invalid_state_error(vm: &VirtualMachine, msg: &str) -> PyBaseExceptionRef { + match vm.import("asyncio.exceptions", 0) { + Ok(module) => { + match vm.get_attribute_opt(module, vm.ctx.intern_str("InvalidStateError")) { + Ok(Some(exc_type)) => match exc_type.call((msg,), vm) { + Ok(exc) => exc.downcast().unwrap(), + Err(_) => vm.new_runtime_error(msg.to_string()), + }, + _ => vm.new_runtime_error(msg.to_string()), + } + } + Err(_) => vm.new_runtime_error(msg.to_string()), + } + } + + fn get_copy_context(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let contextvars = vm.import("contextvars", 0)?; + let copy_context = vm + .get_attribute_opt(contextvars, vm.ctx.intern_str("copy_context"))? + .ok_or_else(|| vm.new_attribute_error("copy_context not found"))?; + copy_context.call((), vm) + } + + fn get_cancelled_error_type(vm: &VirtualMachine) -> PyResult<PyTypeRef> { + let module = vm.import("asyncio.exceptions", 0)?; + let exc_type = vm + .get_attribute_opt(module, vm.ctx.intern_str("CancelledError"))? + .ok_or_else(|| vm.new_attribute_error("CancelledError not found"))?; + exc_type + .downcast() + .map_err(|_| vm.new_type_error("CancelledError is not a type")) + } + + fn is_cancelled_error(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> bool { + match get_cancelled_error_type(vm) { + Ok(cancelled_error) => exc.fast_isinstance(&cancelled_error), + Err(_) => false, + } + } + + fn is_cancelled_error_obj(obj: &PyObjectRef, vm: &VirtualMachine) -> bool { + match get_cancelled_error_type(vm) { + Ok(cancelled_error) => obj.fast_isinstance(&cancelled_error), + Err(_) => false, + } + } +} diff --git a/crates/stdlib/src/_opcode.rs b/crates/stdlib/src/_opcode.rs new file mode 100644 index 00000000000..ba1e6120fc0 --- /dev/null +++ b/crates/stdlib/src/_opcode.rs @@ -0,0 +1,341 @@ +pub(crate) use _opcode::module_def; + +#[pymodule] +mod _opcode { + use crate::vm::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyInt, PyIntRef}, + bytecode::{AnyInstruction, Instruction, InstructionMetadata, PseudoInstruction}, + }; + use core::ops::Deref; + + #[derive(Clone, Copy)] + struct Opcode(AnyInstruction); + + impl Deref for Opcode { + type Target = AnyInstruction; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl TryFrom<i32> for Opcode { + type Error = (); + + fn try_from(value: i32) -> Result<Self, Self::Error> { + Ok(Self( + u16::try_from(value) + .map_err(|_| ())? + .try_into() + .map_err(|_| ())?, + )) + } + } + + impl Opcode { + // https://github.com/python/cpython/blob/v3.14.2/Include/opcode_ids.h#L252 + const HAVE_ARGUMENT: i32 = 43; + + pub fn try_from_pyint(raw: PyIntRef, vm: &VirtualMachine) -> PyResult<Self> { + let instruction = raw + .try_to_primitive::<u16>(vm) + .and_then(|v| { + AnyInstruction::try_from(v).map_err(|_| { + vm.new_exception_empty(vm.ctx.exceptions.value_error.to_owned()) + }) + }) + .map_err(|_| vm.new_value_error("invalid opcode or oparg"))?; + + Ok(Self(instruction)) + } + + const fn inner(self) -> AnyInstruction { + self.0 + } + + /// Check if opcode is valid (can be converted to an AnyInstruction) + #[must_use] + pub fn is_valid(opcode: i32) -> bool { + Self::try_from(opcode).is_ok() + } + + /// Check if instruction has an argument + #[must_use] + pub fn has_arg(opcode: i32) -> bool { + Self::is_valid(opcode) && opcode > Self::HAVE_ARGUMENT + } + + /// Check if instruction uses co_consts + #[must_use] + pub fn has_const(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real(Instruction::LoadConst { .. })) + ) + } + + /// Check if instruction uses co_names + #[must_use] + pub fn has_name(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real( + Instruction::DeleteAttr { .. } + | Instruction::DeleteGlobal { .. } + | Instruction::DeleteName { .. } + | Instruction::ImportFrom { .. } + | Instruction::ImportName { .. } + | Instruction::LoadAttr { .. } + | Instruction::LoadGlobal { .. } + | Instruction::LoadName { .. } + | Instruction::StoreAttr { .. } + | Instruction::StoreGlobal { .. } + | Instruction::StoreName { .. } + )) + ) + } + + /// Check if instruction is a jump + #[must_use] + pub fn has_jump(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real( + Instruction::ForIter { .. } + | Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::Send { .. } + ) | AnyInstruction::Pseudo(PseudoInstruction::Jump { .. })) + ) + } + + /// Check if instruction uses co_freevars/co_cellvars + #[must_use] + pub fn has_free(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real( + Instruction::DeleteDeref { .. } + | Instruction::LoadFromDictOrDeref { .. } + | Instruction::LoadDeref { .. } + | Instruction::StoreDeref { .. } + )) + ) + } + + /// Check if instruction uses co_varnames (local variables) + #[must_use] + pub fn has_local(opcode: i32) -> bool { + matches!( + Self::try_from(opcode).map(|op| op.inner()), + Ok(AnyInstruction::Real( + Instruction::DeleteFast { .. } + | Instruction::LoadFast { .. } + | Instruction::LoadFastAndClear { .. } + | Instruction::StoreFast { .. } + | Instruction::StoreFastLoadFast { .. } + )) + ) + } + + /// Check if instruction has exception info + #[must_use] + pub fn has_exc(_opcode: i32) -> bool { + // No instructions have exception info in RustPython + // (exception handling is done via exception table) + false + } + } + + // prepare specialization + #[pyattr] + const ENABLE_SPECIALIZATION: i8 = 1; + #[pyattr] + const ENABLE_SPECIALIZATION_FT: i8 = 1; + + #[derive(FromArgs)] + struct StackEffectArgs { + #[pyarg(positional)] + opcode: PyIntRef, + #[pyarg(positional, optional)] + oparg: Option<PyObjectRef>, + #[pyarg(named, optional)] + jump: Option<PyObjectRef>, + } + + #[pyfunction] + fn stack_effect(args: StackEffectArgs, vm: &VirtualMachine) -> PyResult<i32> { + let oparg = args + .oparg + .map(|v| { + if !v.fast_isinstance(vm.ctx.types.int_type) { + return Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + v.class().name() + ))); + } + v.downcast_ref::<PyInt>() + .ok_or_else(|| { + vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + v.class().name() + )) + })? + .try_to_primitive::<u32>(vm) + }) + .unwrap_or(Ok(0))?; + + let jump = args + .jump + .map(|v| { + v.try_to_bool(vm).map_err(|_| { + vm.new_value_error("stack_effect: jump must be False, True or None") + }) + }) + .unwrap_or(Ok(false))?; + + let opcode = Opcode::try_from_pyint(args.opcode, vm)?; + + let _ = jump; // Python API accepts jump but it's not used + Ok(opcode.stack_effect(oparg)) + } + + #[pyfunction] + fn is_valid(opcode: i32) -> bool { + Opcode::is_valid(opcode) + } + + #[pyfunction] + fn has_arg(opcode: i32) -> bool { + Opcode::has_arg(opcode) + } + + #[pyfunction] + fn has_const(opcode: i32) -> bool { + Opcode::has_const(opcode) + } + + #[pyfunction] + fn has_name(opcode: i32) -> bool { + Opcode::has_name(opcode) + } + + #[pyfunction] + fn has_jump(opcode: i32) -> bool { + Opcode::has_jump(opcode) + } + + #[pyfunction] + fn has_free(opcode: i32) -> bool { + Opcode::has_free(opcode) + } + + #[pyfunction] + fn has_local(opcode: i32) -> bool { + Opcode::has_local(opcode) + } + + #[pyfunction] + fn has_exc(opcode: i32) -> bool { + Opcode::has_exc(opcode) + } + + #[pyfunction] + fn get_intrinsic1_descs(vm: &VirtualMachine) -> Vec<PyObjectRef> { + [ + "INTRINSIC_1_INVALID", + "INTRINSIC_PRINT", + "INTRINSIC_IMPORT_STAR", + "INTRINSIC_STOPITERATION_ERROR", + "INTRINSIC_ASYNC_GEN_WRAP", + "INTRINSIC_UNARY_POSITIVE", + "INTRINSIC_LIST_TO_TUPLE", + "INTRINSIC_TYPEVAR", + "INTRINSIC_PARAMSPEC", + "INTRINSIC_TYPEVARTUPLE", + "INTRINSIC_SUBSCRIPT_GENERIC", + "INTRINSIC_TYPEALIAS", + ] + .into_iter() + .map(|x| vm.ctx.new_str(x).into()) + .collect() + } + + #[pyfunction] + fn get_intrinsic2_descs(vm: &VirtualMachine) -> Vec<PyObjectRef> { + [ + "INTRINSIC_2_INVALID", + "INTRINSIC_PREP_RERAISE_STAR", + "INTRINSIC_TYPEVAR_WITH_BOUND", + "INTRINSIC_TYPEVAR_WITH_CONSTRAINTS", + "INTRINSIC_SET_FUNCTION_TYPE_PARAMS", + "INTRINSIC_SET_TYPEPARAM_DEFAULT", + ] + .into_iter() + .map(|x| vm.ctx.new_str(x).into()) + .collect() + } + + #[pyfunction] + fn get_nb_ops(vm: &VirtualMachine) -> Vec<PyObjectRef> { + [ + ("NB_ADD", "+"), + ("NB_AND", "&"), + ("NB_FLOOR_DIVIDE", "//"), + ("NB_LSHIFT", "<<"), + ("NB_MATRIX_MULTIPLY", "@"), + ("NB_MULTIPLY", "*"), + ("NB_REMAINDER", "%"), + ("NB_OR", "|"), + ("NB_POWER", "**"), + ("NB_RSHIFT", ">>"), + ("NB_SUBTRACT", "-"), + ("NB_TRUE_DIVIDE", "/"), + ("NB_XOR", "^"), + ("NB_INPLACE_ADD", "+="), + ("NB_INPLACE_AND", "&="), + ("NB_INPLACE_FLOOR_DIVIDE", "//="), + ("NB_INPLACE_LSHIFT", "<<="), + ("NB_INPLACE_MATRIX_MULTIPLY", "@="), + ("NB_INPLACE_MULTIPLY", "*="), + ("NB_INPLACE_REMAINDER", "%="), + ("NB_INPLACE_OR", "|="), + ("NB_INPLACE_POWER", "**="), + ("NB_INPLACE_RSHIFT", ">>="), + ("NB_INPLACE_SUBTRACT", "-="), + ("NB_INPLACE_TRUE_DIVIDE", "/="), + ("NB_INPLACE_XOR", "^="), + ("NB_SUBSCR", "[]"), + ] + .into_iter() + .map(|(a, b)| { + vm.ctx + .new_tuple(vec![vm.ctx.new_str(a).into(), vm.ctx.new_str(b).into()]) + .into() + }) + .collect() + } + + #[pyfunction] + fn get_special_method_names(vm: &VirtualMachine) -> Vec<PyObjectRef> { + ["__enter__", "__exit__", "__aenter__", "__aexit__"] + .into_iter() + .map(|x| vm.ctx.new_str(x).into()) + .collect() + } + + #[pyfunction] + fn get_executor( + _code: PyObjectRef, + _offset: i32, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + Ok(vm.ctx.none()) + } + + #[pyfunction] + fn get_specialization_stats(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } +} diff --git a/crates/stdlib/src/_remote_debugging.rs b/crates/stdlib/src/_remote_debugging.rs new file mode 100644 index 00000000000..618ea9fe0a8 --- /dev/null +++ b/crates/stdlib/src/_remote_debugging.rs @@ -0,0 +1,107 @@ +pub(crate) use _remote_debugging::module_def; + +#[pymodule] +mod _remote_debugging { + use crate::vm::{ + Py, PyObjectRef, PyResult, VirtualMachine, + builtins::PyType, + function::FuncArgs, + types::{Constructor, PyStructSequence}, + }; + + #[pystruct_sequence_data] + struct FrameInfoData { + filename: String, + lineno: i64, + funcname: String, + } + + #[pyattr] + #[pystruct_sequence( + name = "FrameInfo", + module = "_remote_debugging", + data = "FrameInfoData" + )] + struct FrameInfo; + + #[pyclass(with(PyStructSequence))] + impl FrameInfo {} + + #[pystruct_sequence_data] + struct TaskInfoData { + task_id: PyObjectRef, + task_name: PyObjectRef, + coroutine_stack: PyObjectRef, + awaited_by: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence(name = "TaskInfo", module = "_remote_debugging", data = "TaskInfoData")] + struct TaskInfo; + + #[pyclass(with(PyStructSequence))] + impl TaskInfo {} + + #[pystruct_sequence_data] + struct CoroInfoData { + call_stack: PyObjectRef, + task_name: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence(name = "CoroInfo", module = "_remote_debugging", data = "CoroInfoData")] + struct CoroInfo; + + #[pyclass(with(PyStructSequence))] + impl CoroInfo {} + + #[pystruct_sequence_data] + struct ThreadInfoData { + thread_id: PyObjectRef, + frame_info: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence( + name = "ThreadInfo", + module = "_remote_debugging", + data = "ThreadInfoData" + )] + struct ThreadInfo; + + #[pyclass(with(PyStructSequence))] + impl ThreadInfo {} + + #[pystruct_sequence_data] + struct AwaitedInfoData { + thread_id: PyObjectRef, + awaited_by: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence( + name = "AwaitedInfo", + module = "_remote_debugging", + data = "AwaitedInfoData" + )] + struct AwaitedInfo; + + #[pyclass(with(PyStructSequence))] + impl AwaitedInfo {} + + #[pyattr] + #[pyclass(name = "RemoteUnwinder", module = "_remote_debugging")] + #[derive(Debug, PyPayload)] + struct RemoteUnwinder {} + + impl Constructor for RemoteUnwinder { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Err(vm.new_not_implemented_error("_remote_debugging is not available")) + } + } + + #[pyclass(with(Constructor))] + impl RemoteUnwinder {} +} diff --git a/crates/stdlib/src/_sqlite3.rs b/crates/stdlib/src/_sqlite3.rs new file mode 100644 index 00000000000..db1350e0b91 --- /dev/null +++ b/crates/stdlib/src/_sqlite3.rs @@ -0,0 +1,3579 @@ +// spell-checker:ignore libsqlite3 threadsafety PYSQLITE decltypes colnames collseq cantinit dirtywal +// spell-checker:ignore corruptfs narg setinputsizes setoutputsize lastrowid arraysize executemany +// spell-checker:ignore blobopen executescript iterdump getlimit setlimit errorcode errorname +// spell-checker:ignore rowid rowcount fetchone fetchmany fetchall errcode errname vtable pagecount +// spell-checker:ignore autocommit libversion toobig errmsg nomem threadsafe longlong vdbe reindex +// spell-checker:ignore savepoint cantopen ioerr nolfs nomem notadb notfound fullpath notempdir vtab +// spell-checker:ignore checkreservedlock noent fstat rdlock shmlock shmmap shmopen shmsize sharedcache +// spell-checker:ignore cantlock commithook foreignkey notnull primarykey gettemppath autoindex convpath +// spell-checker:ignore dbmoved vnode nbytes + +pub(crate) use _sqlite3::module_def; + +#[pymodule] +mod _sqlite3 { + use core::{ + ffi::{CStr, c_int, c_longlong, c_uint, c_void}, + fmt::Debug, + ops::Deref, + ptr::{NonNull, null, null_mut}, + }; + use libsqlite3_sys::{ + SQLITE_BLOB, SQLITE_DETERMINISTIC, SQLITE_FLOAT, SQLITE_INTEGER, SQLITE_NULL, + SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE, SQLITE_OPEN_URI, SQLITE_TEXT, SQLITE_TRACE_STMT, + SQLITE_TRANSIENT, SQLITE_UTF8, sqlite3, sqlite3_aggregate_context, sqlite3_backup_finish, + sqlite3_backup_init, sqlite3_backup_pagecount, sqlite3_backup_remaining, + sqlite3_backup_step, sqlite3_bind_blob, sqlite3_bind_double, sqlite3_bind_int64, + sqlite3_bind_null, sqlite3_bind_parameter_count, sqlite3_bind_parameter_name, + sqlite3_bind_text, sqlite3_blob, sqlite3_blob_bytes, sqlite3_blob_close, sqlite3_blob_open, + sqlite3_blob_read, sqlite3_blob_write, sqlite3_busy_timeout, sqlite3_changes, + sqlite3_column_blob, sqlite3_column_bytes, sqlite3_column_count, sqlite3_column_decltype, + sqlite3_column_double, sqlite3_column_int64, sqlite3_column_name, sqlite3_column_text, + sqlite3_column_type, sqlite3_complete, sqlite3_context, sqlite3_context_db_handle, + sqlite3_create_collation_v2, sqlite3_create_function_v2, sqlite3_create_window_function, + sqlite3_data_count, sqlite3_db_handle, sqlite3_errcode, sqlite3_errmsg, sqlite3_exec, + sqlite3_expanded_sql, sqlite3_extended_errcode, sqlite3_finalize, sqlite3_get_autocommit, + sqlite3_interrupt, sqlite3_last_insert_rowid, sqlite3_libversion, sqlite3_limit, + sqlite3_open_v2, sqlite3_prepare_v2, sqlite3_progress_handler, sqlite3_reset, + sqlite3_result_blob, sqlite3_result_double, sqlite3_result_error, + sqlite3_result_error_nomem, sqlite3_result_error_toobig, sqlite3_result_int64, + sqlite3_result_null, sqlite3_result_text, sqlite3_set_authorizer, sqlite3_sleep, + sqlite3_step, sqlite3_stmt, sqlite3_stmt_busy, sqlite3_stmt_readonly, sqlite3_threadsafe, + sqlite3_total_changes, sqlite3_trace_v2, sqlite3_user_data, sqlite3_value, + sqlite3_value_blob, sqlite3_value_bytes, sqlite3_value_double, sqlite3_value_int64, + sqlite3_value_text, sqlite3_value_type, + }; + use malachite_bigint::Sign; + use rustpython_common::{ + atomic::{Ordering, PyAtomic, Radium}, + hash::PyHash, + lock::{PyMappedMutexGuard, PyMutex, PyMutexGuard}, + static_cell, + }; + use rustpython_vm::{ + __exports::paste, + AsObject, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, VirtualMachine, atomic_func, + builtins::{ + PyBaseException, PyBaseExceptionRef, PyByteArray, PyBytes, PyDict, PyDictRef, PyFloat, + PyInt, PyIntRef, PyModule, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, + PyTypeRef, PyUtf8Str, PyUtf8StrRef, + }, + convert::IntoObject, + function::{ + ArgCallable, ArgIterable, FsPath, FuncArgs, OptionalArg, PyComparisonValue, + PySetterValue, TimeoutSeconds, + }, + object::{Traverse, TraverseFn}, + protocol::{ + PyBuffer, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequence, + PySequenceMethods, + }, + sliceable::{SaturatedSliceIter, SliceableSequenceOp}, + types::{ + AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Hashable, + Initializer, IterNext, Iterable, PyComparisonOp, SelfIter, + }, + utils::ToCString, + }; + use std::thread::ThreadId; + + macro_rules! exceptions { + ($(($x:ident, $base:expr)),*) => { + paste::paste! { + static_cell! { + $( + static [<$x:snake:upper>]: PyTypeRef; + )* + } + $( + #[allow(dead_code)] + fn [<new_ $x:snake>](vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { + vm.new_exception_msg([<$x:snake _type>]().to_owned(), msg.into()) + } + fn [<$x:snake _type>]() -> &'static Py<PyType> { + [<$x:snake:upper>].get().expect("exception type not initialize") + } + )* + fn setup_module_exceptions(module: &PyObject, vm: &VirtualMachine) { + $( + #[allow(clippy::redundant_closure_call)] + let exception = [<$x:snake:upper>].get_or_init(|| { + let base = $base(vm); + vm.ctx.new_exception_type("_sqlite3", stringify!($x), Some(vec![base.to_owned()])) + }); + module.set_attr(stringify!($x), exception.clone().into_object(), vm).unwrap(); + )* + } + } + }; + } + + exceptions!( + (Warning, |vm: &VirtualMachine| vm + .ctx + .exceptions + .exception_type), + (Error, |vm: &VirtualMachine| vm + .ctx + .exceptions + .exception_type), + (InterfaceError, |_| error_type()), + (DatabaseError, |_| error_type()), + (DataError, |_| database_error_type()), + (OperationalError, |_| database_error_type()), + (IntegrityError, |_| database_error_type()), + (InternalError, |_| database_error_type()), + (ProgrammingError, |_| database_error_type()), + (NotSupportedError, |_| database_error_type()) + ); + + #[pyattr] + fn sqlite_version(vm: &VirtualMachine) -> String { + let s = unsafe { sqlite3_libversion() }; + ptr_to_str(s, vm).unwrap().to_owned() + } + + #[pyattr] + fn threadsafety(_: &VirtualMachine) -> c_int { + let mode = unsafe { sqlite3_threadsafe() }; + match mode { + 0 => 0, + 1 => 3, + 2 => 1, + _ => panic!("Unable to interpret SQLite threadsafety mode"), + } + } + + #[pyattr(name = "_deprecated_version")] + const PYSQLITE_VERSION: &str = "2.6.0"; + + #[pyattr] + const PARSE_DECLTYPES: c_int = 1; + #[pyattr] + const PARSE_COLNAMES: c_int = 2; + #[pyattr] + const LEGACY_TRANSACTION_CONTROL: c_int = -1; + + #[pyattr] + use libsqlite3_sys::{ + SQLITE_ALTER_TABLE, SQLITE_ANALYZE, SQLITE_ATTACH, SQLITE_CREATE_INDEX, + SQLITE_CREATE_TABLE, SQLITE_CREATE_TEMP_INDEX, SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_TEMP_TRIGGER, SQLITE_CREATE_TEMP_VIEW, SQLITE_CREATE_TRIGGER, + SQLITE_CREATE_VIEW, SQLITE_CREATE_VTABLE, SQLITE_DELETE, SQLITE_DENY, SQLITE_DETACH, + SQLITE_DROP_INDEX, SQLITE_DROP_TABLE, SQLITE_DROP_TEMP_INDEX, SQLITE_DROP_TEMP_TABLE, + SQLITE_DROP_TEMP_TRIGGER, SQLITE_DROP_TEMP_VIEW, SQLITE_DROP_TRIGGER, SQLITE_DROP_VIEW, + SQLITE_DROP_VTABLE, SQLITE_FUNCTION, SQLITE_IGNORE, SQLITE_INSERT, SQLITE_LIMIT_ATTACHED, + SQLITE_LIMIT_COLUMN, SQLITE_LIMIT_COMPOUND_SELECT, SQLITE_LIMIT_EXPR_DEPTH, + SQLITE_LIMIT_FUNCTION_ARG, SQLITE_LIMIT_LENGTH, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, + SQLITE_LIMIT_SQL_LENGTH, SQLITE_LIMIT_TRIGGER_DEPTH, SQLITE_LIMIT_VARIABLE_NUMBER, + SQLITE_LIMIT_VDBE_OP, SQLITE_LIMIT_WORKER_THREADS, SQLITE_PRAGMA, SQLITE_READ, + SQLITE_RECURSIVE, SQLITE_REINDEX, SQLITE_SAVEPOINT, SQLITE_SELECT, SQLITE_TRANSACTION, + SQLITE_UPDATE, + }; + + macro_rules! error_codes { + ($($x:ident),*) => { + $( + #[allow(unused_imports)] + use libsqlite3_sys::$x; + )* + static ERROR_CODES: &[(&str, c_int)] = &[ + $( + (stringify!($x), libsqlite3_sys::$x), + )* + ]; + }; + } + + error_codes!( + SQLITE_ABORT, + SQLITE_AUTH, + SQLITE_BUSY, + SQLITE_CANTOPEN, + SQLITE_CONSTRAINT, + SQLITE_CORRUPT, + SQLITE_DONE, + SQLITE_EMPTY, + SQLITE_ERROR, + SQLITE_FORMAT, + SQLITE_FULL, + SQLITE_INTERNAL, + SQLITE_INTERRUPT, + SQLITE_IOERR, + SQLITE_LOCKED, + SQLITE_MISMATCH, + SQLITE_MISUSE, + SQLITE_NOLFS, + SQLITE_NOMEM, + SQLITE_NOTADB, + SQLITE_NOTFOUND, + SQLITE_OK, + SQLITE_PERM, + SQLITE_PROTOCOL, + SQLITE_RANGE, + SQLITE_READONLY, + SQLITE_ROW, + SQLITE_SCHEMA, + SQLITE_TOOBIG, + SQLITE_NOTICE, + SQLITE_WARNING, + SQLITE_ABORT_ROLLBACK, + SQLITE_BUSY_RECOVERY, + SQLITE_CANTOPEN_FULLPATH, + SQLITE_CANTOPEN_ISDIR, + SQLITE_CANTOPEN_NOTEMPDIR, + SQLITE_CORRUPT_VTAB, + SQLITE_IOERR_ACCESS, + SQLITE_IOERR_BLOCKED, + SQLITE_IOERR_CHECKRESERVEDLOCK, + SQLITE_IOERR_CLOSE, + SQLITE_IOERR_DELETE, + SQLITE_IOERR_DELETE_NOENT, + SQLITE_IOERR_DIR_CLOSE, + SQLITE_IOERR_DIR_FSYNC, + SQLITE_IOERR_FSTAT, + SQLITE_IOERR_FSYNC, + SQLITE_IOERR_LOCK, + SQLITE_IOERR_NOMEM, + SQLITE_IOERR_RDLOCK, + SQLITE_IOERR_READ, + SQLITE_IOERR_SEEK, + SQLITE_IOERR_SHMLOCK, + SQLITE_IOERR_SHMMAP, + SQLITE_IOERR_SHMOPEN, + SQLITE_IOERR_SHMSIZE, + SQLITE_IOERR_SHORT_READ, + SQLITE_IOERR_TRUNCATE, + SQLITE_IOERR_UNLOCK, + SQLITE_IOERR_WRITE, + SQLITE_LOCKED_SHAREDCACHE, + SQLITE_READONLY_CANTLOCK, + SQLITE_READONLY_RECOVERY, + SQLITE_CONSTRAINT_CHECK, + SQLITE_CONSTRAINT_COMMITHOOK, + SQLITE_CONSTRAINT_FOREIGNKEY, + SQLITE_CONSTRAINT_FUNCTION, + SQLITE_CONSTRAINT_NOTNULL, + SQLITE_CONSTRAINT_PRIMARYKEY, + SQLITE_CONSTRAINT_TRIGGER, + SQLITE_CONSTRAINT_UNIQUE, + SQLITE_CONSTRAINT_VTAB, + SQLITE_READONLY_ROLLBACK, + SQLITE_IOERR_MMAP, + SQLITE_NOTICE_RECOVER_ROLLBACK, + SQLITE_NOTICE_RECOVER_WAL, + SQLITE_BUSY_SNAPSHOT, + SQLITE_IOERR_GETTEMPPATH, + SQLITE_WARNING_AUTOINDEX, + SQLITE_CANTOPEN_CONVPATH, + SQLITE_IOERR_CONVPATH, + SQLITE_CONSTRAINT_ROWID, + SQLITE_READONLY_DBMOVED, + SQLITE_AUTH_USER, + SQLITE_OK_LOAD_PERMANENTLY, + SQLITE_IOERR_VNODE, + SQLITE_IOERR_AUTH, + SQLITE_IOERR_BEGIN_ATOMIC, + SQLITE_IOERR_COMMIT_ATOMIC, + SQLITE_IOERR_ROLLBACK_ATOMIC, + SQLITE_ERROR_MISSING_COLLSEQ, + SQLITE_ERROR_RETRY, + SQLITE_READONLY_CANTINIT, + SQLITE_READONLY_DIRECTORY, + SQLITE_CORRUPT_SEQUENCE, + SQLITE_LOCKED_VTAB, + SQLITE_CANTOPEN_DIRTYWAL, + SQLITE_ERROR_SNAPSHOT, + SQLITE_CANTOPEN_SYMLINK, + SQLITE_CONSTRAINT_PINNED, + SQLITE_OK_SYMLINK, + SQLITE_BUSY_TIMEOUT, + SQLITE_CORRUPT_INDEX, + SQLITE_IOERR_DATA, + SQLITE_IOERR_CORRUPTFS + ); + + /// Autocommit mode setting for sqlite3 connections. + /// - Legacy (default): use isolation_level to control transactions + /// - Enabled: autocommit mode (no automatic transactions) + /// - Disabled: manual commit mode + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + enum AutocommitMode { + #[default] + Legacy, + Enabled, + Disabled, + } + + impl TryFromBorrowedObject<'_> for AutocommitMode { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Self> { + if obj.is(&vm.ctx.true_value) { + Ok(Self::Enabled) + } else if obj.is(&vm.ctx.false_value) { + Ok(Self::Disabled) + } else if let Ok(val) = obj.try_to_value::<c_int>(vm) { + if val == LEGACY_TRANSACTION_CONTROL { + Ok(Self::Legacy) + } else { + Err(vm.new_value_error(format!( + "autocommit must be True, False, or sqlite3.LEGACY_TRANSACTION_CONTROL, not {val}" + ))) + } + } else { + Err(vm.new_type_error(format!( + "autocommit must be True, False, or sqlite3.LEGACY_TRANSACTION_CONTROL, not {}", + obj.class().name() + ))) + } + } + } + + #[derive(FromArgs)] + struct ConnectArgs { + #[pyarg(any)] + database: FsPath, + #[pyarg(any, default = TimeoutSeconds::new(5.0))] + timeout: TimeoutSeconds, + #[pyarg(any, default = 0)] + detect_types: c_int, + #[pyarg(any, default = Some(vm.ctx.empty_str.to_owned()))] + isolation_level: Option<PyStrRef>, + #[pyarg(any, default = true)] + check_same_thread: bool, + #[pyarg(any, default = Connection::class(&vm.ctx).to_owned())] + factory: PyTypeRef, + // TODO: cache statements + #[allow(dead_code)] + #[pyarg(any, default = 0)] + cached_statements: c_int, + #[pyarg(any, default = false)] + uri: bool, + #[pyarg(any, default)] + autocommit: AutocommitMode, + } + + unsafe impl Traverse for ConnectArgs { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.isolation_level.traverse(tracer_fn); + self.factory.traverse(tracer_fn); + } + } + + #[derive(FromArgs)] + struct BackupArgs { + #[pyarg(any)] + target: PyRef<Connection>, + #[pyarg(named, default = -1)] + pages: c_int, + #[pyarg(named, optional)] + progress: Option<ArgCallable>, + #[pyarg(named, optional)] + name: Option<PyStrRef>, + #[pyarg(named, default = 0.250)] + sleep: f64, + } + + unsafe impl Traverse for BackupArgs { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.progress.traverse(tracer_fn); + self.name.traverse(tracer_fn); + } + } + + #[derive(FromArgs)] + struct CreateFunctionArgs { + #[pyarg(any)] + name: PyStrRef, + #[pyarg(any)] + narg: c_int, + #[pyarg(any)] + func: PyObjectRef, + #[pyarg(named, default)] + deterministic: bool, + } + + #[derive(FromArgs)] + struct CreateAggregateArgs { + #[pyarg(any)] + name: PyStrRef, + #[pyarg(positional)] + narg: c_int, + #[pyarg(positional)] + aggregate_class: PyObjectRef, + } + + #[derive(FromArgs)] + struct BlobOpenArgs { + #[pyarg(positional)] + table: PyStrRef, + #[pyarg(positional)] + column: PyStrRef, + #[pyarg(positional)] + row: i64, + #[pyarg(named, default)] + readonly: bool, + #[pyarg(named, default = vm.ctx.new_str("main"))] + name: PyStrRef, + } + + #[derive(FromArgs)] + struct CursorArgs { + #[pyarg(any, default)] + factory: OptionalArg<PyObjectRef>, + } + + struct CallbackData { + obj: NonNull<PyObject>, + vm: *const VirtualMachine, + } + + impl CallbackData { + fn new(obj: PyObjectRef, vm: &VirtualMachine) -> Option<Self> { + (!vm.is_none(&obj)).then_some(Self { + obj: obj.into_raw(), + vm, + }) + } + + fn retrieve(&self) -> (&PyObject, &VirtualMachine) { + unsafe { (self.obj.as_ref(), &*self.vm) } + } + + unsafe extern "C" fn destructor(data: *mut c_void) { + drop(unsafe { Box::from_raw(data.cast::<Self>()) }); + } + + unsafe extern "C" fn func_callback( + context: *mut sqlite3_context, + argc: c_int, + argv: *mut *mut sqlite3_value, + ) { + let context = SqliteContext::from(context); + let (func, vm) = unsafe { (*context.user_data::<Self>()).retrieve() }; + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; + + let f = || -> PyResult<()> { + let db = context.db_handle(); + let args = args + .iter() + .cloned() + .map(|val| value_to_object(val, db, vm)) + .collect::<PyResult<Vec<PyObjectRef>>>()?; + + let val = func.call(args, vm)?; + + context.result_from_object(&val, vm) + }; + + if let Err(exc) = f() { + context.result_exception(vm, exc, "user-defined function raised exception\0") + } + } + + unsafe extern "C" fn step_callback( + context: *mut sqlite3_context, + argc: c_int, + argv: *mut *mut sqlite3_value, + ) { + let context = SqliteContext::from(context); + let (cls, vm) = unsafe { (*context.user_data::<Self>()).retrieve() }; + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; + let instance = context.aggregate_context::<*const PyObject>(); + if unsafe { (*instance).is_null() } { + match cls.call((), vm) { + Ok(obj) => unsafe { *instance = obj.into_raw().as_ptr() }, + Err(exc) => { + return context.result_exception( + vm, + exc, + "user-defined aggregate's '__init__' method raised error\0", + ); + } + } + } + let instance = unsafe { &**instance }; + + Self::call_method_with_args(context, instance, "step", args, vm); + } + + unsafe extern "C" fn finalize_callback(context: *mut sqlite3_context) { + let context = SqliteContext::from(context); + let (_, vm) = unsafe { (*context.user_data::<Self>()).retrieve() }; + let instance = context.aggregate_context::<*const PyObject>(); + let Some(instance) = (unsafe { (*instance).as_ref() }) else { + return; + }; + + Self::callback_result_from_method(context, instance, "finalize", vm); + } + + unsafe extern "C" fn collation_callback( + data: *mut c_void, + a_len: c_int, + a_ptr: *const c_void, + b_len: c_int, + b_ptr: *const c_void, + ) -> c_int { + let (callable, vm) = unsafe { (*data.cast::<Self>()).retrieve() }; + + let f = || -> PyResult<c_int> { + let text1 = ptr_to_string(a_ptr.cast(), a_len, null_mut(), vm)?; + let text1 = vm.ctx.new_str(text1); + let text2 = ptr_to_string(b_ptr.cast(), b_len, null_mut(), vm)?; + let text2 = vm.ctx.new_str(text2); + + let val = callable.call((text1, text2), vm)?; + let Some(val) = val.number().index(vm) else { + return Ok(0); + }; + + let val = match val?.as_bigint().sign() { + Sign::Plus => 1, + Sign::Minus => -1, + Sign::NoSign => 0, + }; + + Ok(val) + }; + + f().unwrap_or(0) + } + + unsafe extern "C" fn value_callback(context: *mut sqlite3_context) { + let context = SqliteContext::from(context); + let (_, vm) = unsafe { (*context.user_data::<Self>()).retrieve() }; + let instance = context.aggregate_context::<*const PyObject>(); + let instance = unsafe { &**instance }; + + Self::callback_result_from_method(context, instance, "value", vm); + } + + unsafe extern "C" fn inverse_callback( + context: *mut sqlite3_context, + argc: c_int, + argv: *mut *mut sqlite3_value, + ) { + let context = SqliteContext::from(context); + let (_, vm) = unsafe { (*context.user_data::<Self>()).retrieve() }; + let args = unsafe { core::slice::from_raw_parts(argv, argc as usize) }; + let instance = context.aggregate_context::<*const PyObject>(); + let instance = unsafe { &**instance }; + + Self::call_method_with_args(context, instance, "inverse", args, vm); + } + + unsafe extern "C" fn authorizer_callback( + data: *mut c_void, + action: c_int, + arg1: *const libc::c_char, + arg2: *const libc::c_char, + db_name: *const libc::c_char, + access: *const libc::c_char, + ) -> c_int { + let (callable, vm) = unsafe { (*data.cast::<Self>()).retrieve() }; + let f = || -> PyResult<c_int> { + let arg1 = ptr_to_str(arg1, vm)?; + let arg2 = ptr_to_str(arg2, vm)?; + let db_name = ptr_to_str(db_name, vm)?; + let access = ptr_to_str(access, vm)?; + + let val = callable.call((action, arg1, arg2, db_name, access), vm)?; + let Some(val) = val.downcast_ref::<PyInt>() else { + return Ok(SQLITE_DENY); + }; + val.try_to_primitive::<c_int>(vm) + }; + + f().unwrap_or(SQLITE_DENY) + } + + unsafe extern "C" fn trace_callback( + _typ: c_uint, + data: *mut c_void, + stmt: *mut c_void, + sql: *mut c_void, + ) -> c_int { + let (callable, vm) = unsafe { (*data.cast::<Self>()).retrieve() }; + let expanded = unsafe { sqlite3_expanded_sql(stmt.cast()) }; + let f = || -> PyResult<()> { + let stmt = ptr_to_str(expanded, vm).or_else(|_| ptr_to_str(sql.cast(), vm))?; + callable.call((stmt,), vm)?; + Ok(()) + }; + let _ = f(); + 0 + } + + unsafe extern "C" fn progress_callback(data: *mut c_void) -> c_int { + let (callable, vm) = unsafe { (*data.cast::<Self>()).retrieve() }; + if let Ok(val) = callable.call((), vm) + && let Ok(val) = val.is_true(vm) + { + return val as c_int; + } + -1 + } + + fn callback_result_from_method( + context: SqliteContext, + instance: &PyObject, + name: &str, + vm: &VirtualMachine, + ) { + let f = || -> PyResult<()> { + let val = vm.call_method(instance, name, ())?; + context.result_from_object(&val, vm) + }; + + if let Err(exc) = f() { + if exc.fast_isinstance(vm.ctx.exceptions.attribute_error) { + context.result_exception( + vm, + exc, + &format!("user-defined aggregate's '{name}' method not defined\0"), + ) + } else { + context.result_exception( + vm, + exc, + &format!("user-defined aggregate's '{name}' method raised error\0"), + ) + } + } + } + + fn call_method_with_args( + context: SqliteContext, + instance: &PyObject, + name: &str, + args: &[*mut sqlite3_value], + vm: &VirtualMachine, + ) { + let f = || -> PyResult<()> { + let db = context.db_handle(); + let args = args + .iter() + .cloned() + .map(|val| value_to_object(val, db, vm)) + .collect::<PyResult<Vec<PyObjectRef>>>()?; + vm.call_method(instance, name, args).map(drop) + }; + + if let Err(exc) = f() { + if exc.fast_isinstance(vm.ctx.exceptions.attribute_error) { + context.result_exception( + vm, + exc, + &format!("user-defined aggregate's '{name}' method not defined\0"), + ) + } else { + context.result_exception( + vm, + exc, + &format!("user-defined aggregate's '{name}' method raised error\0"), + ) + } + } + } + } + + impl Drop for CallbackData { + fn drop(&mut self) { + unsafe { PyObjectRef::from_raw(self.obj) }; + } + } + + #[pyfunction] + fn connect(args: ConnectArgs, vm: &VirtualMachine) -> PyResult { + let factory = args.factory.clone(); + let conn = Connection::py_new(&factory, args, vm)?; + conn.into_ref_with_type(vm, factory).map(Into::into) + } + + #[pyfunction] + fn complete_statement(statement: PyStrRef, vm: &VirtualMachine) -> PyResult<bool> { + let s = statement.to_cstring(vm)?; + let ret = unsafe { sqlite3_complete(s.as_ptr()) }; + Ok(ret == 1) + } + + #[pyfunction] + fn enable_callback_tracebacks(flag: bool) { + enable_traceback().store(flag, Ordering::Relaxed); + } + + #[pyfunction] + fn register_adapter(typ: PyTypeRef, adapter: ArgCallable, vm: &VirtualMachine) -> PyResult<()> { + if typ.is(PyInt::class(&vm.ctx)) + || typ.is(PyFloat::class(&vm.ctx)) + || typ.is(PyStr::class(&vm.ctx)) + || typ.is(PyByteArray::class(&vm.ctx)) + { + let _ = BASE_TYPE_ADAPTED.set(()); + } + let protocol = PrepareProtocol::class(&vm.ctx).to_owned(); + let key = vm.ctx.new_tuple(vec![typ.into(), protocol.into()]); + adapters().set_item(key.as_object(), adapter.into(), vm) + } + + #[pyfunction] + fn register_converter( + typename: PyStrRef, + converter: ArgCallable, + vm: &VirtualMachine, + ) -> PyResult<()> { + let name = typename.expect_str().to_uppercase(); + converters().set_item(&name, converter.into(), vm) + } + + fn _adapt<F>(obj: &PyObject, proto: PyTypeRef, alt: F, vm: &VirtualMachine) -> PyResult + where + F: FnOnce(&PyObject) -> PyResult, + { + let proto = proto.into_object(); + let key = vm + .ctx + .new_tuple(vec![obj.class().to_owned().into(), proto.clone()]); + + if let Some(adapter) = adapters().get_item_opt(key.as_object(), vm)? { + return adapter.call((obj,), vm); + } + if let Ok(adapter) = proto.get_attr("__adapt__", vm) { + match adapter.call((obj,), vm) { + Ok(val) => { + if !vm.is_none(&val) { + return Ok(val); + } + } + Err(exc) => { + if !exc.fast_isinstance(vm.ctx.exceptions.type_error) { + return Err(exc); + } + } + } + } + if let Ok(adapter) = obj.get_attr("__conform__", vm) { + match adapter.call((proto,), vm) { + Ok(val) => { + if !vm.is_none(&val) { + return Ok(val); + } + } + Err(exc) => { + if !exc.fast_isinstance(vm.ctx.exceptions.type_error) { + return Err(exc); + } + } + } + } + + alt(obj) + } + + #[pyfunction] + fn adapt( + obj: PyObjectRef, + proto: OptionalArg<Option<PyTypeRef>>, + alt: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + if matches!(proto, OptionalArg::Present(None)) { + return if let OptionalArg::Present(alt) = alt { + Ok(alt) + } else { + Err(new_programming_error(vm, "can't adapt".to_owned())) + }; + } + + let proto = proto + .flatten() + .unwrap_or_else(|| PrepareProtocol::class(&vm.ctx).to_owned()); + + _adapt( + &obj, + proto, + |_| { + if let OptionalArg::Present(alt) = alt { + Ok(alt) + } else { + Err(new_programming_error(vm, "can't adapt".to_owned())) + } + }, + vm, + ) + } + + fn need_adapt(obj: &PyObject, vm: &VirtualMachine) -> bool { + if BASE_TYPE_ADAPTED.get().is_some() { + true + } else { + let cls = obj.class(); + !(cls.is(vm.ctx.types.int_type) + || cls.is(vm.ctx.types.float_type) + || cls.is(vm.ctx.types.str_type) + || cls.is(vm.ctx.types.bytearray_type)) + } + } + + static_cell! { + static CONVERTERS: PyDictRef; + static ADAPTERS: PyDictRef; + static BASE_TYPE_ADAPTED: (); + static USER_FUNCTION_EXCEPTION: PyAtomicRef<Option<PyBaseException>>; + static ENABLE_TRACEBACK: PyAtomic<bool>; + } + + fn converters() -> &'static Py<PyDict> { + CONVERTERS.get().expect("converters not initialize") + } + + fn adapters() -> &'static Py<PyDict> { + ADAPTERS.get().expect("adapters not initialize") + } + + fn user_function_exception() -> &'static PyAtomicRef<Option<PyBaseException>> { + USER_FUNCTION_EXCEPTION + .get() + .expect("user function exception not initialize") + } + + fn enable_traceback() -> &'static PyAtomic<bool> { + ENABLE_TRACEBACK + .get() + .expect("enable traceback not initialize") + } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + for (name, code) in ERROR_CODES { + let name = vm.ctx.intern_str(*name); + let code = vm.new_pyobj(*code); + module.set_attr(name, code, vm)?; + } + + setup_module_exceptions(module.as_object(), vm); + + let _ = CONVERTERS.set(vm.ctx.new_dict()); + let _ = ADAPTERS.set(vm.ctx.new_dict()); + let _ = USER_FUNCTION_EXCEPTION.set(PyAtomicRef::from(None)); + let _ = ENABLE_TRACEBACK.set(Radium::new(false)); + + module.set_attr("converters", converters().to_owned(), vm)?; + module.set_attr("adapters", adapters().to_owned(), vm)?; + + Ok(()) + } + + #[pyattr] + #[pyclass(name)] + #[derive(PyPayload)] + struct Connection { + db: PyMutex<Option<Sqlite>>, + initialized: PyAtomic<bool>, + detect_types: PyAtomic<c_int>, + isolation_level: PyAtomicRef<Option<PyStr>>, + check_same_thread: PyAtomic<bool>, + thread_ident: PyMutex<ThreadId>, // TODO: Use atomic + row_factory: PyAtomicRef<Option<PyObject>>, + text_factory: PyAtomicRef<PyObject>, + autocommit: PyMutex<AutocommitMode>, + } + + impl Debug for Connection { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + write!(f, "Sqlite3 Connection") + } + } + + impl Constructor for Connection { + type Args = ConnectArgs; + + fn py_new(cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let text_factory = PyStr::class(&vm.ctx).to_owned().into_object(); + + // For non-subclassed Connection, initialize in __new__ + // For subclassed Connection, leave db as None and require __init__ to be called + let is_base_class = cls.is(Connection::class(&vm.ctx)); + + let db = if is_base_class { + // Initialize immediately for base class + Some(Connection::initialize_db(&args, vm)?) + } else { + // For subclasses, require __init__ to be called + None + }; + + let initialized = db.is_some(); + + Ok(Self { + db: PyMutex::new(db), + initialized: Radium::new(initialized), + detect_types: Radium::new(args.detect_types), + isolation_level: PyAtomicRef::from(args.isolation_level), + check_same_thread: Radium::new(args.check_same_thread), + thread_ident: PyMutex::new(std::thread::current().id()), + row_factory: PyAtomicRef::from(None), + text_factory: PyAtomicRef::from(text_factory), + autocommit: PyMutex::new(args.autocommit), + }) + } + } + + impl Callable for Connection { + type Args = FuncArgs; + + fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let _ = zelf.db_lock(vm)?; + + let (sql,): (PyUtf8StrRef,) = args.bind(vm)?; + + if let Some(stmt) = Statement::new(zelf, sql, vm)? { + Ok(stmt.into_ref(&vm.ctx).into()) + } else { + Ok(vm.ctx.none()) + } + } + } + + impl Initializer for Connection { + type Args = ConnectArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let was_initialized = Radium::swap(&zelf.initialized, false, Ordering::AcqRel); + + // Reset factories to their defaults, matching CPython's behavior. + zelf.reset_factories(vm); + + if was_initialized { + zelf.drop_db(); + } + + // Attempt to open the new database before mutating other state so failures leave + // the connection uninitialized (and subsequent operations raise ProgrammingError). + let db = Self::initialize_db(&args, vm)?; + + let ConnectArgs { + detect_types, + isolation_level, + check_same_thread, + autocommit, + .. + } = args; + + zelf.detect_types.store(detect_types, Ordering::Relaxed); + zelf.check_same_thread + .store(check_same_thread, Ordering::Relaxed); + *zelf.autocommit.lock() = autocommit; + *zelf.thread_ident.lock() = std::thread::current().id(); + let _ = unsafe { zelf.isolation_level.swap(isolation_level) }; + + let mut guard = zelf.db.lock(); + *guard = Some(db); + Radium::store(&zelf.initialized, true, Ordering::Release); + Ok(()) + } + } + + #[pyclass(with(Constructor, Callable, Initializer), flags(BASETYPE, HAS_WEAKREF))] + impl Connection { + fn drop_db(&self) { + self.db.lock().take(); + } + + fn reset_factories(&self, vm: &VirtualMachine) { + let default_text_factory = PyStr::class(&vm.ctx).to_owned().into_object(); + let _ = unsafe { self.row_factory.swap(None) }; + let _ = unsafe { self.text_factory.swap(default_text_factory) }; + } + + fn initialize_db(args: &ConnectArgs, vm: &VirtualMachine) -> PyResult<Sqlite> { + let path = args.database.to_cstring(vm)?; + let db = Sqlite::from(SqliteRaw::open(path.as_ptr(), args.uri, vm)?); + let timeout = (args.timeout.to_secs_f64() * 1000.0) as c_int; + db.busy_timeout(timeout); + if let Some(isolation_level) = &args.isolation_level { + begin_statement_ptr_from_isolation_level(isolation_level, vm)?; + } + Ok(db) + } + + fn db_lock(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<'_, Sqlite>> { + self.check_thread(vm)?; + self._db_lock(vm) + } + + fn _db_lock(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<'_, Sqlite>> { + let guard = self.db.lock(); + if guard.is_some() { + Ok(PyMutexGuard::map(guard, |x| unsafe { + x.as_mut().unwrap_unchecked() + })) + } else { + Err(new_programming_error( + vm, + "Base Connection.__init__ not called.".to_owned(), + )) + } + } + + #[pymethod] + fn cursor( + zelf: PyRef<Self>, + args: CursorArgs, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + zelf.db_lock(vm).map(drop)?; + + let factory = match args.factory { + OptionalArg::Present(f) => f, + OptionalArg::Missing => Cursor::class(&vm.ctx).to_owned().into(), + }; + + let cursor = factory.call((zelf.clone(),), vm)?; + + if !cursor.class().fast_issubclass(Cursor::class(&vm.ctx)) { + return Err(vm.new_type_error(format!( + "factory must return a cursor, not {}", + cursor.class() + ))); + } + + if let Some(cursor_ref) = cursor.downcast_ref::<Cursor>() + && let Some(factory) = zelf.row_factory.to_owned() + { + let _ = unsafe { cursor_ref.row_factory.swap(Some(factory)) }; + } + + Ok(cursor) + } + + #[pymethod] + fn blobopen( + zelf: PyRef<Self>, + args: BlobOpenArgs, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Blob>> { + let table = args.table.to_cstring(vm)?; + let column = args.column.to_cstring(vm)?; + let name = args.name.to_cstring(vm)?; + + let db = zelf.db_lock(vm)?; + + let mut blob = null_mut(); + let ret = unsafe { + sqlite3_blob_open( + db.db, + name.as_ptr(), + table.as_ptr(), + column.as_ptr(), + args.row, + (!args.readonly) as c_int, + &mut blob, + ) + }; + db.check(ret, vm)?; + drop(db); + + let blob = SqliteBlob { blob }; + let blob = Blob { + connection: zelf, + inner: PyMutex::new(Some(BlobInner { blob, offset: 0 })), + }; + Ok(blob.into_ref(&vm.ctx)) + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + self.check_thread(vm)?; + self.drop_db(); + Ok(()) + } + + fn is_closed(&self) -> bool { + self.db.lock().is_none() + } + + #[pymethod] + fn commit(&self, vm: &VirtualMachine) -> PyResult<()> { + self.db_lock(vm)?.implicit_commit(vm) + } + + #[pymethod] + fn rollback(&self, vm: &VirtualMachine) -> PyResult<()> { + let db = self.db_lock(vm)?; + if !db.is_autocommit() { + db._exec(b"ROLLBACK\0", vm) + } else { + Ok(()) + } + } + + #[pymethod] + fn execute( + zelf: PyRef<Self>, + sql: PyUtf8StrRef, + parameters: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Cursor>> { + let row_factory = zelf.row_factory.to_owned(); + let cursor = Cursor::new(zelf, row_factory, vm).into_ref(&vm.ctx); + Cursor::execute(cursor, sql, parameters, vm) + } + + #[pymethod] + fn executemany( + zelf: PyRef<Self>, + sql: PyUtf8StrRef, + seq_of_params: ArgIterable, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Cursor>> { + let row_factory = zelf.row_factory.to_owned(); + let cursor = Cursor::new(zelf, row_factory, vm).into_ref(&vm.ctx); + Cursor::executemany(cursor, sql, seq_of_params, vm) + } + + #[pymethod] + fn executescript( + zelf: PyRef<Self>, + script: PyUtf8StrRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Cursor>> { + let row_factory = zelf.row_factory.to_owned(); + Cursor::executescript( + Cursor::new(zelf, row_factory, vm).into_ref(&vm.ctx), + script, + vm, + ) + } + + // TODO: Make it build without clippy::manual_c_str_literals + #[pymethod] + #[allow(clippy::manual_c_str_literals)] + fn backup(zelf: &Py<Self>, args: BackupArgs, vm: &VirtualMachine) -> PyResult<()> { + let BackupArgs { + target, + pages, + progress, + name, + sleep, + } = args; + if zelf.is(&target) { + return Err(vm.new_value_error("target cannot be the same connection instance")); + } + + let pages = if pages == 0 { -1 } else { pages }; + + let name_cstring; + let name_ptr = if let Some(name) = &name { + name_cstring = name.to_cstring(vm)?; + name_cstring.as_ptr() + } else { + b"main\0".as_ptr().cast() + }; + + let sleep_ms = (sleep * 1000.0) as c_int; + + let db = zelf.db_lock(vm)?; + let target_db = target.db_lock(vm)?; + + let handle = unsafe { + sqlite3_backup_init(target_db.db, b"main\0".as_ptr().cast(), db.db, name_ptr) + }; + + if handle.is_null() { + return Err(target_db.error_extended(vm)); + } + + drop(db); + drop(target_db); + + loop { + let ret = unsafe { sqlite3_backup_step(handle, pages) }; + + if let Some(progress) = &progress { + let remaining = unsafe { sqlite3_backup_remaining(handle) }; + let pagecount = unsafe { sqlite3_backup_pagecount(handle) }; + if let Err(err) = progress.invoke((ret, remaining, pagecount), vm) { + unsafe { sqlite3_backup_finish(handle) }; + return Err(err); + } + } + + if ret == SQLITE_BUSY || ret == SQLITE_LOCKED { + unsafe { sqlite3_sleep(sleep_ms) }; + } else if ret != SQLITE_OK { + break; + } + } + + let ret = unsafe { sqlite3_backup_finish(handle) }; + if ret == SQLITE_OK { + Ok(()) + } else { + Err(target.db_lock(vm)?.error_extended(vm)) + } + } + + #[pymethod] + fn create_function(&self, args: CreateFunctionArgs, vm: &VirtualMachine) -> PyResult<()> { + let name = args.name.to_cstring(vm)?; + let flags = if args.deterministic { + SQLITE_UTF8 | SQLITE_DETERMINISTIC + } else { + SQLITE_UTF8 + }; + let db = self.db_lock(vm)?; + let Some(data) = CallbackData::new(args.func, vm) else { + return db.create_function( + name.as_ptr(), + args.narg, + flags, + null_mut(), + None, + None, + None, + None, + vm, + ); + }; + + db.create_function( + name.as_ptr(), + args.narg, + flags, + Box::into_raw(Box::new(data)).cast(), + Some(CallbackData::func_callback), + None, + None, + Some(CallbackData::destructor), + vm, + ) + } + + #[pymethod] + fn create_aggregate(&self, args: CreateAggregateArgs, vm: &VirtualMachine) -> PyResult<()> { + let name = args.name.to_cstring(vm)?; + let db = self.db_lock(vm)?; + let Some(data) = CallbackData::new(args.aggregate_class, vm) else { + return db.create_function( + name.as_ptr(), + args.narg, + SQLITE_UTF8, + null_mut(), + None, + None, + None, + None, + vm, + ); + }; + + db.create_function( + name.as_ptr(), + args.narg, + SQLITE_UTF8, + Box::into_raw(Box::new(data)).cast(), + None, + Some(CallbackData::step_callback), + Some(CallbackData::finalize_callback), + Some(CallbackData::destructor), + vm, + ) + } + + #[pymethod] + fn create_collation( + &self, + name: PyUtf8StrRef, + callable: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let name = name.to_cstring(vm)?; + let db = self.db_lock(vm)?; + let Some(data) = CallbackData::new(callable.clone(), vm) else { + unsafe { + sqlite3_create_collation_v2( + db.db, + name.as_ptr(), + SQLITE_UTF8, + null_mut(), + None, + None, + ); + } + return Ok(()); + }; + let data = Box::into_raw(Box::new(data)); + + if !callable.is_callable() { + return Err(vm.new_type_error("parameter must be callable")); + } + + let ret = unsafe { + sqlite3_create_collation_v2( + db.db, + name.as_ptr(), + SQLITE_UTF8, + data.cast(), + Some(CallbackData::collation_callback), + Some(CallbackData::destructor), + ) + }; + + db.check(ret, vm).inspect_err(|_| { + // create_collation do not call destructor if error occur + let _ = unsafe { Box::from_raw(data) }; + }) + } + + #[pymethod] + fn create_window_function( + &self, + name: PyStrRef, + narg: c_int, + aggregate_class: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let name = name.to_cstring(vm)?; + let db = self.db_lock(vm)?; + let Some(data) = CallbackData::new(aggregate_class, vm) else { + unsafe { + sqlite3_create_window_function( + db.db, + name.as_ptr(), + narg, + SQLITE_UTF8, + null_mut(), + None, + None, + None, + None, + None, + ) + }; + return Ok(()); + }; + + let ret = unsafe { + sqlite3_create_window_function( + db.db, + name.as_ptr(), + narg, + SQLITE_UTF8, + Box::into_raw(Box::new(data)).cast(), + Some(CallbackData::step_callback), + Some(CallbackData::finalize_callback), + Some(CallbackData::value_callback), + Some(CallbackData::inverse_callback), + Some(CallbackData::destructor), + ) + }; + db.check(ret, vm) + .map_err(|_| new_programming_error(vm, "Error creating window function".to_owned())) + } + + #[pymethod] + fn set_authorizer(&self, callable: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let db = self.db_lock(vm)?; + let Some(data) = CallbackData::new(callable, vm) else { + unsafe { sqlite3_set_authorizer(db.db, None, null_mut()) }; + return Ok(()); + }; + + let ret = unsafe { + sqlite3_set_authorizer( + db.db, + Some(CallbackData::authorizer_callback), + Box::into_raw(Box::new(data)).cast(), + ) + }; + db.check(ret, vm).map_err(|_| { + new_operational_error(vm, "Error setting authorizer callback".to_owned()) + }) + } + + #[pymethod] + fn set_trace_callback(&self, callable: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let db = self.db_lock(vm)?; + let Some(data) = CallbackData::new(callable, vm) else { + unsafe { sqlite3_trace_v2(db.db, SQLITE_TRACE_STMT, None, null_mut()) }; + return Ok(()); + }; + + let ret = unsafe { + sqlite3_trace_v2( + db.db, + SQLITE_TRACE_STMT, + Some(CallbackData::trace_callback), + Box::into_raw(Box::new(data)).cast(), + ) + }; + + db.check(ret, vm) + } + + #[pymethod] + fn set_progress_handler( + &self, + callable: PyObjectRef, + n: c_int, + vm: &VirtualMachine, + ) -> PyResult<()> { + let db = self.db_lock(vm)?; + let Some(data) = CallbackData::new(callable, vm) else { + unsafe { sqlite3_progress_handler(db.db, n, None, null_mut()) }; + return Ok(()); + }; + + unsafe { + sqlite3_progress_handler( + db.db, + n, + Some(CallbackData::progress_callback), + Box::into_raw(Box::new(data)).cast(), + ) + }; + + Ok(()) + } + + #[pymethod] + fn iterdump(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let module = vm.import("sqlite3.dump", 0)?; + let func = module.get_attr("_iterdump", vm)?; + func.call((zelf,), vm) + } + + #[pymethod] + fn interrupt(&self, vm: &VirtualMachine) -> PyResult<()> { + // DO NOT check thread safety + self._db_lock(vm).map(|x| x.interrupt()) + } + + #[pymethod] + fn getlimit(&self, category: c_int, vm: &VirtualMachine) -> PyResult<c_int> { + self.db_lock(vm)?.limit(category, -1, vm) + } + + #[pymethod] + fn setlimit(&self, category: c_int, limit: c_int, vm: &VirtualMachine) -> PyResult<c_int> { + self.db_lock(vm)?.limit(category, limit, vm) + } + + #[pymethod] + fn __enter__(zelf: PyRef<Self>) -> PyRef<Self> { + zelf + } + + #[pymethod] + fn __exit__( + &self, + cls: PyObjectRef, + exc: PyObjectRef, + tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if vm.is_none(&cls) && vm.is_none(&exc) && vm.is_none(&tb) { + self.commit(vm) + } else { + self.rollback(vm) + } + } + + #[pygetset] + fn isolation_level(&self) -> Option<PyStrRef> { + self.isolation_level.deref().map(|x| x.to_owned()) + } + #[pygetset(setter)] + fn set_isolation_level( + &self, + value: PySetterValue<Option<PyStrRef>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + if let Some(val_str) = &value { + begin_statement_ptr_from_isolation_level(val_str, vm)?; + } + + // If setting isolation_level to None (auto-commit mode), commit any pending transaction + if value.is_none() { + let db = self.db_lock(vm)?; + if !db.is_autocommit() { + // Keep the lock and call implicit_commit directly to avoid race conditions + db.implicit_commit(vm)?; + } + } + let _ = unsafe { self.isolation_level.swap(value) }; + Ok(()) + } + PySetterValue::Delete => { + Err(vm.new_attribute_error("'isolation_level' attribute cannot be deleted")) + } + } + } + + #[pygetset] + fn autocommit(&self, vm: &VirtualMachine) -> PyObjectRef { + match *self.autocommit.lock() { + AutocommitMode::Enabled => vm.ctx.true_value.clone().into(), + AutocommitMode::Disabled => vm.ctx.false_value.clone().into(), + AutocommitMode::Legacy => vm.ctx.new_int(LEGACY_TRANSACTION_CONTROL).into(), + } + } + #[pygetset(setter)] + fn set_autocommit(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mode = AutocommitMode::try_from_borrowed_object(vm, &val)?; + let db = self.db_lock(vm)?; + + // Handle transaction state based on mode change + match mode { + AutocommitMode::Enabled => { + // If there's a pending transaction, commit it + if !db.is_autocommit() { + db._exec(b"COMMIT�", vm)?; + } + } + AutocommitMode::Disabled => { + // If not in a transaction, begin one + if db.is_autocommit() { + db._exec(b"BEGIN�", vm)?; + } + } + AutocommitMode::Legacy => { + // Legacy mode doesn't change transaction state + } + } + + drop(db); + *self.autocommit.lock() = mode; + Ok(()) + } + + #[pygetset] + fn text_factory(&self) -> PyObjectRef { + self.text_factory.to_owned() + } + #[pygetset(setter)] + fn set_text_factory(&self, val: PyObjectRef) { + let _ = unsafe { self.text_factory.swap(val) }; + } + + #[pygetset] + fn row_factory(&self) -> Option<PyObjectRef> { + self.row_factory.to_owned() + } + #[pygetset(setter)] + fn set_row_factory(&self, val: Option<PyObjectRef>) { + let _ = unsafe { self.row_factory.swap(val) }; + } + + fn check_thread(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.check_same_thread.load(Ordering::Relaxed) { + let creator_id = *self.thread_ident.lock(); + if std::thread::current().id() != creator_id { + return Err(new_programming_error( + vm, + "SQLite objects created in a thread can only be used in that same thread." + .to_owned(), + )); + } + } + Ok(()) + } + + #[pygetset] + fn in_transaction(&self, vm: &VirtualMachine) -> PyResult<bool> { + self._db_lock(vm).map(|x| !x.is_autocommit()) + } + + #[pygetset] + fn total_changes(&self, vm: &VirtualMachine) -> PyResult<c_int> { + self._db_lock(vm).map(|x| x.total_changes()) + } + } + + #[pyattr] + #[pyclass(name, traverse)] + #[derive(Debug, PyPayload)] + struct Cursor { + connection: PyRef<Connection>, + #[pytraverse(skip)] + arraysize: PyAtomic<c_int>, + #[pytraverse(skip)] + row_factory: PyAtomicRef<Option<PyObject>>, + inner: PyMutex<Option<CursorInner>>, + } + + #[derive(Debug, Traverse)] + struct CursorInner { + description: Option<PyTupleRef>, + row_cast_map: Vec<Option<PyObjectRef>>, + #[pytraverse(skip)] + lastrowid: i64, + #[pytraverse(skip)] + rowcount: i64, + statement: Option<PyRef<Statement>>, + #[pytraverse(skip)] + closed: bool, + } + + #[derive(FromArgs)] + struct FetchManyArgs { + #[pyarg(any, name = "size", optional)] + size: Option<c_int>, + } + + #[pyclass( + with(Constructor, Initializer, IterNext, Iterable), + flags(BASETYPE, HAS_WEAKREF) + )] + impl Cursor { + fn new( + connection: PyRef<Connection>, + row_factory: Option<PyObjectRef>, + _vm: &VirtualMachine, + ) -> Self { + Self { + connection, + arraysize: Radium::new(1), + row_factory: PyAtomicRef::from(row_factory), + inner: PyMutex::from(Some(CursorInner { + description: None, + row_cast_map: vec![], + lastrowid: -1, + rowcount: -1, + statement: None, + closed: false, + })), + } + } + + fn new_uninitialized(connection: PyRef<Connection>, _vm: &VirtualMachine) -> Self { + Self { + connection, + arraysize: Radium::new(1), + row_factory: PyAtomicRef::from(None), + inner: PyMutex::from(None), + } + } + + fn check_cursor_state(inner: Option<&CursorInner>, vm: &VirtualMachine) -> PyResult<()> { + match inner { + Some(inner) if inner.closed => Err(new_programming_error( + vm, + "Cannot operate on a closed cursor.".to_owned(), + )), + Some(_) => Ok(()), + None => Err(new_programming_error( + vm, + "Base Cursor.__init__ not called.".to_owned(), + )), + } + } + + fn inner(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<'_, CursorInner>> { + let guard = self.inner.lock(); + Self::check_cursor_state(guard.as_ref(), vm)?; + Ok(PyMutexGuard::map(guard, |x| unsafe { + x.as_mut().unwrap_unchecked() + })) + } + + /// Check if cursor is valid without retaining the lock. + /// Use this when you only need to verify the cursor state but don't need to modify it. + fn check_cursor_valid(&self, vm: &VirtualMachine) -> PyResult<()> { + let guard = self.inner.lock(); + Self::check_cursor_state(guard.as_ref(), vm) + } + + #[pymethod] + fn execute( + zelf: PyRef<Self>, + sql: PyUtf8StrRef, + parameters: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + let mut inner = zelf.inner(vm)?; + + if let Some(stmt) = inner.statement.take() { + stmt.lock().reset(); + } + + let Some(stmt) = Statement::new(&zelf.connection, sql, vm)? else { + drop(inner); + return Ok(zelf); + }; + let stmt = stmt.into_ref(&vm.ctx); + + inner.rowcount = if stmt.is_dml { 0 } else { -1 }; + + let db = zelf.connection.db_lock(vm)?; + + // Start implicit transaction for DML statements unless in autocommit mode + if stmt.is_dml + && db.is_autocommit() + && zelf.connection.isolation_level.deref().is_some() + && *zelf.connection.autocommit.lock() != AutocommitMode::Enabled + { + db.begin_transaction( + zelf.connection + .isolation_level + .deref() + .map(|x| x.to_owned()), + vm, + )?; + } + + let st = stmt.lock(); + let params_needed = st.bind_parameter_count(); + + if let OptionalArg::Present(parameters) = parameters { + st.bind_parameters(&parameters, vm)?; + } else if params_needed > 0 { + let msg = format!( + "Incorrect number of bindings supplied. The current statement uses {}, and 0 were supplied.", + params_needed + ); + return Err(new_programming_error(vm, msg)); + } + + let ret = st.step(); + + if ret != SQLITE_DONE && ret != SQLITE_ROW { + if let Some(exc) = unsafe { user_function_exception().swap(None) } { + return Err(exc); + } + return Err(db.error_extended(vm)); + } + + inner.row_cast_map = zelf.build_row_cast_map(&st, vm)?; + + let detect_types = zelf.connection.detect_types.load(Ordering::Relaxed); + inner.description = st.columns_description(detect_types, vm)?; + + if ret == SQLITE_ROW { + drop(st); + inner.statement = Some(stmt); + } else { + st.reset(); + drop(st); + if stmt.is_dml { + inner.rowcount += db.changes() as i64; + } + } + + inner.lastrowid = db.lastrowid(); + + drop(inner); + drop(db); + Ok(zelf) + } + + #[pymethod] + fn executemany( + zelf: PyRef<Self>, + sql: PyUtf8StrRef, + seq_of_params: ArgIterable, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + let mut inner = zelf.inner(vm)?; + + if let Some(stmt) = inner.statement.take() { + stmt.lock().reset(); + } + + let Some(stmt) = Statement::new(&zelf.connection, sql, vm)? else { + drop(inner); + return Ok(zelf); + }; + let stmt = stmt.into_ref(&vm.ctx); + + let st = stmt.lock(); + + if st.readonly() { + return Err(new_programming_error( + vm, + "executemany() can only execute DML statements.".to_owned(), + )); + } + + let detect_types = zelf.connection.detect_types.load(Ordering::Relaxed); + inner.description = st.columns_description(detect_types, vm)?; + + inner.rowcount = if stmt.is_dml { 0 } else { -1 }; + + let db = zelf.connection.db_lock(vm)?; + + // Start implicit transaction for DML statements unless in autocommit mode + if stmt.is_dml + && db.is_autocommit() + && zelf.connection.isolation_level.deref().is_some() + && *zelf.connection.autocommit.lock() != AutocommitMode::Enabled + { + db.begin_transaction( + zelf.connection + .isolation_level + .deref() + .map(|x| x.to_owned()), + vm, + )?; + } + + let iter = seq_of_params.iter(vm)?; + for params in iter { + let params = params?; + st.bind_parameters(&params, vm)?; + + if !st.step_row_else_done(vm)? { + if stmt.is_dml { + inner.rowcount += db.changes() as i64; + } + st.reset(); + } + + // if let Some(exc) = unsafe { user_function_exception().swap(None) } { + // return Err(exc); + // } + } + + if st.busy() { + drop(st); + inner.statement = Some(stmt); + } + + drop(inner); + drop(db); + Ok(zelf) + } + + #[pymethod] + fn executescript( + zelf: PyRef<Self>, + script: PyUtf8StrRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + zelf.check_cursor_valid(vm)?; + + let db = zelf.connection.db_lock(vm)?; + + db.sql_limit(script.byte_len(), vm)?; + + db.implicit_commit(vm)?; + + let script = script.to_cstring(vm)?; + let mut ptr = script.as_ptr(); + + while let Some(st) = db.prepare(ptr, &mut ptr, vm)? { + while st.step_row_else_done(vm)? {} + } + + drop(db); + Ok(zelf) + } + + #[pymethod] + fn fetchone(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult { + Self::next(zelf, vm).map(|x| match x { + PyIterReturn::Return(row) => row, + PyIterReturn::StopIteration(_) => vm.ctx.none(), + }) + } + + #[pymethod] + fn fetchmany( + zelf: &Py<Self>, + args: FetchManyArgs, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + let max_rows = match args.size { + Some(size) => { + if size < 0 { + return Err(vm.new_value_error("fetchmany may not be negative")); + } + + size + } + None => zelf.arraysize.load(Ordering::Relaxed), + }; + + let mut list = vec![]; + let mut remaining = max_rows; + while remaining > 0 { + match Cursor::next(zelf, vm)? { + PyIterReturn::Return(row) => { + list.push(row); + remaining -= 1; + } + PyIterReturn::StopIteration(_) => break, + } + } + Ok(list) + } + + #[pymethod] + fn fetchall(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let mut list = vec![]; + while let PyIterReturn::Return(row) = Self::next(zelf, vm)? { + list.push(row); + } + Ok(list) + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + // Check if __init__ was called + let mut guard = self.inner.lock(); + + let Some(inner) = guard.as_mut() else { + return Err(new_programming_error( + vm, + "Base Cursor.__init__ not called.".to_owned(), + )); + }; + + if let Some(stmt) = &inner.statement { + stmt.lock().reset(); + } + inner.closed = true; + + Ok(()) + } + + #[pymethod] + fn setinputsizes(&self, _sizes: PyObjectRef) {} + #[pymethod] + fn setoutputsize(&self, _size: PyObjectRef, _column: OptionalArg<PyObjectRef>) {} + + #[pygetset] + fn connection(&self) -> PyRef<Connection> { + self.connection.clone() + } + + #[pygetset] + fn lastrowid(&self, vm: &VirtualMachine) -> PyResult<i64> { + self.inner(vm).map(|x| x.lastrowid) + } + + #[pygetset] + fn rowcount(&self, vm: &VirtualMachine) -> PyResult<i64> { + self.inner(vm).map(|x| x.rowcount) + } + + #[pygetset] + fn description(&self, vm: &VirtualMachine) -> PyResult<Option<PyTupleRef>> { + self.inner(vm).map(|x| x.description.clone()) + } + + #[pygetset] + fn arraysize(&self) -> c_int { + self.arraysize.load(Ordering::Relaxed) + } + + #[pygetset(setter)] + fn set_arraysize(&self, val: c_int, vm: &VirtualMachine) -> PyResult<()> { + if val < 0 { + return Err(vm.new_value_error("arraysize may not be negative")); + } + + self.arraysize.store(val, Ordering::Relaxed); + + Ok(()) + } + + #[pygetset] + fn row_factory(&self) -> Option<PyObjectRef> { + self.row_factory.to_owned() + } + + #[pygetset(setter)] + fn set_row_factory(&self, val: Option<PyObjectRef>) { + let _ = unsafe { self.row_factory.swap(val) }; + } + + fn build_row_cast_map( + &self, + st: &SqliteStatementRaw, + vm: &VirtualMachine, + ) -> PyResult<Vec<Option<PyObjectRef>>> { + let detect_types = self.connection.detect_types.load(Ordering::Relaxed); + if detect_types == 0 { + return Ok(vec![]); + } + + let mut cast_map = vec![]; + let num_cols = st.column_count(); + + for i in 0..num_cols { + if detect_types & PARSE_COLNAMES != 0 { + let col_name = st.column_name(i); + let col_name = ptr_to_str(col_name, vm)?; + let col_name = col_name + .chars() + .skip_while(|&x| x != '[') + .skip(1) + .take_while(|&x| x != ']') + .flat_map(|x| x.to_uppercase()) + .collect::<String>(); + if let Some(converter) = converters().get_item_opt(&col_name, vm)? { + cast_map.push(Some(converter.clone())); + continue; + } + } + if detect_types & PARSE_DECLTYPES != 0 { + let decltype = st.column_decltype(i); + let decltype = ptr_to_str(decltype, vm)?; + if let Some(decltype) = decltype.split_terminator(&[' ', '(']).next() { + let decltype = decltype.to_uppercase(); + if let Some(converter) = converters().get_item_opt(&decltype, vm)? { + cast_map.push(Some(converter.clone())); + continue; + } + } + } + cast_map.push(None); + } + + Ok(cast_map) + } + } + + impl Constructor for Cursor { + type Args = (PyRef<Connection>,); + + fn py_new( + _cls: &Py<PyType>, + (connection,): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self::new_uninitialized(connection, vm)) + } + } + + impl Initializer for Cursor { + type Args = PyRef<Connection>; + + fn init(zelf: PyRef<Self>, _connection: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + let mut guard = zelf.inner.lock(); + if guard.is_some() { + // Already initialized (e.g., from a call to super().__init__) + return Ok(()); + } + *guard = Some(CursorInner { + description: None, + row_cast_map: vec![], + lastrowid: -1, + rowcount: -1, + statement: None, + closed: false, + }); + Ok(()) + } + } + + impl SelfIter for Cursor {} + impl IterNext for Cursor { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + // Check if connection is closed first, and if so, clear statement to release file lock + if zelf.connection.is_closed() { + let mut guard = zelf.inner.lock(); + if let Some(stmt) = guard.as_mut().and_then(|inner| inner.statement.take()) { + stmt.lock().reset(); + } + return Err(new_programming_error( + vm, + "Cannot operate on a closed database.".to_owned(), + )); + } + + let mut inner = zelf.inner(vm)?; + let Some(stmt) = &inner.statement else { + return Ok(PyIterReturn::StopIteration(None)); + }; + let st = stmt.lock(); + let db = zelf.connection.db_lock(vm)?; + // fetch_one_row + + let num_cols = st.data_count(); + + let mut row = Vec::with_capacity(num_cols as usize); + + for i in 0..num_cols { + let val = if let Some(converter) = + inner.row_cast_map.get(i as usize).cloned().flatten() + { + let blob = st.column_blob(i); + if blob.is_null() { + vm.ctx.none() + } else { + let nbytes = st.column_bytes(i); + let blob = unsafe { + core::slice::from_raw_parts(blob.cast::<u8>(), nbytes as usize) + }; + let blob = vm.ctx.new_bytes(blob.to_vec()); + converter.call((blob,), vm)? + } + } else { + let col_type = st.column_type(i); + match col_type { + SQLITE_NULL => vm.ctx.none(), + SQLITE_INTEGER => vm.ctx.new_int(st.column_int(i)).into(), + SQLITE_FLOAT => vm.ctx.new_float(st.column_double(i)).into(), + SQLITE_TEXT => { + let text = + ptr_to_vec(st.column_text(i), st.column_bytes(i), db.db, vm)?; + + let text_factory = zelf.connection.text_factory.to_owned(); + + if text_factory.is(PyStr::class(&vm.ctx)) { + let text = String::from_utf8(text).map_err(|err| { + let col_name = st.column_name(i); + let col_name_str = ptr_to_str(col_name, vm).unwrap_or("?"); + let valid_up_to = err.utf8_error().valid_up_to(); + let text_prefix = String::from_utf8_lossy(&err.as_bytes()[..valid_up_to]); + let msg = format!( + "Could not decode to UTF-8 column '{col_name_str}' with text '{text_prefix}'" + ); + new_operational_error(vm, msg) + })?; + vm.ctx.new_str(text).into() + } else if text_factory.is(PyBytes::class(&vm.ctx)) { + vm.ctx.new_bytes(text).into() + } else if text_factory.is(PyByteArray::class(&vm.ctx)) { + PyByteArray::from(text).into_ref(&vm.ctx).into() + } else { + let bytes = vm.ctx.new_bytes(text); + text_factory.call((bytes,), vm)? + } + } + SQLITE_BLOB => { + let blob = ptr_to_vec( + st.column_blob(i).cast(), + st.column_bytes(i), + db.db, + vm, + )?; + + vm.ctx.new_bytes(blob).into() + } + _ => { + return Err(vm.new_not_implemented_error(format!( + "unknown column type: {col_type}" + ))); + } + } + }; + + row.push(val); + } + + if !st.step_row_else_done(vm)? { + st.reset(); + drop(st); + if stmt.is_dml { + inner.rowcount = db.changes() as i64; + } + inner.statement = None; + } else { + drop(st); + } + + drop(db); + drop(inner); + + let row = vm.ctx.new_tuple(row); + + if let Some(row_factory) = zelf.row_factory.to_owned() { + row_factory + .call((zelf.to_owned(), row), vm) + .map(PyIterReturn::Return) + } else { + Ok(PyIterReturn::Return(row.into())) + } + } + } + + #[pyattr] + #[pyclass(name, traverse)] + #[derive(Debug, PyPayload)] + struct Row { + data: PyTupleRef, + description: PyTupleRef, + } + + #[pyclass( + with(Constructor, Hashable, Comparable, Iterable, AsMapping, AsSequence), + flags(BASETYPE) + )] + impl Row { + #[pymethod] + fn keys(&self, _vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + Ok(self + .description + .iter() + .map(|x| x.downcast_ref::<PyTuple>().unwrap().as_slice()[0].clone()) + .collect()) + } + + fn subscript(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + if let Some(i) = needle.downcast_ref::<PyInt>() { + let i = i.try_to_primitive::<isize>(vm)?; + self.data.getitem_by_index(vm, i) + } else if let Some(name) = needle.downcast_ref::<PyStr>() { + for (obj, i) in self.description.iter().zip(0..) { + let obj = &obj.downcast_ref::<PyTuple>().unwrap().as_slice()[0]; + let Some(obj) = obj.downcast_ref::<PyStr>() else { + break; + }; + let a_iter = name.expect_str().chars().flat_map(|x| x.to_uppercase()); + let b_iter = obj.expect_str().chars().flat_map(|x| x.to_uppercase()); + + if a_iter.eq(b_iter) { + return self.data.getitem_by_index(vm, i); + } + } + Err(vm.new_index_error("No item with that key")) + } else if let Some(slice) = needle.downcast_ref::<PySlice>() { + let list = self.data.getitem_by_slice(vm, slice.to_saturated(vm)?)?; + Ok(vm.ctx.new_tuple(list).into()) + } else { + Err(vm.new_index_error("Index must be int or string")) + } + } + } + + impl Constructor for Row { + type Args = (PyRef<Cursor>, PyTupleRef); + + fn py_new( + _cls: &Py<PyType>, + (cursor, data): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let description = cursor + .inner(vm)? + .description + .clone() + .ok_or_else(|| vm.new_value_error("no description in Cursor"))?; + + Ok(Self { data, description }) + } + } + + impl Hashable for Row { + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + Ok(zelf.description.as_object().hash(vm)? | zelf.data.as_object().hash(vm)?) + } + } + + impl Comparable for Row { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + if let Some(other) = other.downcast_ref::<Self>() { + let eq = vm + .bool_eq(zelf.description.as_object(), other.description.as_object())? + && vm.bool_eq(zelf.data.as_object(), other.data.as_object())?; + Ok(eq.into()) + } else { + Ok(PyComparisonValue::NotImplemented) + } + }) + } + } + + impl Iterable for Row { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Iterable::iter(zelf.data.clone(), vm) + } + } + + impl AsMapping for Row { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: std::sync::LazyLock<PyMappingMethods> = + std::sync::LazyLock::new(|| PyMappingMethods { + length: atomic_func!(|mapping, _vm| Ok(Row::mapping_downcast(mapping) + .data + .len())), + subscript: atomic_func!(|mapping, needle, vm| { + Row::mapping_downcast(mapping).subscript(needle, vm) + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } + } + + impl AsSequence for Row { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: std::sync::LazyLock<PySequenceMethods> = + std::sync::LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(Row::sequence_downcast(seq).data.len())), + item: atomic_func!(|seq, i, vm| Row::sequence_downcast(seq) + .data + .getitem_by_index(vm, i)), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } + } + + #[pyattr] + #[pyclass(module = "sqlite3", name = "Blob", traverse)] + #[derive(Debug, PyPayload)] + struct Blob { + connection: PyRef<Connection>, + #[pytraverse(skip)] + inner: PyMutex<Option<BlobInner>>, + } + + #[derive(Debug)] + struct BlobInner { + blob: SqliteBlob, + offset: c_int, + } + + impl Drop for BlobInner { + fn drop(&mut self) { + unsafe { sqlite3_blob_close(self.blob.blob) }; + } + } + + #[pyclass(flags(DISALLOW_INSTANTIATION), with(AsMapping, AsNumber, AsSequence))] + impl Blob { + #[pymethod] + fn close(&self) { + self.inner.lock().take(); + } + + fn ensure_connection_open(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.connection.is_closed() { + Err(new_programming_error( + vm, + "Cannot operate on a closed database".to_owned(), + )) + } else { + Ok(()) + } + } + + #[pymethod] + fn read( + &self, + length: OptionalArg<c_int>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PyBytes>> { + self.ensure_connection_open(vm)?; + + let mut length = length.unwrap_or(-1); + let mut inner = self.inner(vm)?; + let blob_len = inner.blob.bytes(); + let max_read = blob_len - inner.offset; + + if length < 0 || length > max_read { + length = max_read; + } + + if length == 0 { + Ok(vm.ctx.empty_bytes.clone()) + } else { + let mut buf = Vec::<u8>::with_capacity(length as usize); + let ret = inner + .blob + .read(buf.as_mut_ptr().cast(), length, inner.offset); + self.check(ret, vm)?; + unsafe { buf.set_len(length as usize) }; + inner.offset += length; + Ok(vm.ctx.new_bytes(buf)) + } + } + + #[pymethod] + fn write(&self, data: PyBuffer, vm: &VirtualMachine) -> PyResult<()> { + self.ensure_connection_open(vm)?; + let mut inner = self.inner(vm)?; + let blob_len = inner.blob.bytes(); + let length = Self::expect_write(blob_len, data.desc.len, inner.offset, vm)?; + + let ret = data.contiguous_or_collect(|buf| { + inner.blob.write(buf.as_ptr().cast(), length, inner.offset) + }); + + self.check(ret, vm)?; + inner.offset += length; + Ok(()) + } + + #[pymethod] + fn tell(&self, vm: &VirtualMachine) -> PyResult<c_int> { + self.ensure_connection_open(vm)?; + self.inner(vm).map(|x| x.offset) + } + + #[pymethod] + fn seek( + &self, + mut offset: c_int, + origin: OptionalArg<c_int>, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.ensure_connection_open(vm)?; + let origin = origin.unwrap_or(libc::SEEK_SET); + let mut inner = self.inner(vm)?; + let blob_len = inner.blob.bytes(); + + let overflow_err = || vm.new_overflow_error("seek offset results in overflow"); + + match origin { + libc::SEEK_SET => {} + libc::SEEK_CUR => { + offset = offset.checked_add(inner.offset).ok_or_else(overflow_err)? + } + libc::SEEK_END => offset = offset.checked_add(blob_len).ok_or_else(overflow_err)?, + _ => { + return Err(vm.new_value_error( + "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END", + )); + } + } + + if offset < 0 || offset > blob_len { + Err(vm.new_value_error("offset out of blob range")) + } else { + inner.offset = offset; + Ok(()) + } + } + + #[pymethod] + fn __enter__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + zelf.ensure_connection_open(vm)?; + let _ = zelf.inner(vm)?; + Ok(zelf) + } + + #[pymethod] + fn __exit__(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + self.ensure_connection_open(vm)?; + let _ = self.inner(vm)?; + self.close(); + Ok(()) + } + + fn inner(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<'_, BlobInner>> { + let guard = self.inner.lock(); + if guard.is_some() { + Ok(PyMutexGuard::map(guard, |x| unsafe { + x.as_mut().unwrap_unchecked() + })) + } else { + Err(new_programming_error( + vm, + "Cannot operate on a closed blob.".to_owned(), + )) + } + } + + fn wrapped_index(index: PyIntRef, length: c_int, vm: &VirtualMachine) -> PyResult<c_int> { + let mut index = index.try_to_primitive::<c_int>(vm)?; + if index < 0 { + index += length; + } + if index < 0 || index >= length { + Err(vm.new_index_error("Blob index out of range")) + } else { + Ok(index) + } + } + + fn expect_write( + blob_len: c_int, + length: usize, + offset: c_int, + vm: &VirtualMachine, + ) -> PyResult<c_int> { + let max_write = blob_len - offset; + if length <= max_write as usize { + Ok(length as c_int) + } else { + Err(vm.new_value_error("data longer than blob length")) + } + } + + fn subscript(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + self.ensure_connection_open(vm)?; + let inner = self.inner(vm)?; + if let Some(index) = needle.try_index_opt(vm) { + let blob_len = inner.blob.bytes(); + let index = Self::wrapped_index(index?, blob_len, vm)?; + let mut byte: u8 = 0; + let ret = inner.blob.read_single(&mut byte, index); + self.check(ret, vm).map(|_| vm.ctx.new_int(byte).into()) + } else if let Some(slice) = needle.downcast_ref::<PySlice>() { + let blob_len = inner.blob.bytes(); + let slice = slice.to_saturated(vm)?; + let (range, step, length) = slice.adjust_indices(blob_len as usize); + let mut buf = Vec::<u8>::with_capacity(length); + + if step == 1 { + let ret = inner.blob.read( + buf.as_mut_ptr().cast(), + length as c_int, + range.start as c_int, + ); + self.check(ret, vm)?; + unsafe { buf.set_len(length) }; + } else { + let iter = SaturatedSliceIter::from_adjust_indices(range, step, length); + let mut byte: u8 = 0; + for index in iter { + let ret = inner.blob.read_single(&mut byte, index as c_int); + self.check(ret, vm)?; + buf.push(byte); + } + } + Ok(vm.ctx.new_bytes(buf).into()) + } else { + Err(vm.new_type_error("Blob indices must be integers")) + } + } + + fn ass_subscript( + &self, + needle: &PyObject, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let Some(value) = value else { + return Err(vm.new_type_error("Blob doesn't support slice deletion")); + }; + self.ensure_connection_open(vm)?; + let inner = self.inner(vm)?; + + if let Some(index) = needle.try_index_opt(vm) { + // Handle single item assignment: blob[i] = b + let Some(value) = value.downcast_ref::<PyInt>() else { + return Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + value.class() + ))); + }; + let value = value.try_to_primitive::<u8>(vm)?; + let blob_len = inner.blob.bytes(); + let index = Self::wrapped_index(index?, blob_len, vm)?; + Self::expect_write(blob_len, 1, index, vm)?; + let ret = inner.blob.write_single(value, index); + self.check(ret, vm) + } else if let Some(slice) = needle.downcast_ref::<PySlice>() { + // Handle slice assignment: blob[a:b:c] = b"..." + let value_buf = PyBuffer::try_from_borrowed_object(vm, &value)?; + + let buf = value_buf + .as_contiguous() + .ok_or_else(|| vm.new_buffer_error("underlying buffer is not C-contiguous"))?; + + let blob_len = inner.blob.bytes(); + let slice = slice.to_saturated(vm)?; + let (range, step, slice_len) = slice.adjust_indices(blob_len as usize); + + if step == 0 { + return Err(vm.new_value_error("slice step cannot be zero")); + } + + if buf.len() != slice_len { + return Err(vm.new_index_error("Blob slice assignment is wrong size")); + } + + if slice_len == 0 { + return Ok(()); + } + + if step == 1 { + let ret = inner.blob.write( + buf.as_ptr().cast(), + buf.len() as c_int, + range.start as c_int, + ); + self.check(ret, vm) + } else { + let span_len = range.end - range.start; + let mut temp_buf = vec![0u8; span_len]; + + let ret = inner.blob.read( + temp_buf.as_mut_ptr().cast(), + span_len as c_int, + range.start as c_int, + ); + self.check(ret, vm)?; + + let mut i_in_temp: usize = 0; + for i_in_src in 0..slice_len { + temp_buf[i_in_temp] = buf[i_in_src]; + i_in_temp += step as usize; + } + + let ret = inner.blob.write( + temp_buf.as_ptr().cast(), + span_len as c_int, + range.start as c_int, + ); + self.check(ret, vm) + } + } else { + Err(vm.new_type_error("Blob indices must be integers")) + } + } + + fn check(&self, ret: c_int, vm: &VirtualMachine) -> PyResult<()> { + if ret == SQLITE_OK { + Ok(()) + } else { + Err(self.connection.db_lock(vm)?.error_extended(vm)) + } + } + } + + impl AsMapping for Blob { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: PyMappingMethods = PyMappingMethods { + length: atomic_func!(|mapping, vm| Blob::mapping_downcast(mapping) + .inner(vm) + .map(|x| x.blob.bytes() as usize)), + subscript: atomic_func!(|mapping, needle, vm| { + Blob::mapping_downcast(mapping).subscript(needle, vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + Blob::mapping_downcast(mapping).ass_subscript(needle, value, vm) + }), + }; + &AS_MAPPING + } + } + + impl AsNumber for Blob { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + add: Some(|a, b, vm| { + Err(vm.new_type_error(format!( + "unsupported operand type(s) for +: '{}' and '{}'", + a.class().name(), + b.class().name() + ))) + }), + multiply: Some(|a, b, vm| { + Err(vm.new_type_error(format!( + "unsupported operand type(s) for *: '{}' and '{}'", + a.class().name(), + b.class().name() + ))) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + + impl AsSequence for Blob { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { + length: None, + concat: None, + repeat: None, + item: None, + ass_item: None, + contains: atomic_func!(|seq, _needle, vm| { + Err(vm.new_type_error(format!( + "argument of type '{}' is not iterable", + seq.obj.class().name(), + ))) + }), + inplace_concat: None, + inplace_repeat: None, + }; + &AS_SEQUENCE + } + } + + #[pyattr] + #[pyclass(name)] + #[derive(Debug, PyPayload)] + struct PrepareProtocol {} + + #[pyclass()] + impl PrepareProtocol {} + + #[pyattr] + #[pyclass(module = "sqlite3", name = "Statement")] + #[derive(PyPayload)] + struct Statement { + st: PyMutex<SqliteStatement>, + pub is_dml: bool, + } + + impl Debug for Statement { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + write!( + f, + "{} Statement", + if self.is_dml { "DML" } else { "Non-DML" } + ) + } + } + + #[pyclass(flags(DISALLOW_INSTANTIATION))] + impl Statement { + fn new( + connection: &Connection, + sql: PyUtf8StrRef, + vm: &VirtualMachine, + ) -> PyResult<Option<Self>> { + if sql.as_str().contains('\0') { + return Err(new_programming_error( + vm, + "statement contains a null character.".to_owned(), + )); + } + let sql_cstr = sql.to_cstring(vm)?; + + let db = connection.db_lock(vm)?; + + db.sql_limit(sql.byte_len(), vm)?; + + let mut tail = null(); + let st = db.prepare(sql_cstr.as_ptr(), &mut tail, vm)?; + + let Some(st) = st else { + return Ok(None); + }; + + let tail = unsafe { CStr::from_ptr(tail) }; + let tail = tail.to_bytes(); + if lstrip_sql(tail).is_some() { + return Err(new_programming_error( + vm, + "You can only execute one statement at a time.".to_owned(), + )); + } + + let is_dml = if let Some(head) = lstrip_sql(sql_cstr.as_bytes()) { + head.len() >= 6 + && (head[..6].eq_ignore_ascii_case(b"insert") + || head[..6].eq_ignore_ascii_case(b"update") + || head[..6].eq_ignore_ascii_case(b"delete") + || (head.len() >= 7 && head[..7].eq_ignore_ascii_case(b"replace"))) + } else { + false + }; + + Ok(Some(Self { + st: PyMutex::from(st), + is_dml, + })) + } + + fn lock(&self) -> PyMutexGuard<'_, SqliteStatement> { + self.st.lock() + } + } + + struct Sqlite { + raw: SqliteRaw, + } + + impl From<SqliteRaw> for Sqlite { + fn from(raw: SqliteRaw) -> Self { + Self { raw } + } + } + + // sqlite3_close_v2 is not exported by libsqlite3-sys, so we declare it manually. + // It handles "zombie close" - if there are still unfinalized statements, + // the database will be closed when the last statement is finalized. + unsafe extern "C" { + fn sqlite3_close_v2(db: *mut sqlite3) -> c_int; + } + + impl Drop for Sqlite { + fn drop(&mut self) { + // Use sqlite3_close_v2 for safe closing even with active statements + unsafe { sqlite3_close_v2(self.raw.db) }; + } + } + + impl Deref for Sqlite { + type Target = SqliteRaw; + + fn deref(&self) -> &Self::Target { + &self.raw + } + } + + #[derive(Copy, Clone)] + struct SqliteRaw { + db: *mut sqlite3, + } + + cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for SqliteStatement {} + // unsafe impl Sync for SqliteStatement {} + unsafe impl Send for Sqlite {} + // unsafe impl Sync for Sqlite {} + unsafe impl Send for SqliteBlob {} + } + } + + impl From<SqliteStatementRaw> for SqliteRaw { + fn from(stmt: SqliteStatementRaw) -> Self { + unsafe { + Self { + db: sqlite3_db_handle(stmt.st), + } + } + } + } + + impl SqliteRaw { + fn check(self, ret: c_int, vm: &VirtualMachine) -> PyResult<()> { + if ret == SQLITE_OK { + Ok(()) + } else { + Err(self.error_extended(vm)) + } + } + + fn error_extended(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + let errcode = unsafe { sqlite3_errcode(self.db) }; + let typ = exception_type_from_errcode(errcode, vm); + let extended_errcode = unsafe { sqlite3_extended_errcode(self.db) }; + let errmsg = unsafe { sqlite3_errmsg(self.db) }; + let errmsg = unsafe { CStr::from_ptr(errmsg) }; + let errmsg = errmsg.to_str().unwrap().to_owned(); + + raise_exception(typ.to_owned(), extended_errcode, errmsg, vm) + } + + fn open(path: *const libc::c_char, uri: bool, vm: &VirtualMachine) -> PyResult<Self> { + let mut db = null_mut(); + let ret = unsafe { + sqlite3_open_v2( + path, + &raw mut db, + SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE + | if uri { SQLITE_OPEN_URI } else { 0 }, + null(), + ) + }; + let zelf = Self { db }; + zelf.check(ret, vm).map(|_| zelf) + } + + fn _exec(self, sql: &[u8], vm: &VirtualMachine) -> PyResult<()> { + let ret = + unsafe { sqlite3_exec(self.db, sql.as_ptr().cast(), None, null_mut(), null_mut()) }; + self.check(ret, vm) + } + + fn prepare( + self, + sql: *const libc::c_char, + tail: *mut *const libc::c_char, + vm: &VirtualMachine, + ) -> PyResult<Option<SqliteStatement>> { + let mut st = null_mut(); + let ret = unsafe { sqlite3_prepare_v2(self.db, sql, -1, &mut st, tail) }; + self.check(ret, vm)?; + if st.is_null() { + Ok(None) + } else { + Ok(Some(SqliteStatement::from(SqliteStatementRaw::from(st)))) + } + } + + fn limit(self, category: c_int, limit: c_int, vm: &VirtualMachine) -> PyResult<c_int> { + let old_limit = unsafe { sqlite3_limit(self.db, category, limit) }; + if old_limit >= 0 { + Ok(old_limit) + } else { + Err(new_programming_error( + vm, + "'category' is out of bounds".to_owned(), + )) + } + } + + fn sql_limit(self, len: usize, vm: &VirtualMachine) -> PyResult<()> { + if len <= unsafe { sqlite3_limit(self.db, SQLITE_LIMIT_SQL_LENGTH, -1) } as usize { + Ok(()) + } else { + Err(new_data_error(vm, "query string is too large".to_owned())) + } + } + + fn is_autocommit(self) -> bool { + unsafe { sqlite3_get_autocommit(self.db) != 0 } + } + + fn changes(self) -> c_int { + unsafe { sqlite3_changes(self.db) } + } + + fn total_changes(self) -> c_int { + unsafe { sqlite3_total_changes(self.db) } + } + + fn lastrowid(self) -> c_longlong { + unsafe { sqlite3_last_insert_rowid(self.db) } + } + + fn implicit_commit(self, vm: &VirtualMachine) -> PyResult<()> { + if self.is_autocommit() { + Ok(()) + } else { + self._exec(b"COMMIT\0", vm) + } + } + + fn begin_transaction( + self, + isolation_level: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let Some(isolation_level) = isolation_level else { + return Ok(()); + }; + let mut s = Vec::with_capacity(16); + s.extend(b"BEGIN "); + s.extend(isolation_level.expect_str().bytes()); + s.push(b'\0'); + self._exec(&s, vm) + } + + fn interrupt(self) { + unsafe { sqlite3_interrupt(self.db) } + } + + fn busy_timeout(self, timeout: i32) { + unsafe { sqlite3_busy_timeout(self.db, timeout) }; + } + + #[allow(clippy::too_many_arguments)] + fn create_function( + self, + name: *const libc::c_char, + narg: c_int, + flags: c_int, + data: *mut c_void, + func: Option< + unsafe extern "C" fn( + arg1: *mut sqlite3_context, + arg2: c_int, + arg3: *mut *mut sqlite3_value, + ), + >, + step: Option< + unsafe extern "C" fn( + arg1: *mut sqlite3_context, + arg2: c_int, + arg3: *mut *mut sqlite3_value, + ), + >, + finalize: Option<unsafe extern "C" fn(arg1: *mut sqlite3_context)>, + destroy: Option<unsafe extern "C" fn(arg1: *mut c_void)>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let ret = unsafe { + sqlite3_create_function_v2( + self.db, name, narg, flags, data, func, step, finalize, destroy, + ) + }; + self.check(ret, vm) + .map_err(|_| new_operational_error(vm, "Error creating function".to_owned())) + } + } + + struct SqliteStatement { + raw: SqliteStatementRaw, + } + + impl From<SqliteStatementRaw> for SqliteStatement { + fn from(raw: SqliteStatementRaw) -> Self { + Self { raw } + } + } + + impl Drop for SqliteStatement { + fn drop(&mut self) { + unsafe { + sqlite3_finalize(self.raw.st); + } + } + } + + impl Deref for SqliteStatement { + type Target = SqliteStatementRaw; + + fn deref(&self) -> &Self::Target { + &self.raw + } + } + + #[derive(Copy, Clone)] + struct SqliteStatementRaw { + st: *mut sqlite3_stmt, + } + + impl From<*mut sqlite3_stmt> for SqliteStatementRaw { + fn from(st: *mut sqlite3_stmt) -> Self { + SqliteStatementRaw { st } + } + } + + impl SqliteStatementRaw { + fn step(self) -> c_int { + unsafe { sqlite3_step(self.st) } + } + + fn step_row_else_done(self, vm: &VirtualMachine) -> PyResult<bool> { + let ret = self.step(); + + if let Some(exc) = unsafe { user_function_exception().swap(None) } { + Err(exc) + } else if ret == SQLITE_ROW { + Ok(true) + } else if ret == SQLITE_DONE { + Ok(false) + } else { + Err(SqliteRaw::from(self).error_extended(vm)) + } + } + + fn reset(self) { + unsafe { sqlite3_reset(self.st) }; + } + + fn data_count(self) -> c_int { + unsafe { sqlite3_data_count(self.st) } + } + + fn bind_parameter( + self, + pos: c_int, + parameter: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<()> { + let adapted; + let obj = if need_adapt(parameter, vm) { + adapted = _adapt( + parameter, + PrepareProtocol::class(&vm.ctx).to_owned(), + |x| Ok(x.to_owned()), + vm, + )?; + &adapted + } else { + parameter + }; + + let ret = if vm.is_none(obj) { + unsafe { sqlite3_bind_null(self.st, pos) } + } else if let Some(val) = obj.downcast_ref::<PyInt>() { + let val = val.try_to_primitive::<i64>(vm).map_err(|_| { + vm.new_overflow_error("Python int too large to convert to SQLite INTEGER") + })?; + unsafe { sqlite3_bind_int64(self.st, pos, val) } + } else if let Some(val) = obj.downcast_ref::<PyFloat>() { + let val = val.to_f64(); + unsafe { sqlite3_bind_double(self.st, pos, val) } + } else if let Some(val) = obj.downcast_ref::<PyStr>() { + let val = val.try_as_utf8(vm)?; + let (ptr, len) = str_to_ptr_len(val, vm)?; + unsafe { sqlite3_bind_text(self.st, pos, ptr, len, SQLITE_TRANSIENT()) } + } else if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, obj) { + let (ptr, len) = buffer_to_ptr_len(&buffer, vm)?; + unsafe { sqlite3_bind_blob(self.st, pos, ptr, len, SQLITE_TRANSIENT()) } + } else { + return Err(new_programming_error( + vm, + format!( + "Error binding parameter {}: type '{}' is not supported", + pos, + obj.class() + ), + )); + }; + + if ret == SQLITE_OK { + Ok(()) + } else { + let db = SqliteRaw::from(self); + db.check(ret, vm) + } + } + + fn bind_parameters(self, parameters: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if let Some(dict) = parameters.downcast_ref::<PyDict>() { + self.bind_parameters_name(dict, vm) + } else if let Ok(seq) = parameters.try_sequence(vm) { + self.bind_parameters_sequence(seq, vm) + } else { + Err(new_programming_error( + vm, + "parameters are of unsupported type".to_owned(), + )) + } + } + + fn bind_parameters_name(self, dict: &Py<PyDict>, vm: &VirtualMachine) -> PyResult<()> { + let num_needed = unsafe { sqlite3_bind_parameter_count(self.st) }; + + for i in 1..=num_needed { + let name = unsafe { sqlite3_bind_parameter_name(self.st, i) }; + if name.is_null() { + return Err(new_programming_error(vm, "Binding {} has no name, but you supplied a dictionary (which has only names).".to_owned())); + } + let name = unsafe { name.add(1) }; + let name = ptr_to_str(name, vm)?; + + let val = match dict.get_item_opt(name, vm)? { + Some(val) => val, + None => { + return Err(new_programming_error( + vm, + format!("You did not supply a value for binding parameter :{name}.",), + )); + } + }; + + self.bind_parameter(i, &val, vm)?; + } + Ok(()) + } + + fn bind_parameter_count(self) -> c_int { + unsafe { sqlite3_bind_parameter_count(self.st) } + } + + fn bind_parameters_sequence( + self, + seq: PySequence<'_>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let num_needed = self.bind_parameter_count(); + let num_supplied = seq.length(vm)?; + + if num_supplied != num_needed as usize { + return Err(new_programming_error( + vm, + format!( + "Incorrect number of bindings supplied. The current statement uses {}, and {} were supplied.", + num_needed, num_supplied + ), + )); + } + + for i in 1..=num_needed { + let val = seq.get_item(i as isize - 1, vm)?; + self.bind_parameter(i, &val, vm)?; + } + Ok(()) + } + + fn column_count(self) -> c_int { + unsafe { sqlite3_column_count(self.st) } + } + + fn column_type(self, pos: c_int) -> c_int { + unsafe { sqlite3_column_type(self.st, pos) } + } + + fn column_int(self, pos: c_int) -> i64 { + unsafe { sqlite3_column_int64(self.st, pos) } + } + + fn column_double(self, pos: c_int) -> f64 { + unsafe { sqlite3_column_double(self.st, pos) } + } + + fn column_blob(self, pos: c_int) -> *const c_void { + unsafe { sqlite3_column_blob(self.st, pos) } + } + + fn column_text(self, pos: c_int) -> *const u8 { + unsafe { sqlite3_column_text(self.st, pos) } + } + + fn column_decltype(self, pos: c_int) -> *const libc::c_char { + unsafe { sqlite3_column_decltype(self.st, pos) } + } + + fn column_bytes(self, pos: c_int) -> c_int { + unsafe { sqlite3_column_bytes(self.st, pos) } + } + + fn column_name(self, pos: c_int) -> *const libc::c_char { + unsafe { sqlite3_column_name(self.st, pos) } + } + + fn columns_name(self, detect_types: i32, vm: &VirtualMachine) -> PyResult<Vec<PyStrRef>> { + let count = self.column_count(); + (0..count) + .map(|i| { + let name = self.column_name(i); + let name_str = ptr_to_str(name, vm)?; + + // If PARSE_COLNAMES is enabled, strip everything after the first '[' (and preceding space) + let processed_name = if detect_types & PARSE_COLNAMES != 0 + && let Some(bracket_pos) = name_str.find('[') + { + // Check if there's a single space before '[' and remove it (CPython compatibility) + let end_pos = if bracket_pos > 0 + && name_str.chars().nth(bracket_pos - 1) == Some(' ') + { + bracket_pos - 1 + } else { + bracket_pos + }; + + &name_str[..end_pos] + } else { + name_str + }; + + Ok(vm.ctx.new_str(processed_name)) + }) + .collect() + } + + fn columns_description( + self, + detect_types: i32, + vm: &VirtualMachine, + ) -> PyResult<Option<PyTupleRef>> { + if self.column_count() == 0 { + return Ok(None); + } + let columns = self + .columns_name(detect_types, vm)? + .into_iter() + .map(|s| { + vm.ctx + .new_tuple(vec![ + s.into(), + vm.ctx.none(), + vm.ctx.none(), + vm.ctx.none(), + vm.ctx.none(), + vm.ctx.none(), + vm.ctx.none(), + ]) + .into() + }) + .collect(); + Ok(Some(vm.ctx.new_tuple(columns))) + } + + fn busy(self) -> bool { + unsafe { sqlite3_stmt_busy(self.st) != 0 } + } + + fn readonly(self) -> bool { + unsafe { sqlite3_stmt_readonly(self.st) != 0 } + } + } + + #[derive(Debug, Copy, Clone)] + struct SqliteBlob { + blob: *mut sqlite3_blob, + } + + impl SqliteBlob { + fn bytes(self) -> c_int { + unsafe { sqlite3_blob_bytes(self.blob) } + } + + fn write(self, buf: *const c_void, length: c_int, offset: c_int) -> c_int { + unsafe { sqlite3_blob_write(self.blob, buf, length, offset) } + } + + fn read(self, buf: *mut c_void, length: c_int, offset: c_int) -> c_int { + unsafe { sqlite3_blob_read(self.blob, buf, length, offset) } + } + + fn read_single(self, byte: &mut u8, offset: c_int) -> c_int { + self.read(byte as *mut u8 as *mut _, 1, offset) + } + + fn write_single(self, byte: u8, offset: c_int) -> c_int { + self.write(&byte as *const u8 as *const _, 1, offset) + } + } + + #[derive(Copy, Clone)] + struct SqliteContext { + ctx: *mut sqlite3_context, + } + + impl From<*mut sqlite3_context> for SqliteContext { + fn from(ctx: *mut sqlite3_context) -> Self { + Self { ctx } + } + } + + impl SqliteContext { + fn user_data<T>(self) -> *mut T { + unsafe { sqlite3_user_data(self.ctx).cast() } + } + + fn aggregate_context<T>(self) -> *mut T { + unsafe { + sqlite3_aggregate_context(self.ctx, core::mem::size_of::<T>() as c_int).cast() + } + } + + fn result_exception(self, vm: &VirtualMachine, exc: PyBaseExceptionRef, msg: &str) { + if exc.fast_isinstance(vm.ctx.exceptions.memory_error) { + unsafe { sqlite3_result_error_nomem(self.ctx) } + } else if exc.fast_isinstance(vm.ctx.exceptions.overflow_error) { + unsafe { sqlite3_result_error_toobig(self.ctx) } + } else { + unsafe { sqlite3_result_error(self.ctx, msg.as_ptr().cast(), -1) } + } + if enable_traceback().load(Ordering::Relaxed) { + vm.print_exception(exc); + } + } + + fn db_handle(self) -> *mut sqlite3 { + unsafe { sqlite3_context_db_handle(self.ctx) } + } + + fn result_from_object(self, val: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + unsafe { + if vm.is_none(val) { + sqlite3_result_null(self.ctx) + } else if let Some(val) = val.downcast_ref::<PyInt>() { + sqlite3_result_int64(self.ctx, val.try_to_primitive(vm)?) + } else if let Some(val) = val.downcast_ref::<PyFloat>() { + sqlite3_result_double(self.ctx, val.to_f64()) + } else if let Some(val) = val.downcast_ref::<PyStr>() { + let val = val.try_as_utf8(vm)?; + let (ptr, len) = str_to_ptr_len(val, vm)?; + sqlite3_result_text(self.ctx, ptr, len, SQLITE_TRANSIENT()) + } else if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, val) { + let (ptr, len) = buffer_to_ptr_len(&buffer, vm)?; + sqlite3_result_blob(self.ctx, ptr, len, SQLITE_TRANSIENT()) + } else { + return Err(new_programming_error( + vm, + "result type not support".to_owned(), + )); + } + } + Ok(()) + } + } + + fn value_to_object(val: *mut sqlite3_value, db: *mut sqlite3, vm: &VirtualMachine) -> PyResult { + let obj = unsafe { + match sqlite3_value_type(val) { + SQLITE_INTEGER => vm.ctx.new_int(sqlite3_value_int64(val)).into(), + SQLITE_FLOAT => vm.ctx.new_float(sqlite3_value_double(val)).into(), + SQLITE_TEXT => { + let text = + ptr_to_vec(sqlite3_value_text(val), sqlite3_value_bytes(val), db, vm)?; + let text = String::from_utf8(text) + .map_err(|_| vm.new_value_error("invalid utf-8 with SQLITE_TEXT"))?; + vm.ctx.new_str(text).into() + } + SQLITE_BLOB => { + let blob = ptr_to_vec( + sqlite3_value_blob(val).cast(), + sqlite3_value_bytes(val), + db, + vm, + )?; + vm.ctx.new_bytes(blob).into() + } + _ => vm.ctx.none(), + } + }; + Ok(obj) + } + + fn ptr_to_str<'a>(p: *const libc::c_char, vm: &VirtualMachine) -> PyResult<&'a str> { + if p.is_null() { + return Err(vm.new_memory_error("string pointer is null")); + } + unsafe { CStr::from_ptr(p).to_str() } + .map_err(|_| vm.new_value_error("Invalid UIF-8 codepoint")) + } + + fn ptr_to_string( + p: *const u8, + nbytes: c_int, + db: *mut sqlite3, + vm: &VirtualMachine, + ) -> PyResult<String> { + let s = ptr_to_vec(p, nbytes, db, vm)?; + String::from_utf8(s).map_err(|_| vm.new_value_error("invalid utf-8")) + } + + fn ptr_to_vec( + p: *const u8, + nbytes: c_int, + db: *mut sqlite3, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + if p.is_null() { + if !db.is_null() && unsafe { sqlite3_errcode(db) } == SQLITE_NOMEM { + Err(vm.new_memory_error("sqlite out of memory")) + } else { + Ok(vec![]) + } + } else if nbytes < 0 { + Err(vm.new_system_error("negative size with ptr")) + } else { + Ok(unsafe { core::slice::from_raw_parts(p.cast(), nbytes as usize) }.to_vec()) + } + } + + fn str_to_ptr_len(s: &PyUtf8Str, vm: &VirtualMachine) -> PyResult<(*const libc::c_char, i32)> { + let s_str = s.as_str(); + let len = c_int::try_from(s_str.len()) + .map_err(|_| vm.new_overflow_error("TEXT longer than INT_MAX bytes"))?; + let ptr = s_str.as_ptr().cast(); + Ok((ptr, len)) + } + + fn buffer_to_ptr_len(buffer: &PyBuffer, vm: &VirtualMachine) -> PyResult<(*const c_void, i32)> { + let bytes = buffer + .as_contiguous() + .ok_or_else(|| vm.new_buffer_error("underlying buffer is not C-contiguous"))?; + let len = c_int::try_from(bytes.len()) + .map_err(|_| vm.new_overflow_error("BLOB longer than INT_MAX bytes"))?; + let ptr = bytes.as_ptr().cast(); + Ok((ptr, len)) + } + + fn exception_type_from_errcode(errcode: c_int, vm: &VirtualMachine) -> &'static Py<PyType> { + match errcode { + SQLITE_INTERNAL | SQLITE_NOTFOUND => internal_error_type(), + SQLITE_NOMEM => vm.ctx.exceptions.memory_error, + SQLITE_ERROR | SQLITE_PERM | SQLITE_ABORT | SQLITE_BUSY | SQLITE_LOCKED + | SQLITE_READONLY | SQLITE_INTERRUPT | SQLITE_IOERR | SQLITE_FULL | SQLITE_CANTOPEN + | SQLITE_PROTOCOL | SQLITE_EMPTY | SQLITE_SCHEMA => operational_error_type(), + SQLITE_CORRUPT => database_error_type(), + SQLITE_TOOBIG => data_error_type(), + SQLITE_CONSTRAINT | SQLITE_MISMATCH => integrity_error_type(), + SQLITE_MISUSE | SQLITE_RANGE => interface_error_type(), + _ => database_error_type(), + } + } + + fn name_from_errcode(errcode: c_int) -> &'static str { + for (name, code) in ERROR_CODES { + if *code == errcode { + return name; + } + } + "unknown error code" + } + + fn raise_exception( + typ: PyTypeRef, + errcode: c_int, + msg: String, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + let dict = vm.ctx.new_dict(); + if let Err(e) = dict.set_item("sqlite_errorcode", vm.ctx.new_int(errcode).into(), vm) { + return e; + } + let errname = name_from_errcode(errcode); + if let Err(e) = dict.set_item("sqlite_errorname", vm.ctx.new_str(errname).into(), vm) { + return e; + } + + vm.new_exception_msg_dict(typ, msg.into(), dict) + } + + static BEGIN_STATEMENTS: &[&[u8]] = &[ + b"BEGIN ", + b"BEGIN DEFERRED", + b"BEGIN IMMEDIATE", + b"BEGIN EXCLUSIVE", + ]; + + fn begin_statement_ptr_from_isolation_level( + s: &PyStr, + vm: &VirtualMachine, + ) -> PyResult<*const libc::c_char> { + BEGIN_STATEMENTS + .iter() + .find(|&&x| x[6..].eq_ignore_ascii_case(s.as_bytes())) + .map(|&x| x.as_ptr().cast()) + .ok_or_else(|| { + vm.new_value_error( + "isolation_level string must be '', 'DEFERRED', 'IMMEDIATE', or 'EXCLUSIVE'", + ) + }) + } + + fn lstrip_sql(sql: &[u8]) -> Option<&[u8]> { + let mut pos = 0; + + // This loop is borrowed from the SQLite source code. + while let Some(t_char) = sql.get(pos) { + match t_char { + b' ' | b'\t' | b'\x0c' | b'\n' | b'\r' => { + // Skip whitespace. + pos += 1; + } + b'-' => { + // Skip line comments. + if sql.get(pos + 1) == Some(&b'-') { + pos += 2; + while let Some(&ch) = sql.get(pos) { + if ch == b'\n' { + break; + } + pos += 1; + } + let _ = sql.get(pos)?; + } else { + return Some(&sql[pos..]); + } + } + b'/' => { + // Skip C style comments. + if sql.get(pos + 1) == Some(&b'*') { + pos += 2; + while let Some(&ch) = sql.get(pos) { + if ch == b'*' && sql.get(pos + 1) == Some(&b'/') { + break; + } + pos += 1; + } + let _ = sql.get(pos)?; + pos += 2; + } else { + return Some(&sql[pos..]); + } + } + _ => { + return Some(&sql[pos..]); + } + } + } + + None + } +} diff --git a/crates/stdlib/src/_testconsole.rs b/crates/stdlib/src/_testconsole.rs new file mode 100644 index 00000000000..0db508e3da5 --- /dev/null +++ b/crates/stdlib/src/_testconsole.rs @@ -0,0 +1,76 @@ +pub(crate) use _testconsole::module_def; + +#[pymodule] +mod _testconsole { + use crate::vm::{ + PyObjectRef, PyResult, VirtualMachine, convert::IntoPyException, function::ArgBytesLike, + }; + use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; + + type Handle = windows_sys::Win32::Foundation::HANDLE; + + #[pyfunction] + fn write_input(file: PyObjectRef, s: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Console::{INPUT_RECORD, KEY_EVENT, WriteConsoleInputW}; + + // Get the fd from the file object via fileno() + let fd_obj = vm.call_method(&file, "fileno", ())?; + let fd: i32 = fd_obj.try_into_value(vm)?; + + let handle = unsafe { libc::get_osfhandle(fd) } as Handle; + if handle == INVALID_HANDLE_VALUE { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + + let data = s.borrow_buf(); + let data = &*data; + + // Interpret as UTF-16-LE pairs + if !data.len().is_multiple_of(2) { + return Err(vm.new_value_error("buffer must contain UTF-16-LE data (even length)")); + } + let wchars: Vec<u16> = data + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + + let size = wchars.len() as u32; + + // Create INPUT_RECORD array + let mut records: Vec<INPUT_RECORD> = Vec::with_capacity(wchars.len()); + for &wc in &wchars { + // SAFETY: zeroing and accessing the union field for KEY_EVENT + let mut rec: INPUT_RECORD = unsafe { core::mem::zeroed() }; + rec.EventType = KEY_EVENT as u16; + rec.Event.KeyEvent.bKeyDown = 1; // TRUE + rec.Event.KeyEvent.wRepeatCount = 1; + rec.Event.KeyEvent.uChar.UnicodeChar = wc; + records.push(rec); + } + + let mut total: u32 = 0; + while total < size { + let mut wrote: u32 = 0; + let res = unsafe { + WriteConsoleInputW( + handle, + records[total as usize..].as_ptr(), + size - total, + &mut wrote, + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + total += wrote; + } + + Ok(()) + } + + #[pyfunction] + fn read_output(_file: PyObjectRef) -> Option<()> { + // Stub, same as CPython + None + } +} diff --git a/crates/stdlib/src/_tokenize.rs b/crates/stdlib/src/_tokenize.rs new file mode 100644 index 00000000000..13bd1661305 --- /dev/null +++ b/crates/stdlib/src/_tokenize.rs @@ -0,0 +1,756 @@ +pub(crate) use _tokenize::module_def; + +#[pymodule] +mod _tokenize { + use crate::{ + common::lock::PyRwLock, + vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyBytes, PyStr, PyType}, + convert::ToPyObject, + function::ArgCallable, + protocol::PyIterReturn, + types::{Constructor, IterNext, Iterable, SelfIter}, + }, + }; + use core::fmt; + use ruff_python_ast::PySourceType; + use ruff_python_ast::token::{Token, TokenKind}; + use ruff_python_parser::{ + LexicalErrorType, ParseError, ParseErrorType, parse_unchecked_source, + }; + use ruff_source_file::{LineIndex, LineRanges}; + use ruff_text_size::{Ranged, TextSize}; + + const TOKEN_ENDMARKER: u8 = 0; + const TOKEN_DEDENT: u8 = 6; + const TOKEN_OP: u8 = 55; + const TOKEN_COMMENT: u8 = 65; + const TOKEN_NL: u8 = 66; + + #[pyattr] + #[pyclass(name = "TokenizerIter")] + #[derive(PyPayload)] + pub struct PyTokenizerIter { + readline: ArgCallable, + extra_tokens: bool, + encoding: Option<String>, + state: PyRwLock<TokenizerState>, + } + + impl PyTokenizerIter { + fn readline(&self, vm: &VirtualMachine) -> PyResult<String> { + let raw_line = match self.readline.invoke((), vm) { + Ok(v) => v, + Err(err) => { + if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + return Ok(String::new()); + } + return Err(err); + } + }; + Ok(match &self.encoding { + Some(encoding) => { + let bytes = raw_line + .downcast::<PyBytes>() + .map_err(|_| vm.new_type_error("readline() returned a non-bytes object"))?; + vm.state + .codec_registry + .decode_text(bytes.into(), encoding, None, vm) + .map(|s| s.to_string())? + } + None => raw_line + .downcast::<PyStr>() + .map(|s| s.to_string()) + .map_err(|_| vm.new_type_error("readline() returned a non-string object"))?, + }) + } + } + + impl fmt::Debug for PyTokenizerIter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PyTokenizerIter") + .field("extra_tokens", &self.extra_tokens) + .field("encoding", &self.encoding) + .finish() + } + } + + #[pyclass(with(Constructor, Iterable, IterNext))] + impl PyTokenizerIter {} + + impl Constructor for PyTokenizerIter { + type Args = PyTokenizerIterArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + let Self::Args { + readline, + extra_tokens, + encoding, + } = args; + + Ok(Self { + readline, + extra_tokens, + encoding: encoding.map(|s| s.to_string()), + state: PyRwLock::new(TokenizerState { + phase: TokenizerPhase::Reading { + source: String::new(), + }, + }), + }) + } + } + + impl SelfIter for PyTokenizerIter {} + + impl IterNext for PyTokenizerIter { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let mut state = zelf.state.read().clone(); + + loop { + match &mut state.phase { + TokenizerPhase::Reading { source } => { + let line = zelf.readline(vm)?; + if line.is_empty() { + let accumulated = core::mem::take(source); + let parsed = parse_unchecked_source(&accumulated, PySourceType::Python); + let tokens: Vec<Token> = parsed.tokens().iter().copied().collect(); + let errors: Vec<ParseError> = parsed.errors().to_vec(); + let line_index = LineIndex::from_source_text(&accumulated); + let implicit_nl = !accumulated.ends_with('\n'); + state.phase = TokenizerPhase::Yielding { + source: accumulated, + tokens, + errors, + index: 0, + line_index, + need_implicit_nl: implicit_nl, + pending_fstring_parts: Vec::new(), + pending_empty_fstring_middle: None, + }; + } else { + source.push_str(&line); + } + } + TokenizerPhase::Yielding { .. } => { + let result = emit_next_token(&mut state, zelf.extra_tokens, vm)?; + *zelf.state.write() = state; + return Ok(result); + } + TokenizerPhase::Done => { + return Ok(PyIterReturn::StopIteration(None)); + } + } + } + } + } + + /// Emit the next token from the Yielding phase. + fn emit_next_token( + state: &mut TokenizerState, + extra_tokens: bool, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + let TokenizerPhase::Yielding { + source, + tokens, + errors, + index, + line_index, + need_implicit_nl, + pending_fstring_parts, + pending_empty_fstring_middle, + } = &mut state.phase + else { + unreachable!() + }; + + // Emit pending empty FSTRING_MIDDLE (for format spec nesting) + if let Some((mid_type, mid_line, mid_col, mid_line_str)) = + pending_empty_fstring_middle.take() + { + return Ok(PyIterReturn::Return(make_token_tuple( + vm, + mid_type, + "", + mid_line, + mid_col as isize, + mid_line, + mid_col as isize, + &mid_line_str, + ))); + } + + // Emit any pending fstring sub-tokens first + if let Some((tok_type, tok_str, sl, sc, el, ec)) = pending_fstring_parts.pop() { + let offset: usize = source + .lines() + .take(sl.saturating_sub(1)) + .map(|l| l.len() + 1) + .sum(); + let full_line = source.full_line_str(TextSize::from(offset.min(source.len()) as u32)); + return Ok(PyIterReturn::Return(make_token_tuple( + vm, + tok_type, + &tok_str, + sl, + sc as isize, + el, + ec as isize, + full_line, + ))); + } + + let source_len = TextSize::from(source.len() as u32); + + while *index < tokens.len() { + let token = tokens[*index]; + *index += 1; + let kind = token.kind(); + let range = token.range(); + + // Check for lexical indentation errors. + // Skip when source has tabs — ruff and CPython handle tab + // indentation differently (CPython uses tabsize=8), so ruff may + // report false IndentationErrors for valid mixed-tab code. + if !source.contains('\t') { + for err in errors.iter() { + if !matches!( + err.error, + ParseErrorType::Lexical(LexicalErrorType::IndentationError) + ) { + continue; + } + if err.location.start() <= range.start() && range.start() < err.location.end() { + return Err(raise_indentation_error(vm, err, source, line_index)); + } + } + } + + if kind == TokenKind::EndOfFile { + continue; + } + + if !extra_tokens && matches!(kind, TokenKind::Comment | TokenKind::NonLogicalNewline) { + continue; + } + + let raw_type = token_kind_value(kind); + let token_type = if extra_tokens && raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP { + TOKEN_OP + } else { + raw_type + }; + + let (token_str, start_line, start_col, end_line, end_col, line_str) = + if kind == TokenKind::Dedent { + let last_line = source.lines().count(); + let default_pos = if extra_tokens { + (last_line + 1, 0) + } else { + (last_line, 0) + }; + let (pos, dedent_line) = + next_non_dedent_info(tokens, *index, source, line_index, default_pos); + ("", pos.0, pos.1, pos.0, pos.1, dedent_line) + } else { + let start_lc = line_index.line_column(range.start(), source); + let start_line = start_lc.line.get(); + let start_col = start_lc.column.to_zero_indexed(); + let implicit_newline = range.start() >= source_len; + let in_source = range.end() <= source_len; + + let (s, el, ec) = if kind == TokenKind::Newline { + if extra_tokens { + if implicit_newline { + ("", start_line, start_col + 1) + } else { + let s = if source[range].starts_with('\r') { + "\r\n" + } else { + "\n" + }; + (s, start_line, start_col + s.len()) + } + } else { + ("", start_line, start_col) + } + } else if kind == TokenKind::NonLogicalNewline { + let s = if in_source { &source[range] } else { "" }; + (s, start_line, start_col + s.len()) + } else { + let end_lc = line_index.line_column(range.end(), source); + let s = if in_source { &source[range] } else { "" }; + (s, end_lc.line.get(), end_lc.column.to_zero_indexed()) + }; + let line_str = source.full_line_str(range.start()); + (s, start_line, start_col, el, ec, line_str) + }; + + // Handle FSTRING_MIDDLE/TSTRING_MIDDLE brace unescaping + if matches!(kind, TokenKind::FStringMiddle | TokenKind::TStringMiddle) + && (token_str.contains("{{") || token_str.contains("}}")) + { + let mut parts = + split_fstring_middle(token_str, token_type, start_line, start_col).into_iter(); + let (tt, ts, sl, sc, el, ec) = parts.next().unwrap(); + let rest: Vec<_> = parts.collect(); + for p in rest.into_iter().rev() { + pending_fstring_parts.push(p); + } + return Ok(PyIterReturn::Return(make_token_tuple( + vm, + tt, + &ts, + sl, + sc as isize, + el, + ec as isize, + line_str, + ))); + } + + // After emitting a Rbrace inside an fstring, check if the + // next token is also Rbrace without an intervening FStringMiddle. + // CPython emits an empty FSTRING_MIDDLE in that position. + if kind == TokenKind::Rbrace + && tokens + .get(*index) + .is_some_and(|t| t.kind() == TokenKind::Rbrace) + { + let mid_type = find_fstring_middle_type(tokens, *index); + *pending_empty_fstring_middle = + Some((mid_type, end_line, end_col, line_str.to_string())); + } + + return Ok(PyIterReturn::Return(make_token_tuple( + vm, + token_type, + token_str, + start_line, + start_col as isize, + end_line, + end_col as isize, + line_str, + ))); + } + + // Emit implicit NL before ENDMARKER if source + // doesn't end with newline and last token is Comment + if extra_tokens && core::mem::take(need_implicit_nl) { + let last_tok = tokens + .iter() + .rev() + .find(|t| t.kind() != TokenKind::EndOfFile); + if let Some(last) = last_tok.filter(|t| t.kind() == TokenKind::Comment) { + let end_lc = line_index.line_column(last.range().end(), source); + let nl_line = end_lc.line.get(); + let nl_col = end_lc.column.to_zero_indexed(); + return Ok(PyIterReturn::Return(make_token_tuple( + vm, + TOKEN_NL, + "", + nl_line, + nl_col as isize, + nl_line, + nl_col as isize + 1, + source.full_line_str(last.range().start()), + ))); + } + } + + // Check for unclosed brackets before ENDMARKER — CPython's tokenizer + // raises SyntaxError("EOF in multi-line statement") in this case. + { + let bracket_count: i32 = tokens + .iter() + .map(|t| match t.kind() { + TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace => 1, + TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace => -1, + _ => 0, + }) + .sum(); + if bracket_count > 0 { + let last_line = source.lines().count(); + return Err(raise_syntax_error( + vm, + "EOF in multi-line statement", + last_line + 1, + 0, + )); + } + } + + // All tokens consumed — emit ENDMARKER + let last_line = source.lines().count(); + let (em_line, em_col, em_line_str): (usize, isize, &str) = if extra_tokens { + (last_line + 1, 0, "") + } else { + let last_line_text = + source.full_line_str(TextSize::from(source.len().saturating_sub(1) as u32)); + (last_line, -1, last_line_text) + }; + + let result = make_token_tuple( + vm, + TOKEN_ENDMARKER, + "", + em_line, + em_col, + em_line, + em_col, + em_line_str, + ); + state.phase = TokenizerPhase::Done; + Ok(PyIterReturn::Return(result)) + } + + /// Determine whether to emit FSTRING_MIDDLE (60) or TSTRING_MIDDLE (63) + /// by looking back for the most recent FStringStart/TStringStart. + fn find_fstring_middle_type(tokens: &[Token], index: usize) -> u8 { + let mut depth = 0i32; + for i in (0..index).rev() { + match tokens[i].kind() { + TokenKind::FStringEnd | TokenKind::TStringEnd => depth += 1, + TokenKind::FStringStart => { + if depth == 0 { + return 60; // FSTRING_MIDDLE + } + depth -= 1; + } + TokenKind::TStringStart => { + if depth == 0 { + return 63; // TSTRING_MIDDLE + } + depth -= 1; + } + _ => {} + } + } + 60 // default to FSTRING_MIDDLE + } + + /// Find the next non-DEDENT token's position and source line. + /// Returns ((line, col), line_str). + fn next_non_dedent_info<'a>( + tokens: &[Token], + index: usize, + source: &'a str, + line_index: &LineIndex, + default_pos: (usize, usize), + ) -> ((usize, usize), &'a str) { + for future in &tokens[index..] { + match future.kind() { + TokenKind::Dedent => continue, + TokenKind::EndOfFile => return (default_pos, ""), + _ => { + let flc = line_index.line_column(future.range().start(), source); + let pos = (flc.line.get(), flc.column.to_zero_indexed()); + return (pos, source.full_line_str(future.range().start())); + } + } + } + (default_pos, "") + } + + /// Raise a SyntaxError with the given message and position. + fn raise_syntax_error( + vm: &VirtualMachine, + msg: &str, + lineno: usize, + offset: usize, + ) -> rustpython_vm::builtins::PyBaseExceptionRef { + let exc = vm.new_exception_msg(vm.ctx.exceptions.syntax_error.to_owned(), msg.into()); + let obj = exc.as_object(); + let _ = obj.set_attr("msg", vm.ctx.new_str(msg), vm); + let _ = obj.set_attr("lineno", vm.ctx.new_int(lineno), vm); + let _ = obj.set_attr("offset", vm.ctx.new_int(offset), vm); + let _ = obj.set_attr("filename", vm.ctx.new_str("<string>"), vm); + let _ = obj.set_attr("text", vm.ctx.none(), vm); + exc + } + + /// Raise an IndentationError from a parse error. + fn raise_indentation_error( + vm: &VirtualMachine, + err: &ParseError, + source: &str, + line_index: &LineIndex, + ) -> rustpython_vm::builtins::PyBaseExceptionRef { + let err_lc = line_index.line_column(err.location.start(), source); + let err_line_text = source.full_line_str(err.location.start()); + let err_text = err_line_text.trim_end_matches('\n').trim_end_matches('\r'); + let msg = format!("{}", err.error); + let exc = vm.new_exception_msg( + vm.ctx.exceptions.indentation_error.to_owned(), + msg.clone().into(), + ); + let obj = exc.as_object(); + let _ = obj.set_attr("lineno", vm.ctx.new_int(err_lc.line.get()), vm); + let _ = obj.set_attr("offset", vm.ctx.new_int(err_text.len() as i64 + 1), vm); + let _ = obj.set_attr("msg", vm.ctx.new_str(msg), vm); + let _ = obj.set_attr("filename", vm.ctx.new_str("<string>"), vm); + let _ = obj.set_attr("text", vm.ctx.new_str(err_text), vm); + exc + } + + /// Split an FSTRING_MIDDLE/TSTRING_MIDDLE token containing `{{`/`}}` + /// into multiple unescaped sub-tokens. + /// Returns vec of (type, string, start_line, start_col, end_line, end_col). + fn split_fstring_middle( + raw: &str, + token_type: u8, + start_line: usize, + start_col: usize, + ) -> Vec<(u8, String, usize, usize, usize, usize)> { + let mut parts = Vec::new(); + let mut current = String::new(); + // Track source position (line, col) — these correspond to the + // original source positions (with {{ and }} still doubled) + let mut cur_line = start_line; + let mut cur_col = start_col; + // Track the start position of the current accumulating part + let mut part_start_line = cur_line; + let mut part_start_col = cur_col; + let mut chars = raw.chars().peekable(); + + // Compute end position of the current accumulated text + let end_pos = |current: &str, start_line: usize, start_col: usize| -> (usize, usize) { + let mut el = start_line; + let mut ec = start_col; + for ch in current.chars() { + if ch == '\n' { + el += 1; + ec = 0; + } else { + ec += ch.len_utf8(); + } + } + (el, ec) + }; + + while let Some(ch) = chars.next() { + if ch == '{' && chars.peek() == Some(&'{') { + chars.next(); + current.push('{'); + cur_col += 2; // skip both {{ in source + } else if ch == '}' && chars.peek() == Some(&'}') { + chars.next(); + // Flush accumulated text before }} + if !current.is_empty() { + let (el, ec) = end_pos(&current, part_start_line, part_start_col); + parts.push(( + token_type, + core::mem::take(&mut current), + part_start_line, + part_start_col, + el, + ec, + )); + } + // Emit unescaped '}' at source position of }} + parts.push(( + token_type, + "}".to_string(), + cur_line, + cur_col, + cur_line, + cur_col + 1, + )); + cur_col += 2; // skip both }} in source + part_start_line = cur_line; + part_start_col = cur_col; + } else { + if current.is_empty() { + part_start_line = cur_line; + part_start_col = cur_col; + } + current.push(ch); + if ch == '\n' { + cur_line += 1; + cur_col = 0; + } else { + cur_col += ch.len_utf8(); + } + } + } + + if !current.is_empty() { + let (el, ec) = end_pos(&current, part_start_line, part_start_col); + parts.push((token_type, current, part_start_line, part_start_col, el, ec)); + } + + parts + } + + #[allow(clippy::too_many_arguments)] + fn make_token_tuple( + vm: &VirtualMachine, + token_type: u8, + string: &str, + start_line: usize, + start_col: isize, + end_line: usize, + end_col: isize, + line: &str, + ) -> PyObjectRef { + vm.ctx + .new_tuple(vec![ + token_type.to_pyobject(vm), + vm.ctx.new_str(string).into(), + vm.ctx + .new_tuple(vec![start_line.to_pyobject(vm), start_col.to_pyobject(vm)]) + .into(), + vm.ctx + .new_tuple(vec![end_line.to_pyobject(vm), end_col.to_pyobject(vm)]) + .into(), + vm.ctx.new_str(line).into(), + ]) + .into() + } + + #[derive(FromArgs)] + pub struct PyTokenizerIterArgs { + #[pyarg(positional)] + readline: ArgCallable, + #[pyarg(named)] + extra_tokens: bool, + #[pyarg(named, optional)] + encoding: Option<rustpython_vm::PyRef<PyStr>>, + } + + #[derive(Clone, Debug)] + struct TokenizerState { + phase: TokenizerPhase, + } + + #[derive(Clone, Debug)] + enum TokenizerPhase { + Reading { + source: String, + }, + Yielding { + source: String, + tokens: Vec<Token>, + errors: Vec<ParseError>, + index: usize, + line_index: LineIndex, + need_implicit_nl: bool, + /// Pending sub-tokens from FSTRING_MIDDLE splitting + pending_fstring_parts: Vec<(u8, String, usize, usize, usize, usize)>, + /// Pending empty FSTRING_MIDDLE for format spec nesting: + /// (type, line, col, line_str) + pending_empty_fstring_middle: Option<(u8, usize, usize, String)>, + }, + Done, + } + + const fn token_kind_value(kind: TokenKind) -> u8 { + match kind { + TokenKind::EndOfFile => 0, + TokenKind::Name + | TokenKind::For + | TokenKind::In + | TokenKind::Pass + | TokenKind::Class + | TokenKind::And + | TokenKind::Is + | TokenKind::Raise + | TokenKind::True + | TokenKind::False + | TokenKind::Assert + | TokenKind::Try + | TokenKind::While + | TokenKind::Yield + | TokenKind::Lambda + | TokenKind::None + | TokenKind::Not + | TokenKind::Or + | TokenKind::Break + | TokenKind::Continue + | TokenKind::Global + | TokenKind::Nonlocal + | TokenKind::Return + | TokenKind::Except + | TokenKind::Import + | TokenKind::Case + | TokenKind::Match + | TokenKind::Type + | TokenKind::Await + | TokenKind::With + | TokenKind::Del + | TokenKind::Finally + | TokenKind::From + | TokenKind::Def + | TokenKind::If + | TokenKind::Else + | TokenKind::Elif + | TokenKind::As + | TokenKind::Async => 1, + TokenKind::Int | TokenKind::Complex | TokenKind::Float => 2, + TokenKind::String => 3, + TokenKind::Newline => 4, + TokenKind::NonLogicalNewline => TOKEN_NL, + TokenKind::Indent => 5, + TokenKind::Dedent => 6, + TokenKind::Lpar => 7, + TokenKind::Rpar => 8, + TokenKind::Lsqb => 9, + TokenKind::Rsqb => 10, + TokenKind::Colon => 11, + TokenKind::Comma => 12, + TokenKind::Semi => 13, + TokenKind::Plus => 14, + TokenKind::Minus => 15, + TokenKind::Star => 16, + TokenKind::Slash => 17, + TokenKind::Vbar => 18, + TokenKind::Amper => 19, + TokenKind::Less => 20, + TokenKind::Greater => 21, + TokenKind::Equal => 22, + TokenKind::Dot => 23, + TokenKind::Percent => 24, + TokenKind::Lbrace => 25, + TokenKind::Rbrace => 26, + TokenKind::EqEqual => 27, + TokenKind::NotEqual => 28, + TokenKind::LessEqual => 29, + TokenKind::GreaterEqual => 30, + TokenKind::Tilde => 31, + TokenKind::CircumFlex => 32, + TokenKind::LeftShift => 33, + TokenKind::RightShift => 34, + TokenKind::DoubleStar => 35, + TokenKind::PlusEqual => 36, + TokenKind::MinusEqual => 37, + TokenKind::StarEqual => 38, + TokenKind::SlashEqual => 39, + TokenKind::PercentEqual => 40, + TokenKind::AmperEqual => 41, + TokenKind::VbarEqual => 42, + TokenKind::CircumflexEqual => 43, + TokenKind::LeftShiftEqual => 44, + TokenKind::RightShiftEqual => 45, + TokenKind::DoubleStarEqual => 46, + TokenKind::DoubleSlash => 47, + TokenKind::DoubleSlashEqual => 48, + TokenKind::At => 49, + TokenKind::AtEqual => 50, + TokenKind::Rarrow => 51, + TokenKind::Ellipsis => 52, + TokenKind::ColonEqual => 53, + TokenKind::Exclamation => 54, + TokenKind::FStringStart => 59, + TokenKind::FStringMiddle => 60, + TokenKind::FStringEnd => 61, + TokenKind::Comment => TOKEN_COMMENT, + TokenKind::TStringStart => 62, + TokenKind::TStringMiddle => 63, + TokenKind::TStringEnd => 64, + TokenKind::IpyEscapeCommand | TokenKind::Question | TokenKind::Unknown => 67, // ERRORTOKEN + TokenKind::Lazy => u8::MAX, // Placeholder: RustPython Doesn't support `lazy imports` yet + } + } +} diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs new file mode 100644 index 00000000000..15a64c6d99c --- /dev/null +++ b/crates/stdlib/src/array.rs @@ -0,0 +1,1681 @@ +// spell-checker:ignore typecode tofile tolist fromfile + +pub(crate) use array::module_def; + +#[pymodule(name = "array")] +mod array { + use crate::{ + common::{ + atomic::{self, AtomicUsize}, + lock::{ + PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutex, PyRwLock, + PyRwLockReadGuard, PyRwLockWriteGuard, + }, + str::wchar_t, + }, + vm::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, + builtins::{ + PositionIterInternal, PyByteArray, PyBytes, PyBytesRef, PyDictRef, PyFloat, + PyGenericAlias, PyInt, PyList, PyListRef, PyStr, PyStrRef, PyTupleRef, PyType, + PyTypeRef, PyUtf8StrRef, builtins_iter, + }, + class_or_notimplemented, + convert::{ToPyObject, ToPyResult, TryFromBorrowedObject, TryFromObject}, + function::{ + ArgBytesLike, ArgIntoFloat, ArgIterable, KwArgs, OptionalArg, PyComparisonValue, + }, + protocol::{ + BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, + PyMappingMethods, PySequenceMethods, + }, + sequence::{OptionalRangeArgs, SequenceExt, SequenceMutExt}, + sliceable::{ + SaturatedSlice, SequenceIndex, SequenceIndexOp, SliceableSequenceMutOp, + SliceableSequenceOp, + }, + stdlib::_warnings, + types::{ + AsBuffer, AsMapping, AsSequence, Comparable, Constructor, IterNext, Iterable, + PyComparisonOp, Representable, SelfIter, + }, + }, + }; + use alloc::fmt; + use core::cmp::Ordering; + use itertools::Itertools; + use num_traits::ToPrimitive; + use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + use std::os::raw; + macro_rules! def_array_enum { + ($(($n:ident, $t:ty, $c:literal, $scode:literal)),*$(,)?) => { + #[derive(Debug, Clone)] + pub enum ArrayContentType { + $($n(Vec<$t>),)* + } + + impl ArrayContentType { + fn from_char(c: char) -> Result<Self, String> { + match c { + $($c => Ok(ArrayContentType::$n(Vec::new())),)* + _ => Err( + "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d)".into() + ), + } + } + + const fn typecode(&self) -> char { + match self { + $(ArrayContentType::$n(_) => $c,)* + } + } + + const fn typecode_str(&self) -> &'static str { + match self { + $(ArrayContentType::$n(_) => $scode,)* + } + } + + const fn itemsize_of_typecode(c: char) -> Option<usize> { + match c { + $($c => Some(core::mem::size_of::<$t>()),)* + _ => None, + } + } + + const fn itemsize(&self) -> usize { + match self { + $(ArrayContentType::$n(_) => core::mem::size_of::<$t>(),)* + } + } + + fn addr(&self) -> usize { + match self { + $(ArrayContentType::$n(v) => v.as_ptr() as usize,)* + } + } + + fn len(&self) -> usize { + match self { + $(ArrayContentType::$n(v) => v.len(),)* + } + } + + fn reserve(&mut self, len: usize) { + match self { + $(ArrayContentType::$n(v) => v.reserve(len),)* + } + } + + fn push(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => { + let val = <$t>::try_into_from_object(vm, obj)?; + v.push(val); + })* + } + Ok(()) + } + + fn pop(&mut self, i: isize, vm: &VirtualMachine) -> PyResult { + match self { + $(ArrayContentType::$n(v) => { + let i = v.wrap_index(i).ok_or_else(|| { + vm.new_index_error("pop index out of range".to_owned()) + })?; + v.remove(i).to_pyresult(vm) + })* + } + } + + fn insert( + &mut self, + i: isize, + obj: PyObjectRef, + vm: &VirtualMachine + ) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => { + let val = <$t>::try_into_from_object(vm, obj)?; + v.insert(i.saturated_at(v.len()), val); + })* + } + Ok(()) + } + + fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> usize { + match self { + $(ArrayContentType::$n(v) => { + if let Ok(val) = <$t>::try_into_from_object(vm, obj) { + v.iter().filter(|&&a| a == val).count() + } else { + 0 + } + })* + } + } + + fn clear(&mut self) -> PyResult<()>{ + match self { + $(ArrayContentType::$n(v) => v.clear(),)* + }; + Ok(()) + } + + fn remove(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()>{ + match self { + $(ArrayContentType::$n(v) => { + if let Ok(val) = <$t>::try_into_from_object(vm, obj) { + if let Some(pos) = v.iter().position(|&a| a == val) { + v.remove(pos); + return Ok(()); + } + } + Err(vm.new_value_error("array.remove(x): x not in array".to_owned())) + })* + } + } + + fn frombytes_move(&mut self, b: Vec<u8>) { + match self { + $(ArrayContentType::$n(v) => { + if v.is_empty() { + // safe because every configuration of bytes for the types we + // support are valid + let b = core::mem::ManuallyDrop::new(b); + let ptr = b.as_ptr() as *mut $t; + let len = b.len() / core::mem::size_of::<$t>(); + let capacity = b.capacity() / core::mem::size_of::<$t>(); + *v = unsafe { Vec::from_raw_parts(ptr, len, capacity) }; + } else { + self.frombytes(&b); + } + })* + } + } + + fn frombytes(&mut self, b: &[u8]) { + match self { + $(ArrayContentType::$n(v) => { + // safe because every configuration of bytes for the types we + // support are valid + if b.len() > 0 { + let ptr = b.as_ptr() as *const $t; + let ptr_len = b.len() / core::mem::size_of::<$t>(); + let slice = unsafe { core::slice::from_raw_parts(ptr, ptr_len) }; + v.extend_from_slice(slice); + } + })* + } + } + + fn fromlist(&mut self, list: &PyList, vm: &VirtualMachine) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => { + // convert list before modify self + let mut list: Vec<$t> = list + .borrow_vec() + .iter() + .cloned() + .map(|value| <$t>::try_into_from_object(vm, value)) + .try_collect()?; + v.append(&mut list); + Ok(()) + })* + } + } + + fn get_bytes(&self) -> &[u8] { + match self { + $(ArrayContentType::$n(v) => { + // safe because we're just reading memory as bytes + let ptr = v.as_ptr() as *const u8; + let ptr_len = v.len() * core::mem::size_of::<$t>(); + unsafe { core::slice::from_raw_parts(ptr, ptr_len) } + })* + } + } + + fn get_bytes_mut(&mut self) -> &mut [u8] { + match self { + $(ArrayContentType::$n(v) => { + // safe because we're just reading memory as bytes + let ptr = v.as_ptr() as *mut u8; + let ptr_len = v.len() * core::mem::size_of::<$t>(); + unsafe { core::slice::from_raw_parts_mut(ptr, ptr_len) } + })* + } + } + + fn index( + &self, + obj: PyObjectRef, + start: usize, + stop: usize, + vm: &VirtualMachine + ) -> PyResult<usize> { + match self { + $(ArrayContentType::$n(v) => { + if let Ok(val) = <$t>::try_into_from_object(vm, obj) { + if let Some(pos) = v.iter().take(stop as _).skip(start as _).position(|&elem| elem == val) { + return Ok(pos + start); + } + } + Err(vm.new_value_error("array.index(x): x not in array".to_owned())) + })* + } + } + + fn reverse(&mut self) { + match self { + $(ArrayContentType::$n(v) => v.reverse(),)* + } + } + + fn get( + &self, + i: usize, + vm: &VirtualMachine + ) -> Option<PyResult> { + match self { + $(ArrayContentType::$n(v) => { + v.get(i).map(|x| x.to_pyresult(vm)) + })* + } + } + + fn getitem_by_index(&self, i: isize, vm: &VirtualMachine) -> PyResult { + match self { + $(ArrayContentType::$n(v) => { + v.getitem_by_index(vm, i).map(|x| x.to_pyresult(vm))? + })* + } + } + + fn getitem_by_slice(&self, slice: SaturatedSlice, vm: &VirtualMachine) -> PyResult { + match self { + $(ArrayContentType::$n(v) => { + let r = v.getitem_by_slice(vm, slice)?; + let array = PyArray::from(ArrayContentType::$n(r)); + array.to_pyresult(vm) + })* + } + } + + fn setitem_by_index( + &mut self, + i: isize, + value: PyObjectRef, + vm: &VirtualMachine + ) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => { + let value = <$t>::try_into_from_object(vm, value)?; + v.setitem_by_index(vm, i, value) + })* + } + } + + fn setitem_by_slice( + &mut self, + slice: SaturatedSlice, + items: &ArrayContentType, + vm: &VirtualMachine + ) -> PyResult<()> { + match self { + $(Self::$n(elements) => if let ArrayContentType::$n(items) = items { + elements.setitem_by_slice(vm, slice, items) + } else { + Err(vm.new_type_error( + "bad argument type for built-in operation".to_owned() + )) + },)* + } + } + + fn setitem_by_slice_no_resize( + &mut self, + slice: SaturatedSlice, + items: &ArrayContentType, + vm: &VirtualMachine + ) -> PyResult<()> { + match self { + $(Self::$n(elements) => if let ArrayContentType::$n(items) = items { + elements.setitem_by_slice_no_resize(vm, slice, items) + } else { + Err(vm.new_type_error( + "bad argument type for built-in operation".to_owned() + )) + },)* + } + } + + fn delitem_by_index(&mut self, i: isize, vm: &VirtualMachine) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => { + v.delitem_by_index(vm, i) + })* + } + } + + fn delitem_by_slice(&mut self, slice: SaturatedSlice, vm: &VirtualMachine) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => { + v.delitem_by_slice(vm, slice) + })* + } + } + + fn add(&self, other: &ArrayContentType, vm: &VirtualMachine) -> PyResult<Self> { + match self { + $(ArrayContentType::$n(v) => if let ArrayContentType::$n(other) = other { + let elements = v.iter().chain(other.iter()).cloned().collect(); + Ok(ArrayContentType::$n(elements)) + } else { + Err(vm.new_type_error( + "bad argument type for built-in operation".to_owned() + )) + },)* + } + } + + fn iadd(&mut self, other: &ArrayContentType, vm: &VirtualMachine) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => if let ArrayContentType::$n(other) = other { + v.extend(other); + Ok(()) + } else { + Err(vm.new_type_error( + "can only extend with array of same kind".to_owned() + )) + },)* + } + } + + fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult<Self> { + match self { + $(ArrayContentType::$n(v) => { + // MemoryError instead Overflow Error, hard to says it is right + // but it is how cpython doing right now + let elements = v.mul(vm, value).map_err(|_| vm.new_memory_error("".to_owned()))?; + Ok(ArrayContentType::$n(elements)) + })* + } + } + + fn imul(&mut self, value: isize, vm: &VirtualMachine) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => { + // MemoryError instead Overflow Error, hard to says it is right + // but it is how cpython doing right now + v.imul(vm, value).map_err(|_| vm.new_memory_error("".to_owned())) + })* + } + } + + fn byteswap(&mut self) { + match self { + $(ArrayContentType::$n(v) => { + for element in v.iter_mut() { + let x = element.byteswap(); + *element = x; + } + })* + } + } + + fn repr(&self, class_name: &str, _vm: &VirtualMachine) -> PyResult<String> { + // we don't need ReprGuard here + let s = match self { + $(ArrayContentType::$n(v) => { + if v.is_empty() { + format!("{}('{}')", class_name, $c) + } else { + format!("{}('{}', [{}])", class_name, $c, v.iter().format(", ")) + } + })* + }; + Ok(s) + } + + fn iter<'a, 'vm: 'a>( + &'a self, + vm: &'vm VirtualMachine + ) -> impl Iterator<Item = PyResult> + 'a { + (0..self.len()).map(move |i| self.get(i, vm).unwrap()) + } + + fn cmp(&self, other: &ArrayContentType) -> Result<Option<Ordering>, ()> { + match self { + $(ArrayContentType::$n(v) => { + if let ArrayContentType::$n(other) = other { + Ok(PartialOrd::partial_cmp(v, other)) + } else { + Err(()) + } + })* + } + } + + fn get_objects(&self, vm: &VirtualMachine) -> Vec<PyObjectRef> { + match self { + $(ArrayContentType::$n(v) => { + v.iter().map(|&x| x.to_object(vm)).collect() + })* + } + } + } + }; + } + + def_array_enum!( + (SignedByte, i8, 'b', "b"), + (UnsignedByte, u8, 'B', "B"), + (PyUnicode, WideChar, 'u', "u"), + (SignedShort, raw::c_short, 'h', "h"), + (UnsignedShort, raw::c_ushort, 'H', "H"), + (SignedInt, raw::c_int, 'i', "i"), + (UnsignedInt, raw::c_uint, 'I', "I"), + (SignedLong, raw::c_long, 'l', "l"), + (UnsignedLong, raw::c_ulong, 'L', "L"), + (SignedLongLong, raw::c_longlong, 'q', "q"), + (UnsignedLongLong, raw::c_ulonglong, 'Q', "Q"), + (Float, f32, 'f', "f"), + (Double, f64, 'd', "d"), + ); + + #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] + pub struct WideChar(wchar_t); + + trait ArrayElement: Sized { + fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self>; + fn byteswap(self) -> Self; + fn to_object(self, vm: &VirtualMachine) -> PyObjectRef; + } + + macro_rules! impl_int_element { + ($($t:ty,)*) => {$( + impl ArrayElement for $t { + fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + obj.try_index(vm)?.try_to_primitive(vm) + } + fn byteswap(self) -> Self { + <$t>::swap_bytes(self) + } + fn to_object(self, vm: &VirtualMachine) -> PyObjectRef { + self.to_pyobject(vm) + } + } + )*}; + } + + macro_rules! impl_float_element { + ($(($t:ty, $f_from:path, $f_swap:path, $f_to:path),)*) => {$( + impl ArrayElement for $t { + fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + $f_from(vm, obj) + } + fn byteswap(self) -> Self { + $f_swap(self) + } + fn to_object(self, vm: &VirtualMachine) -> PyObjectRef { + $f_to(self).into_pyobject(vm) + } + } + )*}; + } + + impl_int_element!(i8, u8, i16, u16, i32, u32, i64, u64,); + impl_float_element!( + ( + f32, + f32_try_into_from_object, + f32_swap_bytes, + pyfloat_from_f32 + ), + (f64, f64_try_into_from_object, f64_swap_bytes, PyFloat::from), + ); + + const fn f32_swap_bytes(x: f32) -> f32 { + f32::from_bits(x.to_bits().swap_bytes()) + } + + const fn f64_swap_bytes(x: f64) -> f64 { + f64::from_bits(x.to_bits().swap_bytes()) + } + + fn f32_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<f32> { + ArgIntoFloat::try_from_object(vm, obj).map(|x| x.into_float() as f32) + } + + fn f64_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<f64> { + ArgIntoFloat::try_from_object(vm, obj).map(|x| x.into_float()) + } + + fn pyfloat_from_f32(value: f32) -> PyFloat { + PyFloat::from(value as f64) + } + + impl ArrayElement for WideChar { + fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + PyUtf8StrRef::try_from_object(vm, obj)? + .as_str() + .chars() + .exactly_one() + .map(|ch| Self(ch as _)) + .map_err(|_| vm.new_type_error("array item must be unicode character")) + } + fn byteswap(self) -> Self { + Self(self.0.swap_bytes()) + } + fn to_object(self, _vm: &VirtualMachine) -> PyObjectRef { + unreachable!() + } + } + + fn u32_to_char(ch: u32) -> Result<CodePoint, String> { + CodePoint::from_u32(ch) + .ok_or_else(|| format!("character U+{ch:4x} is not in range [U+0000; U+10ffff]")) + } + + impl TryFrom<WideChar> for CodePoint { + type Error = String; + + fn try_from(ch: WideChar) -> Result<Self, Self::Error> { + // safe because every configuration of bytes for the types we support are valid + u32_to_char(ch.0 as _) + } + } + + impl ToPyResult for WideChar { + fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { + Ok(CodePoint::try_from(self) + .map_err(|e| vm.new_unicode_encode_error(e))? + .to_pyobject(vm)) + } + } + + impl fmt::Display for WideChar { + fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { + unreachable!("`repr(array('u'))` calls `PyStr::repr`") + } + } + + #[pyattr] + #[pyattr(name = "ArrayType")] + #[pyclass(name = "array")] + #[derive(Debug, PyPayload)] + pub struct PyArray { + array: PyRwLock<ArrayContentType>, + exports: AtomicUsize, + } + + pub type PyArrayRef = PyRef<PyArray>; + + impl From<ArrayContentType> for PyArray { + fn from(array: ArrayContentType) -> Self { + Self { + array: PyRwLock::new(array), + exports: AtomicUsize::new(0), + } + } + } + + #[derive(FromArgs)] + pub struct ArrayNewArgs { + #[pyarg(positional)] + spec: PyUtf8StrRef, + #[pyarg(positional, optional)] + init: OptionalArg<PyObjectRef>, + } + + impl Constructor for PyArray { + type Args = (ArrayNewArgs, KwArgs); + + fn py_new( + cls: &Py<PyType>, + (ArrayNewArgs { spec, init }, kwargs): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let spec = spec.as_str().chars().exactly_one().map_err(|_| { + vm.new_type_error("array() argument 1 must be a unicode character, not str") + })?; + + if cls.is(Self::class(&vm.ctx)) && !kwargs.is_empty() { + return Err(vm.new_type_error("array.array() takes no keyword arguments")); + } + + if spec == 'u' { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + "The 'u' type code is deprecated and will be removed in Python 3.16".to_owned(), + 1, + vm, + )?; + } + + let mut array = + ArrayContentType::from_char(spec).map_err(|err| vm.new_value_error(err))?; + + if let OptionalArg::Present(init) = init { + if let Some(init) = init.downcast_ref::<Self>() { + match (spec, init.read().typecode()) { + (spec, ch) if spec == ch => array.frombytes(&init.get_bytes()), + (spec, 'u') => { + return Err(vm.new_type_error(format!( + "cannot use a unicode array to initialize an array with typecode '{spec}'" + ))) + } + _ => { + for obj in init.read().iter(vm) { + array.push(obj?, vm)?; + } + } + } + } else if let Some(wtf8) = init.downcast_ref::<PyStr>() { + if spec == 'u' { + let bytes = Self::_unicode_to_wchar_bytes(wtf8.as_wtf8(), array.itemsize()); + array.frombytes_move(bytes); + } else { + return Err(vm.new_type_error(format!( + "cannot use a str to initialize an array with typecode '{spec}'" + ))); + } + } else if init.downcastable::<PyBytes>() || init.downcastable::<PyByteArray>() { + init.try_bytes_like(vm, |x| array.frombytes(x))?; + } else if let Ok(iter) = ArgIterable::try_from_object(vm, init.clone()) { + for obj in iter.iter(vm)? { + array.push(obj?, vm)?; + } + } else { + init.try_bytes_like(vm, |x| array.frombytes(x))?; + } + } + + Ok(Self::from(array)) + } + } + + #[pyclass( + flags(BASETYPE, HAS_WEAKREF), + with( + Comparable, + AsBuffer, + AsMapping, + AsSequence, + Iterable, + Constructor, + Representable + ) + )] + impl PyArray { + fn read(&self) -> PyRwLockReadGuard<'_, ArrayContentType> { + self.array.read() + } + + fn write(&self) -> PyRwLockWriteGuard<'_, ArrayContentType> { + self.array.write() + } + + #[pygetset] + fn typecode(&self, vm: &VirtualMachine) -> PyStrRef { + vm.ctx + .intern_str(self.read().typecode().to_string()) + .to_owned() + } + + #[pygetset] + fn itemsize(&self) -> usize { + self.read().itemsize() + } + + #[pymethod] + fn append(zelf: &Py<Self>, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?.push(x, vm) + } + + #[pymethod] + fn clear(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?.clear() + } + + #[pymethod] + fn buffer_info(&self) -> (usize, usize) { + let array = self.read(); + (array.addr(), array.len()) + } + + #[pymethod] + fn count(&self, x: PyObjectRef, vm: &VirtualMachine) -> usize { + self.read().count(x, vm) + } + + #[pymethod] + fn remove(zelf: &Py<Self>, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?.remove(x, vm) + } + + #[pymethod] + fn extend(zelf: &Py<Self>, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut w = zelf.try_resizable(vm)?; + if zelf.is(&obj) { + w.imul(2, vm) + } else if let Some(array) = obj.downcast_ref::<Self>() { + w.iadd(&array.read(), vm) + } else { + let iter = ArgIterable::try_from_object(vm, obj)?; + // zelf.extend_from_iterable(iter, vm) + for obj in iter.iter(vm)? { + w.push(obj?, vm)?; + } + Ok(()) + } + } + + fn _wchar_bytes_to_string( + bytes: &[u8], + item_size: usize, + vm: &VirtualMachine, + ) -> PyResult<Wtf8Buf> { + if item_size == 2 { + // safe because every configuration of bytes for the types we support are valid + let utf16 = unsafe { + core::slice::from_raw_parts( + bytes.as_ptr() as *const u16, + bytes.len() / core::mem::size_of::<u16>(), + ) + }; + Ok(Wtf8Buf::from_wide(utf16)) + } else { + // safe because every configuration of bytes for the types we support are valid + let chars = unsafe { + core::slice::from_raw_parts( + bytes.as_ptr() as *const u32, + bytes.len() / core::mem::size_of::<u32>(), + ) + }; + chars + .iter() + .map(|&ch| { + // cpython issue 17223 + u32_to_char(ch).map_err(|msg| vm.new_value_error(msg)) + }) + .try_collect() + } + } + + fn _unicode_to_wchar_bytes(wtf8: &Wtf8, item_size: usize) -> Vec<u8> { + if item_size == 2 { + wtf8.encode_wide().flat_map(|ch| ch.to_ne_bytes()).collect() + } else { + wtf8.code_points() + .flat_map(|ch| ch.to_u32().to_ne_bytes()) + .collect() + } + } + + #[pymethod] + fn fromunicode(zelf: &Py<Self>, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let wtf8: &Wtf8 = obj.try_to_value(vm).map_err(|_| { + vm.new_type_error(format!( + "fromunicode() argument must be str, not {}", + obj.class().name() + )) + })?; + if zelf.read().typecode() != 'u' { + return Err( + vm.new_value_error("fromunicode() may only be called on unicode type arrays") + ); + } + let mut w = zelf.try_resizable(vm)?; + let bytes = Self::_unicode_to_wchar_bytes(wtf8, w.itemsize()); + w.frombytes_move(bytes); + Ok(()) + } + + #[pymethod] + fn tounicode(&self, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let array = self.array.read(); + if array.typecode() != 'u' { + return Err( + vm.new_value_error("tounicode() may only be called on unicode type arrays") + ); + } + let bytes = array.get_bytes(); + Self::_wchar_bytes_to_string(bytes, self.itemsize(), vm) + } + + fn _from_bytes(&self, b: &[u8], itemsize: usize, vm: &VirtualMachine) -> PyResult<()> { + if !b.len().is_multiple_of(itemsize) { + return Err(vm.new_value_error("bytes length not a multiple of item size")); + } + if b.len() / itemsize > 0 { + self.try_resizable(vm)?.frombytes(b); + } + Ok(()) + } + + #[pymethod] + fn frombytes(&self, b: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { + let b = b.borrow_buf(); + let itemsize = self.read().itemsize(); + self._from_bytes(&b, itemsize, vm) + } + + #[pymethod] + fn fromfile(&self, f: PyObjectRef, n: isize, vm: &VirtualMachine) -> PyResult<()> { + let itemsize = self.itemsize(); + if n < 0 { + return Err(vm.new_value_error("negative count")); + } + let n = vm.check_repeat_or_overflow_error(itemsize, n)?; + let n_bytes = n * itemsize; + + let b = vm.call_method(&f, "read", (n_bytes,))?; + let b = b + .downcast::<PyBytes>() + .map_err(|_| vm.new_type_error("read() didn't return bytes"))?; + + let not_enough_bytes = b.len() != n_bytes; + + self._from_bytes(b.as_bytes(), itemsize, vm)?; + + if not_enough_bytes { + Err(vm.new_exception_msg( + vm.ctx.exceptions.eof_error.to_owned(), + "read() didn't return enough bytes".into(), + )) + } else { + Ok(()) + } + } + + #[pymethod] + fn byteswap(&self) { + self.write().byteswap(); + } + + #[pymethod] + fn index( + &self, + x: PyObjectRef, + range: OptionalRangeArgs, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let (start, stop) = range.saturate(self.__len__(), vm)?; + self.read().index(x, start, stop, vm) + } + + #[pymethod] + fn insert(zelf: &Py<Self>, i: isize, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut w = zelf.try_resizable(vm)?; + w.insert(i, x, vm) + } + + #[pymethod] + fn pop(zelf: &Py<Self>, i: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult { + let mut w = zelf.try_resizable(vm)?; + if w.len() == 0 { + Err(vm.new_index_error("pop from empty array")) + } else { + w.pop(i.unwrap_or(-1), vm) + } + } + + #[pymethod] + pub(crate) fn tobytes(&self) -> Vec<u8> { + self.read().get_bytes().to_vec() + } + + #[pymethod] + fn tofile(&self, f: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + /* Write 64K blocks at a time */ + /* XXX Make the block size settable */ + const BLOCKSIZE: usize = 64 * 1024; + + let bytes = { + let bytes = self.read(); + bytes.get_bytes().to_vec() + }; + + for b in bytes.chunks(BLOCKSIZE) { + let b = PyBytes::from(b.to_vec()).into_ref(&vm.ctx); + vm.call_method(&f, "write", (b,))?; + } + Ok(()) + } + + pub(crate) fn get_bytes(&self) -> PyMappedRwLockReadGuard<'_, [u8]> { + PyRwLockReadGuard::map(self.read(), |a| a.get_bytes()) + } + + pub(crate) fn get_bytes_mut(&self) -> PyMappedRwLockWriteGuard<'_, [u8]> { + PyRwLockWriteGuard::map(self.write(), |a| a.get_bytes_mut()) + } + + #[pymethod] + fn tolist(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let array = self.read(); + let mut v = Vec::with_capacity(array.len()); + for obj in array.iter(vm) { + v.push(obj?); + } + Ok(v) + } + + #[pymethod] + fn fromlist(zelf: &Py<Self>, list: PyListRef, vm: &VirtualMachine) -> PyResult<()> { + zelf.try_resizable(vm)?.fromlist(&list, vm) + } + + #[pymethod] + fn reverse(&self) { + self.write().reverse() + } + + #[pymethod] + fn __copy__(&self) -> Self { + self.array.read().clone().into() + } + + #[pymethod] + fn __deepcopy__(&self, _memo: PyObjectRef) -> Self { + self.__copy__() + } + + fn getitem_inner(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { + SequenceIndex::Int(i) => self.read().getitem_by_index(i, vm), + SequenceIndex::Slice(slice) => self.read().getitem_by_slice(slice, vm), + } + } + + fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.getitem_inner(&needle, vm) + } + + fn setitem_inner( + zelf: &Py<Self>, + needle: &PyObject, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { + SequenceIndex::Int(i) => zelf.write().setitem_by_index(i, value, vm), + SequenceIndex::Slice(slice) => { + let cloned; + let guard; + let items = if zelf.is(&value) { + cloned = zelf.read().clone(); + &cloned + } else { + match value.downcast_ref::<Self>() { + Some(array) => { + guard = array.read(); + &*guard + } + None => { + return Err(vm.new_type_error(format!( + "can only assign array (not \"{}\") to array slice", + value.class() + ))); + } + } + }; + if let Ok(mut w) = zelf.try_resizable(vm) { + w.setitem_by_slice(slice, items, vm) + } else { + zelf.write().setitem_by_slice_no_resize(slice, items, vm) + } + } + } + } + + fn __setitem__( + zelf: &Py<Self>, + needle: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + Self::setitem_inner(zelf, &needle, value, vm) + } + + fn delitem_inner(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { + SequenceIndex::Int(i) => self.try_resizable(vm)?.delitem_by_index(i, vm), + SequenceIndex::Slice(slice) => self.try_resizable(vm)?.delitem_by_slice(slice, vm), + } + } + + fn __delitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.delitem_inner(&needle, vm) + } + + fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + if let Some(other) = other.downcast_ref::<Self>() { + self.read() + .add(&other.read(), vm) + .map(|array| Self::from(array).into_ref(&vm.ctx)) + } else { + Err(vm.new_type_error(format!( + "can only append array (not \"{}\") to array", + other.class().name() + ))) + } + } + + fn __iadd__( + zelf: PyRef<Self>, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + if zelf.is(&other) { + zelf.try_resizable(vm)?.imul(2, vm)?; + } else if let Some(other) = other.downcast_ref::<Self>() { + zelf.try_resizable(vm)?.iadd(&other.read(), vm)?; + } else { + return Err(vm.new_type_error(format!( + "can only extend array with array (not \"{}\")", + other.class().name() + ))); + } + Ok(zelf) + } + + fn __mul__(&self, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + self.read() + .mul(value, vm) + .map(|x| Self::from(x).into_ref(&vm.ctx)) + } + + fn __imul__(zelf: PyRef<Self>, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + zelf.try_resizable(vm)?.imul(value, vm)?; + Ok(zelf) + } + + pub(crate) fn __len__(&self) -> usize { + self.read().len() + } + + fn array_eq(&self, other: &Self, vm: &VirtualMachine) -> PyResult<bool> { + // we cannot use zelf.is(other) for shortcut because if we contenting a + // float value NaN we always return False even they are the same object. + if self.__len__() != other.__len__() { + return Ok(false); + } + let array_a = self.read(); + let array_b = other.read(); + + // fast path for same ArrayContentType type + if let Ok(ord) = array_a.cmp(&array_b) { + return Ok(ord == Some(Ordering::Equal)); + } + + let iter = Iterator::zip(array_a.iter(vm), array_b.iter(vm)); + + for (a, b) in iter { + if !vm.bool_eq(&*a?, &*b?)? { + return Ok(false); + } + } + Ok(true) + } + + #[pymethod] + fn __reduce_ex__( + zelf: &Py<Self>, + proto: usize, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, PyTupleRef, Option<PyDictRef>)> { + if proto < 3 { + return Self::__reduce__(zelf, vm); + } + let array = zelf.read(); + let cls = zelf.class().to_owned(); + let typecode = vm.ctx.new_str(array.typecode_str()); + let bytes = vm.ctx.new_bytes(array.get_bytes().to_vec()); + let code = MachineFormatCode::from_typecode(array.typecode()).unwrap(); + let code = PyInt::from(u8::from(code)).into_pyobject(vm); + let module = vm.import("array", 0)?; + let func = module.get_attr("_array_reconstructor", vm)?; + Ok(( + func, + vm.new_tuple((cls, typecode, code, bytes)), + zelf.as_object().dict(), + )) + } + + #[pymethod] + fn __reduce__( + zelf: &Py<Self>, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, PyTupleRef, Option<PyDictRef>)> { + let array = zelf.read(); + let cls = zelf.class().to_owned(); + let typecode = vm.ctx.new_str(array.typecode_str()); + let values = if array.typecode() == 'u' { + let s = Self::_wchar_bytes_to_string(array.get_bytes(), array.itemsize(), vm)?; + s.code_points().map(|x| x.to_pyobject(vm)).collect() + } else { + array.get_objects(vm) + }; + let values = vm.ctx.new_list(values); + Ok(( + cls.into(), + vm.new_tuple((typecode, values)), + zelf.as_object().dict(), + )) + } + + fn __contains__(&self, value: PyObjectRef, vm: &VirtualMachine) -> bool { + let array = self.array.read(); + for element in array + .iter(vm) + .map(|x| x.expect("Expected to be checked by array.len() and read lock.")) + { + if let Ok(true) = + element.rich_compare_bool(value.as_object(), PyComparisonOp::Eq, vm) + { + return true; + } + } + + false + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl Comparable for PyArray { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + // TODO: deduplicate this logic with sequence::cmp in sequence.rs. Maybe make it generic? + + // we cannot use zelf.is(other) for shortcut because if we contenting a + // float value NaN we always return False even they are the same object. + let other = class_or_notimplemented!(Self, other); + + if let PyComparisonValue::Implemented(x) = + op.eq_only(|| Ok(zelf.array_eq(other, vm)?.into()))? + { + return Ok(x.into()); + } + + let array_a = zelf.read(); + let array_b = other.read(); + + let res = match array_a.cmp(&array_b) { + // fast path for same ArrayContentType type + Ok(partial_ord) => partial_ord.is_some_and(|ord| op.eval_ord(ord)), + Err(()) => { + let iter = Iterator::zip(array_a.iter(vm), array_b.iter(vm)); + + for (a, b) in iter { + let ret = match op { + PyComparisonOp::Lt | PyComparisonOp::Le => { + vm.bool_seq_lt(&*a?, &*b?)? + } + PyComparisonOp::Gt | PyComparisonOp::Ge => { + vm.bool_seq_gt(&*a?, &*b?)? + } + _ => unreachable!(), + }; + if let Some(v) = ret { + return Ok(PyComparisonValue::Implemented(v)); + } + } + + // fallback: + op.eval_ord(array_a.len().cmp(&array_b.len())) + } + }; + + Ok(res.into()) + } + } + + impl AsBuffer for PyArray { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let array = zelf.read(); + let buf = PyBuffer::new( + zelf.to_owned().into(), + BufferDescriptor::format( + array.len() * array.itemsize(), + false, + array.itemsize(), + array.typecode_str().into(), + ), + &BUFFER_METHODS, + ); + Ok(buf) + } + } + + impl Representable for PyArray { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let class = zelf.class(); + let class_name = class.name(); + if zelf.read().typecode() == 'u' { + if zelf.__len__() == 0 { + return Ok(format!("{class_name}('u')")); + } + let to_unicode = zelf.tounicode(vm)?; + let escape = crate::vm::literal::escape::UnicodeEscape::new_repr(&to_unicode); + return Ok(format!("{}('u', {})", class_name, escape.str_repr())); + } + zelf.read().repr(&class_name, vm) + } + } + + static BUFFER_METHODS: BufferMethods = BufferMethods { + obj_bytes: |buffer| buffer.obj_as::<PyArray>().get_bytes().into(), + obj_bytes_mut: |buffer| buffer.obj_as::<PyArray>().get_bytes_mut().into(), + release: |buffer| { + buffer + .obj_as::<PyArray>() + .exports + .fetch_sub(1, atomic::Ordering::Release); + }, + retain: |buffer| { + buffer + .obj_as::<PyArray>() + .exports + .fetch_add(1, atomic::Ordering::Release); + }, + }; + + impl AsMapping for PyArray { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: PyMappingMethods = PyMappingMethods { + length: atomic_func!(|mapping, _vm| Ok( + PyArray::mapping_downcast(mapping).__len__() + )), + subscript: atomic_func!(|mapping, needle, vm| { + PyArray::mapping_downcast(mapping).getitem_inner(needle, vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyArray::mapping_downcast(mapping); + if let Some(value) = value { + PyArray::setitem_inner(zelf, needle, value, vm) + } else { + zelf.delitem_inner(needle, vm) + } + }), + }; + &AS_MAPPING + } + } + + impl AsSequence for PyArray { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyArray::sequence_downcast(seq).__len__())), + concat: atomic_func!(|seq, other, vm| { + let zelf = PyArray::sequence_downcast(seq); + PyArray::__add__(zelf, other.to_owned(), vm).map(|x| x.into()) + }), + repeat: atomic_func!(|seq, n, vm| { + PyArray::sequence_downcast(seq) + .__mul__(n, vm) + .map(|x| x.into()) + }), + item: atomic_func!(|seq, i, vm| { + PyArray::sequence_downcast(seq) + .read() + .getitem_by_index(i, vm) + }), + ass_item: atomic_func!(|seq, i, value, vm| { + let zelf = PyArray::sequence_downcast(seq); + if let Some(value) = value { + zelf.write().setitem_by_index(i, value, vm) + } else { + zelf.write().delitem_by_index(i, vm) + } + }), + contains: atomic_func!(|seq, target, vm| { + let zelf = PyArray::sequence_downcast(seq); + Ok(zelf.__contains__(target.to_owned(), vm)) + }), + inplace_concat: atomic_func!(|seq, other, vm| { + let zelf = PyArray::sequence_downcast(seq).to_owned(); + PyArray::__iadd__(zelf, other.to_owned(), vm).map(|x| x.into()) + }), + inplace_repeat: atomic_func!(|seq, n, vm| { + let zelf = PyArray::sequence_downcast(seq).to_owned(); + PyArray::__imul__(zelf, n, vm).map(|x| x.into()) + }), + }; + &AS_SEQUENCE + } + } + + impl Iterable for PyArray { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyArrayIter { + internal: PyMutex::new(PositionIterInternal::new(zelf, 0)), + } + .into_pyobject(vm)) + } + } + + impl BufferResizeGuard for PyArray { + type Resizable<'a> = PyRwLockWriteGuard<'a, ArrayContentType>; + + fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>> { + let w = self.write(); + (self.exports.load(atomic::Ordering::SeqCst) == 0).then_some(w) + } + } + + #[pyattr] + #[pyclass(name = "arrayiterator", traverse)] + #[derive(Debug, PyPayload)] + pub struct PyArrayIter { + internal: PyMutex<PositionIterInternal<PyArrayRef>>, + } + + #[pyclass(with(IterNext, Iterable), flags(HAS_DICT, DISALLOW_INSTANTIATION))] + impl PyArrayIter { + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.internal + .lock() + .set_state(state, |obj, pos| pos.min(obj.__len__()), vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) + } + } + + impl SelfIter for PyArrayIter {} + impl IterNext for PyArrayIter { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.internal.lock().next(|array, pos| { + Ok(match array.read().get(pos, vm) { + Some(item) => PyIterReturn::Return(item?), + None => PyIterReturn::StopIteration(None), + }) + }) + } + } + + #[derive(FromArgs)] + struct ReconstructorArgs { + #[pyarg(positional)] + arraytype: PyTypeRef, + #[pyarg(positional)] + typecode: PyUtf8StrRef, + #[pyarg(positional)] + mformat_code: MachineFormatCode, + #[pyarg(positional)] + items: PyBytesRef, + } + + #[derive(Debug, Copy, Clone, Eq, PartialEq)] + #[repr(u8)] + enum MachineFormatCode { + Int8 { signed: bool }, // 0, 1 + Int16 { signed: bool, big_endian: bool }, // 2, 3, 4, 5 + Int32 { signed: bool, big_endian: bool }, // 6, 7, 8, 9 + Int64 { signed: bool, big_endian: bool }, // 10, 11, 12, 13 + Ieee754Float { big_endian: bool }, // 14, 15 + Ieee754Double { big_endian: bool }, // 16, 17 + Utf16 { big_endian: bool }, // 18, 19 + Utf32 { big_endian: bool }, // 20, 21 + } + + impl From<MachineFormatCode> for u8 { + fn from(code: MachineFormatCode) -> Self { + use MachineFormatCode::*; + match code { + Int8 { signed } => signed as Self, + Int16 { signed, big_endian } => 2 + signed as Self * 2 + big_endian as Self, + Int32 { signed, big_endian } => 6 + signed as Self * 2 + big_endian as Self, + Int64 { signed, big_endian } => 10 + signed as Self * 2 + big_endian as Self, + Ieee754Float { big_endian } => 14 + big_endian as Self, + Ieee754Double { big_endian } => 16 + big_endian as Self, + Utf16 { big_endian } => 18 + big_endian as Self, + Utf32 { big_endian } => 20 + big_endian as Self, + } + } + } + + impl TryFrom<u8> for MachineFormatCode { + type Error = u8; + + fn try_from(code: u8) -> Result<Self, Self::Error> { + let big_endian = !code.is_multiple_of(2); + let signed = match code { + 0 | 1 => code != 0, + 2..=13 => (code - 2) % 4 >= 2, + _ => false, + }; + match code { + 0..=1 => Ok(Self::Int8 { signed }), + 2..=5 => Ok(Self::Int16 { signed, big_endian }), + 6..=9 => Ok(Self::Int32 { signed, big_endian }), + 10..=13 => Ok(Self::Int64 { signed, big_endian }), + 14..=15 => Ok(Self::Ieee754Float { big_endian }), + 16..=17 => Ok(Self::Ieee754Double { big_endian }), + 18..=19 => Ok(Self::Utf16 { big_endian }), + 20..=21 => Ok(Self::Utf32 { big_endian }), + _ => Err(code), + } + } + } + + impl<'a> TryFromBorrowedObject<'a> for MachineFormatCode { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + obj.try_to_ref::<PyInt>(vm) + .map_err(|_| { + vm.new_type_error(format!( + "an integer is required (got type {})", + obj.class().name() + )) + })? + .try_to_primitive::<i32>(vm)? + .to_u8() + .unwrap_or(u8::MAX) + .try_into() + .map_err(|_| { + vm.new_value_error("third argument must be a valid machine format code.") + }) + } + } + + impl MachineFormatCode { + fn from_typecode(code: char) -> Option<Self> { + use core::mem::size_of; + let signed = code.is_ascii_uppercase(); + let big_endian = cfg!(target_endian = "big"); + let int_size = match code { + 'b' | 'B' => return Some(Self::Int8 { signed }), + 'u' => { + return match size_of::<wchar_t>() { + 2 => Some(Self::Utf16 { big_endian }), + 4 => Some(Self::Utf32 { big_endian }), + _ => None, + }; + } + 'f' => { + // Copied from CPython + const Y: f32 = 16711938.0; + return match &Y.to_ne_bytes() { + b"\x4b\x7f\x01\x02" => Some(Self::Ieee754Float { big_endian: true }), + b"\x02\x01\x7f\x4b" => Some(Self::Ieee754Float { big_endian: false }), + _ => None, + }; + } + 'd' => { + // Copied from CPython + const Y: f64 = 9006104071832581.0; + return match &Y.to_ne_bytes() { + b"\x43\x3f\xff\x01\x02\x03\x04\x05" => { + Some(Self::Ieee754Double { big_endian: true }) + } + b"\x05\x04\x03\x02\x01\xff\x3f\x43" => { + Some(Self::Ieee754Double { big_endian: false }) + } + _ => None, + }; + } + _ => ArrayContentType::itemsize_of_typecode(code)? as u8, + }; + match int_size { + 2 => Some(Self::Int16 { signed, big_endian }), + 4 => Some(Self::Int32 { signed, big_endian }), + 8 => Some(Self::Int64 { signed, big_endian }), + _ => None, + } + } + const fn item_size(self) -> usize { + match self { + Self::Int8 { .. } => 1, + Self::Int16 { .. } | Self::Utf16 { .. } => 2, + Self::Int32 { .. } | Self::Utf32 { .. } | Self::Ieee754Float { .. } => 4, + Self::Int64 { .. } | Self::Ieee754Double { .. } => 8, + } + } + } + + fn check_array_type(typ: PyTypeRef, vm: &VirtualMachine) -> PyResult<PyTypeRef> { + if !typ.fast_issubclass(PyArray::class(&vm.ctx)) { + return Err( + vm.new_type_error(format!("{} is not a subtype of array.array", typ.name())) + ); + } + Ok(typ) + } + + fn check_type_code(spec: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<ArrayContentType> { + let spec = spec.as_str().chars().exactly_one().map_err(|_| { + vm.new_type_error( + "_array_reconstructor() argument 2 must be a unicode character, not str", + ) + })?; + ArrayContentType::from_char(spec) + .map_err(|_| vm.new_value_error("second argument must be a valid type code")) + } + + macro_rules! chunk_to_obj { + ($BYTE:ident, $TY:ty, $BIG_ENDIAN:ident) => {{ + let b = <[u8; ::core::mem::size_of::<$TY>()]>::try_from($BYTE).unwrap(); + if $BIG_ENDIAN { + <$TY>::from_be_bytes(b) + } else { + <$TY>::from_le_bytes(b) + } + }}; + ($VM:ident, $BYTE:ident, $TY:ty, $BIG_ENDIAN:ident) => { + chunk_to_obj!($BYTE, $TY, $BIG_ENDIAN).to_pyobject($VM) + }; + ($VM:ident, $BYTE:ident, $SIGNED_TY:ty, $UNSIGNED_TY:ty, $SIGNED:ident, $BIG_ENDIAN:ident) => {{ + let b = <[u8; ::core::mem::size_of::<$SIGNED_TY>()]>::try_from($BYTE).unwrap(); + match ($SIGNED, $BIG_ENDIAN) { + (false, false) => <$UNSIGNED_TY>::from_le_bytes(b).to_pyobject($VM), + (false, true) => <$UNSIGNED_TY>::from_be_bytes(b).to_pyobject($VM), + (true, false) => <$SIGNED_TY>::from_le_bytes(b).to_pyobject($VM), + (true, true) => <$SIGNED_TY>::from_be_bytes(b).to_pyobject($VM), + } + }}; + } + + #[pyfunction] + fn _array_reconstructor(args: ReconstructorArgs, vm: &VirtualMachine) -> PyResult<PyArrayRef> { + let cls = check_array_type(args.arraytype, vm)?; + let mut array = check_type_code(args.typecode, vm)?; + let format = args.mformat_code; + let bytes = args.items.as_bytes(); + if !bytes.len().is_multiple_of(format.item_size()) { + return Err(vm.new_value_error("bytes length not a multiple of item size")); + } + if MachineFormatCode::from_typecode(array.typecode()) == Some(format) { + array.frombytes(bytes); + return PyArray::from(array).into_ref_with_type(vm, cls); + } + if !matches!( + format, + MachineFormatCode::Utf16 { .. } | MachineFormatCode::Utf32 { .. } + ) { + array.reserve(bytes.len() / format.item_size()); + } + let mut chunks = bytes.chunks(format.item_size()); + match format { + MachineFormatCode::Ieee754Float { big_endian } => { + chunks.try_for_each(|b| array.push(chunk_to_obj!(vm, b, f32, big_endian), vm))? + } + MachineFormatCode::Ieee754Double { big_endian } => { + chunks.try_for_each(|b| array.push(chunk_to_obj!(vm, b, f64, big_endian), vm))? + } + MachineFormatCode::Int8 { signed } => chunks + .try_for_each(|b| array.push(chunk_to_obj!(vm, b, i8, u8, signed, false), vm))?, + MachineFormatCode::Int16 { signed, big_endian } => chunks.try_for_each(|b| { + array.push(chunk_to_obj!(vm, b, i16, u16, signed, big_endian), vm) + })?, + MachineFormatCode::Int32 { signed, big_endian } => chunks.try_for_each(|b| { + array.push(chunk_to_obj!(vm, b, i32, u32, signed, big_endian), vm) + })?, + MachineFormatCode::Int64 { signed, big_endian } => chunks.try_for_each(|b| { + array.push(chunk_to_obj!(vm, b, i64, u64, signed, big_endian), vm) + })?, + MachineFormatCode::Utf16 { big_endian } => { + let utf16: Vec<_> = chunks.map(|b| chunk_to_obj!(b, u16, big_endian)).collect(); + let s = String::from_utf16(&utf16) + .map_err(|_| vm.new_unicode_encode_error("items cannot decode as utf16"))?; + let bytes = PyArray::_unicode_to_wchar_bytes((*s).as_ref(), array.itemsize()); + array.frombytes_move(bytes); + } + MachineFormatCode::Utf32 { big_endian } => { + let s: Wtf8Buf = chunks + .map(|b| chunk_to_obj!(b, u32, big_endian)) + .map(|ch| u32_to_char(ch).map_err(|msg| vm.new_value_error(msg))) + .try_collect()?; + let bytes = PyArray::_unicode_to_wchar_bytes(&s, array.itemsize()); + array.frombytes_move(bytes); + } + }; + PyArray::from(array).into_ref_with_type(vm, cls) + } + + // Register array.array as collections.abc.MutableSequence + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::vm::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + + let array_type = module + .get_attr("array", vm) + .expect("array module has array type"); + + // vm.import returns the top-level module, so we need to get abc submodule + let collections_abc = vm.import("collections.abc", 0)?; + let abc = collections_abc.get_attr("abc", vm)?; + let mutable_sequence = abc.get_attr("MutableSequence", vm)?; + let register = mutable_sequence.get_attr("register", vm)?; + register.call((array_type,), vm)?; + + Ok(()) + } +} diff --git a/crates/stdlib/src/binascii.rs b/crates/stdlib/src/binascii.rs new file mode 100644 index 00000000000..9af56d4eea7 --- /dev/null +++ b/crates/stdlib/src/binascii.rs @@ -0,0 +1,874 @@ +// spell-checker:ignore hexlify unhexlify uuencodes CRCTAB rlecode rledecode + +pub(super) use decl::crc32; +pub(crate) use decl::module_def; +use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef, convert::ToPyException}; + +const PAD: u8 = 61u8; +const MAXLINESIZE: usize = 76; // Excluding the CRLF + +#[pymodule(name = "binascii")] +mod decl { + use super::{MAXLINESIZE, PAD}; + use crate::vm::{ + PyResult, VirtualMachine, + builtins::{PyIntRef, PyTypeRef}, + convert::ToPyException, + function::{ArgAsciiBuffer, ArgBytesLike, OptionalArg}, + }; + use base64::Engine; + use itertools::Itertools; + + #[pyattr(name = "Error", once)] + pub(super) fn error_type(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "binascii", + "Error", + Some(vec![vm.ctx.exceptions.value_error.to_owned()]), + ) + } + + #[pyattr(name = "Incomplete", once)] + fn incomplete_type(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type("binascii", "Incomplete", None) + } + + fn hex_nibble(n: u8) -> u8 { + match n { + 0..=9 => b'0' + n, + 10..=15 => b'a' + (n - 10), + _ => unreachable!(), + } + } + + #[pyfunction(name = "b2a_hex")] + #[pyfunction] + fn hexlify( + data: ArgBytesLike, + sep: OptionalArg<ArgAsciiBuffer>, + bytes_per_sep: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + let bytes_per_sep = bytes_per_sep.unwrap_or(1); + + data.with_ref(|bytes| { + // Get separator character if provided + let sep_char = if let OptionalArg::Present(sep_buf) = sep { + sep_buf.with_ref(|sep_bytes| { + if sep_bytes.len() != 1 { + return Err(vm.new_value_error("sep must be length 1.")); + } + let sep_char = sep_bytes[0]; + if !sep_char.is_ascii() { + return Err(vm.new_value_error("sep must be ASCII.")); + } + Ok(Some(sep_char)) + })? + } else { + None + }; + + // If no separator or bytes_per_sep is 0, use simple hexlify + if sep_char.is_none() || bytes_per_sep == 0 || bytes.is_empty() { + let mut hex = Vec::<u8>::with_capacity(bytes.len() * 2); + for b in bytes { + hex.push(hex_nibble(b >> 4)); + hex.push(hex_nibble(b & 0xf)); + } + return Ok(hex); + } + + let sep_char = sep_char.unwrap(); + let abs_bytes_per_sep = bytes_per_sep.unsigned_abs(); + + // If separator interval is >= data length, no separators needed + if abs_bytes_per_sep >= bytes.len() { + let mut hex = Vec::<u8>::with_capacity(bytes.len() * 2); + for b in bytes { + hex.push(hex_nibble(b >> 4)); + hex.push(hex_nibble(b & 0xf)); + } + return Ok(hex); + } + + // Calculate result length + let num_separators = (bytes.len() - 1) / abs_bytes_per_sep; + let result_len = bytes.len() * 2 + num_separators; + let mut hex = vec![0u8; result_len]; + + if bytes_per_sep < 0 { + // Left-to-right processing (negative bytes_per_sep) + let mut i = 0; // input index + let mut j = 0; // output index + let chunks = bytes.len() / abs_bytes_per_sep; + + // Process complete chunks + for _ in 0..chunks { + for _ in 0..abs_bytes_per_sep { + let b = bytes[i]; + hex[j] = hex_nibble(b >> 4); + hex[j + 1] = hex_nibble(b & 0xf); + i += 1; + j += 2; + } + if i < bytes.len() { + hex[j] = sep_char; + j += 1; + } + } + + // Process remaining bytes + while i < bytes.len() { + let b = bytes[i]; + hex[j] = hex_nibble(b >> 4); + hex[j + 1] = hex_nibble(b & 0xf); + i += 1; + j += 2; + } + } else { + // Right-to-left processing (positive bytes_per_sep) + let mut i = bytes.len() as isize - 1; // input index + let mut j = result_len as isize - 1; // output index + let chunks = bytes.len() / abs_bytes_per_sep; + + // Process complete chunks from right + for _ in 0..chunks { + for _ in 0..abs_bytes_per_sep { + let b = bytes[i as usize]; + hex[j as usize] = hex_nibble(b & 0xf); + hex[(j - 1) as usize] = hex_nibble(b >> 4); + i -= 1; + j -= 2; + } + if i >= 0 { + hex[j as usize] = sep_char; + j -= 1; + } + } + + // Process remaining bytes + while i >= 0 { + let b = bytes[i as usize]; + hex[j as usize] = hex_nibble(b & 0xf); + hex[(j - 1) as usize] = hex_nibble(b >> 4); + i -= 1; + j -= 2; + } + } + + Ok(hex) + }) + } + + const fn unhex_nibble(c: u8) -> Option<u8> { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } + } + + #[pyfunction(name = "a2b_hex")] + #[pyfunction] + fn unhexlify(data: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + data.with_ref(|hex_bytes| { + if hex_bytes.len() % 2 != 0 { + return Err(super::new_binascii_error( + "Odd-length string".to_owned(), + vm, + )); + } + + let mut unhex = Vec::<u8>::with_capacity(hex_bytes.len() / 2); + for (n1, n2) in hex_bytes.iter().tuples() { + if let (Some(n1), Some(n2)) = (unhex_nibble(*n1), unhex_nibble(*n2)) { + unhex.push((n1 << 4) | n2); + } else { + return Err(super::new_binascii_error( + "Non-hexadecimal digit found".to_owned(), + vm, + )); + } + } + + Ok(unhex) + }) + } + + #[pyfunction] + pub(crate) fn crc32(data: ArgBytesLike, init: OptionalArg<PyIntRef>) -> u32 { + let init = init.map_or(0, |i| i.as_u32_mask()); + + let mut hasher = crc32fast::Hasher::new_with_initial(init); + data.with_ref(|bytes| { + hasher.update(bytes); + hasher.finalize() + }) + } + + #[pyfunction] + pub(crate) fn crc_hqx(data: ArgBytesLike, init: PyIntRef) -> u32 { + const CRCTAB_HQX: [u16; 256] = [ + 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, + 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, + 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, + 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, + 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, + 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5, + 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, + 0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, + 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4, + 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, + 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 0xff9f, + 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, + 0xd10c, 0xc12d, 0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, + 0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, 0x1290, + 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, + 0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, + 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, + 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, + 0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d, + 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37, 0x7a16, + 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, + 0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, + 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, + 0x3eb2, 0x0ed1, 0x1ef0, + ]; + + let mut crc = init.as_u32_mask() & 0xffff; + + data.with_ref(|buf| { + for byte in buf { + crc = + ((crc << 8) & 0xFF00) ^ CRCTAB_HQX[((crc >> 8) as u8 ^ (byte)) as usize] as u32; + } + }); + + crc + } + + #[derive(FromArgs)] + struct NewlineArg { + #[pyarg(named, default = true)] + newline: bool, + } + + #[derive(FromArgs)] + struct A2bBase64Args { + #[pyarg(any)] + s: ArgAsciiBuffer, + #[pyarg(named, default = false)] + strict_mode: bool, + } + + #[pyfunction] + fn a2b_base64(args: A2bBase64Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + #[rustfmt::skip] + // Converts between ASCII and base-64 characters. The index of a given number yields the + // number in ASCII while the value of said index yields the number in base-64. For example + // "=" is 61 in ASCII but 0 (since it's the pad character) in base-64, so BASE64_TABLE[61] == 0 + const BASE64_TABLE: [i8; 256] = [ + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63, + 52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, /* Note PAD->0 */ + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1, + -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + 41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1, + + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, + ]; + + let A2bBase64Args { s, strict_mode } = args; + s.with_ref(|b| { + if b.is_empty() { + return Ok(vec![]); + } + + if strict_mode && b[0] == PAD { + return Err(base64::DecodeError::InvalidByte(0, 61)); + } + + let mut decoded: Vec<u8> = vec![]; + + let mut quad_pos = 0; // position in the nibble + let mut pads = 0; + let mut left_char: u8 = 0; + let mut padding_started = false; + for (i, &el) in b.iter().enumerate() { + if el == PAD { + padding_started = true; + + pads += 1; + if quad_pos >= 2 && quad_pos + pads >= 4 { + if strict_mode && i + 1 < b.len() { + // Represents excess data after padding error + return Err(base64::DecodeError::InvalidLastSymbol(i, PAD)); + } + + return Ok(decoded); + } + + continue; + } + + let binary_char = BASE64_TABLE[el as usize]; + if binary_char >= 64 || binary_char == -1 { + if strict_mode { + // Represents non-base64 data error + return Err(base64::DecodeError::InvalidByte(i, el)); + } + continue; + } + + if strict_mode && padding_started { + // Represents discontinuous padding error + return Err(base64::DecodeError::InvalidByte(i, PAD)); + } + pads = 0; + + // Decode individual ASCII character + match quad_pos { + 0 => { + quad_pos = 1; + left_char = binary_char as u8; + } + 1 => { + quad_pos = 2; + decoded.push((left_char << 2) | (binary_char >> 4) as u8); + left_char = (binary_char & 0x0f) as u8; + } + 2 => { + quad_pos = 3; + decoded.push((left_char << 4) | (binary_char >> 2) as u8); + left_char = (binary_char & 0x03) as u8; + } + 3 => { + quad_pos = 0; + decoded.push((left_char << 6) | binary_char as u8); + left_char = 0; + } + _ => unsafe { + // quad_pos is only assigned in this match statement to constants + core::hint::unreachable_unchecked() + }, + } + } + + match quad_pos { + 0 => Ok(decoded), + 1 => Err(base64::DecodeError::InvalidLastSymbol( + decoded.len() / 3 * 4 + 1, + 0, + )), + _ => Err(base64::DecodeError::InvalidLength(quad_pos)), + } + }) + .map_err(|err| super::Base64DecodeError(err).to_pyexception(vm)) + } + + #[pyfunction] + fn b2a_base64(data: ArgBytesLike, NewlineArg { newline }: NewlineArg) -> Vec<u8> { + // https://stackoverflow.com/questions/63916821 + let mut encoded = data + .with_ref(|b| base64::engine::general_purpose::STANDARD.encode(b)) + .into_bytes(); + if newline { + encoded.push(b'\n'); + } + encoded + } + + #[inline] + fn uu_a2b_read(c: &u8, vm: &VirtualMachine) -> PyResult<u8> { + // Check the character for legality + // The 64 instead of the expected 63 is because + // there are a few uuencodes out there that use + // '`' as zero instead of space. + if !(b' '..=(b' ' + 64)).contains(c) { + if [b'\r', b'\n'].contains(c) { + return Ok(0); + } + return Err(super::new_binascii_error("Illegal char".to_owned(), vm)); + } + Ok((*c - b' ') & 0x3f) + } + + #[derive(FromArgs)] + struct A2bQpArgs { + #[pyarg(any)] + data: ArgAsciiBuffer, + #[pyarg(named, default = false)] + header: bool, + } + #[pyfunction] + fn a2b_qp(args: A2bQpArgs) -> PyResult<Vec<u8>> { + let s = args.data; + let header = args.header; + s.with_ref(|buffer| { + let len = buffer.len(); + let mut out_data = Vec::with_capacity(len); + + let mut idx = 0; + + while idx < len { + if buffer[idx] == b'=' { + idx += 1; + if idx >= len { + break; + } + // Soft line breaks + if (buffer[idx] == b'\n') || (buffer[idx] == b'\r') { + if buffer[idx] != b'\n' { + while idx < len && buffer[idx] != b'\n' { + idx += 1; + } + } + if idx < len { + idx += 1; + } + } else if buffer[idx] == b'=' { + // broken case from broken python qp + out_data.push(b'='); + idx += 1; + } else if idx + 1 < len + && ((buffer[idx] >= b'A' && buffer[idx] <= b'F') + || (buffer[idx] >= b'a' && buffer[idx] <= b'f') + || (buffer[idx] >= b'0' && buffer[idx] <= b'9')) + && ((buffer[idx + 1] >= b'A' && buffer[idx + 1] <= b'F') + || (buffer[idx + 1] >= b'a' && buffer[idx + 1] <= b'f') + || (buffer[idx + 1] >= b'0' && buffer[idx + 1] <= b'9')) + { + // hex val + if let (Some(ch1), Some(ch2)) = + (unhex_nibble(buffer[idx]), unhex_nibble(buffer[idx + 1])) + { + out_data.push((ch1 << 4) | ch2); + } + idx += 2; + } else { + out_data.push(b'='); + } + } else if header && buffer[idx] == b'_' { + out_data.push(b' '); + idx += 1; + } else { + out_data.push(buffer[idx]); + idx += 1; + } + } + + Ok(out_data) + }) + } + + #[derive(FromArgs)] + struct B2aQpArgs { + #[pyarg(any)] + data: ArgBytesLike, + #[pyarg(named, default = false)] + quotetabs: bool, + #[pyarg(named, default = true)] + istext: bool, + #[pyarg(named, default = false)] + header: bool, + } + + #[pyfunction] + fn b2a_qp(args: B2aQpArgs) -> PyResult<Vec<u8>> { + let s = args.data; + let quotetabs = args.quotetabs; + let istext = args.istext; + let header = args.header; + s.with_ref(|buf| { + let buflen = buf.len(); + let mut line_len = 0; + let mut out_data_len = 0; + let mut crlf = false; + let mut ch; + + let mut in_idx; + let mut out_idx; + + in_idx = 0; + while in_idx < buflen { + if buf[in_idx] == b'\n' { + break; + } + in_idx += 1; + } + if buflen > 0 && in_idx < buflen && buf[in_idx - 1] == b'\r' { + crlf = true; + } + + in_idx = 0; + while in_idx < buflen { + let mut delta = 0; + if (buf[in_idx] > 126) + || (buf[in_idx] == b'=') + || (header && buf[in_idx] == b'_') + || (buf[in_idx] == b'.' + && line_len == 0 + && (in_idx + 1 == buflen + || buf[in_idx + 1] == b'\n' + || buf[in_idx + 1] == b'\r' + || buf[in_idx + 1] == 0)) + || (!istext && ((buf[in_idx] == b'\r') || (buf[in_idx] == b'\n'))) + || ((buf[in_idx] == b'\t' || buf[in_idx] == b' ') && (in_idx + 1 == buflen)) + || ((buf[in_idx] < 33) + && (buf[in_idx] != b'\r') + && (buf[in_idx] != b'\n') + && (quotetabs || ((buf[in_idx] != b'\t') && (buf[in_idx] != b' ')))) + { + if (line_len + 3) >= MAXLINESIZE { + line_len = 0; + delta += if crlf { 3 } else { 2 }; + } + line_len += 3; + delta += 3; + in_idx += 1; + } else if istext + && ((buf[in_idx] == b'\n') + || ((in_idx + 1 < buflen) + && (buf[in_idx] == b'\r') + && (buf[in_idx + 1] == b'\n'))) + { + line_len = 0; + // Protect against whitespace on end of line + if (in_idx != 0) && ((buf[in_idx - 1] == b' ') || (buf[in_idx - 1] == b'\t')) { + delta += 2; + } + delta += if crlf { 2 } else { 1 }; + in_idx += if buf[in_idx] == b'\r' { 2 } else { 1 }; + } else { + if (in_idx + 1 != buflen) + && (buf[in_idx + 1] != b'\n') + && (line_len + 1) >= MAXLINESIZE + { + line_len = 0; + delta += if crlf { 3 } else { 2 }; + } + line_len += 1; + delta += 1; + in_idx += 1; + } + out_data_len += delta; + } + + let mut out_data = Vec::with_capacity(out_data_len); + in_idx = 0; + out_idx = 0; + line_len = 0; + + while in_idx < buflen { + if (buf[in_idx] > 126) + || (buf[in_idx] == b'=') + || (header && buf[in_idx] == b'_') + || ((buf[in_idx] == b'.') + && (line_len == 0) + && (in_idx + 1 == buflen + || buf[in_idx + 1] == b'\n' + || buf[in_idx + 1] == b'\r' + || buf[in_idx + 1] == 0)) + || (!istext && ((buf[in_idx] == b'\r') || (buf[in_idx] == b'\n'))) + || ((buf[in_idx] == b'\t' || buf[in_idx] == b' ') && (in_idx + 1 == buflen)) + || ((buf[in_idx] < 33) + && (buf[in_idx] != b'\r') + && (buf[in_idx] != b'\n') + && (quotetabs || ((buf[in_idx] != b'\t') && (buf[in_idx] != b' ')))) + { + if (line_len + 3) >= MAXLINESIZE { + // MAXLINESIZE = 76 + out_data.push(b'='); + out_idx += 1; + if crlf { + out_data.push(b'\r'); + out_idx += 1; + } + out_data.push(b'\n'); + out_idx += 1; + line_len = 0; + } + out_data.push(b'='); + out_idx += 1; + + ch = hex_nibble(buf[in_idx] >> 4); + if (b'a'..=b'f').contains(&ch) { + ch -= b' '; + } + out_data.push(ch); + ch = hex_nibble(buf[in_idx] & 0xf); + if (b'a'..=b'f').contains(&ch) { + ch -= b' '; + } + out_data.push(ch); + + out_idx += 2; + in_idx += 1; + line_len += 3; + } else if istext + && ((buf[in_idx] == b'\n') + || ((in_idx + 1 < buflen) + && (buf[in_idx] == b'\r') + && (buf[in_idx + 1] == b'\n'))) + { + line_len = 0; + if (out_idx != 0) + && ((out_data[out_idx - 1] == b' ') || (out_data[out_idx - 1] == b'\t')) + { + ch = hex_nibble(out_data[out_idx - 1] >> 4); + if (b'a'..=b'f').contains(&ch) { + ch -= b' '; + } + out_data.push(ch); + ch = hex_nibble(out_data[out_idx - 1] & 0xf); + if (b'a'..=b'f').contains(&ch) { + ch -= b' '; + } + out_data.push(ch); + out_data[out_idx - 1] = b'='; + out_idx += 2; + } + + if crlf { + out_data.push(b'\r'); + out_idx += 1; + } + out_data.push(b'\n'); + out_idx += 1; + in_idx += if buf[in_idx] == b'\r' { 2 } else { 1 }; + } else { + if (in_idx + 1 != buflen) && (buf[in_idx + 1] != b'\n') && (line_len + 1) >= 76 + { + // MAXLINESIZE = 76 + out_data.push(b'='); + out_idx += 1; + if crlf { + out_data.push(b'\r'); + out_idx += 1; + } + out_data.push(b'\n'); + out_idx += 1; + line_len = 0; + } + line_len += 1; + if header && buf[in_idx] == b' ' { + out_data.push(b'_'); + out_idx += 1; + in_idx += 1; + } else { + out_data.push(buf[in_idx]); + out_idx += 1; + in_idx += 1; + } + } + } + Ok(out_data) + }) + } + + #[pyfunction] + fn rlecode_hqx(s: ArgAsciiBuffer) -> PyResult<Vec<u8>> { + const RUN_CHAR: u8 = 0x90; // b'\x90' + s.with_ref(|buffer| { + let len = buffer.len(); + let mut out_data = Vec::<u8>::with_capacity((len * 2) + 2); + + let mut idx = 0; + while idx < len { + let ch = buffer[idx]; + + if ch == RUN_CHAR { + out_data.push(RUN_CHAR); + out_data.push(0); + return Ok(out_data); + } else { + let mut in_end = idx + 1; + while in_end < len && buffer[in_end] == ch && in_end < idx + 255 { + in_end += 1; + } + if in_end - idx > 3 { + out_data.push(ch); + out_data.push(RUN_CHAR); + out_data.push(((in_end - idx) % 256) as u8); + idx = in_end - 1; + } else { + out_data.push(ch); + } + } + idx += 1; + } + Ok(out_data) + }) + } + + #[pyfunction] + fn rledecode_hqx(s: ArgAsciiBuffer) -> PyResult<Vec<u8>> { + const RUN_CHAR: u8 = 0x90; //b'\x90' + s.with_ref(|buffer| { + let len = buffer.len(); + let mut out_data = Vec::<u8>::with_capacity(len); + let mut idx = 0; + + out_data.push(buffer[idx]); + idx += 1; + + while idx < len { + if buffer[idx] == RUN_CHAR { + if buffer[idx + 1] == 0 { + out_data.push(RUN_CHAR); + } else { + let ch = buffer[idx - 1]; + let range = buffer[idx + 1]; + idx += 1; + for _ in 1..range { + out_data.push(ch); + } + } + } else { + out_data.push(buffer[idx]); + } + idx += 1; + } + Ok(out_data) + }) + } + + #[pyfunction] + fn a2b_uu(s: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + s.with_ref(|b| { + // First byte: binary data length (in bytes) + let length = if b.is_empty() { + ((-0x20i32) & 0x3fi32) as usize + } else { + ((b[0] - b' ') & 0x3f) as usize + }; + + // Allocate the buffer + let mut res = Vec::<u8>::with_capacity(length); + let trailing_garbage_error = + || Err(super::new_binascii_error("Trailing garbage".to_owned(), vm)); + + for chunk in b.get(1..).unwrap_or_default().chunks(4) { + let (char_a, char_b, char_c, char_d) = { + let mut chunk = chunk + .iter() + .map(|x| uu_a2b_read(x, vm)) + .collect::<Result<Vec<_>, _>>()?; + while chunk.len() < 4 { + chunk.push(0); + } + (chunk[0], chunk[1], chunk[2], chunk[3]) + }; + + if res.len() < length { + res.push((char_a << 2) | (char_b >> 4)); + } else if char_a != 0 || char_b != 0 { + return trailing_garbage_error(); + } + + if res.len() < length { + res.push(((char_b & 0xf) << 4) | (char_c >> 2)); + } else if char_c != 0 { + return trailing_garbage_error(); + } + + if res.len() < length { + res.push(((char_c & 0x3) << 6) | char_d); + } else if char_d != 0 { + return trailing_garbage_error(); + } + } + + let remaining_length = length - res.len(); + if remaining_length > 0 { + res.extend(vec![0; remaining_length]); + } + Ok(res) + }) + } + + #[derive(FromArgs)] + struct BacktickArg { + #[pyarg(named, default = false)] + backtick: bool, + } + + #[pyfunction] + fn b2a_uu( + data: ArgBytesLike, + BacktickArg { backtick }: BacktickArg, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + #[inline] + const fn uu_b2a(num: u8, backtick: bool) -> u8 { + if backtick && num == 0 { + 0x60 + } else { + b' ' + num + } + } + + data.with_ref(|b| { + let length = b.len(); + if length > 45 { + return Err(super::new_binascii_error( + "At most 45 bytes at once".to_owned(), + vm, + )); + } + let mut res = Vec::<u8>::with_capacity(2 + length.div_ceil(3) * 4); + res.push(uu_b2a(length as u8, backtick)); + + for chunk in b.chunks(3) { + let char_a = *chunk.first().unwrap(); + let char_b = *chunk.get(1).unwrap_or(&0); + let char_c = *chunk.get(2).unwrap_or(&0); + + res.push(uu_b2a(char_a >> 2, backtick)); + res.push(uu_b2a(((char_a & 0x3) << 4) | (char_b >> 4), backtick)); + res.push(uu_b2a(((char_b & 0xf) << 2) | (char_c >> 6), backtick)); + res.push(uu_b2a(char_c & 0x3f, backtick)); + } + + res.push(0xau8); + Ok(res) + }) + } +} + +struct Base64DecodeError(base64::DecodeError); + +fn new_binascii_error(msg: String, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_exception_msg(decl::error_type(vm), msg.into()) +} + +impl ToPyException for Base64DecodeError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + use base64::DecodeError::*; + let message = match &self.0 { + InvalidByte(0, PAD) => "Leading padding not allowed".to_owned(), + InvalidByte(_, PAD) => "Discontinuous padding not allowed".to_owned(), + InvalidByte(_, _) => "Only base64 data is allowed".to_owned(), + InvalidLastSymbol(_, PAD) => "Excess data after padding".to_owned(), + InvalidLastSymbol(length, _) => { + format!( + "Invalid base64-encoded string: number of data characters {length} cannot be 1 more than a multiple of 4" + ) + } + // TODO: clean up errors + InvalidLength(_) => "Incorrect padding".to_owned(), + InvalidPadding => "Incorrect padding".to_owned(), + }; + new_binascii_error(format!("error decoding base64: {message}"), vm) + } +} diff --git a/stdlib/src/bisect.rs b/crates/stdlib/src/bisect.rs similarity index 95% rename from stdlib/src/bisect.rs rename to crates/stdlib/src/bisect.rs index aaab65d7888..69b6e8aee46 100644 --- a/stdlib/src/bisect.rs +++ b/crates/stdlib/src/bisect.rs @@ -1,11 +1,11 @@ -pub(crate) use _bisect::make_module; +pub(crate) use _bisect::module_def; #[pymodule] mod _bisect { use crate::vm::{ + PyObjectRef, PyResult, VirtualMachine, function::{ArgIndex, OptionalArg}, types::PyComparisonOp, - PyObjectRef, PyResult, VirtualMachine, }; #[derive(FromArgs)] @@ -24,7 +24,7 @@ mod _bisect { #[inline] fn handle_default(arg: OptionalArg<ArgIndex>, vm: &VirtualMachine) -> PyResult<Option<isize>> { arg.into_option() - .map(|v| v.try_to_primitive(vm)) + .map(|v| v.into_int_ref().try_to_primitive(vm)) .transpose() } @@ -46,8 +46,7 @@ mod _bisect { // Default is always a Some so we can safely unwrap. let lo = handle_default(lo, vm)? .map(|value| { - usize::try_from(value) - .map_err(|_| vm.new_value_error("lo must be non-negative".to_owned())) + usize::try_from(value).map_err(|_| vm.new_value_error("lo must be non-negative")) }) .unwrap_or(Ok(0))?; let hi = handle_default(hi, vm)? diff --git a/crates/stdlib/src/blake2.rs b/crates/stdlib/src/blake2.rs new file mode 100644 index 00000000000..53687c7027a --- /dev/null +++ b/crates/stdlib/src/blake2.rs @@ -0,0 +1,19 @@ +// spell-checker:ignore usedforsecurity HASHXOF + +pub(crate) use _blake2::module_def; + +#[pymodule] +mod _blake2 { + use crate::hashlib::_hashlib::{BlakeHashArgs, local_blake2b, local_blake2s}; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; + + #[pyfunction] + fn blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_blake2b(args, vm)?.into_pyobject(vm)) + } + + #[pyfunction] + fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_blake2s(args, vm)?.into_pyobject(vm)) + } +} diff --git a/crates/stdlib/src/bz2.rs b/crates/stdlib/src/bz2.rs new file mode 100644 index 00000000000..575f33c4b8f --- /dev/null +++ b/crates/stdlib/src/bz2.rs @@ -0,0 +1,197 @@ +// spell-checker:ignore compresslevel + +pub(crate) use _bz2::module_def; + +#[pymodule] +mod _bz2 { + use crate::compression::{ + DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, + }; + use crate::vm::{ + Py, VirtualMachine, + builtins::{PyBytesRef, PyType}, + common::lock::PyMutex, + function::{ArgBytesLike, OptionalArg}, + object::PyResult, + types::Constructor, + }; + use alloc::fmt; + use bzip2::{Decompress, Status, write::BzEncoder}; + use rustpython_vm::convert::ToPyException; + use std::io::Write; + + const BUFSIZ: usize = 8192; + + #[pyattr] + #[pyclass(name = "BZ2Decompressor")] + #[derive(PyPayload)] + struct BZ2Decompressor { + state: PyMutex<DecompressState<Decompress>>, + } + + impl Decompressor for Decompress { + type Flush = (); + type Status = Status; + type Error = bzip2::Error; + + fn total_in(&self) -> u64 { + self.total_in() + } + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + (): Self::Flush, + ) -> Result<Self::Status, Self::Error> { + self.decompress_vec(input, output) + } + } + + impl DecompressStatus for Status { + fn is_stream_end(&self) -> bool { + *self == Self::StreamEnd + } + } + + impl fmt::Debug for BZ2Decompressor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "_bz2.BZ2Decompressor") + } + } + + impl Constructor for BZ2Decompressor { + type Args = (); + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + state: PyMutex::new(DecompressState::new(Decompress::new(false), vm)), + }) + } + } + + #[pyclass(with(Constructor))] + impl BZ2Decompressor { + #[pymethod] + fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let max_length = args.max_length(); + let data = &*args.data(); + + let mut state = self.state.lock(); + state + .decompress(data, max_length, BUFSIZ, vm) + .map_err(|e| match e { + DecompressError::Decompress(err) => vm.new_os_error(err.to_string()), + DecompressError::Eof(err) => err.to_pyexception(vm), + }) + } + + #[pygetset] + fn eof(&self) -> bool { + self.state.lock().eof() + } + + #[pygetset] + fn unused_data(&self) -> PyBytesRef { + self.state.lock().unused_data() + } + + #[pygetset] + fn needs_input(&self) -> bool { + // False if the decompress() method can provide more + // decompressed data before requiring new uncompressed input. + self.state.lock().needs_input() + } + + #[pymethod(name = "__reduce__")] + fn reduce(&self, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("cannot pickle '_bz2.BZ2Decompressor' object")) + } + + // TODO: mro()? + } + + struct CompressorState { + flushed: bool, + encoder: Option<BzEncoder<Vec<u8>>>, + } + + #[pyattr] + #[pyclass(name = "BZ2Compressor")] + #[derive(PyPayload)] + struct BZ2Compressor { + state: PyMutex<CompressorState>, + } + + impl fmt::Debug for BZ2Compressor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "_bz2.BZ2Compressor") + } + } + + impl Constructor for BZ2Compressor { + type Args = (OptionalArg<i32>,); + + fn py_new( + _cls: &Py<PyType>, + (compresslevel,): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + // TODO: seriously? + // compresslevel.unwrap_or(bzip2::Compression::best().level().try_into().unwrap()); + let compresslevel = compresslevel.unwrap_or(9); + let level = match compresslevel { + valid_level @ 1..=9 => bzip2::Compression::new(valid_level as u32), + _ => { + return Err(vm.new_value_error("compresslevel must be between 1 and 9")); + } + }; + + Ok(Self { + state: PyMutex::new(CompressorState { + flushed: false, + encoder: Some(BzEncoder::new(Vec::new(), level)), + }), + }) + } + } + + // TODO: return partial results from compress() instead of returning everything in flush() + #[pyclass(with(Constructor))] + impl BZ2Compressor { + #[pymethod] + fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + let mut state = self.state.lock(); + if state.flushed { + return Err(vm.new_value_error("Compressor has been flushed")); + } + + // let CompressorState { flushed, encoder } = &mut *state; + let CompressorState { encoder, .. } = &mut *state; + + // TODO: handle Err + data.with_ref(|input_bytes| encoder.as_mut().unwrap().write_all(input_bytes).unwrap()); + Ok(vm.ctx.new_bytes(Vec::new())) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + let mut state = self.state.lock(); + if state.flushed { + return Err(vm.new_value_error("Repeated call to flush()")); + } + + // let CompressorState { flushed, encoder } = &mut *state; + let CompressorState { encoder, .. } = &mut *state; + + // TODO: handle Err + let out = encoder.take().unwrap().finish().unwrap(); + state.flushed = true; + Ok(vm.ctx.new_bytes(out.to_vec())) + } + + #[pymethod(name = "__reduce__")] + fn reduce(&self, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("cannot pickle '_bz2.BZ2Compressor' object")) + } + } +} diff --git a/crates/stdlib/src/cmath.rs b/crates/stdlib/src/cmath.rs new file mode 100644 index 00000000000..e7ea317d212 --- /dev/null +++ b/crates/stdlib/src/cmath.rs @@ -0,0 +1,170 @@ +pub(crate) use cmath::module_def; + +#[pymodule] +mod cmath { + use crate::vm::{ + PyResult, VirtualMachine, + function::{ArgIntoComplex, ArgIntoFloat, OptionalArg}, + }; + use num_complex::Complex64; + + use crate::math::pymath_exception; + + // Constants + #[pyattr(name = "e")] + const E: f64 = pymath::cmath::E; + #[pyattr(name = "pi")] + const PI: f64 = pymath::cmath::PI; + #[pyattr(name = "tau")] + const TAU: f64 = pymath::cmath::TAU; + #[pyattr(name = "inf")] + const INF: f64 = pymath::cmath::INF; + #[pyattr(name = "nan")] + const NAN: f64 = pymath::cmath::NAN; + #[pyattr(name = "infj")] + const INFJ: Complex64 = pymath::cmath::INFJ; + #[pyattr(name = "nanj")] + const NANJ: Complex64 = pymath::cmath::NANJ; + + #[pyfunction] + fn phase(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<f64> { + pymath::cmath::phase(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn polar(x: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<(f64, f64)> { + pymath::cmath::polar(x.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn rect(r: ArgIntoFloat, phi: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::rect(r.into_float(), phi.into_float()) + .map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn isinf(z: ArgIntoComplex) -> bool { + pymath::cmath::isinf(z.into_complex()) + } + + #[pyfunction] + fn isfinite(z: ArgIntoComplex) -> bool { + pymath::cmath::isfinite(z.into_complex()) + } + + #[pyfunction] + fn isnan(z: ArgIntoComplex) -> bool { + pymath::cmath::isnan(z.into_complex()) + } + + #[pyfunction] + fn exp(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::exp(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn sqrt(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::sqrt(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn sin(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::sin(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn asin(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::asin(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn cos(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::cos(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn acos(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::acos(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn log( + z: ArgIntoComplex, + base: OptionalArg<ArgIntoComplex>, + vm: &VirtualMachine, + ) -> PyResult<Complex64> { + pymath::cmath::log( + z.into_complex(), + base.into_option().map(|b| b.into_complex()), + ) + .map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn log10(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::log10(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn acosh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::acosh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn atan(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::atan(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn atanh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::atanh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn tan(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::tan(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn tanh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::tanh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn sinh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::sinh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn cosh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::cosh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn asinh(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { + pymath::cmath::asinh(z.into_complex()).map_err(|err| pymath_exception(err, vm)) + } + + #[derive(FromArgs)] + struct IsCloseArgs { + #[pyarg(positional)] + a: ArgIntoComplex, + #[pyarg(positional)] + b: ArgIntoComplex, + #[pyarg(named, optional)] + rel_tol: OptionalArg<ArgIntoFloat>, + #[pyarg(named, optional)] + abs_tol: OptionalArg<ArgIntoFloat>, + } + + #[pyfunction] + fn isclose(args: IsCloseArgs, vm: &VirtualMachine) -> PyResult<bool> { + let a = args.a.into_complex(); + let b = args.b.into_complex(); + let rel_tol = args.rel_tol.into_option().map(|v| v.into_float()); + let abs_tol = args.abs_tol.into_option().map(|v| v.into_float()); + + pymath::cmath::isclose(a, b, rel_tol, abs_tol) + .map_err(|_| vm.new_value_error("tolerances must be non-negative")) + } +} diff --git a/crates/stdlib/src/compression.rs b/crates/stdlib/src/compression.rs new file mode 100644 index 00000000000..ae47ebbff70 --- /dev/null +++ b/crates/stdlib/src/compression.rs @@ -0,0 +1,378 @@ +// spell-checker:ignore chunker + +//! internal shared module for compression libraries + +use crate::vm::function::{ArgBytesLike, ArgSize, OptionalArg}; +use crate::vm::{ + PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef}, + convert::ToPyException, +}; + +pub const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; +// TODO: don't hardcode +const CHUNKSIZE: usize = u32::MAX as usize; + +#[derive(FromArgs)] +pub struct DecompressArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(any, optional)] + max_length: OptionalArg<ArgSize>, +} + +impl DecompressArgs { + pub fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { + self.data.borrow_buf() + } + pub fn raw_max_length(&self) -> Option<isize> { + self.max_length.into_option().map(|ArgSize { value }| value) + } + + // negative is None + pub fn max_length(&self) -> Option<usize> { + self.max_length + .into_option() + .and_then(|ArgSize { value }| usize::try_from(value).ok()) + } +} + +pub trait Decompressor { + type Flush: DecompressFlushKind; + type Status: DecompressStatus; + type Error; + + fn total_in(&self) -> u64; + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: Self::Flush, + ) -> Result<Self::Status, Self::Error>; + fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { + Err(err) + } +} + +pub trait DecompressStatus { + fn is_stream_end(&self) -> bool; +} + +pub trait DecompressFlushKind: Copy { + const SYNC: Self; +} + +impl DecompressFlushKind for () { + const SYNC: Self = (); +} + +pub const fn flush_sync<T: DecompressFlushKind>(_final_chunk: bool) -> T { + T::SYNC +} + +#[derive(Clone)] +pub struct Chunker<'a> { + data1: &'a [u8], + data2: &'a [u8], +} +impl<'a> Chunker<'a> { + pub const fn new(data: &'a [u8]) -> Self { + Self { + data1: data, + data2: &[], + } + } + pub const fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { + if data1.is_empty() { + Self { + data1: data2, + data2: &[], + } + } else { + Self { data1, data2 } + } + } + pub const fn len(&self) -> usize { + self.data1.len() + self.data2.len() + } + pub const fn is_empty(&self) -> bool { + self.data1.is_empty() + } + pub fn to_vec(&self) -> Vec<u8> { + [self.data1, self.data2].concat() + } + pub fn chunk(&self) -> &'a [u8] { + self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) + } + pub fn advance(&mut self, consumed: usize) { + self.data1 = &self.data1[consumed..]; + if self.data1.is_empty() { + self.data1 = core::mem::take(&mut self.data2); + } + } +} + +pub fn _decompress<D: Decompressor>( + data: &[u8], + d: &mut D, + bufsize: usize, + max_length: Option<usize>, + calc_flush: impl Fn(bool) -> D::Flush, +) -> Result<(Vec<u8>, bool), D::Error> { + let mut data = Chunker::new(data); + _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) +} + +pub fn _decompress_chunks<D: Decompressor>( + data: &mut Chunker<'_>, + d: &mut D, + bufsize: usize, + max_length: Option<usize>, + calc_flush: impl Fn(bool) -> D::Flush, +) -> Result<(Vec<u8>, bool), D::Error> { + if data.is_empty() { + return Ok((Vec::new(), true)); + } + let max_length = max_length.unwrap_or(usize::MAX); + let mut buf = Vec::new(); + + 'outer: loop { + let chunk = data.chunk(); + let flush = calc_flush(chunk.len() == data.len()); + loop { + let additional = core::cmp::min(bufsize, max_length - buf.capacity()); + if additional == 0 { + return Ok((buf, false)); + } + buf.reserve_exact(additional); + + let prev_in = d.total_in(); + let res = d.decompress_vec(chunk, &mut buf, flush); + let consumed = d.total_in() - prev_in; + + data.advance(consumed as usize); + + match res { + Ok(status) => { + let stream_end = status.is_stream_end(); + if stream_end || data.is_empty() { + // we've reached the end of the stream, we're done + buf.shrink_to_fit(); + return Ok((buf, stream_end)); + } else if !chunk.is_empty() && consumed == 0 { + // we're gonna need a bigger buffer + continue; + } else { + // next chunk + continue 'outer; + } + } + Err(e) => { + d.maybe_set_dict(e)?; + // now try the next chunk + continue 'outer; + } + }; + } + } +} + +pub trait Compressor { + type Status: CompressStatusKind; + type Flush: CompressFlushKind; + const CHUNKSIZE: usize; + const DEF_BUF_SIZE: usize; + + fn compress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: Self::Flush, + vm: &VirtualMachine, + ) -> PyResult<Self::Status>; + + fn total_in(&mut self) -> usize; + + fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef; +} + +pub trait CompressFlushKind: Copy { + const NONE: Self; + const FINISH: Self; + + fn to_usize(self) -> usize; +} + +pub trait CompressStatusKind: Copy { + const EOF: Self; + + fn to_usize(self) -> usize; +} + +#[derive(Debug)] +pub struct CompressState<C: Compressor> { + compressor: Option<C>, +} + +impl<C: Compressor> CompressState<C> { + pub const fn new(compressor: C) -> Self { + Self { + compressor: Some(compressor), + } + } + + fn get_compressor(&mut self, vm: &VirtualMachine) -> PyResult<&mut C> { + self.compressor + .as_mut() + .ok_or_else(|| C::new_error(USE_AFTER_FINISH_ERR, vm)) + } + + pub fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut buf = Vec::new(); + let compressor = self.get_compressor(vm)?; + + for mut chunk in data.chunks(C::CHUNKSIZE) { + while !chunk.is_empty() { + buf.reserve(C::DEF_BUF_SIZE); + let prev_in = compressor.total_in(); + compressor.compress_vec(chunk, &mut buf, C::Flush::NONE, vm)?; + let consumed = compressor.total_in() - prev_in; + chunk = &chunk[consumed..]; + } + } + + buf.shrink_to_fit(); + Ok(buf) + } + + pub fn flush(&mut self, mode: C::Flush, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut buf = Vec::new(); + let compressor = self.get_compressor(vm)?; + + let status = loop { + if buf.len() == buf.capacity() { + buf.reserve(C::DEF_BUF_SIZE); + } + let status = compressor.compress_vec(&[], &mut buf, mode, vm)?; + if buf.len() != buf.capacity() { + break status; + } + }; + + if status.to_usize() == C::Status::EOF.to_usize() { + if mode.to_usize() == C::Flush::FINISH.to_usize() { + self.compressor = None; + } else { + return Err(C::new_error("unexpected eof", vm)); + } + } + + buf.shrink_to_fit(); + Ok(buf) + } +} + +#[derive(Debug)] +pub struct DecompressState<D> { + decompress: D, + unused_data: PyBytesRef, + input_buffer: Vec<u8>, + eof: bool, + needs_input: bool, +} + +impl<D: Decompressor> DecompressState<D> { + pub fn new(decompress: D, vm: &VirtualMachine) -> Self { + Self { + decompress, + unused_data: vm.ctx.empty_bytes.clone(), + input_buffer: Vec::new(), + eof: false, + needs_input: true, + } + } + + pub const fn eof(&self) -> bool { + self.eof + } + + #[cfg_attr(target_os = "android", allow(dead_code))] + pub const fn decompressor(&self) -> &D { + &self.decompress + } + + pub fn unused_data(&self) -> PyBytesRef { + self.unused_data.clone() + } + + pub const fn needs_input(&self) -> bool { + self.needs_input + } + + pub fn decompress( + &mut self, + data: &[u8], + max_length: Option<usize>, + bufsize: usize, + vm: &VirtualMachine, + ) -> Result<Vec<u8>, DecompressError<D::Error>> { + if self.eof { + return Err(DecompressError::Eof(EofError)); + } + + let input_buffer = &mut self.input_buffer; + let d = &mut self.decompress; + + let mut chunks = Chunker::chain(input_buffer, data); + + let prev_len = chunks.len(); + let (ret, stream_end) = + match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; + let consumed = prev_len - chunks.len(); + + self.eof |= stream_end; + + if self.eof { + self.needs_input = false; + if !chunks.is_empty() { + self.unused_data = vm.ctx.new_bytes(chunks.to_vec()); + } + } else if chunks.is_empty() { + input_buffer.clear(); + self.needs_input = true; + } else { + self.needs_input = false; + if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { + input_buffer.clear(); + input_buffer.extend_from_slice(&data[n_consumed_from_data..]); + } else { + input_buffer.drain(..consumed); + input_buffer.extend_from_slice(data); + } + } + + ret.map_err(DecompressError::Decompress) + } +} + +pub enum DecompressError<E> { + Decompress(E), + Eof(EofError), +} + +impl<E> From<E> for DecompressError<E> { + fn from(err: E) -> Self { + Self::Decompress(err) + } +} + +pub struct EofError; + +impl ToPyException for EofError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_eof_error("End of stream already reached") + } +} diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs new file mode 100644 index 00000000000..1fc98edb5a7 --- /dev/null +++ b/crates/stdlib/src/contextvars.rs @@ -0,0 +1,655 @@ +pub(crate) use _contextvars::module_def; + +use crate::vm::PyRef; +use _contextvars::PyContext; +use core::cell::RefCell; + +thread_local! { + // TODO: Vec doesn't seem to match copy behavior + static CONTEXTS: RefCell<Vec<PyRef<PyContext>>> = RefCell::default(); +} + +#[pymodule] +mod _contextvars { + use crate::vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, + builtins::{PyGenericAlias, PyList, PyStrRef, PyType, PyTypeRef}, + class::StaticType, + common::{hash::PyHash, wtf8::Wtf8Buf}, + function::{ArgCallable, FuncArgs, OptionalArg}, + protocol::{PyMappingMethods, PySequenceMethods}, + types::{AsMapping, AsSequence, Constructor, Hashable, Iterable, Representable}, + }; + use core::{ + cell::{Cell, RefCell, UnsafeCell}, + sync::atomic::Ordering, + }; + use crossbeam_utils::atomic::AtomicCell; + use indexmap::IndexMap; + use rustpython_common::lock::LazyLock; + + // TODO: Real hamt implementation + type Hamt = IndexMap<PyRef<ContextVar>, PyObjectRef, ahash::RandomState>; + + #[pyclass(no_attr, name = "Hamt", module = "contextvars")] + #[derive(Debug, PyPayload)] + pub(crate) struct HamtObject { + hamt: RefCell<Hamt>, + } + + #[pyclass] + impl HamtObject {} + + impl Default for HamtObject { + fn default() -> Self { + Self { + hamt: RefCell::new(Hamt::default()), + } + } + } + + unsafe impl Sync for HamtObject {} + + #[derive(Debug)] + struct ContextInner { + idx: Cell<usize>, + vars: PyRef<HamtObject>, + // PyObject *ctx_weakreflist; + entered: Cell<bool>, + } + + unsafe impl Sync for ContextInner {} + + #[pyattr] + #[pyclass(name = "Context")] + #[derive(Debug, PyPayload)] + pub(crate) struct PyContext { + // not to confuse with vm::Context + inner: ContextInner, + } + + impl PyContext { + fn empty(vm: &VirtualMachine) -> Self { + Self { + inner: ContextInner { + idx: Cell::new(usize::MAX), + vars: HamtObject::default().into_ref(&vm.ctx), + entered: Cell::new(false), + }, + } + } + + fn borrow_vars(&self) -> impl core::ops::Deref<Target = Hamt> + '_ { + self.inner.vars.hamt.borrow() + } + + fn borrow_vars_mut(&self) -> impl core::ops::DerefMut<Target = Hamt> + '_ { + self.inner.vars.hamt.borrow_mut() + } + + fn enter(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + if zelf.inner.entered.get() { + let msg = format!( + "cannot enter context: {} is already entered", + zelf.as_object().repr(vm)? + ); + return Err(vm.new_runtime_error(msg)); + } + + super::CONTEXTS.with_borrow_mut(|ctxs| { + zelf.inner.idx.set(ctxs.len()); + ctxs.push(zelf.to_owned()); + }); + zelf.inner.entered.set(true); + + Ok(()) + } + + fn exit(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + if !zelf.inner.entered.get() { + let msg = format!( + "cannot exit context: {} is not entered", + zelf.as_object().repr(vm)? + ); + return Err(vm.new_runtime_error(msg)); + } + + super::CONTEXTS.with_borrow_mut(|ctxs| { + let err_msg = + "cannot exit context: thread state references a different context object"; + ctxs.pop_if(|ctx| ctx.get_id() == zelf.get_id()) + .map(drop) + .ok_or_else(|| vm.new_runtime_error(err_msg)) + })?; + zelf.inner.entered.set(false); + + Ok(()) + } + + fn current(vm: &VirtualMachine) -> PyRef<Self> { + super::CONTEXTS.with_borrow_mut(|ctxs| { + if let Some(ctx) = ctxs.last() { + ctx.clone() + } else { + let ctx = Self::empty(vm); + ctx.inner.idx.set(0); + ctx.inner.entered.set(true); + let ctx = ctx.into_ref(&vm.ctx); + ctxs.push(ctx); + ctxs[0].clone() + } + }) + } + + fn contains(&self, needle: &Py<ContextVar>) -> PyResult<bool> { + let vars = self.borrow_vars(); + Ok(vars.get(needle).is_some()) + } + + fn get_inner(&self, needle: &Py<ContextVar>) -> Option<PyObjectRef> { + let vars = self.borrow_vars(); + vars.get(needle).map(|o| o.to_owned()) + } + } + + #[pyclass(with(Constructor, AsMapping, AsSequence, Iterable))] + impl PyContext { + #[pymethod] + fn run( + zelf: &Py<Self>, + callable: ArgCallable, + args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult { + Self::enter(zelf, vm)?; + let result = callable.invoke(args, vm); + Self::exit(zelf, vm)?; + result + } + + #[pymethod] + fn copy(&self, vm: &VirtualMachine) -> Self { + // Deep copy the vars - clone the underlying Hamt data, not just the PyRef + let vars_copy = HamtObject { + hamt: RefCell::new(self.inner.vars.hamt.borrow().clone()), + }; + Self { + inner: ContextInner { + idx: Cell::new(usize::MAX), + vars: vars_copy.into_ref(&vm.ctx), + entered: Cell::new(false), + }, + } + } + + fn __getitem__( + &self, + var: PyRef<ContextVar>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let vars = self.borrow_vars(); + let item = vars + .get(&*var) + .ok_or_else(|| vm.new_key_error(var.into()))?; + Ok(item.to_owned()) + } + + fn __len__(&self) -> usize { + self.borrow_vars().len() + } + + #[pymethod] + fn get( + &self, + key: PyRef<ContextVar>, + default: OptionalArg<PyObjectRef>, + ) -> PyResult<Option<PyObjectRef>> { + let found = self.get_inner(&key); + let result = if let Some(found) = found { + Some(found.to_owned()) + } else { + default.into_option() + }; + Ok(result) + } + + // TODO: wrong return type + #[pymethod] + fn keys(zelf: &Py<Self>) -> Vec<PyObjectRef> { + let vars = zelf.borrow_vars(); + vars.keys().map(|key| key.to_owned().into()).collect() + } + + // TODO: wrong return type + #[pymethod] + fn values(zelf: PyRef<Self>) -> Vec<PyObjectRef> { + let vars = zelf.borrow_vars(); + vars.values().map(|value| value.to_owned()).collect() + } + + // TODO: wrong return type + #[pymethod] + fn items(zelf: PyRef<Self>, vm: &VirtualMachine) -> Vec<PyObjectRef> { + let vars = zelf.borrow_vars(); + vars.iter() + .map(|(k, v)| vm.ctx.new_tuple(vec![k.clone().into(), v.clone()]).into()) + .collect() + } + } + + impl Constructor for PyContext { + type Args = (); + fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self::empty(vm)) + } + } + + impl AsMapping for PyContext { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: PyMappingMethods = PyMappingMethods { + length: atomic_func!(|mapping, _vm| Ok( + PyContext::mapping_downcast(mapping).__len__() + )), + subscript: atomic_func!(|mapping, needle, vm| { + let needle = needle.try_to_value(vm)?; + let found = PyContext::mapping_downcast(mapping).get_inner(needle); + if let Some(found) = found { + Ok(found.to_owned()) + } else { + Err(vm.new_key_error(needle.to_owned().into())) + } + }), + ass_subscript: None, + }; + &AS_MAPPING + } + } + + impl AsSequence for PyContext { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + contains: atomic_func!(|seq, target, vm| { + let target = target.try_to_value(vm)?; + PyContext::sequence_downcast(seq).contains(target) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } + } + + impl Iterable for PyContext { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let vars = zelf.borrow_vars(); + let keys: Vec<PyObjectRef> = vars.keys().map(|k| k.clone().into()).collect(); + let list = vm.ctx.new_list(keys); + <PyList as Iterable>::iter(list, vm) + } + } + + #[pyattr] + #[pyclass(name, traverse)] + #[derive(PyPayload)] + struct ContextVar { + #[pytraverse(skip)] + name: String, + default: Option<PyObjectRef>, + #[pytraverse(skip)] + cached: AtomicCell<Option<ContextVarCache>>, + #[pytraverse(skip)] + cached_id: core::sync::atomic::AtomicUsize, // cached_tsid in CPython + #[pytraverse(skip)] + hash: UnsafeCell<PyHash>, + } + + impl core::fmt::Debug for ContextVar { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("ContextVar").finish() + } + } + + unsafe impl Sync for ContextVar {} + + impl PartialEq for ContextVar { + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self, other) + } + } + impl Eq for ContextVar {} + + #[derive(Debug)] + struct ContextVarCache { + object: PyObjectRef, // value; cached in CPython + idx: usize, // Context index; cached_tsver in CPython + } + + impl ContextVar { + fn delete(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + zelf.cached.store(None); + + let ctx = PyContext::current(vm); + + let mut vars = ctx.borrow_vars_mut(); + if vars.swap_remove(zelf).is_none() { + // TODO: + // PyErr_SetObject(PyExc_LookupError, (PyObject *)var); + let msg = zelf.as_object().repr(vm)?.as_wtf8().to_owned(); + return Err(vm.new_lookup_error(msg)); + } + + Ok(()) + } + + // contextvar_set in CPython + fn set_inner(zelf: &Py<Self>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let ctx = PyContext::current(vm); + + let mut vars = ctx.borrow_vars_mut(); + vars.insert(zelf.to_owned(), value.clone()); + + zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst); + + let cache = ContextVarCache { + object: value, + idx: ctx.inner.idx.get(), + }; + zelf.cached.store(Some(cache)); + + Ok(()) + } + + fn generate_hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyHash { + let name_hash = vm.state.hash_secret.hash_str(&zelf.name); + let pointer_hash = crate::common::hash::hash_pointer(zelf.as_object().get_id()); + pointer_hash ^ name_hash + } + } + + #[pyclass(with(Constructor, Hashable, Representable))] + impl ContextVar { + #[pygetset] + fn name(&self) -> String { + self.name.clone() + } + + #[pymethod] + fn get( + zelf: &Py<Self>, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + let found = super::CONTEXTS.with_borrow(|ctxs| { + let ctx = ctxs.last()?; + let cached_ptr = zelf.cached.as_ptr(); + debug_assert!(!cached_ptr.is_null()); + if let Some(cached) = unsafe { &*cached_ptr } + && zelf.cached_id.load(Ordering::SeqCst) == ctx.get_id() + && cached.idx + 1 == ctxs.len() + { + return Some(cached.object.clone()); + } + let vars = ctx.borrow_vars(); + let obj = vars.get(zelf)?; + zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst); + + // TODO: ensure cached is not changed + let _removed = zelf.cached.swap(Some(ContextVarCache { + object: obj.clone(), + idx: ctxs.len() - 1, + })); + + Some(obj.clone()) + }); + + let value = if let Some(value) = found { + value + } else if let Some(default) = default.into_option() { + default + } else if let Some(default) = zelf.default.as_ref() { + default.clone() + } else { + let msg = zelf.as_object().repr(vm)?; + return Err(vm.new_lookup_error(msg.as_wtf8().to_owned())); + }; + Ok(Some(value)) + } + + #[pymethod] + fn set( + zelf: &Py<Self>, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<ContextToken>> { + let ctx = PyContext::current(vm); + + let old_value = ctx.borrow_vars().get(zelf).map(|v| v.to_owned()); + let token = ContextToken { + ctx: ctx.to_owned(), + var: zelf.to_owned(), + old_value, + used: false.into(), + }; + + // ctx.vars borrow must be released + Self::set_inner(zelf, value, vm)?; + + Ok(token.into_ref(&vm.ctx)) + } + + #[pymethod] + fn reset(zelf: &Py<Self>, token: PyRef<ContextToken>, vm: &VirtualMachine) -> PyResult<()> { + if token.used.get() { + let msg = format!("{} has already been used once", token.as_object().repr(vm)?); + return Err(vm.new_runtime_error(msg)); + } + + if !zelf.is(&token.var) { + let msg = format!( + "{} was created by a different ContextVar", + token.var.as_object().repr(vm)? + ); + return Err(vm.new_value_error(msg)); + } + + let ctx = PyContext::current(vm); + if !ctx.is(&token.ctx) { + let msg = format!( + "{} was created in a different Context", + token.var.as_object().repr(vm)? + ); + return Err(vm.new_value_error(msg)); + } + + token.used.set(true); + + if let Some(old_value) = &token.old_value { + Self::set_inner(zelf, old_value.clone(), vm)?; + } else { + Self::delete(zelf, vm)?; + } + Ok(()) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + #[derive(FromArgs)] + struct ContextVarOptions { + #[pyarg(positional)] + #[allow(dead_code)] // TODO: RUSTPYTHON + name: PyStrRef, + #[pyarg(any, optional)] + #[allow(dead_code)] // TODO: RUSTPYTHON + default: OptionalArg<PyObjectRef>, + } + + impl Constructor for ContextVar { + type Args = ContextVarOptions; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let args: Self::Args = args.bind(vm)?; + let var = Self { + name: args.name.to_string(), + default: args.default.into_option(), + cached_id: 0.into(), + cached: AtomicCell::new(None), + hash: UnsafeCell::new(0), + }; + let py_var = var.into_ref_with_type(vm, cls)?; + + unsafe { + // SAFETY: py_var is not exposed to python memory model yet + *py_var.hash.get() = Self::generate_hash(&py_var, vm) + }; + Ok(py_var.into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + impl core::hash::Hash for ContextVar { + #[inline] + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { + unsafe { *self.hash.get() }.hash(state) + } + } + + impl Hashable for ContextVar { + #[inline] + fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyHash> { + Ok(unsafe { *zelf.hash.get() }) + } + } + + impl Representable for ContextVar { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + // unimplemented!("<ContextVar name={{}} default={{}} at {{}}") + Ok(format!( + "<ContextVar name={} default={:?} at {:#x}>", + zelf.name.as_str(), + zelf.default + .as_ref() + .and_then(|default| default.str(vm).ok()), + zelf.get_id() + )) + } + } + + #[pyattr] + #[pyclass(name = "Token")] + #[derive(Debug, PyPayload)] + struct ContextToken { + ctx: PyRef<PyContext>, // tok_ctx in CPython + var: PyRef<ContextVar>, // tok_var in CPython + old_value: Option<PyObjectRef>, // tok_oldval in CPython + used: Cell<bool>, + } + + unsafe impl Sync for ContextToken {} + + #[pyclass(with(Constructor, Representable))] + impl ContextToken { + #[pygetset] + fn var(&self, _vm: &VirtualMachine) -> PyRef<ContextVar> { + self.var.clone() + } + + #[pygetset] + fn old_value(&self, _vm: &VirtualMachine) -> PyObjectRef { + match &self.old_value { + Some(value) => value.clone(), + None => ContextTokenMissing::static_type().to_owned().into(), + } + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + #[pymethod] + fn __enter__(zelf: PyRef<Self>) -> PyRef<Self> { + zelf + } + + #[pymethod] + fn __exit__( + zelf: &Py<Self>, + _ty: PyObjectRef, + _val: PyObjectRef, + _tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + ContextVar::reset(&zelf.var, zelf.to_owned(), vm) + } + } + + impl Constructor for ContextToken { + type Args = FuncArgs; + + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_runtime_error("Tokens can only be created by ContextVars")) + } + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + impl Representable for ContextToken { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let used = if zelf.used.get() { " used" } else { "" }; + let var = Representable::repr_wtf8(&zelf.var, vm)?; + let ptr = zelf.as_object().get_id() as *const u8; + let mut result = Wtf8Buf::from(format!("<Token{used} var=")); + result.push_wtf8(&var); + result.push_str(&format!(" at {ptr:p}>")); + Ok(result) + } + } + + #[pyclass(no_attr, name = "Token.MISSING")] + #[derive(Debug, PyPayload)] + pub(super) struct ContextTokenMissing {} + + #[pyclass(with(Representable))] + impl ContextTokenMissing {} + + impl Representable for ContextTokenMissing { + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("<Token.MISSING>".to_owned()) + } + } + + #[pyfunction] + fn copy_context(vm: &VirtualMachine) -> PyContext { + PyContext::current(vm).copy(vm) + } + + // Set Token.MISSING attribute + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::vm::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + + let token_type = module.get_attr("Token", vm)?; + token_type.set_attr("MISSING", ContextTokenMissing::static_type().to_owned(), vm)?; + + Ok(()) + } +} diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs new file mode 100644 index 00000000000..a1147fd1cbb --- /dev/null +++ b/crates/stdlib/src/csv.rs @@ -0,0 +1,1146 @@ +pub(crate) use _csv::module_def; + +#[pymodule] +mod _csv { + use crate::common::lock::PyMutex; + use crate::vm::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, + builtins::{PyBaseExceptionRef, PyInt, PyNone, PyStr, PyType, PyTypeRef, PyUtf8StrRef}, + function::{ArgIterable, ArgumentError, FromArgs, FuncArgs, OptionalArg}, + protocol::{PyIter, PyIterReturn}, + raise_if_stop, + types::{Constructor, IterNext, Iterable, SelfIter}, + }; + use alloc::fmt; + use csv_core::Terminator; + use itertools::{self, Itertools}; + use parking_lot::Mutex; + use rustpython_common::{lock::LazyLock, wtf8::Wtf8Buf}; + use rustpython_vm::{match_class, sliceable::SliceableSequenceOp}; + use std::collections::HashMap; + + #[pyattr] + const QUOTE_MINIMAL: i32 = QuoteStyle::Minimal as i32; + #[pyattr] + const QUOTE_ALL: i32 = QuoteStyle::All as i32; + #[pyattr] + const QUOTE_NONNUMERIC: i32 = QuoteStyle::Nonnumeric as i32; + #[pyattr] + const QUOTE_NONE: i32 = QuoteStyle::None as i32; + #[pyattr] + const QUOTE_STRINGS: i32 = QuoteStyle::Strings as i32; + #[pyattr] + const QUOTE_NOTNULL: i32 = QuoteStyle::Notnull as i32; + #[pyattr(name = "__version__")] + const __VERSION__: &str = "1.0"; + + #[pyattr(name = "Error", once)] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_csv", + "Error", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + static GLOBAL_HASHMAP: LazyLock<Mutex<HashMap<String, PyDialect>>> = LazyLock::new(|| { + let m = HashMap::new(); + Mutex::new(m) + }); + static GLOBAL_FIELD_LIMIT: LazyLock<Mutex<isize>> = LazyLock::new(|| Mutex::new(131072)); + + fn new_csv_error(vm: &VirtualMachine, msg: impl Into<Wtf8Buf>) -> PyBaseExceptionRef { + vm.new_exception_msg(super::_csv::error(vm), msg.into()) + } + + #[pyattr] + #[pyclass(module = "csv", name = "Dialect")] + #[derive(Debug, PyPayload, Clone, Copy)] + struct PyDialect { + delimiter: u8, + quotechar: Option<u8>, + escapechar: Option<u8>, + doublequote: bool, + skipinitialspace: bool, + lineterminator: csv_core::Terminator, + quoting: QuoteStyle, + strict: bool, + } + impl Constructor for PyDialect { + type Args = PyObjectRef; + + fn py_new(_cls: &Py<PyType>, ctx: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Self::try_from_object(vm, ctx) + } + } + #[pyclass(with(Constructor))] + impl PyDialect { + #[pygetset] + fn delimiter(&self, vm: &VirtualMachine) -> PyRef<PyStr> { + vm.ctx.new_str(format!("{}", self.delimiter as char)) + } + #[pygetset] + fn quotechar(&self, vm: &VirtualMachine) -> Option<PyRef<PyStr>> { + Some(vm.ctx.new_str(format!("{}", self.quotechar? as char))) + } + #[pygetset] + const fn doublequote(&self) -> bool { + self.doublequote + } + #[pygetset] + const fn skipinitialspace(&self) -> bool { + self.skipinitialspace + } + #[pygetset] + fn lineterminator(&self, vm: &VirtualMachine) -> PyRef<PyStr> { + match self.lineterminator { + Terminator::CRLF => vm.ctx.new_str("\r\n".to_string()).to_owned(), + Terminator::Any(t) => vm.ctx.new_str(format!("{}", t as char)).to_owned(), + _ => unreachable!(), + } + } + #[pygetset] + fn quoting(&self) -> isize { + self.quoting.into() + } + #[pygetset] + fn escapechar(&self, vm: &VirtualMachine) -> Option<PyRef<PyStr>> { + Some(vm.ctx.new_str(format!("{}", self.escapechar? as char))) + } + #[pygetset(name = "strict")] + const fn get_strict(&self) -> bool { + self.strict + } + } + /// Parses the delimiter from a Python object and returns its ASCII value. + /// + /// This function attempts to extract the 'delimiter' attribute from the given Python object and ensures that the attribute is a single-character string. If successful, it returns the ASCII value of the character. If the attribute is not a single-character string, an error is returned. + /// + /// # Arguments + /// + /// * `vm` - A reference to the VirtualMachine, used for executing Python code and manipulating Python objects. + /// * `obj` - A reference to the PyObjectRef from which the 'delimiter' attribute is to be parsed. + /// + /// # Returns + /// + /// If successful, returns a `PyResult<u8>` representing the ASCII value of the 'delimiter' attribute. If unsuccessful, returns a `PyResult` containing an error message. + /// + /// # Errors + /// + /// This function can return the following errors: + /// + /// * If the 'delimiter' attribute is not a single-character string, a type error is returned. + /// * If the 'obj' is not of string type and does not have a 'delimiter' attribute, a type error is returned. + fn parse_delimiter_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<u8> { + if let Ok(attr) = obj.get_attr("delimiter", vm) { + parse_delimiter_from_obj(vm, &attr) + } else { + match_class!(match obj.to_owned() { + s @ PyStr => { + Ok(s.as_bytes().iter().copied().exactly_one().map_err(|_| { + vm.new_type_error(format!( + r#""delimiter" must be a unicode character, not a string of length {}"#, + s.len() + )) + })?) + } + attr => { + let msg = format!( + r#""delimiter" must be a unicode character, not {}"#, + attr.class() + ); + Err(vm.new_type_error(msg)) + } + }) + } + } + + fn parse_quotechar_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Option<u8>> { + match_class!(match obj.get_attr("quotechar", vm)? { + s @ PyStr => { + Ok(Some(s.as_bytes().iter().copied().exactly_one().map_err(|_| { + new_csv_error(vm, format!(r#""quotechar" must be a unicode character or None, not a string of length {}"#, s.len())) + })?)) + } + _n @ PyNone => { + Ok(None) + } + attr => { + Err(new_csv_error( + vm, + format!( + r#""quotechar" must be a unicode character or None, not {}"#, + attr.class() + ), + )) + } + }) + } + fn parse_escapechar_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Option<u8>> { + match_class!(match obj.get_attr("escapechar", vm)? { + s @ PyStr => { + Ok(Some(s.as_bytes().iter().copied().exactly_one().map_err(|_| { + new_csv_error( + vm, + format!(r#""escapechar" must be a unicode character or None, not a string of length {}"#, s.len()), + ) + })?)) + } + _n @ PyNone => { + Ok(None) + } + attr => { + let msg = format!( + r#""escapechar" must be a unicode character or None, not {}"#, + attr.class() + ); + Err(vm.new_type_error(msg.to_owned())) + } + }) + } + fn prase_lineterminator_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Terminator> { + match_class!(match obj.get_attr("lineterminator", vm)? { + s @ PyStr => { + Ok(if s.as_bytes().eq(b"\r\n") { + csv_core::Terminator::CRLF + } else if let Some(t) = s.as_bytes().first() { + // Due to limitations in the current implementation within csv_core + // the support for multiple characters in lineterminator is not complete. + // only capture the first character + csv_core::Terminator::Any(*t) + } else { + return Err(new_csv_error(vm, r#""lineterminator" must be a string"#)); + }) + } + attr => { + Err(vm.new_type_error(format!( + r#""lineterminator" must be a string, not {}"#, + attr.class() + ))) + } + }) + } + fn prase_quoting_from_obj(vm: &VirtualMachine, obj: &PyObject) -> PyResult<QuoteStyle> { + match_class!(match obj.get_attr("quoting", vm)? { + i @ PyInt => { + Ok(i.try_to_primitive::<isize>(vm)?.try_into().map_err(|_| { + let msg = r#"bad "quoting" value"#; + vm.new_type_error(msg.to_owned()) + })?) + } + attr => { + let msg = format!(r#""quoting" must be string or None, not {}"#, attr.class()); + Err(vm.new_type_error(msg.to_owned())) + } + }) + } + impl TryFromObject for PyDialect { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let delimiter = parse_delimiter_from_obj(vm, &obj)?; + let quotechar = parse_quotechar_from_obj(vm, &obj)?; + let escapechar = parse_escapechar_from_obj(vm, &obj)?; + let doublequote = obj.get_attr("doublequote", vm)?.try_to_bool(vm)?; + let skipinitialspace = obj.get_attr("skipinitialspace", vm)?.try_to_bool(vm)?; + let lineterminator = prase_lineterminator_from_obj(vm, &obj)?; + let quoting = prase_quoting_from_obj(vm, &obj)?; + let strict = if let Ok(t) = obj.get_attr("strict", vm) { + t.try_to_bool(vm).unwrap_or(false) + } else { + false + }; + + Ok(Self { + delimiter, + quotechar, + escapechar, + doublequote, + skipinitialspace, + lineterminator, + quoting, + strict, + }) + } + } + + #[pyfunction] + fn register_dialect( + name: PyObjectRef, + dialect: OptionalArg<PyObjectRef>, + opts: FormatOptions, + // TODO: handle quote style, etc + mut _rest: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + let name = name + .downcast::<PyStr>() + .map_err(|_| vm.new_type_error("argument 0 must be a string"))?; + let name: PyUtf8StrRef = name.try_into_utf8(vm)?; + let dialect = match dialect { + OptionalArg::Present(d) => PyDialect::try_from_object(vm, d) + .map_err(|_| vm.new_type_error("argument 1 must be a dialect object"))?, + OptionalArg::Missing => opts.result(vm)?, + }; + let dialect = opts.update_py_dialect(dialect); + GLOBAL_HASHMAP + .lock() + .insert(name.as_str().to_owned(), dialect); + Ok(()) + } + + #[pyfunction] + fn get_dialect( + name: PyObjectRef, + mut _rest: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<PyDialect> { + let name = name.downcast::<PyStr>().map_err(|obj| { + new_csv_error( + vm, + format!("argument 0 must be a string, not '{}'", obj.class()), + ) + })?; + let name: PyUtf8StrRef = name.try_into_utf8(vm)?; + let g = GLOBAL_HASHMAP.lock(); + if let Some(dialect) = g.get(name.as_str()) { + return Ok(*dialect); + } + Err(new_csv_error(vm, "unknown dialect")) + } + + #[pyfunction] + fn unregister_dialect( + name: PyObjectRef, + mut _rest: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + let name = name.downcast::<PyStr>().map_err(|obj| { + new_csv_error( + vm, + format!("argument 0 must be a string, not '{}'", obj.class()), + ) + })?; + let name: PyUtf8StrRef = name.try_into_utf8(vm)?; + let mut g = GLOBAL_HASHMAP.lock(); + if let Some(_removed) = g.remove(name.as_str()) { + return Ok(()); + } + Err(new_csv_error(vm, "unknown dialect")) + } + + #[pyfunction] + fn list_dialects( + rest: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<rustpython_vm::builtins::PyListRef> { + if !rest.args.is_empty() || !rest.kwargs.is_empty() { + return Err(vm.new_type_error("too many argument")); + } + let g = GLOBAL_HASHMAP.lock(); + let t = g + .keys() + .cloned() + .map(|x| vm.ctx.new_str(x).into()) + .collect_vec(); + // .iter().map(|x| vm.ctx.new_str(x.clone()).into_pyobject(vm)).collect_vec(); + Ok(vm.ctx.new_list(t)) + } + + #[pyfunction] + fn field_size_limit(rest: FuncArgs, vm: &VirtualMachine) -> PyResult<isize> { + let old_size = GLOBAL_FIELD_LIMIT.lock().to_owned(); + if !rest.args.is_empty() { + let arg_len = rest.args.len(); + if arg_len != 1 { + return Err(vm.new_type_error(format!( + "field_size_limit() takes at most 1 argument ({arg_len} given)" + ))); + } + let Ok(new_size) = rest.args.first().unwrap().try_int(vm) else { + return Err(vm.new_type_error("limit must be an integer")); + }; + *GLOBAL_FIELD_LIMIT.lock() = new_size.try_to_primitive::<isize>(vm)?; + } + Ok(old_size) + } + + #[pyfunction] + fn reader( + iter: PyIter, + options: FormatOptions, + // TODO: handle quote style, etc + _rest: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<Reader> { + Ok(Reader { + iter, + state: PyMutex::new(ReadState { + buffer: vec![0; 1024], + output_ends: vec![0; 16], + reader: options.to_reader(), + skipinitialspace: options.get_skipinitialspace(), + delimiter: options.get_delimiter(), + line_num: 0, + }), + dialect: options.result(vm)?, + }) + } + + #[pyfunction] + fn writer( + file: PyObjectRef, + options: FormatOptions, + // TODO: handle quote style, etc + _rest: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<Writer> { + let write = match vm.get_attribute_opt(file.clone(), "write")? { + Some(write_meth) => write_meth, + None if file.is_callable() => file, + None => { + return Err(vm.new_type_error("argument 1 must have a \"write\" method")); + } + }; + + Ok(Writer { + write, + state: PyMutex::new(WriteState { + buffer: vec![0; 1024], + writer: options.to_writer(), + }), + dialect: options.result(vm)?, + }) + } + + #[inline] + fn resize_buf<T: num_traits::PrimInt>(buf: &mut Vec<T>) { + let new_size = buf.len() * 2; + buf.resize(new_size, T::zero()); + } + + #[repr(i32)] + #[derive(Debug, Clone, Copy)] + pub enum QuoteStyle { + Minimal = 0, + All = 1, + Nonnumeric = 2, + None = 3, + Strings = 4, + Notnull = 5, + } + impl From<QuoteStyle> for csv_core::QuoteStyle { + fn from(val: QuoteStyle) -> Self { + match val { + QuoteStyle::Minimal => Self::Always, + QuoteStyle::All => Self::Always, + QuoteStyle::Nonnumeric => Self::NonNumeric, + QuoteStyle::None => Self::Never, + QuoteStyle::Strings => todo!(), + QuoteStyle::Notnull => todo!(), + } + } + } + impl TryFromObject for QuoteStyle { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let num = obj.try_int(vm)?.try_to_primitive::<isize>(vm)?; + num.try_into().map_err(|_| { + vm.new_value_error("can not convert to QuoteStyle enum from input argument") + }) + } + } + impl TryFrom<isize> for QuoteStyle { + type Error = (); + fn try_from(num: isize) -> Result<Self, Self::Error> { + match num { + 0 => Ok(Self::Minimal), + 1 => Ok(Self::All), + 2 => Ok(Self::Nonnumeric), + 3 => Ok(Self::None), + 4 => Ok(Self::Strings), + 5 => Ok(Self::Notnull), + _ => Err(()), + } + } + } + impl From<QuoteStyle> for isize { + fn from(val: QuoteStyle) -> Self { + match val { + QuoteStyle::Minimal => 0, + QuoteStyle::All => 1, + QuoteStyle::Nonnumeric => 2, + QuoteStyle::None => 3, + QuoteStyle::Strings => 4, + QuoteStyle::Notnull => 5, + } + } + } + + enum DialectItem { + Str(String), + Obj(PyDialect), + None, + } + + struct FormatOptions { + dialect: DialectItem, + delimiter: Option<u8>, + quotechar: Option<Option<u8>>, + escapechar: Option<u8>, + doublequote: Option<bool>, + skipinitialspace: Option<bool>, + lineterminator: Option<csv_core::Terminator>, + quoting: Option<QuoteStyle>, + strict: Option<bool>, + } + impl Default for FormatOptions { + fn default() -> Self { + Self { + dialect: DialectItem::None, + delimiter: None, + quotechar: None, + escapechar: None, + doublequote: None, + skipinitialspace: None, + lineterminator: None, + quoting: None, + strict: None, + } + } + } + /// prase a dialect item from a Python argument and returns a `DialectItem` or an `ArgumentError`. + /// + /// This function takes a reference to the VirtualMachine and a PyObjectRef as input and attempts to parse a dialect item from the provided Python argument. It returns a `DialectItem` if successful, or an `ArgumentError` if unsuccessful. + /// + /// # Arguments + /// + /// * `vm` - A reference to the VirtualMachine, used for executing Python code and manipulating Python objects. + /// * `obj` - The PyObjectRef from which the dialect item is to be parsed. + /// + /// # Returns + /// + /// If successful, returns a `Result<DialectItem, ArgumentError>` representing the parsed dialect item. If unsuccessful, returns an `ArgumentError`. + /// + /// # Errors + /// + /// This function can return the following errors: + /// + /// * If the provided object is a PyStr, it returns a `DialectItem::Str` containing the string value. + /// * If the provided object is PyNone, it returns an `ArgumentError` with the message "InvalidKeywordArgument('dialect')". + /// * If the provided object is a PyType, it attempts to create a PyDialect from the object and returns a `DialectItem::Obj` containing the PyDialect if successful. If unsuccessful, it returns an `ArgumentError` with the message "InvalidKeywordArgument('dialect')". + /// * If the provided object is none of the above types, it attempts to create a PyDialect from the object and returns a `DialectItem::Obj` containing the PyDialect if successful. If unsuccessful, it returns an `ArgumentError` with the message "InvalidKeywordArgument('dialect')". + fn prase_dialect_item_from_arg( + vm: &VirtualMachine, + obj: PyObjectRef, + ) -> Result<DialectItem, ArgumentError> { + match_class!(match obj { + s @ PyStr => { + let s = s.try_into_utf8(vm).map_err(ArgumentError::Exception)?; + Ok(DialectItem::Str(s.as_str().to_owned())) + } + PyNone => { + Err(ArgumentError::InvalidKeywordArgument("dialect".to_string())) + } + t @ PyType => { + let temp = t + .as_object() + .call(vec![], vm) + .map_err(|_e| ArgumentError::InvalidKeywordArgument("dialect".to_string()))?; + Ok(DialectItem::Obj( + PyDialect::try_from_object(vm, temp).map_err(|_| { + ArgumentError::InvalidKeywordArgument("dialect".to_string()) + })?, + )) + } + obj => { + if let Ok(cur_dialect_item) = PyDialect::try_from_object(vm, obj) { + Ok(DialectItem::Obj(cur_dialect_item)) + } else { + let msg = "dialect".to_string(); + Err(ArgumentError::InvalidKeywordArgument(msg)) + } + } + }) + } + + impl FromArgs for FormatOptions { + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { + let mut res = Self::default(); + if let Some(dialect) = args.kwargs.swap_remove("dialect") { + res.dialect = prase_dialect_item_from_arg(vm, dialect)?; + } else if let Some(dialect) = args.args.first() { + res.dialect = prase_dialect_item_from_arg(vm, dialect.clone())?; + } else { + res.dialect = DialectItem::None; + }; + + if let Some(delimiter) = args.kwargs.swap_remove("delimiter") { + res.delimiter = Some(parse_delimiter_from_obj(vm, &delimiter)?); + } + + if let Some(escapechar) = args.kwargs.swap_remove("escapechar") { + res.escapechar = match_class!(match escapechar { + s @ PyStr => + Some(s.as_bytes().iter().copied().exactly_one().map_err(|_| { + let msg = r#""escapechar" must be a 1-character string"#; + vm.new_type_error(msg.to_owned()) + })?), + _ => None, + }) + }; + if let Some(lineterminator) = args.kwargs.swap_remove("lineterminator") { + res.lineterminator = Some(csv_core::Terminator::Any( + lineterminator + .try_to_value::<&str>(vm)? + .bytes() + .exactly_one() + .map_err(|_| { + let msg = r#""lineterminator" must be a 1-character string"#; + vm.new_type_error(msg.to_owned()) + })?, + )) + }; + if let Some(doublequote) = args.kwargs.swap_remove("doublequote") { + res.doublequote = Some(doublequote.try_to_bool(vm).map_err(|_| { + let msg = r#""doublequote" must be a bool"#; + vm.new_type_error(msg.to_owned()) + })?) + }; + if let Some(skipinitialspace) = args.kwargs.swap_remove("skipinitialspace") { + res.skipinitialspace = Some(skipinitialspace.try_to_bool(vm).map_err(|_| { + let msg = r#""skipinitialspace" must be a bool"#; + vm.new_type_error(msg.to_owned()) + })?) + }; + if let Some(quoting) = args.kwargs.swap_remove("quoting") { + res.quoting = match_class!(match quoting { + i @ PyInt => + Some(i.try_to_primitive::<isize>(vm)?.try_into().map_err(|_e| { + ArgumentError::InvalidKeywordArgument("quoting".to_string()) + })?), + _ => { + // let msg = r#""quoting" must be a int enum"#; + return Err(ArgumentError::InvalidKeywordArgument("quoting".to_string())); + } + }); + }; + if let Some(quotechar) = args.kwargs.swap_remove("quotechar") { + res.quotechar = match_class!(match quotechar { + s @ PyStr => Some(Some(s.as_bytes().iter().copied().exactly_one().map_err( + |_| { + let msg = r#""quotechar" must be a 1-character string"#; + vm.new_type_error(msg.to_owned()) + } + )?)), + PyNone => { + if let Some(QuoteStyle::All) = res.quoting { + let msg = "quotechar must be set if quoting enabled"; + return Err(ArgumentError::Exception( + vm.new_type_error(msg.to_owned()), + )); + } + Some(None) + } + _o => { + let msg = r#"quotechar"#; + return Err( + rustpython_vm::function::ArgumentError::InvalidKeywordArgument( + msg.to_string(), + ), + ); + } + }) + }; + if let Some(strict) = args.kwargs.swap_remove("strict") { + res.strict = Some(strict.try_to_bool(vm).map_err(|_| { + let msg = r#""strict" must be a int enum"#; + vm.new_type_error(msg.to_owned()) + })?) + }; + + if let Some(last_arg) = args.kwargs.pop() { + let msg = format!( + r#"'{}' is an invalid keyword argument for this function"#, + last_arg.0 + ); + return Err(rustpython_vm::function::ArgumentError::InvalidKeywordArgument(msg)); + } + Ok(res) + } + } + + impl FormatOptions { + const fn update_py_dialect(&self, mut res: PyDialect) -> PyDialect { + macro_rules! check_and_fill { + ($res:ident, $e:ident) => {{ + if let Some(t) = self.$e { + $res.$e = t; + } + }}; + } + check_and_fill!(res, delimiter); + // check_and_fill!(res, quotechar); + check_and_fill!(res, delimiter); + check_and_fill!(res, doublequote); + check_and_fill!(res, skipinitialspace); + if let Some(t) = self.escapechar { + res.escapechar = Some(t); + }; + if let Some(t) = self.quotechar { + if let Some(u) = t { + res.quotechar = Some(u); + } else { + res.quotechar = None; + } + }; + check_and_fill!(res, quoting); + check_and_fill!(res, lineterminator); + check_and_fill!(res, strict); + res + } + + fn result(&self, vm: &VirtualMachine) -> PyResult<PyDialect> { + match &self.dialect { + DialectItem::Str(name) => { + let g = GLOBAL_HASHMAP.lock(); + if let Some(dialect) = g.get(name) { + Ok(self.update_py_dialect(*dialect)) + } else { + Err(new_csv_error(vm, format!("{name} is not registered."))) + } + // TODO + // Maybe need to update the obj from HashMap + } + DialectItem::Obj(o) => Ok(self.update_py_dialect(*o)), + DialectItem::None => { + let g = GLOBAL_HASHMAP.lock(); + let res = *g.get("excel").unwrap(); + Ok(self.update_py_dialect(res)) + } + } + } + fn get_skipinitialspace(&self) -> bool { + let mut skipinitialspace = match &self.dialect { + DialectItem::Str(name) => { + let g = GLOBAL_HASHMAP.lock(); + if let Some(dialect) = g.get(name) { + dialect.skipinitialspace + // RustPython todo + // todo! Perfecting the remaining attributes. + } else { + false + } + } + DialectItem::Obj(obj) => obj.skipinitialspace, + _ => false, + }; + if let Some(attr) = self.skipinitialspace { + skipinitialspace = attr + } + skipinitialspace + } + fn get_delimiter(&self) -> u8 { + let mut delimiter = match &self.dialect { + DialectItem::Str(name) => { + let g = GLOBAL_HASHMAP.lock(); + if let Some(dialect) = g.get(name) { + dialect.delimiter + // RustPython todo + // todo! Perfecting the remaining attributes. + } else { + b',' + } + } + DialectItem::Obj(obj) => obj.delimiter, + _ => b',', + }; + if let Some(attr) = self.delimiter { + delimiter = attr + } + delimiter + } + fn to_reader(&self) -> csv_core::Reader { + let mut builder = csv_core::ReaderBuilder::new(); + let mut reader = match &self.dialect { + DialectItem::Str(name) => { + let g = GLOBAL_HASHMAP.lock(); + if let Some(dialect) = g.get(name) { + let mut builder = builder + .delimiter(dialect.delimiter) + .double_quote(dialect.doublequote); + if let Some(t) = dialect.quotechar { + builder = builder.quote(t); + } + builder + // RustPython todo + // todo! Perfecting the remaining attributes. + } else { + &mut builder + } + } + DialectItem::Obj(obj) => { + let mut builder = builder + .delimiter(obj.delimiter) + .double_quote(obj.doublequote); + if let Some(t) = obj.quotechar { + builder = builder.quote(t); + } + builder + } + _ => { + let name = "excel"; + let g = GLOBAL_HASHMAP.lock(); + let dialect = g.get(name).unwrap(); + let mut builder = builder + .delimiter(dialect.delimiter) + .double_quote(dialect.doublequote); + if let Some(quotechar) = dialect.quotechar { + builder = builder.quote(quotechar); + } + builder + } + }; + + if let Some(t) = self.delimiter { + reader = reader.delimiter(t); + } + if let Some(t) = self.quotechar { + if let Some(u) = t { + reader = reader.quote(u); + } else { + reader = reader.quoting(false); + } + } else { + match self.quoting { + Some(QuoteStyle::None) => { + reader = reader.quoting(false); + } + // None => reader = reader.quoting(true), + _ => reader = reader.quoting(true), + } + } + + if let Some(t) = self.lineterminator { + reader = reader.terminator(t); + } + if let Some(t) = self.doublequote { + reader = reader.double_quote(t); + } + if self.escapechar.is_some() { + reader = reader.escape(self.escapechar); + } + reader = match self.lineterminator { + Some(u) => reader.terminator(u), + None => reader.terminator(Terminator::CRLF), + }; + reader.build() + } + fn to_writer(&self) -> csv_core::Writer { + let mut builder = csv_core::WriterBuilder::new(); + let mut writer = match &self.dialect { + DialectItem::Str(name) => { + let g = GLOBAL_HASHMAP.lock(); + if let Some(dialect) = g.get(name) { + let mut builder = builder + .delimiter(dialect.delimiter) + .double_quote(dialect.doublequote) + .terminator(dialect.lineterminator); + if let Some(t) = dialect.quotechar { + builder = builder.quote(t); + } + builder + + // RustPython todo + // todo! Perfecting the remaining attributes. + } else { + &mut builder + } + } + DialectItem::Obj(obj) => { + let mut builder = builder + .delimiter(obj.delimiter) + .double_quote(obj.doublequote) + .terminator(obj.lineterminator); + if let Some(t) = obj.quotechar { + builder = builder.quote(t); + } + builder + } + _ => &mut builder, + }; + if let Some(t) = self.delimiter { + writer = writer.delimiter(t); + } + if let Some(t) = self.quotechar { + if let Some(u) = t { + writer = writer.quote(u); + } else { + todo!() + } + } + if let Some(t) = self.doublequote { + writer = writer.double_quote(t); + } + writer = match self.lineterminator { + Some(u) => writer.terminator(u), + None => writer.terminator(Terminator::CRLF), + }; + if let Some(e) = self.escapechar { + writer = writer.escape(e); + } + if let Some(e) = self.quoting { + writer = writer.quote_style(e.into()); + } + writer.build() + } + } + + struct ReadState { + buffer: Vec<u8>, + output_ends: Vec<usize>, + reader: csv_core::Reader, + skipinitialspace: bool, + delimiter: u8, + line_num: u64, + } + + #[pyclass(no_attr, module = "_csv", name = "reader", traverse)] + #[derive(PyPayload)] + pub(super) struct Reader { + iter: PyIter, + #[pytraverse(skip)] + state: PyMutex<ReadState>, + #[pytraverse(skip)] + dialect: PyDialect, + } + + impl fmt::Debug for Reader { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "_csv.reader") + } + } + + #[pyclass(with(IterNext, Iterable), flags(DISALLOW_INSTANTIATION))] + impl Reader { + #[pygetset] + fn line_num(&self) -> u64 { + self.state.lock().line_num + } + #[pygetset] + const fn dialect(&self, _vm: &VirtualMachine) -> PyDialect { + self.dialect + } + } + impl SelfIter for Reader {} + impl IterNext for Reader { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let string = raise_if_stop!(zelf.iter.next(vm)?); + let string = string.downcast::<PyStr>().map_err(|obj| { + new_csv_error( + vm, + format!( + "iterator should return strings, not {} (the file should be opened in text mode)", + obj.class().name() + ), + ) + })?; + let input = string.as_bytes(); + if input.is_empty() || input.starts_with(b"\n") { + return Ok(PyIterReturn::Return(vm.ctx.new_list(vec![]).into())); + } + let mut state = zelf.state.lock(); + let ReadState { + buffer, + output_ends, + reader, + skipinitialspace, + delimiter, + line_num, + } = &mut *state; + + let mut input_offset = 0; + let mut output_offset = 0; + let mut output_ends_offset = 0; + let field_limit = GLOBAL_FIELD_LIMIT.lock().to_owned(); + #[inline] + fn trim_spaces(input: &[u8]) -> &[u8] { + let trimmed_start = input.iter().position(|&x| x != b' ').unwrap_or(input.len()); + let trimmed_end = input + .iter() + .rposition(|&x| x != b' ') + .map(|i| i + 1) + .unwrap_or(0); + &input[trimmed_start..trimmed_end] + } + let input = if *skipinitialspace { + let t = input.split(|x| x == delimiter); + t.map(|x| { + let trimmed = trim_spaces(x); + String::from_utf8(trimmed.to_vec()).unwrap() + }) + .join(format!("{}", *delimiter as char).as_str()) + } else { + String::from_utf8(input.to_vec()).unwrap() + }; + loop { + let (res, n_read, n_written, n_ends) = reader.read_record( + &input.as_bytes()[input_offset..], + &mut buffer[output_offset..], + &mut output_ends[output_ends_offset..], + ); + input_offset += n_read; + output_offset += n_written; + output_ends_offset += n_ends; + match res { + csv_core::ReadRecordResult::InputEmpty => {} + csv_core::ReadRecordResult::OutputFull => resize_buf(buffer), + csv_core::ReadRecordResult::OutputEndsFull => resize_buf(output_ends), + csv_core::ReadRecordResult::Record => break, + csv_core::ReadRecordResult::End => { + return Ok(PyIterReturn::StopIteration(None)); + } + } + } + let rest = &input.as_bytes()[input_offset..]; + if !rest.iter().all(|&c| matches!(c, b'\r' | b'\n')) { + return Err(new_csv_error( + vm, + "new-line character seen in unquoted field - \ + do you need to open the file in universal-newline mode?" + .to_owned(), + )); + } + + let mut prev_end = 0; + let out: Vec<PyObjectRef> = output_ends[..output_ends_offset] + .iter() + .map(|&end| { + let range = prev_end..end; + if range.len() > field_limit as usize { + return Err(new_csv_error(vm, "filed too long to read".to_string())); + } + prev_end = end; + let s = core::str::from_utf8(&buffer[range.clone()]) + // not sure if this is possible - the input was all strings + .map_err(|_e| vm.new_unicode_decode_error("csv not utf8"))?; + // Rustpython TODO! + // Incomplete implementation + if let QuoteStyle::Nonnumeric = zelf.dialect.quoting { + if let Ok(t) = + String::from_utf8(trim_spaces(&buffer[range.clone()]).to_vec()) + .unwrap() + .parse::<i64>() + { + Ok(vm.ctx.new_int(t).into()) + } else { + Ok(vm.ctx.new_str(s).into()) + } + } else { + Ok(vm.ctx.new_str(s).into()) + } + }) + .collect::<Result<_, _>>()?; + // Removes the last null item before the line terminator, if there is a separator before the line terminator, + // todo! + // if out.last().unwrap().length(vm).unwrap() == 0 { + // out.pop(); + // } + *line_num += 1; + Ok(PyIterReturn::Return(vm.ctx.new_list(out).into())) + } + } + + struct WriteState { + buffer: Vec<u8>, + writer: csv_core::Writer, + } + + #[pyclass(no_attr, module = "_csv", name = "writer", traverse)] + #[derive(PyPayload)] + pub(super) struct Writer { + write: PyObjectRef, + #[pytraverse(skip)] + state: PyMutex<WriteState>, + #[pytraverse(skip)] + dialect: PyDialect, + } + + impl fmt::Debug for Writer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "_csv.writer") + } + } + + #[pyclass(flags(DISALLOW_INSTANTIATION))] + impl Writer { + #[pygetset(name = "dialect")] + const fn get_dialect(&self, _vm: &VirtualMachine) -> PyDialect { + self.dialect + } + #[pymethod] + fn writerow(&self, row: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let mut state = self.state.lock(); + let WriteState { buffer, writer } = &mut *state; + + let mut buffer_offset = 0; + + macro_rules! handle_res { + ($x:expr) => {{ + let (res, n_written) = $x; + buffer_offset += n_written; + match res { + csv_core::WriteResult::InputEmpty => break, + csv_core::WriteResult::OutputFull => resize_buf(buffer), + } + }}; + } + + let row = ArgIterable::try_from_object(vm, row.clone()).map_err(|_e| { + new_csv_error(vm, format!("\'{}\' object is not iterable", row.class())) + })?; + let mut first_flag = true; + for field in row.iter(vm)? { + let field: PyObjectRef = field?; + let stringified; + let data: &[u8] = match_class!(match field { + ref s @ PyStr => s.as_bytes(), + crate::builtins::PyNone => b"", + ref obj => { + stringified = obj.str(vm)?; + stringified.as_bytes() + } + }); + let mut input_offset = 0; + if first_flag { + first_flag = false; + } else { + loop { + handle_res!(writer.delimiter(&mut buffer[buffer_offset..])); + } + } + + loop { + let (res, n_read, n_written) = + writer.field(&data[input_offset..], &mut buffer[buffer_offset..]); + input_offset += n_read; + handle_res!((res, n_written)); + } + } + + loop { + handle_res!(writer.terminator(&mut buffer[buffer_offset..])); + } + let s = core::str::from_utf8(&buffer[..buffer_offset]) + .map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?; + + self.write.call((s,), vm) + } + + #[pymethod] + fn writerows(&self, rows: ArgIterable, vm: &VirtualMachine) -> PyResult<()> { + for row in rows.iter(vm)? { + self.writerow(row?, vm)?; + } + Ok(()) + } + } +} diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs new file mode 100644 index 00000000000..9c4373c312e --- /dev/null +++ b/crates/stdlib/src/faulthandler.rs @@ -0,0 +1,1333 @@ +pub(crate) use decl::module_def; + +#[allow(static_mut_refs)] // TODO: group code only with static mut refs +#[pymodule(name = "faulthandler")] +mod decl { + use crate::vm::{ + PyObjectRef, PyResult, VirtualMachine, + frame::Frame, + function::{ArgIntoFloat, OptionalArg}, + }; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicBool, AtomicI32, Ordering}; + use core::time::Duration; + use parking_lot::{Condvar, Mutex}; + #[cfg(any(unix, windows))] + use rustpython_common::os::{get_errno, set_errno}; + use std::thread; + + /// fault_handler_t + #[cfg(unix)] + struct FaultHandler { + signum: libc::c_int, + enabled: bool, + name: &'static str, + previous: libc::sigaction, + } + + #[cfg(windows)] + struct FaultHandler { + signum: libc::c_int, + enabled: bool, + name: &'static str, + previous: libc::sighandler_t, + } + + #[cfg(unix)] + impl FaultHandler { + const fn new(signum: libc::c_int, name: &'static str) -> Self { + Self { + signum, + enabled: false, + name, + // SAFETY: sigaction is a C struct that can be zero-initialized + previous: unsafe { core::mem::zeroed() }, + } + } + } + + #[cfg(windows)] + impl FaultHandler { + const fn new(signum: libc::c_int, name: &'static str) -> Self { + Self { + signum, + enabled: false, + name, + previous: 0, + } + } + } + + /// faulthandler_handlers[] + /// Number of fatal signals + #[cfg(unix)] + const FAULTHANDLER_NSIGNALS: usize = 5; + #[cfg(windows)] + const FAULTHANDLER_NSIGNALS: usize = 4; + + // Signal handlers use mutable statics matching faulthandler.c implementation. + #[cfg(unix)] + static mut FAULTHANDLER_HANDLERS: [FaultHandler; FAULTHANDLER_NSIGNALS] = [ + FaultHandler::new(libc::SIGBUS, "Bus error"), + FaultHandler::new(libc::SIGILL, "Illegal instruction"), + FaultHandler::new(libc::SIGFPE, "Floating-point exception"), + FaultHandler::new(libc::SIGABRT, "Aborted"), + FaultHandler::new(libc::SIGSEGV, "Segmentation fault"), + ]; + + #[cfg(windows)] + static mut FAULTHANDLER_HANDLERS: [FaultHandler; FAULTHANDLER_NSIGNALS] = [ + FaultHandler::new(libc::SIGILL, "Illegal instruction"), + FaultHandler::new(libc::SIGFPE, "Floating-point exception"), + FaultHandler::new(libc::SIGABRT, "Aborted"), + FaultHandler::new(libc::SIGSEGV, "Segmentation fault"), + ]; + + /// fatal_error state + struct FatalErrorState { + enabled: AtomicBool, + fd: AtomicI32, + all_threads: AtomicBool, + } + + static FATAL_ERROR: FatalErrorState = FatalErrorState { + enabled: AtomicBool::new(false), + fd: AtomicI32::new(2), // stderr by default + all_threads: AtomicBool::new(true), + }; + + #[cfg(feature = "threading")] + type ThreadFrameSlot = Arc<rustpython_vm::vm::thread::ThreadSlot>; + + // Watchdog thread state for dump_traceback_later + struct WatchdogState { + cancel: bool, + fd: i32, + timeout_us: u64, + repeat: bool, + exit: bool, + header: String, + #[cfg(feature = "threading")] + thread_frame_slots: Vec<(u64, ThreadFrameSlot)>, + } + + type WatchdogHandle = Arc<(Mutex<WatchdogState>, Condvar)>; + static WATCHDOG: Mutex<Option<WatchdogHandle>> = Mutex::new(None); + + // Signal-safe output functions + + // PUTS macro + #[cfg(any(unix, windows))] + fn puts(fd: i32, s: &str) { + let _ = unsafe { + #[cfg(windows)] + { + libc::write(fd, s.as_ptr() as *const libc::c_void, s.len() as u32) + } + #[cfg(not(windows))] + { + libc::write(fd, s.as_ptr() as *const libc::c_void, s.len()) + } + }; + } + + #[cfg(any(unix, windows))] + fn puts_bytes(fd: i32, s: &[u8]) { + let _ = unsafe { + #[cfg(windows)] + { + libc::write(fd, s.as_ptr() as *const libc::c_void, s.len() as u32) + } + #[cfg(not(windows))] + { + libc::write(fd, s.as_ptr() as *const libc::c_void, s.len()) + } + }; + } + + // _Py_DumpHexadecimal (traceback.c) + #[cfg(any(unix, windows))] + fn dump_hexadecimal(fd: i32, value: u64, width: usize) { + const HEX_CHARS: &[u8; 16] = b"0123456789abcdef"; + let mut buf = [0u8; 18]; // "0x" + 16 hex digits + buf[0] = b'0'; + buf[1] = b'x'; + + for i in 0..width { + let digit = ((value >> (4 * (width - 1 - i))) & 0xf) as usize; + buf[2 + i] = HEX_CHARS[digit]; + } + + let _ = unsafe { + #[cfg(windows)] + { + libc::write(fd, buf.as_ptr() as *const libc::c_void, (2 + width) as u32) + } + #[cfg(not(windows))] + { + libc::write(fd, buf.as_ptr() as *const libc::c_void, 2 + width) + } + }; + } + + // _Py_DumpDecimal (traceback.c) + #[cfg(any(unix, windows))] + fn dump_decimal(fd: i32, value: usize) { + let mut buf = [0u8; 20]; + let mut v = value; + let mut i = buf.len(); + + if v == 0 { + puts(fd, "0"); + return; + } + + while v > 0 { + i -= 1; + buf[i] = b'0' + (v % 10) as u8; + v /= 10; + } + + let len = buf.len() - i; + let _ = unsafe { + #[cfg(windows)] + { + libc::write(fd, buf[i..].as_ptr() as *const libc::c_void, len as u32) + } + #[cfg(not(windows))] + { + libc::write(fd, buf[i..].as_ptr() as *const libc::c_void, len) + } + }; + } + + /// Get current thread ID + #[cfg(unix)] + fn current_thread_id() -> u64 { + unsafe { libc::pthread_self() as u64 } + } + + #[cfg(windows)] + fn current_thread_id() -> u64 { + unsafe { windows_sys::Win32::System::Threading::GetCurrentThreadId() as u64 } + } + + // write_thread_id (traceback.c:1240-1256) + #[cfg(any(unix, windows))] + fn write_thread_id(fd: i32, thread_id: u64, is_current: bool) { + if is_current { + puts(fd, "Current thread "); + } else { + puts(fd, "Thread "); + } + dump_hexadecimal(fd, thread_id, core::mem::size_of::<usize>() * 2); + puts(fd, " (most recent call first):\n"); + } + + /// Dump the current thread's live frame chain to fd (signal-safe). + /// Walks the `Frame.previous` pointer chain starting from the + /// thread-local current frame pointer. + #[cfg(any(unix, windows))] + fn dump_live_frames(fd: i32) { + const MAX_FRAME_DEPTH: usize = 100; + + let mut frame_ptr = crate::vm::vm::thread::get_current_frame(); + if frame_ptr.is_null() { + puts(fd, " <no Python frame>\n"); + return; + } + let mut depth = 0; + while !frame_ptr.is_null() && depth < MAX_FRAME_DEPTH { + let frame = unsafe { &*frame_ptr }; + dump_frame_from_raw(fd, frame); + frame_ptr = frame.previous_frame(); + depth += 1; + } + if depth >= MAX_FRAME_DEPTH && !frame_ptr.is_null() { + puts(fd, " ...\n"); + } + } + + /// Dump a single frame's info to fd (signal-safe), reading live data. + #[cfg(any(unix, windows))] + fn dump_frame_from_raw(fd: i32, frame: &Frame) { + let filename = frame.code.source_path().as_str(); + let funcname = frame.code.obj_name.as_str(); + let lasti = frame.lasti(); + let lineno = if lasti == 0 { + frame.code.first_line_number.map(|n| n.get()).unwrap_or(1) as u32 + } else { + let idx = (lasti as usize).saturating_sub(1); + if idx < frame.code.locations.len() { + frame.code.locations[idx].0.line.get() as u32 + } else { + frame.code.first_line_number.map(|n| n.get()).unwrap_or(0) as u32 + } + }; + + puts(fd, " File \""); + dump_ascii(fd, filename); + puts(fd, "\", line "); + dump_decimal(fd, lineno as usize); + puts(fd, " in "); + dump_ascii(fd, funcname); + puts(fd, "\n"); + } + + // faulthandler_dump_traceback (signal-safe, for fatal errors) + #[cfg(any(unix, windows))] + fn faulthandler_dump_traceback(fd: i32, all_threads: bool) { + static REENTRANT: AtomicBool = AtomicBool::new(false); + + if REENTRANT.swap(true, Ordering::SeqCst) { + return; + } + + // Write thread header + if all_threads { + write_thread_id(fd, current_thread_id(), true); + } else { + puts(fd, "Stack (most recent call first):\n"); + } + + dump_live_frames(fd); + + REENTRANT.store(false, Ordering::SeqCst); + } + + /// MAX_STRING_LENGTH in traceback.c + const MAX_STRING_LENGTH: usize = 500; + + /// Truncate a UTF-8 string to at most `max_bytes` without splitting a + /// multi-byte codepoint. Signal-safe (no allocation, no panic). + #[cfg(any(unix, windows))] + fn safe_truncate(s: &str, max_bytes: usize) -> (&str, bool) { + if s.len() <= max_bytes { + return (s, false); + } + let mut end = max_bytes; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + (&s[..end], true) + } + + /// Write a string to fd, truncating with "..." if it exceeds MAX_STRING_LENGTH. + /// Mirrors `_Py_DumpASCII` truncation behavior. + #[cfg(any(unix, windows))] + fn dump_ascii(fd: i32, s: &str) { + let (truncated_s, was_truncated) = safe_truncate(s, MAX_STRING_LENGTH); + puts(fd, truncated_s); + if was_truncated { + puts(fd, "..."); + } + } + + /// Write a frame's info to an fd using signal-safe I/O. + #[cfg(any(unix, windows))] + fn dump_frame_from_ref(fd: i32, frame: &crate::vm::Py<Frame>) { + let funcname = frame.code.obj_name.as_str(); + let filename = frame.code.source_path().as_str(); + let lineno = if frame.lasti() == 0 { + frame.code.first_line_number.map(|n| n.get()).unwrap_or(1) as u32 + } else { + frame.current_location().line.get() as u32 + }; + + puts(fd, " File \""); + dump_ascii(fd, filename); + puts(fd, "\", line "); + dump_decimal(fd, lineno as usize); + puts(fd, " in "); + dump_ascii(fd, funcname); + puts(fd, "\n"); + } + + /// Dump traceback for a thread given its frame stack (for cross-thread dumping). + /// # Safety + /// Each `FramePtr` must point to a live frame (caller holds the Mutex). + #[cfg(all(any(unix, windows), feature = "threading"))] + fn dump_traceback_thread_frames( + fd: i32, + thread_id: u64, + is_current: bool, + frames: &[rustpython_vm::vm::FramePtr], + ) { + write_thread_id(fd, thread_id, is_current); + + if frames.is_empty() { + puts(fd, " <no Python frame>\n"); + } else { + for fp in frames.iter().rev() { + // SAFETY: caller holds the Mutex, so the owning thread can't pop. + dump_frame_from_ref(fd, unsafe { fp.as_ref() }); + } + } + } + + #[derive(FromArgs)] + struct DumpTracebackArgs { + #[pyarg(any, default)] + file: OptionalArg<PyObjectRef>, + #[pyarg(any, default = true)] + all_threads: bool, + } + + #[pyfunction] + fn dump_traceback(args: DumpTracebackArgs, vm: &VirtualMachine) -> PyResult<()> { + let fd = get_fd_from_file_opt(args.file, vm)?; + + #[cfg(any(unix, windows))] + { + if args.all_threads { + dump_all_threads(fd, vm); + } else { + puts(fd, "Stack (most recent call first):\n"); + let frames = vm.frames.borrow(); + for fp in frames.iter().rev() { + // SAFETY: the frame is alive while it's in the Vec + dump_frame_from_ref(fd, unsafe { fp.as_ref() }); + } + } + } + + #[cfg(not(any(unix, windows)))] + { + let _ = (fd, args.all_threads); + } + + Ok(()) + } + + /// Dump tracebacks of all threads. + #[cfg(any(unix, windows))] + fn dump_all_threads(fd: i32, vm: &VirtualMachine) { + // Get all threads' frame stacks from the shared registry + #[cfg(feature = "threading")] + { + let current_tid = rustpython_vm::stdlib::_thread::get_ident(); + let registry = vm.state.thread_frames.lock(); + + // First dump non-current threads, then current thread last + for (&tid, slot) in registry.iter() { + if tid == current_tid { + continue; + } + let frames_guard = slot.frames.lock(); + dump_traceback_thread_frames(fd, tid, false, &frames_guard); + puts(fd, "\n"); + } + + // Now dump current thread (use vm.frames for most up-to-date data) + write_thread_id(fd, current_tid, true); + let frames = vm.frames.borrow(); + if frames.is_empty() { + puts(fd, " <no Python frame>\n"); + } else { + for fp in frames.iter().rev() { + dump_frame_from_ref(fd, unsafe { fp.as_ref() }); + } + } + } + + #[cfg(not(feature = "threading"))] + { + write_thread_id(fd, current_thread_id(), true); + let frames = vm.frames.borrow(); + for fp in frames.iter().rev() { + dump_frame_from_ref(fd, unsafe { fp.as_ref() }); + } + } + } + + #[derive(FromArgs)] + #[allow(unused)] + struct EnableArgs { + #[pyarg(any, default)] + file: OptionalArg<PyObjectRef>, + #[pyarg(any, default = true)] + all_threads: bool, + } + + // faulthandler_py_enable + #[pyfunction] + fn enable(args: EnableArgs, vm: &VirtualMachine) -> PyResult<()> { + // Get file descriptor + let fd = get_fd_from_file_opt(args.file, vm)?; + + // Store fd and all_threads in global state + FATAL_ERROR.fd.store(fd, Ordering::Relaxed); + FATAL_ERROR + .all_threads + .store(args.all_threads, Ordering::Relaxed); + + // Install signal handlers + if !faulthandler_enable_internal() { + return Err(vm.new_runtime_error("Failed to enable faulthandler")); + } + + Ok(()) + } + + // Signal handlers + + /// faulthandler_disable_fatal_handler (faulthandler.c:310-321) + #[cfg(unix)] + unsafe fn faulthandler_disable_fatal_handler(handler: &mut FaultHandler) { + if !handler.enabled { + return; + } + handler.enabled = false; + unsafe { + libc::sigaction(handler.signum, &handler.previous, core::ptr::null_mut()); + } + } + + #[cfg(windows)] + unsafe fn faulthandler_disable_fatal_handler(handler: &mut FaultHandler) { + if !handler.enabled { + return; + } + handler.enabled = false; + unsafe { + libc::signal(handler.signum, handler.previous); + } + } + + // faulthandler_fatal_error + #[cfg(unix)] + extern "C" fn faulthandler_fatal_error(signum: libc::c_int) { + let save_errno = get_errno(); + + if !FATAL_ERROR.enabled.load(Ordering::Relaxed) { + return; + } + + let fd = FATAL_ERROR.fd.load(Ordering::Relaxed); + + let handler = unsafe { + FAULTHANDLER_HANDLERS + .iter_mut() + .find(|h| h.signum == signum) + }; + + if let Some(h) = handler { + // Disable handler (restores previous) + unsafe { + faulthandler_disable_fatal_handler(h); + } + + puts(fd, "Fatal Python error: "); + puts(fd, h.name); + puts(fd, "\n\n"); + } else { + puts(fd, "Fatal Python error from unexpected signum: "); + dump_decimal(fd, signum as usize); + puts(fd, "\n\n"); + } + + let all_threads = FATAL_ERROR.all_threads.load(Ordering::Relaxed); + faulthandler_dump_traceback(fd, all_threads); + + set_errno(save_errno); + + // Reset to default handler and re-raise to ensure process terminates. + // We cannot just restore the previous handler because Rust's runtime + // may have installed its own SIGSEGV handler (for stack overflow detection) + // that doesn't terminate the process on software-raised signals. + unsafe { + libc::signal(signum, libc::SIG_DFL); + libc::raise(signum); + } + + // Fallback if raise() somehow didn't terminate the process + unsafe { + libc::_exit(1); + } + } + + // faulthandler_fatal_error for Windows + #[cfg(windows)] + extern "C" fn faulthandler_fatal_error(signum: libc::c_int) { + let save_errno = get_errno(); + + if !FATAL_ERROR.enabled.load(Ordering::Relaxed) { + return; + } + + let fd = FATAL_ERROR.fd.load(Ordering::Relaxed); + + let handler = unsafe { + FAULTHANDLER_HANDLERS + .iter_mut() + .find(|h| h.signum == signum) + }; + + if let Some(h) = handler { + unsafe { + faulthandler_disable_fatal_handler(h); + } + puts(fd, "Fatal Python error: "); + puts(fd, h.name); + puts(fd, "\n\n"); + } else { + puts(fd, "Fatal Python error from unexpected signum: "); + dump_decimal(fd, signum as usize); + puts(fd, "\n\n"); + } + + let all_threads = FATAL_ERROR.all_threads.load(Ordering::Relaxed); + faulthandler_dump_traceback(fd, all_threads); + + set_errno(save_errno); + + unsafe { + libc::signal(signum, libc::SIG_DFL); + libc::raise(signum); + } + + // Fallback + std::process::exit(1); + } + + // Windows vectored exception handler (faulthandler.c:417-480) + #[cfg(windows)] + static EXC_HANDLER: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + + #[cfg(windows)] + fn faulthandler_ignore_exception(code: u32) -> bool { + // bpo-30557: ignore exceptions which are not errors + if (code & 0x80000000) == 0 { + return true; + } + // bpo-31701: ignore MSC and COM exceptions + if code == 0xE06D7363 || code == 0xE0434352 { + return true; + } + false + } + + #[cfg(windows)] + unsafe extern "system" fn faulthandler_exc_handler( + exc_info: *mut windows_sys::Win32::System::Diagnostics::Debug::EXCEPTION_POINTERS, + ) -> i32 { + const EXCEPTION_CONTINUE_SEARCH: i32 = 0; + + if !FATAL_ERROR.enabled.load(Ordering::Relaxed) { + return EXCEPTION_CONTINUE_SEARCH; + } + + let record = unsafe { &*(*exc_info).ExceptionRecord }; + let code = record.ExceptionCode as u32; + + if faulthandler_ignore_exception(code) { + return EXCEPTION_CONTINUE_SEARCH; + } + + let fd = FATAL_ERROR.fd.load(Ordering::Relaxed); + + puts(fd, "Windows fatal exception: "); + match code { + 0xC0000005 => puts(fd, "access violation"), + 0xC000008C => puts(fd, "float divide by zero"), + 0xC0000091 => puts(fd, "float overflow"), + 0xC0000094 => puts(fd, "int divide by zero"), + 0xC0000095 => puts(fd, "integer overflow"), + 0xC0000006 => puts(fd, "page error"), + 0xC00000FD => puts(fd, "stack overflow"), + 0xC000001D => puts(fd, "illegal instruction"), + _ => { + puts(fd, "code "); + dump_hexadecimal(fd, code as u64, 8); + } + } + puts(fd, "\n\n"); + + // Disable SIGSEGV handler for access violations to avoid double output + if code == 0xC0000005 { + unsafe { + for handler in FAULTHANDLER_HANDLERS.iter_mut() { + if handler.signum == libc::SIGSEGV { + faulthandler_disable_fatal_handler(handler); + break; + } + } + } + } + + let all_threads = FATAL_ERROR.all_threads.load(Ordering::Relaxed); + faulthandler_dump_traceback(fd, all_threads); + + EXCEPTION_CONTINUE_SEARCH + } + + // faulthandler_enable + #[cfg(unix)] + fn faulthandler_enable_internal() -> bool { + if FATAL_ERROR.enabled.load(Ordering::Relaxed) { + return true; + } + + unsafe { + for handler in FAULTHANDLER_HANDLERS.iter_mut() { + if handler.enabled { + continue; + } + + let mut action: libc::sigaction = core::mem::zeroed(); + action.sa_sigaction = faulthandler_fatal_error as *const () as libc::sighandler_t; + // SA_NODEFER flag + action.sa_flags = libc::SA_NODEFER; + + if libc::sigaction(handler.signum, &action, &mut handler.previous) != 0 { + return false; + } + + handler.enabled = true; + } + } + + FATAL_ERROR.enabled.store(true, Ordering::Relaxed); + true + } + + #[cfg(windows)] + fn faulthandler_enable_internal() -> bool { + if FATAL_ERROR.enabled.load(Ordering::Relaxed) { + return true; + } + + unsafe { + for handler in FAULTHANDLER_HANDLERS.iter_mut() { + if handler.enabled { + continue; + } + + handler.previous = libc::signal( + handler.signum, + faulthandler_fatal_error as *const () as libc::sighandler_t, + ); + + // SIG_ERR is -1 as sighandler_t (which is usize on Windows) + if handler.previous == libc::SIG_ERR as libc::sighandler_t { + return false; + } + + handler.enabled = true; + } + } + + // Register Windows vectored exception handler + #[cfg(windows)] + { + use windows_sys::Win32::System::Diagnostics::Debug::AddVectoredExceptionHandler; + let h = unsafe { AddVectoredExceptionHandler(1, Some(faulthandler_exc_handler)) }; + EXC_HANDLER.store(h as usize, Ordering::Relaxed); + } + + FATAL_ERROR.enabled.store(true, Ordering::Relaxed); + true + } + + // faulthandler_disable + #[cfg(any(unix, windows))] + fn faulthandler_disable_internal() { + if !FATAL_ERROR.enabled.swap(false, Ordering::Relaxed) { + return; + } + + unsafe { + for handler in FAULTHANDLER_HANDLERS.iter_mut() { + faulthandler_disable_fatal_handler(handler); + } + } + + // Remove Windows vectored exception handler + #[cfg(windows)] + { + use windows_sys::Win32::System::Diagnostics::Debug::RemoveVectoredExceptionHandler; + let h = EXC_HANDLER.swap(0, Ordering::Relaxed); + if h != 0 { + unsafe { + RemoveVectoredExceptionHandler(h as *mut core::ffi::c_void); + } + } + } + } + + #[cfg(not(any(unix, windows)))] + fn faulthandler_enable_internal() -> bool { + FATAL_ERROR.enabled.store(true, Ordering::Relaxed); + true + } + + #[cfg(not(any(unix, windows)))] + fn faulthandler_disable_internal() { + FATAL_ERROR.enabled.store(false, Ordering::Relaxed); + } + + // faulthandler_disable_py + #[pyfunction] + fn disable() -> bool { + let was_enabled = FATAL_ERROR.enabled.load(Ordering::Relaxed); + faulthandler_disable_internal(); + was_enabled + } + + // faulthandler_is_enabled + #[pyfunction] + fn is_enabled() -> bool { + FATAL_ERROR.enabled.load(Ordering::Relaxed) + } + + fn format_timeout(timeout_us: u64) -> String { + let sec = timeout_us / 1_000_000; + let us = timeout_us % 1_000_000; + let min = sec / 60; + let sec = sec % 60; + let hour = min / 60; + let min = min % 60; + + // Match Python's timedelta str format: H:MM:SS.ffffff (no leading zero for hours) + if us != 0 { + format!("Timeout ({}:{:02}:{:02}.{:06})!\n", hour, min, sec, us) + } else { + format!("Timeout ({}:{:02}:{:02})!\n", hour, min, sec) + } + } + + fn get_fd_from_file_opt(file: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<i32> { + match file { + OptionalArg::Present(f) if !vm.is_none(&f) => { + // Check if it's an integer (file descriptor) + if let Ok(fd) = f.try_to_value::<i32>(vm) { + if fd < 0 { + return Err(vm.new_value_error("file is not a valid file descriptor")); + } + return Ok(fd); + } + // Try to get fileno() from file object + let fileno = vm.call_method(&f, "fileno", ())?; + let fd: i32 = fileno.try_to_value(vm)?; + if fd < 0 { + return Err(vm.new_value_error("file is not a valid file descriptor")); + } + // Try to flush the file + let _ = vm.call_method(&f, "flush", ()); + Ok(fd) + } + _ => { + // file=None or file not passed: fall back to sys.stderr + let stderr = vm.sys_module.get_attr("stderr", vm)?; + if vm.is_none(&stderr) { + return Err(vm.new_runtime_error("sys.stderr is None")); + } + let fileno = vm.call_method(&stderr, "fileno", ())?; + let fd: i32 = fileno.try_to_value(vm)?; + let _ = vm.call_method(&stderr, "flush", ()); + Ok(fd) + } + } + } + + fn watchdog_thread(state: WatchdogHandle) { + let (lock, cvar) = &*state; + + loop { + // Hold lock across wait_timeout to avoid race condition + let mut guard = lock.lock(); + if guard.cancel { + return; + } + let timeout = Duration::from_micros(guard.timeout_us); + cvar.wait_for(&mut guard, timeout); + + // Check if cancelled after wait + if guard.cancel { + return; + } + + // Extract values before releasing lock for I/O + let repeat = guard.repeat; + let exit = guard.exit; + let fd = guard.fd; + let header = guard.header.clone(); + #[cfg(feature = "threading")] + let thread_frame_slots = guard.thread_frame_slots.clone(); + drop(guard); // Release lock before I/O + + // Timeout occurred, dump traceback + #[cfg(target_arch = "wasm32")] + let _ = (exit, fd, &header); + + #[cfg(not(target_arch = "wasm32"))] + { + puts_bytes(fd, header.as_bytes()); + + // Use thread frame slots when threading is enabled (includes all threads). + // Fall back to live frame walking for non-threaded builds. + #[cfg(feature = "threading")] + { + for (tid, slot) in &thread_frame_slots { + let frames = slot.frames.lock(); + dump_traceback_thread_frames(fd, *tid, false, &frames); + } + } + #[cfg(not(feature = "threading"))] + { + write_thread_id(fd, current_thread_id(), false); + dump_live_frames(fd); + } + + if exit { + std::process::exit(1); + } + } + + if !repeat { + return; + } + } + } + + #[derive(FromArgs)] + #[allow(unused)] + struct DumpTracebackLaterArgs { + #[pyarg(positional, error_msg = "timeout must be a number (int or float)")] + timeout: ArgIntoFloat, + #[pyarg(any, default = false)] + repeat: bool, + #[pyarg(any, default)] + file: OptionalArg<PyObjectRef>, + #[pyarg(any, default = false)] + exit: bool, + } + + #[pyfunction] + fn dump_traceback_later(args: DumpTracebackLaterArgs, vm: &VirtualMachine) -> PyResult<()> { + let timeout: f64 = args.timeout.into_float(); + + if timeout <= 0.0 { + return Err(vm.new_value_error("timeout must be greater than 0")); + } + + let fd = get_fd_from_file_opt(args.file, vm)?; + + // Convert timeout to microseconds + let timeout_us = (timeout * 1_000_000.0) as u64; + if timeout_us == 0 { + return Err(vm.new_value_error("timeout must be greater than 0")); + } + + let header = format_timeout(timeout_us); + + // Snapshot thread frame slots so watchdog can dump tracebacks + #[cfg(feature = "threading")] + let thread_frame_slots: Vec<(u64, ThreadFrameSlot)> = { + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .map(|(&id, slot)| (id, Arc::clone(slot))) + .collect() + }; + + // Cancel any previous watchdog + cancel_dump_traceback_later(); + + // Create new watchdog state + let state = Arc::new(( + Mutex::new(WatchdogState { + cancel: false, + fd, + timeout_us, + repeat: args.repeat, + exit: args.exit, + header, + #[cfg(feature = "threading")] + thread_frame_slots, + }), + Condvar::new(), + )); + + // Store the state + { + let mut watchdog = WATCHDOG.lock(); + *watchdog = Some(Arc::clone(&state)); + } + + // Start watchdog thread + thread::spawn(move || { + watchdog_thread(state); + }); + + Ok(()) + } + + #[pyfunction] + fn cancel_dump_traceback_later() { + let state = { + let mut watchdog = WATCHDOG.lock(); + watchdog.take() + }; + + if let Some(state) = state { + let (lock, cvar) = &*state; + { + let mut guard = lock.lock(); + guard.cancel = true; + } + cvar.notify_all(); + } + } + + #[cfg(unix)] + mod user_signals { + use parking_lot::Mutex; + + const NSIG: usize = 64; + + #[derive(Clone, Copy)] + pub struct UserSignal { + pub enabled: bool, + pub fd: i32, + pub all_threads: bool, + pub chain: bool, + pub previous: libc::sigaction, + } + + impl Default for UserSignal { + fn default() -> Self { + Self { + enabled: false, + fd: 2, // stderr + all_threads: true, + chain: false, + // SAFETY: sigaction is a C struct that can be zero-initialized + previous: unsafe { core::mem::zeroed() }, + } + } + } + + static USER_SIGNALS: Mutex<Option<Vec<UserSignal>>> = Mutex::new(None); + + pub fn get_user_signal(signum: usize) -> Option<UserSignal> { + let guard = USER_SIGNALS.lock(); + guard.as_ref().and_then(|v| v.get(signum).cloned()) + } + + pub fn set_user_signal(signum: usize, signal: UserSignal) { + let mut guard = USER_SIGNALS.lock(); + if guard.is_none() { + *guard = Some(vec![UserSignal::default(); NSIG]); + } + if let Some(ref mut v) = *guard + && signum < v.len() + { + v[signum] = signal; + } + } + + pub fn clear_user_signal(signum: usize) -> Option<UserSignal> { + let mut guard = USER_SIGNALS.lock(); + if let Some(ref mut v) = *guard + && signum < v.len() + && v[signum].enabled + { + let old = v[signum]; + v[signum] = UserSignal::default(); + return Some(old); + } + None + } + + pub fn is_enabled(signum: usize) -> bool { + let guard = USER_SIGNALS.lock(); + guard + .as_ref() + .and_then(|v| v.get(signum)) + .is_some_and(|s| s.enabled) + } + } + + #[cfg(unix)] + extern "C" fn faulthandler_user_signal(signum: libc::c_int) { + let save_errno = get_errno(); + + let user = match user_signals::get_user_signal(signum as usize) { + Some(u) if u.enabled => u, + _ => return, + }; + + faulthandler_dump_traceback(user.fd, user.all_threads); + + if user.chain { + // Restore the previous handler and re-raise + unsafe { + libc::sigaction(signum, &user.previous, core::ptr::null_mut()); + } + set_errno(save_errno); + unsafe { + libc::raise(signum); + } + // Re-install our handler with the same flags as register() + let save_errno2 = get_errno(); + unsafe { + let mut action: libc::sigaction = core::mem::zeroed(); + action.sa_sigaction = faulthandler_user_signal as *const () as libc::sighandler_t; + action.sa_flags = libc::SA_NODEFER; + libc::sigaction(signum, &action, core::ptr::null_mut()); + } + set_errno(save_errno2); + } + } + + #[cfg(unix)] + fn check_signum(signum: i32, vm: &VirtualMachine) -> PyResult<()> { + // Check if it's a fatal signal (faulthandler.c uses faulthandler_handlers array) + let is_fatal = unsafe { FAULTHANDLER_HANDLERS.iter().any(|h| h.signum == signum) }; + if is_fatal { + return Err(vm.new_runtime_error(format!( + "signal {} cannot be registered, use enable() instead", + signum + ))); + } + + // Check if signal is in valid range + if !(1..64).contains(&signum) { + return Err(vm.new_value_error("signal number out of range")); + } + + Ok(()) + } + + #[cfg(unix)] + #[derive(FromArgs)] + #[allow(unused)] + struct RegisterArgs { + #[pyarg(positional)] + signum: i32, + #[pyarg(any, default)] + file: OptionalArg<PyObjectRef>, + #[pyarg(any, default = true)] + all_threads: bool, + #[pyarg(any, default = false)] + chain: bool, + } + + #[cfg(unix)] + #[pyfunction] + fn register(args: RegisterArgs, vm: &VirtualMachine) -> PyResult<()> { + check_signum(args.signum, vm)?; + + let fd = get_fd_from_file_opt(args.file, vm)?; + + let signum = args.signum as usize; + + // Get current handler to save as previous + let previous = if !user_signals::is_enabled(signum) { + unsafe { + let mut action: libc::sigaction = core::mem::zeroed(); + action.sa_sigaction = faulthandler_user_signal as *const () as libc::sighandler_t; + // SA_RESTART by default; SA_NODEFER only when chaining + // (faulthandler.c:860-864) + action.sa_flags = if args.chain { + libc::SA_NODEFER + } else { + libc::SA_RESTART + }; + + let mut prev: libc::sigaction = core::mem::zeroed(); + if libc::sigaction(args.signum, &action, &mut prev) != 0 { + return Err(vm.new_os_error(format!( + "Failed to register signal handler for signal {}", + args.signum + ))); + } + prev + } + } else { + // Already registered, keep previous handler + user_signals::get_user_signal(signum) + .map(|u| u.previous) + .unwrap_or(unsafe { core::mem::zeroed() }) + }; + + user_signals::set_user_signal( + signum, + user_signals::UserSignal { + enabled: true, + fd, + all_threads: args.all_threads, + chain: args.chain, + previous, + }, + ); + + Ok(()) + } + + #[cfg(unix)] + #[pyfunction] + fn unregister(signum: i32, vm: &VirtualMachine) -> PyResult<bool> { + check_signum(signum, vm)?; + + if let Some(old) = user_signals::clear_user_signal(signum as usize) { + // Restore previous handler + unsafe { + libc::sigaction(signum, &old.previous, core::ptr::null_mut()); + } + Ok(true) + } else { + Ok(false) + } + } + + // Test functions for faulthandler testing + + #[pyfunction] + fn _read_null(_vm: &VirtualMachine) { + #[cfg(not(target_arch = "wasm32"))] + { + suppress_crash_report(); + + unsafe { + let ptr: *const i32 = core::ptr::null(); + core::ptr::read_volatile(ptr); + } + } + } + + #[derive(FromArgs)] + #[allow(dead_code)] + struct SigsegvArgs { + #[pyarg(any, default = false)] + release_gil: bool, + } + + #[pyfunction] + fn _sigsegv(_args: SigsegvArgs, _vm: &VirtualMachine) { + #[cfg(not(target_arch = "wasm32"))] + { + suppress_crash_report(); + + // Write to NULL pointer to trigger a real hardware SIGSEGV, + // matching CPython's *((volatile int *)NULL) = 0; + // Using raise(SIGSEGV) doesn't work reliably because Rust's runtime + // installs its own signal handler that may swallow software signals. + unsafe { + let ptr: *mut i32 = core::ptr::null_mut(); + core::ptr::write_volatile(ptr, 0); + } + } + } + + #[pyfunction] + fn _sigabrt(_vm: &VirtualMachine) { + #[cfg(not(target_arch = "wasm32"))] + { + suppress_crash_report(); + + unsafe { + libc::abort(); + } + } + } + + #[pyfunction] + fn _sigfpe(_vm: &VirtualMachine) { + #[cfg(not(target_arch = "wasm32"))] + { + suppress_crash_report(); + + unsafe { + libc::raise(libc::SIGFPE); + } + } + } + + #[pyfunction] + fn _fatal_error_c_thread() { + // This would call Py_FatalError in a new C thread + // For RustPython, we just panic in a new thread + #[cfg(not(target_arch = "wasm32"))] + { + suppress_crash_report(); + std::thread::spawn(|| { + panic!("Fatal Python error: in new thread"); + }); + // Wait a bit for the thread to panic + std::thread::sleep(core::time::Duration::from_secs(1)); + } + } + + #[cfg(not(target_arch = "wasm32"))] + fn suppress_crash_report() { + #[cfg(windows)] + { + use windows_sys::Win32::System::Diagnostics::Debug::{ + SEM_NOGPFAULTERRORBOX, SetErrorMode, + }; + unsafe { + let mode = SetErrorMode(SEM_NOGPFAULTERRORBOX); + SetErrorMode(mode | SEM_NOGPFAULTERRORBOX); + } + } + + #[cfg(unix)] + { + // Disable core dumps + #[cfg(not(any(target_os = "redox", target_os = "wasi")))] + { + use libc::{RLIMIT_CORE, rlimit, setrlimit}; + let rl = rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + unsafe { + let _ = setrlimit(RLIMIT_CORE, &rl); + } + } + } + } + + // Windows-specific constants + #[cfg(windows)] + #[pyattr] + const _EXCEPTION_ACCESS_VIOLATION: u32 = 0xC0000005; + + #[cfg(windows)] + #[pyattr] + const _EXCEPTION_INT_DIVIDE_BY_ZERO: u32 = 0xC0000094; + + #[cfg(windows)] + #[pyattr] + const _EXCEPTION_STACK_OVERFLOW: u32 = 0xC00000FD; + + #[cfg(windows)] + #[pyattr] + const _EXCEPTION_NONCONTINUABLE: u32 = 0x00000001; + + #[cfg(windows)] + #[pyattr] + const _EXCEPTION_NONCONTINUABLE_EXCEPTION: u32 = 0xC0000025; + + #[cfg(windows)] + #[derive(FromArgs)] + struct RaiseExceptionArgs { + #[pyarg(positional)] + code: u32, + #[pyarg(positional, default = 0)] + flags: u32, + } + + #[cfg(windows)] + #[pyfunction] + fn _raise_exception(args: RaiseExceptionArgs, _vm: &VirtualMachine) { + use windows_sys::Win32::System::Diagnostics::Debug::RaiseException; + + suppress_crash_report(); + unsafe { + RaiseException(args.code, args.flags, 0, core::ptr::null()); + } + } +} diff --git a/stdlib/src/fcntl.rs b/crates/stdlib/src/fcntl.rs similarity index 84% rename from stdlib/src/fcntl.rs rename to crates/stdlib/src/fcntl.rs index ee73e503977..0f75a09ba0f 100644 --- a/stdlib/src/fcntl.rs +++ b/crates/stdlib/src/fcntl.rs @@ -1,12 +1,14 @@ -pub(crate) use fcntl::make_module; +// spell-checker:disable + +pub(crate) use fcntl::module_def; #[pymodule] mod fcntl { use crate::vm::{ + PyResult, VirtualMachine, builtins::PyIntRef, function::{ArgMemoryBuffer, ArgStrOrBytesLike, Either, OptionalArg}, - stdlib::{io, os}, - PyResult, VirtualMachine, + stdlib::_io, }; // TODO: supply these from <asm-generic/fnctl.h> (please file an issue/PR upstream): @@ -20,7 +22,7 @@ mod fcntl { // I_LINK, I_UNLINK, I_PLINK, I_PUNLINK #[pyattr] - use libc::{FD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL}; + use libc::{F_GETFD, F_GETFL, F_SETFD, F_SETFL, FD_CLOEXEC}; #[cfg(not(target_os = "wasi"))] #[pyattr] @@ -45,7 +47,7 @@ mod fcntl { #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] #[pyattr] use libc::{ - F_ADD_SEALS, F_GETLEASE, F_GETPIPE_SZ, F_GET_SEALS, F_NOTIFY, F_SEAL_GROW, F_SEAL_SEAL, + F_ADD_SEALS, F_GET_SEALS, F_GETLEASE, F_GETPIPE_SZ, F_NOTIFY, F_SEAL_GROW, F_SEAL_SEAL, F_SEAL_SHRINK, F_SEAL_WRITE, F_SETLEASE, F_SETPIPE_SZ, }; @@ -55,7 +57,7 @@ mod fcntl { #[pyfunction] fn fcntl( - io::Fildes(fd): io::Fildes, + _io::Fildes(fd): _io::Fildes, cmd: i32, arg: OptionalArg<Either<ArgStrOrBytesLike, PyIntRef>>, vm: &VirtualMachine, @@ -68,12 +70,12 @@ mod fcntl { let s = arg.borrow_bytes(); arg_len = s.len(); buf.get_mut(..arg_len) - .ok_or_else(|| vm.new_value_error("fcntl string arg too long".to_owned()))? + .ok_or_else(|| vm.new_value_error("fcntl string arg too long"))? .copy_from_slice(&s) } let ret = unsafe { libc::fcntl(fd, cmd, buf.as_mut_ptr()) }; if ret < 0 { - return Err(os::errno_err(vm)); + return Err(vm.new_last_errno_error()); } return Ok(vm.ctx.new_bytes(buf[..arg_len].to_vec()).into()); } @@ -82,19 +84,23 @@ mod fcntl { }; let ret = unsafe { libc::fcntl(fd, cmd, int as i32) }; if ret < 0 { - return Err(os::errno_err(vm)); + return Err(vm.new_last_errno_error()); } Ok(vm.new_pyobj(ret)) } #[pyfunction] fn ioctl( - io::Fildes(fd): io::Fildes, - request: u32, + _io::Fildes(fd): _io::Fildes, + request: i64, arg: OptionalArg<Either<Either<ArgMemoryBuffer, ArgStrOrBytesLike>, i32>>, mutate_flag: OptionalArg<bool>, vm: &VirtualMachine, ) -> PyResult { + // Convert to unsigned - handles both positive u32 values and negative i32 values + // that represent the same bit pattern (e.g., TIOCSWINSZ on some platforms). + // First truncate to u32 (takes lower 32 bits), then zero-extend to c_ulong. + let request = (request as u32) as libc::c_ulong; let arg = arg.unwrap_or_else(|| Either::B(0)); match arg { Either::A(buf_kind) => { @@ -102,7 +108,7 @@ mod fcntl { let mut buf = [0u8; BUF_SIZE + 1]; // nul byte let mut fill_buf = |b: &[u8]| { if b.len() > BUF_SIZE { - return Err(vm.new_value_error("fcntl string arg too long".to_owned())); + return Err(vm.new_value_error("fcntl string arg too long")); } buf[..b.len()].copy_from_slice(b); Ok(b.len()) @@ -115,7 +121,7 @@ mod fcntl { let ret = unsafe { libc::ioctl(fd, request as _, arg_buf.as_mut_ptr()) }; if ret < 0 { - return Err(os::errno_err(vm)); + return Err(vm.new_last_errno_error()); } return Ok(vm.ctx.new_int(ret).into()); } @@ -126,14 +132,14 @@ mod fcntl { }; let ret = unsafe { libc::ioctl(fd, request as _, buf.as_mut_ptr()) }; if ret < 0 { - return Err(os::errno_err(vm)); + return Err(vm.new_last_errno_error()); } Ok(vm.ctx.new_bytes(buf[..buf_len].to_vec()).into()) } Either::B(i) => { let ret = unsafe { libc::ioctl(fd, request as _, i) }; if ret < 0 { - return Err(os::errno_err(vm)); + return Err(vm.new_last_errno_error()); } Ok(vm.ctx.new_int(ret).into()) } @@ -143,11 +149,11 @@ mod fcntl { // XXX: at the time of writing, wasi and redox don't have the necessary constants/function #[cfg(not(any(target_os = "wasi", target_os = "redox")))] #[pyfunction] - fn flock(io::Fildes(fd): io::Fildes, operation: i32, vm: &VirtualMachine) -> PyResult { + fn flock(_io::Fildes(fd): _io::Fildes, operation: i32, vm: &VirtualMachine) -> PyResult { let ret = unsafe { libc::flock(fd, operation) }; // TODO: add support for platforms that don't have a builtin `flock` syscall if ret < 0 { - return Err(os::errno_err(vm)); + return Err(vm.new_last_errno_error()); } Ok(vm.ctx.new_int(ret).into()) } @@ -156,7 +162,7 @@ mod fcntl { #[cfg(not(any(target_os = "wasi", target_os = "redox")))] #[pyfunction] fn lockf( - io::Fildes(fd): io::Fildes, + _io::Fildes(fd): _io::Fildes, cmd: i32, len: OptionalArg<PyIntRef>, start: OptionalArg<PyIntRef>, @@ -171,7 +177,7 @@ mod fcntl { }; } - let mut l: libc::flock = unsafe { std::mem::zeroed() }; + let mut l: libc::flock = unsafe { core::mem::zeroed() }; l.l_type = if cmd == libc::LOCK_UN { try_into_l_type!(libc::F_UNLCK) } else if (cmd & libc::LOCK_SH) != 0 { @@ -179,7 +185,7 @@ mod fcntl { } else if (cmd & libc::LOCK_EX) != 0 { try_into_l_type!(libc::F_WRLCK) } else { - return Err(vm.new_value_error("unrecognized lockf argument".to_owned())); + return Err(vm.new_value_error("unrecognized lockf argument")); }?; l.l_start = match start { OptionalArg::Present(s) => s.try_to_primitive(vm)?, @@ -207,7 +213,7 @@ mod fcntl { ) }; if ret < 0 { - return Err(os::errno_err(vm)); + return Err(vm.new_last_errno_error()); } Ok(vm.ctx.new_int(ret).into()) } diff --git a/stdlib/src/grp.rs b/crates/stdlib/src/grp.rs similarity index 76% rename from stdlib/src/grp.rs rename to crates/stdlib/src/grp.rs index 959c984be89..eec901b0e57 100644 --- a/stdlib/src/grp.rs +++ b/crates/stdlib/src/grp.rs @@ -1,37 +1,40 @@ -pub(crate) use grp::make_module; +// spell-checker:disable +pub(crate) use grp::module_def; #[pymodule] mod grp { use crate::vm::{ - builtins::{PyIntRef, PyListRef, PyStrRef}, + PyObjectRef, PyResult, VirtualMachine, + builtins::{PyIntRef, PyListRef, PyUtf8StrRef}, convert::{IntoPyException, ToPyObject}, exceptions, types::PyStructSequence, - PyObjectRef, PyResult, VirtualMachine, }; + use core::ptr::NonNull; use nix::unistd; - use std::ptr::NonNull; - #[pyattr] - #[pyclass(module = "grp", name = "struct_group", traverse)] - #[derive(PyStructSequence)] - struct Group { - #[pytraverse(skip)] + #[pystruct_sequence_data] + struct GroupData { gr_name: String, - #[pytraverse(skip)] gr_passwd: String, - #[pytraverse(skip)] gr_gid: u32, gr_mem: PyListRef, } + + #[pyattr] + #[pystruct_sequence(name = "struct_group", module = "grp", data = "GroupData")] + struct PyGroup; + #[pyclass(with(PyStructSequence))] - impl Group { + impl PyGroup {} + + impl GroupData { fn from_unistd_group(group: unistd::Group, vm: &VirtualMachine) -> Self { - let cstr_lossy = |s: std::ffi::CString| { + let cstr_lossy = |s: alloc::ffi::CString| { s.into_string() .unwrap_or_else(|e| e.into_cstring().to_string_lossy().into_owned()) }; - Group { + GroupData { gr_name: group.name, gr_passwd: cstr_lossy(group.passwd), gr_gid: group.gid.as_raw(), @@ -43,7 +46,7 @@ mod grp { } #[pyfunction] - fn getgrgid(gid: PyIntRef, vm: &VirtualMachine) -> PyResult<Group> { + fn getgrgid(gid: PyIntRef, vm: &VirtualMachine) -> PyResult<GroupData> { let gr_gid = gid.as_bigint(); let gid = libc::gid_t::try_from(gr_gid) .map(unistd::Gid::from_raw) @@ -60,11 +63,11 @@ mod grp { .into(), ) })?; - Ok(Group::from_unistd_group(group, vm)) + Ok(GroupData::from_unistd_group(group, vm)) } #[pyfunction] - fn getgrnam(name: PyStrRef, vm: &VirtualMachine) -> PyResult<Group> { + fn getgrnam(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<GroupData> { let gr_name = name.as_str(); if gr_name.contains('\0') { return Err(exceptions::cstring_error(vm)); @@ -77,20 +80,20 @@ mod grp { .into(), ) })?; - Ok(Group::from_unistd_group(group, vm)) + Ok(GroupData::from_unistd_group(group, vm)) } #[pyfunction] fn getgrall(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { // setgrent, getgrent, etc are not thread safe. Could use fgetgrent_r, but this is easier - static GETGRALL: parking_lot::Mutex<()> = parking_lot::const_mutex(()); + static GETGRALL: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); let _guard = GETGRALL.lock(); let mut list = Vec::new(); unsafe { libc::setgrent() }; while let Some(ptr) = NonNull::new(unsafe { libc::getgrent() }) { let group = unistd::Group::from(unsafe { ptr.as_ref() }); - let group = Group::from_unistd_group(group, vm).to_pyobject(vm); + let group = GroupData::from_unistd_group(group, vm).to_pyobject(vm); list.push(group); } unsafe { libc::endgrent() }; diff --git a/crates/stdlib/src/hashlib.rs b/crates/stdlib/src/hashlib.rs new file mode 100644 index 00000000000..584ed1714d5 --- /dev/null +++ b/crates/stdlib/src/hashlib.rs @@ -0,0 +1,970 @@ +// spell-checker:ignore usedforsecurity HASHXOF hashopenssl dklen +// NOTE: Function names like `openssl_md5` match CPython's `_hashopenssl.c` interface +// for compatibility, but the implementation uses pure Rust crates (md5, sha2, etc.), +// not OpenSSL. + +pub(crate) use _hashlib::module_def; + +#[pymodule] +pub mod _hashlib { + use crate::common::lock::PyRwLock; + use crate::vm::{ + Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyBytes, PyFrozenSet, PyStr, PyTypeRef, PyUtf8StrRef, PyValueError, + }, + class::StaticType, + function::{ArgBytesLike, ArgStrOrBytesLike, FuncArgs, OptionalArg}, + types::{Constructor, Representable}, + }; + use blake2::{Blake2b512, Blake2s256}; + use digest::{DynDigest, OutputSizeUser, core_api::BlockSizeUser}; + use digest::{ExtendableOutput, Update}; + use dyn_clone::{DynClone, clone_trait_object}; + use hmac::Mac; + use md5::Md5; + use sha1::Sha1; + use sha2::{Sha224, Sha256, Sha384, Sha512}; + use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; + + const HASH_ALGORITHMS: &[&str] = &[ + "md5", + "sha1", + "sha224", + "sha256", + "sha384", + "sha512", + "sha3_224", + "sha3_256", + "sha3_384", + "sha3_512", + "shake_128", + "shake_256", + "blake2b", + "blake2s", + ]; + + #[pyattr] + const _GIL_MINSIZE: usize = 2048; + + #[pyattr] + #[pyexception(name = "UnsupportedDigestmodError", base = PyValueError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct UnsupportedDigestmodError(PyValueError); + + #[pyattr] + fn openssl_md_meth_names(vm: &VirtualMachine) -> PyObjectRef { + PyFrozenSet::from_iter( + vm, + HASH_ALGORITHMS.iter().map(|n| vm.ctx.new_str(*n).into()), + ) + .expect("failed to create openssl_md_meth_names frozenset") + .into_ref(&vm.ctx) + .into() + } + + #[pyattr] + fn _constructors(vm: &VirtualMachine) -> PyObjectRef { + let dict = vm.ctx.new_dict(); + for name in HASH_ALGORITHMS { + let s = vm.ctx.new_str(*name); + dict.set_item(&*s, s.clone().into(), vm).unwrap(); + } + dict.into() + } + + #[derive(FromArgs, Debug)] + #[allow(unused)] + struct NewHashArgs { + #[pyarg(positional)] + name: PyUtf8StrRef, + #[pyarg(any, optional)] + data: OptionalArg<ArgBytesLike>, + #[pyarg(named, default = true)] + usedforsecurity: bool, + #[pyarg(named, optional)] + string: OptionalArg<ArgBytesLike>, + } + + #[derive(FromArgs)] + #[allow(unused)] + pub struct BlakeHashArgs { + #[pyarg(any, optional)] + pub data: OptionalArg<ArgBytesLike>, + #[pyarg(named, default = true)] + usedforsecurity: bool, + #[pyarg(named, optional)] + pub string: OptionalArg<ArgBytesLike>, + } + + impl From<NewHashArgs> for BlakeHashArgs { + fn from(args: NewHashArgs) -> Self { + Self { + data: args.data, + usedforsecurity: args.usedforsecurity, + string: args.string, + } + } + } + + #[derive(FromArgs, Debug)] + #[allow(unused)] + pub struct HashArgs { + #[pyarg(any, optional)] + pub data: OptionalArg<ArgBytesLike>, + #[pyarg(named, default = true)] + usedforsecurity: bool, + #[pyarg(named, optional)] + pub string: OptionalArg<ArgBytesLike>, + } + + impl From<NewHashArgs> for HashArgs { + fn from(args: NewHashArgs) -> Self { + Self { + data: args.data, + usedforsecurity: args.usedforsecurity, + string: args.string, + } + } + } + + const KECCAK_WIDTH_BITS: usize = 1600; + + fn keccak_suffix(name: &str) -> Option<u8> { + match name { + "sha3_224" | "sha3_256" | "sha3_384" | "sha3_512" => Some(0x06), + "shake_128" | "shake_256" => Some(0x1f), + _ => None, + } + } + + fn keccak_rate_bits(name: &str, block_size: usize) -> Option<usize> { + keccak_suffix(name).map(|_| block_size * 8) + } + + fn keccak_capacity_bits(name: &str, block_size: usize) -> Option<usize> { + keccak_rate_bits(name, block_size).map(|rate| KECCAK_WIDTH_BITS - rate) + } + + fn missing_hash_attribute<T>(vm: &VirtualMachine, class_name: &str, attr: &str) -> PyResult<T> { + Err(vm.new_attribute_error(format!("'{class_name}' object has no attribute '{attr}'"))) + } + + #[derive(FromArgs)] + #[allow(unused)] + struct XofDigestArgs { + #[pyarg(positional)] + length: isize, + } + + impl XofDigestArgs { + // Match CPython's SHAKE output guard in Modules/sha3module.c. + const MAX_SHAKE_OUTPUT_LENGTH: usize = 1 << 29; + + fn length(&self, vm: &VirtualMachine) -> PyResult<usize> { + let length = usize::try_from(self.length) + .map_err(|_| vm.new_value_error("length must be non-negative"))?; + if length >= Self::MAX_SHAKE_OUTPUT_LENGTH { + return Err(vm.new_value_error("length is too large")); + } + Ok(length) + } + } + + #[derive(FromArgs)] + #[allow(unused)] + struct HmacDigestArgs { + #[pyarg(positional)] + key: ArgBytesLike, + #[pyarg(positional)] + msg: ArgBytesLike, + #[pyarg(positional)] + digest: PyObjectRef, + } + + #[derive(FromArgs)] + #[allow(unused)] + struct Pbkdf2HmacArgs { + #[pyarg(any)] + hash_name: PyUtf8StrRef, + #[pyarg(any)] + password: ArgBytesLike, + #[pyarg(any)] + salt: ArgBytesLike, + #[pyarg(any)] + iterations: i64, + #[pyarg(any, optional)] + dklen: OptionalArg<PyObjectRef>, + } + + fn resolve_data( + data: OptionalArg<ArgBytesLike>, + string: OptionalArg<ArgBytesLike>, + vm: &VirtualMachine, + ) -> PyResult<OptionalArg<ArgBytesLike>> { + match (data.into_option(), string.into_option()) { + (Some(d), None) => Ok(OptionalArg::Present(d)), + (None, Some(s)) => Ok(OptionalArg::Present(s)), + (None, None) => Ok(OptionalArg::Missing), + (Some(_), Some(_)) => Err(vm.new_type_error( + "'data' and 'string' are mutually exclusive \ + and support for 'string' keyword parameter \ + is slated for removal in a future version.", + )), + } + } + + fn resolve_digestmod(digestmod: &PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + if let Some(name) = digestmod.downcast_ref::<PyStr>() + && let Some(name_str) = name.to_str() + { + return Ok(name_str.to_lowercase()); + } + if let Ok(name_obj) = digestmod.get_attr("__name__", vm) + && let Some(name) = name_obj.downcast_ref::<PyStr>() + && let Some(name_str) = name.to_str() + && let Some(algo) = name_str.strip_prefix("openssl_") + { + return Ok(algo.to_owned()); + } + Err(vm.new_exception_msg( + UnsupportedDigestmodError::static_type().to_owned(), + "unsupported digestmod".into(), + )) + } + + fn hash_digest_size(name: &str) -> Option<usize> { + match name { + "md5" => Some(16), + "sha1" => Some(20), + "sha224" => Some(28), + "sha256" => Some(32), + "sha384" => Some(48), + "sha512" => Some(64), + "sha3_224" => Some(28), + "sha3_256" => Some(32), + "sha3_384" => Some(48), + "sha3_512" => Some(64), + "blake2b" => Some(64), + "blake2s" => Some(32), + _ => None, + } + } + + fn unsupported_hash(name: &str, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_exception_msg( + UnsupportedDigestmodError::static_type().to_owned(), + format!("unsupported hash type {name}").into(), + ) + } + + // Object-safe HMAC trait for type-erased dispatch + trait DynHmac: Send + Sync { + fn dyn_update(&mut self, data: &[u8]); + fn dyn_finalize(&self) -> Vec<u8>; + fn dyn_clone(&self) -> Box<dyn DynHmac>; + } + + struct TypedHmac<D>(D); + + impl<D> DynHmac for TypedHmac<D> + where + D: Mac + Clone + Send + Sync + 'static, + { + fn dyn_update(&mut self, data: &[u8]) { + Mac::update(&mut self.0, data); + } + + fn dyn_finalize(&self) -> Vec<u8> { + self.0.clone().finalize().into_bytes().to_vec() + } + + fn dyn_clone(&self) -> Box<dyn DynHmac> { + Box::new(TypedHmac(self.0.clone())) + } + } + + #[pyattr] + #[pyclass(module = "_hashlib", name = "HMAC")] + #[derive(PyPayload)] + pub struct PyHmac { + algo_name: String, + digest_size: usize, + block_size: usize, + ctx: PyRwLock<Box<dyn DynHmac>>, + } + + impl core::fmt::Debug for PyHmac { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "HMAC {}", self.algo_name) + } + } + + #[pyclass(with(Representable), flags(IMMUTABLETYPE))] + impl PyHmac { + #[pyslot] + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create '_hashlib.HMAC' instances")) + } + + #[pygetset] + fn name(&self) -> String { + format!("hmac-{}", self.algo_name) + } + + #[pygetset] + fn digest_size(&self) -> usize { + self.digest_size + } + + #[pygetset] + fn block_size(&self) -> usize { + self.block_size + } + + #[pymethod] + fn update(&self, msg: ArgBytesLike) { + msg.with_ref(|bytes| self.ctx.write().dyn_update(bytes)); + } + + #[pymethod] + fn digest(&self) -> PyBytes { + self.ctx.read().dyn_finalize().into() + } + + #[pymethod] + fn hexdigest(&self) -> String { + hex::encode(self.ctx.read().dyn_finalize()) + } + + #[pymethod] + fn copy(&self) -> Self { + Self { + algo_name: self.algo_name.clone(), + digest_size: self.digest_size, + block_size: self.block_size, + ctx: PyRwLock::new(self.ctx.read().dyn_clone()), + } + } + } + + impl Representable for PyHmac { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} HMAC object @ {:#x}>", + zelf.algo_name, zelf as *const _ as usize + )) + } + } + + #[pyattr] + #[pyclass(module = "_hashlib", name = "HASH")] + #[derive(PyPayload)] + pub struct PyHasher { + pub name: String, + pub ctx: PyRwLock<HashWrapper>, + } + + impl core::fmt::Debug for PyHasher { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "HASH {}", self.name) + } + } + + #[pyclass(with(Representable), flags(IMMUTABLETYPE))] + impl PyHasher { + fn new(name: &str, d: HashWrapper) -> Self { + Self { + name: name.to_owned(), + ctx: PyRwLock::new(d), + } + } + + #[pyslot] + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create '_hashlib.HASH' instances")) + } + + #[pygetset] + fn name(&self) -> String { + self.name.clone() + } + + #[pygetset] + fn digest_size(&self) -> usize { + self.ctx.read().digest_size() + } + + #[pygetset] + fn block_size(&self) -> usize { + self.ctx.read().block_size() + } + + #[pygetset] + fn _capacity_bits(&self, vm: &VirtualMachine) -> PyResult<usize> { + let block_size = self.ctx.read().block_size(); + match keccak_capacity_bits(&self.name, block_size) { + Some(capacity) => Ok(capacity), + None => missing_hash_attribute(vm, "HASH", "_capacity_bits"), + } + } + + #[pygetset] + fn _rate_bits(&self, vm: &VirtualMachine) -> PyResult<usize> { + let block_size = self.ctx.read().block_size(); + match keccak_rate_bits(&self.name, block_size) { + Some(rate) => Ok(rate), + None => missing_hash_attribute(vm, "HASH", "_rate_bits"), + } + } + + #[pygetset] + fn _suffix(&self, vm: &VirtualMachine) -> PyResult<PyBytes> { + match keccak_suffix(&self.name) { + Some(suffix) => Ok(vec![suffix].into()), + None => missing_hash_attribute(vm, "HASH", "_suffix"), + } + } + + #[pymethod] + fn update(&self, data: ArgBytesLike) { + data.with_ref(|bytes| self.ctx.write().update(bytes)); + } + + #[pymethod] + fn digest(&self) -> PyBytes { + self.ctx.read().finalize().into() + } + + #[pymethod] + fn hexdigest(&self) -> String { + hex::encode(self.ctx.read().finalize()) + } + + #[pymethod] + fn copy(&self) -> Self { + Self::new(&self.name, self.ctx.read().clone()) + } + } + + impl Representable for PyHasher { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} _hashlib.HASH object @ {:#x}>", + zelf.name, zelf as *const _ as usize + )) + } + } + + #[pyattr] + #[pyclass(module = "_hashlib", name = "HASHXOF")] + #[derive(PyPayload)] + pub struct PyHasherXof { + name: String, + ctx: PyRwLock<HashXofWrapper>, + } + + impl core::fmt::Debug for PyHasherXof { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "HASHXOF {}", self.name) + } + } + + #[pyclass(with(Representable), flags(IMMUTABLETYPE))] + impl PyHasherXof { + fn new(name: &str, d: HashXofWrapper) -> Self { + Self { + name: name.to_owned(), + ctx: PyRwLock::new(d), + } + } + + #[pyslot] + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create '_hashlib.HASHXOF' instances")) + } + + #[pygetset] + fn name(&self) -> String { + self.name.clone() + } + + #[pygetset] + const fn digest_size(&self) -> usize { + 0 + } + + #[pygetset] + fn block_size(&self) -> usize { + self.ctx.read().block_size() + } + + #[pygetset] + fn _capacity_bits(&self, vm: &VirtualMachine) -> PyResult<usize> { + let block_size = self.ctx.read().block_size(); + match keccak_capacity_bits(&self.name, block_size) { + Some(capacity) => Ok(capacity), + None => missing_hash_attribute(vm, "HASHXOF", "_capacity_bits"), + } + } + + #[pygetset] + fn _rate_bits(&self, vm: &VirtualMachine) -> PyResult<usize> { + let block_size = self.ctx.read().block_size(); + match keccak_rate_bits(&self.name, block_size) { + Some(rate) => Ok(rate), + None => missing_hash_attribute(vm, "HASHXOF", "_rate_bits"), + } + } + + #[pygetset] + fn _suffix(&self, vm: &VirtualMachine) -> PyResult<PyBytes> { + match keccak_suffix(&self.name) { + Some(suffix) => Ok(vec![suffix].into()), + None => missing_hash_attribute(vm, "HASHXOF", "_suffix"), + } + } + + #[pymethod] + fn update(&self, data: ArgBytesLike) { + data.with_ref(|bytes| self.ctx.write().update(bytes)); + } + + #[pymethod] + fn digest(&self, args: XofDigestArgs, vm: &VirtualMachine) -> PyResult<PyBytes> { + Ok(self.ctx.read().finalize_xof(args.length(vm)?).into()) + } + + #[pymethod] + fn hexdigest(&self, args: XofDigestArgs, vm: &VirtualMachine) -> PyResult<String> { + Ok(hex::encode(self.ctx.read().finalize_xof(args.length(vm)?))) + } + + #[pymethod] + fn copy(&self) -> Self { + Self::new(&self.name, self.ctx.read().clone()) + } + } + + impl Representable for PyHasherXof { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} _hashlib.HASHXOF object @ {:#x}>", + zelf.name, zelf as *const _ as usize + )) + } + } + + #[pyfunction(name = "new")] + fn hashlib_new(args: NewHashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let data = resolve_data(args.data, args.string, vm)?; + match args.name.as_str().to_lowercase().as_str() { + "md5" => Ok(PyHasher::new("md5", HashWrapper::new::<Md5>(data)).into_pyobject(vm)), + "sha1" => Ok(PyHasher::new("sha1", HashWrapper::new::<Sha1>(data)).into_pyobject(vm)), + "sha224" => { + Ok(PyHasher::new("sha224", HashWrapper::new::<Sha224>(data)).into_pyobject(vm)) + } + "sha256" => { + Ok(PyHasher::new("sha256", HashWrapper::new::<Sha256>(data)).into_pyobject(vm)) + } + "sha384" => { + Ok(PyHasher::new("sha384", HashWrapper::new::<Sha384>(data)).into_pyobject(vm)) + } + "sha512" => { + Ok(PyHasher::new("sha512", HashWrapper::new::<Sha512>(data)).into_pyobject(vm)) + } + "sha3_224" => { + Ok(PyHasher::new("sha3_224", HashWrapper::new::<Sha3_224>(data)).into_pyobject(vm)) + } + "sha3_256" => { + Ok(PyHasher::new("sha3_256", HashWrapper::new::<Sha3_256>(data)).into_pyobject(vm)) + } + "sha3_384" => { + Ok(PyHasher::new("sha3_384", HashWrapper::new::<Sha3_384>(data)).into_pyobject(vm)) + } + "sha3_512" => { + Ok(PyHasher::new("sha3_512", HashWrapper::new::<Sha3_512>(data)).into_pyobject(vm)) + } + "shake_128" => Ok( + PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(data)) + .into_pyobject(vm), + ), + "shake_256" => Ok( + PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(data)) + .into_pyobject(vm), + ), + "blake2b" => Ok( + PyHasher::new("blake2b", HashWrapper::new::<Blake2b512>(data)).into_pyobject(vm), + ), + "blake2s" => Ok( + PyHasher::new("blake2s", HashWrapper::new::<Blake2s256>(data)).into_pyobject(vm), + ), + other => Err(vm.new_value_error(format!("Unknown hashing algorithm: {other}"))), + } + } + + #[pyfunction(name = "openssl_md5")] + pub fn local_md5(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("md5", HashWrapper::new::<Md5>(data))) + } + + #[pyfunction(name = "openssl_sha1")] + pub fn local_sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha1", HashWrapper::new::<Sha1>(data))) + } + + #[pyfunction(name = "openssl_sha224")] + pub fn local_sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha224", HashWrapper::new::<Sha224>(data))) + } + + #[pyfunction(name = "openssl_sha256")] + pub fn local_sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha256", HashWrapper::new::<Sha256>(data))) + } + + #[pyfunction(name = "openssl_sha384")] + pub fn local_sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha384", HashWrapper::new::<Sha384>(data))) + } + + #[pyfunction(name = "openssl_sha512")] + pub fn local_sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new("sha512", HashWrapper::new::<Sha512>(data))) + } + + #[pyfunction(name = "openssl_sha3_224")] + pub fn local_sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "sha3_224", + HashWrapper::new::<Sha3_224>(data), + )) + } + + #[pyfunction(name = "openssl_sha3_256")] + pub fn local_sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "sha3_256", + HashWrapper::new::<Sha3_256>(data), + )) + } + + #[pyfunction(name = "openssl_sha3_384")] + pub fn local_sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "sha3_384", + HashWrapper::new::<Sha3_384>(data), + )) + } + + #[pyfunction(name = "openssl_sha3_512")] + pub fn local_sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "sha3_512", + HashWrapper::new::<Sha3_512>(data), + )) + } + + #[pyfunction(name = "openssl_shake_128")] + pub fn local_shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasherXof> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasherXof::new( + "shake_128", + HashXofWrapper::new_shake_128(data), + )) + } + + #[pyfunction(name = "openssl_shake_256")] + pub fn local_shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyHasherXof> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasherXof::new( + "shake_256", + HashXofWrapper::new_shake_256(data), + )) + } + + #[pyfunction(name = "openssl_blake2b")] + pub fn local_blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "blake2b", + HashWrapper::new::<Blake2b512>(data), + )) + } + + #[pyfunction(name = "openssl_blake2s")] + pub fn local_blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult<PyHasher> { + let data = resolve_data(args.data, args.string, vm)?; + Ok(PyHasher::new( + "blake2s", + HashWrapper::new::<Blake2s256>(data), + )) + } + + #[pyfunction] + fn get_fips_mode() -> i32 { + 0 + } + + #[pyfunction] + fn compare_digest( + a: ArgStrOrBytesLike, + b: ArgStrOrBytesLike, + vm: &VirtualMachine, + ) -> PyResult<bool> { + use constant_time_eq::constant_time_eq; + + match (&a, &b) { + (ArgStrOrBytesLike::Str(a), ArgStrOrBytesLike::Str(b)) => { + if !a.isascii() || !b.isascii() { + return Err(vm.new_type_error( + "comparing strings with non-ASCII characters is not supported", + )); + } + Ok(constant_time_eq(a.as_bytes(), b.as_bytes())) + } + (ArgStrOrBytesLike::Buf(a), ArgStrOrBytesLike::Buf(b)) => { + Ok(a.with_ref(|a| b.with_ref(|b| constant_time_eq(a, b)))) + } + _ => Err(vm.new_type_error(format!( + "a bytes-like object is required, not '{}'", + b.as_object().class().name() + ))), + } + } + + #[derive(FromArgs, Debug)] + #[allow(unused)] + pub struct NewHMACHashArgs { + #[pyarg(positional)] + key: ArgBytesLike, + #[pyarg(any, optional)] + msg: OptionalArg<Option<ArgBytesLike>>, + #[pyarg(named, optional)] + digestmod: OptionalArg<PyObjectRef>, + } + + #[pyfunction] + fn hmac_new(args: NewHMACHashArgs, vm: &VirtualMachine) -> PyResult<PyHmac> { + let digestmod = args + .digestmod + .into_option() + .ok_or_else(|| vm.new_type_error("Missing required parameter 'digestmod'."))?; + let name = resolve_digestmod(&digestmod, vm)?; + + let key_buf = args.key.borrow_buf(); + let msg_data = args.msg.flatten(); + + macro_rules! make_hmac { + ($hash_ty:ty) => {{ + let mut mac = <hmac::Hmac<$hash_ty> as Mac>::new_from_slice(&key_buf) + .map_err(|_| vm.new_value_error("invalid key length".to_owned()))?; + if let Some(ref m) = msg_data { + m.with_ref(|bytes| Mac::update(&mut mac, bytes)); + } + Ok(PyHmac { + algo_name: name, + digest_size: <$hash_ty as OutputSizeUser>::output_size(), + block_size: <$hash_ty as BlockSizeUser>::block_size(), + ctx: PyRwLock::new(Box::new(TypedHmac(mac))), + }) + }}; + } + + match name.as_str() { + "md5" => make_hmac!(Md5), + "sha1" => make_hmac!(Sha1), + "sha224" => make_hmac!(Sha224), + "sha256" => make_hmac!(Sha256), + "sha384" => make_hmac!(Sha384), + "sha512" => make_hmac!(Sha512), + "sha3_224" => make_hmac!(Sha3_224), + "sha3_256" => make_hmac!(Sha3_256), + "sha3_384" => make_hmac!(Sha3_384), + "sha3_512" => make_hmac!(Sha3_512), + _ => Err(unsupported_hash(&name, vm)), + } + } + + #[pyfunction] + fn hmac_digest(args: HmacDigestArgs, vm: &VirtualMachine) -> PyResult<PyBytes> { + let name = resolve_digestmod(&args.digest, vm)?; + + let key_buf = args.key.borrow_buf(); + let msg_buf = args.msg.borrow_buf(); + + macro_rules! do_hmac { + ($hash_ty:ty) => {{ + let mut mac = <hmac::Hmac<$hash_ty> as Mac>::new_from_slice(&key_buf) + .map_err(|_| vm.new_value_error("invalid key length".to_owned()))?; + Mac::update(&mut mac, &msg_buf); + Ok(mac.finalize().into_bytes().to_vec().into()) + }}; + } + + match name.as_str() { + "md5" => do_hmac!(Md5), + "sha1" => do_hmac!(Sha1), + "sha224" => do_hmac!(Sha224), + "sha256" => do_hmac!(Sha256), + "sha384" => do_hmac!(Sha384), + "sha512" => do_hmac!(Sha512), + "sha3_224" => do_hmac!(Sha3_224), + "sha3_256" => do_hmac!(Sha3_256), + "sha3_384" => do_hmac!(Sha3_384), + "sha3_512" => do_hmac!(Sha3_512), + _ => Err(unsupported_hash(&name, vm)), + } + } + + #[pyfunction] + fn pbkdf2_hmac(args: Pbkdf2HmacArgs, vm: &VirtualMachine) -> PyResult<PyBytes> { + let name = args.hash_name.as_str().to_lowercase(); + + if args.iterations < 1 { + return Err(vm.new_value_error("iteration value must be greater than 0.")); + } + let rounds = u32::try_from(args.iterations) + .map_err(|_| vm.new_overflow_error("iteration value is too great."))?; + + let dklen: usize = match args.dklen.into_option() { + Some(obj) if vm.is_none(&obj) => { + hash_digest_size(&name).ok_or_else(|| unsupported_hash(&name, vm))? + } + Some(obj) => { + let len: i64 = obj.try_into_value(vm)?; + if len < 1 { + return Err(vm.new_value_error("key length must be greater than 0.")); + } + usize::try_from(len) + .map_err(|_| vm.new_overflow_error("key length is too great."))? + } + None => hash_digest_size(&name).ok_or_else(|| unsupported_hash(&name, vm))?, + }; + + let password_buf = args.password.borrow_buf(); + let salt_buf = args.salt.borrow_buf(); + let mut dk = vec![0u8; dklen]; + + macro_rules! do_pbkdf2 { + ($hash_ty:ty) => {{ + pbkdf2::pbkdf2_hmac::<$hash_ty>(&password_buf, &salt_buf, rounds, &mut dk); + Ok(dk.into()) + }}; + } + + match name.as_str() { + "md5" => do_pbkdf2!(Md5), + "sha1" => do_pbkdf2!(Sha1), + "sha224" => do_pbkdf2!(Sha224), + "sha256" => do_pbkdf2!(Sha256), + "sha384" => do_pbkdf2!(Sha384), + "sha512" => do_pbkdf2!(Sha512), + "sha3_224" => do_pbkdf2!(Sha3_224), + "sha3_256" => do_pbkdf2!(Sha3_256), + "sha3_384" => do_pbkdf2!(Sha3_384), + "sha3_512" => do_pbkdf2!(Sha3_512), + _ => Err(unsupported_hash(&name, vm)), + } + } + + pub trait ThreadSafeDynDigest: DynClone + DynDigest + Sync + Send {} + impl<T> ThreadSafeDynDigest for T where T: DynClone + DynDigest + Sync + Send {} + + clone_trait_object!(ThreadSafeDynDigest); + + #[derive(Clone)] + pub struct HashWrapper { + block_size: usize, + inner: Box<dyn ThreadSafeDynDigest>, + } + + impl HashWrapper { + pub fn new<D>(data: OptionalArg<ArgBytesLike>) -> Self + where + D: ThreadSafeDynDigest + BlockSizeUser + Default + 'static, + { + let mut h = Self { + block_size: D::block_size(), + inner: Box::<D>::default(), + }; + if let OptionalArg::Present(d) = data { + d.with_ref(|bytes| h.update(bytes)); + } + h + } + + fn update(&mut self, data: &[u8]) { + self.inner.update(data); + } + + const fn block_size(&self) -> usize { + self.block_size + } + + fn digest_size(&self) -> usize { + self.inner.output_size() + } + + fn finalize(&self) -> Vec<u8> { + let cloned = self.inner.box_clone(); + cloned.finalize().into_vec() + } + } + + #[derive(Clone)] + pub enum HashXofWrapper { + Shake128(Shake128), + Shake256(Shake256), + } + + impl HashXofWrapper { + pub fn new_shake_128(data: OptionalArg<ArgBytesLike>) -> Self { + let mut h = Self::Shake128(Shake128::default()); + if let OptionalArg::Present(d) = data { + d.with_ref(|bytes| h.update(bytes)); + } + h + } + + pub fn new_shake_256(data: OptionalArg<ArgBytesLike>) -> Self { + let mut h = Self::Shake256(Shake256::default()); + if let OptionalArg::Present(d) = data { + d.with_ref(|bytes| h.update(bytes)); + } + h + } + + fn update(&mut self, data: &[u8]) { + match self { + Self::Shake128(h) => h.update(data), + Self::Shake256(h) => h.update(data), + } + } + + fn block_size(&self) -> usize { + match self { + Self::Shake128(_) => Shake128::block_size(), + Self::Shake256(_) => Shake256::block_size(), + } + } + + fn finalize_xof(&self, length: usize) -> Vec<u8> { + match self { + Self::Shake128(h) => h.clone().finalize_boxed(length).into_vec(), + Self::Shake256(h) => h.clone().finalize_boxed(length).into_vec(), + } + } + } +} diff --git a/crates/stdlib/src/json.rs b/crates/stdlib/src/json.rs new file mode 100644 index 00000000000..41c3dead090 --- /dev/null +++ b/crates/stdlib/src/json.rs @@ -0,0 +1,728 @@ +pub(crate) use _json::module_def; +mod machinery; + +#[pymodule] +mod _json { + use super::machinery; + use crate::vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyStrRef, PyType, PyUtf8StrRef}, + convert::ToPyResult, + function::{IntoFuncArgs, OptionalArg}, + protocol::PyIterReturn, + types::{Callable, Constructor}, + }; + use core::str::FromStr; + use malachite_bigint::BigInt; + use rustpython_common::wtf8::Wtf8Buf; + use std::collections::HashMap; + + /// Skip JSON whitespace characters (space, tab, newline, carriage return). + /// Works with a byte slice and returns the number of bytes skipped. + /// Since all JSON whitespace chars are ASCII, bytes == chars. + #[inline] + fn skip_whitespace(bytes: &[u8]) -> usize { + flame_guard!("_json::skip_whitespace"); + let mut count = 0; + for &b in bytes { + match b { + b' ' | b'\t' | b'\n' | b'\r' => count += 1, + _ => break, + } + } + count + } + + /// Check if a byte slice starts with a given ASCII pattern. + #[inline] + fn starts_with_bytes(bytes: &[u8], pattern: &[u8]) -> bool { + bytes.len() >= pattern.len() && &bytes[..pattern.len()] == pattern + } + + #[pyattr(name = "make_scanner")] + #[pyclass(name = "Scanner", traverse)] + #[derive(Debug, PyPayload)] + struct JsonScanner { + #[pytraverse(skip)] + strict: bool, + object_hook: Option<PyObjectRef>, + object_pairs_hook: Option<PyObjectRef>, + parse_float: Option<PyObjectRef>, + parse_int: Option<PyObjectRef>, + parse_constant: PyObjectRef, + ctx: PyObjectRef, + } + + impl Constructor for JsonScanner { + type Args = PyObjectRef; + + fn py_new(_cls: &Py<PyType>, ctx: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let strict = ctx.get_attr("strict", vm)?.try_to_bool(vm)?; + let object_hook = vm.option_if_none(ctx.get_attr("object_hook", vm)?); + let object_pairs_hook = vm.option_if_none(ctx.get_attr("object_pairs_hook", vm)?); + let parse_float = ctx.get_attr("parse_float", vm)?; + let parse_float = if vm.is_none(&parse_float) || parse_float.is(vm.ctx.types.float_type) + { + None + } else { + Some(parse_float) + }; + let parse_int = ctx.get_attr("parse_int", vm)?; + let parse_int = if vm.is_none(&parse_int) || parse_int.is(vm.ctx.types.int_type) { + None + } else { + Some(parse_int) + }; + let parse_constant = ctx.get_attr("parse_constant", vm)?; + + Ok(Self { + strict, + object_hook, + object_pairs_hook, + parse_float, + parse_int, + parse_constant, + ctx, + }) + } + } + + #[pyclass(with(Callable, Constructor))] + impl JsonScanner { + fn parse( + &self, + pystr: PyUtf8StrRef, + char_idx: usize, + byte_idx: usize, + scan_once: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + flame_guard!("JsonScanner::parse"); + let bytes = pystr.as_bytes(); + let wtf8 = pystr.as_wtf8(); + + let first_byte = match bytes.get(byte_idx) { + Some(&b) => b, + None => { + return Ok(PyIterReturn::StopIteration(Some( + vm.ctx.new_int(char_idx).into(), + ))); + } + }; + + match first_byte { + b'"' => { + // Parse string - pass slice starting after the quote + let (wtf8_result, chars_consumed, _bytes_consumed) = + machinery::scanstring(&wtf8[byte_idx + 1..], char_idx + 1, self.strict) + .map_err(|e| py_decode_error(e, pystr.clone().into_wtf8(), vm))?; + let end_char_idx = char_idx + 1 + chars_consumed; + return Ok(PyIterReturn::Return( + vm.new_tuple((wtf8_result, end_char_idx)).into(), + )); + } + b'{' => { + // Parse object in Rust + let mut memo = HashMap::new(); + return self + .parse_object(pystr, char_idx + 1, byte_idx + 1, &scan_once, &mut memo, vm) + .map(|(obj, end_char, _end_byte)| { + PyIterReturn::Return(vm.new_tuple((obj, end_char)).into()) + }); + } + b'[' => { + // Parse array in Rust + let mut memo = HashMap::new(); + return self + .parse_array(pystr, char_idx + 1, byte_idx + 1, &scan_once, &mut memo, vm) + .map(|(obj, end_char, _end_byte)| { + PyIterReturn::Return(vm.new_tuple((obj, end_char)).into()) + }); + } + _ => {} + } + + let s = &pystr.as_str()[byte_idx..]; + + macro_rules! parse_const { + ($s:literal, $val:expr) => { + if s.starts_with($s) { + return Ok(PyIterReturn::Return( + vm.new_tuple(($val, char_idx + $s.len())).into(), + )); + } + }; + } + + parse_const!("null", vm.ctx.none()); + parse_const!("true", true); + parse_const!("false", false); + + if let Some((res, len)) = self.parse_number(s, vm) { + return Ok(PyIterReturn::Return( + vm.new_tuple((res?, char_idx + len)).into(), + )); + } + + macro_rules! parse_constant { + ($s:literal) => { + if s.starts_with($s) { + return Ok(PyIterReturn::Return( + vm.new_tuple(( + self.parse_constant.call(($s,), vm)?, + char_idx + $s.len(), + )) + .into(), + )); + } + }; + } + + parse_constant!("NaN"); + parse_constant!("Infinity"); + parse_constant!("-Infinity"); + + Ok(PyIterReturn::StopIteration(Some( + vm.ctx.new_int(char_idx).into(), + ))) + } + + fn parse_number(&self, s: &str, vm: &VirtualMachine) -> Option<(PyResult, usize)> { + flame_guard!("JsonScanner::parse_number"); + let mut has_neg = false; + let mut has_decimal = false; + let mut has_exponent = false; + let mut has_e_sign = false; + let mut i = 0; + for c in s.chars() { + match c { + '-' if i == 0 => has_neg = true, + n if n.is_ascii_digit() => {} + '.' if !has_decimal => has_decimal = true, + 'e' | 'E' if !has_exponent => has_exponent = true, + '+' | '-' if !has_e_sign => has_e_sign = true, + _ => break, + } + i += 1; + } + if i == 0 || (i == 1 && has_neg) { + return None; + } + let buf = &s[..i]; + let ret = if has_decimal || has_exponent { + // float + if let Some(ref parse_float) = self.parse_float { + parse_float.call((buf,), vm) + } else { + Ok(vm.ctx.new_float(f64::from_str(buf).unwrap()).into()) + } + } else if let Some(ref parse_int) = self.parse_int { + parse_int.call((buf,), vm) + } else { + Ok(vm.new_pyobj(BigInt::from_str(buf).unwrap())) + }; + Some((ret, buf.len())) + } + + /// Parse a JSON object starting after the opening '{'. + /// Returns (parsed_object, end_char_index, end_byte_index). + fn parse_object( + &self, + pystr: PyUtf8StrRef, + start_char_idx: usize, + start_byte_idx: usize, + scan_once: &PyObjectRef, + memo: &mut HashMap<String, PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + flame_guard!("JsonScanner::parse_object"); + + let bytes = pystr.as_bytes(); + let wtf8 = pystr.as_wtf8(); + let mut char_idx = start_char_idx; + let mut byte_idx = start_byte_idx; + + // Skip initial whitespace + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Check for empty object + match bytes.get(byte_idx) { + Some(b'}') => { + return self.finalize_object(vec![], char_idx + 1, byte_idx + 1, vm); + } + Some(b'"') => { + // Continue to parse first key + } + _ => { + return Err(self.make_decode_error( + "Expecting property name enclosed in double quotes", + pystr, + char_idx, + vm, + )); + } + } + + let mut pairs: Vec<(PyObjectRef, PyObjectRef)> = Vec::new(); + + loop { + // We're now at '"', skip it + char_idx += 1; + byte_idx += 1; + + // Parse key string using scanstring with byte slice + let (key_wtf8, chars_consumed, bytes_consumed) = + machinery::scanstring(&wtf8[byte_idx..], char_idx, self.strict) + .map_err(|e| py_decode_error(e, pystr.clone().into_wtf8(), vm))?; + + char_idx += chars_consumed; + byte_idx += bytes_consumed; + + // Key memoization - reuse existing key strings + let key_str = key_wtf8.to_string(); + let key: PyObjectRef = match memo.get(&key_str) { + Some(cached) => cached.clone().into(), + None => { + let py_key = vm.ctx.new_str(key_str.clone()); + memo.insert(key_str, py_key.clone()); + py_key.into() + } + }; + + // Skip whitespace after key + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Expect ':' delimiter + match bytes.get(byte_idx) { + Some(b':') => { + char_idx += 1; + byte_idx += 1; + } + _ => { + return Err(self.make_decode_error( + "Expecting ':' delimiter", + pystr, + char_idx, + vm, + )); + } + } + + // Skip whitespace after ':' + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Parse value recursively + let (value, value_char_end, value_byte_end) = + self.call_scan_once(scan_once, pystr.clone(), char_idx, byte_idx, memo, vm)?; + + pairs.push((key, value)); + char_idx = value_char_end; + byte_idx = value_byte_end; + + // Skip whitespace after value + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Check for ',' or '}' + match bytes.get(byte_idx) { + Some(b'}') => { + char_idx += 1; + byte_idx += 1; + break; + } + Some(b',') => { + let comma_char_idx = char_idx; + char_idx += 1; + byte_idx += 1; + + // Skip whitespace after comma + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Next must be '"' + match bytes.get(byte_idx) { + Some(b'"') => { + // Continue to next key-value pair + } + Some(b'}') => { + // Trailing comma before end of object + return Err(self.make_decode_error( + "Illegal trailing comma before end of object", + pystr, + comma_char_idx, + vm, + )); + } + _ => { + return Err(self.make_decode_error( + "Expecting property name enclosed in double quotes", + pystr, + char_idx, + vm, + )); + } + } + } + _ => { + return Err(self.make_decode_error( + "Expecting ',' delimiter", + pystr, + char_idx, + vm, + )); + } + } + } + + self.finalize_object(pairs, char_idx, byte_idx, vm) + } + + /// Parse a JSON array starting after the opening '['. + /// Returns (parsed_array, end_char_index, end_byte_index). + fn parse_array( + &self, + pystr: PyUtf8StrRef, + start_char_idx: usize, + start_byte_idx: usize, + scan_once: &PyObjectRef, + memo: &mut HashMap<String, PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + flame_guard!("JsonScanner::parse_array"); + + let bytes = pystr.as_bytes(); + let mut char_idx = start_char_idx; + let mut byte_idx = start_byte_idx; + + // Skip initial whitespace + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Check for empty array + if bytes.get(byte_idx) == Some(&b']') { + return Ok((vm.ctx.new_list(vec![]).into(), char_idx + 1, byte_idx + 1)); + } + + let mut values: Vec<PyObjectRef> = Vec::new(); + + loop { + // Parse value + let (value, value_char_end, value_byte_end) = + self.call_scan_once(scan_once, pystr.clone(), char_idx, byte_idx, memo, vm)?; + + values.push(value); + char_idx = value_char_end; + byte_idx = value_byte_end; + + // Skip whitespace after value + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + match bytes.get(byte_idx) { + Some(b']') => { + char_idx += 1; + byte_idx += 1; + break; + } + Some(b',') => { + let comma_char_idx = char_idx; + char_idx += 1; + byte_idx += 1; + + // Skip whitespace after comma + let ws = skip_whitespace(&bytes[byte_idx..]); + char_idx += ws; + byte_idx += ws; + + // Check for trailing comma + if bytes.get(byte_idx) == Some(&b']') { + return Err(self.make_decode_error( + "Illegal trailing comma before end of array", + pystr, + comma_char_idx, + vm, + )); + } + } + _ => { + return Err(self.make_decode_error( + "Expecting ',' delimiter", + pystr, + char_idx, + vm, + )); + } + } + } + + Ok((vm.ctx.new_list(values).into(), char_idx, byte_idx)) + } + + /// Finalize object construction with hooks. + fn finalize_object( + &self, + pairs: Vec<(PyObjectRef, PyObjectRef)>, + end_char_idx: usize, + end_byte_idx: usize, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + let result = if let Some(ref pairs_hook) = self.object_pairs_hook { + // object_pairs_hook takes priority - pass list of tuples + let pairs_list: Vec<PyObjectRef> = pairs + .into_iter() + .map(|(k, v)| vm.new_tuple((k, v)).into()) + .collect(); + pairs_hook.call((vm.ctx.new_list(pairs_list),), vm)? + } else { + // Build a dict from pairs + let dict = vm.ctx.new_dict(); + for (key, value) in pairs { + dict.set_item(&*key, value, vm)?; + } + + // Apply object_hook if present + let dict_obj: PyObjectRef = dict.into(); + if let Some(ref hook) = self.object_hook { + hook.call((dict_obj,), vm)? + } else { + dict_obj + } + }; + + Ok((result, end_char_idx, end_byte_idx)) + } + + /// Call scan_once and handle the result. + /// Returns (value, end_char_idx, end_byte_idx). + fn call_scan_once( + &self, + scan_once: &PyObjectRef, + pystr: PyUtf8StrRef, + char_idx: usize, + byte_idx: usize, + memo: &mut HashMap<String, PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, usize, usize)> { + let bytes = pystr.as_bytes(); + let wtf8 = pystr.as_wtf8(); + let s = pystr.as_str(); + + let first_byte = match bytes.get(byte_idx) { + Some(&b) => b, + None => return Err(self.make_decode_error("Expecting value", pystr, char_idx, vm)), + }; + + match first_byte { + b'"' => { + // String - pass slice starting after the quote + let (wtf8_result, chars_consumed, bytes_consumed) = + machinery::scanstring(&wtf8[byte_idx + 1..], char_idx + 1, self.strict) + .map_err(|e| py_decode_error(e, pystr.clone().into_wtf8(), vm))?; + let py_str = vm.ctx.new_str(wtf8_result.to_string()); + Ok(( + py_str.into(), + char_idx + 1 + chars_consumed, + byte_idx + 1 + bytes_consumed, + )) + } + b'{' => { + // Object + self.parse_object(pystr, char_idx + 1, byte_idx + 1, scan_once, memo, vm) + } + b'[' => { + // Array + self.parse_array(pystr, char_idx + 1, byte_idx + 1, scan_once, memo, vm) + } + b'n' if starts_with_bytes(&bytes[byte_idx..], b"null") => { + // null + Ok((vm.ctx.none(), char_idx + 4, byte_idx + 4)) + } + b't' if starts_with_bytes(&bytes[byte_idx..], b"true") => { + // true + Ok((vm.ctx.new_bool(true).into(), char_idx + 4, byte_idx + 4)) + } + b'f' if starts_with_bytes(&bytes[byte_idx..], b"false") => { + // false + Ok((vm.ctx.new_bool(false).into(), char_idx + 5, byte_idx + 5)) + } + b'N' if starts_with_bytes(&bytes[byte_idx..], b"NaN") => { + // NaN + let result = self.parse_constant.call(("NaN",), vm)?; + Ok((result, char_idx + 3, byte_idx + 3)) + } + b'I' if starts_with_bytes(&bytes[byte_idx..], b"Infinity") => { + // Infinity + let result = self.parse_constant.call(("Infinity",), vm)?; + Ok((result, char_idx + 8, byte_idx + 8)) + } + b'-' => { + // -Infinity or negative number + if starts_with_bytes(&bytes[byte_idx..], b"-Infinity") { + let result = self.parse_constant.call(("-Infinity",), vm)?; + return Ok((result, char_idx + 9, byte_idx + 9)); + } + // Negative number - numbers are ASCII so len == bytes + if let Some((result, len)) = self.parse_number(&s[byte_idx..], vm) { + return Ok((result?, char_idx + len, byte_idx + len)); + } + Err(self.make_decode_error("Expecting value", pystr, char_idx, vm)) + } + b'0'..=b'9' => { + // Positive number - numbers are ASCII so len == bytes + if let Some((result, len)) = self.parse_number(&s[byte_idx..], vm) { + return Ok((result?, char_idx + len, byte_idx + len)); + } + Err(self.make_decode_error("Expecting value", pystr, char_idx, vm)) + } + _ => { + // Fall back to scan_once for unrecognized input + // Note: This path requires char_idx for Python compatibility + let result = scan_once.call((pystr.clone(), char_idx as isize), vm); + + match result { + Ok(tuple) => { + use crate::vm::builtins::PyTupleRef; + let tuple: PyTupleRef = tuple.try_into_value(vm)?; + if tuple.len() != 2 { + return Err(vm.new_value_error("scan_once must return 2-tuple")); + } + let value = tuple.as_slice()[0].clone(); + let end_char_idx: isize = tuple.as_slice()[1].try_to_value(vm)?; + // For fallback, we need to calculate byte_idx from char_idx + // This is expensive but fallback should be rare + let end_byte_idx = s + .char_indices() + .nth(end_char_idx as usize) + .map(|(i, _)| i) + .unwrap_or(s.len()); + Ok((value, end_char_idx as usize, end_byte_idx)) + } + Err(err) if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) => { + Err(self.make_decode_error("Expecting value", pystr, char_idx, vm)) + } + Err(err) => Err(err), + } + } + } + } + + /// Create a decode error. + fn make_decode_error( + &self, + msg: &str, + s: PyUtf8StrRef, + pos: usize, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + let err = machinery::DecodeError::new(msg, pos); + py_decode_error(err, s.into_wtf8(), vm) + } + } + + impl Callable for JsonScanner { + type Args = (PyStrRef, isize); + fn call(zelf: &Py<Self>, (pystr, char_idx): Self::Args, vm: &VirtualMachine) -> PyResult { + if char_idx < 0 { + return Err(vm.new_value_error("idx cannot be negative")); + } + let char_idx = char_idx as usize; + let pystr = pystr.try_into_utf8(vm)?; + let s = pystr.as_str(); + + // Calculate byte index from char index (O(char_idx) but only at entry point) + let byte_idx = if char_idx == 0 { + 0 + } else { + match s.char_indices().nth(char_idx) { + Some((byte_i, _)) => byte_i, + None => { + // char_idx is beyond the string length + return PyIterReturn::StopIteration(Some(vm.ctx.new_int(char_idx).into())) + .to_pyresult(vm); + } + } + }; + + zelf.parse(pystr, char_idx, byte_idx, zelf.to_owned().into(), vm) + .and_then(|x| x.to_pyresult(vm)) + } + } + + fn encode_string(s: &str, ascii_only: bool) -> String { + flame_guard!("_json::encode_string"); + let mut buf = Vec::<u8>::with_capacity(s.len() + 2); + machinery::write_json_string(s, ascii_only, &mut buf) + // SAFETY: writing to a vec can't fail + .unwrap_or_else(|_| unsafe { core::hint::unreachable_unchecked() }); + // SAFETY: we only output valid utf8 from write_json_string + unsafe { String::from_utf8_unchecked(buf) } + } + + #[pyfunction] + fn encode_basestring(s: PyUtf8StrRef) -> String { + encode_string(s.as_str(), false) + } + + #[pyfunction] + fn encode_basestring_ascii(s: PyUtf8StrRef) -> String { + encode_string(s.as_str(), true) + } + + fn py_decode_error( + e: machinery::DecodeError, + s: PyStrRef, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + let get_error = || -> PyResult<_> { + let cls = vm.try_class("json", "JSONDecodeError")?; + let exc = PyType::call(&cls, (e.msg, s, e.pos).into_args(vm), vm)?; + exc.try_into_value(vm) + }; + match get_error() { + Ok(x) | Err(x) => x, + } + } + + #[pyfunction] + fn scanstring( + s: PyStrRef, + end: usize, + strict: OptionalArg<bool>, + vm: &VirtualMachine, + ) -> PyResult<(Wtf8Buf, usize)> { + flame_guard!("_json::scanstring"); + let wtf8 = s.as_wtf8(); + + // Convert char index `end` to byte index + let byte_idx = if end == 0 { + 0 + } else { + wtf8.code_point_indices() + .nth(end) + .map(|(i, _)| i) + .ok_or_else(|| { + py_decode_error( + machinery::DecodeError::new("Unterminated string starting at", end - 1), + s.clone(), + vm, + ) + })? + }; + + let (result, chars_consumed, _bytes_consumed) = + machinery::scanstring(&wtf8[byte_idx..], end, strict.unwrap_or(true)) + .map_err(|e| py_decode_error(e, s, vm))?; + + Ok((result, end + chars_consumed)) + } +} diff --git a/crates/stdlib/src/json/machinery.rs b/crates/stdlib/src/json/machinery.rs new file mode 100644 index 00000000000..2102d437396 --- /dev/null +++ b/crates/stdlib/src/json/machinery.rs @@ -0,0 +1,271 @@ +// spell-checker:ignore LOJKINE +// derived from https://github.com/lovasoa/json_in_type + +// BSD 2-Clause License +// +// Copyright (c) 2018, Ophir LOJKINE +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use std::io; + +use itertools::Itertools; +use memchr::memchr2; +use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; + +static ESCAPE_CHARS: [&str; 0x20] = [ + "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", "\\u0007", "\\b", + "\\t", "\\n", "\\u000", "\\f", "\\r", "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", + "\\u0013", "\\u0014", "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", + "\\u001", "\\u001c", "\\u001d", "\\u001e", "\\u001f", +]; + +// This bitset represents which bytes can be copied as-is to a JSON string (0) +// And which one need to be escaped (1) +// The characters that need escaping are 0x00 to 0x1F, 0x22 ("), 0x5C (\), 0x7F (DEL) +// Non-ASCII unicode characters can be safely included in a JSON string +#[allow( + clippy::unusual_byte_groupings, + reason = "groups of 16 are intentional here" +)] +static NEEDS_ESCAPING_BITSET: [u64; 4] = [ + //fedcba9876543210_fedcba9876543210_fedcba9876543210_fedcba9876543210 + 0b0000000000000000_0000000000000100_1111111111111111_1111111111111111, // 3_2_1_0 + 0b1000000000000000_0000000000000000_0001000000000000_0000000000000000, // 7_6_5_4 + 0b0000000000000000_0000000000000000_0000000000000000_0000000000000000, // B_A_9_8 + 0b0000000000000000_0000000000000000_0000000000000000_0000000000000000, // F_E_D_C +]; + +#[inline(always)] +fn json_escaped_char(c: u8) -> Option<&'static str> { + let bitset_value = NEEDS_ESCAPING_BITSET[(c / 64) as usize] & (1 << (c % 64)); + if bitset_value == 0 { + None + } else { + Some(match c { + x if x < 0x20 => ESCAPE_CHARS[c as usize], + b'\\' => "\\\\", + b'\"' => "\\\"", + 0x7F => "\\u007f", + _ => unreachable!(), + }) + } +} + +pub fn write_json_string<W: io::Write>(s: &str, ascii_only: bool, w: &mut W) -> io::Result<()> { + w.write_all(b"\"")?; + let mut write_start_idx = 0; + let bytes = s.as_bytes(); + if ascii_only { + for (idx, c) in s.char_indices() { + if c.is_ascii() { + if let Some(escaped) = json_escaped_char(c as u8) { + w.write_all(&bytes[write_start_idx..idx])?; + w.write_all(escaped.as_bytes())?; + write_start_idx = idx + 1; + } + } else { + w.write_all(&bytes[write_start_idx..idx])?; + write_start_idx = idx + c.len_utf8(); + // codepoints outside the BMP get 2 '\uxxxx' sequences to represent them + for point in c.encode_utf16(&mut [0; 2]) { + write!(w, "\\u{point:04x}")?; + } + } + } + } else { + for (idx, c) in s.bytes().enumerate() { + if let Some(escaped) = json_escaped_char(c) { + w.write_all(&bytes[write_start_idx..idx])?; + w.write_all(escaped.as_bytes())?; + write_start_idx = idx + 1; + } + } + } + w.write_all(&bytes[write_start_idx..])?; + w.write_all(b"\"") +} + +#[derive(Debug)] +pub struct DecodeError { + pub msg: String, + pub pos: usize, +} +impl DecodeError { + pub fn new(msg: impl Into<String>, pos: usize) -> Self { + let msg = msg.into(); + Self { msg, pos } + } +} + +enum StrOrChar<'a> { + Str(&'a Wtf8), + Char(CodePoint), +} +impl StrOrChar<'_> { + const fn len(&self) -> usize { + match self { + StrOrChar::Str(s) => s.len(), + StrOrChar::Char(c) => c.len_wtf8(), + } + } +} +/// Scan a JSON string starting right after the opening quote. +/// +/// # Arguments +/// * `s` - The string slice starting at the first character after the opening `"` +/// * `char_offset` - The character index where this slice starts (for error messages) +/// * `strict` - Whether to reject control characters +/// +/// # Returns +/// * `Ok((result, chars_consumed, bytes_consumed))` - The decoded string and how much was consumed +/// * `Err(DecodeError)` - If the string is malformed +pub fn scanstring<'a>( + s: &'a Wtf8, + char_offset: usize, + strict: bool, +) -> Result<(Wtf8Buf, usize, usize), DecodeError> { + flame_guard!("machinery::scanstring"); + let unterminated_err = || DecodeError::new("Unterminated string starting at", char_offset - 1); + + let bytes = s.as_bytes(); + + // Fast path: use memchr to find " or \ quickly + if let Some(pos) = memchr2(b'"', b'\\', bytes) + && bytes[pos] == b'"' + { + let content_bytes = &bytes[..pos]; + + // In strict mode, check for control characters (0x00-0x1F) + let has_control_char = strict && content_bytes.iter().any(|&b| b < 0x20); + + if !has_control_char { + flame_guard!("machinery::scanstring::fast_path"); + let result_slice = &s[..pos]; + let char_count = result_slice.code_points().count(); + let mut out = Wtf8Buf::with_capacity(pos); + out.push_wtf8(result_slice); + // +1 for the closing quote + return Ok((out, char_count + 1, pos + 1)); + } + } + + // Slow path: chunk-based parsing for strings with escapes or control chars + flame_guard!("machinery::scanstring::slow_path"); + let mut chunks: Vec<StrOrChar<'a>> = Vec::new(); + let mut output_len = 0usize; + let mut push_chunk = |chunk: StrOrChar<'a>| { + output_len += chunk.len(); + chunks.push(chunk); + }; + + let mut chars = s.code_point_indices().enumerate().peekable(); + let mut chunk_start: usize = 0; + + while let Some((char_i, (byte_i, c))) = chars.next() { + match c.to_char_lossy() { + '"' => { + push_chunk(StrOrChar::Str(&s[chunk_start..byte_i])); + flame_guard!("machinery::scanstring::assemble_chunks"); + let mut out = Wtf8Buf::with_capacity(output_len); + for x in chunks { + match x { + StrOrChar::Str(s) => out.push_wtf8(s), + StrOrChar::Char(c) => out.push(c), + } + } + // +1 for the closing quote + return Ok((out, char_i + 1, byte_i + 1)); + } + '\\' => { + push_chunk(StrOrChar::Str(&s[chunk_start..byte_i])); + let (next_char_i, (_, c)) = chars.next().ok_or_else(unterminated_err)?; + let esc = match c.to_char_lossy() { + '"' => "\"", + '\\' => "\\", + '/' => "/", + 'b' => "\x08", + 'f' => "\x0c", + 'n' => "\n", + 'r' => "\r", + 't' => "\t", + 'u' => { + let mut uni = decode_unicode(&mut chars, char_offset + char_i)?; + chunk_start = byte_i + 6; + if let Some(lead) = uni.to_lead_surrogate() { + // uni is a surrogate -- try to find its pair + let mut chars2 = chars.clone(); + if let Some(((_, (byte_pos2, _)), (_, _))) = chars2 + .next_tuple() + .filter(|((_, (_, c1)), (_, (_, c2)))| *c1 == '\\' && *c2 == 'u') + { + let uni2 = + decode_unicode(&mut chars2, char_offset + next_char_i + 1)?; + if let Some(trail) = uni2.to_trail_surrogate() { + // ok, we found what we were looking for -- \uXXXX\uXXXX, both surrogates + uni = lead.merge(trail).into(); + chunk_start = byte_pos2 + 6; + chars = chars2; + } + } + } + push_chunk(StrOrChar::Char(uni)); + continue; + } + _ => { + return Err(DecodeError::new( + format!("Invalid \\escape: {c:?}"), + char_offset + char_i, + )); + } + }; + chunk_start = byte_i + 2; + push_chunk(StrOrChar::Str(esc.as_ref())); + } + '\x00'..='\x1f' if strict => { + return Err(DecodeError::new( + format!("Invalid control character {c:?} at"), + char_offset + char_i, + )); + } + _ => {} + } + } + Err(unterminated_err()) +} + +#[inline] +fn decode_unicode<I>(it: &mut I, pos: usize) -> Result<CodePoint, DecodeError> +where + I: Iterator<Item = (usize, (usize, CodePoint))>, +{ + flame_guard!("machinery::decode_unicode"); + let err = || DecodeError::new("Invalid \\uXXXX escape", pos); + let mut uni = 0u16; + for _ in 0..4 { + let (_, (_, c)) = it.next().ok_or_else(err)?; + let d = c.to_char().and_then(|c| c.to_digit(16)).ok_or_else(err)? as u16; + uni = (uni << 4) | d; + } + Ok(uni.into()) +} diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs new file mode 100644 index 00000000000..941de413bfc --- /dev/null +++ b/crates/stdlib/src/lib.rs @@ -0,0 +1,255 @@ +// to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of +// how `mod` works, but we want this sometimes for pymodule declarations + +#![allow(clippy::module_inception)] + +#[macro_use] +extern crate rustpython_derive; +extern crate alloc; + +#[macro_use] +pub(crate) mod macros; + +mod _asyncio; +mod _remote_debugging; +pub mod array; +mod binascii; +mod bisect; +mod bz2; +mod cmath; +mod compression; // internal module +mod contextvars; +mod csv; +#[cfg(not(any(target_os = "android", target_arch = "wasm32")))] +mod lzma; +mod zlib; + +mod blake2; +mod hashlib; +mod md5; +mod sha1; +mod sha256; +mod sha3; +mod sha512; + +mod json; + +#[cfg(all( + feature = "host_env", + not(any(target_os = "ios", target_arch = "wasm32")) +))] +mod locale; + +mod _opcode; +#[path = "_tokenize.rs"] +mod _tokenize; +mod math; +#[cfg(all(feature = "host_env", any(unix, windows)))] +mod mmap; +mod pyexpat; +mod pystruct; +mod random; +mod statistics; +mod suggestions; +// TODO: maybe make this an extension module, if we ever get those +// mod re; +#[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] +pub mod socket; +#[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] +mod syslog; +mod unicodedata; + +#[cfg(feature = "host_env")] +mod faulthandler; +#[cfg(all(feature = "host_env", any(unix, target_os = "wasi")))] +mod fcntl; +#[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] +mod multiprocessing; +#[cfg(all( + feature = "host_env", + unix, + not(target_os = "redox"), + not(target_os = "android") +))] +mod posixshmem; +#[cfg(all(feature = "host_env", unix))] +mod posixsubprocess; +// libc is missing constants on redox +#[cfg(all( + feature = "sqlite", + not(any(target_os = "android", target_arch = "wasm32")) +))] +mod _sqlite3; +#[cfg(all(feature = "host_env", windows))] +mod _testconsole; +#[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "android", target_os = "redox")) +))] +mod grp; +#[cfg(all(feature = "host_env", windows))] +mod overlapped; +#[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] +mod resource; +#[cfg(all(feature = "host_env", target_os = "macos"))] +mod scproxy; +#[cfg(all(feature = "host_env", any(unix, windows, target_os = "wasi")))] +mod select; + +#[cfg(all( + feature = "host_env", + not(target_arch = "wasm32"), + feature = "ssl-openssl" +))] +mod openssl; +#[cfg(all( + feature = "host_env", + not(target_arch = "wasm32"), + feature = "ssl-rustls" +))] +mod ssl; +#[cfg(all(feature = "ssl-openssl", feature = "ssl-rustls"))] +compile_error!("features \"ssl-openssl\" and \"ssl-rustls\" are mutually exclusive"); + +#[cfg(all( + feature = "host_env", + unix, + not(target_os = "redox"), + not(target_os = "ios") +))] +mod termios; +#[cfg(all( + feature = "host_env", + not(any( + target_os = "android", + target_os = "ios", + target_os = "windows", + target_arch = "wasm32", + target_os = "redox", + )) +))] +mod uuid; + +#[cfg(all(feature = "host_env", feature = "tkinter"))] +mod tkinter; + +use rustpython_common as common; +use rustpython_vm as vm; + +use crate::vm::{Context, builtins}; + +/// Returns module definitions for multi-phase init modules. +/// These modules are added to sys.modules BEFORE their exec function runs, +/// allowing safe circular imports. +pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> { + vec![ + _asyncio::module_def(ctx), + _opcode::module_def(ctx), + _remote_debugging::module_def(ctx), + array::module_def(ctx), + binascii::module_def(ctx), + bisect::module_def(ctx), + blake2::module_def(ctx), + bz2::module_def(ctx), + cmath::module_def(ctx), + contextvars::module_def(ctx), + csv::module_def(ctx), + #[cfg(feature = "host_env")] + faulthandler::module_def(ctx), + #[cfg(all(feature = "host_env", any(unix, target_os = "wasi")))] + fcntl::module_def(ctx), + #[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "android", target_os = "redox")) + ))] + grp::module_def(ctx), + hashlib::module_def(ctx), + json::module_def(ctx), + #[cfg(all( + feature = "host_env", + not(any(target_os = "ios", target_arch = "wasm32")) + ))] + locale::module_def(ctx), + #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] + lzma::module_def(ctx), + math::module_def(ctx), + md5::module_def(ctx), + #[cfg(all(feature = "host_env", any(unix, windows)))] + mmap::module_def(ctx), + #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] + multiprocessing::module_def(ctx), + #[cfg(all( + feature = "host_env", + not(target_arch = "wasm32"), + feature = "ssl-openssl" + ))] + openssl::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + _testconsole::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + overlapped::module_def(ctx), + #[cfg(all(feature = "host_env", unix))] + posixsubprocess::module_def(ctx), + #[cfg(all( + feature = "host_env", + unix, + not(target_os = "redox"), + not(target_os = "android") + ))] + posixshmem::module_def(ctx), + pyexpat::module_def(ctx), + pystruct::module_def(ctx), + random::module_def(ctx), + #[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] + resource::module_def(ctx), + #[cfg(all(feature = "host_env", target_os = "macos"))] + scproxy::module_def(ctx), + #[cfg(all(feature = "host_env", any(unix, windows, target_os = "wasi")))] + select::module_def(ctx), + sha1::module_def(ctx), + sha256::module_def(ctx), + sha3::module_def(ctx), + sha512::module_def(ctx), + #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] + socket::module_def(ctx), + #[cfg(all( + feature = "sqlite", + not(any(target_os = "android", target_arch = "wasm32")) + ))] + _sqlite3::module_def(ctx), + #[cfg(all( + feature = "host_env", + not(target_arch = "wasm32"), + feature = "ssl-rustls" + ))] + ssl::module_def(ctx), + statistics::module_def(ctx), + suggestions::module_def(ctx), + _tokenize::module_def(ctx), + #[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] + syslog::module_def(ctx), + #[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "ios", target_os = "redox")) + ))] + termios::module_def(ctx), + #[cfg(all(feature = "host_env", feature = "tkinter"))] + tkinter::module_def(ctx), + unicodedata::module_def(ctx), + #[cfg(all( + feature = "host_env", + not(any( + target_os = "android", + target_os = "ios", + target_os = "windows", + target_arch = "wasm32", + target_os = "redox" + )) + ))] + uuid::module_def(ctx), + zlib::module_def(ctx), + ] +} diff --git a/crates/stdlib/src/locale.rs b/crates/stdlib/src/locale.rs new file mode 100644 index 00000000000..a22c6afe57e --- /dev/null +++ b/crates/stdlib/src/locale.rs @@ -0,0 +1,332 @@ +// spell-checker:ignore abday abmon yesexpr noexpr CRNCYSTR RADIXCHAR AMPM THOUSEP + +pub(crate) use _locale::module_def; + +#[cfg(windows)] +#[repr(C)] +struct lconv { + decimal_point: *mut libc::c_char, + thousands_sep: *mut libc::c_char, + grouping: *mut libc::c_char, + int_curr_symbol: *mut libc::c_char, + currency_symbol: *mut libc::c_char, + mon_decimal_point: *mut libc::c_char, + mon_thousands_sep: *mut libc::c_char, + mon_grouping: *mut libc::c_char, + positive_sign: *mut libc::c_char, + negative_sign: *mut libc::c_char, + int_frac_digits: libc::c_char, + frac_digits: libc::c_char, + p_cs_precedes: libc::c_char, + p_sep_by_space: libc::c_char, + n_cs_precedes: libc::c_char, + n_sep_by_space: libc::c_char, + p_sign_posn: libc::c_char, + n_sign_posn: libc::c_char, + int_p_cs_precedes: libc::c_char, + int_p_sep_by_space: libc::c_char, + int_n_cs_precedes: libc::c_char, + int_n_sep_by_space: libc::c_char, + int_p_sign_posn: libc::c_char, + int_n_sign_posn: libc::c_char, +} + +#[cfg(windows)] +unsafe extern "C" { + fn localeconv() -> *mut lconv; +} + +#[cfg(unix)] +use libc::localeconv; + +#[pymodule] +mod _locale { + use alloc::ffi::CString; + use core::{ffi::CStr, ptr}; + use rustpython_vm::{ + PyObjectRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyIntRef, PyListRef, PyTypeRef, PyUtf8StrRef}, + convert::ToPyException, + function::OptionalArg, + }; + #[cfg(windows)] + use windows_sys::Win32::Globalization::GetACP; + + #[cfg(all( + unix, + not(any(target_os = "ios", target_os = "android", target_os = "redox")) + ))] + #[pyattr] + use libc::{ + ABDAY_1, ABDAY_2, ABDAY_3, ABDAY_4, ABDAY_5, ABDAY_6, ABDAY_7, ABMON_1, ABMON_2, ABMON_3, + ABMON_4, ABMON_5, ABMON_6, ABMON_7, ABMON_8, ABMON_9, ABMON_10, ABMON_11, ABMON_12, + ALT_DIGITS, AM_STR, CODESET, CRNCYSTR, D_FMT, D_T_FMT, DAY_1, DAY_2, DAY_3, DAY_4, DAY_5, + DAY_6, DAY_7, ERA, ERA_D_FMT, ERA_D_T_FMT, ERA_T_FMT, MON_1, MON_2, MON_3, MON_4, MON_5, + MON_6, MON_7, MON_8, MON_9, MON_10, MON_11, MON_12, NOEXPR, PM_STR, RADIXCHAR, T_FMT, + T_FMT_AMPM, THOUSEP, YESEXPR, + }; + + #[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))] + #[pyattr] + use libc::LC_MESSAGES; + + #[pyattr] + use libc::{LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, LC_NUMERIC, LC_TIME}; + + #[pyattr(name = "CHAR_MAX")] + fn char_max(vm: &VirtualMachine) -> PyIntRef { + vm.ctx.new_int(libc::c_char::MAX) + } + + unsafe fn copy_grouping(group: *const libc::c_char, vm: &VirtualMachine) -> PyListRef { + let mut group_vec: Vec<PyObjectRef> = Vec::new(); + if group.is_null() { + return vm.ctx.new_list(group_vec); + } + + unsafe { + let mut ptr = group; + while ![0, libc::c_char::MAX].contains(&*ptr) { + let val = vm.ctx.new_int(*ptr); + group_vec.push(val.into()); + ptr = ptr.add(1); + } + } + // https://github.com/python/cpython/blob/677320348728ce058fa3579017e985af74a236d4/Modules/_localemodule.c#L80 + if !group_vec.is_empty() { + group_vec.push(vm.ctx.new_int(0).into()); + } + vm.ctx.new_list(group_vec) + } + + unsafe fn pystr_from_raw_cstr(vm: &VirtualMachine, raw_ptr: *const libc::c_char) -> PyResult { + let slice = unsafe { CStr::from_ptr(raw_ptr) }; + + // Fast path: ASCII/UTF-8 + if let Ok(s) = slice.to_str() { + return Ok(vm.new_pyobj(s)); + } + + // On Windows, locale strings use the ANSI code page encoding + #[cfg(windows)] + { + use windows_sys::Win32::Globalization::{CP_ACP, MultiByteToWideChar}; + let bytes = slice.to_bytes(); + unsafe { + let len = MultiByteToWideChar( + CP_ACP, + 0, + bytes.as_ptr(), + bytes.len() as i32, + ptr::null_mut(), + 0, + ); + if len > 0 { + let mut wide = vec![0u16; len as usize]; + MultiByteToWideChar( + CP_ACP, + 0, + bytes.as_ptr(), + bytes.len() as i32, + wide.as_mut_ptr(), + len, + ); + return Ok(vm.new_pyobj(String::from_utf16_lossy(&wide))); + } + } + } + + Ok(vm.new_pyobj(String::from_utf8_lossy(slice.to_bytes()).into_owned())) + } + + #[pyattr(name = "Error", once)] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "locale", + "Error", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + #[pyfunction] + fn strcoll(string1: PyUtf8StrRef, string2: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + let cstr1 = CString::new(string1.as_str()).map_err(|e| e.to_pyexception(vm))?; + let cstr2 = CString::new(string2.as_str()).map_err(|e| e.to_pyexception(vm))?; + Ok(vm.new_pyobj(unsafe { libc::strcoll(cstr1.as_ptr(), cstr2.as_ptr()) })) + } + + #[pyfunction] + fn strxfrm(string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + // https://github.com/python/cpython/blob/eaae563b6878aa050b4ad406b67728b6b066220e/Modules/_localemodule.c#L390-L442 + let n1 = string.byte_len() + 1; + let mut buff = vec![0u8; n1]; + + let cstr = CString::new(string.as_str()).map_err(|e| e.to_pyexception(vm))?; + let n2 = unsafe { libc::strxfrm(buff.as_mut_ptr() as _, cstr.as_ptr(), n1) }; + buff = vec![0u8; n2 + 1]; + unsafe { + libc::strxfrm(buff.as_mut_ptr() as _, cstr.as_ptr(), n2 + 1); + } + Ok(vm.new_pyobj(String::from_utf8(buff).expect("strxfrm returned invalid utf-8 string"))) + } + + #[pyfunction] + fn localeconv(vm: &VirtualMachine) -> PyResult<PyDictRef> { + let result = vm.ctx.new_dict(); + + unsafe { + macro_rules! set_string_field { + ($lc:expr, $field:ident) => {{ + result.set_item( + stringify!($field), + pystr_from_raw_cstr(vm, (*$lc).$field)?, + vm, + )? + }}; + } + + macro_rules! set_int_field { + ($lc:expr, $field:ident) => {{ result.set_item(stringify!($field), vm.new_pyobj((*$lc).$field), vm)? }}; + } + + macro_rules! set_group_field { + ($lc:expr, $field:ident) => {{ + result.set_item( + stringify!($field), + copy_grouping((*$lc).$field, vm).into(), + vm, + )? + }}; + } + + let lc = super::localeconv(); + set_group_field!(lc, mon_grouping); + set_group_field!(lc, grouping); + set_int_field!(lc, int_frac_digits); + set_int_field!(lc, frac_digits); + set_int_field!(lc, p_cs_precedes); + set_int_field!(lc, p_sep_by_space); + set_int_field!(lc, n_cs_precedes); + set_int_field!(lc, p_sign_posn); + set_int_field!(lc, n_sign_posn); + set_string_field!(lc, decimal_point); + set_string_field!(lc, thousands_sep); + set_string_field!(lc, int_curr_symbol); + set_string_field!(lc, currency_symbol); + set_string_field!(lc, mon_decimal_point); + set_string_field!(lc, mon_thousands_sep); + set_int_field!(lc, n_sep_by_space); + set_string_field!(lc, positive_sign); + set_string_field!(lc, negative_sign); + } + Ok(result) + } + + #[derive(FromArgs)] + struct LocaleArgs { + #[pyarg(any)] + category: i32, + #[pyarg(any, optional)] + locale: OptionalArg<Option<PyUtf8StrRef>>, + } + + /// Maximum code page encoding name length on Windows + #[cfg(windows)] + const MAX_CP_LEN: usize = 15; + + /// Check if the encoding part of a locale string is too long (Windows only) + #[cfg(windows)] + fn check_locale_name(locale: &str) -> bool { + if let Some(dot_pos) = locale.find('.') { + let encoding_part = &locale[dot_pos + 1..]; + // Find the end of encoding (could be followed by '@' modifier) + let encoding_len = encoding_part.find('@').unwrap_or(encoding_part.len()); + encoding_len <= MAX_CP_LEN + } else { + true + } + } + + /// Check locale names for LC_ALL (handles semicolon-separated locales) + #[cfg(windows)] + fn check_locale_name_all(locale: &str) -> bool { + for part in locale.split(';') { + if !check_locale_name(part) { + return false; + } + } + true + } + + #[pyfunction] + fn setlocale(args: LocaleArgs, vm: &VirtualMachine) -> PyResult { + let error = error(vm); + if cfg!(windows) && (args.category < LC_ALL || args.category > LC_TIME) { + return Err(vm.new_exception_msg(error, "unsupported locale setting".into())); + } + unsafe { + let result = match args.locale.flatten() { + None => libc::setlocale(args.category, ptr::null()), + Some(locale) => { + let locale_str = locale.as_str(); + // On Windows, validate encoding name length + #[cfg(windows)] + { + let valid = if args.category == LC_ALL { + check_locale_name_all(locale_str) + } else { + check_locale_name(locale_str) + }; + if !valid { + return Err( + vm.new_exception_msg(error, "unsupported locale setting".into()) + ); + } + } + let c_locale: CString = + CString::new(locale_str).map_err(|e| e.to_pyexception(vm))?; + libc::setlocale(args.category, c_locale.as_ptr()) + } + }; + if result.is_null() { + return Err(vm.new_exception_msg(error, "unsupported locale setting".into())); + } + pystr_from_raw_cstr(vm, result) + } + } + + /// Get the current locale encoding. + #[pyfunction] + fn getencoding() -> String { + #[cfg(windows)] + { + // On Windows, use GetACP() to get the ANSI code page + let acp = unsafe { GetACP() }; + format!("cp{}", acp) + } + #[cfg(not(windows))] + { + // On Unix, use nl_langinfo(CODESET) or fallback to UTF-8 + #[cfg(all( + unix, + not(any(target_os = "ios", target_os = "android", target_os = "redox")) + ))] + { + unsafe { + let codeset = libc::nl_langinfo(libc::CODESET); + if !codeset.is_null() + && let Ok(s) = CStr::from_ptr(codeset).to_str() + && !s.is_empty() + { + return s.to_string(); + } + } + "UTF-8".to_string() + } + #[cfg(any(target_os = "ios", target_os = "android", target_os = "redox"))] + { + "UTF-8".to_string() + } + } + } +} diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs new file mode 100644 index 00000000000..b98cfb1bf94 --- /dev/null +++ b/crates/stdlib/src/lzma.rs @@ -0,0 +1,850 @@ +// spell-checker:ignore ARMTHUMB memlimit + +pub(crate) use _lzma::module_def; + +#[pymodule] +mod _lzma { + use crate::compression::{ + CompressFlushKind, CompressState, CompressStatusKind, Compressor, DecompressArgs, + DecompressError, DecompressState, DecompressStatus, Decompressor, + }; + use alloc::fmt; + use liblzma::stream::{ + Action, Check, Error, Filters, LzmaOptions, MatchFinder, Mode, Status, Stream, + TELL_ANY_CHECK, TELL_NO_CHECK, + }; + // lzma_check, lzma_mode, lzma_match_finder have platform-dependent signedness + // (i32 on Windows, u32 elsewhere). Define as fixed-type const to avoid mismatch. + #[pyattr] + use liblzma_sys::{ + LZMA_FILTER_ARM as FILTER_ARM, LZMA_FILTER_ARMTHUMB as FILTER_ARMTHUMB, + LZMA_FILTER_DELTA as FILTER_DELTA, LZMA_FILTER_IA64 as FILTER_IA64, + LZMA_FILTER_LZMA1 as FILTER_LZMA1, LZMA_FILTER_LZMA2 as FILTER_LZMA2, + LZMA_FILTER_POWERPC as FILTER_POWERPC, LZMA_FILTER_SPARC as FILTER_SPARC, + LZMA_FILTER_X86 as FILTER_X86, + }; + #[pyattr] + use liblzma_sys::{ + LZMA_PRESET_DEFAULT as PRESET_DEFAULT, LZMA_PRESET_EXTREME as PRESET_EXTREME, + }; + use rustpython_common::lock::PyMutex; + use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytesRef, PyDict, PyType, PyTypeRef}; + use rustpython_vm::convert::ToPyException; + use rustpython_vm::function::ArgBytesLike; + use rustpython_vm::types::Constructor; + use rustpython_vm::{Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; + + const BUFSIZ: usize = 8192; + + // liblzma_sys enum types have platform-dependent signedness; `as _` normalizes to i32 + #[pyattr] + const CHECK_NONE: i32 = liblzma_sys::LZMA_CHECK_NONE as _; + #[pyattr] + const CHECK_CRC32: i32 = liblzma_sys::LZMA_CHECK_CRC32 as _; + #[pyattr] + const CHECK_CRC64: i32 = liblzma_sys::LZMA_CHECK_CRC64 as _; + #[pyattr] + const CHECK_SHA256: i32 = liblzma_sys::LZMA_CHECK_SHA256 as _; + #[pyattr] + const CHECK_ID_MAX: i32 = 15; + #[pyattr] + const CHECK_UNKNOWN: i32 = CHECK_ID_MAX + 1; + + #[pyattr] + const MF_HC3: i32 = liblzma_sys::LZMA_MF_HC3 as _; + #[pyattr] + const MF_HC4: i32 = liblzma_sys::LZMA_MF_HC4 as _; + #[pyattr] + const MF_BT2: i32 = liblzma_sys::LZMA_MF_BT2 as _; + #[pyattr] + const MF_BT3: i32 = liblzma_sys::LZMA_MF_BT3 as _; + #[pyattr] + const MF_BT4: i32 = liblzma_sys::LZMA_MF_BT4 as _; + + #[pyattr] + const MODE_FAST: i32 = liblzma_sys::LZMA_MODE_FAST as _; + #[pyattr] + const MODE_NORMAL: i32 = liblzma_sys::LZMA_MODE_NORMAL as _; + + enum Format { + Auto = 0, + Xz = 1, + Alone = 2, + Raw = 3, + } + + #[pyattr] + const FORMAT_AUTO: i32 = Format::Auto as i32; + #[pyattr] + const FORMAT_XZ: i32 = Format::Xz as i32; + #[pyattr] + const FORMAT_ALONE: i32 = Format::Alone as i32; + #[pyattr] + const FORMAT_RAW: i32 = Format::Raw as i32; + + #[pyattr(once, name = "LZMAError")] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "lzma", + "LZMAError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + fn new_lzma_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef { + let msg: String = message.into(); + vm.new_exception_msg(vm.class("lzma", "LZMAError"), msg.into()) + } + + fn catch_lzma_error(err: Error, vm: &VirtualMachine) -> PyBaseExceptionRef { + match err { + Error::UnsupportedCheck => new_lzma_error("Unsupported integrity check", vm), + Error::Mem => vm.new_memory_error(""), + Error::MemLimit => new_lzma_error("Memory usage limit exceeded", vm), + Error::Format => new_lzma_error("Input format not supported by decoder", vm), + Error::Options => new_lzma_error("Invalid or unsupported options", vm), + Error::Data => new_lzma_error("Corrupt input data", vm), + Error::Program => new_lzma_error("Internal error", vm), + Error::NoCheck => new_lzma_error("Corrupt input data", vm), + } + } + + fn int_to_check(check: i32) -> Option<Check> { + if check == -1 { + return Some(Check::Crc64); + } + match check { + CHECK_NONE => Some(Check::None), + CHECK_CRC32 => Some(Check::Crc32), + CHECK_CRC64 => Some(Check::Crc64), + CHECK_SHA256 => Some(Check::Sha256), + _ => None, + } + } + + fn u32_to_mode(val: u32) -> Option<Mode> { + match val as i32 { + MODE_FAST => Some(Mode::Fast), + MODE_NORMAL => Some(Mode::Normal), + _ => None, + } + } + + fn u32_to_mf(val: u32) -> Option<MatchFinder> { + match val as i32 { + MF_HC3 => Some(MatchFinder::HashChain3), + MF_HC4 => Some(MatchFinder::HashChain4), + MF_BT2 => Some(MatchFinder::BinaryTree2), + MF_BT3 => Some(MatchFinder::BinaryTree3), + MF_BT4 => Some(MatchFinder::BinaryTree4), + _ => None, + } + } + + struct LzmaStream { + stream: Stream, + check: i32, + header_buf: [u8; 8], + header_collected: u8, + track_header: bool, + } + + impl LzmaStream { + fn new(stream: Stream, check: i32, track_header: bool) -> Self { + Self { + stream, + check, + header_buf: [0u8; 8], + header_collected: 0, + track_header, + } + } + } + + impl Decompressor for LzmaStream { + type Flush = (); + type Status = Status; + type Error = Error; + + fn total_in(&self) -> u64 { + self.stream.total_in() + } + + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + (): Self::Flush, + ) -> Result<Self::Status, Self::Error> { + if self.track_header && self.header_collected < 8 { + let need = (8 - self.header_collected) as usize; + let n = need.min(input.len()); + self.header_buf[self.header_collected as usize..][..n].copy_from_slice(&input[..n]); + self.header_collected += n as u8; + } + + match self.stream.process_vec(input, output, Action::Run) { + Ok(Status::GetCheck) => { + if self.header_collected >= 8 { + self.check = (self.header_buf[7] & 0x0F) as i32; + } + Ok(Status::Ok) + } + Err(Error::NoCheck) => { + self.check = CHECK_NONE; + Ok(Status::Ok) + } + other => other, + } + } + } + + impl DecompressStatus for Status { + fn is_stream_end(&self) -> bool { + *self == Status::StreamEnd + } + } + + fn get_dict_opt_u32( + spec: &PyObjectRef, + key: &str, + vm: &VirtualMachine, + ) -> PyResult<Option<u32>> { + let dict = spec.downcast_ref::<PyDict>().ok_or_else(|| { + vm.new_type_error("Filter specifier must be a dict or dict-like object") + })?; + match dict.get_item_opt(key, vm)? { + Some(obj) => Ok(Some(obj.try_into_value::<u32>(vm)?)), + None => Ok(None), + } + } + + fn get_dict_opt_u64( + spec: &PyObjectRef, + key: &str, + vm: &VirtualMachine, + ) -> PyResult<Option<u64>> { + let dict = spec.downcast_ref::<PyDict>().ok_or_else(|| { + vm.new_type_error("Filter specifier must be a dict or dict-like object") + })?; + match dict.get_item_opt(key, vm)? { + Some(obj) => Ok(Some(obj.try_into_value::<u64>(vm)?)), + None => Ok(None), + } + } + + fn parse_filter_spec_lzma(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult<LzmaOptions> { + let preset = get_dict_opt_u32(spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT); + + let mut opts = LzmaOptions::new_preset(preset) + .map_err(|_| new_lzma_error(format!("Invalid compression preset: {preset}"), vm))?; + + if let Some(v) = get_dict_opt_u32(spec, "dict_size", vm)? { + opts.dict_size(v); + } + if let Some(v) = get_dict_opt_u32(spec, "lc", vm)? { + opts.literal_context_bits(v); + } + if let Some(v) = get_dict_opt_u32(spec, "lp", vm)? { + opts.literal_position_bits(v); + } + if let Some(v) = get_dict_opt_u32(spec, "pb", vm)? { + opts.position_bits(v); + } + if let Some(v) = get_dict_opt_u32(spec, "mode", vm)? { + let mode = u32_to_mode(v) + .ok_or_else(|| vm.new_value_error("Invalid filter specifier for LZMA filter"))?; + opts.mode(mode); + } + if let Some(v) = get_dict_opt_u32(spec, "nice_len", vm)? { + opts.nice_len(v); + } + if let Some(v) = get_dict_opt_u32(spec, "mf", vm)? { + let mf = u32_to_mf(v) + .ok_or_else(|| vm.new_value_error("Invalid filter specifier for LZMA filter"))?; + opts.match_finder(mf); + } + if let Some(v) = get_dict_opt_u32(spec, "depth", vm)? { + opts.depth(v); + } + + Ok(opts) + } + + fn parse_filter_spec_delta(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult<u32> { + let dist = get_dict_opt_u32(spec, "dist", vm)?.unwrap_or(1); + if dist == 0 || dist > 256 { + return Err(vm.new_value_error("Invalid filter specifier for delta filter")); + } + Ok(dist) + } + + fn parse_filter_spec_bcj(spec: &PyObjectRef, vm: &VirtualMachine) -> PyResult<u32> { + Ok(get_dict_opt_u32(spec, "start_offset", vm)?.unwrap_or(0)) + } + + fn add_bcj_filter( + filters: &mut Filters, + filter_id: u64, + start_offset: u32, + ) -> Result<(), Error> { + if start_offset == 0 { + match filter_id { + FILTER_X86 => { + filters.x86(); + } + FILTER_POWERPC => { + filters.powerpc(); + } + FILTER_IA64 => { + filters.ia64(); + } + FILTER_ARM => { + filters.arm(); + } + FILTER_ARMTHUMB => { + filters.arm_thumb(); + } + FILTER_SPARC => { + filters.sparc(); + } + _ => unreachable!(), + } + Ok(()) + } else { + let props = start_offset.to_le_bytes(); + match filter_id { + FILTER_X86 => { + filters.x86_properties(&props)?; + } + FILTER_POWERPC => { + filters.powerpc_properties(&props)?; + } + FILTER_IA64 => { + filters.ia64_properties(&props)?; + } + FILTER_ARM => { + filters.arm_properties(&props)?; + } + FILTER_ARMTHUMB => { + filters.arm_thumb_properties(&props)?; + } + FILTER_SPARC => { + filters.sparc_properties(&props)?; + } + _ => unreachable!(), + } + Ok(()) + } + } + + fn parse_filter_chain_spec( + filter_specs: Vec<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<Filters> { + const LZMA_FILTERS_MAX: usize = 4; + if filter_specs.len() > LZMA_FILTERS_MAX { + return Err(new_lzma_error( + format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"), + vm, + )); + } + + let mut filters = Filters::new(); + for spec in &filter_specs { + let filter_id = get_dict_opt_u64(spec, "id", vm)? + .ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?; + + match filter_id { + FILTER_LZMA1 => { + let opts = parse_filter_spec_lzma(spec, vm)?; + filters.lzma1(&opts); + } + FILTER_LZMA2 => { + let opts = parse_filter_spec_lzma(spec, vm)?; + filters.lzma2(&opts); + } + FILTER_DELTA => { + let dist = parse_filter_spec_delta(spec, vm)?; + filters + .delta_properties(&[(dist - 1) as u8]) + .map_err(|e| catch_lzma_error(e, vm))?; + } + FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB + | FILTER_SPARC => { + let start_offset = parse_filter_spec_bcj(spec, vm)?; + add_bcj_filter(&mut filters, filter_id, start_offset) + .map_err(|e| catch_lzma_error(e, vm))?; + } + _ => { + return Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}"))); + } + } + } + + Ok(filters) + } + + const DEFAULT_LC: u32 = liblzma_sys::LZMA_LC_DEFAULT; + const DEFAULT_LP: u32 = liblzma_sys::LZMA_LP_DEFAULT; + const DEFAULT_PB: u32 = liblzma_sys::LZMA_PB_DEFAULT; + const DICT_POW2: [u8; 10] = [18, 20, 21, 22, 22, 23, 23, 24, 25, 26]; + + fn preset_dict_size(preset: u32) -> u32 { + let level = (preset & liblzma_sys::LZMA_PRESET_LEVEL_MASK) as usize; + if level > 9 { + return 0; + } + 1u32 << DICT_POW2[level] + } + + fn lzma2_dict_size_from_prop(prop: u8) -> u32 { + if prop > 40 { + return u32::MAX; + } + if prop == 40 { + return u32::MAX; + } + let prop = prop as u32; + (2 | (prop & 1)) << (prop / 2 + 11) + } + + fn lzma2_prop_from_dict_size(dict_size: u32) -> u8 { + if dict_size == u32::MAX { + return 40; + } + for i in 0u8..40 { + if lzma2_dict_size_from_prop(i) >= dict_size { + return i; + } + } + 40 + } + + fn encode_lzma1_properties(lc: u32, lp: u32, pb: u32, dict_size: u32) -> Vec<u8> { + let mut result = vec![0u8; 5]; + result[0] = ((pb * 5 + lp) * 9 + lc) as u8; + result[1..5].copy_from_slice(&dict_size.to_le_bytes()); + result + } + + fn decode_lzma1_properties(props: &[u8]) -> Option<(u32, u32, u32, u32)> { + if props.len() < 5 { + return None; + } + let mut d = props[0] as u32; + let lc = d % 9; + d /= 9; + let lp = d % 5; + let pb = d / 5; + let dict_size = u32::from_le_bytes([props[1], props[2], props[3], props[4]]); + Some((lc, lp, pb, dict_size)) + } + + fn build_filter_spec( + filter_id: u64, + props: &[u8], + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let dict = vm.ctx.new_dict(); + dict.set_item("id", vm.new_pyobj(filter_id), vm)?; + + match filter_id { + FILTER_LZMA1 => { + let (lc, lp, pb, dict_size) = decode_lzma1_properties(props) + .ok_or_else(|| new_lzma_error("Invalid or unsupported options", vm))?; + dict.set_item("lc", vm.new_pyobj(lc), vm)?; + dict.set_item("lp", vm.new_pyobj(lp), vm)?; + dict.set_item("pb", vm.new_pyobj(pb), vm)?; + dict.set_item("dict_size", vm.new_pyobj(dict_size), vm)?; + } + FILTER_LZMA2 => { + if props.len() != 1 { + return Err(new_lzma_error("Invalid or unsupported options", vm)); + } + let dict_size = lzma2_dict_size_from_prop(props[0]); + dict.set_item("dict_size", vm.new_pyobj(dict_size), vm)?; + } + FILTER_DELTA => { + if props.len() != 1 { + return Err(new_lzma_error("Invalid or unsupported options", vm)); + } + let dist = props[0] as u32 + 1; + dict.set_item("dist", vm.new_pyobj(dist), vm)?; + } + FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB + | FILTER_SPARC => { + if props.is_empty() { + // default: no start_offset + } else if props.len() == 4 { + let start_offset = u32::from_le_bytes([props[0], props[1], props[2], props[3]]); + dict.set_item("start_offset", vm.new_pyobj(start_offset), vm)?; + } else { + return Err(new_lzma_error("Invalid or unsupported options", vm)); + } + } + _ => { + return Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}"))); + } + } + + Ok(dict.into()) + } + + #[pyfunction] + fn is_check_supported(check_id: i32) -> bool { + unsafe { liblzma_sys::lzma_check_is_supported(check_id as _) != 0 } + } + + #[pyfunction] + fn _encode_filter_properties( + filter_spec: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + let filter_id = get_dict_opt_u64(&filter_spec, "id", vm)? + .ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?; + + match filter_id { + FILTER_LZMA1 => { + let preset = + get_dict_opt_u32(&filter_spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT); + let lc = get_dict_opt_u32(&filter_spec, "lc", vm)?.unwrap_or(DEFAULT_LC); + let lp = get_dict_opt_u32(&filter_spec, "lp", vm)?.unwrap_or(DEFAULT_LP); + let pb = get_dict_opt_u32(&filter_spec, "pb", vm)?.unwrap_or(DEFAULT_PB); + let dict_size = get_dict_opt_u32(&filter_spec, "dict_size", vm)? + .unwrap_or_else(|| preset_dict_size(preset)); + Ok(encode_lzma1_properties(lc, lp, pb, dict_size)) + } + FILTER_LZMA2 => { + let preset = + get_dict_opt_u32(&filter_spec, "preset", vm)?.unwrap_or(PRESET_DEFAULT); + let dict_size = get_dict_opt_u32(&filter_spec, "dict_size", vm)? + .unwrap_or_else(|| preset_dict_size(preset)); + Ok(vec![lzma2_prop_from_dict_size(dict_size)]) + } + FILTER_DELTA => { + let dist = get_dict_opt_u32(&filter_spec, "dist", vm)?.unwrap_or(1); + if dist == 0 || dist > 256 { + return Err(vm.new_value_error("Invalid filter specifier for delta filter")); + } + Ok(vec![(dist - 1) as u8]) + } + FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB + | FILTER_SPARC => { + let start_offset = get_dict_opt_u32(&filter_spec, "start_offset", vm)?.unwrap_or(0); + if start_offset == 0 { + Ok(vec![]) + } else { + Ok(start_offset.to_le_bytes().to_vec()) + } + } + _ => Err(vm.new_value_error(format!("Invalid filter ID: {filter_id}"))), + } + } + + #[pyfunction] + fn _decode_filter_properties( + filter_id: u64, + encoded_props: ArgBytesLike, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let props = encoded_props.borrow_buf(); + build_filter_spec(filter_id, &props, vm) + } + + #[pyattr] + #[pyclass(name = "LZMADecompressor")] + #[derive(PyPayload)] + struct LZMADecompressor { + state: PyMutex<DecompressState<LzmaStream>>, + } + + impl fmt::Debug for LZMADecompressor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "_lzma.LZMADecompressor") + } + } + + #[derive(FromArgs)] + pub struct LZMADecompressorConstructorArgs { + #[pyarg(any, default = FORMAT_AUTO)] + format: i32, + #[pyarg(any, optional)] + memlimit: Option<u64>, + #[pyarg(any, optional)] + filters: Option<Vec<PyObjectRef>>, + } + + impl Constructor for LZMADecompressor { + type Args = LZMADecompressorConstructorArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if args.format == FORMAT_RAW && args.memlimit.is_some() { + return Err(vm.new_value_error("Cannot specify memory limit with FORMAT_RAW")); + } + + if args.format == FORMAT_RAW && args.filters.is_none() { + return Err(vm.new_value_error("Must specify filters for FORMAT_RAW")); + } + if args.format != FORMAT_RAW && args.filters.is_some() { + return Err(vm.new_value_error("Cannot specify filters except with FORMAT_RAW")); + } + + let memlimit = args.memlimit.unwrap_or(u64::MAX); + let decoder_flags = TELL_ANY_CHECK | TELL_NO_CHECK; + + let lzma_stream = match args.format { + FORMAT_AUTO => { + let stream = Stream::new_auto_decoder(memlimit, decoder_flags) + .map_err(|e| catch_lzma_error(e, vm))?; + LzmaStream::new(stream, CHECK_UNKNOWN, true) + } + FORMAT_XZ => { + let stream = Stream::new_stream_decoder(memlimit, decoder_flags) + .map_err(|e| catch_lzma_error(e, vm))?; + LzmaStream::new(stream, CHECK_UNKNOWN, true) + } + FORMAT_ALONE => { + let stream = + Stream::new_lzma_decoder(memlimit).map_err(|e| catch_lzma_error(e, vm))?; + LzmaStream::new(stream, CHECK_NONE, false) + } + FORMAT_RAW => { + let filter_specs = args.filters.unwrap(); // safe: checked above + let filters = parse_filter_chain_spec(filter_specs, vm)?; + let stream = + Stream::new_raw_decoder(&filters).map_err(|e| catch_lzma_error(e, vm))?; + LzmaStream::new(stream, CHECK_NONE, false) + } + _ => { + return Err( + vm.new_value_error(format!("Invalid container format: {}", args.format)) + ); + } + }; + + Ok(Self { + state: PyMutex::new(DecompressState::new(lzma_stream, vm)), + }) + } + } + + #[pyclass(with(Constructor))] + impl LZMADecompressor { + #[pymethod] + fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let max_length = args.max_length(); + let data = &*args.data(); + + let mut state = self.state.lock(); + state + .decompress(data, max_length, BUFSIZ, vm) + .map_err(|e| match e { + DecompressError::Decompress(err) => catch_lzma_error(err, vm), + DecompressError::Eof(err) => err.to_pyexception(vm), + }) + } + + #[pygetset] + fn check(&self) -> i32 { + self.state.lock().decompressor().check + } + + #[pygetset] + fn eof(&self) -> bool { + self.state.lock().eof() + } + + #[pygetset] + fn unused_data(&self) -> PyBytesRef { + self.state.lock().unused_data() + } + + #[pygetset] + fn needs_input(&self) -> bool { + self.state.lock().needs_input() + } + } + + struct CompressorInner { + stream: Stream, + } + + impl CompressStatusKind for Status { + const EOF: Self = Status::StreamEnd; + + fn to_usize(self) -> usize { + self as usize + } + } + + impl CompressFlushKind for Action { + const NONE: Self = Action::Run; + const FINISH: Self = Action::Finish; + + fn to_usize(self) -> usize { + self as usize + } + } + + impl Compressor for CompressorInner { + type Status = Status; + type Flush = Action; + const CHUNKSIZE: usize = u32::MAX as usize; + const DEF_BUF_SIZE: usize = 16 * 1024; + + fn compress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: Self::Flush, + vm: &VirtualMachine, + ) -> PyResult<Self::Status> { + self.stream + .process_vec(input, output, flush) + .map_err(|e| catch_lzma_error(e, vm)) + } + + fn total_in(&mut self) -> usize { + self.stream.total_in() as usize + } + + fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef { + new_lzma_error(message, vm) + } + } + + impl CompressorInner { + fn new(stream: Stream) -> Self { + Self { stream } + } + } + + #[pyattr] + #[pyclass(name = "LZMACompressor")] + #[derive(PyPayload)] + struct LZMACompressor { + state: PyMutex<CompressState<CompressorInner>>, + } + + impl fmt::Debug for LZMACompressor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "_lzma.LZMACompressor") + } + } + + impl LZMACompressor { + fn init_xz( + check: i32, + preset: u32, + filters: Option<Vec<PyObjectRef>>, + vm: &VirtualMachine, + ) -> PyResult<Stream> { + let real_check = + int_to_check(check).ok_or_else(|| vm.new_value_error("Invalid check value"))?; + if let Some(filter_specs) = filters { + let filters = parse_filter_chain_spec(filter_specs, vm)?; + Stream::new_stream_encoder(&filters, real_check) + .map_err(|e| catch_lzma_error(e, vm)) + } else { + Stream::new_easy_encoder(preset, real_check).map_err(|e| catch_lzma_error(e, vm)) + } + } + + fn init_alone( + preset: u32, + filter_specs: Option<Vec<PyObjectRef>>, + vm: &VirtualMachine, + ) -> PyResult<Stream> { + if let Some(_filter_specs) = filter_specs { + // TODO: validate single LZMA1 filter and use its options + let options = LzmaOptions::new_preset(preset).map_err(|_| { + new_lzma_error(format!("Invalid compression preset: {preset}"), vm) + })?; + Stream::new_lzma_encoder(&options).map_err(|e| catch_lzma_error(e, vm)) + } else { + let options = LzmaOptions::new_preset(preset).map_err(|_| { + new_lzma_error(format!("Invalid compression preset: {preset}"), vm) + })?; + Stream::new_lzma_encoder(&options).map_err(|e| catch_lzma_error(e, vm)) + } + } + + fn init_raw( + filter_specs: Option<Vec<PyObjectRef>>, + vm: &VirtualMachine, + ) -> PyResult<Stream> { + let filter_specs = filter_specs + .ok_or_else(|| vm.new_value_error("Must specify filters for FORMAT_RAW"))?; + let filters = parse_filter_chain_spec(filter_specs, vm)?; + Stream::new_raw_encoder(&filters).map_err(|e| catch_lzma_error(e, vm)) + } + } + + #[derive(FromArgs)] + pub struct LZMACompressorConstructorArgs { + #[pyarg(any, default = FORMAT_XZ)] + format: i32, + #[pyarg(any, default = -1)] + check: i32, + #[pyarg(any, optional)] + preset: Option<PyObjectRef>, + #[pyarg(any, optional)] + filters: Option<Vec<PyObjectRef>>, + } + + impl Constructor for LZMACompressor { + type Args = LZMACompressorConstructorArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if args.format != FORMAT_XZ && args.check != -1 && args.check != CHECK_NONE { + return Err(new_lzma_error( + "Integrity checks are only supported by FORMAT_XZ", + vm, + )); + } + + if args.preset.is_some() && args.filters.is_some() { + return Err(new_lzma_error( + "Cannot specify both preset and filter chain", + vm, + )); + } + + let preset: u32 = match &args.preset { + Some(obj) => obj.clone().try_into_value(vm)?, + None => PRESET_DEFAULT, + }; + + let stream = match args.format { + FORMAT_XZ => Self::init_xz(args.check, preset, args.filters, vm)?, + FORMAT_ALONE => Self::init_alone(preset, args.filters, vm)?, + FORMAT_RAW => Self::init_raw(args.filters, vm)?, + _ => { + return Err( + vm.new_value_error(format!("Invalid container format: {}", args.format)) + ); + } + }; + + Ok(Self { + state: PyMutex::new(CompressState::new(CompressorInner::new(stream))), + }) + } + } + + #[pyclass(with(Constructor))] + impl LZMACompressor { + #[pymethod] + fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut state = self.state.lock(); + state.compress(&data.borrow_buf(), vm) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut state = self.state.lock(); + state.flush(Action::Finish, vm) + } + } +} diff --git a/crates/stdlib/src/macros.rs b/crates/stdlib/src/macros.rs new file mode 100644 index 00000000000..385f4b1c4ab --- /dev/null +++ b/crates/stdlib/src/macros.rs @@ -0,0 +1,7 @@ +#[macro_export] +macro_rules! flame_guard { + ($name:expr) => { + #[cfg(feature = "flame-it")] + let _guard = ::flame::start_guard($name); + }; +} diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs new file mode 100644 index 00000000000..b2ef4a42ba7 --- /dev/null +++ b/crates/stdlib/src/math.rs @@ -0,0 +1,961 @@ +pub(crate) use math::module_def; + +use crate::vm::{VirtualMachine, builtins::PyBaseExceptionRef}; + +#[pymodule] +mod math { + use crate::vm::{ + AsObject, PyObject, PyObjectRef, PyRef, PyResult, VirtualMachine, + builtins::{PyFloat, PyInt, PyIntRef, PyStrInterned, try_bigint_to_f64, try_f64_to_bigint}, + function::{ArgIndex, ArgIntoFloat, ArgIterable, Either, OptionalArg, PosArgs}, + identifier, + }; + use malachite_bigint::BigInt; + use num_traits::{Signed, ToPrimitive}; + + use super::{float_repr, pymath_exception}; + + // Constants + #[pyattr] + use core::f64::consts::{E as e, PI as pi, TAU as tau}; + + #[pyattr(name = "inf")] + const INF: f64 = f64::INFINITY; + #[pyattr(name = "nan")] + const NAN: f64 = f64::NAN; + + // Number theory functions: + #[pyfunction] + fn fabs(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::fabs(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn isfinite(x: ArgIntoFloat) -> bool { + pymath::math::isfinite(x.into_float()) + } + + #[pyfunction] + fn isinf(x: ArgIntoFloat) -> bool { + pymath::math::isinf(x.into_float()) + } + + #[pyfunction] + fn isnan(x: ArgIntoFloat) -> bool { + pymath::math::isnan(x.into_float()) + } + + #[derive(FromArgs)] + struct IsCloseArgs { + #[pyarg(positional)] + a: ArgIntoFloat, + #[pyarg(positional)] + b: ArgIntoFloat, + #[pyarg(named, optional)] + rel_tol: OptionalArg<ArgIntoFloat>, + #[pyarg(named, optional)] + abs_tol: OptionalArg<ArgIntoFloat>, + } + + #[pyfunction] + fn isclose(args: IsCloseArgs, vm: &VirtualMachine) -> PyResult<bool> { + let a = args.a.into_float(); + let b = args.b.into_float(); + let rel_tol = args.rel_tol.into_option().map(|v| v.into_float()); + let abs_tol = args.abs_tol.into_option().map(|v| v.into_float()); + + pymath::math::isclose(a, b, rel_tol, abs_tol) + .map_err(|_| vm.new_value_error("tolerances must be non-negative")) + } + + #[pyfunction] + fn copysign(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::copysign(x.into_float(), y.into_float()) + .map_err(|err| pymath_exception(err, vm)) + } + + // Power and logarithmic functions: + #[pyfunction] + fn exp(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::exp(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn exp2(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::exp2(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn expm1(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::expm1(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn log(x: PyObjectRef, base: OptionalArg<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { + let base = base.into_option().map(|v| v.into_float()); + // Check base first for proper error messages + if let Some(b) = base { + if b <= 0.0 { + return Err(vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(b) + ))); + } + if b == 1.0 { + return Err(vm.new_value_error("math domain error")); + } + } + // Handle BigInt specially for large values (only for actual int type, not float) + if let Some(i) = x.downcast_ref::<PyInt>() { + return pymath::math::log_bigint(i.as_bigint(), base).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("expected a positive input"), + _ => pymath_exception(err, vm), + }); + } + let val = x.try_float(vm)?.to_f64(); + pymath::math::log(val, base).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn log1p(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::log1p(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn log2(x: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { + // Handle BigInt specially for large values (only for actual int type, not float) + if let Some(i) = x.downcast_ref::<PyInt>() { + return pymath::math::log2_bigint(i.as_bigint()).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("expected a positive input"), + _ => pymath_exception(err, vm), + }); + } + let val = x.try_float(vm)?.to_f64(); + pymath::math::log2(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn log10(x: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { + // Handle BigInt specially for large values (only for actual int type, not float) + if let Some(i) = x.downcast_ref::<PyInt>() { + return pymath::math::log10_bigint(i.as_bigint()).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("expected a positive input"), + _ => pymath_exception(err, vm), + }); + } + let val = x.try_float(vm)?.to_f64(); + pymath::math::log10(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a positive input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn pow(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::pow(x.into_float(), y.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn sqrt(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::sqrt(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a nonnegative input, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + + // Trigonometric functions: + #[pyfunction] + fn acos(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::acos(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a number in range from -1 up to 1, got {}", + float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn asin(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::asin(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a number in range from -1 up to 1, got {}", + float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn atan(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::atan(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn atan2(y: ArgIntoFloat, x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::atan2(y.into_float(), x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn cos(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::cos(val).map_err(|err| match err { + pymath::Error::EDOM => { + vm.new_value_error(format!("expected a finite input, got {}", float_repr(val))) + } + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn hypot(coordinates: PosArgs<ArgIntoFloat>) -> f64 { + let coords = ArgIntoFloat::vec_into_f64(coordinates.into_vec()); + pymath::math::hypot(&coords) + } + + #[pyfunction] + fn dist(p: Vec<ArgIntoFloat>, q: Vec<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { + let p = ArgIntoFloat::vec_into_f64(p); + let q = ArgIntoFloat::vec_into_f64(q); + if p.len() != q.len() { + return Err(vm.new_value_error("both points must have the same number of dimensions")); + } + Ok(pymath::math::dist(&p, &q)) + } + + #[pyfunction] + fn sin(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::sin(val).map_err(|err| match err { + pymath::Error::EDOM => { + vm.new_value_error(format!("expected a finite input, got {}", float_repr(val))) + } + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn tan(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::tan(val).map_err(|err| match err { + pymath::Error::EDOM => { + vm.new_value_error(format!("expected a finite input, got {}", float_repr(val))) + } + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn degrees(x: ArgIntoFloat) -> f64 { + pymath::math::degrees(x.into_float()) + } + + #[pyfunction] + fn radians(x: ArgIntoFloat) -> f64 { + pymath::math::radians(x.into_float()) + } + + // Hyperbolic functions: + + #[pyfunction] + fn acosh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::acosh(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn asinh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::asinh(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn atanh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::atanh(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a number between -1 and 1, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn cosh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::cosh(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn sinh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::sinh(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn tanh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::tanh(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + // Special functions: + #[pyfunction] + fn erf(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::erf(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn erfc(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::erfc(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn gamma(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + let val = x.into_float(); + pymath::math::gamma(val).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error(format!( + "expected a noninteger or positive integer, got {}", + super::float_repr(val) + )), + _ => pymath_exception(err, vm), + }) + } + + #[pyfunction] + fn lgamma(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::lgamma(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + fn try_magic_method( + func_name: &'static PyStrInterned, + vm: &VirtualMachine, + value: &PyObject, + ) -> PyResult { + let method = vm.get_method_or_type_error(value.to_owned(), func_name, || { + format!( + "type '{}' doesn't define '{}' method", + value.class().name(), + func_name.as_str(), + ) + })?; + method.call((), vm) + } + + #[pyfunction] + fn trunc(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_magic_method(identifier!(vm, __trunc__), vm, &x) + } + + #[pyfunction] + fn ceil(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Only call __ceil__ if the class defines it - if it exists but is not callable, + // the error should be propagated (not fall back to float conversion) + if x.class().has_attr(identifier!(vm, __ceil__)) { + return try_magic_method(identifier!(vm, __ceil__), vm, &x); + } + // __ceil__ not defined - fall back to float conversion + if let Some(v) = x.try_float_opt(vm) { + let v = try_f64_to_bigint(v?.to_f64().ceil(), vm)?; + return Ok(vm.ctx.new_int(v).into()); + } + Err(vm.new_type_error(format!( + "type '{}' doesn't define '__ceil__' method", + x.class().name(), + ))) + } + + #[pyfunction] + fn floor(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Only call __floor__ if the class defines it - if it exists but is not callable, + // the error should be propagated (not fall back to float conversion) + if x.class().has_attr(identifier!(vm, __floor__)) { + return try_magic_method(identifier!(vm, __floor__), vm, &x); + } + // __floor__ not defined - fall back to float conversion + if let Some(v) = x.try_float_opt(vm) { + let v = try_f64_to_bigint(v?.to_f64().floor(), vm)?; + return Ok(vm.ctx.new_int(v).into()); + } + Err(vm.new_type_error(format!( + "type '{}' doesn't define '__floor__' method", + x.class().name(), + ))) + } + + #[pyfunction] + fn frexp(x: ArgIntoFloat) -> (f64, i32) { + pymath::math::frexp(x.into_float()) + } + + #[pyfunction] + fn ldexp( + x: Either<PyRef<PyFloat>, PyIntRef>, + i: PyIntRef, + vm: &VirtualMachine, + ) -> PyResult<f64> { + let value = match x { + Either::A(f) => f.to_f64(), + Either::B(z) => try_bigint_to_f64(z.as_bigint(), vm)?, + }; + pymath::math::ldexp_bigint(value, i.as_bigint()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn cbrt(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::cbrt(x.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn fsum(seq: ArgIterable<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { + let values: Result<Vec<f64>, _> = + seq.iter(vm)?.map(|r| r.map(|v| v.into_float())).collect(); + pymath::math::fsum(values?).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn modf(x: ArgIntoFloat) -> (f64, f64) { + pymath::math::modf(x.into_float()) + } + + #[derive(FromArgs)] + struct NextAfterArgs { + #[pyarg(positional)] + x: ArgIntoFloat, + #[pyarg(positional)] + y: ArgIntoFloat, + #[pyarg(named, optional)] + steps: OptionalArg<ArgIndex>, + } + + #[pyfunction] + fn nextafter(arg: NextAfterArgs, vm: &VirtualMachine) -> PyResult<f64> { + let x = arg.x.into_float(); + let y = arg.y.into_float(); + + let steps = match arg.steps.into_option() { + Some(steps) => { + let steps: i64 = steps.into_int_ref().try_to_primitive(vm)?; + if steps < 0 { + return Err(vm.new_value_error("steps must be a non-negative integer")); + } + Some(steps as u64) + } + None => None, + }; + Ok(pymath::math::nextafter(x, y, steps)) + } + + #[pyfunction] + fn ulp(x: ArgIntoFloat) -> f64 { + pymath::math::ulp(x.into_float()) + } + + #[pyfunction(name = "fmod")] + fn py_fmod(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::fmod(x.into_float(), y.into_float()).map_err(|err| pymath_exception(err, vm)) + } + + #[pyfunction] + fn remainder(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { + pymath::math::remainder(x.into_float(), y.into_float()) + .map_err(|err| pymath_exception(err, vm)) + } + + #[derive(FromArgs)] + struct ProdArgs { + #[pyarg(positional)] + iterable: ArgIterable<PyObjectRef>, + #[pyarg(named, optional)] + start: OptionalArg<PyObjectRef>, + } + + #[pyfunction] + fn prod(args: ProdArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + use crate::vm::builtins::PyInt; + + let iter = args.iterable; + let start = args.start; + + // Check if start is provided and what type it is (exact types only, not subclasses) + let (mut obj_result, start_is_int, start_is_float) = match &start { + OptionalArg::Present(s) => { + let is_int = s.class().is(vm.ctx.types.int_type); + let is_float = s.class().is(vm.ctx.types.float_type); + (Some(s.clone()), is_int, is_float) + } + OptionalArg::Missing => (None, true, false), // Default is int 1 + }; + + let mut item_iter = iter.iter(vm)?; + + // Integer fast path + if start_is_int && !start_is_float { + let mut int_result: i64 = match &start { + OptionalArg::Present(s) => { + if let Some(i) = s.downcast_ref::<PyInt>() { + match i.as_bigint().try_into() { + Ok(v) => v, + Err(_) => { + // Start overflows i64, fall through to generic path + obj_result = Some(s.clone()); + i64::MAX // Will be ignored + } + } + } else { + 1 + } + } + OptionalArg::Missing => 1, + }; + + if obj_result.is_none() { + loop { + let item = match item_iter.next() { + Some(r) => r?, + None => return Ok(vm.ctx.new_int(int_result).into()), + }; + + // Only use fast path for exact int type (not subclasses) + if item.class().is(vm.ctx.types.int_type) + && let Some(int_item) = item.downcast_ref::<PyInt>() + && let Ok(b) = int_item.as_bigint().try_into() as Result<i64, _> + && let Some(product) = int_result.checked_mul(b) + { + int_result = product; + continue; + } + + // Overflow or non-int: restore to PyObject and continue + obj_result = Some(vm.ctx.new_int(int_result).into()); + let temp = vm._mul(obj_result.as_ref().unwrap(), &item)?; + obj_result = Some(temp); + break; + } + } + } + + // Float fast path + let obj_float = obj_result + .as_ref() + .and_then(|obj| obj.clone().downcast::<PyFloat>().ok()); + if obj_float.is_some() || start_is_float { + let mut flt_result: f64 = if let Some(ref f) = obj_float { + f.to_f64() + } else if start_is_float && let OptionalArg::Present(s) = &start { + s.downcast_ref::<PyFloat>() + .map(|f| f.to_f64()) + .unwrap_or(1.0) + } else { + 1.0 + }; + + loop { + let item = match item_iter.next() { + Some(r) => r?, + None => return Ok(vm.ctx.new_float(flt_result).into()), + }; + + // Only use fast path for exact float/int types (not subclasses) + if item.class().is(vm.ctx.types.float_type) + && let Some(f) = item.downcast_ref::<PyFloat>() + { + flt_result *= f.to_f64(); + continue; + } + if item.class().is(vm.ctx.types.int_type) + && let Some(i) = item.downcast_ref::<PyInt>() + && let Ok(v) = i.as_bigint().try_into() as Result<i64, _> + { + flt_result *= v as f64; + continue; + } + + // Non-exact-float/int: restore and continue with generic path + obj_result = Some(vm.ctx.new_float(flt_result).into()); + let temp = vm._mul(obj_result.as_ref().unwrap(), &item)?; + obj_result = Some(temp); + break; + } + } + + // Generic path for remaining items + let mut result = obj_result.unwrap_or_else(|| vm.ctx.new_int(1).into()); + for item in item_iter { + let item = item?; + result = vm._mul(&result, &item)?; + } + + Ok(result) + } + + #[pyfunction] + fn sumprod( + p: ArgIterable<PyObjectRef>, + q: ArgIterable<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + use crate::vm::builtins::PyInt; + + let mut p_iter = p.iter(vm)?; + let mut q_iter = q.iter(vm)?; + + // Fast path state + let mut int_path_enabled = true; + let mut int_total: i64 = 0; + let mut int_total_in_use = false; + let mut flt_p_values: Vec<f64> = Vec::new(); + let mut flt_q_values: Vec<f64> = Vec::new(); + + // Fallback accumulator for generic Python path + let mut obj_total: Option<PyObjectRef> = None; + + loop { + let m_p = p_iter.next(); + let m_q = q_iter.next(); + + let (p_i, q_i, finished) = match (m_p, m_q) { + (Some(r_p), Some(r_q)) => (Some(r_p?), Some(r_q?), false), + (None, None) => (None, None, true), + _ => return Err(vm.new_value_error("Inputs are not the same length")), + }; + + // Integer fast path (only for exact int types, not subclasses) + if int_path_enabled { + if !finished { + let (p_i, q_i) = (p_i.as_ref().unwrap(), q_i.as_ref().unwrap()); + if p_i.class().is(vm.ctx.types.int_type) + && q_i.class().is(vm.ctx.types.int_type) + && let (Some(p_int), Some(q_int)) = + (p_i.downcast_ref::<PyInt>(), q_i.downcast_ref::<PyInt>()) + && let (Ok(p_val), Ok(q_val)) = ( + p_int.as_bigint().try_into() as Result<i64, _>, + q_int.as_bigint().try_into() as Result<i64, _>, + ) + && let Some(prod) = p_val.checked_mul(q_val) + && let Some(new_total) = int_total.checked_add(prod) + { + int_total = new_total; + int_total_in_use = true; + continue; + } + } + // Finalize int path + int_path_enabled = false; + if int_total_in_use { + let int_obj: PyObjectRef = vm.ctx.new_int(int_total).into(); + obj_total = Some(match obj_total { + Some(total) => vm._add(&total, &int_obj)?, + None => int_obj, + }); + int_total = 0; + int_total_in_use = false; + } + } + + // Float fast path - only when at least one value is exact float type + // (not subclasses, to preserve custom __mul__/__add__ behavior) + { + if !finished { + let (p_i, q_i) = (p_i.as_ref().unwrap(), q_i.as_ref().unwrap()); + + let p_is_exact_float = p_i.class().is(vm.ctx.types.float_type); + let q_is_exact_float = q_i.class().is(vm.ctx.types.float_type); + let p_is_exact_int = p_i.class().is(vm.ctx.types.int_type); + let q_is_exact_int = q_i.class().is(vm.ctx.types.int_type); + let p_is_exact_numeric = p_is_exact_float || p_is_exact_int; + let q_is_exact_numeric = q_is_exact_float || q_is_exact_int; + let has_exact_float = p_is_exact_float || q_is_exact_float; + + // Only use float path if at least one is exact float and both are exact int/float + if has_exact_float && p_is_exact_numeric && q_is_exact_numeric { + let p_flt = if let Some(f) = p_i.downcast_ref::<PyFloat>() { + Some(f.to_f64()) + } else if let Some(i) = p_i.downcast_ref::<PyInt>() { + // PyLong_AsDouble fails for integers too large for f64 + try_bigint_to_f64(i.as_bigint(), vm).ok() + } else { + None + }; + + let q_flt = if let Some(f) = q_i.downcast_ref::<PyFloat>() { + Some(f.to_f64()) + } else if let Some(i) = q_i.downcast_ref::<PyInt>() { + // PyLong_AsDouble fails for integers too large for f64 + try_bigint_to_f64(i.as_bigint(), vm).ok() + } else { + None + }; + + if let (Some(p_val), Some(q_val)) = (p_flt, q_flt) { + flt_p_values.push(p_val); + flt_q_values.push(q_val); + continue; + } + } + } + // Finalize float path + if !flt_p_values.is_empty() { + let flt_result = pymath::math::sumprod(&flt_p_values, &flt_q_values) + .map_err(|err| pymath_exception(err, vm))?; + let flt_obj: PyObjectRef = vm.ctx.new_float(flt_result).into(); + obj_total = Some(match obj_total { + Some(total) => vm._add(&total, &flt_obj)?, + None => flt_obj, + }); + flt_p_values.clear(); + flt_q_values.clear(); + } + } + + if finished { + break; + } + + // Generic Python path + let (p_i, q_i) = (p_i.unwrap(), q_i.unwrap()); + + // Collect current + remaining elements + let p_remaining: Result<Vec<PyObjectRef>, _> = + core::iter::once(Ok(p_i)).chain(p_iter).collect(); + let q_remaining: Result<Vec<PyObjectRef>, _> = + core::iter::once(Ok(q_i)).chain(q_iter).collect(); + let (p_vec, q_vec) = (p_remaining?, q_remaining?); + + if p_vec.len() != q_vec.len() { + return Err(vm.new_value_error("Inputs are not the same length")); + } + + let mut total = obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into()); + for (p_item, q_item) in p_vec.into_iter().zip(q_vec) { + let prod = vm._mul(&p_item, &q_item)?; + total = vm._add(&total, &prod)?; + } + return Ok(total); + } + + Ok(obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into())) + } + + #[pyfunction] + fn fma( + x: ArgIntoFloat, + y: ArgIntoFloat, + z: ArgIntoFloat, + vm: &VirtualMachine, + ) -> PyResult<f64> { + pymath::math::fma(x.into_float(), y.into_float(), z.into_float()).map_err(|err| match err { + pymath::Error::EDOM => vm.new_value_error("invalid operation in fma"), + pymath::Error::ERANGE => vm.new_overflow_error("overflow in fma"), + }) + } + + // Integer functions: + + #[pyfunction] + fn isqrt(x: ArgIndex, vm: &VirtualMachine) -> PyResult<BigInt> { + let value = x.into_int_ref(); + pymath::math::integer::isqrt(value.as_bigint()) + .map_err(|_| vm.new_value_error("isqrt() argument must be nonnegative")) + } + + #[pyfunction] + fn gcd(args: PosArgs<ArgIndex>) -> BigInt { + let ints: Vec<_> = args + .into_vec() + .into_iter() + .map(|x| x.into_int_ref()) + .collect(); + let refs: Vec<_> = ints.iter().map(|x| x.as_bigint()).collect(); + pymath::math::integer::gcd(&refs) + } + + #[pyfunction] + fn lcm(args: PosArgs<ArgIndex>) -> BigInt { + let ints: Vec<_> = args + .into_vec() + .into_iter() + .map(|x| x.into_int_ref()) + .collect(); + let refs: Vec<_> = ints.iter().map(|x| x.as_bigint()).collect(); + pymath::math::integer::lcm(&refs) + } + + #[pyfunction] + fn factorial(x: PyIntRef, vm: &VirtualMachine) -> PyResult<BigInt> { + // Check for negative before overflow - negative values are always invalid + if x.as_bigint().is_negative() { + return Err(vm.new_value_error("factorial() not defined for negative values")); + } + let n: i64 = x.try_to_primitive(vm).map_err(|_| { + vm.new_overflow_error("factorial() argument should not exceed 9223372036854775807") + })?; + pymath::math::integer::factorial(n) + .map(|r| r.into()) + .map_err(|_| vm.new_value_error("factorial() not defined for negative values")) + } + + #[pyfunction] + fn perm( + n: ArgIndex, + k: OptionalArg<Option<ArgIndex>>, + vm: &VirtualMachine, + ) -> PyResult<BigInt> { + let n_int = n.into_int_ref(); + let n_big = n_int.as_bigint(); + + if n_big.is_negative() { + return Err(vm.new_value_error("n must be a non-negative integer")); + } + + // k = None means k = n (factorial) + let k_int = k.flatten().map(|k| k.into_int_ref()); + let k_big: Option<&BigInt> = k_int.as_ref().map(|k| k.as_bigint()); + + if let Some(k_val) = k_big { + if k_val.is_negative() { + return Err(vm.new_value_error("k must be a non-negative integer")); + } + if k_val > n_big { + return Ok(BigInt::from(0u8)); + } + } + + // Convert k to u64 (required by pymath) + let ki: u64 = match k_big { + None => match n_big.to_u64() { + Some(n) => n, + None => { + return Err(vm.new_overflow_error(format!("n must not exceed {}", u64::MAX))); + } + }, + Some(k_val) => match k_val.to_u64() { + Some(k) => k, + None => { + return Err(vm.new_overflow_error(format!("k must not exceed {}", u64::MAX))); + } + }, + }; + + // Fast path: n fits in i64 + if let Some(ni) = n_big.to_i64() + && ni >= 0 + && ki > 1 + { + let result = pymath::math::integer::perm(ni, Some(ki as i64)) + .map_err(|_| vm.new_value_error("perm() error"))?; + return Ok(result.into()); + } + + // BigInt path: use perm_bigint + let result = pymath::math::perm_bigint(n_big, ki); + Ok(result.into()) + } + + #[pyfunction] + fn comb(n: ArgIndex, k: ArgIndex, vm: &VirtualMachine) -> PyResult<BigInt> { + let n_int = n.into_int_ref(); + let n_big = n_int.as_bigint(); + let k_int = k.into_int_ref(); + let k_big = k_int.as_bigint(); + + if n_big.is_negative() { + return Err(vm.new_value_error("n must be a non-negative integer")); + } + if k_big.is_negative() { + return Err(vm.new_value_error("k must be a non-negative integer")); + } + + // Fast path: n fits in i64 + if let Some(ni) = n_big.to_i64() + && ni >= 0 + { + // k overflow or k > n means result is 0 + let ki = match k_big.to_i64() { + Some(k) if k >= 0 && k <= ni => k, + _ => return Ok(BigInt::from(0u8)), + }; + // Apply symmetry: use min(k, n-k) + let ki = ki.min(ni - ki); + if ki > 1 { + let result = pymath::math::integer::comb(ni, ki) + .map_err(|_| vm.new_value_error("comb() error"))?; + return Ok(result.into()); + } + // ki <= 1 cases + if ki == 0 { + return Ok(BigInt::from(1u8)); + } + return Ok(n_big.clone()); // ki == 1 + } + + // BigInt path: n doesn't fit in i64 + // Apply symmetry: k = min(k, n - k) + let n_minus_k = n_big - k_big; + if n_minus_k.is_negative() { + return Ok(BigInt::from(0u8)); + } + let effective_k = if &n_minus_k < k_big { + &n_minus_k + } else { + k_big + }; + + // k must fit in u64 + let ki: u64 = match effective_k.to_u64() { + Some(k) => k, + None => { + return Err( + vm.new_overflow_error(format!("min(n - k, k) must not exceed {}", u64::MAX)) + ); + } + }; + + let result = pymath::math::comb_bigint(n_big, ki); + Ok(result.into()) + } +} + +pub(crate) fn pymath_exception(err: pymath::Error, vm: &VirtualMachine) -> PyBaseExceptionRef { + match err { + pymath::Error::EDOM => vm.new_value_error("math domain error"), + pymath::Error::ERANGE => vm.new_overflow_error("math range error"), + } +} + +/// Format a float in Python style (ensures trailing .0 for integers). +fn float_repr(value: f64) -> String { + if value.is_nan() { + "nan".to_owned() + } else if value.is_infinite() { + if value.is_sign_positive() { + "inf".to_owned() + } else { + "-inf".to_owned() + } + } else { + let s = format!("{}", value); + // If no decimal point and not in scientific notation, add .0 + if !s.contains('.') && !s.contains('e') && !s.contains('E') { + format!("{}.0", s) + } else { + s + } + } +} diff --git a/crates/stdlib/src/md5.rs b/crates/stdlib/src/md5.rs new file mode 100644 index 00000000000..2ff6cd24ff7 --- /dev/null +++ b/crates/stdlib/src/md5.rs @@ -0,0 +1,12 @@ +pub(crate) use _md5::module_def; + +#[pymodule] +mod _md5 { + use crate::hashlib::_hashlib::{HashArgs, local_md5}; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; + + #[pyfunction] + fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_md5(args, vm)?.into_pyobject(vm)) + } +} diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs new file mode 100644 index 00000000000..d441a3dd887 --- /dev/null +++ b/crates/stdlib/src/mmap.rs @@ -0,0 +1,1625 @@ +// spell-checker:disable +//! mmap module +pub(crate) use mmap::module_def; + +#[pymodule] +mod mmap { + use crate::common::{ + borrow::{BorrowedValue, BorrowedValueMut}, + lock::{MapImmutable, PyMutex, PyMutexGuard}, + }; + use crate::vm::{ + AsObject, FromArgs, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, VirtualMachine, atomic_func, + builtins::{PyBytes, PyBytesRef, PyInt, PyIntRef, PyType, PyTypeRef}, + byte::{bytes_from_object, value_from_object}, + convert::ToPyException, + function::{ArgBytesLike, FuncArgs, OptionalArg}, + protocol::{ + BufferDescriptor, BufferMethods, PyBuffer, PyMappingMethods, PySequenceMethods, + }, + sliceable::{SaturatedSlice, SequenceIndex, SequenceIndexOp}, + types::{AsBuffer, AsMapping, AsSequence, Constructor, Representable}, + }; + use core::ops::{Deref, DerefMut}; + use crossbeam_utils::atomic::AtomicCell; + use memmap2::{Mmap, MmapMut, MmapOptions}; + use num_traits::Signed; + use std::io::{self, Write}; + + #[cfg(unix)] + use nix::{sys::stat::fstat, unistd}; + #[cfg(unix)] + use rustpython_common::crt_fd; + + #[cfg(windows)] + use rustpython_common::suppress_iph; + #[cfg(windows)] + use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle, RawHandle}; + #[cfg(windows)] + use windows_sys::Win32::{ + Foundation::{ + CloseHandle, DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE, INVALID_HANDLE_VALUE, + }, + Storage::FileSystem::{FILE_BEGIN, GetFileSize, SetEndOfFile, SetFilePointerEx}, + System::Memory::{ + CreateFileMappingW, FILE_MAP_COPY, FILE_MAP_READ, FILE_MAP_WRITE, FlushViewOfFile, + MapViewOfFile, PAGE_READONLY, PAGE_READWRITE, PAGE_WRITECOPY, UnmapViewOfFile, + }, + System::Threading::GetCurrentProcess, + }; + + #[cfg(unix)] + fn validate_advice(vm: &VirtualMachine, advice: i32) -> PyResult<i32> { + match advice { + libc::MADV_NORMAL + | libc::MADV_RANDOM + | libc::MADV_SEQUENTIAL + | libc::MADV_WILLNEED + | libc::MADV_DONTNEED => Ok(advice), + #[cfg(any( + target_os = "linux", + target_os = "macos", + target_os = "ios", + target_os = "freebsd" + ))] + libc::MADV_FREE => Ok(advice), + #[cfg(target_os = "linux")] + libc::MADV_DONTFORK + | libc::MADV_DOFORK + | libc::MADV_MERGEABLE + | libc::MADV_UNMERGEABLE + | libc::MADV_HUGEPAGE + | libc::MADV_NOHUGEPAGE + | libc::MADV_REMOVE + | libc::MADV_DONTDUMP + | libc::MADV_DODUMP + | libc::MADV_HWPOISON => Ok(advice), + #[cfg(target_os = "freebsd")] + libc::MADV_NOSYNC + | libc::MADV_AUTOSYNC + | libc::MADV_NOCORE + | libc::MADV_CORE + | libc::MADV_PROTECT => Ok(advice), + _ => Err(vm.new_value_error("Not a valid Advice value")), + } + } + + #[repr(C)] + #[derive(PartialEq, Eq, Debug)] + enum AccessMode { + Default = 0, + Read = 1, + Write = 2, + Copy = 3, + } + + impl<'a> TryFromBorrowedObject<'a> for AccessMode { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + let i = u32::try_from_borrowed_object(vm, obj)?; + Ok(match i { + 0 => Self::Default, + 1 => Self::Read, + 2 => Self::Write, + 3 => Self::Copy, + _ => return Err(vm.new_value_error("Not a valid AccessMode value")), + }) + } + } + + #[cfg(unix)] + #[pyattr] + use libc::{ + MADV_DONTNEED, MADV_NORMAL, MADV_RANDOM, MADV_SEQUENTIAL, MADV_WILLNEED, MAP_ANON, + MAP_ANONYMOUS, MAP_PRIVATE, MAP_SHARED, PROT_EXEC, PROT_READ, PROT_WRITE, + }; + + #[cfg(target_os = "macos")] + #[pyattr] + use libc::{MADV_FREE_REUSABLE, MADV_FREE_REUSE}; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "fuchsia", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple" + ))] + #[pyattr] + use libc::MADV_FREE; + + #[cfg(target_os = "linux")] + #[pyattr] + use libc::{ + MADV_DODUMP, MADV_DOFORK, MADV_DONTDUMP, MADV_DONTFORK, MADV_HUGEPAGE, MADV_HWPOISON, + MADV_MERGEABLE, MADV_NOHUGEPAGE, MADV_REMOVE, MADV_UNMERGEABLE, + }; + + #[cfg(any( + target_os = "android", + all( + target_os = "linux", + any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "s390x", + target_arch = "x86", + target_arch = "x86_64", + target_arch = "sparc64" + ) + ) + ))] + #[pyattr] + use libc::MADV_SOFT_OFFLINE; + + #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))] + #[pyattr] + use libc::{MAP_DENYWRITE, MAP_EXECUTABLE, MAP_POPULATE}; + + // MAP_STACK is available on Linux, OpenBSD, and NetBSD + #[cfg(any(target_os = "linux", target_os = "openbsd", target_os = "netbsd"))] + #[pyattr] + use libc::MAP_STACK; + + // FreeBSD-specific MADV constants + #[cfg(target_os = "freebsd")] + #[pyattr] + use libc::{MADV_AUTOSYNC, MADV_CORE, MADV_NOCORE, MADV_NOSYNC, MADV_PROTECT}; + + #[pyattr] + const ACCESS_DEFAULT: u32 = AccessMode::Default as u32; + #[pyattr] + const ACCESS_READ: u32 = AccessMode::Read as u32; + #[pyattr] + const ACCESS_WRITE: u32 = AccessMode::Write as u32; + #[pyattr] + const ACCESS_COPY: u32 = AccessMode::Copy as u32; + + #[cfg(not(target_arch = "wasm32"))] + #[pyattr(name = "PAGESIZE", once)] + fn page_size(_vm: &VirtualMachine) -> usize { + page_size::get() + } + + #[cfg(not(target_arch = "wasm32"))] + #[pyattr(name = "ALLOCATIONGRANULARITY", once)] + fn granularity(_vm: &VirtualMachine) -> usize { + page_size::get_granularity() + } + + #[pyattr(name = "error", once)] + fn error_type(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.exceptions.os_error.to_owned() + } + + /// Named file mapping on Windows using raw Win32 APIs. + /// Supports tagname parameter for inter-process shared memory. + #[cfg(windows)] + struct NamedMmap { + map_handle: HANDLE, + view_ptr: *mut u8, + len: usize, + } + + #[cfg(windows)] + // SAFETY: The memory mapping is managed by the OS and is safe to share + // across threads. Access is synchronized by PyMutex in PyMmap. + unsafe impl Send for NamedMmap {} + #[cfg(windows)] + unsafe impl Sync for NamedMmap {} + + #[cfg(windows)] + impl core::fmt::Debug for NamedMmap { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("NamedMmap") + .field("map_handle", &self.map_handle) + .field("view_ptr", &self.view_ptr) + .field("len", &self.len) + .finish() + } + } + + #[cfg(windows)] + impl Drop for NamedMmap { + fn drop(&mut self) { + unsafe { + if !self.view_ptr.is_null() { + UnmapViewOfFile( + windows_sys::Win32::System::Memory::MEMORY_MAPPED_VIEW_ADDRESS { + Value: self.view_ptr as *mut _, + }, + ); + } + if !self.map_handle.is_null() { + CloseHandle(self.map_handle); + } + } + } + } + + #[derive(Debug)] + enum MmapObj { + Write(MmapMut), + Read(Mmap), + #[cfg(windows)] + Named(NamedMmap), + } + + impl MmapObj { + fn as_slice(&self) -> &[u8] { + match self { + MmapObj::Read(mmap) => &mmap[..], + MmapObj::Write(mmap) => &mmap[..], + #[cfg(windows)] + MmapObj::Named(named) => unsafe { + core::slice::from_raw_parts(named.view_ptr, named.len) + }, + } + } + } + + #[pyattr] + #[pyclass(name = "mmap")] + #[derive(Debug, PyPayload)] + struct PyMmap { + closed: AtomicCell<bool>, + mmap: PyMutex<Option<MmapObj>>, + #[cfg(unix)] + fd: AtomicCell<i32>, + #[cfg(windows)] + handle: AtomicCell<isize>, // HANDLE is isize on Windows + offset: i64, + size: AtomicCell<usize>, + pos: AtomicCell<usize>, // relative to offset + exports: AtomicCell<usize>, + access: AccessMode, + } + + impl PyMmap { + /// Close the underlying file handle/descriptor if open + fn close_handle(&self) { + #[cfg(unix)] + { + let fd = self.fd.swap(-1); + if fd >= 0 { + unsafe { libc::close(fd) }; + } + } + #[cfg(windows)] + { + let handle = self.handle.swap(INVALID_HANDLE_VALUE as isize); + if handle != INVALID_HANDLE_VALUE as isize { + unsafe { CloseHandle(handle as HANDLE) }; + } + } + } + } + + impl Drop for PyMmap { + fn drop(&mut self) { + self.close_handle(); + } + } + + #[cfg(unix)] + #[derive(FromArgs)] + struct MmapNewArgs { + #[pyarg(any)] + fileno: i32, + #[pyarg(any)] + length: isize, + #[pyarg(any, default = libc::MAP_SHARED)] + flags: libc::c_int, + #[pyarg(any, default = libc::PROT_WRITE | libc::PROT_READ)] + prot: libc::c_int, + #[pyarg(any, default = AccessMode::Default)] + access: AccessMode, + #[pyarg(any, default = 0)] + offset: i64, + } + + #[cfg(windows)] + #[derive(FromArgs)] + struct MmapNewArgs { + #[pyarg(any)] + fileno: i32, + #[pyarg(any)] + length: isize, + #[pyarg(any, default)] + tagname: Option<PyObjectRef>, + #[pyarg(any, default = AccessMode::Default)] + access: AccessMode, + #[pyarg(any, default = 0)] + offset: i64, + } + + impl MmapNewArgs { + /// Validate mmap constructor arguments + fn validate_new_args(&self, vm: &VirtualMachine) -> PyResult<usize> { + if self.length < 0 { + return Err(vm.new_overflow_error("memory mapped length must be positive")); + } + if self.offset < 0 { + return Err(vm.new_overflow_error("memory mapped offset must be positive")); + } + Ok(self.length as usize) + } + } + + #[derive(FromArgs)] + pub struct FlushOptions { + #[pyarg(positional, default)] + offset: Option<isize>, + #[pyarg(positional, default)] + size: Option<isize>, + } + + impl FlushOptions { + fn values(self, len: usize) -> Option<(usize, usize)> { + let offset = match self.offset { + Some(o) if o < 0 => return None, + Some(o) => o as usize, + None => 0, + }; + + let size = match self.size { + Some(s) if s < 0 => return None, + Some(s) => s as usize, + None => len, + }; + + if len.checked_sub(offset)? < size { + return None; + } + + Some((offset, size)) + } + } + + #[derive(FromArgs, Clone)] + pub struct FindOptions { + #[pyarg(positional)] + sub: Vec<u8>, + #[pyarg(positional, default)] + start: Option<isize>, + #[pyarg(positional, default)] + end: Option<isize>, + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[derive(FromArgs)] + pub struct AdviseOptions { + #[pyarg(positional)] + option: libc::c_int, + #[pyarg(positional, default)] + start: Option<PyIntRef>, + #[pyarg(positional, default)] + length: Option<PyIntRef>, + } + + #[cfg(all(unix, not(target_os = "redox")))] + impl AdviseOptions { + fn values(self, len: usize, vm: &VirtualMachine) -> PyResult<(libc::c_int, usize, usize)> { + let start = self + .start + .map(|s| { + s.try_to_primitive::<usize>(vm) + .ok() + .filter(|s| *s < len) + .ok_or_else(|| vm.new_value_error("madvise start out of bounds")) + }) + .transpose()? + .unwrap_or(0); + let length = self + .length + .map(|s| { + s.try_to_primitive::<usize>(vm) + .map_err(|_| vm.new_value_error("madvise length invalid")) + }) + .transpose()? + .unwrap_or(len); + + if isize::MAX as usize - start < length { + return Err(vm.new_overflow_error("madvise length too large")); + } + + let length = if start + length > len { + len - start + } else { + length + }; + + Ok((self.option, start, length)) + } + } + + impl Constructor for PyMmap { + type Args = MmapNewArgs; + + #[cfg(unix)] + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + use libc::{MAP_PRIVATE, MAP_SHARED, PROT_READ, PROT_WRITE}; + + let mut map_size = args.validate_new_args(vm)?; + let MmapNewArgs { + fileno: fd, + flags, + prot, + access, + offset, + .. + } = args; + + if (access != AccessMode::Default) + && ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))) + { + return Err(vm.new_value_error("mmap can't specify both access and flags, prot.")); + } + + // TODO: memmap2 doesn't support mapping with prot and flags right now + let (_flags, _prot, access) = match access { + AccessMode::Read => (MAP_SHARED, PROT_READ, access), + AccessMode::Write => (MAP_SHARED, PROT_READ | PROT_WRITE, access), + AccessMode::Copy => (MAP_PRIVATE, PROT_READ | PROT_WRITE, access), + AccessMode::Default => { + let access = if (prot & PROT_READ) != 0 && (prot & PROT_WRITE) != 0 { + access + } else if (prot & PROT_WRITE) != 0 { + AccessMode::Write + } else { + AccessMode::Read + }; + (flags, prot, access) + } + }; + + let fd = unsafe { crt_fd::Borrowed::try_borrow_raw(fd) }; + + // macOS: Issue #11277: fsync(2) is not enough on OS X - a special, OS X specific + // fcntl(2) is necessary to force DISKSYNC and get around mmap(2) bug + #[cfg(target_os = "macos")] + if let Ok(fd) = fd { + use std::os::fd::AsRawFd; + unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_FULLFSYNC) }; + } + + if let Ok(fd) = fd { + let metadata = fstat(fd) + .map_err(|err| io::Error::from_raw_os_error(err as i32).to_pyexception(vm))?; + let file_len = metadata.st_size as i64; + + if map_size == 0 { + if file_len == 0 { + return Err(vm.new_value_error("cannot mmap an empty file")); + } + + if offset > file_len { + return Err(vm.new_value_error("mmap offset is greater than file size")); + } + + map_size = (file_len - offset) + .try_into() + .map_err(|_| vm.new_value_error("mmap length is too large"))?; + } else if offset > file_len || file_len - offset < map_size as i64 { + return Err(vm.new_value_error("mmap length is greater than file size")); + } + } + + let mut mmap_opt = MmapOptions::new(); + let mmap_opt = mmap_opt.offset(offset as u64).len(map_size); + + let (fd, mmap) = || -> std::io::Result<_> { + if let Ok(fd) = fd { + let new_fd: crt_fd::Owned = unistd::dup(fd)?.into(); + let mmap = match access { + AccessMode::Default | AccessMode::Write => { + MmapObj::Write(unsafe { mmap_opt.map_mut(&new_fd) }?) + } + AccessMode::Read => MmapObj::Read(unsafe { mmap_opt.map(&new_fd) }?), + AccessMode::Copy => MmapObj::Write(unsafe { mmap_opt.map_copy(&new_fd) }?), + }; + Ok((Some(new_fd), mmap)) + } else { + let mmap = MmapObj::Write(mmap_opt.map_anon()?); + Ok((None, mmap)) + } + }() + .map_err(|e| e.to_pyexception(vm))?; + + Ok(Self { + closed: AtomicCell::new(false), + mmap: PyMutex::new(Some(mmap)), + fd: AtomicCell::new(fd.map_or(-1, |fd| fd.into_raw())), + offset, + size: AtomicCell::new(map_size), + pos: AtomicCell::new(0), + exports: AtomicCell::new(0), + access, + }) + } + + #[cfg(windows)] + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let mut map_size = args.validate_new_args(vm)?; + let MmapNewArgs { + fileno, + tagname, + access, + offset, + .. + } = args; + + // Parse tagname: None or a string + let tag_str: Option<String> = match tagname { + Some(ref obj) if !vm.is_none(obj) => { + let s = obj + .try_to_value::<String>(vm) + .map_err(|_| vm.new_type_error("tagname must be a string or None"))?; + if s.contains('\0') { + return Err(vm.new_value_error("tagname must not contain null characters")); + } + Some(s) + } + _ => None, + }; + + // Get file handle from fileno + // fileno -1 or 0 means anonymous mapping + let fh: Option<HANDLE> = if fileno != -1 && fileno != 0 { + // Convert CRT file descriptor to Windows HANDLE + // Use suppress_iph! to avoid crashes when the fd is invalid. + // This is critical because socket fds wrapped via _open_osfhandle + // may cause crashes in _get_osfhandle on Windows. + // See Python bug https://bugs.python.org/issue30114 + let handle = unsafe { suppress_iph!(libc::get_osfhandle(fileno)) }; + // Check for invalid handle value (-1 on Windows) + if handle == -1 || handle == INVALID_HANDLE_VALUE as isize { + return Err(vm.new_os_error(format!("Invalid file descriptor: {}", fileno))); + } + Some(handle as HANDLE) + } else { + None + }; + + // Get file size if we have a file handle and map_size is 0 + let mut duplicated_handle: HANDLE = INVALID_HANDLE_VALUE; + if let Some(fh) = fh { + // Duplicate handle so Python code can close the original + let mut new_handle: HANDLE = INVALID_HANDLE_VALUE; + let result = unsafe { + DuplicateHandle( + GetCurrentProcess(), + fh, + GetCurrentProcess(), + &mut new_handle, + 0, + 0, // not inheritable + DUPLICATE_SAME_ACCESS, + ) + }; + if result == 0 { + return Err(io::Error::last_os_error().to_pyexception(vm)); + } + duplicated_handle = new_handle; + + // Get file size + let mut high: u32 = 0; + let low = unsafe { GetFileSize(fh, &mut high) }; + if low == u32::MAX { + let err = io::Error::last_os_error(); + if err.raw_os_error() != Some(0) { + unsafe { CloseHandle(duplicated_handle) }; + return Err(err.to_pyexception(vm)); + } + } + let file_len = ((high as i64) << 32) | (low as i64); + + if map_size == 0 { + if file_len == 0 { + unsafe { CloseHandle(duplicated_handle) }; + return Err(vm.new_value_error("cannot mmap an empty file")); + } + if offset >= file_len { + unsafe { CloseHandle(duplicated_handle) }; + return Err(vm.new_value_error("mmap offset is greater than file size")); + } + if file_len - offset > isize::MAX as i64 { + unsafe { CloseHandle(duplicated_handle) }; + return Err(vm.new_value_error("mmap length is too large")); + } + map_size = (file_len - offset) as usize; + } else { + // If map_size > file_len, extend the file (Windows behavior) + let required_size = offset.checked_add(map_size as i64).ok_or_else(|| { + unsafe { CloseHandle(duplicated_handle) }; + vm.new_overflow_error("mmap size would cause file size overflow") + })?; + if required_size > file_len { + // Extend file using SetFilePointerEx + SetEndOfFile + let result = unsafe { + SetFilePointerEx( + duplicated_handle, + required_size, + core::ptr::null_mut(), + FILE_BEGIN, + ) + }; + if result == 0 { + let err = io::Error::last_os_error(); + unsafe { CloseHandle(duplicated_handle) }; + return Err(err.to_pyexception(vm)); + } + let result = unsafe { SetEndOfFile(duplicated_handle) }; + if result == 0 { + let err = io::Error::last_os_error(); + unsafe { CloseHandle(duplicated_handle) }; + return Err(err.to_pyexception(vm)); + } + } + } + } + + // When tagname is provided, use raw Win32 APIs for named shared memory + if let Some(ref tag) = tag_str { + let (fl_protect, desired_access) = match access { + AccessMode::Default | AccessMode::Write => (PAGE_READWRITE, FILE_MAP_WRITE), + AccessMode::Read => (PAGE_READONLY, FILE_MAP_READ), + AccessMode::Copy => (PAGE_WRITECOPY, FILE_MAP_COPY), + }; + + let fh = if let Some(fh) = fh { + // Close the duplicated handle - we'll use the original + // file handle for CreateFileMappingW + if duplicated_handle != INVALID_HANDLE_VALUE { + unsafe { CloseHandle(duplicated_handle) }; + } + fh + } else { + INVALID_HANDLE_VALUE + }; + + let tag_wide: Vec<u16> = tag.encode_utf16().chain(core::iter::once(0)).collect(); + + let total_size = (offset as u64) + .checked_add(map_size as u64) + .ok_or_else(|| vm.new_overflow_error("mmap offset plus size would overflow"))?; + let size_hi = (total_size >> 32) as u32; + let size_lo = total_size as u32; + + let map_handle = unsafe { + CreateFileMappingW( + fh, + core::ptr::null(), + fl_protect, + size_hi, + size_lo, + tag_wide.as_ptr(), + ) + }; + if map_handle.is_null() { + return Err(io::Error::last_os_error().to_pyexception(vm)); + } + + let off_hi = (offset as u64 >> 32) as u32; + let off_lo = offset as u32; + + let view = + unsafe { MapViewOfFile(map_handle, desired_access, off_hi, off_lo, map_size) }; + if view.Value.is_null() { + unsafe { CloseHandle(map_handle) }; + return Err(io::Error::last_os_error().to_pyexception(vm)); + } + + let named = NamedMmap { + map_handle, + view_ptr: view.Value as *mut u8, + len: map_size, + }; + + return Ok(Self { + closed: AtomicCell::new(false), + mmap: PyMutex::new(Some(MmapObj::Named(named))), + handle: AtomicCell::new(INVALID_HANDLE_VALUE as isize), + offset, + size: AtomicCell::new(map_size), + pos: AtomicCell::new(0), + exports: AtomicCell::new(0), + access, + }); + } + + let mut mmap_opt = MmapOptions::new(); + let mmap_opt = mmap_opt.offset(offset as u64).len(map_size); + + let (handle, mmap) = if duplicated_handle != INVALID_HANDLE_VALUE { + // Safety: We just duplicated this handle and it's valid + let owned_handle = + unsafe { OwnedHandle::from_raw_handle(duplicated_handle as RawHandle) }; + + let mmap_result = match access { + AccessMode::Default | AccessMode::Write => { + unsafe { mmap_opt.map_mut(&owned_handle) }.map(MmapObj::Write) + } + AccessMode::Read => unsafe { mmap_opt.map(&owned_handle) }.map(MmapObj::Read), + AccessMode::Copy => { + unsafe { mmap_opt.map_copy(&owned_handle) }.map(MmapObj::Write) + } + }; + + let mmap = mmap_result.map_err(|e| e.to_pyexception(vm))?; + + // Keep the handle alive + let raw = owned_handle.as_raw_handle() as isize; + core::mem::forget(owned_handle); + (raw, mmap) + } else { + // Anonymous mapping + let mmap = mmap_opt.map_anon().map_err(|e| e.to_pyexception(vm))?; + (INVALID_HANDLE_VALUE as isize, MmapObj::Write(mmap)) + }; + + Ok(Self { + closed: AtomicCell::new(false), + mmap: PyMutex::new(Some(mmap)), + handle: AtomicCell::new(handle), + offset, + size: AtomicCell::new(map_size), + pos: AtomicCell::new(0), + exports: AtomicCell::new(0), + access, + }) + } + } + + static BUFFER_METHODS: BufferMethods = BufferMethods { + obj_bytes: |buffer| buffer.obj_as::<PyMmap>().as_bytes(), + obj_bytes_mut: |buffer| buffer.obj_as::<PyMmap>().as_bytes_mut(), + release: |buffer| { + buffer.obj_as::<PyMmap>().exports.fetch_sub(1); + }, + retain: |buffer| { + buffer.obj_as::<PyMmap>().exports.fetch_add(1); + }, + }; + + impl AsBuffer for PyMmap { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let readonly = matches!(zelf.access, AccessMode::Read); + let buf = PyBuffer::new( + zelf.to_owned().into(), + BufferDescriptor::simple(zelf.__len__(), readonly), + &BUFFER_METHODS, + ); + + Ok(buf) + } + } + + impl AsMapping for PyMmap { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: PyMappingMethods = PyMappingMethods { + length: atomic_func!( + |mapping, _vm| Ok(PyMmap::mapping_downcast(mapping).__len__()) + ), + subscript: atomic_func!(|mapping, needle, vm| { + PyMmap::mapping_downcast(mapping).getitem_inner(needle, vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyMmap::mapping_downcast(mapping); + if let Some(value) = value { + PyMmap::setitem_inner(zelf, needle, value, vm) + } else { + Err(vm + .new_type_error("mmap object doesn't support item deletion".to_owned())) + } + }), + }; + &AS_MAPPING + } + } + + impl AsSequence for PyMmap { + fn as_sequence() -> &'static PySequenceMethods { + use rustpython_common::lock::LazyLock; + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyMmap::sequence_downcast(seq).__len__())), + item: atomic_func!(|seq, i, vm| { + let zelf = PyMmap::sequence_downcast(seq); + zelf.getitem_by_index(i, vm) + }), + ass_item: atomic_func!(|seq, i, value, vm| { + let zelf = PyMmap::sequence_downcast(seq); + if let Some(value) = value { + PyMmap::setitem_by_index(zelf, i, value, vm) + } else { + Err(vm + .new_type_error("mmap object doesn't support item deletion".to_owned())) + } + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } + } + + #[pyclass( + with(Constructor, AsMapping, AsSequence, AsBuffer, Representable), + flags(BASETYPE, HAS_WEAKREF) + )] + impl PyMmap { + fn as_bytes_mut(&self) -> BorrowedValueMut<'_, [u8]> { + PyMutexGuard::map(self.mmap.lock(), |m| { + match m.as_mut().expect("mmap closed or invalid") { + MmapObj::Read(_) => panic!("mmap can't modify a readonly memory map."), + MmapObj::Write(mmap) => &mut mmap[..], + #[cfg(windows)] + MmapObj::Named(named) => unsafe { + core::slice::from_raw_parts_mut(named.view_ptr, named.len) + }, + } + }) + .into() + } + + fn as_bytes(&self) -> BorrowedValue<'_, [u8]> { + PyMutexGuard::map_immutable(self.mmap.lock(), |m| { + m.as_ref().expect("mmap closed or invalid").as_slice() + }) + .into() + } + + fn __len__(&self) -> usize { + self.size.load() + } + + #[inline] + fn pos(&self) -> usize { + self.pos.load() + } + + #[inline] + fn advance_pos(&self, step: usize) { + self.pos.store(self.pos() + step); + } + + #[inline] + fn try_writable<R>( + &self, + vm: &VirtualMachine, + f: impl FnOnce(&mut [u8]) -> R, + ) -> PyResult<R> { + if matches!(self.access, AccessMode::Read) { + return Err(vm.new_type_error("mmap can't modify a readonly memory map.")); + } + + match self.check_valid(vm)?.deref_mut().as_mut().unwrap() { + MmapObj::Write(mmap) => Ok(f(&mut mmap[..])), + #[cfg(windows)] + MmapObj::Named(named) => Ok(f(unsafe { + core::slice::from_raw_parts_mut(named.view_ptr, named.len) + })), + _ => unreachable!("already checked"), + } + } + + fn check_valid(&self, vm: &VirtualMachine) -> PyResult<PyMutexGuard<'_, Option<MmapObj>>> { + let m = self.mmap.lock(); + + if m.is_none() { + return Err(vm.new_value_error("mmap closed or invalid")); + } + + Ok(m) + } + + /// TODO: impl resize + #[allow(dead_code)] + fn check_resizeable(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.exports.load() > 0 { + return Err(vm.new_buffer_error("mmap can't resize with extant buffers exported.")); + } + + if self.access == AccessMode::Write || self.access == AccessMode::Default { + return Ok(()); + } + + Err(vm.new_type_error("mmap can't resize a readonly or copy-on-write memory map.")) + } + + #[pygetset] + fn closed(&self) -> bool { + self.closed.load() + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.closed() { + return Ok(()); + } + + if self.exports.load() > 0 { + return Err(vm.new_buffer_error("cannot close exported pointers exist.")); + } + + let mut mmap = self.mmap.lock(); + self.closed.store(true); + *mmap = None; + + self.close_handle(); + + Ok(()) + } + + fn get_find_range(&self, options: FindOptions) -> (usize, usize) { + let size = self.__len__(); + let start = options + .start + .map(|start| start.saturated_at(size)) + .unwrap_or_else(|| self.pos()); + let end = options + .end + .map(|end| end.saturated_at(size)) + .unwrap_or(size); + (start, end) + } + + #[pymethod] + fn find(&self, options: FindOptions, vm: &VirtualMachine) -> PyResult<PyInt> { + let (start, end) = self.get_find_range(options.clone()); + + let sub = &options.sub; + + // returns start position for empty string + if sub.is_empty() { + return Ok(PyInt::from(start as isize)); + } + + let mmap = self.check_valid(vm)?; + let buf = &mmap.as_ref().unwrap().as_slice()[start..end]; + let pos = buf.windows(sub.len()).position(|window| window == sub); + + Ok(pos.map_or(PyInt::from(-1isize), |i| PyInt::from(start + i))) + } + + #[pymethod] + fn rfind(&self, options: FindOptions, vm: &VirtualMachine) -> PyResult<PyInt> { + let (start, end) = self.get_find_range(options.clone()); + + let sub = &options.sub; + // returns start position for empty string + if sub.is_empty() { + return Ok(PyInt::from(start as isize)); + } + + let mmap = self.check_valid(vm)?; + let buf = &mmap.as_ref().unwrap().as_slice()[start..end]; + let pos = buf.windows(sub.len()).rposition(|window| window == sub); + + Ok(pos.map_or(PyInt::from(-1isize), |i| PyInt::from(start + i))) + } + + #[pymethod] + fn flush(&self, options: FlushOptions, vm: &VirtualMachine) -> PyResult<()> { + let (offset, size) = options + .values(self.__len__()) + .ok_or_else(|| vm.new_value_error("flush values out of range"))?; + + if self.access == AccessMode::Read || self.access == AccessMode::Copy { + return Ok(()); + } + + match self.check_valid(vm)?.deref().as_ref().unwrap() { + MmapObj::Read(_mmap) => {} + MmapObj::Write(mmap) => { + mmap.flush_range(offset, size) + .map_err(|e| e.to_pyexception(vm))?; + } + #[cfg(windows)] + MmapObj::Named(named) => { + let ptr = unsafe { named.view_ptr.add(offset) }; + let result = unsafe { FlushViewOfFile(ptr as *const _, size) }; + if result == 0 { + return Err(io::Error::last_os_error().to_pyexception(vm)); + } + } + } + + Ok(()) + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pymethod] + fn madvise(&self, options: AdviseOptions, vm: &VirtualMachine) -> PyResult<()> { + let (option, start, length) = options.values(self.__len__(), vm)?; + let advice = validate_advice(vm, option)?; + + let guard = self.check_valid(vm)?; + let mmap = guard.deref().as_ref().unwrap(); + let ptr = match mmap { + MmapObj::Read(m) => m.as_ptr(), + MmapObj::Write(m) => m.as_ptr(), + }; + + // Apply madvise to the specified range (start, length) + let ptr_with_offset = unsafe { ptr.add(start) }; + let result = + unsafe { libc::madvise(ptr_with_offset as *mut libc::c_void, length, advice) }; + if result != 0 { + return Err(io::Error::last_os_error().to_pyexception(vm)); + } + + Ok(()) + } + + #[pymethod(name = "move")] + fn move_( + &self, + dest: PyIntRef, + src: PyIntRef, + cnt: PyIntRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + fn args( + dest: PyIntRef, + src: PyIntRef, + cnt: PyIntRef, + size: usize, + vm: &VirtualMachine, + ) -> Option<(usize, usize, usize)> { + if dest.as_bigint().is_negative() + || src.as_bigint().is_negative() + || cnt.as_bigint().is_negative() + { + return None; + } + let dest = dest.try_to_primitive(vm).ok()?; + let src = src.try_to_primitive(vm).ok()?; + let cnt = cnt.try_to_primitive(vm).ok()?; + if size - dest < cnt || size - src < cnt { + return None; + } + Some((dest, src, cnt)) + } + + let size = self.__len__(); + let (dest, src, cnt) = args(dest, src, cnt, size, vm) + .ok_or_else(|| vm.new_value_error("source, destination, or count out of range"))?; + + let dest_end = dest + cnt; + let src_end = src + cnt; + + self.try_writable(vm, |mmap| { + let src_buf = mmap[src..src_end].to_vec(); + (&mut mmap[dest..dest_end]) + .write(&src_buf) + .map_err(|e| e.to_pyexception(vm))?; + Ok(()) + })? + } + + #[pymethod] + fn read(&self, n: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + let num_bytes = n + .map(|obj| { + let class = obj.class().to_owned(); + obj.try_into_value::<Option<isize>>(vm).map_err(|_| { + vm.new_type_error(format!( + "read argument must be int or None, not {}", + class.name() + )) + }) + }) + .transpose()? + .flatten(); + let mmap = self.check_valid(vm)?; + let pos = self.pos(); + let remaining = self.__len__().saturating_sub(pos); + let num_bytes = num_bytes + .filter(|&n| n >= 0 && (n as usize) <= remaining) + .map(|n| n as usize) + .unwrap_or(remaining); + + let end_pos = pos + num_bytes; + let bytes = mmap.deref().as_ref().unwrap().as_slice()[pos..end_pos].to_vec(); + + let result = PyBytes::from(bytes).into_ref(&vm.ctx); + + self.advance_pos(num_bytes); + + Ok(result) + } + + #[pymethod] + fn read_byte(&self, vm: &VirtualMachine) -> PyResult<PyIntRef> { + let pos = self.pos(); + if pos >= self.__len__() { + return Err(vm.new_value_error("read byte out of range")); + } + + let b = self.check_valid(vm)?.deref().as_ref().unwrap().as_slice()[pos]; + + self.advance_pos(1); + + Ok(PyInt::from(b).into_ref(&vm.ctx)) + } + + #[pymethod] + fn readline(&self, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + let pos = self.pos(); + let mmap = self.check_valid(vm)?; + + let remaining = self.__len__().saturating_sub(pos); + if remaining == 0 { + return Ok(PyBytes::from(vec![]).into_ref(&vm.ctx)); + } + + let slice = mmap.as_ref().unwrap().as_slice(); + let eof = slice[pos..].iter().position(|&x| x == b'\n'); + + let end_pos = if let Some(i) = eof { + pos + i + 1 + } else { + self.__len__() + }; + + let bytes = slice[pos..end_pos].to_vec(); + + let result = PyBytes::from(bytes).into_ref(&vm.ctx); + + self.advance_pos(end_pos - pos); + + Ok(result) + } + + #[cfg(unix)] + #[pymethod] + fn resize(&self, _newsize: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + self.check_resizeable(vm)?; + // TODO: implement using mremap on Linux + Err(vm.new_system_error("mmap: resizing not available--no mremap()")) + } + + #[cfg(windows)] + #[pymethod] + fn resize(&self, newsize: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + self.check_resizeable(vm)?; + + let newsize: usize = newsize + .try_to_primitive(vm) + .map_err(|_| vm.new_value_error("new size out of range"))?; + + if newsize == 0 { + return Err(vm.new_value_error("new size must be positive")); + } + + let handle = self.handle.load(); + + // Get the lock on mmap + let mut mmap_guard = self.mmap.lock(); + + // Check if this is a Named mmap - these cannot be resized + if let Some(MmapObj::Named(_)) = mmap_guard.as_ref() { + return Err(vm.new_system_error("mmap: cannot resize a named memory mapping")); + } + + let is_anonymous = handle == INVALID_HANDLE_VALUE as isize; + + if is_anonymous { + // For anonymous mmap, we need to: + // 1. Create a new anonymous mmap with the new size + // 2. Copy data from old mmap to new mmap + // 3. Replace the old mmap + + let old_size = self.size.load(); + let copy_size = core::cmp::min(old_size, newsize); + + // Create new anonymous mmap + let mut new_mmap_opts = MmapOptions::new(); + let mut new_mmap = new_mmap_opts + .len(newsize) + .map_anon() + .map_err(|e| e.to_pyexception(vm))?; + + // Copy data from old mmap to new mmap + if let Some(old_mmap) = mmap_guard.as_ref() { + let src = &old_mmap.as_slice()[..copy_size]; + new_mmap[..copy_size].copy_from_slice(src); + } + + *mmap_guard = Some(MmapObj::Write(new_mmap)); + self.size.store(newsize); + } else { + // File-backed mmap resize + + // Drop the current mmap to release the file mapping + *mmap_guard = None; + + // Resize the file + let required_size = self.offset + newsize as i64; + let result = unsafe { + SetFilePointerEx( + handle as HANDLE, + required_size, + core::ptr::null_mut(), + FILE_BEGIN, + ) + }; + if result == 0 { + // Restore original mmap on error + let err = io::Error::last_os_error(); + self.try_restore_mmap(&mut mmap_guard, handle as HANDLE, self.size.load()); + return Err(err.to_pyexception(vm)); + } + + let result = unsafe { SetEndOfFile(handle as HANDLE) }; + if result == 0 { + let err = io::Error::last_os_error(); + self.try_restore_mmap(&mut mmap_guard, handle as HANDLE, self.size.load()); + return Err(err.to_pyexception(vm)); + } + + // Create new mmap with the new size + let new_mmap = + Self::create_mmap_windows(handle as HANDLE, self.offset, newsize, &self.access) + .map_err(|e| e.to_pyexception(vm))?; + + *mmap_guard = Some(new_mmap); + self.size.store(newsize); + } + + // Adjust position if it's beyond the new size + let pos = self.pos.load(); + if pos > newsize { + self.pos.store(newsize); + } + + Ok(()) + } + + #[pymethod] + fn seek( + &self, + dist: isize, + whence: OptionalArg<libc::c_int>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let how = whence.unwrap_or(0); + let size = self.__len__(); + + let new_pos = match how { + 0 => dist, // relative to start + 1 => { + // relative to current position + let pos = self.pos(); + if (((isize::MAX as usize) - pos) as isize) < dist { + return Err(vm.new_value_error("seek out of range")); + } + pos as isize + dist + } + 2 => { + // relative to end + if (((isize::MAX as usize) - size) as isize) < dist { + return Err(vm.new_value_error("seek out of range")); + } + size as isize + dist + } + _ => return Err(vm.new_value_error("unknown seek type")), + }; + + if new_pos < 0 || (new_pos as usize) > size { + return Err(vm.new_value_error("seek out of range")); + } + + self.pos.store(new_pos as usize); + + Ok(()) + } + + #[cfg(unix)] + #[pymethod] + fn size(&self, vm: &VirtualMachine) -> std::io::Result<PyIntRef> { + let fd = unsafe { crt_fd::Borrowed::try_borrow_raw(self.fd.load())? }; + let file_len = fstat(fd)?.st_size; + Ok(PyInt::from(file_len).into_ref(&vm.ctx)) + } + + #[cfg(windows)] + #[pymethod] + fn size(&self, vm: &VirtualMachine) -> PyResult<PyIntRef> { + let handle = self.handle.load(); + if handle == INVALID_HANDLE_VALUE as isize { + // Anonymous mapping, return the mmap size + return Ok(PyInt::from(self.__len__()).into_ref(&vm.ctx)); + } + + let mut high: u32 = 0; + let low = unsafe { GetFileSize(handle as HANDLE, &mut high) }; + if low == u32::MAX { + let err = io::Error::last_os_error(); + if err.raw_os_error() != Some(0) { + return Err(err.to_pyexception(vm)); + } + } + let file_len = ((high as i64) << 32) | (low as i64); + Ok(PyInt::from(file_len).into_ref(&vm.ctx)) + } + + #[pymethod] + fn tell(&self) -> PyResult<usize> { + Ok(self.pos()) + } + + #[pymethod] + fn write(&self, bytes: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyIntRef> { + let pos = self.pos(); + let size = self.__len__(); + + let data = bytes.borrow_buf(); + + if pos > size || size - pos < data.len() { + return Err(vm.new_value_error("data out of range")); + } + + let len = self.try_writable(vm, |mmap| { + (&mut mmap[pos..(pos + data.len())]) + .write(&data) + .map_err(|err| err.to_pyexception(vm))?; + Ok(data.len()) + })??; + + self.advance_pos(len); + + Ok(PyInt::from(len).into_ref(&vm.ctx)) + } + + #[pymethod] + fn write_byte(&self, byte: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let b = value_from_object(vm, &byte)?; + + let pos = self.pos(); + let size = self.__len__(); + + if pos >= size { + return Err(vm.new_value_error("write byte out of range")); + } + + self.try_writable(vm, |mmap| { + mmap[pos] = b; + })?; + + self.advance_pos(1); + + Ok(()) + } + + fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + self.getitem_inner(&needle, vm) + } + + fn __setitem__( + zelf: &Py<Self>, + needle: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + Self::setitem_inner(zelf, &needle, value, vm) + } + + #[pymethod] + fn __enter__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let _m = zelf.check_valid(vm)?; + Ok(zelf.to_owned()) + } + + #[pymethod] + fn __exit__(zelf: &Py<Self>, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + zelf.close(vm) + } + + #[cfg(windows)] + #[pymethod] + fn __sizeof__(&self) -> usize { + core::mem::size_of::<Self>() + } + } + + impl PyMmap { + #[cfg(windows)] + fn create_mmap_windows( + handle: HANDLE, + offset: i64, + size: usize, + access: &AccessMode, + ) -> io::Result<MmapObj> { + use std::fs::File; + + // Create an owned handle wrapper for memmap2 + // We need to create a File from the handle + let file = unsafe { File::from_raw_handle(handle as RawHandle) }; + + let mut mmap_opt = MmapOptions::new(); + let mmap_opt = mmap_opt.offset(offset as u64).len(size); + + let result = match access { + AccessMode::Default | AccessMode::Write => { + unsafe { mmap_opt.map_mut(&file) }.map(MmapObj::Write) + } + AccessMode::Read => unsafe { mmap_opt.map(&file) }.map(MmapObj::Read), + AccessMode::Copy => unsafe { mmap_opt.map_copy(&file) }.map(MmapObj::Write), + }; + + // Don't close the file handle - we're borrowing it + core::mem::forget(file); + + result + } + + /// Try to restore mmap after a failed resize operation. + /// Returns true if restoration succeeded, false otherwise. + /// If restoration fails, marks the mmap as closed. + #[cfg(windows)] + fn try_restore_mmap(&self, mmap_guard: &mut Option<MmapObj>, handle: HANDLE, size: usize) { + match Self::create_mmap_windows(handle, self.offset, size, &self.access) { + Ok(mmap) => *mmap_guard = Some(mmap), + Err(_) => self.closed.store(true), + } + } + + fn getitem_by_index(&self, i: isize, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let i = i + .wrapped_at(self.__len__()) + .ok_or_else(|| vm.new_index_error("mmap index out of range"))?; + + let b = self.check_valid(vm)?.deref().as_ref().unwrap().as_slice()[i]; + + Ok(PyInt::from(b).into_ref(&vm.ctx).into()) + } + + fn getitem_by_slice( + &self, + slice: &SaturatedSlice, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let (range, step, slice_len) = slice.adjust_indices(self.__len__()); + + let mmap = self.check_valid(vm)?; + let slice_data = mmap.deref().as_ref().unwrap().as_slice(); + + if slice_len == 0 { + return Ok(PyBytes::from(vec![]).into_ref(&vm.ctx).into()); + } else if step == 1 { + return Ok(PyBytes::from(slice_data[range].to_vec()) + .into_ref(&vm.ctx) + .into()); + } + + let mut result_buf = Vec::with_capacity(slice_len); + if step.is_negative() { + for i in range.rev().step_by(step.unsigned_abs()) { + result_buf.push(slice_data[i]); + } + } else { + for i in range.step_by(step.unsigned_abs()) { + result_buf.push(slice_data[i]); + } + } + Ok(PyBytes::from(result_buf).into_ref(&vm.ctx).into()) + } + + fn getitem_inner(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + match SequenceIndex::try_from_borrowed_object(vm, needle, "mmap")? { + SequenceIndex::Int(i) => self.getitem_by_index(i, vm), + SequenceIndex::Slice(slice) => self.getitem_by_slice(&slice, vm), + } + } + + fn setitem_inner( + zelf: &Py<Self>, + needle: &PyObject, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + match SequenceIndex::try_from_borrowed_object(vm, needle, "mmap")? { + SequenceIndex::Int(i) => Self::setitem_by_index(zelf, i, value, vm), + SequenceIndex::Slice(slice) => Self::setitem_by_slice(zelf, &slice, value, vm), + } + } + + fn setitem_by_index( + &self, + i: isize, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let i: usize = i + .wrapped_at(self.__len__()) + .ok_or_else(|| vm.new_index_error("mmap index out of range"))?; + + let b = value_from_object(vm, &value)?; + + self.try_writable(vm, |mmap| { + mmap[i] = b; + })?; + + Ok(()) + } + + fn setitem_by_slice( + &self, + slice: &SaturatedSlice, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let (range, step, slice_len) = slice.adjust_indices(self.__len__()); + + let bytes = bytes_from_object(vm, &value)?; + + if bytes.len() != slice_len { + return Err(vm.new_index_error("mmap slice assignment is wrong size")); + } + + if slice_len == 0 { + // do nothing + Ok(()) + } else if step == 1 { + self.try_writable(vm, |mmap| { + (&mut mmap[range]) + .write(&bytes) + .map_err(|err| err.to_pyexception(vm))?; + Ok(()) + })? + } else { + let mut bi = 0; // bytes index + if step.is_negative() { + for i in range.rev().step_by(step.unsigned_abs()) { + self.try_writable(vm, |mmap| { + mmap[i] = bytes[bi]; + })?; + bi += 1; + } + } else { + for i in range.step_by(step.unsigned_abs()) { + self.try_writable(vm, |mmap| { + mmap[i] = bytes[bi]; + })?; + bi += 1; + } + } + Ok(()) + } + } + } + + impl Representable for PyMmap { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let mmap = zelf.mmap.lock(); + + if mmap.is_none() { + return Ok("<mmap.mmap closed=True>".to_owned()); + } + + let access_str = match zelf.access { + AccessMode::Default => "ACCESS_DEFAULT", + AccessMode::Read => "ACCESS_READ", + AccessMode::Write => "ACCESS_WRITE", + AccessMode::Copy => "ACCESS_COPY", + }; + + let repr = format!( + "<mmap.mmap closed=False, access={}, length={}, pos={}, offset={}>", + access_str, + zelf.__len__(), + zelf.pos(), + zelf.offset + ); + + Ok(repr) + } + } +} diff --git a/crates/stdlib/src/multiprocessing.rs b/crates/stdlib/src/multiprocessing.rs new file mode 100644 index 00000000000..26d1bea8859 --- /dev/null +++ b/crates/stdlib/src/multiprocessing.rs @@ -0,0 +1,1151 @@ +pub(crate) use _multiprocessing::module_def; + +#[cfg(windows)] +#[pymodule] +mod _multiprocessing { + use crate::vm::{ + Context, FromArgs, Py, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyDict, PyType, PyTypeRef}, + function::{ArgBytesLike, FuncArgs, KwArgs}, + types::Constructor, + }; + use core::sync::atomic::{AtomicI32, AtomicU32, Ordering}; + use windows_sys::Win32::Foundation::{ + CloseHandle, ERROR_TOO_MANY_POSTS, HANDLE, INVALID_HANDLE_VALUE, WAIT_FAILED, + WAIT_OBJECT_0, WAIT_TIMEOUT, + }; + use windows_sys::Win32::Networking::WinSock::{self, SOCKET}; + use windows_sys::Win32::System::Threading::{ + CreateSemaphoreW, GetCurrentThreadId, INFINITE, ReleaseSemaphore, WaitForSingleObjectEx, + }; + + // These match the values in Lib/multiprocessing/synchronize.py + const RECURSIVE_MUTEX: i32 = 0; + const SEMAPHORE: i32 = 1; + + macro_rules! ismine { + ($self:expr) => { + $self.count.load(Ordering::Acquire) > 0 + && $self.last_tid.load(Ordering::Acquire) == unsafe { GetCurrentThreadId() } + }; + } + + #[derive(FromArgs)] + struct SemLockNewArgs { + #[pyarg(positional)] + kind: i32, + #[pyarg(positional)] + value: i32, + #[pyarg(positional)] + maxvalue: i32, + #[pyarg(positional)] + name: String, + #[pyarg(positional)] + unlink: bool, + } + + #[pyattr] + #[pyclass(name = "SemLock", module = "_multiprocessing")] + #[derive(Debug, PyPayload)] + struct SemLock { + handle: SemHandle, + kind: i32, + maxvalue: i32, + name: Option<String>, + last_tid: AtomicU32, + count: AtomicI32, + } + + #[derive(Debug)] + struct SemHandle { + raw: HANDLE, + } + + unsafe impl Send for SemHandle {} + unsafe impl Sync for SemHandle {} + + impl SemHandle { + fn create(value: i32, maxvalue: i32, vm: &VirtualMachine) -> PyResult<Self> { + let handle = + unsafe { CreateSemaphoreW(core::ptr::null(), value, maxvalue, core::ptr::null()) }; + if handle == 0 as HANDLE { + return Err(vm.new_last_os_error()); + } + Ok(SemHandle { raw: handle }) + } + + #[inline] + fn as_raw(&self) -> HANDLE { + self.raw + } + } + + impl Drop for SemHandle { + fn drop(&mut self) { + if self.raw != 0 as HANDLE && self.raw != INVALID_HANDLE_VALUE { + unsafe { + CloseHandle(self.raw); + } + } + } + } + + /// _GetSemaphoreValue - get value of semaphore by briefly acquiring and releasing + fn get_semaphore_value(handle: HANDLE) -> Result<i32, ()> { + match unsafe { WaitForSingleObjectEx(handle, 0, 0) } { + WAIT_OBJECT_0 => { + let mut previous: i32 = 0; + if unsafe { ReleaseSemaphore(handle, 1, &mut previous) } == 0 { + return Err(()); + } + Ok(previous + 1) + } + WAIT_TIMEOUT => Ok(0), + _ => Err(()), + } + } + + #[pyclass(with(Constructor), flags(BASETYPE))] + impl SemLock { + #[pygetset] + fn handle(&self) -> isize { + self.handle.as_raw() as isize + } + + #[pygetset] + fn kind(&self) -> i32 { + self.kind + } + + #[pygetset] + fn maxvalue(&self) -> i32 { + self.maxvalue + } + + #[pygetset] + fn name(&self) -> Option<String> { + self.name.clone() + } + + #[pymethod] + fn acquire(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<bool> { + let blocking: bool = args + .kwargs + .get("block") + .or_else(|| args.args.first()) + .map(|o| o.clone().try_to_bool(vm)) + .transpose()? + .unwrap_or(true); + + let timeout_obj = args + .kwargs + .get("timeout") + .or_else(|| args.args.get(1)) + .cloned(); + + // Calculate timeout in milliseconds + let full_msecs: u32 = if !blocking { + 0 + } else if timeout_obj.as_ref().is_none_or(|o| vm.is_none(o)) { + INFINITE + } else { + let timeout: f64 = timeout_obj.unwrap().try_float(vm)?.to_f64(); + let timeout = timeout * 1000.0; // convert to ms + if timeout < 0.0 { + 0 + } else if timeout >= 0.5 * INFINITE as f64 { + return Err(vm.new_overflow_error("timeout is too large")); + } else { + (timeout + 0.5) as u32 + } + }; + + // Check whether we already own the lock + if self.kind == RECURSIVE_MUTEX && ismine!(self) { + self.count.fetch_add(1, Ordering::Release); + return Ok(true); + } + + // Check whether we can acquire without blocking + match unsafe { WaitForSingleObjectEx(self.handle.as_raw(), 0, 0) } { + WAIT_OBJECT_0 => { + self.last_tid + .store(unsafe { GetCurrentThreadId() }, Ordering::Release); + self.count.fetch_add(1, Ordering::Release); + return Ok(true); + } + WAIT_FAILED => return Err(vm.new_last_os_error()), + _ => {} + } + + // Poll with signal checking (CPython uses WaitForMultipleObjectsEx + // with sigint_event; we poll since RustPython has no sigint event) + let poll_ms: u32 = 100; + let mut elapsed: u32 = 0; + loop { + let wait_ms = if full_msecs == INFINITE { + poll_ms + } else { + let remaining = full_msecs.saturating_sub(elapsed); + if remaining == 0 { + return Ok(false); + } + remaining.min(poll_ms) + }; + + let res = unsafe { WaitForSingleObjectEx(self.handle.as_raw(), wait_ms, 0) }; + + match res { + WAIT_OBJECT_0 => { + self.last_tid + .store(unsafe { GetCurrentThreadId() }, Ordering::Release); + self.count.fetch_add(1, Ordering::Release); + return Ok(true); + } + WAIT_TIMEOUT => { + vm.check_signals()?; + if full_msecs != INFINITE { + elapsed = elapsed.saturating_add(wait_ms); + } + } + WAIT_FAILED => return Err(vm.new_last_os_error()), + _ => { + return Err(vm.new_runtime_error(format!( + "WaitForSingleObject() gave unrecognized value {res}" + ))); + } + } + } + } + + #[pymethod] + fn release(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.kind == RECURSIVE_MUTEX { + if !ismine!(self) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.assertion_error.to_owned(), + "attempt to release recursive lock not owned by thread".into(), + )); + } + if self.count.load(Ordering::Acquire) > 1 { + self.count.fetch_sub(1, Ordering::Release); + return Ok(()); + } + } + + if unsafe { ReleaseSemaphore(self.handle.as_raw(), 1, core::ptr::null_mut()) } == 0 { + let err = unsafe { windows_sys::Win32::Foundation::GetLastError() }; + if err == ERROR_TOO_MANY_POSTS { + return Err(vm.new_value_error("semaphore or lock released too many times")); + } + return Err(vm.new_last_os_error()); + } + + self.count.fetch_sub(1, Ordering::Release); + Ok(()) + } + + #[pymethod(name = "__enter__")] + fn enter(&self, vm: &VirtualMachine) -> PyResult<bool> { + self.acquire( + FuncArgs::new::<Vec<_>, KwArgs>( + vec![vm.ctx.new_bool(true).into()], + KwArgs::default(), + ), + vm, + ) + } + + #[pymethod] + fn __exit__(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + self.release(vm) + } + + #[pyclassmethod(name = "_rebuild")] + fn rebuild( + cls: PyTypeRef, + handle: isize, + kind: i32, + maxvalue: i32, + name: Option<String>, + vm: &VirtualMachine, + ) -> PyResult { + // On Windows, _rebuild receives the handle directly (no sem_open) + let zelf = SemLock { + handle: SemHandle { + raw: handle as HANDLE, + }, + kind, + maxvalue, + name, + last_tid: AtomicU32::new(0), + count: AtomicI32::new(0), + }; + zelf.into_ref_with_type(vm, cls).map(Into::into) + } + + #[pymethod] + fn _after_fork(&self) { + self.count.store(0, Ordering::Release); + self.last_tid.store(0, Ordering::Release); + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'SemLock' object")) + } + + #[pymethod] + fn _count(&self) -> i32 { + self.count.load(Ordering::Acquire) + } + + #[pymethod] + fn _is_mine(&self) -> bool { + ismine!(self) + } + + #[pymethod] + fn _get_value(&self, vm: &VirtualMachine) -> PyResult<i32> { + get_semaphore_value(self.handle.as_raw()).map_err(|_| vm.new_last_os_error()) + } + + #[pymethod] + fn _is_zero(&self, vm: &VirtualMachine) -> PyResult<bool> { + let val = + get_semaphore_value(self.handle.as_raw()).map_err(|_| vm.new_last_os_error())?; + Ok(val == 0) + } + + #[extend_class] + fn extend_class(ctx: &Context, class: &Py<PyType>) { + class.set_attr( + ctx.intern_str("RECURSIVE_MUTEX"), + ctx.new_int(RECURSIVE_MUTEX).into(), + ); + class.set_attr(ctx.intern_str("SEMAPHORE"), ctx.new_int(SEMAPHORE).into()); + class.set_attr( + ctx.intern_str("SEM_VALUE_MAX"), + ctx.new_int(i32::MAX).into(), + ); + } + } + + impl Constructor for SemLock { + type Args = SemLockNewArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if args.kind != RECURSIVE_MUTEX && args.kind != SEMAPHORE { + return Err(vm.new_value_error("unrecognized kind")); + } + if args.maxvalue <= 0 { + return Err(vm.new_value_error("maxvalue must be positive")); + } + if args.value < 0 || args.value > args.maxvalue { + return Err(vm.new_value_error("invalid value")); + } + + let handle = SemHandle::create(args.value, args.maxvalue, vm)?; + let name = if args.unlink { None } else { Some(args.name) }; + + Ok(SemLock { + handle, + kind: args.kind, + maxvalue: args.maxvalue, + name, + last_tid: AtomicU32::new(0), + count: AtomicI32::new(0), + }) + } + } + + // On Windows, sem_unlink is a no-op + #[pyfunction] + fn sem_unlink(_name: String) {} + + #[pyattr] + fn flags(vm: &VirtualMachine) -> PyRef<PyDict> { + // On Windows, no HAVE_SEM_OPEN / HAVE_SEM_TIMEDWAIT / HAVE_BROKEN_SEM_GETVALUE + vm.ctx.new_dict() + } + + #[pyfunction] + fn closesocket(socket: usize, vm: &VirtualMachine) -> PyResult<()> { + let res = unsafe { WinSock::closesocket(socket as SOCKET) }; + if res != 0 { + Err(vm.new_last_os_error()) + } else { + Ok(()) + } + } + + #[pyfunction] + fn recv(socket: usize, size: usize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut buf = vec![0u8; size]; + let n_read = + unsafe { WinSock::recv(socket as SOCKET, buf.as_mut_ptr() as *mut _, size as i32, 0) }; + if n_read < 0 { + Err(vm.new_last_os_error()) + } else { + buf.truncate(n_read as usize); + Ok(buf) + } + } + + #[pyfunction] + fn send(socket: usize, buf: ArgBytesLike, vm: &VirtualMachine) -> PyResult<libc::c_int> { + let ret = buf.with_ref(|b| unsafe { + WinSock::send(socket as SOCKET, b.as_ptr() as *const _, b.len() as i32, 0) + }); + if ret < 0 { + Err(vm.new_last_os_error()) + } else { + Ok(ret) + } + } +} + +// Unix platforms (Linux, macOS, etc.) +// macOS has broken sem_timedwait/sem_getvalue - we use polled fallback +#[cfg(unix)] +#[pymodule] +mod _multiprocessing { + use crate::vm::{ + Context, FromArgs, Py, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyDict, PyType, PyTypeRef}, + function::{FuncArgs, KwArgs}, + types::Constructor, + }; + use alloc::ffi::CString; + use core::sync::atomic::{AtomicI32, AtomicU64, Ordering}; + use libc::sem_t; + use nix::errno::Errno; + + /// Error type for sem_timedwait operations + #[cfg(target_vendor = "apple")] + enum SemWaitError { + Timeout, + SignalException(PyBaseExceptionRef), + OsError(Errno), + } + + /// macOS fallback for sem_timedwait using select + sem_trywait polling + /// Matches sem_timedwait_save in semaphore.c + #[cfg(target_vendor = "apple")] + fn sem_timedwait_polled( + sem: *mut sem_t, + deadline: &libc::timespec, + vm: &VirtualMachine, + ) -> Result<(), SemWaitError> { + let mut delay: u64 = 0; + + loop { + // poll: try to acquire + if unsafe { libc::sem_trywait(sem) } == 0 { + return Ok(()); + } + let err = Errno::last(); + if err != Errno::EAGAIN { + return Err(SemWaitError::OsError(err)); + } + + // get current time + let mut now = libc::timeval { + tv_sec: 0, + tv_usec: 0, + }; + if unsafe { libc::gettimeofday(&mut now, core::ptr::null_mut()) } < 0 { + return Err(SemWaitError::OsError(Errno::last())); + } + + // check for timeout + let deadline_usec = deadline.tv_sec * 1_000_000 + deadline.tv_nsec / 1000; + #[allow(clippy::unnecessary_cast)] + let now_usec = now.tv_sec as i64 * 1_000_000 + now.tv_usec as i64; + + if now_usec >= deadline_usec { + return Err(SemWaitError::Timeout); + } + + // calculate how much time is left + let difference = (deadline_usec - now_usec) as u64; + + // check delay not too long -- maximum is 20 msecs + delay += 1000; + if delay > 20000 { + delay = 20000; + } + if delay > difference { + delay = difference; + } + + // sleep using select + let mut tv_delay = libc::timeval { + tv_sec: (delay / 1_000_000) as _, + tv_usec: (delay % 1_000_000) as _, + }; + vm.allow_threads(|| unsafe { + libc::select( + 0, + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + &mut tv_delay, + ) + }); + + // check for signals - preserve the exception (e.g., KeyboardInterrupt) + if let Err(exc) = vm.check_signals() { + return Err(SemWaitError::SignalException(exc)); + } + } + } + + // These match the values in Lib/multiprocessing/synchronize.py + const RECURSIVE_MUTEX: i32 = 0; + const SEMAPHORE: i32 = 1; + + // #define ISMINE(o) (o->count > 0 && PyThread_get_thread_ident() == o->last_tid) + macro_rules! ismine { + ($self:expr) => { + $self.count.load(Ordering::Acquire) > 0 + && $self.last_tid.load(Ordering::Acquire) == current_thread_id() + }; + } + + #[derive(FromArgs)] + struct SemLockNewArgs { + #[pyarg(positional)] + kind: i32, + #[pyarg(positional)] + value: i32, + #[pyarg(positional)] + maxvalue: i32, + #[pyarg(positional)] + name: String, + #[pyarg(positional)] + unlink: bool, + } + + #[pyattr] + #[pyclass(name = "SemLock", module = "_multiprocessing")] + #[derive(Debug, PyPayload)] + struct SemLock { + handle: SemHandle, + kind: i32, + maxvalue: i32, + name: Option<String>, + last_tid: AtomicU64, // unsigned long + count: AtomicI32, // int + } + + #[derive(Debug)] + struct SemHandle { + raw: *mut sem_t, + } + + unsafe impl Send for SemHandle {} + unsafe impl Sync for SemHandle {} + + impl SemHandle { + fn create( + name: &str, + value: u32, + unlink: bool, + vm: &VirtualMachine, + ) -> PyResult<(Self, Option<String>)> { + let cname = semaphore_name(vm, name)?; + // SEM_CREATE(name, val, max) sem_open(name, O_CREAT | O_EXCL, 0600, val) + let raw = unsafe { + libc::sem_open(cname.as_ptr(), libc::O_CREAT | libc::O_EXCL, 0o600, value) + }; + if raw == libc::SEM_FAILED { + let err = Errno::last(); + return Err(os_error(vm, err)); + } + if unlink { + // SEM_UNLINK(name) sem_unlink(name) + unsafe { + libc::sem_unlink(cname.as_ptr()); + } + Ok((SemHandle { raw }, None)) + } else { + Ok((SemHandle { raw }, Some(name.to_owned()))) + } + } + + fn open_existing(name: &str, vm: &VirtualMachine) -> PyResult<Self> { + let cname = semaphore_name(vm, name)?; + let raw = unsafe { libc::sem_open(cname.as_ptr(), 0) }; + if raw == libc::SEM_FAILED { + let err = Errno::last(); + return Err(os_error(vm, err)); + } + Ok(SemHandle { raw }) + } + + #[inline] + fn as_ptr(&self) -> *mut sem_t { + self.raw + } + } + + impl Drop for SemHandle { + fn drop(&mut self) { + // Guard against default/uninitialized state. + // Note: SEM_FAILED is (sem_t*)-1, not null, but valid handles are never null + // and SEM_FAILED is never stored (error is returned immediately on sem_open failure). + if !self.raw.is_null() { + // SEM_CLOSE(sem) sem_close(sem) + unsafe { + libc::sem_close(self.raw); + } + } + } + } + + #[pyclass(with(Constructor), flags(BASETYPE))] + impl SemLock { + #[pygetset] + fn handle(&self) -> isize { + self.handle.as_ptr() as isize + } + + #[pygetset] + fn kind(&self) -> i32 { + self.kind + } + + #[pygetset] + fn maxvalue(&self) -> i32 { + self.maxvalue + } + + #[pygetset] + fn name(&self) -> Option<String> { + self.name.clone() + } + + /// Acquire the semaphore/lock. + // _multiprocessing_SemLock_acquire_impl + #[pymethod] + fn acquire(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<bool> { + // block=True, timeout=None + + let blocking: bool = args + .kwargs + .get("block") + .or_else(|| args.args.first()) + .map(|o| o.clone().try_to_bool(vm)) + .transpose()? + .unwrap_or(true); + + let timeout_obj = args + .kwargs + .get("timeout") + .or_else(|| args.args.get(1)) + .cloned(); + + if self.kind == RECURSIVE_MUTEX && ismine!(self) { + self.count.fetch_add(1, Ordering::Release); + return Ok(true); + } + + // timeout_obj != Py_None + let use_deadline = timeout_obj.as_ref().is_some_and(|o| !vm.is_none(o)); + + let deadline = if use_deadline { + let timeout_obj = timeout_obj.unwrap(); + // This accepts both int and float, converting to f64 + let timeout: f64 = timeout_obj.try_float(vm)?.to_f64(); + let timeout = if timeout < 0.0 { 0.0 } else { timeout }; + + let mut tv = libc::timeval { + tv_sec: 0, + tv_usec: 0, + }; + let res = unsafe { libc::gettimeofday(&mut tv, core::ptr::null_mut()) }; + if res < 0 { + return Err(vm.new_os_error("gettimeofday failed".to_string())); + } + + // deadline calculation: + // long sec = (long) timeout; + // long nsec = (long) (1e9 * (timeout - sec) + 0.5); + // deadline.tv_sec = now.tv_sec + sec; + // deadline.tv_nsec = now.tv_usec * 1000 + nsec; + // deadline.tv_sec += (deadline.tv_nsec / 1000000000); + // deadline.tv_nsec %= 1000000000; + let sec = timeout as libc::c_long; + let nsec = (1e9 * (timeout - sec as f64) + 0.5) as libc::c_long; + let mut deadline = libc::timespec { + tv_sec: tv.tv_sec + sec as libc::time_t, + tv_nsec: (tv.tv_usec as libc::c_long * 1000 + nsec) as _, + }; + deadline.tv_sec += (deadline.tv_nsec / 1_000_000_000) as libc::time_t; + deadline.tv_nsec %= 1_000_000_000; + Some(deadline) + } else { + None + }; + + // Check whether we can acquire without releasing the GIL and blocking + let mut res; + loop { + res = unsafe { libc::sem_trywait(self.handle.as_ptr()) }; + if res >= 0 { + break; + } + let err = Errno::last(); + if err == Errno::EINTR { + vm.check_signals()?; + continue; + } + break; + } + + // if (res < 0 && errno == EAGAIN && blocking) + if res < 0 && Errno::last() == Errno::EAGAIN && blocking { + // Couldn't acquire immediately, need to block. + // + // Save errno inside the allow_threads closure, before + // attach_thread() runs — matches CPython which saves + // `err = errno` before Py_END_ALLOW_THREADS. + + #[cfg(not(target_vendor = "apple"))] + { + let mut saved_errno; + loop { + let sem_ptr = self.handle.as_ptr(); + // Py_BEGIN_ALLOW_THREADS / Py_END_ALLOW_THREADS + let (r, e) = if let Some(ref dl) = deadline { + vm.allow_threads(|| { + let r = unsafe { libc::sem_timedwait(sem_ptr, dl) }; + ( + r, + if r < 0 { + Errno::last() + } else { + Errno::from_raw(0) + }, + ) + }) + } else { + vm.allow_threads(|| { + let r = unsafe { libc::sem_wait(sem_ptr) }; + ( + r, + if r < 0 { + Errno::last() + } else { + Errno::from_raw(0) + }, + ) + }) + }; + res = r; + saved_errno = e; + + if res >= 0 { + break; + } + if saved_errno == Errno::EINTR { + vm.check_signals()?; + continue; + } + break; + } + if res < 0 { + return handle_wait_error(vm, saved_errno); + } + } + #[cfg(target_vendor = "apple")] + { + // macOS: use polled fallback since sem_timedwait is not available + if let Some(ref dl) = deadline { + match sem_timedwait_polled(self.handle.as_ptr(), dl, vm) { + Ok(()) => {} + Err(SemWaitError::Timeout) => { + return Ok(false); + } + Err(SemWaitError::SignalException(exc)) => { + return Err(exc); + } + Err(SemWaitError::OsError(e)) => { + return Err(os_error(vm, e)); + } + } + } else { + // No timeout: use sem_wait (available on macOS) + let mut saved_errno; + loop { + let sem_ptr = self.handle.as_ptr(); + let (r, e) = vm.allow_threads(|| { + let r = unsafe { libc::sem_wait(sem_ptr) }; + ( + r, + if r < 0 { + Errno::last() + } else { + Errno::from_raw(0) + }, + ) + }); + res = r; + saved_errno = e; + if res >= 0 { + break; + } + if saved_errno == Errno::EINTR { + vm.check_signals()?; + continue; + } + break; + } + if res < 0 { + return handle_wait_error(vm, saved_errno); + } + } + } + } else if res < 0 { + // Non-blocking path failed, or blocking=false + let err = Errno::last(); + match err { + Errno::EAGAIN | Errno::ETIMEDOUT => return Ok(false), + Errno::EINTR => { + return vm.check_signals().map(|_| false); + } + _ => return Err(os_error(vm, err)), + } + } + + self.count.fetch_add(1, Ordering::Release); + self.last_tid.store(current_thread_id(), Ordering::Release); + + Ok(true) + } + + /// Release the semaphore/lock. + // _multiprocessing_SemLock_release_impl + #[pymethod] + fn release(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.kind == RECURSIVE_MUTEX { + // if (!ISMINE(self)) + if !ismine!(self) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.assertion_error.to_owned(), + "attempt to release recursive lock not owned by thread".into(), + )); + } + // if (self->count > 1) { --self->count; Py_RETURN_NONE; } + if self.count.load(Ordering::Acquire) > 1 { + self.count.fetch_sub(1, Ordering::Release); + return Ok(()); + } + // assert(self->count == 1); + } else { + // SEMAPHORE case: check value before releasing + #[cfg(not(target_vendor = "apple"))] + { + // Linux: use sem_getvalue + let mut sval: libc::c_int = 0; + let res = unsafe { libc::sem_getvalue(self.handle.as_ptr(), &mut sval) }; + if res < 0 { + return Err(os_error(vm, Errno::last())); + } + if sval >= self.maxvalue { + return Err(vm.new_value_error("semaphore or lock released too many times")); + } + } + #[cfg(target_vendor = "apple")] + { + // macOS: HAVE_BROKEN_SEM_GETVALUE + // We will only check properly the maxvalue == 1 case + if self.maxvalue == 1 { + // make sure that already locked + if unsafe { libc::sem_trywait(self.handle.as_ptr()) } < 0 { + if Errno::last() != Errno::EAGAIN { + return Err(os_error(vm, Errno::last())); + } + // it is already locked as expected + } else { + // it was not locked so undo wait and raise + if unsafe { libc::sem_post(self.handle.as_ptr()) } < 0 { + return Err(os_error(vm, Errno::last())); + } + return Err( + vm.new_value_error("semaphore or lock released too many times") + ); + } + } + } + } + + let res = unsafe { libc::sem_post(self.handle.as_ptr()) }; + if res < 0 { + return Err(os_error(vm, Errno::last())); + } + + self.count.fetch_sub(1, Ordering::Release); + Ok(()) + } + + /// Enter the semaphore/lock (context manager). + // _multiprocessing_SemLock___enter___impl + #[pymethod(name = "__enter__")] + fn enter(&self, vm: &VirtualMachine) -> PyResult<bool> { + // return _multiprocessing_SemLock_acquire_impl(self, 1, Py_None); + self.acquire( + FuncArgs::new::<Vec<_>, KwArgs>( + vec![vm.ctx.new_bool(true).into()], + KwArgs::default(), + ), + vm, + ) + } + + /// Exit the semaphore/lock (context manager). + // _multiprocessing_SemLock___exit___impl + #[pymethod] + fn __exit__(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + self.release(vm) + } + + /// Rebuild a SemLock from pickled state. + // _multiprocessing_SemLock__rebuild_impl + #[pyclassmethod(name = "_rebuild")] + fn rebuild( + cls: PyTypeRef, + _handle: isize, + kind: i32, + maxvalue: i32, + name: Option<String>, + vm: &VirtualMachine, + ) -> PyResult { + let Some(ref name_str) = name else { + return Err(vm.new_value_error("cannot rebuild SemLock without name")); + }; + let handle = SemHandle::open_existing(name_str, vm)?; + // return newsemlockobject(type, handle, kind, maxvalue, name_copy); + let zelf = SemLock { + handle, + kind, + maxvalue, + name, + last_tid: AtomicU64::new(0), + count: AtomicI32::new(0), + }; + zelf.into_ref_with_type(vm, cls).map(Into::into) + } + + /// Rezero the net acquisition count after fork(). + // _multiprocessing_SemLock__after_fork_impl + #[pymethod] + fn _after_fork(&self) { + self.count.store(0, Ordering::Release); + // Also reset last_tid for safety + self.last_tid.store(0, Ordering::Release); + } + + /// SemLock objects cannot be pickled directly. + /// Use multiprocessing.synchronize.SemLock wrapper which handles pickling. + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'SemLock' object")) + } + + /// Num of `acquire()`s minus num of `release()`s for this process. + // _multiprocessing_SemLock__count_impl + #[pymethod] + fn _count(&self) -> i32 { + self.count.load(Ordering::Acquire) + } + + /// Whether the lock is owned by this thread. + // _multiprocessing_SemLock__is_mine_impl + #[pymethod] + fn _is_mine(&self) -> bool { + ismine!(self) + } + + /// Get the value of the semaphore. + // _multiprocessing_SemLock__get_value_impl + #[pymethod] + fn _get_value(&self, vm: &VirtualMachine) -> PyResult<i32> { + #[cfg(not(target_vendor = "apple"))] + { + // Linux: use sem_getvalue + let mut sval: libc::c_int = 0; + let res = unsafe { libc::sem_getvalue(self.handle.as_ptr(), &mut sval) }; + if res < 0 { + return Err(os_error(vm, Errno::last())); + } + // some posix implementations use negative numbers to indicate + // the number of waiting threads + Ok(if sval < 0 { 0 } else { sval }) + } + #[cfg(target_vendor = "apple")] + { + // macOS: HAVE_BROKEN_SEM_GETVALUE - raise NotImplementedError + Err(vm.new_not_implemented_error(String::new())) + } + } + + /// Return whether semaphore has value zero. + // _multiprocessing_SemLock__is_zero_impl + #[pymethod] + fn _is_zero(&self, vm: &VirtualMachine) -> PyResult<bool> { + #[cfg(not(target_vendor = "apple"))] + { + Ok(self._get_value(vm)? == 0) + } + #[cfg(target_vendor = "apple")] + { + // macOS: HAVE_BROKEN_SEM_GETVALUE + // Try to acquire - if EAGAIN, value is 0 + if unsafe { libc::sem_trywait(self.handle.as_ptr()) } < 0 { + if Errno::last() == Errno::EAGAIN { + return Ok(true); + } + return Err(os_error(vm, Errno::last())); + } + // Successfully acquired - undo and return false + if unsafe { libc::sem_post(self.handle.as_ptr()) } < 0 { + return Err(os_error(vm, Errno::last())); + } + Ok(false) + } + } + + #[extend_class] + fn extend_class(ctx: &Context, class: &Py<PyType>) { + class.set_attr( + ctx.intern_str("RECURSIVE_MUTEX"), + ctx.new_int(RECURSIVE_MUTEX).into(), + ); + class.set_attr(ctx.intern_str("SEMAPHORE"), ctx.new_int(SEMAPHORE).into()); + // SEM_VALUE_MAX from system, or INT_MAX if negative + // We use a reasonable default + let sem_value_max: i32 = unsafe { + let val = libc::sysconf(libc::_SC_SEM_VALUE_MAX); + if val < 0 || val > i32::MAX as libc::c_long { + i32::MAX + } else { + val as i32 + } + }; + class.set_attr( + ctx.intern_str("SEM_VALUE_MAX"), + ctx.new_int(sem_value_max).into(), + ); + } + } + + impl Constructor for SemLock { + type Args = SemLockNewArgs; + + // Create a new SemLock. + // _multiprocessing_SemLock_impl + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if args.kind != RECURSIVE_MUTEX && args.kind != SEMAPHORE { + return Err(vm.new_value_error("unrecognized kind")); + } + // Value validation + if args.value < 0 || args.value > args.maxvalue { + return Err(vm.new_value_error("invalid value")); + } + + let value = args.value as u32; + let (handle, name) = SemHandle::create(&args.name, value, args.unlink, vm)?; + + // return newsemlockobject(type, handle, kind, maxvalue, name_copy); + Ok(SemLock { + handle, + kind: args.kind, + maxvalue: args.maxvalue, + name, + last_tid: AtomicU64::new(0), + count: AtomicI32::new(0), + }) + } + } + + /// Function to unlink semaphore names. + // _PyMp_sem_unlink. + #[pyfunction] + fn sem_unlink(name: String, vm: &VirtualMachine) -> PyResult<()> { + let cname = semaphore_name(vm, &name)?; + let res = unsafe { libc::sem_unlink(cname.as_ptr()) }; + if res < 0 { + return Err(os_error(vm, Errno::last())); + } + Ok(()) + } + + /// Module-level flags dict. + #[pyattr] + fn flags(vm: &VirtualMachine) -> PyRef<PyDict> { + let flags = vm.ctx.new_dict(); + // HAVE_SEM_OPEN is always 1 on Unix (we wouldn't be here otherwise) + flags + .set_item("HAVE_SEM_OPEN", vm.ctx.new_int(1).into(), vm) + .unwrap(); + + #[cfg(not(target_vendor = "apple"))] + { + // Linux: HAVE_SEM_TIMEDWAIT is available + flags + .set_item("HAVE_SEM_TIMEDWAIT", vm.ctx.new_int(1).into(), vm) + .unwrap(); + } + + #[cfg(target_vendor = "apple")] + { + // macOS: sem_getvalue is broken + flags + .set_item("HAVE_BROKEN_SEM_GETVALUE", vm.ctx.new_int(1).into(), vm) + .unwrap(); + } + + flags + } + + fn semaphore_name(vm: &VirtualMachine, name: &str) -> PyResult<CString> { + // POSIX semaphore names must start with / + let mut full = String::with_capacity(name.len() + 1); + if !name.starts_with('/') { + full.push('/'); + } + full.push_str(name); + CString::new(full).map_err(|_| vm.new_value_error("embedded null character")) + } + + fn handle_wait_error(vm: &VirtualMachine, saved_errno: Errno) -> PyResult<bool> { + match saved_errno { + Errno::EAGAIN | Errno::ETIMEDOUT => Ok(false), + Errno::EINTR => vm.check_signals().map(|_| false), + _ => Err(os_error(vm, saved_errno)), + } + } + + fn os_error(vm: &VirtualMachine, err: Errno) -> PyBaseExceptionRef { + // _PyMp_SetError maps to PyErr_SetFromErrno + let exc_type = match err { + Errno::EEXIST => vm.ctx.exceptions.file_exists_error.to_owned(), + Errno::ENOENT => vm.ctx.exceptions.file_not_found_error.to_owned(), + _ => vm.ctx.exceptions.os_error.to_owned(), + }; + vm.new_os_subtype_error(exc_type, Some(err as i32), err.desc().to_owned()) + .upcast() + } + + /// Get current thread identifier. + /// PyThread_get_thread_ident on Unix (pthread_self). + fn current_thread_id() -> u64 { + unsafe { libc::pthread_self() as u64 } + } +} + +#[cfg(all(not(unix), not(windows)))] +#[pymodule] +mod _multiprocessing {} diff --git a/crates/stdlib/src/openssl.rs b/crates/stdlib/src/openssl.rs new file mode 100644 index 00000000000..00461f1b468 --- /dev/null +++ b/crates/stdlib/src/openssl.rs @@ -0,0 +1,4177 @@ +// spell-checker:disable + +mod cert; + +// SSL exception types (shared with rustls backend) +#[path = "ssl/error.rs"] +mod ssl_error; + +// Conditional compilation for OpenSSL version-specific error codes +cfg_if::cfg_if! { + if #[cfg(ossl310)] { + // OpenSSL 3.1.0+ + mod ssl_data_31; + use ssl_data_31 as ssl_data; + } else if #[cfg(ossl300)] { + // OpenSSL 3.0.0+ + mod ssl_data_300; + use ssl_data_300 as ssl_data; + } else { + // OpenSSL 1.1.1+ (fallback) + mod ssl_data_111; + use ssl_data_111 as ssl_data; + } +} + +pub(crate) use _ssl::module_def; + +use openssl_probe::ProbeResult; +use rustpython_common::lock::LazyLock; + +// define our own copy of ProbeResult so we can handle the vendor case +// easily, without having to have a bunch of cfgs +cfg_if::cfg_if! { + if #[cfg(openssl_vendored)] { + static PROBE: LazyLock<ProbeResult> = LazyLock::new(openssl_probe::probe); + } else { + static PROBE: LazyLock<ProbeResult> = LazyLock::new(|| ProbeResult { cert_file: None, cert_dir: vec![] }); + } +} + +fn probe() -> &'static ProbeResult { + &PROBE +} + +#[allow(non_upper_case_globals)] +#[pymodule(with( + cert::ssl_cert, + ssl_error::ssl_error, + #[cfg(ossl101)] ossl101, + #[cfg(ossl111)] ossl111, + #[cfg(windows)] windows))] +mod _ssl { + use super::{bio, probe}; + + // Import error types and helpers used in this module (others are exposed via pymodule(with(...))) + use super::ssl_error::{ + PySSLCertVerificationError, PySSLError, create_ssl_eof_error, create_ssl_want_read_error, + create_ssl_want_write_error, + }; + use crate::{ + common::lock::{ + PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, + }, + socket::{self, PySocket}, + vm::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{ + PyBaseException, PyBaseExceptionRef, PyBytesRef, PyListRef, PyModule, PyStrRef, + PyType, PyWeak, + }, + class_or_notimplemented, + convert::ToPyException, + exceptions, + function::{ + ArgBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike, Either, FsPath, OptionalArg, + PyComparisonValue, + }, + types::{Comparable, Constructor, PyComparisonOp}, + utils::ToCString, + }, + }; + use crossbeam_utils::atomic::AtomicCell; + use foreign_types_shared::{ForeignType, ForeignTypeRef}; + use openssl::{ + asn1::{Asn1Object, Asn1ObjectRef}, + error::ErrorStack, + nid::Nid, + ssl::{self, SslContextBuilder, SslOptions, SslVerifyMode}, + x509::X509, + }; + use openssl_sys as sys; + use rustpython_vm::ospath::OsPath; + use std::{ + ffi::CStr, + fmt, + io::{Read, Write}, + path::{Path, PathBuf}, + time::Instant, + }; + + // Import certificate types from parent module + use super::cert::{self, cert_to_certificate, cert_to_py}; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + // if openssl is vendored, it doesn't know the locations + // of system certificates - cache the probe result now. + #[cfg(openssl_vendored)] + rustpython_common::lock::LazyLock::force(&super::PROBE); + + __module_exec(vm, module); + Ok(()) + } + + // Re-export PySSLCertificate to make it available in the _ssl module + // It will be automatically exposed to Python via #[pyclass] + #[allow(unused_imports)] + use super::cert::PySSLCertificate; + + // Constants + #[pyattr] + use sys::{ + // SSL Alert Descriptions that are exported by openssl_sys + SSL_AD_DECODE_ERROR, + SSL_AD_ILLEGAL_PARAMETER, + SSL_AD_UNRECOGNIZED_NAME, + // SSL_ERROR_INVALID_ERROR_CODE, + SSL_ERROR_SSL, + // SSL_ERROR_WANT_X509_LOOKUP, + SSL_ERROR_SYSCALL, + SSL_ERROR_WANT_CONNECT, + SSL_ERROR_WANT_READ, + SSL_ERROR_WANT_WRITE, + SSL_ERROR_ZERO_RETURN, + SSL_OP_CIPHER_SERVER_PREFERENCE as OP_CIPHER_SERVER_PREFERENCE, + SSL_OP_ENABLE_MIDDLEBOX_COMPAT as OP_ENABLE_MIDDLEBOX_COMPAT, + SSL_OP_LEGACY_SERVER_CONNECT as OP_LEGACY_SERVER_CONNECT, + SSL_OP_NO_SSLv2 as OP_NO_SSLv2, + SSL_OP_NO_SSLv3 as OP_NO_SSLv3, + SSL_OP_NO_TICKET as OP_NO_TICKET, + SSL_OP_NO_TLSv1 as OP_NO_TLSv1, + SSL_OP_SINGLE_DH_USE as OP_SINGLE_DH_USE, + SSL_OP_SINGLE_ECDH_USE as OP_SINGLE_ECDH_USE, + X509_V_FLAG_ALLOW_PROXY_CERTS as VERIFY_ALLOW_PROXY_CERTS, + X509_V_FLAG_CRL_CHECK as VERIFY_CRL_CHECK_LEAF, + X509_V_FLAG_PARTIAL_CHAIN as VERIFY_X509_PARTIAL_CHAIN, + X509_V_FLAG_TRUSTED_FIRST as VERIFY_X509_TRUSTED_FIRST, + X509_V_FLAG_X509_STRICT as VERIFY_X509_STRICT, + }; + + // SSL Alert Descriptions (RFC 5246 and extensions) + // Hybrid approach: use openssl_sys constants where available, hardcode others + #[pyattr] + const ALERT_DESCRIPTION_CLOSE_NOTIFY: libc::c_int = 0; + #[pyattr] + const ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: libc::c_int = 10; + #[pyattr] + const ALERT_DESCRIPTION_BAD_RECORD_MAC: libc::c_int = 20; + #[pyattr] + const ALERT_DESCRIPTION_RECORD_OVERFLOW: libc::c_int = 22; + #[pyattr] + const ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: libc::c_int = 30; + #[pyattr] + const ALERT_DESCRIPTION_HANDSHAKE_FAILURE: libc::c_int = 40; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE: libc::c_int = 42; + #[pyattr] + const ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: libc::c_int = 43; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_REVOKED: libc::c_int = 44; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: libc::c_int = 45; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: libc::c_int = 46; + #[pyattr] + const ALERT_DESCRIPTION_ILLEGAL_PARAMETER: libc::c_int = SSL_AD_ILLEGAL_PARAMETER; + #[pyattr] + const ALERT_DESCRIPTION_UNKNOWN_CA: libc::c_int = 48; + #[pyattr] + const ALERT_DESCRIPTION_ACCESS_DENIED: libc::c_int = 49; + #[pyattr] + const ALERT_DESCRIPTION_DECODE_ERROR: libc::c_int = SSL_AD_DECODE_ERROR; + #[pyattr] + const ALERT_DESCRIPTION_DECRYPT_ERROR: libc::c_int = 51; + #[pyattr] + const ALERT_DESCRIPTION_PROTOCOL_VERSION: libc::c_int = 70; + #[pyattr] + const ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: libc::c_int = 71; + #[pyattr] + const ALERT_DESCRIPTION_INTERNAL_ERROR: libc::c_int = 80; + #[pyattr] + const ALERT_DESCRIPTION_USER_CANCELLED: libc::c_int = 90; + #[pyattr] + const ALERT_DESCRIPTION_NO_RENEGOTIATION: libc::c_int = 100; + #[pyattr] + const ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: libc::c_int = 110; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: libc::c_int = 111; + #[pyattr] + const ALERT_DESCRIPTION_UNRECOGNIZED_NAME: libc::c_int = SSL_AD_UNRECOGNIZED_NAME; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: libc::c_int = 113; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: libc::c_int = 114; + #[pyattr] + const ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: libc::c_int = 115; + + // CRL verification constants + #[pyattr] + const VERIFY_CRL_CHECK_CHAIN: libc::c_ulong = + sys::X509_V_FLAG_CRL_CHECK | sys::X509_V_FLAG_CRL_CHECK_ALL; + + // taken from CPython, should probably be kept up to date with their version if it ever changes + #[pyattr] + const _DEFAULT_CIPHERS: &str = + "DEFAULT:!aNULL:!eNULL:!MD5:!3DES:!DES:!RC4:!IDEA:!SEED:!aDSS:!SRP:!PSK"; + // #[pyattr] PROTOCOL_SSLv2: u32 = SslVersion::Ssl2 as u32; // unsupported + // #[pyattr] PROTOCOL_SSLv3: u32 = SslVersion::Ssl3 as u32; + #[pyattr] + const PROTOCOL_SSLv23: u32 = SslVersion::Tls as u32; + #[pyattr] + const PROTOCOL_TLS: u32 = SslVersion::Tls as u32; + #[pyattr] + const PROTOCOL_TLS_CLIENT: u32 = SslVersion::TlsClient as u32; + #[pyattr] + const PROTOCOL_TLS_SERVER: u32 = SslVersion::TlsServer as u32; + #[pyattr] + const PROTOCOL_TLSv1: u32 = SslVersion::Tls1 as u32; + #[pyattr] + const PROTOCOL_TLSv1_1: u32 = SslVersion::Tls1_1 as u32; + #[pyattr] + const PROTOCOL_TLSv1_2: u32 = SslVersion::Tls1_2 as u32; + #[pyattr] + const PROTO_MINIMUM_SUPPORTED: i32 = ProtoVersion::MinSupported as i32; + #[pyattr] + const PROTO_SSLv3: i32 = ProtoVersion::Ssl3 as i32; + #[pyattr] + const PROTO_TLSv1: i32 = ProtoVersion::Tls1 as i32; + #[pyattr] + const PROTO_TLSv1_1: i32 = ProtoVersion::Tls1_1 as i32; + #[pyattr] + const PROTO_TLSv1_2: i32 = ProtoVersion::Tls1_2 as i32; + #[pyattr] + const PROTO_TLSv1_3: i32 = ProtoVersion::Tls1_3 as i32; + #[pyattr] + const PROTO_MAXIMUM_SUPPORTED: i32 = ProtoVersion::MaxSupported as i32; + #[pyattr] + const OP_ALL: libc::c_ulong = (sys::SSL_OP_ALL & !sys::SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) as _; + #[pyattr] + const HAS_TLS_UNIQUE: bool = true; + #[pyattr] + const CERT_NONE: u32 = CertRequirements::None as u32; + #[pyattr] + const CERT_OPTIONAL: u32 = CertRequirements::Optional as u32; + #[pyattr] + const CERT_REQUIRED: u32 = CertRequirements::Required as u32; + #[pyattr] + const VERIFY_DEFAULT: u32 = 0; + #[pyattr] + const HAS_SNI: bool = true; + #[pyattr] + const HAS_ECDH: bool = true; + #[pyattr] + const HAS_NPN: bool = false; + #[pyattr] + const HAS_ALPN: bool = true; + #[pyattr] + const HAS_SSLv2: bool = false; + #[pyattr] + const HAS_SSLv3: bool = false; + #[pyattr] + const HAS_TLSv1: bool = true; + #[pyattr] + const HAS_TLSv1_1: bool = true; + #[pyattr] + const HAS_TLSv1_2: bool = true; + #[pyattr] + const HAS_TLSv1_3: bool = cfg!(ossl111); + #[pyattr] + const HAS_PSK: bool = true; + #[pyattr] + const HAS_PHA: bool = cfg!(ossl111); + + // Encoding constants for Certificate.public_bytes() + #[pyattr] + pub(crate) const ENCODING_PEM: i32 = sys::X509_FILETYPE_PEM; + #[pyattr] + pub(crate) const ENCODING_DER: i32 = sys::X509_FILETYPE_ASN1; + #[pyattr] + const ENCODING_PEM_AUX: i32 = sys::X509_FILETYPE_PEM + 0x100; + + // OpenSSL error codes for unexpected EOF detection + const ERR_LIB_SSL: i32 = 20; + const SSL_R_UNEXPECTED_EOF_WHILE_READING: i32 = 294; + + // SSL_VERIFY constants for post-handshake authentication + #[cfg(ossl111)] + const SSL_VERIFY_POST_HANDSHAKE: libc::c_int = 0x20; + + // the openssl version from the API headers + + #[pyattr(name = "OPENSSL_VERSION")] + fn openssl_version(_vm: &VirtualMachine) -> &str { + openssl::version::version() + } + #[pyattr(name = "OPENSSL_VERSION_NUMBER")] + fn openssl_version_number(_vm: &VirtualMachine) -> i64 { + openssl::version::number() + } + #[pyattr(name = "OPENSSL_VERSION_INFO")] + fn openssl_version_info(_vm: &VirtualMachine) -> OpensslVersionInfo { + parse_version_info(openssl::version::number()) + } + + #[pyattr(name = "_OPENSSL_API_VERSION")] + fn _openssl_api_version(_vm: &VirtualMachine) -> OpensslVersionInfo { + let openssl_api_version = i64::from_str_radix(env!("OPENSSL_API_VERSION"), 16) + .expect("OPENSSL_API_VERSION is malformed"); + parse_version_info(openssl_api_version) + } + + type OpensslVersionInfo = (u8, u8, u8, u8, u8); + const fn parse_version_info(mut n: i64) -> OpensslVersionInfo { + let status = (n & 0xF) as u8; + n >>= 4; + let patch = (n & 0xFF) as u8; + n >>= 8; + let fix = (n & 0xFF) as u8; + n >>= 8; + let minor = (n & 0xFF) as u8; + n >>= 8; + let major = (n & 0xFF) as u8; + (major, minor, fix, patch, status) + } + + #[derive(Copy, Clone, num_enum::IntoPrimitive, num_enum::TryFromPrimitive, PartialEq)] + #[repr(i32)] + enum SslVersion { + Ssl2, + Ssl3 = 1, + Tls, + Tls1, + Tls1_1, + Tls1_2, + TlsClient = 0x10, + TlsServer, + } + + #[derive(Copy, Clone, num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] + #[repr(i32)] + enum ProtoVersion { + MinSupported = -2, + Ssl3 = sys::SSL3_VERSION, + Tls1 = sys::TLS1_VERSION, + Tls1_1 = sys::TLS1_1_VERSION, + Tls1_2 = sys::TLS1_2_VERSION, + #[cfg(ossl111)] + Tls1_3 = sys::TLS1_3_VERSION, + #[cfg(not(ossl111))] + Tls1_3 = 0x304, + MaxSupported = -1, + } + + #[derive(num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] + #[repr(i32)] + enum CertRequirements { + None, + Optional, + Required, + } + + #[derive(Debug, PartialEq)] + enum SslServerOrClient { + Client, + Server, + } + + unsafe fn ptr2obj(ptr: *mut sys::ASN1_OBJECT) -> Option<Asn1Object> { + if ptr.is_null() { + None + } else { + Some(unsafe { Asn1Object::from_ptr(ptr) }) + } + } + + fn _txt2obj(s: &CStr, no_name: bool) -> Option<Asn1Object> { + unsafe { ptr2obj(sys::OBJ_txt2obj(s.as_ptr(), i32::from(no_name))) } + } + fn _nid2obj(nid: Nid) -> Option<Asn1Object> { + unsafe { ptr2obj(sys::OBJ_nid2obj(nid.as_raw())) } + } + + type PyNid = (libc::c_int, String, String, Option<String>); + fn obj2py(obj: &Asn1ObjectRef, vm: &VirtualMachine) -> PyResult<PyNid> { + let nid = obj.nid(); + let short_name = nid + .short_name() + .map_err(|_| vm.new_value_error("NID has no short name"))? + .to_owned(); + let long_name = nid + .long_name() + .map_err(|_| vm.new_value_error("NID has no long name"))? + .to_owned(); + Ok(( + nid.as_raw(), + short_name, + long_name, + cert::obj2txt(obj, true), + )) + } + + #[derive(FromArgs)] + struct Txt2ObjArgs { + txt: PyStrRef, + #[pyarg(any, default = false)] + name: bool, + } + + #[pyfunction] + fn txt2obj(args: Txt2ObjArgs, vm: &VirtualMachine) -> PyResult<PyNid> { + _txt2obj(&args.txt.to_cstring(vm)?, !args.name) + .as_deref() + .ok_or_else(|| vm.new_value_error(format!("unknown object '{}'", args.txt))) + .and_then(|obj| obj2py(obj, vm)) + } + + #[pyfunction] + fn nid2obj(nid: libc::c_int, vm: &VirtualMachine) -> PyResult<PyNid> { + _nid2obj(Nid::from_raw(nid)) + .as_deref() + .ok_or_else(|| vm.new_value_error(format!("unknown NID {nid}"))) + .and_then(|obj| obj2py(obj, vm)) + } + + // Lazily compute and cache cert file/dir paths + static CERT_PATHS: LazyLock<(PathBuf, PathBuf)> = LazyLock::new(|| { + fn path_from_cstr(c: &CStr) -> PathBuf { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + std::ffi::OsStr::from_bytes(c.to_bytes()).into() + } + #[cfg(windows)] + { + // Use lossy conversion for potential non-UTF8 + PathBuf::from(c.to_string_lossy().as_ref()) + } + } + + let probe = probe(); + let cert_file = probe + .cert_file + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| { + path_from_cstr(unsafe { CStr::from_ptr(sys::X509_get_default_cert_file()) }) + }); + let cert_dir = probe + .cert_dir + .first() + .map(PathBuf::from) + .unwrap_or_else(|| { + path_from_cstr(unsafe { CStr::from_ptr(sys::X509_get_default_cert_dir()) }) + }); + (cert_file, cert_dir) + }); + + fn get_cert_file_dir() -> (&'static Path, &'static Path) { + let (cert_file, cert_dir) = &*CERT_PATHS; + (cert_file.as_path(), cert_dir.as_path()) + } + + // Lazily compute and cache cert environment variable names + static CERT_ENV_NAMES: LazyLock<(String, String)> = LazyLock::new(|| { + let cert_file_env = unsafe { CStr::from_ptr(sys::X509_get_default_cert_file_env()) } + .to_string_lossy() + .into_owned(); + let cert_dir_env = unsafe { CStr::from_ptr(sys::X509_get_default_cert_dir_env()) } + .to_string_lossy() + .into_owned(); + (cert_file_env, cert_dir_env) + }); + + #[pyfunction] + fn get_default_verify_paths( + vm: &VirtualMachine, + ) -> PyResult<(&'static str, PyObjectRef, &'static str, PyObjectRef)> { + let (cert_file_env, cert_dir_env) = &*CERT_ENV_NAMES; + let (cert_file, cert_dir) = get_cert_file_dir(); + let cert_file = OsPath::new_str(cert_file).filename(vm); + let cert_dir = OsPath::new_str(cert_dir).filename(vm); + Ok(( + cert_file_env.as_str(), + cert_file, + cert_dir_env.as_str(), + cert_dir, + )) + } + + #[pyfunction(name = "RAND_status")] + fn rand_status() -> i32 { + unsafe { sys::RAND_status() } + } + + #[pyfunction(name = "RAND_add")] + fn rand_add(string: ArgStrOrBytesLike, entropy: f64) { + let f = |b: &[u8]| { + for buf in b.chunks(libc::c_int::MAX as usize) { + unsafe { sys::RAND_add(buf.as_ptr() as *const _, buf.len() as _, entropy) } + } + }; + f(&string.borrow_bytes()) + } + + #[pyfunction(name = "RAND_bytes")] + fn rand_bytes(n: i32, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + if n < 0 { + return Err(vm.new_value_error("num must be positive")); + } + let mut buf = vec![0; n as usize]; + openssl::rand::rand_bytes(&mut buf).map_err(|e| convert_openssl_error(vm, e))?; + Ok(buf) + } + + // Callback data stored in SSL ex_data for SNI/msg callbacks + struct SniCallbackData { + ssl_context: PyRef<PySslContext>, + // Use weak reference to avoid reference cycle: + // PySslSocket -> SslStream -> SSL -> ex_data -> SniCallbackData -> PySslSocket + ssl_socket_weak: PyRef<PyWeak>, + } + + // Thread-local storage for VirtualMachine pointer during handshake + // SNI callback is only called during handshake which is synchronous + thread_local! { + static HANDSHAKE_VM: core::cell::Cell<Option<*const VirtualMachine>> = const { core::cell::Cell::new(None) }; + // SSL pointer during handshake - needed because connection lock is held during handshake + // and callbacks may need to access SSL without acquiring the lock + static HANDSHAKE_SSL_PTR: core::cell::Cell<Option<*mut sys::SSL>> = const { core::cell::Cell::new(None) }; + } + + // RAII guard to set/clear thread-local handshake context + struct HandshakeVmGuard { + _ssl_ptr: *mut sys::SSL, + } + + impl HandshakeVmGuard { + fn new(vm: &VirtualMachine, ssl_ptr: *mut sys::SSL) -> Self { + HANDSHAKE_VM.with(|cell| cell.set(Some(vm as *const _))); + HANDSHAKE_SSL_PTR.with(|cell| cell.set(Some(ssl_ptr))); + HandshakeVmGuard { _ssl_ptr: ssl_ptr } + } + } + + impl Drop for HandshakeVmGuard { + fn drop(&mut self) { + HANDSHAKE_VM.with(|cell| cell.set(None)); + HANDSHAKE_SSL_PTR.with(|cell| cell.set(None)); + } + } + + // Get SSL pointer - either from thread-local (during handshake) or from connection + fn get_ssl_ptr_for_context_change(connection: &PyRwLock<SslConnection>) -> *mut sys::SSL { + // First check if we're in a handshake callback (lock already held) + if let Some(ptr) = HANDSHAKE_SSL_PTR.with(|cell| cell.get()) { + return ptr; + } + // Otherwise, acquire the lock normally + connection.read().ssl().as_ptr() + } + + // Get or create an ex_data index for SNI callback data + fn get_sni_ex_data_index() -> libc::c_int { + use rustpython_common::lock::LazyLock; + static SNI_EX_DATA_IDX: LazyLock<libc::c_int> = LazyLock::new(|| unsafe { + sys::SSL_get_ex_new_index( + 0, + std::ptr::null_mut(), + None, + None, + Some(sni_callback_data_free), + ) + }); + *SNI_EX_DATA_IDX + } + + // Free function for callback data + // NOTE: We don't free the data here because it's managed manually in do_handshake + // to avoid use-after-free when the SSL object is dropped after timeout + unsafe extern "C" fn sni_callback_data_free( + _parent: *mut libc::c_void, + _ptr: *mut libc::c_void, + _ad: *mut sys::CRYPTO_EX_DATA, + _idx: libc::c_int, + _argl: libc::c_long, + _argp: *mut libc::c_void, + ) { + // Intentionally empty - data is freed in cleanup_sni_ex_data() + } + + // Clean up SNI callback data from SSL ex_data + // Called after handshake to free the data and release references + unsafe fn cleanup_sni_ex_data(ssl_ptr: *mut sys::SSL) { + unsafe { + let idx = get_sni_ex_data_index(); + let data_ptr = sys::SSL_get_ex_data(ssl_ptr, idx); + if !data_ptr.is_null() { + // Free the Box<SniCallbackData> - this releases references to context and socket + let _ = Box::from_raw(data_ptr as *mut SniCallbackData); + // Clear the ex_data to prevent double-free + sys::SSL_set_ex_data(ssl_ptr, idx, std::ptr::null_mut()); + } + } + } + + // Get or create an ex_data index for msg_callback data + fn get_msg_callback_ex_data_index() -> libc::c_int { + use rustpython_common::lock::LazyLock; + static MSG_CB_EX_DATA_IDX: LazyLock<libc::c_int> = LazyLock::new(|| unsafe { + sys::SSL_get_ex_new_index( + 0, + std::ptr::null_mut(), + None, + None, + Some(msg_callback_data_free), + ) + }); + *MSG_CB_EX_DATA_IDX + } + + // Free function for msg_callback data - called by OpenSSL when SSL is freed + unsafe extern "C" fn msg_callback_data_free( + _parent: *mut libc::c_void, + ptr: *mut libc::c_void, + _ad: *mut sys::CRYPTO_EX_DATA, + _idx: libc::c_int, + _argl: libc::c_long, + _argp: *mut libc::c_void, + ) { + if !ptr.is_null() { + unsafe { + // Reconstruct PyObjectRef and drop to decrement reference count + let raw = std::ptr::NonNull::new_unchecked(ptr as *mut PyObject); + let _ = PyObjectRef::from_raw(raw); + } + } + } + + // SNI callback function called by OpenSSL + unsafe extern "C" fn _servername_callback( + ssl_ptr: *mut sys::SSL, + al: *mut libc::c_int, + arg: *mut libc::c_void, + ) -> libc::c_int { + const SSL_TLSEXT_ERR_OK: libc::c_int = 0; + const SSL_TLSEXT_ERR_ALERT_FATAL: libc::c_int = 2; + const SSL_AD_INTERNAL_ERROR: libc::c_int = 80; + const TLSEXT_NAMETYPE_host_name: libc::c_int = 0; + + if arg.is_null() { + return SSL_TLSEXT_ERR_OK; + } + + unsafe { + let ctx = &*(arg as *const PySslContext); + + // Get the callback + let callback_opt = ctx.sni_callback.lock().clone(); + let Some(callback) = callback_opt else { + return SSL_TLSEXT_ERR_OK; + }; + + // Get callback data from SSL ex_data + let idx = get_sni_ex_data_index(); + let data_ptr = sys::SSL_get_ex_data(ssl_ptr, idx); + if data_ptr.is_null() { + return SSL_TLSEXT_ERR_ALERT_FATAL; + } + + let callback_data = &*(data_ptr as *const SniCallbackData); + + // Get VM from thread-local storage (set by HandshakeVmGuard in do_handshake) + let Some(vm_ptr) = HANDSHAKE_VM.with(|cell| cell.get()) else { + // VM not available - this shouldn't happen during handshake + *al = SSL_AD_INTERNAL_ERROR; + return SSL_TLSEXT_ERR_ALERT_FATAL; + }; + let vm = &*vm_ptr; + + // Get server name + let servername = sys::SSL_get_servername(ssl_ptr, TLSEXT_NAMETYPE_host_name); + let server_name_arg = if servername.is_null() { + vm.ctx.none() + } else { + let name_cstr = std::ffi::CStr::from_ptr(servername); + match name_cstr.to_str() { + Ok(name_str) => vm.ctx.new_str(name_str).into(), + Err(_) => vm.ctx.none(), + } + }; + + // Get SSL socket from callback data via weak reference + let ssl_socket_obj = callback_data + .ssl_socket_weak + .upgrade() + .unwrap_or_else(|| vm.ctx.none()); + + // Call the Python callback + match callback.call( + ( + ssl_socket_obj, + server_name_arg, + callback_data.ssl_context.to_owned(), + ), + vm, + ) { + Ok(result) => { + // Check return value type (must be None or integer) + if vm.is_none(&result) { + // None is OK + SSL_TLSEXT_ERR_OK + } else { + // Try to convert to integer + match result.try_to_value::<i32>(vm) { + Ok(alert_code) => { + // Valid integer - use as alert code + *al = alert_code; + SSL_TLSEXT_ERR_ALERT_FATAL + } + Err(_) => { + // Type conversion failed - raise TypeError + let type_error = vm.new_type_error(format!( + "servername callback must return None or an integer, not '{}'", + result.class().name() + )); + vm.run_unraisable(type_error, None, result); + *al = SSL_AD_INTERNAL_ERROR; + SSL_TLSEXT_ERR_ALERT_FATAL + } + } + } + } + Err(exc) => { + // Log the exception but don't propagate it + vm.run_unraisable(exc, None, vm.ctx.none()); + *al = SSL_AD_INTERNAL_ERROR; + SSL_TLSEXT_ERR_ALERT_FATAL + } + } + } + } + + // OpenSSL record type constants for msg_callback + const SSL3_RT_CHANGE_CIPHER_SPEC: i32 = 20; + const SSL3_RT_ALERT: i32 = 21; + const SSL3_RT_HANDSHAKE: i32 = 22; + const SSL3_RT_HEADER: i32 = 256; + const SSL3_RT_INNER_CONTENT_TYPE: i32 = 257; + // Special value for change cipher spec (CPython compatibility) + const SSL3_MT_CHANGE_CIPHER_SPEC: i32 = 0x0101; + + // Message callback function called by OpenSSL + // Called during SSL operations to report protocol messages. + // debughelpers.c:_PySSL_msg_callback + unsafe extern "C" fn _msg_callback( + write_p: libc::c_int, + mut version: libc::c_int, + content_type: libc::c_int, + buf: *const libc::c_void, + len: usize, + ssl_ptr: *mut sys::SSL, + _arg: *mut libc::c_void, + ) { + if ssl_ptr.is_null() { + return; + } + + unsafe { + // Get SSL socket from ex_data using the dedicated index + let idx = get_msg_callback_ex_data_index(); + let ssl_socket_ptr = sys::SSL_get_ex_data(ssl_ptr, idx); + if ssl_socket_ptr.is_null() { + return; + } + + // ssl_socket_ptr is a pointer to Box<Py<PySslSocket>>, set in _wrap_socket/_wrap_bio + let ssl_socket: &Py<PySslSocket> = &*(ssl_socket_ptr as *const Py<PySslSocket>); + + // Get the callback from the context + let callback_opt = ssl_socket.ctx.read().msg_callback.lock().clone(); + let Some(callback) = callback_opt else { + return; + }; + + // Get VM from thread-local storage (set by HandshakeVmGuard in do_handshake) + let Some(vm_ptr) = HANDSHAKE_VM.with(|cell| cell.get()) else { + // VM not available - this shouldn't happen during handshake + return; + }; + let vm = &*vm_ptr; + + // Get SSL socket owner object + let ssl_socket_obj = ssl_socket + .owner + .read() + .as_ref() + .and_then(|weak| weak.upgrade()) + .unwrap_or_else(|| vm.ctx.none()); + + // Create the message bytes + let buf_slice = std::slice::from_raw_parts(buf as *const u8, len); + let msg_bytes = vm.ctx.new_bytes(buf_slice.to_vec()); + + // Determine direction string + let direction_str = if write_p != 0 { "write" } else { "read" }; + + // Calculate msg_type based on content_type (debughelpers.c behavior) + let msg_type = match content_type { + SSL3_RT_CHANGE_CIPHER_SPEC => SSL3_MT_CHANGE_CIPHER_SPEC, + SSL3_RT_ALERT => { + // byte 1 is alert type + if len >= 2 { buf_slice[1] as i32 } else { -1 } + } + SSL3_RT_HANDSHAKE => { + // byte 0 is handshake type + if !buf_slice.is_empty() { + buf_slice[0] as i32 + } else { + -1 + } + } + SSL3_RT_HEADER => { + // Frame header: version in bytes 1..2, type in byte 0 + if len >= 3 { + version = ((buf_slice[1] as i32) << 8) | (buf_slice[2] as i32); + buf_slice[0] as i32 + } else { + -1 + } + } + SSL3_RT_INNER_CONTENT_TYPE => { + // Inner content type in byte 0 + if !buf_slice.is_empty() { + buf_slice[0] as i32 + } else { + -1 + } + } + _ => -1, + }; + + // Call the Python callback + // Signature: callback(conn, direction, version, content_type, msg_type, data) + match callback.call( + ( + ssl_socket_obj, + vm.ctx.new_str(direction_str), + vm.ctx.new_int(version), + vm.ctx.new_int(content_type), + vm.ctx.new_int(msg_type), + msg_bytes, + ), + vm, + ) { + Ok(_) => {} + Err(exc) => { + // Log the exception but don't propagate it + vm.run_unraisable(exc, None, vm.ctx.none()); + } + } + } + } + + #[pyfunction(name = "RAND_pseudo_bytes")] + fn rand_pseudo_bytes(n: i32, vm: &VirtualMachine) -> PyResult<(Vec<u8>, bool)> { + if n < 0 { + return Err(vm.new_value_error("num must be positive")); + } + let mut buf = vec![0; n as usize]; + let ret = unsafe { sys::RAND_bytes(buf.as_mut_ptr(), n) }; + match ret { + 0 | 1 => Ok((buf, ret == 1)), + _ => Err(convert_openssl_error(vm, ErrorStack::get())), + } + } + + #[pyattr] + #[pyclass(module = "ssl", name = "_SSLContext")] + #[derive(PyPayload)] + struct PySslContext { + ctx: PyRwLock<SslContextBuilder>, + check_hostname: AtomicCell<bool>, + protocol: SslVersion, + post_handshake_auth: PyMutex<bool>, + sni_callback: PyMutex<Option<PyObjectRef>>, + msg_callback: PyMutex<Option<PyObjectRef>>, + psk_client_callback: PyMutex<Option<PyObjectRef>>, + psk_server_callback: PyMutex<Option<PyObjectRef>>, + psk_identity_hint: PyMutex<Option<String>>, + } + + impl fmt::Debug for PySslContext { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("_SSLContext") + } + } + + fn builder_as_ctx(x: &SslContextBuilder) -> &ssl::SslContextRef { + unsafe { ssl::SslContextRef::from_ptr(x.as_ptr()) } + } + + impl Constructor for PySslContext { + type Args = i32; + + fn py_new( + _cls: &Py<PyType>, + proto_version: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let proto = SslVersion::try_from(proto_version) + .map_err(|_| vm.new_value_error("invalid protocol version"))?; + let method = match proto { + // SslVersion::Ssl3 => unsafe { ssl::SslMethod::from_ptr(sys::SSLv3_method()) }, + SslVersion::Tls => ssl::SslMethod::tls(), + SslVersion::Tls1 => ssl::SslMethod::tls(), + SslVersion::Tls1_1 => ssl::SslMethod::tls(), + SslVersion::Tls1_2 => ssl::SslMethod::tls(), + SslVersion::TlsClient => ssl::SslMethod::tls_client(), + SslVersion::TlsServer => ssl::SslMethod::tls_server(), + _ => return Err(vm.new_value_error("invalid protocol version")), + }; + let mut builder = + SslContextBuilder::new(method).map_err(|e| convert_openssl_error(vm, e))?; + + #[cfg(target_os = "android")] + android::load_client_ca_list(vm, &mut builder)?; + + let check_hostname = proto == SslVersion::TlsClient; + builder.set_verify(if check_hostname { + SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT + } else { + SslVerifyMode::NONE + }); + + // Start with OP_ALL but remove options that CPython doesn't include by default + let mut options = SslOptions::ALL & !SslOptions::DONT_INSERT_EMPTY_FRAGMENTS; + if proto != SslVersion::Ssl2 { + options |= SslOptions::NO_SSLV2; + } + if proto != SslVersion::Ssl3 { + options |= SslOptions::NO_SSLV3; + } + options |= SslOptions::NO_COMPRESSION; + options |= SslOptions::CIPHER_SERVER_PREFERENCE; + options |= SslOptions::SINGLE_DH_USE; + options |= SslOptions::SINGLE_ECDH_USE; + options |= SslOptions::ENABLE_MIDDLEBOX_COMPAT; + builder.set_options(options); + // Remove NO_TLSv1 and NO_TLSv1_1 which newer OpenSSL adds to OP_ALL + builder.clear_options(SslOptions::NO_TLSV1 | SslOptions::NO_TLSV1_1); + + let mode = ssl::SslMode::ACCEPT_MOVING_WRITE_BUFFER | ssl::SslMode::AUTO_RETRY; + builder.set_mode(mode); + + #[cfg(ossl111)] + unsafe { + sys::SSL_CTX_set_post_handshake_auth(builder.as_ptr(), 0); + } + + // Note: Unlike some other implementations, we do NOT set session_id_context at the + // context level. CPython sets it only on individual SSL objects (server-side only). + // This matches CPython's behavior in _ssl.c where SSL_set_session_id_context is called + // in newPySSLSocket() at line 862, not during context creation. + + // Set protocol version limits based on the protocol version + unsafe { + let ctx_ptr = builder.as_ptr(); + match proto { + SslVersion::Tls1 => { + sys::SSL_CTX_set_min_proto_version(ctx_ptr, sys::TLS1_VERSION); + sys::SSL_CTX_set_max_proto_version(ctx_ptr, sys::TLS1_VERSION); + } + SslVersion::Tls1_1 => { + sys::SSL_CTX_set_min_proto_version(ctx_ptr, sys::TLS1_1_VERSION); + sys::SSL_CTX_set_max_proto_version(ctx_ptr, sys::TLS1_1_VERSION); + } + SslVersion::Tls1_2 => { + sys::SSL_CTX_set_min_proto_version(ctx_ptr, sys::TLS1_2_VERSION); + sys::SSL_CTX_set_max_proto_version(ctx_ptr, sys::TLS1_2_VERSION); + } + _ => { + // For Tls, TlsClient, TlsServer, use default (no restrictions) + } + } + } + + // Set default verify flags: VERIFY_X509_TRUSTED_FIRST + unsafe { + let ctx_ptr = builder.as_ptr(); + let param = sys::SSL_CTX_get0_param(ctx_ptr); + sys::X509_VERIFY_PARAM_set_flags(param, sys::X509_V_FLAG_TRUSTED_FIRST); + } + + Ok(PySslContext { + ctx: PyRwLock::new(builder), + check_hostname: AtomicCell::new(check_hostname), + protocol: proto, + post_handshake_auth: PyMutex::new(false), + sni_callback: PyMutex::new(None), + msg_callback: PyMutex::new(None), + psk_client_callback: PyMutex::new(None), + psk_server_callback: PyMutex::new(None), + psk_identity_hint: PyMutex::new(None), + }) + } + } + + #[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(Constructor))] + impl PySslContext { + fn builder(&self) -> PyRwLockWriteGuard<'_, SslContextBuilder> { + self.ctx.write() + } + fn ctx(&self) -> PyMappedRwLockReadGuard<'_, ssl::SslContextRef> { + PyRwLockReadGuard::map(self.ctx.read(), builder_as_ctx) + } + + #[pygetset] + fn post_handshake_auth(&self) -> bool { + *self.post_handshake_auth.lock() + } + #[pygetset(setter)] + fn set_post_handshake_auth( + &self, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let value = value.ok_or_else(|| vm.new_attribute_error("cannot delete attribute"))?; + *self.post_handshake_auth.lock() = value.is_true(vm)?; + Ok(()) + } + + #[cfg(ossl110)] + #[pygetset] + fn security_level(&self) -> i32 { + unsafe { SSL_CTX_get_security_level(self.ctx().as_ptr()) } + } + + #[pymethod] + fn set_ciphers(&self, cipherlist: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let ciphers = cipherlist.as_str(); + if ciphers.contains('\0') { + return Err(exceptions::cstring_error(vm)); + } + self.builder() + .set_cipher_list(ciphers) + .map_err(|_| new_ssl_error(vm, "No cipher can be selected.")) + } + + #[pymethod] + fn get_ciphers(&self, vm: &VirtualMachine) -> PyResult<PyListRef> { + let ctx = self.ctx(); + let ssl = ssl::Ssl::new(&ctx).map_err(|e| convert_openssl_error(vm, e))?; + + unsafe { + let ciphers_ptr = SSL_get_ciphers(ssl.as_ptr()); + if ciphers_ptr.is_null() { + return Ok(vm.ctx.new_list(vec![])); + } + + let num_ciphers = sys::OPENSSL_sk_num(ciphers_ptr as *const _); + let mut result = Vec::new(); + + for i in 0..num_ciphers { + let cipher_ptr = + sys::OPENSSL_sk_value(ciphers_ptr as *const _, i) as *const sys::SSL_CIPHER; + let cipher = ssl::SslCipherRef::from_ptr(cipher_ptr as *mut _); + + let (name, version, bits) = cipher_to_tuple(cipher); + let dict = vm.ctx.new_dict(); + dict.set_item("name", vm.ctx.new_str(name).into(), vm)?; + dict.set_item("protocol", vm.ctx.new_str(version).into(), vm)?; + dict.set_item("secret_bits", vm.ctx.new_int(bits).into(), vm)?; + + // Add description field + let description = cipher_description(cipher_ptr); + dict.set_item("description", vm.ctx.new_str(description).into(), vm)?; + + result.push(dict.into()); + } + + Ok(vm.ctx.new_list(result)) + } + } + + #[pymethod] + fn set_ecdh_curve( + &self, + name: Either<PyStrRef, ArgBytesLike>, + vm: &VirtualMachine, + ) -> PyResult<()> { + use openssl::ec::{EcGroup, EcKey}; + + // Convert name to CString, supporting both str and bytes + let name_cstr = match name { + Either::A(s) => { + if s.as_str().contains('\0') { + return Err(exceptions::cstring_error(vm)); + } + s.to_cstring(vm)? + } + Either::B(b) => std::ffi::CString::new(b.borrow_buf().to_vec()) + .map_err(|_| exceptions::cstring_error(vm))?, + }; + + // Find the NID for the curve name using OBJ_sn2nid + let nid_raw = unsafe { sys::OBJ_sn2nid(name_cstr.as_ptr()) }; + if nid_raw == 0 { + return Err(vm.new_value_error("unknown curve name")); + } + let nid = Nid::from_raw(nid_raw); + + // Create EC key from the curve + let group = EcGroup::from_curve_name(nid).map_err(|e| convert_openssl_error(vm, e))?; + let key = EcKey::from_group(&group).map_err(|e| convert_openssl_error(vm, e))?; + + // Set the temporary ECDH key + self.builder() + .set_tmp_ecdh(&key) + .map_err(|e| convert_openssl_error(vm, e)) + } + + #[pygetset] + fn options(&self) -> libc::c_ulong { + self.ctx.read().options().bits() as _ + } + #[pygetset(setter)] + fn set_options(&self, new_opts: i64, vm: &VirtualMachine) -> PyResult<()> { + if new_opts < 0 { + return Err(vm.new_value_error("invalid options value")); + } + let new_opts = new_opts as libc::c_ulong; + let mut ctx = self.builder(); + // Get current options + let current = ctx.options().bits() as libc::c_ulong; + + // Calculate options to clear and set + let clear = current & !new_opts; + let set = !current & new_opts; + + // Clear options first (using raw FFI since openssl crate doesn't expose clear_options) + if clear != 0 { + unsafe { + sys::SSL_CTX_clear_options(ctx.as_ptr(), clear); + } + } + + // Then set new options + if set != 0 { + ctx.set_options(SslOptions::from_bits_truncate(set as _)); + } + Ok(()) + } + #[pygetset] + fn protocol(&self) -> i32 { + self.protocol as i32 + } + #[pygetset] + fn verify_mode(&self) -> i32 { + let mode = self.ctx().verify_mode(); + if mode == SslVerifyMode::NONE { + CertRequirements::None.into() + } else if mode == SslVerifyMode::PEER { + CertRequirements::Optional.into() + } else if mode == SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT { + CertRequirements::Required.into() + } else { + unreachable!() + } + } + #[pygetset(setter)] + fn set_verify_mode(&self, cert: i32, vm: &VirtualMachine) -> PyResult<()> { + let mut ctx = self.builder(); + let cert_req = CertRequirements::try_from(cert) + .map_err(|_| vm.new_value_error("invalid value for verify_mode"))?; + let mode = match cert_req { + CertRequirements::None if self.check_hostname.load() => { + return Err(vm.new_value_error( + "Cannot set verify_mode to CERT_NONE when check_hostname is enabled.", + )); + } + CertRequirements::None => SslVerifyMode::NONE, + CertRequirements::Optional => SslVerifyMode::PEER, + CertRequirements::Required => { + SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT + } + }; + ctx.set_verify(mode); + Ok(()) + } + #[pygetset] + fn verify_flags(&self) -> libc::c_ulong { + unsafe { + let ctx_ptr = self.ctx().as_ptr(); + let param = sys::SSL_CTX_get0_param(ctx_ptr); + sys::X509_VERIFY_PARAM_get_flags(param) + } + } + #[pygetset(setter)] + fn set_verify_flags(&self, new_flags: libc::c_ulong, vm: &VirtualMachine) -> PyResult<()> { + unsafe { + let ctx_ptr = self.ctx().as_ptr(); + let param = sys::SSL_CTX_get0_param(ctx_ptr); + let flags = sys::X509_VERIFY_PARAM_get_flags(param); + let clear = flags & !new_flags; + let set = !flags & new_flags; + + if clear != 0 && sys::X509_VERIFY_PARAM_clear_flags(param, clear) == 0 { + return Err(new_ssl_error(vm, "Failed to clear verify flags")); + } + if set != 0 && sys::X509_VERIFY_PARAM_set_flags(param, set) == 0 { + return Err(new_ssl_error(vm, "Failed to set verify flags")); + } + Ok(()) + } + } + #[pygetset] + fn check_hostname(&self) -> bool { + self.check_hostname.load() + } + #[pygetset(setter)] + fn set_check_hostname(&self, ch: bool) { + let mut ctx = self.builder(); + if ch && builder_as_ctx(&ctx).verify_mode() == SslVerifyMode::NONE { + ctx.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT); + } + self.check_hostname.store(ch); + } + + // PY_PROTO_MINIMUM_SUPPORTED = -2, PY_PROTO_MAXIMUM_SUPPORTED = -1 + #[pygetset] + fn minimum_version(&self) -> i32 { + let ctx = self.ctx(); + let version = unsafe { sys::SSL_CTX_get_min_proto_version(ctx.as_ptr()) }; + if version == 0 { + -2 // PY_PROTO_MINIMUM_SUPPORTED + } else { + version + } + } + #[pygetset(setter)] + fn set_minimum_version(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + // Handle special values + let proto_version = match value { + -2 => { + // PY_PROTO_MINIMUM_SUPPORTED -> use minimum available (TLS 1.2) + sys::TLS1_2_VERSION + } + -1 => { + // PY_PROTO_MAXIMUM_SUPPORTED -> use maximum available + // For max on min_proto_version, we use the newest available + sys::TLS1_3_VERSION + } + _ => value, + }; + + let ctx = self.builder(); + let result = unsafe { sys::SSL_CTX_set_min_proto_version(ctx.as_ptr(), proto_version) }; + if result == 0 { + return Err(vm.new_value_error("invalid protocol version")); + } + Ok(()) + } + + #[pygetset] + fn maximum_version(&self) -> i32 { + let ctx = self.ctx(); + let version = unsafe { sys::SSL_CTX_get_max_proto_version(ctx.as_ptr()) }; + if version == 0 { + -1 // PY_PROTO_MAXIMUM_SUPPORTED + } else { + version + } + } + #[pygetset(setter)] + fn set_maximum_version(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + // Handle special values + let proto_version = match value { + -1 => { + // PY_PROTO_MAXIMUM_SUPPORTED -> use 0 for OpenSSL (means no limit) + 0 + } + -2 => { + // PY_PROTO_MINIMUM_SUPPORTED -> use minimum available (TLS 1.2) + sys::TLS1_2_VERSION + } + _ => value, + }; + + let ctx = self.builder(); + let result = unsafe { sys::SSL_CTX_set_max_proto_version(ctx.as_ptr(), proto_version) }; + if result == 0 { + return Err(vm.new_value_error("invalid protocol version")); + } + Ok(()) + } + + #[pygetset] + fn num_tickets(&self, _vm: &VirtualMachine) -> PyResult<usize> { + // Only supported for TLS 1.3 + #[cfg(ossl110)] + { + let ctx = self.ctx(); + let num = unsafe { sys::SSL_CTX_get_num_tickets(ctx.as_ptr()) }; + Ok(num) + } + #[cfg(not(ossl110))] + { + Ok(0) + } + } + #[pygetset(setter)] + fn set_num_tickets(&self, value: isize, vm: &VirtualMachine) -> PyResult<()> { + // Check for negative values + if value < 0 { + return Err(vm.new_value_error("num_tickets must be a non-negative integer")); + } + + // Check that this is a server context + if self.protocol != SslVersion::TlsServer { + return Err(vm.new_value_error("SSLContext is not a server context.")); + } + + #[cfg(ossl110)] + { + let ctx = self.builder(); + let result = unsafe { sys::SSL_CTX_set_num_tickets(ctx.as_ptr(), value as usize) }; + if result != 1 { + return Err(vm.new_value_error("failed to set num tickets.")); + } + Ok(()) + } + #[cfg(not(ossl110))] + { + let _ = (value, vm); + Ok(()) + } + } + + #[pymethod] + fn set_default_verify_paths(&self, vm: &VirtualMachine) -> PyResult<()> { + cfg_if::cfg_if! { + if #[cfg(openssl_vendored)] { + let (cert_file, cert_dir) = get_cert_file_dir(); + self.builder() + .load_verify_locations(Some(cert_file), Some(cert_dir)) + .map_err(|e| convert_openssl_error(vm, e)) + } else { + self.builder() + .set_default_verify_paths() + .map_err(|e| convert_openssl_error(vm, e)) + } + } + } + + #[pymethod] + fn _set_alpn_protocols(&self, protos: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { + #[cfg(ossl102)] + { + let mut ctx = self.builder(); + let server = protos.with_ref(|pbuf| { + if pbuf.len() > libc::c_uint::MAX as usize { + return Err(vm.new_overflow_error(format!( + "protocols longer than {} bytes", + libc::c_uint::MAX + ))); + } + ctx.set_alpn_protos(pbuf) + .map_err(|e| convert_openssl_error(vm, e))?; + Ok(pbuf.to_vec()) + })?; + ctx.set_alpn_select_callback(move |_, client| { + let proto = + ssl::select_next_proto(&server, client).ok_or(ssl::AlpnError::NOACK)?; + let pos = memchr::memmem::find(client, proto) + .expect("selected alpn proto should be present in client protos"); + Ok(&client[pos..pos + proto.len()]) + }); + Ok(()) + } + #[cfg(not(ossl102))] + { + Err(vm.new_not_implemented_error( + "The NPN extension requires OpenSSL 1.0.1 or later.", + )) + } + } + + #[pymethod] + fn set_psk_client_callback( + &self, + callback: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Cannot add PSK client callback to a server context + if self.protocol == SslVersion::TlsServer { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Cannot add PSK client callback to a PROTOCOL_TLS_SERVER context" + .to_owned(), + ) + .upcast()); + } + + if vm.is_none(&callback) { + *self.psk_client_callback.lock() = None; + unsafe { + sys::SSL_CTX_set_psk_client_callback(self.builder().as_ptr(), None); + } + } else { + if !callback.is_callable() { + return Err(vm.new_type_error("callback must be callable")); + } + *self.psk_client_callback.lock() = Some(callback); + // Note: The actual callback will be invoked via SSL app_data mechanism + // when do_handshake is called + } + Ok(()) + } + + #[pymethod] + fn set_psk_server_callback( + &self, + callback: PyObjectRef, + identity_hint: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Cannot add PSK server callback to a client context + if self.protocol == SslVersion::TlsClient { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Cannot add PSK server callback to a PROTOCOL_TLS_CLIENT context" + .to_owned(), + ) + .upcast()); + } + + if vm.is_none(&callback) { + *self.psk_server_callback.lock() = None; + *self.psk_identity_hint.lock() = None; + unsafe { + sys::SSL_CTX_set_psk_server_callback(self.builder().as_ptr(), None); + } + } else { + if !callback.is_callable() { + return Err(vm.new_type_error("callback must be callable")); + } + *self.psk_server_callback.lock() = Some(callback); + if let OptionalArg::Present(hint) = identity_hint { + *self.psk_identity_hint.lock() = Some(hint.as_str().to_owned()); + } + // Note: The actual callback will be invoked via SSL app_data mechanism + } + Ok(()) + } + + #[pymethod] + fn load_verify_locations( + &self, + args: LoadVerifyLocationsArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let (None, None, None) = (&args.cafile, &args.capath, &args.cadata) { + return Err(vm.new_type_error("cafile, capath and cadata cannot be all omitted")); + } + + #[cold] + fn invalid_cadata(vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_type_error("cadata should be an ASCII string or a bytes-like object") + } + + let mut ctx = self.builder(); + + // validate cadata type and load cadata + if let Some(cadata) = args.cadata { + let (certs, is_pem) = match cadata { + Either::A(s) => { + if !s.as_str().is_ascii() { + return Err(invalid_cadata(vm)); + } + (X509::stack_from_pem(s.as_bytes()), true) + } + Either::B(b) => (b.with_ref(x509_stack_from_der), false), + }; + let certs = certs.map_err(|e| convert_openssl_error(vm, e))?; + + // If no certificates were loaded, raise an error + if certs.is_empty() { + let msg = if is_pem { + "no start line: cadata does not contain a certificate" + } else { + "not enough data: cadata does not contain a certificate" + }; + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + msg.to_owned(), + ) + .upcast()); + } + + let store = ctx.cert_store_mut(); + for cert in certs { + store + .add_cert(cert) + .map_err(|e| convert_openssl_error(vm, e))?; + } + } + + if args.cafile.is_some() || args.capath.is_some() { + let cafile_path = args.cafile.map(|p| p.to_path_buf(vm)).transpose()?; + let capath_path = args.capath.map(|p| p.to_path_buf(vm)).transpose()?; + // Check file/directory existence before calling OpenSSL to get proper errno + if let Some(ref path) = cafile_path + && !path.exists() + { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + format!("No such file or directory: '{}'", path.display()), + ) + .upcast()); + } + if let Some(ref path) = capath_path + && !path.exists() + { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + format!("No such file or directory: '{}'", path.display()), + ) + .upcast()); + } + ctx.load_verify_locations(cafile_path.as_deref(), capath_path.as_deref()) + .map_err(|e| convert_openssl_error(vm, e))?; + } + + Ok(()) + } + + #[pymethod] + fn get_ca_certs( + &self, + args: GetCertArgs, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + let binary_form = args.binary_form.unwrap_or(false); + let ctx = self.ctx(); + #[cfg(ossl300)] + let certs = ctx.cert_store().all_certificates(); + #[cfg(not(ossl300))] + let certs = ctx.cert_store().objects().iter().filter_map(|x| x.x509()); + + // Filter to only include CA certificates (Basic Constraints: CA=TRUE) + let certs = certs + .into_iter() + .filter(|cert| { + unsafe { + // X509_check_ca() returns 1 for CA certificates + X509_check_ca(cert.as_ptr()) == 1 + } + }) + .map(|ref cert| cert_to_py(vm, cert, binary_form)) + .collect::<Result<Vec<_>, _>>()?; + Ok(certs) + } + + #[pymethod] + fn cert_store_stats(&self, vm: &VirtualMachine) -> PyResult { + let ctx = self.ctx(); + let store_ptr = unsafe { sys::SSL_CTX_get_cert_store(ctx.as_ptr()) }; + + if store_ptr.is_null() { + return Err(vm.new_memory_error("failed to get cert store")); + } + + let objs_ptr = unsafe { sys::X509_STORE_get0_objects(store_ptr) }; + if objs_ptr.is_null() { + return Err(vm.new_memory_error("failed to query cert store")); + } + + let mut x509_count = 0; + let mut crl_count = 0; + let mut ca_count = 0; + + unsafe { + let num_objs = sys::OPENSSL_sk_num(objs_ptr as *const _); + for i in 0..num_objs { + let obj_ptr = + sys::OPENSSL_sk_value(objs_ptr as *const _, i) as *const sys::X509_OBJECT; + let obj_type = X509_OBJECT_get_type(obj_ptr); + + match obj_type { + X509_LU_X509 => { + x509_count += 1; + let x509_ptr = sys::X509_OBJECT_get0_X509(obj_ptr); + // X509_check_ca returns non-zero for any CA type + if !x509_ptr.is_null() && X509_check_ca(x509_ptr) != 0 { + ca_count += 1; + } + } + X509_LU_CRL => { + crl_count += 1; + } + _ => { + // Ignore unrecognized types + } + } + } + // Note: No need to free objs_ptr as X509_STORE_get0_objects returns + // a pointer to internal data that should not be freed by the caller + } + + let dict = vm.ctx.new_dict(); + dict.set_item("x509", vm.ctx.new_int(x509_count).into(), vm)?; + dict.set_item("crl", vm.ctx.new_int(crl_count).into(), vm)?; + dict.set_item("x509_ca", vm.ctx.new_int(ca_count).into(), vm)?; + Ok(dict.into()) + } + + #[pymethod] + fn session_stats(&self, vm: &VirtualMachine) -> PyResult { + let ctx = self.ctx(); + let ctx_ptr = ctx.as_ptr(); + + let dict = vm.ctx.new_dict(); + + macro_rules! add_stat { + ($key:expr, $func:ident) => { + let value = unsafe { $func(ctx_ptr) }; + dict.set_item($key, vm.ctx.new_int(value).into(), vm)?; + }; + } + + add_stat!("number", SSL_CTX_sess_number); + add_stat!("connect", SSL_CTX_sess_connect); + add_stat!("connect_good", SSL_CTX_sess_connect_good); + add_stat!("connect_renegotiate", SSL_CTX_sess_connect_renegotiate); + add_stat!("accept", SSL_CTX_sess_accept); + add_stat!("accept_good", SSL_CTX_sess_accept_good); + add_stat!("accept_renegotiate", SSL_CTX_sess_accept_renegotiate); + add_stat!("hits", SSL_CTX_sess_hits); + add_stat!("misses", SSL_CTX_sess_misses); + add_stat!("timeouts", SSL_CTX_sess_timeouts); + add_stat!("cache_full", SSL_CTX_sess_cache_full); + + Ok(dict.into()) + } + + #[pymethod] + fn load_dh_params(&self, filepath: FsPath, vm: &VirtualMachine) -> PyResult<()> { + let path = filepath.to_path_buf(vm)?; + + // Open the file using fopen (cross-platform) + let fp = + rustpython_common::fileutils::fopen(path.as_path(), "rb").map_err(|e| { + match e.kind() { + std::io::ErrorKind::NotFound => vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + e.to_string(), + ) + .upcast(), + _ => vm.new_os_error(e.to_string()), + } + })?; + + // Read DH parameters + let dh = unsafe { + PEM_read_DHparams( + fp, + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + unsafe { + libc::fclose(fp); + } + + if dh.is_null() { + return Err(convert_openssl_error(vm, ErrorStack::get())); + } + + // Set temporary DH parameters + let ctx = self.builder(); + let result = unsafe { sys::SSL_CTX_set_tmp_dh(ctx.as_ptr(), dh) }; + unsafe { + sys::DH_free(dh); + } + + if result != 1 { + return Err(convert_openssl_error(vm, ErrorStack::get())); + } + + Ok(()) + } + + #[pygetset] + fn sni_callback(&self) -> Option<PyObjectRef> { + self.sni_callback.lock().clone() + } + + #[pygetset(setter)] + fn set_sni_callback( + &self, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if this is a server context + if self.protocol == SslVersion::TlsClient { + return Err(vm.new_value_error("sni_callback cannot be set on TLS_CLIENT context")); + } + + let mut callback_guard = self.sni_callback.lock(); + + if let Some(callback_obj) = value { + if !vm.is_none(&callback_obj) { + // Check if callable + if !callback_obj.is_callable() { + return Err(vm.new_type_error("not a callable object")); + } + + // Set the callback + *callback_guard = Some(callback_obj); + + // Set OpenSSL callback + unsafe { + sys::SSL_CTX_set_tlsext_servername_callback__fixed_rust( + self.ctx().as_ptr(), + Some(_servername_callback), + ); + sys::SSL_CTX_set_tlsext_servername_arg( + self.ctx().as_ptr(), + self as *const _ as *mut _, + ); + } + } else { + // Clear callback + *callback_guard = None; + unsafe { + sys::SSL_CTX_set_tlsext_servername_callback__fixed_rust( + self.ctx().as_ptr(), + None, + ); + } + } + } else { + // Clear callback + *callback_guard = None; + unsafe { + sys::SSL_CTX_set_tlsext_servername_callback__fixed_rust( + self.ctx().as_ptr(), + None, + ); + } + } + + Ok(()) + } + + #[pymethod] + fn set_servername_callback( + &self, + callback: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.set_sni_callback(callback, vm) + } + + #[pygetset(name = "_msg_callback")] + fn msg_callback(&self) -> Option<PyObjectRef> { + self.msg_callback.lock().clone() + } + + #[pygetset(setter, name = "_msg_callback")] + fn set_msg_callback( + &self, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut callback_guard = self.msg_callback.lock(); + + if let Some(callback_obj) = value { + if !vm.is_none(&callback_obj) { + // Check if callable + if !callback_obj.is_callable() { + return Err(vm.new_type_error("not a callable object")); + } + + // Set the callback + *callback_guard = Some(callback_obj); + + // Set OpenSSL callback + unsafe { + SSL_CTX_set_msg_callback(self.ctx().as_ptr(), Some(_msg_callback)); + } + } else { + // Clear callback + *callback_guard = None; + unsafe { + SSL_CTX_set_msg_callback(self.ctx().as_ptr(), None); + } + } + } else { + // Clear callback when value is None + *callback_guard = None; + unsafe { + SSL_CTX_set_msg_callback(self.ctx().as_ptr(), None); + } + } + + Ok(()) + } + + #[pymethod] + fn load_cert_chain(&self, args: LoadCertChainArgs, vm: &VirtualMachine) -> PyResult<()> { + use openssl::pkey::PKey; + use std::cell::RefCell; + + let LoadCertChainArgs { + certfile, + keyfile, + password, + } = args; + + let mut ctx = self.builder(); + let key_path = keyfile.map(|path| path.to_path_buf(vm)).transpose()?; + let cert_path = certfile.to_path_buf(vm)?; + + // Check file existence before calling OpenSSL to get proper errno + if !cert_path.exists() { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + format!("No such file or directory: '{}'", cert_path.display()), + ) + .upcast()); + } + if let Some(ref kp) = key_path + && !kp.exists() + { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(libc::ENOENT), + format!("No such file or directory: '{}'", kp.display()), + ) + .upcast()); + } + + // Load certificate chain + ctx.set_certificate_chain_file(&cert_path) + .map_err(|e| convert_openssl_error(vm, e))?; + + // Load private key - handle password if provided + let key_file_path = key_path.as_ref().unwrap_or(&cert_path); + + // PEM_BUFSIZE = 1024 (maximum password length in OpenSSL) + const PEM_BUFSIZE: usize = 1024; + + // Read key file data + let key_data = std::fs::read(key_file_path) + .map_err(|e| crate::vm::convert::ToPyException::to_pyexception(&e, vm))?; + + let pkey = if let Some(ref pw_obj) = password { + if pw_obj.is_callable() { + // Callable password - use callback that calls Python function + // Store any Python error that occurs in the callback + let py_error: RefCell<Option<PyBaseExceptionRef>> = RefCell::new(None); + + let result = PKey::private_key_from_pem_callback(&key_data, |buf| { + // Call the Python password callback + let pw_result = pw_obj.call((), vm); + match pw_result { + Ok(result) => { + // Extract password bytes + match Self::extract_password_bytes( + &result, + "password callback must return a string", + vm, + ) { + Ok(pw) => { + // Check password length + if pw.len() > PEM_BUFSIZE { + *py_error.borrow_mut() = + Some(vm.new_value_error(format!( + "password cannot be longer than {} bytes", + PEM_BUFSIZE + ))); + return Err(openssl::error::ErrorStack::get()); + } + let len = core::cmp::min(pw.len(), buf.len()); + buf[..len].copy_from_slice(&pw[..len]); + Ok(len) + } + Err(e) => { + *py_error.borrow_mut() = Some(e); + Err(openssl::error::ErrorStack::get()) + } + } + } + Err(e) => { + *py_error.borrow_mut() = Some(e); + Err(openssl::error::ErrorStack::get()) + } + } + }); + + // Check for Python error first + if let Some(py_err) = py_error.into_inner() { + return Err(py_err); + } + + result.map_err(|e| convert_openssl_error(vm, e))? + } else { + // Direct password (string/bytes) + let pw = Self::extract_password_bytes( + pw_obj, + "password should be a string or bytes", + vm, + )?; + + // Check password length + if pw.len() > PEM_BUFSIZE { + return Err(vm.new_value_error(format!( + "password cannot be longer than {} bytes", + PEM_BUFSIZE + ))); + } + + PKey::private_key_from_pem_passphrase(&key_data, &pw) + .map_err(|e| convert_openssl_error(vm, e))? + } + } else { + // No password - use SSL_CTX_use_PrivateKey_file directly for correct error messages + ctx.set_private_key_file(key_file_path, ssl::SslFiletype::PEM) + .map_err(|e| convert_openssl_error(vm, e))?; + + // Verify key matches certificate and return early + return ctx + .check_private_key() + .map_err(|e| convert_openssl_error(vm, e)); + }; + + ctx.set_private_key(&pkey) + .map_err(|e| convert_openssl_error(vm, e))?; + + // Verify key matches certificate + ctx.check_private_key() + .map_err(|e| convert_openssl_error(vm, e)) + } + + // Helper to extract password bytes from string/bytes/bytearray + fn extract_password_bytes( + obj: &PyObject, + bad_type_error: &str, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + use crate::vm::builtins::{PyByteArray, PyBytes, PyStr}; + + if let Some(s) = obj.downcast_ref::<PyStr>() { + Ok(s.as_str().as_bytes().to_vec()) + } else if let Some(b) = obj.downcast_ref::<PyBytes>() { + Ok(b.as_bytes().to_vec()) + } else if let Some(ba) = obj.downcast_ref::<PyByteArray>() { + Ok(ba.borrow_buf().to_vec()) + } else { + Err(vm.new_type_error(bad_type_error.to_owned())) + } + } + + // Helper function to create SSL socket + // = CPython's newPySSLSocket() + fn new_py_ssl_socket( + ctx_ref: PyRef<PySslContext>, + server_side: bool, + server_hostname: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<(ssl::Ssl, SslServerOrClient, Option<PyStrRef>)> { + // Validate socket type and context protocol + if server_side && ctx_ref.protocol == SslVersion::TlsClient { + return Err(new_ssl_error( + vm, + "Cannot create a server socket with a PROTOCOL_TLS_CLIENT context", + )); + } + if !server_side && ctx_ref.protocol == SslVersion::TlsServer { + return Err(new_ssl_error( + vm, + "Cannot create a client socket with a PROTOCOL_TLS_SERVER context", + )); + } + + // Create SSL object + let mut ssl = + ssl::Ssl::new(&ctx_ref.ctx()).map_err(|e| convert_openssl_error(vm, e))?; + + // Set session id context for server-side sockets + let socket_type = if server_side { + unsafe { + const SID_CTX: &[u8] = b"Python"; + let ret = SSL_set_session_id_context( + ssl.as_ptr(), + SID_CTX.as_ptr(), + SID_CTX.len() as libc::c_uint, + ); + if ret == 0 { + return Err(convert_openssl_error(vm, ErrorStack::get())); + } + } + SslServerOrClient::Server + } else { + SslServerOrClient::Client + }; + + // Configure server hostname + if let Some(hostname) = &server_hostname { + let hostname_str = hostname.as_str(); + if hostname_str.is_empty() || hostname_str.starts_with('.') { + return Err(vm.new_value_error( + "server_hostname cannot be an empty string or start with a leading dot.", + )); + } + if hostname_str.contains('\0') { + return Err(vm.new_type_error("embedded null character")); + } + let ip = hostname_str.parse::<std::net::IpAddr>(); + if ip.is_err() { + ssl.set_hostname(hostname_str) + .map_err(|e| convert_openssl_error(vm, e))?; + } + if ctx_ref.check_hostname.load() { + if let Ok(ip) = ip { + ssl.param_mut() + .set_ip(ip) + .map_err(|e| convert_openssl_error(vm, e))?; + } else { + ssl.param_mut() + .set_host(hostname_str) + .map_err(|e| convert_openssl_error(vm, e))?; + } + } + } + + // Configure post-handshake authentication + #[cfg(ossl111)] + if *ctx_ref.post_handshake_auth.lock() { + unsafe { + if server_side { + // Server socket: add SSL_VERIFY_POST_HANDSHAKE flag + // Only in combination with SSL_VERIFY_PEER + let mode = sys::SSL_get_verify_mode(ssl.as_ptr()); + if (mode & sys::SSL_VERIFY_PEER as libc::c_int) != 0 { + sys::SSL_set_verify( + ssl.as_ptr(), + mode | SSL_VERIFY_POST_HANDSHAKE, + None, + ); + } + } else { + // Client socket: call SSL_set_post_handshake_auth + SSL_set_post_handshake_auth(ssl.as_ptr(), 1); + } + } + } + + // Set connect/accept state + if server_side { + ssl.set_accept_state(); + } else { + ssl.set_connect_state(); + } + + Ok((ssl, socket_type, server_hostname)) + } + + #[pymethod] + fn _wrap_socket( + zelf: PyRef<Self>, + args: WrapSocketArgs, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + // Use common helper function + let (ssl, socket_type, server_hostname) = + Self::new_py_ssl_socket(zelf.clone(), args.server_side, args.server_hostname, vm)?; + + // Create SslStream with socket + let stream = ssl::SslStream::new(ssl, SocketStream(args.sock.clone())) + .map_err(|e| convert_openssl_error(vm, e))?; + + let py_ssl_socket = PySslSocket { + ctx: PyRwLock::new(zelf.clone()), + connection: PyRwLock::new(SslConnection::Socket(stream)), + socket_type, + server_hostname, + owner: PyRwLock::new(args.owner.map(|o| o.downgrade(None, vm)).transpose()?), + }; + + // Convert to PyRef (heap allocation) to avoid use-after-free + let py_ref = + py_ssl_socket.into_ref_with_type(vm, PySslSocket::class(&vm.ctx).to_owned())?; + + // Check if SNI callback is configured (minimize lock time) + let has_sni_callback = zelf.sni_callback.lock().is_some(); + + // Set up ex_data for callbacks + unsafe { + let ssl_ptr = py_ref.connection.read().ssl().as_ptr(); + + // Clone and store via into_raw() - increments refcount and returns stable pointer + // The refcount will be decremented by msg_callback_data_free when SSL is freed + let cloned: PyObjectRef = py_ref.clone().into(); + let raw_ptr = cloned.into_raw(); + let msg_cb_idx = get_msg_callback_ex_data_index(); + sys::SSL_set_ex_data(ssl_ptr, msg_cb_idx, raw_ptr.as_ptr() as *mut _); + + // Set SNI callback data if needed + if has_sni_callback { + let ssl_socket_weak = py_ref.as_object().downgrade(None, vm)?; + // Store callback data in SSL ex_data - use weak reference to avoid cycle + let callback_data = Box::new(SniCallbackData { + ssl_context: zelf.clone(), + ssl_socket_weak, + }); + let sni_idx = get_sni_ex_data_index(); + sys::SSL_set_ex_data(ssl_ptr, sni_idx, Box::into_raw(callback_data) as *mut _); + } + } + + // Set session if provided + if let Some(session) = args.session + && !vm.is_none(&session) + { + py_ref.set_session(session, vm)?; + } + + Ok(py_ref.into()) + } + + #[pymethod] + fn _wrap_bio( + zelf: PyRef<Self>, + args: WrapBioArgs, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + // Use common helper function + let (ssl, socket_type, server_hostname) = + Self::new_py_ssl_socket(zelf.clone(), args.server_side, args.server_hostname, vm)?; + + // Create BioStream wrapper + let bio_stream = BioStream { + inbio: args.incoming, + outbio: args.outgoing, + }; + + // Create SslStream with BioStream + let stream = + ssl::SslStream::new(ssl, bio_stream).map_err(|e| convert_openssl_error(vm, e))?; + + let py_ssl_socket = PySslSocket { + ctx: PyRwLock::new(zelf.clone()), + connection: PyRwLock::new(SslConnection::Bio(stream)), + socket_type, + server_hostname, + owner: PyRwLock::new(args.owner.map(|o| o.downgrade(None, vm)).transpose()?), + }; + + // Convert to PyRef (heap allocation) to avoid use-after-free + let py_ref = + py_ssl_socket.into_ref_with_type(vm, PySslSocket::class(&vm.ctx).to_owned())?; + + // Check if SNI callback is configured (minimize lock time) + let has_sni_callback = zelf.sni_callback.lock().is_some(); + + // Set up ex_data for callbacks + unsafe { + let ssl_ptr = py_ref.connection.read().ssl().as_ptr(); + + // Clone and store via into_raw() - increments refcount and returns stable pointer + // The refcount will be decremented by msg_callback_data_free when SSL is freed + let cloned: PyObjectRef = py_ref.clone().into(); + let raw_ptr = cloned.into_raw(); + let msg_cb_idx = get_msg_callback_ex_data_index(); + sys::SSL_set_ex_data(ssl_ptr, msg_cb_idx, raw_ptr.as_ptr() as *mut _); + + // Set SNI callback data if needed + if has_sni_callback { + let ssl_socket_weak = py_ref.as_object().downgrade(None, vm)?; + // Store callback data in SSL ex_data - use weak reference to avoid cycle + let callback_data = Box::new(SniCallbackData { + ssl_context: zelf.clone(), + ssl_socket_weak, + }); + let sni_idx = get_sni_ex_data_index(); + sys::SSL_set_ex_data(ssl_ptr, sni_idx, Box::into_raw(callback_data) as *mut _); + } + } + + // Set session if provided + if let Some(session) = args.session + && !vm.is_none(&session) + { + py_ref.set_session(session, vm)?; + } + + Ok(py_ref.into()) + } + } + + #[derive(FromArgs)] + #[allow(dead_code)] // Fields will be used when _wrap_bio is fully implemented + struct WrapBioArgs { + incoming: PyRef<PySslMemoryBio>, + outgoing: PyRef<PySslMemoryBio>, + server_side: bool, + #[pyarg(any, default)] + server_hostname: Option<PyStrRef>, + #[pyarg(named, default)] + owner: Option<PyObjectRef>, + #[pyarg(named, default)] + session: Option<PyObjectRef>, + } + + #[derive(FromArgs)] + struct WrapSocketArgs { + sock: PyRef<PySocket>, + server_side: bool, + #[pyarg(any, default)] + server_hostname: Option<PyStrRef>, + #[pyarg(named, default)] + owner: Option<PyObjectRef>, + #[pyarg(named, default)] + session: Option<PyObjectRef>, + } + + #[derive(FromArgs)] + struct LoadVerifyLocationsArgs { + #[pyarg(any, default)] + cafile: Option<FsPath>, + #[pyarg(any, default)] + capath: Option<FsPath>, + #[pyarg(any, default)] + cadata: Option<Either<PyStrRef, ArgBytesLike>>, + } + + #[derive(FromArgs)] + struct LoadCertChainArgs { + certfile: FsPath, + #[pyarg(any, optional)] + keyfile: Option<FsPath>, + #[pyarg(any, optional)] + password: Option<PyObjectRef>, + } + + #[derive(FromArgs)] + struct GetCertArgs { + #[pyarg(any, optional)] + binary_form: OptionalArg<bool>, + } + + // Err is true if the socket is blocking + type SocketDeadline = Result<Instant, bool>; + + enum SelectRet { + Nonblocking, + TimedOut, + Closed, + Ok, + } + + #[derive(Clone, Copy)] + enum SslNeeds { + Read, + Write, + } + + struct SocketStream(PyRef<PySocket>); + + impl SocketStream { + fn timeout_deadline(&self) -> SocketDeadline { + self.0.get_timeout().map(|d| Instant::now() + d) + } + + fn select(&self, needs: SslNeeds, deadline: &SocketDeadline) -> SelectRet { + let sock = match self.0.sock_opt() { + Some(s) => s, + None => return SelectRet::Closed, + }; + // For blocking sockets without timeout, call sock_select with None timeout + // to actually block waiting for data instead of busy-looping + let timeout = match &deadline { + Ok(deadline) => match deadline.checked_duration_since(Instant::now()) { + Some(d) => Some(d), + None => return SelectRet::TimedOut, + }, + Err(true) => None, // Blocking: no timeout, wait indefinitely + Err(false) => return SelectRet::Nonblocking, + }; + let res = socket::sock_select( + &sock, + match needs { + SslNeeds::Read => socket::SelectKind::Read, + SslNeeds::Write => socket::SelectKind::Write, + }, + timeout, + ); + match res { + Ok(true) => SelectRet::TimedOut, + _ => SelectRet::Ok, + } + } + + fn socket_needs( + &self, + err: &ssl::Error, + deadline: &SocketDeadline, + ) -> (Option<SslNeeds>, SelectRet) { + let needs = match err.code() { + ssl::ErrorCode::WANT_READ => Some(SslNeeds::Read), + ssl::ErrorCode::WANT_WRITE => Some(SslNeeds::Write), + _ => None, + }; + let state = needs.map_or(SelectRet::Ok, |needs| self.select(needs, deadline)); + (needs, state) + } + } + + fn socket_closed_error(vm: &VirtualMachine) -> PyBaseExceptionRef { + new_ssl_error(vm, "Underlying socket has been closed.") + } + + // BIO stream wrapper to implement Read/Write traits for MemoryBIO + struct BioStream { + inbio: PyRef<PySslMemoryBio>, + outbio: PyRef<PySslMemoryBio>, + } + + impl Read for BioStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { + // Read from incoming MemoryBIO + unsafe { + let nbytes = sys::BIO_read( + self.inbio.bio, + buf.as_mut_ptr() as *mut _, + buf.len().min(i32::MAX as usize) as i32, + ); + if nbytes < 0 { + // BIO_read returns -1 on error or when no data is available + // Check if it's a retry condition (WANT_READ) + Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "BIO has no data available", + )) + } else { + Ok(nbytes as usize) + } + } + } + } + + impl Write for BioStream { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + // Write to outgoing MemoryBIO + unsafe { + let nbytes = sys::BIO_write( + self.outbio.bio, + buf.as_ptr() as *const _, + buf.len().min(i32::MAX as usize) as i32, + ); + if nbytes < 0 { + return Err(std::io::Error::other("BIO write failed")); + } + Ok(nbytes as usize) + } + } + + fn flush(&mut self) -> std::io::Result<()> { + // MemoryBIO doesn't need flushing + Ok(()) + } + } + + // Enum to represent different SSL connection modes + enum SslConnection { + Socket(ssl::SslStream<SocketStream>), + Bio(ssl::SslStream<BioStream>), + } + + impl SslConnection { + // Get a reference to the SSL object + fn ssl(&self) -> &ssl::SslRef { + match self { + SslConnection::Socket(stream) => stream.ssl(), + SslConnection::Bio(stream) => stream.ssl(), + } + } + + // Get underlying socket stream reference (only for socket mode) + fn get_ref(&self) -> Option<&SocketStream> { + match self { + SslConnection::Socket(stream) => Some(stream.get_ref()), + SslConnection::Bio(_) => None, + } + } + + // Check if this is in BIO mode + fn is_bio(&self) -> bool { + matches!(self, SslConnection::Bio(_)) + } + + // Perform SSL handshake + fn do_handshake(&mut self) -> Result<(), ssl::Error> { + match self { + SslConnection::Socket(stream) => stream.do_handshake(), + SslConnection::Bio(stream) => stream.do_handshake(), + } + } + + // Write data to SSL connection + fn ssl_write(&mut self, buf: &[u8]) -> Result<usize, ssl::Error> { + match self { + SslConnection::Socket(stream) => stream.ssl_write(buf), + SslConnection::Bio(stream) => stream.ssl_write(buf), + } + } + + // Read data from SSL connection + fn ssl_read(&mut self, buf: &mut [u8]) -> Result<usize, ssl::Error> { + match self { + SslConnection::Socket(stream) => stream.ssl_read(buf), + SslConnection::Bio(stream) => stream.ssl_read(buf), + } + } + + // Get SSL shutdown state + fn get_shutdown(&mut self) -> ssl::ShutdownState { + match self { + SslConnection::Socket(stream) => stream.get_shutdown(), + SslConnection::Bio(stream) => stream.get_shutdown(), + } + } + + // Check if incoming BIO has EOF (for BIO mode only) + fn is_bio_eof(&self) -> bool { + match self { + SslConnection::Socket(_) => false, + SslConnection::Bio(stream) => stream.get_ref().inbio.eof_written.load(), + } + } + } + + #[pyattr] + #[pyclass(module = "ssl", name = "_SSLSocket", traverse)] + #[derive(PyPayload)] + struct PySslSocket { + ctx: PyRwLock<PyRef<PySslContext>>, + #[pytraverse(skip)] + connection: PyRwLock<SslConnection>, + #[pytraverse(skip)] + socket_type: SslServerOrClient, + server_hostname: Option<PyStrRef>, + owner: PyRwLock<Option<PyRef<PyWeak>>>, + } + + impl fmt::Debug for PySslSocket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("_SSLSocket") + } + } + + #[pyclass(flags(IMMUTABLETYPE))] + impl PySslSocket { + #[pygetset] + fn owner(&self) -> Option<PyObjectRef> { + self.owner.read().as_ref().and_then(|weak| weak.upgrade()) + } + #[pygetset(setter)] + fn set_owner(&self, owner: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut lock = self.owner.write(); + lock.take(); + *lock = Some(owner.downgrade(None, vm)?); + Ok(()) + } + #[pygetset] + fn server_side(&self) -> bool { + self.socket_type == SslServerOrClient::Server + } + #[pygetset] + fn context(&self) -> PyRef<PySslContext> { + self.ctx.read().clone() + } + #[pygetset(setter)] + fn set_context(&self, value: PyRef<PySslContext>, vm: &VirtualMachine) -> PyResult<()> { + // Get SSL pointer - use thread-local during handshake to avoid deadlock + // (connection lock is already held during handshake) + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + + // Set the new SSL_CTX on the SSL object + unsafe { + let result = SSL_set_SSL_CTX(ssl_ptr, value.ctx().as_ptr()); + if result.is_null() { + return Err(vm.new_runtime_error("Failed to set SSL context")); + } + } + + // Update self.ctx to the new context + *self.ctx.write() = value; + Ok(()) + } + #[pygetset] + fn server_hostname(&self) -> Option<PyStrRef> { + self.server_hostname.clone() + } + + #[pymethod] + fn getpeercert( + &self, + args: GetCertArgs, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + let binary = args.binary_form.unwrap_or(false); + let stream = self.connection.read(); + if !stream.ssl().is_init_finished() { + return Err(vm.new_value_error("handshake not done yet")); + } + + let peer_cert = stream.ssl().peer_certificate(); + let Some(cert) = peer_cert else { + return Ok(None); + }; + + if binary { + // Return DER-encoded certificate + cert_to_py(vm, &cert, true).map(Some) + } else { + // Check verify_mode + unsafe { + let ssl_ctx = sys::SSL_get_SSL_CTX(stream.ssl().as_ptr()); + let verify_mode = sys::SSL_CTX_get_verify_mode(ssl_ctx); + if (verify_mode & sys::SSL_VERIFY_PEER as libc::c_int) == 0 { + // Return empty dict when SSL_VERIFY_PEER is not set + Ok(Some(vm.ctx.new_dict().into())) + } else { + // Return decoded certificate + cert_to_py(vm, &cert, false).map(Some) + } + } + } + } + + #[pymethod] + fn get_unverified_chain(&self, vm: &VirtualMachine) -> PyResult<Option<PyListRef>> { + let stream = self.connection.read(); + let ssl = stream.ssl(); + let Some(chain) = ssl.peer_cert_chain() else { + return Ok(None); + }; + + // Return Certificate objects + let mut certs: Vec<PyObjectRef> = chain + .iter() + .map(|cert| unsafe { + sys::X509_up_ref(cert.as_ptr()); + let owned = X509::from_ptr(cert.as_ptr()); + cert_to_certificate(vm, owned) + }) + .collect::<PyResult<_>>()?; + + // SSL_get_peer_cert_chain does not include peer cert for server-side sockets + // Add it manually at the beginning + if matches!(self.socket_type, SslServerOrClient::Server) + && let Some(peer_cert) = ssl.peer_certificate() + { + let peer_obj = cert_to_certificate(vm, peer_cert)?; + certs.insert(0, peer_obj); + } + + Ok(Some(vm.ctx.new_list(certs))) + } + + #[pymethod] + fn get_verified_chain(&self, vm: &VirtualMachine) -> PyResult<Option<PyListRef>> { + let stream = self.connection.read(); + unsafe { + let chain = sys::SSL_get0_verified_chain(stream.ssl().as_ptr()); + if chain.is_null() { + return Ok(None); + } + + let num_certs = sys::OPENSSL_sk_num(chain as *const _); + + let mut certs = Vec::with_capacity(num_certs as usize); + // Return Certificate objects + for i in 0..num_certs { + let cert_ptr = sys::OPENSSL_sk_value(chain as *const _, i) as *mut sys::X509; + if cert_ptr.is_null() { + continue; + } + // Clone the X509 certificate to create an owned copy + sys::X509_up_ref(cert_ptr); + let owned_cert = X509::from_ptr(cert_ptr); + let cert_obj = cert_to_certificate(vm, owned_cert)?; + certs.push(cert_obj); + } + + Ok(if certs.is_empty() { + None + } else { + Some(vm.ctx.new_list(certs)) + }) + } + } + + #[pymethod] + fn version(&self) -> Option<&'static str> { + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + // Return None if handshake is not complete (CPython behavior) + if unsafe { sys::SSL_is_init_finished(ssl_ptr) } == 0 { + return None; + } + let v = unsafe { ssl::SslRef::from_ptr(ssl_ptr).version_str() }; + if v == "unknown" { None } else { Some(v) } + } + + #[pymethod] + fn cipher(&self) -> Option<CipherTuple> { + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + unsafe { ssl::SslRef::from_ptr(ssl_ptr).current_cipher() }.map(cipher_to_tuple) + } + + #[pymethod] + fn pending(&self) -> i32 { + let stream = self.connection.read(); + unsafe { sys::SSL_pending(stream.ssl().as_ptr()) } + } + + #[pymethod] + fn shared_ciphers(&self, vm: &VirtualMachine) -> Option<PyListRef> { + #[cfg(ossl110)] + { + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + unsafe { + let server_ciphers = SSL_get_ciphers(ssl_ptr); + if server_ciphers.is_null() { + return None; + } + + let client_ciphers = SSL_get_client_ciphers(ssl_ptr); + if client_ciphers.is_null() { + return None; + } + + let mut result = Vec::new(); + let num_server = sys::OPENSSL_sk_num(server_ciphers as *const _); + let num_client = sys::OPENSSL_sk_num(client_ciphers as *const _); + + for i in 0..num_server { + let server_cipher_ptr = sys::OPENSSL_sk_value(server_ciphers as *const _, i) + as *const sys::SSL_CIPHER; + + // Check if client supports this cipher by comparing pointers + let mut found = false; + for j in 0..num_client { + let client_cipher_ptr = + sys::OPENSSL_sk_value(client_ciphers as *const _, j) + as *const sys::SSL_CIPHER; + + if server_cipher_ptr == client_cipher_ptr { + found = true; + break; + } + } + + if found { + let cipher = ssl::SslCipherRef::from_ptr(server_cipher_ptr as *mut _); + let (name, version, bits) = cipher_to_tuple(cipher); + let tuple = vm.new_tuple(( + vm.ctx.new_str(name), + vm.ctx.new_str(version), + vm.ctx.new_int(bits), + )); + result.push(tuple.into()); + } + } + + if result.is_empty() { + None + } else { + Some(vm.ctx.new_list(result)) + } + } + } + #[cfg(not(ossl110))] + { + let _ = vm; + None + } + } + + #[pymethod] + fn selected_alpn_protocol(&self) -> Option<String> { + #[cfg(ossl102)] + { + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + unsafe { + let mut out: *const libc::c_uchar = core::ptr::null(); + let mut outlen: libc::c_uint = 0; + + sys::SSL_get0_alpn_selected(ssl_ptr, &mut out, &mut outlen); + + if out.is_null() { + None + } else { + let slice = std::slice::from_raw_parts(out, outlen as usize); + Some(String::from_utf8_lossy(slice).into_owned()) + } + } + } + #[cfg(not(ossl102))] + { + None + } + } + + #[pymethod] + fn get_channel_binding( + &self, + cb_type: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<PyBytesRef>> { + const CB_MAXLEN: usize = 512; + + let cb_type_str = cb_type.as_ref().map_or("tls-unique", |s| s.as_str()); + + if cb_type_str != "tls-unique" { + return Err(vm.new_value_error(format!( + "Unsupported channel binding type '{}'", + cb_type_str + ))); + } + + let stream = self.connection.read(); + let ssl_ptr = stream.ssl().as_ptr(); + + unsafe { + let session_reused = sys::SSL_session_reused(ssl_ptr) != 0; + let is_client = matches!(self.socket_type, SslServerOrClient::Client); + + // Use XOR logic from CPython + let use_finished = session_reused ^ is_client; + + let mut buf = vec![0u8; CB_MAXLEN]; + let len = if use_finished { + sys::SSL_get_finished(ssl_ptr, buf.as_mut_ptr() as *mut _, CB_MAXLEN) + } else { + sys::SSL_get_peer_finished(ssl_ptr, buf.as_mut_ptr() as *mut _, CB_MAXLEN) + }; + + if len == 0 { + Ok(None) + } else { + buf.truncate(len); + Ok(Some(vm.ctx.new_bytes(buf))) + } + } + } + + #[pymethod] + fn verify_client_post_handshake(&self, vm: &VirtualMachine) -> PyResult<()> { + #[cfg(ossl111)] + { + let stream = self.connection.read(); + let result = unsafe { SSL_verify_client_post_handshake(stream.ssl().as_ptr()) }; + if result == 0 { + Err(convert_openssl_error(vm, openssl::error::ErrorStack::get())) + } else { + Ok(()) + } + } + #[cfg(not(ossl111))] + { + Err(vm.new_not_implemented_error( + "Post-handshake auth is not supported by your OpenSSL version.", + )) + } + } + + #[pymethod] + fn shutdown(&self, vm: &VirtualMachine) -> PyResult<Option<PyRef<PySocket>>> { + let stream = self.connection.read(); + let ssl_ptr = stream.ssl().as_ptr(); + + // BIO mode: just try shutdown once and raise SSLWantReadError if needed + if stream.is_bio() { + let ret = unsafe { sys::SSL_shutdown(ssl_ptr) }; + if ret < 0 { + let err = unsafe { sys::SSL_get_error(ssl_ptr, ret) }; + if err == sys::SSL_ERROR_WANT_READ { + return Err(create_ssl_want_read_error(vm).upcast()); + } else if err == sys::SSL_ERROR_WANT_WRITE { + return Err(create_ssl_want_write_error(vm).upcast()); + } else { + return Err(new_ssl_error( + vm, + format!("SSL shutdown failed: error code {}", err), + )); + } + } else if ret == 0 { + // Sent close-notify, waiting for peer's - raise SSLWantReadError + return Err(create_ssl_want_read_error(vm).upcast()); + } + return Ok(None); + } + + // Socket mode: loop with select to wait for peer's close-notify + let socket_stream = stream.get_ref().expect("get_ref() failed for socket mode"); + let deadline = socket_stream.timeout_deadline(); + + // Track how many times we've seen ret == 0 (max 2 tries) + let mut zeros = 0; + + loop { + let ret = unsafe { sys::SSL_shutdown(ssl_ptr) }; + + // ret > 0: complete shutdown + if ret > 0 { + break; + } + + // ret == 0: sent our close-notify, need to receive peer's + if ret == 0 { + zeros += 1; + if zeros > 1 { + // Already tried twice, break out (legacy behavior) + break; + } + // Wait briefly for peer's close_notify before retrying + match socket_stream.select(SslNeeds::Read, &deadline) { + SelectRet::TimedOut => { + return Err(socket::timeout_error_msg( + vm, + "The read operation timed out".to_string(), + ) + .upcast()); + } + SelectRet::Closed => { + return Err(socket_closed_error(vm)); + } + SelectRet::Nonblocking => { + // Non-blocking socket: return SSLWantReadError + return Err(create_ssl_want_read_error(vm).upcast()); + } + SelectRet::Ok => { + // Data available, continue to retry + } + } + continue; + } + + // ret < 0: error or would-block + let err = unsafe { sys::SSL_get_error(ssl_ptr, ret) }; + + let needs = if err == sys::SSL_ERROR_WANT_READ { + SslNeeds::Read + } else if err == sys::SSL_ERROR_WANT_WRITE { + SslNeeds::Write + } else { + // Real error + return Err(new_ssl_error( + vm, + format!("SSL shutdown failed: error code {}", err), + )); + }; + + // Wait on the socket + match socket_stream.select(needs, &deadline) { + SelectRet::TimedOut => { + let msg = if err == sys::SSL_ERROR_WANT_READ { + "The read operation timed out" + } else { + "The write operation timed out" + }; + return Err(socket::timeout_error_msg(vm, msg.to_string()).upcast()); + } + SelectRet::Closed => { + return Err(socket_closed_error(vm)); + } + SelectRet::Nonblocking => { + // Non-blocking socket, raise SSLWantReadError/SSLWantWriteError + if err == sys::SSL_ERROR_WANT_READ { + return Err(create_ssl_want_read_error(vm).upcast()); + } else { + return Err(create_ssl_want_write_error(vm).upcast()); + } + } + SelectRet::Ok => { + // Socket is ready, retry shutdown + continue; + } + } + } + + // Return the underlying socket + Ok(Some(socket_stream.0.clone())) + } + + #[cfg(osslconf = "OPENSSL_NO_COMP")] + #[pymethod] + fn compression(&self) -> Option<&'static str> { + None + } + #[cfg(not(osslconf = "OPENSSL_NO_COMP"))] + #[pymethod] + fn compression(&self) -> Option<&'static str> { + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + let comp_method = unsafe { sys::SSL_get_current_compression(ssl_ptr) }; + if comp_method.is_null() { + return None; + } + let typ = unsafe { sys::COMP_get_type(comp_method) }; + let nid = Nid::from_raw(typ); + if nid == Nid::UNDEF { + return None; + } + nid.short_name().ok() + } + + #[pymethod] + fn do_handshake(&self, vm: &VirtualMachine) -> PyResult<()> { + let mut stream = self.connection.write(); + let ssl_ptr = stream.ssl().as_ptr(); + + // Set up thread-local VM and SSL pointer for callbacks + // This allows callbacks to access SSL without acquiring the connection lock + let _vm_guard = HandshakeVmGuard::new(vm, ssl_ptr); + + // BIO mode: no timeout/select logic, just do handshake + if stream.is_bio() { + let result = stream.do_handshake().map_err(|e| { + let exc = convert_ssl_error(vm, e); + // If it's a cert verification error, set verify info + if exc.class().is(PySSLCertVerificationError::class(&vm.ctx)) { + set_verify_error_info(&exc, ssl_ptr, vm); + } + exc + }); + // Clean up SNI ex_data after handshake (success or failure) + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; + return result; + } + + // Socket mode: handle timeout and blocking + let timeout = stream + .get_ref() + .expect("handshake called in bio mode; should only be called in socket mode") + .timeout_deadline(); + loop { + let err = match stream.do_handshake() { + Ok(()) => { + // Clean up SNI ex_data after successful handshake + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; + return Ok(()); + } + Err(e) => e, + }; + let (needs, state) = stream + .get_ref() + .expect("handshake called in bio mode; should only be called in socket mode") + .socket_needs(&err, &timeout); + match state { + SelectRet::TimedOut => { + // Clean up SNI ex_data before returning error + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; + return Err(socket::timeout_error_msg( + vm, + "The handshake operation timed out".to_owned(), + ) + .upcast()); + } + SelectRet::Closed => { + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; + return Err(socket_closed_error(vm)); + } + SelectRet::Nonblocking => {} + SelectRet::Ok => { + // For blocking sockets, select() has completed successfully + // Continue the handshake loop (matches CPython's SOCKET_IS_BLOCKING behavior) + if needs.is_some() { + continue; + } + } + } + let exc = convert_ssl_error(vm, err); + // If it's a cert verification error, set verify info + if exc.class().is(PySSLCertVerificationError::class(&vm.ctx)) { + set_verify_error_info(&exc, ssl_ptr, vm); + } + // Clean up SNI ex_data before returning error + // SAFETY: ssl_ptr is valid for the lifetime of stream + unsafe { cleanup_sni_ex_data(ssl_ptr) }; + return Err(exc); + } + } + + #[pymethod] + fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { + let mut stream = self.connection.write(); + let data = data.borrow_buf(); + let data = &*data; + + // BIO mode: no timeout/select logic + if stream.is_bio() { + return stream.ssl_write(data).map_err(|e| convert_ssl_error(vm, e)); + } + + // Socket mode: handle timeout and blocking + let socket_ref = stream + .get_ref() + .expect("write called in bio mode; should only be called in socket mode"); + let timeout = socket_ref.timeout_deadline(); + let state = socket_ref.select(SslNeeds::Write, &timeout); + match state { + SelectRet::TimedOut => { + return Err(socket::timeout_error_msg( + vm, + "The write operation timed out".to_owned(), + ) + .upcast()); + } + SelectRet::Closed => return Err(socket_closed_error(vm)), + _ => {} + } + loop { + let err = match stream.ssl_write(data) { + Ok(len) => return Ok(len), + Err(e) => e, + }; + let (needs, state) = stream + .get_ref() + .expect("write called in bio mode; should only be called in socket mode") + .socket_needs(&err, &timeout); + match state { + SelectRet::TimedOut => { + return Err(socket::timeout_error_msg( + vm, + "The write operation timed out".to_owned(), + ) + .upcast()); + } + SelectRet::Closed => return Err(socket_closed_error(vm)), + SelectRet::Nonblocking => {} + SelectRet::Ok => { + // For blocking sockets, select() has completed successfully + // Continue the write loop (matches CPython's SOCKET_IS_BLOCKING behavior) + if needs.is_some() { + continue; + } + } + } + return Err(convert_ssl_error(vm, err)); + } + } + + #[pygetset] + fn session(&self, _vm: &VirtualMachine) -> PyResult<Option<PySslSession>> { + let stream = self.connection.read(); + unsafe { + // Use SSL_get1_session which returns an owned reference (ref count already incremented) + let session_ptr = SSL_get1_session(stream.ssl().as_ptr()); + if session_ptr.is_null() { + Ok(None) + } else { + Ok(Some(PySslSession { + session: session_ptr, + ctx: self.ctx.read().clone(), + })) + } + } + } + + #[pygetset(setter)] + fn set_session(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Check if value is SSLSession type + let session = value + .downcast_ref::<PySslSession>() + .ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?; + + // Check if session refers to the same SSLContext + if !std::ptr::eq( + self.ctx.read().ctx.read().as_ptr(), + session.ctx.ctx.read().as_ptr(), + ) { + return Err(vm.new_value_error("Session refers to a different SSLContext.")); + } + + // Check if this is a client socket + if self.socket_type != SslServerOrClient::Client { + return Err(vm.new_value_error("Cannot set session for server-side SSLSocket.")); + } + + // Check if handshake is not finished + let stream = self.connection.read(); + unsafe { + if sys::SSL_is_init_finished(stream.ssl().as_ptr()) != 0 { + return Err(vm.new_value_error("Cannot set session after handshake.")); + } + + let ret = sys::SSL_set_session(stream.ssl().as_ptr(), session.session); + if ret == 0 { + return Err(convert_openssl_error(vm, ErrorStack::get())); + } + } + + Ok(()) + } + + #[pygetset] + fn session_reused(&self) -> bool { + // Use thread-local SSL pointer during handshake to avoid deadlock + let ssl_ptr = get_ssl_ptr_for_context_change(&self.connection); + unsafe { sys::SSL_session_reused(ssl_ptr) != 0 } + } + + #[pymethod] + fn read( + &self, + n: isize, + buffer: OptionalArg<ArgMemoryBuffer>, + vm: &VirtualMachine, + ) -> PyResult { + // Handle negative n: + // - If buffer is None and n < 0: raise ValueError + // - If buffer is present and n <= 0: use buffer length + // This matches _ssl__SSLSocket_read_impl in CPython + let read_len: usize = match &buffer { + OptionalArg::Present(buf) => { + let buf_len = buf.borrow_buf_mut().len(); + if n <= 0 || (n as usize) > buf_len { + buf_len + } else { + n as usize + } + } + OptionalArg::Missing => { + if n < 0 { + return Err(vm.new_value_error("size should not be negative")); + } + n as usize + } + }; + + // Special case: reading 0 bytes should return empty bytes immediately + if read_len == 0 { + return if buffer.is_present() { + Ok(vm.ctx.new_int(0).into()) + } else { + Ok(vm.ctx.new_bytes(vec![]).into()) + }; + } + + let mut stream = self.connection.write(); + let mut inner_buffer = if let OptionalArg::Present(buffer) = &buffer { + Either::A(buffer.borrow_buf_mut()) + } else { + Either::B(vec![0u8; read_len]) + }; + let buf = match &mut inner_buffer { + Either::A(b) => &mut **b, + Either::B(b) => b.as_mut_slice(), + }; + let buf = match buf.get_mut(..read_len) { + Some(b) => b, + None => buf, + }; + + // BIO mode: no timeout/select logic + let count = if stream.is_bio() { + match stream.ssl_read(buf) { + Ok(count) => count, + Err(e) => { + // Handle ZERO_RETURN (EOF) - raise SSLEOFError + if e.code() == ssl::ErrorCode::ZERO_RETURN { + return Err(create_ssl_eof_error(vm).upcast()); + } + // If WANT_READ and the incoming BIO has EOF written, + // this is an unexpected EOF (transport closed without TLS close_notify) + if e.code() == ssl::ErrorCode::WANT_READ && stream.is_bio_eof() { + return Err(create_ssl_eof_error(vm).upcast()); + } + return Err(convert_ssl_error(vm, e)); + } + } + } else { + // Socket mode: handle timeout and blocking + let timeout = stream + .get_ref() + .expect("read called in bio mode; should only be called in socket mode") + .timeout_deadline(); + loop { + let err = match stream.ssl_read(buf) { + Ok(count) => break count, + Err(e) => e, + }; + if err.code() == ssl::ErrorCode::ZERO_RETURN + && stream.get_shutdown() == ssl::ShutdownState::RECEIVED + { + break 0; + } + let (needs, state) = stream + .get_ref() + .expect("read called in bio mode; should only be called in socket mode") + .socket_needs(&err, &timeout); + match state { + SelectRet::TimedOut => { + return Err(socket::timeout_error_msg( + vm, + "The read operation timed out".to_owned(), + ) + .upcast()); + } + SelectRet::Closed => return Err(socket_closed_error(vm)), + SelectRet::Nonblocking => {} + SelectRet::Ok => { + // For blocking sockets, select() has completed successfully + // Continue the read loop (matches CPython's SOCKET_IS_BLOCKING behavior) + if needs.is_some() { + continue; + } + } + } + return Err(convert_ssl_error(vm, err)); + } + }; + let ret = match inner_buffer { + Either::A(_buf) => vm.ctx.new_int(count).into(), + Either::B(mut buf) => { + buf.truncate(count); + buf.shrink_to_fit(); + vm.ctx.new_bytes(buf).into() + } + }; + Ok(ret) + } + } + + #[pyattr] + #[pyclass(module = "ssl", name = "SSLSession")] + #[derive(PyPayload)] + struct PySslSession { + session: *mut sys::SSL_SESSION, + ctx: PyRef<PySslContext>, + } + + impl fmt::Debug for PySslSession { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("SSLSession") + } + } + + impl Drop for PySslSession { + fn drop(&mut self) { + if !self.session.is_null() { + unsafe { + sys::SSL_SESSION_free(self.session); + } + } + } + } + + unsafe impl Send for PySslSession {} + unsafe impl Sync for PySslSession {} + + impl Comparable for PySslSession { + fn cmp( + zelf: &Py<Self>, + other: &crate::vm::PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let other = class_or_notimplemented!(Self, other); + + if !matches!(op, PyComparisonOp::Eq | PyComparisonOp::Ne) { + return Ok(PyComparisonValue::NotImplemented); + } + let mut eq = unsafe { + let mut self_len: libc::c_uint = 0; + let mut other_len: libc::c_uint = 0; + let self_id = sys::SSL_SESSION_get_id(zelf.session, &mut self_len); + let other_id = sys::SSL_SESSION_get_id(other.session, &mut other_len); + + if self_len != other_len { + false + } else { + let self_slice = std::slice::from_raw_parts(self_id, self_len as usize); + let other_slice = std::slice::from_raw_parts(other_id, other_len as usize); + self_slice == other_slice + } + }; + if matches!(op, PyComparisonOp::Ne) { + eq = !eq; + } + Ok(PyComparisonValue::Implemented(eq)) + } + } + + #[pyattr] + #[pyclass(module = "ssl", name = "MemoryBIO")] + #[derive(PyPayload)] + struct PySslMemoryBio { + bio: *mut sys::BIO, + eof_written: AtomicCell<bool>, + } + + impl fmt::Debug for PySslMemoryBio { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("MemoryBIO") + } + } + + impl Drop for PySslMemoryBio { + fn drop(&mut self) { + if !self.bio.is_null() { + unsafe { + sys::BIO_free_all(self.bio); + } + } + } + } + + unsafe impl Send for PySslMemoryBio {} + unsafe impl Sync for PySslMemoryBio {} + + // OpenSSL functions not in openssl-sys + + unsafe extern "C" { + // X509_check_ca returns 1 for CA certificates, 0 otherwise + fn X509_check_ca(x: *const sys::X509) -> libc::c_int; + } + + unsafe extern "C" { + fn SSL_get_ciphers(ssl: *const sys::SSL) -> *const sys::stack_st_SSL_CIPHER; + } + + #[cfg(ossl110)] + unsafe extern "C" { + fn SSL_get_client_ciphers(ssl: *const sys::SSL) -> *const sys::stack_st_SSL_CIPHER; + } + + #[cfg(ossl111)] + unsafe extern "C" { + fn SSL_verify_client_post_handshake(ssl: *const sys::SSL) -> libc::c_int; + fn SSL_set_post_handshake_auth(ssl: *mut sys::SSL, val: libc::c_int); + } + + #[cfg(ossl110)] + unsafe extern "C" { + fn SSL_CTX_get_security_level(ctx: *const sys::SSL_CTX) -> libc::c_int; + } + + unsafe extern "C" { + fn SSL_set_SSL_CTX(ssl: *mut sys::SSL, ctx: *mut sys::SSL_CTX) -> *mut sys::SSL_CTX; + } + + // Message callback type + #[allow(non_camel_case_types)] + type SSL_CTX_msg_callback = Option< + unsafe extern "C" fn( + write_p: libc::c_int, + version: libc::c_int, + content_type: libc::c_int, + buf: *const libc::c_void, + len: usize, + ssl: *mut sys::SSL, + arg: *mut libc::c_void, + ), + >; + + unsafe extern "C" { + fn SSL_CTX_set_msg_callback(ctx: *mut sys::SSL_CTX, cb: SSL_CTX_msg_callback); + } + + #[cfg(ossl110)] + unsafe extern "C" { + fn SSL_SESSION_has_ticket(session: *const sys::SSL_SESSION) -> libc::c_int; + fn SSL_SESSION_get_ticket_lifetime_hint(session: *const sys::SSL_SESSION) -> libc::c_ulong; + } + + // X509 object types + const X509_LU_X509: libc::c_int = 1; + const X509_LU_CRL: libc::c_int = 2; + + unsafe extern "C" { + fn X509_OBJECT_get_type(obj: *const sys::X509_OBJECT) -> libc::c_int; + fn SSL_set_session_id_context( + ssl: *mut sys::SSL, + sid_ctx: *const libc::c_uchar, + sid_ctx_len: libc::c_uint, + ) -> libc::c_int; + fn SSL_get1_session(ssl: *const sys::SSL) -> *mut sys::SSL_SESSION; + } + + // SSL session statistics constants (used with SSL_CTX_ctrl) + const SSL_CTRL_SESS_NUMBER: libc::c_int = 20; + const SSL_CTRL_SESS_CONNECT: libc::c_int = 21; + const SSL_CTRL_SESS_CONNECT_GOOD: libc::c_int = 22; + const SSL_CTRL_SESS_CONNECT_RENEGOTIATE: libc::c_int = 23; + const SSL_CTRL_SESS_ACCEPT: libc::c_int = 24; + const SSL_CTRL_SESS_ACCEPT_GOOD: libc::c_int = 25; + const SSL_CTRL_SESS_ACCEPT_RENEGOTIATE: libc::c_int = 26; + const SSL_CTRL_SESS_HIT: libc::c_int = 27; + const SSL_CTRL_SESS_MISSES: libc::c_int = 29; + const SSL_CTRL_SESS_TIMEOUTS: libc::c_int = 30; + const SSL_CTRL_SESS_CACHE_FULL: libc::c_int = 31; + + // SSL session statistics functions (implemented as macros in OpenSSL) + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_number(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { sys::SSL_CTX_ctrl(ctx as *mut _, SSL_CTRL_SESS_NUMBER, 0, std::ptr::null_mut()) } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_connect(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { + sys::SSL_CTX_ctrl( + ctx as *mut _, + SSL_CTRL_SESS_CONNECT, + 0, + std::ptr::null_mut(), + ) + } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_connect_good(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { + sys::SSL_CTX_ctrl( + ctx as *mut _, + SSL_CTRL_SESS_CONNECT_GOOD, + 0, + std::ptr::null_mut(), + ) + } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_connect_renegotiate(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { + sys::SSL_CTX_ctrl( + ctx as *mut _, + SSL_CTRL_SESS_CONNECT_RENEGOTIATE, + 0, + std::ptr::null_mut(), + ) + } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_accept(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { sys::SSL_CTX_ctrl(ctx as *mut _, SSL_CTRL_SESS_ACCEPT, 0, std::ptr::null_mut()) } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_accept_good(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { + sys::SSL_CTX_ctrl( + ctx as *mut _, + SSL_CTRL_SESS_ACCEPT_GOOD, + 0, + std::ptr::null_mut(), + ) + } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_accept_renegotiate(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { + sys::SSL_CTX_ctrl( + ctx as *mut _, + SSL_CTRL_SESS_ACCEPT_RENEGOTIATE, + 0, + std::ptr::null_mut(), + ) + } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_hits(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { sys::SSL_CTX_ctrl(ctx as *mut _, SSL_CTRL_SESS_HIT, 0, std::ptr::null_mut()) } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_misses(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { sys::SSL_CTX_ctrl(ctx as *mut _, SSL_CTRL_SESS_MISSES, 0, std::ptr::null_mut()) } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_timeouts(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { + sys::SSL_CTX_ctrl( + ctx as *mut _, + SSL_CTRL_SESS_TIMEOUTS, + 0, + std::ptr::null_mut(), + ) + } + } + + #[allow(non_snake_case)] + unsafe fn SSL_CTX_sess_cache_full(ctx: *const sys::SSL_CTX) -> libc::c_long { + unsafe { + sys::SSL_CTX_ctrl( + ctx as *mut _, + SSL_CTRL_SESS_CACHE_FULL, + 0, + std::ptr::null_mut(), + ) + } + } + + // DH parameters functions + unsafe extern "C" { + fn PEM_read_DHparams( + fp: *mut libc::FILE, + x: *mut *mut sys::DH, + cb: *mut libc::c_void, + u: *mut libc::c_void, + ) -> *mut sys::DH; + } + + // OpenSSL BIO helper functions + // These are typically macros in OpenSSL, implemented via BIO_ctrl + const BIO_CTRL_PENDING: libc::c_int = 10; + const BIO_CTRL_SET_EOF: libc::c_int = 2; + + #[allow(non_snake_case)] + unsafe fn BIO_ctrl_pending(bio: *mut sys::BIO) -> usize { + unsafe { sys::BIO_ctrl(bio, BIO_CTRL_PENDING, 0, std::ptr::null_mut()) as usize } + } + + #[allow(non_snake_case)] + unsafe fn BIO_set_mem_eof_return(bio: *mut sys::BIO, eof: libc::c_int) -> libc::c_int { + unsafe { + sys::BIO_ctrl( + bio, + BIO_CTRL_SET_EOF, + eof as libc::c_long, + std::ptr::null_mut(), + ) as libc::c_int + } + } + + #[allow(non_snake_case)] + unsafe fn BIO_clear_retry_flags(bio: *mut sys::BIO) { + unsafe { + sys::BIO_clear_flags(bio, sys::BIO_FLAGS_RWS | sys::BIO_FLAGS_SHOULD_RETRY); + } + } + + impl Constructor for PySslMemoryBio { + type Args = (); + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + unsafe { + let bio = sys::BIO_new(sys::BIO_s_mem()); + if bio.is_null() { + return Err(vm.new_memory_error("failed to allocate BIO")); + } + + sys::BIO_set_retry_read(bio); + BIO_set_mem_eof_return(bio, -1); + + Ok(PySslMemoryBio { + bio, + eof_written: AtomicCell::new(false), + }) + } + } + } + + #[pyclass(flags(IMMUTABLETYPE), with(Constructor))] + impl PySslMemoryBio { + #[pygetset] + fn pending(&self) -> usize { + unsafe { BIO_ctrl_pending(self.bio) } + } + + #[pygetset] + fn eof(&self) -> bool { + let pending = unsafe { BIO_ctrl_pending(self.bio) }; + pending == 0 && self.eof_written.load() + } + + #[pymethod] + fn read(&self, size: OptionalArg<i32>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + unsafe { + let avail = BIO_ctrl_pending(self.bio).min(i32::MAX as usize) as i32; + let len = size.unwrap_or(-1); + let len = if len < 0 || len > avail { avail } else { len }; + + // When no data available, return empty bytes (CPython behavior) + // CPython returns empty bytes directly without calling BIO_read() + if len == 0 { + return Ok(Vec::new()); + } + + let mut buf = vec![0u8; len as usize]; + let nbytes = sys::BIO_read(self.bio, buf.as_mut_ptr() as *mut _, len); + + if nbytes < 0 { + return Err(convert_openssl_error(vm, ErrorStack::get())); + } + + buf.truncate(nbytes as usize); + Ok(buf) + } + } + + #[pymethod] + fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<i32> { + if self.eof_written.load() { + return Err(new_ssl_error(vm, "cannot write() after write_eof()")); + } + + data.with_ref(|buf| unsafe { + if buf.len() > i32::MAX as usize { + return Err( + vm.new_overflow_error(format!("string longer than {} bytes", i32::MAX)) + ); + } + + let nbytes = sys::BIO_write(self.bio, buf.as_ptr() as *const _, buf.len() as i32); + if nbytes < 0 { + return Err(convert_openssl_error(vm, ErrorStack::get())); + } + + Ok(nbytes) + }) + } + + #[pymethod] + fn write_eof(&self) { + self.eof_written.store(true); + unsafe { + BIO_clear_retry_flags(self.bio); + BIO_set_mem_eof_return(self.bio, 0); + } + } + } + + #[pyclass(flags(IMMUTABLETYPE), with(Comparable))] + impl PySslSession { + #[pygetset] + fn time(&self) -> i64 { + unsafe { + #[cfg(ossl330)] + { + sys::SSL_SESSION_get_time(self.session) as i64 + } + #[cfg(not(ossl330))] + { + sys::SSL_SESSION_get_time(self.session) as i64 + } + } + } + + #[pygetset] + fn timeout(&self) -> i64 { + unsafe { sys::SSL_SESSION_get_timeout(self.session) as i64 } + } + + #[pygetset] + fn ticket_lifetime_hint(&self) -> u64 { + // SSL_SESSION_get_ticket_lifetime_hint available in OpenSSL 1.1.0+ + #[cfg(ossl110)] + { + unsafe { SSL_SESSION_get_ticket_lifetime_hint(self.session) as u64 } + } + #[cfg(not(ossl110))] + { + // Not available in older OpenSSL versions + 0 + } + } + + #[pygetset] + fn id(&self, vm: &VirtualMachine) -> PyBytesRef { + unsafe { + let mut len: libc::c_uint = 0; + let id_ptr = sys::SSL_SESSION_get_id(self.session, &mut len); + let id_slice = std::slice::from_raw_parts(id_ptr, len as usize); + vm.ctx.new_bytes(id_slice.to_vec()) + } + } + + #[pygetset] + fn has_ticket(&self) -> bool { + // SSL_SESSION_has_ticket available in OpenSSL 1.1.0+ + #[cfg(ossl110)] + { + unsafe { SSL_SESSION_has_ticket(self.session) != 0 } + } + #[cfg(not(ossl110))] + { + // Not available in older OpenSSL versions + false + } + } + } + + /// Helper function to create SSL error with proper OSError subtype handling + fn new_ssl_error(vm: &VirtualMachine, msg: impl ToString) -> PyBaseExceptionRef { + vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), None, msg.to_string()) + .upcast() + } + + #[track_caller] + pub(crate) fn convert_openssl_error( + vm: &VirtualMachine, + err: ErrorStack, + ) -> PyBaseExceptionRef { + match err.errors().last() { + Some(e) => { + // Check if this is a system library error (errno-based) + let lib = sys::ERR_GET_LIB(e.code()); + + if lib == sys::ERR_LIB_SYS { + // A system error is being reported; reason is set to errno + let reason = sys::ERR_GET_REASON(e.code()); + + // errno 2 = ENOENT = FileNotFoundError + let exc_type = if reason == 2 { + vm.ctx.exceptions.file_not_found_error.to_owned() + } else { + vm.ctx.exceptions.os_error.to_owned() + }; + return vm.new_os_subtype_error(exc_type, Some(reason), "").upcast(); + } + + let caller = std::panic::Location::caller(); + let (file, line) = (caller.file(), caller.line()); + let file = file + .rsplit_once(&['/', '\\'][..]) + .map_or(file, |(_, basename)| basename); + + // Get error codes - same approach as CPython + let lib = sys::ERR_GET_LIB(e.code()); + let reason = sys::ERR_GET_REASON(e.code()); + + // Look up error mnemonic from our static tables + // CPython uses dict lookup: err_codes_to_names[(lib, reason)] + let key = super::ssl_data::encode_error_key(lib, reason); + let errstr = super::ssl_data::ERROR_CODES + .get(&key) + .copied() + .or_else(|| { + // Fallback: use OpenSSL's error string + e.reason() + }) + .unwrap_or("unknown error"); + + // Check if this is a certificate verification error + // ERR_LIB_SSL = 20 (from _ssl_data_300.h) + // SSL_R_CERTIFICATE_VERIFY_FAILED = 134 (from _ssl_data_300.h) + let is_cert_verify_error = lib == 20 && reason == 134; + + // Look up library name from our static table + // CPython uses: lib_codes_to_names[lib] + let lib_name = super::ssl_data::LIBRARY_CODES.get(&(lib as u32)).copied(); + + // Use SSLCertVerificationError for certificate verification failures + let cls = if is_cert_verify_error { + PySSLCertVerificationError::class(&vm.ctx).to_owned() + } else { + PySSLError::class(&vm.ctx).to_owned() + }; + + // Build message + let msg = if let Some(lib_str) = lib_name { + format!("[{lib_str}] {errstr} ({file}:{line})") + } else { + format!("{errstr} ({file}:{line})") + }; + + // Create exception instance + let reason = sys::ERR_GET_REASON(e.code()); + let exc = vm.new_os_subtype_error(cls, Some(reason), msg); + let exc_obj: PyObjectRef = exc.upcast::<PyBaseException>().into(); + + // Set reason attribute (always set, even if just the error string) + let reason_value = vm.ctx.new_str(errstr); + let _ = exc_obj.set_attr("reason", reason_value, vm); + + // Set library attribute (None if not available) + let library_value: PyObjectRef = if let Some(lib_str) = lib_name { + vm.ctx.new_str(lib_str).into() + } else { + vm.ctx.none() + }; + let _ = exc_obj.set_attr("library", library_value, vm); + + // For SSLCertVerificationError, set verify_code and verify_message + // Note: These will be set to None here, and can be updated by the caller + // if they have access to the SSL object + if is_cert_verify_error { + let _ = exc_obj.set_attr("verify_code", vm.ctx.none(), vm); + let _ = exc_obj.set_attr("verify_message", vm.ctx.none(), vm); + } + + // Convert back to PyBaseExceptionRef + exc_obj.downcast().expect( + "exc_obj is created as PyBaseExceptionRef and must downcast successfully", + ) + } + None => { + let cls = PySSLError::class(&vm.ctx).to_owned(); + vm.new_os_subtype_error(cls, None, "unknown SSL error") + .upcast() + } + } + } + + // Helper function to set verify_code and verify_message on SSLCertVerificationError + fn set_verify_error_info( + exc: &Py<PyBaseException>, + ssl_ptr: *const sys::SSL, + vm: &VirtualMachine, + ) { + // Get verify result + let verify_code = unsafe { sys::SSL_get_verify_result(ssl_ptr) }; + let verify_code_obj = vm.ctx.new_int(verify_code); + + // Get verify message + let verify_message = unsafe { + let verify_str = sys::X509_verify_cert_error_string(verify_code); + if verify_str.is_null() { + vm.ctx.none() + } else { + let c_str = std::ffi::CStr::from_ptr(verify_str); + vm.ctx.new_str(c_str.to_string_lossy()).into() + } + }; + + let exc_obj = exc.as_object(); + let _ = exc_obj.set_attr("verify_code", verify_code_obj, vm); + let _ = exc_obj.set_attr("verify_message", verify_message, vm); + } + #[track_caller] + fn convert_ssl_error( + vm: &VirtualMachine, + e: impl std::borrow::Borrow<ssl::Error>, + ) -> PyBaseExceptionRef { + let e = e.borrow(); + let (cls, msg) = match e.code() { + ssl::ErrorCode::WANT_READ => { + return create_ssl_want_read_error(vm).upcast(); + } + ssl::ErrorCode::WANT_WRITE => { + return create_ssl_want_write_error(vm).upcast(); + } + ssl::ErrorCode::SYSCALL => match e.io_error() { + Some(io_err) => return io_err.to_pyexception(vm), + // When no I/O error and OpenSSL error queue is empty, + // this is an EOF in violation of protocol -> SSLEOFError + None => { + return create_ssl_eof_error(vm).upcast(); + } + }, + ssl::ErrorCode::SSL => { + // Check for OpenSSL 3.0 SSL_R_UNEXPECTED_EOF_WHILE_READING + if let Some(ssl_err) = e.ssl_error() { + // In OpenSSL 3.0+, unexpected EOF is reported as SSL_ERROR_SSL + // with this specific reason code instead of SSL_ERROR_SYSCALL + unsafe { + let err_code = sys::ERR_peek_last_error(); + let reason = sys::ERR_GET_REASON(err_code); + let lib = sys::ERR_GET_LIB(err_code); + if lib == ERR_LIB_SSL && reason == SSL_R_UNEXPECTED_EOF_WHILE_READING { + return create_ssl_eof_error(vm).upcast(); + } + } + return convert_openssl_error(vm, ssl_err.clone()); + } + ( + PySSLError::class(&vm.ctx).to_owned(), + "A failure in the SSL library occurred", + ) + } + _ => ( + PySSLError::class(&vm.ctx).to_owned(), + "A failure in the SSL library occurred", + ), + }; + vm.new_os_subtype_error(cls, None, msg).upcast() + } + + // SSL_FILETYPE_ASN1 part of _add_ca_certs in CPython + fn x509_stack_from_der(der: &[u8]) -> Result<Vec<X509>, ErrorStack> { + unsafe { + openssl::init(); + let bio = bio::MemBioSlice::new(der)?; + + let mut certs = vec![]; + let mut was_bio_eof = false; + + loop { + // Check for EOF before attempting to parse (like CPython's _add_ca_certs) + // BIO_ctrl with BIO_CTRL_EOF returns 1 if EOF, 0 otherwise + if sys::BIO_ctrl(bio.as_ptr(), sys::BIO_CTRL_EOF, 0, std::ptr::null_mut()) != 0 { + was_bio_eof = true; + break; + } + + let cert = sys::d2i_X509_bio(bio.as_ptr(), std::ptr::null_mut()); + if cert.is_null() { + // Parse error (not just EOF) + break; + } + certs.push(X509::from_ptr(cert)); + } + + // If we loaded some certs but didn't reach EOF, there's garbage data + // (like cacert_der + b"A") - this is an error + if !certs.is_empty() && !was_bio_eof { + // Return the error from the last failed parse attempt + return Err(ErrorStack::get()); + } + + // Clear any errors (including parse errors when no certs loaded) + // Let the caller decide how to handle empty results + sys::ERR_clear_error(); + Ok(certs) + } + } + + type CipherTuple = (&'static str, &'static str, i32); + + fn cipher_to_tuple(cipher: &ssl::SslCipherRef) -> CipherTuple { + (cipher.name(), cipher.version(), cipher.bits().secret) + } + + fn cipher_description(cipher: *const sys::SSL_CIPHER) -> String { + unsafe { + // SSL_CIPHER_description writes up to 128 bytes + let mut buf = vec![0u8; 256]; + let result = sys::SSL_CIPHER_description( + cipher, + buf.as_mut_ptr() as *mut libc::c_char, + buf.len() as i32, + ); + if result.is_null() { + return String::from("No description available"); + } + // Find the null terminator + let len = buf.iter().position(|&c| c == 0).unwrap_or(buf.len()); + String::from_utf8_lossy(&buf[..len]).trim().to_string() + } + } + + impl Read for SocketStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { + let mut socket: &PySocket = &self.0; + socket.read(buf) + } + } + + impl Write for SocketStream { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + let mut socket: &PySocket = &self.0; + socket.write(buf) + } + fn flush(&mut self) -> std::io::Result<()> { + let mut socket: &PySocket = &self.0; + socket.flush() + } + } + + #[cfg(target_os = "android")] + mod android { + use super::convert_openssl_error; + use crate::vm::{VirtualMachine, builtins::PyBaseExceptionRef}; + use openssl::{ + ssl::SslContextBuilder, + x509::{X509, store::X509StoreBuilder}, + }; + use std::{ + fs::{File, read_dir}, + io::Read, + path::Path, + }; + + static CERT_DIR: &'static str = "/system/etc/security/cacerts"; + + pub(super) fn load_client_ca_list( + vm: &VirtualMachine, + b: &mut SslContextBuilder, + ) -> Result<(), PyBaseExceptionRef> { + let root = Path::new(CERT_DIR); + if !root.is_dir() { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + None, + CERT_DIR.to_string(), + ) + .upcast()); + } + + let mut combined_pem = String::new(); + let entries = read_dir(root) + .map_err(|err| vm.new_os_error(format!("read cert root: {}", err)))?; + for entry in entries { + let entry = + entry.map_err(|err| vm.new_os_error(format!("iter cert root: {}", err)))?; + + let path = entry.path(); + if !path.is_file() { + continue; + } + + File::open(&path) + .and_then(|mut file| file.read_to_string(&mut combined_pem)) + .map_err(|err| { + vm.new_os_error(format!("open cert file {}: {}", path.display(), err)) + })?; + + combined_pem.push('\n'); + } + + let mut store_b = + X509StoreBuilder::new().map_err(|err| convert_openssl_error(vm, err))?; + let x509_vec = X509::stack_from_pem(combined_pem.as_bytes()) + .map_err(|err| convert_openssl_error(vm, err))?; + for x509 in x509_vec { + store_b + .add_cert(x509) + .map_err(|err| convert_openssl_error(vm, err))?; + } + b.set_cert_store(store_b.build()); + + Ok(()) + } + } +} + +#[allow(non_upper_case_globals)] +#[cfg(ossl101)] +#[pymodule(sub)] +mod ossl101 { + #[pyattr] + use openssl_sys::{ + SSL_OP_NO_COMPRESSION as OP_NO_COMPRESSION, SSL_OP_NO_TLSv1_1 as OP_NO_TLSv1_1, + SSL_OP_NO_TLSv1_2 as OP_NO_TLSv1_2, + }; +} + +#[allow(non_upper_case_globals)] +#[cfg(ossl111)] +#[pymodule(sub)] +mod ossl111 { + #[pyattr] + use openssl_sys::SSL_OP_NO_TLSv1_3 as OP_NO_TLSv1_3; +} + +#[cfg(windows)] +#[pymodule(sub)] +mod windows { + use crate::{ + common::ascii, + vm::{ + PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyFrozenSet, PyStrRef}, + convert::ToPyException, + }, + }; + + #[pyfunction] + fn enum_certificates(store_name: PyStrRef, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + use schannel::{RawPointer, cert_context::ValidUses, cert_store::CertStore}; + use windows_sys::Win32::Security::Cryptography; + + // TODO: check every store for it, not just 2 of them: + // https://github.com/python/cpython/blob/3.8/Modules/_ssl.c#L5603-L5610 + let open_fns = [CertStore::open_current_user, CertStore::open_local_machine]; + let stores = open_fns + .iter() + .filter_map(|open| open(store_name.as_str()).ok()) + .collect::<Vec<_>>(); + let certs = stores.iter().flat_map(|s| s.certs()).map(|c| { + let cert = vm.ctx.new_bytes(c.to_der().to_owned()); + let enc_type = unsafe { + let ptr = c.as_ptr() as *const Cryptography::CERT_CONTEXT; + (*ptr).dwCertEncodingType + }; + let enc_type = match enc_type { + Cryptography::X509_ASN_ENCODING => vm.new_pyobj(ascii!("x509_asn")), + Cryptography::PKCS_7_ASN_ENCODING => vm.new_pyobj(ascii!("pkcs_7_asn")), + other => vm.new_pyobj(other), + }; + let usage: PyObjectRef = match c.valid_uses().map_err(|e| e.to_pyexception(vm))? { + ValidUses::All => vm.ctx.new_bool(true).into(), + ValidUses::Oids(oids) => PyFrozenSet::from_iter( + vm, + oids.into_iter().map(|oid| vm.ctx.new_str(oid).into()), + )? + .into_ref(&vm.ctx) + .into(), + }; + Ok(vm.new_tuple((cert, enc_type, usage)).into()) + }); + let certs: Vec<PyObjectRef> = certs.collect::<PyResult<Vec<_>>>()?; + Ok(certs) + } +} + +mod bio { + //! based off rust-openssl's private `bio` module + + use libc::c_int; + use openssl::error::ErrorStack; + use openssl_sys as sys; + use std::marker::PhantomData; + + pub struct MemBioSlice<'a>(*mut sys::BIO, PhantomData<&'a [u8]>); + + impl Drop for MemBioSlice<'_> { + fn drop(&mut self) { + unsafe { + sys::BIO_free_all(self.0); + } + } + } + + impl<'a> MemBioSlice<'a> { + pub fn new(buf: &'a [u8]) -> Result<MemBioSlice<'a>, ErrorStack> { + openssl::init(); + + assert!(buf.len() <= c_int::MAX as usize); + let bio = unsafe { sys::BIO_new_mem_buf(buf.as_ptr() as *const _, buf.len() as c_int) }; + if bio.is_null() { + return Err(ErrorStack::get()); + } + + Ok(MemBioSlice(bio, PhantomData)) + } + + pub fn as_ptr(&self) -> *mut sys::BIO { + self.0 + } + } +} diff --git a/crates/stdlib/src/openssl/cert.rs b/crates/stdlib/src/openssl/cert.rs new file mode 100644 index 00000000000..b63d824a837 --- /dev/null +++ b/crates/stdlib/src/openssl/cert.rs @@ -0,0 +1,366 @@ +pub(super) use ssl_cert::{PySSLCertificate, cert_to_certificate, cert_to_py, obj2txt}; + +// Certificate type for SSL module + +#[pymodule(sub)] +pub(crate) mod ssl_cert { + use crate::{ + common::{ascii, hash::PyHash}, + vm::{ + Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + class_or_notimplemented, + convert::{ToPyException, ToPyObject}, + function::{FsPath, OptionalArg, PyComparisonValue}, + types::{Comparable, Hashable, PyComparisonOp, Representable}, + }, + }; + use foreign_types_shared::ForeignTypeRef; + use openssl::{ + asn1::Asn1ObjectRef, + nid::Nid, + x509::{self, X509, X509Ref}, + }; + use openssl_sys as sys; + use std::fmt; + + // Import constants and error converter from _ssl module + use crate::openssl::_ssl::{ENCODING_DER, ENCODING_PEM, convert_openssl_error}; + + pub(crate) fn obj2txt(obj: &Asn1ObjectRef, no_name: bool) -> Option<String> { + let no_name = i32::from(no_name); + let ptr = obj.as_ptr(); + let b = unsafe { + let buflen = sys::OBJ_obj2txt(std::ptr::null_mut(), 0, ptr, no_name); + assert!(buflen >= 0); + if buflen == 0 { + return None; + } + let buflen = buflen as usize; + let mut buf = Vec::<u8>::with_capacity(buflen + 1); + let ret = sys::OBJ_obj2txt( + buf.as_mut_ptr() as *mut libc::c_char, + buf.capacity() as _, + ptr, + no_name, + ); + assert!(ret >= 0); + // SAFETY: OBJ_obj2txt initialized the buffer successfully + buf.set_len(buflen); + buf + }; + let s = String::from_utf8(b) + .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()); + Some(s) + } + + #[pyattr] + #[pyclass(module = "ssl", name = "Certificate")] + #[derive(PyPayload)] + pub(crate) struct PySSLCertificate { + pub(crate) cert: X509, + } + + impl fmt::Debug for PySSLCertificate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Certificate") + } + } + + #[pyclass(with(Comparable, Hashable, Representable))] + impl PySSLCertificate { + #[pymethod] + fn public_bytes( + &self, + format: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let format = format.unwrap_or(ENCODING_PEM); + + match format { + ENCODING_DER => { + // DER encoding + let der = self + .cert + .to_der() + .map_err(|e| convert_openssl_error(vm, e))?; + Ok(vm.ctx.new_bytes(der).into()) + } + ENCODING_PEM => { + // PEM encoding - returns string + let pem = self + .cert + .to_pem() + .map_err(|e| convert_openssl_error(vm, e))?; + let pem_str = String::from_utf8(pem) + .map_err(|_| vm.new_value_error("Invalid UTF-8 in PEM"))?; + Ok(vm.ctx.new_str(pem_str).into()) + } + _ => Err(vm.new_value_error("Unsupported format")), + } + } + + #[pymethod] + fn get_info(&self, vm: &VirtualMachine) -> PyResult { + cert_to_dict(vm, &self.cert) + } + } + + impl Comparable for PySSLCertificate { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let other = class_or_notimplemented!(Self, other); + + // Only support equality comparison + if !matches!(op, PyComparisonOp::Eq | PyComparisonOp::Ne) { + return Ok(PyComparisonValue::NotImplemented); + } + + // Compare DER encodings + let self_der = zelf + .cert + .to_der() + .map_err(|e| convert_openssl_error(vm, e))?; + let other_der = other + .cert + .to_der() + .map_err(|e| convert_openssl_error(vm, e))?; + + let eq = self_der == other_der; + Ok(op.eval_ord(eq.cmp(&true)).into()) + } + } + + impl Hashable for PySSLCertificate { + fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyHash> { + // Use subject name hash as certificate hash + let hash = unsafe { sys::X509_subject_name_hash(zelf.cert.as_ptr()) }; + Ok(hash as PyHash) + } + } + + impl Representable for PySSLCertificate { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + // Build subject string like "CN=localhost, O=Python" + let subject = zelf.cert.subject_name(); + let mut parts: Vec<String> = Vec::new(); + for entry in subject.entries() { + // Use short name (SN) if available, otherwise use OID + let name = match entry.object().nid().short_name() { + Ok(sn) => sn.to_string(), + Err(_) => obj2txt(entry.object(), true).unwrap_or_default(), + }; + if let Ok(value) = entry.data().as_utf8() { + parts.push(format!("{}={}", name, value)); + } + } + if parts.is_empty() { + Ok("<Certificate>".to_string()) + } else { + Ok(format!("<Certificate '{}'>", parts.join(", "))) + } + } + } + + fn name_to_py(vm: &VirtualMachine, name: &x509::X509NameRef) -> PyResult { + let list = name + .entries() + .map(|entry| { + let txt = obj2txt(entry.object(), false).to_pyobject(vm); + let asn1_str = entry.data(); + let data_bytes = asn1_str.as_slice(); + let data = match std::str::from_utf8(data_bytes) { + Ok(s) => vm.ctx.new_str(s.to_owned()), + Err(_) => vm + .ctx + .new_str(String::from_utf8_lossy(data_bytes).into_owned()), + }; + Ok(vm.new_tuple(((txt, data),)).into()) + }) + .collect::<Result<_, _>>()?; + Ok(vm.ctx.new_tuple(list).into()) + } + + // Helper to convert X509 to dict (for getpeercert with binary=False) + fn cert_to_dict(vm: &VirtualMachine, cert: &X509Ref) -> PyResult { + let dict = vm.ctx.new_dict(); + + dict.set_item("subject", name_to_py(vm, cert.subject_name())?, vm)?; + dict.set_item("issuer", name_to_py(vm, cert.issuer_name())?, vm)?; + // X.509 version: OpenSSL uses 0-based (0=v1, 1=v2, 2=v3) but Python uses 1-based (1=v1, 2=v2, 3=v3) + dict.set_item("version", vm.new_pyobj(cert.version() + 1), vm)?; + + let serial_num = cert + .serial_number() + .to_bn() + .and_then(|bn| bn.to_hex_str()) + .map_err(|e| convert_openssl_error(vm, e))?; + // Serial number must have even length (each byte = 2 hex chars) + // BigNum::to_hex_str() strips leading zeros, so we need to pad + let serial_str = serial_num.to_string(); + let serial_str = if serial_str.len() % 2 == 1 { + format!("0{}", serial_str) + } else { + serial_str + }; + dict.set_item("serialNumber", vm.ctx.new_str(serial_str).into(), vm)?; + + dict.set_item( + "notBefore", + vm.ctx.new_str(cert.not_before().to_string()).into(), + vm, + )?; + dict.set_item( + "notAfter", + vm.ctx.new_str(cert.not_after().to_string()).into(), + vm, + )?; + + if let Some(names) = cert.subject_alt_names() { + let san: Vec<PyObjectRef> = names + .iter() + .map(|gen_name| { + if let Some(email) = gen_name.email() { + vm.new_tuple((ascii!("email"), email)).into() + } else if let Some(dnsname) = gen_name.dnsname() { + vm.new_tuple((ascii!("DNS"), dnsname)).into() + } else if let Some(ip) = gen_name.ipaddress() { + // Parse IP address properly (IPv4 or IPv6) + let ip_str = if ip.len() == 4 { + // IPv4 + format!("{}.{}.{}.{}", ip[0], ip[1], ip[2], ip[3]) + } else if ip.len() == 16 { + // IPv6 - format with all zeros visible (not compressed) + let ip_addr = + std::net::Ipv6Addr::from(<[u8; 16]>::try_from(&ip[0..16]).unwrap()); + let s = ip_addr.segments(); + format!( + "{:X}:{:X}:{:X}:{:X}:{:X}:{:X}:{:X}:{:X}", + s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7] + ) + } else { + // Fallback for unexpected length + String::from_utf8_lossy(ip).into_owned() + }; + vm.new_tuple((ascii!("IP Address"), ip_str)).into() + } else if let Some(uri) = gen_name.uri() { + vm.new_tuple((ascii!("URI"), uri)).into() + } else { + // Handle DirName, Registered ID, and othername + // Check if this is a directory name + if let Some(dirname) = gen_name.directory_name() + && let Ok(py_name) = name_to_py(vm, dirname) + { + return vm.new_tuple((ascii!("DirName"), py_name)).into(); + } + + // Check for Registered ID (GEN_RID) + // Access raw GENERAL_NAME to check type + let ptr = gen_name.as_ptr(); + unsafe { + if (*ptr).type_ == sys::GEN_RID { + // d is ASN1_OBJECT* for GEN_RID + let oid_ptr = (*ptr).d as *const sys::ASN1_OBJECT; + if !oid_ptr.is_null() { + let oid_ref = Asn1ObjectRef::from_ptr(oid_ptr as *mut _); + if let Some(oid_str) = obj2txt(oid_ref, true) { + return vm + .new_tuple((ascii!("Registered ID"), oid_str)) + .into(); + } + } + } + } + + // For othername and other unsupported types + vm.new_tuple((ascii!("othername"), ascii!("<unsupported>"))) + .into() + } + }) + .collect(); + dict.set_item("subjectAltName", vm.ctx.new_tuple(san).into(), vm)?; + }; + + // Authority Information Access: OCSP URIs + if let Ok(ocsp_list) = cert.ocsp_responders() + && !ocsp_list.is_empty() + { + let uris: Vec<PyObjectRef> = ocsp_list + .iter() + .map(|s| vm.ctx.new_str(s.to_string()).into()) + .collect(); + dict.set_item("OCSP", vm.ctx.new_tuple(uris).into(), vm)?; + } + + // Authority Information Access: CA Issuers URIs + if let Some(aia) = cert.authority_info() { + let ca_issuers: Vec<PyObjectRef> = aia + .iter() + .filter_map(|ad| { + // Check if method is CA Issuers (NID_ad_ca_issuers) + if ad.method().nid() != Nid::AD_CA_ISSUERS { + return None; + } + // Get URI from location + ad.location() + .uri() + .map(|uri| vm.ctx.new_str(uri.to_owned()).into()) + }) + .collect(); + if !ca_issuers.is_empty() { + dict.set_item("caIssuers", vm.ctx.new_tuple(ca_issuers).into(), vm)?; + } + } + + // CRL Distribution Points + if let Some(crl_dps) = cert.crl_distribution_points() { + let mut crl_uris: Vec<PyObjectRef> = Vec::new(); + for dp in crl_dps.iter() { + if let Some(dp_name) = dp.distpoint() + && let Some(fullname) = dp_name.fullname() + { + for gn in fullname.iter() { + if let Some(uri) = gn.uri() { + crl_uris.push(vm.ctx.new_str(uri.to_owned()).into()); + } + } + } + } + if !crl_uris.is_empty() { + dict.set_item( + "crlDistributionPoints", + vm.ctx.new_tuple(crl_uris).into(), + vm, + )?; + } + } + + Ok(dict.into()) + } + + // Helper to create Certificate object from X509 + pub(crate) fn cert_to_certificate(vm: &VirtualMachine, cert: X509) -> PyResult { + Ok(PySSLCertificate { cert }.into_ref(&vm.ctx).into()) + } + + // For getpeercert() - returns bytes or dict depending on binary flag + pub(crate) fn cert_to_py(vm: &VirtualMachine, cert: &X509Ref, binary: bool) -> PyResult { + if binary { + let b = cert.to_der().map_err(|e| convert_openssl_error(vm, e))?; + Ok(vm.ctx.new_bytes(b).into()) + } else { + cert_to_dict(vm, cert) + } + } + + #[pyfunction] + pub(crate) fn _test_decode_cert(path: FsPath, vm: &VirtualMachine) -> PyResult { + let path = path.to_path_buf(vm)?; + let pem = std::fs::read(path).map_err(|e| e.to_pyexception(vm))?; + let x509 = X509::from_pem(&pem).map_err(|e| convert_openssl_error(vm, e))?; + cert_to_py(vm, &x509, false) + } +} diff --git a/crates/stdlib/src/openssl/ssl_data_111.rs b/crates/stdlib/src/openssl/ssl_data_111.rs new file mode 100644 index 00000000000..2d5f56f4855 --- /dev/null +++ b/crates/stdlib/src/openssl/ssl_data_111.rs @@ -0,0 +1,1347 @@ +// File generated by tools/make_ssl_data_rs.py +// Generated on 2025-10-29T07:17:23.692784+00:00 +// Source: OpenSSL from /tmp/openssl-1.1.1 +// spell-checker: disable + +use phf::phf_map; + +// Maps lib_code -> library name +// Example: 20 -> "SSL" +pub static LIBRARY_CODES: phf::Map<u32, &'static str> = phf_map! { + 1u32 => "NONE", + 2u32 => "SYS", + 3u32 => "BN", + 4u32 => "RSA", + 5u32 => "DH", + 6u32 => "EVP", + 7u32 => "BUF", + 8u32 => "OBJ", + 9u32 => "PEM", + 10u32 => "DSA", + 11u32 => "X509", + 12u32 => "METH", + 13u32 => "ASN1", + 14u32 => "CONF", + 15u32 => "CRYPTO", + 16u32 => "EC", + 20u32 => "SSL", + 21u32 => "SSL23", + 22u32 => "SSL2", + 23u32 => "SSL3", + 30u32 => "RSAREF", + 31u32 => "PROXY", + 32u32 => "BIO", + 33u32 => "PKCS7", + 34u32 => "X509V3", + 35u32 => "PKCS12", + 36u32 => "RAND", + 37u32 => "DSO", + 38u32 => "ENGINE", + 39u32 => "OCSP", + 40u32 => "UI", + 41u32 => "COMP", + 42u32 => "ECDSA", + 43u32 => "ECDH", + 44u32 => "OSSL_STORE", + 45u32 => "FIPS", + 46u32 => "CMS", + 47u32 => "TS", + 48u32 => "HMAC", + 49u32 => "JPAKE", + 50u32 => "CT", + 51u32 => "ASYNC", + 52u32 => "KDF", + 53u32 => "SM2", + 128u32 => "USER", +}; + +// Maps encoded (lib, reason) -> error mnemonic +// Example: encode_error_key(20, 134) -> "CERTIFICATE_VERIFY_FAILED" +// Key encoding: (lib << 32) | reason +pub static ERROR_CODES: phf::Map<u64, &'static str> = phf_map! { + 55834575019u64 => "ADDING_OBJECT", + 55834575051u64 => "ASN1_PARSE_ERROR", + 55834575052u64 => "ASN1_SIG_PARSE_ERROR", + 55834574948u64 => "AUX_ERROR", + 55834574950u64 => "BAD_OBJECT_HEADER", + 55834575078u64 => "BAD_TEMPLATE", + 55834575062u64 => "BMPSTRING_IS_WRONG_LENGTH", + 55834574953u64 => "BN_LIB", + 55834574954u64 => "BOOLEAN_IS_WRONG_LENGTH", + 55834574955u64 => "BUFFER_TOO_SMALL", + 55834574956u64 => "CIPHER_HAS_NO_OBJECT_IDENTIFIER", + 55834575065u64 => "CONTEXT_NOT_INITIALISED", + 55834574957u64 => "DATA_IS_WRONG", + 55834574958u64 => "DECODE_ERROR", + 55834575022u64 => "DEPTH_EXCEEDED", + 55834575046u64 => "DIGEST_AND_KEY_TYPE_NOT_SUPPORTED", + 55834574960u64 => "ENCODE_ERROR", + 55834575021u64 => "ERROR_GETTING_TIME", + 55834575020u64 => "ERROR_LOADING_SECTION", + 55834574962u64 => "ERROR_SETTING_CIPHER_PARAMS", + 55834574963u64 => "EXPECTING_AN_INTEGER", + 55834574964u64 => "EXPECTING_AN_OBJECT", + 55834574967u64 => "EXPLICIT_LENGTH_MISMATCH", + 55834574968u64 => "EXPLICIT_TAG_NOT_CONSTRUCTED", + 55834574969u64 => "FIELD_MISSING", + 55834574970u64 => "FIRST_NUM_TOO_LARGE", + 55834574971u64 => "HEADER_TOO_LONG", + 55834575023u64 => "ILLEGAL_BITSTRING_FORMAT", + 55834575024u64 => "ILLEGAL_BOOLEAN", + 55834574972u64 => "ILLEGAL_CHARACTERS", + 55834575025u64 => "ILLEGAL_FORMAT", + 55834575026u64 => "ILLEGAL_HEX", + 55834575027u64 => "ILLEGAL_IMPLICIT_TAG", + 55834575028u64 => "ILLEGAL_INTEGER", + 55834575074u64 => "ILLEGAL_NEGATIVE_VALUE", + 55834575029u64 => "ILLEGAL_NESTED_TAGGING", + 55834574973u64 => "ILLEGAL_NULL", + 55834575030u64 => "ILLEGAL_NULL_VALUE", + 55834575031u64 => "ILLEGAL_OBJECT", + 55834574974u64 => "ILLEGAL_OPTIONAL_ANY", + 55834575018u64 => "ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE", + 55834575069u64 => "ILLEGAL_PADDING", + 55834574975u64 => "ILLEGAL_TAGGED_ANY", + 55834575032u64 => "ILLEGAL_TIME_VALUE", + 55834575070u64 => "ILLEGAL_ZERO_CONTENT", + 55834575033u64 => "INTEGER_NOT_ASCII_FORMAT", + 55834574976u64 => "INTEGER_TOO_LARGE_FOR_LONG", + 55834575068u64 => "INVALID_BIT_STRING_BITS_LEFT", + 55834574977u64 => "INVALID_BMPSTRING_LENGTH", + 55834574978u64 => "INVALID_DIGIT", + 55834575053u64 => "INVALID_MIME_TYPE", + 55834575034u64 => "INVALID_MODIFIER", + 55834575035u64 => "INVALID_NUMBER", + 55834575064u64 => "INVALID_OBJECT_ENCODING", + 55834575075u64 => "INVALID_SCRYPT_PARAMETERS", + 55834574979u64 => "INVALID_SEPARATOR", + 55834575066u64 => "INVALID_STRING_TABLE_VALUE", + 55834574981u64 => "INVALID_UNIVERSALSTRING_LENGTH", + 55834574982u64 => "INVALID_UTF8STRING", + 55834575067u64 => "INVALID_VALUE", + 55834575036u64 => "LIST_ERROR", + 55834575054u64 => "MIME_NO_CONTENT_TYPE", + 55834575055u64 => "MIME_PARSE_ERROR", + 55834575056u64 => "MIME_SIG_PARSE_ERROR", + 55834574985u64 => "MISSING_EOC", + 55834574986u64 => "MISSING_SECOND_NUMBER", + 55834575037u64 => "MISSING_VALUE", + 55834574987u64 => "MSTRING_NOT_UNIVERSAL", + 55834574988u64 => "MSTRING_WRONG_TAG", + 55834575045u64 => "NESTED_ASN1_STRING", + 55834575049u64 => "NESTED_TOO_DEEP", + 55834574989u64 => "NON_HEX_CHARACTERS", + 55834575038u64 => "NOT_ASCII_FORMAT", + 55834574990u64 => "NOT_ENOUGH_DATA", + 55834575057u64 => "NO_CONTENT_TYPE", + 55834574991u64 => "NO_MATCHING_CHOICE_TYPE", + 55834575058u64 => "NO_MULTIPART_BODY_FAILURE", + 55834575059u64 => "NO_MULTIPART_BOUNDARY", + 55834575060u64 => "NO_SIG_CONTENT_TYPE", + 55834574992u64 => "NULL_IS_WRONG_LENGTH", + 55834575039u64 => "OBJECT_NOT_ASCII_FORMAT", + 55834574993u64 => "ODD_NUMBER_OF_CHARS", + 55834574995u64 => "SECOND_NUMBER_TOO_LARGE", + 55834574996u64 => "SEQUENCE_LENGTH_MISMATCH", + 55834574997u64 => "SEQUENCE_NOT_CONSTRUCTED", + 55834575040u64 => "SEQUENCE_OR_SET_NEEDS_CONFIG", + 55834574998u64 => "SHORT_LINE", + 55834575061u64 => "SIG_INVALID_MIME_TYPE", + 55834575050u64 => "STREAMING_NOT_SUPPORTED", + 55834574999u64 => "STRING_TOO_LONG", + 55834575000u64 => "STRING_TOO_SHORT", + 55834575002u64 => "THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", + 55834575041u64 => "TIME_NOT_ASCII_FORMAT", + 55834575071u64 => "TOO_LARGE", + 55834575003u64 => "TOO_LONG", + 55834575072u64 => "TOO_SMALL", + 55834575004u64 => "TYPE_NOT_CONSTRUCTED", + 55834575043u64 => "TYPE_NOT_PRIMITIVE", + 55834575007u64 => "UNEXPECTED_EOC", + 55834575063u64 => "UNIVERSALSTRING_IS_WRONG_LENGTH", + 55834575008u64 => "UNKNOWN_FORMAT", + 55834575009u64 => "UNKNOWN_MESSAGE_DIGEST_ALGORITHM", + 55834575010u64 => "UNKNOWN_OBJECT_TYPE", + 55834575011u64 => "UNKNOWN_PUBLIC_KEY_TYPE", + 55834575047u64 => "UNKNOWN_SIGNATURE_ALGORITHM", + 55834575042u64 => "UNKNOWN_TAG", + 55834575012u64 => "UNSUPPORTED_ANY_DEFINED_BY_TYPE", + 55834575076u64 => "UNSUPPORTED_CIPHER", + 55834575015u64 => "UNSUPPORTED_PUBLIC_KEY_TYPE", + 55834575044u64 => "UNSUPPORTED_TYPE", + 55834575073u64 => "WRONG_INTEGER_TYPE", + 55834575048u64 => "WRONG_PUBLIC_KEY_TYPE", + 55834575016u64 => "WRONG_TAG", + 219043332197u64 => "FAILED_TO_SET_POOL", + 219043332198u64 => "FAILED_TO_SWAP_CONTEXT", + 219043332201u64 => "INIT_FAILED", + 219043332199u64 => "INVALID_POOL_SIZE", + 137438953572u64 => "ACCEPT_ERROR", + 137438953613u64 => "ADDRINFO_ADDR_IS_NOT_AF_INET", + 137438953601u64 => "AMBIGUOUS_HOST_OR_SERVICE", + 137438953573u64 => "BAD_FOPEN_MODE", + 137438953596u64 => "BROKEN_PIPE", + 137438953575u64 => "CONNECT_ERROR", + 137438953579u64 => "GETHOSTBYNAME_ADDR_IS_NOT_AF_INET", + 137438953604u64 => "GETSOCKNAME_ERROR", + 137438953605u64 => "GETSOCKNAME_TRUNCATED_ADDRESS", + 137438953606u64 => "GETTING_SOCKTYPE", + 137438953597u64 => "INVALID_ARGUMENT", + 137438953607u64 => "INVALID_SOCKET", + 137438953595u64 => "IN_USE", + 137438953574u64 => "LENGTH_TOO_LONG", + 137438953608u64 => "LISTEN_V6_ONLY", + 137438953614u64 => "LOOKUP_RETURNED_NOTHING", + 137438953602u64 => "MALFORMED_HOST_OR_SERVICE", + 137438953582u64 => "NBIO_CONNECT_ERROR", + 137438953615u64 => "NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", + 137438953616u64 => "NO_HOSTNAME_OR_SERVICE_SPECIFIED", + 137438953585u64 => "NO_PORT_DEFINED", + 137438953600u64 => "NO_SUCH_FILE", + 137438953587u64 => "NULL_PARAMETER", + 137438953589u64 => "UNABLE_TO_BIND_SOCKET", + 137438953590u64 => "UNABLE_TO_CREATE_SOCKET", + 137438953609u64 => "UNABLE_TO_KEEPALIVE", + 137438953591u64 => "UNABLE_TO_LISTEN_SOCKET", + 137438953610u64 => "UNABLE_TO_NODELAY", + 137438953611u64 => "UNABLE_TO_REUSEADDR", + 137438953617u64 => "UNAVAILABLE_IP_FAMILY", + 137438953592u64 => "UNINITIALIZED", + 137438953612u64 => "UNKNOWN_INFO_TYPE", + 137438953618u64 => "UNSUPPORTED_IP_FAMILY", + 137438953593u64 => "UNSUPPORTED_METHOD", + 137438953603u64 => "UNSUPPORTED_PROTOCOL_FAMILY", + 137438953598u64 => "WRITE_TO_READ_ONLY_BIO", + 137438953594u64 => "WSASTARTUP", + 12884901988u64 => "ARG2_LT_ARG3", + 12884901989u64 => "BAD_RECIPROCAL", + 12884902002u64 => "BIGNUM_TOO_LONG", + 12884902006u64 => "BITS_TOO_SMALL", + 12884901990u64 => "CALLED_WITH_EVEN_MODULUS", + 12884901991u64 => "DIV_BY_ZERO", + 12884901992u64 => "ENCODING_ERROR", + 12884901993u64 => "EXPAND_ON_STATIC_BIGNUM_DATA", + 12884901998u64 => "INPUT_NOT_REDUCED", + 12884901994u64 => "INVALID_LENGTH", + 12884902003u64 => "INVALID_RANGE", + 12884902007u64 => "INVALID_SHIFT", + 12884901999u64 => "NOT_A_SQUARE", + 12884901995u64 => "NOT_INITIALIZED", + 12884901996u64 => "NO_INVERSE", + 12884902004u64 => "NO_SOLUTION", + 12884902005u64 => "PRIVATE_KEY_TOO_LARGE", + 12884902000u64 => "P_IS_NOT_PRIME", + 12884902001u64 => "TOO_MANY_ITERATIONS", + 12884901997u64 => "TOO_MANY_TEMPORARY_VARIABLES", + 197568495715u64 => "ADD_SIGNER_ERROR", + 197568495777u64 => "ATTRIBUTE_ERROR", + 197568495791u64 => "CERTIFICATE_ALREADY_PRESENT", + 197568495776u64 => "CERTIFICATE_HAS_NO_KEYID", + 197568495716u64 => "CERTIFICATE_VERIFY_ERROR", + 197568495717u64 => "CIPHER_INITIALISATION_ERROR", + 197568495718u64 => "CIPHER_PARAMETER_INITIALISATION_ERROR", + 197568495719u64 => "CMS_DATAFINAL_ERROR", + 197568495720u64 => "CMS_LIB", + 197568495786u64 => "CONTENTIDENTIFIER_MISMATCH", + 197568495721u64 => "CONTENT_NOT_FOUND", + 197568495787u64 => "CONTENT_TYPE_MISMATCH", + 197568495722u64 => "CONTENT_TYPE_NOT_COMPRESSED_DATA", + 197568495723u64 => "CONTENT_TYPE_NOT_ENVELOPED_DATA", + 197568495724u64 => "CONTENT_TYPE_NOT_SIGNED_DATA", + 197568495725u64 => "CONTENT_VERIFY_ERROR", + 197568495726u64 => "CTRL_ERROR", + 197568495727u64 => "CTRL_FAILURE", + 197568495728u64 => "DECRYPT_ERROR", + 197568495729u64 => "ERROR_GETTING_PUBLIC_KEY", + 197568495730u64 => "ERROR_READING_MESSAGEDIGEST_ATTRIBUTE", + 197568495731u64 => "ERROR_SETTING_KEY", + 197568495732u64 => "ERROR_SETTING_RECIPIENTINFO", + 197568495733u64 => "INVALID_ENCRYPTED_KEY_LENGTH", + 197568495792u64 => "INVALID_KEY_ENCRYPTION_PARAMETER", + 197568495734u64 => "INVALID_KEY_LENGTH", + 197568495735u64 => "MD_BIO_INIT_ERROR", + 197568495736u64 => "MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH", + 197568495737u64 => "MESSAGEDIGEST_WRONG_LENGTH", + 197568495788u64 => "MSGSIGDIGEST_ERROR", + 197568495778u64 => "MSGSIGDIGEST_VERIFICATION_FAILURE", + 197568495779u64 => "MSGSIGDIGEST_WRONG_LENGTH", + 197568495780u64 => "NEED_ONE_SIGNER", + 197568495781u64 => "NOT_A_SIGNED_RECEIPT", + 197568495738u64 => "NOT_ENCRYPTED_DATA", + 197568495739u64 => "NOT_KEK", + 197568495797u64 => "NOT_KEY_AGREEMENT", + 197568495740u64 => "NOT_KEY_TRANSPORT", + 197568495793u64 => "NOT_PWRI", + 197568495741u64 => "NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 197568495742u64 => "NO_CIPHER", + 197568495743u64 => "NO_CONTENT", + 197568495789u64 => "NO_CONTENT_TYPE", + 197568495744u64 => "NO_DEFAULT_DIGEST", + 197568495745u64 => "NO_DIGEST_SET", + 197568495746u64 => "NO_KEY", + 197568495790u64 => "NO_KEY_OR_CERT", + 197568495747u64 => "NO_MATCHING_DIGEST", + 197568495748u64 => "NO_MATCHING_RECIPIENT", + 197568495782u64 => "NO_MATCHING_SIGNATURE", + 197568495783u64 => "NO_MSGSIGDIGEST", + 197568495794u64 => "NO_PASSWORD", + 197568495749u64 => "NO_PRIVATE_KEY", + 197568495750u64 => "NO_PUBLIC_KEY", + 197568495784u64 => "NO_RECEIPT_REQUEST", + 197568495751u64 => "NO_SIGNERS", + 197568495752u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 197568495785u64 => "RECEIPT_DECODE_ERROR", + 197568495753u64 => "RECIPIENT_ERROR", + 197568495754u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 197568495755u64 => "SIGNFINAL_ERROR", + 197568495756u64 => "SMIME_TEXT_ERROR", + 197568495757u64 => "STORE_INIT_ERROR", + 197568495758u64 => "TYPE_NOT_COMPRESSED_DATA", + 197568495759u64 => "TYPE_NOT_DATA", + 197568495760u64 => "TYPE_NOT_DIGESTED_DATA", + 197568495761u64 => "TYPE_NOT_ENCRYPTED_DATA", + 197568495762u64 => "TYPE_NOT_ENVELOPED_DATA", + 197568495763u64 => "UNABLE_TO_FINALIZE_CONTEXT", + 197568495764u64 => "UNKNOWN_CIPHER", + 197568495765u64 => "UNKNOWN_DIGEST_ALGORITHM", + 197568495766u64 => "UNKNOWN_ID", + 197568495767u64 => "UNSUPPORTED_COMPRESSION_ALGORITHM", + 197568495810u64 => "UNSUPPORTED_CONTENT_ENCRYPTION_ALGORITHM", + 197568495768u64 => "UNSUPPORTED_CONTENT_TYPE", + 197568495769u64 => "UNSUPPORTED_KEK_ALGORITHM", + 197568495795u64 => "UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM", + 197568495771u64 => "UNSUPPORTED_RECIPIENTINFO_TYPE", + 197568495770u64 => "UNSUPPORTED_RECIPIENT_TYPE", + 197568495772u64 => "UNSUPPORTED_TYPE", + 197568495773u64 => "UNWRAP_ERROR", + 197568495796u64 => "UNWRAP_FAILURE", + 197568495774u64 => "VERIFICATION_FAILURE", + 197568495775u64 => "WRAP_ERROR", + 176093659235u64 => "ZLIB_DEFLATE_ERROR", + 176093659236u64 => "ZLIB_INFLATE_ERROR", + 176093659237u64 => "ZLIB_NOT_SUPPORTED", + 60129542254u64 => "ERROR_LOADING_DSO", + 60129542259u64 => "LIST_CANNOT_BE_NULL", + 60129542244u64 => "MISSING_CLOSE_SQUARE_BRACKET", + 60129542245u64 => "MISSING_EQUAL_SIGN", + 60129542256u64 => "MISSING_INIT_FUNCTION", + 60129542253u64 => "MODULE_INITIALIZATION_ERROR", + 60129542246u64 => "NO_CLOSE_BRACE", + 60129542249u64 => "NO_CONF", + 60129542250u64 => "NO_CONF_OR_ENVIRONMENT_VARIABLE", + 60129542251u64 => "NO_SECTION", + 60129542258u64 => "NO_SUCH_FILE", + 60129542252u64 => "NO_VALUE", + 60129542265u64 => "NUMBER_TOO_LARGE", + 60129542255u64 => "RECURSIVE_DIRECTORY_INCLUDE", + 60129542261u64 => "SSL_COMMAND_SECTION_EMPTY", + 60129542262u64 => "SSL_COMMAND_SECTION_NOT_FOUND", + 60129542263u64 => "SSL_SECTION_EMPTY", + 60129542264u64 => "SSL_SECTION_NOT_FOUND", + 60129542247u64 => "UNABLE_TO_CREATE_NEW_SECTION", + 60129542257u64 => "UNKNOWN_MODULE_NAME", + 60129542260u64 => "VARIABLE_EXPANSION_TOO_LONG", + 60129542248u64 => "VARIABLE_HAS_NO_VALUE", + 64424509541u64 => "FIPS_MODE_NOT_SUPPORTED", + 64424509542u64 => "ILLEGAL_HEX_DIGIT", + 64424509543u64 => "ODD_NUMBER_OF_DIGITS", + 214748364908u64 => "BASE64_DECODE_ERROR", + 214748364900u64 => "INVALID_LOG_ID_LENGTH", + 214748364909u64 => "LOG_CONF_INVALID", + 214748364910u64 => "LOG_CONF_INVALID_KEY", + 214748364911u64 => "LOG_CONF_MISSING_DESCRIPTION", + 214748364912u64 => "LOG_CONF_MISSING_KEY", + 214748364913u64 => "LOG_KEY_INVALID", + 214748364916u64 => "SCT_FUTURE_TIMESTAMP", + 214748364904u64 => "SCT_INVALID", + 214748364907u64 => "SCT_INVALID_SIGNATURE", + 214748364905u64 => "SCT_LIST_INVALID", + 214748364914u64 => "SCT_LOG_ID_MISMATCH", + 214748364906u64 => "SCT_NOT_SET", + 214748364915u64 => "SCT_UNSUPPORTED_VERSION", + 214748364901u64 => "UNRECOGNIZED_SIGNATURE_NID", + 214748364902u64 => "UNSUPPORTED_ENTRY_TYPE", + 214748364903u64 => "UNSUPPORTED_VERSION", + 21474836581u64 => "BAD_GENERATOR", + 21474836589u64 => "BN_DECODE_ERROR", + 21474836586u64 => "BN_ERROR", + 21474836595u64 => "CHECK_INVALID_J_VALUE", + 21474836596u64 => "CHECK_INVALID_Q_VALUE", + 21474836602u64 => "CHECK_PUBKEY_INVALID", + 21474836603u64 => "CHECK_PUBKEY_TOO_LARGE", + 21474836604u64 => "CHECK_PUBKEY_TOO_SMALL", + 21474836597u64 => "CHECK_P_NOT_PRIME", + 21474836598u64 => "CHECK_P_NOT_SAFE_PRIME", + 21474836599u64 => "CHECK_Q_NOT_PRIME", + 21474836584u64 => "DECODE_ERROR", + 21474836590u64 => "INVALID_PARAMETER_NAME", + 21474836594u64 => "INVALID_PARAMETER_NID", + 21474836582u64 => "INVALID_PUBKEY", + 21474836592u64 => "KDF_PARAMETER_ERROR", + 21474836588u64 => "KEYS_NOT_SET", + 21474836605u64 => "MISSING_PUBKEY", + 21474836583u64 => "MODULUS_TOO_LARGE", + 21474836600u64 => "NOT_SUITABLE_GENERATOR", + 21474836587u64 => "NO_PARAMETERS_SET", + 21474836580u64 => "NO_PRIVATE_VALUE", + 21474836585u64 => "PARAMETER_ENCODING_ERROR", + 21474836591u64 => "PEER_KEY_ERROR", + 21474836593u64 => "SHARED_INFO_ERROR", + 21474836601u64 => "UNABLE_TO_CHECK_GENERATOR", + 42949673062u64 => "BAD_Q_VALUE", + 42949673068u64 => "BN_DECODE_ERROR", + 42949673069u64 => "BN_ERROR", + 42949673064u64 => "DECODE_ERROR", + 42949673066u64 => "INVALID_DIGEST_TYPE", + 42949673072u64 => "INVALID_PARAMETERS", + 42949673061u64 => "MISSING_PARAMETERS", + 42949673071u64 => "MISSING_PRIVATE_KEY", + 42949673063u64 => "MODULUS_TOO_LARGE", + 42949673067u64 => "NO_PARAMETERS_SET", + 42949673065u64 => "PARAMETER_ENCODING_ERROR", + 42949673073u64 => "Q_NOT_PRIME", + 42949673070u64 => "SEED_LEN_SMALL", + 158913790052u64 => "CTRL_FAILED", + 158913790062u64 => "DSO_ALREADY_LOADED", + 158913790065u64 => "EMPTY_FILE_STRUCTURE", + 158913790066u64 => "FAILURE", + 158913790053u64 => "FILENAME_TOO_BIG", + 158913790054u64 => "FINISH_FAILED", + 158913790067u64 => "INCORRECT_FILE_SYNTAX", + 158913790055u64 => "LOAD_FAILED", + 158913790061u64 => "NAME_TRANSLATION_FAILED", + 158913790063u64 => "NO_FILENAME", + 158913790056u64 => "NULL_HANDLE", + 158913790064u64 => "SET_FILENAME_FAILED", + 158913790057u64 => "STACK_ERROR", + 158913790058u64 => "SYM_FAILURE", + 158913790059u64 => "UNLOAD_FAILED", + 158913790060u64 => "UNSUPPORTED", + 68719476851u64 => "ASN1_ERROR", + 68719476892u64 => "BAD_SIGNATURE", + 68719476880u64 => "BIGNUM_OUT_OF_RANGE", + 68719476836u64 => "BUFFER_TOO_SMALL", + 68719476901u64 => "CANNOT_INVERT", + 68719476882u64 => "COORDINATES_OUT_OF_RANGE", + 68719476896u64 => "CURVE_DOES_NOT_SUPPORT_ECDH", + 68719476895u64 => "CURVE_DOES_NOT_SUPPORT_SIGNING", + 68719476853u64 => "D2I_ECPKPARAMETERS_FAILURE", + 68719476878u64 => "DECODE_ERROR", + 68719476854u64 => "DISCRIMINANT_IS_ZERO", + 68719476855u64 => "EC_GROUP_NEW_BY_NAME_FAILURE", + 68719476879u64 => "FIELD_TOO_LARGE", + 68719476883u64 => "GF2M_NOT_SUPPORTED", + 68719476856u64 => "GROUP2PKPARAMETERS_FAILURE", + 68719476857u64 => "I2D_ECPKPARAMETERS_FAILURE", + 68719476837u64 => "INCOMPATIBLE_OBJECTS", + 68719476848u64 => "INVALID_ARGUMENT", + 68719476846u64 => "INVALID_COMPRESSED_POINT", + 68719476845u64 => "INVALID_COMPRESSION_BIT", + 68719476877u64 => "INVALID_CURVE", + 68719476887u64 => "INVALID_DIGEST", + 68719476874u64 => "INVALID_DIGEST_TYPE", + 68719476838u64 => "INVALID_ENCODING", + 68719476839u64 => "INVALID_FIELD", + 68719476840u64 => "INVALID_FORM", + 68719476858u64 => "INVALID_GROUP_ORDER", + 68719476852u64 => "INVALID_KEY", + 68719476897u64 => "INVALID_OUTPUT_LENGTH", + 68719476869u64 => "INVALID_PEER_KEY", + 68719476868u64 => "INVALID_PENTANOMIAL_BASIS", + 68719476859u64 => "INVALID_PRIVATE_KEY", + 68719476873u64 => "INVALID_TRINOMIAL_BASIS", + 68719476884u64 => "KDF_PARAMETER_ERROR", + 68719476876u64 => "KEYS_NOT_SET", + 68719476872u64 => "LADDER_POST_FAILURE", + 68719476889u64 => "LADDER_PRE_FAILURE", + 68719476898u64 => "LADDER_STEP_FAILURE", + 68719476903u64 => "MISSING_OID", + 68719476860u64 => "MISSING_PARAMETERS", + 68719476861u64 => "MISSING_PRIVATE_KEY", + 68719476893u64 => "NEED_NEW_SETUP_VALUES", + 68719476871u64 => "NOT_A_NIST_PRIME", + 68719476862u64 => "NOT_IMPLEMENTED", + 68719476847u64 => "NOT_INITIALIZED", + 68719476875u64 => "NO_PARAMETERS_SET", + 68719476890u64 => "NO_PRIVATE_VALUE", + 68719476888u64 => "OPERATION_NOT_SUPPORTED", + 68719476870u64 => "PASSED_NULL_PARAMETER", + 68719476885u64 => "PEER_KEY_ERROR", + 68719476863u64 => "PKPARAMETERS2GROUP_FAILURE", + 68719476891u64 => "POINT_ARITHMETIC_FAILURE", + 68719476842u64 => "POINT_AT_INFINITY", + 68719476899u64 => "POINT_COORDINATES_BLIND_FAILURE", + 68719476843u64 => "POINT_IS_NOT_ON_CURVE", + 68719476894u64 => "RANDOM_NUMBER_GENERATION_FAILED", + 68719476886u64 => "SHARED_INFO_ERROR", + 68719476844u64 => "SLOT_FULL", + 68719476849u64 => "UNDEFINED_GENERATOR", + 68719476864u64 => "UNDEFINED_ORDER", + 68719476900u64 => "UNKNOWN_COFACTOR", + 68719476865u64 => "UNKNOWN_GROUP", + 68719476850u64 => "UNKNOWN_ORDER", + 68719476867u64 => "UNSUPPORTED_FIELD", + 68719476881u64 => "WRONG_CURVE_PARAMETERS", + 68719476866u64 => "WRONG_ORDER", + 163208757348u64 => "ALREADY_LOADED", + 163208757381u64 => "ARGUMENT_IS_NOT_A_NUMBER", + 163208757382u64 => "CMD_NOT_EXECUTABLE", + 163208757383u64 => "COMMAND_TAKES_INPUT", + 163208757384u64 => "COMMAND_TAKES_NO_INPUT", + 163208757351u64 => "CONFLICTING_ENGINE_ID", + 163208757367u64 => "CTRL_COMMAND_NOT_IMPLEMENTED", + 163208757352u64 => "DSO_FAILURE", + 163208757380u64 => "DSO_NOT_FOUND", + 163208757396u64 => "ENGINES_SECTION_ERROR", + 163208757350u64 => "ENGINE_CONFIGURATION_ERROR", + 163208757353u64 => "ENGINE_IS_NOT_IN_LIST", + 163208757397u64 => "ENGINE_SECTION_ERROR", + 163208757376u64 => "FAILED_LOADING_PRIVATE_KEY", + 163208757377u64 => "FAILED_LOADING_PUBLIC_KEY", + 163208757354u64 => "FINISH_FAILED", + 163208757356u64 => "ID_OR_NAME_MISSING", + 163208757357u64 => "INIT_FAILED", + 163208757358u64 => "INTERNAL_LIST_ERROR", + 163208757391u64 => "INVALID_ARGUMENT", + 163208757385u64 => "INVALID_CMD_NAME", + 163208757386u64 => "INVALID_CMD_NUMBER", + 163208757399u64 => "INVALID_INIT_VALUE", + 163208757398u64 => "INVALID_STRING", + 163208757365u64 => "NOT_INITIALISED", + 163208757360u64 => "NOT_LOADED", + 163208757368u64 => "NO_CONTROL_FUNCTION", + 163208757392u64 => "NO_INDEX", + 163208757373u64 => "NO_LOAD_FUNCTION", + 163208757378u64 => "NO_REFERENCE", + 163208757364u64 => "NO_SUCH_ENGINE", + 163208757394u64 => "UNIMPLEMENTED_CIPHER", + 163208757395u64 => "UNIMPLEMENTED_DIGEST", + 163208757349u64 => "UNIMPLEMENTED_PUBLIC_KEY_METHOD", + 163208757393u64 => "VERSION_INCOMPATIBILITY", + 25769803919u64 => "AES_KEY_SETUP_FAILED", + 25769803952u64 => "ARIA_KEY_SETUP_FAILED", + 25769803876u64 => "BAD_DECRYPT", + 25769803971u64 => "BAD_KEY_LENGTH", + 25769803931u64 => "BUFFER_TOO_SMALL", + 25769803933u64 => "CAMELLIA_KEY_SETUP_FAILED", + 25769803898u64 => "CIPHER_PARAMETER_ERROR", + 25769803923u64 => "COMMAND_NOT_SUPPORTED", + 25769803949u64 => "COPY_ERROR", + 25769803908u64 => "CTRL_NOT_IMPLEMENTED", + 25769803909u64 => "CTRL_OPERATION_NOT_IMPLEMENTED", + 25769803914u64 => "DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH", + 25769803890u64 => "DECODE_ERROR", + 25769803877u64 => "DIFFERENT_KEY_TYPES", + 25769803929u64 => "DIFFERENT_PARAMETERS", + 25769803941u64 => "ERROR_LOADING_SECTION", + 25769803942u64 => "ERROR_SETTING_FIPS_MODE", + 25769803950u64 => "EXPECTING_AN_HMAC_KEY", + 25769803903u64 => "EXPECTING_AN_RSA_KEY", + 25769803904u64 => "EXPECTING_A_DH_KEY", + 25769803905u64 => "EXPECTING_A_DSA_KEY", + 25769803918u64 => "EXPECTING_A_EC_KEY", + 25769803940u64 => "EXPECTING_A_POLY1305_KEY", + 25769803951u64 => "EXPECTING_A_SIPHASH_KEY", + 25769803943u64 => "FIPS_MODE_NOT_SUPPORTED", + 25769803958u64 => "GET_RAW_KEY_FAILED", + 25769803947u64 => "ILLEGAL_SCRYPT_PARAMETERS", + 25769803910u64 => "INITIALIZATION_ERROR", + 25769803887u64 => "INPUT_NOT_INITIALIZED", + 25769803928u64 => "INVALID_DIGEST", + 25769803944u64 => "INVALID_FIPS_MODE", + 25769803970u64 => "INVALID_IV_LENGTH", + 25769803939u64 => "INVALID_KEY", + 25769803906u64 => "INVALID_KEY_LENGTH", + 25769803924u64 => "INVALID_OPERATION", + 25769803896u64 => "KEYGEN_FAILURE", + 25769803956u64 => "KEY_SETUP_FAILED", + 25769803948u64 => "MEMORY_LIMIT_EXCEEDED", + 25769803935u64 => "MESSAGE_DIGEST_IS_NULL", + 25769803920u64 => "METHOD_NOT_SUPPORTED", + 25769803879u64 => "MISSING_PARAMETERS", + 25769803954u64 => "NOT_XOF_OR_INVALID_LENGTH", + 25769803907u64 => "NO_CIPHER_SET", + 25769803934u64 => "NO_DEFAULT_DIGEST", + 25769803915u64 => "NO_DIGEST_SET", + 25769803930u64 => "NO_KEY_SET", + 25769803925u64 => "NO_OPERATION_SET", + 25769803953u64 => "ONLY_ONESHOT_SUPPORTED", + 25769803926u64 => "OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", + 25769803927u64 => "OPERATON_NOT_INITIALIZED", + 25769803960u64 => "OUTPUT_WOULD_OVERFLOW", + 25769803938u64 => "PARTIALLY_OVERLAPPING", + 25769803957u64 => "PBKDF2_ERROR", + 25769803955u64 => "PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", + 25769803921u64 => "PRIVATE_KEY_DECODE_ERROR", + 25769803922u64 => "PRIVATE_KEY_ENCODE_ERROR", + 25769803882u64 => "PUBLIC_KEY_NOT_RSA", + 25769803936u64 => "UNKNOWN_CIPHER", + 25769803937u64 => "UNKNOWN_DIGEST", + 25769803945u64 => "UNKNOWN_OPTION", + 25769803897u64 => "UNKNOWN_PBE_ALGORITHM", + 25769803932u64 => "UNSUPPORTED_ALGORITHM", + 25769803883u64 => "UNSUPPORTED_CIPHER", + 25769803899u64 => "UNSUPPORTED_KEYLENGTH", + 25769803900u64 => "UNSUPPORTED_KEY_DERIVATION_FUNCTION", + 25769803884u64 => "UNSUPPORTED_KEY_SIZE", + 25769803911u64 => "UNSUPPORTED_NUMBER_OF_ROUNDS", + 25769803901u64 => "UNSUPPORTED_PRF", + 25769803894u64 => "UNSUPPORTED_PRIVATE_KEY_ALGORITHM", + 25769803902u64 => "UNSUPPORTED_SALT_TYPE", + 25769803946u64 => "WRAP_MODE_NOT_ALLOWED", + 25769803885u64 => "WRONG_FINAL_BLOCK_LENGTH", + 25769803959u64 => "XTS_DUPLICATED_KEYS", + 223338299492u64 => "INVALID_DIGEST", + 223338299501u64 => "MISSING_ITERATION_COUNT", + 223338299496u64 => "MISSING_KEY", + 223338299497u64 => "MISSING_MESSAGE_DIGEST", + 223338299493u64 => "MISSING_PARAMETER", + 223338299502u64 => "MISSING_PASS", + 223338299503u64 => "MISSING_SALT", + 223338299499u64 => "MISSING_SECRET", + 223338299498u64 => "MISSING_SEED", + 223338299495u64 => "UNKNOWN_PARAMETER_TYPE", + 223338299500u64 => "VALUE_ERROR", + 223338299494u64 => "VALUE_MISSING", + 34359738470u64 => "OID_EXISTS", + 34359738469u64 => "UNKNOWN_NID", + 167503724645u64 => "CERTIFICATE_VERIFY_ERROR", + 167503724646u64 => "DIGEST_ERR", + 167503724666u64 => "ERROR_IN_NEXTUPDATE_FIELD", + 167503724667u64 => "ERROR_IN_THISUPDATE_FIELD", + 167503724665u64 => "ERROR_PARSING_URL", + 167503724647u64 => "MISSING_OCSPSIGNING_USAGE", + 167503724668u64 => "NEXTUPDATE_BEFORE_THISUPDATE", + 167503724648u64 => "NOT_BASIC_RESPONSE", + 167503724649u64 => "NO_CERTIFICATES_IN_CHAIN", + 167503724652u64 => "NO_RESPONSE_DATA", + 167503724653u64 => "NO_REVOKED_TIME", + 167503724674u64 => "NO_SIGNER_KEY", + 167503724654u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 167503724672u64 => "REQUEST_NOT_SIGNED", + 167503724655u64 => "RESPONSE_CONTAINS_NO_REVOCATION_DATA", + 167503724656u64 => "ROOT_CA_NOT_TRUSTED", + 167503724658u64 => "SERVER_RESPONSE_ERROR", + 167503724659u64 => "SERVER_RESPONSE_PARSE_ERROR", + 167503724661u64 => "SIGNATURE_FAILURE", + 167503724662u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 167503724669u64 => "STATUS_EXPIRED", + 167503724670u64 => "STATUS_NOT_YET_VALID", + 167503724671u64 => "STATUS_TOO_OLD", + 167503724663u64 => "UNKNOWN_MESSAGE_DIGEST", + 167503724664u64 => "UNKNOWN_NID", + 167503724673u64 => "UNSUPPORTED_REQUESTORNAME_TYPE", + 188978561131u64 => "AMBIGUOUS_CONTENT_TYPE", + 188978561139u64 => "BAD_PASSWORD_READ", + 188978561137u64 => "ERROR_VERIFYING_PKCS12_MAC", + 188978561145u64 => "FINGERPRINT_SIZE_DOES_NOT_MATCH_DIGEST", + 188978561130u64 => "INVALID_SCHEME", + 188978561136u64 => "IS_NOT_A", + 188978561140u64 => "LOADER_INCOMPLETE", + 188978561141u64 => "LOADING_STARTED", + 188978561124u64 => "NOT_A_CERTIFICATE", + 188978561125u64 => "NOT_A_CRL", + 188978561126u64 => "NOT_A_KEY", + 188978561127u64 => "NOT_A_NAME", + 188978561128u64 => "NOT_PARAMETERS", + 188978561138u64 => "PASSPHRASE_CALLBACK_ERROR", + 188978561132u64 => "PATH_MUST_BE_ABSOLUTE", + 188978561143u64 => "SEARCH_ONLY_SUPPORTED_FOR_DIRECTORIES", + 188978561133u64 => "UI_PROCESS_INTERRUPTED_OR_CANCELLED", + 188978561129u64 => "UNREGISTERED_SCHEME", + 188978561134u64 => "UNSUPPORTED_CONTENT_TYPE", + 188978561142u64 => "UNSUPPORTED_OPERATION", + 188978561144u64 => "UNSUPPORTED_SEARCH_TYPE", + 188978561135u64 => "URI_AUTHORITY_UNSUPPORTED", + 38654705764u64 => "BAD_BASE64_DECODE", + 38654705765u64 => "BAD_DECRYPT", + 38654705766u64 => "BAD_END_LINE", + 38654705767u64 => "BAD_IV_CHARS", + 38654705780u64 => "BAD_MAGIC_NUMBER", + 38654705768u64 => "BAD_PASSWORD_READ", + 38654705781u64 => "BAD_VERSION_NUMBER", + 38654705782u64 => "BIO_WRITE_FAILURE", + 38654705791u64 => "CIPHER_IS_NULL", + 38654705779u64 => "ERROR_CONVERTING_PRIVATE_KEY", + 38654705783u64 => "EXPECTING_PRIVATE_KEY_BLOB", + 38654705784u64 => "EXPECTING_PUBLIC_KEY_BLOB", + 38654705792u64 => "HEADER_TOO_LONG", + 38654705785u64 => "INCONSISTENT_HEADER", + 38654705786u64 => "KEYBLOB_HEADER_PARSE_ERROR", + 38654705787u64 => "KEYBLOB_TOO_SHORT", + 38654705793u64 => "MISSING_DEK_IV", + 38654705769u64 => "NOT_DEK_INFO", + 38654705770u64 => "NOT_ENCRYPTED", + 38654705771u64 => "NOT_PROC_TYPE", + 38654705772u64 => "NO_START_LINE", + 38654705773u64 => "PROBLEMS_GETTING_PASSWORD", + 38654705788u64 => "PVK_DATA_TOO_SHORT", + 38654705789u64 => "PVK_TOO_SHORT", + 38654705775u64 => "READ_KEY", + 38654705776u64 => "SHORT_HEADER", + 38654705794u64 => "UNEXPECTED_DEK_IV", + 38654705777u64 => "UNSUPPORTED_CIPHER", + 38654705778u64 => "UNSUPPORTED_ENCRYPTION", + 38654705790u64 => "UNSUPPORTED_KEY_COMPONENTS", + 38654705774u64 => "UNSUPPORTED_PUBLIC_KEY_TYPE", + 150323855460u64 => "CANT_PACK_STRUCTURE", + 150323855481u64 => "CONTENT_TYPE_NOT_DATA", + 150323855461u64 => "DECODE_ERROR", + 150323855462u64 => "ENCODE_ERROR", + 150323855463u64 => "ENCRYPT_ERROR", + 150323855480u64 => "ERROR_SETTING_ENCRYPTED_DATA_TYPE", + 150323855464u64 => "INVALID_NULL_ARGUMENT", + 150323855465u64 => "INVALID_NULL_PKCS12_POINTER", + 150323855466u64 => "IV_GEN_ERROR", + 150323855467u64 => "KEY_GEN_ERROR", + 150323855468u64 => "MAC_ABSENT", + 150323855469u64 => "MAC_GENERATION_ERROR", + 150323855470u64 => "MAC_SETUP_ERROR", + 150323855471u64 => "MAC_STRING_SET_ERROR", + 150323855473u64 => "MAC_VERIFY_FAILURE", + 150323855474u64 => "PARSE_ERROR", + 150323855475u64 => "PKCS12_ALGOR_CIPHERINIT_ERROR", + 150323855476u64 => "PKCS12_CIPHERFINAL_ERROR", + 150323855477u64 => "PKCS12_PBE_CRYPT_ERROR", + 150323855478u64 => "UNKNOWN_DIGEST_ALGORITHM", + 150323855479u64 => "UNSUPPORTED_PKCS12_MODE", + 141733920885u64 => "CERTIFICATE_VERIFY_ERROR", + 141733920912u64 => "CIPHER_HAS_NO_OBJECT_IDENTIFIER", + 141733920884u64 => "CIPHER_NOT_INITIALIZED", + 141733920886u64 => "CONTENT_AND_DATA_PRESENT", + 141733920920u64 => "CTRL_ERROR", + 141733920887u64 => "DECRYPT_ERROR", + 141733920869u64 => "DIGEST_FAILURE", + 141733920917u64 => "ENCRYPTION_CTRL_FAILURE", + 141733920918u64 => "ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 141733920888u64 => "ERROR_ADDING_RECIPIENT", + 141733920889u64 => "ERROR_SETTING_CIPHER", + 141733920911u64 => "INVALID_NULL_POINTER", + 141733920923u64 => "INVALID_SIGNED_DATA_TYPE", + 141733920890u64 => "NO_CONTENT", + 141733920919u64 => "NO_DEFAULT_DIGEST", + 141733920922u64 => "NO_MATCHING_DIGEST_TYPE_FOUND", + 141733920883u64 => "NO_RECIPIENT_MATCHES_CERTIFICATE", + 141733920891u64 => "NO_SIGNATURES_ON_DATA", + 141733920910u64 => "NO_SIGNERS", + 141733920872u64 => "OPERATION_NOT_SUPPORTED_ON_THIS_TYPE", + 141733920892u64 => "PKCS7_ADD_SIGNATURE_ERROR", + 141733920921u64 => "PKCS7_ADD_SIGNER_ERROR", + 141733920913u64 => "PKCS7_DATASIGN", + 141733920895u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 141733920873u64 => "SIGNATURE_FAILURE", + 141733920896u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 141733920915u64 => "SIGNING_CTRL_FAILURE", + 141733920916u64 => "SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 141733920897u64 => "SMIME_TEXT_ERROR", + 141733920874u64 => "UNABLE_TO_FIND_CERTIFICATE", + 141733920875u64 => "UNABLE_TO_FIND_MEM_BIO", + 141733920876u64 => "UNABLE_TO_FIND_MESSAGE_DIGEST", + 141733920877u64 => "UNKNOWN_DIGEST_TYPE", + 141733920878u64 => "UNKNOWN_OPERATION", + 141733920879u64 => "UNSUPPORTED_CIPHER_TYPE", + 141733920880u64 => "UNSUPPORTED_CONTENT_TYPE", + 141733920881u64 => "WRONG_CONTENT_TYPE", + 141733920882u64 => "WRONG_PKCS7_TYPE", + 154618822758u64 => "ADDITIONAL_INPUT_TOO_LONG", + 154618822759u64 => "ALREADY_INSTANTIATED", + 154618822761u64 => "ARGUMENT_OUT_OF_RANGE", + 154618822777u64 => "CANNOT_OPEN_FILE", + 154618822785u64 => "DRBG_ALREADY_INITIALIZED", + 154618822760u64 => "DRBG_NOT_INITIALISED", + 154618822762u64 => "ENTROPY_INPUT_TOO_LONG", + 154618822780u64 => "ENTROPY_OUT_OF_RANGE", + 154618822783u64 => "ERROR_ENTROPY_POOL_WAS_IGNORED", + 154618822763u64 => "ERROR_INITIALISING_DRBG", + 154618822764u64 => "ERROR_INSTANTIATING_DRBG", + 154618822765u64 => "ERROR_RETRIEVING_ADDITIONAL_INPUT", + 154618822766u64 => "ERROR_RETRIEVING_ENTROPY", + 154618822767u64 => "ERROR_RETRIEVING_NONCE", + 154618822782u64 => "FAILED_TO_CREATE_LOCK", + 154618822757u64 => "FUNC_NOT_IMPLEMENTED", + 154618822779u64 => "FWRITE_ERROR", + 154618822768u64 => "GENERATE_ERROR", + 154618822769u64 => "INTERNAL_ERROR", + 154618822770u64 => "IN_ERROR_STATE", + 154618822778u64 => "NOT_A_REGULAR_FILE", + 154618822771u64 => "NOT_INSTANTIATED", + 154618822784u64 => "NO_DRBG_IMPLEMENTATION_SELECTED", + 154618822786u64 => "PARENT_LOCKING_NOT_ENABLED", + 154618822787u64 => "PARENT_STRENGTH_TOO_WEAK", + 154618822772u64 => "PERSONALISATION_STRING_TOO_LONG", + 154618822789u64 => "PREDICTION_RESISTANCE_NOT_SUPPORTED", + 154618822756u64 => "PRNG_NOT_SEEDED", + 154618822781u64 => "RANDOM_POOL_OVERFLOW", + 154618822790u64 => "RANDOM_POOL_UNDERFLOW", + 154618822773u64 => "REQUEST_TOO_LARGE_FOR_DRBG", + 154618822774u64 => "RESEED_ERROR", + 154618822775u64 => "SELFTEST_FAILURE", + 154618822791u64 => "TOO_LITTLE_NONCE_REQUESTED", + 154618822792u64 => "TOO_MUCH_NONCE_REQUESTED", + 154618822788u64 => "UNSUPPORTED_DRBG_FLAGS", + 154618822776u64 => "UNSUPPORTED_DRBG_TYPE", + 17179869284u64 => "ALGORITHM_MISMATCH", + 17179869285u64 => "BAD_E_VALUE", + 17179869286u64 => "BAD_FIXED_HEADER_DECRYPT", + 17179869287u64 => "BAD_PAD_BYTE_COUNT", + 17179869288u64 => "BAD_SIGNATURE", + 17179869290u64 => "BLOCK_TYPE_IS_NOT_01", + 17179869291u64 => "BLOCK_TYPE_IS_NOT_02", + 17179869292u64 => "DATA_GREATER_THAN_MOD_LEN", + 17179869293u64 => "DATA_TOO_LARGE", + 17179869294u64 => "DATA_TOO_LARGE_FOR_KEY_SIZE", + 17179869316u64 => "DATA_TOO_LARGE_FOR_MODULUS", + 17179869295u64 => "DATA_TOO_SMALL", + 17179869306u64 => "DATA_TOO_SMALL_FOR_KEY_SIZE", + 17179869342u64 => "DIGEST_DOES_NOT_MATCH", + 17179869329u64 => "DIGEST_NOT_ALLOWED", + 17179869296u64 => "DIGEST_TOO_BIG_FOR_RSA_KEY", + 17179869308u64 => "DMP1_NOT_CONGRUENT_TO_D", + 17179869309u64 => "DMQ1_NOT_CONGRUENT_TO_D", + 17179869307u64 => "D_E_NOT_CONGRUENT_TO_1", + 17179869317u64 => "FIRST_OCTET_INVALID", + 17179869328u64 => "ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", + 17179869341u64 => "INVALID_DIGEST", + 17179869327u64 => "INVALID_DIGEST_LENGTH", + 17179869321u64 => "INVALID_HEADER", + 17179869344u64 => "INVALID_LABEL", + 17179869315u64 => "INVALID_MESSAGE_LENGTH", + 17179869340u64 => "INVALID_MGF1_MD", + 17179869351u64 => "INVALID_MULTI_PRIME_KEY", + 17179869345u64 => "INVALID_OAEP_PARAMETERS", + 17179869322u64 => "INVALID_PADDING", + 17179869325u64 => "INVALID_PADDING_MODE", + 17179869333u64 => "INVALID_PSS_PARAMETERS", + 17179869330u64 => "INVALID_PSS_SALTLEN", + 17179869334u64 => "INVALID_SALT_LENGTH", + 17179869323u64 => "INVALID_TRAILER", + 17179869326u64 => "INVALID_X931_DIGEST", + 17179869310u64 => "IQMP_NOT_INVERSE_OF_Q", + 17179869349u64 => "KEY_PRIME_NUM_INVALID", + 17179869304u64 => "KEY_SIZE_TOO_SMALL", + 17179869318u64 => "LAST_OCTET_INVALID", + 17179869336u64 => "MGF1_DIGEST_NOT_ALLOWED", + 17179869363u64 => "MISSING_PRIVATE_KEY", + 17179869289u64 => "MODULUS_TOO_LARGE", + 17179869352u64 => "MP_COEFFICIENT_NOT_INVERSE_OF_R", + 17179869353u64 => "MP_EXPONENT_NOT_CONGRUENT_TO_D", + 17179869354u64 => "MP_R_NOT_PRIME", + 17179869324u64 => "NO_PUBLIC_EXPONENT", + 17179869297u64 => "NULL_BEFORE_BLOCK_MISSING", + 17179869356u64 => "N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES", + 17179869311u64 => "N_DOES_NOT_EQUAL_P_Q", + 17179869305u64 => "OAEP_DECODING_ERROR", + 17179869332u64 => "OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", + 17179869298u64 => "PADDING_CHECK_FAILED", + 17179869343u64 => "PKCS_DECODING_ERROR", + 17179869348u64 => "PSS_SALTLEN_TOO_SMALL", + 17179869312u64 => "P_NOT_PRIME", + 17179869313u64 => "Q_NOT_PRIME", + 17179869314u64 => "RSA_OPERATIONS_NOT_SUPPORTED", + 17179869320u64 => "SLEN_CHECK_FAILED", + 17179869319u64 => "SLEN_RECOVERY_FAILED", + 17179869299u64 => "SSLV3_ROLLBACK_ATTACK", + 17179869300u64 => "THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", + 17179869301u64 => "UNKNOWN_ALGORITHM_TYPE", + 17179869350u64 => "UNKNOWN_DIGEST", + 17179869335u64 => "UNKNOWN_MASK_DIGEST", + 17179869302u64 => "UNKNOWN_PADDING_TYPE", + 17179869346u64 => "UNSUPPORTED_ENCRYPTION_TYPE", + 17179869347u64 => "UNSUPPORTED_LABEL_SOURCE", + 17179869337u64 => "UNSUPPORTED_MASK_ALGORITHM", + 17179869338u64 => "UNSUPPORTED_MASK_PARAMETER", + 17179869339u64 => "UNSUPPORTED_SIGNATURE_TYPE", + 17179869331u64 => "VALUE_MISSING", + 17179869303u64 => "WRONG_SIGNATURE_LENGTH", + 227633266788u64 => "ASN1_ERROR", + 227633266789u64 => "BAD_SIGNATURE", + 227633266795u64 => "BUFFER_TOO_SMALL", + 227633266798u64 => "DIST_ID_TOO_LARGE", + 227633266800u64 => "ID_NOT_SET", + 227633266799u64 => "ID_TOO_LARGE", + 227633266796u64 => "INVALID_CURVE", + 227633266790u64 => "INVALID_DIGEST", + 227633266791u64 => "INVALID_DIGEST_TYPE", + 227633266792u64 => "INVALID_ENCODING", + 227633266793u64 => "INVALID_FIELD", + 227633266797u64 => "NO_PARAMETERS_SET", + 227633266794u64 => "USER_ID_TOO_LARGE", + 85899346211u64 => "APPLICATION_DATA_AFTER_CLOSE_NOTIFY", + 85899346020u64 => "APP_DATA_IN_HANDSHAKE", + 85899346192u64 => "ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT", + 85899346063u64 => "AT_LEAST_TLS_1_0_NEEDED_IN_FIPS_MODE", + 85899346078u64 => "AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", + 85899346023u64 => "BAD_CHANGE_CIPHER_SPEC", + 85899346106u64 => "BAD_CIPHER", + 85899346310u64 => "BAD_DATA", + 85899346026u64 => "BAD_DATA_RETURNED_BY_CALLBACK", + 85899346027u64 => "BAD_DECOMPRESSION", + 85899346022u64 => "BAD_DH_VALUE", + 85899346031u64 => "BAD_DIGEST_LENGTH", + 85899346153u64 => "BAD_EARLY_DATA", + 85899346224u64 => "BAD_ECC_CERT", + 85899346226u64 => "BAD_ECPOINT", + 85899346030u64 => "BAD_EXTENSION", + 85899346252u64 => "BAD_HANDSHAKE_LENGTH", + 85899346156u64 => "BAD_HANDSHAKE_STATE", + 85899346025u64 => "BAD_HELLO_REQUEST", + 85899346183u64 => "BAD_HRR_VERSION", + 85899346028u64 => "BAD_KEY_SHARE", + 85899346042u64 => "BAD_KEY_UPDATE", + 85899346212u64 => "BAD_LEGACY_VERSION", + 85899346191u64 => "BAD_LENGTH", + 85899346160u64 => "BAD_PACKET", + 85899346035u64 => "BAD_PACKET_LENGTH", + 85899346036u64 => "BAD_PROTOCOL_VERSION_NUMBER", + 85899346139u64 => "BAD_PSK", + 85899346034u64 => "BAD_PSK_IDENTITY", + 85899346363u64 => "BAD_RECORD_TYPE", + 85899346039u64 => "BAD_RSA_ENCRYPT", + 85899346043u64 => "BAD_SIGNATURE", + 85899346267u64 => "BAD_SRP_A_LENGTH", + 85899346291u64 => "BAD_SRP_PARAMETERS", + 85899346272u64 => "BAD_SRTP_MKI_VALUE", + 85899346273u64 => "BAD_SRTP_PROTECTION_PROFILE_LIST", + 85899346044u64 => "BAD_SSL_FILETYPE", + 85899346304u64 => "BAD_VALUE", + 85899346047u64 => "BAD_WRITE_RETRY", + 85899346173u64 => "BINDER_DOES_NOT_VERIFY", + 85899346048u64 => "BIO_NOT_SET", + 85899346049u64 => "BLOCK_CIPHER_PAD_IS_WRONG", + 85899346050u64 => "BN_LIB", + 85899346154u64 => "CALLBACK_FAILED", + 85899346029u64 => "CANNOT_CHANGE_CIPHER", + 85899346051u64 => "CA_DN_LENGTH_MISMATCH", + 85899346317u64 => "CA_KEY_TOO_SMALL", + 85899346318u64 => "CA_MD_TOO_WEAK", + 85899346053u64 => "CCS_RECEIVED_EARLY", + 85899346054u64 => "CERTIFICATE_VERIFY_FAILED", + 85899346297u64 => "CERT_CB_ERROR", + 85899346055u64 => "CERT_LENGTH_MISMATCH", + 85899346138u64 => "CIPHERSUITE_DIGEST_HAS_CHANGED", + 85899346057u64 => "CIPHER_CODE_WRONG_LENGTH", + 85899346058u64 => "CIPHER_OR_HASH_UNAVAILABLE", + 85899346146u64 => "CLIENTHELLO_TLSEXT", + 85899346060u64 => "COMPRESSED_LENGTH_TOO_LONG", + 85899346263u64 => "COMPRESSION_DISABLED", + 85899346061u64 => "COMPRESSION_FAILURE", + 85899346227u64 => "COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE", + 85899346062u64 => "COMPRESSION_LIBRARY_ERROR", + 85899346064u64 => "CONNECTION_TYPE_NOT_SET", + 85899346087u64 => "CONTEXT_NOT_DANE_ENABLED", + 85899346320u64 => "COOKIE_GEN_CALLBACK_FAILURE", + 85899346228u64 => "COOKIE_MISMATCH", + 85899346126u64 => "CUSTOM_EXT_HANDLER_ALREADY_INSTALLED", + 85899346092u64 => "DANE_ALREADY_ENABLED", + 85899346093u64 => "DANE_CANNOT_OVERRIDE_MTYPE_FULL", + 85899346095u64 => "DANE_NOT_ENABLED", + 85899346100u64 => "DANE_TLSA_BAD_CERTIFICATE", + 85899346104u64 => "DANE_TLSA_BAD_CERTIFICATE_USAGE", + 85899346109u64 => "DANE_TLSA_BAD_DATA_LENGTH", + 85899346112u64 => "DANE_TLSA_BAD_DIGEST_LENGTH", + 85899346120u64 => "DANE_TLSA_BAD_MATCHING_TYPE", + 85899346121u64 => "DANE_TLSA_BAD_PUBLIC_KEY", + 85899346122u64 => "DANE_TLSA_BAD_SELECTOR", + 85899346123u64 => "DANE_TLSA_NULL_DATA", + 85899346065u64 => "DATA_BETWEEN_CCS_AND_FINISHED", + 85899346066u64 => "DATA_LENGTH_TOO_LONG", + 85899346067u64 => "DECRYPTION_FAILED", + 85899346201u64 => "DECRYPTION_FAILED_OR_BAD_RECORD_MAC", + 85899346314u64 => "DH_KEY_TOO_SMALL", + 85899346068u64 => "DH_PUBLIC_VALUE_LENGTH_IS_WRONG", + 85899346069u64 => "DIGEST_CHECK_FAILED", + 85899346254u64 => "DTLS_MESSAGE_TOO_BIG", + 85899346229u64 => "DUPLICATE_COMPRESSION_ID", + 85899346238u64 => "ECC_CERT_NOT_FOR_SIGNING", + 85899346294u64 => "ECDH_REQUIRED_FOR_SUITEB_MODE", + 85899346319u64 => "EE_KEY_TOO_SMALL", + 85899346274u64 => "EMPTY_SRTP_PROTECTION_PROFILE_LIST", + 85899346070u64 => "ENCRYPTED_LENGTH_TOO_LONG", + 85899346071u64 => "ERROR_IN_RECEIVED_CIPHER_LIST", + 85899346124u64 => "ERROR_SETTING_TLSA_BASE_DOMAIN", + 85899346114u64 => "EXCEEDS_MAX_FRAGMENT_SIZE", + 85899346072u64 => "EXCESSIVE_MESSAGE_SIZE", + 85899346199u64 => "EXTENSION_NOT_RECEIVED", + 85899346073u64 => "EXTRA_DATA_IN_MESSAGE", + 85899346083u64 => "EXT_LENGTH_MISMATCH", + 85899346325u64 => "FAILED_TO_INIT_ASYNC", + 85899346321u64 => "FRAGMENTED_CLIENT_HELLO", + 85899346074u64 => "GOT_A_FIN_BEFORE_A_CCS", + 85899346075u64 => "HTTPS_PROXY_REQUEST", + 85899346076u64 => "HTTP_REQUEST", + 85899346082u64 => "ILLEGAL_POINT_COMPRESSION", + 85899346300u64 => "ILLEGAL_SUITEB_DIGEST", + 85899346293u64 => "INAPPROPRIATE_FALLBACK", + 85899346260u64 => "INCONSISTENT_COMPRESSION", + 85899346142u64 => "INCONSISTENT_EARLY_DATA_ALPN", + 85899346151u64 => "INCONSISTENT_EARLY_DATA_SNI", + 85899346024u64 => "INCONSISTENT_EXTMS", + 85899346161u64 => "INSUFFICIENT_SECURITY", + 85899346125u64 => "INVALID_ALERT", + 85899346180u64 => "INVALID_CCS_MESSAGE", + 85899346158u64 => "INVALID_CERTIFICATE_OR_ALG", + 85899346200u64 => "INVALID_COMMAND", + 85899346261u64 => "INVALID_COMPRESSION_ALGORITHM", + 85899346203u64 => "INVALID_CONFIG", + 85899346033u64 => "INVALID_CONFIGURATION_NAME", + 85899346202u64 => "INVALID_CONTEXT", + 85899346132u64 => "INVALID_CT_VALIDATION_TYPE", + 85899346040u64 => "INVALID_KEY_UPDATE_TYPE", + 85899346094u64 => "INVALID_MAX_EARLY_DATA", + 85899346305u64 => "INVALID_NULL_CMD_NAME", + 85899346322u64 => "INVALID_SEQUENCE_NUMBER", + 85899346308u64 => "INVALID_SERVERINFO_DATA", + 85899346919u64 => "INVALID_SESSION_ID", + 85899346277u64 => "INVALID_SRP_USERNAME", + 85899346248u64 => "INVALID_STATUS_RESPONSE", + 85899346245u64 => "INVALID_TICKET_KEYS_LENGTH", + 85899346079u64 => "LENGTH_MISMATCH", + 85899346324u64 => "LENGTH_TOO_LONG", + 85899346080u64 => "LENGTH_TOO_SHORT", + 85899346194u64 => "LIBRARY_BUG", + 85899346081u64 => "LIBRARY_HAS_NO_CIPHERS", + 85899346085u64 => "MISSING_DSA_SIGNING_CERT", + 85899346301u64 => "MISSING_ECDSA_SIGNING_CERT", + 85899346176u64 => "MISSING_FATAL", + 85899346210u64 => "MISSING_PARAMETERS", + 85899346230u64 => "MISSING_PSK_KEX_MODES_EXTENSION", + 85899346088u64 => "MISSING_RSA_CERTIFICATE", + 85899346089u64 => "MISSING_RSA_ENCRYPTING_CERT", + 85899346090u64 => "MISSING_RSA_SIGNING_CERT", + 85899346032u64 => "MISSING_SIGALGS_EXTENSION", + 85899346141u64 => "MISSING_SIGNING_CERT", + 85899346278u64 => "MISSING_SRP_PARAM", + 85899346129u64 => "MISSING_SUPPORTED_GROUPS_EXTENSION", + 85899346091u64 => "MISSING_TMP_DH_KEY", + 85899346231u64 => "MISSING_TMP_ECDH_KEY", + 85899346213u64 => "MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA", + 85899346102u64 => "NOT_ON_RECORD_BOUNDARY", + 85899346209u64 => "NOT_REPLACING_CERTIFICATE", + 85899346204u64 => "NOT_SERVER", + 85899346155u64 => "NO_APPLICATION_PROTOCOL", + 85899346096u64 => "NO_CERTIFICATES_RETURNED", + 85899346097u64 => "NO_CERTIFICATE_ASSIGNED", + 85899346099u64 => "NO_CERTIFICATE_SET", + 85899346134u64 => "NO_CHANGE_FOLLOWING_HRR", + 85899346101u64 => "NO_CIPHERS_AVAILABLE", + 85899346103u64 => "NO_CIPHERS_SPECIFIED", + 85899346105u64 => "NO_CIPHER_MATCH", + 85899346251u64 => "NO_CLIENT_CERT_METHOD", + 85899346107u64 => "NO_COMPRESSION_SPECIFIED", + 85899346207u64 => "NO_COOKIE_CALLBACK_SET", + 85899346250u64 => "NO_GOST_CERTIFICATE_SENT_BY_PEER", + 85899346108u64 => "NO_METHOD_SPECIFIED", + 85899346309u64 => "NO_PEM_EXTENSIONS", + 85899346110u64 => "NO_PRIVATE_KEY_ASSIGNED", + 85899346111u64 => "NO_PROTOCOLS_AVAILABLE", + 85899346259u64 => "NO_RENEGOTIATION", + 85899346244u64 => "NO_REQUIRED_DIGEST", + 85899346113u64 => "NO_SHARED_CIPHER", + 85899346330u64 => "NO_SHARED_GROUPS", + 85899346296u64 => "NO_SHARED_SIGNATURE_ALGORITHMS", + 85899346279u64 => "NO_SRTP_PROFILES", + 85899346021u64 => "NO_SUITABLE_KEY_SHARE", + 85899346038u64 => "NO_SUITABLE_SIGNATURE_ALGORITHM", + 85899346136u64 => "NO_VALID_SCTS", + 85899346323u64 => "NO_VERIFY_COOKIE_CALLBACK", + 85899346115u64 => "NULL_SSL_CTX", + 85899346116u64 => "NULL_SSL_METHOD_PASSED", + 85899346214u64 => "OCSP_CALLBACK_FAILURE", + 85899346117u64 => "OLD_SESSION_CIPHER_NOT_RETURNED", + 85899346264u64 => "OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED", + 85899346157u64 => "OVERFLOW_ERROR", + 85899346118u64 => "PACKET_LENGTH_TOO_LONG", + 85899346147u64 => "PARSE_TLSEXT", + 85899346190u64 => "PATH_TOO_LONG", + 85899346119u64 => "PEER_DID_NOT_RETURN_A_CERTIFICATE", + 85899346311u64 => "PEM_NAME_BAD_PREFIX", + 85899346312u64 => "PEM_NAME_TOO_SHORT", + 85899346326u64 => "PIPELINE_FAILURE", + 85899346198u64 => "POST_HANDSHAKE_AUTH_ENCODING_ERR", + 85899346208u64 => "PRIVATE_KEY_MISMATCH", + 85899346127u64 => "PROTOCOL_IS_SHUTDOWN", + 85899346143u64 => "PSK_IDENTITY_NOT_FOUND", + 85899346144u64 => "PSK_NO_CLIENT_CB", + 85899346145u64 => "PSK_NO_SERVER_CB", + 85899346131u64 => "READ_BIO_NOT_SET", + 85899346232u64 => "READ_TIMEOUT_EXPIRED", + 85899346133u64 => "RECORD_LENGTH_MISMATCH", + 85899346218u64 => "RECORD_TOO_SMALL", + 85899346255u64 => "RENEGOTIATE_EXT_TOO_LONG", + 85899346256u64 => "RENEGOTIATION_ENCODING_ERR", + 85899346257u64 => "RENEGOTIATION_MISMATCH", + 85899346205u64 => "REQUEST_PENDING", + 85899346206u64 => "REQUEST_SENT", + 85899346135u64 => "REQUIRED_CIPHER_MISSING", + 85899346262u64 => "REQUIRED_COMPRESSION_ALGORITHM_MISSING", + 85899346265u64 => "SCSV_RECEIVED_WHEN_RENEGOTIATING", + 85899346128u64 => "SCT_VERIFICATION_FAILED", + 85899346195u64 => "SERVERHELLO_TLSEXT", + 85899346197u64 => "SESSION_ID_CONTEXT_UNINITIALIZED", + 85899346327u64 => "SHUTDOWN_WHILE_IN_INIT", + 85899346280u64 => "SIGNATURE_ALGORITHMS_ERROR", + 85899346140u64 => "SIGNATURE_FOR_NON_SIGNING_CERTIFICATE", + 85899346281u64 => "SRP_A_CALC", + 85899346282u64 => "SRTP_COULD_NOT_ALLOCATE_PROFILES", + 85899346283u64 => "SRTP_PROTECTION_PROFILE_LIST_TOO_LONG", + 85899346284u64 => "SRTP_UNKNOWN_PROTECTION_PROFILE", + 85899346152u64 => "SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH", + 85899346239u64 => "SSL3_EXT_INVALID_SERVERNAME", + 85899346240u64 => "SSL3_EXT_INVALID_SERVERNAME_TYPE", + 85899346220u64 => "SSL3_SESSION_ID_TOO_LONG", + 85899346962u64 => "SSLV3_ALERT_BAD_CERTIFICATE", + 85899346940u64 => "SSLV3_ALERT_BAD_RECORD_MAC", + 85899346965u64 => "SSLV3_ALERT_CERTIFICATE_EXPIRED", + 85899346964u64 => "SSLV3_ALERT_CERTIFICATE_REVOKED", + 85899346966u64 => "SSLV3_ALERT_CERTIFICATE_UNKNOWN", + 85899346950u64 => "SSLV3_ALERT_DECOMPRESSION_FAILURE", + 85899346960u64 => "SSLV3_ALERT_HANDSHAKE_FAILURE", + 85899346967u64 => "SSLV3_ALERT_ILLEGAL_PARAMETER", + 85899346961u64 => "SSLV3_ALERT_NO_CERTIFICATE", + 85899346930u64 => "SSLV3_ALERT_UNEXPECTED_MESSAGE", + 85899346963u64 => "SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", + 85899346037u64 => "SSL_COMMAND_SECTION_EMPTY", + 85899346045u64 => "SSL_COMMAND_SECTION_NOT_FOUND", + 85899346148u64 => "SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION", + 85899346149u64 => "SSL_HANDSHAKE_FAILURE", + 85899346150u64 => "SSL_LIBRARY_HAS_NO_CIPHERS", + 85899346292u64 => "SSL_NEGATIVE_LENGTH", + 85899346046u64 => "SSL_SECTION_EMPTY", + 85899346056u64 => "SSL_SECTION_NOT_FOUND", + 85899346221u64 => "SSL_SESSION_ID_CALLBACK_FAILED", + 85899346222u64 => "SSL_SESSION_ID_CONFLICT", + 85899346193u64 => "SSL_SESSION_ID_CONTEXT_TOO_LONG", + 85899346223u64 => "SSL_SESSION_ID_HAS_BAD_LENGTH", + 85899346328u64 => "SSL_SESSION_ID_TOO_LONG", + 85899346130u64 => "SSL_SESSION_VERSION_MISMATCH", + 85899346041u64 => "STILL_IN_INIT", + 85899347036u64 => "TLSV13_ALERT_CERTIFICATE_REQUIRED", + 85899347029u64 => "TLSV13_ALERT_MISSING_EXTENSION", + 85899346969u64 => "TLSV1_ALERT_ACCESS_DENIED", + 85899346970u64 => "TLSV1_ALERT_DECODE_ERROR", + 85899346941u64 => "TLSV1_ALERT_DECRYPTION_FAILED", + 85899346971u64 => "TLSV1_ALERT_DECRYPT_ERROR", + 85899346980u64 => "TLSV1_ALERT_EXPORT_RESTRICTION", + 85899347006u64 => "TLSV1_ALERT_INAPPROPRIATE_FALLBACK", + 85899346991u64 => "TLSV1_ALERT_INSUFFICIENT_SECURITY", + 85899347000u64 => "TLSV1_ALERT_INTERNAL_ERROR", + 85899347020u64 => "TLSV1_ALERT_NO_RENEGOTIATION", + 85899346990u64 => "TLSV1_ALERT_PROTOCOL_VERSION", + 85899346942u64 => "TLSV1_ALERT_RECORD_OVERFLOW", + 85899346968u64 => "TLSV1_ALERT_UNKNOWN_CA", + 85899347010u64 => "TLSV1_ALERT_USER_CANCELLED", + 85899347034u64 => "TLSV1_BAD_CERTIFICATE_HASH_VALUE", + 85899347033u64 => "TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", + 85899347031u64 => "TLSV1_CERTIFICATE_UNOBTAINABLE", + 85899347032u64 => "TLSV1_UNRECOGNIZED_NAME", + 85899347030u64 => "TLSV1_UNSUPPORTED_EXTENSION", + 85899346285u64 => "TLS_HEARTBEAT_PEER_DOESNT_ACCEPT", + 85899346286u64 => "TLS_HEARTBEAT_PENDING", + 85899346287u64 => "TLS_ILLEGAL_EXPORTER_LABEL", + 85899346077u64 => "TLS_INVALID_ECPOINTFORMAT_LIST", + 85899346052u64 => "TOO_MANY_KEY_UPDATES", + 85899346329u64 => "TOO_MANY_WARN_ALERTS", + 85899346084u64 => "TOO_MUCH_EARLY_DATA", + 85899346234u64 => "UNABLE_TO_FIND_ECDH_PARAMETERS", + 85899346159u64 => "UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS", + 85899346162u64 => "UNABLE_TO_LOAD_SSL3_MD5_ROUTINES", + 85899346163u64 => "UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES", + 85899346182u64 => "UNEXPECTED_CCS_MESSAGE", + 85899346098u64 => "UNEXPECTED_END_OF_EARLY_DATA", + 85899346164u64 => "UNEXPECTED_MESSAGE", + 85899346165u64 => "UNEXPECTED_RECORD", + 85899346196u64 => "UNINITIALIZED", + 85899346166u64 => "UNKNOWN_ALERT_TYPE", + 85899346167u64 => "UNKNOWN_CERTIFICATE_TYPE", + 85899346168u64 => "UNKNOWN_CIPHER_RETURNED", + 85899346169u64 => "UNKNOWN_CIPHER_TYPE", + 85899346306u64 => "UNKNOWN_CMD_NAME", + 85899346059u64 => "UNKNOWN_COMMAND", + 85899346288u64 => "UNKNOWN_DIGEST", + 85899346170u64 => "UNKNOWN_KEY_EXCHANGE_TYPE", + 85899346171u64 => "UNKNOWN_PKEY_TYPE", + 85899346172u64 => "UNKNOWN_PROTOCOL", + 85899346174u64 => "UNKNOWN_SSL_VERSION", + 85899346175u64 => "UNKNOWN_STATE", + 85899346258u64 => "UNSAFE_LEGACY_RENEGOTIATION_DISABLED", + 85899346137u64 => "UNSOLICITED_EXTENSION", + 85899346177u64 => "UNSUPPORTED_COMPRESSION_ALGORITHM", + 85899346235u64 => "UNSUPPORTED_ELLIPTIC_CURVE", + 85899346178u64 => "UNSUPPORTED_PROTOCOL", + 85899346179u64 => "UNSUPPORTED_SSL_VERSION", + 85899346249u64 => "UNSUPPORTED_STATUS_TYPE", + 85899346289u64 => "USE_SRTP_NOT_NEGOTIATED", + 85899346086u64 => "VERSION_TOO_HIGH", + 85899346316u64 => "VERSION_TOO_LOW", + 85899346303u64 => "WRONG_CERTIFICATE_TYPE", + 85899346181u64 => "WRONG_CIPHER_RETURNED", + 85899346298u64 => "WRONG_CURVE", + 85899346184u64 => "WRONG_SIGNATURE_LENGTH", + 85899346185u64 => "WRONG_SIGNATURE_SIZE", + 85899346290u64 => "WRONG_SIGNATURE_TYPE", + 85899346186u64 => "WRONG_SSL_VERSION", + 85899346187u64 => "WRONG_VERSION_NUMBER", + 85899346188u64 => "X509_LIB", + 85899346189u64 => "X509_VERIFICATION_SETUP_PROBLEMS", + 201863463044u64 => "BAD_PKCS7_TYPE", + 201863463045u64 => "BAD_TYPE", + 201863463049u64 => "CANNOT_LOAD_CERT", + 201863463050u64 => "CANNOT_LOAD_KEY", + 201863463012u64 => "CERTIFICATE_VERIFY_ERROR", + 201863463039u64 => "COULD_NOT_SET_ENGINE", + 201863463027u64 => "COULD_NOT_SET_TIME", + 201863463046u64 => "DETACHED_CONTENT", + 201863463028u64 => "ESS_ADD_SIGNING_CERT_ERROR", + 201863463051u64 => "ESS_ADD_SIGNING_CERT_V2_ERROR", + 201863463013u64 => "ESS_SIGNING_CERTIFICATE_ERROR", + 201863463014u64 => "INVALID_NULL_POINTER", + 201863463029u64 => "INVALID_SIGNER_CERTIFICATE_PURPOSE", + 201863463015u64 => "MESSAGE_IMPRINT_MISMATCH", + 201863463016u64 => "NONCE_MISMATCH", + 201863463017u64 => "NONCE_NOT_RETURNED", + 201863463018u64 => "NO_CONTENT", + 201863463019u64 => "NO_TIME_STAMP_TOKEN", + 201863463030u64 => "PKCS7_ADD_SIGNATURE_ERROR", + 201863463031u64 => "PKCS7_ADD_SIGNED_ATTR_ERROR", + 201863463041u64 => "PKCS7_TO_TS_TST_INFO_FAILED", + 201863463020u64 => "POLICY_MISMATCH", + 201863463032u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 201863463033u64 => "RESPONSE_SETUP_ERROR", + 201863463021u64 => "SIGNATURE_FAILURE", + 201863463022u64 => "THERE_MUST_BE_ONE_SIGNER", + 201863463034u64 => "TIME_SYSCALL_ERROR", + 201863463042u64 => "TOKEN_NOT_PRESENT", + 201863463043u64 => "TOKEN_PRESENT", + 201863463023u64 => "TSA_NAME_MISMATCH", + 201863463024u64 => "TSA_UNTRUSTED", + 201863463035u64 => "TST_INFO_SETUP_ERROR", + 201863463036u64 => "TS_DATASIGN", + 201863463037u64 => "UNACCEPTABLE_POLICY", + 201863463038u64 => "UNSUPPORTED_MD_ALGORITHM", + 201863463025u64 => "UNSUPPORTED_VERSION", + 201863463047u64 => "VAR_BAD_VALUE", + 201863463048u64 => "VAR_LOOKUP_FAILURE", + 201863463026u64 => "WRONG_CONTENT_TYPE", + 171798691944u64 => "COMMON_OK_AND_CANCEL_CHARACTERS", + 171798691942u64 => "INDEX_TOO_LARGE", + 171798691943u64 => "INDEX_TOO_SMALL", + 171798691945u64 => "NO_RESULT_BUFFER", + 171798691947u64 => "PROCESSING_ERROR", + 171798691940u64 => "RESULT_TOO_LARGE", + 171798691941u64 => "RESULT_TOO_SMALL", + 171798691949u64 => "SYSASSIGN_ERROR", + 171798691950u64 => "SYSDASSGN_ERROR", + 171798691951u64 => "SYSQIOW_ERROR", + 171798691946u64 => "UNKNOWN_CONTROL_COMMAND", + 171798691948u64 => "UNKNOWN_TTYGET_ERRNO_VALUE", + 171798691952u64 => "USER_DATA_DUPLICATION_UNSUPPORTED", + 146028888182u64 => "BAD_IP_ADDRESS", + 146028888183u64 => "BAD_OBJECT", + 146028888164u64 => "BN_DEC2BN_ERROR", + 146028888165u64 => "BN_TO_ASN1_INTEGER_ERROR", + 146028888213u64 => "DIRNAME_ERROR", + 146028888224u64 => "DISTPOINT_ALREADY_SET", + 146028888197u64 => "DUPLICATE_ZONE_ID", + 146028888195u64 => "ERROR_CONVERTING_ZONE", + 146028888208u64 => "ERROR_CREATING_EXTENSION", + 146028888192u64 => "ERROR_IN_EXTENSION", + 146028888201u64 => "EXPECTED_A_SECTION_NAME", + 146028888209u64 => "EXTENSION_EXISTS", + 146028888179u64 => "EXTENSION_NAME_ERROR", + 146028888166u64 => "EXTENSION_NOT_FOUND", + 146028888167u64 => "EXTENSION_SETTING_NOT_SUPPORTED", + 146028888180u64 => "EXTENSION_VALUE_ERROR", + 146028888215u64 => "ILLEGAL_EMPTY_EXTENSION", + 146028888216u64 => "INCORRECT_POLICY_SYNTAX_TAG", + 146028888226u64 => "INVALID_ASNUMBER", + 146028888227u64 => "INVALID_ASRANGE", + 146028888168u64 => "INVALID_BOOLEAN_STRING", + 146028888169u64 => "INVALID_EXTENSION_STRING", + 146028888229u64 => "INVALID_INHERITANCE", + 146028888230u64 => "INVALID_IPADDRESS", + 146028888225u64 => "INVALID_MULTIPLE_RDNS", + 146028888170u64 => "INVALID_NAME", + 146028888171u64 => "INVALID_NULL_ARGUMENT", + 146028888172u64 => "INVALID_NULL_NAME", + 146028888173u64 => "INVALID_NULL_VALUE", + 146028888204u64 => "INVALID_NUMBER", + 146028888205u64 => "INVALID_NUMBERS", + 146028888174u64 => "INVALID_OBJECT_IDENTIFIER", + 146028888202u64 => "INVALID_OPTION", + 146028888198u64 => "INVALID_POLICY_IDENTIFIER", + 146028888217u64 => "INVALID_PROXY_POLICY_SETTING", + 146028888210u64 => "INVALID_PURPOSE", + 146028888228u64 => "INVALID_SAFI", + 146028888199u64 => "INVALID_SECTION", + 146028888207u64 => "INVALID_SYNTAX", + 146028888190u64 => "ISSUER_DECODE_ERROR", + 146028888188u64 => "MISSING_VALUE", + 146028888206u64 => "NEED_ORGANIZATION_AND_NUMBERS", + 146028888200u64 => "NO_CONFIG_DATABASE", + 146028888185u64 => "NO_ISSUER_CERTIFICATE", + 146028888191u64 => "NO_ISSUER_DETAILS", + 146028888203u64 => "NO_POLICY_IDENTIFIER", + 146028888218u64 => "NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED", + 146028888178u64 => "NO_PUBLIC_KEY", + 146028888189u64 => "NO_SUBJECT_DETAILS", + 146028888212u64 => "OPERATION_NOT_DEFINED", + 146028888211u64 => "OTHERNAME_ERROR", + 146028888219u64 => "POLICY_LANGUAGE_ALREADY_DEFINED", + 146028888220u64 => "POLICY_PATH_LENGTH", + 146028888221u64 => "POLICY_PATH_LENGTH_ALREADY_DEFINED", + 146028888223u64 => "POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", + 146028888214u64 => "SECTION_NOT_FOUND", + 146028888186u64 => "UNABLE_TO_GET_ISSUER_DETAILS", + 146028888187u64 => "UNABLE_TO_GET_ISSUER_KEYID", + 146028888175u64 => "UNKNOWN_BIT_STRING_ARGUMENT", + 146028888193u64 => "UNKNOWN_EXTENSION", + 146028888194u64 => "UNKNOWN_EXTENSION_NAME", + 146028888184u64 => "UNKNOWN_OPTION", + 146028888181u64 => "UNSUPPORTED_OPTION", + 146028888231u64 => "UNSUPPORTED_TYPE", + 146028888196u64 => "USER_TOO_LONG", + 47244640366u64 => "AKID_MISMATCH", + 47244640389u64 => "BAD_SELECTOR", + 47244640356u64 => "BAD_X509_FILETYPE", + 47244640374u64 => "BASE64_DECODE_ERROR", + 47244640370u64 => "CANT_CHECK_DH_KEY", + 47244640357u64 => "CERT_ALREADY_IN_HASH_TABLE", + 47244640383u64 => "CRL_ALREADY_DELTA", + 47244640387u64 => "CRL_VERIFY_FAILURE", + 47244640384u64 => "IDP_MISMATCH", + 47244640394u64 => "INVALID_ATTRIBUTES", + 47244640369u64 => "INVALID_DIRECTORY", + 47244640375u64 => "INVALID_FIELD_NAME", + 47244640379u64 => "INVALID_TRUST", + 47244640385u64 => "ISSUER_MISMATCH", + 47244640371u64 => "KEY_TYPE_MISMATCH", + 47244640372u64 => "KEY_VALUES_MISMATCH", + 47244640359u64 => "LOADING_CERT_DIR", + 47244640360u64 => "LOADING_DEFAULTS", + 47244640380u64 => "METHOD_NOT_SUPPORTED", + 47244640390u64 => "NAME_TOO_LONG", + 47244640388u64 => "NEWER_CRL_NOT_NEWER", + 47244640391u64 => "NO_CERTIFICATE_FOUND", + 47244640392u64 => "NO_CERTIFICATE_OR_CRL_FOUND", + 47244640361u64 => "NO_CERT_SET_FOR_US_TO_VERIFY", + 47244640393u64 => "NO_CRL_FOUND", + 47244640386u64 => "NO_CRL_NUMBER", + 47244640381u64 => "PUBLIC_KEY_DECODE_ERROR", + 47244640382u64 => "PUBLIC_KEY_ENCODE_ERROR", + 47244640362u64 => "SHOULD_RETRY", + 47244640363u64 => "UNABLE_TO_FIND_PARAMETERS_IN_CHAIN", + 47244640364u64 => "UNABLE_TO_GET_CERTS_PUBLIC_KEY", + 47244640373u64 => "UNKNOWN_KEY_TYPE", + 47244640365u64 => "UNKNOWN_NID", + 47244640377u64 => "UNKNOWN_PURPOSE_ID", + 47244640376u64 => "UNKNOWN_TRUST_ID", + 47244640367u64 => "UNSUPPORTED_ALGORITHM", + 47244640368u64 => "WRONG_LOOKUP_TYPE", + 47244640378u64 => "WRONG_TYPE", +}; + +/// Helper function to create encoded key from (lib, reason) pair +#[inline] +pub fn encode_error_key(lib: i32, reason: i32) -> u64 { + ((lib as u64) << 32) | (reason as u64 & 0xFFFFFFFF) +} diff --git a/crates/stdlib/src/openssl/ssl_data_300.rs b/crates/stdlib/src/openssl/ssl_data_300.rs new file mode 100644 index 00000000000..0f657eb1b01 --- /dev/null +++ b/crates/stdlib/src/openssl/ssl_data_300.rs @@ -0,0 +1,1759 @@ +// File generated by tools/make_ssl_data_rs.py +// Generated on 2025-10-29T07:17:23.737586+00:00 +// Source: OpenSSL from /tmp/openssl-3.0 +// spell-checker: disable + +use phf::phf_map; + +// Maps lib_code -> library name +// Example: 20 -> "SSL" +pub static LIBRARY_CODES: phf::Map<u32, &'static str> = phf_map! { + 0u32 => "MASK", + 1u32 => "NONE", + 2u32 => "SYS", + 3u32 => "BN", + 4u32 => "RSA", + 5u32 => "DH", + 6u32 => "EVP", + 7u32 => "BUF", + 8u32 => "OBJ", + 9u32 => "PEM", + 10u32 => "DSA", + 11u32 => "X509", + 12u32 => "METH", + 13u32 => "ASN1", + 14u32 => "CONF", + 15u32 => "CRYPTO", + 16u32 => "EC", + 20u32 => "SSL", + 21u32 => "SSL23", + 22u32 => "SSL2", + 23u32 => "SSL3", + 30u32 => "RSAREF", + 31u32 => "PROXY", + 32u32 => "BIO", + 33u32 => "PKCS7", + 34u32 => "X509V3", + 35u32 => "PKCS12", + 36u32 => "RAND", + 37u32 => "DSO", + 38u32 => "ENGINE", + 39u32 => "OCSP", + 40u32 => "UI", + 41u32 => "COMP", + 42u32 => "ECDSA", + 43u32 => "ECDH", + 44u32 => "OSSL_STORE", + 45u32 => "FIPS", + 46u32 => "CMS", + 47u32 => "TS", + 48u32 => "HMAC", + 49u32 => "JPAKE", + 50u32 => "CT", + 51u32 => "ASYNC", + 52u32 => "KDF", + 53u32 => "SM2", + 54u32 => "ESS", + 55u32 => "PROP", + 56u32 => "CRMF", + 57u32 => "PROV", + 58u32 => "CMP", + 59u32 => "OSSL_ENCODER", + 60u32 => "OSSL_DECODER", + 61u32 => "HTTP", + 128u32 => "USER", +}; + +// Maps encoded (lib, reason) -> error mnemonic +// Example: encode_error_key(20, 134) -> "CERTIFICATE_VERIFY_FAILED" +// Key encoding: (lib << 32) | reason +pub static ERROR_CODES: phf::Map<u64, &'static str> = phf_map! { + 55834575019u64 => "ADDING_OBJECT", + 55834575051u64 => "ASN1_PARSE_ERROR", + 55834575052u64 => "ASN1_SIG_PARSE_ERROR", + 55834574948u64 => "AUX_ERROR", + 55834574950u64 => "BAD_OBJECT_HEADER", + 55834575078u64 => "BAD_TEMPLATE", + 55834575062u64 => "BMPSTRING_IS_WRONG_LENGTH", + 55834574953u64 => "BN_LIB", + 55834574954u64 => "BOOLEAN_IS_WRONG_LENGTH", + 55834574955u64 => "BUFFER_TOO_SMALL", + 55834574956u64 => "CIPHER_HAS_NO_OBJECT_IDENTIFIER", + 55834575065u64 => "CONTEXT_NOT_INITIALISED", + 55834574957u64 => "DATA_IS_WRONG", + 55834574958u64 => "DECODE_ERROR", + 55834575022u64 => "DEPTH_EXCEEDED", + 55834575046u64 => "DIGEST_AND_KEY_TYPE_NOT_SUPPORTED", + 55834574960u64 => "ENCODE_ERROR", + 55834575021u64 => "ERROR_GETTING_TIME", + 55834575020u64 => "ERROR_LOADING_SECTION", + 55834574962u64 => "ERROR_SETTING_CIPHER_PARAMS", + 55834574963u64 => "EXPECTING_AN_INTEGER", + 55834574964u64 => "EXPECTING_AN_OBJECT", + 55834574967u64 => "EXPLICIT_LENGTH_MISMATCH", + 55834574968u64 => "EXPLICIT_TAG_NOT_CONSTRUCTED", + 55834574969u64 => "FIELD_MISSING", + 55834574970u64 => "FIRST_NUM_TOO_LARGE", + 55834574971u64 => "HEADER_TOO_LONG", + 55834575023u64 => "ILLEGAL_BITSTRING_FORMAT", + 55834575024u64 => "ILLEGAL_BOOLEAN", + 55834574972u64 => "ILLEGAL_CHARACTERS", + 55834575025u64 => "ILLEGAL_FORMAT", + 55834575026u64 => "ILLEGAL_HEX", + 55834575027u64 => "ILLEGAL_IMPLICIT_TAG", + 55834575028u64 => "ILLEGAL_INTEGER", + 55834575074u64 => "ILLEGAL_NEGATIVE_VALUE", + 55834575029u64 => "ILLEGAL_NESTED_TAGGING", + 55834574973u64 => "ILLEGAL_NULL", + 55834575030u64 => "ILLEGAL_NULL_VALUE", + 55834575031u64 => "ILLEGAL_OBJECT", + 55834574974u64 => "ILLEGAL_OPTIONAL_ANY", + 55834575018u64 => "ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE", + 55834575069u64 => "ILLEGAL_PADDING", + 55834574975u64 => "ILLEGAL_TAGGED_ANY", + 55834575032u64 => "ILLEGAL_TIME_VALUE", + 55834575070u64 => "ILLEGAL_ZERO_CONTENT", + 55834575033u64 => "INTEGER_NOT_ASCII_FORMAT", + 55834574976u64 => "INTEGER_TOO_LARGE_FOR_LONG", + 55834575068u64 => "INVALID_BIT_STRING_BITS_LEFT", + 55834574977u64 => "INVALID_BMPSTRING_LENGTH", + 55834574978u64 => "INVALID_DIGIT", + 55834575053u64 => "INVALID_MIME_TYPE", + 55834575034u64 => "INVALID_MODIFIER", + 55834575035u64 => "INVALID_NUMBER", + 55834575064u64 => "INVALID_OBJECT_ENCODING", + 55834575075u64 => "INVALID_SCRYPT_PARAMETERS", + 55834574979u64 => "INVALID_SEPARATOR", + 55834575066u64 => "INVALID_STRING_TABLE_VALUE", + 55834574981u64 => "INVALID_UNIVERSALSTRING_LENGTH", + 55834574982u64 => "INVALID_UTF8STRING", + 55834575067u64 => "INVALID_VALUE", + 55834575079u64 => "LENGTH_TOO_LONG", + 55834575036u64 => "LIST_ERROR", + 55834575054u64 => "MIME_NO_CONTENT_TYPE", + 55834575055u64 => "MIME_PARSE_ERROR", + 55834575056u64 => "MIME_SIG_PARSE_ERROR", + 55834574985u64 => "MISSING_EOC", + 55834574986u64 => "MISSING_SECOND_NUMBER", + 55834575037u64 => "MISSING_VALUE", + 55834574987u64 => "MSTRING_NOT_UNIVERSAL", + 55834574988u64 => "MSTRING_WRONG_TAG", + 55834575045u64 => "NESTED_ASN1_STRING", + 55834575049u64 => "NESTED_TOO_DEEP", + 55834574989u64 => "NON_HEX_CHARACTERS", + 55834575038u64 => "NOT_ASCII_FORMAT", + 55834574990u64 => "NOT_ENOUGH_DATA", + 55834575057u64 => "NO_CONTENT_TYPE", + 55834574991u64 => "NO_MATCHING_CHOICE_TYPE", + 55834575058u64 => "NO_MULTIPART_BODY_FAILURE", + 55834575059u64 => "NO_MULTIPART_BOUNDARY", + 55834575060u64 => "NO_SIG_CONTENT_TYPE", + 55834574992u64 => "NULL_IS_WRONG_LENGTH", + 55834575039u64 => "OBJECT_NOT_ASCII_FORMAT", + 55834574993u64 => "ODD_NUMBER_OF_CHARS", + 55834574995u64 => "SECOND_NUMBER_TOO_LARGE", + 55834574996u64 => "SEQUENCE_LENGTH_MISMATCH", + 55834574997u64 => "SEQUENCE_NOT_CONSTRUCTED", + 55834575040u64 => "SEQUENCE_OR_SET_NEEDS_CONFIG", + 55834574998u64 => "SHORT_LINE", + 55834575061u64 => "SIG_INVALID_MIME_TYPE", + 55834575050u64 => "STREAMING_NOT_SUPPORTED", + 55834574999u64 => "STRING_TOO_LONG", + 55834575000u64 => "STRING_TOO_SHORT", + 55834575002u64 => "THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", + 55834575041u64 => "TIME_NOT_ASCII_FORMAT", + 55834575071u64 => "TOO_LARGE", + 55834575003u64 => "TOO_LONG", + 55834575072u64 => "TOO_SMALL", + 55834575004u64 => "TYPE_NOT_CONSTRUCTED", + 55834575043u64 => "TYPE_NOT_PRIMITIVE", + 55834575007u64 => "UNEXPECTED_EOC", + 55834575063u64 => "UNIVERSALSTRING_IS_WRONG_LENGTH", + 55834575077u64 => "UNKNOWN_DIGEST", + 55834575008u64 => "UNKNOWN_FORMAT", + 55834575009u64 => "UNKNOWN_MESSAGE_DIGEST_ALGORITHM", + 55834575010u64 => "UNKNOWN_OBJECT_TYPE", + 55834575011u64 => "UNKNOWN_PUBLIC_KEY_TYPE", + 55834575047u64 => "UNKNOWN_SIGNATURE_ALGORITHM", + 55834575042u64 => "UNKNOWN_TAG", + 55834575012u64 => "UNSUPPORTED_ANY_DEFINED_BY_TYPE", + 55834575076u64 => "UNSUPPORTED_CIPHER", + 55834575015u64 => "UNSUPPORTED_PUBLIC_KEY_TYPE", + 55834575044u64 => "UNSUPPORTED_TYPE", + 55834575073u64 => "WRONG_INTEGER_TYPE", + 55834575048u64 => "WRONG_PUBLIC_KEY_TYPE", + 55834575016u64 => "WRONG_TAG", + 219043332197u64 => "FAILED_TO_SET_POOL", + 219043332198u64 => "FAILED_TO_SWAP_CONTEXT", + 219043332201u64 => "INIT_FAILED", + 219043332199u64 => "INVALID_POOL_SIZE", + 137438953572u64 => "ACCEPT_ERROR", + 137438953613u64 => "ADDRINFO_ADDR_IS_NOT_AF_INET", + 137438953601u64 => "AMBIGUOUS_HOST_OR_SERVICE", + 137438953573u64 => "BAD_FOPEN_MODE", + 137438953596u64 => "BROKEN_PIPE", + 137438953575u64 => "CONNECT_ERROR", + 137438953619u64 => "CONNECT_TIMEOUT", + 137438953579u64 => "GETHOSTBYNAME_ADDR_IS_NOT_AF_INET", + 137438953604u64 => "GETSOCKNAME_ERROR", + 137438953605u64 => "GETSOCKNAME_TRUNCATED_ADDRESS", + 137438953606u64 => "GETTING_SOCKTYPE", + 137438953597u64 => "INVALID_ARGUMENT", + 137438953607u64 => "INVALID_SOCKET", + 137438953595u64 => "IN_USE", + 137438953574u64 => "LENGTH_TOO_LONG", + 137438953608u64 => "LISTEN_V6_ONLY", + 137438953614u64 => "LOOKUP_RETURNED_NOTHING", + 137438953602u64 => "MALFORMED_HOST_OR_SERVICE", + 137438953582u64 => "NBIO_CONNECT_ERROR", + 137438953615u64 => "NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", + 137438953616u64 => "NO_HOSTNAME_OR_SERVICE_SPECIFIED", + 137438953585u64 => "NO_PORT_DEFINED", + 137438953600u64 => "NO_SUCH_FILE", + 137438953576u64 => "TRANSFER_ERROR", + 137438953577u64 => "TRANSFER_TIMEOUT", + 137438953589u64 => "UNABLE_TO_BIND_SOCKET", + 137438953590u64 => "UNABLE_TO_CREATE_SOCKET", + 137438953609u64 => "UNABLE_TO_KEEPALIVE", + 137438953591u64 => "UNABLE_TO_LISTEN_SOCKET", + 137438953610u64 => "UNABLE_TO_NODELAY", + 137438953611u64 => "UNABLE_TO_REUSEADDR", + 137438953617u64 => "UNAVAILABLE_IP_FAMILY", + 137438953592u64 => "UNINITIALIZED", + 137438953612u64 => "UNKNOWN_INFO_TYPE", + 137438953618u64 => "UNSUPPORTED_IP_FAMILY", + 137438953593u64 => "UNSUPPORTED_METHOD", + 137438953603u64 => "UNSUPPORTED_PROTOCOL_FAMILY", + 137438953598u64 => "WRITE_TO_READ_ONLY_BIO", + 137438953594u64 => "WSASTARTUP", + 12884901988u64 => "ARG2_LT_ARG3", + 12884901989u64 => "BAD_RECIPROCAL", + 12884902002u64 => "BIGNUM_TOO_LONG", + 12884902006u64 => "BITS_TOO_SMALL", + 12884901990u64 => "CALLED_WITH_EVEN_MODULUS", + 12884901991u64 => "DIV_BY_ZERO", + 12884901992u64 => "ENCODING_ERROR", + 12884901993u64 => "EXPAND_ON_STATIC_BIGNUM_DATA", + 12884901998u64 => "INPUT_NOT_REDUCED", + 12884901994u64 => "INVALID_LENGTH", + 12884902003u64 => "INVALID_RANGE", + 12884902007u64 => "INVALID_SHIFT", + 12884901999u64 => "NOT_A_SQUARE", + 12884901995u64 => "NOT_INITIALIZED", + 12884901996u64 => "NO_INVERSE", + 12884902009u64 => "NO_PRIME_CANDIDATE", + 12884902004u64 => "NO_SOLUTION", + 12884902008u64 => "NO_SUITABLE_DIGEST", + 12884902005u64 => "PRIVATE_KEY_TOO_LARGE", + 12884902000u64 => "P_IS_NOT_PRIME", + 12884902001u64 => "TOO_MANY_ITERATIONS", + 12884901997u64 => "TOO_MANY_TEMPORARY_VARIABLES", + 249108103307u64 => "ALGORITHM_NOT_SUPPORTED", + 249108103335u64 => "BAD_CHECKAFTER_IN_POLLREP", + 249108103276u64 => "BAD_REQUEST_ID", + 249108103324u64 => "CERTHASH_UNMATCHED", + 249108103277u64 => "CERTID_NOT_FOUND", + 249108103337u64 => "CERTIFICATE_NOT_ACCEPTED", + 249108103280u64 => "CERTIFICATE_NOT_FOUND", + 249108103325u64 => "CERTREQMSG_NOT_FOUND", + 249108103281u64 => "CERTRESPONSE_NOT_FOUND", + 249108103282u64 => "CERT_AND_KEY_DO_NOT_MATCH", + 249108103349u64 => "CHECKAFTER_OUT_OF_RANGE", + 249108103344u64 => "ENCOUNTERED_KEYUPDATEWARNING", + 249108103330u64 => "ENCOUNTERED_WAITING", + 249108103283u64 => "ERROR_CALCULATING_PROTECTION", + 249108103284u64 => "ERROR_CREATING_CERTCONF", + 249108103285u64 => "ERROR_CREATING_CERTREP", + 249108103331u64 => "ERROR_CREATING_CERTREQ", + 249108103286u64 => "ERROR_CREATING_ERROR", + 249108103287u64 => "ERROR_CREATING_GENM", + 249108103288u64 => "ERROR_CREATING_GENP", + 249108103290u64 => "ERROR_CREATING_PKICONF", + 249108103291u64 => "ERROR_CREATING_POLLREP", + 249108103292u64 => "ERROR_CREATING_POLLREQ", + 249108103293u64 => "ERROR_CREATING_RP", + 249108103294u64 => "ERROR_CREATING_RR", + 249108103275u64 => "ERROR_PARSING_PKISTATUS", + 249108103326u64 => "ERROR_PROCESSING_MESSAGE", + 249108103295u64 => "ERROR_PROTECTING_MESSAGE", + 249108103296u64 => "ERROR_SETTING_CERTHASH", + 249108103328u64 => "ERROR_UNEXPECTED_CERTCONF", + 249108103308u64 => "ERROR_VALIDATING_PROTECTION", + 249108103339u64 => "ERROR_VALIDATING_SIGNATURE", + 249108103332u64 => "FAILED_BUILDING_OWN_CHAIN", + 249108103309u64 => "FAILED_EXTRACTING_PUBKEY", + 249108103278u64 => "FAILURE_OBTAINING_RANDOM", + 249108103297u64 => "FAIL_INFO_OUT_OF_RANGE", + 249108103268u64 => "INVALID_ARGS", + 249108103342u64 => "INVALID_OPTION", + 249108103333u64 => "MISSING_CERTID", + 249108103298u64 => "MISSING_KEY_INPUT_FOR_CREATING_PROTECTION", + 249108103310u64 => "MISSING_KEY_USAGE_DIGITALSIGNATURE", + 249108103289u64 => "MISSING_P10CSR", + 249108103334u64 => "MISSING_PBM_SECRET", + 249108103299u64 => "MISSING_PRIVATE_KEY", + 249108103358u64 => "MISSING_PRIVATE_KEY_FOR_POPO", + 249108103311u64 => "MISSING_PROTECTION", + 249108103351u64 => "MISSING_PUBLIC_KEY", + 249108103336u64 => "MISSING_REFERENCE_CERT", + 249108103346u64 => "MISSING_SECRET", + 249108103279u64 => "MISSING_SENDER_IDENTIFICATION", + 249108103347u64 => "MISSING_TRUST_ANCHOR", + 249108103312u64 => "MISSING_TRUST_STORE", + 249108103329u64 => "MULTIPLE_REQUESTS_NOT_SUPPORTED", + 249108103338u64 => "MULTIPLE_RESPONSES_NOT_SUPPORTED", + 249108103270u64 => "MULTIPLE_SAN_SOURCES", + 249108103362u64 => "NO_STDIO", + 249108103313u64 => "NO_SUITABLE_SENDER_CERT", + 249108103271u64 => "NULL_ARGUMENT", + 249108103314u64 => "PKIBODY_ERROR", + 249108103300u64 => "PKISTATUSINFO_NOT_FOUND", + 249108103340u64 => "POLLING_FAILED", + 249108103315u64 => "POTENTIALLY_INVALID_CERTIFICATE", + 249108103348u64 => "RECEIVED_ERROR", + 249108103316u64 => "RECIPNONCE_UNMATCHED", + 249108103317u64 => "REQUEST_NOT_ACCEPTED", + 249108103350u64 => "REQUEST_REJECTED_BY_SERVER", + 249108103318u64 => "SENDER_GENERALNAME_TYPE_NOT_SUPPORTED", + 249108103319u64 => "SRVCERT_DOES_NOT_VALIDATE_MSG", + 249108103352u64 => "TOTAL_TIMEOUT", + 249108103320u64 => "TRANSACTIONID_UNMATCHED", + 249108103327u64 => "TRANSFER_ERROR", + 249108103301u64 => "UNEXPECTED_PKIBODY", + 249108103353u64 => "UNEXPECTED_PKISTATUS", + 249108103321u64 => "UNEXPECTED_PVNO", + 249108103302u64 => "UNKNOWN_ALGORITHM_ID", + 249108103303u64 => "UNKNOWN_CERT_TYPE", + 249108103354u64 => "UNKNOWN_PKISTATUS", + 249108103304u64 => "UNSUPPORTED_ALGORITHM", + 249108103305u64 => "UNSUPPORTED_KEY_TYPE", + 249108103322u64 => "UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC", + 249108103343u64 => "VALUE_TOO_LARGE", + 249108103345u64 => "VALUE_TOO_SMALL", + 249108103306u64 => "WRONG_ALGORITHM_OID", + 249108103357u64 => "WRONG_CERTID", + 249108103355u64 => "WRONG_CERTID_IN_RP", + 249108103323u64 => "WRONG_PBM_VALUE", + 249108103356u64 => "WRONG_RP_COMPONENT_COUNT", + 249108103341u64 => "WRONG_SERIAL_IN_RP", + 197568495715u64 => "ADD_SIGNER_ERROR", + 197568495777u64 => "ATTRIBUTE_ERROR", + 197568495791u64 => "CERTIFICATE_ALREADY_PRESENT", + 197568495776u64 => "CERTIFICATE_HAS_NO_KEYID", + 197568495716u64 => "CERTIFICATE_VERIFY_ERROR", + 197568495800u64 => "CIPHER_AEAD_SET_TAG_ERROR", + 197568495801u64 => "CIPHER_GET_TAG", + 197568495717u64 => "CIPHER_INITIALISATION_ERROR", + 197568495718u64 => "CIPHER_PARAMETER_INITIALISATION_ERROR", + 197568495719u64 => "CMS_DATAFINAL_ERROR", + 197568495720u64 => "CMS_LIB", + 197568495786u64 => "CONTENTIDENTIFIER_MISMATCH", + 197568495721u64 => "CONTENT_NOT_FOUND", + 197568495787u64 => "CONTENT_TYPE_MISMATCH", + 197568495722u64 => "CONTENT_TYPE_NOT_COMPRESSED_DATA", + 197568495723u64 => "CONTENT_TYPE_NOT_ENVELOPED_DATA", + 197568495724u64 => "CONTENT_TYPE_NOT_SIGNED_DATA", + 197568495725u64 => "CONTENT_VERIFY_ERROR", + 197568495726u64 => "CTRL_ERROR", + 197568495727u64 => "CTRL_FAILURE", + 197568495803u64 => "DECODE_ERROR", + 197568495728u64 => "DECRYPT_ERROR", + 197568495729u64 => "ERROR_GETTING_PUBLIC_KEY", + 197568495730u64 => "ERROR_READING_MESSAGEDIGEST_ATTRIBUTE", + 197568495731u64 => "ERROR_SETTING_KEY", + 197568495732u64 => "ERROR_SETTING_RECIPIENTINFO", + 197568495799u64 => "ESS_SIGNING_CERTID_MISMATCH_ERROR", + 197568495733u64 => "INVALID_ENCRYPTED_KEY_LENGTH", + 197568495792u64 => "INVALID_KEY_ENCRYPTION_PARAMETER", + 197568495734u64 => "INVALID_KEY_LENGTH", + 197568495806u64 => "INVALID_LABEL", + 197568495807u64 => "INVALID_OAEP_PARAMETERS", + 197568495802u64 => "KDF_PARAMETER_ERROR", + 197568495735u64 => "MD_BIO_INIT_ERROR", + 197568495736u64 => "MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH", + 197568495737u64 => "MESSAGEDIGEST_WRONG_LENGTH", + 197568495788u64 => "MSGSIGDIGEST_ERROR", + 197568495778u64 => "MSGSIGDIGEST_VERIFICATION_FAILURE", + 197568495779u64 => "MSGSIGDIGEST_WRONG_LENGTH", + 197568495780u64 => "NEED_ONE_SIGNER", + 197568495781u64 => "NOT_A_SIGNED_RECEIPT", + 197568495738u64 => "NOT_ENCRYPTED_DATA", + 197568495739u64 => "NOT_KEK", + 197568495797u64 => "NOT_KEY_AGREEMENT", + 197568495740u64 => "NOT_KEY_TRANSPORT", + 197568495793u64 => "NOT_PWRI", + 197568495741u64 => "NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 197568495742u64 => "NO_CIPHER", + 197568495743u64 => "NO_CONTENT", + 197568495789u64 => "NO_CONTENT_TYPE", + 197568495744u64 => "NO_DEFAULT_DIGEST", + 197568495745u64 => "NO_DIGEST_SET", + 197568495746u64 => "NO_KEY", + 197568495790u64 => "NO_KEY_OR_CERT", + 197568495747u64 => "NO_MATCHING_DIGEST", + 197568495748u64 => "NO_MATCHING_RECIPIENT", + 197568495782u64 => "NO_MATCHING_SIGNATURE", + 197568495783u64 => "NO_MSGSIGDIGEST", + 197568495794u64 => "NO_PASSWORD", + 197568495749u64 => "NO_PRIVATE_KEY", + 197568495750u64 => "NO_PUBLIC_KEY", + 197568495784u64 => "NO_RECEIPT_REQUEST", + 197568495751u64 => "NO_SIGNERS", + 197568495804u64 => "PEER_KEY_ERROR", + 197568495752u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 197568495785u64 => "RECEIPT_DECODE_ERROR", + 197568495753u64 => "RECIPIENT_ERROR", + 197568495805u64 => "SHARED_INFO_ERROR", + 197568495754u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 197568495755u64 => "SIGNFINAL_ERROR", + 197568495756u64 => "SMIME_TEXT_ERROR", + 197568495757u64 => "STORE_INIT_ERROR", + 197568495758u64 => "TYPE_NOT_COMPRESSED_DATA", + 197568495759u64 => "TYPE_NOT_DATA", + 197568495760u64 => "TYPE_NOT_DIGESTED_DATA", + 197568495761u64 => "TYPE_NOT_ENCRYPTED_DATA", + 197568495762u64 => "TYPE_NOT_ENVELOPED_DATA", + 197568495763u64 => "UNABLE_TO_FINALIZE_CONTEXT", + 197568495764u64 => "UNKNOWN_CIPHER", + 197568495765u64 => "UNKNOWN_DIGEST_ALGORITHM", + 197568495766u64 => "UNKNOWN_ID", + 197568495767u64 => "UNSUPPORTED_COMPRESSION_ALGORITHM", + 197568495810u64 => "UNSUPPORTED_CONTENT_ENCRYPTION_ALGORITHM", + 197568495768u64 => "UNSUPPORTED_CONTENT_TYPE", + 197568495808u64 => "UNSUPPORTED_ENCRYPTION_TYPE", + 197568495769u64 => "UNSUPPORTED_KEK_ALGORITHM", + 197568495795u64 => "UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM", + 197568495809u64 => "UNSUPPORTED_LABEL_SOURCE", + 197568495771u64 => "UNSUPPORTED_RECIPIENTINFO_TYPE", + 197568495770u64 => "UNSUPPORTED_RECIPIENT_TYPE", + 197568495811u64 => "UNSUPPORTED_SIGNATURE_ALGORITHM", + 197568495772u64 => "UNSUPPORTED_TYPE", + 197568495773u64 => "UNWRAP_ERROR", + 197568495796u64 => "UNWRAP_FAILURE", + 197568495774u64 => "VERIFICATION_FAILURE", + 197568495775u64 => "WRAP_ERROR", + 176093659235u64 => "ZLIB_DEFLATE_ERROR", + 176093659236u64 => "ZLIB_INFLATE_ERROR", + 176093659237u64 => "ZLIB_NOT_SUPPORTED", + 60129542254u64 => "ERROR_LOADING_DSO", + 60129542266u64 => "INVALID_PRAGMA", + 60129542259u64 => "LIST_CANNOT_BE_NULL", + 60129542267u64 => "MANDATORY_BRACES_IN_VARIABLE_EXPANSION", + 60129542244u64 => "MISSING_CLOSE_SQUARE_BRACKET", + 60129542245u64 => "MISSING_EQUAL_SIGN", + 60129542256u64 => "MISSING_INIT_FUNCTION", + 60129542253u64 => "MODULE_INITIALIZATION_ERROR", + 60129542246u64 => "NO_CLOSE_BRACE", + 60129542249u64 => "NO_CONF", + 60129542250u64 => "NO_CONF_OR_ENVIRONMENT_VARIABLE", + 60129542251u64 => "NO_SECTION", + 60129542258u64 => "NO_SUCH_FILE", + 60129542252u64 => "NO_VALUE", + 60129542265u64 => "NUMBER_TOO_LARGE", + 60129542268u64 => "OPENSSL_CONF_REFERENCES_MISSING_SECTION", + 60129542255u64 => "RECURSIVE_DIRECTORY_INCLUDE", + 60129542270u64 => "RECURSIVE_SECTION_REFERENCE", + 60129542269u64 => "RELATIVE_PATH", + 60129542261u64 => "SSL_COMMAND_SECTION_EMPTY", + 60129542262u64 => "SSL_COMMAND_SECTION_NOT_FOUND", + 60129542263u64 => "SSL_SECTION_EMPTY", + 60129542264u64 => "SSL_SECTION_NOT_FOUND", + 60129542247u64 => "UNABLE_TO_CREATE_NEW_SECTION", + 60129542257u64 => "UNKNOWN_MODULE_NAME", + 60129542260u64 => "VARIABLE_EXPANSION_TOO_LONG", + 60129542248u64 => "VARIABLE_HAS_NO_VALUE", + 240518168676u64 => "BAD_PBM_ITERATIONCOUNT", + 240518168678u64 => "CRMFERROR", + 240518168679u64 => "ERROR", + 240518168680u64 => "ERROR_DECODING_CERTIFICATE", + 240518168681u64 => "ERROR_DECRYPTING_CERTIFICATE", + 240518168682u64 => "ERROR_DECRYPTING_SYMMETRIC_KEY", + 240518168683u64 => "FAILURE_OBTAINING_RANDOM", + 240518168684u64 => "ITERATIONCOUNT_BELOW_100", + 240518168677u64 => "MALFORMED_IV", + 240518168685u64 => "NULL_ARGUMENT", + 240518168689u64 => "POPOSKINPUT_NOT_SUPPORTED", + 240518168693u64 => "POPO_INCONSISTENT_PUBLIC_KEY", + 240518168697u64 => "POPO_MISSING", + 240518168694u64 => "POPO_MISSING_PUBLIC_KEY", + 240518168695u64 => "POPO_MISSING_SUBJECT", + 240518168696u64 => "POPO_RAVERIFIED_NOT_ACCEPTED", + 240518168686u64 => "SETTING_MAC_ALGOR_FAILURE", + 240518168687u64 => "SETTING_OWF_ALGOR_FAILURE", + 240518168688u64 => "UNSUPPORTED_ALGORITHM", + 240518168690u64 => "UNSUPPORTED_CIPHER", + 240518168691u64 => "UNSUPPORTED_METHOD_FOR_CREATING_POPO", + 240518168692u64 => "UNSUPPORTED_POPO_METHOD", + 64424509557u64 => "BAD_ALGORITHM_NAME", + 64424509558u64 => "CONFLICTING_NAMES", + 64424509561u64 => "HEX_STRING_TOO_SHORT", + 64424509542u64 => "ILLEGAL_HEX_DIGIT", + 64424509546u64 => "INSUFFICIENT_DATA_SPACE", + 64424509547u64 => "INSUFFICIENT_PARAM_SIZE", + 64424509548u64 => "INSUFFICIENT_SECURE_DATA_SPACE", + 64424509562u64 => "INVALID_NEGATIVE_VALUE", + 64424509549u64 => "INVALID_NULL_ARGUMENT", + 64424509550u64 => "INVALID_OSSL_PARAM_TYPE", + 64424509543u64 => "ODD_NUMBER_OF_DIGITS", + 64424509544u64 => "PROVIDER_ALREADY_EXISTS", + 64424509545u64 => "PROVIDER_SECTION_ERROR", + 64424509559u64 => "RANDOM_SECTION_ERROR", + 64424509551u64 => "SECURE_MALLOC_FAILURE", + 64424509552u64 => "STRING_TOO_LONG", + 64424509553u64 => "TOO_MANY_BYTES", + 64424509554u64 => "TOO_MANY_RECORDS", + 64424509556u64 => "TOO_SMALL_BUFFER", + 64424509560u64 => "UNKNOWN_NAME_IN_RANDOM_SECTION", + 64424509555u64 => "ZERO_LENGTH_NUMBER", + 214748364908u64 => "BASE64_DECODE_ERROR", + 214748364900u64 => "INVALID_LOG_ID_LENGTH", + 214748364909u64 => "LOG_CONF_INVALID", + 214748364910u64 => "LOG_CONF_INVALID_KEY", + 214748364911u64 => "LOG_CONF_MISSING_DESCRIPTION", + 214748364912u64 => "LOG_CONF_MISSING_KEY", + 214748364913u64 => "LOG_KEY_INVALID", + 214748364916u64 => "SCT_FUTURE_TIMESTAMP", + 214748364904u64 => "SCT_INVALID", + 214748364907u64 => "SCT_INVALID_SIGNATURE", + 214748364905u64 => "SCT_LIST_INVALID", + 214748364914u64 => "SCT_LOG_ID_MISMATCH", + 214748364906u64 => "SCT_NOT_SET", + 214748364915u64 => "SCT_UNSUPPORTED_VERSION", + 214748364901u64 => "UNRECOGNIZED_SIGNATURE_NID", + 214748364902u64 => "UNSUPPORTED_ENTRY_TYPE", + 214748364903u64 => "UNSUPPORTED_VERSION", + 21474836607u64 => "BAD_FFC_PARAMETERS", + 21474836581u64 => "BAD_GENERATOR", + 21474836589u64 => "BN_DECODE_ERROR", + 21474836586u64 => "BN_ERROR", + 21474836595u64 => "CHECK_INVALID_J_VALUE", + 21474836596u64 => "CHECK_INVALID_Q_VALUE", + 21474836602u64 => "CHECK_PUBKEY_INVALID", + 21474836603u64 => "CHECK_PUBKEY_TOO_LARGE", + 21474836604u64 => "CHECK_PUBKEY_TOO_SMALL", + 21474836597u64 => "CHECK_P_NOT_PRIME", + 21474836598u64 => "CHECK_P_NOT_SAFE_PRIME", + 21474836599u64 => "CHECK_Q_NOT_PRIME", + 21474836584u64 => "DECODE_ERROR", + 21474836590u64 => "INVALID_PARAMETER_NAME", + 21474836594u64 => "INVALID_PARAMETER_NID", + 21474836582u64 => "INVALID_PUBKEY", + 21474836608u64 => "INVALID_SECRET", + 21474836592u64 => "KDF_PARAMETER_ERROR", + 21474836588u64 => "KEYS_NOT_SET", + 21474836605u64 => "MISSING_PUBKEY", + 21474836583u64 => "MODULUS_TOO_LARGE", + 21474836606u64 => "MODULUS_TOO_SMALL", + 21474836600u64 => "NOT_SUITABLE_GENERATOR", + 21474836587u64 => "NO_PARAMETERS_SET", + 21474836580u64 => "NO_PRIVATE_VALUE", + 21474836585u64 => "PARAMETER_ENCODING_ERROR", + 21474836591u64 => "PEER_KEY_ERROR", + 21474836610u64 => "Q_TOO_LARGE", + 21474836593u64 => "SHARED_INFO_ERROR", + 21474836601u64 => "UNABLE_TO_CHECK_GENERATOR", + 42949673074u64 => "BAD_FFC_PARAMETERS", + 42949673062u64 => "BAD_Q_VALUE", + 42949673068u64 => "BN_DECODE_ERROR", + 42949673069u64 => "BN_ERROR", + 42949673064u64 => "DECODE_ERROR", + 42949673066u64 => "INVALID_DIGEST_TYPE", + 42949673072u64 => "INVALID_PARAMETERS", + 42949673061u64 => "MISSING_PARAMETERS", + 42949673071u64 => "MISSING_PRIVATE_KEY", + 42949673063u64 => "MODULUS_TOO_LARGE", + 42949673067u64 => "NO_PARAMETERS_SET", + 42949673065u64 => "PARAMETER_ENCODING_ERROR", + 42949673075u64 => "P_NOT_PRIME", + 42949673073u64 => "Q_NOT_PRIME", + 42949673070u64 => "SEED_LEN_SMALL", + 42949673076u64 => "TOO_MANY_RETRIES", + 158913790052u64 => "CTRL_FAILED", + 158913790062u64 => "DSO_ALREADY_LOADED", + 158913790065u64 => "EMPTY_FILE_STRUCTURE", + 158913790066u64 => "FAILURE", + 158913790053u64 => "FILENAME_TOO_BIG", + 158913790054u64 => "FINISH_FAILED", + 158913790067u64 => "INCORRECT_FILE_SYNTAX", + 158913790055u64 => "LOAD_FAILED", + 158913790061u64 => "NAME_TRANSLATION_FAILED", + 158913790063u64 => "NO_FILENAME", + 158913790056u64 => "NULL_HANDLE", + 158913790064u64 => "SET_FILENAME_FAILED", + 158913790057u64 => "STACK_ERROR", + 158913790058u64 => "SYM_FAILURE", + 158913790059u64 => "UNLOAD_FAILED", + 158913790060u64 => "UNSUPPORTED", + 68719476851u64 => "ASN1_ERROR", + 68719476892u64 => "BAD_SIGNATURE", + 68719476880u64 => "BIGNUM_OUT_OF_RANGE", + 68719476836u64 => "BUFFER_TOO_SMALL", + 68719476901u64 => "CANNOT_INVERT", + 68719476882u64 => "COORDINATES_OUT_OF_RANGE", + 68719476896u64 => "CURVE_DOES_NOT_SUPPORT_ECDH", + 68719476906u64 => "CURVE_DOES_NOT_SUPPORT_ECDSA", + 68719476895u64 => "CURVE_DOES_NOT_SUPPORT_SIGNING", + 68719476878u64 => "DECODE_ERROR", + 68719476854u64 => "DISCRIMINANT_IS_ZERO", + 68719476855u64 => "EC_GROUP_NEW_BY_NAME_FAILURE", + 68719476863u64 => "EXPLICIT_PARAMS_NOT_SUPPORTED", + 68719476902u64 => "FAILED_MAKING_PUBLIC_KEY", + 68719476879u64 => "FIELD_TOO_LARGE", + 68719476883u64 => "GF2M_NOT_SUPPORTED", + 68719476856u64 => "GROUP2PKPARAMETERS_FAILURE", + 68719476857u64 => "I2D_ECPKPARAMETERS_FAILURE", + 68719476837u64 => "INCOMPATIBLE_OBJECTS", + 68719476904u64 => "INVALID_A", + 68719476848u64 => "INVALID_ARGUMENT", + 68719476905u64 => "INVALID_B", + 68719476907u64 => "INVALID_COFACTOR", + 68719476846u64 => "INVALID_COMPRESSED_POINT", + 68719476845u64 => "INVALID_COMPRESSION_BIT", + 68719476877u64 => "INVALID_CURVE", + 68719476887u64 => "INVALID_DIGEST", + 68719476874u64 => "INVALID_DIGEST_TYPE", + 68719476838u64 => "INVALID_ENCODING", + 68719476839u64 => "INVALID_FIELD", + 68719476840u64 => "INVALID_FORM", + 68719476909u64 => "INVALID_GENERATOR", + 68719476858u64 => "INVALID_GROUP_ORDER", + 68719476852u64 => "INVALID_KEY", + 68719476853u64 => "INVALID_LENGTH", + 68719476910u64 => "INVALID_NAMED_GROUP_CONVERSION", + 68719476897u64 => "INVALID_OUTPUT_LENGTH", + 68719476908u64 => "INVALID_P", + 68719476869u64 => "INVALID_PEER_KEY", + 68719476868u64 => "INVALID_PENTANOMIAL_BASIS", + 68719476859u64 => "INVALID_PRIVATE_KEY", + 68719476911u64 => "INVALID_SEED", + 68719476873u64 => "INVALID_TRINOMIAL_BASIS", + 68719476884u64 => "KDF_PARAMETER_ERROR", + 68719476876u64 => "KEYS_NOT_SET", + 68719476872u64 => "LADDER_POST_FAILURE", + 68719476889u64 => "LADDER_PRE_FAILURE", + 68719476898u64 => "LADDER_STEP_FAILURE", + 68719476903u64 => "MISSING_OID", + 68719476860u64 => "MISSING_PARAMETERS", + 68719476861u64 => "MISSING_PRIVATE_KEY", + 68719476893u64 => "NEED_NEW_SETUP_VALUES", + 68719476871u64 => "NOT_A_NIST_PRIME", + 68719476862u64 => "NOT_IMPLEMENTED", + 68719476847u64 => "NOT_INITIALIZED", + 68719476875u64 => "NO_PARAMETERS_SET", + 68719476890u64 => "NO_PRIVATE_VALUE", + 68719476888u64 => "OPERATION_NOT_SUPPORTED", + 68719476870u64 => "PASSED_NULL_PARAMETER", + 68719476885u64 => "PEER_KEY_ERROR", + 68719476891u64 => "POINT_ARITHMETIC_FAILURE", + 68719476842u64 => "POINT_AT_INFINITY", + 68719476899u64 => "POINT_COORDINATES_BLIND_FAILURE", + 68719476843u64 => "POINT_IS_NOT_ON_CURVE", + 68719476894u64 => "RANDOM_NUMBER_GENERATION_FAILED", + 68719476886u64 => "SHARED_INFO_ERROR", + 68719476844u64 => "SLOT_FULL", + 68719476912u64 => "TOO_MANY_RETRIES", + 68719476849u64 => "UNDEFINED_GENERATOR", + 68719476864u64 => "UNDEFINED_ORDER", + 68719476900u64 => "UNKNOWN_COFACTOR", + 68719476865u64 => "UNKNOWN_GROUP", + 68719476850u64 => "UNKNOWN_ORDER", + 68719476867u64 => "UNSUPPORTED_FIELD", + 68719476881u64 => "WRONG_CURVE_PARAMETERS", + 68719476866u64 => "WRONG_ORDER", + 163208757348u64 => "ALREADY_LOADED", + 163208757381u64 => "ARGUMENT_IS_NOT_A_NUMBER", + 163208757382u64 => "CMD_NOT_EXECUTABLE", + 163208757383u64 => "COMMAND_TAKES_INPUT", + 163208757384u64 => "COMMAND_TAKES_NO_INPUT", + 163208757351u64 => "CONFLICTING_ENGINE_ID", + 163208757367u64 => "CTRL_COMMAND_NOT_IMPLEMENTED", + 163208757352u64 => "DSO_FAILURE", + 163208757380u64 => "DSO_NOT_FOUND", + 163208757396u64 => "ENGINES_SECTION_ERROR", + 163208757350u64 => "ENGINE_CONFIGURATION_ERROR", + 163208757353u64 => "ENGINE_IS_NOT_IN_LIST", + 163208757397u64 => "ENGINE_SECTION_ERROR", + 163208757376u64 => "FAILED_LOADING_PRIVATE_KEY", + 163208757377u64 => "FAILED_LOADING_PUBLIC_KEY", + 163208757354u64 => "FINISH_FAILED", + 163208757356u64 => "ID_OR_NAME_MISSING", + 163208757357u64 => "INIT_FAILED", + 163208757358u64 => "INTERNAL_LIST_ERROR", + 163208757391u64 => "INVALID_ARGUMENT", + 163208757385u64 => "INVALID_CMD_NAME", + 163208757386u64 => "INVALID_CMD_NUMBER", + 163208757399u64 => "INVALID_INIT_VALUE", + 163208757398u64 => "INVALID_STRING", + 163208757365u64 => "NOT_INITIALISED", + 163208757360u64 => "NOT_LOADED", + 163208757368u64 => "NO_CONTROL_FUNCTION", + 163208757392u64 => "NO_INDEX", + 163208757373u64 => "NO_LOAD_FUNCTION", + 163208757378u64 => "NO_REFERENCE", + 163208757364u64 => "NO_SUCH_ENGINE", + 163208757394u64 => "UNIMPLEMENTED_CIPHER", + 163208757395u64 => "UNIMPLEMENTED_DIGEST", + 163208757349u64 => "UNIMPLEMENTED_PUBLIC_KEY_METHOD", + 163208757393u64 => "VERSION_INCOMPATIBILITY", + 231928234091u64 => "EMPTY_ESS_CERT_ID_LIST", + 231928234087u64 => "ESS_CERT_DIGEST_ERROR", + 231928234088u64 => "ESS_CERT_ID_NOT_FOUND", + 231928234089u64 => "ESS_CERT_ID_WRONG_ORDER", + 231928234090u64 => "ESS_DIGEST_ALG_UNKNOWN", + 231928234086u64 => "ESS_SIGNING_CERTIFICATE_ERROR", + 231928234084u64 => "ESS_SIGNING_CERT_ADD_ERROR", + 231928234085u64 => "ESS_SIGNING_CERT_V2_ADD_ERROR", + 231928234092u64 => "MISSING_SIGNING_CERTIFICATE_ATTRIBUTE", + 25769803919u64 => "AES_KEY_SETUP_FAILED", + 25769803952u64 => "ARIA_KEY_SETUP_FAILED", + 25769803976u64 => "BAD_ALGORITHM_NAME", + 25769803876u64 => "BAD_DECRYPT", + 25769803971u64 => "BAD_KEY_LENGTH", + 25769803931u64 => "BUFFER_TOO_SMALL", + 25769804001u64 => "CACHE_CONSTANTS_FAILED", + 25769803933u64 => "CAMELLIA_KEY_SETUP_FAILED", + 25769803973u64 => "CANNOT_GET_PARAMETERS", + 25769803974u64 => "CANNOT_SET_PARAMETERS", + 25769803960u64 => "CIPHER_NOT_GCM_MODE", + 25769803898u64 => "CIPHER_PARAMETER_ERROR", + 25769803923u64 => "COMMAND_NOT_SUPPORTED", + 25769803977u64 => "CONFLICTING_ALGORITHM_NAME", + 25769803949u64 => "COPY_ERROR", + 25769803908u64 => "CTRL_NOT_IMPLEMENTED", + 25769803909u64 => "CTRL_OPERATION_NOT_IMPLEMENTED", + 25769803914u64 => "DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH", + 25769803890u64 => "DECODE_ERROR", + 25769803986u64 => "DEFAULT_QUERY_PARSE_ERROR", + 25769803877u64 => "DIFFERENT_KEY_TYPES", + 25769803929u64 => "DIFFERENT_PARAMETERS", + 25769803941u64 => "ERROR_LOADING_SECTION", + 25769803950u64 => "EXPECTING_AN_HMAC_KEY", + 25769803903u64 => "EXPECTING_AN_RSA_KEY", + 25769803904u64 => "EXPECTING_A_DH_KEY", + 25769803905u64 => "EXPECTING_A_DSA_KEY", + 25769803995u64 => "EXPECTING_A_ECX_KEY", + 25769803918u64 => "EXPECTING_A_EC_KEY", + 25769803940u64 => "EXPECTING_A_POLY1305_KEY", + 25769803951u64 => "EXPECTING_A_SIPHASH_KEY", + 25769803964u64 => "FINAL_ERROR", + 25769803990u64 => "GENERATE_ERROR", + 25769803958u64 => "GET_RAW_KEY_FAILED", + 25769803947u64 => "ILLEGAL_SCRYPT_PARAMETERS", + 25769803980u64 => "INACCESSIBLE_DOMAIN_PARAMETERS", + 25769803979u64 => "INACCESSIBLE_KEY", + 25769803910u64 => "INITIALIZATION_ERROR", + 25769803887u64 => "INPUT_NOT_INITIALIZED", + 25769803961u64 => "INVALID_CUSTOM_LENGTH", + 25769803928u64 => "INVALID_DIGEST", + 25769803970u64 => "INVALID_IV_LENGTH", + 25769803939u64 => "INVALID_KEY", + 25769803906u64 => "INVALID_KEY_LENGTH", + 25769803997u64 => "INVALID_LENGTH", + 25769803994u64 => "INVALID_NULL_ALGORITHM", + 25769803924u64 => "INVALID_OPERATION", + 25769803969u64 => "INVALID_PROVIDER_FUNCTIONS", + 25769803962u64 => "INVALID_SALT_LENGTH", + 25769803999u64 => "INVALID_SECRET_LENGTH", + 25769803996u64 => "INVALID_SEED_LENGTH", + 25769803998u64 => "INVALID_VALUE", + 25769803981u64 => "KEYMGMT_EXPORT_FAILURE", + 25769803956u64 => "KEY_SETUP_FAILED", + 25769803989u64 => "LOCKING_NOT_SUPPORTED", + 25769803948u64 => "MEMORY_LIMIT_EXCEEDED", + 25769803935u64 => "MESSAGE_DIGEST_IS_NULL", + 25769803920u64 => "METHOD_NOT_SUPPORTED", + 25769803879u64 => "MISSING_PARAMETERS", + 25769803966u64 => "NOT_ABLE_TO_COPY_CTX", + 25769803954u64 => "NOT_XOF_OR_INVALID_LENGTH", + 25769803907u64 => "NO_CIPHER_SET", + 25769803934u64 => "NO_DEFAULT_DIGEST", + 25769803915u64 => "NO_DIGEST_SET", + 25769803982u64 => "NO_IMPORT_FUNCTION", + 25769803975u64 => "NO_KEYMGMT_AVAILABLE", + 25769803972u64 => "NO_KEYMGMT_PRESENT", + 25769803930u64 => "NO_KEY_SET", + 25769803925u64 => "NO_OPERATION_SET", + 25769803984u64 => "NULL_MAC_PKEY_CTX", + 25769803953u64 => "ONLY_ONESHOT_SUPPORTED", + 25769803927u64 => "OPERATION_NOT_INITIALIZED", + 25769803926u64 => "OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", + 25769803978u64 => "OUTPUT_WOULD_OVERFLOW", + 25769803963u64 => "PARAMETER_TOO_LARGE", + 25769803938u64 => "PARTIALLY_OVERLAPPING", + 25769803957u64 => "PBKDF2_ERROR", + 25769803955u64 => "PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", + 25769803921u64 => "PRIVATE_KEY_DECODE_ERROR", + 25769803922u64 => "PRIVATE_KEY_ENCODE_ERROR", + 25769803882u64 => "PUBLIC_KEY_NOT_RSA", + 25769804003u64 => "SETTING_XOF_FAILED", + 25769803985u64 => "SET_DEFAULT_PROPERTY_FAILURE", + 25769803959u64 => "TOO_MANY_RECORDS", + 25769803988u64 => "UNABLE_TO_ENABLE_LOCKING", + 25769803991u64 => "UNABLE_TO_GET_MAXIMUM_REQUEST_SIZE", + 25769803992u64 => "UNABLE_TO_GET_RANDOM_STRENGTH", + 25769803987u64 => "UNABLE_TO_LOCK_CONTEXT", + 25769803993u64 => "UNABLE_TO_SET_CALLBACKS", + 25769803936u64 => "UNKNOWN_CIPHER", + 25769803937u64 => "UNKNOWN_DIGEST", + 25769803983u64 => "UNKNOWN_KEY_TYPE", + 25769803945u64 => "UNKNOWN_OPTION", + 25769803897u64 => "UNKNOWN_PBE_ALGORITHM", + 25769803932u64 => "UNSUPPORTED_ALGORITHM", + 25769803883u64 => "UNSUPPORTED_CIPHER", + 25769803899u64 => "UNSUPPORTED_KEYLENGTH", + 25769803900u64 => "UNSUPPORTED_KEY_DERIVATION_FUNCTION", + 25769803884u64 => "UNSUPPORTED_KEY_SIZE", + 25769804000u64 => "UNSUPPORTED_KEY_TYPE", + 25769803911u64 => "UNSUPPORTED_NUMBER_OF_ROUNDS", + 25769803901u64 => "UNSUPPORTED_PRF", + 25769803894u64 => "UNSUPPORTED_PRIVATE_KEY_ALGORITHM", + 25769803902u64 => "UNSUPPORTED_SALT_TYPE", + 25769803965u64 => "UPDATE_ERROR", + 25769803946u64 => "WRAP_MODE_NOT_ALLOWED", + 25769803885u64 => "WRONG_FINAL_BLOCK_LENGTH", + 25769803967u64 => "XTS_DATA_UNIT_IS_TOO_LARGE", + 25769803968u64 => "XTS_DUPLICATED_KEYS", + 261993005164u64 => "ASN1_LEN_EXCEEDS_MAX_RESP_LEN", + 261993005156u64 => "CONNECT_FAILURE", + 261993005165u64 => "ERROR_PARSING_ASN1_LENGTH", + 261993005175u64 => "ERROR_PARSING_CONTENT_LENGTH", + 261993005157u64 => "ERROR_PARSING_URL", + 261993005159u64 => "ERROR_RECEIVING", + 261993005158u64 => "ERROR_SENDING", + 261993005184u64 => "FAILED_READING_DATA", + 261993005182u64 => "HEADER_PARSE_ERROR", + 261993005176u64 => "INCONSISTENT_CONTENT_LENGTH", + 261993005179u64 => "INVALID_PORT_NUMBER", + 261993005181u64 => "INVALID_URL_PATH", + 261993005180u64 => "INVALID_URL_SCHEME", + 261993005173u64 => "MAX_RESP_LEN_EXCEEDED", + 261993005166u64 => "MISSING_ASN1_ENCODING", + 261993005177u64 => "MISSING_CONTENT_TYPE", + 261993005167u64 => "MISSING_REDIRECT_LOCATION", + 261993005161u64 => "RECEIVED_ERROR", + 261993005162u64 => "RECEIVED_WRONG_HTTP_VERSION", + 261993005168u64 => "REDIRECTION_FROM_HTTPS_TO_HTTP", + 261993005172u64 => "REDIRECTION_NOT_ENABLED", + 261993005169u64 => "RESPONSE_LINE_TOO_LONG", + 261993005160u64 => "RESPONSE_PARSE_ERROR", + 261993005185u64 => "RETRY_TIMEOUT", + 261993005183u64 => "SERVER_CANCELED_CONNECTION", + 261993005178u64 => "SOCK_NOT_SUPPORTED", + 261993005170u64 => "STATUS_CODE_UNSUPPORTED", + 261993005163u64 => "TLS_NOT_ENABLED", + 261993005171u64 => "TOO_MANY_REDIRECTIONS", + 261993005174u64 => "UNEXPECTED_CONTENT_TYPE", + 34359738470u64 => "OID_EXISTS", + 34359738469u64 => "UNKNOWN_NID", + 34359738471u64 => "UNKNOWN_OBJECT_NAME", + 167503724645u64 => "CERTIFICATE_VERIFY_ERROR", + 167503724646u64 => "DIGEST_ERR", + 167503724650u64 => "DIGEST_NAME_ERR", + 167503724651u64 => "DIGEST_SIZE_ERR", + 167503724666u64 => "ERROR_IN_NEXTUPDATE_FIELD", + 167503724667u64 => "ERROR_IN_THISUPDATE_FIELD", + 167503724647u64 => "MISSING_OCSPSIGNING_USAGE", + 167503724668u64 => "NEXTUPDATE_BEFORE_THISUPDATE", + 167503724648u64 => "NOT_BASIC_RESPONSE", + 167503724649u64 => "NO_CERTIFICATES_IN_CHAIN", + 167503724652u64 => "NO_RESPONSE_DATA", + 167503724653u64 => "NO_REVOKED_TIME", + 167503724674u64 => "NO_SIGNER_KEY", + 167503724654u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 167503724672u64 => "REQUEST_NOT_SIGNED", + 167503724655u64 => "RESPONSE_CONTAINS_NO_REVOCATION_DATA", + 167503724656u64 => "ROOT_CA_NOT_TRUSTED", + 167503724661u64 => "SIGNATURE_FAILURE", + 167503724662u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 167503724669u64 => "STATUS_EXPIRED", + 167503724670u64 => "STATUS_NOT_YET_VALID", + 167503724671u64 => "STATUS_TOO_OLD", + 167503724663u64 => "UNKNOWN_MESSAGE_DIGEST", + 167503724664u64 => "UNKNOWN_NID", + 167503724673u64 => "UNSUPPORTED_REQUESTORNAME_TYPE", + 257698037861u64 => "COULD_NOT_DECODE_OBJECT", + 257698037862u64 => "DECODER_NOT_FOUND", + 257698037860u64 => "MISSING_GET_PARAMS", + 253403070565u64 => "ENCODER_NOT_FOUND", + 253403070564u64 => "INCORRECT_PROPERTY_QUERY", + 253403070566u64 => "MISSING_GET_PARAMS", + 188978561131u64 => "AMBIGUOUS_CONTENT_TYPE", + 188978561139u64 => "BAD_PASSWORD_READ", + 188978561137u64 => "ERROR_VERIFYING_PKCS12_MAC", + 188978561145u64 => "FINGERPRINT_SIZE_DOES_NOT_MATCH_DIGEST", + 188978561130u64 => "INVALID_SCHEME", + 188978561136u64 => "IS_NOT_A", + 188978561140u64 => "LOADER_INCOMPLETE", + 188978561141u64 => "LOADING_STARTED", + 188978561124u64 => "NOT_A_CERTIFICATE", + 188978561125u64 => "NOT_A_CRL", + 188978561127u64 => "NOT_A_NAME", + 188978561126u64 => "NOT_A_PRIVATE_KEY", + 188978561146u64 => "NOT_A_PUBLIC_KEY", + 188978561128u64 => "NOT_PARAMETERS", + 188978561147u64 => "NO_LOADERS_FOUND", + 188978561138u64 => "PASSPHRASE_CALLBACK_ERROR", + 188978561132u64 => "PATH_MUST_BE_ABSOLUTE", + 188978561143u64 => "SEARCH_ONLY_SUPPORTED_FOR_DIRECTORIES", + 188978561133u64 => "UI_PROCESS_INTERRUPTED_OR_CANCELLED", + 188978561129u64 => "UNREGISTERED_SCHEME", + 188978561134u64 => "UNSUPPORTED_CONTENT_TYPE", + 188978561142u64 => "UNSUPPORTED_OPERATION", + 188978561144u64 => "UNSUPPORTED_SEARCH_TYPE", + 188978561135u64 => "URI_AUTHORITY_UNSUPPORTED", + 38654705764u64 => "BAD_BASE64_DECODE", + 38654705765u64 => "BAD_DECRYPT", + 38654705766u64 => "BAD_END_LINE", + 38654705767u64 => "BAD_IV_CHARS", + 38654705780u64 => "BAD_MAGIC_NUMBER", + 38654705768u64 => "BAD_PASSWORD_READ", + 38654705781u64 => "BAD_VERSION_NUMBER", + 38654705782u64 => "BIO_WRITE_FAILURE", + 38654705791u64 => "CIPHER_IS_NULL", + 38654705779u64 => "ERROR_CONVERTING_PRIVATE_KEY", + 38654705795u64 => "EXPECTING_DSS_KEY_BLOB", + 38654705783u64 => "EXPECTING_PRIVATE_KEY_BLOB", + 38654705784u64 => "EXPECTING_PUBLIC_KEY_BLOB", + 38654705796u64 => "EXPECTING_RSA_KEY_BLOB", + 38654705792u64 => "HEADER_TOO_LONG", + 38654705785u64 => "INCONSISTENT_HEADER", + 38654705786u64 => "KEYBLOB_HEADER_PARSE_ERROR", + 38654705787u64 => "KEYBLOB_TOO_SHORT", + 38654705793u64 => "MISSING_DEK_IV", + 38654705769u64 => "NOT_DEK_INFO", + 38654705770u64 => "NOT_ENCRYPTED", + 38654705771u64 => "NOT_PROC_TYPE", + 38654705772u64 => "NO_START_LINE", + 38654705773u64 => "PROBLEMS_GETTING_PASSWORD", + 38654705788u64 => "PVK_DATA_TOO_SHORT", + 38654705789u64 => "PVK_TOO_SHORT", + 38654705775u64 => "READ_KEY", + 38654705776u64 => "SHORT_HEADER", + 38654705794u64 => "UNEXPECTED_DEK_IV", + 38654705777u64 => "UNSUPPORTED_CIPHER", + 38654705778u64 => "UNSUPPORTED_ENCRYPTION", + 38654705790u64 => "UNSUPPORTED_KEY_COMPONENTS", + 38654705774u64 => "UNSUPPORTED_PUBLIC_KEY_TYPE", + 150323855460u64 => "CANT_PACK_STRUCTURE", + 150323855481u64 => "CONTENT_TYPE_NOT_DATA", + 150323855461u64 => "DECODE_ERROR", + 150323855462u64 => "ENCODE_ERROR", + 150323855463u64 => "ENCRYPT_ERROR", + 150323855480u64 => "ERROR_SETTING_ENCRYPTED_DATA_TYPE", + 150323855464u64 => "INVALID_NULL_ARGUMENT", + 150323855465u64 => "INVALID_NULL_PKCS12_POINTER", + 150323855472u64 => "INVALID_TYPE", + 150323855466u64 => "IV_GEN_ERROR", + 150323855467u64 => "KEY_GEN_ERROR", + 150323855468u64 => "MAC_ABSENT", + 150323855469u64 => "MAC_GENERATION_ERROR", + 150323855470u64 => "MAC_SETUP_ERROR", + 150323855471u64 => "MAC_STRING_SET_ERROR", + 150323855473u64 => "MAC_VERIFY_FAILURE", + 150323855474u64 => "PARSE_ERROR", + 150323855476u64 => "PKCS12_CIPHERFINAL_ERROR", + 150323855478u64 => "UNKNOWN_DIGEST_ALGORITHM", + 150323855479u64 => "UNSUPPORTED_PKCS12_MODE", + 141733920885u64 => "CERTIFICATE_VERIFY_ERROR", + 141733920912u64 => "CIPHER_HAS_NO_OBJECT_IDENTIFIER", + 141733920884u64 => "CIPHER_NOT_INITIALIZED", + 141733920886u64 => "CONTENT_AND_DATA_PRESENT", + 141733920920u64 => "CTRL_ERROR", + 141733920887u64 => "DECRYPT_ERROR", + 141733920869u64 => "DIGEST_FAILURE", + 141733920917u64 => "ENCRYPTION_CTRL_FAILURE", + 141733920918u64 => "ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 141733920888u64 => "ERROR_ADDING_RECIPIENT", + 141733920889u64 => "ERROR_SETTING_CIPHER", + 141733920911u64 => "INVALID_NULL_POINTER", + 141733920923u64 => "INVALID_SIGNED_DATA_TYPE", + 141733920890u64 => "NO_CONTENT", + 141733920919u64 => "NO_DEFAULT_DIGEST", + 141733920922u64 => "NO_MATCHING_DIGEST_TYPE_FOUND", + 141733920883u64 => "NO_RECIPIENT_MATCHES_CERTIFICATE", + 141733920891u64 => "NO_SIGNATURES_ON_DATA", + 141733920910u64 => "NO_SIGNERS", + 141733920872u64 => "OPERATION_NOT_SUPPORTED_ON_THIS_TYPE", + 141733920892u64 => "PKCS7_ADD_SIGNATURE_ERROR", + 141733920921u64 => "PKCS7_ADD_SIGNER_ERROR", + 141733920913u64 => "PKCS7_DATASIGN", + 141733920895u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 141733920873u64 => "SIGNATURE_FAILURE", + 141733920896u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 141733920915u64 => "SIGNING_CTRL_FAILURE", + 141733920916u64 => "SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 141733920897u64 => "SMIME_TEXT_ERROR", + 141733920874u64 => "UNABLE_TO_FIND_CERTIFICATE", + 141733920875u64 => "UNABLE_TO_FIND_MEM_BIO", + 141733920876u64 => "UNABLE_TO_FIND_MESSAGE_DIGEST", + 141733920877u64 => "UNKNOWN_DIGEST_TYPE", + 141733920878u64 => "UNKNOWN_OPERATION", + 141733920879u64 => "UNSUPPORTED_CIPHER_TYPE", + 141733920880u64 => "UNSUPPORTED_CONTENT_TYPE", + 141733920881u64 => "WRONG_CONTENT_TYPE", + 141733920882u64 => "WRONG_PKCS7_TYPE", + 236223201380u64 => "NAME_TOO_LONG", + 236223201381u64 => "NOT_AN_ASCII_CHARACTER", + 236223201382u64 => "NOT_AN_HEXADECIMAL_DIGIT", + 236223201383u64 => "NOT_AN_IDENTIFIER", + 236223201384u64 => "NOT_AN_OCTAL_DIGIT", + 236223201385u64 => "NOT_A_DECIMAL_DIGIT", + 236223201386u64 => "NO_MATCHING_STRING_DELIMITER", + 236223201387u64 => "NO_VALUE", + 236223201388u64 => "PARSE_FAILED", + 236223201389u64 => "STRING_TOO_LONG", + 236223201390u64 => "TRAILING_CHARACTERS", + 244813136056u64 => "ADDITIONAL_INPUT_TOO_LONG", + 244813136045u64 => "ALGORITHM_MISMATCH", + 244813136057u64 => "ALREADY_INSTANTIATED", + 244813135972u64 => "BAD_DECRYPT", + 244813136013u64 => "BAD_ENCODING", + 244813136014u64 => "BAD_LENGTH", + 244813136033u64 => "BAD_TLS_CLIENT_VERSION", + 244813136032u64 => "BN_ERROR", + 244813135974u64 => "CIPHER_OPERATION_FAILED", + 244813136077u64 => "DERIVATION_FUNCTION_INIT_FAILED", + 244813136046u64 => "DIGEST_NOT_ALLOWED", + 244813136058u64 => "ENTROPY_SOURCE_STRENGTH_TOO_WEAK", + 244813136060u64 => "ERROR_INSTANTIATING_DRBG", + 244813136061u64 => "ERROR_RETRIEVING_ENTROPY", + 244813136062u64 => "ERROR_RETRIEVING_NONCE", + 244813136036u64 => "FAILED_DURING_DERIVATION", + 244813136052u64 => "FAILED_TO_CREATE_LOCK", + 244813136034u64 => "FAILED_TO_DECRYPT", + 244813135993u64 => "FAILED_TO_GENERATE_KEY", + 244813135975u64 => "FAILED_TO_GET_PARAMETER", + 244813135976u64 => "FAILED_TO_SET_PARAMETER", + 244813136047u64 => "FAILED_TO_SIGN", + 244813136099u64 => "FIPS_MODULE_CONDITIONAL_ERROR", + 244813136096u64 => "FIPS_MODULE_ENTERING_ERROR_STATE", + 244813136097u64 => "FIPS_MODULE_IN_ERROR_STATE", + 244813136063u64 => "GENERATE_ERROR", + 244813136037u64 => "ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", + 244813136082u64 => "INDICATOR_INTEGRITY_FAILURE", + 244813136053u64 => "INSUFFICIENT_DRBG_STRENGTH", + 244813135980u64 => "INVALID_AAD", + 244813136083u64 => "INVALID_CONFIG_DATA", + 244813136029u64 => "INVALID_CONSTANT_LENGTH", + 244813136048u64 => "INVALID_CURVE", + 244813135983u64 => "INVALID_CUSTOM_LENGTH", + 244813135987u64 => "INVALID_DATA", + 244813135994u64 => "INVALID_DIGEST", + 244813136038u64 => "INVALID_DIGEST_LENGTH", + 244813136090u64 => "INVALID_DIGEST_SIZE", + 244813136102u64 => "INVALID_INPUT_LENGTH", + 244813135995u64 => "INVALID_ITERATION_COUNT", + 244813135981u64 => "INVALID_IV_LENGTH", + 244813136030u64 => "INVALID_KEY", + 244813135977u64 => "INVALID_KEY_LENGTH", + 244813136023u64 => "INVALID_MAC", + 244813136039u64 => "INVALID_MGF1_MD", + 244813135997u64 => "INVALID_MODE", + 244813136089u64 => "INVALID_OUTPUT_LENGTH", + 244813136040u64 => "INVALID_PADDING_MODE", + 244813136070u64 => "INVALID_PUBINFO", + 244813135984u64 => "INVALID_SALT_LENGTH", + 244813136026u64 => "INVALID_SEED_LENGTH", + 244813136051u64 => "INVALID_SIGNATURE_SIZE", + 244813136084u64 => "INVALID_STATE", + 244813135982u64 => "INVALID_TAG", + 244813135990u64 => "INVALID_TAG_LENGTH", + 244813136072u64 => "INVALID_UKM_LENGTH", + 244813136042u64 => "INVALID_X931_DIGEST", + 244813136064u64 => "IN_ERROR_STATE", + 244813135973u64 => "KEY_SETUP_FAILED", + 244813136043u64 => "KEY_SIZE_TOO_SMALL", + 244813136074u64 => "LENGTH_TOO_LARGE", + 244813136075u64 => "MISMATCHING_DOMAIN_PARAMETERS", + 244813136016u64 => "MISSING_CEK_ALG", + 244813136027u64 => "MISSING_CIPHER", + 244813136085u64 => "MISSING_CONFIG_DATA", + 244813136028u64 => "MISSING_CONSTANT", + 244813136000u64 => "MISSING_KEY", + 244813136022u64 => "MISSING_MAC", + 244813136001u64 => "MISSING_MESSAGE_DIGEST", + 244813136081u64 => "MISSING_OID", + 244813136002u64 => "MISSING_PASS", + 244813136003u64 => "MISSING_SALT", + 244813136004u64 => "MISSING_SECRET", + 244813136012u64 => "MISSING_SEED", + 244813136005u64 => "MISSING_SESSION_ID", + 244813136006u64 => "MISSING_TYPE", + 244813136007u64 => "MISSING_XCGHASH", + 244813136086u64 => "MODULE_INTEGRITY_FAILURE", + 244813136093u64 => "NOT_A_PRIVATE_KEY", + 244813136092u64 => "NOT_A_PUBLIC_KEY", + 244813136065u64 => "NOT_INSTANTIATED", + 244813136098u64 => "NOT_PARAMETERS", + 244813136008u64 => "NOT_SUPPORTED", + 244813135985u64 => "NOT_XOF_OR_INVALID_LENGTH", + 244813135986u64 => "NO_KEY_SET", + 244813136049u64 => "NO_PARAMETERS_SET", + 244813136050u64 => "OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", + 244813135978u64 => "OUTPUT_BUFFER_TOO_SMALL", + 244813136100u64 => "PARENT_CANNOT_GENERATE_RANDOM_NUMBERS", + 244813136059u64 => "PARENT_CANNOT_SUPPLY_ENTROPY_SEED", + 244813136054u64 => "PARENT_LOCKING_NOT_ENABLED", + 244813136066u64 => "PARENT_STRENGTH_TOO_WEAK", + 244813136091u64 => "PATH_MUST_BE_ABSOLUTE", + 244813136067u64 => "PERSONALISATION_STRING_TOO_LONG", + 244813136044u64 => "PSS_SALTLEN_TOO_SMALL", + 244813136068u64 => "REQUEST_TOO_LARGE_FOR_DRBG", + 244813136078u64 => "REQUIRE_CTR_MODE_CIPHER", + 244813136069u64 => "RESEED_ERROR", + 244813136094u64 => "SEARCH_ONLY_SUPPORTED_FOR_DIRECTORIES", + 244813136101u64 => "SEED_SOURCES_MUST_NOT_HAVE_A_PARENT", + 244813136087u64 => "SELF_TEST_KAT_FAILURE", + 244813136088u64 => "SELF_TEST_POST_FAILURE", + 244813135992u64 => "TAG_NOT_NEEDED", + 244813135991u64 => "TAG_NOT_SET", + 244813135998u64 => "TOO_MANY_RECORDS", + 244813136079u64 => "UNABLE_TO_FIND_CIPHERS", + 244813136071u64 => "UNABLE_TO_GET_PARENT_STRENGTH", + 244813136031u64 => "UNABLE_TO_GET_PASSPHRASE", + 244813136080u64 => "UNABLE_TO_INITIALISE_CIPHERS", + 244813136019u64 => "UNABLE_TO_LOAD_SHA256", + 244813136073u64 => "UNABLE_TO_LOCK_PARENT", + 244813136076u64 => "UNABLE_TO_RESEED", + 244813136017u64 => "UNSUPPORTED_CEK_ALG", + 244813136025u64 => "UNSUPPORTED_KEY_SIZE", + 244813136009u64 => "UNSUPPORTED_MAC_TYPE", + 244813136024u64 => "UNSUPPORTED_NUMBER_OF_ROUNDS", + 244813136095u64 => "URI_AUTHORITY_UNSUPPORTED", + 244813136010u64 => "VALUE_ERROR", + 244813135979u64 => "WRONG_FINAL_BLOCK_LENGTH", + 244813136011u64 => "WRONG_OUTPUT_BUFFER_SIZE", + 244813136055u64 => "XOF_DIGESTS_NOT_ALLOWED", + 244813136020u64 => "XTS_DATA_UNIT_IS_TOO_LARGE", + 244813136021u64 => "XTS_DUPLICATED_KEYS", + 154618822758u64 => "ADDITIONAL_INPUT_TOO_LONG", + 154618822759u64 => "ALREADY_INSTANTIATED", + 154618822761u64 => "ARGUMENT_OUT_OF_RANGE", + 154618822777u64 => "CANNOT_OPEN_FILE", + 154618822785u64 => "DRBG_ALREADY_INITIALIZED", + 154618822760u64 => "DRBG_NOT_INITIALISED", + 154618822762u64 => "ENTROPY_INPUT_TOO_LONG", + 154618822780u64 => "ENTROPY_OUT_OF_RANGE", + 154618822783u64 => "ERROR_ENTROPY_POOL_WAS_IGNORED", + 154618822763u64 => "ERROR_INITIALISING_DRBG", + 154618822764u64 => "ERROR_INSTANTIATING_DRBG", + 154618822765u64 => "ERROR_RETRIEVING_ADDITIONAL_INPUT", + 154618822766u64 => "ERROR_RETRIEVING_ENTROPY", + 154618822767u64 => "ERROR_RETRIEVING_NONCE", + 154618822782u64 => "FAILED_TO_CREATE_LOCK", + 154618822757u64 => "FUNC_NOT_IMPLEMENTED", + 154618822779u64 => "FWRITE_ERROR", + 154618822768u64 => "GENERATE_ERROR", + 154618822795u64 => "INSUFFICIENT_DRBG_STRENGTH", + 154618822769u64 => "INTERNAL_ERROR", + 154618822770u64 => "IN_ERROR_STATE", + 154618822778u64 => "NOT_A_REGULAR_FILE", + 154618822771u64 => "NOT_INSTANTIATED", + 154618822784u64 => "NO_DRBG_IMPLEMENTATION_SELECTED", + 154618822786u64 => "PARENT_LOCKING_NOT_ENABLED", + 154618822787u64 => "PARENT_STRENGTH_TOO_WEAK", + 154618822772u64 => "PERSONALISATION_STRING_TOO_LONG", + 154618822789u64 => "PREDICTION_RESISTANCE_NOT_SUPPORTED", + 154618822756u64 => "PRNG_NOT_SEEDED", + 154618822781u64 => "RANDOM_POOL_OVERFLOW", + 154618822790u64 => "RANDOM_POOL_UNDERFLOW", + 154618822773u64 => "REQUEST_TOO_LARGE_FOR_DRBG", + 154618822774u64 => "RESEED_ERROR", + 154618822775u64 => "SELFTEST_FAILURE", + 154618822791u64 => "TOO_LITTLE_NONCE_REQUESTED", + 154618822792u64 => "TOO_MUCH_NONCE_REQUESTED", + 154618822799u64 => "UNABLE_TO_CREATE_DRBG", + 154618822800u64 => "UNABLE_TO_FETCH_DRBG", + 154618822797u64 => "UNABLE_TO_GET_PARENT_RESEED_PROP_COUNTER", + 154618822794u64 => "UNABLE_TO_GET_PARENT_STRENGTH", + 154618822796u64 => "UNABLE_TO_LOCK_PARENT", + 154618822788u64 => "UNSUPPORTED_DRBG_FLAGS", + 154618822776u64 => "UNSUPPORTED_DRBG_TYPE", + 17179869284u64 => "ALGORITHM_MISMATCH", + 17179869285u64 => "BAD_E_VALUE", + 17179869286u64 => "BAD_FIXED_HEADER_DECRYPT", + 17179869287u64 => "BAD_PAD_BYTE_COUNT", + 17179869288u64 => "BAD_SIGNATURE", + 17179869290u64 => "BLOCK_TYPE_IS_NOT_01", + 17179869291u64 => "BLOCK_TYPE_IS_NOT_02", + 17179869292u64 => "DATA_GREATER_THAN_MOD_LEN", + 17179869293u64 => "DATA_TOO_LARGE", + 17179869294u64 => "DATA_TOO_LARGE_FOR_KEY_SIZE", + 17179869316u64 => "DATA_TOO_LARGE_FOR_MODULUS", + 17179869295u64 => "DATA_TOO_SMALL", + 17179869306u64 => "DATA_TOO_SMALL_FOR_KEY_SIZE", + 17179869342u64 => "DIGEST_DOES_NOT_MATCH", + 17179869329u64 => "DIGEST_NOT_ALLOWED", + 17179869296u64 => "DIGEST_TOO_BIG_FOR_RSA_KEY", + 17179869308u64 => "DMP1_NOT_CONGRUENT_TO_D", + 17179869309u64 => "DMQ1_NOT_CONGRUENT_TO_D", + 17179869307u64 => "D_E_NOT_CONGRUENT_TO_1", + 17179869317u64 => "FIRST_OCTET_INVALID", + 17179869328u64 => "ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", + 17179869341u64 => "INVALID_DIGEST", + 17179869327u64 => "INVALID_DIGEST_LENGTH", + 17179869321u64 => "INVALID_HEADER", + 17179869355u64 => "INVALID_KEYPAIR", + 17179869357u64 => "INVALID_KEY_LENGTH", + 17179869344u64 => "INVALID_LABEL", + 17179869365u64 => "INVALID_LENGTH", + 17179869315u64 => "INVALID_MESSAGE_LENGTH", + 17179869340u64 => "INVALID_MGF1_MD", + 17179869358u64 => "INVALID_MODULUS", + 17179869351u64 => "INVALID_MULTI_PRIME_KEY", + 17179869345u64 => "INVALID_OAEP_PARAMETERS", + 17179869322u64 => "INVALID_PADDING", + 17179869325u64 => "INVALID_PADDING_MODE", + 17179869333u64 => "INVALID_PSS_PARAMETERS", + 17179869330u64 => "INVALID_PSS_SALTLEN", + 17179869359u64 => "INVALID_REQUEST", + 17179869334u64 => "INVALID_SALT_LENGTH", + 17179869360u64 => "INVALID_STRENGTH", + 17179869323u64 => "INVALID_TRAILER", + 17179869326u64 => "INVALID_X931_DIGEST", + 17179869310u64 => "IQMP_NOT_INVERSE_OF_Q", + 17179869349u64 => "KEY_PRIME_NUM_INVALID", + 17179869304u64 => "KEY_SIZE_TOO_SMALL", + 17179869318u64 => "LAST_OCTET_INVALID", + 17179869336u64 => "MGF1_DIGEST_NOT_ALLOWED", + 17179869363u64 => "MISSING_PRIVATE_KEY", + 17179869289u64 => "MODULUS_TOO_LARGE", + 17179869352u64 => "MP_COEFFICIENT_NOT_INVERSE_OF_R", + 17179869353u64 => "MP_EXPONENT_NOT_CONGRUENT_TO_D", + 17179869354u64 => "MP_R_NOT_PRIME", + 17179869324u64 => "NO_PUBLIC_EXPONENT", + 17179869297u64 => "NULL_BEFORE_BLOCK_MISSING", + 17179869356u64 => "N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES", + 17179869311u64 => "N_DOES_NOT_EQUAL_P_Q", + 17179869305u64 => "OAEP_DECODING_ERROR", + 17179869332u64 => "OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", + 17179869298u64 => "PADDING_CHECK_FAILED", + 17179869361u64 => "PAIRWISE_TEST_FAILURE", + 17179869343u64 => "PKCS_DECODING_ERROR", + 17179869348u64 => "PSS_SALTLEN_TOO_SMALL", + 17179869362u64 => "PUB_EXPONENT_OUT_OF_RANGE", + 17179869312u64 => "P_NOT_PRIME", + 17179869313u64 => "Q_NOT_PRIME", + 17179869364u64 => "RANDOMNESS_SOURCE_STRENGTH_INSUFFICIENT", + 17179869314u64 => "RSA_OPERATIONS_NOT_SUPPORTED", + 17179869320u64 => "SLEN_CHECK_FAILED", + 17179869319u64 => "SLEN_RECOVERY_FAILED", + 17179869299u64 => "SSLV3_ROLLBACK_ATTACK", + 17179869300u64 => "THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", + 17179869301u64 => "UNKNOWN_ALGORITHM_TYPE", + 17179869350u64 => "UNKNOWN_DIGEST", + 17179869335u64 => "UNKNOWN_MASK_DIGEST", + 17179869302u64 => "UNKNOWN_PADDING_TYPE", + 17179869346u64 => "UNSUPPORTED_ENCRYPTION_TYPE", + 17179869347u64 => "UNSUPPORTED_LABEL_SOURCE", + 17179869337u64 => "UNSUPPORTED_MASK_ALGORITHM", + 17179869338u64 => "UNSUPPORTED_MASK_PARAMETER", + 17179869339u64 => "UNSUPPORTED_SIGNATURE_TYPE", + 17179869331u64 => "VALUE_MISSING", + 17179869303u64 => "WRONG_SIGNATURE_LENGTH", + 227633266788u64 => "ASN1_ERROR", + 227633266789u64 => "BAD_SIGNATURE", + 227633266795u64 => "BUFFER_TOO_SMALL", + 227633266798u64 => "DIST_ID_TOO_LARGE", + 227633266800u64 => "ID_NOT_SET", + 227633266799u64 => "ID_TOO_LARGE", + 227633266796u64 => "INVALID_CURVE", + 227633266790u64 => "INVALID_DIGEST", + 227633266791u64 => "INVALID_DIGEST_TYPE", + 227633266792u64 => "INVALID_ENCODING", + 227633266793u64 => "INVALID_FIELD", + 227633266801u64 => "INVALID_PRIVATE_KEY", + 227633266797u64 => "NO_PARAMETERS_SET", + 227633266794u64 => "USER_ID_TOO_LARGE", + 85899346211u64 => "APPLICATION_DATA_AFTER_CLOSE_NOTIFY", + 85899346020u64 => "APP_DATA_IN_HANDSHAKE", + 85899346192u64 => "ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT", + 85899346078u64 => "AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", + 85899346023u64 => "BAD_CHANGE_CIPHER_SPEC", + 85899346106u64 => "BAD_CIPHER", + 85899346310u64 => "BAD_DATA", + 85899346026u64 => "BAD_DATA_RETURNED_BY_CALLBACK", + 85899346027u64 => "BAD_DECOMPRESSION", + 85899346022u64 => "BAD_DH_VALUE", + 85899346031u64 => "BAD_DIGEST_LENGTH", + 85899346153u64 => "BAD_EARLY_DATA", + 85899346224u64 => "BAD_ECC_CERT", + 85899346226u64 => "BAD_ECPOINT", + 85899346030u64 => "BAD_EXTENSION", + 85899346252u64 => "BAD_HANDSHAKE_LENGTH", + 85899346156u64 => "BAD_HANDSHAKE_STATE", + 85899346025u64 => "BAD_HELLO_REQUEST", + 85899346183u64 => "BAD_HRR_VERSION", + 85899346028u64 => "BAD_KEY_SHARE", + 85899346042u64 => "BAD_KEY_UPDATE", + 85899346212u64 => "BAD_LEGACY_VERSION", + 85899346191u64 => "BAD_LENGTH", + 85899346160u64 => "BAD_PACKET", + 85899346035u64 => "BAD_PACKET_LENGTH", + 85899346036u64 => "BAD_PROTOCOL_VERSION_NUMBER", + 85899346139u64 => "BAD_PSK", + 85899346034u64 => "BAD_PSK_IDENTITY", + 85899346363u64 => "BAD_RECORD_TYPE", + 85899346039u64 => "BAD_RSA_ENCRYPT", + 85899346043u64 => "BAD_SIGNATURE", + 85899346267u64 => "BAD_SRP_A_LENGTH", + 85899346291u64 => "BAD_SRP_PARAMETERS", + 85899346272u64 => "BAD_SRTP_MKI_VALUE", + 85899346273u64 => "BAD_SRTP_PROTECTION_PROFILE_LIST", + 85899346044u64 => "BAD_SSL_FILETYPE", + 85899346304u64 => "BAD_VALUE", + 85899346047u64 => "BAD_WRITE_RETRY", + 85899346173u64 => "BINDER_DOES_NOT_VERIFY", + 85899346048u64 => "BIO_NOT_SET", + 85899346049u64 => "BLOCK_CIPHER_PAD_IS_WRONG", + 85899346050u64 => "BN_LIB", + 85899346154u64 => "CALLBACK_FAILED", + 85899346029u64 => "CANNOT_CHANGE_CIPHER", + 85899346219u64 => "CANNOT_GET_GROUP_NAME", + 85899346051u64 => "CA_DN_LENGTH_MISMATCH", + 85899346317u64 => "CA_KEY_TOO_SMALL", + 85899346318u64 => "CA_MD_TOO_WEAK", + 85899346053u64 => "CCS_RECEIVED_EARLY", + 85899346054u64 => "CERTIFICATE_VERIFY_FAILED", + 85899346297u64 => "CERT_CB_ERROR", + 85899346055u64 => "CERT_LENGTH_MISMATCH", + 85899346138u64 => "CIPHERSUITE_DIGEST_HAS_CHANGED", + 85899346057u64 => "CIPHER_CODE_WRONG_LENGTH", + 85899346146u64 => "CLIENTHELLO_TLSEXT", + 85899346060u64 => "COMPRESSED_LENGTH_TOO_LONG", + 85899346263u64 => "COMPRESSION_DISABLED", + 85899346061u64 => "COMPRESSION_FAILURE", + 85899346227u64 => "COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE", + 85899346062u64 => "COMPRESSION_LIBRARY_ERROR", + 85899346064u64 => "CONNECTION_TYPE_NOT_SET", + 85899346087u64 => "CONTEXT_NOT_DANE_ENABLED", + 85899346320u64 => "COOKIE_GEN_CALLBACK_FAILURE", + 85899346228u64 => "COOKIE_MISMATCH", + 85899346216u64 => "COPY_PARAMETERS_FAILED", + 85899346126u64 => "CUSTOM_EXT_HANDLER_ALREADY_INSTALLED", + 85899346092u64 => "DANE_ALREADY_ENABLED", + 85899346093u64 => "DANE_CANNOT_OVERRIDE_MTYPE_FULL", + 85899346095u64 => "DANE_NOT_ENABLED", + 85899346100u64 => "DANE_TLSA_BAD_CERTIFICATE", + 85899346104u64 => "DANE_TLSA_BAD_CERTIFICATE_USAGE", + 85899346109u64 => "DANE_TLSA_BAD_DATA_LENGTH", + 85899346112u64 => "DANE_TLSA_BAD_DIGEST_LENGTH", + 85899346120u64 => "DANE_TLSA_BAD_MATCHING_TYPE", + 85899346121u64 => "DANE_TLSA_BAD_PUBLIC_KEY", + 85899346122u64 => "DANE_TLSA_BAD_SELECTOR", + 85899346123u64 => "DANE_TLSA_NULL_DATA", + 85899346065u64 => "DATA_BETWEEN_CCS_AND_FINISHED", + 85899346066u64 => "DATA_LENGTH_TOO_LONG", + 85899346067u64 => "DECRYPTION_FAILED", + 85899346201u64 => "DECRYPTION_FAILED_OR_BAD_RECORD_MAC", + 85899346314u64 => "DH_KEY_TOO_SMALL", + 85899346068u64 => "DH_PUBLIC_VALUE_LENGTH_IS_WRONG", + 85899346069u64 => "DIGEST_CHECK_FAILED", + 85899346254u64 => "DTLS_MESSAGE_TOO_BIG", + 85899346229u64 => "DUPLICATE_COMPRESSION_ID", + 85899346238u64 => "ECC_CERT_NOT_FOR_SIGNING", + 85899346294u64 => "ECDH_REQUIRED_FOR_SUITEB_MODE", + 85899346319u64 => "EE_KEY_TOO_SMALL", + 85899346274u64 => "EMPTY_SRTP_PROTECTION_PROFILE_LIST", + 85899346070u64 => "ENCRYPTED_LENGTH_TOO_LONG", + 85899346071u64 => "ERROR_IN_RECEIVED_CIPHER_LIST", + 85899346124u64 => "ERROR_SETTING_TLSA_BASE_DOMAIN", + 85899346114u64 => "EXCEEDS_MAX_FRAGMENT_SIZE", + 85899346072u64 => "EXCESSIVE_MESSAGE_SIZE", + 85899346199u64 => "EXTENSION_NOT_RECEIVED", + 85899346073u64 => "EXTRA_DATA_IN_MESSAGE", + 85899346083u64 => "EXT_LENGTH_MISMATCH", + 85899346325u64 => "FAILED_TO_INIT_ASYNC", + 85899346321u64 => "FRAGMENTED_CLIENT_HELLO", + 85899346074u64 => "GOT_A_FIN_BEFORE_A_CCS", + 85899346075u64 => "HTTPS_PROXY_REQUEST", + 85899346076u64 => "HTTP_REQUEST", + 85899346082u64 => "ILLEGAL_POINT_COMPRESSION", + 85899346300u64 => "ILLEGAL_SUITEB_DIGEST", + 85899346293u64 => "INAPPROPRIATE_FALLBACK", + 85899346260u64 => "INCONSISTENT_COMPRESSION", + 85899346142u64 => "INCONSISTENT_EARLY_DATA_ALPN", + 85899346151u64 => "INCONSISTENT_EARLY_DATA_SNI", + 85899346024u64 => "INCONSISTENT_EXTMS", + 85899346161u64 => "INSUFFICIENT_SECURITY", + 85899346125u64 => "INVALID_ALERT", + 85899346180u64 => "INVALID_CCS_MESSAGE", + 85899346158u64 => "INVALID_CERTIFICATE_OR_ALG", + 85899346200u64 => "INVALID_COMMAND", + 85899346261u64 => "INVALID_COMPRESSION_ALGORITHM", + 85899346203u64 => "INVALID_CONFIG", + 85899346033u64 => "INVALID_CONFIGURATION_NAME", + 85899346202u64 => "INVALID_CONTEXT", + 85899346132u64 => "INVALID_CT_VALIDATION_TYPE", + 85899346040u64 => "INVALID_KEY_UPDATE_TYPE", + 85899346094u64 => "INVALID_MAX_EARLY_DATA", + 85899346305u64 => "INVALID_NULL_CMD_NAME", + 85899346322u64 => "INVALID_SEQUENCE_NUMBER", + 85899346308u64 => "INVALID_SERVERINFO_DATA", + 85899346919u64 => "INVALID_SESSION_ID", + 85899346277u64 => "INVALID_SRP_USERNAME", + 85899346248u64 => "INVALID_STATUS_RESPONSE", + 85899346245u64 => "INVALID_TICKET_KEYS_LENGTH", + 85899346253u64 => "LEGACY_SIGALG_DISALLOWED_OR_UNSUPPORTED", + 85899346079u64 => "LENGTH_MISMATCH", + 85899346324u64 => "LENGTH_TOO_LONG", + 85899346080u64 => "LENGTH_TOO_SHORT", + 85899346194u64 => "LIBRARY_BUG", + 85899346081u64 => "LIBRARY_HAS_NO_CIPHERS", + 85899346085u64 => "MISSING_DSA_SIGNING_CERT", + 85899346301u64 => "MISSING_ECDSA_SIGNING_CERT", + 85899346176u64 => "MISSING_FATAL", + 85899346210u64 => "MISSING_PARAMETERS", + 85899346230u64 => "MISSING_PSK_KEX_MODES_EXTENSION", + 85899346088u64 => "MISSING_RSA_CERTIFICATE", + 85899346089u64 => "MISSING_RSA_ENCRYPTING_CERT", + 85899346090u64 => "MISSING_RSA_SIGNING_CERT", + 85899346032u64 => "MISSING_SIGALGS_EXTENSION", + 85899346141u64 => "MISSING_SIGNING_CERT", + 85899346278u64 => "MISSING_SRP_PARAM", + 85899346129u64 => "MISSING_SUPPORTED_GROUPS_EXTENSION", + 85899346091u64 => "MISSING_TMP_DH_KEY", + 85899346231u64 => "MISSING_TMP_ECDH_KEY", + 85899346213u64 => "MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA", + 85899346102u64 => "NOT_ON_RECORD_BOUNDARY", + 85899346209u64 => "NOT_REPLACING_CERTIFICATE", + 85899346204u64 => "NOT_SERVER", + 85899346155u64 => "NO_APPLICATION_PROTOCOL", + 85899346096u64 => "NO_CERTIFICATES_RETURNED", + 85899346097u64 => "NO_CERTIFICATE_ASSIGNED", + 85899346099u64 => "NO_CERTIFICATE_SET", + 85899346134u64 => "NO_CHANGE_FOLLOWING_HRR", + 85899346101u64 => "NO_CIPHERS_AVAILABLE", + 85899346103u64 => "NO_CIPHERS_SPECIFIED", + 85899346105u64 => "NO_CIPHER_MATCH", + 85899346251u64 => "NO_CLIENT_CERT_METHOD", + 85899346107u64 => "NO_COMPRESSION_SPECIFIED", + 85899346207u64 => "NO_COOKIE_CALLBACK_SET", + 85899346250u64 => "NO_GOST_CERTIFICATE_SENT_BY_PEER", + 85899346108u64 => "NO_METHOD_SPECIFIED", + 85899346309u64 => "NO_PEM_EXTENSIONS", + 85899346110u64 => "NO_PRIVATE_KEY_ASSIGNED", + 85899346111u64 => "NO_PROTOCOLS_AVAILABLE", + 85899346259u64 => "NO_RENEGOTIATION", + 85899346244u64 => "NO_REQUIRED_DIGEST", + 85899346113u64 => "NO_SHARED_CIPHER", + 85899346330u64 => "NO_SHARED_GROUPS", + 85899346296u64 => "NO_SHARED_SIGNATURE_ALGORITHMS", + 85899346279u64 => "NO_SRTP_PROFILES", + 85899346217u64 => "NO_SUITABLE_DIGEST_ALGORITHM", + 85899346215u64 => "NO_SUITABLE_GROUPS", + 85899346021u64 => "NO_SUITABLE_KEY_SHARE", + 85899346038u64 => "NO_SUITABLE_SIGNATURE_ALGORITHM", + 85899346136u64 => "NO_VALID_SCTS", + 85899346323u64 => "NO_VERIFY_COOKIE_CALLBACK", + 85899346115u64 => "NULL_SSL_CTX", + 85899346116u64 => "NULL_SSL_METHOD_PASSED", + 85899346225u64 => "OCSP_CALLBACK_FAILURE", + 85899346117u64 => "OLD_SESSION_CIPHER_NOT_RETURNED", + 85899346264u64 => "OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED", + 85899346157u64 => "OVERFLOW_ERROR", + 85899346118u64 => "PACKET_LENGTH_TOO_LONG", + 85899346147u64 => "PARSE_TLSEXT", + 85899346190u64 => "PATH_TOO_LONG", + 85899346119u64 => "PEER_DID_NOT_RETURN_A_CERTIFICATE", + 85899346311u64 => "PEM_NAME_BAD_PREFIX", + 85899346312u64 => "PEM_NAME_TOO_SHORT", + 85899346326u64 => "PIPELINE_FAILURE", + 85899346198u64 => "POST_HANDSHAKE_AUTH_ENCODING_ERR", + 85899346208u64 => "PRIVATE_KEY_MISMATCH", + 85899346127u64 => "PROTOCOL_IS_SHUTDOWN", + 85899346143u64 => "PSK_IDENTITY_NOT_FOUND", + 85899346144u64 => "PSK_NO_CLIENT_CB", + 85899346145u64 => "PSK_NO_SERVER_CB", + 85899346131u64 => "READ_BIO_NOT_SET", + 85899346232u64 => "READ_TIMEOUT_EXPIRED", + 85899346133u64 => "RECORD_LENGTH_MISMATCH", + 85899346218u64 => "RECORD_TOO_SMALL", + 85899346255u64 => "RENEGOTIATE_EXT_TOO_LONG", + 85899346256u64 => "RENEGOTIATION_ENCODING_ERR", + 85899346257u64 => "RENEGOTIATION_MISMATCH", + 85899346205u64 => "REQUEST_PENDING", + 85899346206u64 => "REQUEST_SENT", + 85899346135u64 => "REQUIRED_CIPHER_MISSING", + 85899346262u64 => "REQUIRED_COMPRESSION_ALGORITHM_MISSING", + 85899346265u64 => "SCSV_RECEIVED_WHEN_RENEGOTIATING", + 85899346128u64 => "SCT_VERIFICATION_FAILED", + 85899346195u64 => "SERVERHELLO_TLSEXT", + 85899346197u64 => "SESSION_ID_CONTEXT_UNINITIALIZED", + 85899346327u64 => "SHUTDOWN_WHILE_IN_INIT", + 85899346280u64 => "SIGNATURE_ALGORITHMS_ERROR", + 85899346140u64 => "SIGNATURE_FOR_NON_SIGNING_CERTIFICATE", + 85899346281u64 => "SRP_A_CALC", + 85899346282u64 => "SRTP_COULD_NOT_ALLOCATE_PROFILES", + 85899346283u64 => "SRTP_PROTECTION_PROFILE_LIST_TOO_LONG", + 85899346284u64 => "SRTP_UNKNOWN_PROTECTION_PROFILE", + 85899346152u64 => "SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH", + 85899346239u64 => "SSL3_EXT_INVALID_SERVERNAME", + 85899346240u64 => "SSL3_EXT_INVALID_SERVERNAME_TYPE", + 85899346220u64 => "SSL3_SESSION_ID_TOO_LONG", + 85899346962u64 => "SSLV3_ALERT_BAD_CERTIFICATE", + 85899346940u64 => "SSLV3_ALERT_BAD_RECORD_MAC", + 85899346965u64 => "SSLV3_ALERT_CERTIFICATE_EXPIRED", + 85899346964u64 => "SSLV3_ALERT_CERTIFICATE_REVOKED", + 85899346966u64 => "SSLV3_ALERT_CERTIFICATE_UNKNOWN", + 85899346950u64 => "SSLV3_ALERT_DECOMPRESSION_FAILURE", + 85899346960u64 => "SSLV3_ALERT_HANDSHAKE_FAILURE", + 85899346967u64 => "SSLV3_ALERT_ILLEGAL_PARAMETER", + 85899346961u64 => "SSLV3_ALERT_NO_CERTIFICATE", + 85899346930u64 => "SSLV3_ALERT_UNEXPECTED_MESSAGE", + 85899346963u64 => "SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", + 85899346037u64 => "SSL_COMMAND_SECTION_EMPTY", + 85899346045u64 => "SSL_COMMAND_SECTION_NOT_FOUND", + 85899346148u64 => "SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION", + 85899346149u64 => "SSL_HANDSHAKE_FAILURE", + 85899346150u64 => "SSL_LIBRARY_HAS_NO_CIPHERS", + 85899346292u64 => "SSL_NEGATIVE_LENGTH", + 85899346046u64 => "SSL_SECTION_EMPTY", + 85899346056u64 => "SSL_SECTION_NOT_FOUND", + 85899346221u64 => "SSL_SESSION_ID_CALLBACK_FAILED", + 85899346222u64 => "SSL_SESSION_ID_CONFLICT", + 85899346193u64 => "SSL_SESSION_ID_CONTEXT_TOO_LONG", + 85899346223u64 => "SSL_SESSION_ID_HAS_BAD_LENGTH", + 85899346328u64 => "SSL_SESSION_ID_TOO_LONG", + 85899346130u64 => "SSL_SESSION_VERSION_MISMATCH", + 85899346041u64 => "STILL_IN_INIT", + 85899347036u64 => "TLSV13_ALERT_CERTIFICATE_REQUIRED", + 85899347029u64 => "TLSV13_ALERT_MISSING_EXTENSION", + 85899346969u64 => "TLSV1_ALERT_ACCESS_DENIED", + 85899346970u64 => "TLSV1_ALERT_DECODE_ERROR", + 85899346941u64 => "TLSV1_ALERT_DECRYPTION_FAILED", + 85899346971u64 => "TLSV1_ALERT_DECRYPT_ERROR", + 85899346980u64 => "TLSV1_ALERT_EXPORT_RESTRICTION", + 85899347006u64 => "TLSV1_ALERT_INAPPROPRIATE_FALLBACK", + 85899346991u64 => "TLSV1_ALERT_INSUFFICIENT_SECURITY", + 85899347000u64 => "TLSV1_ALERT_INTERNAL_ERROR", + 85899347040u64 => "TLSV1_ALERT_NO_APPLICATION_PROTOCOL", + 85899347020u64 => "TLSV1_ALERT_NO_RENEGOTIATION", + 85899346990u64 => "TLSV1_ALERT_PROTOCOL_VERSION", + 85899346942u64 => "TLSV1_ALERT_RECORD_OVERFLOW", + 85899346968u64 => "TLSV1_ALERT_UNKNOWN_CA", + 85899347035u64 => "TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", + 85899347010u64 => "TLSV1_ALERT_USER_CANCELLED", + 85899347034u64 => "TLSV1_BAD_CERTIFICATE_HASH_VALUE", + 85899347033u64 => "TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", + 85899347031u64 => "TLSV1_CERTIFICATE_UNOBTAINABLE", + 85899347032u64 => "TLSV1_UNRECOGNIZED_NAME", + 85899347030u64 => "TLSV1_UNSUPPORTED_EXTENSION", + 85899346287u64 => "TLS_ILLEGAL_EXPORTER_LABEL", + 85899346077u64 => "TLS_INVALID_ECPOINTFORMAT_LIST", + 85899346052u64 => "TOO_MANY_KEY_UPDATES", + 85899346329u64 => "TOO_MANY_WARN_ALERTS", + 85899346084u64 => "TOO_MUCH_EARLY_DATA", + 85899346234u64 => "UNABLE_TO_FIND_ECDH_PARAMETERS", + 85899346159u64 => "UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS", + 85899346162u64 => "UNABLE_TO_LOAD_SSL3_MD5_ROUTINES", + 85899346163u64 => "UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES", + 85899346182u64 => "UNEXPECTED_CCS_MESSAGE", + 85899346098u64 => "UNEXPECTED_END_OF_EARLY_DATA", + 85899346214u64 => "UNEXPECTED_EOF_WHILE_READING", + 85899346164u64 => "UNEXPECTED_MESSAGE", + 85899346165u64 => "UNEXPECTED_RECORD", + 85899346196u64 => "UNINITIALIZED", + 85899346166u64 => "UNKNOWN_ALERT_TYPE", + 85899346167u64 => "UNKNOWN_CERTIFICATE_TYPE", + 85899346168u64 => "UNKNOWN_CIPHER_RETURNED", + 85899346169u64 => "UNKNOWN_CIPHER_TYPE", + 85899346306u64 => "UNKNOWN_CMD_NAME", + 85899346059u64 => "UNKNOWN_COMMAND", + 85899346288u64 => "UNKNOWN_DIGEST", + 85899346170u64 => "UNKNOWN_KEY_EXCHANGE_TYPE", + 85899346171u64 => "UNKNOWN_PKEY_TYPE", + 85899346172u64 => "UNKNOWN_PROTOCOL", + 85899346174u64 => "UNKNOWN_SSL_VERSION", + 85899346175u64 => "UNKNOWN_STATE", + 85899346258u64 => "UNSAFE_LEGACY_RENEGOTIATION_DISABLED", + 85899346137u64 => "UNSOLICITED_EXTENSION", + 85899346177u64 => "UNSUPPORTED_COMPRESSION_ALGORITHM", + 85899346235u64 => "UNSUPPORTED_ELLIPTIC_CURVE", + 85899346178u64 => "UNSUPPORTED_PROTOCOL", + 85899346179u64 => "UNSUPPORTED_SSL_VERSION", + 85899346249u64 => "UNSUPPORTED_STATUS_TYPE", + 85899346289u64 => "USE_SRTP_NOT_NEGOTIATED", + 85899346086u64 => "VERSION_TOO_HIGH", + 85899346316u64 => "VERSION_TOO_LOW", + 85899346303u64 => "WRONG_CERTIFICATE_TYPE", + 85899346181u64 => "WRONG_CIPHER_RETURNED", + 85899346298u64 => "WRONG_CURVE", + 85899346184u64 => "WRONG_SIGNATURE_LENGTH", + 85899346185u64 => "WRONG_SIGNATURE_SIZE", + 85899346290u64 => "WRONG_SIGNATURE_TYPE", + 85899346186u64 => "WRONG_SSL_VERSION", + 85899346187u64 => "WRONG_VERSION_NUMBER", + 85899346188u64 => "X509_LIB", + 85899346189u64 => "X509_VERIFICATION_SETUP_PROBLEMS", + 201863463044u64 => "BAD_PKCS7_TYPE", + 201863463045u64 => "BAD_TYPE", + 201863463049u64 => "CANNOT_LOAD_CERT", + 201863463050u64 => "CANNOT_LOAD_KEY", + 201863463012u64 => "CERTIFICATE_VERIFY_ERROR", + 201863463039u64 => "COULD_NOT_SET_ENGINE", + 201863463027u64 => "COULD_NOT_SET_TIME", + 201863463046u64 => "DETACHED_CONTENT", + 201863463028u64 => "ESS_ADD_SIGNING_CERT_ERROR", + 201863463051u64 => "ESS_ADD_SIGNING_CERT_V2_ERROR", + 201863463013u64 => "ESS_SIGNING_CERTIFICATE_ERROR", + 201863463014u64 => "INVALID_NULL_POINTER", + 201863463029u64 => "INVALID_SIGNER_CERTIFICATE_PURPOSE", + 201863463015u64 => "MESSAGE_IMPRINT_MISMATCH", + 201863463016u64 => "NONCE_MISMATCH", + 201863463017u64 => "NONCE_NOT_RETURNED", + 201863463018u64 => "NO_CONTENT", + 201863463019u64 => "NO_TIME_STAMP_TOKEN", + 201863463030u64 => "PKCS7_ADD_SIGNATURE_ERROR", + 201863463031u64 => "PKCS7_ADD_SIGNED_ATTR_ERROR", + 201863463041u64 => "PKCS7_TO_TS_TST_INFO_FAILED", + 201863463020u64 => "POLICY_MISMATCH", + 201863463032u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 201863463033u64 => "RESPONSE_SETUP_ERROR", + 201863463021u64 => "SIGNATURE_FAILURE", + 201863463022u64 => "THERE_MUST_BE_ONE_SIGNER", + 201863463034u64 => "TIME_SYSCALL_ERROR", + 201863463042u64 => "TOKEN_NOT_PRESENT", + 201863463043u64 => "TOKEN_PRESENT", + 201863463023u64 => "TSA_NAME_MISMATCH", + 201863463024u64 => "TSA_UNTRUSTED", + 201863463035u64 => "TST_INFO_SETUP_ERROR", + 201863463036u64 => "TS_DATASIGN", + 201863463037u64 => "UNACCEPTABLE_POLICY", + 201863463038u64 => "UNSUPPORTED_MD_ALGORITHM", + 201863463025u64 => "UNSUPPORTED_VERSION", + 201863463047u64 => "VAR_BAD_VALUE", + 201863463048u64 => "VAR_LOOKUP_FAILURE", + 201863463026u64 => "WRONG_CONTENT_TYPE", + 171798691944u64 => "COMMON_OK_AND_CANCEL_CHARACTERS", + 171798691942u64 => "INDEX_TOO_LARGE", + 171798691943u64 => "INDEX_TOO_SMALL", + 171798691945u64 => "NO_RESULT_BUFFER", + 171798691947u64 => "PROCESSING_ERROR", + 171798691940u64 => "RESULT_TOO_LARGE", + 171798691941u64 => "RESULT_TOO_SMALL", + 171798691949u64 => "SYSASSIGN_ERROR", + 171798691950u64 => "SYSDASSGN_ERROR", + 171798691951u64 => "SYSQIOW_ERROR", + 171798691946u64 => "UNKNOWN_CONTROL_COMMAND", + 171798691948u64 => "UNKNOWN_TTYGET_ERRNO_VALUE", + 171798691952u64 => "USER_DATA_DUPLICATION_UNSUPPORTED", + 146028888182u64 => "BAD_IP_ADDRESS", + 146028888183u64 => "BAD_OBJECT", + 146028888164u64 => "BN_DEC2BN_ERROR", + 146028888165u64 => "BN_TO_ASN1_INTEGER_ERROR", + 146028888213u64 => "DIRNAME_ERROR", + 146028888224u64 => "DISTPOINT_ALREADY_SET", + 146028888197u64 => "DUPLICATE_ZONE_ID", + 146028888233u64 => "EMPTY_KEY_USAGE", + 146028888195u64 => "ERROR_CONVERTING_ZONE", + 146028888208u64 => "ERROR_CREATING_EXTENSION", + 146028888192u64 => "ERROR_IN_EXTENSION", + 146028888201u64 => "EXPECTED_A_SECTION_NAME", + 146028888209u64 => "EXTENSION_EXISTS", + 146028888179u64 => "EXTENSION_NAME_ERROR", + 146028888166u64 => "EXTENSION_NOT_FOUND", + 146028888167u64 => "EXTENSION_SETTING_NOT_SUPPORTED", + 146028888180u64 => "EXTENSION_VALUE_ERROR", + 146028888215u64 => "ILLEGAL_EMPTY_EXTENSION", + 146028888216u64 => "INCORRECT_POLICY_SYNTAX_TAG", + 146028888226u64 => "INVALID_ASNUMBER", + 146028888227u64 => "INVALID_ASRANGE", + 146028888168u64 => "INVALID_BOOLEAN_STRING", + 146028888222u64 => "INVALID_CERTIFICATE", + 146028888172u64 => "INVALID_EMPTY_NAME", + 146028888169u64 => "INVALID_EXTENSION_STRING", + 146028888229u64 => "INVALID_INHERITANCE", + 146028888230u64 => "INVALID_IPADDRESS", + 146028888225u64 => "INVALID_MULTIPLE_RDNS", + 146028888170u64 => "INVALID_NAME", + 146028888171u64 => "INVALID_NULL_ARGUMENT", + 146028888173u64 => "INVALID_NULL_VALUE", + 146028888204u64 => "INVALID_NUMBER", + 146028888205u64 => "INVALID_NUMBERS", + 146028888174u64 => "INVALID_OBJECT_IDENTIFIER", + 146028888202u64 => "INVALID_OPTION", + 146028888198u64 => "INVALID_POLICY_IDENTIFIER", + 146028888217u64 => "INVALID_PROXY_POLICY_SETTING", + 146028888210u64 => "INVALID_PURPOSE", + 146028888228u64 => "INVALID_SAFI", + 146028888199u64 => "INVALID_SECTION", + 146028888207u64 => "INVALID_SYNTAX", + 146028888190u64 => "ISSUER_DECODE_ERROR", + 146028888188u64 => "MISSING_VALUE", + 146028888206u64 => "NEED_ORGANIZATION_AND_NUMBERS", + 146028888232u64 => "NEGATIVE_PATHLEN", + 146028888200u64 => "NO_CONFIG_DATABASE", + 146028888185u64 => "NO_ISSUER_CERTIFICATE", + 146028888191u64 => "NO_ISSUER_DETAILS", + 146028888203u64 => "NO_POLICY_IDENTIFIER", + 146028888218u64 => "NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED", + 146028888178u64 => "NO_PUBLIC_KEY", + 146028888189u64 => "NO_SUBJECT_DETAILS", + 146028888212u64 => "OPERATION_NOT_DEFINED", + 146028888211u64 => "OTHERNAME_ERROR", + 146028888219u64 => "POLICY_LANGUAGE_ALREADY_DEFINED", + 146028888220u64 => "POLICY_PATH_LENGTH", + 146028888221u64 => "POLICY_PATH_LENGTH_ALREADY_DEFINED", + 146028888223u64 => "POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", + 146028888214u64 => "SECTION_NOT_FOUND", + 146028888186u64 => "UNABLE_TO_GET_ISSUER_DETAILS", + 146028888187u64 => "UNABLE_TO_GET_ISSUER_KEYID", + 146028888175u64 => "UNKNOWN_BIT_STRING_ARGUMENT", + 146028888193u64 => "UNKNOWN_EXTENSION", + 146028888194u64 => "UNKNOWN_EXTENSION_NAME", + 146028888184u64 => "UNKNOWN_OPTION", + 146028888181u64 => "UNSUPPORTED_OPTION", + 146028888231u64 => "UNSUPPORTED_TYPE", + 146028888196u64 => "USER_TOO_LONG", + 47244640366u64 => "AKID_MISMATCH", + 47244640389u64 => "BAD_SELECTOR", + 47244640356u64 => "BAD_X509_FILETYPE", + 47244640374u64 => "BASE64_DECODE_ERROR", + 47244640370u64 => "CANT_CHECK_DH_KEY", + 47244640395u64 => "CERTIFICATE_VERIFICATION_FAILED", + 47244640357u64 => "CERT_ALREADY_IN_HASH_TABLE", + 47244640383u64 => "CRL_ALREADY_DELTA", + 47244640387u64 => "CRL_VERIFY_FAILURE", + 47244640396u64 => "DUPLICATE_ATTRIBUTE", + 47244640397u64 => "ERROR_GETTING_MD_BY_NID", + 47244640398u64 => "ERROR_USING_SIGINF_SET", + 47244640384u64 => "IDP_MISMATCH", + 47244640394u64 => "INVALID_ATTRIBUTES", + 47244640369u64 => "INVALID_DIRECTORY", + 47244640399u64 => "INVALID_DISTPOINT", + 47244640375u64 => "INVALID_FIELD_NAME", + 47244640379u64 => "INVALID_TRUST", + 47244640385u64 => "ISSUER_MISMATCH", + 47244640371u64 => "KEY_TYPE_MISMATCH", + 47244640372u64 => "KEY_VALUES_MISMATCH", + 47244640359u64 => "LOADING_CERT_DIR", + 47244640360u64 => "LOADING_DEFAULTS", + 47244640380u64 => "METHOD_NOT_SUPPORTED", + 47244640390u64 => "NAME_TOO_LONG", + 47244640388u64 => "NEWER_CRL_NOT_NEWER", + 47244640391u64 => "NO_CERTIFICATE_FOUND", + 47244640392u64 => "NO_CERTIFICATE_OR_CRL_FOUND", + 47244640361u64 => "NO_CERT_SET_FOR_US_TO_VERIFY", + 47244640393u64 => "NO_CRL_FOUND", + 47244640386u64 => "NO_CRL_NUMBER", + 47244640381u64 => "PUBLIC_KEY_DECODE_ERROR", + 47244640382u64 => "PUBLIC_KEY_ENCODE_ERROR", + 47244640362u64 => "SHOULD_RETRY", + 47244640363u64 => "UNABLE_TO_FIND_PARAMETERS_IN_CHAIN", + 47244640364u64 => "UNABLE_TO_GET_CERTS_PUBLIC_KEY", + 47244640373u64 => "UNKNOWN_KEY_TYPE", + 47244640365u64 => "UNKNOWN_NID", + 47244640377u64 => "UNKNOWN_PURPOSE_ID", + 47244640400u64 => "UNKNOWN_SIGID_ALGS", + 47244640376u64 => "UNKNOWN_TRUST_ID", + 47244640367u64 => "UNSUPPORTED_ALGORITHM", + 47244640368u64 => "WRONG_LOOKUP_TYPE", + 47244640378u64 => "WRONG_TYPE", +}; + +/// Helper function to create encoded key from (lib, reason) pair +#[inline] +pub fn encode_error_key(lib: i32, reason: i32) -> u64 { + ((lib as u64) << 32) | (reason as u64 & 0xFFFFFFFF) +} diff --git a/crates/stdlib/src/openssl/ssl_data_31.rs b/crates/stdlib/src/openssl/ssl_data_31.rs new file mode 100644 index 00000000000..4b80de539ec --- /dev/null +++ b/crates/stdlib/src/openssl/ssl_data_31.rs @@ -0,0 +1,1770 @@ +// File generated by tools/make_ssl_data_rs.py +// Generated on 2025-10-29T07:17:16.621749+00:00 +// Source: OpenSSL from /tmp/openssl-3.1 +// spell-checker: disable + +use phf::phf_map; + +// Maps lib_code -> library name +// Example: 20 -> "SSL" +pub static LIBRARY_CODES: phf::Map<u32, &'static str> = phf_map! { + 0u32 => "MASK", + 1u32 => "NONE", + 2u32 => "SYS", + 3u32 => "BN", + 4u32 => "RSA", + 5u32 => "DH", + 6u32 => "EVP", + 7u32 => "BUF", + 8u32 => "OBJ", + 9u32 => "PEM", + 10u32 => "DSA", + 11u32 => "X509", + 12u32 => "METH", + 13u32 => "ASN1", + 14u32 => "CONF", + 15u32 => "CRYPTO", + 16u32 => "EC", + 20u32 => "SSL", + 21u32 => "SSL23", + 22u32 => "SSL2", + 23u32 => "SSL3", + 30u32 => "RSAREF", + 31u32 => "PROXY", + 32u32 => "BIO", + 33u32 => "PKCS7", + 34u32 => "X509V3", + 35u32 => "PKCS12", + 36u32 => "RAND", + 37u32 => "DSO", + 38u32 => "ENGINE", + 39u32 => "OCSP", + 40u32 => "UI", + 41u32 => "COMP", + 42u32 => "ECDSA", + 43u32 => "ECDH", + 44u32 => "OSSL_STORE", + 45u32 => "FIPS", + 46u32 => "CMS", + 47u32 => "TS", + 48u32 => "HMAC", + 49u32 => "JPAKE", + 50u32 => "CT", + 51u32 => "ASYNC", + 52u32 => "KDF", + 53u32 => "SM2", + 54u32 => "ESS", + 55u32 => "PROP", + 56u32 => "CRMF", + 57u32 => "PROV", + 58u32 => "CMP", + 59u32 => "OSSL_ENCODER", + 60u32 => "OSSL_DECODER", + 61u32 => "HTTP", + 128u32 => "USER", +}; + +// Maps encoded (lib, reason) -> error mnemonic +// Example: encode_error_key(20, 134) -> "CERTIFICATE_VERIFY_FAILED" +// Key encoding: (lib << 32) | reason +pub static ERROR_CODES: phf::Map<u64, &'static str> = phf_map! { + 55834575019u64 => "ADDING_OBJECT", + 55834575051u64 => "ASN1_PARSE_ERROR", + 55834575052u64 => "ASN1_SIG_PARSE_ERROR", + 55834574948u64 => "AUX_ERROR", + 55834574950u64 => "BAD_OBJECT_HEADER", + 55834575078u64 => "BAD_TEMPLATE", + 55834575062u64 => "BMPSTRING_IS_WRONG_LENGTH", + 55834574953u64 => "BN_LIB", + 55834574954u64 => "BOOLEAN_IS_WRONG_LENGTH", + 55834574955u64 => "BUFFER_TOO_SMALL", + 55834574956u64 => "CIPHER_HAS_NO_OBJECT_IDENTIFIER", + 55834575065u64 => "CONTEXT_NOT_INITIALISED", + 55834574957u64 => "DATA_IS_WRONG", + 55834574958u64 => "DECODE_ERROR", + 55834575022u64 => "DEPTH_EXCEEDED", + 55834575046u64 => "DIGEST_AND_KEY_TYPE_NOT_SUPPORTED", + 55834574960u64 => "ENCODE_ERROR", + 55834575021u64 => "ERROR_GETTING_TIME", + 55834575020u64 => "ERROR_LOADING_SECTION", + 55834574962u64 => "ERROR_SETTING_CIPHER_PARAMS", + 55834574963u64 => "EXPECTING_AN_INTEGER", + 55834574964u64 => "EXPECTING_AN_OBJECT", + 55834574967u64 => "EXPLICIT_LENGTH_MISMATCH", + 55834574968u64 => "EXPLICIT_TAG_NOT_CONSTRUCTED", + 55834574969u64 => "FIELD_MISSING", + 55834574970u64 => "FIRST_NUM_TOO_LARGE", + 55834574971u64 => "HEADER_TOO_LONG", + 55834575023u64 => "ILLEGAL_BITSTRING_FORMAT", + 55834575024u64 => "ILLEGAL_BOOLEAN", + 55834574972u64 => "ILLEGAL_CHARACTERS", + 55834575025u64 => "ILLEGAL_FORMAT", + 55834575026u64 => "ILLEGAL_HEX", + 55834575027u64 => "ILLEGAL_IMPLICIT_TAG", + 55834575028u64 => "ILLEGAL_INTEGER", + 55834575074u64 => "ILLEGAL_NEGATIVE_VALUE", + 55834575029u64 => "ILLEGAL_NESTED_TAGGING", + 55834574973u64 => "ILLEGAL_NULL", + 55834575030u64 => "ILLEGAL_NULL_VALUE", + 55834575031u64 => "ILLEGAL_OBJECT", + 55834574974u64 => "ILLEGAL_OPTIONAL_ANY", + 55834575018u64 => "ILLEGAL_OPTIONS_ON_ITEM_TEMPLATE", + 55834575069u64 => "ILLEGAL_PADDING", + 55834574975u64 => "ILLEGAL_TAGGED_ANY", + 55834575032u64 => "ILLEGAL_TIME_VALUE", + 55834575070u64 => "ILLEGAL_ZERO_CONTENT", + 55834575033u64 => "INTEGER_NOT_ASCII_FORMAT", + 55834574976u64 => "INTEGER_TOO_LARGE_FOR_LONG", + 55834575068u64 => "INVALID_BIT_STRING_BITS_LEFT", + 55834574977u64 => "INVALID_BMPSTRING_LENGTH", + 55834574978u64 => "INVALID_DIGIT", + 55834575053u64 => "INVALID_MIME_TYPE", + 55834575034u64 => "INVALID_MODIFIER", + 55834575035u64 => "INVALID_NUMBER", + 55834575064u64 => "INVALID_OBJECT_ENCODING", + 55834575075u64 => "INVALID_SCRYPT_PARAMETERS", + 55834574979u64 => "INVALID_SEPARATOR", + 55834575066u64 => "INVALID_STRING_TABLE_VALUE", + 55834574981u64 => "INVALID_UNIVERSALSTRING_LENGTH", + 55834574982u64 => "INVALID_UTF8STRING", + 55834575067u64 => "INVALID_VALUE", + 55834575079u64 => "LENGTH_TOO_LONG", + 55834575036u64 => "LIST_ERROR", + 55834575054u64 => "MIME_NO_CONTENT_TYPE", + 55834575055u64 => "MIME_PARSE_ERROR", + 55834575056u64 => "MIME_SIG_PARSE_ERROR", + 55834574985u64 => "MISSING_EOC", + 55834574986u64 => "MISSING_SECOND_NUMBER", + 55834575037u64 => "MISSING_VALUE", + 55834574987u64 => "MSTRING_NOT_UNIVERSAL", + 55834574988u64 => "MSTRING_WRONG_TAG", + 55834575045u64 => "NESTED_ASN1_STRING", + 55834575049u64 => "NESTED_TOO_DEEP", + 55834574989u64 => "NON_HEX_CHARACTERS", + 55834575038u64 => "NOT_ASCII_FORMAT", + 55834574990u64 => "NOT_ENOUGH_DATA", + 55834575057u64 => "NO_CONTENT_TYPE", + 55834574991u64 => "NO_MATCHING_CHOICE_TYPE", + 55834575058u64 => "NO_MULTIPART_BODY_FAILURE", + 55834575059u64 => "NO_MULTIPART_BOUNDARY", + 55834575060u64 => "NO_SIG_CONTENT_TYPE", + 55834574992u64 => "NULL_IS_WRONG_LENGTH", + 55834575039u64 => "OBJECT_NOT_ASCII_FORMAT", + 55834574993u64 => "ODD_NUMBER_OF_CHARS", + 55834574995u64 => "SECOND_NUMBER_TOO_LARGE", + 55834574996u64 => "SEQUENCE_LENGTH_MISMATCH", + 55834574997u64 => "SEQUENCE_NOT_CONSTRUCTED", + 55834575040u64 => "SEQUENCE_OR_SET_NEEDS_CONFIG", + 55834574998u64 => "SHORT_LINE", + 55834575061u64 => "SIG_INVALID_MIME_TYPE", + 55834575050u64 => "STREAMING_NOT_SUPPORTED", + 55834574999u64 => "STRING_TOO_LONG", + 55834575000u64 => "STRING_TOO_SHORT", + 55834575002u64 => "THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", + 55834575041u64 => "TIME_NOT_ASCII_FORMAT", + 55834575071u64 => "TOO_LARGE", + 55834575003u64 => "TOO_LONG", + 55834575072u64 => "TOO_SMALL", + 55834575004u64 => "TYPE_NOT_CONSTRUCTED", + 55834575043u64 => "TYPE_NOT_PRIMITIVE", + 55834575007u64 => "UNEXPECTED_EOC", + 55834575063u64 => "UNIVERSALSTRING_IS_WRONG_LENGTH", + 55834575077u64 => "UNKNOWN_DIGEST", + 55834575008u64 => "UNKNOWN_FORMAT", + 55834575009u64 => "UNKNOWN_MESSAGE_DIGEST_ALGORITHM", + 55834575010u64 => "UNKNOWN_OBJECT_TYPE", + 55834575011u64 => "UNKNOWN_PUBLIC_KEY_TYPE", + 55834575047u64 => "UNKNOWN_SIGNATURE_ALGORITHM", + 55834575042u64 => "UNKNOWN_TAG", + 55834575012u64 => "UNSUPPORTED_ANY_DEFINED_BY_TYPE", + 55834575076u64 => "UNSUPPORTED_CIPHER", + 55834575015u64 => "UNSUPPORTED_PUBLIC_KEY_TYPE", + 55834575044u64 => "UNSUPPORTED_TYPE", + 55834575073u64 => "WRONG_INTEGER_TYPE", + 55834575048u64 => "WRONG_PUBLIC_KEY_TYPE", + 55834575016u64 => "WRONG_TAG", + 219043332197u64 => "FAILED_TO_SET_POOL", + 219043332198u64 => "FAILED_TO_SWAP_CONTEXT", + 219043332201u64 => "INIT_FAILED", + 219043332199u64 => "INVALID_POOL_SIZE", + 137438953572u64 => "ACCEPT_ERROR", + 137438953613u64 => "ADDRINFO_ADDR_IS_NOT_AF_INET", + 137438953601u64 => "AMBIGUOUS_HOST_OR_SERVICE", + 137438953573u64 => "BAD_FOPEN_MODE", + 137438953596u64 => "BROKEN_PIPE", + 137438953575u64 => "CONNECT_ERROR", + 137438953619u64 => "CONNECT_TIMEOUT", + 137438953579u64 => "GETHOSTBYNAME_ADDR_IS_NOT_AF_INET", + 137438953604u64 => "GETSOCKNAME_ERROR", + 137438953605u64 => "GETSOCKNAME_TRUNCATED_ADDRESS", + 137438953606u64 => "GETTING_SOCKTYPE", + 137438953597u64 => "INVALID_ARGUMENT", + 137438953607u64 => "INVALID_SOCKET", + 137438953595u64 => "IN_USE", + 137438953574u64 => "LENGTH_TOO_LONG", + 137438953608u64 => "LISTEN_V6_ONLY", + 137438953614u64 => "LOOKUP_RETURNED_NOTHING", + 137438953602u64 => "MALFORMED_HOST_OR_SERVICE", + 137438953582u64 => "NBIO_CONNECT_ERROR", + 137438953615u64 => "NO_ACCEPT_ADDR_OR_SERVICE_SPECIFIED", + 137438953616u64 => "NO_HOSTNAME_OR_SERVICE_SPECIFIED", + 137438953585u64 => "NO_PORT_DEFINED", + 137438953600u64 => "NO_SUCH_FILE", + 137438953576u64 => "TRANSFER_ERROR", + 137438953577u64 => "TRANSFER_TIMEOUT", + 137438953589u64 => "UNABLE_TO_BIND_SOCKET", + 137438953590u64 => "UNABLE_TO_CREATE_SOCKET", + 137438953609u64 => "UNABLE_TO_KEEPALIVE", + 137438953591u64 => "UNABLE_TO_LISTEN_SOCKET", + 137438953610u64 => "UNABLE_TO_NODELAY", + 137438953611u64 => "UNABLE_TO_REUSEADDR", + 137438953617u64 => "UNAVAILABLE_IP_FAMILY", + 137438953592u64 => "UNINITIALIZED", + 137438953612u64 => "UNKNOWN_INFO_TYPE", + 137438953618u64 => "UNSUPPORTED_IP_FAMILY", + 137438953593u64 => "UNSUPPORTED_METHOD", + 137438953603u64 => "UNSUPPORTED_PROTOCOL_FAMILY", + 137438953598u64 => "WRITE_TO_READ_ONLY_BIO", + 137438953594u64 => "WSASTARTUP", + 12884901988u64 => "ARG2_LT_ARG3", + 12884901989u64 => "BAD_RECIPROCAL", + 12884902002u64 => "BIGNUM_TOO_LONG", + 12884902006u64 => "BITS_TOO_SMALL", + 12884901990u64 => "CALLED_WITH_EVEN_MODULUS", + 12884901991u64 => "DIV_BY_ZERO", + 12884901992u64 => "ENCODING_ERROR", + 12884901993u64 => "EXPAND_ON_STATIC_BIGNUM_DATA", + 12884901998u64 => "INPUT_NOT_REDUCED", + 12884901994u64 => "INVALID_LENGTH", + 12884902003u64 => "INVALID_RANGE", + 12884902007u64 => "INVALID_SHIFT", + 12884901999u64 => "NOT_A_SQUARE", + 12884901995u64 => "NOT_INITIALIZED", + 12884901996u64 => "NO_INVERSE", + 12884902009u64 => "NO_PRIME_CANDIDATE", + 12884902004u64 => "NO_SOLUTION", + 12884902008u64 => "NO_SUITABLE_DIGEST", + 12884902005u64 => "PRIVATE_KEY_TOO_LARGE", + 12884902000u64 => "P_IS_NOT_PRIME", + 12884902001u64 => "TOO_MANY_ITERATIONS", + 12884901997u64 => "TOO_MANY_TEMPORARY_VARIABLES", + 249108103307u64 => "ALGORITHM_NOT_SUPPORTED", + 249108103335u64 => "BAD_CHECKAFTER_IN_POLLREP", + 249108103276u64 => "BAD_REQUEST_ID", + 249108103324u64 => "CERTHASH_UNMATCHED", + 249108103277u64 => "CERTID_NOT_FOUND", + 249108103337u64 => "CERTIFICATE_NOT_ACCEPTED", + 249108103280u64 => "CERTIFICATE_NOT_FOUND", + 249108103325u64 => "CERTREQMSG_NOT_FOUND", + 249108103281u64 => "CERTRESPONSE_NOT_FOUND", + 249108103282u64 => "CERT_AND_KEY_DO_NOT_MATCH", + 249108103349u64 => "CHECKAFTER_OUT_OF_RANGE", + 249108103344u64 => "ENCOUNTERED_KEYUPDATEWARNING", + 249108103330u64 => "ENCOUNTERED_WAITING", + 249108103283u64 => "ERROR_CALCULATING_PROTECTION", + 249108103284u64 => "ERROR_CREATING_CERTCONF", + 249108103285u64 => "ERROR_CREATING_CERTREP", + 249108103331u64 => "ERROR_CREATING_CERTREQ", + 249108103286u64 => "ERROR_CREATING_ERROR", + 249108103287u64 => "ERROR_CREATING_GENM", + 249108103288u64 => "ERROR_CREATING_GENP", + 249108103290u64 => "ERROR_CREATING_PKICONF", + 249108103291u64 => "ERROR_CREATING_POLLREP", + 249108103292u64 => "ERROR_CREATING_POLLREQ", + 249108103293u64 => "ERROR_CREATING_RP", + 249108103294u64 => "ERROR_CREATING_RR", + 249108103275u64 => "ERROR_PARSING_PKISTATUS", + 249108103326u64 => "ERROR_PROCESSING_MESSAGE", + 249108103295u64 => "ERROR_PROTECTING_MESSAGE", + 249108103296u64 => "ERROR_SETTING_CERTHASH", + 249108103328u64 => "ERROR_UNEXPECTED_CERTCONF", + 249108103308u64 => "ERROR_VALIDATING_PROTECTION", + 249108103339u64 => "ERROR_VALIDATING_SIGNATURE", + 249108103332u64 => "FAILED_BUILDING_OWN_CHAIN", + 249108103309u64 => "FAILED_EXTRACTING_PUBKEY", + 249108103278u64 => "FAILURE_OBTAINING_RANDOM", + 249108103297u64 => "FAIL_INFO_OUT_OF_RANGE", + 249108103268u64 => "INVALID_ARGS", + 249108103342u64 => "INVALID_OPTION", + 249108103333u64 => "MISSING_CERTID", + 249108103298u64 => "MISSING_KEY_INPUT_FOR_CREATING_PROTECTION", + 249108103310u64 => "MISSING_KEY_USAGE_DIGITALSIGNATURE", + 249108103289u64 => "MISSING_P10CSR", + 249108103334u64 => "MISSING_PBM_SECRET", + 249108103299u64 => "MISSING_PRIVATE_KEY", + 249108103358u64 => "MISSING_PRIVATE_KEY_FOR_POPO", + 249108103311u64 => "MISSING_PROTECTION", + 249108103351u64 => "MISSING_PUBLIC_KEY", + 249108103336u64 => "MISSING_REFERENCE_CERT", + 249108103346u64 => "MISSING_SECRET", + 249108103279u64 => "MISSING_SENDER_IDENTIFICATION", + 249108103347u64 => "MISSING_TRUST_ANCHOR", + 249108103312u64 => "MISSING_TRUST_STORE", + 249108103329u64 => "MULTIPLE_REQUESTS_NOT_SUPPORTED", + 249108103338u64 => "MULTIPLE_RESPONSES_NOT_SUPPORTED", + 249108103270u64 => "MULTIPLE_SAN_SOURCES", + 249108103362u64 => "NO_STDIO", + 249108103313u64 => "NO_SUITABLE_SENDER_CERT", + 249108103271u64 => "NULL_ARGUMENT", + 249108103314u64 => "PKIBODY_ERROR", + 249108103300u64 => "PKISTATUSINFO_NOT_FOUND", + 249108103340u64 => "POLLING_FAILED", + 249108103315u64 => "POTENTIALLY_INVALID_CERTIFICATE", + 249108103348u64 => "RECEIVED_ERROR", + 249108103316u64 => "RECIPNONCE_UNMATCHED", + 249108103317u64 => "REQUEST_NOT_ACCEPTED", + 249108103350u64 => "REQUEST_REJECTED_BY_SERVER", + 249108103318u64 => "SENDER_GENERALNAME_TYPE_NOT_SUPPORTED", + 249108103319u64 => "SRVCERT_DOES_NOT_VALIDATE_MSG", + 249108103352u64 => "TOTAL_TIMEOUT", + 249108103320u64 => "TRANSACTIONID_UNMATCHED", + 249108103327u64 => "TRANSFER_ERROR", + 249108103301u64 => "UNEXPECTED_PKIBODY", + 249108103353u64 => "UNEXPECTED_PKISTATUS", + 249108103321u64 => "UNEXPECTED_PVNO", + 249108103302u64 => "UNKNOWN_ALGORITHM_ID", + 249108103303u64 => "UNKNOWN_CERT_TYPE", + 249108103354u64 => "UNKNOWN_PKISTATUS", + 249108103304u64 => "UNSUPPORTED_ALGORITHM", + 249108103305u64 => "UNSUPPORTED_KEY_TYPE", + 249108103322u64 => "UNSUPPORTED_PROTECTION_ALG_DHBASEDMAC", + 249108103343u64 => "VALUE_TOO_LARGE", + 249108103345u64 => "VALUE_TOO_SMALL", + 249108103306u64 => "WRONG_ALGORITHM_OID", + 249108103357u64 => "WRONG_CERTID", + 249108103355u64 => "WRONG_CERTID_IN_RP", + 249108103323u64 => "WRONG_PBM_VALUE", + 249108103356u64 => "WRONG_RP_COMPONENT_COUNT", + 249108103341u64 => "WRONG_SERIAL_IN_RP", + 197568495715u64 => "ADD_SIGNER_ERROR", + 197568495777u64 => "ATTRIBUTE_ERROR", + 197568495791u64 => "CERTIFICATE_ALREADY_PRESENT", + 197568495776u64 => "CERTIFICATE_HAS_NO_KEYID", + 197568495716u64 => "CERTIFICATE_VERIFY_ERROR", + 197568495800u64 => "CIPHER_AEAD_SET_TAG_ERROR", + 197568495801u64 => "CIPHER_GET_TAG", + 197568495717u64 => "CIPHER_INITIALISATION_ERROR", + 197568495718u64 => "CIPHER_PARAMETER_INITIALISATION_ERROR", + 197568495719u64 => "CMS_DATAFINAL_ERROR", + 197568495720u64 => "CMS_LIB", + 197568495786u64 => "CONTENTIDENTIFIER_MISMATCH", + 197568495721u64 => "CONTENT_NOT_FOUND", + 197568495787u64 => "CONTENT_TYPE_MISMATCH", + 197568495722u64 => "CONTENT_TYPE_NOT_COMPRESSED_DATA", + 197568495723u64 => "CONTENT_TYPE_NOT_ENVELOPED_DATA", + 197568495724u64 => "CONTENT_TYPE_NOT_SIGNED_DATA", + 197568495725u64 => "CONTENT_VERIFY_ERROR", + 197568495726u64 => "CTRL_ERROR", + 197568495727u64 => "CTRL_FAILURE", + 197568495803u64 => "DECODE_ERROR", + 197568495728u64 => "DECRYPT_ERROR", + 197568495729u64 => "ERROR_GETTING_PUBLIC_KEY", + 197568495730u64 => "ERROR_READING_MESSAGEDIGEST_ATTRIBUTE", + 197568495731u64 => "ERROR_SETTING_KEY", + 197568495732u64 => "ERROR_SETTING_RECIPIENTINFO", + 197568495799u64 => "ESS_SIGNING_CERTID_MISMATCH_ERROR", + 197568495733u64 => "INVALID_ENCRYPTED_KEY_LENGTH", + 197568495792u64 => "INVALID_KEY_ENCRYPTION_PARAMETER", + 197568495734u64 => "INVALID_KEY_LENGTH", + 197568495806u64 => "INVALID_LABEL", + 197568495807u64 => "INVALID_OAEP_PARAMETERS", + 197568495802u64 => "KDF_PARAMETER_ERROR", + 197568495735u64 => "MD_BIO_INIT_ERROR", + 197568495736u64 => "MESSAGEDIGEST_ATTRIBUTE_WRONG_LENGTH", + 197568495737u64 => "MESSAGEDIGEST_WRONG_LENGTH", + 197568495788u64 => "MSGSIGDIGEST_ERROR", + 197568495778u64 => "MSGSIGDIGEST_VERIFICATION_FAILURE", + 197568495779u64 => "MSGSIGDIGEST_WRONG_LENGTH", + 197568495780u64 => "NEED_ONE_SIGNER", + 197568495781u64 => "NOT_A_SIGNED_RECEIPT", + 197568495738u64 => "NOT_ENCRYPTED_DATA", + 197568495739u64 => "NOT_KEK", + 197568495797u64 => "NOT_KEY_AGREEMENT", + 197568495740u64 => "NOT_KEY_TRANSPORT", + 197568495793u64 => "NOT_PWRI", + 197568495741u64 => "NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 197568495742u64 => "NO_CIPHER", + 197568495743u64 => "NO_CONTENT", + 197568495789u64 => "NO_CONTENT_TYPE", + 197568495744u64 => "NO_DEFAULT_DIGEST", + 197568495745u64 => "NO_DIGEST_SET", + 197568495746u64 => "NO_KEY", + 197568495790u64 => "NO_KEY_OR_CERT", + 197568495747u64 => "NO_MATCHING_DIGEST", + 197568495748u64 => "NO_MATCHING_RECIPIENT", + 197568495782u64 => "NO_MATCHING_SIGNATURE", + 197568495783u64 => "NO_MSGSIGDIGEST", + 197568495794u64 => "NO_PASSWORD", + 197568495749u64 => "NO_PRIVATE_KEY", + 197568495750u64 => "NO_PUBLIC_KEY", + 197568495784u64 => "NO_RECEIPT_REQUEST", + 197568495751u64 => "NO_SIGNERS", + 197568495804u64 => "PEER_KEY_ERROR", + 197568495752u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 197568495785u64 => "RECEIPT_DECODE_ERROR", + 197568495753u64 => "RECIPIENT_ERROR", + 197568495805u64 => "SHARED_INFO_ERROR", + 197568495754u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 197568495755u64 => "SIGNFINAL_ERROR", + 197568495756u64 => "SMIME_TEXT_ERROR", + 197568495757u64 => "STORE_INIT_ERROR", + 197568495758u64 => "TYPE_NOT_COMPRESSED_DATA", + 197568495759u64 => "TYPE_NOT_DATA", + 197568495760u64 => "TYPE_NOT_DIGESTED_DATA", + 197568495761u64 => "TYPE_NOT_ENCRYPTED_DATA", + 197568495762u64 => "TYPE_NOT_ENVELOPED_DATA", + 197568495763u64 => "UNABLE_TO_FINALIZE_CONTEXT", + 197568495764u64 => "UNKNOWN_CIPHER", + 197568495765u64 => "UNKNOWN_DIGEST_ALGORITHM", + 197568495766u64 => "UNKNOWN_ID", + 197568495767u64 => "UNSUPPORTED_COMPRESSION_ALGORITHM", + 197568495810u64 => "UNSUPPORTED_CONTENT_ENCRYPTION_ALGORITHM", + 197568495768u64 => "UNSUPPORTED_CONTENT_TYPE", + 197568495808u64 => "UNSUPPORTED_ENCRYPTION_TYPE", + 197568495769u64 => "UNSUPPORTED_KEK_ALGORITHM", + 197568495795u64 => "UNSUPPORTED_KEY_ENCRYPTION_ALGORITHM", + 197568495809u64 => "UNSUPPORTED_LABEL_SOURCE", + 197568495771u64 => "UNSUPPORTED_RECIPIENTINFO_TYPE", + 197568495770u64 => "UNSUPPORTED_RECIPIENT_TYPE", + 197568495811u64 => "UNSUPPORTED_SIGNATURE_ALGORITHM", + 197568495772u64 => "UNSUPPORTED_TYPE", + 197568495773u64 => "UNWRAP_ERROR", + 197568495796u64 => "UNWRAP_FAILURE", + 197568495774u64 => "VERIFICATION_FAILURE", + 197568495775u64 => "WRAP_ERROR", + 176093659235u64 => "ZLIB_DEFLATE_ERROR", + 176093659236u64 => "ZLIB_INFLATE_ERROR", + 176093659237u64 => "ZLIB_NOT_SUPPORTED", + 60129542254u64 => "ERROR_LOADING_DSO", + 60129542266u64 => "INVALID_PRAGMA", + 60129542259u64 => "LIST_CANNOT_BE_NULL", + 60129542267u64 => "MANDATORY_BRACES_IN_VARIABLE_EXPANSION", + 60129542244u64 => "MISSING_CLOSE_SQUARE_BRACKET", + 60129542245u64 => "MISSING_EQUAL_SIGN", + 60129542256u64 => "MISSING_INIT_FUNCTION", + 60129542253u64 => "MODULE_INITIALIZATION_ERROR", + 60129542246u64 => "NO_CLOSE_BRACE", + 60129542249u64 => "NO_CONF", + 60129542250u64 => "NO_CONF_OR_ENVIRONMENT_VARIABLE", + 60129542251u64 => "NO_SECTION", + 60129542258u64 => "NO_SUCH_FILE", + 60129542252u64 => "NO_VALUE", + 60129542265u64 => "NUMBER_TOO_LARGE", + 60129542268u64 => "OPENSSL_CONF_REFERENCES_MISSING_SECTION", + 60129542255u64 => "RECURSIVE_DIRECTORY_INCLUDE", + 60129542270u64 => "RECURSIVE_SECTION_REFERENCE", + 60129542269u64 => "RELATIVE_PATH", + 60129542261u64 => "SSL_COMMAND_SECTION_EMPTY", + 60129542262u64 => "SSL_COMMAND_SECTION_NOT_FOUND", + 60129542263u64 => "SSL_SECTION_EMPTY", + 60129542264u64 => "SSL_SECTION_NOT_FOUND", + 60129542247u64 => "UNABLE_TO_CREATE_NEW_SECTION", + 60129542257u64 => "UNKNOWN_MODULE_NAME", + 60129542260u64 => "VARIABLE_EXPANSION_TOO_LONG", + 60129542248u64 => "VARIABLE_HAS_NO_VALUE", + 240518168676u64 => "BAD_PBM_ITERATIONCOUNT", + 240518168678u64 => "CRMFERROR", + 240518168679u64 => "ERROR", + 240518168680u64 => "ERROR_DECODING_CERTIFICATE", + 240518168681u64 => "ERROR_DECRYPTING_CERTIFICATE", + 240518168682u64 => "ERROR_DECRYPTING_SYMMETRIC_KEY", + 240518168683u64 => "FAILURE_OBTAINING_RANDOM", + 240518168684u64 => "ITERATIONCOUNT_BELOW_100", + 240518168677u64 => "MALFORMED_IV", + 240518168685u64 => "NULL_ARGUMENT", + 240518168689u64 => "POPOSKINPUT_NOT_SUPPORTED", + 240518168693u64 => "POPO_INCONSISTENT_PUBLIC_KEY", + 240518168697u64 => "POPO_MISSING", + 240518168694u64 => "POPO_MISSING_PUBLIC_KEY", + 240518168695u64 => "POPO_MISSING_SUBJECT", + 240518168696u64 => "POPO_RAVERIFIED_NOT_ACCEPTED", + 240518168686u64 => "SETTING_MAC_ALGOR_FAILURE", + 240518168687u64 => "SETTING_OWF_ALGOR_FAILURE", + 240518168688u64 => "UNSUPPORTED_ALGORITHM", + 240518168690u64 => "UNSUPPORTED_CIPHER", + 240518168691u64 => "UNSUPPORTED_METHOD_FOR_CREATING_POPO", + 240518168692u64 => "UNSUPPORTED_POPO_METHOD", + 64424509557u64 => "BAD_ALGORITHM_NAME", + 64424509558u64 => "CONFLICTING_NAMES", + 64424509561u64 => "HEX_STRING_TOO_SHORT", + 64424509542u64 => "ILLEGAL_HEX_DIGIT", + 64424509546u64 => "INSUFFICIENT_DATA_SPACE", + 64424509547u64 => "INSUFFICIENT_PARAM_SIZE", + 64424509548u64 => "INSUFFICIENT_SECURE_DATA_SPACE", + 64424509567u64 => "INTEGER_OVERFLOW", + 64424509562u64 => "INVALID_NEGATIVE_VALUE", + 64424509549u64 => "INVALID_NULL_ARGUMENT", + 64424509550u64 => "INVALID_OSSL_PARAM_TYPE", + 64424509571u64 => "NO_PARAMS_TO_MERGE", + 64424509568u64 => "NO_SPACE_FOR_TERMINATING_NULL", + 64424509543u64 => "ODD_NUMBER_OF_DIGITS", + 64424509563u64 => "PARAM_CANNOT_BE_REPRESENTED_EXACTLY", + 64424509564u64 => "PARAM_NOT_INTEGER_TYPE", + 64424509569u64 => "PARAM_OF_INCOMPATIBLE_TYPE", + 64424509565u64 => "PARAM_UNSIGNED_INTEGER_NEGATIVE_VALUE_UNSUPPORTED", + 64424509570u64 => "PARAM_UNSUPPORTED_FLOATING_POINT_FORMAT", + 64424509566u64 => "PARAM_VALUE_TOO_LARGE_FOR_DESTINATION", + 64424509544u64 => "PROVIDER_ALREADY_EXISTS", + 64424509545u64 => "PROVIDER_SECTION_ERROR", + 64424509559u64 => "RANDOM_SECTION_ERROR", + 64424509551u64 => "SECURE_MALLOC_FAILURE", + 64424509552u64 => "STRING_TOO_LONG", + 64424509553u64 => "TOO_MANY_BYTES", + 64424509554u64 => "TOO_MANY_RECORDS", + 64424509556u64 => "TOO_SMALL_BUFFER", + 64424509560u64 => "UNKNOWN_NAME_IN_RANDOM_SECTION", + 64424509555u64 => "ZERO_LENGTH_NUMBER", + 214748364908u64 => "BASE64_DECODE_ERROR", + 214748364900u64 => "INVALID_LOG_ID_LENGTH", + 214748364909u64 => "LOG_CONF_INVALID", + 214748364910u64 => "LOG_CONF_INVALID_KEY", + 214748364911u64 => "LOG_CONF_MISSING_DESCRIPTION", + 214748364912u64 => "LOG_CONF_MISSING_KEY", + 214748364913u64 => "LOG_KEY_INVALID", + 214748364916u64 => "SCT_FUTURE_TIMESTAMP", + 214748364904u64 => "SCT_INVALID", + 214748364907u64 => "SCT_INVALID_SIGNATURE", + 214748364905u64 => "SCT_LIST_INVALID", + 214748364914u64 => "SCT_LOG_ID_MISMATCH", + 214748364906u64 => "SCT_NOT_SET", + 214748364915u64 => "SCT_UNSUPPORTED_VERSION", + 214748364901u64 => "UNRECOGNIZED_SIGNATURE_NID", + 214748364902u64 => "UNSUPPORTED_ENTRY_TYPE", + 214748364903u64 => "UNSUPPORTED_VERSION", + 21474836607u64 => "BAD_FFC_PARAMETERS", + 21474836581u64 => "BAD_GENERATOR", + 21474836589u64 => "BN_DECODE_ERROR", + 21474836586u64 => "BN_ERROR", + 21474836595u64 => "CHECK_INVALID_J_VALUE", + 21474836596u64 => "CHECK_INVALID_Q_VALUE", + 21474836602u64 => "CHECK_PUBKEY_INVALID", + 21474836603u64 => "CHECK_PUBKEY_TOO_LARGE", + 21474836604u64 => "CHECK_PUBKEY_TOO_SMALL", + 21474836597u64 => "CHECK_P_NOT_PRIME", + 21474836598u64 => "CHECK_P_NOT_SAFE_PRIME", + 21474836599u64 => "CHECK_Q_NOT_PRIME", + 21474836584u64 => "DECODE_ERROR", + 21474836590u64 => "INVALID_PARAMETER_NAME", + 21474836594u64 => "INVALID_PARAMETER_NID", + 21474836582u64 => "INVALID_PUBKEY", + 21474836608u64 => "INVALID_SECRET", + 21474836592u64 => "KDF_PARAMETER_ERROR", + 21474836588u64 => "KEYS_NOT_SET", + 21474836605u64 => "MISSING_PUBKEY", + 21474836583u64 => "MODULUS_TOO_LARGE", + 21474836606u64 => "MODULUS_TOO_SMALL", + 21474836600u64 => "NOT_SUITABLE_GENERATOR", + 21474836587u64 => "NO_PARAMETERS_SET", + 21474836580u64 => "NO_PRIVATE_VALUE", + 21474836585u64 => "PARAMETER_ENCODING_ERROR", + 21474836591u64 => "PEER_KEY_ERROR", + 21474836610u64 => "Q_TOO_LARGE", + 21474836593u64 => "SHARED_INFO_ERROR", + 21474836601u64 => "UNABLE_TO_CHECK_GENERATOR", + 42949673074u64 => "BAD_FFC_PARAMETERS", + 42949673062u64 => "BAD_Q_VALUE", + 42949673068u64 => "BN_DECODE_ERROR", + 42949673069u64 => "BN_ERROR", + 42949673064u64 => "DECODE_ERROR", + 42949673066u64 => "INVALID_DIGEST_TYPE", + 42949673072u64 => "INVALID_PARAMETERS", + 42949673061u64 => "MISSING_PARAMETERS", + 42949673071u64 => "MISSING_PRIVATE_KEY", + 42949673063u64 => "MODULUS_TOO_LARGE", + 42949673067u64 => "NO_PARAMETERS_SET", + 42949673065u64 => "PARAMETER_ENCODING_ERROR", + 42949673075u64 => "P_NOT_PRIME", + 42949673073u64 => "Q_NOT_PRIME", + 42949673070u64 => "SEED_LEN_SMALL", + 42949673076u64 => "TOO_MANY_RETRIES", + 158913790052u64 => "CTRL_FAILED", + 158913790062u64 => "DSO_ALREADY_LOADED", + 158913790065u64 => "EMPTY_FILE_STRUCTURE", + 158913790066u64 => "FAILURE", + 158913790053u64 => "FILENAME_TOO_BIG", + 158913790054u64 => "FINISH_FAILED", + 158913790067u64 => "INCORRECT_FILE_SYNTAX", + 158913790055u64 => "LOAD_FAILED", + 158913790061u64 => "NAME_TRANSLATION_FAILED", + 158913790063u64 => "NO_FILENAME", + 158913790056u64 => "NULL_HANDLE", + 158913790064u64 => "SET_FILENAME_FAILED", + 158913790057u64 => "STACK_ERROR", + 158913790058u64 => "SYM_FAILURE", + 158913790059u64 => "UNLOAD_FAILED", + 158913790060u64 => "UNSUPPORTED", + 68719476851u64 => "ASN1_ERROR", + 68719476892u64 => "BAD_SIGNATURE", + 68719476880u64 => "BIGNUM_OUT_OF_RANGE", + 68719476836u64 => "BUFFER_TOO_SMALL", + 68719476901u64 => "CANNOT_INVERT", + 68719476882u64 => "COORDINATES_OUT_OF_RANGE", + 68719476896u64 => "CURVE_DOES_NOT_SUPPORT_ECDH", + 68719476906u64 => "CURVE_DOES_NOT_SUPPORT_ECDSA", + 68719476895u64 => "CURVE_DOES_NOT_SUPPORT_SIGNING", + 68719476878u64 => "DECODE_ERROR", + 68719476854u64 => "DISCRIMINANT_IS_ZERO", + 68719476855u64 => "EC_GROUP_NEW_BY_NAME_FAILURE", + 68719476863u64 => "EXPLICIT_PARAMS_NOT_SUPPORTED", + 68719476902u64 => "FAILED_MAKING_PUBLIC_KEY", + 68719476879u64 => "FIELD_TOO_LARGE", + 68719476883u64 => "GF2M_NOT_SUPPORTED", + 68719476856u64 => "GROUP2PKPARAMETERS_FAILURE", + 68719476857u64 => "I2D_ECPKPARAMETERS_FAILURE", + 68719476837u64 => "INCOMPATIBLE_OBJECTS", + 68719476904u64 => "INVALID_A", + 68719476848u64 => "INVALID_ARGUMENT", + 68719476905u64 => "INVALID_B", + 68719476907u64 => "INVALID_COFACTOR", + 68719476846u64 => "INVALID_COMPRESSED_POINT", + 68719476845u64 => "INVALID_COMPRESSION_BIT", + 68719476877u64 => "INVALID_CURVE", + 68719476887u64 => "INVALID_DIGEST", + 68719476874u64 => "INVALID_DIGEST_TYPE", + 68719476838u64 => "INVALID_ENCODING", + 68719476839u64 => "INVALID_FIELD", + 68719476840u64 => "INVALID_FORM", + 68719476909u64 => "INVALID_GENERATOR", + 68719476858u64 => "INVALID_GROUP_ORDER", + 68719476852u64 => "INVALID_KEY", + 68719476853u64 => "INVALID_LENGTH", + 68719476910u64 => "INVALID_NAMED_GROUP_CONVERSION", + 68719476897u64 => "INVALID_OUTPUT_LENGTH", + 68719476908u64 => "INVALID_P", + 68719476869u64 => "INVALID_PEER_KEY", + 68719476868u64 => "INVALID_PENTANOMIAL_BASIS", + 68719476859u64 => "INVALID_PRIVATE_KEY", + 68719476911u64 => "INVALID_SEED", + 68719476873u64 => "INVALID_TRINOMIAL_BASIS", + 68719476884u64 => "KDF_PARAMETER_ERROR", + 68719476876u64 => "KEYS_NOT_SET", + 68719476872u64 => "LADDER_POST_FAILURE", + 68719476889u64 => "LADDER_PRE_FAILURE", + 68719476898u64 => "LADDER_STEP_FAILURE", + 68719476903u64 => "MISSING_OID", + 68719476860u64 => "MISSING_PARAMETERS", + 68719476861u64 => "MISSING_PRIVATE_KEY", + 68719476893u64 => "NEED_NEW_SETUP_VALUES", + 68719476871u64 => "NOT_A_NIST_PRIME", + 68719476862u64 => "NOT_IMPLEMENTED", + 68719476847u64 => "NOT_INITIALIZED", + 68719476875u64 => "NO_PARAMETERS_SET", + 68719476890u64 => "NO_PRIVATE_VALUE", + 68719476888u64 => "OPERATION_NOT_SUPPORTED", + 68719476870u64 => "PASSED_NULL_PARAMETER", + 68719476885u64 => "PEER_KEY_ERROR", + 68719476891u64 => "POINT_ARITHMETIC_FAILURE", + 68719476842u64 => "POINT_AT_INFINITY", + 68719476899u64 => "POINT_COORDINATES_BLIND_FAILURE", + 68719476843u64 => "POINT_IS_NOT_ON_CURVE", + 68719476894u64 => "RANDOM_NUMBER_GENERATION_FAILED", + 68719476886u64 => "SHARED_INFO_ERROR", + 68719476844u64 => "SLOT_FULL", + 68719476912u64 => "TOO_MANY_RETRIES", + 68719476849u64 => "UNDEFINED_GENERATOR", + 68719476864u64 => "UNDEFINED_ORDER", + 68719476900u64 => "UNKNOWN_COFACTOR", + 68719476865u64 => "UNKNOWN_GROUP", + 68719476850u64 => "UNKNOWN_ORDER", + 68719476867u64 => "UNSUPPORTED_FIELD", + 68719476881u64 => "WRONG_CURVE_PARAMETERS", + 68719476866u64 => "WRONG_ORDER", + 163208757348u64 => "ALREADY_LOADED", + 163208757381u64 => "ARGUMENT_IS_NOT_A_NUMBER", + 163208757382u64 => "CMD_NOT_EXECUTABLE", + 163208757383u64 => "COMMAND_TAKES_INPUT", + 163208757384u64 => "COMMAND_TAKES_NO_INPUT", + 163208757351u64 => "CONFLICTING_ENGINE_ID", + 163208757367u64 => "CTRL_COMMAND_NOT_IMPLEMENTED", + 163208757352u64 => "DSO_FAILURE", + 163208757380u64 => "DSO_NOT_FOUND", + 163208757396u64 => "ENGINES_SECTION_ERROR", + 163208757350u64 => "ENGINE_CONFIGURATION_ERROR", + 163208757353u64 => "ENGINE_IS_NOT_IN_LIST", + 163208757397u64 => "ENGINE_SECTION_ERROR", + 163208757376u64 => "FAILED_LOADING_PRIVATE_KEY", + 163208757377u64 => "FAILED_LOADING_PUBLIC_KEY", + 163208757354u64 => "FINISH_FAILED", + 163208757356u64 => "ID_OR_NAME_MISSING", + 163208757357u64 => "INIT_FAILED", + 163208757358u64 => "INTERNAL_LIST_ERROR", + 163208757391u64 => "INVALID_ARGUMENT", + 163208757385u64 => "INVALID_CMD_NAME", + 163208757386u64 => "INVALID_CMD_NUMBER", + 163208757399u64 => "INVALID_INIT_VALUE", + 163208757398u64 => "INVALID_STRING", + 163208757365u64 => "NOT_INITIALISED", + 163208757360u64 => "NOT_LOADED", + 163208757368u64 => "NO_CONTROL_FUNCTION", + 163208757392u64 => "NO_INDEX", + 163208757373u64 => "NO_LOAD_FUNCTION", + 163208757378u64 => "NO_REFERENCE", + 163208757364u64 => "NO_SUCH_ENGINE", + 163208757394u64 => "UNIMPLEMENTED_CIPHER", + 163208757395u64 => "UNIMPLEMENTED_DIGEST", + 163208757349u64 => "UNIMPLEMENTED_PUBLIC_KEY_METHOD", + 163208757393u64 => "VERSION_INCOMPATIBILITY", + 231928234091u64 => "EMPTY_ESS_CERT_ID_LIST", + 231928234087u64 => "ESS_CERT_DIGEST_ERROR", + 231928234088u64 => "ESS_CERT_ID_NOT_FOUND", + 231928234089u64 => "ESS_CERT_ID_WRONG_ORDER", + 231928234090u64 => "ESS_DIGEST_ALG_UNKNOWN", + 231928234086u64 => "ESS_SIGNING_CERTIFICATE_ERROR", + 231928234084u64 => "ESS_SIGNING_CERT_ADD_ERROR", + 231928234085u64 => "ESS_SIGNING_CERT_V2_ADD_ERROR", + 231928234092u64 => "MISSING_SIGNING_CERTIFICATE_ATTRIBUTE", + 25769803919u64 => "AES_KEY_SETUP_FAILED", + 25769803952u64 => "ARIA_KEY_SETUP_FAILED", + 25769803976u64 => "BAD_ALGORITHM_NAME", + 25769803876u64 => "BAD_DECRYPT", + 25769803971u64 => "BAD_KEY_LENGTH", + 25769803931u64 => "BUFFER_TOO_SMALL", + 25769804001u64 => "CACHE_CONSTANTS_FAILED", + 25769803933u64 => "CAMELLIA_KEY_SETUP_FAILED", + 25769803973u64 => "CANNOT_GET_PARAMETERS", + 25769803974u64 => "CANNOT_SET_PARAMETERS", + 25769803960u64 => "CIPHER_NOT_GCM_MODE", + 25769803898u64 => "CIPHER_PARAMETER_ERROR", + 25769803923u64 => "COMMAND_NOT_SUPPORTED", + 25769803977u64 => "CONFLICTING_ALGORITHM_NAME", + 25769803949u64 => "COPY_ERROR", + 25769803908u64 => "CTRL_NOT_IMPLEMENTED", + 25769803909u64 => "CTRL_OPERATION_NOT_IMPLEMENTED", + 25769803914u64 => "DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH", + 25769803890u64 => "DECODE_ERROR", + 25769803986u64 => "DEFAULT_QUERY_PARSE_ERROR", + 25769803877u64 => "DIFFERENT_KEY_TYPES", + 25769803929u64 => "DIFFERENT_PARAMETERS", + 25769803941u64 => "ERROR_LOADING_SECTION", + 25769803950u64 => "EXPECTING_AN_HMAC_KEY", + 25769803903u64 => "EXPECTING_AN_RSA_KEY", + 25769803904u64 => "EXPECTING_A_DH_KEY", + 25769803905u64 => "EXPECTING_A_DSA_KEY", + 25769803995u64 => "EXPECTING_A_ECX_KEY", + 25769803918u64 => "EXPECTING_A_EC_KEY", + 25769803940u64 => "EXPECTING_A_POLY1305_KEY", + 25769803951u64 => "EXPECTING_A_SIPHASH_KEY", + 25769803964u64 => "FINAL_ERROR", + 25769803990u64 => "GENERATE_ERROR", + 25769803958u64 => "GET_RAW_KEY_FAILED", + 25769803947u64 => "ILLEGAL_SCRYPT_PARAMETERS", + 25769803980u64 => "INACCESSIBLE_DOMAIN_PARAMETERS", + 25769803979u64 => "INACCESSIBLE_KEY", + 25769803910u64 => "INITIALIZATION_ERROR", + 25769803887u64 => "INPUT_NOT_INITIALIZED", + 25769803961u64 => "INVALID_CUSTOM_LENGTH", + 25769803928u64 => "INVALID_DIGEST", + 25769803970u64 => "INVALID_IV_LENGTH", + 25769803939u64 => "INVALID_KEY", + 25769803906u64 => "INVALID_KEY_LENGTH", + 25769803997u64 => "INVALID_LENGTH", + 25769803994u64 => "INVALID_NULL_ALGORITHM", + 25769803924u64 => "INVALID_OPERATION", + 25769803969u64 => "INVALID_PROVIDER_FUNCTIONS", + 25769803962u64 => "INVALID_SALT_LENGTH", + 25769803999u64 => "INVALID_SECRET_LENGTH", + 25769803996u64 => "INVALID_SEED_LENGTH", + 25769803998u64 => "INVALID_VALUE", + 25769803981u64 => "KEYMGMT_EXPORT_FAILURE", + 25769803956u64 => "KEY_SETUP_FAILED", + 25769803989u64 => "LOCKING_NOT_SUPPORTED", + 25769803948u64 => "MEMORY_LIMIT_EXCEEDED", + 25769803935u64 => "MESSAGE_DIGEST_IS_NULL", + 25769803920u64 => "METHOD_NOT_SUPPORTED", + 25769803879u64 => "MISSING_PARAMETERS", + 25769803966u64 => "NOT_ABLE_TO_COPY_CTX", + 25769803954u64 => "NOT_XOF_OR_INVALID_LENGTH", + 25769803907u64 => "NO_CIPHER_SET", + 25769803934u64 => "NO_DEFAULT_DIGEST", + 25769803915u64 => "NO_DIGEST_SET", + 25769803982u64 => "NO_IMPORT_FUNCTION", + 25769803975u64 => "NO_KEYMGMT_AVAILABLE", + 25769803972u64 => "NO_KEYMGMT_PRESENT", + 25769803930u64 => "NO_KEY_SET", + 25769803925u64 => "NO_OPERATION_SET", + 25769803984u64 => "NULL_MAC_PKEY_CTX", + 25769803953u64 => "ONLY_ONESHOT_SUPPORTED", + 25769803927u64 => "OPERATION_NOT_INITIALIZED", + 25769803926u64 => "OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", + 25769803978u64 => "OUTPUT_WOULD_OVERFLOW", + 25769803963u64 => "PARAMETER_TOO_LARGE", + 25769803938u64 => "PARTIALLY_OVERLAPPING", + 25769803957u64 => "PBKDF2_ERROR", + 25769803955u64 => "PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", + 25769803921u64 => "PRIVATE_KEY_DECODE_ERROR", + 25769803922u64 => "PRIVATE_KEY_ENCODE_ERROR", + 25769803882u64 => "PUBLIC_KEY_NOT_RSA", + 25769804003u64 => "SETTING_XOF_FAILED", + 25769803985u64 => "SET_DEFAULT_PROPERTY_FAILURE", + 25769803959u64 => "TOO_MANY_RECORDS", + 25769803988u64 => "UNABLE_TO_ENABLE_LOCKING", + 25769803991u64 => "UNABLE_TO_GET_MAXIMUM_REQUEST_SIZE", + 25769803992u64 => "UNABLE_TO_GET_RANDOM_STRENGTH", + 25769803987u64 => "UNABLE_TO_LOCK_CONTEXT", + 25769803993u64 => "UNABLE_TO_SET_CALLBACKS", + 25769803936u64 => "UNKNOWN_CIPHER", + 25769803937u64 => "UNKNOWN_DIGEST", + 25769803983u64 => "UNKNOWN_KEY_TYPE", + 25769803945u64 => "UNKNOWN_OPTION", + 25769803897u64 => "UNKNOWN_PBE_ALGORITHM", + 25769803932u64 => "UNSUPPORTED_ALGORITHM", + 25769803883u64 => "UNSUPPORTED_CIPHER", + 25769803899u64 => "UNSUPPORTED_KEYLENGTH", + 25769803900u64 => "UNSUPPORTED_KEY_DERIVATION_FUNCTION", + 25769803884u64 => "UNSUPPORTED_KEY_SIZE", + 25769804000u64 => "UNSUPPORTED_KEY_TYPE", + 25769803911u64 => "UNSUPPORTED_NUMBER_OF_ROUNDS", + 25769803901u64 => "UNSUPPORTED_PRF", + 25769803894u64 => "UNSUPPORTED_PRIVATE_KEY_ALGORITHM", + 25769803902u64 => "UNSUPPORTED_SALT_TYPE", + 25769803965u64 => "UPDATE_ERROR", + 25769803946u64 => "WRAP_MODE_NOT_ALLOWED", + 25769803885u64 => "WRONG_FINAL_BLOCK_LENGTH", + 25769803967u64 => "XTS_DATA_UNIT_IS_TOO_LARGE", + 25769803968u64 => "XTS_DUPLICATED_KEYS", + 261993005164u64 => "ASN1_LEN_EXCEEDS_MAX_RESP_LEN", + 261993005156u64 => "CONNECT_FAILURE", + 261993005165u64 => "ERROR_PARSING_ASN1_LENGTH", + 261993005175u64 => "ERROR_PARSING_CONTENT_LENGTH", + 261993005157u64 => "ERROR_PARSING_URL", + 261993005159u64 => "ERROR_RECEIVING", + 261993005158u64 => "ERROR_SENDING", + 261993005184u64 => "FAILED_READING_DATA", + 261993005182u64 => "HEADER_PARSE_ERROR", + 261993005176u64 => "INCONSISTENT_CONTENT_LENGTH", + 261993005179u64 => "INVALID_PORT_NUMBER", + 261993005181u64 => "INVALID_URL_PATH", + 261993005180u64 => "INVALID_URL_SCHEME", + 261993005173u64 => "MAX_RESP_LEN_EXCEEDED", + 261993005166u64 => "MISSING_ASN1_ENCODING", + 261993005177u64 => "MISSING_CONTENT_TYPE", + 261993005167u64 => "MISSING_REDIRECT_LOCATION", + 261993005161u64 => "RECEIVED_ERROR", + 261993005162u64 => "RECEIVED_WRONG_HTTP_VERSION", + 261993005168u64 => "REDIRECTION_FROM_HTTPS_TO_HTTP", + 261993005172u64 => "REDIRECTION_NOT_ENABLED", + 261993005169u64 => "RESPONSE_LINE_TOO_LONG", + 261993005160u64 => "RESPONSE_PARSE_ERROR", + 261993005185u64 => "RETRY_TIMEOUT", + 261993005183u64 => "SERVER_CANCELED_CONNECTION", + 261993005178u64 => "SOCK_NOT_SUPPORTED", + 261993005170u64 => "STATUS_CODE_UNSUPPORTED", + 261993005163u64 => "TLS_NOT_ENABLED", + 261993005171u64 => "TOO_MANY_REDIRECTIONS", + 261993005174u64 => "UNEXPECTED_CONTENT_TYPE", + 34359738470u64 => "OID_EXISTS", + 34359738469u64 => "UNKNOWN_NID", + 34359738471u64 => "UNKNOWN_OBJECT_NAME", + 167503724645u64 => "CERTIFICATE_VERIFY_ERROR", + 167503724646u64 => "DIGEST_ERR", + 167503724650u64 => "DIGEST_NAME_ERR", + 167503724651u64 => "DIGEST_SIZE_ERR", + 167503724666u64 => "ERROR_IN_NEXTUPDATE_FIELD", + 167503724667u64 => "ERROR_IN_THISUPDATE_FIELD", + 167503724647u64 => "MISSING_OCSPSIGNING_USAGE", + 167503724668u64 => "NEXTUPDATE_BEFORE_THISUPDATE", + 167503724648u64 => "NOT_BASIC_RESPONSE", + 167503724649u64 => "NO_CERTIFICATES_IN_CHAIN", + 167503724652u64 => "NO_RESPONSE_DATA", + 167503724653u64 => "NO_REVOKED_TIME", + 167503724674u64 => "NO_SIGNER_KEY", + 167503724654u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 167503724672u64 => "REQUEST_NOT_SIGNED", + 167503724655u64 => "RESPONSE_CONTAINS_NO_REVOCATION_DATA", + 167503724656u64 => "ROOT_CA_NOT_TRUSTED", + 167503724661u64 => "SIGNATURE_FAILURE", + 167503724662u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 167503724669u64 => "STATUS_EXPIRED", + 167503724670u64 => "STATUS_NOT_YET_VALID", + 167503724671u64 => "STATUS_TOO_OLD", + 167503724663u64 => "UNKNOWN_MESSAGE_DIGEST", + 167503724664u64 => "UNKNOWN_NID", + 167503724673u64 => "UNSUPPORTED_REQUESTORNAME_TYPE", + 257698037861u64 => "COULD_NOT_DECODE_OBJECT", + 257698037862u64 => "DECODER_NOT_FOUND", + 257698037860u64 => "MISSING_GET_PARAMS", + 253403070565u64 => "ENCODER_NOT_FOUND", + 253403070564u64 => "INCORRECT_PROPERTY_QUERY", + 253403070566u64 => "MISSING_GET_PARAMS", + 188978561131u64 => "AMBIGUOUS_CONTENT_TYPE", + 188978561139u64 => "BAD_PASSWORD_READ", + 188978561137u64 => "ERROR_VERIFYING_PKCS12_MAC", + 188978561145u64 => "FINGERPRINT_SIZE_DOES_NOT_MATCH_DIGEST", + 188978561130u64 => "INVALID_SCHEME", + 188978561136u64 => "IS_NOT_A", + 188978561140u64 => "LOADER_INCOMPLETE", + 188978561141u64 => "LOADING_STARTED", + 188978561124u64 => "NOT_A_CERTIFICATE", + 188978561125u64 => "NOT_A_CRL", + 188978561127u64 => "NOT_A_NAME", + 188978561126u64 => "NOT_A_PRIVATE_KEY", + 188978561146u64 => "NOT_A_PUBLIC_KEY", + 188978561128u64 => "NOT_PARAMETERS", + 188978561147u64 => "NO_LOADERS_FOUND", + 188978561138u64 => "PASSPHRASE_CALLBACK_ERROR", + 188978561132u64 => "PATH_MUST_BE_ABSOLUTE", + 188978561143u64 => "SEARCH_ONLY_SUPPORTED_FOR_DIRECTORIES", + 188978561133u64 => "UI_PROCESS_INTERRUPTED_OR_CANCELLED", + 188978561129u64 => "UNREGISTERED_SCHEME", + 188978561134u64 => "UNSUPPORTED_CONTENT_TYPE", + 188978561142u64 => "UNSUPPORTED_OPERATION", + 188978561144u64 => "UNSUPPORTED_SEARCH_TYPE", + 188978561135u64 => "URI_AUTHORITY_UNSUPPORTED", + 38654705764u64 => "BAD_BASE64_DECODE", + 38654705765u64 => "BAD_DECRYPT", + 38654705766u64 => "BAD_END_LINE", + 38654705767u64 => "BAD_IV_CHARS", + 38654705780u64 => "BAD_MAGIC_NUMBER", + 38654705768u64 => "BAD_PASSWORD_READ", + 38654705781u64 => "BAD_VERSION_NUMBER", + 38654705782u64 => "BIO_WRITE_FAILURE", + 38654705791u64 => "CIPHER_IS_NULL", + 38654705779u64 => "ERROR_CONVERTING_PRIVATE_KEY", + 38654705795u64 => "EXPECTING_DSS_KEY_BLOB", + 38654705783u64 => "EXPECTING_PRIVATE_KEY_BLOB", + 38654705784u64 => "EXPECTING_PUBLIC_KEY_BLOB", + 38654705796u64 => "EXPECTING_RSA_KEY_BLOB", + 38654705792u64 => "HEADER_TOO_LONG", + 38654705785u64 => "INCONSISTENT_HEADER", + 38654705786u64 => "KEYBLOB_HEADER_PARSE_ERROR", + 38654705787u64 => "KEYBLOB_TOO_SHORT", + 38654705793u64 => "MISSING_DEK_IV", + 38654705769u64 => "NOT_DEK_INFO", + 38654705770u64 => "NOT_ENCRYPTED", + 38654705771u64 => "NOT_PROC_TYPE", + 38654705772u64 => "NO_START_LINE", + 38654705773u64 => "PROBLEMS_GETTING_PASSWORD", + 38654705788u64 => "PVK_DATA_TOO_SHORT", + 38654705789u64 => "PVK_TOO_SHORT", + 38654705775u64 => "READ_KEY", + 38654705776u64 => "SHORT_HEADER", + 38654705794u64 => "UNEXPECTED_DEK_IV", + 38654705777u64 => "UNSUPPORTED_CIPHER", + 38654705778u64 => "UNSUPPORTED_ENCRYPTION", + 38654705790u64 => "UNSUPPORTED_KEY_COMPONENTS", + 38654705774u64 => "UNSUPPORTED_PUBLIC_KEY_TYPE", + 150323855460u64 => "CANT_PACK_STRUCTURE", + 150323855481u64 => "CONTENT_TYPE_NOT_DATA", + 150323855461u64 => "DECODE_ERROR", + 150323855462u64 => "ENCODE_ERROR", + 150323855463u64 => "ENCRYPT_ERROR", + 150323855480u64 => "ERROR_SETTING_ENCRYPTED_DATA_TYPE", + 150323855464u64 => "INVALID_NULL_ARGUMENT", + 150323855465u64 => "INVALID_NULL_PKCS12_POINTER", + 150323855472u64 => "INVALID_TYPE", + 150323855466u64 => "IV_GEN_ERROR", + 150323855467u64 => "KEY_GEN_ERROR", + 150323855468u64 => "MAC_ABSENT", + 150323855469u64 => "MAC_GENERATION_ERROR", + 150323855470u64 => "MAC_SETUP_ERROR", + 150323855471u64 => "MAC_STRING_SET_ERROR", + 150323855473u64 => "MAC_VERIFY_FAILURE", + 150323855474u64 => "PARSE_ERROR", + 150323855476u64 => "PKCS12_CIPHERFINAL_ERROR", + 150323855478u64 => "UNKNOWN_DIGEST_ALGORITHM", + 150323855479u64 => "UNSUPPORTED_PKCS12_MODE", + 141733920885u64 => "CERTIFICATE_VERIFY_ERROR", + 141733920912u64 => "CIPHER_HAS_NO_OBJECT_IDENTIFIER", + 141733920884u64 => "CIPHER_NOT_INITIALIZED", + 141733920886u64 => "CONTENT_AND_DATA_PRESENT", + 141733920920u64 => "CTRL_ERROR", + 141733920887u64 => "DECRYPT_ERROR", + 141733920869u64 => "DIGEST_FAILURE", + 141733920917u64 => "ENCRYPTION_CTRL_FAILURE", + 141733920918u64 => "ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 141733920888u64 => "ERROR_ADDING_RECIPIENT", + 141733920889u64 => "ERROR_SETTING_CIPHER", + 141733920911u64 => "INVALID_NULL_POINTER", + 141733920923u64 => "INVALID_SIGNED_DATA_TYPE", + 141733920890u64 => "NO_CONTENT", + 141733920919u64 => "NO_DEFAULT_DIGEST", + 141733920922u64 => "NO_MATCHING_DIGEST_TYPE_FOUND", + 141733920883u64 => "NO_RECIPIENT_MATCHES_CERTIFICATE", + 141733920891u64 => "NO_SIGNATURES_ON_DATA", + 141733920910u64 => "NO_SIGNERS", + 141733920872u64 => "OPERATION_NOT_SUPPORTED_ON_THIS_TYPE", + 141733920892u64 => "PKCS7_ADD_SIGNATURE_ERROR", + 141733920921u64 => "PKCS7_ADD_SIGNER_ERROR", + 141733920913u64 => "PKCS7_DATASIGN", + 141733920895u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 141733920873u64 => "SIGNATURE_FAILURE", + 141733920896u64 => "SIGNER_CERTIFICATE_NOT_FOUND", + 141733920915u64 => "SIGNING_CTRL_FAILURE", + 141733920916u64 => "SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE", + 141733920897u64 => "SMIME_TEXT_ERROR", + 141733920874u64 => "UNABLE_TO_FIND_CERTIFICATE", + 141733920875u64 => "UNABLE_TO_FIND_MEM_BIO", + 141733920876u64 => "UNABLE_TO_FIND_MESSAGE_DIGEST", + 141733920877u64 => "UNKNOWN_DIGEST_TYPE", + 141733920878u64 => "UNKNOWN_OPERATION", + 141733920879u64 => "UNSUPPORTED_CIPHER_TYPE", + 141733920880u64 => "UNSUPPORTED_CONTENT_TYPE", + 141733920881u64 => "WRONG_CONTENT_TYPE", + 141733920882u64 => "WRONG_PKCS7_TYPE", + 236223201380u64 => "NAME_TOO_LONG", + 236223201381u64 => "NOT_AN_ASCII_CHARACTER", + 236223201382u64 => "NOT_AN_HEXADECIMAL_DIGIT", + 236223201383u64 => "NOT_AN_IDENTIFIER", + 236223201384u64 => "NOT_AN_OCTAL_DIGIT", + 236223201385u64 => "NOT_A_DECIMAL_DIGIT", + 236223201386u64 => "NO_MATCHING_STRING_DELIMITER", + 236223201387u64 => "NO_VALUE", + 236223201388u64 => "PARSE_FAILED", + 236223201389u64 => "STRING_TOO_LONG", + 236223201390u64 => "TRAILING_CHARACTERS", + 244813136056u64 => "ADDITIONAL_INPUT_TOO_LONG", + 244813136045u64 => "ALGORITHM_MISMATCH", + 244813136057u64 => "ALREADY_INSTANTIATED", + 244813135972u64 => "BAD_DECRYPT", + 244813136013u64 => "BAD_ENCODING", + 244813136014u64 => "BAD_LENGTH", + 244813136033u64 => "BAD_TLS_CLIENT_VERSION", + 244813136032u64 => "BN_ERROR", + 244813135974u64 => "CIPHER_OPERATION_FAILED", + 244813136077u64 => "DERIVATION_FUNCTION_INIT_FAILED", + 244813136046u64 => "DIGEST_NOT_ALLOWED", + 244813136105u64 => "EMS_NOT_ENABLED", + 244813136058u64 => "ENTROPY_SOURCE_STRENGTH_TOO_WEAK", + 244813136060u64 => "ERROR_INSTANTIATING_DRBG", + 244813136061u64 => "ERROR_RETRIEVING_ENTROPY", + 244813136062u64 => "ERROR_RETRIEVING_NONCE", + 244813136036u64 => "FAILED_DURING_DERIVATION", + 244813136052u64 => "FAILED_TO_CREATE_LOCK", + 244813136034u64 => "FAILED_TO_DECRYPT", + 244813135993u64 => "FAILED_TO_GENERATE_KEY", + 244813135975u64 => "FAILED_TO_GET_PARAMETER", + 244813135976u64 => "FAILED_TO_SET_PARAMETER", + 244813136047u64 => "FAILED_TO_SIGN", + 244813136099u64 => "FIPS_MODULE_CONDITIONAL_ERROR", + 244813136096u64 => "FIPS_MODULE_ENTERING_ERROR_STATE", + 244813136097u64 => "FIPS_MODULE_IN_ERROR_STATE", + 244813136063u64 => "GENERATE_ERROR", + 244813136037u64 => "ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", + 244813136082u64 => "INDICATOR_INTEGRITY_FAILURE", + 244813136053u64 => "INSUFFICIENT_DRBG_STRENGTH", + 244813135980u64 => "INVALID_AAD", + 244813136083u64 => "INVALID_CONFIG_DATA", + 244813136029u64 => "INVALID_CONSTANT_LENGTH", + 244813136048u64 => "INVALID_CURVE", + 244813135983u64 => "INVALID_CUSTOM_LENGTH", + 244813135987u64 => "INVALID_DATA", + 244813135994u64 => "INVALID_DIGEST", + 244813136038u64 => "INVALID_DIGEST_LENGTH", + 244813136090u64 => "INVALID_DIGEST_SIZE", + 244813136102u64 => "INVALID_INPUT_LENGTH", + 244813135995u64 => "INVALID_ITERATION_COUNT", + 244813135981u64 => "INVALID_IV_LENGTH", + 244813136030u64 => "INVALID_KEY", + 244813135977u64 => "INVALID_KEY_LENGTH", + 244813136023u64 => "INVALID_MAC", + 244813136039u64 => "INVALID_MGF1_MD", + 244813135997u64 => "INVALID_MODE", + 244813136089u64 => "INVALID_OUTPUT_LENGTH", + 244813136040u64 => "INVALID_PADDING_MODE", + 244813136070u64 => "INVALID_PUBINFO", + 244813135984u64 => "INVALID_SALT_LENGTH", + 244813136026u64 => "INVALID_SEED_LENGTH", + 244813136051u64 => "INVALID_SIGNATURE_SIZE", + 244813136084u64 => "INVALID_STATE", + 244813135982u64 => "INVALID_TAG", + 244813135990u64 => "INVALID_TAG_LENGTH", + 244813136072u64 => "INVALID_UKM_LENGTH", + 244813136042u64 => "INVALID_X931_DIGEST", + 244813136064u64 => "IN_ERROR_STATE", + 244813135973u64 => "KEY_SETUP_FAILED", + 244813136043u64 => "KEY_SIZE_TOO_SMALL", + 244813136074u64 => "LENGTH_TOO_LARGE", + 244813136075u64 => "MISMATCHING_DOMAIN_PARAMETERS", + 244813136016u64 => "MISSING_CEK_ALG", + 244813136027u64 => "MISSING_CIPHER", + 244813136085u64 => "MISSING_CONFIG_DATA", + 244813136028u64 => "MISSING_CONSTANT", + 244813136000u64 => "MISSING_KEY", + 244813136022u64 => "MISSING_MAC", + 244813136001u64 => "MISSING_MESSAGE_DIGEST", + 244813136081u64 => "MISSING_OID", + 244813136002u64 => "MISSING_PASS", + 244813136003u64 => "MISSING_SALT", + 244813136004u64 => "MISSING_SECRET", + 244813136012u64 => "MISSING_SEED", + 244813136005u64 => "MISSING_SESSION_ID", + 244813136006u64 => "MISSING_TYPE", + 244813136007u64 => "MISSING_XCGHASH", + 244813136086u64 => "MODULE_INTEGRITY_FAILURE", + 244813136093u64 => "NOT_A_PRIVATE_KEY", + 244813136092u64 => "NOT_A_PUBLIC_KEY", + 244813136065u64 => "NOT_INSTANTIATED", + 244813136098u64 => "NOT_PARAMETERS", + 244813136008u64 => "NOT_SUPPORTED", + 244813135985u64 => "NOT_XOF_OR_INVALID_LENGTH", + 244813135986u64 => "NO_KEY_SET", + 244813136049u64 => "NO_PARAMETERS_SET", + 244813136050u64 => "OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", + 244813135978u64 => "OUTPUT_BUFFER_TOO_SMALL", + 244813136100u64 => "PARENT_CANNOT_GENERATE_RANDOM_NUMBERS", + 244813136059u64 => "PARENT_CANNOT_SUPPLY_ENTROPY_SEED", + 244813136054u64 => "PARENT_LOCKING_NOT_ENABLED", + 244813136066u64 => "PARENT_STRENGTH_TOO_WEAK", + 244813136091u64 => "PATH_MUST_BE_ABSOLUTE", + 244813136067u64 => "PERSONALISATION_STRING_TOO_LONG", + 244813136044u64 => "PSS_SALTLEN_TOO_SMALL", + 244813136068u64 => "REQUEST_TOO_LARGE_FOR_DRBG", + 244813136078u64 => "REQUIRE_CTR_MODE_CIPHER", + 244813136069u64 => "RESEED_ERROR", + 244813136094u64 => "SEARCH_ONLY_SUPPORTED_FOR_DIRECTORIES", + 244813136101u64 => "SEED_SOURCES_MUST_NOT_HAVE_A_PARENT", + 244813136087u64 => "SELF_TEST_KAT_FAILURE", + 244813136088u64 => "SELF_TEST_POST_FAILURE", + 244813135992u64 => "TAG_NOT_NEEDED", + 244813135991u64 => "TAG_NOT_SET", + 244813135998u64 => "TOO_MANY_RECORDS", + 244813136079u64 => "UNABLE_TO_FIND_CIPHERS", + 244813136071u64 => "UNABLE_TO_GET_PARENT_STRENGTH", + 244813136031u64 => "UNABLE_TO_GET_PASSPHRASE", + 244813136080u64 => "UNABLE_TO_INITIALISE_CIPHERS", + 244813136019u64 => "UNABLE_TO_LOAD_SHA256", + 244813136073u64 => "UNABLE_TO_LOCK_PARENT", + 244813136076u64 => "UNABLE_TO_RESEED", + 244813136017u64 => "UNSUPPORTED_CEK_ALG", + 244813136025u64 => "UNSUPPORTED_KEY_SIZE", + 244813136009u64 => "UNSUPPORTED_MAC_TYPE", + 244813136024u64 => "UNSUPPORTED_NUMBER_OF_ROUNDS", + 244813136095u64 => "URI_AUTHORITY_UNSUPPORTED", + 244813136010u64 => "VALUE_ERROR", + 244813135979u64 => "WRONG_FINAL_BLOCK_LENGTH", + 244813136011u64 => "WRONG_OUTPUT_BUFFER_SIZE", + 244813136055u64 => "XOF_DIGESTS_NOT_ALLOWED", + 244813136020u64 => "XTS_DATA_UNIT_IS_TOO_LARGE", + 244813136021u64 => "XTS_DUPLICATED_KEYS", + 154618822758u64 => "ADDITIONAL_INPUT_TOO_LONG", + 154618822759u64 => "ALREADY_INSTANTIATED", + 154618822761u64 => "ARGUMENT_OUT_OF_RANGE", + 154618822777u64 => "CANNOT_OPEN_FILE", + 154618822785u64 => "DRBG_ALREADY_INITIALIZED", + 154618822760u64 => "DRBG_NOT_INITIALISED", + 154618822762u64 => "ENTROPY_INPUT_TOO_LONG", + 154618822780u64 => "ENTROPY_OUT_OF_RANGE", + 154618822783u64 => "ERROR_ENTROPY_POOL_WAS_IGNORED", + 154618822763u64 => "ERROR_INITIALISING_DRBG", + 154618822764u64 => "ERROR_INSTANTIATING_DRBG", + 154618822765u64 => "ERROR_RETRIEVING_ADDITIONAL_INPUT", + 154618822766u64 => "ERROR_RETRIEVING_ENTROPY", + 154618822767u64 => "ERROR_RETRIEVING_NONCE", + 154618822782u64 => "FAILED_TO_CREATE_LOCK", + 154618822757u64 => "FUNC_NOT_IMPLEMENTED", + 154618822779u64 => "FWRITE_ERROR", + 154618822768u64 => "GENERATE_ERROR", + 154618822795u64 => "INSUFFICIENT_DRBG_STRENGTH", + 154618822769u64 => "INTERNAL_ERROR", + 154618822793u64 => "INVALID_PROPERTY_QUERY", + 154618822770u64 => "IN_ERROR_STATE", + 154618822778u64 => "NOT_A_REGULAR_FILE", + 154618822771u64 => "NOT_INSTANTIATED", + 154618822784u64 => "NO_DRBG_IMPLEMENTATION_SELECTED", + 154618822786u64 => "PARENT_LOCKING_NOT_ENABLED", + 154618822787u64 => "PARENT_STRENGTH_TOO_WEAK", + 154618822772u64 => "PERSONALISATION_STRING_TOO_LONG", + 154618822789u64 => "PREDICTION_RESISTANCE_NOT_SUPPORTED", + 154618822756u64 => "PRNG_NOT_SEEDED", + 154618822781u64 => "RANDOM_POOL_OVERFLOW", + 154618822790u64 => "RANDOM_POOL_UNDERFLOW", + 154618822773u64 => "REQUEST_TOO_LARGE_FOR_DRBG", + 154618822774u64 => "RESEED_ERROR", + 154618822775u64 => "SELFTEST_FAILURE", + 154618822791u64 => "TOO_LITTLE_NONCE_REQUESTED", + 154618822792u64 => "TOO_MUCH_NONCE_REQUESTED", + 154618822799u64 => "UNABLE_TO_CREATE_DRBG", + 154618822800u64 => "UNABLE_TO_FETCH_DRBG", + 154618822797u64 => "UNABLE_TO_GET_PARENT_RESEED_PROP_COUNTER", + 154618822794u64 => "UNABLE_TO_GET_PARENT_STRENGTH", + 154618822796u64 => "UNABLE_TO_LOCK_PARENT", + 154618822788u64 => "UNSUPPORTED_DRBG_FLAGS", + 154618822776u64 => "UNSUPPORTED_DRBG_TYPE", + 17179869284u64 => "ALGORITHM_MISMATCH", + 17179869285u64 => "BAD_E_VALUE", + 17179869286u64 => "BAD_FIXED_HEADER_DECRYPT", + 17179869287u64 => "BAD_PAD_BYTE_COUNT", + 17179869288u64 => "BAD_SIGNATURE", + 17179869290u64 => "BLOCK_TYPE_IS_NOT_01", + 17179869291u64 => "BLOCK_TYPE_IS_NOT_02", + 17179869292u64 => "DATA_GREATER_THAN_MOD_LEN", + 17179869293u64 => "DATA_TOO_LARGE", + 17179869294u64 => "DATA_TOO_LARGE_FOR_KEY_SIZE", + 17179869316u64 => "DATA_TOO_LARGE_FOR_MODULUS", + 17179869295u64 => "DATA_TOO_SMALL", + 17179869306u64 => "DATA_TOO_SMALL_FOR_KEY_SIZE", + 17179869342u64 => "DIGEST_DOES_NOT_MATCH", + 17179869329u64 => "DIGEST_NOT_ALLOWED", + 17179869296u64 => "DIGEST_TOO_BIG_FOR_RSA_KEY", + 17179869308u64 => "DMP1_NOT_CONGRUENT_TO_D", + 17179869309u64 => "DMQ1_NOT_CONGRUENT_TO_D", + 17179869307u64 => "D_E_NOT_CONGRUENT_TO_1", + 17179869317u64 => "FIRST_OCTET_INVALID", + 17179869328u64 => "ILLEGAL_OR_UNSUPPORTED_PADDING_MODE", + 17179869341u64 => "INVALID_DIGEST", + 17179869327u64 => "INVALID_DIGEST_LENGTH", + 17179869321u64 => "INVALID_HEADER", + 17179869355u64 => "INVALID_KEYPAIR", + 17179869357u64 => "INVALID_KEY_LENGTH", + 17179869344u64 => "INVALID_LABEL", + 17179869365u64 => "INVALID_LENGTH", + 17179869315u64 => "INVALID_MESSAGE_LENGTH", + 17179869340u64 => "INVALID_MGF1_MD", + 17179869358u64 => "INVALID_MODULUS", + 17179869351u64 => "INVALID_MULTI_PRIME_KEY", + 17179869345u64 => "INVALID_OAEP_PARAMETERS", + 17179869322u64 => "INVALID_PADDING", + 17179869325u64 => "INVALID_PADDING_MODE", + 17179869333u64 => "INVALID_PSS_PARAMETERS", + 17179869330u64 => "INVALID_PSS_SALTLEN", + 17179869359u64 => "INVALID_REQUEST", + 17179869334u64 => "INVALID_SALT_LENGTH", + 17179869360u64 => "INVALID_STRENGTH", + 17179869323u64 => "INVALID_TRAILER", + 17179869326u64 => "INVALID_X931_DIGEST", + 17179869310u64 => "IQMP_NOT_INVERSE_OF_Q", + 17179869349u64 => "KEY_PRIME_NUM_INVALID", + 17179869304u64 => "KEY_SIZE_TOO_SMALL", + 17179869318u64 => "LAST_OCTET_INVALID", + 17179869336u64 => "MGF1_DIGEST_NOT_ALLOWED", + 17179869363u64 => "MISSING_PRIVATE_KEY", + 17179869289u64 => "MODULUS_TOO_LARGE", + 17179869352u64 => "MP_COEFFICIENT_NOT_INVERSE_OF_R", + 17179869353u64 => "MP_EXPONENT_NOT_CONGRUENT_TO_D", + 17179869354u64 => "MP_R_NOT_PRIME", + 17179869324u64 => "NO_PUBLIC_EXPONENT", + 17179869297u64 => "NULL_BEFORE_BLOCK_MISSING", + 17179869356u64 => "N_DOES_NOT_EQUAL_PRODUCT_OF_PRIMES", + 17179869311u64 => "N_DOES_NOT_EQUAL_P_Q", + 17179869305u64 => "OAEP_DECODING_ERROR", + 17179869332u64 => "OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE", + 17179869298u64 => "PADDING_CHECK_FAILED", + 17179869361u64 => "PAIRWISE_TEST_FAILURE", + 17179869343u64 => "PKCS_DECODING_ERROR", + 17179869348u64 => "PSS_SALTLEN_TOO_SMALL", + 17179869362u64 => "PUB_EXPONENT_OUT_OF_RANGE", + 17179869312u64 => "P_NOT_PRIME", + 17179869313u64 => "Q_NOT_PRIME", + 17179869364u64 => "RANDOMNESS_SOURCE_STRENGTH_INSUFFICIENT", + 17179869314u64 => "RSA_OPERATIONS_NOT_SUPPORTED", + 17179869320u64 => "SLEN_CHECK_FAILED", + 17179869319u64 => "SLEN_RECOVERY_FAILED", + 17179869299u64 => "SSLV3_ROLLBACK_ATTACK", + 17179869300u64 => "THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD", + 17179869301u64 => "UNKNOWN_ALGORITHM_TYPE", + 17179869350u64 => "UNKNOWN_DIGEST", + 17179869335u64 => "UNKNOWN_MASK_DIGEST", + 17179869302u64 => "UNKNOWN_PADDING_TYPE", + 17179869346u64 => "UNSUPPORTED_ENCRYPTION_TYPE", + 17179869347u64 => "UNSUPPORTED_LABEL_SOURCE", + 17179869337u64 => "UNSUPPORTED_MASK_ALGORITHM", + 17179869338u64 => "UNSUPPORTED_MASK_PARAMETER", + 17179869339u64 => "UNSUPPORTED_SIGNATURE_TYPE", + 17179869331u64 => "VALUE_MISSING", + 17179869303u64 => "WRONG_SIGNATURE_LENGTH", + 227633266788u64 => "ASN1_ERROR", + 227633266789u64 => "BAD_SIGNATURE", + 227633266795u64 => "BUFFER_TOO_SMALL", + 227633266798u64 => "DIST_ID_TOO_LARGE", + 227633266800u64 => "ID_NOT_SET", + 227633266799u64 => "ID_TOO_LARGE", + 227633266796u64 => "INVALID_CURVE", + 227633266790u64 => "INVALID_DIGEST", + 227633266791u64 => "INVALID_DIGEST_TYPE", + 227633266792u64 => "INVALID_ENCODING", + 227633266793u64 => "INVALID_FIELD", + 227633266801u64 => "INVALID_PRIVATE_KEY", + 227633266797u64 => "NO_PARAMETERS_SET", + 227633266794u64 => "USER_ID_TOO_LARGE", + 85899346211u64 => "APPLICATION_DATA_AFTER_CLOSE_NOTIFY", + 85899346020u64 => "APP_DATA_IN_HANDSHAKE", + 85899346192u64 => "ATTEMPT_TO_REUSE_SESSION_IN_DIFFERENT_CONTEXT", + 85899346078u64 => "AT_LEAST_TLS_1_2_NEEDED_IN_SUITEB_MODE", + 85899346023u64 => "BAD_CHANGE_CIPHER_SPEC", + 85899346106u64 => "BAD_CIPHER", + 85899346310u64 => "BAD_DATA", + 85899346026u64 => "BAD_DATA_RETURNED_BY_CALLBACK", + 85899346027u64 => "BAD_DECOMPRESSION", + 85899346022u64 => "BAD_DH_VALUE", + 85899346031u64 => "BAD_DIGEST_LENGTH", + 85899346153u64 => "BAD_EARLY_DATA", + 85899346224u64 => "BAD_ECC_CERT", + 85899346226u64 => "BAD_ECPOINT", + 85899346030u64 => "BAD_EXTENSION", + 85899346252u64 => "BAD_HANDSHAKE_LENGTH", + 85899346156u64 => "BAD_HANDSHAKE_STATE", + 85899346025u64 => "BAD_HELLO_REQUEST", + 85899346183u64 => "BAD_HRR_VERSION", + 85899346028u64 => "BAD_KEY_SHARE", + 85899346042u64 => "BAD_KEY_UPDATE", + 85899346212u64 => "BAD_LEGACY_VERSION", + 85899346191u64 => "BAD_LENGTH", + 85899346160u64 => "BAD_PACKET", + 85899346035u64 => "BAD_PACKET_LENGTH", + 85899346036u64 => "BAD_PROTOCOL_VERSION_NUMBER", + 85899346139u64 => "BAD_PSK", + 85899346034u64 => "BAD_PSK_IDENTITY", + 85899346363u64 => "BAD_RECORD_TYPE", + 85899346039u64 => "BAD_RSA_ENCRYPT", + 85899346043u64 => "BAD_SIGNATURE", + 85899346267u64 => "BAD_SRP_A_LENGTH", + 85899346291u64 => "BAD_SRP_PARAMETERS", + 85899346272u64 => "BAD_SRTP_MKI_VALUE", + 85899346273u64 => "BAD_SRTP_PROTECTION_PROFILE_LIST", + 85899346044u64 => "BAD_SSL_FILETYPE", + 85899346304u64 => "BAD_VALUE", + 85899346047u64 => "BAD_WRITE_RETRY", + 85899346173u64 => "BINDER_DOES_NOT_VERIFY", + 85899346048u64 => "BIO_NOT_SET", + 85899346049u64 => "BLOCK_CIPHER_PAD_IS_WRONG", + 85899346050u64 => "BN_LIB", + 85899346154u64 => "CALLBACK_FAILED", + 85899346029u64 => "CANNOT_CHANGE_CIPHER", + 85899346219u64 => "CANNOT_GET_GROUP_NAME", + 85899346051u64 => "CA_DN_LENGTH_MISMATCH", + 85899346317u64 => "CA_KEY_TOO_SMALL", + 85899346318u64 => "CA_MD_TOO_WEAK", + 85899346053u64 => "CCS_RECEIVED_EARLY", + 85899346054u64 => "CERTIFICATE_VERIFY_FAILED", + 85899346297u64 => "CERT_CB_ERROR", + 85899346055u64 => "CERT_LENGTH_MISMATCH", + 85899346138u64 => "CIPHERSUITE_DIGEST_HAS_CHANGED", + 85899346057u64 => "CIPHER_CODE_WRONG_LENGTH", + 85899346146u64 => "CLIENTHELLO_TLSEXT", + 85899346060u64 => "COMPRESSED_LENGTH_TOO_LONG", + 85899346263u64 => "COMPRESSION_DISABLED", + 85899346061u64 => "COMPRESSION_FAILURE", + 85899346227u64 => "COMPRESSION_ID_NOT_WITHIN_PRIVATE_RANGE", + 85899346062u64 => "COMPRESSION_LIBRARY_ERROR", + 85899346064u64 => "CONNECTION_TYPE_NOT_SET", + 85899346087u64 => "CONTEXT_NOT_DANE_ENABLED", + 85899346320u64 => "COOKIE_GEN_CALLBACK_FAILURE", + 85899346228u64 => "COOKIE_MISMATCH", + 85899346216u64 => "COPY_PARAMETERS_FAILED", + 85899346126u64 => "CUSTOM_EXT_HANDLER_ALREADY_INSTALLED", + 85899346092u64 => "DANE_ALREADY_ENABLED", + 85899346093u64 => "DANE_CANNOT_OVERRIDE_MTYPE_FULL", + 85899346095u64 => "DANE_NOT_ENABLED", + 85899346100u64 => "DANE_TLSA_BAD_CERTIFICATE", + 85899346104u64 => "DANE_TLSA_BAD_CERTIFICATE_USAGE", + 85899346109u64 => "DANE_TLSA_BAD_DATA_LENGTH", + 85899346112u64 => "DANE_TLSA_BAD_DIGEST_LENGTH", + 85899346120u64 => "DANE_TLSA_BAD_MATCHING_TYPE", + 85899346121u64 => "DANE_TLSA_BAD_PUBLIC_KEY", + 85899346122u64 => "DANE_TLSA_BAD_SELECTOR", + 85899346123u64 => "DANE_TLSA_NULL_DATA", + 85899346065u64 => "DATA_BETWEEN_CCS_AND_FINISHED", + 85899346066u64 => "DATA_LENGTH_TOO_LONG", + 85899346067u64 => "DECRYPTION_FAILED", + 85899346201u64 => "DECRYPTION_FAILED_OR_BAD_RECORD_MAC", + 85899346314u64 => "DH_KEY_TOO_SMALL", + 85899346068u64 => "DH_PUBLIC_VALUE_LENGTH_IS_WRONG", + 85899346069u64 => "DIGEST_CHECK_FAILED", + 85899346254u64 => "DTLS_MESSAGE_TOO_BIG", + 85899346229u64 => "DUPLICATE_COMPRESSION_ID", + 85899346238u64 => "ECC_CERT_NOT_FOR_SIGNING", + 85899346294u64 => "ECDH_REQUIRED_FOR_SUITEB_MODE", + 85899346319u64 => "EE_KEY_TOO_SMALL", + 85899346274u64 => "EMPTY_SRTP_PROTECTION_PROFILE_LIST", + 85899346070u64 => "ENCRYPTED_LENGTH_TOO_LONG", + 85899346071u64 => "ERROR_IN_RECEIVED_CIPHER_LIST", + 85899346124u64 => "ERROR_SETTING_TLSA_BASE_DOMAIN", + 85899346114u64 => "EXCEEDS_MAX_FRAGMENT_SIZE", + 85899346072u64 => "EXCESSIVE_MESSAGE_SIZE", + 85899346199u64 => "EXTENSION_NOT_RECEIVED", + 85899346073u64 => "EXTRA_DATA_IN_MESSAGE", + 85899346083u64 => "EXT_LENGTH_MISMATCH", + 85899346325u64 => "FAILED_TO_INIT_ASYNC", + 85899346321u64 => "FRAGMENTED_CLIENT_HELLO", + 85899346074u64 => "GOT_A_FIN_BEFORE_A_CCS", + 85899346075u64 => "HTTPS_PROXY_REQUEST", + 85899346076u64 => "HTTP_REQUEST", + 85899346082u64 => "ILLEGAL_POINT_COMPRESSION", + 85899346300u64 => "ILLEGAL_SUITEB_DIGEST", + 85899346293u64 => "INAPPROPRIATE_FALLBACK", + 85899346260u64 => "INCONSISTENT_COMPRESSION", + 85899346142u64 => "INCONSISTENT_EARLY_DATA_ALPN", + 85899346151u64 => "INCONSISTENT_EARLY_DATA_SNI", + 85899346024u64 => "INCONSISTENT_EXTMS", + 85899346161u64 => "INSUFFICIENT_SECURITY", + 85899346125u64 => "INVALID_ALERT", + 85899346180u64 => "INVALID_CCS_MESSAGE", + 85899346158u64 => "INVALID_CERTIFICATE_OR_ALG", + 85899346200u64 => "INVALID_COMMAND", + 85899346261u64 => "INVALID_COMPRESSION_ALGORITHM", + 85899346203u64 => "INVALID_CONFIG", + 85899346033u64 => "INVALID_CONFIGURATION_NAME", + 85899346202u64 => "INVALID_CONTEXT", + 85899346132u64 => "INVALID_CT_VALIDATION_TYPE", + 85899346040u64 => "INVALID_KEY_UPDATE_TYPE", + 85899346094u64 => "INVALID_MAX_EARLY_DATA", + 85899346305u64 => "INVALID_NULL_CMD_NAME", + 85899346322u64 => "INVALID_SEQUENCE_NUMBER", + 85899346308u64 => "INVALID_SERVERINFO_DATA", + 85899346919u64 => "INVALID_SESSION_ID", + 85899346277u64 => "INVALID_SRP_USERNAME", + 85899346248u64 => "INVALID_STATUS_RESPONSE", + 85899346245u64 => "INVALID_TICKET_KEYS_LENGTH", + 85899346253u64 => "LEGACY_SIGALG_DISALLOWED_OR_UNSUPPORTED", + 85899346079u64 => "LENGTH_MISMATCH", + 85899346324u64 => "LENGTH_TOO_LONG", + 85899346080u64 => "LENGTH_TOO_SHORT", + 85899346194u64 => "LIBRARY_BUG", + 85899346081u64 => "LIBRARY_HAS_NO_CIPHERS", + 85899346085u64 => "MISSING_DSA_SIGNING_CERT", + 85899346301u64 => "MISSING_ECDSA_SIGNING_CERT", + 85899346176u64 => "MISSING_FATAL", + 85899346210u64 => "MISSING_PARAMETERS", + 85899346230u64 => "MISSING_PSK_KEX_MODES_EXTENSION", + 85899346088u64 => "MISSING_RSA_CERTIFICATE", + 85899346089u64 => "MISSING_RSA_ENCRYPTING_CERT", + 85899346090u64 => "MISSING_RSA_SIGNING_CERT", + 85899346032u64 => "MISSING_SIGALGS_EXTENSION", + 85899346141u64 => "MISSING_SIGNING_CERT", + 85899346278u64 => "MISSING_SRP_PARAM", + 85899346129u64 => "MISSING_SUPPORTED_GROUPS_EXTENSION", + 85899346091u64 => "MISSING_TMP_DH_KEY", + 85899346231u64 => "MISSING_TMP_ECDH_KEY", + 85899346213u64 => "MIXED_HANDSHAKE_AND_NON_HANDSHAKE_DATA", + 85899346102u64 => "NOT_ON_RECORD_BOUNDARY", + 85899346209u64 => "NOT_REPLACING_CERTIFICATE", + 85899346204u64 => "NOT_SERVER", + 85899346155u64 => "NO_APPLICATION_PROTOCOL", + 85899346096u64 => "NO_CERTIFICATES_RETURNED", + 85899346097u64 => "NO_CERTIFICATE_ASSIGNED", + 85899346099u64 => "NO_CERTIFICATE_SET", + 85899346134u64 => "NO_CHANGE_FOLLOWING_HRR", + 85899346101u64 => "NO_CIPHERS_AVAILABLE", + 85899346103u64 => "NO_CIPHERS_SPECIFIED", + 85899346105u64 => "NO_CIPHER_MATCH", + 85899346251u64 => "NO_CLIENT_CERT_METHOD", + 85899346107u64 => "NO_COMPRESSION_SPECIFIED", + 85899346207u64 => "NO_COOKIE_CALLBACK_SET", + 85899346250u64 => "NO_GOST_CERTIFICATE_SENT_BY_PEER", + 85899346108u64 => "NO_METHOD_SPECIFIED", + 85899346309u64 => "NO_PEM_EXTENSIONS", + 85899346110u64 => "NO_PRIVATE_KEY_ASSIGNED", + 85899346111u64 => "NO_PROTOCOLS_AVAILABLE", + 85899346259u64 => "NO_RENEGOTIATION", + 85899346244u64 => "NO_REQUIRED_DIGEST", + 85899346113u64 => "NO_SHARED_CIPHER", + 85899346330u64 => "NO_SHARED_GROUPS", + 85899346296u64 => "NO_SHARED_SIGNATURE_ALGORITHMS", + 85899346279u64 => "NO_SRTP_PROFILES", + 85899346217u64 => "NO_SUITABLE_DIGEST_ALGORITHM", + 85899346215u64 => "NO_SUITABLE_GROUPS", + 85899346021u64 => "NO_SUITABLE_KEY_SHARE", + 85899346038u64 => "NO_SUITABLE_SIGNATURE_ALGORITHM", + 85899346136u64 => "NO_VALID_SCTS", + 85899346323u64 => "NO_VERIFY_COOKIE_CALLBACK", + 85899346115u64 => "NULL_SSL_CTX", + 85899346116u64 => "NULL_SSL_METHOD_PASSED", + 85899346225u64 => "OCSP_CALLBACK_FAILURE", + 85899346117u64 => "OLD_SESSION_CIPHER_NOT_RETURNED", + 85899346264u64 => "OLD_SESSION_COMPRESSION_ALGORITHM_NOT_RETURNED", + 85899346157u64 => "OVERFLOW_ERROR", + 85899346118u64 => "PACKET_LENGTH_TOO_LONG", + 85899346147u64 => "PARSE_TLSEXT", + 85899346190u64 => "PATH_TOO_LONG", + 85899346119u64 => "PEER_DID_NOT_RETURN_A_CERTIFICATE", + 85899346311u64 => "PEM_NAME_BAD_PREFIX", + 85899346312u64 => "PEM_NAME_TOO_SHORT", + 85899346326u64 => "PIPELINE_FAILURE", + 85899346198u64 => "POST_HANDSHAKE_AUTH_ENCODING_ERR", + 85899346208u64 => "PRIVATE_KEY_MISMATCH", + 85899346127u64 => "PROTOCOL_IS_SHUTDOWN", + 85899346143u64 => "PSK_IDENTITY_NOT_FOUND", + 85899346144u64 => "PSK_NO_CLIENT_CB", + 85899346145u64 => "PSK_NO_SERVER_CB", + 85899346131u64 => "READ_BIO_NOT_SET", + 85899346232u64 => "READ_TIMEOUT_EXPIRED", + 85899346133u64 => "RECORD_LENGTH_MISMATCH", + 85899346218u64 => "RECORD_TOO_SMALL", + 85899346255u64 => "RENEGOTIATE_EXT_TOO_LONG", + 85899346256u64 => "RENEGOTIATION_ENCODING_ERR", + 85899346257u64 => "RENEGOTIATION_MISMATCH", + 85899346205u64 => "REQUEST_PENDING", + 85899346206u64 => "REQUEST_SENT", + 85899346135u64 => "REQUIRED_CIPHER_MISSING", + 85899346262u64 => "REQUIRED_COMPRESSION_ALGORITHM_MISSING", + 85899346265u64 => "SCSV_RECEIVED_WHEN_RENEGOTIATING", + 85899346128u64 => "SCT_VERIFICATION_FAILED", + 85899346195u64 => "SERVERHELLO_TLSEXT", + 85899346197u64 => "SESSION_ID_CONTEXT_UNINITIALIZED", + 85899346327u64 => "SHUTDOWN_WHILE_IN_INIT", + 85899346280u64 => "SIGNATURE_ALGORITHMS_ERROR", + 85899346140u64 => "SIGNATURE_FOR_NON_SIGNING_CERTIFICATE", + 85899346281u64 => "SRP_A_CALC", + 85899346282u64 => "SRTP_COULD_NOT_ALLOCATE_PROFILES", + 85899346283u64 => "SRTP_PROTECTION_PROFILE_LIST_TOO_LONG", + 85899346284u64 => "SRTP_UNKNOWN_PROTECTION_PROFILE", + 85899346152u64 => "SSL3_EXT_INVALID_MAX_FRAGMENT_LENGTH", + 85899346239u64 => "SSL3_EXT_INVALID_SERVERNAME", + 85899346240u64 => "SSL3_EXT_INVALID_SERVERNAME_TYPE", + 85899346220u64 => "SSL3_SESSION_ID_TOO_LONG", + 85899346962u64 => "SSLV3_ALERT_BAD_CERTIFICATE", + 85899346940u64 => "SSLV3_ALERT_BAD_RECORD_MAC", + 85899346965u64 => "SSLV3_ALERT_CERTIFICATE_EXPIRED", + 85899346964u64 => "SSLV3_ALERT_CERTIFICATE_REVOKED", + 85899346966u64 => "SSLV3_ALERT_CERTIFICATE_UNKNOWN", + 85899346950u64 => "SSLV3_ALERT_DECOMPRESSION_FAILURE", + 85899346960u64 => "SSLV3_ALERT_HANDSHAKE_FAILURE", + 85899346967u64 => "SSLV3_ALERT_ILLEGAL_PARAMETER", + 85899346961u64 => "SSLV3_ALERT_NO_CERTIFICATE", + 85899346930u64 => "SSLV3_ALERT_UNEXPECTED_MESSAGE", + 85899346963u64 => "SSLV3_ALERT_UNSUPPORTED_CERTIFICATE", + 85899346037u64 => "SSL_COMMAND_SECTION_EMPTY", + 85899346045u64 => "SSL_COMMAND_SECTION_NOT_FOUND", + 85899346148u64 => "SSL_CTX_HAS_NO_DEFAULT_SSL_VERSION", + 85899346149u64 => "SSL_HANDSHAKE_FAILURE", + 85899346150u64 => "SSL_LIBRARY_HAS_NO_CIPHERS", + 85899346292u64 => "SSL_NEGATIVE_LENGTH", + 85899346046u64 => "SSL_SECTION_EMPTY", + 85899346056u64 => "SSL_SECTION_NOT_FOUND", + 85899346221u64 => "SSL_SESSION_ID_CALLBACK_FAILED", + 85899346222u64 => "SSL_SESSION_ID_CONFLICT", + 85899346193u64 => "SSL_SESSION_ID_CONTEXT_TOO_LONG", + 85899346223u64 => "SSL_SESSION_ID_HAS_BAD_LENGTH", + 85899346328u64 => "SSL_SESSION_ID_TOO_LONG", + 85899346130u64 => "SSL_SESSION_VERSION_MISMATCH", + 85899346041u64 => "STILL_IN_INIT", + 85899347036u64 => "TLSV13_ALERT_CERTIFICATE_REQUIRED", + 85899347029u64 => "TLSV13_ALERT_MISSING_EXTENSION", + 85899346969u64 => "TLSV1_ALERT_ACCESS_DENIED", + 85899346970u64 => "TLSV1_ALERT_DECODE_ERROR", + 85899346941u64 => "TLSV1_ALERT_DECRYPTION_FAILED", + 85899346971u64 => "TLSV1_ALERT_DECRYPT_ERROR", + 85899346980u64 => "TLSV1_ALERT_EXPORT_RESTRICTION", + 85899347006u64 => "TLSV1_ALERT_INAPPROPRIATE_FALLBACK", + 85899346991u64 => "TLSV1_ALERT_INSUFFICIENT_SECURITY", + 85899347000u64 => "TLSV1_ALERT_INTERNAL_ERROR", + 85899347040u64 => "TLSV1_ALERT_NO_APPLICATION_PROTOCOL", + 85899347020u64 => "TLSV1_ALERT_NO_RENEGOTIATION", + 85899346990u64 => "TLSV1_ALERT_PROTOCOL_VERSION", + 85899346942u64 => "TLSV1_ALERT_RECORD_OVERFLOW", + 85899346968u64 => "TLSV1_ALERT_UNKNOWN_CA", + 85899347035u64 => "TLSV1_ALERT_UNKNOWN_PSK_IDENTITY", + 85899347010u64 => "TLSV1_ALERT_USER_CANCELLED", + 85899347034u64 => "TLSV1_BAD_CERTIFICATE_HASH_VALUE", + 85899347033u64 => "TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE", + 85899347031u64 => "TLSV1_CERTIFICATE_UNOBTAINABLE", + 85899347032u64 => "TLSV1_UNRECOGNIZED_NAME", + 85899347030u64 => "TLSV1_UNSUPPORTED_EXTENSION", + 85899346287u64 => "TLS_ILLEGAL_EXPORTER_LABEL", + 85899346077u64 => "TLS_INVALID_ECPOINTFORMAT_LIST", + 85899346052u64 => "TOO_MANY_KEY_UPDATES", + 85899346329u64 => "TOO_MANY_WARN_ALERTS", + 85899346084u64 => "TOO_MUCH_EARLY_DATA", + 85899346234u64 => "UNABLE_TO_FIND_ECDH_PARAMETERS", + 85899346159u64 => "UNABLE_TO_FIND_PUBLIC_KEY_PARAMETERS", + 85899346162u64 => "UNABLE_TO_LOAD_SSL3_MD5_ROUTINES", + 85899346163u64 => "UNABLE_TO_LOAD_SSL3_SHA1_ROUTINES", + 85899346182u64 => "UNEXPECTED_CCS_MESSAGE", + 85899346098u64 => "UNEXPECTED_END_OF_EARLY_DATA", + 85899346214u64 => "UNEXPECTED_EOF_WHILE_READING", + 85899346164u64 => "UNEXPECTED_MESSAGE", + 85899346165u64 => "UNEXPECTED_RECORD", + 85899346196u64 => "UNINITIALIZED", + 85899346166u64 => "UNKNOWN_ALERT_TYPE", + 85899346167u64 => "UNKNOWN_CERTIFICATE_TYPE", + 85899346168u64 => "UNKNOWN_CIPHER_RETURNED", + 85899346169u64 => "UNKNOWN_CIPHER_TYPE", + 85899346306u64 => "UNKNOWN_CMD_NAME", + 85899346059u64 => "UNKNOWN_COMMAND", + 85899346288u64 => "UNKNOWN_DIGEST", + 85899346170u64 => "UNKNOWN_KEY_EXCHANGE_TYPE", + 85899346171u64 => "UNKNOWN_PKEY_TYPE", + 85899346172u64 => "UNKNOWN_PROTOCOL", + 85899346174u64 => "UNKNOWN_SSL_VERSION", + 85899346175u64 => "UNKNOWN_STATE", + 85899346258u64 => "UNSAFE_LEGACY_RENEGOTIATION_DISABLED", + 85899346137u64 => "UNSOLICITED_EXTENSION", + 85899346177u64 => "UNSUPPORTED_COMPRESSION_ALGORITHM", + 85899346235u64 => "UNSUPPORTED_ELLIPTIC_CURVE", + 85899346178u64 => "UNSUPPORTED_PROTOCOL", + 85899346179u64 => "UNSUPPORTED_SSL_VERSION", + 85899346249u64 => "UNSUPPORTED_STATUS_TYPE", + 85899346289u64 => "USE_SRTP_NOT_NEGOTIATED", + 85899346086u64 => "VERSION_TOO_HIGH", + 85899346316u64 => "VERSION_TOO_LOW", + 85899346303u64 => "WRONG_CERTIFICATE_TYPE", + 85899346181u64 => "WRONG_CIPHER_RETURNED", + 85899346298u64 => "WRONG_CURVE", + 85899346184u64 => "WRONG_SIGNATURE_LENGTH", + 85899346185u64 => "WRONG_SIGNATURE_SIZE", + 85899346290u64 => "WRONG_SIGNATURE_TYPE", + 85899346186u64 => "WRONG_SSL_VERSION", + 85899346187u64 => "WRONG_VERSION_NUMBER", + 85899346188u64 => "X509_LIB", + 85899346189u64 => "X509_VERIFICATION_SETUP_PROBLEMS", + 201863463044u64 => "BAD_PKCS7_TYPE", + 201863463045u64 => "BAD_TYPE", + 201863463049u64 => "CANNOT_LOAD_CERT", + 201863463050u64 => "CANNOT_LOAD_KEY", + 201863463012u64 => "CERTIFICATE_VERIFY_ERROR", + 201863463039u64 => "COULD_NOT_SET_ENGINE", + 201863463027u64 => "COULD_NOT_SET_TIME", + 201863463046u64 => "DETACHED_CONTENT", + 201863463028u64 => "ESS_ADD_SIGNING_CERT_ERROR", + 201863463051u64 => "ESS_ADD_SIGNING_CERT_V2_ERROR", + 201863463013u64 => "ESS_SIGNING_CERTIFICATE_ERROR", + 201863463014u64 => "INVALID_NULL_POINTER", + 201863463029u64 => "INVALID_SIGNER_CERTIFICATE_PURPOSE", + 201863463015u64 => "MESSAGE_IMPRINT_MISMATCH", + 201863463016u64 => "NONCE_MISMATCH", + 201863463017u64 => "NONCE_NOT_RETURNED", + 201863463018u64 => "NO_CONTENT", + 201863463019u64 => "NO_TIME_STAMP_TOKEN", + 201863463030u64 => "PKCS7_ADD_SIGNATURE_ERROR", + 201863463031u64 => "PKCS7_ADD_SIGNED_ATTR_ERROR", + 201863463041u64 => "PKCS7_TO_TS_TST_INFO_FAILED", + 201863463020u64 => "POLICY_MISMATCH", + 201863463032u64 => "PRIVATE_KEY_DOES_NOT_MATCH_CERTIFICATE", + 201863463033u64 => "RESPONSE_SETUP_ERROR", + 201863463021u64 => "SIGNATURE_FAILURE", + 201863463022u64 => "THERE_MUST_BE_ONE_SIGNER", + 201863463034u64 => "TIME_SYSCALL_ERROR", + 201863463042u64 => "TOKEN_NOT_PRESENT", + 201863463043u64 => "TOKEN_PRESENT", + 201863463023u64 => "TSA_NAME_MISMATCH", + 201863463024u64 => "TSA_UNTRUSTED", + 201863463035u64 => "TST_INFO_SETUP_ERROR", + 201863463036u64 => "TS_DATASIGN", + 201863463037u64 => "UNACCEPTABLE_POLICY", + 201863463038u64 => "UNSUPPORTED_MD_ALGORITHM", + 201863463025u64 => "UNSUPPORTED_VERSION", + 201863463047u64 => "VAR_BAD_VALUE", + 201863463048u64 => "VAR_LOOKUP_FAILURE", + 201863463026u64 => "WRONG_CONTENT_TYPE", + 171798691944u64 => "COMMON_OK_AND_CANCEL_CHARACTERS", + 171798691942u64 => "INDEX_TOO_LARGE", + 171798691943u64 => "INDEX_TOO_SMALL", + 171798691945u64 => "NO_RESULT_BUFFER", + 171798691947u64 => "PROCESSING_ERROR", + 171798691940u64 => "RESULT_TOO_LARGE", + 171798691941u64 => "RESULT_TOO_SMALL", + 171798691949u64 => "SYSASSIGN_ERROR", + 171798691950u64 => "SYSDASSGN_ERROR", + 171798691951u64 => "SYSQIOW_ERROR", + 171798691946u64 => "UNKNOWN_CONTROL_COMMAND", + 171798691948u64 => "UNKNOWN_TTYGET_ERRNO_VALUE", + 171798691952u64 => "USER_DATA_DUPLICATION_UNSUPPORTED", + 146028888182u64 => "BAD_IP_ADDRESS", + 146028888183u64 => "BAD_OBJECT", + 146028888164u64 => "BN_DEC2BN_ERROR", + 146028888165u64 => "BN_TO_ASN1_INTEGER_ERROR", + 146028888213u64 => "DIRNAME_ERROR", + 146028888224u64 => "DISTPOINT_ALREADY_SET", + 146028888197u64 => "DUPLICATE_ZONE_ID", + 146028888233u64 => "EMPTY_KEY_USAGE", + 146028888195u64 => "ERROR_CONVERTING_ZONE", + 146028888208u64 => "ERROR_CREATING_EXTENSION", + 146028888192u64 => "ERROR_IN_EXTENSION", + 146028888201u64 => "EXPECTED_A_SECTION_NAME", + 146028888209u64 => "EXTENSION_EXISTS", + 146028888179u64 => "EXTENSION_NAME_ERROR", + 146028888166u64 => "EXTENSION_NOT_FOUND", + 146028888167u64 => "EXTENSION_SETTING_NOT_SUPPORTED", + 146028888180u64 => "EXTENSION_VALUE_ERROR", + 146028888215u64 => "ILLEGAL_EMPTY_EXTENSION", + 146028888216u64 => "INCORRECT_POLICY_SYNTAX_TAG", + 146028888226u64 => "INVALID_ASNUMBER", + 146028888227u64 => "INVALID_ASRANGE", + 146028888168u64 => "INVALID_BOOLEAN_STRING", + 146028888222u64 => "INVALID_CERTIFICATE", + 146028888172u64 => "INVALID_EMPTY_NAME", + 146028888169u64 => "INVALID_EXTENSION_STRING", + 146028888229u64 => "INVALID_INHERITANCE", + 146028888230u64 => "INVALID_IPADDRESS", + 146028888225u64 => "INVALID_MULTIPLE_RDNS", + 146028888170u64 => "INVALID_NAME", + 146028888171u64 => "INVALID_NULL_ARGUMENT", + 146028888173u64 => "INVALID_NULL_VALUE", + 146028888204u64 => "INVALID_NUMBER", + 146028888205u64 => "INVALID_NUMBERS", + 146028888174u64 => "INVALID_OBJECT_IDENTIFIER", + 146028888202u64 => "INVALID_OPTION", + 146028888198u64 => "INVALID_POLICY_IDENTIFIER", + 146028888217u64 => "INVALID_PROXY_POLICY_SETTING", + 146028888210u64 => "INVALID_PURPOSE", + 146028888228u64 => "INVALID_SAFI", + 146028888199u64 => "INVALID_SECTION", + 146028888207u64 => "INVALID_SYNTAX", + 146028888190u64 => "ISSUER_DECODE_ERROR", + 146028888188u64 => "MISSING_VALUE", + 146028888206u64 => "NEED_ORGANIZATION_AND_NUMBERS", + 146028888232u64 => "NEGATIVE_PATHLEN", + 146028888200u64 => "NO_CONFIG_DATABASE", + 146028888185u64 => "NO_ISSUER_CERTIFICATE", + 146028888191u64 => "NO_ISSUER_DETAILS", + 146028888203u64 => "NO_POLICY_IDENTIFIER", + 146028888218u64 => "NO_PROXY_CERT_POLICY_LANGUAGE_DEFINED", + 146028888178u64 => "NO_PUBLIC_KEY", + 146028888189u64 => "NO_SUBJECT_DETAILS", + 146028888212u64 => "OPERATION_NOT_DEFINED", + 146028888211u64 => "OTHERNAME_ERROR", + 146028888219u64 => "POLICY_LANGUAGE_ALREADY_DEFINED", + 146028888220u64 => "POLICY_PATH_LENGTH", + 146028888221u64 => "POLICY_PATH_LENGTH_ALREADY_DEFINED", + 146028888223u64 => "POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", + 146028888214u64 => "SECTION_NOT_FOUND", + 146028888186u64 => "UNABLE_TO_GET_ISSUER_DETAILS", + 146028888187u64 => "UNABLE_TO_GET_ISSUER_KEYID", + 146028888175u64 => "UNKNOWN_BIT_STRING_ARGUMENT", + 146028888193u64 => "UNKNOWN_EXTENSION", + 146028888194u64 => "UNKNOWN_EXTENSION_NAME", + 146028888184u64 => "UNKNOWN_OPTION", + 146028888181u64 => "UNSUPPORTED_OPTION", + 146028888231u64 => "UNSUPPORTED_TYPE", + 146028888196u64 => "USER_TOO_LONG", + 47244640366u64 => "AKID_MISMATCH", + 47244640389u64 => "BAD_SELECTOR", + 47244640356u64 => "BAD_X509_FILETYPE", + 47244640374u64 => "BASE64_DECODE_ERROR", + 47244640370u64 => "CANT_CHECK_DH_KEY", + 47244640395u64 => "CERTIFICATE_VERIFICATION_FAILED", + 47244640357u64 => "CERT_ALREADY_IN_HASH_TABLE", + 47244640383u64 => "CRL_ALREADY_DELTA", + 47244640387u64 => "CRL_VERIFY_FAILURE", + 47244640396u64 => "DUPLICATE_ATTRIBUTE", + 47244640397u64 => "ERROR_GETTING_MD_BY_NID", + 47244640398u64 => "ERROR_USING_SIGINF_SET", + 47244640384u64 => "IDP_MISMATCH", + 47244640394u64 => "INVALID_ATTRIBUTES", + 47244640369u64 => "INVALID_DIRECTORY", + 47244640399u64 => "INVALID_DISTPOINT", + 47244640375u64 => "INVALID_FIELD_NAME", + 47244640379u64 => "INVALID_TRUST", + 47244640385u64 => "ISSUER_MISMATCH", + 47244640371u64 => "KEY_TYPE_MISMATCH", + 47244640372u64 => "KEY_VALUES_MISMATCH", + 47244640359u64 => "LOADING_CERT_DIR", + 47244640360u64 => "LOADING_DEFAULTS", + 47244640380u64 => "METHOD_NOT_SUPPORTED", + 47244640390u64 => "NAME_TOO_LONG", + 47244640388u64 => "NEWER_CRL_NOT_NEWER", + 47244640391u64 => "NO_CERTIFICATE_FOUND", + 47244640392u64 => "NO_CERTIFICATE_OR_CRL_FOUND", + 47244640361u64 => "NO_CERT_SET_FOR_US_TO_VERIFY", + 47244640393u64 => "NO_CRL_FOUND", + 47244640386u64 => "NO_CRL_NUMBER", + 47244640381u64 => "PUBLIC_KEY_DECODE_ERROR", + 47244640382u64 => "PUBLIC_KEY_ENCODE_ERROR", + 47244640362u64 => "SHOULD_RETRY", + 47244640363u64 => "UNABLE_TO_FIND_PARAMETERS_IN_CHAIN", + 47244640364u64 => "UNABLE_TO_GET_CERTS_PUBLIC_KEY", + 47244640373u64 => "UNKNOWN_KEY_TYPE", + 47244640365u64 => "UNKNOWN_NID", + 47244640377u64 => "UNKNOWN_PURPOSE_ID", + 47244640400u64 => "UNKNOWN_SIGID_ALGS", + 47244640376u64 => "UNKNOWN_TRUST_ID", + 47244640367u64 => "UNSUPPORTED_ALGORITHM", + 47244640368u64 => "WRONG_LOOKUP_TYPE", + 47244640378u64 => "WRONG_TYPE", +}; + +/// Helper function to create encoded key from (lib, reason) pair +#[inline] +pub fn encode_error_key(lib: i32, reason: i32) -> u64 { + ((lib as u64) << 32) | (reason as u64 & 0xFFFFFFFF) +} diff --git a/crates/stdlib/src/overlapped.rs b/crates/stdlib/src/overlapped.rs new file mode 100644 index 00000000000..76d18cb7a9a --- /dev/null +++ b/crates/stdlib/src/overlapped.rs @@ -0,0 +1,1988 @@ +// spell-checker:disable + +pub(crate) use _overlapped::module_def; + +#[allow(non_snake_case)] +#[pymodule] +mod _overlapped { + // straight-forward port of Modules/overlapped.c + + use crate::vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef, PyModule, PyStrRef, PyTupleRef, PyType}, + common::lock::PyMutex, + convert::ToPyObject, + function::OptionalArg, + object::{Traverse, TraverseFn}, + protocol::PyBuffer, + types::{Constructor, Destructor}, + }; + use windows_sys::Win32::{ + Foundation::{self, GetLastError, HANDLE}, + Networking::WinSock::{AF_INET, AF_INET6, SOCKADDR, SOCKADDR_IN, SOCKADDR_IN6}, + System::IO::OVERLAPPED, + }; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + let _ = vm.import("_socket", 0)?; + initialize_winsock_extensions(vm)?; + __module_exec(vm, module); + Ok(()) + } + + #[pyattr] + use windows_sys::Win32::{ + Foundation::{ + ERROR_IO_PENDING, ERROR_NETNAME_DELETED, ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, + ERROR_PORT_UNREACHABLE, ERROR_SEM_TIMEOUT, + }, + Networking::WinSock::{ + SO_UPDATE_ACCEPT_CONTEXT, SO_UPDATE_CONNECT_CONTEXT, TF_REUSE_SOCKET, + }, + System::Threading::INFINITE, + }; + + #[pyattr] + const INVALID_HANDLE_VALUE: isize = + unsafe { core::mem::transmute(windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE) }; + + #[pyattr] + const NULL: isize = 0; + + // Function pointers for Winsock extension functions + static ACCEPT_EX: std::sync::OnceLock<usize> = std::sync::OnceLock::new(); + static CONNECT_EX: std::sync::OnceLock<usize> = std::sync::OnceLock::new(); + static DISCONNECT_EX: std::sync::OnceLock<usize> = std::sync::OnceLock::new(); + static TRANSMIT_FILE: std::sync::OnceLock<usize> = std::sync::OnceLock::new(); + + fn initialize_winsock_extensions(vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Networking::WinSock::{ + INVALID_SOCKET, IPPROTO_TCP, SIO_GET_EXTENSION_FUNCTION_POINTER, SOCK_STREAM, + SOCKET_ERROR, WSAGetLastError, WSAIoctl, closesocket, socket, + }; + + // GUIDs for extension functions + const WSAID_ACCEPTEX: windows_sys::core::GUID = windows_sys::core::GUID { + data1: 0xb5367df1, + data2: 0xcbac, + data3: 0x11cf, + data4: [0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92], + }; + const WSAID_CONNECTEX: windows_sys::core::GUID = windows_sys::core::GUID { + data1: 0x25a207b9, + data2: 0xddf3, + data3: 0x4660, + data4: [0x8e, 0xe9, 0x76, 0xe5, 0x8c, 0x74, 0x06, 0x3e], + }; + const WSAID_DISCONNECTEX: windows_sys::core::GUID = windows_sys::core::GUID { + data1: 0x7fda2e11, + data2: 0x8630, + data3: 0x436f, + data4: [0xa0, 0x31, 0xf5, 0x36, 0xa6, 0xee, 0xc1, 0x57], + }; + const WSAID_TRANSMITFILE: windows_sys::core::GUID = windows_sys::core::GUID { + data1: 0xb5367df0, + data2: 0xcbac, + data3: 0x11cf, + data4: [0x95, 0xca, 0x00, 0x80, 0x5f, 0x48, 0xa1, 0x92], + }; + + // Check all four locks to prevent partial initialization + if ACCEPT_EX.get().is_some() + && CONNECT_EX.get().is_some() + && DISCONNECT_EX.get().is_some() + && TRANSMIT_FILE.get().is_some() + { + return Ok(()); + } + + let s = unsafe { socket(AF_INET as i32, SOCK_STREAM, IPPROTO_TCP) }; + if s == INVALID_SOCKET { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + + let mut dw_bytes: u32 = 0; + + macro_rules! get_extension { + ($guid:expr, $lock:expr) => {{ + let mut func_ptr: usize = 0; + let ret = unsafe { + WSAIoctl( + s, + SIO_GET_EXTENSION_FUNCTION_POINTER, + &$guid as *const _ as *const _, + core::mem::size_of_val(&$guid) as u32, + &mut func_ptr as *mut _ as *mut _, + core::mem::size_of::<usize>() as u32, + &mut dw_bytes, + core::ptr::null_mut(), + None, + ) + }; + if ret == SOCKET_ERROR { + let err = unsafe { WSAGetLastError() } as u32; + unsafe { closesocket(s) }; + return Err(set_from_windows_err(err, vm)); + } + let _ = $lock.set(func_ptr); + }}; + } + + get_extension!(WSAID_ACCEPTEX, ACCEPT_EX); + get_extension!(WSAID_CONNECTEX, CONNECT_EX); + get_extension!(WSAID_DISCONNECTEX, DISCONNECT_EX); + get_extension!(WSAID_TRANSMITFILE, TRANSMIT_FILE); + + unsafe { closesocket(s) }; + Ok(()) + } + + #[pyattr] + #[pyclass(name, traverse)] + #[derive(PyPayload)] + struct Overlapped { + inner: PyMutex<OverlappedInner>, + } + + struct OverlappedInner { + overlapped: OVERLAPPED, + handle: HANDLE, + error: u32, + data: OverlappedData, + } + + unsafe impl Sync for OverlappedInner {} + unsafe impl Send for OverlappedInner {} + + unsafe impl Traverse for OverlappedInner { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + match &self.data { + OverlappedData::Read(buf) | OverlappedData::Accept(buf) => { + buf.traverse(traverse_fn); + } + OverlappedData::ReadInto(buf) | OverlappedData::Write(buf) => { + buf.traverse(traverse_fn); + } + OverlappedData::WriteTo(wt) => { + wt.buf.traverse(traverse_fn); + } + OverlappedData::ReadFrom(rf) => { + if let Some(result) = &rf.result { + result.traverse(traverse_fn); + } + rf.allocated_buffer.traverse(traverse_fn); + } + OverlappedData::ReadFromInto(rfi) => { + if let Some(result) = &rfi.result { + result.traverse(traverse_fn); + } + rfi.user_buffer.traverse(traverse_fn); + } + _ => {} + } + } + } + + impl core::fmt::Debug for Overlapped { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let zelf = self.inner.lock(); + f.debug_struct("Overlapped") + .field("handle", &zelf.handle) + .field("error", &zelf.error) + .field("data", &zelf.data) + .finish() + } + } + + #[derive(Debug)] + enum OverlappedData { + None, + NotStarted, + Read(PyBytesRef), + // Fields below store buffers that must be kept alive during async operations + #[allow(dead_code)] + ReadInto(PyBuffer), + #[allow(dead_code)] + Write(PyBuffer), + #[allow(dead_code)] + Accept(PyBytesRef), + Connect(Vec<u8>), // Store address bytes to keep them alive during async operation + Disconnect, + ConnectNamedPipe, + #[allow(dead_code)] // Reserved for named pipe support + WaitNamedPipeAndConnect, + TransmitFile, + ReadFrom(OverlappedReadFrom), + WriteTo(OverlappedWriteTo), // Store address bytes for WSASendTo + ReadFromInto(OverlappedReadFromInto), + } + + struct OverlappedReadFrom { + // A (buffer, (host, port)) tuple + result: Option<PyObjectRef>, + // The actual read buffer + allocated_buffer: PyBytesRef, + address: SOCKADDR_IN6, + address_length: i32, + } + + impl core::fmt::Debug for OverlappedReadFrom { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("OverlappedReadFrom") + .field("result", &self.result) + .field("allocated_buffer", &self.allocated_buffer) + .field("address_length", &self.address_length) + .finish() + } + } + + struct OverlappedReadFromInto { + // A (number of bytes read, (host, port)) tuple + result: Option<PyObjectRef>, + /* Buffer passed by the user */ + user_buffer: PyBuffer, + address: SOCKADDR_IN6, + address_length: i32, + } + + impl core::fmt::Debug for OverlappedReadFromInto { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("OverlappedReadFromInto") + .field("result", &self.result) + .field("user_buffer", &self.user_buffer) + .field("address_length", &self.address_length) + .finish() + } + } + + struct OverlappedWriteTo { + buf: PyBuffer, + address: Vec<u8>, // Keep address alive during async operation + } + + impl core::fmt::Debug for OverlappedWriteTo { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("OverlappedWriteTo") + .field("buf", &self.buf) + .field("address", &self.address.len()) + .finish() + } + } + + fn mark_as_completed(ov: &mut OVERLAPPED) { + ov.Internal = 0; + if !ov.hEvent.is_null() { + unsafe { windows_sys::Win32::System::Threading::SetEvent(ov.hEvent) }; + } + } + + fn set_from_windows_err(err: u32, vm: &VirtualMachine) -> PyBaseExceptionRef { + let err = if err == 0 { + unsafe { GetLastError() } + } else { + err + }; + let errno = crate::vm::common::os::winerror_to_errno(err as i32); + let message = std::io::Error::from_raw_os_error(err as i32).to_string(); + let exc = vm.new_errno_error(errno, message); + let _ = exc + .as_object() + .set_attr("winerror", err.to_pyobject(vm), vm); + exc.upcast() + } + + fn HasOverlappedIoCompleted(overlapped: &OVERLAPPED) -> bool { + overlapped.Internal != (Foundation::STATUS_PENDING as usize) + } + + /// Parse a Python address tuple to SOCKADDR + fn parse_address(addr_obj: &PyTupleRef, vm: &VirtualMachine) -> PyResult<(Vec<u8>, i32)> { + use windows_sys::Win32::Networking::WinSock::{WSAGetLastError, WSAStringToAddressW}; + + match addr_obj.len() { + 2 => { + // IPv4: (host, port) + let host: PyStrRef = addr_obj[0].clone().try_into_value(vm)?; + let port: u16 = addr_obj[1].clone().try_to_value(vm)?; + + let mut addr: SOCKADDR_IN = unsafe { core::mem::zeroed() }; + addr.sin_family = AF_INET; + + let host_wide: Vec<u16> = host.as_wtf8().encode_wide().chain([0]).collect(); + let mut addr_len = core::mem::size_of::<SOCKADDR_IN>() as i32; + + let ret = unsafe { + WSAStringToAddressW( + host_wide.as_ptr(), + AF_INET as i32, + core::ptr::null(), + &mut addr as *mut _ as *mut SOCKADDR, + &mut addr_len, + ) + }; + + if ret < 0 { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + + // Restore port (WSAStringToAddressW overwrites it) + addr.sin_port = port.to_be(); + + let bytes = unsafe { + core::slice::from_raw_parts( + &addr as *const _ as *const u8, + core::mem::size_of::<SOCKADDR_IN>(), + ) + }; + Ok((bytes.to_vec(), addr_len)) + } + 4 => { + // IPv6: (host, port, flowinfo, scope_id) + let host: PyStrRef = addr_obj[0].clone().try_into_value(vm)?; + let port: u16 = addr_obj[1].clone().try_to_value(vm)?; + let flowinfo: u32 = addr_obj[2].clone().try_to_value(vm)?; + let scope_id: u32 = addr_obj[3].clone().try_to_value(vm)?; + + let mut addr: SOCKADDR_IN6 = unsafe { core::mem::zeroed() }; + addr.sin6_family = AF_INET6; + + let host_wide: Vec<u16> = host.as_wtf8().encode_wide().chain([0]).collect(); + let mut addr_len = core::mem::size_of::<SOCKADDR_IN6>() as i32; + + let ret = unsafe { + WSAStringToAddressW( + host_wide.as_ptr(), + AF_INET6 as i32, + core::ptr::null(), + &mut addr as *mut _ as *mut SOCKADDR, + &mut addr_len, + ) + }; + + if ret < 0 { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + + // Restore fields that WSAStringToAddressW might overwrite + addr.sin6_port = port.to_be(); + addr.sin6_flowinfo = flowinfo; + addr.Anonymous.sin6_scope_id = scope_id; + + let bytes = unsafe { + core::slice::from_raw_parts( + &addr as *const _ as *const u8, + core::mem::size_of::<SOCKADDR_IN6>(), + ) + }; + Ok((bytes.to_vec(), addr_len)) + } + _ => Err(vm.new_value_error("illegal address_as_bytes argument")), + } + } + + /// Parse a SOCKADDR_IN6 (which can also hold IPv4 addresses) to a Python address tuple + fn unparse_address(addr: &SOCKADDR_IN6, _addr_len: i32, vm: &VirtualMachine) -> PyResult { + use core::net::{Ipv4Addr, Ipv6Addr}; + + unsafe { + let family = addr.sin6_family; + if family == AF_INET { + // IPv4 address stored in SOCKADDR_IN6 structure + let addr_in = &*(addr as *const SOCKADDR_IN6 as *const SOCKADDR_IN); + let ip_bytes = addr_in.sin_addr.S_un.S_un_b; + let ip_str = + Ipv4Addr::new(ip_bytes.s_b1, ip_bytes.s_b2, ip_bytes.s_b3, ip_bytes.s_b4) + .to_string(); + let port = u16::from_be(addr_in.sin_port); + Ok((ip_str, port).to_pyobject(vm)) + } else if family == AF_INET6 { + // IPv6 address + let ip_bytes = addr.sin6_addr.u.Byte; + let ip_str = Ipv6Addr::from(ip_bytes).to_string(); + let port = u16::from_be(addr.sin6_port); + let flowinfo = u32::from_be(addr.sin6_flowinfo); + let scope_id = addr.Anonymous.sin6_scope_id; + Ok((ip_str, port, flowinfo, scope_id).to_pyobject(vm)) + } else { + Err(vm.new_value_error("recvfrom returned unsupported address family")) + } + } + } + + #[pyclass(with(Constructor, Destructor))] + impl Overlapped { + #[pygetset] + fn address(&self, _vm: &VirtualMachine) -> usize { + let inner = self.inner.lock(); + &inner.overlapped as *const _ as usize + } + + #[pygetset] + fn pending(&self, _vm: &VirtualMachine) -> bool { + let inner = self.inner.lock(); + !HasOverlappedIoCompleted(&inner.overlapped) + && !matches!(inner.data, OverlappedData::NotStarted) + } + + #[pygetset] + fn error(&self, _vm: &VirtualMachine) -> u32 { + let inner = self.inner.lock(); + inner.error + } + + #[pygetset] + fn event(&self, _vm: &VirtualMachine) -> isize { + let inner = self.inner.lock(); + inner.overlapped.hEvent as isize + } + + #[pymethod] + fn cancel(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + let inner = zelf.inner.lock(); + if matches!( + inner.data, + OverlappedData::NotStarted | OverlappedData::WaitNamedPipeAndConnect + ) { + return Ok(()); + } + let ret = if !HasOverlappedIoCompleted(&inner.overlapped) { + unsafe { + windows_sys::Win32::System::IO::CancelIoEx(inner.handle, &inner.overlapped) + } + } else { + 1 + }; + // CancelIoEx returns ERROR_NOT_FOUND if the I/O completed in-between + if ret == 0 && unsafe { GetLastError() } != Foundation::ERROR_NOT_FOUND { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + #[pymethod] + fn getresult(zelf: &Py<Self>, wait: OptionalArg<bool>, vm: &VirtualMachine) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + + let mut inner = zelf.inner.lock(); + let wait = wait.unwrap_or(false); + + // Check operation state + if matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation not yet attempted")); + } + if matches!(inner.data, OverlappedData::NotStarted) { + return Err(vm.new_value_error("operation failed to start")); + } + + // Get the result + let mut transferred: u32 = 0; + let ret = unsafe { + windows_sys::Win32::System::IO::GetOverlappedResult( + inner.handle, + &inner.overlapped, + &mut transferred, + if wait { 1 } else { 0 }, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + // Handle errors + match err { + ERROR_SUCCESS | ERROR_MORE_DATA => {} + ERROR_BROKEN_PIPE => { + let allow_broken_pipe = match &inner.data { + OverlappedData::Read(_) | OverlappedData::ReadInto(_) => true, + OverlappedData::ReadFrom(_) => true, + OverlappedData::ReadFromInto(rfi) => rfi.result.is_some(), + _ => false, + }; + if !allow_broken_pipe { + return Err(set_from_windows_err(err, vm)); + } + } + _ => return Err(set_from_windows_err(err, vm)), + } + + // Return result based on operation type + match &mut inner.data { + OverlappedData::Read(buf) => { + let len = buf.as_bytes().len(); + let result = if transferred as usize != len { + let resized = vm + .ctx + .new_bytes(buf.as_bytes()[..transferred as usize].to_vec()); + *buf = resized.clone(); + resized + } else { + buf.clone() + }; + Ok(result.into()) + } + OverlappedData::ReadFrom(rf) => { + let len = rf.allocated_buffer.as_bytes().len(); + let resized_buf = if transferred as usize != len { + let resized = vm.ctx.new_bytes( + rf.allocated_buffer.as_bytes()[..transferred as usize].to_vec(), + ); + rf.allocated_buffer = resized.clone(); + resized + } else { + rf.allocated_buffer.clone() + }; + let addr_tuple = unparse_address(&rf.address, rf.address_length, vm)?; + if let Some(result) = &rf.result { + return Ok(result.clone()); + } + let result = vm.ctx.new_tuple(vec![resized_buf.into(), addr_tuple]); + rf.result = Some(result.clone().into()); + Ok(result.into()) + } + OverlappedData::ReadFromInto(rfi) => { + let addr_tuple = unparse_address(&rfi.address, rfi.address_length, vm)?; + if let Some(result) = &rfi.result { + return Ok(result.clone()); + } + let result = vm + .ctx + .new_tuple(vec![vm.ctx.new_int(transferred).into(), addr_tuple]); + rfi.result = Some(result.clone().into()); + Ok(result.into()) + } + _ => Ok(vm.ctx.new_int(transferred).into()), + } + } + + // ReadFile + #[pymethod] + fn ReadFile(zelf: &Py<Self>, handle: isize, size: u32, vm: &VirtualMachine) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Storage::FileSystem::ReadFile; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + #[cfg(target_pointer_width = "32")] + let size = core::cmp::min(size, isize::MAX as u32); + + let buf = vec![0u8; core::cmp::max(size, 1) as usize]; + let buf = vm.ctx.new_bytes(buf); + inner.handle = handle as HANDLE; + inner.data = OverlappedData::Read(buf.clone()); + + let mut nread: u32 = 0; + let ret = unsafe { + ReadFile( + handle as HANDLE, + buf.as_bytes().as_ptr() as *mut _, + size, + &mut nread, + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // ReadFileInto + #[pymethod] + fn ReadFileInto( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Storage::FileSystem::ReadFile; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large")); + } + + // For async read, buffer must be contiguous - we can't use a temporary copy + // because Windows writes data directly to the buffer after this call returns + let Some(contiguous) = buf.as_contiguous_mut() else { + return Err(vm.new_buffer_error("buffer is not contiguous")); + }; + + inner.data = OverlappedData::ReadInto(buf.clone()); + + let mut nread: u32 = 0; + let ret = unsafe { + ReadFile( + handle as HANDLE, + contiguous.as_ptr() as *mut _, + buf_len as u32, + &mut nread, + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSARecv + #[pymethod] + fn WSARecv( + zelf: &Py<Self>, + handle: isize, + size: u32, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSARecv}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + let mut flags = flags.unwrap_or(0); + + #[cfg(target_pointer_width = "32")] + let size = core::cmp::min(size, isize::MAX as u32); + + let buf = vec![0u8; core::cmp::max(size, 1) as usize]; + let buf = vm.ctx.new_bytes(buf); + inner.handle = handle as HANDLE; + inner.data = OverlappedData::Read(buf.clone()); + + let wsabuf = WSABUF { + buf: buf.as_bytes().as_ptr() as *mut _, + len: size, + }; + let mut nread: u32 = 0; + + let ret = unsafe { + WSARecv( + handle as _, + &wsabuf, + 1, + &mut nread, + &mut flags, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSARecvInto + #[pymethod] + fn WSARecvInto( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSARecv}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + let mut flags = flags; + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large")); + } + + let Some(contiguous) = buf.as_contiguous_mut() else { + return Err(vm.new_buffer_error("buffer is not contiguous")); + }; + + inner.data = OverlappedData::ReadInto(buf.clone()); + + let wsabuf = WSABUF { + buf: contiguous.as_ptr() as *mut _, + len: buf_len as u32, + }; + let mut nread: u32 = 0; + + let ret = unsafe { + WSARecv( + handle as _, + &wsabuf, + 1, + &mut nread, + &mut flags, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WriteFile + #[pymethod] + fn WriteFile( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Storage::FileSystem::WriteFile; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large")); + } + + // For async write, buffer must be contiguous - we can't use a temporary copy + // because Windows reads from the buffer after this call returns + let Some(contiguous) = buf.as_contiguous() else { + return Err(vm.new_buffer_error("buffer is not contiguous")); + }; + + inner.data = OverlappedData::Write(buf.clone()); + + let mut written: u32 = 0; + let ret = unsafe { + WriteFile( + handle as HANDLE, + contiguous.as_ptr() as *const _, + buf_len as u32, + &mut written, + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSASend + #[pymethod] + fn WSASend( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSASend}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large")); + } + + let Some(contiguous) = buf.as_contiguous() else { + return Err(vm.new_buffer_error("buffer is not contiguous")); + }; + + inner.data = OverlappedData::Write(buf.clone()); + + let wsabuf = WSABUF { + buf: contiguous.as_ptr() as *mut _, + len: buf_len as u32, + }; + let mut written: u32 = 0; + + let ret = unsafe { + WSASend( + handle as _, + &wsabuf, + 1, + &mut written, + flags, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // AcceptEx + #[pymethod] + fn AcceptEx( + zelf: &Py<Self>, + listen_socket: isize, + accept_socket: isize, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::WSAGetLastError; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + // Buffer size: local address + remote address + let size = core::mem::size_of::<SOCKADDR_IN6>() + 16; + let buf = vec![0u8; size * 2]; + let buf = vm.ctx.new_bytes(buf); + + inner.handle = listen_socket as HANDLE; + inner.data = OverlappedData::Accept(buf.clone()); + + let mut bytes_received: u32 = 0; + + type AcceptExFn = unsafe extern "system" fn( + sListenSocket: usize, + sAcceptSocket: usize, + lpOutputBuffer: *mut core::ffi::c_void, + dwReceiveDataLength: u32, + dwLocalAddressLength: u32, + dwRemoteAddressLength: u32, + lpdwBytesReceived: *mut u32, + lpOverlapped: *mut OVERLAPPED, + ) -> i32; + + let accept_ex: AcceptExFn = unsafe { core::mem::transmute(*ACCEPT_EX.get().unwrap()) }; + + let ret = unsafe { + accept_ex( + listen_socket as _, + accept_socket as _, + buf.as_bytes().as_ptr() as *mut _, + 0, + size as u32, + size as u32, + &mut bytes_received, + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { WSAGetLastError() as u32 } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // ConnectEx + #[pymethod] + fn ConnectEx( + zelf: &Py<Self>, + socket: isize, + address: PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::WSAGetLastError; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + let (addr_bytes, addr_len) = parse_address(&address, vm)?; + + inner.handle = socket as HANDLE; + // Store addr_bytes in OverlappedData to keep it alive during async operation + inner.data = OverlappedData::Connect(addr_bytes); + + type ConnectExFn = unsafe extern "system" fn( + s: usize, + name: *const SOCKADDR, + namelen: i32, + lpSendBuffer: *const core::ffi::c_void, + dwSendDataLength: u32, + lpdwBytesSent: *mut u32, + lpOverlapped: *mut OVERLAPPED, + ) -> i32; + + let connect_ex: ConnectExFn = + unsafe { core::mem::transmute(*CONNECT_EX.get().unwrap()) }; + + // Get pointer to the stored address data + let addr_ptr = match &inner.data { + OverlappedData::Connect(bytes) => bytes.as_ptr(), + _ => unreachable!(), + }; + + let ret = unsafe { + connect_ex( + socket as _, + addr_ptr as *const SOCKADDR, + addr_len, + core::ptr::null(), + 0, + core::ptr::null_mut(), + &mut inner.overlapped, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { WSAGetLastError() as u32 } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // DisconnectEx + #[pymethod] + fn DisconnectEx( + zelf: &Py<Self>, + socket: isize, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::WSAGetLastError; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + inner.handle = socket as HANDLE; + inner.data = OverlappedData::Disconnect; + + type DisconnectExFn = unsafe extern "system" fn( + s: usize, + lpOverlapped: *mut OVERLAPPED, + dwFlags: u32, + dwReserved: u32, + ) -> i32; + + let disconnect_ex: DisconnectExFn = + unsafe { core::mem::transmute(*DISCONNECT_EX.get().unwrap()) }; + + let ret = unsafe { disconnect_ex(socket as _, &mut inner.overlapped, flags, 0) }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { WSAGetLastError() as u32 } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // TransmitFile + #[allow( + clippy::too_many_arguments, + reason = "mirrors Windows TransmitFile argument structure" + )] + #[pymethod] + fn TransmitFile( + zelf: &Py<Self>, + socket: isize, + file: isize, + offset: u32, + offset_high: u32, + count_to_write: u32, + count_per_send: u32, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::WSAGetLastError; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + inner.handle = socket as HANDLE; + inner.data = OverlappedData::TransmitFile; + inner.overlapped.Anonymous.Anonymous.Offset = offset; + inner.overlapped.Anonymous.Anonymous.OffsetHigh = offset_high; + + type TransmitFileFn = unsafe extern "system" fn( + hSocket: usize, + hFile: HANDLE, + nNumberOfBytesToWrite: u32, + nNumberOfBytesPerSend: u32, + lpOverlapped: *mut OVERLAPPED, + lpTransmitBuffers: *const core::ffi::c_void, + dwReserved: u32, + ) -> i32; + + let transmit_file: TransmitFileFn = + unsafe { core::mem::transmute(*TRANSMIT_FILE.get().unwrap()) }; + + let ret = unsafe { + transmit_file( + socket as _, + file as HANDLE, + count_to_write, + count_per_send, + &mut inner.overlapped, + core::ptr::null(), + flags, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { WSAGetLastError() as u32 } + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // ConnectNamedPipe + #[pymethod] + fn ConnectNamedPipe(zelf: &Py<Self>, pipe: isize, vm: &VirtualMachine) -> PyResult<bool> { + use windows_sys::Win32::Foundation::{ + ERROR_IO_PENDING, ERROR_PIPE_CONNECTED, ERROR_SUCCESS, + }; + use windows_sys::Win32::System::Pipes::ConnectNamedPipe; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + inner.handle = pipe as HANDLE; + inner.data = OverlappedData::ConnectNamedPipe; + + let ret = unsafe { ConnectNamedPipe(pipe as HANDLE, &mut inner.overlapped) }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + inner.error = err; + + match err { + ERROR_PIPE_CONNECTED => { + mark_as_completed(&mut inner.overlapped); + Ok(true) + } + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(false), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSASendTo + #[pymethod] + fn WSASendTo( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + flags: u32, + address: PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ERROR_IO_PENDING, ERROR_SUCCESS}; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSASendTo}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + let (addr_bytes, addr_len) = parse_address(&address, vm)?; + + inner.handle = handle as HANDLE; + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large")); + } + + let Some(contiguous) = buf.as_contiguous() else { + return Err(vm.new_buffer_error("buffer is not contiguous")); + }; + + // Store both buffer and address in OverlappedData to keep them alive + inner.data = OverlappedData::WriteTo(OverlappedWriteTo { + buf: buf.clone(), + address: addr_bytes, + }); + + let wsabuf = WSABUF { + buf: contiguous.as_ptr() as *mut _, + len: buf_len as u32, + }; + let mut written: u32 = 0; + + // Get pointer to the stored address data + let addr_ptr = match &inner.data { + OverlappedData::WriteTo(wt) => wt.address.as_ptr(), + _ => unreachable!(), + }; + + let ret = unsafe { + WSASendTo( + handle as _, + &wsabuf, + 1, + &mut written, + flags, + addr_ptr as *const SOCKADDR, + addr_len, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_SUCCESS | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSARecvFrom + #[pymethod] + fn WSARecvFrom( + zelf: &Py<Self>, + handle: isize, + size: u32, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSARecvFrom}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + let mut flags = flags.unwrap_or(0); + + #[cfg(target_pointer_width = "32")] + let size = core::cmp::min(size, isize::MAX as u32); + + let buf = vec![0u8; core::cmp::max(size, 1) as usize]; + let buf = vm.ctx.new_bytes(buf); + inner.handle = handle as HANDLE; + + let address: SOCKADDR_IN6 = unsafe { core::mem::zeroed() }; + let address_length = core::mem::size_of::<SOCKADDR_IN6>() as i32; + + inner.data = OverlappedData::ReadFrom(OverlappedReadFrom { + result: None, + allocated_buffer: buf.clone(), + address, + address_length, + }); + + let wsabuf = WSABUF { + buf: buf.as_bytes().as_ptr() as *mut _, + len: size, + }; + let mut nread: u32 = 0; + + // Get mutable reference to address in inner.data + let (addr_ptr, addr_len_ptr) = match &mut inner.data { + OverlappedData::ReadFrom(rf) => ( + &mut rf.address as *mut SOCKADDR_IN6, + &mut rf.address_length as *mut i32, + ), + _ => unreachable!(), + }; + + let ret = unsafe { + WSARecvFrom( + handle as _, + &wsabuf, + 1, + &mut nread, + &mut flags, + addr_ptr as *mut SOCKADDR, + addr_len_ptr, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + + // WSARecvFromInto + #[pymethod] + fn WSARecvFromInto( + zelf: &Py<Self>, + handle: isize, + buf: PyBuffer, + size: u32, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult { + use windows_sys::Win32::Foundation::{ + ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, + }; + use windows_sys::Win32::Networking::WinSock::{WSABUF, WSAGetLastError, WSARecvFrom}; + + let mut inner = zelf.inner.lock(); + if !matches!(inner.data, OverlappedData::None) { + return Err(vm.new_value_error("operation already attempted")); + } + + let mut flags = flags.unwrap_or(0); + inner.handle = handle as HANDLE; + + let Some(contiguous) = buf.as_contiguous_mut() else { + return Err(vm.new_buffer_error("buffer is not contiguous")); + }; + + let buf_len = buf.desc.len; + if buf_len > u32::MAX as usize { + return Err(vm.new_value_error("buffer too large")); + } + + let address: SOCKADDR_IN6 = unsafe { core::mem::zeroed() }; + let address_length = core::mem::size_of::<SOCKADDR_IN6>() as i32; + + inner.data = OverlappedData::ReadFromInto(OverlappedReadFromInto { + result: None, + user_buffer: buf.clone(), + address, + address_length, + }); + + let wsabuf = WSABUF { + buf: contiguous.as_ptr() as *mut _, + len: size, + }; + let mut nread: u32 = 0; + + // Get mutable reference to address in inner.data + let (addr_ptr, addr_len_ptr) = match &mut inner.data { + OverlappedData::ReadFromInto(rfi) => ( + &mut rfi.address as *mut SOCKADDR_IN6, + &mut rfi.address_length as *mut i32, + ), + _ => unreachable!(), + }; + + let ret = unsafe { + WSARecvFrom( + handle as _, + &wsabuf, + 1, + &mut nread, + &mut flags, + addr_ptr as *mut SOCKADDR, + addr_len_ptr, + &mut inner.overlapped, + None, + ) + }; + + let err = if ret < 0 { + unsafe { WSAGetLastError() as u32 } + } else { + ERROR_SUCCESS + }; + inner.error = err; + + match err { + ERROR_BROKEN_PIPE => { + mark_as_completed(&mut inner.overlapped); + Err(set_from_windows_err(err, vm)) + } + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_IO_PENDING => Ok(vm.ctx.none()), + _ => { + inner.data = OverlappedData::NotStarted; + Err(set_from_windows_err(err, vm)) + } + } + } + } + + impl Constructor for Overlapped { + type Args = (OptionalArg<isize>,); + + fn py_new(_cls: &Py<PyType>, (event,): Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let mut event = event.unwrap_or(INVALID_HANDLE_VALUE); + + if event == INVALID_HANDLE_VALUE { + event = unsafe { + windows_sys::Win32::System::Threading::CreateEventW( + core::ptr::null(), + Foundation::TRUE, + Foundation::FALSE, + core::ptr::null(), + ) as isize + }; + if event == NULL { + return Err(set_from_windows_err(0, vm)); + } + } + + let mut overlapped: OVERLAPPED = unsafe { core::mem::zeroed() }; + if event != NULL { + overlapped.hEvent = event as HANDLE; + } + let inner = OverlappedInner { + overlapped, + handle: NULL as HANDLE, + error: 0, + data: OverlappedData::None, + }; + Ok(Overlapped { + inner: PyMutex::new(inner), + }) + } + } + + impl Destructor for Overlapped { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Foundation::{ + ERROR_NOT_FOUND, ERROR_OPERATION_ABORTED, ERROR_SUCCESS, + }; + use windows_sys::Win32::System::IO::{CancelIoEx, GetOverlappedResult}; + + let mut inner = zelf.inner.lock(); + let olderr = unsafe { GetLastError() }; + + // Cancel pending I/O and wait for completion + if !HasOverlappedIoCompleted(&inner.overlapped) + && !matches!(inner.data, OverlappedData::NotStarted) + { + let cancelled = unsafe { CancelIoEx(inner.handle, &inner.overlapped) } != 0; + let mut transferred: u32 = 0; + let ret = unsafe { + GetOverlappedResult( + inner.handle, + &inner.overlapped, + &mut transferred, + if cancelled { 1 } else { 0 }, + ) + }; + + let err = if ret != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + match err { + ERROR_SUCCESS | ERROR_NOT_FOUND | ERROR_OPERATION_ABORTED => {} + _ => { + let msg = format!( + "{:?} still has pending operation at deallocation, the process may crash", + zelf + ); + let exc = vm.new_runtime_error(msg); + let err_msg = Some(format!( + "Exception ignored while deallocating overlapped operation {:?}", + zelf + )); + let obj: PyObjectRef = zelf.to_owned().into(); + vm.run_unraisable(exc, err_msg, obj); + } + } + } + + // Close the event handle + if !inner.overlapped.hEvent.is_null() { + unsafe { + Foundation::CloseHandle(inner.overlapped.hEvent); + } + inner.overlapped.hEvent = core::ptr::null_mut(); + } + + // Restore last error + unsafe { Foundation::SetLastError(olderr) }; + + Ok(()) + } + } + + #[pyfunction] + fn ConnectPipe(address: String, vm: &VirtualMachine) -> PyResult<isize> { + use windows_sys::Win32::Foundation::{GENERIC_READ, GENERIC_WRITE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_OVERLAPPED, OPEN_EXISTING, + }; + + let address_wide: Vec<u16> = address.encode_utf16().chain(core::iter::once(0)).collect(); + + let handle = unsafe { + CreateFileW( + address_wide.as_ptr(), + GENERIC_READ | GENERIC_WRITE, + 0, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_OVERLAPPED, + core::ptr::null_mut(), + ) + }; + + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { + return Err(set_from_windows_err(0, vm)); + } + + Ok(handle as isize) + } + + #[pyfunction] + fn CreateIoCompletionPort( + handle: isize, + port: isize, + key: usize, + concurrency: u32, + vm: &VirtualMachine, + ) -> PyResult<isize> { + let r = unsafe { + windows_sys::Win32::System::IO::CreateIoCompletionPort( + handle as HANDLE, + port as HANDLE, + key, + concurrency, + ) as isize + }; + if r == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(r) + } + + #[pyfunction] + fn GetQueuedCompletionStatus(port: isize, msecs: u32, vm: &VirtualMachine) -> PyResult { + let mut bytes_transferred = 0; + let mut completion_key = 0; + let mut overlapped: *mut OVERLAPPED = core::ptr::null_mut(); + let ret = unsafe { + windows_sys::Win32::System::IO::GetQueuedCompletionStatus( + port as HANDLE, + &mut bytes_transferred, + &mut completion_key, + &mut overlapped, + msecs, + ) + }; + let err = if ret != 0 { + Foundation::ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + if overlapped.is_null() { + if err == Foundation::WAIT_TIMEOUT { + return Ok(vm.ctx.none()); + } else { + return Err(set_from_windows_err(err, vm)); + } + } + + let value = vm.ctx.new_tuple(vec![ + err.to_pyobject(vm), + bytes_transferred.to_pyobject(vm), + completion_key.to_pyobject(vm), + (overlapped as usize).to_pyobject(vm), + ]); + Ok(value.into()) + } + + #[pyfunction] + fn PostQueuedCompletionStatus( + port: isize, + bytes: u32, + key: usize, + address: usize, + vm: &VirtualMachine, + ) -> PyResult<()> { + let ret = unsafe { + windows_sys::Win32::System::IO::PostQueuedCompletionStatus( + port as HANDLE, + bytes, + key, + address as *mut OVERLAPPED, + ) + }; + if ret == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + // Registry to track callback data for proper cleanup + // Uses Arc for reference counting to prevent use-after-free when callback + // and UnregisterWait race - the data stays alive until both are done + static WAIT_CALLBACK_REGISTRY: std::sync::OnceLock< + std::sync::Mutex<std::collections::HashMap<isize, alloc::sync::Arc<PostCallbackData>>>, + > = std::sync::OnceLock::new(); + + fn wait_callback_registry() -> &'static std::sync::Mutex< + std::collections::HashMap<isize, alloc::sync::Arc<PostCallbackData>>, + > { + WAIT_CALLBACK_REGISTRY + .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new())) + } + + // Callback data for RegisterWaitWithQueue + // Uses Arc to ensure the data stays alive while callback is executing + struct PostCallbackData { + completion_port: HANDLE, + overlapped: *mut OVERLAPPED, + } + + // SAFETY: The pointers are handles/addresses passed from Python and are + // only used to call Windows APIs. They are not dereferenced as Rust pointers. + unsafe impl Send for PostCallbackData {} + unsafe impl Sync for PostCallbackData {} + + unsafe extern "system" fn post_to_queue_callback( + parameter: *mut core::ffi::c_void, + timer_or_wait_fired: bool, + ) { + // Reconstruct Arc from raw pointer - this gives us ownership of one reference + // The Arc prevents use-after-free since we own a reference count + let data = unsafe { alloc::sync::Arc::from_raw(parameter as *const PostCallbackData) }; + + unsafe { + let _ = windows_sys::Win32::System::IO::PostQueuedCompletionStatus( + data.completion_port, + if timer_or_wait_fired { 1 } else { 0 }, + 0, + data.overlapped, + ); + } + // Arc is dropped here, decrementing refcount + // Memory is freed only when all references (callback + registry) are gone + } + + #[pyfunction] + fn RegisterWaitWithQueue( + object: isize, + completion_port: isize, + overlapped: usize, + timeout: u32, + vm: &VirtualMachine, + ) -> PyResult<isize> { + use windows_sys::Win32::System::Threading::{ + RegisterWaitForSingleObject, WT_EXECUTEINWAITTHREAD, WT_EXECUTEONLYONCE, + }; + + let data = alloc::sync::Arc::new(PostCallbackData { + completion_port: completion_port as HANDLE, + overlapped: overlapped as *mut OVERLAPPED, + }); + + // Create raw pointer for the callback - this increments refcount + let data_ptr = alloc::sync::Arc::into_raw(data.clone()); + + let mut new_wait_object: HANDLE = core::ptr::null_mut(); + let ret = unsafe { + RegisterWaitForSingleObject( + &mut new_wait_object, + object as HANDLE, + Some(post_to_queue_callback), + data_ptr as *mut _, + timeout, + WT_EXECUTEINWAITTHREAD | WT_EXECUTEONLYONCE, + ) + }; + + if ret == 0 { + // Registration failed - reconstruct Arc to drop the extra reference + unsafe { + let _ = alloc::sync::Arc::from_raw(data_ptr); + } + return Err(set_from_windows_err(0, vm)); + } + + // Store in registry for cleanup tracking + let wait_handle = new_wait_object as isize; + if let Ok(mut registry) = wait_callback_registry().lock() { + registry.insert(wait_handle, data); + } + + Ok(wait_handle) + } + + // Helper to cleanup callback data when unregistering + // Just removes from registry - Arc ensures memory stays alive if callback is running + fn cleanup_wait_callback_data(wait_handle: isize) { + if let Ok(mut registry) = wait_callback_registry().lock() { + // Removing from registry drops one Arc reference + // If callback already ran, this frees the memory + // If callback is still pending/running, it holds the other reference + registry.remove(&wait_handle); + } + } + + #[pyfunction] + fn UnregisterWait(wait_handle: isize, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Threading::UnregisterWait; + + let ret = unsafe { UnregisterWait(wait_handle as HANDLE) }; + // Cleanup callback data regardless of UnregisterWait result + // (callback may have already fired, or may never fire) + cleanup_wait_callback_data(wait_handle); + if ret == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + #[pyfunction] + fn UnregisterWaitEx(wait_handle: isize, event: isize, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Threading::UnregisterWaitEx; + + let ret = unsafe { UnregisterWaitEx(wait_handle as HANDLE, event as HANDLE) }; + // Cleanup callback data regardless of UnregisterWaitEx result + cleanup_wait_callback_data(wait_handle); + if ret == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + #[pyfunction] + fn BindLocal(socket: isize, family: i32, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Networking::WinSock::{ + INADDR_ANY, SOCKET_ERROR, WSAGetLastError, bind, + }; + + let ret = if family == AF_INET as i32 { + let mut addr: SOCKADDR_IN = unsafe { core::mem::zeroed() }; + addr.sin_family = AF_INET; + addr.sin_port = 0; + addr.sin_addr.S_un.S_addr = INADDR_ANY; + unsafe { + bind( + socket as _, + &addr as *const _ as *const SOCKADDR, + core::mem::size_of::<SOCKADDR_IN>() as i32, + ) + } + } else if family == AF_INET6 as i32 { + // in6addr_any is all zeros, which we have from zeroed() + let mut addr: SOCKADDR_IN6 = unsafe { core::mem::zeroed() }; + addr.sin6_family = AF_INET6; + addr.sin6_port = 0; + unsafe { + bind( + socket as _, + &addr as *const _ as *const SOCKADDR, + core::mem::size_of::<SOCKADDR_IN6>() as i32, + ) + } + } else { + return Err(vm.new_value_error("expected tuple of length 2 or 4")); + }; + + if ret == SOCKET_ERROR { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + Ok(()) + } + + #[pyfunction] + fn FormatMessage(error_code: u32, _vm: &VirtualMachine) -> PyResult<String> { + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::System::Diagnostics::Debug::{ + FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM, + FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, + }; + + // LANG_NEUTRAL = 0, SUBLANG_DEFAULT = 1 + const LANG_NEUTRAL: u32 = 0; + const SUBLANG_DEFAULT: u32 = 1; + + let mut buffer: *mut u16 = core::ptr::null_mut(); + + let len = unsafe { + FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_IGNORE_INSERTS, + core::ptr::null(), + error_code, + (SUBLANG_DEFAULT << 10) | LANG_NEUTRAL, + &mut buffer as *mut _ as *mut u16, + 0, + core::ptr::null(), + ) + }; + + if len == 0 || buffer.is_null() { + if !buffer.is_null() { + unsafe { LocalFree(buffer as *mut _) }; + } + return Ok(format!("unknown error code {}", error_code)); + } + + // Convert to Rust string, trimming trailing whitespace + let slice = unsafe { core::slice::from_raw_parts(buffer, len as usize) }; + let msg = String::from_utf16_lossy(slice).trim_end().to_string(); + + unsafe { LocalFree(buffer as *mut _) }; + + Ok(msg) + } + + #[pyfunction] + fn WSAConnect(socket: isize, address: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Networking::WinSock::{SOCKET_ERROR, WSAConnect, WSAGetLastError}; + + let (addr_bytes, addr_len) = parse_address(&address, vm)?; + + let ret = unsafe { + WSAConnect( + socket as _, + addr_bytes.as_ptr() as *const SOCKADDR, + addr_len, + core::ptr::null(), + core::ptr::null_mut(), + core::ptr::null(), + core::ptr::null(), + ) + }; + + if ret == SOCKET_ERROR { + let err = unsafe { WSAGetLastError() } as u32; + return Err(set_from_windows_err(err, vm)); + } + Ok(()) + } + + #[pyfunction] + fn CreateEvent( + event_attributes: PyObjectRef, + manual_reset: bool, + initial_state: bool, + name: Option<String>, + vm: &VirtualMachine, + ) -> PyResult<isize> { + if !vm.is_none(&event_attributes) { + return Err(vm.new_value_error("EventAttributes must be None")); + } + + let name_wide: Option<Vec<u16>> = + name.map(|n| n.encode_utf16().chain(core::iter::once(0)).collect()); + let name_ptr = name_wide + .as_ref() + .map(|n| n.as_ptr()) + .unwrap_or(core::ptr::null()); + + let event = unsafe { + windows_sys::Win32::System::Threading::CreateEventW( + core::ptr::null(), + if manual_reset { 1 } else { 0 }, + if initial_state { 1 } else { 0 }, + name_ptr, + ) as isize + }; + if event == NULL { + return Err(set_from_windows_err(0, vm)); + } + Ok(event) + } + + #[pyfunction] + fn SetEvent(handle: isize, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { windows_sys::Win32::System::Threading::SetEvent(handle as HANDLE) }; + if ret == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } + + #[pyfunction] + fn ResetEvent(handle: isize, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { windows_sys::Win32::System::Threading::ResetEvent(handle as HANDLE) }; + if ret == 0 { + return Err(set_from_windows_err(0, vm)); + } + Ok(()) + } +} diff --git a/crates/stdlib/src/posixshmem.rs b/crates/stdlib/src/posixshmem.rs new file mode 100644 index 00000000000..f5481619bba --- /dev/null +++ b/crates/stdlib/src/posixshmem.rs @@ -0,0 +1,52 @@ +#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] +pub(crate) use _posixshmem::module_def; + +#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))] +#[pymodule] +mod _posixshmem { + use alloc::ffi::CString; + + use crate::{ + common::os::errno_io_error, + vm::{ + FromArgs, PyResult, VirtualMachine, builtins::PyUtf8StrRef, convert::IntoPyException, + }, + }; + + #[derive(FromArgs)] + struct ShmOpenArgs { + #[pyarg(any)] + name: PyUtf8StrRef, + #[pyarg(any)] + flags: libc::c_int, + #[pyarg(any, default = 0o600)] + mode: libc::mode_t, + } + + #[pyfunction] + fn shm_open(args: ShmOpenArgs, vm: &VirtualMachine) -> PyResult<libc::c_int> { + let name = CString::new(args.name.as_str()).map_err(|e| e.into_pyexception(vm))?; + let mode: libc::c_uint = args.mode as _; + #[cfg(target_os = "freebsd")] + let mode = mode.try_into().unwrap(); + // SAFETY: `name` is a NUL-terminated string and `shm_open` does not write through it. + let fd = unsafe { libc::shm_open(name.as_ptr(), args.flags, mode) }; + if fd == -1 { + Err(errno_io_error().into_pyexception(vm)) + } else { + Ok(fd) + } + } + + #[pyfunction] + fn shm_unlink(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<()> { + let name = CString::new(name.as_str()).map_err(|e| e.into_pyexception(vm))?; + // SAFETY: `name` is a valid NUL-terminated string and `shm_unlink` only reads it. + let ret = unsafe { libc::shm_unlink(name.as_ptr()) }; + if ret == -1 { + Err(errno_io_error().into_pyexception(vm)) + } else { + Ok(()) + } + } +} diff --git a/crates/stdlib/src/posixsubprocess.rs b/crates/stdlib/src/posixsubprocess.rs new file mode 100644 index 00000000000..fec2ceb16d5 --- /dev/null +++ b/crates/stdlib/src/posixsubprocess.rs @@ -0,0 +1,537 @@ +// spell-checker:disable + +use crate::vm::{ + builtins::PyListRef, + function::ArgSequence, + ospath::OsPath, + stdlib::posix, + {PyObjectRef, PyResult, TryFromObject, VirtualMachine}, +}; +use itertools::Itertools; +use nix::{ + errno::Errno, + unistd::{self, Pid}, +}; +use std::{ + io::prelude::*, + os::fd::{AsFd, AsRawFd, BorrowedFd, IntoRawFd, OwnedFd, RawFd}, +}; +use unistd::{Gid, Uid}; + +use alloc::ffi::CString; + +use core::{convert::Infallible as Never, ffi::CStr, marker::PhantomData, ops::Deref}; + +pub(crate) use _posixsubprocess::module_def; + +#[pymodule] +mod _posixsubprocess { + use rustpython_vm::{AsObject, TryFromBorrowedObject}; + + use super::*; + use crate::vm::{PyResult, VirtualMachine, convert::IntoPyException}; + + #[pyfunction] + fn fork_exec(args: ForkExecArgs<'_>, vm: &VirtualMachine) -> PyResult<libc::pid_t> { + // Check for interpreter shutdown when preexec_fn is used + if args.preexec_fn.is_some() + && vm + .state + .finalizing + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(vm.new_python_finalization_error( + "preexec_fn not supported at interpreter shutdown".to_owned(), + )); + } + + let extra_groups = args + .groups_list + .as_ref() + .map(|l| Vec::<Gid>::try_from_borrowed_object(vm, l.as_object())) + .transpose()?; + let argv = CharPtrVec::from_iter(args.args.iter()); + let envp = args.env_list.as_ref().map(CharPtrVec::from_iter); + let procargs = ProcArgs { + argv: &argv, + envp: envp.as_deref(), + extra_groups: extra_groups.as_deref(), + }; + match unsafe { nix::unistd::fork() }.map_err(|err| err.into_pyexception(vm))? { + nix::unistd::ForkResult::Child => exec(&args, procargs, vm), + nix::unistd::ForkResult::Parent { child } => Ok(child.as_raw()), + } + } +} + +macro_rules! gen_args { + ($($field:ident: $t:ty),*$(,)?) => { + #[derive(FromArgs)] + struct ForkExecArgs<'fd> { + $(#[pyarg(positional)] $field: $t,)* + } + }; +} + +struct CStrPathLike { + s: CString, +} +impl TryFromObject for CStrPathLike { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let s = OsPath::try_from_object(vm, obj)?.into_cstring(vm)?; + Ok(Self { s }) + } +} +impl AsRef<CStr> for CStrPathLike { + fn as_ref(&self) -> &CStr { + &self.s + } +} + +#[derive(Default)] +struct CharPtrVec<'a> { + vec: Vec<*const libc::c_char>, + marker: PhantomData<Vec<&'a CStr>>, +} + +impl<'a, T: AsRef<CStr>> FromIterator<&'a T> for CharPtrVec<'a> { + fn from_iter<I: IntoIterator<Item = &'a T>>(iter: I) -> Self { + let vec = iter + .into_iter() + .map(|x| x.as_ref().as_ptr()) + .chain(core::iter::once(core::ptr::null())) + .collect(); + Self { + vec, + marker: PhantomData, + } + } +} + +impl<'a> Deref for CharPtrVec<'a> { + type Target = CharPtrSlice<'a>; + fn deref(&self) -> &Self::Target { + unsafe { + &*(self.vec.as_slice() as *const [*const libc::c_char] as *const CharPtrSlice<'a>) + } + } +} + +#[repr(transparent)] +struct CharPtrSlice<'a> { + marker: PhantomData<[&'a CStr]>, + slice: [*const libc::c_char], +} + +impl CharPtrSlice<'_> { + const fn as_ptr(&self) -> *const *const libc::c_char { + self.slice.as_ptr() + } +} + +#[derive(Copy, Clone)] +struct Fd(BorrowedFd<'static>); + +impl TryFromObject for Fd { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + match MaybeFd::try_from_object(vm, obj)? { + MaybeFd::Valid(fd) => Ok(fd), + MaybeFd::Invalid => Err(vm.new_value_error("invalid fd")), + } + } +} + +impl Write for Fd { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + Ok(unistd::write(self, buf)?) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl AsRawFd for Fd { + fn as_raw_fd(&self) -> RawFd { + self.0.as_raw_fd() + } +} + +impl IntoRawFd for Fd { + fn into_raw_fd(self) -> RawFd { + self.0.as_raw_fd() + } +} + +impl AsFd for Fd { + fn as_fd(&self) -> BorrowedFd<'_> { + self.0.as_fd() + } +} + +impl From<OwnedFd> for Fd { + fn from(fd: OwnedFd) -> Self { + Self(unsafe { BorrowedFd::borrow_raw(fd.into_raw_fd()) }) + } +} + +#[derive(Copy, Clone)] +enum MaybeFd { + Valid(Fd), + Invalid, +} + +impl TryFromObject for MaybeFd { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let fd = i32::try_from_object(vm, obj)?; + Ok(if fd == -1 { + MaybeFd::Invalid + } else { + MaybeFd::Valid(Fd(unsafe { BorrowedFd::borrow_raw(fd) })) + }) + } +} + +impl AsRawFd for MaybeFd { + fn as_raw_fd(&self) -> RawFd { + match self { + MaybeFd::Valid(fd) => fd.as_raw_fd(), + MaybeFd::Invalid => -1, + } + } +} + +// impl + +gen_args! { + args: ArgSequence<CStrPathLike> /* list */, + exec_list: ArgSequence<CStrPathLike> /* list */, + close_fds: bool, + fds_to_keep: ArgSequence<BorrowedFd<'fd>>, + cwd: Option<CStrPathLike>, + env_list: Option<ArgSequence<CStrPathLike>>, + p2cread: MaybeFd, + p2cwrite: MaybeFd, + c2pread: MaybeFd, + c2pwrite: MaybeFd, + errread: MaybeFd, + errwrite: MaybeFd, + errpipe_read: Fd, + errpipe_write: Fd, + restore_signals: bool, + call_setsid: bool, + pgid_to_set: libc::pid_t, + gid: Option<Gid>, + groups_list: Option<PyListRef>, + uid: Option<Uid>, + child_umask: i32, + preexec_fn: Option<PyObjectRef>, +} + +// can't reallocate inside of exec(), so we reallocate prior to fork() and pass this along +struct ProcArgs<'a> { + argv: &'a CharPtrSlice<'a>, + envp: Option<&'a CharPtrSlice<'a>>, + extra_groups: Option<&'a [Gid]>, +} + +fn exec(args: &ForkExecArgs<'_>, procargs: ProcArgs<'_>, vm: &VirtualMachine) -> ! { + let mut ctx = ExecErrorContext::NoExec; + match exec_inner(args, procargs, &mut ctx, vm) { + Ok(x) => match x {}, + Err(e) => { + let mut pipe = args.errpipe_write; + if matches!(ctx, ExecErrorContext::PreExec) { + // For preexec_fn errors, use SubprocessError format (errno=0) + let _ = write!(pipe, "SubprocessError:0:{}", ctx.as_msg()); + } else { + // errno is written in hex format + let _ = write!(pipe, "OSError:{:x}:{}", e as i32, ctx.as_msg()); + } + std::process::exit(255) + } + } +} + +enum ExecErrorContext { + NoExec, + ChDir, + PreExec, + Exec, +} + +impl ExecErrorContext { + const fn as_msg(&self) -> &'static str { + match self { + Self::NoExec => "noexec", + Self::ChDir => "noexec:chdir", + Self::PreExec => "Exception occurred in preexec_fn.", + Self::Exec => "", + } + } +} + +fn exec_inner( + args: &ForkExecArgs<'_>, + procargs: ProcArgs<'_>, + ctx: &mut ExecErrorContext, + vm: &VirtualMachine, +) -> nix::Result<Never> { + for &fd in args.fds_to_keep.as_slice() { + if fd.as_raw_fd() != args.errpipe_write.as_raw_fd() { + posix::set_inheritable(fd, true)? + } + } + + for &fd in &[args.p2cwrite, args.c2pread, args.errread] { + if let MaybeFd::Valid(fd) = fd { + unistd::close(fd)?; + } + } + unistd::close(args.errpipe_read)?; + + let c2pwrite = match args.c2pwrite { + MaybeFd::Valid(c2pwrite) if c2pwrite.as_raw_fd() == 0 => { + let fd = unistd::dup(c2pwrite)?; + posix::set_inheritable(fd.as_fd(), true)?; + MaybeFd::Valid(fd.into()) + } + fd => fd, + }; + + let mut errwrite = args.errwrite; + loop { + match errwrite { + MaybeFd::Valid(fd) if fd.as_raw_fd() == 0 || fd.as_raw_fd() == 1 => { + let fd = unistd::dup(fd)?; + posix::set_inheritable(fd.as_fd(), true)?; + errwrite = MaybeFd::Valid(fd.into()); + } + _ => break, + } + } + + fn dup_into_stdio<F>(fd: MaybeFd, io_fd: i32, dup2_stdio: F) -> nix::Result<()> + where + F: Fn(Fd) -> nix::Result<()>, + { + match fd { + MaybeFd::Valid(fd) if fd.as_raw_fd() == io_fd => { + posix::set_inheritable(fd.as_fd(), true) + } + MaybeFd::Valid(fd) => dup2_stdio(fd), + MaybeFd::Invalid => Ok(()), + } + } + dup_into_stdio(args.p2cread, 0, unistd::dup2_stdin)?; + dup_into_stdio(c2pwrite, 1, unistd::dup2_stdout)?; + dup_into_stdio(errwrite, 2, unistd::dup2_stderr)?; + + if let Some(ref cwd) = args.cwd { + unistd::chdir(cwd.s.as_c_str()).inspect_err(|_| *ctx = ExecErrorContext::ChDir)? + } + + if args.child_umask >= 0 { + unsafe { libc::umask(args.child_umask as libc::mode_t) }; + } + + if args.restore_signals { + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + libc::signal(libc::SIGXFSZ, libc::SIG_DFL); + } + } + + if args.call_setsid { + unistd::setsid()?; + } + + if args.pgid_to_set > -1 { + unistd::setpgid(Pid::from_raw(0), Pid::from_raw(args.pgid_to_set))?; + } + + if let Some(_groups) = procargs.extra_groups { + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "redox")))] + unistd::setgroups(_groups)?; + } + + if let Some(gid) = args.gid.filter(|x| x.as_raw() != u32::MAX) { + let ret = unsafe { libc::setregid(gid.as_raw(), gid.as_raw()) }; + nix::Error::result(ret)?; + } + + if let Some(uid) = args.uid.filter(|x| x.as_raw() != u32::MAX) { + let ret = unsafe { libc::setreuid(uid.as_raw(), uid.as_raw()) }; + nix::Error::result(ret)?; + } + + // Call preexec_fn after all process setup but before closing FDs + if let Some(ref preexec_fn) = args.preexec_fn { + match preexec_fn.call((), vm) { + Ok(_) => {} + Err(_e) => { + // Cannot safely stringify exception after fork + *ctx = ExecErrorContext::PreExec; + return Err(Errno::UnknownErrno); + } + } + } + + *ctx = ExecErrorContext::Exec; + + if args.close_fds { + close_fds(KeepFds { + above: 2, + keep: &args.fds_to_keep, + }); + } + + let mut first_err = None; + for exec in args.exec_list.as_slice() { + // not using nix's versions of these functions because those allocate the char-ptr array, + // and we can't allocate + if let Some(envp) = procargs.envp { + unsafe { libc::execve(exec.s.as_ptr(), procargs.argv.as_ptr(), envp.as_ptr()) }; + } else { + unsafe { libc::execv(exec.s.as_ptr(), procargs.argv.as_ptr()) }; + } + let e = Errno::last(); + if e != Errno::ENOENT && e != Errno::ENOTDIR && first_err.is_none() { + first_err = Some(e) + } + } + Err(first_err.unwrap_or_else(Errno::last)) +} + +#[derive(Copy, Clone)] +struct KeepFds<'a> { + above: i32, + keep: &'a [BorrowedFd<'a>], +} + +impl KeepFds<'_> { + fn should_keep(self, fd: i32) -> bool { + fd > self.above + && self + .keep + .binary_search_by_key(&fd, BorrowedFd::as_raw_fd) + .is_err() + } +} + +fn close_fds(keep: KeepFds<'_>) { + #[cfg(not(target_os = "redox"))] + if close_dir_fds(keep).is_ok() { + return; + } + #[cfg(target_os = "redox")] + if close_filetable_fds(keep).is_ok() { + return; + } + close_fds_brute_force(keep) +} + +#[cfg(not(target_os = "redox"))] +fn close_dir_fds(keep: KeepFds<'_>) -> nix::Result<()> { + use nix::{dir::Dir, fcntl::OFlag}; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple", + ))] + let fd_dir_name = c"/dev/fd"; + + #[cfg(any(target_os = "linux", target_os = "android"))] + let fd_dir_name = c"/proc/self/fd"; + + let mut dir = Dir::open( + fd_dir_name, + OFlag::O_RDONLY | OFlag::O_DIRECTORY, + nix::sys::stat::Mode::empty(), + )?; + let dirfd = dir.as_raw_fd(); + 'outer: for e in dir.iter() { + let e = e?; + let mut parser = IntParser::default(); + for &c in e.file_name().to_bytes() { + if parser.feed(c).is_err() { + continue 'outer; + } + } + let fd = parser.num; + if fd != dirfd && keep.should_keep(fd) { + let _ = unistd::close(fd); + } + } + Ok(()) +} + +#[cfg(target_os = "redox")] +fn close_filetable_fds(keep: KeepFds<'_>) -> nix::Result<()> { + use nix::fcntl; + use std::os::fd::{FromRawFd, OwnedFd}; + let filetable = fcntl::open( + c"/scheme/thisproc/current/filetable", + fcntl::OFlag::O_RDONLY, + nix::sys::stat::Mode::empty(), + )?; + let read_one = || -> nix::Result<_> { + let mut byte = 0; + let n = nix::unistd::read(&filetable, std::slice::from_mut(&mut byte))?; + Ok((n > 0).then_some(byte)) + }; + while let Some(c) = read_one()? { + let mut parser = IntParser::default(); + if parser.feed(c).is_err() { + continue; + } + let done = loop { + let Some(c) = read_one()? else { break true }; + if parser.feed(c).is_err() { + break false; + } + }; + + let fd = parser.num as i32; + if fd != filetable.as_raw_fd() && keep.should_keep(fd) { + let _ = unistd::close(fd); + } + if done { + break; + } + } + Ok(()) +} + +fn close_fds_brute_force(keep: KeepFds<'_>) { + let max_fd = nix::unistd::sysconf(nix::unistd::SysconfVar::OPEN_MAX) + .ok() + .flatten() + .unwrap_or(256) as i32; + let fds = itertools::chain![ + Some(keep.above), + keep.keep.iter().map(BorrowedFd::as_raw_fd), + Some(max_fd) + ]; + for fd in fds.tuple_windows().flat_map(|(start, end)| start + 1..end) { + unsafe { libc::close(fd) }; + } +} + +#[derive(Default)] +struct IntParser { + num: i32, +} + +struct NonDigit; +impl IntParser { + fn feed(&mut self, c: u8) -> Result<(), NonDigit> { + let digit = (c as char).to_digit(10).ok_or(NonDigit)?; + self.num *= 10; + self.num += digit as i32; + Ok(()) + } +} diff --git a/crates/stdlib/src/pyexpat.rs b/crates/stdlib/src/pyexpat.rs new file mode 100644 index 00000000000..40418f54c30 --- /dev/null +++ b/crates/stdlib/src/pyexpat.rs @@ -0,0 +1,438 @@ +//! Pyexpat builtin module + +// spell-checker: ignore libexpat + +pub(crate) use _pyexpat::module_def; + +macro_rules! create_property { + ($ctx: expr, $attributes: expr, $name: expr, $class: expr, $element: ident) => { + let attr = $ctx.new_static_getset( + $name, + $class, + move |this: &PyExpatLikeXmlParser| this.$element.read().clone(), + move |this: &PyExpatLikeXmlParser, func: PyObjectRef| *this.$element.write() = func, + ); + + $attributes.insert($ctx.intern_str($name), attr.into()); + }; +} + +macro_rules! create_bool_property { + ($ctx: expr, $attributes: expr, $name: expr, $class: expr, $element: ident) => { + let attr = $ctx.new_static_getset( + $name, + $class, + move |this: &PyExpatLikeXmlParser| this.$element.read().clone(), + move |this: &PyExpatLikeXmlParser, + value: PyObjectRef, + vm: &VirtualMachine| + -> PyResult<()> { + let bool_value = value.is_true(vm)?; + *this.$element.write() = vm.ctx.new_bool(bool_value).into(); + Ok(()) + }, + ); + + $attributes.insert($ctx.intern_str($name), attr.into()); + }; +} + +#[pymodule(name = "pyexpat")] +mod _pyexpat { + use crate::vm::{ + Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBytesRef, PyException, PyModule, PyStr, PyStrRef, PyType, PyUtf8StrRef}, + extend_module, + function::{ArgBytesLike, Either, IntoFuncArgs, OptionalArg}, + types::Constructor, + }; + use rustpython_common::lock::PyRwLock; + use std::io::Cursor; + use xml::reader::XmlEvent; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + // Add submodules + let model = super::_model::module_def(&vm.ctx).create_module(vm)?; + let errors = super::_errors::module_def(&vm.ctx).create_module(vm)?; + + extend_module!(vm, module, { + "model" => model, + "errors" => errors, + }); + + Ok(()) + } + + type MutableObject = PyRwLock<PyObjectRef>; + + #[pyattr(name = "version_info")] + pub const VERSION_INFO: (u32, u32, u32) = (2, 7, 1); + + #[pyattr] + #[pyclass(name = "xmlparser", module = false, traverse)] + #[derive(Debug, PyPayload)] + pub struct PyExpatLikeXmlParser { + #[pytraverse(skip)] + namespace_separator: Option<String>, + start_element: MutableObject, + end_element: MutableObject, + character_data: MutableObject, + entity_decl: MutableObject, + buffer_text: MutableObject, + namespace_prefixes: MutableObject, + ordered_attributes: MutableObject, + specified_attributes: MutableObject, + intern: MutableObject, + // Additional handlers (stubs for compatibility) + processing_instruction: MutableObject, + unparsed_entity_decl: MutableObject, + notation_decl: MutableObject, + start_namespace_decl: MutableObject, + end_namespace_decl: MutableObject, + comment: MutableObject, + start_cdata_section: MutableObject, + end_cdata_section: MutableObject, + default: MutableObject, + default_expand: MutableObject, + not_standalone: MutableObject, + external_entity_ref: MutableObject, + start_doctype_decl: MutableObject, + end_doctype_decl: MutableObject, + xml_decl: MutableObject, + element_decl: MutableObject, + attlist_decl: MutableObject, + skipped_entity: MutableObject, + } + type PyExpatLikeXmlParserRef = PyRef<PyExpatLikeXmlParser>; + + #[inline] + fn invoke_handler<T>(vm: &VirtualMachine, handler: &MutableObject, args: T) + where + T: IntoFuncArgs, + { + // Clone the handler while holding the read lock, then release the lock + let handler = handler.read().clone(); + handler.call(args, vm).ok(); + } + + #[pyclass] + impl PyExpatLikeXmlParser { + fn new( + namespace_separator: Option<String>, + intern: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyExpatLikeXmlParserRef> { + let intern_dict = intern.unwrap_or_else(|| vm.ctx.new_dict().into()); + Ok(Self { + namespace_separator, + start_element: MutableObject::new(vm.ctx.none()), + end_element: MutableObject::new(vm.ctx.none()), + character_data: MutableObject::new(vm.ctx.none()), + entity_decl: MutableObject::new(vm.ctx.none()), + buffer_text: MutableObject::new(vm.ctx.new_bool(false).into()), + namespace_prefixes: MutableObject::new(vm.ctx.new_bool(false).into()), + ordered_attributes: MutableObject::new(vm.ctx.new_bool(false).into()), + specified_attributes: MutableObject::new(vm.ctx.new_bool(false).into()), + intern: MutableObject::new(intern_dict), + // Additional handlers (stubs for compatibility) + processing_instruction: MutableObject::new(vm.ctx.none()), + unparsed_entity_decl: MutableObject::new(vm.ctx.none()), + notation_decl: MutableObject::new(vm.ctx.none()), + start_namespace_decl: MutableObject::new(vm.ctx.none()), + end_namespace_decl: MutableObject::new(vm.ctx.none()), + comment: MutableObject::new(vm.ctx.none()), + start_cdata_section: MutableObject::new(vm.ctx.none()), + end_cdata_section: MutableObject::new(vm.ctx.none()), + default: MutableObject::new(vm.ctx.none()), + default_expand: MutableObject::new(vm.ctx.none()), + not_standalone: MutableObject::new(vm.ctx.none()), + external_entity_ref: MutableObject::new(vm.ctx.none()), + start_doctype_decl: MutableObject::new(vm.ctx.none()), + end_doctype_decl: MutableObject::new(vm.ctx.none()), + xml_decl: MutableObject::new(vm.ctx.none()), + element_decl: MutableObject::new(vm.ctx.none()), + attlist_decl: MutableObject::new(vm.ctx.none()), + skipped_entity: MutableObject::new(vm.ctx.none()), + } + .into_ref(&vm.ctx)) + } + + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { + let mut attributes = class.attributes.write(); + + create_property!(ctx, attributes, "StartElementHandler", class, start_element); + create_property!(ctx, attributes, "EndElementHandler", class, end_element); + create_property!( + ctx, + attributes, + "CharacterDataHandler", + class, + character_data + ); + create_property!(ctx, attributes, "EntityDeclHandler", class, entity_decl); + create_bool_property!(ctx, attributes, "buffer_text", class, buffer_text); + create_bool_property!( + ctx, + attributes, + "namespace_prefixes", + class, + namespace_prefixes + ); + create_bool_property!( + ctx, + attributes, + "ordered_attributes", + class, + ordered_attributes + ); + create_bool_property!( + ctx, + attributes, + "specified_attributes", + class, + specified_attributes + ); + create_property!(ctx, attributes, "intern", class, intern); + // Additional handlers (stubs for compatibility) + create_property!( + ctx, + attributes, + "ProcessingInstructionHandler", + class, + processing_instruction + ); + create_property!( + ctx, + attributes, + "UnparsedEntityDeclHandler", + class, + unparsed_entity_decl + ); + create_property!(ctx, attributes, "NotationDeclHandler", class, notation_decl); + create_property!( + ctx, + attributes, + "StartNamespaceDeclHandler", + class, + start_namespace_decl + ); + create_property!( + ctx, + attributes, + "EndNamespaceDeclHandler", + class, + end_namespace_decl + ); + create_property!(ctx, attributes, "CommentHandler", class, comment); + create_property!( + ctx, + attributes, + "StartCdataSectionHandler", + class, + start_cdata_section + ); + create_property!( + ctx, + attributes, + "EndCdataSectionHandler", + class, + end_cdata_section + ); + create_property!(ctx, attributes, "DefaultHandler", class, default); + create_property!( + ctx, + attributes, + "DefaultHandlerExpand", + class, + default_expand + ); + create_property!( + ctx, + attributes, + "NotStandaloneHandler", + class, + not_standalone + ); + create_property!( + ctx, + attributes, + "ExternalEntityRefHandler", + class, + external_entity_ref + ); + create_property!( + ctx, + attributes, + "StartDoctypeDeclHandler", + class, + start_doctype_decl + ); + create_property!( + ctx, + attributes, + "EndDoctypeDeclHandler", + class, + end_doctype_decl + ); + create_property!(ctx, attributes, "XmlDeclHandler", class, xml_decl); + create_property!(ctx, attributes, "ElementDeclHandler", class, element_decl); + create_property!(ctx, attributes, "AttlistDeclHandler", class, attlist_decl); + create_property!( + ctx, + attributes, + "SkippedEntityHandler", + class, + skipped_entity + ); + } + + fn create_config(&self) -> xml::ParserConfig { + xml::ParserConfig::new() + .cdata_to_characters(true) + .coalesce_characters(false) + .whitespace_to_characters(true) + } + + /// Construct element name with namespace if separator is set + fn make_name(&self, name: &xml::name::OwnedName) -> String { + match (&self.namespace_separator, &name.namespace) { + (Some(sep), Some(ns)) => format!("{}{}{}", ns, sep, name.local_name), + _ => name.local_name.clone(), + } + } + + fn do_parse<T>( + &self, + vm: &VirtualMachine, + parser: xml::EventReader<T>, + ) -> Result<(), xml::reader::Error> + where + T: std::io::Read, + { + for e in parser { + match e { + Ok(XmlEvent::StartElement { + name, attributes, .. + }) => { + let dict = vm.ctx.new_dict(); + for attribute in attributes { + let attr_name = self.make_name(&attribute.name); + dict.set_item( + attr_name.as_str(), + vm.ctx.new_str(attribute.value).into(), + vm, + ) + .unwrap(); + } + + let name_str = PyStr::from(self.make_name(&name)).into_ref(&vm.ctx); + invoke_handler(vm, &self.start_element, (name_str, dict)); + } + Ok(XmlEvent::EndElement { name, .. }) => { + let name_str = PyStr::from(self.make_name(&name)).into_ref(&vm.ctx); + invoke_handler(vm, &self.end_element, (name_str,)); + } + Ok(XmlEvent::Characters(chars)) => { + let str = PyStr::from(chars).into_ref(&vm.ctx); + invoke_handler(vm, &self.character_data, (str,)); + } + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + #[pymethod(name = "Parse")] + fn parse( + &self, + data: Either<PyStrRef, PyBytesRef>, + _isfinal: OptionalArg<bool>, + vm: &VirtualMachine, + ) -> PyResult<i32> { + let bytes = match data { + Either::A(s) => s.as_bytes().to_vec(), + Either::B(b) => b.as_bytes().to_vec(), + }; + // Empty data is valid - used to finalize parsing + if bytes.is_empty() { + return Ok(1); + } + let reader = Cursor::<Vec<u8>>::new(bytes); + let parser = self.create_config().create_reader(reader); + // Note: xml-rs is stricter than libexpat; some errors are silently ignored + // to maintain compatibility with existing Python code + let _ = self.do_parse(vm, parser); + Ok(1) + } + + #[pymethod(name = "ParseFile")] + fn parse_file(&self, file: PyObjectRef, vm: &VirtualMachine) -> PyResult<i32> { + let read_res = vm.call_method(&file, "read", ())?; + let bytes_like = ArgBytesLike::try_from_object(vm, read_res)?; + let buf = bytes_like.borrow_buf().to_vec(); + if buf.is_empty() { + return Ok(1); + } + let reader = Cursor::new(buf); + let parser = self.create_config().create_reader(reader); + // Note: xml-rs is stricter than libexpat; some errors are silently ignored + let _ = self.do_parse(vm, parser); + Ok(1) + } + } + + #[derive(FromArgs)] + struct ParserCreateArgs { + #[pyarg(any, optional)] + encoding: Option<PyStrRef>, + #[pyarg(any, optional)] + namespace_separator: Option<PyUtf8StrRef>, + #[pyarg(any, optional)] + intern: Option<PyObjectRef>, + } + + #[pyfunction(name = "ParserCreate")] + fn parser_create( + args: ParserCreateArgs, + vm: &VirtualMachine, + ) -> PyResult<PyExpatLikeXmlParserRef> { + // Validate namespace_separator: must be at most one character + let ns_sep = match args.namespace_separator { + Some(ref s) => { + if s.as_str().chars().count() > 1 { + return Err(vm.new_value_error( + "namespace_separator must be at most one character, omitted, or None", + )); + } + Some(s.as_str().to_owned()) + } + None => None, + }; + + // encoding parameter is currently not used (xml-rs handles encoding from XML declaration) + let _ = args.encoding; + + PyExpatLikeXmlParser::new(ns_sep, args.intern, vm) + } + + // TODO: Tie this exception to the module's state. + #[pyattr] + #[pyattr(name = "error")] + #[pyexception(name = "ExpatError", base = PyException)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyExpatError(PyException); + + #[pyexception] + impl PyExpatError {} +} + +#[pymodule(name = "model")] +mod _model {} + +#[pymodule(name = "errors")] +mod _errors {} diff --git a/stdlib/src/pystruct.rs b/crates/stdlib/src/pystruct.rs similarity index 80% rename from stdlib/src/pystruct.rs rename to crates/stdlib/src/pystruct.rs index 2d83e9570d6..9c5d67f396c 100644 --- a/stdlib/src/pystruct.rs +++ b/crates/stdlib/src/pystruct.rs @@ -1,24 +1,25 @@ //! Python struct module. //! -//! Docs: https://docs.python.org/3/library/struct.html +//! Docs: <https://docs.python.org/3/library/struct.html> //! //! Use this rust module to do byte packing: -//! https://docs.rs/byteorder/1.2.6/byteorder/ +//! <https://docs.rs/byteorder/1.2.6/byteorder/> -pub(crate) use _struct::make_module; +pub(crate) use _struct::module_def; #[pymodule] pub(crate) mod _struct { use crate::vm::{ - buffer::{new_struct_error, struct_error_type, FormatSpec}, - builtins::{PyBytes, PyStr, PyStrRef, PyTupleRef, PyTypeRef}, + AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + buffer::{FormatSpec, new_struct_error, struct_error_type}, + builtins::{PyBytes, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef}, function::{ArgBytesLike, ArgMemoryBuffer, PosArgs}, match_class, protocol::PyIterReturn, - types::{Constructor, IterNext, Iterable, SelfIter}, - AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + types::{Constructor, IterNext, Iterable, Representable, SelfIter}, }; use crossbeam_utils::atomic::AtomicCell; + use rustpython_common::wtf8::{Wtf8Buf, wtf8_concat}; #[derive(Traverse)] struct IntoStructFormatBytes(PyStrRef); @@ -27,30 +28,25 @@ pub(crate) mod _struct { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { // CPython turns str to bytes but we do reversed way here // The only performance difference is this transition cost - let fmt = match_class! { - match obj { - s @ PyStr => if s.is_ascii() { - Some(s) - } else { - None - }, - b @ PyBytes => if b.is_ascii() { - Some(unsafe { - PyStr::new_ascii_unchecked(b.as_bytes().to_vec()) - }.into_ref(&vm.ctx)) - } else { - None - }, - other => return Err(vm.new_type_error(format!("Struct() argument 1 must be a str or bytes object, not {}", other.class().name()))), - } - }.ok_or_else(|| vm.new_unicode_decode_error("Struct format must be a ascii string".to_owned()))?; - Ok(IntoStructFormatBytes(fmt)) + let fmt = match_class!(match obj { + s @ PyStr => s.isascii().then_some(s), + b @ PyBytes => ascii::AsciiStr::from_ascii(&b) + .ok() + .map(|s| vm.ctx.new_str(s)), + other => + return Err(vm.new_type_error(format!( + "Struct() argument 1 must be a str or bytes object, not {}", + other.class().name() + ))), + }) + .ok_or_else(|| vm.new_unicode_decode_error("Struct format must be a ascii string"))?; + Ok(Self(fmt)) } } impl IntoStructFormatBytes { fn format_spec(&self, vm: &VirtualMachine) -> PyResult<FormatSpec> { - FormatSpec::parse(self.0.as_str().as_bytes(), vm) + FormatSpec::parse(self.0.as_bytes(), vm) } } @@ -76,7 +72,7 @@ pub(crate) mod _struct { } else { ("unpack_from", "unpacking") }; - if offset >= buffer_len { + if offset + needed > buffer_len { let msg = format!( "{op} requires a buffer of at least {required} bytes for {op_action} {needed} \ bytes at offset {offset} (actual buffer size is {buffer_len})", @@ -137,7 +133,7 @@ pub(crate) mod _struct { #[derive(FromArgs)] struct UpdateFromArgs { buffer: ArgBytesLike, - #[pyarg(any, default = "0")] + #[pyarg(any, default = 0)] offset: isize, } @@ -166,17 +162,17 @@ pub(crate) mod _struct { } impl UnpackIterator { - fn new( + fn with_buffer( vm: &VirtualMachine, format_spec: FormatSpec, buffer: ArgBytesLike, - ) -> PyResult<UnpackIterator> { + ) -> PyResult<Self> { if format_spec.size == 0 { Err(new_struct_error( vm, "cannot iteratively unpack with a struct of length 0".to_owned(), )) - } else if buffer.len() % format_spec.size != 0 { + } else if !buffer.len().is_multiple_of(format_spec.size) { Err(new_struct_error( vm, format!( @@ -185,7 +181,7 @@ pub(crate) mod _struct { ), )) } else { - Ok(UnpackIterator { + Ok(Self { format_spec, buffer, offset: AtomicCell::new(0), @@ -194,14 +190,15 @@ pub(crate) mod _struct { } } - #[pyclass(with(IterNext, Iterable))] + #[pyclass(with(IterNext, Iterable), flags(DISALLOW_INSTANTIATION))] impl UnpackIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { + #[pymethod] + fn __length_hint__(&self) -> usize { self.buffer.len().saturating_sub(self.offset.load()) / self.format_spec.size } } impl SelfIter for UnpackIterator {} + impl IterNext for UnpackIterator { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { let size = zelf.format_spec.size; @@ -225,7 +222,7 @@ pub(crate) mod _struct { vm: &VirtualMachine, ) -> PyResult<UnpackIterator> { let format_spec = fmt.format_spec(vm)?; - UnpackIterator::new(vm, format_spec, buffer) + UnpackIterator::with_buffer(vm, format_spec, buffer) } #[pyfunction] @@ -245,16 +242,14 @@ pub(crate) mod _struct { impl Constructor for PyStruct { type Args = IntoStructFormatBytes; - fn py_new(cls: PyTypeRef, fmt: Self::Args, vm: &VirtualMachine) -> PyResult { + fn py_new(_cls: &Py<PyType>, fmt: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { let spec = fmt.format_spec(vm)?; let format = fmt.0; - PyStruct { spec, format } - .into_ref_with_type(vm, cls) - .map(Into::into) + Ok(Self { spec, format }) } } - #[pyclass(with(Constructor))] + #[pyclass(with(Constructor, Representable))] impl PyStruct { #[pygetset] fn format(&self) -> PyStrRef { @@ -263,7 +258,7 @@ pub(crate) mod _struct { #[pygetset] #[inline] - fn size(&self) -> usize { + const fn size(&self) -> usize { self.spec.size } @@ -305,14 +300,21 @@ pub(crate) mod _struct { buffer: ArgBytesLike, vm: &VirtualMachine, ) -> PyResult<UnpackIterator> { - UnpackIterator::new(vm, self.spec.clone(), buffer) + UnpackIterator::with_buffer(vm, self.spec.clone(), buffer) + } + } + + impl Representable for PyStruct { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + Ok(wtf8_concat!("Struct('", zelf.format.as_wtf8(), "')")) } } // seems weird that this is part of the "public" API, but whatever // TODO: implement a format code->spec cache like CPython does? #[pyfunction] - fn _clearcache() {} + const fn _clearcache() {} #[pyattr(name = "error")] fn error_type(vm: &VirtualMachine) -> PyTypeRef { diff --git a/crates/stdlib/src/random.rs b/crates/stdlib/src/random.rs new file mode 100644 index 00000000000..52d45f16b2b --- /dev/null +++ b/crates/stdlib/src/random.rs @@ -0,0 +1,144 @@ +//! Random module. + +pub(crate) use _random::module_def; + +#[pymodule] +mod _random { + use crate::common::lock::PyMutex; + use crate::vm::{ + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyInt, PyTupleRef}, + convert::ToPyException, + function::OptionalOption, + types::{Constructor, Initializer}, + }; + use itertools::Itertools; + use malachite_bigint::{BigInt, BigUint, Sign}; + use mt19937::MT19937; + use num_traits::{Signed, Zero}; + use rand_core::{RngCore, SeedableRng}; + use rustpython_vm::types::DefaultConstructor; + + #[pyattr] + #[pyclass(name = "Random")] + #[derive(Debug, PyPayload, Default)] + struct PyRandom { + rng: PyMutex<MT19937>, + } + + impl DefaultConstructor for PyRandom {} + + impl Initializer for PyRandom { + type Args = OptionalOption; + + fn init(zelf: PyRef<Self>, x: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + zelf.seed(x, vm) + } + } + + #[pyclass(flags(BASETYPE), with(Constructor, Initializer))] + impl PyRandom { + #[pymethod] + fn random(&self) -> f64 { + let mut rng = self.rng.lock(); + mt19937::gen_res53(&mut *rng) + } + + #[pymethod] + fn seed(&self, n: OptionalOption<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { + *self.rng.lock() = match n.flatten() { + Some(n) => { + // Fallback to using hash if object isn't Int-like. + let (_, mut key) = match n.downcast::<PyInt>() { + Ok(n) => n.as_bigint().abs(), + Err(obj) => BigInt::from(obj.hash(vm)?).abs(), + } + .to_u32_digits(); + if cfg!(target_endian = "big") { + key.reverse(); + } + let key = if key.is_empty() { &[0] } else { key.as_slice() }; + MT19937::new_with_slice_seed(key) + } + None => MT19937::try_from_os_rng() + .map_err(|e| std::io::Error::from(e).to_pyexception(vm))?, + }; + Ok(()) + } + + #[pymethod] + fn getrandbits(&self, k: PyObjectRef, vm: &VirtualMachine) -> PyResult<BigInt> { + let k_int = k.try_index(vm)?; + let k_bigint = k_int.as_bigint(); + if k_bigint.is_negative() { + return Err(vm.new_value_error("number of bits must be non-negative")); + } + let k: isize = k_int + .try_to_primitive(vm) + .map_err(|_| vm.new_overflow_error("getrandbits: number of bits too large"))?; + match k { + 0 => Ok(BigInt::zero()), + mut k => { + let mut rng = self.rng.lock(); + let mut gen_u32 = |k| { + let r = rng.next_u32(); + if k < 32 { r >> (32 - k) } else { r } + }; + + let words = (k - 1) / 32 + 1; + let word_array = (0..words) + .map(|_| { + let word = gen_u32(k); + k = k.wrapping_sub(32); + word + }) + .collect::<Vec<_>>(); + + let uint = BigUint::new(word_array); + // very unlikely but might as well check + let sign = if uint.is_zero() { + Sign::NoSign + } else { + Sign::Plus + }; + Ok(BigInt::from_biguint(sign, uint)) + } + } + } + + #[pymethod] + fn getstate(&self, vm: &VirtualMachine) -> PyTupleRef { + let rng = self.rng.lock(); + vm.new_tuple( + rng.get_state() + .iter() + .copied() + .chain([rng.get_index() as u32]) + .map(|i| vm.ctx.new_int(i).into()) + .collect::<Vec<PyObjectRef>>(), + ) + } + + #[pymethod] + fn setstate(&self, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + let state: &[_; mt19937::N + 1] = state + .as_slice() + .try_into() + .map_err(|_| vm.new_value_error("state vector is the wrong size"))?; + let (index, state) = state.split_last().unwrap(); + let index: usize = index.try_to_value(vm)?; + if index > mt19937::N { + return Err(vm.new_value_error("invalid state")); + } + let state: [u32; mt19937::N] = state + .iter() + .map(|i| i.try_to_value(vm)) + .process_results(|it| it.collect_array())? + .unwrap(); + let mut rng = self.rng.lock(); + rng.set_state(&state); + rng.set_index(index); + Ok(()) + } + } +} diff --git a/stdlib/src/re.rs b/crates/stdlib/src/re.rs similarity index 95% rename from stdlib/src/re.rs rename to crates/stdlib/src/re.rs index 5417e03fc72..c72039f10c5 100644 --- a/stdlib/src/re.rs +++ b/crates/stdlib/src/re.rs @@ -1,4 +1,4 @@ -pub(crate) use re::make_module; +pub(crate) use re::module_def; #[pymodule] mod re { @@ -9,10 +9,11 @@ mod re { * system. */ use crate::vm::{ + PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyInt, PyIntRef, PyStr, PyStrRef}, convert::{ToPyObject, TryFromObject}, function::{OptionalArg, PosArgs}, - match_class, PyObjectRef, PyResult, PyPayload, VirtualMachine, + match_class, }; use num_traits::Signed; use regex::bytes::{Captures, Regex, RegexBuilder}; @@ -158,11 +159,9 @@ mod re { } fn do_sub(pattern: &PyPattern, repl: PyStrRef, search_text: PyStrRef, limit: usize) -> String { - let out = pattern.regex.replacen( - search_text.as_str().as_bytes(), - limit, - repl.as_str().as_bytes(), - ); + let out = pattern + .regex + .replacen(search_text.as_bytes(), limit, repl.as_bytes()); String::from_utf8_lossy(&out).into_owned() } @@ -172,21 +171,21 @@ mod re { regex_text.push_str(pattern.regex.as_str()); let regex = Regex::new(&regex_text).unwrap(); regex - .captures(search_text.as_str().as_bytes()) + .captures(search_text.as_bytes()) .map(|captures| create_match(search_text.clone(), captures)) } fn do_search(regex: &PyPattern, search_text: PyStrRef) -> Option<PyMatch> { regex .regex - .captures(search_text.as_str().as_bytes()) + .captures(search_text.as_bytes()) .map(|captures| create_match(search_text.clone(), captures)) } fn do_findall(vm: &VirtualMachine, pattern: &PyPattern, search_text: PyStrRef) -> PyResult { let out = pattern .regex - .captures_iter(search_text.as_str().as_bytes()) + .captures_iter(search_text.as_bytes()) .map(|captures| match captures.len() { 1 => { let full = captures.get(0).unwrap().as_bytes(); @@ -232,7 +231,7 @@ mod re { .map(|i| i.try_to_primitive::<usize>(vm)) .transpose()? .unwrap_or(0); - let text = search_text.as_str().as_bytes(); + let text = search_text.as_bytes(); // essentially Regex::split, but it outputs captures as well let mut output = Vec::new(); let mut last = 0; @@ -266,7 +265,7 @@ mod re { fn make_regex(vm: &VirtualMachine, pattern: &str, flags: PyRegexFlags) -> PyResult<PyPattern> { let unicode = if flags.unicode && flags.ascii { - return Err(vm.new_value_error("ASCII and UNICODE flags are incompatible".to_owned())); + return Err(vm.new_value_error("ASCII and UNICODE flags are incompatible")); } else { !flags.ascii }; @@ -318,7 +317,7 @@ mod re { #[pyfunction] fn purge(_vm: &VirtualMachine) {} - #[pyclass] + #[pyclass(flags(HAS_WEAKREF))] impl PyPattern { #[pymethod(name = "match")] fn match_(&self, text: PyStrRef) -> Option<PyMatch> { @@ -332,9 +331,7 @@ mod re { #[pymethod] fn sub(&self, repl: PyStrRef, text: PyStrRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let replaced_text = self - .regex - .replace_all(text.as_str().as_bytes(), repl.as_str().as_bytes()); + let replaced_text = self.regex.replace_all(text.as_bytes(), repl.as_bytes()); let replaced_text = String::from_utf8_lossy(&replaced_text).into_owned(); Ok(vm.ctx.new_str(replaced_text)) } diff --git a/stdlib/src/resource.rs b/crates/stdlib/src/resource.rs similarity index 85% rename from stdlib/src/resource.rs rename to crates/stdlib/src/resource.rs index 075e191284d..34c8161e0cd 100644 --- a/stdlib/src/resource.rs +++ b/crates/stdlib/src/resource.rs @@ -1,18 +1,21 @@ -pub(crate) use resource::make_module; +// spell-checker:disable + +pub(crate) use resource::module_def; #[pymodule] mod resource { use crate::vm::{ + PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, convert::{ToPyException, ToPyObject}, - stdlib::os, types::PyStructSequence, - PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, }; - use std::{io, mem}; + use core::mem; + use std::io; cfg_if::cfg_if! { if #[cfg(target_os = "android")] { - use libc::RLIM_NLIMITS; + #[expect(deprecated)] + const RLIM_NLIMITS: i32 = libc::RLIM_NLIMITS; } else { // This constant isn't abi-stable across os versions, so we just // pick a high number so we don't get false positive ValueErrors and just bubble up the @@ -24,8 +27,8 @@ mod resource { // TODO: RLIMIT_OFILE, #[pyattr] use libc::{ - RLIMIT_AS, RLIMIT_CORE, RLIMIT_CPU, RLIMIT_DATA, RLIMIT_FSIZE, RLIMIT_MEMLOCK, - RLIMIT_NOFILE, RLIMIT_NPROC, RLIMIT_RSS, RLIMIT_STACK, RLIM_INFINITY, + RLIM_INFINITY, RLIMIT_AS, RLIMIT_CORE, RLIMIT_CPU, RLIMIT_DATA, RLIMIT_FSIZE, + RLIMIT_MEMLOCK, RLIMIT_NOFILE, RLIMIT_NPROC, RLIMIT_RSS, RLIMIT_STACK, }; #[cfg(any(target_os = "linux", target_os = "android", target_os = "emscripten"))] @@ -53,17 +56,15 @@ mod resource { #[pyattr] use libc::RLIMIT_VMEM; - #[cfg(any(target_os = "linux", target_os = "emscripten", target_os = "freebds"))] + #[cfg(any(target_os = "linux", target_os = "emscripten", target_os = "freebsd"))] #[pyattr] use libc::RUSAGE_THREAD; #[cfg(not(any(target_os = "windows", target_os = "redox")))] #[pyattr] use libc::{RUSAGE_CHILDREN, RUSAGE_SELF}; - #[pyattr] - #[pyclass(name = "struct_rusage")] - #[derive(PyStructSequence)] - struct Rusage { + #[pystruct_sequence_data] + struct RUsageData { ru_utime: f64, ru_stime: f64, ru_maxrss: libc::c_long, @@ -82,15 +83,19 @@ mod resource { ru_nivcsw: libc::c_long, } + #[pyattr] + #[pystruct_sequence(name = "struct_rusage", module = "resource", data = "RUsageData")] + struct PyRUsage; + #[pyclass(with(PyStructSequence))] - impl Rusage {} + impl PyRUsage {} - impl From<libc::rusage> for Rusage { + impl From<libc::rusage> for RUsageData { fn from(rusage: libc::rusage) -> Self { let tv = |tv: libc::timeval| tv.tv_sec as f64 + (tv.tv_usec as f64 / 1_000_000.0); - Rusage { + Self { ru_utime: tv(rusage.ru_utime), - ru_stime: tv(rusage.ru_utime), + ru_stime: tv(rusage.ru_stime), ru_maxrss: rusage.ru_maxrss, ru_ixrss: rusage.ru_ixrss, ru_idrss: rusage.ru_idrss, @@ -110,7 +115,7 @@ mod resource { } #[pyfunction] - fn getrusage(who: i32, vm: &VirtualMachine) -> PyResult<Rusage> { + fn getrusage(who: i32, vm: &VirtualMachine) -> PyResult<RUsageData> { let res = unsafe { let mut rusage = mem::MaybeUninit::<libc::rusage>::uninit(); if libc::getrusage(who, rusage.as_mut_ptr()) == -1 { @@ -119,9 +124,9 @@ mod resource { Ok(rusage.assume_init()) } }; - res.map(Rusage::from).map_err(|e| { + res.map(RUsageData::from).map_err(|e| { if e.kind() == io::ErrorKind::InvalidInput { - vm.new_value_error("invalid who parameter".to_owned()) + vm.new_value_error("invalid who parameter") } else { e.to_pyexception(vm) } @@ -137,7 +142,7 @@ mod resource { rlim_cur: cur & RLIM_INFINITY, rlim_max: max & RLIM_INFINITY, })), - _ => Err(vm.new_value_error("expected a tuple of 2 integers".to_owned())), + _ => Err(vm.new_value_error("expected a tuple of 2 integers")), } } } @@ -151,12 +156,12 @@ mod resource { fn getrlimit(resource: i32, vm: &VirtualMachine) -> PyResult<Limits> { #[allow(clippy::unnecessary_cast)] if resource < 0 || resource >= RLIM_NLIMITS as i32 { - return Err(vm.new_value_error("invalid resource specified".to_owned())); + return Err(vm.new_value_error("invalid resource specified")); } let rlimit = unsafe { let mut rlimit = mem::MaybeUninit::<libc::rlimit>::uninit(); if libc::getrlimit(resource as _, rlimit.as_mut_ptr()) == -1 { - return Err(os::errno_err(vm)); + return Err(vm.new_last_errno_error()); } rlimit.assume_init() }; @@ -167,7 +172,7 @@ mod resource { fn setrlimit(resource: i32, limits: Limits, vm: &VirtualMachine) -> PyResult<()> { #[allow(clippy::unnecessary_cast)] if resource < 0 || resource >= RLIM_NLIMITS as i32 { - return Err(vm.new_value_error("invalid resource specified".to_owned())); + return Err(vm.new_value_error("invalid resource specified")); } let res = unsafe { if libc::setrlimit(resource as _, &limits.0) == -1 { @@ -178,10 +183,10 @@ mod resource { }; res.map_err(|e| match e.kind() { io::ErrorKind::InvalidInput => { - vm.new_value_error("current limit exceeds maximum limit".to_owned()) + vm.new_value_error("current limit exceeds maximum limit") } io::ErrorKind::PermissionDenied => { - vm.new_value_error("not allowed to raise maximum limit".to_owned()) + vm.new_value_error("not allowed to raise maximum limit") } _ => e.to_pyexception(vm), }) diff --git a/stdlib/src/scproxy.rs b/crates/stdlib/src/scproxy.rs similarity index 77% rename from stdlib/src/scproxy.rs rename to crates/stdlib/src/scproxy.rs index 7108f50d8fb..09e7cdc6046 100644 --- a/stdlib/src/scproxy.rs +++ b/crates/stdlib/src/scproxy.rs @@ -1,13 +1,13 @@ -pub(crate) use _scproxy::make_module; +pub(crate) use _scproxy::module_def; #[pymodule] mod _scproxy { // straight-forward port of Modules/_scproxy.c use crate::vm::{ - builtins::{PyDictRef, PyStr}, + Py, PyResult, VirtualMachine, + builtins::{PyDict, PyDictRef, PyStr}, convert::ToPyObject, - PyResult, VirtualMachine, }; use system_configuration::core_foundation::{ array::CFArray, @@ -22,7 +22,7 @@ mod _scproxy { fn proxy_dict() -> Option<CFDictionary<CFString, CFType>> { // Py_BEGIN_ALLOW_THREADS - let proxy_dict = unsafe { SCDynamicStoreCopyProxies(std::ptr::null()) }; + let proxy_dict = unsafe { SCDynamicStoreCopyProxies(core::ptr::null()) }; // Py_END_ALLOW_THREADS if proxy_dict.is_null() { None @@ -56,10 +56,7 @@ mod _scproxy { .map(|s| { unsafe { CFType::from_void(*s) } .downcast::<CFString>() - .map(|s| { - let a_string: std::borrow::Cow<str> = (&s).into(); - PyStr::from(a_string.into_owned()) - }) + .map(|s| PyStr::from(s.to_string())) .to_pyobject(vm) }) .collect(); @@ -77,7 +74,7 @@ mod _scproxy { let result = vm.ctx.new_dict(); - let set_proxy = |result: &PyDictRef, + let set_proxy = |result: &Py<PyDict>, proto: &str, enabled_key: CFStringRef, host_key: CFStringRef, @@ -89,23 +86,22 @@ mod _scproxy { .and_then(|v| v.downcast::<CFNumber>()) .and_then(|v| v.to_i32()) .unwrap_or(0); - if enabled { - if let Some(host) = proxy_dict + if enabled + && let Some(host) = proxy_dict .find(host_key) .and_then(|v| v.downcast::<CFString>()) + { + let h = alloc::borrow::Cow::<str>::from(&host); + let v = if let Some(port) = proxy_dict + .find(port_key) + .and_then(|v| v.downcast::<CFNumber>()) + .and_then(|v| v.to_i32()) { - let h = std::borrow::Cow::<str>::from(&host); - let v = if let Some(port) = proxy_dict - .find(port_key) - .and_then(|v| v.downcast::<CFNumber>()) - .and_then(|v| v.to_i32()) - { - format!("http://{h}:{port}") - } else { - format!("http://{h}") - }; - result.set_item(proto, vm.new_pyobj(v), vm)?; - } + format!("http://{h}:{port}") + } else { + format!("http://{h}") + }; + result.set_item(proto, vm.new_pyobj(v), vm)?; } Ok(()) }; diff --git a/crates/stdlib/src/select.rs b/crates/stdlib/src/select.rs new file mode 100644 index 00000000000..b52144247f7 --- /dev/null +++ b/crates/stdlib/src/select.rs @@ -0,0 +1,745 @@ +// spell-checker:disable + +pub(crate) use decl::module_def; + +use crate::vm::{ + PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, +}; +use core::mem; +use std::io; + +#[cfg(unix)] +mod platform { + pub use libc::{FD_ISSET, FD_SET, FD_SETSIZE, FD_ZERO, fd_set, select, timeval}; + pub use std::os::unix::io::RawFd; + + pub const fn check_err(x: i32) -> bool { + x < 0 + } +} + +#[allow(non_snake_case)] +#[cfg(windows)] +mod platform { + pub use WinSock::{FD_SET as fd_set, FD_SETSIZE, SOCKET as RawFd, TIMEVAL as timeval, select}; + use windows_sys::Win32::Networking::WinSock; + + // based off winsock2.h: https://gist.github.com/piscisaureus/906386#file-winsock2-h-L128-L141 + + pub unsafe fn FD_SET(fd: RawFd, set: *mut fd_set) { + unsafe { + let mut slot = (&raw mut (*set).fd_array).cast::<RawFd>(); + let fd_count = (*set).fd_count; + for _ in 0..fd_count { + if *slot == fd { + return; + } + slot = slot.add(1); + } + // slot == &fd_array[fd_count] at this point + if fd_count < FD_SETSIZE { + *slot = fd as RawFd; + (*set).fd_count += 1; + } + } + } + + pub unsafe fn FD_ZERO(set: *mut fd_set) { + unsafe { (*set).fd_count = 0 }; + } + + pub unsafe fn FD_ISSET(fd: RawFd, set: *mut fd_set) -> bool { + use WinSock::__WSAFDIsSet; + unsafe { __WSAFDIsSet(fd as _, set) != 0 } + } + + pub fn check_err(x: i32) -> bool { + x == WinSock::SOCKET_ERROR + } +} + +#[cfg(target_os = "wasi")] +mod platform { + pub use libc::{FD_SETSIZE, timeval}; + pub use std::os::fd::RawFd; + + pub fn check_err(x: i32) -> bool { + x < 0 + } + + #[repr(C)] + pub struct fd_set { + __nfds: usize, + __fds: [libc::c_int; FD_SETSIZE], + } + + #[allow(non_snake_case)] + pub unsafe fn FD_ISSET(fd: RawFd, set: *const fd_set) -> bool { + let set = unsafe { &*set }; + let n = set.__nfds; + for p in &set.__fds[..n] { + if *p == fd { + return true; + } + } + false + } + + #[allow(non_snake_case)] + pub unsafe fn FD_SET(fd: RawFd, set: *mut fd_set) { + let set = unsafe { &mut *set }; + let n = set.__nfds; + for p in &set.__fds[..n] { + if *p == fd { + return; + } + } + set.__nfds = n + 1; + set.__fds[n] = fd; + } + + #[allow(non_snake_case)] + pub unsafe fn FD_ZERO(set: *mut fd_set) { + let set = unsafe { &mut *set }; + set.__nfds = 0; + } + + unsafe extern "C" { + pub fn select( + nfds: libc::c_int, + readfds: *mut fd_set, + writefds: *mut fd_set, + errorfds: *mut fd_set, + timeout: *const timeval, + ) -> libc::c_int; + } +} + +use platform::RawFd; +pub use platform::timeval; + +#[derive(Traverse)] +struct Selectable { + obj: PyObjectRef, + #[pytraverse(skip)] + fno: RawFd, +} + +impl TryFromObject for Selectable { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let fno = obj.try_to_value(vm).or_else(|_| { + let meth = vm.get_method_or_type_error( + obj.clone(), + vm.ctx.interned_str("fileno").unwrap(), + || "select arg must be an int or object with a fileno() method".to_owned(), + )?; + meth.call((), vm)?.try_into_value(vm) + })?; + Ok(Self { obj, fno }) + } +} + +// Keep it in a MaybeUninit, since on windows FD_ZERO doesn't actually zero the whole thing +#[repr(transparent)] +pub struct FdSet(mem::MaybeUninit<platform::fd_set>); + +impl FdSet { + pub fn new() -> Self { + // it's just ints, and all the code that's actually + // interacting with it is in C, so it's safe to zero + let mut fdset = core::mem::MaybeUninit::zeroed(); + unsafe { platform::FD_ZERO(fdset.as_mut_ptr()) }; + Self(fdset) + } + + pub fn insert(&mut self, fd: RawFd) { + unsafe { platform::FD_SET(fd, self.0.as_mut_ptr()) }; + } + + pub fn contains(&mut self, fd: RawFd) -> bool { + unsafe { platform::FD_ISSET(fd, self.0.as_mut_ptr()) } + } + + pub fn clear(&mut self) { + unsafe { platform::FD_ZERO(self.0.as_mut_ptr()) }; + } + + pub fn highest(&mut self) -> Option<RawFd> { + (0..platform::FD_SETSIZE as RawFd) + .rev() + .find(|&i| self.contains(i)) + } +} + +pub fn select( + nfds: libc::c_int, + readfds: &mut FdSet, + writefds: &mut FdSet, + errfds: &mut FdSet, + timeout: Option<&mut timeval>, +) -> io::Result<i32> { + let timeout = match timeout { + Some(tv) => tv as *mut timeval, + None => core::ptr::null_mut(), + }; + let ret = unsafe { + platform::select( + nfds, + readfds.0.as_mut_ptr(), + writefds.0.as_mut_ptr(), + errfds.0.as_mut_ptr(), + timeout, + ) + }; + if platform::check_err(ret) { + Err(io::Error::last_os_error()) + } else { + Ok(ret) + } +} + +fn sec_to_timeval(sec: f64) -> timeval { + timeval { + tv_sec: sec.trunc() as _, + tv_usec: (sec.fract() * 1e6) as _, + } +} + +#[pymodule(name = "select")] +mod decl { + use super::*; + use crate::vm::{ + Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyModule, PyTypeRef}, + convert::ToPyException, + function::{Either, OptionalOption}, + stdlib::time, + }; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + #[cfg(windows)] + crate::vm::windows::init_winsock(); + + #[cfg(unix)] + { + use crate::vm::class::PyClassImpl; + poll::PyPoll::make_static_type(); + } + + __module_exec(vm, module); + Ok(()) + } + + #[pyattr] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.exceptions.os_error.to_owned() + } + + #[pyfunction] + fn select( + rlist: PyObjectRef, + wlist: PyObjectRef, + xlist: PyObjectRef, + timeout: OptionalOption<Either<f64, isize>>, + vm: &VirtualMachine, + ) -> PyResult<(PyListRef, PyListRef, PyListRef)> { + let mut timeout = timeout.flatten().map(|e| match e { + Either::A(f) => f, + Either::B(i) => i as f64, + }); + if let Some(timeout) = timeout + && timeout < 0.0 + { + return Err(vm.new_value_error("timeout must be positive")); + } + let deadline = timeout.map(|s| time::time(vm).unwrap() + s); + + let seq2set = |list: &PyObject| -> PyResult<(Vec<Selectable>, FdSet)> { + let v: Vec<Selectable> = list.try_to_value(vm)?; + let mut fds = FdSet::new(); + for fd in &v { + fds.insert(fd.fno); + } + Ok((v, fds)) + }; + + let (rlist, mut r) = seq2set(&rlist)?; + let (wlist, mut w) = seq2set(&wlist)?; + let (xlist, mut x) = seq2set(&xlist)?; + + if rlist.is_empty() && wlist.is_empty() && xlist.is_empty() { + let empty = vm.ctx.new_list(vec![]); + return Ok((empty.clone(), empty.clone(), empty)); + } + + let nfds: i32 = [&mut r, &mut w, &mut x] + .iter_mut() + .filter_map(|set| set.highest()) + .max() + .map_or(0, |n| n + 1) as _; + + loop { + let mut tv = timeout.map(sec_to_timeval); + let res = vm.allow_threads(|| super::select(nfds, &mut r, &mut w, &mut x, tv.as_mut())); + + match res { + Ok(_) => break, + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) => return Err(err.to_pyexception(vm)), + } + + vm.check_signals()?; + + if let Some(ref mut timeout) = timeout { + *timeout = deadline.unwrap() - time::time(vm).unwrap(); + if *timeout < 0.0 { + r.clear(); + w.clear(); + x.clear(); + break; + } + // retry select() if we haven't reached the deadline yet + } + } + + let set2list = |list: Vec<Selectable>, mut set: FdSet| { + vm.ctx.new_list( + list.into_iter() + .filter(|fd| set.contains(fd.fno)) + .map(|fd| fd.obj) + .collect(), + ) + }; + + let rlist = set2list(rlist, r); + let wlist = set2list(wlist, w); + let xlist = set2list(xlist, x); + + Ok((rlist, wlist, xlist)) + } + + #[cfg(unix)] + #[pyfunction] + fn poll() -> poll::PyPoll { + poll::PyPoll::default() + } + + #[cfg(unix)] + #[pyattr] + use libc::{POLLERR, POLLHUP, POLLIN, POLLNVAL, POLLOUT, POLLPRI}; + + #[cfg(unix)] + pub(super) mod poll { + use super::*; + use crate::vm::{ + AsObject, PyPayload, + builtins::PyFloat, + common::lock::PyMutex, + convert::{IntoPyException, ToPyObject}, + function::OptionalArg, + stdlib::_io::Fildes, + }; + use core::{convert::TryFrom, time::Duration}; + use libc::pollfd; + use num_traits::{Signed, ToPrimitive}; + use std::time::Instant; + + #[derive(Default)] + pub(super) struct TimeoutArg<const MILLIS: bool>(pub Option<Duration>); + + impl<const MILLIS: bool> TryFromObject for TimeoutArg<MILLIS> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let timeout = if vm.is_none(&obj) { + None + } else if let Some(float) = obj.downcast_ref::<PyFloat>() { + let float = float.to_f64(); + if float.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)")); + } + if float.is_sign_negative() { + None + } else { + let secs = if MILLIS { float * 1000.0 } else { float }; + Some(Duration::from_secs_f64(secs)) + } + } else if let Some(int) = obj.try_index_opt(vm).transpose()? { + if int.as_bigint().is_negative() { + None + } else { + let n = int + .as_bigint() + .to_u64() + .ok_or_else(|| vm.new_overflow_error("value out of range"))?; + Some(if MILLIS { + Duration::from_millis(n) + } else { + Duration::from_secs(n) + }) + } + } else { + return Err(vm.new_type_error(format!( + "expected an int or float for duration, got {}", + obj.class() + ))); + }; + Ok(Self(timeout)) + } + } + + #[pyclass(module = "select", name = "poll")] + #[derive(Default, Debug, PyPayload)] + pub struct PyPoll { + // keep sorted + fds: PyMutex<Vec<pollfd>>, + } + + #[inline] + fn search(fds: &[pollfd], fd: i32) -> Result<usize, usize> { + fds.binary_search_by_key(&fd, |pfd| pfd.fd) + } + + fn insert_fd(fds: &mut Vec<pollfd>, fd: i32, events: i16) { + match search(fds, fd) { + Ok(i) => fds[i].events = events, + Err(i) => fds.insert( + i, + pollfd { + fd, + events, + revents: 0, + }, + ), + } + } + + fn get_fd_mut(fds: &mut [pollfd], fd: i32) -> Option<&mut pollfd> { + search(fds, fd).ok().map(move |i| &mut fds[i]) + } + + fn remove_fd(fds: &mut Vec<pollfd>, fd: i32) -> Option<pollfd> { + search(fds, fd).ok().map(|i| fds.remove(i)) + } + + // new EventMask type + #[derive(Copy, Clone)] + #[repr(transparent)] + pub struct EventMask(pub i16); + + impl TryFromObject for EventMask { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + use crate::builtins::PyInt; + let int = obj + .downcast::<PyInt>() + .map_err(|_| vm.new_type_error("argument must be an integer"))?; + + let val = int.as_bigint(); + if val.is_negative() { + return Err(vm.new_value_error("negative event mask")); + } + + // Try converting to i16, should raise OverflowError if too large + let mask = i16::try_from(val) + .map_err(|_| vm.new_overflow_error("event mask value out of range"))?; + + Ok(Self(mask)) + } + } + + const DEFAULT_EVENTS: i16 = libc::POLLIN | libc::POLLPRI | libc::POLLOUT; + + #[pyclass] + impl PyPoll { + #[pymethod] + fn register( + &self, + Fildes(fd): Fildes, + eventmask: OptionalArg<EventMask>, + ) -> PyResult<()> { + let mask = match eventmask { + OptionalArg::Present(event_mask) => event_mask.0, + OptionalArg::Missing => DEFAULT_EVENTS, + }; + insert_fd(&mut self.fds.lock(), fd, mask); + Ok(()) + } + + #[pymethod] + fn modify( + &self, + Fildes(fd): Fildes, + eventmask: EventMask, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut fds = self.fds.lock(); + // CPython raises KeyError if fd is not registered, match that behavior + let pfd = get_fd_mut(&mut fds, fd) + .ok_or_else(|| vm.new_key_error(vm.ctx.new_int(fd).into()))?; + pfd.events = eventmask.0; + Ok(()) + } + + #[pymethod] + fn unregister(&self, Fildes(fd): Fildes, vm: &VirtualMachine) -> PyResult<()> { + let removed = remove_fd(&mut self.fds.lock(), fd); + removed + .map(drop) + .ok_or_else(|| vm.new_key_error(vm.ctx.new_int(fd).into())) + } + + #[pymethod] + fn poll( + &self, + timeout: OptionalArg<TimeoutArg<true>>, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + let mut fds = self.fds.lock(); + let TimeoutArg(timeout) = timeout.unwrap_or_default(); + let timeout_ms = match timeout { + Some(d) => i32::try_from(d.as_millis()) + .map_err(|_| vm.new_overflow_error("value out of range"))?, + None => -1i32, + }; + let deadline = timeout.map(|d| Instant::now() + d); + let mut poll_timeout = timeout_ms; + loop { + let res = vm.allow_threads(|| unsafe { + libc::poll(fds.as_mut_ptr(), fds.len() as _, poll_timeout) + }); + match nix::Error::result(res) { + Ok(_) => break, + Err(nix::Error::EINTR) => vm.check_signals()?, + Err(e) => return Err(e.into_pyexception(vm)), + } + if let Some(d) = deadline { + if let Some(remaining) = d.checked_duration_since(Instant::now()) { + poll_timeout = remaining.as_millis() as i32; + } else { + break; + } + } + } + Ok(fds + .iter() + .filter(|pfd| pfd.revents != 0) + .map(|pfd| (pfd.fd, pfd.revents & 0xfff).to_pyobject(vm)) + .collect()) + } + } + } + + #[cfg(any(target_os = "linux", target_os = "android", target_os = "redox"))] + #[pyattr(name = "epoll", once)] + fn epoll(_vm: &VirtualMachine) -> PyTypeRef { + use crate::vm::class::PyClassImpl; + epoll::PyEpoll::make_static_type() + } + + #[cfg(any(target_os = "linux", target_os = "android", target_os = "redox"))] + #[pyattr] + use libc::{ + EPOLL_CLOEXEC, EPOLLERR, EPOLLEXCLUSIVE, EPOLLHUP, EPOLLIN, EPOLLMSG, EPOLLONESHOT, + EPOLLOUT, EPOLLPRI, EPOLLRDBAND, EPOLLRDHUP, EPOLLRDNORM, EPOLLWAKEUP, EPOLLWRBAND, + EPOLLWRNORM, + }; + #[cfg(any(target_os = "linux", target_os = "android", target_os = "redox"))] + #[pyattr] + const EPOLLET: u32 = libc::EPOLLET as u32; + + #[cfg(any(target_os = "linux", target_os = "android", target_os = "redox"))] + pub(super) mod epoll { + use super::*; + use crate::vm::{ + Py, PyPayload, PyRef, + builtins::PyType, + common::lock::{PyRwLock, PyRwLockReadGuard}, + convert::{IntoPyException, ToPyObject}, + function::OptionalArg, + stdlib::_io::Fildes, + types::Constructor, + }; + use core::ops::Deref; + use rustix::event::epoll::{self, EventData, EventFlags}; + use std::os::fd::{AsRawFd, IntoRawFd, OwnedFd}; + use std::time::Instant; + + #[pyclass(module = "select", name = "epoll")] + #[derive(Debug, rustpython_vm::PyPayload)] + pub struct PyEpoll { + epoll_fd: PyRwLock<Option<OwnedFd>>, + } + + #[derive(FromArgs)] + pub struct EpollNewArgs { + #[pyarg(any, default = -1)] + sizehint: i32, + #[pyarg(any, default = 0)] + flags: i32, + } + + impl Constructor for PyEpoll { + type Args = EpollNewArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if let ..=-2 | 0 = args.sizehint { + return Err(vm.new_value_error("negative sizehint")); + } + if !matches!(args.flags, 0 | libc::EPOLL_CLOEXEC) { + return Err(vm.new_os_error("invalid flags".to_owned())); + } + Self::new().map_err(|e| e.into_pyexception(vm)) + } + } + + #[derive(FromArgs)] + struct EpollPollArgs { + #[pyarg(any, default)] + timeout: poll::TimeoutArg<false>, + #[pyarg(any, default = -1)] + maxevents: i32, + } + + #[pyclass(with(Constructor))] + impl PyEpoll { + fn new() -> std::io::Result<Self> { + let epoll_fd = epoll::create(epoll::CreateFlags::CLOEXEC)?; + let epoll_fd = Some(epoll_fd).into(); + Ok(Self { epoll_fd }) + } + + #[pymethod] + fn close(&self) -> std::io::Result<()> { + let fd = self.epoll_fd.write().take(); + if let Some(fd) = fd { + nix::unistd::close(fd.into_raw_fd())?; + } + Ok(()) + } + + #[pygetset] + fn closed(&self) -> bool { + self.epoll_fd.read().is_none() + } + + fn get_epoll( + &self, + vm: &VirtualMachine, + ) -> PyResult<impl Deref<Target = OwnedFd> + '_> { + PyRwLockReadGuard::try_map(self.epoll_fd.read(), |x| x.as_ref()) + .map_err(|_| vm.new_value_error("I/O operation on closed epoll object")) + } + + #[pymethod] + fn fileno(&self, vm: &VirtualMachine) -> PyResult<i32> { + self.get_epoll(vm).map(|epoll_fd| epoll_fd.as_raw_fd()) + } + + #[pyclassmethod] + fn fromfd(cls: PyTypeRef, fd: OwnedFd, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let epoll_fd = Some(fd).into(); + Self { epoll_fd }.into_ref_with_type(vm, cls) + } + + #[pymethod] + fn register( + &self, + fd: Fildes, + eventmask: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let events = match eventmask { + OptionalArg::Present(mask) => EventFlags::from_bits_retain(mask), + OptionalArg::Missing => EventFlags::IN | EventFlags::PRI | EventFlags::OUT, + }; + let epoll_fd = &*self.get_epoll(vm)?; + let data = EventData::new_u64(fd.as_raw_fd() as u64); + epoll::add(epoll_fd, fd, data, events).map_err(|e| e.into_pyexception(vm)) + } + + #[pymethod] + fn modify(&self, fd: Fildes, eventmask: u32, vm: &VirtualMachine) -> PyResult<()> { + let events = EventFlags::from_bits_retain(eventmask); + let epoll_fd = &*self.get_epoll(vm)?; + let data = EventData::new_u64(fd.as_raw_fd() as u64); + epoll::modify(epoll_fd, fd, data, events).map_err(|e| e.into_pyexception(vm)) + } + + #[pymethod] + fn unregister(&self, fd: Fildes, vm: &VirtualMachine) -> PyResult<()> { + let epoll_fd = &*self.get_epoll(vm)?; + epoll::delete(epoll_fd, fd).map_err(|e| e.into_pyexception(vm)) + } + + #[pymethod] + fn poll(&self, args: EpollPollArgs, vm: &VirtualMachine) -> PyResult<PyListRef> { + let poll::TimeoutArg(timeout) = args.timeout; + let maxevents = args.maxevents; + + let mut poll_timeout = + timeout + .map(rustix::event::Timespec::try_from) + .transpose() + .map_err(|_| vm.new_overflow_error("timeout is too large"))?; + + let deadline = timeout.map(|d| Instant::now() + d); + let maxevents = match maxevents { + ..-1 => { + return Err(vm.new_value_error(format!( + "maxevents must be greater than 0, got {maxevents}" + ))); + } + -1 => libc::FD_SETSIZE - 1, + _ => maxevents as usize, + }; + + let mut events = Vec::<epoll::Event>::with_capacity(maxevents); + + let epoll = &*self.get_epoll(vm)?; + + loop { + events.clear(); + match vm.allow_threads(|| { + epoll::wait( + epoll, + rustix::buffer::spare_capacity(&mut events), + poll_timeout.as_ref(), + ) + }) { + Ok(_) => break, + Err(rustix::io::Errno::INTR) => vm.check_signals()?, + Err(e) => return Err(e.into_pyexception(vm)), + } + if let Some(deadline) = deadline { + if let Some(new_timeout) = deadline.checked_duration_since(Instant::now()) { + poll_timeout = Some(new_timeout.try_into().unwrap()); + } else { + break; + } + } + } + + let ret = events + .iter() + .map(|ev| (ev.data.u64() as i32, { ev.flags }.bits()).to_pyobject(vm)) + .collect(); + + Ok(vm.ctx.new_list(ret)) + } + + #[pymethod] + fn __enter__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + zelf.get_epoll(vm)?; + Ok(zelf) + } + + #[pymethod] + fn __exit__( + &self, + _exc_type: OptionalArg, + _exc_value: OptionalArg, + _exc_tb: OptionalArg, + ) -> std::io::Result<()> { + self.close() + } + } + } +} diff --git a/crates/stdlib/src/sha1.rs b/crates/stdlib/src/sha1.rs new file mode 100644 index 00000000000..3e3d4928c79 --- /dev/null +++ b/crates/stdlib/src/sha1.rs @@ -0,0 +1,12 @@ +pub(crate) use _sha1::module_def; + +#[pymodule] +mod _sha1 { + use crate::hashlib::_hashlib::{HashArgs, local_sha1}; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; + + #[pyfunction] + fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha1(args, vm)?.into_pyobject(vm)) + } +} diff --git a/crates/stdlib/src/sha256.rs b/crates/stdlib/src/sha256.rs new file mode 100644 index 00000000000..b4c26dc0dd6 --- /dev/null +++ b/crates/stdlib/src/sha256.rs @@ -0,0 +1,23 @@ +#[pymodule] +mod _sha256 { + use crate::hashlib::_hashlib::{HashArgs, local_sha224, local_sha256}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; + + #[pyfunction] + fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha224(args, vm)?.into_pyobject(vm)) + } + + #[pyfunction] + fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha256(args, vm)?.into_pyobject(vm)) + } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } +} + +pub(crate) use _sha256::module_def; diff --git a/crates/stdlib/src/sha3.rs b/crates/stdlib/src/sha3.rs new file mode 100644 index 00000000000..0eb2dfa84d5 --- /dev/null +++ b/crates/stdlib/src/sha3.rs @@ -0,0 +1,40 @@ +pub(crate) use _sha3::module_def; + +#[pymodule] +mod _sha3 { + use crate::hashlib::_hashlib::{ + HashArgs, local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128, + local_shake_256, + }; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; + + #[pyfunction] + fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha3_224(args, vm)?.into_pyobject(vm)) + } + + #[pyfunction] + fn sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha3_256(args, vm)?.into_pyobject(vm)) + } + + #[pyfunction] + fn sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha3_384(args, vm)?.into_pyobject(vm)) + } + + #[pyfunction] + fn sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha3_512(args, vm)?.into_pyobject(vm)) + } + + #[pyfunction] + fn shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_shake_128(args, vm)?.into_pyobject(vm)) + } + + #[pyfunction] + fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_shake_256(args, vm)?.into_pyobject(vm)) + } +} diff --git a/crates/stdlib/src/sha512.rs b/crates/stdlib/src/sha512.rs new file mode 100644 index 00000000000..b7c6f02ed66 --- /dev/null +++ b/crates/stdlib/src/sha512.rs @@ -0,0 +1,23 @@ +#[pymodule] +mod _sha512 { + use crate::hashlib::_hashlib::{HashArgs, local_sha384, local_sha512}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; + + #[pyfunction] + fn sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha384(args, vm)?.into_pyobject(vm)) + } + + #[pyfunction] + fn sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha512(args, vm)?.into_pyobject(vm)) + } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } +} + +pub(crate) use _sha512::module_def; diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs new file mode 100644 index 00000000000..d3fe59144ef --- /dev/null +++ b/crates/stdlib/src/socket.rs @@ -0,0 +1,3498 @@ +// spell-checker:disable + +pub(crate) use _socket::module_def; + +#[cfg(feature = "ssl")] +pub(super) use _socket::{PySocket, SelectKind, sock_select, timeout_error_msg}; + +#[pymodule] +mod _socket { + use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard}; + use crate::vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyListRef, PyModule, PyOSError, PyStrRef, PyTupleRef, PyTypeRef, + PyUtf8StrRef, + }, + common::os::ErrorExt, + convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject}, + function::{ + ArgBytesLike, ArgIntoFloat, ArgMemoryBuffer, ArgStrOrBytesLike, Either, FsPath, + OptionalArg, OptionalOption, + }, + types::{Constructor, DefaultConstructor, Destructor, Initializer, Representable}, + utils::ToCString, + }; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + #[cfg(windows)] + crate::vm::windows::init_winsock(); + + __module_exec(vm, module); + Ok(()) + } + use core::{ + mem::MaybeUninit, + net::{Ipv4Addr, Ipv6Addr, SocketAddr}, + time::Duration, + }; + use crossbeam_utils::atomic::AtomicCell; + use num_traits::ToPrimitive; + use socket2::Socket; + use std::{ + ffi, + io::{self, Read, Write}, + net::{self, Shutdown, ToSocketAddrs}, + time::Instant, + }; + + #[cfg(unix)] + use libc as c; + #[cfg(windows)] + mod c { + pub use windows_sys::Win32::NetworkManagement::IpHelper::{if_indextoname, if_nametoindex}; + pub use windows_sys::Win32::Networking::WinSock::{ + INADDR_ANY, INADDR_BROADCAST, INADDR_LOOPBACK, INADDR_NONE, + }; + + pub use windows_sys::Win32::Networking::WinSock::{ + AF_APPLETALK, AF_DECnet, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, + AI_NUMERICSERV, AI_V4MAPPED, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_HDRINCL, + IP_MULTICAST_IF, IP_MULTICAST_LOOP, IP_MULTICAST_TTL, IP_OPTIONS, IP_RECVDSTADDR, + IP_TOS, IP_TTL, IPPORT_RESERVED, IPPROTO_AH, IPPROTO_DSTOPTS, IPPROTO_EGP, IPPROTO_ESP, + IPPROTO_FRAGMENT, IPPROTO_GGP, IPPROTO_HOPOPTS, IPPROTO_ICMP, IPPROTO_ICMPV6, + IPPROTO_IDP, IPPROTO_IGMP, IPPROTO_IP, IPPROTO_IP as IPPROTO_IPIP, IPPROTO_IPV4, + IPPROTO_IPV6, IPPROTO_ND, IPPROTO_NONE, IPPROTO_PIM, IPPROTO_PUP, IPPROTO_RAW, + IPPROTO_ROUTING, IPPROTO_TCP, IPPROTO_UDP, IPV6_CHECKSUM, IPV6_DONTFRAG, IPV6_HOPLIMIT, + IPV6_HOPOPTS, IPV6_JOIN_GROUP, IPV6_LEAVE_GROUP, IPV6_MULTICAST_HOPS, + IPV6_MULTICAST_IF, IPV6_MULTICAST_LOOP, IPV6_PKTINFO, IPV6_RECVRTHDR, IPV6_RECVTCLASS, + IPV6_RTHDR, IPV6_TCLASS, IPV6_UNICAST_HOPS, IPV6_V6ONLY, MSG_BCAST, MSG_CTRUNC, + MSG_DONTROUTE, MSG_MCAST, MSG_OOB, MSG_PEEK, MSG_TRUNC, MSG_WAITALL, NI_DGRAM, + NI_MAXHOST, NI_MAXSERV, NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, + RCVALL_IPLEVEL, RCVALL_OFF, RCVALL_ON, RCVALL_SOCKETLEVELONLY, SD_BOTH as SHUT_RDWR, + SD_RECEIVE as SHUT_RD, SD_SEND as SHUT_WR, SIO_KEEPALIVE_VALS, SIO_LOOPBACK_FAST_PATH, + SIO_RCVALL, SO_BROADCAST, SO_ERROR, SO_KEEPALIVE, SO_LINGER, SO_OOBINLINE, SO_RCVBUF, + SO_REUSEADDR, SO_SNDBUF, SO_TYPE, SO_USELOOPBACK, SOCK_DGRAM, SOCK_RAW, SOCK_RDM, + SOCK_SEQPACKET, SOCK_STREAM, SOL_SOCKET, SOMAXCONN, TCP_NODELAY, WSAEBADF, + WSAECONNRESET, WSAENOTSOCK, WSAEWOULDBLOCK, + }; + pub use windows_sys::Win32::Networking::WinSock::{ + INVALID_SOCKET, SOCKET_ERROR, WSA_FLAG_OVERLAPPED, WSADuplicateSocketW, + WSAGetLastError, WSAIoctl, WSAPROTOCOL_INFOW, WSASocketW, + }; + pub use windows_sys::Win32::Networking::WinSock::{ + SO_REUSEADDR as SO_EXCLUSIVEADDRUSE, getprotobyname, getservbyname, getservbyport, + getsockopt, setsockopt, + }; + pub use windows_sys::Win32::Networking::WinSock::{ + WSA_NOT_ENOUGH_MEMORY as EAI_MEMORY, WSAEAFNOSUPPORT as EAI_FAMILY, + WSAEINVAL as EAI_BADFLAGS, WSAESOCKTNOSUPPORT as EAI_SOCKTYPE, + WSAHOST_NOT_FOUND as EAI_NODATA, WSAHOST_NOT_FOUND as EAI_NONAME, + WSANO_RECOVERY as EAI_FAIL, WSATRY_AGAIN as EAI_AGAIN, + WSATYPE_NOT_FOUND as EAI_SERVICE, + }; + pub const IF_NAMESIZE: usize = + windows_sys::Win32::NetworkManagement::Ndis::IF_MAX_STRING_SIZE as _; + pub const AF_UNSPEC: i32 = windows_sys::Win32::Networking::WinSock::AF_UNSPEC as _; + pub const AF_INET: i32 = windows_sys::Win32::Networking::WinSock::AF_INET as _; + pub const AF_INET6: i32 = windows_sys::Win32::Networking::WinSock::AF_INET6 as _; + pub const AI_PASSIVE: i32 = windows_sys::Win32::Networking::WinSock::AI_PASSIVE as _; + pub const AI_NUMERICHOST: i32 = + windows_sys::Win32::Networking::WinSock::AI_NUMERICHOST as _; + pub const FROM_PROTOCOL_INFO: i32 = -1; + } + // constants + #[pyattr(name = "has_ipv6")] + const HAS_IPV6: bool = true; + #[pyattr] + // put IPPROTO_MAX later + use c::{ + AF_INET, AF_INET6, AF_UNSPEC, INADDR_ANY, INADDR_LOOPBACK, INADDR_NONE, IPPROTO_ICMP, + IPPROTO_ICMPV6, IPPROTO_IP, IPPROTO_IPV6, IPPROTO_TCP, IPPROTO_TCP as SOL_TCP, IPPROTO_UDP, + MSG_CTRUNC, MSG_DONTROUTE, MSG_OOB, MSG_PEEK, MSG_TRUNC, MSG_WAITALL, NI_DGRAM, NI_MAXHOST, + NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, SHUT_RD, SHUT_RDWR, SHUT_WR, + SO_BROADCAST, SO_ERROR, SO_KEEPALIVE, SO_LINGER, SO_OOBINLINE, SO_RCVBUF, SO_REUSEADDR, + SO_SNDBUF, SO_TYPE, SOCK_DGRAM, SOCK_STREAM, SOL_SOCKET, TCP_NODELAY, + }; + + #[cfg(not(target_os = "redox"))] + #[pyattr] + use c::{ + AF_APPLETALK, AF_DECnet, AF_IPX, IPPROTO_AH, IPPROTO_DSTOPTS, IPPROTO_EGP, IPPROTO_ESP, + IPPROTO_FRAGMENT, IPPROTO_HOPOPTS, IPPROTO_IDP, IPPROTO_IGMP, IPPROTO_IPIP, IPPROTO_NONE, + IPPROTO_PIM, IPPROTO_PUP, IPPROTO_RAW, IPPROTO_ROUTING, + }; + + #[cfg(unix)] + #[pyattr] + use c::{AF_UNIX, SO_REUSEPORT}; + + #[pyattr] + use c::{AI_ADDRCONFIG, AI_NUMERICHOST, AI_NUMERICSERV, AI_PASSIVE}; + + #[cfg(not(target_os = "redox"))] + #[pyattr] + use c::{SOCK_RAW, SOCK_RDM, SOCK_SEQPACKET}; + + #[cfg(target_os = "android")] + #[pyattr] + use c::{SOL_ATALK, SOL_AX25, SOL_IPX, SOL_NETROM, SOL_ROSE}; + + #[cfg(target_os = "freebsd")] + #[pyattr] + use c::SO_SETFIB; + + #[cfg(target_os = "linux")] + #[pyattr] + use c::{ + CAN_BCM, CAN_EFF_FLAG, CAN_EFF_MASK, CAN_ERR_FLAG, CAN_ERR_MASK, CAN_ISOTP, CAN_J1939, + CAN_RAW, CAN_RAW_ERR_FILTER, CAN_RAW_FD_FRAMES, CAN_RAW_FILTER, CAN_RAW_JOIN_FILTERS, + CAN_RAW_LOOPBACK, CAN_RAW_RECV_OWN_MSGS, CAN_RTR_FLAG, CAN_SFF_MASK, IPPROTO_MPTCP, + J1939_IDLE_ADDR, J1939_MAX_UNICAST_ADDR, J1939_NLA_BYTES_ACKED, J1939_NLA_PAD, + J1939_NO_ADDR, J1939_NO_NAME, J1939_NO_PGN, J1939_PGN_ADDRESS_CLAIMED, + J1939_PGN_ADDRESS_COMMANDED, J1939_PGN_MAX, J1939_PGN_PDU1_MAX, J1939_PGN_REQUEST, + SCM_J1939_DEST_ADDR, SCM_J1939_DEST_NAME, SCM_J1939_ERRQUEUE, SCM_J1939_PRIO, + SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, SO_J1939_SEND_PRIO, SOL_CAN_BASE, + SOL_CAN_RAW, + }; + + // CAN BCM opcodes + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_SETUP: i32 = 1; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_DELETE: i32 = 2; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_READ: i32 = 3; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_SEND: i32 = 4; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_SETUP: i32 = 5; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_DELETE: i32 = 6; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_READ: i32 = 7; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_STATUS: i32 = 8; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_EXPIRED: i32 = 9; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_STATUS: i32 = 10; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_TIMEOUT: i32 = 11; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_CHANGED: i32 = 12; + + // CAN BCM flags (linux/can/bcm.h) + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_SETTIMER: i32 = 0x0001; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_STARTTIMER: i32 = 0x0002; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_COUNTEVT: i32 = 0x0004; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_ANNOUNCE: i32 = 0x0008; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_CP_CAN_ID: i32 = 0x0010; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_FILTER_ID: i32 = 0x0020; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_CHECK_DLC: i32 = 0x0040; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_NO_AUTOTIMER: i32 = 0x0080; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_ANNOUNCE_RESUME: i32 = 0x0100; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_TX_RESET_MULTI_IDX: i32 = 0x0200; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_RX_RTR_FRAME: i32 = 0x0400; + #[cfg(target_os = "linux")] + #[pyattr] + const CAN_BCM_CAN_FD_FRAME: i32 = 0x0800; + + #[cfg(all(target_os = "linux", target_env = "gnu"))] + #[pyattr] + use c::SOL_RDS; + + #[cfg(target_os = "netbsd")] + #[pyattr] + use c::IPPROTO_VRRP; + + #[cfg(target_vendor = "apple")] + #[pyattr] + use c::{AF_SYSTEM, PF_SYSTEM, SYSPROTO_CONTROL, TCP_KEEPALIVE}; + + // RFC3542 IPv6 socket options for macOS (netinet6/in6.h) + // Not available in libc, define manually + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVHOPLIMIT: i32 = 37; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVRTHDR: i32 = 38; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVHOPOPTS: i32 = 39; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVDSTOPTS: i32 = 40; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_USE_MIN_MTU: i32 = 42; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RECVPATHMTU: i32 = 43; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_PATHMTU: i32 = 44; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_NEXTHOP: i32 = 48; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_HOPOPTS: i32 = 49; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_DSTOPTS: i32 = 50; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RTHDR: i32 = 51; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RTHDRDSTOPTS: i32 = 57; + #[cfg(target_vendor = "apple")] + #[pyattr] + const IPV6_RTHDR_TYPE_0: i32 = 0; + + #[cfg(windows)] + #[pyattr] + use c::{ + IPPORT_RESERVED, IPPROTO_IPV4, RCVALL_IPLEVEL, RCVALL_OFF, RCVALL_ON, + RCVALL_SOCKETLEVELONLY, SIO_KEEPALIVE_VALS, SIO_LOOPBACK_FAST_PATH, SIO_RCVALL, + SO_EXCLUSIVEADDRUSE, + }; + + #[cfg(not(windows))] + #[pyattr] + const IPPORT_RESERVED: i32 = 1024; + + #[pyattr] + const IPPORT_USERRESERVED: i32 = 5000; + + #[cfg(any(unix, target_os = "android"))] + #[pyattr] + use c::{ + EAI_SYSTEM, MSG_EOR, SO_ACCEPTCONN, SO_DEBUG, SO_DONTROUTE, SO_RCVLOWAT, SO_RCVTIMEO, + SO_SNDLOWAT, SO_SNDTIMEO, + }; + + #[cfg(any(target_os = "android", target_os = "linux"))] + #[pyattr] + use c::{ + ALG_OP_DECRYPT, ALG_OP_ENCRYPT, ALG_SET_AEAD_ASSOCLEN, ALG_SET_AEAD_AUTHSIZE, ALG_SET_IV, + ALG_SET_KEY, ALG_SET_OP, IP_DEFAULT_MULTICAST_LOOP, IP_RECVOPTS, IP_RETOPTS, IPV6_DSTOPTS, + IPV6_NEXTHOP, IPV6_PATHMTU, IPV6_RECVDSTOPTS, IPV6_RECVHOPLIMIT, IPV6_RECVHOPOPTS, + IPV6_RECVPATHMTU, IPV6_RTHDRDSTOPTS, NETLINK_CRYPTO, NETLINK_DNRTMSG, NETLINK_FIREWALL, + NETLINK_IP6_FW, NETLINK_NFLOG, NETLINK_ROUTE, NETLINK_USERSOCK, NETLINK_XFRM, SO_PASSSEC, + SO_PEERSEC, SOL_ALG, + }; + + #[cfg(any(target_os = "android", target_vendor = "apple"))] + #[pyattr] + use c::{AI_DEFAULT, AI_MASK, AI_V4MAPPED_CFG}; + + #[cfg(any(target_os = "freebsd", target_os = "netbsd"))] + #[pyattr] + use c::MSG_NOTIFICATION; + + #[cfg(any(target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + use c::TCP_USER_TIMEOUT; + + #[cfg(any(unix, target_os = "android", windows))] + #[pyattr] + use c::{ + INADDR_BROADCAST, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_MULTICAST_IF, + IP_MULTICAST_LOOP, IP_MULTICAST_TTL, IP_TTL, IPV6_MULTICAST_HOPS, IPV6_MULTICAST_IF, + IPV6_MULTICAST_LOOP, IPV6_UNICAST_HOPS, IPV6_V6ONLY, + }; + + #[cfg(any(unix, target_os = "android", windows))] + #[pyattr] + const INADDR_UNSPEC_GROUP: u32 = 0xe0000000; + + #[cfg(any(unix, target_os = "android", windows))] + #[pyattr] + const INADDR_ALLHOSTS_GROUP: u32 = 0xe0000001; + + #[cfg(any(unix, target_os = "android", windows))] + #[pyattr] + const INADDR_MAX_LOCAL_GROUP: u32 = 0xe00000ff; + + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + use c::{ + AF_ALG, AF_ASH, AF_ATMPVC, AF_ATMSVC, AF_AX25, AF_BRIDGE, AF_CAN, AF_ECONET, AF_IRDA, + AF_LLC, AF_NETBEUI, AF_NETLINK, AF_NETROM, AF_PACKET, AF_PPPOX, AF_RDS, AF_SECURITY, + AF_TIPC, AF_VSOCK, AF_WANPIPE, AF_X25, IP_TRANSPARENT, MSG_CONFIRM, MSG_ERRQUEUE, + MSG_FASTOPEN, MSG_MORE, PF_CAN, PF_PACKET, PF_RDS, SCM_CREDENTIALS, SO_BINDTODEVICE, + SO_MARK, SOL_IP, SOL_TIPC, SOL_UDP, TCP_CORK, TCP_DEFER_ACCEPT, TCP_LINGER2, TCP_QUICKACK, + TCP_SYNCNT, TCP_WINDOW_CLAMP, + }; + + // gated on presence of AF_VSOCK: + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + const SO_VM_SOCKETS_BUFFER_SIZE: u32 = 0; + + // gated on presence of AF_VSOCK: + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + const SO_VM_SOCKETS_BUFFER_MIN_SIZE: u32 = 1; + + // gated on presence of AF_VSOCK: + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + const SO_VM_SOCKETS_BUFFER_MAX_SIZE: u32 = 2; + + // gated on presence of AF_VSOCK: + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + const VMADDR_CID_ANY: u32 = 0xffffffff; // 0xffffffff + + // gated on presence of AF_VSOCK: + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + const VMADDR_PORT_ANY: u32 = 0xffffffff; // 0xffffffff + + // gated on presence of AF_VSOCK: + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + const VMADDR_CID_HOST: u32 = 2; + + // gated on presence of AF_VSOCK: + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + #[pyattr] + const VM_SOCKETS_INVALID_VERSION: u32 = 0xffffffff; // 0xffffffff + + // TODO: gated on https://github.com/rust-lang/libc/pull/1662 + // // gated on presence of AF_VSOCK: + // #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + // #[pyattr(name = "IOCTL_VM_SOCKETS_GET_LOCAL_CID", once)] + // fn ioctl_vm_sockets_get_local_cid(_vm: &VirtualMachine) -> i32 { + // c::_IO(7, 0xb9) + // } + + #[cfg(not(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] + #[pyattr] + const SOL_IP: i32 = 0; + + #[cfg(not(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] + #[pyattr] + const SOL_UDP: i32 = 17; + + #[cfg(any(target_os = "android", target_os = "linux", windows))] + #[pyattr] + use c::{IP_OPTIONS, IPV6_HOPOPTS, IPV6_RECVRTHDR, IPV6_RTHDR}; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_vendor = "apple" + ))] + #[pyattr] + use c::{IPPROTO_HELLO, IPPROTO_XTP, LOCAL_PEERCRED, MSG_EOF}; + + #[cfg(any(target_os = "netbsd", target_os = "openbsd", windows))] + #[pyattr] + use c::{MSG_BCAST, MSG_MCAST}; + + #[cfg(any( + target_os = "android", + target_os = "fuchsia", + target_os = "freebsd", + target_os = "linux" + ))] + #[pyattr] + use c::{IPPROTO_UDPLITE, TCP_CONGESTION}; + + #[cfg(any( + target_os = "android", + target_os = "fuchsia", + target_os = "freebsd", + target_os = "linux" + ))] + #[pyattr] + const UDPLITE_SEND_CSCOV: i32 = 10; + + #[cfg(any( + target_os = "android", + target_os = "fuchsia", + target_os = "freebsd", + target_os = "linux" + ))] + #[pyattr] + const UDPLITE_RECV_CSCOV: i32 = 11; + + #[cfg(any( + target_os = "android", + target_os = "fuchsia", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyattr] + use c::AF_KEY; + + #[cfg(any( + target_os = "android", + target_os = "fuchsia", + target_os = "linux", + target_os = "redox" + ))] + #[pyattr] + use c::SO_DOMAIN; + + #[cfg(any( + target_os = "android", + target_os = "fuchsia", + all( + target_os = "linux", + any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "loongarch64", + target_arch = "mips", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" + ) + ), + target_os = "redox" + ))] + #[pyattr] + use c::SO_PRIORITY; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" + ))] + #[pyattr] + use c::IPPROTO_MOBILE; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_vendor = "apple" + ))] + #[pyattr] + use c::SCM_CREDS; + + #[cfg(any( + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_vendor = "apple" + ))] + #[pyattr] + use c::TCP_FASTOPEN; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + all( + target_os = "linux", + any( + target_arch = "aarch64", + target_arch = "x86", + target_arch = "loongarch64", + target_arch = "mips", + target_arch = "powerpc", + target_arch = "powerpc64", + target_arch = "riscv64", + target_arch = "s390x", + target_arch = "x86_64" + ) + ), + target_os = "redox" + ))] + #[pyattr] + use c::SO_PROTOCOL; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::IPV6_DONTFRAG; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "fuchsia", + target_os = "linux", + target_os = "redox" + ))] + #[pyattr] + use c::{SO_PASSCRED, SO_PEERCRED}; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd" + ))] + #[pyattr] + use c::TCP_INFO; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_vendor = "apple" + ))] + #[pyattr] + use c::IP_RECVTOS; + + #[cfg(any( + target_os = "android", + target_os = "netbsd", + target_os = "redox", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::NI_MAXSERV; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple" + ))] + #[pyattr] + use c::{IPPROTO_EON, IPPROTO_IPCOMP}; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::IPPROTO_ND; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::{IPV6_CHECKSUM, IPV6_HOPLIMIT}; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd" + ))] + #[pyattr] + use c::IPPROTO_SCTP; // also in windows + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::{AI_ALL, AI_V4MAPPED}; + + #[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::EAI_NODATA; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::{ + AF_LINK, IP_RECVDSTADDR, IPPROTO_GGP, IPV6_JOIN_GROUP, IPV6_LEAVE_GROUP, SO_USELOOPBACK, + }; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd" + ))] + #[pyattr] + use c::{MSG_CMSG_CLOEXEC, MSG_NOSIGNAL}; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd", + target_os = "redox" + ))] + #[pyattr] + use c::TCP_KEEPIDLE; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd", + target_vendor = "apple" + ))] + #[pyattr] + use c::{TCP_KEEPCNT, TCP_KEEPINTVL}; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] + #[pyattr] + use c::{SOCK_CLOEXEC, SOCK_NONBLOCK}; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple" + ))] + #[pyattr] + use c::{ + AF_ROUTE, AF_SNA, EAI_OVERFLOW, IPPROTO_GRE, IPPROTO_RSVP, IPPROTO_TP, IPV6_RECVPKTINFO, + MSG_DONTWAIT, SCM_RIGHTS, TCP_MAXSEG, + }; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::IPV6_PKTINFO; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::AI_CANONNAME; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple", + windows + ))] + #[pyattr] + use c::{ + EAI_AGAIN, EAI_BADFLAGS, EAI_FAIL, EAI_FAMILY, EAI_MEMORY, EAI_NONAME, EAI_SERVICE, + EAI_SOCKTYPE, IP_HDRINCL, IP_TOS, IPV6_RECVTCLASS, IPV6_TCLASS, SOMAXCONN, + }; + + #[cfg(not(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_vendor = "apple", + windows + )))] + #[pyattr] + const SOMAXCONN: i32 = 5; // Common value + + // HERE IS WHERE THE BLUETOOTH CONSTANTS START + // TODO: there should be a more intelligent way of detecting bluetooth on a platform. + // CPython uses header-detection, but blocks NetBSD and DragonFly BSD + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyattr] + use c::AF_BLUETOOTH; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyattr] + const BDADDR_ANY: &str = "00:00:00:00:00:00"; + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyattr] + const BDADDR_LOCAL: &str = "00:00:00:FF:FF:FF"; + // HERE IS WHERE THE BLUETOOTH CONSTANTS END + + #[cfg(windows)] + #[pyattr] + use windows_sys::Win32::Networking::WinSock::{ + IPPROTO_CBT, IPPROTO_ICLFXBM, IPPROTO_IGP, IPPROTO_L2TP, IPPROTO_PGM, IPPROTO_RDP, + IPPROTO_SCTP, IPPROTO_ST, + }; + + #[pyattr] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.exceptions.os_error.to_owned() + } + + #[pyattr] + fn timeout(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.exceptions.timeout_error.to_owned() + } + + #[pyattr(once)] + fn herror(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "socket", + "herror", + Some(vec![vm.ctx.exceptions.os_error.to_owned()]), + ) + } + #[pyattr(once)] + fn gaierror(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "socket", + "gaierror", + Some(vec![vm.ctx.exceptions.os_error.to_owned()]), + ) + } + + #[pyfunction] + const fn htonl(x: u32) -> u32 { + u32::to_be(x) + } + + #[pyfunction] + const fn htons(x: u16) -> u16 { + u16::to_be(x) + } + + #[pyfunction] + const fn ntohl(x: u32) -> u32 { + u32::from_be(x) + } + + #[pyfunction] + const fn ntohs(x: u16) -> u16 { + u16::from_be(x) + } + + #[cfg(unix)] + type RawSocket = std::os::unix::io::RawFd; + #[cfg(windows)] + type RawSocket = std::os::windows::raw::SOCKET; + + #[cfg(unix)] + macro_rules! errcode { + ($e:ident) => { + c::$e + }; + } + #[cfg(windows)] + macro_rules! errcode { + ($e:ident) => { + paste::paste!(c::[<WSA $e>]) + }; +} + + #[cfg(windows)] + use windows_sys::Win32::NetworkManagement::IpHelper; + + fn get_raw_sock(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<RawSocket> { + #[cfg(unix)] + type CastFrom = libc::c_long; + #[cfg(windows)] + type CastFrom = libc::c_longlong; + + // should really just be to_index() but test_socket tests the error messages explicitly + if obj.fast_isinstance(vm.ctx.types.float_type) { + return Err(vm.new_type_error("integer argument expected, got float")); + } + let int = obj + .try_index_opt(vm) + .unwrap_or_else(|| Err(vm.new_type_error("an integer is required")))?; + int.try_to_primitive::<CastFrom>(vm) + .map(|sock| sock as RawSocket) + } + + #[cfg(target_os = "linux")] + #[derive(FromArgs)] + struct SendmsgAfalgArgs { + #[pyarg(any, default)] + msg: Vec<ArgBytesLike>, + #[pyarg(named)] + op: u32, + #[pyarg(named, default)] + iv: Option<ArgBytesLike>, + #[pyarg(named, default)] + assoclen: OptionalArg<isize>, + #[pyarg(named, default)] + flags: i32, + } + + #[pyattr(name = "socket")] + #[pyattr(name = "SocketType")] + #[pyclass(name = "socket")] + #[derive(Debug, PyPayload)] + pub struct PySocket { + kind: AtomicCell<i32>, + family: AtomicCell<i32>, + proto: AtomicCell<i32>, + pub(crate) timeout: AtomicCell<f64>, + sock: PyRwLock<Option<Socket>>, + } + + const _: () = assert!(core::mem::size_of::<Option<Socket>>() == core::mem::size_of::<Socket>()); + + impl Default for PySocket { + fn default() -> Self { + Self { + kind: AtomicCell::default(), + family: AtomicCell::default(), + proto: AtomicCell::default(), + timeout: AtomicCell::new(-1.0), + sock: PyRwLock::new(None), + } + } + } + + #[cfg(windows)] + const CLOSED_ERR: i32 = c::WSAENOTSOCK; + #[cfg(unix)] + const CLOSED_ERR: i32 = c::EBADF; + + impl Read for &PySocket { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { + (&mut &*self.sock()?).read(buf) + } + } + + impl Write for &PySocket { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + (&mut &*self.sock()?).write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + (&mut &*self.sock()?).flush() + } + } + + impl PySocket { + pub fn sock_opt(&self) -> Option<PyMappedRwLockReadGuard<'_, Socket>> { + let sock = PyRwLockReadGuard::try_map(self.sock.read(), |sock| sock.as_ref()); + sock.ok() + } + + pub fn sock(&self) -> io::Result<PyMappedRwLockReadGuard<'_, Socket>> { + self.sock_opt() + .ok_or_else(|| io::Error::from_raw_os_error(CLOSED_ERR)) + } + + fn init_inner( + &self, + family: i32, + socket_kind: i32, + proto: i32, + sock: Socket, + ) -> io::Result<()> { + self.family.store(family); + // Mask out SOCK_NONBLOCK and SOCK_CLOEXEC flags from stored type + // to ensure consistent cross-platform behavior + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] + let masked_kind = socket_kind & !(c::SOCK_NONBLOCK | c::SOCK_CLOEXEC); + #[cfg(not(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + )))] + let masked_kind = socket_kind; + self.kind.store(masked_kind); + self.proto.store(proto); + let mut s = self.sock.write(); + let sock = s.insert(sock); + // If SOCK_NONBLOCK is set, use timeout 0 (non-blocking) + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] + let timeout = if socket_kind & c::SOCK_NONBLOCK != 0 { + 0.0 + } else { + DEFAULT_TIMEOUT.load() + }; + #[cfg(not(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + )))] + let timeout = DEFAULT_TIMEOUT.load(); + self.timeout.store(timeout); + if timeout >= 0.0 { + sock.set_nonblocking(true)?; + } + Ok(()) + } + + /// returns Err(blocking) + pub fn get_timeout(&self) -> Result<Duration, bool> { + let timeout = self.timeout.load(); + if timeout > 0.0 { + Ok(Duration::from_secs_f64(timeout)) + } else { + Err(timeout != 0.0) + } + } + + fn sock_op<F, R>( + &self, + vm: &VirtualMachine, + select: SelectKind, + f: F, + ) -> Result<R, IoOrPyException> + where + F: FnMut() -> io::Result<R>, + { + let timeout = self.get_timeout().ok(); + self.sock_op_timeout_err(vm, select, timeout, f) + } + + fn sock_op_timeout_err<F, R>( + &self, + vm: &VirtualMachine, + select: SelectKind, + timeout: Option<Duration>, + mut f: F, + ) -> Result<R, IoOrPyException> + where + F: FnMut() -> io::Result<R>, + { + let deadline = timeout.map(Deadline::new); + + loop { + if deadline.is_some() || matches!(select, SelectKind::Connect) { + let interval = deadline.as_ref().map(|d| d.time_until()).transpose()?; + let sock = self.sock()?; + let res = vm.allow_threads(|| sock_select(&sock, select, interval)); + match res { + Ok(true) => return Err(IoOrPyException::Timeout), + Err(e) if e.kind() == io::ErrorKind::Interrupted => { + vm.check_signals()?; + continue; + } + Err(e) => return Err(e.into()), + Ok(false) => {} // no timeout, continue as normal + } + } + + let err = loop { + // Detach thread state around the blocking syscall so + // stop-the-world can park this thread (e.g. before fork). + match vm.allow_threads(&mut f) { + Ok(x) => return Ok(x), + Err(e) if e.kind() == io::ErrorKind::Interrupted => vm.check_signals()?, + Err(e) => break e, + } + }; + if timeout.is_some() && err.kind() == io::ErrorKind::WouldBlock { + continue; + } + return Err(err.into()); + } + } + + fn extract_address( + &self, + addr: PyObjectRef, + caller: &str, + vm: &VirtualMachine, + ) -> Result<socket2::SockAddr, IoOrPyException> { + let family = self.family.load(); + match family { + #[cfg(unix)] + c::AF_UNIX => { + use crate::vm::function::ArgStrOrBytesLike; + use std::os::unix::ffi::OsStrExt; + let buf = ArgStrOrBytesLike::try_from_object(vm, addr)?; + let bytes = &*buf.borrow_bytes(); + let path = match &buf { + ArgStrOrBytesLike::Buf(_) => ffi::OsStr::from_bytes(bytes).into(), + ArgStrOrBytesLike::Str(s) => vm.fsencode(s)?, + }; + socket2::SockAddr::unix(path) + .map_err(|_| vm.new_os_error("AF_UNIX path too long".to_owned()).into()) + } + c::AF_INET => { + let tuple: PyTupleRef = addr.downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_INET address must be tuple, not {}", + caller, + obj.class().name() + )) + })?; + if tuple.len() != 2 { + return Err(vm + .new_type_error("AF_INET address must be a pair (host, post)") + .into()); + } + let addr = Address::from_tuple(&tuple, vm)?; + let mut addr4 = get_addr(vm, addr.host, c::AF_INET)?; + match &mut addr4 { + SocketAddr::V4(addr4) => { + addr4.set_port(addr.port); + } + SocketAddr::V6(_) => unreachable!(), + } + Ok(addr4.into()) + } + c::AF_INET6 => { + let tuple: PyTupleRef = addr.downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_INET6 address must be tuple, not {}", + caller, + obj.class().name() + )) + })?; + match tuple.len() { + 2..=4 => {} + _ => return Err(vm.new_type_error( + "AF_INET6 address must be a tuple (host, port[, flowinfo[, scopeid]])", + ).into()), + } + let (addr, flowinfo, scopeid) = Address::from_tuple_ipv6(&tuple, vm)?; + let mut addr6 = get_addr(vm, addr.host, c::AF_INET6)?; + match &mut addr6 { + SocketAddr::V6(addr6) => { + addr6.set_port(addr.port); + addr6.set_flowinfo(flowinfo); + addr6.set_scope_id(scopeid); + } + SocketAddr::V4(_) => unreachable!(), + } + Ok(addr6.into()) + } + #[cfg(target_os = "linux")] + c::AF_CAN => { + let tuple: PyTupleRef = addr.downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_CAN address must be tuple, not {}", + caller, + obj.class().name() + )) + })?; + if tuple.is_empty() || tuple.len() > 2 { + return Err(vm + .new_type_error( + "AF_CAN address must be a tuple (interface,) or (interface, addr)", + ) + .into()); + } + let interface: PyStrRef = tuple[0].clone().downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_CAN interface must be str, not {}", + caller, + obj.class().name() + )) + })?; + let interface = interface.try_into_utf8(vm).map_err(IoOrPyException::from)?; + let ifname = interface.as_str(); + + // Get interface index + let ifindex = if ifname.is_empty() { + 0 // Bind to all CAN interfaces + } else { + // Check interface name length (IFNAMSIZ is typically 16) + if ifname.len() >= 16 { + return Err(vm + .new_os_error("interface name too long".to_owned()) + .into()); + } + let cstr = alloc::ffi::CString::new(ifname) + .map_err(|_| vm.new_os_error("invalid interface name".to_owned()))?; + let idx = unsafe { libc::if_nametoindex(cstr.as_ptr()) }; + if idx == 0 { + return Err(io::Error::last_os_error().into()); + } + idx as i32 + }; + + // Create sockaddr_can + let mut storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; + let can_addr = + &mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_can; + unsafe { + (*can_addr).can_family = libc::AF_CAN as libc::sa_family_t; + (*can_addr).can_ifindex = ifindex; + } + let storage: socket2::SockAddrStorage = + unsafe { core::mem::transmute(storage) }; + Ok(unsafe { + socket2::SockAddr::new( + storage, + core::mem::size_of::<libc::sockaddr_can>() as libc::socklen_t, + ) + }) + } + #[cfg(target_os = "linux")] + c::AF_ALG => { + let tuple: PyTupleRef = addr.downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_ALG address must be tuple, not {}", + caller, + obj.class().name() + )) + })?; + if tuple.len() != 2 { + return Err(vm + .new_type_error("AF_ALG address must be a tuple (type, name)") + .into()); + } + let alg_type: PyStrRef = tuple[0].clone().downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_ALG type must be str, not {}", + caller, + obj.class().name() + )) + })?; + let alg_name: PyStrRef = tuple[1].clone().downcast().map_err(|obj| { + vm.new_type_error(format!( + "{}(): AF_ALG name must be str, not {}", + caller, + obj.class().name() + )) + })?; + + let alg_type = alg_type.try_into_utf8(vm).map_err(IoOrPyException::from)?; + let alg_name = alg_name.try_into_utf8(vm).map_err(IoOrPyException::from)?; + let type_str = alg_type.as_str(); + let name_str = alg_name.as_str(); + + // salg_type is 14 bytes, salg_name is 64 bytes + if type_str.len() >= 14 { + return Err(vm.new_value_error("type too long").into()); + } + if name_str.len() >= 64 { + return Err(vm.new_value_error("name too long").into()); + } + + // Create sockaddr_alg + let mut storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; + let alg_addr = + &mut storage as *mut libc::sockaddr_storage as *mut libc::sockaddr_alg; + unsafe { + (*alg_addr).salg_family = libc::AF_ALG as libc::sa_family_t; + // Copy type string + for (i, b) in type_str.bytes().enumerate() { + (*alg_addr).salg_type[i] = b; + } + // Copy name string + for (i, b) in name_str.bytes().enumerate() { + (*alg_addr).salg_name[i] = b; + } + } + let storage: socket2::SockAddrStorage = + unsafe { core::mem::transmute(storage) }; + Ok(unsafe { + socket2::SockAddr::new( + storage, + core::mem::size_of::<libc::sockaddr_alg>() as libc::socklen_t, + ) + }) + } + _ => Err(vm.new_os_error(format!("{caller}(): bad family")).into()), + } + } + + fn connect_inner( + &self, + address: PyObjectRef, + caller: &str, + vm: &VirtualMachine, + ) -> Result<(), IoOrPyException> { + let sock_addr = self.extract_address(address, caller, vm)?; + + let sock = self.sock()?; + let err = match vm.allow_threads(|| sock.connect(&sock_addr)) { + Ok(()) => return Ok(()), + Err(e) => e, + }; + + let wait_connect = if err.kind() == io::ErrorKind::Interrupted { + vm.check_signals()?; + self.timeout.load() != 0.0 + } else { + #[cfg(unix)] + use c::EINPROGRESS; + #[cfg(windows)] + use c::WSAEWOULDBLOCK as EINPROGRESS; + + self.timeout.load() > 0.0 && err.raw_os_error() == Some(EINPROGRESS) + }; + + if wait_connect { + // basically, connect() is async, and it registers an "error" on the socket when it's + // done connecting. SelectKind::Connect fills the errorfds fd_set, so if we wake up + // from poll and the error is EISCONN then we know that the connect is done + self.sock_op(vm, SelectKind::Connect, || { + let sock = self.sock()?; + let err = sock.take_error()?; + match err { + Some(e) if e.posix_errno() == libc::EISCONN => Ok(()), + Some(e) => Err(e), + // TODO: is this accurate? + None => Ok(()), + } + }) + } else { + Err(err.into()) + } + } + } + + impl DefaultConstructor for PySocket {} + + #[derive(FromArgs)] + pub struct SocketInitArgs { + #[pyarg(any, optional)] + family: OptionalArg<i32>, + #[pyarg(any, optional)] + r#type: OptionalArg<i32>, + #[pyarg(any, optional)] + proto: OptionalArg<i32>, + #[pyarg(any, optional)] + fileno: OptionalOption<PyObjectRef>, + } + + impl Initializer for PySocket { + type Args = SocketInitArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + Self::_init(zelf, args, vm).map_err(|e| e.into_pyexception(vm)) + } + } + + impl Representable for PySocket { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<socket object, fd={}, family={}, type={}, proto={}>", + zelf.fileno(), + zelf.family.load(), + zelf.kind.load(), + zelf.proto.load(), + )) + } + } + + impl Destructor for PySocket { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Emit ResourceWarning if socket is still open + if zelf.sock.read().is_some() { + let laddr = if let Ok(sock) = zelf.sock() + && let Ok(addr) = sock.local_addr() + && let Ok(repr) = get_addr_tuple(&addr, vm).repr(vm) + { + format!(", laddr={}", repr.as_wtf8()) + } else { + String::new() + }; + + let msg = format!( + "unclosed <socket.socket fd={}, family={}, type={}, proto={}{}>", + zelf.fileno(), + zelf.family.load(), + zelf.kind.load(), + zelf.proto.load(), + laddr + ); + let _ = crate::vm::warn::warn( + vm.ctx.new_str(msg).into(), + Some(vm.ctx.exceptions.resource_warning.to_owned()), + 1, + None, + vm, + ); + } + let _ = zelf.close(); + Ok(()) + } + } + + #[pyclass( + with(Constructor, Initializer, Representable, Destructor), + flags(BASETYPE) + )] + impl PySocket { + fn _init( + zelf: PyRef<Self>, + args: <Self as Initializer>::Args, + vm: &VirtualMachine, + ) -> Result<(), IoOrPyException> { + let mut family = args.family.unwrap_or(-1); + let mut socket_kind = args.r#type.unwrap_or(-1); + let mut proto = args.proto.unwrap_or(-1); + + let fileno = args.fileno; + let sock; + + // On Windows, fileno can be bytes from socket.share() for fromshare() + #[cfg(windows)] + if let Some(fileno_obj) = fileno.flatten() { + use crate::vm::builtins::PyBytes; + if let Ok(bytes) = fileno_obj.clone().downcast::<PyBytes>() { + let bytes_data = bytes.as_bytes(); + let expected_size = core::mem::size_of::<c::WSAPROTOCOL_INFOW>(); + + if bytes_data.len() != expected_size { + return Err(vm + .new_value_error(format!( + "socket descriptor string has wrong size, should be {} bytes", + expected_size + )) + .into()); + } + + let mut info: c::WSAPROTOCOL_INFOW = unsafe { core::mem::zeroed() }; + unsafe { + core::ptr::copy_nonoverlapping( + bytes_data.as_ptr(), + &mut info as *mut c::WSAPROTOCOL_INFOW as *mut u8, + expected_size, + ); + } + + let fd = unsafe { + c::WSASocketW( + c::FROM_PROTOCOL_INFO, + c::FROM_PROTOCOL_INFO, + c::FROM_PROTOCOL_INFO, + &info, + 0, + c::WSA_FLAG_OVERLAPPED, + ) + }; + + if fd == c::INVALID_SOCKET { + return Err(Self::wsa_error().into()); + } + + crate::vm::stdlib::nt::raw_set_handle_inheritable(fd as _, false)?; + + family = info.iAddressFamily; + socket_kind = info.iSocketType; + proto = info.iProtocol; + + sock = unsafe { sock_from_raw_unchecked(fd as RawSocket) }; + return Ok(zelf.init_inner(family, socket_kind, proto, sock)?); + } + + // Not bytes, treat as regular fileno + let fileno = get_raw_sock(fileno_obj, vm)?; + sock = sock_from_raw(fileno, vm)?; + match sock.local_addr() { + Ok(addr) if family == -1 => family = addr.family() as i32, + Err(e) + if family == -1 + || matches!( + e.raw_os_error(), + Some(errcode!(ENOTSOCK)) | Some(errcode!(EBADF)) + ) => + { + core::mem::forget(sock); + return Err(e.into()); + } + _ => {} + } + if socket_kind == -1 { + socket_kind = sock.r#type().map_err(|e| e.into_pyexception(vm))?.into(); + } + proto = 0; + return Ok(zelf.init_inner(family, socket_kind, proto, sock)?); + } + + #[cfg(not(windows))] + let fileno = fileno + .flatten() + .map(|obj| get_raw_sock(obj, vm)) + .transpose()?; + #[cfg(not(windows))] + if let Some(fileno) = fileno { + sock = sock_from_raw(fileno, vm)?; + match sock.local_addr() { + Ok(addr) if family == -1 => family = addr.family() as i32, + Err(e) + if family == -1 + || matches!( + e.raw_os_error(), + Some(errcode!(ENOTSOCK)) | Some(errcode!(EBADF)) + ) => + { + core::mem::forget(sock); + return Err(e.into()); + } + _ => {} + } + if socket_kind == -1 { + socket_kind = sock.r#type().map_err(|e| e.into_pyexception(vm))?.into(); + } + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "linux", + ))] { + if proto == -1 { + proto = sock.protocol()?.map_or(0, Into::into); + } + } else { + proto = 0; + } + } + return Ok(zelf.init_inner(family, socket_kind, proto, sock)?); + } + + // No fileno provided, create new socket + { + if family == -1 { + family = c::AF_INET as _ + } + if socket_kind == -1 { + socket_kind = c::SOCK_STREAM + } + if proto == -1 { + proto = 0 + } + sock = Socket::new(family.into(), socket_kind.into(), Some(proto.into()))?; + }; + Ok(zelf.init_inner(family, socket_kind, proto, sock)?) + } + + #[pymethod] + fn connect( + &self, + address: PyObjectRef, + vm: &VirtualMachine, + ) -> Result<(), IoOrPyException> { + self.connect_inner(address, "connect", vm) + } + + #[pymethod] + fn connect_ex(&self, address: PyObjectRef, vm: &VirtualMachine) -> PyResult<i32> { + match self.connect_inner(address, "connect_ex", vm) { + Ok(()) => Ok(0), + Err(err) => err.errno(), + } + } + + #[pymethod] + fn bind(&self, address: PyObjectRef, vm: &VirtualMachine) -> Result<(), IoOrPyException> { + let sock_addr = self.extract_address(address, "bind", vm)?; + Ok(self.sock()?.bind(&sock_addr)?) + } + + #[pymethod] + fn listen(&self, backlog: OptionalArg<i32>) -> io::Result<()> { + let backlog = backlog.unwrap_or(128); + let backlog = if backlog < 0 { 0 } else { backlog }; + self.sock()?.listen(backlog) + } + + #[pymethod] + fn _accept( + &self, + vm: &VirtualMachine, + ) -> Result<(RawSocket, PyObjectRef), IoOrPyException> { + // Use accept_raw() instead of accept() to avoid socket2's set_common_flags() + // which tries to set SO_NOSIGPIPE and fails with EINVAL on Unix domain sockets on macOS + let (sock, addr) = self.sock_op(vm, SelectKind::Read, || self.sock()?.accept_raw())?; + let fd = into_sock_fileno(sock); + Ok((fd, get_addr_tuple(&addr, vm))) + } + + #[pymethod] + fn recv( + &self, + bufsize: usize, + flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> Result<Vec<u8>, IoOrPyException> { + let flags = flags.unwrap_or(0); + let mut buffer = Vec::with_capacity(bufsize); + let sock = self.sock()?; + let n = self.sock_op(vm, SelectKind::Read, || { + sock.recv_with_flags(buffer.spare_capacity_mut(), flags) + })?; + unsafe { buffer.set_len(n) }; + Ok(buffer) + } + + #[pymethod] + fn recv_into( + &self, + buf: ArgMemoryBuffer, + nbytes: OptionalArg<isize>, + flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> Result<usize, IoOrPyException> { + let flags = flags.unwrap_or(0); + let sock = self.sock()?; + let mut buf = buf.borrow_buf_mut(); + let buf = &mut *buf; + + // Handle nbytes parameter + let read_len = if let OptionalArg::Present(nbytes) = nbytes { + let nbytes = nbytes + .to_usize() + .ok_or_else(|| vm.new_value_error("negative buffersize in recv_into"))?; + nbytes.min(buf.len()) + } else { + buf.len() + }; + + let buf = &mut buf[..read_len]; + self.sock_op(vm, SelectKind::Read, || { + sock.recv_with_flags(unsafe { slice_as_uninit(buf) }, flags) + }) + } + + #[pymethod] + fn recvfrom( + &self, + bufsize: isize, + flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> Result<(Vec<u8>, PyObjectRef), IoOrPyException> { + let flags = flags.unwrap_or(0); + let bufsize = bufsize + .to_usize() + .ok_or_else(|| vm.new_value_error("negative buffersize in recvfrom"))?; + let mut buffer = Vec::with_capacity(bufsize); + let (n, addr) = self.sock_op(vm, SelectKind::Read, || { + self.sock()? + .recv_from_with_flags(buffer.spare_capacity_mut(), flags) + })?; + unsafe { buffer.set_len(n) }; + Ok((buffer, get_addr_tuple(&addr, vm))) + } + + #[pymethod] + fn recvfrom_into( + &self, + buf: ArgMemoryBuffer, + nbytes: OptionalArg<isize>, + flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> Result<(usize, PyObjectRef), IoOrPyException> { + let mut buf = buf.borrow_buf_mut(); + let buf = &mut *buf; + let buf = match nbytes { + OptionalArg::Present(i) => { + let i = i.to_usize().ok_or_else(|| { + vm.new_value_error("negative buffersize in recvfrom_into") + })?; + buf.get_mut(..i).ok_or_else(|| { + vm.new_value_error("nbytes is greater than the length of the buffer") + })? + } + OptionalArg::Missing => buf, + }; + let flags = flags.unwrap_or(0); + let sock = self.sock()?; + let (n, addr) = self.sock_op(vm, SelectKind::Read, || { + sock.recv_from_with_flags(unsafe { slice_as_uninit(buf) }, flags) + })?; + Ok((n, get_addr_tuple(&addr, vm))) + } + + #[pymethod] + fn send( + &self, + bytes: ArgBytesLike, + flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> Result<usize, IoOrPyException> { + let flags = flags.unwrap_or(0); + let buf = bytes.borrow_buf(); + let buf = &*buf; + self.sock_op(vm, SelectKind::Write, || { + self.sock()?.send_with_flags(buf, flags) + }) + } + + #[pymethod] + fn sendall( + &self, + bytes: ArgBytesLike, + flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> Result<(), IoOrPyException> { + let flags = flags.unwrap_or(0); + + let timeout = self.get_timeout().ok(); + + let deadline = timeout.map(Deadline::new); + + let buf = bytes.borrow_buf(); + let buf = &*buf; + let mut buf_offset = 0; + // now we have like 3 layers of interrupt loop :) + while buf_offset < buf.len() { + let interval = deadline.as_ref().map(|d| d.time_until()).transpose()?; + self.sock_op_timeout_err(vm, SelectKind::Write, interval, || { + let subbuf = &buf[buf_offset..]; + buf_offset += self.sock()?.send_with_flags(subbuf, flags)?; + Ok(()) + })?; + vm.check_signals()?; + } + Ok(()) + } + + #[pymethod] + fn sendto( + &self, + bytes: ArgBytesLike, + arg2: PyObjectRef, + arg3: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> Result<usize, IoOrPyException> { + // signature is bytes[, flags], address + let (flags, address) = match arg3 { + OptionalArg::Present(arg3) => { + // should just be i32::try_from_obj but tests check for error message + let int = arg2 + .try_index_opt(vm) + .unwrap_or_else(|| Err(vm.new_type_error("an integer is required")))?; + let flags = int.try_to_primitive::<i32>(vm)?; + (flags, arg3) + } + OptionalArg::Missing => (0, arg2), + }; + let addr = self.extract_address(address, "sendto", vm)?; + let buf = bytes.borrow_buf(); + let buf = &*buf; + self.sock_op(vm, SelectKind::Write, || { + self.sock()?.send_to_with_flags(buf, &addr, flags) + }) + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pymethod] + fn sendmsg( + &self, + buffers: Vec<ArgBytesLike>, + ancdata: OptionalArg, + flags: OptionalArg<i32>, + addr: OptionalOption, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let flags = flags.unwrap_or(0); + let mut msg = socket2::MsgHdr::new(); + + let sockaddr; + if let Some(addr) = addr.flatten() { + sockaddr = self + .extract_address(addr, "sendmsg", vm) + .map_err(|e| e.into_pyexception(vm))?; + msg = msg.with_addr(&sockaddr); + } + + let buffers = buffers + .iter() + .map(|buf| buf.borrow_buf()) + .collect::<Vec<_>>(); + let buffers = buffers + .iter() + .map(|buf| io::IoSlice::new(buf)) + .collect::<Vec<_>>(); + msg = msg.with_buffers(&buffers); + + let control_buf; + if let OptionalArg::Present(ancdata) = ancdata { + let cmsgs = vm.extract_elements_with( + &ancdata, + |obj| -> PyResult<(i32, i32, ArgBytesLike)> { + let seq: Vec<PyObjectRef> = obj.try_into_value(vm)?; + let [lvl, typ, data]: [PyObjectRef; 3] = seq + .try_into() + .map_err(|_| vm.new_type_error("expected a sequence of length 3"))?; + Ok(( + lvl.try_into_value(vm)?, + typ.try_into_value(vm)?, + data.try_into_value(vm)?, + )) + }, + )?; + control_buf = Self::pack_cmsgs_to_send(&cmsgs, vm)?; + if !control_buf.is_empty() { + msg = msg.with_control(&control_buf); + } + } + + self.sock_op(vm, SelectKind::Write, || { + let sock = self.sock()?; + sock.sendmsg(&msg, flags) + }) + .map_err(|e| e.into_pyexception(vm)) + } + + /// sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) -> int + /// + /// Set operation mode and target IV for an AF_ALG socket. + #[cfg(target_os = "linux")] + #[pymethod] + fn sendmsg_afalg(&self, args: SendmsgAfalgArgs, vm: &VirtualMachine) -> PyResult<usize> { + let msg = args.msg; + let op = args.op; + let iv = args.iv; + let flags = args.flags; + + // Validate assoclen - must be non-negative if provided + let assoclen: Option<u32> = match args.assoclen { + OptionalArg::Present(val) if val < 0 => { + return Err(vm.new_type_error("assoclen must be non-negative")); + } + OptionalArg::Present(val) => Some(val as u32), + OptionalArg::Missing => None, + }; + + // Build control messages for AF_ALG + let mut control_buf = Vec::new(); + + // Add ALG_SET_OP control message + { + let op_bytes = op.to_ne_bytes(); + let space = + unsafe { libc::CMSG_SPACE(core::mem::size_of::<u32>() as u32) } as usize; + let old_len = control_buf.len(); + control_buf.resize(old_len + space, 0u8); + + let cmsg = control_buf[old_len..].as_mut_ptr() as *mut libc::cmsghdr; + unsafe { + (*cmsg).cmsg_len = libc::CMSG_LEN(core::mem::size_of::<u32>() as u32) as _; + (*cmsg).cmsg_level = libc::SOL_ALG; + (*cmsg).cmsg_type = libc::ALG_SET_OP; + let data = libc::CMSG_DATA(cmsg); + core::ptr::copy_nonoverlapping(op_bytes.as_ptr(), data, op_bytes.len()); + } + } + + // Add ALG_SET_IV control message if iv is provided + if let Some(iv_data) = iv { + let iv_bytes = iv_data.borrow_buf(); + // struct af_alg_iv { __u32 ivlen; __u8 iv[]; } + let iv_struct_size = 4 + iv_bytes.len(); + let space = unsafe { libc::CMSG_SPACE(iv_struct_size as u32) } as usize; + let old_len = control_buf.len(); + control_buf.resize(old_len + space, 0u8); + + let cmsg = control_buf[old_len..].as_mut_ptr() as *mut libc::cmsghdr; + unsafe { + (*cmsg).cmsg_len = libc::CMSG_LEN(iv_struct_size as u32) as _; + (*cmsg).cmsg_level = libc::SOL_ALG; + (*cmsg).cmsg_type = libc::ALG_SET_IV; + let data = libc::CMSG_DATA(cmsg); + // Write ivlen + let ivlen = (iv_bytes.len() as u32).to_ne_bytes(); + core::ptr::copy_nonoverlapping(ivlen.as_ptr(), data, 4); + // Write iv + core::ptr::copy_nonoverlapping(iv_bytes.as_ptr(), data.add(4), iv_bytes.len()); + } + } + + // Add ALG_SET_AEAD_ASSOCLEN control message if assoclen is provided + if let Some(assoclen_val) = assoclen { + let assoclen_bytes = assoclen_val.to_ne_bytes(); + let space = + unsafe { libc::CMSG_SPACE(core::mem::size_of::<u32>() as u32) } as usize; + let old_len = control_buf.len(); + control_buf.resize(old_len + space, 0u8); + + let cmsg = control_buf[old_len..].as_mut_ptr() as *mut libc::cmsghdr; + unsafe { + (*cmsg).cmsg_len = libc::CMSG_LEN(core::mem::size_of::<u32>() as u32) as _; + (*cmsg).cmsg_level = libc::SOL_ALG; + (*cmsg).cmsg_type = libc::ALG_SET_AEAD_ASSOCLEN; + let data = libc::CMSG_DATA(cmsg); + core::ptr::copy_nonoverlapping( + assoclen_bytes.as_ptr(), + data, + assoclen_bytes.len(), + ); + } + } + + // Build buffers + let buffers = msg.iter().map(|buf| buf.borrow_buf()).collect::<Vec<_>>(); + let iovecs: Vec<libc::iovec> = buffers + .iter() + .map(|buf| libc::iovec { + iov_base: buf.as_ptr() as *mut _, + iov_len: buf.len(), + }) + .collect(); + + // Set up msghdr + let mut msghdr: libc::msghdr = unsafe { core::mem::zeroed() }; + msghdr.msg_iov = iovecs.as_ptr() as *mut _; + msghdr.msg_iovlen = iovecs.len() as _; + if !control_buf.is_empty() { + msghdr.msg_control = control_buf.as_mut_ptr() as *mut _; + msghdr.msg_controllen = control_buf.len() as _; + } + + self.sock_op(vm, SelectKind::Write, || { + let sock = self.sock()?; + let fd = sock_fileno(&sock); + let ret = unsafe { libc::sendmsg(fd as libc::c_int, &msghdr, flags) }; + if ret < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ret as usize) + } + }) + .map_err(|e| e.into_pyexception(vm)) + } + + /// recvmsg(bufsize[, ancbufsize[, flags]]) -> (data, ancdata, msg_flags, address) + /// + /// Receive normal data and ancillary data from the socket. + #[cfg(all(unix, not(target_os = "redox")))] + #[pymethod] + fn recvmsg( + &self, + bufsize: isize, + ancbufsize: OptionalArg<isize>, + flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<PyTupleRef> { + use core::mem::MaybeUninit; + + if bufsize < 0 { + return Err(vm.new_value_error("negative buffer size in recvmsg")); + } + let bufsize = bufsize as usize; + + let ancbufsize = ancbufsize.unwrap_or(0); + if ancbufsize < 0 { + return Err(vm.new_value_error("negative ancillary buffer size in recvmsg")); + } + let ancbufsize = ancbufsize as usize; + let flags = flags.unwrap_or(0); + + // Allocate buffers + let mut data_buf: Vec<MaybeUninit<u8>> = vec![MaybeUninit::uninit(); bufsize]; + let mut anc_buf: Vec<MaybeUninit<u8>> = vec![MaybeUninit::uninit(); ancbufsize]; + let mut addr_storage: libc::sockaddr_storage = unsafe { core::mem::zeroed() }; + + // Set up iovec + let mut iov = [libc::iovec { + iov_base: data_buf.as_mut_ptr().cast(), + iov_len: bufsize, + }]; + + // Set up msghdr + let mut msg: libc::msghdr = unsafe { core::mem::zeroed() }; + msg.msg_name = (&mut addr_storage as *mut libc::sockaddr_storage).cast(); + msg.msg_namelen = core::mem::size_of::<libc::sockaddr_storage>() as libc::socklen_t; + msg.msg_iov = iov.as_mut_ptr(); + msg.msg_iovlen = 1; + if ancbufsize > 0 { + msg.msg_control = anc_buf.as_mut_ptr().cast(); + msg.msg_controllen = ancbufsize as _; + } + + let n = self + .sock_op(vm, SelectKind::Read, || { + let sock = self.sock()?; + let fd = sock_fileno(&sock); + let ret = unsafe { libc::recvmsg(fd as libc::c_int, &mut msg, flags) }; + if ret < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ret as usize) + } + }) + .map_err(|e| e.into_pyexception(vm))?; + + // Build data bytes + let data = unsafe { + data_buf.set_len(n); + core::mem::transmute::<Vec<MaybeUninit<u8>>, Vec<u8>>(data_buf) + }; + + // Build ancdata list + let ancdata = Self::parse_ancillary_data(&msg, vm)?; + + // Build address tuple + let address = if msg.msg_namelen > 0 { + let storage: socket2::SockAddrStorage = + unsafe { core::mem::transmute(addr_storage) }; + let addr = unsafe { socket2::SockAddr::new(storage, msg.msg_namelen) }; + get_addr_tuple(&addr, vm) + } else { + vm.ctx.none() + }; + + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytes(data).into(), + ancdata, + vm.ctx.new_int(msg.msg_flags).into(), + address, + ])) + } + + /// Parse ancillary data from a received message header + #[cfg(all(unix, not(target_os = "redox")))] + fn parse_ancillary_data(msg: &libc::msghdr, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let mut result = Vec::new(); + + // Calculate buffer end for truncation handling + let ctrl_buf = msg.msg_control as *const u8; + let ctrl_end = unsafe { ctrl_buf.add(msg.msg_controllen as _) }; + + let mut cmsg: *mut libc::cmsghdr = unsafe { libc::CMSG_FIRSTHDR(msg) }; + while !cmsg.is_null() { + let cmsg_ref = unsafe { &*cmsg }; + let data_ptr = unsafe { libc::CMSG_DATA(cmsg) }; + + // Calculate data length, respecting buffer truncation + let data_len_from_cmsg = + cmsg_ref.cmsg_len as usize - (data_ptr as usize - cmsg as usize); + let available = ctrl_end as usize - data_ptr as usize; + let data_len = data_len_from_cmsg.min(available); + + let data = unsafe { core::slice::from_raw_parts(data_ptr, data_len) }; + + let tuple = vm.ctx.new_tuple(vec![ + vm.ctx.new_int(cmsg_ref.cmsg_level).into(), + vm.ctx.new_int(cmsg_ref.cmsg_type).into(), + vm.ctx.new_bytes(data.to_vec()).into(), + ]); + + result.push(tuple.into()); + + cmsg = unsafe { libc::CMSG_NXTHDR(msg, cmsg) }; + } + + Ok(vm.ctx.new_list(result).into()) + } + + // based on nix's implementation + #[cfg(all(unix, not(target_os = "redox")))] + fn pack_cmsgs_to_send( + cmsgs: &[(i32, i32, ArgBytesLike)], + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + use core::{mem, ptr}; + + if cmsgs.is_empty() { + return Ok(vec![]); + } + + let capacity = cmsgs + .iter() + .map(|(_, _, buf)| buf.len()) + .try_fold(0, |sum, len| { + let space = checked_cmsg_space(len).ok_or_else(|| { + vm.new_os_error("ancillary data item too large".to_owned()) + })?; + usize::checked_add(sum, space) + .ok_or_else(|| vm.new_os_error("too much ancillary data".to_owned())) + })?; + + let mut cmsg_buffer = vec![0u8; capacity]; + + // make a dummy msghdr so we can use the CMSG_* apis + let mut mhdr = unsafe { mem::zeroed::<libc::msghdr>() }; + mhdr.msg_control = cmsg_buffer.as_mut_ptr().cast(); + mhdr.msg_controllen = capacity as _; + + let mut pmhdr: *mut libc::cmsghdr = unsafe { libc::CMSG_FIRSTHDR(&mhdr) }; + for (lvl, typ, buf) in cmsgs { + if pmhdr.is_null() { + return Err(vm.new_runtime_error( + "unexpected NULL result from CMSG_FIRSTHDR/CMSG_NXTHDR", + )); + } + let data = &*buf.borrow_buf(); + assert_eq!(data.len(), buf.len()); + // Safe because we know that pmhdr is valid, and we initialized it with + // sufficient space + unsafe { + (*pmhdr).cmsg_level = *lvl; + (*pmhdr).cmsg_type = *typ; + (*pmhdr).cmsg_len = libc::CMSG_LEN(data.len() as _) as _; + ptr::copy_nonoverlapping(data.as_ptr(), libc::CMSG_DATA(pmhdr), data.len()); + } + + // Safe because mhdr is valid + pmhdr = unsafe { libc::CMSG_NXTHDR(&mhdr, pmhdr) }; + } + + Ok(cmsg_buffer) + } + + #[pymethod] + fn close(&self) -> io::Result<()> { + let sock = self.sock.write().take(); + if let Some(sock) = sock { + close_inner(into_sock_fileno(sock))?; + } + Ok(()) + } + + #[pymethod] + #[inline] + fn detach(&self) -> i64 { + let sock = self.sock.write().take(); + sock.map_or(INVALID_SOCKET as i64, |s| into_sock_fileno(s) as i64) + } + + #[pymethod] + fn fileno(&self) -> i64 { + self.sock + .read() + .as_ref() + .map_or(INVALID_SOCKET as i64, |s| sock_fileno(s) as i64) + } + + #[pymethod] + fn getsockname(&self, vm: &VirtualMachine) -> std::io::Result<PyObjectRef> { + let addr = self.sock()?.local_addr()?; + + Ok(get_addr_tuple(&addr, vm)) + } + + #[pymethod] + fn getpeername(&self, vm: &VirtualMachine) -> std::io::Result<PyObjectRef> { + let addr = self.sock()?.peer_addr()?; + + Ok(get_addr_tuple(&addr, vm)) + } + + #[pymethod] + fn gettimeout(&self) -> Option<f64> { + let timeout = self.timeout.load(); + if timeout >= 0.0 { Some(timeout) } else { None } + } + + #[pymethod] + fn setblocking(&self, block: bool) -> io::Result<()> { + self.timeout.store(if block { -1.0 } else { 0.0 }); + self.sock()?.set_nonblocking(!block) + } + + #[pymethod] + fn getblocking(&self) -> bool { + self.timeout.load() != 0.0 + } + + #[pymethod] + fn settimeout(&self, timeout: Option<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<()> { + let timeout = match timeout { + Some(t) => { + let f = t.into_float(); + if f.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)")); + } + if f < 0.0 || !f.is_finite() { + return Err(vm.new_value_error("Timeout value out of range")); + } + Some(f) + } + None => None, + }; + self.timeout.store(timeout.unwrap_or(-1.0)); + // even if timeout is > 0 the socket needs to be nonblocking in order for us to select() on + // it + self.sock() + .map_err(|e| e.into_pyexception(vm))? + .set_nonblocking(timeout.is_some()) + .map_err(|e| e.into_pyexception(vm)) + } + + #[pymethod] + fn getsockopt( + &self, + level: i32, + name: i32, + buflen: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> Result<PyObjectRef, IoOrPyException> { + let sock = self.sock()?; + let fd = sock_fileno(&sock); + let buflen = buflen.unwrap_or(0); + if buflen == 0 { + let mut flag: libc::c_int = 0; + let mut flagsize = core::mem::size_of::<libc::c_int>() as _; + let ret = unsafe { + c::getsockopt( + fd as _, + level, + name, + &mut flag as *mut libc::c_int as *mut _, + &mut flagsize, + ) + }; + if ret < 0 { + return Err(crate::common::os::errno_io_error().into()); + } + Ok(vm.ctx.new_int(flag).into()) + } else { + if buflen <= 0 || buflen > 1024 { + return Err(vm + .new_os_error("getsockopt buflen out of range".to_owned()) + .into()); + } + let mut buf = vec![0u8; buflen as usize]; + let mut buflen = buflen as _; + let ret = unsafe { + c::getsockopt( + fd as _, + level, + name, + buf.as_mut_ptr() as *mut _, + &mut buflen, + ) + }; + if ret < 0 { + return Err(crate::common::os::errno_io_error().into()); + } + buf.truncate(buflen as usize); + Ok(vm.ctx.new_bytes(buf).into()) + } + } + + #[pymethod] + fn setsockopt( + &self, + level: i32, + name: i32, + value: Option<Either<ArgBytesLike, i32>>, + optlen: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> Result<(), IoOrPyException> { + let sock = self.sock()?; + let fd = sock_fileno(&sock); + let ret = match (value, optlen) { + (Some(Either::A(b)), OptionalArg::Missing) => b.with_ref(|b| unsafe { + c::setsockopt(fd as _, level, name, b.as_ptr() as *const _, b.len() as _) + }), + (Some(Either::B(ref val)), OptionalArg::Missing) => unsafe { + c::setsockopt( + fd as _, + level, + name, + val as *const i32 as *const _, + core::mem::size_of::<i32>() as _, + ) + }, + (None, OptionalArg::Present(optlen)) => unsafe { + c::setsockopt(fd as _, level, name, core::ptr::null(), optlen as _) + }, + _ => { + return Err(vm + .new_type_error("expected the value arg xor the optlen arg") + .into()); + } + }; + if ret < 0 { + Err(crate::common::os::errno_io_error().into()) + } else { + Ok(()) + } + } + + #[pymethod] + fn shutdown(&self, how: i32, vm: &VirtualMachine) -> Result<(), IoOrPyException> { + let how = match how { + c::SHUT_RD => Shutdown::Read, + c::SHUT_WR => Shutdown::Write, + c::SHUT_RDWR => Shutdown::Both, + _ => { + return Err(vm + .new_value_error("`how` must be SHUT_RD, SHUT_WR, or SHUT_RDWR") + .into()); + } + }; + Ok(self.sock()?.shutdown(how)?) + } + + #[cfg(windows)] + fn wsa_error() -> io::Error { + io::Error::from_raw_os_error(unsafe { c::WSAGetLastError() }) + } + + #[cfg(windows)] + #[pymethod] + fn ioctl( + &self, + cmd: PyObjectRef, + option: PyObjectRef, + vm: &VirtualMachine, + ) -> Result<u32, IoOrPyException> { + use crate::vm::builtins::PyInt; + use crate::vm::convert::TryFromObject; + + let sock = self.sock()?; + let fd = sock_fileno(&sock); + let mut recv: u32 = 0; + + // Convert cmd to u32, returning ValueError for invalid/negative values + let cmd_int = cmd + .downcast::<PyInt>() + .map_err(|_| vm.new_type_error("an integer is required"))?; + let cmd_val = cmd_int.as_bigint(); + let cmd: u32 = cmd_val + .to_u32() + .ok_or_else(|| vm.new_value_error(format!("invalid ioctl command {}", cmd_val)))?; + + match cmd { + c::SIO_RCVALL | c::SIO_LOOPBACK_FAST_PATH => { + // Option must be an integer, not None + if vm.is_none(&option) { + return Err(vm + .new_type_error("an integer is required (got type NoneType)") + .into()); + } + let option_val: u32 = TryFromObject::try_from_object(vm, option)?; + let ret = unsafe { + c::WSAIoctl( + fd as _, + cmd, + &option_val as *const u32 as *const _, + core::mem::size_of::<u32>() as u32, + core::ptr::null_mut(), + 0, + &mut recv, + core::ptr::null_mut(), + None, + ) + }; + if ret == c::SOCKET_ERROR { + return Err(Self::wsa_error().into()); + } + Ok(recv) + } + c::SIO_KEEPALIVE_VALS => { + let tuple: PyTupleRef = option + .downcast() + .map_err(|_| vm.new_type_error("SIO_KEEPALIVE_VALS requires a tuple"))?; + if tuple.len() != 3 { + return Err(vm + .new_type_error( + "SIO_KEEPALIVE_VALS requires (onoff, keepalivetime, keepaliveinterval)", + ) + .into()); + } + + #[repr(C)] + struct TcpKeepalive { + onoff: u32, + keepalivetime: u32, + keepaliveinterval: u32, + } + + let ka = TcpKeepalive { + onoff: TryFromObject::try_from_object(vm, tuple[0].clone())?, + keepalivetime: TryFromObject::try_from_object(vm, tuple[1].clone())?, + keepaliveinterval: TryFromObject::try_from_object(vm, tuple[2].clone())?, + }; + + let ret = unsafe { + c::WSAIoctl( + fd as _, + cmd, + &ka as *const TcpKeepalive as *const _, + core::mem::size_of::<TcpKeepalive>() as u32, + core::ptr::null_mut(), + 0, + &mut recv, + core::ptr::null_mut(), + None, + ) + }; + if ret == c::SOCKET_ERROR { + return Err(Self::wsa_error().into()); + } + Ok(recv) + } + _ => Err(vm + .new_value_error(format!("invalid ioctl command {}", cmd)) + .into()), + } + } + + #[cfg(windows)] + #[pymethod] + fn share(&self, process_id: u32, _vm: &VirtualMachine) -> Result<Vec<u8>, IoOrPyException> { + let sock = self.sock()?; + let fd = sock_fileno(&sock); + + let mut info: MaybeUninit<c::WSAPROTOCOL_INFOW> = MaybeUninit::uninit(); + + let ret = unsafe { c::WSADuplicateSocketW(fd as _, process_id, info.as_mut_ptr()) }; + + if ret == c::SOCKET_ERROR { + return Err(Self::wsa_error().into()); + } + + let info = unsafe { info.assume_init() }; + let bytes = unsafe { + core::slice::from_raw_parts( + &info as *const c::WSAPROTOCOL_INFOW as *const u8, + core::mem::size_of::<c::WSAPROTOCOL_INFOW>(), + ) + }; + + Ok(bytes.to_vec()) + } + + #[pygetset(name = "type")] + fn kind(&self) -> i32 { + self.kind.load() + } + + #[pygetset] + fn family(&self) -> i32 { + self.family.load() + } + + #[pygetset] + fn proto(&self) -> i32 { + self.proto.load() + } + } + + struct Address { + host: PyUtf8StrRef, + port: u16, + } + + impl ToSocketAddrs for Address { + type Iter = alloc::vec::IntoIter<SocketAddr>; + fn to_socket_addrs(&self) -> io::Result<Self::Iter> { + (self.host.as_str(), self.port).to_socket_addrs() + } + } + + impl TryFromObject for Address { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let tuple = PyTupleRef::try_from_object(vm, obj)?; + if tuple.len() != 2 { + Err(vm.new_type_error("Address tuple should have only 2 values")) + } else { + Self::from_tuple(&tuple, vm) + } + } + } + + impl Address { + fn from_tuple(tuple: &[PyObjectRef], vm: &VirtualMachine) -> PyResult<Self> { + let host = PyStrRef::try_from_object(vm, tuple[0].clone())?; + let host = host.try_into_utf8(vm)?; + let port = i32::try_from_borrowed_object(vm, &tuple[1])?; + let port = port + .to_u16() + .ok_or_else(|| vm.new_overflow_error("port must be 0-65535."))?; + Ok(Self { host, port }) + } + + fn from_tuple_ipv6( + tuple: &[PyObjectRef], + vm: &VirtualMachine, + ) -> PyResult<(Self, u32, u32)> { + let addr = Self::from_tuple(tuple, vm)?; + let flowinfo = tuple + .get(2) + .map(|obj| u32::try_from_borrowed_object(vm, obj)) + .transpose()? + .unwrap_or(0); + let scopeid = tuple + .get(3) + .map(|obj| u32::try_from_borrowed_object(vm, obj)) + .transpose()? + .unwrap_or(0); + if flowinfo > 0xfffff { + return Err(vm.new_overflow_error("flowinfo must be 0-1048575.")); + } + Ok((addr, flowinfo, scopeid)) + } + } + + fn get_ip_addr_tuple(addr: &SocketAddr, vm: &VirtualMachine) -> PyObjectRef { + match addr { + SocketAddr::V4(addr) => (addr.ip().to_string(), addr.port()).to_pyobject(vm), + SocketAddr::V6(addr) => ( + addr.ip().to_string(), + addr.port(), + addr.flowinfo(), + addr.scope_id(), + ) + .to_pyobject(vm), + } + } + + fn get_addr_tuple(addr: &socket2::SockAddr, vm: &VirtualMachine) -> PyObjectRef { + if let Some(addr) = addr.as_socket() { + return get_ip_addr_tuple(&addr, vm); + } + #[cfg(unix)] + if addr.is_unix() { + use std::os::unix::ffi::OsStrExt; + if let Some(abstractpath) = addr.as_abstract_namespace() { + return vm.ctx.new_bytes([b"\0", abstractpath].concat()).into(); + } + // necessary on macos + let path = ffi::OsStr::as_bytes(addr.as_pathname().unwrap_or("".as_ref()).as_ref()); + let nul_pos = memchr::memchr(b'\0', path).unwrap_or(path.len()); + let path = ffi::OsStr::from_bytes(&path[..nul_pos]); + return vm.fsdecode(path).into(); + } + #[cfg(target_os = "linux")] + { + let family = addr.family(); + if family == libc::AF_CAN as libc::sa_family_t { + // AF_CAN address: (interface_name,) or (interface_name, can_id) + let can_addr = unsafe { &*(addr.as_ptr() as *const libc::sockaddr_can) }; + let ifindex = can_addr.can_ifindex; + let ifname = if ifindex == 0 { + String::new() + } else { + let mut buf = [0u8; libc::IF_NAMESIZE]; + let ret = unsafe { + libc::if_indextoname( + ifindex as libc::c_uint, + buf.as_mut_ptr() as *mut libc::c_char, + ) + }; + if ret.is_null() { + String::new() + } else { + let nul_pos = memchr::memchr(b'\0', &buf).unwrap_or(buf.len()); + String::from_utf8_lossy(&buf[..nul_pos]).into_owned() + } + }; + return vm.ctx.new_tuple(vec![vm.ctx.new_str(ifname).into()]).into(); + } + if family == libc::AF_ALG as libc::sa_family_t { + // AF_ALG address: (type, name) + let alg_addr = unsafe { &*(addr.as_ptr() as *const libc::sockaddr_alg) }; + let type_bytes = &alg_addr.salg_type; + let name_bytes = &alg_addr.salg_name; + let type_nul = memchr::memchr(b'\0', type_bytes).unwrap_or(type_bytes.len()); + let name_nul = memchr::memchr(b'\0', name_bytes).unwrap_or(name_bytes.len()); + let type_str = String::from_utf8_lossy(&type_bytes[..type_nul]).into_owned(); + let name_str = String::from_utf8_lossy(&name_bytes[..name_nul]).into_owned(); + return vm + .ctx + .new_tuple(vec![ + vm.ctx.new_str(type_str).into(), + vm.ctx.new_str(name_str).into(), + ]) + .into(); + } + } + // TODO: support more address families + (String::new(), 0).to_pyobject(vm) + } + + #[pyfunction] + fn gethostname(vm: &VirtualMachine) -> PyResult<PyStrRef> { + gethostname::gethostname() + .into_string() + .map(|hostname| vm.ctx.new_str(hostname)) + .map_err(|err| vm.new_os_error(err.into_string().unwrap())) + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyfunction] + fn sethostname(hostname: PyUtf8StrRef) -> nix::Result<()> { + nix::unistd::sethostname(hostname.as_str()) + } + + #[pyfunction] + fn inet_aton(ip_string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + ip_string + .as_str() + .parse::<Ipv4Addr>() + .map(|ip_addr| Vec::<u8>::from(ip_addr.octets())) + .map_err(|_| { + vm.new_os_error("illegal IP address string passed to inet_aton".to_owned()) + }) + } + + #[pyfunction] + fn inet_ntoa(packed_ip: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let packed_ip = packed_ip.borrow_buf(); + let packed_ip = <&[u8; 4]>::try_from(&*packed_ip) + .map_err(|_| vm.new_os_error("packed IP wrong length for inet_ntoa".to_owned()))?; + Ok(vm.ctx.new_str(Ipv4Addr::from(*packed_ip).to_string())) + } + + fn cstr_opt_as_ptr(x: &OptionalArg<ffi::CString>) -> *const libc::c_char { + x.as_ref().map_or_else(core::ptr::null, |s| s.as_ptr()) + } + + #[pyfunction] + fn getservbyname( + servicename: PyStrRef, + protocolname: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<u16> { + let cstr_name = servicename.to_cstring(vm)?; + let cstr_proto = protocolname + .as_ref() + .map(|s| s.to_cstring(vm)) + .transpose()?; + let cstr_proto = cstr_opt_as_ptr(&cstr_proto); + let serv = unsafe { c::getservbyname(cstr_name.as_ptr() as _, cstr_proto as _) }; + if serv.is_null() { + return Err(vm.new_os_error("service/proto not found".to_owned())); + } + let port = unsafe { (*serv).s_port }; + Ok(u16::from_be(port as u16)) + } + + #[pyfunction] + fn getservbyport( + port: i32, + protocolname: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<String> { + let port = port + .to_u16() + .ok_or_else(|| vm.new_overflow_error("getservbyport: port must be 0-65535."))?; + let cstr_proto = protocolname + .as_ref() + .map(|s| s.to_cstring(vm)) + .transpose()?; + let cstr_proto = cstr_opt_as_ptr(&cstr_proto); + let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto as _) }; + if serv.is_null() { + return Err(vm.new_os_error("port/proto not found".to_owned())); + } + let s = unsafe { ffi::CStr::from_ptr((*serv).s_name as _) }; + Ok(s.to_string_lossy().into_owned()) + } + + unsafe fn slice_as_uninit<T>(v: &mut [T]) -> &mut [MaybeUninit<T>] { + unsafe { &mut *(v as *mut [T] as *mut [MaybeUninit<T>]) } + } + + enum IoOrPyException { + Timeout, + Py(PyBaseExceptionRef), + Io(io::Error), + } + impl From<PyBaseExceptionRef> for IoOrPyException { + fn from(exc: PyBaseExceptionRef) -> Self { + Self::Py(exc) + } + } + impl From<PyRef<PyOSError>> for IoOrPyException { + fn from(exc: PyRef<PyOSError>) -> Self { + Self::Py(exc.upcast()) + } + } + impl From<io::Error> for IoOrPyException { + fn from(err: io::Error) -> Self { + Self::Io(err) + } + } + impl IoOrPyException { + fn errno(self) -> PyResult<i32> { + match self { + Self::Timeout => Ok(errcode!(EWOULDBLOCK)), + Self::Io(err) => Ok(err.posix_errno()), + Self::Py(exc) => Err(exc), + } + } + } + impl IntoPyException for IoOrPyException { + #[inline] + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + match self { + Self::Timeout => timeout_error(vm).upcast(), + Self::Py(exc) => exc, + Self::Io(err) => err.into_pyexception(vm), + } + } + } + + #[derive(Copy, Clone)] + pub(crate) enum SelectKind { + Read, + Write, + Connect, + } + + /// returns true if timed out + pub(crate) fn sock_select( + sock: &Socket, + kind: SelectKind, + interval: Option<Duration>, + ) -> io::Result<bool> { + #[cfg(unix)] + { + use nix::poll::*; + use std::os::fd::AsFd; + let events = match kind { + SelectKind::Read => PollFlags::POLLIN, + SelectKind::Write => PollFlags::POLLOUT, + SelectKind::Connect => PollFlags::POLLOUT | PollFlags::POLLERR, + }; + let mut pollfd = [PollFd::new(sock.as_fd(), events)]; + let timeout = match interval { + Some(d) => d.try_into().unwrap_or(PollTimeout::MAX), + None => PollTimeout::NONE, + }; + let ret = poll(&mut pollfd, timeout)?; + Ok(ret == 0) + } + #[cfg(windows)] + { + use crate::select; + + let fd = sock_fileno(sock); + + let mut reads = select::FdSet::new(); + let mut writes = select::FdSet::new(); + let mut errs = select::FdSet::new(); + + let fd = fd as usize; + match kind { + SelectKind::Read => reads.insert(fd), + SelectKind::Write => writes.insert(fd), + SelectKind::Connect => { + writes.insert(fd); + errs.insert(fd); + } + } + + let mut interval = interval.map(|dur| select::timeval { + tv_sec: dur.as_secs() as _, + tv_usec: dur.subsec_micros() as _, + }); + + select::select( + fd as i32 + 1, + &mut reads, + &mut writes, + &mut errs, + interval.as_mut(), + ) + .map(|ret| ret == 0) + } + } + + #[derive(FromArgs)] + struct GAIOptions { + #[pyarg(positional)] + host: Option<ArgStrOrBytesLike>, + #[pyarg(positional)] + port: Option<Either<ArgStrOrBytesLike, i32>>, + + #[pyarg(positional, default = c::AF_UNSPEC)] + family: i32, + #[pyarg(positional, default = 0)] + ty: i32, + #[pyarg(positional, default = 0)] + proto: i32, + #[pyarg(positional, default = 0)] + flags: i32, + } + + #[pyfunction] + fn getaddrinfo( + opts: GAIOptions, + vm: &VirtualMachine, + ) -> Result<Vec<PyObjectRef>, IoOrPyException> { + let hints = dns_lookup::AddrInfoHints { + socktype: opts.ty, + protocol: opts.proto, + address: opts.family, + flags: opts.flags, + }; + + // Encode host: str uses IDNA encoding, bytes must be valid UTF-8 + let host_encoded: Option<String> = match opts.host.as_ref() { + Some(ArgStrOrBytesLike::Str(s)) => { + let encoded = + vm.state + .codec_registry + .encode_text(s.to_owned(), "idna", None, vm)?; + let host_str = core::str::from_utf8(encoded.as_bytes()) + .map_err(|_| vm.new_runtime_error("idna output is not utf8"))?; + Some(host_str.to_owned()) + } + Some(ArgStrOrBytesLike::Buf(b)) => { + let bytes = b.borrow_buf(); + let host_str = core::str::from_utf8(&bytes) + .map_err(|_| vm.new_unicode_decode_error("host bytes is not utf8"))?; + Some(host_str.to_owned()) + } + None => None, + }; + let host = host_encoded.as_deref(); + + // Encode port: str/bytes as service name, int as port number + let port_encoded: Option<String> = match opts.port.as_ref() { + Some(Either::A(sb)) => { + let port_str = match sb { + ArgStrOrBytesLike::Str(s) => { + // For str, check for surrogates and raise UnicodeEncodeError if found + s.to_str() + .ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed"))? + .to_owned() + } + ArgStrOrBytesLike::Buf(b) => { + // For bytes, check if it's valid UTF-8 + let bytes = b.borrow_buf(); + core::str::from_utf8(&bytes) + .map_err(|_| vm.new_unicode_decode_error("port is not utf8"))? + .to_owned() + } + }; + Some(port_str) + } + Some(Either::B(i)) => Some(i.to_string()), + None => None, + }; + let port = port_encoded.as_deref(); + + let addrs = dns_lookup::getaddrinfo(host, port, Some(hints)) + .map_err(|err| convert_socket_error(vm, err, SocketError::GaiError))?; + + let list = addrs + .map(|ai| { + ai.map(|ai| { + vm.new_tuple(( + ai.address, + ai.socktype, + ai.protocol, + ai.canonname, + get_ip_addr_tuple(&ai.sockaddr, vm), + )) + .into() + }) + }) + .collect::<io::Result<Vec<_>>>()?; + Ok(list) + } + + #[pyfunction] + fn gethostbyaddr( + addr: PyUtf8StrRef, + vm: &VirtualMachine, + ) -> Result<(String, PyListRef, PyListRef), IoOrPyException> { + let addr = get_addr(vm, addr, c::AF_UNSPEC)?; + let (hostname, _) = dns_lookup::getnameinfo(&addr, 0) + .map_err(|e| convert_socket_error(vm, e, SocketError::HError))?; + Ok(( + hostname, + vm.ctx.new_list(vec![]), + vm.ctx + .new_list(vec![vm.ctx.new_str(addr.ip().to_string()).into()]), + )) + } + + #[pyfunction] + fn gethostbyname(name: PyUtf8StrRef, vm: &VirtualMachine) -> Result<String, IoOrPyException> { + let addr = get_addr(vm, name, c::AF_INET)?; + match addr { + SocketAddr::V4(ip) => Ok(ip.ip().to_string()), + _ => unreachable!(), + } + } + + #[pyfunction] + fn gethostbyname_ex( + name: PyUtf8StrRef, + vm: &VirtualMachine, + ) -> Result<(String, PyListRef, PyListRef), IoOrPyException> { + let addr = get_addr(vm, name, c::AF_INET)?; + let (hostname, _) = dns_lookup::getnameinfo(&addr, 0) + .map_err(|e| convert_socket_error(vm, e, SocketError::HError))?; + Ok(( + hostname, + vm.ctx.new_list(vec![]), + vm.ctx + .new_list(vec![vm.ctx.new_str(addr.ip().to_string()).into()]), + )) + } + + #[pyfunction] + fn inet_pton(af_inet: i32, ip_string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + static ERROR_MSG: &str = "illegal IP address string passed to inet_pton"; + let ip_addr = match af_inet { + c::AF_INET => ip_string + .as_str() + .parse::<Ipv4Addr>() + .map_err(|_| vm.new_os_error(ERROR_MSG.to_owned()))? + .octets() + .to_vec(), + c::AF_INET6 => ip_string + .as_str() + .parse::<Ipv6Addr>() + .map_err(|_| vm.new_os_error(ERROR_MSG.to_owned()))? + .octets() + .to_vec(), + _ => return Err(vm.new_os_error("Address family not supported by protocol".to_owned())), + }; + Ok(ip_addr) + } + + #[pyfunction] + fn inet_ntop(af_inet: i32, packed_ip: ArgBytesLike, vm: &VirtualMachine) -> PyResult<String> { + let packed_ip = packed_ip.borrow_buf(); + match af_inet { + c::AF_INET => { + let packed_ip = <&[u8; 4]>::try_from(&*packed_ip).map_err(|_| { + vm.new_value_error("invalid length of packed IP address string") + })?; + Ok(Ipv4Addr::from(*packed_ip).to_string()) + } + c::AF_INET6 => { + let packed_ip = <&[u8; 16]>::try_from(&*packed_ip).map_err(|_| { + vm.new_value_error("invalid length of packed IP address string") + })?; + Ok(get_ipv6_addr_str(Ipv6Addr::from(*packed_ip))) + } + _ => Err(vm.new_value_error(format!("unknown address family {af_inet}"))), + } + } + + #[pyfunction] + fn getprotobyname(name: PyStrRef, vm: &VirtualMachine) -> PyResult { + let cstr = name.to_cstring(vm)?; + let proto = unsafe { c::getprotobyname(cstr.as_ptr() as _) }; + if proto.is_null() { + return Err(vm.new_os_error("protocol not found".to_owned())); + } + let num = unsafe { (*proto).p_proto }; + Ok(vm.ctx.new_int(num).into()) + } + + #[pyfunction] + fn getnameinfo( + address: PyTupleRef, + flags: i32, + vm: &VirtualMachine, + ) -> Result<(String, String), IoOrPyException> { + match address.len() { + 2..=4 => {} + _ => { + return Err(vm.new_type_error("illegal sockaddr argument").into()); + } + } + let (addr, flowinfo, scopeid) = Address::from_tuple_ipv6(&address, vm)?; + let hints = dns_lookup::AddrInfoHints { + address: c::AF_UNSPEC, + socktype: c::SOCK_DGRAM, + flags: c::AI_NUMERICHOST, + protocol: 0, + }; + let service = addr.port.to_string(); + let host_str = addr.host.as_str(); + let mut res = dns_lookup::getaddrinfo(Some(host_str), Some(&service), Some(hints)) + .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError))? + .filter_map(Result::ok); + let mut ainfo = res.next().unwrap(); + if res.next().is_some() { + return Err(vm + .new_os_error("sockaddr resolved to multiple addresses".to_owned()) + .into()); + } + match &mut ainfo.sockaddr { + SocketAddr::V4(_) => { + if address.len() != 2 { + return Err(vm + .new_os_error("IPv4 sockaddr must be 2 tuple".to_owned()) + .into()); + } + } + SocketAddr::V6(addr) => { + addr.set_flowinfo(flowinfo); + addr.set_scope_id(scopeid); + } + } + dns_lookup::getnameinfo(&ainfo.sockaddr, flags) + .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError)) + } + + #[cfg(unix)] + #[pyfunction] + fn socketpair( + family: OptionalArg<i32>, + socket_kind: OptionalArg<i32>, + proto: OptionalArg<i32>, + ) -> Result<(PySocket, PySocket), IoOrPyException> { + let family = family.unwrap_or(libc::AF_UNIX); + let socket_kind = socket_kind.unwrap_or(libc::SOCK_STREAM); + let proto = proto.unwrap_or(0); + let (a, b) = Socket::pair(family.into(), socket_kind.into(), Some(proto.into()))?; + let py_a = PySocket::default(); + py_a.init_inner(family, socket_kind, proto, a)?; + let py_b = PySocket::default(); + py_b.init_inner(family, socket_kind, proto, b)?; + Ok((py_a, py_b)) + } + + #[cfg(all(unix, not(target_os = "redox")))] + type IfIndex = c::c_uint; + #[cfg(windows)] + type IfIndex = u32; // NET_IFINDEX but windows-sys 0.59 doesn't have it + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn if_nametoindex(name: FsPath, vm: &VirtualMachine) -> PyResult<IfIndex> { + let name = name.to_cstring(vm)?; + // in case 'if_nametoindex' does not set errno + crate::common::os::set_errno(libc::ENODEV); + let ret = unsafe { c::if_nametoindex(name.as_ptr() as _) }; + if ret == 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(ret) + } + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn if_indextoname(index: IfIndex, vm: &VirtualMachine) -> PyResult<String> { + let mut buf = [0; c::IF_NAMESIZE + 1]; + // in case 'if_indextoname' does not set errno + crate::common::os::set_errno(libc::ENXIO); + let ret = unsafe { c::if_indextoname(index, buf.as_mut_ptr()) }; + if ret.is_null() { + Err(vm.new_last_errno_error()) + } else { + let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as _) }; + Ok(buf.to_string_lossy().into_owned()) + } + } + + #[cfg(any( + windows, + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "ios", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd", + ))] + #[pyfunction] + fn if_nameindex(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + #[cfg(not(windows))] + { + let list = nix::net::if_::if_nameindex() + .map_err(|err| err.into_pyexception(vm))? + .to_slice() + .iter() + .map(|iface| { + let tup: (u32, String) = + (iface.index(), iface.name().to_string_lossy().into_owned()); + tup.to_pyobject(vm) + }) + .collect(); + + Ok(list) + } + #[cfg(windows)] + { + use windows_sys::Win32::NetworkManagement::Ndis::NET_LUID_LH; + + let table = MibTable::get_raw().map_err(|err| err.into_pyexception(vm))?; + let list = table.as_slice().iter().map(|entry| { + let name = + get_name(&entry.InterfaceLuid).map_err(|err| err.into_pyexception(vm))?; + let tup = (entry.InterfaceIndex, name.to_string_lossy()); + Ok(tup.to_pyobject(vm)) + }); + let list = list.collect::<PyResult<_>>()?; + return Ok(list); + + fn get_name(luid: &NET_LUID_LH) -> io::Result<widestring::WideCString> { + let mut buf = [0; c::IF_NAMESIZE + 1]; + let ret = unsafe { + IpHelper::ConvertInterfaceLuidToNameW(luid, buf.as_mut_ptr(), buf.len()) + }; + if ret == 0 { + Ok(widestring::WideCString::from_ustr_truncate( + widestring::WideStr::from_slice(&buf[..]), + )) + } else { + Err(io::Error::from_raw_os_error(ret as i32)) + } + } + struct MibTable { + ptr: core::ptr::NonNull<IpHelper::MIB_IF_TABLE2>, + } + impl MibTable { + fn get_raw() -> io::Result<Self> { + let mut ptr = core::ptr::null_mut(); + let ret = unsafe { IpHelper::GetIfTable2Ex(IpHelper::MibIfTableRaw, &mut ptr) }; + if ret == 0 { + let ptr = unsafe { core::ptr::NonNull::new_unchecked(ptr) }; + Ok(Self { ptr }) + } else { + Err(io::Error::from_raw_os_error(ret as i32)) + } + } + } + impl MibTable { + fn as_slice(&self) -> &[IpHelper::MIB_IF_ROW2] { + unsafe { + let p = self.ptr.as_ptr(); + let ptr = &raw const (*p).Table as *const IpHelper::MIB_IF_ROW2; + core::slice::from_raw_parts(ptr, (*p).NumEntries as usize) + } + } + } + impl Drop for MibTable { + fn drop(&mut self) { + unsafe { IpHelper::FreeMibTable(self.ptr.as_ptr() as *mut _) }; + } + } + } + } + + fn get_addr( + vm: &VirtualMachine, + pyname: PyUtf8StrRef, + af: i32, + ) -> Result<SocketAddr, IoOrPyException> { + let name = pyname.as_str(); + if name.is_empty() { + let hints = dns_lookup::AddrInfoHints { + address: af, + socktype: c::SOCK_DGRAM, + flags: c::AI_PASSIVE, + protocol: 0, + }; + let mut res = dns_lookup::getaddrinfo(None, Some("0"), Some(hints)) + .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError))?; + let ainfo = res.next().unwrap()?; + if res.next().is_some() { + return Err(vm + .new_os_error("wildcard resolved to multiple address".to_owned()) + .into()); + } + return Ok(ainfo.sockaddr); + } + if name == "255.255.255.255" || name == "<broadcast>" { + match af { + c::AF_INET | c::AF_UNSPEC => {} + _ => { + return Err(vm + .new_os_error("address family mismatched".to_owned()) + .into()); + } + } + return Ok(SocketAddr::V4(net::SocketAddrV4::new( + c::INADDR_BROADCAST.into(), + 0, + ))); + } + if let c::AF_INET | c::AF_UNSPEC = af + && let Ok(addr) = name.parse::<Ipv4Addr>() + { + return Ok(SocketAddr::V4(net::SocketAddrV4::new(addr, 0))); + } + if matches!(af, c::AF_INET | c::AF_UNSPEC) + && !name.contains('%') + && let Ok(addr) = name.parse::<Ipv6Addr>() + { + return Ok(SocketAddr::V6(net::SocketAddrV6::new(addr, 0, 0, 0))); + } + let hints = dns_lookup::AddrInfoHints { + address: af, + ..Default::default() + }; + let name = vm + .state + .codec_registry + .encode_text(pyname.into_wtf8(), "idna", None, vm)?; + let name = core::str::from_utf8(name.as_bytes()) + .map_err(|_| vm.new_runtime_error("idna output is not utf8"))?; + let mut res = dns_lookup::getaddrinfo(Some(name), None, Some(hints)) + .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError))?; + Ok(res.next().unwrap().map(|ainfo| ainfo.sockaddr)?) + } + + fn sock_from_raw(fileno: RawSocket, vm: &VirtualMachine) -> PyResult<Socket> { + let invalid = { + cfg_if::cfg_if! { + if #[cfg(windows)] { + fileno == INVALID_SOCKET + } else { + fileno < 0 + } + } + }; + if invalid { + return Err(vm.new_value_error("negative file descriptor")); + } + Ok(unsafe { sock_from_raw_unchecked(fileno) }) + } + /// SAFETY: fileno must not be equal to INVALID_SOCKET + unsafe fn sock_from_raw_unchecked(fileno: RawSocket) -> Socket { + #[cfg(unix)] + { + use std::os::unix::io::FromRawFd; + unsafe { Socket::from_raw_fd(fileno) } + } + #[cfg(windows)] + { + use std::os::windows::io::FromRawSocket; + unsafe { Socket::from_raw_socket(fileno) } + } + } + pub(super) fn sock_fileno(sock: &Socket) -> RawSocket { + #[cfg(unix)] + { + use std::os::unix::io::AsRawFd; + sock.as_raw_fd() + } + #[cfg(windows)] + { + use std::os::windows::io::AsRawSocket; + sock.as_raw_socket() + } + } + fn into_sock_fileno(sock: Socket) -> RawSocket { + #[cfg(unix)] + { + use std::os::unix::io::IntoRawFd; + sock.into_raw_fd() + } + #[cfg(windows)] + { + use std::os::windows::io::IntoRawSocket; + sock.into_raw_socket() + } + } + + pub(super) const INVALID_SOCKET: RawSocket = { + #[cfg(unix)] + { + -1 + } + #[cfg(windows)] + { + windows_sys::Win32::Networking::WinSock::INVALID_SOCKET as RawSocket + } + }; + + fn convert_socket_error( + vm: &VirtualMachine, + err: dns_lookup::LookupError, + err_kind: SocketError, + ) -> IoOrPyException { + if let dns_lookup::LookupErrorKind::System = err.kind() { + return io::Error::from(err).into(); + } + let strerr = { + #[cfg(unix)] + { + let s = match err_kind { + SocketError::GaiError => unsafe { + ffi::CStr::from_ptr(libc::gai_strerror(err.error_num())) + }, + SocketError::HError => unsafe { + ffi::CStr::from_ptr(libc::hstrerror(err.error_num())) + }, + }; + s.to_str().unwrap() + } + #[cfg(windows)] + { + "getaddrinfo failed" + } + }; + let exception_cls = match err_kind { + SocketError::GaiError => gaierror(vm), + SocketError::HError => herror(vm), + }; + vm.new_os_subtype_error(exception_cls, Some(err.error_num()), strerr) + .into() + } + + fn timeout_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + timeout_error_msg(vm, "timed out".to_owned()) + } + pub(crate) fn timeout_error_msg(vm: &VirtualMachine, msg: String) -> PyRef<PyOSError> { + vm.new_os_subtype_error(timeout(vm), None, msg) + } + + fn get_ipv6_addr_str(ipv6: Ipv6Addr) -> String { + match ipv6.to_ipv4() { + // instead of "::0.0.ddd.ddd" it's "::xxxx" + Some(v4) if !ipv6.is_unspecified() && matches!(v4.octets(), [0, 0, _, _]) => { + format!("::{:x}", u32::from(v4)) + } + _ => ipv6.to_string(), + } + } + + pub(crate) struct Deadline { + deadline: Instant, + } + + impl Deadline { + fn new(timeout: Duration) -> Self { + Self { + deadline: Instant::now() + timeout, + } + } + fn time_until(&self) -> Result<Duration, IoOrPyException> { + self.deadline + .checked_duration_since(Instant::now()) + // past the deadline already + .ok_or(IoOrPyException::Timeout) + } + } + + static DEFAULT_TIMEOUT: AtomicCell<f64> = AtomicCell::new(-1.0); + + #[pyfunction] + fn getdefaulttimeout() -> Option<f64> { + let timeout = DEFAULT_TIMEOUT.load(); + if timeout >= 0.0 { Some(timeout) } else { None } + } + + #[pyfunction] + fn setdefaulttimeout(timeout: Option<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<()> { + let val = match timeout { + Some(t) => { + let f = t.into_float(); + if f.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)")); + } + if f < 0.0 || !f.is_finite() { + return Err(vm.new_value_error("Timeout value out of range")); + } + f + } + None => -1.0, + }; + DEFAULT_TIMEOUT.store(val); + Ok(()) + } + + #[pyfunction] + fn dup(x: PyObjectRef, vm: &VirtualMachine) -> Result<RawSocket, IoOrPyException> { + let sock = get_raw_sock(x, vm)?; + let sock = core::mem::ManuallyDrop::new(sock_from_raw(sock, vm)?); + let newsock = sock.try_clone()?; + let fd = into_sock_fileno(newsock); + #[cfg(windows)] + crate::vm::stdlib::nt::raw_set_handle_inheritable(fd as _, false)?; + Ok(fd) + } + + #[pyfunction] + fn close(x: PyObjectRef, vm: &VirtualMachine) -> Result<(), IoOrPyException> { + Ok(close_inner(get_raw_sock(x, vm)?)?) + } + + fn close_inner(x: RawSocket) -> io::Result<()> { + #[cfg(unix)] + use libc::close; + #[cfg(windows)] + use windows_sys::Win32::Networking::WinSock::closesocket as close; + let ret = unsafe { close(x as _) }; + if ret < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() != Some(errcode!(ECONNRESET)) { + return Err(err); + } + } + Ok(()) + } + + enum SocketError { + HError, + GaiError, + } + + #[cfg(all(unix, not(target_os = "redox")))] + fn checked_cmsg_len(len: usize) -> Option<usize> { + // SAFETY: CMSG_LEN is always safe + let cmsg_len = |length| unsafe { libc::CMSG_LEN(length) }; + if len as u64 > (i32::MAX as u64 - cmsg_len(0) as u64) { + return None; + } + let res = cmsg_len(len as _) as usize; + if res > i32::MAX as usize || res < len { + return None; + } + Some(res) + } + + #[cfg(all(unix, not(target_os = "redox")))] + fn checked_cmsg_space(len: usize) -> Option<usize> { + // SAFETY: CMSG_SPACE is always safe + let cmsg_space = |length| unsafe { libc::CMSG_SPACE(length) }; + if len as u64 > (i32::MAX as u64 - cmsg_space(1) as u64) { + return None; + } + let res = cmsg_space(len as _) as usize; + if res > i32::MAX as usize || res < len { + return None; + } + Some(res) + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyfunction(name = "CMSG_LEN")] + fn cmsg_len(length: usize, vm: &VirtualMachine) -> PyResult<usize> { + checked_cmsg_len(length) + .ok_or_else(|| vm.new_overflow_error("CMSG_LEN() argument out of range")) + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyfunction(name = "CMSG_SPACE")] + fn cmsg_space(length: usize, vm: &VirtualMachine) -> PyResult<usize> { + checked_cmsg_space(length) + .ok_or_else(|| vm.new_overflow_error("CMSG_SPACE() argument out of range")) + } +} diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs new file mode 100644 index 00000000000..06c0010a79d --- /dev/null +++ b/crates/stdlib/src/ssl.rs @@ -0,0 +1,5176 @@ +// spell-checker: ignore ssleof aesccm aesgcm capath getblocking setblocking ENDTLS TLSEXT + +//! Pure Rust SSL/TLS implementation using rustls +//! +//! This module provides SSL/TLS support without requiring C dependencies. +//! It implements the Python ssl module API using: +//! - rustls: TLS protocol implementation +//! - x509-parser/x509-cert: Certificate parsing +//! - ring: Cryptographic primitives +//! - rustls-platform-verifier: Platform-native certificate verification +//! +//! DO NOT add openssl dependency here. +//! +//! Warning: This library contains AI-generated code and comments. Do not trust any code or comment without verification. Please have a qualified expert review the code and remove this notice after review. + +// OID (Object Identifier) management module +mod oid; + +// Certificate operations module (parsing, validation, conversion) +mod cert; + +// OpenSSL compatibility layer (abstracts rustls operations) +mod compat; + +// SSL exception types (shared with openssl backend) +mod error; + +pub(crate) use _ssl::module_def; + +#[allow(non_snake_case)] +#[allow(non_upper_case_globals)] +#[pymodule(with(error::ssl_error))] +mod _ssl { + use crate::{ + common::{ + hash::PyHash, + lock::{PyMutex, PyRwLock}, + }, + socket::{PySocket, SelectKind, sock_select, timeout_error_msg}, + vm::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyBytesRef, PyListRef, PyStrRef, PyType, PyTypeRef, + PyUtf8StrRef, + }, + convert::IntoPyException, + function::{ + ArgBytesLike, ArgMemoryBuffer, Either, FuncArgs, OptionalArg, PyComparisonValue, + }, + stdlib::_warnings, + types::{Comparable, Constructor, Hashable, PyComparisonOp, Representable}, + }, + }; + + // Import error types used in this module (others are exposed via pymodule(with(...))) + use super::error::{ + PySSLError, create_ssl_eof_error, create_ssl_want_read_error, create_ssl_want_write_error, + create_ssl_zero_return_error, + }; + use alloc::sync::Arc; + use core::{ + hash::{Hash, Hasher}, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, + }; + use rustls::crypto::aws_lc_rs::ALL_CIPHER_SUITES; + use std::{ + collections::{HashMap, hash_map::DefaultHasher}, + io::BufRead, + time::SystemTime, + }; + + // Rustls imports + use parking_lot::{Mutex as ParkingMutex, RwLock as ParkingRwLock}; + use pem_rfc7468::{LineEnding, encode_string}; + use rustls::{ + ClientConfig, ClientConnection, RootCertStore, ServerConfig, ServerConnection, + client::{ClientSessionMemoryCache, ClientSessionStore}, + crypto::SupportedKxGroup, + pki_types::{CertificateDer, CertificateRevocationListDer, PrivateKeyDer, ServerName}, + server::{ClientHello, ResolvesServerCert}, + sign::CertifiedKey, + version::{TLS12, TLS13}, + }; + use sha2::{Digest, Sha256}; + + // Import certificate operations module + use super::cert; + + // Import OID module + use super::oid; + + // Import compat module (OpenSSL compatibility layer) + use super::compat::{ + ClientConfigOptions, MultiCertResolver, ProtocolSettings, ServerConfigOptions, SslError, + TlsConnection, create_client_config, create_server_config, curve_name_to_kx_group, + extract_cipher_info, get_cipher_encryption_desc, is_blocking_io_error, + normalize_cipher_name, ssl_do_handshake, + }; + + // Type aliases for better readability + // Additional type alias for certificate/key pairs (SessionCache and SniCertName defined below) + + /// Certificate and private key pair used in SSL contexts + type CertKeyPair = (Arc<CertifiedKey>, PrivateKeyDer<'static>); + + // Constants matching Python ssl module + + // SSL/TLS Protocol versions + #[pyattr] + const PROTOCOL_TLS: i32 = 2; // Auto-negotiate best version + #[pyattr] + const PROTOCOL_SSLv23: i32 = PROTOCOL_TLS; // Alias for PROTOCOL_TLS + #[pyattr] + const PROTOCOL_TLS_CLIENT: i32 = 16; + #[pyattr] + const PROTOCOL_TLS_SERVER: i32 = 17; + + // Note: rustls doesn't support TLS 1.0/1.1 for security reasons + // These are defined for API compatibility but will raise errors if used + #[pyattr] + const PROTOCOL_TLSv1: i32 = 3; + #[pyattr] + const PROTOCOL_TLSv1_1: i32 = 4; + #[pyattr] + const PROTOCOL_TLSv1_2: i32 = 5; + #[pyattr] + const PROTOCOL_TLSv1_3: i32 = 6; + + // Protocol version constants for TLSVersion enum + #[pyattr] + const PROTO_SSLv3: i32 = 0x0300; + #[pyattr] + const PROTO_TLSv1: i32 = 0x0301; + #[pyattr] + const PROTO_TLSv1_1: i32 = 0x0302; + #[pyattr] + const PROTO_TLSv1_2: i32 = 0x0303; + #[pyattr] + const PROTO_TLSv1_3: i32 = 0x0304; + + // Minimum and maximum supported protocol versions for rustls + // Use special values -2 and -1 to avoid enum name conflicts + #[pyattr] + const PROTO_MINIMUM_SUPPORTED: i32 = -2; // special value + #[pyattr] + const PROTO_MAXIMUM_SUPPORTED: i32 = -1; // special value + + // Internal constants for rustls actual supported versions + // rustls only supports TLS 1.2 and TLS 1.3 + const MINIMUM_VERSION: i32 = PROTO_TLSv1_2; // 0x0303 + const MAXIMUM_VERSION: i32 = PROTO_TLSv1_3; // 0x0304 + + // Buffer sizes and limits (OpenSSL/CPython compatibility) + const PEM_BUFSIZE: usize = 1024; + // OpenSSL: ssl/ssl_local.h + const SSL3_RT_MAX_PLAIN_LENGTH: usize = 16384; + // SSL session cache size (common practice, similar to OpenSSL defaults) + const SSL_SESSION_CACHE_SIZE: usize = 256; + + // Certificate verification modes + #[pyattr] + const CERT_NONE: i32 = 0; + #[pyattr] + const CERT_OPTIONAL: i32 = 1; + #[pyattr] + const CERT_REQUIRED: i32 = 2; + + // Certificate requirements + #[pyattr] + const VERIFY_DEFAULT: i32 = 0; + #[pyattr] + const VERIFY_CRL_CHECK_LEAF: i32 = 4; + #[pyattr] + const VERIFY_CRL_CHECK_CHAIN: i32 = 12; + #[pyattr] + const VERIFY_X509_STRICT: i32 = 32; + #[pyattr] + const VERIFY_ALLOW_PROXY_CERTS: i32 = 64; + #[pyattr] + const VERIFY_X509_TRUSTED_FIRST: i32 = 32768; + #[pyattr] + const VERIFY_X509_PARTIAL_CHAIN: i32 = 0x80000; + + // Options (OpenSSL-compatible flags, mostly no-op in rustls) + #[pyattr] + const OP_NO_SSLv2: i32 = 0x00000000; // Not supported anyway + #[pyattr] + const OP_NO_SSLv3: i32 = 0x02000000; + #[pyattr] + const OP_NO_TLSv1: i32 = 0x04000000; + #[pyattr] + const OP_NO_TLSv1_1: i32 = 0x10000000; + #[pyattr] + const OP_NO_TLSv1_2: i32 = 0x08000000; + #[pyattr] + const OP_NO_TLSv1_3: i32 = 0x20000000; + #[pyattr] + const OP_NO_COMPRESSION: i32 = 0x00020000; + #[pyattr] + const OP_CIPHER_SERVER_PREFERENCE: i32 = 0x00400000; + #[pyattr] + const OP_SINGLE_DH_USE: i32 = 0x00000000; // No-op in rustls + #[pyattr] + const OP_SINGLE_ECDH_USE: i32 = 0x00000000; // No-op in rustls + #[pyattr] + const OP_NO_TICKET: i32 = 0x00004000; + #[pyattr] + const OP_LEGACY_SERVER_CONNECT: i32 = 0x00000004; + #[pyattr] + const OP_NO_RENEGOTIATION: i32 = 0x40000000; + #[pyattr] + const OP_IGNORE_UNEXPECTED_EOF: i32 = 0x00000080; + #[pyattr] + const OP_ENABLE_MIDDLEBOX_COMPAT: i32 = 0x00100000; + #[pyattr] + const OP_ALL: i32 = 0x00000BFB; // Combined "safe" options (reduced for i32, excluding OP_LEGACY_SERVER_CONNECT for OpenSSL 3.0.0+ compatibility) + + // Alert types (matching _TLSAlertType enum) + #[pyattr] + const ALERT_DESCRIPTION_CLOSE_NOTIFY: i32 = 0; + #[pyattr] + const ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: i32 = 10; + #[pyattr] + const ALERT_DESCRIPTION_BAD_RECORD_MAC: i32 = 20; + #[pyattr] + const ALERT_DESCRIPTION_DECRYPTION_FAILED: i32 = 21; + #[pyattr] + const ALERT_DESCRIPTION_RECORD_OVERFLOW: i32 = 22; + #[pyattr] + const ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: i32 = 30; + #[pyattr] + const ALERT_DESCRIPTION_HANDSHAKE_FAILURE: i32 = 40; + #[pyattr] + const ALERT_DESCRIPTION_NO_CERTIFICATE: i32 = 41; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE: i32 = 42; + #[pyattr] + const ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: i32 = 43; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_REVOKED: i32 = 44; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: i32 = 45; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: i32 = 46; + #[pyattr] + const ALERT_DESCRIPTION_ILLEGAL_PARAMETER: i32 = 47; + #[pyattr] + const ALERT_DESCRIPTION_UNKNOWN_CA: i32 = 48; + #[pyattr] + const ALERT_DESCRIPTION_ACCESS_DENIED: i32 = 49; + #[pyattr] + const ALERT_DESCRIPTION_DECODE_ERROR: i32 = 50; + #[pyattr] + const ALERT_DESCRIPTION_DECRYPT_ERROR: i32 = 51; + #[pyattr] + const ALERT_DESCRIPTION_EXPORT_RESTRICTION: i32 = 60; + #[pyattr] + const ALERT_DESCRIPTION_PROTOCOL_VERSION: i32 = 70; + #[pyattr] + const ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: i32 = 71; + #[pyattr] + const ALERT_DESCRIPTION_INTERNAL_ERROR: i32 = 80; + #[pyattr] + const ALERT_DESCRIPTION_INAPPROPRIATE_FALLBACK: i32 = 86; + #[pyattr] + const ALERT_DESCRIPTION_USER_CANCELLED: i32 = 90; + #[pyattr] + const ALERT_DESCRIPTION_NO_RENEGOTIATION: i32 = 100; + #[pyattr] + const ALERT_DESCRIPTION_MISSING_EXTENSION: i32 = 109; + #[pyattr] + const ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: i32 = 110; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: i32 = 111; + #[pyattr] + const ALERT_DESCRIPTION_UNRECOGNIZED_NAME: i32 = 112; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: i32 = 113; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: i32 = 114; + #[pyattr] + const ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: i32 = 115; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_REQUIRED: i32 = 116; + #[pyattr] + const ALERT_DESCRIPTION_NO_APPLICATION_PROTOCOL: i32 = 120; + + // Version info - reporting as OpenSSL 3.3.0 for compatibility + #[pyattr] + const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; // OpenSSL 3.3.0 (808452096) + #[pyattr] + const OPENSSL_VERSION: &str = "OpenSSL 3.3.0 (rustls/0.23)"; + #[pyattr] + const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release + #[pyattr] + const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release + + // Default cipher list for rustls - using modern secure ciphers + #[pyattr] + const _DEFAULT_CIPHERS: &str = + "TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256"; + + // Has features + #[pyattr] + const HAS_SNI: bool = true; + #[pyattr] + const HAS_TLS_UNIQUE: bool = false; // Not supported + #[pyattr] + const HAS_ECDH: bool = true; + #[pyattr] + const HAS_NPN: bool = false; // Deprecated, use ALPN + #[pyattr] + const HAS_ALPN: bool = true; + #[pyattr] + const HAS_PSK: bool = false; // PSK not supported in rustls + #[pyattr] + const HAS_SSLv2: bool = false; + #[pyattr] + const HAS_SSLv3: bool = false; + #[pyattr] + const HAS_TLSv1: bool = false; // Not supported for security + #[pyattr] + const HAS_TLSv1_1: bool = false; // Not supported for security + #[pyattr] + const HAS_TLSv1_2: bool = true; // rustls supports TLS 1.2 + #[pyattr] + const HAS_TLSv1_3: bool = true; + #[pyattr] + const HAS_PHA: bool = false; // Post-Handshake Auth not supported in rustls + + // Encoding constants (matching OpenSSL) + #[pyattr] + const ENCODING_PEM: i32 = 1; + #[pyattr] + const ENCODING_DER: i32 = 2; + #[pyattr] + const ENCODING_PEM_AUX: i32 = 0x101; // PEM + 0x100 + + /// Validate server hostname for TLS SNI + /// + /// Checks that the hostname: + /// - Is not empty + /// - Does not start with a dot + /// - Is not an IP address (SNI requires DNS names) + /// - Does not contain null bytes + /// - Does not exceed 253 characters (DNS limit) + /// + /// Returns Ok(()) if validation passes, or an appropriate error. + fn validate_hostname(hostname: &str, vm: &VirtualMachine) -> PyResult<()> { + if hostname.is_empty() { + return Err(vm.new_value_error("server_hostname cannot be an empty string")); + } + + if hostname.starts_with('.') { + return Err(vm.new_value_error("server_hostname cannot start with a dot")); + } + + // IP addresses are allowed as server_hostname + // SNI will not be sent for IP addresses + + if hostname.contains('\0') { + return Err(vm.new_type_error("embedded null character")); + } + + if hostname.len() > 253 { + return Err(vm.new_value_error("server_hostname is too long (maximum 253 characters)")); + } + + Ok(()) + } + + // SNI certificate resolver that uses shared mutable state + // The Python SNI callback updates this state, and resolve() reads from it + #[derive(Debug)] + struct SniCertResolver { + // SNI state: (certificate, server_name) + sni_state: Arc<ParkingMutex<SniCertName>>, + } + + impl ResolvesServerCert for SniCertResolver { + fn resolve(&self, client_hello: ClientHello<'_>) -> Option<Arc<CertifiedKey>> { + let mut state = self.sni_state.lock(); + + // Extract and store SNI from client hello for later use + if let Some(sni) = client_hello.server_name() { + state.1 = Some(sni.to_string()); + } else { + state.1 = None; + } + + // Return the current certificate (may have been updated by Python callback) + Some(state.0.clone()) + } + } + + // Session data structure for tracking TLS sessions + #[derive(Debug, Clone)] + struct SessionData { + #[allow(dead_code)] + server_name: String, + session_id: Vec<u8>, + creation_time: SystemTime, + lifetime: u64, + } + + // Type alias to simplify complex session cache type + type SessionCache = Arc<ParkingRwLock<HashMap<Vec<u8>, Arc<ParkingMutex<SessionData>>>>>; + + // Type alias for SNI state + type SniCertName = (Arc<CertifiedKey>, Option<String>); + + // SESSION EMULATION IMPLEMENTATION + // + // IMPORTANT: This is an EMULATION of CPython's SSL session management. + // Rustls 0.23 does NOT expose session data (ticket bytes, session IDs, etc.) + // through public APIs. All session value fields are private. + // + // LIMITATIONS: + // - Session IDs are generated from metadata (server name + timestamp hash) + // NOT actual TLS session IDs + // - Ticket data is not stored (Rustls keeps it internally) + // - Session resumption works (via Rustls's automatic mechanism) + // but we can't access the actual session state + // + // This implementation provides: + // ✓ session.id - synthetic ID based on metadata + // ✓ session.time - creation timestamp + // ✓ session.timeout - default lifetime value + // ✓ session.has_ticket - always True when session exists + // ✓ session_reused - tracked via handshake_kind() + // ✗ Actual TLS session ID/ticket data - NOT ACCESSIBLE + + // Generate a synthetic session ID from server name and timestamp + // NOTE: This is NOT the actual TLS session ID, just a unique identifier + fn generate_session_id_from_metadata(server_name: &str, time: &SystemTime) -> Vec<u8> { + let mut hasher = Sha256::new(); + hasher.update(server_name.as_bytes()); + hasher.update( + time.duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + .to_le_bytes(), + ); + hasher.finalize()[..16].to_vec() + } + + // Custom ClientSessionStore that tracks session metadata for Python access + // NOTE: This wraps ClientSessionMemoryCache and records metadata when sessions are stored + #[derive(Debug)] + struct PythonClientSessionStore { + inner: Arc<ClientSessionMemoryCache>, + session_cache: SessionCache, + } + + impl ClientSessionStore for PythonClientSessionStore { + fn set_kx_hint(&self, server_name: ServerName<'static>, group: rustls::NamedGroup) { + self.inner.set_kx_hint(server_name, group); + } + + fn kx_hint(&self, server_name: &ServerName<'_>) -> Option<rustls::NamedGroup> { + self.inner.kx_hint(server_name) + } + + fn set_tls12_session( + &self, + server_name: ServerName<'static>, + value: rustls::client::Tls12ClientSessionValue, + ) { + // Store in inner cache for actual resumption (Rustls handles this) + self.inner.set_tls12_session(server_name.clone(), value); + + // Record metadata in Python-accessible cache + // NOTE: We can't access value.session_id or value.ticket (private fields) + // So we generate a synthetic ID from metadata + let creation_time = SystemTime::now(); + let server_name_str = server_name.to_str(); + let session_data = SessionData { + server_name: server_name_str.as_ref().to_string(), + session_id: generate_session_id_from_metadata( + server_name_str.as_ref(), + &creation_time, + ), + creation_time, + lifetime: 7200, // TLS 1.2 default session lifetime + }; + + let key = server_name_str.as_bytes().to_vec(); + self.session_cache + .write() + .insert(key, Arc::new(ParkingMutex::new(session_data))); + } + + fn tls12_session( + &self, + server_name: &ServerName<'_>, + ) -> Option<rustls::client::Tls12ClientSessionValue> { + self.inner.tls12_session(server_name) + } + + fn remove_tls12_session(&self, server_name: &ServerName<'static>) { + self.inner.remove_tls12_session(server_name); + + // Also remove from Python cache + let key = server_name.to_str().as_bytes().to_vec(); + self.session_cache.write().remove(&key); + } + + fn insert_tls13_ticket( + &self, + server_name: ServerName<'static>, + value: rustls::client::Tls13ClientSessionValue, + ) { + // Store in inner cache for actual resumption (Rustls handles this) + self.inner.insert_tls13_ticket(server_name.clone(), value); + + // Record metadata in Python-accessible cache + // NOTE: We can't access value.ticket or value.lifetime_secs (private fields) + // So we use default values + let creation_time = SystemTime::now(); + let server_name_str = server_name.to_str(); + let session_data = SessionData { + server_name: server_name_str.to_string(), + session_id: generate_session_id_from_metadata( + server_name_str.as_ref(), + &creation_time, + ), + creation_time, + lifetime: 7200, // Default TLS 1.3 ticket lifetime (Rustls uses this) + }; + + let key = server_name_str.as_bytes().to_vec(); + self.session_cache + .write() + .insert(key, Arc::new(ParkingMutex::new(session_data))); + } + + fn take_tls13_ticket( + &self, + server_name: &ServerName<'static>, + ) -> Option<rustls::client::Tls13ClientSessionValue> { + self.inner.take_tls13_ticket(server_name) + } + } + + /// Parse length-prefixed ALPN protocol list + /// + /// Format: [len1, proto1..., len2, proto2..., ...] + /// + /// This is the wire format used by Python's ssl.py when calling _set_alpn_protocols(). + /// Each protocol is prefixed with a single byte indicating its length. + /// + /// # Arguments + /// * `bytes` - The length-prefixed protocol data + /// * `vm` - VirtualMachine for error creation + /// + /// # Returns + /// * `Ok(Vec<Vec<u8>>)` - List of protocol names as byte vectors + /// * `Err(PyBaseExceptionRef)` - ValueError with detailed error message + fn parse_length_prefixed_alpn(bytes: &[u8], vm: &VirtualMachine) -> PyResult<Vec<Vec<u8>>> { + let mut alpn_list = Vec::new(); + let mut offset = 0; + + while offset < bytes.len() { + // Check if we can read the length byte + if offset + 1 > bytes.len() { + return Err(vm.new_value_error(format!( + "Invalid ALPN protocol data: unexpected end at offset {offset}", + ))); + } + + let proto_len = bytes[offset] as usize; + offset += 1; + + // Validate protocol length + if proto_len == 0 { + return Err(vm.new_value_error(format!( + "Invalid ALPN protocol data: protocol length cannot be 0 at offset {}", + offset - 1 + ))); + } + + // Check if we have enough bytes for the protocol data + if offset + proto_len > bytes.len() { + return Err(vm.new_value_error(format!( + "Invalid ALPN protocol data: expected {} bytes at offset {}, but only {} bytes remain", + proto_len, offset, bytes.len() - offset + ))); + } + + // Extract protocol bytes + let proto = bytes[offset..offset + proto_len].to_vec(); + alpn_list.push(proto); + offset += proto_len; + } + + Ok(alpn_list) + } + + /// Parse OpenSSL cipher string to rustls SupportedCipherSuite list + /// + /// Supports patterns like: + /// - "AES128" → filters for AES_128 + /// - "AES256" → filters for AES_256 + /// - "AES128:AES256" → both + /// - "ECDHE+AESGCM" → ECDHE AND AESGCM (both conditions must match) + /// - "ALL" or "DEFAULT" → all available + /// - "!MD5" → exclusion (ignored, rustls doesn't support weak ciphers anyway) + fn parse_cipher_string(cipher_str: &str) -> Result<Vec<rustls::SupportedCipherSuite>, String> { + if cipher_str.is_empty() { + return Err("No cipher can be selected".to_string()); + } + + let all_suites = ALL_CIPHER_SUITES; + let mut selected = Vec::new(); + + for part in cipher_str.split(':') { + let part = part.trim(); + + // Skip exclusions (rustls doesn't support these) + if part.starts_with('!') { + continue; + } + + // Skip priority markers starting with + + if part.starts_with('+') { + continue; + } + + // Match pattern + match part { + "ALL" | "DEFAULT" | "HIGH" => { + // Add all available cipher suites + selected.extend_from_slice(all_suites); + } + _ => { + // Check if this is a compound pattern with + (AND condition) + // e.g., "ECDHE+AESGCM" means ECDHE AND AESGCM + let patterns: Vec<&str> = part.split('+').collect(); + + let mut found_any = false; + for suite in all_suites { + let name = format!("{:?}", suite.suite()); + + // Check if all patterns match (AND condition) + let matches = patterns.iter().all(|&pattern| { + // Handle common OpenSSL pattern variations + if pattern.contains("AES128") { + name.contains("AES_128") + } else if pattern.contains("AES256") { + name.contains("AES_256") + } else if pattern == "AESGCM" { + // AESGCM: AES with GCM mode + name.contains("AES") && name.contains("GCM") + } else if pattern == "AESCCM" { + // AESCCM: AES with CCM mode + name.contains("AES") && name.contains("CCM") + } else if pattern == "CHACHA20" { + name.contains("CHACHA20") + } else if pattern == "ECDHE" { + name.contains("ECDHE") + } else if pattern == "DHE" { + // DHE but not ECDHE + name.contains("DHE") && !name.contains("ECDHE") + } else if pattern == "ECDH" { + // ECDH but not ECDHE + name.contains("ECDH") && !name.contains("ECDHE") + } else if pattern == "DH" { + // DH but not DHE or ECDH + name.contains("DH") + && !name.contains("DHE") + && !name.contains("ECDH") + } else if pattern == "RSA" { + name.contains("RSA") + } else if pattern == "AES" { + name.contains("AES") + } else if pattern == "ECDSA" { + name.contains("ECDSA") + } else { + // Direct substring match for other patterns + name.contains(pattern) + } + }); + + if matches { + selected.push(*suite); + found_any = true; + } + } + + if !found_any { + // No matching cipher suite found - warn but continue + } + } + } + } + + // Remove duplicates + selected.dedup_by_key(|s| s.suite()); + + if selected.is_empty() { + Err("No cipher can be selected".to_string()) + } else { + Ok(selected) + } + } + + // SSLContext - manages TLS configuration + #[pyattr] + #[pyclass(name = "_SSLContext", module = "ssl", traverse)] + #[derive(Debug, PyPayload)] + struct PySSLContext { + #[pytraverse(skip)] + protocol: i32, + #[pytraverse(skip)] + check_hostname: PyRwLock<bool>, + #[pytraverse(skip)] + verify_mode: PyRwLock<i32>, + #[pytraverse(skip)] + verify_flags: PyRwLock<i32>, + // Rustls configuration (built lazily) + #[allow(dead_code)] + #[pytraverse(skip)] + client_config: PyRwLock<Option<Arc<ClientConfig>>>, + #[allow(dead_code)] + #[pytraverse(skip)] + server_config: PyRwLock<Option<Arc<ServerConfig>>>, + // Certificate store + #[pytraverse(skip)] + root_certs: PyRwLock<RootCertStore>, + // Store full CA certificates for get_ca_certs() + // RootCertStore only keeps TrustAnchors, not full certificates + #[pytraverse(skip)] + ca_certs_der: PyRwLock<Vec<Vec<u8>>>, + // Store CA certificates from capath for lazy loading simulation + // (CPython only returns these in get_ca_certs() after they're used in handshake) + #[pytraverse(skip)] + capath_certs_der: PyRwLock<Vec<Vec<u8>>>, + // Certificate Revocation Lists for CRL checking + #[pytraverse(skip)] + crls: PyRwLock<Vec<CertificateRevocationListDer<'static>>>, + // Server certificate/key pairs (supports multiple for RSA+ECC dual mode) + // OpenSSL allows multiple cert/key pairs to be loaded, and selects the appropriate + // one based on client capabilities during handshake + // Stored as (CertifiedKey, PrivateKeyDer) to support both server and client usage + #[pytraverse(skip)] + cert_keys: PyRwLock<Vec<CertKeyPair>>, + // Options + #[allow(dead_code)] + #[pytraverse(skip)] + options: PyRwLock<i32>, + // ALPN protocols + #[allow(dead_code)] + #[pytraverse(skip)] + alpn_protocols: PyRwLock<Vec<Vec<u8>>>, + // ALPN strict matching flag + // When false (default), mimics OpenSSL behavior: no ALPN negotiation failure + // When true, requires ALPN match (Rustls default behavior) + #[allow(dead_code)] + #[pytraverse(skip)] + require_alpn_match: PyRwLock<bool>, + // TLS 1.3 features + #[pytraverse(skip)] + post_handshake_auth: PyRwLock<bool>, + #[pytraverse(skip)] + num_tickets: PyRwLock<i32>, + // Protocol version limits + #[pytraverse(skip)] + minimum_version: PyRwLock<i32>, + #[pytraverse(skip)] + maximum_version: PyRwLock<i32>, + // SNI callback for server-side (contains PyObjectRef - needs GC tracking) + sni_callback: PyRwLock<Option<PyObjectRef>>, + // Message callback for debugging (contains PyObjectRef - needs GC tracking) + msg_callback: PyRwLock<Option<PyObjectRef>>, + // ECDH curve name for key exchange + #[pytraverse(skip)] + ecdh_curve: PyRwLock<Option<String>>, + // Certificate statistics for cert_store_stats() + #[pytraverse(skip)] + ca_cert_count: PyRwLock<usize>, // Number of CA certificates + #[pytraverse(skip)] + x509_cert_count: PyRwLock<usize>, // Total number of certificates + // Session management + #[pytraverse(skip)] + client_session_cache: SessionCache, + // Rustls session store for actual TLS session resumption + #[pytraverse(skip)] + rustls_session_store: Arc<PythonClientSessionStore>, + // Rustls server session store for server-side session resumption + #[pytraverse(skip)] + rustls_server_session_store: Arc<rustls::server::ServerSessionMemoryCache>, + // Shared ticketer for TLS 1.2 session tickets + #[pytraverse(skip)] + server_ticketer: Arc<dyn rustls::server::ProducesTickets>, + // Server-side session statistics + #[pytraverse(skip)] + accept_count: AtomicUsize, // Total number of accepts + #[pytraverse(skip)] + session_hits: AtomicUsize, // Number of session reuses + // Cipher suite selection + /// Selected cipher suites (None = use all rustls defaults) + #[pytraverse(skip)] + selected_ciphers: PyRwLock<Option<Vec<rustls::SupportedCipherSuite>>>, + } + + #[derive(FromArgs)] + struct WrapSocketArgs { + sock: PyObjectRef, + server_side: bool, + #[pyarg(positional, optional)] + server_hostname: OptionalArg<Option<PyUtf8StrRef>>, + #[pyarg(named, optional)] + owner: OptionalArg<PyObjectRef>, + #[pyarg(named, optional)] + session: OptionalArg<PyObjectRef>, + } + + #[derive(FromArgs)] + struct WrapBioArgs { + incoming: PyRef<PyMemoryBIO>, + outgoing: PyRef<PyMemoryBIO>, + #[pyarg(named, optional)] + server_side: OptionalArg<bool>, + #[pyarg(named, optional)] + server_hostname: OptionalArg<Option<PyUtf8StrRef>>, + #[pyarg(named, optional)] + owner: OptionalArg<PyObjectRef>, + #[pyarg(named, optional)] + session: OptionalArg<PyObjectRef>, + } + + #[derive(FromArgs)] + struct LoadVerifyLocationsArgs { + #[pyarg(any, optional, error_msg = "path should be a str or bytes")] + cafile: OptionalArg<Option<Either<PyStrRef, ArgBytesLike>>>, + #[pyarg(any, optional, error_msg = "path should be a str or bytes")] + capath: OptionalArg<Option<Either<PyStrRef, ArgBytesLike>>>, + #[pyarg(any, optional, error_msg = "cadata should be a str or bytes")] + cadata: OptionalArg<Option<Either<PyStrRef, ArgBytesLike>>>, + } + + #[derive(FromArgs)] + struct LoadCertChainArgs { + #[pyarg(any, error_msg = "path should be a str or bytes")] + certfile: Either<PyStrRef, ArgBytesLike>, + #[pyarg(any, optional, error_msg = "path should be a str or bytes")] + keyfile: OptionalArg<Option<Either<PyStrRef, ArgBytesLike>>>, + #[pyarg(any, optional)] + password: OptionalArg<PyObjectRef>, + } + + #[derive(FromArgs)] + struct GetCertArgs { + #[pyarg(any, optional)] + binary_form: OptionalArg<bool>, + } + + #[pyclass(with(Constructor, Representable), flags(BASETYPE))] + impl PySSLContext { + // Helper method to convert DER certificate bytes to Python dict + fn cert_der_to_dict(&self, vm: &VirtualMachine, cert_der: &[u8]) -> PyResult<PyObjectRef> { + cert::cert_der_to_dict_helper(vm, cert_der) + } + + #[pygetset] + fn check_hostname(&self) -> bool { + *self.check_hostname.read() + } + + #[pygetset(setter)] + fn set_check_hostname(&self, value: bool) { + *self.check_hostname.write() = value; + // When check_hostname is enabled, ensure verify_mode is at least CERT_REQUIRED + if value { + let current_verify_mode = *self.verify_mode.read(); + if current_verify_mode == CERT_NONE { + *self.verify_mode.write() = CERT_REQUIRED; + } + } + } + + #[pygetset] + fn verify_mode(&self) -> i32 { + *self.verify_mode.read() + } + + #[pygetset(setter)] + fn set_verify_mode(&self, mode: i32, vm: &VirtualMachine) -> PyResult<()> { + if !(CERT_NONE..=CERT_REQUIRED).contains(&mode) { + return Err(vm.new_value_error("invalid verify mode")); + } + // Cannot set CERT_NONE when check_hostname is enabled + if mode == CERT_NONE && *self.check_hostname.read() { + return Err(vm.new_value_error( + "Cannot set verify_mode to CERT_NONE when check_hostname is enabled", + )); + } + *self.verify_mode.write() = mode; + Ok(()) + } + + #[pygetset] + fn protocol(&self) -> i32 { + self.protocol + } + + #[pygetset] + fn verify_flags(&self) -> i32 { + *self.verify_flags.read() + } + + #[pygetset(setter)] + fn set_verify_flags(&self, value: i32) { + *self.verify_flags.write() = value; + } + + #[pygetset] + fn post_handshake_auth(&self) -> bool { + *self.post_handshake_auth.read() + } + + #[pygetset(setter)] + fn set_post_handshake_auth(&self, value: bool) { + *self.post_handshake_auth.write() = value; + } + + #[pygetset] + fn num_tickets(&self) -> i32 { + *self.num_tickets.read() + } + + #[pygetset(setter)] + fn set_num_tickets(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + if value < 0 { + return Err(vm.new_value_error("num_tickets must be a non-negative integer")); + } + if self.protocol != PROTOCOL_TLS_SERVER { + return Err( + vm.new_value_error("num_tickets can only be set on server-side contexts") + ); + } + *self.num_tickets.write() = value; + Ok(()) + } + + #[pygetset] + fn options(&self) -> i32 { + *self.options.read() + } + + #[pygetset(setter)] + fn set_options(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + // Validate that the value is non-negative + if value < 0 { + return Err(vm.new_value_error("options must be non-negative")); + } + + // Deprecated SSL/TLS protocol version options + let opt_no = OP_NO_SSLv2 + | OP_NO_SSLv3 + | OP_NO_TLSv1 + | OP_NO_TLSv1_1 + | OP_NO_TLSv1_2 + | OP_NO_TLSv1_3; + + // Get current options and calculate newly set bits + let old_opts = *self.options.read(); + let set = !old_opts & value; // Bits being newly set + + // Warn if any deprecated options are being newly set + if (set & opt_no) != 0 { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + "ssl.OP_NO_SSL*/ssl.OP_NO_TLS* options are deprecated".to_owned(), + 2, // stack_level = 2 + vm, + )?; + } + + *self.options.write() = value; + Ok(()) + } + + #[pygetset] + fn minimum_version(&self) -> i32 { + let v = *self.minimum_version.read(); + // return MINIMUM_SUPPORTED if value is 0 + if v == 0 { PROTO_MINIMUM_SUPPORTED } else { v } + } + + #[pygetset(setter)] + fn set_minimum_version(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + // Validate that the value is a valid TLS version constant + // Valid values: 0 (default), -2 (MINIMUM_SUPPORTED), -1 (MAXIMUM_SUPPORTED), + // or 0x0300-0x0304 (SSLv3-TLSv1.3) + if value != 0 + && value != -2 + && value != -1 + && !(PROTO_SSLv3..=PROTO_TLSv1_3).contains(&value) + { + return Err(vm.new_value_error(format!("invalid protocol version: {value}"))); + } + // Convert special values to rustls actual supported versions + // MINIMUM_SUPPORTED (-2) -> 0 (auto-negotiate) + // MAXIMUM_SUPPORTED (-1) -> MAXIMUM_VERSION (TLSv1.3) + let normalized_value = match value { + PROTO_MINIMUM_SUPPORTED => 0, // Auto-negotiate + PROTO_MAXIMUM_SUPPORTED => MAXIMUM_VERSION, // TLSv1.3 + _ => value, + }; + *self.minimum_version.write() = normalized_value; + Ok(()) + } + + #[pygetset] + fn maximum_version(&self) -> i32 { + let v = *self.maximum_version.read(); + // return MAXIMUM_SUPPORTED if value is 0 + if v == 0 { PROTO_MAXIMUM_SUPPORTED } else { v } + } + + #[pygetset(setter)] + fn set_maximum_version(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + // Validate that the value is a valid TLS version constant + // Valid values: 0 (default), -2 (MINIMUM_SUPPORTED), -1 (MAXIMUM_SUPPORTED), + // or 0x0300-0x0304 (SSLv3-TLSv1.3) + if value != 0 + && value != -2 + && value != -1 + && !(PROTO_SSLv3..=PROTO_TLSv1_3).contains(&value) + { + return Err(vm.new_value_error(format!("invalid protocol version: {value}"))); + } + // Convert special values to rustls actual supported versions + // MAXIMUM_SUPPORTED (-1) -> 0 (auto-negotiate) + // MINIMUM_SUPPORTED (-2) -> MINIMUM_VERSION (TLSv1.2) + let normalized_value = match value { + PROTO_MAXIMUM_SUPPORTED => 0, // Auto-negotiate + PROTO_MINIMUM_SUPPORTED => MINIMUM_VERSION, // TLSv1.2 + _ => value, + }; + *self.maximum_version.write() = normalized_value; + Ok(()) + } + + #[pymethod] + fn load_cert_chain(&self, args: LoadCertChainArgs, vm: &VirtualMachine) -> PyResult<()> { + // Parse certfile argument (str or bytes) to path + let cert_path = Self::parse_path_arg(&args.certfile, vm)?; + + // Parse keyfile argument (default to certfile if not provided) + let key_path = match args.keyfile { + OptionalArg::Present(Some(ref k)) => Self::parse_path_arg(k, vm)?, + _ => cert_path.clone(), + }; + + // Parse password argument (str, bytes-like, or callable) + // Callable passwords are NOT invoked immediately (lazy evaluation) + let (password_str, password_callable) = + Self::parse_password_argument(&args.password, vm)?; + + // Validate immediate password length (limit: PEM_BUFSIZE = 1024 bytes) + if let Some(ref pwd) = password_str + && pwd.len() > PEM_BUFSIZE + { + return Err(vm.new_value_error(format!( + "password cannot be longer than {PEM_BUFSIZE} bytes", + ))); + } + + // First attempt: Load with immediate password (or None if callable) + let mut result = + cert::load_cert_chain_from_file(&cert_path, &key_path, password_str.as_deref()); + + // If failed and callable exists, invoke it and retry + // This implements lazy evaluation: callable only invoked if password is actually needed + if result.is_err() + && let Some(callable) = password_callable + { + // Invoke callable - exceptions propagate naturally + let pwd_result = callable.call((), vm)?; + + // Convert callable result to string + let password_from_callable = if let Ok(pwd_str) = + PyUtf8StrRef::try_from_object(vm, pwd_result.clone()) + { + pwd_str.as_str().to_owned() + } else if let Ok(pwd_bytes_like) = ArgBytesLike::try_from_object(vm, pwd_result) { + String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()).map_err(|_| { + vm.new_type_error("password callback returned invalid UTF-8 bytes") + })? + } else { + return Err( + vm.new_type_error("password callback must return a string or bytes") + ); + }; + + // Validate callable password length + if password_from_callable.len() > PEM_BUFSIZE { + return Err(vm.new_value_error(format!( + "password cannot be longer than {PEM_BUFSIZE} bytes", + ))); + } + + // Retry with callable password + result = cert::load_cert_chain_from_file( + &cert_path, + &key_path, + Some(&password_from_callable), + ); + } + + // Process result + let (certs, key) = result.map_err(|e| { + // Try to downcast to io::Error to preserve errno information + if let Ok(io_err) = e.downcast::<std::io::Error>() { + match io_err.kind() { + // File access errors (NotFound, PermissionDenied) - preserve errno + std::io::ErrorKind::NotFound | std::io::ErrorKind::PermissionDenied => { + io_err.into_pyexception(vm) + } + // Other io::Error types + std::io::ErrorKind::Other => { + let msg = io_err.to_string(); + if msg.contains("Failed to decrypt") || msg.contains("wrong password") { + // Wrong password error + vm.new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + msg, + ) + .upcast() + } else { + // [SSL] PEM lib + super::compat::SslError::create_ssl_error_with_reason( + vm, + Some("SSL"), + "", + "PEM lib", + ) + } + } + // PEM parsing errors - [SSL] PEM lib + _ => super::compat::SslError::create_ssl_error_with_reason( + vm, + Some("SSL"), + "", + "PEM lib", + ), + } + } else { + // Unknown error type - [SSL] PEM lib + super::compat::SslError::create_ssl_error_with_reason( + vm, + Some("SSL"), + "", + "PEM lib", + ) + } + })?; + + // Validate certificate and key match + cert::validate_cert_key_match(&certs, &key).map_err(|e| { + let msg = if e.contains("key values mismatch") { + "[SSL: KEY_VALUES_MISMATCH] key values mismatch".to_owned() + } else { + e + }; + vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), Some(0), msg) + .upcast() + })?; + + // Auto-build certificate chain: if only leaf cert is in file, try to add CA certs + // This matches OpenSSL behavior where it automatically includes intermediate/CA certs + let mut full_chain = certs.clone(); + if full_chain.len() == 1 { + // Only have leaf cert, try to build chain from CA certs + let ca_certs_der = self.ca_certs_der.read(); + if !ca_certs_der.is_empty() { + // Use build_verified_chain to construct full chain + let chain_result = cert::build_verified_chain(&full_chain, &ca_certs_der); + if chain_result.len() > 1 { + // Successfully built a longer chain + full_chain = chain_result.into_iter().map(CertificateDer::from).collect(); + } + } + } + + // Additional validation: Create CertifiedKey to ensure rustls accepts it + let signing_key = + rustls::crypto::aws_lc_rs::sign::any_supported_type(&key).map_err(|_| { + vm.new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "[SSL: KEY_VALUES_MISMATCH] key values mismatch", + ) + .upcast() + })?; + + let certified_key = CertifiedKey::new(full_chain.clone(), signing_key); + if certified_key.keys_match().is_err() { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "[SSL: KEY_VALUES_MISMATCH] key values mismatch", + ) + .upcast()); + } + + // Add cert/key pair to collection (OpenSSL allows multiple cert/key pairs) + // Store both CertifiedKey (for server) and PrivateKeyDer (for client mTLS) + let cert_der = &full_chain[0]; + let mut cert_keys = self.cert_keys.write(); + + // Remove any existing cert/key pair with the same certificate + // (This allows updating cert/key pair without duplicating) + cert_keys.retain(|(existing, _)| &existing.cert[0] != cert_der); + + // Add new cert/key pair as tuple + cert_keys.push((Arc::new(certified_key), key)); + + Ok(()) + } + + #[pymethod] + fn load_verify_locations( + &self, + args: LoadVerifyLocationsArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check that at least one argument is provided + let has_cafile = matches!(&args.cafile, OptionalArg::Present(Some(_))); + let has_capath = matches!(&args.capath, OptionalArg::Present(Some(_))); + let has_cadata = matches!(&args.cadata, OptionalArg::Present(Some(_))); + + if !has_cafile && !has_capath && !has_cadata { + return Err(vm.new_type_error("cafile, capath and cadata cannot be all omitted")); + } + + // Parse arguments BEFORE acquiring locks to reduce lock scope + let cafile_path = if let OptionalArg::Present(Some(ref cafile_obj)) = args.cafile { + Some(Self::parse_path_arg(cafile_obj, vm)?) + } else { + None + }; + + let capath_dir = if let OptionalArg::Present(Some(ref capath_obj)) = args.capath { + Some(Self::parse_path_arg(capath_obj, vm)?) + } else { + None + }; + + let cadata_parsed = if let OptionalArg::Present(Some(ref cadata_obj)) = args.cadata { + let is_string = matches!(cadata_obj, Either::A(_)); + let data_vec = self.parse_cadata_arg(cadata_obj, vm)?; + Some((data_vec, is_string)) + } else { + None + }; + + // Check for CRL before acquiring main locks + let (crl_opt, cafile_is_crl) = if let Some(ref path) = cafile_path { + let crl = self.load_crl_from_file(path, vm)?; + let is_crl = crl.is_some(); + (crl, is_crl) + } else { + (None, false) + }; + + // If it's a CRL, just add it (separate lock, no conflict with root_store) + if let Some(crl) = crl_opt { + self.crls.write().push(crl); + } + + // Now acquire write locks for certificate loading + let mut root_store = self.root_certs.write(); + let mut ca_certs_der = self.ca_certs_der.write(); + + // Load from file (if not CRL) + if let Some(ref path) = cafile_path + && !cafile_is_crl + { + // Not a CRL, load as certificate + let stats = + self.load_certs_from_file_helper(&mut root_store, &mut ca_certs_der, path, vm)?; + self.update_cert_stats(stats); + } + + // Load from directory (don't add to ca_certs_der) + if let Some(ref dir_path) = capath_dir { + let stats = self.load_certs_from_dir_helper(&mut root_store, dir_path, vm)?; + self.update_cert_stats(stats); + } + + // Load from bytes or str + if let Some((ref data_vec, is_string)) = cadata_parsed { + let stats = self.load_certs_from_bytes_helper( + &mut root_store, + &mut ca_certs_der, + data_vec, + is_string, // PEM only for strings + vm, + )?; + self.update_cert_stats(stats); + } + + Ok(()) + } + + /// Helper: Get path from Python's os.environ + fn get_env_path( + environ: &PyObject, + var_name: &str, + vm: &VirtualMachine, + ) -> PyResult<String> { + let path_obj = environ.get_item(var_name, vm)?; + path_obj.try_into_value(vm) + } + + /// Helper: Try to load certificates from Python's os.environ variables + /// + /// Returns true if certificates were successfully loaded. + /// + /// We use Python's os.environ instead of Rust's std::env + /// because Python code can modify os.environ at runtime (e.g., + /// `os.environ['SSL_CERT_FILE'] = '/path'`), but rustls-native-certs uses + /// std::env which only sees the process environment at startup. + fn try_load_from_python_environ( + &self, + loader: &mut cert::CertLoader<'_>, + vm: &VirtualMachine, + ) -> PyResult<bool> { + use std::path::Path; + + let os_module = vm.import("os", 0)?; + let environ = os_module.get_attr("environ", vm)?; + + // Try SSL_CERT_FILE first + if let Ok(cert_file) = Self::get_env_path(&environ, "SSL_CERT_FILE", vm) + && Path::new(&cert_file).exists() + && let Ok(stats) = loader.load_from_file(&cert_file) + { + self.update_cert_stats(stats); + return Ok(true); + } + + // Try SSL_CERT_DIR (only if SSL_CERT_FILE didn't work) + if let Ok(cert_dir) = Self::get_env_path(&environ, "SSL_CERT_DIR", vm) + && Path::new(&cert_dir).is_dir() + && let Ok(stats) = loader.load_from_dir(&cert_dir) + { + self.update_cert_stats(stats); + return Ok(true); + } + + Ok(false) + } + + /// Helper: Load system certificates using rustls-native-certs + /// + /// This uses platform-specific methods: + /// - Linux: openssl-probe to find certificate files + /// - macOS: Keychain API + /// - Windows: System certificate store (ROOT + CA stores) + fn load_system_certificates( + &self, + store: &mut rustls::RootCertStore, + vm: &VirtualMachine, + ) -> PyResult<()> { + #[cfg(windows)] + { + // Windows: Use schannel to load from both ROOT and CA stores + use schannel::cert_store::CertStore; + + let store_names = ["ROOT", "CA"]; + let open_fns = [CertStore::open_current_user, CertStore::open_local_machine]; + + for store_name in store_names { + for open_fn in &open_fns { + if let Ok(cert_store) = open_fn(store_name) { + for cert_ctx in cert_store.certs() { + let der_bytes = cert_ctx.to_der(); + let cert = + rustls::pki_types::CertificateDer::from(der_bytes.to_vec()); + let is_ca = cert::is_ca_certificate(cert.as_ref()); + if store.add(cert).is_ok() { + *self.x509_cert_count.write() += 1; + if is_ca { + *self.ca_cert_count.write() += 1; + } + } + } + } + } + } + + if *self.x509_cert_count.read() == 0 { + return Err(vm.new_os_error("Failed to load certificates from Windows store")); + } + + Ok(()) + } + + #[cfg(not(windows))] + { + let result = rustls_native_certs::load_native_certs(); + + // Load successfully found certificates + for cert in result.certs { + let is_ca = cert::is_ca_certificate(cert.as_ref()); + if store.add(cert).is_ok() { + *self.x509_cert_count.write() += 1; + if is_ca { + *self.ca_cert_count.write() += 1; + } + } + } + + // If there were errors but some certs loaded, just continue + // If NO certs loaded and there were errors, report the first error + if *self.x509_cert_count.read() == 0 && !result.errors.is_empty() { + return Err(vm.new_os_error(format!( + "Failed to load native certificates: {}", + result.errors[0] + ))); + } + + Ok(()) + } + } + + #[pymethod] + fn load_default_certs( + &self, + _purpose: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut store = self.root_certs.write(); + + #[cfg(windows)] + { + // Windows: Load system certificates first, then additionally load from env + // see: test_load_default_certs_env_windows + let _ = self.load_system_certificates(&mut store, vm); + + let mut lazy_ca_certs = Vec::new(); + let mut loader = cert::CertLoader::new(&mut store, &mut lazy_ca_certs); + let _ = self.try_load_from_python_environ(&mut loader, vm)?; + } + + #[cfg(not(windows))] + { + // Non-Windows: Try env vars first; only fallback to system certs if not set + // see: test_load_default_certs_env + let mut lazy_ca_certs = Vec::new(); + let mut loader = cert::CertLoader::new(&mut store, &mut lazy_ca_certs); + let loaded = self.try_load_from_python_environ(&mut loader, vm)?; + + if !loaded { + let _ = self.load_system_certificates(&mut store, vm); + } + } + + // If no certificates were loaded from system, fallback to webpki-roots (Mozilla CA bundle) + // This ensures we always have some trusted root certificates even if system cert loading fails + if *self.x509_cert_count.read() == 0 { + use webpki_roots; + + // webpki_roots provides TLS_SERVER_ROOTS as &[TrustAnchor] + // We can use extend() to add them to the RootCertStore + let webpki_count = webpki_roots::TLS_SERVER_ROOTS.len(); + store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + + *self.x509_cert_count.write() += webpki_count; + *self.ca_cert_count.write() += webpki_count; + } + + Ok(()) + } + + #[pymethod] + fn set_alpn_protocols(&self, protocols: PyListRef, vm: &VirtualMachine) -> PyResult<()> { + let mut alpn_list = Vec::new(); + for item in protocols.borrow_vec().iter() { + let bytes = ArgBytesLike::try_from_object(vm, item.clone())?; + alpn_list.push(bytes.borrow_buf().to_vec()); + } + *self.alpn_protocols.write() = alpn_list; + Ok(()) + } + + #[pymethod] + fn _set_alpn_protocols(&self, protos: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { + let bytes = protos.borrow_buf(); + let alpn_list = parse_length_prefixed_alpn(&bytes, vm)?; + *self.alpn_protocols.write() = alpn_list; + Ok(()) + } + + #[pymethod] + fn set_ciphers(&self, ciphers: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<()> { + let cipher_str = ciphers.as_str(); + + // Parse cipher string and store selected ciphers + let selected_ciphers = parse_cipher_string(cipher_str).map_err(|e| { + vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), None, e) + .upcast() + })?; + + // Store in context + *self.selected_ciphers.write() = Some(selected_ciphers); + + Ok(()) + } + + #[pymethod] + fn get_ciphers(&self, vm: &VirtualMachine) -> PyResult<PyListRef> { + // Dynamically generate cipher list from rustls ALL_CIPHER_SUITES + // This automatically includes all cipher suites supported by the current rustls version + + let cipher_list = ALL_CIPHER_SUITES + .iter() + .map(|suite| { + // Extract cipher information using unified helper + let cipher_info = extract_cipher_info(suite); + + // Convert to OpenSSL-style name + // e.g., "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" -> "ECDHE-RSA-AES128-GCM-SHA256" + let openssl_name = normalize_cipher_name(&cipher_info.name); + + // Determine key exchange and auth methods + let (kx, auth) = if cipher_info.protocol == "TLSv1.3" { + // TLS 1.3 doesn't distinguish - all use modern algos + ("any", "any") + } else if cipher_info.name.contains("ECDHE") { + // TLS 1.2 with ECDHE + let auth = if cipher_info.name.contains("ECDSA") { + "ECDSA" + } else if cipher_info.name.contains("RSA") { + "RSA" + } else { + "any" + }; + ("ECDH", auth) + } else { + ("any", "any") + }; + + // Build description string + // Format: "{name} {protocol} Kx={kx} Au={auth} Enc={enc} Mac={mac}" + let enc = get_cipher_encryption_desc(&openssl_name); + + let description = format!( + "{} {} Kx={} Au={} Enc={} Mac=AEAD", + openssl_name, cipher_info.protocol, kx, auth, enc + ); + + // Create cipher dict + let dict = vm.ctx.new_dict(); + dict.set_item("name", vm.ctx.new_str(openssl_name).into(), vm) + .unwrap(); + dict.set_item("protocol", vm.ctx.new_str(cipher_info.protocol).into(), vm) + .unwrap(); + dict.set_item("id", vm.ctx.new_int(0).into(), vm).unwrap(); // Placeholder ID + dict.set_item("strength_bits", vm.ctx.new_int(cipher_info.bits).into(), vm) + .unwrap(); + dict.set_item("alg_bits", vm.ctx.new_int(cipher_info.bits).into(), vm) + .unwrap(); + dict.set_item("description", vm.ctx.new_str(description).into(), vm) + .unwrap(); + dict.into() + }) + .collect::<Vec<_>>(); + + Ok(PyListRef::from(vm.ctx.new_list(cipher_list))) + } + + #[pymethod] + fn set_default_verify_paths(&self, vm: &VirtualMachine) -> PyResult<()> { + // Just call load_default_certs + self.load_default_certs(OptionalArg::Missing, vm) + } + + #[pymethod] + fn cert_store_stats(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Use the certificate counters that are updated in load_verify_locations + let x509_count = *self.x509_cert_count.read() as i32; + let ca_count = *self.ca_cert_count.read() as i32; + + let dict = vm.ctx.new_dict(); + dict.set_item("x509", vm.ctx.new_int(x509_count).into(), vm)?; + dict.set_item("crl", vm.ctx.new_int(0).into(), vm)?; // CRL not supported + dict.set_item("x509_ca", vm.ctx.new_int(ca_count).into(), vm)?; + Ok(dict.into()) + } + + #[pymethod] + fn session_stats(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Return session statistics + // NOTE: This is a partial implementation - rustls doesn't expose all OpenSSL stats + let dict = vm.ctx.new_dict(); + + // Number of sessions currently in the cache + let session_count = self.client_session_cache.read().len() as i32; + dict.set_item("number", vm.ctx.new_int(session_count).into(), vm)?; + + // Client-side statistics (not tracked separately in this implementation) + dict.set_item("connect", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("connect_good", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("connect_renegotiate", vm.ctx.new_int(0).into(), vm)?; // rustls doesn't support renegotiation + + // Server-side statistics + let accept_count = self.accept_count.load(Ordering::SeqCst) as i32; + dict.set_item("accept", vm.ctx.new_int(accept_count).into(), vm)?; + dict.set_item("accept_good", vm.ctx.new_int(accept_count).into(), vm)?; // Assume all accepts are good + dict.set_item("accept_renegotiate", vm.ctx.new_int(0).into(), vm)?; // rustls doesn't support renegotiation + + // Session reuse statistics + let hits = self.session_hits.load(Ordering::SeqCst) as i32; + dict.set_item("hits", vm.ctx.new_int(hits).into(), vm)?; + + // Misses, timeouts, and cache_full are not tracked in this implementation + dict.set_item("misses", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("timeouts", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("cache_full", vm.ctx.new_int(0).into(), vm)?; + + Ok(dict.into()) + } + + #[pygetset] + fn sni_callback(&self) -> Option<PyObjectRef> { + self.sni_callback.read().clone() + } + + #[pygetset(setter)] + fn set_sni_callback( + &self, + callback: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Validate callback is callable or None + if let Some(ref cb) = callback + && !cb.is(vm.ctx.types.none_type) + && !cb.is_callable() + { + return Err(vm.new_type_error("sni_callback must be callable or None")); + } + *self.sni_callback.write() = callback; + Ok(()) + } + + #[pymethod] + fn set_servername_callback( + &self, + callback: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Alias for set_sni_callback + self.set_sni_callback(callback, vm) + } + + #[pygetset] + fn security_level(&self) -> i32 { + // rustls uses a fixed security level + // Return 2 which is a reasonable default (equivalent to OpenSSL 1.1.0+ level 2) + 2 + } + + #[pygetset] + fn _msg_callback(&self) -> Option<PyObjectRef> { + self.msg_callback.read().clone() + } + + #[pygetset(setter)] + fn set__msg_callback( + &self, + callback: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Validate callback is callable or None + if let Some(ref cb) = callback + && !cb.is(vm.ctx.types.none_type) + && !cb.is_callable() + { + return Err(vm.new_type_error("msg_callback must be callable or None")); + } + *self.msg_callback.write() = callback; + Ok(()) + } + + #[pymethod] + fn get_ca_certs(&self, args: GetCertArgs, vm: &VirtualMachine) -> PyResult<PyListRef> { + let binary_form = args.binary_form.unwrap_or(false); + let ca_certs_der = self.ca_certs_der.read(); + + let mut certs = Vec::new(); + for cert_der in ca_certs_der.iter() { + // Parse certificate to check if it's a CA and get info + match x509_parser::parse_x509_certificate(cert_der) { + Ok((_, cert)) => { + // Check if this is a CA certificate (BasicConstraints: CA=TRUE) + let is_ca = if let Ok(Some(bc_ext)) = cert.basic_constraints() { + bc_ext.value.ca + } else { + false + }; + + // Only include CA certificates + if !is_ca { + continue; + } + + if binary_form { + // Return DER-encoded certificate as bytes + certs.push(vm.ctx.new_bytes(cert_der.clone()).into()); + } else { + // Return certificate as dict (use helper from _test_decode_cert) + let dict = self.cert_der_to_dict(vm, cert_der)?; + certs.push(dict); + } + } + Err(_) => { + // Skip invalid certificates + continue; + } + } + } + + Ok(PyListRef::from(vm.ctx.new_list(certs))) + } + + #[pymethod] + fn load_dh_params(&self, filepath: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Validate filepath is not None + if vm.is_none(&filepath) { + return Err(vm.new_type_error("DH params filepath cannot be None")); + } + + // Validate filepath is str or bytes + let path_str = if let Ok(s) = PyUtf8StrRef::try_from_object(vm, filepath.clone()) { + s.as_str().to_owned() + } else if let Ok(b) = ArgBytesLike::try_from_object(vm, filepath) { + String::from_utf8(b.borrow_buf().to_vec()) + .map_err(|_| vm.new_value_error("Invalid path encoding"))? + } else { + return Err(vm.new_type_error("DH params filepath must be str or bytes")); + }; + + // Check if file exists + if !std::path::Path::new(&path_str).exists() { + // Create FileNotFoundError with errno=ENOENT (2) + let exc = vm.new_os_subtype_error( + vm.ctx.exceptions.file_not_found_error.to_owned(), + Some(2), // errno = ENOENT (2) + "No such file or directory", + ); + // Set filename attribute + let _ = exc + .as_object() + .set_attr("filename", vm.ctx.new_str(path_str.clone()), vm); + return Err(exc.upcast()); + } + + // Validate that the file contains DH parameters + // Read the file and check for DH PARAMETERS header + let contents = + std::fs::read_to_string(&path_str).map_err(|e| vm.new_os_error(e.to_string()))?; + + if !contents.contains("BEGIN DH PARAMETERS") + && !contents.contains("BEGIN X9.42 DH PARAMETERS") + { + // File exists but doesn't contain DH parameters - raise SSLError + // [PEM: NO_START_LINE] no start line + return Err(super::compat::SslError::create_ssl_error_with_reason( + vm, + Some("PEM"), + "NO_START_LINE", + "[PEM: NO_START_LINE] no start line", + )); + } + + // rustls doesn't use DH parameters (it uses ECDHE for key exchange) + // This is a no-op for compatibility with OpenSSL-based code + Ok(()) + } + + #[pymethod] + fn set_ecdh_curve(&self, name: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Validate name is not None + if vm.is_none(&name) { + return Err(vm.new_type_error("ECDH curve name cannot be None")); + } + + // Validate name is str or bytes + let curve_name = if let Ok(s) = PyUtf8StrRef::try_from_object(vm, name.clone()) { + s.as_str().to_owned() + } else if let Ok(b) = ArgBytesLike::try_from_object(vm, name) { + String::from_utf8(b.borrow_buf().to_vec()) + .map_err(|_| vm.new_value_error("Invalid curve name encoding"))? + } else { + return Err(vm.new_type_error("ECDH curve name must be str or bytes")); + }; + + // Validate curve name (common curves for compatibility) + // rustls supports: X25519, secp256r1 (prime256v1), secp384r1 + let valid_curves = [ + "prime256v1", + "secp256r1", + "prime384v1", + "secp384r1", + "prime521v1", + "secp521r1", + "X25519", + "x25519", + "x448", // For future compatibility + ]; + + if !valid_curves.contains(&curve_name.as_str()) { + return Err(vm.new_value_error(format!("unknown curve name '{curve_name}'"))); + } + + // Store the curve name to be used during handshake + // This will limit the key exchange groups offered/accepted + *self.ecdh_curve.write() = Some(curve_name); + Ok(()) + } + + #[pymethod] + fn _wrap_socket( + zelf: PyRef<Self>, + args: WrapSocketArgs, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PySSLSocket>> { + // Convert server_hostname to Option<String> + // Handle both missing argument and None value + let hostname = match args.server_hostname.into_option().flatten() { + Some(hostname_str) => { + let hostname = hostname_str.as_str(); + + // Validate hostname + if hostname.is_empty() { + return Err(vm.new_value_error("server_hostname cannot be an empty string")); + } + + // Check if it starts with a dot + if hostname.starts_with('.') { + return Err(vm.new_value_error("server_hostname cannot start with a dot")); + } + + // IP addresses are allowed + // SNI will not be sent for IP addresses + + // Check for NULL bytes + if hostname.contains('\0') { + return Err(vm.new_type_error("embedded null character")); + } + + Some(hostname.to_string()) + } + None => None, + }; + + // Validate socket type and context protocol + if args.server_side && zelf.protocol == PROTOCOL_TLS_CLIENT { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Cannot create a server socket with a PROTOCOL_TLS_CLIENT context", + ) + .upcast()); + } + if !args.server_side && zelf.protocol == PROTOCOL_TLS_SERVER { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Cannot create a client socket with a PROTOCOL_TLS_SERVER context", + ) + .upcast()); + } + + // Create _SSLSocket instance + let ssl_socket = PySSLSocket { + sock: args.sock.clone(), + context: PyRwLock::new(zelf), + server_side: args.server_side, + server_hostname: PyRwLock::new(hostname), + connection: PyMutex::new(None), + handshake_done: PyMutex::new(false), + session_was_reused: PyMutex::new(false), + owner: PyRwLock::new(args.owner.into_option()), + // Filter out Python None objects - only store actual SSLSession objects + session: PyRwLock::new(args.session.into_option().filter(|s| !vm.is_none(s))), + verified_chain: PyRwLock::new(None), + incoming_bio: None, + outgoing_bio: None, + sni_state: PyRwLock::new(None), + pending_context: PyRwLock::new(None), + client_hello_buffer: PyMutex::new(None), + shutdown_state: PyMutex::new(ShutdownState::NotStarted), + pending_tls_output: PyMutex::new(Vec::new()), + write_buffered_len: PyMutex::new(0), + deferred_cert_error: Arc::new(ParkingRwLock::new(None)), + }; + + // Create PyRef with correct type + let ssl_socket_ref = ssl_socket + .into_ref_with_type(vm, vm.class("_ssl", "_SSLSocket")) + .map_err(|_| vm.new_type_error("Failed to create SSLSocket"))?; + + Ok(ssl_socket_ref) + } + + #[pymethod] + fn _wrap_bio( + zelf: PyRef<Self>, + args: WrapBioArgs, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PySSLSocket>> { + // Convert server_hostname to Option<String> + // Handle both missing argument and None value + let hostname = match args.server_hostname.into_option().flatten() { + Some(hostname_str) => { + let hostname = hostname_str.as_str(); + validate_hostname(hostname, vm)?; + Some(hostname.to_string()) + } + None => None, + }; + + // Extract server_side value + let server_side = args.server_side.unwrap_or(false); + + // Validate socket type and context protocol + if server_side && zelf.protocol == PROTOCOL_TLS_CLIENT { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Cannot create a server socket with a PROTOCOL_TLS_CLIENT context", + ) + .upcast()); + } + if !server_side && zelf.protocol == PROTOCOL_TLS_SERVER { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Cannot create a client socket with a PROTOCOL_TLS_SERVER context", + ) + .upcast()); + } + + // Create _SSLSocket instance with BIO mode + let ssl_socket = PySSLSocket { + sock: vm.ctx.none(), // No socket in BIO mode + context: PyRwLock::new(zelf), + server_side, + server_hostname: PyRwLock::new(hostname), + connection: PyMutex::new(None), + handshake_done: PyMutex::new(false), + session_was_reused: PyMutex::new(false), + owner: PyRwLock::new(args.owner.into_option()), + // Filter out Python None objects - only store actual SSLSession objects + session: PyRwLock::new(args.session.into_option().filter(|s| !vm.is_none(s))), + verified_chain: PyRwLock::new(None), + incoming_bio: Some(args.incoming), + outgoing_bio: Some(args.outgoing), + sni_state: PyRwLock::new(None), + pending_context: PyRwLock::new(None), + client_hello_buffer: PyMutex::new(None), + shutdown_state: PyMutex::new(ShutdownState::NotStarted), + pending_tls_output: PyMutex::new(Vec::new()), + write_buffered_len: PyMutex::new(0), + deferred_cert_error: Arc::new(ParkingRwLock::new(None)), + }; + + let ssl_socket_ref = ssl_socket + .into_ref_with_type(vm, vm.class("_ssl", "_SSLSocket")) + .map_err(|_| vm.new_type_error("Failed to create SSLSocket"))?; + + Ok(ssl_socket_ref) + } + + // Helper functions (private): + + /// Parse path argument (str or bytes) to string + fn parse_path_arg( + arg: &Either<PyStrRef, ArgBytesLike>, + vm: &VirtualMachine, + ) -> PyResult<String> { + match arg { + Either::A(s) => Ok(s.clone().try_into_utf8(vm)?.as_str().to_owned()), + Either::B(b) => String::from_utf8(b.borrow_buf().to_vec()) + .map_err(|_| vm.new_value_error("path contains invalid UTF-8")), + } + } + + /// Parse password argument (str, bytes-like, or callable) + /// + /// Returns (immediate_password, callable) where: + /// - immediate_password: Some(string) if password is str/bytes, None if callable + /// - callable: Some(PyObjectRef) if password is callable, None otherwise + fn parse_password_argument( + password: &OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<(Option<String>, Option<PyObjectRef>)> { + match password { + OptionalArg::Present(p) => { + if vm.is_none(p) { + return Ok((None, None)); + } + + // Try string + if let Ok(pwd_str) = PyUtf8StrRef::try_from_object(vm, p.clone()) { + Ok((Some(pwd_str.as_str().to_owned()), None)) + } + // Try bytes-like + else if let Ok(pwd_bytes_like) = ArgBytesLike::try_from_object(vm, p.clone()) + { + let pwd = String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()) + .map_err(|_| vm.new_type_error("password bytes must be valid UTF-8"))?; + Ok((Some(pwd), None)) + } + // Try callable + else if p.is_callable() { + Ok((None, Some(p.clone()))) + } else { + Err(vm.new_type_error("password should be a string or callable")) + } + } + _ => Ok((None, None)), + } + } + + /// Helper: Load certificates from file into existing store + fn load_certs_from_file_helper( + &self, + root_store: &mut RootCertStore, + ca_certs_der: &mut Vec<Vec<u8>>, + path: &str, + vm: &VirtualMachine, + ) -> PyResult<cert::CertStats> { + let mut loader = cert::CertLoader::new(root_store, ca_certs_der); + loader.load_from_file(path).map_err(|e| { + // Preserve errno for file access errors (NotFound, PermissionDenied) + match e.kind() { + std::io::ErrorKind::NotFound | std::io::ErrorKind::PermissionDenied => { + e.into_pyexception(vm) + } + // PEM parsing errors + _ => super::compat::SslError::create_ssl_error_with_reason( + vm, + Some("X509"), + "", + "PEM lib", + ), + } + }) + } + + /// Helper: Load certificates from directory into existing store + fn load_certs_from_dir_helper( + &self, + root_store: &mut RootCertStore, + path: &str, + vm: &VirtualMachine, + ) -> PyResult<cert::CertStats> { + // Load certs and store them in capath_certs_der for lazy loading simulation + // (CPython only returns these in get_ca_certs() after they're used in handshake) + let mut capath_certs = Vec::new(); + let mut loader = cert::CertLoader::new(root_store, &mut capath_certs); + let stats = loader + .load_from_dir(path) + .map_err(|e| e.into_pyexception(vm))?; + + // Store loaded certs for potential tracking after handshake + *self.capath_certs_der.write() = capath_certs; + + Ok(stats) + } + + /// Helper: Load certificates from bytes into existing store + fn load_certs_from_bytes_helper( + &self, + root_store: &mut RootCertStore, + ca_certs_der: &mut Vec<Vec<u8>>, + data: &[u8], + pem_only: bool, + vm: &VirtualMachine, + ) -> PyResult<cert::CertStats> { + let mut loader = cert::CertLoader::new(root_store, ca_certs_der); + // treat_all_as_ca=true: CPython counts all certificates loaded via cadata as CA certs + // regardless of their Basic Constraints extension + // pem_only=true for string input + loader + .load_from_bytes_ex(data, true, pem_only) + .map_err(|e| { + // Preserve specific error messages from cert.rs + let err_msg = e.to_string(); + if err_msg.contains("no start line") { + vm.new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "no start line: cadata does not contain a certificate", + ) + .upcast() + } else if err_msg.contains("not enough data") { + vm.new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "not enough data: cadata does not contain a certificate", + ) + .upcast() + } else { + vm.new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + err_msg, + ) + .upcast() + } + }) + } + + /// Helper: Try to parse data as CRL (PEM or DER format) + fn try_parse_crl( + &self, + data: &[u8], + ) -> Result<CertificateRevocationListDer<'static>, String> { + // Try PEM format first + let mut cursor = std::io::Cursor::new(data); + let mut crl_iter = rustls_pemfile::crls(&mut cursor); + if let Some(Ok(crl)) = crl_iter.next() { + return Ok(crl); + } + + // Try DER format + // Basic validation: CRL should start with SEQUENCE tag (0x30) + if !data.is_empty() && data[0] == 0x30 { + return Ok(CertificateRevocationListDer::from(data.to_vec())); + } + + Err("Not a valid CRL file".to_string()) + } + + /// Helper: Load CRL from file + fn load_crl_from_file( + &self, + path: &str, + vm: &VirtualMachine, + ) -> PyResult<Option<CertificateRevocationListDer<'static>>> { + let data = std::fs::read(path).map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound | std::io::ErrorKind::PermissionDenied => { + e.into_pyexception(vm) + } + _ => vm.new_os_error(e.to_string()), + })?; + + match self.try_parse_crl(&data) { + Ok(crl) => Ok(Some(crl)), + Err(_) => Ok(None), // Not a CRL file, might be a cert file + } + } + + /// Helper: Parse cadata argument (str or bytes) + fn parse_cadata_arg( + &self, + arg: &Either<PyStrRef, ArgBytesLike>, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + match arg { + Either::A(s) => Ok(s.clone().try_into_utf8(vm)?.as_str().as_bytes().to_vec()), + Either::B(b) => Ok(b.borrow_buf().to_vec()), + } + } + + /// Helper: Update certificate statistics + fn update_cert_stats(&self, stats: cert::CertStats) { + *self.x509_cert_count.write() += stats.total_certs; + *self.ca_cert_count.write() += stats.ca_certs; + } + } + + impl Representable for PySSLContext { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!("<SSLContext(protocol={})>", zelf.protocol)) + } + } + + impl Constructor for PySSLContext { + type Args = (i32,); + + fn py_new( + _cls: &Py<PyType>, + (protocol,): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + // Validate protocol + match protocol { + PROTOCOL_TLS | PROTOCOL_TLS_CLIENT | PROTOCOL_TLS_SERVER | PROTOCOL_TLSv1_2 + | PROTOCOL_TLSv1_3 => { + // Valid protocols + } + PROTOCOL_TLSv1 | PROTOCOL_TLSv1_1 => { + return Err(vm.new_value_error( + "TLS 1.0 and 1.1 are not supported by rustls for security reasons", + )); + } + _ => { + return Err(vm.new_value_error(format!("invalid protocol version: {protocol}"))); + } + } + + // Set default options + // OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 | OP_NO_COMPRESSION | + // OP_CIPHER_SERVER_PREFERENCE | OP_SINGLE_DH_USE | OP_SINGLE_ECDH_USE | + // OP_ENABLE_MIDDLEBOX_COMPAT + let default_options = OP_ALL + | OP_NO_SSLv2 + | OP_NO_SSLv3 + | OP_NO_COMPRESSION + | OP_CIPHER_SERVER_PREFERENCE + | OP_SINGLE_DH_USE + | OP_SINGLE_ECDH_USE + | OP_ENABLE_MIDDLEBOX_COMPAT; + + // Set default verify_mode based on protocol + // PROTOCOL_TLS_CLIENT defaults to CERT_REQUIRED + // PROTOCOL_TLS_SERVER defaults to CERT_NONE + let default_verify_mode = if protocol == PROTOCOL_TLS_CLIENT { + CERT_REQUIRED + } else { + CERT_NONE + }; + + // Set default verify_flags based on protocol + // Both PROTOCOL_TLS_CLIENT and PROTOCOL_TLS_SERVER only set VERIFY_X509_TRUSTED_FIRST + // Note: VERIFY_X509_PARTIAL_CHAIN and VERIFY_X509_STRICT are NOT set here + // - they're only added by create_default_context() in Python's ssl.py + let default_verify_flags = VERIFY_DEFAULT | VERIFY_X509_TRUSTED_FIRST; + + // Set minimum and maximum protocol versions based on protocol constant + // specific protocol versions fix both min and max + let (min_version, max_version) = match protocol { + PROTOCOL_TLSv1_2 => (PROTO_TLSv1_2, PROTO_TLSv1_2), // Only TLS 1.2 + PROTOCOL_TLSv1_3 => (PROTO_TLSv1_3, PROTO_TLSv1_3), // Only TLS 1.3 + _ => (PROTO_MINIMUM_SUPPORTED, PROTO_MAXIMUM_SUPPORTED), // Auto-negotiate + }; + + // IMPORTANT: Create shared session cache BEFORE PySSLContext + // Both client_session_cache and PythonClientSessionStore.session_cache + // MUST point to the same HashMap to ensure Python-level and Rustls-level + // sessions are synchronized + let shared_session_cache = Arc::new(ParkingRwLock::new(HashMap::new())); + let rustls_client_store = Arc::new(PythonClientSessionStore { + inner: Arc::new(rustls::client::ClientSessionMemoryCache::new( + SSL_SESSION_CACHE_SIZE, + )), + session_cache: shared_session_cache.clone(), + }); + + Ok(PySSLContext { + protocol, + check_hostname: PyRwLock::new(protocol == PROTOCOL_TLS_CLIENT), + verify_mode: PyRwLock::new(default_verify_mode), + verify_flags: PyRwLock::new(default_verify_flags), + client_config: PyRwLock::new(None), + server_config: PyRwLock::new(None), + root_certs: PyRwLock::new(RootCertStore::empty()), + ca_certs_der: PyRwLock::new(Vec::new()), + capath_certs_der: PyRwLock::new(Vec::new()), + crls: PyRwLock::new(Vec::new()), + cert_keys: PyRwLock::new(Vec::new()), + options: PyRwLock::new(default_options), + alpn_protocols: PyRwLock::new(Vec::new()), + require_alpn_match: PyRwLock::new(false), + post_handshake_auth: PyRwLock::new(false), + num_tickets: PyRwLock::new(2), // TLS 1.3 default + minimum_version: PyRwLock::new(min_version), + maximum_version: PyRwLock::new(max_version), + sni_callback: PyRwLock::new(None), + msg_callback: PyRwLock::new(None), + ecdh_curve: PyRwLock::new(None), + ca_cert_count: PyRwLock::new(0), + x509_cert_count: PyRwLock::new(0), + // Use the shared cache created above + client_session_cache: shared_session_cache, + rustls_session_store: rustls_client_store, + rustls_server_session_store: rustls::server::ServerSessionMemoryCache::new( + SSL_SESSION_CACHE_SIZE, + ), + server_ticketer: rustls::crypto::aws_lc_rs::Ticketer::new() + .expect("Failed to create shared ticketer for TLS 1.2 session resumption"), + accept_count: AtomicUsize::new(0), + session_hits: AtomicUsize::new(0), + selected_ciphers: PyRwLock::new(None), + }) + } + } + + // SSLSocket - represents a TLS-wrapped socket + #[pyattr] + #[pyclass(name = "_SSLSocket", module = "ssl", traverse)] + #[derive(Debug, PyPayload)] + pub(crate) struct PySSLSocket { + // Underlying socket + sock: PyObjectRef, + // SSL context + context: PyRwLock<PyRef<PySSLContext>>, + // Server-side or client-side + #[pytraverse(skip)] + server_side: bool, + // Server hostname for SNI + #[pytraverse(skip)] + server_hostname: PyRwLock<Option<String>>, + // TLS connection state + #[pytraverse(skip)] + connection: PyMutex<Option<TlsConnection>>, + // Handshake completed flag + #[pytraverse(skip)] + handshake_done: PyMutex<bool>, + // Session was reused (for session resumption tracking) + #[pytraverse(skip)] + session_was_reused: PyMutex<bool>, + // Owner (SSLSocket instance that owns this _SSLSocket) + owner: PyRwLock<Option<PyObjectRef>>, + // Session for resumption + session: PyRwLock<Option<PyObjectRef>>, + // Verified certificate chain (built during verification) + #[allow(dead_code)] + #[pytraverse(skip)] + verified_chain: PyRwLock<Option<Vec<CertificateDer<'static>>>>, + // MemoryBIO mode (optional) + incoming_bio: Option<PyRef<PyMemoryBIO>>, + outgoing_bio: Option<PyRef<PyMemoryBIO>>, + // SNI certificate resolver state (for server-side only) + #[pytraverse(skip)] + sni_state: PyRwLock<Option<Arc<ParkingMutex<SniCertName>>>>, + // Pending context change (for SNI callback deferred handling) + pending_context: PyRwLock<Option<PyRef<PySSLContext>>>, + // Buffer to store ClientHello for connection recreation + #[pytraverse(skip)] + client_hello_buffer: PyMutex<Option<Vec<u8>>>, + // Shutdown state for tracking close-notify exchange + #[pytraverse(skip)] + shutdown_state: PyMutex<ShutdownState>, + // Pending TLS output buffer for non-blocking sockets + // Stores unsent TLS bytes when sock_send() would block + // This prevents data loss when write_tls() drains rustls' internal buffer + // but the socket cannot accept all the data immediately + #[pytraverse(skip)] + pub(crate) pending_tls_output: PyMutex<Vec<u8>>, + // Tracks bytes already buffered in rustls for the current write operation + // Prevents duplicate writes when retrying after WantWrite/WantRead + #[pytraverse(skip)] + pub(crate) write_buffered_len: PyMutex<usize>, + // Deferred client certificate verification error (for TLS 1.3) + // Stores error message if client cert verification failed during handshake + // Error is raised on first I/O operation after handshake + // Using Arc to share with the certificate verifier + #[pytraverse(skip)] + deferred_cert_error: Arc<ParkingRwLock<Option<String>>>, + } + + // Shutdown state for tracking close-notify exchange + #[derive(Debug, Clone, Copy, PartialEq)] + enum ShutdownState { + NotStarted, // unwrap() not called yet + SentCloseNotify, // close-notify sent, waiting for peer's response + Completed, // unwrap() completed successfully + } + + #[pyclass(with(Constructor, Representable), flags(BASETYPE))] + impl PySSLSocket { + // Check if this is BIO mode + pub(crate) fn is_bio_mode(&self) -> bool { + self.incoming_bio.is_some() && self.outgoing_bio.is_some() + } + + // Get incoming BIO reference (for EOF checking) + pub(crate) fn incoming_bio(&self) -> Option<PyObjectRef> { + self.incoming_bio.as_ref().map(|bio| bio.clone().into()) + } + + // Check for deferred certificate verification errors (TLS 1.3) + // If an error exists, raise it and clear it from storage + fn check_deferred_cert_error(&self, vm: &VirtualMachine) -> PyResult<()> { + let error_opt = self.deferred_cert_error.read().clone(); + if let Some(error_msg) = error_opt { + // Clear the error so it's only raised once + *self.deferred_cert_error.write() = None; + // Raise OSError with the stored error message + return Err(vm.new_os_error(error_msg)); + } + Ok(()) + } + + // Get socket timeout as Duration + pub(crate) fn get_socket_timeout(&self, vm: &VirtualMachine) -> PyResult<Option<Duration>> { + if self.is_bio_mode() { + return Ok(None); + } + + // Get timeout from socket + let timeout_obj = self.sock.get_attr("gettimeout", vm)?.call((), vm)?; + + // timeout can be None (blocking), 0.0 (non-blocking), or positive float + if vm.is_none(&timeout_obj) { + // None means blocking forever + Ok(None) + } else { + let timeout_float: f64 = timeout_obj.try_into_value(vm)?; + if timeout_float <= 0.0 { + // 0 means non-blocking + Ok(Some(Duration::from_secs(0))) + } else { + // Positive timeout + Ok(Some(Duration::from_secs_f64(timeout_float))) + } + } + } + + // Create and store a session object after successful handshake + fn create_session_after_handshake(&self, vm: &VirtualMachine) -> PyResult<()> { + // Only create session for client-side connections + if self.server_side { + return Ok(()); + } + + // Check if session already exists + let session_opt = self.session.read().clone(); + if let Some(ref s) = session_opt { + if vm.is_none(s) { + } else { + return Ok(()); + } + } + + // Get server hostname + let server_name = self.server_hostname.read().clone(); + + // Try to get session data from context's session cache + // IMPORTANT: Acquire and release locks quickly to avoid deadlock + let context = self.context.read(); + let session_cache_arc = context.client_session_cache.clone(); + drop(context); // Release context lock ASAP + + let (session_id, creation_time, lifetime) = if let Some(ref name) = server_name { + let key = name.as_bytes().to_vec(); + + // Clone the data we need while holding the lock, then immediately release + let session_data_opt = { + let cache_guard = session_cache_arc.read(); + cache_guard.get(&key).cloned() // Clone Arc<PyMutex<SessionData>> + }; // Lock released here + + if let Some(session_data_arc) = session_data_opt { + let data = session_data_arc.lock(); + let result = (data.session_id.clone(), data.creation_time, data.lifetime); + drop(data); // Explicit unlock + result + } else { + // Create new session ID if not in cache + let time = std::time::SystemTime::now(); + (generate_session_id_from_metadata(name, &time), time, 7200) + } + } else { + // No server name, use defaults + let time = std::time::SystemTime::now(); + (vec![0; 16], time, 7200) + }; + + // Create a new SSLSession object with real metadata + let session = PySSLSession { + // Use dummy session data to indicate we have a ticket + // TLS 1.2+ always uses session tickets/resumption + session_data: vec![1], // Non-empty to indicate has_ticket=True + session_id, + creation_time, + lifetime, + }; + + let py_session = session.into_pyobject(vm); + + *self.session.write() = Some(py_session); + + Ok(()) + } + + // Complete handshake and create session + /// Track which CA certificate from capath was used to verify peer + /// + /// This simulates lazy loading behavior: capath certificates + /// are only added to get_ca_certs() after they're actually used in a handshake. + fn track_used_ca_from_capath(&self) -> Result<(), String> { + // Extract capath_certs, releasing context lock quickly + let capath_certs = { + let context = self.context.read(); + let certs = context.capath_certs_der.read(); + if certs.is_empty() { + return Ok(()); + } + certs.clone() + }; + + // Extract peer certificates, releasing connection lock quickly + let top_cert_der = { + let conn_guard = self.connection.lock(); + let conn = conn_guard.as_ref().ok_or("No connection")?; + let peer_certs = conn.peer_certificates().ok_or("No peer certificates")?; + if peer_certs.is_empty() { + return Ok(()); + } + peer_certs + .iter() + .map(|c| c.as_ref().to_vec()) + .next_back() + .expect("is_empty checked above") + }; + + // Get the top certificate in the chain (closest to root) + // Note: Server usually doesn't send the root CA, so we check the last cert's issuer + let (_, top_cert) = x509_parser::parse_x509_certificate(&top_cert_der) + .map_err(|e| format!("Failed to parse top cert: {e}"))?; + + let top_issuer = top_cert.issuer(); + + // Find matching CA in capath certs (skip unparseable certificates) + let matching_ca = capath_certs.iter().find_map(|ca_der| { + let (_, ca) = x509_parser::parse_x509_certificate(ca_der).ok()?; + // Check if this CA is self-signed (root CA) and matches the issuer + (ca.subject() == ca.issuer() && ca.subject() == top_issuer).then(|| ca_der.clone()) + }); + + // Update ca_certs_der if we found a match + if let Some(ca_der) = matching_ca { + let context = self.context.read(); + let mut ca_certs_der = context.ca_certs_der.write(); + if !ca_certs_der.iter().any(|c| c == &ca_der) { + ca_certs_der.push(ca_der); + } + } + + Ok(()) + } + + fn complete_handshake(&self, vm: &VirtualMachine) -> PyResult<()> { + *self.handshake_done.lock() = true; + + // Check if session was resumed - get value and release lock immediately + let was_resumed = self + .connection + .lock() + .as_ref() + .map(|conn| conn.is_session_resumed()) + .unwrap_or(false); + + *self.session_was_reused.lock() = was_resumed; + + // Update context session statistics if server-side + if self.server_side { + let context = self.context.read(); + // Increment accept count for every successful server handshake + context.accept_count.fetch_add(1, Ordering::SeqCst); + // Increment hits count if session was resumed + if was_resumed { + context.session_hits.fetch_add(1, Ordering::SeqCst); + } + } + + // Track CA certificate used during handshake (client-side only) + // This simulates lazy loading behavior for capath certificates + if !self.server_side { + // Don't fail handshake if tracking fails + let _ = self.track_used_ca_from_capath(); + } + + self.create_session_after_handshake(vm)?; + Ok(()) + } + + // Internal implementation with timeout control + pub(crate) fn sock_wait_for_io_impl( + &self, + kind: SelectKind, + vm: &VirtualMachine, + ) -> PyResult<bool> { + if self.is_bio_mode() { + // BIO mode doesn't use select + return Ok(false); + } + + // Get timeout + let timeout = self.get_socket_timeout(vm)?; + + // Check for non-blocking mode (timeout = 0) + if let Some(t) = timeout + && t.is_zero() + { + // Non-blocking mode - don't use select + return Ok(false); + } + + // Use select with the effective timeout + let py_socket: PyRef<PySocket> = self.sock.clone().try_into_value(vm)?; + let socket = py_socket + .sock() + .map_err(|e| vm.new_os_error(format!("Failed to get socket: {e}")))?; + + let timed_out = sock_select(&socket, kind, timeout) + .map_err(|e| vm.new_os_error(format!("select failed: {e}")))?; + + Ok(timed_out) + } + + // Internal implementation with explicit timeout override + pub(crate) fn sock_wait_for_io_with_timeout( + &self, + kind: SelectKind, + timeout: Option<core::time::Duration>, + vm: &VirtualMachine, + ) -> PyResult<bool> { + if self.is_bio_mode() { + // BIO mode doesn't use select + return Ok(false); + } + + if let Some(t) = timeout + && t.is_zero() + { + // Non-blocking mode - don't use select + return Ok(false); + } + + let py_socket: PyRef<PySocket> = self.sock.clone().try_into_value(vm)?; + let socket = py_socket + .sock() + .map_err(|e| vm.new_os_error(format!("Failed to get socket: {e}")))?; + + let timed_out = sock_select(&socket, kind, timeout) + .map_err(|e| vm.new_os_error(format!("select failed: {e}")))?; + + Ok(timed_out) + } + + // SNI (Server Name Indication) Helper Methods: + // These methods support the server-side handshake SNI callback mechanism + + /// Check if this is the first read during handshake (for SNI callback) + /// Returns true if we haven't processed ClientHello yet, regardless of SNI presence + pub(crate) fn is_first_sni_read(&self) -> bool { + self.client_hello_buffer.lock().is_none() + } + + /// Check if SNI callback is configured + pub(crate) fn has_sni_callback(&self) -> bool { + // Nested read locks are safe + self.context.read().sni_callback.read().is_some() + } + + /// Save ClientHello data from PyObjectRef for potential connection recreation + pub(crate) fn save_client_hello_from_bytes(&self, bytes_data: &[u8]) { + *self.client_hello_buffer.lock() = Some(bytes_data.to_vec()); + } + + /// Get the extracted SNI name from resolver + pub(crate) fn get_extracted_sni_name(&self) -> Option<String> { + // Clone the Arc option to avoid nested lock (sni_state.read -> arc.lock) + let sni_state_opt = self.sni_state.read().clone(); + sni_state_opt.as_ref().and_then(|arc| arc.lock().1.clone()) + } + + /// Invoke the Python SNI callback + pub(crate) fn invoke_sni_callback( + &self, + sni_name: Option<&str>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let callback = self + .context + .read() + .sni_callback + .read() + .clone() + .ok_or_else(|| vm.new_value_error("SNI callback not set"))?; + + let ssl_sock = self.owner.read().clone().unwrap_or(vm.ctx.none()); + let server_name_py: PyObjectRef = match sni_name { + Some(name) => vm.ctx.new_str(name.to_string()).into(), + None => vm.ctx.none(), + }; + let initial_context: PyObjectRef = self.context.read().clone().into(); + + // catches exceptions from the callback and reports them as unraisable + let result = match callback.call((ssl_sock, server_name_py, initial_context), vm) { + Ok(result) => result, + Err(exc) => { + vm.run_unraisable( + exc, + Some("in ssl servername callback".to_owned()), + callback.clone(), + ); + // Return SSL error like SSL_TLSEXT_ERR_ALERT_FATAL + let ssl_exc: PyBaseExceptionRef = vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "SNI callback raised exception", + ) + .upcast(); + let _ = ssl_exc.as_object().set_attr( + "reason", + vm.ctx.new_str("TLSV1_ALERT_INTERNAL_ERROR"), + vm, + ); + return Err(ssl_exc); + } + }; + + // Check return value type (must be None or integer) + if !vm.is_none(&result) { + // Try to convert to integer + if result.try_to_value::<i32>(vm).is_err() { + // Type conversion failed - raise TypeError as unraisable + let type_error = vm.new_type_error(format!( + "servername callback must return None or an integer, not '{}'", + result.class().name() + )); + vm.run_unraisable(type_error, None, result.clone()); + + // Return SSL error with reason set to TLSV1_ALERT_INTERNAL_ERROR + // + // RUSTLS API LIMITATION: + // We cannot send a TLS InternalError alert to the client here because: + // 1. Rustls does not provide a public API like send_fatal_alert() + // 2. This method is called AFTER dropping the connection lock (to prevent deadlock) + // 3. By the time we detect the error, the connection is no longer available + // + // CPython/OpenSSL behavior: + // - SNI callback runs inside SSL_do_handshake with connection active + // - Sets *al = SSL_AD_INTERNAL_ERROR + // - OpenSSL automatically sends alert before returning + // + // RustPython/Rustls behavior: + // - SNI callback runs after dropping connection lock (deadlock prevention) + // - Exception has _reason='TLSV1_ALERT_INTERNAL_ERROR' for error reporting + // - TCP connection closes without sending TLS alert to client + // + // If rustls adds send_fatal_alert() API in the future, we should: + // - Re-acquire connection lock after callback + // - Call: connection.send_fatal_alert(AlertDescription::InternalError) + // - Then close connection + let exc: PyBaseExceptionRef = vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "SNI callback returned invalid type", + ) + .upcast(); + let _ = exc.as_object().set_attr( + "reason", + vm.ctx.new_str("TLSV1_ALERT_INTERNAL_ERROR"), + vm, + ); + return Err(exc); + } + } + + Ok(()) + } + + // Helper to call socket methods, bypassing any SSL wrapper + pub(crate) fn sock_recv(&self, size: usize, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // In BIO mode, read from incoming BIO (flags not supported) + if let Some(ref bio) = self.incoming_bio { + let bio_obj: PyObjectRef = bio.clone().into(); + let read_method = bio_obj.get_attr("read", vm)?; + return read_method.call((vm.ctx.new_int(size),), vm); + } + + // Normal socket mode + let socket_mod = vm.import("socket", 0)?; + let socket_class = socket_mod.get_attr("socket", vm)?; + + // Call socket.socket.recv(self.sock, size, flags) + let recv_method = socket_class.get_attr("recv", vm)?; + recv_method.call((self.sock.clone(), vm.ctx.new_int(size)), vm) + } + + /// Peek at socket data without consuming it (MSG_PEEK). + /// Used during TLS shutdown to avoid consuming post-TLS cleartext data. + pub(crate) fn sock_peek(&self, size: usize, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let socket_mod = vm.import("socket", 0)?; + let socket_class = socket_mod.get_attr("socket", vm)?; + let recv_method = socket_class.get_attr("recv", vm)?; + let msg_peek = socket_mod.get_attr("MSG_PEEK", vm)?; + recv_method.call((self.sock.clone(), vm.ctx.new_int(size), msg_peek), vm) + } + + /// Socket send - just sends data, caller must handle pending flush + /// Use flush_pending_tls_output before this if ordering is important + pub(crate) fn sock_send(&self, data: &[u8], vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // In BIO mode, write to outgoing BIO + if let Some(ref bio) = self.outgoing_bio { + let bio_obj: PyObjectRef = bio.clone().into(); + let write_method = bio_obj.get_attr("write", vm)?; + return write_method.call((vm.ctx.new_bytes(data.to_vec()),), vm); + } + + // Normal socket mode + let socket_mod = vm.import("socket", 0)?; + let socket_class = socket_mod.get_attr("socket", vm)?; + + // Call socket.socket.send(self.sock, data) + let send_method = socket_class.get_attr("send", vm)?; + send_method.call((self.sock.clone(), vm.ctx.new_bytes(data.to_vec())), vm) + } + + /// Flush any pending TLS output data to the socket + /// Optional deadline parameter allows respecting a read deadline during flush + pub(crate) fn flush_pending_tls_output( + &self, + vm: &VirtualMachine, + deadline: Option<std::time::Instant>, + ) -> PyResult<()> { + let mut pending = self.pending_tls_output.lock(); + if pending.is_empty() { + return Ok(()); + } + + let socket_timeout = self.get_socket_timeout(vm)?; + let is_non_blocking = socket_timeout.map(|t| t.is_zero()).unwrap_or(false); + + let mut sent_total = 0; + + while sent_total < pending.len() { + // Calculate timeout: use deadline if provided, otherwise use socket timeout + let timeout_to_use = if let Some(dl) = deadline { + let now = std::time::Instant::now(); + if now >= dl { + // Deadline already passed + *pending = pending[sent_total..].to_vec(); + return Err( + timeout_error_msg(vm, "The operation timed out".to_string()).upcast() + ); + } + Some(dl - now) + } else { + socket_timeout + }; + + // Use sock_select directly with calculated timeout + let py_socket: PyRef<PySocket> = self.sock.clone().try_into_value(vm)?; + let socket = py_socket + .sock() + .map_err(|e| vm.new_os_error(format!("Failed to get socket: {e}")))?; + let timed_out = sock_select(&socket, SelectKind::Write, timeout_to_use) + .map_err(|e| vm.new_os_error(format!("select failed: {e}")))?; + + if timed_out { + // Keep unsent data in pending buffer + *pending = pending[sent_total..].to_vec(); + if is_non_blocking { + return Err(create_ssl_want_write_error(vm).upcast()); + } + return Err( + timeout_error_msg(vm, "The write operation timed out".to_string()).upcast(), + ); + } + + match self.sock_send(&pending[sent_total..], vm) { + Ok(result) => { + let sent: usize = result.try_to_value::<isize>(vm)?.try_into().unwrap_or(0); + if sent == 0 { + if is_non_blocking { + // Keep unsent data in pending buffer + *pending = pending[sent_total..].to_vec(); + return Err(create_ssl_want_write_error(vm).upcast()); + } + // Socket said ready but sent 0 bytes - retry + continue; + } + sent_total += sent; + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + if is_non_blocking { + // Keep unsent data in pending buffer + *pending = pending[sent_total..].to_vec(); + return Err(create_ssl_want_write_error(vm).upcast()); + } + continue; + } + // Keep unsent data in pending buffer for other errors too + *pending = pending[sent_total..].to_vec(); + return Err(e); + } + } + } + + // All data sent successfully + pending.clear(); + Ok(()) + } + + /// Send TLS output data to socket, saving unsent bytes to pending buffer + /// This prevents data loss when rustls' write_tls() drains its internal buffer + /// but the socket cannot accept all the data immediately + fn send_tls_output(&self, buf: Vec<u8>, vm: &VirtualMachine) -> PyResult<()> { + if buf.is_empty() { + return Ok(()); + } + + let timeout = self.get_socket_timeout(vm)?; + let is_non_blocking = timeout.map(|t| t.is_zero()).unwrap_or(false); + + let mut sent_total = 0; + while sent_total < buf.len() { + let timed_out = self.sock_wait_for_io_impl(SelectKind::Write, vm)?; + if timed_out { + // Save unsent data to pending buffer + self.pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err( + timeout_error_msg(vm, "The write operation timed out".to_string()).upcast(), + ); + } + + match self.sock_send(&buf[sent_total..], vm) { + Ok(result) => { + let sent: usize = result.try_to_value::<isize>(vm)?.try_into().unwrap_or(0); + if sent == 0 { + if is_non_blocking { + // Save unsent data to pending buffer + self.pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(create_ssl_want_write_error(vm).upcast()); + } + continue; + } + sent_total += sent; + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + if is_non_blocking { + // Save unsent data to pending buffer + self.pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(create_ssl_want_write_error(vm).upcast()); + } + continue; + } + // Save unsent data for other errors too + self.pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(e); + } + } + } + + Ok(()) + } + + /// Flush all pending TLS output data, respecting socket timeout + /// Used during handshake completion and shutdown() to ensure all data is sent + pub(crate) fn blocking_flush_all_pending(&self, vm: &VirtualMachine) -> PyResult<()> { + // Get socket timeout to respect during flush + let timeout = self.get_socket_timeout(vm)?; + if timeout.map(|t| t.is_zero()).unwrap_or(false) { + return self.flush_pending_tls_output(vm, None); + } + + loop { + let pending_data = { + let pending = self.pending_tls_output.lock(); + if pending.is_empty() { + return Ok(()); + } + pending.clone() + }; + + // Wait for socket to be writable, respecting socket timeout + let py_socket: PyRef<PySocket> = self.sock.clone().try_into_value(vm)?; + let socket = py_socket + .sock() + .map_err(|e| vm.new_os_error(format!("Failed to get socket: {e}")))?; + let timed_out = sock_select(&socket, SelectKind::Write, timeout) + .map_err(|e| vm.new_os_error(format!("select failed: {e}")))?; + + if timed_out { + return Err( + timeout_error_msg(vm, "The write operation timed out".to_string()).upcast(), + ); + } + + // Try to send pending data (use raw to avoid recursion) + match self.sock_send(&pending_data, vm) { + Ok(result) => { + let sent: usize = result.try_to_value::<isize>(vm)?.try_into().unwrap_or(0); + if sent > 0 { + let mut pending = self.pending_tls_output.lock(); + pending.drain(..sent); + } + // If sent == 0, loop will retry with sock_select + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + continue; + } + return Err(e); + } + } + } + } + + // Helper function to convert Python PROTO_* constants to rustls versions + fn get_rustls_versions( + minimum: i32, + maximum: i32, + options: i32, + ) -> &'static [&'static rustls::SupportedProtocolVersion] { + // Rustls only supports TLS 1.2 and 1.3 + // PROTO_TLSv1_2 = 0x0303, PROTO_TLSv1_3 = 0x0304 + // PROTO_MINIMUM_SUPPORTED = -2, PROTO_MAXIMUM_SUPPORTED = -1 + // If minimum and maximum are 0, use default (both TLS 1.2 and 1.3) + + // Static arrays for single-version configurations + static TLS12_ONLY: &[&rustls::SupportedProtocolVersion] = &[&TLS12]; + static TLS13_ONLY: &[&rustls::SupportedProtocolVersion] = &[&TLS13]; + + // Normalize special values: -2 (MINIMUM_SUPPORTED) → TLS 1.2, -1 (MAXIMUM_SUPPORTED) → TLS 1.3 + let min = if minimum == -2 { + PROTO_TLSv1_2 + } else { + minimum + }; + let max = if maximum == -1 { + PROTO_TLSv1_3 + } else { + maximum + }; + + // Check if versions are disabled by options + let tls12_disabled = (options & OP_NO_TLSv1_2) != 0; + let tls13_disabled = (options & OP_NO_TLSv1_3) != 0; + + let want_tls12 = (min == 0 || min <= PROTO_TLSv1_2) + && (max == 0 || max >= PROTO_TLSv1_2) + && !tls12_disabled; + let want_tls13 = (min == 0 || min <= PROTO_TLSv1_3) + && (max == 0 || max >= PROTO_TLSv1_3) + && !tls13_disabled; + + match (want_tls12, want_tls13) { + (true, true) => rustls::DEFAULT_VERSIONS, // Both TLS 1.2 and 1.3 + (true, false) => TLS12_ONLY, // Only TLS 1.2 + (false, true) => TLS13_ONLY, // Only TLS 1.3 + (false, false) => rustls::DEFAULT_VERSIONS, // Fallback to default + } + } + + /// Helper: Prepare TLS versions from context settings + fn prepare_tls_versions(&self) -> &'static [&'static rustls::SupportedProtocolVersion] { + let ctx = self.context.read(); + let min_ver = *ctx.minimum_version.read(); + let max_ver = *ctx.maximum_version.read(); + let options = *ctx.options.read(); + Self::get_rustls_versions(min_ver, max_ver, options) + } + + /// Helper: Prepare KX groups (ECDH curve) from context settings + fn prepare_kx_groups( + &self, + vm: &VirtualMachine, + ) -> PyResult<Option<Vec<&'static dyn SupportedKxGroup>>> { + let ctx = self.context.read(); + let ecdh_curve = ctx.ecdh_curve.read().clone(); + drop(ctx); + + if let Some(ref curve_name) = ecdh_curve { + match curve_name_to_kx_group(curve_name) { + Ok(groups) => Ok(Some(groups)), + Err(e) => Err(vm.new_value_error(format!("Failed to set ECDH curve: {e}"))), + } + } else { + Ok(None) + } + } + + /// Helper: Prepare all common protocol settings (versions, KX groups, ciphers, ALPN) + fn prepare_protocol_settings(&self, vm: &VirtualMachine) -> PyResult<ProtocolSettings> { + let ctx = self.context.read(); + let versions = self.prepare_tls_versions(); + let kx_groups = self.prepare_kx_groups(vm)?; + let cipher_suites = ctx.selected_ciphers.read().clone(); + let alpn_protocols = ctx.alpn_protocols.read().clone(); + + Ok(ProtocolSettings { + versions, + kx_groups, + cipher_suites, + alpn_protocols, + }) + } + + /// Initialize server-side TLS connection with configuration + /// + /// This method handles all server-side setup including: + /// - Certificate and key validation + /// - Client authentication configuration + /// - SNI (Server Name Indication) setup + /// - ALPN protocol negotiation + /// - Session resumption configuration + /// + /// Returns the configured ServerConnection. + fn initialize_server_connection( + &self, + conn_guard: &mut Option<TlsConnection>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let ctx = self.context.read(); + let cert_keys = ctx.cert_keys.read(); + + if cert_keys.is_empty() { + return Err(vm.new_value_error( + "Server-side connection requires certificate and key (use load_cert_chain)", + )); + } + + // Clone cert_keys for use in config + // PrivateKeyDer doesn't implement Clone, use clone_key() + let cert_keys_clone: Vec<CertKeyPair> = cert_keys + .iter() + .map(|(ck, pk)| (ck.clone(), pk.clone_key())) + .collect(); + drop(cert_keys); + + // Prepare common protocol settings (TLS versions, ECDH curve, cipher suites, ALPN) + let protocol_settings = self.prepare_protocol_settings(vm)?; + let min_ver = *ctx.minimum_version.read(); + + // Check if client certificate verification is required + let verify_mode = *ctx.verify_mode.read(); + let root_store = ctx.root_certs.read(); + let pha_enabled = *ctx.post_handshake_auth.read(); + + // Check if TLS 1.3 is being used + let is_tls13 = min_ver >= PROTO_TLSv1_3; + + // For TLS 1.3: always use deferred validation for client certificates + // For TLS 1.2: use immediate validation during handshake + let use_deferred_validation = is_tls13 + && !pha_enabled + && (verify_mode == CERT_REQUIRED || verify_mode == CERT_OPTIONAL); + + // For TLS 1.3 + PHA: if PHA is enabled, don't request cert in initial handshake + // The certificate will be requested later via verify_client_post_handshake() + let request_initial_cert = if pha_enabled { + // PHA enabled: don't request cert initially (will use PHA later) + false + } else if verify_mode == CERT_REQUIRED || verify_mode == CERT_OPTIONAL { + // PHA not enabled or TLS 1.2: request cert in initial handshake + true + } else { + // CERT_NONE + false + }; + + // Check if SNI callback is set + let sni_callback = ctx.sni_callback.read().clone(); + let use_sni_resolver = sni_callback.is_some(); + + // Create SNI state if needed (to be stored in PySSLSocket later) + // For SNI, use the first cert_key pair as the initial certificate + let sni_state: Option<Arc<ParkingMutex<SniCertName>>> = if use_sni_resolver { + // Use first cert_key as initial certificate for SNI + // Extract CertifiedKey from tuple + let (first_cert_key, _) = &cert_keys_clone[0]; + let first_cert_key = first_cert_key.clone(); + + // Check if we already have existing SNI state (from previous connection) + let existing_sni_state = self.sni_state.read().clone(); + + if let Some(sni_state_arc) = existing_sni_state { + // Reuse existing Arc and update its contents + // This is crucial: rustls SniCertResolver holds references to this Arc + let mut state = sni_state_arc.lock(); + state.0 = first_cert_key; + state.1 = None; // Reset SNI name for new connection + drop(state); + + // Return the existing Arc (not a new one!) + Some(sni_state_arc) + } else { + // First connection: create new SNI state + Some(Arc::new(ParkingMutex::new((first_cert_key, None)))) + } + } else { + None + }; + + // Determine which cert resolver to use + // Priority: SNI > Multi-cert/Single-cert via MultiCertResolver + let cert_resolver: Option<Arc<dyn ResolvesServerCert>> = if use_sni_resolver { + // SNI takes precedence - use first cert_key for initial setup + sni_state.as_ref().map(|sni_state_arc| { + Arc::new(SniCertResolver { + sni_state: sni_state_arc.clone(), + }) as Arc<dyn ResolvesServerCert> + }) + } else { + // Use MultiCertResolver for all cases (single or multiple certs) + // Extract CertifiedKey from tuples for MultiCertResolver + let cert_keys_only: Vec<Arc<CertifiedKey>> = + cert_keys_clone.iter().map(|(ck, _)| ck.clone()).collect(); + Some(Arc::new(MultiCertResolver::new(cert_keys_only))) + }; + + // Extract cert_chain and private_key from first cert_key + // + // Note: Since we always use cert_resolver now, these values won't actually be used + // by create_server_config. But we still need to provide them for the API signature. + let (first_cert_key, _) = &cert_keys_clone[0]; + let certs_clone = first_cert_key.cert.clone(); + + // Provide a dummy key since cert_resolver will handle cert selection + let key_clone = PrivateKeyDer::Pkcs8(Vec::new().into()); + + // Get shared server session storage and ticketer from context + let server_session_storage = ctx.rustls_server_session_store.clone(); + let server_ticketer = ctx.server_ticketer.clone(); + + // Build server config using compat helper + let config_options = ServerConfigOptions { + protocol_settings, + cert_chain: certs_clone, + private_key: key_clone, + root_store: if request_initial_cert { + Some(root_store.clone()) + } else { + None + }, + request_client_cert: request_initial_cert, + use_deferred_validation, + cert_resolver, + deferred_cert_error: if use_deferred_validation { + Some(self.deferred_cert_error.clone()) + } else { + None + }, + session_storage: Some(server_session_storage), + ticketer: Some(server_ticketer), + }; + + drop(root_store); + + // Check if we have a cached ServerConfig + let cached_config_arc = ctx.server_config.read().clone(); + drop(ctx); + + let config_arc = if let Some(cached) = cached_config_arc { + // Don't use cache when SNI is enabled, because each connection needs + // a fresh SniCertResolver with the correct Arc references + if use_sni_resolver { + let config = + create_server_config(config_options).map_err(|e| vm.new_value_error(e))?; + Arc::new(config) + } else { + cached + } + } else { + let config = + create_server_config(config_options).map_err(|e| vm.new_value_error(e))?; + let config_arc = Arc::new(config); + + // Cache the ServerConfig for future connections + let ctx = self.context.read(); + *ctx.server_config.write() = Some(config_arc.clone()); + drop(ctx); + + config_arc + }; + + let conn = ServerConnection::new(config_arc).map_err(|e| { + vm.new_value_error(format!("Failed to create server connection: {e}")) + })?; + + *conn_guard = Some(TlsConnection::Server(conn)); + + // If ClientHello buffer exists (from SNI callback), re-inject it + if let Some(ref hello_data) = *self.client_hello_buffer.lock() + && let Some(TlsConnection::Server(ref mut server)) = *conn_guard + { + let mut cursor = std::io::Cursor::new(hello_data.as_slice()); + let _ = server.read_tls(&mut cursor); + + // Process the re-injected ClientHello + let _ = server.process_new_packets(); + + // DON'T clear buffer - keep it to prevent callback from being invoked again + // The buffer being non-empty signals that SNI callback was already processed + } + + // Store SNI state if we're using SNI resolver + if let Some(sni_state_arc) = sni_state { + *self.sni_state.write() = Some(sni_state_arc); + } + + Ok(()) + } + + #[pymethod] + fn do_handshake(&self, vm: &VirtualMachine) -> PyResult<()> { + // Check if handshake already done + if *self.handshake_done.lock() { + return Ok(()); + } + + let mut conn_guard = self.connection.lock(); + + // Initialize connection if not already done + if conn_guard.is_none() { + // Check for pending context change (from SNI callback) + if let Some(new_ctx) = self.pending_context.write().take() { + *self.context.write() = new_ctx; + } + + if self.server_side { + // Server-side connection - delegate to helper method + self.initialize_server_connection(&mut conn_guard, vm)?; + } else { + // Client-side connection + let ctx = self.context.read(); + + // Prepare common protocol settings (TLS versions, ECDH curve, cipher suites, ALPN) + let protocol_settings = self.prepare_protocol_settings(vm)?; + + // Clone values we need before building config + let verify_mode = *ctx.verify_mode.read(); + let root_store_clone = ctx.root_certs.read().clone(); + let ca_certs_der_clone = ctx.ca_certs_der.read().clone(); + + // For client mTLS: extract cert_chain and private_key from first cert_key (if any) + // Now we store both CertifiedKey and PrivateKeyDer as tuple + let cert_keys_guard = ctx.cert_keys.read(); + let (cert_chain_clone, private_key_opt) = if !cert_keys_guard.is_empty() { + let (first_cert_key, private_key) = &cert_keys_guard[0]; + let certs = first_cert_key.cert.clone(); + (certs, Some(private_key.clone_key())) + } else { + (Vec::new(), None) + }; + drop(cert_keys_guard); + + let check_hostname = *ctx.check_hostname.read(); + let verify_flags = *ctx.verify_flags.read(); + + // Get session store before dropping ctx + let session_store = ctx.rustls_session_store.clone(); + + // Get CRLs for revocation checking + let crls_clone = ctx.crls.read().clone(); + + // Drop ctx early to avoid borrow conflicts + drop(ctx); + + // Build client config using compat helper + let config_options = ClientConfigOptions { + protocol_settings, + root_store: if verify_mode != CERT_NONE { + Some(root_store_clone) + } else { + None + }, + ca_certs_der: ca_certs_der_clone, + cert_chain: if !cert_chain_clone.is_empty() { + Some(cert_chain_clone) + } else { + None + }, + private_key: private_key_opt, + verify_server_cert: verify_mode != CERT_NONE, + check_hostname, + verify_flags, + session_store: Some(session_store), + crls: crls_clone, + }; + + let config = + create_client_config(config_options).map_err(|e| vm.new_value_error(e))?; + + // Parse server name for SNI + // Convert to ServerName + use rustls::pki_types::ServerName; + let hostname_opt = self.server_hostname.read().clone(); + + let server_name = if let Some(ref hostname) = hostname_opt { + // Use the provided hostname for SNI + ServerName::try_from(hostname.clone()).map_err(|e| { + vm.new_value_error(format!("Invalid server hostname: {e:?}")) + })? + } else { + // When server_hostname=None, use an IP address to suppress SNI + // no hostname = no SNI extension + ServerName::IpAddress( + core::net::IpAddr::V4(core::net::Ipv4Addr::new(127, 0, 0, 1)).into(), + ) + }; + + let conn = ClientConnection::new(Arc::new(config), server_name.clone()) + .map_err(|e| { + vm.new_value_error(format!("Failed to create client connection: {e}")) + })?; + + *conn_guard = Some(TlsConnection::Client(conn)); + } + } + + // Perform the actual handshake by exchanging data with the socket/BIO + + let conn = conn_guard.as_mut().expect("unreachable"); + let is_client = matches!(conn, TlsConnection::Client(_)); + let handshake_result = ssl_do_handshake(conn, self, vm); + drop(conn_guard); + + if is_client { + // CLIENT is simple - no SNI callback handling needed + handshake_result.map_err(|e| e.into_py_err(vm))?; + self.complete_handshake(vm)?; + Ok(()) + } else { + // Use OpenSSL-compatible handshake for server + // Handle SNI callback restart + match handshake_result { + Ok(()) => { + // Handshake completed successfully + self.complete_handshake(vm)?; + Ok(()) + } + Err(SslError::SniCallbackRestart) => { + // SNI detected - need to call callback and recreate connection + + // Get the SNI name that was extracted (may be None if client didn't send SNI) + let sni_name = self.get_extracted_sni_name(); + + // Now safe to call Python callback (no locks held) + self.invoke_sni_callback(sni_name.as_deref(), vm)?; + + // Clear connection to trigger recreation + *self.connection.lock() = None; + + // Recursively call do_handshake to recreate with new context + self.do_handshake(vm) + } + Err(e) => { + // Other errors - convert to Python exception + Err(e.into_py_err(vm)) + } + } + } + } + + #[pymethod] + fn read( + &self, + len: OptionalArg<isize>, + buffer: OptionalArg<ArgMemoryBuffer>, + vm: &VirtualMachine, + ) -> PyResult { + // Convert len to usize, defaulting to 1024 if not provided + // -1 means read all available data (treat as large buffer size) + let len_val = len.unwrap_or(PEM_BUFSIZE as isize); + let mut len = if len_val == -1 { + // -1 is only valid when a buffer is provided + match &buffer { + OptionalArg::Present(buf_arg) => buf_arg.len(), + OptionalArg::Missing => { + return Err(vm.new_value_error("negative read length")); + } + } + } else if len_val < 0 { + return Err(vm.new_value_error("negative read length")); + } else { + len_val as usize + }; + + // if buffer is provided, limit len to buffer size + if let OptionalArg::Present(buf_arg) = &buffer { + let buf_len = buf_arg.len(); + if len_val <= 0 || len > buf_len { + len = buf_len; + } + } + + // return empty bytes immediately for len=0 + if len == 0 { + return match buffer { + OptionalArg::Present(_) => Ok(vm.ctx.new_int(0).into()), + OptionalArg::Missing => Ok(vm.ctx.new_bytes(vec![]).into()), + }; + } + + // Ensure handshake is done - if not, complete it first + // This matches OpenSSL behavior where SSL_read() auto-completes handshake + if !*self.handshake_done.lock() { + self.do_handshake(vm)?; + } + + // Check if connection has been shut down + // Only block after shutdown is COMPLETED, not during shutdown process + let shutdown_state = *self.shutdown_state.lock(); + if shutdown_state == ShutdownState::Completed { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "cannot read after shutdown", + ) + .upcast()); + } + + // Helper function to handle return value based on buffer presence + let return_data = |data: Vec<u8>, + buffer_arg: &OptionalArg<ArgMemoryBuffer>, + vm: &VirtualMachine| + -> PyResult<PyObjectRef> { + match buffer_arg { + OptionalArg::Present(buf_arg) => { + // Write into buffer and return number of bytes written + let n = data.len(); + if n > 0 { + let mut buf = buf_arg.borrow_buf_mut(); + let buf_slice = &mut *buf; + let copy_len = n.min(buf_slice.len()); + buf_slice[..copy_len].copy_from_slice(&data[..copy_len]); + } + Ok(vm.ctx.new_int(n).into()) + } + OptionalArg::Missing => { + // Return bytes object + Ok(vm.ctx.new_bytes(data).into()) + } + } + }; + + // Use compat layer for unified read logic with proper EOF handling + // This matches SSL_read_ex() approach + let mut buf = vec![0u8; len]; + let read_result = { + let mut conn_guard = self.connection.lock(); + let conn = conn_guard + .as_mut() + .ok_or_else(|| vm.new_value_error("Connection not established"))?; + crate::ssl::compat::ssl_read(conn, &mut buf, self, vm) + }; + match read_result { + Ok(n) => { + // Check for deferred certificate verification errors (TLS 1.3) + // Must be checked AFTER ssl_read, as the error is set during I/O + self.check_deferred_cert_error(vm)?; + buf.truncate(n); + return_data(buf, &buffer, vm) + } + Err(crate::ssl::compat::SslError::Eof) => { + // If plaintext is still buffered, return it before EOF. + let pending = { + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(conn) => conn, + None => return Err(create_ssl_eof_error(vm).upcast()), + }; + + let mut reader = conn.reader(); + reader.fill_buf().map(|buf| buf.len()).unwrap_or(0) + }; + if pending > 0 { + let mut buf = vec![0u8; pending.min(len)]; + let read_retry = { + let mut conn_guard = self.connection.lock(); + let conn = conn_guard + .as_mut() + .ok_or_else(|| vm.new_value_error("Connection not established"))?; + crate::ssl::compat::ssl_read(conn, &mut buf, self, vm) + }; + if let Ok(n) = read_retry { + buf.truncate(n); + return return_data(buf, &buffer, vm); + } + } + // EOF occurred in violation of protocol (unexpected closure) + Err(create_ssl_eof_error(vm).upcast()) + } + Err(crate::ssl::compat::SslError::ZeroReturn) => { + // If plaintext is still buffered, return it before clean EOF. + let pending = { + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(conn) => conn, + None => return Err(create_ssl_zero_return_error(vm).upcast()), + }; + + let mut reader = conn.reader(); + reader.fill_buf().map(|buf| buf.len()).unwrap_or(0) + }; + if pending > 0 { + let mut buf = vec![0u8; pending.min(len)]; + let read_retry = { + let mut conn_guard = self.connection.lock(); + let conn = conn_guard + .as_mut() + .ok_or_else(|| vm.new_value_error("Connection not established"))?; + crate::ssl::compat::ssl_read(conn, &mut buf, self, vm) + }; + if let Ok(n) = read_retry { + buf.truncate(n); + return return_data(buf, &buffer, vm); + } + } + // Clean closure via close_notify from peer. + // If we already sent close_notify (unwrap was called), + // raise SSLZeroReturnError (bidirectional shutdown). + // Otherwise return empty bytes, which callers (asyncore, + // asyncio sslproto) interpret as EOF. + let our_shutdown_state = *self.shutdown_state.lock(); + if our_shutdown_state == ShutdownState::SentCloseNotify + || our_shutdown_state == ShutdownState::Completed + { + Err(create_ssl_zero_return_error(vm).upcast()) + } else { + return_data(vec![], &buffer, vm) + } + } + Err(crate::ssl::compat::SslError::WantRead) => { + // Non-blocking mode: would block + Err(create_ssl_want_read_error(vm).upcast()) + } + Err(crate::ssl::compat::SslError::WantWrite) => { + // Non-blocking mode: would block on write + Err(create_ssl_want_write_error(vm).upcast()) + } + Err(crate::ssl::compat::SslError::Timeout(msg)) => { + Err(timeout_error_msg(vm, msg).upcast()) + } + Err(crate::ssl::compat::SslError::Py(e)) => { + // Python exception - pass through + Err(e) + } + Err(e) => { + // Other SSL errors + Err(e.into_py_err(vm)) + } + } + } + + #[pymethod] + fn pending(&self) -> PyResult<usize> { + // Returns the number of already decrypted bytes available for read + // This is critical for asyncore's readable() method which checks socket.pending() > 0 + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(c) => c, + None => return Ok(0), // No connection established yet + }; + + // Use rustls Reader's fill_buf() to check buffered plaintext + // fill_buf() returns a reference to buffered data without consuming it + // This matches OpenSSL's SSL_pending() behavior + let mut reader = conn.reader(); + match reader.fill_buf() { + Ok(buf) => Ok(buf.len()), + Err(_) => { + // WouldBlock or other errors mean no data available + // Return 0 like OpenSSL does when buffer is empty + Ok(0) + } + } + } + + #[pymethod] + fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { + let data_bytes = data.borrow_buf(); + let data_len = data_bytes.len(); + + if data_len == 0 { + return Ok(0); + } + + // Ensure handshake is done (SSL_write auto-completes handshake) + if !*self.handshake_done.lock() { + self.do_handshake(vm)?; + } + + // Check shutdown state + // Only block after shutdown is COMPLETED, not during shutdown process + if *self.shutdown_state.lock() == ShutdownState::Completed { + return Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "cannot write after shutdown", + ) + .upcast()); + } + + // Call ssl_write (matches CPython's SSL_write_ex loop) + let result = { + let mut conn_guard = self.connection.lock(); + let conn = conn_guard + .as_mut() + .ok_or_else(|| vm.new_value_error("Connection not established"))?; + + crate::ssl::compat::ssl_write(conn, data_bytes.as_ref(), self, vm) + }; + + match result { + Ok(n) => { + self.check_deferred_cert_error(vm)?; + Ok(n) + } + Err(crate::ssl::compat::SslError::WantRead) => { + Err(create_ssl_want_read_error(vm).upcast()) + } + Err(crate::ssl::compat::SslError::WantWrite) => { + Err(create_ssl_want_write_error(vm).upcast()) + } + Err(crate::ssl::compat::SslError::Timeout(msg)) => { + Err(timeout_error_msg(vm, msg).upcast()) + } + Err(e) => Err(e.into_py_err(vm)), + } + } + + #[pymethod] + fn getpeercert( + &self, + args: GetCertArgs, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + let binary = args.binary_form.unwrap_or(false); + + // Check if handshake is complete + if !*self.handshake_done.lock() { + return Err(vm.new_value_error("handshake not done yet")); + } + + // Extract DER bytes from connection, releasing lock quickly + let der_bytes = { + let conn_guard = self.connection.lock(); + let conn = conn_guard + .as_ref() + .ok_or_else(|| vm.new_value_error("No TLS connection established"))?; + + let Some(peer_certificates) = conn.peer_certificates() else { + return Ok(None); + }; + let cert = peer_certificates + .first() + .ok_or_else(|| vm.new_value_error("No peer certificate available"))?; + cert.as_ref().to_vec() + }; + + if binary { + // Return DER-encoded certificate as bytes + return Ok(Some(vm.ctx.new_bytes(der_bytes).into())); + } + + // Dictionary mode: check verify_mode + let verify_mode = *self.context.read().verify_mode.read(); + + if verify_mode == CERT_NONE { + // Return empty dict when CERT_NONE + return Ok(Some(vm.ctx.new_dict().into())); + } + + // Parse DER certificate and convert to dict (outside lock) + let (_, cert) = x509_parser::parse_x509_certificate(&der_bytes) + .map_err(|e| vm.new_value_error(format!("Failed to parse certificate: {e}")))?; + + cert::cert_to_dict(vm, &cert).map(Some) + } + + #[pymethod] + fn cipher(&self) -> Option<(String, String, i32)> { + // Extract cipher suite, releasing lock quickly + let suite = { + let conn_guard = self.connection.lock(); + conn_guard.as_ref()?.negotiated_cipher_suite()? + }; + + // Extract cipher information outside the lock + let cipher_info = extract_cipher_info(&suite); + + // Note: returns a 3-tuple (name, protocol_version, bits) + // The 'description' field is part of get_ciphers() output, not cipher() + Some(( + cipher_info.name, + cipher_info.protocol.to_string(), + cipher_info.bits, + )) + } + + #[pymethod] + fn version(&self) -> Option<String> { + // Extract cipher suite, releasing lock quickly + let suite = { + let conn_guard = self.connection.lock(); + conn_guard.as_ref()?.negotiated_cipher_suite()? + }; + + // Convert to string outside the lock + let version_str = match suite.version().version { + rustls::ProtocolVersion::TLSv1_2 => "TLSv1.2", + rustls::ProtocolVersion::TLSv1_3 => "TLSv1.3", + _ => "Unknown", + }; + + Some(version_str.to_string()) + } + + #[pymethod] + fn selected_alpn_protocol(&self) -> Option<String> { + let conn_guard = self.connection.lock(); + let conn = conn_guard.as_ref()?; + + let alpn_bytes = conn.alpn_protocol()?; + + // Null byte protocol (vec![0u8]) means no actual ALPN match (fallback protocol) + if alpn_bytes.is_empty() || alpn_bytes == [0u8] { + return None; + } + + // Convert bytes to string + String::from_utf8(alpn_bytes.to_vec()).ok() + } + + #[pymethod] + fn selected_npn_protocol(&self) -> Option<String> { + // NPN (Next Protocol Negotiation) is the predecessor to ALPN + // It was deprecated in favor of ALPN (RFC 7301) + // Rustls doesn't support NPN, only ALPN + // Return None to indicate NPN is not supported + None + } + + #[pygetset] + fn owner(&self) -> Option<PyObjectRef> { + self.owner.read().clone() + } + + #[pygetset(setter)] + fn set_owner(&self, owner: PyObjectRef, _vm: &VirtualMachine) -> PyResult<()> { + *self.owner.write() = Some(owner); + Ok(()) + } + + #[pygetset] + fn server_side(&self) -> bool { + self.server_side + } + + #[pygetset] + fn context(&self) -> PyRef<PySSLContext> { + self.context.read().clone() + } + + #[pygetset(setter)] + fn set_context(&self, value: PyRef<PySSLContext>, _vm: &VirtualMachine) -> PyResult<()> { + // Update context reference immediately + // SSL_set_SSL_CTX allows context changes at any time, + // even after handshake completion + *self.context.write() = value; + + // Clear pending context as we've applied the change + *self.pending_context.write() = None; + + Ok(()) + } + + #[pygetset] + fn server_hostname(&self) -> Option<String> { + self.server_hostname.read().clone() + } + + #[pygetset(setter)] + fn set_server_hostname( + &self, + value: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if handshake is already done + if *self.handshake_done.lock() { + return Err( + vm.new_value_error("Cannot set server_hostname on socket after handshake") + ); + } + + // Validate hostname + let hostname_string = value + .map(|s| { + validate_hostname(s.as_str(), vm)?; + Ok::<String, _>(s.as_str().to_owned()) + }) + .transpose()?; + + *self.server_hostname.write() = hostname_string; + Ok(()) + } + + #[pygetset] + fn session(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Return the stored session object if any + let sess = self.session.read().clone(); + if let Some(s) = sess { + Ok(s) + } else { + Ok(vm.ctx.none()) + } + } + + #[pygetset(setter)] + fn set_session(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Validate that value is an SSLSession + if !value.is(vm.ctx.types.none_type) { + // Try to downcast to SSLSession to validate + let _ = value + .downcast_ref::<PySSLSession>() + .ok_or_else(|| vm.new_type_error("Value is not a SSLSession."))?; + } + + // Check if this is a client socket + if self.server_side { + return Err(vm.new_value_error("Cannot set session for server-side SSLSocket")); + } + + // Check if handshake is already done + if *self.handshake_done.lock() { + return Err(vm.new_value_error("Cannot set session after handshake.")); + } + + // Store the session for potential use during handshake + *self.session.write() = if value.is(vm.ctx.types.none_type) { + None + } else { + Some(value) + }; + + Ok(()) + } + + #[pygetset] + fn session_reused(&self) -> bool { + // Return the tracked session reuse status + *self.session_was_reused.lock() + } + + #[pymethod] + fn compression(&self) -> Option<&'static str> { + // rustls doesn't support compression + None + } + + #[pymethod] + fn get_unverified_chain(&self, vm: &VirtualMachine) -> PyResult<Option<PyListRef>> { + // Get peer certificates from the connection + let conn_guard = self.connection.lock(); + let conn = conn_guard + .as_ref() + .ok_or_else(|| vm.new_value_error("Handshake not completed"))?; + + let certs = conn.peer_certificates(); + + let Some(certs) = certs else { + return Ok(None); + }; + + // Convert to list of Certificate objects + let cert_list: Vec<PyObjectRef> = certs + .iter() + .map(|cert_der| { + let cert_bytes = cert_der.as_ref().to_vec(); + PySSLCertificate { + der_bytes: cert_bytes, + } + .into_ref(&vm.ctx) + .into() + }) + .collect(); + + Ok(Some(vm.ctx.new_list(cert_list))) + } + + #[pymethod] + fn get_verified_chain(&self, vm: &VirtualMachine) -> PyResult<Option<PyListRef>> { + // Get peer certificates (what peer sent during handshake) + let conn_guard = self.connection.lock(); + let Some(ref conn) = *conn_guard else { + return Ok(None); + }; + + let peer_certs = conn.peer_certificates(); + + let Some(peer_certs_slice) = peer_certs else { + return Ok(None); + }; + + // Build the verified chain using cert module + let ctx_guard = self.context.read(); + let ca_certs_der = ctx_guard.ca_certs_der.read(); + + let chain_der = cert::build_verified_chain(peer_certs_slice, &ca_certs_der); + + // Convert DER chain to Python list of Certificate objects + let cert_list: Vec<PyObjectRef> = chain_der + .into_iter() + .map(|der_bytes| PySSLCertificate { der_bytes }.into_ref(&vm.ctx).into()) + .collect(); + + Ok(Some(vm.ctx.new_list(cert_list))) + } + + #[pymethod] + fn shutdown(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Check current shutdown state + let current_state = *self.shutdown_state.lock(); + + // If already completed, return immediately + if current_state == ShutdownState::Completed { + if self.is_bio_mode() { + return Ok(vm.ctx.none()); + } + return Ok(self.sock.clone()); + } + + // Get connection + let mut conn_guard = self.connection.lock(); + let conn = conn_guard + .as_mut() + .ok_or_else(|| vm.new_value_error("Connection not established"))?; + + let is_bio = self.is_bio_mode(); + + // Step 1: Send our close_notify if not already sent + if current_state == ShutdownState::NotStarted { + // First, flush ALL pending TLS data BEFORE sending close_notify + // This is CRITICAL - close_notify must come AFTER all application data + // Otherwise data loss occurs when peer receives close_notify first + + // Step 1a: Flush any pending TLS records from rustls internal buffer + // This ensures all application data is converted to TLS records + while conn.wants_write() { + let mut buf = Vec::new(); + conn.write_tls(&mut buf) + .map_err(|e| vm.new_os_error(format!("TLS write failed: {e}")))?; + if !buf.is_empty() { + self.send_tls_output(buf, vm)?; + } + } + + // Step 1b: Flush pending_tls_output buffer to socket + if !is_bio { + // Socket mode: blocking flush to ensure data order + // Must complete before sending close_notify + self.blocking_flush_all_pending(vm)?; + } else { + // BIO mode: non-blocking flush (caller handles pending data) + let _ = self.flush_pending_tls_output(vm, None); + } + + conn.send_close_notify(); + + // Write close_notify to outgoing buffer/BIO + self.write_pending_tls(conn, vm)?; + // Ensure close_notify and any pending TLS data are flushed + if !is_bio { + self.flush_pending_tls_output(vm, None)?; + } + + // Update state + *self.shutdown_state.lock() = ShutdownState::SentCloseNotify; + } + + // Step 2: Try to read and process peer's close_notify + + // First check if we already have peer's close_notify + // This can happen if it was received during a previous read() call + let mut peer_closed = self.check_peer_closed(conn, vm)?; + + // If peer hasn't closed yet, try to read from socket + if !peer_closed { + // Check socket timeout mode + let timeout_mode = if !is_bio { + // Get socket timeout + match self.sock.get_attr("gettimeout", vm) { + Ok(method) => match method.call((), vm) { + Ok(timeout) => { + if vm.is_none(&timeout) { + // timeout=None means blocking + Some(None) + } else if let Ok(t) = timeout.try_float(vm).map(|f| f.to_f64()) { + if t == 0.0 { + // timeout=0 means non-blocking + Some(Some(0.0)) + } else { + // timeout>0 means timeout mode + Some(Some(t)) + } + } else { + None + } + } + Err(_) => None, + }, + Err(_) => None, + } + } else { + None // BIO mode + }; + + if is_bio { + // In BIO mode: non-blocking read attempt + if self.try_read_close_notify(conn, vm)? { + peer_closed = true; + } + } else if let Some(timeout) = timeout_mode { + match timeout { + Some(0.0) => { + // Non-blocking: return immediately after sending close_notify. + // Don't wait for peer's close_notify to avoid blocking. + drop(conn_guard); + // Best-effort flush; WouldBlock is expected in non-blocking mode. + // Other errors indicate close_notify may not have been sent, + // but we still complete shutdown to avoid inconsistent state. + let _ = self.flush_pending_tls_output(vm, None); + *self.shutdown_state.lock() = ShutdownState::Completed; + *self.connection.lock() = None; + return Ok(self.sock.clone()); + } + _ => { + // Blocking or timeout mode: wait for peer's close_notify. + // This is proper TLS shutdown - we should receive peer's + // close_notify before closing the connection. + drop(conn_guard); + + // Flush our close_notify first + if timeout.is_none() { + self.blocking_flush_all_pending(vm)?; + } else { + self.flush_pending_tls_output(vm, None)?; + } + + // Calculate deadline for timeout mode + let deadline = timeout.map(|t| { + std::time::Instant::now() + core::time::Duration::from_secs_f64(t) + }); + + // Wait for peer's close_notify + loop { + // Re-acquire connection lock for each iteration + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(c) => c, + None => break, // Connection already closed + }; + + // Check if peer already sent close_notify + if self.check_peer_closed(conn, vm)? { + break; + } + + drop(conn_guard); + + // Check timeout + let remaining_timeout = if let Some(dl) = deadline { + let now = std::time::Instant::now(); + if now >= dl { + // Timeout reached - raise TimeoutError + return Err(timeout_error_msg( + vm, + "The read operation timed out".to_string(), + ) + .upcast()); + } + Some(dl - now) + } else { + None // Blocking mode: no timeout + }; + + // Wait for socket to be readable + let timed_out = self.sock_wait_for_io_with_timeout( + SelectKind::Read, + remaining_timeout, + vm, + )?; + + if timed_out { + // Timeout waiting for peer's close_notify + return Err(timeout_error_msg( + vm, + "The read operation timed out".to_string(), + ) + .upcast()); + } + + // Try to read data from socket + let mut conn_guard = self.connection.lock(); + let conn = match conn_guard.as_mut() { + Some(c) => c, + None => break, + }; + + // Read and process any incoming TLS data + match self.try_read_close_notify(conn, vm) { + Ok(closed) => { + if closed { + break; + } + // Check again after processing + if self.check_peer_closed(conn, vm)? { + break; + } + } + Err(_) => { + // Socket error - peer likely closed connection + break; + } + } + } + + // Shutdown complete + *self.shutdown_state.lock() = ShutdownState::Completed; + *self.connection.lock() = None; + return Ok(self.sock.clone()); + } + } + } + + // Step 3: Check again if peer has sent close_notify (non-blocking/BIO mode only) + if !peer_closed { + peer_closed = self.check_peer_closed(conn, vm)?; + } + } + + drop(conn_guard); // Release lock before returning + + if !peer_closed { + // Still waiting for peer's close-notify + // Raise SSLWantReadError to signal app needs to transfer data + // This is correct for non-blocking sockets and BIO mode + return Err(create_ssl_want_read_error(vm).upcast()); + } + // Both close-notify exchanged, shutdown complete + *self.shutdown_state.lock() = ShutdownState::Completed; + + if is_bio { + return Ok(vm.ctx.none()); + } + Ok(self.sock.clone()) + } + + // Helper: Write all pending TLS data (including close_notify) to outgoing buffer/BIO + fn write_pending_tls(&self, conn: &mut TlsConnection, vm: &VirtualMachine) -> PyResult<()> { + // First, flush any previously pending TLS output + // Must succeed before sending new data to maintain order + self.flush_pending_tls_output(vm, None)?; + + loop { + if !conn.wants_write() { + break; + } + + let mut buf = vec![0u8; SSL3_RT_MAX_PLAIN_LENGTH]; + let written = conn + .write_tls(&mut buf.as_mut_slice()) + .map_err(|e| vm.new_os_error(format!("TLS write failed: {e}")))?; + + if written == 0 { + break; + } + + // Send TLS data, saving unsent bytes to pending buffer if needed + self.send_tls_output(buf[..written].to_vec(), vm)?; + } + + Ok(()) + } + + // Helper: Try to read incoming data from socket/BIO + // Returns true if peer closed connection (with or without close_notify) + fn try_read_close_notify( + &self, + conn: &mut TlsConnection, + vm: &VirtualMachine, + ) -> PyResult<bool> { + // In socket mode, peek first to avoid consuming post-TLS cleartext + // data. During STARTTLS, after close_notify exchange, the socket + // transitions to cleartext. Without peeking, sock_recv may consume + // cleartext data meant for the application after unwrap(). + if self.incoming_bio.is_none() { + return self.try_read_close_notify_socket(conn, vm); + } + + // BIO mode: read from incoming BIO + match self.sock_recv(SSL3_RT_MAX_PLAIN_LENGTH, vm) { + Ok(bytes_obj) => { + let bytes = ArgBytesLike::try_from_object(vm, bytes_obj)?; + let data = bytes.borrow_buf(); + + if data.is_empty() { + if let Some(ref bio) = self.incoming_bio { + // BIO mode: check if EOF was signaled via write_eof() + let bio_obj: PyObjectRef = bio.clone().into(); + let eof_attr = bio_obj.get_attr("eof", vm)?; + let is_eof = eof_attr.try_to_bool(vm)?; + if !is_eof { + return Ok(false); + } + } + return Ok(true); + } + + let data_slice: &[u8] = data.as_ref(); + let mut cursor = std::io::Cursor::new(data_slice); + let _ = conn.read_tls(&mut cursor); + let _ = conn.process_new_packets(); + Ok(false) + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Ok(false); + } + Ok(true) + } + } + } + + /// Socket-mode close_notify reader that respects TLS record boundaries. + /// Uses MSG_PEEK to inspect data before consuming, preventing accidental + /// consumption of post-TLS cleartext data during STARTTLS transitions. + /// + /// Equivalent to OpenSSL's `SSL_set_read_ahead(ssl, 0)` — rustls has no + /// such knob, so we enforce record-level reads manually via peek. + fn try_read_close_notify_socket( + &self, + conn: &mut TlsConnection, + vm: &VirtualMachine, + ) -> PyResult<bool> { + // Peek at the first 5 bytes (TLS record header size) + let peeked_obj = match self.sock_peek(5, vm) { + Ok(obj) => obj, + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Ok(false); + } + return Ok(true); + } + }; + + let peeked = ArgBytesLike::try_from_object(vm, peeked_obj)?; + let peek_data = peeked.borrow_buf(); + + if peek_data.is_empty() { + return Ok(true); // EOF + } + + // TLS record content types: ChangeCipherSpec(20), Alert(21), + // Handshake(22), ApplicationData(23) + let content_type = peek_data[0]; + if !(20..=23).contains(&content_type) { + // Not a TLS record - post-TLS cleartext data. + // Peer has completed TLS shutdown; don't consume this data. + return Ok(true); + } + + // Determine how many bytes to read for exactly one TLS record + let recv_size = if peek_data.len() >= 5 { + let record_length = u16::from_be_bytes([peek_data[3], peek_data[4]]) as usize; + 5 + record_length + } else { + // Partial header available - read just these bytes for now + peek_data.len() + }; + + drop(peek_data); + drop(peeked); + + // Now consume exactly one TLS record from the socket + match self.sock_recv(recv_size, vm) { + Ok(bytes_obj) => { + let bytes = ArgBytesLike::try_from_object(vm, bytes_obj)?; + let data = bytes.borrow_buf(); + + if data.is_empty() { + return Ok(true); + } + + let data_slice: &[u8] = data.as_ref(); + let mut cursor = std::io::Cursor::new(data_slice); + let _ = conn.read_tls(&mut cursor); + let _ = conn.process_new_packets(); + Ok(false) + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Ok(false); + } + Ok(true) + } + } + } + + // Helper: Check if peer has sent close_notify + fn check_peer_closed( + &self, + conn: &mut TlsConnection, + vm: &VirtualMachine, + ) -> PyResult<bool> { + // Process any remaining packets and check peer_has_closed + let io_state = conn + .process_new_packets() + .map_err(|e| vm.new_os_error(format!("Failed to process packets: {e}")))?; + + Ok(io_state.peer_has_closed()) + } + + #[pymethod] + fn shared_ciphers(&self, vm: &VirtualMachine) -> Option<PyListRef> { + // Return None for client-side sockets + if !self.server_side { + return None; + } + + // Check if handshake completed + if !*self.handshake_done.lock() { + return None; + } + + // Get negotiated cipher suite from rustls + let conn_guard = self.connection.lock(); + let conn = conn_guard.as_ref()?; + + let suite = conn.negotiated_cipher_suite()?; + + // Extract cipher information using unified helper + let cipher_info = extract_cipher_info(&suite); + + // Return as list with single tuple (name, version, bits) + let tuple = vm.ctx.new_tuple(vec![ + vm.ctx.new_str(cipher_info.name).into(), + vm.ctx.new_str(cipher_info.protocol).into(), + vm.ctx.new_int(cipher_info.bits).into(), + ]); + Some(vm.ctx.new_list(vec![tuple.into()])) + } + + #[pymethod] + fn verify_client_post_handshake(&self, vm: &VirtualMachine) -> PyResult<()> { + // TLS 1.3 post-handshake authentication + // This is only valid for server-side TLS 1.3 connections + + // Check if this is a server-side socket + if !self.server_side { + return Err(vm.new_value_error( + "Cannot perform post-handshake authentication on client-side socket", + )); + } + + // Check if handshake has been completed + if !*self.handshake_done.lock() { + return Err(vm.new_value_error( + "Handshake must be completed before post-handshake authentication", + )); + } + + // Check connection exists and protocol version + let conn_guard = self.connection.lock(); + if let Some(conn) = conn_guard.as_ref() { + let version = match conn { + TlsConnection::Client(_) => { + return Err(vm.new_value_error( + "Post-handshake authentication requires server socket", + )); + } + TlsConnection::Server(server) => server.protocol_version(), + }; + + // Post-handshake auth is only available in TLS 1.3 + if version != Some(rustls::ProtocolVersion::TLSv1_3) { + // Get SSLError class from ssl module (not _ssl) + // ssl.py imports _ssl.SSLError as ssl.SSLError + let ssl_mod = vm.import("ssl", 0)?; + let ssl_error_class = ssl_mod.get_attr("SSLError", vm)?; + + // Create SSLError instance with message containing WRONG_SSL_VERSION + let msg = "[SSL: WRONG_SSL_VERSION] wrong ssl version"; + let args = vm.ctx.new_tuple(vec![vm.ctx.new_str(msg).into()]); + let exc = ssl_error_class.call((args,), vm)?; + + return Err(exc + .downcast() + .map_err(|_| vm.new_type_error("Failed to create SSLError"))?); + } + } else { + return Err(vm.new_value_error("No SSL connection established")); + } + + // rustls doesn't provide an API for post-handshake authentication. + // The rustls TLS library does not support requesting client certificates + // after the initial handshake is completed. + // Raise SSLError instead of NotImplementedError for compatibility + Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + "Post-handshake authentication is not supported by the rustls backend. \ + The rustls TLS library does not provide an API to request client certificates \ + after the initial handshake. Consider requesting the client certificate \ + during the initial handshake by setting the appropriate verify_mode before \ + calling do_handshake().", + ) + .upcast()) + } + + #[pymethod] + fn get_channel_binding( + &self, + cb_type: OptionalArg<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<PyBytesRef>> { + let cb_type_str = cb_type.as_ref().map_or("tls-unique", |s| s.as_str()); + + // rustls doesn't support channel binding (tls-unique, tls-server-end-point, etc.) + // This is because: + // 1. tls-unique requires access to TLS Finished messages, which rustls doesn't expose + // 2. tls-server-end-point requires the server certificate, which we don't track here + // 3. TLS 1.3 deprecated tls-unique anyway + // + // For compatibility, we'll return None (no channel binding available) + // rather than raising an error + + if cb_type_str != "tls-unique" { + return Err(vm.new_value_error(format!( + "Unsupported channel binding type '{cb_type_str}'", + ))); + } + + // Return None to indicate channel binding is not available + // This matches the behavior when the handshake hasn't completed yet + Ok(None) + } + } + + impl Representable for PySSLSocket { + #[inline] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("<SSLSocket>".to_owned()) + } + } + + impl Constructor for PySSLSocket { + type Args = (); + + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error( + "Cannot directly instantiate SSLSocket, use SSLContext.wrap_socket()", + )) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + // Clean up SSL socket resources on drop + impl Drop for PySSLSocket { + fn drop(&mut self) { + // Only clear connection state. + // Do NOT clear pending_tls_output - it may contain data that hasn't + // been flushed to the socket yet. SSLSocket._real_close() in Python + // doesn't call shutdown(), so when the socket is closed, pending TLS + // data would be lost if we clear it here. + // All fields (Vec, primitives) are automatically freed when the + // struct is dropped, so explicit clearing is unnecessary. + let _ = self.connection.lock().take(); + } + } + + // MemoryBIO - provides in-memory buffer for SSL/TLS I/O + #[pyattr] + #[pyclass(name = "MemoryBIO", module = "ssl")] + #[derive(Debug, PyPayload)] + struct PyMemoryBIO { + // Internal buffer + buffer: PyMutex<Vec<u8>>, + // EOF flag + eof: PyRwLock<bool>, + } + + #[pyclass(with(Constructor), flags(BASETYPE))] + impl PyMemoryBIO { + #[pymethod] + fn read(&self, len: OptionalArg<i32>, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + let mut buffer = self.buffer.lock(); + + if buffer.is_empty() && *self.eof.read() { + // Return empty bytes at EOF + return Ok(vm.ctx.new_bytes(vec![])); + } + + let read_len = match len { + OptionalArg::Present(n) if n >= 0 => n as usize, + OptionalArg::Present(n) => { + return Err(vm.new_value_error(format!("negative read length: {n}"))); + } + OptionalArg::Missing => buffer.len(), // Read all available + }; + + let actual_len = read_len.min(buffer.len()); + let data = buffer.drain(..actual_len).collect::<Vec<u8>>(); + + Ok(vm.ctx.new_bytes(data)) + } + + #[pymethod] + fn write(&self, buf: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + // Check if it's a memoryview and if it's contiguous + if let Ok(mem_view) = buf.get_attr("c_contiguous", vm) { + // It's a memoryview, check if contiguous + let is_contiguous: bool = mem_view.try_to_bool(vm)?; + if !is_contiguous { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.buffer_error.to_owned(), + "non-contiguous buffer is not supported".into(), + )); + } + } + + // Convert to bytes-like object + let bytes_like = ArgBytesLike::try_from_object(vm, buf)?; + let data = bytes_like.borrow_buf(); + let len = data.len(); + + let mut buffer = self.buffer.lock(); + buffer.extend_from_slice(&data); + + Ok(len) + } + + #[pymethod] + fn write_eof(&self, _vm: &VirtualMachine) -> PyResult<()> { + *self.eof.write() = true; + Ok(()) + } + + #[pygetset] + fn pending(&self) -> i32 { + self.buffer.lock().len() as i32 + } + + #[pygetset] + fn eof(&self) -> bool { + // EOF is true only when buffer is empty AND write_eof has been called + let pending = self.buffer.lock().len(); + pending == 0 && *self.eof.read() + } + } + + impl Representable for PyMemoryBIO { + #[inline] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("<MemoryBIO>".to_owned()) + } + } + + impl Constructor for PyMemoryBIO { + type Args = (); + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(PyMemoryBIO { + buffer: PyMutex::new(Vec::new()), + eof: PyRwLock::new(false), + }) + } + } + + // SSLSession - represents a cached SSL session + // NOTE: This is an EMULATION - actual session data is managed by Rustls internally + #[pyattr] + #[pyclass(name = "SSLSession", module = "ssl")] + #[derive(Debug, PyPayload)] + struct PySSLSession { + // Session data - serialized rustls session (EMULATED - kept empty) + session_data: Vec<u8>, + // Session ID - synthetic ID generated from metadata (NOT actual TLS session ID) + #[allow(dead_code)] + session_id: Vec<u8>, + // Session metadata + creation_time: std::time::SystemTime, + // Lifetime in seconds (default 7200 = 2 hours) + lifetime: u64, + } + + #[pyclass(flags(BASETYPE))] + impl PySSLSession { + #[pygetset] + fn time(&self) -> i64 { + // Return session creation time as Unix timestamp + self.creation_time + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 + } + + #[pygetset] + fn timeout(&self) -> i64 { + // Return session timeout/lifetime in seconds + self.lifetime as i64 + } + + #[pygetset] + fn ticket_lifetime_hint(&self) -> i64 { + // Return ticket lifetime hint (same as timeout for rustls) + self.lifetime as i64 + } + + #[pygetset] + fn id(&self, vm: &VirtualMachine) -> PyBytesRef { + // Return session ID (hash of session data for uniqueness) + + let mut hasher = DefaultHasher::new(); + self.session_data.hash(&mut hasher); + let hash = hasher.finish(); + + // Convert hash to bytes + vm.ctx.new_bytes(hash.to_be_bytes().to_vec()) + } + + #[pygetset] + fn has_ticket(&self) -> bool { + // For rustls, if we have session data, we have a ticket + !self.session_data.is_empty() + } + } + + impl Representable for PySSLSession { + #[inline] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("<SSLSession>".to_owned()) + } + } + + // Helper functions + + // OID module already imported at top of _ssl module + + #[derive(FromArgs)] + struct Txt2ObjArgs { + txt: PyUtf8StrRef, + #[pyarg(named, optional)] + name: OptionalArg<bool>, + } + + #[pyfunction] + fn txt2obj(args: Txt2ObjArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let txt = args.txt.as_str(); + let name = args.name.unwrap_or(false); + + // If name=False (default), only accept OID strings + // If name=True, accept both names and OID strings + let entry = if txt + .chars() + .next() + .map(|c| c.is_ascii_digit()) + .unwrap_or(false) + { + // Looks like an OID string (starts with digit) + oid::find_by_oid_string(txt) + } else if name { + // name=True: allow shortname/longname lookup + oid::find_by_name(txt) + } else { + // name=False: only OID strings allowed, not names + None + }; + + let entry = entry.ok_or_else(|| vm.new_value_error(format!("unknown object '{txt}'")))?; + + // Return tuple: (nid, shortname, longname, oid) + Ok(vm + .new_tuple(( + vm.ctx.new_int(entry.nid), + vm.ctx.new_str(entry.short_name), + vm.ctx.new_str(entry.long_name), + vm.ctx.new_str(entry.oid_string()), + )) + .into()) + } + + #[pyfunction] + fn nid2obj(nid: i32, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let entry = oid::find_by_nid(nid) + .ok_or_else(|| vm.new_value_error(format!("unknown NID {nid}")))?; + + // Return tuple: (nid, shortname, longname, oid) + Ok(vm + .new_tuple(( + vm.ctx.new_int(entry.nid), + vm.ctx.new_str(entry.short_name), + vm.ctx.new_str(entry.long_name), + vm.ctx.new_str(entry.oid_string()), + )) + .into()) + } + + #[pyfunction] + fn get_default_verify_paths(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Return default certificate paths as a tuple + // Lib/ssl.py expects: (openssl_cafile_env, openssl_cafile, openssl_capath_env, openssl_capath) + // parts[0] = environment variable name for cafile + // parts[1] = default cafile path + // parts[2] = environment variable name for capath + // parts[3] = default capath path + + // Common default paths for different platforms + // These match the first candidates that rustls-native-certs/openssl-probe checks + #[cfg(target_os = "macos")] + let (default_cafile, default_capath) = { + // macOS primarily uses Keychain API, but provides fallback paths + // for compatibility and when Keychain access fails + (Some("/etc/ssl/cert.pem"), Some("/etc/ssl/certs")) + }; + + #[cfg(target_os = "linux")] + let (default_cafile, default_capath) = { + // Linux: matches openssl-probe's first candidate (/etc/ssl/cert.pem) + // openssl-probe checks multiple locations at runtime, but we return + // OpenSSL's compile-time default + (Some("/etc/ssl/cert.pem"), Some("/etc/ssl/certs")) + }; + + #[cfg(windows)] + let (default_cafile, default_capath) = { + // Windows uses certificate store, not file paths + // Return empty strings to avoid None being passed to os.path.isfile() + (Some(""), Some("")) + }; + + #[cfg(not(any(target_os = "macos", target_os = "linux", windows)))] + let (default_cafile, default_capath): (Option<&str>, Option<&str>) = (None, None); + + let tuple = vm.ctx.new_tuple(vec![ + vm.ctx.new_str("SSL_CERT_FILE").into(), // openssl_cafile_env + default_cafile + .map(|s| vm.ctx.new_str(s).into()) + .unwrap_or_else(|| vm.ctx.none()), // openssl_cafile + vm.ctx.new_str("SSL_CERT_DIR").into(), // openssl_capath_env + default_capath + .map(|s| vm.ctx.new_str(s).into()) + .unwrap_or_else(|| vm.ctx.none()), // openssl_capath + ]); + Ok(tuple.into()) + } + + #[pyfunction] + fn RAND_status() -> i32 { + 1 // Always have good randomness with aws-lc-rs + } + + #[pyfunction] + fn RAND_add(_string: PyObjectRef, _entropy: f64) { + // No-op: aws-lc-rs handles its own entropy + // Accept any type (str, bytes, bytearray) + } + + #[pyfunction] + fn RAND_bytes(n: i64, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + use aws_lc_rs::rand::{SecureRandom, SystemRandom}; + + // Validate n is not negative + if n < 0 { + return Err(vm.new_value_error("num must be positive")); + } + + let n_usize = n as usize; + let rng = SystemRandom::new(); + let mut buf = vec![0u8; n_usize]; + rng.fill(&mut buf) + .map_err(|_| vm.new_os_error("Failed to generate random bytes"))?; + Ok(PyBytesRef::from(vm.ctx.new_bytes(buf))) + } + + #[pyfunction] + fn RAND_pseudo_bytes(n: i64, vm: &VirtualMachine) -> PyResult<(PyBytesRef, bool)> { + // In rustls/aws-lc-rs, all random bytes are cryptographically strong + let bytes = RAND_bytes(n, vm)?; + Ok((bytes, true)) + } + + /// Test helper to decode a certificate from a file path + /// + /// This is a simplified wrapper around cert_der_to_dict_helper that handles + /// file reading and PEM/DER auto-detection. Used by test suite. + #[pyfunction] + fn _test_decode_cert(path: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Read certificate file + let path_str = path.as_str(); + let cert_data = std::fs::read(path_str).map_err(|e| { + vm.new_os_error(format!( + "Failed to read certificate file {}: {}", + path_str, e + )) + })?; + + // Auto-detect PEM vs DER format + let cert_der = if cert_data + .windows(27) + .any(|w| w == b"-----BEGIN CERTIFICATE-----") + { + // Parse PEM format + let mut cursor = std::io::Cursor::new(&cert_data); + rustls_pemfile::certs(&mut cursor) + .find_map(|r| r.ok()) + .ok_or_else(|| vm.new_value_error("No valid certificate found in PEM file"))? + .to_vec() + } else { + // Assume DER format + cert_data + }; + + // Reuse the comprehensive helper function + cert::cert_der_to_dict_helper(vm, &cert_der) + } + + #[pyfunction] + fn DER_cert_to_PEM_cert(der_cert: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let der_bytes = der_cert.borrow_buf(); + let bytes_slice: &[u8] = der_bytes.as_ref(); + + // Use pem-rfc7468 for RFC 7468 compliant PEM encoding + let pem_str = encode_string("CERTIFICATE", LineEnding::LF, bytes_slice) + .map_err(|e| vm.new_value_error(format!("PEM encoding failed: {e}")))?; + + Ok(vm.ctx.new_str(pem_str)) + } + + #[pyfunction] + fn PEM_cert_to_DER_cert(pem_cert: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + // Parse PEM format + let mut cursor = std::io::Cursor::new(pem_cert.as_bytes()); + let mut certs = rustls_pemfile::certs(&mut cursor); + + if let Some(Ok(cert)) = certs.next() { + Ok(vm.ctx.new_bytes(cert.to_vec())) + } else { + Err(vm.new_value_error("Failed to parse PEM certificate")) + } + } + + // Windows-specific certificate store enumeration functions + #[cfg(windows)] + #[pyfunction] + fn enum_certificates( + store_name: PyUtf8StrRef, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + use schannel::{RawPointer, cert_context::ValidUses, cert_store::CertStore}; + use windows_sys::Win32::Security::Cryptography; + + let store_name_str = store_name.as_str(); + + // Try both Current User and Local Machine stores + let open_fns = [CertStore::open_current_user, CertStore::open_local_machine]; + let stores = open_fns + .iter() + .filter_map(|open| open(store_name_str).ok()) + .collect::<Vec<_>>(); + + // If no stores could be opened, raise OSError + if stores.is_empty() { + return Err(vm.new_os_error(format!( + "failed to open certificate store {:?}", + store_name_str + ))); + } + + let certs = stores.iter().flat_map(|s| s.certs()).map(|c| { + let cert = vm.ctx.new_bytes(c.to_der().to_owned()); + let enc_type = unsafe { + let ptr = c.as_ptr() as *const Cryptography::CERT_CONTEXT; + (*ptr).dwCertEncodingType + }; + let enc_type = match enc_type { + Cryptography::X509_ASN_ENCODING => vm.new_pyobj("x509_asn"), + Cryptography::PKCS_7_ASN_ENCODING => vm.new_pyobj("pkcs_7_asn"), + other => vm.new_pyobj(other), + }; + let usage: PyObjectRef = match c.valid_uses() { + Ok(ValidUses::All) => vm.ctx.new_bool(true).into(), + Ok(ValidUses::Oids(oids)) => { + match crate::builtins::PyFrozenSet::from_iter( + vm, + oids.into_iter().map(|oid| vm.ctx.new_str(oid).into()), + ) { + Ok(set) => set.into_ref(&vm.ctx).into(), + Err(_) => vm.ctx.new_bool(true).into(), + } + } + Err(_) => vm.ctx.new_bool(true).into(), + }; + Ok(vm.new_tuple((cert, enc_type, usage)).into()) + }); + certs.collect::<PyResult<Vec<_>>>() + } + + #[cfg(windows)] + #[pyfunction] + fn enum_crls(store_name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + use windows_sys::Win32::Security::Cryptography::{ + CRL_CONTEXT, CertCloseStore, CertEnumCRLsInStore, CertOpenSystemStoreW, + X509_ASN_ENCODING, + }; + + let store_name_str = store_name.as_str(); + let store_name_wide: Vec<u16> = store_name_str + .encode_utf16() + .chain(core::iter::once(0)) + .collect(); + + // Open system store + let store = unsafe { CertOpenSystemStoreW(0, store_name_wide.as_ptr()) }; + + if store.is_null() { + return Err(vm.new_os_error(format!( + "failed to open certificate store {:?}", + store_name_str + ))); + } + + let mut result = Vec::new(); + + let mut crl_context: *const CRL_CONTEXT = core::ptr::null(); + loop { + crl_context = unsafe { CertEnumCRLsInStore(store, crl_context) }; + if crl_context.is_null() { + break; + } + + let crl = unsafe { &*crl_context }; + let crl_bytes = + unsafe { core::slice::from_raw_parts(crl.pbCrlEncoded, crl.cbCrlEncoded as usize) }; + + let enc_type = if crl.dwCertEncodingType == X509_ASN_ENCODING { + vm.new_pyobj("x509_asn") + } else { + vm.new_pyobj(crl.dwCertEncodingType) + }; + + result.push( + vm.new_tuple((vm.ctx.new_bytes(crl_bytes.to_vec()), enc_type)) + .into(), + ); + } + + unsafe { CertCloseStore(store, 0) }; + + Ok(result) + } + + // Certificate type for SSL module (pure Rust implementation) + #[pyattr] + #[pyclass(module = "_ssl", name = "Certificate")] + #[derive(Debug, PyPayload)] + pub struct PySSLCertificate { + // Store the raw DER bytes + der_bytes: Vec<u8>, + } + + impl PySSLCertificate { + // Parse the certificate lazily + fn parse(&self) -> Result<x509_parser::certificate::X509Certificate<'_>, String> { + match x509_parser::parse_x509_certificate(&self.der_bytes) { + Ok((_, cert)) => Ok(cert), + Err(e) => Err(format!("Failed to parse certificate: {e}")), + } + } + } + + #[pyclass(with(Comparable, Hashable, Representable))] + impl PySSLCertificate { + #[pymethod] + fn public_bytes( + &self, + format: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let format = format.unwrap_or(ENCODING_PEM); + + match format { + x if x == ENCODING_DER => { + // Return DER bytes directly + Ok(vm.ctx.new_bytes(self.der_bytes.clone()).into()) + } + x if x == ENCODING_PEM => { + // Convert DER to PEM using RFC 7468 compliant encoding + let pem_str = encode_string("CERTIFICATE", LineEnding::LF, &self.der_bytes) + .map_err(|e| vm.new_value_error(format!("PEM encoding failed: {e}")))?; + Ok(vm.ctx.new_str(pem_str).into()) + } + _ => Err(vm.new_value_error("Unsupported format")), + } + } + + #[pymethod] + fn get_info(&self, vm: &VirtualMachine) -> PyResult { + let cert = self.parse().map_err(|e| vm.new_value_error(e))?; + cert::cert_to_dict(vm, &cert) + } + } + + // Implement Comparable trait for PySSLCertificate + impl Comparable for PySSLCertificate { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + if let Some(other_cert) = other.downcast_ref::<Self>() { + Ok((zelf.der_bytes == other_cert.der_bytes).into()) + } else { + Ok(PyComparisonValue::NotImplemented) + } + }) + } + } + + // Implement Hashable trait for PySSLCertificate + impl Hashable for PySSLCertificate { + fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyHash> { + let mut hasher = DefaultHasher::new(); + zelf.der_bytes.hash(&mut hasher); + Ok(hasher.finish() as PyHash) + } + } + + // Implement Representable trait for PySSLCertificate + impl Representable for PySSLCertificate { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + // Try to parse and show subject + match zelf.parse() { + Ok(cert) => { + let subject = cert.subject(); + // Get CN if available + let cn = subject + .iter_common_name() + .next() + .and_then(|attr| attr.as_str().ok()) + .unwrap_or("Unknown"); + Ok(format!("<Certificate(subject=CN={cn})>")) + } + Err(_) => Ok("<Certificate(invalid)>".to_owned()), + } + } + } +} diff --git a/crates/stdlib/src/ssl/cert.rs b/crates/stdlib/src/ssl/cert.rs new file mode 100644 index 00000000000..cd39972cf41 --- /dev/null +++ b/crates/stdlib/src/ssl/cert.rs @@ -0,0 +1,1776 @@ +// cspell: ignore accessdescs + +//! Certificate parsing, validation, and conversion utilities for SSL/TLS +//! +//! This module provides reusable functions for working with X.509 certificates: +//! - Parsing PEM/DER encoded certificates +//! - Validating certificate properties (CA status, etc.) +//! - Converting certificates to Python dict format +//! - Building and verifying certificate chains +//! - Loading certificates from files, directories, and bytes + +use alloc::sync::Arc; +use chrono::{DateTime, Utc}; +use parking_lot::RwLock as ParkingRwLock; +use rustls::{ + DigitallySignedStruct, RootCertStore, SignatureScheme, + client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime}, + server::danger::{ClientCertVerified, ClientCertVerifier}, +}; +use rustpython_vm::{PyObjectRef, PyResult, VirtualMachine}; +use std::collections::HashSet; +use x509_parser::prelude::*; + +use super::compat::{VERIFY_X509_PARTIAL_CHAIN, VERIFY_X509_STRICT}; + +// Certificate Verification Constants + +/// All supported signature schemes for certificate verification +/// +/// This list includes all modern signature algorithms supported by rustls. +/// Used by verifiers that accept any signature scheme (NoVerifier, EmptyRootStoreVerifier). +const ALL_SIGNATURE_SCHEMES: &[SignatureScheme] = &[ + SignatureScheme::RSA_PKCS1_SHA256, + SignatureScheme::RSA_PKCS1_SHA384, + SignatureScheme::RSA_PKCS1_SHA512, + SignatureScheme::ECDSA_NISTP256_SHA256, + SignatureScheme::ECDSA_NISTP384_SHA384, + SignatureScheme::ECDSA_NISTP521_SHA512, + SignatureScheme::RSA_PSS_SHA256, + SignatureScheme::RSA_PSS_SHA384, + SignatureScheme::RSA_PSS_SHA512, + SignatureScheme::ED25519, +]; + +// Error Handling Utilities + +/// Certificate loading error types with specific error messages +/// +/// This module provides consistent error creation functions for certificate +/// operations, reducing code duplication and ensuring uniform error messages +/// across the codebase. +mod cert_error { + use alloc::sync::Arc; + use core::fmt::{Debug, Display}; + use std::io; + + /// Create InvalidData error with formatted message + pub fn invalid_data(msg: impl Into<String>) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, msg.into()) + } + + /// PEM parsing error variants + pub mod pem { + use super::*; + + pub fn no_start_line(context: &str) -> io::Error { + invalid_data(format!("no start line: {context}")) + } + + pub fn parse_failed(e: impl Display) -> io::Error { + invalid_data(format!("Failed to parse PEM certificate: {e}")) + } + + pub fn parse_failed_debug(e: impl Debug) -> io::Error { + invalid_data(format!("Failed to parse PEM certificate: {e:?}")) + } + + pub fn invalid_cert() -> io::Error { + invalid_data("No certificates found in certificate file") + } + } + + /// DER parsing error variants + pub mod der { + use super::*; + + pub fn not_enough_data(context: &str) -> io::Error { + invalid_data(format!("not enough data: {context}")) + } + + pub fn parse_failed(e: impl Display) -> io::Error { + invalid_data(format!("Failed to parse DER certificate: {e}")) + } + } + + /// Private key error variants + pub mod key { + use super::*; + + pub fn not_found(context: &str) -> io::Error { + invalid_data(format!("No private key found in {context}")) + } + + pub fn parse_failed(e: impl Display) -> io::Error { + invalid_data(format!("Failed to parse private key: {e}")) + } + + pub fn parse_encrypted_failed(e: impl Display) -> io::Error { + invalid_data(format!("Failed to parse encrypted private key: {e}")) + } + + pub fn decrypt_failed(e: impl Display) -> io::Error { + io::Error::other(format!( + "Failed to decrypt private key (wrong password?): {e}", + )) + } + } + + /// Convert error message to rustls::Error with InvalidCertificate wrapper + pub fn to_rustls_invalid_cert(msg: impl Into<String>) -> rustls::Error { + rustls::Error::InvalidCertificate(rustls::CertificateError::Other(rustls::OtherError( + Arc::new(invalid_data(msg)), + ))) + } + + /// Convert error message to rustls::Error with InvalidCertificate wrapper and custom ErrorKind + pub fn to_rustls_cert_error(kind: io::ErrorKind, msg: impl Into<String>) -> rustls::Error { + rustls::Error::InvalidCertificate(rustls::CertificateError::Other(rustls::OtherError( + Arc::new(io::Error::new(kind, msg.into())), + ))) + } +} + +// Helper Functions for Certificate Parsing + +/// Map X.509 OID to human-readable attribute name +/// +/// Converts common X.509 Distinguished Name OIDs to their standard names. +/// Returns the OID string itself if not recognized. +fn oid_to_attribute_name(oid_str: &str) -> &str { + match oid_str { + "2.5.4.3" => "commonName", + "2.5.4.6" => "countryName", + "2.5.4.7" => "localityName", + "2.5.4.8" => "stateOrProvinceName", + "2.5.4.10" => "organizationName", + "2.5.4.11" => "organizationalUnitName", + "1.2.840.113549.1.9.1" => "emailAddress", + _ => oid_str, + } +} + +/// Format IP address (IPv4 or IPv6) to string +/// +/// Formats raw IP address bytes according to standard notation: +/// - IPv4: dotted decimal (e.g., "192.0.2.1") +/// - IPv6: colon-separated hex (e.g., "2001:DB8:0:0:0:0:0:1") +fn format_ip_address(ip: &[u8]) -> String { + if ip.len() == 4 { + // IPv4 + format!("{}.{}.{}.{}", ip[0], ip[1], ip[2], ip[3]) + } else if ip.len() == 16 { + // IPv6 - format in full form without compression (uppercase) + // CPython returns IPv6 in full form: 2001:DB8:0:0:0:0:0:1 (not 2001:db8::1) + let segments = [ + u16::from_be_bytes([ip[0], ip[1]]), + u16::from_be_bytes([ip[2], ip[3]]), + u16::from_be_bytes([ip[4], ip[5]]), + u16::from_be_bytes([ip[6], ip[7]]), + u16::from_be_bytes([ip[8], ip[9]]), + u16::from_be_bytes([ip[10], ip[11]]), + u16::from_be_bytes([ip[12], ip[13]]), + u16::from_be_bytes([ip[14], ip[15]]), + ]; + format!( + "{:X}:{:X}:{:X}:{:X}:{:X}:{:X}:{:X}:{:X}", + segments[0], + segments[1], + segments[2], + segments[3], + segments[4], + segments[5], + segments[6], + segments[7] + ) + } else { + // Unknown format - return as debug string + format!("{ip:?}") + } +} + +/// Format ASN.1 time to string +/// +/// Formats certificate validity dates in the format: +/// "Mon DD HH:MM:SS YYYY GMT" +fn format_asn1_time(time: &x509_parser::time::ASN1Time) -> String { + let timestamp = time.timestamp(); + DateTime::<Utc>::from_timestamp(timestamp, 0) + .expect("ASN1Time must be valid timestamp") + .format("%b %e %H:%M:%S %Y GMT") + .to_string() +} + +/// Format certificate serial number to hexadecimal string with even padding +/// +/// Converts a BigUint serial number to uppercase hex string, ensuring +/// even length by prepending '0' if necessary. +fn format_serial_number(serial: &num_bigint::BigUint) -> String { + let mut serial_str = serial.to_str_radix(16).to_uppercase(); + if serial_str.len() % 2 == 1 { + serial_str.insert(0, '0'); + } + serial_str +} + +/// Normalize wildcard hostname by stripping "*." prefix +/// +/// Returns the normalized hostname without the wildcard prefix. +/// Used for wildcard certificate matching. +fn normalize_wildcard_hostname(hostname: &str) -> &str { + hostname.strip_prefix("*.").unwrap_or(hostname) +} + +/// Process Subject Alternative Name (SAN) general names into Python tuples +/// +/// Converts X.509 GeneralName entries into Python tuple format. +/// Returns a vector of PyObjectRef tuples in the format: (type, value) +fn process_san_general_names( + vm: &VirtualMachine, + general_names: &[GeneralName<'_>], +) -> Vec<PyObjectRef> { + general_names + .iter() + .filter_map(|name| match name { + GeneralName::DNSName(dns) => Some(vm.new_tuple(("DNS", *dns)).into()), + GeneralName::IPAddress(ip) => { + let ip_str = format_ip_address(ip); + Some(vm.new_tuple(("IP Address", ip_str)).into()) + } + GeneralName::RFC822Name(email) => Some(vm.new_tuple(("email", *email)).into()), + GeneralName::URI(uri) => Some(vm.new_tuple(("URI", *uri)).into()), + GeneralName::DirectoryName(dn) => { + let dn_str = format!("{dn}"); + Some(vm.new_tuple(("DirName", dn_str)).into()) + } + GeneralName::RegisteredID(oid) => { + let oid_str = oid.to_string(); + Some(vm.new_tuple(("Registered ID", oid_str)).into()) + } + GeneralName::OtherName(oid, value) => { + let oid_str = oid.to_string(); + let value_str = format!("{value:?}"); + Some( + vm.new_tuple(("othername", format!("{oid_str}:{value_str}"))) + .into(), + ) + } + _ => None, + }) + .collect() +} + +// Certificate Validation and Parsing + +/// Check if a certificate is a CA certificate by examining the Basic Constraints extension +/// +/// Returns `true` if the certificate has Basic Constraints with CA=true, +/// `false` otherwise (including parse errors or missing extension). +/// This matches OpenSSL's X509_check_ca() behavior. +pub fn is_ca_certificate(cert_der: &[u8]) -> bool { + // Parse the certificate + let Ok((_, cert)) = X509Certificate::from_der(cert_der) else { + return false; + }; + + // Check Basic Constraints extension + // If extension exists and CA=true, it's a CA certificate + // Otherwise (no extension or CA=false), it's NOT a CA certificate + if let Ok(Some(ext)) = cert.basic_constraints() { + return ext.value.ca; + } + + // No Basic Constraints extension -> NOT a CA certificate + // (matches OpenSSL X509_check_ca() behavior) + false +} + +/// Convert an X509Name to Python nested tuple format for SSL certificate dicts +/// +/// Format: ((('CN', 'example.com'),), (('O', 'Example Org'),), ...) +fn name_to_py(vm: &VirtualMachine, name: &x509_parser::x509::X509Name<'_>) -> PyResult { + let list: Vec<PyObjectRef> = name + .iter() + .flat_map(|rdn| { + // Each RDN can have multiple attributes + rdn.iter() + .map(|attr| { + let oid_str = attr.attr_type().to_id_string(); + let value_str = attr.attr_value().as_str().unwrap_or("").to_string(); + let key = oid_to_attribute_name(&oid_str); + + vm.new_tuple((vm.new_tuple((vm.ctx.new_str(key), vm.ctx.new_str(value_str))),)) + .into() + }) + .collect::<Vec<_>>() + }) + .collect(); + + Ok(vm.ctx.new_tuple(list).into()) +} + +/// Convert DER-encoded certificate to Python dict (for getpeercert with binary_form=False) +/// +/// Returns a dict with fields: subject, issuer, version, serialNumber, +/// notBefore, notAfter, subjectAltName (if present) +pub fn cert_to_dict( + vm: &VirtualMachine, + cert: &x509_parser::certificate::X509Certificate<'_>, +) -> PyResult { + let dict = vm.ctx.new_dict(); + + // Subject and Issuer + dict.set_item("subject", name_to_py(vm, cert.subject())?, vm)?; + dict.set_item("issuer", name_to_py(vm, cert.issuer())?, vm)?; + + // Version (X.509 v3 = version 2 in the cert, but Python uses 3) + dict.set_item( + "version", + vm.ctx.new_int(cert.version().0 as i32 + 1).into(), + vm, + )?; + + // Serial number - hex format with even length + let serial = format_serial_number(&cert.serial); + dict.set_item("serialNumber", vm.ctx.new_str(serial).into(), vm)?; + + // Validity dates - format with GMT using chrono + dict.set_item( + "notBefore", + vm.ctx + .new_str(format_asn1_time(&cert.validity().not_before)) + .into(), + vm, + )?; + dict.set_item( + "notAfter", + vm.ctx + .new_str(format_asn1_time(&cert.validity().not_after)) + .into(), + vm, + )?; + + // Subject Alternative Names (if present) + if let Ok(Some(san_ext)) = cert.subject_alternative_name() { + let san_list = process_san_general_names(vm, &san_ext.value.general_names); + + if !san_list.is_empty() { + dict.set_item("subjectAltName", vm.ctx.new_tuple(san_list).into(), vm)?; + } + } + + Ok(dict.into()) +} + +/// Convert DER-encoded certificate to Python dict (for get_ca_certs) +/// +/// Similar to cert_to_dict but includes additional fields like crlDistributionPoints +/// and uses CPython's specific ordering: issuer, notAfter, notBefore, serialNumber, subject, version +pub fn cert_der_to_dict_helper(vm: &VirtualMachine, cert_der: &[u8]) -> PyResult<PyObjectRef> { + // Parse the certificate using x509-parser + let (_, cert) = x509_parser::parse_x509_certificate(cert_der) + .map_err(|e| vm.new_value_error(format!("Failed to parse certificate: {e}")))?; + + // Helper to convert X509Name to nested tuple format + let name_to_tuple = |name: &x509_parser::x509::X509Name<'_>| -> PyResult { + let mut entries = Vec::new(); + for rdn in name.iter() { + for attr in rdn.iter() { + let oid_str = attr.attr_type().to_id_string(); + + // Get value as bytes and convert to string + let value_str = if let Ok(s) = attr.attr_value().as_str() { + s.to_string() + } else { + let value_bytes = attr.attr_value().data; + match core::str::from_utf8(value_bytes) { + Ok(s) => s.to_string(), + Err(_) => String::from_utf8_lossy(value_bytes).into_owned(), + } + }; + + let key = oid_to_attribute_name(&oid_str); + + let entry = + vm.new_tuple((vm.ctx.new_str(key.to_string()), vm.ctx.new_str(value_str))); + entries.push(vm.new_tuple((entry,)).into()); + } + } + Ok(vm.ctx.new_tuple(entries).into()) + }; + + let dict = vm.ctx.new_dict(); + + // CPython ordering: issuer, notAfter, notBefore, serialNumber, subject, version + dict.set_item("issuer", name_to_tuple(cert.issuer())?, vm)?; + + // Validity - format with GMT using chrono + dict.set_item( + "notAfter", + vm.ctx + .new_str(format_asn1_time(&cert.validity().not_after)) + .into(), + vm, + )?; + dict.set_item( + "notBefore", + vm.ctx + .new_str(format_asn1_time(&cert.validity().not_before)) + .into(), + vm, + )?; + + // Serial number - hex format with even length + let serial = format_serial_number(&cert.serial); + dict.set_item("serialNumber", vm.ctx.new_str(serial).into(), vm)?; + + dict.set_item("subject", name_to_tuple(cert.subject())?, vm)?; + + // Version + dict.set_item( + "version", + vm.ctx.new_int(cert.version().0 as i32 + 1).into(), + vm, + )?; + + // Authority Information Access (OCSP and caIssuers) - use x509-parser's extensions_map + let mut ocsp_urls = Vec::new(); + let mut ca_issuer_urls = Vec::new(); + let mut crl_urls = Vec::new(); + + if let Ok(ext_map) = cert.tbs_certificate.extensions_map() { + use x509_parser::extensions::{GeneralName, ParsedExtension}; + use x509_parser::oid_registry::{ + OID_PKIX_AUTHORITY_INFO_ACCESS, OID_X509_EXT_CRL_DISTRIBUTION_POINTS, + }; + + // Authority Information Access + if let Some(ext) = ext_map.get(&OID_PKIX_AUTHORITY_INFO_ACCESS) + && let ParsedExtension::AuthorityInfoAccess(aia) = &ext.parsed_extension() + { + for desc in &aia.accessdescs { + if let GeneralName::URI(uri) = &desc.access_location { + let method_str = desc.access_method.to_id_string(); + if method_str == "1.3.6.1.5.5.7.48.1" { + // OCSP + ocsp_urls.push(vm.ctx.new_str(uri.to_string()).into()); + } else if method_str == "1.3.6.1.5.5.7.48.2" { + // caIssuers + ca_issuer_urls.push(vm.ctx.new_str(uri.to_string()).into()); + } + } + } + } + + // CRL Distribution Points + if let Some(ext) = ext_map.get(&OID_X509_EXT_CRL_DISTRIBUTION_POINTS) + && let ParsedExtension::CRLDistributionPoints(cdp) = &ext.parsed_extension() + { + for dp in cdp.points.iter() { + if let Some(dist_point) = &dp.distribution_point { + use x509_parser::extensions::DistributionPointName; + if let DistributionPointName::FullName(names) = dist_point { + for name in names { + if let GeneralName::URI(uri) = name { + crl_urls.push(vm.ctx.new_str(uri.to_string()).into()); + } + } + } + } + } + } + } + + if !ocsp_urls.is_empty() { + dict.set_item("OCSP", vm.ctx.new_tuple(ocsp_urls).into(), vm)?; + } + if !ca_issuer_urls.is_empty() { + dict.set_item("caIssuers", vm.ctx.new_tuple(ca_issuer_urls).into(), vm)?; + } + if !crl_urls.is_empty() { + dict.set_item( + "crlDistributionPoints", + vm.ctx.new_tuple(crl_urls).into(), + vm, + )?; + } + + // Subject Alternative Names + if let Ok(Some(san_ext)) = cert.subject_alternative_name() { + let mut san_entries = Vec::new(); + for name in &san_ext.value.general_names { + use x509_parser::extensions::GeneralName; + match name { + GeneralName::DNSName(dns) => { + san_entries.push(vm.new_tuple(("DNS", *dns)).into()); + } + GeneralName::IPAddress(ip) => { + let ip_str = format_ip_address(ip); + san_entries.push(vm.new_tuple(("IP Address", ip_str)).into()); + } + GeneralName::RFC822Name(email) => { + san_entries.push(vm.new_tuple(("email", *email)).into()); + } + GeneralName::URI(uri) => { + san_entries.push(vm.new_tuple(("URI", *uri)).into()); + } + GeneralName::OtherName(_oid, _data) => { + // OtherName is not fully supported, mark as unsupported + san_entries.push(vm.new_tuple(("othername", "<unsupported>")).into()); + } + GeneralName::DirectoryName(name) => { + // Convert X509Name to nested tuple format + let dir_tuple = name_to_tuple(name)?; + san_entries.push(vm.new_tuple(("DirName", dir_tuple)).into()); + } + GeneralName::RegisteredID(oid) => { + // Convert OID to string representation + let oid_str = oid.to_id_string(); + san_entries.push(vm.new_tuple(("Registered ID", oid_str)).into()); + } + _ => {} + } + } + if !san_entries.is_empty() { + dict.set_item("subjectAltName", vm.ctx.new_tuple(san_entries).into(), vm)?; + } + } + + Ok(dict.into()) +} + +/// Build a verified certificate chain by adding CA certificates from the trust store +/// +/// Takes peer certificates (from TLS handshake) and extends the chain by finding +/// issuer certificates from the trust store until reaching a root certificate. +/// +/// Returns the complete chain as DER-encoded bytes. +pub fn build_verified_chain( + peer_certs: &[CertificateDer<'static>], + ca_certs_der: &[Vec<u8>], +) -> Vec<Vec<u8>> { + let mut chain_der: Vec<Vec<u8>> = Vec::new(); + + // Start with peer certificates (what was sent during handshake) + for cert in peer_certs { + chain_der.push(cert.as_ref().to_vec()); + } + + // Keep adding issuers until we reach a root or can't find the issuer + while let Some(der) = chain_der.last() { + let last_cert_der = der; + + // Parse the last certificate in the chain + let (_, last_cert) = match X509Certificate::from_der(last_cert_der) { + Ok(parsed) => parsed, + Err(_) => break, + }; + + // Check if it's self-signed (root certificate) + if last_cert.subject() == last_cert.issuer() { + // This is a root certificate, we're done + break; + } + + // Try to find the issuer in the trust store + let issuer_name = last_cert.issuer(); + let mut found_issuer = false; + + for ca_der in ca_certs_der.iter() { + let (_, ca_cert) = match X509Certificate::from_der(ca_der) { + Ok(parsed) => parsed, + Err(_) => continue, + }; + + // Check if this CA's subject matches the certificate's issuer + if ca_cert.subject() == issuer_name { + // Check if we already have this certificate in the chain + if !chain_der.iter().any(|existing| existing == ca_der) { + chain_der.push(ca_der.clone()); + found_issuer = true; + break; + } + } + } + + if !found_issuer { + // Can't find issuer, stop here + break; + } + } + + chain_der +} + +/// Statistics from certificate loading operations +#[derive(Debug, Clone, Default)] +pub struct CertStats { + pub total_certs: usize, + pub ca_certs: usize, +} + +/// Certificate loader that handles PEM/DER parsing and validation +/// +/// This structure encapsulates the common pattern of loading certificates +/// from various sources (files, directories, bytes) and adding them to +/// a RootCertStore while tracking statistics. +/// +/// Duplicate certificates are detected and only counted once. +pub struct CertLoader<'a> { + store: &'a mut RootCertStore, + ca_certs_der: &'a mut Vec<Vec<u8>>, + seen_certs: HashSet<Vec<u8>>, +} + +impl<'a> CertLoader<'a> { + /// Create a new CertLoader with references to the store and DER cache + pub fn new(store: &'a mut RootCertStore, ca_certs_der: &'a mut Vec<Vec<u8>>) -> Self { + // Initialize seen_certs with existing certificates + let seen_certs = ca_certs_der.iter().cloned().collect(); + Self { + store, + ca_certs_der, + seen_certs, + } + } + + /// Load certificates from a file (supports both PEM and DER formats) + /// + /// Returns statistics about loaded certificates + pub fn load_from_file(&mut self, path: &str) -> Result<CertStats, std::io::Error> { + let contents = std::fs::read(path)?; + self.load_from_bytes(&contents) + } + + /// Load certificates from a directory + /// + /// Reads all files in the directory and attempts to parse them as certificates. + /// Invalid files are silently skipped (matches OpenSSL capath behavior). + pub fn load_from_dir(&mut self, dir_path: &str) -> Result<CertStats, std::io::Error> { + let entries = std::fs::read_dir(dir_path)?; + let mut stats = CertStats::default(); + + for entry in entries { + let entry = entry?; + let path = entry.path(); + + // Skip directories and process all files + // OpenSSL capath uses hash-based naming like "4e1295a3.0" + if path.is_file() + && let Ok(contents) = std::fs::read(&path) + { + // Ignore errors for individual files (some may not be certs) + if let Ok(file_stats) = self.load_from_bytes(&contents) { + stats.total_certs += file_stats.total_certs; + stats.ca_certs += file_stats.ca_certs; + } + } + } + + Ok(stats) + } + + /// Helper: Add a certificate to the store with duplicate checking + /// + /// Returns true if the certificate was added (not a duplicate), false if it was a duplicate. + fn add_cert_to_store( + &mut self, + cert_bytes: Vec<u8>, + cert_der: CertificateDer<'static>, + treat_all_as_ca: bool, + stats: &mut CertStats, + ) -> bool { + // Check for duplicates using HashSet + if !self.seen_certs.insert(cert_bytes.clone()) { + return false; // Duplicate certificate - skip + } + + // Determine if this is a CA certificate + let is_ca = if treat_all_as_ca { + true + } else { + is_ca_certificate(&cert_bytes) + }; + + // Store full DER for get_ca_certs() + self.ca_certs_der.push(cert_bytes); + + // Add to trust store (rustls may handle duplicates internally) + let _ = self.store.add(cert_der); + + // Update statistics + stats.total_certs += 1; + if is_ca { + stats.ca_certs += 1; + } + + true + } + + /// Load certificates from byte slice (auto-detects PEM vs DER format) + /// + /// Tries to parse as PEM first, falls back to DER if that fails. + /// Duplicate certificates are detected and only counted once. + /// + /// If `treat_all_as_ca` is true, all certificates are counted as CA certificates + /// regardless of their Basic Constraints (this matches + /// load_verify_locations with cadata parameter). + /// + /// If `pem_only` is true, only PEM parsing is attempted (for string input) + pub fn load_from_bytes_ex( + &mut self, + data: &[u8], + treat_all_as_ca: bool, + pem_only: bool, + ) -> Result<CertStats, std::io::Error> { + let mut stats = CertStats::default(); + + // Try to parse as PEM first + let mut cursor = std::io::Cursor::new(data); + let certs_iter = rustls_pemfile::certs(&mut cursor); + + let mut found_any = false; + let mut first_pem_error = None; // Store first PEM parsing error + for cert_result in certs_iter { + match cert_result { + Ok(cert) => { + found_any = true; + let cert_bytes = cert.to_vec(); + + // Validate that this is actually a valid X.509 certificate + // rustls_pemfile only does base64 decoding, not X.509 validation + if let Err(e) = X509Certificate::from_der(&cert_bytes) { + // Invalid X.509 certificate + return Err(cert_error::pem::parse_failed_debug(e)); + } + + // Add certificate using helper method (handles duplicates) + self.add_cert_to_store(cert_bytes, cert, treat_all_as_ca, &mut stats); + // Helper returns false for duplicates (skip counting) + } + Err(e) if !found_any => { + // PEM parsing failed on first certificate + if pem_only { + // For string input (PEM only), return "no start line" error + return Err(cert_error::pem::no_start_line( + "cadata does not contain a certificate", + )); + } + // Store the error and break to try DER format below + first_pem_error = Some(e); + break; + } + Err(e) => { + // PEM parsing failed after some certs were loaded + return Err(cert_error::pem::parse_failed(e)); + } + } + } + + // If PEM parsing found nothing, try DER format (unless pem_only) + // DER can have multiple certificates concatenated, so parse them sequentially + if !found_any && stats.total_certs == 0 { + // If we had a PEM parsing error, return it instead of trying DER fallback + // This ensures that malformed PEM files (like badcert.pem) raise an error + if let Some(e) = first_pem_error { + return Err(cert_error::pem::parse_failed(e)); + } + + // For PEM-only mode (string input), don't fallback to DER + if pem_only { + return Err(cert_error::pem::no_start_line( + "cadata does not contain a certificate", + )); + } + let mut remaining = data; + let mut loaded_count = 0; + + while !remaining.is_empty() { + match X509Certificate::from_der(remaining) { + Ok((rest, _parsed_cert)) => { + // Extract the DER bytes for this certificate + // Length = total remaining - bytes left after parsing + let cert_len = remaining.len() - rest.len(); + let cert_bytes = &remaining[..cert_len]; + let cert_der = CertificateDer::from(cert_bytes.to_vec()); + + // Add certificate using helper method (handles duplicates) + self.add_cert_to_store( + cert_bytes.to_vec(), + cert_der, + treat_all_as_ca, + &mut stats, + ); + + loaded_count += 1; + remaining = rest; // Move to next certificate + } + Err(e) => { + if loaded_count == 0 { + // Failed to parse first certificate - invalid data + return Err(cert_error::der::not_enough_data( + "cadata does not contain a certificate", + )); + } else { + // Loaded some certificates but failed on subsequent data (garbage) + return Err(cert_error::der::parse_failed(e)); + } + } + } + } + + // If we somehow got here with no certificates loaded + if loaded_count == 0 { + return Err(cert_error::der::not_enough_data( + "cadata does not contain a certificate", + )); + } + } + + Ok(stats) + } + + /// Load certificates from byte slice (auto-detects PEM vs DER format) + /// + /// This is a convenience wrapper that calls load_from_bytes_ex with treat_all_as_ca=false + /// and pem_only=false. + pub fn load_from_bytes(&mut self, data: &[u8]) -> Result<CertStats, std::io::Error> { + self.load_from_bytes_ex(data, false, false) + } +} + +// NoVerifier: disables certificate verification (for CERT_NONE mode) +#[derive(Debug)] +pub struct NoVerifier; + +impl ServerCertVerifier for NoVerifier { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result<ServerCertVerified, rustls::Error> { + // Accept all certificates without verification + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + // Accept all signatures without verification + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + // Accept all signatures without verification + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec<SignatureScheme> { + ALL_SIGNATURE_SCHEMES.to_vec() + } +} + +// HostnameIgnoringVerifier: verifies certificate chain but ignores hostname +// This is used when check_hostname=False but verify_mode != CERT_NONE +// +// Unlike the previous implementation that used an inner WebPkiServerVerifier, +// this version uses webpki directly to verify only the certificate chain, +// completely bypassing hostname verification. +#[derive(Debug)] +pub struct HostnameIgnoringVerifier { + inner: Arc<dyn ServerCertVerifier>, +} + +impl HostnameIgnoringVerifier { + /// Create a new HostnameIgnoringVerifier with a pre-built verifier + /// This is useful when you need to configure the verifier with CRLs or other options + pub fn new_with_verifier(inner: Arc<dyn ServerCertVerifier>) -> Self { + Self { inner } + } +} + +impl ServerCertVerifier for HostnameIgnoringVerifier { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, // Intentionally ignored + ocsp_response: &[u8], + now: UnixTime, + ) -> Result<ServerCertVerified, rustls::Error> { + // Extract a hostname from the certificate to pass to inner verifier + // The inner verifier will validate certificate chain, trust anchors, etc. + // but may fail on hostname mismatch - we'll catch and ignore that error + let dummy_hostname = extract_first_dns_name(end_entity) + .unwrap_or_else(|| ServerName::try_from("localhost").expect("localhost is valid")); + + // Call inner verifier for full certificate validation + match self.inner.verify_server_cert( + end_entity, + intermediates, + &dummy_hostname, + ocsp_response, + now, + ) { + Ok(verified) => Ok(verified), + Err(e) => { + // Check if the error is a hostname mismatch + // If so, ignore it (that's the whole point of HostnameIgnoringVerifier) + match e { + rustls::Error::InvalidCertificate( + rustls::CertificateError::NotValidForName, + ) + | rustls::Error::InvalidCertificate( + rustls::CertificateError::NotValidForNameContext { .. }, + ) => { + // Hostname mismatch - this is expected and acceptable + // The certificate chain, trust anchor, and expiry are valid + Ok(ServerCertVerified::assertion()) + } + _ => { + // Other errors (expired cert, untrusted CA, etc.) should propagate + Err(e) + } + } + } + } + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec<SignatureScheme> { + self.inner.supported_verify_schemes() + } +} + +// Helper function to extract the first DNS name from a certificate +fn extract_first_dns_name(cert_der: &CertificateDer<'_>) -> Option<ServerName<'static>> { + let (_, cert) = X509Certificate::from_der(cert_der.as_ref()).ok()?; + + // Try Subject Alternative Names first + if let Ok(Some(san_ext)) = cert.subject_alternative_name() { + for name in &san_ext.value.general_names { + if let x509_parser::extensions::GeneralName::DNSName(dns) = name { + // Remove wildcard prefix if present (e.g., "*.example.com" → "example.com") + // This allows us to use the domain for certificate chain verification + // when check_hostname=False + let dns_str = dns.to_string(); + let normalized_dns = normalize_wildcard_hostname(&dns_str); + + match ServerName::try_from(normalized_dns.to_string()) { + Ok(server_name) => { + return Some(server_name); + } + Err(_e) => { + // Continue to next + } + } + } + } + } + + // Fallback to Common Name + for rdn in cert.subject().iter() { + for attr in rdn.iter() { + if attr.attr_type() == &x509_parser::oid_registry::OID_X509_COMMON_NAME + && let Ok(cn) = attr.attr_value().as_str() + { + // Remove wildcard prefix if present + let normalized_cn = normalize_wildcard_hostname(cn); + + match ServerName::try_from(normalized_cn.to_string()) { + Ok(server_name) => { + return Some(server_name); + } + Err(_e) => {} + } + } + } + } + + None +} + +// Custom client certificate verifier for TLS 1.3 deferred validation +// This verifier always succeeds during handshake but stores verification errors +// for later retrieval during I/O operations +#[derive(Debug)] +pub struct DeferredClientCertVerifier { + // The actual verifier that performs validation + inner: Arc<dyn ClientCertVerifier>, + // Shared storage for deferred error message + deferred_error: Arc<ParkingRwLock<Option<String>>>, +} + +impl DeferredClientCertVerifier { + pub fn new( + inner: Arc<dyn ClientCertVerifier>, + deferred_error: Arc<ParkingRwLock<Option<String>>>, + ) -> Self { + Self { + inner, + deferred_error, + } + } +} + +impl ClientCertVerifier for DeferredClientCertVerifier { + fn offer_client_auth(&self) -> bool { + self.inner.offer_client_auth() + } + + fn client_auth_mandatory(&self) -> bool { + // Delegate to inner verifier to respect CERT_REQUIRED mode + // This ensures client certificates are mandatory when verify_mode=CERT_REQUIRED + self.inner.client_auth_mandatory() + } + + fn root_hint_subjects(&self) -> &[rustls::DistinguishedName] { + self.inner.root_hint_subjects() + } + + fn verify_client_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + now: UnixTime, + ) -> Result<ClientCertVerified, rustls::Error> { + // Perform the actual verification + let result = self + .inner + .verify_client_cert(end_entity, intermediates, now); + + // If verification failed, store the error for the server's Python code + // AND return the error so rustls sends the appropriate TLS alert + if let Err(ref e) = result { + let error_msg = format!("certificate verify failed: {e}"); + *self.deferred_error.write() = Some(error_msg); + // Return the error to rustls so it sends the alert to the client + return result; + } + + result + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec<SignatureScheme> { + self.inner.supported_verify_schemes() + } +} + +// Public Utility Functions + +/// Load certificate chain and private key from files +/// +/// This function loads a certificate chain from `cert_path` and a private key +/// from `key_path`. If `password` is provided, it will be used to decrypt +/// an encrypted private key. +/// +/// Returns (certificate_chain, private_key) on success. +/// +/// # Arguments +/// * `cert_path` - Path to certificate file (PEM or DER format) +/// * `key_path` - Path to private key file (PEM or DER format, optionally encrypted) +/// * `password` - Optional password for encrypted private key +/// +/// # Errors +/// Returns error if: +/// - Files cannot be read +/// - Certificate or key cannot be parsed +/// - Password is incorrect for encrypted key +pub(super) fn load_cert_chain_from_file( + cert_path: &str, + key_path: &str, + password: Option<&str>, +) -> Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>), Box<dyn core::error::Error>> { + // Load certificate file - preserve io::Error for errno + let cert_contents = std::fs::read(cert_path)?; + + // Parse certificates (PEM format) + let mut cert_cursor = std::io::Cursor::new(&cert_contents); + let certs: Vec<CertificateDer<'static>> = rustls_pemfile::certs(&mut cert_cursor) + .collect::<Result<Vec<_>, _>>() + .map_err(cert_error::pem::parse_failed)?; + + if certs.is_empty() { + return Err(Box::new(cert_error::pem::invalid_cert())); + } + + // Load private key file - preserve io::Error for errno + let key_contents = std::fs::read(key_path)?; + + // Parse private key (supports PKCS8, RSA, EC formats) + let private_key = if let Some(pwd) = password { + // Try to parse as encrypted PKCS#8 + use der::SecretDocument; + use pkcs8::EncryptedPrivateKeyInfo; + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + let pem_str = String::from_utf8_lossy(&key_contents); + + // Extract just the ENCRYPTED PRIVATE KEY block if present + // (file may contain multiple PEM blocks like key + certificate) + let encrypted_key_pem = if let Some(start) = + pem_str.find("-----BEGIN ENCRYPTED PRIVATE KEY-----") + { + if let Some(end_marker) = pem_str[start..].find("-----END ENCRYPTED PRIVATE KEY-----") { + let end = start + end_marker + "-----END ENCRYPTED PRIVATE KEY-----".len(); + Some(&pem_str[start..end]) + } else { + None + } + } else { + None + }; + + // Try to decode and decrypt PEM-encoded encrypted private key using pkcs8's PEM support + let decrypted_key_result = if let Some(key_pem) = encrypted_key_pem { + match SecretDocument::from_pem(key_pem) { + Ok((label, doc)) => { + if label == "ENCRYPTED PRIVATE KEY" { + // Parse encrypted key info from DER + match EncryptedPrivateKeyInfo::try_from(doc.as_bytes()) { + Ok(encrypted_key) => { + // Decrypt with password + match encrypted_key.decrypt(pwd.as_bytes()) { + Ok(decrypted) => { + // Convert decrypted SecretDocument to PrivateKeyDer + let key_vec: Vec<u8> = decrypted.as_bytes().to_vec(); + let pkcs8_key: PrivatePkcs8KeyDer<'static> = key_vec.into(); + Some(PrivateKeyDer::Pkcs8(pkcs8_key)) + } + Err(e) => { + return Err(Box::new(cert_error::key::decrypt_failed(e))); + } + } + } + Err(e) => { + return Err(Box::new(cert_error::key::parse_encrypted_failed(e))); + } + } + } else { + None + } + } + Err(_) => None, + } + } else { + None + }; + + match decrypted_key_result { + Some(key) => key, + None => { + // Not encrypted PKCS#8, try as unencrypted key + // (password might have been provided for an unencrypted key) + let mut key_cursor = std::io::Cursor::new(&key_contents); + match rustls_pemfile::private_key(&mut key_cursor) { + Ok(Some(key)) => key, + Ok(None) => { + return Err(Box::new(cert_error::key::not_found("key file"))); + } + Err(e) => { + return Err(Box::new(cert_error::key::parse_failed(e))); + } + } + } + } + } else { + // No password provided - try to parse unencrypted key + let mut key_cursor = std::io::Cursor::new(&key_contents); + match rustls_pemfile::private_key(&mut key_cursor) { + Ok(Some(key)) => key, + Ok(None) => { + return Err(Box::new(cert_error::key::not_found("key file"))); + } + Err(e) => { + return Err(Box::new(cert_error::key::parse_failed(e))); + } + } + }; + + Ok((certs, private_key)) +} + +/// Validate that a certificate and private key match +/// +/// This function checks that the public key in the certificate matches +/// the provided private key. This is a basic sanity check to prevent +/// configuration errors. +/// +/// # Arguments +/// * `certs` - Certificate chain (first certificate is the leaf) +/// * `private_key` - Private key to validate against +/// +/// # Errors +/// Returns error if: +/// - Certificate chain is empty +/// - Public key extraction fails +/// - Keys don't match +/// +/// Note: This is a simplified validation. Full validation would require +/// signing and verifying a test message, which is complex with rustls. +pub fn validate_cert_key_match( + certs: &[CertificateDer<'_>], + private_key: &PrivateKeyDer<'_>, +) -> Result<(), String> { + if certs.is_empty() { + return Err("Certificate chain is empty".to_string()); + } + + // For rustls, the actual validation happens when creating CertifiedKey + // We can attempt to create a signing key to verify the key is valid + use rustls::crypto::aws_lc_rs::sign::any_supported_type; + + match any_supported_type(private_key) { + Ok(_signing_key) => { + // If we can create a signing key, the private key is valid + // Rustls will validate the cert-key match when building config + Ok(()) + } + Err(_) => Err("PEM lib".to_string()), + } +} + +/// StrictCertVerifier: wraps a ServerCertVerifier and adds RFC 5280 strict validation +/// +/// When VERIFY_X509_STRICT flag is set, performs additional validation: +/// - Checks for Authority Key Identifier (AKI) extension (required by RFC 5280 Section 4.2.1.1) +/// - Validates other RFC 5280 compliance requirements +/// +/// This matches X509_V_FLAG_X509_STRICT behavior in OpenSSL. +#[derive(Debug)] +pub struct StrictCertVerifier { + inner: Arc<dyn ServerCertVerifier>, + verify_flags: i32, +} + +impl StrictCertVerifier { + /// Create a new StrictCertVerifier + /// + /// # Arguments + /// * `inner` - The underlying verifier to wrap + /// * `verify_flags` - SSL verification flags (e.g., VERIFY_X509_STRICT) + pub fn new(inner: Arc<dyn ServerCertVerifier>, verify_flags: i32) -> Self { + Self { + inner, + verify_flags, + } + } + + /// Check if a certificate has the Authority Key Identifier extension + /// + /// RFC 5280 Section 4.2.1.1 states that conforming CAs MUST include this + /// extension in all certificates except self-signed certificates. + fn check_aki_present(cert_der: &[u8]) -> Result<(), String> { + let (_, cert) = X509Certificate::from_der(cert_der) + .map_err(|e| format!("Failed to parse certificate: {e}"))?; + + // Check for Authority Key Identifier extension (OID 2.5.29.35) + let has_aki = cert + .tbs_certificate + .extensions() + .iter() + .any(|ext| ext.oid == oid_registry::OID_X509_EXT_AUTHORITY_KEY_IDENTIFIER); + + if !has_aki { + return Err( + "certificate verification failed: certificate missing required Authority Key Identifier extension" + .to_string(), + ); + } + + Ok(()) + } +} + +impl ServerCertVerifier for StrictCertVerifier { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + server_name: &ServerName<'_>, + ocsp_response: &[u8], + now: UnixTime, + ) -> Result<ServerCertVerified, rustls::Error> { + // First, perform the standard verification + let result = self.inner.verify_server_cert( + end_entity, + intermediates, + server_name, + ocsp_response, + now, + )?; + + // If VERIFY_X509_STRICT flag is set, perform additional validation + if self.verify_flags & VERIFY_X509_STRICT != 0 { + // Check end entity certificate for AKI + // RFC 5280 Section 4.2.1.1: self-signed certificates are exempt from AKI requirement + if !is_self_signed(end_entity) { + Self::check_aki_present(end_entity.as_ref()) + .map_err(cert_error::to_rustls_invalid_cert)?; + } + + // Check intermediate certificates for AKI + for intermediate in intermediates { + Self::check_aki_present(intermediate.as_ref()) + .map_err(cert_error::to_rustls_invalid_cert)?; + } + } + + Ok(result) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec<SignatureScheme> { + self.inner.supported_verify_schemes() + } +} + +/// EmptyRootStoreVerifier: used when verify_mode != CERT_NONE but no CA certs are loaded +/// +/// This verifier always fails certificate verification with UnknownIssuer error, +/// when no root certificates are available. +/// This allows the SSL context to be created successfully, but handshake will fail +/// with a proper SSLCertVerificationError (verify_code=20, UNABLE_TO_GET_ISSUER_CERT_LOCALLY). +#[derive(Debug)] +pub struct EmptyRootStoreVerifier; + +impl ServerCertVerifier for EmptyRootStoreVerifier { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result<ServerCertVerified, rustls::Error> { + // Always fail with UnknownIssuer - when no CA certs loaded + // This will be mapped to X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY (20) + Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::UnknownIssuer, + )) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + // Accept signatures during handshake - the cert verification will fail anyway + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + // Accept signatures during handshake - the cert verification will fail anyway + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec<SignatureScheme> { + ALL_SIGNATURE_SCHEMES.to_vec() + } +} + +/// CRLCheckVerifier: Wraps a verifier to enforce CRL checking when flags are set +/// +/// This verifier ensures that when CRL checking flags are set (VERIFY_CRL_CHECK_LEAF = 4) +/// but no CRLs have been loaded, the verification fails with UnknownRevocationStatus. +/// This matches X509_V_FLAG_CRL_CHECK without loaded CRLs +/// causes "unable to get CRL" error. +#[derive(Debug)] +pub struct CRLCheckVerifier { + inner: Arc<dyn ServerCertVerifier>, + has_crls: bool, + crl_check_enabled: bool, +} + +impl CRLCheckVerifier { + pub fn new( + inner: Arc<dyn ServerCertVerifier>, + has_crls: bool, + crl_check_enabled: bool, + ) -> Self { + Self { + inner, + has_crls, + crl_check_enabled, + } + } +} + +impl ServerCertVerifier for CRLCheckVerifier { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + server_name: &ServerName<'_>, + ocsp_response: &[u8], + now: UnixTime, + ) -> Result<ServerCertVerified, rustls::Error> { + // If CRL checking is enabled but no CRLs are loaded, fail with UnknownRevocationStatus + // X509_V_ERR_UNABLE_TO_GET_CRL (3) + if self.crl_check_enabled && !self.has_crls { + return Err(rustls::Error::InvalidCertificate( + rustls::CertificateError::UnknownRevocationStatus, + )); + } + + // Otherwise, delegate to inner verifier + self.inner + .verify_server_cert(end_entity, intermediates, server_name, ocsp_response, now) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec<SignatureScheme> { + self.inner.supported_verify_schemes() + } +} + +/// Partial Chain Verifier - Handles VERIFY_X509_PARTIAL_CHAIN flag +/// +/// OpenSSL's X509_V_FLAG_PARTIAL_CHAIN allows verification to succeed if any certificate +/// in the presented chain is found in the trust store, not just the root CA. This is useful +/// for trusting intermediate certificates or self-signed certificates directly. +/// +/// rustls's WebPkiServerVerifier doesn't support this behavior by default, so we wrap it +/// to add partial chain support when the flag is set. +/// +/// Behavior: +/// 1. Try standard verification first (full chain to trusted root) +/// 2. If that fails and VERIFY_X509_PARTIAL_CHAIN is set: +/// - Check if the end-entity certificate is in the trust store +/// - If yes, accept the certificate as trusted +/// +/// This matches accepting self-signed certificates that +/// are explicitly loaded via load_verify_locations(). +#[derive(Debug)] +pub struct PartialChainVerifier { + inner: Arc<dyn ServerCertVerifier>, + ca_certs_der: Vec<Vec<u8>>, + verify_flags: i32, +} + +impl PartialChainVerifier { + pub fn new( + inner: Arc<dyn ServerCertVerifier>, + ca_certs_der: Vec<Vec<u8>>, + verify_flags: i32, + ) -> Self { + Self { + inner, + ca_certs_der, + verify_flags, + } + } +} + +impl ServerCertVerifier for PartialChainVerifier { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + server_name: &ServerName<'_>, + ocsp_response: &[u8], + now: UnixTime, + ) -> Result<ServerCertVerified, rustls::Error> { + // Try standard verification first + match self.inner.verify_server_cert( + end_entity, + intermediates, + server_name, + ocsp_response, + now, + ) { + Ok(result) => Ok(result), + Err(e) => { + // If verification failed, check if the end-entity certificate is in the trust store + // OpenSSL behavior: + // 1. Self-signed certs in trust store: ALWAYS trusted (flag not required) + // 2. Non-self-signed end-entity certs in trust store: require VERIFY_X509_PARTIAL_CHAIN + // 3. Intermediate certs in trust store: require VERIFY_X509_PARTIAL_CHAIN + let end_entity_der = end_entity.as_ref(); + if self + .ca_certs_der + .iter() + .any(|cert_der| cert_der.as_slice() == end_entity_der) + { + // End-entity certificate is in the trust store + // Check if this is a self-signed certificate + let is_self_signed_cert = is_self_signed(end_entity); + + // Self-signed: always trust (OpenSSL behavior) + // Non-self-signed: require VERIFY_X509_PARTIAL_CHAIN flag + if is_self_signed_cert || (self.verify_flags & VERIFY_X509_PARTIAL_CHAIN != 0) { + // Certificate is trusted, but still perform hostname verification + verify_hostname(end_entity, server_name)?; + return Ok(ServerCertVerified::assertion()); + } + } + // No match found or non-self-signed without flag - return original error + Err(e) + } + } + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result<HandshakeSignatureValid, rustls::Error> { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec<SignatureScheme> { + self.inner.supported_verify_schemes() + } +} + +// Hostname Verification: + +/// Check if a certificate is self-signed by comparing issuer and subject. +/// Returns true if the certificate is self-signed (issuer == subject). +fn is_self_signed(cert_der: &CertificateDer<'_>) -> bool { + use x509_parser::prelude::*; + + // Parse the certificate + let Ok((_, cert)) = X509Certificate::from_der(cert_der.as_ref()) else { + // If we can't parse it, assume it's not self-signed (conservative approach) + return false; + }; + + // Compare issuer and subject + // A certificate is self-signed if issuer == subject + cert.issuer() == cert.subject() +} + +/// Verify that a certificate is valid for the given hostname/IP address. +/// This function checks Subject Alternative Names (SAN) and Common Name (CN). +fn verify_hostname( + cert_der: &CertificateDer<'_>, + server_name: &ServerName<'_>, +) -> Result<(), rustls::Error> { + use x509_parser::extensions::GeneralName; + use x509_parser::prelude::*; + + // Parse the certificate + let (_, cert) = X509Certificate::from_der(cert_der.as_ref()).map_err(|e| { + cert_error::to_rustls_invalid_cert(format!( + "Failed to parse certificate for hostname verification: {e}" + )) + })?; + + match server_name { + ServerName::DnsName(dns) => { + let expected_name = dns.as_ref(); + + // 1. Check Subject Alternative Names (SAN) - preferred method + if let Ok(Some(san_ext)) = cert.subject_alternative_name() { + for name in &san_ext.value.general_names { + if let GeneralName::DNSName(dns_name) = name + && hostname_matches(expected_name, dns_name) + { + return Ok(()); + } + } + } + + // 2. Fallback to Common Name (CN) - deprecated but still checked for compatibility + for rdn in cert.subject().iter() { + for attr in rdn.iter() { + if attr.attr_type() == &x509_parser::oid_registry::OID_X509_COMMON_NAME + && let Ok(cn) = attr.attr_value().as_str() + && hostname_matches(expected_name, cn) + { + return Ok(()); + } + } + } + + // No match found - return error + Err(cert_error::to_rustls_invalid_cert(format!( + "Hostname mismatch: certificate is not valid for '{expected_name}'", + ))) + } + ServerName::IpAddress(ip) => verify_ip_address(&cert, ip), + _ => { + // Unknown server name type + Err(cert_error::to_rustls_cert_error( + std::io::ErrorKind::InvalidInput, + "Unsupported server name type for hostname verification", + )) + } + } +} + +/// Match a hostname against a pattern, supporting wildcard certificates (*.example.com). +/// Implements RFC 6125 wildcard matching rules: +/// - Wildcard must be in the leftmost label +/// - Wildcard must be the only character in that label +/// - Wildcard must match at least one character +fn hostname_matches(expected: &str, pattern: &str) -> bool { + // Wildcard matching for *.example.com + if let Some(pattern_base) = pattern.strip_prefix("*.") { + // Find the first dot in expected hostname + if let Some(dot_pos) = expected.find('.') { + let expected_base = &expected[dot_pos + 1..]; + + // The base domains must match (case insensitive) + // and the leftmost label must not be empty + return dot_pos > 0 && expected_base.eq_ignore_ascii_case(pattern_base); + } + + // No dot in expected, can't match wildcard + return false; + } + + // Exact match (case insensitive per RFC 4343) + expected.eq_ignore_ascii_case(pattern) +} + +/// Verify that a certificate is valid for the given IP address. +/// Checks Subject Alternative Names for IP Address entries. +fn verify_ip_address( + cert: &X509Certificate<'_>, + expected_ip: &rustls::pki_types::IpAddr, +) -> Result<(), rustls::Error> { + use core::net::IpAddr; + use x509_parser::extensions::GeneralName; + + // Convert rustls IpAddr to std::net::IpAddr for comparison + let expected_std_ip: IpAddr = match expected_ip { + rustls::pki_types::IpAddr::V4(octets) => IpAddr::V4(core::net::Ipv4Addr::from(*octets)), + rustls::pki_types::IpAddr::V6(octets) => IpAddr::V6(core::net::Ipv6Addr::from(*octets)), + }; + + // Check Subject Alternative Names for IP addresses + if let Ok(Some(san_ext)) = cert.subject_alternative_name() { + for name in &san_ext.value.general_names { + if let GeneralName::IPAddress(cert_ip_bytes) = name { + // Parse the IP address from the certificate + let cert_ip = match cert_ip_bytes.len() { + 4 => { + // IPv4 + if let Ok(octets) = <[u8; 4]>::try_from(*cert_ip_bytes) { + IpAddr::V4(core::net::Ipv4Addr::from(octets)) + } else { + continue; + } + } + 16 => { + // IPv6 + if let Ok(octets) = <[u8; 16]>::try_from(*cert_ip_bytes) { + IpAddr::V6(core::net::Ipv6Addr::from(octets)) + } else { + continue; + } + } + _ => continue, // Invalid IP address length + }; + + if cert_ip == expected_std_ip { + return Ok(()); + } + } + } + } + + // No matching IP address found + Err(cert_error::to_rustls_invalid_cert(format!( + "IP address mismatch: certificate is not valid for '{expected_std_ip}'", + ))) +} diff --git a/crates/stdlib/src/ssl/compat.rs b/crates/stdlib/src/ssl/compat.rs new file mode 100644 index 00000000000..14812375a8b --- /dev/null +++ b/crates/stdlib/src/ssl/compat.rs @@ -0,0 +1,2409 @@ +// spell-checker: ignore webpki ssleof sslerror akid certsign sslerr aesgcm + +// OpenSSL compatibility layer for rustls +// +// This module provides OpenSSL-like abstractions over rustls APIs, +// making the code more readable and maintainable. Each function is named +// after its OpenSSL equivalent (e.g., ssl_do_handshake corresponds to SSL_do_handshake). + +// SSL error code data tables (shared with OpenSSL backend for compatibility) +// These map OpenSSL error codes to human-readable strings +#[path = "../openssl/ssl_data_31.rs"] +mod ssl_data; + +use crate::socket::{SelectKind, timeout_error_msg}; +use crate::vm::VirtualMachine; +use alloc::sync::Arc; +use parking_lot::RwLock as ParkingRwLock; +use rustls::RootCertStore; +use rustls::client::ClientConfig; +use rustls::client::ClientConnection; +use rustls::crypto::SupportedKxGroup; +use rustls::pki_types::{CertificateDer, CertificateRevocationListDer, PrivateKeyDer}; +use rustls::server::ResolvesServerCert; +use rustls::server::ServerConfig; +use rustls::server::ServerConnection; +use rustls::sign::CertifiedKey; +use rustpython_vm::builtins::{PyBaseException, PyBaseExceptionRef}; +use rustpython_vm::convert::IntoPyException; +use rustpython_vm::function::ArgBytesLike; +use rustpython_vm::{AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject}; +use std::io::Read; +use std::sync::Once; + +// Import PySSLSocket from parent module +use super::_ssl::PySSLSocket; + +// Import error types and helper functions from error module +use super::error::{ + PySSLCertVerificationError, PySSLError, create_ssl_eof_error, create_ssl_syscall_error, + create_ssl_want_read_error, create_ssl_want_write_error, create_ssl_zero_return_error, +}; + +// SSL Verification Flags +/// VERIFY_X509_STRICT flag for RFC 5280 strict compliance +/// When set, performs additional validation including AKI extension checks +pub const VERIFY_X509_STRICT: i32 = 0x20; + +/// VERIFY_X509_PARTIAL_CHAIN flag for partial chain validation +/// When set, accept certificates if any certificate in the chain is in the trust store +/// (not just root CAs). This matches OpenSSL's X509_V_FLAG_PARTIAL_CHAIN behavior. +pub const VERIFY_X509_PARTIAL_CHAIN: i32 = 0x80000; + +// CryptoProvider Initialization: + +/// Ensure the default CryptoProvider is installed (thread-safe, runs once) +/// +/// This is necessary because rustls 0.23+ requires a process-level CryptoProvider +/// to be installed before using default_provider(). We use Once to ensure this +/// happens exactly once, even if called from multiple threads. +static INIT_PROVIDER: Once = Once::new(); + +fn ensure_default_provider() { + INIT_PROVIDER.call_once(|| { + let _ = rustls::crypto::CryptoProvider::install_default( + rustls::crypto::aws_lc_rs::default_provider(), + ); + }); +} + +// OpenSSL Constants: + +// OpenSSL TLS record maximum plaintext size (ssl/ssl_local.h) +// #define SSL3_RT_MAX_PLAIN_LENGTH 16384 +const SSL3_RT_MAX_PLAIN_LENGTH: usize = 16384; + +// OpenSSL error library codes (include/openssl/err.h) +// #define ERR_LIB_SSL 20 +const ERR_LIB_SSL: i32 = 20; + +// OpenSSL SSL error reason codes (include/openssl/sslerr.h) +// #define SSL_R_NO_SHARED_CIPHER 193 +const SSL_R_NO_SHARED_CIPHER: i32 = 193; + +// OpenSSL X509 verification flags (include/openssl/x509_vfy.h) +// #define X509_V_FLAG_CRL_CHECK 4 +const X509_V_FLAG_CRL_CHECK: i32 = 4; + +// X509 Certificate Verification Error Codes (OpenSSL Compatible): +// +// These constants match OpenSSL's X509_V_ERR_* values for certificate +// verification. They are used to map rustls certificate errors to OpenSSL +// error codes for compatibility. + +pub use x509::{ + X509_V_ERR_CERT_HAS_EXPIRED, X509_V_ERR_CERT_NOT_YET_VALID, X509_V_ERR_CERT_REVOKED, + X509_V_ERR_HOSTNAME_MISMATCH, X509_V_ERR_INVALID_PURPOSE, X509_V_ERR_IP_ADDRESS_MISMATCH, + X509_V_ERR_UNABLE_TO_GET_CRL, X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY, + X509_V_ERR_UNSPECIFIED, +}; + +#[allow(dead_code)] +mod x509 { + pub const X509_V_OK: i32 = 0; + pub const X509_V_ERR_UNSPECIFIED: i32 = 1; + pub const X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT: i32 = 2; + pub const X509_V_ERR_UNABLE_TO_GET_CRL: i32 = 3; + pub const X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE: i32 = 4; + pub const X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE: i32 = 5; + pub const X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY: i32 = 6; + pub const X509_V_ERR_CERT_SIGNATURE_FAILURE: i32 = 7; + pub const X509_V_ERR_CRL_SIGNATURE_FAILURE: i32 = 8; + pub const X509_V_ERR_CERT_NOT_YET_VALID: i32 = 9; + pub const X509_V_ERR_CERT_HAS_EXPIRED: i32 = 10; + pub const X509_V_ERR_CRL_NOT_YET_VALID: i32 = 11; + pub const X509_V_ERR_CRL_HAS_EXPIRED: i32 = 12; + pub const X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD: i32 = 13; + pub const X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD: i32 = 14; + pub const X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD: i32 = 15; + pub const X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD: i32 = 16; + pub const X509_V_ERR_OUT_OF_MEM: i32 = 17; + pub const X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT: i32 = 18; + pub const X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: i32 = 19; + pub const X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY: i32 = 20; + pub const X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE: i32 = 21; + pub const X509_V_ERR_CERT_CHAIN_TOO_LONG: i32 = 22; + pub const X509_V_ERR_CERT_REVOKED: i32 = 23; + pub const X509_V_ERR_INVALID_CA: i32 = 24; + pub const X509_V_ERR_PATH_LENGTH_EXCEEDED: i32 = 25; + pub const X509_V_ERR_INVALID_PURPOSE: i32 = 26; + pub const X509_V_ERR_CERT_UNTRUSTED: i32 = 27; + pub const X509_V_ERR_CERT_REJECTED: i32 = 28; + pub const X509_V_ERR_SUBJECT_ISSUER_MISMATCH: i32 = 29; + pub const X509_V_ERR_AKID_SKID_MISMATCH: i32 = 30; + pub const X509_V_ERR_AKID_ISSUER_SERIAL_MISMATCH: i32 = 31; + pub const X509_V_ERR_KEYUSAGE_NO_CERTSIGN: i32 = 32; + pub const X509_V_ERR_UNABLE_TO_GET_CRL_ISSUER: i32 = 33; + pub const X509_V_ERR_UNHANDLED_CRITICAL_EXTENSION: i32 = 34; + pub const X509_V_ERR_KEYUSAGE_NO_CRL_SIGN: i32 = 35; + pub const X509_V_ERR_UNHANDLED_CRITICAL_CRL_EXTENSION: i32 = 36; + pub const X509_V_ERR_INVALID_NON_CA: i32 = 37; + pub const X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED: i32 = 38; + pub const X509_V_ERR_KEYUSAGE_NO_DIGITAL_SIGNATURE: i32 = 39; + pub const X509_V_ERR_PROXY_CERTIFICATES_NOT_ALLOWED: i32 = 40; + pub const X509_V_ERR_INVALID_EXTENSION: i32 = 41; + pub const X509_V_ERR_INVALID_POLICY_EXTENSION: i32 = 42; + pub const X509_V_ERR_NO_EXPLICIT_POLICY: i32 = 43; + pub const X509_V_ERR_DIFFERENT_CRL_SCOPE: i32 = 44; + pub const X509_V_ERR_UNSUPPORTED_EXTENSION_FEATURE: i32 = 45; + pub const X509_V_ERR_UNNESTED_RESOURCE: i32 = 46; + pub const X509_V_ERR_PERMITTED_VIOLATION: i32 = 47; + pub const X509_V_ERR_EXCLUDED_VIOLATION: i32 = 48; + pub const X509_V_ERR_SUBTREE_MINMAX: i32 = 49; + pub const X509_V_ERR_APPLICATION_VERIFICATION: i32 = 50; + pub const X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE: i32 = 51; + pub const X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX: i32 = 52; + pub const X509_V_ERR_UNSUPPORTED_NAME_SYNTAX: i32 = 53; + pub const X509_V_ERR_CRL_PATH_VALIDATION_ERROR: i32 = 54; + pub const X509_V_ERR_HOSTNAME_MISMATCH: i32 = 62; + pub const X509_V_ERR_EMAIL_MISMATCH: i32 = 63; + pub const X509_V_ERR_IP_ADDRESS_MISMATCH: i32 = 64; +} + +// Certificate Error Conversion Functions: + +/// Convert rustls CertificateError to X509 verification code and message +/// +/// Maps rustls certificate errors to OpenSSL X509_V_ERR_* codes for compatibility. +/// Returns (verify_code, verify_message) tuple. +fn rustls_cert_error_to_verify_info(cert_err: &rustls::CertificateError) -> (i32, &'static str) { + use rustls::CertificateError; + + match cert_err { + CertificateError::UnknownIssuer => ( + X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY, + "unable to get local issuer certificate", + ), + CertificateError::Expired => (X509_V_ERR_CERT_HAS_EXPIRED, "certificate has expired"), + CertificateError::NotValidYet => ( + X509_V_ERR_CERT_NOT_YET_VALID, + "certificate is not yet valid", + ), + CertificateError::Revoked => (X509_V_ERR_CERT_REVOKED, "certificate revoked"), + CertificateError::UnknownRevocationStatus => ( + X509_V_ERR_UNABLE_TO_GET_CRL, + "unable to get certificate CRL", + ), + CertificateError::InvalidPurpose => ( + X509_V_ERR_INVALID_PURPOSE, + "unsupported certificate purpose", + ), + CertificateError::Other(other_err) => { + // Check if this is a hostname mismatch error from our verify_hostname function + let err_msg = format!("{other_err:?}"); + if err_msg.contains("Hostname mismatch") || err_msg.contains("not valid for") { + ( + X509_V_ERR_HOSTNAME_MISMATCH, + "Hostname mismatch, certificate is not valid for", + ) + } else if err_msg.contains("IP address mismatch") { + ( + X509_V_ERR_IP_ADDRESS_MISMATCH, + "IP address mismatch, certificate is not valid for", + ) + } else { + (X509_V_ERR_UNSPECIFIED, "certificate verification failed") + } + } + _ => (X509_V_ERR_UNSPECIFIED, "certificate verification failed"), + } +} + +/// Create SSLCertVerificationError with proper attributes +/// +/// Matches CPython's _ssl.c fill_and_set_sslerror() behavior. +/// This function creates a Python SSLCertVerificationError exception with verify_code +/// and verify_message attributes set appropriately for the given rustls certificate error. +/// +/// # Note +/// If attribute setting fails (extremely rare), returns the exception without attributes +pub(super) fn create_ssl_cert_verification_error( + vm: &VirtualMachine, + cert_err: &rustls::CertificateError, +) -> PyResult<PyBaseExceptionRef> { + let (verify_code, verify_message) = rustls_cert_error_to_verify_info(cert_err); + + let msg = + format!("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: {verify_message}",); + + let exc = vm.new_os_subtype_error( + PySSLCertVerificationError::class(&vm.ctx).to_owned(), + None, + msg, + ); + + // Set verify_code and verify_message attributes + // Ignore errors as they're extremely rare (e.g., out of memory) + exc.as_object().set_attr( + "verify_code", + vm.ctx.new_int(verify_code).as_object().to_owned(), + vm, + )?; + exc.as_object().set_attr( + "verify_message", + vm.ctx.new_str(verify_message).as_object().to_owned(), + vm, + )?; + + exc.as_object() + .set_attr("library", vm.ctx.new_str("SSL").as_object().to_owned(), vm)?; + exc.as_object().set_attr( + "reason", + vm.ctx + .new_str("CERTIFICATE_VERIFY_FAILED") + .as_object() + .to_owned(), + vm, + )?; + + Ok(exc.upcast()) +} + +/// Unified TLS connection type (client or server) +#[derive(Debug)] +pub(super) enum TlsConnection { + Client(ClientConnection), + Server(ServerConnection), +} + +impl TlsConnection { + /// Check if handshake is in progress + pub fn is_handshaking(&self) -> bool { + match self { + TlsConnection::Client(conn) => conn.is_handshaking(), + TlsConnection::Server(conn) => conn.is_handshaking(), + } + } + + /// Check if connection wants to read data + pub fn wants_read(&self) -> bool { + match self { + TlsConnection::Client(conn) => conn.wants_read(), + TlsConnection::Server(conn) => conn.wants_read(), + } + } + + /// Check if connection wants to write data + pub fn wants_write(&self) -> bool { + match self { + TlsConnection::Client(conn) => conn.wants_write(), + TlsConnection::Server(conn) => conn.wants_write(), + } + } + + /// Read TLS data from socket + pub fn read_tls(&mut self, reader: &mut dyn std::io::Read) -> std::io::Result<usize> { + match self { + TlsConnection::Client(conn) => conn.read_tls(reader), + TlsConnection::Server(conn) => conn.read_tls(reader), + } + } + + /// Write TLS data to socket + pub fn write_tls(&mut self, writer: &mut dyn std::io::Write) -> std::io::Result<usize> { + match self { + TlsConnection::Client(conn) => conn.write_tls(writer), + TlsConnection::Server(conn) => conn.write_tls(writer), + } + } + + /// Process new TLS packets + pub fn process_new_packets(&mut self) -> Result<rustls::IoState, rustls::Error> { + match self { + TlsConnection::Client(conn) => conn.process_new_packets(), + TlsConnection::Server(conn) => conn.process_new_packets(), + } + } + + /// Get reader for plaintext data (rustls native type) + pub fn reader(&mut self) -> rustls::Reader<'_> { + match self { + TlsConnection::Client(conn) => conn.reader(), + TlsConnection::Server(conn) => conn.reader(), + } + } + + /// Get writer for plaintext data (rustls native type) + pub fn writer(&mut self) -> rustls::Writer<'_> { + match self { + TlsConnection::Client(conn) => conn.writer(), + TlsConnection::Server(conn) => conn.writer(), + } + } + + /// Check if session was resumed + pub fn is_session_resumed(&self) -> bool { + use rustls::HandshakeKind; + match self { + TlsConnection::Client(conn) => { + matches!(conn.handshake_kind(), Some(HandshakeKind::Resumed)) + } + TlsConnection::Server(conn) => { + matches!(conn.handshake_kind(), Some(HandshakeKind::Resumed)) + } + } + } + + /// Send close_notify alert + pub fn send_close_notify(&mut self) { + match self { + TlsConnection::Client(conn) => conn.send_close_notify(), + TlsConnection::Server(conn) => conn.send_close_notify(), + } + } + + /// Get negotiated ALPN protocol + pub fn alpn_protocol(&self) -> Option<&[u8]> { + match self { + TlsConnection::Client(conn) => conn.alpn_protocol(), + TlsConnection::Server(conn) => conn.alpn_protocol(), + } + } + + /// Get negotiated cipher suite + pub fn negotiated_cipher_suite(&self) -> Option<rustls::SupportedCipherSuite> { + match self { + TlsConnection::Client(conn) => conn.negotiated_cipher_suite(), + TlsConnection::Server(conn) => conn.negotiated_cipher_suite(), + } + } + + /// Get peer certificates + pub fn peer_certificates(&self) -> Option<&[rustls::pki_types::CertificateDer<'static>]> { + match self { + TlsConnection::Client(conn) => conn.peer_certificates(), + TlsConnection::Server(conn) => conn.peer_certificates(), + } + } +} + +/// Error types matching OpenSSL error codes +#[derive(Debug)] +pub(super) enum SslError { + /// SSL_ERROR_WANT_READ + WantRead, + /// SSL_ERROR_WANT_WRITE + WantWrite, + /// SSL_ERROR_SYSCALL + Syscall(String), + /// SSL_ERROR_SSL + Ssl(String), + /// SSL_ERROR_ZERO_RETURN (clean closure with close_notify) + ZeroReturn, + /// Unexpected EOF without close_notify (protocol violation) + Eof, + /// Non-TLS data received before handshake completed + PreauthData, + /// Certificate verification error + CertVerification(rustls::CertificateError), + /// I/O error + Io(std::io::Error), + /// Timeout error (socket.timeout) + Timeout(String), + /// SNI callback triggered - need to restart handshake + SniCallbackRestart, + /// Python exception (pass through directly) + Py(PyBaseExceptionRef), + /// TLS alert received with OpenSSL-compatible error code + AlertReceived { lib: i32, reason: i32 }, + /// NO_SHARED_CIPHER error (OpenSSL SSL_R_NO_SHARED_CIPHER) + NoCipherSuites, +} + +impl SslError { + /// Convert TLS alert code to OpenSSL error reason code + /// OpenSSL uses reason = 1000 + alert_code for TLS alerts + fn alert_to_openssl_reason(alert: rustls::AlertDescription) -> i32 { + // AlertDescription can be converted to u8 via as u8 cast + 1000 + (u8::from(alert) as i32) + } + + /// Convert rustls error to SslError + pub fn from_rustls(err: rustls::Error) -> Self { + match err { + rustls::Error::InvalidCertificate(cert_err) => SslError::CertVerification(cert_err), + rustls::Error::AlertReceived(alert_desc) => { + // Map TLS alerts to OpenSSL-compatible error codes + // lib = 20 (ERR_LIB_SSL), reason = 1000 + alert_code + match alert_desc { + rustls::AlertDescription::CloseNotify => { + // Special case: close_notify is handled as ZeroReturn + SslError::ZeroReturn + } + _ => { + // All other alerts: convert to OpenSSL error code + // This includes InternalError (80 -> reason 1080) + SslError::AlertReceived { + lib: ERR_LIB_SSL, + reason: Self::alert_to_openssl_reason(alert_desc), + } + } + } + } + // OpenSSL 3.0 changed transport EOF from SSL_ERROR_SYSCALL with + // zero return value to SSL_ERROR_SSL with SSL_R_UNEXPECTED_EOF_WHILE_READING. + // In rustls, these cases correspond to unexpected connection closure: + rustls::Error::InvalidMessage(_) => { + // UnexpectedMessage, CorruptMessage, etc. → SSLEOFError + // Matches CPython's "EOF occurred in violation of protocol" + SslError::Eof + } + rustls::Error::PeerIncompatible(peer_err) => { + // Check for specific incompatibility types + use rustls::PeerIncompatible; + match peer_err { + PeerIncompatible::NoCipherSuitesInCommon => { + // Maps to OpenSSL SSL_R_NO_SHARED_CIPHER (lib=20, reason=193) + SslError::NoCipherSuites + } + _ => { + // Other protocol incompatibilities → SSLEOFError + SslError::Eof + } + } + } + _ => SslError::Ssl(format!("{err}")), + } + } + + /// Create SSLError with library and reason from string values + /// + /// This is the base helper for creating SSLError with _library and _reason + /// attributes when you already have the string values. + /// + /// # Arguments + /// * `vm` - Virtual machine reference + /// * `library` - Library name (e.g., "PEM", "SSL") + /// * `reason` - Error reason (e.g., "PEM lib", "NO_SHARED_CIPHER") + /// * `message` - Main error message + /// + /// # Returns + /// PyBaseExceptionRef with _library and _reason attributes set + /// + /// # Note + /// If attribute setting fails (extremely rare), returns the exception without attributes + pub(super) fn create_ssl_error_with_reason( + vm: &VirtualMachine, + library: Option<&str>, + reason: &str, + message: impl Into<String>, + ) -> PyBaseExceptionRef { + let msg = message.into(); + // SSLError args should be (errno, message) format + // FIXME: Use 1 as generic SSL error code + let exc = vm.new_os_subtype_error(PySSLError::class(&vm.ctx).to_owned(), Some(1), msg); + + // Set library and reason attributes + // Ignore errors as they're extremely rare (e.g., out of memory) + let library_obj = match library { + Some(lib) => vm.ctx.new_str(lib).as_object().to_owned(), + None => vm.ctx.none(), + }; + let _ = exc.as_object().set_attr("library", library_obj, vm); + let _ = + exc.as_object() + .set_attr("reason", vm.ctx.new_str(reason).as_object().to_owned(), vm); + + exc.upcast() + } + + /// Create SSLError with library and reason from ssl_data codes + /// + /// This helper converts OpenSSL numeric error codes to Python SSLError exceptions + /// with proper _library and _reason attributes by looking up the error strings + /// in ssl_data tables, then delegates to create_ssl_error_with_reason. + /// + /// # Arguments + /// * `vm` - Virtual machine reference + /// * `lib` - OpenSSL library code (e.g., ERR_LIB_SSL = 20) + /// * `reason` - OpenSSL reason code (e.g., SSL_R_NO_SHARED_CIPHER = 193) + /// + /// # Returns + /// PyBaseExceptionRef with _library and _reason attributes set + fn create_ssl_error_from_codes( + vm: &VirtualMachine, + lib: i32, + reason: i32, + ) -> PyBaseExceptionRef { + // Look up error strings from ssl_data tables + let key = ssl_data::encode_error_key(lib, reason); + let reason_str = ssl_data::ERROR_CODES + .get(&key) + .copied() + .unwrap_or("unknown error"); + + let lib_str = ssl_data::LIBRARY_CODES + .get(&(lib as u32)) + .copied() + .unwrap_or("UNKNOWN"); + + // Delegate to create_ssl_error_with_reason for actual exception creation + Self::create_ssl_error_with_reason( + vm, + Some(lib_str), + reason_str, + format!("[SSL] {reason_str}"), + ) + } + + /// Convert to Python exception + pub fn into_py_err(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + match self { + SslError::WantRead => create_ssl_want_read_error(vm).upcast(), + SslError::WantWrite => create_ssl_want_write_error(vm).upcast(), + SslError::Timeout(msg) => timeout_error_msg(vm, msg).upcast(), + SslError::Syscall(msg) => { + // SSLSyscallError with errno=SSL_ERROR_SYSCALL (5) + create_ssl_syscall_error(vm, msg).upcast() + } + SslError::Ssl(msg) => vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + format!("SSL error: {msg}"), + ) + .upcast(), + SslError::ZeroReturn => create_ssl_zero_return_error(vm).upcast(), + SslError::Eof => create_ssl_eof_error(vm).upcast(), + SslError::PreauthData => { + // Non-TLS data received before handshake + Self::create_ssl_error_with_reason( + vm, + None, + "before TLS handshake with data", + "before TLS handshake with data", + ) + } + SslError::CertVerification(cert_err) => { + // Use the proper cert verification error creator + create_ssl_cert_verification_error(vm, &cert_err).expect("unlikely to happen") + } + SslError::Io(err) => err.into_pyexception(vm), + SslError::SniCallbackRestart => { + // This should be handled at PySSLSocket level + unreachable!("SniCallbackRestart should not reach Python layer") + } + SslError::Py(exc) => exc, + SslError::AlertReceived { lib, reason } => { + Self::create_ssl_error_from_codes(vm, lib, reason) + } + SslError::NoCipherSuites => { + // OpenSSL error: lib=20 (ERR_LIB_SSL), reason=193 (SSL_R_NO_SHARED_CIPHER) + Self::create_ssl_error_from_codes(vm, ERR_LIB_SSL, SSL_R_NO_SHARED_CIPHER) + } + } + } +} + +pub type SslResult<T> = Result<T, SslError>; +/// Common protocol settings shared between client and server connections +#[derive(Debug)] +pub struct ProtocolSettings { + pub versions: &'static [&'static rustls::SupportedProtocolVersion], + pub kx_groups: Option<Vec<&'static dyn rustls::crypto::SupportedKxGroup>>, + pub cipher_suites: Option<Vec<rustls::SupportedCipherSuite>>, + pub alpn_protocols: Vec<Vec<u8>>, +} + +/// Options for creating a server TLS configuration +#[derive(Debug)] +pub struct ServerConfigOptions { + /// Common protocol settings (versions, ALPN, KX groups, cipher suites) + pub protocol_settings: ProtocolSettings, + /// Server certificate chain + pub cert_chain: Vec<CertificateDer<'static>>, + /// Server private key + pub private_key: PrivateKeyDer<'static>, + /// Root certificates for client verification (if required) + pub root_store: Option<RootCertStore>, + /// Whether to request client certificate + pub request_client_cert: bool, + /// Whether to use deferred client certificate validation (TLS 1.3) + pub use_deferred_validation: bool, + /// Custom certificate resolver (for SNI support) + pub cert_resolver: Option<Arc<dyn ResolvesServerCert>>, + /// Deferred certificate error storage (for TLS 1.3) + pub deferred_cert_error: Option<Arc<ParkingRwLock<Option<String>>>>, + /// Session storage for server-side session resumption + pub session_storage: Option<Arc<rustls::server::ServerSessionMemoryCache>>, + /// Shared ticketer for TLS 1.2 session tickets (stateless resumption) + pub ticketer: Option<Arc<dyn rustls::server::ProducesTickets>>, +} + +/// Options for creating a client TLS configuration +#[derive(Debug)] +pub struct ClientConfigOptions { + /// Common protocol settings (versions, ALPN, KX groups, cipher suites) + pub protocol_settings: ProtocolSettings, + /// Root certificates for server verification + pub root_store: Option<RootCertStore>, + /// DER-encoded CA certificates (for partial chain verification) + pub ca_certs_der: Vec<Vec<u8>>, + /// Client certificate chain (for mTLS) + pub cert_chain: Option<Vec<CertificateDer<'static>>>, + /// Client private key (for mTLS) + pub private_key: Option<PrivateKeyDer<'static>>, + /// Whether to verify server certificates (CERT_NONE disables verification) + pub verify_server_cert: bool, + /// Whether to check hostname against certificate (check_hostname) + pub check_hostname: bool, + /// SSL verification flags (e.g., VERIFY_X509_STRICT) + pub verify_flags: i32, + /// Session store for client-side session resumption + pub session_store: Option<Arc<dyn rustls::client::ClientSessionStore>>, + /// Certificate Revocation Lists for CRL checking + pub crls: Vec<CertificateRevocationListDer<'static>>, +} + +/// Create custom CryptoProvider with specified cipher suites and key exchange groups +/// +/// This helper function consolidates the duplicated CryptoProvider creation logic +/// for both server and client configurations. +fn create_custom_crypto_provider( + cipher_suites: Option<Vec<rustls::SupportedCipherSuite>>, + kx_groups: Option<Vec<&'static dyn rustls::crypto::SupportedKxGroup>>, +) -> Arc<rustls::crypto::CryptoProvider> { + use rustls::crypto::aws_lc_rs::{ALL_CIPHER_SUITES, ALL_KX_GROUPS}; + let default_provider = rustls::crypto::aws_lc_rs::default_provider(); + + Arc::new(rustls::crypto::CryptoProvider { + cipher_suites: cipher_suites.unwrap_or_else(|| ALL_CIPHER_SUITES.to_vec()), + kx_groups: kx_groups.unwrap_or_else(|| ALL_KX_GROUPS.to_vec()), + signature_verification_algorithms: default_provider.signature_verification_algorithms, + secure_random: default_provider.secure_random, + key_provider: default_provider.key_provider, + }) +} + +/// Create a server TLS configuration +/// +/// This abstracts the complex rustls ServerConfig building logic, +/// matching SSL_CTX initialization for server sockets. +pub(super) fn create_server_config(options: ServerConfigOptions) -> Result<ServerConfig, String> { + use rustls::server::WebPkiClientVerifier; + + // Ensure default CryptoProvider is installed + ensure_default_provider(); + + // Create custom crypto provider using helper function + let custom_provider = create_custom_crypto_provider( + options.protocol_settings.cipher_suites.clone(), + options.protocol_settings.kx_groups.clone(), + ); + + // Step 1: Build the appropriate client cert verifier based on settings + let client_cert_verifier: Option<Arc<dyn rustls::server::danger::ClientCertVerifier>> = + if let Some(root_store) = options.root_store { + if options.request_client_cert { + // Client certificate verification required + let base_verifier = WebPkiClientVerifier::builder(Arc::new(root_store)) + .build() + .map_err(|e| format!("Failed to create client verifier: {e}"))?; + + if options.use_deferred_validation { + // TLS 1.3: Use deferred validation + if let Some(deferred_error) = options.deferred_cert_error { + use crate::ssl::cert::DeferredClientCertVerifier; + let deferred_verifier = + DeferredClientCertVerifier::new(base_verifier, deferred_error); + Some(Arc::new(deferred_verifier)) + } else { + // No deferred error storage provided, use immediate validation + Some(base_verifier) + } + } else { + // TLS 1.2 or non-deferred: Use immediate validation + Some(base_verifier) + } + } else { + // No client authentication + None + } + } else { + // No root store - no client authentication + None + }; + + // Step 2: Create ServerConfig builder once with the selected verifier + let builder = ServerConfig::builder_with_provider(custom_provider.clone()) + .with_protocol_versions(options.protocol_settings.versions) + .map_err(|e| format!("Failed to create server config builder: {e}"))?; + + let builder = if let Some(verifier) = client_cert_verifier { + builder.with_client_cert_verifier(verifier) + } else { + builder.with_no_client_auth() + }; + + // Add certificate + let mut config = if let Some(resolver) = options.cert_resolver { + // Use custom cert resolver (e.g., for SNI) + builder.with_cert_resolver(resolver) + } else { + // Use single certificate + builder + .with_single_cert(options.cert_chain, options.private_key) + .map_err(|e| format!("Failed to set server certificate: {e}"))? + }; + + // Set ALPN protocols with fallback + apply_alpn_with_fallback( + &mut config.alpn_protocols, + &options.protocol_settings.alpn_protocols, + ); + + // Set session storage for server-side session resumption (TLS 1.3) + if let Some(session_storage) = options.session_storage { + config.session_storage = session_storage; + } + + // Set ticketer for TLS 1.2 session tickets (stateless resumption) + if let Some(ticketer) = options.ticketer { + config.ticketer = ticketer.clone(); + } + + Ok(config) +} + +/// Build WebPki verifier with CRL support +/// +/// This helper function consolidates the duplicated CRL setup logic for both +/// check_hostname=True and check_hostname=False cases. +fn build_webpki_verifier_with_crls( + root_store: Arc<RootCertStore>, + crls: Vec<CertificateRevocationListDer<'static>>, + verify_flags: i32, +) -> Result<Arc<dyn rustls::client::danger::ServerCertVerifier>, String> { + use rustls::client::WebPkiServerVerifier; + + let mut verifier_builder = WebPkiServerVerifier::builder(root_store); + + // Check if CRL verification is requested + let crl_check_requested = verify_flags & X509_V_FLAG_CRL_CHECK != 0; + let has_crls = !crls.is_empty(); + + // Add CRLs if provided OR if CRL checking is explicitly requested + // (even with empty CRLs, rustls will fail verification if CRL checking is enabled) + if has_crls || crl_check_requested { + verifier_builder = verifier_builder.with_crls(crls); + + // Check if we should only verify end-entity (leaf) certificates + if verify_flags & X509_V_FLAG_CRL_CHECK != 0 { + verifier_builder = verifier_builder.only_check_end_entity_revocation(); + } + } + + let webpki_verifier = verifier_builder + .build() + .map_err(|e| format!("Failed to build WebPkiServerVerifier: {e}"))?; + + Ok(webpki_verifier as Arc<dyn rustls::client::danger::ServerCertVerifier>) +} + +/// Apply verifier wrappers (CRLCheckVerifier and StrictCertVerifier) +/// +/// This helper function consolidates the duplicated verifier wrapping logic. +fn apply_verifier_wrappers( + verifier: Arc<dyn rustls::client::danger::ServerCertVerifier>, + verify_flags: i32, + has_crls: bool, + ca_certs_der: Vec<Vec<u8>>, +) -> Arc<dyn rustls::client::danger::ServerCertVerifier> { + let crl_check_requested = verify_flags & X509_V_FLAG_CRL_CHECK != 0; + + // Wrap with CRLCheckVerifier to enforce CRL checking when flags are set + let verifier = if crl_check_requested { + use crate::ssl::cert::CRLCheckVerifier; + Arc::new(CRLCheckVerifier::new( + verifier, + has_crls, + crl_check_requested, + )) + } else { + verifier + }; + + // Always use PartialChainVerifier when trust store is not empty + // This allows self-signed certificates in trust store to be trusted + // (OpenSSL behavior: self-signed certs are always trusted, non-self-signed require flag) + let verifier = if !ca_certs_der.is_empty() { + use crate::ssl::cert::PartialChainVerifier; + Arc::new(PartialChainVerifier::new( + verifier, + ca_certs_der, + verify_flags, + )) + } else { + verifier + }; + + // Wrap with StrictCertVerifier if VERIFY_X509_STRICT flag is set + if verify_flags & VERIFY_X509_STRICT != 0 { + Arc::new(super::cert::StrictCertVerifier::new(verifier, verify_flags)) + } else { + verifier + } +} + +/// Apply ALPN protocols +/// +/// OpenSSL 1.1.0f+ allows ALPN negotiation to fail without aborting handshake. +/// rustls follows RFC 7301 strictly and rejects connections with no matching protocol. +/// To emulate OpenSSL behavior, we add a special fallback protocol (null byte). +fn apply_alpn_with_fallback(config_alpn: &mut Vec<Vec<u8>>, alpn_protocols: &[Vec<u8>]) { + if !alpn_protocols.is_empty() { + *config_alpn = alpn_protocols.to_vec(); + config_alpn.push(vec![0u8]); // Add null byte as fallback marker + } +} + +/// Create a client TLS configuration +/// +/// This abstracts the complex rustls ClientConfig building logic, +/// matching SSL_CTX initialization for client sockets. +pub(super) fn create_client_config(options: ClientConfigOptions) -> Result<ClientConfig, String> { + // Ensure default CryptoProvider is installed + ensure_default_provider(); + + // Create custom crypto provider using helper function + let custom_provider = create_custom_crypto_provider( + options.protocol_settings.cipher_suites.clone(), + options.protocol_settings.kx_groups.clone(), + ); + + // Step 1: Build the appropriate verifier based on verification settings + let verifier: Arc<dyn rustls::client::danger::ServerCertVerifier> = if options + .verify_server_cert + { + // Verify server certificates + let root_store = options + .root_store + .ok_or("Root store required for server verification")?; + + let root_store_arc = Arc::new(root_store); + + // Check if root_store is empty (no CA certs loaded) + // CPython allows this and fails during handshake with SSLCertVerificationError + if root_store_arc.is_empty() { + // Use EmptyRootStoreVerifier - always fails with UnknownIssuer during handshake + use crate::ssl::cert::EmptyRootStoreVerifier; + Arc::new(EmptyRootStoreVerifier) + } else { + // Calculate has_crls once for both hostname verification paths + let has_crls = !options.crls.is_empty(); + + if options.check_hostname { + // Default behavior: verify both certificate chain and hostname + let base_verifier = build_webpki_verifier_with_crls( + root_store_arc.clone(), + options.crls, + options.verify_flags, + )?; + + // Apply CRL and Strict verifier wrappers using helper function + apply_verifier_wrappers( + base_verifier, + options.verify_flags, + has_crls, + options.ca_certs_der.clone(), + ) + } else { + // check_hostname=False: verify certificate chain but ignore hostname + use crate::ssl::cert::HostnameIgnoringVerifier; + + // Build verifier with CRL support using helper function + let webpki_verifier = build_webpki_verifier_with_crls( + root_store_arc.clone(), + options.crls, + options.verify_flags, + )?; + + // Apply CRL verifier wrapper if needed (without Strict wrapper yet) + let crl_check_requested = options.verify_flags & X509_V_FLAG_CRL_CHECK != 0; + let verifier = if crl_check_requested { + use crate::ssl::cert::CRLCheckVerifier; + Arc::new(CRLCheckVerifier::new( + webpki_verifier, + has_crls, + crl_check_requested, + )) as Arc<dyn rustls::client::danger::ServerCertVerifier> + } else { + webpki_verifier + }; + + // Wrap with PartialChainVerifier if VERIFY_X509_PARTIAL_CHAIN is set + const VERIFY_X509_PARTIAL_CHAIN: i32 = 0x80000; + let verifier = if options.verify_flags & VERIFY_X509_PARTIAL_CHAIN != 0 { + use crate::ssl::cert::PartialChainVerifier; + Arc::new(PartialChainVerifier::new( + verifier, + options.ca_certs_der.clone(), + options.verify_flags, + )) as Arc<dyn rustls::client::danger::ServerCertVerifier> + } else { + verifier + }; + + // Wrap with HostnameIgnoringVerifier to bypass hostname checking + let hostname_ignoring_verifier: Arc< + dyn rustls::client::danger::ServerCertVerifier, + > = Arc::new(HostnameIgnoringVerifier::new_with_verifier(verifier)); + + // Apply Strict verifier wrapper once at the end if needed + if options.verify_flags & VERIFY_X509_STRICT != 0 { + Arc::new(crate::ssl::cert::StrictCertVerifier::new( + hostname_ignoring_verifier, + options.verify_flags, + )) + } else { + hostname_ignoring_verifier + } + } + } + } else { + // CERT_NONE: disable all verification + use crate::ssl::cert::NoVerifier; + Arc::new(NoVerifier) + }; + + // Step 2: Create ClientConfig builder once with the selected verifier + let builder = ClientConfig::builder_with_provider(custom_provider.clone()) + .with_protocol_versions(options.protocol_settings.versions) + .map_err(|e| format!("Failed to create client config builder: {e}"))? + .dangerous() + .with_custom_certificate_verifier(verifier); + + // Add client certificate if provided (mTLS) + let mut config = + if let (Some(cert_chain), Some(private_key)) = (options.cert_chain, options.private_key) { + builder + .with_client_auth_cert(cert_chain, private_key) + .map_err(|e| format!("Failed to set client certificate: {e}"))? + } else { + builder.with_no_client_auth() + }; + + // Set ALPN protocols + apply_alpn_with_fallback( + &mut config.alpn_protocols, + &options.protocol_settings.alpn_protocols, + ); + + // Set session resumption + if let Some(session_store) = options.session_store { + use rustls::client::Resumption; + config.resumption = Resumption::store(session_store); + } + + Ok(config) +} + +/// Helper function - check if error is BlockingIOError +pub(super) fn is_blocking_io_error(err: &Py<PyBaseException>, vm: &VirtualMachine) -> bool { + err.fast_isinstance(vm.ctx.exceptions.blocking_io_error) +} + +// Socket I/O Helper Functions + +/// Send all bytes to socket, handling partial sends with blocking wait +/// +/// Loops until all bytes are sent. For blocking sockets, this will wait +/// until all data is sent. For non-blocking sockets, returns WantWrite +/// if no progress can be made. +/// Optional deadline parameter allows respecting a read deadline during flush. +fn send_all_bytes( + socket: &PySSLSocket, + buf: Vec<u8>, + vm: &VirtualMachine, + deadline: Option<std::time::Instant>, +) -> SslResult<()> { + // First, flush any previously pending TLS data with deadline + socket + .flush_pending_tls_output(vm, deadline) + .map_err(SslError::Py)?; + + if buf.is_empty() { + return Ok(()); + } + + let mut sent_total = 0; + while sent_total < buf.len() { + // Check deadline before each send attempt + if let Some(dl) = deadline + && std::time::Instant::now() >= dl + { + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::Timeout("The operation timed out".to_string())); + } + + // Wait for socket to be writable before sending + let timed_out = if let Some(dl) = deadline { + let now = std::time::Instant::now(); + if now >= dl { + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::Timeout( + "The write operation timed out".to_string(), + )); + } + socket + .sock_wait_for_io_with_timeout(SelectKind::Write, Some(dl - now), vm) + .map_err(SslError::Py)? + } else { + socket + .sock_wait_for_io_impl(SelectKind::Write, vm) + .map_err(SslError::Py)? + }; + if timed_out { + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::Timeout( + "The write operation timed out".to_string(), + )); + } + + match socket.sock_send(&buf[sent_total..], vm) { + Ok(result) => { + let sent: usize = result + .try_to_value::<isize>(vm) + .map_err(SslError::Py)? + .try_into() + .map_err(|_| SslError::Syscall("Invalid send return value".to_string()))?; + if sent == 0 { + // No progress - save unsent bytes to pending buffer + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::WantWrite); + } + sent_total += sent; + } + Err(e) => { + if is_blocking_io_error(&e, vm) { + // Save unsent bytes to pending buffer + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::WantWrite); + } + // For other errors, also save unsent bytes + socket + .pending_tls_output + .lock() + .extend_from_slice(&buf[sent_total..]); + return Err(SslError::Py(e)); + } + } + } + Ok(()) +} + +// Handshake Helper Functions + +/// Write TLS handshake data to socket/BIO +/// +/// Drains all pending TLS data from rustls and sends it to the peer. +/// Returns whether any progress was made. +fn handshake_write_loop( + conn: &mut TlsConnection, + socket: &PySSLSocket, + force_initial_write: bool, + vm: &VirtualMachine, +) -> SslResult<bool> { + let mut made_progress = false; + + // Flush any previously pending TLS data before generating new output + // Must succeed before sending new data to maintain order + socket + .flush_pending_tls_output(vm, None) + .map_err(SslError::Py)?; + + while conn.wants_write() || force_initial_write { + if force_initial_write && !conn.wants_write() { + // No data to write on first iteration - break to avoid infinite loop + break; + } + + let mut buf = Vec::new(); + let written = conn + .write_tls(&mut buf as &mut dyn std::io::Write) + .map_err(SslError::Io)?; + + if written > 0 && !buf.is_empty() { + // Send all bytes to socket, handling partial sends + send_all_bytes(socket, buf, vm, None)?; + made_progress = true; + } else if written == 0 { + // No data written but wants_write is true - should not happen normally + // Break to avoid infinite loop + break; + } + + // Check if there's more to write + if !conn.wants_write() { + break; + } + } + + Ok(made_progress) +} + +/// Read TLS handshake data from socket/BIO +/// +/// Waits for and reads TLS records from the peer, handling SNI callback setup. +/// Returns (made_progress, is_first_sni_read). +/// TLS record header size (content_type + version + length). +const TLS_RECORD_HEADER_SIZE: usize = 5; + +/// Read exactly one TLS record from the TCP socket. +/// +/// OpenSSL reads one TLS record at a time (no read-ahead by default). +/// Rustls, however, consumes all available TCP data when fed via read_tls(). +/// If a close_notify or other control record arrives alongside application +/// data, the eager read drains the TCP buffer, leaving the control record in +/// rustls's internal buffer where select() cannot see it. This causes +/// asyncore-based servers (which rely on select() for readability) to miss +/// the data and the peer times out. +/// +/// Fix: peek at the TCP buffer to find the first complete TLS record boundary +/// and recv() only that many bytes. Any remaining data stays in the kernel +/// buffer and remains visible to select(). +fn recv_one_tls_record(socket: &PySSLSocket, vm: &VirtualMachine) -> SslResult<PyObjectRef> { + // Peek at what is available without consuming it. + let peeked_obj = match socket.sock_peek(SSL3_RT_MAX_PLAIN_LENGTH, vm) { + Ok(d) => d, + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Err(SslError::WantRead); + } + return Err(SslError::Py(e)); + } + }; + + let peeked = ArgBytesLike::try_from_object(vm, peeked_obj) + .map_err(|_| SslError::Syscall("Expected bytes-like object from peek".to_string()))?; + let peeked_bytes = peeked.borrow_buf(); + + if peeked_bytes.is_empty() { + // Empty peek means the peer has closed the TCP connection (FIN). + // Non-blocking sockets would have returned EAGAIN/EWOULDBLOCK + // (caught above as WantRead), so empty bytes here always means EOF. + return Err(SslError::Eof); + } + + if peeked_bytes.len() < TLS_RECORD_HEADER_SIZE { + // Not enough data for a TLS record header yet. + // Read all available bytes so rustls can buffer the partial header; + // this avoids busy-waiting because the kernel buffer is now empty + // and select() will only wake us when new data arrives. + return socket.sock_recv(peeked_bytes.len(), vm).map_err(|e| { + if is_blocking_io_error(&e, vm) { + SslError::WantRead + } else { + SslError::Py(e) + } + }); + } + + // Parse the TLS record length from the header. + let record_body_len = u16::from_be_bytes([peeked_bytes[3], peeked_bytes[4]]) as usize; + let total_record_size = TLS_RECORD_HEADER_SIZE + record_body_len; + + let recv_size = if peeked_bytes.len() >= total_record_size { + // Complete record available — consume exactly one record. + total_record_size + } else { + // Incomplete record — consume everything so the kernel buffer is + // drained and select() will block until more data arrives. + peeked_bytes.len() + }; + + // Must drop the borrow before calling sock_recv (which re-enters Python). + drop(peeked_bytes); + drop(peeked); + + socket.sock_recv(recv_size, vm).map_err(|e| { + if is_blocking_io_error(&e, vm) { + SslError::WantRead + } else { + SslError::Py(e) + } + }) +} + +/// Read a single TLS record for post-handshake I/O while preserving the +/// SSL-vs-socket error precedence from the old sock_recv() path. +fn recv_one_tls_record_for_data( + conn: &mut TlsConnection, + socket: &PySSLSocket, + vm: &VirtualMachine, +) -> SslResult<PyObjectRef> { + match recv_one_tls_record(socket, vm) { + Ok(data) => Ok(data), + Err(SslError::Eof) => { + if let Err(rustls_err) = conn.process_new_packets() { + return Err(SslError::from_rustls(rustls_err)); + } + Ok(vm.ctx.new_bytes(vec![]).into()) + } + Err(SslError::Py(e)) => { + if let Err(rustls_err) = conn.process_new_packets() { + return Err(SslError::from_rustls(rustls_err)); + } + if is_connection_closed_error(&e, vm) { + return Err(SslError::Eof); + } + Err(SslError::Py(e)) + } + Err(e) => Err(e), + } +} + +fn handshake_read_data( + conn: &mut TlsConnection, + socket: &PySSLSocket, + is_bio: bool, + is_server: bool, + vm: &VirtualMachine, +) -> SslResult<(bool, bool)> { + if !conn.wants_read() { + return Ok((false, false)); + } + + // SERVER-SPECIFIC: Check if this is the first read (for SNI callback) + // Must check BEFORE reading data, so we can detect first time + let is_first_sni_read = is_server && socket.is_first_sni_read(); + + // Wait for data in socket mode + if !is_bio { + let timed_out = socket + .sock_wait_for_io_impl(SelectKind::Read, vm) + .map_err(SslError::Py)?; + + if timed_out { + // This should rarely happen now - only if socket itself has a timeout + // and we're waiting for required handshake data + return Err(SslError::Timeout("timed out".to_string())); + } + } + + let data_obj = if !is_bio { + // In socket mode, read one TLS record at a time to avoid consuming + // application data that may arrive alongside the final handshake + // record. This matches OpenSSL's default (no read-ahead) behaviour + // and keeps remaining data in the kernel buffer where select() can + // detect it. + recv_one_tls_record(socket, vm)? + } else { + match socket.sock_recv(SSL3_RT_MAX_PLAIN_LENGTH, vm) { + Ok(d) => d, + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Err(SslError::WantRead); + } + if !conn.wants_write() && e.fast_isinstance(vm.ctx.exceptions.timeout_error) { + return Ok((false, false)); + } + return Err(SslError::Py(e)); + } + } + }; + + // SERVER-SPECIFIC: Save ClientHello on first read for potential connection recreation + if is_first_sni_read { + // Extract bytes from PyObjectRef + use rustpython_vm::builtins::PyBytes; + if let Some(bytes_obj) = data_obj.downcast_ref::<PyBytes>() { + socket.save_client_hello_from_bytes(bytes_obj.as_bytes()); + } + } + + // Feed data to rustls + ssl_read_tls_records(conn, data_obj, is_bio, vm)?; + + Ok((true, is_first_sni_read)) +} + +/// Handle handshake completion for server-side TLS 1.3 +/// +/// Tries to send NewSessionTicket in non-blocking mode to avoid deadlocks. +/// Returns true if handshake is complete and we should exit. +fn handle_handshake_complete( + conn: &mut TlsConnection, + socket: &PySSLSocket, + _is_server: bool, + vm: &VirtualMachine, +) -> SslResult<bool> { + if conn.is_handshaking() { + return Ok(false); // Not complete yet + } + + // Handshake is complete! + // + // Different behavior for BIO mode vs socket mode: + // + // BIO mode (CPython-compatible): + // - Python code calls outgoing.read() to get pending data + // - We just return here and let Python handle the data + // + // Socket mode (rustls-specific): + // - OpenSSL automatically writes to socket in SSL_do_handshake() + // - We must explicitly call write_tls() to send pending data + // - Without this, client hangs waiting for server's NewSessionTicket + + if socket.is_bio_mode() { + // BIO mode: Write pending data to outgoing BIO (one-time drain) + // Python's ssl_io_loop will read from outgoing BIO + if conn.wants_write() { + // Call write_tls ONCE to drain pending data + // Do NOT loop on wants_write() - avoid infinite loop/deadlock + let tls_data = ssl_write_tls_records(conn)?; + if !tls_data.is_empty() { + send_all_bytes(socket, tls_data, vm, None)?; + } + + // IMPORTANT: Don't check wants_write() again! + // Python's ssl_io_loop will call do_handshake() again if needed + } + } else if conn.wants_write() { + // Send all pending data (e.g., TLS 1.3 NewSessionTicket) to socket + // Must drain ALL rustls buffer - don't break on WantWrite + while conn.wants_write() { + let tls_data = ssl_write_tls_records(conn)?; + if tls_data.is_empty() { + break; + } + match send_all_bytes(socket, tls_data, vm, None) { + Ok(()) => {} + Err(SslError::WantWrite) => { + // Socket buffer full, data saved to pending_tls_output + // Flush pending and continue draining rustls buffer + socket + .blocking_flush_all_pending(vm) + .map_err(SslError::Py)?; + } + Err(e) => return Err(e), + } + } + } + + // CRITICAL: Ensure all pending TLS data is sent before returning + // TLS 1.3 Finished must reach server before handshake is considered complete + // Without this, server may not process application data + if !socket.is_bio_mode() { + // Flush pending_tls_output to ensure all TLS data reaches the server + socket + .blocking_flush_all_pending(vm) + .map_err(SslError::Py)?; + } + + Ok(true) +} + +/// Try to read plaintext data from TLS connection buffer +/// +/// Returns Ok(Some(n)) if n bytes were read, Ok(None) if would block, +/// or Err on real errors. +fn try_read_plaintext(conn: &mut TlsConnection, buf: &mut [u8]) -> SslResult<Option<usize>> { + let mut reader = conn.reader(); + match reader.read(buf) { + Ok(0) => { + // EOF from TLS connection + Ok(Some(0)) + } + Ok(n) => { + // Successfully read n bytes + Ok(Some(n)) + } + Err(e) if e.kind() != std::io::ErrorKind::WouldBlock => { + // Real error + Err(SslError::Io(e)) + } + Err(_) => { + // WouldBlock - no plaintext available + Ok(None) + } + } +} + +/// Equivalent to OpenSSL's SSL_do_handshake() +/// +/// Performs TLS handshake by exchanging data with the peer until completion. +/// This abstracts away the low-level rustls read_tls/write_tls loop. +/// +/// = SSL_do_handshake() +pub(super) fn ssl_do_handshake( + conn: &mut TlsConnection, + socket: &PySSLSocket, + vm: &VirtualMachine, +) -> SslResult<()> { + // Check if handshake is already done + if !conn.is_handshaking() { + return Ok(()); + } + + let is_bio = socket.is_bio_mode(); + let is_server = matches!(conn, TlsConnection::Server(_)); + let mut first_iteration = true; // Track if this is the first loop iteration + let mut iteration_count = 0; + + loop { + iteration_count += 1; + let mut made_progress = false; + + // IMPORTANT: In BIO mode, force initial write even if wants_write() is false + // rustls requires write_tls() to be called to generate ClientHello/ServerHello + let force_initial_write = is_bio && first_iteration; + + // Write TLS handshake data to socket/BIO + let write_progress = handshake_write_loop(conn, socket, force_initial_write, vm)?; + made_progress |= write_progress; + + // Read TLS handshake data from socket/BIO + let (read_progress, is_first_sni_read) = + handshake_read_data(conn, socket, is_bio, is_server, vm)?; + made_progress |= read_progress; + + // Process TLS packets (state machine) + if let Err(e) = conn.process_new_packets() { + // Send close_notify on error + if !is_bio { + conn.send_close_notify(); + // Flush any pending TLS data before sending close_notify + let _ = socket.flush_pending_tls_output(vm, None); + // Actually send the close_notify alert using send_all_bytes + // for proper partial send handling and retry logic + if let Ok(alert_data) = ssl_write_tls_records(conn) + && !alert_data.is_empty() + { + let _ = send_all_bytes(socket, alert_data, vm, None); + } + } + + // InvalidMessage during handshake means non-TLS data was received + // before the handshake completed (e.g., HTTP request to TLS server) + if matches!(e, rustls::Error::InvalidMessage(_)) { + return Err(SslError::PreauthData); + } + + // Certificate verification errors are already handled by from_rustls + + return Err(SslError::from_rustls(e)); + } + + // SERVER-SPECIFIC: Check SNI callback after processing packets + // SNI name is extracted during process_new_packets() + // Invoke callback on FIRST read if callback is configured, regardless of SNI presence + if is_server && is_first_sni_read && socket.has_sni_callback() { + // IMPORTANT: Do NOT call the callback here! + // The connection lock is still held, which would cause deadlock. + // Return SniCallbackRestart to signal do_handshake to: + // 1. Drop conn_guard + // 2. Call the callback (with Some(name) or None) + // 3. Restart handshake + return Err(SslError::SniCallbackRestart); + } + + // Check if handshake is complete and handle post-handshake processing + // CRITICAL: We do NOT check wants_read() - this matches CPython/OpenSSL behavior! + if handle_handshake_complete(conn, socket, is_server, vm)? { + return Ok(()); + } + + // In BIO mode, stop after one iteration + if is_bio { + // Before returning WANT error, write any pending TLS data to BIO + // This is critical: if wants_write is true after process_new_packets, + // we need to write that data to the outgoing BIO before returning + if conn.wants_write() { + // Write all pending TLS data to outgoing BIO + loop { + let mut buf = vec![0u8; SSL3_RT_MAX_PLAIN_LENGTH]; + let n = match conn.write_tls(&mut buf.as_mut_slice()) { + Ok(n) => n, + Err(_) => break, + }; + if n == 0 { + break; + } + // Send to outgoing BIO + send_all_bytes(socket, buf[..n].to_vec(), vm, None)?; + // Check if there's more to write + if !conn.wants_write() { + break; + } + } + // After writing, check if we still want more + // If all data was written, wants_write may now be false + if conn.wants_write() { + // Still need more - return WANT_WRITE + return Err(SslError::WantWrite); + } + // Otherwise fall through to check wants_read + } + + // Check if we need to read + if conn.wants_read() { + return Err(SslError::WantRead); + } + break; + } + + // Mark that we've completed the first iteration + first_iteration = false; + + // Improved loop termination logic: + // Continue looping if: + // 1. Rustls wants more I/O (wants_read or wants_write), OR + // 2. We made progress in this iteration + // + // This is more robust than just checking made_progress, because: + // - Rustls may need multiple iterations to process TLS state machine + // - Network delays may cause temporary "no progress" situations + // - wants_read/wants_write accurately reflect Rustls internal state + let should_continue = conn.wants_read() || conn.wants_write() || made_progress; + + if !should_continue { + break; + } + + // Safety check: prevent truly infinite loops (should never happen) + if iteration_count > 1000 { + break; + } + } + + // If we exit the loop without completing handshake, return appropriate error + if conn.is_handshaking() { + // For non-blocking sockets, return WantRead/WantWrite to signal caller + // should retry when socket is ready. This matches OpenSSL behavior. + if conn.wants_write() { + return Err(SslError::WantWrite); + } + if conn.wants_read() { + return Err(SslError::WantRead); + } + // Neither wants_read nor wants_write - this is a real error + Err(SslError::Syscall(format!( + "SSL handshake failed: incomplete after {iteration_count} iterations", + ))) + } else { + // Handshake completed successfully (shouldn't reach here normally) + Ok(()) + } +} + +/// Equivalent to OpenSSL's SSL_read() +/// +/// Reads application data from TLS connection. +/// Automatically handles TLS record I/O as needed. +/// +/// = SSL_read_ex() +pub(super) fn ssl_read( + conn: &mut TlsConnection, + buf: &mut [u8], + socket: &PySSLSocket, + vm: &VirtualMachine, +) -> SslResult<usize> { + let is_bio = socket.is_bio_mode(); + + // Get socket timeout and calculate deadline (= _PyDeadline_Init) + let deadline = if !is_bio { + match socket.get_socket_timeout(vm).map_err(SslError::Py)? { + Some(timeout) if !timeout.is_zero() => Some(std::time::Instant::now() + timeout), + _ => None, // None = blocking (no deadline), Some(0) = non-blocking (handled below) + } + } else { + None // BIO mode has no deadline + }; + + // CRITICAL: Flush any pending TLS output before reading + // This ensures data from previous write() calls is sent before we wait for response. + // Without this, write() may leave data in pending_tls_output (if socket buffer was full), + // and read() would timeout waiting for a response that the server never received. + if !is_bio { + socket + .flush_pending_tls_output(vm, deadline) + .map_err(SslError::Py)?; + } + + // Loop to handle TLS records and post-handshake messages + // Matches SSL_read behavior which loops until data is available + // - CPython uses OpenSSL's SSL_read which loops on SSL_ERROR_WANT_READ/WANT_WRITE + // - We use rustls which requires manual read_tls/process_new_packets loop + // - No iteration limit: relies on deadline and blocking I/O + // - Blocking sockets: sock_select() and recv() wait at kernel level (no CPU busy-wait) + // - Non-blocking sockets: immediate return on first WantRead + // - Deadline prevents timeout issues + + loop { + // Check deadline + if let Some(deadline) = deadline + && std::time::Instant::now() >= deadline + { + // Timeout expired + return Err(SslError::Timeout( + "The read operation timed out".to_string(), + )); + } + // Check if we need to read more TLS records BEFORE trying plaintext read + // This ensures we don't miss data that's already been processed + let needs_more_tls = conn.wants_read(); + + // Try to read plaintext from rustls buffer + if let Some(n) = try_read_plaintext(conn, buf)? { + if n == 0 { + // EOF from TLS - close_notify received + // Return ZeroReturn so Python raises SSLZeroReturnError + return Err(SslError::ZeroReturn); + } + return Ok(n); + } + + // No plaintext available and rustls doesn't want to read more TLS records + if !needs_more_tls { + // Check if connection needs to write data first (e.g., TLS key update, renegotiation) + // This mirrors the handshake logic which checks both wants_read() and wants_write() + if conn.wants_write() && !is_bio { + // Check deadline BEFORE attempting flush + if let Some(deadline) = deadline + && std::time::Instant::now() >= deadline + { + return Err(SslError::Timeout( + "The read operation timed out".to_string(), + )); + } + + // Flush pending TLS data before continuing + // CRITICAL: Pass deadline so flush respects read timeout + let tls_data = ssl_write_tls_records(conn)?; + if !tls_data.is_empty() { + // Use best-effort send - don't fail READ just because WRITE couldn't complete + match send_all_bytes(socket, tls_data, vm, deadline) { + Ok(()) => {} + Err(SslError::WantWrite) => { + // Socket buffer full - acceptable during READ operation + // Pending data will be sent on next write/read call + } + Err(SslError::Timeout(_)) => { + // Timeout during flush is acceptable during READ + // Pending data stays buffered for next operation + } + Err(e) => return Err(e), + } + } + + // Check deadline AFTER flush attempt + if let Some(deadline) = deadline + && std::time::Instant::now() >= deadline + { + return Err(SslError::Timeout( + "The read operation timed out".to_string(), + )); + } + + // After flushing, rustls may want to read again - continue loop + continue; + } + + // BIO mode: check for EOF + if is_bio && let Some(bio_obj) = socket.incoming_bio() { + let is_eof = bio_obj + .get_attr("eof", vm) + .and_then(|v| v.try_into_value::<bool>(vm)) + .unwrap_or(false); + if is_eof { + return Err(SslError::Eof); + } + } + + // For non-blocking sockets, return WantRead so caller can poll and retry. + // For blocking sockets (or sockets with timeout), wait for more data. + if !is_bio { + let timeout = socket.get_socket_timeout(vm).map_err(SslError::Py)?; + if let Some(t) = timeout + && t.is_zero() + { + // Non-blocking socket: check if peer has closed before returning WantRead + // If close_notify was received, we should return ZeroReturn (EOF), not WantRead + // This is critical for asyncore-based applications that rely on recv() returning + // 0 or raising SSL_ERROR_ZERO_RETURN to detect connection close. + let io_state = conn.process_new_packets().map_err(SslError::from_rustls)?; + if io_state.peer_has_closed() { + return Err(SslError::ZeroReturn); + } + // Non-blocking socket: return immediately + return Err(SslError::WantRead); + } + // Blocking socket or socket with timeout: try to read more data from socket. + // Even though rustls says it doesn't want to read, more TLS records may arrive. + // Use single-record reading to avoid consuming close_notify alongside data. + let data = recv_one_tls_record_for_data(conn, socket, vm)?; + + let bytes_read = data + .clone() + .try_into_value::<rustpython_vm::builtins::PyBytes>(vm) + .map(|b| b.as_bytes().len()) + .unwrap_or(0); + + if bytes_read == 0 { + // No more data available - check if this is clean shutdown or unexpected EOF + // If close_notify was already received, return ZeroReturn (clean closure) + // Otherwise, return Eof (unexpected EOF) + let io_state = conn.process_new_packets().map_err(SslError::from_rustls)?; + if io_state.peer_has_closed() { + return Err(SslError::ZeroReturn); + } + return Err(SslError::Eof); + } + + // Feed data to rustls and process + ssl_read_tls_records(conn, data, false, vm)?; + conn.process_new_packets().map_err(SslError::from_rustls)?; + + // Continue loop to try reading plaintext + continue; + } + + return Err(SslError::WantRead); + } + + // Read and process TLS records + match ssl_ensure_data_available(conn, socket, vm) { + Ok(_bytes_read) => { + // Successfully read and processed TLS data + // Continue loop to try reading plaintext + } + Err(SslError::Io(ref io_err)) if io_err.to_string().contains("message buffer full") => { + // This case should be rare now that ssl_read_tls_records handles buffer full + // Just continue loop to try again + continue; + } + Err(e) => { + // Other errors - check for buffered plaintext before propagating + match try_read_plaintext(conn, buf)? { + Some(n) if n > 0 => { + // Have buffered plaintext - return it successfully + return Ok(n); + } + _ => { + // No buffered data - propagate the error + return Err(e); + } + } + } + } + } +} + +/// Equivalent to OpenSSL's SSL_write() +/// +/// Writes application data to TLS connection. +/// Automatically handles TLS record I/O as needed. +/// +/// = SSL_write_ex() +pub(super) fn ssl_write( + conn: &mut TlsConnection, + data: &[u8], + socket: &PySSLSocket, + vm: &VirtualMachine, +) -> SslResult<usize> { + if data.is_empty() { + return Ok(0); + } + + let is_bio = socket.is_bio_mode(); + + // Get socket timeout and calculate deadline (= _PyDeadline_Init) + let deadline = if !is_bio { + match socket.get_socket_timeout(vm).map_err(SslError::Py)? { + Some(timeout) if !timeout.is_zero() => Some(std::time::Instant::now() + timeout), + _ => None, + } + } else { + None + }; + + // Flush any pending TLS output before writing new data + if !is_bio { + socket + .flush_pending_tls_output(vm, deadline) + .map_err(SslError::Py)?; + } + + // Check if we already have data buffered from a previous retry + // (prevents duplicate writes when retrying after WantWrite/WantRead) + let already_buffered = *socket.write_buffered_len.lock(); + + // Only write plaintext if not already buffered + // Track how much we wrote for partial write handling + let mut bytes_written_to_rustls = 0usize; + + if already_buffered == 0 { + // Write plaintext to rustls (= SSL_write_ex internal buffer write) + bytes_written_to_rustls = { + let mut writer = conn.writer(); + use std::io::Write; + // Use write() instead of write_all() to support partial writes. + // In BIO mode (asyncio), when the internal buffer is full, + // we want to write as much as possible and return that count, + // rather than failing completely. + match writer.write(data) { + Ok(0) if !data.is_empty() => { + // Buffer is full and nothing could be written. + // In BIO mode, return WantWrite so the caller can + // drain the outgoing BIO and retry. + if is_bio { + return Err(SslError::WantWrite); + } + return Err(SslError::Syscall("Write failed: buffer full".to_string())); + } + Ok(n) => n, + Err(e) => { + if is_bio { + // In BIO mode, treat write errors as WantWrite + return Err(SslError::WantWrite); + } + return Err(SslError::Syscall(format!("Write failed: {e}"))); + } + } + }; + // Mark data as buffered (only the portion we actually wrote) + *socket.write_buffered_len.lock() = bytes_written_to_rustls; + } else if already_buffered != data.len() { + // Caller is retrying with different data - this is a protocol error + // Clear the buffer state and return an SSL error (bad write retry) + *socket.write_buffered_len.lock() = 0; + return Err(SslError::Ssl("bad write retry".to_string())); + } + // else: already_buffered == data.len(), this is a valid retry + + // Loop to send TLS records, handling WANT_READ/WANT_WRITE + // Matches CPython's do-while loop on SSL_ERROR_WANT_READ/WANT_WRITE + loop { + // Check deadline + if let Some(dl) = deadline + && std::time::Instant::now() >= dl + { + return Err(SslError::Timeout( + "The write operation timed out".to_string(), + )); + } + + // Check if rustls has TLS data to send + if !conn.wants_write() { + // All TLS data sent successfully + break; + } + + // Get TLS records from rustls + let tls_data = ssl_write_tls_records(conn)?; + if tls_data.is_empty() { + break; + } + + // Send TLS data to socket + match send_all_bytes(socket, tls_data, vm, deadline) { + Ok(()) => { + // Successfully sent, continue loop to check for more data + } + Err(SslError::WantWrite) => { + // Non-blocking socket would block - return WANT_WRITE + // If we had a partial write to rustls, return partial success + // instead of error to match OpenSSL partial-write semantics + if bytes_written_to_rustls > 0 && bytes_written_to_rustls < data.len() { + *socket.write_buffered_len.lock() = 0; + return Ok(bytes_written_to_rustls); + } + // Keep write_buffered_len set so we don't re-buffer on retry + return Err(SslError::WantWrite); + } + Err(SslError::WantRead) => { + // Need to read before write can complete (e.g., renegotiation) + if is_bio { + // If we had a partial write to rustls, return partial success + if bytes_written_to_rustls > 0 && bytes_written_to_rustls < data.len() { + *socket.write_buffered_len.lock() = 0; + return Ok(bytes_written_to_rustls); + } + // Keep write_buffered_len set so we don't re-buffer on retry + return Err(SslError::WantRead); + } + // For socket mode, try to read TLS data + let recv_result = socket.sock_recv(4096, vm).map_err(SslError::Py)?; + ssl_read_tls_records(conn, recv_result, false, vm)?; + conn.process_new_packets().map_err(SslError::from_rustls)?; + // Continue loop + } + Err(e @ SslError::Timeout(_)) => { + // If we had a partial write to rustls, return partial success + if bytes_written_to_rustls > 0 && bytes_written_to_rustls < data.len() { + *socket.write_buffered_len.lock() = 0; + return Ok(bytes_written_to_rustls); + } + // Preserve buffered state so retry doesn't duplicate data + // (send_all_bytes saved unsent TLS bytes to pending_tls_output) + return Err(e); + } + Err(e) => { + // Clear buffer state on error + *socket.write_buffered_len.lock() = 0; + return Err(e); + } + } + } + + // Final flush to ensure all data is sent + if !is_bio { + socket + .flush_pending_tls_output(vm, deadline) + .map_err(SslError::Py)?; + } + + // Determine how many bytes we actually wrote + let actual_written = if bytes_written_to_rustls > 0 { + // Fresh write: return what we wrote to rustls + bytes_written_to_rustls + } else if already_buffered > 0 { + // Retry of previous write: return the full buffered amount + already_buffered + } else { + data.len() + }; + + // Write completed successfully - clear buffer state + *socket.write_buffered_len.lock() = 0; + + Ok(actual_written) +} + +// Helper functions (private-ish, used by public SSL functions) + +/// Write TLS records from rustls to socket +fn ssl_write_tls_records(conn: &mut TlsConnection) -> SslResult<Vec<u8>> { + let mut buf = Vec::new(); + let n = conn + .write_tls(&mut buf as &mut dyn std::io::Write) + .map_err(SslError::Io)?; + + if n > 0 { Ok(buf) } else { Ok(Vec::new()) } +} + +/// Read TLS records from socket to rustls +fn ssl_read_tls_records( + conn: &mut TlsConnection, + data: PyObjectRef, + is_bio: bool, + vm: &VirtualMachine, +) -> SslResult<()> { + // Convert PyObject to bytes-like (supports bytes, bytearray, etc.) + let bytes = ArgBytesLike::try_from_object(vm, data) + .map_err(|_| SslError::Syscall("Expected bytes-like object".to_string()))?; + + let bytes_data = bytes.borrow_buf(); + + if bytes_data.is_empty() { + // different error for BIO vs socket mode + if is_bio { + // In BIO mode, no data means WANT_READ + return Err(SslError::WantRead); + } else { + // In socket mode, empty recv() means TCP EOF (FIN received) + // Need to distinguish: + // 1. Clean shutdown: received TLS close_notify → return ZeroReturn (0 bytes) + // 2. Unexpected EOF: no close_notify → return Eof (SSLEOFError) + // + // SSL_ERROR_ZERO_RETURN vs SSL_ERROR_EOF logic + // CPython checks SSL_get_shutdown() & SSL_RECEIVED_SHUTDOWN + // + // Process any buffered TLS records (may contain close_notify) + match conn.process_new_packets() { + Ok(io_state) => { + if io_state.peer_has_closed() { + // Received close_notify - normal SSL closure (SSL_ERROR_ZERO_RETURN) + return Err(SslError::ZeroReturn); + } else { + // No close_notify - ragged EOF (SSL_ERROR_EOF → SSLEOFError) + // CPython raises SSLEOFError here, which SSLSocket.read() handles + // based on suppress_ragged_eofs setting + return Err(SslError::Eof); + } + } + Err(e) => return Err(SslError::from_rustls(e)), + } + } + } + + // Feed all received data to read_tls - loop to consume all data + // read_tls may not consume all data in one call, and buffer may become full + let mut offset = 0; + while offset < bytes_data.len() { + let remaining = &bytes_data[offset..]; + let mut cursor = std::io::Cursor::new(remaining); + + match conn.read_tls(&mut cursor) { + Ok(read_bytes) => { + if read_bytes == 0 { + // Buffer is full - process existing packets to make room + conn.process_new_packets().map_err(SslError::from_rustls)?; + + // Try again - if we still can't consume, break + let mut retry_cursor = std::io::Cursor::new(remaining); + match conn.read_tls(&mut retry_cursor) { + Ok(0) => { + // Still can't consume - break to avoid infinite loop + break; + } + Ok(n) => { + offset += n; + } + Err(e) => { + return Err(SslError::Io(e)); + } + } + } else { + offset += read_bytes; + } + } + Err(e) => { + // Check if it's a buffer full error (unlikely but handle it) + if e.to_string().contains("buffer full") { + conn.process_new_packets().map_err(SslError::from_rustls)?; + continue; + } + // Real error - propagate it + return Err(SslError::Io(e)); + } + } + } + + Ok(()) +} + +/// Check if an exception is a connection closed error +/// In SSL context, these errors indicate unexpected connection termination without proper TLS shutdown +fn is_connection_closed_error(exc: &Py<PyBaseException>, vm: &VirtualMachine) -> bool { + use rustpython_vm::stdlib::errno::errors; + + // Check for ConnectionAbortedError, ConnectionResetError (Python exception types) + if exc.fast_isinstance(vm.ctx.exceptions.connection_aborted_error) + || exc.fast_isinstance(vm.ctx.exceptions.connection_reset_error) + { + return true; + } + + // Also check OSError with specific errno values (ECONNABORTED, ECONNRESET) + if exc.fast_isinstance(vm.ctx.exceptions.os_error) + && let Ok(errno) = exc.as_object().get_attr("errno", vm) + && let Ok(errno_int) = errno.try_int(vm) + && let Ok(errno_val) = errno_int.try_to_primitive::<i32>(vm) + { + return errno_val == errors::ECONNABORTED || errno_val == errors::ECONNRESET; + } + false +} + +/// Ensure TLS data is available for reading +/// Returns the number of bytes read from the socket +fn ssl_ensure_data_available( + conn: &mut TlsConnection, + socket: &PySSLSocket, + vm: &VirtualMachine, +) -> SslResult<usize> { + // Unlike OpenSSL's SSL_read, rustls requires explicit I/O + if conn.wants_read() { + let is_bio = socket.is_bio_mode(); + + // For non-BIO mode (regular sockets), check if socket is ready first + // PERFORMANCE OPTIMIZATION: Only use select for sockets with timeout + // - Blocking sockets (timeout=None): Skip select, recv() will block naturally + // - Timeout sockets: Use select to enforce timeout + // - Non-blocking sockets: Skip select, recv() will return EAGAIN immediately + if !is_bio { + let timeout = socket.get_socket_timeout(vm).map_err(SslError::Py)?; + + // Only use select if socket has a positive timeout + if let Some(t) = timeout + && !t.is_zero() + { + // Socket has timeout - use select to enforce it + let timed_out = socket + .sock_wait_for_io_impl(SelectKind::Read, vm) + .map_err(SslError::Py)?; + if timed_out { + // Socket not ready within timeout - raise socket.timeout + return Err(SslError::Timeout( + "The read operation timed out".to_string(), + )); + } + } + // else: non-blocking socket (timeout=0) or blocking socket (timeout=None) - skip select + } + + // Read one TLS record at a time for non-BIO sockets (matching + // OpenSSL's default no-read-ahead behaviour). This prevents + // consuming a close_notify that arrives alongside application data, + // keeping it in the kernel buffer where select() can detect it. + let data = if !is_bio { + recv_one_tls_record_for_data(conn, socket, vm)? + } else { + match socket.sock_recv(2048, vm) { + Ok(data) => data, + Err(e) => { + if is_blocking_io_error(&e, vm) { + return Err(SslError::WantRead); + } + if let Err(rustls_err) = conn.process_new_packets() { + return Err(SslError::from_rustls(rustls_err)); + } + if is_connection_closed_error(&e, vm) { + return Err(SslError::Eof); + } + return Err(SslError::Py(e)); + } + } + }; + + // Get the size of received data + let bytes_read = data + .clone() + .try_into_value::<rustpython_vm::builtins::PyBytes>(vm) + .map(|b| b.as_bytes().len()) + .unwrap_or(0); + + // Check if BIO has EOF set (incoming BIO closed) + let is_eof = if is_bio { + // Check incoming BIO's eof property + if let Some(bio_obj) = socket.incoming_bio() { + bio_obj + .get_attr("eof", vm) + .and_then(|v| v.try_into_value::<bool>(vm)) + .unwrap_or(false) + } else { + false + } + } else { + false + }; + + // If BIO EOF is set and no data available, treat as connection EOF + if is_eof && bytes_read == 0 { + return Err(SslError::Eof); + } + + // Feed data to rustls and process packets + ssl_read_tls_records(conn, data, is_bio, vm)?; + + // Process any packets we successfully read + conn.process_new_packets().map_err(SslError::from_rustls)?; + + Ok(bytes_read) + } else { + // No data to read + Ok(0) + } +} + +// Multi-Certificate Resolver for RSA/ECC Support + +/// Multi-certificate resolver that selects appropriate certificate based on client capabilities +/// +/// This resolver implements OpenSSL's behavior of supporting multiple certificate/key pairs +/// (e.g., one RSA and one ECC) and selecting the appropriate one based on the client's +/// supported signature algorithms during the TLS handshake. +/// +/// OpenSSL's SSL_CTX_use_certificate_chain_file can be called multiple +/// times to add different certificate types, and OpenSSL automatically selects the best one. +#[derive(Debug)] +pub(super) struct MultiCertResolver { + cert_keys: Vec<Arc<CertifiedKey>>, +} + +impl MultiCertResolver { + /// Create a new multi-certificate resolver + pub fn new(cert_keys: Vec<Arc<CertifiedKey>>) -> Self { + Self { cert_keys } + } +} + +impl ResolvesServerCert for MultiCertResolver { + fn resolve(&self, client_hello: rustls::server::ClientHello<'_>) -> Option<Arc<CertifiedKey>> { + // Get the signature schemes supported by the client + let client_schemes = client_hello.signature_schemes(); + + // Try to find a certificate that matches the client's signature schemes + for cert_key in &self.cert_keys { + // Check if this certificate's signing key is compatible with any of the + // client's supported signature schemes + if let Some(_scheme) = cert_key.key.choose_scheme(client_schemes) { + return Some(cert_key.clone()); + } + } + + // If no perfect match, return the first certificate as fallback + // (This matches OpenSSL's behavior of using the first loaded cert if negotiation fails) + self.cert_keys.first().cloned() + } +} + +// Helper Functions for OpenSSL Compatibility: + +/// Normalize cipher suite name for OpenSSL compatibility +/// +/// Converts rustls cipher names to OpenSSL format: +/// - TLS_AES_256_GCM_SHA384 → AES256-GCM-SHA384 +/// - Replaces "AES-256" with "AES256" and "AES-128" with "AES128" +pub(super) fn normalize_cipher_name(rustls_name: &str) -> String { + rustls_name + .strip_prefix("TLS_") + .unwrap_or(rustls_name) + .replace("_WITH_", "_") + .replace('_', "-") + .replace("AES-256", "AES256") + .replace("AES-128", "AES128") +} + +/// Get cipher key size in bits from cipher suite name +/// +/// Returns: +/// - 256 for AES-256 and ChaCha20 +/// - 128 for AES-128 +/// - 0 for unknown ciphers +pub(super) fn get_cipher_key_bits(cipher_name: &str) -> i32 { + if cipher_name.contains("256") || cipher_name.contains("CHACHA20") { + 256 + } else if cipher_name.contains("128") { + 128 + } else { + 0 + } +} + +/// Get encryption algorithm description from cipher name +/// +/// Returns human-readable encryption description for OpenSSL compatibility +pub(super) fn get_cipher_encryption_desc(cipher_name: &str) -> &'static str { + if cipher_name.contains("AES256") { + "AESGCM(256)" + } else if cipher_name.contains("AES128") { + "AESGCM(128)" + } else if cipher_name.contains("CHACHA20") { + "CHACHA20-POLY1305(256)" + } else { + "Unknown" + } +} + +/// Normalize rustls cipher suite name to IANA standard format +/// +/// Converts rustls Debug format names to IANA standard: +/// - "TLS13_AES_256_GCM_SHA384" -> "TLS_AES_256_GCM_SHA384" +/// - Other names remain unchanged +pub(super) fn normalize_rustls_cipher_name(rustls_name: &str) -> String { + if rustls_name.starts_with("TLS13_") { + rustls_name.replace("TLS13_", "TLS_") + } else { + rustls_name.to_string() + } +} + +/// Convert rustls protocol version to string representation +/// +/// Returns the TLS version string +/// - TLSv1.2, TLSv1.3, or "Unknown" +pub(super) fn get_protocol_version_str(version: &rustls::SupportedProtocolVersion) -> &'static str { + match version.version { + rustls::ProtocolVersion::TLSv1_2 => "TLSv1.2", + rustls::ProtocolVersion::TLSv1_3 => "TLSv1.3", + _ => "Unknown", + } +} + +/// Cipher suite information +/// +/// Contains all relevant cipher information extracted from a rustls CipherSuite +pub(super) struct CipherInfo { + /// IANA standard cipher name (e.g., "TLS_AES_256_GCM_SHA384") + pub name: String, + /// TLS protocol version (e.g., "TLSv1.2", "TLSv1.3") + pub protocol: &'static str, + /// Key size in bits (e.g., 128, 256) + pub bits: i32, +} + +/// Extract cipher information from a rustls CipherSuite +/// +/// This consolidates the common cipher extraction logic used across +/// get_ciphers(), cipher(), and shared_ciphers() methods. +pub(super) fn extract_cipher_info(suite: &rustls::SupportedCipherSuite) -> CipherInfo { + let rustls_name = format!("{:?}", suite.suite()); + let name = normalize_rustls_cipher_name(&rustls_name); + let protocol = get_protocol_version_str(suite.version()); + let bits = get_cipher_key_bits(&name); + + CipherInfo { + name, + protocol, + bits, + } +} + +/// Convert curve name to rustls key exchange group +/// +/// Maps OpenSSL curve names (e.g., "prime256v1", "secp384r1") to rustls KxGroups. +/// Returns an error if the curve is not supported by rustls. +pub(super) fn curve_name_to_kx_group( + curve: &str, +) -> Result<Vec<&'static dyn SupportedKxGroup>, String> { + // Get the default crypto provider's key exchange groups + let provider = rustls::crypto::aws_lc_rs::default_provider(); + let all_groups = &provider.kx_groups; + + match curve { + // P-256 (also known as secp256r1 or prime256v1) + "prime256v1" | "secp256r1" => { + // Find SECP256R1 in the provider's groups + all_groups + .iter() + .find(|g| g.name() == rustls::NamedGroup::secp256r1) + .map(|g| vec![*g]) + .ok_or_else(|| "secp256r1 not supported by crypto provider".to_owned()) + } + // P-384 (also known as secp384r1 or prime384v1) + "secp384r1" | "prime384v1" => all_groups + .iter() + .find(|g| g.name() == rustls::NamedGroup::secp384r1) + .map(|g| vec![*g]) + .ok_or_else(|| "secp384r1 not supported by crypto provider".to_owned()), + // X25519 + "X25519" | "x25519" => all_groups + .iter() + .find(|g| g.name() == rustls::NamedGroup::X25519) + .map(|g| vec![*g]) + .ok_or_else(|| "X25519 not supported by crypto provider".to_owned()), + // P-521 (also known as secp521r1 or prime521v1) + // Now supported with aws-lc-rs crypto provider + "prime521v1" | "secp521r1" => all_groups + .iter() + .find(|g| g.name() == rustls::NamedGroup::secp521r1) + .map(|g| vec![*g]) + .ok_or_else(|| "secp521r1 not supported by crypto provider".to_owned()), + // X448 + // Now supported with aws-lc-rs crypto provider + "X448" | "x448" => all_groups + .iter() + .find(|g| g.name() == rustls::NamedGroup::X448) + .map(|g| vec![*g]) + .ok_or_else(|| "X448 not supported by crypto provider".to_owned()), + _ => Err(format!("unknown curve name '{curve}'")), + } +} diff --git a/crates/stdlib/src/ssl/error.rs b/crates/stdlib/src/ssl/error.rs new file mode 100644 index 00000000000..cbc59e0e8f6 --- /dev/null +++ b/crates/stdlib/src/ssl/error.rs @@ -0,0 +1,146 @@ +// SSL exception types shared between ssl (rustls) and openssl backends + +pub(crate) use ssl_error::*; + +#[pymodule(sub)] +pub(crate) mod ssl_error { + use crate::vm::{ + Py, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBaseException, PyOSError, PyStrRef}, + types::Constructor, + }; + + // Error type constants - exposed as pyattr and available for internal use + #[pyattr] + pub(crate) const SSL_ERROR_NONE: i32 = 0; + #[pyattr] + pub(crate) const SSL_ERROR_SSL: i32 = 1; + #[pyattr] + pub(crate) const SSL_ERROR_WANT_READ: i32 = 2; + #[pyattr] + pub(crate) const SSL_ERROR_WANT_WRITE: i32 = 3; + #[pyattr] + pub(crate) const SSL_ERROR_WANT_X509_LOOKUP: i32 = 4; + #[pyattr] + pub(crate) const SSL_ERROR_SYSCALL: i32 = 5; + #[pyattr] + pub(crate) const SSL_ERROR_ZERO_RETURN: i32 = 6; + #[pyattr] + pub(crate) const SSL_ERROR_WANT_CONNECT: i32 = 7; + #[pyattr] + pub(crate) const SSL_ERROR_EOF: i32 = 8; + #[pyattr] + pub(crate) const SSL_ERROR_INVALID_ERROR_CODE: i32 = 10; + + #[pyattr] + #[pyexception(name = "SSLError", base = PyOSError)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLError(PyOSError); + + #[pyexception] + impl PySSLError { + // Returns strerror attribute if available, otherwise str(args) + #[pymethod] + fn __str__(exc: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + use crate::vm::AsObject; + // Try to get strerror attribute first (OSError compatibility) + if let Ok(strerror) = exc.as_object().get_attr("strerror", vm) + && !vm.is_none(&strerror) + { + return strerror.str(vm); + } + + // Otherwise return str(args) + let args = exc.args(); + if args.len() == 1 { + args.as_slice()[0].str(vm) + } else { + args.as_object().str(vm) + } + } + } + + #[pyattr] + #[pyexception(name = "SSLZeroReturnError", base = PySSLError)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLZeroReturnError(PySSLError); + + #[pyexception] + impl PySSLZeroReturnError {} + + #[pyattr] + #[pyexception(name = "SSLWantReadError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLWantReadError(PySSLError); + + #[pyattr] + #[pyexception(name = "SSLWantWriteError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLWantWriteError(PySSLError); + + #[pyattr] + #[pyexception(name = "SSLSyscallError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLSyscallError(PySSLError); + + #[pyattr] + #[pyexception(name = "SSLEOFError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLEOFError(PySSLError); + + #[pyattr] + #[pyexception(name = "SSLCertVerificationError", base = PySSLError, impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySSLCertVerificationError(PySSLError); + + // Helper functions to create SSL exceptions with proper errno attribute + pub fn create_ssl_want_read_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLWantReadError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_WANT_READ), + "The operation did not complete (read)", + ) + } + + pub fn create_ssl_want_write_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLWantWriteError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_WANT_WRITE), + "The operation did not complete (write)", + ) + } + + pub fn create_ssl_eof_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLEOFError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_EOF), + "EOF occurred in violation of protocol", + ) + } + + pub fn create_ssl_zero_return_error(vm: &VirtualMachine) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLZeroReturnError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_ZERO_RETURN), + "TLS/SSL connection has been closed (EOF)", + ) + } + + pub fn create_ssl_syscall_error( + vm: &VirtualMachine, + msg: impl Into<String>, + ) -> PyRef<PyOSError> { + vm.new_os_subtype_error( + PySSLSyscallError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_SYSCALL), + msg.into(), + ) + } +} diff --git a/crates/stdlib/src/ssl/oid.rs b/crates/stdlib/src/ssl/oid.rs new file mode 100644 index 00000000000..d85898c0f79 --- /dev/null +++ b/crates/stdlib/src/ssl/oid.rs @@ -0,0 +1,465 @@ +// spell-checker: disable + +//! OID (Object Identifier) management for SSL/TLS +//! +//! This module provides OID lookup functionality compatible with CPython's ssl module. +//! It uses oid-registry crate for well-known OIDs while maintaining NID (Numerical Identifier) +//! mappings for CPython compatibility. + +use oid_registry::asn1_rs::Oid; +use std::collections::HashMap; + +/// OID entry with openssl-compatible metadata +#[derive(Debug, Clone)] +pub struct OidEntry { + /// NID (OpenSSL Numerical Identifier) - must match CPython/OpenSSL values + pub nid: i32, + /// Short name (e.g., "CN", "serverAuth") + pub short_name: &'static str, + /// Long name/description (e.g., "commonName", "TLS Web Server Authentication") + pub long_name: &'static str, + /// OID reference (static or dynamic) + pub oid: OidRef, +} + +/// OID reference - either from oid-registry or runtime-created +#[derive(Debug, Clone)] +pub enum OidRef { + /// Static OID from oid-registry crate (stored as value) + Static(Oid<'static>), + /// OID string (for OIDs not in oid-registry) - parsed on demand + String(&'static str), +} + +impl OidEntry { + /// Create entry from oid-registry static constant + pub fn from_static( + nid: i32, + short_name: &'static str, + long_name: &'static str, + oid: &Oid<'static>, + ) -> Self { + Self { + nid, + short_name, + long_name, + oid: OidRef::Static(oid.clone()), + } + } + + /// Create entry from OID string (for OIDs not in oid-registry) + pub const fn from_string( + nid: i32, + short_name: &'static str, + long_name: &'static str, + oid_str: &'static str, + ) -> Self { + Self { + nid, + short_name, + long_name, + oid: OidRef::String(oid_str), + } + } + + /// Get OID as string (e.g., "2.5.4.3") + pub fn oid_string(&self) -> String { + match &self.oid { + OidRef::Static(oid) => oid.to_id_string(), + OidRef::String(s) => s.to_string(), + } + } +} + +/// OID table with multiple indices for fast lookup +pub struct OidTable { + /// All entries + entries: Vec<OidEntry>, + /// NID -> index mapping + nid_to_idx: HashMap<i32, usize>, + /// Short name -> index mapping + short_name_to_idx: HashMap<&'static str, usize>, + /// Long name -> index mapping (case-insensitive) + long_name_to_idx: HashMap<String, usize>, + /// OID string -> index mapping + oid_str_to_idx: HashMap<String, usize>, +} + +impl OidTable { + fn build() -> Self { + let entries = build_oid_entries(); + let mut nid_to_idx = HashMap::with_capacity(entries.len()); + let mut short_name_to_idx = HashMap::with_capacity(entries.len()); + let mut long_name_to_idx = HashMap::with_capacity(entries.len()); + let mut oid_str_to_idx = HashMap::with_capacity(entries.len()); + + for (idx, entry) in entries.iter().enumerate() { + nid_to_idx.insert(entry.nid, idx); + short_name_to_idx.insert(entry.short_name, idx); + long_name_to_idx.insert(entry.long_name.to_lowercase(), idx); + oid_str_to_idx.insert(entry.oid_string(), idx); + } + + Self { + entries, + nid_to_idx, + short_name_to_idx, + long_name_to_idx, + oid_str_to_idx, + } + } + + pub fn find_by_nid(&self, nid: i32) -> Option<&OidEntry> { + self.nid_to_idx.get(&nid).map(|&idx| &self.entries[idx]) + } + + pub fn find_by_oid_string(&self, oid_str: &str) -> Option<&OidEntry> { + self.oid_str_to_idx + .get(oid_str) + .map(|&idx| &self.entries[idx]) + } + + pub fn find_by_name(&self, name: &str) -> Option<&OidEntry> { + // Try short name first (exact match) + self.short_name_to_idx + .get(name) + .or_else(|| { + // Try long name (case-insensitive) + self.long_name_to_idx.get(&name.to_lowercase()) + }) + .map(|&idx| &self.entries[idx]) + } +} + +/// Global OID table +static OID_TABLE: rustpython_common::lock::LazyLock<OidTable> = + rustpython_common::lock::LazyLock::new(OidTable::build); + +/// Macro to define OID entry using oid-registry constant +macro_rules! oid_static { + ($nid:expr, $short:expr, $long:expr, $oid_const:path) => { + OidEntry::from_static($nid, $short, $long, &$oid_const) + }; +} + +/// Macro to define OID entry from string +macro_rules! oid_string { + ($nid:expr, $short:expr, $long:expr, $oid_str:expr) => { + OidEntry::from_string($nid, $short, $long, $oid_str) + }; +} + +/// Build the complete OID table +fn build_oid_entries() -> Vec<OidEntry> { + vec![ + // Priority 1: X.509 DN Attributes (OpenSSL NID values) + // These NIDs MUST match OpenSSL for CPython compatibility + oid_static!(13, "CN", "commonName", oid_registry::OID_X509_COMMON_NAME), + oid_static!(14, "C", "countryName", oid_registry::OID_X509_COUNTRY_NAME), + oid_static!( + 15, + "L", + "localityName", + oid_registry::OID_X509_LOCALITY_NAME + ), + oid_static!( + 16, + "ST", + "stateOrProvinceName", + oid_registry::OID_X509_STATE_OR_PROVINCE_NAME + ), + oid_static!( + 17, + "O", + "organizationName", + oid_registry::OID_X509_ORGANIZATION_NAME + ), + oid_static!( + 18, + "OU", + "organizationalUnitName", + oid_registry::OID_X509_ORGANIZATIONAL_UNIT + ), + oid_static!(41, "name", "name", oid_registry::OID_X509_NAME), + oid_static!(42, "GN", "givenName", oid_registry::OID_X509_GIVEN_NAME), + oid_static!(43, "initials", "initials", oid_registry::OID_X509_INITIALS), + oid_static!( + 4, + "serialNumber", + "serialNumber", + oid_registry::OID_X509_SERIALNUMBER + ), + oid_static!(100, "surname", "surname", oid_registry::OID_X509_SURNAME), + // emailAddress is special - it's in PKCS#9, not X.509 + oid_static!( + 48, + "emailAddress", + "emailAddress", + oid_registry::OID_PKCS9_EMAIL_ADDRESS + ), + // Priority 2: X.509 Extensions (Critical ones) + oid_static!( + 82, + "subjectKeyIdentifier", + "X509v3 Subject Key Identifier", + oid_registry::OID_X509_EXT_SUBJECT_KEY_IDENTIFIER + ), + oid_static!( + 83, + "keyUsage", + "X509v3 Key Usage", + oid_registry::OID_X509_EXT_KEY_USAGE + ), + oid_static!( + 85, + "subjectAltName", + "X509v3 Subject Alternative Name", + oid_registry::OID_X509_EXT_SUBJECT_ALT_NAME + ), + oid_static!( + 86, + "issuerAltName", + "X509v3 Issuer Alternative Name", + oid_registry::OID_X509_EXT_ISSUER_ALT_NAME + ), + oid_static!( + 87, + "basicConstraints", + "X509v3 Basic Constraints", + oid_registry::OID_X509_EXT_BASIC_CONSTRAINTS + ), + oid_static!( + 88, + "crlNumber", + "X509v3 CRL Number", + oid_registry::OID_X509_EXT_CRL_NUMBER + ), + oid_static!( + 90, + "authorityKeyIdentifier", + "X509v3 Authority Key Identifier", + oid_registry::OID_X509_EXT_AUTHORITY_KEY_IDENTIFIER + ), + oid_static!( + 126, + "extendedKeyUsage", + "X509v3 Extended Key Usage", + oid_registry::OID_X509_EXT_EXTENDED_KEY_USAGE + ), + oid_static!( + 103, + "crlDistributionPoints", + "X509v3 CRL Distribution Points", + oid_registry::OID_X509_EXT_CRL_DISTRIBUTION_POINTS + ), + oid_static!( + 89, + "certificatePolicies", + "X509v3 Certificate Policies", + oid_registry::OID_X509_EXT_CERTIFICATE_POLICIES + ), + oid_static!( + 177, + "authorityInfoAccess", + "Authority Information Access", + oid_registry::OID_PKIX_AUTHORITY_INFO_ACCESS + ), + oid_static!( + 105, + "nameConstraints", + "X509v3 Name Constraints", + oid_registry::OID_X509_EXT_NAME_CONSTRAINTS + ), + // Priority 3: Extended Key Usage OIDs (not in oid-registry) + // These are defined in RFC 5280 but not in oid-registry, so we use strings + oid_string!( + 129, + "serverAuth", + "TLS Web Server Authentication", + "1.3.6.1.5.5.7.3.1" + ), + oid_string!( + 130, + "clientAuth", + "TLS Web Client Authentication", + "1.3.6.1.5.5.7.3.2" + ), + oid_string!(131, "codeSigning", "Code Signing", "1.3.6.1.5.5.7.3.3"), + oid_string!( + 132, + "emailProtection", + "E-mail Protection", + "1.3.6.1.5.5.7.3.4" + ), + oid_string!(133, "timeStamping", "Time Stamping", "1.3.6.1.5.5.7.3.8"), + oid_string!(180, "OCSPSigning", "OCSP Signing", "1.3.6.1.5.5.7.3.9"), + // Priority 4: Signature Algorithms + oid_static!( + 6, + "rsaEncryption", + "rsaEncryption", + oid_registry::OID_PKCS1_RSAENCRYPTION + ), + oid_static!( + 65, + "sha1WithRSAEncryption", + "sha1WithRSAEncryption", + oid_registry::OID_PKCS1_SHA1WITHRSA + ), + oid_static!( + 668, + "sha256WithRSAEncryption", + "sha256WithRSAEncryption", + oid_registry::OID_PKCS1_SHA256WITHRSA + ), + oid_static!( + 669, + "sha384WithRSAEncryption", + "sha384WithRSAEncryption", + oid_registry::OID_PKCS1_SHA384WITHRSA + ), + oid_static!( + 670, + "sha512WithRSAEncryption", + "sha512WithRSAEncryption", + oid_registry::OID_PKCS1_SHA512WITHRSA + ), + oid_static!( + 408, + "id-ecPublicKey", + "id-ecPublicKey", + oid_registry::OID_KEY_TYPE_EC_PUBLIC_KEY + ), + oid_static!( + 794, + "ecdsa-with-SHA256", + "ecdsa-with-SHA256", + oid_registry::OID_SIG_ECDSA_WITH_SHA256 + ), + oid_static!( + 795, + "ecdsa-with-SHA384", + "ecdsa-with-SHA384", + oid_registry::OID_SIG_ECDSA_WITH_SHA384 + ), + oid_static!( + 796, + "ecdsa-with-SHA512", + "ecdsa-with-SHA512", + oid_registry::OID_SIG_ECDSA_WITH_SHA512 + ), + // Priority 5: Hash Algorithms + oid_string!(64, "sha1", "sha1", "1.3.14.3.2.26"), + oid_static!(672, "sha256", "sha256", oid_registry::OID_NIST_HASH_SHA256), + oid_static!(673, "sha384", "sha384", oid_registry::OID_NIST_HASH_SHA384), + oid_static!(674, "sha512", "sha512", oid_registry::OID_NIST_HASH_SHA512), + oid_string!(675, "sha224", "sha224", "2.16.840.1.101.3.4.2.4"), + // Priority 6: Elliptic Curve OIDs + oid_static!(714, "secp256r1", "secp256r1", oid_registry::OID_EC_P256), + oid_string!(715, "secp384r1", "secp384r1", "1.3.132.0.34"), + oid_string!(716, "secp521r1", "secp521r1", "1.3.132.0.35"), + oid_string!(1172, "X25519", "X25519", "1.3.101.110"), + oid_string!(1173, "Ed25519", "Ed25519", "1.3.101.112"), + // Additional useful OIDs + oid_string!( + 183, + "subjectInfoAccess", + "Subject Information Access", + "1.3.6.1.5.5.7.1.11" + ), + oid_string!(920, "OCSP", "OCSP", "1.3.6.1.5.5.7.48.1"), + oid_string!(921, "caIssuers", "CA Issuers", "1.3.6.1.5.5.7.48.2"), + ] +} + +// Public API Functions + +/// Find OID entry by NID +pub fn find_by_nid(nid: i32) -> Option<&'static OidEntry> { + OID_TABLE.find_by_nid(nid) +} + +/// Find OID entry by OID string (e.g., "2.5.4.3") +pub fn find_by_oid_string(oid_str: &str) -> Option<&'static OidEntry> { + OID_TABLE.find_by_oid_string(oid_str) +} + +/// Find OID entry by name (short or long name) +pub fn find_by_name(name: &str) -> Option<&'static OidEntry> { + OID_TABLE.find_by_name(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_find_by_nid() { + let entry = find_by_nid(13).unwrap(); + assert_eq!(entry.short_name, "CN"); + assert_eq!(entry.long_name, "commonName"); + assert_eq!(entry.oid_string(), "2.5.4.3"); + } + + #[test] + fn test_find_by_oid_string() { + let entry = find_by_oid_string("2.5.4.3").unwrap(); + assert_eq!(entry.nid, 13); + assert_eq!(entry.short_name, "CN"); + } + + #[test] + fn test_find_by_name_short() { + let entry = find_by_name("CN").unwrap(); + assert_eq!(entry.nid, 13); + assert_eq!(entry.oid_string(), "2.5.4.3"); + } + + #[test] + fn test_find_by_name_long() { + let entry = find_by_name("commonName").unwrap(); + assert_eq!(entry.nid, 13); + assert_eq!(entry.short_name, "CN"); + } + + #[test] + fn test_find_by_name_case_insensitive() { + let entry = find_by_name("COMMONNAME").unwrap(); + assert_eq!(entry.nid, 13); + } + + #[test] + fn test_subject_alt_name() { + let entry = find_by_nid(85).unwrap(); + assert_eq!(entry.short_name, "subjectAltName"); + assert_eq!(entry.oid_string(), "2.5.29.17"); + } + + #[test] + fn test_server_auth_eku() { + let entry = find_by_nid(129).unwrap(); + assert_eq!(entry.short_name, "serverAuth"); + assert_eq!(entry.oid_string(), "1.3.6.1.5.5.7.3.1"); + } + + #[test] + fn test_no_duplicate_nids() { + let table = &*OID_TABLE; + assert_eq!( + table.entries.len(), + table.nid_to_idx.len(), + "Duplicate NIDs detected!" + ); + } + + #[test] + fn test_oid_count() { + let table = &*OID_TABLE; + // We should have 50+ OIDs defined + assert!( + table.entries.len() >= 50, + "Expected at least 50 OIDs, got {}", + table.entries.len() + ); + } +} diff --git a/stdlib/src/statistics.rs b/crates/stdlib/src/statistics.rs similarity index 88% rename from stdlib/src/statistics.rs rename to crates/stdlib/src/statistics.rs index 356bfd66f9c..2f7eb85284a 100644 --- a/stdlib/src/statistics.rs +++ b/crates/stdlib/src/statistics.rs @@ -1,13 +1,16 @@ -pub(crate) use _statistics::make_module; +pub(crate) use _statistics::module_def; #[pymodule] mod _statistics { - use crate::vm::{function::ArgIntoFloat, PyResult, VirtualMachine}; + use crate::vm::{PyResult, VirtualMachine, function::ArgIntoFloat}; // See https://github.com/python/cpython/blob/6846d6712a0894f8e1a91716c11dd79f42864216/Modules/_statisticsmodule.c#L28-L120 - #[allow(clippy::excessive_precision)] + #[allow( + clippy::excessive_precision, + reason = "constants are kept at CPython precision" + )] fn normal_dist_inv_cdf(p: f64, mu: f64, sigma: f64) -> Option<f64> { - if p <= 0.0 || p >= 1.0 || sigma <= 0.0 { + if p <= 0.0 || p >= 1.0 { return None; } @@ -53,7 +56,10 @@ mod _statistics { let r = (-(r.ln())).sqrt(); let num; let den; - #[allow(clippy::excessive_precision)] + #[allow( + clippy::excessive_precision, + reason = "piecewise polynomial coefficients match CPython" + )] if r <= 5.0 { let r = r - 1.6; // Hash sum-49.33206503301610289036 @@ -126,7 +132,7 @@ mod _statistics { sigma: ArgIntoFloat, vm: &VirtualMachine, ) -> PyResult<f64> { - normal_dist_inv_cdf(*p, *mu, *sigma) - .ok_or_else(|| vm.new_value_error("inv_cdf undefined for these parameters".to_owned())) + normal_dist_inv_cdf(p.into_float(), mu.into_float(), sigma.into_float()) + .ok_or_else(|| vm.new_value_error("inv_cdf undefined for these parameters")) } } diff --git a/crates/stdlib/src/suggestions.rs b/crates/stdlib/src/suggestions.rs new file mode 100644 index 00000000000..e0667dfb553 --- /dev/null +++ b/crates/stdlib/src/suggestions.rs @@ -0,0 +1,20 @@ +pub(crate) use _suggestions::module_def; + +#[pymodule] +mod _suggestions { + use rustpython_vm::VirtualMachine; + + use crate::vm::PyObjectRef; + + #[pyfunction] + fn _generate_suggestions( + candidates: Vec<PyObjectRef>, + name: PyObjectRef, + vm: &VirtualMachine, + ) -> PyObjectRef { + match crate::vm::suggestion::calculate_suggestions(candidates.iter(), &name) { + Some(suggestion) => suggestion.into(), + None => vm.ctx.none(), + } + } +} diff --git a/stdlib/src/syslog.rs b/crates/stdlib/src/syslog.rs similarity index 84% rename from stdlib/src/syslog.rs rename to crates/stdlib/src/syslog.rs index 9879f3ffaf0..8f446a8e161 100644 --- a/stdlib/src/syslog.rs +++ b/crates/stdlib/src/syslog.rs @@ -1,17 +1,18 @@ -// spell-checker:ignore logoption openlog setlogmask upto +// spell-checker:ignore logoption openlog setlogmask upto NDELAY ODELAY -pub(crate) use syslog::make_module; +pub(crate) use syslog::module_def; #[pymodule(name = "syslog")] mod syslog { use crate::common::lock::PyRwLock; use crate::vm::{ + PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyStr, PyStrRef}, function::{OptionalArg, OptionalOption}, utils::ToCString, - PyObjectRef, PyPayload, PyResult, VirtualMachine, }; - use std::{ffi::CStr, os::raw::c_char}; + use core::ffi::CStr; + use std::os::raw::c_char; #[pyattr] use libc::{ @@ -26,16 +27,16 @@ mod syslog { use libc::{LOG_AUTHPRIV, LOG_CRON, LOG_PERROR}; fn get_argv(vm: &VirtualMachine) -> Option<PyStrRef> { - if let Some(argv) = vm.state.settings.argv.first() { - if !argv.is_empty() { - return Some( - PyStr::from(match argv.find('\\') { - Some(value) => &argv[value..], - None => argv, - }) - .into_ref(&vm.ctx), - ); - } + if let Some(argv) = vm.state.config.settings.argv.first() + && !argv.is_empty() + { + return Some( + PyStr::from(match argv.find('\\') { + Some(value) => &argv[value..], + None => argv, + }) + .into_ref(&vm.ctx), + ); } None } @@ -49,8 +50,8 @@ mod syslog { impl GlobalIdent { fn as_ptr(&self) -> *const c_char { match self { - GlobalIdent::Explicit(ref cstr) => cstr.as_ptr(), - GlobalIdent::Implicit => std::ptr::null(), + Self::Explicit(cstr) => cstr.as_ptr(), + Self::Implicit => core::ptr::null(), } } } @@ -135,13 +136,13 @@ mod syslog { #[inline] #[pyfunction(name = "LOG_MASK")] - fn log_mask(pri: i32) -> i32 { + const fn log_mask(pri: i32) -> i32 { pri << 1 } #[inline] #[pyfunction(name = "LOG_UPTO")] - fn log_upto(pri: i32) -> i32 { + const fn log_upto(pri: i32) -> i32 { (1 << (pri + 1)) - 1 } } diff --git a/stdlib/src/termios.rs b/crates/stdlib/src/termios.rs similarity index 85% rename from stdlib/src/termios.rs rename to crates/stdlib/src/termios.rs index 637e9185a6d..de402724434 100644 --- a/stdlib/src/termios.rs +++ b/crates/stdlib/src/termios.rs @@ -1,11 +1,14 @@ -pub(crate) use self::termios::make_module; +// spell-checker:disable + +pub(crate) use self::termios::module_def; #[pymodule] mod termios { use crate::vm::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytes, PyInt, PyListRef, PyTypeRef}, + common::os::ErrorExt, convert::ToPyObject, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, }; use termios::Termios; @@ -54,9 +57,9 @@ mod termios { ))] #[pyattr] use libc::{ - FIONCLEX, FIONREAD, TIOCEXCL, TIOCMBIC, TIOCMBIS, TIOCMGET, TIOCMSET, TIOCM_CAR, TIOCM_CD, - TIOCM_CTS, TIOCM_DSR, TIOCM_DTR, TIOCM_LE, TIOCM_RI, TIOCM_RNG, TIOCM_RTS, TIOCM_SR, - TIOCM_ST, TIOCNXCL, TIOCSCTTY, + FIONCLEX, FIONREAD, TIOCEXCL, TIOCM_CAR, TIOCM_CD, TIOCM_CTS, TIOCM_DSR, TIOCM_DTR, + TIOCM_LE, TIOCM_RI, TIOCM_RNG, TIOCM_RTS, TIOCM_SR, TIOCM_ST, TIOCMBIC, TIOCMBIS, TIOCMGET, + TIOCMSET, TIOCNXCL, TIOCSCTTY, }; #[cfg(any(target_os = "android", target_os = "linux"))] #[pyattr] @@ -99,12 +102,6 @@ mod termios { ))] #[pyattr] use termios::os::target::TCSASOFT; - #[cfg(any(target_os = "android", target_os = "linux"))] - #[pyattr] - use termios::os::target::{ - B1000000, B1152000, B1500000, B2000000, B2500000, B3000000, B3500000, B4000000, B500000, - B576000, CBAUDEX, - }; #[cfg(any( target_os = "android", target_os = "freebsd", @@ -115,6 +112,12 @@ mod termios { ))] #[pyattr] use termios::os::target::{B460800, B921600}; + #[cfg(any(target_os = "android", target_os = "linux"))] + #[pyattr] + use termios::os::target::{ + B500000, B576000, B1000000, B1152000, B1500000, B2000000, B2500000, B3000000, B3500000, + B4000000, CBAUDEX, + }; #[cfg(any( target_os = "android", target_os = "illumos", @@ -153,16 +156,17 @@ mod termios { use termios::os::target::{VSWTCH, VSWTCH as VSWTC}; #[pyattr] use termios::{ + B0, B50, B75, B110, B134, B150, B200, B300, B600, B1200, B1800, B2400, B4800, B9600, + B19200, B38400, BRKINT, CLOCAL, CREAD, CS5, CS6, CS7, CS8, CSIZE, CSTOPB, ECHO, ECHOE, + ECHOK, ECHONL, HUPCL, ICANON, ICRNL, IEXTEN, IGNBRK, IGNCR, IGNPAR, INLCR, INPCK, ISIG, + ISTRIP, IXANY, IXOFF, IXON, NOFLSH, OCRNL, ONLCR, ONLRET, ONOCR, OPOST, PARENB, PARMRK, + PARODD, TCIFLUSH, TCIOFF, TCIOFLUSH, TCION, TCOFLUSH, TCOOFF, TCOON, TCSADRAIN, TCSAFLUSH, + TCSANOW, TOSTOP, VEOF, VEOL, VERASE, VINTR, VKILL, VMIN, VQUIT, VSTART, VSTOP, VSUSP, + VTIME, os::target::{ - B115200, B230400, B57600, CRTSCTS, ECHOCTL, ECHOKE, ECHOPRT, EXTA, EXTB, FLUSHO, + B57600, B115200, B230400, CRTSCTS, ECHOCTL, ECHOKE, ECHOPRT, EXTA, EXTB, FLUSHO, IMAXBEL, NCCS, PENDIN, VDISCARD, VEOL2, VLNEXT, VREPRINT, VWERASE, }, - B0, B110, B1200, B134, B150, B1800, B19200, B200, B2400, B300, B38400, B4800, B50, B600, - B75, B9600, BRKINT, CLOCAL, CREAD, CS5, CS6, CS7, CS8, CSIZE, CSTOPB, ECHO, ECHOE, ECHOK, - ECHONL, HUPCL, ICANON, ICRNL, IEXTEN, IGNBRK, IGNCR, IGNPAR, INLCR, INPCK, ISIG, ISTRIP, - IXANY, IXOFF, IXON, NOFLSH, OCRNL, ONLCR, ONLRET, ONOCR, OPOST, PARENB, PARMRK, PARODD, - TCIFLUSH, TCIOFF, TCIOFLUSH, TCION, TCOFLUSH, TCOOFF, TCOON, TCSADRAIN, TCSAFLUSH, TCSANOW, - TOSTOP, VEOF, VEOL, VERASE, VINTR, VKILL, VMIN, VQUIT, VSTART, VSTOP, VSUSP, VTIME, }; #[pyfunction] @@ -194,9 +198,7 @@ mod termios { fn tcsetattr(fd: i32, when: i32, attributes: PyListRef, vm: &VirtualMachine) -> PyResult<()> { let [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] = <&[PyObjectRef; 7]>::try_from(&*attributes.borrow_vec()) - .map_err(|_| { - vm.new_type_error("tcsetattr, arg 3: must be 7 element list".to_owned()) - })? + .map_err(|_| vm.new_type_error("tcsetattr, arg 3: must be 7 element list"))? .clone(); let mut termios = Termios::from_fd(fd).map_err(|e| termios_error(e, vm))?; termios.c_iflag = iflag.try_into_value(vm)?; @@ -215,13 +217,16 @@ mod termios { )) })?; for (cc, x) in termios.c_cc.iter_mut().zip(cc.iter()) { - *cc = if let Some(c) = x.payload::<PyBytes>().filter(|b| b.as_bytes().len() == 1) { + *cc = if let Some(c) = x + .downcast_ref::<PyBytes>() + .filter(|b| b.as_bytes().len() == 1) + { c.as_bytes()[0] as _ - } else if let Some(i) = x.payload::<PyInt>() { + } else if let Some(i) = x.downcast_ref::<PyInt>() { i.try_to_primitive(vm)? } else { return Err(vm.new_type_error( - "tcsetattr: elements of attributes must be characters or integers".to_owned(), + "tcsetattr: elements of attributes must be characters or integers", )); }; } @@ -256,13 +261,12 @@ mod termios { } fn termios_error(err: std::io::Error, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception( + vm.new_os_subtype_error( error_type(vm), - vec![ - err.raw_os_error().to_pyobject(vm), - vm.ctx.new_str(err.to_string()).into(), - ], + Some(err.posix_errno()), + vm.ctx.new_str(err.to_string()), ) + .upcast() } #[pyattr(name = "error", once)] diff --git a/crates/stdlib/src/tkinter.rs b/crates/stdlib/src/tkinter.rs new file mode 100644 index 00000000000..b258002c129 --- /dev/null +++ b/crates/stdlib/src/tkinter.rs @@ -0,0 +1,553 @@ +// spell-checker:ignore createcommand + +pub(crate) use self::_tkinter::module_def; + +#[pymodule] +mod _tkinter { + use rustpython_vm::convert::IntoPyException; + use rustpython_vm::types::Constructor; + use rustpython_vm::{Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; + + use rustpython_vm::builtins::{PyInt, PyStr, PyType}; + use std::{ffi, ptr}; + + use crate::builtins::PyTypeRef; + use rustpython_common::atomic::AtomicBool; + use rustpython_common::atomic::Ordering; + + #[cfg(windows)] + fn _get_tcl_lib_path() -> String { + // TODO: fix packaging + String::from(r"C:\ActiveTcl\lib") + } + + #[pyattr(name = "TclError", once)] + fn tcl_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_tkinter", + "TclError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + #[pyattr(name = "TkError", once)] + fn tk_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_tkinter", + "TkError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + #[pyattr(once, name = "TK_VERSION")] + fn tk_version(_vm: &VirtualMachine) -> String { + format!("{}.{}", 8, 6) + } + + #[pyattr(once, name = "TCL_VERSION")] + fn tcl_version(_vm: &VirtualMachine) -> String { + format!( + "{}.{}", + tk_sys::TCL_MAJOR_VERSION, + tk_sys::TCL_MINOR_VERSION + ) + } + + #[pyattr] + #[pyclass(name = "TclObject")] + #[derive(PyPayload)] + struct TclObject { + value: *mut tk_sys::Tcl_Obj, + } + + impl core::fmt::Debug for TclObject { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "TclObject") + } + } + + unsafe impl Send for TclObject {} + unsafe impl Sync for TclObject {} + + #[pyclass] + impl TclObject {} + + static QUIT_MAIN_LOOP: AtomicBool = AtomicBool::new(false); + static ERROR_IN_CMD: AtomicBool = AtomicBool::new(false); + + #[pyattr] + #[pyclass(name = "tkapp")] + #[derive(PyPayload)] + struct TkApp { + // Tcl_Interp *interp; + interpreter: *mut tk_sys::Tcl_Interp, + // int wantobjects; + want_objects: bool, + // int threaded; /* True if tcl_platform[threaded] */ + threaded: bool, + // Tcl_ThreadId thread_id; + thread_id: Option<tk_sys::Tcl_ThreadId>, + // int dispatching; + dispatching: bool, + // PyObject *trace; + trace: Option<()>, + // /* We cannot include tclInt.h, as this is internal. + // So we cache interesting types here. */ + old_boolean_type: *const tk_sys::Tcl_ObjType, + boolean_type: *const tk_sys::Tcl_ObjType, + byte_array_type: *const tk_sys::Tcl_ObjType, + double_type: *const tk_sys::Tcl_ObjType, + int_type: *const tk_sys::Tcl_ObjType, + wide_int_type: *const tk_sys::Tcl_ObjType, + bignum_type: *const tk_sys::Tcl_ObjType, + list_type: *const tk_sys::Tcl_ObjType, + string_type: *const tk_sys::Tcl_ObjType, + utf32_string_type: *const tk_sys::Tcl_ObjType, + pixel_type: *const tk_sys::Tcl_ObjType, + } + + unsafe impl Send for TkApp {} + unsafe impl Sync for TkApp {} + + impl core::fmt::Debug for TkApp { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "TkApp") + } + } + + #[derive(FromArgs, Debug)] + struct TkAppConstructorArgs { + #[pyarg(any)] + screen_name: Option<String>, + #[pyarg(any)] + _base_name: Option<String>, + #[pyarg(any)] + class_name: String, + #[pyarg(any)] + interactive: i32, + #[pyarg(any)] + wantobjects: i32, + #[pyarg(any, default = true)] + want_tk: bool, + #[pyarg(any)] + sync: i32, + #[pyarg(any)] + use_: Option<String>, + } + + impl Constructor for TkApp { + type Args = TkAppConstructorArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + create(args, vm) + } + } + + fn varname_converter(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + // if let Ok(bytes) = obj.bytes(vm) { + // todo!() + // } + + // str + if let Some(str) = obj.downcast_ref::<PyStr>() { + return Ok(str.as_str().to_string()); + } + + if let Some(_tcl_obj) = obj.downcast_ref::<TclObject>() { + // Assume that the Tcl object has a method to retrieve a string. + // return tcl_obj. + todo!(); + } + + // Construct an error message using the type name (truncated to 50 characters). + Err(vm.new_type_error(format!( + "must be str, bytes or Tcl_Obj, not {:.50}", + obj.obj_type().str(vm)?.as_str() + ))) + } + + #[derive(Debug, FromArgs)] + struct TkAppGetVarArgs { + #[pyarg(any)] + name: PyObjectRef, + #[pyarg(any, default)] + name2: Option<String>, + } + + // TODO: DISALLOW_INSTANTIATION + #[pyclass(with(Constructor))] + impl TkApp { + fn tcl_obj_to_bool(&self, obj: *mut tk_sys::Tcl_Obj) -> bool { + let mut res = -1; + unsafe { + if tk_sys::Tcl_GetBooleanFromObj(self.interpreter, obj, &mut res) + != tk_sys::TCL_OK as i32 + { + panic!("Tcl_GetBooleanFromObj failed"); + } + } + assert!(res == 0 || res == 1); + res != 0 + } + + fn tcl_obj_to_pyobject( + &self, + obj: *mut tk_sys::Tcl_Obj, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let type_ptr = unsafe { (*obj).typePtr }; + if type_ptr.is_null() { + return self.unicode_from_object(obj, vm); + } else if type_ptr == self.old_boolean_type || type_ptr == self.boolean_type { + return Ok(vm.ctx.new_bool(self.tcl_obj_to_bool(obj)).into()); + } else if type_ptr == self.string_type + || type_ptr == self.utf32_string_type + || type_ptr == self.pixel_type + { + return self.unicode_from_object(obj, vm); + } + // TODO: handle other types + + Ok(TclObject { value: obj }.into_pyobject(vm)) + } + + fn unicode_from_string( + s: *mut ffi::c_char, + size: usize, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + // terribly unsafe + let s = unsafe { std::slice::from_raw_parts(s, size) } + .to_vec() + .into_iter() + .map(|c| c as u8) + .collect::<Vec<u8>>(); + let s = String::from_utf8(s).unwrap(); + Ok(PyObjectRef::from(vm.ctx.new_str(s))) + } + + fn unicode_from_object( + &self, + obj: *mut tk_sys::Tcl_Obj, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let type_ptr = unsafe { (*obj).typePtr }; + if !type_ptr.is_null() + && !self.interpreter.is_null() + && (type_ptr == self.string_type || type_ptr == self.utf32_string_type) + { + let len = ptr::null_mut(); + let data = unsafe { tk_sys::Tcl_GetUnicodeFromObj(obj, len) }; + return if size_of::<tk_sys::Tcl_UniChar>() == 2 { + let v = unsafe { std::slice::from_raw_parts(data as *const u16, len as usize) }; + let s = String::from_utf16(v).unwrap(); + Ok(PyObjectRef::from(vm.ctx.new_str(s))) + } else { + let v = unsafe { std::slice::from_raw_parts(data as *const u32, len as usize) }; + let s = widestring::U32String::from_vec(v).to_string_lossy(); + Ok(PyObjectRef::from(vm.ctx.new_str(s))) + }; + } + let len = ptr::null_mut(); + let s = unsafe { tk_sys::Tcl_GetStringFromObj(obj, len) }; + Self::unicode_from_string(s, len as _, vm) + } + + fn var_invoke(&self) { + if self.threaded && self.thread_id != Some(unsafe { tk_sys::Tcl_GetCurrentThread() }) { + // TODO: do stuff + } + } + + fn inner_getvar( + &self, + args: TkAppGetVarArgs, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let TkAppGetVarArgs { name, name2 } = args; + // TODO: technically not thread safe + let name = varname_converter(name, vm)?; + + let name = ffi::CString::new(name).map_err(|e| e.into_pyexception(vm))?; + let name2 = + ffi::CString::new(name2.unwrap_or_default()).map_err(|e| e.into_pyexception(vm))?; + let name2_ptr = if name2.is_empty() { + ptr::null() + } else { + name2.as_ptr() + }; + let res = unsafe { + tk_sys::Tcl_GetVar2Ex( + self.interpreter, + name.as_ptr() as _, + name2_ptr as _, + flags as _, + ) + }; + if res.is_null() { + // TODO: Should be tk error + unsafe { + let err_obj = tk_sys::Tcl_GetObjResult(self.interpreter); + let err_str_obj = tk_sys::Tcl_GetString(err_obj); + let err_cstr = ffi::CStr::from_ptr(err_str_obj as _); + return Err(vm.new_type_error(format!("{err_cstr:?}"))); + } + } + let res = if self.want_objects { + self.tcl_obj_to_pyobject(res, vm) + } else { + self.unicode_from_object(res, vm) + }?; + Ok(res) + } + + #[pymethod] + fn getvar(&self, args: TkAppGetVarArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + self.var_invoke(); + self.inner_getvar(args, tk_sys::TCL_LEAVE_ERR_MSG, vm) + } + + #[pymethod] + fn globalgetvar( + &self, + args: TkAppGetVarArgs, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + self.var_invoke(); + self.inner_getvar( + args, + tk_sys::TCL_LEAVE_ERR_MSG | tk_sys::TCL_GLOBAL_ONLY, + vm, + ) + } + + #[pymethod] + fn getint(&self, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if let Some(int) = arg.downcast_ref::<PyInt>() { + return Ok(PyObjectRef::from(vm.ctx.new_int(int.as_bigint().clone()))); + } + + if let Some(obj) = arg.downcast_ref::<TclObject>() { + let value = obj.value; + unsafe { tk_sys::Tcl_IncrRefCount(value) }; + } else { + todo!(); + } + todo!(); + } + // TODO: Fix arguments + #[pymethod] + fn mainloop(&self, threshold: Option<i32>) -> PyResult<()> { + let threshold = threshold.unwrap_or(0); + // self.dispatching = true; + QUIT_MAIN_LOOP.store(false, Ordering::Relaxed); + while unsafe { tk_sys::Tk_GetNumMainWindows() } > threshold + && !QUIT_MAIN_LOOP.load(Ordering::Relaxed) + && !ERROR_IN_CMD.load(Ordering::Relaxed) + { + if self.threaded { + unsafe { tk_sys::Tcl_DoOneEvent(0 as _) }; + } else { + unsafe { tk_sys::Tcl_DoOneEvent(tk_sys::TCL_DONT_WAIT as _) }; + // TODO: sleep for the proper time + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + Ok(()) + } + + #[pymethod] + fn quit(&self) { + QUIT_MAIN_LOOP.store(true, Ordering::Relaxed); + } + } + + #[pyfunction] + fn create(args: TkAppConstructorArgs, vm: &VirtualMachine) -> PyResult<TkApp> { + unsafe { + let interp = tk_sys::Tcl_CreateInterp(); + let want_objects = args.wantobjects != 0; + let threaded = !{ + let part1 = String::from("tcl_platform"); + let part2 = String::from("threaded"); + let part1 = ffi::CString::new(part1).map_err(|e| e.into_pyexception(vm))?; + let part2 = ffi::CString::new(part2).map_err(|e| e.into_pyexception(vm))?; + let part1_ptr = part1.as_ptr(); + let part2_ptr = part2.as_ptr(); + tk_sys::Tcl_GetVar2Ex( + interp, + part1_ptr as _, + part2_ptr as _, + tk_sys::TCL_GLOBAL_ONLY as ffi::c_int, + ) + } + .is_null(); + let thread_id = tk_sys::Tcl_GetCurrentThread(); + let dispatching = false; + let trace = None; + // TODO: Handle threaded build + let bool_str = String::from("oldBoolean"); + let old_boolean_type = tk_sys::Tcl_GetObjType(bool_str.as_ptr() as _); + let (boolean_type, byte_array_type) = { + let true_str = String::from("true"); + let mut value = *tk_sys::Tcl_NewStringObj(true_str.as_ptr() as _, -1); + let mut bool_value = 0; + tk_sys::Tcl_GetBooleanFromObj(interp, &mut value, &mut bool_value); + let boolean_type = value.typePtr; + tk_sys::Tcl_DecrRefCount(&mut value); + + let mut value = + *tk_sys::Tcl_NewByteArrayObj(&bool_value as *const i32 as *const u8, 1); + let byte_array_type = value.typePtr; + tk_sys::Tcl_DecrRefCount(&mut value); + (boolean_type, byte_array_type) + }; + let double_str = String::from("double"); + let double_type = tk_sys::Tcl_GetObjType(double_str.as_ptr() as _); + let int_str = String::from("int"); + let int_type = tk_sys::Tcl_GetObjType(int_str.as_ptr() as _); + let int_type = if int_type.is_null() { + let mut value = *tk_sys::Tcl_NewWideIntObj(0); + let res = value.typePtr; + tk_sys::Tcl_DecrRefCount(&mut value); + res + } else { + int_type + }; + let wide_int_str = String::from("wideInt"); + let wide_int_type = tk_sys::Tcl_GetObjType(wide_int_str.as_ptr() as _); + let bignum_str = String::from("bignum"); + let bignum_type = tk_sys::Tcl_GetObjType(bignum_str.as_ptr() as _); + let list_str = String::from("list"); + let list_type = tk_sys::Tcl_GetObjType(list_str.as_ptr() as _); + let string_str = String::from("string"); + let string_type = tk_sys::Tcl_GetObjType(string_str.as_ptr() as _); + let utf32_str = String::from("utf32"); + let utf32_string_type = tk_sys::Tcl_GetObjType(utf32_str.as_ptr() as _); + let pixel_str = String::from("pixel"); + let pixel_type = tk_sys::Tcl_GetObjType(pixel_str.as_ptr() as _); + + let exit_str = String::from("exit"); + tk_sys::Tcl_DeleteCommand(interp, exit_str.as_ptr() as _); + + if let Some(name) = args.screen_name { + tk_sys::Tcl_SetVar2( + interp, + "env".as_ptr() as _, + "DISPLAY".as_ptr() as _, + name.as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } + + if args.interactive != 0 { + tk_sys::Tcl_SetVar2( + interp, + "tcl_interactive".as_ptr() as _, + ptr::null(), + "1".as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } else { + tk_sys::Tcl_SetVar2( + interp, + "tcl_interactive".as_ptr() as _, + ptr::null(), + "0".as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } + + let argv0 = args.class_name.clone().to_lowercase(); + tk_sys::Tcl_SetVar2( + interp, + "argv0".as_ptr() as _, + ptr::null(), + argv0.as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + + if !args.want_tk { + tk_sys::Tcl_SetVar2( + interp, + "_tkinter_skip_tk_init".as_ptr() as _, + ptr::null(), + "1".as_ptr() as _, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } + + if args.sync != 0 || args.use_.is_some() { + let mut argv = String::with_capacity(4); + if args.sync != 0 { + argv.push_str("-sync"); + } + if args.use_.is_some() { + if args.sync != 0 { + argv.push(' '); + } + argv.push_str("-use "); + argv.push_str(&args.use_.unwrap()); + } + argv.push('\0'); + let argv_ptr = argv.as_ptr() as *mut *mut i8; + tk_sys::Tcl_SetVar2( + interp, + "argv".as_ptr() as _, + ptr::null(), + argv_ptr as *const i8, + tk_sys::TCL_GLOBAL_ONLY as i32, + ); + } + + #[cfg(windows)] + { + let ret = std::env::var("TCL_LIBRARY"); + if ret.is_err() { + let loc = _get_tcl_lib_path(); + std::env::set_var("TCL_LIBRARY", loc); + } + } + + // Bindgen cannot handle Tcl_AppInit + if tk_sys::Tcl_Init(interp) != tk_sys::TCL_OK as ffi::c_int { + todo!("Tcl_Init failed"); + } + + Ok(TkApp { + interpreter: interp, + want_objects, + threaded, + thread_id: Some(thread_id), + dispatching, + trace, + old_boolean_type, + boolean_type, + byte_array_type, + double_type, + int_type, + wide_int_type, + bignum_type, + list_type, + string_type, + utf32_string_type, + pixel_type, + }) + } + } + + #[pyattr] + const READABLE: i32 = tk_sys::TCL_READABLE as i32; + #[pyattr] + const WRITABLE: i32 = tk_sys::TCL_WRITABLE as i32; + #[pyattr] + const EXCEPTION: i32 = tk_sys::TCL_EXCEPTION as i32; + + #[pyattr] + const TIMER_EVENTS: i32 = tk_sys::TCL_TIMER_EVENTS as i32; + #[pyattr] + const IDLE_EVENTS: i32 = tk_sys::TCL_IDLE_EVENTS as i32; + #[pyattr] + const DONT_WAIT: i32 = tk_sys::TCL_DONT_WAIT as i32; +} diff --git a/crates/stdlib/src/unicodedata.rs b/crates/stdlib/src/unicodedata.rs new file mode 100644 index 00000000000..5664fd0c36e --- /dev/null +++ b/crates/stdlib/src/unicodedata.rs @@ -0,0 +1,375 @@ +/* Access to the unicode database. + See also: https://docs.python.org/3/library/unicodedata.html +*/ + +// spell-checker:ignore nfkc unistr unidata + +pub(crate) use unicodedata::module_def; + +use crate::vm::{ + PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject, +}; + +enum NormalizeForm { + Nfc, + Nfkc, + Nfd, + Nfkd, +} + +impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + obj.try_value_with( + |form: &PyStr| match form.as_bytes() { + b"NFC" => Ok(Self::Nfc), + b"NFKC" => Ok(Self::Nfkc), + b"NFD" => Ok(Self::Nfd), + b"NFKD" => Ok(Self::Nfkd), + _ => Err(vm.new_value_error("invalid normalization form")), + }, + vm, + ) + } +} + +#[pymodule] +mod unicodedata { + use super::NormalizeForm::*; + use crate::vm::{ + Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyModule, PyStrRef}, + function::OptionalArg, + }; + use itertools::Itertools; + use rustpython_common::wtf8::{CodePoint, Wtf8Buf}; + use ucd::{Codepoint, DecompositionType, EastAsianWidth, Number, NumericType}; + use unic_char_property::EnumeratedCharProperty; + use unic_normal::StrNormalForm; + use unic_ucd_age::{Age, UNICODE_VERSION, UnicodeVersion}; + use unic_ucd_bidi::BidiClass; + use unic_ucd_category::GeneralCategory; + use unicode_bidi_mirroring::is_mirroring; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + // Add UCD methods as module-level functions + let ucd: PyObjectRef = Ucd::new(UNICODE_VERSION).into_ref(&vm.ctx).into(); + + for attr in [ + "category", + "lookup", + "name", + "bidirectional", + "combining", + "decimal", + "decomposition", + "digit", + "east_asian_width", + "is_normalized", + "mirrored", + "normalize", + "numeric", + ] { + module.set_attr(attr, ucd.get_attr(attr, vm)?, vm)?; + } + + Ok(()) + } + + #[pyattr] + #[pyclass(name = "UCD")] + #[derive(Debug, PyPayload)] + pub(super) struct Ucd { + unic_version: UnicodeVersion, + } + + impl Ucd { + pub const fn new(unic_version: UnicodeVersion) -> Self { + Self { unic_version } + } + + fn check_age(&self, c: CodePoint) -> bool { + c.to_char() + .is_none_or(|c| Age::of(c).is_some_and(|age| age.actual() <= self.unic_version)) + } + + fn extract_char( + &self, + character: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<Option<CodePoint>> { + let c = character + .as_wtf8() + .code_points() + .exactly_one() + .map_err(|_| vm.new_type_error("argument must be an unicode character, not str"))?; + + Ok(self.check_age(c).then_some(c)) + } + } + + #[pyclass(flags(DISALLOW_INSTANTIATION))] + impl Ucd { + #[pymethod] + fn category(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + Ok(self + .extract_char(character, vm)? + .map_or(GeneralCategory::Unassigned, |c| { + c.to_char() + .map_or(GeneralCategory::Surrogate, GeneralCategory::of) + }) + .abbr_name() + .to_owned()) + } + + #[pymethod] + fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + if let Some(name_str) = name.to_str() + && let Some(character) = unicode_names2::character(name_str) + && self.check_age(character.into()) + { + return Ok(character.to_string()); + } + Err(vm.new_key_error( + vm.ctx + .new_str(format!("undefined character name '{name}'")) + .into(), + )) + } + + #[pymethod] + fn name( + &self, + character: PyStrRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let c = self.extract_char(character, vm)?; + + if let Some(c) = c + && self.check_age(c) + && let Some(name) = c.to_char().and_then(unicode_names2::name) + { + return Ok(vm.ctx.new_str(name.to_string()).into()); + } + default.ok_or_else(|| vm.new_value_error("no such name")) + } + + #[pymethod] + fn bidirectional( + &self, + character: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<&'static str> { + let bidi = match self.extract_char(character, vm)? { + Some(c) => c + .to_char() + .map_or(BidiClass::LeftToRight, BidiClass::of) + .abbr_name(), + None => "", + }; + Ok(bidi) + } + + /// NOTE: This function uses 9.0.0 database instead of 3.2.0 + #[pymethod] + fn east_asian_width( + &self, + character: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<&'static str> { + Ok(self + .extract_char(character, vm)? + .and_then(|c| c.to_char()) + .map_or(EastAsianWidth::Neutral, |c| c.east_asian_width()) + .abbr_name()) + } + + #[pymethod] + fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<Wtf8Buf> { + let text = unistr.as_wtf8(); + let normalized_text = match form { + Nfc => text.map_utf8(|s| s.nfc()).collect(), + Nfkc => text.map_utf8(|s| s.nfkc()).collect(), + Nfd => text.map_utf8(|s| s.nfd()).collect(), + Nfkd => text.map_utf8(|s| s.nfkd()).collect(), + }; + Ok(normalized_text) + } + + #[pymethod] + fn is_normalized(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<bool> { + let text = unistr.as_wtf8(); + let normalized: Wtf8Buf = match form { + Nfc => text.map_utf8(|s| s.nfc()).collect(), + Nfkc => text.map_utf8(|s| s.nfkc()).collect(), + Nfd => text.map_utf8(|s| s.nfd()).collect(), + Nfkd => text.map_utf8(|s| s.nfkd()).collect(), + }; + Ok(text == &*normalized) + } + + #[pymethod] + fn mirrored(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> { + match self.extract_char(character, vm)? { + Some(c) => { + if let Some(ch) = c.to_char() { + // Check if the character is mirrored in bidirectional text using Unicode standard + Ok(if is_mirroring(ch) { 1 } else { 0 }) + } else { + Ok(0) + } + } + None => Ok(0), + } + } + + #[pymethod] + fn combining(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> { + Ok(self + .extract_char(character, vm)? + .and_then(|c| c.to_char()) + .map_or(0, |ch| ch.canonical_combining_class() as i32)) + } + + #[pymethod] + fn decomposition(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + let ch = match self.extract_char(character, vm)?.and_then(|c| c.to_char()) { + Some(ch) => ch, + None => return Ok(String::new()), + }; + let chars: Vec<char> = ch.decomposition_map().collect(); + // If decomposition maps to just the character itself, there's no decomposition + if chars.len() == 1 && chars[0] == ch { + return Ok(String::new()); + } + let hex_parts = chars.iter().map(|c| format!("{:04X}", *c as u32)).join(" "); + let tag = match ch.decomposition_type() { + Some(DecompositionType::Canonical) | None => return Ok(hex_parts), + Some(dt) => decomposition_type_tag(dt), + }; + Ok(format!("<{tag}> {hex_parts}")) + } + + #[pymethod] + fn digit( + &self, + character: PyStrRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char()); + if let Some(ch) = ch + && matches!( + ch.numeric_type(), + Some(NumericType::Decimal) | Some(NumericType::Digit) + ) + && let Some(Number::Integer(n)) = ch.numeric_value() + { + return Ok(vm.ctx.new_int(n).into()); + } + default.ok_or_else(|| vm.new_value_error("not a digit")) + } + + #[pymethod] + fn decimal( + &self, + character: PyStrRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char()); + if let Some(ch) = ch + && ch.numeric_type() == Some(NumericType::Decimal) + && let Some(Number::Integer(n)) = ch.numeric_value() + { + return Ok(vm.ctx.new_int(n).into()); + } + default.ok_or_else(|| vm.new_value_error("not a decimal")) + } + + #[pymethod] + fn numeric( + &self, + character: PyStrRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let ch = self.extract_char(character, vm)?.and_then(|c| c.to_char()); + if let Some(ch) = ch { + match ch.numeric_value() { + Some(Number::Integer(n)) => { + return Ok(vm.ctx.new_float(n as f64).into()); + } + Some(Number::Rational(num, den)) => { + return Ok(vm.ctx.new_float(num as f64 / den as f64).into()); + } + None => {} + } + } + default.ok_or_else(|| vm.new_value_error("not a numeric character")) + } + + #[pygetset] + fn unidata_version(&self) -> String { + self.unic_version.to_string() + } + } + + fn decomposition_type_tag(dt: DecompositionType) -> &'static str { + match dt { + DecompositionType::Canonical => "canonical", + DecompositionType::Compat => "compat", + DecompositionType::Circle => "circle", + DecompositionType::Final => "final", + DecompositionType::Font => "font", + DecompositionType::Fraction => "fraction", + DecompositionType::Initial => "initial", + DecompositionType::Isolated => "isolated", + DecompositionType::Medial => "medial", + DecompositionType::Narrow => "narrow", + DecompositionType::Nobreak => "noBreak", + DecompositionType::Small => "small", + DecompositionType::Square => "square", + DecompositionType::Sub => "sub", + DecompositionType::Super => "super", + DecompositionType::Vertical => "vertical", + DecompositionType::Wide => "wide", + } + } + + trait EastAsianWidthAbbrName { + fn abbr_name(&self) -> &'static str; + } + + impl EastAsianWidthAbbrName for EastAsianWidth { + fn abbr_name(&self) -> &'static str { + match self { + Self::Narrow => "Na", + Self::Wide => "W", + Self::Neutral => "N", + Self::Ambiguous => "A", + Self::FullWidth => "F", + Self::HalfWidth => "H", + } + } + } + + #[pyattr] + fn ucd_3_2_0(vm: &VirtualMachine) -> PyRef<Ucd> { + Ucd { + unic_version: UnicodeVersion { + major: 3, + minor: 2, + micro: 0, + }, + } + .into_ref(&vm.ctx) + } + + #[pyattr] + fn unidata_version(_vm: &VirtualMachine) -> String { + UNICODE_VERSION.to_string() + } +} diff --git a/crates/stdlib/src/uuid.rs b/crates/stdlib/src/uuid.rs new file mode 100644 index 00000000000..3bc01610d43 --- /dev/null +++ b/crates/stdlib/src/uuid.rs @@ -0,0 +1,36 @@ +pub(crate) use _uuid::module_def; + +#[pymodule] +mod _uuid { + use crate::{builtins::PyNone, vm::VirtualMachine}; + use mac_address::get_mac_address; + use std::sync::OnceLock; + use uuid::{Context, Uuid, timestamp::Timestamp}; + + fn get_node_id() -> [u8; 6] { + match get_mac_address() { + Ok(Some(_ma)) => get_mac_address().unwrap().unwrap().bytes(), + // os_random is expensive, but this is only ever called once + _ => rustpython_common::rand::os_random::<6>(), + } + } + + #[pyfunction] + fn generate_time_safe() -> (Vec<u8>, PyNone) { + static CONTEXT: Context = Context::new(0); + let ts = Timestamp::now(&CONTEXT); + + static NODE_ID: OnceLock<[u8; 6]> = OnceLock::new(); + let unique_node_id = NODE_ID.get_or_init(get_node_id); + + (Uuid::new_v1(ts, unique_node_id).as_bytes().to_vec(), PyNone) + } + + #[pyattr] + fn has_uuid_generate_time_safe(_vm: &VirtualMachine) -> u32 { + 0 + } + + #[pyattr(name = "has_stable_extractable_node")] + const HAS_STABLE_EXTRACTABLE_NODE: bool = false; +} diff --git a/crates/stdlib/src/zlib.rs b/crates/stdlib/src/zlib.rs new file mode 100644 index 00000000000..35a617ed152 --- /dev/null +++ b/crates/stdlib/src/zlib.rs @@ -0,0 +1,628 @@ +// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker + +pub(crate) use zlib::module_def; + +#[pymodule] +mod zlib { + use crate::compression::{ + _decompress, CompressFlushKind, CompressState, CompressStatusKind, Compressor, + DecompressArgs, DecompressError, DecompressFlushKind, DecompressState, DecompressStatus, + Decompressor, USE_AFTER_FINISH_ERR, flush_sync, + }; + use crate::vm::{ + Py, PyObject, PyPayload, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef, PyIntRef, PyType, PyTypeRef}, + common::lock::PyMutex, + convert::{ToPyException, TryFromBorrowedObject}, + function::{ArgBytesLike, ArgPrimitiveIndex, ArgSize, OptionalArg}, + types::Constructor, + }; + use adler32::RollingAdler32 as Adler32; + use flate2::{ + Compress, Compression, Decompress, FlushCompress, FlushDecompress, Status, + write::ZlibEncoder, + }; + use std::io::Write; + + #[pyattr] + use libz_sys::{ + Z_BEST_COMPRESSION, Z_BEST_SPEED, Z_BLOCK, Z_DEFAULT_COMPRESSION, Z_DEFAULT_STRATEGY, + Z_DEFLATED as DEFLATED, Z_FILTERED, Z_FINISH, Z_FIXED, Z_FULL_FLUSH, Z_HUFFMAN_ONLY, + Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_RLE, Z_SYNC_FLUSH, Z_TREES, + }; + + #[pyattr(name = "__version__")] + const __VERSION__: &str = "1.0"; + + // we're statically linking libz-rs, so the compile-time and runtime + // versions will always be the same + #[pyattr(name = "ZLIB_RUNTIME_VERSION")] + #[pyattr] + const ZLIB_VERSION: &str = unsafe { + match core::ffi::CStr::from_ptr(libz_sys::zlibVersion()).to_str() { + Ok(s) => s, + Err(_) => unreachable!(), + } + }; + + // copied from zlibmodule.c (commit 530f506ac91338) + #[pyattr] + const MAX_WBITS: i8 = 15; + #[pyattr] + const DEF_BUF_SIZE: usize = 16 * 1024; + #[pyattr] + const DEF_MEM_LEVEL: u8 = 8; + + #[pyattr(once)] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "zlib", + "error", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + #[pyfunction] + fn adler32(data: ArgBytesLike, begin_state: OptionalArg<PyIntRef>) -> u32 { + data.with_ref(|data| { + let begin_state = begin_state.map_or(1, |i| i.as_u32_mask()); + + let mut hasher = Adler32::from_value(begin_state); + hasher.update_buffer(data); + hasher.hash() + }) + } + + #[pyfunction] + fn crc32(data: ArgBytesLike, begin_state: OptionalArg<PyIntRef>) -> u32 { + crate::binascii::crc32(data, begin_state) + } + + #[derive(FromArgs)] + struct PyFuncCompressArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(any, default = Level::new(Z_DEFAULT_COMPRESSION))] + level: Level, + #[pyarg(any, default = ArgPrimitiveIndex { value: MAX_WBITS })] + wbits: ArgPrimitiveIndex<i8>, + } + + /// Returns a bytes object containing compressed data. + #[pyfunction] + fn compress(args: PyFuncCompressArgs, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + let PyFuncCompressArgs { + data, + level, + ref wbits, + } = args; + let level = level.ok_or_else(|| new_zlib_error("Bad compression level", vm))?; + + let compress = InitOptions::new(wbits.value, vm)?.compress(level); + let mut encoder = ZlibEncoder::new_with_compress(Vec::new(), compress); + data.with_ref(|input_bytes| encoder.write_all(input_bytes).unwrap()); + let encoded_bytes = encoder.finish().unwrap(); + Ok(vm.ctx.new_bytes(encoded_bytes)) + } + + enum InitOptions { + Standard { + header: bool, + // [De]Compress::new_with_window_bits is only enabled for zlib; miniz_oxide doesn't + // support wbits (yet?) + wbits: u8, + }, + Gzip { + wbits: u8, + }, + } + + impl InitOptions { + fn new(wbits: i8, vm: &VirtualMachine) -> PyResult<Self> { + let header = wbits > 0; + let wbits = wbits.unsigned_abs(); + match wbits { + // TODO: wbits = 0 should be a valid option: + // > windowBits can also be zero to request that inflate use the window size in + // > the zlib header of the compressed stream. + // but flate2 doesn't expose it + // 0 => ... + 9..=15 => Ok(Self::Standard { header, wbits }), + 25..=31 => Ok(Self::Gzip { wbits: wbits - 16 }), + _ => Err(vm.new_value_error("Invalid initialization option")), + } + } + + fn decompress(self) -> Decompress { + match self { + Self::Standard { header, wbits } => Decompress::new_with_window_bits(header, wbits), + Self::Gzip { wbits } => Decompress::new_gzip(wbits), + } + } + fn compress(self, level: Compression) -> Compress { + match self { + Self::Standard { header, wbits } => { + Compress::new_with_window_bits(level, header, wbits) + } + Self::Gzip { wbits } => Compress::new_gzip(level, wbits), + } + } + } + + #[derive(FromArgs)] + struct PyFuncDecompressArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(any, default = ArgPrimitiveIndex { value: MAX_WBITS })] + wbits: ArgPrimitiveIndex<i8>, + #[pyarg(any, default = ArgPrimitiveIndex { value: DEF_BUF_SIZE })] + bufsize: ArgPrimitiveIndex<usize>, + } + + /// Returns a bytes object containing the uncompressed data. + #[pyfunction] + fn decompress(args: PyFuncDecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let PyFuncDecompressArgs { + data, + wbits, + bufsize, + } = args; + data.with_ref(|data| { + let mut d = InitOptions::new(wbits.value, vm)?.decompress(); + let (buf, stream_end) = _decompress(data, &mut d, bufsize.value, None, flush_sync) + .map_err(|e| new_zlib_error(e.to_string(), vm))?; + if !stream_end { + return Err(new_zlib_error( + "Error -5 while decompressing data: incomplete or truncated stream", + vm, + )); + } + Ok(buf) + }) + } + + #[derive(FromArgs)] + struct DecompressobjArgs { + #[pyarg(any, default = ArgPrimitiveIndex { value: MAX_WBITS })] + wbits: ArgPrimitiveIndex<i8>, + #[pyarg(any, optional)] + zdict: OptionalArg<ArgBytesLike>, + } + + #[pyfunction] + fn decompressobj(args: DecompressobjArgs, vm: &VirtualMachine) -> PyResult<PyDecompress> { + let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); + let zdict = args.zdict.into_option(); + if let Some(dict) = &zdict + && args.wbits.value < 0 + { + dict.with_ref(|d| decompress.set_dictionary(d)) + .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; + } + let inner = PyDecompressInner { + decompress: Some(DecompressWithDict { decompress, zdict }), + eof: false, + unused_data: vm.ctx.empty_bytes.clone(), + unconsumed_tail: vm.ctx.empty_bytes.clone(), + }; + Ok(PyDecompress { + inner: PyMutex::new(inner), + }) + } + + #[derive(Debug)] + struct PyDecompressInner { + decompress: Option<DecompressWithDict>, + eof: bool, + unused_data: PyBytesRef, + unconsumed_tail: PyBytesRef, + } + + #[pyattr] + #[pyclass(name = "Decompress")] + #[derive(Debug, PyPayload)] + struct PyDecompress { + inner: PyMutex<PyDecompressInner>, + } + + #[pyclass(flags(DISALLOW_INSTANTIATION))] + impl PyDecompress { + #[pygetset] + fn eof(&self) -> bool { + self.inner.lock().eof + } + + #[pygetset] + fn unused_data(&self) -> PyBytesRef { + self.inner.lock().unused_data.clone() + } + + #[pygetset] + fn unconsumed_tail(&self) -> PyBytesRef { + self.inner.lock().unconsumed_tail.clone() + } + + fn decompress_inner( + inner: &mut PyDecompressInner, + data: &[u8], + bufsize: usize, + max_length: Option<usize>, + is_flush: bool, + vm: &VirtualMachine, + ) -> PyResult<(PyResult<Vec<u8>>, bool)> { + let Some(d) = &mut inner.decompress else { + return Err(new_zlib_error(USE_AFTER_FINISH_ERR, vm)); + }; + + let prev_in = d.total_in(); + let res = if is_flush { + // if is_flush: ignore zdict, finish if final chunk + let calc_flush = |final_chunk| { + if final_chunk { + FlushDecompress::Finish + } else { + FlushDecompress::None + } + }; + _decompress(data, &mut d.decompress, bufsize, max_length, calc_flush) + } else { + _decompress(data, d, bufsize, max_length, flush_sync) + } + .map_err(|e| new_zlib_error(e.to_string(), vm)); + let (ret, stream_end) = match res { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; + let consumed = (d.total_in() - prev_in) as usize; + + // save unused input + let unconsumed = &data[consumed..]; + if !unconsumed.is_empty() { + if stream_end { + let unused = [inner.unused_data.as_bytes(), unconsumed].concat(); + inner.unused_data = vm.ctx.new_pyref(unused); + } else { + inner.unconsumed_tail = vm.ctx.new_bytes(unconsumed.to_vec()); + } + } else if !inner.unconsumed_tail.is_empty() { + inner.unconsumed_tail = vm.ctx.empty_bytes.clone(); + } + + Ok((ret, stream_end)) + } + + #[pymethod] + fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let max_length: usize = args + .raw_max_length() + .unwrap_or(0) + .try_into() + .map_err(|_| vm.new_value_error("max_length must be non-negative"))?; + let max_length = (max_length != 0).then_some(max_length); + let data = &*args.data(); + + let inner = &mut *self.inner.lock(); + + let (ret, stream_end) = + Self::decompress_inner(inner, data, DEF_BUF_SIZE, max_length, false, vm)?; + + inner.eof |= stream_end; + + ret + } + + #[pymethod] + fn flush(&self, length: OptionalArg<ArgSize>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let length = match length { + OptionalArg::Present(ArgSize { value }) if value <= 0 => { + return Err(vm.new_value_error("length must be greater than zero")); + } + OptionalArg::Present(ArgSize { value }) => value as usize, + OptionalArg::Missing => DEF_BUF_SIZE, + }; + + let inner = &mut *self.inner.lock(); + let data = core::mem::replace(&mut inner.unconsumed_tail, vm.ctx.empty_bytes.clone()); + + let (ret, _) = Self::decompress_inner(inner, &data, length, None, true, vm)?; + + if inner.eof { + inner.decompress = None; + } + + ret + } + } + + #[derive(FromArgs)] + #[allow(dead_code)] // FIXME: use args + struct CompressobjArgs { + #[pyarg(any, default = Level::new(Z_DEFAULT_COMPRESSION))] + level: Level, + // only DEFLATED is valid right now, it's w/e + #[pyarg(any, default = DEFLATED)] + method: i32, + #[pyarg(any, default = ArgPrimitiveIndex { value: MAX_WBITS })] + wbits: ArgPrimitiveIndex<i8>, + #[pyarg(any, name = "memLevel", default = DEF_MEM_LEVEL)] + mem_level: u8, + #[pyarg(any, default = Z_DEFAULT_STRATEGY)] + strategy: i32, + #[pyarg(any, optional)] + zdict: Option<ArgBytesLike>, + } + + #[pyfunction] + fn compressobj(args: CompressobjArgs, vm: &VirtualMachine) -> PyResult<PyCompress> { + let CompressobjArgs { + level, + wbits, + zdict, + .. + } = args; + let level = level.ok_or_else(|| vm.new_value_error("invalid initialization option"))?; + #[allow(unused_mut)] + let mut compress = InitOptions::new(wbits.value, vm)?.compress(level); + if let Some(zdict) = zdict { + zdict.with_ref(|zdict| compress.set_dictionary(zdict).unwrap()); + } + Ok(PyCompress { + inner: PyMutex::new(CompressState::new(CompressInner::new(compress))), + }) + } + + #[derive(Debug)] + struct CompressInner { + compress: Compress, + } + + #[pyattr] + #[pyclass(name = "Compress")] + #[derive(Debug, PyPayload)] + struct PyCompress { + inner: PyMutex<CompressState<CompressInner>>, + } + + #[pyclass(flags(DISALLOW_INSTANTIATION))] + impl PyCompress { + #[pymethod] + fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut inner = self.inner.lock(); + data.with_ref(|b| inner.compress(b, vm)) + } + + #[pymethod] + fn flush(&self, mode: OptionalArg<i32>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mode = match mode.unwrap_or(Z_FINISH) { + Z_NO_FLUSH => return Ok(vec![]), + Z_PARTIAL_FLUSH => FlushCompress::Partial, + Z_SYNC_FLUSH => FlushCompress::Sync, + Z_FULL_FLUSH => FlushCompress::Full, + Z_FINISH => FlushCompress::Finish, + _ => return Err(new_zlib_error("invalid mode", vm)), + }; + self.inner.lock().flush(mode, vm) + } + + // TODO: This is an optional feature of Compress + // #[pymethod] + // #[pymethod(name = "__copy__")] + // #[pymethod(name = "__deepcopy__")] + // fn copy(&self) -> Self { + // todo!("<flate2::Compress as Clone>") + // } + } + + const CHUNKSIZE: usize = u32::MAX as usize; + + impl CompressInner { + const fn new(compress: Compress) -> Self { + Self { compress } + } + } + + impl CompressStatusKind for Status { + const EOF: Self = Self::StreamEnd; + + fn to_usize(self) -> usize { + self as usize + } + } + + impl CompressFlushKind for FlushCompress { + const NONE: Self = Self::None; + const FINISH: Self = Self::Finish; + + fn to_usize(self) -> usize { + self as usize + } + } + + impl Compressor for CompressInner { + type Status = Status; + type Flush = FlushCompress; + const CHUNKSIZE: usize = CHUNKSIZE; + const DEF_BUF_SIZE: usize = DEF_BUF_SIZE; + + fn compress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: Self::Flush, + vm: &VirtualMachine, + ) -> PyResult<Self::Status> { + self.compress + .compress_vec(input, output, flush) + .map_err(|_| new_zlib_error("error while compressing", vm)) + } + + fn total_in(&mut self) -> usize { + self.compress.total_in() as usize + } + + fn new_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef { + new_zlib_error(message, vm) + } + } + + fn new_zlib_error(message: impl Into<String>, vm: &VirtualMachine) -> PyBaseExceptionRef { + let msg: String = message.into(); + vm.new_exception_msg(vm.class("zlib", "error"), msg.into()) + } + + struct Level(Option<flate2::Compression>); + + impl Level { + fn new(level: i32) -> Self { + let compression = match level { + Z_DEFAULT_COMPRESSION => Compression::default(), + valid_level @ Z_NO_COMPRESSION..=Z_BEST_COMPRESSION => { + Compression::new(valid_level as u32) + } + _ => return Self(None), + }; + Self(Some(compression)) + } + + fn ok_or_else( + self, + f: impl FnOnce() -> PyBaseExceptionRef, + ) -> PyResult<flate2::Compression> { + self.0.ok_or_else(f) + } + } + + impl<'a> TryFromBorrowedObject<'a> for Level { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + let int: i32 = obj.try_index(vm)?.try_to_primitive(vm)?; + Ok(Self::new(int)) + } + } + + #[pyattr] + #[pyclass(name = "_ZlibDecompressor")] + #[derive(Debug, PyPayload)] + struct ZlibDecompressor { + inner: PyMutex<DecompressState<DecompressWithDict>>, + } + + #[derive(Debug)] + struct DecompressWithDict { + decompress: Decompress, + zdict: Option<ArgBytesLike>, + } + + impl DecompressStatus for Status { + fn is_stream_end(&self) -> bool { + *self == Self::StreamEnd + } + } + + impl DecompressFlushKind for FlushDecompress { + const SYNC: Self = Self::Sync; + } + + impl Decompressor for Decompress { + type Flush = FlushDecompress; + type Status = Status; + type Error = flate2::DecompressError; + + fn total_in(&self) -> u64 { + self.total_in() + } + + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: Self::Flush, + ) -> Result<Self::Status, Self::Error> { + self.decompress_vec(input, output, flush) + } + } + + impl Decompressor for DecompressWithDict { + type Flush = FlushDecompress; + type Status = Status; + type Error = flate2::DecompressError; + + fn total_in(&self) -> u64 { + self.decompress.total_in() + } + + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: Self::Flush, + ) -> Result<Self::Status, Self::Error> { + self.decompress.decompress_vec(input, output, flush) + } + + fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { + let zdict = err.needs_dictionary().and(self.zdict.as_ref()).ok_or(err)?; + self.decompress.set_dictionary(&zdict.borrow_buf())?; + Ok(()) + } + } + + // impl Deconstruct + + impl Constructor for ZlibDecompressor { + type Args = DecompressobjArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); + let zdict = args.zdict.into_option(); + if let Some(dict) = &zdict + && args.wbits.value < 0 + { + dict.with_ref(|d| decompress.set_dictionary(d)) + .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; + } + let inner = DecompressState::new(DecompressWithDict { decompress, zdict }, vm); + Ok(Self { + inner: PyMutex::new(inner), + }) + } + } + + #[pyclass(with(Constructor))] + impl ZlibDecompressor { + #[pygetset] + fn eof(&self) -> bool { + self.inner.lock().eof() + } + + #[pygetset] + fn unused_data(&self) -> PyBytesRef { + self.inner.lock().unused_data() + } + + #[pygetset] + fn needs_input(&self) -> bool { + self.inner.lock().needs_input() + } + + #[pymethod] + fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let max_length = args.max_length(); + let data = &*args.data(); + + let inner = &mut *self.inner.lock(); + + inner + .decompress(data, max_length, DEF_BUF_SIZE, vm) + .map_err(|e| match e { + DecompressError::Decompress(err) => new_zlib_error(err.to_string(), vm), + DecompressError::Eof(err) => err.to_pyexception(vm), + }) + } + + // TODO: Wait for getstate pyslot to be fixed + // #[pyslot] + // fn getstate(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyObject> { + // Err(vm.new_type_error("cannot serialize '_ZlibDecompressor' object".to_owned())) + // } + } +} diff --git a/crates/venvlauncher/Cargo.toml b/crates/venvlauncher/Cargo.toml new file mode 100644 index 00000000000..ac3ea106b7a --- /dev/null +++ b/crates/venvlauncher/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "rustpython-venvlauncher" +description = "Lightweight venv launcher for RustPython" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +# Free-threaded variants (RustPython uses Py_GIL_DISABLED=true) +[[bin]] +name = "venvlaunchert" +path = "src/main.rs" + +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = [ + "Win32_Foundation", + "Win32_System_Threading", + "Win32_System_Environment", + "Win32_Storage_FileSystem", + "Win32_System_Console", + "Win32_Security", +] } + +[lints] +workspace = true diff --git a/crates/venvlauncher/build.rs b/crates/venvlauncher/build.rs new file mode 100644 index 00000000000..404f46a484f --- /dev/null +++ b/crates/venvlauncher/build.rs @@ -0,0 +1,21 @@ +//! Build script for venvlauncher +//! +//! Sets the Windows subsystem to GUI for venvwlauncher variants. +//! Only MSVC toolchain is supported on Windows (same as CPython). + +fn main() { + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap(); + let target_env = std::env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); + + // Only apply on Windows with MSVC toolchain + if target_os == "windows" && target_env == "msvc" { + let exe_name = std::env::var("CARGO_BIN_NAME").unwrap_or_default(); + + // venvwlauncher and venvwlaunchert should be Windows GUI applications + // (no console window) + if exe_name.contains("venvw") { + println!("cargo:rustc-link-arg=/SUBSYSTEM:WINDOWS"); + println!("cargo:rustc-link-arg=/ENTRY:mainCRTStartup"); + } + } +} diff --git a/crates/venvlauncher/src/main.rs b/crates/venvlauncher/src/main.rs new file mode 100644 index 00000000000..7087e791e37 --- /dev/null +++ b/crates/venvlauncher/src/main.rs @@ -0,0 +1,155 @@ +//! RustPython venv launcher +//! +//! A lightweight launcher that reads pyvenv.cfg and delegates execution +//! to the actual Python interpreter. This mimics CPython's venvlauncher.c. +//! Windows only. + +#[cfg(not(windows))] +compile_error!("venvlauncher is only supported on Windows"); + +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +fn main() -> ExitCode { + match run() { + Ok(code) => ExitCode::from(code as u8), + Err(e) => { + eprintln!("venvlauncher error: {}", e); + ExitCode::from(1) + } + } +} + +fn run() -> Result<u32, Box<dyn core::error::Error>> { + // 1. Get own executable path + let exe_path = env::current_exe()?; + let exe_name = exe_path + .file_name() + .ok_or("Failed to get executable name")? + .to_string_lossy(); + + // 2. Determine target executable name based on launcher name + // pythonw.exe / venvwlauncher -> pythonw.exe (GUI, no console) + // python.exe / venvlauncher -> python.exe (console) + let exe_name_lower = exe_name.to_lowercase(); + let target_exe = if exe_name_lower.contains("pythonw") || exe_name_lower.contains("venvw") { + "pythonw.exe" + } else { + "python.exe" + }; + + // 3. Find pyvenv.cfg + // The launcher is in Scripts/ directory, pyvenv.cfg is in parent (venv root) + let scripts_dir = exe_path.parent().ok_or("Failed to get Scripts directory")?; + let venv_dir = scripts_dir.parent().ok_or("Failed to get venv directory")?; + let cfg_path = venv_dir.join("pyvenv.cfg"); + + if !cfg_path.exists() { + return Err(format!("pyvenv.cfg not found: {}", cfg_path.display()).into()); + } + + // 4. Parse home= from pyvenv.cfg + let home = read_home(&cfg_path)?; + + // 5. Locate python executable in home directory + let python_path = PathBuf::from(&home).join(target_exe); + if !python_path.exists() { + return Err(format!("Python not found: {}", python_path.display()).into()); + } + + // 6. Set __PYVENV_LAUNCHER__ environment variable + // This tells Python it was launched from a venv + // SAFETY: We are in a single-threaded context (program entry point) + unsafe { + env::set_var("__PYVENV_LAUNCHER__", &exe_path); + } + + // 7. Launch Python with same arguments + let args: Vec<String> = env::args().skip(1).collect(); + launch_process(&python_path, &args) +} + +/// Parse the `home=` value from pyvenv.cfg +fn read_home(cfg_path: &Path) -> Result<String, Box<dyn core::error::Error>> { + let content = fs::read_to_string(cfg_path)?; + + for line in content.lines() { + let line = line.trim(); + // Skip comments and empty lines + if line.is_empty() || line.starts_with('#') { + continue; + } + + // Look for "home = <path>" or "home=<path>" + if let Some(rest) = line.strip_prefix("home") { + let rest = rest.trim_start(); + if let Some(value) = rest.strip_prefix('=') { + return Ok(value.trim().to_string()); + } + } + } + + Err("'home' key not found in pyvenv.cfg".into()) +} + +/// Launch the Python process and wait for it to complete +fn launch_process(exe: &Path, args: &[String]) -> Result<u32, Box<dyn core::error::Error>> { + use std::process::Command; + + let status = Command::new(exe).args(args).status()?; + + Ok(status.code().unwrap_or(1) as u32) +} + +#[cfg(all(test, windows))] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn test_read_home() { + let temp_dir = std::env::temp_dir(); + let cfg_path = temp_dir.join("test_pyvenv.cfg"); + + let mut file = fs::File::create(&cfg_path).unwrap(); + writeln!(file, "home = C:\\Python314").unwrap(); + writeln!(file, "include-system-site-packages = false").unwrap(); + writeln!(file, "version = 3.14.0").unwrap(); + + let home = read_home(&cfg_path).unwrap(); + assert_eq!(home, "C:\\Python314"); + + fs::remove_file(&cfg_path).unwrap(); + } + + #[test] + fn test_read_home_no_spaces() { + let temp_dir = std::env::temp_dir(); + let cfg_path = temp_dir.join("test_pyvenv2.cfg"); + + let mut file = fs::File::create(&cfg_path).unwrap(); + writeln!(file, "home=C:\\Python313").unwrap(); + + let home = read_home(&cfg_path).unwrap(); + assert_eq!(home, "C:\\Python313"); + + fs::remove_file(&cfg_path).unwrap(); + } + + #[test] + fn test_read_home_with_comments() { + let temp_dir = std::env::temp_dir(); + let cfg_path = temp_dir.join("test_pyvenv3.cfg"); + + let mut file = fs::File::create(&cfg_path).unwrap(); + writeln!(file, "# This is a comment").unwrap(); + writeln!(file, "home = D:\\RustPython").unwrap(); + + let home = read_home(&cfg_path).unwrap(); + assert_eq!(home, "D:\\RustPython"); + + fs::remove_file(&cfg_path).unwrap(); + } +} diff --git a/crates/vm/Cargo.toml b/crates/vm/Cargo.toml new file mode 100644 index 00000000000..5f7e901b834 --- /dev/null +++ b/crates/vm/Cargo.toml @@ -0,0 +1,155 @@ +[package] +name = "rustpython-vm" +description = "RustPython virtual machine." +include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"] +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[features] +default = ["compiler", "wasmbind", "gc", "host_env", "stdio"] +host_env = [] +stdio = [] +importlib = [] +encodings = ["importlib"] +vm-tracing-logging = [] +flame-it = ["flame", "flamer"] +freeze-stdlib = ["encodings"] +jit = ["rustpython-jit"] +threading = ["rustpython-common/threading"] +gc = [] +compiler = ["parser", "codegen", "rustpython-compiler"] +ast = ["ruff_python_ast", "ruff_text_size"] +codegen = ["rustpython-codegen", "ast"] +parser = ["ast"] +serde = ["dep:serde"] +wasmbind = ["rustpython-common/wasm_js", "chrono/wasmbind", "wasm-bindgen"] + +[dependencies] +rustpython-compiler = { workspace = true, optional = true } +rustpython-codegen = { workspace = true, optional = true } +rustpython-common = { workspace = true } +rustpython-derive = { workspace = true } +rustpython-jit = { workspace = true, optional = true } + +ruff_python_ast = { workspace = true, optional = true } +ruff_python_parser = { workspace = true } +ruff_text_size = { workspace = true, optional = true } +rustpython-compiler-core = { workspace = true } +rustpython-literal = { workspace = true } +rustpython-sre_engine = { workspace = true } + +ascii = { workspace = true } +ahash = { workspace = true } +bitflags = { workspace = true } +bstr = { workspace = true } +cfg-if = { workspace = true } +crossbeam-utils = { workspace = true } +chrono = { workspace = true } +constant_time_eq = { workspace = true } +flame = { workspace = true, optional = true } +getrandom = { workspace = true } +hex = { workspace = true } +indexmap = { workspace = true } +itertools = { workspace = true } +is-macro = { workspace = true } +libc = { workspace = true } +log = { workspace = true } +malachite-bigint = { workspace = true } +num-complex = { workspace = true } +num-integer = { workspace = true } +num-traits = { workspace = true } +num_enum = { workspace = true } +parking_lot = { workspace = true } +paste = { workspace = true } +scoped-tls = { workspace = true } +scopeguard = { workspace = true } +serde = { workspace = true, optional = true } +static_assertions = { workspace = true } +strum = { workspace = true } +strum_macros = { workspace = true } +thiserror = { workspace = true } +thread_local = { workspace = true } +memchr = { workspace = true } + +caseless = "0.2.2" +flamer = { version = "0.5", optional = true } +half = "2" +psm = "0.1" +optional = { workspace = true } +result-like = "0.5.0" +timsort = "0.1.2" + +## unicode stuff +# TODO: use unic for this; needed for title case: +# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 +unicode-casing = { workspace = true } +# update version all at the same time +unic-ucd-bidi = { workspace = true } +unic-ucd-category = { workspace = true } +unic-ucd-ident = { workspace = true } + +[target.'cfg(unix)'.dependencies] +rustix = { workspace = true } +nix = { workspace = true } +exitcode = "1.1.2" +uname = "0.1.1" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +rustyline = { workspace = true } +which = "8" +errno = "0.3" +widestring = { workspace = true } + +[target.'cfg(all(any(target_os = "linux", target_os = "macos", target_os = "windows", target_os = "android"), not(any(target_env = "musl", target_env = "sgx"))))'.dependencies] +libffi = { workspace = true, features = ["system"] } +libloading = "0.9" + +[target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] +num_cpus = "1.17.0" + +[target.'cfg(windows)'.dependencies] +junction = { workspace = true } + +[target.'cfg(windows)'.dependencies.windows-sys] +workspace = true +features = [ + "Win32_Foundation", + "Win32_Globalization", + "Win32_Media_Audio", + "Win32_Networking_WinSock", + "Win32_Security", + "Win32_Security_Authorization", + "Win32_Storage_FileSystem", + "Win32_System_Console", + "Win32_System_Diagnostics_Debug", + "Win32_System_Environment", + "Win32_System_IO", + "Win32_System_Ioctl", + "Win32_System_Kernel", + "Win32_System_LibraryLoader", + "Win32_System_Memory", + "Win32_System_Performance", + "Win32_System_Pipes", + "Win32_System_Registry", + "Win32_System_SystemInformation", + "Win32_System_SystemServices", + "Win32_System_Threading", + "Win32_System_Time", + "Win32_System_WindowsProgramming", + "Win32_UI_Shell", + "Win32_UI_WindowsAndMessaging", +] + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +wasm-bindgen = { workspace = true, optional = true } + +[build-dependencies] +glob = { workspace = true } +itertools = { workspace = true } + +[lints] +workspace = true diff --git a/vm/Lib/README.md b/crates/vm/Lib/README.md similarity index 100% rename from vm/Lib/README.md rename to crates/vm/Lib/README.md diff --git a/crates/vm/Lib/core_modules/codecs.py b/crates/vm/Lib/core_modules/codecs.py new file mode 120000 index 00000000000..4a96fdd182d --- /dev/null +++ b/crates/vm/Lib/core_modules/codecs.py @@ -0,0 +1 @@ +../../../../Lib/codecs.py \ No newline at end of file diff --git a/crates/vm/Lib/core_modules/copyreg.py b/crates/vm/Lib/core_modules/copyreg.py new file mode 120000 index 00000000000..6f4f0d4d445 --- /dev/null +++ b/crates/vm/Lib/core_modules/copyreg.py @@ -0,0 +1 @@ +../../../../Lib/copyreg.py \ No newline at end of file diff --git a/crates/vm/Lib/core_modules/encodings_ascii.py b/crates/vm/Lib/core_modules/encodings_ascii.py new file mode 120000 index 00000000000..c0507e75b03 --- /dev/null +++ b/crates/vm/Lib/core_modules/encodings_ascii.py @@ -0,0 +1 @@ +../../../../Lib/encodings/ascii.py \ No newline at end of file diff --git a/crates/vm/Lib/core_modules/encodings_utf_8.py b/crates/vm/Lib/core_modules/encodings_utf_8.py new file mode 120000 index 00000000000..0a82f3b4a41 --- /dev/null +++ b/crates/vm/Lib/core_modules/encodings_utf_8.py @@ -0,0 +1 @@ +../../../../Lib/encodings/utf_8.py \ No newline at end of file diff --git a/crates/vm/Lib/python_builtins/__hello__.py b/crates/vm/Lib/python_builtins/__hello__.py new file mode 120000 index 00000000000..e7dedd3d0aa --- /dev/null +++ b/crates/vm/Lib/python_builtins/__hello__.py @@ -0,0 +1 @@ +../../../../Lib/__hello__.py \ No newline at end of file diff --git a/crates/vm/Lib/python_builtins/__phello__ b/crates/vm/Lib/python_builtins/__phello__ new file mode 120000 index 00000000000..21833dd18d8 --- /dev/null +++ b/crates/vm/Lib/python_builtins/__phello__ @@ -0,0 +1 @@ +../../../../Lib/__phello__/ \ No newline at end of file diff --git a/crates/vm/Lib/python_builtins/_frozen_importlib.py b/crates/vm/Lib/python_builtins/_frozen_importlib.py new file mode 120000 index 00000000000..9d752e80dec --- /dev/null +++ b/crates/vm/Lib/python_builtins/_frozen_importlib.py @@ -0,0 +1 @@ +../../../../Lib/importlib/_bootstrap.py \ No newline at end of file diff --git a/crates/vm/Lib/python_builtins/_frozen_importlib_external.py b/crates/vm/Lib/python_builtins/_frozen_importlib_external.py new file mode 120000 index 00000000000..a6b01510674 --- /dev/null +++ b/crates/vm/Lib/python_builtins/_frozen_importlib_external.py @@ -0,0 +1 @@ +../../../../Lib/importlib/_bootstrap_external.py \ No newline at end of file diff --git a/crates/vm/Lib/python_builtins/_thread.py b/crates/vm/Lib/python_builtins/_thread.py new file mode 120000 index 00000000000..9079ca9fda3 --- /dev/null +++ b/crates/vm/Lib/python_builtins/_thread.py @@ -0,0 +1 @@ +../../../../Lib/_dummy_thread.py \ No newline at end of file diff --git a/crates/vm/build.rs b/crates/vm/build.rs new file mode 100644 index 00000000000..f76bf3f5cbd --- /dev/null +++ b/crates/vm/build.rs @@ -0,0 +1,74 @@ +use itertools::Itertools; +use std::{env, io::prelude::*, path::PathBuf, process::Command}; + +fn main() { + let frozen_libs = if cfg!(feature = "freeze-stdlib") { + "Lib/*/*.py" + } else { + "Lib/python_builtins/*.py" + }; + for entry in glob::glob(frozen_libs).expect("Lib/ exists?").flatten() { + let display = entry.display(); + println!("cargo:rerun-if-changed={display}"); + } + println!("cargo:rerun-if-changed=../../Lib/importlib/_bootstrap.py"); + + println!("cargo:rustc-env=RUSTPYTHON_GIT_HASH={}", git_hash()); + println!( + "cargo:rustc-env=RUSTPYTHON_GIT_TIMESTAMP={}", + git_timestamp() + ); + println!("cargo:rustc-env=RUSTPYTHON_GIT_TAG={}", git_tag()); + println!("cargo:rustc-env=RUSTPYTHON_GIT_BRANCH={}", git_branch()); + println!("cargo:rustc-env=RUSTC_VERSION={}", rustc_version()); + + println!( + "cargo:rustc-env=RUSTPYTHON_TARGET_TRIPLE={}", + env::var("TARGET").unwrap() + ); + + let mut env_path = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + env_path.push("env_vars.rs"); + let mut f = std::fs::File::create(env_path).unwrap(); + write!( + f, + "sysvars! {{ {} }}", + std::env::vars_os().format_with(", ", |(k, v), f| f(&format_args!("{k:?} => {v:?}"))) + ) + .unwrap(); +} + +fn git_hash() -> String { + git(&["rev-parse", "--short", "HEAD"]) +} + +fn git_timestamp() -> String { + git(&["log", "-1", "--format=%ct"]) +} + +fn git_tag() -> String { + git(&["describe", "--all", "--always", "--dirty"]) +} + +fn git_branch() -> String { + git(&["name-rev", "--name-only", "HEAD"]) +} + +fn git(args: &[&str]) -> String { + command("git", args) +} + +fn rustc_version() -> String { + let rustc = env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + command(rustc, &["-V"]) +} + +fn command(cmd: impl AsRef<std::ffi::OsStr>, args: &[&str]) -> String { + match Command::new(cmd).args(args).output() { + Ok(output) => match String::from_utf8(output.stdout) { + Ok(s) => s, + Err(err) => format!("(output error: {err})"), + }, + Err(err) => format!("(command error: {err})"), + } +} diff --git a/crates/vm/src/anystr.rs b/crates/vm/src/anystr.rs new file mode 100644 index 00000000000..79b62a58abf --- /dev/null +++ b/crates/vm/src/anystr.rs @@ -0,0 +1,462 @@ +use crate::{ + Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyIntRef, PyTuple}, + convert::TryFromBorrowedObject, + function::OptionalOption, +}; +use num_traits::{cast::ToPrimitive, sign::Signed}; + +use core::ops::Range; + +#[derive(FromArgs)] +pub struct SplitArgs<T: TryFromObject> { + #[pyarg(any, default)] + sep: Option<T>, + #[pyarg(any, default = -1)] + maxsplit: isize, +} + +#[derive(FromArgs)] +pub struct SplitLinesArgs { + #[pyarg(any, default = false)] + pub keepends: bool, +} + +#[derive(FromArgs)] +pub struct ExpandTabsArgs { + #[pyarg(any, default = 8)] + tabsize: isize, +} + +impl ExpandTabsArgs { + pub fn tabsize(&self) -> usize { + self.tabsize.to_usize().unwrap_or(0) + } +} + +#[derive(FromArgs)] +pub struct StartsEndsWithArgs { + #[pyarg(positional)] + affix: PyObjectRef, + #[pyarg(positional, default)] + start: Option<PyIntRef>, + #[pyarg(positional, default)] + end: Option<PyIntRef>, +} + +impl StartsEndsWithArgs { + pub fn get_value(self, len: usize) -> (PyObjectRef, Option<Range<usize>>) { + let range = if self.start.is_some() || self.end.is_some() { + Some(adjust_indices(self.start, self.end, len)) + } else { + None + }; + (self.affix, range) + } + + #[inline] + pub fn prepare<S, F>(self, s: &S, len: usize, substr: F) -> Option<(PyObjectRef, &S)> + where + S: ?Sized + AnyStr, + F: Fn(&S, Range<usize>) -> &S, + { + let (affix, range) = self.get_value(len); + let substr = if let Some(range) = range { + if !range.is_normal() { + return None; + } + substr(s, range) + } else { + s + }; + Some((affix, substr)) + } +} + +fn saturate_to_isize(py_int: PyIntRef) -> isize { + let big = py_int.as_bigint(); + big.to_isize().unwrap_or_else(|| { + if big.is_negative() { + isize::MIN + } else { + isize::MAX + } + }) +} + +// help get optional string indices +pub fn adjust_indices(start: Option<PyIntRef>, end: Option<PyIntRef>, len: usize) -> Range<usize> { + let mut start = start.map_or(0, saturate_to_isize); + let mut end = end.map_or(len as isize, saturate_to_isize); + if end > len as isize { + end = len as isize; + } else if end < 0 { + end += len as isize; + if end < 0 { + end = 0; + } + } + if start < 0 { + start += len as isize; + if start < 0 { + start = 0; + } + } + start as usize..end as usize +} + +pub trait StringRange { + fn is_normal(&self) -> bool; +} + +impl StringRange for Range<usize> { + fn is_normal(&self) -> bool { + self.start <= self.end + } +} + +pub trait AnyStrWrapper<S: AnyStr + ?Sized> { + fn as_ref(&self) -> Option<&S>; + fn is_empty(&self) -> bool; +} + +pub trait AnyStrContainer<S> +where + S: ?Sized, +{ + fn new() -> Self; + fn with_capacity(capacity: usize) -> Self; + fn push_str(&mut self, s: &S); +} + +pub trait AnyChar: Copy { + fn is_lowercase(self) -> bool; + fn is_uppercase(self) -> bool; + fn bytes_len(self) -> usize; +} + +pub trait AnyStr { + type Char: AnyChar; + type Container: AnyStrContainer<Self> + Extend<Self::Char>; + + fn to_container(&self) -> Self::Container; + fn as_bytes(&self) -> &[u8]; + fn elements(&self) -> impl Iterator<Item = Self::Char>; + fn get_bytes(&self, range: Range<usize>) -> &Self; + // FIXME: get_chars is expensive for str + fn get_chars(&self, range: Range<usize>) -> &Self; + fn bytes_len(&self) -> usize; + // NOTE: str::chars().count() consumes the O(n) time. But pystr::char_len does cache. + // So using chars_len directly is too expensive and the below method shouldn't be implemented. + // fn chars_len(&self) -> usize; + fn is_empty(&self) -> bool; + + fn py_add(&self, other: &Self) -> Self::Container { + let mut new = Self::Container::with_capacity(self.bytes_len() + other.bytes_len()); + new.push_str(self); + new.push_str(other); + new + } + + fn py_split<T, SP, SN, SW>( + &self, + args: SplitArgs<T>, + vm: &VirtualMachine, + full_obj: impl FnOnce() -> PyObjectRef, + split: SP, + splitn: SN, + split_whitespace: SW, + ) -> PyResult<Vec<PyObjectRef>> + where + T: TryFromObject + AnyStrWrapper<Self>, + SP: Fn(&Self, &Self, &VirtualMachine) -> Vec<PyObjectRef>, + SN: Fn(&Self, &Self, usize, &VirtualMachine) -> Vec<PyObjectRef>, + SW: Fn(&Self, isize, &VirtualMachine) -> Vec<PyObjectRef>, + { + if args.sep.as_ref().is_some_and(|sep| sep.is_empty()) { + return Err(vm.new_value_error("empty separator")); + } + let splits = if let Some(pattern) = args.sep { + let Some(pattern) = pattern.as_ref() else { + return Ok(vec![full_obj()]); + }; + if args.maxsplit < 0 { + split(self, pattern, vm) + } else { + splitn(self, pattern, (args.maxsplit + 1) as usize, vm) + } + } else { + split_whitespace(self, args.maxsplit, vm) + }; + Ok(splits) + } + fn py_split_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef; + fn py_rsplit_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef; + + #[inline] + fn py_starts_ends_with<'a, T, F>( + &self, + affix: &'a PyObject, + func_name: &str, + py_type_name: &str, + func: F, + vm: &VirtualMachine, + ) -> PyResult<bool> + where + T: TryFromBorrowedObject<'a>, + F: Fn(&Self, T) -> bool, + { + single_or_tuple_any( + affix, + &|s: T| Ok(func(self, s)), + &|o| { + format!( + "{} first arg must be {} or a tuple of {}, not {}", + func_name, + py_type_name, + py_type_name, + o.class(), + ) + }, + vm, + ) + } + + #[inline] + fn py_strip<'a, S, FC, FD>( + &'a self, + chars: OptionalOption<S>, + func_chars: FC, + func_default: FD, + ) -> &'a Self + where + S: AnyStrWrapper<Self>, + FC: Fn(&'a Self, &Self) -> &'a Self, + FD: Fn(&'a Self) -> &'a Self, + { + let chars = chars.flatten(); + match chars { + Some(chars) => { + if let Some(chars) = chars.as_ref() { + func_chars(self, chars) + } else { + self + } + } + None => func_default(self), + } + } + + #[inline] + fn py_find<F>(&self, needle: &Self, range: Range<usize>, find: F) -> Option<usize> + where + F: Fn(&Self, &Self) -> Option<usize>, + { + if range.is_normal() { + let start = range.start; + let index = find(self.get_chars(range), needle)?; + Some(start + index) + } else { + None + } + } + + #[inline] + fn py_count<F>(&self, needle: &Self, range: Range<usize>, count: F) -> usize + where + F: Fn(&Self, &Self) -> usize, + { + if range.is_normal() { + count(self.get_chars(range), needle) + } else { + 0 + } + } + + fn py_pad(&self, left: usize, right: usize, fillchar: Self::Char) -> Self::Container { + let mut u = Self::Container::with_capacity( + (left + right) * fillchar.bytes_len() + self.bytes_len(), + ); + u.extend(core::iter::repeat_n(fillchar, left)); + u.push_str(self); + u.extend(core::iter::repeat_n(fillchar, right)); + u + } + + fn py_center(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { + let marg = width - len; + let left = marg / 2 + (marg & width & 1); + self.py_pad(left, marg - left, fillchar) + } + + fn py_ljust(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { + self.py_pad(0, width - len, fillchar) + } + + fn py_rjust(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { + self.py_pad(width - len, 0, fillchar) + } + + fn py_join( + &self, + mut iter: impl core::iter::Iterator<Item = PyResult<impl AnyStrWrapper<Self> + TryFromObject>>, + ) -> PyResult<Self::Container> { + let mut joined = if let Some(elem) = iter.next() { + elem?.as_ref().unwrap().to_container() + } else { + return Ok(Self::Container::new()); + }; + for elem in iter { + let elem = elem?; + joined.push_str(self); + joined.push_str(elem.as_ref().unwrap()); + } + Ok(joined) + } + + fn py_partition<'a, F, S>( + &'a self, + sub: &Self, + split: F, + vm: &VirtualMachine, + ) -> PyResult<(Self::Container, bool, Self::Container)> + where + F: Fn() -> S, + S: core::iter::Iterator<Item = &'a Self>, + { + if sub.is_empty() { + return Err(vm.new_value_error("empty separator")); + } + + let mut sp = split(); + let front = sp.next().unwrap().to_container(); + let (has_mid, back) = if let Some(back) = sp.next() { + (true, back.to_container()) + } else { + (false, Self::Container::new()) + }; + Ok((front, has_mid, back)) + } + + fn py_removeprefix<FC>(&self, prefix: &Self, prefix_len: usize, is_prefix: FC) -> &Self + where + FC: Fn(&Self, &Self) -> bool, + { + //if self.py_starts_with(prefix) { + if is_prefix(self, prefix) { + self.get_bytes(prefix_len..self.bytes_len()) + } else { + self + } + } + + fn py_removesuffix<FC>(&self, suffix: &Self, suffix_len: usize, is_suffix: FC) -> &Self + where + FC: Fn(&Self, &Self) -> bool, + { + if is_suffix(self, suffix) { + self.get_bytes(0..self.bytes_len() - suffix_len) + } else { + self + } + } + + // TODO: remove this function from anystr. + // See https://github.com/RustPython/RustPython/pull/4709/files#r1141013993 + fn py_bytes_splitlines<FW, W>(&self, options: SplitLinesArgs, into_wrapper: FW) -> Vec<W> + where + FW: Fn(&Self) -> W, + { + let keep = options.keepends as usize; + let mut elements = Vec::new(); + let mut last_i = 0; + let mut enumerated = self.as_bytes().iter().enumerate().peekable(); + while let Some((i, ch)) = enumerated.next() { + let (end_len, i_diff) = match *ch { + b'\n' => (keep, 1), + b'\r' => { + let is_rn = enumerated.next_if(|(_, ch)| **ch == b'\n').is_some(); + if is_rn { (keep + keep, 2) } else { (keep, 1) } + } + _ => continue, + }; + let range = last_i..i + end_len; + last_i = i + i_diff; + elements.push(into_wrapper(self.get_bytes(range))); + } + if last_i != self.bytes_len() { + elements.push(into_wrapper(self.get_bytes(last_i..self.bytes_len()))); + } + elements + } + + fn py_zfill(&self, width: isize) -> Vec<u8> { + let width = width.to_usize().unwrap_or(0); + rustpython_common::str::zfill(self.as_bytes(), width) + } + + // Unified form of CPython functions: + // _Py_bytes_islower + // unicode_islower_impl + fn py_islower(&self) -> bool { + let mut lower = false; + for c in self.elements() { + if c.is_uppercase() { + return false; + } else if !lower && c.is_lowercase() { + lower = true + } + } + lower + } + + // Unified form of CPython functions: + // Py_bytes_isupper + // unicode_isupper_impl + fn py_isupper(&self) -> bool { + let mut upper = false; + for c in self.elements() { + if c.is_lowercase() { + return false; + } else if !upper && c.is_uppercase() { + upper = true + } + } + upper + } +} + +/// Tests that the predicate is True on a single value, or if the value is a tuple a tuple, then +/// test that any of the values contained within the tuples satisfies the predicate. Type parameter +/// T specifies the type that is expected, if the input value is not of that type or a tuple of +/// values of that type, then a TypeError is raised. +pub fn single_or_tuple_any<'a, T, F, M>( + obj: &'a PyObject, + predicate: &F, + message: &M, + vm: &VirtualMachine, +) -> PyResult<bool> +where + T: TryFromBorrowedObject<'a>, + F: Fn(T) -> PyResult<bool>, + M: Fn(&PyObject) -> String, +{ + match obj.try_to_value::<T>(vm) { + Ok(single) => (predicate)(single), + Err(_) => { + let tuple: &Py<PyTuple> = obj + .try_to_value(vm) + .map_err(|_| vm.new_type_error((message)(obj)))?; + for obj in tuple { + if single_or_tuple_any(obj, predicate, message, vm)? { + return Ok(true); + } + } + Ok(false) + } + } +} diff --git a/crates/vm/src/buffer.rs b/crates/vm/src/buffer.rs new file mode 100644 index 00000000000..db58c909bca --- /dev/null +++ b/crates/vm/src/buffer.rs @@ -0,0 +1,730 @@ +use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef, PyTuple, PyTupleRef, PyTypeRef}, + common::{static_cell, str::wchar_t}, + convert::ToPyObject, + function::{ArgBytesLike, ArgIntoBool, ArgIntoFloat}, +}; +use alloc::fmt; +use core::{iter::Peekable, mem}; +use half::f16; +use itertools::Itertools; +use malachite_bigint::BigInt; +use num_traits::{PrimInt, ToPrimitive}; +use std::os::raw; + +type PackFunc = fn(&VirtualMachine, PyObjectRef, &mut [u8]) -> PyResult<()>; +type UnpackFunc = fn(&VirtualMachine, &[u8]) -> PyObjectRef; + +static OVERFLOW_MSG: &str = "total struct size too long"; // not a const to reduce code size + +#[derive(Debug, Copy, Clone, PartialEq)] +pub(crate) enum Endianness { + Native, + Little, + Big, + Host, +} + +impl Endianness { + /// Parse endianness + /// See also: https://docs.python.org/3/library/struct.html?highlight=struct#byte-order-size-and-alignment + fn parse<I>(chars: &mut Peekable<I>) -> Self + where + I: Sized + Iterator<Item = u8>, + { + let e = match chars.peek() { + Some(b'@') => Self::Native, + Some(b'=') => Self::Host, + Some(b'<') => Self::Little, + Some(b'>') | Some(b'!') => Self::Big, + _ => return Self::Native, + }; + chars.next().unwrap(); + e + } +} + +trait ByteOrder { + fn convert<I: PrimInt>(i: I) -> I; +} +enum BigEndian {} +impl ByteOrder for BigEndian { + fn convert<I: PrimInt>(i: I) -> I { + i.to_be() + } +} +enum LittleEndian {} +impl ByteOrder for LittleEndian { + fn convert<I: PrimInt>(i: I) -> I { + i.to_le() + } +} + +#[cfg(target_endian = "big")] +type NativeEndian = BigEndian; +#[cfg(target_endian = "little")] +type NativeEndian = LittleEndian; + +#[derive(Copy, Clone, num_enum::TryFromPrimitive)] +#[repr(u8)] +pub(crate) enum FormatType { + Pad = b'x', + SByte = b'b', + UByte = b'B', + Char = b'c', + WideChar = b'u', + Str = b's', + Pascal = b'p', + Short = b'h', + UShort = b'H', + Int = b'i', + UInt = b'I', + Long = b'l', + ULong = b'L', + SSizeT = b'n', + SizeT = b'N', + LongLong = b'q', + ULongLong = b'Q', + Bool = b'?', + Half = b'e', + Float = b'f', + Double = b'd', + LongDouble = b'g', + VoidP = b'P', + PyObject = b'O', +} + +impl fmt::Debug for FormatType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&(*self as u8 as char), f) + } +} + +impl FormatType { + fn info(self, e: Endianness) -> &'static FormatInfo { + use FormatType::*; + use mem::{align_of, size_of}; + macro_rules! native_info { + ($t:ty) => {{ + &FormatInfo { + size: size_of::<$t>(), + align: align_of::<$t>(), + pack: Some(<$t as Packable>::pack::<NativeEndian>), + unpack: Some(<$t as Packable>::unpack::<NativeEndian>), + } + }}; + } + macro_rules! nonnative_info { + ($t:ty, $end:ty) => {{ + &FormatInfo { + size: size_of::<$t>(), + align: 0, + pack: Some(<$t as Packable>::pack::<$end>), + unpack: Some(<$t as Packable>::unpack::<$end>), + } + }}; + } + macro_rules! match_nonnative { + ($zelf:expr, $end:ty) => {{ + match $zelf { + Pad | Str | Pascal => &FormatInfo { + size: size_of::<u8>(), + align: 0, + pack: None, + unpack: None, + }, + SByte => nonnative_info!(i8, $end), + UByte => nonnative_info!(u8, $end), + Char => &FormatInfo { + size: size_of::<u8>(), + align: 0, + pack: Some(pack_char), + unpack: Some(unpack_char), + }, + Short => nonnative_info!(i16, $end), + UShort => nonnative_info!(u16, $end), + Int | Long => nonnative_info!(i32, $end), + UInt | ULong => nonnative_info!(u32, $end), + LongLong => nonnative_info!(i64, $end), + ULongLong => nonnative_info!(u64, $end), + Bool => nonnative_info!(bool, $end), + Half => nonnative_info!(f16, $end), + Float => nonnative_info!(f32, $end), + Double => nonnative_info!(f64, $end), + LongDouble => nonnative_info!(f64, $end), // long double same as double + PyObject => nonnative_info!(usize, $end), // pointer size + _ => unreachable!(), // size_t or void* + } + }}; + } + match e { + Endianness::Native => match self { + Pad | Str | Pascal => &FormatInfo { + size: size_of::<raw::c_char>(), + align: 0, + pack: None, + unpack: None, + }, + SByte => native_info!(raw::c_schar), + UByte => native_info!(raw::c_uchar), + Char => &FormatInfo { + size: size_of::<raw::c_char>(), + align: 0, + pack: Some(pack_char), + unpack: Some(unpack_char), + }, + WideChar => native_info!(wchar_t), + Short => native_info!(raw::c_short), + UShort => native_info!(raw::c_ushort), + Int => native_info!(raw::c_int), + UInt => native_info!(raw::c_uint), + Long => native_info!(raw::c_long), + ULong => native_info!(raw::c_ulong), + SSizeT => native_info!(isize), // ssize_t == isize + SizeT => native_info!(usize), // size_t == usize + LongLong => native_info!(raw::c_longlong), + ULongLong => native_info!(raw::c_ulonglong), + Bool => native_info!(bool), + Half => native_info!(f16), + Float => native_info!(raw::c_float), + Double => native_info!(raw::c_double), + LongDouble => native_info!(raw::c_double), // long double same as double for now + VoidP => native_info!(*mut raw::c_void), + PyObject => native_info!(*mut raw::c_void), // pointer to PyObject + }, + Endianness::Big => match_nonnative!(self, BigEndian), + Endianness::Little => match_nonnative!(self, LittleEndian), + Endianness::Host => match_nonnative!(self, NativeEndian), + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct FormatCode { + pub repeat: usize, + pub code: FormatType, + pub info: &'static FormatInfo, + pub pre_padding: usize, +} + +impl FormatCode { + pub const fn arg_count(&self) -> usize { + match self.code { + FormatType::Pad => 0, + FormatType::Str | FormatType::Pascal => 1, + _ => self.repeat, + } + } + + pub fn parse<I>( + chars: &mut Peekable<I>, + endianness: Endianness, + ) -> Result<(Vec<Self>, usize, usize), String> + where + I: Sized + Iterator<Item = u8>, + { + let mut offset = 0isize; + let mut arg_count = 0usize; + let mut codes = vec![]; + while chars.peek().is_some() { + // Skip whitespace before repeat count or format char + while let Some(b' ' | b'\t' | b'\n' | b'\r') = chars.peek() { + chars.next(); + } + + // determine repeat operator: + let repeat = match chars.peek() { + Some(b'0'..=b'9') => { + let mut repeat = 0isize; + while let Some(b'0'..=b'9') = chars.peek() { + if let Some(c) = chars.next() { + let current_digit = c - b'0'; + repeat = repeat + .checked_mul(10) + .and_then(|r| r.checked_add(current_digit as _)) + .ok_or_else(|| OVERFLOW_MSG.to_owned())?; + } + } + repeat + } + _ => 1, + }; + + // determine format char: + let c = match chars.next() { + Some(c) => c, + None => { + // If we have a repeat count but only whitespace follows, error + if repeat != 1 { + return Err("repeat count given without format specifier".to_owned()); + } + // Otherwise, we're done parsing + break; + } + }; + + // Check for embedded null character + if c == 0 { + return Err("embedded null character".to_owned()); + } + + // PEP3118: Handle extended format specifiers + // T{...} - struct, X{} - function pointer, (...) - array shape, :name: - field name + if c == b'T' || c == b'X' { + // Skip struct/function pointer: consume until matching '}' + if chars.peek() == Some(&b'{') { + chars.next(); // consume '{' + let mut depth = 1; + while depth > 0 { + match chars.next() { + Some(b'{') => depth += 1, + Some(b'}') => depth -= 1, + None => return Err("unmatched '{' in format".to_owned()), + _ => {} + } + } + continue; + } + } + + if c == b'(' { + // Skip array shape: consume until matching ')' + let mut depth = 1; + while depth > 0 { + match chars.next() { + Some(b'(') => depth += 1, + Some(b')') => depth -= 1, + None => return Err("unmatched '(' in format".to_owned()), + _ => {} + } + } + continue; + } + + if c == b':' { + // Skip field name: consume until next ':' + loop { + match chars.next() { + Some(b':') => break, + None => return Err("unmatched ':' in format".to_owned()), + _ => {} + } + } + continue; + } + + if c == b'{' + || c == b'}' + || c == b'&' + || c == b'<' + || c == b'>' + || c == b'@' + || c == b'=' + || c == b'!' + { + // Skip standalone braces (pointer targets, etc.), pointer prefix, and nested endianness markers + continue; + } + + let code = FormatType::try_from(c) + .ok() + .filter(|c| match c { + FormatType::SSizeT | FormatType::SizeT | FormatType::VoidP => { + endianness == Endianness::Native + } + _ => true, + }) + .ok_or_else(|| "bad char in struct format".to_owned())?; + + let info = code.info(endianness); + + let padding = compensate_alignment(offset as usize, info.align) + .ok_or_else(|| OVERFLOW_MSG.to_owned())?; + offset = padding + .to_isize() + .and_then(|extra| offset.checked_add(extra)) + .ok_or_else(|| OVERFLOW_MSG.to_owned())?; + + let code = Self { + repeat: repeat as usize, + code, + info, + pre_padding: padding, + }; + arg_count += code.arg_count(); + codes.push(code); + + offset = (info.size as isize) + .checked_mul(repeat) + .and_then(|item_size| offset.checked_add(item_size)) + .ok_or_else(|| OVERFLOW_MSG.to_owned())?; + } + + Ok((codes, offset as usize, arg_count)) + } +} + +const fn compensate_alignment(offset: usize, align: usize) -> Option<usize> { + if align != 0 && offset != 0 { + // a % b == a & (b-1) if b is a power of 2 + (align - 1).checked_sub((offset - 1) & (align - 1)) + } else { + // alignment is already all good + Some(0) + } +} + +pub(crate) struct FormatInfo { + pub size: usize, + pub align: usize, + pub pack: Option<PackFunc>, + pub unpack: Option<UnpackFunc>, +} +impl fmt::Debug for FormatInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FormatInfo") + .field("size", &self.size) + .field("align", &self.align) + .finish() + } +} + +#[derive(Debug, Clone)] +pub struct FormatSpec { + #[allow(dead_code)] + pub(crate) endianness: Endianness, + pub(crate) codes: Vec<FormatCode>, + pub size: usize, + pub arg_count: usize, +} + +impl FormatSpec { + pub fn parse(fmt: &[u8], vm: &VirtualMachine) -> PyResult<Self> { + let mut chars = fmt.iter().copied().peekable(); + + // First determine "@", "<", ">","!" or "=" + let endianness = Endianness::parse(&mut chars); + + // Now, analyze struct string further: + let (codes, size, arg_count) = + FormatCode::parse(&mut chars, endianness).map_err(|err| new_struct_error(vm, err))?; + + Ok(Self { + endianness, + codes, + size, + arg_count, + }) + } + + pub fn pack(&self, args: Vec<PyObjectRef>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + // Create data vector: + let mut data = vec![0; self.size]; + + self.pack_into(&mut data, args, vm)?; + + Ok(data) + } + + pub fn pack_into( + &self, + mut buffer: &mut [u8], + args: Vec<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + if self.arg_count != args.len() { + return Err(new_struct_error( + vm, + format!( + "pack expected {} items for packing (got {})", + self.codes.len(), + args.len() + ), + )); + } + + let mut args = args.into_iter(); + // Loop over all opcodes: + for code in &self.codes { + buffer = &mut buffer[code.pre_padding..]; + debug!("code: {code:?}"); + match code.code { + FormatType::Str => { + let (buf, rest) = buffer.split_at_mut(code.repeat); + pack_string(vm, args.next().unwrap(), buf)?; + buffer = rest; + } + FormatType::Pascal => { + let (buf, rest) = buffer.split_at_mut(code.repeat); + pack_pascal(vm, args.next().unwrap(), buf)?; + buffer = rest; + } + FormatType::Pad => { + let (pad_buf, rest) = buffer.split_at_mut(code.repeat); + for el in pad_buf { + *el = 0 + } + buffer = rest; + } + _ => { + let pack = code.info.pack.unwrap(); + for arg in args.by_ref().take(code.repeat) { + let (item_buf, rest) = buffer.split_at_mut(code.info.size); + pack(vm, arg, item_buf)?; + buffer = rest; + } + } + } + } + + Ok(()) + } + + pub fn unpack(&self, mut data: &[u8], vm: &VirtualMachine) -> PyResult<PyTupleRef> { + if self.size != data.len() { + return Err(new_struct_error( + vm, + format!("unpack requires a buffer of {} bytes", self.size), + )); + } + + let mut items = Vec::with_capacity(self.arg_count); + for code in &self.codes { + data = &data[code.pre_padding..]; + debug!("unpack code: {code:?}"); + match code.code { + FormatType::Pad => { + data = &data[code.repeat..]; + } + FormatType::Str => { + let (str_data, rest) = data.split_at(code.repeat); + // string is just stored inline + items.push(vm.ctx.new_bytes(str_data.to_vec()).into()); + data = rest; + } + FormatType::Pascal => { + let (str_data, rest) = data.split_at(code.repeat); + items.push(unpack_pascal(vm, str_data)); + data = rest; + } + _ => { + let unpack = code.info.unpack.unwrap(); + for _ in 0..code.repeat { + let (item_data, rest) = data.split_at(code.info.size); + items.push(unpack(vm, item_data)); + data = rest; + } + } + }; + } + + Ok(PyTuple::new_ref(items, &vm.ctx)) + } + + #[inline] + pub const fn size(&self) -> usize { + self.size + } +} + +trait Packable { + fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()>; + fn unpack<E: ByteOrder>(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef; +} + +trait PackInt: PrimInt { + fn pack_int<E: ByteOrder>(self, data: &mut [u8]); + fn unpack_int<E: ByteOrder>(data: &[u8]) -> Self; +} + +macro_rules! make_pack_prim_int { + ($T:ty) => { + impl PackInt for $T { + fn pack_int<E: ByteOrder>(self, data: &mut [u8]) { + let i = E::convert(self); + data.copy_from_slice(&i.to_ne_bytes()); + } + #[inline] + fn unpack_int<E: ByteOrder>(data: &[u8]) -> Self { + let mut x = [0; core::mem::size_of::<$T>()]; + x.copy_from_slice(data); + E::convert(<$T>::from_ne_bytes(x)) + } + } + + impl Packable for $T { + fn pack<E: ByteOrder>( + vm: &VirtualMachine, + arg: PyObjectRef, + data: &mut [u8], + ) -> PyResult<()> { + let i: $T = get_int_or_index(vm, arg)?; + i.pack_int::<E>(data); + Ok(()) + } + + fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { + let i = <$T>::unpack_int::<E>(rdr); + vm.ctx.new_int(i).into() + } + } + }; +} + +fn get_int_or_index<T>(vm: &VirtualMachine, arg: PyObjectRef) -> PyResult<T> +where + T: PrimInt + for<'a> TryFrom<&'a BigInt>, +{ + let index = arg + .try_index_opt(vm) + .unwrap_or_else(|| Err(new_struct_error(vm, "required argument is not an integer")))?; + index + .try_to_primitive(vm) + .map_err(|_| new_struct_error(vm, "argument out of range")) +} + +make_pack_prim_int!(i8); +make_pack_prim_int!(u8); +make_pack_prim_int!(i16); +make_pack_prim_int!(u16); +make_pack_prim_int!(i32); +make_pack_prim_int!(u32); +make_pack_prim_int!(i64); +make_pack_prim_int!(u64); +make_pack_prim_int!(usize); +make_pack_prim_int!(isize); + +macro_rules! make_pack_float { + ($T:ty) => { + impl Packable for $T { + fn pack<E: ByteOrder>( + vm: &VirtualMachine, + arg: PyObjectRef, + data: &mut [u8], + ) -> PyResult<()> { + let f = ArgIntoFloat::try_from_object(vm, arg)?.into_float() as $T; + f.to_bits().pack_int::<E>(data); + Ok(()) + } + + fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { + let i = PackInt::unpack_int::<E>(rdr); + <$T>::from_bits(i).to_pyobject(vm) + } + } + }; +} + +make_pack_float!(f32); +make_pack_float!(f64); + +impl Packable for f16 { + fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { + let f_64 = ArgIntoFloat::try_from_object(vm, arg)?.into_float(); + // "from_f64 should be preferred in any non-`const` context" except it gives the wrong result :/ + let f_16 = Self::from_f64_const(f_64); + if f_16.is_infinite() != f_64.is_infinite() { + return Err(vm.new_overflow_error("float too large to pack with e format")); + } + f_16.to_bits().pack_int::<E>(data); + Ok(()) + } + + fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { + let i = PackInt::unpack_int::<E>(rdr); + Self::from_bits(i).to_f64().to_pyobject(vm) + } +} + +impl Packable for *mut raw::c_void { + fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { + usize::pack::<E>(vm, arg, data) + } + + fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { + usize::unpack::<E>(vm, rdr) + } +} + +impl Packable for bool { + fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { + let v = ArgIntoBool::try_from_object(vm, arg)?.into_bool() as u8; + v.pack_int::<E>(data); + Ok(()) + } + + fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { + let i = u8::unpack_int::<E>(rdr); + vm.ctx.new_bool(i != 0).into() + } +} + +fn pack_char(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { + let v = PyBytesRef::try_from_object(vm, arg)?; + let ch = *v + .as_bytes() + .iter() + .exactly_one() + .map_err(|_| new_struct_error(vm, "char format requires a bytes object of length 1"))?; + data[0] = ch; + Ok(()) +} + +fn pack_string(vm: &VirtualMachine, arg: PyObjectRef, buf: &mut [u8]) -> PyResult<()> { + let b = ArgBytesLike::try_from_object(vm, arg)?; + b.with_ref(|data| write_string(buf, data)); + Ok(()) +} + +fn pack_pascal(vm: &VirtualMachine, arg: PyObjectRef, buf: &mut [u8]) -> PyResult<()> { + if buf.is_empty() { + return Ok(()); + } + let b = ArgBytesLike::try_from_object(vm, arg)?; + b.with_ref(|data| { + let string_length = core::cmp::min(core::cmp::min(data.len(), 255), buf.len() - 1); + buf[0] = string_length as u8; + write_string(&mut buf[1..], data); + }); + Ok(()) +} + +fn write_string(buf: &mut [u8], data: &[u8]) { + let len_from_data = core::cmp::min(data.len(), buf.len()); + buf[..len_from_data].copy_from_slice(&data[..len_from_data]); + for byte in &mut buf[len_from_data..] { + *byte = 0 + } +} + +fn unpack_char(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef { + vm.ctx.new_bytes(vec![data[0]]).into() +} + +fn unpack_pascal(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef { + let (&len, data) = match data.split_first() { + Some(x) => x, + None => { + // cpython throws an internal SystemError here + return vm.ctx.new_bytes(vec![]).into(); + } + }; + let len = core::cmp::min(len as usize, data.len()); + vm.ctx.new_bytes(data[..len].to_vec()).into() +} + +// XXX: are those functions expected to be placed here? +pub fn struct_error_type(vm: &VirtualMachine) -> &'static PyTypeRef { + static_cell! { + static INSTANCE: PyTypeRef; + } + INSTANCE.get_or_init(|| vm.ctx.new_exception_type("struct", "error", None)) +} + +pub fn new_struct_error(vm: &VirtualMachine, msg: impl Into<String>) -> PyBaseExceptionRef { + // can't just STRUCT_ERROR.get().unwrap() cause this could be called before from buffer + // machinery, independent of whether _struct was ever imported + let msg: String = msg.into(); + vm.new_exception_msg(struct_error_type(vm).clone(), msg.into()) +} diff --git a/crates/vm/src/builtins/asyncgenerator.rs b/crates/vm/src/builtins/asyncgenerator.rs new file mode 100644 index 00000000000..dcb1c6d6f81 --- /dev/null +++ b/crates/vm/src/builtins/asyncgenerator.rs @@ -0,0 +1,821 @@ +use super::{PyCode, PyGenerator, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; +use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::PyBaseExceptionRef, + class::PyClassImpl, + common::lock::PyMutex, + coroutine::{Coro, warn_deprecated_throw_signature}, + frame::FrameRef, + function::OptionalArg, + object::{Traverse, TraverseFn}, + protocol::PyIterReturn, + types::{Destructor, IterNext, Iterable, Representable, SelfIter}, +}; + +use crossbeam_utils::atomic::AtomicCell; + +#[pyclass(name = "async_generator", module = false, traverse = "manual")] +#[derive(Debug)] +pub struct PyAsyncGen { + inner: Coro, + running_async: AtomicCell<bool>, + // whether hooks have been initialized + ag_hooks_inited: AtomicCell<bool>, + // ag_origin_or_finalizer - stores the finalizer callback + ag_finalizer: PyMutex<Option<PyObjectRef>>, +} + +unsafe impl Traverse for PyAsyncGen { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.inner.traverse(tracer_fn); + self.ag_finalizer.traverse(tracer_fn); + } +} +type PyAsyncGenRef = PyRef<PyAsyncGen>; + +impl PyPayload for PyAsyncGen { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.async_generator + } +} + +#[pyclass( + flags(DISALLOW_INSTANTIATION, HAS_WEAKREF), + with(PyRef, Representable, Destructor) +)] +impl PyAsyncGen { + pub const fn as_coro(&self) -> &Coro { + &self.inner + } + + pub fn new(frame: FrameRef, name: PyStrRef, qualname: PyStrRef) -> Self { + Self { + inner: Coro::new(frame, name, qualname), + running_async: AtomicCell::new(false), + ag_hooks_inited: AtomicCell::new(false), + ag_finalizer: PyMutex::new(None), + } + } + + /// Initialize async generator hooks. + /// Returns Ok(()) if successful, Err if firstiter hook raised an exception. + fn init_hooks(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // = async_gen_init_hooks + if zelf.ag_hooks_inited.load() { + return Ok(()); + } + + zelf.ag_hooks_inited.store(true); + + // Get and store finalizer from VM + let finalizer = vm.async_gen_finalizer.borrow().clone(); + if let Some(finalizer) = finalizer { + *zelf.ag_finalizer.lock() = Some(finalizer); + } + + // Call firstiter hook + let firstiter = vm.async_gen_firstiter.borrow().clone(); + if let Some(firstiter) = firstiter { + let obj: PyObjectRef = zelf.to_owned().into(); + firstiter.call((obj,), vm)?; + } + + Ok(()) + } + + /// Call finalizer hook if set. + fn call_finalizer(zelf: &Py<Self>, vm: &VirtualMachine) { + let finalizer = zelf.ag_finalizer.lock().clone(); + if let Some(finalizer) = finalizer + && !zelf.inner.closed.load() + { + // Create a strong reference for the finalizer call. + // This keeps the object alive during the finalizer execution. + let obj: PyObjectRef = zelf.to_owned().into(); + + // Call the finalizer. Any exceptions are handled as unraisable. + if let Err(e) = finalizer.call((obj,), vm) { + vm.run_unraisable(e, Some("async generator finalizer".to_owned()), finalizer); + } + } + } + + #[pygetset] + fn __name__(&self) -> PyStrRef { + self.inner.name() + } + + #[pygetset(setter)] + fn set___name__(&self, name: PyStrRef) { + self.inner.set_name(name) + } + + #[pygetset] + fn __qualname__(&self) -> PyStrRef { + self.inner.qualname() + } + + #[pygetset(setter)] + fn set___qualname__(&self, qualname: PyStrRef) { + self.inner.set_qualname(qualname) + } + + #[pygetset] + fn ag_await(&self, _vm: &VirtualMachine) -> Option<PyObjectRef> { + self.inner.frame().yield_from_target() + } + #[pygetset] + fn ag_frame(&self, _vm: &VirtualMachine) -> Option<FrameRef> { + if self.inner.closed() { + None + } else { + Some(self.inner.frame()) + } + } + #[pygetset] + fn ag_running(&self, _vm: &VirtualMachine) -> bool { + self.inner.running() + } + #[pygetset] + fn ag_code(&self, _vm: &VirtualMachine) -> PyRef<PyCode> { + self.inner.frame().code.clone() + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +#[pyclass] +impl PyRef<PyAsyncGen> { + #[pymethod] + const fn __aiter__(self, _vm: &VirtualMachine) -> Self { + self + } + + #[pymethod] + fn __anext__(self, vm: &VirtualMachine) -> PyResult<PyAsyncGenASend> { + PyAsyncGen::init_hooks(&self, vm)?; + Ok(PyAsyncGenASend { + ag: self, + state: AtomicCell::new(AwaitableState::Init), + value: vm.ctx.none(), + }) + } + + #[pymethod] + fn asend(self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyAsyncGenASend> { + PyAsyncGen::init_hooks(&self, vm)?; + Ok(PyAsyncGenASend { + ag: self, + state: AtomicCell::new(AwaitableState::Init), + value, + }) + } + + #[pymethod] + fn athrow( + self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<PyAsyncGenAThrow> { + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; + PyAsyncGen::init_hooks(&self, vm)?; + Ok(PyAsyncGenAThrow { + ag: self, + aclose: false, + state: AtomicCell::new(AwaitableState::Init), + value: ( + exc_type, + exc_val.unwrap_or_none(vm), + exc_tb.unwrap_or_none(vm), + ), + }) + } + + #[pymethod] + fn aclose(self, vm: &VirtualMachine) -> PyResult<PyAsyncGenAThrow> { + PyAsyncGen::init_hooks(&self, vm)?; + Ok(PyAsyncGenAThrow { + ag: self, + aclose: true, + state: AtomicCell::new(AwaitableState::Init), + value: ( + vm.ctx.exceptions.generator_exit.to_owned().into(), + vm.ctx.none(), + vm.ctx.none(), + ), + }) + } +} + +impl Representable for PyAsyncGen { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + Ok(zelf.inner.repr(zelf.as_object(), zelf.get_id(), vm)) + } +} + +#[pyclass( + module = false, + name = "async_generator_wrapped_value", + traverse = "manual" +)] +#[derive(Debug)] +pub(crate) struct PyAsyncGenWrappedValue(pub PyObjectRef); + +unsafe impl Traverse for PyAsyncGenWrappedValue { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.traverse(tracer_fn); + } +} + +impl PyPayload for PyAsyncGenWrappedValue { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.async_generator_wrapped_value + } +} + +#[pyclass] +impl PyAsyncGenWrappedValue {} + +impl PyAsyncGenWrappedValue { + fn unbox(ag: &PyAsyncGen, val: PyResult<PyIterReturn>, vm: &VirtualMachine) -> PyResult { + let (closed, async_done) = match &val { + Ok(PyIterReturn::StopIteration(_)) => (true, true), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.generator_exit) => (true, true), + Err(_) => (false, true), + _ => (false, false), + }; + if closed { + ag.inner.closed.store(true); + } + if async_done { + ag.running_async.store(false); + } + let val = val?.into_async_pyresult(vm)?; + match_class!(match val { + val @ Self => { + ag.running_async.store(false); + Err(vm.new_stop_iteration(Some(val.0.clone()))) + } + val => Ok(val), + }) + } +} + +#[derive(Debug, Clone, Copy)] +enum AwaitableState { + Init, + Iter, + Closed, +} + +#[pyclass(module = false, name = "async_generator_asend", traverse = "manual")] +#[derive(Debug)] +pub(crate) struct PyAsyncGenASend { + ag: PyAsyncGenRef, + state: AtomicCell<AwaitableState>, + value: PyObjectRef, +} + +unsafe impl Traverse for PyAsyncGenASend { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.ag.traverse(tracer_fn); + self.value.traverse(tracer_fn); + } +} + +impl PyPayload for PyAsyncGenASend { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.async_generator_asend + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PyAsyncGenASend { + #[pymethod(name = "__await__")] + const fn r#await(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyRef<Self> { + zelf + } + + #[pymethod] + fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let val = match self.state.load() { + AwaitableState::Closed => { + return Err( + vm.new_runtime_error("cannot reuse already awaited __anext__()/asend()") + ); + } + AwaitableState::Iter => val, // already running, all good + AwaitableState::Init => { + if self.ag.running_async.load() { + return Err( + vm.new_runtime_error("anext(): asynchronous generator is already running") + ); + } + self.ag.running_async.store(true); + self.state.store(AwaitableState::Iter); + if vm.is_none(&val) { + self.value.clone() + } else { + val + } + } + }; + let res = self.ag.inner.send(self.ag.as_object(), val, vm); + let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); + if res.is_err() { + self.set_closed(); + } + res + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + match self.state.load() { + AwaitableState::Closed => { + return Err( + vm.new_runtime_error("cannot reuse already awaited __anext__()/asend()") + ); + } + AwaitableState::Init => { + if self.ag.running_async.load() { + self.state.store(AwaitableState::Closed); + return Err( + vm.new_runtime_error("anext(): asynchronous generator is already running") + ); + } + self.ag.running_async.store(true); + self.state.store(AwaitableState::Iter); + } + AwaitableState::Iter => {} + } + + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; + let res = self.ag.inner.throw( + self.ag.as_object(), + exc_type, + exc_val.unwrap_or_none(vm), + exc_tb.unwrap_or_none(vm), + vm, + ); + let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); + if res.is_err() { + self.set_closed(); + } + res + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + if matches!(self.state.load(), AwaitableState::Closed) { + return Ok(()); + } + let result = self.throw( + vm.ctx.exceptions.generator_exit.to_owned().into(), + OptionalArg::Missing, + OptionalArg::Missing, + vm, + ); + match result { + Ok(_) => Err(vm.new_runtime_error("coroutine ignored GeneratorExit")), + Err(e) + if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) + || e.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) + || e.fast_isinstance(vm.ctx.exceptions.generator_exit) => + { + Ok(()) + } + Err(e) => Err(e), + } + } + + fn set_closed(&self) { + self.state.store(AwaitableState::Closed); + } +} + +impl SelfIter for PyAsyncGenASend {} +impl IterNext for PyAsyncGenASend { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + PyIterReturn::from_pyresult(zelf.send(vm.ctx.none(), vm), vm) + } +} + +#[pyclass(module = false, name = "async_generator_athrow", traverse = "manual")] +#[derive(Debug)] +pub(crate) struct PyAsyncGenAThrow { + ag: PyAsyncGenRef, + aclose: bool, + state: AtomicCell<AwaitableState>, + value: (PyObjectRef, PyObjectRef, PyObjectRef), +} + +unsafe impl Traverse for PyAsyncGenAThrow { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.ag.traverse(tracer_fn); + self.value.traverse(tracer_fn); + } +} + +impl PyPayload for PyAsyncGenAThrow { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.async_generator_athrow + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PyAsyncGenAThrow { + #[pymethod(name = "__await__")] + const fn r#await(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyRef<Self> { + zelf + } + + #[pymethod] + fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.state.load() { + AwaitableState::Closed => { + Err(vm.new_runtime_error("cannot reuse already awaited aclose()/athrow()")) + } + AwaitableState::Init => { + if self.ag.running_async.load() { + self.state.store(AwaitableState::Closed); + let msg = if self.aclose { + "aclose(): asynchronous generator is already running" + } else { + "athrow(): asynchronous generator is already running" + }; + return Err(vm.new_runtime_error(msg.to_owned())); + } + if self.ag.inner.closed() { + self.state.store(AwaitableState::Closed); + return Err(vm.new_stop_iteration(None)); + } + if !vm.is_none(&val) { + return Err(vm.new_runtime_error( + "can't send non-None value to a just-started async generator", + )); + } + self.state.store(AwaitableState::Iter); + self.ag.running_async.store(true); + + let (ty, val, tb) = self.value.clone(); + let ret = self.ag.inner.throw(self.ag.as_object(), ty, val, tb, vm); + let ret = if self.aclose { + if self.ignored_close(&ret) { + Err(self.yield_close(vm)) + } else { + ret.and_then(|o| o.into_async_pyresult(vm)) + } + } else { + PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) + }; + ret.map_err(|e| self.check_error(e, vm)) + } + AwaitableState::Iter => { + let ret = self.ag.inner.send(self.ag.as_object(), val, vm); + if self.aclose { + match ret { + Ok(PyIterReturn::Return(v)) + if v.downcastable::<PyAsyncGenWrappedValue>() => + { + Err(self.yield_close(vm)) + } + other => other + .and_then(|o| o.into_async_pyresult(vm)) + .map_err(|e| self.check_error(e, vm)), + } + } else { + PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) + } + } + } + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + match self.state.load() { + AwaitableState::Closed => { + return Err(vm.new_runtime_error("cannot reuse already awaited aclose()/athrow()")); + } + AwaitableState::Init => { + if self.ag.running_async.load() { + self.state.store(AwaitableState::Closed); + let msg = if self.aclose { + "aclose(): asynchronous generator is already running" + } else { + "athrow(): asynchronous generator is already running" + }; + return Err(vm.new_runtime_error(msg.to_owned())); + } + if self.ag.inner.closed() { + self.state.store(AwaitableState::Closed); + return Err(vm.new_stop_iteration(None)); + } + self.ag.running_async.store(true); + self.state.store(AwaitableState::Iter); + } + AwaitableState::Iter => {} + } + + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; + let ret = self.ag.inner.throw( + self.ag.as_object(), + exc_type, + exc_val.unwrap_or_none(vm), + exc_tb.unwrap_or_none(vm), + vm, + ); + let res = if self.aclose { + if self.ignored_close(&ret) { + Err(self.yield_close(vm)) + } else { + ret.and_then(|o| o.into_async_pyresult(vm)) + } + } else { + PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) + }; + res.map_err(|e| self.check_error(e, vm)) + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + if matches!(self.state.load(), AwaitableState::Closed) { + return Ok(()); + } + let result = self.throw( + vm.ctx.exceptions.generator_exit.to_owned().into(), + OptionalArg::Missing, + OptionalArg::Missing, + vm, + ); + match result { + Ok(_) => Err(vm.new_runtime_error("coroutine ignored GeneratorExit")), + Err(e) + if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) + || e.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) + || e.fast_isinstance(vm.ctx.exceptions.generator_exit) => + { + Ok(()) + } + Err(e) => Err(e), + } + } + + fn ignored_close(&self, res: &PyResult<PyIterReturn>) -> bool { + res.as_ref().is_ok_and(|v| match v { + PyIterReturn::Return(obj) => obj.downcastable::<PyAsyncGenWrappedValue>(), + PyIterReturn::StopIteration(_) => false, + }) + } + fn yield_close(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.ag.running_async.store(false); + self.ag.inner.closed.store(true); + self.state.store(AwaitableState::Closed); + vm.new_runtime_error("async generator ignored GeneratorExit") + } + fn check_error(&self, exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.ag.running_async.store(false); + self.ag.inner.closed.store(true); + self.state.store(AwaitableState::Closed); + if self.aclose + && (exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) + || exc.fast_isinstance(vm.ctx.exceptions.generator_exit)) + { + vm.new_stop_iteration(None) + } else { + exc + } + } +} + +impl SelfIter for PyAsyncGenAThrow {} +impl IterNext for PyAsyncGenAThrow { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + PyIterReturn::from_pyresult(zelf.send(vm.ctx.none(), vm), vm) + } +} + +/// Awaitable wrapper for anext() builtin with default value. +/// When StopAsyncIteration is raised, it converts it to StopIteration(default). +#[pyclass(module = false, name = "anext_awaitable", traverse = "manual")] +#[derive(Debug)] +pub struct PyAnextAwaitable { + wrapped: PyObjectRef, + default_value: PyObjectRef, + state: AtomicCell<AwaitableState>, +} + +unsafe impl Traverse for PyAnextAwaitable { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.wrapped.traverse(tracer_fn); + self.default_value.traverse(tracer_fn); + } +} + +impl PyPayload for PyAnextAwaitable { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.anext_awaitable + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PyAnextAwaitable { + pub fn new(wrapped: PyObjectRef, default_value: PyObjectRef) -> Self { + Self { + wrapped, + default_value, + state: AtomicCell::new(AwaitableState::Init), + } + } + + #[pymethod(name = "__await__")] + fn r#await(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyRef<Self> { + zelf + } + + fn check_closed(&self, vm: &VirtualMachine) -> PyResult<()> { + if let AwaitableState::Closed = self.state.load() { + return Err(vm.new_runtime_error("cannot reuse already awaited __anext__()/asend()")); + } + Ok(()) + } + + /// Get the awaitable iterator from wrapped object. + // = anextawaitable_getiter. + fn get_awaitable_iter(&self, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyCoroutine; + use crate::protocol::PyIter; + + let wrapped = &self.wrapped; + + // If wrapped is already an async_generator_asend, it's an iterator + if wrapped.class().is(vm.ctx.types.async_generator_asend) + || wrapped.class().is(vm.ctx.types.async_generator_athrow) + { + return Ok(wrapped.clone()); + } + + // _PyCoro_GetAwaitableIter equivalent + let awaitable = if wrapped.class().is(vm.ctx.types.coroutine_type) { + // Coroutine - get __await__ later + wrapped.clone() + } else { + // Check for generator with CO_ITERABLE_COROUTINE flag + if let Some(generator) = wrapped.downcast_ref::<PyGenerator>() + && generator + .as_coro() + .frame() + .code + .flags + .contains(crate::bytecode::CodeFlags::ITERABLE_COROUTINE) + { + // Return the generator itself as the iterator + return Ok(wrapped.clone()); + } + // Try to get __await__ method + if let Some(await_method) = vm.get_method(wrapped.clone(), identifier!(vm, __await__)) { + await_method?.call((), vm)? + } else { + return Err(vm.new_type_error(format!( + "'{}' object can't be awaited", + wrapped.class().name() + ))); + } + }; + + // If awaitable is a coroutine, get its __await__ + if awaitable.class().is(vm.ctx.types.coroutine_type) { + let coro_await = vm.call_method(&awaitable, "__await__", ())?; + // Check that __await__ returned an iterator + if !PyIter::check(&coro_await) { + return Err(vm.new_type_error("__await__ returned a non-iterable")); + } + return Ok(coro_await); + } + + // Check the result is an iterator, not a coroutine + if awaitable.downcast_ref::<PyCoroutine>().is_some() { + return Err(vm.new_type_error("__await__() returned a coroutine")); + } + + // Check that the result is an iterator + if !PyIter::check(&awaitable) { + return Err(vm.new_type_error(format!( + "__await__() returned non-iterator of type '{}'", + awaitable.class().name() + ))); + } + + Ok(awaitable) + } + + #[pymethod] + fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.check_closed(vm)?; + self.state.store(AwaitableState::Iter); + let awaitable = self.get_awaitable_iter(vm)?; + let result = vm.call_method(&awaitable, "send", (val,)); + self.handle_result(result, vm) + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.check_closed(vm)?; + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; + self.state.store(AwaitableState::Iter); + let awaitable = self.get_awaitable_iter(vm)?; + let result = vm.call_method( + &awaitable, + "throw", + ( + exc_type, + exc_val.unwrap_or_none(vm), + exc_tb.unwrap_or_none(vm), + ), + ); + self.handle_result(result, vm) + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + self.state.store(AwaitableState::Closed); + if let Ok(awaitable) = self.get_awaitable_iter(vm) { + let _ = vm.call_method(&awaitable, "close", ()); + } + Ok(()) + } + + /// Convert StopAsyncIteration to StopIteration(default_value) + fn handle_result(&self, result: PyResult, vm: &VirtualMachine) -> PyResult { + match result { + Ok(value) => Ok(value), + Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) => { + Err(vm.new_stop_iteration(Some(self.default_value.clone()))) + } + Err(exc) => Err(exc), + } + } +} + +impl SelfIter for PyAnextAwaitable {} +impl IterNext for PyAnextAwaitable { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + PyIterReturn::from_pyresult(zelf.send(vm.ctx.none(), vm), vm) + } +} + +/// _PyGen_Finalize for async generators +impl Destructor for PyAsyncGen { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Generator is already closed, nothing to do + if zelf.inner.closed.load() { + return Ok(()); + } + + // Call the async generator finalizer hook if set. + Self::call_finalizer(zelf, vm); + + Ok(()) + } +} + +impl Drop for PyAsyncGen { + fn drop(&mut self) { + self.inner.frame().clear_generator(); + } +} + +pub fn init(ctx: &'static Context) { + PyAsyncGen::extend_class(ctx, ctx.types.async_generator); + PyAsyncGenASend::extend_class(ctx, ctx.types.async_generator_asend); + PyAsyncGenAThrow::extend_class(ctx, ctx.types.async_generator_athrow); + PyAnextAwaitable::extend_class(ctx, ctx.types.anext_awaitable); +} diff --git a/crates/vm/src/builtins/bool.rs b/crates/vm/src/builtins/bool.rs new file mode 100644 index 00000000000..db37eee6ed1 --- /dev/null +++ b/crates/vm/src/builtins/bool.rs @@ -0,0 +1,205 @@ +use super::{PyInt, PyStrRef, PyType, PyTypeRef, PyUtf8StrRef}; +use crate::common::format::FormatSpec; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, + class::PyClassImpl, + convert::{IntoPyException, ToPyObject, ToPyResult}, + function::{FuncArgs, OptionalArg}, + protocol::PyNumberMethods, + types::{AsNumber, Constructor, Representable}, +}; +use core::fmt::{Debug, Formatter}; +use num_traits::Zero; + +impl ToPyObject for bool { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_bool(self).into() + } +} + +impl<'a> TryFromBorrowedObject<'a> for bool { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + // Python takes integers as a legit bool value + match obj.downcast_ref::<PyInt>() { + Some(int_obj) => { + let int_val = int_obj.as_bigint(); + Ok(!int_val.is_zero()) + } + None => { + Err(vm.new_type_error(format!("Expected type bool, not {}", obj.class().name()))) + } + } + } +} + +impl PyObjectRef { + /// Convert Python bool into Rust bool. + pub fn try_to_bool(self, vm: &VirtualMachine) -> PyResult<bool> { + if self.is(&vm.ctx.true_value) { + return Ok(true); + } + if self.is(&vm.ctx.false_value) { + return Ok(false); + } + + let slots = &self.class().slots; + + // 1. Try nb_bool slot first + if let Some(nb_bool) = slots.as_number.boolean.load() { + return nb_bool(self.as_object().number(), vm); + } + + // 2. Try mp_length slot (mapping protocol) + if let Some(mp_length) = slots.as_mapping.length.load() { + let len = mp_length(self.as_object().mapping_unchecked(), vm)?; + return Ok(len != 0); + } + + // 3. Try sq_length slot (sequence protocol) + if let Some(sq_length) = slots.as_sequence.length.load() { + let len = sq_length(self.as_object().sequence_unchecked(), vm)?; + return Ok(len != 0); + } + + // 4. Default: objects without __bool__ or __len__ are truthy + Ok(true) + } +} + +#[pyclass(name = "bool", module = false, base = PyInt, ctx = "bool_type")] +#[repr(transparent)] +pub struct PyBool(pub PyInt); + +impl Debug for PyBool { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + let value = !self.0.as_bigint().is_zero(); + write!(f, "PyBool({})", value) + } +} + +impl Constructor for PyBool { + type Args = OptionalArg<PyObjectRef>; + + fn slot_new(zelf: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let x: Self::Args = args.bind(vm)?; + if !zelf.fast_isinstance(vm.ctx.types.type_type) { + let actual_class = zelf.class(); + let actual_type = &actual_class.name(); + return Err(vm.new_type_error(format!( + "requires a 'type' object but received a '{actual_type}'" + ))); + } + let val = x.map_or(Ok(false), |val| val.try_to_bool(vm))?; + Ok(vm.ctx.new_bool(val).into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +#[pyclass(with(Constructor, AsNumber, Representable), flags(_MATCH_SELF))] +impl PyBool { + #[pymethod] + fn __format__(obj: PyObjectRef, spec: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<String> { + let new_bool = obj.try_to_bool(vm)?; + FormatSpec::parse(spec.as_str()) + .and_then(|format_spec| format_spec.format_bool(new_bool)) + .map_err(|err| err.into_pyexception(vm)) + } +} + +impl PyBool { + pub(crate) fn __or__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if lhs.fast_isinstance(vm.ctx.types.bool_type) + && rhs.fast_isinstance(vm.ctx.types.bool_type) + { + let lhs = get_value(&lhs); + let rhs = get_value(&rhs); + (lhs || rhs).to_pyobject(vm) + } else if let Some(lhs) = lhs.downcast_ref::<PyInt>() { + lhs.__or__(rhs).to_pyobject(vm) + } else { + vm.ctx.not_implemented() + } + } + + pub(crate) fn __and__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if lhs.fast_isinstance(vm.ctx.types.bool_type) + && rhs.fast_isinstance(vm.ctx.types.bool_type) + { + let lhs = get_value(&lhs); + let rhs = get_value(&rhs); + (lhs && rhs).to_pyobject(vm) + } else if let Some(lhs) = lhs.downcast_ref::<PyInt>() { + lhs.__and__(rhs).to_pyobject(vm) + } else { + vm.ctx.not_implemented() + } + } + + pub(crate) fn __xor__(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + if lhs.fast_isinstance(vm.ctx.types.bool_type) + && rhs.fast_isinstance(vm.ctx.types.bool_type) + { + let lhs = get_value(&lhs); + let rhs = get_value(&rhs); + (lhs ^ rhs).to_pyobject(vm) + } else if let Some(lhs) = lhs.downcast_ref::<PyInt>() { + lhs.__xor__(rhs).to_pyobject(vm) + } else { + vm.ctx.not_implemented() + } + } +} + +impl AsNumber for PyBool { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + and: Some(|a, b, vm| PyBool::__and__(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + xor: Some(|a, b, vm| PyBool::__xor__(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + or: Some(|a, b, vm| PyBool::__or__(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + ..PyInt::AS_NUMBER + }; + &AS_NUMBER + } +} + +impl Representable for PyBool { + #[inline] + fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let name = if get_value(zelf.as_object()) { + vm.ctx.names.True + } else { + vm.ctx.names.False + }; + Ok(name.to_owned()) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use slot_repr instead") + } +} + +pub(crate) fn init(context: &'static Context) { + PyBool::extend_class(context, context.types.bool_type); +} + +// pub fn not(vm: &VirtualMachine, obj: &PyObject) -> PyResult<bool> { +// if obj.fast_isinstance(vm.ctx.types.bool_type) { +// let value = get_value(obj); +// Ok(!value) +// } else { +// Err(vm.new_type_error(format!("Can only invert a bool, on {:?}", obj))) +// } +// } + +// Retrieve inner int value: +pub(crate) fn get_value(obj: &PyObject) -> bool { + !obj.downcast_ref::<PyBool>() + .unwrap() + .0 + .as_bigint() + .is_zero() +} diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs new file mode 100644 index 00000000000..1326febd000 --- /dev/null +++ b/crates/vm/src/builtins/builtin_func.rs @@ -0,0 +1,285 @@ +use super::{PyStrInterned, PyStrRef, PyType, type_}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + common::wtf8::Wtf8, + convert::TryFromObject, + function::{FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags, PyNativeFn}, + types::{Callable, Comparable, PyComparisonOp, Representable}, +}; +use alloc::fmt; + +// PyCFunctionObject in CPython +#[repr(C)] +#[pyclass(name = "builtin_function_or_method", module = false)] +pub struct PyNativeFunction { + pub(crate) value: &'static PyMethodDef, + pub(crate) zelf: Option<PyObjectRef>, + pub(crate) module: Option<&'static PyStrInterned>, // None for bound method + /// Prevent HeapMethodDef from being freed while this function references it + pub(crate) _method_def_owner: Option<PyObjectRef>, +} + +impl PyPayload for PyNativeFunction { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.builtin_function_or_method_type + } +} + +impl fmt::Debug for PyNativeFunction { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "builtin function {}.{} ({:?}) self as instance of {:?}", + self.module.map_or(Wtf8::new("<unknown>"), |m| m.as_wtf8()), + self.value.name, + self.value.flags, + self.zelf.as_ref().map(|z| z.class().name().to_owned()) + ) + } +} + +impl PyNativeFunction { + pub const fn with_module(mut self, module: &'static PyStrInterned) -> Self { + self.module = Some(module); + self + } + + pub fn into_ref(self, ctx: &Context) -> PyRef<Self> { + PyRef::new_ref( + self, + ctx.types.builtin_function_or_method_type.to_owned(), + None, + ) + } + + // PyCFunction_GET_SELF + pub fn get_self(&self) -> Option<&PyObject> { + if self.value.flags.contains(PyMethodFlags::STATIC) { + return None; + } + self.zelf.as_deref() + } + + pub const fn as_func(&self) -> &'static dyn PyNativeFn { + self.value.func + } +} + +impl Callable for PyNativeFunction { + type Args = FuncArgs; + #[inline] + fn call(zelf: &Py<Self>, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if let Some(z) = &zelf.zelf { + // STATIC methods store the class in zelf for qualname/repr purposes, + // but should not prepend it to args (the Rust function doesn't expect it). + if !zelf.value.flags.contains(PyMethodFlags::STATIC) { + args.prepend_arg(z.clone()); + } + } + (zelf.value.func)(vm, args) + } +} + +// meth_richcompare in CPython +impl Comparable for PyNativeFunction { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + if let Some(other) = other.downcast_ref::<Self>() { + let eq = match (zelf.zelf.as_ref(), other.zelf.as_ref()) { + (Some(z), Some(o)) => z.is(o), + (None, None) => true, + _ => false, + }; + let eq = eq && core::ptr::eq(zelf.value, other.value); + Ok(eq.into()) + } else { + Ok(PyComparisonValue::NotImplemented) + } + }) + } +} + +// meth_repr in CPython +impl Representable for PyNativeFunction { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + if let Some(bound) = zelf + .zelf + .as_ref() + .filter(|b| !b.class().is(vm.ctx.types.module_type)) + { + Ok(format!( + "<built-in method {} of {} object at {:#x}>", + zelf.value.name, + bound.class().name(), + bound.get_id() + )) + } else { + Ok(format!("<built-in function {}>", zelf.value.name)) + } + } +} + +#[pyclass( + with(Callable, Comparable, Representable), + flags(HAS_DICT, HAS_WEAKREF, DISALLOW_INSTANTIATION) +)] +impl PyNativeFunction { + #[pygetset] + fn __module__(zelf: NativeFunctionOrMethod) -> Option<&'static PyStrInterned> { + zelf.0.module + } + + #[pygetset] + fn __name__(zelf: NativeFunctionOrMethod) -> &'static str { + zelf.0.value.name + } + + // meth_get__qualname__ in CPython + #[pygetset] + fn __qualname__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let zelf = zelf.0; + let qualname = if let Some(bound) = &zelf.zelf { + if bound.class().is(vm.ctx.types.module_type) { + return Ok(vm.ctx.intern_str(zelf.value.name).to_owned()); + } + let prefix = if bound.class().is(vm.ctx.types.type_type) { + // m_self is a type: use PyType_GetQualName(m_self) + bound.get_attr("__qualname__", vm)?.str(vm)?.to_string() + } else { + // m_self is an instance: use Py_TYPE(m_self).__qualname__ + bound.class().name().to_string() + }; + vm.ctx.new_str(format!("{}.{}", prefix, &zelf.value.name)) + } else { + vm.ctx.intern_str(zelf.value.name).to_owned() + }; + Ok(qualname) + } + + #[pygetset] + fn __doc__(zelf: NativeFunctionOrMethod) -> Option<&'static str> { + zelf.0.value.doc + } + + // meth_get__self__ in CPython + #[pygetset] + fn __self__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyObjectRef { + zelf.0.zelf.clone().unwrap_or_else(|| vm.ctx.none()) + } + + // meth_reduce in CPython + #[pymethod] + fn __reduce__(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyResult { + let zelf = zelf.0; + if zelf.zelf.is_none() || zelf.module.is_some() { + Ok(vm.ctx.new_str(zelf.value.name).into()) + } else { + let getattr = vm.builtins.get_attr("getattr", vm)?; + let target = zelf.zelf.clone().unwrap(); + Ok(vm.new_tuple((getattr, (target, zelf.value.name))).into()) + } + } + + #[pymethod] + fn __reduce_ex__(zelf: PyObjectRef, _ver: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.call_special_method(&zelf, identifier!(vm, __reduce__), ()) + } + + #[pygetset] + fn __text_signature__(zelf: NativeFunctionOrMethod) -> Option<&'static str> { + let doc = zelf.0.value.doc?; + let signature = type_::get_text_signature_from_internal_doc(zelf.0.value.name, doc)?; + Some(signature) + } +} + +// PyCMethodObject in CPython +// repr(C) ensures `func` is at offset 0, allowing safe cast from PyNativeMethod to PyNativeFunction +#[repr(C)] +#[pyclass(name = "builtin_function_or_method", module = false, base = PyNativeFunction, ctx = "builtin_function_or_method_type")] +pub struct PyNativeMethod { + pub(crate) func: PyNativeFunction, + pub(crate) class: &'static Py<PyType>, // TODO: the actual life is &'self +} + +// All Python-visible behavior (getters, slots) is registered by PyNativeFunction::extend_class. +// PyNativeMethod only extends the Rust-side struct with the defining class reference. +// The func field at offset 0 (#[repr(C)]) allows NativeFunctionOrMethod to read it safely. +#[pyclass(flags(HAS_DICT, HAS_WEAKREF, DISALLOW_INSTANTIATION))] +impl PyNativeMethod {} + +impl fmt::Debug for PyNativeMethod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "builtin method of {:?} with {:?}", + &*self.class.name(), + &self.func + ) + } +} + +/// Vectorcall for builtin functions (PEP 590). +/// Avoids `prepend_arg` O(n) shift by building args with self at front. +fn vectorcall_native_function( + zelf_obj: &PyObject, + args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py<PyNativeFunction> = zelf_obj.downcast_ref().unwrap(); + + // Build FuncArgs with self already at position 0 (no insert(0) needed) + let needs_self = zelf + .zelf + .as_ref() + .is_some_and(|_| !zelf.value.flags.contains(PyMethodFlags::STATIC)); + + let func_args = if needs_self { + let self_obj = zelf.zelf.as_ref().unwrap().clone(); + let mut all_args = Vec::with_capacity(args.len() + 1); + all_args.push(self_obj); + all_args.extend(args); + FuncArgs::from_vectorcall(&all_args, nargs + 1, kwnames) + } else { + FuncArgs::from_vectorcall(&args, nargs, kwnames) + }; + + (zelf.value.func)(vm, func_args) +} + +pub fn init(context: &'static Context) { + PyNativeFunction::extend_class(context, context.types.builtin_function_or_method_type); + context + .types + .builtin_function_or_method_type + .slots + .vectorcall + .store(Some(vectorcall_native_function)); +} + +/// Wrapper that provides access to the common PyNativeFunction data +/// for both PyNativeFunction and PyNativeMethod (which has func as its first field). +struct NativeFunctionOrMethod(PyRef<PyNativeFunction>); + +impl TryFromObject for NativeFunctionOrMethod { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let class = vm.ctx.types.builtin_function_or_method_type; + if obj.fast_isinstance(class) { + // Both PyNativeFunction and PyNativeMethod share the same type now. + // PyNativeMethod has `func: PyNativeFunction` as its first field, + // so we can safely treat the data pointer as PyNativeFunction for reading. + Ok(Self(unsafe { obj.downcast_unchecked() })) + } else { + Err(vm.new_downcast_type_error(class, &obj)) + } + } +} diff --git a/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs similarity index 79% rename from vm/src/builtins/bytearray.rs rename to crates/vm/src/builtins/bytearray.rs index 9ac0fb54f23..dec5cacc972 100644 --- a/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -1,15 +1,17 @@ //! Implementation of the python bytearray object. use super::{ - PositionIterInternal, PyBytes, PyBytesRef, PyDictRef, PyIntRef, PyStrRef, PyTuple, PyTupleRef, - PyType, PyTypeRef, + PositionIterInternal, PyBytes, PyBytesRef, PyDictRef, PyGenericAlias, PyIntRef, PyStrRef, + PyTuple, PyTupleRef, PyType, PyTypeRef, iter::builtins_iter, }; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, anystr::{self, AnyStr}, atomic_func, byte::{bytes_from_object, value_from_object}, - bytesinner::{ - bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, - ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, + bytes_inner::{ + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, + ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, bytes_decode, }, class::PyClassImpl, common::{ @@ -21,8 +23,7 @@ use crate::{ }, convert::{ToPyObject, ToPyResult}, function::{ - ArgBytesLike, ArgIterable, ArgSize, Either, FuncArgs, OptionalArg, OptionalOption, - PyComparisonValue, + ArgBytesLike, ArgIterable, ArgSize, Either, OptionalArg, OptionalOption, PyComparisonValue, }, protocol::{ BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, @@ -30,14 +31,13 @@ use crate::{ }, sliceable::{SequenceIndex, SliceableSequenceMutOp, SliceableSequenceOp}, types::{ - AsBuffer, AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Initializer, - IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, + AsBuffer, AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, + DefaultConstructor, Initializer, IterNext, Iterable, PyComparisonOp, Representable, + SelfIter, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, }; use bstr::ByteSlice; -use std::mem::size_of; +use core::mem::size_of; #[pyclass(module = false, name = "bytearray", unhashable = true)] #[derive(Debug, Default)] @@ -67,18 +67,19 @@ impl PyPayload for PyByteArray { } /// Fill bytearray class methods dictionary. -pub(crate) fn init(context: &Context) { +pub(crate) fn init(context: &'static Context) { PyByteArray::extend_class(context, context.types.bytearray_type); PyByteArrayIterator::extend_class(context, context.types.bytearray_iterator_type); } impl PyByteArray { + #[deprecated(note = "use PyByteArray::from(...).into_ref() instead")] pub fn new_ref(data: Vec<u8>, ctx: &Context) -> PyRef<Self> { - PyRef::new_ref(Self::from(data), ctx.types.bytearray_type.to_owned(), None) + Self::from(data).into_ref(ctx) } - fn from_inner(inner: PyBytesInner) -> Self { - PyByteArray { + const fn from_inner(inner: PyBytesInner) -> Self { + Self { inner: PyRwLock::new(inner), exports: AtomicUsize::new(0), } @@ -134,7 +135,7 @@ impl PyByteArray { SequenceIndex::Slice(slice) => self .borrow_buf() .getitem_by_slice(vm, slice) - .map(|x| Self::new_ref(x, &vm.ctx).into()), + .map(|x| vm.ctx.new_bytearray(x).into()), } } @@ -169,7 +170,7 @@ impl PyByteArray { } #[pyclass( - flags(BASETYPE), + flags(BASETYPE, _MATCH_SELF), with( Py, PyRef, @@ -200,28 +201,25 @@ impl PyByteArray { self.inner.write() } - #[pymethod(magic)] - fn alloc(&self) -> usize { + #[pymethod] + fn __alloc__(&self) -> usize { self.inner().capacity() } - #[pymethod(magic)] - fn len(&self) -> usize { + fn __len__(&self) -> usize { self.borrow_buf().len() } - #[pymethod(magic)] - fn sizeof(&self) -> usize { + #[pymethod] + fn __sizeof__(&self) -> usize { size_of::<Self>() + self.borrow_buf().len() * size_of::<u8>() } - #[pymethod(magic)] - fn add(&self, other: ArgBytesLike) -> Self { + fn __add__(&self, other: ArgBytesLike) -> Self { self.inner().add(&other.borrow_buf()).into() } - #[pymethod(magic)] - fn contains( + fn __contains__( &self, needle: Either<PyBytesInner, PyIntRef>, vm: &VirtualMachine, @@ -229,21 +227,22 @@ impl PyByteArray { self.inner().contains(needle, vm) } - #[pymethod(magic)] - fn iadd(zelf: PyRef<Self>, other: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + fn __iadd__( + zelf: PyRef<Self>, + other: ArgBytesLike, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { zelf.try_resizable(vm)? .elements .extend(&*other.borrow_buf()); Ok(zelf) } - #[pymethod(magic)] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) } - #[pymethod(magic)] - pub fn delitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + pub fn __delitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self._delitem(&needle, vm) } @@ -323,37 +322,25 @@ impl PyByteArray { } #[pyclassmethod] - fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = PyBytesInner::fromhex(string.as_str(), vm)?; + fn fromhex(cls: PyTypeRef, string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let bytes = PyBytesInner::fromhex_object(string, vm)?; let bytes = vm.ctx.new_bytes(bytes); let args = vec![bytes.into()].into(); PyType::call(&cls, args, vm) } #[pymethod] - fn center( - &self, - options: ByteInnerPaddingOptions, - vm: &VirtualMachine, - ) -> PyResult<PyByteArray> { + fn center(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner().center(options, vm)?.into()) } #[pymethod] - fn ljust( - &self, - options: ByteInnerPaddingOptions, - vm: &VirtualMachine, - ) -> PyResult<PyByteArray> { + fn ljust(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner().ljust(options, vm)?.into()) } #[pymethod] - fn rjust( - &self, - options: ByteInnerPaddingOptions, - vm: &VirtualMachine, - ) -> PyResult<PyByteArray> { + fn rjust(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner().rjust(options, vm)?.into()) } @@ -363,7 +350,7 @@ impl PyByteArray { } #[pymethod] - fn join(&self, iter: ArgIterable<PyBytesInner>, vm: &VirtualMachine) -> PyResult<PyByteArray> { + fn join(&self, iter: ArgIterable<PyBytesInner>, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner().join(iter, vm)?.into()) } @@ -375,7 +362,7 @@ impl PyByteArray { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "endswith", "bytes", @@ -396,7 +383,7 @@ impl PyByteArray { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "startswith", "bytes", @@ -414,7 +401,7 @@ impl PyByteArray { #[pymethod] fn index(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult<usize> { let index = self.inner().find(options, |h, n| h.find(n), vm)?; - index.ok_or_else(|| vm.new_value_error("substring not found".to_owned())) + index.ok_or_else(|| vm.new_value_error("substring not found")) } #[pymethod] @@ -426,15 +413,11 @@ impl PyByteArray { #[pymethod] fn rindex(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult<usize> { let index = self.inner().find(options, |h, n| h.rfind(n), vm)?; - index.ok_or_else(|| vm.new_value_error("substring not found".to_owned())) + index.ok_or_else(|| vm.new_value_error("substring not found")) } #[pymethod] - fn translate( - &self, - options: ByteInnerTranslateOptions, - vm: &VirtualMachine, - ) -> PyResult<PyByteArray> { + fn translate(&self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner().translate(options, vm)?.into()) } @@ -459,11 +442,8 @@ impl PyByteArray { options: ByteInnerSplitOptions, vm: &VirtualMachine, ) -> PyResult<Vec<PyObjectRef>> { - self.inner().split( - options, - |s, vm| Self::new_ref(s.to_vec(), &vm.ctx).into(), - vm, - ) + self.inner() + .split(options, |s, vm| vm.ctx.new_bytearray(s.to_vec()).into(), vm) } #[pymethod] @@ -472,11 +452,8 @@ impl PyByteArray { options: ByteInnerSplitOptions, vm: &VirtualMachine, ) -> PyResult<Vec<PyObjectRef>> { - self.inner().rsplit( - options, - |s, vm| Self::new_ref(s.to_vec(), &vm.ctx).into(), - vm, - ) + self.inner() + .rsplit(options, |s, vm| vm.ctx.new_bytearray(s.to_vec()).into(), vm) } #[pymethod] @@ -486,9 +463,10 @@ impl PyByteArray { let value = self.inner(); let (front, has_mid, back) = value.partition(&sep, vm)?; Ok(vm.new_tuple(( - Self::new_ref(front.to_vec(), &vm.ctx), - Self::new_ref(if has_mid { sep.elements } else { Vec::new() }, &vm.ctx), - Self::new_ref(back.to_vec(), &vm.ctx), + vm.ctx.new_bytearray(front.to_vec()), + vm.ctx + .new_bytearray(if has_mid { sep.elements } else { Vec::new() }), + vm.ctx.new_bytearray(back.to_vec()), ))) } @@ -497,9 +475,10 @@ impl PyByteArray { let value = self.inner(); let (back, has_mid, front) = value.rpartition(&sep, vm)?; Ok(vm.new_tuple(( - Self::new_ref(front.to_vec(), &vm.ctx), - Self::new_ref(if has_mid { sep.elements } else { Vec::new() }, &vm.ctx), - Self::new_ref(back.to_vec(), &vm.ctx), + vm.ctx.new_bytearray(front.to_vec()), + vm.ctx + .new_bytearray(if has_mid { sep.elements } else { Vec::new() }), + vm.ctx.new_bytearray(back.to_vec()), ))) } @@ -511,7 +490,7 @@ impl PyByteArray { #[pymethod] fn splitlines(&self, options: anystr::SplitLinesArgs, vm: &VirtualMachine) -> Vec<PyObjectRef> { self.inner() - .splitlines(options, |x| Self::new_ref(x.to_vec(), &vm.ctx).into()) + .splitlines(options, |x| vm.ctx.new_bytearray(x.to_vec()).into()) } #[pymethod] @@ -526,7 +505,7 @@ impl PyByteArray { new: PyBytesInner, count: OptionalArg<isize>, vm: &VirtualMachine, - ) -> PyResult<PyByteArray> { + ) -> PyResult<Self> { Ok(self.inner().replace(old, new, count, vm)?.into()) } @@ -540,39 +519,44 @@ impl PyByteArray { self.inner().title().into() } - #[pymethod(name = "__rmul__")] - #[pymethod(magic)] - fn mul(&self, value: ArgSize, vm: &VirtualMachine) -> PyResult<Self> { + fn __mul__(&self, value: ArgSize, vm: &VirtualMachine) -> PyResult<Self> { self.repeat(value.into(), vm) } - #[pymethod(magic)] - fn imul(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + fn __imul__(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { Self::irepeat(&zelf, value.into(), vm)?; Ok(zelf) } - #[pymethod(name = "__mod__")] - fn mod_(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyByteArray> { + fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { let formatted = self.inner().cformat(values, vm)?; Ok(formatted.into()) } - #[pymethod(magic)] - fn rmod(&self, _values: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() - } - #[pymethod] fn reverse(&self) { self.borrow_buf_mut().reverse(); } + + #[pymethod] + fn resize(&self, size: isize, vm: &VirtualMachine) -> PyResult<()> { + if size < 0 { + return Err(vm.new_value_error("bytearray.resize(): new size must be >= 0")); + } + self.try_resizable(vm)?.elements.resize(size as usize, 0); + Ok(()) + } + + // TODO: Uncomment when Python adds __class_getitem__ to bytearray + // #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } } #[pyclass] impl Py<PyByteArray> { - #[pymethod(magic)] - fn setitem( + fn __setitem__( &self, needle: PyObjectRef, value: PyObjectRef, @@ -586,7 +570,7 @@ impl Py<PyByteArray> { let elements = &mut self.try_resizable(vm)?.elements; let index = elements .wrap_index(index.unwrap_or(-1)) - .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; + .ok_or_else(|| vm.new_index_error("index out of range"))?; Ok(elements.remove(index)) } @@ -612,7 +596,7 @@ impl Py<PyByteArray> { let elements = &mut self.try_resizable(vm)?.elements; let index = elements .find_byte(value) - .ok_or_else(|| vm.new_value_error("value not found in bytearray".to_owned()))?; + .ok_or_else(|| vm.new_value_error("value not found in bytearray"))?; elements.remove(index); Ok(()) } @@ -634,17 +618,17 @@ impl Py<PyByteArray> { Ok(()) } - #[pymethod(magic)] - fn reduce_ex( + #[pymethod] + fn __reduce_ex__( &self, _proto: usize, vm: &VirtualMachine, ) -> (PyTypeRef, PyTupleRef, Option<PyDictRef>) { - Self::reduce(self, vm) + self.__reduce__(vm) } - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option<PyDictRef>) { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option<PyDictRef>) { let bytes = PyBytes::from(self.borrow_buf().to_vec()).to_pyobject(vm); ( self.class().to_owned(), @@ -657,11 +641,7 @@ impl Py<PyByteArray> { #[pyclass] impl PyRef<PyByteArray> { #[pymethod] - fn lstrip( - self, - chars: OptionalOption<PyBytesInner>, - vm: &VirtualMachine, - ) -> PyRef<PyByteArray> { + fn lstrip(self, chars: OptionalOption<PyBytesInner>, vm: &VirtualMachine) -> Self { let inner = self.inner(); let stripped = inner.lstrip(chars); let elements = &inner.elements; @@ -674,11 +654,7 @@ impl PyRef<PyByteArray> { } #[pymethod] - fn rstrip( - self, - chars: OptionalOption<PyBytesInner>, - vm: &VirtualMachine, - ) -> PyRef<PyByteArray> { + fn rstrip(self, chars: OptionalOption<PyBytesInner>, vm: &VirtualMachine) -> Self { let inner = self.inner(); let stripped = inner.rstrip(chars); let elements = &inner.elements; @@ -696,15 +672,7 @@ impl PyRef<PyByteArray> { } } -impl Constructor for PyByteArray { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - PyByteArray::default() - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} +impl DefaultConstructor for PyByteArray {} impl Initializer for PyByteArray { type Args = ByteInnerNewOptions; @@ -712,7 +680,7 @@ impl Initializer for PyByteArray { fn init(zelf: PyRef<Self>, options: Self::Args, vm: &VirtualMachine) -> PyResult<()> { // First unpack bytearray and *then* get a lock to set it. let mut inner = options.get_bytearray_inner(vm)?; - std::mem::swap(&mut *zelf.inner_mut(), &mut inner); + core::mem::swap(&mut *zelf.inner_mut(), &mut inner); Ok(()) } } @@ -757,7 +725,7 @@ impl AsBuffer for PyByteArray { fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { Ok(PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor::simple(zelf.len(), false), + BufferDescriptor::simple(zelf.__len__(), false), &BUFFER_METHODS, )) } @@ -775,16 +743,18 @@ impl BufferResizeGuard for PyByteArray { impl AsMapping for PyByteArray { fn as_mapping() -> &'static PyMappingMethods { static AS_MAPPING: PyMappingMethods = PyMappingMethods { - length: atomic_func!(|mapping, _vm| Ok(PyByteArray::mapping_downcast(mapping).len())), + length: atomic_func!(|mapping, _vm| Ok( + PyByteArray::mapping_downcast(mapping).__len__() + )), subscript: atomic_func!(|mapping, needle, vm| { - PyByteArray::mapping_downcast(mapping).getitem(needle.to_owned(), vm) + PyByteArray::mapping_downcast(mapping).__getitem__(needle.to_owned(), vm) }), ass_subscript: atomic_func!(|mapping, needle, value, vm| { let zelf = PyByteArray::mapping_downcast(mapping); if let Some(value) = value { - Py::setitem(zelf, needle.to_owned(), value, vm) + zelf.__setitem__(needle.to_owned(), value, vm) } else { - zelf.delitem(needle.to_owned(), vm) + zelf.__delitem__(needle.to_owned(), vm) } }), }; @@ -795,7 +765,7 @@ impl AsMapping for PyByteArray { impl AsSequence for PyByteArray { fn as_sequence() -> &'static PySequenceMethods { static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyByteArray::sequence_downcast(seq).len())), + length: atomic_func!(|seq, _vm| Ok(PyByteArray::sequence_downcast(seq).__len__())), concat: atomic_func!(|seq, other, vm| { PyByteArray::sequence_downcast(seq) .inner() @@ -824,12 +794,12 @@ impl AsSequence for PyByteArray { contains: atomic_func!(|seq, other, vm| { let other = <Either<PyBytesInner, PyIntRef>>::try_from_object(vm, other.to_owned())?; - PyByteArray::sequence_downcast(seq).contains(other, vm) + PyByteArray::sequence_downcast(seq).__contains__(other, vm) }), inplace_concat: atomic_func!(|seq, other, vm| { let other = ArgBytesLike::try_from_object(vm, other.to_owned())?; let zelf = PyByteArray::sequence_downcast(seq).to_owned(); - PyByteArray::iadd(zelf, other, vm).map(|x| x.into()) + PyByteArray::__iadd__(zelf, other, vm).map(|x| x.into()) }), inplace_repeat: atomic_func!(|seq, n, vm| { let zelf = PyByteArray::sequence_downcast(seq).to_owned(); @@ -846,7 +816,7 @@ impl AsNumber for PyByteArray { static AS_NUMBER: PyNumberMethods = PyNumberMethods { remainder: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::<PyByteArray>() { - a.mod_(b.to_owned(), vm).to_pyresult(vm) + a.__mod__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } @@ -875,10 +845,6 @@ impl Representable for PyByteArray { } } -// fn set_value(obj: &PyObject, value: Vec<u8>) { -// obj.borrow_mut().kind = PyObjectPayload::Bytes { value }; -// } - #[pyclass(module = false, name = "bytearray_iterator")] #[derive(Debug)] pub struct PyByteArrayIterator { @@ -886,34 +852,37 @@ pub struct PyByteArrayIterator { } impl PyPayload for PyByteArrayIterator { + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.bytearray_iterator_type } } -#[pyclass(with(Constructor, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyByteArrayIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().length_hint(|obj| obj.len()) + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().length_hint(|obj| obj.__len__()) } - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.internal .lock() - .set_state(state, |obj, pos| pos.min(obj.len()), vm) + .set_state(state, |obj, pos| pos.min(obj.__len__()), vm) } } -impl Unconstructible for PyByteArrayIterator {} - impl SelfIter for PyByteArrayIterator {} impl IterNext for PyByteArrayIterator { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { diff --git a/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs similarity index 75% rename from vm/src/builtins/bytes.rs rename to crates/vm/src/builtins/bytes.rs index 351eb7ba8a3..9e5d7d0ae51 100644 --- a/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -1,18 +1,23 @@ use super::{ - PositionIterInternal, PyDictRef, PyIntRef, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, + PositionIterInternal, PyDictRef, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, PyTupleRef, + PyType, PyTypeRef, iter::builtins_iter, }; +use crate::common::lock::LazyLock; use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, TryFromObject, VirtualMachine, anystr::{self, AnyStr}, atomic_func, - bytesinner::{ - bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, - ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, + bytes_inner::{ + ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, + ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, bytes_decode, }, class::PyClassImpl, common::{hash::PyHash, lock::PyMutex}, convert::{ToPyObject, ToPyResult}, function::{ - ArgBytesLike, ArgIndex, ArgIterable, Either, OptionalArg, OptionalOption, PyComparisonValue, + ArgBytesLike, ArgIndex, ArgIterable, Either, FuncArgs, OptionalArg, OptionalOption, + PyComparisonValue, }, protocol::{ BufferDescriptor, BufferMethods, PyBuffer, PyIterReturn, PyMappingMethods, PyNumberMethods, @@ -21,14 +26,11 @@ use crate::{ sliceable::{SequenceIndex, SliceableSequenceOp}, types::{ AsBuffer, AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Hashable, - IterNext, Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, + IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, VirtualMachine, }; use bstr::ByteSlice; -use once_cell::sync::Lazy; -use std::{mem::size_of, ops::Deref}; +use core::{mem::size_of, ops::Deref}; #[pyclass(module = false, name = "bytes")] #[derive(Clone, Debug)] @@ -78,27 +80,83 @@ impl AsRef<[u8]> for PyBytesRef { } impl PyPayload for PyBytes { + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.bytes_type } } -pub(crate) fn init(context: &Context) { +pub(crate) fn init(context: &'static Context) { PyBytes::extend_class(context, context.types.bytes_type); PyBytesIterator::extend_class(context, context.types.bytes_iterator_type); } impl Constructor for PyBytes { - type Args = ByteInnerNewOptions; + type Args = Vec<u8>; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let options: ByteInnerNewOptions = args.bind(vm)?; + + // Optimizations for exact bytes type + if cls.is(vm.ctx.types.bytes_type) { + // Return empty bytes singleton + if options.source.is_missing() + && options.encoding.is_missing() + && options.errors.is_missing() + { + return Ok(vm.ctx.empty_bytes.clone().into()); + } + + // Return exact bytes as-is + if let OptionalArg::Present(ref obj) = options.source + && options.encoding.is_missing() + && options.errors.is_missing() + && let Ok(b) = obj.clone().downcast_exact::<PyBytes>(vm) + { + return Ok(b.into_pyref().into()); + } + } + + // Handle __bytes__ method - may return PyBytes directly + if let OptionalArg::Present(ref obj) = options.source + && options.encoding.is_missing() + && options.errors.is_missing() + && let Some(bytes_method) = vm.get_method(obj.clone(), identifier!(vm, __bytes__)) + { + let bytes = bytes_method?.call((), vm)?; + // If exact bytes type and __bytes__ returns bytes, use it directly + if cls.is(vm.ctx.types.bytes_type) + && let Ok(b) = bytes.clone().downcast::<PyBytes>() + { + return Ok(b.into()); + } + // Otherwise convert to Vec<u8> + let inner = PyBytesInner::try_from_borrowed_object(vm, &bytes)?; + let payload = Self::py_new(&cls, inner.elements, vm)?; + return payload.into_ref_with_type(vm, cls).map(Into::into); + } - fn py_new(cls: PyTypeRef, options: Self::Args, vm: &VirtualMachine) -> PyResult { - options.get_bytes(cls, vm).to_pyresult(vm) + // Fallback to get_bytearray_inner + let elements = options.get_bytearray_inner(vm)?.elements; + + // Return empty bytes singleton for exact bytes types + if elements.is_empty() && cls.is(vm.ctx.types.bytes_type) { + return Ok(vm.ctx.empty_bytes.clone().into()); + } + + let payload = Self::py_new(&cls, elements, vm)?; + payload.into_ref_with_type(vm, cls).map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, elements: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self::from(elements)) } } impl PyBytes { + #[deprecated(note = "use PyBytes::from(...).into_ref() instead")] pub fn new_ref(data: Vec<u8>, ctx: &Context) -> PyRef<Self> { - PyRef::new_ref(Self::from(data), ctx.types.bytes_type.to_owned(), None) + Self::from(data).into_ref(ctx) } fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { @@ -114,7 +172,7 @@ impl PyBytes { } impl PyRef<PyBytes> { - fn repeat(self, count: isize, vm: &VirtualMachine) -> PyResult<PyRef<PyBytes>> { + fn repeat(self, count: isize, vm: &VirtualMachine) -> PyResult<Self> { if count == 1 && self.class().is(vm.ctx.types.bytes_type) { // Special case: when some `bytes` is multiplied by `1`, // nothing really happens, we need to return an object itself @@ -129,7 +187,8 @@ impl PyRef<PyBytes> { } #[pyclass( - flags(BASETYPE), + itemsize = 1, + flags(BASETYPE, _MATCH_SELF), with( Py, PyRef, @@ -145,14 +204,13 @@ impl PyRef<PyBytes> { ) )] impl PyBytes { - #[pymethod(magic)] #[inline] - pub fn len(&self) -> usize { + pub const fn __len__(&self) -> usize { self.inner.len() } #[inline] - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.inner.is_empty() } @@ -161,18 +219,16 @@ impl PyBytes { self.inner.as_bytes() } - #[pymethod(magic)] - fn sizeof(&self) -> usize { + #[pymethod] + fn __sizeof__(&self) -> usize { size_of::<Self>() + self.len() * size_of::<u8>() } - #[pymethod(magic)] - fn add(&self, other: ArgBytesLike) -> Vec<u8> { + fn __add__(&self, other: ArgBytesLike) -> Vec<u8> { self.inner.add(&other.borrow_buf()) } - #[pymethod(magic)] - fn contains( + fn __contains__( &self, needle: Either<PyBytesInner, PyIntRef>, vm: &VirtualMachine, @@ -185,8 +241,7 @@ impl PyBytes { PyBytesInner::maketrans(from, to, vm) } - #[pymethod(magic)] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { self._getitem(&needle, vm) } @@ -261,24 +316,24 @@ impl PyBytes { } #[pyclassmethod] - fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = PyBytesInner::fromhex(string.as_str(), vm)?; + fn fromhex(cls: PyTypeRef, string: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let bytes = PyBytesInner::fromhex_object(string, vm)?; let bytes = vm.ctx.new_bytes(bytes).into(); PyType::call(&cls, vec![bytes].into(), vm) } #[pymethod] - fn center(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<PyBytes> { + fn center(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner.center(options, vm)?.into()) } #[pymethod] - fn ljust(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<PyBytes> { + fn ljust(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner.ljust(options, vm)?.into()) } #[pymethod] - fn rjust(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<PyBytes> { + fn rjust(&self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner.rjust(options, vm)?.into()) } @@ -288,7 +343,7 @@ impl PyBytes { } #[pymethod] - fn join(&self, iter: ArgIterable<PyBytesInner>, vm: &VirtualMachine) -> PyResult<PyBytes> { + fn join(&self, iter: ArgIterable<PyBytesInner>, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner.join(iter, vm)?.into()) } @@ -299,7 +354,7 @@ impl PyBytes { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "endswith", "bytes", @@ -319,7 +374,7 @@ impl PyBytes { Some(x) => x, None => return Ok(false), }; - substr.py_startsendswith( + substr.py_starts_ends_with( &affix, "startswith", "bytes", @@ -337,7 +392,7 @@ impl PyBytes { #[pymethod] fn index(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult<usize> { let index = self.inner.find(options, |h, n| h.find(n), vm)?; - index.ok_or_else(|| vm.new_value_error("substring not found".to_owned())) + index.ok_or_else(|| vm.new_value_error("substring not found")) } #[pymethod] @@ -349,15 +404,11 @@ impl PyBytes { #[pymethod] fn rindex(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult<usize> { let index = self.inner.find(options, |h, n| h.rfind(n), vm)?; - index.ok_or_else(|| vm.new_value_error("substring not found".to_owned())) + index.ok_or_else(|| vm.new_value_error("substring not found")) } #[pymethod] - fn translate( - &self, - options: ByteInnerTranslateOptions, - vm: &VirtualMachine, - ) -> PyResult<PyBytes> { + fn translate(&self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult<Self> { Ok(self.inner.translate(options, vm)?.into()) } @@ -449,7 +500,7 @@ impl PyBytes { new: PyBytesInner, count: OptionalArg<isize>, vm: &VirtualMachine, - ) -> PyResult<PyBytes> { + ) -> PyResult<Self> { Ok(self.inner.replace(old, new, count, vm)?.into()) } @@ -458,43 +509,41 @@ impl PyBytes { self.inner.title().into() } - #[pymethod(name = "__rmul__")] - #[pymethod(magic)] - fn mul(zelf: PyRef<Self>, value: ArgIndex, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.repeat(value.try_to_primitive(vm)?, vm) + fn __mul__(zelf: PyRef<Self>, value: ArgIndex, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + zelf.repeat(value.into_int_ref().try_to_primitive(vm)?, vm) } - #[pymethod(name = "__mod__")] - fn mod_(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyBytes> { + fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { let formatted = self.inner.cformat(values, vm)?; Ok(formatted.into()) } - #[pymethod(magic)] - fn rmod(&self, _values: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() - } - - #[pymethod(magic)] - fn getnewargs(&self, vm: &VirtualMachine) -> PyTupleRef { + #[pymethod] + fn __getnewargs__(&self, vm: &VirtualMachine) -> PyTupleRef { let param: Vec<PyObjectRef> = self.elements().map(|x| x.to_pyobject(vm)).collect(); PyTuple::new_ref(param, &vm.ctx) } + + // TODO: Uncomment when Python adds __class_getitem__ to bytes + // #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } } #[pyclass] impl Py<PyBytes> { - #[pymethod(magic)] - fn reduce_ex( + #[pymethod] + fn __reduce_ex__( &self, _proto: usize, vm: &VirtualMachine, ) -> (PyTypeRef, PyTupleRef, Option<PyDictRef>) { - Self::reduce(self, vm) + self.__reduce__(vm) } - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option<PyDictRef>) { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option<PyDictRef>) { let bytes = PyBytes::from(self.to_vec()).to_pyobject(vm); ( self.class().to_owned(), @@ -506,8 +555,8 @@ impl Py<PyBytes> { #[pyclass] impl PyRef<PyBytes> { - #[pymethod(magic)] - fn bytes(self, vm: &VirtualMachine) -> PyRef<PyBytes> { + #[pymethod] + fn __bytes__(self, vm: &VirtualMachine) -> Self { if self.is(vm.ctx.types.bytes_type) { self } else { @@ -516,7 +565,7 @@ impl PyRef<PyBytes> { } #[pymethod] - fn lstrip(self, chars: OptionalOption<PyBytesInner>, vm: &VirtualMachine) -> PyRef<PyBytes> { + fn lstrip(self, chars: OptionalOption<PyBytesInner>, vm: &VirtualMachine) -> Self { let stripped = self.inner.lstrip(chars); if stripped == self.as_bytes() { self @@ -526,7 +575,7 @@ impl PyRef<PyBytes> { } #[pymethod] - fn rstrip(self, chars: OptionalOption<PyBytesInner>, vm: &VirtualMachine) -> PyRef<PyBytes> { + fn rstrip(self, chars: OptionalOption<PyBytesInner>, vm: &VirtualMachine) -> Self { let stripped = self.inner.rstrip(chars); if stripped == self.as_bytes() { self @@ -541,7 +590,7 @@ impl PyRef<PyBytes> { /// Other possible values are 'ignore', 'replace' /// For a list of possible encodings, /// see https://docs.python.org/3/library/codecs.html#standard-encodings - /// currently, only 'utf-8' and 'ascii' emplemented + /// currently, only 'utf-8' and 'ascii' implemented #[pymethod] fn decode(self, args: DecodeArgs, vm: &VirtualMachine) -> PyResult<PyStrRef> { bytes_decode(self.into(), args, vm) @@ -568,7 +617,7 @@ impl AsBuffer for PyBytes { impl AsMapping for PyBytes { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy<PyMappingMethods> = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { length: atomic_func!(|mapping, _vm| Ok(PyBytes::mapping_downcast(mapping).len())), subscript: atomic_func!( |mapping, needle, vm| PyBytes::mapping_downcast(mapping)._getitem(needle, vm) @@ -581,7 +630,7 @@ impl AsMapping for PyBytes { impl AsSequence for PyBytes { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, _vm| Ok(PyBytes::sequence_downcast(seq).len())), concat: atomic_func!(|seq, other, vm| { PyBytes::sequence_downcast(seq) @@ -604,7 +653,7 @@ impl AsSequence for PyBytes { contains: atomic_func!(|seq, other, vm| { let other = <Either<PyBytesInner, PyIntRef>>::try_from_object(vm, other.to_owned())?; - PyBytes::sequence_downcast(seq).contains(other, vm) + PyBytes::sequence_downcast(seq).__contains__(other, vm) }), ..PySequenceMethods::NOT_IMPLEMENTED }); @@ -617,7 +666,7 @@ impl AsNumber for PyBytes { static AS_NUMBER: PyNumberMethods = PyNumberMethods { remainder: Some(|a, b, vm| { if let Some(a) = a.downcast_ref::<PyBytes>() { - a.mod_(b.to_owned(), vm).to_pyresult(vm) + a.__mod__(b.to_owned(), vm).to_pyresult(vm) } else { Ok(vm.ctx.not_implemented()) } @@ -683,33 +732,37 @@ pub struct PyBytesIterator { } impl PyPayload for PyBytesIterator { + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.bytes_iterator_type } } -#[pyclass(with(Constructor, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyBytesIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { + #[pymethod] + fn __length_hint__(&self) -> usize { self.internal.lock().length_hint(|obj| obj.len()) } - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.internal .lock() .set_state(state, |obj, pos| pos.min(obj.len()), vm) } } -impl Unconstructible for PyBytesIterator {} impl SelfIter for PyBytesIterator {} impl IterNext for PyBytesIterator { diff --git a/crates/vm/src/builtins/capsule.rs b/crates/vm/src/builtins/capsule.rs new file mode 100644 index 00000000000..c9c7fd2849d --- /dev/null +++ b/crates/vm/src/builtins/capsule.rs @@ -0,0 +1,33 @@ +use super::PyType; +use crate::{Context, Py, PyPayload, PyResult, class::PyClassImpl, types::Representable}; + +/// PyCapsule - a container for C pointers. +/// In RustPython, this is a minimal implementation for compatibility. +#[pyclass(module = false, name = "PyCapsule")] +#[derive(Debug, Clone, Copy)] +pub struct PyCapsule { + // Capsules store opaque pointers; we don't expose the actual pointer functionality + // since RustPython doesn't have the same C extension model as CPython. + _private: (), +} + +impl PyPayload for PyCapsule { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.capsule_type + } +} + +#[pyclass(with(Representable), flags(DISALLOW_INSTANTIATION))] +impl PyCapsule {} + +impl Representable for PyCapsule { + #[inline] + fn repr_str(_zelf: &Py<Self>, _vm: &crate::VirtualMachine) -> PyResult<String> { + Ok("<capsule object>".to_string()) + } +} + +pub fn init(context: &'static Context) { + PyCapsule::extend_class(context, context.types.capsule_type); +} diff --git a/crates/vm/src/builtins/classmethod.rs b/crates/vm/src/builtins/classmethod.rs new file mode 100644 index 00000000000..8955d31ce40 --- /dev/null +++ b/crates/vm/src/builtins/classmethod.rs @@ -0,0 +1,227 @@ +use super::{PyBoundMethod, PyGenericAlias, PyStr, PyType, PyTypeRef}; +use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + common::lock::PyMutex, + function::{FuncArgs, PySetterValue}, + types::{Constructor, GetDescriptor, Initializer, Representable}, +}; + +/// classmethod(function) -> method +/// +/// Convert a function to be a class method. +/// +/// A class method receives the class as implicit first argument, +/// just like an instance method receives the instance. +/// To declare a class method, use this idiom: +/// +/// class C: +/// @classmethod +/// def f(cls, arg1, arg2, ...): +/// ... +/// +/// It can be called either on the class (e.g. C.f()) or on an instance +/// (e.g. C().f()). The instance is ignored except for its class. +/// If a class method is called for a derived class, the derived class +/// object is passed as the implied first argument. +/// +/// Class methods are different than C++ or Java static methods. +/// If you want those, see the staticmethod builtin. +#[pyclass(module = false, name = "classmethod")] +#[derive(Debug)] +pub struct PyClassMethod { + callable: PyMutex<PyObjectRef>, +} + +impl From<PyObjectRef> for PyClassMethod { + fn from(callable: PyObjectRef) -> Self { + Self { + callable: PyMutex::new(callable), + } + } +} + +impl PyPayload for PyClassMethod { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.classmethod_type + } +} + +impl GetDescriptor for PyClassMethod { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let (zelf, _obj) = Self::_unwrap(&zelf, obj, vm)?; + let cls = cls.unwrap_or_else(|| _obj.class().to_owned().into()); + let callable = zelf.callable.lock().clone(); + Ok(PyBoundMethod::new(cls, callable).into_ref(&vm.ctx).into()) + } +} + +impl Constructor for PyClassMethod { + type Args = PyObjectRef; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let callable: Self::Args = args.bind(vm)?; + // Create a dictionary to hold copied attributes + let dict = vm.ctx.new_dict(); + + // Copy attributes from the callable to the dict + // This is similar to functools.wraps in CPython + if let Ok(doc) = callable.get_attr("__doc__", vm) { + dict.set_item(identifier!(vm.ctx, __doc__), doc, vm)?; + } + if let Ok(name) = callable.get_attr("__name__", vm) { + dict.set_item(identifier!(vm.ctx, __name__), name, vm)?; + } + if let Ok(qualname) = callable.get_attr("__qualname__", vm) { + dict.set_item(identifier!(vm.ctx, __qualname__), qualname, vm)?; + } + if let Ok(module) = callable.get_attr("__module__", vm) { + dict.set_item(identifier!(vm.ctx, __module__), module, vm)?; + } + if let Ok(annotations) = callable.get_attr("__annotations__", vm) { + dict.set_item(identifier!(vm.ctx, __annotations__), annotations, vm)?; + } + + // Create PyClassMethod instance with the pre-populated dict + let classmethod = Self { + callable: PyMutex::new(callable), + }; + + let result = PyRef::new_ref(classmethod, cls, Some(dict)); + Ok(PyObjectRef::from(result)) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyClassMethod { + type Args = PyObjectRef; + + fn init(zelf: PyRef<Self>, callable: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + *zelf.callable.lock() = callable; + Ok(()) + } +} + +impl PyClassMethod { + #[deprecated(note = "use PyClassMethod::from(...).into_ref() instead")] + pub fn new_ref(callable: PyObjectRef, ctx: &Context) -> PyRef<Self> { + Self::from(callable).into_ref(ctx) + } +} + +#[pyclass( + with(GetDescriptor, Constructor, Initializer, Representable), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) +)] +impl PyClassMethod { + #[pygetset] + fn __func__(&self) -> PyObjectRef { + self.callable.lock().clone() + } + + #[pygetset] + fn __wrapped__(&self) -> PyObjectRef { + self.callable.lock().clone() + } + + #[pygetset] + fn __module__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__module__", vm) + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__qualname__", vm) + } + + #[pygetset] + fn __name__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__name__", vm) + } + + #[pygetset] + fn __annotations__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__annotations__", vm) + } + + #[pygetset(setter)] + fn set___annotations__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => self.callable.lock().set_attr("__annotations__", v, vm), + PySetterValue::Delete => Ok(()), // Silently ignore delete like CPython + } + } + + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__annotate__", vm) + } + + #[pygetset(setter)] + fn set___annotate__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => self.callable.lock().set_attr("__annotate__", v, vm), + PySetterValue::Delete => Ok(()), // Silently ignore delete like CPython + } + } + + #[pygetset] + fn __isabstractmethod__(&self, vm: &VirtualMachine) -> PyObjectRef { + match vm.get_attribute_opt(self.callable.lock().clone(), "__isabstractmethod__") { + Ok(Some(is_abstract)) => is_abstract, + _ => vm.ctx.new_bool(false).into(), + } + } + + #[pygetset(setter)] + fn set___isabstractmethod__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.callable + .lock() + .set_attr("__isabstractmethod__", value, vm)?; + Ok(()) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +impl Representable for PyClassMethod { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let callable = zelf.callable.lock().repr(vm).unwrap(); + let class = Self::class(&vm.ctx); + + let repr = match ( + class + .__qualname__(vm) + .downcast_ref::<PyStr>() + .map(|n| n.as_wtf8()), + class + .__module__(vm) + .downcast_ref::<PyStr>() + .map(|m| m.as_wtf8()), + ) { + (None, _) => return Err(vm.new_type_error("Unknown qualified name")), + (Some(qualname), Some(module)) if module != "builtins" => { + format!("<{module}.{qualname}({callable})>") + } + _ => format!("<{}({})>", class.slot_name(), callable), + }; + Ok(repr) + } +} + +pub(crate) fn init(context: &'static Context) { + PyClassMethod::extend_class(context, context.types.classmethod_type); +} diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs new file mode 100644 index 00000000000..a7ef4c08a2d --- /dev/null +++ b/crates/vm/src/builtins/code.rs @@ -0,0 +1,1278 @@ +//! Infamous code object. The python class `code` + +use super::{PyBytesRef, PyStrRef, PyTupleRef, PyType}; +use crate::common::lock::PyMutex; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::PyStrInterned, + bytecode::{self, AsBag, BorrowedConstant, CodeFlags, Constant, ConstantBag, Instruction}, + class::{PyClassImpl, StaticType}, + convert::{ToPyException, ToPyObject}, + frozen, + function::OptionalArg, + types::{Constructor, Representable}, +}; +use alloc::fmt; +use core::{ + borrow::Borrow, + ops::Deref, + sync::atomic::{AtomicPtr, AtomicU64, Ordering}, +}; +use malachite_bigint::BigInt; +use num_traits::Zero; +use rustpython_compiler_core::{OneIndexed, bytecode::CodeUnits, bytecode::PyCodeLocationInfoKind}; + +/// State for iterating through code address ranges +struct PyCodeAddressRange<'a> { + ar_start: i32, + ar_end: i32, + ar_line: i32, + computed_line: i32, + reader: LineTableReader<'a>, +} + +impl<'a> PyCodeAddressRange<'a> { + fn new(linetable: &'a [u8], first_line: i32) -> Self { + PyCodeAddressRange { + ar_start: 0, + ar_end: 0, + ar_line: -1, + computed_line: first_line, + reader: LineTableReader::new(linetable), + } + } + + /// Check if this is a NO_LINE marker (code 15) + fn is_no_line_marker(byte: u8) -> bool { + (byte >> 3) == 0x1f + } + + /// Advance to next address range + fn advance(&mut self) -> bool { + if self.reader.at_end() { + return false; + } + + let first_byte = match self.reader.read_byte() { + Some(b) => b, + None => return false, + }; + + if (first_byte & 0x80) == 0 { + return false; // Invalid linetable + } + + let code = (first_byte >> 3) & 0x0f; + let length = ((first_byte & 0x07) + 1) as i32; + + // Get line delta for this entry + let line_delta = self.get_line_delta(code); + + // Update computed line + self.computed_line += line_delta; + + // Check for NO_LINE marker + if Self::is_no_line_marker(first_byte) { + self.ar_line = -1; + } else { + self.ar_line = self.computed_line; + } + + // Update address range + self.ar_start = self.ar_end; + self.ar_end += length * 2; // sizeof(_Py_CODEUNIT) = 2 + + // Skip remaining bytes for this entry + while !self.reader.at_end() { + if let Some(b) = self.reader.peek_byte() { + if (b & 0x80) != 0 { + break; + } + self.reader.read_byte(); + } else { + break; + } + } + + true + } + + fn get_line_delta(&mut self, code: u8) -> i32 { + let kind = match PyCodeLocationInfoKind::from_code(code) { + Some(k) => k, + None => return 0, + }; + + match kind { + PyCodeLocationInfoKind::None => 0, // NO_LINE marker + PyCodeLocationInfoKind::Long => { + let delta = self.reader.read_signed_varint(); + // Skip end_line, col, end_col + self.reader.read_varint(); + self.reader.read_varint(); + self.reader.read_varint(); + delta + } + PyCodeLocationInfoKind::NoColumns => self.reader.read_signed_varint(), + PyCodeLocationInfoKind::OneLine0 => { + self.reader.read_byte(); // Skip column + self.reader.read_byte(); // Skip end column + 0 + } + PyCodeLocationInfoKind::OneLine1 => { + self.reader.read_byte(); // Skip column + self.reader.read_byte(); // Skip end column + 1 + } + PyCodeLocationInfoKind::OneLine2 => { + self.reader.read_byte(); // Skip column + self.reader.read_byte(); // Skip end column + 2 + } + _ if kind.is_short() => { + self.reader.read_byte(); // Skip column byte + 0 + } + _ => 0, + } + } +} + +#[derive(FromArgs)] +pub struct ReplaceArgs { + #[pyarg(named, optional)] + co_posonlyargcount: OptionalArg<u32>, + #[pyarg(named, optional)] + co_argcount: OptionalArg<u32>, + #[pyarg(named, optional)] + co_kwonlyargcount: OptionalArg<u32>, + #[pyarg(named, optional)] + co_filename: OptionalArg<PyStrRef>, + #[pyarg(named, optional)] + co_firstlineno: OptionalArg<u32>, + #[pyarg(named, optional)] + co_consts: OptionalArg<Vec<PyObjectRef>>, + #[pyarg(named, optional)] + co_name: OptionalArg<PyStrRef>, + #[pyarg(named, optional)] + co_names: OptionalArg<Vec<PyObjectRef>>, + #[pyarg(named, optional)] + co_flags: OptionalArg<u32>, + #[pyarg(named, optional)] + co_varnames: OptionalArg<Vec<PyObjectRef>>, + #[pyarg(named, optional)] + co_nlocals: OptionalArg<u32>, + #[pyarg(named, optional)] + co_stacksize: OptionalArg<u32>, + #[pyarg(named, optional)] + co_code: OptionalArg<crate::builtins::PyBytesRef>, + #[pyarg(named, optional)] + co_linetable: OptionalArg<crate::builtins::PyBytesRef>, + #[pyarg(named, optional)] + co_exceptiontable: OptionalArg<crate::builtins::PyBytesRef>, + #[pyarg(named, optional)] + co_freevars: OptionalArg<Vec<PyObjectRef>>, + #[pyarg(named, optional)] + co_cellvars: OptionalArg<Vec<PyObjectRef>>, + #[pyarg(named, optional)] + co_qualname: OptionalArg<PyStrRef>, +} + +#[derive(Clone)] +#[repr(transparent)] +pub struct Literal(PyObjectRef); + +impl Borrow<PyObject> for Literal { + fn borrow(&self) -> &PyObject { + &self.0 + } +} + +impl From<Literal> for PyObjectRef { + fn from(obj: Literal) -> Self { + obj.0 + } +} + +fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<'_, Literal> { + match_class!(match obj { + ref i @ super::int::PyInt => { + let value = i.as_bigint(); + if obj.class().is(super::bool_::PyBool::static_type()) { + BorrowedConstant::Boolean { + value: !value.is_zero(), + } + } else { + BorrowedConstant::Integer { value } + } + } + ref f @ super::float::PyFloat => BorrowedConstant::Float { value: f.to_f64() }, + ref c @ super::complex::PyComplex => BorrowedConstant::Complex { + value: c.to_complex() + }, + ref s @ super::pystr::PyStr => BorrowedConstant::Str { value: s.as_wtf8() }, + ref b @ super::bytes::PyBytes => BorrowedConstant::Bytes { + value: b.as_bytes() + }, + ref c @ PyCode => { + BorrowedConstant::Code { code: &c.code } + } + ref t @ super::tuple::PyTuple => { + let elements = t.as_slice(); + // SAFETY: Literal is repr(transparent) over PyObjectRef, and a Literal tuple only ever + // has other literals as elements + let elements = unsafe { &*(elements as *const [PyObjectRef] as *const [Literal]) }; + BorrowedConstant::Tuple { elements } + } + super::singletons::PyNone => BorrowedConstant::None, + super::slice::PyEllipsis => BorrowedConstant::Ellipsis, + _ => panic!("unexpected payload for constant python value"), + }) +} + +impl Constant for Literal { + type Name = &'static PyStrInterned; + fn borrow_constant(&self) -> BorrowedConstant<'_, Self> { + borrow_obj_constant(&self.0) + } +} + +impl<'a> AsBag for &'a Context { + type Bag = PyObjBag<'a>; + fn as_bag(self) -> PyObjBag<'a> { + PyObjBag(self) + } +} + +impl<'a> AsBag for &'a VirtualMachine { + type Bag = PyObjBag<'a>; + fn as_bag(self) -> PyObjBag<'a> { + PyObjBag(&self.ctx) + } +} + +#[derive(Clone, Copy)] +pub struct PyObjBag<'a>(pub &'a Context); + +impl ConstantBag for PyObjBag<'_> { + type Constant = Literal; + + fn make_constant<C: Constant>(&self, constant: BorrowedConstant<'_, C>) -> Self::Constant { + let ctx = self.0; + let obj = match constant { + BorrowedConstant::Integer { value } => ctx.new_bigint(value).into(), + BorrowedConstant::Float { value } => ctx.new_float(value).into(), + BorrowedConstant::Complex { value } => ctx.new_complex(value).into(), + BorrowedConstant::Str { value } if value.len() <= 20 => { + ctx.intern_str(value).to_object() + } + BorrowedConstant::Str { value } => ctx.new_str(value).into(), + BorrowedConstant::Bytes { value } => ctx.new_bytes(value.to_vec()).into(), + BorrowedConstant::Boolean { value } => ctx.new_bool(value).into(), + BorrowedConstant::Code { code } => ctx.new_code(code.map_clone_bag(self)).into(), + BorrowedConstant::Tuple { elements } => { + let elements = elements + .iter() + .map(|constant| self.make_constant(constant.borrow_constant()).0) + .collect(); + ctx.new_tuple(elements).into() + } + BorrowedConstant::None => ctx.none(), + BorrowedConstant::Ellipsis => ctx.ellipsis.clone().into(), + }; + + Literal(obj) + } + + fn make_name(&self, name: &str) -> &'static PyStrInterned { + self.0.intern_str(name) + } + + fn make_int(&self, value: BigInt) -> Self::Constant { + Literal(self.0.new_int(value).into()) + } + + fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant { + Literal(self.0.new_tuple(elements.map(|lit| lit.0).collect()).into()) + } + + fn make_code(&self, code: CodeObject) -> Self::Constant { + Literal(self.0.new_code(code).into()) + } +} + +pub type CodeObject = bytecode::CodeObject<Literal>; + +pub trait IntoCodeObject { + fn into_code_object(self, ctx: &Context) -> CodeObject; +} + +impl IntoCodeObject for CodeObject { + fn into_code_object(self, _ctx: &Context) -> Self { + self + } +} + +impl IntoCodeObject for bytecode::CodeObject { + fn into_code_object(self, ctx: &Context) -> CodeObject { + self.map_bag(PyObjBag(ctx)) + } +} + +impl<B: AsRef<[u8]>> IntoCodeObject for frozen::FrozenCodeObject<B> { + fn into_code_object(self, ctx: &Context) -> CodeObject { + self.decode(ctx) + } +} + +/// Per-code-object monitoring data (_PyCoMonitoringData). +/// Stores original opcodes displaced by INSTRUMENTED_LINE / INSTRUMENTED_INSTRUCTION. +pub struct CoMonitoringData { + /// Original opcodes at positions with INSTRUMENTED_LINE. + /// Indexed by instruction index. 0 = not instrumented for LINE. + pub line_opcodes: Vec<u8>, + + /// Original opcodes at positions with INSTRUMENTED_INSTRUCTION. + /// Indexed by instruction index. 0 = not instrumented for INSTRUCTION. + pub per_instruction_opcodes: Vec<u8>, +} + +#[pyclass(module = false, name = "code")] +pub struct PyCode { + pub code: CodeObject, + source_path: AtomicPtr<PyStrInterned>, + /// Version counter for lazy re-instrumentation. + /// Compared against `PyGlobalState::instrumentation_version` at RESUME. + pub instrumentation_version: AtomicU64, + /// Side-table for INSTRUMENTED_LINE / INSTRUMENTED_INSTRUCTION. + pub monitoring_data: PyMutex<Option<CoMonitoringData>>, + /// Whether adaptive counters have been initialized (lazy quickening). + pub quickened: core::sync::atomic::AtomicBool, +} + +impl Deref for PyCode { + type Target = CodeObject; + fn deref(&self) -> &Self::Target { + &self.code + } +} + +impl PyCode { + pub fn new(code: CodeObject) -> Self { + let sp = code.source_path as *const PyStrInterned as *mut PyStrInterned; + Self { + code, + source_path: AtomicPtr::new(sp), + instrumentation_version: AtomicU64::new(0), + monitoring_data: PyMutex::new(None), + quickened: core::sync::atomic::AtomicBool::new(false), + } + } + + pub fn source_path(&self) -> &'static PyStrInterned { + // SAFETY: always points to a valid &'static PyStrInterned (interned strings are never deallocated) + unsafe { &*self.source_path.load(Ordering::Relaxed) } + } + + pub fn set_source_path(&self, new: &'static PyStrInterned) { + self.source_path.store( + new as *const PyStrInterned as *mut PyStrInterned, + Ordering::Relaxed, + ); + } + pub fn from_pyc_path(path: &std::path::Path, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let name = match path.file_stem() { + Some(stem) => stem.display().to_string(), + None => "".to_owned(), + }; + let content = std::fs::read(path).map_err(|e| e.to_pyexception(vm))?; + Self::from_pyc( + &content, + Some(&name), + Some(&path.display().to_string()), + Some("<source>"), + vm, + ) + } + pub fn from_pyc( + pyc_bytes: &[u8], + name: Option<&str>, + bytecode_path: Option<&str>, + source_path: Option<&str>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + if !crate::import::check_pyc_magic_number_bytes(pyc_bytes) { + return Err(vm.new_value_error("pyc bytes has wrong MAGIC")); + } + let bootstrap_external = vm.import("_frozen_importlib_external", 0)?; + let compile_bytecode = bootstrap_external.get_attr("_compile_bytecode", vm)?; + // 16 is the pyc header length + let Some((_, code_bytes)) = pyc_bytes.split_at_checked(16) else { + return Err(vm.new_value_error(format!( + "pyc_bytes header is broken. 16 bytes expected but {} bytes given.", + pyc_bytes.len() + ))); + }; + let code_bytes_obj = vm.ctx.new_bytes(code_bytes.to_vec()); + let compiled = + compile_bytecode.call((code_bytes_obj, name, bytecode_path, source_path), vm)?; + compiled.try_downcast(vm) + } +} + +impl fmt::Debug for PyCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "code: {:?}", self.code) + } +} + +impl PyPayload for PyCode { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.code_type + } +} + +impl Representable for PyCode { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let code = &zelf.code; + Ok(format!( + "<code object {} at {:#x} file {:?}, line {}>", + code.obj_name, + zelf.get_id(), + zelf.source_path().as_str(), + code.first_line_number.map_or(-1, |n| n.get() as i32) + )) + } +} + +// Arguments for code object constructor +#[derive(FromArgs)] +pub struct PyCodeNewArgs { + argcount: u32, + posonlyargcount: u32, + kwonlyargcount: u32, + nlocals: u32, + stacksize: u32, + flags: u32, + co_code: PyBytesRef, + consts: PyTupleRef, + names: PyTupleRef, + varnames: PyTupleRef, + filename: PyStrRef, + name: PyStrRef, + qualname: PyStrRef, + firstlineno: i32, + linetable: PyBytesRef, + exceptiontable: PyBytesRef, + freevars: PyTupleRef, + cellvars: PyTupleRef, +} + +impl Constructor for PyCode { + type Args = PyCodeNewArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + // Convert names tuple to vector of interned strings + let names: Box<[&'static PyStrInterned]> = args + .names + .iter() + .map(|obj| { + let s = obj + .downcast_ref::<super::pystr::PyStr>() + .ok_or_else(|| vm.new_type_error("names must be tuple of strings"))?; + Ok(vm.ctx.intern_str(s.as_wtf8())) + }) + .collect::<PyResult<Vec<_>>>()? + .into_boxed_slice(); + + let varnames: Box<[&'static PyStrInterned]> = args + .varnames + .iter() + .map(|obj| { + let s = obj + .downcast_ref::<super::pystr::PyStr>() + .ok_or_else(|| vm.new_type_error("varnames must be tuple of strings"))?; + Ok(vm.ctx.intern_str(s.as_wtf8())) + }) + .collect::<PyResult<Vec<_>>>()? + .into_boxed_slice(); + + let cellvars: Box<[&'static PyStrInterned]> = args + .cellvars + .iter() + .map(|obj| { + let s = obj + .downcast_ref::<super::pystr::PyStr>() + .ok_or_else(|| vm.new_type_error("cellvars must be tuple of strings"))?; + Ok(vm.ctx.intern_str(s.as_wtf8())) + }) + .collect::<PyResult<Vec<_>>>()? + .into_boxed_slice(); + + let freevars: Box<[&'static PyStrInterned]> = args + .freevars + .iter() + .map(|obj| { + let s = obj + .downcast_ref::<super::pystr::PyStr>() + .ok_or_else(|| vm.new_type_error("freevars must be tuple of strings"))?; + Ok(vm.ctx.intern_str(s.as_wtf8())) + }) + .collect::<PyResult<Vec<_>>>()? + .into_boxed_slice(); + + // Check nlocals matches varnames length + if args.nlocals as usize != varnames.len() { + return Err(vm.new_value_error(format!( + "nlocals ({}) != len(varnames) ({})", + args.nlocals, + varnames.len() + ))); + } + + // Parse and validate bytecode from bytes + let bytecode_bytes = args.co_code.as_bytes(); + let instructions = CodeUnits::try_from(bytecode_bytes) + .map_err(|e| vm.new_value_error(format!("invalid bytecode: {}", e)))?; + + // Convert constants + let constants = args + .consts + .iter() + .map(|obj| { + // Convert PyObject to Literal constant. For now, just wrap it + Literal(obj.clone()) + }) + .collect(); + + // Create locations (start and end pairs) + let row = if args.firstlineno > 0 { + OneIndexed::new(args.firstlineno as usize).unwrap_or(OneIndexed::MIN) + } else { + OneIndexed::MIN + }; + let loc = rustpython_compiler_core::SourceLocation { + line: row, + character_offset: OneIndexed::from_zero_indexed(0), + }; + let locations: Box< + [( + rustpython_compiler_core::SourceLocation, + rustpython_compiler_core::SourceLocation, + )], + > = vec![(loc, loc); instructions.len()].into_boxed_slice(); + + // Build the CodeObject + let code = CodeObject { + instructions, + locations, + flags: CodeFlags::from_bits_truncate(args.flags), + posonlyarg_count: args.posonlyargcount, + arg_count: args.argcount, + kwonlyarg_count: args.kwonlyargcount, + source_path: vm.ctx.intern_str(args.filename.as_wtf8()), + first_line_number: if args.firstlineno > 0 { + OneIndexed::new(args.firstlineno as usize) + } else { + None + }, + max_stackdepth: args.stacksize, + obj_name: vm.ctx.intern_str(args.name.as_wtf8()), + qualname: vm.ctx.intern_str(args.qualname.as_wtf8()), + cell2arg: None, // TODO: reuse `fn cell2arg` + constants, + names, + varnames, + cellvars, + freevars, + linetable: args.linetable.as_bytes().to_vec().into_boxed_slice(), + exceptiontable: args.exceptiontable.as_bytes().to_vec().into_boxed_slice(), + }; + + Ok(PyCode::new(code)) + } +} + +#[pyclass(with(Representable, Constructor), flags(HAS_WEAKREF))] +impl PyCode { + #[pygetset] + const fn co_posonlyargcount(&self) -> usize { + self.code.posonlyarg_count as usize + } + + #[pygetset] + const fn co_argcount(&self) -> usize { + self.code.arg_count as usize + } + + #[pygetset] + const fn co_stacksize(&self) -> u32 { + self.code.max_stackdepth + } + + #[pygetset] + pub fn co_filename(&self) -> PyStrRef { + self.source_path().to_owned() + } + + #[pygetset] + pub fn co_cellvars(&self, vm: &VirtualMachine) -> PyTupleRef { + let cellvars = self + .cellvars + .iter() + .map(|name| name.to_pyobject(vm)) + .collect(); + vm.ctx.new_tuple(cellvars) + } + + #[pygetset] + fn co_nlocals(&self) -> usize { + self.code.varnames.len() + } + + #[pygetset] + fn co_firstlineno(&self) -> u32 { + self.code.first_line_number.map_or(0, |n| n.get() as _) + } + + #[pygetset] + const fn co_kwonlyargcount(&self) -> usize { + self.code.kwonlyarg_count as usize + } + + #[pygetset] + fn co_consts(&self, vm: &VirtualMachine) -> PyTupleRef { + let consts = self.code.constants.iter().map(|x| x.0.clone()).collect(); + vm.ctx.new_tuple(consts) + } + + #[pygetset] + fn co_name(&self) -> PyStrRef { + self.code.obj_name.to_owned() + } + #[pygetset] + fn co_qualname(&self) -> PyStrRef { + self.code.qualname.to_owned() + } + + #[pygetset] + fn co_names(&self, vm: &VirtualMachine) -> PyTupleRef { + let names = self + .code + .names + .deref() + .iter() + .map(|name| name.to_pyobject(vm)) + .collect(); + vm.ctx.new_tuple(names) + } + + #[pygetset] + const fn co_flags(&self) -> u32 { + self.code.flags.bits() + } + + #[pygetset] + pub fn co_varnames(&self, vm: &VirtualMachine) -> PyTupleRef { + let varnames = self.code.varnames.iter().map(|s| s.to_object()).collect(); + vm.ctx.new_tuple(varnames) + } + + #[pygetset] + pub fn co_code(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { + vm.ctx.new_bytes(self.code.instructions.original_bytes()) + } + + #[pygetset] + pub fn _co_code_adaptive(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { + // Return current (possibly quickened/specialized) bytecode + let bytes = unsafe { + core::slice::from_raw_parts( + self.code.instructions.as_ptr() as *const u8, + self.code.instructions.len() * 2, + ) + }; + vm.ctx.new_bytes(bytes.to_vec()) + } + + #[pygetset] + pub fn co_freevars(&self, vm: &VirtualMachine) -> PyTupleRef { + let names = self + .code + .freevars + .deref() + .iter() + .map(|name| name.to_pyobject(vm)) + .collect(); + vm.ctx.new_tuple(names) + } + + #[pygetset] + pub fn co_linetable(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { + // Return the actual linetable from the code object + vm.ctx.new_bytes(self.code.linetable.to_vec()) + } + + #[pygetset] + pub fn co_exceptiontable(&self, vm: &VirtualMachine) -> crate::builtins::PyBytesRef { + // Return the actual exception table from the code object + vm.ctx.new_bytes(self.code.exceptiontable.to_vec()) + } + + #[pymethod] + pub fn co_lines(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // TODO: Implement lazy iterator (lineiterator) like CPython for better performance + // Currently returns eager list for simplicity + + // Return an iterator over (start_offset, end_offset, lineno) tuples + let linetable = self.code.linetable.as_ref(); + let mut lines = Vec::new(); + + if !linetable.is_empty() { + let first_line = self.code.first_line_number.map_or(0, |n| n.get() as i32); + let mut range = PyCodeAddressRange::new(linetable, first_line); + + // Process all address ranges and merge consecutive entries with same line + let mut pending_entry: Option<(i32, i32, i32)> = None; + + while range.advance() { + let start = range.ar_start; + let end = range.ar_end; + let line = range.ar_line; + + if let Some((prev_start, _, prev_line)) = pending_entry { + if prev_line == line { + // Same line, extend the range + pending_entry = Some((prev_start, end, prev_line)); + } else { + // Different line, emit the previous entry + let tuple = if prev_line == -1 { + vm.ctx.new_tuple(vec![ + vm.ctx.new_int(prev_start).into(), + vm.ctx.new_int(start).into(), + vm.ctx.none(), + ]) + } else { + vm.ctx.new_tuple(vec![ + vm.ctx.new_int(prev_start).into(), + vm.ctx.new_int(start).into(), + vm.ctx.new_int(prev_line).into(), + ]) + }; + lines.push(tuple.into()); + pending_entry = Some((start, end, line)); + } + } else { + // First entry + pending_entry = Some((start, end, line)); + } + } + + // Emit the last pending entry + if let Some((start, end, line)) = pending_entry { + let tuple = if line == -1 { + vm.ctx.new_tuple(vec![ + vm.ctx.new_int(start).into(), + vm.ctx.new_int(end).into(), + vm.ctx.none(), + ]) + } else { + vm.ctx.new_tuple(vec![ + vm.ctx.new_int(start).into(), + vm.ctx.new_int(end).into(), + vm.ctx.new_int(line).into(), + ]) + }; + lines.push(tuple.into()); + } + } + + let list = vm.ctx.new_list(lines); + vm.call_method(list.as_object(), "__iter__", ()) + } + + #[pymethod] + pub fn co_positions(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Return an iterator over (line, end_line, column, end_column) tuples for each instruction + let linetable = self.code.linetable.as_ref(); + let mut positions = Vec::new(); + + if !linetable.is_empty() { + let mut reader = LineTableReader::new(linetable); + let mut line = self.code.first_line_number.map_or(0, |n| n.get() as i32); + + while !reader.at_end() { + let first_byte = match reader.read_byte() { + Some(b) => b, + None => break, + }; + + if (first_byte & 0x80) == 0 { + break; // Invalid linetable + } + + let code = (first_byte >> 3) & 0x0f; + let length = ((first_byte & 0x07) + 1) as i32; + + let kind = match PyCodeLocationInfoKind::from_code(code) { + Some(k) => k, + None => break, // Invalid code + }; + + let (line_delta, end_line_delta, column, end_column): ( + i32, + i32, + Option<i32>, + Option<i32>, + ) = match kind { + PyCodeLocationInfoKind::None => { + // No location - all values are None + (0, 0, None, None) + } + PyCodeLocationInfoKind::Long => { + // Long form + let delta = reader.read_signed_varint(); + let end_line_delta = reader.read_varint() as i32; + + let col = reader.read_varint(); + let column = if col == 0 { + None + } else { + Some((col - 1) as i32) + }; + + let end_col = reader.read_varint(); + let end_column = if end_col == 0 { + None + } else { + Some((end_col - 1) as i32) + }; + + // endline = line + end_line_delta (will be computed after line update) + (delta, end_line_delta, column, end_column) + } + PyCodeLocationInfoKind::NoColumns => { + // No column form + let delta = reader.read_signed_varint(); + (delta, 0, None, None) // endline will be same as line (delta = 0) + } + PyCodeLocationInfoKind::OneLine0 + | PyCodeLocationInfoKind::OneLine1 + | PyCodeLocationInfoKind::OneLine2 => { + // One-line form - endline = line + let col = reader.read_byte().unwrap_or(0) as i32; + let end_col = reader.read_byte().unwrap_or(0) as i32; + let delta = kind.one_line_delta().unwrap_or(0); + (delta, 0, Some(col), Some(end_col)) // endline = line (delta = 0) + } + _ if kind.is_short() => { + // Short form - endline = line + let col_data = reader.read_byte().unwrap_or(0); + let col_group = kind.short_column_group().unwrap_or(0); + let col = ((col_group as i32) << 3) | ((col_data >> 4) as i32); + let end_col = col + (col_data & 0x0f) as i32; + (0, 0, Some(col), Some(end_col)) // endline = line (delta = 0) + } + _ => (0, 0, None, None), + }; + + // Update line number + line += line_delta; + + // Generate position tuples for each instruction covered by this entry + for _ in 0..length { + // Handle special case for no location (code 15) + let final_line = if kind == PyCodeLocationInfoKind::None { + None + } else { + Some(line) + }; + + let final_endline = if kind == PyCodeLocationInfoKind::None { + None + } else { + Some(line + end_line_delta) + }; + + let line_obj = final_line.to_pyobject(vm); + let end_line_obj = final_endline.to_pyobject(vm); + let column_obj = column.to_pyobject(vm); + let end_column_obj = end_column.to_pyobject(vm); + + let tuple = + vm.ctx + .new_tuple(vec![line_obj, end_line_obj, column_obj, end_column_obj]); + positions.push(tuple.into()); + } + } + } + + let list = vm.ctx.new_list(positions); + vm.call_method(list.as_object(), "__iter__", ()) + } + + #[pymethod] + pub fn co_branches(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let instructions = &self.code.instructions; + let mut branches = Vec::new(); + let mut extended_arg: u32 = 0; + + for (i, unit) in instructions.iter().enumerate() { + // De-instrument: use base opcode for instrumented variants + let op = unit.op.to_base().unwrap_or(unit.op); + let raw_arg = u32::from(u8::from(unit.arg)); + + if matches!(op, Instruction::ExtendedArg) { + extended_arg = (extended_arg | raw_arg) << 8; + continue; + } + + let oparg = extended_arg | raw_arg; + extended_arg = 0; + + let caches = op.cache_entries(); + let (src, left, right) = match op { + Instruction::ForIter { .. } => { + // left = fall-through past CACHE entries (continue iteration) + // right = past END_FOR (iterator exhausted, skip cleanup) + // arg is relative forward from after instruction+caches + let after_cache = i + 1 + caches; + let target = after_cache + oparg as usize; + let right = if matches!( + instructions.get(target).map(|u| u.op), + Some(Instruction::EndFor) | Some(Instruction::InstrumentedEndFor) + ) { + (target + 1) * 2 + } else { + target * 2 + }; + (i * 2, after_cache * 2, right) + } + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } => { + // left = fall-through past CACHE entries (skip NOT_TAKEN if present) + // right = jump target (relative forward from after instruction+caches) + let after_cache = i + 1 + caches; + let next_op = instructions + .get(after_cache) + .map(|u| u.op.to_base().unwrap_or(u.op)); + let fallthrough = if matches!(next_op, Some(Instruction::NotTaken)) { + (after_cache + 1) * 2 + } else { + after_cache * 2 + }; + let right_target = after_cache + oparg as usize; + (i * 2, fallthrough, right_target * 2) + } + Instruction::EndAsyncFor => { + // src = END_SEND position (next_i - oparg) + let next_i = i + 1; + let Some(src_i) = next_i.checked_sub(oparg as usize) else { + continue; + }; + // left = fall-through past NOT_TAKEN + (src_i * 2, (src_i + 2) * 2, next_i * 2) + } + _ => continue, + }; + + let tuple = vm.ctx.new_tuple(vec![ + vm.ctx.new_int(src).into(), + vm.ctx.new_int(left).into(), + vm.ctx.new_int(right).into(), + ]); + branches.push(tuple.into()); + } + + let list = vm.ctx.new_list(branches); + vm.call_method(list.as_object(), "__iter__", ()) + } + + #[pymethod] + pub fn replace(&self, args: ReplaceArgs, vm: &VirtualMachine) -> PyResult<Self> { + let ReplaceArgs { + co_posonlyargcount, + co_argcount, + co_kwonlyargcount, + co_filename, + co_firstlineno, + co_consts, + co_name, + co_names, + co_flags, + co_varnames, + co_nlocals, + co_stacksize, + co_code, + co_linetable, + co_exceptiontable, + co_freevars, + co_cellvars, + co_qualname, + } = args; + let posonlyarg_count = match co_posonlyargcount { + OptionalArg::Present(posonlyarg_count) => posonlyarg_count, + OptionalArg::Missing => self.code.posonlyarg_count, + }; + + let arg_count = match co_argcount { + OptionalArg::Present(arg_count) => arg_count, + OptionalArg::Missing => self.code.arg_count, + }; + + let source_path = match co_filename { + OptionalArg::Present(source_path) => source_path, + OptionalArg::Missing => self.source_path().to_owned(), + }; + + let first_line_number = match co_firstlineno { + OptionalArg::Present(first_line_number) => OneIndexed::new(first_line_number as _), + OptionalArg::Missing => self.code.first_line_number, + }; + + let kwonlyarg_count = match co_kwonlyargcount { + OptionalArg::Present(kwonlyarg_count) => kwonlyarg_count, + OptionalArg::Missing => self.code.kwonlyarg_count, + }; + + let constants = match co_consts { + OptionalArg::Present(constants) => constants, + OptionalArg::Missing => self.code.constants.iter().map(|x| x.0.clone()).collect(), + }; + + let obj_name = match co_name { + OptionalArg::Present(obj_name) => obj_name, + OptionalArg::Missing => self.code.obj_name.to_owned(), + }; + + let names = match co_names { + OptionalArg::Present(names) => names, + OptionalArg::Missing => self + .code + .names + .deref() + .iter() + .map(|name| name.to_pyobject(vm)) + .collect(), + }; + + let flags = match co_flags { + OptionalArg::Present(flags) => flags, + OptionalArg::Missing => self.code.flags.bits(), + }; + + let varnames = match co_varnames { + OptionalArg::Present(varnames) => varnames, + OptionalArg::Missing => self.code.varnames.iter().map(|s| s.to_object()).collect(), + }; + + let qualname = match co_qualname { + OptionalArg::Present(qualname) => qualname, + OptionalArg::Missing => self.code.qualname.to_owned(), + }; + + let max_stackdepth = match co_stacksize { + OptionalArg::Present(stacksize) => stacksize, + OptionalArg::Missing => self.code.max_stackdepth, + }; + + let instructions = match co_code { + OptionalArg::Present(code_bytes) => { + // Parse and validate bytecode from bytes + CodeUnits::try_from(code_bytes.as_bytes()) + .map_err(|e| vm.new_value_error(format!("invalid bytecode: {}", e)))? + } + OptionalArg::Missing => self.code.instructions.clone(), + }; + + let cellvars = match co_cellvars { + OptionalArg::Present(cellvars) => cellvars + .into_iter() + .map(|o| o.as_interned_str(vm).unwrap()) + .collect(), + OptionalArg::Missing => self.code.cellvars.clone(), + }; + + let freevars = match co_freevars { + OptionalArg::Present(freevars) => freevars + .into_iter() + .map(|o| o.as_interned_str(vm).unwrap()) + .collect(), + OptionalArg::Missing => self.code.freevars.clone(), + }; + + // Validate co_nlocals if provided + if let OptionalArg::Present(nlocals) = co_nlocals + && nlocals as usize != varnames.len() + { + return Err(vm.new_value_error(format!( + "co_nlocals ({}) != len(co_varnames) ({})", + nlocals, + varnames.len() + ))); + } + + // Handle linetable and exceptiontable + let linetable = match co_linetable { + OptionalArg::Present(linetable) => linetable.as_bytes().to_vec().into_boxed_slice(), + OptionalArg::Missing => self.code.linetable.clone(), + }; + + let exceptiontable = match co_exceptiontable { + OptionalArg::Present(exceptiontable) => { + exceptiontable.as_bytes().to_vec().into_boxed_slice() + } + OptionalArg::Missing => self.code.exceptiontable.clone(), + }; + + let new_code = CodeObject { + flags: CodeFlags::from_bits_truncate(flags), + posonlyarg_count, + arg_count, + kwonlyarg_count, + source_path: source_path.as_object().as_interned_str(vm).unwrap(), + first_line_number, + obj_name: obj_name.as_object().as_interned_str(vm).unwrap(), + qualname: qualname.as_object().as_interned_str(vm).unwrap(), + + max_stackdepth, + instructions, + // FIXME: invalid locations. Actually locations is a duplication of linetable. + // It can be removed once we move every other code to use linetable only. + locations: self.code.locations.clone(), + constants: constants.into_iter().map(Literal).collect(), + names: names + .into_iter() + .map(|o| o.as_interned_str(vm).unwrap()) + .collect(), + varnames: varnames + .into_iter() + .map(|o| o.as_interned_str(vm).unwrap()) + .collect(), + cellvars, + freevars, + cell2arg: self.code.cell2arg.clone(), + linetable, + exceptiontable, + }; + + Ok(PyCode::new(new_code)) + } + + #[pymethod] + fn _varname_from_oparg(&self, opcode: i32, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let idx_err = |vm: &VirtualMachine| vm.new_index_error("tuple index out of range"); + + let idx = usize::try_from(opcode).map_err(|_| idx_err(vm))?; + + let varnames_len = self.code.varnames.len(); + let cellvars_len = self.code.cellvars.len(); + + let name = if idx < varnames_len { + // Index in varnames + self.code.varnames.get(idx).ok_or_else(|| idx_err(vm))? + } else if idx < varnames_len + cellvars_len { + // Index in cellvars + self.code + .cellvars + .get(idx - varnames_len) + .ok_or_else(|| idx_err(vm))? + } else { + // Index in freevars + self.code + .freevars + .get(idx - varnames_len - cellvars_len) + .ok_or_else(|| idx_err(vm))? + }; + Ok(name.to_object()) + } +} + +impl fmt::Display for PyCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (**self).fmt(f) + } +} + +impl ToPyObject for CodeObject { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_code(self).into() + } +} + +impl ToPyObject for bytecode::CodeObject { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_code(self).into() + } +} + +// Helper struct for reading linetable +struct LineTableReader<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> LineTableReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + + fn read_byte(&mut self) -> Option<u8> { + if self.pos < self.data.len() { + let byte = self.data[self.pos]; + self.pos += 1; + Some(byte) + } else { + None + } + } + + fn peek_byte(&self) -> Option<u8> { + if self.pos < self.data.len() { + Some(self.data[self.pos]) + } else { + None + } + } + + fn read_varint(&mut self) -> u32 { + if let Some(first) = self.read_byte() { + let mut val = (first & 0x3f) as u32; + let mut shift = 0; + let mut byte = first; + while (byte & 0x40) != 0 { + if let Some(next) = self.read_byte() { + shift += 6; + val |= ((next & 0x3f) as u32) << shift; + byte = next; + } else { + break; + } + } + val + } else { + 0 + } + } + + fn read_signed_varint(&mut self) -> i32 { + let uval = self.read_varint(); + if uval & 1 != 0 { + -((uval >> 1) as i32) + } else { + (uval >> 1) as i32 + } + } + + fn at_end(&self) -> bool { + self.pos >= self.data.len() + } +} + +pub fn init(ctx: &'static Context) { + PyCode::extend_class(ctx, ctx.types.code_type); +} diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs new file mode 100644 index 00000000000..3e905b6ec27 --- /dev/null +++ b/crates/vm/src/builtins/complex.rs @@ -0,0 +1,509 @@ +use super::{PyStr, PyType, PyTypeRef, float}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::PyUtf8StrRef, + class::PyClassImpl, + common::{format::FormatSpec, wtf8::Wtf8Buf}, + convert::{IntoPyException, ToPyObject, ToPyResult}, + function::{FuncArgs, OptionalArg, PyComparisonValue}, + protocol::PyNumberMethods, + stdlib::_warnings, + types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, +}; +use core::cell::Cell; +use core::num::Wrapping; +use core::ptr::NonNull; +use num_complex::Complex64; +use num_traits::Zero; +use rustpython_common::hash; + +/// Create a complex number from a real part and an optional imaginary part. +/// +/// This is equivalent to (real + imag*1j) where imag defaults to 0. +#[pyclass(module = false, name = "complex")] +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct PyComplex { + value: Complex64, +} + +// spell-checker:ignore MAXFREELIST +thread_local! { + static COMPLEX_FREELIST: Cell<crate::object::FreeList<PyComplex>> = const { Cell::new(crate::object::FreeList::new()) }; +} + +impl PyPayload for PyComplex { + const MAX_FREELIST: usize = 100; + const HAS_FREELIST: bool = true; + + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.complex_type + } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + COMPLEX_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> { + COMPLEX_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } +} + +impl ToPyObject for Complex64 { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + PyComplex::from(self).to_pyobject(vm) + } +} + +impl From<Complex64> for PyComplex { + fn from(value: Complex64) -> Self { + Self { value } + } +} + +impl PyObjectRef { + /// Tries converting a python object into a complex, returns an option of whether the complex + /// and whether the object was a complex originally or coerced into one + pub fn try_complex(&self, vm: &VirtualMachine) -> PyResult<Option<(Complex64, bool)>> { + if let Some(complex) = self.downcast_ref_if_exact::<PyComplex>(vm) { + return Ok(Some((complex.value, true))); + } + if let Some(method) = vm.get_method(self.clone(), identifier!(vm, __complex__)) { + let result = method?.call((), vm)?; + + let ret_class = result.class().to_owned(); + if let Some(ret) = result.downcast_ref::<PyComplex>() { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + format!( + "__complex__ returned non-complex (type {ret_class}). \ + The ability to return an instance of a strict subclass of complex \ + is deprecated, and may be removed in a future version of Python." + ), + 1, + vm, + )?; + + return Ok(Some((ret.value, true))); + } else { + return match result.downcast_ref::<PyComplex>() { + Some(complex_obj) => Ok(Some((complex_obj.value, true))), + None => Err(vm.new_type_error(format!( + "__complex__ returned non-complex (type '{}')", + result.class().name() + ))), + }; + } + } + // `complex` does not have a `__complex__` by default, so subclasses might not either, + // use the actual stored value in this case + if let Some(complex) = self.downcast_ref::<PyComplex>() { + return Ok(Some((complex.value, true))); + } + if let Some(float) = self.try_float_opt(vm) { + return Ok(Some((Complex64::new(float?.to_f64(), 0.0), false))); + } + Ok(None) + } +} + +pub fn init(context: &'static Context) { + PyComplex::extend_class(context, context.types.complex_type); +} + +fn to_op_complex(value: &PyObject, vm: &VirtualMachine) -> PyResult<Option<Complex64>> { + let r = if let Some(complex) = value.downcast_ref::<PyComplex>() { + Some(complex.value) + } else { + float::to_op_float(value, vm)?.map(|float| Complex64::new(float, 0.0)) + }; + Ok(r) +} + +fn inner_div(v1: Complex64, v2: Complex64, vm: &VirtualMachine) -> PyResult<Complex64> { + if v2.is_zero() { + return Err(vm.new_zero_division_error("complex division by zero")); + } + + Ok(v1.fdiv(v2)) +} + +fn inner_pow(v1: Complex64, v2: Complex64, vm: &VirtualMachine) -> PyResult<Complex64> { + if v1.is_zero() { + return if v2.re < 0.0 || v2.im != 0.0 { + let msg = format!("{v1} cannot be raised to a negative or complex power"); + Err(vm.new_zero_division_error(msg)) + } else if v2.is_zero() { + Ok(Complex64::new(1.0, 0.0)) + } else { + Ok(Complex64::new(0.0, 0.0)) + }; + } + + let ans = powc(v1, v2); + if ans.is_infinite() && !(v1.is_infinite() || v2.is_infinite()) { + Err(vm.new_overflow_error("complex exponentiation overflow")) + } else { + Ok(ans) + } +} + +// num-complex changed their powc() implementation in 0.4.4, making it incompatible +// with what the regression tests expect. this is that old formula. +fn powc(a: Complex64, exp: Complex64) -> Complex64 { + let (r, theta) = a.to_polar(); + if r.is_zero() { + return Complex64::new(r, r); + } + Complex64::from_polar( + r.powf(exp.re) * (-exp.im * theta).exp(), + exp.re * theta + exp.im * r.ln(), + ) +} + +impl Constructor for PyComplex { + type Args = ComplexArgs; + + fn slot_new(cls: PyTypeRef, func_args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Optimization: return exact complex as-is (only when imag is not provided) + if cls.is(vm.ctx.types.complex_type) + && func_args.args.len() == 1 + && func_args.kwargs.is_empty() + && func_args.args[0].class().is(vm.ctx.types.complex_type) + { + return Ok(func_args.args[0].clone()); + } + + let args: Self::Args = func_args.bind(vm)?; + let payload = Self::py_new(&cls, args, vm)?; + payload.into_ref_with_type(vm, cls).map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let imag_missing = args.imag.is_missing(); + let (real, real_was_complex) = match args.real { + OptionalArg::Missing => (Complex64::new(0.0, 0.0), false), + OptionalArg::Present(val) => { + if let Some(c) = val.try_complex(vm)? { + c + } else if let Some(s) = val.downcast_ref::<PyStr>() { + if args.imag.is_present() { + return Err(vm.new_type_error( + "complex() can't take second arg if first is a string", + )); + } + let (re, im) = s + .to_str() + .and_then(rustpython_literal::complex::parse_str) + .ok_or_else(|| vm.new_value_error("complex() arg is a malformed string"))?; + return Ok(Self::from(Complex64 { re, im })); + } else { + return Err(vm.new_type_error(format!( + "complex() first argument must be a string or a number, not '{}'", + val.class().name() + ))); + } + } + }; + + let (imag, imag_was_complex) = match args.imag { + // Copy the imaginary from the real to the real of the imaginary + // if an imaginary argument is not passed in + OptionalArg::Missing => (Complex64::new(real.im, 0.0), false), + OptionalArg::Present(obj) => { + if let Some(c) = obj.try_complex(vm)? { + c + } else if obj.class().fast_issubclass(vm.ctx.types.str_type) { + return Err(vm.new_type_error("complex() second arg can't be a string")); + } else { + return Err(vm.new_type_error(format!( + "complex() second argument must be a number, not '{}'", + obj.class().name() + ))); + } + } + }; + + let final_real = if imag_was_complex { + real.re - imag.im + } else { + real.re + }; + + let final_imag = if real_was_complex && !imag_missing { + imag.re + real.im + } else { + imag.re + }; + let value = Complex64::new(final_real, final_imag); + Ok(Self::from(value)) + } +} + +impl PyComplex { + #[deprecated(note = "use PyComplex::from(...).into_ref() instead")] + pub fn new_ref(value: Complex64, ctx: &Context) -> PyRef<Self> { + Self::from(value).into_ref(ctx) + } + + pub const fn to_complex64(self) -> Complex64 { + self.value + } + + pub const fn to_complex(&self) -> Complex64 { + self.value + } + + fn number_op<F, R>(a: &PyObject, b: &PyObject, op: F, vm: &VirtualMachine) -> PyResult + where + F: FnOnce(Complex64, Complex64, &VirtualMachine) -> R, + R: ToPyResult, + { + if let (Some(a), Some(b)) = (to_op_complex(a, vm)?, to_op_complex(b, vm)?) { + op(a, b, vm).to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + } + + fn complex_real_binop<CCF, RCF, CRF, R>( + a: &PyObject, + b: &PyObject, + cc_op: CCF, + cr_op: CRF, + rc_op: RCF, + vm: &VirtualMachine, + ) -> PyResult + where + CCF: FnOnce(Complex64, Complex64) -> R, + CRF: FnOnce(Complex64, f64) -> R, + RCF: FnOnce(f64, Complex64) -> R, + R: ToPyResult, + { + let value = match (a.downcast_ref::<PyComplex>(), b.downcast_ref::<PyComplex>()) { + // complex + complex + (Some(a_complex), Some(b_complex)) => cc_op(a_complex.value, b_complex.value), + (Some(a_complex), None) => { + let Some(b_real) = float::to_op_float(b, vm)? else { + return Ok(vm.ctx.not_implemented()); + }; + + // complex + real + cr_op(a_complex.value, b_real) + } + (None, Some(b_complex)) => { + let Some(a_real) = float::to_op_float(a, vm)? else { + return Ok(vm.ctx.not_implemented()); + }; + + // real + complex + rc_op(a_real, b_complex.value) + } + (None, None) => return Ok(vm.ctx.not_implemented()), + }; + value.to_pyresult(vm) + } +} + +#[pyclass( + flags(BASETYPE), + with(PyRef, Comparable, Hashable, Constructor, AsNumber, Representable) +)] +impl PyComplex { + #[pygetset] + const fn real(&self) -> f64 { + self.value.re + } + + #[pygetset] + const fn imag(&self) -> f64 { + self.value.im + } + + #[pymethod] + fn conjugate(&self) -> Complex64 { + self.value.conj() + } + + #[pymethod] + const fn __getnewargs__(&self) -> (f64, f64) { + let Complex64 { re, im } = self.value; + (re, im) + } + + #[pymethod] + fn __format__(zelf: &Py<Self>, spec: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + // Empty format spec: equivalent to str(self) + if spec.is_empty() { + return Ok(zelf.as_object().str(vm)?.as_wtf8().to_owned()); + } + let format_spec = + FormatSpec::parse(spec.as_str()).map_err(|err| err.into_pyexception(vm))?; + let result = if format_spec.has_locale_format() { + let locale = crate::format::get_locale_info(); + format_spec.format_complex_locale(&zelf.value, &locale) + } else { + format_spec.format_complex(&zelf.value) + }; + result + .map(Wtf8Buf::from_string) + .map_err(|err| err.into_pyexception(vm)) + } +} + +#[pyclass] +impl PyRef<PyComplex> { + #[pymethod] + fn __complex__(self, vm: &VirtualMachine) -> Self { + if self.is(vm.ctx.types.complex_type) { + self + } else { + PyComplex::from(self.value).into_ref(&vm.ctx) + } + } +} + +impl Comparable for PyComplex { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let result = if let Some(other) = other.downcast_ref::<Self>() { + if zelf.value.re.is_nan() + && zelf.value.im.is_nan() + && other.value.re.is_nan() + && other.value.im.is_nan() + { + true + } else { + zelf.value == other.value + } + } else { + match float::to_op_float(other, vm) { + Ok(Some(other)) => zelf.value == other.into(), + Err(_) => false, + Ok(None) => return Ok(PyComparisonValue::NotImplemented), + } + }; + Ok(PyComparisonValue::Implemented(result)) + }) + } +} + +impl Hashable for PyComplex { + #[inline] + fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<hash::PyHash> { + let value = zelf.value; + + let re_hash = + hash::hash_float(value.re).unwrap_or_else(|| hash::hash_object_id(zelf.get_id())); + + let im_hash = + hash::hash_float(value.im).unwrap_or_else(|| hash::hash_object_id(zelf.get_id())); + + let Wrapping(ret) = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(hash::IMAG); + Ok(hash::fix_sentinel(ret)) + } +} + +impl AsNumber for PyComplex { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + add: Some(|a, b, vm| { + PyComplex::complex_real_binop( + a, + b, + |a, b| a + b, + |a_complex, b_real| Complex64::new(a_complex.re + b_real, a_complex.im), + |a_real, b_complex| Complex64::new(a_real + b_complex.re, b_complex.im), + vm, + ) + }), + subtract: Some(|a, b, vm| { + PyComplex::complex_real_binop( + a, + b, + |a, b| a - b, + |a_complex, b_real| Complex64::new(a_complex.re - b_real, a_complex.im), + |a_real, b_complex| Complex64::new(a_real - b_complex.re, -b_complex.im), + vm, + ) + }), + multiply: Some(|a, b, vm| PyComplex::number_op(a, b, |a, b, _vm| a * b, vm)), + power: Some(|a, b, c, vm| { + if vm.is_none(c) { + PyComplex::number_op(a, b, inner_pow, vm) + } else { + Err(vm.new_value_error(String::from("complex modulo"))) + } + }), + negative: Some(|number, vm| { + let value = PyComplex::number_downcast(number).value; + (-value).to_pyresult(vm) + }), + positive: Some(|number, vm| { + PyComplex::number_downcast_exact(number, vm).to_pyresult(vm) + }), + absolute: Some(|number, vm| { + let value = PyComplex::number_downcast(number).value; + let result = value.norm(); + // Check for overflow: hypot returns inf for finite inputs that overflow + if result.is_infinite() && value.re.is_finite() && value.im.is_finite() { + return Err(vm.new_overflow_error("absolute value too large")); + } + result.to_pyresult(vm) + }), + boolean: Some(|number, _vm| Ok(!PyComplex::number_downcast(number).value.is_zero())), + true_divide: Some(|a, b, vm| PyComplex::number_op(a, b, inner_div, vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + + fn clone_exact(zelf: &Py<Self>, vm: &VirtualMachine) -> PyRef<Self> { + vm.ctx.new_complex(zelf.value) + } +} + +impl Representable for PyComplex { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + // TODO: when you fix this, move it to rustpython_common::complex::repr and update + // ast/src/unparse.rs + impl Display for Constant in ast/src/constant.rs + let Complex64 { re, im } = zelf.value; + Ok(rustpython_literal::complex::to_string(re, im)) + } +} + +#[derive(FromArgs)] +pub struct ComplexArgs { + #[pyarg(any, optional)] + real: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + imag: OptionalArg<PyObjectRef>, +} diff --git a/crates/vm/src/builtins/coroutine.rs b/crates/vm/src/builtins/coroutine.rs new file mode 100644 index 00000000000..9746dddda87 --- /dev/null +++ b/crates/vm/src/builtins/coroutine.rs @@ -0,0 +1,253 @@ +use super::{PyCode, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; +use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + coroutine::{Coro, warn_deprecated_throw_signature}, + frame::FrameRef, + function::OptionalArg, + object::{Traverse, TraverseFn}, + protocol::PyIterReturn, + types::{Destructor, IterNext, Iterable, Representable, SelfIter}, +}; +use crossbeam_utils::atomic::AtomicCell; + +#[pyclass(module = false, name = "coroutine", traverse = "manual")] +#[derive(Debug)] +// PyCoro_Type in CPython +pub struct PyCoroutine { + inner: Coro, +} + +unsafe impl Traverse for PyCoroutine { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.inner.traverse(tracer_fn); + } +} + +impl PyPayload for PyCoroutine { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.coroutine_type + } +} + +#[pyclass( + flags(DISALLOW_INSTANTIATION, HAS_WEAKREF), + with(Py, IterNext, Representable, Destructor) +)] +impl PyCoroutine { + pub const fn as_coro(&self) -> &Coro { + &self.inner + } + + pub fn new(frame: FrameRef, name: PyStrRef, qualname: PyStrRef) -> Self { + Self { + inner: Coro::new(frame, name, qualname), + } + } + + #[pygetset] + fn __name__(&self) -> PyStrRef { + self.inner.name() + } + + #[pygetset(setter)] + fn set___name__(&self, name: PyStrRef) { + self.inner.set_name(name) + } + + #[pygetset] + fn __qualname__(&self) -> PyStrRef { + self.inner.qualname() + } + + #[pygetset(setter)] + fn set___qualname__(&self, qualname: PyStrRef) { + self.inner.set_qualname(qualname) + } + + #[pymethod(name = "__await__")] + fn r#await(zelf: PyRef<Self>) -> PyCoroutineWrapper { + PyCoroutineWrapper { + coro: zelf, + closed: AtomicCell::new(false), + } + } + + #[pygetset] + fn cr_await(&self, _vm: &VirtualMachine) -> Option<PyObjectRef> { + self.inner.frame().yield_from_target() + } + #[pygetset] + fn cr_frame(&self, _vm: &VirtualMachine) -> Option<FrameRef> { + if self.inner.closed() { + None + } else { + Some(self.inner.frame()) + } + } + #[pygetset] + fn cr_running(&self, _vm: &VirtualMachine) -> bool { + self.inner.running() + } + #[pygetset] + fn cr_code(&self, _vm: &VirtualMachine) -> PyRef<PyCode> { + self.inner.frame().code.clone() + } + // TODO: coroutine origin tracking: + // https://docs.python.org/3/library/sys.html#sys.set_coroutine_origin_tracking_depth + #[pygetset] + const fn cr_origin(&self, _vm: &VirtualMachine) -> Option<(PyStrRef, usize, PyStrRef)> { + None + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +#[pyclass] +impl Py<PyCoroutine> { + #[pymethod] + fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + self.inner.send(self.as_object(), value, vm) + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; + self.inner.throw( + self.as_object(), + exc_type, + exc_val.unwrap_or_none(vm), + exc_tb.unwrap_or_none(vm), + vm, + ) + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + self.inner.close(self.as_object(), vm) + } +} + +impl Representable for PyCoroutine { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + Ok(zelf.inner.repr(zelf.as_object(), zelf.get_id(), vm)) + } +} + +impl SelfIter for PyCoroutine {} +impl IterNext for PyCoroutine { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.send(vm.ctx.none(), vm) + } +} + +impl Destructor for PyCoroutine { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + if zelf.inner.closed() || zelf.inner.running() { + return Ok(()); + } + if zelf.inner.frame().lasti() == 0 { + zelf.inner.closed.store(true); + return Ok(()); + } + if let Err(e) = zelf.inner.close(zelf.as_object(), vm) { + vm.run_unraisable(e, None, zelf.as_object().to_owned()); + } + Ok(()) + } +} + +#[pyclass(module = false, name = "coroutine_wrapper", traverse = "manual")] +#[derive(Debug)] +// PyCoroWrapper_Type in CPython +pub struct PyCoroutineWrapper { + coro: PyRef<PyCoroutine>, + closed: AtomicCell<bool>, +} + +unsafe impl Traverse for PyCoroutineWrapper { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.coro.traverse(tracer_fn); + } +} + +impl PyPayload for PyCoroutineWrapper { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.coroutine_wrapper_type + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PyCoroutineWrapper { + fn check_closed(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.closed.load() { + return Err(vm.new_runtime_error("cannot reuse already awaited coroutine")); + } + Ok(()) + } + + #[pymethod] + fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + self.check_closed(vm)?; + let result = self.coro.send(val, vm); + // Mark as closed if exhausted + if let Ok(PyIterReturn::StopIteration(_)) = &result { + self.closed.store(true); + } + result + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + self.check_closed(vm)?; + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; + let result = self.coro.throw(exc_type, exc_val, exc_tb, vm); + // Mark as closed if exhausted + if let Ok(PyIterReturn::StopIteration(_)) = &result { + self.closed.store(true); + } + result + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + self.closed.store(true); + self.coro.close(vm) + } +} + +impl SelfIter for PyCoroutineWrapper {} +impl IterNext for PyCoroutineWrapper { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + Self::send(zelf, vm.ctx.none(), vm) + } +} + +impl Drop for PyCoroutine { + fn drop(&mut self) { + self.inner.frame().clear_generator(); + } +} + +pub fn init(ctx: &'static Context) { + PyCoroutine::extend_class(ctx, ctx.types.coroutine_type); + PyCoroutineWrapper::extend_class(ctx, ctx.types.coroutine_wrapper_type); +} diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs new file mode 100644 index 00000000000..acfe58d723b --- /dev/null +++ b/crates/vm/src/builtins/descriptor.rs @@ -0,0 +1,958 @@ +use super::{PyStr, PyStrInterned, PyType}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyTypeRef, builtin_func::PyNativeMethod, type_}, + class::PyClassImpl, + common::hash::PyHash, + convert::{ToPyObject, ToPyResult}, + function::{ArgSize, FuncArgs, PyMethodDef, PyMethodFlags, PySetterValue}, + protocol::{PyNumberBinaryFunc, PyNumberTernaryFunc, PyNumberUnaryFunc}, + types::{ + Callable, Comparable, DelFunc, DescrGetFunc, DescrSetFunc, GenericMethod, GetDescriptor, + GetattroFunc, HashFunc, Hashable, InitFunc, IterFunc, IterNextFunc, MapAssSubscriptFunc, + MapLenFunc, MapSubscriptFunc, PyComparisonOp, Representable, RichCompareFunc, + SeqAssItemFunc, SeqConcatFunc, SeqContainsFunc, SeqItemFunc, SeqLenFunc, SeqRepeatFunc, + SetattroFunc, StringifyFunc, + }, +}; +use rustpython_common::lock::PyRwLock; + +#[derive(Debug)] +pub struct PyDescriptor { + pub typ: &'static Py<PyType>, + pub name: &'static PyStrInterned, + pub qualname: PyRwLock<Option<String>>, +} + +#[derive(Debug)] +pub struct PyDescriptorOwned { + pub typ: PyRef<PyType>, + pub name: &'static PyStrInterned, + pub qualname: PyRwLock<Option<String>>, +} + +#[pyclass(name = "method_descriptor", module = false)] +pub struct PyMethodDescriptor { + pub common: PyDescriptor, + pub method: &'static PyMethodDef, + // vectorcall: vector_call_func, + pub objclass: &'static Py<PyType>, // TODO: move to tp_members + /// Prevent HeapMethodDef from being freed while this descriptor references it + pub(crate) _method_def_owner: Option<PyObjectRef>, +} + +impl PyMethodDescriptor { + pub fn new(method: &'static PyMethodDef, typ: &'static Py<PyType>, ctx: &Context) -> Self { + Self { + common: PyDescriptor { + typ, + name: ctx.intern_str(method.name), + qualname: PyRwLock::new(None), + }, + method, + objclass: typ, + _method_def_owner: None, + } + } +} + +impl PyPayload for PyMethodDescriptor { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.method_descriptor_type + } +} + +impl core::fmt::Debug for PyMethodDescriptor { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "method descriptor for '{}'", self.common.name) + } +} + +impl GetDescriptor for PyMethodDescriptor { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let descr = Self::_as_pyref(&zelf, vm).unwrap(); + let bound = match obj { + Some(obj) => { + if descr.method.flags.contains(PyMethodFlags::METHOD) { + if cls.is_some_and(|c| c.fast_isinstance(vm.ctx.types.type_type)) { + obj + } else { + return Err(vm.new_type_error(format!( + "descriptor '{}' needs a type, not '{}', as arg 2", + descr.common.name.as_str(), + obj.class().name() + ))); + } + } else if descr.method.flags.contains(PyMethodFlags::CLASS) { + obj.class().to_owned().into() + } else { + obj + } + } + None if descr.method.flags.contains(PyMethodFlags::CLASS) => cls.unwrap(), + None => return Ok(zelf), + }; + Ok(descr.bind(bound, &vm.ctx).into()) + } +} + +impl Callable for PyMethodDescriptor { + type Args = FuncArgs; + #[inline] + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + (zelf.method.func)(vm, args) + } +} + +impl PyMethodDescriptor { + pub fn bind(&self, obj: PyObjectRef, ctx: &Context) -> PyRef<PyNativeMethod> { + self.method.build_bound_method(ctx, obj, self.common.typ) + } +} + +#[pyclass( + with(GetDescriptor, Callable, Representable), + flags(METHOD_DESCRIPTOR, DISALLOW_INSTANTIATION) +)] +impl PyMethodDescriptor { + #[pygetset] + const fn __name__(&self) -> &'static PyStrInterned { + self.common.name + } + + #[pygetset] + fn __qualname__(&self) -> String { + format!("{}.{}", self.common.typ.name(), &self.common.name) + } + + #[pygetset] + const fn __doc__(&self) -> Option<&'static str> { + self.method.doc + } + + #[pygetset] + fn __text_signature__(&self) -> Option<String> { + self.method.doc.and_then(|doc| { + type_::get_text_signature_from_internal_doc(self.method.name, doc) + .map(|signature| signature.to_string()) + }) + } + + #[pygetset] + fn __objclass__(&self) -> PyTypeRef { + self.objclass.to_owned() + } + + #[pymethod] + fn __reduce__( + &self, + vm: &VirtualMachine, + ) -> (Option<PyObjectRef>, (Option<PyObjectRef>, &'static str)) { + let builtins_getattr = vm.builtins.get_attr("getattr", vm).ok(); + let classname = vm.builtins.get_attr(&self.common.typ.__name__(vm), vm).ok(); + (builtins_getattr, (classname, self.method.name)) + } +} + +impl Representable for PyMethodDescriptor { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<method '{}' of '{}' objects>", + &zelf.method.name, + zelf.common.typ.name() + )) + } +} + +#[derive(Debug)] +pub enum MemberKind { + Object = 6, + Bool = 14, + ObjectEx = 16, +} + +pub type MemberSetterFunc = Option<fn(&VirtualMachine, PyObjectRef, PySetterValue) -> PyResult<()>>; + +pub enum MemberGetter { + Getter(fn(&VirtualMachine, PyObjectRef) -> PyResult), + Offset(usize), +} + +pub enum MemberSetter { + Setter(MemberSetterFunc), + Offset(usize), +} + +pub struct PyMemberDef { + pub name: String, + pub kind: MemberKind, + pub getter: MemberGetter, + pub setter: MemberSetter, + pub doc: Option<String>, +} + +impl PyMemberDef { + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.getter { + MemberGetter::Getter(getter) => (getter)(vm, obj), + MemberGetter::Offset(offset) => get_slot_from_object(obj, offset, self, vm), + } + } + + fn set( + &self, + obj: PyObjectRef, + value: PySetterValue<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match self.setter { + MemberSetter::Setter(setter) => match setter { + Some(setter) => (setter)(vm, obj, value), + None => Err(vm.new_attribute_error("readonly attribute")), + }, + MemberSetter::Offset(offset) => set_slot_at_object(obj, offset, self, value, vm), + } + } +} + +impl core::fmt::Debug for PyMemberDef { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyMemberDef") + .field("name", &self.name) + .field("kind", &self.kind) + .field("doc", &self.doc) + .finish() + } +} + +// = PyMemberDescrObject +#[pyclass(name = "member_descriptor", module = false)] +#[derive(Debug)] +pub struct PyMemberDescriptor { + pub common: PyDescriptorOwned, + pub member: PyMemberDef, +} + +impl PyPayload for PyMemberDescriptor { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.member_descriptor_type + } +} + +fn calculate_qualname(descr: &PyDescriptorOwned, vm: &VirtualMachine) -> PyResult<Option<String>> { + if let Some(qualname) = vm.get_attribute_opt(descr.typ.clone().into(), "__qualname__")? { + let str = qualname.downcast::<PyStr>().map_err(|_| { + vm.new_type_error("<descriptor>.__objclass__.__qualname__ is not a unicode object") + })?; + Ok(Some(format!("{}.{}", str, descr.name))) + } else { + Ok(None) + } +} + +#[pyclass(with(GetDescriptor, Representable), flags(DISALLOW_INSTANTIATION))] +impl PyMemberDescriptor { + #[pymember] + fn __objclass__(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf: &Py<Self> = zelf.try_to_value(vm)?; + Ok(zelf.common.typ.clone().into()) + } + + #[pymember] + fn __name__(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf: &Py<Self> = zelf.try_to_value(vm)?; + Ok(zelf.common.name.to_owned().into()) + } + + #[pygetset] + fn __doc__(&self) -> Option<String> { + self.member.doc.to_owned() + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyResult<Option<String>> { + let qualname = self.common.qualname.read(); + Ok(if qualname.is_none() { + drop(qualname); + let calculated = calculate_qualname(&self.common, vm)?; + calculated.clone_into(&mut self.common.qualname.write()); + calculated + } else { + qualname.to_owned() + }) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let builtins_getattr = vm.builtins.get_attr("getattr", vm)?; + Ok(vm + .ctx + .new_tuple(vec![ + builtins_getattr, + vm.ctx + .new_tuple(vec![ + self.common.typ.clone().into(), + vm.ctx.new_str(self.common.name.as_str()).into(), + ]) + .into(), + ]) + .into()) + } + + #[pyslot] + fn descr_set( + zelf: &PyObject, + obj: PyObjectRef, + value: PySetterValue<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let zelf = Self::_as_pyref(zelf, vm)?; + + if !obj.class().fast_issubclass(&zelf.common.typ) { + return Err(vm.new_type_error(format!( + "descriptor '{}' for '{}' objects doesn't apply to a '{}' object", + zelf.common.name, + zelf.common.typ.name(), + obj.class().name() + ))); + } + + zelf.member.set(obj, value, vm) + } +} + +// PyMember_GetOne +fn get_slot_from_object( + obj: PyObjectRef, + offset: usize, + member: &PyMemberDef, + vm: &VirtualMachine, +) -> PyResult { + let slot = match member.kind { + MemberKind::Object => obj.get_slot(offset).unwrap_or_else(|| vm.ctx.none()), + MemberKind::Bool => obj + .get_slot(offset) + .unwrap_or_else(|| vm.ctx.new_bool(false).into()), + MemberKind::ObjectEx => obj.get_slot(offset).ok_or_else(|| { + vm.new_no_attribute_error(obj.clone(), vm.ctx.new_str(member.name.clone())) + })?, + }; + Ok(slot) +} + +// PyMember_SetOne +fn set_slot_at_object( + obj: PyObjectRef, + offset: usize, + member: &PyMemberDef, + value: PySetterValue, + vm: &VirtualMachine, +) -> PyResult<()> { + match member.kind { + MemberKind::Object => match value { + PySetterValue::Assign(v) => { + obj.set_slot(offset, Some(v)); + } + PySetterValue::Delete => { + obj.set_slot(offset, None); + } + }, + MemberKind::Bool => { + match value { + PySetterValue::Assign(v) => { + if !v.class().is(vm.ctx.types.bool_type) { + return Err(vm.new_type_error("attribute value type must be bool")); + } + obj.set_slot(offset, Some(v)) + } + PySetterValue::Delete => { + return Err(vm.new_type_error("can't delete numeric/char attribute")); + } + }; + } + MemberKind::ObjectEx => match value { + PySetterValue::Assign(v) => { + obj.set_slot(offset, Some(v)); + } + PySetterValue::Delete => { + if obj.get_slot(offset).is_none() { + return Err(vm.new_attribute_error(member.name.clone())); + } + obj.set_slot(offset, None); + } + }, + } + + Ok(()) +} + +impl Representable for PyMemberDescriptor { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<member '{}' of '{}' objects>", + zelf.common.name, + zelf.common.typ.name(), + )) + } +} + +impl GetDescriptor for PyMemberDescriptor { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let descr = Self::_as_pyref(&zelf, vm)?; + match obj { + Some(x) => { + if !x.class().fast_issubclass(&descr.common.typ) { + return Err(vm.new_type_error(format!( + "descriptor '{}' for '{}' objects doesn't apply to a '{}' object", + descr.common.name, + descr.common.typ.name(), + x.class().name() + ))); + } + descr.member.get(x, vm) + } + None => Ok(zelf), + } + } +} + +/// Vectorcall for method_descriptor: calls native method directly +fn vectorcall_method_descriptor( + zelf_obj: &PyObject, + args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py<PyMethodDescriptor> = zelf_obj.downcast_ref().unwrap(); + let func_args = FuncArgs::from_vectorcall_owned(args, nargs, kwnames); + (zelf.method.func)(vm, func_args) +} + +/// Vectorcall for wrapper_descriptor: calls wrapped slot function +fn vectorcall_wrapper( + zelf_obj: &PyObject, + mut args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py<PyWrapper> = zelf_obj.downcast_ref().unwrap(); + // First positional arg is self + if nargs == 0 { + return Err(vm.new_type_error(format!( + "descriptor '{}' of '{}' object needs an argument", + zelf.name.as_str(), + zelf.typ.name() + ))); + } + let obj = args.remove(0); + if !obj.fast_isinstance(zelf.typ) { + return Err(vm.new_type_error(format!( + "descriptor '{}' requires a '{}' object but received a '{}'", + zelf.name.as_str(), + zelf.typ.name(), + obj.class().name() + ))); + } + let rest = FuncArgs::from_vectorcall_owned(args, nargs - 1, kwnames); + zelf.wrapped.call(obj, rest, vm) +} + +pub fn init(ctx: &'static Context) { + PyMemberDescriptor::extend_class(ctx, ctx.types.member_descriptor_type); + PyMethodDescriptor::extend_class(ctx, ctx.types.method_descriptor_type); + ctx.types + .method_descriptor_type + .slots + .vectorcall + .store(Some(vectorcall_method_descriptor)); + PyWrapper::extend_class(ctx, ctx.types.wrapper_descriptor_type); + ctx.types + .wrapper_descriptor_type + .slots + .vectorcall + .store(Some(vectorcall_wrapper)); + PyMethodWrapper::extend_class(ctx, ctx.types.method_wrapper_type); +} + +// PyWrapper - wrapper_descriptor + +/// Each variant knows how to call the wrapped function with proper types +#[derive(Clone, Copy)] +pub enum SlotFunc { + // Basic slots + Init(InitFunc), + Hash(HashFunc), + Str(StringifyFunc), + Repr(StringifyFunc), + Iter(IterFunc), + IterNext(IterNextFunc), + Call(GenericMethod), + Del(DelFunc), + + // Attribute access slots + GetAttro(GetattroFunc), + SetAttro(SetattroFunc), // __setattr__ + DelAttro(SetattroFunc), // __delattr__ (same func type, different PySetterValue) + + // Rich comparison slots (with comparison op) + RichCompare(RichCompareFunc, PyComparisonOp), + + // Descriptor slots + DescrGet(DescrGetFunc), + DescrSet(DescrSetFunc), // __set__ + DescrDel(DescrSetFunc), // __delete__ (same func type, different PySetterValue) + + // Sequence sub-slots (sq_*) + SeqLength(SeqLenFunc), + SeqConcat(SeqConcatFunc), + SeqRepeat(SeqRepeatFunc), + SeqItem(SeqItemFunc), + SeqSetItem(SeqAssItemFunc), // __setitem__ (same func type, value = Some) + SeqDelItem(SeqAssItemFunc), // __delitem__ (same func type, value = None) + SeqContains(SeqContainsFunc), + + // Mapping sub-slots (mp_*) + MapLength(MapLenFunc), + MapSubscript(MapSubscriptFunc), + MapSetSubscript(MapAssSubscriptFunc), // __setitem__ (same func type, value = Some) + MapDelSubscript(MapAssSubscriptFunc), // __delitem__ (same func type, value = None) + + // Number sub-slots (nb_*) - grouped by signature + NumBoolean(PyNumberUnaryFunc<bool>), // __bool__ + NumUnary(PyNumberUnaryFunc), // __int__, __float__, __index__ + NumBinary(PyNumberBinaryFunc), // __add__, __sub__, __mul__, etc. + NumBinaryRight(PyNumberBinaryFunc), // __radd__, __rsub__, etc. (swapped args) + NumTernary(PyNumberTernaryFunc), // __pow__ + NumTernaryRight(PyNumberTernaryFunc), // __rpow__ (swapped first two args) +} + +impl core::fmt::Debug for SlotFunc { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + SlotFunc::Init(_) => write!(f, "SlotFunc::Init(...)"), + SlotFunc::Hash(_) => write!(f, "SlotFunc::Hash(...)"), + SlotFunc::Str(_) => write!(f, "SlotFunc::Str(...)"), + SlotFunc::Repr(_) => write!(f, "SlotFunc::Repr(...)"), + SlotFunc::Iter(_) => write!(f, "SlotFunc::Iter(...)"), + SlotFunc::IterNext(_) => write!(f, "SlotFunc::IterNext(...)"), + SlotFunc::Call(_) => write!(f, "SlotFunc::Call(...)"), + SlotFunc::Del(_) => write!(f, "SlotFunc::Del(...)"), + SlotFunc::GetAttro(_) => write!(f, "SlotFunc::GetAttro(...)"), + SlotFunc::SetAttro(_) => write!(f, "SlotFunc::SetAttro(...)"), + SlotFunc::DelAttro(_) => write!(f, "SlotFunc::DelAttro(...)"), + SlotFunc::RichCompare(_, op) => write!(f, "SlotFunc::RichCompare(..., {:?})", op), + SlotFunc::DescrGet(_) => write!(f, "SlotFunc::DescrGet(...)"), + SlotFunc::DescrSet(_) => write!(f, "SlotFunc::DescrSet(...)"), + SlotFunc::DescrDel(_) => write!(f, "SlotFunc::DescrDel(...)"), + // Sequence sub-slots + SlotFunc::SeqLength(_) => write!(f, "SlotFunc::SeqLength(...)"), + SlotFunc::SeqConcat(_) => write!(f, "SlotFunc::SeqConcat(...)"), + SlotFunc::SeqRepeat(_) => write!(f, "SlotFunc::SeqRepeat(...)"), + SlotFunc::SeqItem(_) => write!(f, "SlotFunc::SeqItem(...)"), + SlotFunc::SeqSetItem(_) => write!(f, "SlotFunc::SeqSetItem(...)"), + SlotFunc::SeqDelItem(_) => write!(f, "SlotFunc::SeqDelItem(...)"), + SlotFunc::SeqContains(_) => write!(f, "SlotFunc::SeqContains(...)"), + // Mapping sub-slots + SlotFunc::MapLength(_) => write!(f, "SlotFunc::MapLength(...)"), + SlotFunc::MapSubscript(_) => write!(f, "SlotFunc::MapSubscript(...)"), + SlotFunc::MapSetSubscript(_) => write!(f, "SlotFunc::MapSetSubscript(...)"), + SlotFunc::MapDelSubscript(_) => write!(f, "SlotFunc::MapDelSubscript(...)"), + // Number sub-slots + SlotFunc::NumBoolean(_) => write!(f, "SlotFunc::NumBoolean(...)"), + SlotFunc::NumUnary(_) => write!(f, "SlotFunc::NumUnary(...)"), + SlotFunc::NumBinary(_) => write!(f, "SlotFunc::NumBinary(...)"), + SlotFunc::NumBinaryRight(_) => write!(f, "SlotFunc::NumBinaryRight(...)"), + SlotFunc::NumTernary(_) => write!(f, "SlotFunc::NumTernary(...)"), + SlotFunc::NumTernaryRight(_) => write!(f, "SlotFunc::NumTernaryRight(...)"), + } + } +} + +impl SlotFunc { + /// Call the wrapped slot function with proper type handling + pub fn call(&self, obj: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + match self { + SlotFunc::Init(func) => { + func(obj, args, vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::Hash(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err(vm.new_type_error("__hash__() takes no arguments (1 given)")); + } + let hash = func(&obj, vm)?; + Ok(vm.ctx.new_int(hash).into()) + } + SlotFunc::Repr(func) | SlotFunc::Str(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + let name = match self { + SlotFunc::Repr(_) => "__repr__", + SlotFunc::Str(_) => "__str__", + _ => unreachable!(), + }; + return Err(vm.new_type_error(format!("{name}() takes no arguments (1 given)"))); + } + let s = func(&obj, vm)?; + Ok(s.into()) + } + SlotFunc::Iter(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err(vm.new_type_error("__iter__() takes no arguments (1 given)")); + } + func(obj, vm) + } + SlotFunc::IterNext(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err(vm.new_type_error("__next__() takes no arguments (1 given)")); + } + func(&obj, vm).to_pyresult(vm) + } + SlotFunc::Call(func) => func(&obj, args, vm), + SlotFunc::Del(func) => { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err(vm.new_type_error("__del__() takes no arguments (1 given)")); + } + func(&obj, vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::GetAttro(func) => { + let (name,): (PyRef<PyStr>,) = args.bind(vm)?; + func(&obj, &name, vm) + } + SlotFunc::SetAttro(func) => { + let (name, value): (PyRef<PyStr>, PyObjectRef) = args.bind(vm)?; + func(&obj, &name, PySetterValue::Assign(value), vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::DelAttro(func) => { + let (name,): (PyRef<PyStr>,) = args.bind(vm)?; + func(&obj, &name, PySetterValue::Delete, vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::RichCompare(func, op) => { + let (other,): (PyObjectRef,) = args.bind(vm)?; + func(&obj, &other, *op, vm).map(|r| match r { + crate::function::Either::A(obj) => obj, + crate::function::Either::B(cmp_val) => cmp_val.to_pyobject(vm), + }) + } + SlotFunc::DescrGet(func) => { + let (instance, owner): (PyObjectRef, crate::function::OptionalArg<PyObjectRef>) = + args.bind(vm)?; + let owner = owner.into_option(); + let instance_opt = if vm.is_none(&instance) { + None + } else { + Some(instance) + }; + func(obj, instance_opt, owner, vm) + } + SlotFunc::DescrSet(func) => { + let (instance, value): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + func(&obj, instance, PySetterValue::Assign(value), vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::DescrDel(func) => { + let (instance,): (PyObjectRef,) = args.bind(vm)?; + func(&obj, instance, PySetterValue::Delete, vm)?; + Ok(vm.ctx.none()) + } + // Sequence sub-slots + SlotFunc::SeqLength(func) => { + args.bind::<()>(vm)?; + let len = func(obj.sequence_unchecked(), vm)?; + Ok(vm.ctx.new_int(len).into()) + } + SlotFunc::SeqConcat(func) => { + let (other,): (PyObjectRef,) = args.bind(vm)?; + func(obj.sequence_unchecked(), &other, vm) + } + SlotFunc::SeqRepeat(func) => { + let (n,): (ArgSize,) = args.bind(vm)?; + func(obj.sequence_unchecked(), n.into(), vm) + } + SlotFunc::SeqItem(func) => { + let (index,): (isize,) = args.bind(vm)?; + func(obj.sequence_unchecked(), index, vm) + } + SlotFunc::SeqSetItem(func) => { + let (index, value): (isize, PyObjectRef) = args.bind(vm)?; + func(obj.sequence_unchecked(), index, Some(value), vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::SeqDelItem(func) => { + let (index,): (isize,) = args.bind(vm)?; + func(obj.sequence_unchecked(), index, None, vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::SeqContains(func) => { + let (item,): (PyObjectRef,) = args.bind(vm)?; + let result = func(obj.sequence_unchecked(), &item, vm)?; + Ok(vm.ctx.new_bool(result).into()) + } + // Mapping sub-slots + SlotFunc::MapLength(func) => { + args.bind::<()>(vm)?; + let len = func(obj.mapping_unchecked(), vm)?; + Ok(vm.ctx.new_int(len).into()) + } + SlotFunc::MapSubscript(func) => { + let (key,): (PyObjectRef,) = args.bind(vm)?; + func(obj.mapping_unchecked(), &key, vm) + } + SlotFunc::MapSetSubscript(func) => { + let (key, value): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + func(obj.mapping_unchecked(), &key, Some(value), vm)?; + Ok(vm.ctx.none()) + } + SlotFunc::MapDelSubscript(func) => { + let (key,): (PyObjectRef,) = args.bind(vm)?; + func(obj.mapping_unchecked(), &key, None, vm)?; + Ok(vm.ctx.none()) + } + // Number sub-slots + SlotFunc::NumBoolean(func) => { + args.bind::<()>(vm)?; + let result = func(obj.number(), vm)?; + Ok(vm.ctx.new_bool(result).into()) + } + SlotFunc::NumUnary(func) => { + args.bind::<()>(vm)?; + func(obj.number(), vm) + } + SlotFunc::NumBinary(func) => { + let (other,): (PyObjectRef,) = args.bind(vm)?; + func(&obj, &other, vm) + } + SlotFunc::NumBinaryRight(func) => { + let (other,): (PyObjectRef,) = args.bind(vm)?; + func(&other, &obj, vm) // Swapped: other op obj + } + SlotFunc::NumTernary(func) => { + let (y, z): (PyObjectRef, crate::function::OptionalArg<PyObjectRef>) = + args.bind(vm)?; + let z = z.unwrap_or_else(|| vm.ctx.none()); + func(&obj, &y, &z, vm) + } + SlotFunc::NumTernaryRight(func) => { + let (y, z): (PyObjectRef, crate::function::OptionalArg<PyObjectRef>) = + args.bind(vm)?; + let z = z.unwrap_or_else(|| vm.ctx.none()); + func(&y, &obj, &z, vm) // Swapped: y ** obj % z + } + } + } +} + +/// wrapper_descriptor: wraps a slot function as a Python method +// = PyWrapperDescrObject +#[pyclass(name = "wrapper_descriptor", module = false)] +#[derive(Debug)] +pub struct PyWrapper { + pub typ: &'static Py<PyType>, + pub name: &'static PyStrInterned, + pub wrapped: SlotFunc, + pub doc: Option<&'static str>, +} + +impl PyPayload for PyWrapper { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.wrapper_descriptor_type + } +} + +impl GetDescriptor for PyWrapper { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + match obj { + None => Ok(zelf), + Some(obj) => { + let zelf = zelf.downcast::<Self>().unwrap(); + Ok(PyMethodWrapper { wrapper: zelf, obj }.into_pyobject(vm)) + } + } + } +} + +impl Callable for PyWrapper { + type Args = FuncArgs; + + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // list.__init__(l, [1,2,3]) form - first arg is self + let (obj, rest): (PyObjectRef, FuncArgs) = args.bind(vm)?; + + if !obj.fast_isinstance(zelf.typ) { + return Err(vm.new_type_error(format!( + "descriptor '{}' requires a '{}' object but received a '{}'", + zelf.name.as_str(), + zelf.typ.name(), + obj.class().name() + ))); + } + + zelf.wrapped.call(obj, rest, vm) + } +} + +#[pyclass( + with(GetDescriptor, Callable, Representable), + flags(DISALLOW_INSTANTIATION) +)] +impl PyWrapper { + #[pygetset] + fn __name__(&self) -> &'static PyStrInterned { + self.name + } + + #[pygetset] + fn __qualname__(&self) -> String { + format!("{}.{}", self.typ.name(), self.name) + } + + #[pygetset] + fn __objclass__(&self) -> PyTypeRef { + self.typ.to_owned() + } + + #[pygetset] + fn __doc__(&self) -> Option<&'static str> { + self.doc + } +} + +impl Representable for PyWrapper { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<slot wrapper '{}' of '{}' objects>", + zelf.name.as_str(), + zelf.typ.name() + )) + } +} + +// PyMethodWrapper - method-wrapper + +/// method-wrapper: a slot wrapper bound to an instance +/// Returned when accessing l.__init__ on an instance +#[pyclass(name = "method-wrapper", module = false, traverse)] +#[derive(Debug)] +pub struct PyMethodWrapper { + pub wrapper: PyRef<PyWrapper>, + #[pytraverse(skip)] + pub obj: PyObjectRef, +} + +impl PyPayload for PyMethodWrapper { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.method_wrapper_type + } +} + +impl Callable for PyMethodWrapper { + type Args = FuncArgs; + + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // bpo-37619: Check type compatibility before calling wrapped slot + if !zelf.obj.fast_isinstance(zelf.wrapper.typ) { + return Err(vm.new_type_error(format!( + "descriptor '{}' requires a '{}' object but received a '{}'", + zelf.wrapper.name.as_str(), + zelf.wrapper.typ.name(), + zelf.obj.class().name() + ))); + } + zelf.wrapper.wrapped.call(zelf.obj.clone(), args, vm) + } +} + +#[pyclass( + with(Callable, Representable, Hashable, Comparable), + flags(DISALLOW_INSTANTIATION) +)] +impl PyMethodWrapper { + #[pygetset] + fn __self__(&self) -> PyObjectRef { + self.obj.clone() + } + + #[pygetset] + fn __name__(&self) -> &'static PyStrInterned { + self.wrapper.name + } + + #[pygetset] + fn __objclass__(&self) -> PyTypeRef { + self.wrapper.typ.to_owned() + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let builtins_getattr = vm.builtins.get_attr("getattr", vm)?; + Ok(vm + .ctx + .new_tuple(vec![ + builtins_getattr, + vm.ctx + .new_tuple(vec![ + zelf.obj.clone(), + vm.ctx.new_str(zelf.wrapper.name.as_str()).into(), + ]) + .into(), + ]) + .into()) + } +} + +impl Representable for PyMethodWrapper { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<method-wrapper '{}' of {} object at {:#x}>", + zelf.wrapper.name.as_str(), + zelf.obj.class().name(), + zelf.obj.get_id() + )) + } +} + +impl Hashable for PyMethodWrapper { + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + let obj_hash = zelf.obj.hash(vm)?; + let wrapper_hash = zelf.wrapper.as_object().get_id() as PyHash; + Ok(obj_hash ^ wrapper_hash) + } +} + +impl Comparable for PyMethodWrapper { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<crate::function::PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + let eq = zelf.wrapper.is(&other.wrapper) && vm.bool_eq(&zelf.obj, &other.obj)?; + Ok(eq.into()) + }) + } +} diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs new file mode 100644 index 00000000000..0e64e9e66ac --- /dev/null +++ b/crates/vm/src/builtins/dict.rs @@ -0,0 +1,1447 @@ +use super::{ + IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, PyMappingProxy, PySet, + PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set::PySetInner, +}; +use crate::common::lock::LazyLock; +use crate::object::{Traverse, TraverseFn}; +use crate::{ + AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromObject, atomic_func, + builtins::{ + PyTuple, + iter::{builtins_iter, builtins_reversed}, + type_::PyAttributes, + }, + class::{PyClassDef, PyClassImpl}, + common::ascii, + dict_inner::{self, DictKey}, + function::{ArgIterable, KwArgs, OptionalArg, PyArithmeticValue::*, PyComparisonValue}, + iter::PyExactSizeIterator, + protocol::{PyIterIter, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + recursion::ReprGuard, + types::{ + AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, DefaultConstructor, + Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, + }, + vm::VirtualMachine, +}; +use alloc::fmt; +use core::cell::Cell; +use core::ptr::NonNull; +use rustpython_common::lock::PyMutex; +use rustpython_common::wtf8::Wtf8Buf; + +pub type DictContentType = dict_inner::Dict; + +#[pyclass(module = false, name = "dict", unhashable = true, traverse = "manual")] +#[derive(Default)] +pub struct PyDict { + entries: DictContentType, +} +pub type PyDictRef = PyRef<PyDict>; + +// SAFETY: Traverse properly visits all owned PyObjectRefs +unsafe impl Traverse for PyDict { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.entries.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec<PyObjectRef>) { + // Pop all entries and collect both keys and values + for (key, value) in self.entries.drain_entries() { + out.push(key); + out.push(value); + } + } +} + +impl fmt::Debug for PyDict { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO: implement more detailed, non-recursive Debug formatter + f.write_str("dict") + } +} + +thread_local! { + static DICT_FREELIST: Cell<crate::object::FreeList<PyDict>> = const { Cell::new(crate::object::FreeList::new()) }; +} + +impl PyPayload for PyDict { + const MAX_FREELIST: usize = 80; + const HAS_FREELIST: bool = true; + + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.dict_type + } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + DICT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> { + DICT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } +} + +impl PyDict { + #[deprecated(note = "use PyDict::default().into_ref() instead")] + pub fn new_ref(ctx: &Context) -> PyRef<Self> { + Self::default().into_ref(ctx) + } + + /// escape hatch to access the underlying data structure directly. prefer adding a method on + /// PyDict instead of using this + pub(crate) const fn _as_dict_inner(&self) -> &DictContentType { + &self.entries + } + + /// Monotonically increasing version for mutation tracking. + pub(crate) fn version(&self) -> u64 { + self.entries.version() + } + + /// Returns all keys as a Vec, atomically under a single read lock. + /// Thread-safe: prevents "dictionary changed size during iteration" errors. + pub fn keys_vec(&self) -> Vec<PyObjectRef> { + self.entries.keys() + } + + /// Returns all values as a Vec, atomically under a single read lock. + /// Thread-safe: prevents "dictionary changed size during iteration" errors. + pub fn values_vec(&self) -> Vec<PyObjectRef> { + self.entries.values() + } + + /// Returns all items as a Vec, atomically under a single read lock. + /// Thread-safe: prevents "dictionary changed size during iteration" errors. + pub fn items_vec(&self) -> Vec<(PyObjectRef, PyObjectRef)> { + self.entries.items() + } + + // Used in update and ior. + pub(crate) fn merge_object(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let casted: Result<PyRefExact<Self>, _> = other.downcast_exact(vm); + let other = match casted { + Ok(dict_other) => return self.merge_dict(dict_other.into_pyref(), vm), + Err(other) => other, + }; + let dict = &self.entries; + // Use get_attr to properly invoke __getattribute__ for proxy objects + let keys_result = other.get_attr(vm.ctx.intern_str("keys"), vm); + let has_keys = match keys_result { + Ok(keys_method) => { + let keys = keys_method.call((), vm)?.get_iter(vm)?; + while let PyIterReturn::Return(key) = keys.next(vm)? { + let val = other.get_item(&*key, vm)?; + dict.insert(vm, &*key, val)?; + } + true + } + Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => false, + Err(e) => return Err(e), + }; + if !has_keys { + let iter = other.get_iter(vm)?; + loop { + fn err(vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_value_error("Iterator must have exactly two elements") + } + let element = match iter.next(vm)? { + PyIterReturn::Return(obj) => obj, + PyIterReturn::StopIteration(_) => break, + }; + let elem_iter = element.get_iter(vm)?; + let key = elem_iter.next(vm)?.into_result().map_err(|_| err(vm))?; + let value = elem_iter.next(vm)?.into_result().map_err(|_| err(vm))?; + if matches!(elem_iter.next(vm)?, PyIterReturn::Return(_)) { + return Err(err(vm)); + } + dict.insert(vm, &*key, value)?; + } + } + Ok(()) + } + + fn merge_dict(&self, dict_other: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { + let dict = &self.entries; + let dict_size = &dict_other.size(); + for (key, value) in &dict_other { + dict.insert(vm, &*key, value)?; + } + if dict_other.entries.has_changed_size(dict_size) { + return Err(vm.new_runtime_error("dict mutated during update")); + } + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.entries.len() == 0 + } + + /// Set item variant which can be called with multiple + /// key types, such as str to name a notable one. + pub(crate) fn inner_setitem<K: DictKey + ?Sized>( + &self, + key: &K, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.entries.insert(vm, key, value) + } + + pub(crate) fn inner_delitem<K: DictKey + ?Sized>( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.entries.delete(vm, key) + } + + pub fn get_or_insert( + &self, + vm: &VirtualMachine, + key: PyObjectRef, + default: impl FnOnce() -> PyObjectRef, + ) -> PyResult { + self.entries.setdefault(vm, &*key, default) + } + + pub fn from_attributes(attrs: PyAttributes, vm: &VirtualMachine) -> PyResult<Self> { + let entries = DictContentType::default(); + + for (key, value) in attrs { + entries.insert(vm, key, value)?; + } + + Ok(Self { entries }) + } + + pub fn contains_key<K: DictKey + ?Sized>(&self, key: &K, vm: &VirtualMachine) -> bool { + self.entries.contains(vm, key).unwrap() + } + + pub fn size(&self) -> dict_inner::DictSize { + self.entries.size() + } +} + +// Python dict methods: +#[allow(clippy::len_without_is_empty)] +#[pyclass( + with( + Py, + PyRef, + Constructor, + Initializer, + Comparable, + Iterable, + AsSequence, + AsNumber, + AsMapping, + Representable + ), + flags(BASETYPE, MAPPING, _MATCH_SELF) +)] +impl PyDict { + #[pyclassmethod] + fn fromkeys( + class: PyTypeRef, + iterable: ArgIterable, + value: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let value = value.unwrap_or_none(vm); + let d = PyType::call(&class, ().into(), vm)?; + match d.downcast_exact::<Self>(vm) { + Ok(pydict) => { + for key in iterable.iter(vm)? { + pydict.__setitem__(key?, value.clone(), vm)?; + } + Ok(pydict.into_pyref().into()) + } + Err(pyobj) => { + for key in iterable.iter(vm)? { + pyobj.set_item(&*key?, value.clone(), vm)?; + } + Ok(pyobj) + } + } + } + + pub fn __len__(&self) -> usize { + self.entries.len() + } + + #[pymethod] + fn __sizeof__(&self) -> usize { + core::mem::size_of::<Self>() + self.entries.sizeof() + } + + fn __contains__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + self.entries.contains(vm, &*key) + } + + fn __delitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner_delitem(&*key, vm) + } + + #[pymethod] + pub fn clear(&self) { + self.entries.clear() + } + + fn __setitem__( + &self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.inner_setitem(&*key, value, vm) + } + + #[pymethod] + fn get( + &self, + key: PyObjectRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + match self.entries.get(vm, &*key)? { + Some(value) => Ok(value), + None => Ok(default.unwrap_or_none(vm)), + } + } + + #[pymethod] + fn setdefault( + &self, + key: PyObjectRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + self.entries + .setdefault(vm, &*key, || default.unwrap_or_none(vm)) + } + + #[pymethod] + pub fn copy(&self) -> Self { + Self { + entries: self.entries.clone(), + } + } + + #[pymethod] + fn update( + &self, + dict_obj: OptionalArg<PyObjectRef>, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let OptionalArg::Present(dict_obj) = dict_obj { + self.merge_object(dict_obj, vm)?; + } + for (key, value) in kwargs { + self.entries.insert(vm, &key, value)?; + } + Ok(()) + } + + fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let other_dict: Result<PyDictRef, _> = other.downcast(); + if let Ok(other) = other_dict { + let self_cp = self.copy(); + self_cp.merge_dict(other, vm)?; + return Ok(self_cp.into_pyobject(vm)); + } + Ok(vm.ctx.not_implemented()) + } + + #[pymethod] + fn pop( + &self, + key: PyObjectRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + match self.entries.pop(vm, &*key)? { + Some(value) => Ok(value), + None => default.ok_or_else(|| vm.new_key_error(key)), + } + } + + #[pymethod] + fn popitem(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { + let (key, value) = self.entries.pop_back().ok_or_else(|| { + let err_msg = vm + .ctx + .new_str(ascii!("popitem(): dictionary is empty")) + .into(); + vm.new_key_error(err_msg) + })?; + Ok((key, value)) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +#[pyclass] +impl Py<PyDict> { + fn inner_cmp( + &self, + other: &Self, + op: PyComparisonOp, + item: bool, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + if op == PyComparisonOp::Ne { + return Self::inner_cmp(self, other, PyComparisonOp::Eq, item, vm) + .map(|x| x.map(|eq| !eq)); + } + if !op.eval_ord(self.__len__().cmp(&other.__len__())) { + return Ok(Implemented(false)); + } + let (superset, subset) = if self.__len__() < other.__len__() { + (other, self) + } else { + (self, other) + }; + for (k, v1) in subset { + match superset.get_item_opt(&*k, vm)? { + Some(v2) => { + if v1.is(&v2) { + continue; + } + if item && !vm.bool_eq(&v1, &v2)? { + return Ok(Implemented(false)); + } + } + None => { + return Ok(Implemented(false)); + } + } + } + Ok(Implemented(true)) + } + + #[cfg_attr(feature = "flame-it", flame("PyDictRef"))] + fn __getitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner_getitem(&*key, vm) + } +} + +#[pyclass] +impl PyRef<PyDict> { + #[pymethod] + const fn keys(self) -> PyDictKeys { + PyDictKeys::new(self) + } + + #[pymethod] + const fn values(self) -> PyDictValues { + PyDictValues::new(self) + } + + #[pymethod] + const fn items(self) -> PyDictItems { + PyDictItems::new(self) + } + + #[pymethod] + fn __reversed__(self) -> PyDictReverseKeyIterator { + PyDictReverseKeyIterator::new(self) + } + + fn __ior__(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { + self.merge_object(other, vm)?; + Ok(self) + } + + fn __ror__(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let other_dict: Result<Self, _> = other.downcast(); + if let Ok(other) = other_dict { + let other_cp = other.copy(); + other_cp.merge_dict(self, vm)?; + return Ok(other_cp.into_pyobject(vm)); + } + Ok(vm.ctx.not_implemented()) + } +} + +impl DefaultConstructor for PyDict {} + +impl Initializer for PyDict { + type Args = (OptionalArg<PyObjectRef>, KwArgs); + + fn init( + zelf: PyRef<Self>, + (dict_obj, kwargs): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<()> { + zelf.update(dict_obj, kwargs, vm) + } +} + +impl AsMapping for PyDict { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: PyMappingMethods = PyMappingMethods { + length: atomic_func!(|mapping, _vm| Ok(PyDict::mapping_downcast(mapping).__len__())), + subscript: atomic_func!(|mapping, needle, vm| { + PyDict::mapping_downcast(mapping).inner_getitem(needle, vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyDict::mapping_downcast(mapping); + if let Some(value) = value { + zelf.inner_setitem(needle, value, vm) + } else { + zelf.inner_delitem(needle, vm) + } + }), + }; + &AS_MAPPING + } +} + +impl AsSequence for PyDict { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + contains: atomic_func!(|seq, target, vm| PyDict::sequence_downcast(seq) + .entries + .contains(vm, target)), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsNumber for PyDict { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PyDict>() { + PyDict::__or__(a, b.to_pyobject(vm), vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + inplace_or: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PyDict>() { + a.to_owned() + .__ior__(b.to_pyobject(vm), vm) + .map(|d| d.into()) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Comparable for PyDict { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + zelf.inner_cmp(other, PyComparisonOp::Eq, true, vm) + }) + } +} + +impl Iterable for PyDict { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyDictKeyIterator::new(zelf).into_pyobject(vm)) + } +} + +impl Representable for PyDict { + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let s = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let mut result = Wtf8Buf::from("{"); + let mut first = true; + for (key, value) in zelf { + if !first { + result.push_str(", "); + } + first = false; + result.push_wtf8(key.repr(vm)?.as_wtf8()); + result.push_str(": "); + result.push_wtf8(value.repr(vm)?.as_wtf8()); + } + result.push_char('}'); + vm.ctx.new_str(result) + } else { + vm.ctx.intern_str("{...}").to_owned() + }; + Ok(s) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } +} + +impl Py<PyDict> { + #[inline] + fn exact_dict(&self, vm: &VirtualMachine) -> bool { + self.class().is(vm.ctx.types.dict_type) + } + + fn missing_opt<K: DictKey + ?Sized>( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + vm.get_method(self.to_owned().into(), identifier!(vm, __missing__)) + .map(|methods| methods?.call((key.to_pyobject(vm),), vm)) + .transpose() + } + + #[inline] + fn inner_getitem<K: DictKey + ?Sized>( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + if let Some(value) = self.entries.get(vm, key)? { + Ok(value) + } else if let Some(value) = self.missing_opt(key, vm)? { + Ok(value) + } else { + Err(vm.new_key_error(key.to_pyobject(vm))) + } + } + + /// Take a python dictionary and convert it to attributes. + pub fn to_attributes(&self, vm: &VirtualMachine) -> PyAttributes { + let mut attrs = PyAttributes::default(); + for (key, value) in self { + let key: PyRefExact<PyStr> = key.downcast_exact(vm).expect("dict has non-string keys"); + attrs.insert(vm.ctx.intern_str(key), value); + } + attrs + } + + pub fn get_item_opt<K: DictKey + ?Sized>( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + if self.exact_dict(vm) { + self.entries.get(vm, key) + // FIXME: check __missing__? + } else { + match self.as_object().get_item(key, vm) { + Ok(value) => Ok(Some(value)), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + self.missing_opt(key, vm) + } + Err(e) => Err(e), + } + } + } + + /// Return a cached-entry hint for exact dict fast paths. + pub(crate) fn hint_for_key<K: DictKey + ?Sized>( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult<Option<u16>> { + if self.exact_dict(vm) { + self.entries.hint_for_key(vm, key) + } else { + Ok(None) + } + } + + /// Fast lookup using a cached entry index hint. + pub(crate) fn get_item_opt_hint<K: DictKey + ?Sized>( + &self, + key: &K, + hint: u16, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + if self.exact_dict(vm) { + self.entries.get_hint(vm, key, usize::from(hint)) + } else { + self.get_item_opt(key, vm) + } + } + + pub fn get_item<K: DictKey + ?Sized>(&self, key: &K, vm: &VirtualMachine) -> PyResult { + if self.exact_dict(vm) { + self.inner_getitem(key, vm) + } else { + self.as_object().get_item(key, vm) + } + } + + pub fn set_item<K: DictKey + ?Sized>( + &self, + key: &K, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if self.exact_dict(vm) { + self.inner_setitem(key, value, vm) + } else { + self.as_object().set_item(key, value, vm) + } + } + + pub fn del_item<K: DictKey + ?Sized>(&self, key: &K, vm: &VirtualMachine) -> PyResult<()> { + if self.exact_dict(vm) { + self.inner_delitem(key, vm) + } else { + self.as_object().del_item(key, vm) + } + } + + pub fn pop_item<K: DictKey + ?Sized>( + &self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + if self.exact_dict(vm) { + self.entries.remove_if_exists(vm, key) + } else { + let value = self.as_object().get_item(key, vm)?; + self.as_object().del_item(key, vm)?; + Ok(Some(value)) + } + } + + pub fn get_chain<K: DictKey + ?Sized>( + &self, + other: &Self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + let self_exact = self.exact_dict(vm); + let other_exact = other.exact_dict(vm); + if self_exact && other_exact { + // SAFETY: exact_dict checks passed + let self_exact = unsafe { PyExact::ref_unchecked(self) }; + let other_exact = unsafe { PyExact::ref_unchecked(other) }; + self_exact.get_chain_exact(other_exact, key, vm) + } else if let Some(value) = self.get_item_opt(key, vm)? { + Ok(Some(value)) + } else { + other.get_item_opt(key, vm) + } + } +} + +impl PyExact<PyDict> { + /// Look up `key` in `self`, falling back to `other`. + /// Both dicts must be exact `dict` types (enforced by `PyExact`). + pub(crate) fn get_chain_exact<K: DictKey + ?Sized>( + &self, + other: &Self, + key: &K, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + debug_assert!(self.class().is(vm.ctx.types.dict_type)); + debug_assert!(other.class().is(vm.ctx.types.dict_type)); + self.entries.get_chain(&other.entries, vm, key) + } +} + +// Implement IntoIterator so that we can easily iterate dictionaries from rust code. +impl IntoIterator for PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIntoIter; + + fn into_iter(self) -> Self::IntoIter { + DictIntoIter::new(self) + } +} + +impl<'a> IntoIterator for &'a PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + DictIter::new(self) + } +} + +impl<'a> IntoIterator for &'a Py<PyDict> { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + DictIter::new(self) + } +} + +impl<'a> IntoIterator for &'a PyDict { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + DictIter::new(self) + } +} + +pub struct DictIntoIter { + dict: PyDictRef, + position: usize, +} + +impl DictIntoIter { + pub const fn new(dict: PyDictRef) -> Self { + Self { dict, position: 0 } + } +} + +impl Iterator for DictIntoIter { + type Item = (PyObjectRef, PyObjectRef); + + fn next(&mut self) -> Option<Self::Item> { + let (position, key, value) = self.dict.entries.next_entry(self.position)?; + self.position = position; + Some((key, value)) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + let l = self.len(); + (l, Some(l)) + } +} +impl ExactSizeIterator for DictIntoIter { + fn len(&self) -> usize { + self.dict.entries.len_from_entry_index(self.position) + } +} + +pub struct DictIter<'a> { + dict: &'a PyDict, + position: usize, +} + +impl<'a> DictIter<'a> { + pub const fn new(dict: &'a PyDict) -> Self { + DictIter { dict, position: 0 } + } +} + +impl Iterator for DictIter<'_> { + type Item = (PyObjectRef, PyObjectRef); + + fn next(&mut self) -> Option<Self::Item> { + let (position, key, value) = self.dict.entries.next_entry(self.position)?; + self.position = position; + Some((key, value)) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + let l = self.len(); + (l, Some(l)) + } +} +impl ExactSizeIterator for DictIter<'_> { + fn len(&self) -> usize { + self.dict.entries.len_from_entry_index(self.position) + } +} + +#[pyclass] +trait DictView: PyPayload + PyClassDef + Iterable + Representable { + type ReverseIter: PyPayload + core::fmt::Debug; + + fn dict(&self) -> &Py<PyDict>; + fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef; + + fn __len__(&self) -> usize { + self.dict().__len__() + } + + #[pymethod] + fn __reversed__(&self) -> Self::ReverseIter; +} + +macro_rules! dict_view { + ( $name: ident, $iter_name: ident, $reverse_iter_name: ident, + $class: ident, $iter_class: ident, $reverse_iter_class: ident, + $class_name: literal, $iter_class_name: literal, $reverse_iter_class_name: literal, + $result_fn: expr) => { + #[pyclass(module = false, name = $class_name)] + #[derive(Debug)] + pub(crate) struct $name { + pub dict: PyDictRef, + } + + impl $name { + pub const fn new(dict: PyDictRef) -> Self { + $name { dict } + } + } + + impl DictView for $name { + type ReverseIter = $reverse_iter_name; + + fn dict(&self) -> &Py<PyDict> { + &self.dict + } + + fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef { + #[allow(clippy::redundant_closure_call)] + $result_fn(vm, key, value) + } + + fn __reversed__(&self) -> Self::ReverseIter { + $reverse_iter_name::new(self.dict.clone()) + } + } + + impl Iterable for $name { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok($iter_name::new(zelf.dict.clone()).into_pyobject(vm)) + } + } + + impl PyPayload for $name { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.$class + } + } + + impl Representable for $name { + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let s = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let mut result = Wtf8Buf::from(format!("{}([", Self::NAME)); + let mut first = true; + for (key, value) in zelf.dict().clone() { + if !first { + result.push_str(", "); + } + first = false; + result.push_wtf8(Self::item(vm, key, value).repr(vm)?.as_wtf8()); + } + result.push_str("])"); + vm.ctx.new_str(result) + } else { + vm.ctx.intern_str("{...}").to_owned() + }; + Ok(s) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } + } + + #[pyclass(module = false, name = $iter_class_name)] + #[derive(Debug)] + pub(crate) struct $iter_name { + pub size: dict_inner::DictSize, + pub internal: PyMutex<PositionIterInternal<PyDictRef>>, + } + + impl PyPayload for $iter_name { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.$iter_class + } + } + + #[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] + impl $iter_name { + fn new(dict: PyDictRef) -> Self { + $iter_name { + size: dict.size(), + internal: PyMutex::new(PositionIterInternal::new(dict, 0)), + } + } + + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().length_hint(|_| self.size.entries_size) + } + + #[allow(clippy::redundant_closure_call)] + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let iter = builtins_iter(vm); + let internal = self.internal.lock(); + let entries = match &internal.status { + IterStatus::Active(dict) => dict + .into_iter() + .map(|(key, value)| ($result_fn)(vm, key, value)) + .collect::<Vec<_>>(), + IterStatus::Exhausted => vec![], + }; + vm.new_tuple((iter, (vm.ctx.new_list(entries),))) + } + } + + impl SelfIter for $iter_name {} + impl IterNext for $iter_name { + #[allow(clippy::redundant_closure_call)] + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let mut internal = zelf.internal.lock(); + let next = if let IterStatus::Active(dict) = &internal.status { + if dict.entries.has_changed_size(&zelf.size) { + internal.status = IterStatus::Exhausted; + return Err( + vm.new_runtime_error("dictionary changed size during iteration") + ); + } + match dict.entries.next_entry(internal.position) { + Some((position, key, value)) => { + internal.position = position; + PyIterReturn::Return(($result_fn)(vm, key, value)) + } + None => { + internal.status = IterStatus::Exhausted; + PyIterReturn::StopIteration(None) + } + } + } else { + PyIterReturn::StopIteration(None) + }; + Ok(next) + } + } + + #[pyclass(module = false, name = $reverse_iter_class_name)] + #[derive(Debug)] + pub(crate) struct $reverse_iter_name { + pub size: dict_inner::DictSize, + internal: PyMutex<PositionIterInternal<PyDictRef>>, + } + + impl PyPayload for $reverse_iter_name { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.$reverse_iter_class + } + } + + #[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] + impl $reverse_iter_name { + fn new(dict: PyDictRef) -> Self { + let size = dict.size(); + let position = size.entries_size.saturating_sub(1); + $reverse_iter_name { + size, + internal: PyMutex::new(PositionIterInternal::new(dict, position)), + } + } + + #[allow(clippy::redundant_closure_call)] + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let iter = builtins_reversed(vm); + let internal = self.internal.lock(); + // TODO: entries must be reversed too + let entries = match &internal.status { + IterStatus::Active(dict) => dict + .into_iter() + .map(|(key, value)| ($result_fn)(vm, key, value)) + .collect::<Vec<_>>(), + IterStatus::Exhausted => vec![], + }; + vm.new_tuple((iter, (vm.ctx.new_list(entries),))) + } + + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal + .lock() + .rev_length_hint(|_| self.size.entries_size) + } + } + impl SelfIter for $reverse_iter_name {} + impl IterNext for $reverse_iter_name { + #[allow(clippy::redundant_closure_call)] + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let mut internal = zelf.internal.lock(); + let next = if let IterStatus::Active(dict) = &internal.status { + if dict.entries.has_changed_size(&zelf.size) { + internal.status = IterStatus::Exhausted; + return Err( + vm.new_runtime_error("dictionary changed size during iteration") + ); + } + match dict.entries.prev_entry(internal.position) { + Some((position, key, value)) => { + if internal.position == position { + internal.status = IterStatus::Exhausted; + } else { + internal.position = position; + } + PyIterReturn::Return(($result_fn)(vm, key, value)) + } + None => { + internal.status = IterStatus::Exhausted; + PyIterReturn::StopIteration(None) + } + } + } else { + PyIterReturn::StopIteration(None) + }; + Ok(next) + } + } + }; +} + +dict_view! { + PyDictKeys, + PyDictKeyIterator, + PyDictReverseKeyIterator, + dict_keys_type, + dict_keyiterator_type, + dict_reversekeyiterator_type, + "dict_keys", + "dict_keyiterator", + "dict_reversekeyiterator", + |_vm: &VirtualMachine, key: PyObjectRef, _value: PyObjectRef| key +} + +dict_view! { + PyDictValues, + PyDictValueIterator, + PyDictReverseValueIterator, + dict_values_type, + dict_valueiterator_type, + dict_reversevalueiterator_type, + "dict_values", + "dict_valueiterator", + "dict_reversevalueiterator", + |_vm: &VirtualMachine, _key: PyObjectRef, value: PyObjectRef| value +} + +dict_view! { + PyDictItems, + PyDictItemIterator, + PyDictReverseItemIterator, + dict_items_type, + dict_itemiterator_type, + dict_reverseitemiterator_type, + "dict_items", + "dict_itemiterator", + "dict_reverseitemiterator", + |vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef| + vm.new_tuple((key, value)).into() +} + +// Set operations defined on set-like views of the dictionary. +#[pyclass] +trait ViewSetOps: DictView { + fn to_set(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PySetInner> { + let len = zelf.dict().__len__(); + let zelf: PyObjectRef = Self::iter(zelf, vm)?; + let iter = PyIterIter::new(vm, zelf, Some(len)); + PySetInner::from_iter(iter, vm) + } + + fn __xor__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { + let zelf = Self::to_set(zelf, vm)?; + let inner = zelf.symmetric_difference(other, vm)?; + Ok(PySet { inner }) + } + + fn __and__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { + let zelf = Self::to_set(zelf, vm)?; + let inner = zelf.intersection(other, vm)?; + Ok(PySet { inner }) + } + + fn __or__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { + let zelf = Self::to_set(zelf, vm)?; + let inner = zelf.union(other, vm)?; + Ok(PySet { inner }) + } + + fn __sub__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { + let zelf = Self::to_set(zelf, vm)?; + let inner = zelf.difference(other, vm)?; + Ok(PySet { inner }) + } + + fn __rsub__(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { + let left = PySetInner::from_iter(other.iter(vm)?, vm)?; + let right = ArgIterable::try_from_object(vm, Self::iter(zelf, vm)?)?; + let inner = left.difference(right, vm)?; + Ok(PySet { inner }) + } + + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + match_class!(match other { + ref dictview @ Self => { + return zelf.dict().inner_cmp( + dictview.dict(), + op, + !zelf.class().is(vm.ctx.types.dict_keys_type), + vm, + ); + } + ref _set @ PySet => { + let inner = Self::to_set(zelf.to_owned(), vm)?; + let zelf_set = PySet { inner }.into_pyobject(vm); + return PySet::cmp(zelf_set.downcast_ref().unwrap(), other, op, vm); + } + ref _dictitems @ PyDictItems => {} + ref _dictkeys @ PyDictKeys => {} + _ => { + return Ok(NotImplemented); + } + }); + let lhs: Vec<PyObjectRef> = zelf.as_object().to_owned().try_into_value(vm)?; + let rhs: Vec<PyObjectRef> = other.to_owned().try_into_value(vm)?; + lhs.iter() + .richcompare(rhs.iter(), op, vm) + .map(PyComparisonValue::Implemented) + } + + #[pymethod] + fn isdisjoint(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + // TODO: to_set is an expensive operation. After merging #3316 rewrite implementation using PySequence_Contains. + let zelf = Self::to_set(zelf, vm)?; + let result = zelf.isdisjoint(other, vm)?; + Ok(result) + } +} + +impl ViewSetOps for PyDictKeys {} +#[pyclass( + flags(DISALLOW_INSTANTIATION), + with( + DictView, + Comparable, + Iterable, + ViewSetOps, + AsSequence, + AsNumber, + Representable + ) +)] +impl PyDictKeys { + fn __contains__(zelf: PyObjectRef, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + zelf.sequence_unchecked().contains(&key, vm) + } + + #[pygetset] + fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { + PyMappingProxy::from(zelf.dict().to_owned()) + } +} + +impl Comparable for PyDictKeys { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + ViewSetOps::cmp(zelf, other, op, vm) + } +} + +impl AsSequence for PyDictKeys { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyDictKeys::sequence_downcast(seq).__len__())), + contains: atomic_func!(|seq, target, vm| { + PyDictKeys::sequence_downcast(seq) + .dict + .entries + .contains(vm, target) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsNumber for PyDictKeys { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + subtract: Some(set_inner_number_subtract), + and: Some(set_inner_number_and), + xor: Some(set_inner_number_xor), + or: Some(set_inner_number_or), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl ViewSetOps for PyDictItems {} +#[pyclass( + flags(DISALLOW_INSTANTIATION), + with( + DictView, + Comparable, + Iterable, + ViewSetOps, + AsSequence, + AsNumber, + Representable + ) +)] +impl PyDictItems { + fn __contains__(zelf: PyObjectRef, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + zelf.sequence_unchecked().contains(&needle, vm) + } + #[pygetset] + fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { + PyMappingProxy::from(zelf.dict().to_owned()) + } +} + +impl Comparable for PyDictItems { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + ViewSetOps::cmp(zelf, other, op, vm) + } +} + +impl AsSequence for PyDictItems { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyDictItems::sequence_downcast(seq).__len__())), + contains: atomic_func!(|seq, target, vm| { + let needle: &Py<PyTuple> = match target.downcast_ref() { + Some(needle) => needle, + None => return Ok(false), + }; + if needle.len() != 2 { + return Ok(false); + } + + let zelf = PyDictItems::sequence_downcast(seq); + let key = &needle[0]; + if !zelf.dict.__contains__(key.to_owned(), vm)? { + return Ok(false); + } + let value = &needle[1]; + let found = zelf.dict().__getitem__(key.to_owned(), vm)?; + vm.identical_or_equal(&found, value) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsNumber for PyDictItems { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + subtract: Some(set_inner_number_subtract), + and: Some(set_inner_number_and), + xor: Some(set_inner_number_xor), + or: Some(set_inner_number_or), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +#[pyclass( + flags(DISALLOW_INSTANTIATION), + with(DictView, Iterable, AsSequence, Representable) +)] +impl PyDictValues { + #[pygetset] + fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { + PyMappingProxy::from(zelf.dict().to_owned()) + } +} + +impl AsSequence for PyDictValues { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyDictValues::sequence_downcast(seq).__len__())), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +fn set_inner_number_op<F>(a: &PyObject, b: &PyObject, f: F, vm: &VirtualMachine) -> PyResult +where + F: FnOnce(PySetInner, ArgIterable) -> PyResult<PySetInner>, +{ + let a = PySetInner::from_iter( + ArgIterable::try_from_object(vm, a.to_owned())?.iter(vm)?, + vm, + )?; + let b = ArgIterable::try_from_object(vm, b.to_owned())?; + Ok(PySet { inner: f(a, b)? }.into_pyobject(vm)) +} + +fn set_inner_number_subtract(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { + set_inner_number_op(a, b, |a, b| a.difference(b, vm), vm) +} + +fn set_inner_number_and(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { + set_inner_number_op(a, b, |a, b| a.intersection(b, vm), vm) +} + +fn set_inner_number_xor(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { + set_inner_number_op(a, b, |a, b| a.symmetric_difference(b, vm), vm) +} + +fn set_inner_number_or(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { + set_inner_number_op(a, b, |a, b| a.union(b, vm), vm) +} + +pub(crate) fn init(context: &'static Context) { + PyDict::extend_class(context, context.types.dict_type); + PyDictKeys::extend_class(context, context.types.dict_keys_type); + PyDictKeyIterator::extend_class(context, context.types.dict_keyiterator_type); + PyDictReverseKeyIterator::extend_class(context, context.types.dict_reversekeyiterator_type); + PyDictValues::extend_class(context, context.types.dict_values_type); + PyDictValueIterator::extend_class(context, context.types.dict_valueiterator_type); + PyDictReverseValueIterator::extend_class(context, context.types.dict_reversevalueiterator_type); + PyDictItems::extend_class(context, context.types.dict_items_type); + PyDictItemIterator::extend_class(context, context.types.dict_itemiterator_type); + PyDictReverseItemIterator::extend_class(context, context.types.dict_reverseitemiterator_type); +} diff --git a/crates/vm/src/builtins/enumerate.rs b/crates/vm/src/builtins/enumerate.rs new file mode 100644 index 00000000000..3cca47ce2d3 --- /dev/null +++ b/crates/vm/src/builtins/enumerate.rs @@ -0,0 +1,150 @@ +use super::{ + IterStatus, PositionIterInternal, PyGenericAlias, PyIntRef, PyTupleRef, PyType, PyTypeRef, + iter::builtins_reversed, +}; +use crate::common::lock::{PyMutex, PyRwLock}; +use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + class::PyClassImpl, + convert::ToPyObject, + function::OptionalArg, + protocol::{PyIter, PyIterReturn}, + raise_if_stop, + types::{Constructor, IterNext, Iterable, SelfIter}, +}; +use malachite_bigint::BigInt; +use num_traits::Zero; + +#[pyclass(module = false, name = "enumerate", traverse)] +#[derive(Debug)] +pub struct PyEnumerate { + #[pytraverse(skip)] + counter: PyRwLock<BigInt>, + iterable: PyIter, +} + +impl PyPayload for PyEnumerate { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.enumerate_type + } +} + +#[derive(FromArgs)] +pub struct EnumerateArgs { + #[pyarg(any)] + iterable: PyIter, + #[pyarg(any, optional)] + start: OptionalArg<PyIntRef>, +} + +impl Constructor for PyEnumerate { + type Args = EnumerateArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { iterable, start }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + let counter = start.map_or_else(BigInt::zero, |start| start.as_bigint().clone()); + Ok(Self { + counter: PyRwLock::new(counter), + iterable, + }) + } +} + +#[pyclass(with(Py, IterNext, Iterable, Constructor), flags(BASETYPE))] +impl PyEnumerate { + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +#[pyclass] +impl Py<PyEnumerate> { + #[pymethod] + fn __reduce__(&self) -> (PyTypeRef, (PyIter, BigInt)) { + ( + self.class().to_owned(), + (self.iterable.clone(), self.counter.read().clone()), + ) + } +} + +impl SelfIter for PyEnumerate {} + +impl IterNext for PyEnumerate { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let next_obj = raise_if_stop!(zelf.iterable.next(vm)?); + let mut counter = zelf.counter.write(); + let position = counter.clone(); + *counter += 1; + Ok(PyIterReturn::Return((position, next_obj).to_pyobject(vm))) + } +} + +#[pyclass(module = false, name = "reversed", traverse)] +#[derive(Debug)] +pub struct PyReverseSequenceIterator { + internal: PyMutex<PositionIterInternal<PyObjectRef>>, +} + +impl PyPayload for PyReverseSequenceIterator { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.reverse_iter_type + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PyReverseSequenceIterator { + pub const fn new(obj: PyObjectRef, len: usize) -> Self { + let position = len.saturating_sub(1); + Self { + internal: PyMutex::new(PositionIterInternal::new(obj, position)), + } + } + + #[pymethod] + fn __length_hint__(&self, vm: &VirtualMachine) -> PyResult<usize> { + let internal = self.internal.lock(); + if let IterStatus::Active(obj) = &internal.status + && internal.position <= obj.length(vm)? + { + return Ok(internal.position + 1); + } + Ok(0) + } + + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.internal.lock().set_state(state, |_, pos| pos, vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_reversed(vm); + self.internal.lock().reduce( + func, + |x| x.clone(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) + } +} + +impl SelfIter for PyReverseSequenceIterator {} +impl IterNext for PyReverseSequenceIterator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.internal + .lock() + .rev_next(|obj, pos| PyIterReturn::from_getitem_result(obj.get_item(&pos, vm), vm)) + } +} + +pub fn init(context: &'static Context) { + PyEnumerate::extend_class(context, context.types.enumerate_type); + PyReverseSequenceIterator::extend_class(context, context.types.reverse_iter_type); +} diff --git a/crates/vm/src/builtins/filter.rs b/crates/vm/src/builtins/filter.rs new file mode 100644 index 00000000000..56d951f8a79 --- /dev/null +++ b/crates/vm/src/builtins/filter.rs @@ -0,0 +1,76 @@ +use super::{PyType, PyTypeRef}; +use crate::{ + Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + class::PyClassImpl, + protocol::{PyIter, PyIterReturn}, + raise_if_stop, + types::{Constructor, IterNext, Iterable, SelfIter}, +}; + +#[pyclass(module = false, name = "filter", traverse)] +#[derive(Debug)] +pub struct PyFilter { + predicate: PyObjectRef, + iterator: PyIter, +} + +impl PyPayload for PyFilter { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.filter_type + } +} + +impl Constructor for PyFilter { + type Args = (PyObjectRef, PyIter); + + fn py_new( + _cls: &Py<PyType>, + (function, iterator): Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + predicate: function, + iterator, + }) + } +} + +#[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] +impl PyFilter { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> (PyTypeRef, (PyObjectRef, PyIter)) { + ( + vm.ctx.types.filter_type.to_owned(), + (self.predicate.clone(), self.iterator.clone()), + ) + } +} + +impl SelfIter for PyFilter {} + +impl IterNext for PyFilter { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let predicate = &zelf.predicate; + loop { + let next_obj = raise_if_stop!(zelf.iterator.next(vm)?); + let predicate_value = if vm.is_none(predicate) { + next_obj.clone() + } else { + // the predicate itself can raise StopIteration which does stop the filter iteration + raise_if_stop!(PyIterReturn::from_pyresult( + predicate.call((next_obj.clone(),), vm), + vm + )?) + }; + + if predicate_value.try_to_bool(vm)? { + return Ok(PyIterReturn::Return(next_obj)); + } + } + } +} + +pub fn init(context: &'static Context) { + PyFilter::extend_class(context, context.types.filter_type); +} diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs new file mode 100644 index 00000000000..4d7d7c7101a --- /dev/null +++ b/crates/vm/src/builtins/float.rs @@ -0,0 +1,531 @@ +use super::{ + PyByteArray, PyBytes, PyInt, PyIntRef, PyStr, PyType, PyTypeRef, PyUtf8StrRef, + try_bigint_to_f64, +}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, TryFromObject, VirtualMachine, + class::PyClassImpl, + common::{float_ops, format::FormatSpec, hash, wtf8::Wtf8Buf}, + convert::{IntoPyException, ToPyObject, ToPyResult}, + function::{ + ArgBytesLike, FuncArgs, OptionalArg, OptionalOption, PyArithmeticValue::*, + PyComparisonValue, + }, + protocol::PyNumberMethods, + types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, +}; +use core::cell::Cell; +use core::ptr::NonNull; +use malachite_bigint::{BigInt, ToBigInt}; +use num_complex::Complex64; +use num_traits::{Signed, ToPrimitive, Zero}; +use rustpython_common::int::float_to_ratio; + +#[pyclass(module = false, name = "float")] +#[derive(Debug, Copy, Clone, PartialEq)] +pub struct PyFloat { + value: f64, +} + +impl PyFloat { + pub const fn to_f64(&self) -> f64 { + self.value + } +} + +thread_local! { + static FLOAT_FREELIST: Cell<crate::object::FreeList<PyFloat>> = const { Cell::new(crate::object::FreeList::new()) }; +} + +impl PyPayload for PyFloat { + const MAX_FREELIST: usize = 100; + const HAS_FREELIST: bool = true; + + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.float_type + } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + FLOAT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> { + FLOAT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } +} + +impl ToPyObject for f64 { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_float(self).into() + } +} +impl ToPyObject for f32 { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_float(f64::from(self)).into() + } +} + +impl From<f64> for PyFloat { + fn from(value: f64) -> Self { + Self { value } + } +} + +pub(crate) fn to_op_float(obj: &PyObject, vm: &VirtualMachine) -> PyResult<Option<f64>> { + let v = if let Some(float) = obj.downcast_ref::<PyFloat>() { + Some(float.value) + } else if let Some(int) = obj.downcast_ref::<PyInt>() { + Some(try_bigint_to_f64(int.as_bigint(), vm)?) + } else { + None + }; + Ok(v) +} + +macro_rules! impl_try_from_object_float { + ($($t:ty),*) => { + $(impl TryFromObject for $t { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + PyRef::<PyFloat>::try_from_object(vm, obj).map(|f| f.to_f64() as $t) + } + })* + }; +} + +impl_try_from_object_float!(f32, f64); + +fn inner_div(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<f64> { + float_ops::div(v1, v2).ok_or_else(|| vm.new_zero_division_error("division by zero")) +} + +fn inner_mod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<f64> { + float_ops::mod_(v1, v2).ok_or_else(|| vm.new_zero_division_error("division by zero")) +} + +pub fn try_to_bigint(value: f64, vm: &VirtualMachine) -> PyResult<BigInt> { + match value.to_bigint() { + Some(int) => Ok(int), + None => { + if value.is_infinite() { + Err(vm + .new_overflow_error("OverflowError: cannot convert float infinity to integer")) + } else if value.is_nan() { + Err(vm.new_value_error("ValueError: cannot convert float NaN to integer")) + } else { + // unreachable unless BigInt has a bug + unreachable!( + "A finite float value failed to be converted to bigint: {}", + value + ) + } + } + } +} + +fn inner_floordiv(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<f64> { + float_ops::floordiv(v1, v2).ok_or_else(|| vm.new_zero_division_error("division by zero")) +} + +fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { + float_ops::divmod(v1, v2).ok_or_else(|| vm.new_zero_division_error("division by zero")) +} + +pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { + if v1.is_zero() && v2.is_sign_negative() { + let msg = "zero to a negative power"; + Err(vm.new_zero_division_error(msg.to_owned())) + } else if v1.is_sign_negative() && (v2.floor() - v2).abs() > f64::EPSILON { + let v1 = Complex64::new(v1, 0.); + let v2 = Complex64::new(v2, 0.); + Ok(v1.powc(v2).to_pyobject(vm)) + } else { + Ok(v1.powf(v2).to_pyobject(vm)) + } +} + +impl Constructor for PyFloat { + type Args = OptionalArg<PyObjectRef>; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Optimization: return exact float as-is + if cls.is(vm.ctx.types.float_type) + && args.kwargs.is_empty() + && let Some(first) = args.args.first() + && first.class().is(vm.ctx.types.float_type) + { + return Ok(first.clone()); + } + + let arg: Self::Args = args.bind(vm)?; + let payload = Self::py_new(&cls, arg, vm)?; + payload.into_ref_with_type(vm, cls).map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, arg: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let float_val = match arg { + OptionalArg::Missing => 0.0, + OptionalArg::Present(val) => { + if let Some(f) = val.try_float_opt(vm) { + f?.value + } else { + float_from_string(val, vm)? + } + } + }; + Ok(Self::from(float_val)) + } +} + +fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { + let (bytearray, buffer, buffer_lock, mapped_string); + let b = if let Some(s) = val.downcast_ref::<PyStr>() { + use crate::common::str::PyKindStr; + match s.as_str_kind() { + PyKindStr::Ascii(s) => s.trim().as_bytes(), + PyKindStr::Utf8(s) => { + mapped_string = s + .trim() + .chars() + .map(|c| { + if let Some(n) = rustpython_common::str::char_to_decimal(c) { + char::from_digit(n.into(), 10).unwrap() + } else if c.is_whitespace() { + ' ' + } else { + c + } + }) + .collect::<String>(); + mapped_string.as_bytes() + } + // if there are surrogates, it's not gonna parse anyway, + // so we can just choose a known bad value + PyKindStr::Wtf8(_) => b"", + } + } else if let Some(bytes) = val.downcast_ref::<PyBytes>() { + bytes.as_bytes() + } else if let Some(buf) = val.downcast_ref::<PyByteArray>() { + bytearray = buf.borrow_buf(); + &*bytearray + } else if let Ok(b) = ArgBytesLike::try_from_borrowed_object(vm, &val) { + buffer = b; + buffer_lock = buffer.borrow_buf(); + &*buffer_lock + } else { + return Err(vm.new_type_error(format!( + "float() argument must be a string or a number, not '{}'", + val.class().name() + ))); + }; + crate::literal::float::parse_bytes(b).ok_or_else(|| { + val.repr(vm) + .map(|repr| vm.new_value_error(format!("could not convert string to float: {repr}"))) + .unwrap_or_else(|e| e) + }) +} + +#[pyclass( + flags(BASETYPE, _MATCH_SELF), + with(Comparable, Hashable, Constructor, AsNumber, Representable) +)] +impl PyFloat { + #[pymethod] + fn __format__(zelf: &Py<Self>, spec: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + // Empty format spec: equivalent to str(self) + if spec.is_empty() { + return Ok(zelf.as_object().str(vm)?.as_wtf8().to_owned()); + } + let format_spec = + FormatSpec::parse(spec.as_str()).map_err(|err| err.into_pyexception(vm))?; + let result = if format_spec.has_locale_format() { + let locale = crate::format::get_locale_info(); + format_spec.format_float_locale(zelf.value, &locale) + } else { + format_spec.format_float(zelf.value) + }; + result + .map(Wtf8Buf::from_string) + .map_err(|err| err.into_pyexception(vm)) + } + + #[pystaticmethod] + fn __getformat__(spec: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<String> { + if !matches!(spec.as_str(), "double" | "float") { + return Err( + vm.new_value_error("__getformat__() argument 1 must be 'double' or 'float'") + ); + } + + const BIG_ENDIAN: bool = cfg!(target_endian = "big"); + + Ok(if BIG_ENDIAN { + "IEEE, big-endian" + } else { + "IEEE, little-endian" + } + .to_owned()) + } + + #[pymethod] + fn __trunc__(&self, vm: &VirtualMachine) -> PyResult<BigInt> { + try_to_bigint(self.value, vm) + } + + #[pymethod] + fn __floor__(&self, vm: &VirtualMachine) -> PyResult<BigInt> { + try_to_bigint(self.value.floor(), vm) + } + + #[pymethod] + fn __ceil__(&self, vm: &VirtualMachine) -> PyResult<BigInt> { + try_to_bigint(self.value.ceil(), vm) + } + + #[pymethod] + fn __round__(&self, ndigits: OptionalOption<PyIntRef>, vm: &VirtualMachine) -> PyResult { + let ndigits = ndigits.flatten(); + let value = if let Some(ndigits) = ndigits { + let ndigits = ndigits.as_bigint(); + let ndigits = match ndigits.to_i32() { + Some(n) => n, + None if ndigits.is_positive() => i32::MAX, + None => i32::MIN, + }; + let float = float_ops::round_float_digits(self.value, ndigits) + .ok_or_else(|| vm.new_overflow_error("overflow occurred during round"))?; + vm.ctx.new_float(float).into() + } else { + let fract = self.value.fract(); + let value = if (fract.abs() - 0.5).abs() < f64::EPSILON { + if self.value.trunc() % 2.0 == 0.0 { + self.value - fract + } else { + self.value + fract + } + } else { + self.value.round() + }; + let int = try_to_bigint(value, vm)?; + vm.ctx.new_int(int).into() + }; + Ok(value) + } + + #[pygetset] + const fn real(zelf: PyRef<Self>) -> PyRef<Self> { + zelf + } + + #[pygetset] + const fn imag(&self) -> f64 { + 0.0f64 + } + + #[pymethod] + const fn conjugate(zelf: PyRef<Self>) -> PyRef<Self> { + zelf + } + + #[pymethod] + fn is_integer(&self) -> bool { + crate::literal::float::is_integer(self.value) + } + + #[pymethod] + fn as_integer_ratio(&self, vm: &VirtualMachine) -> PyResult<(PyIntRef, PyIntRef)> { + let value = self.value; + + float_to_ratio(value) + .map(|(numer, denom)| (vm.ctx.new_bigint(&numer), vm.ctx.new_bigint(&denom))) + .ok_or_else(|| { + if value.is_infinite() { + vm.new_overflow_error("cannot convert Infinity to integer ratio") + } else if value.is_nan() { + vm.new_value_error("cannot convert NaN to integer ratio") + } else { + unreachable!("finite float must able to convert to integer ratio") + } + }) + } + + #[pyclassmethod] + fn from_number(cls: PyTypeRef, number: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if number.class().is(vm.ctx.types.float_type) && cls.is(vm.ctx.types.float_type) { + return Ok(number); + } + + let value = number.try_float(vm)?.to_f64(); + let result = vm.ctx.new_float(value); + if cls.is(vm.ctx.types.float_type) { + Ok(result.into()) + } else { + PyType::call(&cls, vec![result.into()].into(), vm) + } + } + + #[pyclassmethod] + fn fromhex(cls: PyTypeRef, string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + let result = crate::literal::float::from_hex(string.as_str().trim()) + .ok_or_else(|| vm.new_value_error("invalid hexadecimal floating-point string"))?; + PyType::call(&cls, vec![vm.ctx.new_float(result).into()].into(), vm) + } + + #[pymethod] + fn hex(&self) -> String { + crate::literal::float::to_hex(self.value) + } + + #[pymethod] + fn __getnewargs__(&self, vm: &VirtualMachine) -> PyObjectRef { + (self.value,).to_pyobject(vm) + } +} + +impl Comparable for PyFloat { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let ret = if let Some(other) = other.downcast_ref::<Self>() { + zelf.value + .partial_cmp(&other.value) + .map_or_else(|| op == PyComparisonOp::Ne, |ord| op.eval_ord(ord)) + } else if let Some(other) = other.downcast_ref::<PyInt>() { + let a = zelf.to_f64(); + let b = other.as_bigint(); + match op { + PyComparisonOp::Lt => float_ops::lt_int(a, b), + PyComparisonOp::Le => { + if let (Some(a_int), Some(b_float)) = (a.to_bigint(), b.to_f64()) { + a <= b_float && a_int <= *b + } else { + float_ops::lt_int(a, b) + } + } + PyComparisonOp::Eq => float_ops::eq_int(a, b), + PyComparisonOp::Ne => !float_ops::eq_int(a, b), + PyComparisonOp::Ge => { + if let (Some(a_int), Some(b_float)) = (a.to_bigint(), b.to_f64()) { + a >= b_float && a_int >= *b + } else { + float_ops::gt_int(a, b) + } + } + PyComparisonOp::Gt => float_ops::gt_int(a, b), + } + } else { + return Ok(NotImplemented); + }; + Ok(Implemented(ret)) + } +} + +impl Hashable for PyFloat { + #[inline] + fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<hash::PyHash> { + Ok(hash::hash_float(zelf.to_f64()).unwrap_or_else(|| hash::hash_object_id(zelf.get_id()))) + } +} + +impl AsNumber for PyFloat { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + add: Some(|a, b, vm| PyFloat::number_op(a, b, |a, b, _vm| a + b, vm)), + subtract: Some(|a, b, vm| PyFloat::number_op(a, b, |a, b, _vm| a - b, vm)), + multiply: Some(|a, b, vm| PyFloat::number_op(a, b, |a, b, _vm| a * b, vm)), + remainder: Some(|a, b, vm| PyFloat::number_op(a, b, inner_mod, vm)), + divmod: Some(|a, b, vm| PyFloat::number_op(a, b, inner_divmod, vm)), + power: Some(|a, b, c, vm| { + if vm.is_none(c) { + PyFloat::number_op(a, b, float_pow, vm) + } else { + Err(vm.new_type_error( + "pow() 3rd argument not allowed unless all arguments are integers", + )) + } + }), + negative: Some(|num, vm| { + let value = PyFloat::number_downcast(num).value; + (-value).to_pyresult(vm) + }), + positive: Some(|num, vm| PyFloat::number_downcast_exact(num, vm).to_pyresult(vm)), + absolute: Some(|num, vm| { + let value = PyFloat::number_downcast(num).value; + value.abs().to_pyresult(vm) + }), + boolean: Some(|num, _vm| Ok(!PyFloat::number_downcast(num).value.is_zero())), + int: Some(|num, vm| { + let value = PyFloat::number_downcast(num).value; + try_to_bigint(value, vm).map(|x| PyInt::from(x).into_pyobject(vm)) + }), + float: Some(|num, vm| Ok(PyFloat::number_downcast_exact(num, vm).into())), + floor_divide: Some(|a, b, vm| PyFloat::number_op(a, b, inner_floordiv, vm)), + true_divide: Some(|a, b, vm| PyFloat::number_op(a, b, inner_div, vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + + #[inline] + fn clone_exact(zelf: &Py<Self>, vm: &VirtualMachine) -> PyRef<Self> { + vm.ctx.new_float(zelf.value) + } +} + +impl Representable for PyFloat { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(crate::literal::float::to_string(zelf.value)) + } +} + +impl PyFloat { + fn number_op<F, R>(a: &PyObject, b: &PyObject, op: F, vm: &VirtualMachine) -> PyResult + where + F: FnOnce(f64, f64, &VirtualMachine) -> R, + R: ToPyResult, + { + if let (Some(a), Some(b)) = (to_op_float(a, vm)?, to_op_float(b, vm)?) { + op(a, b, vm).to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + } +} + +// Retrieve inner float value: +#[cfg(feature = "serde")] +pub(crate) fn get_value(obj: &PyObject) -> f64 { + obj.downcast_ref::<PyFloat>().unwrap().value +} + +#[rustfmt::skip] // to avoid line splitting +pub fn init(context: &'static Context) { + PyFloat::extend_class(context, context.types.float_type); +} diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs new file mode 100644 index 00000000000..4601eee4467 --- /dev/null +++ b/crates/vm/src/builtins/frame.rs @@ -0,0 +1,756 @@ +/*! The python `frame` type. + +*/ + +use super::{PyCode, PyDictRef, PyIntRef, PyStrRef}; +use crate::{ + Context, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + frame::{Frame, FrameOwner, FrameRef}, + function::PySetterValue, + types::Representable, +}; +use num_traits::Zero; +use rustpython_compiler_core::bytecode::{ + self, Constant, Instruction, InstructionMetadata, StackEffect, +}; +use stack_analysis::*; + +/// Stack state analysis for safe line-number jumps. +/// +/// Models the evaluation stack as a 64-bit integer, encoding the kind of each +/// stack entry in 3-bit blocks. Used by `set_f_lineno` to verify that a jump +/// is safe and to determine how many values need to be popped. +pub(crate) mod stack_analysis { + use super::*; + + const BITS_PER_BLOCK: u32 = 3; + const MASK: i64 = (1 << BITS_PER_BLOCK) - 1; // 0b111 + const MAX_STACK_ENTRIES: u32 = 63 / BITS_PER_BLOCK; // 21 + const WILL_OVERFLOW: u64 = 1u64 << ((MAX_STACK_ENTRIES - 1) * BITS_PER_BLOCK); + + pub const EMPTY_STACK: i64 = 0; + pub const UNINITIALIZED: i64 = -2; + pub const OVERFLOWED: i64 = -1; + + /// Kind of a stack entry. + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + #[repr(i64)] + pub enum Kind { + Iterator = 1, + Except = 2, + Object = 3, + Null = 4, + Lasti = 5, + } + + impl Kind { + fn from_i64(v: i64) -> Option<Self> { + match v { + 1 => Some(Kind::Iterator), + 2 => Some(Kind::Except), + 3 => Some(Kind::Object), + 4 => Some(Kind::Null), + 5 => Some(Kind::Lasti), + _ => None, + } + } + } + + pub fn push_value(stack: i64, kind: i64) -> i64 { + if (stack as u64) >= WILL_OVERFLOW { + OVERFLOWED + } else { + (stack << BITS_PER_BLOCK) | kind + } + } + + pub fn pop_value(stack: i64) -> i64 { + stack >> BITS_PER_BLOCK + } + + pub fn top_of_stack(stack: i64) -> i64 { + stack & MASK + } + + fn peek(stack: i64, n: u32) -> i64 { + debug_assert!(n >= 1); + (stack >> (BITS_PER_BLOCK * (n - 1))) & MASK + } + + fn stack_swap(stack: i64, n: u32) -> i64 { + debug_assert!(n >= 1); + let to_swap = peek(stack, n); + let top = top_of_stack(stack); + let shift = BITS_PER_BLOCK * (n - 1); + let replaced_low = (stack & !(MASK << shift)) | (top << shift); + (replaced_low & !MASK) | to_swap + } + + fn pop_to_level(mut stack: i64, level: u32) -> i64 { + if level == 0 { + return EMPTY_STACK; + } + let max_item: i64 = (1 << BITS_PER_BLOCK) - 1; + let level_max_stack = max_item << ((level - 1) * BITS_PER_BLOCK); + while stack > level_max_stack { + stack = pop_value(stack); + } + stack + } + + fn compatible_kind(from: i64, to: i64) -> bool { + if to == 0 { + return false; + } + if to == Kind::Object as i64 { + return from != Kind::Null as i64; + } + if to == Kind::Null as i64 { + return true; + } + from == to + } + + pub fn compatible_stack(from_stack: i64, to_stack: i64) -> bool { + if from_stack < 0 || to_stack < 0 { + return false; + } + let mut from = from_stack; + let mut to = to_stack; + while from > to { + from = pop_value(from); + } + while from != 0 { + let from_top = top_of_stack(from); + let to_top = top_of_stack(to); + if !compatible_kind(from_top, to_top) { + return false; + } + from = pop_value(from); + to = pop_value(to); + } + to == 0 + } + + pub fn explain_incompatible_stack(to_stack: i64) -> &'static str { + debug_assert!(to_stack != 0); + if to_stack == OVERFLOWED { + return "stack is too deep to analyze"; + } + if to_stack == UNINITIALIZED { + return "can't jump into an exception handler, or code may be unreachable"; + } + match Kind::from_i64(top_of_stack(to_stack)) { + Some(Kind::Except) => "can't jump into an 'except' block as there's no exception", + Some(Kind::Lasti) => "can't jump into a re-raising block as there's no location", + Some(Kind::Iterator) => "can't jump into the body of a for loop", + _ => "incompatible stacks", + } + } + + /// Analyze bytecode and compute the stack state at each instruction index. + pub fn mark_stacks<C: Constant>(code: &bytecode::CodeObject<C>) -> Vec<i64> { + let instructions = &*code.instructions; + let len = instructions.len(); + + let mut stacks = vec![UNINITIALIZED; len + 1]; + stacks[0] = EMPTY_STACK; + + let mut todo = true; + while todo { + todo = false; + + let mut i = 0; + while i < len { + let mut next_stack = stacks[i]; + let mut opcode = instructions[i].op; + let mut oparg: u32 = 0; + + // Accumulate EXTENDED_ARG prefixes + while matches!(opcode, Instruction::ExtendedArg) { + oparg = (oparg << 8) | u32::from(u8::from(instructions[i].arg)); + i += 1; + if i >= len { + break; + } + stacks[i] = next_stack; + opcode = instructions[i].op; + } + if i >= len { + break; + } + oparg = (oparg << 8) | u32::from(u8::from(instructions[i].arg)); + + // De-instrument and de-specialize: get the underlying base instruction + let opcode = opcode.to_base().unwrap_or(opcode).deoptimize(); + + let caches = opcode.cache_entries(); + let next_i = i + 1 + caches; + + if next_stack == UNINITIALIZED { + i = next_i; + continue; + } + + match opcode { + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } => { + // Relative forward: target = after_caches + delta + let j = next_i + oparg as usize; + next_stack = pop_value(next_stack); + let target_stack = next_stack; + if j < stacks.len() && stacks[j] == UNINITIALIZED { + stacks[j] = target_stack; + } + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::Send { .. } => { + // Relative forward: target = after_caches + delta + let j = next_i + oparg as usize; + if j < stacks.len() && stacks[j] == UNINITIALIZED { + stacks[j] = next_stack; + } + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::JumpForward { .. } => { + // Relative forward: target = after_caches + delta + let j = next_i + oparg as usize; + if j < stacks.len() && stacks[j] == UNINITIALIZED { + stacks[j] = next_stack; + } + } + Instruction::JumpBackward { .. } + | Instruction::JumpBackwardNoInterrupt { .. } => { + // Relative backward: target = after_caches - delta + let j = next_i - oparg as usize; + if j < stacks.len() && stacks[j] == UNINITIALIZED { + stacks[j] = next_stack; + if j < i { + todo = true; + } + } + } + Instruction::GetIter | Instruction::GetAIter => { + next_stack = push_value(pop_value(next_stack), Kind::Iterator as i64); + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::ForIter { .. } => { + // Fall-through (iteration continues): pushes the next value + let body_stack = push_value(next_stack, Kind::Object as i64); + if next_i < stacks.len() { + stacks[next_i] = body_stack; + } + // Exhaustion path: relative forward from after_caches + let mut j = next_i + oparg as usize; + if j < instructions.len() { + let target_op = + instructions[j].op.to_base().unwrap_or(instructions[j].op); + if matches!(target_op, Instruction::EndFor) { + j += 1; + } + } + if j < stacks.len() && stacks[j] == UNINITIALIZED { + stacks[j] = next_stack; + } + } + Instruction::EndAsyncFor => { + next_stack = pop_value(pop_value(next_stack)); + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::PushExcInfo => { + next_stack = push_value(next_stack, Kind::Except as i64); + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::PopExcept => { + next_stack = pop_value(next_stack); + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::ReturnValue => { + // End of block, no fall-through + } + Instruction::RaiseVarargs { .. } => { + // End of block, no fall-through + } + Instruction::Reraise { .. } => { + // End of block, no fall-through + } + Instruction::PushNull => { + next_stack = push_value(next_stack, Kind::Null as i64); + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::LoadGlobal { .. } => { + next_stack = push_value(next_stack, Kind::Object as i64); + if oparg & 1 != 0 { + next_stack = push_value(next_stack, Kind::Null as i64); + } + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::LoadAttr { .. } => { + // LoadAttr: pops object, pushes result + // If oparg & 1, it also pushes Null (method load) + let attr_oparg = oparg; + if attr_oparg & 1 != 0 { + next_stack = pop_value(next_stack); + next_stack = push_value(next_stack, Kind::Object as i64); + next_stack = push_value(next_stack, Kind::Null as i64); + } + // else: default stack_effect handles it + else { + let effect: StackEffect = opcode.stack_effect_info(oparg); + let popped = effect.popped() as i64; + let pushed = effect.pushed() as i64; + for _ in 0..popped { + next_stack = pop_value(next_stack); + } + for _ in 0..pushed { + next_stack = push_value(next_stack, Kind::Object as i64); + } + } + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::Swap { .. } => { + let n = oparg; + next_stack = stack_swap(next_stack, n); + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + Instruction::Copy { .. } => { + let n = oparg; + next_stack = push_value(next_stack, peek(next_stack, n)); + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + _ => { + // Default: use stack_effect + let effect: StackEffect = opcode.stack_effect_info(oparg); + let popped = effect.popped() as i64; + let pushed = effect.pushed() as i64; + let mut ns = next_stack; + for _ in 0..popped { + ns = pop_value(ns); + } + for _ in 0..pushed { + ns = push_value(ns, Kind::Object as i64); + } + next_stack = ns; + if next_i < stacks.len() { + stacks[next_i] = next_stack; + } + } + } + i = next_i; + } + + // Scan exception table + let exception_table = bytecode::decode_exception_table(&code.exceptiontable); + for entry in &exception_table { + let start_offset = entry.start as usize; + let handler = entry.target as usize; + let level = entry.depth as u32; + let has_lasti = entry.push_lasti; + + if start_offset < stacks.len() + && stacks[start_offset] != UNINITIALIZED + && handler < stacks.len() + && stacks[handler] == UNINITIALIZED + { + todo = true; + let mut target_stack = pop_to_level(stacks[start_offset], level); + if has_lasti { + target_stack = push_value(target_stack, Kind::Lasti as i64); + } + target_stack = push_value(target_stack, Kind::Except as i64); + stacks[handler] = target_stack; + } + } + } + + stacks + } + + /// Build a mapping from instruction index to line number. + /// Returns -1 for indices with no line start. + pub fn mark_lines<C: Constant>(code: &bytecode::CodeObject<C>) -> Vec<i32> { + let len = code.instructions.len(); + let mut line_starts = vec![-1i32; len]; + let mut last_line: i32 = -1; + + for (i, (loc, _)) in code.locations.iter().enumerate() { + if i >= len { + break; + } + let line = loc.line.get() as i32; + if line != last_line && line > 0 { + line_starts[i] = line; + last_line = line; + } + } + line_starts + } + + /// Find the first line number >= `line` that has code. + pub fn first_line_not_before(lines: &[i32], line: i32) -> i32 { + let mut result = i32::MAX; + for &l in lines { + if l >= line && l < result { + result = l; + } + } + if result == i32::MAX { -1 } else { result } + } +} + +pub fn init(context: &'static Context) { + Frame::extend_class(context, context.types.frame_type); +} + +impl Representable for Frame { + #[inline] + fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + const REPR: &str = "<frame object at .. >"; + Ok(vm.ctx.intern_str(REPR).to_owned()) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION), with(Py))] +impl Frame { + #[pygetset] + fn f_globals(&self) -> PyDictRef { + self.globals.clone() + } + + #[pygetset] + fn f_builtins(&self) -> PyObjectRef { + self.builtins.clone() + } + + #[pygetset] + fn f_locals(&self, vm: &VirtualMachine) -> PyResult { + let result = self.locals(vm).map(Into::into); + self.locals_dirty + .store(true, core::sync::atomic::Ordering::Release); + result + } + + #[pygetset] + pub fn f_code(&self) -> PyRef<PyCode> { + self.code.clone() + } + + #[pygetset] + fn f_lasti(&self) -> u32 { + // Return byte offset (each instruction is 2 bytes) for compatibility + self.lasti() * 2 + } + + #[pygetset] + pub fn f_lineno(&self) -> usize { + // If lasti is 0, execution hasn't started yet - use first line number + // Similar to PyCode_Addr2Line which returns co_firstlineno for addr_q < 0 + if self.lasti() == 0 { + self.code.first_line_number.map(|n| n.get()).unwrap_or(1) + } else { + self.current_location().line.get() + } + } + + #[pygetset(setter)] + fn set_f_lineno(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let l_new_lineno = match value { + PySetterValue::Assign(val) => { + let line_ref: PyIntRef = val + .downcast() + .map_err(|_| vm.new_value_error("lineno must be an integer"))?; + line_ref + .try_to_primitive::<i32>(vm) + .map_err(|_| vm.new_value_error("lineno must be an integer"))? + } + PySetterValue::Delete => { + return Err(vm.new_type_error("can't delete f_lineno attribute")); + } + }; + + let first_line = self + .code + .first_line_number + .map(|n| n.get() as i32) + .unwrap_or(1); + + if l_new_lineno < first_line { + return Err(vm.new_value_error(format!( + "line {l_new_lineno} comes before the current code block" + ))); + } + + let py_code: &PyCode = &self.code; + let code = &py_code.code; + let lines = mark_lines(code); + + // Find the first line >= target that has actual code + let new_lineno = first_line_not_before(&lines, l_new_lineno); + if new_lineno < 0 { + return Err(vm.new_value_error(format!( + "line {l_new_lineno} comes after the current code block" + ))); + } + + let stacks = mark_stacks(code); + let len = self.code.instructions.len(); + + // lasti points past the current instruction (already incremented). + // stacks[lasti - 1] gives the stack state before executing the + // instruction that triggered this trace event, which is the current + // evaluation stack. + let current_lasti = self.lasti() as usize; + let start_idx = current_lasti.saturating_sub(1); + let start_stack = if start_idx < stacks.len() { + stacks[start_idx] + } else { + OVERFLOWED + }; + let mut best_stack = OVERFLOWED; + let mut best_addr: i32 = -1; + let mut err: i32 = -1; + let mut msg = "cannot find bytecode for specified line"; + + for i in 0..len { + if lines[i] == new_lineno { + let target_stack = stacks[i]; + if compatible_stack(start_stack, target_stack) { + err = 0; + if target_stack > best_stack { + best_stack = target_stack; + best_addr = i as i32; + } + } else if err < 0 { + if start_stack == OVERFLOWED { + msg = "stack to deep to analyze"; + } else if start_stack == UNINITIALIZED { + msg = "can't jump from unreachable code"; + } else { + msg = explain_incompatible_stack(target_stack); + err = 1; + } + } + } + } + + if err != 0 { + return Err(vm.new_value_error(msg.to_owned())); + } + + // Count how many entries to pop + let mut pop_count = 0usize; + { + let mut s = start_stack; + while s > best_stack { + pop_count += 1; + s = pop_value(s); + } + } + + // Store the pending unwind for the execution loop to perform. + // We cannot pop stack entries here because the execution loop + // holds the state mutex, and trying to lock it again would deadlock. + self.set_pending_stack_pops(pop_count as u32); + self.set_pending_unwind_from_stack(start_stack); + + // Set lasti to best_addr. The executor will read lasti and execute + // the instruction at that index next. + self.set_lasti(best_addr as u32); + Ok(()) + } + + #[pygetset] + fn f_trace(&self) -> PyObjectRef { + let boxed = self.trace.lock(); + boxed.clone() + } + + #[pygetset(setter)] + fn set_f_trace(&self, value: PySetterValue, vm: &VirtualMachine) { + let mut storage = self.trace.lock(); + *storage = value.unwrap_or_none(vm); + } + + #[pymember(type = "bool")] + fn f_trace_lines(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf: FrameRef = zelf.downcast().unwrap_or_else(|_| unreachable!()); + + let boxed = zelf.trace_lines.lock(); + Ok(vm.ctx.new_bool(*boxed).into()) + } + + #[pymember(type = "bool", setter)] + fn set_f_trace_lines( + vm: &VirtualMachine, + zelf: PyObjectRef, + value: PySetterValue, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + let zelf: FrameRef = zelf.downcast().unwrap_or_else(|_| unreachable!()); + + let value: PyIntRef = value + .downcast() + .map_err(|_| vm.new_type_error("attribute value type must be bool"))?; + + let mut trace_lines = zelf.trace_lines.lock(); + *trace_lines = !value.as_bigint().is_zero(); + + Ok(()) + } + PySetterValue::Delete => Err(vm.new_type_error("can't delete numeric/char attribute")), + } + } + + #[pymember(type = "bool")] + fn f_trace_opcodes(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf: FrameRef = zelf.downcast().unwrap_or_else(|_| unreachable!()); + let trace_opcodes = zelf.trace_opcodes.lock(); + Ok(vm.ctx.new_bool(*trace_opcodes).into()) + } + + #[pymember(type = "bool", setter)] + fn set_f_trace_opcodes( + vm: &VirtualMachine, + zelf: PyObjectRef, + value: PySetterValue, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + let zelf: FrameRef = zelf.downcast().unwrap_or_else(|_| unreachable!()); + + let value: PyIntRef = value + .downcast() + .map_err(|_| vm.new_type_error("attribute value type must be bool"))?; + + let mut trace_opcodes = zelf.trace_opcodes.lock(); + *trace_opcodes = !value.as_bigint().is_zero(); + + // TODO: Implement the equivalent of _PyEval_SetOpcodeTrace() + + Ok(()) + } + PySetterValue::Delete => Err(vm.new_type_error("can't delete numeric/char attribute")), + } + } +} + +#[pyclass] +impl Py<Frame> { + #[pymethod] + // = frame_clear_impl + fn clear(&self, vm: &VirtualMachine) -> PyResult<()> { + let owner = FrameOwner::from_i8(self.owner.load(core::sync::atomic::Ordering::Acquire)); + match owner { + FrameOwner::Generator => { + // Generator frame: check if suspended (lasti > 0 means + // FRAME_SUSPENDED). lasti == 0 means FRAME_CREATED and + // can be cleared. + if self.lasti() != 0 { + return Err(vm.new_runtime_error("cannot clear a suspended frame")); + } + } + FrameOwner::Thread => { + // Thread-owned frame: always executing, cannot clear. + return Err(vm.new_runtime_error("cannot clear an executing frame")); + } + FrameOwner::FrameObject => { + // Detached frame: safe to clear. + } + } + + // Clear fastlocals + // SAFETY: Frame is not executing (detached or stopped). + { + let fastlocals = unsafe { self.fastlocals_mut() }; + for slot in fastlocals.iter_mut() { + *slot = None; + } + } + + // Clear the evaluation stack and cell references + self.clear_stack_and_cells(); + + // Clear temporary refs + self.temporary_refs.lock().clear(); + + Ok(()) + } + + #[pygetset] + fn f_generator(&self) -> Option<PyObjectRef> { + self.generator.to_owned() + } + + #[pygetset] + pub fn f_back(&self, vm: &VirtualMachine) -> Option<PyRef<Frame>> { + let previous = self.previous_frame(); + if previous.is_null() { + return None; + } + + if let Some(frame) = vm + .frames + .borrow() + .iter() + .find(|fp| { + // SAFETY: the caller keeps the FrameRef alive while it's in the Vec + let py: &crate::Py<Frame> = unsafe { fp.as_ref() }; + let ptr: *const Frame = &**py; + core::ptr::eq(ptr, previous) + }) + .map(|fp| unsafe { fp.as_ref() }.to_owned()) + { + return Some(frame); + } + + #[cfg(feature = "threading")] + { + let registry = vm.state.thread_frames.lock(); + for slot in registry.values() { + let frames = slot.frames.lock(); + // SAFETY: the owning thread can't pop while we hold the Mutex, + // so FramePtr is valid for the duration of the lock. + if let Some(frame) = frames.iter().find_map(|fp| { + let f = unsafe { fp.as_ref() }; + let ptr: *const Frame = &**f; + core::ptr::eq(ptr, previous).then(|| f.to_owned()) + }) { + return Some(frame); + } + } + } + + None + } +} diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs new file mode 100644 index 00000000000..7a6a3ef278f --- /dev/null +++ b/crates/vm/src/builtins/function.rs @@ -0,0 +1,1450 @@ +#[cfg(feature = "jit")] +mod jit; + +use super::{ + PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyModule, PyStr, PyStrRef, PyTuple, + PyTupleRef, PyType, +}; +use crate::common::lock::PyMutex; +use crate::function::ArgMapping; +use crate::object::{PyAtomicRef, Traverse, TraverseFn}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + bytecode, + class::PyClassImpl, + common::wtf8::{Wtf8Buf, wtf8_concat}, + frame::{Frame, FrameRef}, + function::{FuncArgs, OptionalArg, PyComparisonValue, PySetterValue}, + scope::Scope, + types::{ + Callable, Comparable, Constructor, GetAttr, GetDescriptor, PyComparisonOp, Representable, + }, +}; +use core::sync::atomic::{AtomicU32, Ordering::Relaxed}; +use itertools::Itertools; +#[cfg(feature = "jit")] +use rustpython_jit::CompiledCode; + +fn format_missing_args( + qualname: impl core::fmt::Display, + kind: &str, + missing: &mut Vec<impl core::fmt::Display>, +) -> String { + let count = missing.len(); + let last = if missing.len() > 1 { + missing.pop() + } else { + None + }; + let (and, right): (&str, String) = if let Some(last) = last { + ( + if missing.len() == 1 { + "' and '" + } else { + "', and '" + }, + format!("{last}"), + ) + } else { + ("", String::new()) + }; + format!( + "{qualname}() missing {count} required {kind} argument{}: '{}{}{right}'", + if count == 1 { "" } else { "s" }, + missing.iter().join("', '"), + and, + ) +} + +#[pyclass(module = false, name = "function", traverse = "manual")] +#[derive(Debug)] +pub struct PyFunction { + code: PyAtomicRef<PyCode>, + globals: PyDictRef, + builtins: PyObjectRef, + closure: Option<PyRef<PyTuple<PyCellRef>>>, + defaults_and_kwdefaults: PyMutex<(Option<PyTupleRef>, Option<PyDictRef>)>, + name: PyMutex<PyStrRef>, + qualname: PyMutex<PyStrRef>, + type_params: PyMutex<PyTupleRef>, + annotations: PyMutex<Option<PyDictRef>>, + annotate: PyMutex<Option<PyObjectRef>>, + module: PyMutex<PyObjectRef>, + doc: PyMutex<PyObjectRef>, + func_version: AtomicU32, + #[cfg(feature = "jit")] + jitted_code: PyMutex<Option<CompiledCode>>, +} + +static FUNC_VERSION_COUNTER: AtomicU32 = AtomicU32::new(1); + +/// Atomically allocate the next function version, returning 0 if exhausted. +/// Once the counter wraps to 0, it stays at 0 permanently. +fn next_func_version() -> u32 { + FUNC_VERSION_COUNTER + .fetch_update(Relaxed, Relaxed, |v| (v != 0).then(|| v.wrapping_add(1))) + .unwrap_or(0) +} + +unsafe impl Traverse for PyFunction { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.globals.traverse(tracer_fn); + if let Some(closure) = self.closure.as_ref() { + closure.as_untyped().traverse(tracer_fn); + } + self.defaults_and_kwdefaults.traverse(tracer_fn); + // Traverse additional fields that may contain references + self.type_params.lock().traverse(tracer_fn); + self.annotations.lock().traverse(tracer_fn); + self.module.lock().traverse(tracer_fn); + self.doc.lock().traverse(tracer_fn); + } + + fn clear(&mut self, out: &mut Vec<crate::PyObjectRef>) { + // Pop closure if present (equivalent to Py_CLEAR(func_closure)) + if let Some(closure) = self.closure.take() { + out.push(closure.into()); + } + + // Pop defaults and kwdefaults + if let Some(mut guard) = self.defaults_and_kwdefaults.try_lock() { + if let Some(defaults) = guard.0.take() { + out.push(defaults.into()); + } + if let Some(kwdefaults) = guard.1.take() { + out.push(kwdefaults.into()); + } + } + + // Clear annotations and annotate (Py_CLEAR) + if let Some(mut guard) = self.annotations.try_lock() + && let Some(annotations) = guard.take() + { + out.push(annotations.into()); + } + if let Some(mut guard) = self.annotate.try_lock() + && let Some(annotate) = guard.take() + { + out.push(annotate); + } + + // Clear module, doc, and type_params (Py_CLEAR) + if let Some(mut guard) = self.module.try_lock() { + let old_module = + core::mem::replace(&mut *guard, Context::genesis().none.to_owned().into()); + out.push(old_module); + } + if let Some(mut guard) = self.doc.try_lock() { + let old_doc = + core::mem::replace(&mut *guard, Context::genesis().none.to_owned().into()); + out.push(old_doc); + } + if let Some(mut guard) = self.type_params.try_lock() { + let old_type_params = + core::mem::replace(&mut *guard, Context::genesis().empty_tuple.to_owned()); + out.push(old_type_params.into()); + } + + // Replace name and qualname with empty string to break potential str subclass cycles + // name and qualname could be str subclasses, so they could have reference cycles + if let Some(mut guard) = self.name.try_lock() { + let old_name = core::mem::replace(&mut *guard, Context::genesis().empty_str.to_owned()); + out.push(old_name.into()); + } + if let Some(mut guard) = self.qualname.try_lock() { + let old_qualname = + core::mem::replace(&mut *guard, Context::genesis().empty_str.to_owned()); + out.push(old_qualname.into()); + } + + // Note: globals, builtins, code are NOT cleared (required to be non-NULL) + } +} + +impl PyFunction { + #[inline] + pub(crate) fn new( + code: PyRef<PyCode>, + globals: PyDictRef, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let name = PyMutex::new(code.obj_name.to_owned()); + let module = vm.unwrap_or_none(globals.get_item_opt(identifier!(vm, __name__), vm)?); + let builtins = globals.get_item("__builtins__", vm).unwrap_or_else(|_| { + // If not in globals, inherit from current execution context + if let Some(frame) = vm.current_frame() { + frame.builtins.clone() + } else { + vm.builtins.dict().into() + } + }); + // If builtins is a module, use its __dict__ instead + let builtins = if let Some(module) = builtins.downcast_ref::<PyModule>() { + module.dict().into() + } else { + builtins + }; + + // Get docstring from co_consts[0] if HAS_DOCSTRING flag is set + let doc = if code.code.flags.contains(bytecode::CodeFlags::HAS_DOCSTRING) { + code.code + .constants + .first() + .map(|c| c.as_object().to_owned()) + .unwrap_or_else(|| vm.ctx.none()) + } else { + vm.ctx.none() + }; + + let qualname = vm.ctx.new_str(code.qualname.as_str()); + let func = Self { + code: PyAtomicRef::from(code.clone()), + globals, + builtins, + closure: None, + defaults_and_kwdefaults: PyMutex::new((None, None)), + name, + qualname: PyMutex::new(qualname), + type_params: PyMutex::new(vm.ctx.empty_tuple.clone()), + annotations: PyMutex::new(None), + annotate: PyMutex::new(None), + module: PyMutex::new(module), + doc: PyMutex::new(doc), + func_version: AtomicU32::new(next_func_version()), + #[cfg(feature = "jit")] + jitted_code: PyMutex::new(None), + }; + Ok(func) + } + + fn fill_locals_from_args( + &self, + frame: &Frame, + func_args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + let code: &Py<PyCode> = &self.code; + let nargs = func_args.args.len(); + let n_expected_args = code.arg_count as usize; + let total_args = code.arg_count as usize + code.kwonlyarg_count as usize; + // let arg_names = self.code.arg_names(); + + // This parses the arguments from args and kwargs into + // the proper variables keeping into account default values + // and star-args and kwargs. + // See also: PyEval_EvalCodeWithName in cpython: + // https://github.com/python/cpython/blob/main/Python/ceval.c#L3681 + + // SAFETY: Frame was just created and not yet executing. + let fastlocals = unsafe { frame.fastlocals_mut() }; + + let mut args_iter = func_args.args.into_iter(); + + // Copy positional arguments into local variables + // zip short-circuits if either iterator returns None, which is the behavior we want -- + // only fill as much as there is to fill with as much as we have + for (local, arg) in Iterator::zip( + fastlocals.iter_mut().take(n_expected_args), + args_iter.by_ref().take(nargs), + ) { + *local = Some(arg); + } + + let mut vararg_offset = total_args; + // Pack other positional arguments in to *args: + if code.flags.contains(bytecode::CodeFlags::VARARGS) { + let vararg_value = vm.ctx.new_tuple(args_iter.collect()); + fastlocals[vararg_offset] = Some(vararg_value.into()); + vararg_offset += 1; + } else { + // Check the number of positional arguments + if nargs > n_expected_args { + let n_defaults = self + .defaults_and_kwdefaults + .lock() + .0 + .as_ref() + .map_or(0, |d| d.len()); + let n_required = n_expected_args - n_defaults; + let takes_msg = if n_defaults > 0 { + format!("from {} to {}", n_required, n_expected_args) + } else { + n_expected_args.to_string() + }; + + // Count keyword-only arguments that were actually provided + let kw_only_given = if code.kwonlyarg_count > 0 { + let start = code.arg_count as usize; + let end = start + code.kwonlyarg_count as usize; + code.varnames[start..end] + .iter() + .filter(|name| func_args.kwargs.contains_key(name.as_str())) + .count() + } else { + 0 + }; + + let given_msg = if kw_only_given > 0 { + format!( + "{} positional argument{} (and {} keyword-only argument{}) were", + nargs, + if nargs == 1 { "" } else { "s" }, + kw_only_given, + if kw_only_given == 1 { "" } else { "s" }, + ) + } else { + format!("{} {}", nargs, if nargs == 1 { "was" } else { "were" }) + }; + + return Err(vm.new_type_error(format!( + "{}() takes {} positional argument{} but {} given", + self.__qualname__(), + takes_msg, + if n_expected_args == 1 { "" } else { "s" }, + given_msg, + ))); + } + } + + // Do we support `**kwargs` ? + let kwargs = if code.flags.contains(bytecode::CodeFlags::VARKEYWORDS) { + let d = vm.ctx.new_dict(); + fastlocals[vararg_offset] = Some(d.clone().into()); + Some(d) + } else { + None + }; + + let arg_pos = |range: core::ops::Range<_>, name: &str| { + code.varnames + .iter() + .enumerate() + .skip(range.start) + .take(range.end - range.start) + .find(|(_, s)| s.as_str() == name) + .map(|(p, _)| p) + }; + + let mut posonly_passed_as_kwarg = Vec::new(); + // Handle keyword arguments + for (name, value) in func_args.kwargs { + // Check if we have a parameter with this name: + if let Some(pos) = arg_pos(code.posonlyarg_count as usize..total_args, &name) { + let slot = &mut fastlocals[pos]; + if slot.is_some() { + return Err(vm.new_type_error(format!( + "{}() got multiple values for argument '{}'", + self.__qualname__(), + name + ))); + } + *slot = Some(value); + } else if let Some(kwargs) = kwargs.as_ref() { + kwargs.set_item(&name, value, vm)?; + } else if arg_pos(0..code.posonlyarg_count as usize, &name).is_some() { + posonly_passed_as_kwarg.push(name); + } else { + return Err(vm.new_type_error(format!( + "{}() got an unexpected keyword argument '{}'", + self.__qualname__(), + name + ))); + } + } + if !posonly_passed_as_kwarg.is_empty() { + return Err(vm.new_type_error(format!( + "{}() got some positional-only arguments passed as keyword arguments: '{}'", + self.__qualname__(), + posonly_passed_as_kwarg.into_iter().format(", "), + ))); + } + + let mut defaults_and_kwdefaults = None; + // can't be a closure cause it returns a reference to a captured variable :/ + macro_rules! get_defaults { + () => {{ + defaults_and_kwdefaults + .get_or_insert_with(|| self.defaults_and_kwdefaults.lock().clone()) + }}; + } + + // Add missing positional arguments, if we have fewer positional arguments than the + // function definition calls for + if nargs < n_expected_args { + let defaults = get_defaults!().0.as_ref().map(|tup| tup.as_slice()); + let n_defs = defaults.map_or(0, |d| d.len()); + + let n_required = code.arg_count as usize - n_defs; + + // Given the number of defaults available, check all the arguments for which we + // _don't_ have defaults; if any are missing, raise an exception + let mut missing: Vec<_> = (nargs..n_required) + .filter_map(|i| { + if fastlocals[i].is_none() { + Some(&code.varnames[i]) + } else { + None + } + }) + .collect(); + + if !missing.is_empty() { + return Err(vm.new_type_error(format_missing_args( + self.__qualname__(), + "positional", + &mut missing, + ))); + } + + if let Some(defaults) = defaults { + let n = core::cmp::min(nargs, n_expected_args); + let i = n.saturating_sub(n_required); + + // We have sufficient defaults, so iterate over the corresponding names and use + // the default if we don't already have a value + for i in i..defaults.len() { + let slot = &mut fastlocals[n_required + i]; + if slot.is_none() { + *slot = Some(defaults[i].clone()); + } + } + } + }; + + if code.kwonlyarg_count > 0 { + let mut missing = Vec::new(); + // Check if kw only arguments are all present: + for (slot, kwarg) in fastlocals + .iter_mut() + .zip(&*code.varnames) + .skip(code.arg_count as usize) + .take(code.kwonlyarg_count as usize) + .filter(|(slot, _)| slot.is_none()) + { + if let Some(defaults) = &get_defaults!().1 + && let Some(default) = defaults.get_item_opt(&**kwarg, vm)? + { + *slot = Some(default); + continue; + } + + // No default value and not specified. + missing.push(kwarg); + } + + if !missing.is_empty() { + return Err(vm.new_type_error(format_missing_args( + self.__qualname__(), + "keyword-only", + &mut missing, + ))); + } + } + + if let Some(cell2arg) = code.cell2arg.as_deref() { + for (cell_idx, arg_idx) in cell2arg.iter().enumerate().filter(|(_, i)| **i != -1) { + let x = fastlocals[*arg_idx as usize].take(); + frame.set_cell_contents(cell_idx, x); + } + } + + Ok(()) + } + + /// Set function attribute based on MakeFunctionFlags + pub(crate) fn set_function_attribute( + &mut self, + attr: bytecode::MakeFunctionFlag, + attr_value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + use crate::builtins::PyDict; + match attr { + bytecode::MakeFunctionFlag::Defaults => { + let defaults = match attr_value.downcast::<PyTuple>() { + Ok(tuple) => tuple, + Err(obj) => { + return Err(vm.new_type_error(format!( + "__defaults__ must be a tuple, not {}", + obj.class().name() + ))); + } + }; + self.defaults_and_kwdefaults.lock().0 = Some(defaults); + } + bytecode::MakeFunctionFlag::KwOnlyDefaults => { + let kwdefaults = match attr_value.downcast::<PyDict>() { + Ok(dict) => dict, + Err(obj) => { + return Err(vm.new_type_error(format!( + "__kwdefaults__ must be a dict, not {}", + obj.class().name() + ))); + } + }; + self.defaults_and_kwdefaults.lock().1 = Some(kwdefaults); + } + bytecode::MakeFunctionFlag::Annotations => { + let annotations = match attr_value.downcast::<PyDict>() { + Ok(dict) => dict, + Err(obj) => { + return Err(vm.new_type_error(format!( + "__annotations__ must be a dict, not {}", + obj.class().name() + ))); + } + }; + *self.annotations.lock() = Some(annotations); + } + bytecode::MakeFunctionFlag::Closure => { + let closure_tuple = attr_value + .clone() + .downcast_exact::<PyTuple>(vm) + .map_err(|obj| { + vm.new_type_error(format!( + "closure must be a tuple, not {}", + obj.class().name() + )) + })? + .into_pyref(); + + self.closure = Some(closure_tuple.try_into_typed::<PyCell>(vm)?); + } + bytecode::MakeFunctionFlag::TypeParams => { + let type_params = attr_value.clone().downcast::<PyTuple>().map_err(|_| { + vm.new_type_error(format!( + "__type_params__ must be a tuple, not {}", + attr_value.class().name() + )) + })?; + *self.type_params.lock() = type_params; + } + bytecode::MakeFunctionFlag::Annotate => { + if !attr_value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable")); + } + *self.annotate.lock() = Some(attr_value); + } + } + Ok(()) + } +} + +impl Py<PyFunction> { + pub(crate) fn is_optimized_for_call_specialization(&self) -> bool { + self.code.flags.contains(bytecode::CodeFlags::OPTIMIZED) + } + + pub fn invoke_with_locals( + &self, + func_args: FuncArgs, + locals: Option<ArgMapping>, + vm: &VirtualMachine, + ) -> PyResult { + #[cfg(feature = "jit")] + if let Some(jitted_code) = self.jitted_code.lock().as_ref() { + use crate::convert::ToPyObject; + match jit::get_jit_args(self, &func_args, jitted_code, vm) { + Ok(args) => { + return Ok(args.invoke().to_pyobject(vm)); + } + Err(err) => info!( + "jit: function `{}` is falling back to being interpreted because of the \ + error: {}", + self.code.obj_name, err + ), + } + } + + let code: PyRef<PyCode> = (*self.code).to_owned(); + + let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { + None + } else if let Some(locals) = locals { + Some(locals) + } else { + Some(ArgMapping::from_dict_exact(self.globals.clone())) + }; + + let is_gen = code.flags.contains(bytecode::CodeFlags::GENERATOR); + let is_coro = code.flags.contains(bytecode::CodeFlags::COROUTINE); + let use_datastack = !(is_gen || is_coro); + + // Construct frame: + let frame = Frame::new( + code, + Scope::new(locals, self.globals.clone()), + self.builtins.clone(), + self.closure.as_ref().map_or(&[], |c| c.as_slice()), + Some(self.to_owned().into()), + use_datastack, + vm, + ) + .into_ref(&vm.ctx); + + self.fill_locals_from_args(&frame, func_args, vm)?; + match (is_gen, is_coro) { + (true, false) => { + let obj = PyGenerator::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm); + frame.set_generator(&obj); + Ok(obj) + } + (false, true) => { + let obj = PyCoroutine::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm); + frame.set_generator(&obj); + Ok(obj) + } + (true, true) => { + let obj = PyAsyncGen::new(frame.clone(), self.__name__(), self.__qualname__()) + .into_pyobject(vm); + frame.set_generator(&obj); + Ok(obj) + } + (false, false) => { + let result = vm.run_frame(frame.clone()); + // Release data stack memory after frame execution completes. + unsafe { + if let Some(base) = frame.materialize_localsplus() { + vm.datastack_pop(base); + } + } + result + } + } + } + + #[inline(always)] + pub fn invoke(&self, func_args: FuncArgs, vm: &VirtualMachine) -> PyResult { + self.invoke_with_locals(func_args, None, vm) + } + + /// Returns the function version, or 0 if invalidated. + #[inline] + pub fn func_version(&self) -> u32 { + self.func_version.load(Relaxed) + } + + /// Returns the current version, assigning a fresh one if previously invalidated. + /// Returns 0 if the version counter has overflowed. + /// `_PyFunction_GetVersionForCurrentState` + pub fn get_version_for_current_state(&self) -> u32 { + let v = self.func_version.load(Relaxed); + if v != 0 { + return v; + } + let new_v = next_func_version(); + if new_v == 0 { + return 0; + } + self.func_version.store(new_v, Relaxed); + new_v + } + + /// function_kind(SIMPLE_FUNCTION) equivalent for CALL specialization. + /// Returns true if: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonly args. + pub(crate) fn is_simple_for_call_specialization(&self) -> bool { + let code: &Py<PyCode> = &self.code; + let flags = code.flags; + flags.contains(bytecode::CodeFlags::OPTIMIZED) + && !flags.intersects(bytecode::CodeFlags::VARARGS | bytecode::CodeFlags::VARKEYWORDS) + && code.kwonlyarg_count == 0 + } + + /// Check if this function is eligible for exact-args call specialization. + /// Returns true if: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonly args, + /// and effective_nargs matches co_argcount. + pub(crate) fn can_specialize_call(&self, effective_nargs: u32) -> bool { + let code: &Py<PyCode> = &self.code; + let flags = code.flags; + flags.contains(bytecode::CodeFlags::OPTIMIZED) + && !flags.intersects(bytecode::CodeFlags::VARARGS | bytecode::CodeFlags::VARKEYWORDS) + && code.kwonlyarg_count == 0 + && code.arg_count == effective_nargs + } + + /// Runtime guard for CALL_*_EXACT_ARGS specialization: check only argcount. + /// Other invariants are guaranteed by function versioning and specialization-time checks. + #[inline] + pub(crate) fn has_exact_argcount(&self, effective_nargs: u32) -> bool { + self.code.arg_count == effective_nargs + } + + /// Bytes required for this function's frame on RustPython's thread datastack. + /// Returns `None` for generator/coroutine code paths that do not push a + /// regular datastack-backed frame in the fast call path. + pub(crate) fn datastack_frame_size_bytes(&self) -> Option<usize> { + datastack_frame_size_bytes_for_code(&self.code) + } + + pub(crate) fn prepare_exact_args_frame( + &self, + mut args: Vec<PyObjectRef>, + vm: &VirtualMachine, + ) -> FrameRef { + let code: PyRef<PyCode> = (*self.code).to_owned(); + + debug_assert_eq!(args.len(), code.arg_count as usize); + debug_assert!(code.flags.contains(bytecode::CodeFlags::OPTIMIZED)); + debug_assert!( + !code + .flags + .intersects(bytecode::CodeFlags::VARARGS | bytecode::CodeFlags::VARKEYWORDS) + ); + debug_assert_eq!(code.kwonlyarg_count, 0); + debug_assert!( + !code + .flags + .intersects(bytecode::CodeFlags::GENERATOR | bytecode::CodeFlags::COROUTINE) + ); + + let locals = if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) { + None + } else { + Some(ArgMapping::from_dict_exact(self.globals.clone())) + }; + + let frame = Frame::new( + code.clone(), + Scope::new(locals, self.globals.clone()), + self.builtins.clone(), + self.closure.as_ref().map_or(&[], |c| c.as_slice()), + Some(self.to_owned().into()), + true, // Exact-args fast path is only used for non-gen/coro functions. + vm, + ) + .into_ref(&vm.ctx); + + { + let fastlocals = unsafe { frame.fastlocals_mut() }; + for (slot, arg) in fastlocals.iter_mut().zip(args.drain(..)) { + *slot = Some(arg); + } + } + + if let Some(cell2arg) = code.cell2arg.as_deref() { + let fastlocals = unsafe { frame.fastlocals_mut() }; + for (cell_idx, arg_idx) in cell2arg.iter().enumerate().filter(|(_, i)| **i != -1) { + let x = fastlocals[*arg_idx as usize].take(); + frame.set_cell_contents(cell_idx, x); + } + } + + frame + } + + /// Fast path for calling a simple function with exact positional args. + /// Skips FuncArgs allocation, prepend_arg, and fill_locals_from_args. + /// Only valid when: CO_OPTIMIZED, no VARARGS, no VARKEYWORDS, no kwonlyargs, + /// and nargs == co_argcount. + pub fn invoke_exact_args(&self, args: Vec<PyObjectRef>, vm: &VirtualMachine) -> PyResult { + let code: PyRef<PyCode> = (*self.code).to_owned(); + + debug_assert_eq!(args.len(), code.arg_count as usize); + debug_assert!(code.flags.contains(bytecode::CodeFlags::OPTIMIZED)); + debug_assert!( + !code + .flags + .intersects(bytecode::CodeFlags::VARARGS | bytecode::CodeFlags::VARKEYWORDS) + ); + debug_assert_eq!(code.kwonlyarg_count, 0); + + // Generator/coroutine code objects are SIMPLE_FUNCTION in call + // specialization classification, but their call path must still + // go through invoke() to produce generator/coroutine objects. + if code + .flags + .intersects(bytecode::CodeFlags::GENERATOR | bytecode::CodeFlags::COROUTINE) + { + return self.invoke(FuncArgs::from(args), vm); + } + let frame = self.prepare_exact_args_frame(args, vm); + + let result = vm.run_frame(frame.clone()); + unsafe { + if let Some(base) = frame.materialize_localsplus() { + vm.datastack_pop(base); + } + } + result + } +} + +pub(crate) fn datastack_frame_size_bytes_for_code(code: &Py<PyCode>) -> Option<usize> { + if code + .flags + .intersects(bytecode::CodeFlags::GENERATOR | bytecode::CodeFlags::COROUTINE) + { + return None; + } + let nlocalsplus = code + .varnames + .len() + .checked_add(code.cellvars.len())? + .checked_add(code.freevars.len())?; + let capacity = nlocalsplus.checked_add(code.max_stackdepth as usize)?; + capacity.checked_mul(core::mem::size_of::<usize>()) +} + +impl PyPayload for PyFunction { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.function_type + } +} + +#[pyclass( + with(GetDescriptor, Callable, Representable, Constructor), + flags(HAS_DICT, HAS_WEAKREF, METHOD_DESCRIPTOR) +)] +impl PyFunction { + #[pygetset] + fn __code__(&self) -> PyRef<PyCode> { + (*self.code).to_owned() + } + + #[pygetset(setter)] + fn set___code__(&self, code: PyRef<PyCode>, vm: &VirtualMachine) { + #[cfg(feature = "jit")] + let mut jit_guard = self.jitted_code.lock(); + self.code.swap_to_temporary_refs(code, vm); + #[cfg(feature = "jit")] + { + *jit_guard = None; + } + self.func_version.store(0, Relaxed); + } + + #[pygetset] + fn __defaults__(&self) -> Option<PyTupleRef> { + self.defaults_and_kwdefaults.lock().0.clone() + } + #[pygetset(setter)] + fn set___defaults__(&self, defaults: Option<PyTupleRef>) { + self.defaults_and_kwdefaults.lock().0 = defaults; + self.func_version.store(0, Relaxed); + } + + #[pygetset] + fn __kwdefaults__(&self) -> Option<PyDictRef> { + self.defaults_and_kwdefaults.lock().1.clone() + } + #[pygetset(setter)] + fn set___kwdefaults__(&self, kwdefaults: Option<PyDictRef>) { + self.defaults_and_kwdefaults.lock().1 = kwdefaults; + self.func_version.store(0, Relaxed); + } + + // {"__closure__", T_OBJECT, OFF(func_closure), READONLY}, + // {"__doc__", T_OBJECT, OFF(func_doc), 0}, + // {"__globals__", T_OBJECT, OFF(func_globals), READONLY}, + // {"__module__", T_OBJECT, OFF(func_module), 0}, + // {"__builtins__", T_OBJECT, OFF(func_builtins), READONLY}, + #[pymember] + fn __globals__(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf = Self::_as_pyref(&zelf, vm)?; + Ok(zelf.globals.clone().into()) + } + + #[pymember] + fn __closure__(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf = Self::_as_pyref(&zelf, vm)?; + Ok(vm.unwrap_or_none(zelf.closure.clone().map(|x| x.into()))) + } + + #[pymember] + fn __builtins__(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf = Self::_as_pyref(&zelf, vm)?; + Ok(zelf.builtins.clone()) + } + + #[pygetset] + fn __name__(&self) -> PyStrRef { + self.name.lock().clone() + } + + #[pygetset(setter)] + fn set___name__(&self, name: PyStrRef) { + *self.name.lock() = name; + } + + #[pymember] + fn __doc__(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + // When accessed from instance, obj is the PyFunction instance + if let Ok(func) = obj.downcast::<Self>() { + let doc = func.doc.lock(); + Ok(doc.clone()) + } else { + // When accessed from class, return None as there's no instance + Ok(vm.ctx.none()) + } + } + + #[pymember(setter)] + fn set___doc__(vm: &VirtualMachine, zelf: PyObjectRef, value: PySetterValue) -> PyResult<()> { + let zelf: PyRef<Self> = zelf.downcast().unwrap_or_else(|_| unreachable!()); + let value = value.unwrap_or_none(vm); + *zelf.doc.lock() = value; + Ok(()) + } + + #[pygetset] + fn __module__(&self) -> PyObjectRef { + self.module.lock().clone() + } + + #[pygetset(setter)] + fn set___module__(&self, module: PySetterValue<PyObjectRef>, vm: &VirtualMachine) { + *self.module.lock() = module.unwrap_or_none(vm); + } + + #[pygetset] + fn __annotations__(&self, vm: &VirtualMachine) -> PyResult<PyDictRef> { + // First check if we have cached annotations + { + let annotations = self.annotations.lock(); + if let Some(ref ann) = *annotations { + return Ok(ann.clone()); + } + } + + // Check for callable __annotate__ and clone it before calling + let annotate_fn = { + let annotate = self.annotate.lock(); + if let Some(ref func) = *annotate + && func.is_callable() + { + Some(func.clone()) + } else { + None + } + }; + + // Release locks before calling __annotate__ to avoid deadlock + if let Some(annotate_fn) = annotate_fn { + let one = vm.ctx.new_int(1); + let ann_dict = annotate_fn.call((one,), vm)?; + let ann_dict = ann_dict + .downcast::<crate::builtins::PyDict>() + .map_err(|obj| { + vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + obj.class().name() + )) + })?; + + // Cache the result + *self.annotations.lock() = Some(ann_dict.clone()); + return Ok(ann_dict); + } + + // No __annotate__ or not callable, create empty dict + let new_dict = vm.ctx.new_dict(); + *self.annotations.lock() = Some(new_dict.clone()); + Ok(new_dict) + } + + #[pygetset(setter)] + fn set___annotations__( + &self, + value: PySetterValue<Option<PyObjectRef>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(Some(value)) => { + let annotations = value.downcast::<crate::builtins::PyDict>().map_err(|_| { + vm.new_type_error("__annotations__ must be set to a dict object") + })?; + *self.annotations.lock() = Some(annotations); + *self.annotate.lock() = None; + } + PySetterValue::Assign(None) => { + *self.annotations.lock() = None; + *self.annotate.lock() = None; + } + PySetterValue::Delete => { + // del only clears cached annotations; __annotate__ is preserved + *self.annotations.lock() = None; + } + } + Ok(()) + } + + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyObjectRef { + self.annotate + .lock() + .clone() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pygetset(setter)] + fn set___annotate__( + &self, + value: PySetterValue<Option<PyObjectRef>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let annotate = match value { + PySetterValue::Assign(Some(value)) => { + if !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None")); + } + // Clear cached __annotations__ when __annotate__ is set + *self.annotations.lock() = None; + Some(value) + } + PySetterValue::Assign(None) => None, + PySetterValue::Delete => { + return Err(vm.new_type_error("__annotate__ cannot be deleted")); + } + }; + *self.annotate.lock() = annotate; + Ok(()) + } + + #[pygetset] + fn __qualname__(&self) -> PyStrRef { + self.qualname.lock().clone() + } + + #[pygetset(setter)] + fn set___qualname__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + let Ok(qualname) = value.downcast::<PyStr>() else { + return Err(vm.new_type_error("__qualname__ must be set to a string object")); + }; + *self.qualname.lock() = qualname; + } + PySetterValue::Delete => { + return Err(vm.new_type_error("__qualname__ must be set to a string object")); + } + } + Ok(()) + } + + #[pygetset] + fn __type_params__(&self) -> PyTupleRef { + self.type_params.lock().clone() + } + + #[pygetset(setter)] + fn set___type_params__( + &self, + value: PySetterValue<PyTupleRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + *self.type_params.lock() = value; + } + PySetterValue::Delete => { + return Err(vm.new_type_error("__type_params__ must be set to a tuple object")); + } + } + Ok(()) + } + + #[cfg(feature = "jit")] + #[pymethod] + fn __jit__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { + let mut jit_guard = zelf.jitted_code.lock(); + if jit_guard.is_some() { + return Ok(()); + } + let arg_types = jit::get_jit_arg_types(&zelf, vm)?; + let ret_type = jit::jit_ret_type(&zelf, vm)?; + let code: &Py<PyCode> = &zelf.code; + let compiled = rustpython_jit::compile(&code.code, &arg_types, ret_type) + .map_err(|err| jit::new_jit_error(err.to_string(), vm))?; + *jit_guard = Some(compiled); + Ok(()) + } +} + +impl GetDescriptor for PyFunction { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let (_zelf, obj) = Self::_unwrap(&zelf, obj, vm)?; + Ok(if vm.is_none(&obj) && !Self::_cls_is(&cls, obj.class()) { + zelf + } else { + PyBoundMethod::new(obj, zelf).into_ref(&vm.ctx).into() + }) + } +} + +impl Callable for PyFunction { + type Args = FuncArgs; + #[inline] + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + zelf.invoke(args, vm) + } +} + +impl Representable for PyFunction { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<function {} at {:#x}>", + zelf.__qualname__(), + zelf.get_id() + )) + } +} + +#[derive(FromArgs)] +pub struct PyFunctionNewArgs { + #[pyarg(positional)] + code: PyRef<PyCode>, + #[pyarg(positional)] + globals: PyDictRef, + #[pyarg(any, optional, error_msg = "arg 3 (name) must be None or string")] + name: OptionalArg<PyStrRef>, + #[pyarg(any, optional, error_msg = "arg 4 (defaults) must be None or tuple")] + argdefs: Option<PyTupleRef>, + #[pyarg(any, optional, error_msg = "arg 5 (closure) must be None or tuple")] + closure: Option<PyTupleRef>, + #[pyarg(any, optional, error_msg = "arg 6 (kwdefaults) must be None or dict")] + kwdefaults: Option<PyDictRef>, +} + +impl Constructor for PyFunction { + type Args = PyFunctionNewArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + // Handle closure - must be a tuple of cells + let closure = if let Some(closure_tuple) = args.closure { + // Check that closure length matches code's free variables + if closure_tuple.len() != args.code.freevars.len() { + return Err(vm.new_value_error(format!( + "{} requires closure of length {}, not {}", + args.code.obj_name, + args.code.freevars.len(), + closure_tuple.len() + ))); + } + + // Validate that all items are cells and create typed tuple + let typed_closure = closure_tuple.try_into_typed::<PyCell>(vm)?; + Some(typed_closure) + } else if !args.code.freevars.is_empty() { + return Err(vm.new_type_error("arg 5 (closure) must be tuple")); + } else { + None + }; + + let mut func = Self::new(args.code.clone(), args.globals.clone(), vm)?; + // Set function name if provided + if let Some(name) = args.name.into_option() { + *func.name.lock() = name.clone(); + // Also update qualname to match the name + *func.qualname.lock() = name; + } + // Now set additional attributes directly + if let Some(closure_tuple) = closure { + func.closure = Some(closure_tuple); + } + if let Some(argdefs) = args.argdefs { + func.defaults_and_kwdefaults.lock().0 = Some(argdefs); + } + if let Some(kwdefaults) = args.kwdefaults { + func.defaults_and_kwdefaults.lock().1 = Some(kwdefaults); + } + + Ok(func) + } +} + +#[pyclass(module = false, name = "method", traverse)] +#[derive(Debug)] +pub struct PyBoundMethod { + object: PyObjectRef, + function: PyObjectRef, +} + +impl Callable for PyBoundMethod { + type Args = FuncArgs; + #[inline] + fn call(zelf: &Py<Self>, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { + args.prepend_arg(zelf.object.clone()); + zelf.function.call(args, vm) + } +} + +impl Comparable for PyBoundMethod { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + Ok(PyComparisonValue::Implemented( + zelf.function.is(&other.function) && zelf.object.is(&other.object), + )) + }) + } +} + +impl GetAttr for PyBoundMethod { + fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + let class_attr = vm + .ctx + .interned_str(name) + .and_then(|attr_name| zelf.get_class_attr(attr_name)); + if let Some(obj) = class_attr { + return vm.call_if_get_descriptor(&obj, zelf.to_owned().into()); + } + zelf.function.get_attr(name, vm) + } +} + +#[derive(FromArgs)] +pub struct PyBoundMethodNewArgs { + #[pyarg(positional)] + function: PyObjectRef, + #[pyarg(positional)] + object: PyObjectRef, +} + +impl Constructor for PyBoundMethod { + type Args = PyBoundMethodNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { function, object }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self::new(object, function)) + } +} + +impl PyBoundMethod { + pub const fn new(object: PyObjectRef, function: PyObjectRef) -> Self { + Self { object, function } + } + + #[inline] + pub(crate) fn function_obj(&self) -> &PyObjectRef { + &self.function + } + + #[inline] + pub(crate) fn self_obj(&self) -> &PyObjectRef { + &self.object + } + + #[deprecated(note = "Use `Self::new(object, function).into_ref(ctx)` instead")] + pub fn new_ref(object: PyObjectRef, function: PyObjectRef, ctx: &Context) -> PyRef<Self> { + Self::new(object, function).into_ref(ctx) + } +} + +#[pyclass( + with(Callable, Comparable, GetAttr, Constructor, Representable), + flags(IMMUTABLETYPE, HAS_WEAKREF) +)] +impl PyBoundMethod { + #[pymethod] + fn __reduce__( + &self, + vm: &VirtualMachine, + ) -> (Option<PyObjectRef>, (PyObjectRef, Option<PyObjectRef>)) { + let builtins_getattr = vm.builtins.get_attr("getattr", vm).ok(); + let func_self = self.object.clone(); + let func_name = self.function.get_attr("__name__", vm).ok(); + (builtins_getattr, (func_self, func_name)) + } + + #[pygetset] + fn __doc__(&self, vm: &VirtualMachine) -> PyResult { + self.function.get_attr("__doc__", vm) + } + + #[pygetset] + fn __func__(&self) -> PyObjectRef { + self.function.clone() + } + + #[pygetset(name = "__self__")] + fn get_self(&self) -> PyObjectRef { + self.object.clone() + } + + #[pygetset] + fn __module__(&self, vm: &VirtualMachine) -> Option<PyObjectRef> { + self.function.get_attr("__module__", vm).ok() + } +} + +impl PyPayload for PyBoundMethod { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.bound_method_type + } +} + +impl Representable for PyBoundMethod { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let func_name = if let Some(qname) = + vm.get_attribute_opt(zelf.function.clone(), identifier!(vm, __qualname__))? + { + Some(qname) + } else { + vm.get_attribute_opt(zelf.function.clone(), identifier!(vm, __name__))? + }; + let func_name: Option<PyStrRef> = func_name.and_then(|o| o.downcast().ok()); + let object_repr = zelf.object.repr(vm)?; + let name = func_name.as_ref().map_or("?".as_ref(), |s| s.as_wtf8()); + Ok(wtf8_concat!( + "<bound method ", + name, + " of ", + object_repr.as_wtf8(), + ">" + )) + } +} + +#[pyclass(module = false, name = "cell", traverse)] +#[derive(Debug, Default)] +pub(crate) struct PyCell { + contents: PyMutex<Option<PyObjectRef>>, +} +pub(crate) type PyCellRef = PyRef<PyCell>; + +impl PyPayload for PyCell { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.cell_type + } +} + +impl Constructor for PyCell { + type Args = OptionalArg; + + fn py_new(_cls: &Py<PyType>, value: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self::new(value.into_option())) + } +} + +#[pyclass(with(Constructor))] +impl PyCell { + pub const fn new(contents: Option<PyObjectRef>) -> Self { + Self { + contents: PyMutex::new(contents), + } + } + + pub fn get(&self) -> Option<PyObjectRef> { + self.contents.lock().clone() + } + pub fn set(&self, x: Option<PyObjectRef>) { + *self.contents.lock() = x; + } + + #[pygetset] + fn cell_contents(&self, vm: &VirtualMachine) -> PyResult { + self.get() + .ok_or_else(|| vm.new_value_error("Cell is empty")) + } + #[pygetset(setter)] + fn set_cell_contents(&self, x: PySetterValue) { + match x { + PySetterValue::Assign(value) => self.set(Some(value)), + PySetterValue::Delete => self.set(None), + } + } +} + +/// Vectorcall implementation for PyFunction (PEP 590). +/// Takes owned args to avoid cloning when filling fastlocals. +pub(crate) fn vectorcall_function( + zelf_obj: &PyObject, + mut args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py<PyFunction> = zelf_obj.downcast_ref().unwrap(); + let code: &Py<PyCode> = &zelf.code; + + let has_kwargs = kwnames.is_some_and(|kw| !kw.is_empty()); + let is_simple = !has_kwargs + && code.flags.contains(bytecode::CodeFlags::OPTIMIZED) + && !code.flags.contains(bytecode::CodeFlags::VARARGS) + && !code.flags.contains(bytecode::CodeFlags::VARKEYWORDS) + && code.kwonlyarg_count == 0 + && !code + .flags + .intersects(bytecode::CodeFlags::GENERATOR | bytecode::CodeFlags::COROUTINE); + + if is_simple && nargs == code.arg_count as usize { + // FAST PATH: simple positional-only call, exact arg count. + // Move owned args directly into fastlocals — no clone needed. + args.truncate(nargs); + let frame = zelf.prepare_exact_args_frame(args, vm); + + let result = vm.run_frame(frame.clone()); + unsafe { + if let Some(base) = frame.materialize_localsplus() { + vm.datastack_pop(base); + } + } + return result; + } + + // SLOW PATH: construct FuncArgs from owned Vec and delegate to invoke() + let func_args = if has_kwargs { + FuncArgs::from_vectorcall(&args, nargs, kwnames) + } else { + args.truncate(nargs); + FuncArgs::from(args) + }; + zelf.invoke(func_args, vm) +} + +/// Vectorcall implementation for PyBoundMethod (PEP 590). +fn vectorcall_bound_method( + zelf_obj: &PyObject, + mut args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py<PyBoundMethod> = zelf_obj.downcast_ref().unwrap(); + + // Insert self at front of existing Vec (avoids 2nd allocation). + // O(n) memmove is cheaper than a 2nd heap alloc+dealloc for typical arg counts. + args.insert(0, zelf.object.clone()); + let new_nargs = nargs + 1; + zelf.function.vectorcall(args, new_nargs, kwnames, vm) +} + +pub fn init(context: &'static Context) { + PyFunction::extend_class(context, context.types.function_type); + context + .types + .function_type + .slots + .vectorcall + .store(Some(vectorcall_function)); + + PyBoundMethod::extend_class(context, context.types.bound_method_type); + context + .types + .bound_method_type + .slots + .vectorcall + .store(Some(vectorcall_bound_method)); + + PyCell::extend_class(context, context.types.cell_type); +} diff --git a/crates/vm/src/builtins/function/jit.rs b/crates/vm/src/builtins/function/jit.rs new file mode 100644 index 00000000000..56594a0f462 --- /dev/null +++ b/crates/vm/src/builtins/function/jit.rs @@ -0,0 +1,224 @@ +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyCode, PyDict, PyDictRef, PyFunction, PyStrInterned, bool_, float, int, + }, + bytecode::CodeFlags, + convert::ToPyObject, + function::FuncArgs, +}; +use num_traits::ToPrimitive; +use rustpython_jit::{AbiValue, Args, CompiledCode, JitArgumentError, JitType}; + +#[derive(Debug, thiserror::Error)] +pub enum ArgsError { + #[error("wrong number of arguments passed")] + WrongNumberOfArgs, + #[error("argument passed multiple times")] + ArgPassedMultipleTimes, + #[error("not a keyword argument")] + NotAKeywordArg, + #[error("not all arguments passed")] + NotAllArgsPassed, + #[error("integer can't fit into a machine integer")] + IntOverflow, + #[error("type can't be used in a jit function")] + NonJitType, + #[error("{0}")] + JitError(#[from] JitArgumentError), +} + +impl ToPyObject for AbiValue { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + match self { + AbiValue::Int(i) => i.to_pyobject(vm), + AbiValue::Float(f) => f.to_pyobject(vm), + AbiValue::Bool(b) => b.to_pyobject(vm), + _ => unimplemented!(), + } + } +} + +pub fn new_jit_error(msg: String, vm: &VirtualMachine) -> PyBaseExceptionRef { + let jit_error = vm.ctx.exceptions.jit_error.to_owned(); + vm.new_exception_msg(jit_error, msg.into()) +} + +fn get_jit_arg_type(dict: &Py<PyDict>, name: &str, vm: &VirtualMachine) -> PyResult<JitType> { + if let Some(value) = dict.get_item_opt(name, vm)? { + if value.is(vm.ctx.types.int_type) { + Ok(JitType::Int) + } else if value.is(vm.ctx.types.float_type) { + Ok(JitType::Float) + } else if value.is(vm.ctx.types.bool_type) { + Ok(JitType::Bool) + } else { + Err(new_jit_error( + "Jit requires argument to be either int, float or bool".to_owned(), + vm, + )) + } + } else { + Err(new_jit_error( + format!("argument {name} needs annotation"), + vm, + )) + } +} + +pub fn get_jit_arg_types(func: &Py<PyFunction>, vm: &VirtualMachine) -> PyResult<Vec<JitType>> { + let code: &Py<PyCode> = &func.code; + let arg_names = code.arg_names(); + + if code + .flags + .intersects(CodeFlags::VARARGS | CodeFlags::VARKEYWORDS) + { + return Err(new_jit_error( + "Can't jit functions with variable number of arguments".to_owned(), + vm, + )); + } + + if arg_names.args.is_empty() && arg_names.kwonlyargs.is_empty() { + return Ok(Vec::new()); + } + + let func_obj: PyObjectRef = func.as_ref().to_owned(); + let annotations = func_obj.get_attr("__annotations__", vm)?; + if vm.is_none(&annotations) { + Err(new_jit_error( + "Jitting function requires arguments to have annotations".to_owned(), + vm, + )) + } else if let Ok(dict) = PyDictRef::try_from_object(vm, annotations) { + let mut arg_types = Vec::new(); + + for arg in arg_names.args { + arg_types.push(get_jit_arg_type(&dict, arg.as_str(), vm)?); + } + + for arg in arg_names.kwonlyargs { + arg_types.push(get_jit_arg_type(&dict, arg.as_str(), vm)?); + } + + Ok(arg_types) + } else { + Err(vm.new_type_error("Function annotations aren't a dict")) + } +} + +pub fn jit_ret_type(func: &Py<PyFunction>, vm: &VirtualMachine) -> PyResult<Option<JitType>> { + let func_obj: PyObjectRef = func.as_ref().to_owned(); + let annotations = func_obj.get_attr("__annotations__", vm)?; + if vm.is_none(&annotations) { + Err(new_jit_error( + "Jitting function requires return type to have annotations".to_owned(), + vm, + )) + } else if let Ok(dict) = PyDictRef::try_from_object(vm, annotations) { + if dict.contains_key("return", vm) { + get_jit_arg_type(&dict, "return", vm).map_or(Ok(None), |t| Ok(Some(t))) + } else { + Ok(None) + } + } else { + Err(vm.new_type_error("Function annotations aren't a dict")) + } +} + +fn get_jit_value(vm: &VirtualMachine, obj: &PyObject) -> Result<AbiValue, ArgsError> { + // This does exact type checks as subclasses of int/float can't be passed to jitted functions + let cls = obj.class(); + if cls.is(vm.ctx.types.int_type) { + int::get_value(obj) + .to_i64() + .map(AbiValue::Int) + .ok_or(ArgsError::IntOverflow) + } else if cls.is(vm.ctx.types.float_type) { + Ok(AbiValue::Float( + obj.downcast_ref::<float::PyFloat>().unwrap().to_f64(), + )) + } else if cls.is(vm.ctx.types.bool_type) { + Ok(AbiValue::Bool(bool_::get_value(obj))) + } else { + Err(ArgsError::NonJitType) + } +} + +/// Like `fill_locals_from_args` but to populate arguments for calling a jit function. +/// This also doesn't do full error handling but instead return None if anything is wrong. In +/// that case it falls back to the executing the bytecode version which will call +/// `fill_locals_from_args` which will raise the actual exception if needed. +#[cfg(feature = "jit")] +pub(crate) fn get_jit_args<'a>( + func: &PyFunction, + func_args: &FuncArgs, + jitted_code: &'a CompiledCode, + vm: &VirtualMachine, +) -> Result<Args<'a>, ArgsError> { + let mut jit_args = jitted_code.args_builder(); + let nargs = func_args.args.len(); + + let code: &Py<PyCode> = &func.code; + let arg_names = code.arg_names(); + let arg_count = code.arg_count; + let posonlyarg_count = code.posonlyarg_count; + + if nargs > arg_count as usize || nargs < posonlyarg_count as usize { + return Err(ArgsError::WrongNumberOfArgs); + } + + // Add positional arguments + for i in 0..nargs { + jit_args.set(i, get_jit_value(vm, &func_args.args[i])?)?; + } + + // Handle keyword arguments + for (name, value) in &func_args.kwargs { + let arg_pos = + |args: &[&PyStrInterned], name: &str| args.iter().position(|arg| arg.as_str() == name); + if let Some(arg_idx) = arg_pos(arg_names.args, name) { + if jit_args.is_set(arg_idx) { + return Err(ArgsError::ArgPassedMultipleTimes); + } + jit_args.set(arg_idx, get_jit_value(vm, value)?)?; + } else if let Some(kwarg_idx) = arg_pos(arg_names.kwonlyargs, name) { + let arg_idx = kwarg_idx + arg_count as usize; + if jit_args.is_set(arg_idx) { + return Err(ArgsError::ArgPassedMultipleTimes); + } + jit_args.set(arg_idx, get_jit_value(vm, value)?)?; + } else { + return Err(ArgsError::NotAKeywordArg); + } + } + + let (defaults, kwdefaults) = func.defaults_and_kwdefaults.lock().clone(); + + // fill in positional defaults + if let Some(defaults) = defaults { + for (i, default) in defaults.iter().enumerate() { + let arg_idx = i + arg_count as usize - defaults.len(); + if !jit_args.is_set(arg_idx) { + jit_args.set(arg_idx, get_jit_value(vm, default)?)?; + } + } + } + + // fill in keyword only defaults + if let Some(kw_only_defaults) = kwdefaults { + for (i, name) in arg_names.kwonlyargs.iter().enumerate() { + let arg_idx = i + arg_count as usize; + if !jit_args.is_set(arg_idx) { + let default = kw_only_defaults + .get_item(&**name, vm) + .map_err(|_| ArgsError::NotAllArgsPassed) + .and_then(|obj| get_jit_value(vm, &obj))?; + jit_args.set(arg_idx, default)?; + } + } + } + + jit_args.into_args().ok_or(ArgsError::NotAllArgsPassed) +} diff --git a/crates/vm/src/builtins/generator.rs b/crates/vm/src/builtins/generator.rs new file mode 100644 index 00000000000..2eee2fecd0d --- /dev/null +++ b/crates/vm/src/builtins/generator.rs @@ -0,0 +1,178 @@ +/* + * The mythical generator. + */ + +use super::{PyCode, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; +use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + coroutine::{Coro, warn_deprecated_throw_signature}, + frame::FrameRef, + function::OptionalArg, + object::{Traverse, TraverseFn}, + protocol::PyIterReturn, + types::{Destructor, IterNext, Iterable, Representable, SelfIter}, +}; + +#[pyclass(module = false, name = "generator", traverse = "manual")] +#[derive(Debug)] +pub struct PyGenerator { + inner: Coro, +} + +unsafe impl Traverse for PyGenerator { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.inner.traverse(tracer_fn); + } +} + +impl PyPayload for PyGenerator { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.generator_type + } +} + +#[pyclass( + flags(DISALLOW_INSTANTIATION, HAS_WEAKREF), + with(Py, IterNext, Iterable, Representable, Destructor) +)] +impl PyGenerator { + pub const fn as_coro(&self) -> &Coro { + &self.inner + } + + pub fn new(frame: FrameRef, name: PyStrRef, qualname: PyStrRef) -> Self { + Self { + inner: Coro::new(frame, name, qualname), + } + } + + #[pygetset] + fn __name__(&self) -> PyStrRef { + self.inner.name() + } + + #[pygetset(setter)] + fn set___name__(&self, name: PyStrRef) { + self.inner.set_name(name) + } + + #[pygetset] + fn __qualname__(&self) -> PyStrRef { + self.inner.qualname() + } + + #[pygetset(setter)] + fn set___qualname__(&self, qualname: PyStrRef) { + self.inner.set_qualname(qualname) + } + + #[pygetset] + fn gi_frame(&self, _vm: &VirtualMachine) -> Option<FrameRef> { + if self.inner.closed() { + None + } else { + Some(self.inner.frame()) + } + } + + #[pygetset] + fn gi_running(&self, _vm: &VirtualMachine) -> bool { + self.inner.running() + } + + #[pygetset] + fn gi_code(&self, _vm: &VirtualMachine) -> PyRef<PyCode> { + self.inner.frame().code.clone() + } + + #[pygetset] + fn gi_yieldfrom(&self, _vm: &VirtualMachine) -> Option<PyObjectRef> { + self.inner.frame().yield_from_target() + } + + #[pygetset] + fn gi_suspended(&self, _vm: &VirtualMachine) -> bool { + self.inner.suspended() + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +#[pyclass] +impl Py<PyGenerator> { + #[pymethod] + fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + self.inner.send(self.as_object(), value, vm) + } + + #[pymethod] + fn throw( + &self, + exc_type: PyObjectRef, + exc_val: OptionalArg, + exc_tb: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + warn_deprecated_throw_signature(&exc_val, &exc_tb, vm)?; + self.inner.throw( + self.as_object(), + exc_type, + exc_val.unwrap_or_none(vm), + exc_tb.unwrap_or_none(vm), + vm, + ) + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + self.inner.close(self.as_object(), vm) + } +} + +impl Representable for PyGenerator { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + Ok(zelf.inner.repr(zelf.as_object(), zelf.get_id(), vm)) + } +} + +impl SelfIter for PyGenerator {} +impl IterNext for PyGenerator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.send(vm.ctx.none(), vm) + } +} + +impl Destructor for PyGenerator { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // _PyGen_Finalize: close the generator if it's still suspended + if zelf.inner.closed() || zelf.inner.running() { + return Ok(()); + } + // Generator was never started, just mark as closed + if zelf.inner.frame().lasti() == 0 { + zelf.inner.closed.store(true); + return Ok(()); + } + // Throw GeneratorExit to run finally blocks + if let Err(e) = zelf.inner.close(zelf.as_object(), vm) { + vm.run_unraisable(e, None, zelf.as_object().to_owned()); + } + Ok(()) + } +} + +impl Drop for PyGenerator { + fn drop(&mut self) { + self.inner.frame().clear_generator(); + } +} + +pub fn init(ctx: &'static Context) { + PyGenerator::extend_class(ctx, ctx.types.generator_type); +} diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs new file mode 100644 index 00000000000..1564229d186 --- /dev/null +++ b/crates/vm/src/builtins/genericalias.rs @@ -0,0 +1,746 @@ +// spell-checker:ignore iparam gaiterobject +use crate::common::lock::LazyLock; + +use super::type_; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, atomic_func, + builtins::{PyList, PyStr, PyTuple, PyTupleRef, PyType}, + class::PyClassImpl, + common::hash, + convert::ToPyObject, + function::{FuncArgs, PyComparisonValue}, + protocol::{PyMappingMethods, PyNumberMethods}, + types::{ + AsMapping, AsNumber, Callable, Comparable, Constructor, GetAttr, Hashable, IterNext, + Iterable, PyComparisonOp, Representable, + }, +}; +use alloc::fmt; + +// Attributes that are looked up on the GenericAlias itself, not on __origin__ +static ATTR_EXCEPTIONS: [&str; 9] = [ + "__class__", + "__origin__", + "__args__", + "__unpacked__", + "__parameters__", + "__typing_unpacked_tuple_args__", + "__mro_entries__", + "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ + "__reduce__", +]; + +// Attributes that are blocked from being looked up on __origin__ +static ATTR_BLOCKED: [&str; 3] = ["__bases__", "__copy__", "__deepcopy__"]; + +#[pyclass(module = "types", name = "GenericAlias")] +pub struct PyGenericAlias { + origin: PyObjectRef, + args: PyTupleRef, + parameters: PyTupleRef, + starred: bool, // for __unpacked__ attribute +} + +impl fmt::Debug for PyGenericAlias { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("GenericAlias") + } +} + +impl PyPayload for PyGenericAlias { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.generic_alias_type + } +} + +impl Constructor for PyGenericAlias { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("GenericAlias() takes no keyword arguments")); + } + let (origin, arguments): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + let args = if let Ok(tuple) = arguments.try_to_ref::<PyTuple>(vm) { + tuple.to_owned() + } else { + PyTuple::new_ref(vec![arguments], &vm.ctx) + }; + Ok(Self::new(origin, args, false, vm)) + } +} + +#[pyclass( + with( + AsNumber, + AsMapping, + Callable, + Comparable, + Constructor, + GetAttr, + Hashable, + Iterable, + Representable + ), + flags(BASETYPE, HAS_WEAKREF) +)] +impl PyGenericAlias { + pub fn new( + origin: impl Into<PyObjectRef>, + args: PyTupleRef, + starred: bool, + vm: &VirtualMachine, + ) -> Self { + let parameters = make_parameters(&args, vm); + Self { + origin: origin.into(), + args, + parameters, + starred, + } + } + + /// Create a GenericAlias from an origin and PyObjectRef arguments (helper for compatibility) + pub fn from_args( + origin: impl Into<PyObjectRef>, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> Self { + let args = if let Ok(tuple) = args.try_to_ref::<PyTuple>(vm) { + tuple.to_owned() + } else { + PyTuple::new_ref(vec![args], &vm.ctx) + }; + Self::new(origin, args, false, vm) + } + + fn repr(&self, vm: &VirtualMachine) -> PyResult<String> { + fn repr_item(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + if obj.is(&vm.ctx.ellipsis) { + return Ok("...".to_string()); + } + + if vm + .get_attribute_opt(obj.clone(), identifier!(vm, __origin__))? + .is_some() + && vm + .get_attribute_opt(obj.clone(), identifier!(vm, __args__))? + .is_some() + { + return Ok(obj.repr(vm)?.to_string()); + } + + match ( + vm.get_attribute_opt(obj.clone(), identifier!(vm, __qualname__))? + .and_then(|o| o.downcast_ref::<PyStr>().map(|n| n.to_string())), + vm.get_attribute_opt(obj.clone(), identifier!(vm, __module__))? + .and_then(|o| o.downcast_ref::<PyStr>().map(|m| m.to_string())), + ) { + (None, _) | (_, None) => Ok(obj.repr(vm)?.to_string()), + (Some(qualname), Some(module)) => Ok(if module == "builtins" { + qualname + } else { + format!("{module}.{qualname}") + }), + } + } + + fn repr_arg(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + // ParamSpec args can be lists - format their items with repr_item + if obj.class().is(vm.ctx.types.list_type) { + let list = obj.downcast_ref::<crate::builtins::PyList>().unwrap(); + let len = list.borrow_vec().len(); + let mut parts = Vec::with_capacity(len); + // Use indexed access so list mutation during repr causes IndexError + for i in 0..len { + let item = list + .borrow_vec() + .get(i) + .cloned() + .ok_or_else(|| vm.new_index_error("list index out of range"))?; + parts.push(repr_item(item, vm)?); + } + Ok(format!("[{}]", parts.join(", "))) + } else { + repr_item(obj, vm) + } + } + + let repr_str = format!( + "{}[{}]", + repr_item(self.origin.clone(), vm)?, + if self.args.is_empty() { + "()".to_owned() + } else { + self.args + .iter() + .map(|o| repr_arg(o.clone(), vm)) + .collect::<PyResult<Vec<_>>>()? + .join(", ") + } + ); + + // Add * prefix if this is a starred GenericAlias + Ok(if self.starred { + format!("*{repr_str}") + } else { + repr_str + }) + } + + #[pygetset] + fn __parameters__(&self) -> PyObjectRef { + self.parameters.clone().into() + } + + #[pygetset] + fn __args__(&self) -> PyObjectRef { + self.args.clone().into() + } + + #[pygetset] + fn __origin__(&self) -> PyObjectRef { + self.origin.clone() + } + + #[pygetset] + const fn __unpacked__(&self) -> bool { + self.starred + } + + #[pygetset] + fn __typing_unpacked_tuple_args__(&self, vm: &VirtualMachine) -> PyObjectRef { + if self.starred && self.origin.is(vm.ctx.types.tuple_type.as_object()) { + self.args.clone().into() + } else { + vm.ctx.none() + } + } + + fn __getitem__(zelf: PyRef<Self>, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let new_args = subs_parameters( + zelf.to_owned().into(), + zelf.args.clone(), + zelf.parameters.clone(), + needle, + vm, + )?; + + Ok(Self::new(zelf.origin.clone(), new_args, false, vm).into_pyobject(vm)) + } + + #[pymethod] + fn __dir__(&self, vm: &VirtualMachine) -> PyResult<PyList> { + let dir = vm.dir(Some(self.__origin__()))?; + for exc in &ATTR_EXCEPTIONS { + if !dir.__contains__((*exc).to_pyobject(vm), vm)? { + dir.append((*exc).to_pyobject(vm)); + } + } + Ok(dir) + } + + #[pymethod] + fn __reduce__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + if zelf.starred { + // (next, (iter(GenericAlias(origin, args)),)) + let next_fn = vm.builtins.get_attr("next", vm)?; + let non_starred = Self::new(zelf.origin.clone(), zelf.args.clone(), false, vm); + let iter_obj = PyGenericAliasIterator { + obj: crate::common::lock::PyMutex::new(Some(non_starred.into_pyobject(vm))), + } + .into_pyobject(vm); + Ok(PyTuple::new_ref( + vec![next_fn, PyTuple::new_ref(vec![iter_obj], &vm.ctx).into()], + &vm.ctx, + )) + } else { + Ok(PyTuple::new_ref( + vec![ + vm.ctx.types.generic_alias_type.to_owned().into(), + PyTuple::new_ref(vec![zelf.origin.clone(), zelf.args.clone().into()], &vm.ctx) + .into(), + ], + &vm.ctx, + )) + } + } + + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyTupleRef { + PyTuple::new_ref(vec![self.__origin__()], &vm.ctx) + } + + #[pymethod] + fn __instancecheck__(_zelf: PyRef<Self>, _obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("isinstance() argument 2 cannot be a parameterized generic")) + } + + #[pymethod] + fn __subclasscheck__(_zelf: PyRef<Self>, _obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("issubclass() argument 2 cannot be a parameterized generic")) + } + + fn __ror__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + type_::or_(other, zelf, vm) + } + + fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + type_::or_(zelf, other, vm) + } +} + +pub(crate) fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { + make_parameters_from_slice(args.as_slice(), vm) +} + +fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyTupleRef { + let mut parameters: Vec<PyObjectRef> = Vec::with_capacity(args.len()); + + for arg in args { + // We don't want __parameters__ descriptor of a bare Python class. + if arg.class().is(vm.ctx.types.type_type) { + continue; + } + + // Check for __typing_subst__ attribute + if arg.get_attr(identifier!(vm, __typing_subst__), vm).is_ok() { + if tuple_index(&parameters, arg).is_none() { + parameters.push(arg.clone()); + } + } else if let Ok(subparams) = arg.get_attr(identifier!(vm, __parameters__), vm) + && let Ok(sub_params) = subparams.try_to_ref::<PyTuple>(vm) + { + for sub_param in sub_params { + if tuple_index(&parameters, sub_param).is_none() { + parameters.push(sub_param.clone()); + } + } + } else if arg.try_to_ref::<PyTuple>(vm).is_ok() || arg.try_to_ref::<PyList>(vm).is_ok() { + // Recursively extract parameters from lists/tuples (ParamSpec args) + let items: Vec<PyObjectRef> = if let Ok(t) = arg.try_to_ref::<PyTuple>(vm) { + t.as_slice().to_vec() + } else { + let list = arg.downcast_ref::<PyList>().unwrap(); + list.borrow_vec().to_vec() + }; + let sub = make_parameters_from_slice(&items, vm); + for sub_param in sub.iter() { + if tuple_index(&parameters, sub_param).is_none() { + parameters.push(sub_param.clone()); + } + } + } + } + + PyTuple::new_ref(parameters, &vm.ctx) +} + +#[inline] +fn tuple_index(vec: &[PyObjectRef], item: &PyObject) -> Option<usize> { + vec.iter().position(|element| element.is(item)) +} + +fn is_unpacked_typevartuple(arg: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + if arg.class().is(vm.ctx.types.type_type) { + return Ok(false); + } + + if let Ok(attr) = arg.get_attr(identifier!(vm, __typing_is_unpacked_typevartuple__), vm) { + attr.try_to_bool(vm) + } else { + Ok(false) + } +} + +fn subs_tvars( + obj: PyObjectRef, + params: &Py<PyTuple>, + arg_items: &[PyObjectRef], + vm: &VirtualMachine, +) -> PyResult { + obj.get_attr(identifier!(vm, __parameters__), vm) + .ok() + .and_then(|sub_params| { + PyTupleRef::try_from_object(vm, sub_params) + .ok() + .filter(|sub_params| !sub_params.is_empty()) + .map(|sub_params| { + let mut sub_args = Vec::new(); + + for arg in sub_params.iter() { + if let Some(idx) = tuple_index(params.as_slice(), arg) { + let param = &params[idx]; + let substituted_arg = &arg_items[idx]; + + // Check if this is a TypeVarTuple (has tp_iter) + if param.class().slots.iter.load().is_some() + && substituted_arg.try_to_ref::<PyTuple>(vm).is_ok() + { + // TypeVarTuple case - extend with tuple elements + if let Ok(tuple) = substituted_arg.try_to_ref::<PyTuple>(vm) { + for elem in tuple { + sub_args.push(elem.clone()); + } + continue; + } + } + + sub_args.push(substituted_arg.clone()); + } else { + sub_args.push(arg.clone()); + } + } + + let sub_args: PyObjectRef = PyTuple::new_ref(sub_args, &vm.ctx).into(); + obj.get_item(&*sub_args, vm) + }) + }) + .unwrap_or(Ok(obj)) +} + +// CPython's _unpack_args equivalent +fn unpack_args(item: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let mut new_args = Vec::new(); + + let arg_items = if let Ok(tuple) = item.try_to_ref::<PyTuple>(vm) { + tuple.as_slice().to_vec() + } else { + vec![item] + }; + + for item in arg_items { + // Skip PyType objects - they can't be unpacked + if item.class().is(vm.ctx.types.type_type) { + new_args.push(item); + continue; + } + + // Try to get __typing_unpacked_tuple_args__ + if let Ok(sub_args) = item.get_attr(identifier!(vm, __typing_unpacked_tuple_args__), vm) + && !sub_args.is(&vm.ctx.none) + && let Ok(tuple) = sub_args.try_to_ref::<PyTuple>(vm) + { + // Check for ellipsis at the end + let has_ellipsis_at_end = tuple + .as_slice() + .last() + .is_some_and(|item| item.is(&vm.ctx.ellipsis)); + + if !has_ellipsis_at_end { + // Safe to unpack - add all elements's PyList_SetSlice + for arg in tuple { + new_args.push(arg.clone()); + } + continue; + } + } + + // Default case: add the item as-is's PyList_Append + new_args.push(item); + } + + Ok(PyTuple::new_ref(new_args, &vm.ctx)) +} + +// _Py_subs_parameters +pub fn subs_parameters( + alias: PyObjectRef, // = self + args: PyTupleRef, + parameters: PyTupleRef, + item: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<PyTupleRef> { + let n_params = parameters.len(); + if n_params == 0 { + return Err(vm.new_type_error(format!("{} is not a generic class", alias.repr(vm)?))); + } + + // Step 1: Unpack args + let mut item: PyObjectRef = unpack_args(item, vm)?.into(); + + // Step 2: Call __typing_prepare_subst__ on each parameter + for param in parameters.iter() { + if let Ok(prepare) = param.get_attr(identifier!(vm, __typing_prepare_subst__), vm) + && !prepare.is(&vm.ctx.none) + { + // Call prepare(self, item) + item = if item.try_to_ref::<PyTuple>(vm).is_ok() { + prepare.call((alias.clone(), item.clone()), vm)? + } else { + // Create a tuple with the single item's "O(O)" format + let tuple_args = PyTuple::new_ref(vec![item.clone()], &vm.ctx); + prepare.call((alias.clone(), tuple_args.to_pyobject(vm)), vm)? + }; + } + } + + // Step 3: Extract final arg items + let arg_items = if let Ok(tuple) = item.try_to_ref::<PyTuple>(vm) { + tuple.as_slice().to_vec() + } else { + vec![item.clone()] + }; + let n_items = arg_items.len(); + + if n_items != n_params { + return Err(vm.new_type_error(format!( + "Too {} arguments for {}; actual {}, expected {}", + if n_items > n_params { "many" } else { "few" }, + alias.repr(vm)?, + n_items, + n_params + ))); + } + + // Step 4: Replace all type variables + let mut new_args = Vec::new(); + + for arg in args.iter() { + // Skip PyType objects + if arg.class().is(vm.ctx.types.type_type) { + new_args.push(arg.clone()); + continue; + } + + // Recursively substitute params in lists/tuples + let is_list = arg.try_to_ref::<PyList>(vm).is_ok(); + if arg.try_to_ref::<PyTuple>(vm).is_ok() || is_list { + let sub_items: Vec<PyObjectRef> = if let Ok(t) = arg.try_to_ref::<PyTuple>(vm) { + t.as_slice().to_vec() + } else { + arg.downcast_ref::<PyList>().unwrap().borrow_vec().to_vec() + }; + let sub_tuple = PyTuple::new_ref(sub_items, &vm.ctx); + let sub_result = subs_parameters( + alias.clone(), + sub_tuple, + parameters.clone(), + item.clone(), + vm, + )?; + let substituted: PyObjectRef = if is_list { + // Convert tuple back to list + PyList::from(sub_result.as_slice().to_vec()) + .into_ref(&vm.ctx) + .into() + } else { + sub_result.into() + }; + new_args.push(substituted); + continue; + } + + // Check if this is an unpacked TypeVarTuple + let unpack = is_unpacked_typevartuple(arg, vm)?; + + // Try __typing_subst__ method first + let substituted_arg = if let Ok(subst) = arg.get_attr(identifier!(vm, __typing_subst__), vm) + { + if let Some(iparam) = tuple_index(parameters.as_slice(), arg) { + subst.call((arg_items[iparam].clone(),), vm)? + } else { + subs_tvars(arg.clone(), &parameters, &arg_items, vm)? + } + } else { + subs_tvars(arg.clone(), &parameters, &arg_items, vm)? + }; + + if unpack { + if let Ok(tuple) = substituted_arg.try_to_ref::<PyTuple>(vm) { + for elem in tuple { + new_args.push(elem.clone()); + } + } else { + new_args.push(substituted_arg); + } + } else { + new_args.push(substituted_arg); + } + } + + Ok(PyTuple::new_ref(new_args, &vm.ctx)) +} + +impl AsMapping for PyGenericAlias { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + subscript: atomic_func!(|mapping, needle, vm| { + let zelf = PyGenericAlias::mapping_downcast(mapping); + PyGenericAlias::__getitem__(zelf.to_owned(), needle.to_owned(), vm) + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } +} + +impl AsNumber for PyGenericAlias { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| PyGenericAlias::__or__(a.to_owned(), b.to_owned(), vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Callable for PyGenericAlias { + type Args = FuncArgs; + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + zelf.origin.call(args, vm).map(|obj| { + if let Err(exc) = obj.set_attr(identifier!(vm, __orig_class__), zelf.to_owned(), vm) + && !exc.fast_isinstance(vm.ctx.exceptions.attribute_error) + && !exc.fast_isinstance(vm.ctx.exceptions.type_error) + { + return Err(exc); + } + Ok(obj) + })? + } +} + +impl Comparable for PyGenericAlias { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + if zelf.starred != other.starred { + return Ok(PyComparisonValue::Implemented(false)); + } + Ok(PyComparisonValue::Implemented( + zelf.__origin__() + .rich_compare_bool(&other.__origin__(), PyComparisonOp::Eq, vm)? + && zelf.__args__().rich_compare_bool( + &other.__args__(), + PyComparisonOp::Eq, + vm, + )?, + )) + }) + } +} + +impl Hashable for PyGenericAlias { + #[inline] + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { + Ok(zelf.origin.hash(vm)? ^ zelf.args.as_object().hash(vm)?) + } +} + +impl GetAttr for PyGenericAlias { + fn getattro(zelf: &Py<Self>, attr: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + let attr_str = attr.as_wtf8(); + for exc in &ATTR_EXCEPTIONS { + if attr_str == *exc { + return zelf.as_object().generic_getattr(attr, vm); + } + } + for blocked in &ATTR_BLOCKED { + if attr_str == *blocked { + return zelf.as_object().generic_getattr(attr, vm); + } + } + zelf.__origin__().get_attr(attr, vm) + } +} + +impl Representable for PyGenericAlias { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + zelf.repr(vm) + } +} + +impl Iterable for PyGenericAlias { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyGenericAliasIterator { + obj: crate::common::lock::PyMutex::new(Some(zelf.into())), + } + .into_pyobject(vm)) + } +} + +// gaiterobject - yields one starred GenericAlias then exhausts +#[pyclass(module = "types", name = "generic_alias_iterator")] +#[derive(Debug, PyPayload)] +pub struct PyGenericAliasIterator { + obj: crate::common::lock::PyMutex<Option<PyObjectRef>>, +} + +#[pyclass(with(Representable, Iterable, IterNext))] +impl PyGenericAliasIterator { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let iter_fn = vm.builtins.get_attr("iter", vm)?; + let guard = self.obj.lock(); + let arg: PyObjectRef = if let Some(ref obj) = *guard { + // Not yet exhausted: (iter, (obj,)) + PyTuple::new_ref(vec![obj.clone()], &vm.ctx).into() + } else { + // Exhausted: (iter, ((),)) + let empty = PyTuple::new_ref(vec![], &vm.ctx); + PyTuple::new_ref(vec![empty.into()], &vm.ctx).into() + }; + Ok(PyTuple::new_ref(vec![iter_fn, arg], &vm.ctx)) + } +} + +impl Representable for PyGenericAliasIterator { + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("<generic_alias_iterator>".to_owned()) + } +} + +impl Iterable for PyGenericAliasIterator { + fn iter(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + Ok(zelf.into()) + } +} + +impl crate::types::IterNext for PyGenericAliasIterator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<crate::protocol::PyIterReturn> { + use crate::protocol::PyIterReturn; + let mut guard = zelf.obj.lock(); + let obj = match guard.take() { + Some(obj) => obj, + None => return Ok(PyIterReturn::StopIteration(None)), + }; + // Create a starred GenericAlias from the original + let alias = obj + .downcast_ref::<PyGenericAlias>() + .ok_or_else(|| vm.new_type_error("generic_alias_iterator expected GenericAlias"))?; + let starred = PyGenericAlias::new(alias.origin.clone(), alias.args.clone(), true, vm); + Ok(PyIterReturn::Return(starred.into_pyobject(vm))) + } +} + +/// Creates a GenericAlias from type parameters, equivalent to _Py_subscript_generic. +/// This is used for PEP 695 classes to create Generic[T] from type parameters. +// _Py_subscript_generic +pub fn subscript_generic(type_params: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let typing_module = vm.import("typing", 0)?; + let generic_type = typing_module.get_attr("Generic", vm)?; + let generic_alias_class = typing_module.get_attr("_GenericAlias", vm)?; + + let params = if let Ok(tuple) = type_params.try_to_ref::<PyTuple>(vm) { + tuple.to_owned() + } else { + PyTuple::new_ref(vec![type_params], &vm.ctx) + }; + + let args = crate::stdlib::_typing::unpack_typevartuples(&params, vm)?; + + generic_alias_class.call((generic_type, args.to_pyobject(vm)), vm) +} + +pub fn init(context: &'static Context) { + PyGenericAlias::extend_class(context, context.types.generic_alias_type); + PyGenericAliasIterator::extend_class(context, context.types.generic_alias_iterator_type); +} diff --git a/crates/vm/src/builtins/getset.rs b/crates/vm/src/builtins/getset.rs new file mode 100644 index 00000000000..ff132fb7471 --- /dev/null +++ b/crates/vm/src/builtins/getset.rs @@ -0,0 +1,162 @@ +/*! Python `attribute` descriptor class. (PyGetSet) + +*/ +use super::PyType; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::type_::PointerSlot, + class::PyClassImpl, + function::{IntoPyGetterFunc, IntoPySetterFunc, PyGetterFunc, PySetterFunc, PySetterValue}, + types::{GetDescriptor, Representable}, +}; + +#[pyclass(module = false, name = "getset_descriptor")] +pub struct PyGetSet { + name: String, + class: PointerSlot<Py<PyType>>, // A class type freed before getset is non-sense. + getter: Option<PyGetterFunc>, + setter: Option<PySetterFunc>, + // doc: Option<String>, +} + +impl core::fmt::Debug for PyGetSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "PyGetSet {{ name: {}, getter: {}, setter: {} }}", + self.name, + if self.getter.is_some() { + "Some" + } else { + "None" + }, + if self.setter.is_some() { + "Some" + } else { + "None" + }, + ) + } +} + +impl PyPayload for PyGetSet { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.getset_type + } +} + +impl GetDescriptor for PyGetSet { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let (zelf, obj) = match Self::_check(&zelf, obj, vm) { + Some(obj) => obj, + None => return Ok(zelf), + }; + if let Some(ref f) = zelf.getter { + f(vm, obj) + } else { + Err(vm.new_attribute_error(format!( + "attribute '{}' of '{}' objects is not readable", + zelf.name, + Self::class(&vm.ctx).name() + ))) + } + } +} + +impl PyGetSet { + pub fn new(name: String, class: &'static Py<PyType>) -> Self { + Self { + name, + class: PointerSlot::from(class), + getter: None, + setter: None, + } + } + + pub fn with_get<G, X>(mut self, getter: G) -> Self + where + G: IntoPyGetterFunc<X>, + { + self.getter = Some(getter.into_getter()); + self + } + + pub fn with_set<S, X>(mut self, setter: S) -> Self + where + S: IntoPySetterFunc<X>, + { + self.setter = Some(setter.into_setter()); + self + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION), with(GetDescriptor, Representable))] +impl PyGetSet { + // Descriptor methods + + #[pyslot] + fn descr_set( + zelf: &PyObject, + obj: PyObjectRef, + value: PySetterValue<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let zelf = zelf.try_to_ref::<Self>(vm)?; + if let Some(ref f) = zelf.setter { + f(vm, obj, value) + } else { + Err(vm.new_attribute_error(format!( + "attribute '{}' of '{}' objects is not writable", + zelf.name, + obj.class().name() + ))) + } + } + + #[pygetset] + fn __name__(&self) -> String { + self.name.clone() + } + + #[pygetset] + fn __qualname__(&self) -> String { + format!( + "{}.{}", + unsafe { self.class.borrow_static() }.slot_name(), + self.name.clone() + ) + } + + #[pymember] + fn __objclass__(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { + let zelf: &Py<Self> = zelf.try_to_value(vm)?; + Ok(unsafe { zelf.class.borrow_static() }.to_owned().into()) + } +} + +impl Representable for PyGetSet { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let class = unsafe { zelf.class.borrow_static() }; + // Special case for object type + if core::ptr::eq(class, vm.ctx.types.object_type) { + Ok(format!("<attribute '{}'>", zelf.name)) + } else { + Ok(format!( + "<attribute '{}' of '{}' objects>", + zelf.name, + class.name() + )) + } + } +} + +pub(crate) fn init(context: &'static Context) { + PyGetSet::extend_class(context, context.types.getset_type); +} diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs new file mode 100644 index 00000000000..a253506eba1 --- /dev/null +++ b/crates/vm/src/builtins/int.rs @@ -0,0 +1,796 @@ +use super::{PyByteArray, PyBytes, PyStr, PyType, PyTypeRef, float}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromBorrowedObject, VirtualMachine, + builtins::PyUtf8StrRef, + bytes_inner::PyBytesInner, + class::PyClassImpl, + common::{ + format::FormatSpec, + hash, + int::{bigint_to_finite_float, bytes_to_int, true_div}, + wtf8::Wtf8Buf, + }, + convert::{IntoPyException, ToPyObject, ToPyResult}, + function::{ + ArgByteOrder, ArgIntoBool, FuncArgs, OptionalArg, OptionalOption, PyArithmeticValue, + PyComparisonValue, + }, + protocol::{PyNumberMethods, handle_bytes_to_int_err}, + types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, +}; +use alloc::fmt; +use core::cell::Cell; +use core::ops::{Neg, Not}; +use core::ptr::NonNull; +use malachite_bigint::{BigInt, Sign}; +use num_integer::Integer; +use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero}; + +#[pyclass(module = false, name = "int")] +#[derive(Debug)] +pub struct PyInt { + value: BigInt, +} + +impl fmt::Display for PyInt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + BigInt::fmt(&self.value, f) + } +} + +pub type PyIntRef = PyRef<PyInt>; + +impl<T> From<T> for PyInt +where + T: Into<BigInt>, +{ + fn from(v: T) -> Self { + Self { value: v.into() } + } +} + +// spell-checker:ignore MAXFREELIST +thread_local! { + static INT_FREELIST: Cell<crate::object::FreeList<PyInt>> = const { Cell::new(crate::object::FreeList::new()) }; +} + +impl PyPayload for PyInt { + const MAX_FREELIST: usize = 100; + const HAS_FREELIST: bool = true; + + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.int_type + } + + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_int(self.value).into() + } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + INT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> { + INT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } +} + +macro_rules! impl_into_pyobject_int { + ($($t:ty)*) => {$( + impl ToPyObject for $t { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_int(self).into() + } + } + )*}; +} + +impl_into_pyobject_int!(isize i8 i16 i32 i64 i128 usize u8 u16 u32 u64 u128 BigInt); + +macro_rules! impl_try_from_object_int { + ($(($t:ty, $to_prim:ident),)*) => {$( + impl<'a> TryFromBorrowedObject<'a> for $t { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + obj.try_value_with(|int: &PyInt| { + int.try_to_primitive(vm) + }, vm) + } + } + )*}; +} + +impl_try_from_object_int!( + (isize, to_isize), + (i8, to_i8), + (i16, to_i16), + (i32, to_i32), + (i64, to_i64), + (i128, to_i128), + (usize, to_usize), + (u8, to_u8), + (u16, to_u16), + (u32, to_u32), + (u64, to_u64), + (u128, to_u128), +); + +fn inner_pow(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { + if int2.is_negative() { + let v1 = try_to_float(int1, vm)?; + let v2 = try_to_float(int2, vm)?; + float::float_pow(v1, v2, vm) + } else { + let value = if let Some(v2) = int2.to_u64() { + return Ok(vm.ctx.new_int(Pow::pow(int1, v2)).into()); + } else if int1.is_one() { + 1 + } else if int1.is_zero() { + 0 + } else if int1 == &BigInt::from(-1) { + if int2.is_odd() { -1 } else { 1 } + } else { + // missing feature: BigInt exp + // practically, exp over u64 is not possible to calculate anyway + return Ok(vm.ctx.not_implemented()); + }; + Ok(vm.ctx.new_int(value).into()) + } +} + +fn inner_mod(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { + if int2.is_zero() { + Err(vm.new_zero_division_error("division by zero")) + } else { + Ok(vm.ctx.new_int(int1.mod_floor(int2)).into()) + } +} + +fn inner_floordiv(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { + if int2.is_zero() { + Err(vm.new_zero_division_error("division by zero")) + } else { + Ok(vm.ctx.new_int(int1.div_floor(int2)).into()) + } +} + +fn inner_divmod(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { + if int2.is_zero() { + return Err(vm.new_zero_division_error("division by zero")); + } + let (div, modulo) = int1.div_mod_floor(int2); + Ok(vm.new_tuple((div, modulo)).into()) +} + +fn inner_lshift(base: &BigInt, bits: &BigInt, vm: &VirtualMachine) -> PyResult { + inner_shift( + base, + bits, + |base, bits| base << bits, + |bits, vm| { + bits.to_usize() + .ok_or_else(|| vm.new_overflow_error("the number is too large to convert to int")) + }, + vm, + ) +} + +fn inner_rshift(base: &BigInt, bits: &BigInt, vm: &VirtualMachine) -> PyResult { + inner_shift( + base, + bits, + |base, bits| base >> bits, + |bits, _vm| Ok(bits.to_usize().unwrap_or(usize::MAX)), + vm, + ) +} + +fn inner_shift<F, S>( + base: &BigInt, + bits: &BigInt, + shift_op: F, + shift_bits: S, + vm: &VirtualMachine, +) -> PyResult +where + F: Fn(&BigInt, usize) -> BigInt, + S: Fn(&BigInt, &VirtualMachine) -> PyResult<usize>, +{ + if bits.is_negative() { + Err(vm.new_value_error("negative shift count")) + } else if base.is_zero() { + Ok(vm.ctx.new_int(0).into()) + } else { + shift_bits(bits, vm).map(|bits| vm.ctx.new_int(shift_op(base, bits)).into()) + } +} + +fn inner_truediv(i1: &BigInt, i2: &BigInt, vm: &VirtualMachine) -> PyResult { + if i2.is_zero() { + return Err(vm.new_zero_division_error("division by zero")); + } + + let float = true_div(i1, i2); + + if float.is_infinite() { + Err(vm.new_exception_msg( + vm.ctx.exceptions.overflow_error.to_owned(), + "integer division result too large for a float".into(), + )) + } else { + Ok(vm.ctx.new_float(float).into()) + } +} + +impl Constructor for PyInt { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if cls.is(vm.ctx.types.bool_type) { + return Err(vm.new_type_error("int.__new__(bool) is not safe, use bool.__new__()")); + } + + // Optimization: return exact int as-is (only for exact int type, not subclasses) + if cls.is(vm.ctx.types.int_type) + && args.args.len() == 1 + && args.kwargs.is_empty() + && args.args[0].class().is(vm.ctx.types.int_type) + { + return Ok(args.args[0].clone()); + } + + let options: IntOptions = args.bind(vm)?; + let value = if let OptionalArg::Present(val) = options.val_options { + if let OptionalArg::Present(base) = options.base { + let base = base + .try_index(vm)? + .as_bigint() + .to_u32() + .filter(|&v| v == 0 || (2..=36).contains(&v)) + .ok_or_else(|| vm.new_value_error("int() base must be >= 2 and <= 36, or 0"))?; + try_int_radix(&val, base, vm) + } else { + val.try_int(vm).map(|x| x.as_bigint().clone()) + } + } else if let OptionalArg::Present(_) = options.base { + Err(vm.new_type_error("int() missing string argument")) + } else { + Ok(Zero::zero()) + }?; + + Self::with_value(cls, value, vm).map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl PyInt { + fn with_value<T>(cls: PyTypeRef, value: T, vm: &VirtualMachine) -> PyResult<PyRef<Self>> + where + T: Into<BigInt> + ToPrimitive, + { + if cls.is(vm.ctx.types.int_type) { + Ok(vm.ctx.new_int(value)) + } else if cls.is(vm.ctx.types.bool_type) { + Ok(vm.ctx.new_bool(!value.into().eq(&BigInt::zero())).upcast()) + } else { + Self::from(value).into_ref_with_type(vm, cls) + } + } + + pub const fn as_bigint(&self) -> &BigInt { + &self.value + } + + /// Fast decimal string conversion, using i64 path when possible. + #[inline] + pub fn to_str_radix_10(&self) -> String { + match self.value.to_i64() { + Some(i) => i.to_string(), + None => self.value.to_string(), + } + } + + // _PyLong_AsUnsignedLongMask + pub fn as_u32_mask(&self) -> u32 { + let v = self.as_bigint(); + v.to_u32() + .or_else(|| v.to_i32().map(|i| i as u32)) + .unwrap_or_else(|| { + let mut out = 0u32; + for digit in v.iter_u32_digits() { + out = out.wrapping_shl(32) | digit; + } + match v.sign() { + Sign::Minus => out * -1i32 as u32, + _ => out, + } + }) + } + + pub fn try_to_primitive<'a, I>(&'a self, vm: &VirtualMachine) -> PyResult<I> + where + I: PrimInt + TryFrom<&'a BigInt>, + { + // TODO: Python 3.14+: ValueError for negative int to unsigned type + // See stdlib_socket.py socket.htonl(-1) + // + // if I::min_value() == I::zero() && self.as_bigint().sign() == Sign::Minus { + // return Err(vm.new_value_error("Cannot convert negative int".to_owned())); + // } + + I::try_from(self.as_bigint()).map_err(|_| { + vm.new_overflow_error(format!( + "Python int too large to convert to Rust {}", + core::any::type_name::<I>() + )) + }) + } + + #[inline] + fn int_op<F>(&self, other: PyObjectRef, op: F) -> PyArithmeticValue<BigInt> + where + F: Fn(&BigInt, &BigInt) -> BigInt, + { + let r = other + .downcast_ref::<Self>() + .map(|other| op(&self.value, &other.value)); + PyArithmeticValue::from_option(r) + } + + #[inline] + fn general_op<F>(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyResult + where + F: Fn(&BigInt, &BigInt) -> PyResult, + { + if let Some(other) = other.downcast_ref::<Self>() { + op(&self.value, &other.value) + } else { + Ok(vm.ctx.not_implemented()) + } + } +} + +#[pyclass( + itemsize = 4, + flags(BASETYPE, _MATCH_SELF), + with(PyRef, Comparable, Hashable, Constructor, AsNumber, Representable) +)] +impl PyInt { + pub(crate) fn __xor__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { + self.int_op(other, |a, b| a ^ b) + } + + pub(crate) fn __or__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { + self.int_op(other, |a, b| a | b) + } + + pub(crate) fn __and__(&self, other: PyObjectRef) -> PyArithmeticValue<BigInt> { + self.int_op(other, |a, b| a & b) + } + + fn modpow(&self, other: PyObjectRef, modulus: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let modulus = match modulus.downcast_ref::<Self>() { + Some(val) => val.as_bigint(), + None => return Ok(vm.ctx.not_implemented()), + }; + if modulus.is_zero() { + return Err(vm.new_value_error("pow() 3rd argument cannot be 0")); + } + + self.general_op( + other, + |a, b| { + let i = if b.is_negative() { + // modular multiplicative inverse + // based on rust-num/num-integer#10, should hopefully be published soon + fn normalize(a: BigInt, n: &BigInt) -> BigInt { + let a = a % n; + if a.is_negative() { a + n } else { a } + } + fn inverse(a: BigInt, n: &BigInt) -> Option<BigInt> { + use num_integer::*; + let ExtendedGcd { gcd, x: c, .. } = a.extended_gcd(n); + if gcd.is_one() { + Some(normalize(c, n)) + } else { + None + } + } + let a = inverse(a % modulus, modulus).ok_or_else(|| { + vm.new_value_error("base is not invertible for the given modulus") + })?; + let b = -b; + a.modpow(&b, modulus) + } else { + a.modpow(b, modulus) + }; + Ok(vm.ctx.new_int(i).into()) + }, + vm, + ) + } + + #[pymethod] + fn __round__( + zelf: PyRef<Self>, + ndigits: OptionalOption<PyIntRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + if let Some(ndigits) = ndigits.flatten() { + let ndigits = ndigits.as_bigint(); + // round(12345, -2) == 12300 + // If precision >= 0, then any integer is already rounded correctly + if let Some(ndigits) = ndigits.neg().to_u32() + && ndigits > 0 + { + // Work with positive integers and negate at the end if necessary + let sign = if zelf.value.is_negative() { + BigInt::from(-1) + } else { + BigInt::from(1) + }; + let value = zelf.value.abs(); + + // Divide and multiply by the power of 10 to get the approximate answer + let pow10 = BigInt::from(10).pow(ndigits); + let quotient = &value / &pow10; + let rounded = &quotient * &pow10; + + // Malachite division uses floor rounding, Python uses half-even + let remainder = &value - &rounded; + let half_pow10 = &pow10 / BigInt::from(2); + let correction = + if remainder > half_pow10 || (remainder == half_pow10 && quotient.is_odd()) { + pow10 + } else { + BigInt::from(0) + }; + let rounded = (rounded + correction) * sign; + return Ok(vm.ctx.new_int(rounded)); + } + } + Ok(zelf) + } + + #[pymethod] + fn __trunc__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { + zelf.__int__(vm) + } + + #[pymethod] + fn __floor__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { + zelf.__int__(vm) + } + + #[pymethod] + fn __ceil__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { + zelf.__int__(vm) + } + + #[pymethod] + fn __format__(zelf: &Py<Self>, spec: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + // Empty format spec on a subclass: equivalent to str(self) + if spec.is_empty() && !zelf.class().is(vm.ctx.types.int_type) { + return Ok(zelf.as_object().str(vm)?.as_wtf8().to_owned()); + } + let format_spec = + FormatSpec::parse(spec.as_str()).map_err(|err| err.into_pyexception(vm))?; + let result = if format_spec.has_locale_format() { + let locale = crate::format::get_locale_info(); + format_spec.format_int_locale(&zelf.value, &locale) + } else { + format_spec.format_int(&zelf.value) + }; + result + .map(Wtf8Buf::from_string) + .map_err(|err| err.into_pyexception(vm)) + } + + #[pymethod] + fn __sizeof__(&self) -> usize { + core::mem::size_of::<Self>() + (((self.value.bits() + 7) & !7) / 8) as usize + } + + #[pymethod] + fn as_integer_ratio(&self, vm: &VirtualMachine) -> (PyRef<Self>, i32) { + (vm.ctx.new_bigint(&self.value), 1) + } + + #[pymethod] + fn bit_length(&self) -> u64 { + self.value.bits() + } + + #[pymethod] + fn conjugate(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { + zelf.__int__(vm) + } + + #[pyclassmethod] + fn from_bytes( + cls: PyTypeRef, + args: IntFromByteArgs, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + let signed = args.signed.map_or(false, Into::into); + let value = match (args.byteorder, signed) { + (ArgByteOrder::Big, true) => BigInt::from_signed_bytes_be(args.bytes.as_bytes()), + (ArgByteOrder::Big, false) => BigInt::from_bytes_be(Sign::Plus, args.bytes.as_bytes()), + (ArgByteOrder::Little, true) => BigInt::from_signed_bytes_le(args.bytes.as_bytes()), + (ArgByteOrder::Little, false) => { + BigInt::from_bytes_le(Sign::Plus, args.bytes.as_bytes()) + } + }; + Self::with_value(cls, value, vm) + } + + #[pymethod] + fn to_bytes(&self, args: IntToByteArgs, vm: &VirtualMachine) -> PyResult<PyBytes> { + let signed = args.signed.map_or(false, Into::into); + let byte_len = args.length; + + let value = self.as_bigint(); + match value.sign() { + Sign::Minus if !signed => { + return Err(vm.new_overflow_error("can't convert negative int to unsigned")); + } + Sign::NoSign => return Ok(vec![0u8; byte_len].into()), + _ => {} + } + + let mut origin_bytes = match (args.byteorder, signed) { + (ArgByteOrder::Big, true) => value.to_signed_bytes_be(), + (ArgByteOrder::Big, false) => value.to_bytes_be().1, + (ArgByteOrder::Little, true) => value.to_signed_bytes_le(), + (ArgByteOrder::Little, false) => value.to_bytes_le().1, + }; + + let origin_len = origin_bytes.len(); + if origin_len > byte_len { + return Err(vm.new_overflow_error("int too big to convert")); + } + + let mut append_bytes = match value.sign() { + Sign::Minus => vec![255u8; byte_len - origin_len], + _ => vec![0u8; byte_len - origin_len], + }; + + let bytes = match args.byteorder { + ArgByteOrder::Big => { + let mut bytes = append_bytes; + bytes.append(&mut origin_bytes); + bytes + } + ArgByteOrder::Little => { + let mut bytes = origin_bytes; + bytes.append(&mut append_bytes); + bytes + } + }; + Ok(bytes.into()) + } + + #[pygetset] + fn real(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { + zelf.__int__(vm) + } + + #[pygetset] + const fn imag(&self) -> usize { + 0 + } + + #[pygetset] + fn numerator(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { + zelf.__int__(vm) + } + + #[pygetset] + const fn denominator(&self) -> usize { + 1 + } + + #[pymethod] + const fn is_integer(&self) -> bool { + true + } + + #[pymethod] + fn bit_count(&self) -> u32 { + self.value.iter_u32_digits().map(|n| n.count_ones()).sum() + } + + #[pymethod] + fn __getnewargs__(&self, vm: &VirtualMachine) -> PyObjectRef { + (self.value.clone(),).to_pyobject(vm) + } +} + +#[pyclass] +impl PyRef<PyInt> { + pub(crate) fn __int__(self, vm: &VirtualMachine) -> PyRefExact<PyInt> { + self.into_exact_or(&vm.ctx, |zelf| unsafe { + // TODO: this is actually safe. we need better interface + PyRefExact::new_unchecked(vm.ctx.new_bigint(&zelf.value)) + }) + } +} + +impl Comparable for PyInt { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let r = other + .downcast_ref::<Self>() + .map(|other| op.eval_ord(zelf.value.cmp(&other.value))); + Ok(PyComparisonValue::from_option(r)) + } +} + +impl Representable for PyInt { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok(zelf.to_str_radix_10()) + } +} + +impl Hashable for PyInt { + #[inline] + fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<hash::PyHash> { + Ok(hash::hash_bigint(zelf.as_bigint())) + } +} + +impl AsNumber for PyInt { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyInt::AS_NUMBER; + &AS_NUMBER + } + + #[inline] + fn clone_exact(zelf: &Py<Self>, vm: &VirtualMachine) -> PyRef<Self> { + vm.ctx.new_bigint(&zelf.value) + } +} + +impl PyInt { + pub(super) const AS_NUMBER: PyNumberMethods = PyNumberMethods { + add: Some(|a, b, vm| Self::number_op(a, b, |a, b, _vm| a + b, vm)), + subtract: Some(|a, b, vm| Self::number_op(a, b, |a, b, _vm| a - b, vm)), + multiply: Some(|a, b, vm| Self::number_op(a, b, |a, b, _vm| a * b, vm)), + remainder: Some(|a, b, vm| Self::number_op(a, b, inner_mod, vm)), + divmod: Some(|a, b, vm| Self::number_op(a, b, inner_divmod, vm)), + power: Some(|a, b, c, vm| { + if let Some(a) = a.downcast_ref::<Self>() { + if vm.is_none(c) { + a.general_op(b.to_owned(), |a, b| inner_pow(a, b, vm), vm) + } else { + a.modpow(b.to_owned(), c.to_owned(), vm) + } + } else { + Ok(vm.ctx.not_implemented()) + } + }), + negative: Some(|num, vm| (&Self::number_downcast(num).value).neg().to_pyresult(vm)), + positive: Some(|num, vm| Ok(Self::number_downcast_exact(num, vm).into())), + absolute: Some(|num, vm| Self::number_downcast(num).value.abs().to_pyresult(vm)), + boolean: Some(|num, _vm| Ok(!Self::number_downcast(num).value.is_zero())), + invert: Some(|num, vm| (&Self::number_downcast(num).value).not().to_pyresult(vm)), + lshift: Some(|a, b, vm| Self::number_op(a, b, inner_lshift, vm)), + rshift: Some(|a, b, vm| Self::number_op(a, b, inner_rshift, vm)), + and: Some(|a, b, vm| Self::number_op(a, b, |a, b, _vm| a & b, vm)), + xor: Some(|a, b, vm| Self::number_op(a, b, |a, b, _vm| a ^ b, vm)), + or: Some(|a, b, vm| Self::number_op(a, b, |a, b, _vm| a | b, vm)), + int: Some(|num, vm| Ok(Self::number_downcast_exact(num, vm).into())), + float: Some(|num, vm| { + let zelf = Self::number_downcast(num); + try_to_float(&zelf.value, vm).map(|x| vm.ctx.new_float(x).into()) + }), + floor_divide: Some(|a, b, vm| Self::number_op(a, b, inner_floordiv, vm)), + true_divide: Some(|a, b, vm| Self::number_op(a, b, inner_truediv, vm)), + index: Some(|num, vm| Ok(Self::number_downcast_exact(num, vm).into())), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + + fn number_op<F, R>(a: &PyObject, b: &PyObject, op: F, vm: &VirtualMachine) -> PyResult + where + F: FnOnce(&BigInt, &BigInt, &VirtualMachine) -> R, + R: ToPyResult, + { + if let (Some(a), Some(b)) = (a.downcast_ref::<Self>(), b.downcast_ref::<Self>()) { + op(&a.value, &b.value, vm).to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + } +} + +#[derive(FromArgs)] +pub struct IntOptions { + #[pyarg(positional, optional)] + val_options: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + base: OptionalArg<PyObjectRef>, +} + +#[derive(FromArgs)] +struct IntFromByteArgs { + bytes: PyBytesInner, + #[pyarg(any, default = ArgByteOrder::Big)] + byteorder: ArgByteOrder, + #[pyarg(named, optional)] + signed: OptionalArg<ArgIntoBool>, +} + +#[derive(FromArgs)] +struct IntToByteArgs { + #[pyarg(any, default = 1)] + length: usize, + #[pyarg(any, default = ArgByteOrder::Big)] + byteorder: ArgByteOrder, + #[pyarg(named, optional)] + signed: OptionalArg<ArgIntoBool>, +} + +fn try_int_radix(obj: &PyObject, base: u32, vm: &VirtualMachine) -> PyResult<BigInt> { + match_class!(match obj.to_owned() { + string @ PyStr => { + let s = string.as_wtf8().trim(); + bytes_to_int(s.as_bytes(), base, vm.state.int_max_str_digits.load()) + .map_err(|e| handle_bytes_to_int_err(e, obj, vm)) + } + bytes @ PyBytes => { + bytes_to_int(bytes.as_bytes(), base, vm.state.int_max_str_digits.load()) + .map_err(|e| handle_bytes_to_int_err(e, obj, vm)) + } + bytearray @ PyByteArray => { + let inner = bytearray.borrow_buf(); + bytes_to_int(&inner, base, vm.state.int_max_str_digits.load()) + .map_err(|e| handle_bytes_to_int_err(e, obj, vm)) + } + _ => Err(vm.new_type_error("int() can't convert non-string with explicit base")), + }) +} + +// Retrieve inner int value: +pub(crate) fn get_value(obj: &PyObject) -> &BigInt { + &obj.downcast_ref::<PyInt>().unwrap().value +} + +pub fn try_to_float(int: &BigInt, vm: &VirtualMachine) -> PyResult<f64> { + bigint_to_finite_float(int) + .ok_or_else(|| vm.new_overflow_error("int too large to convert to float")) +} + +pub(crate) fn init(context: &'static Context) { + PyInt::extend_class(context, context.types.int_type); +} diff --git a/crates/vm/src/builtins/interpolation.rs b/crates/vm/src/builtins/interpolation.rs new file mode 100644 index 00000000000..b429fb5b91c --- /dev/null +++ b/crates/vm/src/builtins/interpolation.rs @@ -0,0 +1,229 @@ +use super::{ + PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, genericalias::PyGenericAlias, + tuple::IntoPyTuple, +}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + common::hash::PyHash, + convert::ToPyObject, + function::{OptionalArg, PyComparisonValue}, + types::{Comparable, Constructor, Hashable, PyComparisonOp, Representable}, +}; +use itertools::Itertools; +use rustpython_common::wtf8::Wtf8Buf; + +/// Interpolation object for t-strings (PEP 750). +/// +/// Represents an interpolated expression within a template string. +#[pyclass(module = "string.templatelib", name = "Interpolation")] +#[derive(Debug, Clone)] +pub struct PyInterpolation { + pub value: PyObjectRef, + pub expression: PyStrRef, + pub conversion: PyObjectRef, // None or 's', 'r', 'a' + pub format_spec: PyStrRef, +} + +impl PyPayload for PyInterpolation { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.interpolation_type + } +} + +impl PyInterpolation { + pub fn new( + value: PyObjectRef, + expression: PyStrRef, + conversion: PyObjectRef, + format_spec: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<Self> { + // Validate conversion like _PyInterpolation_Build does + let is_valid = vm.is_none(&conversion) + || conversion + .downcast_ref::<PyStr>() + .is_some_and(|s| matches!(s.to_str(), Some("s") | Some("r") | Some("a"))); + if !is_valid { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.system_error.to_owned(), + "Interpolation() argument 'conversion' must be one of 's', 'a' or 'r'".into(), + )); + } + Ok(Self { + value, + expression, + conversion, + format_spec, + }) + } +} + +impl Constructor for PyInterpolation { + type Args = InterpolationArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let conversion: PyObjectRef = if let Some(s) = args.conversion { + let has_flag = s + .as_bytes() + .iter() + .exactly_one() + .ok() + .is_some_and(|s| matches!(*s, b's' | b'r' | b'a')); + if !has_flag { + return Err(vm.new_value_error( + "Interpolation() argument 'conversion' must be one of 's', 'a' or 'r'", + )); + } + s.into() + } else { + vm.ctx.none() + }; + + let expression = args + .expression + .unwrap_or_else(|| vm.ctx.empty_str.to_owned()); + let format_spec = args + .format_spec + .unwrap_or_else(|| vm.ctx.empty_str.to_owned()); + + Ok(PyInterpolation { + value: args.value, + expression, + conversion, + format_spec, + }) + } +} + +#[derive(FromArgs)] +pub struct InterpolationArgs { + #[pyarg(positional)] + value: PyObjectRef, + #[pyarg(any, optional)] + expression: OptionalArg<PyStrRef>, + #[pyarg( + any, + optional, + error_msg = "Interpolation() argument 'conversion' must be str or None" + )] + conversion: Option<PyStrRef>, + #[pyarg(any, optional)] + format_spec: OptionalArg<PyStrRef>, +} + +#[pyclass(with(Constructor, Comparable, Hashable, Representable))] +impl PyInterpolation { + #[pyattr] + fn __match_args__(ctx: &Context) -> PyTupleRef { + ctx.new_tuple(vec![ + ctx.intern_str("value").to_owned().into(), + ctx.intern_str("expression").to_owned().into(), + ctx.intern_str("conversion").to_owned().into(), + ctx.intern_str("format_spec").to_owned().into(), + ]) + } + + #[pygetset] + fn value(&self) -> PyObjectRef { + self.value.clone() + } + + #[pygetset] + fn expression(&self) -> PyStrRef { + self.expression.clone() + } + + #[pygetset] + fn conversion(&self) -> PyObjectRef { + self.conversion.clone() + } + + #[pygetset] + fn format_spec(&self) -> PyStrRef { + self.format_spec.clone() + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { + let cls = zelf.class().to_owned(); + let args = ( + zelf.value.clone(), + zelf.expression.clone(), + zelf.conversion.clone(), + zelf.format_spec.clone(), + ); + (cls, args.to_pyobject(vm)).into_pytuple(vm) + } +} + +impl Comparable for PyInterpolation { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + + let eq = vm.bool_eq(&zelf.value, &other.value)? + && vm.bool_eq(zelf.expression.as_object(), other.expression.as_object())? + && vm.bool_eq(&zelf.conversion, &other.conversion)? + && vm.bool_eq(zelf.format_spec.as_object(), other.format_spec.as_object())?; + + Ok(eq.into()) + }) + } +} + +impl Hashable for PyInterpolation { + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + // Hash based on (value, expression, conversion, format_spec) + let value_hash = zelf.value.hash(vm)?; + let expr_hash = zelf.expression.as_object().hash(vm)?; + let conv_hash = zelf.conversion.hash(vm)?; + let spec_hash = zelf.format_spec.as_object().hash(vm)?; + + // Combine hashes + Ok(value_hash + .wrapping_add(expr_hash.wrapping_mul(3)) + .wrapping_add(conv_hash.wrapping_mul(5)) + .wrapping_add(spec_hash.wrapping_mul(7))) + } +} + +impl Representable for PyInterpolation { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let value_repr = zelf.value.repr(vm)?; + let expr_repr = zelf.expression.repr(vm)?; + let spec_repr = zelf.format_spec.repr(vm)?; + + let mut result = Wtf8Buf::from("Interpolation("); + result.push_wtf8(value_repr.as_wtf8()); + result.push_str(", "); + result.push_str(&expr_repr); + result.push_str(", "); + if vm.is_none(&zelf.conversion) { + result.push_str("None"); + } else { + result.push_wtf8(zelf.conversion.repr(vm)?.as_wtf8()); + } + result.push_str(", "); + result.push_str(&spec_repr); + result.push_char(')'); + + Ok(result) + } +} + +pub fn init(context: &'static Context) { + PyInterpolation::extend_class(context, context.types.interpolation_type); +} diff --git a/crates/vm/src/builtins/iter.rs b/crates/vm/src/builtins/iter.rs new file mode 100644 index 00000000000..f33b0e9bbbb --- /dev/null +++ b/crates/vm/src/builtins/iter.rs @@ -0,0 +1,295 @@ +/* + * iterator types + */ + +use super::{PyInt, PyTupleRef, PyType}; +use crate::{ + Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + class::PyClassImpl, + function::ArgCallable, + object::{Traverse, TraverseFn}, + protocol::PyIterReturn, + types::{IterNext, Iterable, SelfIter}, +}; +use rustpython_common::lock::{PyMutex, PyRwLock, PyRwLockUpgradableReadGuard}; + +/// Marks status of iterator. +#[derive(Debug, Clone)] +pub enum IterStatus<T> { + /// Iterator hasn't raised StopIteration. + Active(T), + /// Iterator has raised StopIteration. + Exhausted, +} + +unsafe impl<T: Traverse> Traverse for IterStatus<T> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + match self { + Self::Active(r) => r.traverse(tracer_fn), + Self::Exhausted => (), + } + } +} + +#[derive(Debug)] +pub struct PositionIterInternal<T> { + pub status: IterStatus<T>, + pub position: usize, +} + +unsafe impl<T: Traverse> Traverse for PositionIterInternal<T> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.status.traverse(tracer_fn) + } +} + +impl<T> PositionIterInternal<T> { + pub const fn new(obj: T, position: usize) -> Self { + Self { + status: IterStatus::Active(obj), + position, + } + } + + pub fn set_state<F>(&mut self, state: PyObjectRef, f: F, vm: &VirtualMachine) -> PyResult<()> + where + F: FnOnce(&T, usize) -> usize, + { + if let IterStatus::Active(obj) = &self.status { + if let Some(i) = state.downcast_ref::<PyInt>() { + let i = i.try_to_primitive(vm).unwrap_or(0); + self.position = f(obj, i); + Ok(()) + } else { + Err(vm.new_type_error("an integer is required.")) + } + } else { + Ok(()) + } + } + + /// Build a pickle-compatible reduce tuple. + /// + /// `func` must be resolved **before** acquiring any lock that guards this + /// `PositionIterInternal`, so that the builtins lookup cannot trigger + /// reentrant iterator access and deadlock. + pub fn reduce<F, E>( + &self, + func: PyObjectRef, + active: F, + empty: E, + vm: &VirtualMachine, + ) -> PyTupleRef + where + F: FnOnce(&T) -> PyObjectRef, + E: FnOnce(&VirtualMachine) -> PyObjectRef, + { + if let IterStatus::Active(obj) = &self.status { + vm.new_tuple((func, (active(obj),), self.position)) + } else { + vm.new_tuple((func, (empty(vm),))) + } + } + + fn _next<F, OP>(&mut self, f: F, op: OP) -> PyResult<PyIterReturn> + where + F: FnOnce(&T, usize) -> PyResult<PyIterReturn>, + OP: FnOnce(&mut Self), + { + if let IterStatus::Active(obj) = &self.status { + let ret = f(obj, self.position); + if let Ok(PyIterReturn::Return(_)) = ret { + op(self); + } else { + self.status = IterStatus::Exhausted; + } + ret + } else { + Ok(PyIterReturn::StopIteration(None)) + } + } + + pub fn next<F>(&mut self, f: F) -> PyResult<PyIterReturn> + where + F: FnOnce(&T, usize) -> PyResult<PyIterReturn>, + { + self._next(f, |zelf| zelf.position += 1) + } + + pub fn rev_next<F>(&mut self, f: F) -> PyResult<PyIterReturn> + where + F: FnOnce(&T, usize) -> PyResult<PyIterReturn>, + { + self._next(f, |zelf| { + if zelf.position == 0 { + zelf.status = IterStatus::Exhausted; + } else { + zelf.position -= 1; + } + }) + } + + pub fn length_hint<F>(&self, f: F) -> usize + where + F: FnOnce(&T) -> usize, + { + if let IterStatus::Active(obj) = &self.status { + f(obj).saturating_sub(self.position) + } else { + 0 + } + } + + pub fn rev_length_hint<F>(&self, f: F) -> usize + where + F: FnOnce(&T) -> usize, + { + if let IterStatus::Active(obj) = &self.status + && self.position <= f(obj) + { + return self.position + 1; + } + 0 + } +} + +pub fn builtins_iter(vm: &VirtualMachine) -> PyObjectRef { + vm.builtins.get_attr("iter", vm).unwrap() +} + +pub fn builtins_reversed(vm: &VirtualMachine) -> PyObjectRef { + vm.builtins.get_attr("reversed", vm).unwrap() +} + +#[pyclass(module = false, name = "iterator", traverse)] +#[derive(Debug)] +pub struct PySequenceIterator { + internal: PyMutex<PositionIterInternal<PyObjectRef>>, +} + +impl PyPayload for PySequenceIterator { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.iter_type + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PySequenceIterator { + pub fn new(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { + let _seq = obj.try_sequence(vm)?; + Ok(Self { + internal: PyMutex::new(PositionIterInternal::new(obj, 0)), + }) + } + + #[pymethod] + fn __length_hint__(&self, vm: &VirtualMachine) -> PyObjectRef { + let internal = self.internal.lock(); + if let IterStatus::Active(obj) = &internal.status { + let seq = obj.sequence_unchecked(); + seq.length(vm) + .map(|x| PyInt::from(x).into_pyobject(vm)) + .unwrap_or_else(|_| vm.ctx.not_implemented()) + } else { + PyInt::from(0).into_pyobject(vm) + } + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) + } + + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.internal.lock().set_state(state, |_, pos| pos, vm) + } +} + +impl SelfIter for PySequenceIterator {} +impl IterNext for PySequenceIterator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.internal.lock().next(|obj, pos| { + let seq = obj.sequence_unchecked(); + PyIterReturn::from_getitem_result(seq.get_item(pos as isize, vm), vm) + }) + } +} + +#[pyclass(module = false, name = "callable_iterator", traverse)] +#[derive(Debug)] +pub struct PyCallableIterator { + sentinel: PyObjectRef, + status: PyRwLock<IterStatus<ArgCallable>>, +} + +impl PyPayload for PyCallableIterator { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.callable_iterator + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PyCallableIterator { + pub const fn new(callable: ArgCallable, sentinel: PyObjectRef) -> Self { + Self { + sentinel, + status: PyRwLock::new(IterStatus::Active(callable)), + } + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + let status = self.status.read(); + if let IterStatus::Active(callable) = &*status { + let callable_obj: PyObjectRef = callable.clone().into(); + vm.new_tuple((func, (callable_obj, self.sentinel.clone()))) + } else { + vm.new_tuple((func, (vm.ctx.empty_tuple.clone(),))) + } + } +} + +impl SelfIter for PyCallableIterator {} +impl IterNext for PyCallableIterator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + // Clone the callable and release the lock before invoking, + // so that reentrant next() calls don't deadlock. + let callable = { + let status = zelf.status.read(); + match &*status { + IterStatus::Active(callable) => callable.clone(), + IterStatus::Exhausted => return Ok(PyIterReturn::StopIteration(None)), + } + }; + + let ret = callable.invoke((), vm)?; + + // Re-check: a reentrant call may have exhausted the iterator. + let status = zelf.status.upgradable_read(); + if !matches!(&*status, IterStatus::Active(_)) { + return Ok(PyIterReturn::StopIteration(None)); + } + + if vm.bool_eq(&ret, &zelf.sentinel)? { + *PyRwLockUpgradableReadGuard::upgrade(status) = IterStatus::Exhausted; + Ok(PyIterReturn::StopIteration(None)) + } else { + Ok(PyIterReturn::Return(ret)) + } + } +} + +pub fn init(context: &'static Context) { + PySequenceIterator::extend_class(context, context.types.iter_type); + PyCallableIterator::extend_class(context, context.types.callable_iterator); +} diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs new file mode 100644 index 00000000000..cdb8a73ead2 --- /dev/null +++ b/crates/vm/src/builtins/list.rs @@ -0,0 +1,777 @@ +use super::{ + PositionIterInternal, PyGenericAlias, PyTupleRef, PyType, PyTypeRef, + iter::{builtins_iter, builtins_reversed}, +}; +use crate::atomic_func; +use crate::common::lock::{ + PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, +}; +use crate::object::{Traverse, TraverseFn}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + builtins::PyStr, + class::PyClassImpl, + convert::ToPyObject, + function::{ArgSize, FuncArgs, OptionalArg, PyComparisonValue}, + iter::PyExactSizeIterator, + protocol::{PyIterReturn, PyMappingMethods, PySequenceMethods}, + recursion::ReprGuard, + sequence::{MutObjectSequenceOp, OptionalRangeArgs, SequenceExt, SequenceMutExt}, + sliceable::{SequenceIndex, SliceableSequenceMutOp, SliceableSequenceOp}, + types::{ + AsMapping, AsSequence, Comparable, Constructor, Initializer, IterNext, Iterable, + PyComparisonOp, Representable, SelfIter, + }, + vm::VirtualMachine, +}; +use rustpython_common::wtf8::Wtf8Buf; + +use alloc::fmt; +use core::cell::Cell; +use core::ops::DerefMut; +use core::ptr::NonNull; + +#[pyclass(module = false, name = "list", unhashable = true, traverse = "manual")] +#[derive(Default)] +pub struct PyList { + elements: PyRwLock<Vec<PyObjectRef>>, +} + +impl fmt::Debug for PyList { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO: implement more detailed, non-recursive Debug formatter + f.write_str("list") + } +} + +impl From<Vec<PyObjectRef>> for PyList { + fn from(elements: Vec<PyObjectRef>) -> Self { + Self { + elements: PyRwLock::new(elements), + } + } +} + +impl FromIterator<PyObjectRef> for PyList { + fn from_iter<T: IntoIterator<Item = PyObjectRef>>(iter: T) -> Self { + Vec::from_iter(iter).into() + } +} + +// SAFETY: Traverse properly visits all owned PyObjectRefs +unsafe impl Traverse for PyList { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.elements.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec<PyObjectRef>) { + // During GC, we use interior mutability to access elements. + // This is safe because during GC collection, the object is unreachable + // and no other code should be accessing it. + if let Some(mut guard) = self.elements.try_write() { + out.extend(guard.drain(..)); + } + } +} + +thread_local! { + static LIST_FREELIST: Cell<crate::object::FreeList<PyList>> = const { Cell::new(crate::object::FreeList::new()) }; +} + +impl PyPayload for PyList { + const MAX_FREELIST: usize = 80; + const HAS_FREELIST: bool = true; + + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.list_type + } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + LIST_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> { + LIST_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } +} + +impl ToPyObject for Vec<PyObjectRef> { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + PyList::from(self).into_ref(&vm.ctx).into() + } +} + +impl PyList { + #[deprecated(note = "use PyList::from(...).into_ref() instead")] + pub fn new_ref(elements: Vec<PyObjectRef>, ctx: &Context) -> PyRef<Self> { + Self::from(elements).into_ref(ctx) + } + + pub fn borrow_vec(&self) -> PyMappedRwLockReadGuard<'_, [PyObjectRef]> { + PyRwLockReadGuard::map(self.elements.read(), |v| &**v) + } + + pub fn borrow_vec_mut(&self) -> PyRwLockWriteGuard<'_, Vec<PyObjectRef>> { + self.elements.write() + } + + fn repeat(&self, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let elements = &*self.borrow_vec(); + let v = elements.mul(vm, n)?; + Ok(Self::from(v).into_ref(&vm.ctx)) + } + + fn irepeat(zelf: PyRef<Self>, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + zelf.borrow_vec_mut().imul(vm, n)?; + Ok(zelf) + } +} + +#[derive(FromArgs, Default, Traverse)] +pub(crate) struct SortOptions { + #[pyarg(named, default)] + key: Option<PyObjectRef>, + #[pytraverse(skip)] + #[pyarg(named, default = false)] + reverse: bool, +} + +pub type PyListRef = PyRef<PyList>; + +#[pyclass( + with( + Constructor, + Initializer, + AsMapping, + Iterable, + Comparable, + AsSequence, + Representable + ), + flags(BASETYPE, SEQUENCE, _MATCH_SELF) +)] +impl PyList { + #[pymethod] + pub(crate) fn append(&self, x: PyObjectRef) { + self.borrow_vec_mut().push(x); + } + + #[pymethod] + pub(crate) fn extend(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut new_elements = x.try_to_value(vm)?; + self.borrow_vec_mut().append(&mut new_elements); + Ok(()) + } + + #[pymethod] + pub(crate) fn insert(&self, position: isize, element: PyObjectRef) { + let mut elements = self.borrow_vec_mut(); + let position = elements.saturate_index(position); + elements.insert(position, element); + } + + fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let other = other.downcast_ref::<Self>().ok_or_else(|| { + vm.new_type_error(format!( + "Cannot add {} and {}", + Self::class(&vm.ctx).name(), + other.class().name() + )) + })?; + let mut elements = self.borrow_vec().to_vec(); + elements.extend(other.borrow_vec().iter().cloned()); + Ok(Self::from(elements).into_ref(&vm.ctx)) + } + + fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + self.concat(&other, vm) + } + + fn inplace_concat( + zelf: &Py<Self>, + other: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let mut seq = extract_cloned(other, Ok, vm)?; + zelf.borrow_vec_mut().append(&mut seq); + Ok(zelf.to_owned().into()) + } + + fn __iadd__( + zelf: PyRef<Self>, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + let mut seq = extract_cloned(&other, Ok, vm)?; + zelf.borrow_vec_mut().append(&mut seq); + Ok(zelf) + } + + #[pymethod] + fn clear(&self) { + let _removed = core::mem::take(self.borrow_vec_mut().deref_mut()); + } + + #[pymethod] + fn copy(&self, vm: &VirtualMachine) -> PyRef<Self> { + Self::from(self.borrow_vec().to_vec()).into_ref(&vm.ctx) + } + + #[allow(clippy::len_without_is_empty)] + pub fn __len__(&self) -> usize { + self.borrow_vec().len() + } + + #[pymethod] + fn __sizeof__(&self) -> usize { + core::mem::size_of::<Self>() + + self.elements.read().capacity() * core::mem::size_of::<PyObjectRef>() + } + + #[pymethod] + fn reverse(&self) { + self.borrow_vec_mut().reverse(); + } + + #[pymethod] + fn __reversed__(zelf: PyRef<Self>) -> PyListReverseIterator { + let position = zelf.__len__().saturating_sub(1); + PyListReverseIterator { + internal: PyMutex::new(PositionIterInternal::new(zelf, position)), + } + } + + fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { + SequenceIndex::Int(i) => { + let vec = self.borrow_vec(); + let pos = vec + .wrap_index(i) + .ok_or_else(|| vm.new_index_error("list index out of range"))?; + Ok(vec.do_get(pos)) + } + SequenceIndex::Slice(slice) => self + .borrow_vec() + .getitem_by_slice(vm, slice) + .map(|x| vm.ctx.new_list(x).into()), + } + } + + fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self._getitem(&needle, vm) + } + + fn _setitem(&self, needle: &PyObject, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { + SequenceIndex::Int(index) => self + .borrow_vec_mut() + .setitem_by_index(vm, index, value) + .map_err(|e| { + if e.class().is(vm.ctx.exceptions.index_error) { + vm.new_index_error("list assignment index out of range".to_owned()) + } else { + e + } + }), + SequenceIndex::Slice(slice) => { + let sec = extract_cloned(&value, Ok, vm)?; + self.borrow_vec_mut().setitem_by_slice(vm, slice, &sec) + } + } + } + + fn __setitem__( + &self, + needle: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self._setitem(&needle, value, vm) + } + + fn __mul__(&self, n: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + self.repeat(n.into(), vm) + } + + fn __imul__(zelf: PyRef<Self>, n: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + Self::irepeat(zelf, n.into(), vm) + } + + #[pymethod] + fn count(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + self.mut_count(vm, &needle) + } + + pub(crate) fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + self.mut_contains(vm, &needle) + } + + #[pymethod] + fn index( + &self, + needle: PyObjectRef, + range: OptionalRangeArgs, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let (start, stop) = range.saturate(self.__len__(), vm)?; + let index = self.mut_index_range(vm, &needle, start..stop)?; + if let Some(index) = index.into() { + Ok(index) + } else { + Err(vm.new_value_error(format!("'{}' is not in list", needle.str(vm)?))) + } + } + + #[pymethod] + fn pop(&self, i: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult { + let mut i = i.into_option().unwrap_or(-1); + let mut elements = self.borrow_vec_mut(); + if i < 0 { + i += elements.len() as isize; + } + if elements.is_empty() { + Err(vm.new_index_error("pop from empty list")) + } else if i < 0 || i as usize >= elements.len() { + Err(vm.new_index_error("pop index out of range")) + } else { + Ok(elements.remove(i as usize)) + } + } + + #[pymethod] + fn remove(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let index = self.mut_index(vm, &needle)?; + + if let Some(index) = index.into() { + // defer delete out of borrow + let is_inside_range = index < self.borrow_vec().len(); + Ok(is_inside_range.then(|| self.borrow_vec_mut().remove(index))) + } else { + Err(vm.new_value_error(format!("'{}' is not in list", needle.str(vm)?))) + } + .map(drop) + } + + fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { + SequenceIndex::Int(i) => self.borrow_vec_mut().delitem_by_index(vm, i), + SequenceIndex::Slice(slice) => self.borrow_vec_mut().delitem_by_slice(vm, slice), + } + } + + fn __delitem__(&self, subscript: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self._delitem(&subscript, vm) + } + + #[pymethod] + pub(crate) fn sort(&self, options: SortOptions, vm: &VirtualMachine) -> PyResult<()> { + // replace list contents with [] for duration of sort. + // this prevents keyfunc from messing with the list and makes it easy to + // check if it tries to append elements to it. + let mut elements = core::mem::take(self.borrow_vec_mut().deref_mut()); + let res = do_sort(vm, &mut elements, options.key, options.reverse); + core::mem::swap(self.borrow_vec_mut().deref_mut(), &mut elements); + res?; + + if !elements.is_empty() { + return Err(vm.new_value_error("list modified during sort")); + } + + Ok(()) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +fn extract_cloned<F, R>(obj: &PyObject, mut f: F, vm: &VirtualMachine) -> PyResult<Vec<R>> +where + F: FnMut(PyObjectRef) -> PyResult<R>, +{ + use crate::builtins::PyTuple; + if let Some(tuple) = obj.downcast_ref_if_exact::<PyTuple>(vm) { + tuple.iter().map(|x| f(x.clone())).collect() + } else if let Some(list) = obj.downcast_ref_if_exact::<PyList>(vm) { + list.borrow_vec().iter().map(|x| f(x.clone())).collect() + } else { + let iter = obj.to_owned().get_iter(vm)?; + let iter = iter.iter::<PyObjectRef>(vm)?; + let len = obj + .sequence_unchecked() + .length_opt(vm) + .transpose()? + .unwrap_or(0); + let mut v = Vec::with_capacity(len); + for x in iter { + v.push(f(x?)?); + } + v.shrink_to_fit(); + Ok(v) + } +} + +impl MutObjectSequenceOp for PyList { + type Inner = [PyObjectRef]; + + fn do_get(index: usize, inner: &[PyObjectRef]) -> Option<&PyObject> { + inner.get(index).map(|r| r.as_ref()) + } + + fn do_lock(&self) -> impl core::ops::Deref<Target = [PyObjectRef]> { + self.borrow_vec() + } +} + +impl Constructor for PyList { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: FuncArgs, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self::default()) + } +} + +impl Initializer for PyList { + type Args = OptionalArg<PyObjectRef>; + + fn init(zelf: PyRef<Self>, iterable: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let mut elements = if let OptionalArg::Present(iterable) = iterable { + iterable.try_to_value(vm)? + } else { + vec![] + }; + core::mem::swap(zelf.borrow_vec_mut().deref_mut(), &mut elements); + Ok(()) + } +} + +impl AsMapping for PyList { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: PyMappingMethods = PyMappingMethods { + length: atomic_func!(|mapping, _vm| Ok(PyList::mapping_downcast(mapping).__len__())), + subscript: atomic_func!( + |mapping, needle, vm| PyList::mapping_downcast(mapping)._getitem(needle, vm) + ), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyList::mapping_downcast(mapping); + if let Some(value) = value { + zelf._setitem(needle, value, vm) + } else { + zelf._delitem(needle, vm) + } + }), + }; + &AS_MAPPING + } +} + +impl AsSequence for PyList { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyList::sequence_downcast(seq).__len__())), + concat: atomic_func!(|seq, other, vm| { + PyList::sequence_downcast(seq) + .concat(other, vm) + .map(|x| x.into()) + }), + repeat: atomic_func!(|seq, n, vm| { + PyList::sequence_downcast(seq) + .repeat(n, vm) + .map(|x| x.into()) + }), + item: atomic_func!(|seq, i, vm| { + let list = PyList::sequence_downcast(seq); + let vec = list.borrow_vec(); + let pos = vec + .wrap_index(i) + .ok_or_else(|| vm.new_index_error("list index out of range"))?; + Ok(vec.do_get(pos)) + }), + ass_item: atomic_func!(|seq, i, value, vm| { + let zelf = PyList::sequence_downcast(seq); + if let Some(value) = value { + zelf.borrow_vec_mut().setitem_by_index(vm, i, value) + } else { + zelf.borrow_vec_mut().delitem_by_index(vm, i) + } + .map_err(|e| { + if e.class().is(vm.ctx.exceptions.index_error) { + vm.new_index_error("list assignment index out of range".to_owned()) + } else { + e + } + }) + }), + contains: atomic_func!(|seq, target, vm| { + let zelf = PyList::sequence_downcast(seq); + zelf.mut_contains(vm, target) + }), + inplace_concat: atomic_func!(|seq, other, vm| { + let zelf = PyList::sequence_downcast(seq); + PyList::inplace_concat(zelf, other, vm) + }), + inplace_repeat: atomic_func!(|seq, n, vm| { + let zelf = PyList::sequence_downcast(seq); + Ok(PyList::irepeat(zelf.to_owned(), n, vm)?.into()) + }), + }; + &AS_SEQUENCE + } +} + +impl Iterable for PyList { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyListIterator { + internal: PyMutex::new(PositionIterInternal::new(zelf, 0)), + } + .into_pyobject(vm)) + } +} + +impl Comparable for PyList { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + if let Some(res) = op.identical_optimization(zelf, other) { + return Ok(res.into()); + } + let other = class_or_notimplemented!(Self, other); + let a = &*zelf.borrow_vec(); + let b = &*other.borrow_vec(); + a.iter() + .richcompare(b.iter(), op, vm) + .map(PyComparisonValue::Implemented) + } +} + +impl Representable for PyList { + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + if zelf.__len__() == 0 { + return Ok(vm.ctx.intern_str("[]").to_owned()); + } + + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + // Clone elements before calling repr to release the read lock. + // Element repr may mutate the list (e.g., list.clear()), which + // needs a write lock and would deadlock if read lock is held. + let mut writer = Wtf8Buf::new(); + writer.push_char('['); + + let mut elements = zelf.borrow_vec().to_vec(); + let mut size = zelf.__len__(); + let mut first = true; + let mut i = 0; + while i < size { + if elements.len() != size { + // `repr` mutated the list. refetch it. + elements = zelf.borrow_vec().to_vec(); + } + + let item = &elements[i]; + + if first { + first = false; + } else { + writer.push_str(", "); + } + + writer.push_wtf8(item.repr(vm)?.as_wtf8()); + + size = zelf.__len__(); // Refetch list size as `repr` may mutate the list. + i += 1; + } + + writer.push_char(']'); + Ok(vm.ctx.new_str(writer)) + } else { + Ok(vm.ctx.intern_str("[...]").to_owned()) + } + } + + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("repr() is overridden directly") + } +} + +fn do_sort( + vm: &VirtualMachine, + values: &mut Vec<PyObjectRef>, + key_func: Option<PyObjectRef>, + reverse: bool, +) -> PyResult<()> { + // CPython uses __lt__ for all comparisons in sort. + // try_sort_by_gt expects is_gt(a, b) = true when a should come AFTER b. + let cmp = |a: &PyObjectRef, b: &PyObjectRef| { + if reverse { + // Descending: a comes after b when a < b + a.rich_compare_bool(b, PyComparisonOp::Lt, vm) + } else { + // Ascending: a comes after b when b < a + b.rich_compare_bool(a, PyComparisonOp::Lt, vm) + } + }; + + if let Some(ref key_func) = key_func { + let mut items = values + .iter() + .map(|x| Ok((x.clone(), key_func.call((x.clone(),), vm)?))) + .collect::<Result<Vec<_>, _>>()?; + timsort::try_sort_by_gt(&mut items, |a, b| cmp(&a.1, &b.1))?; + *values = items.into_iter().map(|(val, _)| val).collect(); + } else { + timsort::try_sort_by_gt(values, cmp)?; + } + + Ok(()) +} + +#[pyclass(module = false, name = "list_iterator", traverse)] +#[derive(Debug)] +pub struct PyListIterator { + internal: PyMutex<PositionIterInternal<PyListRef>>, +} + +impl PyPayload for PyListIterator { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.list_iterator_type + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] +impl PyListIterator { + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().length_hint(|obj| obj.__len__()) + } + + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.internal + .lock() + .set_state(state, |obj, pos| pos.min(obj.__len__()), vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.new_list(Vec::new()).into(), + vm, + ) + } +} + +impl PyListIterator { + /// Fast path for FOR_ITER specialization. + pub(crate) fn fast_next(&self) -> Option<PyObjectRef> { + self.internal + .lock() + .next(|list, pos| { + let vec = list.borrow_vec(); + Ok(PyIterReturn::from_result(vec.get(pos).cloned().ok_or(None))) + }) + .ok() + .and_then(|r| match r { + PyIterReturn::Return(v) => Some(v), + PyIterReturn::StopIteration(_) => None, + }) + } +} + +impl SelfIter for PyListIterator {} +impl IterNext for PyListIterator { + fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.internal.lock().next(|list, pos| { + let vec = list.borrow_vec(); + Ok(PyIterReturn::from_result(vec.get(pos).cloned().ok_or(None))) + }) + } +} + +#[pyclass(module = false, name = "list_reverseiterator", traverse)] +#[derive(Debug)] +pub struct PyListReverseIterator { + internal: PyMutex<PositionIterInternal<PyListRef>>, +} + +impl PyPayload for PyListReverseIterator { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.list_reverseiterator_type + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] +impl PyListReverseIterator { + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().rev_length_hint(|obj| obj.__len__()) + } + + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.internal + .lock() + .set_state(state, |obj, pos| pos.min(obj.__len__()), vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_reversed(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.new_list(Vec::new()).into(), + vm, + ) + } +} + +impl SelfIter for PyListReverseIterator {} +impl IterNext for PyListReverseIterator { + fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.internal.lock().rev_next(|list, pos| { + let vec = list.borrow_vec(); + Ok(PyIterReturn::from_result(vec.get(pos).cloned().ok_or(None))) + }) + } +} + +pub fn init(context: &'static Context) { + let list_type = &context.types.list_type; + PyList::extend_class(context, list_type); + + PyListIterator::extend_class(context, context.types.list_iterator_type); + PyListReverseIterator::extend_class(context, context.types.list_reverseiterator_type); +} diff --git a/crates/vm/src/builtins/map.rs b/crates/vm/src/builtins/map.rs new file mode 100644 index 00000000000..663c1bc94e0 --- /dev/null +++ b/crates/vm/src/builtins/map.rs @@ -0,0 +1,129 @@ +use super::PyType; +use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::PyTupleRef, + class::PyClassImpl, + function::{ArgIntoBool, OptionalArg, PosArgs}, + protocol::{PyIter, PyIterReturn}, + types::{Constructor, IterNext, Iterable, SelfIter}, +}; +use rustpython_common::atomic::{self, PyAtomic, Radium}; + +#[pyclass(module = false, name = "map", traverse)] +#[derive(Debug)] +pub struct PyMap { + mapper: PyObjectRef, + iterators: Vec<PyIter>, + #[pytraverse(skip)] + strict: PyAtomic<bool>, +} + +impl PyPayload for PyMap { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.map_type + } +} + +#[derive(FromArgs)] +pub struct PyMapNewArgs { + #[pyarg(named, optional)] + strict: OptionalArg<bool>, +} + +impl Constructor for PyMap { + type Args = (PyObjectRef, PosArgs<PyIter>, PyMapNewArgs); + + fn py_new( + _cls: &Py<PyType>, + (mapper, iterators, args): Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + let iterators = iterators.into_vec(); + let strict = Radium::new(args.strict.unwrap_or(false)); + Ok(Self { + mapper, + iterators, + strict, + }) + } +} + +#[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] +impl PyMap { + #[pymethod] + fn __length_hint__(&self, vm: &VirtualMachine) -> PyResult<usize> { + self.iterators.iter().try_fold(0, |prev, cur| { + let cur = cur.as_ref().to_owned().length_hint(0, vm)?; + let max = core::cmp::max(prev, cur); + Ok(max) + }) + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let cls = zelf.class().to_owned(); + let mut vec = vec![zelf.mapper.clone()]; + vec.extend(zelf.iterators.iter().map(|o| o.clone().into())); + let tuple_args = vm.ctx.new_tuple(vec); + Ok(if zelf.strict.load(atomic::Ordering::Acquire) { + vm.new_tuple((cls, tuple_args, true)) + } else { + vm.new_tuple((cls, tuple_args)) + }) + } + + #[pymethod] + fn __setstate__(zelf: PyRef<Self>, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if let Ok(obj) = ArgIntoBool::try_from_object(vm, state) { + zelf.strict.store(obj.into(), atomic::Ordering::Release); + } + Ok(()) + } +} + +impl SelfIter for PyMap {} + +impl IterNext for PyMap { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let mut next_objs = Vec::new(); + for (idx, iterator) in zelf.iterators.iter().enumerate() { + let item = match iterator.next(vm)? { + PyIterReturn::Return(obj) => obj, + PyIterReturn::StopIteration(v) => { + if zelf.strict.load(atomic::Ordering::Acquire) { + if idx > 0 { + let plural = if idx == 1 { " " } else { "s 1-" }; + return Err(vm.new_value_error(format!( + "map() argument {} is shorter than argument{}{}", + idx + 1, + plural, + idx, + ))); + } + for (idx, iterator) in zelf.iterators[1..].iter().enumerate() { + if let PyIterReturn::Return(_) = iterator.next(vm)? { + let plural = if idx == 0 { " " } else { "s 1-" }; + return Err(vm.new_value_error(format!( + "map() argument {} is longer than argument{}{}", + idx + 2, + plural, + idx + 1, + ))); + } + } + } + return Ok(PyIterReturn::StopIteration(v)); + } + }; + next_objs.push(item); + } + + // the mapper itself can raise StopIteration which does stop the map iteration + PyIterReturn::from_pyresult(zelf.mapper.call(next_objs, vm), vm) + } +} + +pub fn init(context: &'static Context) { + PyMap::extend_class(context, context.types.map_type); +} diff --git a/crates/vm/src/builtins/mappingproxy.rs b/crates/vm/src/builtins/mappingproxy.rs new file mode 100644 index 00000000000..df9c61e0a58 --- /dev/null +++ b/crates/vm/src/builtins/mappingproxy.rs @@ -0,0 +1,298 @@ +use super::{PyDict, PyDictRef, PyGenericAlias, PyList, PyTuple, PyType, PyTypeRef}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, + class::PyClassImpl, + common::{hash, lock::LazyLock}, + convert::ToPyObject, + function::{ArgMapping, OptionalArg, PyComparisonValue}, + object::{Traverse, TraverseFn}, + protocol::{PyMappingMethods, PyNumberMethods, PySequenceMethods}, + types::{ + AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, Iterable, + PyComparisonOp, Representable, + }, +}; +use rustpython_common::wtf8::{Wtf8Buf, wtf8_concat}; + +#[pyclass(module = false, name = "mappingproxy", traverse)] +#[derive(Debug)] +pub struct PyMappingProxy { + mapping: MappingProxyInner, +} + +#[derive(Debug)] +enum MappingProxyInner { + Class(PyTypeRef), + Mapping(ArgMapping), +} + +unsafe impl Traverse for MappingProxyInner { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + match self { + Self::Class(r) => r.traverse(tracer_fn), + Self::Mapping(arg) => arg.traverse(tracer_fn), + } + } +} + +impl PyPayload for PyMappingProxy { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.mappingproxy_type + } +} + +impl From<PyTypeRef> for PyMappingProxy { + fn from(dict: PyTypeRef) -> Self { + Self { + mapping: MappingProxyInner::Class(dict), + } + } +} + +impl From<PyDictRef> for PyMappingProxy { + fn from(dict: PyDictRef) -> Self { + Self { + mapping: MappingProxyInner::Mapping(ArgMapping::from_dict_exact(dict)), + } + } +} + +impl Constructor for PyMappingProxy { + type Args = PyObjectRef; + + fn py_new(_cls: &Py<PyType>, mapping: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if mapping.mapping_unchecked().check() + && !mapping.downcastable::<PyList>() + && !mapping.downcastable::<PyTuple>() + { + return Ok(Self { + mapping: MappingProxyInner::Mapping(ArgMapping::new(mapping)), + }); + } + Err(vm.new_type_error(format!( + "mappingproxy() argument must be a mapping, not {}", + mapping.class() + ))) + } +} + +#[pyclass(with( + AsMapping, + Iterable, + Constructor, + AsSequence, + Comparable, + Hashable, + AsNumber, + Representable +))] +impl PyMappingProxy { + fn get_inner(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + match &self.mapping { + MappingProxyInner::Class(class) => Ok(key + .as_interned_str(vm) + .and_then(|key| class.attributes.read().get(key).cloned())), + MappingProxyInner::Mapping(mapping) => mapping.mapping().subscript(&*key, vm).map(Some), + } + } + + #[pymethod] + fn get( + &self, + key: PyObjectRef, + default: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + let obj = self.to_object(vm)?; + Ok(Some(vm.call_method( + &obj, + "get", + (key, default.unwrap_or_none(vm)), + )?)) + } + + pub fn __getitem__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.get_inner(key.clone(), vm)? + .ok_or_else(|| vm.new_key_error(key)) + } + + fn _contains(&self, key: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + match &self.mapping { + MappingProxyInner::Class(class) => Ok(key + .as_interned_str(vm) + .is_some_and(|key| class.attributes.read().contains_key(key))), + MappingProxyInner::Mapping(mapping) => { + mapping.obj().sequence_unchecked().contains(key, vm) + } + } + } + + pub fn __contains__(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + self._contains(&key, vm) + } + + fn to_object(&self, vm: &VirtualMachine) -> PyResult { + Ok(match &self.mapping { + MappingProxyInner::Mapping(d) => d.as_ref().to_owned(), + MappingProxyInner::Class(c) => { + PyDict::from_attributes(c.attributes.read().clone(), vm)?.to_pyobject(vm) + } + }) + } + + #[pymethod] + pub fn items(&self, vm: &VirtualMachine) -> PyResult { + let obj = self.to_object(vm)?; + vm.call_method(&obj, identifier!(vm, items).as_str(), ()) + } + + #[pymethod] + pub fn keys(&self, vm: &VirtualMachine) -> PyResult { + let obj = self.to_object(vm)?; + vm.call_method(&obj, identifier!(vm, keys).as_str(), ()) + } + + #[pymethod] + pub fn values(&self, vm: &VirtualMachine) -> PyResult { + let obj = self.to_object(vm)?; + vm.call_method(&obj, identifier!(vm, values).as_str(), ()) + } + + #[pymethod] + pub fn copy(&self, vm: &VirtualMachine) -> PyResult { + match &self.mapping { + MappingProxyInner::Mapping(d) => { + vm.call_method(d.obj(), identifier!(vm, copy).as_str(), ()) + } + MappingProxyInner::Class(c) => { + Ok(PyDict::from_attributes(c.attributes.read().clone(), vm)?.to_pyobject(vm)) + } + } + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + fn __len__(&self, vm: &VirtualMachine) -> PyResult<usize> { + let obj = self.to_object(vm)?; + obj.length(vm) + } + + #[pymethod] + fn __reversed__(&self, vm: &VirtualMachine) -> PyResult { + vm.call_method( + self.to_object(vm)?.as_object(), + identifier!(vm, __reversed__).as_str(), + (), + ) + } + + fn __ior__(&self, _args: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!( + r#""'|=' is not supported by {}; use '|' instead""#, + Self::class(&vm.ctx) + ))) + } + + fn __or__(&self, args: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._or(self.copy(vm)?.as_ref(), args.as_ref()) + } +} + +impl Comparable for PyMappingProxy { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let obj = zelf.to_object(vm)?; + Ok(PyComparisonValue::Implemented( + obj.rich_compare_bool(other, op, vm)?, + )) + } +} + +impl Hashable for PyMappingProxy { + #[inline] + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { + // Delegate hash to the underlying mapping + let obj = zelf.to_object(vm)?; + obj.hash(vm) + } +} + +impl AsMapping for PyMappingProxy { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + length: atomic_func!( + |mapping, vm| PyMappingProxy::mapping_downcast(mapping).__len__(vm) + ), + subscript: atomic_func!(|mapping, needle, vm| { + PyMappingProxy::mapping_downcast(mapping).__getitem__(needle.to_owned(), vm) + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } +} + +impl AsSequence for PyMappingProxy { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, vm| PyMappingProxy::sequence_downcast(seq).__len__(vm)), + contains: atomic_func!( + |seq, target, vm| PyMappingProxy::sequence_downcast(seq)._contains(target, vm) + ), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsNumber for PyMappingProxy { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PyMappingProxy>() { + a.__or__(b.to_pyobject(vm), vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + inplace_or: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PyMappingProxy>() { + a.__ior__(b.to_pyobject(vm), vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Iterable for PyMappingProxy { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let obj = zelf.to_object(vm)?; + let iter = obj.get_iter(vm)?; + Ok(iter.into()) + } +} + +impl Representable for PyMappingProxy { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let obj = zelf.to_object(vm)?; + Ok(wtf8_concat!("mappingproxy(", obj.repr(vm)?.as_wtf8(), ')')) + } +} + +pub fn init(context: &'static Context) { + PyMappingProxy::extend_class(context, context.types.mappingproxy_type) +} diff --git a/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs similarity index 80% rename from vm/src/builtins/memory.rs rename to crates/vm/src/builtins/memory.rs index 2c436ca316f..73eb1f1780b 100644 --- a/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -1,11 +1,13 @@ use super::{ - PositionIterInternal, PyBytes, PyBytesRef, PyInt, PyListRef, PySlice, PyStr, PyStrRef, PyTuple, - PyTupleRef, PyType, PyTypeRef, + PositionIterInternal, PyBytes, PyBytesRef, PyGenericAlias, PyInt, PyListRef, PySlice, PyStr, + PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, PyUtf8StrRef, iter::builtins_iter, }; +use crate::common::lock::LazyLock; use crate::{ - atomic_func, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, TryFromObject, VirtualMachine, atomic_func, buffer::FormatSpec, - bytesinner::bytes_to_hex, + bytes_inner::bytes_to_hex, class::PyClassImpl, common::{ borrow::{BorrowedValue, BorrowedValueMut}, @@ -22,16 +24,13 @@ use crate::{ sliceable::SequenceIndexOp, types::{ AsBuffer, AsMapping, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, + PyComparisonOp, Representable, SelfIter, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, VirtualMachine, }; +use core::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range}; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; -use once_cell::sync::Lazy; use rustpython_common::lock::PyMutex; -use std::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range}; #[derive(FromArgs)] pub struct PyMemoryViewNewArgs { @@ -44,7 +43,7 @@ pub struct PyMemoryView { // avoid double release when memoryview had released the buffer before drop buffer: ManuallyDrop<PyBuffer>, // the released memoryview does not mean the buffer is destroyed - // because the possible another memeoryview is viewing from it + // because the possible another memoryview is viewing from it released: AtomicCell<bool>, // start does NOT mean the bytes before start will not be visited, // it means the point we starting to get the absolute position via @@ -62,9 +61,8 @@ pub struct PyMemoryView { impl Constructor for PyMemoryView { type Args = PyMemoryViewNewArgs; - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let zelf = Self::from_object(&args.object, vm)?; - zelf.into_ref_with_type(vm, cls).map(Into::into) + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Self::from_object(&args.object, vm) } } @@ -76,11 +74,11 @@ impl PyMemoryView { /// this should be the main entrance to create the memoryview /// to avoid the chained memoryview pub fn from_object(obj: &PyObject, vm: &VirtualMachine) -> PyResult<Self> { - if let Some(other) = obj.payload::<Self>() { + if let Some(other) = obj.downcast_ref::<Self>() { Ok(other.new_view()) } else { let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; - PyMemoryView::from_buffer(buffer, vm) + Self::from_buffer(buffer, vm) } } @@ -94,7 +92,7 @@ impl PyMemoryView { let format_spec = Self::parse_format(&buffer.desc.format, vm)?; let desc = buffer.desc.clone(); - Ok(PyMemoryView { + Ok(Self { buffer: ManuallyDrop::new(buffer), released: AtomicCell::new(false), start: 0, @@ -104,7 +102,7 @@ impl PyMemoryView { }) } - /// don't use this function to create the memeoryview if the buffer is exporting + /// don't use this function to create the memoryview if the buffer is exporting /// via another memoryview, use PyMemoryView::new_view() or PyMemoryView::from_object /// to reduce the chain pub fn from_buffer_range( @@ -121,7 +119,7 @@ impl PyMemoryView { /// this should be the only way to create a memoryview from another memoryview pub fn new_view(&self) -> Self { - let zelf = PyMemoryView { + let zelf = Self { buffer: self.buffer.clone(), released: AtomicCell::new(false), start: self.start, @@ -135,7 +133,7 @@ impl PyMemoryView { fn try_not_released(&self, vm: &VirtualMachine) -> PyResult<()> { if self.released.load() { - Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned())) + Err(vm.new_value_error("operation forbidden on released memoryview object")) } else { Ok(()) } @@ -143,14 +141,14 @@ impl PyMemoryView { fn getitem_by_idx(&self, i: isize, vm: &VirtualMachine) -> PyResult { if self.desc.ndim() != 1 { - return Err(vm.new_not_implemented_error( - "multi-dimensional sub-views are not implemented".to_owned(), - )); + return Err( + vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") + ); } let (shape, stride, suboffset) = self.desc.dim_desc[0]; let index = i .wrapped_at(shape) - .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; + .ok_or_else(|| vm.new_index_error("index out of range"))?; let index = index as isize * stride + suboffset; let pos = (index + self.start as isize) as usize; self.unpack_single(pos, vm) @@ -172,12 +170,12 @@ impl PyMemoryView { fn setitem_by_idx(&self, i: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { if self.desc.ndim() != 1 { - return Err(vm.new_not_implemented_error("sub-views are not implemented".to_owned())); + return Err(vm.new_not_implemented_error("sub-views are not implemented")); } let (shape, stride, suboffset) = self.desc.dim_desc[0]; let index = i .wrapped_at(shape) - .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; + .ok_or_else(|| vm.new_index_error("index out of range"))?; let index = index as isize * stride + suboffset; let pos = (index + self.start as isize) as usize; self.pack_single(pos, value, vm) @@ -213,7 +211,7 @@ impl PyMemoryView { .unpack(&bytes[pos..pos + self.desc.itemsize], vm) .map(|x| { if x.len() == 1 { - x.fast_getitem(0) + x[0].to_owned() } else { x.into() } @@ -223,14 +221,14 @@ impl PyMemoryView { fn pos_from_multi_index(&self, indexes: &[isize], vm: &VirtualMachine) -> PyResult<usize> { match indexes.len().cmp(&self.desc.ndim()) { Ordering::Less => { - return Err(vm.new_not_implemented_error("sub-views are not implemented".to_owned())) + return Err(vm.new_not_implemented_error("sub-views are not implemented")); } Ordering::Greater => { return Err(vm.new_type_error(format!( "cannot index {}-dimension view with {}-element tuple", self.desc.ndim(), indexes.len() - ))) + ))); } Ordering::Equal => (), } @@ -261,8 +259,8 @@ impl PyMemoryView { // no suboffset set, stride must be positive self.start += stride as usize * range.start; } - let newlen = range.len(); - self.desc.dim_desc[dim].0 = newlen; + let new_len = range.len(); + self.desc.dim_desc[dim].0 = new_len; } fn init_slice(&mut self, slice: &PySlice, dim: usize, vm: &VirtualMachine) -> PyResult<()> { @@ -331,10 +329,10 @@ impl PyMemoryView { return Ok(false); } - if let Some(other) = other.payload::<Self>() { - if other.released.load() { - return Ok(false); - } + if let Some(other) = other.downcast_ref::<Self>() + && other.released.load() + { + return Ok(false); } let other = match PyBuffer::try_from_borrowed_object(vm, other) { @@ -380,16 +378,12 @@ impl PyMemoryView { } }; ret = vm.bool_eq(&a_val, &b_val); - if let Ok(b) = ret { - !b - } else { - true - } + if let Ok(b) = ret { !b } else { true } }); ret } - fn obj_bytes(&self) -> BorrowedValue<[u8]> { + fn obj_bytes(&self) -> BorrowedValue<'_, [u8]> { if self.desc.is_contiguous() { BorrowedValue::map(self.buffer.obj_bytes(), |x| { &x[self.start..self.start + self.desc.len] @@ -399,7 +393,7 @@ impl PyMemoryView { } } - fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + fn obj_bytes_mut(&self) -> BorrowedValueMut<'_, [u8]> { if self.desc.is_contiguous() { BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { &mut x[self.start..self.start + self.desc.len] @@ -409,7 +403,7 @@ impl PyMemoryView { } } - fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> { + fn as_contiguous(&self) -> Option<BorrowedValue<'_, [u8]>> { self.desc.is_contiguous().then(|| { BorrowedValue::map(self.buffer.obj_bytes(), |x| { &x[self.start..self.start + self.desc.len] @@ -417,7 +411,7 @@ impl PyMemoryView { }) } - fn _as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> { + fn _as_contiguous_mut(&self) -> Option<BorrowedValueMut<'_, [u8]>> { self.desc.is_contiguous().then(|| { BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { &mut x[self.start..self.start + self.desc.len] @@ -495,7 +489,7 @@ impl Py<PyMemoryView> { vm: &VirtualMachine, ) -> PyResult<()> { if self.desc.ndim() != 1 { - return Err(vm.new_not_implemented_error("sub-view are not implemented".to_owned())); + return Err(vm.new_not_implemented_error("sub-view are not implemented")); } let mut dest = self.new_view(); @@ -505,7 +499,7 @@ impl Py<PyMemoryView> { if self.is(&src) { return if !is_equiv_structure(&self.desc, &dest.desc) { Err(vm.new_value_error( - "memoryview assignment: lvalue and rvalue have different structures".to_owned(), + "memoryview assignment: lvalue and rvalue have different structures", )) } else { // assign self[:] to self @@ -525,7 +519,7 @@ impl Py<PyMemoryView> { if !is_equiv_structure(&src.desc, &dest.desc) { return Err(vm.new_value_error( - "memoryview assignment: lvalue and rvalue have different structures".to_owned(), + "memoryview assignment: lvalue and rvalue have different structures", )); } @@ -543,18 +537,26 @@ impl Py<PyMemoryView> { } } -#[pyclass(with( - Py, - Hashable, - Comparable, - AsBuffer, - AsMapping, - AsSequence, - Constructor, - Iterable, - Representable -))] +#[pyclass( + with( + Py, + Hashable, + Comparable, + AsBuffer, + AsMapping, + AsSequence, + Constructor, + Iterable, + Representable + ), + flags(SEQUENCE, HAS_WEAKREF) +)] impl PyMemoryView { + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + #[pymethod] pub fn release(&self) { if self.released.compare_exchange(false, true).is_ok() { @@ -614,13 +616,22 @@ impl PyMemoryView { #[pygetset] fn suboffsets(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { self.try_not_released(vm)?; - Ok(vm.ctx.new_tuple( - self.desc - .dim_desc - .iter() - .map(|(_, _, suboffset)| suboffset.to_pyobject(vm)) - .collect(), - )) + let has_suboffsets = self + .desc + .dim_desc + .iter() + .any(|(_, _, suboffset)| *suboffset != 0); + if has_suboffsets { + Ok(vm.ctx.new_tuple( + self.desc + .dim_desc + .iter() + .map(|(_, _, suboffset)| suboffset.to_pyobject(vm)) + .collect(), + )) + } else { + Ok(vm.ctx.empty_tuple.clone()) + } } #[pygetset] @@ -641,35 +652,34 @@ impl PyMemoryView { #[pygetset] fn f_contiguous(&self, vm: &VirtualMachine) -> PyResult<bool> { - // TODO: fortain order + // TODO: column-major order self.try_not_released(vm) .map(|_| self.desc.ndim() <= 1 && self.desc.is_contiguous()) } - #[pymethod(magic)] - fn enter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + #[pymethod] + fn __enter__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { zelf.try_not_released(vm).map(|_| zelf) } - #[pymethod(magic)] - fn exit(&self, _args: FuncArgs) { + #[pymethod] + fn __exit__(&self, _args: FuncArgs) { self.release(); } - #[pymethod(magic)] - fn getitem(zelf: PyRef<Self>, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn __getitem__(zelf: PyRef<Self>, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { zelf.try_not_released(vm)?; if zelf.desc.ndim() == 0 { // 0-d memoryview can be referenced using mv[...] or mv[()] only if needle.is(&vm.ctx.ellipsis) { return Ok(zelf.into()); } - if let Some(tuple) = needle.payload::<PyTuple>() { - if tuple.is_empty() { - return zelf.unpack_single(0, vm); - } + if let Some(tuple) = needle.downcast_ref::<PyTuple>() + && tuple.is_empty() + { + return zelf.unpack_single(0, vm); } - return Err(vm.new_type_error("invalid indexing of 0-dim memory".to_owned())); + return Err(vm.new_type_error("invalid indexing of 0-dim memory")); } match SubscriptNeedle::try_from_object(vm, needle)? { @@ -679,23 +689,22 @@ impl PyMemoryView { } } - #[pymethod(magic)] - fn delitem(&self, _needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn __delitem__(&self, _needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { if self.desc.readonly { - return Err(vm.new_type_error("cannot modify read-only memory".to_owned())); + return Err(vm.new_type_error("cannot modify read-only memory")); } - Err(vm.new_type_error("cannot delete memory".to_owned())) + Err(vm.new_type_error("cannot delete memory")) } - #[pymethod(magic)] - fn len(&self, vm: &VirtualMachine) -> PyResult<usize> { + fn __len__(&self, vm: &VirtualMachine) -> PyResult<usize> { self.try_not_released(vm)?; - Ok(if self.desc.ndim() == 0 { - 1 + if self.desc.ndim() == 0 { + // 0-dimensional memoryview has no length + Err(vm.new_type_error("0-dim memory has no length")) } else { // shape for dim[0] - self.desc.dim_desc[0].0 - }) + Ok(self.desc.dim_desc[0].0) + } } #[pymethod] @@ -739,14 +748,70 @@ impl PyMemoryView { self.contiguous_or_collect(|x| bytes_to_hex(x, sep, bytes_per_sep, vm)) } - fn cast_to_1d(&self, format: PyStrRef, vm: &VirtualMachine) -> PyResult<Self> { - let format_spec = Self::parse_format(format.as_str(), vm)?; - let itemsize = format_spec.size(); - if self.desc.len % itemsize != 0 { + #[pymethod] + fn count(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + self.try_not_released(vm)?; + if self.desc.ndim() != 1 { return Err( - vm.new_type_error("memoryview: length is not a multiple of itemsize".to_owned()) + vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") ); } + let len = self.desc.dim_desc[0].0; + let mut count = 0; + for i in 0..len { + let item = self.getitem_by_idx(i as isize, vm)?; + if vm.bool_eq(&item, &value)? { + count += 1; + } + } + Ok(count) + } + + #[pymethod] + fn index( + &self, + value: PyObjectRef, + start: OptionalArg<isize>, + stop: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult<usize> { + self.try_not_released(vm)?; + if self.desc.ndim() != 1 { + return Err( + vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") + ); + } + let len = self.desc.dim_desc[0].0; + let start = start.unwrap_or(0); + let stop = stop.unwrap_or(len as isize); + + let start = if start < 0 { + (start + len as isize).max(0) as usize + } else { + (start as usize).min(len) + }; + let stop = if stop < 0 { + (stop + len as isize).max(0) as usize + } else { + (stop as usize).min(len) + }; + + for i in start..stop { + let item = self.getitem_by_idx(i as isize, vm)?; + if vm.bool_eq(&item, &value)? { + return Ok(i); + } + } + Err(vm.new_value_error("memoryview.index(x): x not in memoryview")) + } + + fn cast_to_1d(&self, format: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Self> { + let format_str = format.as_str(); + let format_spec = Self::parse_format(format_str, vm)?; + let itemsize = format_spec.size(); + if !self.desc.len.is_multiple_of(itemsize) { + return Err(vm.new_type_error("memoryview: length is not a multiple of itemsize")); + } Ok(Self { buffer: self.buffer.clone(), @@ -757,7 +822,7 @@ impl PyMemoryView { len: self.desc.len, readonly: self.desc.readonly, itemsize, - format: format.to_string().into(), + format: format_str.to_owned().into(), dim_desc: vec![(self.desc.len / itemsize, itemsize as isize, 0)], }, hash: OnceCell::new(), @@ -768,9 +833,7 @@ impl PyMemoryView { fn cast(&self, args: CastArgs, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { self.try_not_released(vm)?; if !self.desc.is_contiguous() { - return Err(vm.new_type_error( - "memoryview: casts are restricted to C-contiguous views".to_owned(), - )); + return Err(vm.new_type_error("memoryview: casts are restricted to C-contiguous views")); } let CastArgs { format, shape } = args; @@ -778,7 +841,7 @@ impl PyMemoryView { if let OptionalArg::Present(shape) = shape { if self.desc.is_zero_in_shape() { return Err(vm.new_type_error( - "memoryview: cannot cast view with zeros in shape or strides".to_owned(), + "memoryview: cannot cast view with zeros in shape or strides", )); } @@ -800,9 +863,7 @@ impl PyMemoryView { let shape_ndim = shape.len(); // TODO: MAX_NDIM if self.desc.ndim() != 1 && shape_ndim != 1 { - return Err( - vm.new_type_error("memoryview: cast must be 1D -> ND or ND -> 1D".to_owned()) - ); + return Err(vm.new_type_error("memoryview: cast must be 1D -> ND or ND -> 1D")); } let mut other = self.cast_to_1d(format, vm)?; @@ -822,9 +883,7 @@ impl PyMemoryView { let x = usize::try_from_borrowed_object(vm, x)?; if x > isize::MAX as usize / product_shape { - return Err(vm.new_value_error( - "memoryview.cast(): product(shape) > SSIZE_MAX".to_owned(), - )); + return Err(vm.new_value_error("memoryview.cast(): product(shape) > SSIZE_MAX")); } product_shape *= x; dim_descriptor.push((x, 0, 0)); @@ -836,9 +895,9 @@ impl PyMemoryView { } if product_shape != other.desc.len { - return Err(vm.new_type_error( - "memoryview: product(shape) * itemsize != buffer size".to_owned(), - )); + return Err( + vm.new_type_error("memoryview: product(shape) * itemsize != buffer size") + ); } other.desc.dim_desc = dim_descriptor; @@ -852,8 +911,7 @@ impl PyMemoryView { #[pyclass] impl Py<PyMemoryView> { - #[pymethod(magic)] - fn setitem( + fn __setitem__( &self, needle: PyObjectRef, value: PyObjectRef, @@ -861,22 +919,22 @@ impl Py<PyMemoryView> { ) -> PyResult<()> { self.try_not_released(vm)?; if self.desc.readonly { - return Err(vm.new_type_error("cannot modify read-only memory".to_owned())); + return Err(vm.new_type_error("cannot modify read-only memory")); } if value.is(&vm.ctx.none) { - return Err(vm.new_type_error("cannot delete memory".to_owned())); + return Err(vm.new_type_error("cannot delete memory")); } if self.desc.ndim() == 0 { // TODO: merge branches when we got conditional if let if needle.is(&vm.ctx.ellipsis) { return self.pack_single(0, value, vm); - } else if let Some(tuple) = needle.payload::<PyTuple>() { - if tuple.is_empty() { - return self.pack_single(0, value, vm); - } + } else if let Some(tuple) = needle.downcast_ref::<PyTuple>() + && tuple.is_empty() + { + return self.pack_single(0, value, vm); } - return Err(vm.new_type_error("invalid indexing of 0-dim memory".to_owned())); + return Err(vm.new_type_error("invalid indexing of 0-dim memory")); } match SubscriptNeedle::try_from_object(vm, needle)? { SubscriptNeedle::Index(i) => self.setitem_by_idx(i, value, vm), @@ -885,21 +943,21 @@ impl Py<PyMemoryView> { } } - #[pymethod(magic)] - fn reduce_ex(&self, _proto: usize, vm: &VirtualMachine) -> PyResult { - self.reduce(vm) + #[pymethod] + fn __reduce_ex__(&self, _proto: usize, vm: &VirtualMachine) -> PyResult { + self.__reduce__(vm) } - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("cannot pickle 'memoryview' object".to_owned())) + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'memoryview' object")) } } #[derive(FromArgs)] struct CastArgs { #[pyarg(any)] - format: PyStrRef, + format: PyUtf8StrRef, #[pyarg(any, optional)] shape: OptionalArg<Either<PyTupleRef, PyListRef>>, } @@ -914,15 +972,15 @@ enum SubscriptNeedle { impl TryFromObject for SubscriptNeedle { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { // TODO: number protocol - if let Some(i) = obj.payload::<PyInt>() { + if let Some(i) = obj.downcast_ref::<PyInt>() { Ok(Self::Index(i.try_to_primitive(vm)?)) - } else if obj.payload_is::<PySlice>() { + } else if obj.downcastable::<PySlice>() { Ok(Self::Slice(unsafe { obj.downcast_unchecked::<PySlice>() })) } else if let Ok(i) = obj.try_index(vm) { Ok(Self::Index(i.try_to_primitive(vm)?)) } else { - if let Some(tuple) = obj.payload::<PyTuple>() { - if tuple.iter().all(|x| x.payload_is::<PyInt>()) { + if let Some(tuple) = obj.downcast_ref::<PyTuple>() { + if tuple.iter().all(|x| x.downcastable::<PyInt>()) { let v = tuple .iter() .map(|x| { @@ -931,13 +989,13 @@ impl TryFromObject for SubscriptNeedle { }) .try_collect()?; return Ok(Self::MultiIndex(v)); - } else if tuple.iter().all(|x| x.payload_is::<PySlice>()) { + } else if tuple.iter().all(|x| x.downcastable::<PySlice>()) { return Err(vm.new_not_implemented_error( - "multi-dimensional slicing is not implemented".to_owned(), + "multi-dimensional slicing is not implemented", )); } } - Err(vm.new_type_error("memoryview: invalid slice key".to_owned())) + Err(vm.new_type_error("memoryview: invalid slice key")) } } } @@ -952,7 +1010,7 @@ static BUFFER_METHODS: BufferMethods = BufferMethods { impl AsBuffer for PyMemoryView { fn as_buffer(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyBuffer> { if zelf.released.load() { - Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned())) + Err(vm.new_value_error("operation forbidden on released memoryview object")) } else { Ok(PyBuffer::new( zelf.to_owned().into(), @@ -976,15 +1034,15 @@ impl Drop for PyMemoryView { impl AsMapping for PyMemoryView { fn as_mapping() -> &'static PyMappingMethods { static AS_MAPPING: PyMappingMethods = PyMappingMethods { - length: atomic_func!(|mapping, vm| PyMemoryView::mapping_downcast(mapping).len(vm)), + length: atomic_func!(|mapping, vm| PyMemoryView::mapping_downcast(mapping).__len__(vm)), subscript: atomic_func!(|mapping, needle, vm| { let zelf = PyMemoryView::mapping_downcast(mapping); - PyMemoryView::getitem(zelf.to_owned(), needle.to_owned(), vm) + PyMemoryView::__getitem__(zelf.to_owned(), needle.to_owned(), vm) }), ass_subscript: atomic_func!(|mapping, needle, value, vm| { let zelf = PyMemoryView::mapping_downcast(mapping); if let Some(value) = value { - zelf.setitem(needle.to_owned(), value, vm) + zelf.__setitem__(needle.to_owned(), value, vm) } else { Err(vm.new_type_error("cannot delete memory".to_owned())) } @@ -996,11 +1054,11 @@ impl AsMapping for PyMemoryView { impl AsSequence for PyMemoryView { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, vm| { let zelf = PyMemoryView::sequence_downcast(seq); zelf.try_not_released(vm)?; - zelf.len(vm) + zelf.__len__(vm) }), item: atomic_func!(|seq, i, vm| { let zelf = PyMemoryView::sequence_downcast(seq); @@ -1037,21 +1095,21 @@ impl Comparable for PyMemoryView { impl Hashable for PyMemoryView { fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - zelf.hash - .get_or_try_init(|| { - zelf.try_not_released(vm)?; - if !zelf.desc.readonly { - return Err( - vm.new_value_error("cannot hash writable memoryview object".to_owned()) - ); - } - Ok(zelf.contiguous_or_collect(|bytes| vm.state.hash_secret.hash_bytes(bytes))) - }) - .map(|&x| x) + if let Some(val) = zelf.hash.get() { + return Ok(*val); + } + zelf.try_not_released(vm)?; + if !zelf.desc.readonly { + return Err(vm.new_value_error("cannot hash writable memoryview object")); + } + let val = zelf.contiguous_or_collect(|bytes| vm.state.hash_secret.hash_bytes(bytes)); + let _ = zelf.hash.set(val); + Ok(*zelf.hash.get().unwrap()) } } impl PyPayload for PyMemoryView { + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.memoryview_type } @@ -1069,7 +1127,7 @@ impl Representable for PyMemoryView { } } -pub(crate) fn init(ctx: &Context) { +pub(crate) fn init(ctx: &'static Context) { PyMemoryView::extend_class(ctx, ctx.types.memoryview_type); PyMemoryViewIterator::extend_class(ctx, ctx.types.memoryviewiterator_type); } @@ -1081,7 +1139,7 @@ fn format_unpack( ) -> PyResult<PyObjectRef> { format_spec.unpack(bytes, vm).map(|x| { if x.len() == 1 { - x.fast_getitem(0) + x[0].to_owned() } else { x.into() } @@ -1137,22 +1195,25 @@ impl PyPayload for PyMemoryViewIterator { } } -#[pyclass(with(Constructor, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyMemoryViewIterator { - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) } } -impl Unconstructible for PyMemoryViewIterator {} impl SelfIter for PyMemoryViewIterator {} impl IterNext for PyMemoryViewIterator { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { zelf.internal.lock().next(|mv, pos| { - let len = mv.len(vm)?; + let len = mv.__len__(vm)?; Ok(if pos >= len { PyIterReturn::StopIteration(None) } else { diff --git a/crates/vm/src/builtins/mod.rs b/crates/vm/src/builtins/mod.rs new file mode 100644 index 00000000000..099787332c9 --- /dev/null +++ b/crates/vm/src/builtins/mod.rs @@ -0,0 +1,103 @@ +//! This package contains the python basic/builtin types +//! 7 common PyRef type aliases are exposed - [`PyBytesRef`], [`PyDictRef`], [`PyIntRef`], [`PyListRef`], [`PyStrRef`], [`PyTypeRef`], [`PyTupleRef`] +//! Do not add more PyRef type aliases. They will be rare enough to use directly `PyRef<T>`. + +pub(crate) mod asyncgenerator; +pub use asyncgenerator::PyAsyncGen; +pub(crate) mod builtin_func; +pub(crate) mod bytearray; +pub use bytearray::PyByteArray; +pub(crate) mod bytes; +pub use bytes::{PyBytes, PyBytesRef}; +pub(crate) mod capsule; +pub use capsule::PyCapsule; +pub(crate) mod classmethod; +pub use classmethod::PyClassMethod; +pub(crate) mod code; +pub use code::PyCode; +pub(crate) mod complex; +pub use complex::PyComplex; +pub(crate) mod coroutine; +pub use coroutine::PyCoroutine; +pub(crate) mod dict; +pub use dict::{PyDict, PyDictRef}; +pub(crate) mod enumerate; +pub use enumerate::PyEnumerate; +pub(crate) mod filter; +pub use filter::PyFilter; +pub(crate) mod float; +pub use float::PyFloat; +pub(crate) mod frame; +pub(crate) mod function; +pub use function::{PyBoundMethod, PyFunction}; +pub(crate) mod generator; +pub use generator::PyGenerator; +pub(crate) mod genericalias; +pub use genericalias::PyGenericAlias; +pub(crate) mod getset; +pub use getset::PyGetSet; +pub(crate) mod int; +pub use int::{PyInt, PyIntRef}; +pub(crate) mod interpolation; +pub use interpolation::PyInterpolation; +pub(crate) mod iter; +pub use iter::*; +pub(crate) mod list; +pub use list::{PyList, PyListRef}; +pub(crate) mod map; +pub use map::PyMap; +pub(crate) mod mappingproxy; +pub use mappingproxy::PyMappingProxy; +pub(crate) mod memory; +pub use memory::PyMemoryView; +pub(crate) mod module; +pub use module::{PyModule, PyModuleDef, PyModuleSlots}; +pub(crate) mod namespace; +pub use namespace::PyNamespace; +pub(crate) mod object; +pub use object::PyBaseObject; +pub(crate) mod property; +pub use property::PyProperty; +#[path = "bool.rs"] +pub(crate) mod bool_; +pub use bool_::PyBool; +#[path = "str.rs"] +pub(crate) mod pystr; +pub use pystr::{PyStr, PyStrInterned, PyStrRef, PyUtf8Str, PyUtf8StrInterned, PyUtf8StrRef}; +#[path = "super.rs"] +pub(crate) mod super_; +pub use super_::PySuper; +#[path = "type.rs"] +pub(crate) mod type_; +pub use type_::{PyType, PyTypeRef}; +pub(crate) mod range; +pub use range::PyRange; +pub(crate) mod set; +pub use set::{PyFrozenSet, PySet}; +pub(crate) mod singletons; +pub use singletons::{PyNone, PyNotImplemented}; +pub(crate) mod slice; +pub use slice::{PyEllipsis, PySlice}; +pub(crate) mod staticmethod; +pub use staticmethod::PyStaticMethod; +pub(crate) mod template; +pub use template::{PyTemplate, PyTemplateIter}; +pub(crate) mod traceback; +pub use traceback::PyTraceback; +pub(crate) mod tuple; +pub use tuple::{PyTuple, PyTupleRef}; +pub(crate) mod weakproxy; +pub use weakproxy::PyWeakProxy; +pub(crate) mod weakref; +pub use weakref::PyWeak; +pub(crate) mod zip; +pub use zip::PyZip; +#[path = "union.rs"] +pub(crate) mod union_; +pub use union_::{PyUnion, make_union}; +pub(crate) mod descriptor; + +pub use float::try_to_bigint as try_f64_to_bigint; +pub use int::try_to_float as try_bigint_to_f64; + +pub use crate::exceptions::types::*; diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs new file mode 100644 index 00000000000..cabaf1d63cb --- /dev/null +++ b/crates/vm/src/builtins/module.rs @@ -0,0 +1,489 @@ +use super::{PyDict, PyDictRef, PyStr, PyStrRef, PyType, PyTypeRef}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyStrInterned, pystr::AsPyStr}, + class::PyClassImpl, + convert::ToPyObject, + function::{FuncArgs, PyMethodDef, PySetterValue}, + import::{get_spec_file_origin, is_possibly_shadowing_path, is_stdlib_module_name}, + types::{GetAttr, Initializer, Representable}, +}; + +#[pyclass(module = false, name = "module")] +#[derive(Debug)] +pub struct PyModuleDef { + // pub index: usize, + pub name: &'static PyStrInterned, + pub doc: Option<&'static PyStrInterned>, + // pub size: isize, + pub methods: &'static [PyMethodDef], + pub slots: PyModuleSlots, + // traverse: traverse_proc + // clear: inquiry + // free: free_func +} + +pub type ModuleCreate = + fn(&VirtualMachine, &PyObject, &'static PyModuleDef) -> PyResult<PyRef<PyModule>>; +pub type ModuleExec = fn(&VirtualMachine, &Py<PyModule>) -> PyResult<()>; + +#[derive(Default)] +pub struct PyModuleSlots { + pub create: Option<ModuleCreate>, + pub exec: Option<ModuleExec>, +} + +impl core::fmt::Debug for PyModuleSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyModuleSlots") + .field("create", &self.create.is_some()) + .field("exec", &self.exec.is_some()) + .finish() + } +} + +impl PyModuleDef { + /// Create a module from this definition (Phase 1 of multi-phase init). + /// + /// This performs: + /// 1. Create module object (using create slot if provided) + /// 2. Initialize module dict from def + /// 3. Add methods to module + /// + /// Does NOT add to sys.modules or call exec slot. + pub fn create_module(&'static self, vm: &VirtualMachine) -> PyResult<PyRef<PyModule>> { + use crate::PyPayload; + + // Create module (use create slot if provided, else default creation) + let module = if let Some(create) = self.slots.create { + // Custom module creation + let spec = vm.ctx.new_str(self.name.as_str()); + create(vm, spec.as_object(), self)? + } else { + // Default module creation + PyModule::from_def(self).into_ref(&vm.ctx) + }; + + // Initialize module dict and methods + PyModule::__init_dict_from_def(vm, &module); + module.__init_methods(vm)?; + + Ok(module) + } + + /// Execute the module's exec slot (Phase 2 of multi-phase init). + /// + /// Calls the exec slot if present. Returns Ok(()) if no exec slot. + pub fn exec_module(&'static self, vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + if let Some(exec) = self.slots.exec { + exec(vm, module)?; + } + Ok(()) + } +} + +#[allow( + clippy::new_without_default, + reason = "avoid a misleading Default implementation" +)] +#[pyclass(module = false, name = "module")] +#[derive(Debug)] +pub struct PyModule { + // PyObject *md_dict; + pub def: Option<&'static PyModuleDef>, + // state: Any + // weaklist + // for logging purposes after md_dict is cleared + pub name: Option<&'static PyStrInterned>, +} + +impl PyPayload for PyModule { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.module_type + } +} + +#[derive(FromArgs)] +pub struct ModuleInitArgs { + name: PyStrRef, + #[pyarg(any, default)] + doc: Option<PyStrRef>, +} + +impl PyModule { + #[allow(clippy::new_without_default)] + pub const fn new() -> Self { + Self { + def: None, + name: None, + } + } + + pub const fn from_def(def: &'static PyModuleDef) -> Self { + Self { + def: Some(def), + name: Some(def.name), + } + } + + pub fn __init_dict_from_def(vm: &VirtualMachine, module: &Py<Self>) { + let doc = module.def.unwrap().doc.map(|doc| doc.to_owned()); + module.init_dict(module.name.unwrap(), doc, vm); + } +} + +impl Py<PyModule> { + pub fn __init_methods(&self, vm: &VirtualMachine) -> PyResult<()> { + debug_assert!(self.def.is_some()); + for method in self.def.unwrap().methods { + let func = method + .to_function() + .with_module(self.name.unwrap()) + .into_ref(&vm.ctx); + vm.__module_set_attr(self, vm.ctx.intern_str(method.name), func)?; + } + Ok(()) + } + + fn getattr_inner(&self, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + if let Some(attr) = self.as_object().generic_getattr_opt(name, None, vm)? { + return Ok(attr); + } + if let Ok(getattr) = self.dict().get_item(identifier!(vm, __getattr__), vm) { + return getattr.call((name.to_owned(),), vm); + } + let dict = self.dict(); + + // Get the raw __name__ object (may be a str subclass) + let mod_name_obj = dict + .get_item_opt(identifier!(vm, __name__), vm) + .ok() + .flatten(); + let mod_name_str = mod_name_obj.as_ref().and_then(|n| { + n.downcast_ref::<PyStr>() + .map(|s| s.to_string_lossy().into_owned()) + }); + + // If __name__ is not set or not a string, use a simpler error message + let mod_display = match mod_name_str.as_deref() { + Some(s) => s, + None => { + return Err(vm.new_attribute_error(format!("module has no attribute '{name}'"))); + } + }; + + let spec = dict + .get_item_opt(vm.ctx.intern_str("__spec__"), vm) + .ok() + .flatten() + .filter(|s| !vm.is_none(s)); + + let origin = get_spec_file_origin(&spec, vm); + + let is_possibly_shadowing = origin + .as_ref() + .map(|o| is_possibly_shadowing_path(o, vm)) + .unwrap_or(false); + // Use the ORIGINAL __name__ object for stdlib check (may raise TypeError + // if __name__ is an unhashable str subclass) + let is_possibly_shadowing_stdlib = if is_possibly_shadowing { + if let Some(ref mod_name) = mod_name_obj { + is_stdlib_module_name(mod_name, vm)? + } else { + false + } + } else { + false + }; + + if is_possibly_shadowing_stdlib { + let origin = origin.as_ref().unwrap(); + Err(vm.new_attribute_error(format!( + "module '{mod_display}' has no attribute '{name}' \ + (consider renaming '{origin}' since it has the same \ + name as the standard library module named '{mod_display}' \ + and prevents importing that standard library module)" + ))) + } else { + let is_initializing = PyModule::is_initializing(&dict, vm); + if is_initializing { + if is_possibly_shadowing { + let origin = origin.as_ref().unwrap(); + Err(vm.new_attribute_error(format!( + "module '{mod_display}' has no attribute '{name}' \ + (consider renaming '{origin}' if it has the same name \ + as a library you intended to import)" + ))) + } else if let Some(ref origin) = origin { + Err(vm.new_attribute_error(format!( + "partially initialized module '{mod_display}' from '{origin}' \ + has no attribute '{name}' \ + (most likely due to a circular import)" + ))) + } else { + Err(vm.new_attribute_error(format!( + "partially initialized module '{mod_display}' \ + has no attribute '{name}' \ + (most likely due to a circular import)" + ))) + } + } else { + // Check for uninitialized submodule + let submodule_initializing = + is_uninitialized_submodule(mod_name_str.as_ref(), name, vm); + if submodule_initializing { + Err(vm.new_attribute_error(format!( + "cannot access submodule '{name}' of module '{mod_display}' \ + (most likely due to a circular import)" + ))) + } else { + Err(vm.new_attribute_error(format!( + "module '{mod_display}' has no attribute '{name}'" + ))) + } + } + } + } + + // TODO: to be replaced by the commented-out dict method above once dictoffset land + pub fn dict(&self) -> PyDictRef { + self.as_object().dict().unwrap() + } + + // TODO: should be on PyModule, not Py<PyModule> + pub(crate) fn init_dict( + &self, + name: &'static PyStrInterned, + doc: Option<PyStrRef>, + vm: &VirtualMachine, + ) { + let dict = self.dict(); + dict.set_item(identifier!(vm, __name__), name.to_object(), vm) + .expect("Failed to set __name__ on module"); + dict.set_item(identifier!(vm, __doc__), doc.to_pyobject(vm), vm) + .expect("Failed to set __doc__ on module"); + dict.set_item("__package__", vm.ctx.none(), vm) + .expect("Failed to set __package__ on module"); + dict.set_item("__loader__", vm.ctx.none(), vm) + .expect("Failed to set __loader__ on module"); + dict.set_item("__spec__", vm.ctx.none(), vm) + .expect("Failed to set __spec__ on module"); + } + + pub fn get_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult { + let attr_name = attr_name.as_pystr(&vm.ctx); + self.getattr_inner(attr_name, vm) + } + + pub fn set_attr<'a>( + &self, + attr_name: impl AsPyStr<'a>, + attr_value: impl Into<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.as_object().set_attr(attr_name, attr_value, vm) + } +} + +#[pyclass( + with(GetAttr, Initializer, Representable), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) +)] +impl PyModule { + #[pyslot] + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Self::new().into_ref_with_type(vm, cls).map(Into::into) + } + + #[pymethod] + fn __dir__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + // First check if __dict__ attribute exists and is actually a dictionary + let dict_attr = zelf.as_object().get_attr(identifier!(vm, __dict__), vm)?; + let dict = dict_attr + .downcast::<PyDict>() + .map_err(|_| vm.new_type_error("<module>.__dict__ is not a dictionary"))?; + let attrs = dict.into_iter().map(|(k, _v)| k).collect(); + Ok(attrs) + } + + #[pygetset] + fn __annotate__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let dict = zelf.dict(); + // Get __annotate__ from dict; if not present, insert None and return it + // See: module_get_annotate() + if let Some(annotate) = dict.get_item_opt(identifier!(vm, __annotate__), vm)? { + Ok(annotate) + } else { + let none = vm.ctx.none(); + dict.set_item(identifier!(vm, __annotate__), none.clone(), vm)?; + Ok(none) + } + } + + #[pygetset(setter)] + fn set___annotate__( + zelf: &Py<Self>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(value) => { + if !vm.is_none(&value) && !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None")); + } + let dict = zelf.dict(); + dict.set_item(identifier!(vm, __annotate__), value.clone(), vm)?; + // Clear __annotations__ if value is not None + if !vm.is_none(&value) { + dict.del_item(identifier!(vm, __annotations__), vm).ok(); + } + Ok(()) + } + PySetterValue::Delete => Err(vm.new_type_error("cannot delete __annotate__ attribute")), + } + } + + #[pygetset] + fn __annotations__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let dict = zelf.dict(); + + // Check if __annotations__ is already in dict (explicitly set) + if let Some(annotations) = dict.get_item_opt(identifier!(vm, __annotations__), vm)? { + return Ok(annotations); + } + + // Check if module is initializing + let is_initializing = Self::is_initializing(&dict, vm); + + // PEP 649: Get __annotate__ and call it if callable + let annotations = if let Some(annotate) = + dict.get_item_opt(identifier!(vm, __annotate__), vm)? + && annotate.is_callable() + { + // Call __annotate__(1) where 1 is FORMAT_VALUE + let result = annotate.call((1i32,), vm)?; + if !result.class().is(vm.ctx.types.dict_type) { + return Err(vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + result.class().name() + ))); + } + result + } else { + vm.ctx.new_dict().into() + }; + + // Cache result unless module is initializing + if !is_initializing { + dict.set_item(identifier!(vm, __annotations__), annotations.clone(), vm)?; + } + + Ok(annotations) + } + + /// Check if module is initializing via __spec__._initializing + fn is_initializing(dict: &PyDictRef, vm: &VirtualMachine) -> bool { + if let Ok(Some(spec)) = dict.get_item_opt(vm.ctx.intern_str("__spec__"), vm) + && let Ok(initializing) = spec.get_attr(vm.ctx.intern_str("_initializing"), vm) + { + return initializing.try_to_bool(vm).unwrap_or(false); + } + false + } + + #[pygetset(setter)] + fn set___annotations__( + zelf: &Py<Self>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + let dict = zelf.dict(); + match value { + PySetterValue::Assign(value) => { + dict.set_item(identifier!(vm, __annotations__), value, vm)?; + // Clear __annotate__ from dict + dict.del_item(identifier!(vm, __annotate__), vm).ok(); + Ok(()) + } + PySetterValue::Delete => { + if dict.del_item(identifier!(vm, __annotations__), vm).is_err() { + return Err(vm.new_attribute_error("__annotations__")); + } + // Also clear __annotate__ + dict.del_item(identifier!(vm, __annotate__), vm).ok(); + Ok(()) + } + } + } +} + +impl Initializer for PyModule { + type Args = ModuleInitArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + debug_assert!( + zelf.class() + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + ); + zelf.init_dict(vm.ctx.intern_str(args.name.as_wtf8()), args.doc, vm); + Ok(()) + } +} + +impl GetAttr for PyModule { + fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + zelf.getattr_inner(name, vm) + } +} + +impl Representable for PyModule { + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + // Use cached importlib reference (like interp->importlib) + let module_repr = vm.importlib.get_attr("_module_repr", vm)?; + let repr = module_repr.call((zelf.to_owned(),), vm)?; + repr.downcast() + .map_err(|_| vm.new_type_error("_module_repr did not return a string")) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } +} + +pub(crate) fn init(context: &'static Context) { + PyModule::extend_class(context, context.types.module_type); +} + +/// Check if {module_name}.{name} is an uninitialized submodule in sys.modules. +fn is_uninitialized_submodule( + module_name: Option<&String>, + name: &Py<PyStr>, + vm: &VirtualMachine, +) -> bool { + let mod_name = match module_name { + Some(n) => n.as_str(), + None => return false, + }; + let full_name = format!("{mod_name}.{name}"); + let sys_modules = match vm.sys_module.get_attr("modules", vm).ok() { + Some(m) => m, + None => return false, + }; + let sub_mod = match sys_modules.get_item(&full_name, vm).ok() { + Some(m) => m, + None => return false, + }; + let spec = match sub_mod.get_attr("__spec__", vm).ok() { + Some(s) if !vm.is_none(&s) => s, + _ => return false, + }; + spec.get_attr("_initializing", vm) + .ok() + .and_then(|v| v.try_to_bool(vm).ok()) + .unwrap_or(false) +} diff --git a/crates/vm/src/builtins/namespace.rs b/crates/vm/src/builtins/namespace.rs new file mode 100644 index 00000000000..4e872a172a4 --- /dev/null +++ b/crates/vm/src/builtins/namespace.rs @@ -0,0 +1,180 @@ +use super::{PyStr, PyTupleRef, PyType, tuple::IntoPyTuple}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::PyDict, + class::PyClassImpl, + function::{FuncArgs, PyComparisonValue}, + recursion::ReprGuard, + types::{ + Comparable, Constructor, DefaultConstructor, Initializer, PyComparisonOp, Representable, + }, +}; +use rustpython_common::wtf8::Wtf8Buf; + +/// A simple attribute-based namespace. +/// +/// SimpleNamespace(**kwargs) +#[pyclass(module = "types", name = "SimpleNamespace")] +#[derive(Debug, Default)] +pub struct PyNamespace {} + +impl PyPayload for PyNamespace { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.namespace_type + } +} + +impl DefaultConstructor for PyNamespace {} + +#[pyclass( + flags(BASETYPE, HAS_DICT, HAS_WEAKREF), + with(Constructor, Initializer, Comparable, Representable) +)] +impl PyNamespace { + #[pymethod] + fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyTupleRef { + let dict = zelf.as_object().dict().unwrap(); + let obj = zelf.as_object().to_owned(); + let result: (PyObjectRef, PyObjectRef, PyObjectRef) = ( + obj.class().to_owned().into(), + vm.new_tuple(()).into(), + dict.into(), + ); + result.into_pytuple(vm) + } + + #[pymethod] + fn __replace__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() { + return Err(vm.new_type_error("__replace__() takes no positional arguments")); + } + + // Create a new instance of the same type + let cls: PyObjectRef = zelf.class().to_owned().into(); + let result = cls.call((), vm)?; + + // Copy the current namespace dict to the new instance + let src_dict = zelf.dict().unwrap(); + let dst_dict = result.dict().unwrap(); + for (key, value) in src_dict { + dst_dict.set_item(&*key, value, vm)?; + } + + // Update with the provided kwargs + for (name, value) in args.kwargs { + let name = vm.ctx.new_str(name); + result.set_attr(&name, value, vm)?; + } + + Ok(result) + } +} + +impl Initializer for PyNamespace { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // SimpleNamespace accepts 0 or 1 positional argument (a mapping) + if args.args.len() > 1 { + return Err(vm.new_type_error(format!( + "{} expected at most 1 positional argument, got {}", + zelf.class().name(), + args.args.len() + ))); + } + + // If there's a positional argument, treat it as a mapping + if let Some(mapping) = args.args.first() { + // Convert to dict if not already + let dict: PyRef<PyDict> = if let Some(d) = mapping.downcast_ref::<PyDict>() { + d.to_owned() + } else { + // Call dict() on the mapping + let dict_type: PyObjectRef = vm.ctx.types.dict_type.to_owned().into(); + dict_type + .call((mapping.clone(),), vm)? + .downcast() + .map_err(|_| vm.new_type_error("dict() did not return a dict"))? + }; + + // Validate keys are strings and set attributes + for (key, value) in dict.into_iter() { + let key_str = key + .downcast_ref::<crate::builtins::PyStr>() + .ok_or_else(|| { + vm.new_type_error(format!( + "keywords must be strings, not '{}'", + key.class().name() + )) + })?; + zelf.as_object().set_attr(key_str, value, vm)?; + } + } + + // Apply keyword arguments (these override positional mapping values) + for (name, value) in args.kwargs { + let name = vm.ctx.new_str(name); + zelf.as_object().set_attr(&name, value, vm)?; + } + Ok(()) + } +} + +impl Comparable for PyNamespace { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let other = class_or_notimplemented!(Self, other); + let (d1, d2) = ( + zelf.as_object().dict().unwrap(), + other.as_object().dict().unwrap(), + ); + PyDict::cmp(&d1, d2.as_object(), op, vm) + } +} + +impl Representable for PyNamespace { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let o = zelf.as_object(); + let name = if o.class().is(vm.ctx.types.namespace_type) { + "namespace".to_owned() + } else { + o.class().slot_name().to_owned() + }; + + let repr = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let dict = zelf.as_object().dict().unwrap(); + let mut result = Wtf8Buf::from(format!("{name}(")); + let mut first = true; + for (key, value) in dict { + let Some(key_str) = key.downcast_ref::<PyStr>() else { + continue; + }; + if key_str.as_wtf8().is_empty() { + continue; + } + if !first { + result.push_str(", "); + } + first = false; + result.push_wtf8(key_str.as_wtf8()); + result.push_char('='); + result.push_wtf8(value.repr(vm)?.as_wtf8()); + } + result.push_char(')'); + result + } else { + Wtf8Buf::from(format!("{name}(...)")) + }; + Ok(repr) + } +} + +pub fn init(context: &'static Context) { + PyNamespace::extend_class(context, context.types.namespace_type); +} diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs new file mode 100644 index 00000000000..002b05d38f1 --- /dev/null +++ b/crates/vm/src/builtins/object.rs @@ -0,0 +1,759 @@ +use super::{PyDictRef, PyList, PyStr, PyStrRef, PyType, PyTypeRef, PyUtf8StrRef}; +use crate::common::hash::PyHash; +use crate::types::PyTypeFlags; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + convert::ToPyResult, + function::{Either, FuncArgs, PyArithmeticValue, PyComparisonValue, PySetterValue}, + types::{Constructor, Initializer, PyComparisonOp}, +}; +use itertools::Itertools; + +/// object() +/// -- +/// +/// The base class of the class hierarchy. +/// +/// When called, it accepts no arguments and returns a new featureless +/// instance that has no instance attributes and cannot be given any. +#[pyclass(module = false, name = "object")] +#[derive(Debug)] +pub struct PyBaseObject; + +impl PyPayload for PyBaseObject { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.object_type + } +} + +impl Constructor for PyBaseObject { + type Args = FuncArgs; + + // = object_new + fn slot_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() || !args.kwargs.is_empty() { + // Check if type's __new__ != object.__new__ + let tp_new = cls.get_attr(identifier!(vm, __new__)); + let object_new = vm.ctx.types.object_type.get_attr(identifier!(vm, __new__)); + + if let (Some(tp_new), Some(object_new)) = (tp_new, object_new) { + if !tp_new.is(&object_new) { + // Type has its own __new__, so object.__new__ is being called + // with excess args. This is the first error case in CPython + return Err(vm.new_type_error( + "object.__new__() takes exactly one argument (the type to instantiate)", + )); + } + + // If we reach here, tp_new == object_new + // Now check if type's __init__ == object.__init__ + let tp_init = cls.get_attr(identifier!(vm, __init__)); + let object_init = vm.ctx.types.object_type.get_attr(identifier!(vm, __init__)); + + if let (Some(tp_init), Some(object_init)) = (tp_init, object_init) + && tp_init.is(&object_init) + { + // Both __new__ and __init__ are object's versions, + // so the type accepts no arguments + return Err(vm.new_type_error(format!("{}() takes no arguments", cls.name()))); + } + // If tp_init != object_init, then the type has custom __init__ + // which might accept arguments, so we allow it + } + } + + // Ensure that all abstract methods are implemented before instantiating instance. + if let Some(abs_methods) = cls.get_attr(identifier!(vm, __abstractmethods__)) + && let Some(unimplemented_abstract_method_count) = abs_methods.length_opt(vm) + { + let methods: Vec<PyUtf8StrRef> = abs_methods.try_to_value(vm)?; + let methods: String = Itertools::intersperse( + methods.iter().map(|name| name.as_str().to_owned()), + "', '".to_owned(), + ) + .collect(); + + let unimplemented_abstract_method_count = unimplemented_abstract_method_count?; + let name = cls.name().to_string(); + + match unimplemented_abstract_method_count { + 0 => {} + 1 => { + return Err(vm.new_type_error(format!( + "class {name} without an implementation for abstract method '{methods}'" + ))); + } + 2.. => { + return Err(vm.new_type_error(format!( + "class {name} without an implementation for abstract methods '{methods}'" + ))); + } + // TODO: remove `allow` when redox build doesn't complain about it + #[allow(unreachable_patterns)] + _ => unreachable!(), + } + } + + generic_alloc(cls, 0, vm) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +pub(crate) fn generic_alloc(cls: PyTypeRef, _nitems: usize, vm: &VirtualMachine) -> PyResult { + // Only create dict if the class has HAS_DICT flag (i.e., __slots__ was not defined + // or __dict__ is in __slots__) + let dict = if cls + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + { + Some(vm.ctx.new_dict()) + } else { + None + }; + Ok(crate::PyRef::new_ref(PyBaseObject, cls, dict).into()) +} + +impl Initializer for PyBaseObject { + type Args = FuncArgs; + + // object_init: excess_args validation + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + if args.is_empty() { + return Ok(()); + } + + let typ = zelf.class(); + let object_type = &vm.ctx.types.object_type; + + let typ_init = typ.slots.init.load().map(|f| f as usize); + let object_init = object_type.slots.init.load().map(|f| f as usize); + + // if (type->tp_init != object_init) → first error + if typ_init != object_init { + return Err(vm.new_type_error( + "object.__init__() takes exactly one argument (the instance to initialize)", + )); + } + + // if (type->tp_new == object_new) → second error + if let (Some(typ_new), Some(object_new)) = ( + typ.get_attr(identifier!(vm, __new__)), + object_type.get_attr(identifier!(vm, __new__)), + ) && typ_new.is(&object_new) + { + return Err(vm.new_type_error(format!( + "{}.__init__() takes exactly one argument (the instance to initialize)", + typ.name() + ))); + } + + // Both conditions false → OK (e.g., tuple, dict with custom __new__) + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } +} + +// TODO: implement _PyType_GetSlotNames properly +fn type_slot_names(typ: &Py<PyType>, vm: &VirtualMachine) -> PyResult<Option<super::PyListRef>> { + // let attributes = typ.attributes.read(); + // if let Some(slot_names) = attributes.get(identifier!(vm.ctx, __slotnames__)) { + // return match_class!(match slot_names.clone() { + // l @ super::PyList => Ok(Some(l)), + // _n @ super::PyNone => Ok(None), + // _ => Err(vm.new_type_error(format!( + // "{:.200}.__slotnames__ should be a list or None, not {:.200}", + // typ.name(), + // slot_names.class().name() + // ))), + // }); + // } + + let copyreg = vm.import("copyreg", 0)?; + let copyreg_slotnames = copyreg.get_attr("_slotnames", vm)?; + let slot_names = copyreg_slotnames.call((typ.to_owned(),), vm)?; + let result = match_class!(match slot_names { + l @ super::PyList => Some(l), + _n @ super::PyNone => None, + _ => return Err(vm.new_type_error("copyreg._slotnames didn't return a list or None")), + }); + Ok(result) +} + +// object_getstate_default +fn object_getstate_default(obj: &PyObject, required: bool, vm: &VirtualMachine) -> PyResult { + // Check itemsize + if required && obj.class().slots.itemsize > 0 { + return Err(vm.new_type_error(format!("cannot pickle {:.200} objects", obj.class().name()))); + } + + let state = if obj.dict().is_none_or(|d| d.is_empty()) { + vm.ctx.none() + } else { + // let state = object_get_dict(obj.clone(), obj.ctx()).unwrap(); + let Some(state) = obj.dict() else { + return Ok(vm.ctx.none()); + }; + state.into() + }; + + let slot_names = + type_slot_names(obj.class(), vm).map_err(|_| vm.new_type_error("cannot pickle object"))?; + + if required { + // Start with PyBaseObject_Type's basicsize + let mut basicsize = vm.ctx.types.object_type.slots.basicsize; + + // Add __dict__ size if type has dict + if obj.class().slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + basicsize += core::mem::size_of::<PyObjectRef>(); + } + + // Add __weakref__ size if type has weakref support + let has_weakref = if let Some(ref ext) = obj.class().heaptype_ext { + match &ext.slots { + None => true, // Heap type without __slots__ has automatic weakref + Some(slots) => slots.iter().any(|s| s.as_bytes() == b"__weakref__"), + } + } else { + let weakref_name = vm.ctx.intern_str("__weakref__"); + obj.class().attributes.read().contains_key(weakref_name) + }; + if has_weakref { + basicsize += core::mem::size_of::<PyObjectRef>(); + } + + // Add slots size + if let Some(ref slot_names) = slot_names { + basicsize += core::mem::size_of::<PyObjectRef>() * slot_names.__len__(); + } + + // Fail if actual type's basicsize > expected basicsize + if obj.class().slots.basicsize > basicsize { + return Err(vm.new_type_error(format!("cannot pickle '{}' object", obj.class().name()))); + } + } + + if let Some(slot_names) = slot_names { + let slot_names_len = slot_names.__len__(); + if slot_names_len > 0 { + let slots = vm.ctx.new_dict(); + for i in 0..slot_names_len { + let borrowed_names = slot_names.borrow_vec(); + // Check if slotnames changed during iteration + if borrowed_names.len() != slot_names_len { + return Err(vm.new_runtime_error("__slotnames__ changed size during iteration")); + } + let name = borrowed_names[i].downcast_ref::<PyStr>().unwrap(); + let Ok(value) = obj.get_attr(name, vm) else { + continue; + }; + slots.set_item(name.as_wtf8(), value, vm).unwrap(); + } + + if !slots.is_empty() { + return (state, slots).to_pyresult(vm); + } + } + } + + Ok(state) +} + +// object_getstate +// fn object_getstate( +// obj: &PyObject, +// required: bool, +// vm: &VirtualMachine, +// ) -> PyResult { +// let getstate = obj.get_attr(identifier!(vm, __getstate__), vm)?; +// if vm.is_none(&getstate) { +// return Ok(None); +// } + +// let getstate = match getstate.downcast_exact::<PyNativeFunction>(vm) { +// Ok(getstate) +// if getstate +// .get_self() +// .map_or(false, |self_obj| self_obj.is(obj)) +// && std::ptr::addr_eq( +// getstate.as_func() as *const _, +// &PyBaseObject::__getstate__ as &dyn crate::function::PyNativeFn as *const _, +// ) => +// { +// return object_getstate_default(obj, required, vm); +// } +// Ok(getstate) => getstate.into_pyref().into(), +// Err(getstate) => getstate, +// }; +// getstate.call((), vm) +// } + +#[pyclass(with(Constructor, Initializer), flags(BASETYPE))] +impl PyBaseObject { + #[pymethod(raw)] + fn __getstate__(vm: &VirtualMachine, args: FuncArgs) -> PyResult { + let (zelf,): (PyObjectRef,) = args.bind(vm)?; + object_getstate_default(&zelf, false, vm) + } + + #[pyslot] + fn slot_richcompare( + zelf: &PyObject, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<Either<PyObjectRef, PyComparisonValue>> { + Self::cmp(zelf, other, op, vm).map(Either::B) + } + + #[inline(always)] + fn cmp( + zelf: &PyObject, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let res = match op { + PyComparisonOp::Eq => { + if zelf.is(other) { + PyComparisonValue::Implemented(true) + } else { + PyComparisonValue::NotImplemented + } + } + PyComparisonOp::Ne => { + let cmp = zelf.class().slots.richcompare.load().unwrap(); + let value = match cmp(zelf, other, PyComparisonOp::Eq, vm)? { + Either::A(obj) => PyArithmeticValue::from_object(vm, obj) + .map(|obj| obj.try_to_bool(vm)) + .transpose()?, + Either::B(value) => value, + }; + value.map(|v| !v) + } + _ => PyComparisonValue::NotImplemented, + }; + Ok(res) + } + + /// Implement setattr(self, name, value). + #[pymethod] + fn __setattr__( + obj: PyObjectRef, + name: PyStrRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + obj.generic_setattr(&name, PySetterValue::Assign(value), vm) + } + + /// Implement delattr(self, name). + #[pymethod] + fn __delattr__(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + obj.generic_setattr(&name, PySetterValue::Delete, vm) + } + + #[pyslot] + pub(crate) fn slot_setattro( + obj: &PyObject, + attr_name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + obj.generic_setattr(attr_name, value, vm) + } + + /// Return str(self). + #[pyslot] + fn slot_str(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { + // FIXME: try tp_repr first and fallback to object.__repr__ + zelf.repr(vm) + } + + #[pyslot] + fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let class = zelf.class(); + match ( + class + .__qualname__(vm) + .downcast_ref::<PyStr>() + .map(|n| n.as_wtf8()), + class + .__module__(vm) + .downcast_ref::<PyStr>() + .map(|m| m.as_wtf8()), + ) { + (None, _) => Err(vm.new_type_error("Unknown qualified name")), + (Some(qualname), Some(module)) if module != "builtins" => Ok(PyStr::from(format!( + "<{}.{} object at {:#x}>", + module, + qualname, + zelf.get_id() + )) + .into_ref(&vm.ctx)), + _ => Ok(PyStr::from(format!( + "<{} object at {:#x}>", + class.slot_name(), + zelf.get_id() + )) + .into_ref(&vm.ctx)), + } + } + + #[pyclassmethod] + fn __subclasshook__(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() + } + + #[pyclassmethod] + fn __init_subclass__(_cls: PyTypeRef) {} + + #[pymethod] + pub fn __dir__(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyList> { + obj.dir(vm) + } + + #[pymethod] + fn __format__( + obj: PyObjectRef, + format_spec: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + if !format_spec.is_empty() { + return Err(vm.new_type_error(format!( + "unsupported format string passed to {}.__format__", + obj.class().name() + ))); + } + obj.str(vm) + } + + #[pygetset] + fn __class__(obj: PyObjectRef) -> PyTypeRef { + obj.class().to_owned() + } + + #[pygetset(setter)] + fn set___class__( + instance: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value.downcast::<PyType>() { + Ok(cls) => { + let current_cls = instance.class(); + let both_module = current_cls.fast_issubclass(vm.ctx.types.module_type) + && cls.fast_issubclass(vm.ctx.types.module_type); + let both_mutable = !current_cls + .slots + .flags + .has_feature(PyTypeFlags::IMMUTABLETYPE) + && !cls.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE); + // FIXME(#1979) cls instances might have a payload + if both_mutable || both_module { + let has_dict = + |typ: &Py<PyType>| typ.slots.flags.has_feature(PyTypeFlags::HAS_DICT); + let has_weakref = + |typ: &Py<PyType>| typ.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF); + // Compare slots tuples + let slots_equal = match ( + current_cls + .heaptype_ext + .as_ref() + .and_then(|e| e.slots.as_ref()), + cls.heaptype_ext.as_ref().and_then(|e| e.slots.as_ref()), + ) { + (Some(a), Some(b)) => { + a.len() == b.len() + && a.iter() + .zip(b.iter()) + .all(|(x, y)| x.as_wtf8() == y.as_wtf8()) + } + (None, None) => true, + _ => false, + }; + if current_cls.slots.basicsize != cls.slots.basicsize + || !slots_equal + || has_dict(current_cls) != has_dict(&cls) + || has_weakref(current_cls) != has_weakref(&cls) + || current_cls.slots.member_count != cls.slots.member_count + { + return Err(vm.new_type_error(format!( + "__class__ assignment: '{}' object layout differs from '{}'", + cls.name(), + current_cls.name() + ))); + } + instance.set_class(cls, vm); + Ok(()) + } else { + Err(vm.new_type_error( + "__class__ assignment only supported for mutable types or ModuleType subclasses", + )) + } + } + Err(value) => { + let value_class = value.class(); + let type_repr = &value_class.name(); + Err(vm.new_type_error(format!( + "__class__ must be set to a class, not '{type_repr}' object" + ))) + } + } + } + + /// Return getattr(self, name). + #[pyslot] + pub(crate) fn getattro(obj: &PyObject, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + vm_trace!("object.__getattribute__({:?}, {:?})", obj, name); + obj.as_object().generic_getattr(name, vm) + } + + #[pymethod] + fn __getattribute__(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult { + Self::getattro(&obj, &name, vm) + } + + #[pymethod] + fn __reduce__(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + common_reduce(obj, 0, vm) + } + + #[pymethod] + fn __reduce_ex__(obj: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { + let __reduce__ = identifier!(vm, __reduce__); + if let Some(reduce) = vm.get_attribute_opt(obj.clone(), __reduce__)? { + let object_reduce = vm.ctx.types.object_type.get_attr(__reduce__).unwrap(); + let typ_obj: PyObjectRef = obj.class().to_owned().into(); + let class_reduce = typ_obj.get_attr(__reduce__, vm)?; + if !class_reduce.is(&object_reduce) { + return reduce.call((), vm); + } + } + common_reduce(obj, proto, vm) + } + + #[pyslot] + fn slot_hash(zelf: &PyObject, _vm: &VirtualMachine) -> PyResult<PyHash> { + Ok(zelf.get_id() as _) + } + + #[pymethod] + fn __sizeof__(zelf: PyObjectRef) -> usize { + zelf.class().slots.basicsize + } +} + +pub fn object_get_dict(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyDictRef> { + obj.dict() + .ok_or_else(|| vm.new_attribute_error("This object has no __dict__")) +} +pub fn object_set_dict(obj: PyObjectRef, dict: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { + obj.set_dict(dict) + .map_err(|_| vm.new_attribute_error("This object has no __dict__")) +} + +pub fn init(ctx: &'static Context) { + // Manually set alloc/init slots - derive macro doesn't generate extend_slots + // for trait impl that overrides #[pyslot] method + ctx.types.object_type.slots.alloc.store(Some(generic_alloc)); + ctx.types + .object_type + .slots + .init + .store(Some(<PyBaseObject as Initializer>::slot_init)); + PyBaseObject::extend_class(ctx, ctx.types.object_type); +} + +/// Get arguments for __new__ from __getnewargs_ex__ or __getnewargs__ +/// Returns (args, kwargs) tuple where either can be None +fn get_new_arguments( + obj: &PyObject, + vm: &VirtualMachine, +) -> PyResult<(Option<super::PyTupleRef>, Option<super::PyDictRef>)> { + // First try __getnewargs_ex__ + if let Some(getnewargs_ex) = vm.get_special_method(obj, identifier!(vm, __getnewargs_ex__))? { + let newargs = getnewargs_ex.invoke((), vm)?; + + let newargs_tuple: PyRef<super::PyTuple> = newargs.downcast().map_err(|obj| { + vm.new_type_error(format!( + "__getnewargs_ex__ should return a tuple, not '{}'", + obj.class().name() + )) + })?; + + if newargs_tuple.len() != 2 { + return Err(vm.new_value_error(format!( + "__getnewargs_ex__ should return a tuple of length 2, not {}", + newargs_tuple.len() + ))); + } + + let args = newargs_tuple.as_slice()[0].clone(); + let kwargs = newargs_tuple.as_slice()[1].clone(); + + let args_tuple: PyRef<super::PyTuple> = args.downcast().map_err(|obj| { + vm.new_type_error(format!( + "first item of the tuple returned by __getnewargs_ex__ must be a tuple, not '{}'", + obj.class().name() + )) + })?; + + let kwargs_dict: PyRef<super::PyDict> = kwargs.downcast().map_err(|obj| { + vm.new_type_error(format!( + "second item of the tuple returned by __getnewargs_ex__ must be a dict, not '{}'", + obj.class().name() + )) + })?; + + return Ok((Some(args_tuple), Some(kwargs_dict))); + } + + // Fall back to __getnewargs__ + if let Some(getnewargs) = vm.get_special_method(obj, identifier!(vm, __getnewargs__))? { + let args = getnewargs.invoke((), vm)?; + + let args_tuple: PyRef<super::PyTuple> = args.downcast().map_err(|obj| { + vm.new_type_error(format!( + "__getnewargs__ should return a tuple, not '{}'", + obj.class().name() + )) + })?; + + return Ok((Some(args_tuple), None)); + } + + // No __getnewargs_ex__ or __getnewargs__ + Ok((None, None)) +} + +/// Check if __getstate__ is overridden by comparing with object.__getstate__ +fn is_getstate_overridden(obj: &PyObject, vm: &VirtualMachine) -> bool { + let obj_cls = obj.class(); + let object_type = vm.ctx.types.object_type; + + // If the class is object itself, not overridden + if obj_cls.is(object_type) { + return false; + } + + // Check if __getstate__ in the MRO comes from object or elsewhere + // If the type has its own __getstate__, it's overridden + if let Some(getstate) = obj_cls.get_attr(identifier!(vm, __getstate__)) + && let Some(obj_getstate) = object_type.get_attr(identifier!(vm, __getstate__)) + { + return !getstate.is(&obj_getstate); + } + false +} + +/// object_getstate - calls __getstate__ method or default implementation +fn object_getstate(obj: &PyObject, required: bool, vm: &VirtualMachine) -> PyResult { + // If __getstate__ is not overridden, use the default implementation with required flag + if !is_getstate_overridden(obj, vm) { + return object_getstate_default(obj, required, vm); + } + + // __getstate__ is overridden, call it without required + let getstate = obj.get_attr(identifier!(vm, __getstate__), vm)?; + getstate.call((), vm) +} + +/// Get list items iterator if obj is a list (or subclass), None iterator otherwise +fn get_items_iter(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { + let listitems: PyObjectRef = if obj.fast_isinstance(vm.ctx.types.list_type) { + obj.get_iter(vm)?.into() + } else { + vm.ctx.none() + }; + + let dictitems: PyObjectRef = if obj.fast_isinstance(vm.ctx.types.dict_type) { + let items = vm.call_method(obj, "items", ())?; + items.get_iter(vm)?.into() + } else { + vm.ctx.none() + }; + + Ok((listitems, dictitems)) +} + +/// reduce_newobj - creates reduce tuple for protocol >= 2 +fn reduce_newobj(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Check if type has tp_new + let cls = obj.class(); + if cls.slots.new.load().is_none() { + return Err(vm.new_type_error(format!("cannot pickle '{}' object", cls.name()))); + } + + let (args, kwargs) = get_new_arguments(&obj, vm)?; + + let copyreg = vm.import("copyreg", 0)?; + + let has_args = args.is_some(); + + let (newobj, newargs): (PyObjectRef, PyObjectRef) = if kwargs.is_none() + || kwargs.as_ref().is_some_and(|k| k.is_empty()) + { + // Use copyreg.__newobj__ + let newobj = copyreg.get_attr("__newobj__", vm)?; + + let args_vec: Vec<PyObjectRef> = args.map(|a| a.as_slice().to_vec()).unwrap_or_default(); + + // Create (cls, *args) tuple + let mut newargs_vec: Vec<PyObjectRef> = vec![cls.to_owned().into()]; + newargs_vec.extend(args_vec); + let newargs = vm.ctx.new_tuple(newargs_vec); + + (newobj, newargs.into()) + } else { + // args == NULL with non-empty kwargs is BadInternalCall + let Some(args) = args else { + return Err(vm.new_system_error("bad internal call")); + }; + // Use copyreg.__newobj_ex__ + let newobj = copyreg.get_attr("__newobj_ex__", vm)?; + let args_tuple: PyObjectRef = args.into(); + let kwargs_dict: PyObjectRef = kwargs + .map(|k| k.into()) + .unwrap_or_else(|| vm.ctx.new_dict().into()); + + let newargs = vm + .ctx + .new_tuple(vec![cls.to_owned().into(), args_tuple, kwargs_dict]); + (newobj, newargs.into()) + }; + + // Determine if state is required + // required = !(has_args || is_list || is_dict) + let is_list = obj.fast_isinstance(vm.ctx.types.list_type); + let is_dict = obj.fast_isinstance(vm.ctx.types.dict_type); + let required = !(has_args || is_list || is_dict); + + let state = object_getstate(&obj, required, vm)?; + + let (listitems, dictitems) = get_items_iter(&obj, vm)?; + + let result = vm + .ctx + .new_tuple(vec![newobj, newargs, state, listitems, dictitems]); + Ok(result.into()) +} + +fn common_reduce(obj: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { + if proto >= 2 { + reduce_newobj(obj, vm) + } else { + let copyreg = vm.import("copyreg", 0)?; + let reduce_ex = copyreg.get_attr("_reduce_ex", vm)?; + reduce_ex.call((obj, proto), vm) + } +} diff --git a/crates/vm/src/builtins/property.rs b/crates/vm/src/builtins/property.rs new file mode 100644 index 00000000000..d01477dfcbf --- /dev/null +++ b/crates/vm/src/builtins/property.rs @@ -0,0 +1,416 @@ +/*! Python `property` descriptor class. + +*/ +use super::{PyStrRef, PyType}; +use crate::common::lock::PyRwLock; +use crate::function::{IntoFuncArgs, PosArgs}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + function::{FuncArgs, PySetterValue}, + types::{Constructor, GetDescriptor, Initializer}, +}; +use core::sync::atomic::{AtomicBool, Ordering}; + +#[pyclass(module = false, name = "property", traverse)] +#[derive(Debug)] +pub struct PyProperty { + getter: PyRwLock<Option<PyObjectRef>>, + setter: PyRwLock<Option<PyObjectRef>>, + deleter: PyRwLock<Option<PyObjectRef>>, + doc: PyRwLock<Option<PyObjectRef>>, + name: PyRwLock<Option<PyObjectRef>>, + #[pytraverse(skip)] + getter_doc: core::sync::atomic::AtomicBool, +} + +impl PyPayload for PyProperty { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.property_type + } +} + +#[derive(FromArgs)] +pub struct PropertyArgs { + #[pyarg(any, default)] + fget: Option<PyObjectRef>, + #[pyarg(any, default)] + fset: Option<PyObjectRef>, + #[pyarg(any, default)] + fdel: Option<PyObjectRef>, + #[pyarg(any, default)] + doc: Option<PyObjectRef>, + #[pyarg(any, default)] + name: Option<PyStrRef>, +} + +impl GetDescriptor for PyProperty { + fn descr_get( + zelf_obj: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let (zelf, obj) = Self::_unwrap(&zelf_obj, obj, vm)?; + if vm.is_none(&obj) { + Ok(zelf_obj) + } else if let Some(getter) = zelf.getter.read().clone() { + // Clone and release lock before calling Python code to prevent deadlock + getter.call((obj,), vm) + } else { + let error_msg = zelf.format_property_error(&obj, "getter", vm)?; + Err(vm.new_attribute_error(error_msg)) + } + } +} + +#[pyclass( + with(Constructor, Initializer, GetDescriptor), + flags(BASETYPE, HAS_WEAKREF) +)] +impl PyProperty { + // Helper method to get property name + // Returns the name if available, None if not found, or propagates errors + fn get_property_name(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + // First check if name was set via __set_name__ + if let Some(name) = self.name.read().clone() { + return Ok(Some(name)); + } + + // Clone and release lock before calling Python code to prevent deadlock + let Some(getter) = self.getter.read().clone() else { + return Ok(None); + }; + + match getter.get_attr("__name__", vm) { + Ok(name) => Ok(Some(name)), + Err(e) => { + // If it's an AttributeError from the getter, return None + // Otherwise, propagate the original exception (e.g., RuntimeError) + if e.class().is(vm.ctx.exceptions.attribute_error) { + Ok(None) + } else { + Err(e) + } + } + } + } + + // Descriptor methods + + #[pyslot] + fn descr_set( + zelf: &PyObject, + obj: PyObjectRef, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + let zelf = zelf.try_to_ref::<Self>(vm)?; + match value { + PySetterValue::Assign(value) => { + // Clone and release lock before calling Python code to prevent deadlock + if let Some(setter) = zelf.setter.read().clone() { + setter.call((obj, value), vm).map(drop) + } else { + let error_msg = zelf.format_property_error(&obj, "setter", vm)?; + Err(vm.new_attribute_error(error_msg)) + } + } + PySetterValue::Delete => { + // Clone and release lock before calling Python code to prevent deadlock + if let Some(deleter) = zelf.deleter.read().clone() { + deleter.call((obj,), vm).map(drop) + } else { + let error_msg = zelf.format_property_error(&obj, "deleter", vm)?; + Err(vm.new_attribute_error(error_msg)) + } + } + } + } + + // Access functions + + #[pygetset] + fn fget(&self) -> Option<PyObjectRef> { + self.getter.read().clone() + } + + pub(crate) fn get_fget(&self) -> Option<PyObjectRef> { + self.getter.read().clone() + } + + #[pygetset] + fn fset(&self) -> Option<PyObjectRef> { + self.setter.read().clone() + } + + #[pygetset] + fn fdel(&self) -> Option<PyObjectRef> { + self.deleter.read().clone() + } + + #[pygetset(name = "__name__")] + fn name_getter(&self, vm: &VirtualMachine) -> PyResult { + match self.get_property_name(vm)? { + Some(name) => Ok(name), + None => Err(vm.new_attribute_error("'property' object has no attribute '__name__'")), + } + } + + #[pygetset(name = "__name__", setter)] + fn name_setter(&self, value: PyObjectRef) { + *self.name.write() = Some(value); + } + + fn doc_getter(&self) -> Option<PyObjectRef> { + self.doc.read().clone() + } + fn doc_setter(&self, value: Option<PyObjectRef>) { + *self.doc.write() = value; + } + + #[pymethod] + fn __set_name__(&self, args: PosArgs, vm: &VirtualMachine) -> PyResult<()> { + let func_args = args.into_args(vm); + let func_args_len = func_args.args.len(); + let (_owner, name): (PyObjectRef, PyObjectRef) = func_args.bind(vm).map_err(|_e| { + vm.new_type_error(format!( + "__set_name__() takes 2 positional arguments but {func_args_len} were given" + )) + })?; + + *self.name.write() = Some(name); + + Ok(()) + } + + // Python builder functions + + // Helper method to create a new property with updated attributes + fn clone_property_with( + zelf: PyRef<Self>, + new_getter: Option<PyObjectRef>, + new_setter: Option<PyObjectRef>, + new_deleter: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + // Determine doc based on getter_doc flag and whether we're updating the getter + let doc = if zelf.getter_doc.load(Ordering::Relaxed) && new_getter.is_some() { + // If the original property uses getter doc and we have a new getter, + // pass Py_None to let __init__ get the doc from the new getter + Some(vm.ctx.none()) + } else if zelf.getter_doc.load(Ordering::Relaxed) { + // If original used getter_doc but we're not changing the getter, + // pass None to let init get doc from existing getter + Some(vm.ctx.none()) + } else { + // Otherwise use the existing doc + zelf.doc_getter() + }; + + // Create property args with updated values + let args = PropertyArgs { + fget: new_getter.or_else(|| zelf.fget()), + fset: new_setter.or_else(|| zelf.fset()), + fdel: new_deleter.or_else(|| zelf.fdel()), + doc, + name: None, + }; + + // Create new property using py_new and init + let new_prop = Self::slot_new(zelf.class().to_owned(), FuncArgs::default(), vm)?; + let new_prop_ref = new_prop.downcast::<Self>().unwrap(); + Self::init(new_prop_ref.clone(), args, vm)?; + + // Copy the name if it exists + if let Some(name) = zelf.name.read().clone() { + *new_prop_ref.name.write() = Some(name); + } + + Ok(new_prop_ref) + } + + #[pymethod] + fn getter( + zelf: PyRef<Self>, + getter: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + Self::clone_property_with(zelf, getter, None, None, vm) + } + + #[pymethod] + fn setter( + zelf: PyRef<Self>, + setter: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + Self::clone_property_with(zelf, None, setter, None, vm) + } + + #[pymethod] + fn deleter( + zelf: PyRef<Self>, + deleter: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + Self::clone_property_with(zelf, None, None, deleter, vm) + } + + #[pygetset] + fn __isabstractmethod__(&self, vm: &VirtualMachine) -> PyResult { + // Helper to check if a method is abstract + let is_abstract = |method: &PyObject| -> PyResult<bool> { + match method.get_attr("__isabstractmethod__", vm) { + Ok(isabstract) => isabstract.try_to_bool(vm), + Err(_) => Ok(false), + } + }; + + // Clone and release lock before calling Python code to prevent deadlock + // Check getter + if let Some(getter) = self.getter.read().clone() + && is_abstract(&getter)? + { + return Ok(vm.ctx.new_bool(true).into()); + } + + // Check setter + if let Some(setter) = self.setter.read().clone() + && is_abstract(&setter)? + { + return Ok(vm.ctx.new_bool(true).into()); + } + + // Check deleter + if let Some(deleter) = self.deleter.read().clone() + && is_abstract(&deleter)? + { + return Ok(vm.ctx.new_bool(true).into()); + } + + Ok(vm.ctx.new_bool(false).into()) + } + + #[pygetset(setter)] + fn set___isabstractmethod__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Clone and release lock before calling Python code to prevent deadlock + if let Some(getter) = self.getter.read().clone() { + getter.set_attr("__isabstractmethod__", value, vm)?; + } + Ok(()) + } + + // Helper method to format property error messages + #[cold] + fn format_property_error( + &self, + obj: &PyObject, + error_type: &str, + vm: &VirtualMachine, + ) -> PyResult<String> { + let prop_name = self.get_property_name(vm)?; + let obj_type = obj.class(); + let qualname = obj_type.__qualname__(vm); + + match prop_name { + Some(name) => Ok(format!( + "property {} of {} object has no {}", + name.repr(vm)?, + qualname.repr(vm)?, + error_type + )), + None => Ok(format!( + "property of {} object has no {}", + qualname.repr(vm)?, + error_type + )), + } + } +} + +impl Constructor for PyProperty { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: FuncArgs, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + getter: PyRwLock::new(None), + setter: PyRwLock::new(None), + deleter: PyRwLock::new(None), + doc: PyRwLock::new(None), + name: PyRwLock::new(None), + getter_doc: AtomicBool::new(false), + }) + } +} + +impl Initializer for PyProperty { + type Args = PropertyArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Set doc and getter_doc flag + let mut getter_doc = false; + + // Helper to get doc from getter + let get_getter_doc = |fget: &PyObject| -> Option<PyObjectRef> { + fget.get_attr("__doc__", vm) + .ok() + .filter(|doc| !vm.is_none(doc)) + }; + + let doc = match args.doc { + Some(doc) if !vm.is_none(&doc) => Some(doc), + _ => { + // No explicit doc or doc is None, try to get from getter + args.fget.as_ref().and_then(|fget| { + get_getter_doc(fget).inspect(|_| { + getter_doc = true; + }) + }) + } + }; + + // Check if this is a property subclass + let is_exact_property = zelf.class().is(vm.ctx.types.property_type); + + if is_exact_property { + // For exact property type, store doc in the field + *zelf.doc.write() = doc; + } else { + // For property subclass, set __doc__ as an attribute + let doc_to_set = doc.unwrap_or_else(|| vm.ctx.none()); + match zelf.as_object().set_attr("__doc__", doc_to_set, vm) { + Ok(()) => {} + Err(e) if !getter_doc && e.class().is(vm.ctx.exceptions.attribute_error) => { + // Silently ignore AttributeError for backwards compatibility + // (only when not using getter_doc) + } + Err(e) => return Err(e), + } + } + + *zelf.getter.write() = args.fget; + *zelf.setter.write() = args.fset; + *zelf.deleter.write() = args.fdel; + *zelf.name.write() = args.name.map(|a| a.as_object().to_owned()); + zelf.getter_doc.store(getter_doc, Ordering::Relaxed); + + Ok(()) + } +} + +pub(crate) fn init(context: &'static Context) { + PyProperty::extend_class(context, context.types.property_type); + + // This is a bit unfortunate, but this instance attribute overlaps with the + // class __doc__ string.. + extend_class!(context, context.types.property_type, { + "__doc__" => context.new_static_getset( + "__doc__", + context.types.property_type, + PyProperty::doc_getter, + PyProperty::doc_setter, + ), + }); +} diff --git a/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs similarity index 75% rename from vm/src/builtins/range.rs rename to crates/vm/src/builtins/range.rs index 348924a7a97..153a82bb43b 100644 --- a/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -1,25 +1,27 @@ use super::{ - builtins_iter, tuple::tuple_hash, PyInt, PyIntRef, PySlice, PyTupleRef, PyType, PyTypeRef, + PyGenericAlias, PyInt, PyIntRef, PySlice, PyTupleRef, PyType, PyTypeRef, builtins_iter, + tuple::tuple_hash, }; +use crate::common::lock::LazyLock; use crate::{ - atomic_func, + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, atomic_func, class::PyClassImpl, common::hash::PyHash, function::{ArgIndex, FuncArgs, OptionalArg, PyComparisonValue}, - protocol::{PyIterReturn, PyMappingMethods, PySequenceMethods}, + protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, types::{ - AsMapping, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, + AsMapping, AsNumber, AsSequence, Comparable, Hashable, IterNext, Iterable, PyComparisonOp, + Representable, SelfIter, }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, }; +use core::cell::Cell; +use core::cmp::max; +use core::ptr::NonNull; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Signed, ToPrimitive, Zero}; -use once_cell::sync::Lazy; -use std::cmp::max; // Search flag passed to iter_search enum SearchType { @@ -28,19 +30,17 @@ enum SearchType { Index, } -// Note: might be a good idea to merge with _membership_iter_search or generalize (_sequence_iter_check?) -// and place in vm.rs for all sequences to be able to use it. #[inline] fn iter_search( - obj: PyObjectRef, - item: PyObjectRef, + obj: &PyObject, + item: &PyObject, flag: SearchType, vm: &VirtualMachine, ) -> PyResult<usize> { let mut count = 0; let iter = obj.get_iter(vm)?; for element in iter.iter_without_hint::<PyObjectRef>(vm)? { - if vm.bool_eq(&item, &*element?)? { + if vm.bool_eq(item, &*element?)? { match flag { SearchType::Index => return Ok(count), SearchType::Contains => return Ok(1), @@ -53,10 +53,10 @@ fn iter_search( SearchType::Contains => Ok(0), SearchType::Index => Err(vm.new_value_error(format!( "{} not in range", - &item - .repr(vm) - .map(|v| v.as_str().to_owned()) - .unwrap_or_else(|_| "value".to_owned()) + item.repr(vm) + .as_ref() + .map_or("value".as_ref(), |s| s.as_wtf8()) + .to_owned() ))), } } @@ -69,10 +69,49 @@ pub struct PyRange { pub step: PyIntRef, } +// spell-checker:ignore MAXFREELIST +thread_local! { + static RANGE_FREELIST: Cell<crate::object::FreeList<PyRange>> = const { Cell::new(crate::object::FreeList::new()) }; +} + impl PyPayload for PyRange { + const MAX_FREELIST: usize = 6; + const HAS_FREELIST: bool = true; + + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.range_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + RANGE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> { + RANGE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } } impl PyRange { @@ -165,19 +204,31 @@ impl PyRange { } // pub fn get_value(obj: &PyObject) -> PyRange { -// obj.payload::<PyRange>().unwrap().clone() +// obj.downcast_ref::<PyRange>().unwrap().clone() // } -pub fn init(context: &Context) { +pub fn init(context: &'static Context) { PyRange::extend_class(context, context.types.range_type); PyLongRangeIterator::extend_class(context, context.types.long_range_iterator_type); PyRangeIterator::extend_class(context, context.types.range_iterator_type); } -#[pyclass(with(AsMapping, AsSequence, Hashable, Comparable, Iterable, Representable))] +#[pyclass( + with( + Py, + AsMapping, + AsNumber, + AsSequence, + Hashable, + Comparable, + Iterable, + Representable + ), + flags(SEQUENCE) +)] impl PyRange { fn new(cls: PyTypeRef, stop: ArgIndex, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - PyRange { + Self { start: vm.ctx.new_pyref(0), stop: stop.into(), step: vm.ctx.new_pyref(1), @@ -194,9 +245,9 @@ impl PyRange { ) -> PyResult<PyRef<Self>> { let step = step.map_or_else(|| vm.ctx.new_int(1), |step| step.into()); if step.as_bigint().is_zero() { - return Err(vm.new_value_error("range() arg 3 must not be zero".to_owned())); + return Err(vm.new_value_error("range() arg 3 must not be zero")); } - PyRange { + Self { start: start.try_index(vm)?, stop: stop.try_index(vm)?, step, @@ -219,13 +270,13 @@ impl PyRange { self.step.clone() } - #[pymethod(magic)] - fn reversed(&self, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn __reversed__(&self, vm: &VirtualMachine) -> PyResult { let start = self.start.as_bigint(); let step = self.step.as_bigint(); // Use CPython calculation for this: - let length = self.len(); + let length = self.__len__(); let new_stop = start - step; let start = &new_stop + length.clone() * step; let step = -step; @@ -255,38 +306,12 @@ impl PyRange { ) } - #[pymethod(magic)] - fn len(&self) -> BigInt { + fn __len__(&self) -> BigInt { self.compute_length() } - #[pymethod(magic)] - fn bool(&self) -> bool { - !self.is_empty() - } - - #[pymethod(magic)] - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> bool { - // Only accept ints, not subclasses. - if let Some(int) = needle.payload_if_exact::<PyInt>(vm) { - match self.offset(int.as_bigint()) { - Some(ref offset) => offset.is_multiple_of(self.step.as_bigint()), - None => false, - } - } else { - iter_search( - self.clone().into_pyobject(vm), - needle, - SearchType::Contains, - vm, - ) - .unwrap_or(0) - != 0 - } - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef) { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef) { let range_parameters: Vec<PyObjectRef> = [&self.start, &self.stop, &self.step] .iter() .map(|x| x.as_object().to_owned()) @@ -295,45 +320,7 @@ impl PyRange { (vm.ctx.types.range_type.to_owned(), range_parameters_tuple) } - #[pymethod] - fn index(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<BigInt> { - if let Ok(int) = needle.clone().downcast::<PyInt>() { - match self.index_of(int.as_bigint()) { - Some(idx) => Ok(idx), - None => Err(vm.new_value_error(format!("{int} is not in range"))), - } - } else { - // Fallback to iteration. - Ok(BigInt::from_bytes_be( - Sign::Plus, - &iter_search( - self.clone().into_pyobject(vm), - needle, - SearchType::Index, - vm, - )? - .to_be_bytes(), - )) - } - } - - #[pymethod] - fn count(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - if let Ok(int) = item.clone().downcast::<PyInt>() { - if self.index_of(int.as_bigint()).is_some() { - Ok(1) - } else { - Ok(0) - } - } else { - // Dealing with classes who might compare equal with ints in their - // __eq__, slow search. - iter_search(self.clone().into_pyobject(vm), item, SearchType::Count, vm) - } - } - - #[pymethod(magic)] - fn getitem(&self, subscript: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn __getitem__(&self, subscript: PyObjectRef, vm: &VirtualMachine) -> PyResult { match RangeIndex::try_from_object(vm, subscript)? { RangeIndex::Slice(slice) => { let (mut sub_start, mut sub_stop, mut sub_step) = @@ -345,7 +332,7 @@ impl PyRange { sub_start = (sub_start * range_step.as_bigint()) + range_start.as_bigint(); sub_stop = (sub_stop * range_step.as_bigint()) + range_start.as_bigint(); - Ok(PyRange { + Ok(Self { start: vm.ctx.new_pyref(sub_start), stop: vm.ctx.new_pyref(sub_stop), step: vm.ctx.new_pyref(sub_step), @@ -355,7 +342,7 @@ impl PyRange { } RangeIndex::Int(index) => match self.get(index.as_bigint()) { Some(value) => Ok(vm.ctx.new_int(value).into()), - None => Err(vm.new_index_error("range object index out of range".to_owned())), + None => Err(vm.new_index_error("range object index out of range")), }, } } @@ -364,19 +351,76 @@ impl PyRange { fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { let range = if args.args.len() <= 1 { let stop = args.bind(vm)?; - PyRange::new(cls, stop, vm) + Self::new(cls, stop, vm) } else { let (start, stop, step) = args.bind(vm)?; - PyRange::new_from(cls, start, stop, step, vm) + Self::new_from(cls, start, stop, step, vm) }?; Ok(range.into()) } + + // TODO: Uncomment when Python adds __class_getitem__ to range + // #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +#[pyclass] +impl Py<PyRange> { + fn contains_inner(&self, needle: &PyObject, vm: &VirtualMachine) -> bool { + // Only accept ints, not subclasses. + if let Some(int) = needle.downcast_ref_if_exact::<PyInt>(vm) { + match self.offset(int.as_bigint()) { + Some(ref offset) => offset.is_multiple_of(self.step.as_bigint()), + None => false, + } + } else { + iter_search(self.as_object(), needle, SearchType::Contains, vm).unwrap_or(0) != 0 + } + } + + fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> bool { + self.contains_inner(&needle, vm) + } + + #[pymethod] + fn index(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<BigInt> { + if let Ok(int) = needle.clone().downcast::<PyInt>() { + match self.index_of(int.as_bigint()) { + Some(idx) => Ok(idx), + None => Err(vm.new_value_error(format!("{int} is not in range"))), + } + } else { + // Fallback to iteration. + Ok(BigInt::from_bytes_be( + Sign::Plus, + &iter_search(self.as_object(), &needle, SearchType::Index, vm)?.to_be_bytes(), + )) + } + } + + #[pymethod] + fn count(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + if let Ok(int) = item.clone().downcast::<PyInt>() { + let count = if self.index_of(int.as_bigint()).is_some() { + 1 + } else { + 0 + }; + Ok(count) + } else { + // Dealing with classes who might compare equal with ints in their + // __eq__, slow search. + iter_search(self.as_object(), &item, SearchType::Count, vm) + } + } } impl PyRange { fn protocol_length(&self, vm: &VirtualMachine) -> PyResult<usize> { - PyInt::from(self.len()) + PyInt::from(self.__len__()) .try_to_primitive::<isize>(vm) .map(|x| x as usize) } @@ -384,12 +428,12 @@ impl PyRange { impl AsMapping for PyRange { fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy<PyMappingMethods> = Lazy::new(|| PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { length: atomic_func!( |mapping, vm| PyRange::mapping_downcast(mapping).protocol_length(vm) ), subscript: atomic_func!(|mapping, needle, vm| { - PyRange::mapping_downcast(mapping).getitem(needle.to_owned(), vm) + PyRange::mapping_downcast(mapping).__getitem__(needle.to_owned(), vm) }), ..PyMappingMethods::NOT_IMPLEMENTED }); @@ -399,16 +443,16 @@ impl AsMapping for PyRange { impl AsSequence for PyRange { fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { length: atomic_func!(|seq, vm| PyRange::sequence_downcast(seq).protocol_length(vm)), item: atomic_func!(|seq, i, vm| { PyRange::sequence_downcast(seq) .get(&i.into()) .map(|x| PyInt::from(x).into_ref(&vm.ctx).into()) - .ok_or_else(|| vm.new_index_error("index out of range".to_owned())) + .ok_or_else(|| vm.new_index_error("index out of range")) }), contains: atomic_func!(|seq, needle, vm| { - Ok(PyRange::sequence_downcast(seq).contains(needle.to_owned(), vm)) + Ok(PyRange::sequence_downcast(seq).contains_inner(needle, vm)) }), ..PySequenceMethods::NOT_IMPLEMENTED }); @@ -416,6 +460,19 @@ impl AsSequence for PyRange { } } +impl AsNumber for PyRange { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyRange>().unwrap(); + Ok(!zelf.is_empty()) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + impl Hashable for PyRange { fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { let length = zelf.compute_length(); @@ -473,7 +530,7 @@ impl Iterable for PyRange { zelf.start.as_bigint(), zelf.stop.as_bigint(), zelf.step.as_bigint(), - zelf.len(), + zelf.__len__(), ); if let (Some(start), Some(step), Some(_), Some(_)) = ( start.to_isize(), @@ -532,15 +589,16 @@ pub struct PyLongRangeIterator { } impl PyPayload for PyLongRangeIterator { + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.long_range_iterator_type } } -#[pyclass(with(Constructor, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyLongRangeIterator { - #[pymethod(magic)] - fn length_hint(&self) -> BigInt { + #[pymethod] + fn __length_hint__(&self) -> BigInt { let index = BigInt::from(self.index.load()); if index < self.length { self.length.clone() - index @@ -549,14 +607,14 @@ impl PyLongRangeIterator { } } - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.index.store(range_state(&self.length, state, vm)?); Ok(()) } - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { range_iter_reduce( self.start.clone(), self.length.clone(), @@ -566,7 +624,6 @@ impl PyLongRangeIterator { ) } } -impl Unconstructible for PyLongRangeIterator {} impl SelfIter for PyLongRangeIterator {} impl IterNext for PyLongRangeIterator { @@ -597,32 +654,29 @@ pub struct PyRangeIterator { } impl PyPayload for PyRangeIterator { + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.range_iterator_type } } -#[pyclass(with(Constructor, IterNext, Iterable))] +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyRangeIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { + #[pymethod] + fn __length_hint__(&self) -> usize { let index = self.index.load(); - if index < self.length { - self.length - index - } else { - 0 - } + self.length.saturating_sub(index) } - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { self.index .store(range_state(&BigInt::from(self.length), state, vm)?); Ok(()) } - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { range_iter_reduce( BigInt::from(self.start), BigInt::from(self.length), @@ -632,20 +686,26 @@ impl PyRangeIterator { ) } } -impl Unconstructible for PyRangeIterator {} + +impl PyRangeIterator { + /// Fast path for FOR_ITER specialization. Returns the next isize value + /// without allocating PyInt or PyIterReturn. + pub(crate) fn fast_next(&self) -> Option<isize> { + let index = self.index.fetch_add(1); + if index < self.length { + Some(self.start + (index as isize) * self.step) + } else { + None + } + } +} impl SelfIter for PyRangeIterator {} impl IterNext for PyRangeIterator { fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - // TODO: In pathological case (index == usize::MAX) this can wrap around - // (since fetch_add wraps). This would result in the iterator spinning again - // from the beginning. - let index = zelf.index.fetch_add(1); - let r = if index < zelf.length { - let value = zelf.start + (index as isize) * zelf.step; - PyIterReturn::Return(vm.ctx.new_int(value).into()) - } else { - PyIterReturn::StopIteration(None) + let r = match zelf.fast_next() { + Some(value) => PyIterReturn::Return(vm.ctx.new_int(value).into()), + None => PyIterReturn::StopIteration(None), }; Ok(r) } @@ -658,7 +718,7 @@ fn range_iter_reduce( index: usize, vm: &VirtualMachine, ) -> PyResult<PyTupleRef> { - let iter = builtins_iter(vm).to_owned(); + let iter = builtins_iter(vm); let stop = start.clone() + length * step.clone(); let range = PyRange { start: PyInt::from(start).into_ref(&vm.ctx), @@ -670,7 +730,7 @@ fn range_iter_reduce( // Silently clips state (i.e index) in range [0, usize::MAX]. fn range_state(length: &BigInt, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - if let Some(i) = state.payload::<PyInt>() { + if let Some(i) = state.downcast_ref::<PyInt>() { let mut index = i.as_bigint(); let max_usize = BigInt::from(usize::MAX); if index > length { @@ -678,7 +738,7 @@ fn range_state(length: &BigInt, state: PyObjectRef, vm: &VirtualMachine) -> PyRe } Ok(index.to_usize().unwrap_or(0)) } else { - Err(vm.new_type_error("an integer is required.".to_owned())) + Err(vm.new_type_error("an integer is required.")) } } @@ -690,14 +750,14 @@ pub enum RangeIndex { impl TryFromObject for RangeIndex { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { match_class!(match obj { - i @ PyInt => Ok(RangeIndex::Int(i)), - s @ PySlice => Ok(RangeIndex::Slice(s)), + i @ PyInt => Ok(Self::Int(i)), + s @ PySlice => Ok(Self::Slice(s)), obj => { let val = obj.try_index(vm).map_err(|_| vm.new_type_error(format!( "sequence indices be integers or slices or classes that override __index__ operator, not '{}'", obj.class().name() )))?; - Ok(RangeIndex::Int(val)) + Ok(Self::Int(val)) } }) } diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs new file mode 100644 index 00000000000..85e6b37fab0 --- /dev/null +++ b/crates/vm/src/builtins/set.rs @@ -0,0 +1,1428 @@ +/* + * Builtin set type with a sequence of unique items. + */ +use super::{ + IterStatus, PositionIterInternal, PyDict, PyDictRef, PyGenericAlias, PyTupleRef, PyType, + PyTypeRef, builtins_iter, +}; +use crate::common::lock::LazyLock; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + atomic_func, + class::PyClassImpl, + common::{ascii, hash::PyHash, lock::PyMutex, rc::PyRc, wtf8::Wtf8Buf}, + convert::ToPyResult, + dict_inner::{self, DictSize}, + function::{ArgIterable, FuncArgs, OptionalArg, PosArgs, PyArithmeticValue, PyComparisonValue}, + protocol::{PyIterReturn, PyNumberMethods, PySequenceMethods}, + recursion::ReprGuard, + types::AsNumber, + types::{ + AsSequence, Comparable, Constructor, DefaultConstructor, Hashable, Initializer, IterNext, + Iterable, PyComparisonOp, Representable, SelfIter, + }, + utils::collection_repr, + vm::VirtualMachine, +}; +use alloc::fmt; +use core::borrow::Borrow; +use core::ops::Deref; +use rustpython_common::{ + atomic::{Ordering, PyAtomic, Radium}, + hash, +}; + +pub type SetContentType = dict_inner::Dict<()>; + +#[pyclass(module = false, name = "set", unhashable = true, traverse)] +#[derive(Default)] +pub struct PySet { + pub(super) inner: PySetInner, +} + +impl PySet { + #[deprecated(note = "Use `PySet::default().into_ref(ctx)` instead")] + pub fn new_ref(ctx: &Context) -> PyRef<Self> { + Self::default().into_ref(ctx) + } + + pub fn elements(&self) -> Vec<PyObjectRef> { + self.inner.elements() + } + + fn fold_op( + &self, + others: impl core::iter::Iterator<Item = ArgIterable>, + op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, + vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + inner: self.inner.fold_op(others, op, vm)?, + }) + } + + fn op( + &self, + other: AnySet, + op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, + vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + inner: self + .inner + .fold_op(core::iter::once(other.into_iterable(vm)?), op, vm)?, + }) + } +} + +#[pyclass(module = false, name = "frozenset", unhashable = true)] +pub struct PyFrozenSet { + inner: PySetInner, + hash: PyAtomic<PyHash>, +} + +impl Default for PyFrozenSet { + fn default() -> Self { + Self { + inner: PySetInner::default(), + hash: hash::SENTINEL.into(), + } + } +} + +impl PyFrozenSet { + // Also used by ssl.rs windows. + pub fn from_iter( + vm: &VirtualMachine, + it: impl IntoIterator<Item = PyObjectRef>, + ) -> PyResult<Self> { + let inner = PySetInner::default(); + for elem in it { + inner.add(elem, vm)?; + } + // FIXME: empty set check + Ok(Self { + inner, + ..Default::default() + }) + } + + pub fn elements(&self) -> Vec<PyObjectRef> { + self.inner.elements() + } + + fn fold_op( + &self, + others: impl core::iter::Iterator<Item = ArgIterable>, + op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, + vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + inner: self.inner.fold_op(others, op, vm)?, + ..Default::default() + }) + } + + fn op( + &self, + other: AnySet, + op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, + vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + inner: self + .inner + .fold_op(core::iter::once(other.into_iterable(vm)?), op, vm)?, + ..Default::default() + }) + } +} + +impl fmt::Debug for PySet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO: implement more detailed, non-recursive Debug formatter + f.write_str("set") + } +} + +impl fmt::Debug for PyFrozenSet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO: implement more detailed, non-recursive Debug formatter + f.write_str("PyFrozenSet ")?; + f.debug_set().entries(self.elements().iter()).finish() + } +} + +impl PyPayload for PySet { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.set_type + } +} + +impl PyPayload for PyFrozenSet { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.frozenset_type + } +} + +#[derive(Default, Clone)] +pub(super) struct PySetInner { + content: PyRc<SetContentType>, +} + +unsafe impl crate::object::Traverse for PySetInner { + fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn<'_>) { + // FIXME(discord9): Rc means shared ref, so should it be traced? + self.content.traverse(tracer_fn) + } +} + +impl PySetInner { + pub(super) fn from_iter<T>(iter: T, vm: &VirtualMachine) -> PyResult<Self> + where + T: IntoIterator<Item = PyResult<PyObjectRef>>, + { + let set = Self::default(); + for item in iter { + set.add(item?, vm)?; + } + Ok(set) + } + + fn fold_op<O>( + &self, + others: impl core::iter::Iterator<Item = O>, + op: fn(&Self, O, &VirtualMachine) -> PyResult<Self>, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let mut res = self.copy(); + for other in others { + res = op(&res, other, vm)?; + } + Ok(res) + } + + fn len(&self) -> usize { + self.content.len() + } + + fn sizeof(&self) -> usize { + self.content.sizeof() + } + + fn copy(&self) -> Self { + Self { + content: PyRc::new((*self.content).clone()), + } + } + + fn contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + self.retry_op_with_frozenset(needle, vm, |needle, vm| self.content.contains(vm, needle)) + } + + fn compare(&self, other: &Self, op: PyComparisonOp, vm: &VirtualMachine) -> PyResult<bool> { + if op == PyComparisonOp::Ne { + return self.compare(other, PyComparisonOp::Eq, vm).map(|eq| !eq); + } + if !op.eval_ord(self.len().cmp(&other.len())) { + return Ok(false); + } + + let (superset, subset) = match op { + PyComparisonOp::Lt | PyComparisonOp::Le => (other, self), + _ => (self, other), + }; + + for key in subset.elements() { + if !superset.contains(&key, vm)? { + return Ok(false); + } + } + Ok(true) + } + + pub(super) fn union(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<Self> { + let set = self.clone(); + for item in other.iter(vm)? { + set.add(item?, vm)?; + } + + Ok(set) + } + + pub(super) fn intersection(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<Self> { + let set = Self::default(); + for item in other.iter(vm)? { + let obj = item?; + if self.contains(&obj, vm)? { + set.add(obj, vm)?; + } + } + Ok(set) + } + + pub(super) fn difference(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<Self> { + let set = self.copy(); + for item in other.iter(vm)? { + set.content.delete_if_exists(vm, &*item?)?; + } + Ok(set) + } + + pub(super) fn symmetric_difference( + &self, + other: ArgIterable, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let new_inner = self.clone(); + + // We want to remove duplicates in other + let other_set = Self::from_iter(other.iter(vm)?, vm)?; + + for item in other_set.elements() { + new_inner.content.delete_or_insert(vm, &item, ())? + } + + Ok(new_inner) + } + + fn issuperset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + for item in other.iter(vm)? { + if !self.contains(&*item?, vm)? { + return Ok(false); + } + } + Ok(true) + } + + fn issubset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + let other_set = Self::from_iter(other.iter(vm)?, vm)?; + self.compare(&other_set, PyComparisonOp::Le, vm) + } + + pub(super) fn isdisjoint(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + for item in other.iter(vm)? { + if self.contains(&*item?, vm)? { + return Ok(false); + } + } + Ok(true) + } + + fn iter(&self) -> PySetIterator { + PySetIterator { + size: self.content.size(), + internal: PyMutex::new(PositionIterInternal::new(self.content.clone(), 0)), + } + } + + fn repr(&self, class_name: Option<&str>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + collection_repr(class_name, "{", "}", self.elements().iter(), vm) + } + + fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.content.insert(vm, &*item, ()) + } + + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.retry_op_with_frozenset(&item, vm, |item, vm| self.content.delete(vm, item)) + } + + fn discard(&self, item: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + self.retry_op_with_frozenset(item, vm, |item, vm| self.content.delete_if_exists(vm, item)) + } + + fn clear(&self) { + self.content.clear() + } + + fn elements(&self) -> Vec<PyObjectRef> { + self.content.keys() + } + + fn pop(&self, vm: &VirtualMachine) -> PyResult { + // TODO: should be pop_front, but that requires rearranging every index + if let Some((key, _)) = self.content.pop_back() { + Ok(key) + } else { + let err_msg = vm.ctx.new_str(ascii!("pop from an empty set")).into(); + Err(vm.new_key_error(err_msg)) + } + } + + fn update( + &self, + others: impl core::iter::Iterator<Item = ArgIterable>, + vm: &VirtualMachine, + ) -> PyResult<()> { + for iterable in others { + for item in iterable.iter(vm)? { + self.add(item?, vm)?; + } + } + Ok(()) + } + + fn update_internal(&self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // check AnySet + if let Ok(any_set) = AnySet::try_from_object(vm, iterable.to_owned()) { + self.merge_set(any_set, vm) + // check Dict + } else if let Ok(dict) = iterable.to_owned().downcast_exact::<PyDict>(vm) { + self.merge_dict(dict.into_pyref(), vm) + } else { + // add iterable that is not AnySet or Dict + for item in iterable.try_into_value::<ArgIterable>(vm)?.iter(vm)? { + self.add(item?, vm)?; + } + Ok(()) + } + } + + fn merge_set(&self, any_set: AnySet, vm: &VirtualMachine) -> PyResult<()> { + for item in any_set.as_inner().elements() { + self.add(item, vm)?; + } + Ok(()) + } + + fn merge_dict(&self, dict: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { + for (key, _value) in dict { + self.add(key, vm)?; + } + Ok(()) + } + + fn intersection_update( + &self, + others: impl core::iter::Iterator<Item = ArgIterable>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let temp_inner = self.fold_op(others, Self::intersection, vm)?; + self.clear(); + for obj in temp_inner.elements() { + self.add(obj, vm)?; + } + Ok(()) + } + + fn difference_update( + &self, + others: impl core::iter::Iterator<Item = ArgIterable>, + vm: &VirtualMachine, + ) -> PyResult<()> { + for iterable in others { + let items = iterable.iter(vm)?.collect::<Result<Vec<_>, _>>()?; + for item in items { + self.content.delete_if_exists(vm, &*item)?; + } + } + Ok(()) + } + + fn symmetric_difference_update( + &self, + others: impl core::iter::Iterator<Item = ArgIterable>, + vm: &VirtualMachine, + ) -> PyResult<()> { + for iterable in others { + // We want to remove duplicates in iterable + let iterable_set = Self::from_iter(iterable.iter(vm)?, vm)?; + for item in iterable_set.elements() { + self.content.delete_or_insert(vm, &item, ())?; + } + } + Ok(()) + } + + fn hash(&self, vm: &VirtualMachine) -> PyResult<PyHash> { + // Work to increase the bit dispersion for closely spaced hash values. + // This is important because some use cases have many combinations of a + // small number of elements with nearby hashes so that many distinct + // combinations collapse to only a handful of distinct hash values. + const fn _shuffle_bits(h: u64) -> u64 { + ((h ^ 89869747) ^ (h.wrapping_shl(16))).wrapping_mul(3644798167) + } + // Factor in the number of active entries + let mut hash: u64 = (self.len() as u64 + 1).wrapping_mul(1927868237); + // Xor-in shuffled bits from every entry's hash field because xor is + // commutative and a frozenset hash should be independent of order. + hash = self.content.try_fold_keys(hash, |h, element| { + Ok(h ^ _shuffle_bits(element.hash(vm)? as u64)) + })?; + // Disperse patterns arising in nested frozen-sets + hash ^= (hash >> 11) ^ (hash >> 25); + hash = hash.wrapping_mul(69069).wrapping_add(907133923); + // -1 is reserved as an error code + if hash == u64::MAX { + hash = 590923713; + } + Ok(hash as PyHash) + } + + // Run operation, on failure, if item is a set/set subclass, convert it + // into a frozenset and try the operation again. Propagates original error + // on failure to convert and restores item in KeyError on failure (remove). + fn retry_op_with_frozenset<T, F>( + &self, + item: &PyObject, + vm: &VirtualMachine, + op: F, + ) -> PyResult<T> + where + F: Fn(&PyObject, &VirtualMachine) -> PyResult<T>, + { + op(item, vm).or_else(|original_err| { + item.downcast_ref::<PySet>() + // Keep original error around. + .ok_or(original_err) + .and_then(|set| { + op( + &PyFrozenSet { + inner: set.inner.copy(), + ..Default::default() + } + .into_pyobject(vm), + vm, + ) + // If operation raised KeyError, report original set (set.remove) + .map_err(|op_err| { + if op_err.fast_isinstance(vm.ctx.exceptions.key_error) { + vm.new_key_error(item.to_owned()) + } else { + op_err + } + }) + }) + }) + } +} + +fn extract_set(obj: &PyObject) -> Option<&PySetInner> { + match_class!(match obj { + ref set @ PySet => Some(&set.inner), + ref frozen @ PyFrozenSet => Some(&frozen.inner), + _ => None, + }) +} + +fn reduce_set( + zelf: &PyObject, + vm: &VirtualMachine, +) -> PyResult<(PyTypeRef, PyTupleRef, Option<PyDictRef>)> { + Ok(( + zelf.class().to_owned(), + vm.new_tuple((extract_set(zelf) + .unwrap_or(&PySetInner::default()) + .elements(),)), + zelf.dict(), + )) +} + +#[pyclass( + with( + Constructor, + Initializer, + AsSequence, + Comparable, + Iterable, + AsNumber, + Representable + ), + flags(BASETYPE, _MATCH_SELF, HAS_WEAKREF) +)] +impl PySet { + fn __len__(&self) -> usize { + self.inner.len() + } + + fn __contains__(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.contains(needle, vm) + } + + #[pymethod] + fn __sizeof__(&self) -> usize { + core::mem::size_of::<Self>() + self.inner.sizeof() + } + + #[pymethod] + fn copy(&self) -> Self { + Self { + inner: self.inner.copy(), + } + } + + #[pymethod] + fn union(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { + self.fold_op(others.into_iter(), PySetInner::union, vm) + } + + #[pymethod] + fn intersection(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { + self.fold_op(others.into_iter(), PySetInner::intersection, vm) + } + + #[pymethod] + fn difference(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { + self.fold_op(others.into_iter(), PySetInner::difference, vm) + } + + #[pymethod] + fn symmetric_difference( + &self, + others: PosArgs<ArgIterable>, + vm: &VirtualMachine, + ) -> PyResult<Self> { + self.fold_op(others.into_iter(), PySetInner::symmetric_difference, vm) + } + + #[pymethod] + fn issubset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.issubset(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.issuperset(other, vm) + } + + #[pymethod] + fn isdisjoint(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.isdisjoint(other, vm) + } + + fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(self.op( + other, + PySetInner::union, + vm, + )?)) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + fn __and__( + &self, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(self.op( + other, + PySetInner::intersection, + vm, + )?)) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + fn __sub__( + &self, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(self.op( + other, + PySetInner::difference, + vm, + )?)) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + fn __rsub__( + zelf: PyRef<Self>, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(Self { + inner: other + .as_inner() + .difference(ArgIterable::try_from_object(vm, zelf.into())?, vm)?, + })) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + fn __xor__( + &self, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(self.op( + other, + PySetInner::symmetric_difference, + vm, + )?)) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + #[pymethod] + pub fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.add(item, vm) + } + + #[pymethod] + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.remove(item, vm) + } + + #[pymethod] + fn discard(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.discard(&item, vm).map(|_| ()) + } + + #[pymethod] + fn clear(&self) { + self.inner.clear() + } + + #[pymethod] + fn pop(&self, vm: &VirtualMachine) -> PyResult { + self.inner.pop(vm) + } + + fn __ior__(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + zelf.inner.update(set.into_iterable_iter(vm)?, vm)?; + Ok(zelf) + } + + #[pymethod] + fn update(&self, others: PosArgs<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { + for iterable in others { + self.inner.update_internal(iterable, vm)?; + } + Ok(()) + } + + #[pymethod] + fn intersection_update( + &self, + others: PosArgs<ArgIterable>, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.inner.intersection_update(others.into_iter(), vm)?; + Ok(()) + } + + fn __iand__(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + if !set.is(zelf.as_object()) { + zelf.inner + .intersection_update(core::iter::once(set.into_iterable(vm)?), vm)?; + } + Ok(zelf) + } + + #[pymethod] + fn difference_update(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<()> { + self.inner.difference_update(others.into_iter(), vm) + } + + fn __isub__(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + if set.is(zelf.as_object()) { + zelf.inner.clear(); + } else { + zelf.inner + .difference_update(set.into_iterable_iter(vm)?, vm)?; + } + Ok(zelf) + } + + #[pymethod] + fn symmetric_difference_update( + &self, + others: PosArgs<ArgIterable>, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.inner + .symmetric_difference_update(others.into_iter(), vm) + } + + fn __ixor__(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + if set.is(zelf.as_object()) { + zelf.inner.clear(); + } else { + zelf.inner + .symmetric_difference_update(set.into_iterable_iter(vm)?, vm)?; + } + Ok(zelf) + } + + #[pymethod] + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> PyResult<(PyTypeRef, PyTupleRef, Option<PyDictRef>)> { + reduce_set(zelf.as_ref(), vm) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +impl DefaultConstructor for PySet {} + +impl Initializer for PySet { + type Args = OptionalArg<PyObjectRef>; + + fn init(zelf: PyRef<Self>, iterable: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + zelf.clear(); + if let OptionalArg::Present(it) = iterable { + zelf.update(PosArgs::new(vec![it]), vm)?; + } + Ok(()) + } +} + +impl AsSequence for PySet { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PySet::sequence_downcast(seq).__len__())), + contains: atomic_func!( + |seq, needle, vm| PySet::sequence_downcast(seq).__contains__(needle, vm) + ), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl Comparable for PySet { + fn cmp( + zelf: &crate::Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + extract_set(other).map_or(Ok(PyComparisonValue::NotImplemented), |other| { + Ok(zelf.inner.compare(other, op, vm)?.into()) + }) + } +} + +impl Iterable for PySet { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(zelf.inner.iter().into_pyobject(vm)) + } +} + +impl AsNumber for PySet { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + // Binary ops check both operands are sets (like CPython's set_sub, etc.) + // This is needed because __rsub__ swaps operands: a.__rsub__(b) calls subtract(b, a) + subtract: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } + if let Some(a) = a.downcast_ref::<PySet>() { + a.__sub__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + // When called via __rsub__, a might be PyFrozenSet + a.__sub__(b.to_owned(), vm) + .map(|r| { + r.map(|s| PySet { + inner: s.inner.clone(), + }) + }) + .to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + and: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } + if let Some(a) = a.downcast_ref::<PySet>() { + a.__and__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__and__(b.to_owned(), vm) + .map(|r| { + r.map(|s| PySet { + inner: s.inner.clone(), + }) + }) + .to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + xor: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } + if let Some(a) = a.downcast_ref::<PySet>() { + a.__xor__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__xor__(b.to_owned(), vm) + .map(|r| { + r.map(|s| PySet { + inner: s.inner.clone(), + }) + }) + .to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + or: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } + if let Some(a) = a.downcast_ref::<PySet>() { + a.__or__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__or__(b.to_owned(), vm) + .map(|r| { + r.map(|s| PySet { + inner: s.inner.clone(), + }) + }) + .to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + inplace_subtract: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PySet>() { + PySet::__isub__(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) + .to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + inplace_and: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PySet>() { + PySet::__iand__(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) + .to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + inplace_xor: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PySet>() { + PySet::__ixor__(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) + .to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + inplace_or: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PySet>() { + PySet::__ior__(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) + .to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Representable for PySet { + #[inline] + fn repr_wtf8(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let class = zelf.class(); + let borrowed_name = class.name(); + let class_name = borrowed_name.deref(); + if zelf.inner.len() == 0 { + return Ok(Wtf8Buf::from(format!("{class_name}()"))); + } + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let name = (class_name != "set").then_some(class_name); + zelf.inner.repr(name, vm) + } else { + Ok(Wtf8Buf::from(format!("{class_name}(...)"))) + } + } +} + +impl Constructor for PyFrozenSet { + type Args = Vec<PyObjectRef>; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let iterable: OptionalArg<PyObjectRef> = args.bind(vm)?; + + // Optimizations for exact frozenset type + if cls.is(vm.ctx.types.frozenset_type) { + // Return exact frozenset as-is + if let OptionalArg::Present(ref input) = iterable + && let Ok(fs) = input.clone().downcast_exact::<PyFrozenSet>(vm) + { + return Ok(fs.into_pyref().into()); + } + + // Return empty frozenset singleton + if iterable.is_missing() { + return Ok(vm.ctx.empty_frozenset.clone().into()); + } + } + + let elements: Vec<PyObjectRef> = if let OptionalArg::Present(iterable) = iterable { + iterable.try_to_value(vm)? + } else { + vec![] + }; + + // Return empty frozenset singleton for exact frozenset types (when iterable was empty) + if elements.is_empty() && cls.is(vm.ctx.types.frozenset_type) { + return Ok(vm.ctx.empty_frozenset.clone().into()); + } + + let payload = Self::py_new(&cls, elements, vm)?; + payload.into_ref_with_type(vm, cls).map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, elements: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Self::from_iter(vm, elements) + } +} + +#[pyclass( + flags(BASETYPE, _MATCH_SELF, HAS_WEAKREF), + with( + Constructor, + AsSequence, + Hashable, + Comparable, + Iterable, + AsNumber, + Representable + ) +)] +impl PyFrozenSet { + fn __len__(&self) -> usize { + self.inner.len() + } + + fn __contains__(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.contains(needle, vm) + } + + #[pymethod] + fn __sizeof__(&self) -> usize { + core::mem::size_of::<Self>() + self.inner.sizeof() + } + + #[pymethod] + fn copy(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRef<Self> { + if zelf.class().is(vm.ctx.types.frozenset_type) { + zelf + } else { + Self { + inner: zelf.inner.copy(), + ..Default::default() + } + .into_ref(&vm.ctx) + } + } + + #[pymethod] + fn union(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { + self.fold_op(others.into_iter(), PySetInner::union, vm) + } + + #[pymethod] + fn intersection(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { + self.fold_op(others.into_iter(), PySetInner::intersection, vm) + } + + #[pymethod] + fn difference(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { + self.fold_op(others.into_iter(), PySetInner::difference, vm) + } + + #[pymethod] + fn symmetric_difference( + &self, + others: PosArgs<ArgIterable>, + vm: &VirtualMachine, + ) -> PyResult<Self> { + self.fold_op(others.into_iter(), PySetInner::symmetric_difference, vm) + } + + #[pymethod] + fn issubset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.issubset(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.issuperset(other, vm) + } + + #[pymethod] + fn isdisjoint(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { + self.inner.isdisjoint(other, vm) + } + + fn __or__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(set) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(self.op( + set, + PySetInner::union, + vm, + )?)) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + fn __and__( + &self, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(self.op( + other, + PySetInner::intersection, + vm, + )?)) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + fn __sub__( + &self, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(self.op( + other, + PySetInner::difference, + vm, + )?)) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + fn __rsub__( + zelf: PyRef<Self>, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(Self { + inner: other + .as_inner() + .difference(ArgIterable::try_from_object(vm, zelf.into())?, vm)?, + ..Default::default() + })) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + fn __xor__( + &self, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyArithmeticValue<Self>> { + if let Ok(other) = AnySet::try_from_object(vm, other) { + Ok(PyArithmeticValue::Implemented(self.op( + other, + PySetInner::symmetric_difference, + vm, + )?)) + } else { + Ok(PyArithmeticValue::NotImplemented) + } + } + + #[pymethod] + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> PyResult<(PyTypeRef, PyTupleRef, Option<PyDictRef>)> { + reduce_set(zelf.as_ref(), vm) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +impl AsSequence for PyFrozenSet { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyFrozenSet::sequence_downcast(seq).__len__())), + contains: atomic_func!( + |seq, needle, vm| PyFrozenSet::sequence_downcast(seq).__contains__(needle, vm) + ), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl Hashable for PyFrozenSet { + #[inline] + fn hash(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + let hash = match zelf.hash.load(Ordering::Relaxed) { + hash::SENTINEL => { + let hash = zelf.inner.hash(vm)?; + match Radium::compare_exchange( + &zelf.hash, + hash::SENTINEL, + hash::fix_sentinel(hash), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => hash, + Err(prev_stored) => prev_stored, + } + } + hash => hash, + }; + Ok(hash) + } +} + +impl Comparable for PyFrozenSet { + fn cmp( + zelf: &crate::Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + extract_set(other).map_or(Ok(PyComparisonValue::NotImplemented), |other| { + Ok(zelf.inner.compare(other, op, vm)?.into()) + }) + } +} + +impl Iterable for PyFrozenSet { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(zelf.inner.iter().into_pyobject(vm)) + } +} + +impl AsNumber for PyFrozenSet { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + // Binary ops check both operands are sets (like CPython's set_sub, etc.) + // __rsub__ swaps operands. Result type follows first operand's type. + subtract: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } + if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__sub__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PySet>() { + // When called via __rsub__, a might be PySet - return set (not frozenset) + a.__sub__(b.to_owned(), vm).to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + and: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } + if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__and__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PySet>() { + a.__and__(b.to_owned(), vm).to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + xor: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } + if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__xor__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PySet>() { + a.__xor__(b.to_owned(), vm).to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + or: Some(|a, b, vm| { + if !AnySet::check(a, vm) || !AnySet::check(b, vm) { + return Ok(vm.ctx.not_implemented()); + } + if let Some(a) = a.downcast_ref::<PyFrozenSet>() { + a.__or__(b.to_owned(), vm).to_pyresult(vm) + } else if let Some(a) = a.downcast_ref::<PySet>() { + a.__or__(b.to_owned(), vm).to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Representable for PyFrozenSet { + #[inline] + fn repr_wtf8(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let inner = &zelf.inner; + let class = zelf.class(); + let class_name = class.name(); + if inner.len() == 0 { + return Ok(Wtf8Buf::from(format!("{class_name}()"))); + } + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + inner.repr(Some(&class_name), vm) + } else { + Ok(Wtf8Buf::from(format!("{class_name}(...)"))) + } + } +} + +struct AnySet { + object: PyObjectRef, +} + +impl Borrow<PyObject> for AnySet { + #[inline(always)] + fn borrow(&self) -> &PyObject { + &self.object + } +} + +impl AnySet { + /// Check if object is a set or frozenset (including subclasses) + /// Equivalent to CPython's PyAnySet_Check + fn check(obj: &PyObject, vm: &VirtualMachine) -> bool { + let ctx = &vm.ctx; + obj.fast_isinstance(ctx.types.set_type) || obj.fast_isinstance(ctx.types.frozenset_type) + } + + fn into_iterable(self, vm: &VirtualMachine) -> PyResult<ArgIterable> { + self.object.try_into_value(vm) + } + + fn into_iterable_iter( + self, + vm: &VirtualMachine, + ) -> PyResult<impl core::iter::Iterator<Item = ArgIterable>> { + Ok(core::iter::once(self.into_iterable(vm)?)) + } + + fn as_inner(&self) -> &PySetInner { + match_class!(match self.object.as_object() { + ref set @ PySet => &set.inner, + ref frozen @ PyFrozenSet => &frozen.inner, + _ => unreachable!("AnySet is always PySet or PyFrozenSet"), // should not be called. + }) + } +} + +impl TryFromObject for AnySet { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let class = obj.class(); + if class.fast_issubclass(vm.ctx.types.set_type) + || class.fast_issubclass(vm.ctx.types.frozenset_type) + { + Ok(Self { object: obj }) + } else { + Err(vm.new_type_error(format!("{class} is not a subtype of set or frozenset"))) + } + } +} + +#[pyclass(module = false, name = "set_iterator")] +pub(crate) struct PySetIterator { + size: DictSize, + internal: PyMutex<PositionIterInternal<PyRc<SetContentType>>>, +} + +impl fmt::Debug for PySetIterator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO: implement more detailed, non-recursive Debug formatter + f.write_str("set_iterator") + } +} + +impl PyPayload for PySetIterator { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.set_iterator_type + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] +impl PySetIterator { + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().length_hint(|_| self.size.entries_size) + } + + #[pymethod] + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, (PyObjectRef,))> { + let internal = zelf.internal.lock(); + Ok(( + builtins_iter(vm), + (vm.ctx + .new_list(match &internal.status { + IterStatus::Exhausted => vec![], + IterStatus::Active(dict) => { + dict.keys().into_iter().skip(internal.position).collect() + } + }) + .into(),), + )) + } +} + +impl SelfIter for PySetIterator {} +impl IterNext for PySetIterator { + fn next(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let mut internal = zelf.internal.lock(); + let next = if let IterStatus::Active(dict) = &internal.status { + if dict.has_changed_size(&zelf.size) { + internal.status = IterStatus::Exhausted; + return Err(vm.new_runtime_error("set changed size during iteration")); + } + match dict.next_entry(internal.position) { + Some((position, key, _)) => { + internal.position = position; + PyIterReturn::Return(key) + } + None => { + internal.status = IterStatus::Exhausted; + PyIterReturn::StopIteration(None) + } + } + } else { + PyIterReturn::StopIteration(None) + }; + Ok(next) + } +} + +pub fn init(context: &'static Context) { + PySet::extend_class(context, context.types.set_type); + PyFrozenSet::extend_class(context, context.types.frozenset_type); + PySetIterator::extend_class(context, context.types.set_iterator_type); +} diff --git a/crates/vm/src/builtins/singletons.rs b/crates/vm/src/builtins/singletons.rs new file mode 100644 index 00000000000..00d84dfabb6 --- /dev/null +++ b/crates/vm/src/builtins/singletons.rs @@ -0,0 +1,157 @@ +use super::{PyStrRef, PyType, PyTypeRef}; +use crate::{ + Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, + class::PyClassImpl, + common::hash::PyHash, + convert::ToPyObject, + function::{FuncArgs, PyComparisonValue}, + protocol::PyNumberMethods, + types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, +}; + +#[pyclass(module = false, name = "NoneType")] +#[derive(Debug)] +pub struct PyNone; + +impl PyPayload for PyNone { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.none_type + } +} + +// This allows a built-in function to not return a value, mapping to +// Python's behavior of returning `None` in this situation. +impl ToPyObject for () { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } +} + +impl<T: ToPyObject> ToPyObject for Option<T> { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + match self { + Some(x) => x.to_pyobject(vm), + None => vm.ctx.none(), + } + } +} + +impl Constructor for PyNone { + type Args = (); + + fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let _: () = args.bind(vm)?; + Ok(vm.ctx.none.clone().into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unreachable!("None is a singleton") + } +} + +#[pyclass(with(Constructor, AsNumber, Comparable, Hashable, Representable))] +impl PyNone {} + +impl Representable for PyNone { + #[inline] + fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + Ok(vm.ctx.names.None.to_owned()) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } +} + +impl AsNumber for PyNone { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|_number, _vm| Ok(false)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Comparable for PyNone { + fn cmp( + zelf: &Py<Self>, + other: &crate::PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + Ok(op + .identical_optimization(zelf, other) + .map(PyComparisonValue::Implemented) + .unwrap_or(PyComparisonValue::NotImplemented)) + } +} + +impl Hashable for PyNone { + fn hash(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyHash> { + Ok(0xFCA86420) + } +} + +#[pyclass(module = false, name = "NotImplementedType")] +#[derive(Debug)] +pub struct PyNotImplemented; + +impl PyPayload for PyNotImplemented { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.not_implemented_type + } +} + +impl Constructor for PyNotImplemented { + type Args = (); + + fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let _: () = args.bind(vm)?; + Ok(vm.ctx.not_implemented.clone().into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unreachable!("PyNotImplemented is a singleton") + } +} + +#[pyclass(with(Constructor, AsNumber, Representable), flags(IMMUTABLETYPE))] +impl PyNotImplemented { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyStrRef { + vm.ctx.names.NotImplemented.to_owned() + } +} + +impl AsNumber for PyNotImplemented { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|_number, vm| { + Err(vm.new_type_error("NotImplemented should not be used in a boolean context")) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Representable for PyNotImplemented { + #[inline] + fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + Ok(vm.ctx.names.NotImplemented.to_owned()) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } +} + +pub fn init(context: &'static Context) { + PyNone::extend_class(context, context.types.none_type); + PyNotImplemented::extend_class(context, context.types.not_implemented_type); +} diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs new file mode 100644 index 00000000000..9c6fa5b59fe --- /dev/null +++ b/crates/vm/src/builtins/slice.rs @@ -0,0 +1,419 @@ +// sliceobject.{h,c} in CPython +// spell-checker:ignore sliceobject +use core::cell::Cell; +use core::ptr::NonNull; +use rustpython_common::wtf8::{Wtf8Buf, wtf8_concat}; + +use super::{PyGenericAlias, PyStrRef, PyTupleRef, PyType, PyTypeRef}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + common::hash::{PyHash, PyUHash}, + convert::ToPyObject, + function::{ArgIndex, FuncArgs, OptionalArg, PyComparisonValue}, + sliceable::SaturatedSlice, + types::{Comparable, Constructor, Hashable, PyComparisonOp, Representable}, +}; +use malachite_bigint::{BigInt, ToBigInt}; +use num_traits::{One, Signed, Zero}; + +#[pyclass(module = false, name = "slice", unhashable = true, traverse = "manual")] +#[derive(Debug)] +pub struct PySlice { + pub start: Option<PyObjectRef>, + pub stop: PyObjectRef, + pub step: Option<PyObjectRef>, +} + +// SAFETY: Traverse properly visits all owned PyObjectRefs +unsafe impl crate::object::Traverse for PySlice { + fn traverse(&self, traverse_fn: &mut crate::object::TraverseFn<'_>) { + self.start.traverse(traverse_fn); + self.stop.traverse(traverse_fn); + self.step.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec<PyObjectRef>) { + if let Some(start) = self.start.take() { + out.push(start); + } + // stop is not Option, so it will be freed when payload is dropped + // (via drop_in_place on freelist pop, or Box::from_raw on dealloc) + if let Some(step) = self.step.take() { + out.push(step); + } + } +} + +thread_local! { + static SLICE_FREELIST: Cell<crate::object::FreeList<PySlice>> = const { Cell::new(crate::object::FreeList::new()) }; +} + +impl PyPayload for PySlice { + const MAX_FREELIST: usize = 1; + const HAS_FREELIST: bool = true; + + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.slice_type + } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + SLICE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> { + SLICE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } +} + +#[pyclass(with(Comparable, Representable, Hashable))] +impl PySlice { + #[pygetset] + fn start(&self, vm: &VirtualMachine) -> PyObjectRef { + self.start.clone().to_pyobject(vm) + } + + pub(crate) fn start_ref<'a>(&'a self, vm: &'a VirtualMachine) -> &'a PyObject { + match &self.start { + Some(v) => v, + None => vm.ctx.none.as_object(), + } + } + + #[pygetset] + pub(crate) fn stop(&self, _vm: &VirtualMachine) -> PyObjectRef { + self.stop.clone() + } + + #[pygetset] + fn step(&self, vm: &VirtualMachine) -> PyObjectRef { + self.step.clone().to_pyobject(vm) + } + + pub(crate) fn step_ref<'a>(&'a self, vm: &'a VirtualMachine) -> &'a PyObject { + match &self.step { + Some(v) => v, + None => vm.ctx.none.as_object(), + } + } + + pub fn to_saturated(&self, vm: &VirtualMachine) -> PyResult<SaturatedSlice> { + SaturatedSlice::with_slice(self, vm) + } + + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let slice: Self = match args.args.len() { + 0 => { + return Err(vm.new_type_error("slice() must have at least one arguments.")); + } + 1 => { + let stop = args.bind(vm)?; + Self { + start: None, + stop, + step: None, + } + } + _ => { + let (start, stop, step): (PyObjectRef, PyObjectRef, OptionalArg<PyObjectRef>) = + args.bind(vm)?; + Self { + start: Some(start), + stop, + step: step.into_option(), + } + } + }; + slice.into_ref_with_type(vm, cls).map(Into::into) + } + + pub(crate) fn inner_indices( + &self, + length: &BigInt, + vm: &VirtualMachine, + ) -> PyResult<(BigInt, BigInt, BigInt)> { + // Calculate step + let step: BigInt; + if vm.is_none(self.step_ref(vm)) { + step = One::one(); + } else { + // Clone the value, not the reference. + let this_step = self.step(vm).try_index(vm)?; + step = this_step.as_bigint().clone(); + + if step.is_zero() { + return Err(vm.new_value_error("slice step cannot be zero.")); + } + } + + // For convenience + let backwards = step.is_negative(); + + // Each end of the array + let lower = if backwards { + (-1_i8).to_bigint().unwrap() + } else { + Zero::zero() + }; + + let upper = if backwards { + lower.clone() + length + } else { + length.clone() + }; + + // Calculate start + let mut start: BigInt; + if vm.is_none(self.start_ref(vm)) { + // Default + start = if backwards { + upper.clone() + } else { + lower.clone() + }; + } else { + let this_start = self.start(vm).try_index(vm)?; + start = this_start.as_bigint().clone(); + + if start < Zero::zero() { + // From end of array + start += length; + + if start < lower { + start = lower.clone(); + } + } else if start > upper { + start = upper.clone(); + } + } + + // Calculate Stop + let mut stop: BigInt; + if vm.is_none(&self.stop) { + stop = if backwards { lower } else { upper }; + } else { + let this_stop = self.stop(vm).try_index(vm)?; + stop = this_stop.as_bigint().clone(); + + if stop < Zero::zero() { + // From end of array + stop += length; + if stop < lower { + stop = lower; + } + } else if stop > upper { + stop = upper; + } + } + + Ok((start, stop, step)) + } + + #[pymethod] + fn indices(&self, length: ArgIndex, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let length = length.into_int_ref(); + let length = length.as_bigint(); + if length.is_negative() { + return Err(vm.new_value_error("length should not be negative.")); + } + let (start, stop, step) = self.inner_indices(length, vm)?; + Ok(vm.new_tuple((start, stop, step))) + } + + #[allow(clippy::type_complexity)] + #[pymethod] + fn __reduce__( + zelf: PyRef<Self>, + ) -> PyResult<( + PyTypeRef, + (Option<PyObjectRef>, PyObjectRef, Option<PyObjectRef>), + )> { + Ok(( + zelf.class().to_owned(), + (zelf.start.clone(), zelf.stop.clone(), zelf.step.clone()), + )) + } + + // TODO: Uncomment when Python adds __class_getitem__ to slice + // #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +impl Hashable for PySlice { + #[inline] + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + const XXPRIME_1: PyUHash = if cfg!(target_pointer_width = "64") { + 11400714785074694791 + } else { + 2654435761 + }; + const XXPRIME_2: PyUHash = if cfg!(target_pointer_width = "64") { + 14029467366897019727 + } else { + 2246822519 + }; + const XXPRIME_5: PyUHash = if cfg!(target_pointer_width = "64") { + 2870177450012600261 + } else { + 374761393 + }; + const ROTATE: u32 = if cfg!(target_pointer_width = "64") { + 31 + } else { + 13 + }; + + let mut acc = XXPRIME_5; + for part in &[zelf.start_ref(vm), &zelf.stop, zelf.step_ref(vm)] { + let lane = part.hash(vm)? as PyUHash; + if lane == u64::MAX as PyUHash { + return Ok(-1 as PyHash); + } + acc = acc.wrapping_add(lane.wrapping_mul(XXPRIME_2)); + acc = acc.rotate_left(ROTATE); + acc = acc.wrapping_mul(XXPRIME_1); + } + if acc == u64::MAX as PyUHash { + return Ok(1546275796 as PyHash); + } + Ok(acc as PyHash) + } +} + +impl Comparable for PySlice { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let other = class_or_notimplemented!(Self, other); + + let ret = match op { + PyComparisonOp::Lt | PyComparisonOp::Le => None + .or_else(|| { + vm.bool_seq_lt(zelf.start_ref(vm), other.start_ref(vm)) + .transpose() + }) + .or_else(|| vm.bool_seq_lt(&zelf.stop, &other.stop).transpose()) + .or_else(|| { + vm.bool_seq_lt(zelf.step_ref(vm), other.step_ref(vm)) + .transpose() + }) + .unwrap_or_else(|| Ok(op == PyComparisonOp::Le))?, + PyComparisonOp::Eq | PyComparisonOp::Ne => { + let eq = vm.identical_or_equal(zelf.start_ref(vm), other.start_ref(vm))? + && vm.identical_or_equal(&zelf.stop, &other.stop)? + && vm.identical_or_equal(zelf.step_ref(vm), other.step_ref(vm))?; + if op == PyComparisonOp::Ne { !eq } else { eq } + } + PyComparisonOp::Gt | PyComparisonOp::Ge => None + .or_else(|| { + vm.bool_seq_gt(zelf.start_ref(vm), other.start_ref(vm)) + .transpose() + }) + .or_else(|| vm.bool_seq_gt(&zelf.stop, &other.stop).transpose()) + .or_else(|| { + vm.bool_seq_gt(zelf.step_ref(vm), other.step_ref(vm)) + .transpose() + }) + .unwrap_or_else(|| Ok(op == PyComparisonOp::Ge))?, + }; + + Ok(PyComparisonValue::Implemented(ret)) + } +} + +impl Representable for PySlice { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let start_repr = zelf.start_ref(vm).repr(vm)?; + let stop_repr = zelf.stop.repr(vm)?; + let step_repr = zelf.step_ref(vm).repr(vm)?; + + Ok(wtf8_concat!( + "slice(", + start_repr.as_wtf8(), + ", ", + stop_repr.as_wtf8(), + ", ", + step_repr.as_wtf8(), + ")" + )) + } +} + +#[pyclass(module = false, name = "EllipsisType")] +#[derive(Debug)] +pub struct PyEllipsis; + +impl PyPayload for PyEllipsis { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.ellipsis_type + } +} + +impl Constructor for PyEllipsis { + type Args = (); + + fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let _: () = args.bind(vm)?; + Ok(vm.ctx.ellipsis.clone().into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unreachable!("Ellipsis is a singleton") + } +} + +#[pyclass(with(Constructor, Representable), flags(IMMUTABLETYPE))] +impl PyEllipsis { + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyStrRef { + vm.ctx.names.Ellipsis.to_owned() + } +} + +impl Representable for PyEllipsis { + #[inline] + fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + Ok(vm.ctx.names.Ellipsis.to_owned()) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } +} + +pub fn init(ctx: &'static Context) { + PySlice::extend_class(ctx, ctx.types.slice_type); + PyEllipsis::extend_class(ctx, ctx.types.ellipsis_type); +} diff --git a/crates/vm/src/builtins/staticmethod.rs b/crates/vm/src/builtins/staticmethod.rs new file mode 100644 index 00000000000..2554fa816aa --- /dev/null +++ b/crates/vm/src/builtins/staticmethod.rs @@ -0,0 +1,202 @@ +use super::{PyGenericAlias, PyStr, PyType, PyTypeRef}; +use crate::{ + Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + common::lock::PyMutex, + function::{FuncArgs, PySetterValue}, + types::{Callable, Constructor, GetDescriptor, Initializer, Representable}, +}; + +#[pyclass(module = false, name = "staticmethod", traverse)] +#[derive(Debug)] +pub struct PyStaticMethod { + pub callable: PyMutex<PyObjectRef>, +} + +impl PyPayload for PyStaticMethod { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.staticmethod_type + } +} + +impl GetDescriptor for PyStaticMethod { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let (zelf, _obj) = Self::_unwrap(&zelf, obj, vm)?; + Ok(zelf.callable.lock().clone()) + } +} + +impl From<PyObjectRef> for PyStaticMethod { + fn from(callable: PyObjectRef) -> Self { + Self { + callable: PyMutex::new(callable), + } + } +} + +impl Constructor for PyStaticMethod { + type Args = PyObjectRef; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let callable: Self::Args = args.bind(vm)?; + let doc = callable.get_attr("__doc__", vm); + + let result = Self { + callable: PyMutex::new(callable), + } + .into_ref_with_type(vm, cls)?; + let obj = PyObjectRef::from(result); + + if let Ok(doc) = doc { + obj.set_attr("__doc__", doc, vm)?; + } + + Ok(obj) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl PyStaticMethod { + pub fn new(callable: PyObjectRef) -> Self { + Self { + callable: PyMutex::new(callable), + } + } + #[deprecated(note = "use PyStaticMethod::new(...).into_ref() instead")] + pub fn new_ref(callable: PyObjectRef, ctx: &Context) -> PyRef<Self> { + Self::new(callable).into_ref(ctx) + } +} + +impl Initializer for PyStaticMethod { + type Args = PyObjectRef; + + fn init(zelf: PyRef<Self>, callable: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + *zelf.callable.lock() = callable; + Ok(()) + } +} + +#[pyclass( + with(Callable, GetDescriptor, Constructor, Initializer, Representable), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) +)] +impl PyStaticMethod { + #[pygetset] + fn __func__(&self) -> PyObjectRef { + self.callable.lock().clone() + } + + #[pygetset] + fn __wrapped__(&self) -> PyObjectRef { + self.callable.lock().clone() + } + + #[pygetset] + fn __module__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__module__", vm) + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__qualname__", vm) + } + + #[pygetset] + fn __name__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__name__", vm) + } + + #[pygetset] + fn __annotations__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__annotations__", vm) + } + + #[pygetset(setter)] + fn set___annotations__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => self.callable.lock().set_attr("__annotations__", v, vm), + PySetterValue::Delete => Ok(()), // Silently ignore delete like CPython + } + } + + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyResult { + self.callable.lock().get_attr("__annotate__", vm) + } + + #[pygetset(setter)] + fn set___annotate__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + match value { + PySetterValue::Assign(v) => self.callable.lock().set_attr("__annotate__", v, vm), + PySetterValue::Delete => Ok(()), // Silently ignore delete like CPython + } + } + + #[pygetset] + fn __isabstractmethod__(&self, vm: &VirtualMachine) -> PyObjectRef { + match vm.get_attribute_opt(self.callable.lock().clone(), "__isabstractmethod__") { + Ok(Some(is_abstract)) => is_abstract, + _ => vm.ctx.new_bool(false).into(), + } + } + + #[pygetset(setter)] + fn set___isabstractmethod__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.callable + .lock() + .set_attr("__isabstractmethod__", value, vm)?; + Ok(()) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +impl Callable for PyStaticMethod { + type Args = FuncArgs; + #[inline] + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let callable = zelf.callable.lock().clone(); + callable.call(args, vm) + } +} + +impl Representable for PyStaticMethod { + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let callable = zelf.callable.lock().repr(vm).unwrap(); + let class = Self::class(&vm.ctx); + + match ( + class + .__qualname__(vm) + .downcast_ref::<PyStr>() + .map(|n| n.as_wtf8()), + class + .__module__(vm) + .downcast_ref::<PyStr>() + .map(|m| m.as_wtf8()), + ) { + (None, _) => Err(vm.new_type_error("Unknown qualified name")), + (Some(qualname), Some(module)) if module != "builtins" => { + Ok(format!("<{module}.{qualname}({callable})>")) + } + _ => Ok(format!("<{}({})>", class.slot_name(), callable)), + } + } +} + +pub fn init(context: &'static Context) { + PyStaticMethod::extend_class(context, context.types.staticmethod_type); +} diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs new file mode 100644 index 00000000000..d882fa913a5 --- /dev/null +++ b/crates/vm/src/builtins/str.rs @@ -0,0 +1,2672 @@ +use super::{ + PositionIterInternal, PyBytesRef, PyDict, PyTupleRef, PyType, PyTypeRef, + int::{PyInt, PyIntRef}, + iter::{ + IterStatus::{self, Exhausted}, + builtins_iter, + }, +}; +use crate::common::lock::LazyLock; +use crate::{ + AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromBorrowedObject, VirtualMachine, + anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper, adjust_indices}, + atomic_func, + cformat::cformat_string, + class::PyClassImpl, + common::str::{PyKindStr, StrData, StrKind}, + convert::{IntoPyException, ToPyException, ToPyObject, ToPyResult}, + format::{format, format_map}, + function::{ArgIterable, ArgSize, FuncArgs, OptionalArg, OptionalOption, PyComparisonValue}, + intern::PyInterned, + object::{MaybeTraverse, Traverse, TraverseFn}, + protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + sequence::SequenceExt, + sliceable::{SequenceIndex, SliceableSequenceOp}, + types::{ + AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, + PyComparisonOp, Representable, SelfIter, + }, +}; +use alloc::{borrow::Cow, fmt}; +use ascii::{AsciiChar, AsciiStr, AsciiString}; +use bstr::ByteSlice; +use core::{char, mem, ops::Range}; +use itertools::Itertools; +use num_traits::ToPrimitive; +use rustpython_common::{ + ascii, + atomic::{self, PyAtomic, Radium}, + format::{FormatSpec, FormatString, FromTemplate}, + hash, + lock::PyMutex, + str::DeduceStrKind, + wtf8::{CodePoint, Wtf8, Wtf8Buf, Wtf8Chunk, Wtf8Concat}, +}; +use unic_ucd_bidi::BidiClass; +use unic_ucd_category::GeneralCategory; +use unic_ucd_ident::{is_xid_continue, is_xid_start}; +use unicode_casing::CharExt; + +impl<'a> TryFromBorrowedObject<'a> for String { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + obj.try_value_with(|pystr: &PyUtf8Str| Ok(pystr.as_str().to_owned()), vm) + } +} + +impl<'a> TryFromBorrowedObject<'a> for &'a str { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + let pystr: &Py<PyUtf8Str> = TryFromBorrowedObject::try_from_borrowed_object(vm, obj)?; + Ok(pystr.as_str()) + } +} + +impl<'a> TryFromBorrowedObject<'a> for &'a Wtf8 { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + let pystr: &Py<PyStr> = TryFromBorrowedObject::try_from_borrowed_object(vm, obj)?; + Ok(pystr.as_wtf8()) + } +} + +pub type PyStrRef = PyRef<PyStr>; +pub type PyUtf8StrRef = PyRef<PyUtf8Str>; + +#[pyclass(module = false, name = "str")] +pub struct PyStr { + data: StrData, + hash: PyAtomic<hash::PyHash>, +} + +impl fmt::Debug for PyStr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PyStr") + .field("value", &self.as_wtf8()) + .field("kind", &self.data.kind()) + .field("hash", &self.hash) + .finish() + } +} + +impl AsRef<str> for PyStr { + #[track_caller] // <- can remove this once it doesn't panic + fn as_ref(&self) -> &str { + self.to_str().expect("str has surrogates") + } +} + +impl AsRef<str> for Py<PyStr> { + #[track_caller] // <- can remove this once it doesn't panic + fn as_ref(&self) -> &str { + self.to_str().expect("str has surrogates") + } +} + +impl AsRef<str> for PyStrRef { + #[track_caller] // <- can remove this once it doesn't panic + fn as_ref(&self) -> &str { + self.to_str().expect("str has surrogates") + } +} + +impl AsRef<Wtf8> for PyStr { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl AsRef<Wtf8> for Py<PyStr> { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl AsRef<Wtf8> for PyStrRef { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl Wtf8Concat for PyStr { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + buf.push_wtf8(self.as_wtf8()); + } +} + +impl Wtf8Concat for Py<PyStr> { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + buf.push_wtf8(self.as_wtf8()); + } +} + +impl<'a> From<&'a AsciiStr> for PyStr { + fn from(s: &'a AsciiStr) -> Self { + s.to_owned().into() + } +} + +impl From<AsciiString> for PyStr { + fn from(s: AsciiString) -> Self { + s.into_boxed_ascii_str().into() + } +} + +impl From<Box<AsciiStr>> for PyStr { + fn from(s: Box<AsciiStr>) -> Self { + StrData::from(s).into() + } +} + +impl From<AsciiChar> for PyStr { + fn from(ch: AsciiChar) -> Self { + AsciiString::from(ch).into() + } +} + +impl<'a> From<&'a str> for PyStr { + fn from(s: &'a str) -> Self { + s.to_owned().into() + } +} + +impl<'a> From<&'a Wtf8> for PyStr { + fn from(s: &'a Wtf8) -> Self { + s.to_owned().into() + } +} + +impl From<String> for PyStr { + fn from(s: String) -> Self { + s.into_boxed_str().into() + } +} + +impl From<Wtf8Buf> for PyStr { + fn from(w: Wtf8Buf) -> Self { + w.into_box().into() + } +} + +impl From<char> for PyStr { + fn from(ch: char) -> Self { + StrData::from(ch).into() + } +} + +impl From<CodePoint> for PyStr { + fn from(ch: CodePoint) -> Self { + StrData::from(ch).into() + } +} + +impl From<StrData> for PyStr { + fn from(data: StrData) -> Self { + Self { + data, + hash: Radium::new(hash::SENTINEL), + } + } +} + +impl<'a> From<alloc::borrow::Cow<'a, str>> for PyStr { + fn from(s: alloc::borrow::Cow<'a, str>) -> Self { + s.into_owned().into() + } +} + +impl From<Box<str>> for PyStr { + #[inline] + fn from(value: Box<str>) -> Self { + StrData::from(value).into() + } +} + +impl From<Box<Wtf8>> for PyStr { + #[inline] + fn from(value: Box<Wtf8>) -> Self { + StrData::from(value).into() + } +} + +impl Default for PyStr { + fn default() -> Self { + Self { + data: StrData::default(), + hash: Radium::new(hash::SENTINEL), + } + } +} + +impl fmt::Display for PyStr { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.as_wtf8().fmt(f) + } +} + +pub trait AsPyStr<'a> +where + Self: 'a, +{ + #[allow( + clippy::wrong_self_convention, + reason = "this trait is intentionally implemented for references" + )] + fn as_pystr(self, ctx: &Context) -> &'a Py<PyStr>; +} + +impl<'a> AsPyStr<'a> for &'a Py<PyStr> { + #[inline] + fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { + self + } +} + +impl<'a> AsPyStr<'a> for &'a Py<PyUtf8Str> { + #[inline] + fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { + Py::<PyUtf8Str>::as_pystr(self) + } +} + +impl<'a> AsPyStr<'a> for &'a PyStrRef { + #[inline] + fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { + self + } +} + +impl<'a> AsPyStr<'a> for &'a PyUtf8StrRef { + #[inline] + fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { + Py::<PyUtf8Str>::as_pystr(self) + } +} + +impl AsPyStr<'static> for &'static str { + #[inline] + fn as_pystr(self, ctx: &Context) -> &'static Py<PyStr> { + ctx.intern_str(self) + } +} + +impl<'a> AsPyStr<'a> for &'a PyStrInterned { + #[inline] + fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { + self + } +} + +impl<'a> AsPyStr<'a> for &'a PyUtf8StrInterned { + #[inline] + fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { + Py::<PyUtf8Str>::as_pystr(self) + } +} + +#[pyclass(module = false, name = "str_iterator", traverse = "manual")] +#[derive(Debug)] +pub struct PyStrIterator { + internal: PyMutex<(PositionIterInternal<PyStrRef>, usize)>, +} + +unsafe impl Traverse for PyStrIterator { + fn traverse(&self, tracer: &mut TraverseFn<'_>) { + // No need to worry about deadlock, for inner is a PyStr and can't make ref cycle + self.internal.lock().0.traverse(tracer); + } +} + +impl PyPayload for PyStrIterator { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.str_iterator_type + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] +impl PyStrIterator { + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().0.length_hint(|obj| obj.char_len()) + } + + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut internal = self.internal.lock(); + internal.1 = usize::MAX; + internal + .0 + .set_state(state, |obj, pos| pos.min(obj.char_len()), vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + self.internal.lock().0.reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_str.to_owned().into(), + vm, + ) + } +} + +impl SelfIter for PyStrIterator {} + +impl IterNext for PyStrIterator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let mut internal = zelf.internal.lock(); + + if let IterStatus::Active(s) = &internal.0.status { + let value = s.as_wtf8(); + + if internal.1 == usize::MAX { + if let Some((offset, ch)) = value.code_point_indices().nth(internal.0.position) { + internal.0.position += 1; + internal.1 = offset + ch.len_wtf8(); + return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); + } + } else if let Some(value) = value.get(internal.1..) + && let Some(ch) = value.code_points().next() + { + internal.0.position += 1; + internal.1 += ch.len_wtf8(); + return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); + } + internal.0.status = Exhausted; + } + Ok(PyIterReturn::StopIteration(None)) + } +} + +#[derive(FromArgs)] +pub struct StrArgs { + #[pyarg(any, optional)] + object: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + encoding: OptionalArg<PyUtf8StrRef>, + #[pyarg(any, optional)] + errors: OptionalArg<PyUtf8StrRef>, +} + +impl Constructor for PyStr { + type Args = StrArgs; + + fn slot_new(cls: PyTypeRef, func_args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Optimization: return exact str as-is (only when no encoding/errors provided) + if cls.is(vm.ctx.types.str_type) + && func_args.args.len() == 1 + && func_args.kwargs.is_empty() + && func_args.args[0].class().is(vm.ctx.types.str_type) + { + return Ok(func_args.args[0].clone()); + } + + let args: Self::Args = func_args.bind(vm)?; + let payload = Self::py_new(&cls, args, vm)?; + payload.into_ref_with_type(vm, cls).map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + match args.object { + OptionalArg::Present(input) => { + if let OptionalArg::Present(enc) = args.encoding { + let s = vm.state.codec_registry.decode_text( + input, + enc.as_str(), + args.errors.into_option(), + vm, + )?; + Ok(Self::from(s.as_wtf8().to_owned())) + } else { + let s = input.str(vm)?; + Ok(Self::from(s.as_wtf8().to_owned())) + } + } + OptionalArg::Missing => Ok(Self::from(String::new())), + } + } +} + +impl PyStr { + /// # Safety: Given `bytes` must be valid data for given `kind` + unsafe fn new_str_unchecked(data: Box<Wtf8>, kind: StrKind) -> Self { + unsafe { StrData::new_str_unchecked(data, kind) }.into() + } + + unsafe fn new_with_char_len<T: DeduceStrKind + Into<Box<Wtf8>>>(s: T, char_len: usize) -> Self { + let kind = s.str_kind(); + unsafe { StrData::new_with_char_len(s.into(), kind, char_len) }.into() + } + + /// # Safety + /// Given `bytes` must be ascii + pub unsafe fn new_ascii_unchecked(bytes: Vec<u8>) -> Self { + unsafe { AsciiString::from_ascii_unchecked(bytes) }.into() + } + + #[deprecated(note = "use PyStr::from(...).into_ref() instead")] + pub fn new_ref(zelf: impl Into<Self>, ctx: &Context) -> PyRef<Self> { + let zelf = zelf.into(); + zelf.into_ref(ctx) + } + + fn new_substr(&self, s: Wtf8Buf) -> Self { + let kind = if self.kind().is_ascii() || s.is_ascii() { + StrKind::Ascii + } else if self.kind().is_utf8() || s.is_utf8() { + StrKind::Utf8 + } else { + StrKind::Wtf8 + }; + unsafe { + // SAFETY: kind is properly decided for substring + Self::new_str_unchecked(s.into(), kind) + } + } + + #[inline] + pub const fn as_wtf8(&self) -> &Wtf8 { + self.data.as_wtf8() + } + + pub const fn as_bytes(&self) -> &[u8] { + self.data.as_wtf8().as_bytes() + } + + pub fn to_str(&self) -> Option<&str> { + self.data.as_str() + } + + /// Returns `&str` + /// + /// # Panic + /// If the string contains surrogates. + #[inline] + #[track_caller] + pub fn expect_str(&self) -> &str { + self.to_str().expect("PyStr contains surrogates") + } + + pub(crate) fn ensure_valid_utf8(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.is_utf8() { + Ok(()) + } else { + let start = self + .as_wtf8() + .code_points() + .position(|c| c.to_char().is_none()) + .unwrap(); + Err(vm.new_unicode_encode_error_real( + identifier!(vm, utf_8).to_owned(), + vm.ctx.new_str(self.data.clone()), + start, + start + 1, + vm.ctx.new_str("surrogates not allowed"), + )) + } + } + + pub fn to_string_lossy(&self) -> Cow<'_, str> { + self.to_str() + .map(Cow::Borrowed) + .unwrap_or_else(|| self.as_wtf8().to_string_lossy()) + } + + pub const fn kind(&self) -> StrKind { + self.data.kind() + } + + #[inline] + pub fn as_str_kind(&self) -> PyKindStr<'_> { + self.data.as_str_kind() + } + + pub const fn is_utf8(&self) -> bool { + self.kind().is_utf8() + } + + fn char_all<F>(&self, test: F) -> bool + where + F: Fn(char) -> bool, + { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.chars().all(|ch| test(ch.into())), + PyKindStr::Utf8(s) => s.chars().all(test), + PyKindStr::Wtf8(w) => w.code_points().all(|ch| ch.is_char_and(&test)), + } + } + + fn repeat(zelf: PyRef<Self>, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + if value == 0 && zelf.class().is(vm.ctx.types.str_type) { + // Special case: when some `str` is multiplied by `0`, + // returns the empty `str`. + return Ok(vm.ctx.empty_str.to_owned()); + } + if (value == 1 || zelf.is_empty()) && zelf.class().is(vm.ctx.types.str_type) { + // Special case: when some `str` is multiplied by `1` or is the empty `str`, + // nothing really happens, we need to return an object itself + // with the same `id()` to be compatible with CPython. + // This only works for `str` itself, not its subclasses. + return Ok(zelf); + } + zelf.as_wtf8() + .as_bytes() + .mul(vm, value) + .map(|x| Self::from(unsafe { Wtf8Buf::from_bytes_unchecked(x) }).into_ref(&vm.ctx)) + } + + pub fn try_as_utf8<'a>(&'a self, vm: &VirtualMachine) -> PyResult<&'a PyUtf8Str> { + // Check if the string contains surrogates + self.ensure_valid_utf8(vm)?; + // If no surrogates, we can safely cast to PyStr + Ok(unsafe { &*(self as *const _ as *const PyUtf8Str) }) + } +} + +impl Py<PyStr> { + pub fn try_as_utf8<'a>(&'a self, vm: &VirtualMachine) -> PyResult<&'a Py<PyUtf8Str>> { + // Check if the string contains surrogates + self.ensure_valid_utf8(vm)?; + // If no surrogates, we can safely cast to PyStr + Ok(unsafe { &*(self as *const _ as *const Py<PyUtf8Str>) }) + } +} + +#[pyclass( + flags(BASETYPE, _MATCH_SELF), + with( + AsMapping, + AsNumber, + AsSequence, + Representable, + Hashable, + Comparable, + Iterable, + Constructor + ) +)] +impl PyStr { + fn __add__(zelf: PyRef<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(other) = other.downcast_ref::<Self>() { + let bytes = zelf.as_wtf8().py_add(other.as_wtf8()); + Ok(unsafe { + // SAFETY: `kind` is safely decided + let kind = zelf.kind() | other.kind(); + Self::new_str_unchecked(bytes.into(), kind) + } + .to_pyobject(vm)) + } else if let Some(radd) = vm.get_method(other.clone(), identifier!(vm, __radd__)) { + // hack to get around not distinguishing number add from seq concat + radd?.call((zelf,), vm) + } else { + Err(vm.new_type_error(format!( + r#"can only concatenate str (not "{}") to str"#, + other.class().name() + ))) + } + } + + fn _contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + if let Some(needle) = needle.downcast_ref::<Self>() { + Ok(memchr::memmem::find(self.as_bytes(), needle.as_bytes()).is_some()) + } else { + Err(vm.new_type_error(format!( + "'in <string>' requires string as left operand, not {}", + needle.class().name() + ))) + } + } + + fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + self._contains(&needle, vm) + } + + fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + let item = match SequenceIndex::try_from_borrowed_object(vm, needle, "str")? { + SequenceIndex::Int(i) => self.getitem_by_index(vm, i)?.to_pyobject(vm), + SequenceIndex::Slice(slice) => self.getitem_by_slice(vm, slice)?.to_pyobject(vm), + }; + Ok(item) + } + + fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self._getitem(&needle, vm) + } + + #[inline] + pub(crate) fn hash(&self, vm: &VirtualMachine) -> hash::PyHash { + match self.hash.load(atomic::Ordering::Relaxed) { + hash::SENTINEL => self._compute_hash(vm), + hash => hash, + } + } + + #[cold] + fn _compute_hash(&self, vm: &VirtualMachine) -> hash::PyHash { + let hash_val = vm.state.hash_secret.hash_bytes(self.as_bytes()); + debug_assert_ne!(hash_val, hash::SENTINEL); + // spell-checker:ignore cmpxchg + // like with char_len, we don't need a cmpxchg loop, since it'll always be the same value + self.hash.store(hash_val, atomic::Ordering::Relaxed); + hash_val + } + + #[inline] + pub fn byte_len(&self) -> usize { + self.data.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + + #[inline] + pub fn char_len(&self) -> usize { + self.data.char_len() + } + + #[pymethod] + #[inline(always)] + pub const fn isascii(&self) -> bool { + matches!(self.kind(), StrKind::Ascii) + } + + #[pymethod] + fn __sizeof__(&self) -> usize { + core::mem::size_of::<Self>() + self.byte_len() * core::mem::size_of::<u8>() + } + + fn __mul__(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + Self::repeat(zelf, value.into(), vm) + } + + #[inline] + pub(crate) fn repr(&self, vm: &VirtualMachine) -> PyResult<String> { + use crate::literal::escape::UnicodeEscape; + UnicodeEscape::new_repr(self.as_wtf8()) + .str_repr() + .to_string() + .ok_or_else(|| vm.new_overflow_error("string is too long to generate repr")) + } + + #[pymethod] + fn lower(&self) -> Self { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.to_ascii_lowercase().into(), + PyKindStr::Utf8(s) => s.to_lowercase().into(), + PyKindStr::Wtf8(w) => w.to_lowercase().into(), + } + } + + // casefold is much more aggressive than lower + #[pymethod] + fn casefold(&self) -> Self { + match self.as_str_kind() { + PyKindStr::Ascii(s) => caseless::default_case_fold_str(s.as_str()).into(), + PyKindStr::Utf8(s) => caseless::default_case_fold_str(s).into(), + PyKindStr::Wtf8(w) => w + .chunks() + .map(|c| match c { + Wtf8Chunk::Utf8(s) => Wtf8Buf::from_string(caseless::default_case_fold_str(s)), + Wtf8Chunk::Surrogate(c) => Wtf8Buf::from(c), + }) + .collect::<Wtf8Buf>() + .into(), + } + } + + #[pymethod] + fn upper(&self) -> Self { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.to_ascii_uppercase().into(), + PyKindStr::Utf8(s) => s.to_uppercase().into(), + PyKindStr::Wtf8(w) => w.to_uppercase().into(), + } + } + + #[pymethod] + fn capitalize(&self) -> Wtf8Buf { + match self.as_str_kind() { + PyKindStr::Ascii(s) => { + let mut s = s.to_owned(); + if let [first, rest @ ..] = s.as_mut_slice() { + first.make_ascii_uppercase(); + ascii::AsciiStr::make_ascii_lowercase(rest.into()); + } + s.into() + } + PyKindStr::Utf8(s) => { + let mut chars = s.chars(); + let mut out = String::with_capacity(s.len()); + if let Some(c) = chars.next() { + out.extend(c.to_titlecase()); + out.push_str(&chars.as_str().to_lowercase()); + } + out.into() + } + PyKindStr::Wtf8(s) => { + let mut out = Wtf8Buf::with_capacity(s.len()); + let mut chars = s.code_points(); + if let Some(ch) = chars.next() { + match ch.to_char() { + Some(ch) => out.extend(ch.to_titlecase()), + None => out.push(ch), + } + out.push_wtf8(&chars.as_wtf8().to_lowercase()); + } + out + } + } + } + + #[pymethod] + fn split(zelf: &Py<Self>, args: SplitArgs, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let elements = match zelf.as_str_kind() { + PyKindStr::Ascii(s) => s.py_split( + args, + vm, + || zelf.as_object().to_owned(), + |v, s, vm| { + v.as_bytes() + .split_str(s) + .map(|s| unsafe { AsciiStr::from_ascii_unchecked(s) }.to_pyobject(vm)) + .collect() + }, + |v, s, n, vm| { + v.as_bytes() + .splitn_str(n, s) + .map(|s| unsafe { AsciiStr::from_ascii_unchecked(s) }.to_pyobject(vm)) + .collect() + }, + |v, n, vm| { + v.as_bytes().py_split_whitespace(n, |s| { + unsafe { AsciiStr::from_ascii_unchecked(s) }.to_pyobject(vm) + }) + }, + ), + PyKindStr::Utf8(s) => s.py_split( + args, + vm, + || zelf.as_object().to_owned(), + |v, s, vm| v.split(s).map(|s| vm.ctx.new_str(s).into()).collect(), + |v, s, n, vm| v.splitn(n, s).map(|s| vm.ctx.new_str(s).into()).collect(), + |v, n, vm| v.py_split_whitespace(n, |s| vm.ctx.new_str(s).into()), + ), + PyKindStr::Wtf8(w) => w.py_split( + args, + vm, + || zelf.as_object().to_owned(), + |v, s, vm| v.split(s).map(|s| vm.ctx.new_str(s).into()).collect(), + |v, s, n, vm| v.splitn(n, s).map(|s| vm.ctx.new_str(s).into()).collect(), + |v, n, vm| v.py_split_whitespace(n, |s| vm.ctx.new_str(s).into()), + ), + }?; + Ok(elements) + } + + #[pymethod] + fn rsplit(zelf: &Py<Self>, args: SplitArgs, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let mut elements = zelf.as_wtf8().py_split( + args, + vm, + || zelf.as_object().to_owned(), + |v, s, vm| v.rsplit(s).map(|s| vm.ctx.new_str(s).into()).collect(), + |v, s, n, vm| v.rsplitn(n, s).map(|s| vm.ctx.new_str(s).into()).collect(), + |v, n, vm| v.py_rsplit_whitespace(n, |s| vm.ctx.new_str(s).into()), + )?; + // Unlike Python rsplit, Rust rsplitn returns an iterator that + // starts from the end of the string. + elements.reverse(); + Ok(elements) + } + + #[pymethod] + fn strip(&self, chars: OptionalOption<PyStrRef>) -> Self { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s + .py_strip( + chars, + |s, chars| { + let s = s + .as_str() + .trim_matches(|c| memchr::memchr(c as _, chars.as_bytes()).is_some()); + unsafe { AsciiStr::from_ascii_unchecked(s.as_bytes()) } + }, + |s| s.trim(), + ) + .into(), + PyKindStr::Utf8(s) => s + .py_strip( + chars, + |s, chars| s.trim_matches(|c| chars.contains(c)), + |s| s.trim(), + ) + .into(), + PyKindStr::Wtf8(w) => w + .py_strip( + chars, + |s, chars| s.trim_matches(|c| chars.code_points().contains(&c)), + |s| s.trim(), + ) + .into(), + } + } + + #[pymethod] + fn lstrip( + zelf: PyRef<Self>, + chars: OptionalOption<PyStrRef>, + vm: &VirtualMachine, + ) -> PyRef<Self> { + let s = zelf.as_wtf8(); + let stripped = s.py_strip( + chars, + |s, chars| s.trim_start_matches(|c| chars.contains_code_point(c)), + |s| s.trim_start(), + ); + if s == stripped { + zelf + } else { + vm.ctx.new_str(stripped) + } + } + + #[pymethod] + fn rstrip( + zelf: PyRef<Self>, + chars: OptionalOption<PyStrRef>, + vm: &VirtualMachine, + ) -> PyRef<Self> { + let s = zelf.as_wtf8(); + let stripped = s.py_strip( + chars, + |s, chars| s.trim_end_matches(|c| chars.contains_code_point(c)), + |s| s.trim_end(), + ); + if s == stripped { + zelf + } else { + vm.ctx.new_str(stripped) + } + } + + #[pymethod] + fn endswith(&self, options: anystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult<bool> { + let (affix, substr) = + match options.prepare(self.as_wtf8(), self.len(), |s, r| s.get_chars(r)) { + Some(x) => x, + None => return Ok(false), + }; + substr.py_starts_ends_with( + &affix, + "endswith", + "str", + |s, x: &Py<Self>| s.ends_with(x.as_wtf8()), + vm, + ) + } + + #[pymethod] + fn startswith( + &self, + options: anystr::StartsEndsWithArgs, + vm: &VirtualMachine, + ) -> PyResult<bool> { + let (affix, substr) = + match options.prepare(self.as_wtf8(), self.len(), |s, r| s.get_chars(r)) { + Some(x) => x, + None => return Ok(false), + }; + substr.py_starts_ends_with( + &affix, + "startswith", + "str", + |s, x: &Py<Self>| s.starts_with(x.as_wtf8()), + vm, + ) + } + + #[pymethod] + fn removeprefix(&self, pref: PyStrRef) -> Wtf8Buf { + self.as_wtf8() + .py_removeprefix(pref.as_wtf8(), pref.byte_len(), |s, p| s.starts_with(p)) + .to_owned() + } + + #[pymethod] + fn removesuffix(&self, suffix: PyStrRef) -> Wtf8Buf { + self.as_wtf8() + .py_removesuffix(suffix.as_wtf8(), suffix.byte_len(), |s, p| s.ends_with(p)) + .to_owned() + } + + #[pymethod] + fn isalnum(&self) -> bool { + !self.data.is_empty() && self.char_all(char::is_alphanumeric) + } + + #[pymethod] + fn isnumeric(&self) -> bool { + !self.data.is_empty() && self.char_all(char::is_numeric) + } + + #[pymethod] + fn isdigit(&self) -> bool { + // python's isdigit also checks if exponents are digits, these are the unicode codepoints for exponents + !self.data.is_empty() + && self.char_all(|c| { + c.is_ascii_digit() + || matches!(c, '⁰' | '¹' | '²' | '³' | '⁴' | '⁵' | '⁶' | '⁷' | '⁸' | '⁹') + }) + } + + #[pymethod] + fn isdecimal(&self) -> bool { + !self.data.is_empty() + && self.char_all(|c| GeneralCategory::of(c) == GeneralCategory::DecimalNumber) + } + + fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + cformat_string(vm, self.as_wtf8(), values) + } + + #[pymethod] + fn format(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let format_str = + FormatString::from_str(self.as_wtf8()).map_err(|e| e.to_pyexception(vm))?; + format(&format_str, &args, vm) + } + + #[pymethod] + fn format_map(&self, mapping: PyObjectRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let format_string = + FormatString::from_str(self.as_wtf8()).map_err(|err| err.to_pyexception(vm))?; + format_map(&format_string, &mapping, vm) + } + + #[pymethod] + fn __format__( + zelf: PyRef<PyStr>, + spec: PyUtf8StrRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PyStr>> { + if spec.is_empty() { + return if zelf.class().is(vm.ctx.types.str_type) { + Ok(zelf) + } else { + zelf.as_object().str(vm) + }; + } + let zelf = zelf.try_into_utf8(vm)?; + let s = FormatSpec::parse(spec.as_str()) + .and_then(|format_spec| { + format_spec.format_string(&CharLenStr(zelf.as_str(), zelf.char_len())) + }) + .map_err(|err| err.into_pyexception(vm))?; + Ok(vm.ctx.new_str(s)) + } + + #[pymethod] + fn title(&self) -> Wtf8Buf { + let mut title = Wtf8Buf::with_capacity(self.data.len()); + let mut previous_is_cased = false; + for c_orig in self.as_wtf8().code_points() { + let c = c_orig.to_char_lossy(); + if c.is_lowercase() { + if !previous_is_cased { + title.extend(c.to_titlecase()); + } else { + title.push_char(c); + } + previous_is_cased = true; + } else if c.is_uppercase() || c.is_titlecase() { + if previous_is_cased { + title.extend(c.to_lowercase()); + } else { + title.push_char(c); + } + previous_is_cased = true; + } else { + previous_is_cased = false; + title.push(c_orig); + } + } + title + } + + #[pymethod] + fn swapcase(&self) -> Wtf8Buf { + let mut swapped_str = Wtf8Buf::with_capacity(self.data.len()); + for c_orig in self.as_wtf8().code_points() { + let c = c_orig.to_char_lossy(); + // to_uppercase returns an iterator, to_ascii_uppercase returns the char + if c.is_lowercase() { + swapped_str.push_char(c.to_ascii_uppercase()); + } else if c.is_uppercase() { + swapped_str.push_char(c.to_ascii_lowercase()); + } else { + swapped_str.push(c_orig); + } + } + swapped_str + } + + #[pymethod] + fn isalpha(&self) -> bool { + !self.data.is_empty() && self.char_all(char::is_alphabetic) + } + + #[pymethod] + fn replace(&self, args: ReplaceArgs) -> Wtf8Buf { + use core::cmp::Ordering; + + let s = self.as_wtf8(); + let ReplaceArgs { old, new, count } = args; + + match count.cmp(&0) { + Ordering::Less => s.replace(old.as_wtf8(), new.as_wtf8()), + Ordering::Equal => s.to_owned(), + Ordering::Greater => { + let s_is_empty = s.is_empty(); + let old_is_empty = old.is_empty(); + + if s_is_empty && !old_is_empty { + s.to_owned() + } else if s_is_empty && old_is_empty { + new.as_wtf8().to_owned() + } else { + s.replacen(old.as_wtf8(), new.as_wtf8(), count as usize) + } + } + } + } + + #[pymethod] + fn isprintable(&self) -> bool { + self.char_all(|c| c == '\u{0020}' || rustpython_literal::char::is_printable(c)) + } + + #[pymethod] + fn isspace(&self) -> bool { + use unic_ucd_bidi::bidi_class::abbr_names::*; + !self.data.is_empty() + && self.char_all(|c| { + GeneralCategory::of(c) == GeneralCategory::SpaceSeparator + || matches!(BidiClass::of(c), WS | B | S) + }) + } + + // Return true if all cased characters in the string are lowercase and there is at least one cased character, false otherwise. + #[pymethod] + fn islower(&self) -> bool { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.py_islower(), + PyKindStr::Utf8(s) => s.py_islower(), + PyKindStr::Wtf8(w) => w.py_islower(), + } + } + + // Return true if all cased characters in the string are uppercase and there is at least one cased character, false otherwise. + #[pymethod] + fn isupper(&self) -> bool { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s.py_isupper(), + PyKindStr::Utf8(s) => s.py_isupper(), + PyKindStr::Wtf8(w) => w.py_isupper(), + } + } + + #[pymethod] + fn splitlines(&self, args: anystr::SplitLinesArgs, vm: &VirtualMachine) -> Vec<PyObjectRef> { + let into_wrapper = |s: &Wtf8| self.new_substr(s.to_owned()).to_pyobject(vm); + let mut elements = Vec::new(); + let mut last_i = 0; + let self_str = self.as_wtf8(); + let mut enumerated = self_str.code_point_indices().peekable(); + while let Some((i, ch)) = enumerated.next() { + let end_len = match ch.to_char_lossy() { + '\n' => 1, + '\r' => { + let is_rn = enumerated.next_if(|(_, ch)| *ch == '\n').is_some(); + if is_rn { 2 } else { 1 } + } + '\x0b' | '\x0c' | '\x1c' | '\x1d' | '\x1e' | '\u{0085}' | '\u{2028}' + | '\u{2029}' => ch.len_wtf8(), + _ => continue, + }; + let range = if args.keepends { + last_i..i + end_len + } else { + last_i..i + }; + last_i = i + end_len; + elements.push(into_wrapper(&self_str[range])); + } + if last_i != self_str.len() { + elements.push(into_wrapper(&self_str[last_i..])); + } + elements + } + + #[pymethod] + fn join( + zelf: PyRef<Self>, + iterable: ArgIterable<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + let iter = iterable.iter(vm)?; + let joined = match iter.exactly_one() { + Ok(first) => { + let first = first?; + if first.as_object().class().is(vm.ctx.types.str_type) { + return Ok(first); + } else { + first.as_wtf8().to_owned() + } + } + Err(iter) => zelf.as_wtf8().py_join(iter)?, + }; + Ok(vm.ctx.new_str(joined)) + } + + // FIXME: two traversals of str is expensive + #[inline] + fn _to_char_idx(r: &Wtf8, byte_idx: usize) -> usize { + r[..byte_idx].code_points().count() + } + + #[inline] + fn _find<F>(&self, args: FindArgs, find: F) -> Option<usize> + where + F: Fn(&Wtf8, &Wtf8) -> Option<usize>, + { + let (sub, range) = args.get_value(self.len()); + self.as_wtf8().py_find(sub.as_wtf8(), range, find) + } + + #[pymethod] + fn find(&self, args: FindArgs) -> isize { + self._find(args, |r, s| Some(Self::_to_char_idx(r, r.find(s)?))) + .map_or(-1, |v| v as isize) + } + + #[pymethod] + fn rfind(&self, args: FindArgs) -> isize { + self._find(args, |r, s| Some(Self::_to_char_idx(r, r.rfind(s)?))) + .map_or(-1, |v| v as isize) + } + + #[pymethod] + fn index(&self, args: FindArgs, vm: &VirtualMachine) -> PyResult<usize> { + self._find(args, |r, s| Some(Self::_to_char_idx(r, r.find(s)?))) + .ok_or_else(|| vm.new_value_error("substring not found")) + } + + #[pymethod] + fn rindex(&self, args: FindArgs, vm: &VirtualMachine) -> PyResult<usize> { + self._find(args, |r, s| Some(Self::_to_char_idx(r, r.rfind(s)?))) + .ok_or_else(|| vm.new_value_error("substring not found")) + } + + #[pymethod] + fn partition(&self, sep: PyStrRef, vm: &VirtualMachine) -> PyResult { + let (front, has_mid, back) = self.as_wtf8().py_partition( + sep.as_wtf8(), + || self.as_wtf8().splitn(2, sep.as_wtf8()), + vm, + )?; + let partition = ( + self.new_substr(front), + if has_mid { + sep + } else { + vm.ctx.new_str(ascii!("")) + }, + self.new_substr(back), + ); + Ok(partition.to_pyobject(vm)) + } + + #[pymethod] + fn rpartition(&self, sep: PyStrRef, vm: &VirtualMachine) -> PyResult { + let (back, has_mid, front) = self.as_wtf8().py_partition( + sep.as_wtf8(), + || self.as_wtf8().rsplitn(2, sep.as_wtf8()), + vm, + )?; + Ok(( + self.new_substr(front), + if has_mid { + sep + } else { + vm.ctx.empty_str.to_owned() + }, + self.new_substr(back), + ) + .to_pyobject(vm)) + } + + #[pymethod] + fn istitle(&self) -> bool { + if self.data.is_empty() { + return false; + } + + let mut cased = false; + let mut previous_is_cased = false; + for c in self.as_wtf8().code_points().map(CodePoint::to_char_lossy) { + if c.is_uppercase() || c.is_titlecase() { + if previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else if c.is_lowercase() { + if !previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else { + previous_is_cased = false; + } + } + cased + } + + #[pymethod] + fn count(&self, args: FindArgs) -> usize { + let (needle, range) = args.get_value(self.len()); + self.as_wtf8() + .py_count(needle.as_wtf8(), range, |h, n| h.find_iter(n).count()) + } + + #[pymethod] + fn zfill(&self, width: isize) -> Wtf8Buf { + unsafe { + // SAFETY: this is safe-guaranteed because the original self.as_wtf8() is valid wtf8 + Wtf8Buf::from_bytes_unchecked(self.as_wtf8().py_zfill(width)) + } + } + + #[inline] + fn _pad( + &self, + width: isize, + fillchar: OptionalArg<PyStrRef>, + pad: fn(&Wtf8, usize, CodePoint, usize) -> Wtf8Buf, + vm: &VirtualMachine, + ) -> PyResult<Wtf8Buf> { + let fillchar = fillchar.map_or(Ok(' '.into()), |ref s| { + s.as_wtf8().code_points().exactly_one().map_err(|_| { + vm.new_type_error("The fill character must be exactly one character long") + }) + })?; + Ok(if self.len() as isize >= width { + self.as_wtf8().to_owned() + } else { + pad(self.as_wtf8(), width as usize, fillchar, self.len()) + }) + } + + #[pymethod] + fn center( + &self, + width: isize, + fillchar: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<Wtf8Buf> { + self._pad(width, fillchar, AnyStr::py_center, vm) + } + + #[pymethod] + fn ljust( + &self, + width: isize, + fillchar: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<Wtf8Buf> { + self._pad(width, fillchar, AnyStr::py_ljust, vm) + } + + #[pymethod] + fn rjust( + &self, + width: isize, + fillchar: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<Wtf8Buf> { + self._pad(width, fillchar, AnyStr::py_rjust, vm) + } + + #[pymethod] + fn expandtabs(&self, args: anystr::ExpandTabsArgs, vm: &VirtualMachine) -> PyResult<String> { + // TODO: support WTF-8 + Ok(rustpython_common::str::expandtabs( + self.try_as_utf8(vm)?.as_str(), + args.tabsize(), + )) + } + + #[pymethod] + pub fn isidentifier(&self) -> bool { + let Some(s) = self.to_str() else { return false }; + let mut chars = s.chars(); + let is_identifier_start = chars.next().is_some_and(|c| c == '_' || is_xid_start(c)); + // a string is not an identifier if it has whitespace or starts with a number + is_identifier_start && chars.all(is_xid_continue) + } + + // https://docs.python.org/3/library/stdtypes.html#str.translate + #[pymethod] + fn translate(&self, table: PyObjectRef, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + vm.get_method_or_type_error(table.clone(), identifier!(vm, __getitem__), || { + format!("'{}' object is not subscriptable", table.class().name()) + })?; + + let mut translated = Wtf8Buf::new(); + for cp in self.as_wtf8().code_points() { + match table.get_item(&*cp.to_u32().to_pyobject(vm), vm) { + Ok(value) => { + if let Some(text) = value.downcast_ref::<Self>() { + translated.push_wtf8(text.as_wtf8()); + } else if let Some(bigint) = value.downcast_ref::<PyInt>() { + let mapped = bigint + .as_bigint() + .to_u32() + .and_then(CodePoint::from_u32) + .ok_or_else(|| { + vm.new_value_error("character mapping must be in range(0x110000)") + })?; + translated.push(mapped); + } else if !vm.is_none(&value) { + return Err( + vm.new_type_error("character mapping must return integer, None or str") + ); + } + } + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => translated.push(cp), + Err(e) => return Err(e), + } + } + Ok(translated) + } + + #[pystaticmethod] + fn maketrans( + dict_or_str: PyObjectRef, + to_str: OptionalArg<PyStrRef>, + none_str: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult { + let new_dict = vm.ctx.new_dict(); + if let OptionalArg::Present(to_str) = to_str { + match dict_or_str.downcast::<Self>() { + Ok(from_str) => { + if to_str.len() == from_str.len() { + for (c1, c2) in from_str + .as_wtf8() + .code_points() + .zip(to_str.as_wtf8().code_points()) + { + new_dict.set_item( + &*vm.new_pyobj(c1.to_u32()), + vm.new_pyobj(c2.to_u32()), + vm, + )?; + } + if let OptionalArg::Present(none_str) = none_str { + for c in none_str.as_wtf8().code_points() { + new_dict.set_item(&*vm.new_pyobj(c.to_u32()), vm.ctx.none(), vm)?; + } + } + Ok(new_dict.to_pyobject(vm)) + } else { + Err(vm.new_value_error( + "the first two maketrans arguments must have equal length", + )) + } + } + _ => Err(vm.new_type_error( + "first maketrans argument must be a string if there is a second argument", + )), + } + } else { + // dict_str must be a dict + match dict_or_str.downcast::<PyDict>() { + Ok(dict) => { + for (key, val) in dict { + // FIXME: ints are key-compatible + if let Some(num) = key.downcast_ref::<PyInt>() { + new_dict.set_item( + &*num.as_bigint().to_i32().to_pyobject(vm), + val, + vm, + )?; + } else if let Some(string) = key.downcast_ref::<Self>() { + if string.len() == 1 { + let num_value = + string.as_wtf8().code_points().next().unwrap().to_u32(); + new_dict.set_item(&*num_value.to_pyobject(vm), val, vm)?; + } else { + return Err(vm.new_value_error( + "string keys in translate table must be of length 1", + )); + } + } else { + return Err(vm.new_type_error( + "keys in translate table must be strings or integers", + )); + } + } + Ok(new_dict.to_pyobject(vm)) + } + _ => Err(vm.new_value_error( + "if you give only one argument to maketrans it must be a dict", + )), + } + } + } + + #[pymethod] + fn encode(zelf: PyRef<Self>, args: EncodeArgs, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + encode_string(zelf, args.encoding, args.errors, vm) + } + + #[pymethod] + fn __getnewargs__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyObjectRef { + (zelf.as_wtf8(),).to_pyobject(vm) + } + + #[pymethod] + fn __str__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + if zelf.class().is(vm.ctx.types.str_type) { + // Already exact str, just return a reference + Ok(zelf.to_owned()) + } else { + // Subclass, create a new exact str + Ok(PyStr::from(zelf.data.clone()).into_ref(&vm.ctx)) + } + } +} + +impl PyRef<PyStr> { + pub fn is_empty(&self) -> bool { + (**self).is_empty() + } + + pub fn concat_in_place(&mut self, other: &Wtf8, vm: &VirtualMachine) { + if other.is_empty() { + return; + } + let mut s = Wtf8Buf::with_capacity(self.byte_len() + other.len()); + s.push_wtf8(self.as_ref()); + s.push_wtf8(other); + if self.as_object().strong_count() == 1 { + // SAFETY: strong_count()==1 guarantees unique ownership of this PyStr. + // Mutating payload in place preserves semantics while avoiding PyObject reallocation. + unsafe { + let payload = self.payload() as *const PyStr as *mut PyStr; + (*payload).data = PyStr::from(s).data; + (*payload) + .hash + .store(hash::SENTINEL, atomic::Ordering::Relaxed); + } + } else { + *self = PyStr::from(s).into_ref(&vm.ctx); + } + } + + pub fn try_into_utf8(self, vm: &VirtualMachine) -> PyResult<PyRef<PyUtf8Str>> { + self.ensure_valid_utf8(vm)?; + Ok(unsafe { mem::transmute::<Self, PyRef<PyUtf8Str>>(self) }) + } +} + +struct CharLenStr<'a>(&'a str, usize); +impl core::ops::Deref for CharLenStr<'_> { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.0 + } +} +impl crate::common::format::CharLen for CharLenStr<'_> { + fn char_len(&self) -> usize { + self.1 + } +} + +impl Representable for PyStr { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + zelf.repr(vm) + } +} + +impl Hashable for PyStr { + #[inline] + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { + Ok(zelf.hash(vm)) + } +} + +impl Comparable for PyStr { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + if let Some(res) = op.identical_optimization(zelf, other) { + return Ok(res.into()); + } + let other = class_or_notimplemented!(Self, other); + Ok(op.eval_ord(zelf.as_wtf8().cmp(other.as_wtf8())).into()) + } +} + +impl Iterable for PyStr { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyStrIterator { + internal: PyMutex::new((PositionIterInternal::new(zelf, 0), 0)), + } + .into_pyobject(vm)) + } +} + +impl AsMapping for PyStr { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + length: atomic_func!(|mapping, _vm| Ok(PyStr::mapping_downcast(mapping).len())), + subscript: atomic_func!( + |mapping, needle, vm| PyStr::mapping_downcast(mapping)._getitem(needle, vm) + ), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } +} + +impl AsNumber for PyStr { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + add: Some(|a, b, vm| { + let Some(a) = a.downcast_ref::<PyStr>() else { + return Ok(vm.ctx.not_implemented()); + }; + let Some(b) = b.downcast_ref::<PyStr>() else { + return Ok(vm.ctx.not_implemented()); + }; + let bytes = a.as_wtf8().py_add(b.as_wtf8()); + Ok(unsafe { + let kind = a.kind() | b.kind(); + PyStr::new_str_unchecked(bytes.into(), kind) + } + .to_pyobject(vm)) + }), + remainder: Some(|a, b, vm| { + if let Some(a) = a.downcast_ref::<PyStr>() { + a.__mod__(b.to_owned(), vm).to_pyresult(vm) + } else { + Ok(vm.ctx.not_implemented()) + } + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl AsSequence for PyStr { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyStr::sequence_downcast(seq).len())), + concat: atomic_func!(|seq, other, vm| { + let zelf = PyStr::sequence_downcast(seq); + PyStr::__add__(zelf.to_owned(), other.to_owned(), vm) + }), + repeat: atomic_func!(|seq, n, vm| { + let zelf = PyStr::sequence_downcast(seq); + PyStr::repeat(zelf.to_owned(), n, vm).map(|x| x.into()) + }), + item: atomic_func!(|seq, i, vm| { + let zelf = PyStr::sequence_downcast(seq); + zelf.getitem_by_index(vm, i).to_pyresult(vm) + }), + contains: atomic_func!( + |seq, needle, vm| PyStr::sequence_downcast(seq)._contains(needle, vm) + ), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +#[derive(FromArgs)] +struct EncodeArgs { + #[pyarg(any, default)] + encoding: Option<PyUtf8StrRef>, + #[pyarg(any, default)] + errors: Option<PyUtf8StrRef>, +} + +pub(crate) fn encode_string( + s: PyStrRef, + encoding: Option<PyUtf8StrRef>, + errors: Option<PyUtf8StrRef>, + vm: &VirtualMachine, +) -> PyResult<PyBytesRef> { + let encoding = match encoding.as_ref() { + None => crate::codecs::DEFAULT_ENCODING, + Some(s) => s.as_str(), + }; + vm.state.codec_registry.encode_text(s, encoding, errors, vm) +} + +impl PyPayload for PyStr { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.str_type + } +} + +impl ToPyObject for String { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + +impl ToPyObject for Wtf8Buf { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + +impl ToPyObject for char { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + let cp = self as u32; + if cp <= u8::MAX as u32 { + vm.ctx.latin1_char(cp as u8).into() + } else { + vm.ctx.new_str(self).into() + } + } +} + +impl ToPyObject for CodePoint { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + let cp = self.to_u32(); + if cp <= u8::MAX as u32 { + vm.ctx.latin1_char(cp as u8).into() + } else { + vm.ctx.new_str(self).into() + } + } +} + +impl ToPyObject for &str { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + +impl ToPyObject for &String { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self.clone()).into() + } +} + +impl ToPyObject for &Wtf8 { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + +impl ToPyObject for &Wtf8Buf { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self.clone()).into() + } +} + +impl ToPyObject for &AsciiStr { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + +impl ToPyObject for AsciiString { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str(self).into() + } +} + +impl ToPyObject for AsciiChar { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.latin1_char(u8::from(self)).into() + } +} + +type SplitArgs = anystr::SplitArgs<PyStrRef>; + +#[derive(FromArgs)] +pub struct FindArgs { + #[pyarg(positional)] + sub: PyStrRef, + #[pyarg(positional, default)] + start: Option<PyIntRef>, + #[pyarg(positional, default)] + end: Option<PyIntRef>, +} + +impl FindArgs { + fn get_value(self, len: usize) -> (PyStrRef, core::ops::Range<usize>) { + let range = adjust_indices(self.start, self.end, len); + (self.sub, range) + } +} + +#[derive(FromArgs)] +struct ReplaceArgs { + #[pyarg(positional)] + old: PyStrRef, + + #[pyarg(positional)] + new: PyStrRef, + + #[pyarg(any, default = -1)] + count: isize, +} + +pub fn init(ctx: &'static Context) { + PyStr::extend_class(ctx, ctx.types.str_type); + + PyStrIterator::extend_class(ctx, ctx.types.str_iterator_type); +} + +impl SliceableSequenceOp for PyStr { + type Item = CodePoint; + type Sliced = Self; + + fn do_get(&self, index: usize) -> Self::Item { + self.data.nth_char(index) + } + + fn do_slice(&self, range: Range<usize>) -> Self::Sliced { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s[range].into(), + PyKindStr::Utf8(s) => { + let char_len = range.len(); + let out = rustpython_common::str::get_chars(s, range); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } + } + PyKindStr::Wtf8(w) => { + let char_len = range.len(); + let out = rustpython_common::str::get_codepoints(w, range); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } + } + } + } + + fn do_slice_reverse(&self, range: Range<usize>) -> Self::Sliced { + match self.as_str_kind() { + PyKindStr::Ascii(s) => { + let mut out = s[range].to_owned(); + out.as_mut_slice().reverse(); + out.into() + } + PyKindStr::Utf8(s) => { + let char_len = range.len(); + let mut out = String::with_capacity(2 * char_len); + out.extend( + s.chars() + .rev() + .skip(self.char_len() - range.end) + .take(range.len()), + ); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, range.len()) } + } + PyKindStr::Wtf8(w) => { + let char_len = range.len(); + let mut out = Wtf8Buf::with_capacity(2 * char_len); + out.extend( + w.code_points() + .rev() + .skip(self.char_len() - range.end) + .take(range.len()), + ); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } + } + } + } + + fn do_stepped_slice(&self, range: Range<usize>, step: usize) -> Self::Sliced { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s[range] + .as_slice() + .iter() + .copied() + .step_by(step) + .collect::<AsciiString>() + .into(), + PyKindStr::Utf8(s) => { + let char_len = (range.len() / step) + 1; + let mut out = String::with_capacity(2 * char_len); + out.extend(s.chars().skip(range.start).take(range.len()).step_by(step)); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } + } + PyKindStr::Wtf8(w) => { + let char_len = (range.len() / step) + 1; + let mut out = Wtf8Buf::with_capacity(2 * char_len); + out.extend( + w.code_points() + .skip(range.start) + .take(range.len()) + .step_by(step), + ); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } + } + } + } + + fn do_stepped_slice_reverse(&self, range: Range<usize>, step: usize) -> Self::Sliced { + match self.as_str_kind() { + PyKindStr::Ascii(s) => s[range] + .chars() + .rev() + .step_by(step) + .collect::<AsciiString>() + .into(), + PyKindStr::Utf8(s) => { + let char_len = (range.len() / step) + 1; + // not ascii, so the codepoints have to be at least 2 bytes each + let mut out = String::with_capacity(2 * char_len); + out.extend( + s.chars() + .rev() + .skip(self.char_len() - range.end) + .take(range.len()) + .step_by(step), + ); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } + } + PyKindStr::Wtf8(w) => { + let char_len = (range.len() / step) + 1; + // not ascii, so the codepoints have to be at least 2 bytes each + let mut out = Wtf8Buf::with_capacity(2 * char_len); + out.extend( + w.code_points() + .rev() + .skip(self.char_len() - range.end) + .take(range.len()) + .step_by(step), + ); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } + } + } + } + + fn empty() -> Self::Sliced { + Self::default() + } + + fn len(&self) -> usize { + self.char_len() + } +} + +impl AsRef<str> for PyRefExact<PyStr> { + #[track_caller] + fn as_ref(&self) -> &str { + self.to_str().expect("str has surrogates") + } +} + +impl AsRef<str> for PyExact<PyStr> { + #[track_caller] + fn as_ref(&self) -> &str { + self.to_str().expect("str has surrogates") + } +} + +impl AsRef<Wtf8> for PyRefExact<PyStr> { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl AsRef<Wtf8> for PyExact<PyStr> { + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl AnyStrWrapper<Wtf8> for PyStrRef { + fn as_ref(&self) -> Option<&Wtf8> { + Some(self.as_wtf8()) + } + + fn is_empty(&self) -> bool { + self.data.is_empty() + } +} + +impl AnyStrWrapper<str> for PyStrRef { + fn as_ref(&self) -> Option<&str> { + self.data.as_str() + } + + fn is_empty(&self) -> bool { + self.data.is_empty() + } +} + +impl AnyStrWrapper<AsciiStr> for PyStrRef { + fn as_ref(&self) -> Option<&AsciiStr> { + self.data.as_ascii() + } + + fn is_empty(&self) -> bool { + self.data.is_empty() + } +} + +#[repr(transparent)] +#[derive(Debug)] +pub struct PyUtf8Str(PyStr); + +impl fmt::Display for PyUtf8Str { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl MaybeTraverse for PyUtf8Str { + const HAS_TRAVERSE: bool = true; + const HAS_CLEAR: bool = false; + + fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.0.try_traverse(traverse_fn); + } + + fn try_clear(&mut self, _out: &mut Vec<PyObjectRef>) { + // No clear needed for PyUtf8Str + } +} + +impl PyPayload for PyUtf8Str { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.str_type + } + + const PAYLOAD_TYPE_ID: core::any::TypeId = core::any::TypeId::of::<PyStr>(); + + unsafe fn validate_downcastable_from(obj: &PyObject) -> bool { + // SAFETY: we know the object is a PyStr in this context + let wtf8 = unsafe { obj.downcast_unchecked_ref::<PyStr>() }; + wtf8.is_utf8() + } + + fn try_downcast_from(obj: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + let str = obj.try_downcast_ref::<PyStr>(vm)?; + str.ensure_valid_utf8(vm) + } +} + +impl<'a> From<&'a AsciiStr> for PyUtf8Str { + fn from(s: &'a AsciiStr) -> Self { + s.to_owned().into() + } +} + +impl From<AsciiString> for PyUtf8Str { + fn from(s: AsciiString) -> Self { + s.into_boxed_ascii_str().into() + } +} + +impl From<Box<AsciiStr>> for PyUtf8Str { + fn from(s: Box<AsciiStr>) -> Self { + let data = StrData::from(s); + unsafe { Self::from_str_data_unchecked(data) } + } +} + +impl From<AsciiChar> for PyUtf8Str { + fn from(ch: AsciiChar) -> Self { + AsciiString::from(ch).into() + } +} + +impl<'a> From<&'a str> for PyUtf8Str { + fn from(s: &'a str) -> Self { + s.to_owned().into() + } +} + +impl From<String> for PyUtf8Str { + fn from(s: String) -> Self { + s.into_boxed_str().into() + } +} + +impl From<char> for PyUtf8Str { + fn from(ch: char) -> Self { + let data = StrData::from(ch); + unsafe { Self::from_str_data_unchecked(data) } + } +} + +impl<'a> From<alloc::borrow::Cow<'a, str>> for PyUtf8Str { + fn from(s: alloc::borrow::Cow<'a, str>) -> Self { + s.into_owned().into() + } +} + +impl From<Box<str>> for PyUtf8Str { + #[inline] + fn from(value: Box<str>) -> Self { + let data = StrData::from(value); + unsafe { Self::from_str_data_unchecked(data) } + } +} + +impl AsRef<Wtf8> for PyUtf8Str { + #[inline] + fn as_ref(&self) -> &Wtf8 { + self.0.as_wtf8() + } +} + +impl AsRef<str> for PyUtf8Str { + #[inline] + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl PyUtf8Str { + // Create a new `PyUtf8Str` from `StrData` without validation. + // This function must be only used in this module to create conversions. + // # Safety: must be called with a valid UTF-8 string data. + unsafe fn from_str_data_unchecked(data: StrData) -> Self { + Self(PyStr::from(data)) + } + + /// Returns the underlying WTF-8 slice (always valid UTF-8 for this type). + #[inline] + pub fn as_wtf8(&self) -> &Wtf8 { + self.0.as_wtf8() + } + + /// Returns the underlying string slice. + pub fn as_str(&self) -> &str { + debug_assert!( + self.0.is_utf8(), + "PyUtf8Str invariant violated: inner string is not valid UTF-8" + ); + // Safety: This is safe because the type invariant guarantees UTF-8 validity. + unsafe { self.0.to_str().unwrap_unchecked() } + } + + #[inline] + pub fn as_bytes(&self) -> &[u8] { + self.as_str().as_bytes() + } + + #[inline] + pub fn byte_len(&self) -> usize { + self.0.byte_len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + #[inline] + pub fn char_len(&self) -> usize { + self.0.char_len() + } +} + +impl Py<PyUtf8Str> { + /// Upcast to PyStr. + pub fn as_pystr(&self) -> &Py<PyStr> { + unsafe { + // Safety: PyUtf8Str is a wrapper around PyStr, so this cast is safe. + &*(self as *const Self as *const Py<PyStr>) + } + } + + /// Returns the underlying `&str`. + #[inline] + pub fn as_str(&self) -> &str { + self.as_pystr().to_str().unwrap_or_else(|| { + debug_assert!(false, "PyUtf8Str invariant violated"); + // Safety: PyUtf8Str guarantees valid UTF-8 + unsafe { core::hint::unreachable_unchecked() } + }) + } +} + +impl PyRef<PyUtf8Str> { + /// Convert to PyStrRef. Safe because PyUtf8Str is a subtype of PyStr. + pub fn into_wtf8(self) -> PyStrRef { + unsafe { mem::transmute::<Self, PyStrRef>(self) } + } +} + +impl From<PyRef<PyUtf8Str>> for PyRef<PyStr> { + fn from(s: PyRef<PyUtf8Str>) -> Self { + s.into_wtf8() + } +} + +impl PartialEq for PyUtf8Str { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} +impl Eq for PyUtf8Str {} + +impl AnyStrContainer<str> for String { + fn new() -> Self { + Self::new() + } + + fn with_capacity(capacity: usize) -> Self { + Self::with_capacity(capacity) + } + + fn push_str(&mut self, other: &str) { + Self::push_str(self, other) + } +} + +impl anystr::AnyChar for char { + fn is_lowercase(self) -> bool { + self.is_lowercase() + } + + fn is_uppercase(self) -> bool { + self.is_uppercase() + } + + fn bytes_len(self) -> usize { + self.len_utf8() + } +} + +impl AnyStr for str { + type Char = char; + type Container = String; + + fn to_container(&self) -> Self::Container { + self.to_owned() + } + + fn as_bytes(&self) -> &[u8] { + self.as_bytes() + } + + fn elements(&self) -> impl Iterator<Item = char> { + Self::chars(self) + } + + fn get_bytes(&self, range: core::ops::Range<usize>) -> &Self { + &self[range] + } + + fn get_chars(&self, range: core::ops::Range<usize>) -> &Self { + rustpython_common::str::get_chars(self, range) + } + + fn is_empty(&self) -> bool { + Self::is_empty(self) + } + + fn bytes_len(&self) -> usize { + Self::len(self) + } + + fn py_split_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef, + { + // CPython split_whitespace + let mut splits = Vec::new(); + let mut last_offset = 0; + let mut count = maxsplit; + for (offset, _) in self.match_indices(|c: char| c.is_ascii_whitespace() || c == '\x0b') { + if last_offset == offset { + last_offset += 1; + continue; + } + if count == 0 { + break; + } + splits.push(convert(&self[last_offset..offset])); + last_offset = offset + 1; + count -= 1; + } + if last_offset != self.len() { + splits.push(convert(&self[last_offset..])); + } + splits + } + + fn py_rsplit_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef, + { + // CPython rsplit_whitespace + let mut splits = Vec::new(); + let mut last_offset = self.len(); + let mut count = maxsplit; + for (offset, _) in self.rmatch_indices(|c: char| c.is_ascii_whitespace() || c == '\x0b') { + if last_offset == offset + 1 { + last_offset -= 1; + continue; + } + if count == 0 { + break; + } + splits.push(convert(&self[offset + 1..last_offset])); + last_offset = offset; + count -= 1; + } + if last_offset != 0 { + splits.push(convert(&self[..last_offset])); + } + splits + } +} + +impl AnyStrContainer<Wtf8> for Wtf8Buf { + fn new() -> Self { + Self::new() + } + + fn with_capacity(capacity: usize) -> Self { + Self::with_capacity(capacity) + } + + fn push_str(&mut self, other: &Wtf8) { + self.push_wtf8(other) + } +} + +impl anystr::AnyChar for CodePoint { + fn is_lowercase(self) -> bool { + self.is_char_and(char::is_lowercase) + } + fn is_uppercase(self) -> bool { + self.is_char_and(char::is_uppercase) + } + fn bytes_len(self) -> usize { + self.len_wtf8() + } +} + +impl AnyStr for Wtf8 { + type Char = CodePoint; + type Container = Wtf8Buf; + + fn to_container(&self) -> Self::Container { + self.to_owned() + } + + fn as_bytes(&self) -> &[u8] { + self.as_bytes() + } + + fn elements(&self) -> impl Iterator<Item = Self::Char> { + self.code_points() + } + + fn get_bytes(&self, range: core::ops::Range<usize>) -> &Self { + &self[range] + } + + fn get_chars(&self, range: core::ops::Range<usize>) -> &Self { + rustpython_common::str::get_codepoints(self, range) + } + + fn bytes_len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn py_split_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef, + { + // CPython split_whitespace + let mut splits = Vec::new(); + let mut last_offset = 0; + let mut count = maxsplit; + for (offset, _) in self + .code_point_indices() + .filter(|(_, c)| c.is_char_and(|c| c.is_ascii_whitespace() || c == '\x0b')) + { + if last_offset == offset { + last_offset += 1; + continue; + } + if count == 0 { + break; + } + splits.push(convert(&self[last_offset..offset])); + last_offset = offset + 1; + count -= 1; + } + if last_offset != self.len() { + splits.push(convert(&self[last_offset..])); + } + splits + } + + fn py_rsplit_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef, + { + // CPython rsplit_whitespace + let mut splits = Vec::new(); + let mut last_offset = self.len(); + let mut count = maxsplit; + for (offset, _) in self + .code_point_indices() + .rev() + .filter(|(_, c)| c.is_char_and(|c| c.is_ascii_whitespace() || c == '\x0b')) + { + if last_offset == offset + 1 { + last_offset -= 1; + continue; + } + if count == 0 { + break; + } + splits.push(convert(&self[offset + 1..last_offset])); + last_offset = offset; + count -= 1; + } + if last_offset != 0 { + splits.push(convert(&self[..last_offset])); + } + splits + } +} + +impl AnyStrContainer<AsciiStr> for AsciiString { + fn new() -> Self { + Self::new() + } + + fn with_capacity(capacity: usize) -> Self { + Self::with_capacity(capacity) + } + + fn push_str(&mut self, other: &AsciiStr) { + Self::push_str(self, other) + } +} + +impl anystr::AnyChar for ascii::AsciiChar { + fn is_lowercase(self) -> bool { + self.is_lowercase() + } + + fn is_uppercase(self) -> bool { + self.is_uppercase() + } + + fn bytes_len(self) -> usize { + 1 + } +} + +const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; + +impl AnyStr for AsciiStr { + type Char = AsciiChar; + type Container = AsciiString; + + fn to_container(&self) -> Self::Container { + self.to_ascii_string() + } + + fn as_bytes(&self) -> &[u8] { + self.as_bytes() + } + + fn elements(&self) -> impl Iterator<Item = Self::Char> { + self.chars() + } + + fn get_bytes(&self, range: core::ops::Range<usize>) -> &Self { + &self[range] + } + + fn get_chars(&self, range: core::ops::Range<usize>) -> &Self { + &self[range] + } + + fn bytes_len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn py_split_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef, + { + let mut splits = Vec::new(); + let mut count = maxsplit; + let mut haystack = self; + while let Some(offset) = haystack.as_bytes().find_byteset(ASCII_WHITESPACES) { + if offset != 0 { + if count == 0 { + break; + } + splits.push(convert(&haystack[..offset])); + count -= 1; + } + haystack = &haystack[offset + 1..]; + } + if !haystack.is_empty() { + splits.push(convert(haystack)); + } + splits + } + + fn py_rsplit_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef, + { + // CPython rsplit_whitespace + let mut splits = Vec::new(); + let mut count = maxsplit; + let mut haystack = self; + while let Some(offset) = haystack.as_bytes().rfind_byteset(ASCII_WHITESPACES) { + if offset + 1 != haystack.len() { + if count == 0 { + break; + } + splits.push(convert(&haystack[offset + 1..])); + count -= 1; + } + haystack = &haystack[..offset]; + } + if !haystack.is_empty() { + splits.push(convert(haystack)); + } + splits + } +} + +/// The unique reference of interned PyStr +/// Always intended to be used as a static reference +pub type PyStrInterned = PyInterned<PyStr>; + +impl PyStrInterned { + #[inline] + pub fn to_exact(&'static self) -> PyRefExact<PyStr> { + unsafe { PyRefExact::new_unchecked(self.to_owned()) } + } +} + +impl core::fmt::Display for PyStrInterned { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.data.fmt(f) + } +} + +impl AsRef<str> for PyStrInterned { + #[inline(always)] + fn as_ref(&self) -> &str { + self.to_str() + .expect("Interned PyStr should always be valid UTF-8") + } +} + +/// Interned PyUtf8Str — guaranteed UTF-8 at type level. +/// Same layout as `PyStrInterned` due to `#[repr(transparent)]` on both +/// `PyInterned<T>` and `PyUtf8Str`. +pub type PyUtf8StrInterned = PyInterned<PyUtf8Str>; + +impl PyUtf8StrInterned { + /// Returns the underlying `&str`. + #[inline] + pub fn as_str(&self) -> &str { + Py::<PyUtf8Str>::as_str(self) + } + + /// View as `PyStrInterned` (widening: UTF-8 → WTF-8). + #[inline] + pub fn as_interned_str(&self) -> &PyStrInterned { + // Safety: PyUtf8Str is #[repr(transparent)] over PyStr, + // so PyInterned<PyUtf8Str> has the same layout as PyInterned<PyStr>. + unsafe { &*(self as *const Self as *const PyStrInterned) } + } + + /// Narrow a `PyStrInterned` to `PyUtf8StrInterned`. + /// + /// # Safety + /// The caller must ensure that the interned string is valid UTF-8. + #[inline] + pub unsafe fn from_str_interned_unchecked(s: &PyStrInterned) -> &Self { + unsafe { &*(s as *const PyStrInterned as *const Self) } + } +} + +impl core::fmt::Display for PyUtf8StrInterned { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl AsRef<str> for PyUtf8StrInterned { + #[inline(always)] + fn as_ref(&self) -> &str { + self.as_str() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Interpreter; + use rustpython_common::wtf8::Wtf8Buf; + + #[test] + fn str_title() { + let tests = vec![ + (" Hello ", " hello "), + ("Hello ", "hello "), + ("Hello ", "Hello "), + ("Format This As Title String", "fOrMaT thIs aS titLe String"), + ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), + ("Getint", "getInt"), + // spell-checker:disable-next-line + ("Greek Ωppercases ...", "greek ωppercases ..."), + // spell-checker:disable-next-line + ("Greek ῼitlecases ...", "greek ῳitlecases ..."), + ]; + for (title, input) in tests { + assert_eq!(PyStr::from(input).title().as_str(), Ok(title)); + } + } + + #[test] + fn str_istitle() { + let pos = vec![ + "A", + "A Titlecased Line", + "A\nTitlecased Line", + "A Titlecased, Line", + // spell-checker:disable-next-line + "Greek Ωppercases ...", + // spell-checker:disable-next-line + "Greek ῼitlecases ...", + ]; + + for s in pos { + assert!(PyStr::from(s).istitle()); + } + + let neg = vec![ + "", + "a", + "\n", + "Not a capitalized String", + "Not\ta Titlecase String", + "Not--a Titlecase String", + "NOT", + ]; + for s in neg { + assert!(!PyStr::from(s).istitle()); + } + } + + #[test] + fn str_maketrans_and_translate() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let table = vm.ctx.new_dict(); + table + .set_item("a", vm.ctx.new_str("🎅").into(), vm) + .unwrap(); + table.set_item("b", vm.ctx.none(), vm).unwrap(); + table + .set_item("c", vm.ctx.new_str(ascii!("xda")).into(), vm) + .unwrap(); + let translated = + PyStr::maketrans(table.into(), OptionalArg::Missing, OptionalArg::Missing, vm) + .unwrap(); + let text = PyStr::from("abc"); + let translated = text.translate(translated, vm).unwrap(); + assert_eq!(translated, Wtf8Buf::from("🎅xda")); + let translated = text.translate(vm.ctx.new_int(3).into(), vm); + assert_eq!("TypeError", &*translated.unwrap_err().class().name(),); + }) + } +} diff --git a/vm/src/builtins/super.rs b/crates/vm/src/builtins/super.rs similarity index 78% rename from vm/src/builtins/super.rs rename to crates/vm/src/builtins/super.rs index a7eefea321d..01bdfa6749e 100644 --- a/vm/src/builtins/super.rs +++ b/crates/vm/src/builtins/super.rs @@ -1,3 +1,4 @@ +// spell-checker:ignore cmeth /*! Python `super` class. See also [CPython source code.](https://github.com/python/cpython/blob/50b48572d9a90c5bb36e2bef6179548ea927a35a/Objects/typeobject.c#L7663) @@ -5,11 +6,11 @@ See also [CPython source code.](https://github.com/python/cpython/blob/50b48572d use super::{PyStr, PyType, PyTypeRef}; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, common::lock::PyRwLock, function::{FuncArgs, IntoFuncArgs, OptionalArg}, types::{Callable, Constructor, GetAttr, GetDescriptor, Initializer, Representable}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; #[pyclass(module = false, name = "super", traverse)] @@ -29,7 +30,7 @@ impl PySuperInner { let obj = if vm.is_none(&obj) { None } else { - let obj_type = supercheck(typ.clone(), obj.clone(), vm)?; + let obj_type = super_check(typ.clone(), obj.clone(), vm)?; Some((obj, obj_type)) }; Ok(Self { typ, obj }) @@ -37,6 +38,7 @@ impl PySuperInner { } impl PyPayload for PySuper { + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.super_type } @@ -45,22 +47,20 @@ impl PyPayload for PySuper { impl Constructor for PySuper { type Args = FuncArgs; - fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - let obj = PySuper { + fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { inner: PyRwLock::new(PySuperInner::new( vm.ctx.types.object_type.to_owned(), // is this correct? vm.ctx.none(), vm, )?), - } - .into_ref_with_type(vm, cls)?; - Ok(obj.into()) + }) } } #[derive(FromArgs)] pub struct InitArgs { - #[pyarg(positional, optional)] + #[pyarg(positional, optional, error_msg = "super() argument 1 must be a type")] py_type: OptionalArg<PyTypeRef>, #[pyarg(positional, optional)] py_obj: OptionalArg<PyObjectRef>, @@ -80,12 +80,13 @@ impl Initializer for PySuper { } else { let frame = vm .current_frame() - .ok_or_else(|| vm.new_runtime_error("super(): no current frame".to_owned()))?; + .ok_or_else(|| vm.new_runtime_error("super(): no current frame"))?; if frame.code.arg_count == 0 { - return Err(vm.new_runtime_error("super(): no arguments".to_owned())); + return Err(vm.new_runtime_error("super(): no arguments")); } - let obj = frame.fastlocals.lock()[0] + // SAFETY: Frame is current and not concurrently mutated. + let obj = unsafe { frame.fastlocals() }[0] .clone() .or_else(|| { if let Some(cell2arg) = frame.code.cell2arg.as_deref() { @@ -93,20 +94,20 @@ impl Initializer for PySuper { .iter() .enumerate() .find(|(_, arg_idx)| **arg_idx == 0) - .and_then(|(cell_idx, _)| frame.cells_frees[cell_idx].get()) + .and_then(|(cell_idx, _)| frame.get_cell_contents(cell_idx)) } else { None } }) - .ok_or_else(|| vm.new_runtime_error("super(): arg[0] deleted".to_owned()))?; + .ok_or_else(|| vm.new_runtime_error("super(): arg[0] deleted"))?; let mut typ = None; for (i, var) in frame.code.freevars.iter().enumerate() { - if var.as_str() == "__class__" { + if var.as_bytes() == b"__class__" { let i = frame.code.cellvars.len() + i; - let class = frame.cells_frees[i].get().ok_or_else(|| { - vm.new_runtime_error("super(): empty __class__ cell".to_owned()) - })?; + let class = frame + .get_cell_contents(i) + .ok_or_else(|| vm.new_runtime_error("super(): empty __class__ cell"))?; typ = Some(class.downcast().map_err(|o| { vm.new_type_error(format!( "super(): __class__ is not a type ({})", @@ -118,15 +119,15 @@ impl Initializer for PySuper { } let typ = typ.ok_or_else(|| { vm.new_type_error( - "super must be called with 1 argument or from inside class method".to_owned(), + "super must be called with 1 argument or from inside class method", ) })?; (typ, obj) }; - let mut inner = PySuperInner::new(typ, obj, vm)?; - std::mem::swap(&mut inner, &mut zelf.inner.write()); + let inner = PySuperInner::new(typ, obj, vm)?; + *zelf.inner.write() = inner; Ok(()) } @@ -134,13 +135,13 @@ impl Initializer for PySuper { #[pyclass(with(GetAttr, GetDescriptor, Constructor, Initializer, Representable))] impl PySuper { - #[pygetset(magic)] - fn thisclass(&self) -> PyTypeRef { + #[pygetset] + fn __thisclass__(&self) -> PyTypeRef { self.inner.read().typ.clone() } - #[pygetset(magic)] - fn self_class(&self) -> Option<PyTypeRef> { + #[pygetset] + fn __self_class__(&self) -> Option<PyTypeRef> { Some(self.inner.read().obj.as_ref()?.1.clone()) } @@ -160,18 +161,19 @@ impl GetAttr for PySuper { // We want __class__ to return the class of the super object // (i.e. super, or a subclass), not the class of su->obj. - if name.as_str() == "__class__" { + if name.as_bytes() == b"__class__" { return skip(zelf, name); } if let Some(name) = vm.ctx.interned_str(name) { // skip the classes in start_type.mro up to and including zelf.typ - let mro: Vec<_> = start_type - .iter_mro() + let mro: Vec<PyRef<PyType>> = start_type.mro_map_collect(|x| x.to_owned()); + let mro: Vec<_> = mro + .iter() .skip_while(|cls| !cls.is(&zelf.inner.read().typ)) .skip(1) // skip su->type (if any) .collect(); - for cls in mro { + for cls in &mro { if let Some(descr) = cls.get_direct_attr(name) { return vm .call_get_descriptor_specific( @@ -202,7 +204,7 @@ impl GetDescriptor for PySuper { let zelf_class = zelf.as_object().class(); if zelf_class.is(vm.ctx.types.super_type) { let typ = zelf.inner.read().typ.clone(); - Ok(PySuper { + Ok(Self { inner: PyRwLock::new(PySuperInner::new(typ, obj, vm)?), } .into_ref(&vm.ctx) @@ -235,26 +237,39 @@ impl Representable for PySuper { } } -fn supercheck(ty: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTypeRef> { - if let Ok(cls) = obj.clone().downcast::<PyType>() { - if cls.fast_issubclass(&ty) { - return Ok(cls); - } - } +fn super_check(ty: PyTypeRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTypeRef> { + let typ = match obj.clone().downcast::<PyType>() { + Ok(cls) if cls.fast_issubclass(&ty) => return Ok(cls), + Ok(cls) => Some(cls), + Err(_) => None, + }; + if obj.fast_isinstance(&ty) { return Ok(obj.class().to_owned()); } + let class_attr = obj.get_attr("__class__", vm)?; - if let Ok(cls) = class_attr.downcast::<PyType>() { - if !cls.is(&ty) && cls.fast_issubclass(&ty) { - return Ok(cls); - } + if let Ok(cls) = class_attr.downcast::<PyType>() + && !cls.is(&ty) + && cls.fast_issubclass(&ty) + { + return Ok(cls); } - Err(vm - .new_type_error("super(type, obj): obj must be an instance or subtype of type".to_owned())) + + let (type_or_instance, obj_str) = match typ { + Some(t) => ("type", t.name().to_owned()), + None => ("instance of", obj.class().name().to_owned()), + }; + + Err(vm.new_type_error(format!( + "super(type, obj): obj ({} {}) is not an instance or subtype of type ({}).", + type_or_instance, + obj_str, + ty.name(), + ))) } -pub fn init(context: &Context) { +pub fn init(context: &'static Context) { let super_type = &context.types.super_type; PySuper::extend_class(context, super_type); diff --git a/crates/vm/src/builtins/template.rs b/crates/vm/src/builtins/template.rs new file mode 100644 index 00000000000..3f03f9a227a --- /dev/null +++ b/crates/vm/src/builtins/template.rs @@ -0,0 +1,335 @@ +use super::{ + PyStr, PyTupleRef, PyType, PyTypeRef, genericalias::PyGenericAlias, + interpolation::PyInterpolation, +}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, + class::PyClassImpl, + common::lock::LazyLock, + function::{FuncArgs, PyComparisonValue}, + protocol::{PyIterReturn, PySequenceMethods}, + types::{ + AsSequence, Comparable, Constructor, IterNext, Iterable, PyComparisonOp, Representable, + SelfIter, + }, +}; +use rustpython_common::wtf8::{Wtf8Buf, wtf8_concat}; + +/// Template object for t-strings (PEP 750). +/// +/// Represents a template string with interpolated expressions. +#[pyclass(module = "string.templatelib", name = "Template")] +#[derive(Debug, Clone)] +pub struct PyTemplate { + pub strings: PyTupleRef, + pub interpolations: PyTupleRef, +} + +impl PyPayload for PyTemplate { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.template_type + } +} + +impl PyTemplate { + pub fn new(strings: PyTupleRef, interpolations: PyTupleRef) -> Self { + Self { + strings, + interpolations, + } + } +} + +impl Constructor for PyTemplate { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("Template.__new__ only accepts *args arguments")); + } + + let mut strings: Vec<PyObjectRef> = Vec::new(); + let mut interpolations: Vec<PyObjectRef> = Vec::new(); + let mut last_was_str = false; + + for item in args.args.iter() { + if let Ok(s) = item.clone().downcast::<PyStr>() { + if last_was_str { + // Concatenate adjacent strings + if let Some(last) = strings.last_mut() { + let last_str = last.downcast_ref::<PyStr>().unwrap(); + let mut buf = last_str.as_wtf8().to_owned(); + buf.push_wtf8(s.as_wtf8()); + *last = vm.ctx.new_str(buf).into(); + } + } else { + strings.push(s.into()); + } + last_was_str = true; + } else if item.class().is(vm.ctx.types.interpolation_type) { + if !last_was_str { + // Add empty string before interpolation + strings.push(vm.ctx.empty_str.to_owned().into()); + } + interpolations.push(item.clone()); + last_was_str = false; + } else { + return Err(vm.new_type_error(format!( + "Template.__new__ *args need to be of type 'str' or 'Interpolation', got {}", + item.class().name() + ))); + } + } + + if !last_was_str { + // Add trailing empty string + strings.push(vm.ctx.empty_str.to_owned().into()); + } + + Ok(PyTemplate { + strings: vm.ctx.new_tuple(strings), + interpolations: vm.ctx.new_tuple(interpolations), + }) + } +} + +#[pyclass(with(Constructor, Comparable, Iterable, Representable, AsSequence))] +impl PyTemplate { + #[pygetset] + fn strings(&self) -> PyTupleRef { + self.strings.clone() + } + + #[pygetset] + fn interpolations(&self) -> PyTupleRef { + self.interpolations.clone() + } + + #[pygetset] + fn values(&self, vm: &VirtualMachine) -> PyTupleRef { + let values: Vec<PyObjectRef> = self + .interpolations + .iter() + .map(|interp| { + interp + .downcast_ref::<PyInterpolation>() + .map(|i| i.value.clone()) + .unwrap_or_else(|| interp.clone()) + }) + .collect(); + vm.ctx.new_tuple(values) + } + + fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let other = other.downcast_ref::<PyTemplate>().ok_or_else(|| { + vm.new_type_error(format!( + "can only concatenate Template (not '{}') to Template", + other.class().name() + )) + })?; + + // Concatenate the two templates + let mut new_strings: Vec<PyObjectRef> = Vec::new(); + let mut new_interps: Vec<PyObjectRef> = Vec::new(); + + // Add all strings from self except the last one + let self_strings_len = self.strings.len(); + for i in 0..self_strings_len.saturating_sub(1) { + new_strings.push(self.strings.get(i).unwrap().clone()); + } + + // Add all interpolations from self + for interp in self.interpolations.iter() { + new_interps.push(interp.clone()); + } + + // Concatenate last string of self with first string of other + let mut buf = Wtf8Buf::new(); + if let Some(s) = self + .strings + .get(self_strings_len.saturating_sub(1)) + .and_then(|s| s.downcast_ref::<PyStr>()) + { + buf.push_wtf8(s.as_wtf8()); + } + if let Some(s) = other + .strings + .first() + .and_then(|s| s.downcast_ref::<PyStr>()) + { + buf.push_wtf8(s.as_wtf8()); + } + new_strings.push(vm.ctx.new_str(buf).into()); + + // Add remaining strings from other (skip first) + for i in 1..other.strings.len() { + new_strings.push(other.strings.get(i).unwrap().clone()); + } + + // Add all interpolations from other + for interp in other.interpolations.iter() { + new_interps.push(interp.clone()); + } + + let template = PyTemplate { + strings: vm.ctx.new_tuple(new_strings), + interpolations: vm.ctx.new_tuple(new_interps), + }; + + Ok(template.into_ref(&vm.ctx)) + } + + fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + self.concat(&other, vm) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + // Import string.templatelib._template_unpickle + // We need to import string first, then get templatelib from it, + // because import("string.templatelib", 0) with empty from_list returns the top-level module + let string_mod = vm.import("string.templatelib", 0)?; + let templatelib = string_mod.get_attr("templatelib", vm)?; + let unpickle_func = templatelib.get_attr("_template_unpickle", vm)?; + + // Return (func, (strings, interpolations)) + let args = vm.ctx.new_tuple(vec![ + self.strings.clone().into(), + self.interpolations.clone().into(), + ]); + Ok(vm.ctx.new_tuple(vec![unpickle_func, args.into()])) + } +} + +impl AsSequence for PyTemplate { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + concat: atomic_func!(|seq, other, vm| { + let zelf = PyTemplate::sequence_downcast(seq); + zelf.concat(other, vm).map(|t| t.into()) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl Comparable for PyTemplate { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + + let eq = vm.bool_eq(zelf.strings.as_object(), other.strings.as_object())? + && vm.bool_eq( + zelf.interpolations.as_object(), + other.interpolations.as_object(), + )?; + + Ok(eq.into()) + }) + } +} + +impl Iterable for PyTemplate { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyTemplateIter::new(zelf).into_pyobject(vm)) + } +} + +impl Representable for PyTemplate { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let strings_repr = zelf.strings.as_object().repr(vm)?; + let interp_repr = zelf.interpolations.as_object().repr(vm)?; + Ok(wtf8_concat!( + "Template(strings=", + strings_repr.as_wtf8(), + ", interpolations=", + interp_repr.as_wtf8(), + ')', + )) + } +} + +/// Iterator for Template objects +#[pyclass(module = "string.templatelib", name = "TemplateIter")] +#[derive(Debug)] +pub struct PyTemplateIter { + template: PyRef<PyTemplate>, + index: core::sync::atomic::AtomicUsize, + from_strings: core::sync::atomic::AtomicBool, +} + +impl PyPayload for PyTemplateIter { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.template_iter_type + } +} + +impl PyTemplateIter { + fn new(template: PyRef<PyTemplate>) -> Self { + Self { + template, + index: core::sync::atomic::AtomicUsize::new(0), + from_strings: core::sync::atomic::AtomicBool::new(true), + } + } +} + +#[pyclass(with(IterNext, Iterable))] +impl PyTemplateIter {} + +impl SelfIter for PyTemplateIter {} + +impl IterNext for PyTemplateIter { + fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { + use core::sync::atomic::Ordering; + + loop { + let from_strings = zelf.from_strings.load(Ordering::SeqCst); + let index = zelf.index.load(Ordering::SeqCst); + + if from_strings { + if index < zelf.template.strings.len() { + let item = zelf.template.strings.get(index).unwrap(); + zelf.from_strings.store(false, Ordering::SeqCst); + + // Skip empty strings + if let Some(s) = item.downcast_ref::<PyStr>() + && s.as_wtf8().is_empty() + { + continue; + } + return Ok(PyIterReturn::Return(item.clone())); + } else { + return Ok(PyIterReturn::StopIteration(None)); + } + } else if index < zelf.template.interpolations.len() { + let item = zelf.template.interpolations.get(index).unwrap(); + zelf.index.fetch_add(1, Ordering::SeqCst); + zelf.from_strings.store(true, Ordering::SeqCst); + return Ok(PyIterReturn::Return(item.clone())); + } else { + return Ok(PyIterReturn::StopIteration(None)); + } + } + } +} + +pub fn init(context: &'static Context) { + PyTemplate::extend_class(context, context.types.template_type); + PyTemplateIter::extend_class(context, context.types.template_iter_type); +} diff --git a/crates/vm/src/builtins/traceback.rs b/crates/vm/src/builtins/traceback.rs new file mode 100644 index 00000000000..c6eac4e87e7 --- /dev/null +++ b/crates/vm/src/builtins/traceback.rs @@ -0,0 +1,137 @@ +use super::{PyList, PyType}; +use crate::{ + AsObject, Context, Py, PyPayload, PyRef, PyResult, VirtualMachine, class::PyClassImpl, + frame::FrameRef, function::PySetterValue, types::Constructor, +}; +use rustpython_common::lock::PyMutex; +use rustpython_compiler_core::OneIndexed; + +#[pyclass(module = false, name = "traceback", traverse)] +#[derive(Debug)] +pub struct PyTraceback { + pub next: PyMutex<Option<PyTracebackRef>>, + pub frame: FrameRef, + #[pytraverse(skip)] + pub lasti: u32, + #[pytraverse(skip)] + pub lineno: OneIndexed, +} + +pub type PyTracebackRef = PyRef<PyTraceback>; + +impl PyPayload for PyTraceback { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.traceback_type + } +} + +#[pyclass(with(Constructor))] +impl PyTraceback { + pub const fn new( + next: Option<PyRef<Self>>, + frame: FrameRef, + lasti: u32, + lineno: OneIndexed, + ) -> Self { + Self { + next: PyMutex::new(next), + frame, + lasti, + lineno, + } + } + + #[pygetset] + fn tb_frame(&self) -> FrameRef { + self.frame.clone() + } + + #[pygetset] + const fn tb_lasti(&self) -> u32 { + self.lasti + } + + #[pygetset] + const fn tb_lineno(&self) -> usize { + self.lineno.get() + } + + #[pygetset] + fn tb_next(&self) -> Option<PyRef<Self>> { + self.next.lock().as_ref().cloned() + } + + #[pymethod] + fn __dir__(&self, vm: &VirtualMachine) -> PyList { + PyList::from( + ["tb_frame", "tb_next", "tb_lasti", "tb_lineno"] + .iter() + .map(|&s| vm.ctx.new_str(s).into()) + .collect::<Vec<_>>(), + ) + } + + #[pygetset(setter)] + fn set_tb_next( + zelf: &Py<Self>, + value: PySetterValue<Option<PyRef<Self>>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let value = match value { + PySetterValue::Assign(v) => v, + PySetterValue::Delete => { + return Err(vm.new_type_error("can't delete tb_next attribute")); + } + }; + if let Some(ref new_next) = value { + let mut cursor = new_next.clone(); + loop { + if cursor.is(zelf) { + return Err(vm.new_value_error("traceback loop detected")); + } + let next = cursor.next.lock().clone(); + match next { + Some(n) => cursor = n, + None => break, + } + } + } + *zelf.next.lock() = value; + Ok(()) + } +} + +impl Constructor for PyTraceback { + type Args = (Option<PyRef<Self>>, FrameRef, u32, usize); + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let (next, frame, lasti, lineno) = args; + let lineno = + OneIndexed::new(lineno).ok_or_else(|| vm.new_value_error("lineno must be positive"))?; + Ok(Self::new(next, frame, lasti, lineno)) + } +} + +impl PyTracebackRef { + pub fn iter(&self) -> impl Iterator<Item = Self> { + core::iter::successors(Some(self.clone()), |tb| tb.next.lock().clone()) + } +} + +pub fn init(context: &'static Context) { + PyTraceback::extend_class(context, context.types.traceback_type); +} + +#[cfg(feature = "serde")] +impl serde::Serialize for PyTraceback { + fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { + use serde::ser::SerializeStruct; + + let mut struc = s.serialize_struct("PyTraceback", 3)?; + struc.serialize_field("name", self.frame.code.obj_name.as_str())?; + struc.serialize_field("lineno", &self.lineno.get())?; + struc.serialize_field("filename", self.frame.code.source_path().as_str())?; + struc.end() + } +} diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs new file mode 100644 index 00000000000..623f7144796 --- /dev/null +++ b/crates/vm/src/builtins/tuple.rs @@ -0,0 +1,738 @@ +use super::{ + PositionIterInternal, PyGenericAlias, PyStrRef, PyType, PyTypeRef, iter::builtins_iter, +}; +use crate::common::lock::LazyLock; +use crate::common::{ + hash::{PyHash, PyUHash}, + lock::PyMutex, + wtf8::wtf8_concat, +}; +use crate::object::{Traverse, TraverseFn}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + atomic_func, + class::PyClassImpl, + convert::{ToPyObject, TransmuteFromObject}, + function::{ArgSize, FuncArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, + iter::PyExactSizeIterator, + protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + recursion::ReprGuard, + sequence::{OptionalRangeArgs, SequenceExt}, + sliceable::{SequenceIndex, SliceableSequenceOp}, + types::{ + AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, + PyComparisonOp, Representable, SelfIter, + }, + utils::collection_repr, + vm::VirtualMachine, +}; +use alloc::fmt; +use core::cell::Cell; +use core::ptr::NonNull; + +#[pyclass(module = false, name = "tuple", traverse = "manual")] +pub struct PyTuple<R = PyObjectRef> { + elements: Box<[R]>, +} + +impl<R> fmt::Debug for PyTuple<R> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // TODO: implement more informational, non-recursive Debug formatter + f.write_str("tuple") + } +} + +// SAFETY: Traverse properly visits all owned PyObjectRefs +// Note: Only impl for PyTuple<PyObjectRef> (the default) +unsafe impl Traverse for PyTuple { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + self.elements.traverse(traverse_fn); + } + + fn clear(&mut self, out: &mut Vec<PyObjectRef>) { + let elements = core::mem::take(&mut self.elements); + out.extend(elements.into_vec()); + } +} + +// spell-checker:ignore MAXSAVESIZE +/// Per-size freelist storage for tuples, matching tuples[PyTuple_MAXSAVESIZE]. +/// Each bucket caches tuples of a specific element count (index = len - 1). +struct TupleFreeList { + buckets: [Vec<NonNull<PyObject>>; Self::MAX_SAVE_SIZE], +} + +impl TupleFreeList { + /// Largest tuple size to cache on the freelist (sizes 1..=20). + const MAX_SAVE_SIZE: usize = 20; + const fn new() -> Self { + Self { + buckets: [const { Vec::new() }; Self::MAX_SAVE_SIZE], + } + } +} + +impl Default for TupleFreeList { + fn default() -> Self { + Self::new() + } +} + +impl Drop for TupleFreeList { + fn drop(&mut self) { + // Same safety pattern as FreeList<T>::drop — free raw allocation + // without running payload destructors to avoid TLS-after-destruction panics. + let layout = crate::object::pyinner_layout::<PyTuple>(); + for bucket in &mut self.buckets { + for ptr in bucket.drain(..) { + unsafe { + alloc::alloc::dealloc(ptr.as_ptr() as *mut u8, layout); + } + } + } + } +} + +thread_local! { + static TUPLE_FREELIST: Cell<TupleFreeList> = const { Cell::new(TupleFreeList::new()) }; +} + +impl PyPayload for PyTuple { + const MAX_FREELIST: usize = 2000; + const HAS_FREELIST: bool = true; + + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.tuple_type + } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + let len = unsafe { &*(obj as *const crate::Py<PyTuple>) } + .elements + .len(); + if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE { + return false; + } + TUPLE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let bucket = &mut list.buckets[len - 1]; + let stored = if bucket.len() < Self::MAX_FREELIST { + bucket.push(unsafe { NonNull::new_unchecked(obj) }); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(payload: &Self) -> Option<NonNull<PyObject>> { + let len = payload.elements.len(); + if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE { + return None; + } + TUPLE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.buckets[len - 1].pop(); + fl.set(list); + result + }) + .ok() + .flatten() + } +} + +pub trait IntoPyTuple { + fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef; +} + +impl IntoPyTuple for () { + fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef { + vm.ctx.empty_tuple.clone() + } +} + +impl IntoPyTuple for Vec<PyObjectRef> { + fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef { + PyTuple::new_ref(self, &vm.ctx) + } +} + +pub trait FromPyTuple<'a>: Sized { + fn from_pytuple(tuple: &'a PyTuple, vm: &VirtualMachine) -> PyResult<Self>; +} + +macro_rules! impl_from_into_pytuple { + ($($T:ident),+) => { + impl<$($T: ToPyObject),*> IntoPyTuple for ($($T,)*) { + fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef { + #[allow(non_snake_case)] + let ($($T,)*) = self; + PyTuple::new_ref(vec![$($T.to_pyobject(vm)),*], &vm.ctx) + } + } + + // TODO: figure out a way to let PyObjectRef implement TryFromBorrowedObject, and + // have this be a TryFromBorrowedObject bound + impl<'a, $($T: TryFromObject),*> FromPyTuple<'a> for ($($T,)*) { + fn from_pytuple(tuple: &'a PyTuple, vm: &VirtualMachine) -> PyResult<Self> { + #[allow(non_snake_case)] + let &[$(ref $T),+] = tuple.as_slice().try_into().map_err(|_| { + vm.new_type_error(format!("expected tuple with {} elements", impl_from_into_pytuple!(@count $($T)+))) + })?; + Ok(($($T::try_from_object(vm, $T.clone())?,)+)) + + } + } + + impl<$($T: ToPyObject),*> ToPyObject for ($($T,)*) { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + self.into_pytuple(vm).into() + } + } + }; + (@count $($T:ident)+) => { + 0 $(+ impl_from_into_pytuple!(@discard $T))+ + }; + (@discard $T:ident) => { + 1 + }; +} + +impl_from_into_pytuple!(A); +impl_from_into_pytuple!(A, B); +impl_from_into_pytuple!(A, B, C); +impl_from_into_pytuple!(A, B, C, D); +impl_from_into_pytuple!(A, B, C, D, E); +impl_from_into_pytuple!(A, B, C, D, E, F); +impl_from_into_pytuple!(A, B, C, D, E, F, G); + +pub type PyTupleRef = PyRef<PyTuple>; + +impl Constructor for PyTuple { + type Args = Vec<PyObjectRef>; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let iterable: OptionalArg<PyObjectRef> = args.bind(vm)?; + + // Optimizations for exact tuple type + if cls.is(vm.ctx.types.tuple_type) { + // Return exact tuple as-is + if let OptionalArg::Present(ref input) = iterable + && let Ok(tuple) = input.clone().downcast_exact::<PyTuple>(vm) + { + return Ok(tuple.into_pyref().into()); + } + + // Return empty tuple singleton + if iterable.is_missing() { + return Ok(vm.ctx.empty_tuple.clone().into()); + } + } + + let elements = if let OptionalArg::Present(iterable) = iterable { + iterable.try_to_value(vm)? + } else { + vec![] + }; + + // Return empty tuple singleton for exact tuple types (when iterable was empty) + if elements.is_empty() && cls.is(vm.ctx.types.tuple_type) { + return Ok(vm.ctx.empty_tuple.clone().into()); + } + + let payload = Self::py_new(&cls, elements, vm)?; + payload.into_ref_with_type(vm, cls).map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, elements: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + elements: elements.into_boxed_slice(), + }) + } +} + +impl<R> AsRef<[R]> for PyTuple<R> { + fn as_ref(&self) -> &[R] { + &self.elements + } +} + +impl<R> core::ops::Deref for PyTuple<R> { + type Target = [R]; + + fn deref(&self) -> &[R] { + &self.elements + } +} + +impl<'a, R> core::iter::IntoIterator for &'a PyTuple<R> { + type Item = &'a R; + type IntoIter = core::slice::Iter<'a, R>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a, R> core::iter::IntoIterator for &'a Py<PyTuple<R>> { + type Item = &'a R; + type IntoIter = core::slice::Iter<'a, R>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<R> PyTuple<R> { + pub const fn as_slice(&self) -> &[R] { + &self.elements + } + + #[inline] + pub fn len(&self) -> usize { + self.elements.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.elements.is_empty() + } + + #[inline] + pub fn iter(&self) -> core::slice::Iter<'_, R> { + self.elements.iter() + } +} + +impl PyTuple<PyObjectRef> { + // Do not deprecate this. empty_tuple must be checked. + pub fn new_ref(elements: Vec<PyObjectRef>, ctx: &Context) -> PyRef<Self> { + if elements.is_empty() { + ctx.empty_tuple.clone() + } else { + let elements = elements.into_boxed_slice(); + PyRef::new_ref(Self { elements }, ctx.types.tuple_type.to_owned(), None) + } + } + + /// Creating a new tuple with given boxed slice. + /// NOTE: for usual case, you probably want to use PyTuple::new_ref. + /// Calling this function implies trying micro optimization for non-zero-sized tuple. + pub const fn new_unchecked(elements: Box<[PyObjectRef]>) -> Self { + Self { elements } + } + + fn repeat(zelf: PyRef<Self>, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + Ok(if zelf.elements.is_empty() || value == 0 { + vm.ctx.empty_tuple.clone() + } else if value == 1 && zelf.class().is(vm.ctx.types.tuple_type) { + // Special case: when some `tuple` is multiplied by `1`, + // nothing really happens, we need to return an object itself + // with the same `id()` to be compatible with CPython. + // This only works for `tuple` itself, not its subclasses. + zelf + } else { + let v = zelf.elements.mul(vm, value)?; + let elements = v.into_boxed_slice(); + Self { elements }.into_ref(&vm.ctx) + }) + } + + pub fn extract_tuple<'a, T: FromPyTuple<'a>>(&'a self, vm: &VirtualMachine) -> PyResult<T> { + T::from_pytuple(self, vm) + } +} + +impl<T> PyTuple<PyRef<T>> { + pub fn new_ref_typed(elements: Vec<PyRef<T>>, ctx: &Context) -> PyRef<Self> { + // SAFETY: PyRef<T> has the same layout as PyObjectRef + unsafe { + let elements: Vec<PyObjectRef> = + core::mem::transmute::<Vec<PyRef<T>>, Vec<PyObjectRef>>(elements); + let tuple = PyTuple::<PyObjectRef>::new_ref(elements, ctx); + core::mem::transmute::<PyRef<PyTuple>, PyRef<Self>>(tuple) + } + } +} + +#[pyclass( + itemsize = core::mem::size_of::<crate::PyObjectRef>(), + flags(BASETYPE, SEQUENCE, _MATCH_SELF), + with(AsMapping, AsNumber, AsSequence, Hashable, Comparable, Iterable, Constructor, Representable) +)] +impl PyTuple { + fn __add__( + zelf: PyRef<Self>, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyArithmeticValue<PyRef<Self>> { + let added = other.downcast::<Self>().map(|other| { + if other.elements.is_empty() && zelf.class().is(vm.ctx.types.tuple_type) { + zelf + } else if zelf.elements.is_empty() && other.class().is(vm.ctx.types.tuple_type) { + other + } else { + let elements = zelf + .iter() + .chain(other.as_slice()) + .cloned() + .collect::<Box<[_]>>(); + Self { elements }.into_ref(&vm.ctx) + } + }); + PyArithmeticValue::from_option(added.ok()) + } + + #[pymethod] + fn count(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + let mut count: usize = 0; + for element in self { + if vm.identical_or_equal(element, &needle)? { + count += 1; + } + } + Ok(count) + } + + #[inline] + pub const fn __len__(&self) -> usize { + self.elements.len() + } + + fn __mul__(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + Self::repeat(zelf, value.into(), vm) + } + + fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + match SequenceIndex::try_from_borrowed_object(vm, needle, "tuple")? { + SequenceIndex::Int(i) => { + let index = self + .elements + .wrap_index(i) + .ok_or_else(|| vm.new_index_error("tuple index out of range"))?; + Ok(self.elements[index].clone()) + } + SequenceIndex::Slice(slice) => self + .elements + .getitem_by_slice(vm, slice) + .map(|x| vm.ctx.new_tuple(x).into()), + } + } + + fn __getitem__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self._getitem(&needle, vm) + } + + #[pymethod] + fn index( + &self, + needle: PyObjectRef, + range: OptionalRangeArgs, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let (start, stop) = range.saturate(self.len(), vm)?; + for (index, element) in self.elements.iter().enumerate().take(stop).skip(start) { + if vm.identical_or_equal(element, &needle)? { + return Ok(index); + } + } + Err(vm.new_value_error("tuple.index(x): x not in tuple")) + } + + fn _contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + for element in &self.elements { + if vm.identical_or_equal(element, needle)? { + return Ok(true); + } + } + Ok(false) + } + + fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + self._contains(&needle, vm) + } + + #[pymethod] + fn __getnewargs__(zelf: PyRef<Self>, vm: &VirtualMachine) -> (PyTupleRef,) { + // the arguments to pass to tuple() is just one tuple - so we'll be doing tuple(tup), which + // should just return tup, or tuple_subclass(tup), which'll copy/validate (e.g. for a + // structseq) + let tup_arg = if zelf.class().is(vm.ctx.types.tuple_type) { + zelf + } else { + Self::new_ref(zelf.elements.clone().into_vec(), &vm.ctx) + }; + (tup_arg,) + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +impl AsMapping for PyTuple { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + length: atomic_func!(|mapping, _vm| Ok(PyTuple::mapping_downcast(mapping).len())), + subscript: atomic_func!( + |mapping, needle, vm| PyTuple::mapping_downcast(mapping)._getitem(needle, vm) + ), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } +} + +impl AsSequence for PyTuple { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyTuple::sequence_downcast(seq).__len__())), + concat: atomic_func!(|seq, other, vm| { + let zelf = PyTuple::sequence_downcast(seq); + match PyTuple::__add__(zelf.to_owned(), other.to_owned(), vm) { + PyArithmeticValue::Implemented(tuple) => Ok(tuple.into()), + PyArithmeticValue::NotImplemented => Err(vm.new_type_error(format!( + "can only concatenate tuple (not '{}') to tuple", + other.class().name() + ))), + } + }), + repeat: atomic_func!(|seq, n, vm| { + let zelf = PyTuple::sequence_downcast(seq); + PyTuple::repeat(zelf.to_owned(), n, vm).map(|x| x.into()) + }), + item: atomic_func!(|seq, i, vm| { + let zelf = PyTuple::sequence_downcast(seq); + zelf.elements.getitem_by_index(vm, i) + }), + contains: atomic_func!(|seq, needle, vm| { + let zelf = PyTuple::sequence_downcast(seq); + zelf._contains(needle, vm) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsNumber for PyTuple { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyTuple>().unwrap(); + Ok(!zelf.elements.is_empty()) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Hashable for PyTuple { + #[inline] + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + tuple_hash(zelf.as_slice(), vm) + } +} + +impl Comparable for PyTuple { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + if let Some(res) = op.identical_optimization(zelf, other) { + return Ok(res.into()); + } + let other = class_or_notimplemented!(Self, other); + zelf.iter() + .richcompare(other.iter(), op, vm) + .map(PyComparisonValue::Implemented) + } +} + +impl Iterable for PyTuple { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyTupleIterator { + internal: PyMutex::new(PositionIterInternal::new(zelf, 0)), + } + .into_pyobject(vm)) + } +} + +impl Representable for PyTuple { + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let s = if zelf.is_empty() { + vm.ctx.intern_str("()").to_owned() + } else if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let s = if zelf.len() == 1 { + wtf8_concat!("(", zelf.elements[0].repr(vm)?.as_wtf8(), ",)") + } else { + collection_repr(None, "(", ")", zelf.elements.iter(), vm)? + }; + vm.ctx.new_str(s) + } else { + vm.ctx.intern_str("(...)").to_owned() + }; + Ok(s) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } +} + +impl PyRef<PyTuple<PyObjectRef>> { + pub fn try_into_typed<T: PyPayload>( + self, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PyTuple<PyRef<T>>>> { + // Check that all elements are of the correct type + for elem in self.as_slice() { + <PyRef<T> as TransmuteFromObject>::check(vm, elem)?; + } + // SAFETY: We just verified all elements are of type T + Ok(unsafe { core::mem::transmute::<Self, PyRef<PyTuple<PyRef<T>>>>(self) }) + } +} + +impl<T: PyPayload> PyRef<PyTuple<PyRef<T>>> { + pub fn into_untyped(self) -> PyRef<PyTuple> { + // SAFETY: PyTuple<PyRef<T>> has the same layout as PyTuple + unsafe { core::mem::transmute::<Self, PyRef<PyTuple>>(self) } + } +} + +impl<T: PyPayload> Py<PyTuple<PyRef<T>>> { + pub fn as_untyped(&self) -> &Py<PyTuple> { + // SAFETY: PyTuple<PyRef<T>> has the same layout as PyTuple + unsafe { core::mem::transmute::<&Self, &Py<PyTuple>>(self) } + } +} + +impl<T: PyPayload> From<PyRef<PyTuple<PyRef<T>>>> for PyTupleRef { + #[inline] + fn from(tup: PyRef<PyTuple<PyRef<T>>>) -> Self { + tup.into_untyped() + } +} + +#[pyclass(module = false, name = "tuple_iterator", traverse)] +#[derive(Debug)] +pub(crate) struct PyTupleIterator { + internal: PyMutex<PositionIterInternal<PyTupleRef>>, +} + +impl PyPayload for PyTupleIterator { + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.tuple_iterator_type + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] +impl PyTupleIterator { + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().length_hint(|obj| obj.len()) + } + + #[pymethod] + fn __setstate__(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.internal + .lock() + .set_state(state, |obj, pos| pos.min(obj.len()), vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + let func = builtins_iter(vm); + self.internal.lock().reduce( + func, + |x| x.clone().into(), + |vm| vm.ctx.empty_tuple.clone().into(), + vm, + ) + } +} + +impl PyTupleIterator { + /// Fast path for FOR_ITER specialization. + pub(crate) fn fast_next(&self) -> Option<PyObjectRef> { + self.internal + .lock() + .next(|tuple, pos| { + Ok(PyIterReturn::from_result( + tuple.get(pos).cloned().ok_or(None), + )) + }) + .ok() + .and_then(|r| match r { + PyIterReturn::Return(v) => Some(v), + PyIterReturn::StopIteration(_) => None, + }) + } +} + +impl SelfIter for PyTupleIterator {} +impl IterNext for PyTupleIterator { + fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.internal.lock().next(|tuple, pos| { + Ok(PyIterReturn::from_result( + tuple.get(pos).cloned().ok_or(None), + )) + }) + } +} + +pub(crate) fn init(context: &'static Context) { + PyTuple::extend_class(context, context.types.tuple_type); + PyTupleIterator::extend_class(context, context.types.tuple_iterator_type); +} + +pub(super) fn tuple_hash(elements: &[PyObjectRef], vm: &VirtualMachine) -> PyResult<PyHash> { + #[cfg(target_pointer_width = "64")] + const PRIME1: PyUHash = 11400714785074694791; + #[cfg(target_pointer_width = "64")] + const PRIME2: PyUHash = 14029467366897019727; + #[cfg(target_pointer_width = "64")] + const PRIME5: PyUHash = 2870177450012600261; + #[cfg(target_pointer_width = "64")] + const ROTATE: u32 = 31; + + #[cfg(target_pointer_width = "32")] + const PRIME1: PyUHash = 2654435761; + #[cfg(target_pointer_width = "32")] + const PRIME2: PyUHash = 2246822519; + #[cfg(target_pointer_width = "32")] + const PRIME5: PyUHash = 374761393; + #[cfg(target_pointer_width = "32")] + const ROTATE: u32 = 13; + + let mut acc = PRIME5; + let len = elements.len() as PyUHash; + + for val in elements { + let lane = val.hash(vm)? as PyUHash; + acc = acc.wrapping_add(lane.wrapping_mul(PRIME2)); + acc = acc.rotate_left(ROTATE); + acc = acc.wrapping_mul(PRIME1); + } + + acc = acc.wrapping_add(len ^ (PRIME5 ^ 3527539)); + + if acc as PyHash == -1 { + return Ok(1546275796); + } + Ok(acc as PyHash) +} diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs new file mode 100644 index 00000000000..5970c30fbd3 --- /dev/null +++ b/crates/vm/src/builtins/type.rs @@ -0,0 +1,2942 @@ +use super::{ + PyClassMethod, PyDictRef, PyList, PyStaticMethod, PyStr, PyStrInterned, PyStrRef, PyTupleRef, + PyUtf8StrRef, PyWeak, mappingproxy::PyMappingProxy, object, union_, +}; +use crate::{ + AsObject, Context, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromObject, VirtualMachine, + builtins::{ + PyBaseExceptionRef, + descriptor::{ + MemberGetter, MemberKind, MemberSetter, PyDescriptorOwned, PyMemberDef, + PyMemberDescriptor, + }, + function::{PyCellRef, PyFunction}, + tuple::{IntoPyTuple, PyTuple}, + }, + class::{PyClassImpl, StaticType}, + common::{ + ascii, + borrow::BorrowedValue, + lock::{PyMutex, PyRwLock, PyRwLockReadGuard}, + }, + function::{FuncArgs, KwArgs, OptionalArg, PyMethodDef, PySetterValue}, + object::{Traverse, TraverseFn}, + protocol::{PyIterReturn, PyNumberMethods}, + types::{ + AsNumber, Callable, Constructor, GetAttr, Initializer, PyTypeFlags, PyTypeSlots, + Representable, SLOT_DEFS, SetAttr, TypeDataRef, TypeDataRefMut, TypeDataSlot, + }, +}; +use core::{ + any::Any, + borrow::Borrow, + ops::Deref, + pin::Pin, + ptr::NonNull, + sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, Ordering}, +}; +use indexmap::{IndexMap, map::Entry}; +use itertools::Itertools; +use num_traits::ToPrimitive; +use rustpython_common::wtf8::Wtf8; +use std::collections::HashSet; + +#[pyclass(module = false, name = "type", traverse = "manual")] +pub struct PyType { + pub base: Option<PyTypeRef>, + pub bases: PyRwLock<Vec<PyTypeRef>>, + pub mro: PyRwLock<Vec<PyTypeRef>>, + pub subclasses: PyRwLock<Vec<PyRef<PyWeak>>>, + pub attributes: PyRwLock<PyAttributes>, + pub slots: PyTypeSlots, + pub heaptype_ext: Option<Pin<Box<HeapTypeExt>>>, + /// Type version tag for inline caching. 0 means unassigned/invalidated. + pub tp_version_tag: AtomicU32, +} + +/// Monotonic counter for type version tags. Once it reaches `u32::MAX`, +/// `assign_version_tag()` returns 0 permanently, disabling new inline-cache +/// entries but not invalidating correctness (cache misses fall back to the +/// generic path). +static NEXT_TYPE_VERSION: AtomicU32 = AtomicU32::new(1); + +// Method cache (type_cache / MCACHE): direct-mapped cache keyed by +// (tp_version_tag, interned_name_ptr). +// +// Uses a lock-free SeqLock pattern for the read/write protocol: +// - Readers validate sequence/version/name before and after the value read. +// - Writers bracket updates with sequence odd/even transitions. +// No mutex needed on the hot path (cache hit). + +const TYPE_CACHE_SIZE_EXP: u32 = 12; +const TYPE_CACHE_SIZE: usize = 1 << TYPE_CACHE_SIZE_EXP; +const TYPE_CACHE_MASK: usize = TYPE_CACHE_SIZE - 1; + +struct TypeCacheEntry { + /// Sequence lock (odd = write in progress, even = quiescent). + sequence: AtomicU32, + /// tp_version_tag at cache time. 0 = empty/invalid. + version: AtomicU32, + /// Interned attribute name pointer (pointer equality check). + name: AtomicPtr<PyStrInterned>, + /// Cached lookup result as raw pointer. null = empty. + /// The cache holds a **borrowed** pointer (no refcount increment). + /// Safety: `type_cache_clear()` nullifies all entries during GC, + /// and `type_cache_clear_version()` nullifies entries when a type + /// is modified — both before the source dict entry is removed. + /// Types are always part of reference cycles (via `mro` self-reference) + /// so they are always collected by the cyclic GC (never refcount-freed). + value: AtomicPtr<PyObject>, +} + +// SAFETY: TypeCacheEntry is thread-safe: +// - All fields use atomic operations +// - Value pointer is valid as long as version matches (SeqLock pattern) +// - PyObjectRef uses atomic reference counting +unsafe impl Send for TypeCacheEntry {} +unsafe impl Sync for TypeCacheEntry {} + +impl TypeCacheEntry { + fn new() -> Self { + Self { + sequence: AtomicU32::new(0), + version: AtomicU32::new(0), + name: AtomicPtr::new(core::ptr::null_mut()), + value: AtomicPtr::new(core::ptr::null_mut()), + } + } + + #[inline] + fn begin_write(&self) { + let mut seq = self.sequence.load(Ordering::Acquire); + loop { + while (seq & 1) != 0 { + core::hint::spin_loop(); + seq = self.sequence.load(Ordering::Acquire); + } + match self.sequence.compare_exchange_weak( + seq, + seq.wrapping_add(1), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + core::sync::atomic::fence(Ordering::Release); + break; + } + Err(observed) => { + core::hint::spin_loop(); + seq = observed; + } + } + } + } + + #[inline] + fn end_write(&self) { + self.sequence.fetch_add(1, Ordering::Release); + } + + #[inline] + fn begin_read(&self) -> u32 { + let mut sequence = self.sequence.load(Ordering::Acquire); + while (sequence & 1) != 0 { + core::hint::spin_loop(); + sequence = self.sequence.load(Ordering::Acquire); + } + sequence + } + + #[inline] + fn end_read(&self, previous: u32) -> bool { + core::sync::atomic::fence(Ordering::Acquire); + self.sequence.load(Ordering::Relaxed) == previous + } + + /// Null out the cached value pointer. + /// Caller must ensure no concurrent reads can observe this entry + /// (version should be set to 0 first). + fn clear_value(&self) { + self.value.store(core::ptr::null_mut(), Ordering::Relaxed); + } +} + +// std::sync::LazyLock is used here (not crate::common::lock::LazyLock) +// because TYPE_CACHE is a global shared across test threads. The common +// LazyLock delegates to LazyCell in non-threading mode, which is !Sync. +static TYPE_CACHE: std::sync::LazyLock<Box<[TypeCacheEntry]>> = std::sync::LazyLock::new(|| { + (0..TYPE_CACHE_SIZE) + .map(|_| TypeCacheEntry::new()) + .collect::<Vec<_>>() + .into_boxed_slice() +}); + +/// When true, find_name_in_mro skips populating the cache. +/// Set during GC's type_cache_clear to prevent re-population from drops. +static TYPE_CACHE_CLEARING: AtomicBool = AtomicBool::new(false); + +/// MCACHE_HASH: XOR of version and name pointer hash, masked to cache size. +#[inline] +fn type_cache_hash(version: u32, name: &'static PyStrInterned) -> usize { + let name_hash = (name as *const PyStrInterned as usize >> 3) as u32; + ((version ^ name_hash) as usize) & TYPE_CACHE_MASK +} + +/// Invalidate cache entries for a specific version tag. +/// Called from modified() when a type is changed. +fn type_cache_clear_version(version: u32) { + for entry in TYPE_CACHE.iter() { + if entry.version.load(Ordering::Relaxed) == version { + entry.begin_write(); + if entry.version.load(Ordering::Relaxed) == version { + entry.version.store(0, Ordering::Release); + entry.clear_value(); + } + entry.end_write(); + } + } +} + +/// Clear all method cache entries (_PyType_ClearCache). +/// Called during GC collection to nullify borrowed pointers before +/// the collector breaks cycles. +/// +/// Sets TYPE_CACHE_CLEARING to suppress cache re-population during the +/// entire operation, preventing concurrent lookups from repopulating +/// entries while we're clearing them. +pub fn type_cache_clear() { + TYPE_CACHE_CLEARING.store(true, Ordering::Release); + for entry in TYPE_CACHE.iter() { + entry.begin_write(); + entry.version.store(0, Ordering::Release); + entry.clear_value(); + entry.end_write(); + } + TYPE_CACHE_CLEARING.store(false, Ordering::Release); +} + +unsafe impl crate::object::Traverse for PyType { + fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn<'_>) { + self.base.traverse(tracer_fn); + self.bases.traverse(tracer_fn); + self.mro.traverse(tracer_fn); + self.subclasses.traverse(tracer_fn); + self.attributes + .read_recursive() + .iter() + .map(|(_, v)| v.traverse(tracer_fn)) + .count(); + if let Some(ext) = self.heaptype_ext.as_ref() { + ext.specialization_cache.traverse(tracer_fn); + } + } + + /// type_clear: break reference cycles in type objects + fn clear(&mut self, out: &mut Vec<crate::PyObjectRef>) { + if let Some(base) = self.base.take() { + out.push(base.into()); + } + if let Some(mut guard) = self.bases.try_write() { + for base in guard.drain(..) { + out.push(base.into()); + } + } + if let Some(mut guard) = self.mro.try_write() { + for typ in guard.drain(..) { + out.push(typ.into()); + } + } + if let Some(mut guard) = self.subclasses.try_write() { + for weak in guard.drain(..) { + out.push(weak.into()); + } + } + if let Some(mut guard) = self.attributes.try_write() { + for (_, val) in guard.drain(..) { + out.push(val); + } + } + if let Some(ext) = self.heaptype_ext.as_ref() { + ext.specialization_cache.clear_into(out); + } + } +} + +// PyHeapTypeObject in CPython +pub struct HeapTypeExt { + pub name: PyRwLock<PyUtf8StrRef>, + pub qualname: PyRwLock<PyStrRef>, + pub slots: Option<PyRef<PyTuple<PyStrRef>>>, + pub type_data: PyRwLock<Option<TypeDataSlot>>, + pub specialization_cache: TypeSpecializationCache, +} + +pub struct TypeSpecializationCache { + pub init: PyAtomicRef<Option<PyFunction>>, + pub getitem: PyAtomicRef<Option<PyFunction>>, + pub getitem_version: AtomicU32, + // Serialize cache writes/invalidation similar to CPython's BEGIN_TYPE_LOCK. + write_lock: PyMutex<()>, + retired: PyRwLock<Vec<PyObjectRef>>, +} + +impl TypeSpecializationCache { + fn new() -> Self { + Self { + init: PyAtomicRef::from(None::<PyRef<PyFunction>>), + getitem: PyAtomicRef::from(None::<PyRef<PyFunction>>), + getitem_version: AtomicU32::new(0), + write_lock: PyMutex::new(()), + retired: PyRwLock::new(Vec::new()), + } + } + + #[inline] + fn retire_old_function(&self, old: Option<PyRef<PyFunction>>) { + if let Some(old) = old { + self.retired.write().push(old.into()); + } + } + + #[inline] + fn swap_init(&self, new_init: Option<PyRef<PyFunction>>, vm: Option<&VirtualMachine>) { + if let Some(vm) = vm { + // Keep replaced refs alive for the currently executing frame, matching + // CPython-style "old pointer remains valid during ongoing execution" + // without accumulating global retired refs. + self.init.swap_to_temporary_refs(new_init, vm); + return; + } + // SAFETY: old value is moved to `retired`, so it stays alive while + // concurrent readers may still hold borrowed references. + let old = unsafe { self.init.swap(new_init) }; + self.retire_old_function(old); + } + + #[inline] + fn swap_getitem(&self, new_getitem: Option<PyRef<PyFunction>>, vm: Option<&VirtualMachine>) { + if let Some(vm) = vm { + self.getitem.swap_to_temporary_refs(new_getitem, vm); + return; + } + // SAFETY: old value is moved to `retired`, so it stays alive while + // concurrent readers may still hold borrowed references. + let old = unsafe { self.getitem.swap(new_getitem) }; + self.retire_old_function(old); + } + + #[inline] + fn invalidate_for_type_modified(&self) { + let _guard = self.write_lock.lock(); + // _spec_cache contract: type modification invalidates all cached + // specialization functions. + self.swap_init(None, None); + self.swap_getitem(None, None); + } + + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + if let Some(init) = self.init.deref() { + tracer_fn(init.as_object()); + } + if let Some(getitem) = self.getitem.deref() { + tracer_fn(getitem.as_object()); + } + self.retired + .read() + .iter() + .map(|obj| obj.traverse(tracer_fn)) + .count(); + } + + fn clear_into(&self, out: &mut Vec<PyObjectRef>) { + let _guard = self.write_lock.lock(); + let old_init = unsafe { self.init.swap(None) }; + if let Some(old_init) = old_init { + out.push(old_init.into()); + } + let old_getitem = unsafe { self.getitem.swap(None) }; + if let Some(old_getitem) = old_getitem { + out.push(old_getitem.into()); + } + self.getitem_version.store(0, Ordering::Release); + out.extend(self.retired.write().drain(..)); + } +} + +pub struct PointerSlot<T>(NonNull<T>); + +unsafe impl<T> Sync for PointerSlot<T> {} +unsafe impl<T> Send for PointerSlot<T> {} + +impl<T> PointerSlot<T> { + pub const unsafe fn borrow_static(&self) -> &'static T { + unsafe { self.0.as_ref() } + } +} + +impl<T> Clone for PointerSlot<T> { + fn clone(&self) -> Self { + *self + } +} + +impl<T> Copy for PointerSlot<T> {} + +impl<T> From<&'static T> for PointerSlot<T> { + fn from(x: &'static T) -> Self { + Self(NonNull::from(x)) + } +} + +impl<T> AsRef<T> for PointerSlot<T> { + fn as_ref(&self) -> &T { + unsafe { self.0.as_ref() } + } +} + +pub type PyTypeRef = PyRef<PyType>; + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for PyType {} + unsafe impl Sync for PyType {} + } +} + +/// For attributes we do not use a dict, but an IndexMap, which is an Hash Table +/// that maintains order and is compatible with the standard HashMap This is probably +/// faster and only supports strings as keys. +pub type PyAttributes = IndexMap<&'static PyStrInterned, PyObjectRef, ahash::RandomState>; + +unsafe impl Traverse for PyAttributes { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.values().for_each(|v| v.traverse(tracer_fn)); + } +} + +impl core::fmt::Display for PyType { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Display::fmt(&self.name(), f) + } +} + +impl core::fmt::Debug for PyType { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "[PyType {}]", &self.name()) + } +} + +impl PyPayload for PyType { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.type_type + } +} + +fn downcast_qualname(value: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + match value.downcast::<PyStr>() { + Ok(value) => Ok(value), + Err(value) => Err(vm.new_type_error(format!( + "can only assign string to __qualname__, not '{}'", + value.class().name() + ))), + } +} + +fn is_subtype_with_mro(a_mro: &[PyTypeRef], a: &Py<PyType>, b: &Py<PyType>) -> bool { + if a.is(b) { + return true; + } + for item in a_mro { + if item.is(b) { + return true; + } + } + false +} + +impl PyType { + /// Assign a fresh version tag. Returns 0 if the version counter has been + /// exhausted, in which case no new cache entries can be created. + pub fn assign_version_tag(&self) -> u32 { + let v = self.tp_version_tag.load(Ordering::Acquire); + if v != 0 { + return v; + } + + // Assign versions to all direct bases first (MRO invariant). + for base in self.bases.read().iter() { + if base.assign_version_tag() == 0 { + return 0; + } + } + + loop { + let current = NEXT_TYPE_VERSION.load(Ordering::Relaxed); + let Some(next) = current.checked_add(1) else { + return 0; // Overflow: version space exhausted + }; + if NEXT_TYPE_VERSION + .compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + self.tp_version_tag.store(current, Ordering::Release); + return current; + } + } + } + + /// Invalidate this type's version tag and cascade to all subclasses. + pub fn modified(&self) { + if let Some(ext) = self.heaptype_ext.as_ref() { + ext.specialization_cache.invalidate_for_type_modified(); + } + // If already invalidated, all subclasses must also be invalidated + // (guaranteed by the MRO invariant in assign_version_tag). + let old_version = self.tp_version_tag.load(Ordering::Acquire); + if old_version == 0 { + return; + } + self.tp_version_tag.store(0, Ordering::SeqCst); + // Nullify borrowed pointers in cache entries for this version + // so they don't dangle after the dict is modified. + type_cache_clear_version(old_version); + let subclasses = self.subclasses.read(); + for weak_ref in subclasses.iter() { + if let Some(sub) = weak_ref.upgrade() { + sub.downcast_ref::<PyType>().unwrap().modified(); + } + } + } + + pub fn new_simple_heap( + name: &str, + base: &Py<PyType>, + ctx: &Context, + ) -> Result<PyRef<Self>, String> { + Self::new_heap( + name, + vec![base.to_owned()], + Default::default(), + Default::default(), + Self::static_type().to_owned(), + ctx, + ) + } + pub fn new_heap( + name: &str, + bases: Vec<PyRef<Self>>, + attrs: PyAttributes, + mut slots: PyTypeSlots, + metaclass: PyRef<Self>, + ctx: &Context, + ) -> Result<PyRef<Self>, String> { + // TODO: ensure clean slot name + // assert_eq!(slots.name.borrow(), ""); + + // Set HEAPTYPE flag for heap-allocated types + slots.flags |= PyTypeFlags::HEAPTYPE; + + let name_utf8 = ctx.new_utf8_str(name); + let name = name_utf8.clone().into_wtf8(); + let heaptype_ext = HeapTypeExt { + name: PyRwLock::new(name_utf8), + qualname: PyRwLock::new(name), + slots: None, + type_data: PyRwLock::new(None), + specialization_cache: TypeSpecializationCache::new(), + }; + let base = bases[0].clone(); + + Self::new_heap_inner(base, bases, attrs, slots, heaptype_ext, metaclass, ctx) + } + + /// Equivalent to CPython's PyType_Check macro + /// Checks if obj is an instance of type (or its subclass) + pub(crate) fn check(obj: &PyObject) -> Option<&Py<Self>> { + obj.downcast_ref::<Self>() + } + + fn resolve_mro(bases: &[PyRef<Self>]) -> Result<Vec<PyTypeRef>, String> { + // Check for duplicates in bases. + let mut unique_bases = HashSet::new(); + for base in bases { + if !unique_bases.insert(base.get_id()) { + return Err(format!("duplicate base class {}", base.name())); + } + } + + let mros = bases + .iter() + .map(|base| base.mro_map_collect(|t| t.to_owned())) + .collect(); + linearise_mro(mros) + } + + /// Inherit SEQUENCE and MAPPING flags from base classes + /// Check all bases in order and inherit the first SEQUENCE or MAPPING flag found + fn inherit_patma_flags(slots: &mut PyTypeSlots, bases: &[PyRef<Self>]) { + const COLLECTION_FLAGS: PyTypeFlags = PyTypeFlags::from_bits_truncate( + PyTypeFlags::SEQUENCE.bits() | PyTypeFlags::MAPPING.bits(), + ); + + // If flags are already set, don't override + if slots.flags.intersects(COLLECTION_FLAGS) { + return; + } + + // Check each base in order and inherit the first collection flag found + for base in bases { + let base_flags = base.slots.flags & COLLECTION_FLAGS; + if !base_flags.is_empty() { + slots.flags |= base_flags; + return; + } + } + } + + /// Check for __abc_tpflags__ and set the appropriate flags + /// This checks in attrs and all base classes for __abc_tpflags__ + fn check_abc_tpflags( + slots: &mut PyTypeSlots, + attrs: &PyAttributes, + bases: &[PyRef<Self>], + ctx: &Context, + ) { + const COLLECTION_FLAGS: PyTypeFlags = PyTypeFlags::from_bits_truncate( + PyTypeFlags::SEQUENCE.bits() | PyTypeFlags::MAPPING.bits(), + ); + + // Don't override if flags are already set + if slots.flags.intersects(COLLECTION_FLAGS) { + return; + } + + // First check in our own attributes + let abc_tpflags_name = ctx.intern_str("__abc_tpflags__"); + if let Some(abc_tpflags_obj) = attrs.get(abc_tpflags_name) + && let Some(int_obj) = abc_tpflags_obj.downcast_ref::<crate::builtins::int::PyInt>() + { + let flags_val = int_obj.as_bigint().to_i64().unwrap_or(0); + let abc_flags = PyTypeFlags::from_bits_truncate(flags_val as u64); + slots.flags |= abc_flags & COLLECTION_FLAGS; + return; + } + + // Then check in base classes + for base in bases { + if let Some(abc_tpflags_obj) = base.find_name_in_mro(abc_tpflags_name) + && let Some(int_obj) = abc_tpflags_obj.downcast_ref::<crate::builtins::int::PyInt>() + { + let flags_val = int_obj.as_bigint().to_i64().unwrap_or(0); + let abc_flags = PyTypeFlags::from_bits_truncate(flags_val as u64); + slots.flags |= abc_flags & COLLECTION_FLAGS; + return; + } + } + } + + #[allow(clippy::too_many_arguments)] + fn new_heap_inner( + base: PyRef<Self>, + bases: Vec<PyRef<Self>>, + attrs: PyAttributes, + mut slots: PyTypeSlots, + heaptype_ext: HeapTypeExt, + metaclass: PyRef<Self>, + ctx: &Context, + ) -> Result<PyRef<Self>, String> { + let mro = Self::resolve_mro(&bases)?; + + // Inherit HAS_DICT from any base in MRO that has it + // (not just the first base, as any base with __dict__ means subclass needs it too) + if mro + .iter() + .any(|b| b.slots.flags.has_feature(PyTypeFlags::HAS_DICT)) + { + slots.flags |= PyTypeFlags::HAS_DICT + } + + // Inherit HAS_WEAKREF/MANAGED_WEAKREF from any base in MRO that has it + if mro + .iter() + .any(|b| b.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF)) + { + slots.flags |= PyTypeFlags::HAS_WEAKREF | PyTypeFlags::MANAGED_WEAKREF + } + + // Inherit SEQUENCE and MAPPING flags from base classes + Self::inherit_patma_flags(&mut slots, &bases); + + // Check for __abc_tpflags__ from ABCMeta (for collections.abc.Sequence, Mapping, etc.) + Self::check_abc_tpflags(&mut slots, &attrs, &bases, ctx); + + if slots.basicsize == 0 { + slots.basicsize = base.slots.basicsize; + } + + Self::inherit_readonly_slots(&mut slots, &base); + + // Normalize: any type with HAS_WEAKREF gets MANAGED_WEAKREF + if slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) { + slots.flags |= PyTypeFlags::MANAGED_WEAKREF; + } + + if let Some(qualname) = attrs.get(identifier!(ctx, __qualname__)) + && !qualname.fast_isinstance(ctx.types.str_type) + { + return Err(format!( + "type __qualname__ must be a str, not {}", + qualname.class().name() + )); + } + + let new_type = PyRef::new_ref( + Self { + base: Some(base), + bases: PyRwLock::new(bases), + mro: PyRwLock::new(mro), + subclasses: PyRwLock::default(), + attributes: PyRwLock::new(attrs), + slots, + heaptype_ext: Some(Pin::new(Box::new(heaptype_ext))), + tp_version_tag: AtomicU32::new(0), + }, + metaclass, + None, + ); + new_type.mro.write().insert(0, new_type.clone()); + + new_type.init_slots(ctx); + + let weakref_type = super::PyWeak::static_type(); + for base in new_type.bases.read().iter() { + base.subclasses.write().push( + new_type + .as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) + .unwrap(), + ); + } + + Ok(new_type) + } + + pub fn new_static( + base: PyRef<Self>, + attrs: PyAttributes, + mut slots: PyTypeSlots, + metaclass: PyRef<Self>, + ) -> Result<PyRef<Self>, String> { + if base.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + slots.flags |= PyTypeFlags::HAS_DICT + } + if base.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) { + slots.flags |= PyTypeFlags::HAS_WEAKREF | PyTypeFlags::MANAGED_WEAKREF + } + + // Inherit SEQUENCE and MAPPING flags from base class + // For static types, we only have a single base + Self::inherit_patma_flags(&mut slots, core::slice::from_ref(&base)); + + if slots.basicsize == 0 { + slots.basicsize = base.slots.basicsize; + } + + Self::inherit_readonly_slots(&mut slots, &base); + + // Normalize: any type with HAS_WEAKREF gets MANAGED_WEAKREF + if slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) { + slots.flags |= PyTypeFlags::MANAGED_WEAKREF; + } + + let bases = PyRwLock::new(vec![base.clone()]); + let mro = base.mro_map_collect(|x| x.to_owned()); + + let new_type = PyRef::new_ref( + Self { + base: Some(base), + bases, + mro: PyRwLock::new(mro), + subclasses: PyRwLock::default(), + attributes: PyRwLock::new(attrs), + slots, + heaptype_ext: None, + tp_version_tag: AtomicU32::new(0), + }, + metaclass, + None, + ); + + // Static types are not tracked by GC. + // They are immortal and never participate in collectable cycles. + unsafe { + crate::gc_state::gc_state() + .untrack_object(core::ptr::NonNull::from(new_type.as_object())); + } + new_type.as_object().clear_gc_tracked(); + + new_type.mro.write().insert(0, new_type.clone()); + + // Note: inherit_slots is called in PyClassImpl::init_class after + // slots are fully initialized by make_slots() + + Self::set_new(&new_type.slots, &new_type.base); + Self::set_alloc(&new_type.slots, &new_type.base); + + let weakref_type = super::PyWeak::static_type(); + for base in new_type.bases.read().iter() { + base.subclasses.write().push( + new_type + .as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) + .unwrap(), + ); + } + + Ok(new_type) + } + + pub(crate) fn init_slots(&self, ctx: &Context) { + // Inherit slots from MRO (mro[0] is self, so skip it) + let mro: Vec<_> = self.mro.read()[1..].to_vec(); + for base in mro.iter() { + self.inherit_slots(base); + } + + // Wire dunder methods to slots + #[allow(clippy::mutable_key_type)] + let mut slot_name_set = std::collections::HashSet::new(); + + // mro[0] is self, so skip it; self.attributes is checked separately below + for cls in self.mro.read()[1..].iter() { + for &name in cls.attributes.read().keys() { + if name.as_bytes().starts_with(b"__") && name.as_bytes().ends_with(b"__") { + slot_name_set.insert(name); + } + } + } + for &name in self.attributes.read().keys() { + if name.as_bytes().starts_with(b"__") && name.as_bytes().ends_with(b"__") { + slot_name_set.insert(name); + } + } + // Sort for deterministic iteration order (important for slot processing) + let mut slot_names: Vec<_> = slot_name_set.into_iter().collect(); + slot_names.sort_by_key(|name| name.as_str()); + for attr_name in slot_names { + self.update_slot::<true>(attr_name, ctx); + } + + Self::set_new(&self.slots, &self.base); + Self::set_alloc(&self.slots, &self.base); + } + + fn set_new(slots: &PyTypeSlots, base: &Option<PyTypeRef>) { + if slots.flags.contains(PyTypeFlags::DISALLOW_INSTANTIATION) { + slots.new.store(None) + } else if slots.new.load().is_none() { + slots.new.store( + base.as_ref() + .map(|base| base.slots.new.load()) + .unwrap_or(None), + ) + } + } + + fn set_alloc(slots: &PyTypeSlots, base: &Option<PyTypeRef>) { + if slots.alloc.load().is_none() { + slots.alloc.store( + base.as_ref() + .map(|base| base.slots.alloc.load()) + .unwrap_or(None), + ); + } + } + + /// Inherit readonly slots from base type at creation time. + /// These slots are not AtomicCell and must be set before the type is used. + fn inherit_readonly_slots(slots: &mut PyTypeSlots, base: &Self) { + if slots.as_buffer.is_none() { + slots.as_buffer = base.slots.as_buffer; + } + } + + /// Inherit slots from base type. inherit_slots + pub(crate) fn inherit_slots(&self, base: &Self) { + // Use SLOT_DEFS to iterate all slots + // Note: as_buffer is handled in inherit_readonly_slots (not AtomicCell) + for def in SLOT_DEFS { + def.accessor.copyslot_if_none(self, base); + } + } + + // This is used for class initialization where the vm is not yet available. + pub fn set_str_attr<V: Into<PyObjectRef>>( + &self, + attr_name: &str, + value: V, + ctx: impl AsRef<Context>, + ) { + let ctx = ctx.as_ref(); + let attr_name = ctx.intern_str(attr_name); + self.set_attr(attr_name, value.into()) + } + + pub fn set_attr(&self, attr_name: &'static PyStrInterned, value: PyObjectRef) { + // Invalidate caches BEFORE modifying attributes so that borrowed + // pointers in cache entries are nullified while the source objects + // are still alive. + self.modified(); + self.attributes.write().insert(attr_name, value); + } + + /// Internal get_attr implementation for fast lookup on a class. + /// Searches the full MRO (including self) with method cache acceleration. + pub fn get_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { + self.find_name_in_mro(attr_name) + } + + /// Cache __init__ for CALL_ALLOC_AND_ENTER_INIT specialization. + /// The cache is valid only when guarded by the type version check. + pub(crate) fn cache_init_for_specialization( + &self, + init: PyRef<PyFunction>, + tp_version: u32, + vm: &VirtualMachine, + ) -> bool { + let Some(ext) = self.heaptype_ext.as_ref() else { + return false; + }; + if tp_version == 0 { + return false; + } + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return false; + } + let _guard = ext.specialization_cache.write_lock.lock(); + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return false; + } + ext.specialization_cache.swap_init(Some(init), Some(vm)); + true + } + + /// Read cached __init__ for CALL_ALLOC_AND_ENTER_INIT specialization. + pub(crate) fn get_cached_init_for_specialization( + &self, + tp_version: u32, + ) -> Option<PyRef<PyFunction>> { + let ext = self.heaptype_ext.as_ref()?; + if tp_version == 0 { + return None; + } + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return None; + } + ext.specialization_cache + .init + .to_owned_ordering(Ordering::Acquire) + } + + /// Cache __getitem__ for BINARY_OP_SUBSCR_GETITEM specialization. + /// The cache is valid only when guarded by the type version check. + pub(crate) fn cache_getitem_for_specialization( + &self, + getitem: PyRef<PyFunction>, + tp_version: u32, + vm: &VirtualMachine, + ) -> bool { + let Some(ext) = self.heaptype_ext.as_ref() else { + return false; + }; + if tp_version == 0 { + return false; + } + let _guard = ext.specialization_cache.write_lock.lock(); + if self.tp_version_tag.load(Ordering::Acquire) != tp_version { + return false; + } + let func_version = getitem.get_version_for_current_state(); + if func_version == 0 { + return false; + } + ext.specialization_cache + .swap_getitem(Some(getitem), Some(vm)); + ext.specialization_cache + .getitem_version + .store(func_version, Ordering::Relaxed); + true + } + + /// Read cached __getitem__ for BINARY_OP_SUBSCR_GETITEM specialization. + pub(crate) fn get_cached_getitem_for_specialization(&self) -> Option<(PyRef<PyFunction>, u32)> { + let ext = self.heaptype_ext.as_ref()?; + // Match CPython check order: pointer (Acquire) then function version. + let getitem = ext + .specialization_cache + .getitem + .to_owned_ordering(Ordering::Acquire)?; + let cached_version = ext + .specialization_cache + .getitem_version + .load(Ordering::Relaxed); + if cached_version == 0 { + return None; + } + Some((getitem, cached_version)) + } + + pub fn get_direct_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { + self.attributes.read().get(attr_name).cloned() + } + + /// find_name_in_mro with method cache (MCACHE). + /// Looks in tp_dict of types in MRO, bypasses descriptors. + /// + /// Uses a lock-free SeqLock-style pattern: + /// Read: load sequence/version/name → load value + try_to_owned → + /// validate value pointer + sequence + /// Write: sequence(begin) → version=0 → swap value/name → version=assigned → sequence(end) + fn find_name_in_mro(&self, name: &'static PyStrInterned) -> Option<PyObjectRef> { + let version = self.tp_version_tag.load(Ordering::Acquire); + if version != 0 { + let idx = type_cache_hash(version, name); + let entry = &TYPE_CACHE[idx]; + let name_ptr = name as *const _ as *mut _; + loop { + let seq1 = entry.begin_read(); + let v1 = entry.version.load(Ordering::Acquire); + let type_version = self.tp_version_tag.load(Ordering::Acquire); + if v1 != type_version + || !core::ptr::eq(entry.name.load(Ordering::Relaxed), name_ptr) + { + break; + } + let ptr = entry.value.load(Ordering::Acquire); + if ptr.is_null() { + if entry.end_read(seq1) { + break; + } + continue; + } + // _Py_TryIncrefCompare-style validation: + // safe_inc via raw pointer, then ensure source is unchanged. + if let Some(cloned) = unsafe { PyObject::try_to_owned_from_ptr(ptr) } { + let same_ptr = core::ptr::eq(entry.value.load(Ordering::Relaxed), ptr); + if same_ptr && entry.end_read(seq1) { + return Some(cloned); + } + drop(cloned); + continue; + } + break; + } + } + + // Assign version BEFORE the MRO walk so that any concurrent + // modified() call during the walk invalidates this version. + let assigned = if version == 0 { + self.assign_version_tag() + } else { + version + }; + + // MRO walk + let result = self.find_name_in_mro_uncached(name); + + // Only cache positive results. Negative results are not cached to + // avoid stale entries from transient MRO walk failures during + // concurrent type modifications. + if let Some(ref found) = result + && assigned != 0 + && !TYPE_CACHE_CLEARING.load(Ordering::Acquire) + && self.tp_version_tag.load(Ordering::Acquire) == assigned + { + let idx = type_cache_hash(assigned, name); + let entry = &TYPE_CACHE[idx]; + let name_ptr = name as *const _ as *mut _; + entry.begin_write(); + // Invalidate first to prevent readers from seeing partial state + entry.version.store(0, Ordering::Release); + // Store borrowed pointer (no refcount increment). + let new_ptr = &**found as *const PyObject as *mut PyObject; + entry.value.store(new_ptr, Ordering::Relaxed); + entry.name.store(name_ptr, Ordering::Relaxed); + // Activate entry — Release ensures value/name writes are visible + entry.version.store(assigned, Ordering::Release); + entry.end_write(); + } + + result + } + + /// Raw MRO walk without cache. + fn find_name_in_mro_uncached(&self, name: &'static PyStrInterned) -> Option<PyObjectRef> { + for cls in self.mro.read().iter() { + if let Some(value) = cls.attributes.read().get(name) { + return Some(value.clone()); + } + } + None + } + + /// _PyType_LookupRef: look up a name through the MRO without setting an exception. + pub fn lookup_ref(&self, name: &Py<PyStr>, vm: &VirtualMachine) -> Option<PyObjectRef> { + let interned_name = vm.ctx.interned_str(name)?; + self.find_name_in_mro(interned_name) + } + + pub fn get_super_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { + self.mro.read()[1..] + .iter() + .find_map(|class| class.attributes.read().get(attr_name).cloned()) + } + + /// Fast lookup for attribute existence on a class. + pub fn has_attr(&self, attr_name: &'static PyStrInterned) -> bool { + self.has_name_in_mro(attr_name) + } + + /// Check if attribute exists in MRO, using method cache for fast check. + /// Unlike find_name_in_mro, avoids cloning the value on cache hit. + fn has_name_in_mro(&self, name: &'static PyStrInterned) -> bool { + let version = self.tp_version_tag.load(Ordering::Acquire); + if version != 0 { + let idx = type_cache_hash(version, name); + let entry = &TYPE_CACHE[idx]; + let name_ptr = name as *const _ as *mut _; + loop { + let seq1 = entry.begin_read(); + let v1 = entry.version.load(Ordering::Acquire); + let type_version = self.tp_version_tag.load(Ordering::Acquire); + if v1 != type_version + || !core::ptr::eq(entry.name.load(Ordering::Relaxed), name_ptr) + { + break; + } + let ptr = entry.value.load(Ordering::Acquire); + if entry.end_read(seq1) { + if !ptr.is_null() { + return true; + } + break; + } + continue; + } + } + + // Cache miss — use find_name_in_mro which populates cache + self.find_name_in_mro(name).is_some() + } + + pub fn get_attributes(&self) -> PyAttributes { + // Gather all members here: + let mut attributes = PyAttributes::default(); + + // mro[0] is self, so we iterate through the entire MRO in reverse + for bc in self.mro.read().iter().map(|cls| -> &Self { cls }).rev() { + for (name, value) in bc.attributes.read().iter() { + attributes.insert(name.to_owned(), value.clone()); + } + } + + attributes + } + + // bound method for every type + pub(crate) fn __new__(zelf: PyRef<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let (subtype, args): (PyRef<Self>, FuncArgs) = args.bind(vm)?; + if !subtype.fast_issubclass(&zelf) { + return Err(vm.new_type_error(format!( + "{zelf}.__new__({subtype}): {subtype} is not a subtype of {zelf}", + zelf = zelf.name(), + subtype = subtype.name(), + ))); + } + call_slot_new(zelf, subtype, args, vm) + } + + fn name_inner<'a, R: 'a>( + &'a self, + static_f: impl FnOnce(&'static str) -> R, + heap_f: impl FnOnce(&'a HeapTypeExt) -> R, + ) -> R { + if let Some(ref ext) = self.heaptype_ext { + heap_f(ext) + } else { + static_f(self.slots.name) + } + } + + pub fn slot_name(&self) -> BorrowedValue<'_, str> { + self.name_inner( + |name| name.into(), + |ext| { + PyRwLockReadGuard::map(ext.name.read(), |name: &PyUtf8StrRef| -> &str { + name.as_str() + }) + .into() + }, + ) + } + + pub fn name(&self) -> BorrowedValue<'_, str> { + self.name_inner( + |name| name.rsplit_once('.').map_or(name, |(_, name)| name).into(), + |ext| { + PyRwLockReadGuard::map(ext.name.read(), |name: &PyUtf8StrRef| -> &str { + name.as_str() + }) + .into() + }, + ) + } + + // Type Data Slot API - CPython's PyObject_GetTypeData equivalent + + /// Initialize type data for this type. Can only be called once. + /// Returns an error if the type is not a heap type or if data is already initialized. + pub fn init_type_data<T: Any + Send + Sync + 'static>(&self, data: T) -> Result<(), String> { + let ext = self + .heaptype_ext + .as_ref() + .ok_or_else(|| "Cannot set type data on non-heap types".to_string())?; + + let mut type_data = ext.type_data.write(); + if type_data.is_some() { + return Err("Type data already initialized".to_string()); + } + *type_data = Some(TypeDataSlot::new(data)); + Ok(()) + } + + /// Get a read guard to the type data. + /// Returns None if the type is not a heap type, has no data, or the data type doesn't match. + pub fn get_type_data<T: Any + 'static>(&self) -> Option<TypeDataRef<'_, T>> { + self.heaptype_ext + .as_ref() + .and_then(|ext| TypeDataRef::try_new(ext.type_data.read())) + } + + /// Get a write guard to the type data. + /// Returns None if the type is not a heap type, has no data, or the data type doesn't match. + pub fn get_type_data_mut<T: Any + 'static>(&self) -> Option<TypeDataRefMut<'_, T>> { + self.heaptype_ext + .as_ref() + .and_then(|ext| TypeDataRefMut::try_new(ext.type_data.write())) + } + + /// Check if this type has type data of the given type. + pub fn has_type_data<T: Any + 'static>(&self) -> bool { + self.heaptype_ext.as_ref().is_some_and(|ext| { + ext.type_data + .read() + .as_ref() + .is_some_and(|slot| slot.get::<T>().is_some()) + }) + } +} + +impl Py<PyType> { + pub(crate) fn is_subtype(&self, other: &Self) -> bool { + is_subtype_with_mro(&self.mro.read(), self, other) + } + + /// Equivalent to CPython's PyType_CheckExact macro + /// Checks if obj is exactly a type (not a subclass) + pub fn check_exact<'a>(obj: &'a PyObject, vm: &VirtualMachine) -> Option<&'a Self> { + obj.downcast_ref_if_exact::<PyType>(vm) + } + + /// Determines if `subclass` is actually a subclass of `cls`, this doesn't call __subclasscheck__, + /// so only use this if `cls` is known to have not overridden the base __subclasscheck__ magic + /// method. + pub fn fast_issubclass(&self, cls: &impl Borrow<PyObject>) -> bool { + self.as_object().is(cls.borrow()) || self.mro.read()[1..].iter().any(|c| c.is(cls.borrow())) + } + + pub fn mro_map_collect<F, R>(&self, f: F) -> Vec<R> + where + F: Fn(&Self) -> R, + { + self.mro.read().iter().map(|x| x.deref()).map(f).collect() + } + + pub fn mro_collect(&self) -> Vec<PyRef<PyType>> { + self.mro + .read() + .iter() + .map(|x| x.deref()) + .map(|x| x.to_owned()) + .collect() + } + + pub fn iter_base_chain(&self) -> impl Iterator<Item = &Self> { + core::iter::successors(Some(self), |cls| cls.base.as_deref()) + } + + pub fn extend_methods(&'static self, method_defs: &'static [PyMethodDef], ctx: &Context) { + for method_def in method_defs { + let method = method_def.to_proper_method(self, ctx); + self.set_attr(ctx.intern_str(method_def.name), method); + } + } +} + +#[pyclass( + with( + Py, + Constructor, + Initializer, + GetAttr, + SetAttr, + Callable, + AsNumber, + Representable + ), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) +)] +impl PyType { + #[pygetset] + fn __bases__(&self, vm: &VirtualMachine) -> PyTupleRef { + vm.ctx.new_tuple( + self.bases + .read() + .iter() + .map(|x| x.as_object().to_owned()) + .collect(), + ) + } + #[pygetset(setter, name = "__bases__")] + fn set_bases(zelf: &Py<Self>, bases: Vec<PyTypeRef>, vm: &VirtualMachine) -> PyResult<()> { + // TODO: Assigning to __bases__ is only used in typing.NamedTupleMeta.__new__ + // Rather than correctly re-initializing the class, we are skipping a few steps for now + if zelf.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot set '__bases__' attribute of immutable type '{}'", + zelf.name() + ))); + } + if bases.is_empty() { + return Err(vm.new_type_error(format!( + "can only assign non-empty tuple to %s.__bases__, not {}", + zelf.name() + ))); + } + + // TODO: check for mro cycles + + // TODO: Remove this class from all subclass lists + // for base in self.bases.read().iter() { + // let subclasses = base.subclasses.write(); + // // TODO: how to uniquely identify the subclasses to remove? + // } + + *zelf.bases.write() = bases; + // Recursively update the mros of this class and all subclasses + fn update_mro_recursively(cls: &PyType, vm: &VirtualMachine) -> PyResult<()> { + let mut mro = + PyType::resolve_mro(&cls.bases.read()).map_err(|msg| vm.new_type_error(msg))?; + // Preserve self (mro[0]) when updating MRO + mro.insert(0, cls.mro.read()[0].to_owned()); + *cls.mro.write() = mro; + for subclass in cls.subclasses.write().iter() { + let subclass = subclass.upgrade().unwrap(); + let subclass: &Py<PyType> = subclass.downcast_ref().unwrap(); + update_mro_recursively(subclass, vm)?; + } + Ok(()) + } + update_mro_recursively(zelf, vm)?; + + // Invalidate inline caches + zelf.modified(); + + // TODO: do any old slots need to be cleaned up first? + zelf.init_slots(&vm.ctx); + + // Register this type as a subclass of its new bases + let weakref_type = super::PyWeak::static_type(); + for base in zelf.bases.read().iter() { + base.subclasses.write().push( + zelf.as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) + .unwrap(), + ); + } + + Ok(()) + } + + #[pygetset] + fn __base__(&self) -> Option<PyTypeRef> { + self.base.clone() + } + + #[pygetset] + const fn __flags__(&self) -> u64 { + self.slots.flags.bits() + } + + #[pygetset] + fn __basicsize__(&self) -> usize { + crate::object::SIZEOF_PYOBJECT_HEAD + self.slots.basicsize + } + + #[pygetset] + fn __itemsize__(&self) -> usize { + self.slots.itemsize + } + + #[pygetset] + pub fn __name__(&self, vm: &VirtualMachine) -> PyStrRef { + self.name_inner( + |name| { + vm.ctx + .interned_str(name.rsplit_once('.').map_or(name, |(_, name)| name)) + .unwrap_or_else(|| { + panic!( + "static type name must be already interned but {} is not", + self.slot_name() + ) + }) + .to_owned() + }, + |ext| ext.name.read().clone().into_wtf8(), + ) + } + + #[pygetset] + pub fn __qualname__(&self, vm: &VirtualMachine) -> PyObjectRef { + if let Some(ref heap_type) = self.heaptype_ext { + heap_type.qualname.read().clone().into() + } else { + // For static types, return the name + vm.ctx.new_str(self.name().deref()).into() + } + } + + #[pygetset(setter)] + fn set___qualname__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + self.check_set_special_type_attr(identifier!(vm, __qualname__), vm)?; + let value = value.ok_or_else(|| { + vm.new_type_error(format!( + "cannot delete '__qualname__' attribute of immutable type '{}'", + self.name() + )) + })?; + + let str_value = downcast_qualname(value, vm)?; + + let heap_type = self.heaptype_ext.as_ref().ok_or_else(|| { + vm.new_type_error(format!( + "cannot set '__qualname__' attribute of immutable type '{}'", + self.name() + )) + })?; + + // Use std::mem::replace to swap the new value in and get the old value out, + // then drop the old value after releasing the lock + let _old_qualname = { + let mut qualname_guard = heap_type.qualname.write(); + core::mem::replace(&mut *qualname_guard, str_value) + }; + // old_qualname is dropped here, outside the lock scope + + Ok(()) + } + + #[pygetset] + fn __annotate__(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '__annotate__'", + self.name() + ))); + } + + let mut attrs = self.attributes.write(); + // First try __annotate__, in case that's been set explicitly + if let Some(annotate) = attrs.get(identifier!(vm, __annotate__)).cloned() { + return Ok(annotate); + } + // Then try __annotate_func__ + if let Some(annotate) = attrs.get(identifier!(vm, __annotate_func__)).cloned() { + // TODO: Apply descriptor tp_descr_get if needed + return Ok(annotate); + } + // Set __annotate_func__ = None and return None + let none = vm.ctx.none(); + attrs.insert(identifier!(vm, __annotate_func__), none.clone()); + Ok(none) + } + + #[pygetset(setter)] + fn set___annotate__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let value = match value { + PySetterValue::Delete => { + return Err(vm.new_type_error("cannot delete __annotate__ attribute")); + } + PySetterValue::Assign(v) => v, + }; + + if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot set '__annotate__' attribute of immutable type '{}'", + self.name() + ))); + } + + if !vm.is_none(&value) && !value.is_callable() { + return Err(vm.new_type_error("__annotate__ must be callable or None")); + } + + let mut attrs = self.attributes.write(); + // Clear cached annotations only when setting to a new callable + if !vm.is_none(&value) { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } + attrs.insert(identifier!(vm, __annotate_func__), value.clone()); + + Ok(()) + } + + #[pygetset] + fn __annotations__(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let attrs = self.attributes.read(); + if let Some(annotations) = attrs.get(identifier!(vm, __annotations__)).cloned() { + // Ignore the __annotations__ descriptor stored on type itself. + if !annotations.class().is(vm.ctx.types.getset_type) { + if vm.is_none(&annotations) + || annotations.class().is(vm.ctx.types.dict_type) + || self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + { + return Ok(annotations); + } + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '__annotations__'", + self.name() + ))); + } + } + // Then try __annotations_cache__ + if let Some(annotations) = attrs.get(identifier!(vm, __annotations_cache__)).cloned() { + if vm.is_none(&annotations) + || annotations.class().is(vm.ctx.types.dict_type) + || self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + { + return Ok(annotations); + } + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '__annotations__'", + self.name() + ))); + } + drop(attrs); + + if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '__annotations__'", + self.name() + ))); + } + + // Get __annotate__ and call it if callable + let annotate = self.__annotate__(vm)?; + let annotations = if annotate.is_callable() { + // Call __annotate__(1) where 1 is FORMAT_VALUE + let result = annotate.call((1i32,), vm)?; + if !result.class().is(vm.ctx.types.dict_type) { + return Err(vm.new_type_error(format!( + "__annotate__ returned non-dict of type '{}'", + result.class().name() + ))); + } + result + } else { + vm.ctx.new_dict().into() + }; + + // Cache the result in __annotations_cache__ + self.attributes + .write() + .insert(identifier!(vm, __annotations_cache__), annotations.clone()); + Ok(annotations) + } + + #[pygetset(setter)] + fn set___annotations__( + &self, + value: crate::function::PySetterValue<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot set '__annotations__' attribute of immutable type '{}'", + self.name() + ))); + } + + let mut attrs = self.attributes.write(); + let has_annotations = attrs.contains_key(identifier!(vm, __annotations__)); + + match value { + crate::function::PySetterValue::Assign(value) => { + // SET path: store the value (including None) + let key = if has_annotations { + identifier!(vm, __annotations__) + } else { + identifier!(vm, __annotations_cache__) + }; + attrs.insert(key, value); + if has_annotations { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } + } + crate::function::PySetterValue::Delete => { + // DELETE path: remove the key + let removed = if has_annotations { + attrs + .swap_remove(identifier!(vm, __annotations__)) + .is_some() + } else { + attrs + .swap_remove(identifier!(vm, __annotations_cache__)) + .is_some() + }; + if !removed { + return Err(vm.new_attribute_error("__annotations__")); + } + if has_annotations { + attrs.swap_remove(identifier!(vm, __annotations_cache__)); + } + } + } + attrs.swap_remove(identifier!(vm, __annotate_func__)); + attrs.swap_remove(identifier!(vm, __annotate__)); + + Ok(()) + } + + #[pygetset] + pub fn __module__(&self, vm: &VirtualMachine) -> PyObjectRef { + self.attributes + .read() + .get(identifier!(vm, __module__)) + .cloned() + // We need to exclude this method from going into recursion: + .and_then(|found| { + if found.fast_isinstance(vm.ctx.types.getset_type) { + None + } else { + Some(found) + } + }) + .unwrap_or_else(|| { + // For non-heap types, extract module from tp_name (e.g. "typing.TypeAliasType" -> "typing") + let slot_name = self.slot_name(); + if let Some((module, _)) = slot_name.rsplit_once('.') { + vm.ctx.intern_str(module).to_object() + } else { + vm.ctx.new_str(ascii!("builtins")).into() + } + }) + } + + #[pygetset(setter)] + fn set___module__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.check_set_special_type_attr(identifier!(vm, __module__), vm)?; + let mut attributes = self.attributes.write(); + attributes.swap_remove(identifier!(vm, __firstlineno__)); + attributes.insert(identifier!(vm, __module__), value); + Ok(()) + } + + #[pyclassmethod] + fn __prepare__( + _cls: PyTypeRef, + _name: OptionalArg<PyObjectRef>, + _bases: OptionalArg<PyObjectRef>, + _kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyDictRef { + vm.ctx.new_dict() + } + + #[pymethod] + fn __subclasses__(&self) -> PyList { + let mut subclasses = self.subclasses.write(); + subclasses.retain(|x| x.upgrade().is_some()); + PyList::from( + subclasses + .iter() + .map(|x| x.upgrade().unwrap()) + .collect::<Vec<_>>(), + ) + } + + pub fn __ror__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + or_(other, zelf, vm) + } + + pub fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + or_(zelf, other, vm) + } + + #[pygetset] + fn __dict__(zelf: PyRef<Self>) -> PyMappingProxy { + PyMappingProxy::from(zelf) + } + + #[pygetset(setter)] + fn set___dict__(&self, _value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_not_implemented_error( + "Setting __dict__ attribute on a type isn't yet implemented", + )) + } + + fn check_set_special_type_attr( + &self, + name: &PyStrInterned, + vm: &VirtualMachine, + ) -> PyResult<()> { + if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot set '{}' attribute of immutable type '{}'", + name, + self.slot_name() + ))); + } + Ok(()) + } + + #[pygetset(setter)] + fn set___name__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.check_set_special_type_attr(identifier!(vm, __name__), vm)?; + let name = value.downcast::<PyStr>().map_err(|value| { + vm.new_type_error(format!( + "can only assign string to {}.__name__, not '{}'", + self.slot_name(), + value.class().slot_name(), + )) + })?; + if name.as_bytes().contains(&0) { + return Err(vm.new_value_error("type name must not contain null characters")); + } + let name = name.try_into_utf8(vm)?; + + let heap_type = self.heaptype_ext.as_ref().ok_or_else(|| { + vm.new_type_error(format!( + "cannot set '__name__' attribute of immutable type '{}'", + self.slot_name() + )) + })?; + + // Use std::mem::replace to swap the new value in and get the old value out, + // then drop the old value after releasing the lock + let _old_name = { + let mut name_guard = heap_type.name.write(); + core::mem::replace(&mut *name_guard, name) + }; + // old_name is dropped here, outside the lock scope + + Ok(()) + } + + #[pygetset] + fn __text_signature__(&self) -> Option<String> { + self.slots + .doc + .and_then(|doc| get_text_signature_from_internal_doc(&self.name(), doc)) + .map(|signature| signature.to_string()) + } + + #[pygetset] + fn __type_params__(&self, vm: &VirtualMachine) -> PyTupleRef { + let attrs = self.attributes.read(); + let key = identifier!(vm, __type_params__); + if let Some(params) = attrs.get(&key) + && let Ok(tuple) = params.clone().downcast::<PyTuple>() + { + return tuple; + } + // Return empty tuple if not found or not a tuple + vm.ctx.empty_tuple.clone() + } + + #[pygetset(setter)] + fn set___type_params__( + &self, + value: PySetterValue<PyTupleRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + PySetterValue::Assign(ref val) => { + let key = identifier!(vm, __type_params__); + self.check_set_special_type_attr(key, vm)?; + self.modified(); + self.attributes.write().insert(key, val.clone().into()); + } + PySetterValue::Delete => { + // For delete, we still need to check if the type is immutable + if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot delete '__type_params__' attribute of immutable type '{}'", + self.slot_name() + ))); + } + let key = identifier!(vm, __type_params__); + self.modified(); + self.attributes.write().shift_remove(&key); + } + } + Ok(()) + } +} + +impl Constructor for PyType { + type Args = FuncArgs; + + fn slot_new(metatype: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + vm_trace!("type.__new__ {:?}", args); + + let is_type_type = metatype.is(vm.ctx.types.type_type); + if is_type_type && args.args.len() == 1 && args.kwargs.is_empty() { + return Ok(args.args[0].class().to_owned().into()); + } + + if args.args.len() != 3 { + return Err(vm.new_type_error(if is_type_type { + "type() takes 1 or 3 arguments".to_owned() + } else { + format!( + "type.__new__() takes exactly 3 arguments ({} given)", + args.args.len() + ) + })); + } + + let (name, bases, dict, kwargs): (PyStrRef, PyTupleRef, PyDictRef, KwArgs) = + args.clone().bind(vm)?; + + if name.as_bytes().contains(&0) { + return Err(vm.new_value_error("type name must not contain null characters")); + } + let name = name.try_into_utf8(vm)?; + + let (metatype, base, bases, base_is_type) = if bases.is_empty() { + let base = vm.ctx.types.object_type.to_owned(); + (metatype, base.clone(), vec![base], false) + } else { + let bases = bases + .iter() + .map(|obj| { + obj.clone().downcast::<Self>().or_else(|obj| { + if vm + .get_attribute_opt(obj, identifier!(vm, __mro_entries__))? + .is_some() + { + Err(vm.new_type_error( + "type() doesn't support MRO entry resolution; \ + use types.new_class()", + )) + } else { + Err(vm.new_type_error("bases must be types")) + } + }) + }) + .collect::<PyResult<Vec<_>>>()?; + + // Search the bases for the proper metatype to deal with this: + let winner = calculate_meta_class(metatype.clone(), &bases, vm)?; + let metatype = if !winner.is(&metatype) { + if let Some(ref slot_new) = winner.slots.new.load() { + // Pass it to the winner + return slot_new(winner, args, vm); + } + winner + } else { + metatype + }; + + let base = best_base(&bases, vm)?; + let base_is_type = base.is(vm.ctx.types.type_type); + + (metatype, base.to_owned(), bases, base_is_type) + }; + + let qualname = dict + .pop_item(identifier!(vm, __qualname__).as_object(), vm)? + .map(|obj| downcast_qualname(obj, vm)) + .transpose()? + .unwrap_or_else(|| { + // If __qualname__ is not provided, we can use the name as default + name.clone().into_wtf8() + }); + let mut attributes = dict.to_attributes(vm); + + // Check __doc__ for surrogates - raises UnicodeEncodeError during type creation + if let Some(doc) = attributes.get(identifier!(vm, __doc__)) + && let Some(doc_str) = doc.downcast_ref::<PyStr>() + { + doc_str.ensure_valid_utf8(vm)?; + } + + if let Some(f) = attributes.get_mut(identifier!(vm, __init_subclass__)) + && f.class().is(vm.ctx.types.function_type) + { + *f = PyClassMethod::from(f.clone()).into_pyobject(vm); + } + + if let Some(f) = attributes.get_mut(identifier!(vm, __class_getitem__)) + && f.class().is(vm.ctx.types.function_type) + { + *f = PyClassMethod::from(f.clone()).into_pyobject(vm); + } + + if let Some(f) = attributes.get_mut(identifier!(vm, __new__)) + && f.class().is(vm.ctx.types.function_type) + { + *f = PyStaticMethod::from(f.clone()).into_pyobject(vm); + } + + if let Some(current_frame) = vm.current_frame() { + let entry = attributes.entry(identifier!(vm, __module__)); + if matches!(entry, Entry::Vacant(_)) { + let module_name = vm.unwrap_or_none( + current_frame + .globals + .get_item_opt(identifier!(vm, __name__), vm)?, + ); + entry.or_insert(module_name); + } + } + + if attributes.get(identifier!(vm, __eq__)).is_some() + && attributes.get(identifier!(vm, __hash__)).is_none() + { + // if __eq__ exists but __hash__ doesn't, overwrite it with None so it doesn't inherit the default hash + // https://docs.python.org/3/reference/datamodel.html#object.__hash__ + attributes.insert(identifier!(vm, __hash__), vm.ctx.none.clone().into()); + } + + let (heaptype_slots, add_dict, add_weakref): ( + Option<PyRef<PyTuple<PyStrRef>>>, + bool, + bool, + ) = if let Some(x) = attributes.get(identifier!(vm, __slots__)) { + // Check if __slots__ is bytes - not allowed + if x.class().is(vm.ctx.types.bytes_type) { + return Err(vm.new_type_error("__slots__ items must be strings, not 'bytes'")); + } + + let slots = if x.class().is(vm.ctx.types.str_type) { + let x = unsafe { x.downcast_unchecked_ref::<PyStr>() }; + PyTuple::new_ref_typed(vec![x.to_owned()], &vm.ctx) + } else { + let iter = x.get_iter(vm)?; + let elements = { + let mut elements = Vec::new(); + while let PyIterReturn::Return(element) = iter.next(vm)? { + // Check if any slot item is bytes + if element.class().is(vm.ctx.types.bytes_type) { + return Err( + vm.new_type_error("__slots__ items must be strings, not 'bytes'") + ); + } + elements.push(element); + } + elements + }; + let tuple = elements.into_pytuple(vm); + tuple.try_into_typed(vm)? + }; + + // Check if base has itemsize > 0 - can't add arbitrary slots to variable-size types + // Types like int, bytes, tuple have itemsize > 0 and don't allow custom slots + // But types like weakref.ref have itemsize = 0 and DO allow slots + let has_custom_slots = slots + .iter() + .any(|s| !matches!(s.as_bytes(), b"__dict__" | b"__weakref__")); + if has_custom_slots && base.slots.itemsize > 0 { + return Err(vm.new_type_error(format!( + "nonempty __slots__ not supported for subtype of '{}'", + base.name() + ))); + } + + // Validate slot names and track duplicates + let mut seen_dict = false; + let mut seen_weakref = false; + for slot in slots.iter() { + // Use isidentifier for validation (handles Unicode properly) + if !slot.isidentifier() { + return Err(vm.new_type_error("__slots__ must be identifiers")); + } + + let slot_name = slot.as_bytes(); + + // Check for duplicate __dict__ + if slot_name == b"__dict__" { + if seen_dict { + return Err( + vm.new_type_error("__dict__ slot disallowed: we already got one") + ); + } + seen_dict = true; + } + + // Check for duplicate __weakref__ + if slot_name == b"__weakref__" { + if seen_weakref { + return Err( + vm.new_type_error("__weakref__ slot disallowed: we already got one") + ); + } + seen_weakref = true; + } + + // Check if slot name conflicts with class attributes + if attributes.contains_key(vm.ctx.intern_str(slot.as_wtf8())) { + return Err(vm.new_value_error(format!( + "'{}' in __slots__ conflicts with a class variable", + slot.as_wtf8() + ))); + } + } + + // Check if base class already has __dict__ - can't redefine it + if seen_dict && base.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + return Err(vm.new_type_error("__dict__ slot disallowed: we already got one")); + } + + // Check if base class already has __weakref__ - can't redefine it + if seen_weakref && base.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) { + return Err(vm.new_type_error("__weakref__ slot disallowed: we already got one")); + } + + // Check if __dict__ or __weakref__ is in slots + let dict_name = "__dict__"; + let weakref_name = "__weakref__"; + let has_dict = slots.iter().any(|s| s.as_wtf8() == dict_name); + let add_weakref = seen_weakref; + + // Filter out __dict__ and __weakref__ from slots + // (they become descriptors, not member slots) + let filtered_slots = if has_dict || add_weakref { + let filtered: Vec<PyStrRef> = slots + .iter() + .filter(|s| s.as_wtf8() != dict_name && s.as_wtf8() != weakref_name) + .cloned() + .collect(); + PyTuple::new_ref_typed(filtered, &vm.ctx) + } else { + slots + }; + + (Some(filtered_slots), has_dict, add_weakref) + } else { + (None, false, false) + }; + + // FIXME: this is a temporary fix. multi bases with multiple slots will break object + let base_member_count = bases + .iter() + .map(|base| base.slots.member_count) + .max() + .unwrap(); + let heaptype_member_count = heaptype_slots.as_ref().map(|x| x.len()).unwrap_or(0); + let member_count: usize = base_member_count + heaptype_member_count; + + let mut flags = PyTypeFlags::heap_type_flags(); + + // Check if we may add dict + // We can only add a dict if the primary base class doesn't already have one + // In CPython, this checks tp_dictoffset == 0 + let may_add_dict = !base.slots.flags.has_feature(PyTypeFlags::HAS_DICT); + + // Add HAS_DICT and MANAGED_DICT if: + // 1. __slots__ is not defined AND base doesn't have dict, OR + // 2. __dict__ is in __slots__ + if (heaptype_slots.is_none() && may_add_dict) || add_dict { + flags |= PyTypeFlags::HAS_DICT | PyTypeFlags::MANAGED_DICT; + } + + // Add HAS_WEAKREF if: + // 1. __slots__ is not defined (automatic weakref support), OR + // 2. __weakref__ is in __slots__ + let may_add_weakref = !base.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF); + if (heaptype_slots.is_none() && may_add_weakref) || add_weakref { + flags |= PyTypeFlags::HAS_WEAKREF | PyTypeFlags::MANAGED_WEAKREF; + } + + let (slots, heaptype_ext) = { + let slots = PyTypeSlots { + flags, + member_count, + itemsize: base.slots.itemsize, + ..PyTypeSlots::heap_default() + }; + let heaptype_ext = HeapTypeExt { + name: PyRwLock::new(name), + qualname: PyRwLock::new(qualname), + slots: heaptype_slots.clone(), + type_data: PyRwLock::new(None), + specialization_cache: TypeSpecializationCache::new(), + }; + (slots, heaptype_ext) + }; + + let typ = Self::new_heap_inner( + base, + bases, + attributes, + slots, + heaptype_ext, + metatype, + &vm.ctx, + ) + .map_err(|e| vm.new_type_error(e))?; + + if let Some(ref slots) = heaptype_slots { + let mut offset = base_member_count; + let class_name = typ.name().to_string(); + for member in slots.as_slice() { + // Apply name mangling for private attributes (__x -> _ClassName__x) + let member_str = member + .to_str() + .ok_or_else(|| vm.new_type_error("__slots__ must be valid UTF-8 strings"))?; + let mangled_name = mangle_name(&class_name, member_str); + let member_def = PyMemberDef { + name: mangled_name.clone(), + kind: MemberKind::ObjectEx, + getter: MemberGetter::Offset(offset), + setter: MemberSetter::Offset(offset), + doc: None, + }; + let attr_name = vm.ctx.intern_str(mangled_name.as_str()); + let member_descriptor: PyRef<PyMemberDescriptor> = + vm.ctx.new_pyref(PyMemberDescriptor { + common: PyDescriptorOwned { + typ: typ.clone(), + name: attr_name, + qualname: PyRwLock::new(None), + }, + member: member_def, + }); + // __slots__ attributes always get a member descriptor + // (this overrides any inherited attribute from MRO) + typ.set_attr(attr_name, member_descriptor.into()); + offset += 1; + } + } + + if let Some(cell) = typ.attributes.write().get(identifier!(vm, __classcell__)) { + let cell = PyCellRef::try_from_object(vm, cell.clone()).map_err(|_| { + vm.new_type_error(format!( + "__classcell__ must be a nonlocal cell, not {}", + cell.class().name() + )) + })?; + cell.set(Some(typ.clone().into())); + }; + + // All *classes* should have a dict. Exceptions are *instances* of + // classes that define __slots__ and instances of built-in classes + // (with exceptions, e.g function) + // Also, type subclasses don't need their own __dict__ descriptor + // since they inherit it from type + + // Add __dict__ descriptor after type creation to ensure correct __objclass__ + // Only add if: + // 1. base is not type (type subclasses inherit __dict__ from type) + // 2. the class has HAS_DICT flag (i.e., __slots__ was not defined or __dict__ is in __slots__) + // 3. no base class in MRO already provides __dict__ descriptor + if !base_is_type && typ.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + let __dict__ = identifier!(vm, __dict__); + let has_inherited_dict = typ + .mro + .read() + .iter() + .any(|base| base.attributes.read().contains_key(&__dict__)); + if !typ.attributes.read().contains_key(&__dict__) && !has_inherited_dict { + unsafe { + let descriptor = + vm.ctx + .new_getset("__dict__", &typ, subtype_get_dict, subtype_set_dict); + typ.attributes.write().insert(__dict__, descriptor.into()); + } + } + } + + // Add __weakref__ descriptor for types with HAS_WEAKREF + if typ.slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) { + let __weakref__ = vm.ctx.intern_str("__weakref__"); + let has_inherited_weakref = typ + .mro + .read() + .iter() + .any(|base| base.attributes.read().contains_key(&__weakref__)); + if !typ.attributes.read().contains_key(&__weakref__) && !has_inherited_weakref { + unsafe { + let descriptor = vm.ctx.new_getset( + "__weakref__", + &typ, + subtype_get_weakref, + subtype_set_weakref, + ); + typ.attributes + .write() + .insert(__weakref__, descriptor.into()); + } + } + } + + // Set __doc__ to None if not already present in the type's dict + // This matches CPython's behavior in type_dict_set_doc (typeobject.c) + // which ensures every type has a __doc__ entry in its dict + { + let __doc__ = identifier!(vm, __doc__); + if !typ.attributes.read().contains_key(&__doc__) { + typ.attributes.write().insert(__doc__, vm.ctx.none()); + } + } + + // avoid deadlock + let attributes = typ + .attributes + .read() + .iter() + .filter_map(|(name, obj)| { + vm.get_method(obj.clone(), identifier!(vm, __set_name__)) + .map(|res| res.map(|meth| (obj.clone(), name.to_owned(), meth))) + }) + .collect::<PyResult<Vec<_>>>()?; + for (obj, name, set_name) in attributes { + set_name.call((typ.clone(), name), vm).inspect_err(|e| { + // PEP 678: Add a note to the original exception instead of wrapping it + // (Python 3.12+, gh-77757) + let note = format!( + "Error calling __set_name__ on '{}' instance '{}' in '{}'", + obj.class().name(), + name, + typ.name() + ); + // Ignore result - adding a note is best-effort, the original exception is what matters + drop(vm.call_method(e.as_object(), "add_note", (vm.ctx.new_str(note.as_str()),))); + })?; + } + + if let Some(init_subclass) = typ.get_super_attr(identifier!(vm, __init_subclass__)) { + let init_subclass = vm + .call_get_descriptor_specific(&init_subclass, None, Some(typ.clone().into())) + .unwrap_or(Ok(init_subclass))?; + init_subclass.call(kwargs, vm)?; + }; + + Ok(typ.into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +const SIGNATURE_END_MARKER: &str = ")\n--\n\n"; +fn get_signature(doc: &str) -> Option<&str> { + doc.find(SIGNATURE_END_MARKER).map(|index| &doc[..=index]) +} + +fn find_signature<'a>(name: &str, doc: &'a str) -> Option<&'a str> { + let name = name.rsplit('.').next().unwrap(); + let doc = doc.strip_prefix(name)?; + doc.starts_with('(').then_some(doc) +} + +pub(crate) fn get_text_signature_from_internal_doc<'a>( + name: &str, + internal_doc: &'a str, +) -> Option<&'a str> { + find_signature(name, internal_doc).and_then(get_signature) +} + +// _PyType_GetDocFromInternalDoc in CPython +fn get_doc_from_internal_doc<'a>(name: &str, internal_doc: &'a str) -> &'a str { + // Similar to CPython's _PyType_DocWithoutSignature + // If the doc starts with the type name and a '(', it's a signature + if let Some(doc_without_sig) = find_signature(name, internal_doc) { + // Find where the signature ends + if let Some(sig_end_pos) = doc_without_sig.find(SIGNATURE_END_MARKER) { + let after_sig = &doc_without_sig[sig_end_pos + SIGNATURE_END_MARKER.len()..]; + // Return the documentation after the signature, or empty string if none + return after_sig; + } + } + // If no signature found, return the whole doc + internal_doc +} + +impl Initializer for PyType { + type Args = FuncArgs; + + // type_init + fn slot_init(_zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // type.__init__() takes 1 or 3 arguments + if args.args.len() == 1 && !args.kwargs.is_empty() { + return Err(vm.new_type_error("type.__init__() takes no keyword arguments")); + } + if args.args.len() != 1 && args.args.len() != 3 { + return Err(vm.new_type_error("type.__init__() takes 1 or 3 arguments")); + } + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } +} + +impl GetAttr for PyType { + fn getattro(zelf: &Py<Self>, name_str: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + #[cold] + fn attribute_error( + zelf: &Py<PyType>, + name: &Wtf8, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + vm.new_attribute_error(format!( + "type object '{}' has no attribute '{}'", + zelf.slot_name(), + name, + )) + } + + let Some(name) = vm.ctx.interned_str(name_str) else { + return Err(attribute_error(zelf, name_str.as_wtf8(), vm)); + }; + vm_trace!("type.__getattribute__({:?}, {:?})", zelf, name); + let mcl = zelf.class(); + let mcl_attr = mcl.get_attr(name); + + if let Some(ref attr) = mcl_attr { + let attr_class = attr.class(); + let has_descr_set = attr_class.slots.descr_set.load().is_some(); + if has_descr_set { + let descr_get = attr_class.slots.descr_get.load(); + if let Some(descr_get) = descr_get { + let mcl = mcl.to_owned().into(); + return descr_get(attr.clone(), Some(zelf.to_owned().into()), Some(mcl), vm); + } + } + } + + let zelf_attr = zelf.get_attr(name); + + if let Some(attr) = zelf_attr { + let descr_get = attr.class().slots.descr_get.load(); + if let Some(descr_get) = descr_get { + descr_get(attr, None, Some(zelf.to_owned().into()), vm) + } else { + Ok(attr) + } + } else if let Some(attr) = mcl_attr { + vm.call_if_get_descriptor(&attr, zelf.to_owned().into()) + } else { + Err(attribute_error(zelf, name_str.as_wtf8(), vm)) + } + } +} + +#[pyclass] +impl Py<PyType> { + #[pygetset] + fn __mro__(&self) -> PyTuple { + let elements: Vec<PyObjectRef> = self.mro_map_collect(|x| x.as_object().to_owned()); + PyTuple::new_unchecked(elements.into_boxed_slice()) + } + + #[pygetset] + fn __doc__(&self, vm: &VirtualMachine) -> PyResult { + // Similar to CPython's type_get_doc + // For non-heap types (static types), check if there's an internal doc + if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + && let Some(internal_doc) = self.slots.doc + { + // Process internal doc, removing signature if present + let doc_str = get_doc_from_internal_doc(&self.name(), internal_doc); + return Ok(vm.ctx.new_str(doc_str).into()); + } + + // Check if there's a __doc__ in THIS type's dict only (not MRO) + // CPython returns None if __doc__ is not in the type's own dict + if let Some(doc_attr) = self.get_direct_attr(vm.ctx.intern_str("__doc__")) { + // If it's a descriptor, call its __get__ method + let descr_get = doc_attr.class().slots.descr_get.load(); + if let Some(descr_get) = descr_get { + descr_get(doc_attr, None, Some(self.to_owned().into()), vm) + } else { + Ok(doc_attr) + } + } else { + Ok(vm.ctx.none()) + } + } + + #[pygetset(setter)] + fn set___doc__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + // Similar to CPython's type_set_doc + let value = value.ok_or_else(|| { + vm.new_type_error(format!( + "cannot delete '__doc__' attribute of type '{}'", + self.name() + )) + })?; + + // Check if we can set this special type attribute + self.check_set_special_type_attr(identifier!(vm, __doc__), vm)?; + + // Set the __doc__ in the type's dict + self.attributes + .write() + .insert(identifier!(vm, __doc__), value); + + Ok(()) + } + + #[pymethod] + fn __dir__(&self) -> PyList { + let attributes: Vec<PyObjectRef> = self + .get_attributes() + .into_iter() + .map(|(k, _)| k.to_object()) + .collect(); + PyList::from(attributes) + } + + #[pymethod] + fn __instancecheck__(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + // Use real_is_instance to avoid infinite recursion + obj.real_is_instance(self.as_object(), vm) + } + + #[pymethod] + fn __subclasscheck__(&self, subclass: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + // Use real_is_subclass to avoid going through __subclasscheck__ recursion + // This matches CPython's type___subclasscheck___impl which calls _PyObject_RealIsSubclass + subclass.real_is_subclass(self.as_object(), vm) + } + + #[pyclassmethod] + fn __subclasshook__(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() + } + + #[pymethod] + fn mro(&self) -> Vec<PyObjectRef> { + self.mro_map_collect(|cls| cls.to_owned().into()) + } +} + +impl SetAttr for PyType { + fn setattro( + zelf: &Py<Self>, + attr_name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + let attr_name = vm.ctx.intern_str(attr_name.as_wtf8()); + if zelf.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error(format!( + "cannot set '{}' attribute of immutable type '{}'", + attr_name, + zelf.slot_name() + ))); + } + if let Some(attr) = zelf.get_class_attr(attr_name) { + let descr_set = attr.class().slots.descr_set.load(); + if let Some(descriptor) = descr_set { + return descriptor(&attr, zelf.to_owned().into(), value, vm); + } + } + let assign = value.is_assign(); + + // Invalidate inline caches before modifying attributes. + // This ensures other threads see the version invalidation before + // any attribute changes, preventing use-after-free of cached descriptors. + zelf.modified(); + + if let PySetterValue::Assign(value) = value { + zelf.attributes.write().insert(attr_name, value); + } else { + let prev_value = zelf.attributes.write().shift_remove(attr_name); // TODO: swap_remove applicable? + if prev_value.is_none() { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '{}'", + zelf.name(), + attr_name, + ))); + } + } + + if attr_name.as_wtf8().starts_with("__") && attr_name.as_wtf8().ends_with("__") { + if assign { + zelf.update_slot::<true>(attr_name, &vm.ctx); + } else { + zelf.update_slot::<false>(attr_name, &vm.ctx); + } + } + Ok(()) + } +} + +impl Callable for PyType { + type Args = FuncArgs; + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + vm_trace!("type_call: {:?}", zelf); + + if zelf.is(vm.ctx.types.type_type) { + let num_args = args.args.len(); + if num_args == 1 && args.kwargs.is_empty() { + return Ok(args.args[0].obj_type()); + } + if num_args != 3 { + return Err(vm.new_type_error("type() takes 1 or 3 arguments")); + } + } + + let obj = if let Some(slot_new) = zelf.slots.new.load() { + slot_new(zelf.to_owned(), args.clone(), vm)? + } else { + return Err(vm.new_type_error(format!("cannot create '{}' instances", zelf.slots.name))); + }; + + if !obj.class().fast_issubclass(zelf) { + return Ok(obj); + } + + if let Some(init_method) = obj.class().slots.init.load() { + init_method(obj.clone(), args, vm)?; + } + Ok(obj) + } +} + +impl AsNumber for PyType { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| or_(a.to_owned(), b.to_owned(), vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Representable for PyType { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let module = zelf.__module__(vm); + let module = module.downcast_ref::<PyStr>().map(|m| m.as_wtf8()); + + let repr = match module { + Some(module) if module != "builtins" => { + let qualname = zelf.__qualname__(vm); + let qualname = qualname.downcast_ref::<PyStr>().map(|n| n.as_wtf8()); + let name = zelf.name(); + let qualname = qualname.unwrap_or_else(|| name.as_ref()); + format!("<class '{module}.{qualname}'>") + } + _ => format!("<class '{}'>", zelf.slot_name()), + }; + Ok(repr) + } +} + +// = get_builtin_base_with_dict +fn get_builtin_base_with_dict(typ: &Py<PyType>, vm: &VirtualMachine) -> Option<PyTypeRef> { + let mut current = Some(typ.to_owned()); + while let Some(t) = current { + // In CPython: type->tp_dictoffset != 0 && !(type->tp_flags & Py_TPFLAGS_HEAPTYPE) + // Special case: type itself is a builtin with dict support + if t.is(vm.ctx.types.type_type) { + return Some(t); + } + // We check HAS_DICT flag (equivalent to tp_dictoffset != 0) and HEAPTYPE + if t.slots.flags.contains(PyTypeFlags::HAS_DICT) + && !t.slots.flags.contains(PyTypeFlags::HEAPTYPE) + { + return Some(t); + } + current = t.__base__(); + } + None +} + +// = get_dict_descriptor +fn get_dict_descriptor(base: &Py<PyType>, vm: &VirtualMachine) -> Option<PyObjectRef> { + let dict_attr = identifier!(vm, __dict__); + // Use _PyType_Lookup (which is lookup_ref in RustPython) + base.lookup_ref(dict_attr, vm) +} + +// = raise_dict_descr_error +fn raise_dict_descriptor_error(obj: &PyObject, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_type_error(format!( + "this __dict__ descriptor does not support '{}' objects", + obj.class().name() + )) +} + +fn subtype_get_dict(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let base = get_builtin_base_with_dict(obj.class(), vm); + + if let Some(base_type) = base { + if let Some(descr) = get_dict_descriptor(&base_type, vm) { + // Call the descriptor's tp_descr_get + vm.call_get_descriptor(&descr, obj.clone()) + .unwrap_or_else(|| Err(raise_dict_descriptor_error(&obj, vm))) + } else { + Err(raise_dict_descriptor_error(&obj, vm)) + } + } else { + // PyObject_GenericGetDict + object::object_get_dict(obj, vm).map(Into::into) + } +} + +// = subtype_setdict +fn subtype_set_dict(obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let base = get_builtin_base_with_dict(obj.class(), vm); + + if let Some(base_type) = base { + if let Some(descr) = get_dict_descriptor(&base_type, vm) { + // Call the descriptor's tp_descr_set + let descr_set = descr + .class() + .slots + .descr_set + .load() + .ok_or_else(|| raise_dict_descriptor_error(&obj, vm))?; + descr_set(&descr, obj, PySetterValue::Assign(value), vm) + } else { + Err(raise_dict_descriptor_error(&obj, vm)) + } + } else { + // PyObject_GenericSetDict + object::object_set_dict(obj, value.try_into_value(vm)?, vm)?; + Ok(()) + } +} + +// subtype_get_weakref +fn subtype_get_weakref(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Return the first weakref in the weakref list, or None + let weakref = obj.get_weakrefs(); + Ok(weakref.unwrap_or_else(|| vm.ctx.none())) +} + +// subtype_set_weakref: __weakref__ is read-only +fn subtype_set_weakref(obj: PyObjectRef, _value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_attribute_error(format!( + "attribute '__weakref__' of '{}' objects is not writable", + obj.class().name() + ))) +} + +/* + * The magical type type + */ + +/// Vectorcall for PyType (PEP 590). +/// Fast path: type(x) returns x.__class__ without constructing FuncArgs. +fn vectorcall_type( + zelf_obj: &PyObject, + args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, +) -> PyResult { + let zelf: &Py<PyType> = zelf_obj.downcast_ref().unwrap(); + + // type(x) fast path: single positional arg, no kwargs + if zelf.is(vm.ctx.types.type_type) { + let no_kwargs = kwnames.is_none_or(|kw| kw.is_empty()); + if nargs == 1 && no_kwargs { + return Ok(args[0].obj_type()); + } + } + + // Fallback: construct FuncArgs and use standard call + let func_args = FuncArgs::from_vectorcall_owned(args, nargs, kwnames); + PyType::call(zelf, func_args, vm) +} + +pub(crate) fn init(ctx: &'static Context) { + PyType::extend_class(ctx, ctx.types.type_type); + ctx.types + .type_type + .slots + .vectorcall + .store(Some(vectorcall_type)); +} + +pub(crate) fn call_slot_new( + typ: PyTypeRef, + subtype: PyTypeRef, + args: FuncArgs, + vm: &VirtualMachine, +) -> PyResult { + // Check DISALLOW_INSTANTIATION flag on subtype (the type being instantiated) + if subtype + .slots + .flags + .has_feature(PyTypeFlags::DISALLOW_INSTANTIATION) + { + return Err(vm.new_type_error(format!("cannot create '{}' instances", subtype.slot_name()))); + } + + // "is not safe" check (tp_new_wrapper logic) + // Check that the user doesn't do something silly and unsafe like + // object.__new__(dict). To do this, we check that the most derived base + // that's not a heap type is this type. + let mut staticbase = subtype.clone(); + while staticbase.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + if let Some(base) = staticbase.base.as_ref() { + staticbase = base.clone(); + } else { + break; + } + } + + // Check if staticbase's tp_new differs from typ's tp_new + let typ_new = typ.slots.new.load(); + let staticbase_new = staticbase.slots.new.load(); + if typ_new.map(|f| f as usize) != staticbase_new.map(|f| f as usize) { + return Err(vm.new_type_error(format!( + "{}.__new__({}) is not safe, use {}.__new__()", + typ.slot_name(), + subtype.slot_name(), + staticbase.slot_name() + ))); + } + + let slot_new = typ + .slots + .new + .load() + .expect("Should be able to find a new slot somewhere in the mro"); + slot_new(subtype, args, vm) +} + +pub(crate) fn or_(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + union_::or_op(zelf, other, vm) +} + +fn take_next_base(bases: &mut [Vec<PyTypeRef>]) -> Option<PyTypeRef> { + for base in bases.iter() { + let head = base[0].clone(); + if !bases.iter().any(|x| x[1..].iter().any(|x| x.is(&head))) { + // Remove from other heads. + for item in bases.iter_mut() { + if item[0].is(&head) { + item.remove(0); + } + } + + return Some(head); + } + } + + None +} + +fn linearise_mro(mut bases: Vec<Vec<PyTypeRef>>) -> Result<Vec<PyTypeRef>, String> { + vm_trace!("Linearise MRO: {:?}", bases); + // Python requires that the class direct bases are kept in the same order. + // This is called local precedence ordering. + // This means we must verify that for classes A(), B(A) we must reject C(A, B) even though this + // algorithm will allow the mro ordering of [C, B, A, object]. + // To verify this, we make sure non of the direct bases are in the mro of bases after them. + for (i, base_mro) in bases.iter().enumerate() { + let base = &base_mro[0]; // MROs cannot be empty. + for later_mro in &bases[i + 1..] { + // We start at index 1 to skip direct bases. + // This will not catch duplicate bases, but such a thing is already tested for. + if later_mro[1..].iter().any(|cls| cls.is(base)) { + return Err(format!( + "Cannot create a consistent method resolution order (MRO) for bases {}", + bases.iter().map(|x| x.first().unwrap()).format(", ") + )); + } + } + } + + let mut result = vec![]; + while !bases.is_empty() { + let head = take_next_base(&mut bases).ok_or_else(|| { + // Take the head class of each class here. Now that we have reached the problematic bases. + // Because this failed, we assume the lists cannot be empty. + format!( + "Cannot create a consistent method resolution order (MRO) for bases {}", + bases.iter().map(|x| x.first().unwrap()).format(", ") + ) + })?; + + result.push(head); + + bases.retain(|x| !x.is_empty()); + } + Ok(result) +} + +fn calculate_meta_class( + metatype: PyTypeRef, + bases: &[PyTypeRef], + vm: &VirtualMachine, +) -> PyResult<PyTypeRef> { + // = _PyType_CalculateMetaclass + let mut winner = metatype; + for base in bases { + let base_type = base.class(); + + // First try fast_issubclass for PyType instances + if winner.fast_issubclass(base_type) { + continue; + } else if base_type.fast_issubclass(&winner) { + winner = base_type.to_owned(); + continue; + } + + // If fast_issubclass didn't work, fall back to general is_subclass + // This handles cases where metaclasses are not PyType subclasses + let winner_is_subclass = winner.as_object().is_subclass(base_type.as_object(), vm)?; + if winner_is_subclass { + continue; + } + + let base_type_is_subclass = base_type.as_object().is_subclass(winner.as_object(), vm)?; + if base_type_is_subclass { + winner = base_type.to_owned(); + continue; + } + + return Err(vm.new_type_error( + "metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass \ + of the metaclasses of all its bases", + )); + } + Ok(winner) +} + +/// Returns true if the two types have different instance layouts. +fn shape_differs(t1: &Py<PyType>, t2: &Py<PyType>) -> bool { + t1.__basicsize__() != t2.__basicsize__() || t1.slots.itemsize != t2.slots.itemsize +} + +fn solid_base<'a>(typ: &'a Py<PyType>, vm: &VirtualMachine) -> &'a Py<PyType> { + let base = if let Some(base) = &typ.base { + solid_base(base, vm) + } else { + vm.ctx.types.object_type + }; + + if shape_differs(typ, base) { typ } else { base } +} + +fn best_base<'a>(bases: &'a [PyTypeRef], vm: &VirtualMachine) -> PyResult<&'a Py<PyType>> { + let mut base: Option<&Py<PyType>> = None; + let mut winner: Option<&Py<PyType>> = None; + + for base_i in bases { + // if !base_i.fast_issubclass(vm.ctx.types.type_type) { + // println!("base_i type : {}", base_i.name()); + // return Err(vm.new_type_error("best must be types".into())); + // } + + if !base_i.slots.flags.has_feature(PyTypeFlags::BASETYPE) { + return Err(vm.new_type_error(format!( + "type '{}' is not an acceptable base type", + base_i.slot_name() + ))); + } + + let candidate = solid_base(base_i, vm); + if winner.is_none() { + winner = Some(candidate); + base = Some(base_i.deref()); + } else if winner.unwrap().fast_issubclass(candidate) { + // Do nothing + } else if candidate.fast_issubclass(winner.unwrap()) { + winner = Some(candidate); + base = Some(base_i.deref()); + } else { + return Err(vm.new_type_error("multiple bases have instance layout conflict")); + } + } + + debug_assert!(base.is_some()); + Ok(base.unwrap()) +} + +/// Apply Python name mangling for private attributes. +/// `__x` becomes `_ClassName__x` if inside a class. +fn mangle_name(class_name: &str, name: &str) -> String { + // Only mangle names starting with __ and not ending with __ + if !name.starts_with("__") || name.ends_with("__") || name.contains('.') { + return name.to_string(); + } + // Strip leading underscores from class name + let class_name = class_name.trim_start_matches('_'); + format!("_{}{}", class_name, name) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn map_ids(obj: Result<Vec<PyTypeRef>, String>) -> Result<Vec<usize>, String> { + Ok(obj?.into_iter().map(|x| x.get_id()).collect()) + } + + #[test] + fn test_linearise() { + let context = Context::genesis(); + let object = context.types.object_type.to_owned(); + let type_type = context.types.type_type.to_owned(); + + let a = PyType::new_heap( + "A", + vec![object.clone()], + PyAttributes::default(), + Default::default(), + type_type.clone(), + context, + ) + .unwrap(); + let b = PyType::new_heap( + "B", + vec![object.clone()], + PyAttributes::default(), + Default::default(), + type_type, + context, + ) + .unwrap(); + + assert_eq!( + map_ids(linearise_mro(vec![ + vec![object.clone()], + vec![object.clone()] + ])), + map_ids(Ok(vec![object.clone()])) + ); + assert_eq!( + map_ids(linearise_mro(vec![ + vec![a.clone(), object.clone()], + vec![b.clone(), object.clone()], + ])), + map_ids(Ok(vec![a, b, object])) + ); + } +} diff --git a/crates/vm/src/builtins/union.rs b/crates/vm/src/builtins/union.rs new file mode 100644 index 00000000000..0f7ce123721 --- /dev/null +++ b/crates/vm/src/builtins/union.rs @@ -0,0 +1,546 @@ +use super::{genericalias, type_}; +use crate::common::lock::LazyLock; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, + builtins::{PyFrozenSet, PySet, PyStr, PyTuple, PyTupleRef, PyType}, + class::PyClassImpl, + common::hash, + convert::ToPyObject, + function::PyComparisonValue, + protocol::{PyMappingMethods, PyNumberMethods}, + stdlib::_typing::{TypeAliasType, call_typing_func_object}, + types::{AsMapping, AsNumber, Comparable, GetAttr, Hashable, PyComparisonOp, Representable}, +}; +use alloc::fmt; + +const CLS_ATTRS: &[&str] = &["__module__"]; + +#[pyclass(module = "typing", name = "Union", traverse)] +pub struct PyUnion { + args: PyTupleRef, + /// Frozenset of hashable args, or None if all args were hashable + hashable_args: Option<PyRef<PyFrozenSet>>, + /// Tuple of initially unhashable args, or None if all args were hashable + unhashable_args: Option<PyTupleRef>, + parameters: PyTupleRef, +} + +impl fmt::Debug for PyUnion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("UnionObject") + } +} + +impl PyPayload for PyUnion { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.union_type + } +} + +impl PyUnion { + /// Create a new union from dedup result (internal use) + fn from_components(result: UnionComponents, vm: &VirtualMachine) -> PyResult<Self> { + let parameters = make_parameters(&result.args, vm)?; + Ok(Self { + args: result.args, + hashable_args: result.hashable_args, + unhashable_args: result.unhashable_args, + parameters, + }) + } + + /// Direct access to args field (_Py_union_args) + #[inline] + pub fn args(&self) -> &Py<PyTuple> { + &self.args + } + + fn repr(&self, vm: &VirtualMachine) -> PyResult<String> { + fn repr_item(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + if obj.is(vm.ctx.types.none_type) { + return Ok("None".to_string()); + } + + if vm + .get_attribute_opt(obj.clone(), identifier!(vm, __origin__))? + .is_some() + && vm + .get_attribute_opt(obj.clone(), identifier!(vm, __args__))? + .is_some() + { + return Ok(obj.repr(vm)?.to_string()); + } + + match ( + vm.get_attribute_opt(obj.clone(), identifier!(vm, __qualname__))? + .and_then(|o| o.downcast_ref::<PyStr>().map(|n| n.to_string())), + vm.get_attribute_opt(obj.clone(), identifier!(vm, __module__))? + .and_then(|o| o.downcast_ref::<PyStr>().map(|m| m.to_string())), + ) { + (None, _) | (_, None) => Ok(obj.repr(vm)?.to_string()), + (Some(qualname), Some(module)) => Ok(if module == "builtins" { + qualname + } else { + format!("{module}.{qualname}") + }), + } + } + + Ok(self + .args + .iter() + .map(|o| repr_item(o.clone(), vm)) + .collect::<PyResult<Vec<_>>>()? + .join(" | ")) + } +} + +#[pyclass( + flags(DISALLOW_INSTANTIATION, HAS_WEAKREF), + with(Hashable, Comparable, AsMapping, AsNumber, Representable) +)] +impl PyUnion { + #[pygetset] + fn __name__(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str("Union").into() + } + + #[pygetset] + fn __qualname__(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_str("Union").into() + } + + #[pygetset] + fn __origin__(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.union_type.to_owned().into() + } + + #[pygetset] + fn __parameters__(&self) -> PyObjectRef { + self.parameters.clone().into() + } + + #[pygetset] + fn __args__(&self) -> PyObjectRef { + self.args.clone().into() + } + + #[pymethod] + fn __instancecheck__( + zelf: PyRef<Self>, + obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<bool> { + if zelf + .args + .iter() + .any(|x| x.class().is(vm.ctx.types.generic_alias_type)) + { + Err(vm.new_type_error("isinstance() argument 2 cannot be a parameterized generic")) + } else { + obj.is_instance(zelf.__args__().as_object(), vm) + } + } + + #[pymethod] + fn __subclasscheck__( + zelf: PyRef<Self>, + obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<bool> { + if zelf + .args + .iter() + .any(|x| x.class().is(vm.ctx.types.generic_alias_type)) + { + Err(vm.new_type_error("issubclass() argument 2 cannot be a parameterized generic")) + } else { + obj.is_subclass(zelf.__args__().as_object(), vm) + } + } + + fn __or__(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + type_::or_(zelf, other, vm) + } + + #[pymethod] + fn __mro_entries__(zelf: PyRef<Self>, _args: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("Cannot subclass {}", zelf.repr(vm)?))) + } + + #[pyclassmethod] + fn __class_getitem__( + _cls: crate::builtins::PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // Convert args to tuple if not already + let args_tuple = if let Some(tuple) = args.downcast_ref::<PyTuple>() { + tuple.to_owned() + } else { + PyTuple::new_ref(vec![args], &vm.ctx) + }; + + // Check for empty union + if args_tuple.is_empty() { + return Err(vm.new_type_error("Cannot create empty Union")); + } + + // Create union using make_union to properly handle None -> NoneType conversion + make_union(&args_tuple, vm) + } +} + +fn is_unionable(obj: PyObjectRef, vm: &VirtualMachine) -> bool { + let cls = obj.class(); + cls.is(vm.ctx.types.none_type) + || obj.downcastable::<PyType>() + || cls.fast_issubclass(vm.ctx.types.generic_alias_type) + || cls.is(vm.ctx.types.union_type) + || obj.downcast_ref::<TypeAliasType>().is_some() +} + +fn type_check(arg: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Fast path to avoid calling into typing.py + if is_unionable(arg.clone(), vm) { + return Ok(arg); + } + let message_str: PyObjectRef = vm + .ctx + .new_str("Union[arg, ...]: each arg must be a type.") + .into(); + call_typing_func_object(vm, "_type_check", (arg, message_str)) +} + +fn has_union_operands(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> bool { + let union_type = vm.ctx.types.union_type; + a.class().is(union_type) || b.class().is(union_type) +} + +pub fn or_op(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if !has_union_operands(zelf.clone(), other.clone(), vm) + && (!is_unionable(zelf.clone(), vm) || !is_unionable(other.clone(), vm)) + { + return Ok(vm.ctx.not_implemented()); + } + + let left = type_check(zelf, vm)?; + let right = type_check(other, vm)?; + let tuple = PyTuple::new_ref(vec![left, right], &vm.ctx); + make_union(&tuple, vm) +} + +fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let parameters = genericalias::make_parameters(args, vm); + let result = dedup_and_flatten_args(&parameters, vm)?; + Ok(result.args) +} + +fn flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { + let mut total_args = 0; + for arg in args { + if let Some(pyref) = arg.downcast_ref::<PyUnion>() { + total_args += pyref.args.len(); + } else { + total_args += 1; + }; + } + + let mut flattened_args = Vec::with_capacity(total_args); + for arg in args { + if let Some(pyref) = arg.downcast_ref::<PyUnion>() { + flattened_args.extend(pyref.args.iter().cloned()); + } else if vm.is_none(arg) { + flattened_args.push(vm.ctx.types.none_type.to_owned().into()); + } else if arg.downcast_ref::<PyStr>().is_some() { + // Convert string to ForwardRef + match string_to_forwardref(arg.clone(), vm) { + Ok(fr) => flattened_args.push(fr), + Err(_) => flattened_args.push(arg.clone()), + } + } else { + flattened_args.push(arg.clone()); + }; + } + + PyTuple::new_ref(flattened_args, &vm.ctx) +} + +fn string_to_forwardref(arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Import annotationlib.ForwardRef and create a ForwardRef + let annotationlib = vm.import("annotationlib", 0)?; + let forwardref_cls = annotationlib.get_attr("ForwardRef", vm)?; + forwardref_cls.call((arg,), vm) +} + +/// Components for creating a PyUnion after deduplication +struct UnionComponents { + /// All unique args in order + args: PyTupleRef, + /// Frozenset of hashable args (for fast equality comparison) + hashable_args: Option<PyRef<PyFrozenSet>>, + /// Tuple of unhashable args at creation time (for hash error message) + unhashable_args: Option<PyTupleRef>, +} + +fn dedup_and_flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyResult<UnionComponents> { + let args = flatten_args(args, vm); + + // Use set-based deduplication like CPython: + // - For hashable elements: use Python's set semantics (hash + equality) + // - For unhashable elements: use equality comparison + // + // This avoids calling __eq__ when hashes differ, so `int | BadType` + // doesn't raise even if BadType.__eq__ raises. + + let mut new_args: Vec<PyObjectRef> = Vec::with_capacity(args.len()); + + // Track hashable elements using a Python set (uses hash + equality) + let hashable_set = PySet::default().into_ref(&vm.ctx); + let mut hashable_list: Vec<PyObjectRef> = Vec::new(); + let mut unhashable_list: Vec<PyObjectRef> = Vec::new(); + + for arg in &*args { + // Try to hash the element first + match arg.hash(vm) { + Ok(_) => { + // Element is hashable - use set for deduplication + // Set membership uses hash first, then equality only if hashes match + let contains = vm + .call_method(hashable_set.as_ref(), "__contains__", (arg.clone(),)) + .and_then(|r| r.try_to_bool(vm))?; + if !contains { + hashable_set.add(arg.clone(), vm)?; + hashable_list.push(arg.clone()); + new_args.push(arg.clone()); + } + } + Err(_) => { + // Element is unhashable - use equality comparison + let mut is_duplicate = false; + for existing in &unhashable_list { + match existing.rich_compare_bool(arg, PyComparisonOp::Eq, vm) { + Ok(true) => { + is_duplicate = true; + break; + } + Ok(false) => continue, + Err(e) => return Err(e), + } + } + if !is_duplicate { + unhashable_list.push(arg.clone()); + new_args.push(arg.clone()); + } + } + } + } + + new_args.shrink_to_fit(); + + // Create hashable_args frozenset if there are hashable elements + let hashable_args = if !hashable_list.is_empty() { + Some(PyFrozenSet::from_iter(vm, hashable_list.into_iter())?.into_ref(&vm.ctx)) + } else { + None + }; + + // Create unhashable_args tuple if there are unhashable elements + let unhashable_args = if !unhashable_list.is_empty() { + Some(PyTuple::new_ref(unhashable_list, &vm.ctx)) + } else { + None + }; + + Ok(UnionComponents { + args: PyTuple::new_ref(new_args, &vm.ctx), + hashable_args, + unhashable_args, + }) +} + +pub fn make_union(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyResult { + let result = dedup_and_flatten_args(args, vm)?; + Ok(match result.args.len() { + 1 => result.args[0].to_owned(), + _ => PyUnion::from_components(result, vm)?.to_pyobject(vm), + }) +} + +impl PyUnion { + fn getitem(zelf: PyRef<Self>, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let new_args = genericalias::subs_parameters( + zelf.to_owned().into(), + zelf.args.clone(), + zelf.parameters.clone(), + needle, + vm, + )?; + let res; + if new_args.is_empty() { + res = make_union(&new_args, vm)?; + } else { + let mut tmp = new_args[0].to_owned(); + for arg in new_args.iter().skip(1) { + tmp = vm._or(&tmp, arg)?; + } + res = tmp; + } + + Ok(res) + } +} + +impl AsMapping for PyUnion { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + subscript: atomic_func!(|mapping, needle, vm| { + let zelf = PyUnion::mapping_downcast(mapping); + PyUnion::getitem(zelf.to_owned(), needle.to_owned(), vm) + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } +} + +impl AsNumber for PyUnion { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| PyUnion::__or__(a.to_owned(), b.to_owned(), vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Comparable for PyUnion { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + + // Check if lengths are equal + if zelf.args.len() != other.args.len() { + return Ok(PyComparisonValue::Implemented(false)); + } + + // Fast path: if both unions have all hashable args, compare frozensets directly + // Always use Eq here since eq_only handles Ne by negating the result + if zelf.unhashable_args.is_none() + && other.unhashable_args.is_none() + && let (Some(a), Some(b)) = (&zelf.hashable_args, &other.hashable_args) + { + let eq = a + .as_object() + .rich_compare_bool(b.as_object(), PyComparisonOp::Eq, vm)?; + return Ok(PyComparisonValue::Implemented(eq)); + } + + // Slow path: O(n^2) nested loop comparison for unhashable elements + // Check if all elements in zelf.args are in other.args + for arg_a in &*zelf.args { + let mut found = false; + for arg_b in &*other.args { + match arg_a.rich_compare_bool(arg_b, PyComparisonOp::Eq, vm) { + Ok(true) => { + found = true; + break; + } + Ok(false) => continue, + Err(e) => return Err(e), // Propagate comparison errors + } + } + if !found { + return Ok(PyComparisonValue::Implemented(false)); + } + } + + // Check if all elements in other.args are in zelf.args (for symmetry) + for arg_b in &*other.args { + let mut found = false; + for arg_a in &*zelf.args { + match arg_b.rich_compare_bool(arg_a, PyComparisonOp::Eq, vm) { + Ok(true) => { + found = true; + break; + } + Ok(false) => continue, + Err(e) => return Err(e), // Propagate comparison errors + } + } + if !found { + return Ok(PyComparisonValue::Implemented(false)); + } + } + + Ok(PyComparisonValue::Implemented(true)) + }) + } +} + +impl Hashable for PyUnion { + #[inline] + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { + // If there are any unhashable args from creation time, the union is unhashable + if let Some(ref unhashable_args) = zelf.unhashable_args { + let n = unhashable_args.len(); + // Try to hash each previously unhashable arg to get an error + for arg in unhashable_args.iter() { + arg.hash(vm)?; + } + // All previously unhashable args somehow became hashable + // But still raise an error to maintain consistent hashing + return Err(vm.new_type_error(format!( + "union contains {} unhashable element{}", + n, + if n > 1 { "s" } else { "" } + ))); + } + + // If we have a stored frozenset of hashable args, use that + if let Some(ref hashable_args) = zelf.hashable_args { + return PyFrozenSet::hash(hashable_args, vm); + } + + // Fallback: compute hash from args + let mut args_to_hash = Vec::new(); + for arg in &*zelf.args { + match arg.hash(vm) { + Ok(_) => args_to_hash.push(arg.clone()), + Err(e) => return Err(e), + } + } + let set = PyFrozenSet::from_iter(vm, args_to_hash.into_iter())?; + PyFrozenSet::hash(&set.into_ref(&vm.ctx), vm) + } +} + +impl GetAttr for PyUnion { + fn getattro(zelf: &Py<Self>, attr: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + for &exc in CLS_ATTRS { + if *exc == attr.to_string() { + return zelf.as_object().generic_getattr(attr, vm); + } + } + zelf.as_object().get_attr(attr, vm) + } +} + +impl Representable for PyUnion { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + zelf.repr(vm) + } +} + +pub fn init(context: &'static Context) { + let union_type = &context.types.union_type; + PyUnion::extend_class(context, union_type); +} diff --git a/crates/vm/src/builtins/weakproxy.rs b/crates/vm/src/builtins/weakproxy.rs new file mode 100644 index 00000000000..561e69d6180 --- /dev/null +++ b/crates/vm/src/builtins/weakproxy.rs @@ -0,0 +1,363 @@ +use super::{PyStr, PyStrRef, PyType, PyTypeRef, PyWeak}; +use crate::common::lock::LazyLock; +use crate::{ + Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, + class::PyClassImpl, + common::hash::PyHash, + function::{OptionalArg, PyComparisonValue, PySetterValue}, + protocol::{PyIter, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + stdlib::builtins::reversed, + types::{ + AsMapping, AsNumber, AsSequence, Comparable, Constructor, GetAttr, Hashable, IterNext, + Iterable, PyComparisonOp, Representable, SetAttr, + }, +}; + +#[pyclass(module = false, name = "weakproxy", unhashable = true, traverse)] +#[derive(Debug)] +pub struct PyWeakProxy { + weak: PyRef<PyWeak>, +} + +impl PyPayload for PyWeakProxy { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.weakproxy_type + } +} + +#[derive(FromArgs)] +pub struct WeakProxyNewArgs { + #[pyarg(positional)] + referent: PyObjectRef, + #[pyarg(positional, optional)] + callback: OptionalArg<PyObjectRef>, +} + +impl Constructor for PyWeakProxy { + type Args = WeakProxyNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { referent, callback }: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + // using an internal subclass as the class prevents us from getting the generic weakref, + // which would mess up the weakref count + let weak_cls = WEAK_SUBCLASS.get_or_init(|| { + vm.ctx.new_class( + None, + "__weakproxy", + vm.ctx.types.weakref_type.to_owned(), + super::PyWeak::make_slots(), + ) + }); + // TODO: PyWeakProxy should use the same payload as PyWeak + Ok(Self { + weak: referent.downgrade_with_typ(callback.into_option(), weak_cls.clone(), vm)?, + }) + } +} + +crate::common::static_cell! { + static WEAK_SUBCLASS: PyTypeRef; +} + +#[pyclass(with( + GetAttr, + SetAttr, + Constructor, + Comparable, + AsNumber, + AsSequence, + AsMapping, + Representable, + IterNext +))] +impl PyWeakProxy { + fn try_upgrade(&self, vm: &VirtualMachine) -> PyResult { + self.weak.upgrade().ok_or_else(|| new_reference_error(vm)) + } + + #[pymethod] + fn __str__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + zelf.try_upgrade(vm)?.str(vm) + } + + fn len(&self, vm: &VirtualMachine) -> PyResult<usize> { + self.try_upgrade(vm)?.length(vm) + } + + #[pymethod] + fn __bytes__(&self, vm: &VirtualMachine) -> PyResult { + self.try_upgrade(vm)?.bytes(vm) + } + + #[pymethod] + fn __reversed__(&self, vm: &VirtualMachine) -> PyResult { + let obj = self.try_upgrade(vm)?; + reversed(obj, vm) + } + fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + self.try_upgrade(vm)? + .sequence_unchecked() + .contains(&needle, vm) + } + + fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let obj = self.try_upgrade(vm)?; + obj.get_item(&*needle, vm) + } + + fn setitem( + &self, + needle: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let obj = self.try_upgrade(vm)?; + obj.set_item(&*needle, value, vm) + } + + fn delitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let obj = self.try_upgrade(vm)?; + obj.del_item(&*needle, vm) + } +} + +impl Iterable for PyWeakProxy { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let obj = zelf.try_upgrade(vm)?; + Ok(obj.get_iter(vm)?.into()) + } +} + +impl IterNext for PyWeakProxy { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let obj = zelf.try_upgrade(vm)?; + if obj.class().slots.iternext.load().is_none() { + return Err(vm.new_type_error("Weakref proxy referenced a non-iterator".to_owned())); + } + PyIter::new(obj).next(vm) + } +} + +fn new_reference_error(vm: &VirtualMachine) -> PyRef<super::PyBaseException> { + vm.new_exception_msg( + vm.ctx.exceptions.reference_error.to_owned(), + "weakly-referenced object no longer exists".into(), + ) +} + +impl GetAttr for PyWeakProxy { + // TODO: callbacks + fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + let obj = zelf.try_upgrade(vm)?; + obj.get_attr(name, vm) + } +} + +impl SetAttr for PyWeakProxy { + fn setattro( + zelf: &Py<Self>, + attr_name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + let obj = zelf.try_upgrade(vm)?; + obj.call_set_attr(vm, attr_name, value) + } +} + +fn proxy_upgrade(obj: &PyObject, vm: &VirtualMachine) -> PyResult { + obj.downcast_ref::<PyWeakProxy>() + .expect("proxy_upgrade called on non-PyWeakProxy object") + .try_upgrade(vm) +} + +fn proxy_upgrade_opt(obj: &PyObject, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + match obj.downcast_ref::<PyWeakProxy>() { + Some(proxy) => Ok(Some(proxy.try_upgrade(vm)?)), + None => Ok(None), + } +} + +fn proxy_unary_op( + obj: &PyObject, + vm: &VirtualMachine, + op: fn(&VirtualMachine, &PyObject) -> PyResult, +) -> PyResult { + let upgraded = proxy_upgrade(obj, vm)?; + op(vm, &upgraded) +} + +macro_rules! proxy_unary_slot { + ($vm_method:ident) => { + Some(|number, vm| proxy_unary_op(number.obj, vm, |vm, obj| vm.$vm_method(obj))) + }; +} + +fn proxy_binary_op( + a: &PyObject, + b: &PyObject, + vm: &VirtualMachine, + op: fn(&VirtualMachine, &PyObject, &PyObject) -> PyResult, +) -> PyResult { + let a_up = proxy_upgrade_opt(a, vm)?; + let b_up = proxy_upgrade_opt(b, vm)?; + let a_ref = a_up.as_deref().unwrap_or(a); + let b_ref = b_up.as_deref().unwrap_or(b); + op(vm, a_ref, b_ref) +} + +macro_rules! proxy_binary_slot { + ($vm_method:ident) => { + Some(|a, b, vm| proxy_binary_op(a, b, vm, |vm, a, b| vm.$vm_method(a, b))) + }; +} + +fn proxy_ternary_op( + a: &PyObject, + b: &PyObject, + c: &PyObject, + vm: &VirtualMachine, + op: fn(&VirtualMachine, &PyObject, &PyObject, &PyObject) -> PyResult, +) -> PyResult { + let a_up = proxy_upgrade_opt(a, vm)?; + let b_up = proxy_upgrade_opt(b, vm)?; + let c_up = proxy_upgrade_opt(c, vm)?; + let a_ref = a_up.as_deref().unwrap_or(a); + let b_ref = b_up.as_deref().unwrap_or(b); + let c_ref = c_up.as_deref().unwrap_or(c); + op(vm, a_ref, b_ref, c_ref) +} + +macro_rules! proxy_ternary_slot { + ($vm_method:ident) => { + Some(|a, b, c, vm| proxy_ternary_op(a, b, c, vm, |vm, a, b, c| vm.$vm_method(a, b, c))) + }; +} + +impl AsNumber for PyWeakProxy { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: LazyLock<PyNumberMethods> = LazyLock::new(|| PyNumberMethods { + boolean: Some(|number, vm| { + let obj = proxy_upgrade(number.obj, vm)?; + obj.is_true(vm) + }), + int: Some(|number, vm| { + let obj = proxy_upgrade(number.obj, vm)?; + obj.try_int(vm).map(Into::into) + }), + float: Some(|number, vm| { + let obj = proxy_upgrade(number.obj, vm)?; + obj.try_float(vm).map(Into::into) + }), + index: Some(|number, vm| { + let obj = proxy_upgrade(number.obj, vm)?; + obj.try_index(vm).map(Into::into) + }), + negative: proxy_unary_slot!(_neg), + positive: proxy_unary_slot!(_pos), + absolute: proxy_unary_slot!(_abs), + invert: proxy_unary_slot!(_invert), + add: proxy_binary_slot!(_add), + subtract: proxy_binary_slot!(_sub), + multiply: proxy_binary_slot!(_mul), + remainder: proxy_binary_slot!(_mod), + divmod: proxy_binary_slot!(_divmod), + lshift: proxy_binary_slot!(_lshift), + rshift: proxy_binary_slot!(_rshift), + and: proxy_binary_slot!(_and), + xor: proxy_binary_slot!(_xor), + or: proxy_binary_slot!(_or), + floor_divide: proxy_binary_slot!(_floordiv), + true_divide: proxy_binary_slot!(_truediv), + matrix_multiply: proxy_binary_slot!(_matmul), + inplace_add: proxy_binary_slot!(_iadd), + inplace_subtract: proxy_binary_slot!(_isub), + inplace_multiply: proxy_binary_slot!(_imul), + inplace_remainder: proxy_binary_slot!(_imod), + inplace_lshift: proxy_binary_slot!(_ilshift), + inplace_rshift: proxy_binary_slot!(_irshift), + inplace_and: proxy_binary_slot!(_iand), + inplace_xor: proxy_binary_slot!(_ixor), + inplace_or: proxy_binary_slot!(_ior), + inplace_floor_divide: proxy_binary_slot!(_ifloordiv), + inplace_true_divide: proxy_binary_slot!(_itruediv), + inplace_matrix_multiply: proxy_binary_slot!(_imatmul), + power: proxy_ternary_slot!(_pow), + inplace_power: proxy_ternary_slot!(_ipow), + }); + &AS_NUMBER + } +} + +impl Comparable for PyWeakProxy { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + let obj = zelf.try_upgrade(vm)?; + Ok(PyComparisonValue::Implemented( + obj.rich_compare_bool(other, op, vm)?, + )) + } +} + +impl AsSequence for PyWeakProxy { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, vm| PyWeakProxy::sequence_downcast(seq).len(vm)), + contains: atomic_func!(|seq, needle, vm| { + PyWeakProxy::sequence_downcast(seq).__contains__(needle.to_owned(), vm) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsMapping for PyWeakProxy { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: PyMappingMethods = PyMappingMethods { + length: atomic_func!(|mapping, vm| PyWeakProxy::mapping_downcast(mapping).len(vm)), + subscript: atomic_func!(|mapping, needle, vm| { + PyWeakProxy::mapping_downcast(mapping).getitem(needle.to_owned(), vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyWeakProxy::mapping_downcast(mapping); + if let Some(value) = value { + zelf.setitem(needle.to_owned(), value, vm) + } else { + zelf.delitem(needle.to_owned(), vm) + } + }), + }; + &AS_MAPPING + } +} + +impl Representable for PyWeakProxy { + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + zelf.try_upgrade(vm)?.repr(vm) + } + + #[cold] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("use repr instead") + } +} + +pub fn init(context: &'static Context) { + PyWeakProxy::extend_class(context, context.types.weakproxy_type); +} + +impl Hashable for PyWeakProxy { + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + zelf.try_upgrade(vm)?.hash(vm) + } +} diff --git a/crates/vm/src/builtins/weakref.rs b/crates/vm/src/builtins/weakref.rs new file mode 100644 index 00000000000..e1b9545252d --- /dev/null +++ b/crates/vm/src/builtins/weakref.rs @@ -0,0 +1,158 @@ +use super::{PyGenericAlias, PyType, PyTypeRef}; +use crate::common::{ + atomic::{Ordering, Radium}, + hash::{self, PyHash}, +}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + class::PyClassImpl, + function::{FuncArgs, OptionalArg}, + types::{ + Callable, Comparable, Constructor, Hashable, Initializer, PyComparisonOp, Representable, + }, +}; + +pub use crate::object::PyWeak; + +#[derive(FromArgs)] +#[allow(dead_code)] +pub struct WeakNewArgs { + #[pyarg(positional)] + referent: PyObjectRef, + #[pyarg(positional, optional)] + callback: OptionalArg<PyObjectRef>, +} + +impl PyPayload for PyWeak { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.weakref_type + } +} + +impl Callable for PyWeak { + type Args = (); + #[inline] + fn call(zelf: &Py<Self>, _: Self::Args, vm: &VirtualMachine) -> PyResult { + Ok(vm.unwrap_or_none(zelf.upgrade())) + } +} + +impl Constructor for PyWeak { + type Args = WeakNewArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // PyArg_UnpackTuple: only process positional args, ignore kwargs. + // Subclass __init__ will handle extra kwargs. + let mut positional = args.args.into_iter(); + let referent = positional + .next() + .ok_or_else(|| vm.new_type_error("__new__ expected at least 1 argument, got 0"))?; + let callback = positional.next(); + if let Some(_extra) = positional.next() { + return Err(vm.new_type_error(format!( + "__new__ expected at most 2 arguments, got {}", + 3 + positional.count() + ))); + } + let weak = referent.downgrade_with_typ(callback, cls, vm)?; + Ok(weak.into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyWeak { + type Args = WeakNewArgs; + + // weakref_tp_init: accepts args but does nothing (all init done in slot_new) + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + Ok(()) + } +} + +#[pyclass( + with( + Callable, + Hashable, + Comparable, + Constructor, + Initializer, + Representable + ), + flags(BASETYPE) +)] +impl PyWeak { + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } +} + +impl Hashable for PyWeak { + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + let hash = match zelf.hash.load(Ordering::Relaxed) { + hash::SENTINEL => { + let obj = zelf + .upgrade() + .ok_or_else(|| vm.new_type_error("weak object has gone away"))?; + let hash = obj.hash(vm)?; + match Radium::compare_exchange( + &zelf.hash, + hash::SENTINEL, + hash::fix_sentinel(hash), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => hash, + Err(prev_stored) => prev_stored, + } + } + hash => hash, + }; + Ok(hash) + } +} + +impl Comparable for PyWeak { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<crate::function::PyComparisonValue> { + op.eq_only(|| { + let other = class_or_notimplemented!(Self, other); + let both = zelf.upgrade().and_then(|s| other.upgrade().map(|o| (s, o))); + let eq = match both { + Some((a, b)) => vm.bool_eq(&a, &b)?, + None => zelf.is(other), + }; + Ok(eq.into()) + }) + } +} + +impl Representable for PyWeak { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let id = zelf.get_id(); + let repr = if let Some(o) = zelf.upgrade() { + format!( + "<weakref at {:#x}; to '{}' at {:#x}>", + id, + o.class().name(), + o.get_id(), + ) + } else { + format!("<weakref at {id:#x}; dead>") + }; + Ok(repr) + } +} + +pub fn init(context: &'static Context) { + PyWeak::extend_class(context, context.types.weakref_type); +} diff --git a/vm/src/builtins/zip.rs b/crates/vm/src/builtins/zip.rs similarity index 88% rename from vm/src/builtins/zip.rs rename to crates/vm/src/builtins/zip.rs index 56c88f14c61..19c036e7951 100644 --- a/vm/src/builtins/zip.rs +++ b/crates/vm/src/builtins/zip.rs @@ -1,11 +1,11 @@ -use super::{PyType, PyTypeRef}; +use super::PyType; use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyTupleRef, class::PyClassImpl, function::{ArgIntoBool, OptionalArg, PosArgs}, protocol::{PyIter, PyIterReturn}, types::{Constructor, IterNext, Iterable, SelfIter}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; use rustpython_common::atomic::{self, PyAtomic, Radium}; @@ -18,6 +18,7 @@ pub struct PyZip { } impl PyPayload for PyZip { + #[inline] fn class(ctx: &Context) -> &'static Py<PyType> { ctx.types.zip_type } @@ -32,19 +33,21 @@ pub struct PyZipNewArgs { impl Constructor for PyZip { type Args = (PosArgs<PyIter>, PyZipNewArgs); - fn py_new(cls: PyTypeRef, (iterators, args): Self::Args, vm: &VirtualMachine) -> PyResult { + fn py_new( + _cls: &Py<PyType>, + (iterators, args): Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { let iterators = iterators.into_vec(); let strict = Radium::new(args.strict.unwrap_or(false)); - PyZip { iterators, strict } - .into_ref_with_type(vm, cls) - .map(Into::into) + Ok(Self { iterators, strict }) } } #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] impl PyZip { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { let cls = zelf.class().to_owned(); let iterators = zelf .iterators @@ -59,8 +62,8 @@ impl PyZip { }) } - #[pymethod(magic)] - fn setstate(zelf: PyRef<Self>, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + #[pymethod] + fn __setstate__(zelf: PyRef<Self>, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { if let Ok(obj) = ArgIntoBool::try_from_object(vm, state) { zelf.strict.store(obj.into(), atomic::Ordering::Release); } @@ -110,6 +113,6 @@ impl IterNext for PyZip { } } -pub fn init(ctx: &Context) { +pub fn init(ctx: &'static Context) { PyZip::extend_class(ctx, ctx.types.zip_type); } diff --git a/crates/vm/src/byte.rs b/crates/vm/src/byte.rs new file mode 100644 index 00000000000..d9e927cbfa5 --- /dev/null +++ b/crates/vm/src/byte.rs @@ -0,0 +1,25 @@ +//! byte operation APIs +use crate::object::AsObject; +use crate::{PyObject, PyResult, VirtualMachine}; +use num_traits::ToPrimitive; + +pub fn bytes_from_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Vec<u8>> { + if let Ok(elements) = obj.try_bytes_like(vm, |bytes| bytes.to_vec()) { + return Ok(elements); + } + + if !obj.fast_isinstance(vm.ctx.types.str_type) + && let Ok(elements) = vm.map_iterable_object(obj, |x| value_from_object(vm, &x)) + { + return elements; + } + + Err(vm.new_type_error("can assign only bytes, buffers, or iterables of ints in range(0, 256)")) +} + +pub fn value_from_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<u8> { + obj.try_index(vm)? + .as_bigint() + .to_u8() + .ok_or_else(|| vm.new_value_error("byte must be in range(0, 256)")) +} diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs new file mode 100644 index 00000000000..2318415f0fe --- /dev/null +++ b/crates/vm/src/bytes_inner.rs @@ -0,0 +1,1234 @@ +// spell-checker:ignore unchunked +use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, + anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper}, + builtins::{ + PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyInt, PyIntRef, PyStr, PyStrRef, + pystr, pystr::PyUtf8StrRef, + }, + byte::bytes_from_object, + cformat::cformat_bytes, + common::hash, + function::{ArgIterable, Either, OptionalArg, OptionalOption, PyComparisonValue}, + literal::escape::Escape, + protocol::PyBuffer, + sequence::{SequenceExt, SequenceMutExt}, + types::PyComparisonOp, +}; +use bstr::ByteSlice; +use itertools::Itertools; +use malachite_bigint::BigInt; +use num_traits::ToPrimitive; + +const STRING_WITHOUT_ENCODING: &str = "string argument without an encoding"; +const ENCODING_WITHOUT_STRING: &str = "encoding without a string argument"; + +#[derive(Debug, Default, Clone)] +pub struct PyBytesInner { + pub(super) elements: Vec<u8>, +} + +impl From<Vec<u8>> for PyBytesInner { + fn from(elements: Vec<u8>) -> Self { + Self { elements } + } +} + +impl<'a> TryFromBorrowedObject<'a> for PyBytesInner { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + bytes_from_object(vm, obj).map(Self::from) + } +} + +#[derive(FromArgs)] +pub struct ByteInnerNewOptions { + #[pyarg(any, optional)] + pub source: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + pub encoding: OptionalArg<PyUtf8StrRef>, + #[pyarg(any, optional)] + pub errors: OptionalArg<PyUtf8StrRef>, +} + +impl ByteInnerNewOptions { + fn get_value_from_string( + s: PyStrRef, + encoding: PyUtf8StrRef, + errors: OptionalArg<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult<PyBytesInner> { + let bytes = pystr::encode_string(s, Some(encoding), errors.into_option(), vm)?; + Ok(bytes.as_bytes().to_vec().into()) + } + + fn get_value_from_source(source: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyBytesInner> { + bytes_from_object(vm, &source).map(|x| x.into()) + } + + fn get_value_from_size(size: PyIntRef, vm: &VirtualMachine) -> PyResult<PyBytesInner> { + let size = size + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("cannot fit 'int' into an index-sized integer"))?; + let size = if size < 0 { + return Err(vm.new_value_error("negative count")); + } else { + size as usize + }; + Ok(vec![0; size].into()) + } + + fn handle_object_fallback(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyBytesInner> { + match_class!(match obj { + i @ PyInt => { + Self::get_value_from_size(i, vm) + } + _s @ PyStr => Err(vm.new_type_error(STRING_WITHOUT_ENCODING.to_owned())), + obj => { + Self::get_value_from_source(obj, vm) + } + }) + } + + pub fn get_bytearray_inner(self, vm: &VirtualMachine) -> PyResult<PyBytesInner> { + match (self.source, self.encoding, self.errors) { + (OptionalArg::Present(obj), OptionalArg::Missing, OptionalArg::Missing) => { + // Try __index__ first to handle int-like objects that might raise custom exceptions + if let Some(index_result) = obj.try_index_opt(vm) { + match index_result { + Ok(index) => Self::get_value_from_size(index, vm), + Err(e) => { + // Only propagate non-TypeError exceptions + // TypeError means the object doesn't support __index__, so fall back + if e.fast_isinstance(vm.ctx.exceptions.type_error) { + // Fall back to treating as buffer-like object + Self::handle_object_fallback(obj, vm) + } else { + // Propagate other exceptions (e.g., ZeroDivisionError) + Err(e) + } + } + } + } else { + Self::handle_object_fallback(obj, vm) + } + } + (OptionalArg::Present(obj), OptionalArg::Present(encoding), errors) => { + if let Ok(s) = obj.downcast::<PyStr>() { + Self::get_value_from_string(s, encoding, errors, vm) + } else { + Err(vm.new_type_error(ENCODING_WITHOUT_STRING.to_owned())) + } + } + (OptionalArg::Missing, OptionalArg::Missing, OptionalArg::Missing) => { + Ok(PyBytesInner::default()) + } + (OptionalArg::Missing, OptionalArg::Present(_), _) => { + Err(vm.new_type_error(ENCODING_WITHOUT_STRING.to_owned())) + } + (OptionalArg::Missing, _, OptionalArg::Present(_)) => { + Err(vm.new_type_error("errors without a string argument")) + } + (OptionalArg::Present(_), OptionalArg::Missing, OptionalArg::Present(_)) => { + Err(vm.new_type_error(STRING_WITHOUT_ENCODING.to_owned())) + } + } + } +} + +#[derive(FromArgs)] +pub struct ByteInnerFindOptions { + #[pyarg(positional)] + sub: Either<PyBytesInner, PyIntRef>, + #[pyarg(positional, default)] + start: Option<PyIntRef>, + #[pyarg(positional, default)] + end: Option<PyIntRef>, +} + +impl ByteInnerFindOptions { + pub fn get_value( + self, + len: usize, + vm: &VirtualMachine, + ) -> PyResult<(Vec<u8>, core::ops::Range<usize>)> { + let sub = match self.sub { + Either::A(v) => v.elements.to_vec(), + Either::B(int) => vec![int.as_bigint().byte_or(vm)?], + }; + let range = anystr::adjust_indices(self.start, self.end, len); + Ok((sub, range)) + } +} + +#[derive(FromArgs)] +pub struct ByteInnerPaddingOptions { + #[pyarg(positional)] + width: isize, + #[pyarg(positional, optional)] + fillchar: OptionalArg<PyObjectRef>, +} + +impl ByteInnerPaddingOptions { + fn get_value(self, fn_name: &str, vm: &VirtualMachine) -> PyResult<(isize, u8)> { + let fillchar = if let OptionalArg::Present(v) = self.fillchar { + try_as_bytes(v.clone(), |bytes| bytes.iter().copied().exactly_one().ok()) + .flatten() + .ok_or_else(|| { + vm.new_type_error(format!( + "{}() argument 2 must be a byte string of length 1, not {}", + fn_name, + v.class().name() + )) + })? + } else { + b' ' // default is space + }; + + Ok((self.width, fillchar)) + } +} + +#[derive(FromArgs)] +pub struct ByteInnerTranslateOptions { + #[pyarg(positional)] + table: Option<PyObjectRef>, + #[pyarg(any, optional)] + delete: OptionalArg<PyObjectRef>, +} + +impl ByteInnerTranslateOptions { + pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Vec<u8>, Vec<u8>)> { + let table = self.table.map_or_else( + || Ok((0..=u8::MAX).collect::<Vec<u8>>()), + |v| { + let bytes = v + .try_into_value::<PyBytesInner>(vm) + .ok() + .filter(|v| v.elements.len() == 256) + .ok_or_else(|| { + vm.new_value_error("translation table must be 256 characters long") + })?; + Ok(bytes.elements.to_vec()) + }, + )?; + + let delete = match self.delete { + OptionalArg::Present(byte) => { + let byte: PyBytesInner = byte.try_into_value(vm)?; + byte.elements + } + _ => vec![], + }; + + Ok((table, delete)) + } +} + +pub type ByteInnerSplitOptions = anystr::SplitArgs<PyBytesInner>; + +impl PyBytesInner { + #[inline] + pub fn as_bytes(&self) -> &[u8] { + &self.elements + } + + fn new_repr_overflow_error(vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_overflow_error("bytes object is too large to make repr") + } + + pub fn repr_with_name(&self, class_name: &str, vm: &VirtualMachine) -> PyResult<String> { + const DECORATION_LEN: isize = 2 + 3; // 2 for (), 3 for b"" => bytearray(b"") + let escape = crate::literal::escape::AsciiEscape::new_repr(&self.elements); + let len = escape + .layout() + .len + .and_then(|len| (len as isize).checked_add(DECORATION_LEN + class_name.len() as isize)) + .ok_or_else(|| Self::new_repr_overflow_error(vm))? as usize; + let mut buf = String::with_capacity(len); + buf.push_str(class_name); + buf.push('('); + escape.bytes_repr().write(&mut buf).unwrap(); + buf.push(')'); + debug_assert_eq!(buf.len(), len); + Ok(buf) + } + + pub fn repr_bytes(&self, vm: &VirtualMachine) -> PyResult<String> { + let escape = crate::literal::escape::AsciiEscape::new_repr(&self.elements); + let len = 3 + escape + .layout() + .len + .ok_or_else(|| Self::new_repr_overflow_error(vm))?; + let mut buf = String::with_capacity(len); + escape.bytes_repr().write(&mut buf).unwrap(); + debug_assert_eq!(buf.len(), len); + Ok(buf) + } + + #[inline] + pub const fn len(&self) -> usize { + self.elements.len() + } + + #[inline] + pub const fn capacity(&self) -> usize { + self.elements.capacity() + } + + #[inline] + pub const fn is_empty(&self) -> bool { + self.elements.is_empty() + } + + pub fn cmp( + &self, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyComparisonValue { + // TODO: bytes can compare with any object implemented buffer protocol + // but not memoryview, and not equal if compare with unicode str(PyStr) + PyComparisonValue::from_option( + other + .try_bytes_like(vm, |other| op.eval_ord(self.elements.as_slice().cmp(other))) + .ok(), + ) + } + + pub fn hash(&self, vm: &VirtualMachine) -> hash::PyHash { + vm.state.hash_secret.hash_bytes(&self.elements) + } + + pub fn add(&self, other: &[u8]) -> Vec<u8> { + self.elements.py_add(other) + } + + pub fn contains(&self, needle: Either<Self, PyIntRef>, vm: &VirtualMachine) -> PyResult<bool> { + Ok(match needle { + Either::A(byte) => self.elements.contains_str(byte.elements.as_slice()), + Either::B(int) => self.elements.contains(&int.as_bigint().byte_or(vm)?), + }) + } + + pub fn isalnum(&self) -> bool { + !self.elements.is_empty() + && self + .elements + .iter() + .all(|x| char::from(*x).is_alphanumeric()) + } + + pub fn isalpha(&self) -> bool { + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_alphabetic()) + } + + pub fn isascii(&self) -> bool { + self.elements.iter().all(|x| char::from(*x).is_ascii()) + } + + pub fn isdigit(&self) -> bool { + !self.elements.is_empty() + && self + .elements + .iter() + .all(|x| char::from(*x).is_ascii_digit()) + } + + pub fn islower(&self) -> bool { + self.elements.py_islower() + } + + pub fn isupper(&self) -> bool { + self.elements.py_isupper() + } + + pub fn isspace(&self) -> bool { + !self.elements.is_empty() + && self + .elements + .iter() + .all(|x| char::from(*x).is_ascii_whitespace()) + } + + pub fn istitle(&self) -> bool { + if self.elements.is_empty() { + return false; + } + + let mut iter = self.elements.iter().peekable(); + let mut prev_cased = false; + + while let Some(c) = iter.next() { + let current = char::from(*c); + let next = if let Some(k) = iter.peek() { + char::from(**k) + } else if current.is_uppercase() { + return !prev_cased; + } else { + return prev_cased; + }; + + let is_cased = current.to_uppercase().next().unwrap() != current + || current.to_lowercase().next().unwrap() != current; + if (is_cased && next.is_uppercase() && !prev_cased) + || (!is_cased && next.is_lowercase()) + { + return false; + } + + prev_cased = is_cased; + } + + true + } + + pub fn lower(&self) -> Vec<u8> { + self.elements.to_ascii_lowercase() + } + + pub fn upper(&self) -> Vec<u8> { + self.elements.to_ascii_uppercase() + } + + pub fn capitalize(&self) -> Vec<u8> { + let mut new: Vec<u8> = Vec::with_capacity(self.elements.len()); + if let Some((first, second)) = self.elements.split_first() { + new.push(first.to_ascii_uppercase()); + second.iter().for_each(|x| new.push(x.to_ascii_lowercase())); + } + new + } + + pub fn swapcase(&self) -> Vec<u8> { + let mut new: Vec<u8> = Vec::with_capacity(self.elements.len()); + for w in &self.elements { + match w { + b'A'..=b'Z' => new.push(w.to_ascii_lowercase()), + b'a'..=b'z' => new.push(w.to_ascii_uppercase()), + x => new.push(*x), + } + } + new + } + + pub fn hex( + &self, + sep: OptionalArg<Either<PyStrRef, PyBytesRef>>, + bytes_per_sep: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult<String> { + bytes_to_hex(self.elements.as_slice(), sep, bytes_per_sep, vm) + } + + pub fn fromhex(bytes: &[u8], vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut iter = bytes.iter().enumerate(); + let mut result: Vec<u8> = Vec::with_capacity(bytes.len() / 2); + // None means odd number of hex digits, Some(i) means invalid char at position i + let invalid_char: Option<usize> = loop { + let (i, &b) = match iter.next() { + Some(val) => val, + None => { + return Ok(result); + } + }; + + if is_py_ascii_whitespace(b) { + continue; + } + + let top = match b { + b'0'..=b'9' => b - b'0', + b'a'..=b'f' => 10 + b - b'a', + b'A'..=b'F' => 10 + b - b'A', + _ => break Some(i), + }; + + let (i, b) = match iter.next() { + Some(val) => val, + None => break None, // odd number of hex digits + }; + + let bot = match b { + b'0'..=b'9' => b - b'0', + b'a'..=b'f' => 10 + b - b'a', + b'A'..=b'F' => 10 + b - b'A', + _ => break Some(i), + }; + + result.push((top << 4) + bot); + }; + + match invalid_char { + None => Err(vm.new_value_error( + "fromhex() arg must contain an even number of hexadecimal digits", + )), + Some(i) => Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {i}" + ))), + } + } + + /// Parse hex string from str or bytes-like object + pub fn fromhex_object(string: PyObjectRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + if let Some(s) = string.downcast_ref::<PyStr>() { + Self::fromhex(s.as_bytes(), vm) + } else if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &string) { + let borrowed = buffer + .as_contiguous() + .ok_or_else(|| vm.new_buffer_error("fromhex() requires a contiguous buffer"))?; + Self::fromhex(&borrowed, vm) + } else { + Err(vm.new_type_error(format!( + "fromhex() argument must be str or bytes-like, not {}", + string.class().name() + ))) + } + } + + #[inline] + fn _pad( + &self, + options: ByteInnerPaddingOptions, + pad: fn(&[u8], usize, u8, usize) -> Vec<u8>, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + let (width, fillchar) = options.get_value("center", vm)?; + let len = self.len(); + Ok(if len as isize >= width { + Vec::from(&self.elements[..]) + } else { + pad(&self.elements, width as usize, fillchar, len) + }) + } + + pub fn center( + &self, + options: ByteInnerPaddingOptions, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + self._pad(options, AnyStr::py_center, vm) + } + + pub fn ljust( + &self, + options: ByteInnerPaddingOptions, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + self._pad(options, AnyStr::py_ljust, vm) + } + + pub fn rjust( + &self, + options: ByteInnerPaddingOptions, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + self._pad(options, AnyStr::py_rjust, vm) + } + + pub fn count(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult<usize> { + let (needle, range) = options.get_value(self.elements.len(), vm)?; + Ok(self + .elements + .py_count(needle.as_slice(), range, |h, n| h.find_iter(n).count())) + } + + pub fn join(&self, iterable: ArgIterable<Self>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let iter = iterable.iter(vm)?; + self.elements.py_join(iter) + } + + #[inline] + pub fn find<F>( + &self, + options: ByteInnerFindOptions, + find: F, + vm: &VirtualMachine, + ) -> PyResult<Option<usize>> + where + F: Fn(&[u8], &[u8]) -> Option<usize>, + { + let (needle, range) = options.get_value(self.elements.len(), vm)?; + Ok(self.elements.py_find(&needle, range, find)) + } + + pub fn maketrans(from: Self, to: Self, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + if from.len() != to.len() { + return Err(vm.new_value_error("the two maketrans arguments must have equal length")); + } + let mut res = vec![]; + + for i in 0..=u8::MAX { + res.push(if let Some(position) = from.elements.find_byte(i) { + to.elements[position] + } else { + i + }); + } + + Ok(res) + } + + pub fn translate( + &self, + options: ByteInnerTranslateOptions, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + let (table, delete) = options.get_value(vm)?; + + let mut res = if delete.is_empty() { + Vec::with_capacity(self.elements.len()) + } else { + Vec::new() + }; + + for i in &self.elements { + if !delete.contains(i) { + res.push(table[*i as usize]); + } + } + + Ok(res) + } + + pub fn strip(&self, chars: OptionalOption<Self>) -> Vec<u8> { + self.elements + .py_strip( + chars, + |s, chars| s.trim_with(|c| chars.contains(&(c as u8))), + |s| s.trim(), + ) + .to_vec() + } + + pub fn lstrip(&self, chars: OptionalOption<Self>) -> &[u8] { + self.elements.py_strip( + chars, + |s, chars| s.trim_start_with(|c| chars.contains(&(c as u8))), + |s| s.trim_start(), + ) + } + + pub fn rstrip(&self, chars: OptionalOption<Self>) -> &[u8] { + self.elements.py_strip( + chars, + |s, chars| s.trim_end_with(|c| chars.contains(&(c as u8))), + |s| s.trim_end(), + ) + } + + // new in Python 3.9 + pub fn removeprefix(&self, prefix: Self) -> Vec<u8> { + self.elements + .py_removeprefix(&prefix.elements, prefix.elements.len(), |s, p| { + s.starts_with(p) + }) + .to_vec() + } + + // new in Python 3.9 + pub fn removesuffix(&self, suffix: Self) -> Vec<u8> { + self.elements + .py_removesuffix(&suffix.elements, suffix.elements.len(), |s, p| { + s.ends_with(p) + }) + .to_vec() + } + + pub fn split<F>( + &self, + options: ByteInnerSplitOptions, + convert: F, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> + where + F: Fn(&[u8], &VirtualMachine) -> PyObjectRef, + { + let elements = self.elements.py_split( + options, + vm, + || convert(&self.elements, vm), + |v, s, vm| v.split_str(s).map(|v| convert(v, vm)).collect(), + |v, s, n, vm| v.splitn_str(n, s).map(|v| convert(v, vm)).collect(), + |v, n, vm| v.py_split_whitespace(n, |v| convert(v, vm)), + )?; + Ok(elements) + } + + pub fn rsplit<F>( + &self, + options: ByteInnerSplitOptions, + convert: F, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> + where + F: Fn(&[u8], &VirtualMachine) -> PyObjectRef, + { + let mut elements = self.elements.py_split( + options, + vm, + || convert(&self.elements, vm), + |v, s, vm| v.rsplit_str(s).map(|v| convert(v, vm)).collect(), + |v, s, n, vm| v.rsplitn_str(n, s).map(|v| convert(v, vm)).collect(), + |v, n, vm| v.py_rsplit_whitespace(n, |v| convert(v, vm)), + )?; + elements.reverse(); + Ok(elements) + } + + pub fn partition(&self, sub: &Self, vm: &VirtualMachine) -> PyResult<(Vec<u8>, bool, Vec<u8>)> { + self.elements.py_partition( + &sub.elements, + || self.elements.splitn_str(2, &sub.elements), + vm, + ) + } + + pub fn rpartition( + &self, + sub: &Self, + vm: &VirtualMachine, + ) -> PyResult<(Vec<u8>, bool, Vec<u8>)> { + self.elements.py_partition( + &sub.elements, + || self.elements.rsplitn_str(2, &sub.elements), + vm, + ) + } + + pub fn expandtabs(&self, options: anystr::ExpandTabsArgs) -> Vec<u8> { + let tabsize = options.tabsize(); + let mut counter: usize = 0; + let mut res = vec![]; + + if tabsize == 0 { + return self + .elements + .iter() + .copied() + .filter(|x| *x != b'\t') + .collect(); + } + + for i in &self.elements { + if *i == b'\t' { + let len = tabsize - counter % tabsize; + res.extend_from_slice(&vec![b' '; len]); + counter += len; + } else { + res.push(*i); + if *i == b'\r' || *i == b'\n' { + counter = 0; + } else { + counter += 1; + } + } + } + + res + } + + pub fn splitlines<FW, W>(&self, options: anystr::SplitLinesArgs, into_wrapper: FW) -> Vec<W> + where + FW: Fn(&[u8]) -> W, + { + self.elements.py_bytes_splitlines(options, into_wrapper) + } + + pub fn zfill(&self, width: isize) -> Vec<u8> { + self.elements.py_zfill(width) + } + + // len(self)>=1, from="", len(to)>=1, max_count>=1 + fn replace_interleave(&self, to: Self, max_count: Option<usize>) -> Vec<u8> { + let place_count = self.elements.len() + 1; + let count = max_count.map_or(place_count, |v| core::cmp::min(v, place_count)) - 1; + let capacity = self.elements.len() + count * to.len(); + let mut result = Vec::with_capacity(capacity); + let to_slice = to.elements.as_slice(); + result.extend_from_slice(to_slice); + for c in &self.elements[..count] { + result.push(*c); + result.extend_from_slice(to_slice); + } + result.extend_from_slice(&self.elements[count..]); + result + } + + fn replace_delete(&self, from: Self, max_count: Option<usize>) -> Vec<u8> { + let count = count_substring( + self.elements.as_slice(), + from.elements.as_slice(), + max_count, + ); + if count == 0 { + // no matches + return self.elements.clone(); + } + + let result_len = self.len() - (count * from.len()); + debug_assert!(self.len() >= count * from.len()); + + let mut result = Vec::with_capacity(result_len); + let mut last_end = 0; + let mut count = count; + for offset in self.elements.find_iter(&from.elements) { + result.extend_from_slice(&self.elements[last_end..offset]); + last_end = offset + from.len(); + count -= 1; + if count == 0 { + break; + } + } + result.extend_from_slice(&self.elements[last_end..]); + result + } + + pub fn replace_in_place(&self, from: Self, to: Self, max_count: Option<usize>) -> Vec<u8> { + let len = from.len(); + let mut iter = self.elements.find_iter(&from.elements); + + let mut new = if let Some(offset) = iter.next() { + let mut new = self.elements.clone(); + new[offset..offset + len].clone_from_slice(to.elements.as_slice()); + if max_count == Some(1) { + return new; + } else { + new + } + } else { + return self.elements.clone(); + }; + + let mut count = max_count.unwrap_or(usize::MAX) - 1; + for offset in iter { + new[offset..offset + len].clone_from_slice(to.elements.as_slice()); + count -= 1; + if count == 0 { + break; + } + } + new + } + + fn replace_general( + &self, + from: Self, + to: Self, + max_count: Option<usize>, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + let count = count_substring( + self.elements.as_slice(), + from.elements.as_slice(), + max_count, + ); + if count == 0 { + // no matches, return unchanged + return Ok(self.elements.clone()); + } + + // Check for overflow + // result_len = self_len + count * (to_len-from_len) + debug_assert!(count > 0); + if to.len() as isize - from.len() as isize + > (isize::MAX - self.elements.len() as isize) / count as isize + { + return Err(vm.new_overflow_error("replace bytes is too long")); + } + let result_len = (self.elements.len() as isize + + count as isize * (to.len() as isize - from.len() as isize)) + as usize; + + let mut result = Vec::with_capacity(result_len); + let mut last_end = 0; + let mut count = count; + for offset in self.elements.find_iter(&from.elements) { + result.extend_from_slice(&self.elements[last_end..offset]); + result.extend_from_slice(to.elements.as_slice()); + last_end = offset + from.len(); + count -= 1; + if count == 0 { + break; + } + } + result.extend_from_slice(&self.elements[last_end..]); + Ok(result) + } + + pub fn replace( + &self, + from: Self, + to: Self, + max_count: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + // stringlib_replace in CPython + let max_count = match max_count { + OptionalArg::Present(max_count) if max_count >= 0 => { + if max_count == 0 || (self.elements.is_empty() && !from.is_empty()) { + // nothing to do; return the original bytes + return Ok(self.elements.clone()); + } else if self.elements.is_empty() && from.is_empty() { + return Ok(to.elements); + } + Some(max_count as usize) + } + _ => None, + }; + + // Handle zero-length special cases + if from.elements.is_empty() { + if to.elements.is_empty() { + // nothing to do; return the original bytes + return Ok(self.elements.clone()); + } + // insert the 'to' bytes everywhere. + // >>> b"Python".replace(b"", b".") + // b'.P.y.t.h.o.n.' + return Ok(self.replace_interleave(to, max_count)); + } + + // Except for b"".replace(b"", b"A") == b"A" there is no way beyond this + // point for an empty self bytes to generate a non-empty bytes + // Special case so the remaining code always gets a non-empty bytes + if self.elements.is_empty() { + return Ok(self.elements.clone()); + } + + if to.elements.is_empty() { + // delete all occurrences of 'from' bytes + Ok(self.replace_delete(from, max_count)) + } else if from.len() == to.len() { + // Handle special case where both bytes have the same length + Ok(self.replace_in_place(from, to, max_count)) + } else { + // Otherwise use the more generic algorithms + self.replace_general(from, to, max_count, vm) + } + } + + pub fn title(&self) -> Vec<u8> { + let mut res = vec![]; + let mut spaced = true; + + for i in &self.elements { + match i { + b'A'..=b'Z' | b'a'..=b'z' => { + if spaced { + res.push(i.to_ascii_uppercase()); + spaced = false + } else { + res.push(i.to_ascii_lowercase()); + } + } + _ => { + res.push(*i); + spaced = true + } + } + } + + res + } + + pub fn cformat(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + cformat_bytes(vm, self.elements.as_slice(), values) + } + + pub fn mul(&self, n: isize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + self.elements.mul(vm, n) + } + + pub fn imul(&mut self, n: isize, vm: &VirtualMachine) -> PyResult<()> { + self.elements.imul(vm, n) + } + + pub fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let buffer = PyBuffer::try_from_borrowed_object(vm, other)?; + let borrowed = buffer.as_contiguous(); + if let Some(other) = borrowed { + let mut v = Vec::with_capacity(self.elements.len() + other.len()); + v.extend_from_slice(&self.elements); + v.extend_from_slice(&other); + Ok(v) + } else { + let mut v = self.elements.clone(); + buffer.append_to(&mut v); + Ok(v) + } + } +} + +pub fn try_as_bytes<F, R>(obj: PyObjectRef, f: F) -> Option<R> +where + F: Fn(&[u8]) -> R, +{ + match_class!(match obj { + i @ PyBytes => Some(f(i.as_bytes())), + j @ PyByteArray => Some(f(&j.borrow_buf())), + _ => None, + }) +} + +#[inline] +fn count_substring(haystack: &[u8], needle: &[u8], max_count: Option<usize>) -> usize { + let substrings = haystack.find_iter(needle); + if let Some(max_count) = max_count { + core::cmp::min(substrings.take(max_count).count(), max_count) + } else { + substrings.count() + } +} + +pub trait ByteOr: ToPrimitive { + fn byte_or(&self, vm: &VirtualMachine) -> PyResult<u8> { + match self.to_u8() { + Some(value) => Ok(value), + None => Err(vm.new_value_error("byte must be in range(0, 256)")), + } + } +} + +impl ByteOr for BigInt {} + +impl AnyStrWrapper<[u8]> for PyBytesInner { + fn as_ref(&self) -> Option<&[u8]> { + Some(&self.elements) + } + + fn is_empty(&self) -> bool { + self.elements.is_empty() + } +} + +impl AnyStrContainer<[u8]> for Vec<u8> { + fn new() -> Self { + Self::new() + } + + fn with_capacity(capacity: usize) -> Self { + Self::with_capacity(capacity) + } + + fn push_str(&mut self, other: &[u8]) { + self.extend(other) + } +} + +const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; + +impl anystr::AnyChar for u8 { + fn is_lowercase(self) -> bool { + self.is_ascii_lowercase() + } + + fn is_uppercase(self) -> bool { + self.is_ascii_uppercase() + } + + fn bytes_len(self) -> usize { + 1 + } +} + +impl AnyStr for [u8] { + type Char = u8; + type Container = Vec<u8>; + + fn to_container(&self) -> Self::Container { + self.to_vec() + } + + fn as_bytes(&self) -> &[u8] { + self + } + + fn elements(&self) -> impl Iterator<Item = u8> { + self.iter().copied() + } + + fn get_bytes(&self, range: core::ops::Range<usize>) -> &Self { + &self[range] + } + + fn get_chars(&self, range: core::ops::Range<usize>) -> &Self { + &self[range] + } + + fn is_empty(&self) -> bool { + Self::is_empty(self) + } + + fn bytes_len(&self) -> usize { + Self::len(self) + } + + fn py_split_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef, + { + let mut splits = Vec::new(); + let mut count = maxsplit; + let mut haystack = self; + while let Some(offset) = haystack.find_byteset(ASCII_WHITESPACES) { + if offset != 0 { + if count == 0 { + break; + } + splits.push(convert(&haystack[..offset])); + count -= 1; + } + haystack = &haystack[offset + 1..]; + } + if !haystack.is_empty() { + splits.push(convert(haystack)); + } + splits + } + + fn py_rsplit_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> + where + F: Fn(&Self) -> PyObjectRef, + { + let mut splits = Vec::new(); + let mut count = maxsplit; + let mut haystack = self; + while let Some(offset) = haystack.rfind_byteset(ASCII_WHITESPACES) { + if offset + 1 != haystack.len() { + if count == 0 { + break; + } + splits.push(convert(&haystack[offset + 1..])); + count -= 1; + } + haystack = &haystack[..offset]; + } + if !haystack.is_empty() { + splits.push(convert(haystack)); + } + splits + } +} + +#[derive(FromArgs)] +pub struct DecodeArgs { + #[pyarg(any, default)] + encoding: Option<PyUtf8StrRef>, + #[pyarg(any, default)] + errors: Option<PyUtf8StrRef>, +} + +pub fn bytes_decode( + zelf: PyObjectRef, + args: DecodeArgs, + vm: &VirtualMachine, +) -> PyResult<PyStrRef> { + let DecodeArgs { encoding, errors } = args; + let encoding = match encoding.as_ref() { + None => crate::codecs::DEFAULT_ENCODING, + Some(e) => e.as_str(), + }; + vm.state + .codec_registry + .decode_text(zelf, encoding, errors, vm) +} + +fn hex_impl_no_sep(bytes: &[u8]) -> String { + let mut buf: Vec<u8> = vec![0; bytes.len() * 2]; + hex::encode_to_slice(bytes, buf.as_mut_slice()).unwrap(); + unsafe { String::from_utf8_unchecked(buf) } +} + +fn hex_impl(bytes: &[u8], sep: u8, bytes_per_sep: isize) -> String { + let len = bytes.len(); + + let buf = if bytes_per_sep < 0 { + let bytes_per_sep = core::cmp::min(len, (-bytes_per_sep) as usize); + let chunks = (len - 1) / bytes_per_sep; + let chunked = chunks * bytes_per_sep; + let unchunked = len - chunked; + let mut buf = vec![0; len * 2 + chunks]; + let mut j = 0; + for i in (0..chunks).map(|i| i * bytes_per_sep) { + hex::encode_to_slice( + &bytes[i..i + bytes_per_sep], + &mut buf[j..j + bytes_per_sep * 2], + ) + .unwrap(); + j += bytes_per_sep * 2; + buf[j] = sep; + j += 1; + } + hex::encode_to_slice(&bytes[chunked..], &mut buf[j..j + unchunked * 2]).unwrap(); + buf + } else { + let bytes_per_sep = core::cmp::min(len, bytes_per_sep as usize); + let chunks = (len - 1) / bytes_per_sep; + let chunked = chunks * bytes_per_sep; + let unchunked = len - chunked; + let mut buf = vec![0; len * 2 + chunks]; + hex::encode_to_slice(&bytes[..unchunked], &mut buf[..unchunked * 2]).unwrap(); + let mut j = unchunked * 2; + for i in (0..chunks).map(|i| i * bytes_per_sep + unchunked) { + buf[j] = sep; + j += 1; + hex::encode_to_slice( + &bytes[i..i + bytes_per_sep], + &mut buf[j..j + bytes_per_sep * 2], + ) + .unwrap(); + j += bytes_per_sep * 2; + } + buf + }; + + unsafe { String::from_utf8_unchecked(buf) } +} + +pub fn bytes_to_hex( + bytes: &[u8], + sep: OptionalArg<Either<PyStrRef, PyBytesRef>>, + bytes_per_sep: OptionalArg<isize>, + vm: &VirtualMachine, +) -> PyResult<String> { + if bytes.is_empty() { + return Ok("".to_owned()); + } + + if let OptionalArg::Present(sep) = sep { + let bytes_per_sep = bytes_per_sep.unwrap_or(1); + if bytes_per_sep == 0 { + return Ok(hex_impl_no_sep(bytes)); + } + + let s_guard; + let b_guard; + let sep = match &sep { + Either::A(s) => { + s_guard = s.as_wtf8(); + s_guard.as_bytes() + } + Either::B(bytes) => { + b_guard = bytes.as_bytes(); + b_guard + } + }; + + if sep.len() != 1 { + return Err(vm.new_value_error("sep must be length 1.")); + } + let sep = sep[0]; + if sep > 127 { + return Err(vm.new_value_error("sep must be ASCII.")); + } + + Ok(hex_impl(bytes, sep, bytes_per_sep)) + } else { + Ok(hex_impl_no_sep(bytes)) + } +} + +pub const fn is_py_ascii_whitespace(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') +} diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs new file mode 100644 index 00000000000..939b1c7760f --- /dev/null +++ b/crates/vm/src/cformat.rs @@ -0,0 +1,470 @@ +//cspell:ignore bytesobject + +//! Implementation of Printf-Style string formatting +//! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). + +use crate::common::cformat::*; +use crate::common::wtf8::{CodePoint, Wtf8, Wtf8Buf}; +use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, + VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyByteArray, PyBytes, PyFloat, PyInt, PyStr, try_f64_to_bigint, tuple, + }, + function::ArgIntoFloat, + protocol::PyBuffer, + stdlib::builtins, +}; +use itertools::Itertools; +use num_traits::cast::ToPrimitive; + +fn spec_format_bytes( + vm: &VirtualMachine, + spec: &CFormatSpec, + obj: PyObjectRef, +) -> PyResult<Vec<u8>> { + match &spec.format_type { + CFormatType::String(conversion) => match conversion { + // Unlike strings, %r and %a are identical for bytes: the behaviour corresponds to + // %a for strings (not %r) + CFormatConversion::Repr | CFormatConversion::Ascii => { + let b = builtins::ascii(obj, vm)?.into(); + Ok(b) + } + CFormatConversion::Str | CFormatConversion::Bytes => { + if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &obj) { + Ok(buffer.contiguous_or_collect(|bytes| spec.format_bytes(bytes))) + } else { + let bytes = vm + .get_special_method(&obj, identifier!(vm, __bytes__))? + .ok_or_else(|| { + vm.new_type_error(format!( + "%b requires a bytes-like object, or an object that \ + implements __bytes__, not '{}'", + obj.class().name() + )) + })? + .invoke((), vm)?; + let bytes = PyBytes::try_from_borrowed_object(vm, &bytes)?; + Ok(spec.format_bytes(bytes.as_bytes())) + } + } + }, + CFormatType::Number(number_type) => match number_type { + CNumberType::DecimalD | CNumberType::DecimalI | CNumberType::DecimalU => { + match_class!(match &obj { + ref i @ PyInt => { + Ok(spec.format_number(i.as_bigint()).into_bytes()) + } + ref f @ PyFloat => { + Ok(spec + .format_number(&try_f64_to_bigint(f.to_f64(), vm)?) + .into_bytes()) + } + obj => { + if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { + let result = method?.call((), vm)?; + if let Some(i) = result.downcast_ref::<PyInt>() { + return Ok(spec.format_number(i.as_bigint()).into_bytes()); + } + } + Err(vm.new_type_error(format!( + "%{} format: a number is required, not {}", + spec.format_type.to_char(), + obj.class().name() + ))) + } + }) + } + _ => { + if let Some(i) = obj.downcast_ref::<PyInt>() { + Ok(spec.format_number(i.as_bigint()).into_bytes()) + } else { + Err(vm.new_type_error(format!( + "%{} format: an integer is required, not {}", + spec.format_type.to_char(), + obj.class().name() + ))) + } + } + }, + CFormatType::Float(_) => { + let class = obj.class().to_owned(); + let value = ArgIntoFloat::try_from_object(vm, obj).map_err(|e| { + if e.fast_isinstance(vm.ctx.exceptions.type_error) { + // formatfloat in bytesobject.c generates its own specific exception + // text in this case, mirror it here. + vm.new_type_error(format!("float argument required, not {}", class.name())) + } else { + e + } + })?; + Ok(spec.format_float(value.into()).into_bytes()) + } + CFormatType::Character(CCharacterType::Character) => { + if let Some(i) = obj.downcast_ref::<PyInt>() { + let ch = i + .try_to_primitive::<u8>(vm) + .map_err(|_| vm.new_overflow_error("%c arg not in range(256)"))?; + return Ok(spec.format_char(ch)); + } + if let Some(b) = obj.downcast_ref::<PyBytes>() { + if b.len() == 1 { + return Ok(spec.format_char(b.as_bytes()[0])); + } + } else if let Some(ba) = obj.downcast_ref::<PyByteArray>() { + let buf = ba.borrow_buf(); + if buf.len() == 1 { + return Ok(spec.format_char(buf[0])); + } + } + Err(vm.new_type_error("%c requires an integer in range(256) or a single byte")) + } + } +} + +fn spec_format_string( + vm: &VirtualMachine, + spec: &CFormatSpec, + obj: PyObjectRef, + idx: usize, +) -> PyResult<Wtf8Buf> { + match &spec.format_type { + CFormatType::String(conversion) => { + let result = match conversion { + CFormatConversion::Ascii => builtins::ascii(obj, vm)?.into(), + CFormatConversion::Str => obj.str(vm)?.as_wtf8().to_owned(), + CFormatConversion::Repr => obj.repr(vm)?.as_wtf8().to_owned(), + CFormatConversion::Bytes => { + // idx is the position of the %, we want the position of the b + return Err(vm.new_value_error(format!( + "unsupported format character 'b' (0x62) at index {}", + idx + 1 + ))); + } + }; + Ok(spec.format_string(result)) + } + CFormatType::Number(number_type) => match number_type { + CNumberType::DecimalD | CNumberType::DecimalI | CNumberType::DecimalU => { + match_class!(match &obj { + ref i @ PyInt => { + Ok(spec.format_number(i.as_bigint()).into()) + } + ref f @ PyFloat => { + Ok(spec + .format_number(&try_f64_to_bigint(f.to_f64(), vm)?) + .into()) + } + obj => { + if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { + let result = method?.call((), vm)?; + if let Some(i) = result.downcast_ref::<PyInt>() { + return Ok(spec.format_number(i.as_bigint()).into()); + } + } + Err(vm.new_type_error(format!( + "%{} format: a number is required, not {}", + spec.format_type.to_char(), + obj.class().name() + ))) + } + }) + } + _ => { + if let Some(i) = obj.downcast_ref::<PyInt>() { + Ok(spec.format_number(i.as_bigint()).into()) + } else { + Err(vm.new_type_error(format!( + "%{} format: an integer is required, not {}", + spec.format_type.to_char(), + obj.class().name() + ))) + } + } + }, + CFormatType::Float(_) => { + let value = ArgIntoFloat::try_from_object(vm, obj)?; + Ok(spec.format_float(value.into()).into()) + } + CFormatType::Character(CCharacterType::Character) => { + if let Some(i) = obj.downcast_ref::<PyInt>() { + let ch = i + .as_bigint() + .to_u32() + .and_then(CodePoint::from_u32) + .ok_or_else(|| vm.new_overflow_error("%c arg not in range(0x110000)"))?; + return Ok(spec.format_char(ch)); + } + if let Some(s) = obj.downcast_ref::<PyStr>() + && let Ok(ch) = s.as_wtf8().code_points().exactly_one() + { + return Ok(spec.format_char(ch)); + } + Err(vm.new_type_error("%c requires int or char")) + } + } +} + +fn try_update_quantity_from_element( + vm: &VirtualMachine, + element: Option<&PyObject>, +) -> PyResult<CFormatQuantity> { + match element { + Some(width_obj) => { + if let Some(i) = width_obj.downcast_ref::<PyInt>() { + let i = i.try_to_primitive::<i32>(vm)?.unsigned_abs(); + Ok(CFormatQuantity::Amount(i as usize)) + } else { + Err(vm.new_type_error("* wants int")) + } + } + None => Err(vm.new_type_error("not enough arguments for format string")), + } +} + +fn try_conversion_flag_from_tuple( + vm: &VirtualMachine, + element: Option<&PyObject>, +) -> PyResult<CConversionFlags> { + match element { + Some(width_obj) => { + if let Some(i) = width_obj.downcast_ref::<PyInt>() { + let i = i.try_to_primitive::<i32>(vm)?; + let flags = if i < 0 { + CConversionFlags::LEFT_ADJUST + } else { + CConversionFlags::from_bits(0).unwrap() + }; + Ok(flags) + } else { + Err(vm.new_type_error("* wants int")) + } + } + None => Err(vm.new_type_error("not enough arguments for format string")), + } +} + +fn try_update_quantity_from_tuple<'a, I: Iterator<Item = &'a PyObjectRef>>( + vm: &VirtualMachine, + elements: &mut I, + q: &mut Option<CFormatQuantity>, + f: &mut CConversionFlags, +) -> PyResult<()> { + let Some(CFormatQuantity::FromValuesTuple) = q else { + return Ok(()); + }; + let element = elements.next(); + f.insert(try_conversion_flag_from_tuple( + vm, + element.map(|v| v.as_ref()), + )?); + let quantity = try_update_quantity_from_element(vm, element.map(|v| v.as_ref()))?; + *q = Some(quantity); + Ok(()) +} + +fn try_update_precision_from_tuple<'a, I: Iterator<Item = &'a PyObjectRef>>( + vm: &VirtualMachine, + elements: &mut I, + p: &mut Option<CFormatPrecision>, +) -> PyResult<()> { + let Some(CFormatPrecision::Quantity(CFormatQuantity::FromValuesTuple)) = p else { + return Ok(()); + }; + let quantity = try_update_quantity_from_element(vm, elements.next().map(|v| v.as_ref()))?; + *p = Some(CFormatPrecision::Quantity(quantity)); + Ok(()) +} + +fn specifier_error(vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_type_error("format requires a mapping") +} + +pub(crate) fn cformat_bytes( + vm: &VirtualMachine, + format_string: &[u8], + values_obj: PyObjectRef, +) -> PyResult<Vec<u8>> { + let mut format = CFormatBytes::parse_from_bytes(format_string) + .map_err(|err| vm.new_value_error(err.to_string()))?; + let (num_specifiers, mapping_required) = format + .check_specifiers() + .ok_or_else(|| specifier_error(vm))?; + + let mut result = vec![]; + + let is_mapping = values_obj.class().has_attr(identifier!(vm, __getitem__)) + && !values_obj.fast_isinstance(vm.ctx.types.tuple_type) + && !values_obj.fast_isinstance(vm.ctx.types.bytes_type) + && !values_obj.fast_isinstance(vm.ctx.types.bytearray_type); + + if num_specifiers == 0 { + // literal only + return if is_mapping + || values_obj + .downcast_ref::<tuple::PyTuple>() + .is_some_and(|e| e.is_empty()) + { + for (_, part) in format.iter_mut() { + match part { + CFormatPart::Literal(literal) => result.append(literal), + CFormatPart::Spec(_) => unreachable!(), + } + } + Ok(result) + } else { + Err(vm.new_type_error("not all arguments converted during bytes formatting")) + }; + } + + if mapping_required { + // dict + return if is_mapping { + for (_, part) in format { + match part { + CFormatPart::Literal(literal) => result.extend(literal), + CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => { + let key = mapping_key.unwrap(); + let value = values_obj.get_item(&key, vm)?; + let part_result = spec_format_bytes(vm, &spec, value)?; + result.extend(part_result); + } + } + } + Ok(result) + } else { + Err(vm.new_type_error("format requires a mapping")) + }; + } + + // tuple + let values = if let Some(tup) = values_obj.downcast_ref::<tuple::PyTuple>() { + tup.as_slice() + } else { + core::slice::from_ref(&values_obj) + }; + let mut value_iter = values.iter(); + + for (_, part) in format { + match part { + CFormatPart::Literal(literal) => result.extend(literal), + CFormatPart::Spec(CFormatSpecKeyed { mut spec, .. }) => { + try_update_quantity_from_tuple( + vm, + &mut value_iter, + &mut spec.min_field_width, + &mut spec.flags, + )?; + try_update_precision_from_tuple(vm, &mut value_iter, &mut spec.precision)?; + + let value = match value_iter.next() { + Some(obj) => Ok(obj.clone()), + None => Err(vm.new_type_error("not enough arguments for format string")), + }?; + let part_result = spec_format_bytes(vm, &spec, value)?; + result.extend(part_result); + } + } + } + + // check that all arguments were converted + if value_iter.next().is_some() && !is_mapping { + Err(vm.new_type_error("not all arguments converted during bytes formatting")) + } else { + Ok(result) + } +} + +pub(crate) fn cformat_string( + vm: &VirtualMachine, + format_string: &Wtf8, + values_obj: PyObjectRef, +) -> PyResult<Wtf8Buf> { + let format = CFormatWtf8::parse_from_wtf8(format_string) + .map_err(|err| vm.new_value_error(err.to_string()))?; + let (num_specifiers, mapping_required) = format + .check_specifiers() + .ok_or_else(|| specifier_error(vm))?; + + let mut result = Wtf8Buf::new(); + + let is_mapping = values_obj.class().has_attr(identifier!(vm, __getitem__)) + && !values_obj.fast_isinstance(vm.ctx.types.tuple_type) + && !values_obj.fast_isinstance(vm.ctx.types.str_type); + + if num_specifiers == 0 { + // literal only + return if is_mapping + || values_obj + .downcast_ref::<tuple::PyTuple>() + .is_some_and(|e| e.is_empty()) + { + for (_, part) in format.iter() { + match part { + CFormatPart::Literal(literal) => result.push_wtf8(literal), + CFormatPart::Spec(_) => unreachable!(), + } + } + Ok(result) + } else { + Err(vm.new_type_error("not all arguments converted during string formatting")) + }; + } + + if mapping_required { + // dict + return if is_mapping { + for (idx, part) in format { + match part { + CFormatPart::Literal(literal) => result.push_wtf8(&literal), + CFormatPart::Spec(CFormatSpecKeyed { mapping_key, spec }) => { + let value = values_obj.get_item(&mapping_key.unwrap(), vm)?; + let part_result = spec_format_string(vm, &spec, value, idx)?; + result.push_wtf8(&part_result); + } + } + } + Ok(result) + } else { + Err(vm.new_type_error("format requires a mapping")) + }; + } + + // tuple + let values = if let Some(tup) = values_obj.downcast_ref::<tuple::PyTuple>() { + tup.as_slice() + } else { + core::slice::from_ref(&values_obj) + }; + let mut value_iter = values.iter(); + + for (idx, part) in format { + match part { + CFormatPart::Literal(literal) => result.push_wtf8(&literal), + CFormatPart::Spec(CFormatSpecKeyed { mut spec, .. }) => { + try_update_quantity_from_tuple( + vm, + &mut value_iter, + &mut spec.min_field_width, + &mut spec.flags, + )?; + try_update_precision_from_tuple(vm, &mut value_iter, &mut spec.precision)?; + + let value = match value_iter.next() { + Some(obj) => Ok(obj.clone()), + None => Err(vm.new_type_error("not enough arguments for format string")), + }?; + let part_result = spec_format_string(vm, &spec, value, idx)?; + result.push_wtf8(&part_result); + } + } + } + + // check that all arguments were converted + if value_iter.next().is_some() && !is_mapping { + Err(vm.new_type_error("not all arguments converted during string formatting")) + } else { + Ok(result) + } +} diff --git a/crates/vm/src/class.rs b/crates/vm/src/class.rs new file mode 100644 index 00000000000..237e11abbe5 --- /dev/null +++ b/crates/vm/src/class.rs @@ -0,0 +1,263 @@ +//! Utilities to define a new Python class + +use crate::{ + PyPayload, + builtins::{PyBaseObject, PyType, PyTypeRef, descriptor::PyWrapper}, + function::PyMethodDef, + object::Py, + types::{PyTypeFlags, PyTypeSlots, SLOT_DEFS, hash_not_implemented}, + vm::Context, +}; +use rustpython_common::static_cell; + +/// Add slot wrapper descriptors to a type's dict +/// +/// Iterates SLOT_DEFS and creates a PyWrapper for each slot that: +/// 1. Has a function set in the type's slots +/// 2. Doesn't already have an attribute in the type's dict +pub fn add_operators(class: &'static Py<PyType>, ctx: &Context) { + for def in SLOT_DEFS.iter() { + // Skip __new__ - it has special handling + if def.name == "__new__" { + continue; + } + + // Special handling for __hash__ = None + if def.name == "__hash__" + && class + .slots + .hash + .load() + .is_some_and(|h| h as usize == hash_not_implemented as *const () as usize) + { + class.set_attr(ctx.names.__hash__, ctx.none.clone().into()); + continue; + } + + // __getattr__ should only have a wrapper if the type explicitly defines it. + // Unlike __getattribute__, __getattr__ is not present on object by default. + // Both map to TpGetattro, but only __getattribute__ gets a wrapper from the slot. + if def.name == "__getattr__" { + continue; + } + + // Get the slot function wrapped in SlotFunc + let Some(slot_func) = def.accessor.get_slot_func_with_op(&class.slots, def.op) else { + continue; + }; + + // Check if attribute already exists in dict + let attr_name = ctx.intern_str(def.name); + if class.attributes.read().contains_key(attr_name) { + continue; + } + + // Create and add the wrapper + let wrapper = PyWrapper { + typ: class, + name: attr_name, + wrapped: slot_func, + doc: Some(def.doc), + }; + class.set_attr(attr_name, wrapper.into_ref(ctx).into()); + } +} + +pub trait StaticType { + // Ideally, saving PyType is better than PyTypeRef + fn static_cell() -> &'static static_cell::StaticCell<PyTypeRef>; + #[inline] + fn static_metaclass() -> &'static Py<PyType> { + PyType::static_type() + } + #[inline] + fn static_baseclass() -> &'static Py<PyType> { + PyBaseObject::static_type() + } + #[inline] + fn static_type() -> &'static Py<PyType> { + #[cold] + fn fail() -> ! { + panic!( + "static type has not been initialized. e.g. the native types defined in different module may be used before importing library." + ); + } + Self::static_cell().get().unwrap_or_else(|| fail()) + } + fn init_manually(typ: PyTypeRef) -> &'static Py<PyType> { + let cell = Self::static_cell(); + cell.set(typ) + .unwrap_or_else(|_| panic!("double initialization from init_manually")); + cell.get().unwrap() + } + fn init_builtin_type() -> &'static Py<PyType> + where + Self: PyClassImpl, + { + let typ = Self::create_static_type(); + let cell = Self::static_cell(); + cell.set(typ) + .unwrap_or_else(|_| panic!("double initialization of {}", Self::NAME)); + cell.get().unwrap() + } + fn create_static_type() -> PyTypeRef + where + Self: PyClassImpl, + { + PyType::new_static( + Self::static_baseclass().to_owned(), + Default::default(), + Self::make_slots(), + Self::static_metaclass().to_owned(), + ) + .unwrap() + } +} + +pub trait PyClassDef { + const NAME: &'static str; + const MODULE_NAME: Option<&'static str>; + const TP_NAME: &'static str; + const DOC: Option<&'static str> = None; + const BASICSIZE: usize; + const ITEMSIZE: usize = 0; + const UNHASHABLE: bool = false; + + // due to restriction of rust trait system, object.__base__ is None + // but PyBaseObject::Base will be PyBaseObject. + type Base: PyClassDef; +} + +pub trait PyClassImpl: PyClassDef { + const TP_FLAGS: PyTypeFlags = PyTypeFlags::DEFAULT; + + fn extend_class(ctx: &'static Context, class: &'static Py<PyType>) + where + Self: Sized, + { + #[cfg(debug_assertions)] + { + assert!(class.slots.flags.is_created_with_flags()); + } + + let _ = ctx.intern_str(Self::NAME); // intern type name + + if Self::TP_FLAGS.has_feature(PyTypeFlags::HAS_DICT) { + let __dict__ = identifier!(ctx, __dict__); + class.set_attr( + __dict__, + ctx.new_static_getset( + "__dict__", + class, + crate::builtins::object::object_get_dict, + crate::builtins::object::object_set_dict, + ) + .into(), + ); + } + Self::impl_extend_class(ctx, class); + if let Some(doc) = Self::DOC { + // Only set __doc__ if it doesn't already exist (e.g., as a member descriptor) + // This matches CPython's behavior in type_dict_set_doc + let doc_attr_name = identifier!(ctx, __doc__); + if class.attributes.read().get(doc_attr_name).is_none() { + class.set_attr(doc_attr_name, ctx.new_str(doc).into()); + } + } + if let Some(module_name) = Self::MODULE_NAME { + let module_key = identifier!(ctx, __module__); + // Don't overwrite a getset descriptor for __module__ (e.g. TypeAliasType + // has an instance-level __module__ getset that should not be replaced) + let has_getset = class + .attributes + .read() + .get(module_key) + .is_some_and(|v| v.downcastable::<crate::builtins::PyGetSet>()); + if !has_getset { + class.set_attr(module_key, ctx.new_str(module_name).into()); + } + } + + // Don't add __new__ attribute if slot_new is inherited from object + // (Python doesn't add __new__ to __dict__ for inherited slots) + // Exception: object itself should have __new__ in its dict + if let Some(slot_new) = class.slots.new.load() { + let object_new = ctx.types.object_type.slots.new.load(); + let is_object_itself = core::ptr::eq(class, ctx.types.object_type); + let is_inherited_from_object = !is_object_itself + && object_new.is_some_and(|obj_new| slot_new as usize == obj_new as usize); + + if !is_inherited_from_object { + let bound_new = + ctx.slot_new_wrapper + .build_bound_method(ctx, class.to_owned().into(), class); + class.set_attr(identifier!(ctx, __new__), bound_new.into()); + } + } + + // Add slot wrappers using SLOT_DEFS array + add_operators(class, ctx); + + // Inherit slots from base types after slots are fully initialized + for base in class.bases.read().iter() { + class.inherit_slots(base); + } + + class.extend_methods(class.slots.methods, ctx); + } + + fn make_static_type() -> PyTypeRef + where + Self: StaticType + Sized, + { + (*Self::static_cell().get_or_init(|| { + let typ = Self::create_static_type(); + Self::extend_class(Context::genesis(), unsafe { + // typ will be saved in static_cell + let r: &Py<PyType> = &typ; + let r: &'static Py<PyType> = core::mem::transmute(r); + r + }); + typ + })) + .to_owned() + } + + fn impl_extend_class(ctx: &'static Context, class: &'static Py<PyType>); + const METHOD_DEFS: &'static [PyMethodDef]; + fn extend_slots(slots: &mut PyTypeSlots); + + fn make_slots() -> PyTypeSlots { + let mut slots = PyTypeSlots { + flags: Self::TP_FLAGS, + name: Self::TP_NAME, + basicsize: Self::BASICSIZE, + itemsize: Self::ITEMSIZE, + doc: Self::DOC, + methods: Self::METHOD_DEFS, + ..Default::default() + }; + + if Self::UNHASHABLE { + slots.hash.store(Some(hash_not_implemented)); + } + + Self::extend_slots(&mut slots); + slots + } +} + +/// Trait for Python subclasses that can provide a reference to their base type. +/// +/// This trait is automatically implemented by the `#[pyclass]` macro when +/// `base = SomeType` is specified. It provides safe reference access to the +/// base type's payload. +/// +/// For subclasses with `#[repr(transparent)]` +/// which enables ownership transfer via `into_base()`. +pub trait PySubclass: crate::PyPayload { + type Base: crate::PyPayload; + + /// Returns a reference to the base type's payload. + fn as_base(&self) -> &Self::Base; +} diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs new file mode 100644 index 00000000000..87c3a4a0a9d --- /dev/null +++ b/crates/vm/src/codecs.rs @@ -0,0 +1,1232 @@ +use rustpython_common::{ + borrow::BorrowedValue, + encodings::{ + CodecContext, DecodeContext, DecodeErrorHandler, EncodeContext, EncodeErrorHandler, + EncodeReplace, StrBuffer, StrSize, errors, + }, + str::StrKind, + wtf8::{CodePoint, Wtf8, Wtf8Buf}, +}; + +use crate::common::lock::OnceCell; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, + VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyBytes, PyBytesRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyUtf8Str, + PyUtf8StrRef, + }, + common::{ascii, lock::PyRwLock}, + convert::ToPyObject, + function::{ArgBytesLike, PyMethodDef}, +}; +use alloc::borrow::Cow; +use core::ops::{self, Range}; +use std::collections::HashMap; + +pub struct CodecsRegistry { + inner: PyRwLock<RegistryInner>, +} + +struct RegistryInner { + search_path: Vec<PyObjectRef>, + search_cache: HashMap<String, PyCodec>, + errors: HashMap<String, PyObjectRef>, +} + +pub const DEFAULT_ENCODING: &str = "utf-8"; + +#[derive(Clone)] +#[repr(transparent)] +pub struct PyCodec(PyTupleRef); +impl PyCodec { + #[inline] + pub fn from_tuple(tuple: PyTupleRef) -> Result<Self, PyTupleRef> { + if tuple.len() == 4 { + Ok(Self(tuple)) + } else { + Err(tuple) + } + } + #[inline] + pub fn into_tuple(self) -> PyTupleRef { + self.0 + } + #[inline] + pub fn as_tuple(&self) -> &Py<PyTuple> { + &self.0 + } + + #[inline] + pub fn get_encode_func(&self) -> &PyObject { + &self.0[0] + } + #[inline] + pub fn get_decode_func(&self) -> &PyObject { + &self.0[1] + } + + pub fn is_text_codec(&self, vm: &VirtualMachine) -> PyResult<bool> { + let is_text = vm.get_attribute_opt(self.0.clone().into(), "_is_text_encoding")?; + is_text.map_or(Ok(true), |is_text| is_text.try_to_bool(vm)) + } + + pub fn encode( + &self, + obj: PyObjectRef, + errors: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult { + let args = match errors { + Some(errors) => vec![obj, errors.into_wtf8().into()], + None => vec![obj], + }; + let res = self.get_encode_func().call(args, vm)?; + let res = res + .downcast::<PyTuple>() + .ok() + .filter(|tuple| tuple.len() == 2) + .ok_or_else(|| vm.new_type_error("encoder must return a tuple (object, integer)"))?; + // we don't actually care about the integer + Ok(res[0].clone()) + } + + pub fn decode( + &self, + obj: PyObjectRef, + errors: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult { + let args = match errors { + Some(errors) => vec![obj, errors.into_wtf8().into()], + None => vec![obj], + }; + let res = self.get_decode_func().call(args, vm)?; + let res = res + .downcast::<PyTuple>() + .ok() + .filter(|tuple| tuple.len() == 2) + .ok_or_else(|| vm.new_type_error("decoder must return a tuple (object,integer)"))?; + // we don't actually care about the integer + Ok(res[0].clone()) + } + + pub fn get_incremental_encoder( + &self, + errors: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult { + let args = match errors { + Some(e) => vec![e.into()], + None => vec![], + }; + vm.call_method(self.0.as_object(), "incrementalencoder", args) + } + + pub fn get_incremental_decoder( + &self, + errors: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult { + let args = match errors { + Some(e) => vec![e.into()], + None => vec![], + }; + vm.call_method(self.0.as_object(), "incrementaldecoder", args) + } +} + +impl TryFromObject for PyCodec { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + obj.downcast::<PyTuple>() + .ok() + .and_then(|tuple| Self::from_tuple(tuple).ok()) + .ok_or_else(|| vm.new_type_error("codec search functions must return 4-tuples")) + } +} + +impl ToPyObject for PyCodec { + #[inline] + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.0.into() + } +} + +impl CodecsRegistry { + /// Reset the inner RwLock to unlocked state after fork(). + /// + /// # Safety + /// Must only be called after fork() in the child process when no other + /// threads exist. + #[cfg(all(unix, feature = "threading"))] + pub(crate) unsafe fn reinit_after_fork(&self) { + unsafe { crate::common::lock::reinit_rwlock_after_fork(&self.inner) }; + } + + pub(crate) fn new(ctx: &Context) -> Self { + ::rustpython_vm::common::static_cell! { + static METHODS: Box<[PyMethodDef]>; + } + + let methods = METHODS.get_or_init(|| { + crate::define_methods![ + "strict_errors" => strict_errors as EMPTY, + "ignore_errors" => ignore_errors as EMPTY, + "replace_errors" => replace_errors as EMPTY, + "xmlcharrefreplace_errors" => xmlcharrefreplace_errors as EMPTY, + "backslashreplace_errors" => backslashreplace_errors as EMPTY, + "namereplace_errors" => namereplace_errors as EMPTY, + "surrogatepass_errors" => surrogatepass_errors as EMPTY, + "surrogateescape_errors" => surrogateescape_errors as EMPTY + ] + .into_boxed_slice() + }); + + let errors = [ + ("strict", methods[0].build_function(ctx)), + ("ignore", methods[1].build_function(ctx)), + ("replace", methods[2].build_function(ctx)), + ("xmlcharrefreplace", methods[3].build_function(ctx)), + ("backslashreplace", methods[4].build_function(ctx)), + ("namereplace", methods[5].build_function(ctx)), + ("surrogatepass", methods[6].build_function(ctx)), + ("surrogateescape", methods[7].build_function(ctx)), + ]; + let errors = errors + .into_iter() + .map(|(name, f)| (name.to_owned(), f.into())) + .collect(); + let inner = RegistryInner { + search_path: Vec::new(), + search_cache: HashMap::new(), + errors, + }; + Self { + inner: PyRwLock::new(inner), + } + } + + pub fn register(&self, search_function: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if !search_function.is_callable() { + return Err(vm.new_type_error("argument must be callable")); + } + self.inner.write().search_path.push(search_function); + Ok(()) + } + + pub fn unregister(&self, search_function: PyObjectRef) -> PyResult<()> { + let mut inner = self.inner.write(); + // Do nothing if search_path is not created yet or was cleared. + if inner.search_path.is_empty() { + return Ok(()); + } + for (i, item) in inner.search_path.iter().enumerate() { + if item.get_id() == search_function.get_id() { + if !inner.search_cache.is_empty() { + inner.search_cache.clear(); + } + inner.search_path.remove(i); + return Ok(()); + } + } + Ok(()) + } + + pub(crate) fn register_manual(&self, name: &str, codec: PyCodec) -> PyResult<()> { + let name = normalize_encoding_name(name); + self.inner + .write() + .search_cache + .insert(name.into_owned(), codec); + Ok(()) + } + + pub fn lookup(&self, encoding: &str, vm: &VirtualMachine) -> PyResult<PyCodec> { + let encoding = normalize_encoding_name(encoding); + let search_path = { + let inner = self.inner.read(); + if let Some(codec) = inner.search_cache.get(encoding.as_ref()) { + // hit cache + return Ok(codec.clone()); + } + inner.search_path.clone() + }; + let encoding: PyUtf8StrRef = vm.ctx.new_utf8_str(encoding.as_ref()); + for func in search_path { + let res = func.call((encoding.clone(),), vm)?; + let res: Option<PyCodec> = res.try_into_value(vm)?; + if let Some(codec) = res { + let mut inner = self.inner.write(); + // someone might have raced us to this, so use theirs + let codec = inner + .search_cache + .entry(encoding.as_str().to_owned()) + .or_insert(codec); + return Ok(codec.clone()); + } + } + Err(vm.new_lookup_error(format!("unknown encoding: {encoding}"))) + } + + fn _lookup_text_encoding( + &self, + encoding: &str, + generic_func: &str, + vm: &VirtualMachine, + ) -> PyResult<PyCodec> { + let codec = self.lookup(encoding, vm)?; + if codec.is_text_codec(vm)? { + Ok(codec) + } else { + Err(vm.new_lookup_error(format!( + "'{encoding}' is not a text encoding; use {generic_func} to handle arbitrary codecs" + ))) + } + } + + pub fn forget(&self, encoding: &str) -> Option<PyCodec> { + let encoding = normalize_encoding_name(encoding); + self.inner.write().search_cache.remove(encoding.as_ref()) + } + + pub fn encode( + &self, + obj: PyObjectRef, + encoding: &str, + errors: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult { + let codec = self.lookup(encoding, vm)?; + codec.encode(obj, errors, vm).inspect_err(|exc| { + Self::add_codec_note(exc, "encoding", encoding, vm); + }) + } + + pub fn decode( + &self, + obj: PyObjectRef, + encoding: &str, + errors: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult { + let codec = self.lookup(encoding, vm)?; + codec.decode(obj, errors, vm).inspect_err(|exc| { + Self::add_codec_note(exc, "decoding", encoding, vm); + }) + } + + pub fn encode_text( + &self, + obj: PyStrRef, + encoding: &str, + errors: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult<PyBytesRef> { + let codec = self._lookup_text_encoding(encoding, "codecs.encode()", vm)?; + codec + .encode(obj.into(), errors, vm) + .inspect_err(|exc| { + Self::add_codec_note(exc, "encoding", encoding, vm); + })? + .downcast() + .map_err(|obj| { + vm.new_type_error(format!( + "'{}' encoder returned '{}' instead of 'bytes'; use codecs.encode() to \ + encode to arbitrary types", + encoding, + obj.class().name(), + )) + }) + } + + pub fn decode_text( + &self, + obj: PyObjectRef, + encoding: &str, + errors: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + let codec = self._lookup_text_encoding(encoding, "codecs.decode()", vm)?; + codec + .decode(obj, errors, vm) + .inspect_err(|exc| { + Self::add_codec_note(exc, "decoding", encoding, vm); + })? + .downcast() + .map_err(|obj| { + vm.new_type_error(format!( + "'{}' decoder returned '{}' instead of 'str'; use codecs.decode() to \ + decode to arbitrary types", + encoding, + obj.class().name(), + )) + }) + } + + fn add_codec_note( + exc: &crate::builtins::PyBaseExceptionRef, + operation: &str, + encoding: &str, + vm: &VirtualMachine, + ) { + let note = format!("{operation} with '{encoding}' codec failed"); + let _ = vm.call_method(exc.as_object(), "add_note", (vm.ctx.new_str(note),)); + } + + pub fn register_error(&self, name: String, handler: PyObjectRef) -> Option<PyObjectRef> { + self.inner.write().errors.insert(name, handler) + } + + pub fn unregister_error(&self, name: &str, vm: &VirtualMachine) -> PyResult<bool> { + const BUILTIN_ERROR_HANDLERS: &[&str] = &[ + "strict", + "ignore", + "replace", + "xmlcharrefreplace", + "backslashreplace", + "namereplace", + "surrogatepass", + "surrogateescape", + ]; + if BUILTIN_ERROR_HANDLERS.contains(&name) { + return Err(vm.new_value_error(format!( + "cannot un-register built-in error handler '{name}'" + ))); + } + Ok(self.inner.write().errors.remove(name).is_some()) + } + + pub fn lookup_error_opt(&self, name: &str) -> Option<PyObjectRef> { + self.inner.read().errors.get(name).cloned() + } + + pub fn lookup_error(&self, name: &str, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + self.lookup_error_opt(name) + .ok_or_else(|| vm.new_lookup_error(format!("unknown error handler name '{name}'"))) + } +} + +fn normalize_encoding_name(encoding: &str) -> Cow<'_, str> { + // _Py_normalize_encoding: collapse non-alphanumeric/non-dot chars into + // single underscore, strip non-ASCII, lowercase ASCII letters. + let needs_transform = encoding + .bytes() + .any(|b| b.is_ascii_uppercase() || !b.is_ascii_alphanumeric() && b != b'.'); + if !needs_transform { + return encoding.into(); + } + let mut out = String::with_capacity(encoding.len()); + let mut punct = false; + for c in encoding.chars() { + if c.is_ascii_alphanumeric() || c == '.' { + if punct && !out.is_empty() { + out.push('_'); + } + out.push(c.to_ascii_lowercase()); + punct = false; + } else { + punct = true; + } + } + out.into() +} + +#[derive(Eq, PartialEq)] +enum StandardEncoding { + Utf8, + Utf16Be, + Utf16Le, + Utf32Be, + Utf32Le, +} + +impl StandardEncoding { + #[cfg(target_endian = "little")] + const UTF_16_NE: Self = Self::Utf16Le; + #[cfg(target_endian = "big")] + const UTF_16_NE: Self = Self::Utf16Be; + + #[cfg(target_endian = "little")] + const UTF_32_NE: Self = Self::Utf32Le; + #[cfg(target_endian = "big")] + const UTF_32_NE: Self = Self::Utf32Be; + + fn parse(encoding: &str) -> Option<Self> { + if let Some(encoding) = encoding.to_lowercase().strip_prefix("utf") { + let encoding = encoding + .strip_prefix(|c| ['-', '_'].contains(&c)) + .unwrap_or(encoding); + if encoding == "8" { + Some(Self::Utf8) + } else if let Some(encoding) = encoding.strip_prefix("16") { + if encoding.is_empty() { + return Some(Self::UTF_16_NE); + } + let encoding = encoding.strip_prefix(['-', '_']).unwrap_or(encoding); + match encoding { + "be" => Some(Self::Utf16Be), + "le" => Some(Self::Utf16Le), + _ => None, + } + } else if let Some(encoding) = encoding.strip_prefix("32") { + if encoding.is_empty() { + return Some(Self::UTF_32_NE); + } + let encoding = encoding.strip_prefix(['-', '_']).unwrap_or(encoding); + match encoding { + "be" => Some(Self::Utf32Be), + "le" => Some(Self::Utf32Le), + _ => None, + } + } else { + None + } + } else if encoding == "cp65001" { + Some(Self::Utf8) + } else { + None + } + } +} + +struct SurrogatePass; + +impl<'a> EncodeErrorHandler<PyEncodeContext<'a>> for SurrogatePass { + fn handle_encode_error( + &self, + ctx: &mut PyEncodeContext<'a>, + range: Range<StrSize>, + reason: Option<&str>, + ) -> PyResult<(EncodeReplace<PyEncodeContext<'a>>, StrSize)> { + let standard_encoding = StandardEncoding::parse(ctx.encoding) + .ok_or_else(|| ctx.error_encoding(range.clone(), reason))?; + let err_str = &ctx.full_data()[range.start.bytes..range.end.bytes]; + let num_chars = range.end.chars - range.start.chars; + let mut out: Vec<u8> = Vec::with_capacity(num_chars * 4); + for ch in err_str.code_points() { + let c = ch.to_u32(); + let 0xd800..=0xdfff = c else { + // Not a surrogate, fail with original exception + return Err(ctx.error_encoding(range, reason)); + }; + match standard_encoding { + StandardEncoding::Utf8 => out.extend(ch.encode_wtf8(&mut [0; 4]).as_bytes()), + StandardEncoding::Utf16Le => out.extend((c as u16).to_le_bytes()), + StandardEncoding::Utf16Be => out.extend((c as u16).to_be_bytes()), + StandardEncoding::Utf32Le => out.extend(c.to_le_bytes()), + StandardEncoding::Utf32Be => out.extend(c.to_be_bytes()), + } + } + Ok((EncodeReplace::Bytes(ctx.bytes(out)), range.end)) + } +} + +impl<'a> DecodeErrorHandler<PyDecodeContext<'a>> for SurrogatePass { + fn handle_decode_error( + &self, + ctx: &mut PyDecodeContext<'a>, + byte_range: Range<usize>, + reason: Option<&str>, + ) -> PyResult<(PyStrRef, usize)> { + let standard_encoding = StandardEncoding::parse(ctx.encoding) + .ok_or_else(|| ctx.error_decoding(byte_range.clone(), reason))?; + + let s = ctx.full_data(); + debug_assert!(byte_range.start <= 0.max(s.len() - 1)); + debug_assert!(byte_range.end >= 1.min(s.len())); + debug_assert!(byte_range.end <= s.len()); + + // Try decoding a single surrogate character. If there are more, + // let the codec call us again. + let p = &s[byte_range.start..]; + + fn slice<const N: usize>(p: &[u8]) -> Option<[u8; N]> { + p.first_chunk().copied() + } + + let c = match standard_encoding { + StandardEncoding::Utf8 => { + // it's a three-byte code + slice::<3>(p) + .filter(|&[a, b, c]| { + (u32::from(a) & 0xf0) == 0xe0 + && (u32::from(b) & 0xc0) == 0x80 + && (u32::from(c) & 0xc0) == 0x80 + }) + .map(|[a, b, c]| { + ((u32::from(a) & 0x0f) << 12) + + ((u32::from(b) & 0x3f) << 6) + + (u32::from(c) & 0x3f) + }) + } + StandardEncoding::Utf16Le => slice(p).map(u16::from_le_bytes).map(u32::from), + StandardEncoding::Utf16Be => slice(p).map(u16::from_be_bytes).map(u32::from), + StandardEncoding::Utf32Le => slice(p).map(u32::from_le_bytes), + StandardEncoding::Utf32Be => slice(p).map(u32::from_be_bytes), + }; + let byte_length = match standard_encoding { + StandardEncoding::Utf8 => 3, + StandardEncoding::Utf16Be | StandardEncoding::Utf16Le => 2, + StandardEncoding::Utf32Be | StandardEncoding::Utf32Le => 4, + }; + + // !Py_UNICODE_IS_SURROGATE + let c = c + .and_then(CodePoint::from_u32) + .filter(|c| matches!(c.to_u32(), 0xd800..=0xdfff)) + .ok_or_else(|| ctx.error_decoding(byte_range.clone(), reason))?; + + Ok((ctx.string(c.into()), byte_range.start + byte_length)) + } +} + +pub struct PyEncodeContext<'a> { + vm: &'a VirtualMachine, + encoding: &'a str, + data: &'a Py<PyStr>, + pos: StrSize, + exception: OnceCell<PyBaseExceptionRef>, +} + +impl<'a> PyEncodeContext<'a> { + pub fn new(encoding: &'a str, data: &'a Py<PyStr>, vm: &'a VirtualMachine) -> Self { + Self { + vm, + encoding, + data, + pos: StrSize::default(), + exception: OnceCell::new(), + } + } +} + +impl CodecContext for PyEncodeContext<'_> { + type Error = PyBaseExceptionRef; + type StrBuf = PyStrRef; + type BytesBuf = PyBytesRef; + + fn string(&self, s: Wtf8Buf) -> Self::StrBuf { + self.vm.ctx.new_str(s) + } + + fn bytes(&self, b: Vec<u8>) -> Self::BytesBuf { + self.vm.ctx.new_bytes(b) + } +} +impl EncodeContext for PyEncodeContext<'_> { + fn full_data(&self) -> &Wtf8 { + self.data.as_wtf8() + } + + fn data_len(&self) -> StrSize { + StrSize { + bytes: self.data.byte_len(), + chars: self.data.char_len(), + } + } + + fn remaining_data(&self) -> &Wtf8 { + &self.full_data()[self.pos.bytes..] + } + + fn position(&self) -> StrSize { + self.pos + } + + fn restart_from(&mut self, pos: StrSize) -> Result<(), Self::Error> { + if pos.chars > self.data.char_len() { + return Err(self.vm.new_index_error(format!( + "position {} from error handler out of bounds", + pos.chars + ))); + } + assert!( + self.data.as_wtf8().is_code_point_boundary(pos.bytes), + "invalid pos {pos:?} for {:?}", + self.data.as_wtf8() + ); + self.pos = pos; + Ok(()) + } + + fn error_encoding(&self, range: Range<StrSize>, reason: Option<&str>) -> Self::Error { + let vm = self.vm; + match self.exception.get() { + Some(exc) => { + match update_unicode_error_attrs( + exc.as_object(), + range.start.chars, + range.end.chars, + reason, + vm, + ) { + Ok(()) => exc.clone(), + Err(e) => e, + } + } + None => self + .exception + .get_or_init(|| { + let reason = reason.expect( + "should only ever pass reason: None if an exception is already set", + ); + vm.new_unicode_encode_error_real( + vm.ctx.new_str(self.encoding), + self.data.to_owned(), + range.start.chars, + range.end.chars, + vm.ctx.new_str(reason), + ) + }) + .clone(), + } + } +} + +pub struct PyDecodeContext<'a> { + vm: &'a VirtualMachine, + encoding: &'a str, + data: PyDecodeData<'a>, + orig_bytes: Option<&'a Py<PyBytes>>, + pos: usize, + exception: OnceCell<PyBaseExceptionRef>, +} +enum PyDecodeData<'a> { + Original(BorrowedValue<'a, [u8]>), + Modified(PyBytesRef), +} +impl ops::Deref for PyDecodeData<'_> { + type Target = [u8]; + fn deref(&self) -> &Self::Target { + match self { + PyDecodeData::Original(data) => data, + PyDecodeData::Modified(data) => data, + } + } +} + +impl<'a> PyDecodeContext<'a> { + pub fn new(encoding: &'a str, data: &'a ArgBytesLike, vm: &'a VirtualMachine) -> Self { + Self { + vm, + encoding, + data: PyDecodeData::Original(data.borrow_buf()), + orig_bytes: data.as_object().downcast_ref(), + pos: 0, + exception: OnceCell::new(), + } + } +} + +impl CodecContext for PyDecodeContext<'_> { + type Error = PyBaseExceptionRef; + type StrBuf = PyStrRef; + type BytesBuf = PyBytesRef; + + fn string(&self, s: Wtf8Buf) -> Self::StrBuf { + self.vm.ctx.new_str(s) + } + + fn bytes(&self, b: Vec<u8>) -> Self::BytesBuf { + self.vm.ctx.new_bytes(b) + } +} +impl DecodeContext for PyDecodeContext<'_> { + fn full_data(&self) -> &[u8] { + &self.data + } + + fn remaining_data(&self) -> &[u8] { + &self.data[self.pos..] + } + + fn position(&self) -> usize { + self.pos + } + + fn advance(&mut self, by: usize) { + self.pos += by; + } + + fn restart_from(&mut self, pos: usize) -> Result<(), Self::Error> { + if pos > self.data.len() { + return Err(self + .vm + .new_index_error(format!("position {pos} from error handler out of bounds",))); + } + self.pos = pos; + Ok(()) + } + + fn error_decoding(&self, byte_range: Range<usize>, reason: Option<&str>) -> Self::Error { + let vm = self.vm; + + match self.exception.get() { + Some(exc) => { + match update_unicode_error_attrs( + exc.as_object(), + byte_range.start, + byte_range.end, + reason, + vm, + ) { + Ok(()) => exc.clone(), + Err(e) => e, + } + } + None => self + .exception + .get_or_init(|| { + let reason = reason.expect( + "should only ever pass reason: None if an exception is already set", + ); + let data = if let Some(bytes) = self.orig_bytes { + bytes.to_owned() + } else { + vm.ctx.new_bytes(self.data.to_vec()) + }; + vm.new_unicode_decode_error_real( + vm.ctx.new_str(self.encoding), + data, + byte_range.start, + byte_range.end, + vm.ctx.new_str(reason), + ) + }) + .clone(), + } + } +} + +#[derive(strum_macros::EnumString)] +#[strum(serialize_all = "lowercase")] +enum StandardError { + Strict, + Ignore, + Replace, + XmlCharRefReplace, + BackslashReplace, + SurrogatePass, + SurrogateEscape, +} + +impl<'a> EncodeErrorHandler<PyEncodeContext<'a>> for StandardError { + fn handle_encode_error( + &self, + ctx: &mut PyEncodeContext<'a>, + range: Range<StrSize>, + reason: Option<&str>, + ) -> PyResult<(EncodeReplace<PyEncodeContext<'a>>, StrSize)> { + use StandardError::*; + // use errors::*; + match self { + Strict => errors::Strict.handle_encode_error(ctx, range, reason), + Ignore => errors::Ignore.handle_encode_error(ctx, range, reason), + Replace => errors::Replace.handle_encode_error(ctx, range, reason), + XmlCharRefReplace => errors::XmlCharRefReplace.handle_encode_error(ctx, range, reason), + BackslashReplace => errors::BackslashReplace.handle_encode_error(ctx, range, reason), + SurrogatePass => SurrogatePass.handle_encode_error(ctx, range, reason), + SurrogateEscape => errors::SurrogateEscape.handle_encode_error(ctx, range, reason), + } + } +} + +impl<'a> DecodeErrorHandler<PyDecodeContext<'a>> for StandardError { + fn handle_decode_error( + &self, + ctx: &mut PyDecodeContext<'a>, + byte_range: Range<usize>, + reason: Option<&str>, + ) -> PyResult<(PyStrRef, usize)> { + use StandardError::*; + match self { + Strict => errors::Strict.handle_decode_error(ctx, byte_range, reason), + Ignore => errors::Ignore.handle_decode_error(ctx, byte_range, reason), + Replace => errors::Replace.handle_decode_error(ctx, byte_range, reason), + XmlCharRefReplace => Err(ctx + .vm + .new_type_error("don't know how to handle UnicodeDecodeError in error callback")), + BackslashReplace => { + errors::BackslashReplace.handle_decode_error(ctx, byte_range, reason) + } + SurrogatePass => self::SurrogatePass.handle_decode_error(ctx, byte_range, reason), + SurrogateEscape => errors::SurrogateEscape.handle_decode_error(ctx, byte_range, reason), + } + } +} + +pub struct ErrorsHandler<'a> { + errors: &'a Py<PyUtf8Str>, + resolved: OnceCell<ResolvedError>, +} +enum ResolvedError { + Standard(StandardError), + Handler(PyObjectRef), +} + +impl<'a> ErrorsHandler<'a> { + #[inline] + pub fn new(errors: Option<&'a Py<PyUtf8Str>>, vm: &VirtualMachine) -> Self { + match errors { + Some(errors) => Self { + errors, + resolved: OnceCell::new(), + }, + None => Self { + errors: identifier_utf8!(vm, strict), + resolved: OnceCell::from(ResolvedError::Standard(StandardError::Strict)), + }, + } + } + #[inline] + fn resolve(&self, vm: &VirtualMachine) -> PyResult<&ResolvedError> { + if let Some(val) = self.resolved.get() { + return Ok(val); + } + let errors_str = self.errors.as_str(); + let val = if let Ok(standard) = errors_str.parse() { + ResolvedError::Standard(standard) + } else { + vm.state + .codec_registry + .lookup_error(errors_str, vm) + .map(ResolvedError::Handler)? + }; + let _ = self.resolved.set(val); + Ok(self.resolved.get().unwrap()) + } +} +impl StrBuffer for PyStrRef { + fn is_compatible_with(&self, kind: StrKind) -> bool { + self.kind() <= kind + } +} +impl<'a> EncodeErrorHandler<PyEncodeContext<'a>> for ErrorsHandler<'_> { + fn handle_encode_error( + &self, + ctx: &mut PyEncodeContext<'a>, + range: Range<StrSize>, + reason: Option<&str>, + ) -> PyResult<(EncodeReplace<PyEncodeContext<'a>>, StrSize)> { + let vm = ctx.vm; + let handler = match self.resolve(vm)? { + ResolvedError::Standard(standard) => { + return standard.handle_encode_error(ctx, range, reason); + } + ResolvedError::Handler(handler) => handler, + }; + let encode_exc = ctx.error_encoding(range.clone(), reason); + let res = handler.call((encode_exc.clone(),), vm)?; + let tuple_err = + || vm.new_type_error("encoding error handler must return (str/bytes, int) tuple"); + let (replace, restart) = match res.downcast_ref::<PyTuple>().map(|tup| tup.as_slice()) { + Some([replace, restart]) => (replace.clone(), restart), + _ => return Err(tuple_err()), + }; + let replace = match_class!(match replace { + s @ PyStr => EncodeReplace::Str(s), + b @ PyBytes => EncodeReplace::Bytes(b), + _ => return Err(tuple_err()), + }); + let restart = isize::try_from_borrowed_object(vm, restart).map_err(|_| tuple_err())?; + let restart = if restart < 0 { + // will still be out of bounds if it underflows ¯\_(ツ)_/¯ + ctx.data.char_len().wrapping_sub(restart.unsigned_abs()) + } else { + restart as usize + }; + let restart = if restart == range.end.chars { + range.end + } else { + StrSize { + chars: restart, + bytes: ctx + .data + .as_wtf8() + .code_point_indices() + .nth(restart) + .map_or(ctx.data.byte_len(), |(i, _)| i), + } + }; + Ok((replace, restart)) + } +} +impl<'a> DecodeErrorHandler<PyDecodeContext<'a>> for ErrorsHandler<'_> { + fn handle_decode_error( + &self, + ctx: &mut PyDecodeContext<'a>, + byte_range: Range<usize>, + reason: Option<&str>, + ) -> PyResult<(PyStrRef, usize)> { + let vm = ctx.vm; + let handler = match self.resolve(vm)? { + ResolvedError::Standard(standard) => { + return standard.handle_decode_error(ctx, byte_range, reason); + } + ResolvedError::Handler(handler) => handler, + }; + let decode_exc = ctx.error_decoding(byte_range.clone(), reason); + let data_bytes: PyObjectRef = decode_exc.as_object().get_attr("object", vm)?; + let res = handler.call((decode_exc.clone(),), vm)?; + let new_data = decode_exc.as_object().get_attr("object", vm)?; + if !new_data.is(&data_bytes) { + let new_data: PyBytesRef = new_data + .downcast() + .map_err(|_| vm.new_type_error("object attribute must be bytes"))?; + ctx.data = PyDecodeData::Modified(new_data); + } + let data = &*ctx.data; + let tuple_err = || vm.new_type_error("decoding error handler must return (str, int) tuple"); + match res.downcast_ref::<PyTuple>().map(|tup| tup.as_slice()) { + Some([replace, restart]) => { + let replace = replace + .downcast_ref::<PyStr>() + .ok_or_else(tuple_err)? + .to_owned(); + let restart = + isize::try_from_borrowed_object(vm, restart).map_err(|_| tuple_err())?; + let restart = if restart < 0 { + // will still be out of bounds if it underflows ¯\_(ツ)_/¯ + data.len().wrapping_sub(restart.unsigned_abs()) + } else { + restart as usize + }; + Ok((replace, restart)) + } + _ => Err(tuple_err()), + } + } +} + +fn call_native_encode_error<E>( + handler: E, + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> +where + for<'a> E: EncodeErrorHandler<PyEncodeContext<'a>>, +{ + // let err = err. + let range = extract_unicode_error_range(&err, vm)?; + let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; + let s_encoding = PyUtf8StrRef::try_from_object(vm, err.get_attr("encoding", vm)?)?; + let mut ctx = PyEncodeContext { + vm, + encoding: s_encoding.as_str(), + data: &s, + pos: StrSize::default(), + exception: OnceCell::from(err.downcast().unwrap()), + }; + let mut iter = s.as_wtf8().code_point_indices(); + let start = StrSize { + chars: range.start, + bytes: iter.nth(range.start).unwrap().0, + }; + let end = StrSize { + chars: range.end, + bytes: if let Some(n) = range.len().checked_sub(1) { + iter.nth(n).map_or(s.byte_len(), |(i, _)| i) + } else { + start.bytes + }, + }; + let (replace, restart) = handler.handle_encode_error(&mut ctx, start..end, None)?; + let replace = match replace { + EncodeReplace::Str(s) => s.into(), + EncodeReplace::Bytes(b) => b.into(), + }; + Ok((replace, restart.chars)) +} + +fn call_native_decode_error<E>( + handler: E, + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> +where + for<'a> E: DecodeErrorHandler<PyDecodeContext<'a>>, +{ + let range = extract_unicode_error_range(&err, vm)?; + let s = ArgBytesLike::try_from_object(vm, err.get_attr("object", vm)?)?; + let s_encoding = PyUtf8StrRef::try_from_object(vm, err.get_attr("encoding", vm)?)?; + let mut ctx = PyDecodeContext { + vm, + encoding: s_encoding.as_str(), + data: PyDecodeData::Original(s.borrow_buf()), + orig_bytes: s.as_object().downcast_ref(), + pos: 0, + exception: OnceCell::from(err.downcast().unwrap()), + }; + let (replace, restart) = handler.handle_decode_error(&mut ctx, range, None)?; + Ok((replace.into(), restart)) +} + +// this is a hack, for now +fn call_native_translate_error<E>( + handler: E, + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> +where + for<'a> E: EncodeErrorHandler<PyEncodeContext<'a>>, +{ + // let err = err. + let range = extract_unicode_error_range(&err, vm)?; + let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; + let mut ctx = PyEncodeContext { + vm, + encoding: "", + data: &s, + pos: StrSize::default(), + exception: OnceCell::from(err.downcast().unwrap()), + }; + let mut iter = s.as_wtf8().code_point_indices(); + let start = StrSize { + chars: range.start, + bytes: iter.nth(range.start).unwrap().0, + }; + let end = StrSize { + chars: range.end, + bytes: if let Some(n) = range.len().checked_sub(1) { + iter.nth(n).map_or(s.byte_len(), |(i, _)| i) + } else { + start.bytes + }, + }; + let (replace, restart) = handler.handle_encode_error(&mut ctx, start..end, None)?; + let replace = match replace { + EncodeReplace::Str(s) => s.into(), + EncodeReplace::Bytes(b) => b.into(), + }; + Ok((replace, restart.chars)) +} + +// TODO: exceptions with custom payloads +fn extract_unicode_error_range(err: &PyObject, vm: &VirtualMachine) -> PyResult<Range<usize>> { + let start = err.get_attr("start", vm)?; + let start = start.try_into_value(vm)?; + let end = err.get_attr("end", vm)?; + let end = end.try_into_value(vm)?; + Ok(Range { start, end }) +} + +fn update_unicode_error_attrs( + err: &PyObject, + start: usize, + end: usize, + reason: Option<&str>, + vm: &VirtualMachine, +) -> PyResult<()> { + err.set_attr("start", start.to_pyobject(vm), vm)?; + err.set_attr("end", end.to_pyobject(vm), vm)?; + if let Some(reason) = reason { + err.set_attr("reason", reason.to_pyobject(vm), vm)?; + } + Ok(()) +} + +#[inline] +fn is_encode_err(err: &PyObject, vm: &VirtualMachine) -> bool { + err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) +} +#[inline] +fn is_decode_err(err: &PyObject, vm: &VirtualMachine) -> bool { + err.fast_isinstance(vm.ctx.exceptions.unicode_decode_error) +} +#[inline] +fn is_translate_err(err: &PyObject, vm: &VirtualMachine) -> bool { + err.fast_isinstance(vm.ctx.exceptions.unicode_translate_error) +} + +fn bad_err_type(err: PyObjectRef, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_type_error(format!( + "don't know how to handle {} in error callback", + err.class().name() + )) +} + +fn strict_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let err = err + .downcast() + .unwrap_or_else(|_| vm.new_type_error("codec must pass exception instance")); + Err(err) +} + +fn ignore_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) || is_decode_err(&err, vm) || is_translate_err(&err, vm) { + let range = extract_unicode_error_range(&err, vm)?; + Ok((vm.ctx.new_str(ascii!("")).into(), range.end)) + } else { + Err(bad_err_type(err, vm)) + } +} + +fn replace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) { + call_native_encode_error(errors::Replace, err, vm) + } else if is_decode_err(&err, vm) { + call_native_decode_error(errors::Replace, err, vm) + } else if is_translate_err(&err, vm) { + // char::REPLACEMENT_CHARACTER as a str + let replacement_char = "\u{FFFD}"; + let range = extract_unicode_error_range(&err, vm)?; + let replace = replacement_char.repeat(range.end - range.start); + Ok((replace.to_pyobject(vm), range.end)) + } else { + Err(bad_err_type(err, vm)) + } +} + +fn xmlcharrefreplace_errors( + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) { + call_native_encode_error(errors::XmlCharRefReplace, err, vm) + } else { + Err(bad_err_type(err, vm)) + } +} + +fn backslashreplace_errors( + err: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, usize)> { + if is_decode_err(&err, vm) { + call_native_decode_error(errors::BackslashReplace, err, vm) + } else if is_encode_err(&err, vm) { + call_native_encode_error(errors::BackslashReplace, err, vm) + } else if is_translate_err(&err, vm) { + call_native_translate_error(errors::BackslashReplace, err, vm) + } else { + Err(bad_err_type(err, vm)) + } +} + +fn namereplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) { + call_native_encode_error(errors::NameReplace, err, vm) + } else { + Err(bad_err_type(err, vm)) + } +} + +fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) { + call_native_encode_error(SurrogatePass, err, vm) + } else if is_decode_err(&err, vm) { + call_native_decode_error(SurrogatePass, err, vm) + } else { + Err(bad_err_type(err, vm)) + } +} + +fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { + if is_encode_err(&err, vm) { + call_native_encode_error(errors::SurrogateEscape, err, vm) + } else if is_decode_err(&err, vm) { + call_native_decode_error(errors::SurrogateEscape, err, vm) + } else { + Err(bad_err_type(err, vm)) + } +} diff --git a/crates/vm/src/compiler.rs b/crates/vm/src/compiler.rs new file mode 100644 index 00000000000..25fa33302a5 --- /dev/null +++ b/crates/vm/src/compiler.rs @@ -0,0 +1,56 @@ +#[cfg(feature = "codegen")] +pub use rustpython_codegen::CompileOpts; + +#[cfg(feature = "compiler")] +pub use rustpython_compiler::*; + +#[cfg(not(feature = "compiler"))] +pub use rustpython_compiler_core::Mode; + +#[cfg(not(feature = "compiler"))] +pub use rustpython_compiler_core as core; + +#[cfg(not(feature = "compiler"))] +pub use ruff_python_parser as parser; + +#[cfg(not(feature = "compiler"))] +mod error { + #[cfg(all(feature = "parser", feature = "codegen"))] + panic!("Use --features=compiler to enable both parser and codegen"); + + #[derive(Debug, thiserror::Error)] + pub enum CompileErrorType { + #[cfg(feature = "codegen")] + #[error(transparent)] + Codegen(#[from] super::codegen::error::CodegenErrorType), + #[cfg(feature = "parser")] + #[error(transparent)] + Parse(#[from] super::parser::ParseErrorType), + } + + #[derive(Debug, thiserror::Error)] + pub enum CompileError { + #[cfg(feature = "codegen")] + #[error(transparent)] + Codegen(#[from] super::codegen::error::CodegenError), + #[cfg(feature = "parser")] + #[error(transparent)] + Parse(#[from] super::parser::ParseError), + } +} +#[cfg(not(feature = "compiler"))] +pub use error::{CompileError, CompileErrorType}; + +#[cfg(any(feature = "parser", feature = "codegen"))] +impl crate::convert::ToPyException for (CompileError, Option<&str>) { + fn to_pyexception(&self, vm: &crate::VirtualMachine) -> crate::builtins::PyBaseExceptionRef { + vm.new_syntax_error(&self.0, self.1) + } +} + +#[cfg(any(feature = "parser", feature = "codegen"))] +impl crate::convert::ToPyException for (CompileError, Option<&str>, bool) { + fn to_pyexception(&self, vm: &crate::VirtualMachine) -> crate::builtins::PyBaseExceptionRef { + vm.new_syntax_error_maybe_incomplete(&self.0, self.1, self.2) + } +} diff --git a/vm/src/convert/into_object.rs b/crates/vm/src/convert/into_object.rs similarity index 100% rename from vm/src/convert/into_object.rs rename to crates/vm/src/convert/into_object.rs diff --git a/vm/src/convert/mod.rs b/crates/vm/src/convert/mod.rs similarity index 100% rename from vm/src/convert/mod.rs rename to crates/vm/src/convert/mod.rs diff --git a/vm/src/convert/to_pyobject.rs b/crates/vm/src/convert/to_pyobject.rs similarity index 91% rename from vm/src/convert/to_pyobject.rs rename to crates/vm/src/convert/to_pyobject.rs index c61296fe693..840c0c65fbf 100644 --- a/vm/src/convert/to_pyobject.rs +++ b/crates/vm/src/convert/to_pyobject.rs @@ -1,4 +1,4 @@ -use crate::{builtins::PyBaseExceptionRef, PyObjectRef, PyResult, VirtualMachine}; +use crate::{PyObjectRef, PyResult, VirtualMachine, builtins::PyBaseExceptionRef}; /// Implemented by any type that can be returned from a built-in Python function. /// diff --git a/vm/src/convert/transmute_from.rs b/crates/vm/src/convert/transmute_from.rs similarity index 95% rename from vm/src/convert/transmute_from.rs rename to crates/vm/src/convert/transmute_from.rs index 908188f0d10..c1b4b79384e 100644 --- a/vm/src/convert/transmute_from.rs +++ b/crates/vm/src/convert/transmute_from.rs @@ -17,7 +17,7 @@ unsafe impl<T: PyPayload> TransmuteFromObject for PyRef<T> { fn check(vm: &VirtualMachine, obj: &PyObject) -> PyResult<()> { let class = T::class(&vm.ctx); if obj.fast_isinstance(class) { - if obj.payload_is::<T>() { + if obj.downcastable::<T>() { Ok(()) } else { Err(vm.new_downcast_runtime_error(class, obj)) diff --git a/crates/vm/src/convert/try_from.rs b/crates/vm/src/convert/try_from.rs new file mode 100644 index 00000000000..85d6f5e20e3 --- /dev/null +++ b/crates/vm/src/convert/try_from.rs @@ -0,0 +1,162 @@ +use crate::{ + Py, VirtualMachine, + builtins::PyFloat, + object::{AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult}, +}; +use malachite_bigint::Sign; +use num_traits::ToPrimitive; + +/// Implemented by any type that can be created from a Python object. +/// +/// Any type that implements `TryFromObject` is automatically `FromArgs`, and +/// so can be accepted as a argument to a built-in function. +pub trait TryFromObject: Sized { + /// Attempt to convert a Python object to a value of this type. + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self>; +} + +/// Rust-side only version of TryFromObject to reduce unnecessary Rc::clone +impl<T: for<'a> TryFromBorrowedObject<'a>> TryFromObject for T { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + TryFromBorrowedObject::try_from_borrowed_object(vm, &obj) + } +} + +impl PyObjectRef { + pub fn try_into_value<T>(self, vm: &VirtualMachine) -> PyResult<T> + where + T: TryFromObject, + { + T::try_from_object(vm, self) + } +} + +impl PyObject { + pub fn try_to_value<'a, T>(&'a self, vm: &VirtualMachine) -> PyResult<T> + where + T: 'a + TryFromBorrowedObject<'a>, + { + T::try_from_borrowed_object(vm, self) + } + + pub fn try_to_ref<'a, T>(&'a self, vm: &VirtualMachine) -> PyResult<&'a Py<T>> + where + T: 'a + PyPayload, + { + self.try_to_value::<&Py<T>>(vm) + } + + pub fn try_value_with<T, F, R>(&self, f: F, vm: &VirtualMachine) -> PyResult<R> + where + T: PyPayload, + F: Fn(&T) -> PyResult<R>, + { + let class = T::class(&vm.ctx); + let py_ref = if self.fast_isinstance(class) { + self.downcast_ref() + .ok_or_else(|| vm.new_downcast_runtime_error(class, self))? + } else { + return Err(vm.new_downcast_type_error(class, self)); + }; + f(py_ref) + } +} + +/// Lower-cost variation of `TryFromObject` +pub trait TryFromBorrowedObject<'a>: Sized +where + Self: 'a, +{ + /// Attempt to convert a Python object to a value of this type. + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self>; +} + +impl<T> TryFromObject for PyRef<T> +where + T: PyPayload, +{ + #[inline] + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let class = T::class(&vm.ctx); + if obj.fast_isinstance(class) { + T::try_downcast_from(&obj, vm)?; + Ok(unsafe { obj.downcast_unchecked() }) + } else { + Err(vm.new_downcast_type_error(class, &obj)) + } + } +} + +impl TryFromObject for PyObjectRef { + #[inline] + fn try_from_object(_vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + Ok(obj) + } +} + +impl<T: TryFromObject> TryFromObject for Option<T> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + if vm.is_none(&obj) { + Ok(None) + } else { + T::try_from_object(vm, obj).map(Some) + } + } +} + +impl<'a, T: 'a + TryFromObject> TryFromBorrowedObject<'a> for Vec<T> { + fn try_from_borrowed_object(vm: &VirtualMachine, value: &'a PyObject) -> PyResult<Self> { + vm.extract_elements_with(value, |obj| T::try_from_object(vm, obj)) + } +} + +impl<'a, T: PyPayload> TryFromBorrowedObject<'a> for &'a Py<T> { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + let class = T::class(&vm.ctx); + if obj.fast_isinstance(class) { + obj.downcast_ref() + .ok_or_else(|| vm.new_downcast_runtime_error(class, &obj)) + } else { + Err(vm.new_downcast_type_error(class, &obj)) + } + } +} + +impl TryFromObject for core::time::Duration { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + if let Some(float) = obj.downcast_ref::<PyFloat>() { + let f = float.to_f64(); + if f.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)")); + } + if f < 0.0 { + return Err(vm.new_value_error("negative duration")); + } + if !f.is_finite() || f > u64::MAX as f64 { + return Err(vm.new_overflow_error("timestamp too large to convert to C PyTime_t")); + } + // Convert float to Duration using floor rounding (_PyTime_ROUND_FLOOR) + let secs = f.trunc() as u64; + let frac = f.fract(); + // Use floor to round down the nanoseconds + let nanos = (frac * 1_000_000_000.0).floor() as u32; + Ok(Self::new(secs, nanos)) + } else if let Some(int) = obj.try_index_opt(vm) { + let int = int?; + let bigint = int.as_bigint(); + if bigint.sign() == Sign::Minus { + return Err(vm.new_value_error("negative duration")); + } + + let sec = bigint + .to_u64() + .ok_or_else(|| vm.new_value_error("value out of range"))?; + Ok(Self::from_secs(sec)) + } else { + Err(vm.new_type_error(format!( + "expected an int or float for duration, got {}", + obj.class() + ))) + } + } +} diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs new file mode 100644 index 00000000000..07158c48859 --- /dev/null +++ b/crates/vm/src/coroutine.rs @@ -0,0 +1,356 @@ +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::PyStrRef, + common::lock::PyMutex, + exceptions::types::PyBaseException, + frame::{ExecutionResult, Frame, FrameOwner, FrameRef}, + function::OptionalArg, + object::{PyAtomicRef, Traverse, TraverseFn}, + protocol::PyIterReturn, +}; +use crossbeam_utils::atomic::AtomicCell; + +impl ExecutionResult { + /// Turn an ExecutionResult into a PyResult that would be returned from a generator or coroutine + fn into_iter_return(self, vm: &VirtualMachine) -> PyIterReturn { + match self { + Self::Yield(value) => PyIterReturn::Return(value), + Self::Return(value) => { + let arg = if vm.is_none(&value) { + None + } else { + Some(value) + }; + PyIterReturn::StopIteration(arg) + } + } + } +} + +#[derive(Debug)] +pub struct Coro { + frame: FrameRef, + pub closed: AtomicCell<bool>, // TODO: https://github.com/RustPython/RustPython/pull/3183#discussion_r720560652 + running: AtomicCell<bool>, + // code + // _weakreflist + name: PyMutex<PyStrRef>, + qualname: PyMutex<PyStrRef>, + exception: PyAtomicRef<Option<PyBaseException>>, // exc_state +} + +unsafe impl Traverse for Coro { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.frame.traverse(tracer_fn); + self.name.traverse(tracer_fn); + self.qualname.traverse(tracer_fn); + if let Some(exc) = self.exception.deref() { + exc.traverse(tracer_fn); + } + } +} + +fn gen_name(jen: &PyObject, vm: &VirtualMachine) -> &'static str { + let typ = jen.class(); + if typ.is(vm.ctx.types.coroutine_type) { + "coroutine" + } else if typ.is(vm.ctx.types.async_generator) { + "async generator" + } else { + "generator" + } +} + +impl Coro { + pub fn new(frame: FrameRef, name: PyStrRef, qualname: PyStrRef) -> Self { + Self { + frame, + closed: AtomicCell::new(false), + running: AtomicCell::new(false), + exception: PyAtomicRef::from(None), + name: PyMutex::new(name), + qualname: PyMutex::new(qualname), + } + } + + fn maybe_close(&self, res: &PyResult<ExecutionResult>) { + match res { + Ok(ExecutionResult::Return(_)) | Err(_) => { + self.closed.store(true); + // Frame is no longer suspended; allow frame.clear() to succeed. + self.frame.owner.store( + FrameOwner::FrameObject as i8, + core::sync::atomic::Ordering::Release, + ); + } + Ok(ExecutionResult::Yield(_)) => {} + } + } + + fn run_with_context<F>( + &self, + jen: &PyObject, + vm: &VirtualMachine, + func: F, + ) -> PyResult<ExecutionResult> + where + F: FnOnce(&Py<Frame>) -> PyResult<ExecutionResult>, + { + if self.running.compare_exchange(false, true).is_err() { + return Err(vm.new_value_error(format!("{} already executing", gen_name(jen, vm)))); + } + + // SAFETY: running.compare_exchange guarantees exclusive access + let gen_exc = unsafe { self.exception.swap(None) }; + let exception_ptr = &self.exception as *const PyAtomicRef<Option<PyBaseException>>; + + let result = vm.resume_gen_frame(&self.frame, gen_exc, |f| { + let result = func(f); + // SAFETY: exclusive access guaranteed by running flag + let _old = unsafe { (*exception_ptr).swap(vm.current_exception()) }; + result + }); + + self.running.store(false); + result + } + + fn finalize_send_result( + &self, + jen: &PyObject, + result: PyResult<ExecutionResult>, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + self.maybe_close(&result); + match result { + Ok(exec_res) => Ok(exec_res.into_iter_return(vm)), + Err(e) => { + if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + let err = + vm.new_runtime_error(format!("{} raised StopIteration", gen_name(jen, vm))); + err.set___cause__(Some(e)); + Err(err) + } else if jen.class().is(vm.ctx.types.async_generator) + && e.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) + { + let err = vm.new_runtime_error("async generator raised StopAsyncIteration"); + err.set___cause__(Some(e)); + Err(err) + } else { + Err(e) + } + } + } + } + + pub(crate) fn send_none(&self, jen: &PyObject, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + if self.closed.load() { + return Ok(PyIterReturn::StopIteration(None)); + } + self.frame.locals_to_fast(vm)?; + let value = if self.frame.lasti() > 0 { + Some(vm.ctx.none()) + } else { + None + }; + let result = self.run_with_context(jen, vm, |f| f.resume(value, vm)); + self.finalize_send_result(jen, result, vm) + } + + pub fn send( + &self, + jen: &PyObject, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + if self.closed.load() { + return Ok(PyIterReturn::StopIteration(None)); + } + self.frame.locals_to_fast(vm)?; + let value = if self.frame.lasti() > 0 { + Some(value) + } else if !vm.is_none(&value) { + return Err(vm.new_type_error(format!( + "can't send non-None value to a just-started {}", + gen_name(jen, vm), + ))); + } else { + None + }; + let result = self.run_with_context(jen, vm, |f| f.resume(value, vm)); + self.finalize_send_result(jen, result, vm) + } + + pub fn throw( + &self, + jen: &PyObject, + exc_type: PyObjectRef, + exc_val: PyObjectRef, + exc_tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + // Validate throw arguments (_gen_throw) + if exc_type.fast_isinstance(vm.ctx.exceptions.base_exception_type) && !vm.is_none(&exc_val) + { + return Err(vm.new_type_error("instance exception may not have a separate value")); + } + if !vm.is_none(&exc_tb) && !exc_tb.fast_isinstance(vm.ctx.types.traceback_type) { + return Err(vm.new_type_error("throw() third argument must be a traceback object")); + } + if self.closed.load() { + return Err(vm.normalize_exception(exc_type, exc_val, exc_tb)?); + } + // Validate exception type before entering generator context. + // Invalid types propagate to caller without closing the generator. + crate::exceptions::ExceptionCtor::try_from_object(vm, exc_type.clone())?; + let result = self.run_with_context(jen, vm, |f| f.gen_throw(vm, exc_type, exc_val, exc_tb)); + self.maybe_close(&result); + Ok(result?.into_iter_return(vm)) + } + + pub fn close(&self, jen: &PyObject, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if self.closed.load() { + return Ok(vm.ctx.none()); + } + // If generator hasn't started (FRAME_CREATED), just mark as closed + if self.frame.lasti() == 0 { + self.closed.store(true); + return Ok(vm.ctx.none()); + } + let result = self.run_with_context(jen, vm, |f| { + f.gen_throw( + vm, + vm.ctx.exceptions.generator_exit.to_owned().into(), + vm.ctx.none(), + vm.ctx.none(), + ) + }); + self.closed.store(true); + // Release frame locals and stack to free references held by the + // closed generator, matching gen_send_ex2 with close_on_completion. + self.frame.clear_locals_and_stack(); + match result { + Ok(ExecutionResult::Yield(_)) => { + Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", gen_name(jen, vm)))) + } + Err(e) if !is_gen_exit(&e, vm) => Err(e), + Ok(ExecutionResult::Return(value)) => Ok(value), + _ => Ok(vm.ctx.none()), + } + } + + pub fn suspended(&self) -> bool { + !self.closed.load() && !self.running.load() && self.frame.lasti() > 0 + } + + pub fn running(&self) -> bool { + self.running.load() + } + + pub fn closed(&self) -> bool { + self.closed.load() + } + + pub fn frame(&self) -> FrameRef { + self.frame.clone() + } + + pub fn name(&self) -> PyStrRef { + self.name.lock().clone() + } + + pub fn set_name(&self, name: PyStrRef) { + *self.name.lock() = name; + } + + pub fn qualname(&self) -> PyStrRef { + self.qualname.lock().clone() + } + + pub fn set_qualname(&self, qualname: PyStrRef) { + *self.qualname.lock() = qualname; + } + + pub fn repr(&self, jen: &PyObject, id: usize, vm: &VirtualMachine) -> String { + let qualname = self.qualname(); + format!( + "<{} object {} at {:#x}>", + gen_name(jen, vm), + qualname.as_wtf8(), + id + ) + } +} + +pub fn is_gen_exit(exc: &Py<PyBaseException>, vm: &VirtualMachine) -> bool { + exc.fast_isinstance(vm.ctx.exceptions.generator_exit) +} + +/// Get an awaitable iterator from an object. +/// +/// Returns the object itself if it's a coroutine or iterable coroutine (generator with +/// CO_ITERABLE_COROUTINE flag). Otherwise calls `__await__()` and validates the result. +pub fn get_awaitable_iter(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + use crate::builtins::{PyCoroutine, PyGenerator}; + use crate::protocol::PyIter; + + if obj.downcastable::<PyCoroutine>() + || obj.downcast_ref::<PyGenerator>().is_some_and(|g| { + g.as_coro() + .frame() + .code + .flags + .contains(crate::bytecode::CodeFlags::ITERABLE_COROUTINE) + }) + { + return Ok(obj); + } + + if let Some(await_method) = vm.get_method(obj.clone(), identifier!(vm, __await__)) { + let result = await_method?.call((), vm)?; + // __await__() must NOT return a coroutine (PEP 492) + if result.downcastable::<PyCoroutine>() + || result.downcast_ref::<PyGenerator>().is_some_and(|g| { + g.as_coro() + .frame() + .code + .flags + .contains(crate::bytecode::CodeFlags::ITERABLE_COROUTINE) + }) + { + return Err(vm.new_type_error("__await__() returned a coroutine")); + } + if !PyIter::check(&result) { + return Err(vm.new_type_error(format!( + "__await__() returned non-iterator of type '{}'", + result.class().name() + ))); + } + return Ok(result); + } + + Err(vm.new_type_error(format!("'{}' object can't be awaited", obj.class().name()))) +} + +/// Emit DeprecationWarning for the deprecated 3-argument throw() signature. +pub fn warn_deprecated_throw_signature( + exc_val: &OptionalArg, + exc_tb: &OptionalArg, + vm: &VirtualMachine, +) -> PyResult<()> { + if exc_val.is_present() || exc_tb.is_present() { + crate::warn::warn( + vm.ctx + .new_str( + "the (type, val, tb) signature of throw() is deprecated, \ + use throw(val) instead", + ) + .into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + Ok(()) +} diff --git a/crates/vm/src/datastack.rs b/crates/vm/src/datastack.rs new file mode 100644 index 00000000000..b00f1b6dc19 --- /dev/null +++ b/crates/vm/src/datastack.rs @@ -0,0 +1,256 @@ +/// Thread data stack for interpreter frames (`_PyStackChunk` / +/// `tstate->datastack_*`). +/// +/// A linked list of chunks providing bump allocation for frame-local data +/// (localsplus arrays). Normal function calls allocate via `push()` +/// (pointer bump). Generators and coroutines use heap-allocated storage. +use alloc::alloc::{alloc, dealloc}; +use core::alloc::Layout; +use core::ptr; + +/// Minimum chunk size in bytes (`_PY_DATA_STACK_CHUNK_SIZE`). +const MIN_CHUNK_SIZE: usize = 16 * 1024; + +/// Extra headroom (in bytes) to avoid allocating a new chunk for the next +/// frame right after growing. +const MINIMUM_OVERHEAD: usize = 1000 * core::mem::size_of::<usize>(); + +/// Alignment for all data stack allocations. +const ALIGN: usize = 16; + +/// Header for a data stack chunk. The usable data region starts right after +/// this header (aligned to `ALIGN`). +#[repr(C)] +struct DataStackChunk { + /// Previous chunk in the linked list (NULL for the root chunk). + previous: *mut DataStackChunk, + /// Total allocation size in bytes (including this header). + size: usize, + /// Saved `top` offset when a newer chunk was pushed. Used to restore + /// `DataStack::top` when popping back to this chunk. + saved_top: usize, +} + +impl DataStackChunk { + /// Pointer to the first usable byte after the header (aligned). + #[inline(always)] + fn data_start(&self) -> *mut u8 { + let header_end = (self as *const Self as usize) + core::mem::size_of::<Self>(); + let aligned = (header_end + ALIGN - 1) & !(ALIGN - 1); + aligned as *mut u8 + } + + /// Pointer past the last usable byte. + #[inline(always)] + fn data_limit(&self) -> *mut u8 { + unsafe { (self as *const Self as *mut u8).add(self.size) } + } +} + +/// Per-thread data stack for bump-allocating frame-local data. +pub struct DataStack { + /// Current chunk. + chunk: *mut DataStackChunk, + /// Current allocation position within the current chunk. + top: *mut u8, + /// End of usable space in the current chunk. + limit: *mut u8, +} + +impl DataStack { + /// Create a new data stack with an initial root chunk. + pub fn new() -> Self { + let chunk = Self::alloc_chunk(MIN_CHUNK_SIZE, ptr::null_mut()); + let top = unsafe { (*chunk).data_start() }; + let limit = unsafe { (*chunk).data_limit() }; + // Skip one ALIGN-sized slot in the root chunk so that `pop()` never + // frees it (`push_chunk` convention). + let top = unsafe { top.add(ALIGN) }; + Self { chunk, top, limit } + } + + /// Check if the current chunk has at least `size` bytes available. + #[inline(always)] + pub fn has_space(&self, size: usize) -> bool { + let aligned_size = (size + ALIGN - 1) & !(ALIGN - 1); + (self.limit as usize).saturating_sub(self.top as usize) >= aligned_size + } + + /// Allocate `size` bytes from the data stack. + /// + /// Returns a pointer to the allocated region (aligned to `ALIGN`). + /// The caller must call `pop()` with the returned pointer when done + /// (LIFO order). + #[inline(always)] + pub fn push(&mut self, size: usize) -> *mut u8 { + let aligned_size = (size + ALIGN - 1) & !(ALIGN - 1); + unsafe { + if self.top.add(aligned_size) <= self.limit { + let ptr = self.top; + self.top = self.top.add(aligned_size); + ptr + } else { + self.push_slow(aligned_size) + } + } + } + + /// Slow path: allocate a new chunk and push from it. + #[cold] + #[inline(never)] + fn push_slow(&mut self, aligned_size: usize) -> *mut u8 { + let mut chunk_size = MIN_CHUNK_SIZE; + let needed = aligned_size + .checked_add(MINIMUM_OVERHEAD) + .and_then(|v| v.checked_add(core::mem::size_of::<DataStackChunk>())) + .and_then(|v| v.checked_add(ALIGN)) + .expect("DataStack chunk size overflow"); + while chunk_size < needed { + chunk_size = chunk_size + .checked_mul(2) + .expect("DataStack chunk size overflow"); + } + // Save current position in old chunk. + unsafe { + (*self.chunk).saved_top = self.top as usize - self.chunk as usize; + } + let new_chunk = Self::alloc_chunk(chunk_size, self.chunk); + self.chunk = new_chunk; + let start = unsafe { (*new_chunk).data_start() }; + self.limit = unsafe { (*new_chunk).data_limit() }; + self.top = unsafe { start.add(aligned_size) }; + start + } + + /// Pop a previous allocation. `base` must be the pointer returned by + /// `push()`. Calls must be in LIFO order. + /// + /// # Safety + /// `base` must be a valid pointer returned by `push()` on this data stack, + /// and all allocations made after it must already have been popped. + #[inline(always)] + pub unsafe fn pop(&mut self, base: *mut u8) { + debug_assert!(!base.is_null()); + if self.is_in_current_chunk(base) { + // Common case: base is within the current chunk. + self.top = base; + } else { + // base is in a previous chunk — free the current chunk. + unsafe { self.pop_slow(base) }; + } + } + + /// Check if `ptr` falls within the current chunk's data area. + /// Both bounds are checked to handle non-monotonic allocation addresses + /// (e.g. on Windows where newer chunks may be at lower addresses). + #[inline(always)] + fn is_in_current_chunk(&self, ptr: *mut u8) -> bool { + let chunk_start = unsafe { (*self.chunk).data_start() }; + ptr >= chunk_start && ptr <= self.limit + } + + /// Slow path: pop back to a previous chunk. + #[cold] + #[inline(never)] + unsafe fn pop_slow(&mut self, base: *mut u8) { + loop { + let old_chunk = self.chunk; + let prev = unsafe { (*old_chunk).previous }; + debug_assert!(!prev.is_null(), "tried to pop past the root chunk"); + unsafe { Self::free_chunk(old_chunk) }; + self.chunk = prev; + self.limit = unsafe { (*prev).data_limit() }; + if self.is_in_current_chunk(base) { + self.top = base; + return; + } + } + } + + /// Allocate a new chunk. + fn alloc_chunk(size: usize, previous: *mut DataStackChunk) -> *mut DataStackChunk { + let layout = Layout::from_size_align(size, ALIGN).expect("invalid chunk layout"); + let ptr = unsafe { alloc(layout) }; + if ptr.is_null() { + alloc::alloc::handle_alloc_error(layout); + } + let chunk = ptr as *mut DataStackChunk; + unsafe { + (*chunk).previous = previous; + (*chunk).size = size; + (*chunk).saved_top = 0; + } + chunk + } + + /// Free a chunk. + unsafe fn free_chunk(chunk: *mut DataStackChunk) { + let size = unsafe { (*chunk).size }; + let layout = Layout::from_size_align(size, ALIGN).expect("invalid chunk layout"); + unsafe { dealloc(chunk as *mut u8, layout) }; + } +} + +// SAFETY: DataStack is per-thread and not shared. The raw pointers +// it contains point to memory exclusively owned by this DataStack. +unsafe impl Send for DataStack {} + +impl Default for DataStack { + fn default() -> Self { + Self::new() + } +} + +impl Drop for DataStack { + fn drop(&mut self) { + let mut chunk = self.chunk; + while !chunk.is_null() { + let prev = unsafe { (*chunk).previous }; + unsafe { Self::free_chunk(chunk) }; + chunk = prev; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic_push_pop() { + let mut ds = DataStack::new(); + let p1 = ds.push(64); + assert!(!p1.is_null()); + let p2 = ds.push(128); + assert!(!p2.is_null()); + assert!(p2 > p1); + unsafe { + ds.pop(p2); + ds.pop(p1); + } + } + + #[test] + fn cross_chunk_push_pop() { + let mut ds = DataStack::new(); + // Push enough to force a new chunk + let mut ptrs = Vec::new(); + for _ in 0..100 { + ptrs.push(ds.push(1024)); + } + // Pop all in reverse + for p in ptrs.into_iter().rev() { + unsafe { ds.pop(p) }; + } + } + + #[test] + fn alignment() { + let mut ds = DataStack::new(); + for size in [1, 7, 15, 16, 17, 31, 32, 33, 64, 100] { + let p = ds.push(size); + assert_eq!(p as usize % ALIGN, 0, "alignment violated for size {size}"); + unsafe { ds.pop(p) }; + } + } +} diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs new file mode 100644 index 00000000000..763fa856319 --- /dev/null +++ b/crates/vm/src/dict_inner.rs @@ -0,0 +1,1276 @@ +//! Ordered dictionary implementation. +//! Inspired by: <https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html> +//! And: <https://www.youtube.com/watch?v=p33CVV29OG8> +//! And: <http://code.activestate.com/recipes/578375/> + +use crate::{ + AsObject, Py, PyExact, PyObject, PyObjectRef, PyRefExact, PyResult, VirtualMachine, + builtins::{PyBytes, PyInt, PyStr, PyStrInterned, PyStrRef, PyUtf8Str, PyUtf8StrRef}, + convert::ToPyObject, +}; +use crate::{ + common::{ + hash, + lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}, + wtf8::{Wtf8, Wtf8Buf}, + }, + object::{Traverse, TraverseFn}, +}; +use alloc::fmt; +use core::mem::size_of; +use core::ops::ControlFlow; +use core::sync::atomic::{ + AtomicU64, + Ordering::{Acquire, Release}, +}; +use num_traits::ToPrimitive; + +// HashIndex is intended to be same size with hash::PyHash +// but it doesn't mean the values are compatible with actual PyHash value + +/// hash value of an object returned by __hash__ +type HashValue = hash::PyHash; +/// index calculated by resolving collision +type HashIndex = hash::PyHash; +/// index into dict.indices +type IndexIndex = usize; +/// index into dict.entries +type EntryIndex = usize; + +pub struct Dict<T = PyObjectRef> { + inner: PyRwLock<DictInner<T>>, + version: AtomicU64, +} + +unsafe impl<T: Traverse> Traverse for Dict<T> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.inner.traverse(tracer_fn); + } +} + +impl<T> fmt::Debug for Dict<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Debug").finish() + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +struct IndexEntry(i64); + +impl IndexEntry { + const FREE: Self = Self(-1); + const DUMMY: Self = Self(-2); + + /// # Safety + /// idx must not be one of FREE or DUMMY + const unsafe fn from_index_unchecked(idx: usize) -> Self { + debug_assert!((idx as isize) >= 0); + Self(idx as i64) + } + + const fn index(self) -> Option<usize> { + if self.0 >= 0 { + Some(self.0 as usize) + } else { + None + } + } +} + +#[derive(Clone)] +struct DictInner<T> { + used: usize, + filled: usize, + indices: Vec<IndexEntry>, + entries: Vec<Option<DictEntry<T>>>, +} + +unsafe impl<T: Traverse> Traverse for DictInner<T> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.entries + .iter() + .map(|v| { + if let Some(v) = v { + v.key.traverse(tracer_fn); + v.value.traverse(tracer_fn); + } + }) + .count(); + } +} + +impl<T: Clone> Clone for Dict<T> { + fn clone(&self) -> Self { + Self { + inner: PyRwLock::new(self.inner.read().clone()), + version: AtomicU64::new(0), + } + } +} + +impl<T> Default for Dict<T> { + fn default() -> Self { + Self { + inner: PyRwLock::new(DictInner { + used: 0, + filled: 0, + indices: vec![IndexEntry::FREE; 8], + entries: Vec::new(), + }), + version: AtomicU64::new(0), + } + } +} + +#[derive(Clone)] +struct DictEntry<T> { + hash: HashValue, + key: PyObjectRef, + index: IndexIndex, + value: T, +} +static_assertions::assert_eq_size!(DictEntry<PyObjectRef>, Option<DictEntry<PyObjectRef>>); + +#[derive(Debug, PartialEq, Eq)] +pub struct DictSize { + indices_size: usize, + pub entries_size: usize, + pub used: usize, + filled: usize, +} + +struct GenIndexes { + idx: HashIndex, + perturb: HashValue, + mask: HashIndex, +} + +impl GenIndexes { + const fn new(hash: HashValue, mask: HashIndex) -> Self { + let hash = hash.abs(); + Self { + idx: hash, + perturb: hash, + mask, + } + } + + const fn next(&mut self) -> usize { + let prev = self.idx; + self.idx = prev + .wrapping_mul(5) + .wrapping_add(self.perturb) + .wrapping_add(1); + self.perturb >>= 5; + (prev & self.mask) as usize + } +} + +impl<T> DictInner<T> { + fn resize(&mut self, new_size: usize) { + let new_size = { + let mut i = 1; + while i < new_size { + i <<= 1; + } + i + }; + self.indices = vec![IndexEntry::FREE; new_size]; + let mask = (new_size - 1) as i64; + for (entry_idx, entry) in self.entries.iter_mut().enumerate() { + if let Some(entry) = entry { + let mut idxs = GenIndexes::new(entry.hash, mask); + loop { + let index_index = idxs.next(); + unsafe { + // Safety: index is always valid here + // index_index is generated by idxs + // entry_idx is saved one + let idx = self.indices.get_unchecked_mut(index_index); + if *idx == IndexEntry::FREE { + *idx = IndexEntry::from_index_unchecked(entry_idx); + entry.index = index_index; + break; + } + } + } + } else { + //removed entry + } + } + self.filled = self.used; + } + + fn unchecked_push( + &mut self, + index: IndexIndex, + hash_value: HashValue, + key: PyObjectRef, + value: T, + index_entry: IndexEntry, + ) { + let entry = DictEntry { + hash: hash_value, + key, + value, + index, + }; + let entry_index = self.entries.len(); + self.entries.push(Some(entry)); + self.indices[index] = unsafe { + // SAFETY: entry_index is self.entries.len(). it never can + // grow to `usize-2` because hash tables cannot full its index + IndexEntry::from_index_unchecked(entry_index) + }; + self.used += 1; + if let IndexEntry::FREE = index_entry { + self.filled += 1; + if let Some(new_size) = self.should_resize() { + self.resize(new_size) + } + } + } + + const fn size(&self) -> DictSize { + DictSize { + indices_size: self.indices.len(), + entries_size: self.entries.len(), + used: self.used, + filled: self.filled, + } + } + + #[inline] + const fn should_resize(&self) -> Option<usize> { + if self.filled * 3 > self.indices.len() * 2 { + Some(self.used * 2) + } else { + None + } + } + + #[inline] + fn get_entry_checked(&self, idx: EntryIndex, index_index: IndexIndex) -> Option<&DictEntry<T>> { + match self.entries.get(idx) { + Some(Some(entry)) if entry.index == index_index => Some(entry), + _ => None, + } + } +} + +type PopInnerResult<T> = ControlFlow<Option<DictEntry<T>>>; + +impl<T: Clone> Dict<T> { + /// Monotonically increasing version counter for mutation tracking. + pub fn version(&self) -> u64 { + self.version.load(Acquire) + } + + /// Bump the version counter after any mutation. + fn bump_version(&self) { + self.version.fetch_add(1, Release); + } + + fn read(&self) -> PyRwLockReadGuard<'_, DictInner<T>> { + self.inner.read() + } + + fn write(&self) -> PyRwLockWriteGuard<'_, DictInner<T>> { + self.inner.write() + } + + /// Store a key + pub fn insert<K>(&self, vm: &VirtualMachine, key: &K, value: T) -> PyResult<()> + where + K: DictKey + ?Sized, + { + let hash = key.key_hash(vm)?; + let _removed = loop { + let (entry_index, index_index) = self.lookup(vm, key, hash, None)?; + let mut inner = self.write(); + if let Some(index) = entry_index.index() { + // Update existing key + if let Some(entry) = inner.entries.get_mut(index) { + let Some(entry) = entry.as_mut() else { + // The dict was changed since we did lookup. Let's try again. + // this is very rare to happen + // (and seems only happen with very high freq gc, and about one time in 10000 iters) + // but still possible + continue; + }; + if entry.index == index_index { + let removed = core::mem::replace(&mut entry.value, value); + self.bump_version(); + // defer dec RC + break Some(removed); + } else { + // stuff shifted around, let's try again + } + } else { + // The dict was changed since we did lookup. Let's try again. + } + } else { + // New key - validate slot is still what lookup found + if inner.indices.get(index_index) != Some(&entry_index) { + // Dict was resized since lookup, retry + continue; + } + inner.unchecked_push(index_index, hash, key.to_pyobject(vm), value, entry_index); + self.bump_version(); + break None; + } + }; + Ok(()) + } + + pub fn contains<K: DictKey + ?Sized>(&self, vm: &VirtualMachine, key: &K) -> PyResult<bool> { + let key_hash = key.key_hash(vm)?; + let (entry, _) = self.lookup(vm, key, key_hash, None)?; + Ok(entry.index().is_some()) + } + + /// Retrieve a key + #[cfg_attr(feature = "flame-it", flame("Dict"))] + pub fn get<K: DictKey + ?Sized>(&self, vm: &VirtualMachine, key: &K) -> PyResult<Option<T>> { + let hash = key.key_hash(vm)?; + self._get_inner(vm, key, hash) + } + + /// Return a stable entry hint for `key` if present. + /// + /// The hint is the internal entry index and can be used with + /// [`Self::get_hint`]. It is invalidated by dict mutations. + pub fn hint_for_key<K: DictKey + ?Sized>( + &self, + vm: &VirtualMachine, + key: &K, + ) -> PyResult<Option<u16>> { + let hash = key.key_hash(vm)?; + let (entry, _) = self.lookup(vm, key, hash, None)?; + let Some(index) = entry.index() else { + return Ok(None); + }; + Ok(u16::try_from(index).ok()) + } + + /// Fast path lookup using a cached entry index (`hint`). + /// + /// Returns `None` if the hint is stale or the key no longer matches. + pub fn get_hint<K: DictKey + ?Sized>( + &self, + vm: &VirtualMachine, + key: &K, + hint: usize, + ) -> PyResult<Option<T>> { + let (entry_key, entry_value) = { + let inner = self.read(); + let Some(Some(entry)) = inner.entries.get(hint) else { + return Ok(None); + }; + if key.key_is(&entry.key) { + return Ok(Some(entry.value.clone())); + } + (entry.key.clone(), entry.value.clone()) + }; + // key_eq may run Python __eq__, so must be outside the lock. + if key.key_eq(vm, &entry_key)? { + Ok(Some(entry_value)) + } else { + Ok(None) + } + } + + fn _get_inner<K: DictKey + ?Sized>( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + ) -> PyResult<Option<T>> { + let ret = loop { + let (entry, index_index) = self.lookup(vm, key, hash, None)?; + if let Some(index) = entry.index() { + let inner = self.read(); + if let Some(entry) = inner.get_entry_checked(index, index_index) { + break Some(entry.value.clone()); + } else { + // The dict was changed since we did lookup. Let's try again. + continue; + } + } else { + break None; + } + }; + Ok(ret) + } + + pub fn get_chain<K: DictKey + ?Sized>( + &self, + other: &Self, + vm: &VirtualMachine, + key: &K, + ) -> PyResult<Option<T>> { + let hash = key.key_hash(vm)?; + if let Some(x) = self._get_inner(vm, key, hash)? { + Ok(Some(x)) + } else { + other._get_inner(vm, key, hash) + } + } + + pub fn clear(&self) { + let _removed = { + let mut inner = self.write(); + inner.indices.clear(); + inner.indices.resize(8, IndexEntry::FREE); + inner.used = 0; + inner.filled = 0; + self.bump_version(); + // defer dec rc + core::mem::take(&mut inner.entries) + }; + } + + /// Delete a key + pub fn delete<K>(&self, vm: &VirtualMachine, key: &K) -> PyResult<()> + where + K: DictKey + ?Sized, + { + if self.remove_if_exists(vm, key)?.is_some() { + Ok(()) + } else { + Err(vm.new_key_error(key.to_pyobject(vm))) + } + } + + pub fn delete_if_exists<K>(&self, vm: &VirtualMachine, key: &K) -> PyResult<bool> + where + K: DictKey + ?Sized, + { + self.remove_if_exists(vm, key).map(|opt| opt.is_some()) + } + + pub fn delete_if<K, F>(&self, vm: &VirtualMachine, key: &K, pred: F) -> PyResult<bool> + where + K: DictKey + ?Sized, + F: Fn(&T) -> PyResult<bool>, + { + self.remove_if(vm, key, pred).map(|opt| opt.is_some()) + } + + pub fn remove_if_exists<K>(&self, vm: &VirtualMachine, key: &K) -> PyResult<Option<T>> + where + K: DictKey + ?Sized, + { + self.remove_if(vm, key, |_| Ok(true)) + } + + /// pred should be VERY CAREFUL about what it does as it is called while + /// the dict's internal mutex is held + pub(crate) fn remove_if<K, F>( + &self, + vm: &VirtualMachine, + key: &K, + pred: F, + ) -> PyResult<Option<T>> + where + K: DictKey + ?Sized, + F: Fn(&T) -> PyResult<bool>, + { + let hash = key.key_hash(vm)?; + let removed = loop { + let lookup = self.lookup(vm, key, hash, None)?; + match self.pop_inner_if(lookup, &pred)? { + ControlFlow::Break(entry) => break entry, + ControlFlow::Continue(()) => continue, + } + }; + Ok(removed.map(|entry| entry.value)) + } + + pub fn delete_or_insert(&self, vm: &VirtualMachine, key: &PyObject, value: T) -> PyResult<()> { + let hash = key.key_hash(vm)?; + let _removed = loop { + let lookup = self.lookup(vm, key, hash, None)?; + let (entry, index_index) = lookup; + if entry.index().is_some() { + match self.pop_inner(lookup) { + ControlFlow::Break(Some(entry)) => break Some(entry), + _ => continue, + } + } else { + let mut inner = self.write(); + if inner.indices.get(index_index) != Some(&entry) { + continue; + } + inner.unchecked_push(index_index, hash, key.to_owned(), value, entry); + self.bump_version(); + break None; + } + }; + Ok(()) + } + + pub fn setdefault<K, F>(&self, vm: &VirtualMachine, key: &K, default: F) -> PyResult<T> + where + K: DictKey + ?Sized, + F: FnOnce() -> T, + { + let hash = key.key_hash(vm)?; + let mut default = Some(default); + loop { + let (index_entry, index_index) = self.lookup(vm, key, hash, None)?; + if let Some(index) = index_entry.index() { + let inner = self.read(); + if let Some(entry) = inner.get_entry_checked(index, index_index) { + return Ok(entry.value.clone()); + } + continue; + } + let mut inner = self.write(); + if inner.indices.get(index_index) != Some(&index_entry) { + continue; + } + let value = default + .take() + .expect("default must only be computed on insertion")(); + inner.unchecked_push( + index_index, + hash, + key.to_pyobject(vm), + value.clone(), + index_entry, + ); + self.bump_version(); + return Ok(value); + } + } + + #[allow(dead_code)] + pub fn setdefault_entry<K, F>( + &self, + vm: &VirtualMachine, + key: &K, + default: F, + ) -> PyResult<(PyObjectRef, T)> + where + K: DictKey + ?Sized, + F: FnOnce() -> T, + { + let hash = key.key_hash(vm)?; + let mut default = Some(default); + loop { + let (index_entry, index_index) = self.lookup(vm, key, hash, None)?; + if let Some(index) = index_entry.index() { + let inner = self.read(); + if let Some(entry) = inner.get_entry_checked(index, index_index) { + return Ok((entry.key.clone(), entry.value.clone())); + } + continue; + } + let mut inner = self.write(); + if inner.indices.get(index_index) != Some(&index_entry) { + continue; + } + let value = default + .take() + .expect("default must only be computed on insertion")(); + let key_obj = key.to_pyobject(vm); + let ret = (key_obj.clone(), value.clone()); + inner.unchecked_push(index_index, hash, key_obj, value, index_entry); + self.bump_version(); + return Ok(ret); + } + } + + pub fn len(&self) -> usize { + self.read().used + } + + pub fn size(&self) -> DictSize { + self.read().size() + } + + pub fn next_entry(&self, mut position: EntryIndex) -> Option<(usize, PyObjectRef, T)> { + let inner = self.read(); + loop { + let entry = inner.entries.get(position)?; + position += 1; + if let Some(entry) = entry { + break Some((position, entry.key.clone(), entry.value.clone())); + } + } + } + + pub fn prev_entry(&self, mut position: EntryIndex) -> Option<(usize, PyObjectRef, T)> { + let inner = self.read(); + loop { + let entry = inner.entries.get(position)?; + position = position.saturating_sub(1); + if let Some(entry) = entry { + break Some((position, entry.key.clone(), entry.value.clone())); + } + } + } + + pub fn len_from_entry_index(&self, position: EntryIndex) -> usize { + self.read().entries.len().saturating_sub(position) + } + + pub fn has_changed_size(&self, old: &DictSize) -> bool { + let current = self.read().size(); + current != *old + } + + pub fn keys(&self) -> Vec<PyObjectRef> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| v.key.clone())) + .collect() + } + + pub fn values(&self) -> Vec<T> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| v.value.clone())) + .collect() + } + + pub fn items(&self) -> Vec<(PyObjectRef, T)> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| (v.key.clone(), v.value.clone()))) + .collect() + } + + pub fn try_fold_keys<Acc, Fold>(&self, init: Acc, f: Fold) -> PyResult<Acc> + where + Fold: FnMut(Acc, &PyObject) -> PyResult<Acc>, + { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| v.key.as_object())) + .try_fold(init, f) + } + + /// Lookup the index for the given key. + #[cfg_attr(feature = "flame-it", flame("Dict"))] + fn lookup<K: DictKey + ?Sized>( + &self, + vm: &VirtualMachine, + key: &K, + hash_value: HashValue, + mut lock: Option<PyRwLockReadGuard<'_, DictInner<T>>>, + ) -> PyResult<LookupResult> { + let mut idxs = None; + let mut free_slot = None; + let ret = 'outer: loop { + let (entry_key, ret) = { + let inner = lock.take().unwrap_or_else(|| self.read()); + let mask = (inner.indices.len() - 1) as i64; + let idxs = idxs.get_or_insert_with(|| GenIndexes::new(hash_value, mask)); + if idxs.mask != mask { + // Dict was resized since last probe, restart + *idxs = GenIndexes::new(hash_value, mask); + free_slot = None; + } + loop { + let index_index = idxs.next(); + let index_entry = *unsafe { + // Safety: index_index is generated + inner.indices.get_unchecked(index_index) + }; + match index_entry { + IndexEntry::DUMMY => { + if free_slot.is_none() { + free_slot = Some(index_index); + } + } + IndexEntry::FREE => { + let idxs = match free_slot { + Some(free) => (IndexEntry::DUMMY, free), + None => (IndexEntry::FREE, index_index), + }; + return Ok(idxs); + } + idx => { + let entry = unsafe { + // Safety: DUMMY and FREE are already handled above. + // i is always valid and entry always exists. + let i = idx.index().unwrap_unchecked(); + inner.entries.get_unchecked(i).as_ref().unwrap_unchecked() + }; + let ret = (idx, index_index); + if key.key_is(&entry.key) { + break 'outer ret; + } else if entry.hash == hash_value { + break (entry.key.clone(), ret); + } else { + // entry mismatch + } + } + } + // warn!("Perturb value: {}", i); + } + }; + // This comparison needs to be done outside the lock. + if key.key_eq(vm, &entry_key)? { + break 'outer ret; + } else { + // hash collision + } + + // warn!("Perturb value: {}", i); + }; + Ok(ret) + } + + // returns Err(()) if changed since lookup + fn pop_inner(&self, lookup: LookupResult) -> PopInnerResult<T> { + self.pop_inner_if(lookup, |_| Ok::<_, core::convert::Infallible>(true)) + .unwrap_or_else(|x| match x {}) + } + + fn pop_inner_if<E>( + &self, + lookup: LookupResult, + pred: impl Fn(&T) -> Result<bool, E>, + ) -> Result<PopInnerResult<T>, E> { + let (entry_index, index_index) = lookup; + let Some(entry_index) = entry_index.index() else { + return Ok(ControlFlow::Break(None)); + }; + let inner = &mut *self.write(); + let slot = if let Some(slot) = inner.entries.get_mut(entry_index) { + slot + } else { + // The dict was changed since we did lookup. Let's try again. + return Ok(ControlFlow::Continue(())); + }; + match slot { + Some(entry) if entry.index == index_index => { + if !pred(&entry.value)? { + return Ok(ControlFlow::Break(None)); + } + } + // The dict was changed since we did lookup. Let's try again. + _ => return Ok(ControlFlow::Continue(())), + } + *unsafe { + // index_index is result of lookup + inner.indices.get_unchecked_mut(index_index) + } = IndexEntry::DUMMY; + inner.used -= 1; + let removed = slot.take(); + self.bump_version(); + Ok(ControlFlow::Break(removed)) + } + + /// Retrieve and delete a key + pub fn pop<K: DictKey + ?Sized>(&self, vm: &VirtualMachine, key: &K) -> PyResult<Option<T>> { + let hash_value = key.key_hash(vm)?; + let removed = loop { + let lookup = self.lookup(vm, key, hash_value, None)?; + match self.pop_inner(lookup) { + ControlFlow::Break(entry) => break entry.map(|e| e.value), + ControlFlow::Continue(()) => continue, + } + }; + Ok(removed) + } + + pub fn pop_back(&self) -> Option<(PyObjectRef, T)> { + let inner = &mut *self.write(); + let entry = loop { + let entry = inner.entries.pop()?; + if let Some(entry) = entry { + break entry; + } + }; + inner.used -= 1; + *unsafe { + // entry.index always refers valid index + inner.indices.get_unchecked_mut(entry.index) + } = IndexEntry::DUMMY; + self.bump_version(); + Some((entry.key, entry.value)) + } + + pub fn sizeof(&self) -> usize { + let inner = self.read(); + size_of::<Self>() + + size_of::<DictInner<T>>() + + inner.indices.len() * size_of::<i64>() + + inner.entries.len() * size_of::<DictEntry<T>>() + } + + /// Pop all entries from the dict, returning (key, value) pairs. + /// This is used for circular reference resolution in GC. + /// Requires &mut self to avoid lock contention. + pub fn drain_entries(&mut self) -> impl Iterator<Item = (PyObjectRef, T)> + '_ { + let inner = self.inner.get_mut(); + inner.used = 0; + inner.filled = 0; + inner.indices.iter_mut().for_each(|i| *i = IndexEntry::FREE); + inner.entries.drain(..).flatten().map(|e| (e.key, e.value)) + } +} + +type LookupResult = (IndexEntry, IndexIndex); + +/// Types implementing this trait can be used to index +/// the dictionary. Typical use-cases are: +/// - PyObjectRef -> arbitrary python type used as key +/// - str -> string reference used as key, this is often used internally +pub trait DictKey { + type Owned: ToPyObject; + fn _to_owned(&self, vm: &VirtualMachine) -> Self::Owned; + fn to_pyobject(&self, vm: &VirtualMachine) -> PyObjectRef { + self._to_owned(vm).to_pyobject(vm) + } + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue>; + fn key_is(&self, other: &PyObject) -> bool; + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool>; + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize>; +} + +/// Implement trait for PyObjectRef such that we can use python objects +/// to index dictionaries. +impl DictKey for PyObject { + type Owned = PyObjectRef; + #[inline(always)] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + + #[inline(always)] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + self.hash(vm) + } + + #[inline(always)] + fn key_is(&self, other: &PyObject) -> bool { + self.is(other) + } + + #[inline(always)] + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + vm.identical_or_equal(self, other_key) + } + + #[inline] + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + self.try_index(vm)?.try_to_primitive(vm) + } +} + +impl DictKey for Py<PyStr> { + type Owned = PyStrRef; + #[inline(always)] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + + #[inline] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + Ok(self.hash(vm)) + } + + #[inline(always)] + fn key_is(&self, other: &PyObject) -> bool { + self.is(other) + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + if self.is(other_key) { + Ok(true) + } else if let Some(pystr) = other_key.downcast_ref_if_exact::<PyStr>(vm) { + Ok(self.as_wtf8() == pystr.as_wtf8()) + } else { + vm.bool_eq(self.as_object(), other_key) + } + } + + #[inline(always)] + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + self.as_object().key_as_isize(vm) + } +} + +impl DictKey for Py<PyUtf8Str> { + type Owned = PyUtf8StrRef; + #[inline(always)] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + + #[inline] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + self.as_pystr().key_hash(vm) + } + + #[inline(always)] + fn key_is(&self, other: &PyObject) -> bool { + self.as_pystr().key_is(other) + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + self.as_pystr().key_eq(vm, other_key) + } + + #[inline(always)] + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + self.as_pystr().key_as_isize(vm) + } +} + +impl DictKey for PyStrInterned { + type Owned = PyRefExact<PyStr>; + + #[inline] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + let zelf: &'static Self = unsafe { &*(self as *const _) }; + zelf.to_exact() + } + + #[inline] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + (**self).key_hash(vm) + } + + #[inline] + fn key_is(&self, other: &PyObject) -> bool { + (**self).key_is(other) + } + + #[inline] + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + (**self).key_eq(vm, other_key) + } + + #[inline] + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + (**self).key_as_isize(vm) + } +} + +impl DictKey for PyExact<PyStr> { + type Owned = PyRefExact<PyStr>; + + #[inline] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + + #[inline(always)] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + (**self).key_hash(vm) + } + + #[inline(always)] + fn key_is(&self, other: &PyObject) -> bool { + (**self).key_is(other) + } + + #[inline(always)] + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + (**self).key_eq(vm, other_key) + } + + #[inline(always)] + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + (**self).key_as_isize(vm) + } +} + +// AsRef<str> fit this case but not possible in rust 1.46 + +/// Implement trait for the str type, so that we can use strings +/// to index dictionaries. +impl DictKey for str { + type Owned = String; + + #[inline(always)] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + + #[inline] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + // follow a similar route as the hashing of PyStrRef + Ok(vm.state.hash_secret.hash_str(self)) + } + + #[inline(always)] + fn key_is(&self, _other: &PyObject) -> bool { + // No matter who the other pyobject is, we are never the same thing, since + // we are a str, not a pyobject. + false + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + if let Some(pystr) = other_key.downcast_ref_if_exact::<PyStr>(vm) { + Ok(pystr.as_wtf8() == self) + } else { + // Fall back to PyObjectRef implementation. + let s = vm.ctx.new_str(self); + s.key_eq(vm, other_key) + } + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + Err(vm.new_type_error("'str' object cannot be interpreted as an integer")) + } +} + +impl DictKey for String { + type Owned = Self; + + #[inline] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.clone() + } + + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + self.as_str().key_hash(vm) + } + + fn key_is(&self, other: &PyObject) -> bool { + self.as_str().key_is(other) + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + self.as_str().key_eq(vm, other_key) + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + self.as_str().key_as_isize(vm) + } +} + +impl DictKey for Wtf8 { + type Owned = Wtf8Buf; + + #[inline(always)] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + + #[inline] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + // follow a similar route as the hashing of PyStrRef + Ok(vm.state.hash_secret.hash_bytes(self.as_bytes())) + } + + #[inline(always)] + fn key_is(&self, _other: &PyObject) -> bool { + // No matter who the other pyobject is, we are never the same thing, since + // we are a str, not a pyobject. + false + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + if let Some(pystr) = other_key.downcast_ref_if_exact::<PyStr>(vm) { + Ok(pystr.as_wtf8() == self) + } else { + // Fall back to PyObjectRef implementation. + let s = vm.ctx.new_str(self); + s.key_eq(vm, other_key) + } + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + Err(vm.new_type_error("'str' object cannot be interpreted as an integer")) + } +} + +impl DictKey for Wtf8Buf { + type Owned = Self; + + #[inline] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.clone() + } + + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + (**self).key_hash(vm) + } + + fn key_is(&self, other: &PyObject) -> bool { + (**self).key_is(other) + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + (**self).key_eq(vm, other_key) + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + (**self).key_as_isize(vm) + } +} + +impl DictKey for [u8] { + type Owned = Vec<u8>; + + #[inline(always)] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.to_owned() + } + + #[inline] + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + // follow a similar route as the hashing of PyStrRef + Ok(vm.state.hash_secret.hash_bytes(self)) + } + + #[inline(always)] + fn key_is(&self, _other: &PyObject) -> bool { + // No matter who the other pyobject is, we are never the same thing, since + // we are a str, not a pyobject. + false + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + if let Some(pystr) = other_key.downcast_ref_if_exact::<PyBytes>(vm) { + Ok(pystr.as_bytes() == self) + } else { + // Fall back to PyObjectRef implementation. + let s = vm.ctx.new_bytes(self.to_vec()); + s.key_eq(vm, other_key) + } + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + Err(vm.new_type_error("'str' object cannot be interpreted as an integer")) + } +} + +impl DictKey for Vec<u8> { + type Owned = Self; + + #[inline] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + self.clone() + } + + fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { + self.as_slice().key_hash(vm) + } + + fn key_is(&self, other: &PyObject) -> bool { + self.as_slice().key_is(other) + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + self.as_slice().key_eq(vm, other_key) + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { + self.as_slice().key_as_isize(vm) + } +} + +impl DictKey for usize { + type Owned = Self; + + #[inline] + fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { + *self + } + + fn key_hash(&self, _vm: &VirtualMachine) -> PyResult<HashValue> { + Ok(hash::hash_usize(*self)) + } + + fn key_is(&self, _other: &PyObject) -> bool { + false + } + + fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { + if let Some(int) = other_key.downcast_ref_if_exact::<PyInt>(vm) { + if let Some(i) = int.as_bigint().to_usize() { + Ok(i == *self) + } else { + Ok(false) + } + } else { + let int = vm.ctx.new_int(*self); + vm.bool_eq(int.as_ref(), other_key) + } + } + + fn key_as_isize(&self, _vm: &VirtualMachine) -> PyResult<isize> { + Ok(*self as isize) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Interpreter, common::ascii}; + + #[test] + fn test_insert() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let dict = Dict::default(); + assert_eq!(0, dict.len()); + + let key1 = vm.new_pyobj(true); + let value1 = vm.new_pyobj(ascii!("abc")); + dict.insert(vm, &*key1, value1).unwrap(); + assert_eq!(1, dict.len()); + + let key2 = vm.new_pyobj(ascii!("x")); + let value2 = vm.new_pyobj(ascii!("def")); + dict.insert(vm, &*key2, value2.clone()).unwrap(); + assert_eq!(2, dict.len()); + + dict.insert(vm, &*key1, value2.clone()).unwrap(); + assert_eq!(2, dict.len()); + + dict.delete(vm, &*key1).unwrap(); + assert_eq!(1, dict.len()); + + dict.insert(vm, &*key1, value2.clone()).unwrap(); + assert_eq!(2, dict.len()); + + assert!(dict.contains(vm, &*key1).unwrap()); + assert!(dict.contains(vm, "x").unwrap()); + + let val = dict.get(vm, "x").unwrap().unwrap(); + vm.bool_eq(&val, &value2) + .expect("retrieved value must be equal to inserted value."); + }) + } + + macro_rules! hash_tests { + ($($name:ident: $example_hash:expr,)*) => { + $( + #[test] + fn $name() { + check_hash_equivalence($example_hash); + } + )* + } + } + + hash_tests! { + test_abc: "abc", + test_x: "x", + } + + fn check_hash_equivalence(text: &str) { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let value1 = text; + let value2 = vm.new_pyobj(value1.to_owned()); + + let hash1 = value1.key_hash(vm).expect("Hash should not fail."); + let hash2 = value2.key_hash(vm).expect("Hash should not fail."); + assert_eq!(hash1, hash2); + }) + } +} diff --git a/vm/src/eval.rs b/crates/vm/src/eval.rs similarity index 87% rename from vm/src/eval.rs rename to crates/vm/src/eval.rs index 35f27dc9d68..be09b3e4cc9 100644 --- a/vm/src/eval.rs +++ b/crates/vm/src/eval.rs @@ -1,9 +1,9 @@ -use crate::{compiler, scope::Scope, PyResult, VirtualMachine}; +use crate::{PyResult, VirtualMachine, compiler, scope::Scope}; pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult { match vm.compile(source, compiler::Mode::Eval, source_path.to_owned()) { Ok(bytecode) => { - debug!("Code object: {:?}", bytecode); + debug!("Code object: {bytecode:?}"); vm.run_code_obj(bytecode, scope) } Err(err) => Err(vm.new_syntax_error(&err, Some(source))), diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs new file mode 100644 index 00000000000..02342e4003d --- /dev/null +++ b/crates/vm/src/exception_group.rs @@ -0,0 +1,483 @@ +//! ExceptionGroup implementation for Python 3.11+ +//! +//! This module implements BaseExceptionGroup and ExceptionGroup with multiple inheritance support. + +use crate::builtins::{PyList, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef}; +use crate::function::{ArgIterable, FuncArgs}; +use crate::types::{PyTypeFlags, PyTypeSlots}; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, +}; +use core::fmt::Write; +use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; + +use crate::exceptions::types::PyBaseException; + +/// Create dynamic ExceptionGroup type with multiple inheritance +fn create_exception_group(ctx: &Context) -> PyRef<PyType> { + let excs = &ctx.exceptions; + let exception_group_slots = PyTypeSlots { + flags: PyTypeFlags::heap_type_flags() | PyTypeFlags::HAS_DICT, + ..Default::default() + }; + PyType::new_heap( + "ExceptionGroup", + vec![ + excs.base_exception_group.to_owned(), + excs.exception_type.to_owned(), + ], + Default::default(), + exception_group_slots, + ctx.types.type_type.to_owned(), + ctx, + ) + .expect("Failed to create ExceptionGroup type with multiple inheritance") +} + +pub fn exception_group() -> &'static Py<PyType> { + ::rustpython_vm::common::static_cell! { + static CELL: ::rustpython_vm::builtins::PyTypeRef; + } + CELL.get_or_init(|| create_exception_group(Context::genesis())) +} + +pub(super) mod types { + use super::*; + use crate::PyPayload; + use crate::builtins::PyGenericAlias; + use crate::types::{Constructor, Initializer}; + + #[pyexception(name, base = PyBaseException, ctx = "base_exception_group")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyBaseExceptionGroup(PyBaseException); + + #[pyexception(with(Constructor, Initializer))] + impl PyBaseExceptionGroup { + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + #[pymethod] + fn derive( + zelf: PyRef<PyBaseException>, + excs: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let message = zelf.get_arg(0).unwrap_or_else(|| vm.ctx.new_str("").into()); + vm.invoke_exception( + vm.ctx.exceptions.base_exception_group.to_owned(), + vec![message, excs], + ) + .map(|e| e.into()) + } + + #[pymethod] + fn subgroup( + zelf: PyRef<PyBaseException>, + condition: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let matcher = get_condition_matcher(&condition, vm)?; + + // If self matches the condition entirely, return self + let zelf_obj: PyObjectRef = zelf.clone().into(); + if matcher.check(&zelf_obj, vm)? { + return Ok(zelf_obj); + } + + let exceptions = get_exceptions_tuple(&zelf, vm)?; + let mut matching: Vec<PyObjectRef> = Vec::new(); + let mut modified = false; + + for exc in exceptions { + if is_base_exception_group(&exc, vm) { + // Recursive call for nested groups + let subgroup_result = vm.call_method(&exc, "subgroup", (condition.clone(),))?; + if !vm.is_none(&subgroup_result) { + matching.push(subgroup_result.clone()); + } + if !subgroup_result.is(&exc) { + modified = true; + } + } else if matcher.check(&exc, vm)? { + matching.push(exc); + } else { + modified = true; + } + } + + if !modified { + return Ok(zelf.clone().into()); + } + + if matching.is_empty() { + return Ok(vm.ctx.none()); + } + + // Create new group with matching exceptions and copy metadata + derive_and_copy_attributes(&zelf, matching, vm) + } + + #[pymethod] + fn split( + zelf: PyRef<PyBaseException>, + condition: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyTupleRef> { + let matcher = get_condition_matcher(&condition, vm)?; + + // If self matches the condition entirely + let zelf_obj: PyObjectRef = zelf.clone().into(); + if matcher.check(&zelf_obj, vm)? { + return Ok(vm.ctx.new_tuple(vec![zelf_obj, vm.ctx.none()])); + } + + let exceptions = get_exceptions_tuple(&zelf, vm)?; + let mut matching: Vec<PyObjectRef> = Vec::new(); + let mut rest: Vec<PyObjectRef> = Vec::new(); + + for exc in exceptions { + if is_base_exception_group(&exc, vm) { + let result = vm.call_method(&exc, "split", (condition.clone(),))?; + let result_tuple: PyTupleRef = result.try_into_value(vm)?; + let match_part = result_tuple + .first() + .cloned() + .unwrap_or_else(|| vm.ctx.none()); + let rest_part = result_tuple + .get(1) + .cloned() + .unwrap_or_else(|| vm.ctx.none()); + + if !vm.is_none(&match_part) { + matching.push(match_part); + } + if !vm.is_none(&rest_part) { + rest.push(rest_part); + } + } else if matcher.check(&exc, vm)? { + matching.push(exc); + } else { + rest.push(exc); + } + } + + let match_group = if matching.is_empty() { + vm.ctx.none() + } else { + derive_and_copy_attributes(&zelf, matching, vm)? + }; + + let rest_group = if rest.is_empty() { + vm.ctx.none() + } else { + derive_and_copy_attributes(&zelf, rest, vm)? + }; + + Ok(vm.ctx.new_tuple(vec![match_group, rest_group])) + } + + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let message = zelf.get_arg(0).map(|m| m.str(vm)).transpose()?; + + let num_excs = zelf + .get_arg(1) + .and_then(|obj| obj.downcast_ref::<PyTuple>().map(|t| t.len())) + .unwrap_or(0); + + let suffix = if num_excs == 1 { "" } else { "s" }; + let mut result = match message { + Some(s) => s.as_wtf8().to_owned(), + None => Wtf8Buf::new(), + }; + write!(result, " ({num_excs} sub-exception{suffix})").unwrap(); + Ok(vm.ctx.new_str(result)) + } + + #[pyslot] + fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let zelf = zelf + .downcast_ref::<PyBaseException>() + .expect("exception group must be BaseException"); + let class_name = zelf.class().name().to_owned(); + let message = zelf.get_arg(0).map(|m| m.repr(vm)).transpose()?; + + let mut result = Wtf8Buf::new(); + write!(result, "{class_name}(").unwrap(); + let message_wtf8: &Wtf8 = message.as_ref().map_or("''".as_ref(), |s| s.as_wtf8()); + result.push_wtf8(message_wtf8); + result.push_str(", ["); + if let Some(exceptions_obj) = zelf.get_arg(1) { + let iter: ArgIterable<PyObjectRef> = + ArgIterable::try_from_object(vm, exceptions_obj.clone())?; + let mut first = true; + for exc in iter.iter(vm)? { + if !first { + result.push_str(", "); + } + first = false; + result.push_wtf8(exc?.repr(vm)?.as_wtf8()); + } + } + result.push_str("])"); + + Ok(vm.ctx.new_str(result)) + } + } + + impl Constructor for PyBaseExceptionGroup { + type Args = crate::function::PosArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let args: Self::Args = args.bind(vm)?; + let args = args.into_vec(); + // Validate exactly 2 positional arguments + if args.len() != 2 { + return Err(vm.new_type_error(format!( + "BaseExceptionGroup.__new__() takes exactly 2 positional arguments ({} given)", + args.len() + ))); + } + + // Validate message is str + let message = args[0].clone(); + if !message.fast_isinstance(vm.ctx.types.str_type) { + return Err(vm.new_type_error(format!( + "argument 1 must be str, not {}", + message.class().name() + ))); + } + + // Validate exceptions is a sequence (not set or None) + let exceptions_arg = &args[1]; + + // Check for set/frozenset (not a sequence - unordered) + if exceptions_arg.fast_isinstance(vm.ctx.types.set_type) + || exceptions_arg.fast_isinstance(vm.ctx.types.frozenset_type) + { + return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); + } + + // Check for None + if exceptions_arg.is(&vm.ctx.none) { + return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); + } + + let exceptions: Vec<PyObjectRef> = exceptions_arg.try_to_value(vm).map_err(|_| { + vm.new_type_error("second argument (exceptions) must be a sequence") + })?; + + // Validate non-empty + if exceptions.is_empty() { + return Err( + vm.new_value_error("second argument (exceptions) must be a non-empty sequence") + ); + } + + // Validate all items are BaseException instances + let mut has_non_exception = false; + for (i, exc) in exceptions.iter().enumerate() { + if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_value_error(format!( + "Item {} of second argument (exceptions) is not an exception", + i + ))); + } + // Check if any exception is not an Exception subclass + // With dynamic ExceptionGroup (inherits from both BaseExceptionGroup and Exception), + // ExceptionGroup instances are automatically instances of Exception + if !exc.fast_isinstance(vm.ctx.exceptions.exception_type) { + has_non_exception = true; + } + } + + // Get the dynamic ExceptionGroup type + let exception_group_type = crate::exception_group::exception_group(); + + // Determine the actual class to use + let actual_cls = if cls.is(exception_group_type) { + // ExceptionGroup cannot contain BaseExceptions that are not Exception + if has_non_exception { + return Err( + vm.new_type_error("Cannot nest BaseExceptions in an ExceptionGroup") + ); + } + cls + } else if cls.is(vm.ctx.exceptions.base_exception_group) { + // Auto-convert to ExceptionGroup if all are Exception subclasses + if !has_non_exception { + exception_group_type.to_owned() + } else { + cls + } + } else { + // User-defined subclass + if has_non_exception && cls.fast_issubclass(vm.ctx.exceptions.exception_type) { + return Err(vm.new_type_error(format!( + "Cannot nest BaseExceptions in '{}'", + cls.name() + ))); + } + cls + }; + + // Create the exception with (message, exceptions_tuple) as args + let exceptions_tuple = vm.ctx.new_tuple(exceptions); + let init_args = vec![message, exceptions_tuple.into()]; + PyBaseException::new(init_args, vm) + .into_ref_with_type(vm, actual_cls) + .map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + impl Initializer for PyBaseExceptionGroup { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // BaseExceptionGroup_init: no kwargs allowed + if !args.kwargs.is_empty() { + return Err(vm.new_type_error(format!( + "{} does not take keyword arguments", + zelf.class().name() + ))); + } + // Do NOT call PyBaseException::slot_init here. + // slot_new already set args to (message, exceptions_tuple). + // Calling base init would overwrite with original args (message, exceptions_list). + let _ = (zelf, args, vm); + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is overridden") + } + } + + // Helper functions for ExceptionGroup + fn is_base_exception_group(obj: &PyObject, vm: &VirtualMachine) -> bool { + obj.fast_isinstance(vm.ctx.exceptions.base_exception_group) + } + + fn get_exceptions_tuple( + exc: &Py<PyBaseException>, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + let obj = exc + .get_arg(1) + .ok_or_else(|| vm.new_type_error("exceptions must be a tuple"))?; + let tuple = obj + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("exceptions must be a tuple"))?; + Ok(tuple.to_vec()) + } + + enum ConditionMatcher { + Type(PyTypeRef), + Types(Vec<PyTypeRef>), + Callable(PyObjectRef), + } + + fn get_condition_matcher( + condition: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<ConditionMatcher> { + // If it's a type and subclass of BaseException + if let Some(typ) = condition.downcast_ref::<PyType>() + && typ.fast_issubclass(vm.ctx.exceptions.base_exception_type) + { + return Ok(ConditionMatcher::Type(typ.to_owned())); + } + + // If it's a tuple of types + if let Some(tuple) = condition.downcast_ref::<PyTuple>() { + let mut types = Vec::new(); + for item in tuple.iter() { + let typ: PyTypeRef = item.clone().try_into_value(vm).map_err(|_| { + vm.new_type_error( + "expected a function, exception type or tuple of exception types", + ) + })?; + if !typ.fast_issubclass(vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error( + "expected a function, exception type or tuple of exception types", + )); + } + types.push(typ); + } + if !types.is_empty() { + return Ok(ConditionMatcher::Types(types)); + } + } + + // If it's callable (but not a type) + if condition.is_callable() && condition.downcast_ref::<PyType>().is_none() { + return Ok(ConditionMatcher::Callable(condition.to_owned())); + } + + Err(vm.new_type_error("expected a function, exception type or tuple of exception types")) + } + + impl ConditionMatcher { + fn check(&self, exc: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + match self { + ConditionMatcher::Type(typ) => Ok(exc.fast_isinstance(typ)), + ConditionMatcher::Types(types) => Ok(types.iter().any(|t| exc.fast_isinstance(t))), + ConditionMatcher::Callable(func) => { + let result = func.call((exc.to_owned(),), vm)?; + result.try_to_bool(vm) + } + } + } + } + + fn derive_and_copy_attributes( + orig: &Py<PyBaseException>, + excs: Vec<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + // Call derive method to create new group + let excs_seq = vm.ctx.new_list(excs); + let new_group = vm.call_method(orig.as_object(), "derive", (excs_seq,))?; + + // Verify derive returned a BaseExceptionGroup + if !is_base_exception_group(&new_group, vm) { + return Err(vm.new_type_error("derive must return an instance of BaseExceptionGroup")); + } + + // Copy traceback + if let Some(tb) = orig.__traceback__() { + new_group.set_attr("__traceback__", tb, vm)?; + } + + // Copy context + if let Some(ctx) = orig.__context__() { + new_group.set_attr("__context__", ctx, vm)?; + } + + // Copy cause + if let Some(cause) = orig.__cause__() { + new_group.set_attr("__cause__", cause, vm)?; + } + + // Copy notes (if present) - make a copy of the list + if let Ok(notes) = orig.as_object().get_attr("__notes__", vm) + && let Some(notes_list) = notes.downcast_ref::<PyList>() + { + let notes_copy = vm.ctx.new_list(notes_list.borrow_vec().to_vec()); + new_group.set_attr("__notes__", notes_copy, vm)?; + } + + Ok(new_group) + } +} diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs new file mode 100644 index 00000000000..f32005bd348 --- /dev/null +++ b/crates/vm/src/exceptions.rs @@ -0,0 +1,2980 @@ +use self::types::{PyBaseException, PyBaseExceptionRef}; +use crate::common::lock::PyRwLock; +use crate::object::{Traverse, TraverseFn}; +use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{ + PyList, PyNone, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, + traceback::{PyTraceback, PyTracebackRef}, + }, + class::{PyClassImpl, StaticType}, + convert::{IntoPyException, ToPyException, ToPyObject}, + function::{ArgIterable, FuncArgs, IntoFuncArgs, PySetterValue}, + py_io::{self, Write}, + stdlib::sys, + suggestion::offer_suggestions, + types::{Callable, Constructor, Initializer, Representable}, +}; +use crossbeam_utils::atomic::AtomicCell; +use itertools::Itertools; +use std::{ + collections::HashSet, + io::{self, BufRead, BufReader}, +}; + +pub use super::exception_group::exception_group; + +unsafe impl Traverse for PyBaseException { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.traceback.traverse(tracer_fn); + self.cause.traverse(tracer_fn); + self.context.traverse(tracer_fn); + self.args.traverse(tracer_fn); + } +} + +impl core::fmt::Debug for PyBaseException { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // TODO: implement more detailed, non-recursive Debug formatter + f.write_str("PyBaseException") + } +} + +impl PyPayload for PyBaseException { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.exceptions.base_exception_type + } +} + +impl VirtualMachine { + // Why `impl VirtualMachine`? + // These functions are natively free function in CPython - not methods of PyException + + /// Print exception chain by calling sys.excepthook + pub fn print_exception(&self, exc: PyBaseExceptionRef) { + let vm = self; + let write_fallback = |exc, errstr| { + if let Ok(stderr) = sys::get_stderr(vm) { + let mut stderr = py_io::PyWriter(stderr, vm); + // if this fails stderr might be closed -- ignore it + let _ = writeln!(stderr, "{errstr}"); + let _ = self.write_exception(&mut stderr, exc); + } else { + eprintln!("{errstr}\nlost sys.stderr"); + let _ = self.write_exception(&mut py_io::IoWriter(io::stderr()), exc); + } + }; + if let Ok(excepthook) = vm.sys_module.get_attr("excepthook", vm) { + let (exc_type, exc_val, exc_tb) = vm.split_exception(exc.clone()); + if let Err(eh_exc) = excepthook.call((exc_type, exc_val, exc_tb), vm) { + write_fallback(&eh_exc, "Error in sys.excepthook:"); + write_fallback(&exc, "Original exception was:"); + } + } else { + write_fallback(&exc, "missing sys.excepthook"); + } + } + + pub fn write_exception<W: Write>( + &self, + output: &mut W, + exc: &Py<PyBaseException>, + ) -> Result<(), W::Error> { + let seen = &mut HashSet::<usize>::new(); + self.write_exception_recursive(output, exc, seen) + } + + fn write_exception_recursive<W: Write>( + &self, + output: &mut W, + exc: &Py<PyBaseException>, + seen: &mut HashSet<usize>, + ) -> Result<(), W::Error> { + // This function should not be called directly, + // use `wite_exception` as a public interface. + // It is similar to `print_exception_recursive` from `CPython`. + seen.insert(exc.get_id()); + + #[allow(clippy::manual_map)] + if let Some((cause_or_context, msg)) = if let Some(cause) = exc.__cause__() { + // This can be a special case: `raise e from e`, + // we just ignore it and treat like `raise e` without any extra steps. + Some(( + cause, + "\nThe above exception was the direct cause of the following exception:\n", + )) + } else if let Some(context) = exc.__context__() { + // This can be a special case: + // e = ValueError('e') + // e.__context__ = e + // In this case, we just ignore + // `__context__` part from going into recursion. + Some(( + context, + "\nDuring handling of the above exception, another exception occurred:\n", + )) + } else { + None + } { + if !seen.contains(&cause_or_context.get_id()) { + self.write_exception_recursive(output, &cause_or_context, seen)?; + writeln!(output, "{msg}")?; + } else { + seen.insert(cause_or_context.get_id()); + } + } + + self.write_exception_inner(output, exc) + } + + /// Print exception with traceback + pub fn write_exception_inner<W: Write>( + &self, + output: &mut W, + exc: &Py<PyBaseException>, + ) -> Result<(), W::Error> { + let vm = self; + if let Some(tb) = exc.traceback.read().clone() { + writeln!(output, "Traceback (most recent call last):")?; + for tb in tb.iter() { + write_traceback_entry(output, &tb)?; + } + } + + let varargs = exc.args(); + let args_repr = vm.exception_args_as_string(varargs, true); + + let exc_class = exc.class(); + + if exc_class.fast_issubclass(vm.ctx.exceptions.syntax_error) { + return self.write_syntaxerror(output, exc, exc_class, &args_repr); + } + + let exc_name = exc_class.name(); + match args_repr.len() { + 0 => write!(output, "{exc_name}"), + 1 => write!(output, "{}: {}", exc_name, args_repr[0]), + _ => write!( + output, + "{}: ({})", + exc_name, + args_repr.into_iter().format(", "), + ), + }?; + + match offer_suggestions(exc, vm) { + Some(suggestions) => writeln!(output, ". Did you mean: '{suggestions}'?"), + None => writeln!(output), + } + } + + /// Format and write a SyntaxError + /// This logic is derived from TracebackException._format_syntax_error + /// + /// The logic has support for `end_offset` to highlight a range in the source code, + /// but it looks like `end_offset` is not used yet when SyntaxErrors are created. + fn write_syntaxerror<W: Write>( + &self, + output: &mut W, + exc: &Py<PyBaseException>, + exc_type: &Py<PyType>, + args_repr: &[PyRef<PyStr>], + ) -> Result<(), W::Error> { + let vm = self; + debug_assert!(exc_type.fast_issubclass(vm.ctx.exceptions.syntax_error)); + + let getattr = |attr: &'static str| exc.as_object().get_attr(attr, vm).ok(); + + let maybe_lineno = getattr("lineno").map(|obj| { + obj.str(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<lineno str() failed>")) + }); + let maybe_filename = getattr("filename").and_then(|obj| obj.str(vm).ok()); + + let maybe_text = getattr("text").map(|obj| { + obj.str(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<text str() failed>")) + }); + + let mut filename_suffix = String::new(); + + if let Some(lineno) = maybe_lineno { + let filename = match maybe_filename { + Some(filename) => filename, + None => vm.ctx.new_str("<string>"), + }; + writeln!(output, r##" File "{filename}", line {lineno}"##,)?; + } else if let Some(filename) = maybe_filename { + filename_suffix = format!(" ({filename})"); + } + + if let Some(text) = maybe_text { + // if text ends with \n or \r\n, remove it + use rustpython_common::wtf8::CodePoint; + let text_wtf8 = text.as_wtf8(); + let r_text = text_wtf8.trim_end_matches(|cp: CodePoint| { + cp == CodePoint::from_char('\n') || cp == CodePoint::from_char('\r') + }); + let l_text = r_text.trim_start_matches(|cp: CodePoint| { + cp == CodePoint::from_char(' ') + || cp == CodePoint::from_char('\n') + || cp == CodePoint::from_char('\x0c') // \f + }); + let spaces = (r_text.len() - l_text.len()) as isize; + + writeln!(output, " {l_text}")?; + + let maybe_offset: Option<isize> = + getattr("offset").and_then(|obj| obj.try_to_value::<isize>(vm).ok()); + + if let Some(offset) = maybe_offset { + let maybe_end_offset: Option<isize> = + getattr("end_offset").and_then(|obj| obj.try_to_value::<isize>(vm).ok()); + let maybe_end_lineno: Option<isize> = + getattr("end_lineno").and_then(|obj| obj.try_to_value::<isize>(vm).ok()); + let maybe_lineno_int: Option<isize> = + getattr("lineno").and_then(|obj| obj.try_to_value::<isize>(vm).ok()); + + // Only show caret if end_lineno is same as lineno (or not set) + let same_line = match (maybe_lineno_int, maybe_end_lineno) { + (Some(lineno), Some(end_lineno)) => lineno == end_lineno, + _ => true, + }; + + if same_line { + let mut end_offset = match maybe_end_offset { + Some(0) | None => offset, + Some(end_offset) => end_offset, + }; + + if offset == end_offset || end_offset == -1 { + end_offset = offset + 1; + } + + // Convert 1-based column offset to 0-based index into stripped text + let colno = offset - 1 - spaces; + let end_colno = end_offset - 1 - spaces; + if colno >= 0 { + let caret_space = l_text + .code_points() + .take(colno as usize) + .map(|cp| cp.to_char().filter(|c| c.is_whitespace()).unwrap_or(' ')) + .collect::<String>(); + + let mut error_width = end_colno - colno; + if error_width < 1 { + error_width = 1; + } + + writeln!( + output, + " {}{}", + caret_space, + "^".repeat(error_width as usize) + )?; + } + } + } + } + + let exc_name = exc_type.name(); + + match args_repr.len() { + 0 => write!(output, "{exc_name}{filename_suffix}"), + 1 => write!(output, "{}: {}{}", exc_name, args_repr[0], filename_suffix), + _ => write!( + output, + "{}: ({}){}", + exc_name, + args_repr.iter().format(", "), + filename_suffix + ), + }?; + + match offer_suggestions(exc, vm) { + Some(suggestions) => writeln!(output, ". Did you mean: '{suggestions}'?"), + None => writeln!(output), + } + } + + fn exception_args_as_string(&self, varargs: PyTupleRef, str_single: bool) -> Vec<PyStrRef> { + let vm = self; + match varargs.len() { + 0 => vec![], + 1 => { + let args0_repr = if str_single { + varargs[0] + .str(vm) + .unwrap_or_else(|_| PyStr::from("<element str() failed>").into_ref(&vm.ctx)) + } else { + varargs[0].repr(vm).unwrap_or_else(|_| { + PyStr::from("<element repr() failed>").into_ref(&vm.ctx) + }) + }; + vec![args0_repr] + } + _ => varargs + .iter() + .map(|vararg| { + vararg.repr(vm).unwrap_or_else(|_| { + PyStr::from("<element repr() failed>").into_ref(&vm.ctx) + }) + }) + .collect(), + } + } + + pub fn split_exception( + &self, + exc: PyBaseExceptionRef, + ) -> (PyObjectRef, PyObjectRef, PyObjectRef) { + let tb = exc.__traceback__().to_pyobject(self); + let class = exc.class().to_owned(); + (class.into(), exc.into(), tb) + } + + /// Similar to PyErr_NormalizeException in CPython + pub fn normalize_exception( + &self, + exc_type: PyObjectRef, + exc_val: PyObjectRef, + exc_tb: PyObjectRef, + ) -> PyResult<PyBaseExceptionRef> { + let ctor = ExceptionCtor::try_from_object(self, exc_type)?; + let exc = ctor.instantiate_value(exc_val, self)?; + if let Some(tb) = Option::<PyTracebackRef>::try_from_object(self, exc_tb)? { + exc.set_traceback_typed(Some(tb)); + } + Ok(exc) + } + + pub fn invoke_exception( + &self, + cls: PyTypeRef, + args: Vec<PyObjectRef>, + ) -> PyResult<PyBaseExceptionRef> { + // TODO: fast-path built-in exceptions by directly instantiating them? Is that really worth it? + let res = PyType::call(&cls, args.into_args(self), self)?; + res.downcast::<PyBaseException>().map_err(|obj| { + self.new_type_error(format!( + "calling {} should have returned an instance of BaseException, not {}", + cls, + obj.class() + )) + }) + } +} + +fn print_source_line<W: Write>( + output: &mut W, + filename: &str, + lineno: usize, +) -> Result<(), W::Error> { + // TODO: use io.open() method instead, when available, according to https://github.com/python/cpython/blob/main/Python/traceback.c#L393 + // TODO: support different encodings + let file = match std::fs::File::open(filename) { + Ok(file) => file, + Err(_) => return Ok(()), + }; + let file = BufReader::new(file); + + for (i, line) in file.lines().enumerate() { + if i + 1 == lineno { + if let Ok(line) = line { + // Indented with 4 spaces + writeln!(output, " {}", line.trim_start())?; + } + return Ok(()); + } + } + + Ok(()) +} + +/// Print exception occurrence location from traceback element +fn write_traceback_entry<W: Write>( + output: &mut W, + tb_entry: &Py<PyTraceback>, +) -> Result<(), W::Error> { + let filename = tb_entry.frame.code.source_path().as_str(); + writeln!( + output, + r##" File "{}", line {}, in {}"##, + filename.trim_start_matches(r"\\?\"), + tb_entry.lineno, + tb_entry.frame.code.obj_name + )?; + print_source_line(output, filename, tb_entry.lineno.get())?; + + Ok(()) +} + +#[derive(Clone)] +pub enum ExceptionCtor { + Class(PyTypeRef), + Instance(PyBaseExceptionRef), +} + +impl TryFromObject for ExceptionCtor { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + obj.downcast::<PyType>() + .and_then(|cls| { + if cls.fast_issubclass(vm.ctx.exceptions.base_exception_type) { + Ok(Self::Class(cls)) + } else { + Err(cls.into()) + } + }) + .or_else(|obj| obj.downcast::<PyBaseException>().map(Self::Instance)) + .map_err(|obj| { + vm.new_type_error(format!( + "exceptions must be classes or instances deriving from BaseException, not {}", + obj.class().name() + )) + }) + } +} + +impl ExceptionCtor { + pub fn instantiate(self, vm: &VirtualMachine) -> PyResult<PyBaseExceptionRef> { + match self { + Self::Class(cls) => vm.invoke_exception(cls, vec![]), + Self::Instance(exc) => Ok(exc), + } + } + + pub fn instantiate_value( + self, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyBaseExceptionRef> { + let exc_inst = value.clone().downcast::<PyBaseException>().ok(); + match (self, exc_inst) { + // both are instances; which would we choose? + (Self::Instance(_exc_a), Some(_exc_b)) => { + Err(vm.new_type_error("instance exception may not have a separate value")) + } + // if the "type" is an instance and the value isn't, use the "type" + (Self::Instance(exc), None) => Ok(exc), + // if the value is an instance of the type, use the instance value + (Self::Class(cls), Some(exc)) if exc.fast_isinstance(&cls) => Ok(exc), + // otherwise; construct an exception of the type using the value as args + (Self::Class(cls), _) => { + let args = match_class!(match value { + PyNone => vec![], + tup @ PyTuple => tup.to_vec(), + exc @ PyBaseException => exc.args().to_vec(), + obj => vec![obj], + }); + vm.invoke_exception(cls, args) + } + } + } +} + +#[derive(Debug)] +pub struct ExceptionZoo { + pub base_exception_type: &'static Py<PyType>, + pub base_exception_group: &'static Py<PyType>, + pub system_exit: &'static Py<PyType>, + pub keyboard_interrupt: &'static Py<PyType>, + pub generator_exit: &'static Py<PyType>, + pub exception_type: &'static Py<PyType>, + pub stop_iteration: &'static Py<PyType>, + pub stop_async_iteration: &'static Py<PyType>, + pub arithmetic_error: &'static Py<PyType>, + pub floating_point_error: &'static Py<PyType>, + pub overflow_error: &'static Py<PyType>, + pub zero_division_error: &'static Py<PyType>, + pub assertion_error: &'static Py<PyType>, + pub attribute_error: &'static Py<PyType>, + pub buffer_error: &'static Py<PyType>, + pub eof_error: &'static Py<PyType>, + pub import_error: &'static Py<PyType>, + pub module_not_found_error: &'static Py<PyType>, + pub lookup_error: &'static Py<PyType>, + pub index_error: &'static Py<PyType>, + pub key_error: &'static Py<PyType>, + pub memory_error: &'static Py<PyType>, + pub name_error: &'static Py<PyType>, + pub unbound_local_error: &'static Py<PyType>, + pub os_error: &'static Py<PyType>, + pub blocking_io_error: &'static Py<PyType>, + pub child_process_error: &'static Py<PyType>, + pub connection_error: &'static Py<PyType>, + pub broken_pipe_error: &'static Py<PyType>, + pub connection_aborted_error: &'static Py<PyType>, + pub connection_refused_error: &'static Py<PyType>, + pub connection_reset_error: &'static Py<PyType>, + pub file_exists_error: &'static Py<PyType>, + pub file_not_found_error: &'static Py<PyType>, + pub interrupted_error: &'static Py<PyType>, + pub is_a_directory_error: &'static Py<PyType>, + pub not_a_directory_error: &'static Py<PyType>, + pub permission_error: &'static Py<PyType>, + pub process_lookup_error: &'static Py<PyType>, + pub timeout_error: &'static Py<PyType>, + pub reference_error: &'static Py<PyType>, + pub runtime_error: &'static Py<PyType>, + pub not_implemented_error: &'static Py<PyType>, + pub recursion_error: &'static Py<PyType>, + pub python_finalization_error: &'static Py<PyType>, + pub syntax_error: &'static Py<PyType>, + pub incomplete_input_error: &'static Py<PyType>, + pub indentation_error: &'static Py<PyType>, + pub tab_error: &'static Py<PyType>, + pub system_error: &'static Py<PyType>, + pub type_error: &'static Py<PyType>, + pub value_error: &'static Py<PyType>, + pub unicode_error: &'static Py<PyType>, + pub unicode_decode_error: &'static Py<PyType>, + pub unicode_encode_error: &'static Py<PyType>, + pub unicode_translate_error: &'static Py<PyType>, + + #[cfg(feature = "jit")] + pub jit_error: &'static Py<PyType>, + + pub warning: &'static Py<PyType>, + pub deprecation_warning: &'static Py<PyType>, + pub pending_deprecation_warning: &'static Py<PyType>, + pub runtime_warning: &'static Py<PyType>, + pub syntax_warning: &'static Py<PyType>, + pub user_warning: &'static Py<PyType>, + pub future_warning: &'static Py<PyType>, + pub import_warning: &'static Py<PyType>, + pub unicode_warning: &'static Py<PyType>, + pub bytes_warning: &'static Py<PyType>, + pub resource_warning: &'static Py<PyType>, + pub encoding_warning: &'static Py<PyType>, +} + +macro_rules! extend_exception { + ( + $exc_struct:ident, + $ctx:expr, + $class:expr + ) => { + extend_exception!($exc_struct, $ctx, $class, {}); + }; + ( + $exc_struct:ident, + $ctx:expr, + $class:expr, + { $($name:expr => $value:expr),* $(,)* } + ) => { + $exc_struct::extend_class($ctx, $class); + extend_class!($ctx, $class, { + $($name => $value,)* + }); + }; +} + +impl PyBaseException { + pub(crate) fn new(args: Vec<PyObjectRef>, vm: &VirtualMachine) -> Self { + Self { + traceback: PyRwLock::new(None), + cause: PyRwLock::new(None), + context: PyRwLock::new(None), + suppress_context: AtomicCell::new(false), + args: PyRwLock::new(PyTuple::new_ref(args, &vm.ctx)), + } + } + + pub fn get_arg(&self, idx: usize) -> Option<PyObjectRef> { + self.args.read().get(idx).cloned() + } +} + +#[pyclass( + with(Py, PyRef, Constructor, Initializer, Representable), + flags(BASETYPE, HAS_DICT) +)] +impl PyBaseException { + #[pygetset] + pub fn args(&self) -> PyTupleRef { + self.args.read().clone() + } + + #[pygetset(setter)] + fn set_args(&self, args: ArgIterable, vm: &VirtualMachine) -> PyResult<()> { + let args = args.iter(vm)?.collect::<PyResult<Vec<_>>>()?; + *self.args.write() = PyTuple::new_ref(args, &vm.ctx); + Ok(()) + } + + #[pygetset] + pub fn __traceback__(&self) -> Option<PyTracebackRef> { + self.traceback.read().clone() + } + + #[pygetset(setter)] + pub fn set___traceback__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let traceback = if vm.is_none(&value) { + None + } else { + match value.downcast::<PyTraceback>() { + Ok(tb) => Some(tb), + Err(_) => { + return Err(vm.new_type_error("__traceback__ must be a traceback or None")); + } + } + }; + self.set_traceback_typed(traceback); + Ok(()) + } + + // Helper method for internal use that doesn't require PyObjectRef + pub(crate) fn set_traceback_typed(&self, traceback: Option<PyTracebackRef>) { + *self.traceback.write() = traceback; + } + + #[pygetset] + pub fn __cause__(&self) -> Option<PyRef<Self>> { + self.cause.read().clone() + } + + #[pygetset(setter)] + pub fn set___cause__(&self, cause: Option<PyRef<Self>>) { + let mut c = self.cause.write(); + self.set_suppress_context(true); + *c = cause; + } + + #[pygetset] + pub fn __context__(&self) -> Option<PyRef<Self>> { + self.context.read().clone() + } + + #[pygetset(setter)] + pub fn set___context__(&self, context: Option<PyRef<Self>>) { + *self.context.write() = context; + } + + #[pygetset] + pub(super) fn __suppress_context__(&self) -> bool { + self.suppress_context.load() + } + + #[pygetset(name = "__suppress_context__", setter)] + fn set_suppress_context(&self, suppress_context: bool) { + self.suppress_context.store(suppress_context); + } +} + +#[pyclass] +impl Py<PyBaseException> { + #[pymethod] + pub(super) fn __str__(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let str_args = vm.exception_args_as_string(self.args(), true); + Ok(match str_args.into_iter().exactly_one() { + Err(i) if i.len() == 0 => vm.ctx.empty_str.to_owned(), + Ok(s) => s, + Err(i) => PyStr::from(format!("({})", i.format(", "))).into_ref(&vm.ctx), + }) + } +} + +#[pyclass] +impl PyRef<PyBaseException> { + #[pymethod] + fn with_traceback(self, tb: Option<PyTracebackRef>) -> PyResult<Self> { + *self.traceback.write() = tb; + Ok(self) + } + + #[pymethod] + fn add_note(self, note: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let dict = self + .as_object() + .dict() + .ok_or_else(|| vm.new_attribute_error("Exception object has no __dict__"))?; + + let notes = if let Ok(notes) = dict.get_item("__notes__", vm) { + notes + } else { + let new_notes = vm.ctx.new_list(vec![]); + dict.set_item("__notes__", new_notes.clone().into(), vm)?; + new_notes.into() + }; + + let notes = notes + .downcast::<PyList>() + .map_err(|_| vm.new_type_error("__notes__ must be a list"))?; + + notes.borrow_vec_mut().push(note.into()); + Ok(()) + } + + #[pymethod] + fn __reduce__(self, vm: &VirtualMachine) -> PyTupleRef { + if let Some(dict) = self.as_object().dict().filter(|x| !x.is_empty()) { + vm.new_tuple((self.class().to_owned(), self.args(), dict)) + } else { + vm.new_tuple((self.class().to_owned(), self.args())) + } + } + + #[pymethod] + fn __setstate__(self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if !vm.is_none(&state) { + let dict = state + .downcast::<crate::builtins::PyDict>() + .map_err(|_| vm.new_type_error("state is not a dictionary"))?; + + for (key, value) in &dict { + let key_str = key.str(vm)?; + if key_str.as_bytes().starts_with(b"__") { + continue; + } + self.as_object().set_attr(&key_str, value.clone(), vm)?; + } + } + Ok(vm.ctx.none()) + } +} + +impl Constructor for PyBaseException { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if cls.is(Self::class(&vm.ctx)) && !args.kwargs.is_empty() { + return Err(vm.new_type_error("BaseException() takes no keyword arguments")); + } + Self::new(args.args, vm) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: FuncArgs, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyBaseException { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + *zelf.args.write() = PyTuple::new_ref(args.args, &vm.ctx); + Ok(()) + } +} + +impl Representable for PyBaseException { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let repr_args = vm.exception_args_as_string(zelf.args(), false); + let cls = zelf.class(); + Ok(format!("{}({})", cls.name(), repr_args.iter().format(", "))) + } +} + +impl ExceptionZoo { + pub(crate) fn init() -> Self { + use self::types::*; + + let base_exception_type = PyBaseException::init_builtin_type(); + + // Sorted By Hierarchy then alphabetized. + let base_exception_group = PyBaseExceptionGroup::init_builtin_type(); + let system_exit = PySystemExit::init_builtin_type(); + let keyboard_interrupt = PyKeyboardInterrupt::init_builtin_type(); + let generator_exit = PyGeneratorExit::init_builtin_type(); + + let exception_type = PyException::init_builtin_type(); + let stop_iteration = PyStopIteration::init_builtin_type(); + let stop_async_iteration = PyStopAsyncIteration::init_builtin_type(); + let arithmetic_error = PyArithmeticError::init_builtin_type(); + let floating_point_error = PyFloatingPointError::init_builtin_type(); + let overflow_error = PyOverflowError::init_builtin_type(); + let zero_division_error = PyZeroDivisionError::init_builtin_type(); + + let assertion_error = PyAssertionError::init_builtin_type(); + let attribute_error = PyAttributeError::init_builtin_type(); + let buffer_error = PyBufferError::init_builtin_type(); + let eof_error = PyEOFError::init_builtin_type(); + + let import_error = PyImportError::init_builtin_type(); + let module_not_found_error = PyModuleNotFoundError::init_builtin_type(); + + let lookup_error = PyLookupError::init_builtin_type(); + let index_error = PyIndexError::init_builtin_type(); + let key_error = PyKeyError::init_builtin_type(); + + let memory_error = PyMemoryError::init_builtin_type(); + + let name_error = PyNameError::init_builtin_type(); + let unbound_local_error = PyUnboundLocalError::init_builtin_type(); + + // os errors + let os_error = PyOSError::init_builtin_type(); + let blocking_io_error = PyBlockingIOError::init_builtin_type(); + let child_process_error = PyChildProcessError::init_builtin_type(); + + let connection_error = PyConnectionError::init_builtin_type(); + let broken_pipe_error = PyBrokenPipeError::init_builtin_type(); + let connection_aborted_error = PyConnectionAbortedError::init_builtin_type(); + let connection_refused_error = PyConnectionRefusedError::init_builtin_type(); + let connection_reset_error = PyConnectionResetError::init_builtin_type(); + + let file_exists_error = PyFileExistsError::init_builtin_type(); + let file_not_found_error = PyFileNotFoundError::init_builtin_type(); + let interrupted_error = PyInterruptedError::init_builtin_type(); + let is_a_directory_error = PyIsADirectoryError::init_builtin_type(); + let not_a_directory_error = PyNotADirectoryError::init_builtin_type(); + let permission_error = PyPermissionError::init_builtin_type(); + let process_lookup_error = PyProcessLookupError::init_builtin_type(); + let timeout_error = PyTimeoutError::init_builtin_type(); + + let reference_error = PyReferenceError::init_builtin_type(); + + let runtime_error = PyRuntimeError::init_builtin_type(); + let not_implemented_error = PyNotImplementedError::init_builtin_type(); + let recursion_error = PyRecursionError::init_builtin_type(); + let python_finalization_error = PyPythonFinalizationError::init_builtin_type(); + + let syntax_error = PySyntaxError::init_builtin_type(); + let incomplete_input_error = PyIncompleteInputError::init_builtin_type(); + let indentation_error = PyIndentationError::init_builtin_type(); + let tab_error = PyTabError::init_builtin_type(); + + let system_error = PySystemError::init_builtin_type(); + let type_error = PyTypeError::init_builtin_type(); + let value_error = PyValueError::init_builtin_type(); + let unicode_error = PyUnicodeError::init_builtin_type(); + let unicode_decode_error = PyUnicodeDecodeError::init_builtin_type(); + let unicode_encode_error = PyUnicodeEncodeError::init_builtin_type(); + let unicode_translate_error = PyUnicodeTranslateError::init_builtin_type(); + + #[cfg(feature = "jit")] + let jit_error = PyJitError::init_builtin_type(); + + let warning = PyWarning::init_builtin_type(); + let deprecation_warning = PyDeprecationWarning::init_builtin_type(); + let pending_deprecation_warning = PyPendingDeprecationWarning::init_builtin_type(); + let runtime_warning = PyRuntimeWarning::init_builtin_type(); + let syntax_warning = PySyntaxWarning::init_builtin_type(); + let user_warning = PyUserWarning::init_builtin_type(); + let future_warning = PyFutureWarning::init_builtin_type(); + let import_warning = PyImportWarning::init_builtin_type(); + let unicode_warning = PyUnicodeWarning::init_builtin_type(); + let bytes_warning = PyBytesWarning::init_builtin_type(); + let resource_warning = PyResourceWarning::init_builtin_type(); + let encoding_warning = PyEncodingWarning::init_builtin_type(); + + Self { + base_exception_type, + base_exception_group, + system_exit, + keyboard_interrupt, + generator_exit, + exception_type, + stop_iteration, + stop_async_iteration, + arithmetic_error, + floating_point_error, + overflow_error, + zero_division_error, + assertion_error, + attribute_error, + buffer_error, + eof_error, + import_error, + module_not_found_error, + lookup_error, + index_error, + key_error, + memory_error, + name_error, + unbound_local_error, + os_error, + blocking_io_error, + child_process_error, + connection_error, + broken_pipe_error, + connection_aborted_error, + connection_refused_error, + connection_reset_error, + file_exists_error, + file_not_found_error, + interrupted_error, + is_a_directory_error, + not_a_directory_error, + permission_error, + process_lookup_error, + timeout_error, + reference_error, + runtime_error, + not_implemented_error, + recursion_error, + python_finalization_error, + syntax_error, + incomplete_input_error, + indentation_error, + tab_error, + system_error, + type_error, + value_error, + unicode_error, + unicode_decode_error, + unicode_encode_error, + unicode_translate_error, + + #[cfg(feature = "jit")] + jit_error, + + warning, + deprecation_warning, + pending_deprecation_warning, + runtime_warning, + syntax_warning, + user_warning, + future_warning, + import_warning, + unicode_warning, + bytes_warning, + resource_warning, + encoding_warning, + } + } + + // TODO: remove it after fixing `errno` / `winerror` problem + #[allow( + clippy::redundant_clone, + reason = "temporary workaround until errno/winerror handling is fixed" + )] + pub fn extend(ctx: &'static Context) { + use self::types::*; + + let excs = &ctx.exceptions; + + PyBaseException::extend_class(ctx, excs.base_exception_type); + + // Sorted By Hierarchy then alphabetized. + extend_exception!(PyBaseExceptionGroup, ctx, excs.base_exception_group, { + "message" => ctx.new_readonly_getset("message", excs.base_exception_group, make_arg_getter(0)), + "exceptions" => ctx.new_readonly_getset("exceptions", excs.base_exception_group, make_arg_getter(1)), + }); + + extend_exception!(PySystemExit, ctx, excs.system_exit, { + "code" => ctx.new_readonly_getset("code", excs.system_exit, system_exit_code), + }); + extend_exception!(PyKeyboardInterrupt, ctx, excs.keyboard_interrupt); + extend_exception!(PyGeneratorExit, ctx, excs.generator_exit); + + extend_exception!(PyException, ctx, excs.exception_type); + + extend_exception!(PyStopIteration, ctx, excs.stop_iteration, { + "value" => ctx.none(), + }); + extend_exception!(PyStopAsyncIteration, ctx, excs.stop_async_iteration); + + extend_exception!(PyArithmeticError, ctx, excs.arithmetic_error); + extend_exception!(PyFloatingPointError, ctx, excs.floating_point_error); + extend_exception!(PyOverflowError, ctx, excs.overflow_error); + extend_exception!(PyZeroDivisionError, ctx, excs.zero_division_error); + + extend_exception!(PyAssertionError, ctx, excs.assertion_error); + extend_exception!(PyAttributeError, ctx, excs.attribute_error, { + "name" => ctx.none(), + "obj" => ctx.none(), + }); + extend_exception!(PyBufferError, ctx, excs.buffer_error); + extend_exception!(PyEOFError, ctx, excs.eof_error); + + extend_exception!(PyImportError, ctx, excs.import_error, { + "msg" => ctx.new_readonly_getset("msg", excs.import_error, make_arg_getter(0)), + }); + extend_exception!(PyModuleNotFoundError, ctx, excs.module_not_found_error); + + extend_exception!(PyLookupError, ctx, excs.lookup_error); + extend_exception!(PyIndexError, ctx, excs.index_error); + + extend_exception!(PyKeyError, ctx, excs.key_error); + + extend_exception!(PyMemoryError, ctx, excs.memory_error); + extend_exception!(PyNameError, ctx, excs.name_error, { + "name" => ctx.none(), + }); + extend_exception!(PyUnboundLocalError, ctx, excs.unbound_local_error); + + // os errors: + // PyOSError now uses struct fields with pygetset, no need for dynamic attributes + extend_exception!(PyOSError, ctx, excs.os_error); + + extend_exception!(PyBlockingIOError, ctx, excs.blocking_io_error); + extend_exception!(PyChildProcessError, ctx, excs.child_process_error); + + extend_exception!(PyConnectionError, ctx, excs.connection_error); + extend_exception!(PyBrokenPipeError, ctx, excs.broken_pipe_error); + extend_exception!(PyConnectionAbortedError, ctx, excs.connection_aborted_error); + extend_exception!(PyConnectionRefusedError, ctx, excs.connection_refused_error); + extend_exception!(PyConnectionResetError, ctx, excs.connection_reset_error); + + extend_exception!(PyFileExistsError, ctx, excs.file_exists_error); + extend_exception!(PyFileNotFoundError, ctx, excs.file_not_found_error); + extend_exception!(PyInterruptedError, ctx, excs.interrupted_error); + extend_exception!(PyIsADirectoryError, ctx, excs.is_a_directory_error); + extend_exception!(PyNotADirectoryError, ctx, excs.not_a_directory_error); + extend_exception!(PyPermissionError, ctx, excs.permission_error); + extend_exception!(PyProcessLookupError, ctx, excs.process_lookup_error); + extend_exception!(PyTimeoutError, ctx, excs.timeout_error); + + extend_exception!(PyReferenceError, ctx, excs.reference_error); + extend_exception!(PyRuntimeError, ctx, excs.runtime_error); + extend_exception!(PyNotImplementedError, ctx, excs.not_implemented_error); + extend_exception!(PyRecursionError, ctx, excs.recursion_error); + extend_exception!( + PyPythonFinalizationError, + ctx, + excs.python_finalization_error + ); + + extend_exception!(PySyntaxError, ctx, excs.syntax_error, { + "msg" => ctx.new_static_getset( + "msg", + excs.syntax_error, + make_arg_getter(0), + syntax_error_set_msg, + ), + // TODO: members + "filename" => ctx.none(), + "lineno" => ctx.none(), + "end_lineno" => ctx.none(), + "offset" => ctx.none(), + "end_offset" => ctx.none(), + "text" => ctx.none(), + }); + extend_exception!(PyIncompleteInputError, ctx, excs.incomplete_input_error); + extend_exception!(PyIndentationError, ctx, excs.indentation_error); + extend_exception!(PyTabError, ctx, excs.tab_error); + + extend_exception!(PySystemError, ctx, excs.system_error); + extend_exception!(PyTypeError, ctx, excs.type_error); + extend_exception!(PyValueError, ctx, excs.value_error); + extend_exception!(PyUnicodeError, ctx, excs.unicode_error); + extend_exception!(PyUnicodeDecodeError, ctx, excs.unicode_decode_error); + extend_exception!(PyUnicodeEncodeError, ctx, excs.unicode_encode_error); + extend_exception!(PyUnicodeTranslateError, ctx, excs.unicode_translate_error); + + #[cfg(feature = "jit")] + extend_exception!(PyJitError, ctx, excs.jit_error); + + extend_exception!(PyWarning, ctx, excs.warning); + extend_exception!(PyDeprecationWarning, ctx, excs.deprecation_warning); + extend_exception!( + PyPendingDeprecationWarning, + ctx, + excs.pending_deprecation_warning + ); + extend_exception!(PyRuntimeWarning, ctx, excs.runtime_warning); + extend_exception!(PySyntaxWarning, ctx, excs.syntax_warning); + extend_exception!(PyUserWarning, ctx, excs.user_warning); + extend_exception!(PyFutureWarning, ctx, excs.future_warning); + extend_exception!(PyImportWarning, ctx, excs.import_warning); + extend_exception!(PyUnicodeWarning, ctx, excs.unicode_warning); + extend_exception!(PyBytesWarning, ctx, excs.bytes_warning); + extend_exception!(PyResourceWarning, ctx, excs.resource_warning); + extend_exception!(PyEncodingWarning, ctx, excs.encoding_warning); + } +} + +fn make_arg_getter(idx: usize) -> impl Fn(PyBaseExceptionRef) -> Option<PyObjectRef> { + move |exc| exc.get_arg(idx) +} + +fn syntax_error_set_msg( + exc: PyBaseExceptionRef, + value: PySetterValue, + vm: &VirtualMachine, +) -> PyResult<()> { + let mut args = exc.args.write(); + let mut new_args = args.as_slice().to_vec(); + // Ensure the message slot at index 0 always exists for SyntaxError.args. + if new_args.is_empty() { + new_args.push(vm.ctx.none()); + } + match value { + PySetterValue::Assign(value) => new_args[0] = value, + PySetterValue::Delete => new_args[0] = vm.ctx.none(), + } + *args = PyTuple::new_ref(new_args, &vm.ctx); + Ok(()) +} + +fn system_exit_code(exc: PyBaseExceptionRef) -> Option<PyObjectRef> { + // SystemExit.code based on args length: + // - size == 0: code is None + // - size == 1: code is args[0] + // - size > 1: code is args (the whole tuple) + let args = exc.args.read(); + match args.len() { + 0 => None, + 1 => Some(args.first().unwrap().clone()), + _ => Some(args.as_object().to_owned()), + } +} + +#[cfg(feature = "serde")] +pub struct SerializeException<'vm, 's> { + vm: &'vm VirtualMachine, + exc: &'s Py<PyBaseException>, +} + +#[cfg(feature = "serde")] +impl<'vm, 's> SerializeException<'vm, 's> { + pub fn new(vm: &'vm VirtualMachine, exc: &'s Py<PyBaseException>) -> Self { + SerializeException { vm, exc } + } +} + +#[cfg(feature = "serde")] +pub struct SerializeExceptionOwned<'vm> { + vm: &'vm VirtualMachine, + exc: PyBaseExceptionRef, +} + +#[cfg(feature = "serde")] +impl serde::Serialize for SerializeExceptionOwned<'_> { + fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { + let Self { vm, exc } = self; + SerializeException::new(vm, exc).serialize(s) + } +} + +#[cfg(feature = "serde")] +impl serde::Serialize for SerializeException<'_, '_> { + fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { + use serde::ser::*; + + let mut struc = s.serialize_struct("PyBaseException", 7)?; + struc.serialize_field("exc_type", &*self.exc.class().name())?; + let tbs = { + struct Tracebacks(PyTracebackRef); + impl serde::Serialize for Tracebacks { + fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { + let mut s = s.serialize_seq(None)?; + for tb in self.0.iter() { + s.serialize_element(&**tb)?; + } + s.end() + } + } + self.exc.__traceback__().map(Tracebacks) + }; + struc.serialize_field("traceback", &tbs)?; + struc.serialize_field( + "cause", + &self + .exc + .__cause__() + .map(|exc| SerializeExceptionOwned { vm: self.vm, exc }), + )?; + struc.serialize_field( + "context", + &self + .exc + .__context__() + .map(|exc| SerializeExceptionOwned { vm: self.vm, exc }), + )?; + struc.serialize_field("suppress_context", &self.exc.__suppress_context__())?; + + let args = { + struct Args<'vm>(&'vm VirtualMachine, PyTupleRef); + impl serde::Serialize for Args<'_> { + fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { + s.collect_seq( + self.1 + .iter() + .map(|arg| crate::py_serde::PyObjectSerializer::new(self.0, arg)), + ) + } + } + Args(self.vm, self.exc.args()) + }; + struc.serialize_field("args", &args)?; + + let rendered = { + let mut rendered = String::new(); + self.vm + .write_exception(&mut rendered, self.exc) + .map_err(S::Error::custom)?; + rendered + }; + struc.serialize_field("rendered", &rendered)?; + + struc.end() + } +} + +pub fn cstring_error(vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_value_error("embedded null character") +} + +impl ToPyException for alloc::ffi::NulError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + cstring_error(vm) + } +} + +#[cfg(windows)] +impl<C> ToPyException for widestring::error::ContainsNul<C> { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + cstring_error(vm) + } +} + +#[cfg(any(unix, windows, target_os = "wasi"))] +pub(crate) fn errno_to_exc_type(errno: i32, vm: &VirtualMachine) -> Option<&'static Py<PyType>> { + use crate::stdlib::errno::errors; + let excs = &vm.ctx.exceptions; + match errno { + #[allow(unreachable_patterns)] // EAGAIN is sometimes the same as EWOULDBLOCK + errors::EWOULDBLOCK | errors::EAGAIN => Some(excs.blocking_io_error), + errors::EALREADY => Some(excs.blocking_io_error), + errors::EINPROGRESS => Some(excs.blocking_io_error), + errors::EPIPE => Some(excs.broken_pipe_error), + #[cfg(not(target_os = "wasi"))] + errors::ESHUTDOWN => Some(excs.broken_pipe_error), + errors::ECHILD => Some(excs.child_process_error), + errors::ECONNABORTED => Some(excs.connection_aborted_error), + errors::ECONNREFUSED => Some(excs.connection_refused_error), + errors::ECONNRESET => Some(excs.connection_reset_error), + errors::EEXIST => Some(excs.file_exists_error), + errors::ENOENT => Some(excs.file_not_found_error), + errors::EISDIR => Some(excs.is_a_directory_error), + errors::ENOTDIR => Some(excs.not_a_directory_error), + errors::EINTR => Some(excs.interrupted_error), + errors::EACCES => Some(excs.permission_error), + errors::EPERM => Some(excs.permission_error), + errors::ESRCH => Some(excs.process_lookup_error), + errors::ETIMEDOUT => Some(excs.timeout_error), + _ => None, + } +} + +#[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] +pub(crate) fn errno_to_exc_type(_errno: i32, _vm: &VirtualMachine) -> Option<&'static Py<PyType>> { + None +} + +pub(crate) trait ToOSErrorBuilder { + fn to_os_error_builder(&self, vm: &VirtualMachine) -> OSErrorBuilder; +} + +pub(crate) struct OSErrorBuilder { + exc_type: PyTypeRef, + errno: Option<i32>, + strerror: Option<PyObjectRef>, + filename: Option<PyObjectRef>, + #[cfg(windows)] + winerror: Option<PyObjectRef>, + filename2: Option<PyObjectRef>, +} + +impl OSErrorBuilder { + #[must_use] + pub fn with_subtype( + exc_type: PyTypeRef, + errno: Option<i32>, + strerror: impl ToPyObject, + vm: &VirtualMachine, + ) -> Self { + let strerror = strerror.to_pyobject(vm); + Self { + exc_type, + errno, + strerror: Some(strerror), + filename: None, + #[cfg(windows)] + winerror: None, + filename2: None, + } + } + + #[must_use] + pub fn with_errno(errno: i32, strerror: impl ToPyObject, vm: &VirtualMachine) -> Self { + let exc_type = errno_to_exc_type(errno, vm) + .unwrap_or(vm.ctx.exceptions.os_error) + .to_owned(); + Self::with_subtype(exc_type, Some(errno), strerror, vm) + } + + #[must_use] + #[allow(dead_code)] + pub(crate) fn filename(mut self, filename: PyObjectRef) -> Self { + self.filename.replace(filename); + self + } + + #[must_use] + #[allow(dead_code)] + pub(crate) fn filename2(mut self, filename: PyObjectRef) -> Self { + self.filename2.replace(filename); + self + } + + #[must_use] + #[cfg(windows)] + pub(crate) fn winerror(mut self, winerror: PyObjectRef) -> Self { + self.winerror.replace(winerror); + self + } + + /// Strip winerror from the builder. Used for C runtime errors + /// (e.g. `_wopen`, `open`) that should produce `[Errno X]` format + /// instead of `[WinError X]`. + #[must_use] + #[cfg(windows)] + pub(crate) fn without_winerror(mut self) -> Self { + self.winerror = None; + self + } + + pub fn build(self, vm: &VirtualMachine) -> PyRef<types::PyOSError> { + use types::PyOSError; + + let OSErrorBuilder { + exc_type, + errno, + strerror, + filename, + #[cfg(windows)] + winerror, + filename2, + } = self; + + let args = if let Some(errno) = errno { + #[cfg(windows)] + let winerror = winerror.to_pyobject(vm); + #[cfg(not(windows))] + let winerror = vm.ctx.none(); + + vec![ + errno.to_pyobject(vm), + strerror.to_pyobject(vm), + filename.to_pyobject(vm), + winerror, + filename2.to_pyobject(vm), + ] + } else { + vec![strerror.to_pyobject(vm)] + }; + + let payload = PyOSError::py_new(&exc_type, args.clone().into(), vm) + .expect("new_os_error usage error"); + let os_error = payload + .into_ref_with_type(vm, exc_type) + .expect("new_os_error usage error"); + PyOSError::slot_init(os_error.as_object().to_owned(), args.into(), vm) + .expect("new_os_error usage error"); + os_error + } +} + +impl IntoPyException for OSErrorBuilder { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.build(vm).upcast() + } +} + +impl ToOSErrorBuilder for std::io::Error { + fn to_os_error_builder(&self, vm: &VirtualMachine) -> OSErrorBuilder { + use crate::common::os::ErrorExt; + + let errno = self.posix_errno(); + #[cfg(windows)] + let msg = 'msg: { + // Use C runtime's strerror for POSIX errno values. + // For Windows-specific error codes, fall back to FormatMessage. + const MAX_POSIX_ERRNO: i32 = 127; + if errno > 0 && errno <= MAX_POSIX_ERRNO { + let ptr = unsafe { libc::strerror(errno) }; + if !ptr.is_null() { + let s = unsafe { core::ffi::CStr::from_ptr(ptr) }.to_string_lossy(); + if !s.starts_with("Unknown error") { + break 'msg s.into_owned(); + } + } + } + self.to_string() + }; + #[cfg(unix)] + let msg = { + let ptr = unsafe { libc::strerror(errno) }; + if !ptr.is_null() { + unsafe { core::ffi::CStr::from_ptr(ptr) } + .to_string_lossy() + .into_owned() + } else { + self.to_string() + } + }; + #[cfg(not(any(windows, unix)))] + let msg = self.to_string(); + + #[allow(unused_mut)] + let mut builder = OSErrorBuilder::with_errno(errno, msg, vm); + #[cfg(windows)] + if let Some(winerror) = self.raw_os_error() { + builder = builder.winerror(winerror.to_pyobject(vm)); + } + builder + } +} + +impl ToPyException for std::io::Error { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + let builder = self.to_os_error_builder(vm); + builder.into_pyexception(vm) + } +} + +impl IntoPyException for std::io::Error { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + self.to_pyexception(vm) + } +} + +#[cfg(unix)] +impl IntoPyException for nix::Error { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + std::io::Error::from(self).into_pyexception(vm) + } +} + +#[cfg(unix)] +impl IntoPyException for rustix::io::Errno { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + std::io::Error::from(self).into_pyexception(vm) + } +} + +pub(super) mod types { + use crate::common::lock::PyRwLock; + use crate::object::{MaybeTraverse, Traverse, TraverseFn}; + #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] + use crate::{ + AsObject, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + VirtualMachine, + builtins::{ + PyInt, PyStrRef, PyTupleRef, PyType, PyTypeRef, traceback::PyTracebackRef, + tuple::IntoPyTuple, + }, + convert::ToPyResult, + function::{ArgBytesLike, FuncArgs, KwArgs}, + types::{Constructor, Initializer}, + }; + use crossbeam_utils::atomic::AtomicCell; + use itertools::Itertools; + use rustpython_common::{ + str::UnicodeEscapeCodepoint, + wtf8::{Wtf8, Wtf8Buf, wtf8_concat}, + }; + + // Re-export exception group types from dedicated module + pub use crate::exception_group::types::PyBaseExceptionGroup; + + // This module is designed to be used as `use builtins::*;`. + // Do not add any pub symbols not included in builtins module. + // `PyBaseExceptionRef` is the only exception. + + pub type PyBaseExceptionRef = PyRef<PyBaseException>; + + // Sorted By Hierarchy then alphabetized. + + #[pyclass(module = false, name = "BaseException", traverse = "manual")] + pub struct PyBaseException { + pub(super) traceback: PyRwLock<Option<PyTracebackRef>>, + pub(super) cause: PyRwLock<Option<PyRef<Self>>>, + pub(super) context: PyRwLock<Option<PyRef<Self>>>, + pub(super) suppress_context: AtomicCell<bool>, + pub(super) args: PyRwLock<PyTupleRef>, + } + + #[pyexception(name, base = PyBaseException, ctx = "system_exit")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySystemExit(PyBaseException); + + // SystemExit_init: has its own __init__ that sets the code attribute + #[pyexception(with(Initializer))] + impl PySystemExit {} + + impl Initializer for PySystemExit { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Call BaseException_init first (handles args) + PyBaseException::slot_init(zelf, args, vm) + // Note: code is computed dynamically via system_exit_code getter + // so we don't need to set it here explicitly + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(name, base = PyBaseException, ctx = "generator_exit", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyGeneratorExit(PyBaseException); + + #[pyexception(name, base = PyBaseException, ctx = "keyboard_interrupt", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyKeyboardInterrupt(PyBaseException); + + #[pyexception(name, base = PyBaseException, ctx = "exception_type", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyException(PyBaseException); + + #[pyexception(name, base = PyException, ctx = "stop_iteration")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyStopIteration(PyException); + + #[pyexception(with(Initializer))] + impl PyStopIteration {} + + impl Initializer for PyStopIteration { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + zelf.set_attr("value", vm.unwrap_or_none(args.args.first().cloned()), vm)?; + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(name, base = PyException, ctx = "stop_async_iteration", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyStopAsyncIteration(PyException); + + #[pyexception(name, base = PyException, ctx = "arithmetic_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyArithmeticError(PyException); + + #[pyexception(name, base = PyArithmeticError, ctx = "floating_point_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyFloatingPointError(PyArithmeticError); + #[pyexception(name, base = PyArithmeticError, ctx = "overflow_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyOverflowError(PyArithmeticError); + + #[pyexception(name, base = PyArithmeticError, ctx = "zero_division_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyZeroDivisionError(PyArithmeticError); + + #[pyexception(name, base = PyException, ctx = "assertion_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyAssertionError(PyException); + + #[pyexception(name, base = PyException, ctx = "attribute_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyAttributeError(PyException); + + #[pyexception(with(Initializer))] + impl PyAttributeError {} + + impl Initializer for PyAttributeError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Only 'name' and 'obj' kwargs are allowed + let mut kwargs = args.kwargs.clone(); + let name = kwargs.swap_remove("name"); + let obj = kwargs.swap_remove("obj"); + + // Reject unknown kwargs + if let Some(invalid_key) = kwargs.keys().next() { + return Err(vm.new_type_error(format!( + "AttributeError() got an unexpected keyword argument '{invalid_key}'" + ))); + } + + // Pass args without kwargs to BaseException_init + let base_args = FuncArgs::new(args.args.clone(), KwArgs::default()); + PyBaseException::slot_init(zelf.clone(), base_args, vm)?; + + // Set attributes + zelf.set_attr("name", vm.unwrap_or_none(name), vm)?; + zelf.set_attr("obj", vm.unwrap_or_none(obj), vm)?; + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(name, base = PyException, ctx = "buffer_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyBufferError(PyException); + + #[pyexception(name, base = PyException, ctx = "eof_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyEOFError(PyException); + + #[pyexception(name, base = PyException, ctx = "import_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyImportError(PyException); + + #[pyexception(with(Initializer))] + impl PyImportError { + #[pymethod] + fn __reduce__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyTupleRef { + let obj = exc.as_object().to_owned(); + let mut result: Vec<PyObjectRef> = vec![ + obj.class().to_owned().into(), + vm.new_tuple((exc.get_arg(0).unwrap(),)).into(), + ]; + + if let Some(dict) = obj.dict().filter(|x| !x.is_empty()) { + result.push(dict.into()); + } + + result.into_pytuple(vm) + } + } + + impl Initializer for PyImportError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Only 'name', 'path', 'name_from' kwargs are allowed + let mut kwargs = args.kwargs.clone(); + let name = kwargs.swap_remove("name"); + let path = kwargs.swap_remove("path"); + let name_from = kwargs.swap_remove("name_from"); + + // Check for any remaining invalid keyword arguments + if let Some(invalid_key) = kwargs.keys().next() { + return Err(vm.new_type_error(format!( + "'{invalid_key}' is an invalid keyword argument for ImportError" + ))); + } + + let dict = zelf.dict().unwrap(); + dict.set_item("name", vm.unwrap_or_none(name), vm)?; + dict.set_item("path", vm.unwrap_or_none(path), vm)?; + dict.set_item("name_from", vm.unwrap_or_none(name_from), vm)?; + PyBaseException::slot_init(zelf, args, vm) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(name, base = PyImportError, ctx = "module_not_found_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyModuleNotFoundError(PyImportError); + + #[pyexception(name, base = PyException, ctx = "lookup_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyLookupError(PyException); + + #[pyexception(name, base = PyLookupError, ctx = "index_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyIndexError(PyLookupError); + + #[pyexception(name, base = PyLookupError, ctx = "key_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyKeyError(PyLookupError); + + #[pyexception] + impl PyKeyError { + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let args = zelf.args(); + Ok(if args.len() == 1 { + vm.exception_args_as_string(args, false) + .into_iter() + .exactly_one() + .unwrap() + } else { + zelf.__str__(vm)? + }) + } + } + + #[pyexception(name, base = PyException, ctx = "memory_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyMemoryError(PyException); + + #[pyexception(name, base = PyException, ctx = "name_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyNameError(PyException); + + // NameError_init: handles the .name. kwarg + #[pyexception(with(Initializer))] + impl PyNameError {} + + impl Initializer for PyNameError { + type Args = FuncArgs; + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + // Only 'name' kwarg is allowed + let mut kwargs = args.kwargs.clone(); + let name = kwargs.swap_remove("name"); + + // Reject unknown kwargs + if let Some(invalid_key) = kwargs.keys().next() { + return Err(vm.new_type_error(format!( + "NameError() got an unexpected keyword argument '{invalid_key}'" + ))); + } + + // Pass args without kwargs to BaseException_init + let base_args = FuncArgs::new(args.args.clone(), KwArgs::default()); + PyBaseException::slot_init(zelf.clone(), base_args, vm)?; + + // Set name attribute if provided + if let Some(name) = name { + zelf.set_attr("name", name, vm)?; + } + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(name, base = PyNameError, ctx = "unbound_local_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyUnboundLocalError(PyNameError); + + #[pyexception(name, base = PyException, ctx = "os_error")] + #[repr(C)] + pub struct PyOSError { + base: PyException, + errno: PyAtomicRef<Option<PyObject>>, + strerror: PyAtomicRef<Option<PyObject>>, + filename: PyAtomicRef<Option<PyObject>>, + filename2: PyAtomicRef<Option<PyObject>>, + #[cfg(windows)] + winerror: PyAtomicRef<Option<PyObject>>, + // For BlockingIOError: characters written before blocking occurred + // -1 means not set (AttributeError when accessed) + written: AtomicCell<isize>, + } + + impl crate::class::PySubclass for PyOSError { + type Base = PyException; + fn as_base(&self) -> &Self::Base { + &self.base + } + } + + impl core::fmt::Debug for PyOSError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyOSError").finish_non_exhaustive() + } + } + + unsafe impl Traverse for PyOSError { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.base.try_traverse(tracer_fn); + if let Some(obj) = self.errno.deref() { + tracer_fn(obj); + } + if let Some(obj) = self.strerror.deref() { + tracer_fn(obj); + } + if let Some(obj) = self.filename.deref() { + tracer_fn(obj); + } + if let Some(obj) = self.filename2.deref() { + tracer_fn(obj); + } + #[cfg(windows)] + if let Some(obj) = self.winerror.deref() { + tracer_fn(obj); + } + } + } + + // OS Errors: + impl Constructor for PyOSError { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: FuncArgs, vm: &VirtualMachine) -> PyResult<Self> { + let len = args.args.len(); + // CPython only sets errno/strerror when args len is 2-5 + let (errno, strerror) = if (2..=5).contains(&len) { + (Some(args.args[0].clone()), Some(args.args[1].clone())) + } else { + (None, None) + }; + let filename = if (3..=5).contains(&len) { + Some(args.args[2].clone()) + } else { + None + }; + let filename2 = if len == 5 { + args.args.get(4).cloned() + } else { + None + }; + // Truncate args for base exception when 3-5 args + let base_args = if (3..=5).contains(&len) { + args.args[..2].to_vec() + } else { + args.args.to_vec() + }; + let base_exception = PyBaseException::new(base_args, vm); + Ok(Self { + base: PyException(base_exception), + errno: errno.into(), + strerror: strerror.into(), + filename: filename.into(), + filename2: filename2.into(), + #[cfg(windows)] + winerror: None.into(), + written: AtomicCell::new(-1), + }) + } + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // We need this method, because of how `CPython` copies `init` + // from `BaseException` in `SimpleExtendsException` macro. + // See: `BaseException_new` + if *cls.name() == *vm.ctx.exceptions.os_error.name() { + let args_vec = args.args.to_vec(); + let len = args_vec.len(); + if (2..=5).contains(&len) { + let errno = &args_vec[0]; + if let Some(error) = errno + .downcast_ref::<PyInt>() + .and_then(|errno| errno.try_to_primitive::<i32>(vm).ok()) + .and_then(|errno| super::errno_to_exc_type(errno, vm)) + .and_then(|typ| vm.invoke_exception(typ.to_owned(), args_vec).ok()) + { + return error.to_pyresult(vm); + } + } + } + let payload = Self::py_new(&cls, args, vm)?; + payload.into_ref_with_type(vm, cls).map(Into::into) + } + } + + impl Initializer for PyOSError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let len = args.args.len(); + let mut new_args = args; + + // All OSError subclasses use #[repr(transparent)] wrapping PyOSError, + // so we can safely access the PyOSError fields through pointer cast + // SAFETY: All OSError subclasses (FileNotFoundError, etc.) are + // #[repr(transparent)] wrappers around PyOSError with identical memory layout + #[allow(deprecated)] + let exc: &Py<PyOSError> = zelf.downcast_ref::<PyOSError>().unwrap(); + + // Check if this is BlockingIOError - need to handle characters_written + let is_blocking_io_error = + zelf.class() + .is(vm.ctx.exceptions.blocking_io_error.as_ref()); + + // SAFETY: slot_init is called during object initialization, + // so fields are None and swap result can be safely ignored + let mut set_filename = true; + if len <= 5 { + // Only set errno/strerror when args len is 2-5 + if 2 <= len { + let _ = unsafe { exc.errno.swap(Some(new_args.args[0].clone())) }; + let _ = unsafe { exc.strerror.swap(Some(new_args.args[1].clone())) }; + } + if 3 <= len { + let third_arg = &new_args.args[2]; + // BlockingIOError's 3rd argument can be the number of characters written + if is_blocking_io_error + && !vm.is_none(third_arg) + && crate::protocol::PyNumber::check(third_arg) + && let Ok(written) = third_arg.try_index(vm) + && let Ok(n) = written.try_to_primitive::<isize>(vm) + { + exc.written.store(n); + set_filename = false; + // Clear filename that was set in py_new + let _ = unsafe { exc.filename.swap(None) }; + } + if set_filename { + let _ = unsafe { exc.filename.swap(Some(third_arg.clone())) }; + } + } + #[cfg(windows)] + if 4 <= len { + let winerror = new_args.args.get(3).cloned(); + // Store original winerror + let _ = unsafe { exc.winerror.swap(winerror.clone()) }; + + // Convert winerror to errno and update errno + args[0] + if let Some(errno) = winerror + .as_ref() + .and_then(|w| w.downcast_ref::<crate::builtins::PyInt>()) + .and_then(|w| w.try_to_primitive::<i32>(vm).ok()) + .map(crate::common::os::winerror_to_errno) + { + let errno_obj = vm.new_pyobj(errno); + let _ = unsafe { exc.errno.swap(Some(errno_obj.clone())) }; + new_args.args[0] = errno_obj; + } + } + if len == 5 { + let _ = unsafe { exc.filename2.swap(new_args.args.get(4).cloned()) }; + } + } + + // args are truncated to 2 for compatibility (only when 2-5 args and filename is not None) + // truncation happens inside "if (filename && filename != Py_None)" block + let has_filename = exc.filename.to_owned().filter(|f| !vm.is_none(f)).is_some(); + if (3..=5).contains(&len) && has_filename { + new_args.args.truncate(2); + } + PyBaseException::slot_init(zelf, new_args, vm) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(with(Constructor, Initializer))] + impl PyOSError { + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let obj = zelf.as_object(); + + // Get OSError fields directly + let errno_field = obj.get_attr("errno", vm).ok().filter(|v| !vm.is_none(v)); + let strerror = obj.get_attr("strerror", vm).ok().filter(|v| !vm.is_none(v)); + let filename = obj.get_attr("filename", vm).ok().filter(|v| !vm.is_none(v)); + let filename2 = obj + .get_attr("filename2", vm) + .ok() + .filter(|v| !vm.is_none(v)); + #[cfg(windows)] + let winerror = obj.get_attr("winerror", vm).ok().filter(|v| !vm.is_none(v)); + + // Windows: winerror takes priority over errno + #[cfg(windows)] + if let Some(ref win_err) = winerror { + let code = win_err.str(vm)?; + if let Some(ref f) = filename { + let msg = strerror + .as_ref() + .map(|s| s.str(vm)) + .transpose()? + .map(|s| s.to_string()) + .unwrap_or_else(|| "None".to_owned()); + if let Some(ref f2) = filename2 { + return Ok(vm.ctx.new_str(format!( + "[WinError {}] {}: {} -> {}", + code, + msg, + f.repr(vm)?, + f2.repr(vm)? + ))); + } + return Ok(vm.ctx.new_str(format!( + "[WinError {}] {}: {}", + code, + msg, + f.repr(vm)? + ))); + } + // winerror && strerror (no filename) + if let Some(ref s) = strerror { + return Ok(vm + .ctx + .new_str(format!("[WinError {}] {}", code, s.str(vm)?))); + } + } + + // Non-Windows or fallback: use errno + if let Some(ref f) = filename { + let errno_str = errno_field + .as_ref() + .map(|e| e.str(vm)) + .transpose()? + .map(|s| s.to_string()) + .unwrap_or_else(|| "None".to_owned()); + let msg = strerror + .as_ref() + .map(|s| s.str(vm)) + .transpose()? + .map(|s| s.to_string()) + .unwrap_or_else(|| "None".to_owned()); + if let Some(ref f2) = filename2 { + return Ok(vm.ctx.new_str(format!( + "[Errno {}] {}: {} -> {}", + errno_str, + msg, + f.repr(vm)?, + f2.repr(vm)? + ))); + } + return Ok(vm.ctx.new_str(format!( + "[Errno {}] {}: {}", + errno_str, + msg, + f.repr(vm)? + ))); + } + + // errno && strerror (no filename) + if let (Some(e), Some(s)) = (&errno_field, &strerror) { + return Ok(vm + .ctx + .new_str(format!("[Errno {}] {}", e.str(vm)?, s.str(vm)?))); + } + + // fallback to BaseException.__str__ + zelf.__str__(vm) + } + + #[pymethod] + fn __reduce__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyTupleRef { + let args = exc.args(); + let obj = exc.as_object().to_owned(); + let mut result: Vec<PyObjectRef> = vec![obj.class().to_owned().into()]; + + if args.len() >= 2 && args.len() <= 5 { + // SAFETY: len() == 2 is checked so get_arg 1 or 2 won't panic + let errno = exc.get_arg(0).unwrap(); + let msg = exc.get_arg(1).unwrap(); + + if let Ok(filename) = obj.get_attr("filename", vm) { + if !vm.is_none(&filename) { + let mut args_reduced: Vec<PyObjectRef> = vec![errno, msg, filename]; + + let filename2 = obj + .get_attr("filename2", vm) + .ok() + .filter(|f| !vm.is_none(f)); + #[cfg(windows)] + let winerror = obj.get_attr("winerror", vm).ok().filter(|w| !vm.is_none(w)); + + if let Some(filename2) = filename2 { + #[cfg(windows)] + { + args_reduced.push(winerror.unwrap_or_else(|| vm.ctx.none())); + } + #[cfg(not(windows))] + args_reduced.push(vm.ctx.none()); + args_reduced.push(filename2); + } else { + // Diverges from CPython: include winerror even without + // filename2 so it survives pickle round-trips. + #[cfg(windows)] + if let Some(winerror) = winerror { + args_reduced.push(winerror); + } + } + result.push(args_reduced.into_pytuple(vm).into()); + } else { + // filename is None - use original args as-is + // (may contain winerror at position 3) + result.push(args.into()); + } + } else { + result.push(args.into()); + } + } else { + result.push(args.into()); + } + + if let Some(dict) = obj.dict().filter(|x| !x.is_empty()) { + result.push(dict.into()); + } + result.into_pytuple(vm) + } + + // Getters and setters for OSError fields + #[pygetset] + fn errno(&self) -> Option<PyObjectRef> { + self.errno.to_owned() + } + + #[pygetset(setter)] + fn set_errno(&self, value: Option<PyObjectRef>, vm: &VirtualMachine) { + self.errno.swap_to_temporary_refs(value, vm); + } + + #[pygetset] + fn strerror(&self) -> Option<PyObjectRef> { + self.strerror.to_owned() + } + + #[pygetset(setter, name = "strerror")] + fn set_strerror(&self, value: Option<PyObjectRef>, vm: &VirtualMachine) { + self.strerror.swap_to_temporary_refs(value, vm); + } + + #[pygetset] + fn filename(&self) -> Option<PyObjectRef> { + self.filename.to_owned() + } + + #[pygetset(setter)] + fn set_filename(&self, value: Option<PyObjectRef>, vm: &VirtualMachine) { + self.filename.swap_to_temporary_refs(value, vm); + } + + #[pygetset] + fn filename2(&self) -> Option<PyObjectRef> { + self.filename2.to_owned() + } + + #[pygetset(setter)] + fn set_filename2(&self, value: Option<PyObjectRef>, vm: &VirtualMachine) { + self.filename2.swap_to_temporary_refs(value, vm); + } + + #[cfg(windows)] + #[pygetset] + fn winerror(&self) -> Option<PyObjectRef> { + self.winerror.to_owned() + } + + #[cfg(windows)] + #[pygetset(setter)] + fn set_winerror(&self, value: Option<PyObjectRef>, vm: &VirtualMachine) { + self.winerror.swap_to_temporary_refs(value, vm); + } + + #[pygetset] + fn characters_written(&self, vm: &VirtualMachine) -> PyResult<isize> { + let written = self.written.load(); + if written == -1 { + Err(vm.new_attribute_error("characters_written")) + } else { + Ok(written) + } + } + + #[pygetset(setter)] + fn set_characters_written( + &self, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + match value { + None => { + // Deleting the attribute + if self.written.load() == -1 { + Err(vm.new_attribute_error("characters_written")) + } else { + self.written.store(-1); + Ok(()) + } + } + Some(v) => { + let n = v + .try_index(vm)? + .try_to_primitive::<isize>(vm) + .map_err(|_| { + vm.new_value_error("cannot convert characters_written value to isize") + })?; + self.written.store(n); + Ok(()) + } + } + } + } + + #[pyexception(name, base = PyOSError, ctx = "blocking_io_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyBlockingIOError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "child_process_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyChildProcessError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "connection_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyConnectionError(PyOSError); + + #[pyexception(name, base = PyConnectionError, ctx = "broken_pipe_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyBrokenPipeError(PyConnectionError); + + #[pyexception( + name, + base = PyConnectionError, + ctx = "connection_aborted_error", + impl + )] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyConnectionAbortedError(PyConnectionError); + + #[pyexception( + name, + base = PyConnectionError, + ctx = "connection_refused_error", + impl + )] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyConnectionRefusedError(PyConnectionError); + + #[pyexception(name, base = PyConnectionError, ctx = "connection_reset_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyConnectionResetError(PyConnectionError); + + #[pyexception(name, base = PyOSError, ctx = "file_exists_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyFileExistsError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "file_not_found_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyFileNotFoundError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "interrupted_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyInterruptedError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "is_a_directory_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyIsADirectoryError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "not_a_directory_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyNotADirectoryError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "permission_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyPermissionError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "process_lookup_error", impl)] + #[repr(transparent)] + #[derive(Debug)] + pub struct PyProcessLookupError(PyOSError); + + #[pyexception(name, base = PyOSError, ctx = "timeout_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyTimeoutError(PyOSError); + + #[pyexception(name, base = PyException, ctx = "reference_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyReferenceError(PyException); + + #[pyexception(name, base = PyException, ctx = "runtime_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyRuntimeError(PyException); + + #[pyexception(name, base = PyRuntimeError, ctx = "not_implemented_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyNotImplementedError(PyRuntimeError); + + #[pyexception(name, base = PyRuntimeError, ctx = "recursion_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyRecursionError(PyRuntimeError); + + #[pyexception(name, base = PyRuntimeError, ctx = "python_finalization_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyPythonFinalizationError(PyRuntimeError); + + #[pyexception(name, base = PyException, ctx = "syntax_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySyntaxError(PyException); + + #[pyexception(with(Initializer))] + impl PySyntaxError { + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + fn basename(filename: &Wtf8) -> &Wtf8 { + let bytes = filename.as_bytes(); + let pos = if cfg!(windows) { + bytes.iter().rposition(|&b| b == b'/' || b == b'\\') + } else { + bytes.iter().rposition(|&b| b == b'/') + }; + match pos { + // SAFETY: splitting at ASCII byte boundary preserves WTF-8 validity + Some(pos) => unsafe { Wtf8::from_bytes_unchecked(&bytes[pos + 1..]) }, + None => filename, + } + } + + let maybe_lineno = zelf + .as_object() + .get_attr("lineno", vm) + .and_then(|obj| obj.str_utf8(vm)) + .ok(); + let maybe_filename = zelf.as_object().get_attr("filename", vm).ok().map(|obj| { + obj.str(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<filename str() failed>")) + }); + + let msg = match zelf.as_object().get_attr("msg", vm) { + Ok(obj) => obj + .str(vm) + .unwrap_or_else(|_| vm.ctx.new_str("<msg str() failed>")), + Err(_) => { + // Fallback to the base formatting if the msg attribute was deleted or attribute lookup fails for any reason. + return Py::<PyBaseException>::__str__(zelf, vm); + } + }; + + let msg_with_location_info: Wtf8Buf = match (maybe_lineno, maybe_filename) { + (Some(lineno), Some(filename)) => wtf8_concat!( + msg.as_wtf8(), + " (", + basename(filename.as_wtf8()), + ", line ", + lineno.as_str(), + ")" + ), + (Some(lineno), None) => { + wtf8_concat!(msg.as_wtf8(), " (line ", lineno.as_str(), ")") + } + (None, Some(filename)) => { + wtf8_concat!(msg.as_wtf8(), " (", basename(filename.as_wtf8()), ")") + } + (None, None) => msg.as_wtf8().to_owned(), + }; + + Ok(vm.ctx.new_str(msg_with_location_info)) + } + } + + impl Initializer for PySyntaxError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let len = args.args.len(); + let new_args = args; + + zelf.set_attr("print_file_and_line", vm.ctx.none(), vm)?; + + if len == 2 + && let Ok(location_tuple) = new_args.args[1] + .clone() + .downcast::<crate::builtins::PyTuple>() + { + let location_tup_len = location_tuple.len(); + for (i, &attr) in [ + "filename", + "lineno", + "offset", + "text", + "end_lineno", + "end_offset", + ] + .iter() + .enumerate() + { + if location_tup_len > i { + zelf.set_attr(attr, location_tuple[i].to_owned(), vm)?; + } else { + break; + } + } + } + + PyBaseException::slot_init(zelf, new_args, vm) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + // MiddlingExtendsException: inherits __init__ from SyntaxError via MRO + #[pyexception( + name = "_IncompleteInputError", + base = PySyntaxError, + ctx = "incomplete_input_error", + impl + )] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyIncompleteInputError(PySyntaxError); + + #[pyexception(name, base = PySyntaxError, ctx = "indentation_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyIndentationError(PySyntaxError); + + #[pyexception(name, base = PyIndentationError, ctx = "tab_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyTabError(PyIndentationError); + + #[pyexception(name, base = PyException, ctx = "system_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySystemError(PyException); + + #[pyexception(name, base = PyException, ctx = "type_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyTypeError(PyException); + + #[pyexception(name, base = PyException, ctx = "value_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyValueError(PyException); + + #[pyexception(name, base = PyValueError, ctx = "unicode_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyUnicodeError(PyValueError); + + #[pyexception(name, base = PyUnicodeError, ctx = "unicode_decode_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyUnicodeDecodeError(PyUnicodeError); + + #[pyexception(with(Initializer))] + impl PyUnicodeDecodeError { + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let Ok(object) = zelf.as_object().get_attr("object", vm) else { + return Ok(vm.ctx.empty_str.to_owned()); + }; + let object: ArgBytesLike = object.try_into_value(vm)?; + let encoding: PyStrRef = zelf + .as_object() + .get_attr("encoding", vm)? + .try_into_value(vm)?; + let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = zelf + .as_object() + .get_attr("reason", vm)? + .try_into_value(vm)?; + Ok(vm.ctx.new_str(if start < object.len() && end <= object.len() && end == start + 1 { + let b = object.borrow_buf()[start]; + format!( + "'{encoding}' codec can't decode byte {b:#02x} in position {start}: {reason}" + ) + } else { + format!( + "'{encoding}' codec can't decode bytes in position {start}-{}: {reason}", + end - 1, + ) + })) + } + } + + impl Initializer for PyUnicodeDecodeError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + type Args = (PyStrRef, ArgBytesLike, isize, isize, PyStrRef); + let (encoding, object, start, end, reason): Args = args.bind(vm)?; + zelf.set_attr("encoding", encoding, vm)?; + let object_as_bytes = vm.ctx.new_bytes(object.borrow_buf().to_vec()); + zelf.set_attr("object", object_as_bytes, vm)?; + zelf.set_attr("start", vm.ctx.new_int(start), vm)?; + zelf.set_attr("end", vm.ctx.new_int(end), vm)?; + zelf.set_attr("reason", reason, vm)?; + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(name, base = PyUnicodeError, ctx = "unicode_encode_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyUnicodeEncodeError(PyUnicodeError); + + #[pyexception(with(Initializer))] + impl PyUnicodeEncodeError { + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let Ok(object) = zelf.as_object().get_attr("object", vm) else { + return Ok(vm.ctx.empty_str.to_owned()); + }; + let object: PyStrRef = object.try_into_value(vm)?; + let encoding: PyStrRef = zelf + .as_object() + .get_attr("encoding", vm)? + .try_into_value(vm)?; + let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = zelf + .as_object() + .get_attr("reason", vm)? + .try_into_value(vm)?; + Ok(vm.ctx.new_str(if start < object.char_len() && end <= object.char_len() && end == start + 1 { + let ch = object.as_wtf8().code_points().nth(start).unwrap(); + format!( + "'{encoding}' codec can't encode character '{}' in position {start}: {reason}", + UnicodeEscapeCodepoint(ch) + ) + } else { + format!( + "'{encoding}' codec can't encode characters in position {start}-{}: {reason}", + end - 1, + ) + })) + } + } + + impl Initializer for PyUnicodeEncodeError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + type Args = (PyStrRef, PyStrRef, isize, isize, PyStrRef); + let (encoding, object, start, end, reason): Args = args.bind(vm)?; + zelf.set_attr("encoding", encoding, vm)?; + zelf.set_attr("object", object, vm)?; + zelf.set_attr("start", vm.ctx.new_int(start), vm)?; + zelf.set_attr("end", vm.ctx.new_int(end), vm)?; + zelf.set_attr("reason", reason, vm)?; + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyexception(name, base = PyUnicodeError, ctx = "unicode_translate_error")] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyUnicodeTranslateError(PyUnicodeError); + + #[pyexception(with(Initializer))] + impl PyUnicodeTranslateError { + #[pymethod] + fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let Ok(object) = zelf.as_object().get_attr("object", vm) else { + return Ok(vm.ctx.empty_str.to_owned()); + }; + let object: PyStrRef = object.try_into_value(vm)?; + let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; + let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; + let reason: PyStrRef = zelf + .as_object() + .get_attr("reason", vm)? + .try_into_value(vm)?; + Ok(vm.ctx.new_str( + if start < object.char_len() && end <= object.char_len() && end == start + 1 { + let ch = object.as_wtf8().code_points().nth(start).unwrap(); + format!( + "can't translate character '{}' in position {start}: {reason}", + UnicodeEscapeCodepoint(ch) + ) + } else { + format!( + "can't translate characters in position {start}-{}: {reason}", + end - 1, + ) + }, + )) + } + } + + impl Initializer for PyUnicodeTranslateError { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + type Args = (PyStrRef, isize, isize, PyStrRef); + let (object, start, end, reason): Args = args.bind(vm)?; + zelf.set_attr("object", object, vm)?; + zelf.set_attr("start", vm.ctx.new_int(start), vm)?; + zelf.set_attr("end", vm.ctx.new_int(end), vm)?; + zelf.set_attr("reason", reason, vm)?; + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + /// JIT error. + #[cfg(feature = "jit")] + #[pyexception(name, base = PyException, ctx = "jit_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyJitError(PyException); + + // Warnings + #[pyexception(name, base = PyException, ctx = "warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyWarning(PyException); + + #[pyexception(name, base = PyWarning, ctx = "deprecation_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyDeprecationWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "pending_deprecation_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyPendingDeprecationWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "runtime_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyRuntimeWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "syntax_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PySyntaxWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "user_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyUserWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "future_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyFutureWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "import_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyImportWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "unicode_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyUnicodeWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "bytes_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyBytesWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "resource_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyResourceWarning(PyWarning); + + #[pyexception(name, base = PyWarning, ctx = "encoding_warning", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyEncodingWarning(PyWarning); +} + +/// Check if match_type is valid for except* (must be exception type, not ExceptionGroup). +fn check_except_star_type_valid(match_type: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let base_exc: PyObjectRef = vm.ctx.exceptions.base_exception_type.to_owned().into(); + let base_eg: PyObjectRef = vm.ctx.exceptions.base_exception_group.to_owned().into(); + + // Helper to check a single type + let check_one = |exc_type: &PyObjectRef| -> PyResult<()> { + // Must be a subclass of BaseException + if !exc_type.is_subclass(&base_exc, vm)? { + return Err(vm.new_type_error( + "catching classes that do not inherit from BaseException is not allowed", + )); + } + // Must not be a subclass of BaseExceptionGroup + if exc_type.is_subclass(&base_eg, vm)? { + return Err(vm.new_type_error( + "catching ExceptionGroup with except* is not allowed. Use except instead.", + )); + } + Ok(()) + }; + + // If it's a tuple, check each element + if let Ok(tuple) = match_type.clone().downcast::<PyTuple>() { + for item in tuple.iter() { + check_one(item)?; + } + } else { + check_one(match_type)?; + } + Ok(()) +} + +/// Match exception against except* handler type. +/// Returns (rest, match) tuple. +pub fn exception_group_match( + exc_value: &PyObjectRef, + match_type: &PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, PyObjectRef)> { + // Implements _PyEval_ExceptionGroupMatch + + // If exc_value is None, return (None, None) + if vm.is_none(exc_value) { + return Ok((vm.ctx.none(), vm.ctx.none())); + } + + // Validate match_type and reject ExceptionGroup/BaseExceptionGroup + check_except_star_type_valid(match_type, vm)?; + + // Check if exc_value matches match_type + if exc_value.is_instance(match_type, vm)? { + // Full match of exc itself + let is_eg = exc_value.fast_isinstance(vm.ctx.exceptions.base_exception_group); + let matched = if is_eg { + exc_value.clone() + } else { + // Naked exception - wrap it in ExceptionGroup + let excs = vm.ctx.new_tuple(vec![exc_value.clone()]); + let eg_type: PyObjectRef = crate::exception_group::exception_group().to_owned().into(); + let wrapped = eg_type.call((vm.ctx.new_str(""), excs), vm)?; + // Copy traceback from original exception + if let Ok(exc) = exc_value.clone().downcast::<types::PyBaseException>() + && let Some(tb) = exc.__traceback__() + && let Ok(wrapped_exc) = wrapped.clone().downcast::<types::PyBaseException>() + { + let _ = wrapped_exc.set___traceback__(tb.into(), vm); + } + wrapped + }; + return Ok((vm.ctx.none(), matched)); + } + + // Check for partial match if it's an exception group + if exc_value.fast_isinstance(vm.ctx.exceptions.base_exception_group) { + let pair = vm.call_method(exc_value, "split", (match_type.clone(),))?; + if !pair.class().is(vm.ctx.types.tuple_type) { + return Err(vm.new_type_error(format!( + "{}.split must return a tuple, not {}", + exc_value.class().name(), + pair.class().name() + ))); + } + let pair_tuple: PyTupleRef = pair.try_into_value(vm)?; + if pair_tuple.len() < 2 { + return Err(vm.new_type_error(format!( + "{}.split must return a 2-tuple, got tuple of size {}", + exc_value.class().name(), + pair_tuple.len() + ))); + } + let matched = pair_tuple[0].clone(); + let rest = pair_tuple[1].clone(); + return Ok((rest, matched)); + } + + // No match + Ok((exc_value.clone(), vm.ctx.none())) +} + +/// Prepare exception for reraise in except* block. +/// Implements _PyExc_PrepReraiseStar +pub fn prep_reraise_star(orig: PyObjectRef, excs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyList; + + let excs_list = excs + .downcast::<PyList>() + .map_err(|_| vm.new_type_error("expected list for prep_reraise_star"))?; + + let excs_vec: Vec<PyObjectRef> = excs_list.borrow_vec().to_vec(); + + // If no exceptions to process, return None + if excs_vec.is_empty() { + return Ok(vm.ctx.none()); + } + + // Special case: naked exception (not an ExceptionGroup) + // Only one except* clause could have executed, so there's at most one exception to raise + if !orig.fast_isinstance(vm.ctx.exceptions.base_exception_group) { + // Find first non-None exception + let first = excs_vec.into_iter().find(|e| !vm.is_none(e)); + return Ok(first.unwrap_or_else(|| vm.ctx.none())); + } + + // Split excs into raised (new) and reraised (from original) by comparing metadata + let mut raised: Vec<PyObjectRef> = Vec::new(); + let mut reraised: Vec<PyObjectRef> = Vec::new(); + + for exc in excs_vec { + if vm.is_none(&exc) { + continue; + } + // Check if this exception came from the original group + if is_exception_from_orig(&exc, &orig, vm) { + reraised.push(exc); + } else { + raised.push(exc); + } + } + + // If no exceptions to reraise, return None + if raised.is_empty() && reraised.is_empty() { + return Ok(vm.ctx.none()); + } + + // Project reraised exceptions onto original structure to preserve nesting + let reraised_eg = exception_group_projection(&orig, &reraised, vm)?; + + // If no new raised exceptions, just return the reraised projection + if raised.is_empty() { + return Ok(reraised_eg); + } + + // Combine raised with reraised_eg + if !vm.is_none(&reraised_eg) { + raised.push(reraised_eg); + } + + // If only one exception, return it directly + if raised.len() == 1 { + return Ok(raised.into_iter().next().unwrap()); + } + + // Create new ExceptionGroup for multiple exceptions + let excs_tuple = vm.ctx.new_tuple(raised); + let eg_type: PyObjectRef = crate::exception_group::exception_group().to_owned().into(); + eg_type.call((vm.ctx.new_str(""), excs_tuple), vm) +} + +/// Check if an exception came from the original group (for reraise detection). +/// Instead of comparing metadata (which can be modified when caught), we compare +/// leaf exception object IDs. split() preserves leaf exception identity. +fn is_exception_from_orig(exc: &PyObjectRef, orig: &PyObjectRef, vm: &VirtualMachine) -> bool { + // Collect leaf exception IDs from exc + let mut exc_leaf_ids = HashSet::new(); + collect_exception_group_leaf_ids(exc, &mut exc_leaf_ids, vm); + + if exc_leaf_ids.is_empty() { + return false; + } + + // Collect leaf exception IDs from orig + let mut orig_leaf_ids = HashSet::new(); + collect_exception_group_leaf_ids(orig, &mut orig_leaf_ids, vm); + + // If ALL of exc's leaves are in orig's leaves, it's a reraise + exc_leaf_ids.iter().all(|id| orig_leaf_ids.contains(id)) +} + +/// Collect all leaf exception IDs from an exception (group). +fn collect_exception_group_leaf_ids( + exc: &PyObjectRef, + leaf_ids: &mut HashSet<usize>, + vm: &VirtualMachine, +) { + if vm.is_none(exc) { + return; + } + + // If not an exception group, it's a leaf - add its ID + if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_group) { + leaf_ids.insert(exc.get_id()); + return; + } + + // Recurse into exception group's exceptions + if let Ok(excs_attr) = exc.get_attr("exceptions", vm) + && let Ok(tuple) = excs_attr.downcast::<PyTuple>() + { + for e in tuple.iter() { + collect_exception_group_leaf_ids(e, leaf_ids, vm); + } + } +} + +/// Project orig onto keep list, preserving nested structure. +/// Returns an exception group containing only the exceptions from orig +/// that are also in the keep list. +fn exception_group_projection( + orig: &PyObjectRef, + keep: &[PyObjectRef], + vm: &VirtualMachine, +) -> PyResult { + if keep.is_empty() { + return Ok(vm.ctx.none()); + } + + // Collect all leaf IDs from keep list + let mut leaf_ids = HashSet::new(); + for e in keep { + collect_exception_group_leaf_ids(e, &mut leaf_ids, vm); + } + + // Split orig by matching leaf IDs, preserving structure + split_by_leaf_ids(orig, &leaf_ids, vm) +} + +/// Recursively split an exception (group) by leaf IDs. +/// Returns the projection containing only matching leaves with preserved structure. +fn split_by_leaf_ids( + exc: &PyObjectRef, + leaf_ids: &HashSet<usize>, + vm: &VirtualMachine, +) -> PyResult { + if vm.is_none(exc) { + return Ok(vm.ctx.none()); + } + + // If not an exception group, check if it's in our set + if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_group) { + if leaf_ids.contains(&exc.get_id()) { + return Ok(exc.clone()); + } + return Ok(vm.ctx.none()); + } + + // Exception group - recurse and reconstruct + let excs_attr = exc.get_attr("exceptions", vm)?; + let tuple: PyTupleRef = excs_attr.try_into_value(vm)?; + + let mut matched = Vec::new(); + for e in tuple.iter() { + let m = split_by_leaf_ids(e, leaf_ids, vm)?; + if !vm.is_none(&m) { + matched.push(m); + } + } + + if matched.is_empty() { + return Ok(vm.ctx.none()); + } + + // Reconstruct using derive() to preserve the structure (not necessarily the subclass type) + let matched_tuple = vm.ctx.new_tuple(matched); + vm.call_method(exc, "derive", (matched_tuple,)) +} diff --git a/crates/vm/src/format.rs b/crates/vm/src/format.rs new file mode 100644 index 00000000000..95bd893baea --- /dev/null +++ b/crates/vm/src/format.rs @@ -0,0 +1,235 @@ +use crate::{ + PyObject, PyResult, VirtualMachine, + builtins::PyBaseExceptionRef, + convert::{IntoPyException, ToPyException}, + function::FuncArgs, + stdlib::builtins, +}; + +use crate::common::format::*; +use crate::common::wtf8::{Wtf8, Wtf8Buf}; + +/// Get locale information from C `localeconv()` for the 'n' format specifier. +#[cfg(unix)] +pub(crate) fn get_locale_info() -> LocaleInfo { + use core::ffi::CStr; + unsafe { + let lc = libc::localeconv(); + if lc.is_null() { + return LocaleInfo { + thousands_sep: String::new(), + decimal_point: ".".to_string(), + grouping: vec![], + }; + } + let thousands_sep = CStr::from_ptr((*lc).thousands_sep) + .to_string_lossy() + .into_owned(); + let decimal_point = CStr::from_ptr((*lc).decimal_point) + .to_string_lossy() + .into_owned(); + let grouping = parse_grouping((*lc).grouping); + LocaleInfo { + thousands_sep, + decimal_point, + grouping, + } + } +} + +#[cfg(not(unix))] +pub(crate) fn get_locale_info() -> LocaleInfo { + LocaleInfo { + thousands_sep: String::new(), + decimal_point: ".".to_string(), + grouping: vec![], + } +} + +/// Parse C `lconv.grouping` into a `Vec<u8>`. +/// Reads bytes until 0 or CHAR_MAX, then appends 0 (meaning "repeat last group"). +#[cfg(unix)] +unsafe fn parse_grouping(grouping: *const libc::c_char) -> Vec<u8> { + let mut result = Vec::new(); + if grouping.is_null() { + return result; + } + unsafe { + let mut ptr = grouping; + while ![0, libc::c_char::MAX].contains(&*ptr) { + result.push(*ptr as u8); + ptr = ptr.add(1); + } + } + if !result.is_empty() { + result.push(0); + } + result +} + +impl IntoPyException for FormatSpecError { + fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { + match self { + Self::DecimalDigitsTooMany => { + vm.new_value_error("Too many decimal digits in format string") + } + Self::PrecisionTooBig => vm.new_value_error("Precision too big"), + Self::InvalidFormatSpecifier => vm.new_value_error("Invalid format specifier"), + Self::UnspecifiedFormat(c1, c2) => { + let msg = format!("Cannot specify '{c1}' with '{c2}'."); + vm.new_value_error(msg) + } + Self::ExclusiveFormat(c1, c2) => { + let msg = format!("Cannot specify both '{c1}' and '{c2}'."); + vm.new_value_error(msg) + } + Self::UnknownFormatCode(c, s) => { + let msg = format!("Unknown format code '{c}' for object of type '{s}'"); + vm.new_value_error(msg) + } + Self::PrecisionNotAllowed => { + vm.new_value_error("Precision not allowed in integer format specifier") + } + Self::NotAllowed(s) => { + let msg = format!("{s} not allowed with integer format specifier 'c'"); + vm.new_value_error(msg) + } + Self::UnableToConvert => vm.new_value_error("Unable to convert int to float"), + Self::CodeNotInRange => vm.new_overflow_error("%c arg not in range(0x110000)"), + Self::ZeroPadding => { + vm.new_value_error("Zero padding is not allowed in complex format specifier") + } + Self::AlignmentFlag => { + vm.new_value_error("'=' alignment flag is not allowed in complex format specifier") + } + Self::NotImplemented(c, s) => { + let msg = format!("Format code '{c}' for object of type '{s}' not implemented yet"); + vm.new_value_error(msg) + } + } + } +} + +impl ToPyException for FormatParseError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + match self { + Self::UnmatchedBracket => vm.new_value_error("expected '}' before end of string"), + _ => vm.new_value_error("Unexpected error parsing format string"), + } + } +} + +fn format_internal( + vm: &VirtualMachine, + format: &FormatString, + field_func: &mut impl FnMut(FieldType) -> PyResult, +) -> PyResult<Wtf8Buf> { + let mut final_string = Wtf8Buf::new(); + for part in &format.format_parts { + let pystr; + let result_string: &Wtf8 = match part { + FormatPart::Field { + field_name, + conversion_spec, + format_spec, + } => { + let FieldName { field_type, parts } = + FieldName::parse(field_name).map_err(|e| e.to_pyexception(vm))?; + + let mut argument = field_func(field_type)?; + + for name_part in parts { + match name_part { + FieldNamePart::Attribute(attribute) => { + argument = argument.get_attr(&vm.ctx.new_str(attribute), vm)?; + } + FieldNamePart::Index(index) => { + argument = argument.get_item(&index, vm)?; + } + FieldNamePart::StringIndex(index) => { + argument = argument.get_item(&index, vm)?; + } + } + } + + let nested_format = + FormatString::from_str(format_spec).map_err(|e| e.to_pyexception(vm))?; + let format_spec = format_internal(vm, &nested_format, field_func)?; + + let argument = match conversion_spec.and_then(FormatConversion::from_char) { + Some(FormatConversion::Str) => argument.str(vm)?.into(), + Some(FormatConversion::Repr) => argument.repr(vm)?.into(), + Some(FormatConversion::Ascii) => { + vm.ctx.new_str(builtins::ascii(argument, vm)?).into() + } + Some(FormatConversion::Bytes) => { + vm.call_method(&argument, identifier!(vm, decode).as_str(), ())? + } + None => argument, + }; + + // FIXME: compiler can intern specs using parser tree. Then this call can be interned_str + pystr = vm.format(&argument, vm.ctx.new_str(format_spec))?; + pystr.as_wtf8() + } + FormatPart::Literal(literal) => literal, + }; + final_string.push_wtf8(result_string); + } + Ok(final_string) +} + +pub(crate) fn format( + format: &FormatString, + arguments: &FuncArgs, + vm: &VirtualMachine, +) -> PyResult<Wtf8Buf> { + let mut auto_argument_index: usize = 0; + let mut seen_index = false; + format_internal(vm, format, &mut |field_type| match field_type { + FieldType::Auto => { + if seen_index { + return Err(vm.new_value_error( + "cannot switch from manual field specification to automatic field numbering", + )); + } + auto_argument_index += 1; + arguments + .args + .get(auto_argument_index - 1) + .cloned() + .ok_or_else(|| vm.new_index_error("tuple index out of range")) + } + FieldType::Index(index) => { + if auto_argument_index != 0 { + return Err(vm.new_value_error( + "cannot switch from automatic field numbering to manual field specification", + )); + } + seen_index = true; + arguments + .args + .get(index) + .cloned() + .ok_or_else(|| vm.new_index_error("tuple index out of range")) + } + FieldType::Keyword(keyword) => keyword + .as_str() + .ok() + .and_then(|keyword| arguments.get_optional_kwarg(keyword)) + .ok_or_else(|| vm.new_key_error(vm.ctx.new_str(keyword).into())), + }) +} + +pub(crate) fn format_map( + format: &FormatString, + dict: &PyObject, + vm: &VirtualMachine, +) -> PyResult<Wtf8Buf> { + format_internal(vm, format, &mut |field_type| match field_type { + FieldType::Auto | FieldType::Index(_) => { + Err(vm.new_value_error("Format string contains positional fields")) + } + FieldType::Keyword(keyword) => dict.get_item(&keyword, vm), + }) +} diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs new file mode 100644 index 00000000000..16fd9fa88b1 --- /dev/null +++ b/crates/vm/src/frame.rs @@ -0,0 +1,9578 @@ +// spell-checker: ignore compactlong compactlongs + +use crate::anystr::AnyStr; +#[cfg(feature = "flame")] +use crate::bytecode::InstructionMetadata; +use crate::{ + AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, PyStackRef, + TryFromObject, VirtualMachine, + builtins::{ + PyBaseException, PyBaseExceptionRef, PyBaseObject, PyCode, PyCoroutine, PyDict, PyDictRef, + PyFloat, PyFrozenSet, PyGenerator, PyInt, PyInterpolation, PyList, PyModule, PyProperty, + PySet, PySlice, PyStr, PyStrInterned, PyTemplate, PyTraceback, PyType, PyUtf8Str, + asyncgenerator::PyAsyncGenWrappedValue, + builtin_func::PyNativeFunction, + descriptor::{MemberGetter, PyMemberDescriptor, PyMethodDescriptor}, + frame::stack_analysis, + function::{ + PyBoundMethod, PyCell, PyCellRef, PyFunction, datastack_frame_size_bytes_for_code, + vectorcall_function, + }, + list::PyListIterator, + range::PyRangeIterator, + tuple::{PyTuple, PyTupleIterator, PyTupleRef}, + }, + bytecode::{ + self, ADAPTIVE_COOLDOWN_VALUE, Arg, Instruction, LoadAttr, LoadSuperAttr, SpecialMethod, + }, + convert::{ToPyObject, ToPyResult}, + coroutine::Coro, + exceptions::ExceptionCtor, + function::{ArgMapping, Either, FuncArgs, PyMethodFlags}, + object::PyAtomicBorrow, + object::{Traverse, TraverseFn}, + protocol::{PyIter, PyIterReturn}, + scope::Scope, + sliceable::SliceableSequenceOp, + stdlib::{_typing, builtins, sys::monitoring}, + types::{PyComparisonOp, PyTypeFlags}, + vm::{Context, PyMethod}, +}; +use alloc::fmt; +use bstr::ByteSlice; +use core::cell::UnsafeCell; +use core::iter::zip; +use core::sync::atomic; +use core::sync::atomic::AtomicPtr; +use core::sync::atomic::Ordering::{Acquire, Relaxed}; +use indexmap::IndexMap; +use itertools::Itertools; +use malachite_bigint::BigInt; +use num_traits::Zero; +use rustpython_common::atomic::{PyAtomic, Radium}; +use rustpython_common::{ + lock::{OnceCell, PyMutex}, + wtf8::{Wtf8, Wtf8Buf, wtf8_concat}, +}; +use rustpython_compiler_core::SourceLocation; + +pub type FrameRef = PyRef<Frame>; + +/// The reason why we might be unwinding a block. +/// This could be return of function, exception being +/// raised, a break or continue being hit, etc.. +#[derive(Clone, Debug)] +enum UnwindReason { + /// We are returning a value from a return statement. + Returning { value: PyObjectRef }, + + /// We hit an exception, so unwind any try-except and finally blocks. The exception should be + /// on top of the vm exception stack. + Raising { exception: PyBaseExceptionRef }, +} + +/// Tracks who owns a frame. +// = `_PyFrameOwner` +#[repr(i8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum FrameOwner { + /// Being executed by a thread (FRAME_OWNED_BY_THREAD). + Thread = 0, + /// Owned by a generator/coroutine (FRAME_OWNED_BY_GENERATOR). + Generator = 1, + /// Not executing; held only by a frame object or traceback + /// (FRAME_OWNED_BY_FRAME_OBJECT). + FrameObject = 2, +} + +impl FrameOwner { + pub(crate) fn from_i8(v: i8) -> Self { + match v { + 0 => Self::Thread, + 1 => Self::Generator, + _ => Self::FrameObject, + } + } +} + +/// Lock-free mutable storage for frame-internal data. +/// +/// # Safety +/// Frame execution is single-threaded: only one thread at a time executes +/// a given frame (enforced by the owner field and generator running flag). +/// External readers (e.g. `f_locals`) are on the same thread as execution +/// (trace callback) or the frame is not executing. +pub(crate) struct FrameUnsafeCell<T>(UnsafeCell<T>); + +impl<T> FrameUnsafeCell<T> { + fn new(value: T) -> Self { + Self(UnsafeCell::new(value)) + } + + /// # Safety + /// Caller must ensure no concurrent mutable access. + #[inline(always)] + unsafe fn get(&self) -> *mut T { + self.0.get() + } +} + +// SAFETY: Frame execution is single-threaded. See FrameUnsafeCell doc. +#[cfg(feature = "threading")] +unsafe impl<T: Send> Send for FrameUnsafeCell<T> {} +#[cfg(feature = "threading")] +unsafe impl<T: Send> Sync for FrameUnsafeCell<T> {} + +/// Unified storage for local variables and evaluation stack. +/// +/// Memory layout (each slot is `usize`-sized): +/// `[0..nlocalsplus)` — fastlocals (`Option<PyObjectRef>`) +/// `[nlocalsplus..nlocalsplus+stack_top)` — active evaluation stack (`Option<PyStackRef>`) +/// `[nlocalsplus+stack_top..capacity)` — unused stack capacity +/// +/// Both `Option<PyObjectRef>` and `Option<PyStackRef>` are `usize`-sized +/// (niche optimization on NonNull / NonZeroUsize). The raw storage is +/// `usize` to unify them; typed access is provided through methods. +pub struct LocalsPlus { + /// Backing storage. + data: LocalsPlusData, + /// Number of fastlocals slots (nlocals + ncells + nfrees). + nlocalsplus: u32, + /// Current evaluation stack depth. + stack_top: u32, +} + +enum LocalsPlusData { + /// Heap-allocated storage (generators, coroutines, exec/eval frames). + Heap(Box<[usize]>), + /// Data stack allocated storage (normal function calls). + /// The pointer is valid while the enclosing data stack frame is alive. + DataStack { ptr: *mut usize, capacity: usize }, +} + +// SAFETY: DataStack variant points to thread-local DataStack memory. +// Frame execution is single-threaded (enforced by owner field). +#[cfg(feature = "threading")] +unsafe impl Send for LocalsPlusData {} +#[cfg(feature = "threading")] +unsafe impl Sync for LocalsPlusData {} + +const _: () = { + assert!(core::mem::size_of::<Option<PyObjectRef>>() == core::mem::size_of::<usize>()); + // PyStackRef size is checked in object/core.rs +}; + +impl LocalsPlus { + /// Create a new heap-backed LocalsPlus. All slots start as None (0). + fn new(nlocalsplus: usize, stacksize: usize) -> Self { + let capacity = nlocalsplus + .checked_add(stacksize) + .expect("LocalsPlus capacity overflow"); + let nlocalsplus_u32 = u32::try_from(nlocalsplus).expect("nlocalsplus exceeds u32"); + Self { + data: LocalsPlusData::Heap(vec![0usize; capacity].into_boxed_slice()), + nlocalsplus: nlocalsplus_u32, + stack_top: 0, + } + } + + /// Create a new LocalsPlus backed by the thread data stack. + /// All slots are zero-initialized. + /// + /// The caller must call `materialize_localsplus()` when the frame finishes + /// to migrate data to the heap, then `datastack_pop()` to free the memory. + fn new_on_datastack(nlocalsplus: usize, stacksize: usize, vm: &VirtualMachine) -> Self { + let capacity = nlocalsplus + .checked_add(stacksize) + .expect("LocalsPlus capacity overflow"); + let byte_size = capacity + .checked_mul(core::mem::size_of::<usize>()) + .expect("LocalsPlus byte size overflow"); + let nlocalsplus_u32 = u32::try_from(nlocalsplus).expect("nlocalsplus exceeds u32"); + let ptr = vm.datastack_push(byte_size) as *mut usize; + // Zero-initialize all slots (0 = None for both PyObjectRef and PyStackRef). + unsafe { core::ptr::write_bytes(ptr, 0, capacity) }; + Self { + data: LocalsPlusData::DataStack { ptr, capacity }, + nlocalsplus: nlocalsplus_u32, + stack_top: 0, + } + } + + /// Migrate data-stack-backed storage to the heap, preserving all values. + /// Returns the data stack base pointer for `DataStack::pop()`. + /// Returns `None` if already heap-backed. + fn materialize_to_heap(&mut self) -> Option<*mut u8> { + if let LocalsPlusData::DataStack { ptr, capacity } = &self.data { + let base = *ptr as *mut u8; + let heap_data = unsafe { core::slice::from_raw_parts(*ptr, *capacity) } + .to_vec() + .into_boxed_slice(); + self.data = LocalsPlusData::Heap(heap_data); + Some(base) + } else { + None + } + } + + /// Drop all contained values without freeing the backing storage. + fn drop_values(&mut self) { + self.stack_clear(); + let fastlocals = self.fastlocals_mut(); + for slot in fastlocals.iter_mut() { + let _ = slot.take(); + } + } + + // -- Data access helpers -- + + #[inline(always)] + fn data_as_slice(&self) -> &[usize] { + match &self.data { + LocalsPlusData::Heap(b) => b, + LocalsPlusData::DataStack { ptr, capacity } => unsafe { + core::slice::from_raw_parts(*ptr, *capacity) + }, + } + } + + #[inline(always)] + fn data_as_mut_slice(&mut self) -> &mut [usize] { + match &mut self.data { + LocalsPlusData::Heap(b) => b, + LocalsPlusData::DataStack { ptr, capacity } => unsafe { + core::slice::from_raw_parts_mut(*ptr, *capacity) + }, + } + } + + /// Total capacity (fastlocals + stack). + #[inline(always)] + fn capacity(&self) -> usize { + match &self.data { + LocalsPlusData::Heap(b) => b.len(), + LocalsPlusData::DataStack { capacity, .. } => *capacity, + } + } + + /// Stack capacity (max stack depth). + #[inline(always)] + fn stack_capacity(&self) -> usize { + self.capacity() - self.nlocalsplus as usize + } + + // -- Fastlocals access -- + + /// Immutable access to fastlocals as `Option<PyObjectRef>` slice. + #[inline(always)] + fn fastlocals(&self) -> &[Option<PyObjectRef>] { + let data = self.data_as_slice(); + let ptr = data.as_ptr() as *const Option<PyObjectRef>; + unsafe { core::slice::from_raw_parts(ptr, self.nlocalsplus as usize) } + } + + /// Mutable access to fastlocals as `Option<PyObjectRef>` slice. + #[inline(always)] + fn fastlocals_mut(&mut self) -> &mut [Option<PyObjectRef>] { + let nlocalsplus = self.nlocalsplus as usize; + let data = self.data_as_mut_slice(); + let ptr = data.as_mut_ptr() as *mut Option<PyObjectRef>; + unsafe { core::slice::from_raw_parts_mut(ptr, nlocalsplus) } + } + + // -- Stack access -- + + /// Current stack depth. + #[inline(always)] + fn stack_len(&self) -> usize { + self.stack_top as usize + } + + /// Whether the stack is empty. + #[inline(always)] + fn stack_is_empty(&self) -> bool { + self.stack_top == 0 + } + + /// Push a value onto the evaluation stack. + #[inline(always)] + fn stack_push(&mut self, val: Option<PyStackRef>) { + let idx = self.nlocalsplus as usize + self.stack_top as usize; + debug_assert!( + idx < self.capacity(), + "stack overflow: stack_top={}, capacity={}", + self.stack_top, + self.stack_capacity() + ); + let data = self.data_as_mut_slice(); + data[idx] = unsafe { core::mem::transmute::<Option<PyStackRef>, usize>(val) }; + self.stack_top += 1; + } + + /// Try to push; returns Err if stack is full. + #[inline(always)] + fn stack_try_push(&mut self, val: Option<PyStackRef>) -> Result<(), Option<PyStackRef>> { + let idx = self.nlocalsplus as usize + self.stack_top as usize; + if idx >= self.capacity() { + return Err(val); + } + let data = self.data_as_mut_slice(); + data[idx] = unsafe { core::mem::transmute::<Option<PyStackRef>, usize>(val) }; + self.stack_top += 1; + Ok(()) + } + + /// Pop a value from the evaluation stack. + #[inline(always)] + fn stack_pop(&mut self) -> Option<PyStackRef> { + debug_assert!(self.stack_top > 0, "stack underflow"); + self.stack_top -= 1; + let idx = self.nlocalsplus as usize + self.stack_top as usize; + let data = self.data_as_mut_slice(); + let raw = core::mem::replace(&mut data[idx], 0); + unsafe { core::mem::transmute::<usize, Option<PyStackRef>>(raw) } + } + + /// Immutable view of the active stack as `Option<PyStackRef>` slice. + #[inline(always)] + fn stack_as_slice(&self) -> &[Option<PyStackRef>] { + let data = self.data_as_slice(); + let base = self.nlocalsplus as usize; + let ptr = unsafe { (data.as_ptr().add(base)) as *const Option<PyStackRef> }; + unsafe { core::slice::from_raw_parts(ptr, self.stack_top as usize) } + } + + /// Get a reference to a stack slot by index from the bottom. + #[inline(always)] + fn stack_index(&self, idx: usize) -> &Option<PyStackRef> { + debug_assert!(idx < self.stack_top as usize); + let data = self.data_as_slice(); + let raw_idx = self.nlocalsplus as usize + idx; + unsafe { &*(data.as_ptr().add(raw_idx) as *const Option<PyStackRef>) } + } + + /// Get a mutable reference to a stack slot by index from the bottom. + #[inline(always)] + fn stack_index_mut(&mut self, idx: usize) -> &mut Option<PyStackRef> { + debug_assert!(idx < self.stack_top as usize); + let raw_idx = self.nlocalsplus as usize + idx; + let data = self.data_as_mut_slice(); + unsafe { &mut *(data.as_mut_ptr().add(raw_idx) as *mut Option<PyStackRef>) } + } + + /// Get the last stack element (top of stack). + #[inline(always)] + fn stack_last(&self) -> Option<&Option<PyStackRef>> { + if self.stack_top == 0 { + None + } else { + Some(self.stack_index(self.stack_top as usize - 1)) + } + } + + /// Get mutable reference to the last stack element. + #[inline(always)] + fn stack_last_mut(&mut self) -> Option<&mut Option<PyStackRef>> { + if self.stack_top == 0 { + None + } else { + let idx = self.stack_top as usize - 1; + Some(self.stack_index_mut(idx)) + } + } + + /// Swap two stack elements. + #[inline(always)] + fn stack_swap(&mut self, a: usize, b: usize) { + let base = self.nlocalsplus as usize; + let data = self.data_as_mut_slice(); + data.swap(base + a, base + b); + } + + /// Truncate the stack to `new_len` elements, dropping excess values. + fn stack_truncate(&mut self, new_len: usize) { + debug_assert!(new_len <= self.stack_top as usize); + while self.stack_top as usize > new_len { + let _ = self.stack_pop(); + } + } + + /// Clear the stack, dropping all values. + fn stack_clear(&mut self) { + while self.stack_top > 0 { + let _ = self.stack_pop(); + } + } + + /// Drain stack elements from `from` to the end, returning an iterator + /// that yields `Option<PyStackRef>` in forward order and shrinks the stack. + fn stack_drain( + &mut self, + from: usize, + ) -> impl ExactSizeIterator<Item = Option<PyStackRef>> + '_ { + let end = self.stack_top as usize; + debug_assert!(from <= end); + // Reduce stack_top now; the drain iterator owns the elements. + self.stack_top = from as u32; + LocalsPlusStackDrain { + localsplus: self, + current: from, + end, + } + } + + /// Extend the stack with values from an iterator. + fn stack_extend(&mut self, iter: impl Iterator<Item = Option<PyStackRef>>) { + for val in iter { + self.stack_push(val); + } + } +} + +/// Iterator for draining stack elements in forward order. +struct LocalsPlusStackDrain<'a> { + localsplus: &'a mut LocalsPlus, + /// Current read position (stack-relative index). + current: usize, + /// End position (exclusive, stack-relative index). + end: usize, +} + +impl Iterator for LocalsPlusStackDrain<'_> { + type Item = Option<PyStackRef>; + + fn next(&mut self) -> Option<Self::Item> { + if self.current >= self.end { + return None; + } + let idx = self.localsplus.nlocalsplus as usize + self.current; + let data = self.localsplus.data_as_mut_slice(); + let raw = core::mem::replace(&mut data[idx], 0); + self.current += 1; + Some(unsafe { core::mem::transmute::<usize, Option<PyStackRef>>(raw) }) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + let remaining = self.end - self.current; + (remaining, Some(remaining)) + } +} + +impl ExactSizeIterator for LocalsPlusStackDrain<'_> {} + +impl Drop for LocalsPlusStackDrain<'_> { + fn drop(&mut self) { + while self.current < self.end { + let idx = self.localsplus.nlocalsplus as usize + self.current; + let data = self.localsplus.data_as_mut_slice(); + let raw = core::mem::replace(&mut data[idx], 0); + let _ = unsafe { core::mem::transmute::<usize, Option<PyStackRef>>(raw) }; + self.current += 1; + } + } +} + +impl Drop for LocalsPlus { + fn drop(&mut self) { + // drop_values handles both stack and fastlocals. + // For DataStack-backed storage, the caller should have called + // materialize_localsplus() + datastack_pop() before drop. + // If not (e.g. panic), the DataStack memory is leaked but + // values are still dropped safely. + self.drop_values(); + } +} + +unsafe impl Traverse for LocalsPlus { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.fastlocals().traverse(tracer_fn); + self.stack_as_slice().traverse(tracer_fn); + } +} + +/// Lazy locals dict for frames. For NEWLOCALS frames, the dict is +/// only allocated on first access (most function frames never need it). +pub struct FrameLocals { + inner: OnceCell<ArgMapping>, +} + +impl FrameLocals { + /// Create with an already-initialized locals mapping (non-NEWLOCALS frames). + fn with_locals(locals: ArgMapping) -> Self { + let cell = OnceCell::new(); + let _ = cell.set(locals); + Self { inner: cell } + } + + /// Create an empty lazy locals (for NEWLOCALS frames). + /// The dict will be created on first access. + fn lazy() -> Self { + Self { + inner: OnceCell::new(), + } + } + + /// Get the locals mapping, creating it lazily if needed. + #[inline] + pub fn get_or_create(&self, vm: &VirtualMachine) -> &ArgMapping { + self.inner + .get_or_init(|| ArgMapping::from_dict_exact(vm.ctx.new_dict())) + } + + /// Get the locals mapping if already created. + #[inline] + pub fn get(&self) -> Option<&ArgMapping> { + self.inner.get() + } + + #[inline] + pub fn mapping(&self, vm: &VirtualMachine) -> crate::protocol::PyMapping<'_> { + self.get_or_create(vm).mapping() + } + + #[inline] + pub fn clone_mapping(&self, vm: &VirtualMachine) -> ArgMapping { + self.get_or_create(vm).clone() + } + + pub fn into_object(&self, vm: &VirtualMachine) -> PyObjectRef { + self.clone_mapping(vm).into() + } + + pub fn as_object(&self, vm: &VirtualMachine) -> &PyObject { + self.get_or_create(vm).obj() + } +} + +impl fmt::Debug for FrameLocals { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FrameLocals") + .field("initialized", &self.inner.get().is_some()) + .finish() + } +} + +impl Clone for FrameLocals { + fn clone(&self) -> Self { + let cell = OnceCell::new(); + if let Some(locals) = self.inner.get() { + let _ = cell.set(locals.clone()); + } + Self { inner: cell } + } +} + +unsafe impl Traverse for FrameLocals { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + if let Some(locals) = self.inner.get() { + locals.traverse(tracer_fn); + } + } +} + +/// Lightweight execution frame. Not a PyObject. +/// Analogous to CPython's `_PyInterpreterFrame`. +/// +/// Currently always embedded inside a `Frame` PyObject via `FrameUnsafeCell`. +/// In future PRs this will be usable independently for normal function calls +/// (allocated on the Rust stack + DataStack), eliminating PyObject overhead. +pub struct InterpreterFrame { + pub code: PyRef<PyCode>, + pub func_obj: Option<PyObjectRef>, + + /// Unified storage for local variables and evaluation stack. + pub(crate) localsplus: LocalsPlus, + pub locals: FrameLocals, + pub globals: PyDictRef, + pub builtins: PyObjectRef, + + /// index of last instruction ran + pub lasti: PyAtomic<u32>, + /// tracer function for this frame (usually is None) + pub trace: PyMutex<PyObjectRef>, + + /// Previous line number for LINE event suppression. + pub(crate) prev_line: u32, + + // member + pub trace_lines: PyMutex<bool>, + pub trace_opcodes: PyMutex<bool>, + pub temporary_refs: PyMutex<Vec<PyObjectRef>>, + /// Back-reference to owning generator/coroutine/async generator. + /// Borrowed reference (not ref-counted) to avoid Generator↔Frame cycle. + /// Cleared by the generator's Drop impl. + pub generator: PyAtomicBorrow, + /// Previous frame in the call chain for signal-safe traceback walking. + /// Mirrors `_PyInterpreterFrame.previous`. + pub(crate) previous: AtomicPtr<Frame>, + /// Who owns this frame. Mirrors `_PyInterpreterFrame.owner`. + /// Used by `frame.clear()` to reject clearing an executing frame, + /// even when called from a different thread. + pub(crate) owner: atomic::AtomicI8, + /// Set when f_locals is accessed. Cleared after locals_to_fast() sync. + pub(crate) locals_dirty: atomic::AtomicBool, + /// Number of stack entries to pop after set_f_lineno returns to the + /// execution loop. set_f_lineno cannot pop directly because the + /// execution loop holds the state mutex. + pub(crate) pending_stack_pops: PyAtomic<u32>, + /// The encoded stack state that set_f_lineno wants to unwind *from*. + /// Used together with `pending_stack_pops` to identify Except entries + /// that need special exception-state handling. + pub(crate) pending_unwind_from_stack: PyAtomic<i64>, +} + +/// Python-visible frame object. Currently always wraps an `InterpreterFrame`. +/// Analogous to CPython's `PyFrameObject`. +#[pyclass(module = false, name = "frame", traverse = "manual")] +pub struct Frame { + pub(crate) iframe: FrameUnsafeCell<InterpreterFrame>, +} + +impl core::ops::Deref for Frame { + type Target = InterpreterFrame; + /// Transparent access to InterpreterFrame fields. + /// + /// # Safety argument + /// Immutable fields (code, globals, builtins, func_obj, locals) are safe + /// to access at any time. Atomic/mutex fields (lasti, trace, owner, etc.) + /// provide their own synchronization. Mutable fields (localsplus, prev_line) + /// are only mutated during single-threaded execution via `with_exec`. + #[inline(always)] + fn deref(&self) -> &InterpreterFrame { + unsafe { &*self.iframe.get() } + } +} + +impl PyPayload for Frame { + #[inline] + fn class(ctx: &Context) -> &'static Py<PyType> { + ctx.types.frame_type + } +} + +unsafe impl Traverse for Frame { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + // SAFETY: GC traversal does not run concurrently with frame execution. + let iframe = unsafe { &*self.iframe.get() }; + iframe.code.traverse(tracer_fn); + iframe.func_obj.traverse(tracer_fn); + iframe.localsplus.traverse(tracer_fn); + iframe.locals.traverse(tracer_fn); + iframe.globals.traverse(tracer_fn); + iframe.builtins.traverse(tracer_fn); + iframe.trace.traverse(tracer_fn); + iframe.temporary_refs.traverse(tracer_fn); + } +} + +// Running a frame can result in one of the below: +pub enum ExecutionResult { + Return(PyObjectRef), + Yield(PyObjectRef), +} + +/// A valid execution result, or an exception +type FrameResult = PyResult<Option<ExecutionResult>>; + +impl Frame { + pub(crate) fn new( + code: PyRef<PyCode>, + scope: Scope, + builtins: PyObjectRef, + closure: &[PyCellRef], + func_obj: Option<PyObjectRef>, + use_datastack: bool, + vm: &VirtualMachine, + ) -> Self { + let nlocals = code.varnames.len(); + let num_cells = code.cellvars.len(); + let nfrees = closure.len(); + + let nlocalsplus = nlocals + .checked_add(num_cells) + .and_then(|v| v.checked_add(nfrees)) + .expect("Frame::new: nlocalsplus overflow"); + let max_stackdepth = code.max_stackdepth as usize; + let mut localsplus = if use_datastack { + LocalsPlus::new_on_datastack(nlocalsplus, max_stackdepth, vm) + } else { + LocalsPlus::new(nlocalsplus, max_stackdepth) + }; + + // Store cell/free variable objects directly in localsplus + let fastlocals = localsplus.fastlocals_mut(); + for i in 0..num_cells { + fastlocals[nlocals + i] = Some(PyCell::default().into_ref(&vm.ctx).into()); + } + for (i, cell) in closure.iter().enumerate() { + fastlocals[nlocals + num_cells + i] = Some(cell.clone().into()); + } + + let iframe = InterpreterFrame { + localsplus, + locals: match scope.locals { + Some(locals) => FrameLocals::with_locals(locals), + None if code.flags.contains(bytecode::CodeFlags::NEWLOCALS) => FrameLocals::lazy(), + None => { + FrameLocals::with_locals(ArgMapping::from_dict_exact(scope.globals.clone())) + } + }, + globals: scope.globals, + builtins, + code, + func_obj, + lasti: Radium::new(0), + prev_line: 0, + trace: PyMutex::new(vm.ctx.none()), + trace_lines: PyMutex::new(true), + trace_opcodes: PyMutex::new(false), + temporary_refs: PyMutex::new(vec![]), + generator: PyAtomicBorrow::new(), + previous: AtomicPtr::new(core::ptr::null_mut()), + owner: atomic::AtomicI8::new(FrameOwner::FrameObject as i8), + locals_dirty: atomic::AtomicBool::new(false), + pending_stack_pops: Default::default(), + pending_unwind_from_stack: Default::default(), + }; + Self { + iframe: FrameUnsafeCell::new(iframe), + } + } + + /// Access fastlocals immutably. + /// + /// # Safety + /// Caller must ensure no concurrent mutable access (frame not executing, + /// or called from the same thread during trace callback). + #[inline(always)] + pub unsafe fn fastlocals(&self) -> &[Option<PyObjectRef>] { + unsafe { (*self.iframe.get()).localsplus.fastlocals() } + } + + /// Access fastlocals mutably. + /// + /// # Safety + /// Caller must ensure exclusive access (frame not executing). + #[inline(always)] + #[allow(clippy::mut_from_ref)] + pub unsafe fn fastlocals_mut(&self) -> &mut [Option<PyObjectRef>] { + unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() } + } + + /// Migrate data-stack-backed storage to the heap, preserving all values, + /// and return the data stack base pointer for `DataStack::pop()`. + /// Returns `None` if already heap-backed. + /// + /// # Safety + /// Caller must ensure the frame is not executing and the returned + /// pointer is passed to `VirtualMachine::datastack_pop()`. + pub(crate) unsafe fn materialize_localsplus(&self) -> Option<*mut u8> { + unsafe { (*self.iframe.get()).localsplus.materialize_to_heap() } + } + + /// Clear evaluation stack and state-owned cell/free references. + /// For full local/cell cleanup, call `clear_locals_and_stack()`. + pub(crate) fn clear_stack_and_cells(&self) { + // SAFETY: Called when frame is not executing (generator closed). + // Cell refs in fastlocals[nlocals..] are cleared by clear_locals_and_stack(). + unsafe { + (*self.iframe.get()).localsplus.stack_clear(); + } + } + + /// Clear locals and stack after generator/coroutine close. + /// Releases references held by the frame, matching _PyFrame_ClearLocals. + pub(crate) fn clear_locals_and_stack(&self) { + self.clear_stack_and_cells(); + // SAFETY: Frame is not executing (generator closed). + let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() }; + for slot in fastlocals.iter_mut() { + *slot = None; + } + } + + /// Get cell contents by cell index. Reads through fastlocals (no state lock needed). + pub(crate) fn get_cell_contents(&self, cell_idx: usize) -> Option<PyObjectRef> { + let nlocals = self.code.varnames.len(); + // SAFETY: Frame not executing; no concurrent mutation. + let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + fastlocals + .get(nlocals + cell_idx) + .and_then(|slot| slot.as_ref()) + .and_then(|obj| obj.downcast_ref::<PyCell>()) + .and_then(|cell| cell.get()) + } + + /// Set cell contents by cell index. Only safe to call before frame execution starts. + pub(crate) fn set_cell_contents(&self, cell_idx: usize, value: Option<PyObjectRef>) { + let nlocals = self.code.varnames.len(); + // SAFETY: Called before frame execution starts. + let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + fastlocals[nlocals + cell_idx] + .as_ref() + .and_then(|obj| obj.downcast_ref::<PyCell>()) + .expect("cell slot empty or not a PyCell") + .set(value); + } + + /// Store a borrowed back-reference to the owning generator/coroutine. + /// The caller must ensure the generator outlives the frame. + pub fn set_generator(&self, generator: &PyObject) { + self.generator.store(generator); + self.owner + .store(FrameOwner::Generator as i8, atomic::Ordering::Release); + } + + /// Clear the generator back-reference. Called when the generator is finalized. + pub fn clear_generator(&self) { + self.generator.clear(); + self.owner + .store(FrameOwner::FrameObject as i8, atomic::Ordering::Release); + } + + pub fn current_location(&self) -> SourceLocation { + self.code.locations[self.lasti() as usize - 1].0 + } + + /// Get the previous frame pointer for signal-safe traceback walking. + pub fn previous_frame(&self) -> *const Frame { + self.previous.load(atomic::Ordering::Relaxed) + } + + pub fn lasti(&self) -> u32 { + self.lasti.load(Relaxed) + } + + pub fn set_lasti(&self, val: u32) { + self.lasti.store(val, Relaxed); + } + + pub(crate) fn pending_stack_pops(&self) -> u32 { + self.pending_stack_pops.load(Relaxed) + } + + pub(crate) fn set_pending_stack_pops(&self, val: u32) { + self.pending_stack_pops.store(val, Relaxed); + } + + pub(crate) fn pending_unwind_from_stack(&self) -> i64 { + self.pending_unwind_from_stack.load(Relaxed) + } + + pub(crate) fn set_pending_unwind_from_stack(&self, val: i64) { + self.pending_unwind_from_stack.store(val, Relaxed); + } + + /// Sync locals dict back to fastlocals. Called before generator/coroutine resume + /// to apply any modifications made via f_locals. + pub fn locals_to_fast(&self, vm: &VirtualMachine) -> PyResult<()> { + if !self.locals_dirty.load(atomic::Ordering::Acquire) { + return Ok(()); + } + let code = &**self.code; + // SAFETY: Called before generator resume; no concurrent access. + let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals_mut() }; + let locals_map = self.locals.mapping(vm); + for (i, &varname) in code.varnames.iter().enumerate() { + if i >= fastlocals.len() { + break; + } + match locals_map.subscript(varname, vm) { + Ok(value) => fastlocals[i] = Some(value), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} + Err(e) => return Err(e), + } + } + self.locals_dirty.store(false, atomic::Ordering::Release); + Ok(()) + } + + pub fn locals(&self, vm: &VirtualMachine) -> PyResult<ArgMapping> { + // SAFETY: Either the frame is not executing (caller checked owner), + // or we're in a trace callback on the same thread that's executing. + let locals = &self.locals; + let code = &**self.code; + let map = &code.varnames; + let j = core::cmp::min(map.len(), code.varnames.len()); + let locals_map = locals.mapping(vm); + if !code.varnames.is_empty() { + let fastlocals = unsafe { (*self.iframe.get()).localsplus.fastlocals() }; + for (&k, v) in zip(&map[..j], fastlocals) { + match locals_map.ass_subscript(k, v.clone(), vm) { + Ok(()) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} + Err(e) => return Err(e), + } + } + } + if !code.cellvars.is_empty() || !code.freevars.is_empty() { + for (i, &k) in code.cellvars.iter().enumerate() { + let cell_value = self.get_cell_contents(i); + match locals_map.ass_subscript(k, cell_value, vm) { + Ok(()) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} + Err(e) => return Err(e), + } + } + if code.flags.contains(bytecode::CodeFlags::OPTIMIZED) { + for (i, &k) in code.freevars.iter().enumerate() { + let cell_value = self.get_cell_contents(code.cellvars.len() + i); + match locals_map.ass_subscript(k, cell_value, vm) { + Ok(()) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} + Err(e) => return Err(e), + } + } + } + } + Ok(locals.clone_mapping(vm)) + } +} + +impl Py<Frame> { + #[inline(always)] + fn with_exec<R>(&self, vm: &VirtualMachine, f: impl FnOnce(ExecutingFrame<'_>) -> R) -> R { + // SAFETY: Frame execution is single-threaded. Only one thread at a time + // executes a given frame (enforced by the owner field and generator + // running flag). Same safety argument as FastLocals (UnsafeCell). + let iframe = unsafe { &mut *self.iframe.get() }; + let exec = ExecutingFrame { + code: &iframe.code, + localsplus: &mut iframe.localsplus, + locals: &iframe.locals, + globals: &iframe.globals, + builtins: &iframe.builtins, + builtins_dict: if iframe.globals.class().is(vm.ctx.types.dict_type) { + iframe + .builtins + .downcast_ref_if_exact::<PyDict>(vm) + // SAFETY: downcast_ref_if_exact already verified exact type + .map(|d| unsafe { PyExact::ref_unchecked(d) }) + } else { + None + }, + lasti: &iframe.lasti, + object: self, + prev_line: &mut iframe.prev_line, + monitoring_mask: 0, + }; + f(exec) + } + + // #[cfg_attr(feature = "flame-it", flame("Frame"))] + pub fn run(&self, vm: &VirtualMachine) -> PyResult<ExecutionResult> { + self.with_exec(vm, |mut exec| exec.run(vm)) + } + + pub(crate) fn resume( + &self, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<ExecutionResult> { + self.with_exec(vm, |mut exec| { + if let Some(value) = value { + exec.push_value(value) + } + exec.run(vm) + }) + } + + pub(crate) fn gen_throw( + &self, + vm: &VirtualMachine, + exc_type: PyObjectRef, + exc_val: PyObjectRef, + exc_tb: PyObjectRef, + ) -> PyResult<ExecutionResult> { + self.with_exec(vm, |mut exec| exec.gen_throw(vm, exc_type, exc_val, exc_tb)) + } + + pub fn yield_from_target(&self) -> Option<PyObjectRef> { + // If the frame is currently executing (owned by thread), it has no + // yield-from target to report. + let owner = FrameOwner::from_i8(self.owner.load(atomic::Ordering::Acquire)); + if owner == FrameOwner::Thread { + return None; + } + // SAFETY: Frame is not executing, so UnsafeCell access is safe. + let iframe = unsafe { &mut *self.iframe.get() }; + let exec = ExecutingFrame { + code: &iframe.code, + localsplus: &mut iframe.localsplus, + locals: &iframe.locals, + globals: &iframe.globals, + builtins: &iframe.builtins, + builtins_dict: None, + lasti: &iframe.lasti, + object: self, + prev_line: &mut iframe.prev_line, + monitoring_mask: 0, + }; + exec.yield_from_target().map(PyObject::to_owned) + } + + pub fn is_internal_frame(&self) -> bool { + let code = self.f_code(); + let filename = code.co_filename(); + let filename = filename.as_bytes(); + filename.find(b"importlib").is_some() && filename.find(b"_bootstrap").is_some() + } + + pub fn next_external_frame(&self, vm: &VirtualMachine) -> Option<FrameRef> { + let mut frame = self.f_back(vm); + while let Some(ref f) = frame { + if !f.is_internal_frame() { + break; + } + frame = f.f_back(vm); + } + frame + } +} + +/// An executing frame; borrows mutable frame-internal data for the duration +/// of bytecode execution. +struct ExecutingFrame<'a> { + code: &'a PyRef<PyCode>, + localsplus: &'a mut LocalsPlus, + locals: &'a FrameLocals, + globals: &'a PyDictRef, + builtins: &'a PyObjectRef, + /// Cached downcast of builtins to PyDict for fast LOAD_GLOBAL. + /// Only set when both globals and builtins are exact dict types (not + /// subclasses), so that `__missing__` / `__getitem__` overrides are + /// not bypassed. + builtins_dict: Option<&'a PyExact<PyDict>>, + object: &'a Py<Frame>, + lasti: &'a PyAtomic<u32>, + prev_line: &'a mut u32, + /// Cached monitoring events mask. Reloaded at Resume instruction only, + monitoring_mask: u32, +} + +#[inline] +fn specialization_compact_int_value(i: &PyInt, vm: &VirtualMachine) -> Option<isize> { + // _PyLong_IsCompact(): a one-digit PyLong (base 2^30), + // i.e. abs(value) <= 2^30 - 1. + const CPYTHON_COMPACT_LONG_ABS_MAX: i64 = (1i64 << 30) - 1; + let v = i.try_to_primitive::<i64>(vm).ok()?; + if (-CPYTHON_COMPACT_LONG_ABS_MAX..=CPYTHON_COMPACT_LONG_ABS_MAX).contains(&v) { + Some(v as isize) + } else { + None + } +} + +#[inline] +fn compact_int_from_obj(obj: &PyObject, vm: &VirtualMachine) -> Option<isize> { + obj.downcast_ref_if_exact::<PyInt>(vm) + .and_then(|i| specialization_compact_int_value(i, vm)) +} + +#[inline] +fn exact_float_from_obj(obj: &PyObject, vm: &VirtualMachine) -> Option<f64> { + obj.downcast_ref_if_exact::<PyFloat>(vm).map(|f| f.to_f64()) +} + +#[inline] +fn specialization_nonnegative_compact_index(i: &PyInt, vm: &VirtualMachine) -> Option<usize> { + // _PyLong_IsNonNegativeCompact(): a single base-2^30 digit. + const CPYTHON_COMPACT_LONG_MAX: u64 = (1u64 << 30) - 1; + let v = i.try_to_primitive::<u64>(vm).ok()?; + if v <= CPYTHON_COMPACT_LONG_MAX { + Some(v as usize) + } else { + None + } +} + +fn release_datastack_frame(frame: &Py<Frame>, vm: &VirtualMachine) { + unsafe { + if let Some(base) = frame.materialize_localsplus() { + vm.datastack_pop(base); + } + } +} + +type BinaryOpExtendGuard = fn(&PyObject, &PyObject, &VirtualMachine) -> bool; +type BinaryOpExtendAction = fn(&PyObject, &PyObject, &VirtualMachine) -> Option<PyObjectRef>; + +struct BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator, + guard: BinaryOpExtendGuard, + action: BinaryOpExtendAction, +} + +const BINARY_OP_EXTEND_EXTERNAL_CACHE_OFFSET: usize = 1; + +#[inline] +fn compactlongs_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { + compact_int_from_obj(lhs, vm).is_some() && compact_int_from_obj(rhs, vm).is_some() +} + +macro_rules! bitwise_longs_action { + ($name:ident, $op:tt) => { + #[inline] + fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option<PyObjectRef> { + let lhs_val = compact_int_from_obj(lhs, vm)?; + let rhs_val = compact_int_from_obj(rhs, vm)?; + Some(vm.ctx.new_int(lhs_val $op rhs_val).into()) + } + }; +} +bitwise_longs_action!(compactlongs_or, |); +bitwise_longs_action!(compactlongs_and, &); +bitwise_longs_action!(compactlongs_xor, ^); + +#[inline] +fn float_compactlong_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { + exact_float_from_obj(lhs, vm).is_some_and(|f| !f.is_nan()) + && compact_int_from_obj(rhs, vm).is_some() +} + +#[inline] +fn nonzero_float_compactlong_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { + float_compactlong_guard(lhs, rhs, vm) && compact_int_from_obj(rhs, vm).is_some_and(|v| v != 0) +} + +macro_rules! float_long_action { + ($name:ident, $op:tt) => { + #[inline] + fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option<PyObjectRef> { + let lhs_val = exact_float_from_obj(lhs, vm)?; + let rhs_val = compact_int_from_obj(rhs, vm)?; + Some(vm.ctx.new_float(lhs_val $op rhs_val as f64).into()) + } + }; +} +float_long_action!(float_compactlong_add, +); +float_long_action!(float_compactlong_subtract, -); +float_long_action!(float_compactlong_multiply, *); +float_long_action!(float_compactlong_true_div, /); + +#[inline] +fn compactlong_float_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { + compact_int_from_obj(lhs, vm).is_some() + && exact_float_from_obj(rhs, vm).is_some_and(|f| !f.is_nan()) +} + +#[inline] +fn nonzero_compactlong_float_guard(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> bool { + compactlong_float_guard(lhs, rhs, vm) && exact_float_from_obj(rhs, vm).is_some_and(|f| f != 0.0) +} + +macro_rules! long_float_action { + ($name:ident, $op:tt) => { + #[inline] + fn $name(lhs: &PyObject, rhs: &PyObject, vm: &VirtualMachine) -> Option<PyObjectRef> { + let lhs_val = compact_int_from_obj(lhs, vm)?; + let rhs_val = exact_float_from_obj(rhs, vm)?; + Some(vm.ctx.new_float(lhs_val as f64 $op rhs_val).into()) + } + }; +} +long_float_action!(compactlong_float_add, +); +long_float_action!(compactlong_float_subtract, -); +long_float_action!(compactlong_float_multiply, *); +long_float_action!(compactlong_float_true_div, /); + +static BINARY_OP_EXTEND_DESCRIPTORS: &[BinaryOpExtendSpecializationDescr] = &[ + // long-long arithmetic + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Or, + guard: compactlongs_guard, + action: compactlongs_or, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::And, + guard: compactlongs_guard, + action: compactlongs_and, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Xor, + guard: compactlongs_guard, + action: compactlongs_xor, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceOr, + guard: compactlongs_guard, + action: compactlongs_or, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceAnd, + guard: compactlongs_guard, + action: compactlongs_and, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::InplaceXor, + guard: compactlongs_guard, + action: compactlongs_xor, + }, + // float-long arithmetic + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Add, + guard: float_compactlong_guard, + action: float_compactlong_add, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Subtract, + guard: float_compactlong_guard, + action: float_compactlong_subtract, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::TrueDivide, + guard: nonzero_float_compactlong_guard, + action: float_compactlong_true_div, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Multiply, + guard: float_compactlong_guard, + action: float_compactlong_multiply, + }, + // long-float arithmetic + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Add, + guard: compactlong_float_guard, + action: compactlong_float_add, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Subtract, + guard: compactlong_float_guard, + action: compactlong_float_subtract, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::TrueDivide, + guard: nonzero_compactlong_float_guard, + action: compactlong_float_true_div, + }, + BinaryOpExtendSpecializationDescr { + oparg: bytecode::BinaryOperator::Multiply, + guard: compactlong_float_guard, + action: compactlong_float_multiply, + }, +]; + +impl fmt::Debug for ExecutingFrame<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExecutingFrame") + .field("code", self.code) + .field("stack_len", &self.localsplus.stack_len()) + .finish() + } +} + +impl ExecutingFrame<'_> { + #[inline] + fn monitoring_disabled_for_code(&self, vm: &VirtualMachine) -> bool { + self.code.is(&vm.ctx.init_cleanup_code) + } + + fn specialization_new_init_cleanup_frame(&self, vm: &VirtualMachine) -> FrameRef { + Frame::new( + vm.ctx.init_cleanup_code.clone(), + Scope::new( + Some(ArgMapping::from_dict_exact(vm.ctx.new_dict())), + self.globals.clone(), + ), + self.builtins.clone(), + &[], + None, + true, + vm, + ) + .into_ref(&vm.ctx) + } + + fn specialization_run_init_cleanup_shim( + &self, + new_obj: PyObjectRef, + init_func: &Py<PyFunction>, + pos_args: Vec<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let shim = self.specialization_new_init_cleanup_frame(vm); + let shim_result = vm.with_frame_untraced(shim.clone(), |shim| { + shim.with_exec(vm, |mut exec| exec.push_value(new_obj.clone())); + + let mut all_args = Vec::with_capacity(pos_args.len() + 1); + all_args.push(new_obj.clone()); + all_args.extend(pos_args); + + let init_frame = init_func.prepare_exact_args_frame(all_args, vm); + let init_result = vm.run_frame(init_frame.clone()); + release_datastack_frame(&init_frame, vm); + let init_result = init_result?; + + shim.with_exec(vm, |mut exec| exec.push_value(init_result)); + match shim.run(vm)? { + ExecutionResult::Return(value) => Ok(value), + ExecutionResult::Yield(_) => unreachable!("_Py_InitCleanup shim cannot yield"), + } + }); + release_datastack_frame(&shim, vm); + shim_result + } + + #[inline(always)] + fn update_lasti(&mut self, f: impl FnOnce(&mut u32)) { + let mut val = self.lasti.load(Relaxed); + f(&mut val); + self.lasti.store(val, Relaxed); + } + + #[inline(always)] + fn lasti(&self) -> u32 { + self.lasti.load(Relaxed) + } + + /// Access the PyCellRef at the given cell/free variable index. + /// `cell_idx` is 0-based: 0..ncells for cellvars, ncells.. for freevars. + #[inline(always)] + fn cell_ref(&self, cell_idx: usize) -> &PyCell { + let nlocals = self.code.varnames.len(); + self.localsplus.fastlocals()[nlocals + cell_idx] + .as_ref() + .expect("cell slot empty") + .downcast_ref::<PyCell>() + .expect("cell slot is not a PyCell") + } + + /// Perform deferred stack unwinding after set_f_lineno. + /// + /// set_f_lineno cannot pop the value stack directly because the execution + /// loop holds the state mutex. Instead it records the work in + /// `pending_stack_pops` / `pending_unwind_from_stack` and we execute it + /// here, inside the execution loop where we already own the state. + fn unwind_stack_for_lineno(&mut self, pop_count: usize, from_stack: i64, vm: &VirtualMachine) { + let mut cur_stack = from_stack; + for _ in 0..pop_count { + let val = self.pop_value_opt(); + if stack_analysis::top_of_stack(cur_stack) == stack_analysis::Kind::Except as i64 + && let Some(exc_obj) = val + { + if vm.is_none(&exc_obj) { + vm.set_exception(None); + } else { + let exc = exc_obj.downcast::<PyBaseException>().ok(); + vm.set_exception(exc); + } + } + cur_stack = stack_analysis::pop_value(cur_stack); + } + } + + /// Fire 'exception' trace event (sys.settrace) with (type, value, traceback) tuple. + /// Matches `_PyEval_MonitorRaise` → `PY_MONITORING_EVENT_RAISE` → + /// `sys_trace_exception_func` in legacy_tracing.c. + fn fire_exception_trace(&self, exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<()> { + if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + let exc_type: PyObjectRef = exc.class().to_owned().into(); + let exc_value: PyObjectRef = exc.clone().into(); + let exc_tb: PyObjectRef = exc + .__traceback__() + .map(|tb| -> PyObjectRef { tb.into() }) + .unwrap_or_else(|| vm.ctx.none()); + let tuple = vm.ctx.new_tuple(vec![exc_type, exc_value, exc_tb]).into(); + vm.trace_event(crate::protocol::TraceEvent::Exception, Some(tuple))?; + } + Ok(()) + } + + fn run(&mut self, vm: &VirtualMachine) -> PyResult<ExecutionResult> { + flame_guard!(format!( + "Frame::run({obj_name})", + obj_name = self.code.obj_name + )); + // Execute until return or exception: + let mut arg_state = bytecode::OpArgState::default(); + loop { + let idx = self.lasti() as usize; + // Advance lasti past the current instruction BEFORE firing the + // line event. This ensures that f_lineno (which reads + // locations[lasti - 1]) returns the line of the instruction + // being traced, not the previous one. + self.update_lasti(|i| *i += 1); + + // Fire 'line' trace event when line number changes. + // Only fire if this frame has a per-frame trace function set + // (frames entered before sys.settrace() have trace=None). + // Skip RESUME – it should not generate user-visible line events. + if vm.use_tracing.get() + && !vm.is_none(&self.object.trace.lock()) + && !matches!( + self.code.instructions.read_op(idx), + Instruction::Resume { .. } | Instruction::InstrumentedResume + ) + && let Some((loc, _)) = self.code.locations.get(idx) + && loc.line.get() as u32 != *self.prev_line + { + *self.prev_line = loc.line.get() as u32; + vm.trace_event(crate::protocol::TraceEvent::Line, None)?; + // Trace callback may have changed lasti via set_f_lineno. + // Re-read and restart the loop from the new position. + if self.lasti() != (idx as u32 + 1) { + // set_f_lineno defers stack unwinding because we hold + // the state mutex. Perform it now. + let pops = self.object.pending_stack_pops(); + if pops > 0 { + let from_stack = self.object.pending_unwind_from_stack(); + self.unwind_stack_for_lineno(pops as usize, from_stack, vm); + self.object.set_pending_stack_pops(0); + } + arg_state.reset(); + continue; + } + } + let op = self.code.instructions.read_op(idx); + let arg = arg_state.extend(self.code.instructions.read_arg(idx)); + let mut do_extend_arg = false; + let caches = op.cache_entries(); + + // Update prev_line only when tracing or monitoring is active. + // When neither is enabled, prev_line is stale but unused. + if vm.use_tracing.get() { + if !matches!( + op, + Instruction::Resume { .. } + | Instruction::ExtendedArg + | Instruction::InstrumentedLine + ) && let Some((loc, _)) = self.code.locations.get(idx) + { + *self.prev_line = loc.line.get() as u32; + } + + // Fire 'opcode' trace event for sys.settrace when f_trace_opcodes + // is set. Skip RESUME and ExtendedArg + // (_Py_call_instrumentation_instruction). + if !vm.is_none(&self.object.trace.lock()) + && *self.object.trace_opcodes.lock() + && !matches!( + op, + Instruction::Resume { .. } + | Instruction::InstrumentedResume + | Instruction::ExtendedArg + ) + { + vm.trace_event(crate::protocol::TraceEvent::Opcode, None)?; + } + } + + if vm.eval_breaker_tripped() + && let Err(exception) = vm.check_signals() + { + #[cold] + fn handle_signal_exception( + frame: &mut ExecutingFrame<'_>, + exception: PyBaseExceptionRef, + idx: usize, + vm: &VirtualMachine, + ) -> FrameResult { + let (loc, _end_loc) = frame.code.locations[idx]; + let next = exception.__traceback__(); + let new_traceback = + PyTraceback::new(next, frame.object.to_owned(), idx as u32 * 2, loc.line); + exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + vm.contextualize_exception(&exception); + frame.unwind_blocks(vm, UnwindReason::Raising { exception }) + } + match handle_signal_exception(self, exception, idx, vm) { + Ok(None) => {} + Ok(Some(value)) => { + break Ok(value); + } + Err(exception) => { + break Err(exception); + } + } + continue; + } + let lasti_before = self.lasti(); + let result = self.execute_instruction(op, arg, &mut do_extend_arg, vm); + // Skip inline cache entries if instruction fell through (no jump). + if caches > 0 && self.lasti() == lasti_before { + self.update_lasti(|i| *i += caches as u32); + } + match result { + Ok(None) => {} + Ok(Some(value)) => { + break Ok(value); + } + // Instruction raised an exception + Err(exception) => { + #[cold] + fn handle_exception( + frame: &mut ExecutingFrame<'_>, + exception: PyBaseExceptionRef, + idx: usize, + is_reraise: bool, + is_new_raise: bool, + vm: &VirtualMachine, + ) -> FrameResult { + // 1. Extract traceback from exception's '__traceback__' attr. + // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. + // 3. First, try to find handler in exception table + + // RERAISE instructions should not add traceback entries - they're just + // re-raising an already-processed exception + if !is_reraise { + // Check if the exception already has traceback entries before + // we add ours. If it does, it was propagated from a callee + // function and we should not re-contextualize it. + let had_prior_traceback = exception.__traceback__().is_some(); + + // PyTraceBack_Here always adds a new entry without + // checking for duplicates. Each time an exception passes through + // a frame (e.g., in a loop with repeated raise statements), + // a new traceback entry is added. + let (loc, _end_loc) = frame.code.locations[idx]; + let next = exception.__traceback__(); + + let new_traceback = PyTraceback::new( + next, + frame.object.to_owned(), + idx as u32 * 2, + loc.line, + ); + vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.line); + exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + + // _PyErr_SetObject sets __context__ only when the exception + // is first raised. When an exception propagates through frames, + // __context__ must not be overwritten. We contextualize when: + // - It's an explicit raise (raise/raise from) + // - The exception had no prior traceback (originated here) + if is_new_raise || !had_prior_traceback { + vm.contextualize_exception(&exception); + } + } + + // Use exception table for zero-cost exception handling + frame.unwind_blocks(vm, UnwindReason::Raising { exception }) + } + + // Check if this is a RERAISE instruction + // Both AnyInstruction::Raise { kind: Reraise/ReraiseFromStack } and + // AnyInstruction::Reraise are reraise operations that should not add + // new traceback entries. + // EndAsyncFor and CleanupThrow also re-raise non-matching exceptions. + let is_reraise = match op { + Instruction::RaiseVarargs { argc: kind } => matches!( + kind.get(arg), + bytecode::RaiseKind::BareRaise | bytecode::RaiseKind::ReraiseFromStack + ), + Instruction::Reraise { .. } + | Instruction::EndAsyncFor + | Instruction::CleanupThrow => true, + _ => false, + }; + + // Explicit raise instructions (raise/raise from) - these always + // need contextualization even if the exception has prior traceback + let is_new_raise = matches!( + op, + Instruction::RaiseVarargs { argc: kind } + if matches!( + kind.get(arg), + bytecode::RaiseKind::Raise | bytecode::RaiseKind::RaiseCause + ) + ); + + // Fire RAISE or RERAISE monitoring event. + // If the callback raises, replace the original exception. + let exception = { + let mon_events = vm.state.monitoring_events.load(); + if is_reraise { + if mon_events & monitoring::EVENT_RERAISE != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_reraise(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + } + } else if mon_events & monitoring::EVENT_RAISE != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_raise(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + } + }; + + // Fire 'exception' trace event for sys.settrace. + // Only for new raises, not re-raises (matching the + // `error` label that calls _PyEval_MonitorRaise). + if !is_reraise { + self.fire_exception_trace(&exception, vm)?; + } + + match handle_exception(self, exception, idx, is_reraise, is_new_raise, vm) { + Ok(None) => {} + Ok(Some(result)) => break Ok(result), + Err(exception) => { + // Fire PY_UNWIND: exception escapes this frame + let exception = if vm.state.monitoring_events.load() + & monitoring::EVENT_PY_UNWIND + != 0 + { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_py_unwind(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + }; + + // Restore lasti from traceback so frame.f_lineno matches tb_lineno + // The traceback was created with the correct lasti when exception + // was first raised, but frame.lasti may have changed during cleanup + if let Some(tb) = exception.__traceback__() + && core::ptr::eq::<Py<Frame>>(&*tb.frame, self.object) + { + // This traceback entry is for this frame - restore its lasti + // tb.lasti is in bytes (idx * 2), convert back to instruction index + self.update_lasti(|i| *i = tb.lasti / 2); + } + break Err(exception); + } + } + } + } + if !do_extend_arg { + arg_state.reset() + } + } + } + + fn yield_from_target(&self) -> Option<&PyObject> { + // checks gi_frame_state == FRAME_SUSPENDED_YIELD_FROM + // which is set when YIELD_VALUE with oparg >= 1 is executed. + // In RustPython, we check: + // 1. lasti points to RESUME (after YIELD_VALUE) + // 2. The previous instruction was YIELD_VALUE with arg >= 1 + // 3. Stack top is the delegate (receiver) + // + // First check if stack is empty - if so, we can't be in yield-from + if self.localsplus.stack_is_empty() { + return None; + } + let lasti = self.lasti() as usize; + if let Some(unit) = self.code.instructions.get(lasti) { + match &unit.op { + Instruction::Send { .. } => return Some(self.top_value()), + Instruction::Resume { .. } | Instruction::InstrumentedResume => { + // Check if previous instruction was YIELD_VALUE with arg >= 1 + // This indicates yield-from/await context + if lasti > 0 + && let Some(prev_unit) = self.code.instructions.get(lasti - 1) + && matches!( + &prev_unit.op, + Instruction::YieldValue { .. } | Instruction::InstrumentedYieldValue + ) + { + // YIELD_VALUE arg: 0 = direct yield, >= 1 = yield-from/await + // OpArgByte.0 is the raw byte value + if u8::from(prev_unit.arg) >= 1 { + // In yield-from/await context, delegate is on top of stack + return Some(self.top_value()); + } + } + } + _ => {} + } + } + None + } + + /// Handle throw() on a generator/coroutine. + fn gen_throw( + &mut self, + vm: &VirtualMachine, + exc_type: PyObjectRef, + exc_val: PyObjectRef, + exc_tb: PyObjectRef, + ) -> PyResult<ExecutionResult> { + self.monitoring_mask = vm.state.monitoring_events.load(); + // Reset prev_line so that LINE monitoring events fire even if + // the exception handler is on the same line as the yield point. + // In CPython, _Py_call_instrumentation_line has a special case + // for RESUME: it fires LINE even when prev_line == current_line. + // Since gen_throw bypasses RESUME, we reset prev_line instead. + *self.prev_line = 0; + if let Some(jen) = self.yield_from_target() { + // Check if the exception is GeneratorExit (type or instance). + // For GeneratorExit, close the sub-iterator instead of throwing. + let is_gen_exit = if let Some(typ) = exc_type.downcast_ref::<PyType>() { + typ.fast_issubclass(vm.ctx.exceptions.generator_exit) + } else { + exc_type.fast_isinstance(vm.ctx.exceptions.generator_exit) + }; + + if is_gen_exit { + // gen_close_iter: close the sub-iterator + let close_result = if let Some(coro) = self.builtin_coro(jen) { + coro.close(jen, vm).map(|_| ()) + } else if let Some(close_meth) = vm.get_attribute_opt(jen.to_owned(), "close")? { + close_meth.call((), vm).map(|_| ()) + } else { + Ok(()) + }; + if let Err(err) = close_result { + let idx = self.lasti().saturating_sub(1) as usize; + if idx < self.code.locations.len() { + let (loc, _end_loc) = self.code.locations[idx]; + let next = err.__traceback__(); + let new_traceback = PyTraceback::new( + next, + self.object.to_owned(), + idx as u32 * 2, + loc.line, + ); + err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + } + + self.push_value(vm.ctx.none()); + vm.contextualize_exception(&err); + return match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) { + Ok(None) => self.run(vm), + Ok(Some(result)) => Ok(result), + Err(exception) => Err(exception), + }; + } + // Fall through to throw_here to raise GeneratorExit in the generator + } else { + // For non-GeneratorExit, delegate throw to sub-iterator + let thrower = if let Some(coro) = self.builtin_coro(jen) { + Some(Either::A(coro)) + } else { + vm.get_attribute_opt(jen.to_owned(), "throw")? + .map(Either::B) + }; + if let Some(thrower) = thrower { + let ret = match thrower { + Either::A(coro) => coro + .throw(jen, exc_type, exc_val, exc_tb, vm) + .to_pyresult(vm), + Either::B(meth) => meth.call((exc_type, exc_val, exc_tb), vm), + }; + return ret.map(ExecutionResult::Yield).or_else(|err| { + // Add traceback entry for the yield-from/await point. + // gen_send_ex2 resumes the frame with a pending exception, + // which goes through error: → PyTraceBack_Here. We add the + // entry here before calling unwind_blocks. + let idx = self.lasti().saturating_sub(1) as usize; + if idx < self.code.locations.len() { + let (loc, _end_loc) = self.code.locations[idx]; + let next = err.__traceback__(); + let new_traceback = PyTraceback::new( + next, + self.object.to_owned(), + idx as u32 * 2, + loc.line, + ); + err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + } + + self.push_value(vm.ctx.none()); + vm.contextualize_exception(&err); + match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) { + Ok(None) => self.run(vm), + Ok(Some(result)) => Ok(result), + Err(exception) => Err(exception), + } + }); + } + } + } + // throw_here: no delegate has throw method, or not in yield-from + // Validate the exception type first. Invalid types propagate directly to + // the caller. Valid types with failed instantiation (e.g. __new__ returns + // wrong type) get thrown into the generator via PyErr_SetObject path. + let ctor = ExceptionCtor::try_from_object(vm, exc_type)?; + let exception = match ctor.instantiate_value(exc_val, vm) { + Ok(exc) => { + if let Some(tb) = Option::<PyRef<PyTraceback>>::try_from_object(vm, exc_tb)? { + exc.set_traceback_typed(Some(tb)); + } + exc + } + Err(err) => err, + }; + + // Add traceback entry for the generator frame at the yield site + let idx = self.lasti().saturating_sub(1) as usize; + if idx < self.code.locations.len() { + let (loc, _end_loc) = self.code.locations[idx]; + let next = exception.__traceback__(); + let new_traceback = + PyTraceback::new(next, self.object.to_owned(), idx as u32 * 2, loc.line); + exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + } + + // Fire PY_THROW and RAISE events before raising the exception. + // If a monitoring callback fails, its exception replaces the original. + let exception = { + let mon_events = vm.state.monitoring_events.load(); + let exception = if mon_events & monitoring::EVENT_PY_THROW != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_py_throw(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + }; + if mon_events & monitoring::EVENT_RAISE != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_raise(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + } + }; + + // when raising an exception, set __context__ to the current exception + // This is done in _PyErr_SetObject + vm.contextualize_exception(&exception); + + // always pushes Py_None before calling gen_send_ex with exc=1 + // This is needed for exception handler to have correct stack state + self.push_value(vm.ctx.none()); + + match self.unwind_blocks(vm, UnwindReason::Raising { exception }) { + Ok(None) => self.run(vm), + Ok(Some(result)) => Ok(result), + Err(exception) => { + // Fire PY_UNWIND: exception escapes the generator frame. + let exception = + if vm.state.monitoring_events.load() & monitoring::EVENT_PY_UNWIND != 0 { + let offset = idx as u32 * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + match monitoring::fire_py_unwind(vm, self.code, offset, &exc_obj) { + Ok(()) => exception, + Err(monitor_exc) => monitor_exc, + } + } else { + exception + }; + Err(exception) + } + } + } + + fn unbound_cell_exception(&self, i: usize, vm: &VirtualMachine) -> PyBaseExceptionRef { + if let Some(&name) = self.code.cellvars.get(i) { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!("local variable '{name}' referenced before assignment").into(), + ) + } else { + let name = self.code.freevars[i - self.code.cellvars.len()]; + vm.new_name_error( + format!("cannot access free variable '{name}' where it is not associated with a value in enclosing scope"), + name.to_owned(), + ) + } + } + + /// Execute a single instruction. + #[inline(always)] + fn execute_instruction( + &mut self, + instruction: Instruction, + arg: bytecode::OpArg, + extend_arg: &mut bool, + vm: &VirtualMachine, + ) -> FrameResult { + flame_guard!(format!( + "Frame::execute_instruction({})", + instruction.display(arg, &self.code.code).to_string() + )); + + #[cfg(feature = "vm-tracing-logging")] + { + trace!("======="); + /* TODO: + for frame in self.frames.iter() { + trace!(" {:?}", frame); + } + */ + trace!(" {:#?}", self); + trace!( + " Executing op code: {}", + instruction.display(arg, &self.code.code) + ); + trace!("======="); + } + + #[cold] + fn name_error(name: &'static PyStrInterned, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) + } + + match instruction { + Instruction::BinaryOp { op } => { + let op_val = op.get(arg); + self.adaptive(|s, ii, cb| s.specialize_binary_op(vm, op_val, ii, cb)); + self.execute_bin_op(vm, op_val) + } + // Super-instruction for BINARY_OP_ADD_UNICODE + STORE_FAST targeting + // the left local, matching BINARY_OP_INPLACE_ADD_UNICODE shape. + Instruction::BinaryOpInplaceAddUnicode => { + let b = self.top_value(); + let a = self.nth_value(1); + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let target_local = self.binary_op_inplace_unicode_target_local(cache_base, a); + if let (Some(_a_str), Some(_b_str), Some(target_local)) = ( + a.downcast_ref_if_exact::<PyStr>(vm), + b.downcast_ref_if_exact::<PyStr>(vm), + target_local, + ) { + let right = self.pop_value(); + let left = self.pop_value(); + + let local_obj = self.localsplus.fastlocals_mut()[target_local] + .take() + .expect("BINARY_OP_INPLACE_ADD_UNICODE target local missing"); + debug_assert!(local_obj.is(&left)); + let mut local_str = local_obj + .downcast_exact::<PyStr>(vm) + .expect("BINARY_OP_INPLACE_ADD_UNICODE target local not exact str") + .into_pyref(); + drop(left); + let right_str = right + .downcast_ref_if_exact::<PyStr>(vm) + .expect("BINARY_OP_INPLACE_ADD_UNICODE right operand not exact str"); + local_str.concat_in_place(right_str.as_wtf8(), vm); + + self.localsplus.fastlocals_mut()[target_local] = Some(local_str.into()); + self.jump_relative_forward( + 1, + Instruction::BinaryOpInplaceAddUnicode.cache_entries() as u32, + ); + Ok(None) + } else { + self.execute_bin_op(vm, self.binary_op_from_arg(arg)) + } + } + Instruction::BinarySlice => { + // Stack: [container, start, stop] -> [result] + let stop = self.pop_value(); + let start = self.pop_value(); + let container = self.pop_value(); + let slice: PyObjectRef = PySlice { + start: Some(start), + stop, + step: None, + } + .into_ref(&vm.ctx) + .into(); + let result = container.get_item(&*slice, vm)?; + self.push_value(result); + Ok(None) + } + Instruction::BuildList { count: size } => { + let sz = size.get(arg) as usize; + let elements = self.pop_multiple(sz).collect(); + let list_obj = vm.ctx.new_list(elements); + self.push_value(list_obj.into()); + Ok(None) + } + Instruction::BuildMap { count: size } => self.execute_build_map(vm, size.get(arg)), + Instruction::BuildSet { count: size } => { + let set = PySet::default().into_ref(&vm.ctx); + for element in self.pop_multiple(size.get(arg) as usize) { + set.add(element, vm)?; + } + self.push_value(set.into()); + Ok(None) + } + Instruction::BuildSlice { argc } => self.execute_build_slice(vm, argc.get(arg)), + /* + Instruction::ToBool => { + dbg!("Shouldn't be called outside of match statements for now") + let value = self.pop_value(); + // call __bool__ + let result = value.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } + */ + Instruction::BuildString { count: size } => { + let s: Wtf8Buf = self + .pop_multiple(size.get(arg) as usize) + .map(|pyobj| pyobj.downcast::<PyStr>().unwrap()) + .collect(); + self.push_value(vm.ctx.new_str(s).into()); + Ok(None) + } + Instruction::BuildTuple { count: size } => { + let elements = self.pop_multiple(size.get(arg) as usize).collect(); + let list_obj = vm.ctx.new_tuple(elements); + self.push_value(list_obj.into()); + Ok(None) + } + Instruction::BuildTemplate => { + // Stack: [strings_tuple, interpolations_tuple] -> [template] + let interpolations = self.pop_value(); + let strings = self.pop_value(); + + let strings = strings + .downcast::<PyTuple>() + .map_err(|_| vm.new_type_error("BUILD_TEMPLATE expected tuple for strings"))?; + let interpolations = interpolations.downcast::<PyTuple>().map_err(|_| { + vm.new_type_error("BUILD_TEMPLATE expected tuple for interpolations") + })?; + + let template = PyTemplate::new(strings, interpolations); + self.push_value(template.into_pyobject(vm)); + Ok(None) + } + Instruction::BuildInterpolation { format: oparg } => { + // oparg encoding: (conversion << 2) | has_format_spec + // Stack: [value, expression_str, (format_spec)?] -> [interpolation] + let oparg_val = oparg.get(arg); + let has_format_spec = (oparg_val & 1) != 0; + let conversion_code = oparg_val >> 2; + + let format_spec = if has_format_spec { + self.pop_value().downcast::<PyStr>().map_err(|_| { + vm.new_type_error("BUILD_INTERPOLATION expected str for format_spec") + })? + } else { + vm.ctx.empty_str.to_owned() + }; + + let expression = self.pop_value().downcast::<PyStr>().map_err(|_| { + vm.new_type_error("BUILD_INTERPOLATION expected str for expression") + })?; + let value = self.pop_value(); + + // conversion: 0=None, 1=Str, 2=Repr, 3=Ascii + let conversion: PyObjectRef = match conversion_code { + 0 => vm.ctx.none(), + 1 => vm.ctx.new_str("s").into(), + 2 => vm.ctx.new_str("r").into(), + 3 => vm.ctx.new_str("a").into(), + _ => vm.ctx.none(), // should not happen + }; + + let interpolation = + PyInterpolation::new(value, expression, conversion, format_spec, vm)?; + self.push_value(interpolation.into_pyobject(vm)); + Ok(None) + } + Instruction::Call { argc: nargs } => { + // Stack: [callable, self_or_null, arg1, ..., argN] + let nargs_val = nargs.get(arg); + self.adaptive(|s, ii, cb| s.specialize_call(vm, nargs_val, ii, cb)); + self.execute_call_vectorcall(nargs_val, vm) + } + Instruction::CallKw { argc: nargs } => { + let nargs = nargs.get(arg); + self.adaptive(|s, ii, cb| s.specialize_call_kw(vm, nargs, ii, cb)); + // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] + self.execute_call_kw_vectorcall(nargs, vm) + } + Instruction::CallFunctionEx => { + // Stack: [callable, self_or_null, args_tuple, kwargs_or_null] + let args = self.collect_ex_args(vm)?; + self.execute_call(args, vm) + } + Instruction::CallIntrinsic1 { func } => { + let value = self.pop_value(); + let result = self.call_intrinsic_1(func.get(arg), value, vm)?; + self.push_value(result); + Ok(None) + } + Instruction::CallIntrinsic2 { func } => { + let value2 = self.pop_value(); + let value1 = self.pop_value(); + let result = self.call_intrinsic_2(func.get(arg), value1, value2, vm)?; + self.push_value(result); + Ok(None) + } + Instruction::CheckEgMatch => { + let match_type = self.pop_value(); + let exc_value = self.pop_value(); + let (rest, matched) = + crate::exceptions::exception_group_match(&exc_value, &match_type, vm)?; + + // Set matched exception as current exception (if not None) + // This mirrors CPython's PyErr_SetHandledException(match_o) in CHECK_EG_MATCH + if !vm.is_none(&matched) + && let Some(exc) = matched.downcast_ref::<PyBaseException>() + { + vm.set_exception(Some(exc.to_owned())); + } + + self.push_value(rest); + self.push_value(matched); + Ok(None) + } + Instruction::CompareOp { opname: op } => { + let op_val = op.get(arg); + self.adaptive(|s, ii, cb| s.specialize_compare_op(vm, op_val, ii, cb)); + self.execute_compare(vm, op_val) + } + Instruction::ContainsOp { invert } => { + self.adaptive(|s, ii, cb| s.specialize_contains_op(vm, ii, cb)); + let b = self.pop_value(); + let a = self.pop_value(); + + let value = match invert.get(arg) { + bytecode::Invert::No => self._in(vm, &a, &b)?, + bytecode::Invert::Yes => self._not_in(vm, &a, &b)?, + }; + self.push_value(vm.ctx.new_bool(value).into()); + Ok(None) + } + Instruction::ConvertValue { oparg: conversion } => { + self.convert_value(conversion.get(arg), vm) + } + Instruction::Copy { i: index } => { + // CopyItem { index: 1 } copies TOS + // CopyItem { index: 2 } copies second from top + // This is 1-indexed to match CPython + let idx = index.get(arg) as usize; + let stack_len = self.localsplus.stack_len(); + debug_assert!(stack_len >= idx, "CopyItem: stack underflow"); + let value = self.localsplus.stack_index(stack_len - idx).clone(); + self.push_stackref_opt(value); + Ok(None) + } + Instruction::CopyFreeVars { .. } => { + // Free vars are already set up at frame creation time in RustPython + Ok(None) + } + Instruction::DeleteAttr { namei: idx } => self.delete_attr(vm, idx.get(arg)), + Instruction::DeleteDeref { i } => { + self.cell_ref(i.get(arg) as usize).set(None); + Ok(None) + } + Instruction::DeleteFast { var_num } => { + let fastlocals = self.localsplus.fastlocals_mut(); + let idx = var_num.get(arg); + if fastlocals[idx].is_none() { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx] + ) + .into(), + )); + } + fastlocals[idx] = None; + Ok(None) + } + Instruction::DeleteGlobal { namei: idx } => { + let name = self.code.names[idx.get(arg) as usize]; + match self.globals.del_item(name, vm) { + Ok(()) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + return Err(name_error(name, vm)); + } + Err(e) => return Err(e), + } + Ok(None) + } + Instruction::DeleteName { namei: idx } => { + let name = self.code.names[idx.get(arg) as usize]; + let res = self.locals.mapping(vm).ass_subscript(name, None, vm); + + match res { + Ok(()) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + return Err(name_error(name, vm)); + } + Err(e) => return Err(e), + } + Ok(None) + } + Instruction::DeleteSubscr => self.execute_delete_subscript(vm), + Instruction::DictUpdate { i: index } => { + // Stack before: [..., dict, ..., source] (source at TOS) + // Stack after: [..., dict, ...] (source consumed) + // The dict to update is at position TOS-i (before popping source) + + let idx = index.get(arg); + + // Pop the source from TOS + let source = self.pop_value(); + + // Get the dict to update (it's now at TOS-(i-1) after popping source) + let dict = if idx <= 1 { + // DICT_UPDATE 0 or 1: dict is at TOS (after popping source) + self.top_value() + } else { + // DICT_UPDATE n: dict is at TOS-(n-1) + self.nth_value(idx - 1) + }; + + let dict = dict.downcast_ref::<PyDict>().expect("exact dict expected"); + + // For dictionary unpacking {**x}, x must be a mapping + // Check if the object has the mapping protocol (keys method) + if vm + .get_method(source.clone(), vm.ctx.intern_str("keys")) + .is_none() + { + return Err(vm.new_type_error(format!( + "'{}' object is not a mapping", + source.class().name() + ))); + } + + dict.merge_object(source, vm)?; + Ok(None) + } + Instruction::DictMerge { i: index } => { + let source = self.pop_value(); + let idx = index.get(arg); + + // Get the dict to merge into (same logic as DICT_UPDATE) + let dict_ref = if idx <= 1 { + self.top_value() + } else { + self.nth_value(idx - 1) + }; + + let dict: &Py<PyDict> = unsafe { dict_ref.downcast_unchecked_ref() }; + + // Get callable for error messages + // Stack: [callable, self_or_null, args_tuple, kwargs_dict] + let callable = self.nth_value(idx + 2); + let func_str = Self::object_function_str(callable, vm); + + // Check if source is a mapping + if vm + .get_method(source.clone(), vm.ctx.intern_str("keys")) + .is_none() + { + return Err(vm.new_type_error(format!( + "{} argument after ** must be a mapping, not {}", + func_str, + source.class().name() + ))); + } + + // Merge keys, checking for duplicates + let keys_iter = vm.call_method(&source, "keys", ())?; + for key in keys_iter.try_to_value::<Vec<PyObjectRef>>(vm)? { + if dict.contains_key(&*key, vm) { + let key_str = key.str(vm)?; + return Err(vm.new_type_error(format!( + "{} got multiple values for keyword argument '{}'", + func_str, + key_str.as_wtf8() + ))); + } + let value = vm.call_method(&source, "__getitem__", (key.clone(),))?; + dict.set_item(&*key, value, vm)?; + } + Ok(None) + } + Instruction::EndAsyncFor => { + // Pops (awaitable, exc) from stack. + // If exc is StopAsyncIteration, clears it (normal loop end). + // Otherwise re-raises. + let exc = self.pop_value(); + let _awaitable = self.pop_value(); + + let exc = exc + .downcast::<PyBaseException>() + .expect("EndAsyncFor expects exception on stack"); + + if exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) { + // StopAsyncIteration - normal end of async for loop + vm.set_exception(None); + Ok(None) + } else { + // Other exception - re-raise + Err(exc) + } + } + Instruction::ExtendedArg => { + *extend_arg = true; + Ok(None) + } + Instruction::ForIter { .. } => { + // Relative forward jump: target = lasti + caches + delta + let target = bytecode::Label::new(self.lasti() + 1 + u32::from(arg)); + self.adaptive(|s, ii, cb| s.specialize_for_iter(vm, u32::from(arg), ii, cb)); + self.execute_for_iter(vm, target)?; + Ok(None) + } + Instruction::FormatSimple => { + let value = self.pop_value(); + let formatted = vm.format(&value, vm.ctx.new_str(""))?; + self.push_value(formatted.into()); + + Ok(None) + } + Instruction::FormatWithSpec => { + let spec = self.pop_value(); + let value = self.pop_value(); + let formatted = vm.format(&value, spec.downcast::<PyStr>().unwrap())?; + self.push_value(formatted.into()); + + Ok(None) + } + Instruction::GetAIter => { + let aiterable = self.pop_value(); + let aiter = vm.call_special_method(&aiterable, identifier!(vm, __aiter__), ())?; + self.push_value(aiter); + Ok(None) + } + Instruction::GetANext => { + #[cfg(debug_assertions)] // remove when GetANext is fully implemented + let orig_stack_len = self.localsplus.stack_len(); + + let aiter = self.top_value(); + let awaitable = if aiter.class().is(vm.ctx.types.async_generator) { + vm.call_special_method(aiter, identifier!(vm, __anext__), ())? + } else { + if !aiter.has_attr("__anext__", vm).unwrap_or(false) { + // TODO: __anext__ must be protocol + let msg = format!( + "'async for' requires an iterator with __anext__ method, got {:.100}", + aiter.class().name() + ); + return Err(vm.new_type_error(msg)); + } + let next_iter = + vm.call_special_method(aiter, identifier!(vm, __anext__), ())?; + + // _PyCoro_GetAwaitableIter in CPython + fn get_awaitable_iter(next_iter: &PyObject, vm: &VirtualMachine) -> PyResult { + let gen_is_coroutine = |_| { + // TODO: cpython gen_is_coroutine + true + }; + if next_iter.class().is(vm.ctx.types.coroutine_type) + || gen_is_coroutine(next_iter) + { + return Ok(next_iter.to_owned()); + } + // TODO: error handling + vm.call_special_method(next_iter, identifier!(vm, __await__), ()) + } + get_awaitable_iter(&next_iter, vm).map_err(|_| { + vm.new_type_error(format!( + "'async for' received an invalid object from __anext__: {:.200}", + next_iter.class().name() + )) + })? + }; + self.push_value(awaitable); + #[cfg(debug_assertions)] + debug_assert_eq!(orig_stack_len + 1, self.localsplus.stack_len()); + Ok(None) + } + Instruction::GetAwaitable { r#where: oparg } => { + let iterable = self.pop_value(); + + let iter = match crate::coroutine::get_awaitable_iter(iterable.clone(), vm) { + Ok(iter) => iter, + Err(e) => { + // _PyEval_FormatAwaitableError: override error for async with + // when the type doesn't have __await__ + let oparg_val = oparg.get(arg); + if vm + .get_method(iterable.clone(), identifier!(vm, __await__)) + .is_none() + { + if oparg_val == 1 { + return Err(vm.new_type_error(format!( + "'async with' received an object from __aenter__ \ + that does not implement __await__: {}", + iterable.class().name() + ))); + } else if oparg_val == 2 { + return Err(vm.new_type_error(format!( + "'async with' received an object from __aexit__ \ + that does not implement __await__: {}", + iterable.class().name() + ))); + } + } + return Err(e); + } + }; + + // Check if coroutine is already being awaited + if let Some(coro) = iter.downcast_ref::<PyCoroutine>() + && coro.as_coro().frame().yield_from_target().is_some() + { + return Err(vm.new_runtime_error("coroutine is being awaited already")); + } + + self.push_value(iter); + Ok(None) + } + Instruction::GetIter => { + let iterated_obj = self.pop_value(); + let iter_obj = iterated_obj.get_iter(vm)?; + self.push_value(iter_obj.into()); + Ok(None) + } + Instruction::GetYieldFromIter => { + // GET_YIELD_FROM_ITER: prepare iterator for yield from + // If iterable is a coroutine, ensure we're in a coroutine context + // If iterable is a generator, use it directly + // Otherwise, call iter() on it + let iterable = self.pop_value(); + let iter = if iterable.class().is(vm.ctx.types.coroutine_type) { + // Coroutine requires CO_COROUTINE or CO_ITERABLE_COROUTINE flag + if !self.code.flags.intersects( + bytecode::CodeFlags::COROUTINE | bytecode::CodeFlags::ITERABLE_COROUTINE, + ) { + return Err(vm.new_type_error( + "cannot 'yield from' a coroutine object in a non-coroutine generator", + )); + } + iterable + } else if iterable.class().is(vm.ctx.types.generator_type) { + // Generator can be used directly + iterable + } else { + // Otherwise, get iterator + iterable.get_iter(vm)?.into() + }; + self.push_value(iter); + Ok(None) + } + Instruction::GetLen => { + // STACK.append(len(STACK[-1])) + let obj = self.top_value(); + let len = obj.length(vm)?; + self.push_value(vm.ctx.new_int(len).into()); + Ok(None) + } + Instruction::ImportFrom { namei: idx } => { + let obj = self.import_from(vm, idx.get(arg))?; + self.push_value(obj); + Ok(None) + } + Instruction::ImportName { namei: idx } => { + self.import(vm, Some(self.code.names[idx.get(arg) as usize]))?; + Ok(None) + } + Instruction::IsOp { invert } => { + let b = self.pop_value(); + let a = self.pop_value(); + let res = a.is(&b); + + let value = match invert.get(arg) { + bytecode::Invert::No => res, + bytecode::Invert::Yes => !res, + }; + self.push_value(vm.ctx.new_bool(value).into()); + Ok(None) + } + Instruction::JumpForward { .. } => { + self.jump_relative_forward(u32::from(arg), 0); + Ok(None) + } + Instruction::JumpBackward { .. } => { + // CPython rewrites JUMP_BACKWARD to JUMP_BACKWARD_NO_JIT + // when JIT is unavailable. + let instr_idx = self.lasti() as usize - 1; + unsafe { + self.code + .instructions + .replace_op(instr_idx, Instruction::JumpBackwardNoJit); + } + self.jump_relative_backward(u32::from(arg), 1); + Ok(None) + } + Instruction::JumpBackwardJit | Instruction::JumpBackwardNoJit => { + self.jump_relative_backward(u32::from(arg), 1); + Ok(None) + } + Instruction::JumpBackwardNoInterrupt { .. } => { + self.jump_relative_backward(u32::from(arg), 0); + Ok(None) + } + Instruction::ListAppend { i } => { + let item = self.pop_value(); + let obj = self.nth_value(i.get(arg)); + let list: &Py<PyList> = unsafe { + // SAFETY: trust compiler + obj.downcast_unchecked_ref() + }; + list.append(item); + Ok(None) + } + Instruction::ListExtend { i } => { + let iterable = self.pop_value(); + let obj = self.nth_value(i.get(arg)); + let list: &Py<PyList> = unsafe { + // SAFETY: compiler guarantees correct type + obj.downcast_unchecked_ref() + }; + let type_name = iterable.class().name().to_owned(); + // Only rewrite the error if the type is truly not iterable + // (no __iter__ and no __getitem__). Preserve original TypeError + // from custom iterables that raise during iteration. + let not_iterable = iterable.class().slots.iter.load().is_none() + && iterable + .get_class_attr(vm.ctx.intern_str("__getitem__")) + .is_none(); + list.extend(iterable, vm).map_err(|e| { + if not_iterable && e.class().is(vm.ctx.exceptions.type_error) { + vm.new_type_error(format!( + "Value after * must be an iterable, not {type_name}" + )) + } else { + e + } + })?; + Ok(None) + } + Instruction::LoadAttr { namei: idx } => self.load_attr(vm, idx.get(arg)), + Instruction::LoadSuperAttr { namei: idx } => { + let idx_val = idx.get(arg); + self.adaptive(|s, ii, cb| s.specialize_load_super_attr(vm, idx_val, ii, cb)); + self.load_super_attr(vm, idx_val) + } + Instruction::LoadBuildClass => { + let build_class = if let Some(builtins_dict) = self.builtins_dict { + builtins_dict + .get_item_opt(identifier!(vm, __build_class__), vm)? + .ok_or_else(|| { + vm.new_name_error( + "__build_class__ not found", + identifier!(vm, __build_class__).to_owned(), + ) + })? + } else { + self.builtins + .get_item(identifier!(vm, __build_class__), vm) + .map_err(|e| { + if e.fast_isinstance(vm.ctx.exceptions.key_error) { + vm.new_name_error( + "__build_class__ not found", + identifier!(vm, __build_class__).to_owned(), + ) + } else { + e + } + })? + }; + self.push_value(build_class); + Ok(None) + } + Instruction::LoadLocals => { + // Push the locals dict onto the stack + let locals = self.locals.into_object(vm); + self.push_value(locals); + Ok(None) + } + Instruction::LoadFromDictOrDeref { i } => { + // Pop dict from stack (locals or classdict depending on context) + let class_dict = self.pop_value(); + let i = i.get(arg) as usize; + let name = if i < self.code.cellvars.len() { + self.code.cellvars[i] + } else { + self.code.freevars[i - self.code.cellvars.len()] + }; + // Only treat KeyError as "not found", propagate other exceptions + let value = if let Some(dict_obj) = class_dict.downcast_ref::<PyDict>() { + dict_obj.get_item_opt(name, vm)? + } else { + match class_dict.get_item(name, vm) { + Ok(v) => Some(v), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => None, + Err(e) => return Err(e), + } + }; + self.push_value(match value { + Some(v) => v, + None => self + .cell_ref(i) + .get() + .ok_or_else(|| self.unbound_cell_exception(i, vm))?, + }); + Ok(None) + } + Instruction::LoadFromDictOrGlobals { i: idx } => { + // PEP 649: Pop dict from stack (classdict), check there first, then globals + let dict = self.pop_value(); + let name = self.code.names[idx.get(arg) as usize]; + + // Only treat KeyError as "not found", propagate other exceptions + let value = if let Some(dict_obj) = dict.downcast_ref::<PyDict>() { + dict_obj.get_item_opt(name, vm)? + } else { + // Not an exact dict, use mapping protocol + match dict.get_item(name, vm) { + Ok(v) => Some(v), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => None, + Err(e) => return Err(e), + } + }; + + self.push_value(match value { + Some(v) => v, + None => self.load_global_or_builtin(name, vm)?, + }); + Ok(None) + } + Instruction::LoadConst { consti } => { + self.push_value(self.code.constants[consti.get(arg)].clone().into()); + // Mirror CPython's LOAD_CONST family transition. RustPython does + // not currently distinguish immortal constants at runtime. + let instr_idx = self.lasti() as usize - 1; + unsafe { + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadConstMortal); + } + Ok(None) + } + Instruction::LoadConstMortal | Instruction::LoadConstImmortal => { + self.push_value(self.code.constants[u32::from(arg).into()].clone().into()); + Ok(None) + } + Instruction::LoadCommonConstant { idx } => { + use bytecode::CommonConstant; + let value = match idx.get(arg) { + CommonConstant::AssertionError => { + vm.ctx.exceptions.assertion_error.to_owned().into() + } + CommonConstant::NotImplementedError => { + vm.ctx.exceptions.not_implemented_error.to_owned().into() + } + CommonConstant::BuiltinTuple => vm.ctx.types.tuple_type.to_owned().into(), + CommonConstant::BuiltinAll => vm.builtins.get_attr("all", vm)?, + CommonConstant::BuiltinAny => vm.builtins.get_attr("any", vm)?, + }; + self.push_value(value); + Ok(None) + } + Instruction::LoadSmallInt { i: idx } => { + // Push small integer (-5..=256) directly without constant table lookup + let value = vm.ctx.new_int(idx.get(arg) as i32); + self.push_value(value.into()); + Ok(None) + } + Instruction::LoadDeref { i } => { + let idx = i.get(arg) as usize; + let x = self + .cell_ref(idx) + .get() + .ok_or_else(|| self.unbound_cell_exception(idx, vm))?; + self.push_value(x); + Ok(None) + } + Instruction::LoadFast { var_num } => { + #[cold] + fn reference_error( + varname: &'static PyStrInterned, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!("local variable '{varname}' referenced before assignment").into(), + ) + } + let idx = var_num.get(arg); + let x = self.localsplus.fastlocals()[idx] + .clone() + .ok_or_else(|| reference_error(self.code.varnames[idx], vm))?; + self.push_value(x); + Ok(None) + } + Instruction::LoadFastAndClear { var_num } => { + // Load value and clear the slot (for inlined comprehensions) + // If slot is empty, push None (not an error - variable may not exist yet) + let idx = var_num.get(arg); + let x = self.localsplus.fastlocals_mut()[idx] + .take() + .unwrap_or_else(|| vm.ctx.none()); + self.push_value(x); + Ok(None) + } + Instruction::LoadFastCheck { var_num } => { + // Same as LoadFast but explicitly checks for unbound locals + // (LoadFast in RustPython already does this check) + let idx = var_num.get(arg); + let x = self.localsplus.fastlocals()[idx].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx] + ) + .into(), + ) + })?; + self.push_value(x); + Ok(None) + } + Instruction::LoadFastLoadFast { var_nums } => { + // Load two local variables at once + // oparg encoding: (idx1 << 4) | idx2 + let oparg = var_nums.get(arg); + let (idx1, idx2) = oparg.indexes(); + let fastlocals = self.localsplus.fastlocals(); + let x1 = fastlocals[idx1].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx1] + ) + .into(), + ) + })?; + let x2 = fastlocals[idx2].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx2] + ) + .into(), + ) + })?; + self.push_value(x1); + self.push_value(x2); + Ok(None) + } + // Borrow optimization not yet active; falls back to clone. + // push_borrowed() is available but disabled until stack + // lifetime issues at yield/exception points are resolved. + Instruction::LoadFastBorrow { var_num } => { + let idx = var_num.get(arg); + let x = self.localsplus.fastlocals()[idx].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx] + ) + .into(), + ) + })?; + self.push_value(x); + Ok(None) + } + Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { + let oparg = var_nums.get(arg); + let (idx1, idx2) = oparg.indexes(); + let fastlocals = self.localsplus.fastlocals(); + let x1 = fastlocals[idx1].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx1] + ) + .into(), + ) + })?; + let x2 = fastlocals[idx2].clone().ok_or_else(|| { + vm.new_exception_msg( + vm.ctx.exceptions.unbound_local_error.to_owned(), + format!( + "local variable '{}' referenced before assignment", + self.code.varnames[idx2] + ) + .into(), + ) + })?; + self.push_value(x1); + self.push_value(x2); + Ok(None) + } + Instruction::LoadGlobal { namei: idx } => { + let oparg = idx.get(arg); + self.adaptive(|s, ii, cb| s.specialize_load_global(vm, oparg, ii, cb)); + let name = &self.code.names[(oparg >> 1) as usize]; + let x = self.load_global_or_builtin(name, vm)?; + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); + } + Ok(None) + } + Instruction::LoadName { namei: idx } => { + let name = self.code.names[idx.get(arg) as usize]; + let result = self.locals.mapping(vm).subscript(name, vm); + match result { + Ok(x) => self.push_value(x), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + self.push_value(self.load_global_or_builtin(name, vm)?); + } + Err(e) => return Err(e), + } + Ok(None) + } + Instruction::LoadSpecial { method } => { + // Stack effect: 0 (replaces TOS with bound method) + // Input: [..., obj] + // Output: [..., bound_method] + use crate::vm::PyMethod; + + let obj = self.pop_value(); + let oparg = method.get(arg); + let method_name = get_special_method_name(oparg, vm); + + let bound = match vm.get_special_method(&obj, method_name)? { + Some(PyMethod::Function { target, func }) => { + // Create bound method: PyBoundMethod(object=target, function=func) + crate::builtins::PyBoundMethod::new(target, func) + .into_ref(&vm.ctx) + .into() + } + Some(PyMethod::Attribute(bound)) => bound, + None => { + return Err(vm.new_type_error(get_special_method_error_msg( + oparg, + &obj.class().name(), + special_method_can_suggest(&obj, oparg, vm)?, + ))); + } + }; + self.push_value(bound); + Ok(None) + } + Instruction::MakeFunction => self.execute_make_function(vm), + Instruction::MakeCell { .. } => { + // Cell creation is handled at frame creation time in RustPython + Ok(None) + } + Instruction::MapAdd { i } => { + let value = self.pop_value(); + let key = self.pop_value(); + let obj = self.nth_value(i.get(arg)); + let dict: &Py<PyDict> = unsafe { + // SAFETY: trust compiler + obj.downcast_unchecked_ref() + }; + dict.set_item(&*key, value, vm)?; + Ok(None) + } + Instruction::MatchClass { count: nargs } => { + // STACK[-1] is a tuple of keyword attribute names, STACK[-2] is the class being matched against, and STACK[-3] is the match subject. + // nargs is the number of positional sub-patterns. + let kwd_attrs = self.pop_value(); + let kwd_attrs = kwd_attrs.downcast_ref::<PyTuple>().unwrap(); + let cls = self.pop_value(); + let subject = self.pop_value(); + let nargs_val = nargs.get(arg) as usize; + + // Check if subject is an instance of cls + if subject.is_instance(cls.as_ref(), vm)? { + let mut extracted = vec![]; + + // Get __match_args__ for positional arguments if nargs > 0 + if nargs_val > 0 { + // Get __match_args__ from the class + let match_args = + vm.get_attribute_opt(cls.clone(), identifier!(vm, __match_args__))?; + + if let Some(match_args) = match_args { + // Convert to tuple + let match_args = match match_args.downcast_exact::<PyTuple>(vm) { + Ok(tuple) => tuple, + Err(match_args) => { + // __match_args__ must be a tuple + // Get type names for error message + let type_name = cls + .downcast::<crate::builtins::PyType>() + .ok() + .and_then(|t| t.__name__(vm).to_str().map(str::to_owned)) + .unwrap_or_else(|| String::from("?")); + let match_args_type_name = match_args.class().__name__(vm); + return Err(vm.new_type_error(format!( + "{}.__match_args__ must be a tuple (got {})", + type_name, match_args_type_name + ))); + } + }; + + // Check if we have enough match args + if match_args.len() < nargs_val { + return Err(vm.new_type_error(format!( + "class pattern accepts at most {} positional sub-patterns ({} given)", + match_args.len(), + nargs_val + ))); + } + + // Extract positional attributes + for i in 0..nargs_val { + let attr_name = &match_args[i]; + let attr_name_str = match attr_name.downcast_ref::<PyStr>() { + Some(s) => s, + None => { + return Err(vm.new_type_error( + "__match_args__ elements must be strings", + )); + } + }; + match subject.get_attr(attr_name_str, vm) { + Ok(value) => extracted.push(value), + Err(e) + if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => + { + // Missing attribute → non-match + self.push_value(vm.ctx.none()); + return Ok(None); + } + Err(e) => return Err(e), + } + } + } else { + // No __match_args__, check if this is a type with MATCH_SELF behavior + // For built-in types like bool, int, str, list, tuple, dict, etc. + // they match the subject itself as the single positional argument + let is_match_self_type = cls + .downcast::<PyType>() + .is_ok_and(|t| t.slots.flags.contains(PyTypeFlags::_MATCH_SELF)); + + if is_match_self_type { + if nargs_val == 1 { + // Match the subject itself as the single positional argument + extracted.push(subject.clone()); + } else if nargs_val > 1 { + // Too many positional arguments for MATCH_SELF + return Err(vm.new_type_error( + "class pattern accepts at most 1 positional sub-pattern for MATCH_SELF types", + )); + } + } else { + // No __match_args__ and not a MATCH_SELF type + if nargs_val > 0 { + return Err(vm.new_type_error( + "class pattern defines no positional sub-patterns (__match_args__ missing)", + )); + } + } + } + } + + // Extract keyword attributes + for name in kwd_attrs { + let name_str = name.downcast_ref::<PyStr>().unwrap(); + match subject.get_attr(name_str, vm) { + Ok(value) => extracted.push(value), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => { + self.push_value(vm.ctx.none()); + return Ok(None); + } + Err(e) => return Err(e), + } + } + + self.push_value(vm.ctx.new_tuple(extracted).into()); + } else { + // Not an instance, push None + self.push_value(vm.ctx.none()); + } + Ok(None) + } + Instruction::MatchKeys => { + // MATCH_KEYS doesn't pop subject and keys, only reads them + let keys_tuple = self.top_value(); // stack[-1] + let subject = self.nth_value(1); // stack[-2] + + // Check if subject is a mapping and extract values for keys + if subject.class().slots.flags.contains(PyTypeFlags::MAPPING) { + let keys = keys_tuple.downcast_ref::<PyTuple>().unwrap(); + let mut values = Vec::new(); + let mut all_match = true; + + // We use the two argument form of map.get(key, default) for two reasons: + // - Atomically check for a key and get its value without error handling. + // - Don't cause key creation or resizing in dict subclasses like + // collections.defaultdict that define __missing__ (or similar). + // See CPython's _PyEval_MatchKeys + + if let Some(get_method) = vm + .get_method(subject.to_owned(), vm.ctx.intern_str("get")) + .transpose()? + { + let dummy = vm + .ctx + .new_base_object(vm.ctx.types.object_type.to_owned(), None); + + for key in keys { + // value = map.get(key, dummy) + match get_method.call((key.as_object(), dummy.clone()), vm) { + Ok(value) => { + // if value == dummy: key not in map! + if value.is(&dummy) { + all_match = false; + break; + } + values.push(value); + } + Err(e) => return Err(e), + } + } + } else { + // Fallback if .get() method is not available (shouldn't happen for mappings) + for key in keys { + match subject.get_item(key.as_object(), vm) { + Ok(value) => values.push(value), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { + all_match = false; + break; + } + Err(e) => return Err(e), + } + } + } + + if all_match { + // Push values tuple on successful match + self.push_value(vm.ctx.new_tuple(values).into()); + } else { + // No match - push None + self.push_value(vm.ctx.none()); + } + } else { + // Not a mapping - push None + self.push_value(vm.ctx.none()); + } + Ok(None) + } + Instruction::MatchMapping => { + // Pop and push back the subject to keep it on stack + let subject = self.pop_value(); + + // Check if the type has the MAPPING flag + let is_mapping = subject.class().slots.flags.contains(PyTypeFlags::MAPPING); + + self.push_value(subject); + self.push_value(vm.ctx.new_bool(is_mapping).into()); + Ok(None) + } + Instruction::MatchSequence => { + // Pop and push back the subject to keep it on stack + let subject = self.pop_value(); + + // Check if the type has the SEQUENCE flag + let is_sequence = subject.class().slots.flags.contains(PyTypeFlags::SEQUENCE); + + self.push_value(subject); + self.push_value(vm.ctx.new_bool(is_sequence).into()); + Ok(None) + } + Instruction::Nop => Ok(None), + // NOT_TAKEN is a branch prediction hint - functionally a NOP + Instruction::NotTaken => Ok(None), + // CACHE is used by adaptive interpreter for inline caching - NOP for us + Instruction::Cache => Ok(None), + Instruction::ReturnGenerator => { + // In RustPython, generators/coroutines are created in function.rs + // before the frame starts executing. The RETURN_GENERATOR instruction + // pushes None so that the following POP_TOP has something to consume. + // This matches CPython's semantics where the sent value (None for first call) + // is on the stack when the generator resumes. + self.push_value(vm.ctx.none()); + Ok(None) + } + Instruction::PopExcept => { + // Pop prev_exc from value stack and restore it + let prev_exc = self.pop_value(); + if vm.is_none(&prev_exc) { + vm.set_exception(None); + } else if let Ok(exc) = prev_exc.downcast::<PyBaseException>() { + vm.set_exception(Some(exc)); + } + + // NOTE: We do NOT clear the traceback of the exception that was just handled. + // Python preserves exception tracebacks even after the exception is no longer + // the "current exception". This is important for code that catches an exception, + // stores it, and later inspects its traceback. + // Reference cycles (Exception → Traceback → Frame → locals) are handled by + // Python's garbage collector which can detect and break cycles. + + Ok(None) + } + Instruction::PopJumpIfFalse { .. } => self.pop_jump_if_relative(vm, arg, 1, false), + Instruction::PopJumpIfTrue { .. } => self.pop_jump_if_relative(vm, arg, 1, true), + Instruction::PopJumpIfNone { .. } => { + let value = self.pop_value(); + if vm.is_none(&value) { + self.jump_relative_forward(u32::from(arg), 1); + } + Ok(None) + } + Instruction::PopJumpIfNotNone { .. } => { + let value = self.pop_value(); + if !vm.is_none(&value) { + self.jump_relative_forward(u32::from(arg), 1); + } + Ok(None) + } + Instruction::PopTop => { + // Pop value from stack and ignore. + self.pop_value(); + Ok(None) + } + Instruction::EndFor => { + // Pop the next value from stack (cleanup after loop body) + self.pop_value(); + Ok(None) + } + Instruction::PopIter => { + // Pop the iterator from stack (end of for loop) + self.pop_value(); + Ok(None) + } + Instruction::PushNull => { + // Push NULL for self_or_null slot in call protocol + self.push_null(); + Ok(None) + } + Instruction::RaiseVarargs { argc: kind } => self.execute_raise(vm, kind.get(arg)), + Instruction::Resume { .. } | Instruction::ResumeCheck => { + // Lazy quickening: initialize adaptive counters on first execution + if !self.code.quickened.swap(true, atomic::Ordering::Relaxed) { + self.code.instructions.quicken(); + atomic::fence(atomic::Ordering::Release); + } + if self.monitoring_disabled_for_code(vm) { + let global_ver = vm + .state + .instrumentation_version + .load(atomic::Ordering::Acquire); + monitoring::instrument_code(self.code, 0); + self.code + .instrumentation_version + .store(global_ver, atomic::Ordering::Release); + return Ok(None); + } + // Check if bytecode needs re-instrumentation + let global_ver = vm + .state + .instrumentation_version + .load(atomic::Ordering::Acquire); + let code_ver = self + .code + .instrumentation_version + .load(atomic::Ordering::Acquire); + if code_ver != global_ver { + let events = { + let state = vm.state.monitoring.lock(); + state.events_for_code(self.code.get_id()) + }; + monitoring::instrument_code(self.code, events); + self.code + .instrumentation_version + .store(global_ver, atomic::Ordering::Release); + // Re-execute this instruction (it may now be INSTRUMENTED_RESUME) + self.update_lasti(|i| *i -= 1); + } + Ok(None) + } + Instruction::ReturnValue => { + let value = self.pop_value(); + self.unwind_blocks(vm, UnwindReason::Returning { value }) + } + Instruction::SetAdd { i } => { + let item = self.pop_value(); + let obj = self.nth_value(i.get(arg)); + let set: &Py<PySet> = unsafe { + // SAFETY: trust compiler + obj.downcast_unchecked_ref() + }; + set.add(item, vm)?; + Ok(None) + } + Instruction::SetUpdate { i } => { + let iterable = self.pop_value(); + let obj = self.nth_value(i.get(arg)); + let set: &Py<PySet> = unsafe { + // SAFETY: compiler guarantees correct type + obj.downcast_unchecked_ref() + }; + let iter = PyIter::try_from_object(vm, iterable)?; + while let PyIterReturn::Return(item) = iter.next(vm)? { + set.add(item, vm)?; + } + Ok(None) + } + Instruction::PushExcInfo => { + // Stack: [exc] -> [prev_exc, exc] + let exc = self.pop_value(); + let prev_exc = vm + .current_exception() + .map(|e| e.into()) + .unwrap_or_else(|| vm.ctx.none()); + + // Set exc as the current exception + if let Some(exc_ref) = exc.downcast_ref::<PyBaseException>() { + vm.set_exception(Some(exc_ref.to_owned())); + } + + self.push_value(prev_exc); + self.push_value(exc); + Ok(None) + } + Instruction::CheckExcMatch => { + // Stack: [exc, type] -> [exc, bool] + let exc_type = self.pop_value(); + let exc = self.top_value(); + + // Validate that exc_type inherits from BaseException + if let Some(tuple_of_exceptions) = exc_type.downcast_ref::<PyTuple>() { + for exception in tuple_of_exceptions { + if !exception + .is_subclass(vm.ctx.exceptions.base_exception_type.into(), vm)? + { + return Err(vm.new_type_error( + "catching classes that do not inherit from BaseException is not allowed", + )); + } + } + } else if !exc_type.is_subclass(vm.ctx.exceptions.base_exception_type.into(), vm)? { + return Err(vm.new_type_error( + "catching classes that do not inherit from BaseException is not allowed", + )); + } + + let result = exc.is_instance(&exc_type, vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } + Instruction::Reraise { depth } => { + // inst(RERAISE, (values[oparg], exc -- values[oparg])) + // + // Stack layout: [values..., exc] where len(values) == oparg + // RERAISE pops exc and oparg additional values from the stack. + // values[0] is lasti used to set frame->instr_ptr for traceback. + // We skip the lasti update since RustPython's traceback is already correct. + let depth_val = depth.get(arg) as usize; + + // Pop exception from TOS + let exc = self.pop_value(); + + // Pop the depth values (lasti and possibly other items like prev_exc) + for _ in 0..depth_val { + self.pop_value(); + } + + if let Some(exc_ref) = exc.downcast_ref::<PyBaseException>() { + Err(exc_ref.to_owned()) + } else { + // Fallback: use current exception if TOS is not an exception + let exc = vm + .topmost_exception() + .ok_or_else(|| vm.new_runtime_error("No active exception to re-raise"))?; + Err(exc) + } + } + Instruction::SetFunctionAttribute { flag: attr } => { + self.execute_set_function_attribute(vm, attr.get(arg)) + } + Instruction::SetupAnnotations => self.setup_annotations(vm), + Instruction::StoreAttr { namei: idx } => { + let idx_val = idx.get(arg); + self.adaptive(|s, ii, cb| s.specialize_store_attr(vm, idx_val, ii, cb)); + self.store_attr(vm, idx_val) + } + Instruction::StoreDeref { i } => { + let value = self.pop_value(); + self.cell_ref(i.get(arg) as usize).set(Some(value)); + Ok(None) + } + Instruction::StoreFast { var_num } => { + let value = self.pop_value(); + let fastlocals = self.localsplus.fastlocals_mut(); + fastlocals[var_num.get(arg)] = Some(value); + Ok(None) + } + Instruction::StoreFastLoadFast { var_nums } => { + let value = self.pop_value(); + let locals = self.localsplus.fastlocals_mut(); + let oparg = var_nums.get(arg); + let (store_idx, load_idx) = oparg.indexes(); + locals[store_idx] = Some(value); + let load_value = locals[load_idx] + .clone() + .expect("StoreFastLoadFast: load slot should have value after store"); + self.push_value(load_value); + Ok(None) + } + Instruction::StoreFastStoreFast { var_nums } => { + let oparg = var_nums.get(arg); + let (idx1, idx2) = oparg.indexes(); + let value1 = self.pop_value(); + let value2 = self.pop_value(); + let fastlocals = self.localsplus.fastlocals_mut(); + fastlocals[idx1] = Some(value1); + fastlocals[idx2] = Some(value2); + Ok(None) + } + Instruction::StoreGlobal { namei: idx } => { + let value = self.pop_value(); + self.globals + .set_item(self.code.names[idx.get(arg) as usize], value, vm)?; + Ok(None) + } + Instruction::StoreName { namei: idx } => { + let name = self.code.names[idx.get(arg) as usize]; + let value = self.pop_value(); + self.locals + .mapping(vm) + .ass_subscript(name, Some(value), vm)?; + Ok(None) + } + Instruction::StoreSlice => { + // Stack: [value, container, start, stop] -> [] + let stop = self.pop_value(); + let start = self.pop_value(); + let container = self.pop_value(); + let value = self.pop_value(); + let slice: PyObjectRef = PySlice { + start: Some(start), + stop, + step: None, + } + .into_ref(&vm.ctx) + .into(); + container.set_item(&*slice, value, vm)?; + Ok(None) + } + Instruction::StoreSubscr => { + self.adaptive(|s, ii, cb| s.specialize_store_subscr(vm, ii, cb)); + self.execute_store_subscript(vm) + } + Instruction::Swap { i: index } => { + let len = self.localsplus.stack_len(); + debug_assert!(len > 0, "stack underflow in SWAP"); + let i = len - 1; // TOS index + let index_val = index.get(arg) as usize; + // CPython: SWAP(n) swaps TOS with PEEK(n) where PEEK(n) = stack_pointer[-n] + // This means swap TOS with the element at index (len - n) + debug_assert!( + index_val <= len, + "SWAP index {} exceeds stack size {}", + index_val, + len + ); + let j = len - index_val; + self.localsplus.stack_swap(i, j); + Ok(None) + } + Instruction::ToBool => { + self.adaptive(|s, ii, cb| s.specialize_to_bool(vm, ii, cb)); + let obj = self.pop_value(); + let bool_val = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(bool_val).into()); + Ok(None) + } + Instruction::UnpackEx { counts: args } => { + let args = args.get(arg); + self.execute_unpack_ex(vm, args.before, args.after) + } + Instruction::UnpackSequence { count: size } => { + let expected = size.get(arg); + self.adaptive(|s, ii, cb| s.specialize_unpack_sequence(vm, expected, ii, cb)); + self.unpack_sequence(expected, vm) + } + Instruction::WithExceptStart => { + // Stack: [..., __exit__, lasti, prev_exc, exc] + // Call __exit__(type, value, tb) and push result + // __exit__ is at TOS-3 (below lasti, prev_exc, and exc) + let exc = vm.current_exception(); + + let stack_len = self.localsplus.stack_len(); + let exit = expect_unchecked( + self.localsplus.stack_index(stack_len - 4).clone(), + "WithExceptStart: __exit__ is NULL", + ); + + let args = if let Some(ref exc) = exc { + vm.split_exception(exc.clone()) + } else { + (vm.ctx.none(), vm.ctx.none(), vm.ctx.none()) + }; + let exit_res = exit.call(args, vm)?; + // Push result on top of stack + self.push_value(exit_res); + + Ok(None) + } + Instruction::YieldValue { arg: oparg } => { + debug_assert!( + self.localsplus + .stack_as_slice() + .iter() + .flatten() + .all(|sr| !sr.is_borrowed()), + "borrowed refs on stack at yield point" + ); + let value = self.pop_value(); + // arg=0: direct yield (wrapped for async generators) + // arg=1: yield from await/yield-from (NOT wrapped) + let wrap = oparg.get(arg) == 0; + let value = if wrap && self.code.flags.contains(bytecode::CodeFlags::COROUTINE) { + PyAsyncGenWrappedValue(value).into_pyobject(vm) + } else { + value + }; + Ok(Some(ExecutionResult::Yield(value))) + } + Instruction::Send { .. } => { + // (receiver, v -- receiver, retval) + self.adaptive(|s, ii, cb| s.specialize_send(vm, ii, cb)); + let exit_label = bytecode::Label::new(self.lasti() + 1 + u32::from(arg)); + let receiver = self.nth_value(1); + let can_fast_send = !self.specialization_eval_frame_active(vm) + && (receiver.downcast_ref_if_exact::<PyGenerator>(vm).is_some() + || receiver.downcast_ref_if_exact::<PyCoroutine>(vm).is_some()) + && self + .builtin_coro(receiver) + .is_some_and(|coro| !coro.running() && !coro.closed()); + let val = self.pop_value(); + let receiver = self.top_value(); + let ret = if can_fast_send { + let coro = self.builtin_coro(receiver).unwrap(); + if vm.is_none(&val) { + coro.send_none(receiver, vm)? + } else { + coro.send(receiver, val, vm)? + } + } else { + self._send(receiver, val, vm)? + }; + match ret { + PyIterReturn::Return(value) => { + self.push_value(value); + Ok(None) + } + PyIterReturn::StopIteration(value) => { + if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + let stop_exc = vm.new_stop_iteration(value.clone()); + self.fire_exception_trace(&stop_exc, vm)?; + } + let value = vm.unwrap_or_none(value); + self.push_value(value); + self.jump(exit_label); + Ok(None) + } + } + } + Instruction::SendGen => { + let exit_label = bytecode::Label::new(self.lasti() + 1 + u32::from(arg)); + // Stack: [receiver, val] — peek receiver before popping + let receiver = self.nth_value(1); + let can_fast_send = !self.specialization_eval_frame_active(vm) + && (receiver.downcast_ref_if_exact::<PyGenerator>(vm).is_some() + || receiver.downcast_ref_if_exact::<PyCoroutine>(vm).is_some()) + && self + .builtin_coro(receiver) + .is_some_and(|coro| !coro.running() && !coro.closed()); + let val = self.pop_value(); + + if can_fast_send { + let receiver = self.top_value(); + let coro = self.builtin_coro(receiver).unwrap(); + let ret = if vm.is_none(&val) { + coro.send_none(receiver, vm)? + } else { + coro.send(receiver, val, vm)? + }; + match ret { + PyIterReturn::Return(value) => { + self.push_value(value); + return Ok(None); + } + PyIterReturn::StopIteration(value) => { + if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + let stop_exc = vm.new_stop_iteration(value.clone()); + self.fire_exception_trace(&stop_exc, vm)?; + } + let value = vm.unwrap_or_none(value); + self.push_value(value); + self.jump(exit_label); + return Ok(None); + } + } + } + let receiver = self.top_value(); + match self._send(receiver, val, vm)? { + PyIterReturn::Return(value) => { + self.push_value(value); + Ok(None) + } + PyIterReturn::StopIteration(value) => { + if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + let stop_exc = vm.new_stop_iteration(value.clone()); + self.fire_exception_trace(&stop_exc, vm)?; + } + let value = vm.unwrap_or_none(value); + self.push_value(value); + self.jump(exit_label); + Ok(None) + } + } + } + Instruction::EndSend => { + // Stack: (receiver, value) -> (value) + // Pops receiver, leaves value + let value = self.pop_value(); + self.pop_value(); // discard receiver + self.push_value(value); + Ok(None) + } + Instruction::ExitInitCheck => { + // Check that __init__ returned None + let should_be_none = self.pop_value(); + if !vm.is_none(&should_be_none) { + return Err(vm.new_type_error(format!( + "__init__() should return None, not '{}'", + should_be_none.class().name() + ))); + } + Ok(None) + } + Instruction::CleanupThrow => { + // CLEANUP_THROW: (sub_iter, last_sent_val, exc) -> (None, value) OR re-raise + // If StopIteration: pop all 3, extract value, push (None, value) + // Otherwise: pop all 3, return Err(exc) for unwind_blocks to handle + // + // Unlike CPython where exception_unwind pops the triple as part of + // stack cleanup to handler depth, RustPython pops here explicitly + // and lets unwind_blocks find outer handlers. + // Compiler sets handler_depth = base + 2 (before exc is pushed). + + // First peek at exc_value (top of stack) without popping + let exc = self.top_value(); + + // Check if it's a StopIteration + if let Some(exc_ref) = exc.downcast_ref::<PyBaseException>() + && exc_ref.fast_isinstance(vm.ctx.exceptions.stop_iteration) + { + // Extract value from StopIteration + let value = exc_ref.get_arg(0).unwrap_or_else(|| vm.ctx.none()); + // Now pop all three + self.pop_value(); // exc + self.pop_value(); // last_sent_val + self.pop_value(); // sub_iter + self.push_value(vm.ctx.none()); + self.push_value(value); + return Ok(None); + } + + // Re-raise other exceptions: pop all three and return Err(exc) + let exc = self.pop_value(); // exc + self.pop_value(); // last_sent_val + self.pop_value(); // sub_iter + + let exc = exc + .downcast::<PyBaseException>() + .map_err(|_| vm.new_type_error("exception expected"))?; + Err(exc) + } + Instruction::UnaryInvert => { + let a = self.pop_value(); + let value = vm._invert(&a)?; + self.push_value(value); + Ok(None) + } + Instruction::UnaryNegative => { + let a = self.pop_value(); + let value = vm._neg(&a)?; + self.push_value(value); + Ok(None) + } + Instruction::UnaryNot => { + let obj = self.pop_value(); + let value = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(!value).into()); + Ok(None) + } + // Specialized LOAD_ATTR opcodes + Instruction::LoadAttrMethodNoDict => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version + && let Some(func) = self.try_read_cached_descriptor(cache_base, type_version) + { + let owner = self.pop_value(); + self.push_value(func); + self.push_value(owner); + Ok(None) + } else { + self.load_attr_slow(vm, oparg) + } + } + Instruction::LoadAttrMethodLazyDict => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version + && owner.dict().is_none() + && let Some(func) = self.try_read_cached_descriptor(cache_base, type_version) + { + let owner = self.pop_value(); + self.push_value(func); + self.push_value(owner); + Ok(None) + } else { + self.load_attr_slow(vm, oparg) + } + } + Instruction::LoadAttrMethodWithValues => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + let attr_name = self.code.names[oparg.name_idx() as usize]; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version { + // Check instance dict doesn't shadow the method + let shadowed = if let Some(dict) = owner.dict() { + match dict.get_item_opt(attr_name, vm) { + Ok(Some(_)) => true, + Ok(None) => false, + Err(_) => { + // Dict lookup error -> use safe path. + return self.load_attr_slow(vm, oparg); + } + } + } else { + false + }; + + if !shadowed + && let Some(func) = + self.try_read_cached_descriptor(cache_base, type_version) + { + let owner = self.pop_value(); + self.push_value(func); + self.push_value(owner); + return Ok(None); + } + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrInstanceValue => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + let attr_name = self.code.names[oparg.name_idx() as usize]; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version { + // Type version matches — no data descriptor for this attr. + // Try direct dict lookup, skipping full descriptor protocol. + if let Some(dict) = owner.dict() + && let Some(value) = dict.get_item_opt(attr_name, vm)? + { + self.pop_value(); + self.push_value(value); + return Ok(None); + } + // Not in instance dict — fall through to class lookup via slow path + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrWithHint => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + let attr_name = self.code.names[oparg.name_idx() as usize]; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version + && let Some(dict) = owner.dict() + && let Some(value) = dict.get_item_opt(attr_name, vm)? + { + self.pop_value(); + if oparg.is_method() { + self.push_value(value); + self.push_value_opt(None); + } else { + self.push_value(value); + } + return Ok(None); + } + + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrModule => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + let attr_name = self.code.names[oparg.name_idx() as usize]; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version + && let Some(module) = owner.downcast_ref_if_exact::<PyModule>(vm) + && let Ok(value) = module.get_attr(attr_name, vm) + { + self.pop_value(); + if oparg.is_method() { + self.push_value(value); + self.push_value_opt(None); + } else { + self.push_value(value); + } + return Ok(None); + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrNondescriptorNoDict => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version + && let Some(attr) = self.try_read_cached_descriptor(cache_base, type_version) + { + self.pop_value(); + if oparg.is_method() { + self.push_value(attr); + self.push_value_opt(None); + } else { + self.push_value(attr); + } + return Ok(None); + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrNondescriptorWithValues => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + let attr_name = self.code.names[oparg.name_idx() as usize]; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version { + // Instance dict has priority — check if attr is shadowed + if let Some(dict) = owner.dict() + && let Some(value) = dict.get_item_opt(attr_name, vm)? + { + self.pop_value(); + if oparg.is_method() { + self.push_value(value); + self.push_value_opt(None); + } else { + self.push_value(value); + } + return Ok(None); + } + // Not in instance dict — use cached class attr + let Some(attr) = self.try_read_cached_descriptor(cache_base, type_version) + else { + return self.load_attr_slow(vm, oparg); + }; + self.pop_value(); + if oparg.is_method() { + self.push_value(attr); + self.push_value_opt(None); + } else { + self.push_value(attr); + } + return Ok(None); + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrClass => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && let Some(owner_type) = owner.downcast_ref::<PyType>() + && owner_type.tp_version_tag.load(Acquire) == type_version + && let Some(attr) = self.try_read_cached_descriptor(cache_base, type_version) + { + self.pop_value(); + if oparg.is_method() { + self.push_value(attr); + self.push_value_opt(None); + } else { + self.push_value(attr); + } + return Ok(None); + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrClassWithMetaclassCheck => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + let metaclass_version = self.code.instructions.read_cache_u32(cache_base + 3); + + if type_version != 0 + && metaclass_version != 0 + && let Some(owner_type) = owner.downcast_ref::<PyType>() + && owner_type.tp_version_tag.load(Acquire) == type_version + && owner.class().tp_version_tag.load(Acquire) == metaclass_version + && let Some(attr) = self.try_read_cached_descriptor(cache_base, type_version) + { + self.pop_value(); + if oparg.is_method() { + self.push_value(attr); + self.push_value_opt(None); + } else { + self.push_value(attr); + } + return Ok(None); + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrGetattributeOverridden => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + let func_version = self.code.instructions.read_cache_u32(cache_base + 3); + + if !oparg.is_method() + && !self.specialization_eval_frame_active(vm) + && type_version != 0 + && func_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version + && let Some(func_obj) = + self.try_read_cached_descriptor(cache_base, type_version) + && let Some(func) = func_obj.downcast_ref_if_exact::<PyFunction>(vm) + && func.func_version() == func_version + && self.specialization_has_datastack_space_for_func(vm, func) + { + debug_assert!(func.has_exact_argcount(2)); + let owner = self.pop_value(); + let attr_name = self.code.names[oparg.name_idx() as usize].to_owned().into(); + let result = func.invoke_exact_args(vec![owner, attr_name], vm)?; + self.push_value(result); + return Ok(None); + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrSlot => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version { + let slot_offset = + self.code.instructions.read_cache_u32(cache_base + 3) as usize; + if let Some(value) = owner.get_slot(slot_offset) { + self.pop_value(); + if oparg.is_method() { + self.push_value(value); + self.push_value_opt(None); + } else { + self.push_value(value); + } + return Ok(None); + } + // Slot is None → AttributeError (fall through to slow path) + } + self.load_attr_slow(vm, oparg) + } + Instruction::LoadAttrProperty => { + let oparg = LoadAttr::new(u32::from(arg)); + let cache_base = self.lasti() as usize; + + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && !self.specialization_eval_frame_active(vm) + && owner.class().tp_version_tag.load(Acquire) == type_version + && let Some(fget_obj) = + self.try_read_cached_descriptor(cache_base, type_version) + && let Some(func) = fget_obj.downcast_ref_if_exact::<PyFunction>(vm) + && func.can_specialize_call(1) + && self.specialization_has_datastack_space_for_func(vm, func) + { + let owner = self.pop_value(); + let result = func.invoke_exact_args(vec![owner], vm)?; + self.push_value(result); + return Ok(None); + } + self.load_attr_slow(vm, oparg) + } + Instruction::StoreAttrInstanceValue => { + let attr_idx = u32::from(arg); + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let attr_name = self.code.names[attr_idx as usize]; + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version + && let Some(dict) = owner.dict() + { + self.pop_value(); // owner + let value = self.pop_value(); + dict.set_item(attr_name, value, vm)?; + return Ok(None); + } + self.store_attr(vm, attr_idx) + } + Instruction::StoreAttrWithHint => { + let attr_idx = u32::from(arg); + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let attr_name = self.code.names[attr_idx as usize]; + let owner = self.top_value(); + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + + if type_version != 0 + && owner.class().tp_version_tag.load(Acquire) == type_version + && let Some(dict) = owner.dict() + { + self.pop_value(); // owner + let value = self.pop_value(); + dict.set_item(attr_name, value, vm)?; + return Ok(None); + } + self.store_attr(vm, attr_idx) + } + Instruction::StoreAttrSlot => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let type_version = self.code.instructions.read_cache_u32(cache_base + 1); + let version_match = type_version != 0 && { + let owner = self.top_value(); + owner.class().tp_version_tag.load(Acquire) == type_version + }; + + if version_match { + let slot_offset = + self.code.instructions.read_cache_u16(cache_base + 3) as usize; + let owner = self.pop_value(); + let value = self.pop_value(); + owner.set_slot(slot_offset, Some(value)); + return Ok(None); + } + let attr_idx = u32::from(arg); + self.store_attr(vm, attr_idx) + } + Instruction::StoreSubscrListInt => { + // Stack: [value, obj, idx] (TOS=idx, TOS1=obj, TOS2=value) + let idx = self.pop_value(); + let obj = self.pop_value(); + let value = self.pop_value(); + if let Some(list) = obj.downcast_ref_if_exact::<PyList>(vm) + && let Some(int_idx) = idx.downcast_ref_if_exact::<PyInt>(vm) + && let Some(i) = specialization_nonnegative_compact_index(int_idx, vm) + { + let mut vec = list.borrow_vec_mut(); + if i < vec.len() { + vec[i] = value; + return Ok(None); + } + } + obj.set_item(&*idx, value, vm)?; + Ok(None) + } + Instruction::StoreSubscrDict => { + // Stack: [value, obj, idx] (TOS=idx, TOS1=obj, TOS2=value) + let idx = self.pop_value(); + let obj = self.pop_value(); + let value = self.pop_value(); + if let Some(dict) = obj.downcast_ref_if_exact::<PyDict>(vm) { + dict.set_item(&*idx, value, vm)?; + Ok(None) + } else { + obj.set_item(&*idx, value, vm)?; + Ok(None) + } + } + // Specialized BINARY_OP opcodes + Instruction::BinaryOpAddInt => { + self.execute_binary_op_int(vm, |a, b| a + b, bytecode::BinaryOperator::Add) + } + Instruction::BinaryOpSubtractInt => { + self.execute_binary_op_int(vm, |a, b| a - b, bytecode::BinaryOperator::Subtract) + } + Instruction::BinaryOpMultiplyInt => { + self.execute_binary_op_int(vm, |a, b| a * b, bytecode::BinaryOperator::Multiply) + } + Instruction::BinaryOpAddFloat => { + self.execute_binary_op_float(vm, |a, b| a + b, bytecode::BinaryOperator::Add) + } + Instruction::BinaryOpSubtractFloat => { + self.execute_binary_op_float(vm, |a, b| a - b, bytecode::BinaryOperator::Subtract) + } + Instruction::BinaryOpMultiplyFloat => { + self.execute_binary_op_float(vm, |a, b| a * b, bytecode::BinaryOperator::Multiply) + } + Instruction::BinaryOpAddUnicode => { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_str), Some(b_str)) = ( + a.downcast_ref_if_exact::<PyStr>(vm), + b.downcast_ref_if_exact::<PyStr>(vm), + ) { + let result = a_str.as_wtf8().py_add(b_str.as_wtf8()); + self.pop_value(); + self.pop_value(); + self.push_value(result.to_pyobject(vm)); + Ok(None) + } else { + self.execute_bin_op(vm, bytecode::BinaryOperator::Add) + } + } + Instruction::BinaryOpSubscrGetitem => { + let owner = self.nth_value(1); + if !self.specialization_eval_frame_active(vm) + && let Some((func, func_version)) = + owner.class().get_cached_getitem_for_specialization() + && func.func_version() == func_version + && self.specialization_has_datastack_space_for_func(vm, &func) + { + debug_assert!(func.has_exact_argcount(2)); + let sub = self.pop_value(); + let owner = self.pop_value(); + let result = func.invoke_exact_args(vec![owner, sub], vm)?; + self.push_value(result); + return Ok(None); + } + self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) + } + Instruction::BinaryOpExtend => { + let op = self.binary_op_from_arg(arg); + let b = self.top_value(); + let a = self.nth_value(1); + let cache_base = self.lasti() as usize; + if let Some(descr) = self.read_cached_binary_op_extend_descr(cache_base) + && descr.oparg == op + && (descr.guard)(a, b, vm) + && let Some(result) = (descr.action)(a, b, vm) + { + self.pop_value(); + self.pop_value(); + self.push_value(result); + Ok(None) + } else { + self.execute_bin_op(vm, op) + } + } + Instruction::BinaryOpSubscrListInt => { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(list), Some(idx)) = ( + a.downcast_ref_if_exact::<PyList>(vm), + b.downcast_ref_if_exact::<PyInt>(vm), + ) && let Some(i) = specialization_nonnegative_compact_index(idx, vm) + { + let vec = list.borrow_vec(); + if i < vec.len() { + let value = vec.do_get(i); + drop(vec); + self.pop_value(); + self.pop_value(); + self.push_value(value); + return Ok(None); + } + } + self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) + } + Instruction::BinaryOpSubscrTupleInt => { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(tuple), Some(idx)) = ( + a.downcast_ref_if_exact::<PyTuple>(vm), + b.downcast_ref_if_exact::<PyInt>(vm), + ) && let Some(i) = specialization_nonnegative_compact_index(idx, vm) + { + let elements = tuple.as_slice(); + if i < elements.len() { + let value = elements[i].clone(); + self.pop_value(); + self.pop_value(); + self.push_value(value); + return Ok(None); + } + } + self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) + } + Instruction::BinaryOpSubscrDict => { + let b = self.top_value(); + let a = self.nth_value(1); + if let Some(dict) = a.downcast_ref_if_exact::<PyDict>(vm) { + match dict.get_item_opt(b, vm) { + Ok(Some(value)) => { + self.pop_value(); + self.pop_value(); + self.push_value(value); + return Ok(None); + } + Ok(None) => { + let key = self.pop_value(); + self.pop_value(); + return Err(vm.new_key_error(key)); + } + Err(e) => { + return Err(e); + } + } + } + self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) + } + Instruction::BinaryOpSubscrStrInt => { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_str), Some(b_int)) = ( + a.downcast_ref_if_exact::<PyStr>(vm), + b.downcast_ref_if_exact::<PyInt>(vm), + ) && let Some(i) = specialization_nonnegative_compact_index(b_int, vm) + && let Ok(ch) = a_str.getitem_by_index(vm, i as isize) + && ch.is_ascii() + { + let ascii_idx = ch.to_u32() as usize; + self.pop_value(); + self.pop_value(); + self.push_value(vm.ctx.ascii_char_cache[ascii_idx].clone().into()); + return Ok(None); + } + self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) + } + Instruction::BinaryOpSubscrListSlice => { + let b = self.top_value(); + let a = self.nth_value(1); + if a.downcast_ref_if_exact::<PyList>(vm).is_some() + && b.downcast_ref::<PySlice>().is_some() + { + let b_owned = self.pop_value(); + let a_owned = self.pop_value(); + let result = a_owned.get_item(b_owned.as_object(), vm)?; + self.push_value(result); + return Ok(None); + } + self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) + } + Instruction::CallPyExactArgs => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + let nargs: u32 = arg.into(); + if self.specialization_eval_frame_active(vm) { + return self.execute_call_vectorcall(nargs, vm); + } + // Stack: [callable, self_or_null, arg1, ..., argN] + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let callable = self.nth_value(nargs + 1); + if let Some(func) = callable.downcast_ref_if_exact::<PyFunction>(vm) + && func.func_version() == cached_version + && cached_version != 0 + { + let effective_nargs = nargs + u32::from(self_or_null_is_some); + if !func.has_exact_argcount(effective_nargs) { + return self.execute_call_vectorcall(nargs, vm); + } + if !self.specialization_has_datastack_space_for_func(vm, func) { + return self.execute_call_vectorcall(nargs, vm); + } + if self.specialization_call_recursion_guard(vm) { + return self.execute_call_vectorcall(nargs, vm); + } + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs as usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let func = callable.downcast_ref_if_exact::<PyFunction>(vm).unwrap(); + let args = if let Some(self_val) = self_or_null { + let mut all_args = Vec::with_capacity(pos_args.len() + 1); + all_args.push(self_val); + all_args.extend(pos_args); + all_args + } else { + pos_args + }; + let result = func.invoke_exact_args(args, vm)?; + self.push_value(result); + Ok(None) + } else { + self.execute_call_vectorcall(nargs, vm) + } + } + Instruction::CallBoundMethodExactArgs => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + let nargs: u32 = arg.into(); + if self.specialization_eval_frame_active(vm) { + return self.execute_call_vectorcall(nargs, vm); + } + // Stack: [callable, self_or_null(NULL), arg1, ..., argN] + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let callable = self.nth_value(nargs + 1); + if !self_or_null_is_some + && let Some(bound_method) = callable.downcast_ref_if_exact::<PyBoundMethod>(vm) + { + let bound_function = bound_method.function_obj().clone(); + let bound_self = bound_method.self_obj().clone(); + if let Some(func) = bound_function.downcast_ref_if_exact::<PyFunction>(vm) + && func.func_version() == cached_version + && cached_version != 0 + { + if !func.has_exact_argcount(nargs + 1) { + return self.execute_call_vectorcall(nargs, vm); + } + if !self.specialization_has_datastack_space_for_func(vm, func) { + return self.execute_call_vectorcall(nargs, vm); + } + if self.specialization_call_recursion_guard(vm) { + return self.execute_call_vectorcall(nargs, vm); + } + let pos_args: Vec<PyObjectRef> = + self.pop_multiple(nargs as usize).collect(); + self.pop_value_opt(); // null (self_or_null) + self.pop_value(); // callable (bound method) + let mut all_args = Vec::with_capacity(pos_args.len() + 1); + all_args.push(bound_self); + all_args.extend(pos_args); + let result = func.invoke_exact_args(all_args, vm)?; + self.push_value(result); + return Ok(None); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallLen => { + let nargs: u32 = arg.into(); + if nargs == 1 { + // Stack: [callable, null, arg] + let obj = self.pop_value(); // arg + let null = self.pop_value_opt(); + let callable = self.pop_value(); + if null.is_none() + && vm + .callable_cache + .len + .as_ref() + .is_some_and(|len_callable| callable.is(len_callable)) + { + let len = obj.length(vm)?; + self.push_value(vm.ctx.new_int(len).into()); + return Ok(None); + } + // Guard failed — re-push and fallback + self.push_value(callable); + self.push_value_opt(null); + self.push_value(obj); + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallIsinstance => { + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let effective_nargs = nargs + u32::from(self_or_null_is_some); + if effective_nargs == 2 { + let callable = self.nth_value(nargs + 1); + if vm + .callable_cache + .isinstance + .as_ref() + .is_some_and(|isinstance_callable| callable.is(isinstance_callable)) + { + let nargs_usize = nargs as usize; + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + self.pop_value(); // callable + let mut all_args = Vec::with_capacity(2); + if let Some(self_val) = self_or_null { + all_args.push(self_val); + } + all_args.extend(pos_args); + let result = all_args[0].is_instance(&all_args[1], vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + return Ok(None); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallType1 => { + let nargs: u32 = arg.into(); + if nargs == 1 { + // Stack: [callable, null, arg] + let obj = self.pop_value(); + let null = self.pop_value_opt(); + let callable = self.pop_value(); + if null.is_none() && callable.is(vm.ctx.types.type_type.as_object()) { + let tp = obj.class().to_owned().into(); + self.push_value(tp); + return Ok(None); + } + // Guard failed — re-push and fallback + self.push_value(callable); + self.push_value_opt(null); + self.push_value(obj); + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallStr1 => { + let nargs: u32 = arg.into(); + if nargs == 1 { + let obj = self.pop_value(); + let null = self.pop_value_opt(); + let callable = self.pop_value(); + if null.is_none() && callable.is(vm.ctx.types.str_type.as_object()) { + let result = obj.str(vm)?; + self.push_value(result.into()); + return Ok(None); + } + self.push_value(callable); + self.push_value_opt(null); + self.push_value(obj); + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallTuple1 => { + let nargs: u32 = arg.into(); + if nargs == 1 { + let obj = self.pop_value(); + let null = self.pop_value_opt(); + let callable = self.pop_value(); + if null.is_none() && callable.is(vm.ctx.types.tuple_type.as_object()) { + // tuple(x) returns x as-is when x is already an exact tuple + if let Ok(tuple) = obj.clone().downcast_exact::<PyTuple>(vm) { + self.push_value(tuple.into_pyref().into()); + } else { + let elements: Vec<PyObjectRef> = vm.extract_elements_with(&obj, Ok)?; + self.push_value(vm.ctx.new_tuple(elements).into()); + } + return Ok(None); + } + self.push_value(callable); + self.push_value_opt(null); + self.push_value(obj); + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallBuiltinO => { + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let effective_nargs = nargs + u32::from(self_or_null_is_some); + let callable = self.nth_value(nargs + 1); + if let Some(native) = callable.downcast_ref_if_exact::<PyNativeFunction>(vm) { + let call_conv = native.value.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS); + if call_conv == PyMethodFlags::O && effective_nargs == 1 { + let nargs_usize = nargs as usize; + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let mut args_vec = Vec::with_capacity(effective_nargs as usize); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + args_vec.extend(pos_args); + let result = + callable.vectorcall(args_vec, effective_nargs as usize, None, vm)?; + self.push_value(result); + return Ok(None); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallBuiltinFast => { + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let effective_nargs = nargs + u32::from(self_or_null_is_some); + let callable = self.nth_value(nargs + 1); + if let Some(native) = callable.downcast_ref_if_exact::<PyNativeFunction>(vm) { + let call_conv = native.value.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS); + if call_conv == PyMethodFlags::FASTCALL { + let nargs_usize = nargs as usize; + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let mut args_vec = Vec::with_capacity(effective_nargs as usize); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + args_vec.extend(pos_args); + let result = + callable.vectorcall(args_vec, effective_nargs as usize, None, vm)?; + self.push_value(result); + return Ok(None); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallPyGeneral => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + let nargs: u32 = arg.into(); + if self.specialization_eval_frame_active(vm) { + return self.execute_call_vectorcall(nargs, vm); + } + let callable = self.nth_value(nargs + 1); + if let Some(func) = callable.downcast_ref_if_exact::<PyFunction>(vm) + && func.func_version() == cached_version + && cached_version != 0 + { + if self.specialization_call_recursion_guard(vm) { + return self.execute_call_vectorcall(nargs, vm); + } + let nargs_usize = nargs as usize; + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let (args_vec, effective_nargs) = if let Some(self_val) = self_or_null { + let mut v = Vec::with_capacity(nargs_usize + 1); + v.push(self_val); + v.extend(pos_args); + (v, nargs_usize + 1) + } else { + (pos_args, nargs_usize) + }; + let result = + vectorcall_function(&callable, args_vec, effective_nargs, None, vm)?; + self.push_value(result); + Ok(None) + } else { + self.execute_call_vectorcall(nargs, vm) + } + } + Instruction::CallBoundMethodGeneral => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + let nargs: u32 = arg.into(); + if self.specialization_eval_frame_active(vm) { + return self.execute_call_vectorcall(nargs, vm); + } + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let callable = self.nth_value(nargs + 1); + if !self_or_null_is_some + && let Some(bound_method) = callable.downcast_ref_if_exact::<PyBoundMethod>(vm) + { + let bound_function = bound_method.function_obj().clone(); + let bound_self = bound_method.self_obj().clone(); + if let Some(func) = bound_function.downcast_ref_if_exact::<PyFunction>(vm) + && func.func_version() == cached_version + && cached_version != 0 + { + if self.specialization_call_recursion_guard(vm) { + return self.execute_call_vectorcall(nargs, vm); + } + let nargs_usize = nargs as usize; + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + self.pop_value_opt(); // null (self_or_null) + self.pop_value(); // callable (bound method) + let mut args_vec = Vec::with_capacity(nargs_usize + 1); + args_vec.push(bound_self); + args_vec.extend(pos_args); + let result = vectorcall_function( + &bound_function, + args_vec, + nargs_usize + 1, + None, + vm, + )?; + self.push_value(result); + return Ok(None); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallListAppend => { + let nargs: u32 = arg.into(); + if nargs == 1 { + // Stack: [callable, self_or_null, item] + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self.localsplus.stack_index(stack_len - 2).is_some(); + let callable = self.nth_value(2); + let self_is_list = self + .localsplus + .stack_index(stack_len - 2) + .as_ref() + .is_some_and(|obj| obj.downcast_ref::<PyList>().is_some()); + if vm + .callable_cache + .list_append + .as_ref() + .is_some_and(|list_append| callable.is(list_append)) + && self_or_null_is_some + && self_is_list + { + let item = self.pop_value(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + if let Some(list_obj) = self_or_null.as_ref() + && let Some(list) = list_obj.downcast_ref::<PyList>() + { + list.append(item); + // CALL_LIST_APPEND fuses the following POP_TOP. + self.jump_relative_forward( + 1, + Instruction::CallListAppend.cache_entries() as u32, + ); + return Ok(None); + } + self.push_value(callable); + self.push_value_opt(self_or_null); + self.push_value(item); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallMethodDescriptorNoargs => { + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let total_nargs = nargs + u32::from(self_or_null_is_some); + if total_nargs == 1 { + let callable = self.nth_value(nargs + 1); + let self_index = + stack_len - nargs as usize - 1 + usize::from(!self_or_null_is_some); + if let Some(descr) = callable.downcast_ref_if_exact::<PyMethodDescriptor>(vm) + && (descr.method.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS)) + == PyMethodFlags::NOARGS + && self + .localsplus + .stack_index(self_index) + .as_ref() + .is_some_and(|self_obj| self_obj.class().is(descr.objclass)) + { + let func = descr.method.func; + let positional_args: Vec<PyObjectRef> = + self.pop_multiple(nargs as usize).collect(); + let self_or_null = self.pop_value_opt(); + self.pop_value(); // callable + let mut all_args = Vec::with_capacity(total_nargs as usize); + if let Some(self_val) = self_or_null { + all_args.push(self_val); + } + all_args.extend(positional_args); + let args = FuncArgs { + args: all_args, + kwargs: Default::default(), + }; + let result = func(vm, args)?; + self.push_value(result); + return Ok(None); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallMethodDescriptorO => { + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let total_nargs = nargs + u32::from(self_or_null_is_some); + if total_nargs == 2 { + let callable = self.nth_value(nargs + 1); + let self_index = + stack_len - nargs as usize - 1 + usize::from(!self_or_null_is_some); + if let Some(descr) = callable.downcast_ref_if_exact::<PyMethodDescriptor>(vm) + && (descr.method.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS)) + == PyMethodFlags::O + && self + .localsplus + .stack_index(self_index) + .as_ref() + .is_some_and(|self_obj| self_obj.class().is(descr.objclass)) + { + let func = descr.method.func; + let positional_args: Vec<PyObjectRef> = + self.pop_multiple(nargs as usize).collect(); + let self_or_null = self.pop_value_opt(); + self.pop_value(); // callable + let mut all_args = Vec::with_capacity(total_nargs as usize); + if let Some(self_val) = self_or_null { + all_args.push(self_val); + } + all_args.extend(positional_args); + let args = FuncArgs { + args: all_args, + kwargs: Default::default(), + }; + let result = func(vm, args)?; + self.push_value(result); + return Ok(None); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallMethodDescriptorFast => { + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let total_nargs = nargs + u32::from(self_or_null_is_some); + let callable = self.nth_value(nargs + 1); + let self_index = + stack_len - nargs as usize - 1 + usize::from(!self_or_null_is_some); + if total_nargs > 0 + && let Some(descr) = callable.downcast_ref_if_exact::<PyMethodDescriptor>(vm) + && (descr.method.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS)) + == PyMethodFlags::FASTCALL + && self + .localsplus + .stack_index(self_index) + .as_ref() + .is_some_and(|self_obj| self_obj.class().is(descr.objclass)) + { + let func = descr.method.func; + let positional_args: Vec<PyObjectRef> = + self.pop_multiple(nargs as usize).collect(); + let self_or_null = self.pop_value_opt(); + self.pop_value(); // callable + let mut all_args = Vec::with_capacity(total_nargs as usize); + if let Some(self_val) = self_or_null { + all_args.push(self_val); + } + all_args.extend(positional_args); + let args = FuncArgs { + args: all_args, + kwargs: Default::default(), + }; + let result = func(vm, args)?; + self.push_value(result); + return Ok(None); + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallBuiltinClass => { + let nargs: u32 = arg.into(); + let callable = self.nth_value(nargs + 1); + if let Some(cls) = callable.downcast_ref::<PyType>() + && cls.slots.vectorcall.load().is_some() + { + let nargs_usize = nargs as usize; + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let self_is_some = self_or_null.is_some(); + let mut args_vec = Vec::with_capacity(nargs_usize + usize::from(self_is_some)); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + args_vec.extend(pos_args); + let result = callable.vectorcall( + args_vec, + nargs_usize + usize::from(self_is_some), + None, + vm, + )?; + self.push_value(result); + return Ok(None); + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallAllocAndEnterInit => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + let nargs: u32 = arg.into(); + let callable = self.nth_value(nargs + 1); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + if !self.specialization_eval_frame_active(vm) + && !self_or_null_is_some + && cached_version != 0 + && let Some(cls) = callable.downcast_ref::<PyType>() + && cls.tp_version_tag.load(Acquire) == cached_version + && let Some(init_func) = cls.get_cached_init_for_specialization(cached_version) + && let Some(cls_alloc) = cls.slots.alloc.load() + { + // Match CPython's `code->co_framesize + _Py_InitCleanup.co_framesize` + // shape, using RustPython's datastack-backed frame size + // equivalent for the extra shim frame. + let init_cleanup_stack_bytes = + datastack_frame_size_bytes_for_code(&vm.ctx.init_cleanup_code) + .expect("_Py_InitCleanup shim is not a generator/coroutine"); + if !self.specialization_has_datastack_space_for_func_with_extra( + vm, + &init_func, + init_cleanup_stack_bytes, + ) { + return self.execute_call_vectorcall(nargs, vm); + } + // CPython creates `_Py_InitCleanup` + `__init__` frames here. + // Keep the guard conservative and deopt when the effective + // recursion budget for those two frames is not available. + if self.specialization_call_recursion_guard_with_extra_frames(vm, 1) { + return self.execute_call_vectorcall(nargs, vm); + } + // Allocate object directly (tp_new == object.__new__, tp_alloc == generic). + let cls_ref = cls.to_owned(); + let new_obj = cls_alloc(cls_ref, 0, vm)?; + + // Build args: [new_obj, arg1, ..., argN] + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs as usize).collect(); + let _null = self.pop_value_opt(); // self_or_null (None) + let _callable = self.pop_value(); // callable (type) + let result = self + .specialization_run_init_cleanup_shim(new_obj, &init_func, pos_args, vm)?; + self.push_value(result); + return Ok(None); + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallMethodDescriptorFastWithKeywords => { + // Native function interface is uniform regardless of keyword support + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let total_nargs = nargs + u32::from(self_or_null_is_some); + let callable = self.nth_value(nargs + 1); + let self_index = + stack_len - nargs as usize - 1 + usize::from(!self_or_null_is_some); + if total_nargs > 0 + && let Some(descr) = callable.downcast_ref_if_exact::<PyMethodDescriptor>(vm) + && (descr.method.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS)) + == (PyMethodFlags::FASTCALL | PyMethodFlags::KEYWORDS) + && self + .localsplus + .stack_index(self_index) + .as_ref() + .is_some_and(|self_obj| self_obj.class().is(descr.objclass)) + { + let func = descr.method.func; + let positional_args: Vec<PyObjectRef> = + self.pop_multiple(nargs as usize).collect(); + let self_or_null = self.pop_value_opt(); + self.pop_value(); // callable + let mut all_args = Vec::with_capacity(total_nargs as usize); + if let Some(self_val) = self_or_null { + all_args.push(self_val); + } + all_args.extend(positional_args); + let args = FuncArgs { + args: all_args, + kwargs: Default::default(), + }; + let result = func(vm, args)?; + self.push_value(result); + return Ok(None); + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallBuiltinFastWithKeywords => { + // Native function interface is uniform regardless of keyword support + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let effective_nargs = nargs + u32::from(self_or_null_is_some); + let callable = self.nth_value(nargs + 1); + if let Some(native) = callable.downcast_ref_if_exact::<PyNativeFunction>(vm) { + let call_conv = native.value.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS); + if call_conv == (PyMethodFlags::FASTCALL | PyMethodFlags::KEYWORDS) { + let nargs_usize = nargs as usize; + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let mut args_vec = Vec::with_capacity(effective_nargs as usize); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + args_vec.extend(pos_args); + let result = + callable.vectorcall(args_vec, effective_nargs as usize, None, vm)?; + self.push_value(result); + return Ok(None); + } + } + self.execute_call_vectorcall(nargs, vm) + } + Instruction::CallNonPyGeneral => { + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let callable = self.nth_value(nargs + 1); + if callable.downcast_ref_if_exact::<PyFunction>(vm).is_some() + || callable + .downcast_ref_if_exact::<PyBoundMethod>(vm) + .is_some() + { + return self.execute_call_vectorcall(nargs, vm); + } + let nargs_usize = nargs as usize; + let pos_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let mut args_vec = + Vec::with_capacity(nargs_usize + usize::from(self_or_null_is_some)); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + args_vec.extend(pos_args); + let result = callable.vectorcall( + args_vec, + nargs_usize + usize::from(self_or_null_is_some), + None, + vm, + )?; + self.push_value(result); + Ok(None) + } + Instruction::CallKwPy => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + let nargs: u32 = arg.into(); + if self.specialization_eval_frame_active(vm) { + return self.execute_call_kw_vectorcall(nargs, vm); + } + // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] + let callable = self.nth_value(nargs + 2); + if let Some(func) = callable.downcast_ref_if_exact::<PyFunction>(vm) + && func.func_version() == cached_version + && cached_version != 0 + { + if self.specialization_call_recursion_guard(vm) { + return self.execute_call_kw_vectorcall(nargs, vm); + } + let nargs_usize = nargs as usize; + let kwarg_names_obj = self.pop_value(); + let kwarg_names_tuple = kwarg_names_obj + .downcast_ref::<PyTuple>() + .expect("kwarg names should be tuple"); + let kw_count = kwarg_names_tuple.len(); + let all_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let pos_count = nargs_usize - kw_count; + let (args_vec, effective_nargs) = if let Some(self_val) = self_or_null { + let mut v = Vec::with_capacity(nargs_usize + 1); + v.push(self_val); + v.extend(all_args); + (v, pos_count + 1) + } else { + (all_args, pos_count) + }; + let kwnames = kwarg_names_tuple.as_slice(); + let result = vectorcall_function( + &callable, + args_vec, + effective_nargs, + Some(kwnames), + vm, + )?; + self.push_value(result); + return Ok(None); + } + self.execute_call_kw_vectorcall(nargs, vm) + } + Instruction::CallKwBoundMethod => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + let nargs: u32 = arg.into(); + if self.specialization_eval_frame_active(vm) { + return self.execute_call_kw_vectorcall(nargs, vm); + } + // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 2) + .is_some(); + let callable = self.nth_value(nargs + 2); + if !self_or_null_is_some + && let Some(bound_method) = callable.downcast_ref_if_exact::<PyBoundMethod>(vm) + { + let bound_function = bound_method.function_obj().clone(); + let bound_self = bound_method.self_obj().clone(); + if let Some(func) = bound_function.downcast_ref_if_exact::<PyFunction>(vm) + && func.func_version() == cached_version + && cached_version != 0 + { + let nargs_usize = nargs as usize; + let kwarg_names_obj = self.pop_value(); + let kwarg_names_tuple = kwarg_names_obj + .downcast_ref::<PyTuple>() + .expect("kwarg names should be tuple"); + let kw_count = kwarg_names_tuple.len(); + let all_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + self.pop_value_opt(); // null (self_or_null) + self.pop_value(); // callable (bound method) + let pos_count = nargs_usize - kw_count; + let mut args_vec = Vec::with_capacity(nargs_usize + 1); + args_vec.push(bound_self); + args_vec.extend(all_args); + let kwnames = kwarg_names_tuple.as_slice(); + let result = vectorcall_function( + &bound_function, + args_vec, + pos_count + 1, + Some(kwnames), + vm, + )?; + self.push_value(result); + return Ok(None); + } + } + self.execute_call_kw_vectorcall(nargs, vm) + } + Instruction::CallKwNonPy => { + let nargs: u32 = arg.into(); + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 2) + .is_some(); + let callable = self.nth_value(nargs + 2); + if callable.downcast_ref_if_exact::<PyFunction>(vm).is_some() + || callable + .downcast_ref_if_exact::<PyBoundMethod>(vm) + .is_some() + { + return self.execute_call_kw_vectorcall(nargs, vm); + } + let nargs_usize = nargs as usize; + let kwarg_names_obj = self.pop_value(); + let kwarg_names_tuple = kwarg_names_obj + .downcast_ref::<PyTuple>() + .expect("kwarg names should be tuple"); + let kw_count = kwarg_names_tuple.len(); + let all_args: Vec<PyObjectRef> = self.pop_multiple(nargs_usize).collect(); + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + let pos_count = nargs_usize - kw_count; + let mut args_vec = + Vec::with_capacity(nargs_usize + usize::from(self_or_null_is_some)); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + args_vec.extend(all_args); + let result = callable.vectorcall( + args_vec, + pos_count + usize::from(self_or_null_is_some), + Some(kwarg_names_tuple.as_slice()), + vm, + )?; + self.push_value(result); + Ok(None) + } + Instruction::LoadSuperAttrAttr => { + let oparg = u32::from(arg); + let attr_name = self.code.names[(oparg >> 2) as usize]; + // Stack: [global_super, class, self] + let self_obj = self.top_value(); + let class_obj = self.nth_value(1); + let global_super = self.nth_value(2); + // Guard: global_super is builtin super and class is a type + if global_super.is(&vm.ctx.types.super_type.as_object()) + && class_obj.downcast_ref::<PyType>().is_some() + { + let class = class_obj.downcast_ref::<PyType>().unwrap(); + let start_type = self_obj.class(); + // MRO lookup: skip classes up to and including `class`, then search + let mro: Vec<PyRef<PyType>> = start_type.mro_map_collect(|x| x.to_owned()); + let mut found = None; + let mut past_class = false; + for cls in &mro { + if !past_class { + if cls.is(class) { + past_class = true; + } + continue; + } + if let Some(descr) = cls.get_direct_attr(attr_name) { + // Call descriptor __get__ if available + // Pass None for obj when self IS its own type (classmethod) + let obj_arg = if self_obj.is(start_type.as_object()) { + None + } else { + Some(self_obj.to_owned()) + }; + let result = vm + .call_get_descriptor_specific( + &descr, + obj_arg, + Some(start_type.as_object().to_owned()), + ) + .unwrap_or(Ok(descr))?; + found = Some(result); + break; + } + } + if let Some(attr) = found { + self.pop_value(); // self + self.pop_value(); // class + self.pop_value(); // super + self.push_value(attr); + return Ok(None); + } + } + let oparg = LoadSuperAttr::new(oparg); + self.load_super_attr(vm, oparg) + } + Instruction::LoadSuperAttrMethod => { + let oparg = u32::from(arg); + let attr_name = self.code.names[(oparg >> 2) as usize]; + // Stack: [global_super, class, self] + let self_obj = self.top_value(); + let class_obj = self.nth_value(1); + let global_super = self.nth_value(2); + // Guard: global_super is builtin super and class is a type + if global_super.is(&vm.ctx.types.super_type.as_object()) + && class_obj.downcast_ref::<PyType>().is_some() + { + let class = class_obj.downcast_ref::<PyType>().unwrap(); + let self_val = self_obj.to_owned(); + let start_type = self_obj.class(); + // MRO lookup + let mro: Vec<PyRef<PyType>> = start_type.mro_map_collect(|x| x.to_owned()); + let mut found = None; + let mut past_class = false; + for cls in &mro { + if !past_class { + if cls.is(class) { + past_class = true; + } + continue; + } + if let Some(descr) = cls.get_direct_attr(attr_name) { + let descr_cls = descr.class(); + if descr_cls + .slots + .flags + .has_feature(PyTypeFlags::METHOD_DESCRIPTOR) + { + // Method descriptor: push unbound func + self + // CALL will prepend self as first positional arg + found = Some((descr, true)); + } else if let Some(descr_get) = descr_cls.slots.descr_get.load() { + // Has __get__ but not METHOD_DESCRIPTOR: bind it + let bound = descr_get( + descr, + Some(self_val.clone()), + Some(start_type.as_object().to_owned()), + vm, + )?; + found = Some((bound, false)); + } else { + // Plain attribute + found = Some((descr, false)); + } + break; + } + } + if let Some((attr, is_method)) = found { + self.pop_value(); // self + self.pop_value(); // class + self.pop_value(); // super + self.push_value(attr); + if is_method { + self.push_value(self_val); + } else { + self.push_null(); + } + return Ok(None); + } + } + let oparg = LoadSuperAttr::new(oparg); + self.load_super_attr(vm, oparg) + } + Instruction::CompareOpInt => { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_int), Some(b_int)) = ( + a.downcast_ref_if_exact::<PyInt>(vm), + b.downcast_ref_if_exact::<PyInt>(vm), + ) && let (Some(a_val), Some(b_val)) = ( + specialization_compact_int_value(a_int, vm), + specialization_compact_int_value(b_int, vm), + ) { + let op = self.compare_op_from_arg(arg); + let result = op.eval_ord(a_val.cmp(&b_val)); + self.pop_value(); + self.pop_value(); + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } else { + let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) + .unwrap_or(bytecode::ComparisonOperator::Equal); + self.execute_compare(vm, op) + } + } + Instruction::CompareOpFloat => { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_f), Some(b_f)) = ( + a.downcast_ref_if_exact::<PyFloat>(vm), + b.downcast_ref_if_exact::<PyFloat>(vm), + ) { + let op = self.compare_op_from_arg(arg); + let (a, b) = (a_f.to_f64(), b_f.to_f64()); + // Use Rust's IEEE 754 float comparison which handles NaN correctly + let result = match a.partial_cmp(&b) { + Some(ord) => op.eval_ord(ord), + None => op == PyComparisonOp::Ne, // NaN != anything is true + }; + self.pop_value(); + self.pop_value(); + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } else { + let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) + .unwrap_or(bytecode::ComparisonOperator::Equal); + self.execute_compare(vm, op) + } + } + Instruction::CompareOpStr => { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_str), Some(b_str)) = ( + a.downcast_ref_if_exact::<PyStr>(vm), + b.downcast_ref_if_exact::<PyStr>(vm), + ) { + let op = self.compare_op_from_arg(arg); + if op != PyComparisonOp::Eq && op != PyComparisonOp::Ne { + let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) + .unwrap_or(bytecode::ComparisonOperator::Equal); + return self.execute_compare(vm, op); + } + let result = op.eval_ord(a_str.as_wtf8().cmp(b_str.as_wtf8())); + self.pop_value(); + self.pop_value(); + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } else { + let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) + .unwrap_or(bytecode::ComparisonOperator::Equal); + self.execute_compare(vm, op) + } + } + Instruction::ToBoolBool => { + let obj = self.top_value(); + if obj.class().is(vm.ctx.types.bool_type) { + // Already a bool, no-op + Ok(None) + } else { + let obj = self.pop_value(); + let result = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } + } + Instruction::ToBoolInt => { + let obj = self.top_value(); + if let Some(int_val) = obj.downcast_ref_if_exact::<PyInt>(vm) { + let result = !int_val.as_bigint().is_zero(); + self.pop_value(); + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } else { + let obj = self.pop_value(); + let result = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } + } + Instruction::ToBoolNone => { + let obj = self.top_value(); + if obj.class().is(vm.ctx.types.none_type) { + self.pop_value(); + self.push_value(vm.ctx.new_bool(false).into()); + Ok(None) + } else { + let obj = self.pop_value(); + let result = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } + } + Instruction::ToBoolList => { + let obj = self.top_value(); + if let Some(list) = obj.downcast_ref_if_exact::<PyList>(vm) { + let result = !list.borrow_vec().is_empty(); + self.pop_value(); + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } else { + let obj = self.pop_value(); + let result = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } + } + Instruction::ToBoolStr => { + let obj = self.top_value(); + if let Some(s) = obj.downcast_ref_if_exact::<PyStr>(vm) { + let result = !s.is_empty(); + self.pop_value(); + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } else { + let obj = self.pop_value(); + let result = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } + } + Instruction::ToBoolAlwaysTrue => { + // Objects without __bool__ or __len__ are always True. + // Guard: check type version hasn't changed. + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let obj = self.top_value(); + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + if cached_version != 0 && obj.class().tp_version_tag.load(Acquire) == cached_version + { + self.pop_value(); + self.push_value(vm.ctx.new_bool(true).into()); + Ok(None) + } else { + let obj = self.pop_value(); + let result = obj.try_to_bool(vm)?; + self.push_value(vm.ctx.new_bool(result).into()); + Ok(None) + } + } + Instruction::ContainsOpDict => { + let b = self.top_value(); // haystack + if let Some(dict) = b.downcast_ref_if_exact::<PyDict>(vm) { + let a = self.nth_value(1); // needle + let found = dict.get_item_opt(a, vm)?.is_some(); + self.pop_value(); + self.pop_value(); + let invert = bytecode::Invert::try_from(u32::from(arg) as u8) + .unwrap_or(bytecode::Invert::No); + let value = match invert { + bytecode::Invert::No => found, + bytecode::Invert::Yes => !found, + }; + self.push_value(vm.ctx.new_bool(value).into()); + Ok(None) + } else { + let b = self.pop_value(); + let a = self.pop_value(); + let invert = bytecode::Invert::try_from(u32::from(arg) as u8) + .unwrap_or(bytecode::Invert::No); + let value = match invert { + bytecode::Invert::No => self._in(vm, &a, &b)?, + bytecode::Invert::Yes => self._not_in(vm, &a, &b)?, + }; + self.push_value(vm.ctx.new_bool(value).into()); + Ok(None) + } + } + Instruction::ContainsOpSet => { + let b = self.top_value(); // haystack + if b.downcast_ref_if_exact::<PySet>(vm).is_some() + || b.downcast_ref_if_exact::<PyFrozenSet>(vm).is_some() + { + let a = self.nth_value(1); // needle + let found = vm._contains(b, a)?; + self.pop_value(); + self.pop_value(); + let invert = bytecode::Invert::try_from(u32::from(arg) as u8) + .unwrap_or(bytecode::Invert::No); + let value = match invert { + bytecode::Invert::No => found, + bytecode::Invert::Yes => !found, + }; + self.push_value(vm.ctx.new_bool(value).into()); + Ok(None) + } else { + let b = self.pop_value(); + let a = self.pop_value(); + let invert = bytecode::Invert::try_from(u32::from(arg) as u8) + .unwrap_or(bytecode::Invert::No); + let value = match invert { + bytecode::Invert::No => self._in(vm, &a, &b)?, + bytecode::Invert::Yes => self._not_in(vm, &a, &b)?, + }; + self.push_value(vm.ctx.new_bool(value).into()); + Ok(None) + } + } + Instruction::UnpackSequenceTwoTuple => { + let obj = self.top_value(); + if let Some(tuple) = obj.downcast_ref_if_exact::<PyTuple>(vm) { + let elements = tuple.as_slice(); + if elements.len() == 2 { + let e0 = elements[0].clone(); + let e1 = elements[1].clone(); + self.pop_value(); + self.push_value(e1); + self.push_value(e0); + return Ok(None); + } + } + let size = u32::from(arg); + self.unpack_sequence(size, vm) + } + Instruction::UnpackSequenceTuple => { + let size = u32::from(arg) as usize; + let obj = self.top_value(); + if let Some(tuple) = obj.downcast_ref_if_exact::<PyTuple>(vm) { + let elements = tuple.as_slice(); + if elements.len() == size { + let elems: Vec<_> = elements.to_vec(); + self.pop_value(); + for elem in elems.into_iter().rev() { + self.push_value(elem); + } + return Ok(None); + } + } + self.unpack_sequence(size as u32, vm) + } + Instruction::UnpackSequenceList => { + let size = u32::from(arg) as usize; + let obj = self.top_value(); + if let Some(list) = obj.downcast_ref_if_exact::<PyList>(vm) { + let vec = list.borrow_vec(); + if vec.len() == size { + let elems: Vec<_> = vec.to_vec(); + drop(vec); + self.pop_value(); + for elem in elems.into_iter().rev() { + self.push_value(elem); + } + return Ok(None); + } + } + self.unpack_sequence(size as u32, vm) + } + Instruction::ForIterRange => { + let target = bytecode::Label::new(self.lasti() + 1 + u32::from(arg)); + let iter = self.top_value(); + if let Some(range_iter) = iter.downcast_ref_if_exact::<PyRangeIterator>(vm) { + if let Some(value) = range_iter.fast_next() { + self.push_value(vm.ctx.new_int(value).into()); + } else { + self.for_iter_jump_on_exhausted(target); + } + Ok(None) + } else { + self.execute_for_iter(vm, target)?; + Ok(None) + } + } + Instruction::ForIterList => { + let target = bytecode::Label::new(self.lasti() + 1 + u32::from(arg)); + let iter = self.top_value(); + if let Some(list_iter) = iter.downcast_ref_if_exact::<PyListIterator>(vm) { + if let Some(value) = list_iter.fast_next() { + self.push_value(value); + } else { + self.for_iter_jump_on_exhausted(target); + } + Ok(None) + } else { + self.execute_for_iter(vm, target)?; + Ok(None) + } + } + Instruction::ForIterTuple => { + let target = bytecode::Label::new(self.lasti() + 1 + u32::from(arg)); + let iter = self.top_value(); + if let Some(tuple_iter) = iter.downcast_ref_if_exact::<PyTupleIterator>(vm) { + if let Some(value) = tuple_iter.fast_next() { + self.push_value(value); + } else { + self.for_iter_jump_on_exhausted(target); + } + Ok(None) + } else { + self.execute_for_iter(vm, target)?; + Ok(None) + } + } + Instruction::ForIterGen => { + let target = bytecode::Label::new(self.lasti() + 1 + u32::from(arg)); + let iter = self.top_value(); + if self.specialization_eval_frame_active(vm) { + self.execute_for_iter(vm, target)?; + return Ok(None); + } + if let Some(generator) = iter.downcast_ref_if_exact::<PyGenerator>(vm) { + if generator.as_coro().running() || generator.as_coro().closed() { + self.execute_for_iter(vm, target)?; + return Ok(None); + } + match generator.as_coro().send_none(iter, vm) { + Ok(PyIterReturn::Return(value)) => { + self.push_value(value); + } + Ok(PyIterReturn::StopIteration(value)) => { + if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + let stop_exc = vm.new_stop_iteration(value); + self.fire_exception_trace(&stop_exc, vm)?; + } + self.for_iter_jump_on_exhausted(target); + } + Err(e) => return Err(e), + } + Ok(None) + } else { + self.execute_for_iter(vm, target)?; + Ok(None) + } + } + Instruction::LoadGlobalModule => { + let oparg = u32::from(arg); + let cache_base = self.lasti() as usize; + // Keep specialized opcode on guard miss (JUMP_TO_PREDICTED behavior). + let cached_version = self.code.instructions.read_cache_u16(cache_base + 1); + let cached_index = self.code.instructions.read_cache_u16(cache_base + 3); + if let Ok(current_version) = u16::try_from(self.globals.version()) + && cached_version == current_version + { + let name = self.code.names[(oparg >> 1) as usize]; + if let Some(x) = self.globals.get_item_opt_hint(name, cached_index, vm)? { + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); + } + return Ok(None); + } + } + let name = self.code.names[(oparg >> 1) as usize]; + let x = self.load_global_or_builtin(name, vm)?; + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); + } + Ok(None) + } + Instruction::LoadGlobalBuiltin => { + let oparg = u32::from(arg); + let cache_base = self.lasti() as usize; + let cached_globals_ver = self.code.instructions.read_cache_u16(cache_base + 1); + let cached_builtins_ver = self.code.instructions.read_cache_u16(cache_base + 2); + let cached_index = self.code.instructions.read_cache_u16(cache_base + 3); + if let Ok(current_globals_ver) = u16::try_from(self.globals.version()) + && cached_globals_ver == current_globals_ver + && let Some(builtins_dict) = self.builtins.downcast_ref_if_exact::<PyDict>(vm) + && let Ok(current_builtins_ver) = u16::try_from(builtins_dict.version()) + && cached_builtins_ver == current_builtins_ver + { + let name = self.code.names[(oparg >> 1) as usize]; + if let Some(x) = builtins_dict.get_item_opt_hint(name, cached_index, vm)? { + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); + } + return Ok(None); + } + } + let name = self.code.names[(oparg >> 1) as usize]; + let x = self.load_global_or_builtin(name, vm)?; + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); + } + Ok(None) + } + // All INSTRUMENTED_* opcodes delegate to a cold function to keep + // the hot instruction loop free of monitoring overhead. + _ => self.execute_instrumented(instruction, arg, vm), + } + } + + /// Handle all INSTRUMENTED_* opcodes. This function is cold — it only + /// runs when sys.monitoring has rewritten the bytecode. + #[cold] + fn execute_instrumented( + &mut self, + instruction: Instruction, + arg: bytecode::OpArg, + vm: &VirtualMachine, + ) -> FrameResult { + debug_assert!( + instruction.is_instrumented(), + "execute_instrumented called with non-instrumented opcode {instruction:?}" + ); + if self.monitoring_disabled_for_code(vm) { + let global_ver = vm + .state + .instrumentation_version + .load(atomic::Ordering::Acquire); + monitoring::instrument_code(self.code, 0); + self.code + .instrumentation_version + .store(global_ver, atomic::Ordering::Release); + self.update_lasti(|i| *i -= 1); + return Ok(None); + } + self.monitoring_mask = vm.state.monitoring_events.load(); + match instruction { + Instruction::InstrumentedResume => { + // Version check: re-instrument if stale + let global_ver = vm + .state + .instrumentation_version + .load(atomic::Ordering::Acquire); + let code_ver = self + .code + .instrumentation_version + .load(atomic::Ordering::Acquire); + if code_ver != global_ver { + let events = { + let state = vm.state.monitoring.lock(); + state.events_for_code(self.code.get_id()) + }; + monitoring::instrument_code(self.code, events); + self.code + .instrumentation_version + .store(global_ver, atomic::Ordering::Release); + // Re-execute (may have been de-instrumented to base Resume) + self.update_lasti(|i| *i -= 1); + return Ok(None); + } + let resume_type = u32::from(arg); + let offset = (self.lasti() - 1) * 2; + if resume_type == 0 { + if self.monitoring_mask & monitoring::EVENT_PY_START != 0 { + monitoring::fire_py_start(vm, self.code, offset)?; + } + } else if self.monitoring_mask & monitoring::EVENT_PY_RESUME != 0 { + monitoring::fire_py_resume(vm, self.code, offset)?; + } + Ok(None) + } + Instruction::InstrumentedReturnValue => { + let value = self.pop_value(); + if self.monitoring_mask & monitoring::EVENT_PY_RETURN != 0 { + let offset = (self.lasti() - 1) * 2; + monitoring::fire_py_return(vm, self.code, offset, &value)?; + } + self.unwind_blocks(vm, UnwindReason::Returning { value }) + } + Instruction::InstrumentedYieldValue => { + debug_assert!( + self.localsplus + .stack_as_slice() + .iter() + .flatten() + .all(|sr| !sr.is_borrowed()), + "borrowed refs on stack at yield point" + ); + let value = self.pop_value(); + if self.monitoring_mask & monitoring::EVENT_PY_YIELD != 0 { + let offset = (self.lasti() - 1) * 2; + monitoring::fire_py_yield(vm, self.code, offset, &value)?; + } + let oparg = u32::from(arg); + let wrap = oparg == 0; + let value = if wrap && self.code.flags.contains(bytecode::CodeFlags::COROUTINE) { + PyAsyncGenWrappedValue(value).into_pyobject(vm) + } else { + value + }; + Ok(Some(ExecutionResult::Yield(value))) + } + Instruction::InstrumentedCall => { + let args = self.collect_positional_args(u32::from(arg)); + self.execute_call_instrumented(args, vm) + } + Instruction::InstrumentedCallKw => { + let args = self.collect_keyword_args(u32::from(arg)); + self.execute_call_instrumented(args, vm) + } + Instruction::InstrumentedCallFunctionEx => { + let args = self.collect_ex_args(vm)?; + self.execute_call_instrumented(args, vm) + } + Instruction::InstrumentedLoadSuperAttr => { + let oparg = bytecode::LoadSuperAttr::from(u32::from(arg)); + let offset = (self.lasti() - 1) * 2; + // Fire CALL event before super() call + let call_args = if self.monitoring_mask & monitoring::EVENT_CALL != 0 { + let global_super: PyObjectRef = self.nth_value(2).to_owned(); + let arg0 = if oparg.has_class() { + self.nth_value(1).to_owned() + } else { + monitoring::get_missing(vm) + }; + monitoring::fire_call(vm, self.code, offset, &global_super, arg0.clone())?; + Some((global_super, arg0)) + } else { + None + }; + match self.load_super_attr(vm, oparg) { + Ok(result) => { + // Fire C_RETURN on success + if let Some((global_super, arg0)) = call_args { + monitoring::fire_c_return(vm, self.code, offset, &global_super, arg0)?; + } + Ok(result) + } + Err(exc) => { + // Fire C_RAISE on failure + let exc = if let Some((global_super, arg0)) = call_args { + match monitoring::fire_c_raise( + vm, + self.code, + offset, + &global_super, + arg0, + ) { + Ok(()) => exc, + Err(monitor_exc) => monitor_exc, + } + } else { + exc + }; + Err(exc) + } + } + } + Instruction::InstrumentedJumpForward => { + let src_offset = (self.lasti() - 1) * 2; + let target_idx = self.lasti() + u32::from(arg); + let target = bytecode::Label::new(target_idx); + self.jump(target); + if self.monitoring_mask & monitoring::EVENT_JUMP != 0 { + monitoring::fire_jump(vm, self.code, src_offset, target.as_u32() * 2)?; + } + Ok(None) + } + Instruction::InstrumentedJumpBackward => { + let src_offset = (self.lasti() - 1) * 2; + let target_idx = self.lasti() + 1 - u32::from(arg); + let target = bytecode::Label::new(target_idx); + self.jump(target); + if self.monitoring_mask & monitoring::EVENT_JUMP != 0 { + monitoring::fire_jump(vm, self.code, src_offset, target.as_u32() * 2)?; + } + Ok(None) + } + Instruction::InstrumentedForIter => { + let src_offset = (self.lasti() - 1) * 2; + let target = bytecode::Label::new(self.lasti() + 1 + u32::from(arg)); + let continued = self.execute_for_iter(vm, target)?; + if continued { + if self.monitoring_mask & monitoring::EVENT_BRANCH_LEFT != 0 { + let dest_offset = (self.lasti() + 1) * 2; // after caches + monitoring::fire_branch_left(vm, self.code, src_offset, dest_offset)?; + } + } else if self.monitoring_mask & monitoring::EVENT_BRANCH_RIGHT != 0 { + let dest_offset = self.lasti() * 2; + monitoring::fire_branch_right(vm, self.code, src_offset, dest_offset)?; + } + Ok(None) + } + Instruction::InstrumentedEndFor => { + // Stack: [value, receiver(iter), ...] + // PyGen_Check: only fire STOP_ITERATION for generators + let is_gen = self + .nth_value(1) + .downcast_ref::<crate::builtins::PyGenerator>() + .is_some(); + let value = self.pop_value(); + if is_gen && self.monitoring_mask & monitoring::EVENT_STOP_ITERATION != 0 { + let offset = (self.lasti() - 1) * 2; + monitoring::fire_stop_iteration(vm, self.code, offset, &value)?; + } + Ok(None) + } + Instruction::InstrumentedEndSend => { + let value = self.pop_value(); + let receiver = self.pop_value(); + // PyGen_Check || PyCoro_CheckExact + let is_gen_or_coro = receiver + .downcast_ref::<crate::builtins::PyGenerator>() + .is_some() + || receiver + .downcast_ref::<crate::builtins::PyCoroutine>() + .is_some(); + if is_gen_or_coro && self.monitoring_mask & monitoring::EVENT_STOP_ITERATION != 0 { + let offset = (self.lasti() - 1) * 2; + monitoring::fire_stop_iteration(vm, self.code, offset, &value)?; + } + self.push_value(value); + Ok(None) + } + Instruction::InstrumentedPopJumpIfTrue => { + let src_offset = (self.lasti() - 1) * 2; + let target_idx = self.lasti() + 1 + u32::from(arg); + let obj = self.pop_value(); + let value = obj.try_to_bool(vm)?; + if value { + self.jump(bytecode::Label::new(target_idx)); + if self.monitoring_mask & monitoring::EVENT_BRANCH_RIGHT != 0 { + monitoring::fire_branch_right(vm, self.code, src_offset, target_idx * 2)?; + } + } + Ok(None) + } + Instruction::InstrumentedPopJumpIfFalse => { + let src_offset = (self.lasti() - 1) * 2; + let target_idx = self.lasti() + 1 + u32::from(arg); + let obj = self.pop_value(); + let value = obj.try_to_bool(vm)?; + if !value { + self.jump(bytecode::Label::new(target_idx)); + if self.monitoring_mask & monitoring::EVENT_BRANCH_RIGHT != 0 { + monitoring::fire_branch_right(vm, self.code, src_offset, target_idx * 2)?; + } + } + Ok(None) + } + Instruction::InstrumentedPopJumpIfNone => { + let src_offset = (self.lasti() - 1) * 2; + let target_idx = self.lasti() + 1 + u32::from(arg); + let value = self.pop_value(); + if vm.is_none(&value) { + self.jump(bytecode::Label::new(target_idx)); + if self.monitoring_mask & monitoring::EVENT_BRANCH_RIGHT != 0 { + monitoring::fire_branch_right(vm, self.code, src_offset, target_idx * 2)?; + } + } + Ok(None) + } + Instruction::InstrumentedPopJumpIfNotNone => { + let src_offset = (self.lasti() - 1) * 2; + let target_idx = self.lasti() + 1 + u32::from(arg); + let value = self.pop_value(); + if !vm.is_none(&value) { + self.jump(bytecode::Label::new(target_idx)); + if self.monitoring_mask & monitoring::EVENT_BRANCH_RIGHT != 0 { + monitoring::fire_branch_right(vm, self.code, src_offset, target_idx * 2)?; + } + } + Ok(None) + } + Instruction::InstrumentedNotTaken => { + if self.monitoring_mask & monitoring::EVENT_BRANCH_LEFT != 0 { + let not_taken_idx = self.lasti() as usize - 1; + // Scan backwards past CACHE entries to find the branch instruction + let mut branch_idx = not_taken_idx.saturating_sub(1); + while branch_idx > 0 + && matches!( + self.code.instructions.read_op(branch_idx), + Instruction::Cache + ) + { + branch_idx -= 1; + } + let src_offset = (branch_idx as u32) * 2; + let dest_offset = self.lasti() * 2; + monitoring::fire_branch_left(vm, self.code, src_offset, dest_offset)?; + } + Ok(None) + } + Instruction::InstrumentedPopIter => { + // BRANCH_RIGHT is fired by InstrumentedForIter, not here. + self.pop_value(); + Ok(None) + } + Instruction::InstrumentedEndAsyncFor => { + if self.monitoring_mask & monitoring::EVENT_BRANCH_RIGHT != 0 { + let oparg_val = u32::from(arg); + // src = next_instr - oparg (END_SEND position) + let src_offset = (self.lasti() - oparg_val) * 2; + // dest = this_instr + 1 + let dest_offset = self.lasti() * 2; + monitoring::fire_branch_right(vm, self.code, src_offset, dest_offset)?; + } + let exc = self.pop_value(); + let _awaitable = self.pop_value(); + let exc = exc + .downcast::<PyBaseException>() + .expect("EndAsyncFor expects exception on stack"); + if exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) { + vm.set_exception(None); + Ok(None) + } else { + Err(exc) + } + } + Instruction::InstrumentedLine => { + let idx = self.lasti() as usize - 1; + let offset = idx as u32 * 2; + + // Read the full side-table chain before firing any events, + // because a callback may de-instrument and clear the tables. + let (real_op_byte, also_instruction) = { + let data = self.code.monitoring_data.lock(); + let line_op = data.as_ref().map(|d| d.line_opcodes[idx]).unwrap_or(0); + if line_op == u8::from(Instruction::InstrumentedInstruction) { + // LINE wraps INSTRUCTION: resolve the INSTRUCTION side-table too + let inst_op = data + .as_ref() + .map(|d| d.per_instruction_opcodes[idx]) + .unwrap_or(0); + (inst_op, true) + } else { + (line_op, false) + } + }; + debug_assert!( + real_op_byte != 0, + "INSTRUMENTED_LINE at {idx} without stored opcode" + ); + + // Fire LINE event only if line changed + if let Some((loc, _)) = self.code.locations.get(idx) { + let line = loc.line.get() as u32; + if line != *self.prev_line && line > 0 { + *self.prev_line = line; + monitoring::fire_line(vm, self.code, offset, line)?; + } + } + + // If the LINE position also had INSTRUCTION, fire that event too + if also_instruction { + monitoring::fire_instruction(vm, self.code, offset)?; + } + + // Re-dispatch to the real original opcode + let original_op = Instruction::try_from(real_op_byte) + .expect("invalid opcode in side-table chain"); + let lasti_before_dispatch = self.lasti(); + let result = if original_op.to_base().is_some() { + self.execute_instrumented(original_op, arg, vm) + } else { + let mut do_extend_arg = false; + self.execute_instruction(original_op, arg, &mut do_extend_arg, vm) + }; + let orig_caches = original_op.to_base().unwrap_or(original_op).cache_entries(); + if orig_caches > 0 && self.lasti() == lasti_before_dispatch { + self.update_lasti(|i| *i += orig_caches as u32); + } + result + } + Instruction::InstrumentedInstruction => { + let idx = self.lasti() as usize - 1; + let offset = idx as u32 * 2; + + // Get original opcode from side-table + let original_op_byte = { + let data = self.code.monitoring_data.lock(); + data.as_ref() + .map(|d| d.per_instruction_opcodes[idx]) + .unwrap_or(0) + }; + debug_assert!( + original_op_byte != 0, + "INSTRUMENTED_INSTRUCTION at {idx} without stored opcode" + ); + + // Fire INSTRUCTION event + monitoring::fire_instruction(vm, self.code, offset)?; + + // Re-dispatch to original opcode + let original_op = Instruction::try_from(original_op_byte) + .expect("invalid opcode in instruction side-table"); + let lasti_before_dispatch = self.lasti(); + let result = if original_op.to_base().is_some() { + self.execute_instrumented(original_op, arg, vm) + } else { + let mut do_extend_arg = false; + self.execute_instruction(original_op, arg, &mut do_extend_arg, vm) + }; + let orig_caches = original_op.to_base().unwrap_or(original_op).cache_entries(); + if orig_caches > 0 && self.lasti() == lasti_before_dispatch { + self.update_lasti(|i| *i += orig_caches as u32); + } + result + } + _ => { + unreachable!("{instruction:?} instruction should not be executed") + } + } + } + + #[inline] + fn load_global_or_builtin(&self, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + if let Some(builtins_dict) = self.builtins_dict { + // Fast path: both globals and builtins are exact dicts + // SAFETY: builtins_dict is only set when globals is also exact dict + let globals_exact = unsafe { PyExact::ref_unchecked(self.globals.as_ref()) }; + globals_exact + .get_chain_exact(builtins_dict, name, vm)? + .ok_or_else(|| { + vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) + }) + } else { + // Slow path: builtins is not a dict, use generic __getitem__ + if let Some(value) = self.globals.get_item_opt(name, vm)? { + return Ok(value); + } + self.builtins.get_item(name, vm).map_err(|e| { + if e.fast_isinstance(vm.ctx.exceptions.key_error) { + vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) + } else { + e + } + }) + } + } + + #[cfg_attr(feature = "flame-it", flame("Frame"))] + fn import(&mut self, vm: &VirtualMachine, module_name: Option<&Py<PyStr>>) -> PyResult<()> { + let module_name = module_name.unwrap_or(vm.ctx.empty_str); + let top = self.pop_value(); + let from_list = match <Option<PyTupleRef>>::try_from_object(vm, top)? { + Some(from_list) => from_list.try_into_typed::<PyStr>(vm)?, + None => vm.ctx.empty_tuple_typed().to_owned(), + }; + let level = usize::try_from_object(vm, self.pop_value())?; + + let module = vm.import_from(module_name, &from_list, level)?; + + self.push_value(module); + Ok(()) + } + + #[cfg_attr(feature = "flame-it", flame("Frame"))] + fn import_from(&mut self, vm: &VirtualMachine, idx: bytecode::NameIdx) -> PyResult { + let module = self.top_value(); + let name = self.code.names[idx as usize]; + + // Load attribute, and transform any error into import error. + if let Some(obj) = vm.get_attribute_opt(module.to_owned(), name)? { + return Ok(obj); + } + // fallback to importing '{module.__name__}.{name}' from sys.modules + let fallback_module = (|| { + let mod_name = module.get_attr(identifier!(vm, __name__), vm).ok()?; + let mod_name = mod_name.downcast_ref::<PyStr>()?; + let full_mod_name = format!("{mod_name}.{name}"); + let sys_modules = vm.sys_module.get_attr("modules", vm).ok()?; + sys_modules.get_item(&full_mod_name, vm).ok() + })(); + + if let Some(sub_module) = fallback_module { + return Ok(sub_module); + } + + use crate::import::{ + get_spec_file_origin, is_possibly_shadowing_path, is_stdlib_module_name, + }; + + // Get module name for the error message + let mod_name_obj = module.get_attr(identifier!(vm, __name__), vm).ok(); + let mod_name_str = mod_name_obj + .as_ref() + .and_then(|n| n.downcast_ref::<PyUtf8Str>().map(|s| s.as_str().to_owned())); + let module_name = mod_name_str.as_deref().unwrap_or("<unknown module name>"); + + let spec = module + .get_attr("__spec__", vm) + .ok() + .filter(|s| !vm.is_none(s)); + + let origin = get_spec_file_origin(&spec, vm); + + let is_possibly_shadowing = origin + .as_ref() + .map(|o| is_possibly_shadowing_path(o, vm)) + .unwrap_or(false); + let is_possibly_shadowing_stdlib = if is_possibly_shadowing { + if let Some(ref mod_name) = mod_name_obj { + is_stdlib_module_name(mod_name, vm)? + } else { + false + } + } else { + false + }; + + let msg = if is_possibly_shadowing_stdlib { + let origin = origin.as_ref().unwrap(); + format!( + "cannot import name '{name}' from '{module_name}' \ + (consider renaming '{origin}' since it has the same \ + name as the standard library module named '{module_name}' \ + and prevents importing that standard library module)" + ) + } else { + let is_init = is_module_initializing(module, vm); + if is_init { + if is_possibly_shadowing { + let origin = origin.as_ref().unwrap(); + format!( + "cannot import name '{name}' from '{module_name}' \ + (consider renaming '{origin}' if it has the same name \ + as a library you intended to import)" + ) + } else if let Some(ref path) = origin { + format!( + "cannot import name '{name}' from partially initialized module \ + '{module_name}' (most likely due to a circular import) ({path})" + ) + } else { + format!( + "cannot import name '{name}' from partially initialized module \ + '{module_name}' (most likely due to a circular import)" + ) + } + } else if let Some(ref path) = origin { + format!("cannot import name '{name}' from '{module_name}' ({path})") + } else { + format!("cannot import name '{name}' from '{module_name}' (unknown location)") + } + }; + let err = vm.new_import_error(msg, vm.ctx.new_utf8_str(module_name)); + + if let Some(ref path) = origin { + let _ignore = err + .as_object() + .set_attr("path", vm.ctx.new_str(path.as_str()), vm); + } + + // name_from = the attribute name that failed to import (best-effort metadata) + let _ignore = err.as_object().set_attr("name_from", name.to_owned(), vm); + + Err(err) + } + + #[cfg_attr(feature = "flame-it", flame("Frame"))] + fn import_star(&mut self, vm: &VirtualMachine) -> PyResult<()> { + let module = self.pop_value(); + + let Some(dict) = module.dict() else { + return Ok(()); + }; + + let mod_name = module + .get_attr(identifier!(vm, __name__), vm) + .ok() + .and_then(|n| n.downcast::<PyStr>().ok()); + + let require_str = |obj: PyObjectRef, attr: &str| -> PyResult<PyRef<PyStr>> { + obj.downcast().map_err(|obj: PyObjectRef| { + let source = if let Some(ref mod_name) = mod_name { + format!("{}.{attr}", mod_name.as_wtf8()) + } else { + attr.to_owned() + }; + let repr = obj.repr(vm).unwrap_or_else(|_| vm.ctx.new_str("?")); + vm.new_type_error(format!( + "{} in {} must be str, not {}", + repr.as_wtf8(), + source, + obj.class().name() + )) + }) + }; + + let locals_map = self.locals.mapping(vm); + if let Ok(all) = dict.get_item(identifier!(vm, __all__), vm) { + let items: Vec<PyObjectRef> = all.try_to_value(vm)?; + for item in items { + let name = require_str(item, "__all__")?; + let value = module.get_attr(&*name, vm)?; + locals_map.ass_subscript(&name, Some(value), vm)?; + } + } else { + for (k, v) in dict { + let k = require_str(k, "__dict__")?; + if !k.as_bytes().starts_with(b"_") { + locals_map.ass_subscript(&k, Some(v), vm)?; + } + } + } + Ok(()) + } + + /// Unwind blocks. + /// The reason for unwinding gives a hint on what to do when + /// unwinding a block. + /// Optionally returns an exception. + #[cfg_attr(feature = "flame-it", flame("Frame"))] + fn unwind_blocks(&mut self, vm: &VirtualMachine, reason: UnwindReason) -> FrameResult { + // use exception table for exception handling + match reason { + UnwindReason::Raising { exception } => { + // Look up handler in exception table + // lasti points to NEXT instruction (already incremented in run loop) + // The exception occurred at the previous instruction + // Python uses signed int where INSTR_OFFSET() - 1 = -1 before first instruction. + // We use u32, so check for 0 explicitly. + if self.lasti() == 0 { + // No instruction executed yet, no handler can match + return Err(exception); + } + let offset = self.lasti() - 1; + if let Some(entry) = + bytecode::find_exception_handler(&self.code.exceptiontable, offset) + { + // Fire EXCEPTION_HANDLED before setting up handler. + // If the callback raises, the handler is NOT set up and the + // new exception propagates instead. + if vm.state.monitoring_events.load() & monitoring::EVENT_EXCEPTION_HANDLED != 0 + { + let byte_offset = offset * 2; + let exc_obj: PyObjectRef = exception.clone().into(); + monitoring::fire_exception_handled(vm, self.code, byte_offset, &exc_obj)?; + } + + // 1. Pop stack to entry.depth + while self.localsplus.stack_len() > entry.depth as usize { + let _ = self.localsplus.stack_pop(); + } + + // 2. If push_lasti=true (SETUP_CLEANUP), push lasti before exception + // pushes lasti as PyLong + if entry.push_lasti { + self.push_value(vm.ctx.new_int(offset as i32).into()); + } + + // 3. Push exception onto stack + // always push exception, PUSH_EXC_INFO transforms [exc] -> [prev_exc, exc] + // Do NOT call vm.set_exception here! PUSH_EXC_INFO will do it. + // PUSH_EXC_INFO needs to get prev_exc from vm.current_exception() BEFORE setting the new one. + self.push_value(exception.into()); + + // 4. Jump to handler + self.jump(bytecode::Label::new(entry.target)); + + Ok(None) + } else { + // No handler found, propagate exception + Err(exception) + } + } + UnwindReason::Returning { value } => Ok(Some(ExecutionResult::Return(value))), + } + } + + fn execute_store_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { + let idx = self.pop_value(); + let obj = self.pop_value(); + let value = self.pop_value(); + obj.set_item(&*idx, value, vm)?; + Ok(None) + } + + fn execute_delete_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { + let idx = self.pop_value(); + let obj = self.pop_value(); + obj.del_item(&*idx, vm)?; + Ok(None) + } + + fn execute_build_map(&mut self, vm: &VirtualMachine, size: u32) -> FrameResult { + let size = size as usize; + let map_obj = vm.ctx.new_dict(); + for (key, value) in self.pop_multiple(2 * size).tuples() { + map_obj.set_item(&*key, value, vm)?; + } + + self.push_value(map_obj.into()); + Ok(None) + } + + fn execute_build_slice( + &mut self, + vm: &VirtualMachine, + argc: bytecode::BuildSliceArgCount, + ) -> FrameResult { + let step = match argc { + bytecode::BuildSliceArgCount::Two => None, + bytecode::BuildSliceArgCount::Three => Some(self.pop_value()), + }; + let stop = self.pop_value(); + let start = self.pop_value(); + + let obj = PySlice { + start: Some(start), + stop, + step, + } + .into_ref(&vm.ctx); + self.push_value(obj.into()); + Ok(None) + } + + fn collect_positional_args(&mut self, nargs: u32) -> FuncArgs { + FuncArgs { + args: self.pop_multiple(nargs as usize).collect(), + kwargs: IndexMap::new(), + } + } + + fn collect_keyword_args(&mut self, nargs: u32) -> FuncArgs { + let kwarg_names = self + .pop_value() + .downcast::<PyTuple>() + .expect("kwarg names should be tuple of strings"); + let args = self.pop_multiple(nargs as usize); + + let kwarg_names = kwarg_names.as_slice().iter().map(|pyobj| { + pyobj + .downcast_ref::<PyUtf8Str>() + .unwrap() + .as_str() + .to_owned() + }); + FuncArgs::with_kwargs_names(args, kwarg_names) + } + + fn collect_ex_args(&mut self, vm: &VirtualMachine) -> PyResult<FuncArgs> { + let kwargs_or_null = self.pop_value_opt(); + let kwargs = if let Some(kw_obj) = kwargs_or_null { + let mut kwargs = IndexMap::new(); + + // Stack: [callable, self_or_null, args_tuple] + let callable = self.nth_value(2); + let func_str = Self::object_function_str(callable, vm); + + Self::iterate_mapping_keys(vm, &kw_obj, &func_str, |key| { + let key_str = key + .downcast_ref::<PyUtf8Str>() + .ok_or_else(|| vm.new_type_error("keywords must be strings"))?; + let value = kw_obj.get_item(&*key, vm)?; + kwargs.insert(key_str.as_str().to_owned(), value); + Ok(()) + })?; + kwargs + } else { + IndexMap::new() + }; + let args_obj = self.pop_value(); + let args = if let Some(tuple) = args_obj.downcast_ref::<PyTuple>() { + tuple.as_slice().to_vec() + } else { + // Single *arg passed directly; convert to sequence at runtime. + // Stack: [callable, self_or_null] + let callable = self.nth_value(1); + let func_str = Self::object_function_str(callable, vm); + let not_iterable = args_obj.class().slots.iter.load().is_none() + && args_obj + .get_class_attr(vm.ctx.intern_str("__getitem__")) + .is_none(); + args_obj.try_to_value::<Vec<PyObjectRef>>(vm).map_err(|e| { + if not_iterable && e.class().is(vm.ctx.exceptions.type_error) { + vm.new_type_error(format!( + "{} argument after * must be an iterable, not {}", + func_str, + args_obj.class().name() + )) + } else { + e + } + })? + }; + Ok(FuncArgs { args, kwargs }) + } + + /// Returns a display string for a callable object for use in error messages. + /// For objects with `__qualname__`, returns "module.qualname()" or "qualname()". + /// For other objects, returns repr(obj). + fn object_function_str(obj: &PyObject, vm: &VirtualMachine) -> Wtf8Buf { + let repr_fallback = || { + obj.repr(vm) + .as_ref() + .map_or("?".as_ref(), |s| s.as_wtf8()) + .to_owned() + }; + let Ok(qualname) = obj.get_attr(vm.ctx.intern_str("__qualname__"), vm) else { + return repr_fallback(); + }; + let Some(qualname_str) = qualname.downcast_ref::<PyStr>() else { + return repr_fallback(); + }; + if let Ok(module) = obj.get_attr(vm.ctx.intern_str("__module__"), vm) + && let Some(module_str) = module.downcast_ref::<PyStr>() + && module_str.as_bytes() != b"builtins" + { + return wtf8_concat!(module_str.as_wtf8(), ".", qualname_str.as_wtf8(), "()"); + } + wtf8_concat!(qualname_str.as_wtf8(), "()") + } + + /// Helper function to iterate over mapping keys using the keys() method. + /// This ensures proper order preservation for OrderedDict and other custom mappings. + fn iterate_mapping_keys<F>( + vm: &VirtualMachine, + mapping: &PyObject, + func_str: &Wtf8, + mut key_handler: F, + ) -> PyResult<()> + where + F: FnMut(PyObjectRef) -> PyResult<()>, + { + let Some(keys_method) = vm.get_method(mapping.to_owned(), vm.ctx.intern_str("keys")) else { + return Err(vm.new_type_error(format!( + "{} argument after ** must be a mapping, not {}", + func_str, + mapping.class().name() + ))); + }; + + let keys = keys_method?.call((), vm)?.get_iter(vm)?; + while let PyIterReturn::Return(key) = keys.next(vm)? { + key_handler(key)?; + } + Ok(()) + } + + /// Vectorcall dispatch for Instruction::Call (positional args only). + /// Uses vectorcall slot if available, otherwise falls back to FuncArgs. + #[inline] + fn execute_call_vectorcall(&mut self, nargs: u32, vm: &VirtualMachine) -> FrameResult { + let nargs_usize = nargs as usize; + let stack_len = self.localsplus.stack_len(); + debug_assert!( + stack_len >= nargs_usize + 2, + "CALL stack underflow: need callable + self_or_null + {nargs_usize} args, have {stack_len}" + ); + let callable_idx = stack_len - nargs_usize - 2; + let self_or_null_idx = stack_len - nargs_usize - 1; + let args_start = stack_len - nargs_usize; + + // Build args: [self?, arg1, ..., argN] + let self_or_null = self + .localsplus + .stack_index_mut(self_or_null_idx) + .take() + .map(|sr| sr.to_pyobj()); + let has_self = self_or_null.is_some(); + + let effective_nargs = if has_self { + nargs_usize + 1 + } else { + nargs_usize + }; + let mut args_vec = Vec::with_capacity(effective_nargs); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + for stack_idx in args_start..stack_len { + let val = self + .localsplus + .stack_index_mut(stack_idx) + .take() + .unwrap() + .to_pyobj(); + args_vec.push(val); + } + + let callable_obj = self + .localsplus + .stack_index_mut(callable_idx) + .take() + .unwrap() + .to_pyobj(); + self.localsplus.stack_truncate(callable_idx); + + // invoke_vectorcall falls back to FuncArgs if no vectorcall slot + let result = callable_obj.vectorcall(args_vec, effective_nargs, None, vm)?; + self.push_value(result); + Ok(None) + } + + /// Vectorcall dispatch for Instruction::CallKw (positional + keyword args). + #[inline] + fn execute_call_kw_vectorcall(&mut self, nargs: u32, vm: &VirtualMachine) -> FrameResult { + let nargs_usize = nargs as usize; + + // Pop kwarg_names tuple from top of stack + let kwarg_names_obj = self.pop_value(); + let kwarg_names_tuple = kwarg_names_obj + .downcast_ref::<PyTuple>() + .expect("kwarg names should be tuple"); + let kw_count = kwarg_names_tuple.len(); + debug_assert!(kw_count <= nargs_usize, "CALL_KW kw_count exceeds nargs"); + + let stack_len = self.localsplus.stack_len(); + debug_assert!( + stack_len >= nargs_usize + 2, + "CALL_KW stack underflow: need callable + self_or_null + {nargs_usize} args, have {stack_len}" + ); + let callable_idx = stack_len - nargs_usize - 2; + let self_or_null_idx = stack_len - nargs_usize - 1; + let args_start = stack_len - nargs_usize; + + // Build args: [self?, pos_arg1, ..., pos_argM, kw_val1, ..., kw_valK] + let self_or_null = self + .localsplus + .stack_index_mut(self_or_null_idx) + .take() + .map(|sr| sr.to_pyobj()); + let has_self = self_or_null.is_some(); + + let pos_count = nargs_usize + .checked_sub(kw_count) + .expect("CALL_KW: kw_count exceeds nargs"); + let effective_nargs = if has_self { pos_count + 1 } else { pos_count }; + + // Build the full args slice: positional (including self) + kwarg values + let total_args = effective_nargs + kw_count; + let mut args_vec = Vec::with_capacity(total_args); + if let Some(self_val) = self_or_null { + args_vec.push(self_val); + } + for stack_idx in args_start..stack_len { + let val = self + .localsplus + .stack_index_mut(stack_idx) + .take() + .unwrap() + .to_pyobj(); + args_vec.push(val); + } + + let callable_obj = self + .localsplus + .stack_index_mut(callable_idx) + .take() + .unwrap() + .to_pyobj(); + self.localsplus.stack_truncate(callable_idx); + + // invoke_vectorcall falls back to FuncArgs if no vectorcall slot + let kwnames = kwarg_names_tuple.as_slice(); + let result = callable_obj.vectorcall(args_vec, effective_nargs, Some(kwnames), vm)?; + self.push_value(result); + Ok(None) + } + + #[inline] + fn execute_call(&mut self, args: FuncArgs, vm: &VirtualMachine) -> FrameResult { + // Stack: [callable, self_or_null, ...] + let self_or_null = self.pop_value_opt(); // Option<PyObjectRef> + let callable = self.pop_value(); + + let final_args = if let Some(self_val) = self_or_null { + let mut args = args; + args.prepend_arg(self_val); + args + } else { + args + }; + + let value = callable.call(final_args, vm)?; + self.push_value(value); + Ok(None) + } + + /// Instrumented version of execute_call: fires CALL, C_RETURN, and C_RAISE events. + fn execute_call_instrumented(&mut self, args: FuncArgs, vm: &VirtualMachine) -> FrameResult { + let self_or_null = self.pop_value_opt(); + let callable = self.pop_value(); + + let final_args = if let Some(self_val) = self_or_null { + let mut args = args; + args.prepend_arg(self_val); + args + } else { + args + }; + + let is_python_call = callable.downcast_ref_if_exact::<PyFunction>(vm).is_some(); + + // Fire CALL event + let call_arg0 = if self.monitoring_mask & monitoring::EVENT_CALL != 0 { + let arg0 = final_args + .args + .first() + .cloned() + .unwrap_or_else(|| monitoring::get_missing(vm)); + let offset = (self.lasti() - 1) * 2; + monitoring::fire_call(vm, self.code, offset, &callable, arg0.clone())?; + Some(arg0) + } else { + None + }; + + match callable.call(final_args, vm) { + Ok(value) => { + if let Some(arg0) = call_arg0 + && !is_python_call + { + let offset = (self.lasti() - 1) * 2; + monitoring::fire_c_return(vm, self.code, offset, &callable, arg0)?; + } + self.push_value(value); + Ok(None) + } + Err(exc) => { + let exc = if let Some(arg0) = call_arg0 + && !is_python_call + { + let offset = (self.lasti() - 1) * 2; + match monitoring::fire_c_raise(vm, self.code, offset, &callable, arg0) { + Ok(()) => exc, + Err(monitor_exc) => monitor_exc, + } + } else { + exc + }; + Err(exc) + } + } + } + + fn execute_raise(&mut self, vm: &VirtualMachine, kind: bytecode::RaiseKind) -> FrameResult { + let cause = match kind { + bytecode::RaiseKind::RaiseCause => { + let val = self.pop_value(); + Some(if vm.is_none(&val) { + // if the cause arg is none, we clear the cause + None + } else { + // if the cause arg is an exception, we overwrite it + let ctor = ExceptionCtor::try_from_object(vm, val).map_err(|_| { + vm.new_type_error("exception causes must derive from BaseException") + })?; + Some(ctor.instantiate(vm)?) + }) + } + // if there's no cause arg, we keep the cause as is + _ => None, + }; + let exception = match kind { + bytecode::RaiseKind::RaiseCause | bytecode::RaiseKind::Raise => { + ExceptionCtor::try_from_object(vm, self.pop_value())?.instantiate(vm)? + } + bytecode::RaiseKind::BareRaise => { + // RAISE_VARARGS 0: bare `raise` gets exception from VM state + // This is the current exception set by PUSH_EXC_INFO + vm.topmost_exception() + .ok_or_else(|| vm.new_runtime_error("No active exception to reraise"))? + } + bytecode::RaiseKind::ReraiseFromStack => { + // RERAISE: gets exception from stack top + // Used in cleanup blocks where exception is on stack after COPY 3 + let exc = self.pop_value(); + exc.downcast::<PyBaseException>().map_err(|obj| { + vm.new_type_error(format!( + "exceptions must derive from BaseException, not {}", + obj.class().name() + )) + })? + } + }; + #[cfg(debug_assertions)] + debug!("Exception raised: {exception:?} with cause: {cause:?}"); + if let Some(cause) = cause { + exception.set___cause__(cause); + } + Err(exception) + } + + fn builtin_coro<'a>(&self, coro: &'a PyObject) -> Option<&'a Coro> { + match_class!(match coro { + ref g @ PyGenerator => Some(g.as_coro()), + ref c @ PyCoroutine => Some(c.as_coro()), + _ => None, + }) + } + + fn _send( + &self, + jen: &PyObject, + val: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + match self.builtin_coro(jen) { + Some(coro) => coro.send(jen, val, vm), + // TODO: turn return type to PyResult<PyIterReturn> then ExecutionResult will be simplified + None if vm.is_none(&val) => PyIter::new(jen).next(vm), + None => { + let meth = jen.get_attr("send", vm)?; + PyIterReturn::from_pyresult(meth.call((val,), vm), vm) + } + } + } + + fn execute_unpack_ex(&mut self, vm: &VirtualMachine, before: u8, after: u8) -> FrameResult { + let (before, after) = (before as usize, after as usize); + let value = self.pop_value(); + let not_iterable = value.class().slots.iter.load().is_none() + && value + .get_class_attr(vm.ctx.intern_str("__getitem__")) + .is_none(); + let elements: Vec<_> = value.try_to_value(vm).map_err(|e| { + if not_iterable && e.class().is(vm.ctx.exceptions.type_error) { + vm.new_type_error(format!( + "cannot unpack non-iterable {} object", + value.class().name() + )) + } else { + e + } + })?; + let min_expected = before + after; + + let middle = elements.len().checked_sub(min_expected).ok_or_else(|| { + vm.new_value_error(format!( + "not enough values to unpack (expected at least {}, got {})", + min_expected, + elements.len() + )) + })?; + + let mut elements = elements; + // Elements on stack from right-to-left: + self.localsplus.stack_extend( + elements + .drain(before + middle..) + .rev() + .map(|e| Some(PyStackRef::new_owned(e))), + ); + + let middle_elements = elements.drain(before..).collect(); + let t = vm.ctx.new_list(middle_elements); + self.push_value(t.into()); + + // Lastly the first reversed values: + self.localsplus.stack_extend( + elements + .into_iter() + .rev() + .map(|e| Some(PyStackRef::new_owned(e))), + ); + + Ok(None) + } + + #[inline] + fn jump(&mut self, label: bytecode::Label) { + let target_pc = label.as_u32(); + vm_trace!("jump from {:?} to {:?}", self.lasti(), target_pc); + self.update_lasti(|i| *i = target_pc); + } + + /// Jump forward by `delta` code units from after instruction + caches. + /// lasti is already at instruction_index + 1, so after = lasti + caches. + /// + /// Unchecked arithmetic is intentional: the compiler guarantees valid + /// targets, and debug builds will catch overflow via Rust's default checks. + #[inline] + fn jump_relative_forward(&mut self, delta: u32, caches: u32) { + let target = self.lasti() + caches + delta; + self.update_lasti(|i| *i = target); + } + + /// Jump backward by `delta` code units from after instruction + caches. + /// + /// Unchecked arithmetic is intentional: the compiler guarantees valid + /// targets, and debug builds will catch underflow via Rust's default checks. + #[inline] + fn jump_relative_backward(&mut self, delta: u32, caches: u32) { + let target = self.lasti() + caches - delta; + self.update_lasti(|i| *i = target); + } + + #[inline] + fn pop_jump_if_relative( + &mut self, + vm: &VirtualMachine, + arg: bytecode::OpArg, + caches: u32, + flag: bool, + ) -> FrameResult { + let obj = self.pop_value(); + let value = obj.try_to_bool(vm)?; + if value == flag { + self.jump_relative_forward(u32::from(arg), caches); + } + Ok(None) + } + + /// Advance the iterator on top of stack. + /// Returns `true` if iteration continued (item pushed), `false` if exhausted (jumped). + fn execute_for_iter( + &mut self, + vm: &VirtualMachine, + target: bytecode::Label, + ) -> Result<bool, PyBaseExceptionRef> { + let top = self.top_value(); + + // FOR_ITER_RANGE: bypass generic iterator protocol for range iterators + if let Some(range_iter) = top.downcast_ref_if_exact::<PyRangeIterator>(vm) { + if let Some(value) = range_iter.fast_next() { + self.push_value(vm.ctx.new_int(value).into()); + return Ok(true); + } + if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + let stop_exc = vm.new_stop_iteration(None); + self.fire_exception_trace(&stop_exc, vm)?; + } + self.jump(self.for_iter_jump_target(target)); + return Ok(false); + } + + let top_of_stack = PyIter::new(top); + let next_obj = top_of_stack.next(vm); + + match next_obj { + Ok(PyIterReturn::Return(value)) => { + self.push_value(value); + Ok(true) + } + Ok(PyIterReturn::StopIteration(value)) => { + // Fire 'exception' trace event for StopIteration, matching + // FOR_ITER's inline call to _PyEval_MonitorRaise. + if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) { + let stop_exc = vm.new_stop_iteration(value); + self.fire_exception_trace(&stop_exc, vm)?; + } + self.jump(self.for_iter_jump_target(target)); + Ok(false) + } + Err(next_error) => { + self.pop_value(); + Err(next_error) + } + } + } + + /// Compute the jump target for FOR_ITER exhaustion: skip END_FOR and jump to POP_ITER. + fn for_iter_jump_target(&self, target: bytecode::Label) -> bytecode::Label { + let target_idx = target.as_usize(); + if let Some(unit) = self.code.instructions.get(target_idx) + && matches!( + unit.op, + bytecode::Instruction::EndFor | bytecode::Instruction::InstrumentedEndFor + ) + { + return bytecode::Label::new(target.as_u32() + 1); + } + target + } + fn execute_make_function(&mut self, vm: &VirtualMachine) -> FrameResult { + // MakeFunction only takes code object, no flags + let code_obj: PyRef<PyCode> = self + .pop_value() + .downcast() + .expect("Stack value should be code object"); + + // Create function with minimal attributes + let func_obj = PyFunction::new(code_obj, self.globals.clone(), vm)?.into_pyobject(vm); + + self.push_value(func_obj); + Ok(None) + } + + fn execute_set_function_attribute( + &mut self, + vm: &VirtualMachine, + attr: bytecode::MakeFunctionFlag, + ) -> FrameResult { + // SET_FUNCTION_ATTRIBUTE sets attributes on a function + // Stack: [..., attr_value, func] -> [..., func] + // Stack order: func is at -1, attr_value is at -2 + + let func = self.pop_value_opt(); + let attr_value = expect_unchecked(self.replace_top(func), "attr_value must not be null"); + + let func = self.top_value(); + // Get the function reference and call the new method + let func_ref = func + .downcast_ref_if_exact::<PyFunction>(vm) + .expect("SET_FUNCTION_ATTRIBUTE expects function on stack"); + + let payload: &PyFunction = func_ref.payload(); + // SetFunctionAttribute always follows MakeFunction, so at this point + // there are no other references to func. It is therefore safe to treat it as mutable. + unsafe { + let payload_ptr = payload as *const PyFunction as *mut PyFunction; + (*payload_ptr).set_function_attribute(attr, attr_value, vm)?; + }; + + Ok(None) + } + + #[cfg_attr(feature = "flame-it", flame("Frame"))] + fn execute_bin_op(&mut self, vm: &VirtualMachine, op: bytecode::BinaryOperator) -> FrameResult { + let b_ref = &self.pop_value(); + let a_ref = &self.pop_value(); + let value = match op { + // BINARY_OP_ADD_INT / BINARY_OP_SUBTRACT_INT fast paths: + // bypass binary_op1 dispatch for exact int types, use i64 arithmetic + // when possible to avoid BigInt heap allocation. + bytecode::BinaryOperator::Add | bytecode::BinaryOperator::InplaceAdd => { + if let (Some(a), Some(b)) = ( + a_ref.downcast_ref_if_exact::<PyInt>(vm), + b_ref.downcast_ref_if_exact::<PyInt>(vm), + ) { + Ok(self.int_add(a.as_bigint(), b.as_bigint(), vm)) + } else if matches!(op, bytecode::BinaryOperator::Add) { + vm._add(a_ref, b_ref) + } else { + vm._iadd(a_ref, b_ref) + } + } + bytecode::BinaryOperator::Subtract | bytecode::BinaryOperator::InplaceSubtract => { + if let (Some(a), Some(b)) = ( + a_ref.downcast_ref_if_exact::<PyInt>(vm), + b_ref.downcast_ref_if_exact::<PyInt>(vm), + ) { + Ok(self.int_sub(a.as_bigint(), b.as_bigint(), vm)) + } else if matches!(op, bytecode::BinaryOperator::Subtract) { + vm._sub(a_ref, b_ref) + } else { + vm._isub(a_ref, b_ref) + } + } + bytecode::BinaryOperator::Multiply => vm._mul(a_ref, b_ref), + bytecode::BinaryOperator::MatrixMultiply => vm._matmul(a_ref, b_ref), + bytecode::BinaryOperator::Power => vm._pow(a_ref, b_ref, vm.ctx.none.as_object()), + bytecode::BinaryOperator::TrueDivide => vm._truediv(a_ref, b_ref), + bytecode::BinaryOperator::FloorDivide => vm._floordiv(a_ref, b_ref), + bytecode::BinaryOperator::Remainder => vm._mod(a_ref, b_ref), + bytecode::BinaryOperator::Lshift => vm._lshift(a_ref, b_ref), + bytecode::BinaryOperator::Rshift => vm._rshift(a_ref, b_ref), + bytecode::BinaryOperator::Xor => vm._xor(a_ref, b_ref), + bytecode::BinaryOperator::Or => vm._or(a_ref, b_ref), + bytecode::BinaryOperator::And => vm._and(a_ref, b_ref), + bytecode::BinaryOperator::InplaceMultiply => vm._imul(a_ref, b_ref), + bytecode::BinaryOperator::InplaceMatrixMultiply => vm._imatmul(a_ref, b_ref), + bytecode::BinaryOperator::InplacePower => { + vm._ipow(a_ref, b_ref, vm.ctx.none.as_object()) + } + bytecode::BinaryOperator::InplaceTrueDivide => vm._itruediv(a_ref, b_ref), + bytecode::BinaryOperator::InplaceFloorDivide => vm._ifloordiv(a_ref, b_ref), + bytecode::BinaryOperator::InplaceRemainder => vm._imod(a_ref, b_ref), + bytecode::BinaryOperator::InplaceLshift => vm._ilshift(a_ref, b_ref), + bytecode::BinaryOperator::InplaceRshift => vm._irshift(a_ref, b_ref), + bytecode::BinaryOperator::InplaceXor => vm._ixor(a_ref, b_ref), + bytecode::BinaryOperator::InplaceOr => vm._ior(a_ref, b_ref), + bytecode::BinaryOperator::InplaceAnd => vm._iand(a_ref, b_ref), + bytecode::BinaryOperator::Subscr => a_ref.get_item(b_ref.as_object(), vm), + }?; + + self.push_value(value); + Ok(None) + } + + /// Int addition with i64 fast path to avoid BigInt heap allocation. + #[inline] + fn int_add(&self, a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + use num_traits::ToPrimitive; + if let (Some(av), Some(bv)) = (a.to_i64(), b.to_i64()) + && let Some(result) = av.checked_add(bv) + { + return vm.ctx.new_int(result).into(); + } + vm.ctx.new_int(a + b).into() + } + + /// Int subtraction with i64 fast path to avoid BigInt heap allocation. + #[inline] + fn int_sub(&self, a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + use num_traits::ToPrimitive; + if let (Some(av), Some(bv)) = (a.to_i64(), b.to_i64()) + && let Some(result) = av.checked_sub(bv) + { + return vm.ctx.new_int(result).into(); + } + vm.ctx.new_int(a - b).into() + } + + #[cold] + fn setup_annotations(&mut self, vm: &VirtualMachine) -> FrameResult { + let __annotations__ = identifier!(vm, __annotations__); + let locals_obj = self.locals.as_object(vm); + // Try using locals as dict first, if not, fallback to generic method. + let has_annotations = if let Some(d) = locals_obj.downcast_ref_if_exact::<PyDict>(vm) { + d.contains_key(__annotations__, vm) + } else { + self._in(vm, __annotations__.as_object(), locals_obj)? + }; + if !has_annotations { + locals_obj.set_item(__annotations__, vm.ctx.new_dict().into(), vm)?; + } + Ok(None) + } + + /// _PyEval_UnpackIterableStackRef + fn unpack_sequence(&mut self, size: u32, vm: &VirtualMachine) -> FrameResult { + let value = self.pop_value(); + let size = size as usize; + + // Fast path for exact tuple/list types (not subclasses) — push + // elements directly from the slice without intermediate Vec allocation, + // matching UNPACK_SEQUENCE_TUPLE / UNPACK_SEQUENCE_LIST specializations. + let cls = value.class(); + if cls.is(vm.ctx.types.tuple_type) { + let tuple = value.downcast_ref::<PyTuple>().unwrap(); + return self.unpack_fast(tuple.as_slice(), size, vm); + } + if cls.is(vm.ctx.types.list_type) { + let list = value.downcast_ref::<PyList>().unwrap(); + let borrowed = list.borrow_vec(); + return self.unpack_fast(&borrowed, size, vm); + } + + // General path — iterate up to `size + 1` elements to avoid + // consuming the entire iterator (fixes hang on infinite sequences). + let not_iterable = value.class().slots.iter.load().is_none() + && value + .get_class_attr(vm.ctx.intern_str("__getitem__")) + .is_none(); + let iter = PyIter::try_from_object(vm, value.clone()).map_err(|e| { + if not_iterable && e.class().is(vm.ctx.exceptions.type_error) { + vm.new_type_error(format!( + "cannot unpack non-iterable {} object", + value.class().name() + )) + } else { + e + } + })?; + + let mut elements = Vec::with_capacity(size); + for _ in 0..size { + match iter.next(vm)? { + PyIterReturn::Return(item) => elements.push(item), + PyIterReturn::StopIteration(_) => { + return Err(vm.new_value_error(format!( + "not enough values to unpack (expected {size}, got {})", + elements.len() + ))); + } + } + } + + // Check that the iterator is exhausted. + match iter.next(vm)? { + PyIterReturn::Return(_) => { + // For exact dict types, show "got N" using the container's + // size (PyDict_Size). Exact tuple/list are handled by the + // fast path above and never reach here. + let msg = if value.class().is(vm.ctx.types.dict_type) { + if let Ok(got) = value.length(vm) { + if got > size { + format!("too many values to unpack (expected {size}, got {got})") + } else { + format!("too many values to unpack (expected {size})") + } + } else { + format!("too many values to unpack (expected {size})") + } + } else { + format!("too many values to unpack (expected {size})") + }; + Err(vm.new_value_error(msg)) + } + PyIterReturn::StopIteration(_) => { + self.localsplus.stack_extend( + elements + .into_iter() + .rev() + .map(|e| Some(PyStackRef::new_owned(e))), + ); + Ok(None) + } + } + } + + fn unpack_fast( + &mut self, + elements: &[PyObjectRef], + size: usize, + vm: &VirtualMachine, + ) -> FrameResult { + match elements.len().cmp(&size) { + core::cmp::Ordering::Equal => { + for elem in elements.iter().rev() { + self.push_value(elem.clone()); + } + Ok(None) + } + core::cmp::Ordering::Greater => Err(vm.new_value_error(format!( + "too many values to unpack (expected {size}, got {})", + elements.len() + ))), + core::cmp::Ordering::Less => Err(vm.new_value_error(format!( + "not enough values to unpack (expected {size}, got {})", + elements.len() + ))), + } + } + + fn convert_value( + &mut self, + conversion: bytecode::ConvertValueOparg, + vm: &VirtualMachine, + ) -> FrameResult { + use bytecode::ConvertValueOparg; + let value = self.pop_value(); + let value = match conversion { + ConvertValueOparg::Str => value.str(vm)?.into(), + ConvertValueOparg::Repr => value.repr(vm)?.into(), + ConvertValueOparg::Ascii => vm.ctx.new_str(builtins::ascii(value, vm)?).into(), + ConvertValueOparg::None => value, + }; + + self.push_value(value); + Ok(None) + } + + fn _in(&self, vm: &VirtualMachine, needle: &PyObject, haystack: &PyObject) -> PyResult<bool> { + let found = vm._contains(haystack, needle)?; + Ok(found) + } + + #[inline(always)] + fn _not_in( + &self, + vm: &VirtualMachine, + needle: &PyObject, + haystack: &PyObject, + ) -> PyResult<bool> { + Ok(!self._in(vm, needle, haystack)?) + } + + #[cfg_attr(feature = "flame-it", flame("Frame"))] + fn execute_compare( + &mut self, + vm: &VirtualMachine, + op: bytecode::ComparisonOperator, + ) -> FrameResult { + let b = self.pop_value(); + let a = self.pop_value(); + let cmp_op: PyComparisonOp = op.into(); + + // COMPARE_OP_INT: leaf type, cannot recurse — skip rich_compare dispatch + if let (Some(a_int), Some(b_int)) = ( + a.downcast_ref_if_exact::<PyInt>(vm), + b.downcast_ref_if_exact::<PyInt>(vm), + ) { + let result = cmp_op.eval_ord(a_int.as_bigint().cmp(b_int.as_bigint())); + self.push_value(vm.ctx.new_bool(result).into()); + return Ok(None); + } + // COMPARE_OP_FLOAT: leaf type, cannot recurse — skip rich_compare dispatch. + // Falls through on NaN (partial_cmp returns None) for correct != semantics. + if let (Some(a_f), Some(b_f)) = ( + a.downcast_ref_if_exact::<PyFloat>(vm), + b.downcast_ref_if_exact::<PyFloat>(vm), + ) && let Some(ord) = a_f.to_f64().partial_cmp(&b_f.to_f64()) + { + let result = cmp_op.eval_ord(ord); + self.push_value(vm.ctx.new_bool(result).into()); + return Ok(None); + } + + let value = a.rich_compare(b, cmp_op, vm)?; + self.push_value(value); + Ok(None) + } + + /// Read a cached descriptor pointer and validate it against the expected + /// type version, using a lock-free double-check pattern: + /// 1. read pointer → incref (try_to_owned) + /// 2. re-read version + pointer and confirm they still match + /// + /// This matches the read-side pattern used in LOAD_ATTR_METHOD_WITH_VALUES + /// and friends: no read-side lock, relying on the write side to invalidate + /// the version tag before swapping the pointer. + #[inline] + fn try_read_cached_descriptor( + &self, + cache_base: usize, + expected_type_version: u32, + ) -> Option<PyObjectRef> { + let descr_ptr = self.code.instructions.read_cache_ptr(cache_base + 5); + if descr_ptr == 0 { + return None; + } + // SAFETY: `descr_ptr` was a valid `*mut PyObject` when the writer + // stored it, and the writer keeps a strong reference alive in + // `InlineCacheEntry`. `try_to_owned_from_ptr` performs a + // conditional incref that fails if the object is already freed. + let cloned = unsafe { PyObject::try_to_owned_from_ptr(descr_ptr as *mut PyObject) }?; + // Double-check: version tag still matches AND pointer unchanged. + if self.code.instructions.read_cache_u32(cache_base + 1) == expected_type_version + && self.code.instructions.read_cache_ptr(cache_base + 5) == descr_ptr + { + Some(cloned) + } else { + drop(cloned); + None + } + } + + #[inline] + unsafe fn write_cached_descriptor( + &self, + cache_base: usize, + type_version: u32, + descr_ptr: usize, + ) { + // Publish descriptor cache with version-invalidation protocol: + // invalidate version first, then write payload, then publish version. + // Reader double-checks version+ptr after incref, so no writer lock needed. + unsafe { + self.code.instructions.write_cache_u32(cache_base + 1, 0); + self.code + .instructions + .write_cache_ptr(cache_base + 5, descr_ptr); + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + } + } + + #[inline] + unsafe fn write_cached_descriptor_with_metaclass( + &self, + cache_base: usize, + type_version: u32, + metaclass_version: u32, + descr_ptr: usize, + ) { + unsafe { + self.code.instructions.write_cache_u32(cache_base + 1, 0); + self.code + .instructions + .write_cache_u32(cache_base + 3, metaclass_version); + self.code + .instructions + .write_cache_ptr(cache_base + 5, descr_ptr); + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + } + } + + #[inline] + unsafe fn write_cached_binary_op_extend_descr( + &self, + cache_base: usize, + descr: Option<&'static BinaryOpExtendSpecializationDescr>, + ) { + let ptr = descr.map_or(0, |d| { + d as *const BinaryOpExtendSpecializationDescr as usize + }); + unsafe { + self.code + .instructions + .write_cache_ptr(cache_base + BINARY_OP_EXTEND_EXTERNAL_CACHE_OFFSET, ptr); + } + } + + #[inline] + fn read_cached_binary_op_extend_descr( + &self, + cache_base: usize, + ) -> Option<&'static BinaryOpExtendSpecializationDescr> { + let ptr = self + .code + .instructions + .read_cache_ptr(cache_base + BINARY_OP_EXTEND_EXTERNAL_CACHE_OFFSET); + if ptr == 0 { + return None; + } + // SAFETY: We only store pointers to entries in `BINARY_OP_EXTEND_DESCRIPTORS`. + Some(unsafe { &*(ptr as *const BinaryOpExtendSpecializationDescr) }) + } + + #[inline] + fn binary_op_extended_specialization( + &self, + op: bytecode::BinaryOperator, + lhs: &PyObject, + rhs: &PyObject, + vm: &VirtualMachine, + ) -> Option<&'static BinaryOpExtendSpecializationDescr> { + BINARY_OP_EXTEND_DESCRIPTORS + .iter() + .find(|d| d.oparg == op && (d.guard)(lhs, rhs, vm)) + } + + fn load_attr(&mut self, vm: &VirtualMachine, oparg: LoadAttr) -> FrameResult { + self.adaptive(|s, ii, cb| s.specialize_load_attr(vm, oparg, ii, cb)); + self.load_attr_slow(vm, oparg) + } + + fn specialize_load_attr( + &mut self, + _vm: &VirtualMachine, + oparg: LoadAttr, + instr_idx: usize, + cache_base: usize, + ) { + // Pre-check: bail if already specialized by another thread + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::LoadAttr { .. } + ) { + return; + } + let obj = self.top_value(); + let cls = obj.class(); + + // Check if this is a type object (class attribute access) + if obj.downcast_ref::<PyType>().is_some() { + self.specialize_class_load_attr(_vm, oparg, instr_idx, cache_base); + return; + } + + // Only specialize if getattro is the default (PyBaseObject::getattro) + let is_default_getattro = cls + .slots + .getattro + .load() + .is_some_and(|f| f as usize == PyBaseObject::getattro as *const () as usize); + if !is_default_getattro { + let mut type_version = cls.tp_version_tag.load(Acquire); + if type_version == 0 { + type_version = cls.assign_version_tag(); + } + if type_version != 0 + && !oparg.is_method() + && !self.specialization_eval_frame_active(_vm) + && cls.get_attr(identifier!(_vm, __getattr__)).is_none() + && let Some(getattribute) = cls.get_attr(identifier!(_vm, __getattribute__)) + && let Some(func) = getattribute.downcast_ref_if_exact::<PyFunction>(_vm) + && func.can_specialize_call(2) + { + let func_version = func.get_version_for_current_state(); + if func_version != 0 { + let func_ptr = &*getattribute as *const PyObject as usize; + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 3, func_version); + self.write_cached_descriptor(cache_base, type_version, func_ptr); + } + self.specialize_at( + instr_idx, + cache_base, + Instruction::LoadAttrGetattributeOverridden, + ); + return; + } + } + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + // Get or assign type version + let mut type_version = cls.tp_version_tag.load(Acquire); + if type_version == 0 { + type_version = cls.assign_version_tag(); + } + if type_version == 0 { + // Version counter overflow — backoff to avoid re-attempting every execution + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + let attr_name = self.code.names[oparg.name_idx() as usize]; + + // Match CPython: only specialize module attribute loads when the + // current module dict has no __getattr__ override and the attribute is + // already present. + if let Some(module) = obj.downcast_ref_if_exact::<PyModule>(_vm) { + let module_dict = module.dict(); + match ( + module_dict.get_item_opt(identifier!(_vm, __getattr__), _vm), + module_dict.get_item_opt(attr_name, _vm), + ) { + (Ok(None), Ok(Some(_))) => { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + } + self.specialize_at(instr_idx, cache_base, Instruction::LoadAttrModule); + } + (Ok(_), Ok(_)) => self.cooldown_adaptive_at(cache_base), + _ => unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + }, + } + return; + } + + // Look up attr in class via MRO + let cls_attr = cls.get_attr(attr_name); + let class_has_dict = cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT); + + if oparg.is_method() { + // Method specialization + if let Some(ref descr) = cls_attr + && descr + .class() + .slots + .flags + .has_feature(PyTypeFlags::METHOD_DESCRIPTOR) + { + let descr_ptr = &**descr as *const PyObject as usize; + unsafe { + self.write_cached_descriptor(cache_base, type_version, descr_ptr); + } + + let new_op = if !class_has_dict { + Instruction::LoadAttrMethodNoDict + } else if obj.dict().is_none() { + Instruction::LoadAttrMethodLazyDict + } else { + Instruction::LoadAttrMethodWithValues + }; + self.specialize_at(instr_idx, cache_base, new_op); + return; + } + // Can't specialize this method call + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } else { + // Regular attribute access + let has_data_descr = cls_attr.as_ref().is_some_and(|descr| { + let descr_cls = descr.class(); + descr_cls.slots.descr_get.load().is_some() + && descr_cls.slots.descr_set.load().is_some() + }); + let has_descr_get = cls_attr + .as_ref() + .is_some_and(|descr| descr.class().slots.descr_get.load().is_some()); + + if has_data_descr { + // Check for member descriptor (slot access) + if let Some(ref descr) = cls_attr + && let Some(member_descr) = descr.downcast_ref::<PyMemberDescriptor>() + && let MemberGetter::Offset(offset) = member_descr.member.getter + { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + self.code + .instructions + .write_cache_u32(cache_base + 3, offset as u32); + } + self.specialize_at(instr_idx, cache_base, Instruction::LoadAttrSlot); + } else if let Some(ref descr) = cls_attr + && let Some(prop) = descr.downcast_ref::<PyProperty>() + && let Some(fget) = prop.get_fget() + && let Some(func) = fget.downcast_ref_if_exact::<PyFunction>(_vm) + && func.can_specialize_call(1) + && !self.specialization_eval_frame_active(_vm) + { + // Property specialization caches fget directly. + let fget_ptr = &*fget as *const PyObject as usize; + unsafe { + self.write_cached_descriptor(cache_base, type_version, fget_ptr); + } + self.specialize_at(instr_idx, cache_base, Instruction::LoadAttrProperty); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + } else if has_descr_get { + // Non-data descriptor with __get__ — can't specialize + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } else if class_has_dict { + if let Some(ref descr) = cls_attr { + // Plain class attr + class supports dict — check dict first, fallback + let descr_ptr = &**descr as *const PyObject as usize; + unsafe { + self.write_cached_descriptor(cache_base, type_version, descr_ptr); + } + self.specialize_at( + instr_idx, + cache_base, + Instruction::LoadAttrNondescriptorWithValues, + ); + } else { + // Match CPython ABSENT/no-shadow behavior: if the + // attribute is missing on both the class and the current + // instance, keep the generic opcode and just enter + // cooldown instead of specializing a repeated miss path. + let has_instance_attr = if let Some(dict) = obj.dict() { + match dict.get_item_opt(attr_name, _vm) { + Ok(Some(_)) => true, + Ok(None) => false, + Err(_) => { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code + .instructions + .read_adaptive_counter(cache_base), + ), + ); + } + return; + } + } + } else { + false + }; + if has_instance_attr { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + } + self.specialize_at(instr_idx, cache_base, Instruction::LoadAttrWithHint); + } else { + self.cooldown_adaptive_at(cache_base); + } + } + } else if let Some(ref descr) = cls_attr { + // No dict support, plain class attr — cache directly + let descr_ptr = &**descr as *const PyObject as usize; + unsafe { + self.write_cached_descriptor(cache_base, type_version, descr_ptr); + } + self.specialize_at( + instr_idx, + cache_base, + Instruction::LoadAttrNondescriptorNoDict, + ); + } else { + // No dict and no class attr: repeated miss path, so cooldown. + self.cooldown_adaptive_at(cache_base); + } + } + } + + fn specialize_class_load_attr( + &mut self, + _vm: &VirtualMachine, + oparg: LoadAttr, + instr_idx: usize, + cache_base: usize, + ) { + let obj = self.top_value(); + let owner_type = obj.downcast_ref::<PyType>().unwrap(); + + // Get or assign type version for the type object itself + let mut type_version = owner_type.tp_version_tag.load(Acquire); + if type_version == 0 { + type_version = owner_type.assign_version_tag(); + } + if type_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + let attr_name = self.code.names[oparg.name_idx() as usize]; + + // Check metaclass: ensure no data descriptor on metaclass for this name + let mcl = obj.class(); + let mcl_attr = mcl.get_attr(attr_name); + if let Some(ref attr) = mcl_attr { + let attr_class = attr.class(); + if attr_class.slots.descr_set.load().is_some() { + // Data descriptor on metaclass — can't specialize + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + } + let mut metaclass_version = 0; + if !mcl.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + metaclass_version = mcl.tp_version_tag.load(Acquire); + if metaclass_version == 0 { + metaclass_version = mcl.assign_version_tag(); + } + if metaclass_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + } + + // Look up attr in the type's own MRO + let cls_attr = owner_type.get_attr(attr_name); + if let Some(ref descr) = cls_attr { + let descr_class = descr.class(); + let has_descr_get = descr_class.slots.descr_get.load().is_some(); + if !has_descr_get { + // METHOD or NON_DESCRIPTOR — can cache directly + let descr_ptr = &**descr as *const PyObject as usize; + let new_op = if metaclass_version == 0 { + Instruction::LoadAttrClass + } else { + Instruction::LoadAttrClassWithMetaclassCheck + }; + unsafe { + if metaclass_version == 0 { + self.write_cached_descriptor(cache_base, type_version, descr_ptr); + } else { + self.write_cached_descriptor_with_metaclass( + cache_base, + type_version, + metaclass_version, + descr_ptr, + ); + } + } + self.specialize_at(instr_idx, cache_base, new_op); + return; + } + } + + // Can't specialize + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + + fn load_attr_slow(&mut self, vm: &VirtualMachine, oparg: LoadAttr) -> FrameResult { + let attr_name = self.code.names[oparg.name_idx() as usize]; + let parent = self.pop_value(); + + if oparg.is_method() { + // Method call: push [method, self_or_null] + let method = PyMethod::get(parent.clone(), attr_name, vm)?; + match method { + PyMethod::Function { target: _, func } => { + self.push_value(func); + self.push_value(parent); + } + PyMethod::Attribute(val) => { + self.push_value(val); + self.push_null(); + } + } + } else { + // Regular attribute access + let obj = parent.get_attr(attr_name, vm)?; + self.push_value(obj); + } + Ok(None) + } + + fn specialize_binary_op( + &mut self, + vm: &VirtualMachine, + op: bytecode::BinaryOperator, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::BinaryOp { .. } + ) { + return; + } + let b = self.top_value(); + let a = self.nth_value(1); + // `external_cache` in _PyBinaryOpCache is used only by BINARY_OP_EXTEND. + unsafe { + self.write_cached_binary_op_extend_descr(cache_base, None); + } + let mut cached_extend_descr = None; + + let new_op = match op { + bytecode::BinaryOperator::Add => { + if a.downcast_ref_if_exact::<PyInt>(vm).is_some() + && b.downcast_ref_if_exact::<PyInt>(vm).is_some() + { + Some(Instruction::BinaryOpAddInt) + } else if a.downcast_ref_if_exact::<PyFloat>(vm).is_some() + && b.downcast_ref_if_exact::<PyFloat>(vm).is_some() + { + Some(Instruction::BinaryOpAddFloat) + } else if a.downcast_ref_if_exact::<PyStr>(vm).is_some() + && b.downcast_ref_if_exact::<PyStr>(vm).is_some() + { + if self + .binary_op_inplace_unicode_target_local(cache_base, a) + .is_some() + { + Some(Instruction::BinaryOpInplaceAddUnicode) + } else { + Some(Instruction::BinaryOpAddUnicode) + } + } else if let Some(descr) = self.binary_op_extended_specialization(op, a, b, vm) { + cached_extend_descr = Some(descr); + Some(Instruction::BinaryOpExtend) + } else { + None + } + } + bytecode::BinaryOperator::Subtract => { + if a.downcast_ref_if_exact::<PyInt>(vm).is_some() + && b.downcast_ref_if_exact::<PyInt>(vm).is_some() + { + Some(Instruction::BinaryOpSubtractInt) + } else if a.downcast_ref_if_exact::<PyFloat>(vm).is_some() + && b.downcast_ref_if_exact::<PyFloat>(vm).is_some() + { + Some(Instruction::BinaryOpSubtractFloat) + } else if let Some(descr) = self.binary_op_extended_specialization(op, a, b, vm) { + cached_extend_descr = Some(descr); + Some(Instruction::BinaryOpExtend) + } else { + None + } + } + bytecode::BinaryOperator::Multiply => { + if a.downcast_ref_if_exact::<PyInt>(vm).is_some() + && b.downcast_ref_if_exact::<PyInt>(vm).is_some() + { + Some(Instruction::BinaryOpMultiplyInt) + } else if a.downcast_ref_if_exact::<PyFloat>(vm).is_some() + && b.downcast_ref_if_exact::<PyFloat>(vm).is_some() + { + Some(Instruction::BinaryOpMultiplyFloat) + } else if let Some(descr) = self.binary_op_extended_specialization(op, a, b, vm) { + cached_extend_descr = Some(descr); + Some(Instruction::BinaryOpExtend) + } else { + None + } + } + bytecode::BinaryOperator::TrueDivide => { + if let Some(descr) = self.binary_op_extended_specialization(op, a, b, vm) { + cached_extend_descr = Some(descr); + Some(Instruction::BinaryOpExtend) + } else { + None + } + } + bytecode::BinaryOperator::Subscr => { + let b_is_nonnegative_int = b + .downcast_ref_if_exact::<PyInt>(vm) + .is_some_and(|i| specialization_nonnegative_compact_index(i, vm).is_some()); + if a.downcast_ref_if_exact::<PyList>(vm).is_some() && b_is_nonnegative_int { + Some(Instruction::BinaryOpSubscrListInt) + } else if a.downcast_ref_if_exact::<PyTuple>(vm).is_some() && b_is_nonnegative_int { + Some(Instruction::BinaryOpSubscrTupleInt) + } else if a.downcast_ref_if_exact::<PyDict>(vm).is_some() { + Some(Instruction::BinaryOpSubscrDict) + } else if a.downcast_ref_if_exact::<PyStr>(vm).is_some() && b_is_nonnegative_int { + Some(Instruction::BinaryOpSubscrStrInt) + } else if a.downcast_ref_if_exact::<PyList>(vm).is_some() + && b.downcast_ref::<PySlice>().is_some() + { + Some(Instruction::BinaryOpSubscrListSlice) + } else { + let cls = a.class(); + if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + && !self.specialization_eval_frame_active(vm) + && let Some(_getitem) = cls.get_attr(identifier!(vm, __getitem__)) + && let Some(func) = _getitem.downcast_ref_if_exact::<PyFunction>(vm) + && func.can_specialize_call(2) + { + let mut type_version = cls.tp_version_tag.load(Acquire); + if type_version == 0 { + type_version = cls.assign_version_tag(); + } + if type_version != 0 { + if cls.cache_getitem_for_specialization( + func.to_owned(), + type_version, + vm, + ) { + Some(Instruction::BinaryOpSubscrGetitem) + } else { + None + } + } else { + None + } + } else { + None + } + } + } + bytecode::BinaryOperator::InplaceAdd => { + if a.downcast_ref_if_exact::<PyStr>(vm).is_some() + && b.downcast_ref_if_exact::<PyStr>(vm).is_some() + { + if self + .binary_op_inplace_unicode_target_local(cache_base, a) + .is_some() + { + Some(Instruction::BinaryOpInplaceAddUnicode) + } else { + Some(Instruction::BinaryOpAddUnicode) + } + } else if a.downcast_ref_if_exact::<PyInt>(vm).is_some() + && b.downcast_ref_if_exact::<PyInt>(vm).is_some() + { + Some(Instruction::BinaryOpAddInt) + } else if a.downcast_ref_if_exact::<PyFloat>(vm).is_some() + && b.downcast_ref_if_exact::<PyFloat>(vm).is_some() + { + Some(Instruction::BinaryOpAddFloat) + } else { + None + } + } + bytecode::BinaryOperator::InplaceSubtract => { + if a.downcast_ref_if_exact::<PyInt>(vm).is_some() + && b.downcast_ref_if_exact::<PyInt>(vm).is_some() + { + Some(Instruction::BinaryOpSubtractInt) + } else if a.downcast_ref_if_exact::<PyFloat>(vm).is_some() + && b.downcast_ref_if_exact::<PyFloat>(vm).is_some() + { + Some(Instruction::BinaryOpSubtractFloat) + } else { + None + } + } + bytecode::BinaryOperator::InplaceMultiply => { + if a.downcast_ref_if_exact::<PyInt>(vm).is_some() + && b.downcast_ref_if_exact::<PyInt>(vm).is_some() + { + Some(Instruction::BinaryOpMultiplyInt) + } else if a.downcast_ref_if_exact::<PyFloat>(vm).is_some() + && b.downcast_ref_if_exact::<PyFloat>(vm).is_some() + { + Some(Instruction::BinaryOpMultiplyFloat) + } else { + None + } + } + bytecode::BinaryOperator::And + | bytecode::BinaryOperator::Or + | bytecode::BinaryOperator::Xor + | bytecode::BinaryOperator::InplaceAnd + | bytecode::BinaryOperator::InplaceOr + | bytecode::BinaryOperator::InplaceXor => { + if let Some(descr) = self.binary_op_extended_specialization(op, a, b, vm) { + cached_extend_descr = Some(descr); + Some(Instruction::BinaryOpExtend) + } else { + None + } + } + _ => None, + }; + + if matches!(new_op, Some(Instruction::BinaryOpExtend)) { + unsafe { + self.write_cached_binary_op_extend_descr(cache_base, cached_extend_descr); + } + } + self.commit_specialization(instr_idx, cache_base, new_op); + } + + #[inline] + fn binary_op_inplace_unicode_target_local( + &self, + cache_base: usize, + left: &PyObject, + ) -> Option<usize> { + let next_idx = cache_base + Instruction::BinaryOp { op: Arg::marker() }.cache_entries(); + let unit = self.code.instructions.get(next_idx)?; + let next_op = unit.op.to_base().unwrap_or(unit.op); + if !matches!(next_op, Instruction::StoreFast { .. }) { + return None; + } + let local_idx = usize::from(u8::from(unit.arg)); + self.localsplus + .fastlocals() + .get(local_idx) + .and_then(|slot| slot.as_ref()) + .filter(|local| local.is(left)) + .map(|_| local_idx) + } + + /// Adaptive counter: trigger specialization at zero, otherwise advance countdown. + #[inline] + fn adaptive(&mut self, specialize: impl FnOnce(&mut Self, usize, usize)) { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let counter = self.code.instructions.read_adaptive_counter(cache_base); + if bytecode::adaptive_counter_triggers(counter) { + specialize(self, instr_idx, cache_base); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::advance_adaptive_counter(counter), + ); + } + } + } + + /// Install a specialized opcode and set adaptive cooldown bits. + #[inline] + fn specialize_at(&mut self, instr_idx: usize, cache_base: usize, new_op: Instruction) { + unsafe { + self.code + .instructions + .write_adaptive_counter(cache_base, ADAPTIVE_COOLDOWN_VALUE); + self.code.instructions.replace_op(instr_idx, new_op); + } + } + + #[inline] + fn cooldown_adaptive_at(&mut self, cache_base: usize) { + unsafe { + self.code + .instructions + .write_adaptive_counter(cache_base, ADAPTIVE_COOLDOWN_VALUE); + } + } + + /// Commit a specialization result: replace op on success, backoff on failure. + #[inline] + fn commit_specialization( + &mut self, + instr_idx: usize, + cache_base: usize, + new_op: Option<Instruction>, + ) { + if let Some(new_op) = new_op { + self.specialize_at(instr_idx, cache_base, new_op); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + } + + /// Execute a specialized binary op on two int operands. + /// Fallback to generic binary op if either operand is not an exact int. + #[inline] + fn execute_binary_op_int( + &mut self, + vm: &VirtualMachine, + op: impl FnOnce(&BigInt, &BigInt) -> BigInt, + deopt_op: bytecode::BinaryOperator, + ) -> FrameResult { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_int), Some(b_int)) = ( + a.downcast_ref_if_exact::<PyInt>(vm), + b.downcast_ref_if_exact::<PyInt>(vm), + ) { + let result = op(a_int.as_bigint(), b_int.as_bigint()); + self.pop_value(); + self.pop_value(); + self.push_value(vm.ctx.new_bigint(&result).into()); + Ok(None) + } else { + self.execute_bin_op(vm, deopt_op) + } + } + + /// Execute a specialized binary op on two float operands. + /// Fallback to generic binary op if either operand is not an exact float. + #[inline] + fn execute_binary_op_float( + &mut self, + vm: &VirtualMachine, + op: impl FnOnce(f64, f64) -> f64, + deopt_op: bytecode::BinaryOperator, + ) -> FrameResult { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_f), Some(b_f)) = ( + a.downcast_ref_if_exact::<PyFloat>(vm), + b.downcast_ref_if_exact::<PyFloat>(vm), + ) { + let result = op(a_f.to_f64(), b_f.to_f64()); + self.pop_value(); + self.pop_value(); + self.push_value(vm.ctx.new_float(result).into()); + Ok(None) + } else { + self.execute_bin_op(vm, deopt_op) + } + } + + fn specialize_call( + &mut self, + vm: &VirtualMachine, + nargs: u32, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::Call { .. } + ) { + return; + } + // Stack: [callable, self_or_null, arg1, ..., argN] + // callable is at position nargs + 1 from top + // self_or_null is at position nargs from top + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 1) + .is_some(); + let callable = self.nth_value(nargs + 1); + + if let Some(func) = callable.downcast_ref_if_exact::<PyFunction>(vm) { + if self.specialization_eval_frame_active(vm) { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + if !func.is_optimized_for_call_specialization() { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + let version = func.get_version_for_current_state(); + if version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + let effective_nargs = if self_or_null_is_some { + nargs + 1 + } else { + nargs + }; + + let new_op = if func.can_specialize_call(effective_nargs) { + Instruction::CallPyExactArgs + } else { + Instruction::CallPyGeneral + }; + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, version); + } + self.specialize_at(instr_idx, cache_base, new_op); + return; + } + + // Bound Python method object (`method`) specialization. + if !self_or_null_is_some + && let Some(bound_method) = callable.downcast_ref_if_exact::<PyBoundMethod>(vm) + { + if let Some(func) = bound_method + .function_obj() + .downcast_ref_if_exact::<PyFunction>(vm) + { + if self.specialization_eval_frame_active(vm) { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + if !func.is_optimized_for_call_specialization() { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + let version = func.get_version_for_current_state(); + if version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + let new_op = if func.can_specialize_call(nargs + 1) { + Instruction::CallBoundMethodExactArgs + } else { + Instruction::CallBoundMethodGeneral + }; + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, version); + } + self.specialize_at(instr_idx, cache_base, new_op); + } else { + // Match CPython: bound methods wrapping non-Python callables + // are not specialized as CALL_NON_PY_GENERAL. + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + return; + } + + // Try to specialize method descriptor calls + if let Some(descr) = callable.downcast_ref_if_exact::<PyMethodDescriptor>(vm) { + let call_cache_entries = Instruction::CallListAppend.cache_entries(); + let next_idx = cache_base + call_cache_entries; + let next_is_pop_top = if next_idx < self.code.instructions.len() { + let next_op = self.code.instructions.read_op(next_idx); + matches!(next_op.to_base().unwrap_or(next_op), Instruction::PopTop) + } else { + false + }; + + let call_conv = descr.method.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS); + let total_nargs = nargs + u32::from(self_or_null_is_some); + + let new_op = if call_conv == PyMethodFlags::NOARGS { + if total_nargs != 1 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + Instruction::CallMethodDescriptorNoargs + } else if call_conv == PyMethodFlags::O { + if total_nargs != 2 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + if self_or_null_is_some + && nargs == 1 + && next_is_pop_top + && vm + .callable_cache + .list_append + .as_ref() + .is_some_and(|list_append| callable.is(list_append)) + { + Instruction::CallListAppend + } else { + Instruction::CallMethodDescriptorO + } + } else if call_conv == PyMethodFlags::FASTCALL { + Instruction::CallMethodDescriptorFast + } else if call_conv == (PyMethodFlags::FASTCALL | PyMethodFlags::KEYWORDS) { + Instruction::CallMethodDescriptorFastWithKeywords + } else { + Instruction::CallNonPyGeneral + }; + self.specialize_at(instr_idx, cache_base, new_op); + return; + } + + // Try to specialize builtin calls + if let Some(native) = callable.downcast_ref_if_exact::<PyNativeFunction>(vm) { + let effective_nargs = nargs + u32::from(self_or_null_is_some); + let call_conv = native.value.flags + & (PyMethodFlags::VARARGS + | PyMethodFlags::FASTCALL + | PyMethodFlags::NOARGS + | PyMethodFlags::O + | PyMethodFlags::KEYWORDS); + let new_op = if call_conv == PyMethodFlags::O { + if effective_nargs != 1 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + if native.zelf.is_none() + && nargs == 1 + && vm + .callable_cache + .len + .as_ref() + .is_some_and(|len_callable| callable.is(len_callable)) + { + Instruction::CallLen + } else { + Instruction::CallBuiltinO + } + } else if call_conv == PyMethodFlags::FASTCALL { + if native.zelf.is_none() + && effective_nargs == 2 + && vm + .callable_cache + .isinstance + .as_ref() + .is_some_and(|isinstance_callable| callable.is(isinstance_callable)) + { + Instruction::CallIsinstance + } else { + Instruction::CallBuiltinFast + } + } else if call_conv == (PyMethodFlags::FASTCALL | PyMethodFlags::KEYWORDS) { + Instruction::CallBuiltinFastWithKeywords + } else { + Instruction::CallNonPyGeneral + }; + self.specialize_at(instr_idx, cache_base, new_op); + return; + } + + // type/str/tuple(x) and class-call specializations + if let Some(cls) = callable.downcast_ref::<PyType>() { + if cls.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + if !self_or_null_is_some && nargs == 1 { + let new_op = if callable.is(&vm.ctx.types.type_type.as_object()) { + Some(Instruction::CallType1) + } else if callable.is(&vm.ctx.types.str_type.as_object()) { + Some(Instruction::CallStr1) + } else if callable.is(&vm.ctx.types.tuple_type.as_object()) { + Some(Instruction::CallTuple1) + } else { + None + }; + if let Some(new_op) = new_op { + self.specialize_at(instr_idx, cache_base, new_op); + return; + } + } + if cls.slots.vectorcall.load().is_some() { + self.specialize_at(instr_idx, cache_base, Instruction::CallBuiltinClass); + return; + } + self.specialize_at(instr_idx, cache_base, Instruction::CallNonPyGeneral); + return; + } + + // CPython only considers CALL_ALLOC_AND_ENTER_INIT for types whose + // metaclass is exactly `type`. + if !callable.class().is(vm.ctx.types.type_type) { + self.specialize_at(instr_idx, cache_base, Instruction::CallNonPyGeneral); + return; + } + + // CallAllocAndEnterInit: heap type with default __new__ + if !self_or_null_is_some && cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + let object_new = vm.ctx.types.object_type.slots.new.load(); + let cls_new = cls.slots.new.load(); + let object_alloc = vm.ctx.types.object_type.slots.alloc.load(); + let cls_alloc = cls.slots.alloc.load(); + if let (Some(cls_new_fn), Some(obj_new_fn), Some(cls_alloc_fn), Some(obj_alloc_fn)) = + (cls_new, object_new, cls_alloc, object_alloc) + && cls_new_fn as usize == obj_new_fn as usize + && cls_alloc_fn as usize == obj_alloc_fn as usize + { + let init = cls.get_attr(identifier!(vm, __init__)); + let mut version = cls.tp_version_tag.load(Acquire); + if version == 0 { + version = cls.assign_version_tag(); + } + if version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + if let Some(init) = init + && let Some(init_func) = init.downcast_ref_if_exact::<PyFunction>(vm) + && init_func.is_simple_for_call_specialization() + && cls.cache_init_for_specialization(init_func.to_owned(), version, vm) + { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, version); + } + self.specialize_at( + instr_idx, + cache_base, + Instruction::CallAllocAndEnterInit, + ); + return; + } + } + } + self.specialize_at(instr_idx, cache_base, Instruction::CallNonPyGeneral); + return; + } + + // General fallback: specialized non-Python callable path + self.specialize_at(instr_idx, cache_base, Instruction::CallNonPyGeneral); + } + + fn specialize_call_kw( + &mut self, + vm: &VirtualMachine, + nargs: u32, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::CallKw { .. } + ) { + return; + } + // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] + // callable is at position nargs + 2 from top + let stack_len = self.localsplus.stack_len(); + let self_or_null_is_some = self + .localsplus + .stack_index(stack_len - nargs as usize - 2) + .is_some(); + let callable = self.nth_value(nargs + 2); + + if let Some(func) = callable.downcast_ref_if_exact::<PyFunction>(vm) { + if self.specialization_eval_frame_active(vm) { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + if !func.is_optimized_for_call_specialization() { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + let version = func.get_version_for_current_state(); + if version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, version); + } + self.specialize_at(instr_idx, cache_base, Instruction::CallKwPy); + return; + } + + if !self_or_null_is_some + && let Some(bound_method) = callable.downcast_ref_if_exact::<PyBoundMethod>(vm) + { + if let Some(func) = bound_method + .function_obj() + .downcast_ref_if_exact::<PyFunction>(vm) + { + if self.specialization_eval_frame_active(vm) { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + if !func.is_optimized_for_call_specialization() { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + let version = func.get_version_for_current_state(); + if version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, version); + } + self.specialize_at(instr_idx, cache_base, Instruction::CallKwBoundMethod); + } else { + // Match CPython: bound methods wrapping non-Python callables + // are not specialized as CALL_KW_NON_PY. + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + return; + } + + // General fallback: specialized non-Python callable path + self.specialize_at(instr_idx, cache_base, Instruction::CallKwNonPy); + } + + fn specialize_send(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::Send { .. } + ) { + return; + } + // Stack: [receiver, val] — receiver is at position 1 + let receiver = self.nth_value(1); + let is_exact_gen_or_coro = receiver.downcast_ref_if_exact::<PyGenerator>(vm).is_some() + || receiver.downcast_ref_if_exact::<PyCoroutine>(vm).is_some(); + if is_exact_gen_or_coro && !self.specialization_eval_frame_active(vm) { + self.specialize_at(instr_idx, cache_base, Instruction::SendGen); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + } + + fn specialize_load_super_attr( + &mut self, + vm: &VirtualMachine, + oparg: LoadSuperAttr, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::LoadSuperAttr { .. } + ) { + return; + } + // Stack: [global_super, class, self] + let global_super = self.nth_value(2); + let class = self.nth_value(1); + + if !global_super.is(&vm.ctx.types.super_type.as_object()) + || class.downcast_ref::<PyType>().is_none() + { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + let new_op = if oparg.is_load_method() { + Instruction::LoadSuperAttrMethod + } else { + Instruction::LoadSuperAttrAttr + }; + self.specialize_at(instr_idx, cache_base, new_op); + } + + fn specialize_compare_op( + &mut self, + vm: &VirtualMachine, + op: bytecode::ComparisonOperator, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::CompareOp { .. } + ) { + return; + } + let b = self.top_value(); + let a = self.nth_value(1); + + let new_op = if let (Some(a_int), Some(b_int)) = ( + a.downcast_ref_if_exact::<PyInt>(vm), + b.downcast_ref_if_exact::<PyInt>(vm), + ) { + if specialization_compact_int_value(a_int, vm).is_some() + && specialization_compact_int_value(b_int, vm).is_some() + { + Some(Instruction::CompareOpInt) + } else { + None + } + } else if a.downcast_ref_if_exact::<PyFloat>(vm).is_some() + && b.downcast_ref_if_exact::<PyFloat>(vm).is_some() + { + Some(Instruction::CompareOpFloat) + } else if a.downcast_ref_if_exact::<PyStr>(vm).is_some() + && b.downcast_ref_if_exact::<PyStr>(vm).is_some() + && (op == bytecode::ComparisonOperator::Equal + || op == bytecode::ComparisonOperator::NotEqual) + { + Some(Instruction::CompareOpStr) + } else { + None + }; + + self.commit_specialization(instr_idx, cache_base, new_op); + } + + /// Recover the ComparisonOperator from the instruction arg byte. + /// `replace_op` preserves the arg byte, so the original op remains accessible. + fn compare_op_from_arg(&self, arg: bytecode::OpArg) -> PyComparisonOp { + bytecode::ComparisonOperator::try_from(u32::from(arg)) + .unwrap_or(bytecode::ComparisonOperator::Equal) + .into() + } + + /// Recover the BinaryOperator from the instruction arg byte. + /// `replace_op` preserves the arg byte, so the original op remains accessible. + fn binary_op_from_arg(&self, arg: bytecode::OpArg) -> bytecode::BinaryOperator { + bytecode::BinaryOperator::try_from(u32::from(arg)).unwrap_or(bytecode::BinaryOperator::Add) + } + + fn specialize_to_bool(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::ToBool + ) { + return; + } + let obj = self.top_value(); + let cls = obj.class(); + + let new_op = if cls.is(vm.ctx.types.bool_type) { + Some(Instruction::ToBoolBool) + } else if cls.is(PyInt::class(&vm.ctx)) { + Some(Instruction::ToBoolInt) + } else if cls.is(vm.ctx.types.none_type) { + Some(Instruction::ToBoolNone) + } else if cls.is(PyList::class(&vm.ctx)) { + Some(Instruction::ToBoolList) + } else if cls.is(PyStr::class(&vm.ctx)) { + Some(Instruction::ToBoolStr) + } else if cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + && cls.slots.as_number.boolean.load().is_none() + && cls.slots.as_mapping.length.load().is_none() + && cls.slots.as_sequence.length.load().is_none() + { + // Cache type version for ToBoolAlwaysTrue guard + let mut type_version = cls.tp_version_tag.load(Acquire); + if type_version == 0 { + type_version = cls.assign_version_tag(); + } + if type_version != 0 { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + } + self.specialize_at(instr_idx, cache_base, Instruction::ToBoolAlwaysTrue); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + return; + } else { + None + }; + + self.commit_specialization(instr_idx, cache_base, new_op); + } + + fn specialize_for_iter( + &mut self, + vm: &VirtualMachine, + jump_delta: u32, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::ForIter { .. } + ) { + return; + } + let iter = self.top_value(); + + let new_op = if iter.downcast_ref_if_exact::<PyRangeIterator>(vm).is_some() { + Some(Instruction::ForIterRange) + } else if iter.downcast_ref_if_exact::<PyListIterator>(vm).is_some() { + Some(Instruction::ForIterList) + } else if iter.downcast_ref_if_exact::<PyTupleIterator>(vm).is_some() { + Some(Instruction::ForIterTuple) + } else if iter.downcast_ref_if_exact::<PyGenerator>(vm).is_some() + && jump_delta <= i16::MAX as u32 + && self.for_iter_has_end_for_shape(instr_idx, jump_delta) + && !self.specialization_eval_frame_active(vm) + { + Some(Instruction::ForIterGen) + } else { + None + }; + + self.commit_specialization(instr_idx, cache_base, new_op); + } + + #[inline] + fn specialization_eval_frame_active(&self, vm: &VirtualMachine) -> bool { + vm.use_tracing.get() + } + + #[inline] + fn specialization_has_datastack_space_for_func( + &self, + vm: &VirtualMachine, + func: &Py<PyFunction>, + ) -> bool { + self.specialization_has_datastack_space_for_func_with_extra(vm, func, 0) + } + + #[inline] + fn specialization_has_datastack_space_for_func_with_extra( + &self, + vm: &VirtualMachine, + func: &Py<PyFunction>, + extra_bytes: usize, + ) -> bool { + match func.datastack_frame_size_bytes() { + Some(frame_size) => frame_size + .checked_add(extra_bytes) + .is_some_and(|size| vm.datastack_has_space(size)), + None => extra_bytes == 0 || vm.datastack_has_space(extra_bytes), + } + } + + #[inline] + fn specialization_call_recursion_guard(&self, vm: &VirtualMachine) -> bool { + self.specialization_call_recursion_guard_with_extra_frames(vm, 0) + } + + #[inline] + fn specialization_call_recursion_guard_with_extra_frames( + &self, + vm: &VirtualMachine, + extra_frames: usize, + ) -> bool { + vm.current_recursion_depth() + .saturating_add(1) + .saturating_add(extra_frames) + >= vm.recursion_limit.get() + } + + #[inline] + fn for_iter_has_end_for_shape(&self, instr_idx: usize, jump_delta: u32) -> bool { + let target_idx = instr_idx + + 1 + + Instruction::ForIter { + delta: Arg::marker(), + } + .cache_entries() + + jump_delta as usize; + self.code.instructions.get(target_idx).is_some_and(|unit| { + matches!( + unit.op, + Instruction::EndFor | Instruction::InstrumentedEndFor + ) + }) + } + + /// Handle iterator exhaustion in specialized FOR_ITER handlers. + /// Skips END_FOR if present at target and jumps. + fn for_iter_jump_on_exhausted(&mut self, target: bytecode::Label) { + let target_idx = target.as_usize(); + let jump_target = if let Some(unit) = self.code.instructions.get(target_idx) { + if matches!( + unit.op, + bytecode::Instruction::EndFor | bytecode::Instruction::InstrumentedEndFor + ) { + bytecode::Label::new(target.as_u32() + 1) + } else { + target + } + } else { + target + }; + self.jump(jump_target); + } + + fn specialize_load_global( + &mut self, + vm: &VirtualMachine, + oparg: u32, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::LoadGlobal { .. } + ) { + return; + } + let name = self.code.names[(oparg >> 1) as usize]; + let Ok(globals_version) = u16::try_from(self.globals.version()) else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + }; + + if let Ok(Some(globals_hint)) = self.globals.hint_for_key(name, vm) { + unsafe { + self.code + .instructions + .write_cache_u16(cache_base + 1, globals_version); + self.code.instructions.write_cache_u16(cache_base + 2, 0); + self.code + .instructions + .write_cache_u16(cache_base + 3, globals_hint); + } + self.specialize_at(instr_idx, cache_base, Instruction::LoadGlobalModule); + return; + } + + if let Some(builtins_dict) = self.builtins.downcast_ref_if_exact::<PyDict>(vm) + && let Ok(Some(builtins_hint)) = builtins_dict.hint_for_key(name, vm) + && let Ok(builtins_version) = u16::try_from(builtins_dict.version()) + { + unsafe { + self.code + .instructions + .write_cache_u16(cache_base + 1, globals_version); + self.code + .instructions + .write_cache_u16(cache_base + 2, builtins_version); + self.code + .instructions + .write_cache_u16(cache_base + 3, builtins_hint); + } + self.specialize_at(instr_idx, cache_base, Instruction::LoadGlobalBuiltin); + return; + } + + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + + fn specialize_store_subscr( + &mut self, + vm: &VirtualMachine, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::StoreSubscr + ) { + return; + } + // Stack: [value, obj, idx] — obj is TOS-1 + let obj = self.nth_value(1); + let idx = self.top_value(); + + let new_op = if let (Some(list), Some(int_idx)) = ( + obj.downcast_ref_if_exact::<PyList>(vm), + idx.downcast_ref_if_exact::<PyInt>(vm), + ) { + let list_len = list.borrow_vec().len(); + if specialization_nonnegative_compact_index(int_idx, vm).is_some_and(|i| i < list_len) { + Some(Instruction::StoreSubscrListInt) + } else { + None + } + } else if obj.downcast_ref_if_exact::<PyDict>(vm).is_some() { + Some(Instruction::StoreSubscrDict) + } else { + None + }; + + self.commit_specialization(instr_idx, cache_base, new_op); + } + + fn specialize_contains_op(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::ContainsOp { .. } + ) { + return; + } + let haystack = self.top_value(); // b = TOS = haystack + let new_op = if haystack.downcast_ref_if_exact::<PyDict>(vm).is_some() { + Some(Instruction::ContainsOpDict) + } else if haystack.downcast_ref_if_exact::<PySet>(vm).is_some() + || haystack.downcast_ref_if_exact::<PyFrozenSet>(vm).is_some() + { + Some(Instruction::ContainsOpSet) + } else { + None + }; + + self.commit_specialization(instr_idx, cache_base, new_op); + } + + fn specialize_unpack_sequence( + &mut self, + vm: &VirtualMachine, + expected_count: u32, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::UnpackSequence { .. } + ) { + return; + } + let obj = self.top_value(); + let new_op = if let Some(tuple) = obj.downcast_ref_if_exact::<PyTuple>(vm) { + if tuple.len() != expected_count as usize { + None + } else if expected_count == 2 { + Some(Instruction::UnpackSequenceTwoTuple) + } else { + Some(Instruction::UnpackSequenceTuple) + } + } else if let Some(list) = obj.downcast_ref_if_exact::<PyList>(vm) { + if list.borrow_vec().len() == expected_count as usize { + Some(Instruction::UnpackSequenceList) + } else { + None + } + } else { + None + }; + + self.commit_specialization(instr_idx, cache_base, new_op); + } + + fn specialize_store_attr( + &mut self, + vm: &VirtualMachine, + attr_idx: bytecode::NameIdx, + instr_idx: usize, + cache_base: usize, + ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::StoreAttr { .. } + ) { + return; + } + // TOS = owner (the object being assigned to) + let owner = self.top_value(); + let cls = owner.class(); + + // Only specialize if setattr is the default (generic_setattr) + let is_default_setattr = cls + .slots + .setattro + .load() + .is_some_and(|f| f as usize == PyBaseObject::slot_setattro as *const () as usize); + if !is_default_setattr { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + // Get or assign type version + let mut type_version = cls.tp_version_tag.load(Acquire); + if type_version == 0 { + type_version = cls.assign_version_tag(); + } + if type_version == 0 { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + + // Check for data descriptor + let attr_name = self.code.names[attr_idx as usize]; + let cls_attr = cls.get_attr(attr_name); + let has_data_descr = cls_attr.as_ref().is_some_and(|descr| { + let descr_cls = descr.class(); + descr_cls.slots.descr_get.load().is_some() && descr_cls.slots.descr_set.load().is_some() + }); + + if has_data_descr { + // Check for member descriptor (slot access) + if let Some(ref descr) = cls_attr + && let Some(member_descr) = descr.downcast_ref::<PyMemberDescriptor>() + && let MemberGetter::Offset(offset) = member_descr.member.getter + { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + self.code + .instructions + .write_cache_u16(cache_base + 3, offset as u16); + } + self.specialize_at(instr_idx, cache_base, Instruction::StoreAttrSlot); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + } else if let Some(dict) = owner.dict() { + let use_hint = match dict.get_item_opt(attr_name, vm) { + Ok(Some(_)) => true, + Ok(None) => false, + Err(_) => { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + return; + } + }; + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + } + self.specialize_at( + instr_idx, + cache_base, + if use_hint { + Instruction::StoreAttrWithHint + } else { + Instruction::StoreAttrInstanceValue + }, + ); + } else { + unsafe { + self.code.instructions.write_adaptive_counter( + cache_base, + bytecode::adaptive_counter_backoff( + self.code.instructions.read_adaptive_counter(cache_base), + ), + ); + } + } + } + + fn load_super_attr(&mut self, vm: &VirtualMachine, oparg: LoadSuperAttr) -> FrameResult { + let attr_name = self.code.names[oparg.name_idx() as usize]; + + // Stack layout (bottom to top): [super, class, self] + // Pop in LIFO order: self, class, super + let self_obj = self.pop_value(); + let class = self.pop_value(); + let global_super = self.pop_value(); + + // Create super object - pass args based on has_class flag + // When super is shadowed, has_class=false means call with 0 args + let super_obj = if oparg.has_class() { + global_super.call((class.clone(), self_obj.clone()), vm)? + } else { + global_super.call((), vm)? + }; + + if oparg.is_load_method() { + // Method load: push [method, self_or_null] + let method = PyMethod::get(super_obj, attr_name, vm)?; + match method { + PyMethod::Function { target: _, func } => { + self.push_value(func); + self.push_value(self_obj); + } + PyMethod::Attribute(val) => { + self.push_value(val); + self.push_null(); + } + } + } else { + // Regular attribute access + let obj = super_obj.get_attr(attr_name, vm)?; + self.push_value(obj); + } + Ok(None) + } + + fn store_attr(&mut self, vm: &VirtualMachine, attr: bytecode::NameIdx) -> FrameResult { + let attr_name = self.code.names[attr as usize]; + let parent = self.pop_value(); + let value = self.pop_value(); + parent.set_attr(attr_name, value, vm)?; + Ok(None) + } + + fn delete_attr(&mut self, vm: &VirtualMachine, attr: bytecode::NameIdx) -> FrameResult { + let attr_name = self.code.names[attr as usize]; + let parent = self.pop_value(); + parent.del_attr(attr_name, vm)?; + Ok(None) + } + + // Block stack functions removed - exception table handles all exception/cleanup + + #[inline] + #[track_caller] + fn push_stackref_opt(&mut self, obj: Option<PyStackRef>) { + match self.localsplus.stack_try_push(obj) { + Ok(()) => {} + Err(_e) => self.fatal("tried to push value onto stack but overflowed max_stackdepth"), + } + } + + #[inline] + #[track_caller] // not a real track_caller but push_value is less useful for debugging + fn push_value_opt(&mut self, obj: Option<PyObjectRef>) { + self.push_stackref_opt(obj.map(PyStackRef::new_owned)); + } + + #[inline] + #[track_caller] + fn push_value(&mut self, obj: PyObjectRef) { + self.push_stackref_opt(Some(PyStackRef::new_owned(obj))); + } + + /// Push a borrowed reference onto the stack (no refcount increment). + /// + /// # Safety + /// The object must remain alive until the borrowed ref is consumed. + /// The compiler guarantees consumption within the same basic block. + #[inline] + #[track_caller] + #[allow(dead_code)] + unsafe fn push_borrowed(&mut self, obj: &PyObject) { + self.push_stackref_opt(Some(unsafe { PyStackRef::new_borrowed(obj) })); + } + + #[inline] + fn push_null(&mut self) { + self.push_stackref_opt(None); + } + + /// Pop a raw stackref from the stack, returning None if the stack slot is NULL. + #[inline] + fn pop_stackref_opt(&mut self) -> Option<PyStackRef> { + if self.localsplus.stack_is_empty() { + self.fatal("tried to pop from empty stack"); + } + self.localsplus.stack_pop() + } + + /// Pop a raw stackref from the stack. Panics if NULL. + #[inline] + #[track_caller] + fn pop_stackref(&mut self) -> PyStackRef { + expect_unchecked( + self.pop_stackref_opt(), + "pop stackref but null found. This is a compiler bug.", + ) + } + + /// Pop a value from the stack, returning None if the stack slot is NULL. + /// Automatically promotes borrowed refs to owned. + #[inline] + fn pop_value_opt(&mut self) -> Option<PyObjectRef> { + self.pop_stackref_opt().map(|sr| sr.to_pyobj()) + } + + #[inline] + #[track_caller] + fn pop_value(&mut self) -> PyObjectRef { + self.pop_stackref().to_pyobj() + } + + fn call_intrinsic_1( + &mut self, + func: bytecode::IntrinsicFunction1, + arg: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match func { + bytecode::IntrinsicFunction1::Print => { + let displayhook = vm + .sys_module + .get_attr("displayhook", vm) + .map_err(|_| vm.new_runtime_error("lost sys.displayhook"))?; + displayhook.call((arg,), vm) + } + bytecode::IntrinsicFunction1::ImportStar => { + // arg is the module object + self.push_value(arg); // Push module back on stack for import_star + self.import_star(vm)?; + Ok(vm.ctx.none()) + } + bytecode::IntrinsicFunction1::UnaryPositive => vm._pos(&arg), + bytecode::IntrinsicFunction1::SubscriptGeneric => { + // Used for PEP 695: Generic[*type_params] + crate::builtins::genericalias::subscript_generic(arg, vm) + } + bytecode::IntrinsicFunction1::TypeVar => { + let type_var: PyObjectRef = + _typing::TypeVar::new(vm, arg.clone(), vm.ctx.none(), vm.ctx.none()) + .into_ref(&vm.ctx) + .into(); + Ok(type_var) + } + bytecode::IntrinsicFunction1::ParamSpec => { + let param_spec: PyObjectRef = _typing::ParamSpec::new(arg.clone(), vm) + .into_ref(&vm.ctx) + .into(); + Ok(param_spec) + } + bytecode::IntrinsicFunction1::TypeVarTuple => { + let type_var_tuple: PyObjectRef = _typing::TypeVarTuple::new(arg.clone(), vm) + .into_ref(&vm.ctx) + .into(); + Ok(type_var_tuple) + } + bytecode::IntrinsicFunction1::TypeAlias => { + // TypeAlias receives a tuple of (name, type_params, value) + let tuple: PyTupleRef = arg + .downcast() + .map_err(|_| vm.new_type_error("TypeAlias expects a tuple argument"))?; + + if tuple.len() != 3 { + return Err(vm.new_type_error(format!( + "TypeAlias expects exactly 3 arguments, got {}", + tuple.len() + ))); + } + + let name = tuple.as_slice()[0].clone(); + let type_params_obj = tuple.as_slice()[1].clone(); + let compute_value = tuple.as_slice()[2].clone(); + + let type_params: PyTupleRef = if vm.is_none(&type_params_obj) { + vm.ctx.empty_tuple.clone() + } else { + type_params_obj + .downcast() + .map_err(|_| vm.new_type_error("Type params must be a tuple."))? + }; + + let name = name + .downcast::<crate::builtins::PyStr>() + .map_err(|_| vm.new_type_error("TypeAliasType name must be a string"))?; + let type_alias = _typing::TypeAliasType::new(name, type_params, compute_value); + Ok(type_alias.into_ref(&vm.ctx).into()) + } + bytecode::IntrinsicFunction1::ListToTuple => { + // Convert list to tuple + let list = arg + .downcast::<PyList>() + .map_err(|_| vm.new_type_error("LIST_TO_TUPLE expects a list"))?; + Ok(vm.ctx.new_tuple(list.borrow_vec().to_vec()).into()) + } + bytecode::IntrinsicFunction1::StopIterationError => { + // Convert StopIteration to RuntimeError + // Used to ensure async generators don't raise StopIteration directly + // _PyGen_FetchStopIterationValue + // Use fast_isinstance to handle subclasses of StopIteration + if arg.fast_isinstance(vm.ctx.exceptions.stop_iteration) { + Err(vm.new_runtime_error("coroutine raised StopIteration")) + } else { + // If not StopIteration, just re-raise the original exception + Err(arg.downcast().unwrap_or_else(|obj| { + vm.new_runtime_error(format!( + "unexpected exception type: {:?}", + obj.class() + )) + })) + } + } + bytecode::IntrinsicFunction1::AsyncGenWrap => { + // Wrap value for async generator + // Creates an AsyncGenWrappedValue + Ok(crate::builtins::asyncgenerator::PyAsyncGenWrappedValue(arg) + .into_ref(&vm.ctx) + .into()) + } + } + } + + fn call_intrinsic_2( + &mut self, + func: bytecode::IntrinsicFunction2, + arg1: PyObjectRef, + arg2: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match func { + bytecode::IntrinsicFunction2::SetTypeparamDefault => { + crate::stdlib::_typing::set_typeparam_default(arg1, arg2, vm) + } + bytecode::IntrinsicFunction2::SetFunctionTypeParams => { + // arg1 is the function, arg2 is the type params tuple + // Set __type_params__ attribute on the function + arg1.set_attr("__type_params__", arg2, vm)?; + Ok(arg1) + } + bytecode::IntrinsicFunction2::TypeVarWithBound => { + let type_var: PyObjectRef = + _typing::TypeVar::new(vm, arg1.clone(), arg2, vm.ctx.none()) + .into_ref(&vm.ctx) + .into(); + Ok(type_var) + } + bytecode::IntrinsicFunction2::TypeVarWithConstraint => { + let type_var: PyObjectRef = + _typing::TypeVar::new(vm, arg1.clone(), vm.ctx.none(), arg2) + .into_ref(&vm.ctx) + .into(); + Ok(type_var) + } + bytecode::IntrinsicFunction2::PrepReraiseStar => { + // arg1 = orig (original exception) + // arg2 = excs (list of exceptions raised/reraised in except* blocks) + // Returns: exception to reraise, or None if nothing to reraise + crate::exceptions::prep_reraise_star(arg1, arg2, vm) + } + } + } + + /// Pop multiple values from the stack. Panics if any slot is NULL. + fn pop_multiple(&mut self, count: usize) -> impl ExactSizeIterator<Item = PyObjectRef> + '_ { + let stack_len = self.localsplus.stack_len(); + if count > stack_len { + let instr = self.code.instructions.get(self.lasti() as usize); + let op_name = instr + .map(|i| format!("{:?}", i.op)) + .unwrap_or_else(|| "None".to_string()); + panic!( + "Stack underflow in pop_multiple: trying to pop {} elements from stack with {} elements. lasti={}, code={}, op={}, source_path={}", + count, + stack_len, + self.lasti(), + self.code.obj_name, + op_name, + self.code.source_path() + ); + } + self.localsplus.stack_drain(stack_len - count).map(|obj| { + expect_unchecked(obj, "pop_multiple but null found. This is a compiler bug.").to_pyobj() + }) + } + + #[inline] + fn replace_top(&mut self, top: Option<PyObjectRef>) -> Option<PyObjectRef> { + let mut slot = top.map(PyStackRef::new_owned); + let last = self.localsplus.stack_last_mut().unwrap(); + core::mem::swap(last, &mut slot); + slot.map(|sr| sr.to_pyobj()) + } + + #[inline] + #[track_caller] + fn top_value(&self) -> &PyObject { + match self.localsplus.stack_last() { + Some(Some(last)) => last.as_object(), + Some(None) => self.fatal("tried to get top of stack but got NULL"), + None => self.fatal("tried to get top of stack but stack is empty"), + } + } + + #[inline] + #[track_caller] + fn nth_value(&self, depth: u32) -> &PyObject { + let idx = self.localsplus.stack_len() - depth as usize - 1; + match self.localsplus.stack_index(idx) { + Some(obj) => obj.as_object(), + None => unsafe { core::hint::unreachable_unchecked() }, + } + } + + #[cold] + #[inline(never)] + #[track_caller] + fn fatal(&self, msg: &'static str) -> ! { + dbg!(self); + panic!("{msg}") + } +} + +impl fmt::Debug for Frame { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: Debug is best-effort; concurrent mutation is unlikely + // and would only affect debug output. + let iframe = unsafe { &*self.iframe.get() }; + let stack_str = + iframe + .localsplus + .stack_as_slice() + .iter() + .fold(String::new(), |mut s, slot| { + match slot { + Some(elem) if elem.downcastable::<Self>() => { + s.push_str("\n > {frame}"); + } + Some(elem) => { + core::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); + } + None => { + s.push_str("\n > NULL"); + } + } + s + }); + // TODO: fix this up + write!( + f, + "Frame Object {{ \n Stack:{}\n Locals initialized:{}\n}}", + stack_str, + self.locals.get().is_some() + ) + } +} + +/// _PyEval_SpecialMethodCanSuggest +fn special_method_can_suggest( + obj: &PyObjectRef, + oparg: SpecialMethod, + vm: &VirtualMachine, +) -> PyResult<bool> { + Ok(match oparg { + SpecialMethod::Enter | SpecialMethod::Exit => { + vm.get_special_method(obj, get_special_method_name(SpecialMethod::AEnter, vm))? + .is_some() + && vm + .get_special_method(obj, get_special_method_name(SpecialMethod::AExit, vm))? + .is_some() + } + SpecialMethod::AEnter | SpecialMethod::AExit => { + vm.get_special_method(obj, get_special_method_name(SpecialMethod::Enter, vm))? + .is_some() + && vm + .get_special_method(obj, get_special_method_name(SpecialMethod::Exit, vm))? + .is_some() + } + }) +} + +fn get_special_method_name(oparg: SpecialMethod, vm: &VirtualMachine) -> &'static PyStrInterned { + match oparg { + SpecialMethod::Enter => identifier!(vm, __enter__), + SpecialMethod::Exit => identifier!(vm, __exit__), + SpecialMethod::AEnter => identifier!(vm, __aenter__), + SpecialMethod::AExit => identifier!(vm, __aexit__), + } +} + +/// _Py_SpecialMethod _Py_SpecialMethods +fn get_special_method_error_msg( + oparg: SpecialMethod, + class_name: &str, + can_suggest: bool, +) -> String { + if can_suggest { + match oparg { + SpecialMethod::Enter => format!( + "'{class_name}' object does not support the context manager protocol (missed __enter__ method) but it supports the asynchronous context manager protocol. Did you mean to use 'async with'?" + ), + SpecialMethod::Exit => format!( + "'{class_name}' object does not support the context manager protocol (missed __exit__ method) but it supports the asynchronous context manager protocol. Did you mean to use 'async with'?" + ), + SpecialMethod::AEnter => format!( + "'{class_name}' object does not support the asynchronous context manager protocol (missed __aenter__ method) but it supports the context manager protocol. Did you mean to use 'with'?" + ), + SpecialMethod::AExit => format!( + "'{class_name}' object does not support the asynchronous context manager protocol (missed __aexit__ method) but it supports the context manager protocol. Did you mean to use 'with'?" + ), + } + } else { + match oparg { + SpecialMethod::Enter => format!( + "'{class_name}' object does not support the context manager protocol (missed __enter__ method)" + ), + SpecialMethod::Exit => format!( + "'{class_name}' object does not support the context manager protocol (missed __exit__ method)" + ), + SpecialMethod::AEnter => format!( + "'{class_name}' object does not support the asynchronous context manager protocol (missed __aenter__ method)" + ), + SpecialMethod::AExit => format!( + "'{class_name}' object does not support the asynchronous context manager protocol (missed __aexit__ method)" + ), + } + } +} + +fn is_module_initializing(module: &PyObject, vm: &VirtualMachine) -> bool { + let Ok(spec) = module.get_attr(&vm.ctx.new_str("__spec__"), vm) else { + return false; + }; + if vm.is_none(&spec) { + return false; + } + let Ok(initializing_attr) = spec.get_attr(&vm.ctx.new_str("_initializing"), vm) else { + return false; + }; + initializing_attr.try_to_bool(vm).unwrap_or(false) +} + +fn expect_unchecked<T: fmt::Debug>(optional: Option<T>, err_msg: &'static str) -> T { + if cfg!(debug_assertions) { + optional.expect(err_msg) + } else { + unsafe { optional.unwrap_unchecked() } + } +} diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs new file mode 100644 index 00000000000..b39ee6f6bca --- /dev/null +++ b/crates/vm/src/function/argument.rs @@ -0,0 +1,658 @@ +use crate::{ + AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBaseExceptionRef, PyTupleRef, PyTypeRef}, + convert::ToPyObject, + object::{Traverse, TraverseFn}, +}; +use core::ops::RangeInclusive; +use indexmap::IndexMap; +use itertools::Itertools; + +pub trait IntoFuncArgs: Sized { + fn into_args(self, vm: &VirtualMachine) -> FuncArgs; + fn into_method_args(self, obj: PyObjectRef, vm: &VirtualMachine) -> FuncArgs { + let mut args = self.into_args(vm); + args.prepend_arg(obj); + args + } +} + +impl<T> IntoFuncArgs for T +where + T: Into<FuncArgs>, +{ + fn into_args(self, _vm: &VirtualMachine) -> FuncArgs { + self.into() + } +} + +// A tuple of values that each implement `ToPyObject` represents a sequence of +// arguments that can be bound and passed to a built-in function. +macro_rules! into_func_args_from_tuple { + ($(($n:tt, $T:ident)),*) => { + impl<$($T,)*> IntoFuncArgs for ($($T,)*) + where + $($T: ToPyObject,)* + { + #[inline] + fn into_args(self, vm: &VirtualMachine) -> FuncArgs { + let ($($n,)*) = self; + PosArgs::new(vec![$($n.to_pyobject(vm),)*]).into() + } + + #[inline] + fn into_method_args(self, obj: PyObjectRef, vm: &VirtualMachine) -> FuncArgs { + let ($($n,)*) = self; + PosArgs::new(vec![obj, $($n.to_pyobject(vm),)*]).into() + } + } + }; +} + +into_func_args_from_tuple!((v1, T1)); +into_func_args_from_tuple!((v1, T1), (v2, T2)); +into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3)); +into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4)); +into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4), (v5, T5)); +into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4), (v5, T5), (v6, T6)); +// We currently allows only 6 unnamed positional arguments. +// Please use `#[derive(FromArgs)]` and a struct for more complex argument parsing. +// The number of limitation came from: +// https://rust-lang.github.io/rust-clippy/master/index.html#too_many_arguments + +/// The `FuncArgs` struct is one of the most used structs then creating +/// a rust function that can be called from python. It holds both positional +/// arguments, as well as keyword arguments passed to the function. +#[derive(Debug, Default, Clone, Traverse)] +pub struct FuncArgs { + pub args: Vec<PyObjectRef>, + // sorted map, according to https://www.python.org/dev/peps/pep-0468/ + pub kwargs: IndexMap<String, PyObjectRef>, +} + +unsafe impl Traverse for IndexMap<String, PyObjectRef> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.values().for_each(|v| v.traverse(tracer_fn)); + } +} + +/// Conversion from vector of python objects to function arguments. +impl<A> From<A> for FuncArgs +where + A: Into<PosArgs>, +{ + fn from(args: A) -> Self { + Self { + args: args.into().into_vec(), + kwargs: IndexMap::new(), + } + } +} + +impl From<KwArgs> for FuncArgs { + fn from(kwargs: KwArgs) -> Self { + Self { + args: Vec::new(), + kwargs: kwargs.0, + } + } +} + +impl FromArgs for FuncArgs { + fn from_args(_vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { + Ok(core::mem::take(args)) + } +} + +impl FuncArgs { + pub fn new<A, K>(args: A, kwargs: K) -> Self + where + A: Into<PosArgs>, + K: Into<KwArgs>, + { + let PosArgs(args) = args.into(); + let KwArgs(kwargs) = kwargs.into(); + Self { args, kwargs } + } + + pub fn with_kwargs_names<A, KW>(mut args: A, kwarg_names: KW) -> Self + where + A: ExactSizeIterator<Item = PyObjectRef>, + KW: ExactSizeIterator<Item = String>, + { + // last `kwarg_names.len()` elements of args in order of appearance in the call signature + let total_argc = args.len(); + let kwarg_count = kwarg_names.len(); + let pos_arg_count = total_argc - kwarg_count; + + let pos_args = args.by_ref().take(pos_arg_count).collect(); + + let kwargs = kwarg_names.zip_eq(args).collect::<IndexMap<_, _>>(); + + Self { + args: pos_args, + kwargs, + } + } + + /// Create FuncArgs from a vectorcall-style argument slice (PEP 590). + /// `args[..nargs]` are positional, and if `kwnames` is provided, + /// the last `kwnames.len()` entries in `args[nargs..]` are keyword values. + /// Convert borrowed vectorcall args to FuncArgs (clones all values). + pub fn from_vectorcall( + args: &[PyObjectRef], + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + ) -> Self { + debug_assert!(nargs <= args.len()); + debug_assert!(kwnames.is_none_or(|kw| nargs + kw.len() <= args.len())); + let pos_args = args[..nargs].to_vec(); + let kwargs = if let Some(names) = kwnames { + names + .iter() + .zip(&args[nargs..nargs + names.len()]) + .map(|(name, val)| { + let key = name + .downcast_ref::<crate::builtins::PyUtf8Str>() + .expect("kwnames must be strings") + .as_str() + .to_owned(); + (key, val.clone()) + }) + .collect() + } else { + IndexMap::new() + }; + Self { + args: pos_args, + kwargs, + } + } + + /// Convert owned vectorcall args to FuncArgs (moves values, no clone). + pub fn from_vectorcall_owned( + mut args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + ) -> Self { + debug_assert!(nargs <= args.len()); + debug_assert!(kwnames.is_none_or(|kw| nargs + kw.len() <= args.len())); + let kwargs = if let Some(names) = kwnames { + let kw_count = names.len(); + names + .iter() + .zip(args.drain(nargs..nargs + kw_count)) + .map(|(name, val)| { + let key = name + .downcast_ref::<crate::builtins::PyUtf8Str>() + .expect("kwnames must be strings") + .as_str() + .to_owned(); + (key, val) + }) + .collect() + } else { + IndexMap::new() + }; + args.truncate(nargs); + Self { args, kwargs } + } + + pub fn is_empty(&self) -> bool { + self.args.is_empty() && self.kwargs.is_empty() + } + + pub fn prepend_arg(&mut self, item: PyObjectRef) { + self.args.reserve_exact(1); + self.args.insert(0, item) + } + + pub fn shift(&mut self) -> PyObjectRef { + self.args.remove(0) + } + + pub fn get_kwarg(&self, key: &str, default: PyObjectRef) -> PyObjectRef { + self.kwargs + .get(key) + .cloned() + .unwrap_or_else(|| default.clone()) + } + + pub fn get_optional_kwarg(&self, key: &str) -> Option<PyObjectRef> { + self.kwargs.get(key).cloned() + } + + pub fn get_optional_kwarg_with_type( + &self, + key: &str, + ty: PyTypeRef, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + match self.get_optional_kwarg(key) { + Some(kwarg) => { + if kwarg.fast_isinstance(&ty) { + Ok(Some(kwarg)) + } else { + let expected_ty_name = &ty.name(); + let kwarg_class = kwarg.class(); + let actual_ty_name = &kwarg_class.name(); + Err(vm.new_type_error(format!( + "argument of type {expected_ty_name} is required for named parameter `{key}` (got: {actual_ty_name})" + ))) + } + } + None => Ok(None), + } + } + + pub fn take_positional(&mut self) -> Option<PyObjectRef> { + if self.args.is_empty() { + None + } else { + Some(self.args.remove(0)) + } + } + + pub fn take_positional_keyword(&mut self, name: &str) -> Option<PyObjectRef> { + self.take_positional().or_else(|| self.take_keyword(name)) + } + + pub fn take_keyword(&mut self, name: &str) -> Option<PyObjectRef> { + self.kwargs.swap_remove(name) + } + + pub fn remaining_keywords(&mut self) -> impl Iterator<Item = (String, PyObjectRef)> + '_ { + self.kwargs.drain(..) + } + + /// Binds these arguments to their respective values. + /// + /// If there is an insufficient number of arguments, there are leftover + /// arguments after performing the binding, or if an argument is not of + /// the expected type, a TypeError is raised. + /// + /// If the given `FromArgs` includes any conversions, exceptions raised + /// during the conversion will halt the binding and return the error. + pub fn bind<T: FromArgs>(mut self, vm: &VirtualMachine) -> PyResult<T> { + let given_args = self.args.len(); + let bound = T::from_args(vm, &mut self) + .map_err(|e| e.into_exception(T::arity(), given_args, vm))?; + + if !self.args.is_empty() { + Err(vm.new_type_error(format!( + "expected at most {} arguments, got {}", + T::arity().end(), + given_args, + ))) + } else if let Some(err) = self.check_kwargs_empty(vm) { + Err(err) + } else { + Ok(bound) + } + } + + pub fn check_kwargs_empty(&self, vm: &VirtualMachine) -> Option<PyBaseExceptionRef> { + self.kwargs + .keys() + .next() + .map(|k| vm.new_type_error(format!("Unexpected keyword argument {k}"))) + } +} + +/// An error encountered while binding arguments to the parameters of a Python +/// function call. +pub enum ArgumentError { + /// The call provided fewer positional arguments than the function requires. + TooFewArgs, + /// The call provided more positional arguments than the function accepts. + TooManyArgs, + /// The function doesn't accept a keyword argument with the given name. + InvalidKeywordArgument(String), + /// The function require a keyword argument with the given name, but one wasn't provided + RequiredKeywordArgument(String), + /// An exception was raised while binding arguments to the function + /// parameters. + Exception(PyBaseExceptionRef), +} + +impl From<PyBaseExceptionRef> for ArgumentError { + fn from(ex: PyBaseExceptionRef) -> Self { + Self::Exception(ex) + } +} + +impl ArgumentError { + fn into_exception( + self, + arity: RangeInclusive<usize>, + num_given: usize, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + match self { + Self::TooFewArgs => vm.new_type_error(format!( + "expected at least {} arguments, got {}", + arity.start(), + num_given + )), + Self::TooManyArgs => vm.new_type_error(format!( + "expected at most {} arguments, got {}", + arity.end(), + num_given + )), + Self::InvalidKeywordArgument(name) => { + vm.new_type_error(format!("{name} is an invalid keyword argument")) + } + Self::RequiredKeywordArgument(name) => { + vm.new_type_error(format!("Required keyword only argument {name}")) + } + Self::Exception(ex) => ex, + } + } +} + +/// Implemented by any type that can be accepted as a parameter to a built-in +/// function. +/// +pub trait FromArgs: Sized { + /// The range of positional arguments permitted by the function signature. + /// + /// Returns an empty range if not applicable. + fn arity() -> RangeInclusive<usize> { + 0..=0 + } + + /// Extracts this item from the next argument(s). + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError>; +} + +pub trait FromArgOptional { + type Inner: TryFromObject; + fn from_inner(x: Self::Inner) -> Self; +} + +impl<T: TryFromObject> FromArgOptional for OptionalArg<T> { + type Inner = T; + fn from_inner(x: T) -> Self { + Self::Present(x) + } +} + +impl<T: TryFromObject> FromArgOptional for T { + type Inner = Self; + fn from_inner(x: Self) -> Self { + x + } +} + +/// A map of keyword arguments to their values. +/// +/// A built-in function with a `KwArgs` parameter is analogous to a Python +/// function with `**kwargs`. All remaining keyword arguments are extracted +/// (and hence the function will permit an arbitrary number of them). +/// +/// `KwArgs` optionally accepts a generic type parameter to allow type checks +/// or conversions of each argument. +/// +/// Note: +/// +/// KwArgs is only for functions that accept arbitrary keyword arguments. For +/// functions that accept only *specific* named arguments, a rust struct with +/// an appropriate FromArgs implementation must be created. +#[derive(Clone)] +pub struct KwArgs<T = PyObjectRef>(IndexMap<String, T>); + +unsafe impl<T> Traverse for KwArgs<T> +where + T: Traverse, +{ + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.iter().map(|(_, v)| v.traverse(tracer_fn)).count(); + } +} + +impl<T> KwArgs<T> { + pub const fn new(map: IndexMap<String, T>) -> Self { + Self(map) + } + + pub fn pop_kwarg(&mut self, name: &str) -> Option<T> { + self.0.swap_remove(name) + } + + pub fn is_empty(self) -> bool { + self.0.is_empty() + } +} + +impl<T> FromIterator<(String, T)> for KwArgs<T> { + fn from_iter<I: IntoIterator<Item = (String, T)>>(iter: I) -> Self { + Self(iter.into_iter().collect()) + } +} + +impl<T> Default for KwArgs<T> { + fn default() -> Self { + Self(IndexMap::new()) + } +} + +impl<T> FromArgs for KwArgs<T> +where + T: TryFromObject, +{ + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { + let mut kwargs = IndexMap::new(); + for (name, value) in args.remaining_keywords() { + kwargs.insert(name, value.try_into_value(vm)?); + } + Ok(Self(kwargs)) + } +} + +impl<T> IntoIterator for KwArgs<T> { + type Item = (String, T); + type IntoIter = indexmap::map::IntoIter<String, T>; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +/// A list of positional argument values. +/// +/// A built-in function with a `PosArgs` parameter is analogous to a Python +/// function with `*args`. All remaining positional arguments are extracted +/// (and hence the function will permit an arbitrary number of them). +/// +/// `PosArgs` optionally accepts a generic type parameter to allow type checks +/// or conversions of each argument. +#[derive(Clone)] +pub struct PosArgs<T = PyObjectRef>(Vec<T>); + +unsafe impl<T> Traverse for PosArgs<T> +where + T: Traverse, +{ + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.traverse(tracer_fn) + } +} + +impl<T> PosArgs<T> { + pub const fn new(args: Vec<T>) -> Self { + Self(args) + } + + pub fn into_vec(self) -> Vec<T> { + self.0 + } + + pub fn iter(&self) -> core::slice::Iter<'_, T> { + self.0.iter() + } +} + +impl<T> From<Vec<T>> for PosArgs<T> { + fn from(v: Vec<T>) -> Self { + Self(v) + } +} + +impl From<()> for PosArgs<PyObjectRef> { + fn from(_args: ()) -> Self { + Self(Vec::new()) + } +} + +impl<T> AsRef<[T]> for PosArgs<T> { + fn as_ref(&self) -> &[T] { + &self.0 + } +} + +impl<T: PyPayload> PosArgs<PyRef<T>> { + pub fn into_tuple(self, vm: &VirtualMachine) -> PyTupleRef { + vm.ctx + .new_tuple(self.0.into_iter().map(Into::into).collect()) + } +} + +impl<T> FromArgs for PosArgs<T> +where + T: TryFromObject, +{ + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { + let mut varargs = Vec::new(); + while let Some(value) = args.take_positional() { + varargs.push(value.try_into_value(vm)?); + } + Ok(Self(varargs)) + } +} + +impl<T> IntoIterator for PosArgs<T> { + type Item = T; + type IntoIter = alloc::vec::IntoIter<T>; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<T> FromArgs for T +where + T: TryFromObject, +{ + fn arity() -> RangeInclusive<usize> { + 1..=1 + } + + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { + let value = args.take_positional().ok_or(ArgumentError::TooFewArgs)?; + Ok(value.try_into_value(vm)?) + } +} + +/// An argument that may or may not be provided by the caller. +/// +/// This style of argument is not possible in pure Python. +#[derive(Debug, result_like::OptionLike, is_macro::Is)] +pub enum OptionalArg<T = PyObjectRef> { + Present(T), + Missing, +} + +unsafe impl<T> Traverse for OptionalArg<T> +where + T: Traverse, +{ + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + match self { + Self::Present(o) => o.traverse(tracer_fn), + Self::Missing => (), + } + } +} + +impl OptionalArg<PyObjectRef> { + pub fn unwrap_or_none(self, vm: &VirtualMachine) -> PyObjectRef { + self.unwrap_or_else(|| vm.ctx.none()) + } +} + +pub type OptionalOption<T = PyObjectRef> = OptionalArg<Option<T>>; + +impl<T> OptionalOption<T> { + #[inline] + pub fn flatten(self) -> Option<T> { + self.into_option().flatten() + } +} + +impl<T> FromArgs for OptionalArg<T> +where + T: TryFromObject, +{ + fn arity() -> RangeInclusive<usize> { + 0..=1 + } + + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { + let r = if let Some(value) = args.take_positional() { + Self::Present(value.try_into_value(vm)?) + } else { + Self::Missing + }; + Ok(r) + } +} + +// For functions that accept no arguments. Implemented explicitly instead of via +// macro below to avoid unused warnings. +impl FromArgs for () { + fn from_args(_vm: &VirtualMachine, _args: &mut FuncArgs) -> Result<Self, ArgumentError> { + Ok(()) + } +} + +// A tuple of types that each implement `FromArgs` represents a sequence of +// arguments that can be bound and passed to a built-in function. +// +// Technically, a tuple can contain tuples, which can contain tuples, and so on, +// so this actually represents a tree of values to be bound from arguments, but +// in practice this is only used for the top-level parameters. +macro_rules! tuple_from_py_func_args { + ($($T:ident),+) => { + impl<$($T),+> FromArgs for ($($T,)+) + where + $($T: FromArgs),+ + { + fn arity() -> RangeInclusive<usize> { + let mut min = 0; + let mut max = 0; + $( + let (start, end) = $T::arity().into_inner(); + min += start; + max += end; + )+ + min..=max + } + + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { + Ok(($($T::from_args(vm, args)?,)+)) + } + } + }; +} + +// Implement `FromArgs` for up to 7-tuples, allowing built-in functions to bind +// up to 7 top-level parameters (note that `PosArgs`, `KwArgs`, nested tuples, etc. +// count as 1, so this should actually be more than enough). +tuple_from_py_func_args!(A); +tuple_from_py_func_args!(A, B); +tuple_from_py_func_args!(A, B, C); +tuple_from_py_func_args!(A, B, C, D); +tuple_from_py_func_args!(A, B, C, D, E); +tuple_from_py_func_args!(A, B, C, D, E, F); +tuple_from_py_func_args!(A, B, C, D, E, F, G); +tuple_from_py_func_args!(A, B, C, D, E, F, G, H); diff --git a/vm/src/function/arithmetic.rs b/crates/vm/src/function/arithmetic.rs similarity index 87% rename from vm/src/function/arithmetic.rs rename to crates/vm/src/function/arithmetic.rs index b40e31e1b62..0ea15ba3edd 100644 --- a/vm/src/function/arithmetic.rs +++ b/crates/vm/src/function/arithmetic.rs @@ -1,7 +1,7 @@ use crate::{ + VirtualMachine, convert::{ToPyObject, TryFromObject}, object::{AsObject, PyObjectRef, PyResult}, - VirtualMachine, }; #[derive(result_like::OptionLike)] @@ -34,8 +34,8 @@ where { fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { match self { - PyArithmeticValue::Implemented(v) => v.to_pyobject(vm), - PyArithmeticValue::NotImplemented => vm.ctx.not_implemented(), + Self::Implemented(v) => v.to_pyobject(vm), + Self::NotImplemented => vm.ctx.not_implemented(), } } } diff --git a/crates/vm/src/function/buffer.rs b/crates/vm/src/function/buffer.rs new file mode 100644 index 00000000000..c0dd6473bdc --- /dev/null +++ b/crates/vm/src/function/buffer.rs @@ -0,0 +1,205 @@ +use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, + VirtualMachine, + builtins::{PyStr, PyStrRef}, + common::borrow::{BorrowedValue, BorrowedValueMut}, + protocol::PyBuffer, +}; + +// Python/getargs.c + +/// any bytes-like object. Like the `y*` format code for `PyArg_Parse` in CPython. +#[derive(Debug, Traverse)] +pub struct ArgBytesLike(PyBuffer); + +impl PyObject { + pub fn try_bytes_like<R>( + &self, + vm: &VirtualMachine, + f: impl FnOnce(&[u8]) -> R, + ) -> PyResult<R> { + let buffer = PyBuffer::try_from_borrowed_object(vm, self)?; + buffer + .as_contiguous() + .map(|x| f(&x)) + .ok_or_else(|| vm.new_buffer_error("non-contiguous buffer is not a bytes-like object")) + } + + pub fn try_rw_bytes_like<R>( + &self, + vm: &VirtualMachine, + f: impl FnOnce(&mut [u8]) -> R, + ) -> PyResult<R> { + let buffer = PyBuffer::try_from_borrowed_object(vm, self)?; + buffer + .as_contiguous_mut() + .map(|mut x| f(&mut x)) + .ok_or_else(|| vm.new_type_error("buffer is not a read-write bytes-like object")) + } +} + +impl ArgBytesLike { + pub fn borrow_buf(&self) -> BorrowedValue<'_, [u8]> { + unsafe { self.0.contiguous_unchecked() } + } + + pub fn with_ref<F, R>(&self, f: F) -> R + where + F: FnOnce(&[u8]) -> R, + { + f(&self.borrow_buf()) + } + + pub const fn len(&self) -> usize { + self.0.desc.len + } + + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn as_object(&self) -> &PyObject { + &self.0.obj + } +} + +impl From<ArgBytesLike> for PyBuffer { + fn from(buffer: ArgBytesLike) -> Self { + buffer.0 + } +} + +impl From<ArgBytesLike> for PyObjectRef { + fn from(buffer: ArgBytesLike) -> Self { + buffer.as_object().to_owned() + } +} + +impl<'a> TryFromBorrowedObject<'a> for ArgBytesLike { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; + if buffer.desc.is_contiguous() { + Ok(Self(buffer)) + } else { + Err(vm.new_buffer_error("non-contiguous buffer is not a bytes-like object")) + } + } +} + +/// A memory buffer, read-write access. Like the `w*` format code for `PyArg_Parse` in CPython. +#[derive(Debug, Traverse)] +pub struct ArgMemoryBuffer(PyBuffer); + +impl ArgMemoryBuffer { + pub fn borrow_buf_mut(&self) -> BorrowedValueMut<'_, [u8]> { + unsafe { self.0.contiguous_mut_unchecked() } + } + + pub fn with_ref<F, R>(&self, f: F) -> R + where + F: FnOnce(&mut [u8]) -> R, + { + f(&mut self.borrow_buf_mut()) + } + + pub const fn len(&self) -> usize { + self.0.desc.len + } + + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl From<ArgMemoryBuffer> for PyBuffer { + fn from(buffer: ArgMemoryBuffer) -> Self { + buffer.0 + } +} + +impl<'a> TryFromBorrowedObject<'a> for ArgMemoryBuffer { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; + if !buffer.desc.is_contiguous() { + Err(vm.new_buffer_error("non-contiguous buffer is not a bytes-like object")) + } else if buffer.desc.readonly { + Err(vm.new_type_error("buffer is not a read-write bytes-like object")) + } else { + Ok(Self(buffer)) + } + } +} + +/// A text string or bytes-like object. Like the `s*` format code for `PyArg_Parse` in CPython. +pub enum ArgStrOrBytesLike { + Buf(ArgBytesLike), + Str(PyStrRef), +} + +impl ArgStrOrBytesLike { + pub fn as_object(&self) -> &PyObject { + match self { + Self::Buf(b) => b.as_object(), + Self::Str(s) => s.as_object(), + } + } +} + +impl TryFromObject for ArgStrOrBytesLike { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + obj.downcast() + .map(Self::Str) + .or_else(|obj| ArgBytesLike::try_from_object(vm, obj).map(Self::Buf)) + } +} + +impl ArgStrOrBytesLike { + pub fn borrow_bytes(&self) -> BorrowedValue<'_, [u8]> { + match self { + Self::Buf(b) => b.borrow_buf(), + Self::Str(s) => s.as_bytes().into(), + } + } +} + +#[derive(Debug)] +pub enum ArgAsciiBuffer { + String(PyStrRef), + Buffer(ArgBytesLike), +} + +impl TryFromObject for ArgAsciiBuffer { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + match obj.downcast::<PyStr>() { + Ok(string) => { + if string.as_wtf8().is_ascii() { + Ok(Self::String(string)) + } else { + Err(vm.new_value_error("string argument should contain only ASCII characters")) + } + } + Err(obj) => ArgBytesLike::try_from_object(vm, obj).map(ArgAsciiBuffer::Buffer), + } + } +} + +impl ArgAsciiBuffer { + pub fn len(&self) -> usize { + match self { + Self::String(s) => s.as_wtf8().len(), + Self::Buffer(buffer) => buffer.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub fn with_ref<R>(&self, f: impl FnOnce(&[u8]) -> R) -> R { + match self { + Self::String(s) => f(s.as_bytes()), + Self::Buffer(buffer) => buffer.with_ref(f), + } + } +} diff --git a/vm/src/function/builtin.rs b/crates/vm/src/function/builtin.rs similarity index 75% rename from vm/src/function/builtin.rs rename to crates/vm/src/function/builtin.rs index 3faaf594fe5..444df64a8ef 100644 --- a/vm/src/function/builtin.rs +++ b/crates/vm/src/function/builtin.rs @@ -1,13 +1,20 @@ use super::{FromArgs, FuncArgs}; use crate::{ - convert::ToPyResult, object::PyThreadingConstraint, Py, PyPayload, PyRef, PyResult, - VirtualMachine, + Py, PyPayload, PyRef, PyResult, VirtualMachine, convert::ToPyResult, + object::PyThreadingConstraint, }; -use std::marker::PhantomData; +use core::marker::PhantomData; /// A built-in Python function. // PyCFunction in CPython -pub type PyNativeFn = py_dyn_fn!(dyn Fn(&VirtualMachine, FuncArgs) -> PyResult); +pub trait PyNativeFn: + Fn(&VirtualMachine, FuncArgs) -> PyResult + PyThreadingConstraint + 'static +{ +} +impl<F: Fn(&VirtualMachine, FuncArgs) -> PyResult + PyThreadingConstraint + 'static> PyNativeFn + for F +{ +} /// Implemented by types that are or can generate built-in functions. /// @@ -34,40 +41,44 @@ pub trait IntoPyNativeFn<Kind>: Sized + PyThreadingConstraint + 'static { /// `IntoPyNativeFn::into_func()` generates a PyNativeFn that performs the /// appropriate type and arity checking, any requested conversions, and then if /// successful calls the function with the extracted parameters. - fn into_func(self) -> &'static PyNativeFn { - let boxed = Box::new(move |vm: &VirtualMachine, args| self.call(vm, args)); - Box::leak(boxed) + fn into_func(self) -> impl PyNativeFn { + into_func(self) } +} - /// Equivalent to `into_func()`, but accessible as a constant. This is only - /// valid if this function is zero-sized, i.e. that - /// `std::mem::size_of::<F>() == 0`. If it isn't, use of this constant will - /// raise a compile error. - const STATIC_FUNC: &'static PyNativeFn = { - if std::mem::size_of::<Self>() == 0 { - &|vm, args| { - // SAFETY: we just confirmed that Self is zero-sized, so there - // aren't any bytes in it that could be uninit. - #[allow(clippy::uninit_assumed_init)] - let f = unsafe { std::mem::MaybeUninit::<Self>::uninit().assume_init() }; - f.call(vm, args) - } - } else { - panic!("function must be zero-sized to access STATIC_FUNC") +const fn into_func<F: IntoPyNativeFn<Kind>, Kind>(f: F) -> impl PyNativeFn { + move |vm: &VirtualMachine, args| f.call(vm, args) +} + +const fn zst_ref_out_of_thin_air<T: 'static>(x: T) -> &'static T { + // if T is zero-sized, there's no issue forgetting it - even if it does have a Drop impl, it + // would never get called anyway if we consider this semantically a Box::leak(Box::new(x))-type + // operation. if T isn't zero-sized, we don't have to worry about it because we'll fail to compile. + core::mem::forget(x); + const { + if core::mem::size_of::<T>() != 0 { + panic!("can't use a non-zero-sized type here") } - }; + // SAFETY: we just confirmed that T is zero-sized, so we can + // pull a value of it out of thin air. + unsafe { core::ptr::NonNull::<T>::dangling().as_ref() } + } +} + +/// Get the STATIC_FUNC of the passed function. The same +/// requirements of zero-sized-ness apply, see that documentation for details. +/// +/// Equivalent to [`IntoPyNativeFn::into_func()`], but usable in a const context. This is only +/// valid if the function is zero-sized, i.e. that `std::mem::size_of::<F>() == 0`. If you call +/// this function with a non-zero-sized function, it will raise a compile error. +#[inline(always)] +pub const fn static_func<Kind, F: IntoPyNativeFn<Kind>>(f: F) -> &'static dyn PyNativeFn { + zst_ref_out_of_thin_air(into_func(f)) } -/// Get the [`STATIC_FUNC`](IntoPyNativeFn::STATIC_FUNC) of the passed function. The same -/// requirements of zero-sizedness apply, see that documentation for details. #[inline(always)] -pub const fn static_func<Kind, F: IntoPyNativeFn<Kind>>(f: F) -> &'static PyNativeFn { - // if f is zero-sized, there's no issue forgetting it - even if a capture of f does have a Drop - // impl, it would never get called anyway. If you passed it to into_func, it would just get - // Box::leak'd, and as a 'static reference it'll never be dropped. and if f isn't zero-sized, - // we'll never reach this point anyway because we'll fail to compile. - std::mem::forget(f); - F::STATIC_FUNC +pub const fn static_raw_func<F: PyNativeFn>(f: F) -> &'static dyn PyNativeFn { + zst_ref_out_of_thin_air(f) } // TODO: once higher-rank trait bounds are stabilized, remove the `Kind` type @@ -207,16 +218,16 @@ into_py_native_fn_tuple!( #[cfg(test)] mod tests { use super::*; + use core::mem::size_of_val; #[test] fn test_into_native_fn_noalloc() { - let check_zst = |f: &'static PyNativeFn| assert_eq!(std::mem::size_of_val(f), 0); fn py_func(_b: bool, _vm: &crate::VirtualMachine) -> i32 { 1 } - check_zst(py_func.into_func()); + assert_eq!(size_of_val(&py_func.into_func()), 0); let empty_closure = || "foo".to_owned(); - check_zst(empty_closure.into_func()); - check_zst(static_func(empty_closure)); + assert_eq!(size_of_val(&empty_closure.into_func()), 0); + assert_eq!(size_of_val(static_func(empty_closure)), 0); } } diff --git a/vm/src/function/either.rs b/crates/vm/src/function/either.rs similarity index 90% rename from vm/src/function/either.rs rename to crates/vm/src/function/either.rs index ceb79d55c9f..9ee7f028bd2 100644 --- a/vm/src/function/either.rs +++ b/crates/vm/src/function/either.rs @@ -1,7 +1,7 @@ use crate::{ - convert::ToPyObject, AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, convert::ToPyObject, }; -use std::borrow::Borrow; +use core::borrow::Borrow; pub enum Either<A, B> { A(A), @@ -28,7 +28,7 @@ impl<A: AsRef<PyObject>, B: AsRef<PyObject>> AsRef<PyObject> for Either<A, B> { } } -impl<A: Into<PyObjectRef>, B: Into<PyObjectRef>> From<Either<A, B>> for PyObjectRef { +impl<A: Into<Self>, B: Into<Self>> From<Either<A, B>> for PyObjectRef { #[inline(always)] fn from(value: Either<A, B>) -> Self { match value { diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs new file mode 100644 index 00000000000..732fd0ca35a --- /dev/null +++ b/crates/vm/src/function/fspath.rs @@ -0,0 +1,153 @@ +use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBytes, PyBytesRef, PyStrRef}, + convert::{IntoPyException, ToPyObject}, + function::PyStr, + protocol::PyBuffer, +}; +use alloc::borrow::Cow; +use std::{ffi::OsStr, path::PathBuf}; + +/// Helper to implement os.fspath() +#[derive(Clone)] +pub enum FsPath { + Str(PyStrRef), + Bytes(PyBytesRef), +} + +impl FsPath { + pub fn try_from_path_like( + obj: PyObjectRef, + check_for_nul: bool, + vm: &VirtualMachine, + ) -> PyResult<Self> { + Self::try_from( + obj, + check_for_nul, + "expected str, bytes or os.PathLike object", + vm, + ) + } + + // PyOS_FSPath + pub fn try_from( + obj: PyObjectRef, + check_for_nul: bool, + msg: &'static str, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let check_nul = |b: &[u8]| { + if !check_for_nul || memchr::memchr(b'\0', b).is_none() { + Ok(()) + } else { + Err(crate::exceptions::cstring_error(vm)) + } + }; + let match1 = |obj: PyObjectRef| { + let pathlike = match_class!(match obj { + s @ PyStr => { + check_nul(s.as_bytes())?; + Self::Str(s) + } + b @ PyBytes => { + check_nul(&b)?; + Self::Bytes(b) + } + obj => return Ok(Err(obj)), + }); + Ok(Ok(pathlike)) + }; + let obj = match match1(obj)? { + Ok(pathlike) => return Ok(pathlike), + Err(obj) => obj, + }; + let not_pathlike_error = || format!("{msg}, not {}", obj.class().name()); + let method = vm.get_method_or_type_error( + obj.clone(), + identifier!(vm, __fspath__), + not_pathlike_error, + )?; + // If __fspath__ is explicitly set to None, treat it as if it doesn't have __fspath__ + if vm.is_none(&method) { + return Err(vm.new_type_error(not_pathlike_error())); + } + let result = method.call((), vm)?; + match1(result)?.map_err(|result| { + vm.new_type_error(format!( + "expected {}.__fspath__() to return str or bytes, not {}", + obj.class().name(), + result.class().name(), + )) + }) + } + + pub fn as_os_str(&self, vm: &VirtualMachine) -> PyResult<Cow<'_, OsStr>> { + // TODO: FS encodings + match self { + Self::Str(s) => vm.fsencode(s), + Self::Bytes(b) => Self::bytes_as_os_str(b.as_bytes(), vm).map(Cow::Borrowed), + } + } + + pub fn as_bytes(&self) -> &[u8] { + // TODO: FS encodings + match self { + Self::Str(s) => s.as_bytes(), + Self::Bytes(b) => b.as_bytes(), + } + } + + pub fn to_string_lossy(&self) -> Cow<'_, str> { + match self { + Self::Str(s) => s.to_string_lossy(), + Self::Bytes(s) => String::from_utf8_lossy(s), + } + } + + pub fn to_path_buf(&self, vm: &VirtualMachine) -> PyResult<PathBuf> { + let path = match self { + Self::Str(s) => PathBuf::from(vm.fsencode(s)?.as_ref() as &OsStr), + Self::Bytes(b) => PathBuf::from(Self::bytes_as_os_str(b, vm)?), + }; + Ok(path) + } + + pub fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<alloc::ffi::CString> { + alloc::ffi::CString::new(self.as_bytes()).map_err(|e| e.into_pyexception(vm)) + } + + #[cfg(windows)] + pub fn to_wide_cstring(&self, vm: &VirtualMachine) -> PyResult<widestring::WideCString> { + widestring::WideCString::from_os_str(self.as_os_str(vm)?) + .map_err(|err| err.into_pyexception(vm)) + } + + pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { + rustpython_common::os::bytes_as_os_str(b) + .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8")) + } +} + +impl ToPyObject for FsPath { + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + match self { + Self::Str(s) => s.into(), + Self::Bytes(b) => b.into(), + } + } +} + +impl TryFromObject for FsPath { + // PyUnicode_FSDecoder in CPython + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let obj = match obj.try_to_value::<PyBuffer>(vm) { + Ok(buffer) => { + let mut bytes = vec![]; + buffer.append_to(&mut bytes); + vm.ctx.new_bytes(bytes).into() + } + Err(_) => obj, + }; + Self::try_from_path_like(obj, true, vm) + } +} diff --git a/crates/vm/src/function/getset.rs b/crates/vm/src/function/getset.rs new file mode 100644 index 00000000000..e7a6ae5bdec --- /dev/null +++ b/crates/vm/src/function/getset.rs @@ -0,0 +1,244 @@ +/*! Python `attribute` descriptor class. (PyGetSet) + +*/ +use crate::{ + Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + convert::ToPyResult, + function::{BorrowedParam, OwnedParam, RefParam}, + object::PyThreadingConstraint, +}; + +#[derive(result_like::OptionLike, is_macro::Is, Debug)] +pub enum PySetterValue<T = PyObjectRef> { + Assign(T), + Delete, +} + +impl PySetterValue { + pub fn unwrap_or_none(self, vm: &VirtualMachine) -> PyObjectRef { + match self { + Self::Assign(value) => value, + Self::Delete => vm.ctx.none(), + } + } +} + +trait FromPySetterValue +where + Self: Sized, +{ + fn from_setter_value(vm: &VirtualMachine, obj: PySetterValue) -> PyResult<Self>; +} + +impl<T> FromPySetterValue for T +where + T: Sized + TryFromObject, +{ + #[inline] + fn from_setter_value(vm: &VirtualMachine, obj: PySetterValue) -> PyResult<Self> { + let obj = obj.ok_or_else(|| vm.new_type_error("can't delete attribute"))?; + T::try_from_object(vm, obj) + } +} + +impl<T> FromPySetterValue for PySetterValue<T> +where + T: Sized + TryFromObject, +{ + #[inline] + fn from_setter_value(vm: &VirtualMachine, obj: PySetterValue) -> PyResult<Self> { + obj.map(|obj| T::try_from_object(vm, obj)).transpose() + } +} + +pub type PyGetterFunc = Box<py_dyn_fn!(dyn Fn(&VirtualMachine, PyObjectRef) -> PyResult)>; +pub type PySetterFunc = + Box<py_dyn_fn!(dyn Fn(&VirtualMachine, PyObjectRef, PySetterValue) -> PyResult<()>)>; + +pub trait IntoPyGetterFunc<T>: PyThreadingConstraint + Sized + 'static { + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult; + fn into_getter(self) -> PyGetterFunc { + Box::new(move |vm, obj| self.get(obj, vm)) + } +} + +impl<F, T, R> IntoPyGetterFunc<(OwnedParam<T>, R, VirtualMachine)> for F +where + F: Fn(T, &VirtualMachine) -> R + 'static + Send + Sync, + T: TryFromObject, + R: ToPyResult, +{ + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let obj = T::try_from_object(vm, obj)?; + (self)(obj, vm).to_pyresult(vm) + } +} + +impl<F, S, R> IntoPyGetterFunc<(BorrowedParam<S>, R, VirtualMachine)> for F +where + F: Fn(&Py<S>, &VirtualMachine) -> R + 'static + Send + Sync, + S: PyPayload, + R: ToPyResult, +{ + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = PyRef::<S>::try_from_object(vm, obj)?; + (self)(&zelf, vm).to_pyresult(vm) + } +} + +impl<F, S, R> IntoPyGetterFunc<(RefParam<S>, R, VirtualMachine)> for F +where + F: Fn(&S, &VirtualMachine) -> R + 'static + Send + Sync, + S: PyPayload, + R: ToPyResult, +{ + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = PyRef::<S>::try_from_object(vm, obj)?; + (self)(&zelf, vm).to_pyresult(vm) + } +} + +impl<F, T, R> IntoPyGetterFunc<(OwnedParam<T>, R)> for F +where + F: Fn(T) -> R + 'static + Send + Sync, + T: TryFromObject, + R: ToPyResult, +{ + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let obj = T::try_from_object(vm, obj)?; + (self)(obj).to_pyresult(vm) + } +} + +impl<F, S, R> IntoPyGetterFunc<(BorrowedParam<S>, R)> for F +where + F: Fn(&Py<S>) -> R + 'static + Send + Sync, + S: PyPayload, + R: ToPyResult, +{ + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = PyRef::<S>::try_from_object(vm, obj)?; + (self)(&zelf).to_pyresult(vm) + } +} + +impl<F, S, R> IntoPyGetterFunc<(RefParam<S>, R)> for F +where + F: Fn(&S) -> R + 'static + Send + Sync, + S: PyPayload, + R: ToPyResult, +{ + fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = PyRef::<S>::try_from_object(vm, obj)?; + (self)(&zelf).to_pyresult(vm) + } +} + +pub trait IntoPyNoResult { + fn into_noresult(self) -> PyResult<()>; +} + +impl IntoPyNoResult for () { + #[inline] + fn into_noresult(self) -> PyResult<()> { + Ok(()) + } +} + +impl IntoPyNoResult for PyResult<()> { + #[inline] + fn into_noresult(self) -> PyResult<()> { + self + } +} + +pub trait IntoPySetterFunc<T>: PyThreadingConstraint + Sized + 'static { + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()>; + fn into_setter(self) -> PySetterFunc { + Box::new(move |vm, obj, value| self.set(obj, value, vm)) + } +} + +impl<F, T, V, R> IntoPySetterFunc<(OwnedParam<T>, V, R, VirtualMachine)> for F +where + F: Fn(T, V, &VirtualMachine) -> R + 'static + Send + Sync, + T: TryFromObject, + V: FromPySetterValue, + R: IntoPyNoResult, +{ + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let obj = T::try_from_object(vm, obj)?; + let value = V::from_setter_value(vm, value)?; + (self)(obj, value, vm).into_noresult() + } +} + +impl<F, S, V, R> IntoPySetterFunc<(BorrowedParam<S>, V, R, VirtualMachine)> for F +where + F: Fn(&Py<S>, V, &VirtualMachine) -> R + 'static + Send + Sync, + S: PyPayload, + V: FromPySetterValue, + R: IntoPyNoResult, +{ + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let zelf = PyRef::<S>::try_from_object(vm, obj)?; + let value = V::from_setter_value(vm, value)?; + (self)(&zelf, value, vm).into_noresult() + } +} + +impl<F, S, V, R> IntoPySetterFunc<(RefParam<S>, V, R, VirtualMachine)> for F +where + F: Fn(&S, V, &VirtualMachine) -> R + 'static + Send + Sync, + S: PyPayload, + V: FromPySetterValue, + R: IntoPyNoResult, +{ + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let zelf = PyRef::<S>::try_from_object(vm, obj)?; + let value = V::from_setter_value(vm, value)?; + (self)(&zelf, value, vm).into_noresult() + } +} + +impl<F, T, V, R> IntoPySetterFunc<(OwnedParam<T>, V, R)> for F +where + F: Fn(T, V) -> R + 'static + Send + Sync, + T: TryFromObject, + V: FromPySetterValue, + R: IntoPyNoResult, +{ + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let obj = T::try_from_object(vm, obj)?; + let value = V::from_setter_value(vm, value)?; + (self)(obj, value).into_noresult() + } +} + +impl<F, S, V, R> IntoPySetterFunc<(BorrowedParam<S>, V, R)> for F +where + F: Fn(&Py<S>, V) -> R + 'static + Send + Sync, + S: PyPayload, + V: FromPySetterValue, + R: IntoPyNoResult, +{ + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let zelf = PyRef::<S>::try_from_object(vm, obj)?; + let value = V::from_setter_value(vm, value)?; + (self)(&zelf, value).into_noresult() + } +} + +impl<F, S, V, R> IntoPySetterFunc<(RefParam<S>, V, R)> for F +where + F: Fn(&S, V) -> R + 'static + Send + Sync, + S: PyPayload, + V: FromPySetterValue, + R: IntoPyNoResult, +{ + fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let zelf = PyRef::<S>::try_from_object(vm, obj)?; + let value = V::from_setter_value(vm, value)?; + (self)(&zelf, value).into_noresult() + } +} diff --git a/crates/vm/src/function/method.rs b/crates/vm/src/function/method.rs new file mode 100644 index 00000000000..295e4d89adf --- /dev/null +++ b/crates/vm/src/function/method.rs @@ -0,0 +1,325 @@ +use crate::{ + Context, Py, PyObjectRef, PyPayload, PyRef, VirtualMachine, + builtins::{ + PyType, + builtin_func::{PyNativeFunction, PyNativeMethod}, + descriptor::PyMethodDescriptor, + }, + function::{IntoPyNativeFn, PyNativeFn}, +}; + +bitflags::bitflags! { + // METH_XXX flags in CPython + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct PyMethodFlags: u32 { + const VARARGS = 0x0001; + const KEYWORDS = 0x0002; + // METH_NOARGS and METH_O must not be combined with the flags above. + const NOARGS = 0x0004; + const O = 0x0008; + + // METH_CLASS and METH_STATIC are a little different; these control + // the construction of methods for a class. These cannot be used for + // functions in modules. + const CLASS = 0x0010; + const STATIC = 0x0020; + + // METH_COEXIST allows a method to be entered even though a slot has + // already filled the entry. When defined, the flag allows a separate + // method, "__contains__" for example, to coexist with a defined + // slot like sq_contains. + // const COEXIST = 0x0040; + + // if not Py_LIMITED_API + const FASTCALL = 0x0080; + + // This bit is preserved for Stackless Python + // const STACKLESS = 0x0100; + + // METH_METHOD means the function stores an + // additional reference to the class that defines it; + // both self and class are passed to it. + // It uses PyCMethodObject instead of PyCFunctionObject. + // May not be combined with METH_NOARGS, METH_O, METH_CLASS or METH_STATIC. + const METHOD = 0x0200; + } +} + +impl PyMethodFlags { + // FIXME: macro temp + pub const EMPTY: Self = Self::empty(); +} + +#[macro_export] +macro_rules! define_methods { + // TODO: more flexible syntax + ($($name:literal => $func:ident as $flags:ident),+) => { + vec![ $( $crate::function::PyMethodDef { + name: $name, + func: $crate::function::static_func($func), + flags: $crate::function::PyMethodFlags::$flags, + doc: None, + }),+ ] + }; +} + +#[derive(Clone)] +pub struct PyMethodDef { + pub name: &'static str, // TODO: interned + pub func: &'static dyn PyNativeFn, + pub flags: PyMethodFlags, + pub doc: Option<&'static str>, // TODO: interned +} + +impl PyMethodDef { + #[inline] + pub const fn new_const<Kind>( + name: &'static str, + func: impl IntoPyNativeFn<Kind>, + flags: PyMethodFlags, + doc: Option<&'static str>, + ) -> Self { + Self { + name, + func: super::static_func(func), + flags, + doc, + } + } + + #[inline] + pub const fn new_raw_const( + name: &'static str, + func: impl PyNativeFn, + flags: PyMethodFlags, + doc: Option<&'static str>, + ) -> Self { + Self { + name, + func: super::static_raw_func(func), + flags, + doc, + } + } + + pub fn to_proper_method( + &'static self, + class: &'static Py<PyType>, + ctx: &Context, + ) -> PyObjectRef { + if self.flags.contains(PyMethodFlags::METHOD) { + self.build_method(ctx, class).into() + } else if self.flags.contains(PyMethodFlags::CLASS) { + self.build_classmethod(ctx, class).into() + } else if self.flags.contains(PyMethodFlags::STATIC) { + self.build_staticmethod(ctx, class).into() + } else { + unreachable!() + } + } + + pub const fn to_function(&'static self) -> PyNativeFunction { + PyNativeFunction { + zelf: None, + value: self, + module: None, + _method_def_owner: None, + } + } + + pub fn to_method( + &'static self, + class: &'static Py<PyType>, + ctx: &Context, + ) -> PyMethodDescriptor { + PyMethodDescriptor::new(self, class, ctx) + } + + pub const fn to_bound_method( + &'static self, + obj: PyObjectRef, + class: &'static Py<PyType>, + ) -> PyNativeMethod { + PyNativeMethod { + func: PyNativeFunction { + zelf: Some(obj), + value: self, + module: None, + _method_def_owner: None, + }, + class, + } + } + + pub fn build_function(&'static self, ctx: &Context) -> PyRef<PyNativeFunction> { + self.to_function().into_ref(ctx) + } + + pub fn build_bound_function( + &'static self, + ctx: &Context, + obj: PyObjectRef, + ) -> PyRef<PyNativeFunction> { + let function = PyNativeFunction { + zelf: Some(obj), + value: self, + module: None, + _method_def_owner: None, + }; + PyRef::new_ref( + function, + ctx.types.builtin_function_or_method_type.to_owned(), + None, + ) + } + + pub fn build_method( + &'static self, + ctx: &Context, + class: &'static Py<PyType>, + ) -> PyRef<PyMethodDescriptor> { + debug_assert!(self.flags.contains(PyMethodFlags::METHOD)); + let method = self.to_method(class, ctx); + PyRef::new_ref(method, ctx.types.method_descriptor_type.to_owned(), None) + } + + pub fn build_bound_method( + &'static self, + ctx: &Context, + obj: PyObjectRef, + class: &'static Py<PyType>, + ) -> PyRef<PyNativeMethod> { + PyRef::new_ref( + self.to_bound_method(obj, class), + ctx.types.builtin_function_or_method_type.to_owned(), + None, + ) + } + + pub fn build_classmethod( + &'static self, + ctx: &Context, + class: &'static Py<PyType>, + ) -> PyRef<PyMethodDescriptor> { + PyRef::new_ref( + self.to_method(class, ctx), + ctx.types.method_descriptor_type.to_owned(), + None, + ) + } + + pub fn build_staticmethod( + &'static self, + ctx: &Context, + class: &'static Py<PyType>, + ) -> PyRef<PyNativeMethod> { + debug_assert!(self.flags.contains(PyMethodFlags::STATIC)); + // Set zelf to the class (m_self = type for static methods). + // Callable::call skips prepending when STATIC flag is set. + let func = PyNativeFunction { + zelf: Some(class.to_owned().into()), + value: self, + module: None, + _method_def_owner: None, + }; + PyNativeMethod { func, class }.into_ref(ctx) + } + + #[doc(hidden)] + pub const fn __const_concat_arrays<const SUM_LEN: usize>( + method_groups: &[&[Self]], + ) -> [Self; SUM_LEN] { + const NULL_METHOD: PyMethodDef = PyMethodDef { + name: "", + func: &|_, _| unreachable!(), + flags: PyMethodFlags::empty(), + doc: None, + }; + let mut all_methods = [NULL_METHOD; SUM_LEN]; + let mut all_idx = 0; + let mut group_idx = 0; + while group_idx < method_groups.len() { + let group = method_groups[group_idx]; + let mut method_idx = 0; + while method_idx < group.len() { + all_methods[all_idx] = group[method_idx].const_copy(); + method_idx += 1; + all_idx += 1; + } + group_idx += 1; + } + all_methods + } + + const fn const_copy(&self) -> Self { + Self { + name: self.name, + func: self.func, + flags: self.flags, + doc: self.doc, + } + } +} + +impl core::fmt::Debug for PyMethodDef { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyMethodDef") + .field("name", &self.name) + .field( + "func", + &(unsafe { + core::mem::transmute::<&dyn PyNativeFn, [usize; 2]>(self.func)[1] as *const u8 + }), + ) + .field("flags", &self.flags) + .field("doc", &self.doc) + .finish() + } +} + +// This is not a part of CPython API. +// But useful to support dynamically generated methods +#[pyclass(name, module = false, ctx = "method_def")] +#[derive(Debug)] +pub struct HeapMethodDef { + method: PyMethodDef, +} + +impl HeapMethodDef { + pub const fn new(method: PyMethodDef) -> Self { + Self { method } + } +} + +impl Py<HeapMethodDef> { + pub(crate) unsafe fn method(&self) -> &'static PyMethodDef { + unsafe { &*(&self.method as *const _) } + } + + pub fn build_function(&self, vm: &VirtualMachine) -> PyRef<PyNativeFunction> { + let mut function = unsafe { self.method() }.to_function(); + function._method_def_owner = Some(self.to_owned().into()); + PyRef::new_ref( + function, + vm.ctx.types.builtin_function_or_method_type.to_owned(), + None, + ) + } + + pub fn build_method( + &self, + class: &'static Py<PyType>, + vm: &VirtualMachine, + ) -> PyRef<PyMethodDescriptor> { + let mut function = unsafe { self.method() }.to_method(class, &vm.ctx); + function._method_def_owner = Some(self.to_owned().into()); + PyRef::new_ref( + function, + vm.ctx.types.method_descriptor_type.to_owned(), + None, + ) + } +} + +#[pyclass] +impl HeapMethodDef {} diff --git a/crates/vm/src/function/mod.rs b/crates/vm/src/function/mod.rs new file mode 100644 index 00000000000..4be94e3f0be --- /dev/null +++ b/crates/vm/src/function/mod.rs @@ -0,0 +1,49 @@ +mod argument; +mod arithmetic; +mod buffer; +mod builtin; +mod either; +mod fspath; +mod getset; +mod method; +mod number; +mod protocol; +mod time; + +pub use argument::{ + ArgumentError, FromArgOptional, FromArgs, FuncArgs, IntoFuncArgs, KwArgs, OptionalArg, + OptionalOption, PosArgs, +}; +pub use arithmetic::{PyArithmeticValue, PyComparisonValue}; +pub use buffer::{ArgAsciiBuffer, ArgBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike}; +pub use builtin::{IntoPyNativeFn, PyNativeFn, static_func, static_raw_func}; +pub use either::Either; +pub use fspath::FsPath; +pub use getset::PySetterValue; +pub(super) use getset::{IntoPyGetterFunc, IntoPySetterFunc, PyGetterFunc, PySetterFunc}; +pub use method::{HeapMethodDef, PyMethodDef, PyMethodFlags}; +pub use number::{ArgIndex, ArgIntoBool, ArgIntoComplex, ArgIntoFloat, ArgPrimitiveIndex, ArgSize}; +pub use protocol::{ArgCallable, ArgIterable, ArgMapping, ArgSequence}; +pub use time::TimeoutSeconds; + +use crate::{PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject}; +use builtin::{BorrowedParam, OwnedParam, RefParam}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ArgByteOrder { + Big, + Little, +} + +impl<'a> TryFromBorrowedObject<'a> for ArgByteOrder { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + obj.try_value_with( + |s: &PyStr| match s.as_bytes() { + b"big" => Ok(Self::Big), + b"little" => Ok(Self::Little), + _ => Err(vm.new_value_error("byteorder must be either 'little' or 'big'")), + }, + vm, + ) + } +} diff --git a/crates/vm/src/function/number.rs b/crates/vm/src/function/number.rs new file mode 100644 index 00000000000..b53208bcd93 --- /dev/null +++ b/crates/vm/src/function/number.rs @@ -0,0 +1,198 @@ +use super::argument::OptionalArg; +use crate::{AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyIntRef}; +use core::ops::Deref; +use malachite_bigint::BigInt; +use num_complex::Complex64; +use num_traits::PrimInt; + +/// A Python complex-like object. +/// +/// `ArgIntoComplex` implements `FromArgs` so that a built-in function can accept +/// any object that can be transformed into a complex. +/// +/// If the object is not a Python complex object but has a `__complex__()` +/// method, this method will first be called to convert the object into a float. +/// If `__complex__()` is not defined then it falls back to `__float__()`. If +/// `__float__()` is not defined it falls back to `__index__()`. +#[derive(Debug, PartialEq)] +#[repr(transparent)] +pub struct ArgIntoComplex { + value: Complex64, +} + +impl ArgIntoComplex { + #[inline] + pub fn into_complex(self) -> Complex64 { + self.value + } +} + +impl From<ArgIntoComplex> for Complex64 { + fn from(arg: ArgIntoComplex) -> Self { + arg.value + } +} + +impl TryFromObject for ArgIntoComplex { + // Equivalent to PyComplex_AsCComplex + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + // We do not care if it was already a complex. + let (value, _) = obj.try_complex(vm)?.ok_or_else(|| { + vm.new_type_error(format!("must be real number, not {}", obj.class().name())) + })?; + Ok(Self { value }) + } +} + +/// A Python float-like object. +/// +/// `ArgIntoFloat` implements `FromArgs` so that a built-in function can accept +/// any object that can be transformed into a float. +/// +/// If the object is not a Python floating point object but has a `__float__()` +/// method, this method will first be called to convert the object into a float. +/// If `__float__()` is not defined then it falls back to `__index__()`. +#[derive(Debug, PartialEq)] +#[repr(transparent)] +pub struct ArgIntoFloat { + value: f64, +} + +impl ArgIntoFloat { + #[inline] + pub fn into_float(self) -> f64 { + self.value + } + + pub fn vec_into_f64(v: Vec<Self>) -> Vec<f64> { + // TODO: Vec::into_raw_parts once stabilized + let mut v = core::mem::ManuallyDrop::new(v); + let (p, l, c) = (v.as_mut_ptr(), v.len(), v.capacity()); + // SAFETY: IntoPyFloat is repr(transparent) over f64 + unsafe { Vec::from_raw_parts(p.cast(), l, c) } + } +} + +impl From<ArgIntoFloat> for f64 { + fn from(arg: ArgIntoFloat) -> Self { + arg.value + } +} + +impl TryFromObject for ArgIntoFloat { + // Equivalent to PyFloat_AsDouble. + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let value = obj.try_float(vm)?.to_f64(); + Ok(Self { value }) + } +} + +/// A Python bool-like object. +/// +/// `ArgIntoBool` implements `FromArgs` so that a built-in function can accept +/// any object that can be transformed into a boolean. +/// +/// By default an object is considered true unless its class defines either a +/// `__bool__()` method that returns False or a `__len__()` method that returns +/// zero, when called with the object. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct ArgIntoBool { + value: bool, +} + +impl ArgIntoBool { + pub const TRUE: Self = Self { value: true }; + pub const FALSE: Self = Self { value: false }; + + #[inline] + pub fn into_bool(self) -> bool { + self.value + } +} + +impl From<ArgIntoBool> for bool { + fn from(arg: ArgIntoBool) -> Self { + arg.value + } +} + +impl TryFromObject for ArgIntoBool { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + Ok(Self { + value: obj.try_to_bool(vm)?, + }) + } +} + +// Implement ArgIndex to separate between "true" int and int generated by index +#[derive(Debug, Traverse)] +#[repr(transparent)] +pub struct ArgIndex { + value: PyIntRef, +} + +impl ArgIndex { + #[inline] + pub fn into_int_ref(self) -> PyIntRef { + self.value + } +} + +impl AsRef<PyIntRef> for ArgIndex { + fn as_ref(&self) -> &PyIntRef { + &self.value + } +} + +impl From<ArgIndex> for PyIntRef { + fn from(arg: ArgIndex) -> Self { + arg.value + } +} + +impl TryFromObject for ArgIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + Ok(Self { + value: obj.try_index(vm)?, + }) + } +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub struct ArgPrimitiveIndex<T> { + pub value: T, +} + +impl<T> OptionalArg<ArgPrimitiveIndex<T>> { + pub fn into_primitive(self) -> OptionalArg<T> { + self.map(|x| x.value) + } +} + +impl<T> Deref for ArgPrimitiveIndex<T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + +impl<T> TryFromObject for ArgPrimitiveIndex<T> +where + T: PrimInt + for<'a> TryFrom<&'a BigInt>, +{ + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + Ok(Self { + value: obj.try_index(vm)?.try_to_primitive(vm)?, + }) + } +} + +pub type ArgSize = ArgPrimitiveIndex<isize>; + +impl From<ArgSize> for isize { + fn from(arg: ArgSize) -> Self { + arg.value + } +} diff --git a/crates/vm/src/function/protocol.rs b/crates/vm/src/function/protocol.rs new file mode 100644 index 00000000000..402f6d0365b --- /dev/null +++ b/crates/vm/src/function/protocol.rs @@ -0,0 +1,230 @@ +use super::IntoFuncArgs; +use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + builtins::{PyDictRef, iter::PySequenceIterator}, + convert::ToPyObject, + object::{Traverse, TraverseFn}, + protocol::{PyIter, PyIterIter, PyMapping}, + types::GenericMethod, +}; +use core::{borrow::Borrow, marker::PhantomData}; + +#[derive(Clone, Traverse)] +pub struct ArgCallable { + obj: PyObjectRef, + #[pytraverse(skip)] + call: GenericMethod, +} + +impl ArgCallable { + #[inline(always)] + pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { + let args = args.into_args(vm); + (self.call)(&self.obj, args, vm) + } +} + +impl core::fmt::Debug for ArgCallable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("ArgCallable") + .field("obj", &self.obj) + .field("call", &format!("{:08x}", self.call as usize)) + .finish() + } +} + +impl Borrow<PyObject> for ArgCallable { + #[inline(always)] + fn borrow(&self) -> &PyObject { + &self.obj + } +} + +impl AsRef<PyObject> for ArgCallable { + #[inline(always)] + fn as_ref(&self) -> &PyObject { + &self.obj + } +} + +impl From<ArgCallable> for PyObjectRef { + #[inline(always)] + fn from(value: ArgCallable) -> Self { + value.obj + } +} + +impl TryFromObject for ArgCallable { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let Some(callable) = obj.to_callable() else { + return Err( + vm.new_type_error(format!("'{}' object is not callable", obj.class().name())) + ); + }; + let call = callable.call; + Ok(Self { obj, call }) + } +} + +/// An iterable Python object. +/// +/// `ArgIterable` implements `FromArgs` so that a built-in function can accept +/// an object that is required to conform to the Python iterator protocol. +/// +/// ArgIterable can optionally perform type checking and conversions on iterated +/// objects using a generic type parameter that implements `TryFromObject`. +pub struct ArgIterable<T = PyObjectRef> { + iterable: PyObjectRef, + iter_fn: Option<crate::types::IterFunc>, + _item: PhantomData<T>, +} + +unsafe impl<T: Traverse> Traverse for ArgIterable<T> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.iterable.traverse(tracer_fn) + } +} + +impl<T> ArgIterable<T> { + /// Returns an iterator over this sequence of objects. + /// + /// This operation may fail if an exception is raised while invoking the + /// `__iter__` method of the iterable object. + pub fn iter<'a>(&self, vm: &'a VirtualMachine) -> PyResult<PyIterIter<'a, T>> { + let iter = PyIter::new(match self.iter_fn { + Some(f) => f(self.iterable.clone(), vm)?, + None => PySequenceIterator::new(self.iterable.clone(), vm)?.into_pyobject(vm), + }); + iter.into_iter(vm) + } +} + +impl<T> TryFromObject for ArgIterable<T> +where + T: TryFromObject, +{ + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let cls = obj.class(); + let iter_fn = cls.slots.iter.load(); + if iter_fn.is_none() && !cls.has_attr(identifier!(vm, __getitem__)) { + return Err(vm.new_type_error(format!("'{}' object is not iterable", cls.name()))); + } + Ok(Self { + iterable: obj, + iter_fn, + _item: PhantomData, + }) + } +} + +#[derive(Debug, Clone, Traverse)] +pub struct ArgMapping { + obj: PyObjectRef, +} + +impl ArgMapping { + #[inline] + pub const fn new(obj: PyObjectRef) -> Self { + Self { obj } + } + + #[inline(always)] + pub fn from_dict_exact(dict: PyDictRef) -> Self { + Self { obj: dict.into() } + } + + #[inline(always)] + pub fn obj(&self) -> &PyObject { + &self.obj + } + + #[inline(always)] + pub fn mapping(&self) -> PyMapping<'_> { + self.obj.mapping_unchecked() + } +} + +impl Borrow<PyObject> for ArgMapping { + #[inline(always)] + fn borrow(&self) -> &PyObject { + &self.obj + } +} + +impl AsRef<PyObject> for ArgMapping { + #[inline(always)] + fn as_ref(&self) -> &PyObject { + &self.obj + } +} + +impl From<ArgMapping> for PyObjectRef { + #[inline(always)] + fn from(value: ArgMapping) -> Self { + value.obj + } +} + +impl ToPyObject for ArgMapping { + #[inline(always)] + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.obj + } +} + +impl TryFromObject for ArgMapping { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let _mapping = obj.try_mapping(vm)?; + Ok(Self { obj }) + } +} + +// this is not strictly related to PySequence protocol. +#[derive(Clone)] +pub struct ArgSequence<T = PyObjectRef>(Vec<T>); + +unsafe impl<T: Traverse> Traverse for ArgSequence<T> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.traverse(tracer_fn); + } +} + +impl<T> ArgSequence<T> { + #[inline(always)] + pub fn into_vec(self) -> Vec<T> { + self.0 + } + #[inline(always)] + pub fn as_slice(&self) -> &[T] { + &self.0 + } +} + +impl<T> core::ops::Deref for ArgSequence<T> { + type Target = [T]; + #[inline(always)] + fn deref(&self) -> &[T] { + self.as_slice() + } +} + +impl<'a, T> IntoIterator for &'a ArgSequence<T> { + type Item = &'a T; + type IntoIter = core::slice::Iter<'a, T>; + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} +impl<T> IntoIterator for ArgSequence<T> { + type Item = T; + type IntoIter = alloc::vec::IntoIter<T>; + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<T: TryFromObject> TryFromObject for ArgSequence<T> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + obj.try_to_value(vm).map(Self) + } +} diff --git a/crates/vm/src/function/time.rs b/crates/vm/src/function/time.rs new file mode 100644 index 00000000000..29f14495d14 --- /dev/null +++ b/crates/vm/src/function/time.rs @@ -0,0 +1,34 @@ +use crate::{PyObjectRef, PyResult, TryFromObject, VirtualMachine}; + +/// A Python timeout value that accepts both `float` and `int`. +/// +/// `TimeoutSeconds` implements `FromArgs` so that a built-in function can accept +/// timeout parameters given as either `float` or `int`, normalizing them to `f64`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct TimeoutSeconds { + value: f64, +} + +impl TimeoutSeconds { + pub const fn new(secs: f64) -> Self { + Self { value: secs } + } + + #[inline] + pub fn to_secs_f64(self) -> f64 { + self.value + } +} + +impl TryFromObject for TimeoutSeconds { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let value = match super::Either::<f64, i64>::try_from_object(vm, obj)? { + super::Either::A(f) => f, + super::Either::B(i) => i as f64, + }; + if value.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)".to_owned())); + } + Ok(Self { value }) + } +} diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs new file mode 100644 index 00000000000..d86c3d4d560 --- /dev/null +++ b/crates/vm/src/gc_state.rs @@ -0,0 +1,895 @@ +//! Garbage Collection State and Algorithm +//! +//! Generational garbage collection using an intrusive doubly-linked list. + +use crate::common::linked_list::LinkedList; +use crate::common::lock::{PyMutex, PyRwLock}; +use crate::object::{GC_PERMANENT, GC_UNTRACKED, GcLink}; +use crate::{AsObject, PyObject, PyObjectRef}; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use std::collections::HashSet; + +#[cfg(not(target_arch = "wasm32"))] +fn elapsed_secs(start: &std::time::Instant) -> f64 { + start.elapsed().as_secs_f64() +} + +#[cfg(target_arch = "wasm32")] +fn elapsed_secs(_start: &()) -> f64 { + 0.0 +} + +bitflags::bitflags! { + /// GC debug flags (see Include/internal/pycore_gc.h) + #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] + pub struct GcDebugFlags: u32 { + /// Print collection statistics + const STATS = 1 << 0; + /// Print collectable objects + const COLLECTABLE = 1 << 1; + /// Print uncollectable objects + const UNCOLLECTABLE = 1 << 2; + /// Save all garbage in gc.garbage + const SAVEALL = 1 << 5; + /// DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | DEBUG_SAVEALL + const LEAK = Self::COLLECTABLE.bits() | Self::UNCOLLECTABLE.bits() | Self::SAVEALL.bits(); + } +} + +/// Result from a single collection run +#[derive(Debug, Default)] +pub struct CollectResult { + pub collected: usize, + pub uncollectable: usize, + pub candidates: usize, + pub duration: f64, +} + +/// Statistics for a single generation (gc_generation_stats) +#[derive(Debug, Default)] +pub struct GcStats { + pub collections: usize, + pub collected: usize, + pub uncollectable: usize, + pub candidates: usize, + pub duration: f64, +} + +/// A single GC generation with intrusive linked list +pub struct GcGeneration { + /// Number of objects in this generation + count: AtomicUsize, + /// Threshold for triggering collection + threshold: AtomicU32, + /// Collection statistics + stats: PyMutex<GcStats>, +} + +impl GcGeneration { + pub const fn new(threshold: u32) -> Self { + Self { + count: AtomicUsize::new(0), + threshold: AtomicU32::new(threshold), + stats: PyMutex::new(GcStats { + collections: 0, + collected: 0, + uncollectable: 0, + candidates: 0, + duration: 0.0, + }), + } + } + + pub fn count(&self) -> usize { + self.count.load(Ordering::SeqCst) + } + + pub fn threshold(&self) -> u32 { + self.threshold.load(Ordering::SeqCst) + } + + pub fn set_threshold(&self, value: u32) { + self.threshold.store(value, Ordering::SeqCst); + } + + pub fn stats(&self) -> GcStats { + let guard = self.stats.lock(); + GcStats { + collections: guard.collections, + collected: guard.collected, + uncollectable: guard.uncollectable, + candidates: guard.candidates, + duration: guard.duration, + } + } + + pub fn update_stats( + &self, + collected: usize, + uncollectable: usize, + candidates: usize, + duration: f64, + ) { + let mut guard = self.stats.lock(); + guard.collections += 1; + guard.collected += collected; + guard.uncollectable += uncollectable; + guard.candidates += candidates; + guard.duration += duration; + } + + /// Reset the stats mutex to unlocked state after fork(). + /// + /// # Safety + /// Must only be called after fork() in the child process when no other + /// threads exist. + #[cfg(all(unix, feature = "threading"))] + unsafe fn reinit_stats_after_fork(&self) { + unsafe { crate::common::lock::reinit_mutex_after_fork(&self.stats) }; + } +} + +/// Wrapper for NonNull<PyObject> to impl Hash/Eq for use in temporary collection sets. +/// Only used within collect_inner, never shared across threads. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct GcPtr(NonNull<PyObject>); + +/// Global GC state +pub struct GcState { + /// 3 generations (0 = youngest, 2 = oldest) + pub generations: [GcGeneration; 3], + /// Permanent generation (frozen objects) + pub permanent: GcGeneration, + /// GC enabled flag + pub enabled: AtomicBool, + /// Per-generation intrusive linked lists for object tracking. + /// Objects start in gen0, survivors are promoted to gen1, then gen2. + generation_lists: [PyRwLock<LinkedList<GcLink, PyObject>>; 3], + /// Frozen/permanent objects (excluded from normal GC) + permanent_list: PyRwLock<LinkedList<GcLink, PyObject>>, + /// Debug flags + pub debug: AtomicU32, + /// gc.garbage list (uncollectable objects with __del__) + pub garbage: PyMutex<Vec<PyObjectRef>>, + /// gc.callbacks list + pub callbacks: PyMutex<Vec<PyObjectRef>>, + /// Mutex for collection (prevents concurrent collections) + collecting: PyMutex<()>, + /// Allocation counter for gen0 + alloc_count: AtomicUsize, +} + +// SAFETY: All fields are either inherently Send/Sync (atomics, RwLock, Mutex) or protected by PyMutex. +// LinkedList<GcLink, PyObject> is Send+Sync because GcLink's Target (PyObject) is Send+Sync. +#[cfg(feature = "threading")] +unsafe impl Send for GcState {} +#[cfg(feature = "threading")] +unsafe impl Sync for GcState {} + +impl Default for GcState { + fn default() -> Self { + Self::new() + } +} + +impl GcState { + pub fn new() -> Self { + Self { + generations: [ + GcGeneration::new(2000), // young + GcGeneration::new(10), // old[0] + GcGeneration::new(0), // old[1] + ], + permanent: GcGeneration::new(0), + enabled: AtomicBool::new(true), + generation_lists: [ + PyRwLock::new(LinkedList::new()), + PyRwLock::new(LinkedList::new()), + PyRwLock::new(LinkedList::new()), + ], + permanent_list: PyRwLock::new(LinkedList::new()), + debug: AtomicU32::new(0), + garbage: PyMutex::new(Vec::new()), + callbacks: PyMutex::new(Vec::new()), + collecting: PyMutex::new(()), + alloc_count: AtomicUsize::new(0), + } + } + + /// Check if GC is enabled + pub fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::SeqCst) + } + + /// Enable GC + pub fn enable(&self) { + self.enabled.store(true, Ordering::SeqCst); + } + + /// Disable GC + pub fn disable(&self) { + self.enabled.store(false, Ordering::SeqCst); + } + + /// Get debug flags + pub fn get_debug(&self) -> GcDebugFlags { + GcDebugFlags::from_bits_truncate(self.debug.load(Ordering::SeqCst)) + } + + /// Set debug flags + pub fn set_debug(&self, flags: GcDebugFlags) { + self.debug.store(flags.bits(), Ordering::SeqCst); + } + + /// Get thresholds for all generations + pub fn get_threshold(&self) -> (u32, u32, u32) { + ( + self.generations[0].threshold(), + self.generations[1].threshold(), + self.generations[2].threshold(), + ) + } + + /// Set thresholds + pub fn set_threshold(&self, t0: u32, t1: Option<u32>, t2: Option<u32>) { + self.generations[0].set_threshold(t0); + if let Some(t1) = t1 { + self.generations[1].set_threshold(t1); + } + if let Some(t2) = t2 { + self.generations[2].set_threshold(t2); + } + } + + /// Get counts for all generations + pub fn get_count(&self) -> (usize, usize, usize) { + ( + self.generations[0].count(), + self.generations[1].count(), + self.generations[2].count(), + ) + } + + /// Get statistics for all generations + pub fn get_stats(&self) -> [GcStats; 3] { + [ + self.generations[0].stats(), + self.generations[1].stats(), + self.generations[2].stats(), + ] + } + + /// Track a new object (add to gen0). + /// O(1) — intrusive linked list push_front, no hashing. + /// + /// # Safety + /// obj must be a valid pointer to a PyObject + pub unsafe fn track_object(&self, obj: NonNull<PyObject>) { + let obj_ref = unsafe { obj.as_ref() }; + obj_ref.set_gc_tracked(); + obj_ref.set_gc_generation(0); + + self.generation_lists[0].write().push_front(obj); + self.generations[0].count.fetch_add(1, Ordering::SeqCst); + self.alloc_count.fetch_add(1, Ordering::SeqCst); + } + + /// Untrack an object (remove from GC lists). + /// O(1) — intrusive linked list remove by node pointer. + /// + /// # Safety + /// obj must be a valid pointer to a PyObject that is currently tracked. + /// The object's memory must still be valid (pointers are read). + pub unsafe fn untrack_object(&self, obj: NonNull<PyObject>) { + let obj_ref = unsafe { obj.as_ref() }; + + loop { + let obj_gen = obj_ref.gc_generation(); + + let (list_lock, count) = if obj_gen <= 2 { + ( + &self.generation_lists[obj_gen as usize] + as &PyRwLock<LinkedList<GcLink, PyObject>>, + &self.generations[obj_gen as usize].count, + ) + } else if obj_gen == GC_PERMANENT { + (&self.permanent_list, &self.permanent.count) + } else { + return; // GC_UNTRACKED or unknown — already untracked + }; + + let mut list = list_lock.write(); + // Re-check generation under lock (may have changed due to promotion) + if obj_ref.gc_generation() != obj_gen { + drop(list); + continue; // Retry with the updated generation + } + if unsafe { list.remove(obj) }.is_some() { + count.fetch_sub(1, Ordering::SeqCst); + obj_ref.clear_gc_tracked(); + obj_ref.set_gc_generation(GC_UNTRACKED); + } else { + // Object claims to be in this generation but wasn't found in the list. + // This indicates a bug: the object was already removed from the list + // without updating gc_generation, or was never inserted. + eprintln!( + "GC WARNING: untrack_object failed to remove obj={obj:p} from gen={obj_gen}, \ + tracked={}, gc_gen={}", + obj_ref.is_gc_tracked(), + obj_ref.gc_generation() + ); + } + return; + } + } + + /// Get tracked objects (for gc.get_objects) + /// If generation is None, returns all tracked objects. + /// If generation is Some(n), returns objects in generation n only. + pub fn get_objects(&self, generation: Option<i32>) -> Vec<PyObjectRef> { + fn collect_from_list( + list: &LinkedList<GcLink, PyObject>, + ) -> impl Iterator<Item = PyObjectRef> + '_ { + list.iter().filter_map(|obj| obj.try_to_owned()) + } + + match generation { + None => { + // Return all tracked objects from all generations + permanent + let mut result = Vec::new(); + for gen_list in &self.generation_lists { + result.extend(collect_from_list(&gen_list.read())); + } + result.extend(collect_from_list(&self.permanent_list.read())); + result + } + Some(g) if (0..=2).contains(&g) => { + let guard = self.generation_lists[g as usize].read(); + collect_from_list(&guard).collect() + } + _ => Vec::new(), + } + } + + /// Check if automatic GC should run and run it if needed. + /// Called after object allocation. + /// Returns true if GC was run, false otherwise. + pub fn maybe_collect(&self) -> bool { + if !self.is_enabled() { + return false; + } + + // Check gen0 threshold + let count0 = self.generations[0].count.load(Ordering::SeqCst) as u32; + let threshold0 = self.generations[0].threshold(); + if threshold0 > 0 && count0 >= threshold0 { + self.collect(0); + return true; + } + + false + } + + /// Perform garbage collection on the given generation + pub fn collect(&self, generation: usize) -> CollectResult { + self.collect_inner(generation, false) + } + + /// Force collection even if GC is disabled (for manual gc.collect() calls) + pub fn collect_force(&self, generation: usize) -> CollectResult { + self.collect_inner(generation, true) + } + + fn collect_inner(&self, generation: usize, force: bool) -> CollectResult { + if !force && !self.is_enabled() { + return CollectResult::default(); + } + + // Try to acquire the collecting lock + let Some(_guard) = self.collecting.try_lock() else { + return CollectResult::default(); + }; + + #[cfg(not(target_arch = "wasm32"))] + let start_time = std::time::Instant::now(); + #[cfg(target_arch = "wasm32")] + let start_time = (); + + // Memory barrier to ensure visibility of all reference count updates + // from other threads before we start analyzing the object graph. + core::sync::atomic::fence(Ordering::SeqCst); + + let generation = generation.min(2); + let debug = self.get_debug(); + + // Clear the method cache to release strong references that + // might prevent cycle collection (_PyType_ClearCache). + crate::builtins::type_::type_cache_clear(); + + // Step 1: Gather objects from generations 0..=generation + // Hold read locks for the entire scan to prevent concurrent modifications. + let gen_locks: Vec<_> = (0..=generation) + .map(|i| self.generation_lists[i].read()) + .collect(); + + let mut collecting: HashSet<GcPtr> = HashSet::new(); + for gen_list in &gen_locks { + for obj in gen_list.iter() { + if obj.strong_count() > 0 { + collecting.insert(GcPtr(NonNull::from(obj))); + } + } + } + + if collecting.is_empty() { + // Reset counts for generations whose objects were promoted away. + // For gen2 (oldest), survivors stay in-place so don't reset gen2 count. + let reset_end = if generation >= 2 { 2 } else { generation + 1 }; + for i in 0..reset_end { + self.generations[i].count.store(0, Ordering::SeqCst); + } + let duration = elapsed_secs(&start_time); + self.generations[generation].update_stats(0, 0, 0, duration); + return CollectResult { + collected: 0, + uncollectable: 0, + candidates: 0, + duration, + }; + } + + let candidates = collecting.len(); + + if debug.contains(GcDebugFlags::STATS) { + eprintln!( + "gc: collecting {} objects from generations 0..={}", + collecting.len(), + generation + ); + } + + // Step 2: Build gc_refs map (copy reference counts) + let mut gc_refs: std::collections::HashMap<GcPtr, usize> = std::collections::HashMap::new(); + for &ptr in &collecting { + let obj = unsafe { ptr.0.as_ref() }; + gc_refs.insert(ptr, obj.strong_count()); + } + + // Step 3: Subtract internal references + for &ptr in &collecting { + let obj = unsafe { ptr.0.as_ref() }; + if obj.strong_count() == 0 { + continue; + } + let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; + for child_ptr in referent_ptrs { + let gc_ptr = GcPtr(child_ptr); + if collecting.contains(&gc_ptr) + && let Some(refs) = gc_refs.get_mut(&gc_ptr) + { + *refs = refs.saturating_sub(1); + } + } + } + + // Step 4: Find reachable objects (gc_refs > 0) and traverse from them + let mut reachable: HashSet<GcPtr> = HashSet::new(); + let mut worklist: Vec<GcPtr> = Vec::new(); + + for (&ptr, &refs) in &gc_refs { + if refs > 0 { + reachable.insert(ptr); + worklist.push(ptr); + } + } + + while let Some(ptr) = worklist.pop() { + let obj = unsafe { ptr.0.as_ref() }; + if obj.is_gc_tracked() { + let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; + for child_ptr in referent_ptrs { + let gc_ptr = GcPtr(child_ptr); + if collecting.contains(&gc_ptr) && reachable.insert(gc_ptr) { + worklist.push(gc_ptr); + } + } + } + } + + // Step 5: Find unreachable objects + let unreachable: Vec<GcPtr> = collecting.difference(&reachable).copied().collect(); + + if debug.contains(GcDebugFlags::STATS) { + eprintln!( + "gc: {} reachable, {} unreachable", + reachable.len(), + unreachable.len() + ); + } + + // Create strong references while read locks are still held. + // After dropping gen_locks, other threads can untrack+free objects, + // making the raw pointers in `reachable`/`unreachable` dangling. + // Strong refs keep objects alive for later phases. + // + // Use try_to_owned() (CAS-based) instead of strong_count()+to_owned() + // to prevent a TOCTOU race: another thread can dec() the count to 0 + // between the check and the increment, causing a use-after-free when + // the destroying thread eventually frees the memory. + let survivor_refs: Vec<PyObjectRef> = reachable + .iter() + .filter_map(|ptr| { + let obj = unsafe { ptr.0.as_ref() }; + obj.try_to_owned() + }) + .collect(); + + let unreachable_refs: Vec<crate::PyObjectRef> = unreachable + .iter() + .filter_map(|ptr| { + let obj = unsafe { ptr.0.as_ref() }; + obj.try_to_owned() + }) + .collect(); + + if unreachable.is_empty() { + drop(gen_locks); + self.promote_survivors(generation, &survivor_refs); + let reset_end = if generation >= 2 { 2 } else { generation + 1 }; + for i in 0..reset_end { + self.generations[i].count.store(0, Ordering::SeqCst); + } + let duration = elapsed_secs(&start_time); + self.generations[generation].update_stats(0, 0, candidates, duration); + return CollectResult { + collected: 0, + uncollectable: 0, + candidates, + duration, + }; + } + + // Release read locks before finalization phase. + drop(gen_locks); + + // Step 6: Finalize unreachable objects and handle resurrection + + if unreachable_refs.is_empty() { + self.promote_survivors(generation, &survivor_refs); + let reset_end = if generation >= 2 { 2 } else { generation + 1 }; + for i in 0..reset_end { + self.generations[i].count.store(0, Ordering::SeqCst); + } + let duration = elapsed_secs(&start_time); + self.generations[generation].update_stats(0, 0, candidates, duration); + return CollectResult { + collected: 0, + uncollectable: 0, + candidates, + duration, + }; + } + + // 6b: Record initial strong counts (for resurrection detection) + let initial_counts: std::collections::HashMap<GcPtr, usize> = unreachable_refs + .iter() + .map(|obj| { + let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); + (ptr, obj.strong_count()) + }) + .collect(); + + // 6c: Clear existing weakrefs BEFORE calling __del__ + let mut all_callbacks: Vec<(crate::PyRef<crate::object::PyWeak>, crate::PyObjectRef)> = + Vec::new(); + for obj_ref in &unreachable_refs { + let callbacks = obj_ref.gc_clear_weakrefs_collect_callbacks(); + all_callbacks.extend(callbacks); + } + for (wr, cb) in all_callbacks { + if let Some(Err(e)) = crate::vm::thread::with_vm(&cb, |vm| cb.call((wr.clone(),), vm)) { + crate::vm::thread::with_vm(&cb, |vm| { + vm.run_unraisable(e.clone(), Some("weakref callback".to_owned()), cb.clone()); + }); + } + } + + // 6d: Call __del__ on unreachable objects (skip already-finalized). + // try_call_finalizer() internally checks gc_finalized() and sets it, + // so we must NOT set it beforehand. + for obj_ref in &unreachable_refs { + obj_ref.try_call_finalizer(); + } + + // Detect resurrection + let mut resurrected_set: HashSet<GcPtr> = HashSet::new(); + let unreachable_set: HashSet<GcPtr> = unreachable.iter().copied().collect(); + + for obj in &unreachable_refs { + let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); + let initial = initial_counts.get(&ptr).copied().unwrap_or(1); + if obj.strong_count() > initial { + resurrected_set.insert(ptr); + } + } + + // Transitive resurrection + let mut worklist: Vec<GcPtr> = resurrected_set.iter().copied().collect(); + while let Some(ptr) = worklist.pop() { + let obj = unsafe { ptr.0.as_ref() }; + let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; + for child_ptr in referent_ptrs { + let child_gc_ptr = GcPtr(child_ptr); + if unreachable_set.contains(&child_gc_ptr) && resurrected_set.insert(child_gc_ptr) { + worklist.push(child_gc_ptr); + } + } + } + + // Partition into resurrected and truly dead + let (resurrected, truly_dead): (Vec<_>, Vec<_>) = + unreachable_refs.into_iter().partition(|obj| { + let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); + resurrected_set.contains(&ptr) + }); + + if debug.contains(GcDebugFlags::STATS) { + eprintln!( + "gc: {} resurrected, {} truly dead", + resurrected.len(), + truly_dead.len() + ); + } + + // Compute collected count (exclude instance dicts in truly_dead) + let collected = { + let dead_ptrs: HashSet<usize> = truly_dead + .iter() + .map(|obj| obj.as_ref() as *const PyObject as usize) + .collect(); + let instance_dict_count = truly_dead + .iter() + .filter(|obj| { + if let Some(dict_ref) = obj.dict() { + dead_ptrs.contains(&(dict_ref.as_object() as *const PyObject as usize)) + } else { + false + } + }) + .count(); + truly_dead.len() - instance_dict_count + }; + + // Promote survivors to next generation BEFORE tp_clear. + // move_legacy_finalizer_reachable → delete_garbage order ensures + // survivor_refs are dropped before tp_clear, so reachable objects + // aren't kept alive beyond the deferred-drop phase. + self.promote_survivors(generation, &survivor_refs); + drop(survivor_refs); + + // Resurrected objects stay tracked — just drop our references + drop(resurrected); + + if debug.contains(GcDebugFlags::COLLECTABLE) { + for obj in &truly_dead { + eprintln!( + "gc: collectable <{} {:p}>", + obj.class().name(), + obj.as_ref() + ); + } + } + + if debug.contains(GcDebugFlags::SAVEALL) { + let mut garbage_guard = self.garbage.lock(); + for obj_ref in truly_dead.iter() { + garbage_guard.push(obj_ref.clone()); + } + } + + if !truly_dead.is_empty() { + // Break cycles by clearing references (tp_clear) + // Use deferred drop context to prevent stack overflow. + rustpython_common::refcount::with_deferred_drops(|| { + for obj_ref in truly_dead.iter() { + if obj_ref.gc_has_clear() { + let edges = unsafe { obj_ref.gc_clear() }; + drop(edges); + } + } + drop(truly_dead); + }); + } + + // Reset counts for generations whose objects were promoted away. + // For gen2 (oldest), survivors stay in-place so don't reset gen2 count. + let reset_end = if generation >= 2 { 2 } else { generation + 1 }; + for i in 0..reset_end { + self.generations[i].count.store(0, Ordering::SeqCst); + } + + let duration = elapsed_secs(&start_time); + self.generations[generation].update_stats(collected, 0, candidates, duration); + + CollectResult { + collected, + uncollectable: 0, + candidates, + duration, + } + } + + /// Promote surviving objects to the next generation. + /// + /// `survivors` must be strong references (`PyObjectRef`) to keep objects alive, + /// since the generation read locks are released before this is called. + /// + /// Holds both source and destination list locks simultaneously to prevent + /// a race where concurrent `untrack_object` reads a stale `gc_generation` + /// and operates on the wrong list. + fn promote_survivors(&self, from_gen: usize, survivors: &[PyObjectRef]) { + if from_gen >= 2 { + return; // Already in oldest generation + } + + let next_gen = from_gen + 1; + + for obj_ref in survivors { + let obj = obj_ref.as_ref(); + let ptr = NonNull::from(obj); + let obj_gen = obj.gc_generation(); + if obj_gen as usize <= from_gen && obj_gen <= 2 { + let src_gen = obj_gen as usize; + + // Lock both source and destination lists simultaneously. + // Always ascending order (src_gen < next_gen) → no deadlock. + let mut src = self.generation_lists[src_gen].write(); + let mut dst = self.generation_lists[next_gen].write(); + + // Re-check under locks: object might have been untracked concurrently + if obj.gc_generation() != obj_gen || !obj.is_gc_tracked() { + continue; + } + + if unsafe { src.remove(ptr) }.is_some() { + self.generations[src_gen] + .count + .fetch_sub(1, Ordering::SeqCst); + + dst.push_front(ptr); + self.generations[next_gen] + .count + .fetch_add(1, Ordering::SeqCst); + + obj.set_gc_generation(next_gen as u8); + } + } + } + } + + /// Get count of frozen objects + pub fn get_freeze_count(&self) -> usize { + self.permanent.count() + } + + /// Freeze all tracked objects (move to permanent generation). + /// Lock order: generation_lists[i] → permanent_list (consistent with unfreeze). + pub fn freeze(&self) { + let mut count = 0usize; + + for (gen_idx, gen_list) in self.generation_lists.iter().enumerate() { + let mut list = gen_list.write(); + let mut perm = self.permanent_list.write(); + while let Some(ptr) = list.pop_front() { + perm.push_front(ptr); + unsafe { ptr.as_ref().set_gc_generation(GC_PERMANENT) }; + count += 1; + } + self.generations[gen_idx].count.store(0, Ordering::SeqCst); + } + + self.permanent.count.fetch_add(count, Ordering::SeqCst); + } + + /// Unfreeze all objects (move from permanent to gen2). + /// Lock order: generation_lists[2] → permanent_list (consistent with freeze). + pub fn unfreeze(&self) { + let mut count = 0usize; + + { + let mut gen2 = self.generation_lists[2].write(); + let mut perm_list = self.permanent_list.write(); + while let Some(ptr) = perm_list.pop_front() { + gen2.push_front(ptr); + unsafe { ptr.as_ref().set_gc_generation(2) }; + count += 1; + } + self.permanent.count.store(0, Ordering::SeqCst); + } + + self.generations[2].count.fetch_add(count, Ordering::SeqCst); + } + + /// Reset all locks to unlocked state after fork(). + /// + /// After fork(), only the forking thread survives. Any lock held by another + /// thread is permanently stuck. This resets them by zeroing the raw bytes. + /// + /// # Safety + /// Must only be called after fork() in the child process when no other + /// threads exist. The calling thread must NOT hold any of these locks. + #[cfg(all(unix, feature = "threading"))] + pub unsafe fn reinit_after_fork(&self) { + use crate::common::lock::{reinit_mutex_after_fork, reinit_rwlock_after_fork}; + + unsafe { + reinit_mutex_after_fork(&self.collecting); + reinit_mutex_after_fork(&self.garbage); + reinit_mutex_after_fork(&self.callbacks); + + for generation in &self.generations { + generation.reinit_stats_after_fork(); + } + self.permanent.reinit_stats_after_fork(); + + for rw in &self.generation_lists { + reinit_rwlock_after_fork(rw); + } + reinit_rwlock_after_fork(&self.permanent_list); + } + } +} + +/// Get a reference to the GC state. +/// +/// In threading mode this is a true global (OnceLock). +/// In non-threading mode this is thread-local, because PyRwLock/PyMutex +/// use Cell-based locks that are not Sync. +pub fn gc_state() -> &'static GcState { + rustpython_common::static_cell! { + static GC_STATE: GcState; + } + GC_STATE.get_or_init(GcState::new) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gc_state_default() { + let state = GcState::new(); + assert!(state.is_enabled()); + assert_eq!(state.get_debug(), GcDebugFlags::empty()); + assert_eq!(state.get_threshold(), (2000, 10, 0)); + assert_eq!(state.get_count(), (0, 0, 0)); + } + + #[test] + fn test_gc_enable_disable() { + let state = GcState::new(); + assert!(state.is_enabled()); + state.disable(); + assert!(!state.is_enabled()); + state.enable(); + assert!(state.is_enabled()); + } + + #[test] + fn test_gc_threshold() { + let state = GcState::new(); + state.set_threshold(100, Some(20), Some(30)); + assert_eq!(state.get_threshold(), (100, 20, 30)); + } + + #[test] + fn test_gc_debug_flags() { + let state = GcState::new(); + state.set_debug(GcDebugFlags::STATS | GcDebugFlags::COLLECTABLE); + assert_eq!( + state.get_debug(), + GcDebugFlags::STATS | GcDebugFlags::COLLECTABLE + ); + } +} diff --git a/crates/vm/src/getpath.rs b/crates/vm/src/getpath.rs new file mode 100644 index 00000000000..31fa0617b45 --- /dev/null +++ b/crates/vm/src/getpath.rs @@ -0,0 +1,411 @@ +//! Path configuration for RustPython (ref: Modules/getpath.py) +//! +//! This module implements Python path calculation logic following getpath.py. +//! It uses landmark-based search to locate prefix, exec_prefix, and stdlib directories. +//! +//! The main entry point is `init_path_config()` which computes Paths from Settings. + +use crate::vm::{Paths, Settings}; +use std::env; +use std::path::{Path, PathBuf}; + +// Platform-specific landmarks (ref: getpath.py PLATFORM CONSTANTS) + +#[cfg(not(windows))] +mod platform { + use crate::version; + + pub const BUILDDIR_TXT: &str = "pybuilddir.txt"; + pub const BUILD_LANDMARK: &str = "Modules/Setup.local"; + pub const VENV_LANDMARK: &str = "pyvenv.cfg"; + pub const BUILDSTDLIB_LANDMARK: &str = "Lib/os.py"; + + pub fn stdlib_subdir() -> String { + format!("lib/python{}.{}", version::MAJOR, version::MINOR) + } + + pub fn stdlib_landmarks() -> [String; 2] { + let subdir = stdlib_subdir(); + [format!("{}/os.py", subdir), format!("{}/os.pyc", subdir)] + } + + pub fn platstdlib_landmark() -> String { + format!( + "lib/python{}.{}/lib-dynload", + version::MAJOR, + version::MINOR + ) + } + + pub fn zip_landmark() -> String { + format!("lib/python{}{}.zip", version::MAJOR, version::MINOR) + } +} + +#[cfg(windows)] +mod platform { + use crate::version; + + pub const BUILDDIR_TXT: &str = "pybuilddir.txt"; + pub const BUILD_LANDMARK: &str = "Modules\\Setup.local"; + pub const VENV_LANDMARK: &str = "pyvenv.cfg"; + pub const BUILDSTDLIB_LANDMARK: &str = "Lib\\os.py"; + pub const STDLIB_SUBDIR: &str = "Lib"; + + pub fn stdlib_landmarks() -> [String; 2] { + ["Lib\\os.py".into(), "Lib\\os.pyc".into()] + } + + pub fn platstdlib_landmark() -> String { + "DLLs".into() + } + + pub fn zip_landmark() -> String { + format!("python{}{}.zip", version::MAJOR, version::MINOR) + } +} + +// Helper functions (ref: getpath.py HELPER FUNCTIONS) + +/// Search upward from a directory for landmark files/directories +/// Returns the directory where a landmark was found +fn search_up<P, F>(start: P, landmarks: &[&str], test: F) -> Option<PathBuf> +where + P: AsRef<Path>, + F: Fn(&Path) -> bool, +{ + let mut current = start.as_ref().to_path_buf(); + loop { + for landmark in landmarks { + let path = current.join(landmark); + if test(&path) { + return Some(current); + } + } + if !current.pop() { + return None; + } + } +} + +/// Search upward for a file landmark +fn search_up_file<P: AsRef<Path>>(start: P, landmarks: &[&str]) -> Option<PathBuf> { + search_up(start, landmarks, |p| p.is_file()) +} + +/// Search upward for a directory landmark +#[cfg(not(windows))] +fn search_up_dir<P: AsRef<Path>>(start: P, landmarks: &[&str]) -> Option<PathBuf> { + search_up(start, landmarks, |p| p.is_dir()) +} + +// Path computation functions + +/// Compute path configuration from Settings +/// +/// This function should be called before interpreter initialization. +/// It returns a Paths struct with all computed path values. +pub fn init_path_config(settings: &Settings) -> Paths { + let mut paths = Paths::default(); + + // Step 0: Get executable path + let executable = get_executable_path(); + let real_executable = executable + .as_ref() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + + // Step 1: Check for __PYVENV_LAUNCHER__ environment variable + // When launched from a venv launcher, __PYVENV_LAUNCHER__ contains the venv's python.exe path + // In this case: + // - sys.executable should be the launcher path (where user invoked Python) + // - sys._base_executable should be the real Python executable + let exe_dir = if let Ok(launcher) = env::var("__PYVENV_LAUNCHER__") { + paths.executable = launcher.clone(); + paths.base_executable = real_executable; + PathBuf::from(&launcher).parent().map(PathBuf::from) + } else { + paths.executable = real_executable; + executable + .as_ref() + .and_then(|p| p.parent().map(PathBuf::from)) + }; + + // Step 2: Check for venv (pyvenv.cfg) and get 'home' + let (venv_prefix, home_dir) = detect_venv(&exe_dir); + let search_dir = home_dir.clone().or(exe_dir.clone()); + + // Step 3: Check for build directory + let build_prefix = detect_build_directory(&search_dir); + + // Step 4: Calculate prefix via landmark search + // When in venv, search_dir is home_dir, so this gives us the base Python's prefix + let calculated_prefix = calculate_prefix(&search_dir, &build_prefix); + + // Step 5: Set prefix and base_prefix + if venv_prefix.is_some() { + // In venv: prefix = venv directory, base_prefix = original Python's prefix + paths.prefix = venv_prefix + .as_ref() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_else(|| calculated_prefix.clone()); + paths.base_prefix = calculated_prefix; + } else { + // Not in venv: prefix == base_prefix + paths.prefix = calculated_prefix.clone(); + paths.base_prefix = calculated_prefix; + } + + // Step 6: Calculate exec_prefix + paths.exec_prefix = if venv_prefix.is_some() { + // In venv: exec_prefix = prefix (venv directory) + paths.prefix.clone() + } else { + calculate_exec_prefix(&search_dir, &paths.prefix) + }; + paths.base_exec_prefix = paths.base_prefix.clone(); + + // Step 7: Calculate base_executable (if not already set by __PYVENV_LAUNCHER__) + if paths.base_executable.is_empty() { + paths.base_executable = calculate_base_executable(executable.as_ref(), &home_dir); + } + + // Step 8: Build module_search_paths + paths.module_search_paths = + build_module_search_paths(settings, &paths.prefix, &paths.exec_prefix); + + // Step 9: Calculate stdlib_dir + paths.stdlib_dir = calculate_stdlib_dir(&paths.prefix); + + paths +} + +/// Get default prefix value +fn default_prefix() -> String { + std::option_env!("RUSTPYTHON_PREFIX") + .map(String::from) + .unwrap_or_else(|| { + if cfg!(windows) { + "C:".to_owned() + } else { + "/usr/local".to_owned() + } + }) +} + +/// Detect virtual environment by looking for pyvenv.cfg +/// Returns (venv_prefix, home_dir from pyvenv.cfg) +fn detect_venv(exe_dir: &Option<PathBuf>) -> (Option<PathBuf>, Option<PathBuf>) { + // Try exe_dir/../pyvenv.cfg first (standard venv layout: venv/bin/python) + if let Some(dir) = exe_dir + && let Some(venv_dir) = dir.parent() + { + let cfg = venv_dir.join(platform::VENV_LANDMARK); + if cfg.exists() + && let Some(home) = parse_pyvenv_home(&cfg) + { + return (Some(venv_dir.to_path_buf()), Some(PathBuf::from(home))); + } + } + + // Try exe_dir/pyvenv.cfg (alternative layout) + if let Some(dir) = exe_dir { + let cfg = dir.join(platform::VENV_LANDMARK); + if cfg.exists() + && let Some(home) = parse_pyvenv_home(&cfg) + { + return (Some(dir.clone()), Some(PathBuf::from(home))); + } + } + + (None, None) +} + +/// Detect if running from a build directory +fn detect_build_directory(exe_dir: &Option<PathBuf>) -> Option<PathBuf> { + let dir = exe_dir.as_ref()?; + + // Check for pybuilddir.txt (indicates build directory) + if dir.join(platform::BUILDDIR_TXT).exists() { + return Some(dir.clone()); + } + + // Check for Modules/Setup.local (build landmark) + if dir.join(platform::BUILD_LANDMARK).exists() { + return Some(dir.clone()); + } + + // Search up for Lib/os.py (build stdlib landmark) + search_up_file(dir, &[platform::BUILDSTDLIB_LANDMARK]) +} + +/// Calculate prefix by searching for landmarks +fn calculate_prefix(exe_dir: &Option<PathBuf>, build_prefix: &Option<PathBuf>) -> String { + // 1. If build directory detected, use it + if let Some(bp) = build_prefix { + return bp.to_string_lossy().into_owned(); + } + + if let Some(dir) = exe_dir { + // 2. Search for ZIP landmark + let zip = platform::zip_landmark(); + if let Some(prefix) = search_up_file(dir, &[&zip]) { + return prefix.to_string_lossy().into_owned(); + } + + // 3. Search for stdlib landmarks (os.py) + let landmarks = platform::stdlib_landmarks(); + let refs: Vec<&str> = landmarks.iter().map(|s| s.as_str()).collect(); + if let Some(prefix) = search_up_file(dir, &refs) { + return prefix.to_string_lossy().into_owned(); + } + } + + // 4. Fallback to default + default_prefix() +} + +/// Calculate exec_prefix +fn calculate_exec_prefix(exe_dir: &Option<PathBuf>, prefix: &str) -> String { + #[cfg(windows)] + { + // Windows: exec_prefix == prefix + let _ = exe_dir; // silence unused warning + prefix.to_owned() + } + + #[cfg(not(windows))] + { + // POSIX: search for lib-dynload directory + if let Some(dir) = exe_dir { + let landmark = platform::platstdlib_landmark(); + if let Some(exec_prefix) = search_up_dir(dir, &[&landmark]) { + return exec_prefix.to_string_lossy().into_owned(); + } + } + // Fallback: same as prefix + prefix.to_owned() + } +} + +/// Calculate base_executable +fn calculate_base_executable(executable: Option<&PathBuf>, home_dir: &Option<PathBuf>) -> String { + // If in venv and we have home, construct base_executable from home + if let (Some(exe), Some(home)) = (executable, home_dir) + && let Some(exe_name) = exe.file_name() + { + let base = home.join(exe_name); + return base.to_string_lossy().into_owned(); + } + + // Otherwise, base_executable == executable + executable + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Calculate stdlib_dir (sys._stdlib_dir) +/// Returns None if the stdlib directory doesn't exist +fn calculate_stdlib_dir(prefix: &str) -> Option<String> { + #[cfg(not(windows))] + let stdlib_dir = PathBuf::from(prefix).join(platform::stdlib_subdir()); + + #[cfg(windows)] + let stdlib_dir = PathBuf::from(prefix).join(platform::STDLIB_SUBDIR); + + if stdlib_dir.is_dir() { + Some(stdlib_dir.to_string_lossy().into_owned()) + } else { + None + } +} + +/// Build the complete module_search_paths (sys.path) +fn build_module_search_paths(settings: &Settings, prefix: &str, exec_prefix: &str) -> Vec<String> { + let mut paths = Vec::new(); + + // 1. PYTHONPATH/RUSTPYTHONPATH from settings + paths.extend(settings.path_list.iter().cloned()); + + // 2. ZIP file path + let zip_path = PathBuf::from(prefix).join(platform::zip_landmark()); + paths.push(zip_path.to_string_lossy().into_owned()); + + // 3. stdlib and platstdlib directories + #[cfg(not(windows))] + { + // POSIX: stdlib first, then lib-dynload + let stdlib_dir = PathBuf::from(prefix).join(platform::stdlib_subdir()); + paths.push(stdlib_dir.to_string_lossy().into_owned()); + + let platstdlib = PathBuf::from(exec_prefix).join(platform::platstdlib_landmark()); + paths.push(platstdlib.to_string_lossy().into_owned()); + } + + #[cfg(windows)] + { + // Windows: DLLs first, then Lib + let platstdlib = PathBuf::from(exec_prefix).join(platform::platstdlib_landmark()); + paths.push(platstdlib.to_string_lossy().into_owned()); + + let stdlib_dir = PathBuf::from(prefix).join(platform::STDLIB_SUBDIR); + paths.push(stdlib_dir.to_string_lossy().into_owned()); + } + + paths +} + +/// Get the current executable path +fn get_executable_path() -> Option<PathBuf> { + #[cfg(not(target_arch = "wasm32"))] + { + let exec_arg = env::args_os().next()?; + which::which(exec_arg).ok() + } + #[cfg(target_arch = "wasm32")] + { + let exec_arg = env::args().next()?; + Some(PathBuf::from(exec_arg)) + } +} + +/// Parse pyvenv.cfg and extract the 'home' key value +fn parse_pyvenv_home(pyvenv_cfg: &Path) -> Option<String> { + let content = std::fs::read_to_string(pyvenv_cfg).ok()?; + + for line in content.lines() { + if let Some((key, value)) = line.split_once('=') + && key.trim().to_lowercase() == "home" + { + return Some(value.trim().to_string()); + } + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_init_path_config() { + let settings = Settings::default(); + let paths = init_path_config(&settings); + // Just verify it doesn't panic and returns valid paths + assert!(!paths.prefix.is_empty()); + } + + #[test] + fn test_search_up() { + // Test with a path that doesn't have any landmarks + let result = search_up_file(std::env::temp_dir(), &["nonexistent_landmark_xyz"]); + assert!(result.is_none()); + } + + #[test] + fn test_default_prefix() { + let prefix = default_prefix(); + assert!(!prefix.is_empty()); + } +} diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs new file mode 100644 index 00000000000..9d015c8f3b6 --- /dev/null +++ b/crates/vm/src/import.rs @@ -0,0 +1,612 @@ +//! Import mechanics + +use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, + builtins::{PyCode, PyStr, PyUtf8Str, PyUtf8StrRef, traceback::PyTraceback}, + exceptions::types::PyBaseException, + scope::Scope, + vm::{VirtualMachine, resolve_frozen_alias, thread}, +}; + +pub(crate) fn check_pyc_magic_number_bytes(buf: &[u8]) -> bool { + buf.starts_with(&crate::version::PYC_MAGIC_NUMBER_BYTES[..2]) +} + +pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult<PyObjectRef> { + flame_guard!("init importlib"); + + // importlib_bootstrap needs these and it inlines checks to sys.modules before calling into + // import machinery, so this should bring some speedup + #[cfg(all(feature = "threading", not(target_os = "wasi")))] + import_builtin(vm, "_thread")?; + import_builtin(vm, "_warnings")?; + import_builtin(vm, "_weakref")?; + + let importlib = thread::enter_vm(vm, || { + let bootstrap = import_frozen(vm, "_frozen_importlib")?; + let install = bootstrap.get_attr("_install", vm)?; + let imp = import_builtin(vm, "_imp")?; + install.call((vm.sys_module.clone(), imp), vm)?; + Ok(bootstrap) + })?; + vm.import_func = importlib.get_attr(identifier!(vm, __import__), vm)?; + vm.importlib = importlib.clone(); + Ok(importlib) +} + +#[cfg(feature = "host_env")] +pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef) -> PyResult<()> { + use crate::{TryFromObject, builtins::PyListRef}; + + thread::enter_vm(vm, || { + flame_guard!("install_external"); + + // same deal as imports above + import_builtin(vm, crate::stdlib::os::MODULE_NAME)?; + #[cfg(windows)] + import_builtin(vm, "winreg")?; + import_builtin(vm, "_io")?; + import_builtin(vm, "marshal")?; + + let install_external = importlib.get_attr("_install_external_importers", vm)?; + install_external.call((), vm)?; + let zipimport_res = (|| -> PyResult<()> { + let zipimport = vm.import("zipimport", 0)?; + let zipimporter = zipimport.get_attr("zipimporter", vm)?; + let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?; + let path_hooks = PyListRef::try_from_object(vm, path_hooks)?; + path_hooks.insert(0, zipimporter); + Ok(()) + })(); + if zipimport_res.is_err() { + warn!("couldn't init zipimport") + } + Ok(()) + }) +} + +pub fn make_frozen(vm: &VirtualMachine, name: &str) -> PyResult<PyRef<PyCode>> { + let frozen = vm.state.frozen.get(name).ok_or_else(|| { + vm.new_import_error( + format!("No such frozen object named {name}"), + vm.ctx.new_utf8_str(name), + ) + })?; + Ok(vm.ctx.new_code(frozen.code)) +} + +pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { + let frozen = vm.state.frozen.get(module_name).ok_or_else(|| { + vm.new_import_error( + format!("No such frozen object named {module_name}"), + vm.ctx.new_utf8_str(module_name), + ) + })?; + let module = import_code_obj(vm, module_name, vm.ctx.new_code(frozen.code), false)?; + debug_assert!(module.get_attr(identifier!(vm, __name__), vm).is_ok()); + let origname = resolve_frozen_alias(module_name); + module.set_attr("__origname__", vm.ctx.new_utf8_str(origname), vm)?; + Ok(module) +} + +pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + + // Check if already in sys.modules (handles recursive imports) + if let Ok(module) = sys_modules.get_item(module_name, vm) { + return Ok(module); + } + + // Try multi-phase init first (preferred for modules that import other modules) + if let Some(&def) = vm.state.module_defs.get(module_name) { + // Phase 1: Create and initialize module + let module = def.create_module(vm)?; + + // Add to sys.modules BEFORE exec (critical for circular import handling) + sys_modules.set_item(module_name, module.clone().into(), vm)?; + + // Phase 2: Call exec slot (can safely import other modules now) + // If exec fails, remove the partially-initialized module from sys.modules + if let Err(e) = def.exec_module(vm, &module) { + let _ = sys_modules.del_item(module_name, vm); + return Err(e); + } + + return Ok(module.into()); + } + + // Module not found in module_defs + Err(vm.new_import_error( + format!("Cannot import builtin module {module_name}"), + vm.ctx.new_utf8_str(module_name), + )) +} + +#[cfg(feature = "rustpython-compiler")] +pub fn import_file( + vm: &VirtualMachine, + module_name: &str, + file_path: String, + content: &str, +) -> PyResult { + let code = vm + .compile_with_opts( + content, + crate::compiler::Mode::Exec, + file_path, + vm.compile_opts(), + ) + .map_err(|err| vm.new_syntax_error(&err, Some(content)))?; + import_code_obj(vm, module_name, code, true) +} + +#[cfg(feature = "rustpython-compiler")] +pub fn import_source(vm: &VirtualMachine, module_name: &str, content: &str) -> PyResult { + let code = vm + .compile_with_opts( + content, + crate::compiler::Mode::Exec, + "<source>".to_owned(), + vm.compile_opts(), + ) + .map_err(|err| vm.new_syntax_error(&err, Some(content)))?; + import_code_obj(vm, module_name, code, false) +} + +/// If `__spec__._initializing` is true, wait for the module to finish +/// initializing by calling `_lock_unlock_module`. +fn import_ensure_initialized( + module: &PyObjectRef, + name: &str, + vm: &VirtualMachine, +) -> PyResult<()> { + let initializing = match vm.get_attribute_opt(module.clone(), vm.ctx.intern_str("__spec__"))? { + Some(spec) => match vm.get_attribute_opt(spec, vm.ctx.intern_str("_initializing"))? { + Some(v) => v.try_to_bool(vm)?, + None => false, + }, + None => false, + }; + if initializing { + let lock_unlock = vm.importlib.get_attr("_lock_unlock_module", vm)?; + lock_unlock.call((vm.ctx.new_utf8_str(name),), vm)?; + } + Ok(()) +} + +pub fn import_code_obj( + vm: &VirtualMachine, + module_name: &str, + code_obj: PyRef<PyCode>, + set_file_attr: bool, +) -> PyResult { + let attrs = vm.ctx.new_dict(); + attrs.set_item( + identifier!(vm, __name__), + vm.ctx.new_utf8_str(module_name).into(), + vm, + )?; + if set_file_attr { + attrs.set_item( + identifier!(vm, __file__), + code_obj.source_path().to_object(), + vm, + )?; + } + let module = vm.new_module(module_name, attrs.clone(), None); + + // Store module in cache to prevent infinite loop with mutual importing libs: + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + sys_modules.set_item(module_name, module.clone().into(), vm)?; + + // Execute main code in module: + let scope = Scope::with_builtins(None, attrs, vm); + vm.run_code_obj(code_obj, scope)?; + Ok(module.into()) +} + +fn remove_importlib_frames_inner( + vm: &VirtualMachine, + tb: Option<PyRef<PyTraceback>>, + always_trim: bool, +) -> (Option<PyRef<PyTraceback>>, bool) { + let traceback = if let Some(tb) = tb { + tb + } else { + return (None, false); + }; + + let file_name = traceback.frame.code.source_path().as_str(); + + let (inner_tb, mut now_in_importlib) = + remove_importlib_frames_inner(vm, traceback.next.lock().clone(), always_trim); + if file_name == "_frozen_importlib" || file_name == "_frozen_importlib_external" { + if traceback.frame.code.obj_name.as_str() == "_call_with_frames_removed" { + now_in_importlib = true; + } + if always_trim || now_in_importlib { + return (inner_tb, now_in_importlib); + } + } else { + now_in_importlib = false; + } + + ( + Some( + PyTraceback::new( + inner_tb, + traceback.frame.clone(), + traceback.lasti, + traceback.lineno, + ) + .into_ref(&vm.ctx), + ), + now_in_importlib, + ) +} + +// TODO: This function should do nothing on verbose mode. +// TODO: Fix this function after making PyTraceback.next mutable +pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &Py<PyBaseException>) { + if vm.state.config.settings.verbose != 0 { + return; + } + + let always_trim = exc.fast_isinstance(vm.ctx.exceptions.import_error); + + if let Some(tb) = exc.__traceback__() { + let trimmed_tb = remove_importlib_frames_inner(vm, Some(tb), always_trim).0; + exc.set_traceback_typed(trimmed_tb); + } +} + +/// Get origin path from a module spec, checking has_location first. +pub(crate) fn get_spec_file_origin( + spec: &Option<PyObjectRef>, + vm: &VirtualMachine, +) -> Option<String> { + let spec = spec.as_ref()?; + let has_location = spec + .get_attr("has_location", vm) + .ok() + .and_then(|v| v.try_to_bool(vm).ok()) + .unwrap_or(false); + if !has_location { + return None; + } + spec.get_attr("origin", vm).ok().and_then(|origin| { + if vm.is_none(&origin) { + None + } else { + origin + .downcast_ref::<PyStr>() + .and_then(|s| s.to_str().map(|s| s.to_owned())) + } + }) +} + +/// Check if a module file possibly shadows another module of the same name. +/// Compares the module's directory with the original sys.path[0] (derived from sys.argv[0]). +pub(crate) fn is_possibly_shadowing_path(origin: &str, vm: &VirtualMachine) -> bool { + use std::path::Path; + + if vm.state.config.settings.safe_path { + return false; + } + + let origin_path = Path::new(origin); + let parent = match origin_path.parent() { + Some(p) => p, + None => return false, + }; + // For packages (__init__.py), look one directory further up + let root = if origin_path.file_name() == Some("__init__.py".as_ref()) { + parent.parent().unwrap_or(Path::new("")) + } else { + parent + }; + + // Compute original sys.path[0] from sys.argv[0] (the script path). + // See: config->sys_path_0, which is set once + // at initialization and never changes even if sys.path is modified. + let sys_path_0 = (|| -> Option<String> { + let argv = vm.sys_module.get_attr("argv", vm).ok()?; + let argv0 = argv.get_item(&0usize, vm).ok()?; + let argv0_str = argv0.downcast_ref::<PyUtf8Str>()?; + let s = argv0_str.as_str(); + + // For -c and REPL, original sys.path[0] is "" + if s == "-c" || s.is_empty() { + return Some(String::new()); + } + // For scripts, original sys.path[0] is dirname(argv[0]) + Some( + Path::new(s) + .parent() + .and_then(|p| p.to_str()) + .unwrap_or("") + .to_owned(), + ) + })(); + + let sys_path_0 = match sys_path_0 { + Some(p) => p, + None => return false, + }; + + let cmp_path = if sys_path_0.is_empty() { + match std::env::current_dir() { + Ok(d) => d.to_string_lossy().to_string(), + Err(_) => return false, + } + } else { + sys_path_0 + }; + + root.to_str() == Some(cmp_path.as_str()) +} + +/// Check if a module name is in sys.stdlib_module_names. +/// Takes the original __name__ object to preserve str subclass behavior. +/// Propagates errors (e.g. TypeError for unhashable str subclass). +pub(crate) fn is_stdlib_module_name(name: &PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + let stdlib_names = match vm.sys_module.get_attr("stdlib_module_names", vm) { + Ok(names) => names, + Err(_) => return Ok(false), + }; + if !stdlib_names.class().fast_issubclass(vm.ctx.types.set_type) + && !stdlib_names + .class() + .fast_issubclass(vm.ctx.types.frozenset_type) + { + return Ok(false); + } + let result = vm.call_method(&stdlib_names, "__contains__", (name.clone(),))?; + result.try_to_bool(vm) +} + +/// PyImport_ImportModuleLevelObject +pub(crate) fn import_module_level( + name: &Py<PyStr>, + globals: Option<PyObjectRef>, + fromlist: Option<PyObjectRef>, + level: i32, + vm: &VirtualMachine, +) -> PyResult { + if level < 0 { + return Err(vm.new_value_error("level must be >= 0")); + } + + let name_str = match name.to_str() { + Some(s) => s, + None => { + // Name contains surrogates. Like CPython, try sys.modules + // lookup with the Python string key directly. + if level == 0 { + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + return sys_modules.get_item(name, vm).map_err(|_| { + vm.new_import_error(format!("No module named '{}'", name), name.to_owned()) + }); + } + return Err(vm.new_import_error(format!("No module named '{}'", name), name.to_owned())); + } + }; + + // Resolve absolute name + let abs_name = if level > 0 { + // When globals is not provided (Rust None), raise KeyError + // matching resolve_name() where globals==NULL + if globals.is_none() { + return Err(vm.new_key_error(vm.ctx.new_str("'__name__' not in globals").into())); + } + let globals_ref = globals.as_ref().unwrap(); + // When globals is Python None, treat like empty mapping + let empty_dict_obj; + let globals_ref = if vm.is_none(globals_ref) { + empty_dict_obj = vm.ctx.new_dict().into(); + &empty_dict_obj + } else { + globals_ref + }; + let package = calc_package(Some(globals_ref), vm)?; + if package.is_empty() { + return Err(vm.new_import_error( + "attempted relative import with no known parent package", + vm.ctx.new_utf8_str(""), + )); + } + resolve_name(name_str, &package, level as usize, vm)? + } else { + if name_str.is_empty() { + return Err(vm.new_value_error("Empty module name")); + } + name_str.to_owned() + }; + + // import_get_module + import_find_and_load + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + let module = match sys_modules.get_item(&*abs_name, vm) { + Ok(m) if !vm.is_none(&m) => { + import_ensure_initialized(&m, &abs_name, vm)?; + m + } + _ => { + let find_and_load = vm.importlib.get_attr("_find_and_load", vm)?; + let abs_name_obj = vm.ctx.new_utf8_str(&*abs_name); + find_and_load.call((abs_name_obj, vm.import_func.clone()), vm)? + } + }; + + // Handle fromlist + let has_from = match fromlist.as_ref().filter(|fl| !vm.is_none(fl)) { + Some(fl) => fl.clone().try_to_bool(vm)?, + None => false, + }; + + if has_from { + let fromlist = fromlist.unwrap(); + // Only call _handle_fromlist if the module looks like a package + // (has __path__). Non-module objects without __name__/__path__ would + // crash inside _handle_fromlist; IMPORT_FROM handles per-attribute + // errors with proper ImportError conversion. + let has_path = vm + .get_attribute_opt(module.clone(), vm.ctx.intern_str("__path__"))? + .is_some(); + if has_path { + let handle_fromlist = vm.importlib.get_attr("_handle_fromlist", vm)?; + handle_fromlist.call((module, fromlist, vm.import_func.clone()), vm) + } else { + Ok(module) + } + } else if level == 0 || !name_str.is_empty() { + match name_str.find('.') { + None => Ok(module), + Some(dot) => { + let to_return = if level == 0 { + name_str[..dot].to_owned() + } else { + let cut_off = name_str.len() - dot; + abs_name[..abs_name.len() - cut_off].to_owned() + }; + match sys_modules.get_item(&*to_return, vm) { + Ok(m) => Ok(m), + Err(_) if level == 0 => { + // For absolute imports (level 0), try importing the + // parent. Matches _bootstrap.__import__ behavior. + let find_and_load = vm.importlib.get_attr("_find_and_load", vm)?; + let to_return_obj = vm.ctx.new_utf8_str(&*to_return); + find_and_load.call((to_return_obj, vm.import_func.clone()), vm) + } + Err(_) => { + // For relative imports (level > 0), raise KeyError + let to_return_obj: PyObjectRef = vm + .ctx + .new_utf8_str(format!("'{to_return}' not in sys.modules as expected")) + .into(); + Err(vm.new_key_error(to_return_obj)) + } + } + } + } + } else { + Ok(module) + } +} + +/// resolve_name in import.c - resolve relative import name +fn resolve_name(name: &str, package: &str, level: usize, vm: &VirtualMachine) -> PyResult<String> { + // Python: bits = package.rsplit('.', level - 1) + // Rust: rsplitn(level, '.') gives maxsplit=level-1 + let parts: Vec<&str> = package.rsplitn(level, '.').collect(); + if parts.len() < level { + return Err(vm.new_import_error( + "attempted relative import beyond top-level package", + vm.ctx.new_utf8_str(name), + )); + } + // rsplitn returns parts right-to-left, so last() is the leftmost (base) + let base = parts.last().unwrap(); + if name.is_empty() { + Ok(base.to_string()) + } else { + Ok(format!("{base}.{name}")) + } +} + +/// _calc___package__ - calculate package from globals for relative imports +fn calc_package(globals: Option<&PyObjectRef>, vm: &VirtualMachine) -> PyResult<String> { + let globals = globals.ok_or_else(|| { + vm.new_import_error( + "attempted relative import with no known parent package", + vm.ctx.new_utf8_str(""), + ) + })?; + + let package = globals.get_item("__package__", vm).ok(); + let spec = globals.get_item("__spec__", vm).ok(); + + if let Some(ref pkg) = package + && !vm.is_none(pkg) + { + let pkg_str: PyUtf8StrRef = pkg + .clone() + .downcast() + .map_err(|_| vm.new_type_error("package must be a string"))?; + // Warn if __package__ != __spec__.parent + if let Some(ref spec) = spec + && !vm.is_none(spec) + && let Ok(parent) = spec.get_attr("parent", vm) + && !pkg_str.is(&parent) + && pkg_str + .as_object() + .rich_compare_bool(&parent, crate::types::PyComparisonOp::Ne, vm) + .unwrap_or(false) + { + let parent_repr = parent + .repr_utf8(vm) + .map(|s| s.as_str().to_owned()) + .unwrap_or_default(); + let msg = format!( + "__package__ != __spec__.parent ('{}' != {})", + pkg_str.as_str(), + parent_repr + ); + let warn = vm + .import("_warnings", 0) + .and_then(|w| w.get_attr("warn", vm)); + if let Ok(warn_fn) = warn { + let _ = warn_fn.call( + ( + vm.ctx.new_str(msg), + vm.ctx.exceptions.deprecation_warning.to_owned(), + ), + vm, + ); + } + } + return Ok(pkg_str.as_str().to_owned()); + } else if let Some(ref spec) = spec + && !vm.is_none(spec) + && let Ok(parent) = spec.get_attr("parent", vm) + && !vm.is_none(&parent) + { + let parent_str: PyUtf8StrRef = parent + .downcast() + .map_err(|_| vm.new_type_error("package set to non-string"))?; + return Ok(parent_str.as_str().to_owned()); + } + + // Fall back to __name__ and __path__ + let warn = vm + .import("_warnings", 0) + .and_then(|w| w.get_attr("warn", vm)); + if let Ok(warn_fn) = warn { + let _ = warn_fn.call( + ( + vm.ctx.new_str("can't resolve package from __spec__ or __package__, falling back on __name__ and __path__"), + vm.ctx.exceptions.import_warning.to_owned(), + ), + vm, + ); + } + + let mod_name = globals.get_item("__name__", vm).map_err(|_| { + vm.new_import_error( + "attempted relative import with no known parent package", + vm.ctx.new_utf8_str(""), + ) + })?; + let mod_name_str: PyUtf8StrRef = mod_name + .downcast() + .map_err(|_| vm.new_type_error("__name__ must be a string"))?; + let mut package = mod_name_str.as_str().to_owned(); + // If not a package (no __path__), strip last component. + // Uses rpartition('.')[0] semantics: returns empty string when no dot. + if globals.get_item("__path__", vm).is_err() { + package = match package.rfind('.') { + Some(dot) => package[..dot].to_owned(), + None => String::new(), + }; + } + Ok(package) +} diff --git a/crates/vm/src/intern.rs b/crates/vm/src/intern.rs new file mode 100644 index 00000000000..37b971b8dca --- /dev/null +++ b/crates/vm/src/intern.rs @@ -0,0 +1,353 @@ +use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; + +use crate::{ + AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, VirtualMachine, + builtins::{PyStr, PyStrInterned, PyTypeRef}, + common::lock::PyRwLock, + convert::ToPyObject, +}; +use alloc::borrow::ToOwned; +use core::{borrow::Borrow, ops::Deref}; + +#[derive(Debug)] +pub struct StringPool { + inner: PyRwLock<std::collections::HashSet<CachedPyStrRef, ahash::RandomState>>, +} + +impl Default for StringPool { + fn default() -> Self { + Self { + inner: PyRwLock::new(Default::default()), + } + } +} + +impl Clone for StringPool { + fn clone(&self) -> Self { + Self { + inner: PyRwLock::new(self.inner.read().clone()), + } + } +} + +impl StringPool { + /// Reset the inner RwLock to unlocked state after fork(). + /// + /// # Safety + /// Must only be called after fork() in the child process when no other + /// threads exist. + #[cfg(all(unix, feature = "threading"))] + pub(crate) unsafe fn reinit_after_fork(&self) { + unsafe { crate::common::lock::reinit_rwlock_after_fork(&self.inner) }; + } + + #[inline] + pub unsafe fn intern<S: InternableString>( + &self, + s: S, + typ: PyTypeRef, + ) -> &'static PyStrInterned { + if let Some(found) = self.interned(s.as_ref()) { + return found; + } + + #[cold] + fn miss(zelf: &StringPool, s: PyRefExact<PyStr>) -> &'static PyStrInterned { + let cache = CachedPyStrRef { inner: s }; + let inserted = zelf.inner.write().insert(cache.clone()); + if inserted { + let interned = unsafe { cache.as_interned_str() }; + unsafe { interned.as_object().mark_intern() }; + interned + } else { + unsafe { + zelf.inner + .read() + .get(cache.as_ref()) + .expect("inserted is false") + .as_interned_str() + } + } + } + let str_ref = s.into_pyref_exact(typ); + miss(self, str_ref) + } + + #[inline] + pub fn interned<S: MaybeInternedString + ?Sized>( + &self, + s: &S, + ) -> Option<&'static PyStrInterned> { + if let Some(interned) = s.as_interned() { + return Some(interned); + } + self.inner + .read() + .get(s.as_ref()) + .map(|cached| unsafe { cached.as_interned_str() }) + } +} + +#[derive(Debug, Clone)] +#[repr(transparent)] +pub struct CachedPyStrRef { + inner: PyRefExact<PyStr>, +} + +impl core::hash::Hash for CachedPyStrRef { + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { + self.inner.as_wtf8().hash(state) + } +} + +impl PartialEq for CachedPyStrRef { + fn eq(&self, other: &Self) -> bool { + self.inner.as_wtf8() == other.inner.as_wtf8() + } +} + +impl Eq for CachedPyStrRef {} + +impl core::borrow::Borrow<Wtf8> for CachedPyStrRef { + #[inline] + fn borrow(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl AsRef<Wtf8> for CachedPyStrRef { + #[inline] + fn as_ref(&self) -> &Wtf8 { + self.as_wtf8() + } +} + +impl CachedPyStrRef { + /// # Safety + /// the given cache must be alive while returned reference is alive + #[inline] + const unsafe fn as_interned_str(&self) -> &'static PyStrInterned { + unsafe { core::mem::transmute_copy(self) } + } + + #[inline] + fn as_wtf8(&self) -> &Wtf8 { + self.inner.as_wtf8() + } +} + +#[repr(transparent)] +pub struct PyInterned<T> { + inner: Py<T>, +} + +impl PyInterned<PyStr> { + /// Returns `&str` for interned strings. + /// + /// # Panics + /// Panics if the interned string contains unpaired surrogates (WTF-8 content). + /// Most interned strings are valid UTF-8, so this is an ergonomic default. + #[inline] + pub fn as_str(&self) -> &str { + self.inner + .to_str() + .unwrap_or_else(|| panic!("interned str is always valid UTF-8")) + } +} + +impl<T: PyPayload> PyInterned<T> { + #[inline] + pub fn leak(cache: PyRef<T>) -> &'static Self { + unsafe { core::mem::transmute(cache) } + } + + #[inline] + const fn as_ptr(&self) -> *const Py<T> { + self as *const _ as *const _ + } + + #[inline] + pub fn to_owned(&'static self) -> PyRef<T> { + unsafe { (*(&self as *const _ as *const PyRef<T>)).clone() } + } + + #[inline] + pub fn to_object(&'static self) -> PyObjectRef { + self.to_owned().into() + } +} + +impl<T: PyPayload> Borrow<PyObject> for PyInterned<T> { + #[inline(always)] + fn borrow(&self) -> &PyObject { + self.inner.borrow() + } +} + +// NOTE: std::hash::Hash of Self and Self::Borrowed *must* be the same +// This is ok only because PyObject doesn't implement Hash +impl<T: PyPayload> core::hash::Hash for PyInterned<T> { + #[inline(always)] + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { + self.get_id().hash(state) + } +} + +impl<T> AsRef<Py<T>> for PyInterned<T> { + #[inline(always)] + fn as_ref(&self) -> &Py<T> { + &self.inner + } +} + +impl<T> Deref for PyInterned<T> { + type Target = Py<T>; + #[inline(always)] + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl<T: PyPayload> PartialEq for PyInterned<T> { + #[inline(always)] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self, other) + } +} + +impl<T: PyPayload> Eq for PyInterned<T> {} + +impl<T: core::fmt::Debug + PyPayload> core::fmt::Debug for PyInterned<T> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + core::fmt::Debug::fmt(&**self, f)?; + write!(f, "@{:p}", self.as_ptr()) + } +} + +impl<T: PyPayload> ToPyObject for &'static PyInterned<T> { + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.to_owned().into() + } +} + +mod sealed { + use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; + + use crate::{ + builtins::PyStr, + object::{Py, PyExact, PyRefExact}, + }; + + pub trait SealedInternable {} + + impl SealedInternable for String {} + impl SealedInternable for &str {} + impl SealedInternable for Wtf8Buf {} + impl SealedInternable for &Wtf8 {} + impl SealedInternable for PyRefExact<PyStr> {} + + pub trait SealedMaybeInterned {} + + impl SealedMaybeInterned for str {} + impl SealedMaybeInterned for Wtf8 {} + impl SealedMaybeInterned for PyExact<PyStr> {} + impl SealedMaybeInterned for Py<PyStr> {} +} + +/// A sealed marker trait for `DictKey` types that always become an exact instance of `str` +pub trait InternableString: sealed::SealedInternable + ToPyObject + AsRef<Self::Interned> { + type Interned: MaybeInternedString + ?Sized; + fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr>; +} + +impl InternableString for String { + type Interned = str; + #[inline] + fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr> { + let obj = PyRef::new_ref(PyStr::from(self), str_type, None); + unsafe { PyRefExact::new_unchecked(obj) } + } +} + +impl InternableString for &str { + type Interned = str; + #[inline] + fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr> { + self.to_owned().into_pyref_exact(str_type) + } +} + +impl InternableString for Wtf8Buf { + type Interned = Wtf8; + fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr> { + let obj = PyRef::new_ref(PyStr::from(self), str_type, None); + unsafe { PyRefExact::new_unchecked(obj) } + } +} + +impl InternableString for &Wtf8 { + type Interned = Wtf8; + fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr> { + self.to_owned().into_pyref_exact(str_type) + } +} + +impl InternableString for PyRefExact<PyStr> { + type Interned = Py<PyStr>; + #[inline] + fn into_pyref_exact(self, _str_type: PyTypeRef) -> PyRefExact<PyStr> { + self + } +} + +pub trait MaybeInternedString: + AsRef<Wtf8> + crate::dict_inner::DictKey + sealed::SealedMaybeInterned +{ + fn as_interned(&self) -> Option<&'static PyStrInterned>; +} + +impl MaybeInternedString for str { + #[inline(always)] + fn as_interned(&self) -> Option<&'static PyStrInterned> { + None + } +} + +impl MaybeInternedString for Wtf8 { + #[inline(always)] + fn as_interned(&self) -> Option<&'static PyStrInterned> { + None + } +} + +impl MaybeInternedString for PyExact<PyStr> { + #[inline(always)] + fn as_interned(&self) -> Option<&'static PyStrInterned> { + None + } +} + +impl MaybeInternedString for Py<PyStr> { + #[inline(always)] + fn as_interned(&self) -> Option<&'static PyStrInterned> { + if self.as_object().is_interned() { + Some(unsafe { core::mem::transmute::<&Self, &PyInterned<PyStr>>(self) }) + } else { + None + } + } +} + +impl PyObject { + #[inline] + pub fn as_interned_str(&self, vm: &crate::VirtualMachine) -> Option<&'static PyStrInterned> { + let s: Option<&Py<PyStr>> = self.downcast_ref(); + if self.is_interned() { + s.unwrap().as_interned() + } else if let Some(s) = s { + vm.ctx.interned_str(s.as_wtf8()) + } else { + None + } + } +} diff --git a/crates/vm/src/iter.rs b/crates/vm/src/iter.rs new file mode 100644 index 00000000000..1e132437929 --- /dev/null +++ b/crates/vm/src/iter.rs @@ -0,0 +1,53 @@ +use crate::{PyObjectRef, PyResult, types::PyComparisonOp, vm::VirtualMachine}; +use itertools::Itertools; + +pub trait PyExactSizeIterator<'a>: ExactSizeIterator<Item = &'a PyObjectRef> + Sized { + fn eq(self, other: impl PyExactSizeIterator<'a>, vm: &VirtualMachine) -> PyResult<bool> { + let lhs = self; + let rhs = other; + if lhs.len() != rhs.len() { + return Ok(false); + } + for (a, b) in lhs.zip_eq(rhs) { + if !vm.identical_or_equal(a, b)? { + return Ok(false); + } + } + Ok(true) + } + + fn richcompare( + self, + other: impl PyExactSizeIterator<'a>, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<bool> { + let less = match op { + PyComparisonOp::Eq => return PyExactSizeIterator::eq(self, other, vm), + PyComparisonOp::Ne => return PyExactSizeIterator::eq(self, other, vm).map(|eq| !eq), + PyComparisonOp::Lt | PyComparisonOp::Le => true, + PyComparisonOp::Gt | PyComparisonOp::Ge => false, + }; + + let lhs = self; + let rhs = other; + let lhs_len = lhs.len(); + let rhs_len = rhs.len(); + for (a, b) in lhs.zip(rhs) { + if vm.bool_eq(a, b)? { + continue; + } + let ret = if less { + vm.bool_seq_lt(a, b)? + } else { + vm.bool_seq_gt(a, b)? + }; + if let Some(v) = ret { + return Ok(v); + } + } + Ok(op.eval_ord(lhs_len.cmp(&rhs_len))) + } +} + +impl<'a, T> PyExactSizeIterator<'a> for T where T: ExactSizeIterator<Item = &'a PyObjectRef> + Sized {} diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs new file mode 100644 index 00000000000..21762466f13 --- /dev/null +++ b/crates/vm/src/lib.rs @@ -0,0 +1,113 @@ +//! This crate contains most of the python logic. +//! +//! - Interpreter +//! - Import mechanics +//! - Base objects +//! +//! Some stdlib modules are implemented here, but most of them are in the `rustpython-stdlib` module. The + +// to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of +// how `mod` works, but we want this sometimes for pymodule declarations +#![allow(clippy::module_inception)] +// we want to mirror python naming conventions when defining python structs, so that does mean +// uppercase acronyms, e.g. TextIOWrapper instead of TextIoWrapper +#![allow(clippy::upper_case_acronyms)] +#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] +#![doc(html_root_url = "https://docs.rs/rustpython-vm/")] + +#[cfg(feature = "flame-it")] +#[macro_use] +extern crate flamer; + +#[macro_use] +extern crate bitflags; +#[macro_use] +extern crate log; +// extern crate env_logger; +extern crate alloc; + +#[macro_use] +extern crate rustpython_derive; + +extern crate self as rustpython_vm; + +pub use rustpython_derive::*; + +//extern crate eval; use eval::eval::*; +// use py_code_object::{Function, NativeType, PyCodeObject}; + +// This is above everything else so that the defined macros are available everywhere +#[macro_use] +pub(crate) mod macros; + +mod anystr; +pub mod buffer; +pub mod builtins; +pub mod byte; +mod bytes_inner; +pub mod cformat; +pub mod class; +mod codecs; +pub mod compiler; +pub mod convert; +mod coroutine; +pub mod datastack; +mod dict_inner; + +#[cfg(feature = "rustpython-compiler")] +pub mod eval; + +mod exception_group; +pub mod exceptions; +pub mod format; +pub mod frame; +pub mod function; +pub mod getpath; +pub mod import; +mod intern; +pub mod iter; +pub mod object; + +#[cfg(feature = "host_env")] +pub mod ospath; + +pub mod prelude; +pub mod protocol; +pub mod py_io; + +#[cfg(feature = "serde")] +pub mod py_serde; + +pub mod gc_state; +pub mod readline; +pub mod recursion; +pub mod scope; +pub mod sequence; +pub mod signal; +pub mod sliceable; +pub mod stdlib; +pub mod suggestion; +pub mod types; +pub mod utils; +pub mod version; +pub mod vm; +pub mod warn; + +#[cfg(windows)] +pub mod windows; + +pub use self::convert::{TryFromBorrowedObject, TryFromObject}; +pub use self::object::{ + AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, + PyResult, PyStackRef, PyWeakRef, +}; +pub use self::vm::{Context, Interpreter, InterpreterBuilder, Settings, VirtualMachine}; + +pub use rustpython_common as common; +pub use rustpython_compiler_core::{bytecode, frozen}; +pub use rustpython_literal as literal; + +#[doc(hidden)] +pub mod __exports { + pub use paste; +} diff --git a/crates/vm/src/macros.rs b/crates/vm/src/macros.rs new file mode 100644 index 00000000000..4fad50ac8f2 --- /dev/null +++ b/crates/vm/src/macros.rs @@ -0,0 +1,260 @@ +#[macro_export] +macro_rules! extend_module { + ( $vm:expr, $module:expr, { $($name:expr => $value:expr),* $(,)? }) => {{ + $( + $vm.__module_set_attr($module, $vm.ctx.intern_str($name), $value).unwrap(); + )* + }}; +} + +#[macro_export] +macro_rules! py_class { + ( $ctx:expr, $class_name:expr, $class_base:expr, { $($name:tt => $value:expr),* $(,)* }) => { + py_class!($ctx, $class_name, $class_base, $crate::types::PyTypeFlags::BASETYPE, { $($name => $value),* }) + }; + ( $ctx:expr, $class_name:expr, $class_base:expr, $flags:expr, { $($name:tt => $value:expr),* $(,)* }) => { + { + #[allow(unused_mut)] + let mut slots = $crate::types::PyTypeSlots::heap_default(); + slots.flags = $flags; + $($crate::py_class!(@extract_slots($ctx, &mut slots, $name, $value));)* + let py_class = $ctx.new_class(None, $class_name, $class_base, slots); + $($crate::py_class!(@extract_attrs($ctx, &py_class, $name, $value));)* + py_class + } + }; + (@extract_slots($ctx:expr, $slots:expr, (slot $slot_name:ident), $value:expr)) => { + $slots.$slot_name.store(Some($value)); + }; + (@extract_slots($ctx:expr, $class:expr, $name:expr, $value:expr)) => {}; + (@extract_attrs($ctx:expr, $slots:expr, (slot $slot_name:ident), $value:expr)) => {}; + (@extract_attrs($ctx:expr, $class:expr, $name:expr, $value:expr)) => { + $class.set_attr($name, $value); + }; +} + +#[macro_export] +macro_rules! extend_class { + ( $ctx:expr, $class:expr, { $($name:expr => $value:expr),* $(,)* }) => { + $( + $class.set_attr($ctx.intern_str($name), $value.into()); + )* + }; +} + +#[macro_export] +macro_rules! py_namespace { + ( $vm:expr, { $($name:expr => $value:expr),* $(,)* }) => { + { + let namespace = $crate::object::PyPayload::into_ref($crate::builtins::PyNamespace {}, &$vm.ctx); + let obj = $crate::object::AsObject::as_object(&namespace); + $( + obj.generic_setattr($vm.ctx.intern_str($name), $crate::function::PySetterValue::Assign($value.into()), $vm).unwrap(); + )* + namespace + } + } +} + +/// Macro to match on the built-in class of a Python object. +/// +/// Like `match`, `match_class!` must be exhaustive, so a default arm without +/// casting is required. +/// +/// # Examples +/// +/// ``` +/// use malachite_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::match_class; +/// use rustpython_vm::builtins::{PyFloat, PyInt}; +/// use rustpython_vm::{PyPayload}; +/// +/// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { +/// let obj = PyInt::from(0).into_pyobject(vm); +/// assert_eq!( +/// "int", +/// match_class!(match obj { +/// PyInt => "int", +/// PyFloat => "float", +/// _ => "neither", +/// }) +/// ); +/// # }); +/// +/// ``` +/// +/// With a binding to the downcasted type: +/// +/// ``` +/// use malachite_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::match_class; +/// use rustpython_vm::builtins::{PyFloat, PyInt}; +/// use rustpython_vm::{ PyPayload}; +/// +/// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { +/// let obj = PyInt::from(0).into_pyobject(vm); +/// +/// let int_value = match_class!(match obj { +/// i @ PyInt => i.as_bigint().clone(), +/// f @ PyFloat => f.to_f64().to_bigint().unwrap(), +/// obj => panic!("non-numeric object {:?}", obj), +/// }); +/// +/// assert!(int_value.is_zero()); +/// # }); +/// ``` +#[macro_export] +macro_rules! match_class { + // The default arm. + (match ($obj:expr) { _ => $default:expr $(,)? }) => { + $default + }; + + // The default arm, binding the original object to the specified identifier. + (match ($obj:expr) { $binding:ident => $default:expr $(,)? }) => {{ + #[allow(clippy::redundant_locals, reason = "macro arm intentionally binds expression once to a local")] + let $binding = $obj; + $default + }}; + (match ($obj:expr) { ref $binding:ident => $default:expr $(,)? }) => {{ + #[allow(clippy::redundant_locals, reason = "macro arm intentionally binds expression once to a local reference")] + let $binding = &$obj; + $default + }}; + + // An arm taken when the object is an instance of the specified built-in + // class and binding the downcasted object to the specified identifier and + // the target expression is a block. + (match ($obj:expr) { $binding:ident @ $class:ty => $expr:block $($rest:tt)* }) => { + $crate::match_class!(match ($obj) { $binding @ $class => ($expr), $($rest)* }) + }; + (match ($obj:expr) { ref $binding:ident @ $class:ty => $expr:block $($rest:tt)* }) => { + $crate::match_class!(match ($obj) { ref $binding @ $class => ($expr), $($rest)* }) + }; + + // An arm taken when the object is an instance of the specified built-in + // class and binding the downcasted object to the specified identifier. + (match ($obj:expr) { $binding:ident @ $class:ty => $expr:expr, $($rest:tt)* }) => { + match $obj.downcast::<$class>() { + Ok($binding) => $expr, + Err(_obj) => $crate::match_class!(match (_obj) { $($rest)* }), + } + }; + (match ($obj:expr) { ref $binding:ident @ $class:ty => $expr:expr, $($rest:tt)* }) => { + match $obj.downcast_ref::<$class>() { + core::option::Option::Some($binding) => $expr, + core::option::Option::None => $crate::match_class!(match ($obj) { $($rest)* }), + } + }; + + // An arm taken when the object is an instance of the specified built-in + // class and the target expression is a block. + (match ($obj:expr) { $class:ty => $expr:block $($rest:tt)* }) => { + $crate::match_class!(match ($obj) { $class => ($expr), $($rest)* }) + }; + + // An arm taken when the object is an instance of the specified built-in + // class. + (match ($obj:expr) { $class:ty => $expr:expr, $($rest:tt)* }) => { + if $obj.downcastable::<$class>() { + $expr + } else { + $crate::match_class!(match ($obj) { $($rest)* }) + } + }; + + // To allow match expressions without parens around the match target + (match $($rest:tt)*) => { + $crate::match_class!(@parse_match () ($($rest)*)) + }; + (@parse_match ($($target:tt)*) ({ $($inner:tt)* })) => { + $crate::match_class!(match ($($target)*) { $($inner)* }) + }; + (@parse_match ($($target:tt)*) ($next:tt $($rest:tt)*)) => { + $crate::match_class!(@parse_match ($($target)* $next) ($($rest)*)) + }; +} + +#[macro_export] +macro_rules! identifier( + ($as_ctx:expr, $name:ident) => { + $as_ctx.as_ref().names.$name + }; +); + +#[macro_export] +macro_rules! identifier_utf8( + ($as_ctx:expr, $name:ident) => {{ + // Safety: All known identifiers are ascii strings. + let interned = $as_ctx.as_ref().names.$name; + unsafe { $crate::builtins::PyUtf8StrInterned::from_str_interned_unchecked(interned) } + }}; +); + +/// Super detailed logging. Might soon overflow your log buffers +/// Default, this logging is discarded, except when a the `vm-tracing-logging` +/// build feature is enabled. +macro_rules! vm_trace { + ($($arg:tt)+) => { + #[cfg(feature = "vm-tracing-logging")] + trace!($($arg)+); + } +} + +macro_rules! flame_guard { + ($name:expr) => { + #[cfg(feature = "flame-it")] + let _guard = ::flame::start_guard($name); + }; +} + +#[macro_export] +macro_rules! class_or_notimplemented { + ($t:ty, $obj:expr) => {{ + let a: &$crate::PyObject = &*$obj; + match $crate::PyObject::downcast_ref::<$t>(&a) { + Some(pyref) => pyref, + None => return Ok($crate::function::PyArithmeticValue::NotImplemented), + } + }}; +} + +#[macro_export] +macro_rules! named_function { + ($ctx:expr, $module:ident, $func:ident) => {{ + #[allow(unused_variables)] // weird lint, something to do with paste probably + let ctx: &$crate::Context = &$ctx; + $crate::__exports::paste::expr! { + ctx.new_method_def( + stringify!($func), + [<$module _ $func>], + ::rustpython_vm::function::PyMethodFlags::empty(), + ) + .to_function() + .with_module(ctx.intern_str(stringify!($module)).into()) + .into_ref(ctx) + } + }}; +} + +// can't use PyThreadingConstraint for stuff like this since it's not an auto trait, and +// therefore we can't add it ad-hoc to a trait object +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + macro_rules! py_dyn_fn { + (dyn Fn($($arg:ty),*$(,)*) -> $ret:ty) => { + dyn Fn($($arg),*) -> $ret + Send + Sync + 'static + }; + } + } else { + macro_rules! py_dyn_fn { + (dyn Fn($($arg:ty),*$(,)*) -> $ret:ty) => { + dyn Fn($($arg),*) -> $ret + 'static + }; + } + } +} diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs new file mode 100644 index 00000000000..a8d7c09da89 --- /dev/null +++ b/crates/vm/src/object/core.rs @@ -0,0 +1,2566 @@ +//! Essential types for object models +//! +//! +-------------------------+--------------+-----------------------+ +//! | Management | Typed | Untyped | +//! +-------------------------+------------------+-------------------+ +//! | Interpreter-independent | [`Py<T>`] | [`PyObject`] | +//! | Reference-counted | [`PyRef<T>`] | [`PyObjectRef`] | +//! | Weak | [`PyWeakRef<T>`] | [`PyRef<PyWeak>`] | +//! +-------------------------+--------------+-----------------------+ +//! +//! [`PyRef<PyWeak>`] may looking like to be called as PyObjectWeak by the rule, +//! but not to do to remember it is a PyRef object. +use super::{ + PyAtomicRef, + ext::{AsObject, PyRefExact, PyResult}, + payload::PyPayload, +}; +use crate::object::traverse_object::PyObjVTable; +use crate::{ + builtins::{PyDictRef, PyType, PyTypeRef}, + common::{ + atomic::{Ordering, PyAtomic, Radium}, + linked_list::{Link, Pointers}, + lock::PyRwLock, + refcount::RefCount, + }, + vm::VirtualMachine, +}; +use crate::{ + class::StaticType, + object::traverse::{MaybeTraverse, Traverse, TraverseFn}, +}; +use itertools::Itertools; + +use alloc::fmt; + +use core::{ + any::TypeId, + borrow::Borrow, + cell::UnsafeCell, + marker::PhantomData, + mem::ManuallyDrop, + num::NonZeroUsize, + ops::Deref, + ptr::{self, NonNull}, +}; + +// so, PyObjectRef is basically equivalent to `PyRc<PyInner<dyn PyObjectPayload>>`, except it's +// only one pointer in width rather than 2. We do that by manually creating a vtable, and putting +// a &'static reference to it inside the `PyRc` rather than adjacent to it, like trait objects do. +// This can lead to faster code since there's just less data to pass around, as well as because of +// some weird stuff with trait objects, alignment, and padding. +// +// So, every type has an alignment, which means that if you create a value of it it's location in +// memory has to be a multiple of it's alignment. e.g., a type with alignment 4 (like i32) could be +// at 0xb7befbc0, 0xb7befbc4, or 0xb7befbc8, but not 0xb7befbc2. If you have a struct and there are +// 2 fields whose sizes/alignments don't perfectly fit in with each other, e.g.: +// +-------------+-------------+---------------------------+ +// | u16 | ? | i32 | +// | 0x00 | 0x01 | 0x02 | 0x03 | 0x04 | 0x05 | 0x06 | 0x07 | +// +-------------+-------------+---------------------------+ +// There has to be padding in the space between the 2 fields. But, if that field is a trait object +// (like `dyn PyObjectPayload`) we don't *know* how much padding there is between the `payload` +// field and the previous field. So, Rust has to consult the vtable to know the exact offset of +// `payload` in `PyInner<dyn PyObjectPayload>`, which has a huge performance impact when *every +// single payload access* requires a vtable lookup. Thankfully, we're able to avoid that because of +// the way we use PyObjectRef, in that whenever we want to access the payload we (almost) always +// access it from a generic function. So, rather than doing +// +// - check vtable for payload offset +// - get offset in PyInner struct +// - call as_any() method of PyObjectPayload +// - call downcast_ref() method of Any +// we can just do +// - check vtable that typeid matches +// - pointer cast directly to *const PyInner<T> +// +// and at that point the compiler can know the offset of `payload` for us because **we've given it a +// concrete type to work with before we ever access the `payload` field** + +/// A type to just represent "we've erased the type of this object, cast it before you use it" +#[derive(Debug)] +pub(super) struct Erased; + +/// Trashcan mechanism to limit recursive deallocation depth (Py_TRASHCAN). +/// Without this, deeply nested structures (e.g. 200k-deep list) cause stack overflow +/// during deallocation because each level adds a stack frame. +mod trashcan { + use core::cell::Cell; + + /// Maximum nesting depth for deallocation before deferring. + /// CPython uses UNWIND_NO_NESTING = 50. + const TRASHCAN_LIMIT: usize = 50; + + type DeallocFn = unsafe fn(*mut super::PyObject); + type DeallocQueue = Vec<(*mut super::PyObject, DeallocFn)>; + + thread_local! { + static DEALLOC_DEPTH: Cell<usize> = const { Cell::new(0) }; + static DEALLOC_QUEUE: Cell<DeallocQueue> = const { Cell::new(Vec::new()) }; + } + + /// Try to begin deallocation. Returns true if we should proceed, + /// false if the object was deferred (depth exceeded). + #[inline] + pub(super) unsafe fn begin( + obj: *mut super::PyObject, + dealloc: unsafe fn(*mut super::PyObject), + ) -> bool { + DEALLOC_DEPTH.with(|d| { + let depth = d.get(); + if depth >= TRASHCAN_LIMIT { + // Depth exceeded: defer this deallocation + DEALLOC_QUEUE.with(|q| { + let mut queue = q.take(); + queue.push((obj, dealloc)); + q.set(queue); + }); + false + } else { + d.set(depth + 1); + true + } + }) + } + + /// End deallocation and process any deferred objects if at outermost level. + #[inline] + pub(super) unsafe fn end() { + let depth = DEALLOC_DEPTH.with(|d| { + let depth = d.get(); + debug_assert!(depth > 0, "trashcan::end called without matching begin"); + let depth = depth - 1; + d.set(depth); + depth + }); + if depth == 0 { + // Process deferred deallocations iteratively + loop { + let next = DEALLOC_QUEUE.with(|q| { + let mut queue = q.take(); + let item = queue.pop(); + q.set(queue); + item + }); + if let Some((obj, dealloc)) = next { + unsafe { dealloc(obj) }; + } else { + break; + } + } + } + } +} + +/// Default dealloc: handles __del__, weakref clearing, tp_clear, and memory free. +/// Equivalent to subtype_dealloc. +pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) { + let obj_ref = unsafe { &*(obj as *const PyObject) }; + if let Err(()) = obj_ref.drop_slow_inner() { + return; // resurrected by __del__ + } + + // Trashcan: limit recursive deallocation depth to prevent stack overflow + if !unsafe { trashcan::begin(obj, default_dealloc::<T>) } { + return; // deferred to queue + } + + let vtable = obj_ref.0.vtable; + + // Untrack from GC BEFORE deallocation. + // Must happen before memory is freed because intrusive list removal + // reads the object's gc_pointers (prev/next). + if obj_ref.is_gc_tracked() { + let ptr = unsafe { NonNull::new_unchecked(obj) }; + unsafe { + crate::gc_state::gc_state().untrack_object(ptr); + } + // Verify untrack cleared the tracked flag and generation + debug_assert!( + !obj_ref.is_gc_tracked(), + "object still tracked after untrack_object" + ); + debug_assert_eq!( + obj_ref.gc_generation(), + crate::object::GC_UNTRACKED, + "gc_generation not reset after untrack_object" + ); + } + + // Try to store in freelist for reuse BEFORE tp_clear, so that + // size-based freelists (e.g. PyTuple) can read the payload directly. + // Only exact base types (not heaptype or structseq subtypes) go into the freelist. + let typ = obj_ref.class(); + let pushed = if T::HAS_FREELIST + && typ.heaptype_ext.is_none() + && core::ptr::eq(typ, T::class(crate::vm::Context::genesis())) + { + unsafe { T::freelist_push(obj) } + } else { + false + }; + + // Extract child references to break circular refs (tp_clear). + // This runs regardless of freelist push — the object's children must be released. + let mut edges = Vec::new(); + if let Some(clear_fn) = vtable.clear { + unsafe { clear_fn(obj, &mut edges) }; + } + + if !pushed { + // Deallocate the object memory (handles ObjExt prefix if present) + unsafe { PyInner::dealloc(obj as *mut PyInner<T>) }; + } + + // Drop child references - may trigger recursive destruction. + drop(edges); + + // Trashcan: decrement depth and process deferred objects at outermost level + unsafe { trashcan::end() }; +} +pub(super) unsafe fn debug_obj<T: PyPayload + core::fmt::Debug>( + x: &PyObject, + f: &mut fmt::Formatter<'_>, +) -> fmt::Result { + let x = unsafe { &*(x as *const PyObject as *const PyInner<T>) }; + fmt::Debug::fmt(x, f) +} + +/// Call `try_trace` on payload +pub(super) unsafe fn try_traverse_obj<T: PyPayload>(x: &PyObject, tracer_fn: &mut TraverseFn<'_>) { + let x = unsafe { &*(x as *const PyObject as *const PyInner<T>) }; + let payload = &x.payload; + payload.try_traverse(tracer_fn) +} + +/// Call `try_clear` on payload to extract child references (tp_clear) +pub(super) unsafe fn try_clear_obj<T: PyPayload>(x: *mut PyObject, out: &mut Vec<PyObjectRef>) { + let x = unsafe { &mut *(x as *mut PyInner<T>) }; + x.payload.try_clear(out); +} + +bitflags::bitflags! { + /// GC bits for free-threading support (like ob_gc_bits in Py_GIL_DISABLED) + /// These bits are stored in a separate atomic field for lock-free access. + /// See Include/internal/pycore_gc.h + #[derive(Copy, Clone, Debug, Default)] + pub(crate) struct GcBits: u8 { + /// Tracked by the GC + const TRACKED = 1 << 0; + /// tp_finalize was called (prevents __del__ from being called twice) + const FINALIZED = 1 << 1; + /// Object is unreachable (during GC collection) + const UNREACHABLE = 1 << 2; + /// Object is frozen (immutable) + const FROZEN = 1 << 3; + /// Memory the object references is shared between multiple threads + /// and needs special handling when freeing due to possible in-flight lock-free reads + const SHARED = 1 << 4; + /// Memory of the object itself is shared between multiple threads + /// Objects with this bit that are GC objects will automatically be delay-freed + const SHARED_INLINE = 1 << 5; + /// Use deferred reference counting + const DEFERRED = 1 << 6; + } +} + +/// GC generation constants +pub(crate) const GC_UNTRACKED: u8 = 0xFF; +pub(crate) const GC_PERMANENT: u8 = 3; + +/// Link implementation for GC intrusive linked list tracking +pub(crate) struct GcLink; + +// SAFETY: PyObject (PyInner<Erased>) is heap-allocated and pinned in memory +// once created. gc_pointers is at a fixed offset in PyInner. +unsafe impl Link for GcLink { + type Handle = NonNull<PyObject>; + type Target = PyObject; + + fn as_raw(handle: &NonNull<PyObject>) -> NonNull<PyObject> { + *handle + } + + unsafe fn from_raw(ptr: NonNull<PyObject>) -> NonNull<PyObject> { + ptr + } + + unsafe fn pointers(target: NonNull<PyObject>) -> NonNull<Pointers<PyObject>> { + let inner_ptr = target.as_ptr() as *mut PyInner<Erased>; + unsafe { NonNull::new_unchecked(&raw mut (*inner_ptr).gc_pointers) } + } +} + +/// Extension fields for objects that need dict or member slots. +/// Allocated as a prefix before PyInner when needed (prefix allocation pattern). +/// Access via `PyInner::ext_ref()` using negative offset from the object pointer. +/// +/// align(8) ensures size_of::<ObjExt>() is always a multiple of 8, +/// so the offset from Layout::extend equals size_of::<ObjExt>() for any +/// PyInner<T> alignment (important on wasm32 where pointers are 4 bytes +/// but some payloads like PyWeak have align 8 due to i64 fields). +#[repr(C, align(8))] +pub(super) struct ObjExt { + pub(super) dict: Option<InstanceDict>, + pub(super) slots: Box<[PyRwLock<Option<PyObjectRef>>]>, +} + +impl ObjExt { + fn new(dict: Option<PyDictRef>, member_count: usize) -> Self { + Self { + dict: dict.map(InstanceDict::new), + slots: core::iter::repeat_with(|| PyRwLock::new(None)) + .take(member_count) + .collect_vec() + .into_boxed_slice(), + } + } +} + +impl fmt::Debug for ObjExt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[ObjExt]") + } +} + +/// Precomputed offset constants for prefix allocation. +/// All prefix components are align(8) and their sizes are multiples of 8, +/// so Layout::extend adds no inter-padding. +const EXT_OFFSET: usize = core::mem::size_of::<ObjExt>(); +const WEAKREF_OFFSET: usize = core::mem::size_of::<WeakRefList>(); + +const _: () = + assert!(core::mem::size_of::<ObjExt>().is_multiple_of(core::mem::align_of::<ObjExt>())); +const _: () = assert!(core::mem::align_of::<ObjExt>() >= core::mem::align_of::<PyInner<()>>()); +const _: () = assert!( + core::mem::size_of::<WeakRefList>().is_multiple_of(core::mem::align_of::<WeakRefList>()) +); +const _: () = assert!(core::mem::align_of::<WeakRefList>() >= core::mem::align_of::<PyInner<()>>()); + +/// This is an actual python object. It consists of a `typ` which is the +/// python class, and carries some rust payload optionally. This rust +/// payload can be a rust float or rust int in case of float and int objects. +#[repr(C)] +pub(super) struct PyInner<T> { + pub(super) ref_count: RefCount, + pub(super) vtable: &'static PyObjVTable, + /// GC bits for free-threading (like ob_gc_bits) + pub(super) gc_bits: PyAtomic<u8>, + /// GC generation index (0-2=gen, GC_PERMANENT=permanent, GC_UNTRACKED=not tracked). + /// Uses PyAtomic for interior mutability (writes happen through &self under list locks). + pub(super) gc_generation: PyAtomic<u8>, + /// Intrusive linked list pointers for GC generational tracking + pub(super) gc_pointers: Pointers<PyObject>, + + pub(super) typ: PyAtomicRef<PyType>, // __class__ member + + pub(super) payload: T, +} +pub(crate) const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::<PyInner<()>>(); + +impl<T> PyInner<T> { + /// Read type flags and member_count via raw pointers to avoid Stacked Borrows + /// violations during bootstrap, where type objects have self-referential typ pointers. + #[inline(always)] + fn read_type_flags(&self) -> (crate::types::PyTypeFlags, usize) { + let typ_ptr = self.typ.load_raw(); + let slots = unsafe { core::ptr::addr_of!((*typ_ptr).0.payload.slots) }; + let flags = unsafe { core::ptr::addr_of!((*slots).flags).read() }; + let member_count = unsafe { core::ptr::addr_of!((*slots).member_count).read() }; + (flags, member_count) + } + + /// Access the ObjExt prefix at a negative offset from this PyInner. + /// Returns None if this object was allocated without dict/slots. + /// + /// Layout: [ObjExt?][WeakRefList?][PyInner] + /// ObjExt offset depends on whether WeakRefList is also present. + #[inline(always)] + pub(super) fn ext_ref(&self) -> Option<&ObjExt> { + let (flags, member_count) = self.read_type_flags(); + let has_ext = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0; + if !has_ext { + return None; + } + let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF); + let offset = if has_weakref { + WEAKREF_OFFSET + EXT_OFFSET + } else { + EXT_OFFSET + }; + let self_addr = (self as *const Self as *const u8).addr(); + let ext_ptr = core::ptr::with_exposed_provenance::<ObjExt>(self_addr.wrapping_sub(offset)); + Some(unsafe { &*ext_ptr }) + } + + /// Access the WeakRefList prefix at a fixed negative offset from this PyInner. + /// Returns None if the type does not support weakrefs. + /// + /// Layout: [ObjExt?][WeakRefList?][PyInner] + /// WeakRefList is always immediately before PyInner (fixed WEAKREF_OFFSET). + #[inline(always)] + pub(super) fn weakref_list_ref(&self) -> Option<&WeakRefList> { + let (flags, _) = self.read_type_flags(); + if !flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF) { + return None; + } + let self_addr = (self as *const Self as *const u8).addr(); + let ptr = core::ptr::with_exposed_provenance::<WeakRefList>( + self_addr.wrapping_sub(WEAKREF_OFFSET), + ); + Some(unsafe { &*ptr }) + } +} + +impl<T: fmt::Debug> fmt::Debug for PyInner<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[PyObject {:?}]", &self.payload) + } +} + +unsafe impl<T: MaybeTraverse> Traverse for Py<T> { + /// DO notice that call `trace` on `Py<T>` means apply `tracer_fn` on `Py<T>`'s children, + /// not like call `trace` on `PyRef<T>` which apply `tracer_fn` on `PyRef<T>` itself + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.traverse(tracer_fn) + } +} + +unsafe impl Traverse for PyObject { + /// DO notice that call `trace` on `PyObject` means apply `tracer_fn` on `PyObject`'s children, + /// not like call `trace` on `PyObjectRef` which apply `tracer_fn` on `PyObjectRef` itself + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.traverse(tracer_fn) + } +} + +// === Stripe lock for weakref list protection (WEAKREF_LIST_LOCK) === + +#[cfg(feature = "threading")] +mod weakref_lock { + use core::sync::atomic::{AtomicU8, Ordering}; + + const NUM_WEAKREF_LOCKS: usize = 64; + + static LOCKS: [AtomicU8; NUM_WEAKREF_LOCKS] = [const { AtomicU8::new(0) }; NUM_WEAKREF_LOCKS]; + + pub(super) struct WeakrefLockGuard { + idx: usize, + } + + impl Drop for WeakrefLockGuard { + fn drop(&mut self) { + LOCKS[self.idx].store(0, Ordering::Release); + } + } + + pub(super) fn lock(addr: usize) -> WeakrefLockGuard { + let idx = (addr >> 4) % NUM_WEAKREF_LOCKS; + loop { + if LOCKS[idx] + .compare_exchange_weak(0, 1, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return WeakrefLockGuard { idx }; + } + core::hint::spin_loop(); + } + } + + /// Reset all weakref stripe locks after fork in child process. + /// Locks held by parent threads would cause infinite spin in the child. + #[cfg(unix)] + pub(crate) fn reset_all_after_fork() { + for lock in &LOCKS { + lock.store(0, Ordering::Release); + } + } +} + +#[cfg(not(feature = "threading"))] +mod weakref_lock { + pub(super) struct WeakrefLockGuard; + + impl Drop for WeakrefLockGuard { + fn drop(&mut self) {} + } + + pub(super) fn lock(_addr: usize) -> WeakrefLockGuard { + WeakrefLockGuard + } +} + +/// Reset weakref stripe locks after fork. Must be called before any +/// Python code runs in the child process. +#[cfg(all(unix, feature = "threading"))] +pub(crate) fn reset_weakref_locks_after_fork() { + weakref_lock::reset_all_after_fork(); +} + +// === WeakRefList: inline on every object (tp_weaklist) === + +#[repr(C)] +pub(super) struct WeakRefList { + /// Head of the intrusive doubly-linked list of weakrefs. + head: PyAtomic<*mut Py<PyWeak>>, + /// Cached generic weakref (no callback, exact weakref type). + /// Matches try_reuse_basic_ref in weakrefobject.c. + generic: PyAtomic<*mut Py<PyWeak>>, +} + +impl fmt::Debug for WeakRefList { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("WeakRefList").finish_non_exhaustive() + } +} + +/// Unlink a node from the weakref list. Must be called under stripe lock. +/// +/// # Safety +/// `node` must be a valid pointer to a node currently in the list owned by `wrl`. +unsafe fn unlink_weakref(wrl: &WeakRefList, node: NonNull<Py<PyWeak>>) { + unsafe { + let mut ptrs = WeakLink::pointers(node); + let prev = ptrs.as_ref().get_prev(); + let next = ptrs.as_ref().get_next(); + + if let Some(prev) = prev { + WeakLink::pointers(prev).as_mut().set_next(next); + } else { + // node is the head + wrl.head.store( + next.map_or(ptr::null_mut(), |p| p.as_ptr()), + Ordering::Relaxed, + ); + } + if let Some(next) = next { + WeakLink::pointers(next).as_mut().set_prev(prev); + } + + ptrs.as_mut().set_prev(None); + ptrs.as_mut().set_next(None); + } +} + +impl WeakRefList { + pub fn new() -> Self { + Self { + head: Radium::new(ptr::null_mut()), + generic: Radium::new(ptr::null_mut()), + } + } + + /// get_or_create_weakref + fn add( + &self, + obj: &PyObject, + cls: PyTypeRef, + cls_is_weakref: bool, + callback: Option<PyObjectRef>, + dict: Option<PyDictRef>, + ) -> PyRef<PyWeak> { + let is_generic = cls_is_weakref && callback.is_none(); + + // Try reuse under lock first (fast path, no allocation) + { + let _lock = weakref_lock::lock(obj as *const PyObject as usize); + if is_generic { + let generic_ptr = self.generic.load(Ordering::Relaxed); + if !generic_ptr.is_null() { + let generic = unsafe { &*generic_ptr }; + if generic.0.ref_count.safe_inc() { + return unsafe { PyRef::from_raw(generic_ptr) }; + } + } + } + } + + // Allocate OUTSIDE the stripe lock. PyRef::new_ref may trigger + // maybe_collect → GC → WeakRefList::clear on another object that + // hashes to the same stripe, which would deadlock on the spinlock. + let weak_payload = PyWeak { + pointers: Pointers::new(), + wr_object: Radium::new(obj as *const PyObject as *mut PyObject), + callback: UnsafeCell::new(callback), + hash: Radium::new(crate::common::hash::SENTINEL), + }; + let weak = PyRef::new_ref(weak_payload, cls, dict); + + // Re-acquire lock for linked list insertion + let _lock = weakref_lock::lock(obj as *const PyObject as usize); + + // Re-check: another thread may have inserted a generic ref while we + // were allocating outside the lock. If so, reuse it and drop ours. + if is_generic { + let generic_ptr = self.generic.load(Ordering::Relaxed); + if !generic_ptr.is_null() { + let generic = unsafe { &*generic_ptr }; + if generic.0.ref_count.safe_inc() { + // Nullify wr_object so drop_inner won't unlink an + // un-inserted node (which would corrupt the list head). + weak.wr_object.store(ptr::null_mut(), Ordering::Relaxed); + return unsafe { PyRef::from_raw(generic_ptr) }; + } + } + } + + // Insert into linked list under stripe lock + let node_ptr = NonNull::from(&*weak); + unsafe { + let mut ptrs = WeakLink::pointers(node_ptr); + if is_generic { + // Generic ref goes to head (insert_head for basic ref) + let old_head = self.head.load(Ordering::Relaxed); + ptrs.as_mut().set_next(NonNull::new(old_head)); + ptrs.as_mut().set_prev(None); + if let Some(old_head) = NonNull::new(old_head) { + WeakLink::pointers(old_head) + .as_mut() + .set_prev(Some(node_ptr)); + } + self.head.store(node_ptr.as_ptr(), Ordering::Relaxed); + self.generic.store(node_ptr.as_ptr(), Ordering::Relaxed); + } else { + // Non-generic refs go after generic ref (insert_after) + let generic_ptr = self.generic.load(Ordering::Relaxed); + if let Some(after) = NonNull::new(generic_ptr) { + let after_next = WeakLink::pointers(after).as_ref().get_next(); + ptrs.as_mut().set_prev(Some(after)); + ptrs.as_mut().set_next(after_next); + WeakLink::pointers(after).as_mut().set_next(Some(node_ptr)); + if let Some(next) = after_next { + WeakLink::pointers(next).as_mut().set_prev(Some(node_ptr)); + } + } else { + // No generic ref; insert at head + let old_head = self.head.load(Ordering::Relaxed); + ptrs.as_mut().set_next(NonNull::new(old_head)); + ptrs.as_mut().set_prev(None); + if let Some(old_head) = NonNull::new(old_head) { + WeakLink::pointers(old_head) + .as_mut() + .set_prev(Some(node_ptr)); + } + self.head.store(node_ptr.as_ptr(), Ordering::Relaxed); + } + } + } + + weak + } + + /// Clear all weakrefs and call their callbacks. + /// Called when the owner object is being dropped. + // PyObject_ClearWeakRefs + fn clear(&self, obj: &PyObject) { + let obj_addr = obj as *const PyObject as usize; + let _lock = weakref_lock::lock(obj_addr); + + // Clear generic cache + self.generic.store(ptr::null_mut(), Ordering::Relaxed); + + // Walk the list, collecting weakrefs with callbacks + let mut callbacks: Vec<(PyRef<PyWeak>, PyObjectRef)> = Vec::new(); + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + let next = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + + let wr = unsafe { node.as_ref() }; + + // Mark weakref as dead + wr.0.payload + .wr_object + .store(ptr::null_mut(), Ordering::Relaxed); + + // Unlink from list + unsafe { + let mut ptrs = WeakLink::pointers(node); + ptrs.as_mut().set_prev(None); + ptrs.as_mut().set_next(None); + } + + // Collect callback only if we can still acquire a strong ref. + if wr.0.ref_count.safe_inc() { + let wr_ref = unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) }; + let cb = unsafe { wr.0.payload.callback.get().replace(None) }; + if let Some(cb) = cb { + callbacks.push((wr_ref, cb)); + } + } + + current = next; + } + self.head.store(ptr::null_mut(), Ordering::Relaxed); + + // Invoke callbacks outside the lock + drop(_lock); + for (wr, cb) in callbacks { + crate::vm::thread::with_vm(&cb, |vm| { + let _ = cb.call((wr.clone(),), vm); + }); + } + } + + /// Clear all weakrefs but DON'T call callbacks. Instead, return them for later invocation. + /// Used by GC to ensure ALL weakrefs are cleared BEFORE any callbacks are invoked. + /// handle_weakrefs() clears all weakrefs first, then invokes callbacks. + fn clear_for_gc_collect_callbacks(&self, obj: &PyObject) -> Vec<(PyRef<PyWeak>, PyObjectRef)> { + let obj_addr = obj as *const PyObject as usize; + let _lock = weakref_lock::lock(obj_addr); + + // Clear generic cache + self.generic.store(ptr::null_mut(), Ordering::Relaxed); + + let mut callbacks = Vec::new(); + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + let next = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + + let wr = unsafe { node.as_ref() }; + + // Mark weakref as dead + wr.0.payload + .wr_object + .store(ptr::null_mut(), Ordering::Relaxed); + + // Unlink from list + unsafe { + let mut ptrs = WeakLink::pointers(node); + ptrs.as_mut().set_prev(None); + ptrs.as_mut().set_next(None); + } + + // Collect callback without invoking only if we can keep weakref alive. + if wr.0.ref_count.safe_inc() { + let wr_ref = unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) }; + let cb = unsafe { wr.0.payload.callback.get().replace(None) }; + if let Some(cb) = cb { + callbacks.push((wr_ref, cb)); + } + } + + current = next; + } + self.head.store(ptr::null_mut(), Ordering::Relaxed); + + callbacks + } + + fn count(&self, obj: &PyObject) -> usize { + let _lock = weakref_lock::lock(obj as *const PyObject as usize); + let mut count = 0usize; + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + if unsafe { node.as_ref() }.0.ref_count.get() > 0 { + count += 1; + } + current = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + } + count + } + + fn get_weak_references(&self, obj: &PyObject) -> Vec<PyRef<PyWeak>> { + let _lock = weakref_lock::lock(obj as *const PyObject as usize); + let mut v = Vec::new(); + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + let wr = unsafe { node.as_ref() }; + if wr.0.ref_count.safe_inc() { + v.push(unsafe { PyRef::from_raw(wr as *const Py<PyWeak>) }); + } + current = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + } + v + } +} + +impl Default for WeakRefList { + fn default() -> Self { + Self::new() + } +} + +struct WeakLink; +unsafe impl Link for WeakLink { + type Handle = PyRef<PyWeak>; + + type Target = Py<PyWeak>; + + #[inline(always)] + fn as_raw(handle: &PyRef<PyWeak>) -> NonNull<Self::Target> { + NonNull::from(&**handle) + } + + #[inline(always)] + unsafe fn from_raw(ptr: NonNull<Self::Target>) -> Self::Handle { + unsafe { PyRef::from_raw(ptr.as_ptr()) } + } + + #[inline(always)] + unsafe fn pointers(target: NonNull<Self::Target>) -> NonNull<Pointers<Self::Target>> { + // SAFETY: requirements forwarded from caller + unsafe { NonNull::new_unchecked(&raw mut (*target.as_ptr()).0.payload.pointers) } + } +} + +/// PyWeakReference: each weakref holds a direct pointer to its referent. +#[pyclass(name = "weakref", module = false)] +#[derive(Debug)] +pub struct PyWeak { + pointers: Pointers<Py<PyWeak>>, + /// Direct pointer to the referent object, null when dead. + /// Equivalent to wr_object in PyWeakReference. + wr_object: PyAtomic<*mut PyObject>, + /// Protected by stripe lock (keyed on wr_object address). + callback: UnsafeCell<Option<PyObjectRef>>, + pub(crate) hash: PyAtomic<crate::common::hash::PyHash>, +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for PyWeak {} + unsafe impl Sync for PyWeak {} + } +} + +impl PyWeak { + /// _PyWeakref_GET_REF: attempt to upgrade the weakref to a strong reference. + pub(crate) fn upgrade(&self) -> Option<PyObjectRef> { + let obj_ptr = self.wr_object.load(Ordering::Acquire); + if obj_ptr.is_null() { + return None; + } + + let _lock = weakref_lock::lock(obj_ptr as usize); + + // Double-check under lock (clear may have run between our check and lock) + let obj_ptr = self.wr_object.load(Ordering::Relaxed); + if obj_ptr.is_null() { + return None; + } + + unsafe { + if !(*obj_ptr).0.ref_count.safe_inc() { + return None; + } + Some(PyObjectRef::from_raw(NonNull::new_unchecked(obj_ptr))) + } + } + + pub(crate) fn is_dead(&self) -> bool { + self.wr_object.load(Ordering::Acquire).is_null() + } + + /// weakref_dealloc: remove from list if still linked. + fn drop_inner(&self) { + let obj_ptr = self.wr_object.load(Ordering::Acquire); + if obj_ptr.is_null() { + return; // Already cleared by WeakRefList::clear() + } + + let _lock = weakref_lock::lock(obj_ptr as usize); + + // Double-check under lock + let obj_ptr = self.wr_object.load(Ordering::Relaxed); + if obj_ptr.is_null() { + return; // Cleared between our check and lock acquisition + } + + let obj = unsafe { &*obj_ptr }; + // Safety: if a weakref exists pointing to this object, weakref prefix must be present + let wrl = obj.0.weakref_list_ref().unwrap(); + + // Compute our Py<PyWeak> node pointer from payload address + let offset = std::mem::offset_of!(PyInner<Self>, payload); + let py_inner = (self as *const Self) + .cast::<u8>() + .wrapping_sub(offset) + .cast::<PyInner<Self>>(); + let node_ptr = unsafe { NonNull::new_unchecked(py_inner as *mut Py<Self>) }; + + // Unlink from list + unsafe { unlink_weakref(wrl, node_ptr) }; + + // Update generic cache if this was it + if wrl.generic.load(Ordering::Relaxed) == node_ptr.as_ptr() { + wrl.generic.store(ptr::null_mut(), Ordering::Relaxed); + } + + // Mark as dead + self.wr_object.store(ptr::null_mut(), Ordering::Relaxed); + } +} + +impl Drop for PyWeak { + #[inline(always)] + fn drop(&mut self) { + // we do NOT have actual exclusive access! + let me: &Self = self; + me.drop_inner(); + } +} + +impl Py<PyWeak> { + #[inline(always)] + pub fn upgrade(&self) -> Option<PyObjectRef> { + PyWeak::upgrade(self) + } +} + +#[derive(Debug)] +pub(super) struct InstanceDict { + pub(super) d: PyRwLock<PyDictRef>, +} + +impl From<PyDictRef> for InstanceDict { + #[inline(always)] + fn from(d: PyDictRef) -> Self { + Self::new(d) + } +} + +impl InstanceDict { + #[inline] + pub const fn new(d: PyDictRef) -> Self { + Self { + d: PyRwLock::new(d), + } + } + + #[inline] + pub fn get(&self) -> PyDictRef { + self.d.read().clone() + } + + #[inline] + pub fn set(&self, d: PyDictRef) { + self.replace(d); + } + + #[inline] + pub fn replace(&self, d: PyDictRef) -> PyDictRef { + core::mem::replace(&mut self.d.write(), d) + } + + /// Consume the InstanceDict and return the inner PyDictRef. + #[inline] + pub fn into_inner(self) -> PyDictRef { + self.d.into_inner() + } +} + +impl<T: PyPayload> PyInner<T> { + /// Deallocate a PyInner, handling optional prefix(es). + /// Layout: [ObjExt?][WeakRefList?][PyInner<T>] + /// + /// # Safety + /// `ptr` must be a valid pointer from `PyInner::new` and must not be used after this call. + unsafe fn dealloc(ptr: *mut Self) { + unsafe { + let (flags, member_count) = (*ptr).read_type_flags(); + let has_ext = + flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0; + let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF); + + if has_ext || has_weakref { + // Reconstruct the same layout used in new() + let mut layout = core::alloc::Layout::from_size_align(0, 1).unwrap(); + + if has_ext { + layout = layout + .extend(core::alloc::Layout::new::<ObjExt>()) + .unwrap() + .0; + } + if has_weakref { + layout = layout + .extend(core::alloc::Layout::new::<WeakRefList>()) + .unwrap() + .0; + } + let (combined, inner_offset) = + layout.extend(core::alloc::Layout::new::<Self>()).unwrap(); + let combined = combined.pad_to_align(); + + let alloc_ptr = (ptr as *mut u8).sub(inner_offset); + + // Drop PyInner (payload, typ, etc.) + core::ptr::drop_in_place(ptr); + + // Drop ObjExt if present (dict, slots) + if has_ext { + core::ptr::drop_in_place(alloc_ptr as *mut ObjExt); + } + // WeakRefList has no Drop (just raw pointers), no drop_in_place needed + + alloc::alloc::dealloc(alloc_ptr, combined); + } else { + drop(Box::from_raw(ptr)); + } + } + } +} + +impl<T: PyPayload + core::fmt::Debug> PyInner<T> { + /// Allocate a new PyInner, optionally with prefix(es). + /// Returns a raw pointer to the PyInner (NOT the allocation start). + /// Layout: [ObjExt?][WeakRefList?][PyInner<T>] + fn new(payload: T, typ: PyTypeRef, dict: Option<PyDictRef>) -> *mut Self { + let member_count = typ.slots.member_count; + let needs_ext = typ + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + || member_count > 0; + let needs_weakref = typ + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_WEAKREF); + debug_assert!( + needs_ext || dict.is_none(), + "dict passed to type '{}' without HAS_DICT flag", + typ.name() + ); + + if needs_ext || needs_weakref { + // Build layout left-to-right: [ObjExt?][WeakRefList?][PyInner] + let mut layout = core::alloc::Layout::from_size_align(0, 1).unwrap(); + + let ext_start = if needs_ext { + let (combined, offset) = + layout.extend(core::alloc::Layout::new::<ObjExt>()).unwrap(); + layout = combined; + Some(offset) + } else { + None + }; + + let weakref_start = if needs_weakref { + let (combined, offset) = layout + .extend(core::alloc::Layout::new::<WeakRefList>()) + .unwrap(); + layout = combined; + Some(offset) + } else { + None + }; + + let (combined, inner_offset) = + layout.extend(core::alloc::Layout::new::<Self>()).unwrap(); + let combined = combined.pad_to_align(); + + let alloc_ptr = unsafe { alloc::alloc::alloc(combined) }; + if alloc_ptr.is_null() { + alloc::alloc::handle_alloc_error(combined); + } + // Expose provenance so ext_ref()/weakref_list_ref() can reconstruct + alloc_ptr.expose_provenance(); + + unsafe { + if let Some(offset) = ext_start { + let ext_ptr = alloc_ptr.add(offset) as *mut ObjExt; + ext_ptr.write(ObjExt::new(dict, member_count)); + } + + if let Some(offset) = weakref_start { + let weakref_ptr = alloc_ptr.add(offset) as *mut WeakRefList; + weakref_ptr.write(WeakRefList::new()); + } + + let inner_ptr = alloc_ptr.add(inner_offset) as *mut Self; + inner_ptr.write(Self { + ref_count: RefCount::new(), + vtable: PyObjVTable::of::<T>(), + gc_bits: Radium::new(0), + gc_generation: Radium::new(GC_UNTRACKED), + gc_pointers: Pointers::new(), + typ: PyAtomicRef::from(typ), + payload, + }); + inner_ptr + } + } else { + Box::into_raw(Box::new(Self { + ref_count: RefCount::new(), + vtable: PyObjVTable::of::<T>(), + gc_bits: Radium::new(0), + gc_generation: Radium::new(GC_UNTRACKED), + gc_pointers: Pointers::new(), + typ: PyAtomicRef::from(typ), + payload, + })) + } + } +} + +/// Returns the allocation layout for `PyInner<T>`, for use in freelist Drop impls. +pub(crate) const fn pyinner_layout<T: PyPayload>() -> core::alloc::Layout { + core::alloc::Layout::new::<PyInner<T>>() +} + +/// Thread-local freelist storage for reusing object allocations. +/// +/// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw +/// `PyInner<T>` allocations without running payload destructors to avoid +/// accessing already-destroyed thread-local storage (GC state, other freelists). +pub(crate) struct FreeList<T: PyPayload> { + items: Vec<*mut PyObject>, + _marker: core::marker::PhantomData<T>, +} + +impl<T: PyPayload> FreeList<T> { + pub(crate) const fn new() -> Self { + Self { + items: Vec::new(), + _marker: core::marker::PhantomData, + } + } +} + +impl<T: PyPayload> Default for FreeList<T> { + fn default() -> Self { + Self::new() + } +} + +impl<T: PyPayload> Drop for FreeList<T> { + fn drop(&mut self) { + // During thread teardown, we cannot safely run destructors on cached + // objects because their Drop impls may access thread-local storage + // (GC state, other freelists) that is already destroyed. + // Instead, free just the raw allocation. The payload's heap fields + // (BigInt, PyObjectRef, etc.) are leaked, but this is bounded by + // MAX_FREELIST per type per thread. + for ptr in self.items.drain(..) { + unsafe { + alloc::alloc::dealloc(ptr as *mut u8, core::alloc::Layout::new::<PyInner<T>>()); + } + } + } +} + +impl<T: PyPayload> core::ops::Deref for FreeList<T> { + type Target = Vec<*mut PyObject>; + fn deref(&self) -> &Self::Target { + &self.items + } +} + +impl<T: PyPayload> core::ops::DerefMut for FreeList<T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.items + } +} + +/// The `PyObjectRef` is one of the most used types. It is a reference to a +/// python object. A single python object can have multiple references, and +/// this reference counting is accounted for by this type. Use the `.clone()` +/// method to create a new reference and increment the amount of references +/// to the python object by 1. +#[repr(transparent)] +pub struct PyObjectRef { + ptr: NonNull<PyObject>, +} + +impl Clone for PyObjectRef { + #[inline(always)] + fn clone(&self) -> Self { + (**self).to_owned() + } +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for PyObjectRef {} + unsafe impl Sync for PyObjectRef {} + } +} + +#[repr(transparent)] +pub struct PyObject(PyInner<Erased>); + +impl Deref for PyObjectRef { + type Target = PyObject; + + #[inline(always)] + fn deref(&self) -> &PyObject { + unsafe { self.ptr.as_ref() } + } +} + +impl ToOwned for PyObject { + type Owned = PyObjectRef; + + #[inline(always)] + fn to_owned(&self) -> Self::Owned { + self.0.ref_count.inc(); + PyObjectRef { + ptr: NonNull::from(self), + } + } +} + +impl PyObject { + /// Atomically try to create a strong reference. + /// Returns `None` if the strong count is already 0 (object being destroyed). + /// Uses CAS to prevent the TOCTOU race between checking strong_count and + /// incrementing it. + #[inline] + pub fn try_to_owned(&self) -> Option<PyObjectRef> { + if self.0.ref_count.safe_inc() { + Some(PyObjectRef { + ptr: NonNull::from(self), + }) + } else { + None + } + } + + /// Like [`try_to_owned`](Self::try_to_owned), but from a raw pointer. + /// + /// Uses `addr_of!` to access `ref_count` without forming `&PyObject`, + /// minimizing the borrow scope when the pointer may be stale + /// (e.g. cache-hit paths protected by version guards). + /// + /// # Safety + /// `ptr` must point to a live (not yet deallocated) `PyObject`, or to + /// memory whose `ref_count` field is still atomically readable + /// (same guarantee as `_Py_TryIncRefShared`). + #[inline] + pub unsafe fn try_to_owned_from_ptr(ptr: *mut Self) -> Option<PyObjectRef> { + let inner = ptr.cast::<PyInner<Erased>>(); + let ref_count = unsafe { &*core::ptr::addr_of!((*inner).ref_count) }; + if ref_count.safe_inc() { + Some(PyObjectRef { + ptr: unsafe { NonNull::new_unchecked(ptr) }, + }) + } else { + None + } + } +} + +impl PyObjectRef { + #[inline(always)] + pub const fn into_raw(self) -> NonNull<PyObject> { + let ptr = self.ptr; + core::mem::forget(self); + ptr + } + + /// # Safety + /// The raw pointer must have been previously returned from a call to + /// [`PyObjectRef::into_raw`]. The user is responsible for ensuring that the inner data is not + /// dropped more than once due to mishandling the reference count by calling this function + /// too many times. + #[inline(always)] + pub const unsafe fn from_raw(ptr: NonNull<PyObject>) -> Self { + Self { ptr } + } + + /// Attempt to downcast this reference to a subclass. + /// + /// If the downcast fails, the original ref is returned in as `Err` so + /// another downcast can be attempted without unnecessary cloning. + #[inline(always)] + pub fn downcast<T: PyPayload>(self) -> Result<PyRef<T>, Self> { + if self.downcastable::<T>() { + Ok(unsafe { self.downcast_unchecked() }) + } else { + Err(self) + } + } + + pub fn try_downcast<T: PyPayload>(self, vm: &VirtualMachine) -> PyResult<PyRef<T>> { + T::try_downcast_from(&self, vm)?; + Ok(unsafe { self.downcast_unchecked() }) + } + + /// Force to downcast this reference to a subclass. + /// + /// # Safety + /// T must be the exact payload type + #[inline(always)] + pub unsafe fn downcast_unchecked<T>(self) -> PyRef<T> { + // PyRef::from_obj_unchecked(self) + // manual impl to avoid assertion + let obj = ManuallyDrop::new(self); + PyRef { + ptr: obj.ptr.cast(), + } + } + + // ideally we'd be able to define these in pyobject.rs, but method visibility rules are weird + + /// Attempt to downcast this reference to the specific class that is associated `T`. + /// + /// If the downcast fails, the original ref is returned in as `Err` so + /// another downcast can be attempted without unnecessary cloning. + #[inline] + pub fn downcast_exact<T: PyPayload>(self, vm: &VirtualMachine) -> Result<PyRefExact<T>, Self> { + if self.class().is(T::class(&vm.ctx)) { + // TODO: is this always true? + assert!( + self.downcastable::<T>(), + "obj.__class__ is T::class() but payload is not T" + ); + // SAFETY: just asserted that downcastable::<T>() + Ok(unsafe { PyRefExact::new_unchecked(PyRef::from_obj_unchecked(self)) }) + } else { + Err(self) + } + } +} + +impl PyObject { + /// Returns the WeakRefList if the type supports weakrefs (HAS_WEAKREF). + /// The WeakRefList is stored as a separate prefix before PyInner, + /// independent from ObjExt (dict/slots). + #[inline(always)] + fn weak_ref_list(&self) -> Option<&WeakRefList> { + self.0.weakref_list_ref() + } + + /// Returns the first weakref in the weakref list, if any. + pub(crate) fn get_weakrefs(&self) -> Option<PyObjectRef> { + let wrl = self.weak_ref_list()?; + let _lock = weakref_lock::lock(self as *const PyObject as usize); + let head_ptr = wrl.head.load(Ordering::Relaxed); + if head_ptr.is_null() { + None + } else { + let head = unsafe { &*head_ptr }; + if head.0.ref_count.safe_inc() { + Some(unsafe { PyRef::from_raw(head_ptr) }.into()) + } else { + None + } + } + } + + pub(crate) fn downgrade_with_weakref_typ_opt( + &self, + callback: Option<PyObjectRef>, + // a reference to weakref_type **specifically** + typ: PyTypeRef, + ) -> Option<PyRef<PyWeak>> { + self.weak_ref_list() + .map(|wrl| wrl.add(self, typ, true, callback, None)) + } + + pub(crate) fn downgrade_with_typ( + &self, + callback: Option<PyObjectRef>, + typ: PyTypeRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PyWeak>> { + // Check HAS_WEAKREF flag first + if !self + .class() + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_WEAKREF) + { + return Err(vm.new_type_error(format!( + "cannot create weak reference to '{}' object", + self.class().name() + ))); + } + let dict = if typ + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + { + Some(vm.ctx.new_dict()) + } else { + None + }; + let cls_is_weakref = typ.is(vm.ctx.types.weakref_type); + let wrl = self.weak_ref_list().ok_or_else(|| { + vm.new_type_error(format!( + "cannot create weak reference to '{}' object", + self.class().name() + )) + })?; + Ok(wrl.add(self, typ, cls_is_weakref, callback, dict)) + } + + pub fn downgrade( + &self, + callback: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PyWeak>> { + self.downgrade_with_typ(callback, vm.ctx.types.weakref_type.to_owned(), vm) + } + + pub fn get_weak_references(&self) -> Option<Vec<PyRef<PyWeak>>> { + self.weak_ref_list() + .map(|wrl| wrl.get_weak_references(self)) + } + + #[deprecated(note = "use downcastable instead")] + #[inline(always)] + pub fn payload_is<T: PyPayload>(&self) -> bool { + self.0.vtable.typeid == T::PAYLOAD_TYPE_ID + } + + /// Force to return payload as T. + /// + /// # Safety + /// The actual payload type must be T. + #[deprecated(note = "use downcast_unchecked_ref instead")] + #[inline(always)] + pub const unsafe fn payload_unchecked<T: PyPayload>(&self) -> &T { + // we cast to a PyInner<T> first because we don't know T's exact offset because of + // varying alignment, but once we get a PyInner<T> the compiler can get it for us + let inner = unsafe { &*(&self.0 as *const PyInner<Erased> as *const PyInner<T>) }; + &inner.payload + } + + #[deprecated(note = "use downcast_ref instead")] + #[inline(always)] + pub fn payload<T: PyPayload>(&self) -> Option<&T> { + #[allow(deprecated)] + if self.payload_is::<T>() { + #[allow(deprecated)] + Some(unsafe { self.payload_unchecked() }) + } else { + None + } + } + + #[inline(always)] + pub fn class(&self) -> &Py<PyType> { + self.0.typ.deref() + } + + pub fn set_class(&self, typ: PyTypeRef, vm: &VirtualMachine) { + self.0.typ.swap_to_temporary_refs(typ, vm); + } + + #[deprecated(note = "use downcast_ref_if_exact instead")] + #[inline(always)] + pub fn payload_if_exact<T: PyPayload>(&self, vm: &VirtualMachine) -> Option<&T> { + if self.class().is(T::class(&vm.ctx)) { + #[allow(deprecated)] + self.payload() + } else { + None + } + } + + #[inline(always)] + fn instance_dict(&self) -> Option<&InstanceDict> { + self.0.ext_ref().and_then(|ext| ext.dict.as_ref()) + } + + #[inline(always)] + pub fn dict(&self) -> Option<PyDictRef> { + self.instance_dict().map(|d| d.get()) + } + + /// Set the dict field. Returns `Err(dict)` if this object does not have a dict field + /// in the first place. + pub fn set_dict(&self, dict: PyDictRef) -> Result<(), PyDictRef> { + match self.instance_dict() { + Some(d) => { + d.set(dict); + Ok(()) + } + None => Err(dict), + } + } + + #[deprecated(note = "use downcast_ref instead")] + #[inline(always)] + pub fn payload_if_subclass<T: crate::PyPayload>(&self, vm: &VirtualMachine) -> Option<&T> { + if self.class().fast_issubclass(T::class(&vm.ctx)) { + #[allow(deprecated)] + self.payload() + } else { + None + } + } + + #[inline] + pub(crate) fn typeid(&self) -> TypeId { + self.0.vtable.typeid + } + + /// Check if this object can be downcast to T. + #[inline(always)] + pub fn downcastable<T: PyPayload>(&self) -> bool { + self.typeid() == T::PAYLOAD_TYPE_ID && unsafe { T::validate_downcastable_from(self) } + } + + /// Attempt to downcast this reference to a subclass. + pub fn try_downcast_ref<'a, T: PyPayload>( + &'a self, + vm: &VirtualMachine, + ) -> PyResult<&'a Py<T>> { + T::try_downcast_from(self, vm)?; + Ok(unsafe { self.downcast_unchecked_ref::<T>() }) + } + + /// Attempt to downcast this reference to a subclass. + #[inline(always)] + pub fn downcast_ref<T: PyPayload>(&self) -> Option<&Py<T>> { + if self.downcastable::<T>() { + // SAFETY: just checked that the payload is T, and PyRef is repr(transparent) over + // PyObjectRef + Some(unsafe { self.downcast_unchecked_ref::<T>() }) + } else { + None + } + } + + #[inline(always)] + pub fn downcast_ref_if_exact<T: PyPayload>(&self, vm: &VirtualMachine) -> Option<&Py<T>> { + self.class() + .is(T::class(&vm.ctx)) + .then(|| unsafe { self.downcast_unchecked_ref::<T>() }) + } + + /// # Safety + /// T must be the exact payload type + #[inline(always)] + pub unsafe fn downcast_unchecked_ref<T: PyPayload>(&self) -> &Py<T> { + debug_assert!(self.downcastable::<T>()); + // SAFETY: requirements forwarded from caller + unsafe { &*(self as *const Self as *const Py<T>) } + } + + #[inline(always)] + pub fn strong_count(&self) -> usize { + self.0.ref_count.get() + } + + #[inline] + pub fn weak_count(&self) -> Option<usize> { + self.weak_ref_list().map(|wrl| wrl.count(self)) + } + + #[inline(always)] + pub const fn as_raw(&self) -> *const Self { + self + } + + /// Check if the object has been finalized (__del__ already called). + /// _PyGC_FINALIZED in Py_GIL_DISABLED mode. + #[inline] + pub(crate) fn gc_finalized(&self) -> bool { + GcBits::from_bits_retain(self.0.gc_bits.load(Ordering::Relaxed)).contains(GcBits::FINALIZED) + } + + /// Mark the object as finalized. Should be called before __del__. + /// _PyGC_SET_FINALIZED in Py_GIL_DISABLED mode. + #[inline] + pub(crate) fn set_gc_finalized(&self) { + self.set_gc_bit(GcBits::FINALIZED); + } + + /// Set a GC bit atomically. + #[inline] + pub(crate) fn set_gc_bit(&self, bit: GcBits) { + self.0.gc_bits.fetch_or(bit.bits(), Ordering::Relaxed); + } + + /// Get the GC generation index for this object. + #[inline] + pub(crate) fn gc_generation(&self) -> u8 { + self.0.gc_generation.load(Ordering::Relaxed) + } + + /// Set the GC generation index for this object. + /// Must only be called while holding the generation list's write lock. + #[inline] + pub(crate) fn set_gc_generation(&self, generation: u8) { + self.0.gc_generation.store(generation, Ordering::Relaxed); + } + + /// _PyObject_GC_TRACK + #[inline] + pub(crate) fn set_gc_tracked(&self) { + self.set_gc_bit(GcBits::TRACKED); + } + + /// _PyObject_GC_UNTRACK + #[inline] + pub(crate) fn clear_gc_tracked(&self) { + self.0 + .gc_bits + .fetch_and(!GcBits::TRACKED.bits(), Ordering::Relaxed); + } + + #[inline(always)] // the outer function is never inlined + fn drop_slow_inner(&self) -> Result<(), ()> { + // __del__ is mostly not implemented + #[inline(never)] + #[cold] + fn call_slot_del( + zelf: &PyObject, + slot_del: fn(&PyObject, &VirtualMachine) -> PyResult<()>, + ) -> Result<(), ()> { + let ret = crate::vm::thread::with_vm(zelf, |vm| { + // Temporarily resurrect (0→2) so ref_count stays positive + // during __del__, preventing safe_inc from seeing 0. + zelf.0.ref_count.inc_by(2); + + if let Err(e) = slot_del(zelf, vm) { + let del_method = zelf.get_class_attr(identifier!(vm, __del__)).unwrap(); + vm.run_unraisable(e, None, del_method); + } + + // Undo the temporary resurrection. Always remove both + // temporary refs; the second dec returns true only when + // ref_count drops to 0 (no resurrection). + let _ = zelf.0.ref_count.dec(); + zelf.0.ref_count.dec() + }); + match ret { + // the decref set ref_count back to 0 + Some(true) => Ok(()), + // we've been resurrected by __del__ + Some(false) => Err(()), + None => Ok(()), + } + } + + // __del__ should only be called once (like _PyGC_FINALIZED check in GIL_DISABLED) + // We call __del__ BEFORE clearing weakrefs to allow the finalizer to access + // the object's weak references if needed. + let del = self.class().slots.del.load(); + if let Some(slot_del) = del + && !self.gc_finalized() + { + self.set_gc_finalized(); + call_slot_del(self, slot_del)?; + } + + // Clear weak refs AFTER __del__. + // Note: This differs from GC behavior which clears weakrefs before finalizers, + // but for direct deallocation (drop_slow_inner), we need to allow the finalizer + // to run without triggering use-after-free from WeakRefList operations. + if let Some(wrl) = self.weak_ref_list() { + wrl.clear(self); + } + + Ok(()) + } + + /// _Py_Dealloc: dispatch to type's dealloc + #[inline(never)] + unsafe fn drop_slow(ptr: NonNull<Self>) { + let dealloc = unsafe { ptr.as_ref().0.vtable.dealloc }; + unsafe { dealloc(ptr.as_ptr()) } + } + + /// # Safety + /// This call will make the object live forever. + pub(crate) unsafe fn mark_intern(&self) { + self.0.ref_count.leak(); + } + + pub(crate) fn is_interned(&self) -> bool { + self.0.ref_count.is_leaked() + } + + pub(crate) fn get_slot(&self, offset: usize) -> Option<PyObjectRef> { + self.0.ext_ref().unwrap().slots[offset].read().clone() + } + + pub(crate) fn set_slot(&self, offset: usize, value: Option<PyObjectRef>) { + *self.0.ext_ref().unwrap().slots[offset].write() = value; + } + + /// _PyObject_GC_IS_TRACKED + pub fn is_gc_tracked(&self) -> bool { + GcBits::from_bits_retain(self.0.gc_bits.load(Ordering::Relaxed)).contains(GcBits::TRACKED) + } + + /// Get the referents (objects directly referenced) of this object. + /// Uses the full traverse including dict and slots. + pub fn gc_get_referents(&self) -> Vec<PyObjectRef> { + let mut result = Vec::new(); + self.0.traverse(&mut |child: &PyObject| { + result.push(child.to_owned()); + }); + result + } + + /// Call __del__ if present, without triggering object deallocation. + /// Used by GC to call finalizers before breaking cycles. + /// This allows proper resurrection detection. + /// PyObject_CallFinalizerFromDealloc + pub fn try_call_finalizer(&self) { + let del = self.class().slots.del.load(); + if let Some(slot_del) = del + && !self.gc_finalized() + { + // Mark as finalized BEFORE calling __del__ to prevent double-call + // This ensures drop_slow_inner() won't call __del__ again + self.set_gc_finalized(); + let result = crate::vm::thread::with_vm(self, |vm| { + if let Err(e) = slot_del(self, vm) + && let Some(del_method) = self.get_class_attr(identifier!(vm, __del__)) + { + vm.run_unraisable(e, None, del_method); + } + }); + let _ = result; + } + } + + /// Clear weakrefs but collect callbacks instead of calling them. + /// This is used by GC to ensure ALL weakrefs are cleared BEFORE any callbacks run. + /// Returns collected callbacks as (PyRef<PyWeak>, callback) pairs. + // = handle_weakrefs + pub fn gc_clear_weakrefs_collect_callbacks(&self) -> Vec<(PyRef<PyWeak>, PyObjectRef)> { + if let Some(wrl) = self.weak_ref_list() { + wrl.clear_for_gc_collect_callbacks(self) + } else { + vec![] + } + } + + /// Get raw pointers to referents without incrementing reference counts. + /// This is used during GC to avoid reference count manipulation. + /// tp_traverse visits objects without incref + /// + /// # Safety + /// The returned pointers are only valid as long as the object is alive + /// and its contents haven't been modified. + pub unsafe fn gc_get_referent_ptrs(&self) -> Vec<NonNull<PyObject>> { + let mut result = Vec::new(); + // Traverse the entire object including dict and slots + self.0.traverse(&mut |child: &PyObject| { + result.push(NonNull::from(child)); + }); + result + } + + /// Pop edges from this object for cycle breaking. + /// Returns extracted child references that were removed from this object (tp_clear). + /// This is used during garbage collection to break circular references. + /// + /// # Safety + /// - ptr must be a valid pointer to a PyObject + /// - The caller must have exclusive access (no other references exist) + /// - This is only safe during GC when the object is unreachable + pub unsafe fn gc_clear_raw(ptr: *mut PyObject) -> Vec<PyObjectRef> { + let mut result = Vec::new(); + let obj = unsafe { &*ptr }; + + // 1. Clear payload-specific references (vtable.clear / tp_clear) + if let Some(clear_fn) = obj.0.vtable.clear { + unsafe { clear_fn(ptr, &mut result) }; + } + + // 2. Clear dict and member slots (subtype_clear) + // Detach the dict via Py_CLEAR(*_PyObject_GetDictPtr(self)) — NULL + // the pointer without clearing dict contents. The dict may still be + // referenced by other live objects (e.g. function.__globals__). + let (flags, member_count) = obj.0.read_type_flags(); + let has_ext = flags.has_feature(crate::types::PyTypeFlags::HAS_DICT) || member_count > 0; + if has_ext { + let has_weakref = flags.has_feature(crate::types::PyTypeFlags::HAS_WEAKREF); + let offset = if has_weakref { + WEAKREF_OFFSET + EXT_OFFSET + } else { + EXT_OFFSET + }; + let self_addr = (ptr as *const u8).addr(); + let ext_ptr = + core::ptr::with_exposed_provenance_mut::<ObjExt>(self_addr.wrapping_sub(offset)); + let ext = unsafe { &mut *ext_ptr }; + if let Some(old_dict) = ext.dict.take() { + // Get the dict ref before dropping InstanceDict + let dict_ref = old_dict.into_inner(); + result.push(dict_ref.into()); + } + for slot in ext.slots.iter() { + if let Some(val) = slot.write().take() { + result.push(val); + } + } + } + + result + } + + /// Clear this object for cycle breaking (tp_clear). + /// This version takes &self but should only be called during GC + /// when exclusive access is guaranteed. + /// + /// # Safety + /// - The caller must guarantee exclusive access (no other references exist) + /// - This is only safe during GC when the object is unreachable + pub unsafe fn gc_clear(&self) -> Vec<PyObjectRef> { + // SAFETY: During GC collection, this object is unreachable (gc_refs == 0), + // meaning no other code has a reference to it. The only references are + // internal cycle references which we're about to break. + unsafe { Self::gc_clear_raw(self as *const _ as *mut PyObject) } + } + + /// Check if this object has clear capability (tp_clear) + // Py_TPFLAGS_HAVE_GC types have tp_clear + pub fn gc_has_clear(&self) -> bool { + self.0.vtable.clear.is_some() + || self + .0 + .ext_ref() + .is_some_and(|ext| ext.dict.is_some() || !ext.slots.is_empty()) + } +} + +impl Borrow<PyObject> for PyObjectRef { + #[inline(always)] + fn borrow(&self) -> &PyObject { + self + } +} + +impl AsRef<PyObject> for PyObjectRef { + #[inline(always)] + fn as_ref(&self) -> &PyObject { + self + } +} + +impl<'a, T: PyPayload> From<&'a Py<T>> for &'a PyObject { + #[inline(always)] + fn from(py_ref: &'a Py<T>) -> Self { + py_ref.as_object() + } +} + +impl Drop for PyObjectRef { + #[inline] + fn drop(&mut self) { + if self.0.ref_count.dec() { + unsafe { PyObject::drop_slow(self.ptr) } + } + } +} + +impl fmt::Debug for PyObject { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // SAFETY: the vtable contains functions that accept payload types that always match up + // with the payload of the object + unsafe { (self.0.vtable.debug)(self, f) } + } +} + +impl fmt::Debug for PyObjectRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.as_object().fmt(f) + } +} + +const STACKREF_BORROW_TAG: usize = 1; + +/// A tagged stack reference to a Python object. +/// +/// Uses the lowest bit of the pointer to distinguish owned vs borrowed: +/// - bit 0 = 0 → **owned**: refcount was incremented; Drop will decrement. +/// - bit 0 = 1 → **borrowed**: no refcount change; Drop is a no-op. +/// +/// Same size as `PyObjectRef` (one pointer-width). `PyObject` is at least +/// 8-byte aligned, so the low bit is always available for tagging. +/// +/// Uses `NonZeroUsize` so that `Option<PyStackRef>` has the same size as +/// `PyStackRef` via niche optimization (matching `Option<PyObjectRef>`). +#[repr(transparent)] +pub struct PyStackRef { + bits: NonZeroUsize, +} + +impl PyStackRef { + /// Create an owned stack reference, consuming the `PyObjectRef`. + /// Refcount is NOT incremented — ownership is transferred. + #[inline(always)] + pub fn new_owned(obj: PyObjectRef) -> Self { + let ptr = obj.into_raw(); + let bits = ptr.as_ptr() as usize; + debug_assert!( + bits & STACKREF_BORROW_TAG == 0, + "PyObject pointer must be aligned" + ); + Self { + // SAFETY: valid PyObject pointers are never null + bits: unsafe { NonZeroUsize::new_unchecked(bits) }, + } + } + + /// Create a borrowed stack reference from a `&PyObject`. + /// + /// # Safety + /// The caller must guarantee that the pointed-to object lives at least as + /// long as this `PyStackRef`. In practice the compiler guarantees that + /// borrowed refs are consumed within the same basic block, before any + /// `STORE_FAST`/`DELETE_FAST` could overwrite the source slot. + #[inline(always)] + pub unsafe fn new_borrowed(obj: &PyObject) -> Self { + let bits = (obj as *const PyObject as usize) | STACKREF_BORROW_TAG; + Self { + // SAFETY: valid PyObject pointers are never null, and ORing with 1 keeps it non-zero + bits: unsafe { NonZeroUsize::new_unchecked(bits) }, + } + } + + /// Whether this is a borrowed (non-owning) reference. + #[inline(always)] + pub fn is_borrowed(&self) -> bool { + self.bits.get() & STACKREF_BORROW_TAG != 0 + } + + /// Get a `&PyObject` reference. Works for both owned and borrowed. + #[inline(always)] + pub fn as_object(&self) -> &PyObject { + unsafe { &*((self.bits.get() & !STACKREF_BORROW_TAG) as *const PyObject) } + } + + /// Convert to an owned `PyObjectRef`. + /// + /// * If **borrowed** → increments refcount, forgets self. + /// * If **owned** → reconstructs `PyObjectRef` from the raw pointer, forgets self. + #[inline(always)] + pub fn to_pyobj(self) -> PyObjectRef { + let obj = if self.is_borrowed() { + self.as_object().to_owned() // inc refcount + } else { + let ptr = unsafe { NonNull::new_unchecked(self.bits.get() as *mut PyObject) }; + unsafe { PyObjectRef::from_raw(ptr) } + }; + core::mem::forget(self); // don't run Drop + obj + } + + /// Promote a borrowed ref to owned **in place** (increments refcount, + /// clears the borrow tag). No-op if already owned. + #[inline(always)] + pub fn promote(&mut self) { + if self.is_borrowed() { + self.as_object().0.ref_count.inc(); + // SAFETY: clearing the low bit of a non-null pointer keeps it non-zero + self.bits = + unsafe { NonZeroUsize::new_unchecked(self.bits.get() & !STACKREF_BORROW_TAG) }; + } + } +} + +impl Drop for PyStackRef { + #[inline] + fn drop(&mut self) { + if !self.is_borrowed() { + // Owned: decrement refcount (potentially deallocate). + let ptr = unsafe { NonNull::new_unchecked(self.bits.get() as *mut PyObject) }; + drop(unsafe { PyObjectRef::from_raw(ptr) }); + } + // Borrowed: nothing to do. + } +} + +impl core::ops::Deref for PyStackRef { + type Target = PyObject; + + #[inline(always)] + fn deref(&self) -> &PyObject { + self.as_object() + } +} + +impl Clone for PyStackRef { + /// Cloning always produces an **owned** reference (increments refcount). + #[inline(always)] + fn clone(&self) -> Self { + Self::new_owned(self.as_object().to_owned()) + } +} + +impl fmt::Debug for PyStackRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_borrowed() { + write!(f, "PyStackRef(borrowed, ")?; + } else { + write!(f, "PyStackRef(owned, ")?; + } + self.as_object().fmt(f)?; + write!(f, ")") + } +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl Send for PyStackRef {} + unsafe impl Sync for PyStackRef {} + } +} + +// Ensure Option<PyStackRef> uses niche optimization and matches Option<PyObjectRef> in size +const _: () = assert!( + core::mem::size_of::<Option<PyStackRef>>() == core::mem::size_of::<Option<PyObjectRef>>() +); +const _: () = + assert!(core::mem::size_of::<Option<PyStackRef>>() == core::mem::size_of::<PyStackRef>()); + +#[repr(transparent)] +pub struct Py<T>(PyInner<T>); + +impl<T: PyPayload> Py<T> { + pub fn downgrade( + &self, + callback: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyWeakRef<T>> { + Ok(PyWeakRef { + weak: self.as_object().downgrade(callback, vm)?, + _marker: PhantomData, + }) + } + + #[inline] + pub fn payload(&self) -> &T { + &self.0.payload + } +} + +impl<T> ToOwned for Py<T> { + type Owned = PyRef<T>; + + #[inline(always)] + fn to_owned(&self) -> Self::Owned { + self.0.ref_count.inc(); + PyRef { + ptr: NonNull::from(self), + } + } +} + +impl<T> Deref for Py<T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + &self.0.payload + } +} + +impl<T: PyPayload> Borrow<PyObject> for Py<T> { + #[inline(always)] + fn borrow(&self) -> &PyObject { + unsafe { &*(&self.0 as *const PyInner<T> as *const PyObject) } + } +} + +impl<T> core::hash::Hash for Py<T> +where + T: core::hash::Hash + PyPayload, +{ + #[inline] + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { + self.deref().hash(state) + } +} + +impl<T> PartialEq for Py<T> +where + T: PartialEq + PyPayload, +{ + #[inline] + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl<T> Eq for Py<T> where T: Eq + PyPayload {} + +impl<T> AsRef<PyObject> for Py<T> +where + T: PyPayload, +{ + #[inline(always)] + fn as_ref(&self) -> &PyObject { + self.borrow() + } +} + +impl<T: PyPayload + core::fmt::Debug> fmt::Debug for Py<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (**self).fmt(f) + } +} + +/// A reference to a Python object. +/// +/// Note that a `PyRef<T>` can only deref to a shared / immutable reference. +/// It is the payload type's responsibility to handle (possibly concurrent) +/// mutability with locks or concurrent data structures if required. +/// +/// A `PyRef<T>` can be directly returned from a built-in function to handle +/// situations (such as when implementing in-place methods such as `__iadd__`) +/// where a reference to the same object must be returned. +#[repr(transparent)] +pub struct PyRef<T> { + ptr: NonNull<Py<T>>, +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl<T> Send for PyRef<T> {} + unsafe impl<T> Sync for PyRef<T> {} + } +} + +impl<T: fmt::Debug> fmt::Debug for PyRef<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (**self).fmt(f) + } +} + +impl<T> Drop for PyRef<T> { + #[inline] + fn drop(&mut self) { + if self.0.ref_count.dec() { + unsafe { PyObject::drop_slow(self.ptr.cast::<PyObject>()) } + } + } +} + +impl<T> Clone for PyRef<T> { + #[inline(always)] + fn clone(&self) -> Self { + (**self).to_owned() + } +} + +impl<T: PyPayload> PyRef<T> { + // #[inline(always)] + // pub(crate) const fn into_non_null(self) -> NonNull<Py<T>> { + // let ptr = self.ptr; + // std::mem::forget(self); + // ptr + // } + + #[inline(always)] + pub(crate) const unsafe fn from_non_null(ptr: NonNull<Py<T>>) -> Self { + Self { ptr } + } + + /// # Safety + /// The raw pointer must point to a valid `Py<T>` object + #[inline(always)] + pub(crate) const unsafe fn from_raw(raw: *const Py<T>) -> Self { + unsafe { Self::from_non_null(NonNull::new_unchecked(raw as *mut _)) } + } + + /// Safety: payload type of `obj` must be `T` + #[inline(always)] + unsafe fn from_obj_unchecked(obj: PyObjectRef) -> Self { + debug_assert!(obj.downcast_ref::<T>().is_some()); + let obj = ManuallyDrop::new(obj); + Self { + ptr: obj.ptr.cast(), + } + } + + pub const fn leak(pyref: Self) -> &'static Py<T> { + let ptr = pyref.ptr; + core::mem::forget(pyref); + unsafe { ptr.as_ref() } + } +} + +impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> { + #[inline(always)] + pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option<PyDictRef>) -> Self { + let has_dict = dict.is_some(); + let is_heaptype = typ.heaptype_ext.is_some(); + + // Try to reuse from freelist (no dict, no heaptype) + let cached = if !has_dict && !is_heaptype { + unsafe { T::freelist_pop(&payload) } + } else { + None + }; + + let ptr = if let Some(cached) = cached { + let inner = cached.as_ptr() as *mut PyInner<T>; + unsafe { + core::ptr::write(&mut (*inner).ref_count, RefCount::new()); + (*inner).gc_bits.store(0, Ordering::Relaxed); + core::ptr::drop_in_place(&mut (*inner).payload); + core::ptr::write(&mut (*inner).payload, payload); + // Freelist only stores exact base types (push-side filter), + // but subtypes sharing the same Rust payload (e.g. structseq) + // may pop entries. Update typ if it differs. + let cached_typ: *const Py<PyType> = &*(*inner).typ; + if core::ptr::eq(cached_typ, &*typ) { + drop(typ); + } else { + let _old = (*inner).typ.swap(typ); + } + } + unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) } + } else { + let inner = PyInner::new(payload, typ, dict); + unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) } + }; + + // Track object if: + // - HAS_TRAVERSE is true (Rust payload implements Traverse), OR + // - has instance dict (user-defined class instances), OR + // - heap type (all heap type instances are GC-tracked, like Py_TPFLAGS_HAVE_GC) + if <T as crate::object::MaybeTraverse>::HAS_TRAVERSE || has_dict || is_heaptype { + let gc = crate::gc_state::gc_state(); + unsafe { + gc.track_object(ptr.cast()); + } + // Check if automatic GC should run + gc.maybe_collect(); + } + + Self { ptr } + } +} + +impl<T: crate::class::PySubclass + core::fmt::Debug> PyRef<T> +where + T::Base: core::fmt::Debug, +{ + /// Converts this reference to the base type (ownership transfer). + /// # Safety + /// T and T::Base must have compatible layouts in size_of::<T::Base>() bytes. + #[inline] + pub fn into_base(self) -> PyRef<T::Base> { + let obj: PyObjectRef = self.into(); + match obj.downcast() { + Ok(base_ref) => base_ref, + Err(_) => unsafe { core::hint::unreachable_unchecked() }, + } + } + #[inline] + pub fn upcast<U: PyPayload + StaticType>(self) -> PyRef<U> + where + T: StaticType, + { + debug_assert!(T::static_type().is_subtype(U::static_type())); + let obj: PyObjectRef = self.into(); + match obj.downcast::<U>() { + Ok(upcast_ref) => upcast_ref, + Err(_) => unsafe { core::hint::unreachable_unchecked() }, + } + } +} + +impl<T: crate::class::PySubclass> Py<T> { + /// Converts `&Py<T>` to `&Py<T::Base>`. + #[inline] + pub fn to_base(&self) -> &Py<T::Base> { + debug_assert!(self.as_object().downcast_ref::<T::Base>().is_some()); + // SAFETY: T is #[repr(transparent)] over T::Base, + // so Py<T> and Py<T::Base> have the same layout. + unsafe { &*(self as *const Py<T> as *const Py<T::Base>) } + } + + /// Converts `&Py<T>` to `&Py<U>` where U is an ancestor type. + #[inline] + pub fn upcast_ref<U: PyPayload + StaticType>(&self) -> &Py<U> + where + T: StaticType, + { + debug_assert!(T::static_type().is_subtype(U::static_type())); + // SAFETY: T is a subtype of U, so Py<T> can be viewed as Py<U>. + unsafe { &*(self as *const Py<T> as *const Py<U>) } + } +} + +impl<T> Borrow<PyObject> for PyRef<T> +where + T: PyPayload, +{ + #[inline(always)] + fn borrow(&self) -> &PyObject { + (**self).as_object() + } +} + +impl<T> AsRef<PyObject> for PyRef<T> +where + T: PyPayload, +{ + #[inline(always)] + fn as_ref(&self) -> &PyObject { + self.borrow() + } +} + +impl<T> From<PyRef<T>> for PyObjectRef { + #[inline] + fn from(value: PyRef<T>) -> Self { + let me = ManuallyDrop::new(value); + Self { ptr: me.ptr.cast() } + } +} + +impl<T> Borrow<Py<T>> for PyRef<T> { + #[inline(always)] + fn borrow(&self) -> &Py<T> { + self + } +} + +impl<T> AsRef<Py<T>> for PyRef<T> { + #[inline(always)] + fn as_ref(&self) -> &Py<T> { + self + } +} + +impl<T> Deref for PyRef<T> { + type Target = Py<T>; + + #[inline(always)] + fn deref(&self) -> &Py<T> { + unsafe { self.ptr.as_ref() } + } +} + +impl<T> core::hash::Hash for PyRef<T> +where + T: core::hash::Hash + PyPayload, +{ + #[inline] + fn hash<H: core::hash::Hasher>(&self, state: &mut H) { + self.deref().hash(state) + } +} + +impl<T> PartialEq for PyRef<T> +where + T: PartialEq + PyPayload, +{ + #[inline] + fn eq(&self, other: &Self) -> bool { + self.deref().eq(other.deref()) + } +} + +impl<T> Eq for PyRef<T> where T: Eq + PyPayload {} + +#[repr(transparent)] +pub struct PyWeakRef<T: PyPayload> { + weak: PyRef<PyWeak>, + _marker: PhantomData<T>, +} + +impl<T: PyPayload> PyWeakRef<T> { + pub fn upgrade(&self) -> Option<PyRef<T>> { + self.weak + .upgrade() + // SAFETY: PyWeakRef<T> was always created from a PyRef<T>, so the object is T + .map(|obj| unsafe { PyRef::from_obj_unchecked(obj) }) + } +} + +/// Partially initialize a struct, ensuring that all fields are +/// either given values or explicitly left uninitialized +macro_rules! partially_init { + ( + $ty:path {$($init_field:ident: $init_value:expr),*$(,)?}, + Uninit { $($uninit_field:ident),*$(,)? }$(,)? + ) => {{ + // check all the fields are there but *don't* actually run it + + #[allow(clippy::diverging_sub_expression, reason = "intentional compile-time field check in an unreachable branch")] + if false { + #[allow(invalid_value, dead_code, unreachable_code)] + let _ = {$ty { + $($init_field: $init_value,)* + $($uninit_field: unreachable!(),)* + }}; + } + let mut m = ::core::mem::MaybeUninit::<$ty>::uninit(); + #[allow(unused_unsafe)] + unsafe { + $(::core::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* + } + m + }}; +} + +pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { + use crate::{builtins::object, class::PyClassImpl}; + use core::mem::MaybeUninit; + + // `type` inherits from `object` + // and both `type` and `object are instances of `type`. + // to produce this circular dependency, we need an unsafe block. + // (and yes, this will never get dropped. TODO?) + let (type_type, object_type) = { + // We cast between these 2 types, so make sure (at compile time) that there's no change in + // layout when we wrap PyInner<PyTypeObj> in MaybeUninit<> + static_assertions::assert_eq_size!(MaybeUninit<PyInner<PyType>>, PyInner<PyType>); + static_assertions::assert_eq_align!(MaybeUninit<PyInner<PyType>>, PyInner<PyType>); + + let type_payload = PyType { + base: None, + bases: PyRwLock::default(), + mro: PyRwLock::default(), + subclasses: PyRwLock::default(), + attributes: PyRwLock::new(Default::default()), + slots: PyType::make_slots(), + heaptype_ext: None, + tp_version_tag: core::sync::atomic::AtomicU32::new(0), + }; + let object_payload = PyType { + base: None, + bases: PyRwLock::default(), + mro: PyRwLock::default(), + subclasses: PyRwLock::default(), + attributes: PyRwLock::new(Default::default()), + slots: object::PyBaseObject::make_slots(), + heaptype_ext: None, + tp_version_tag: core::sync::atomic::AtomicU32::new(0), + }; + // Both type_type and object_type are instances of `type`, which has + // HAS_DICT and HAS_WEAKREF, so they need both ObjExt and WeakRefList prefixes. + // Layout: [ObjExt][WeakRefList][PyInner<PyType>] + let alloc_type_with_prefixes = || -> *mut MaybeUninit<PyInner<PyType>> { + let inner_layout = core::alloc::Layout::new::<MaybeUninit<PyInner<PyType>>>(); + let ext_layout = core::alloc::Layout::new::<ObjExt>(); + let weakref_layout = core::alloc::Layout::new::<WeakRefList>(); + + let (layout, weakref_offset) = ext_layout.extend(weakref_layout).unwrap(); + let (combined, inner_offset) = layout.extend(inner_layout).unwrap(); + let combined = combined.pad_to_align(); + + let alloc_ptr = unsafe { alloc::alloc::alloc(combined) }; + if alloc_ptr.is_null() { + alloc::alloc::handle_alloc_error(combined); + } + alloc_ptr.expose_provenance(); + + unsafe { + let ext_ptr = alloc_ptr as *mut ObjExt; + ext_ptr.write(ObjExt::new(None, 0)); + + let weakref_ptr = alloc_ptr.add(weakref_offset) as *mut WeakRefList; + weakref_ptr.write(WeakRefList::new()); + + alloc_ptr.add(inner_offset) as *mut MaybeUninit<PyInner<PyType>> + } + }; + + let type_type_ptr = alloc_type_with_prefixes(); + unsafe { + type_type_ptr.write(partially_init!( + PyInner::<PyType> { + ref_count: RefCount::new(), + vtable: PyObjVTable::of::<PyType>(), + gc_bits: Radium::new(0), + gc_generation: Radium::new(GC_UNTRACKED), + gc_pointers: Pointers::new(), + payload: type_payload, + }, + Uninit { typ } + )); + } + + let object_type_ptr = alloc_type_with_prefixes(); + unsafe { + object_type_ptr.write(partially_init!( + PyInner::<PyType> { + ref_count: RefCount::new(), + vtable: PyObjVTable::of::<PyType>(), + gc_bits: Radium::new(0), + gc_generation: Radium::new(GC_UNTRACKED), + gc_pointers: Pointers::new(), + payload: object_payload, + }, + Uninit { typ }, + )); + } + + let object_type_ptr = object_type_ptr as *mut PyInner<PyType>; + let type_type_ptr = type_type_ptr as *mut PyInner<PyType>; + + unsafe { + (*type_type_ptr).ref_count.inc(); + let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); + ptr::write(&mut (*object_type_ptr).typ, PyAtomicRef::from(type_type)); + (*type_type_ptr).ref_count.inc(); + let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); + ptr::write(&mut (*type_type_ptr).typ, PyAtomicRef::from(type_type)); + + let object_type = PyTypeRef::from_raw(object_type_ptr.cast()); + // object's mro is [object] + (*object_type_ptr).payload.mro = PyRwLock::new(vec![object_type.clone()]); + + (*type_type_ptr).payload.bases = PyRwLock::new(vec![object_type.clone()]); + (*type_type_ptr).payload.base = Some(object_type.clone()); + + let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); + // type's mro is [type, object] + (*type_type_ptr).payload.mro = + PyRwLock::new(vec![type_type.clone(), object_type.clone()]); + + (type_type, object_type) + } + }; + + let weakref_type = PyType { + base: Some(object_type.clone()), + bases: PyRwLock::new(vec![object_type.clone()]), + mro: PyRwLock::new(vec![object_type.clone()]), + subclasses: PyRwLock::default(), + attributes: PyRwLock::default(), + slots: PyWeak::make_slots(), + heaptype_ext: None, + tp_version_tag: core::sync::atomic::AtomicU32::new(0), + }; + let weakref_type = PyRef::new_ref(weakref_type, type_type.clone(), None); + // Static type: untrack from GC (was tracked by new_ref because PyType has HAS_TRAVERSE) + unsafe { + crate::gc_state::gc_state() + .untrack_object(core::ptr::NonNull::from(weakref_type.as_object())); + } + weakref_type.as_object().clear_gc_tracked(); + // weakref's mro is [weakref, object] + weakref_type.mro.write().insert(0, weakref_type.clone()); + + object_type.subclasses.write().push( + type_type + .as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.clone()) + .unwrap(), + ); + + object_type.subclasses.write().push( + weakref_type + .as_object() + .downgrade_with_weakref_typ_opt(None, weakref_type.clone()) + .unwrap(), + ); + + (type_type, object_type, weakref_type) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn miri_test_type_initialization() { + let _ = init_type_hierarchy(); + } + + #[test] + fn miri_test_drop() { + //cspell:ignore dfghjkl + let ctx = crate::Context::genesis(); + let obj = ctx.new_bytes(b"dfghjkl".to_vec()); + drop(obj); + } +} diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs new file mode 100644 index 00000000000..e39d1c7765f --- /dev/null +++ b/crates/vm/src/object/ext.rs @@ -0,0 +1,701 @@ +use super::{ + core::{Py, PyObject, PyObjectRef, PyRef}, + payload::PyPayload, +}; +use crate::common::{ + atomic::{Ordering, PyAtomic, Radium}, + lock::PyRwLockReadGuard, +}; +use crate::{ + VirtualMachine, + builtins::{PyBaseExceptionRef, PyStrInterned, PyType}, + convert::{IntoPyException, ToPyObject, ToPyResult, TryFromObject}, + vm::Context, +}; +use alloc::fmt; + +use core::{ + borrow::Borrow, + marker::PhantomData, + ops::Deref, + ptr::{NonNull, null_mut}, +}; + +/* Python objects and references. + +Okay, so each python object itself is an class itself (PyObject). Each +python object can have several references to it (PyObjectRef). These +references are Rc (reference counting) rust smart pointers. So when +all references are destroyed, the object itself also can be cleaned up. +Basically reference counting, but then done by rust. + +*/ + +/* + * Good reference: https://github.com/ProgVal/pythonvm-rust/blob/master/src/objects/mod.rs + */ + +/// Use this type for functions which return a python object or an exception. +/// Both the python object and the python exception are `PyObjectRef` types +/// since exceptions are also python objects. +pub type PyResult<T = PyObjectRef> = Result<T, PyBaseExceptionRef>; // A valid value, or an exception + +impl<T: fmt::Display> fmt::Display for PyRef<T> +where + T: PyPayload + fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<T: fmt::Display> fmt::Display for Py<T> +where + T: PyPayload + fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +#[repr(transparent)] +pub struct PyExact<T> { + inner: Py<T>, +} + +impl<T: PyPayload> PyExact<T> { + /// # Safety + /// Given reference must be exact type of payload T + #[inline(always)] + pub const unsafe fn ref_unchecked(r: &Py<T>) -> &Self { + unsafe { &*(r as *const _ as *const Self) } + } +} + +impl<T: PyPayload> Deref for PyExact<T> { + type Target = Py<T>; + + #[inline(always)] + fn deref(&self) -> &Py<T> { + &self.inner + } +} + +impl<T: PyPayload> Borrow<PyObject> for PyExact<T> { + #[inline(always)] + fn borrow(&self) -> &PyObject { + self.inner.borrow() + } +} + +impl<T: PyPayload> AsRef<PyObject> for PyExact<T> { + #[inline(always)] + fn as_ref(&self) -> &PyObject { + self.inner.as_ref() + } +} + +impl<T: PyPayload> Borrow<Py<T>> for PyExact<T> { + #[inline(always)] + fn borrow(&self) -> &Py<T> { + &self.inner + } +} + +impl<T: PyPayload> AsRef<Py<T>> for PyExact<T> { + #[inline(always)] + fn as_ref(&self) -> &Py<T> { + &self.inner + } +} + +impl<T: PyPayload> alloc::borrow::ToOwned for PyExact<T> { + type Owned = PyRefExact<T>; + + fn to_owned(&self) -> Self::Owned { + let owned = self.inner.to_owned(); + unsafe { PyRefExact::new_unchecked(owned) } + } +} + +impl<T: PyPayload> PyRef<T> { + pub fn into_exact_or( + self, + ctx: &Context, + f: impl FnOnce(Self) -> PyRefExact<T>, + ) -> PyRefExact<T> { + if self.class().is(T::class(ctx)) { + unsafe { PyRefExact::new_unchecked(self) } + } else { + f(self) + } + } +} + +/// PyRef but guaranteed not to be a subtype instance +#[derive(Debug)] +#[repr(transparent)] +pub struct PyRefExact<T: PyPayload> { + inner: PyRef<T>, +} + +impl<T: PyPayload> PyRefExact<T> { + /// # Safety + /// obj must have exact type for the payload + pub const unsafe fn new_unchecked(obj: PyRef<T>) -> Self { + Self { inner: obj } + } + + pub fn into_pyref(self) -> PyRef<T> { + self.inner + } +} + +impl<T: PyPayload> Clone for PyRefExact<T> { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { inner } + } +} + +impl<T: PyPayload> TryFromObject for PyRefExact<T> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let target_cls = T::class(&vm.ctx); + let cls = obj.class(); + if cls.is(target_cls) { + let obj = obj + .downcast() + .map_err(|obj| vm.new_downcast_runtime_error(target_cls, &obj))?; + Ok(Self { inner: obj }) + } else if cls.fast_issubclass(target_cls) { + Err(vm.new_type_error(format!( + "Expected an exact instance of '{}', not a subclass '{}'", + target_cls.name(), + cls.name(), + ))) + } else { + Err(vm.new_type_error(format!( + "Expected type '{}', not '{}'", + target_cls.name(), + cls.name(), + ))) + } + } +} + +impl<T: PyPayload> Deref for PyRefExact<T> { + type Target = PyExact<T>; + + #[inline(always)] + fn deref(&self) -> &PyExact<T> { + unsafe { PyExact::ref_unchecked(self.inner.deref()) } + } +} + +impl<T: PyPayload> Borrow<PyObject> for PyRefExact<T> { + #[inline(always)] + fn borrow(&self) -> &PyObject { + self.inner.borrow() + } +} + +impl<T: PyPayload> AsRef<PyObject> for PyRefExact<T> { + #[inline(always)] + fn as_ref(&self) -> &PyObject { + self.inner.as_ref() + } +} + +impl<T: PyPayload> Borrow<Py<T>> for PyRefExact<T> { + #[inline(always)] + fn borrow(&self) -> &Py<T> { + self.inner.borrow() + } +} + +impl<T: PyPayload> AsRef<Py<T>> for PyRefExact<T> { + #[inline(always)] + fn as_ref(&self) -> &Py<T> { + self.inner.as_ref() + } +} + +impl<T: PyPayload> Borrow<PyExact<T>> for PyRefExact<T> { + #[inline(always)] + fn borrow(&self) -> &PyExact<T> { + self + } +} + +impl<T: PyPayload> AsRef<PyExact<T>> for PyRefExact<T> { + #[inline(always)] + fn as_ref(&self) -> &PyExact<T> { + self + } +} + +impl<T: PyPayload> ToPyObject for PyRefExact<T> { + #[inline(always)] + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.inner.into() + } +} + +pub struct PyAtomicRef<T> { + inner: PyAtomic<*mut u8>, + _phantom: PhantomData<T>, +} + +impl<T> Drop for PyAtomicRef<T> { + fn drop(&mut self) { + // SAFETY: We are dropping the atomic reference, so we can safely + // release the pointer. + unsafe { + let ptr = Radium::swap(&self.inner, null_mut(), Ordering::Relaxed); + if let Some(ptr) = NonNull::<PyObject>::new(ptr.cast()) { + let _: PyObjectRef = PyObjectRef::from_raw(ptr); + } + } + } +} + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + unsafe impl<T: Send + PyPayload> Send for PyAtomicRef<T> {} + unsafe impl<T: Sync + PyPayload> Sync for PyAtomicRef<T> {} + unsafe impl<T: Send + PyPayload> Send for PyAtomicRef<Option<T>> {} + unsafe impl<T: Sync + PyPayload> Sync for PyAtomicRef<Option<T>> {} + unsafe impl Send for PyAtomicRef<PyObject> {} + unsafe impl Sync for PyAtomicRef<PyObject> {} + unsafe impl Send for PyAtomicRef<Option<PyObject>> {} + unsafe impl Sync for PyAtomicRef<Option<PyObject>> {} + } +} + +impl<T: fmt::Debug> fmt::Debug for PyAtomicRef<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "PyAtomicRef(")?; + unsafe { + self.inner + .load(Ordering::Relaxed) + .cast::<T>() + .as_ref() + .fmt(f) + }?; + write!(f, ")") + } +} + +impl<T: PyPayload> From<PyRef<T>> for PyAtomicRef<T> { + fn from(pyref: PyRef<T>) -> Self { + let py = PyRef::leak(pyref); + let ptr = py as *const _ as *mut u8; + // Expose provenance so we can re-derive via with_exposed_provenance + // without Stacked Borrows tag restrictions during bootstrap + ptr.expose_provenance(); + Self { + inner: Radium::new(ptr), + _phantom: Default::default(), + } + } +} + +impl<T: PyPayload> Deref for PyAtomicRef<T> { + type Target = Py<T>; + + fn deref(&self) -> &Self::Target { + unsafe { + self.inner + .load(Ordering::Relaxed) + .cast::<Py<T>>() + .as_ref() + .unwrap_unchecked() + } + } +} + +impl<T: PyPayload> PyAtomicRef<T> { + /// Load the raw pointer without creating a reference. + /// Avoids Stacked Borrows retag, safe for use during bootstrap + /// when type objects have self-referential pointers being mutated. + #[inline(always)] + pub(super) fn load_raw(&self) -> *const Py<T> { + self.inner.load(Ordering::Relaxed).cast::<Py<T>>() + } + + /// # Safety + /// The caller is responsible to keep the returned PyRef alive + /// until no more reference can be used via PyAtomicRef::deref() + #[must_use] + pub unsafe fn swap(&self, pyref: PyRef<T>) -> PyRef<T> { + let py = PyRef::leak(pyref) as *const Py<T> as *mut _; + let old = Radium::swap(&self.inner, py, Ordering::AcqRel); + unsafe { PyRef::from_raw(old.cast()) } + } + + pub fn swap_to_temporary_refs(&self, pyref: PyRef<T>, vm: &VirtualMachine) { + let old = unsafe { self.swap(pyref) }; + if let Some(frame) = vm.current_frame() { + frame.temporary_refs.lock().push(old.into()); + } + } +} + +impl<T: PyPayload> From<Option<PyRef<T>>> for PyAtomicRef<Option<T>> { + fn from(opt_ref: Option<PyRef<T>>) -> Self { + let val = opt_ref + .map(|x| PyRef::leak(x) as *const Py<T> as *mut _) + .unwrap_or(null_mut()); + Self { + inner: Radium::new(val), + _phantom: Default::default(), + } + } +} + +impl<T: PyPayload> PyAtomicRef<Option<T>> { + pub fn deref(&self) -> Option<&Py<T>> { + self.deref_ordering(Ordering::Relaxed) + } + + pub fn deref_ordering(&self, ordering: Ordering) -> Option<&Py<T>> { + unsafe { self.inner.load(ordering).cast::<Py<T>>().as_ref() } + } + + pub fn to_owned(&self) -> Option<PyRef<T>> { + self.to_owned_ordering(Ordering::Relaxed) + } + + pub fn to_owned_ordering(&self, ordering: Ordering) -> Option<PyRef<T>> { + self.deref_ordering(ordering).map(|x| x.to_owned()) + } + + /// # Safety + /// The caller is responsible to keep the returned PyRef alive + /// until no more reference can be used via PyAtomicRef::deref() + #[must_use] + pub unsafe fn swap(&self, opt_ref: Option<PyRef<T>>) -> Option<PyRef<T>> { + let val = opt_ref + .map(|x| PyRef::leak(x) as *const Py<T> as *mut _) + .unwrap_or(null_mut()); + let old = Radium::swap(&self.inner, val, Ordering::AcqRel); + unsafe { old.cast::<Py<T>>().as_ref().map(|x| PyRef::from_raw(x)) } + } + + pub fn swap_to_temporary_refs(&self, opt_ref: Option<PyRef<T>>, vm: &VirtualMachine) { + let Some(old) = (unsafe { self.swap(opt_ref) }) else { + return; + }; + if let Some(frame) = vm.current_frame() { + frame.temporary_refs.lock().push(old.into()); + } + } +} + +impl From<PyObjectRef> for PyAtomicRef<PyObject> { + fn from(obj: PyObjectRef) -> Self { + let obj = obj.into_raw(); + Self { + inner: Radium::new(obj.cast().as_ptr()), + _phantom: Default::default(), + } + } +} + +impl Deref for PyAtomicRef<PyObject> { + type Target = PyObject; + + fn deref(&self) -> &Self::Target { + unsafe { + self.inner + .load(Ordering::Relaxed) + .cast::<PyObject>() + .as_ref() + .unwrap_unchecked() + } + } +} + +impl PyAtomicRef<PyObject> { + /// # Safety + /// The caller is responsible to keep the returned PyRef alive + /// until no more reference can be used via PyAtomicRef::deref() + #[must_use] + pub unsafe fn swap(&self, obj: PyObjectRef) -> PyObjectRef { + let obj = obj.into_raw(); + let old = Radium::swap(&self.inner, obj.cast().as_ptr(), Ordering::AcqRel); + unsafe { PyObjectRef::from_raw(NonNull::new_unchecked(old.cast())) } + } + + pub fn swap_to_temporary_refs(&self, obj: PyObjectRef, vm: &VirtualMachine) { + let old = unsafe { self.swap(obj) }; + if let Some(frame) = vm.current_frame() { + frame.temporary_refs.lock().push(old); + } + } +} + +impl From<Option<PyObjectRef>> for PyAtomicRef<Option<PyObject>> { + fn from(obj: Option<PyObjectRef>) -> Self { + let val = obj + .map(|x| x.into_raw().as_ptr().cast()) + .unwrap_or(null_mut()); + Self { + inner: Radium::new(val), + _phantom: Default::default(), + } + } +} + +impl PyAtomicRef<Option<PyObject>> { + pub fn deref(&self) -> Option<&PyObject> { + self.deref_ordering(Ordering::Relaxed) + } + + pub fn deref_ordering(&self, ordering: Ordering) -> Option<&PyObject> { + unsafe { self.inner.load(ordering).cast::<PyObject>().as_ref() } + } + + pub fn to_owned(&self) -> Option<PyObjectRef> { + self.to_owned_ordering(Ordering::Relaxed) + } + + pub fn to_owned_ordering(&self, ordering: Ordering) -> Option<PyObjectRef> { + self.deref_ordering(ordering).map(|x| x.to_owned()) + } + + /// # Safety + /// The caller is responsible to keep the returned PyRef alive + /// until no more reference can be used via PyAtomicRef::deref() + #[must_use] + pub unsafe fn swap(&self, obj: Option<PyObjectRef>) -> Option<PyObjectRef> { + let val = obj + .map(|x| x.into_raw().as_ptr().cast()) + .unwrap_or(null_mut()); + let old = Radium::swap(&self.inner, val, Ordering::AcqRel); + unsafe { NonNull::new(old.cast::<PyObject>()).map(|x| PyObjectRef::from_raw(x)) } + } + + pub fn swap_to_temporary_refs(&self, obj: Option<PyObjectRef>, vm: &VirtualMachine) { + let Some(old) = (unsafe { self.swap(obj) }) else { + return; + }; + if let Some(frame) = vm.current_frame() { + frame.temporary_refs.lock().push(old); + } + } +} + +/// Atomic borrowed (non-ref-counted) optional reference to a Python object. +/// Unlike `PyAtomicRef`, this does NOT own the reference. +/// The pointed-to object must outlive this reference. +pub struct PyAtomicBorrow { + inner: PyAtomic<*mut u8>, +} + +// Safety: Access patterns ensure the pointed-to object outlives this reference. +// The owner (generator/coroutine) clears this in its Drop impl before deallocation. +unsafe impl Send for PyAtomicBorrow {} +unsafe impl Sync for PyAtomicBorrow {} + +impl PyAtomicBorrow { + pub fn new() -> Self { + Self { + inner: Radium::new(null_mut()), + } + } + + pub fn store(&self, obj: &PyObject) { + let ptr = obj as *const PyObject as *mut u8; + Radium::store(&self.inner, ptr, Ordering::Relaxed); + } + + pub fn load(&self) -> Option<&PyObject> { + let ptr = Radium::load(&self.inner, Ordering::Relaxed); + if ptr.is_null() { + None + } else { + Some(unsafe { &*(ptr as *const PyObject) }) + } + } + + pub fn clear(&self) { + Radium::store(&self.inner, null_mut(), Ordering::Relaxed); + } + + pub fn to_owned(&self) -> Option<PyObjectRef> { + self.load().map(|obj| obj.to_owned()) + } +} + +impl Default for PyAtomicBorrow { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for PyAtomicBorrow { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "PyAtomicBorrow({:?})", + Radium::load(&self.inner, Ordering::Relaxed) + ) + } +} + +pub trait AsObject +where + Self: Borrow<PyObject>, +{ + #[inline(always)] + fn as_object(&self) -> &PyObject { + self.borrow() + } + + #[inline(always)] + fn get_id(&self) -> usize { + self.as_object().unique_id() + } + + #[inline(always)] + fn is<T>(&self, other: &T) -> bool + where + T: AsObject, + { + self.get_id() == other.get_id() + } + + #[inline(always)] + fn class(&self) -> &Py<PyType> { + self.as_object().class() + } + + fn get_class_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { + self.class().get_attr(attr_name) + } + + /// Determines if `obj` actually an instance of `cls`, this doesn't call __instancecheck__, so only + /// use this if `cls` is known to have not overridden the base __instancecheck__ magic method. + #[inline] + fn fast_isinstance(&self, cls: &Py<PyType>) -> bool { + self.class().fast_issubclass(cls) + } +} + +impl<T> AsObject for T where T: Borrow<PyObject> {} + +impl PyObject { + #[inline(always)] + fn unique_id(&self) -> usize { + self as *const Self as usize + } +} + +// impl<T: ?Sized> Borrow<PyObject> for PyRc<T> { +// #[inline(always)] +// fn borrow(&self) -> &PyObject { +// unsafe { &*(&**self as *const T as *const PyObject) } +// } +// } + +/// A borrow of a reference to a Python object. This avoids having clone the `PyRef<T>`/ +/// `PyObjectRef`, which isn't that cheap as that increments the atomic reference counter. +// TODO: check if we still need this +#[allow(dead_code)] +pub struct PyLease<'a, T: PyPayload> { + inner: PyRwLockReadGuard<'a, PyRef<T>>, +} + +impl<T: PyPayload> PyLease<'_, T> { + #[inline(always)] + pub fn into_owned(self) -> PyRef<T> { + self.inner.clone() + } +} + +impl<T: PyPayload> Borrow<PyObject> for PyLease<'_, T> { + #[inline(always)] + fn borrow(&self) -> &PyObject { + self.inner.as_ref() + } +} + +impl<T: PyPayload> Deref for PyLease<'_, T> { + type Target = PyRef<T>; + #[inline(always)] + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl<T> fmt::Display for PyLease<'_, T> +where + T: PyPayload + fmt::Display, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +impl<T: PyPayload> ToPyObject for PyRef<T> { + #[inline(always)] + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.into() + } +} + +impl ToPyObject for PyObjectRef { + #[inline(always)] + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self + } +} + +impl ToPyObject for &PyObject { + #[inline(always)] + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.to_owned() + } +} + +// Allows a built-in function to return any built-in object payload without +// explicitly implementing `ToPyObject`. +impl<T> ToPyObject for T +where + T: PyPayload + core::fmt::Debug + Sized, +{ + #[inline(always)] + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + PyPayload::into_pyobject(self, vm) + } +} + +impl<T> ToPyResult for T +where + T: ToPyObject, +{ + #[inline(always)] + fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { + Ok(self.to_pyobject(vm)) + } +} + +impl<T, E> ToPyResult for Result<T, E> +where + T: ToPyObject, + E: IntoPyException, +{ + #[inline(always)] + fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { + self.map(|res| T::to_pyobject(res, vm)) + .map_err(|e| E::into_pyexception(e, vm)) + } +} + +impl IntoPyException for PyBaseExceptionRef { + #[inline(always)] + fn into_pyexception(self, _vm: &VirtualMachine) -> PyBaseExceptionRef { + self + } +} diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs new file mode 100644 index 00000000000..56db97aef1d --- /dev/null +++ b/crates/vm/src/object/mod.rs @@ -0,0 +1,12 @@ +mod core; +mod ext; +mod payload; +mod traverse; +mod traverse_object; + +pub use self::core::*; +pub use self::ext::*; +pub use self::payload::*; +pub(crate) use core::SIZEOF_PYOBJECT_HEAD; +pub(crate) use core::{GC_PERMANENT, GC_UNTRACKED, GcLink}; +pub use traverse::{MaybeTraverse, Traverse, TraverseFn}; diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs new file mode 100644 index 00000000000..a615123c680 --- /dev/null +++ b/crates/vm/src/object/payload.rs @@ -0,0 +1,176 @@ +use crate::object::{MaybeTraverse, Py, PyObjectRef, PyRef, PyResult}; +use crate::{ + PyObject, PyRefExact, + builtins::{PyBaseExceptionRef, PyType, PyTypeRef}, + types::PyTypeFlags, + vm::{Context, VirtualMachine}, +}; +use core::ptr::NonNull; + +cfg_if::cfg_if! { + if #[cfg(feature = "threading")] { + pub trait PyThreadingConstraint: Send + Sync {} + impl<T: Send + Sync> PyThreadingConstraint for T {} + } else { + pub trait PyThreadingConstraint {} + impl<T> PyThreadingConstraint for T {} + } +} + +#[cold] +pub(crate) fn cold_downcast_type_error( + vm: &VirtualMachine, + class: &Py<PyType>, + obj: &PyObject, +) -> PyBaseExceptionRef { + vm.new_downcast_type_error(class, obj) +} + +pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { + const PAYLOAD_TYPE_ID: core::any::TypeId = core::any::TypeId::of::<Self>(); + + /// # Safety + /// This function should only be called if `payload_type_id` matches the type of `obj`. + #[inline] + unsafe fn validate_downcastable_from(_obj: &PyObject) -> bool { + true + } + + fn try_downcast_from(obj: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if obj.downcastable::<Self>() { + return Ok(()); + } + + let class = Self::class(&vm.ctx); + Err(cold_downcast_type_error(vm, class, obj)) + } + + fn class(ctx: &Context) -> &'static Py<PyType>; + + /// Whether this type has a freelist. Types with freelists require + /// immediate (non-deferred) GC untracking during dealloc to prevent + /// race conditions when the object is reused. + const HAS_FREELIST: bool = false; + + /// Maximum number of objects to keep in the freelist. + const MAX_FREELIST: usize = 0; + + /// Try to push a dead object onto this type's freelist for reuse. + /// Returns true if the object was stored (caller must NOT free the memory). + /// Called before tp_clear, so the payload is still intact. + /// + /// # Safety + /// `obj` must be a valid pointer to a `PyInner<Self>` with refcount 0. + /// The payload is still initialized and can be read for bucket selection. + #[inline] + unsafe fn freelist_push(_obj: *mut PyObject) -> bool { + false + } + + /// Try to pop a pre-allocated object from this type's freelist. + /// The returned pointer still has the old payload; the caller must + /// reinitialize `ref_count`, `gc_bits`, and `payload`. + /// + /// # Safety + /// The returned pointer (if Some) must point to a valid `PyInner<Self>` + /// whose payload is still initialized from a previous allocation. The caller + /// will drop and overwrite `payload` before reuse. + #[inline] + unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> { + None + } + + #[inline] + fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef + where + Self: core::fmt::Debug, + { + self.into_ref(&vm.ctx).into() + } + + #[inline] + fn _into_ref(self, cls: PyTypeRef, ctx: &Context) -> PyRef<Self> + where + Self: core::fmt::Debug, + { + let dict = if cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { + Some(ctx.new_dict()) + } else { + None + }; + PyRef::new_ref(self, cls, dict) + } + + #[inline] + fn into_exact_ref(self, ctx: &Context) -> PyRefExact<Self> + where + Self: core::fmt::Debug, + { + unsafe { + // Self::into_ref() always returns exact typed PyRef + PyRefExact::new_unchecked(self.into_ref(ctx)) + } + } + + #[inline] + fn into_ref(self, ctx: &Context) -> PyRef<Self> + where + Self: core::fmt::Debug, + { + let cls = Self::class(ctx); + self._into_ref(cls.to_owned(), ctx) + } + + #[inline] + fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult<PyRef<Self>> + where + Self: core::fmt::Debug, + { + let exact_class = Self::class(&vm.ctx); + if cls.fast_issubclass(exact_class) { + if exact_class.slots.basicsize != cls.slots.basicsize { + #[cold] + #[inline(never)] + fn _into_ref_size_error( + vm: &VirtualMachine, + cls: &Py<PyType>, + exact_class: &Py<PyType>, + ) -> PyBaseExceptionRef { + vm.new_type_error(format!( + "cannot create '{}' instance: size differs from base type '{}'", + cls.name(), + exact_class.name() + )) + } + return Err(_into_ref_size_error(vm, &cls, exact_class)); + } + Ok(self._into_ref(cls, &vm.ctx)) + } else { + #[cold] + #[inline(never)] + fn _into_ref_with_type_error( + vm: &VirtualMachine, + cls: &Py<PyType>, + exact_class: &Py<PyType>, + ) -> PyBaseExceptionRef { + vm.new_type_error(format!( + "'{}' is not a subtype of '{}'", + &cls.name(), + exact_class.name() + )) + } + Err(_into_ref_with_type_error(vm, &cls, exact_class)) + } + } +} + +pub trait PyObjectPayload: + PyPayload + core::any::Any + core::fmt::Debug + MaybeTraverse + PyThreadingConstraint + 'static +{ +} + +impl<T: PyPayload + core::fmt::Debug + 'static> PyObjectPayload for T {} + +pub trait SlotOffset { + fn offset() -> usize; +} diff --git a/crates/vm/src/object/traverse.rs b/crates/vm/src/object/traverse.rs new file mode 100644 index 00000000000..9a5ae324baf --- /dev/null +++ b/crates/vm/src/object/traverse.rs @@ -0,0 +1,244 @@ +use core::ptr::NonNull; + +use rustpython_common::lock::{PyMutex, PyRwLock}; + +use crate::{ + AsObject, PyObject, PyObjectRef, PyRef, PyStackRef, function::Either, object::PyObjectPayload, +}; + +pub type TraverseFn<'a> = dyn FnMut(&PyObject) + 'a; + +/// This trait is used as a "Optional Trait"(I 'd like to use `Trace?` but it's not allowed yet) for PyObjectPayload type +/// +/// impl for PyObjectPayload, `pyclass` proc macro will handle the actual dispatch if type impl `Trace` +/// Every PyObjectPayload impl `MaybeTrace`, which may or may not be traceable +pub trait MaybeTraverse { + /// if is traceable, will be used by vtable to determine + const HAS_TRAVERSE: bool = false; + /// if has clear implementation for circular reference resolution (tp_clear) + const HAS_CLEAR: bool = false; + // if this type is traceable, then call with tracer_fn, default to do nothing + fn try_traverse(&self, traverse_fn: &mut TraverseFn<'_>); + // if this type has clear, extract child refs for circular reference resolution (tp_clear) + fn try_clear(&mut self, _out: &mut Vec<PyObjectRef>) {} +} + +/// Type that need traverse it's children should impl [`Traverse`] (not [`MaybeTraverse`]) +/// # Safety +/// Please carefully read [`Traverse::traverse()`] and follow the guideline +pub unsafe trait Traverse { + /// impl `traverse()` with caution! Following those guideline so traverse doesn't cause memory error!: + /// - Make sure that every owned object(Every PyObjectRef/PyRef) is called with traverse_fn **at most once**. + /// If some field is not called, the worst results is just memory leak, + /// but if some field is called repeatedly, panic and deadlock can happen. + /// + /// - _**DO NOT**_ clone a [`PyObjectRef`] or [`PyRef<T>`] in [`Traverse::traverse()`] + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>); + + /// Extract all owned child PyObjectRefs for circular reference resolution (tp_clear). + /// Called just before object deallocation to break circular references. + /// Default implementation does nothing. + fn clear(&mut self, _out: &mut Vec<PyObjectRef>) {} +} + +unsafe impl Traverse for PyObjectRef { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + traverse_fn(self) + } +} + +unsafe impl Traverse for PyStackRef { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + traverse_fn(self.as_object()) + } +} + +unsafe impl<T: PyObjectPayload> Traverse for PyRef<T> { + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + traverse_fn(self.as_object()) + } +} + +unsafe impl Traverse for () { + fn traverse(&self, _traverse_fn: &mut TraverseFn<'_>) {} +} + +unsafe impl<T: Traverse> Traverse for Option<T> { + #[inline] + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + if let Some(v) = self { + v.traverse(traverse_fn); + } + } +} + +unsafe impl<T> Traverse for [T] +where + T: Traverse, +{ + #[inline] + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + for elem in self { + elem.traverse(traverse_fn); + } + } +} + +unsafe impl<T> Traverse for Box<[T]> +where + T: Traverse, +{ + #[inline] + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + for elem in &**self { + elem.traverse(traverse_fn); + } + } +} + +unsafe impl<T> Traverse for Vec<T> +where + T: Traverse, +{ + #[inline] + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + for elem in self { + elem.traverse(traverse_fn); + } + } +} + +unsafe impl<T: Traverse> Traverse for PyRwLock<T> { + #[inline] + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + // if can't get a lock, this means something else is holding the lock, + // but since gc stopped the world, during gc the lock is always held + // so it is safe to ignore those in gc + if let Some(inner) = self.try_read_recursive() { + inner.traverse(traverse_fn) + } + } +} + +/// Safety: We can't hold lock during traverse it's child because it may cause deadlock. +/// TODO(discord9): check if this is thread-safe to do +/// (Outside of gc phase, only incref/decref will call trace, +/// and refcnt is atomic, so it should be fine?) +unsafe impl<T: Traverse> Traverse for PyMutex<T> { + #[inline] + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + let mut chs: Vec<NonNull<PyObject>> = Vec::new(); + if let Some(obj) = self.try_lock() { + obj.traverse(&mut |ch| { + chs.push(NonNull::from(ch)); + }) + } + chs.iter() + .map(|ch| { + // Safety: during gc, this should be fine, because nothing should write during gc's tracing? + let ch = unsafe { ch.as_ref() }; + traverse_fn(ch); + }) + .count(); + } +} + +macro_rules! trace_tuple { + ($(($NAME: ident, $NUM: tt)),*) => { + unsafe impl<$($NAME: Traverse),*> Traverse for ($($NAME),*) { + #[inline] + fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { + $( + self.$NUM.traverse(traverse_fn); + )* + } + } + + }; +} + +unsafe impl<A: Traverse, B: Traverse> Traverse for Either<A, B> { + #[inline] + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + match self { + Self::A(a) => a.traverse(tracer_fn), + Self::B(b) => b.traverse(tracer_fn), + } + } +} + +// only tuple with 12 elements or less is supported, +// because long tuple is extremely rare in almost every case +unsafe impl<A: Traverse> Traverse for (A,) { + #[inline] + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.traverse(tracer_fn); + } +} + +trace_tuple!((A, 0), (B, 1)); +trace_tuple!((A, 0), (B, 1), (C, 2)); +trace_tuple!((A, 0), (B, 1), (C, 2), (D, 3)); +trace_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4)); +trace_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4), (F, 5)); +trace_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4), (F, 5), (G, 6)); +trace_tuple!( + (A, 0), + (B, 1), + (C, 2), + (D, 3), + (E, 4), + (F, 5), + (G, 6), + (H, 7) +); +trace_tuple!( + (A, 0), + (B, 1), + (C, 2), + (D, 3), + (E, 4), + (F, 5), + (G, 6), + (H, 7), + (I, 8) +); +trace_tuple!( + (A, 0), + (B, 1), + (C, 2), + (D, 3), + (E, 4), + (F, 5), + (G, 6), + (H, 7), + (I, 8), + (J, 9) +); +trace_tuple!( + (A, 0), + (B, 1), + (C, 2), + (D, 3), + (E, 4), + (F, 5), + (G, 6), + (H, 7), + (I, 8), + (J, 9), + (K, 10) +); +trace_tuple!( + (A, 0), + (B, 1), + (C, 2), + (D, 3), + (E, 4), + (F, 5), + (G, 6), + (H, 7), + (I, 8), + (J, 9), + (K, 10), + (L, 11) +); diff --git a/crates/vm/src/object/traverse_object.rs b/crates/vm/src/object/traverse_object.rs new file mode 100644 index 00000000000..f5614b3502a --- /dev/null +++ b/crates/vm/src/object/traverse_object.rs @@ -0,0 +1,99 @@ +use alloc::fmt; +use core::any::TypeId; + +use crate::{ + PyObject, PyObjectRef, + object::{ + Erased, InstanceDict, MaybeTraverse, PyInner, PyObjectPayload, debug_obj, default_dealloc, + try_clear_obj, try_traverse_obj, + }, +}; + +use super::{Traverse, TraverseFn}; + +pub(in crate::object) struct PyObjVTable { + pub(in crate::object) typeid: TypeId, + /// dealloc: handles __del__, weakref clearing, and memory free. + pub(in crate::object) dealloc: unsafe fn(*mut PyObject), + pub(in crate::object) debug: unsafe fn(&PyObject, &mut fmt::Formatter<'_>) -> fmt::Result, + pub(in crate::object) trace: Option<unsafe fn(&PyObject, &mut TraverseFn<'_>)>, + /// Clear for circular reference resolution (tp_clear). + /// Called just before deallocation to extract child references. + pub(in crate::object) clear: Option<unsafe fn(*mut PyObject, &mut Vec<PyObjectRef>)>, +} + +impl PyObjVTable { + pub const fn of<T: PyObjectPayload>() -> &'static Self { + &Self { + typeid: T::PAYLOAD_TYPE_ID, + dealloc: default_dealloc::<T>, + debug: debug_obj::<T>, + trace: const { + if T::HAS_TRAVERSE { + Some(try_traverse_obj::<T>) + } else { + None + } + }, + clear: const { + if T::HAS_CLEAR { + Some(try_clear_obj::<T>) + } else { + None + } + }, + } + } +} + +unsafe impl Traverse for InstanceDict { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.d.traverse(tracer_fn) + } +} + +unsafe impl Traverse for PyInner<Erased> { + /// Because PyObject hold a `PyInner<Erased>`, so we need to trace it + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + // For heap type instances, traverse the type reference. + // PyAtomicRef holds a strong reference (via PyRef::leak), so GC must + // account for it to correctly detect instance ↔ type cycles. + // Static types are always alive and don't need this. + let typ = &*self.typ; + if typ.heaptype_ext.is_some() { + // Safety: Py<PyType> and PyObject share the same memory layout + let typ_obj: &PyObject = unsafe { &*(typ as *const _ as *const PyObject) }; + tracer_fn(typ_obj); + } + // Traverse ObjExt prefix fields (dict and slots) if present + if let Some(ext) = self.ext_ref() { + ext.dict.traverse(tracer_fn); + ext.slots.traverse(tracer_fn); + } + + if let Some(f) = self.vtable.trace { + unsafe { + let zelf = &*(self as *const Self as *const PyObject); + f(zelf, tracer_fn) + } + }; + } +} + +unsafe impl<T: MaybeTraverse> Traverse for PyInner<T> { + /// Type is known, so we can call `try_trace` directly instead of using erased type vtable + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + // For heap type instances, traverse the type reference (same as erased version) + let typ = &*self.typ; + if typ.heaptype_ext.is_some() { + let typ_obj: &PyObject = unsafe { &*(typ as *const _ as *const PyObject) }; + tracer_fn(typ_obj); + } + // Traverse ObjExt prefix fields (dict and slots) if present + if let Some(ext) = self.ext_ref() { + ext.dict.traverse(tracer_fn); + ext.slots.traverse(tracer_fn); + } + T::try_traverse(&self.payload, tracer_fn); + } +} diff --git a/crates/vm/src/ospath.rs b/crates/vm/src/ospath.rs new file mode 100644 index 00000000000..d3123a87acb --- /dev/null +++ b/crates/vm/src/ospath.rs @@ -0,0 +1,347 @@ +use rustpython_common::crt_fd; + +use crate::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyBytes, PyStr}, + class::StaticType, + convert::{IntoPyException, ToPyException, ToPyObject, TryFromObject}, + function::FsPath, +}; +use std::path::{Path, PathBuf}; + +/// path_converter +#[derive(Clone, Copy, Default)] +pub struct PathConverter { + /// Function name for error messages (e.g., "rename") + pub function_name: Option<&'static str>, + /// Argument name for error messages (e.g., "src", "dst") + pub argument_name: Option<&'static str>, + /// If true, embedded null characters are allowed + pub non_strict: bool, +} + +impl PathConverter { + pub const fn new() -> Self { + Self { + function_name: None, + argument_name: None, + non_strict: false, + } + } + + pub const fn function(mut self, name: &'static str) -> Self { + self.function_name = Some(name); + self + } + + pub const fn argument(mut self, name: &'static str) -> Self { + self.argument_name = Some(name); + self + } + + pub const fn non_strict(mut self) -> Self { + self.non_strict = true; + self + } + + /// Generate error message prefix like "rename: " + fn error_prefix(&self) -> String { + match self.function_name { + Some(func) => format!("{}: ", func), + None => String::new(), + } + } + + /// Get argument name for error messages, defaults to "path" + fn arg_name(&self) -> &'static str { + self.argument_name.unwrap_or("path") + } + + /// Format a type error message + fn type_error_msg(&self, type_name: &str, allow_fd: bool) -> String { + let expected = if allow_fd { + "string, bytes, os.PathLike or integer" + } else { + "string, bytes or os.PathLike" + }; + format!( + "{}{} should be {}, not {}", + self.error_prefix(), + self.arg_name(), + expected, + type_name + ) + } + + /// Convert to OsPathOrFd (path or file descriptor) + pub(crate) fn try_path_or_fd<'fd>( + &self, + obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<OsPathOrFd<'fd>> { + // Handle fd (before __fspath__ check, like CPython) + if let Some(int) = obj.try_index_opt(vm) { + // Warn if bool is used as a file descriptor + if obj + .class() + .is(crate::builtins::bool_::PyBool::static_type()) + { + crate::stdlib::_warnings::warn( + vm.ctx.exceptions.runtime_warning, + "bool is used as a file descriptor".to_owned(), + 1, + vm, + )?; + } + let fd = int?.try_to_primitive(vm)?; + return unsafe { crt_fd::Borrowed::try_borrow_raw(fd) } + .map(OsPathOrFd::Fd) + .map_err(|e| e.into_pyexception(vm)); + } + + self.try_path_inner(obj, true, vm).map(OsPathOrFd::Path) + } + + /// Convert to OsPath only (no fd support) + fn try_path_inner( + &self, + obj: PyObjectRef, + allow_fd: bool, + vm: &VirtualMachine, + ) -> PyResult<OsPath> { + // Try direct str/bytes match + let obj = match self.try_match_str_bytes(obj.clone(), vm)? { + Ok(path) => return Ok(path), + Err(obj) => obj, + }; + + // Call __fspath__ + let type_error_msg = || self.type_error_msg(&obj.class().name(), allow_fd); + let method = + vm.get_method_or_type_error(obj.clone(), identifier!(vm, __fspath__), type_error_msg)?; + if vm.is_none(&method) { + return Err(vm.new_type_error(type_error_msg())); + } + let result = method.call((), vm)?; + + // Match __fspath__ result + self.try_match_str_bytes(result.clone(), vm)?.map_err(|_| { + vm.new_type_error(format!( + "{}expected {}.__fspath__() to return str or bytes, not {}", + self.error_prefix(), + obj.class().name(), + result.class().name(), + )) + }) + } + + /// Try to match str or bytes, returns Err(obj) if neither + fn try_match_str_bytes( + &self, + obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<Result<OsPath, PyObjectRef>> { + let check_nul = |b: &[u8]| { + if self.non_strict || memchr::memchr(b'\0', b).is_none() { + Ok(()) + } else { + Err(vm.new_value_error(format!( + "{}embedded null character in {}", + self.error_prefix(), + self.arg_name() + ))) + } + }; + + match_class!(match obj { + s @ PyStr => { + check_nul(s.as_bytes())?; + let path = vm.fsencode(&s)?.into_owned(); + Ok(Ok(OsPath { + path, + origin: Some(s.into()), + })) + } + b @ PyBytes => { + check_nul(&b)?; + let path = FsPath::bytes_as_os_str(&b, vm)?.to_owned(); + Ok(Ok(OsPath { + path, + origin: Some(b.into()), + })) + } + obj => Ok(Err(obj)), + }) + } + + /// Convert to OsPath directly + pub fn try_path(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<OsPath> { + self.try_path_inner(obj, false, vm) + } +} + +/// path_t output - the converted path +#[derive(Clone)] +pub struct OsPath { + pub path: std::ffi::OsString, + /// Original Python object for identity preservation in OSError + pub(super) origin: Option<PyObjectRef>, +} + +#[derive(Debug, Copy, Clone)] +pub enum OutputMode { + String, + Bytes, +} + +impl OutputMode { + pub(super) fn process_path(self, path: impl Into<PathBuf>, vm: &VirtualMachine) -> PyObjectRef { + fn inner(mode: OutputMode, path: PathBuf, vm: &VirtualMachine) -> PyObjectRef { + match mode { + OutputMode::String => vm.fsdecode(path).into(), + OutputMode::Bytes => vm + .ctx + .new_bytes(path.into_os_string().into_encoded_bytes()) + .into(), + } + } + inner(self, path.into(), vm) + } +} + +impl OsPath { + pub fn new_str(path: impl Into<std::ffi::OsString>) -> Self { + let path = path.into(); + Self { path, origin: None } + } + + pub(crate) fn from_fspath(fspath: FsPath, vm: &VirtualMachine) -> PyResult<Self> { + let path = fspath.as_os_str(vm)?.into_owned(); + let origin = match fspath { + FsPath::Str(s) => s.into(), + FsPath::Bytes(b) => b.into(), + }; + Ok(Self { + path, + origin: Some(origin), + }) + } + + /// Convert an object to OsPath using the os.fspath-style error message. + /// This is used by open() which should report "expected str, bytes or os.PathLike object, not" + /// instead of "should be string, bytes or os.PathLike, not". + pub(crate) fn try_from_fspath(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { + let fspath = FsPath::try_from_path_like(obj, true, vm)?; + Self::from_fspath(fspath, vm) + } + + pub fn as_path(&self) -> &Path { + Path::new(&self.path) + } + + pub fn into_bytes(self) -> Vec<u8> { + self.path.into_encoded_bytes() + } + + pub fn to_string_lossy(&self) -> alloc::borrow::Cow<'_, str> { + self.path.to_string_lossy() + } + + pub fn into_cstring(self, vm: &VirtualMachine) -> PyResult<alloc::ffi::CString> { + alloc::ffi::CString::new(self.into_bytes()).map_err(|err| err.to_pyexception(vm)) + } + + #[cfg(windows)] + pub fn to_wide_cstring(&self, vm: &VirtualMachine) -> PyResult<widestring::WideCString> { + widestring::WideCString::from_os_str(&self.path).map_err(|err| err.to_pyexception(vm)) + } + + pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef { + if let Some(ref origin) = self.origin { + origin.clone() + } else { + // Default to string when no origin (e.g., from new_str) + OutputMode::String.process_path(self.path.clone(), vm) + } + } + + /// Get the output mode based on origin type (bytes -> Bytes, otherwise -> String) + pub fn mode(&self) -> OutputMode { + match &self.origin { + Some(obj) if obj.downcast_ref::<PyBytes>().is_some() => OutputMode::Bytes, + _ => OutputMode::String, + } + } +} + +impl AsRef<Path> for OsPath { + fn as_ref(&self) -> &Path { + self.as_path() + } +} + +impl TryFromObject for OsPath { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + PathConverter::new().try_path(obj, vm) + } +} + +// path_t with allow_fd in CPython +#[derive(Clone)] +pub(crate) enum OsPathOrFd<'fd> { + Path(OsPath), + Fd(crt_fd::Borrowed<'fd>), +} + +impl TryFromObject for OsPathOrFd<'_> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + PathConverter::new().try_path_or_fd(obj, vm) + } +} + +impl From<OsPath> for OsPathOrFd<'_> { + fn from(path: OsPath) -> Self { + Self::Path(path) + } +} + +impl OsPathOrFd<'_> { + pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef { + match self { + Self::Path(path) => path.filename(vm), + Self::Fd(fd) => fd.to_pyobject(vm), + } + } +} + +impl crate::exceptions::OSErrorBuilder { + #[must_use] + pub(crate) fn with_filename<'a>( + error: &std::io::Error, + filename: impl Into<OsPathOrFd<'a>>, + vm: &VirtualMachine, + ) -> crate::builtins::PyBaseExceptionRef { + // TODO: return type to PyRef<PyOSError> + use crate::exceptions::ToOSErrorBuilder; + let builder = error.to_os_error_builder(vm); + let builder = builder.filename(filename.into().filename(vm)); + builder.build(vm).upcast() + } + + /// Like `with_filename`, but strips winerror on Windows. + /// Use for C runtime errors (open, fstat, etc.) that should produce + /// `[Errno X]` format instead of `[WinError X]`. + #[must_use] + pub(crate) fn with_filename_from_errno<'a>( + error: &std::io::Error, + filename: impl Into<OsPathOrFd<'a>>, + vm: &VirtualMachine, + ) -> crate::builtins::PyBaseExceptionRef { + use crate::exceptions::ToOSErrorBuilder; + let builder = error.to_os_error_builder(vm); + #[cfg(windows)] + let builder = builder.without_winerror(); + let builder = builder.filename(filename.into().filename(vm)); + builder.build(vm).upcast() + } +} diff --git a/crates/vm/src/prelude.rs b/crates/vm/src/prelude.rs new file mode 100644 index 00000000000..b277f1468ab --- /dev/null +++ b/crates/vm/src/prelude.rs @@ -0,0 +1,11 @@ +//! The prelude imports the various objects and traits. +//! +//! The intention is that one can include `use rustpython_vm::prelude::*`. + +pub use crate::{ + object::{ + AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + PyWeakRef, + }, + vm::{Context, Interpreter, Settings, VirtualMachine}, +}; diff --git a/crates/vm/src/protocol/buffer.rs b/crates/vm/src/protocol/buffer.rs new file mode 100644 index 00000000000..0fe4d15458b --- /dev/null +++ b/crates/vm/src/protocol/buffer.rs @@ -0,0 +1,458 @@ +//! Buffer protocol +//! <https://docs.python.org/3/c-api/buffer.html> + +use crate::{ + Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, + common::{ + borrow::{BorrowedValue, BorrowedValueMut}, + lock::{MapImmutable, PyMutex, PyMutexGuard}, + }, + object::PyObjectPayload, + sliceable::SequenceIndexOp, +}; +use alloc::borrow::Cow; +use core::{fmt::Debug, ops::Range}; +use itertools::Itertools; + +pub struct BufferMethods { + pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<'_, [u8]>, + pub obj_bytes_mut: fn(&PyBuffer) -> BorrowedValueMut<'_, [u8]>, + pub release: fn(&PyBuffer), + pub retain: fn(&PyBuffer), +} + +impl Debug for BufferMethods { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("BufferMethods") + .field("obj_bytes", &(self.obj_bytes as usize)) + .field("obj_bytes_mut", &(self.obj_bytes_mut as usize)) + .field("release", &(self.release as usize)) + .field("retain", &(self.retain as usize)) + .finish() + } +} + +#[derive(Debug, Clone, Traverse)] +pub struct PyBuffer { + pub obj: PyObjectRef, + #[pytraverse(skip)] + pub desc: BufferDescriptor, + #[pytraverse(skip)] + methods: &'static BufferMethods, +} + +impl PyBuffer { + pub fn new(obj: PyObjectRef, desc: BufferDescriptor, methods: &'static BufferMethods) -> Self { + let zelf = Self { + obj, + desc: desc.validate(), + methods, + }; + zelf.retain(); + zelf + } + + pub fn as_contiguous(&self) -> Option<BorrowedValue<'_, [u8]>> { + self.desc + .is_contiguous() + .then(|| unsafe { self.contiguous_unchecked() }) + } + + pub fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<'_, [u8]>> { + (!self.desc.readonly && self.desc.is_contiguous()) + .then(|| unsafe { self.contiguous_mut_unchecked() }) + } + + pub fn from_byte_vector(bytes: Vec<u8>, vm: &VirtualMachine) -> Self { + let bytes_len = bytes.len(); + Self::new( + PyPayload::into_pyobject(VecBuffer::from(bytes), vm), + BufferDescriptor::simple(bytes_len, true), + &VEC_BUFFER_METHODS, + ) + } + + /// # Safety + /// assume the buffer is contiguous + pub unsafe fn contiguous_unchecked(&self) -> BorrowedValue<'_, [u8]> { + self.obj_bytes() + } + + /// # Safety + /// assume the buffer is contiguous and writable + pub unsafe fn contiguous_mut_unchecked(&self) -> BorrowedValueMut<'_, [u8]> { + self.obj_bytes_mut() + } + + pub fn append_to(&self, buf: &mut Vec<u8>) { + if let Some(bytes) = self.as_contiguous() { + buf.extend_from_slice(&bytes); + } else { + let bytes = &*self.obj_bytes(); + self.desc.for_each_segment(true, |range| { + buf.extend_from_slice(&bytes[range.start as usize..range.end as usize]) + }); + } + } + + pub fn contiguous_or_collect<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R { + let borrowed; + let mut collected; + let v = if let Some(bytes) = self.as_contiguous() { + borrowed = bytes; + &*borrowed + } else { + collected = vec![]; + self.append_to(&mut collected); + &collected + }; + f(v) + } + + pub fn obj_as<T: PyObjectPayload>(&self) -> &Py<T> { + unsafe { self.obj.downcast_unchecked_ref() } + } + + pub fn obj_bytes(&self) -> BorrowedValue<'_, [u8]> { + (self.methods.obj_bytes)(self) + } + + pub fn obj_bytes_mut(&self) -> BorrowedValueMut<'_, [u8]> { + (self.methods.obj_bytes_mut)(self) + } + + pub fn release(&self) { + (self.methods.release)(self) + } + + pub fn retain(&self) { + (self.methods.retain)(self) + } + + // drop PyBuffer without calling release + // after this function, the owner should use forget() + // or wrap PyBuffer in the ManuallyDrop to prevent drop() + pub(crate) unsafe fn drop_without_release(&mut self) { + // SAFETY: requirements forwarded from caller + unsafe { + core::ptr::drop_in_place(&mut self.obj); + core::ptr::drop_in_place(&mut self.desc); + } + } +} + +impl<'a> TryFromBorrowedObject<'a> for PyBuffer { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + let cls = obj.class(); + if let Some(f) = cls.slots.as_buffer { + return f(obj, vm); + } + Err(vm.new_type_error(format!( + "a bytes-like object is required, not '{}'", + cls.name() + ))) + } +} + +impl Drop for PyBuffer { + fn drop(&mut self) { + self.release(); + } +} + +#[derive(Debug, Clone)] +pub struct BufferDescriptor { + /// product(shape) * itemsize + /// bytes length, but not the length for obj_bytes() even is contiguous + pub len: usize, + pub readonly: bool, + pub itemsize: usize, + pub format: Cow<'static, str>, + /// (shape, stride, suboffset) for each dimension + pub dim_desc: Vec<(usize, isize, isize)>, + // TODO: flags +} + +impl BufferDescriptor { + pub fn simple(bytes_len: usize, readonly: bool) -> Self { + Self { + len: bytes_len, + readonly, + itemsize: 1, + format: Cow::Borrowed("B"), + dim_desc: vec![(bytes_len, 1, 0)], + } + } + + pub fn format( + bytes_len: usize, + readonly: bool, + itemsize: usize, + format: Cow<'static, str>, + ) -> Self { + Self { + len: bytes_len, + readonly, + itemsize, + format, + dim_desc: vec![(bytes_len / itemsize, itemsize as isize, 0)], + } + } + + #[cfg(debug_assertions)] + pub fn validate(self) -> Self { + // ndim=0 is valid for scalar types (e.g., ctypes Structure) + if self.ndim() == 0 { + // Empty structures (len=0) can have itemsize=0 + if self.len > 0 { + assert!(self.itemsize != 0); + } + assert!(self.itemsize == self.len); + } else { + let mut shape_product = 1; + let has_zero_dim = self.dim_desc.iter().any(|(s, _, _)| *s == 0); + for (shape, stride, suboffset) in self.dim_desc.iter().cloned() { + shape_product *= shape; + assert!(suboffset >= 0); + // For empty arrays (any dimension is 0), strides can be 0 + if !has_zero_dim { + assert!(stride != 0); + } + } + assert!(shape_product * self.itemsize == self.len); + } + self + } + + #[cfg(not(debug_assertions))] + pub fn validate(self) -> Self { + self + } + + pub fn ndim(&self) -> usize { + self.dim_desc.len() + } + + pub fn is_contiguous(&self) -> bool { + if self.len == 0 { + return true; + } + let mut sd = self.itemsize; + for (shape, stride, _) in self.dim_desc.iter().cloned().rev() { + if shape > 1 && stride != sd as isize { + return false; + } + sd *= shape; + } + true + } + + /// this function do not check the bound + /// panic if indices.len() != ndim + pub fn fast_position(&self, indices: &[usize]) -> isize { + let mut pos = 0; + for (i, (_, stride, suboffset)) in indices + .iter() + .cloned() + .zip_eq(self.dim_desc.iter().cloned()) + { + pos += i as isize * stride + suboffset; + } + pos + } + + /// panic if indices.len() != ndim + pub fn position(&self, indices: &[isize], vm: &VirtualMachine) -> PyResult<isize> { + let mut pos = 0; + for (i, (shape, stride, suboffset)) in indices + .iter() + .cloned() + .zip_eq(self.dim_desc.iter().cloned()) + { + let i = i.wrapped_at(shape).ok_or_else(|| { + vm.new_index_error(format!("index out of bounds on dimension {i}")) + })?; + pos += i as isize * stride + suboffset; + } + Ok(pos) + } + + pub fn for_each_segment<F>(&self, try_contiguous: bool, mut f: F) + where + F: FnMut(Range<isize>), + { + if self.ndim() == 0 { + f(0..self.itemsize as isize); + return; + } + if try_contiguous && self.is_last_dim_contiguous() { + self._for_each_segment::<_, true>(0, 0, &mut f); + } else { + self._for_each_segment::<_, false>(0, 0, &mut f); + } + } + + fn _for_each_segment<F, const CONTIGUOUS: bool>(&self, mut index: isize, dim: usize, f: &mut F) + where + F: FnMut(Range<isize>), + { + let (shape, stride, suboffset) = self.dim_desc[dim]; + if dim + 1 == self.ndim() { + if CONTIGUOUS { + f(index..index + (shape * self.itemsize) as isize); + } else { + for _ in 0..shape { + let pos = index + suboffset; + f(pos..pos + self.itemsize as isize); + index += stride; + } + } + return; + } + for _ in 0..shape { + self._for_each_segment::<F, CONTIGUOUS>(index + suboffset, dim + 1, f); + index += stride; + } + } + + /// zip two BufferDescriptor with the same shape + pub fn zip_eq<F>(&self, other: &Self, try_contiguous: bool, mut f: F) + where + F: FnMut(Range<isize>, Range<isize>) -> bool, + { + if self.ndim() == 0 { + f(0..self.itemsize as isize, 0..other.itemsize as isize); + return; + } + if try_contiguous && self.is_last_dim_contiguous() { + self._zip_eq::<_, true>(other, 0, 0, 0, &mut f); + } else { + self._zip_eq::<_, false>(other, 0, 0, 0, &mut f); + } + } + + fn _zip_eq<F, const CONTIGUOUS: bool>( + &self, + other: &Self, + mut a_index: isize, + mut b_index: isize, + dim: usize, + f: &mut F, + ) where + F: FnMut(Range<isize>, Range<isize>) -> bool, + { + let (shape, a_stride, a_suboffset) = self.dim_desc[dim]; + let (_b_shape, b_stride, b_suboffset) = other.dim_desc[dim]; + debug_assert_eq!(shape, _b_shape); + if dim + 1 == self.ndim() { + if CONTIGUOUS { + if f( + a_index..a_index + (shape * self.itemsize) as isize, + b_index..b_index + (shape * other.itemsize) as isize, + ) { + return; + } + } else { + for _ in 0..shape { + let a_pos = a_index + a_suboffset; + let b_pos = b_index + b_suboffset; + if f( + a_pos..a_pos + self.itemsize as isize, + b_pos..b_pos + other.itemsize as isize, + ) { + return; + } + a_index += a_stride; + b_index += b_stride; + } + } + return; + } + + for _ in 0..shape { + self._zip_eq::<F, CONTIGUOUS>( + other, + a_index + a_suboffset, + b_index + b_suboffset, + dim + 1, + f, + ); + a_index += a_stride; + b_index += b_stride; + } + } + + fn is_last_dim_contiguous(&self) -> bool { + let (_, stride, suboffset) = self.dim_desc[self.ndim() - 1]; + suboffset == 0 && stride == self.itemsize as isize + } + + pub fn is_zero_in_shape(&self) -> bool { + self.dim_desc.iter().any(|(shape, _, _)| *shape == 0) + } + + // TODO: support column-major order +} + +pub trait BufferResizeGuard { + type Resizable<'a>: 'a + where + Self: 'a; + fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>>; + fn try_resizable(&self, vm: &VirtualMachine) -> PyResult<Self::Resizable<'_>> { + self.try_resizable_opt().ok_or_else(|| { + vm.new_buffer_error("Existing exports of data: object cannot be re-sized") + }) + } +} + +#[pyclass(module = false, name = "vec_buffer")] +#[derive(Debug, PyPayload)] +pub struct VecBuffer { + data: PyMutex<Vec<u8>>, +} + +#[pyclass(flags(BASETYPE, DISALLOW_INSTANTIATION))] +impl VecBuffer { + pub fn take(&self) -> Vec<u8> { + core::mem::take(&mut self.data.lock()) + } +} + +impl From<Vec<u8>> for VecBuffer { + fn from(data: Vec<u8>) -> Self { + Self { + data: PyMutex::new(data), + } + } +} + +impl PyRef<VecBuffer> { + pub fn into_pybuffer(self, readonly: bool) -> PyBuffer { + let len = self.data.lock().len(); + PyBuffer::new( + self.into(), + BufferDescriptor::simple(len, readonly), + &VEC_BUFFER_METHODS, + ) + } + + pub fn into_pybuffer_with_descriptor(self, desc: BufferDescriptor) -> PyBuffer { + PyBuffer::new(self.into(), desc, &VEC_BUFFER_METHODS) + } +} + +static VEC_BUFFER_METHODS: BufferMethods = BufferMethods { + obj_bytes: |buffer| { + PyMutexGuard::map_immutable(buffer.obj_as::<VecBuffer>().data.lock(), |x| x.as_slice()) + .into() + }, + obj_bytes_mut: |buffer| { + PyMutexGuard::map(buffer.obj_as::<VecBuffer>().data.lock(), |x| { + x.as_mut_slice() + }) + .into() + }, + release: |_| {}, + retain: |_| {}, +}; diff --git a/crates/vm/src/protocol/callable.rs b/crates/vm/src/protocol/callable.rs new file mode 100644 index 00000000000..6ff988abbe6 --- /dev/null +++ b/crates/vm/src/protocol/callable.rs @@ -0,0 +1,274 @@ +use crate::{ + builtins::{PyBoundMethod, PyFunction}, + function::{FuncArgs, IntoFuncArgs}, + types::{GenericMethod, VectorCallFunc}, + {PyObject, PyObjectRef, PyResult, VirtualMachine}, +}; + +impl PyObject { + #[inline] + pub fn to_callable(&self) -> Option<PyCallable<'_>> { + PyCallable::new(self) + } + + #[inline] + pub fn is_callable(&self) -> bool { + self.to_callable().is_some() + } + + /// PyObject_Call*Arg* series + #[inline] + pub fn call(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { + let args = args.into_args(vm); + self.call_with_args(args, vm) + } + + /// PyObject_Call + pub fn call_with_args(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let Some(callable) = self.to_callable() else { + return Err( + vm.new_type_error(format!("'{}' object is not callable", self.class().name())) + ); + }; + vm_trace!("Invoke: {:?} {:?}", callable, args); + callable.invoke(args, vm) + } + + /// Vectorcall: call with owned positional args + optional kwnames. + /// Falls back to FuncArgs-based call if no vectorcall slot. + #[inline] + pub fn vectorcall( + &self, + args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, + ) -> PyResult { + let Some(callable) = self.to_callable() else { + return Err( + vm.new_type_error(format!("'{}' object is not callable", self.class().name())) + ); + }; + callable.invoke_vectorcall(args, nargs, kwnames, vm) + } +} + +#[derive(Debug)] +pub struct PyCallable<'a> { + pub obj: &'a PyObject, + pub call: GenericMethod, + pub vectorcall: Option<VectorCallFunc>, +} + +impl<'a> PyCallable<'a> { + pub fn new(obj: &'a PyObject) -> Option<Self> { + let slots = &obj.class().slots; + let call = slots.call.load()?; + let vectorcall = slots.vectorcall.load(); + Some(PyCallable { + obj, + call, + vectorcall, + }) + } + + pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { + let args = args.into_args(vm); + if !vm.use_tracing.get() { + return (self.call)(self.obj, args, vm); + } + // Python functions get 'call'/'return' events from with_frame(). + // Bound methods delegate to the inner callable, which fires its own events. + // All other callables (built-in functions, etc.) get 'c_call'/'c_return'/'c_exception'. + let is_python_callable = self.obj.downcast_ref::<PyFunction>().is_some() + || self.obj.downcast_ref::<PyBoundMethod>().is_some(); + if is_python_callable { + (self.call)(self.obj, args, vm) + } else { + let callable = self.obj.to_owned(); + vm.trace_event(TraceEvent::CCall, Some(callable.clone()))?; + let result = (self.call)(self.obj, args, vm); + if result.is_ok() { + vm.trace_event(TraceEvent::CReturn, Some(callable))?; + } else { + let _ = vm.trace_event(TraceEvent::CException, Some(callable)); + } + result + } + } + + /// Vectorcall dispatch: use vectorcall slot if available, else fall back to FuncArgs. + #[inline] + pub fn invoke_vectorcall( + &self, + args: Vec<PyObjectRef>, + nargs: usize, + kwnames: Option<&[PyObjectRef]>, + vm: &VirtualMachine, + ) -> PyResult { + if let Some(vc) = self.vectorcall { + if !vm.use_tracing.get() { + return vc(self.obj, args, nargs, kwnames, vm); + } + let is_python_callable = self.obj.downcast_ref::<PyFunction>().is_some() + || self.obj.downcast_ref::<PyBoundMethod>().is_some(); + if is_python_callable { + vc(self.obj, args, nargs, kwnames, vm) + } else { + let callable = self.obj.to_owned(); + vm.trace_event(TraceEvent::CCall, Some(callable.clone()))?; + let result = vc(self.obj, args, nargs, kwnames, vm); + if result.is_ok() { + vm.trace_event(TraceEvent::CReturn, Some(callable))?; + } else { + let _ = vm.trace_event(TraceEvent::CException, Some(callable)); + } + result + } + } else { + // Fallback: convert owned Vec to FuncArgs (move, no clone) + let func_args = FuncArgs::from_vectorcall_owned(args, nargs, kwnames); + self.invoke(func_args, vm) + } + } +} + +/// Trace events for sys.settrace and sys.setprofile. +pub(crate) enum TraceEvent { + Call, + Return, + Exception, + Line, + Opcode, + CCall, + CReturn, + CException, +} + +impl TraceEvent { + /// Whether sys.settrace receives this event. + fn is_trace_event(&self) -> bool { + matches!( + self, + Self::Call | Self::Return | Self::Exception | Self::Line | Self::Opcode + ) + } + + /// Whether sys.setprofile receives this event. + /// In legacy_tracing.c, profile callbacks are only registered for + /// PY_RETURN, PY_UNWIND, C_CALL, C_RETURN, C_RAISE. + fn is_profile_event(&self) -> bool { + matches!( + self, + Self::Call | Self::Return | Self::CCall | Self::CReturn | Self::CException + ) + } + + /// Whether this event is dispatched only when f_trace_opcodes is set. + pub(crate) fn is_opcode_event(&self) -> bool { + matches!(self, Self::Opcode) + } +} + +impl core::fmt::Display for TraceEvent { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + use TraceEvent::*; + match self { + Call => write!(f, "call"), + Return => write!(f, "return"), + Exception => write!(f, "exception"), + Line => write!(f, "line"), + Opcode => write!(f, "opcode"), + CCall => write!(f, "c_call"), + CReturn => write!(f, "c_return"), + CException => write!(f, "c_exception"), + } + } +} + +impl VirtualMachine { + /// Call registered trace function. + /// + /// Returns the trace function's return value: + /// - `Some(obj)` if the trace function returned a non-None value + /// - `None` if it returned Python None or no trace function was active + /// + /// In CPython's trace protocol: + /// - For 'call' events: the return value determines the per-frame `f_trace` + /// - For 'line'/'return' events: the return value can update `f_trace` + #[inline] + pub(crate) fn trace_event( + &self, + event: TraceEvent, + arg: Option<PyObjectRef>, + ) -> PyResult<Option<PyObjectRef>> { + if self.use_tracing.get() { + self._trace_event_inner(event, arg) + } else { + Ok(None) + } + } + fn _trace_event_inner( + &self, + event: TraceEvent, + arg: Option<PyObjectRef>, + ) -> PyResult<Option<PyObjectRef>> { + let trace_func = self.trace_func.borrow().to_owned(); + let profile_func = self.profile_func.borrow().to_owned(); + if self.is_none(&trace_func) && self.is_none(&profile_func) { + return Ok(None); + } + + let is_trace_event = event.is_trace_event(); + let is_profile_event = event.is_profile_event(); + let is_opcode_event = event.is_opcode_event(); + + let Some(frame_ref) = self.current_frame() else { + return Ok(None); + }; + + // Opcode events are only dispatched when f_trace_opcodes is set. + if is_opcode_event && !*frame_ref.trace_opcodes.lock() { + return Ok(None); + } + + let frame: PyObjectRef = frame_ref.into(); + let event = self.ctx.new_str(event.to_string()).into(); + let args = vec![frame, event, arg.unwrap_or_else(|| self.ctx.none())]; + + let mut trace_result = None; + + // temporarily disable tracing, during the call to the + // tracing function itself. + if is_trace_event && !self.is_none(&trace_func) { + self.use_tracing.set(false); + let res = trace_func.call(args.clone(), self); + self.use_tracing.set(true); + match res { + Ok(result) => { + if !self.is_none(&result) { + trace_result = Some(result); + } + } + Err(e) => { + // trace_trampoline behavior: clear per-frame f_trace + // and propagate the error. + if let Some(frame_ref) = self.current_frame() { + *frame_ref.trace.lock() = self.ctx.none(); + } + return Err(e); + } + } + } + + if is_profile_event && !self.is_none(&profile_func) { + self.use_tracing.set(false); + let res = profile_func.call(args, self); + self.use_tracing.set(true); + if res.is_err() { + *self.profile_func.borrow_mut() = self.ctx.none(); + } + } + Ok(trace_result) + } +} diff --git a/crates/vm/src/protocol/iter.rs b/crates/vm/src/protocol/iter.rs new file mode 100644 index 00000000000..aa6ab6769cd --- /dev/null +++ b/crates/vm/src/protocol/iter.rs @@ -0,0 +1,291 @@ +use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + builtins::iter::PySequenceIterator, + convert::{ToPyObject, ToPyResult}, + object::{Traverse, TraverseFn}, +}; +use core::borrow::Borrow; +use core::ops::Deref; + +/// Iterator Protocol +// https://docs.python.org/3/c-api/iter.html +#[derive(Debug, Clone)] +#[repr(transparent)] +pub struct PyIter<O = PyObjectRef>(O) +where + O: Borrow<PyObject>; + +unsafe impl<O: Borrow<PyObject>> Traverse for PyIter<O> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.0.borrow().traverse(tracer_fn); + } +} + +impl PyIter<PyObjectRef> { + pub fn check(obj: &PyObject) -> bool { + obj.class().slots.iternext.load().is_some() + } +} + +impl<O> PyIter<O> +where + O: Borrow<PyObject>, +{ + pub const fn new(obj: O) -> Self { + Self(obj) + } + pub fn next(&self, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let iternext = self + .0 + .borrow() + .class() + .slots + .iternext + .load() + .ok_or_else(|| { + vm.new_type_error(format!( + "'{}' object is not an iterator", + self.0.borrow().class().name() + )) + })?; + iternext(self.0.borrow(), vm) + } + + pub fn iter<'a, 'b, U>( + &'b self, + vm: &'a VirtualMachine, + ) -> PyResult<PyIterIter<'a, U, &'b PyObject>> { + let length_hint = vm.length_hint_opt(self.as_ref().to_owned())?; + Ok(PyIterIter::new(vm, self.0.borrow(), length_hint)) + } + + pub fn iter_without_hint<'a, 'b, U>( + &'b self, + vm: &'a VirtualMachine, + ) -> PyResult<PyIterIter<'a, U, &'b PyObject>> { + Ok(PyIterIter::new(vm, self.0.borrow(), None)) + } +} + +impl PyIter<PyObjectRef> { + /// Returns an iterator over this sequence of objects. + pub fn into_iter<U>(self, vm: &VirtualMachine) -> PyResult<PyIterIter<'_, U, PyObjectRef>> { + let length_hint = vm.length_hint_opt(self.as_object().to_owned())?; + Ok(PyIterIter::new(vm, self.0, length_hint)) + } +} + +impl From<PyIter<Self>> for PyObjectRef { + fn from(value: PyIter<Self>) -> Self { + value.0 + } +} + +impl<O> Borrow<PyObject> for PyIter<O> +where + O: Borrow<PyObject>, +{ + #[inline(always)] + fn borrow(&self) -> &PyObject { + self.0.borrow() + } +} + +impl<O> AsRef<PyObject> for PyIter<O> +where + O: Borrow<PyObject>, +{ + #[inline(always)] + fn as_ref(&self) -> &PyObject { + self.0.borrow() + } +} + +impl<O> Deref for PyIter<O> +where + O: Borrow<PyObject>, +{ + type Target = PyObject; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + self.0.borrow() + } +} + +impl ToPyObject for PyIter<PyObjectRef> { + #[inline(always)] + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + self.into() + } +} + +impl TryFromObject for PyIter<PyObjectRef> { + // This helper function is called at multiple places. First, it is called + // in the vm when a for loop is entered. Next, it is used when the builtin + // function 'iter' is called. + fn try_from_object(vm: &VirtualMachine, iter_target: PyObjectRef) -> PyResult<Self> { + let get_iter = iter_target.class().slots.iter.load(); + if let Some(get_iter) = get_iter { + let iter = get_iter(iter_target, vm)?; + if Self::check(&iter) { + Ok(Self(iter)) + } else { + Err(vm.new_type_error(format!( + "iter() returned non-iterator of type '{}'", + iter.class().name() + ))) + } + } else if let Ok(seq_iter) = PySequenceIterator::new(iter_target.clone(), vm) { + Ok(Self(seq_iter.into_pyobject(vm))) + } else { + Err(vm.new_type_error(format!( + "'{}' object is not iterable", + iter_target.class().name() + ))) + } + } +} + +#[derive(result_like::ResultLike)] +pub enum PyIterReturn<T = PyObjectRef> { + Return(T), + StopIteration(Option<PyObjectRef>), +} + +unsafe impl<T: Traverse> Traverse for PyIterReturn<T> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + match self { + Self::Return(r) => r.traverse(tracer_fn), + Self::StopIteration(Some(obj)) => obj.traverse(tracer_fn), + _ => (), + } + } +} + +impl PyIterReturn { + pub fn from_pyresult(result: PyResult, vm: &VirtualMachine) -> PyResult<Self> { + match result { + Ok(obj) => Ok(Self::Return(obj)), + Err(err) if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) => { + let args = err.get_arg(0); + Ok(Self::StopIteration(args)) + } + Err(err) => Err(err), + } + } + + pub fn from_getitem_result(result: PyResult, vm: &VirtualMachine) -> PyResult<Self> { + match result { + Ok(obj) => Ok(Self::Return(obj)), + Err(err) if err.fast_isinstance(vm.ctx.exceptions.index_error) => { + Ok(Self::StopIteration(None)) + } + Err(err) if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) => { + let args = err.get_arg(0); + Ok(Self::StopIteration(args)) + } + Err(err) => Err(err), + } + } + + pub fn into_async_pyresult(self, vm: &VirtualMachine) -> PyResult { + match self { + Self::Return(obj) => Ok(obj), + Self::StopIteration(v) => Err({ + let args = if let Some(v) = v { vec![v] } else { Vec::new() }; + vm.new_exception(vm.ctx.exceptions.stop_async_iteration.to_owned(), args) + }), + } + } +} + +impl ToPyResult for PyIterReturn { + fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { + match self { + Self::Return(obj) => Ok(obj), + Self::StopIteration(v) => Err(vm.new_stop_iteration(v)), + } + } +} + +impl ToPyResult for PyResult<PyIterReturn> { + fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { + self?.to_pyresult(vm) + } +} + +// Typical rust `Iter` object for `PyIter` +pub struct PyIterIter<'a, T, O = PyObjectRef> +where + O: Borrow<PyObject>, +{ + vm: &'a VirtualMachine, + obj: O, // creating PyIter<O> is zero-cost + length_hint: Option<usize>, + _phantom: core::marker::PhantomData<T>, +} + +unsafe impl<T, O> Traverse for PyIterIter<'_, T, O> +where + O: Traverse + Borrow<PyObject>, +{ + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.obj.traverse(tracer_fn) + } +} + +impl<'a, T, O> PyIterIter<'a, T, O> +where + O: Borrow<PyObject>, +{ + pub const fn new(vm: &'a VirtualMachine, obj: O, length_hint: Option<usize>) -> Self { + Self { + vm, + obj, + length_hint, + _phantom: core::marker::PhantomData, + } + } +} + +impl<T, O> Iterator for PyIterIter<'_, T, O> +where + T: TryFromObject, + O: Borrow<PyObject>, +{ + type Item = PyResult<T>; + + fn next(&mut self) -> Option<Self::Item> { + let imp = |next: PyResult<PyIterReturn>| -> PyResult<Option<T>> { + let Some(obj) = next?.into_result().ok() else { + return Ok(None); + }; + Ok(Some(T::try_from_object(self.vm, obj)?)) + }; + let next = PyIter::new(self.obj.borrow()).next(self.vm); + imp(next).transpose() + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + (self.length_hint.unwrap_or(0), self.length_hint) + } +} + +/// Macro to handle `PyIterReturn` values in iterator implementations. +/// +/// Extracts the object from `PyIterReturn::Return(obj)` or performs early return +/// for `PyIterReturn::StopIteration(v)`. This macro should only be used within +/// functions that return `PyResult<PyIterReturn>`. +#[macro_export] +macro_rules! raise_if_stop { + ($input:expr) => { + match $input { + $crate::protocol::PyIterReturn::Return(obj) => obj, + $crate::protocol::PyIterReturn::StopIteration(v) => { + return Ok($crate::protocol::PyIterReturn::StopIteration(v)) + } + } + }; +} diff --git a/crates/vm/src/protocol/mapping.rs b/crates/vm/src/protocol/mapping.rs new file mode 100644 index 00000000000..6c200043e35 --- /dev/null +++ b/crates/vm/src/protocol/mapping.rs @@ -0,0 +1,213 @@ +use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{ + PyDict, PyStrInterned, + dict::{PyDictItems, PyDictKeys, PyDictValues}, + }, + convert::ToPyResult, + object::{Traverse, TraverseFn}, +}; +use crossbeam_utils::atomic::AtomicCell; + +// Mapping protocol +// https://docs.python.org/3/c-api/mapping.html + +#[allow(clippy::type_complexity)] +#[derive(Default)] +pub struct PyMappingSlots { + pub length: AtomicCell<Option<fn(PyMapping<'_>, &VirtualMachine) -> PyResult<usize>>>, + pub subscript: AtomicCell<Option<fn(PyMapping<'_>, &PyObject, &VirtualMachine) -> PyResult>>, + pub ass_subscript: AtomicCell< + Option<fn(PyMapping<'_>, &PyObject, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>>, + >, +} + +impl core::fmt::Debug for PyMappingSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PyMappingSlots") + } +} + +impl PyMappingSlots { + pub fn has_subscript(&self) -> bool { + self.subscript.load().is_some() + } + + /// Copy from static PyMappingMethods + pub fn copy_from(&self, methods: &PyMappingMethods) { + if let Some(f) = methods.length { + self.length.store(Some(f)); + } + if let Some(f) = methods.subscript { + self.subscript.store(Some(f)); + } + if let Some(f) = methods.ass_subscript { + self.ass_subscript.store(Some(f)); + } + } +} + +#[allow(clippy::type_complexity)] +#[derive(Default)] +pub struct PyMappingMethods { + pub length: Option<fn(PyMapping<'_>, &VirtualMachine) -> PyResult<usize>>, + pub subscript: Option<fn(PyMapping<'_>, &PyObject, &VirtualMachine) -> PyResult>, + pub ass_subscript: + Option<fn(PyMapping<'_>, &PyObject, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>>, +} + +impl core::fmt::Debug for PyMappingMethods { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PyMappingMethods") + } +} + +impl PyMappingMethods { + pub const NOT_IMPLEMENTED: Self = Self { + length: None, + subscript: None, + ass_subscript: None, + }; +} + +impl PyObject { + pub fn mapping_unchecked(&self) -> PyMapping<'_> { + PyMapping { obj: self } + } + + pub fn try_mapping(&self, vm: &VirtualMachine) -> PyResult<PyMapping<'_>> { + let mapping = self.mapping_unchecked(); + if mapping.check() { + Ok(mapping) + } else { + Err(vm.new_type_error(format!("{} is not a mapping object", self.class()))) + } + } +} + +#[derive(Copy, Clone)] +pub struct PyMapping<'a> { + pub obj: &'a PyObject, +} + +unsafe impl Traverse for PyMapping<'_> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.obj.traverse(tracer_fn) + } +} + +impl AsRef<PyObject> for PyMapping<'_> { + #[inline(always)] + fn as_ref(&self) -> &PyObject { + self.obj + } +} + +impl PyMapping<'_> { + #[inline] + pub fn slots(&self) -> &PyMappingSlots { + &self.obj.class().slots.as_mapping + } + + #[inline] + pub fn check(&self) -> bool { + self.slots().has_subscript() + } + + pub fn length_opt(self, vm: &VirtualMachine) -> Option<PyResult<usize>> { + self.slots().length.load().map(|f| f(self, vm)) + } + + pub fn length(self, vm: &VirtualMachine) -> PyResult<usize> { + self.length_opt(vm).ok_or_else(|| { + vm.new_type_error(format!( + "object of type '{}' has no len() or not a mapping", + self.obj.class() + )) + })? + } + + pub fn subscript(self, needle: &impl AsObject, vm: &VirtualMachine) -> PyResult { + self._subscript(needle.as_object(), vm) + } + + pub fn ass_subscript( + self, + needle: &impl AsObject, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + self._ass_subscript(needle.as_object(), value, vm) + } + + fn _subscript(self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { + let f = + self.slots().subscript.load().ok_or_else(|| { + vm.new_type_error(format!("{} is not a mapping", self.obj.class())) + })?; + f(self, needle, vm) + } + + fn _ass_subscript( + self, + needle: &PyObject, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let f = self.slots().ass_subscript.load().ok_or_else(|| { + vm.new_type_error(format!( + "'{}' object does not support item assignment", + self.obj.class() + )) + })?; + f(self, needle, value, vm) + } + + pub fn keys(self, vm: &VirtualMachine) -> PyResult { + if let Some(dict) = self.obj.downcast_ref_if_exact::<PyDict>(vm) { + PyDictKeys::new(dict.to_owned()).to_pyresult(vm) + } else { + self.method_output_as_list(identifier!(vm, keys), vm) + } + } + + pub fn values(self, vm: &VirtualMachine) -> PyResult { + if let Some(dict) = self.obj.downcast_ref_if_exact::<PyDict>(vm) { + PyDictValues::new(dict.to_owned()).to_pyresult(vm) + } else { + self.method_output_as_list(identifier!(vm, values), vm) + } + } + + pub fn items(self, vm: &VirtualMachine) -> PyResult { + if let Some(dict) = self.obj.downcast_ref_if_exact::<PyDict>(vm) { + PyDictItems::new(dict.to_owned()).to_pyresult(vm) + } else { + self.method_output_as_list(identifier!(vm, items), vm) + } + } + + fn method_output_as_list( + self, + method_name: &'static PyStrInterned, + vm: &VirtualMachine, + ) -> PyResult { + let meth_output = vm.call_method(self.obj, method_name.as_str(), ())?; + if meth_output.is(vm.ctx.types.list_type) { + return Ok(meth_output); + } + + let iter = meth_output.get_iter(vm).map_err(|_| { + vm.new_type_error(format!( + "{}.{}() returned a non-iterable (type {})", + self.obj.class(), + method_name.as_str(), + meth_output.class() + )) + })?; + + // TODO + // PySequence::from(&iter).list(vm).map(|x| x.into()) + vm.ctx.new_list(iter.try_to_value(vm)?).to_pyresult(vm) + } +} diff --git a/crates/vm/src/protocol/mod.rs b/crates/vm/src/protocol/mod.rs new file mode 100644 index 00000000000..411aa4dfad3 --- /dev/null +++ b/crates/vm/src/protocol/mod.rs @@ -0,0 +1,18 @@ +mod buffer; +mod callable; +mod iter; +mod mapping; +mod number; +mod object; +mod sequence; + +pub use buffer::{BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, VecBuffer}; +pub use callable::PyCallable; +pub(crate) use callable::TraceEvent; +pub use iter::{PyIter, PyIterIter, PyIterReturn}; +pub use mapping::{PyMapping, PyMappingMethods, PyMappingSlots}; +pub use number::{ + PyNumber, PyNumberBinaryFunc, PyNumberBinaryOp, PyNumberMethods, PyNumberSlots, + PyNumberTernaryFunc, PyNumberTernaryOp, PyNumberUnaryFunc, handle_bytes_to_int_err, +}; +pub use sequence::{PySequence, PySequenceMethods, PySequenceSlots}; diff --git a/crates/vm/src/protocol/number.rs b/crates/vm/src/protocol/number.rs new file mode 100644 index 00000000000..36dbd5b8843 --- /dev/null +++ b/crates/vm/src/protocol/number.rs @@ -0,0 +1,701 @@ +use core::ops::Deref; + +use crossbeam_utils::atomic::AtomicCell; + +use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, + VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyByteArray, PyBytes, PyComplex, PyFloat, PyInt, PyIntRef, PyStr, int, + }, + common::int::{BytesToIntError, bytes_to_int}, + function::ArgBytesLike, + object::{Traverse, TraverseFn}, + stdlib::_warnings, +}; + +pub type PyNumberUnaryFunc<R = PyObjectRef> = fn(PyNumber<'_>, &VirtualMachine) -> PyResult<R>; +pub type PyNumberBinaryFunc = fn(&PyObject, &PyObject, &VirtualMachine) -> PyResult; +pub type PyNumberTernaryFunc = fn(&PyObject, &PyObject, &PyObject, &VirtualMachine) -> PyResult; + +impl PyObject { + #[inline] + pub const fn number(&self) -> PyNumber<'_> { + PyNumber { obj: self } + } + + pub fn try_index_opt(&self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { + if let Some(i) = self.downcast_ref_if_exact::<PyInt>(vm) { + Some(Ok(i.to_owned())) + } else if let Some(i) = self.downcast_ref::<PyInt>() { + Some(Ok(vm.ctx.new_bigint(i.as_bigint()))) + } else { + self.number().index(vm) + } + } + + #[inline] + pub fn try_index(&self, vm: &VirtualMachine) -> PyResult<PyIntRef> { + self.try_index_opt(vm).transpose()?.ok_or_else(|| { + vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + self.class() + )) + }) + } + + pub fn try_int(&self, vm: &VirtualMachine) -> PyResult<PyIntRef> { + fn try_convert(obj: &PyObject, lit: &[u8], vm: &VirtualMachine) -> PyResult<PyIntRef> { + let base = 10; + let digit_limit = vm.state.int_max_str_digits.load(); + + let i = bytes_to_int(lit, base, digit_limit) + .map_err(|e| handle_bytes_to_int_err(e, obj, vm))?; + Ok(PyInt::from(i).into_ref(&vm.ctx)) + } + + if let Some(i) = self.downcast_ref_if_exact::<PyInt>(vm) { + Ok(i.to_owned()) + } else if let Some(i) = self.number().int(vm).or_else(|| self.try_index_opt(vm)) { + i + } else if let Ok(Some(f)) = vm.get_special_method(self, identifier!(vm, __trunc__)) { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + "The delegation of int() to __trunc__ is deprecated.".to_owned(), + 1, + vm, + )?; + let ret = f.invoke((), vm)?; + ret.try_index(vm).map_err(|_| { + vm.new_type_error(format!( + "__trunc__ returned non-Integral (type {})", + ret.class() + )) + }) + } else if let Some(s) = self.downcast_ref::<PyStr>() { + try_convert(self, s.as_wtf8().trim().as_bytes(), vm) + } else if let Some(bytes) = self.downcast_ref::<PyBytes>() { + try_convert(self, bytes, vm) + } else if let Some(bytearray) = self.downcast_ref::<PyByteArray>() { + try_convert(self, &bytearray.borrow_buf(), vm) + } else if let Ok(buffer) = ArgBytesLike::try_from_borrowed_object(vm, self) { + // TODO: replace to PyBuffer + try_convert(self, &buffer.borrow_buf(), vm) + } else { + Err(vm.new_type_error(format!( + "int() argument must be a string, a bytes-like object or a real number, not '{}'", + self.class() + ))) + } + } + + pub fn try_float_opt(&self, vm: &VirtualMachine) -> Option<PyResult<PyRef<PyFloat>>> { + if let Some(float) = self.downcast_ref_if_exact::<PyFloat>(vm) { + Some(Ok(float.to_owned())) + } else if let Some(f) = self.number().float(vm) { + Some(f) + } else { + self.try_index_opt(vm) + .map(|i| Ok(vm.ctx.new_float(int::try_to_float(i?.as_bigint(), vm)?))) + } + } + + #[inline] + pub fn try_float(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyFloat>> { + self.try_float_opt(vm).ok_or_else(|| { + vm.new_type_error(format!("must be real number, not {}", self.class())) + })? + } +} + +#[derive(Default)] +pub struct PyNumberMethods { + /* Number implementations must check *both* + arguments for proper type and implement the necessary conversions + in the slot functions themselves. */ + pub add: Option<PyNumberBinaryFunc>, + pub subtract: Option<PyNumberBinaryFunc>, + pub multiply: Option<PyNumberBinaryFunc>, + pub remainder: Option<PyNumberBinaryFunc>, + pub divmod: Option<PyNumberBinaryFunc>, + pub power: Option<PyNumberTernaryFunc>, + pub negative: Option<PyNumberUnaryFunc>, + pub positive: Option<PyNumberUnaryFunc>, + pub absolute: Option<PyNumberUnaryFunc>, + pub boolean: Option<PyNumberUnaryFunc<bool>>, + pub invert: Option<PyNumberUnaryFunc>, + pub lshift: Option<PyNumberBinaryFunc>, + pub rshift: Option<PyNumberBinaryFunc>, + pub and: Option<PyNumberBinaryFunc>, + pub xor: Option<PyNumberBinaryFunc>, + pub or: Option<PyNumberBinaryFunc>, + pub int: Option<PyNumberUnaryFunc>, + pub float: Option<PyNumberUnaryFunc>, + + pub inplace_add: Option<PyNumberBinaryFunc>, + pub inplace_subtract: Option<PyNumberBinaryFunc>, + pub inplace_multiply: Option<PyNumberBinaryFunc>, + pub inplace_remainder: Option<PyNumberBinaryFunc>, + pub inplace_power: Option<PyNumberTernaryFunc>, + pub inplace_lshift: Option<PyNumberBinaryFunc>, + pub inplace_rshift: Option<PyNumberBinaryFunc>, + pub inplace_and: Option<PyNumberBinaryFunc>, + pub inplace_xor: Option<PyNumberBinaryFunc>, + pub inplace_or: Option<PyNumberBinaryFunc>, + + pub floor_divide: Option<PyNumberBinaryFunc>, + pub true_divide: Option<PyNumberBinaryFunc>, + pub inplace_floor_divide: Option<PyNumberBinaryFunc>, + pub inplace_true_divide: Option<PyNumberBinaryFunc>, + + pub index: Option<PyNumberUnaryFunc>, + + pub matrix_multiply: Option<PyNumberBinaryFunc>, + pub inplace_matrix_multiply: Option<PyNumberBinaryFunc>, +} + +impl PyNumberMethods { + /// this is NOT a global variable + pub const NOT_IMPLEMENTED: Self = Self { + add: None, + subtract: None, + multiply: None, + remainder: None, + divmod: None, + power: None, + negative: None, + positive: None, + absolute: None, + boolean: None, + invert: None, + lshift: None, + rshift: None, + and: None, + xor: None, + or: None, + int: None, + float: None, + inplace_add: None, + inplace_subtract: None, + inplace_multiply: None, + inplace_remainder: None, + inplace_power: None, + inplace_lshift: None, + inplace_rshift: None, + inplace_and: None, + inplace_xor: None, + inplace_or: None, + floor_divide: None, + true_divide: None, + inplace_floor_divide: None, + inplace_true_divide: None, + index: None, + matrix_multiply: None, + inplace_matrix_multiply: None, + }; + + pub fn not_implemented() -> &'static Self { + static GLOBAL_NOT_IMPLEMENTED: PyNumberMethods = PyNumberMethods::NOT_IMPLEMENTED; + &GLOBAL_NOT_IMPLEMENTED + } +} + +#[derive(Copy, Clone)] +pub enum PyNumberBinaryOp { + Add, + Subtract, + Multiply, + Remainder, + Divmod, + Lshift, + Rshift, + And, + Xor, + Or, + InplaceAdd, + InplaceSubtract, + InplaceMultiply, + InplaceRemainder, + InplaceLshift, + InplaceRshift, + InplaceAnd, + InplaceXor, + InplaceOr, + FloorDivide, + TrueDivide, + InplaceFloorDivide, + InplaceTrueDivide, + MatrixMultiply, + InplaceMatrixMultiply, +} + +#[derive(Copy, Clone)] +pub enum PyNumberTernaryOp { + Power, + InplacePower, +} + +#[derive(Default)] +pub struct PyNumberSlots { + pub add: AtomicCell<Option<PyNumberBinaryFunc>>, + pub subtract: AtomicCell<Option<PyNumberBinaryFunc>>, + pub multiply: AtomicCell<Option<PyNumberBinaryFunc>>, + pub remainder: AtomicCell<Option<PyNumberBinaryFunc>>, + pub divmod: AtomicCell<Option<PyNumberBinaryFunc>>, + pub power: AtomicCell<Option<PyNumberTernaryFunc>>, + pub negative: AtomicCell<Option<PyNumberUnaryFunc>>, + pub positive: AtomicCell<Option<PyNumberUnaryFunc>>, + pub absolute: AtomicCell<Option<PyNumberUnaryFunc>>, + pub boolean: AtomicCell<Option<PyNumberUnaryFunc<bool>>>, + pub invert: AtomicCell<Option<PyNumberUnaryFunc>>, + pub lshift: AtomicCell<Option<PyNumberBinaryFunc>>, + pub rshift: AtomicCell<Option<PyNumberBinaryFunc>>, + pub and: AtomicCell<Option<PyNumberBinaryFunc>>, + pub xor: AtomicCell<Option<PyNumberBinaryFunc>>, + pub or: AtomicCell<Option<PyNumberBinaryFunc>>, + pub int: AtomicCell<Option<PyNumberUnaryFunc>>, + pub float: AtomicCell<Option<PyNumberUnaryFunc>>, + + // Right variants (internal - not exposed in SlotAccessor) + pub right_add: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_subtract: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_remainder: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_divmod: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_power: AtomicCell<Option<PyNumberTernaryFunc>>, + pub right_lshift: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_rshift: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_and: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_xor: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_or: AtomicCell<Option<PyNumberBinaryFunc>>, + + pub inplace_add: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_subtract: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_remainder: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_power: AtomicCell<Option<PyNumberTernaryFunc>>, + pub inplace_lshift: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_rshift: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_and: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_xor: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_or: AtomicCell<Option<PyNumberBinaryFunc>>, + + pub floor_divide: AtomicCell<Option<PyNumberBinaryFunc>>, + pub true_divide: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_floor_divide: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_true_divide: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_floor_divide: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_true_divide: AtomicCell<Option<PyNumberBinaryFunc>>, + + pub index: AtomicCell<Option<PyNumberUnaryFunc>>, + + pub matrix_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, + pub right_matrix_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, + pub inplace_matrix_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, +} + +impl From<&PyNumberMethods> for PyNumberSlots { + fn from(value: &PyNumberMethods) -> Self { + // right_* slots use the same function as left ops for native types + Self { + add: AtomicCell::new(value.add), + subtract: AtomicCell::new(value.subtract), + multiply: AtomicCell::new(value.multiply), + remainder: AtomicCell::new(value.remainder), + divmod: AtomicCell::new(value.divmod), + power: AtomicCell::new(value.power), + negative: AtomicCell::new(value.negative), + positive: AtomicCell::new(value.positive), + absolute: AtomicCell::new(value.absolute), + boolean: AtomicCell::new(value.boolean), + invert: AtomicCell::new(value.invert), + lshift: AtomicCell::new(value.lshift), + rshift: AtomicCell::new(value.rshift), + and: AtomicCell::new(value.and), + xor: AtomicCell::new(value.xor), + or: AtomicCell::new(value.or), + int: AtomicCell::new(value.int), + float: AtomicCell::new(value.float), + right_add: AtomicCell::new(value.add), + right_subtract: AtomicCell::new(value.subtract), + right_multiply: AtomicCell::new(value.multiply), + right_remainder: AtomicCell::new(value.remainder), + right_divmod: AtomicCell::new(value.divmod), + right_power: AtomicCell::new(value.power), + right_lshift: AtomicCell::new(value.lshift), + right_rshift: AtomicCell::new(value.rshift), + right_and: AtomicCell::new(value.and), + right_xor: AtomicCell::new(value.xor), + right_or: AtomicCell::new(value.or), + inplace_add: AtomicCell::new(value.inplace_add), + inplace_subtract: AtomicCell::new(value.inplace_subtract), + inplace_multiply: AtomicCell::new(value.inplace_multiply), + inplace_remainder: AtomicCell::new(value.inplace_remainder), + inplace_power: AtomicCell::new(value.inplace_power), + inplace_lshift: AtomicCell::new(value.inplace_lshift), + inplace_rshift: AtomicCell::new(value.inplace_rshift), + inplace_and: AtomicCell::new(value.inplace_and), + inplace_xor: AtomicCell::new(value.inplace_xor), + inplace_or: AtomicCell::new(value.inplace_or), + floor_divide: AtomicCell::new(value.floor_divide), + true_divide: AtomicCell::new(value.true_divide), + right_floor_divide: AtomicCell::new(value.floor_divide), + right_true_divide: AtomicCell::new(value.true_divide), + inplace_floor_divide: AtomicCell::new(value.inplace_floor_divide), + inplace_true_divide: AtomicCell::new(value.inplace_true_divide), + index: AtomicCell::new(value.index), + matrix_multiply: AtomicCell::new(value.matrix_multiply), + right_matrix_multiply: AtomicCell::new(value.matrix_multiply), + inplace_matrix_multiply: AtomicCell::new(value.inplace_matrix_multiply), + } + } +} + +impl PyNumberSlots { + /// Copy from static PyNumberMethods + pub fn copy_from(&self, methods: &PyNumberMethods) { + if let Some(f) = methods.add { + self.add.store(Some(f)); + self.right_add.store(Some(f)); + } + if let Some(f) = methods.subtract { + self.subtract.store(Some(f)); + self.right_subtract.store(Some(f)); + } + if let Some(f) = methods.multiply { + self.multiply.store(Some(f)); + self.right_multiply.store(Some(f)); + } + if let Some(f) = methods.remainder { + self.remainder.store(Some(f)); + self.right_remainder.store(Some(f)); + } + if let Some(f) = methods.divmod { + self.divmod.store(Some(f)); + self.right_divmod.store(Some(f)); + } + if let Some(f) = methods.power { + self.power.store(Some(f)); + self.right_power.store(Some(f)); + } + if let Some(f) = methods.negative { + self.negative.store(Some(f)); + } + if let Some(f) = methods.positive { + self.positive.store(Some(f)); + } + if let Some(f) = methods.absolute { + self.absolute.store(Some(f)); + } + if let Some(f) = methods.boolean { + self.boolean.store(Some(f)); + } + if let Some(f) = methods.invert { + self.invert.store(Some(f)); + } + if let Some(f) = methods.lshift { + self.lshift.store(Some(f)); + self.right_lshift.store(Some(f)); + } + if let Some(f) = methods.rshift { + self.rshift.store(Some(f)); + self.right_rshift.store(Some(f)); + } + if let Some(f) = methods.and { + self.and.store(Some(f)); + self.right_and.store(Some(f)); + } + if let Some(f) = methods.xor { + self.xor.store(Some(f)); + self.right_xor.store(Some(f)); + } + if let Some(f) = methods.or { + self.or.store(Some(f)); + self.right_or.store(Some(f)); + } + if let Some(f) = methods.int { + self.int.store(Some(f)); + } + if let Some(f) = methods.float { + self.float.store(Some(f)); + } + if let Some(f) = methods.inplace_add { + self.inplace_add.store(Some(f)); + } + if let Some(f) = methods.inplace_subtract { + self.inplace_subtract.store(Some(f)); + } + if let Some(f) = methods.inplace_multiply { + self.inplace_multiply.store(Some(f)); + } + if let Some(f) = methods.inplace_remainder { + self.inplace_remainder.store(Some(f)); + } + if let Some(f) = methods.inplace_power { + self.inplace_power.store(Some(f)); + } + if let Some(f) = methods.inplace_lshift { + self.inplace_lshift.store(Some(f)); + } + if let Some(f) = methods.inplace_rshift { + self.inplace_rshift.store(Some(f)); + } + if let Some(f) = methods.inplace_and { + self.inplace_and.store(Some(f)); + } + if let Some(f) = methods.inplace_xor { + self.inplace_xor.store(Some(f)); + } + if let Some(f) = methods.inplace_or { + self.inplace_or.store(Some(f)); + } + if let Some(f) = methods.floor_divide { + self.floor_divide.store(Some(f)); + self.right_floor_divide.store(Some(f)); + } + if let Some(f) = methods.true_divide { + self.true_divide.store(Some(f)); + self.right_true_divide.store(Some(f)); + } + if let Some(f) = methods.inplace_floor_divide { + self.inplace_floor_divide.store(Some(f)); + } + if let Some(f) = methods.inplace_true_divide { + self.inplace_true_divide.store(Some(f)); + } + if let Some(f) = methods.index { + self.index.store(Some(f)); + } + if let Some(f) = methods.matrix_multiply { + self.matrix_multiply.store(Some(f)); + self.right_matrix_multiply.store(Some(f)); + } + if let Some(f) = methods.inplace_matrix_multiply { + self.inplace_matrix_multiply.store(Some(f)); + } + } + + pub fn left_binary_op(&self, op_slot: PyNumberBinaryOp) -> Option<PyNumberBinaryFunc> { + use PyNumberBinaryOp::*; + match op_slot { + Add => self.add.load(), + Subtract => self.subtract.load(), + Multiply => self.multiply.load(), + Remainder => self.remainder.load(), + Divmod => self.divmod.load(), + Lshift => self.lshift.load(), + Rshift => self.rshift.load(), + And => self.and.load(), + Xor => self.xor.load(), + Or => self.or.load(), + InplaceAdd => self.inplace_add.load(), + InplaceSubtract => self.inplace_subtract.load(), + InplaceMultiply => self.inplace_multiply.load(), + InplaceRemainder => self.inplace_remainder.load(), + InplaceLshift => self.inplace_lshift.load(), + InplaceRshift => self.inplace_rshift.load(), + InplaceAnd => self.inplace_and.load(), + InplaceXor => self.inplace_xor.load(), + InplaceOr => self.inplace_or.load(), + FloorDivide => self.floor_divide.load(), + TrueDivide => self.true_divide.load(), + InplaceFloorDivide => self.inplace_floor_divide.load(), + InplaceTrueDivide => self.inplace_true_divide.load(), + MatrixMultiply => self.matrix_multiply.load(), + InplaceMatrixMultiply => self.inplace_matrix_multiply.load(), + } + } + + pub fn right_binary_op(&self, op_slot: PyNumberBinaryOp) -> Option<PyNumberBinaryFunc> { + use PyNumberBinaryOp::*; + match op_slot { + Add => self.right_add.load(), + Subtract => self.right_subtract.load(), + Multiply => self.right_multiply.load(), + Remainder => self.right_remainder.load(), + Divmod => self.right_divmod.load(), + Lshift => self.right_lshift.load(), + Rshift => self.right_rshift.load(), + And => self.right_and.load(), + Xor => self.right_xor.load(), + Or => self.right_or.load(), + FloorDivide => self.right_floor_divide.load(), + TrueDivide => self.right_true_divide.load(), + MatrixMultiply => self.right_matrix_multiply.load(), + _ => None, + } + } + + pub fn left_ternary_op(&self, op_slot: PyNumberTernaryOp) -> Option<PyNumberTernaryFunc> { + use PyNumberTernaryOp::*; + match op_slot { + Power => self.power.load(), + InplacePower => self.inplace_power.load(), + } + } + + pub fn right_ternary_op(&self, op_slot: PyNumberTernaryOp) -> Option<PyNumberTernaryFunc> { + use PyNumberTernaryOp::*; + match op_slot { + Power => self.right_power.load(), + _ => None, + } + } +} +#[derive(Copy, Clone)] +pub struct PyNumber<'a> { + pub obj: &'a PyObject, +} + +unsafe impl Traverse for PyNumber<'_> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.obj.traverse(tracer_fn) + } +} + +impl Deref for PyNumber<'_> { + type Target = PyObject; + + fn deref(&self) -> &Self::Target { + self.obj + } +} + +impl<'a> PyNumber<'a> { + // PyNumber_Check - slots are now inherited + pub fn check(obj: &PyObject) -> bool { + let methods = &obj.class().slots.as_number; + let has_number = methods.int.load().is_some() + || methods.index.load().is_some() + || methods.float.load().is_some(); + has_number || obj.downcastable::<PyComplex>() + } +} + +impl PyNumber<'_> { + // PyIndex_Check + pub fn is_index(self) -> bool { + self.class().slots.as_number.index.load().is_some() + } + + #[inline] + pub fn int(self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { + self.class().slots.as_number.int.load().map(|f| { + let ret = f(self, vm)?; + + if let Some(ret) = ret.downcast_ref_if_exact::<PyInt>(vm) { + return Ok(ret.to_owned()); + } + + let ret_class = ret.class().to_owned(); + if let Some(ret) = ret.downcast_ref::<PyInt>() { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + format!( + "__int__ returned non-int (type {ret_class}). \ + The ability to return an instance of a strict subclass of int \ + is deprecated, and may be removed in a future version of Python." + ), + 1, + vm, + )?; + + Ok(ret.to_owned()) + } else { + Err(vm.new_type_error(format!( + "{}.__int__ returned non-int(type {})", + self.class(), + ret_class + ))) + } + }) + } + + #[inline] + pub fn index(self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { + self.class().slots.as_number.index.load().map(|f| { + let ret = f(self, vm)?; + + if let Some(ret) = ret.downcast_ref_if_exact::<PyInt>(vm) { + return Ok(ret.to_owned()); + } + + let ret_class = ret.class().to_owned(); + if let Some(ret) = ret.downcast_ref::<PyInt>() { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + format!( + "__index__ returned non-int (type {ret_class}). \ + The ability to return an instance of a strict subclass of int \ + is deprecated, and may be removed in a future version of Python." + ), + 1, + vm, + )?; + + Ok(ret.to_owned()) + } else { + Err(vm.new_type_error(format!( + "{}.__index__ returned non-int(type {})", + self.class(), + ret_class + ))) + } + }) + } + + #[inline] + pub fn float(self, vm: &VirtualMachine) -> Option<PyResult<PyRef<PyFloat>>> { + self.class().slots.as_number.float.load().map(|f| { + let ret = f(self, vm)?; + + if let Some(ret) = ret.downcast_ref_if_exact::<PyFloat>(vm) { + return Ok(ret.to_owned()); + } + + let ret_class = ret.class().to_owned(); + if let Some(ret) = ret.downcast_ref::<PyFloat>() { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + format!( + "__float__ returned non-float (type {ret_class}). \ + The ability to return an instance of a strict subclass of float \ + is deprecated, and may be removed in a future version of Python." + ), + 1, + vm, + )?; + + Ok(ret.to_owned()) + } else { + Err(vm.new_type_error(format!( + "{}.__float__ returned non-float(type {})", + self.class(), + ret_class + ))) + } + }) + } +} + +pub fn handle_bytes_to_int_err( + e: BytesToIntError, + obj: &PyObject, + vm: &VirtualMachine, +) -> PyBaseExceptionRef { + match e { + BytesToIntError::InvalidLiteral { base } => vm.new_value_error(format!( + "invalid literal for int() with base {base}: {}", + match obj.repr(vm) { + Ok(v) => v, + Err(err) => return err, + }, + )), + BytesToIntError::InvalidBase => { + vm.new_value_error("int() base must be >= 2 and <= 36, or 0") + } + BytesToIntError::DigitLimit { got, limit } => vm.new_value_error(format!( +"Exceeds the limit ({limit} digits) for integer string conversion: value has {got} digits; use sys.set_int_max_str_digits() to increase the limit" + )), + } +} diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs new file mode 100644 index 00000000000..922acaa411b --- /dev/null +++ b/crates/vm/src/protocol/object.rs @@ -0,0 +1,804 @@ +//! Object Protocol +//! <https://docs.python.org/3/c-api/object.html> + +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{ + PyBytes, PyDict, PyDictRef, PyGenericAlias, PyInt, PyList, PyStr, PyTuple, PyTupleRef, + PyType, PyTypeRef, PyUtf8Str, pystr::AsPyStr, + }, + common::{hash::PyHash, str::to_ascii}, + convert::{ToPyObject, ToPyResult}, + dict_inner::DictKey, + function::{Either, FuncArgs, PyArithmeticValue, PySetterValue}, + object::PyPayload, + protocol::PyIter, + types::{Constructor, PyComparisonOp}, +}; + +// RustPython doesn't need these items +// PyObject *Py_NotImplemented +// Py_RETURN_NOTIMPLEMENTED + +impl PyObjectRef { + // int PyObject_Print(PyObject *o, FILE *fp, int flags) + + // PyObject *PyObject_GenericGetDict(PyObject *o, void *context) + // int PyObject_GenericSetDict(PyObject *o, PyObject *value, void *context) + + #[inline(always)] + pub fn rich_compare(self, other: Self, op_id: PyComparisonOp, vm: &VirtualMachine) -> PyResult { + self._cmp(&other, op_id, vm).map(|res| res.to_pyobject(vm)) + } + + pub fn bytes(self, vm: &VirtualMachine) -> PyResult { + let bytes_type = vm.ctx.types.bytes_type; + match self.downcast_exact::<PyInt>(vm) { + Ok(int) => Err(vm.new_downcast_type_error(bytes_type, &int)), + Err(obj) => { + let args = FuncArgs::from(vec![obj]); + <PyBytes as Constructor>::slot_new(bytes_type.to_owned(), args, vm) + } + } + } + + // const hash_not_implemented: fn(&PyObject, &VirtualMachine) ->PyResult<PyHash> = crate::types::Unhashable::slot_hash; + + pub fn is_true(self, vm: &VirtualMachine) -> PyResult<bool> { + self.try_to_bool(vm) + } + + pub fn not(self, vm: &VirtualMachine) -> PyResult<bool> { + self.is_true(vm).map(|x| !x) + } + + pub fn length_hint(self, defaultvalue: usize, vm: &VirtualMachine) -> PyResult<usize> { + Ok(vm.length_hint_opt(self)?.unwrap_or(defaultvalue)) + } + + // PyObject *PyObject_Dir(PyObject *o) + pub fn dir(self, vm: &VirtualMachine) -> PyResult<PyList> { + let attributes = self.class().get_attributes(); + + let dict = PyDict::from_attributes(attributes, vm)?.into_ref(&vm.ctx); + + if let Some(object_dict) = self.dict() { + vm.call_method( + dict.as_object(), + identifier!(vm, update).as_str(), + (object_dict,), + )?; + } + + let attributes: Vec<_> = dict.into_iter().map(|(k, _v)| k).collect(); + + Ok(PyList::from(attributes)) + } +} + +impl PyObject { + /// Takes an object and returns an iterator for it. + /// This is typically a new iterator but if the argument is an iterator, this + /// returns itself. + pub fn get_iter(&self, vm: &VirtualMachine) -> PyResult<PyIter> { + // PyObject_GetIter + PyIter::try_from_object(vm, self.to_owned()) + } + + // PyObject *PyObject_GetAIter(PyObject *o) + pub fn get_aiter(&self, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyCoroutine; + + // Check if object has __aiter__ method + let aiter_method = self.class().get_attr(identifier!(vm, __aiter__)); + let Some(_aiter_method) = aiter_method else { + return Err(vm.new_type_error(format!( + "'{}' object is not an async iterable", + self.class().name() + ))); + }; + + // Call __aiter__ + let iterator = vm.call_special_method(self, identifier!(vm, __aiter__), ())?; + + // Check that __aiter__ did not return a coroutine + if iterator.downcast_ref::<PyCoroutine>().is_some() { + return Err(vm.new_type_error( + "'async_iterator' object cannot be interpreted as an async iterable; \ + perhaps you forgot to call aiter()?", + )); + } + + // Check that the result is an async iterator (has __anext__) + if !iterator.class().has_attr(identifier!(vm, __anext__)) { + return Err(vm.new_type_error(format!( + "'{}' object is not an async iterator", + iterator.class().name() + ))); + } + + Ok(iterator) + } + + pub fn has_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult<bool> { + self.get_attr(attr_name, vm).map(|o| !vm.is_none(&o)) + } + + /// Get an attribute by name. + /// `attr_name` can be a `&str`, `String`, or `PyStrRef`. + pub fn get_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult { + let attr_name = attr_name.as_pystr(&vm.ctx); + self.get_attr_inner(attr_name, vm) + } + + // get_attribute should be used for full attribute access (usually from user code). + #[cfg_attr(feature = "flame-it", flame("PyObjectRef"))] + #[inline] + pub(crate) fn get_attr_inner(&self, attr_name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + vm_trace!("object.__getattribute__: {:?} {:?}", self, attr_name); + let getattro = self.class().slots.getattro.load().unwrap(); + getattro(self, attr_name, vm).inspect_err(|exc| { + vm.set_attribute_error_context(exc, self.to_owned(), attr_name.to_owned()); + }) + } + + pub fn call_set_attr( + &self, + vm: &VirtualMachine, + attr_name: &Py<PyStr>, + attr_value: PySetterValue, + ) -> PyResult<()> { + let setattro = { + let cls = self.class(); + cls.slots.setattro.load().ok_or_else(|| { + let has_getattr = cls.slots.getattro.load().is_some(); + vm.new_type_error(format!( + "'{}' object has {} attributes ({} {})", + cls.name(), + if has_getattr { "only read-only" } else { "no" }, + if attr_value.is_assign() { + "assign to" + } else { + "del" + }, + attr_name + )) + })? + }; + setattro(self, attr_name, attr_value, vm) + } + + pub fn set_attr<'a>( + &self, + attr_name: impl AsPyStr<'a>, + attr_value: impl Into<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let attr_name = attr_name.as_pystr(&vm.ctx); + let attr_value = attr_value.into(); + self.call_set_attr(vm, attr_name, PySetterValue::Assign(attr_value)) + } + + // int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) + #[cfg_attr(feature = "flame-it", flame)] + pub fn generic_setattr( + &self, + attr_name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + vm_trace!("object.__setattr__({:?}, {}, {:?})", self, attr_name, value); + if let Some(attr) = vm + .ctx + .interned_str(attr_name) + .and_then(|attr_name| self.get_class_attr(attr_name)) + { + let descr_set = attr.class().slots.descr_set.load(); + if let Some(descriptor) = descr_set { + return descriptor(&attr, self.to_owned(), value, vm); + } + } + + if let Some(dict) = self.dict() { + if let PySetterValue::Assign(value) = value { + dict.set_item(attr_name, value, vm)?; + } else { + dict.del_item(attr_name, vm).map_err(|e| { + if e.fast_isinstance(vm.ctx.exceptions.key_error) { + vm.new_no_attribute_error(self.to_owned(), attr_name.to_owned()) + } else { + e + } + })?; + } + Ok(()) + } else { + Err(vm.new_no_attribute_error(self.to_owned(), attr_name.to_owned())) + } + } + + pub fn generic_getattr(&self, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + self.generic_getattr_opt(name, None, vm)? + .ok_or_else(|| vm.new_no_attribute_error(self.to_owned(), name.to_owned())) + } + + /// CPython _PyObject_GenericGetAttrWithDict + pub fn generic_getattr_opt( + &self, + name_str: &Py<PyStr>, + dict: Option<PyDictRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + let name = name_str.as_wtf8(); + let obj_cls = self.class(); + let cls_attr_name = vm.ctx.interned_str(name_str); + let cls_attr = match cls_attr_name.and_then(|name| obj_cls.get_attr(name)) { + Some(descr) => { + let descr_cls = descr.class(); + let descr_get = descr_cls.slots.descr_get.load(); + if let Some(descr_get) = descr_get + && descr_cls.slots.descr_set.load().is_some() + { + let cls = obj_cls.to_owned().into(); + return descr_get(descr, Some(self.to_owned()), Some(cls), vm).map(Some); + } + Some((descr, descr_get)) + } + None => None, + }; + + let dict = dict.or_else(|| self.dict()); + + let attr = if let Some(dict) = dict { + dict.get_item_opt(name, vm)? + } else { + None + }; + + if let Some(obj_attr) = attr { + Ok(Some(obj_attr)) + } else if let Some((attr, descr_get)) = cls_attr { + match descr_get { + Some(descr_get) => { + let cls = obj_cls.to_owned().into(); + descr_get(attr, Some(self.to_owned()), Some(cls), vm).map(Some) + } + None => Ok(Some(attr)), + } + } else { + Ok(None) + } + } + + pub fn del_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult<()> { + let attr_name = attr_name.as_pystr(&vm.ctx); + self.call_set_attr(vm, attr_name, PySetterValue::Delete) + } + + // Perform a comparison, raising TypeError when the requested comparison + // operator is not supported. + // see: PyObject_RichCompare / do_richcompare + #[inline] // called by ExecutingFrame::execute_compare with const op + fn _cmp( + &self, + other: &Self, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<Either<PyObjectRef, bool>> { + // Single recursion guard for the entire comparison + // (do_richcompare in Objects/object.c). + vm.with_recursion("in comparison", || self._cmp_inner(other, op, vm)) + } + + fn _cmp_inner( + &self, + other: &Self, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<Either<PyObjectRef, bool>> { + let swapped = op.swapped(); + let call_cmp = |obj: &Self, other: &Self, op| { + let cmp = obj.class().slots.richcompare.load().unwrap(); + let r = match cmp(obj, other, op, vm)? { + Either::A(obj) => PyArithmeticValue::from_object(vm, obj).map(Either::A), + Either::B(arithmetic) => arithmetic.map(Either::B), + }; + Ok(r) + }; + + let mut checked_reverse_op = false; + let is_strict_subclass = { + let self_class = self.class(); + let other_class = other.class(); + !self_class.is(other_class) && other_class.fast_issubclass(self_class) + }; + if is_strict_subclass { + let res = call_cmp(other, self, swapped)?; + checked_reverse_op = true; + if let PyArithmeticValue::Implemented(x) = res { + return Ok(x); + } + } + if let PyArithmeticValue::Implemented(x) = call_cmp(self, other, op)? { + return Ok(x); + } + if !checked_reverse_op { + let res = call_cmp(other, self, swapped)?; + if let PyArithmeticValue::Implemented(x) = res { + return Ok(x); + } + } + match op { + PyComparisonOp::Eq => Ok(Either::B(self.is(&other))), + PyComparisonOp::Ne => Ok(Either::B(!self.is(&other))), + _ => Err(vm.new_type_error(format!( + "'{}' not supported between instances of '{}' and '{}'", + op.operator_token(), + self.class().name(), + other.class().name() + ))), + } + } + #[inline(always)] + pub fn rich_compare_bool( + &self, + other: &Self, + op_id: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<bool> { + match self._cmp(other, op_id, vm)? { + Either::A(obj) => obj.try_to_bool(vm), + Either::B(other) => Ok(other), + } + } + + pub fn repr_utf8(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyUtf8Str>> { + self.repr(vm)?.try_into_utf8(vm) + } + + pub fn repr(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + vm.with_recursion("while getting the repr of an object", || { + self.class().slots.repr.load().map_or_else( + || { + Err(vm.new_runtime_error(format!( + "BUG: object of type '{}' has no __repr__ method. This is a bug in RustPython.", + self.class().name() + ))) + }, + |repr| repr(self, vm), + ) + }) + } + + pub fn ascii(&self, vm: &VirtualMachine) -> PyResult<ascii::AsciiString> { + let repr = self.repr(vm)?; + let ascii = to_ascii(repr.as_wtf8()); + Ok(ascii) + } + + pub fn str_utf8(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyUtf8Str>> { + self.str(vm)?.try_into_utf8(vm) + } + pub fn str(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let obj = match self.to_owned().downcast_exact::<PyStr>(vm) { + Ok(s) => return Ok(s.into_pyref()), + Err(obj) => obj, + }; + // Fast path for exact int: skip __str__ method resolution + let obj = match obj.downcast_exact::<PyInt>(vm) { + Ok(int) => { + return Ok(vm.ctx.new_str(int.to_str_radix_10())); + } + Err(obj) => obj, + }; + // TODO: replace to obj.class().slots.str + let str_method = match vm.get_special_method(&obj, identifier!(vm, __str__))? { + Some(str_method) => str_method, + None => return obj.repr(vm), + }; + let s = str_method.invoke((), vm)?; + s.downcast::<PyStr>().map_err(|obj| { + vm.new_type_error(format!( + "__str__ returned non-string (type {})", + obj.class().name() + )) + }) + } + + // Equivalent to CPython's check_class. Returns Ok(()) if cls is a valid class, + // Err with TypeError if not. Uses abstract_get_bases internally. + fn check_class<F>(&self, vm: &VirtualMachine, msg: F) -> PyResult<()> + where + F: Fn() -> String, + { + let cls = self; + match cls.abstract_get_bases(vm)? { + Some(_bases) => Ok(()), // Has __bases__, it's a valid class + None => { + // No __bases__ or __bases__ is not a tuple + Err(vm.new_type_error(msg())) + } + } + } + + /// abstract_get_bases() has logically 4 return states: + /// 1. getattr(cls, '__bases__') could raise an AttributeError + /// 2. getattr(cls, '__bases__') could raise some other exception + /// 3. getattr(cls, '__bases__') could return a tuple + /// 4. getattr(cls, '__bases__') could return something other than a tuple + /// + /// Only state #3 returns Some(tuple). AttributeErrors are masked by returning None. + /// If an object other than a tuple comes out of __bases__, then again, None is returned. + /// Other exceptions are propagated. + fn abstract_get_bases(&self, vm: &VirtualMachine) -> PyResult<Option<PyTupleRef>> { + match vm.get_attribute_opt(self.to_owned(), identifier!(vm, __bases__))? { + Some(bases) => { + // Check if it's a tuple + match PyTupleRef::try_from_object(vm, bases) { + Ok(tuple) => Ok(Some(tuple)), + Err(_) => Ok(None), // Not a tuple, return None + } + } + None => Ok(None), // AttributeError was masked + } + } + + fn abstract_issubclass(&self, cls: &Self, vm: &VirtualMachine) -> PyResult<bool> { + // Store the current derived class to check + let mut bases: PyTupleRef; + let mut derived = self; + + // First loop: handle single inheritance without recursion + let bases = loop { + if derived.is(cls) { + return Ok(true); + } + + let Some(derived_bases) = derived.abstract_get_bases(vm)? else { + return Ok(false); + }; + + let n = derived_bases.len(); + match n { + 0 => return Ok(false), + 1 => { + // Avoid recursion in the single inheritance case + // Get the next derived class and continue the loop + bases = derived_bases; + derived = &bases.as_slice()[0]; + continue; + } + _ => { + // Multiple inheritance - handle recursively + break derived_bases; + } + } + }; + + let n = bases.len(); + // At this point we know n >= 2 + debug_assert!(n >= 2); + + for i in 0..n { + let result = vm.with_recursion("in __issubclass__", || { + bases.as_slice()[i].abstract_issubclass(cls, vm) + })?; + if result { + return Ok(true); + } + } + + Ok(false) + } + + fn recursive_issubclass(&self, cls: &Self, vm: &VirtualMachine) -> PyResult<bool> { + // Fast path for both being types (matches CPython's PyType_Check) + if let Some(cls) = PyType::check(cls) + && let Some(derived) = PyType::check(self) + { + // PyType_IsSubtype equivalent + return Ok(derived.is_subtype(cls)); + } + // Check if derived is a class + self.check_class(vm, || { + format!("issubclass() arg 1 must be a class, not {}", self.class()) + })?; + + // Check if cls is a class, tuple, or union (matches CPython's order and message) + if !cls.class().is(vm.ctx.types.union_type) { + cls.check_class(vm, || { + format!( + "issubclass() arg 2 must be a class, a tuple of classes, or a union, not {}", + cls.class() + ) + })?; + } + + self.abstract_issubclass(cls, vm) + } + + /// Real issubclass check without going through __subclasscheck__ + /// This is equivalent to CPython's _PyObject_RealIsSubclass which just calls recursive_issubclass + pub fn real_is_subclass(&self, cls: &Self, vm: &VirtualMachine) -> PyResult<bool> { + self.recursive_issubclass(cls, vm) + } + + /// Determines if `self` is a subclass of `cls`, either directly, indirectly or virtually + /// via the __subclasscheck__ magic method. + /// PyObject_IsSubclass/object_issubclass + pub fn is_subclass(&self, cls: &Self, vm: &VirtualMachine) -> PyResult<bool> { + let derived = self; + // PyType_CheckExact(cls) + if cls.class().is(vm.ctx.types.type_type) { + if derived.is(cls) { + return Ok(true); + } + return derived.recursive_issubclass(cls, vm); + } + + // Check for Union type - CPython handles this before tuple + let cls = if cls.class().is(vm.ctx.types.union_type) { + // Get the __args__ attribute which contains the union members + // Match CPython's _Py_union_args which directly accesses the args field + let union = cls + .downcast_ref::<crate::builtins::PyUnion>() + .expect("union is already checked"); + union.args().as_object() + } else { + cls + }; + + // Check if cls is a tuple + if let Some(tuple) = cls.downcast_ref::<PyTuple>() { + for item in tuple { + if vm.with_recursion("in __subclasscheck__", || derived.is_subclass(item, vm))? { + return Ok(true); + } + } + return Ok(false); + } + + // Check for __subclasscheck__ method using lookup_special (matches CPython) + if let Some(checker) = cls.lookup_special(identifier!(vm, __subclasscheck__), vm) { + let res = vm.with_recursion("in __subclasscheck__", || { + checker.call((derived.to_owned(),), vm) + })?; + return res.try_to_bool(vm); + } + + derived.recursive_issubclass(cls, vm) + } + + // _PyObject_RealIsInstance + pub(crate) fn real_is_instance(&self, cls: &Self, vm: &VirtualMachine) -> PyResult<bool> { + self.object_isinstance(cls, vm) + } + + /// Real isinstance check without going through __instancecheck__ + /// This is equivalent to CPython's _PyObject_RealIsInstance/object_isinstance + fn object_isinstance(&self, cls: &Self, vm: &VirtualMachine) -> PyResult<bool> { + if let Ok(cls) = cls.try_to_ref::<PyType>(vm) { + // PyType_Check(cls) - cls is a type object + let mut retval = self.class().is_subtype(cls); + if !retval + && let Some(i_cls) = + vm.get_attribute_opt(self.to_owned(), identifier!(vm, __class__))? + && let Ok(i_cls_type) = PyTypeRef::try_from_object(vm, i_cls) + && !i_cls_type.is(self.class()) + { + retval = i_cls_type.is_subtype(cls); + } + Ok(retval) + } else { + // Not a type object, check if it's a valid class + cls.check_class(vm, || { + format!( + "isinstance() arg 2 must be a type, a tuple of types, or a union, not {}", + cls.class() + ) + })?; + + if let Some(i_cls) = + vm.get_attribute_opt(self.to_owned(), identifier!(vm, __class__))? + { + i_cls.abstract_issubclass(cls, vm) + } else { + Ok(false) + } + } + } + + /// Determines if `self` is an instance of `cls`, either directly, indirectly or virtually via + /// the __instancecheck__ magic method. + pub fn is_instance(&self, cls: &Self, vm: &VirtualMachine) -> PyResult<bool> { + self.object_recursive_isinstance(cls, vm) + } + + // This is object_recursive_isinstance from CPython's Objects/abstract.c + fn object_recursive_isinstance(&self, cls: &Self, vm: &VirtualMachine) -> PyResult<bool> { + // PyObject_TypeCheck(inst, (PyTypeObject *)cls) + // This is an exact check of the type + if self.class().is(cls) { + return Ok(true); + } + + // PyType_CheckExact(cls) optimization + if cls.class().is(vm.ctx.types.type_type) { + // When cls is exactly a type (not a subclass), use object_isinstance + // to avoid going through __instancecheck__ (matches CPython behavior) + return self.object_isinstance(cls, vm); + } + + // Check for Union type (e.g., int | str) - CPython checks this before tuple + let cls = if cls.class().is(vm.ctx.types.union_type) { + // Match CPython's _Py_union_args which directly accesses the args field + let union = cls + .try_to_ref::<crate::builtins::PyUnion>(vm) + .expect("checked by is"); + union.args().as_object() + } else { + cls + }; + + // Check if cls is a tuple + if let Some(tuple) = cls.downcast_ref::<PyTuple>() { + for item in tuple { + if vm.with_recursion("in __instancecheck__", || { + self.object_recursive_isinstance(item, vm) + })? { + return Ok(true); + } + } + return Ok(false); + } + + // Check for __instancecheck__ method using lookup_special (matches CPython) + if let Some(checker) = cls.lookup_special(identifier!(vm, __instancecheck__), vm) { + let res = vm.with_recursion("in __instancecheck__", || { + checker.call((self.to_owned(),), vm) + })?; + return res.try_to_bool(vm); + } + + // Fall back to object_isinstance (without going through __instancecheck__ again) + self.object_isinstance(cls, vm) + } + + pub fn hash(&self, vm: &VirtualMachine) -> PyResult<PyHash> { + if let Some(hash) = self.class().slots.hash.load() { + return hash(self, vm); + } + + Err(vm.new_exception_msg( + vm.ctx.exceptions.type_error.to_owned(), + format!("unhashable type: '{}'", self.class().name()).into(), + )) + } + + // type protocol + // PyObject *PyObject_Type(PyObject *o) + pub fn obj_type(&self) -> PyObjectRef { + self.class().to_owned().into() + } + + // int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) + pub fn type_check(&self, typ: &Py<PyType>) -> bool { + self.fast_isinstance(typ) + } + + pub fn length_opt(&self, vm: &VirtualMachine) -> Option<PyResult<usize>> { + self.sequence_unchecked() + .length_opt(vm) + .or_else(|| self.mapping_unchecked().length_opt(vm)) + } + + pub fn length(&self, vm: &VirtualMachine) -> PyResult<usize> { + self.length_opt(vm).ok_or_else(|| { + vm.new_type_error(format!( + "object of type '{}' has no len()", + self.class().name() + )) + })? + } + + pub fn get_item<K: DictKey + ?Sized>(&self, needle: &K, vm: &VirtualMachine) -> PyResult { + if let Some(dict) = self.downcast_ref_if_exact::<PyDict>(vm) { + return dict.get_item(needle, vm); + } + + let needle = needle.to_pyobject(vm); + + if let Ok(mapping) = self.try_mapping(vm) { + mapping.subscript(&needle, vm) + } else if let Ok(seq) = self.try_sequence(vm) { + let i = needle.key_as_isize(vm)?; + seq.get_item(i, vm) + } else { + if self.class().fast_issubclass(vm.ctx.types.type_type) { + if self.is(vm.ctx.types.type_type) { + return PyGenericAlias::from_args(self.class().to_owned(), needle, vm) + .to_pyresult(vm); + } + + if let Some(class_getitem) = + vm.get_attribute_opt(self.to_owned(), identifier!(vm, __class_getitem__))? + && !vm.is_none(&class_getitem) + { + return class_getitem.call((needle,), vm); + } + return Err(vm.new_type_error(format!( + "type '{}' is not subscriptable", + self.downcast_ref::<PyType>().unwrap().name() + ))); + } + Err(vm.new_type_error(format!("'{}' object is not subscriptable", self.class()))) + } + } + + pub fn set_item<K: DictKey + ?Sized>( + &self, + needle: &K, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let Some(dict) = self.downcast_ref_if_exact::<PyDict>(vm) { + return dict.set_item(needle, value, vm); + } + + let mapping = self.mapping_unchecked(); + if let Some(f) = mapping.slots().ass_subscript.load() { + let needle = needle.to_pyobject(vm); + return f(mapping, &needle, Some(value), vm); + } + + let seq = self.sequence_unchecked(); + if let Some(f) = seq.slots().ass_item.load() { + let i = needle.key_as_isize(vm)?; + return f(seq, i, Some(value), vm); + } + + Err(vm.new_type_error(format!( + "'{}' does not support item assignment", + self.class() + ))) + } + + pub fn del_item<K: DictKey + ?Sized>(&self, needle: &K, vm: &VirtualMachine) -> PyResult<()> { + if let Some(dict) = self.downcast_ref_if_exact::<PyDict>(vm) { + return dict.del_item(needle, vm); + } + + let mapping = self.mapping_unchecked(); + if let Some(f) = mapping.slots().ass_subscript.load() { + let needle = needle.to_pyobject(vm); + return f(mapping, &needle, None, vm); + } + let seq = self.sequence_unchecked(); + if let Some(f) = seq.slots().ass_item.load() { + let i = needle.key_as_isize(vm)?; + return f(seq, i, None, vm); + } + + Err(vm.new_type_error(format!("'{}' does not support item deletion", self.class()))) + } + + /// Equivalent to CPython's _PyObject_LookupSpecial + /// Looks up a special method in the type's MRO without checking instance dict. + /// Returns None if not found (masking AttributeError like CPython). + pub fn lookup_special(&self, attr: &Py<PyStr>, vm: &VirtualMachine) -> Option<PyObjectRef> { + let obj_cls = self.class(); + + // Use PyType::lookup_ref (equivalent to CPython's _PyType_LookupRef) + let res = obj_cls.lookup_ref(attr, vm)?; + + // If it's a descriptor, call its __get__ method + let descr_get = res.class().slots.descr_get.load(); + if let Some(descr_get) = descr_get { + let obj_cls = obj_cls.to_owned().into(); + // CPython ignores exceptions in _PyObject_LookupSpecial and returns NULL + descr_get(res, Some(self.to_owned()), Some(obj_cls), vm).ok() + } else { + Some(res) + } + } +} diff --git a/crates/vm/src/protocol/sequence.rs b/crates/vm/src/protocol/sequence.rs new file mode 100644 index 00000000000..cee46a29089 --- /dev/null +++ b/crates/vm/src/protocol/sequence.rs @@ -0,0 +1,412 @@ +use crate::{ + PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, + builtins::{PyList, PyListRef, PySlice, PyTuple, PyTupleRef}, + convert::ToPyObject, + function::PyArithmeticValue, + object::{Traverse, TraverseFn}, + protocol::PyNumberBinaryOp, +}; +use crossbeam_utils::atomic::AtomicCell; +use itertools::Itertools; + +// Sequence Protocol +// https://docs.python.org/3/c-api/sequence.html + +#[allow(clippy::type_complexity)] +#[derive(Default)] +pub struct PySequenceSlots { + pub length: AtomicCell<Option<fn(PySequence<'_>, &VirtualMachine) -> PyResult<usize>>>, + pub concat: AtomicCell<Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult>>, + pub repeat: AtomicCell<Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>>, + pub item: AtomicCell<Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>>, + pub ass_item: AtomicCell< + Option<fn(PySequence<'_>, isize, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>>, + >, + pub contains: + AtomicCell<Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult<bool>>>, + pub inplace_concat: + AtomicCell<Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult>>, + pub inplace_repeat: AtomicCell<Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>>, +} + +impl core::fmt::Debug for PySequenceSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PySequenceSlots") + } +} + +impl PySequenceSlots { + pub fn has_item(&self) -> bool { + self.item.load().is_some() + } + + /// Copy from static PySequenceMethods + pub fn copy_from(&self, methods: &PySequenceMethods) { + if let Some(f) = methods.length { + self.length.store(Some(f)); + } + if let Some(f) = methods.concat { + self.concat.store(Some(f)); + } + if let Some(f) = methods.repeat { + self.repeat.store(Some(f)); + } + if let Some(f) = methods.item { + self.item.store(Some(f)); + } + if let Some(f) = methods.ass_item { + self.ass_item.store(Some(f)); + } + if let Some(f) = methods.contains { + self.contains.store(Some(f)); + } + if let Some(f) = methods.inplace_concat { + self.inplace_concat.store(Some(f)); + } + if let Some(f) = methods.inplace_repeat { + self.inplace_repeat.store(Some(f)); + } + } +} + +#[allow(clippy::type_complexity)] +#[derive(Default)] +pub struct PySequenceMethods { + pub length: Option<fn(PySequence<'_>, &VirtualMachine) -> PyResult<usize>>, + pub concat: Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult>, + pub repeat: Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>, + pub item: Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>, + pub ass_item: + Option<fn(PySequence<'_>, isize, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>>, + pub contains: Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult<bool>>, + pub inplace_concat: Option<fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult>, + pub inplace_repeat: Option<fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult>, +} + +impl core::fmt::Debug for PySequenceMethods { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PySequenceMethods") + } +} + +impl PySequenceMethods { + pub const NOT_IMPLEMENTED: Self = Self { + length: None, + concat: None, + repeat: None, + item: None, + ass_item: None, + contains: None, + inplace_concat: None, + inplace_repeat: None, + }; +} + +impl PyObject { + #[inline] + pub fn sequence_unchecked(&self) -> PySequence<'_> { + PySequence { obj: self } + } + + pub fn try_sequence(&self, vm: &VirtualMachine) -> PyResult<PySequence<'_>> { + let seq = self.sequence_unchecked(); + if seq.check() { + Ok(seq) + } else { + Err(vm.new_type_error(format!("'{}' is not a sequence", self.class()))) + } + } +} + +#[derive(Copy, Clone)] +pub struct PySequence<'a> { + pub obj: &'a PyObject, +} + +unsafe impl Traverse for PySequence<'_> { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.obj.traverse(tracer_fn) + } +} + +impl PySequence<'_> { + #[inline] + pub fn slots(&self) -> &PySequenceSlots { + &self.obj.class().slots.as_sequence + } + + pub fn check(&self) -> bool { + self.slots().has_item() + } + + pub fn length_opt(self, vm: &VirtualMachine) -> Option<PyResult<usize>> { + self.slots().length.load().map(|f| f(self, vm)) + } + + pub fn length(self, vm: &VirtualMachine) -> PyResult<usize> { + self.length_opt(vm).ok_or_else(|| { + vm.new_type_error(format!( + "'{}' is not a sequence or has no len()", + self.obj.class() + )) + })? + } + + pub fn concat(self, other: &PyObject, vm: &VirtualMachine) -> PyResult { + if let Some(f) = self.slots().concat.load() { + return f(self, other, vm); + } + + // if both arguments appear to be sequences, try fallback to __add__ + if self.check() && other.sequence_unchecked().check() { + let ret = vm.binary_op1(self.obj, other, PyNumberBinaryOp::Add)?; + if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { + return Ok(ret); + } + } + + Err(vm.new_type_error(format!( + "'{}' object can't be concatenated", + self.obj.class() + ))) + } + + pub fn repeat(self, n: isize, vm: &VirtualMachine) -> PyResult { + if let Some(f) = self.slots().repeat.load() { + return f(self, n, vm); + } + + // fallback to __mul__ + if self.check() { + let ret = vm.binary_op1(self.obj, &n.to_pyobject(vm), PyNumberBinaryOp::Multiply)?; + if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { + return Ok(ret); + } + } + + Err(vm.new_type_error(format!("'{}' object can't be repeated", self.obj.class()))) + } + + pub fn inplace_concat(self, other: &PyObject, vm: &VirtualMachine) -> PyResult { + if let Some(f) = self.slots().inplace_concat.load() { + return f(self, other, vm); + } + if let Some(f) = self.slots().concat.load() { + return f(self, other, vm); + } + + // if both arguments appear to be sequences, try fallback to __iadd__ + if self.check() && other.sequence_unchecked().check() { + let ret = vm._iadd(self.obj, other)?; + if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { + return Ok(ret); + } + } + + Err(vm.new_type_error(format!( + "'{}' object can't be concatenated", + self.obj.class() + ))) + } + + pub fn inplace_repeat(self, n: isize, vm: &VirtualMachine) -> PyResult { + if let Some(f) = self.slots().inplace_repeat.load() { + return f(self, n, vm); + } + if let Some(f) = self.slots().repeat.load() { + return f(self, n, vm); + } + + if self.check() { + let ret = vm._imul(self.obj, &n.to_pyobject(vm))?; + if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { + return Ok(ret); + } + } + + Err(vm.new_type_error(format!("'{}' object can't be repeated", self.obj.class()))) + } + + pub fn get_item(self, i: isize, vm: &VirtualMachine) -> PyResult { + if let Some(f) = self.slots().item.load() { + return f(self, i, vm); + } + Err(vm.new_type_error(format!( + "'{}' is not a sequence or does not support indexing", + self.obj.class() + ))) + } + + fn _ass_item(self, i: isize, value: Option<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { + if let Some(f) = self.slots().ass_item.load() { + return f(self, i, value, vm); + } + Err(vm.new_type_error(format!( + "'{}' is not a sequence or doesn't support item {}", + self.obj.class(), + if value.is_some() { + "assignment" + } else { + "deletion" + } + ))) + } + + pub fn set_item(self, i: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self._ass_item(i, Some(value), vm) + } + + pub fn del_item(self, i: isize, vm: &VirtualMachine) -> PyResult<()> { + self._ass_item(i, None, vm) + } + + pub fn get_slice(&self, start: isize, stop: isize, vm: &VirtualMachine) -> PyResult { + if let Ok(mapping) = self.obj.try_mapping(vm) { + let slice = PySlice { + start: Some(start.to_pyobject(vm)), + stop: stop.to_pyobject(vm), + step: None, + }; + mapping.subscript(&slice.into_pyobject(vm), vm) + } else { + Err(vm.new_type_error(format!("'{}' object is unsliceable", self.obj.class()))) + } + } + + fn _ass_slice( + &self, + start: isize, + stop: isize, + value: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mapping = self.obj.mapping_unchecked(); + if let Some(f) = mapping.slots().ass_subscript.load() { + let slice = PySlice { + start: Some(start.to_pyobject(vm)), + stop: stop.to_pyobject(vm), + step: None, + }; + f(mapping, &slice.into_pyobject(vm), value, vm) + } else { + Err(vm.new_type_error(format!( + "'{}' object doesn't support slice {}", + self.obj.class(), + if value.is_some() { + "assignment" + } else { + "deletion" + } + ))) + } + } + + pub fn set_slice( + &self, + start: isize, + stop: isize, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self._ass_slice(start, stop, Some(value), vm) + } + + pub fn del_slice(&self, start: isize, stop: isize, vm: &VirtualMachine) -> PyResult<()> { + self._ass_slice(start, stop, None, vm) + } + + pub fn tuple(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + if let Some(tuple) = self.obj.downcast_ref_if_exact::<PyTuple>(vm) { + Ok(tuple.to_owned()) + } else if let Some(list) = self.obj.downcast_ref_if_exact::<PyList>(vm) { + Ok(vm.ctx.new_tuple(list.borrow_vec().to_vec())) + } else { + let iter = self.obj.to_owned().get_iter(vm)?; + let iter = iter.iter(vm)?; + Ok(vm.ctx.new_tuple(iter.try_collect()?)) + } + } + + pub fn list(&self, vm: &VirtualMachine) -> PyResult<PyListRef> { + let list = vm.ctx.new_list(self.obj.try_to_value(vm)?); + Ok(list) + } + + pub fn count(&self, target: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { + let mut n = 0; + + let iter = self.obj.to_owned().get_iter(vm)?; + let iter = iter.iter::<PyObjectRef>(vm)?; + + for elem in iter { + let elem = elem?; + if vm.bool_eq(&elem, target)? { + if n == isize::MAX as usize { + return Err(vm.new_overflow_error("index exceeds C integer size")); + } + n += 1; + } + } + + Ok(n) + } + + pub fn index(&self, target: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { + let mut index: isize = -1; + + let iter = self.obj.to_owned().get_iter(vm)?; + let iter = iter.iter::<PyObjectRef>(vm)?; + + for elem in iter { + if index == isize::MAX { + return Err(vm.new_overflow_error("index exceeds C integer size")); + } + index += 1; + + let elem = elem?; + if vm.bool_eq(&elem, target)? { + return Ok(index as usize); + } + } + + Err(vm.new_value_error("sequence.index(x): x not in sequence")) + } + + pub fn extract<F, R>(&self, mut f: F, vm: &VirtualMachine) -> PyResult<Vec<R>> + where + F: FnMut(&PyObject) -> PyResult<R>, + { + if let Some(tuple) = self.obj.downcast_ref_if_exact::<PyTuple>(vm) { + tuple.iter().map(|x| f(x.as_ref())).collect() + } else if let Some(list) = self.obj.downcast_ref_if_exact::<PyList>(vm) { + list.borrow_vec().iter().map(|x| f(x.as_ref())).collect() + } else { + let iter = self.obj.to_owned().get_iter(vm)?; + let iter = iter.iter::<PyObjectRef>(vm)?; + let len = self.length(vm).unwrap_or(0); + let mut v = Vec::with_capacity(len); + for x in iter { + v.push(f(x?.as_ref())?); + } + v.shrink_to_fit(); + Ok(v) + } + } + + pub fn contains(self, target: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + if let Some(f) = self.slots().contains.load() { + return f(self, target, vm); + } + + let iter = self.obj.to_owned().get_iter(vm)?; + let iter = iter.iter::<PyObjectRef>(vm)?; + + for elem in iter { + let elem = elem?; + if vm.bool_eq(&elem, target)? { + return Ok(true); + } + } + Ok(false) + } +} diff --git a/crates/vm/src/py_io.rs b/crates/vm/src/py_io.rs new file mode 100644 index 00000000000..5649463b30e --- /dev/null +++ b/crates/vm/src/py_io.rs @@ -0,0 +1,102 @@ +use crate::{ + PyObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytes, PyStr}, + common::ascii, +}; +use alloc::fmt; +use core::ops; +use std::io; + +pub trait Write { + type Error; + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> Result<(), Self::Error>; +} + +#[repr(transparent)] +pub struct IoWriter<T>(pub T); + +impl<T> IoWriter<T> { + pub fn from_ref(x: &mut T) -> &mut Self { + // SAFETY: IoWriter is repr(transparent) over T + unsafe { &mut *(x as *mut T as *mut Self) } + } +} + +impl<T> ops::Deref for IoWriter<T> { + type Target = T; + fn deref(&self) -> &T { + &self.0 + } +} +impl<T> ops::DerefMut for IoWriter<T> { + fn deref_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +impl<W> Write for IoWriter<W> +where + W: io::Write, +{ + type Error = io::Error; + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> io::Result<()> { + <W as io::Write>::write_fmt(&mut self.0, args) + } +} + +impl Write for String { + type Error = fmt::Error; + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> fmt::Result { + <Self as fmt::Write>::write_fmt(self, args) + } +} + +pub struct PyWriter<'vm>(pub PyObjectRef, pub &'vm VirtualMachine); + +impl Write for PyWriter<'_> { + type Error = PyBaseExceptionRef; + fn write_fmt(&mut self, args: fmt::Arguments<'_>) -> Result<(), Self::Error> { + let PyWriter(obj, vm) = self; + vm.call_method(obj, "write", (args.to_string(),)).map(drop) + } +} + +pub fn file_readline(obj: &PyObject, size: Option<usize>, vm: &VirtualMachine) -> PyResult { + let args = size.map_or_else(Vec::new, |size| vec![vm.ctx.new_int(size).into()]); + let ret = vm.call_method(obj, "readline", args)?; + let eof_err = || { + vm.new_exception( + vm.ctx.exceptions.eof_error.to_owned(), + vec![vm.ctx.new_str(ascii!("EOF when reading a line")).into()], + ) + }; + let ret = match_class!(match ret { + s @ PyStr => { + // Use as_wtf8() to handle strings with surrogates (e.g., surrogateescape) + let s_wtf8 = s.as_wtf8(); + if s_wtf8.is_empty() { + return Err(eof_err()); + } + // '\n' is ASCII, so we can check bytes directly + if s_wtf8.as_bytes().last() == Some(&b'\n') { + let no_nl = &s_wtf8[..s_wtf8.len() - 1]; + vm.ctx.new_str(no_nl).into() + } else { + s.into() + } + } + b @ PyBytes => { + let buf = b.as_bytes(); + if buf.is_empty() { + return Err(eof_err()); + } + if buf.last() == Some(&b'\n') { + vm.ctx.new_bytes(buf[..buf.len() - 1].to_owned()).into() + } else { + b.into() + } + } + _ => return Err(vm.new_type_error("object.readline() returned non-string".to_owned())), + }); + Ok(ret) +} diff --git a/vm/src/py_serde.rs b/crates/vm/src/py_serde.rs similarity index 92% rename from vm/src/py_serde.rs rename to crates/vm/src/py_serde.rs index 10a2acc9cf6..945068113f1 100644 --- a/vm/src/py_serde.rs +++ b/crates/vm/src/py_serde.rs @@ -3,7 +3,7 @@ use num_traits::sign::Signed; use serde::de::{DeserializeSeed, Visitor}; use serde::ser::{Serialize, SerializeMap, SerializeSeq}; -use crate::builtins::{bool_, dict::PyDictRef, float, int, list::PyList, tuple::PyTuple, PyStr}; +use crate::builtins::{PyStr, bool_, dict::PyDictRef, float, int, list::PyList, tuple::PyTuple}; use crate::{AsObject, PyObject, PyObjectRef, VirtualMachine}; #[inline] @@ -22,7 +22,7 @@ where pub fn deserialize<'de, D>( vm: &'de VirtualMachine, deserializer: D, -) -> Result<<PyObjectDeserializer as DeserializeSeed>::Value, D::Error> +) -> Result<<PyObjectDeserializer<'de> as DeserializeSeed<'de>>::Value, D::Error> where D: serde::Deserializer<'de>, { @@ -41,7 +41,7 @@ impl<'s> PyObjectSerializer<'s> { PyObjectSerializer { pyobject, vm } } - fn clone_with_object(&self, pyobject: &'s PyObjectRef) -> PyObjectSerializer { + fn clone_with_object(&self, pyobject: &'s PyObjectRef) -> PyObjectSerializer<'_> { PyObjectSerializer { pyobject, vm: self.vm, @@ -49,7 +49,7 @@ impl<'s> PyObjectSerializer<'s> { } } -impl<'s> serde::Serialize for PyObjectSerializer<'s> { +impl serde::Serialize for PyObjectSerializer<'_> { fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: serde::Serializer, @@ -62,7 +62,7 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { } seq.end() }; - if let Some(s) = self.pyobject.payload::<PyStr>() { + if let Some(s) = self.pyobject.downcast_ref::<PyStr>() { serializer.serialize_str(s.as_ref()) } else if self.pyobject.fast_isinstance(self.vm.ctx.types.float_type) { serializer.serialize_f64(float::get_value(self.pyobject)) @@ -80,9 +80,9 @@ impl<'s> serde::Serialize for PyObjectSerializer<'s> { } else { serializer.serialize_i64(v.to_i64().ok_or_else(int_too_large)?) } - } else if let Some(list) = self.pyobject.payload_if_subclass::<PyList>(self.vm) { + } else if let Some(list) = self.pyobject.downcast_ref::<PyList>() { serialize_seq_elements(serializer, &list.borrow_vec()) - } else if let Some(tuple) = self.pyobject.payload_if_subclass::<PyTuple>(self.vm) { + } else if let Some(tuple) = self.pyobject.downcast_ref::<PyTuple>() { serialize_seq_elements(serializer, tuple) } else if self.pyobject.fast_isinstance(self.vm.ctx.types.dict_type) { let dict: PyDictRef = self.pyobject.to_owned().downcast().unwrap(); @@ -130,7 +130,7 @@ impl<'de> DeserializeSeed<'de> for PyObjectDeserializer<'de> { impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { type Value = PyObjectRef; - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter.write_str("a type that can deserialize in Python") } diff --git a/crates/vm/src/readline.rs b/crates/vm/src/readline.rs new file mode 100644 index 00000000000..d62d520ecbd --- /dev/null +++ b/crates/vm/src/readline.rs @@ -0,0 +1,186 @@ +//! Readline interface for REPLs +//! +//! This module provides a common interface for reading lines from the console, with support for history and completion. +//! It uses the [`rustyline`] crate on non-WASM platforms and a custom implementation on WASM platforms. + +use std::{io, path::Path}; + +type OtherError = Box<dyn core::error::Error>; +type OtherResult<T> = Result<T, OtherError>; + +pub enum ReadlineResult { + Line(String), + Eof, + Interrupt, + Io(std::io::Error), + #[cfg(unix)] + OsError(nix::Error), + Other(OtherError), +} + +#[allow(unused)] +mod basic_readline { + use super::*; + + pub trait Helper {} + impl<T> Helper for T {} + + pub struct Readline<H: Helper> { + helper: H, + } + + impl<H: Helper> Readline<H> { + pub const fn new(helper: H) -> Self { + Self { helper } + } + + pub fn load_history(&mut self, _path: &Path) -> OtherResult<()> { + Ok(()) + } + + pub fn save_history(&mut self, _path: &Path) -> OtherResult<()> { + Ok(()) + } + + pub fn add_history_entry(&mut self, _entry: &str) -> OtherResult<()> { + Ok(()) + } + + pub fn readline(&mut self, prompt: &str) -> ReadlineResult { + use std::io::prelude::*; + print!("{prompt}"); + if let Err(e) = io::stdout().flush() { + return ReadlineResult::Io(e); + } + + let next_line = io::stdin().lock().lines().next(); + match next_line { + Some(Ok(line)) => ReadlineResult::Line(line), + None => ReadlineResult::Eof, + Some(Err(e)) if e.kind() == io::ErrorKind::Interrupted => ReadlineResult::Interrupt, + Some(Err(e)) => ReadlineResult::Io(e), + } + } + } +} + +#[cfg(not(target_arch = "wasm32"))] +mod rustyline_readline { + use super::*; + + pub trait Helper: rustyline::Helper {} + impl<T: rustyline::Helper> Helper for T {} + + /// Readline: the REPL + pub struct Readline<H: Helper> { + repl: rustyline::Editor<H, rustyline::history::DefaultHistory>, + } + + #[cfg(windows)] + const EOF_CHAR: &str = "\u{001A}"; + + impl<H: Helper> Readline<H> { + pub fn new(helper: H) -> Self { + use rustyline::*; + let mut repl = Editor::with_config( + Config::builder() + .completion_type(CompletionType::List) + .tab_stop(8) + .bracketed_paste(false) // multi-line paste + .build(), + ) + .expect("failed to initialize line editor"); + repl.set_helper(Some(helper)); + + // Bind CTRL + Z to insert EOF character on Windows + #[cfg(windows)] + { + repl.bind_sequence( + KeyEvent::new('z', Modifiers::CTRL), + EventHandler::Simple(Cmd::Insert(1, EOF_CHAR.into())), + ); + } + + Self { repl } + } + + pub fn load_history(&mut self, path: &Path) -> OtherResult<()> { + self.repl.load_history(path)?; + Ok(()) + } + + pub fn save_history(&mut self, path: &Path) -> OtherResult<()> { + if !path.exists() + && let Some(parent) = path.parent() + { + std::fs::create_dir_all(parent)?; + } + self.repl.save_history(path)?; + Ok(()) + } + + pub fn add_history_entry(&mut self, entry: &str) -> OtherResult<()> { + self.repl.add_history_entry(entry)?; + Ok(()) + } + + pub fn readline(&mut self, prompt: &str) -> ReadlineResult { + use rustyline::error::ReadlineError; + loop { + break match self.repl.readline(prompt) { + Ok(line) => { + // Check for CTRL + Z on Windows + #[cfg(windows)] + { + use std::io::IsTerminal; + + let trimmed = line.trim_end_matches(&['\r', '\n'][..]); + if trimmed == EOF_CHAR && io::stdin().is_terminal() { + return ReadlineResult::Eof; + } + } + ReadlineResult::Line(line) + } + Err(ReadlineError::Interrupted) => ReadlineResult::Interrupt, + Err(ReadlineError::Eof) => ReadlineResult::Eof, + Err(ReadlineError::Io(e)) => ReadlineResult::Io(e), + Err(ReadlineError::Signal(_)) => continue, + #[cfg(unix)] + Err(ReadlineError::Errno(num)) => ReadlineResult::OsError(num), + Err(e) => ReadlineResult::Other(e.into()), + }; + } + } + } +} + +#[cfg(target_arch = "wasm32")] +use basic_readline as readline_inner; +#[cfg(not(target_arch = "wasm32"))] +use rustyline_readline as readline_inner; + +pub use readline_inner::Helper; + +pub struct Readline<H: Helper>(readline_inner::Readline<H>); + +impl<H: Helper> Readline<H> { + pub fn new(helper: H) -> Self { + Self(readline_inner::Readline::new(helper)) + } + + pub fn load_history(&mut self, path: &Path) -> OtherResult<()> { + self.0.load_history(path) + } + + pub fn save_history(&mut self, path: &Path) -> OtherResult<()> { + self.0.save_history(path) + } + + pub fn add_history_entry(&mut self, entry: &str) -> OtherResult<()> { + self.0.add_history_entry(entry) + } + + pub fn readline(&mut self, prompt: &str) -> ReadlineResult { + self.0.readline(prompt) + } +} diff --git a/vm/src/recursion.rs b/crates/vm/src/recursion.rs similarity index 96% rename from vm/src/recursion.rs rename to crates/vm/src/recursion.rs index c7b75e58817..7392cca4ded 100644 --- a/vm/src/recursion.rs +++ b/crates/vm/src/recursion.rs @@ -23,7 +23,7 @@ impl<'vm> ReprGuard<'vm> { } } -impl<'vm> Drop for ReprGuard<'vm> { +impl Drop for ReprGuard<'_> { fn drop(&mut self) { self.vm.repr_guards.borrow_mut().remove(&self.id); } diff --git a/vm/src/scope.rs b/crates/vm/src/scope.rs similarity index 89% rename from vm/src/scope.rs rename to crates/vm/src/scope.rs index 47a1e5e3ffd..5c6d92a8deb 100644 --- a/vm/src/scope.rs +++ b/crates/vm/src/scope.rs @@ -1,14 +1,14 @@ -use crate::{builtins::PyDictRef, function::ArgMapping, VirtualMachine}; -use std::fmt; +use crate::{VirtualMachine, builtins::PyDictRef, function::ArgMapping}; +use alloc::fmt; #[derive(Clone)] pub struct Scope { - pub locals: ArgMapping, + pub locals: Option<ArgMapping>, pub globals: PyDictRef, } impl fmt::Debug for Scope { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // TODO: have a more informative Debug impl that DOESN'T recurse and cause a stack overflow f.write_str("Scope") } @@ -16,25 +16,26 @@ impl fmt::Debug for Scope { impl Scope { #[inline] - pub fn new(locals: Option<ArgMapping>, globals: PyDictRef) -> Scope { - let locals = locals.unwrap_or_else(|| ArgMapping::from_dict_exact(globals.clone())); - Scope { locals, globals } + pub fn new(locals: Option<ArgMapping>, globals: PyDictRef) -> Self { + Self { locals, globals } } pub fn with_builtins( locals: Option<ArgMapping>, globals: PyDictRef, vm: &VirtualMachine, - ) -> Scope { + ) -> Self { if !globals.contains_key("__builtins__", vm) { globals .set_item("__builtins__", vm.builtins.clone().into(), vm) .unwrap(); } - Scope::new(locals, globals) + // For module-level code, locals defaults to globals + let locals = locals.or_else(|| Some(ArgMapping::from_dict_exact(globals.clone()))); + Self::new(locals, globals) } - // pub fn get_locals(&self) -> &PyDictRef { + // pub fn get_locals(&self) -> &Py<PyDict> { // match self.locals.first() { // Some(dict) => dict, // None => &self.globals, @@ -141,7 +142,7 @@ impl Scope { // impl Sealed for super::PyStrRef {} // } // pub trait PyName: -// sealed::Sealed + crate::dictdatatype::DictKey + Clone + ToPyObject +// sealed::Sealed + crate::dict_inner::DictKey + Clone + ToPyObject // { // } // impl PyName for str {} diff --git a/crates/vm/src/sequence.rs b/crates/vm/src/sequence.rs new file mode 100644 index 00000000000..0bc12fd2631 --- /dev/null +++ b/crates/vm/src/sequence.rs @@ -0,0 +1,165 @@ +use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, + builtins::PyIntRef, + function::OptionalArg, + sliceable::SequenceIndexOp, + types::PyComparisonOp, + vm::{MAX_MEMORY_SIZE, VirtualMachine}, +}; +use core::ops::{Deref, Range}; +use optional::Optioned; + +pub trait MutObjectSequenceOp { + type Inner: ?Sized; + + fn do_get(index: usize, inner: &Self::Inner) -> Option<&PyObject>; + fn do_lock(&self) -> impl Deref<Target = Self::Inner>; + + fn mut_count(&self, vm: &VirtualMachine, needle: &PyObject) -> PyResult<usize> { + let mut count = 0; + self._mut_iter_equal_skeleton::<_, false>(vm, needle, 0..isize::MAX as usize, || { + count += 1 + })?; + Ok(count) + } + + fn mut_index_range( + &self, + vm: &VirtualMachine, + needle: &PyObject, + range: Range<usize>, + ) -> PyResult<Optioned<usize>> { + self._mut_iter_equal_skeleton::<_, true>(vm, needle, range, || {}) + } + + fn mut_index(&self, vm: &VirtualMachine, needle: &PyObject) -> PyResult<Optioned<usize>> { + self.mut_index_range(vm, needle, 0..isize::MAX as usize) + } + + fn mut_contains(&self, vm: &VirtualMachine, needle: &PyObject) -> PyResult<bool> { + self.mut_index(vm, needle).map(|x| x.is_some()) + } + + fn _mut_iter_equal_skeleton<F, const SHORT: bool>( + &self, + vm: &VirtualMachine, + needle: &PyObject, + range: Range<usize>, + mut f: F, + ) -> PyResult<Optioned<usize>> + where + F: FnMut(), + { + let mut borrower = None; + let mut i = range.start; + + let index = loop { + if i >= range.end { + break Optioned::<usize>::none(); + } + let guard = if let Some(x) = borrower.take() { + x + } else { + self.do_lock() + }; + + let elem = if let Some(x) = Self::do_get(i, &guard) { + x + } else { + break Optioned::<usize>::none(); + }; + + if elem.is(needle) { + f(); + if SHORT { + break Optioned::<usize>::some(i); + } + borrower = Some(guard); + } else { + let elem = elem.to_owned(); + drop(guard); + + if elem.rich_compare_bool(needle, PyComparisonOp::Eq, vm)? { + f(); + if SHORT { + break Optioned::<usize>::some(i); + } + } + } + i += 1; + }; + + Ok(index) + } +} + +pub trait SequenceExt<T: Clone> +where + Self: AsRef<[T]>, +{ + fn mul(&self, vm: &VirtualMachine, n: isize) -> PyResult<Vec<T>> { + let n = vm.check_repeat_or_overflow_error(self.as_ref().len(), n)?; + + if n > 1 && core::mem::size_of_val(self.as_ref()) >= MAX_MEMORY_SIZE / n { + return Err(vm.new_memory_error("")); + } + + let mut v = Vec::with_capacity(n * self.as_ref().len()); + for _ in 0..n { + v.extend_from_slice(self.as_ref()); + } + Ok(v) + } +} + +impl<T: Clone> SequenceExt<T> for [T] {} + +pub trait SequenceMutExt<T: Clone> +where + Self: AsRef<[T]>, +{ + fn as_vec_mut(&mut self) -> &mut Vec<T>; + + fn imul(&mut self, vm: &VirtualMachine, n: isize) -> PyResult<()> { + let n = vm.check_repeat_or_overflow_error(self.as_ref().len(), n)?; + if n == 0 { + self.as_vec_mut().clear(); + } else if n != 1 { + let mut sample = self.as_vec_mut().clone(); + if n != 2 { + self.as_vec_mut().reserve(sample.len() * (n - 1)); + for _ in 0..n - 2 { + self.as_vec_mut().extend_from_slice(&sample); + } + } + self.as_vec_mut().append(&mut sample); + } + Ok(()) + } +} + +impl<T: Clone> SequenceMutExt<T> for Vec<T> { + fn as_vec_mut(&mut self) -> &mut Self { + self + } +} + +#[derive(FromArgs)] +pub struct OptionalRangeArgs { + #[pyarg(positional, optional)] + start: OptionalArg<PyObjectRef>, + #[pyarg(positional, optional)] + stop: OptionalArg<PyObjectRef>, +} + +impl OptionalRangeArgs { + pub fn saturate(self, len: usize, vm: &VirtualMachine) -> PyResult<(usize, usize)> { + let saturate = |obj: PyObjectRef| -> PyResult<_> { + obj.try_into_value(vm) + .map(|int: PyIntRef| int.as_bigint().saturated_at(len)) + }; + let start = self.start.map_or(Ok(0), saturate)?; + let stop = self.stop.map_or(Ok(len), saturate)?; + Ok((start, stop)) + } +} diff --git a/crates/vm/src/signal.rs b/crates/vm/src/signal.rs new file mode 100644 index 00000000000..3caf8cb8e30 --- /dev/null +++ b/crates/vm/src/signal.rs @@ -0,0 +1,188 @@ +#![cfg_attr(target_os = "wasi", allow(dead_code))] +use crate::{PyObjectRef, PyResult, VirtualMachine}; +use alloc::fmt; +use core::cell::{Cell, RefCell}; +#[cfg(windows)] +use core::sync::atomic::AtomicIsize; +use core::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; + +pub(crate) const NSIG: usize = 64; + +pub(crate) fn new_signal_handlers() -> Box<RefCell<[Option<PyObjectRef>; NSIG]>> { + Box::new(const { RefCell::new([const { None }; NSIG]) }) +} +static ANY_TRIGGERED: AtomicBool = AtomicBool::new(false); +// hack to get around const array repeat expressions, rust issue #79270 +#[allow( + clippy::declare_interior_mutable_const, + reason = "workaround for const array repeat limitation (rust issue #79270)" +)] +const ATOMIC_FALSE: AtomicBool = AtomicBool::new(false); +pub(crate) static TRIGGERS: [AtomicBool; NSIG] = [ATOMIC_FALSE; NSIG]; + +#[cfg(windows)] +static SIGINT_EVENT: AtomicIsize = AtomicIsize::new(0); + +thread_local! { + /// Prevent recursive signal handler invocation. When a Python signal + /// handler is running, new signals are deferred until it completes. + static IN_SIGNAL_HANDLER: Cell<bool> = const { Cell::new(false) }; +} + +struct SignalHandlerGuard; + +impl Drop for SignalHandlerGuard { + fn drop(&mut self) { + IN_SIGNAL_HANDLER.with(|h| h.set(false)); + } +} + +#[cfg_attr(feature = "flame-it", flame)] +#[inline(always)] +pub fn check_signals(vm: &VirtualMachine) -> PyResult<()> { + if vm.signal_handlers.get().is_none() { + return Ok(()); + } + + // Read-only check first: avoids cache-line invalidation on every + // instruction when no signal is pending (the common case). + if !ANY_TRIGGERED.load(Ordering::Relaxed) { + return Ok(()); + } + // Atomic RMW only when a signal is actually pending. + if !ANY_TRIGGERED.swap(false, Ordering::Acquire) { + return Ok(()); + } + + trigger_signals(vm) +} + +#[inline(never)] +#[cold] +fn trigger_signals(vm: &VirtualMachine) -> PyResult<()> { + if IN_SIGNAL_HANDLER.with(|h| h.replace(true)) { + // Already inside a signal handler — defer pending signals + set_triggered(); + return Ok(()); + } + let _guard = SignalHandlerGuard; + + // unwrap should never fail since we check above + let signal_handlers = vm.signal_handlers.get().unwrap().borrow(); + for (signum, trigger) in TRIGGERS.iter().enumerate().skip(1) { + let triggered = trigger.swap(false, Ordering::Relaxed); + if triggered + && let Some(handler) = &signal_handlers[signum] + && let Some(callable) = handler.to_callable() + { + callable.invoke((signum, vm.ctx.none()), vm)?; + } + } + if let Some(signal_rx) = &vm.signal_rx { + for f in signal_rx.rx.try_iter() { + f(vm)?; + } + } + Ok(()) +} + +pub(crate) fn set_triggered() { + ANY_TRIGGERED.store(true, Ordering::Release); +} + +#[inline(always)] +pub(crate) fn is_triggered() -> bool { + ANY_TRIGGERED.load(Ordering::Relaxed) +} + +/// Reset all signal trigger state after fork in child process. +/// Stale triggers from the parent must not fire in the child. +#[cfg(unix)] +#[cfg(feature = "host_env")] +pub(crate) fn clear_after_fork() { + ANY_TRIGGERED.store(false, Ordering::Release); + for trigger in &TRIGGERS { + trigger.store(false, Ordering::Relaxed); + } +} + +pub fn assert_in_range(signum: i32, vm: &VirtualMachine) -> PyResult<()> { + if (1..NSIG as i32).contains(&signum) { + Ok(()) + } else { + Err(vm.new_value_error("signal number out of range")) + } +} + +/// Similar to `PyErr_SetInterruptEx` in CPython +/// +/// Missing signal handler for the given signal number is silently ignored. +#[allow(dead_code)] +#[cfg(all(not(target_arch = "wasm32"), feature = "host_env"))] +pub fn set_interrupt_ex(signum: i32, vm: &VirtualMachine) -> PyResult<()> { + use crate::stdlib::_signal::_signal::{SIG_DFL, SIG_IGN, run_signal}; + assert_in_range(signum, vm)?; + + match signum as usize { + SIG_DFL | SIG_IGN => Ok(()), + _ => { + // interrupt the main thread with given signal number + run_signal(signum); + Ok(()) + } + } +} + +pub type UserSignal = Box<dyn FnOnce(&VirtualMachine) -> PyResult<()> + Send>; + +#[derive(Clone, Debug)] +pub struct UserSignalSender { + tx: mpsc::Sender<UserSignal>, +} + +#[derive(Debug)] +pub struct UserSignalReceiver { + rx: mpsc::Receiver<UserSignal>, +} + +impl UserSignalSender { + pub fn send(&self, sig: UserSignal) -> Result<(), UserSignalSendError> { + self.tx + .send(sig) + .map_err(|mpsc::SendError(sig)| UserSignalSendError(sig))?; + set_triggered(); + Ok(()) + } +} + +pub struct UserSignalSendError(pub UserSignal); + +impl fmt::Debug for UserSignalSendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("UserSignalSendError") + .finish_non_exhaustive() + } +} + +impl fmt::Display for UserSignalSendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("sending a signal to a exited vm") + } +} + +pub fn user_signal_channel() -> (UserSignalSender, UserSignalReceiver) { + let (tx, rx) = mpsc::channel(); + (UserSignalSender { tx }, UserSignalReceiver { rx }) +} + +#[cfg(windows)] +pub fn set_sigint_event(handle: isize) { + SIGINT_EVENT.store(handle, Ordering::Release); +} + +#[cfg(windows)] +pub fn get_sigint_event() -> Option<isize> { + let handle = SIGINT_EVENT.load(Ordering::Acquire); + if handle == 0 { None } else { Some(handle) } +} diff --git a/vm/src/sliceable.rs b/crates/vm/src/sliceable.rs similarity index 92% rename from vm/src/sliceable.rs rename to crates/vm/src/sliceable.rs index 257d325651d..e416f5a1b49 100644 --- a/vm/src/sliceable.rs +++ b/crates/vm/src/sliceable.rs @@ -1,11 +1,11 @@ // export through sliceable module, not slice. use crate::{ - builtins::{int::PyInt, slice::PySlice}, PyObject, PyResult, VirtualMachine, + builtins::{int::PyInt, slice::PySlice}, }; +use core::ops::Range; use malachite_bigint::BigInt; use num_traits::{Signed, ToPrimitive}; -use std::ops::Range; pub trait SliceableSequenceMutOp where @@ -34,7 +34,7 @@ where let pos = self .as_ref() .wrap_index(index) - .ok_or_else(|| vm.new_index_error("assignment index out of range".to_owned()))?; + .ok_or_else(|| vm.new_index_error("assignment index out of range"))?; self.do_set(pos, value); Ok(()) } @@ -47,8 +47,7 @@ where ) -> PyResult<()> { let (range, step, slice_len) = slice.adjust_indices(self.as_ref().len()); if slice_len != items.len() { - Err(vm - .new_buffer_error("Existing exports of data: object cannot be re-sized".to_owned())) + Err(vm.new_buffer_error("Existing exports of data: object cannot be re-sized")) } else if step == 1 { self.do_set_range(range, items); Ok(()) @@ -90,7 +89,7 @@ where let pos = self .as_ref() .wrap_index(index) - .ok_or_else(|| vm.new_index_error("assignment index out of range".to_owned()))?; + .ok_or_else(|| vm.new_index_error("assignment index out of range"))?; self.do_delete(pos); Ok(()) } @@ -206,7 +205,7 @@ pub trait SliceableSequenceOp { fn getitem_by_index(&self, vm: &VirtualMachine, index: isize) -> PyResult<Self::Item> { let pos = self .wrap_index(index) - .ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?; + .ok_or_else(|| vm.new_index_error("index out of range"))?; Ok(self.do_get(pos)) } } @@ -264,21 +263,17 @@ impl SequenceIndex { obj: &PyObject, type_name: &str, ) -> PyResult<Self> { - if let Some(i) = obj.payload::<PyInt>() { + if let Some(i) = obj.downcast_ref::<PyInt>() { // TODO: number protocol i.try_to_primitive(vm) - .map_err(|_| { - vm.new_index_error("cannot fit 'int' into an index-sized integer".to_owned()) - }) + .map_err(|_| vm.new_index_error("cannot fit 'int' into an index-sized integer")) .map(Self::Int) - } else if let Some(slice) = obj.payload::<PySlice>() { + } else if let Some(slice) = obj.downcast_ref::<PySlice>() { slice.to_saturated(vm).map(Self::Slice) } else if let Some(i) = obj.try_index_opt(vm) { // TODO: __index__ for indices is no more supported? i?.try_to_primitive(vm) - .map_err(|_| { - vm.new_index_error("cannot fit 'int' into an index-sized integer".to_owned()) - }) + .map_err(|_| vm.new_index_error("cannot fit 'int' into an index-sized integer")) .map(Self::Int) } else { Err(vm.new_type_error(format!( @@ -311,7 +306,7 @@ impl SequenceIndexOp for isize { let mut p = *self; if p < 0 { // casting to isize is ok because it is used by wrapping_add - p = p.wrapping_add(len as isize); + p = p.wrapping_add(len as Self); } if p < 0 || (p as usize) >= len { None @@ -331,6 +326,7 @@ impl SequenceIndexOp for BigInt { self.try_into().unwrap_or(len) } } + fn wrapped_at(&self, _len: usize) -> Option<usize> { unimplemented!("please add one once we need it") } @@ -355,15 +351,10 @@ impl SaturatedSlice { pub fn with_slice(slice: &PySlice, vm: &VirtualMachine) -> PyResult<Self> { let step = to_isize_index(vm, slice.step_ref(vm))?.unwrap_or(1); if step == 0 { - return Err(vm.new_value_error("slice step cannot be zero".to_owned())); + return Err(vm.new_value_error("slice step cannot be zero")); } - let start = to_isize_index(vm, slice.start_ref(vm))?.unwrap_or_else(|| { - if step.is_negative() { - isize::MAX - } else { - 0 - } - }); + let start = to_isize_index(vm, slice.start_ref(vm))? + .unwrap_or_else(|| if step.is_negative() { isize::MAX } else { 0 }); let stop = to_isize_index(vm, &slice.stop(vm))?.unwrap_or_else(|| { if step.is_negative() { @@ -424,7 +415,7 @@ impl SaturatedSliceIter { Self::from_adjust_indices(range, step, len) } - pub fn from_adjust_indices(range: Range<usize>, step: isize, len: usize) -> Self { + pub const fn from_adjust_indices(range: Range<usize>, step: isize, len: usize) -> Self { let index = if step.is_negative() { range.end as isize - 1 } else { @@ -433,7 +424,7 @@ impl SaturatedSliceIter { Self { index, step, len } } - pub fn positive_order(mut self) -> Self { + pub const fn positive_order(mut self) -> Self { if self.step.is_negative() { self.index += self.step * self.len.saturating_sub(1) as isize; self.step = self.step.saturating_abs() @@ -465,9 +456,7 @@ fn to_isize_index(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Option<isize> return Ok(None); } let result = obj.try_index_opt(vm).unwrap_or_else(|| { - Err(vm.new_type_error( - "slice indices must be integers or None or have an __index__ method".to_owned(), - )) + Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method")) })?; let value = result.as_bigint(); let is_negative = value.is_negative(); diff --git a/crates/vm/src/stdlib/_abc.rs b/crates/vm/src/stdlib/_abc.rs new file mode 100644 index 00000000000..28b3399ec4b --- /dev/null +++ b/crates/vm/src/stdlib/_abc.rs @@ -0,0 +1,479 @@ +//! Implementation of the `_abc` module. +//! +//! This module provides the C implementation of Abstract Base Classes (ABCs) +//! as defined in PEP 3119. + +pub(crate) use _abc::module_def; + +#[pymodule] +mod _abc { + use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyFrozenSet, PyList, PySet, PyStr, PyTupleRef, PyTypeRef, PyWeak}, + common::lock::PyRwLock, + convert::ToPyObject, + protocol::PyIterReturn, + types::Constructor, + }; + use core::sync::atomic::{AtomicU64, Ordering}; + + // Global invalidation counter + static ABC_INVALIDATION_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn get_invalidation_counter() -> u64 { + ABC_INVALIDATION_COUNTER.load(Ordering::SeqCst) + } + + fn increment_invalidation_counter() { + ABC_INVALIDATION_COUNTER.fetch_add(1, Ordering::SeqCst); + } + + /// Internal state held by ABC machinery. + #[pyattr] + #[pyclass(name = "_abc_data", module = "_abc")] + #[derive(Debug, PyPayload)] + struct AbcData { + // WeakRef sets for registry and caches + registry: PyRwLock<Option<PyRef<PySet>>>, + cache: PyRwLock<Option<PyRef<PySet>>>, + negative_cache: PyRwLock<Option<PyRef<PySet>>>, + negative_cache_version: AtomicU64, + } + + #[pyclass(with(Constructor))] + impl AbcData { + fn new() -> Self { + AbcData { + registry: PyRwLock::new(None), + cache: PyRwLock::new(None), + negative_cache: PyRwLock::new(None), + negative_cache_version: AtomicU64::new(get_invalidation_counter()), + } + } + + fn get_cache_version(&self) -> u64 { + self.negative_cache_version.load(Ordering::SeqCst) + } + + fn set_cache_version(&self, version: u64) { + self.negative_cache_version.store(version, Ordering::SeqCst); + } + } + + impl Constructor for AbcData { + type Args = (); + + fn py_new( + _cls: &crate::Py<crate::builtins::PyType>, + _args: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(AbcData::new()) + } + } + + /// Get the _abc_impl attribute from an ABC class + fn get_impl(cls: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<AbcData>> { + let impl_obj = cls.get_attr("_abc_impl", vm)?; + impl_obj + .downcast::<AbcData>() + .map_err(|_| vm.new_type_error("_abc_impl is set to a wrong type")) + } + + /// Check if obj is in the weak set + fn in_weak_set( + set_lock: &PyRwLock<Option<PyRef<PySet>>>, + obj: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<bool> { + let set_opt = set_lock.read(); + let set = match &*set_opt { + Some(s) if !s.elements().is_empty() => s.clone(), + _ => return Ok(false), + }; + drop(set_opt); + + // Create a weak reference to the object + let weak_ref = match obj.downgrade(None, vm) { + Ok(w) => w, + Err(e) => { + // If we can't create a weakref (e.g., TypeError), the object can't be in the set + if e.class().is(vm.ctx.exceptions.type_error) { + return Ok(false); + } + return Err(e); + } + }; + + // Use vm.call_method to call __contains__ + let weak_ref_obj: PyObjectRef = weak_ref.into(); + vm.call_method(set.as_ref(), "__contains__", (weak_ref_obj,))? + .try_to_bool(vm) + } + + /// Add obj to the weak set + fn add_to_weak_set( + set_lock: &PyRwLock<Option<PyRef<PySet>>>, + obj: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut set_opt = set_lock.write(); + let set = match &*set_opt { + Some(s) => s.clone(), + None => { + let new_set = PySet::default().into_ref(&vm.ctx); + *set_opt = Some(new_set.clone()); + new_set + } + }; + drop(set_opt); + + // Create a weak reference to the object + let weak_ref = obj.downgrade(None, vm)?; + set.add(weak_ref.into(), vm)?; + Ok(()) + } + + /// Returns the current ABC cache token. + #[pyfunction] + fn get_cache_token() -> u64 { + get_invalidation_counter() + } + + /// Compute set of abstract method names. + fn compute_abstract_methods(cls: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + let mut abstracts = Vec::new(); + + // Stage 1: direct abstract methods + let ns = cls.get_attr("__dict__", vm)?; + let items = vm.call_method(&ns, "items", ())?; + let iter = items.get_iter(vm)?; + + while let PyIterReturn::Return(item) = iter.next(vm)? { + let tuple: PyTupleRef = item + .downcast() + .map_err(|_| vm.new_type_error("items() returned non-tuple"))?; + let elements = tuple.as_slice(); + if elements.len() != 2 { + return Err(vm.new_type_error("items() returned item which size is not 2")); + } + let key = &elements[0]; + let value = &elements[1]; + + // Check if value has __isabstractmethod__ = True + if let Ok(is_abstract) = value.get_attr("__isabstractmethod__", vm) + && is_abstract.try_to_bool(vm)? + { + abstracts.push(key.clone()); + } + } + + // Stage 2: inherited abstract methods + let bases: PyTupleRef = cls + .get_attr("__bases__", vm)? + .downcast() + .map_err(|_| vm.new_type_error("__bases__ is not a tuple"))?; + + for base in bases.iter() { + if let Ok(base_abstracts) = base.get_attr("__abstractmethods__", vm) { + let iter = base_abstracts.get_iter(vm)?; + while let PyIterReturn::Return(key) = iter.next(vm)? { + // Try to get the attribute from cls - key should be a string + if let Some(key_str) = key.downcast_ref::<PyStr>() + && let Some(value) = vm.get_attribute_opt(cls.to_owned(), key_str)? + && let Ok(is_abstract) = value.get_attr("__isabstractmethod__", vm) + && is_abstract.try_to_bool(vm)? + { + abstracts.push(key); + } + } + } + } + + // Set __abstractmethods__ + let abstracts_set = PyFrozenSet::from_iter(vm, abstracts.into_iter())?; + cls.set_attr("__abstractmethods__", abstracts_set.into_pyobject(vm), vm)?; + + Ok(()) + } + + /// Internal ABC helper for class set-up. Should be never used outside abc module. + #[pyfunction] + fn _abc_init(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + compute_abstract_methods(&cls, vm)?; + + // Set up inheritance registry + let data = AbcData::new(); + cls.set_attr("_abc_impl", data.to_pyobject(vm), vm)?; + + Ok(()) + } + + /// Internal ABC helper for subclass registration. Should be never used outside abc module. + #[pyfunction] + fn _abc_register( + cls: PyObjectRef, + subclass: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + // Type check + if !subclass.class().fast_issubclass(vm.ctx.types.type_type) { + return Err(vm.new_type_error("Can only register classes")); + } + + // Check if already a subclass + if subclass.is_subclass(&cls, vm)? { + return Ok(subclass); + } + + // Check for cycles + if cls.is_subclass(&subclass, vm)? { + return Err(vm.new_runtime_error("Refusing to create an inheritance cycle")); + } + + // Add to registry + let impl_data = get_impl(&cls, vm)?; + add_to_weak_set(&impl_data.registry, &subclass, vm)?; + + // Invalidate negative cache + increment_invalidation_counter(); + + Ok(subclass) + } + + /// Internal ABC helper for instance checks. Should be never used outside abc module. + #[pyfunction] + fn _abc_instancecheck( + cls: PyObjectRef, + instance: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let impl_data = get_impl(&cls, vm)?; + + // Get instance.__class__ + let subclass = instance.get_attr("__class__", vm)?; + + // Check cache + if in_weak_set(&impl_data.cache, &subclass, vm)? { + return Ok(vm.ctx.true_value.clone().into()); + } + + let subtype: PyObjectRef = instance.class().to_owned().into(); + if subtype.is(&subclass) { + let invalidation_counter = get_invalidation_counter(); + if impl_data.get_cache_version() == invalidation_counter + && in_weak_set(&impl_data.negative_cache, &subclass, vm)? + { + return Ok(vm.ctx.false_value.clone().into()); + } + // Fall back to __subclasscheck__ + return vm.call_method(&cls, "__subclasscheck__", (subclass,)); + } + + // Call __subclasscheck__ on subclass + let result = vm.call_method(&cls, "__subclasscheck__", (subclass.clone(),))?; + + match result.clone().try_to_bool(vm) { + Ok(true) => Ok(result), + Ok(false) => { + // Also try with subtype + vm.call_method(&cls, "__subclasscheck__", (subtype,)) + } + Err(e) => Err(e), + } + } + + /// Check if subclass is in registry (recursive) + fn subclasscheck_check_registry( + impl_data: &AbcData, + subclass: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<Option<bool>> { + // Fast path: check if subclass is in weakref directly + if in_weak_set(&impl_data.registry, subclass, vm)? { + return Ok(Some(true)); + } + + let registry_opt = impl_data.registry.read(); + let registry = match &*registry_opt { + Some(s) => s.clone(), + None => return Ok(None), + }; + drop(registry_opt); + + // Make a local copy to protect against concurrent modifications + let registry_copy = PyFrozenSet::from_iter(vm, registry.elements().into_iter())?; + + for weak_ref_obj in registry_copy.elements() { + if let Ok(weak_ref) = weak_ref_obj.downcast::<PyWeak>() + && let Some(rkey) = weak_ref.upgrade() + && subclass.to_owned().is_subclass(&rkey, vm)? + { + add_to_weak_set(&impl_data.cache, subclass, vm)?; + return Ok(Some(true)); + } + } + + Ok(None) + } + + /// Internal ABC helper for subclass checks. Should be never used outside abc module. + #[pyfunction] + fn _abc_subclasscheck( + cls: PyObjectRef, + subclass: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<bool> { + // Type check + if !subclass.class().fast_issubclass(vm.ctx.types.type_type) { + return Err(vm.new_type_error("issubclass() arg 1 must be a class")); + } + + let impl_data = get_impl(&cls, vm)?; + + // 1. Check cache + if in_weak_set(&impl_data.cache, &subclass, vm)? { + return Ok(true); + } + + // 2. Check negative cache; may have to invalidate + let invalidation_counter = get_invalidation_counter(); + if impl_data.get_cache_version() < invalidation_counter { + // Invalidate the negative cache + // Clone set ref and drop lock before calling into VM to avoid reentrancy + let set = impl_data.negative_cache.read().clone(); + if let Some(ref set) = set { + vm.call_method(set.as_ref(), "clear", ())?; + } + impl_data.set_cache_version(invalidation_counter); + } else if in_weak_set(&impl_data.negative_cache, &subclass, vm)? { + return Ok(false); + } + + // 3. Check the subclass hook + let ok = vm.call_method(&cls, "__subclasshook__", (subclass.clone(),))?; + if ok.is(&vm.ctx.true_value) { + add_to_weak_set(&impl_data.cache, &subclass, vm)?; + return Ok(true); + } + if ok.is(&vm.ctx.false_value) { + add_to_weak_set(&impl_data.negative_cache, &subclass, vm)?; + return Ok(false); + } + if !ok.is(&vm.ctx.not_implemented) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.assertion_error.to_owned(), + "__subclasshook__ must return either False, True, or NotImplemented".into(), + )); + } + + // 4. Check if it's a direct subclass + let subclass_type: PyTypeRef = subclass + .clone() + .downcast() + .map_err(|_| vm.new_type_error("expected a type object"))?; + let cls_type: PyTypeRef = cls + .clone() + .downcast() + .map_err(|_| vm.new_type_error("expected a type object"))?; + if subclass_type.fast_issubclass(&cls_type) { + add_to_weak_set(&impl_data.cache, &subclass, vm)?; + return Ok(true); + } + + // 5. Check if it's a subclass of a registered class (recursive) + if let Some(result) = subclasscheck_check_registry(&impl_data, &subclass, vm)? { + return Ok(result); + } + + // 6. Check if it's a subclass of a subclass (recursive) + let subclasses: PyRef<PyList> = vm + .call_method(&cls, "__subclasses__", ())? + .downcast() + .map_err(|_| vm.new_type_error("__subclasses__() must return a list"))?; + + for scls in subclasses.borrow_vec().iter() { + if subclass.is_subclass(scls, vm)? { + add_to_weak_set(&impl_data.cache, &subclass, vm)?; + return Ok(true); + } + } + + // No dice; update negative cache + add_to_weak_set(&impl_data.negative_cache, &subclass, vm)?; + Ok(false) + } + + /// Internal ABC helper for cache and registry debugging. + #[pyfunction] + fn _get_dump(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let impl_data = get_impl(&cls, vm)?; + + let registry = { + let r = impl_data.registry.read(); + match &*r { + Some(s) => { + // Use copy method to get a shallow copy + vm.call_method(s.as_ref(), "copy", ())? + } + None => PySet::default().to_pyobject(vm), + } + }; + + let cache = { + let c = impl_data.cache.read(); + match &*c { + Some(s) => vm.call_method(s.as_ref(), "copy", ())?, + None => PySet::default().to_pyobject(vm), + } + }; + + let negative_cache = { + let nc = impl_data.negative_cache.read(); + match &*nc { + Some(s) => vm.call_method(s.as_ref(), "copy", ())?, + None => PySet::default().to_pyobject(vm), + } + }; + + let version = impl_data.get_cache_version(); + + Ok(vm.ctx.new_tuple(vec![ + registry, + cache, + negative_cache, + vm.ctx.new_int(version).into(), + ])) + } + + /// Internal ABC helper to reset registry of a given class. + #[pyfunction] + fn _reset_registry(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let impl_data = get_impl(&cls, vm)?; + // Clone set ref and drop lock before calling into VM to avoid reentrancy + let set = impl_data.registry.read().clone(); + if let Some(ref set) = set { + vm.call_method(set.as_ref(), "clear", ())?; + } + Ok(()) + } + + /// Internal ABC helper to reset both caches of a given class. + #[pyfunction] + fn _reset_caches(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let impl_data = get_impl(&cls, vm)?; + + // Clone set refs and drop locks before calling into VM to avoid reentrancy + let cache = impl_data.cache.read().clone(); + if let Some(ref set) = cache { + vm.call_method(set.as_ref(), "clear", ())?; + } + + let negative_cache = impl_data.negative_cache.read().clone(); + if let Some(ref set) = negative_cache { + vm.call_method(set.as_ref(), "clear", ())?; + } + + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs new file mode 100644 index 00000000000..73819e257c1 --- /dev/null +++ b/crates/vm/src/stdlib/_ast.rs @@ -0,0 +1,838 @@ +//! `ast` standard module for abstract syntax trees. + +//! +//! This module makes use of the parser logic, and translates all ast nodes +//! into python ast.AST objects. + +pub(crate) use python::_ast::module_def; + +mod pyast; + +use crate::builtins::{PyInt, PyStr}; +use crate::stdlib::_ast::module::{Mod, ModFunctionType, ModInteractive}; +use crate::stdlib::_ast::node::BoxedSlice; +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + TryFromObject, VirtualMachine, + builtins::PyIntRef, + builtins::{PyDict, PyModule, PyType, PyUtf8StrRef}, + class::{PyClassImpl, StaticType}, + compiler::{CompileError, ParseError}, + convert::ToPyObject, +}; +use node::Node; +use ruff_python_ast as ast; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustpython_compiler_core::{ + LineIndex, OneIndexed, PositionEncoding, SourceFile, SourceFileBuilder, SourceLocation, +}; + +#[cfg(feature = "parser")] +use ruff_python_parser as parser; + +#[cfg(feature = "codegen")] +use rustpython_codegen as codegen; + +pub(crate) use python::_ast::NodeAst; + +mod python; +mod repr; +mod validate; + +mod argument; +mod basic; +mod constant; +mod elif_else_clause; +mod exception; +mod expression; +mod module; +mod node; +mod operator; +mod other; +mod parameter; +mod pattern; +mod statement; +mod string; +mod type_ignore; +mod type_parameters; + +/// Return the cached singleton instance for an operator/context node type, +/// or create a new instance if none exists. +fn singleton_node_to_object(vm: &VirtualMachine, node_type: &'static Py<PyType>) -> PyObjectRef { + if let Some(instance) = node_type.get_attr(vm.ctx.intern_str("_instance")) { + return instance; + } + NodeAst + .into_ref_with_type(vm, node_type.to_owned()) + .unwrap() + .into() +} + +fn get_node_field(vm: &VirtualMachine, obj: &PyObject, field: &'static str, typ: &str) -> PyResult { + vm.get_attribute_opt(obj.to_owned(), field)? + .ok_or_else(|| vm.new_type_error(format!(r#"required field "{field}" missing from {typ}"#))) +} + +fn get_node_field_opt( + vm: &VirtualMachine, + obj: &PyObject, + field: &'static str, +) -> PyResult<Option<PyObjectRef>> { + Ok(vm + .get_attribute_opt(obj.to_owned(), field)? + .filter(|obj| !vm.is_none(obj))) +} + +fn get_int_field( + vm: &VirtualMachine, + obj: &PyObject, + field: &'static str, + typ: &str, +) -> PyResult<PyRefExact<PyInt>> { + get_node_field(vm, obj, field, typ)? + .downcast_exact(vm) + .map_err(|_| vm.new_type_error(format!(r#"field "{field}" must have integer type"#))) +} + +struct PySourceRange { + start: PySourceLocation, + end: PySourceLocation, +} + +pub struct PySourceLocation { + row: Row, + column: Column, +} + +impl PySourceLocation { + const fn to_source_location(&self) -> SourceLocation { + SourceLocation { + line: self.row.get_one_indexed(), + character_offset: self.column.get_one_indexed(), + } + } +} + +/// A one-based index into the lines. +#[derive(Clone, Copy)] +struct Row(OneIndexed); + +impl Row { + const fn get(self) -> usize { + self.0.get() + } + + const fn get_one_indexed(self) -> OneIndexed { + self.0 + } +} + +/// An UTF-8 index into the line. +#[derive(Clone, Copy)] +struct Column(TextSize); + +impl Column { + const fn get(self) -> usize { + self.0.to_usize() + } + + const fn get_one_indexed(self) -> OneIndexed { + OneIndexed::from_zero_indexed(self.get()) + } +} + +fn text_range_to_source_range(source_file: &SourceFile, text_range: TextRange) -> PySourceRange { + let index = LineIndex::from_source_text(source_file.clone().source_text()); + let source = &source_file.source_text(); + + if source.is_empty() { + return PySourceRange { + start: PySourceLocation { + row: Row(OneIndexed::from_zero_indexed(0)), + column: Column(TextSize::new(0)), + }, + end: PySourceLocation { + row: Row(OneIndexed::from_zero_indexed(0)), + column: Column(TextSize::new(0)), + }, + }; + } + + let start_row = index.line_index(text_range.start()); + let end_row = index.line_index(text_range.end()); + let start_col = text_range.start() - index.line_start(start_row, source); + let end_col = text_range.end() - index.line_start(end_row, source); + + PySourceRange { + start: PySourceLocation { + row: Row(start_row), + column: Column(start_col), + }, + end: PySourceLocation { + row: Row(end_row), + column: Column(end_col), + }, + } +} + +fn get_opt_int_field( + vm: &VirtualMachine, + obj: &PyObject, + field: &'static str, +) -> PyResult<Option<PyRefExact<PyInt>>> { + match get_node_field_opt(vm, obj, field)? { + Some(val) => val + .downcast_exact(vm) + .map(Some) + .map_err(|_| vm.new_type_error(format!(r#"field "{field}" must have integer type"#))), + None => Ok(None), + } +} + +fn range_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + name: &str, +) -> PyResult<TextRange> { + let start_row = get_int_field(vm, &object, "lineno", name)?; + let start_column = get_int_field(vm, &object, "col_offset", name)?; + // end_lineno and end_col_offset are optional, default to start values + let end_row = + get_opt_int_field(vm, &object, "end_lineno")?.unwrap_or_else(|| start_row.clone()); + let end_column = + get_opt_int_field(vm, &object, "end_col_offset")?.unwrap_or_else(|| start_column.clone()); + + // lineno=0 or negative values as a special case (no location info). + // Use default values (line 1, col 0) when lineno <= 0. + let start_row_val: i32 = start_row.try_to_primitive(vm)?; + let end_row_val: i32 = end_row.try_to_primitive(vm)?; + let start_col_val: i32 = start_column.try_to_primitive(vm)?; + let end_col_val: i32 = end_column.try_to_primitive(vm)?; + + if start_row_val > end_row_val { + return Err(vm.new_value_error(format!( + "AST node line range ({}, {}) is not valid", + start_row_val, end_row_val + ))); + } + if (start_row_val < 0 && end_row_val != start_row_val) + || (start_col_val < 0 && end_col_val != start_col_val) + { + return Err(vm.new_value_error(format!( + "AST node column range ({}, {}) for line range ({}, {}) is not valid", + start_col_val, end_col_val, start_row_val, end_row_val + ))); + } + if start_row_val == end_row_val && start_col_val > end_col_val { + return Err(vm.new_value_error(format!( + "line {}, column {}-{} is not a valid range", + start_row_val, start_col_val, end_col_val + ))); + } + + let location = PySourceRange { + start: PySourceLocation { + row: Row(if start_row_val > 0 { + OneIndexed::new(start_row_val as usize).unwrap_or(OneIndexed::MIN) + } else { + OneIndexed::MIN + }), + column: Column(TextSize::new(start_col_val.max(0) as u32)), + }, + end: PySourceLocation { + row: Row(if end_row_val > 0 { + OneIndexed::new(end_row_val as usize).unwrap_or(OneIndexed::MIN) + } else { + OneIndexed::MIN + }), + column: Column(TextSize::new(end_col_val.max(0) as u32)), + }, + }; + + Ok(source_range_to_text_range(source_file, location)) +} + +fn source_range_to_text_range(source_file: &SourceFile, location: PySourceRange) -> TextRange { + let index = LineIndex::from_source_text(source_file.clone().source_text()); + let source = &source_file.source_text(); + + if source.is_empty() { + return TextRange::new(TextSize::new(0), TextSize::new(0)); + } + + let start = index.offset( + location.start.to_source_location(), + source, + PositionEncoding::Utf8, + ); + let end = index.offset( + location.end.to_source_location(), + source, + PositionEncoding::Utf8, + ); + + TextRange::new(start, end) +} + +fn node_add_location( + dict: &Py<PyDict>, + range: TextRange, + vm: &VirtualMachine, + source_file: &SourceFile, +) { + let range = text_range_to_source_range(source_file, range); + dict.set_item("lineno", vm.ctx.new_int(range.start.row.get()).into(), vm) + .unwrap(); + dict.set_item( + "col_offset", + vm.ctx.new_int(range.start.column.get()).into(), + vm, + ) + .unwrap(); + dict.set_item("end_lineno", vm.ctx.new_int(range.end.row.get()).into(), vm) + .unwrap(); + dict.set_item( + "end_col_offset", + vm.ctx.new_int(range.end.column.get()).into(), + vm, + ) + .unwrap(); +} + +/// Return the expected AST mod type class for a compile() mode string. +pub(crate) fn mode_type_and_name(mode: &str) -> Option<(PyRef<PyType>, &'static str)> { + match mode { + "exec" => Some((pyast::NodeModModule::make_static_type(), "Module")), + "eval" => Some((pyast::NodeModExpression::make_static_type(), "Expression")), + "single" => Some((pyast::NodeModInteractive::make_static_type(), "Interactive")), + "func_type" => Some(( + pyast::NodeModFunctionType::make_static_type(), + "FunctionType", + )), + _ => None, + } +} + +/// Create an empty `arguments` AST node (no parameters). +fn empty_arguments_object(vm: &VirtualMachine) -> PyObjectRef { + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + for list_field in [ + "posonlyargs", + "args", + "kwonlyargs", + "kw_defaults", + "defaults", + ] { + dict.set_item(list_field, vm.ctx.new_list(vec![]).into(), vm) + .unwrap(); + } + for none_field in ["vararg", "kwarg"] { + dict.set_item(none_field, vm.ctx.none(), vm).unwrap(); + } + node.into() +} + +#[cfg(feature = "parser")] +pub(crate) fn parse( + vm: &VirtualMachine, + source: &str, + mode: parser::Mode, + optimize: u8, + target_version: Option<ast::PythonVersion>, + type_comments: bool, +) -> Result<PyObjectRef, CompileError> { + let source_file = SourceFileBuilder::new("".to_owned(), source.to_owned()).finish(); + let mut options = parser::ParseOptions::from(mode); + let target_version = target_version.unwrap_or(ast::PythonVersion::PY314); + options = options.with_target_version(target_version); + let parsed = parser::parse(source, options).map_err(|parse_error| { + let range = text_range_to_source_range(&source_file, parse_error.location); + ParseError { + error: parse_error.error, + raw_location: parse_error.location, + location: range.start.to_source_location(), + end_location: range.end.to_source_location(), + source_path: "<unknown>".to_string(), + is_unclosed_bracket: false, + } + })?; + + if let Some(error) = parsed.unsupported_syntax_errors().first() { + let range = text_range_to_source_range(&source_file, error.range()); + return Err(ParseError { + error: parser::ParseErrorType::OtherError(error.to_string()), + raw_location: error.range(), + location: range.start.to_source_location(), + end_location: range.end.to_source_location(), + source_path: "<unknown>".to_string(), + is_unclosed_bracket: false, + } + .into()); + } + + let mut top = parsed.into_syntax(); + if optimize > 0 { + fold_match_value_constants(&mut top); + } + if optimize >= 2 { + strip_docstrings(&mut top); + } + let top = match top { + ast::Mod::Module(m) => Mod::Module(m), + ast::Mod::Expression(e) => Mod::Expression(e), + }; + let obj = top.ast_to_object(vm, &source_file); + if type_comments && obj.class().is(pyast::NodeModModule::static_type()) { + let type_ignores = type_ignores_from_source(vm, source)?; + let dict = obj.as_object().dict().unwrap(); + dict.set_item("type_ignores", vm.ctx.new_list(type_ignores).into(), vm) + .unwrap(); + } + Ok(obj) +} + +#[cfg(feature = "parser")] +pub(crate) fn wrap_interactive(vm: &VirtualMachine, module_obj: PyObjectRef) -> PyResult { + if !module_obj.class().is(pyast::NodeModModule::static_type()) { + return Err(vm.new_type_error("expected Module node")); + } + let body = get_node_field(vm, &module_obj, "body", "Module")?; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModInteractive::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body, vm).unwrap(); + Ok(node.into()) +} + +#[cfg(feature = "parser")] +pub(crate) fn parse_func_type( + vm: &VirtualMachine, + source: &str, + optimize: u8, + target_version: Option<ast::PythonVersion>, +) -> Result<PyObjectRef, CompileError> { + let _ = optimize; + let _ = target_version; + let source = source.trim(); + let mut depth = 0i32; + let mut split_at = None; + let mut chars = source.chars().peekable(); + let mut idx = 0usize; + while let Some(ch) = chars.next() { + match ch { + '(' | '[' | '{' => depth += 1, + ')' | ']' | '}' => depth -= 1, + '-' if depth == 0 && chars.peek() == Some(&'>') => { + split_at = Some(idx); + break; + } + _ => {} + } + idx += ch.len_utf8(); + } + + let Some(split_at) = split_at else { + return Err(ParseError { + error: parser::ParseErrorType::OtherError("invalid func_type".to_owned()), + raw_location: TextRange::default(), + location: SourceLocation::default(), + end_location: SourceLocation::default(), + source_path: "<unknown>".to_owned(), + is_unclosed_bracket: false, + } + .into()); + }; + + let left = source[..split_at].trim(); + let right = source[split_at + 2..].trim(); + + let parse_expr = |expr_src: &str| -> Result<ast::Expr, CompileError> { + let source_file = SourceFileBuilder::new("".to_owned(), expr_src.to_owned()).finish(); + let parsed = parser::parse_expression(expr_src).map_err(|parse_error| { + let range = text_range_to_source_range(&source_file, parse_error.location); + ParseError { + error: parse_error.error, + raw_location: parse_error.location, + location: range.start.to_source_location(), + end_location: range.end.to_source_location(), + source_path: "<unknown>".to_string(), + is_unclosed_bracket: false, + } + })?; + Ok(*parsed.into_syntax().body) + }; + + let arg_expr = parse_expr(left)?; + let returns = parse_expr(right)?; + + let argtypes: Vec<ast::Expr> = match arg_expr { + ast::Expr::Tuple(tup) => tup.elts, + ast::Expr::Name(_) | ast::Expr::Subscript(_) | ast::Expr::Attribute(_) => vec![arg_expr], + other => vec![other], + }; + + let func_type = ModFunctionType { + argtypes: argtypes.into_boxed_slice(), + returns, + range: TextRange::default(), + }; + let source_file = SourceFileBuilder::new("".to_owned(), source.to_owned()).finish(); + Ok(func_type.ast_to_object(vm, &source_file)) +} + +fn type_ignores_from_source( + vm: &VirtualMachine, + source: &str, +) -> Result<Vec<PyObjectRef>, CompileError> { + let mut ignores = Vec::new(); + for (idx, line) in source.lines().enumerate() { + let Some(pos) = line.find("#") else { + continue; + }; + let comment = &line[pos + 1..]; + let comment = comment.trim_start(); + let Some(rest) = comment.strip_prefix("type: ignore") else { + continue; + }; + let tag = rest.trim_start(); + let tag = if tag.is_empty() { "" } else { tag }; + let node = NodeAst + .into_ref_with_type( + vm, + pyast::NodeTypeIgnoreTypeIgnore::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + let lineno = idx + 1; + dict.set_item("lineno", vm.ctx.new_int(lineno).into(), vm) + .unwrap(); + dict.set_item("tag", vm.ctx.new_str(tag).into(), vm) + .unwrap(); + ignores.push(node.into()); + } + Ok(ignores) +} + +#[cfg(feature = "parser")] +fn fold_match_value_constants(top: &mut ast::Mod) { + match top { + ast::Mod::Module(module) => fold_stmts(&mut module.body), + ast::Mod::Expression(_expr) => {} + } +} + +#[cfg(feature = "parser")] +fn strip_docstrings(top: &mut ast::Mod) { + match top { + ast::Mod::Module(module) => strip_docstring_in_body(&mut module.body), + ast::Mod::Expression(_expr) => {} + } +} + +#[cfg(feature = "parser")] +fn strip_docstring_in_body(body: &mut Vec<ast::Stmt>) { + if let Some(range) = take_docstring(body) + && body.is_empty() + { + let start_offset = range.start(); + let end_offset = start_offset + TextSize::from(4); + let pass_range = TextRange::new(start_offset, end_offset); + body.push(ast::Stmt::Pass(ast::StmtPass { + node_index: Default::default(), + range: pass_range, + })); + } + for stmt in body { + match stmt { + ast::Stmt::FunctionDef(def) => strip_docstring_in_body(&mut def.body), + ast::Stmt::ClassDef(def) => strip_docstring_in_body(&mut def.body), + _ => {} + } + } +} + +#[cfg(feature = "parser")] +fn take_docstring(body: &mut Vec<ast::Stmt>) -> Option<TextRange> { + let ast::Stmt::Expr(expr_stmt) = body.first()? else { + return None; + }; + if matches!(expr_stmt.value.as_ref(), ast::Expr::StringLiteral(_)) { + let range = expr_stmt.range; + body.remove(0); + return Some(range); + } + None +} + +#[cfg(feature = "parser")] +fn fold_stmts(stmts: &mut [ast::Stmt]) { + for stmt in stmts { + fold_stmt(stmt); + } +} + +#[cfg(feature = "parser")] +fn fold_stmt(stmt: &mut ast::Stmt) { + use ast::Stmt; + match stmt { + Stmt::FunctionDef(def) => fold_stmts(&mut def.body), + Stmt::ClassDef(def) => fold_stmts(&mut def.body), + Stmt::For(stmt) => { + fold_stmts(&mut stmt.body); + fold_stmts(&mut stmt.orelse); + } + Stmt::While(stmt) => { + fold_stmts(&mut stmt.body); + fold_stmts(&mut stmt.orelse); + } + Stmt::If(stmt) => { + fold_stmts(&mut stmt.body); + for clause in &mut stmt.elif_else_clauses { + fold_stmts(&mut clause.body); + } + } + Stmt::With(stmt) => { + fold_stmts(&mut stmt.body); + } + Stmt::Try(stmt) => { + fold_stmts(&mut stmt.body); + fold_stmts(&mut stmt.orelse); + fold_stmts(&mut stmt.finalbody); + } + Stmt::Match(stmt) => { + for case in &mut stmt.cases { + fold_pattern(&mut case.pattern); + if let Some(expr) = case.guard.as_deref_mut() { + fold_expr(expr); + } + fold_stmts(&mut case.body); + } + } + _ => {} + } +} + +#[cfg(feature = "parser")] +fn fold_pattern(pattern: &mut ast::Pattern) { + use ast::Pattern; + match pattern { + Pattern::MatchValue(value) => fold_expr(&mut value.value), + Pattern::MatchSequence(seq) => { + for pattern in &mut seq.patterns { + fold_pattern(pattern); + } + } + Pattern::MatchMapping(mapping) => { + for key in &mut mapping.keys { + fold_expr(key); + } + for pattern in &mut mapping.patterns { + fold_pattern(pattern); + } + } + Pattern::MatchClass(class) => { + for pattern in &mut class.arguments.patterns { + fold_pattern(pattern); + } + for keyword in &mut class.arguments.keywords { + fold_pattern(&mut keyword.pattern); + } + } + Pattern::MatchAs(match_as) => { + if let Some(pattern) = match_as.pattern.as_deref_mut() { + fold_pattern(pattern); + } + } + Pattern::MatchOr(match_or) => { + for pattern in &mut match_or.patterns { + fold_pattern(pattern); + } + } + Pattern::MatchSingleton(_) | Pattern::MatchStar(_) => {} + } +} + +#[cfg(feature = "parser")] +fn fold_expr(expr: &mut ast::Expr) { + use ast::Expr; + if let Expr::UnaryOp(unary) = expr { + fold_expr(&mut unary.operand); + if matches!(unary.op, ast::UnaryOp::USub) + && let Expr::NumberLiteral(number_literal) = unary.operand.as_ref() + { + let number = match &number_literal.value { + ast::Number::Int(value) => { + if *value == ast::Int::ZERO { + Some(ast::Number::Int(ast::Int::ZERO)) + } else { + None + } + } + ast::Number::Float(value) => Some(ast::Number::Float(-value)), + ast::Number::Complex { real, imag } => Some(ast::Number::Complex { + real: -real, + imag: -imag, + }), + }; + if let Some(number) = number { + *expr = Expr::NumberLiteral(ast::ExprNumberLiteral { + node_index: unary.node_index.clone(), + range: unary.range, + value: number, + }); + return; + } + } + } + if let Expr::BinOp(binop) = expr { + fold_expr(&mut binop.left); + fold_expr(&mut binop.right); + + let Expr::NumberLiteral(left) = binop.left.as_ref() else { + return; + }; + let Expr::NumberLiteral(right) = binop.right.as_ref() else { + return; + }; + + if let Some(number) = fold_number_binop(&left.value, &binop.op, &right.value) { + *expr = Expr::NumberLiteral(ast::ExprNumberLiteral { + node_index: binop.node_index.clone(), + range: binop.range, + value: number, + }); + } + } +} + +#[cfg(feature = "parser")] +fn fold_number_binop( + left: &ast::Number, + op: &ast::Operator, + right: &ast::Number, +) -> Option<ast::Number> { + let (left_real, left_imag, left_is_complex) = number_to_complex(left)?; + let (right_real, right_imag, right_is_complex) = number_to_complex(right)?; + + if !(left_is_complex || right_is_complex) { + return None; + } + + match op { + ast::Operator::Add => Some(ast::Number::Complex { + real: left_real + right_real, + imag: left_imag + right_imag, + }), + ast::Operator::Sub => Some(ast::Number::Complex { + real: left_real - right_real, + imag: left_imag - right_imag, + }), + _ => None, + } +} + +#[cfg(feature = "parser")] +fn number_to_complex(number: &ast::Number) -> Option<(f64, f64, bool)> { + match number { + ast::Number::Complex { real, imag } => Some((*real, *imag, true)), + ast::Number::Float(value) => Some((*value, 0.0, false)), + ast::Number::Int(value) => value.as_i64().map(|value| (value as f64, 0.0, false)), + } +} + +#[cfg(feature = "codegen")] +pub(crate) fn compile( + vm: &VirtualMachine, + object: PyObjectRef, + filename: &str, + mode: crate::compiler::Mode, + optimize: Option<u8>, +) -> PyResult { + let mut opts = vm.compile_opts(); + if let Some(optimize) = optimize { + opts.optimize = optimize; + } + + let source_file = SourceFileBuilder::new(filename.to_owned(), "".to_owned()).finish(); + let ast: Mod = Node::ast_from_object(vm, &source_file, object)?; + validate::validate_mod(vm, &ast)?; + let ast = match ast { + Mod::Module(m) => ast::Mod::Module(m), + Mod::Interactive(ModInteractive { range, body }) => ast::Mod::Module(ast::ModModule { + node_index: Default::default(), + range, + body, + }), + Mod::Expression(e) => ast::Mod::Expression(e), + Mod::FunctionType(_) => todo!(), + }; + // TODO: create a textual representation of the ast + let text = ""; + let source_file = SourceFileBuilder::new(filename, text).finish(); + let code = codegen::compile::compile_top(ast, source_file, mode, opts) + .map_err(|err| vm.new_syntax_error(&err.into(), None))?; // FIXME source + Ok(vm.ctx.new_code(code).into()) +} + +#[cfg(feature = "codegen")] +pub(crate) fn validate_ast_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<()> { + let source_file = SourceFileBuilder::new("<ast>".to_owned(), "".to_owned()).finish(); + let ast: Mod = Node::ast_from_object(vm, &source_file, object)?; + validate::validate_mod(vm, &ast)?; + Ok(()) +} + +// Used by builtins::compile() +pub const PY_CF_ONLY_AST: i32 = 0x0400; + +// The following flags match the values from Include/cpython/compile.h +// Caveat emptor: These flags are undocumented on purpose and depending +// on their effect outside the standard library is **unsupported**. +pub const PY_CF_SOURCE_IS_UTF8: i32 = 0x0100; +pub const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200; +pub const PY_CF_IGNORE_COOKIE: i32 = 0x0800; +pub const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000; +pub const PY_CF_OPTIMIZED_AST: i32 = 0x8000 | PY_CF_ONLY_AST; +pub const PY_CF_TYPE_COMMENTS: i32 = 0x1000; +pub const PY_CF_ALLOW_TOP_LEVEL_AWAIT: i32 = 0x2000; + +// __future__ flags - sync with Lib/__future__.py +// TODO: These flags aren't being used in rust code +// CO_FUTURE_ANNOTATIONS does make a difference in the codegen, +// so it should be used in compile(). +// see compiler/codegen/src/compile.rs +const CO_NESTED: i32 = 0x0010; +const CO_GENERATOR_ALLOWED: i32 = 0; +const CO_FUTURE_DIVISION: i32 = 0x20000; +const CO_FUTURE_ABSOLUTE_IMPORT: i32 = 0x40000; +const CO_FUTURE_WITH_STATEMENT: i32 = 0x80000; +const CO_FUTURE_PRINT_FUNCTION: i32 = 0x100000; +const CO_FUTURE_UNICODE_LITERALS: i32 = 0x200000; +const CO_FUTURE_BARRY_AS_BDFL: i32 = 0x400000; +const CO_FUTURE_GENERATOR_STOP: i32 = 0x800000; +const CO_FUTURE_ANNOTATIONS: i32 = 0x1000000; + +// Used by builtins::compile() - the summary of all flags +pub const PY_COMPILE_FLAGS_MASK: i32 = PY_CF_ONLY_AST + | PY_CF_SOURCE_IS_UTF8 + | PY_CF_DONT_IMPLY_DEDENT + | PY_CF_IGNORE_COOKIE + | PY_CF_ALLOW_TOP_LEVEL_AWAIT + | PY_CF_ALLOW_INCOMPLETE_INPUT + | PY_CF_OPTIMIZED_AST + | PY_CF_TYPE_COMMENTS + | CO_NESTED + | CO_GENERATOR_ALLOWED + | CO_FUTURE_DIVISION + | CO_FUTURE_ABSOLUTE_IMPORT + | CO_FUTURE_WITH_STATEMENT + | CO_FUTURE_PRINT_FUNCTION + | CO_FUTURE_UNICODE_LITERALS + | CO_FUTURE_BARRY_AS_BDFL + | CO_FUTURE_GENERATOR_STOP + | CO_FUTURE_ANNOTATIONS; diff --git a/crates/vm/src/stdlib/_ast/argument.rs b/crates/vm/src/stdlib/_ast/argument.rs new file mode 100644 index 00000000000..626024f5bd6 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/argument.rs @@ -0,0 +1,166 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +pub(super) struct PositionalArguments { + pub range: TextRange, + pub args: Box<[ast::Expr]>, +} + +impl Node for PositionalArguments { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { args, range: _ } = self; + BoxedSlice(args).ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let args: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; + Ok(Self { + args: args.0, + range: TextRange::default(), // TODO + }) + } +} + +pub(super) struct KeywordArguments { + pub range: TextRange, + pub keywords: Box<[ast::Keyword]>, +} + +impl Node for KeywordArguments { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { keywords, range: _ } = self; + // TODO: use range + BoxedSlice(keywords).ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let keywords: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; + Ok(Self { + keywords: keywords.0, + range: TextRange::default(), // TODO + }) + } +} + +pub(super) fn merge_function_call_arguments( + pos_args: PositionalArguments, + key_args: KeywordArguments, +) -> ast::Arguments { + let range = pos_args.range.cover(key_args.range); + + ast::Arguments { + node_index: Default::default(), + range, + args: pos_args.args, + keywords: key_args.keywords, + } +} + +pub(super) fn split_function_call_arguments( + args: ast::Arguments, +) -> (PositionalArguments, KeywordArguments) { + let ast::Arguments { + node_index: _, + range: _, + args, + keywords, + } = args; + + let positional_arguments_range = args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(); + // debug_assert!(range.contains_range(positional_arguments_range)); + let positional_arguments = PositionalArguments { + range: positional_arguments_range, + args, + }; + + let keyword_arguments_range = keywords + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(); + // debug_assert!(range.contains_range(keyword_arguments_range)); + let keyword_arguments = KeywordArguments { + range: keyword_arguments_range, + keywords, + }; + + (positional_arguments, keyword_arguments) +} + +pub(super) fn split_class_def_args( + args: Option<Box<ast::Arguments>>, +) -> (Option<PositionalArguments>, Option<KeywordArguments>) { + let args = match args { + None => return (None, None), + Some(args) => *args, + }; + let ast::Arguments { + node_index: _, + range: _, + args, + keywords, + } = args; + + let positional_arguments_range = args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(); + // debug_assert!(range.contains_range(positional_arguments_range)); + let positional_arguments = PositionalArguments { + range: positional_arguments_range, + args, + }; + + let keyword_arguments_range = keywords + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(); + // debug_assert!(range.contains_range(keyword_arguments_range)); + let keyword_arguments = KeywordArguments { + range: keyword_arguments_range, + keywords, + }; + + (Some(positional_arguments), Some(keyword_arguments)) +} + +pub(super) fn merge_class_def_args( + positional_arguments: Option<PositionalArguments>, + keyword_arguments: Option<KeywordArguments>, +) -> Option<Box<ast::Arguments>> { + if positional_arguments.is_none() && keyword_arguments.is_none() { + return None; + } + + let args = if let Some(positional_arguments) = positional_arguments { + positional_arguments.args + } else { + vec![].into_boxed_slice() + }; + let keywords = if let Some(keyword_arguments) = keyword_arguments { + keyword_arguments.keywords + } else { + vec![].into_boxed_slice() + }; + + Some(Box::new(ast::Arguments { + node_index: Default::default(), + range: Default::default(), // TODO + args, + keywords, + })) +} diff --git a/crates/vm/src/stdlib/_ast/basic.rs b/crates/vm/src/stdlib/_ast/basic.rs new file mode 100644 index 00000000000..28e4a6803ee --- /dev/null +++ b/crates/vm/src/stdlib/_ast/basic.rs @@ -0,0 +1,50 @@ +use super::*; +use rustpython_codegen::compile::ruff_int_to_bigint; +use rustpython_compiler_core::SourceFile; + +impl Node for ast::Identifier { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + let id = self.as_str(); + vm.ctx.intern_str(id).to_object() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let py_str = PyUtf8StrRef::try_from_object(vm, object)?; + Ok(Self::new(py_str.as_str(), TextRange::default())) + } +} + +impl Node for ast::Int { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + vm.ctx.new_int(ruff_int_to_bigint(&self).unwrap()).into() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + // FIXME: performance + let value: PyIntRef = object.try_into_value(vm)?; + let value = value.as_bigint().to_string(); + Ok(value.parse().unwrap()) + } +} + +impl Node for bool { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + vm.ctx.new_int(self as u8).into() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + i32::try_from_object(vm, object).map(|i| i != 0) + } +} diff --git a/crates/vm/src/stdlib/_ast/constant.rs b/crates/vm/src/stdlib/_ast/constant.rs new file mode 100644 index 00000000000..1030a037f17 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/constant.rs @@ -0,0 +1,447 @@ +use super::*; +use crate::builtins::{PyComplex, PyFrozenSet, PyTuple}; +use ast::str_prefix::StringLiteralPrefix; +use rustpython_compiler_core::SourceFile; + +#[derive(Debug)] +pub(super) struct Constant { + pub(super) range: TextRange, + pub(super) value: ConstantLiteral, +} + +impl Constant { + pub(super) fn new_str( + value: impl Into<Box<str>>, + prefix: StringLiteralPrefix, + range: TextRange, + ) -> Self { + let value = value.into(); + Self { + range, + value: ConstantLiteral::Str { value, prefix }, + } + } + + pub(super) const fn new_int(value: ast::Int, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Int(value), + } + } + + pub(super) const fn new_float(value: f64, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Float(value), + } + } + + pub(super) const fn new_complex(real: f64, imag: f64, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Complex { real, imag }, + } + } + + pub(super) const fn new_bytes(value: Box<[u8]>, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Bytes(value), + } + } + + pub(super) const fn new_bool(value: bool, range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Bool(value), + } + } + + pub(super) const fn new_none(range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::None, + } + } + + pub(super) const fn new_ellipsis(range: TextRange) -> Self { + Self { + range, + value: ConstantLiteral::Ellipsis, + } + } + + pub(crate) fn into_expr(self) -> ast::Expr { + constant_to_ruff_expr(self) + } +} + +#[derive(Debug)] +pub(crate) enum ConstantLiteral { + None, + Bool(bool), + Str { + value: Box<str>, + prefix: StringLiteralPrefix, + }, + Bytes(Box<[u8]>), + Int(ast::Int), + Tuple(Vec<ConstantLiteral>), + FrozenSet(Vec<ConstantLiteral>), + Float(f64), + Complex { + real: f64, + imag: f64, + }, + Ellipsis, +} + +// constructor +impl Node for Constant { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { range, value } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprConstant::static_type().to_owned()) + .unwrap(); + let kind = match &value { + ConstantLiteral::Str { + prefix: StringLiteralPrefix::Unicode, + .. + } => vm.ctx.new_str("u").into(), + _ => vm.ctx.none(), + }; + let value = value.ast_to_object(vm, source_file); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value, vm).unwrap(); + dict.set_item("kind", kind, vm).unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let value_object = get_node_field(vm, &object, "value", "Constant")?; + let value = Node::ast_from_object(vm, source_file, value_object)?; + + Ok(Self { + value, + // kind: get_node_field_opt(_vm, &_object, "kind")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(vm, source_file, object, "Constant")?, + }) + } +} + +impl Node for ConstantLiteral { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::None => vm.ctx.none(), + Self::Bool(value) => vm.ctx.new_bool(value).to_pyobject(vm), + Self::Str { value, .. } => vm.ctx.new_str(value).to_pyobject(vm), + Self::Bytes(value) => vm.ctx.new_bytes(value.into()).to_pyobject(vm), + Self::Int(value) => value.ast_to_object(vm, source_file), + Self::Tuple(value) => { + let value = value + .into_iter() + .map(|c| c.ast_to_object(vm, source_file)) + .collect(); + vm.ctx.new_tuple(value).to_pyobject(vm) + } + Self::FrozenSet(value) => PyFrozenSet::from_iter( + vm, + value.into_iter().map(|c| c.ast_to_object(vm, source_file)), + ) + .unwrap() + .into_pyobject(vm), + Self::Float(value) => vm.ctx.new_float(value).into_pyobject(vm), + Self::Complex { real, imag } => vm + .ctx + .new_complex(num_complex::Complex::new(real, imag)) + .into_pyobject(vm), + Self::Ellipsis => vm.ctx.ellipsis.clone().into(), + } + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + value_object: PyObjectRef, + ) -> PyResult<Self> { + let cls = value_object.class(); + let value = if cls.is(vm.ctx.types.none_type) { + Self::None + } else if cls.is(vm.ctx.types.bool_type) { + Self::Bool(if value_object.is(&vm.ctx.true_value) { + true + } else if value_object.is(&vm.ctx.false_value) { + false + } else { + value_object.try_to_value(vm)? + }) + } else if cls.is(vm.ctx.types.str_type) { + Self::Str { + value: value_object.try_to_value::<String>(vm)?.into(), + prefix: StringLiteralPrefix::Empty, + } + } else if cls.is(vm.ctx.types.bytes_type) { + Self::Bytes(value_object.try_to_value::<Vec<u8>>(vm)?.into()) + } else if cls.is(vm.ctx.types.int_type) { + Self::Int(Node::ast_from_object(vm, source_file, value_object)?) + } else if cls.is(vm.ctx.types.tuple_type) { + let tuple = value_object.downcast::<PyTuple>().map_err(|obj| { + vm.new_type_error(format!( + "Expected type {}, not {}", + PyTuple::static_type().name(), + obj.class().name() + )) + })?; + let tuple = tuple + .into_iter() + .cloned() + .map(|object| Node::ast_from_object(vm, source_file, object)) + .collect::<PyResult<_>>()?; + Self::Tuple(tuple) + } else if cls.is(vm.ctx.types.frozenset_type) { + let set = value_object.downcast::<PyFrozenSet>().unwrap(); + let elements = set + .elements() + .into_iter() + .map(|object| Node::ast_from_object(vm, source_file, object)) + .collect::<PyResult<_>>()?; + Self::FrozenSet(elements) + } else if cls.is(vm.ctx.types.float_type) { + let float = value_object.try_into_value(vm)?; + Self::Float(float) + } else if cls.is(vm.ctx.types.complex_type) { + let complex = value_object.try_complex(vm)?; + let complex = match complex { + None => { + return Err(vm.new_type_error(format!( + "Expected type {}, not {}", + PyComplex::static_type().name(), + value_object.class().name() + ))); + } + Some((value, _was_coerced)) => value, + }; + Self::Complex { + real: complex.re, + imag: complex.im, + } + } else if cls.is(vm.ctx.types.ellipsis_type) { + Self::Ellipsis + } else { + return Err(vm.new_type_error(format!( + "got an invalid type in Constant: {}", + value_object.class().name() + ))); + }; + Ok(value) + } +} + +fn constant_to_ruff_expr(value: Constant) -> ast::Expr { + let Constant { value, range } = value; + match value { + ConstantLiteral::None => ast::Expr::NoneLiteral(ast::ExprNoneLiteral { + node_index: Default::default(), + range, + }), + ConstantLiteral::Bool(value) => ast::Expr::BooleanLiteral(ast::ExprBooleanLiteral { + node_index: Default::default(), + range, + value, + }), + ConstantLiteral::Str { value, prefix } => { + ast::Expr::StringLiteral(ast::ExprStringLiteral { + node_index: Default::default(), + range, + value: ast::StringLiteralValue::single(ast::StringLiteral { + node_index: Default::default(), + range, + value, + flags: ast::StringLiteralFlags::empty().with_prefix(prefix), + }), + }) + } + ConstantLiteral::Bytes(value) => { + ast::Expr::BytesLiteral(ast::ExprBytesLiteral { + node_index: Default::default(), + range, + value: ast::BytesLiteralValue::single(ast::BytesLiteral { + node_index: Default::default(), + range, + value, + flags: ast::BytesLiteralFlags::empty(), // TODO + }), + }) + } + ConstantLiteral::Int(value) => ast::Expr::NumberLiteral(ast::ExprNumberLiteral { + node_index: Default::default(), + range, + value: ast::Number::Int(value), + }), + ConstantLiteral::Tuple(value) => ast::Expr::Tuple(ast::ExprTuple { + node_index: Default::default(), + range, + elts: value + .into_iter() + .map(|value| { + constant_to_ruff_expr(Constant { + range: TextRange::default(), + value, + }) + }) + .collect(), + ctx: ast::ExprContext::Load, + // TODO: Does this matter? + parenthesized: true, + }), + ConstantLiteral::FrozenSet(value) => { + let args = if value.is_empty() { + Vec::new() + } else { + vec![ast::Expr::Set(ast::ExprSet { + node_index: Default::default(), + range: TextRange::default(), + elts: value + .into_iter() + .map(|value| { + constant_to_ruff_expr(Constant { + range: TextRange::default(), + value, + }) + }) + .collect(), + })] + }; + ast::Expr::Call(ast::ExprCall { + node_index: Default::default(), + range, + func: Box::new(ast::Expr::Name(ast::ExprName { + node_index: Default::default(), + range: TextRange::default(), + id: ast::name::Name::new_static("frozenset"), + ctx: ast::ExprContext::Load, + })), + arguments: ast::Arguments { + node_index: Default::default(), + range, + args: args.into(), + keywords: Box::default(), + }, + }) + } + ConstantLiteral::Float(value) => ast::Expr::NumberLiteral(ast::ExprNumberLiteral { + node_index: Default::default(), + range, + value: ast::Number::Float(value), + }), + ConstantLiteral::Complex { real, imag } => { + ast::Expr::NumberLiteral(ast::ExprNumberLiteral { + node_index: Default::default(), + range, + value: ast::Number::Complex { real, imag }, + }) + } + ConstantLiteral::Ellipsis => ast::Expr::EllipsisLiteral(ast::ExprEllipsisLiteral { + node_index: Default::default(), + range, + }), + } +} + +pub(super) fn number_literal_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + constant: ast::ExprNumberLiteral, +) -> PyObjectRef { + let ast::ExprNumberLiteral { + node_index: _, + range, + value, + } = constant; + let c = match value { + ast::Number::Int(n) => Constant::new_int(n, range), + ast::Number::Float(n) => Constant::new_float(n, range), + ast::Number::Complex { real, imag } => Constant::new_complex(real, imag, range), + }; + c.ast_to_object(vm, source_file) +} + +pub(super) fn string_literal_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + constant: ast::ExprStringLiteral, +) -> PyObjectRef { + let ast::ExprStringLiteral { + node_index: _, + range, + value, + } = constant; + let prefix = value + .iter() + .next() + .map_or(StringLiteralPrefix::Empty, |part| part.flags.prefix()); + let c = Constant::new_str(value.to_str(), prefix, range); + c.ast_to_object(vm, source_file) +} + +pub(super) fn bytes_literal_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + constant: ast::ExprBytesLiteral, +) -> PyObjectRef { + let ast::ExprBytesLiteral { + node_index: _, + range, + value, + } = constant; + let bytes = value.as_slice().iter().flat_map(|b| b.value.iter()); + let c = Constant::new_bytes(bytes.copied().collect(), range); + c.ast_to_object(vm, source_file) +} + +pub(super) fn boolean_literal_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + constant: ast::ExprBooleanLiteral, +) -> PyObjectRef { + let ast::ExprBooleanLiteral { + node_index: _, + range, + value, + } = constant; + let c = Constant::new_bool(value, range); + c.ast_to_object(vm, source_file) +} + +pub(super) fn none_literal_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + constant: ast::ExprNoneLiteral, +) -> PyObjectRef { + let ast::ExprNoneLiteral { + node_index: _, + range, + } = constant; + let c = Constant::new_none(range); + c.ast_to_object(vm, source_file) +} + +pub(super) fn ellipsis_literal_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + constant: ast::ExprEllipsisLiteral, +) -> PyObjectRef { + let ast::ExprEllipsisLiteral { + node_index: _, + range, + } = constant; + let c = Constant::new_ellipsis(range); + c.ast_to_object(vm, source_file) +} diff --git a/crates/vm/src/stdlib/_ast/elif_else_clause.rs b/crates/vm/src/stdlib/_ast/elif_else_clause.rs new file mode 100644 index 00000000000..0afdbc02ac1 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/elif_else_clause.rs @@ -0,0 +1,104 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +pub(super) fn ast_to_object( + clause: ast::ElifElseClause, + mut rest: alloc::vec::IntoIter<ast::ElifElseClause>, + vm: &VirtualMachine, + source_file: &SourceFile, +) -> PyObjectRef { + let ast::ElifElseClause { + node_index: _, + range, + test, + body, + } = clause; + let Some(test) = test else { + assert!(rest.len() == 0); + return body.ast_to_object(vm, source_file); + }; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeStmtIf::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + + dict.set_item("test", test.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + + let orelse = if let Some(next) = rest.next() { + if next.test.is_some() { + let next = ast::ElifElseClause { + range: TextRange::new(next.range.start(), range.end()), + ..next + }; + vm.ctx + .new_list(vec![ast_to_object(next, rest, vm, source_file)]) + .into() + } else { + next.body.ast_to_object(vm, source_file) + } + } else { + vm.ctx.new_list(vec![]).into() + }; + dict.set_item("orelse", orelse, vm).unwrap(); + + node_add_location(&dict, range, vm, source_file); + node.into() +} + +pub(super) fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, +) -> PyResult<ast::StmtIf> { + let test = Node::ast_from_object(vm, source_file, get_node_field(vm, &object, "test", "If")?)?; + let body = Node::ast_from_object(vm, source_file, get_node_field(vm, &object, "body", "If")?)?; + let orelse: Vec<ast::Stmt> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "orelse", "If")?, + )?; + let range = range_from_object(vm, source_file, object, "If")?; + + let elif_else_clauses = if orelse.is_empty() { + vec![] + } else if let [ast::Stmt::If(_)] = &*orelse { + let Some(ast::Stmt::If(ast::StmtIf { + node_index: _, + range, + test, + body, + mut elif_else_clauses, + })) = orelse.into_iter().next() + else { + unreachable!() + }; + elif_else_clauses.insert( + 0, + ast::ElifElseClause { + node_index: Default::default(), + range, + test: Some(*test), + body, + }, + ); + elif_else_clauses + } else { + vec![ast::ElifElseClause { + node_index: Default::default(), + range, + test: None, + body: orelse, + }] + }; + + Ok(ast::StmtIf { + node_index: Default::default(), + test, + body, + elif_else_clauses, + range, + }) +} diff --git a/crates/vm/src/stdlib/_ast/exception.rs b/crates/vm/src/stdlib/_ast/exception.rs new file mode 100644 index 00000000000..2daabecc84c --- /dev/null +++ b/crates/vm/src/stdlib/_ast/exception.rs @@ -0,0 +1,82 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +// sum +impl Node for ast::ExceptHandler { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::ExceptHandler(cons) => cons.ast_to_object(vm, source_file), + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok( + if cls.is(pyast::NodeExceptHandlerExceptHandler::static_type()) { + Self::ExceptHandler(ast::ExceptHandlerExceptHandler::ast_from_object( + vm, + source_file, + object, + )?) + } else { + return Err(vm.new_type_error(format!( + "expected some sort of excepthandler, but got {}", + object.repr(vm)? + ))); + }, + ) + } +} + +// constructor +impl Node for ast::ExceptHandlerExceptHandler { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + type_, + name, + body, + range, + } = self; + let node = NodeAst + .into_ref_with_type( + vm, + pyast::NodeExceptHandlerExceptHandler::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("type", type_.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + type_: get_node_field_opt(vm, &object, "type")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + name: get_node_field_opt(vm, &object, "name")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + body: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "body", "ExceptHandler")?, + )?, + range: range_from_object(vm, source_file, object, "ExceptHandler")?, + }) + } +} diff --git a/crates/vm/src/stdlib/_ast/expression.rs b/crates/vm/src/stdlib/_ast/expression.rs new file mode 100644 index 00000000000..cbc47dde9fb --- /dev/null +++ b/crates/vm/src/stdlib/_ast/expression.rs @@ -0,0 +1,1337 @@ +use super::*; +use crate::stdlib::_ast::{ + argument::{merge_function_call_arguments, split_function_call_arguments}, + constant::Constant, + string::JoinedStr, +}; +use rustpython_compiler_core::SourceFile; + +// sum +impl Node for ast::Expr { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::BoolOp(cons) => cons.ast_to_object(vm, source_file), + Self::Name(cons) => cons.ast_to_object(vm, source_file), + Self::BinOp(cons) => cons.ast_to_object(vm, source_file), + Self::UnaryOp(cons) => cons.ast_to_object(vm, source_file), + Self::Lambda(cons) => cons.ast_to_object(vm, source_file), + Self::If(cons) => cons.ast_to_object(vm, source_file), + Self::Dict(cons) => cons.ast_to_object(vm, source_file), + Self::Set(cons) => cons.ast_to_object(vm, source_file), + Self::ListComp(cons) => cons.ast_to_object(vm, source_file), + Self::SetComp(cons) => cons.ast_to_object(vm, source_file), + Self::DictComp(cons) => cons.ast_to_object(vm, source_file), + Self::Generator(cons) => cons.ast_to_object(vm, source_file), + Self::Await(cons) => cons.ast_to_object(vm, source_file), + Self::Yield(cons) => cons.ast_to_object(vm, source_file), + Self::YieldFrom(cons) => cons.ast_to_object(vm, source_file), + Self::Compare(cons) => cons.ast_to_object(vm, source_file), + Self::Call(cons) => cons.ast_to_object(vm, source_file), + Self::Attribute(cons) => cons.ast_to_object(vm, source_file), + Self::Subscript(cons) => cons.ast_to_object(vm, source_file), + Self::Starred(cons) => cons.ast_to_object(vm, source_file), + Self::List(cons) => cons.ast_to_object(vm, source_file), + Self::Tuple(cons) => cons.ast_to_object(vm, source_file), + Self::Slice(cons) => cons.ast_to_object(vm, source_file), + Self::NumberLiteral(cons) => constant::number_literal_to_object(vm, source_file, cons), + Self::StringLiteral(cons) => constant::string_literal_to_object(vm, source_file, cons), + Self::FString(cons) => string::fstring_to_object(vm, source_file, cons), + Self::TString(cons) => string::tstring_to_object(vm, source_file, cons), + Self::BytesLiteral(cons) => constant::bytes_literal_to_object(vm, source_file, cons), + Self::BooleanLiteral(cons) => { + constant::boolean_literal_to_object(vm, source_file, cons) + } + Self::NoneLiteral(cons) => constant::none_literal_to_object(vm, source_file, cons), + Self::EllipsisLiteral(cons) => { + constant::ellipsis_literal_to_object(vm, source_file, cons) + } + Self::Named(cons) => cons.ast_to_object(vm, source_file), + Self::IpyEscapeCommand(_) => { + unimplemented!("IPython escape command is not allowed in Python AST") + } + } + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeExprBoolOp::static_type()) { + Self::BoolOp(ast::ExprBoolOp::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprNamedExpr::static_type()) { + Self::Named(ast::ExprNamed::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprBinOp::static_type()) { + Self::BinOp(ast::ExprBinOp::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprUnaryOp::static_type()) { + Self::UnaryOp(ast::ExprUnaryOp::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprLambda::static_type()) { + Self::Lambda(ast::ExprLambda::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprIfExp::static_type()) { + Self::If(ast::ExprIf::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprDict::static_type()) { + Self::Dict(ast::ExprDict::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprSet::static_type()) { + Self::Set(ast::ExprSet::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprListComp::static_type()) { + Self::ListComp(ast::ExprListComp::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprSetComp::static_type()) { + Self::SetComp(ast::ExprSetComp::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprDictComp::static_type()) { + Self::DictComp(ast::ExprDictComp::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprGeneratorExp::static_type()) { + Self::Generator(ast::ExprGenerator::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodeExprAwait::static_type()) { + Self::Await(ast::ExprAwait::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprYield::static_type()) { + Self::Yield(ast::ExprYield::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprYieldFrom::static_type()) { + Self::YieldFrom(ast::ExprYieldFrom::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodeExprCompare::static_type()) { + Self::Compare(ast::ExprCompare::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprCall::static_type()) { + Self::Call(ast::ExprCall::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprAttribute::static_type()) { + Self::Attribute(ast::ExprAttribute::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodeExprSubscript::static_type()) { + Self::Subscript(ast::ExprSubscript::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodeExprStarred::static_type()) { + Self::Starred(ast::ExprStarred::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprName::static_type()) { + Self::Name(ast::ExprName::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprList::static_type()) { + Self::List(ast::ExprList::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprTuple::static_type()) { + Self::Tuple(ast::ExprTuple::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprSlice::static_type()) { + Self::Slice(ast::ExprSlice::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeExprConstant::static_type()) { + Constant::ast_from_object(vm, source_file, object)?.into_expr() + } else if cls.is(pyast::NodeExprJoinedStr::static_type()) { + JoinedStr::ast_from_object(vm, source_file, object)?.into_expr() + } else if cls.is(pyast::NodeExprTemplateStr::static_type()) { + let template = string::TemplateStr::ast_from_object(vm, source_file, object)?; + return string::template_str_to_expr(vm, template); + } else if cls.is(pyast::NodeExprInterpolation::static_type()) { + let interpolation = + string::TStringInterpolation::ast_from_object(vm, source_file, object)?; + return string::interpolation_to_expr(vm, interpolation); + } else { + return Err(vm.new_type_error(format!( + "expected some sort of expr, but got {}", + object.repr(vm)? + ))); + }) + } +} + +// constructor +impl Node for ast::ExprBoolOp { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + op, + values, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprBoolOp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("op", op.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("values", values.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + op: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "op", "BoolOp")?, + )?, + values: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "values", "BoolOp")?, + )?, + range: range_from_object(vm, source_file, object, "BoolOp")?, + }) + } +} + +// constructor +impl Node for ast::ExprNamed { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + target, + value, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprNamedExpr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + target: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "target", "NamedExpr")?, + )?, + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "NamedExpr")?, + )?, + range: range_from_object(vm, source_file, object, "NamedExpr")?, + }) + } +} + +// constructor +impl Node for ast::ExprBinOp { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + left, + op, + right, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprBinOp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("left", left.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("op", op.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("right", right.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + left: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "left", "BinOp")?, + )?, + op: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "op", "BinOp")?, + )?, + right: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "right", "BinOp")?, + )?, + range: range_from_object(vm, source_file, object, "BinOp")?, + }) + } +} + +// constructor +impl Node for ast::ExprUnaryOp { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + op, + operand, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprUnaryOp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("op", op.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("operand", operand.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + op: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "op", "UnaryOp")?, + )?, + operand: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "operand", "UnaryOp")?, + )?, + range: range_from_object(vm, source_file, object, "UnaryOp")?, + }) + } +} + +// constructor +impl Node for ast::ExprLambda { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + parameters, + body, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprLambda::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + // Lambda with no parameters should have an empty arguments object, not None + let args = match parameters { + Some(params) => params.ast_to_object(vm, source_file), + None => empty_arguments_object(vm), + }; + dict.set_item("args", args, vm).unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + parameters: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "args", "Lambda")?, + )?, + body: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "body", "Lambda")?, + )?, + range: range_from_object(vm, source_file, object, "Lambda")?, + }) + } +} + +// constructor +impl Node for ast::ExprIf { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + test, + body, + orelse, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprIfExp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("test", test.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("orelse", orelse.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + test: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "test", "IfExp")?, + )?, + body: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "body", "IfExp")?, + )?, + orelse: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "orelse", "IfExp")?, + )?, + range: range_from_object(vm, source_file, object, "IfExp")?, + }) + } +} + +// constructor +impl Node for ast::ExprDict { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + items, + range, + } = self; + let (keys, values) = + items + .into_iter() + .fold((vec![], vec![]), |(mut keys, mut values), item| { + keys.push(item.key); + values.push(item.value); + (keys, values) + }); + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprDict::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("keys", keys.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("values", values.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let keys: Vec<Option<ast::Expr>> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "keys", "Dict")?, + )?; + let values: Vec<_> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "values", "Dict")?, + )?; + if keys.len() != values.len() { + return Err(vm.new_value_error("Dict doesn't have the same number of keys as values")); + } + let items = keys + .into_iter() + .zip(values) + .map(|(key, value)| ast::DictItem { key, value }) + .collect(); + Ok(Self { + node_index: Default::default(), + items, + range: range_from_object(vm, source_file, object, "Dict")?, + }) + } +} + +// constructor +impl Node for ast::ExprSet { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + elts, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprSet::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elts", elts.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + elts: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "elts", "Set")?, + )?, + range: range_from_object(vm, source_file, object, "Set")?, + }) + } +} + +// constructor +impl Node for ast::ExprListComp { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + elt, + generators, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprListComp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elt", elt.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("generators", generators.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + elt: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "elt", "ListComp")?, + )?, + generators: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "generators", "ListComp")?, + )?, + range: range_from_object(vm, source_file, object, "ListComp")?, + }) + } +} + +// constructor +impl Node for ast::ExprSetComp { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + elt, + generators, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprSetComp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elt", elt.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("generators", generators.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + elt: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "elt", "SetComp")?, + )?, + generators: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "generators", "SetComp")?, + )?, + range: range_from_object(vm, source_file, object, "SetComp")?, + }) + } +} + +// constructor +impl Node for ast::ExprDictComp { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + key, + value, + generators, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprDictComp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("key", key.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("generators", generators.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + key: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "key", "DictComp")?, + )?, + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "DictComp")?, + )?, + generators: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "generators", "DictComp")?, + )?, + range: range_from_object(vm, source_file, object, "DictComp")?, + }) + } +} + +// constructor +impl Node for ast::ExprGenerator { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + elt, + generators, + range, + parenthesized, + } = self; + let range = if parenthesized { + range + } else { + TextRange::new( + range + .start() + .saturating_sub(ruff_text_size::TextSize::from(1)), + range.end() + ruff_text_size::TextSize::from(1), + ) + }; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprGeneratorExp::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elt", elt.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("generators", generators.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + elt: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "elt", "GeneratorExp")?, + )?, + generators: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "generators", "GeneratorExp")?, + )?, + range: range_from_object(vm, source_file, object, "GeneratorExp")?, + // TODO: Is this correct? + parenthesized: true, + }) + } +} + +// constructor +impl Node for ast::ExprAwait { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprAwait::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "Await")?, + )?, + range: range_from_object(vm, source_file, object, "Await")?, + }) + } +} + +// constructor +impl Node for ast::ExprYield { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprYield::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: get_node_field_opt(vm, &object, "value")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "Yield")?, + }) + } +} + +// constructor +impl Node for ast::ExprYieldFrom { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprYieldFrom::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "YieldFrom")?, + )?, + range: range_from_object(vm, source_file, object, "YieldFrom")?, + }) + } +} + +// constructor +impl Node for ast::ExprCompare { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + left, + ops, + comparators, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprCompare::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("left", left.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("ops", BoxedSlice(ops).ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item( + "comparators", + BoxedSlice(comparators).ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + left: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "left", "Compare")?, + )?, + ops: { + let ops: BoxedSlice<_> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "ops", "Compare")?, + )?; + ops.0 + }, + comparators: { + let comparators: BoxedSlice<_> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "comparators", "Compare")?, + )?; + comparators.0 + }, + range: range_from_object(vm, source_file, object, "Compare")?, + }) + } +} + +// constructor +impl Node for ast::ExprCall { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + func, + arguments, + range, + } = self; + let (positional_arguments, keyword_arguments) = split_function_call_arguments(arguments); + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprCall::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("func", func.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item( + "args", + positional_arguments.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + dict.set_item( + "keywords", + keyword_arguments.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + func: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "func", "Call")?, + )?, + arguments: merge_function_call_arguments( + Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "args", "Call")?, + )?, + Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "keywords", "Call")?, + )?, + ), + range: range_from_object(vm, source_file, object, "Call")?, + }) + } +} + +// constructor +impl Node for ast::ExprAttribute { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + attr, + ctx, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprAttribute::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("attr", attr.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "Attribute")?, + )?, + attr: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "attr", "Attribute")?, + )?, + ctx: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "ctx", "Attribute")?, + )?, + range: range_from_object(vm, source_file, object, "Attribute")?, + }) + } +} + +// constructor +impl Node for ast::ExprSubscript { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + slice, + ctx, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprSubscript::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("slice", slice.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "Subscript")?, + )?, + slice: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "slice", "Subscript")?, + )?, + ctx: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "ctx", "Subscript")?, + )?, + range: range_from_object(vm, source_file, object, "Subscript")?, + }) + } +} + +// constructor +impl Node for ast::ExprStarred { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + ctx, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprStarred::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "Starred")?, + )?, + ctx: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "ctx", "Starred")?, + )?, + range: range_from_object(vm, source_file, object, "Starred")?, + }) + } +} + +// constructor +impl Node for ast::ExprName { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + id, + ctx, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprName::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("id", id.to_pyobject(vm), vm).unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + id: Node::ast_from_object(vm, source_file, get_node_field(vm, &object, "id", "Name")?)?, + ctx: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "ctx", "Name")?, + )?, + range: range_from_object(vm, source_file, object, "Name")?, + }) + } +} + +// constructor +impl Node for ast::ExprList { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + elts, + ctx, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprList::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elts", elts.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + elts: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "elts", "List")?, + )?, + ctx: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "ctx", "List")?, + )?, + range: range_from_object(vm, source_file, object, "List")?, + }) + } +} + +// constructor +impl Node for ast::ExprTuple { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + elts, + ctx, + range: _range, + parenthesized: _, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprTuple::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("elts", elts.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("ctx", ctx.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + elts: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "elts", "Tuple")?, + )?, + ctx: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "ctx", "Tuple")?, + )?, + range: range_from_object(vm, source_file, object, "Tuple")?, + parenthesized: true, // TODO: is this correct? + }) + } +} + +// constructor +impl Node for ast::ExprSlice { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + lower, + upper, + step, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprSlice::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("lower", lower.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("upper", upper.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("step", step.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + lower: get_node_field_opt(vm, &object, "lower")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + upper: get_node_field_opt(vm, &object, "upper")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + step: get_node_field_opt(vm, &object, "step")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "Slice")?, + }) + } +} + +// sum +impl Node for ast::ExprContext { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + let node_type = match self { + Self::Load => pyast::NodeExprContextLoad::static_type(), + Self::Store => pyast::NodeExprContextStore::static_type(), + Self::Del => pyast::NodeExprContextDel::static_type(), + Self::Invalid => { + unimplemented!("Invalid expression context is not allowed in Python AST") + } + }; + singleton_node_to_object(vm, node_type) + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeExprContextLoad::static_type()) { + Self::Load + } else if cls.is(pyast::NodeExprContextStore::static_type()) { + Self::Store + } else if cls.is(pyast::NodeExprContextDel::static_type()) { + Self::Del + } else { + return Err(vm.new_type_error(format!( + "expected some sort of expr_context, but got {}", + object.repr(vm)? + ))); + }) + } +} + +// product +impl Node for ast::Comprehension { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + target, + iter, + ifs, + is_async, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeComprehension::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("iter", iter.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("ifs", ifs.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("is_async", is_async.ast_to_object(vm, source_file), vm) + .unwrap(); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + target: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "target", "comprehension")?, + )?, + iter: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "iter", "comprehension")?, + )?, + ifs: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "ifs", "comprehension")?, + )?, + is_async: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "is_async", "comprehension")?, + )?, + range: Default::default(), + }) + } +} diff --git a/crates/vm/src/stdlib/_ast/module.rs b/crates/vm/src/stdlib/_ast/module.rs new file mode 100644 index 00000000000..b4c2468d33b --- /dev/null +++ b/crates/vm/src/stdlib/_ast/module.rs @@ -0,0 +1,236 @@ +use super::*; +use crate::stdlib::_ast::type_ignore::TypeIgnore; +use rustpython_compiler_core::SourceFile; + +/// Represents the different types of Python module structures. +/// +/// This enum is used to represent the various possible forms of a Python module +/// in an Abstract Syntax Tree (AST). It can correspond to: +/// +/// - `Module`: A standard Python script, containing a sequence of statements +/// (e.g., assignments, function calls), possibly with type ignores. +/// - `Interactive`: A representation of code executed in an interactive +/// Python session (e.g., the REPL or Jupyter notebooks), where statements +/// are evaluated one at a time. +/// - `Expression`: A single expression without any surrounding statements. +/// This is typically used in scenarios like `eval()` or in expression-only +/// contexts. +/// - `FunctionType`: A function signature with argument and return type +/// annotations, representing the type hints of a function (e.g., `def add(x: int, y: int) -> int`). +pub(super) enum Mod { + Module(ast::ModModule), + Interactive(ModInteractive), + Expression(ast::ModExpression), + FunctionType(ModFunctionType), +} + +// sum +impl Node for Mod { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::Module(cons) => cons.ast_to_object(vm, source_file), + Self::Interactive(cons) => cons.ast_to_object(vm, source_file), + Self::Expression(cons) => cons.ast_to_object(vm, source_file), + Self::FunctionType(cons) => cons.ast_to_object(vm, source_file), + } + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeModModule::static_type()) { + Self::Module(ast::ModModule::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeModInteractive::static_type()) { + Self::Interactive(ModInteractive::ast_from_object(vm, source_file, object)?) + } else if cls.is(pyast::NodeModExpression::static_type()) { + Self::Expression(ast::ModExpression::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodeModFunctionType::static_type()) { + Self::FunctionType(ModFunctionType::ast_from_object(vm, source_file, object)?) + } else { + return Err(vm.new_type_error(format!( + "expected some sort of mod, but got {}", + object.repr(vm)? + ))); + }) + } +} + +// constructor +impl Node for ast::ModModule { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + body, + // type_ignores, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModModule::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + // TODO: Improve ruff API + // ruff ignores type_ignore comments currently. + let type_ignores: Vec<TypeIgnore> = vec![]; + dict.set_item( + "type_ignores", + type_ignores.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + let _ = range; + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + body: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "body", "Module")?, + )?, + // type_ignores: Node::ast_from_object( + // _vm, + // get_node_field(_vm, &_object, "type_ignores", "Module")?, + // )?, + range: Default::default(), + }) + } +} + +pub(super) struct ModInteractive { + pub(crate) range: TextRange, + pub(crate) body: Vec<ast::Stmt>, +} + +// constructor +impl Node for ModInteractive { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { body, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModInteractive::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + let _ = range; + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + body: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "body", "Interactive")?, + )?, + range: Default::default(), + }) + } +} + +// constructor +impl Node for ast::ModExpression { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + body, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModExpression::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + let _ = range; + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + body: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "body", "Expression")?, + )?, + range: Default::default(), + }) + } +} + +pub(super) struct ModFunctionType { + pub(crate) argtypes: Box<[ast::Expr]>, + pub(crate) returns: ast::Expr, + pub(crate) range: TextRange, +} + +// constructor +impl Node for ModFunctionType { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + argtypes, + returns, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeModFunctionType::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "argtypes", + BoxedSlice(argtypes).ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + dict.set_item("returns", returns.ast_to_object(vm, source_file), vm) + .unwrap(); + let _ = range; + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + argtypes: { + let argtypes: BoxedSlice<_> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "argtypes", "FunctionType")?, + )?; + argtypes.0 + }, + returns: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "returns", "FunctionType")?, + )?, + range: Default::default(), + }) + } +} diff --git a/crates/vm/src/stdlib/_ast/node.rs b/crates/vm/src/stdlib/_ast/node.rs new file mode 100644 index 00000000000..6f74e71511f --- /dev/null +++ b/crates/vm/src/stdlib/_ast/node.rs @@ -0,0 +1,94 @@ +use crate::{PyObjectRef, PyResult, VirtualMachine}; +use rustpython_compiler_core::SourceFile; + +pub(crate) trait Node: Sized { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef; + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self>; + + /// Used in `Option::ast_from_object`; if `true`, that impl will return None. + fn is_none(&self) -> bool { + false + } +} + +impl<T: Node> Node for Vec<T> { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + vm.ctx + .new_list( + self.into_iter() + .map(|node| node.ast_to_object(vm, source_file)) + .collect(), + ) + .into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + vm.extract_elements_with(&object, |obj| Node::ast_from_object(vm, source_file, obj)) + } +} + +impl<T: Node> Node for Box<T> { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + (*self).ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + T::ast_from_object(vm, source_file, object).map(Self::new) + } + + fn is_none(&self) -> bool { + (**self).is_none() + } +} + +impl<T: Node> Node for Option<T> { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Some(node) => node.ast_to_object(vm, source_file), + None => vm.ctx.none(), + } + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + if vm.is_none(&object) { + Ok(None) + } else { + let x = T::ast_from_object(vm, source_file, object)?; + Ok((!x.is_none()).then_some(x)) + } + } +} + +pub(super) struct BoxedSlice<T>(pub(super) Box<[T]>); + +impl<T: Node> Node for BoxedSlice<T> { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + self.0.into_vec().ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self( + <Vec<T> as Node>::ast_from_object(vm, source_file, object)?.into_boxed_slice(), + )) + } +} diff --git a/crates/vm/src/stdlib/_ast/operator.rs b/crates/vm/src/stdlib/_ast/operator.rs new file mode 100644 index 00000000000..09e63b5d6ce --- /dev/null +++ b/crates/vm/src/stdlib/_ast/operator.rs @@ -0,0 +1,181 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +// sum +impl Node for ast::BoolOp { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + let node_type = match self { + Self::And => pyast::NodeBoolOpAnd::static_type(), + Self::Or => pyast::NodeBoolOpOr::static_type(), + }; + singleton_node_to_object(vm, node_type) + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeBoolOpAnd::static_type()) { + Self::And + } else if cls.is(pyast::NodeBoolOpOr::static_type()) { + Self::Or + } else { + return Err(vm.new_type_error(format!( + "expected some sort of boolop, but got {}", + object.repr(vm)? + ))); + }) + } +} + +// sum +impl Node for ast::Operator { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + let node_type = match self { + Self::Add => pyast::NodeOperatorAdd::static_type(), + Self::Sub => pyast::NodeOperatorSub::static_type(), + Self::Mult => pyast::NodeOperatorMult::static_type(), + Self::MatMult => pyast::NodeOperatorMatMult::static_type(), + Self::Div => pyast::NodeOperatorDiv::static_type(), + Self::Mod => pyast::NodeOperatorMod::static_type(), + Self::Pow => pyast::NodeOperatorPow::static_type(), + Self::LShift => pyast::NodeOperatorLShift::static_type(), + Self::RShift => pyast::NodeOperatorRShift::static_type(), + Self::BitOr => pyast::NodeOperatorBitOr::static_type(), + Self::BitXor => pyast::NodeOperatorBitXor::static_type(), + Self::BitAnd => pyast::NodeOperatorBitAnd::static_type(), + Self::FloorDiv => pyast::NodeOperatorFloorDiv::static_type(), + }; + singleton_node_to_object(vm, node_type) + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeOperatorAdd::static_type()) { + Self::Add + } else if cls.is(pyast::NodeOperatorSub::static_type()) { + Self::Sub + } else if cls.is(pyast::NodeOperatorMult::static_type()) { + Self::Mult + } else if cls.is(pyast::NodeOperatorMatMult::static_type()) { + Self::MatMult + } else if cls.is(pyast::NodeOperatorDiv::static_type()) { + Self::Div + } else if cls.is(pyast::NodeOperatorMod::static_type()) { + Self::Mod + } else if cls.is(pyast::NodeOperatorPow::static_type()) { + Self::Pow + } else if cls.is(pyast::NodeOperatorLShift::static_type()) { + Self::LShift + } else if cls.is(pyast::NodeOperatorRShift::static_type()) { + Self::RShift + } else if cls.is(pyast::NodeOperatorBitOr::static_type()) { + Self::BitOr + } else if cls.is(pyast::NodeOperatorBitXor::static_type()) { + Self::BitXor + } else if cls.is(pyast::NodeOperatorBitAnd::static_type()) { + Self::BitAnd + } else if cls.is(pyast::NodeOperatorFloorDiv::static_type()) { + Self::FloorDiv + } else { + return Err(vm.new_type_error(format!( + "expected some sort of operator, but got {}", + object.repr(vm)? + ))); + }) + } +} + +// sum +impl Node for ast::UnaryOp { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + let node_type = match self { + Self::Invert => pyast::NodeUnaryOpInvert::static_type(), + Self::Not => pyast::NodeUnaryOpNot::static_type(), + Self::UAdd => pyast::NodeUnaryOpUAdd::static_type(), + Self::USub => pyast::NodeUnaryOpUSub::static_type(), + }; + singleton_node_to_object(vm, node_type) + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeUnaryOpInvert::static_type()) { + Self::Invert + } else if cls.is(pyast::NodeUnaryOpNot::static_type()) { + Self::Not + } else if cls.is(pyast::NodeUnaryOpUAdd::static_type()) { + Self::UAdd + } else if cls.is(pyast::NodeUnaryOpUSub::static_type()) { + Self::USub + } else { + return Err(vm.new_type_error(format!( + "expected some sort of unaryop, but got {}", + object.repr(vm)? + ))); + }) + } +} + +// sum +impl Node for ast::CmpOp { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + let node_type = match self { + Self::Eq => pyast::NodeCmpOpEq::static_type(), + Self::NotEq => pyast::NodeCmpOpNotEq::static_type(), + Self::Lt => pyast::NodeCmpOpLt::static_type(), + Self::LtE => pyast::NodeCmpOpLtE::static_type(), + Self::Gt => pyast::NodeCmpOpGt::static_type(), + Self::GtE => pyast::NodeCmpOpGtE::static_type(), + Self::Is => pyast::NodeCmpOpIs::static_type(), + Self::IsNot => pyast::NodeCmpOpIsNot::static_type(), + Self::In => pyast::NodeCmpOpIn::static_type(), + Self::NotIn => pyast::NodeCmpOpNotIn::static_type(), + }; + singleton_node_to_object(vm, node_type) + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeCmpOpEq::static_type()) { + Self::Eq + } else if cls.is(pyast::NodeCmpOpNotEq::static_type()) { + Self::NotEq + } else if cls.is(pyast::NodeCmpOpLt::static_type()) { + Self::Lt + } else if cls.is(pyast::NodeCmpOpLtE::static_type()) { + Self::LtE + } else if cls.is(pyast::NodeCmpOpGt::static_type()) { + Self::Gt + } else if cls.is(pyast::NodeCmpOpGtE::static_type()) { + Self::GtE + } else if cls.is(pyast::NodeCmpOpIs::static_type()) { + Self::Is + } else if cls.is(pyast::NodeCmpOpIsNot::static_type()) { + Self::IsNot + } else if cls.is(pyast::NodeCmpOpIn::static_type()) { + Self::In + } else if cls.is(pyast::NodeCmpOpNotIn::static_type()) { + Self::NotIn + } else { + return Err(vm.new_type_error(format!( + "expected some sort of cmpop, but got {}", + object.repr(vm)? + ))); + }) + } +} diff --git a/crates/vm/src/stdlib/_ast/other.rs b/crates/vm/src/stdlib/_ast/other.rs new file mode 100644 index 00000000000..5c0803ac594 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/other.rs @@ -0,0 +1,151 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +impl Node for ast::ConversionFlag { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + vm.ctx.new_int(self as i8).into() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + // Python's AST uses ASCII codes: 's', 'r', 'a', -1=None + // Note: 255 is -1i8 as u8 (ruff's ConversionFlag::None) + match i32::try_from_object(vm, object)? { + -1 | 255 => Ok(Self::None), + x if x == b's' as i32 => Ok(Self::Str), + x if x == b'r' as i32 => Ok(Self::Repr), + x if x == b'a' as i32 => Ok(Self::Ascii), + _ => Err(vm.new_value_error("invalid conversion flag")), + } + } +} + +// /// This is just a string, not strictly an AST node. But it makes AST conversions easier. +impl Node for ast::name::Name { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + vm.ctx.intern_str(self.as_str()).to_object() + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + match object.downcast::<PyStr>() { + Ok(name) => Ok(Self::new(name)), + Err(_) => Err(vm.new_value_error("expected str for name")), + } + } +} + +impl Node for ast::Decorator { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + ast::Expr::ast_to_object(self.expression, vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let expression = ast::Expr::ast_from_object(vm, source_file, object)?; + let range = expression.range(); + Ok(Self { + node_index: Default::default(), + expression, + range, + }) + } +} + +// product +impl Node for ast::Alias { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + asname, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeAlias::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("asname", asname.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, _range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + name: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "name", "alias")?, + )?, + asname: get_node_field_opt(vm, &object, "asname")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "alias")?, + }) + } +} + +// product +impl Node for ast::WithItem { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + context_expr, + optional_vars, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeWithItem::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "context_expr", + context_expr.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + dict.set_item( + "optional_vars", + optional_vars.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + context_expr: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "context_expr", "withitem")?, + )?, + optional_vars: get_node_field_opt(vm, &object, "optional_vars")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: Default::default(), + }) + } +} diff --git a/crates/vm/src/stdlib/_ast/parameter.rs b/crates/vm/src/stdlib/_ast/parameter.rs new file mode 100644 index 00000000000..15ff237e50d --- /dev/null +++ b/crates/vm/src/stdlib/_ast/parameter.rs @@ -0,0 +1,426 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +// product +impl Node for ast::Parameters { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + posonlyargs, + args, + vararg, + kwonlyargs, + kwarg, + range, + } = self; + let (posonlyargs, args, defaults) = + extract_positional_parameter_defaults(posonlyargs, args); + let (kwonlyargs, kw_defaults) = extract_keyword_parameter_defaults(kwonlyargs); + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeArguments::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "posonlyargs", + posonlyargs.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + dict.set_item("args", args.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("vararg", vararg.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("kwonlyargs", kwonlyargs.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item( + "kw_defaults", + kw_defaults.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + dict.set_item("kwarg", kwarg.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("defaults", defaults.ast_to_object(vm, source_file), vm) + .unwrap(); + let _ = range; + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let kwonlyargs = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "kwonlyargs", "arguments")?, + )?; + let kw_defaults = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "kw_defaults", "arguments")?, + )?; + let kwonlyargs = merge_keyword_parameter_defaults(vm, kwonlyargs, kw_defaults)?; + + let posonlyargs = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "posonlyargs", "arguments")?, + )?; + let args = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "args", "arguments")?, + )?; + let defaults = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "defaults", "arguments")?, + )?; + let (posonlyargs, args) = + merge_positional_parameter_defaults(vm, posonlyargs, args, defaults)?; + + Ok(Self { + node_index: Default::default(), + posonlyargs, + args, + vararg: get_node_field_opt(vm, &object, "vararg")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + kwonlyargs, + kwarg: get_node_field_opt(vm, &object, "kwarg")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: Default::default(), + }) + } + + fn is_none(&self) -> bool { + self.is_empty() + } +} + +// product +impl Node for ast::Parameter { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + annotation, + // type_comment, + range, + } = self; + + // ruff covers the ** in range but python expects it to start at the ident + let range = TextRange::new(name.start(), range.end()); + + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeArg::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("arg", name.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item( + "annotation", + annotation.ast_to_object(_vm, source_file), + _vm, + ) + .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); + node_add_location(&dict, range, _vm, source_file); + node.into() + } + + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + name: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "arg", "arg")?, + )?, + annotation: get_node_field_opt(_vm, &_object, "annotation")? + .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + .transpose()?, + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(_vm, source_file, _object, "arg")?, + }) + } +} + +// product +impl Node for ast::Keyword { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + arg, + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeKeyword::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("arg", arg.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + arg: get_node_field_opt(_vm, &_object, "arg")? + .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + .transpose()?, + value: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "value", "keyword")?, + )?, + range: range_from_object(_vm, source_file, _object, "keyword")?, + }) + } +} + +struct PositionalParameters { + pub _range: TextRange, // TODO: Use this + pub args: Box<[ast::Parameter]>, +} + +impl Node for PositionalParameters { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + BoxedSlice(self.args).ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let args: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; + Ok(Self { + args: args.0, + _range: TextRange::default(), // TODO + }) + } +} + +struct KeywordParameters { + pub _range: TextRange, // TODO: Use this + pub keywords: Box<[ast::Parameter]>, +} + +impl Node for KeywordParameters { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + BoxedSlice(self.keywords).ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let keywords: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; + Ok(Self { + keywords: keywords.0, + _range: TextRange::default(), // TODO + }) + } +} + +struct ParameterDefaults { + pub _range: TextRange, // TODO: Use this + defaults: Box<[Option<Box<ast::Expr>>]>, +} + +impl Node for ParameterDefaults { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + BoxedSlice(self.defaults).ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let defaults: BoxedSlice<_> = Node::ast_from_object(vm, source_file, object)?; + Ok(Self { + defaults: defaults.0, + _range: TextRange::default(), // TODO + }) + } +} + +fn extract_positional_parameter_defaults( + pos_only_args: Vec<ast::ParameterWithDefault>, + args: Vec<ast::ParameterWithDefault>, +) -> ( + PositionalParameters, + PositionalParameters, + ParameterDefaults, +) { + let mut defaults = vec![]; + defaults.extend(pos_only_args.iter().map(|item| item.default.clone())); + defaults.extend(args.iter().map(|item| item.default.clone())); + // If some positional parameters have no default value, + // the "defaults" list contains only the defaults of the last "n" parameters. + // Remove all positional parameters without a default value. + defaults.retain(Option::is_some); + let defaults = ParameterDefaults { + _range: defaults + .iter() + .flatten() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + defaults: defaults.into_boxed_slice(), + }; + + let pos_only_args = PositionalParameters { + _range: pos_only_args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + args: { + let pos_only_args: Vec<_> = pos_only_args + .iter() + .map(|item| item.parameter.clone()) + .collect(); + pos_only_args.into_boxed_slice() + }, + }; + + let args = PositionalParameters { + _range: args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + args: { + let args: Vec<_> = args.iter().map(|item| item.parameter.clone()).collect(); + args.into_boxed_slice() + }, + }; + + (pos_only_args, args, defaults) +} + +/// Merges the keyword parameters with their default values, opposite of [`extract_positional_parameter_defaults`]. +fn merge_positional_parameter_defaults( + vm: &VirtualMachine, + posonlyargs: PositionalParameters, + args: PositionalParameters, + defaults: ParameterDefaults, +) -> PyResult<( + Vec<ast::ParameterWithDefault>, + Vec<ast::ParameterWithDefault>, +)> { + let posonlyargs = posonlyargs.args; + let args = args.args; + let defaults = defaults.defaults; + + let mut posonlyargs: Vec<_> = <Box<[_]> as IntoIterator>::into_iter(posonlyargs) + .map(|parameter| ast::ParameterWithDefault { + node_index: Default::default(), + range: Default::default(), + parameter, + default: None, + }) + .collect(); + let mut args: Vec<_> = <Box<[_]> as IntoIterator>::into_iter(args) + .map(|parameter| ast::ParameterWithDefault { + node_index: Default::default(), + range: Default::default(), + parameter, + default: None, + }) + .collect(); + + // If an argument has a default value, insert it + // Note that "defaults" will only contain default values for the last "n" parameters + // so we need to skip the first "total_argument_count - n" arguments. + let total_args = posonlyargs.len() + args.len(); + if defaults.len() > total_args { + return Err(vm.new_value_error("more positional defaults than args on arguments")); + } + let default_argument_count = total_args - defaults.len(); + for (arg, default) in posonlyargs + .iter_mut() + .chain(args.iter_mut()) + .skip(default_argument_count) + .zip(defaults) + { + arg.default = default; + } + + Ok((posonlyargs, args)) +} + +fn extract_keyword_parameter_defaults( + kw_only_args: Vec<ast::ParameterWithDefault>, +) -> (KeywordParameters, ParameterDefaults) { + let mut defaults = vec![]; + defaults.extend(kw_only_args.iter().map(|item| item.default.clone())); + let defaults = ParameterDefaults { + _range: defaults + .iter() + .flatten() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + defaults: defaults.into_boxed_slice(), + }; + + let kw_only_args = KeywordParameters { + _range: kw_only_args + .iter() + .map(|item| item.range()) + .reduce(|acc, next| acc.cover(next)) + .unwrap_or_default(), + keywords: { + let kw_only_args: Vec<_> = kw_only_args + .iter() + .map(|item| item.parameter.clone()) + .collect(); + kw_only_args.into_boxed_slice() + }, + }; + + (kw_only_args, defaults) +} + +/// Merges the keyword parameters with their default values, opposite of [`extract_keyword_parameter_defaults`]. +fn merge_keyword_parameter_defaults( + vm: &VirtualMachine, + kw_only_args: KeywordParameters, + defaults: ParameterDefaults, +) -> PyResult<Vec<ast::ParameterWithDefault>> { + if kw_only_args.keywords.len() != defaults.defaults.len() { + return Err( + vm.new_value_error("length of kwonlyargs is not the same as kw_defaults on arguments") + ); + } + Ok(core::iter::zip(kw_only_args.keywords, defaults.defaults) + .map(|(parameter, default)| ast::ParameterWithDefault { + node_index: Default::default(), + parameter, + default, + range: Default::default(), + }) + .collect()) +} diff --git a/crates/vm/src/stdlib/_ast/pattern.rs b/crates/vm/src/stdlib/_ast/pattern.rs new file mode 100644 index 00000000000..621c849e812 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/pattern.rs @@ -0,0 +1,578 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +// product +impl Node for ast::MatchCase { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + pattern, + guard, + body, + range: _, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeMatchCase::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("pattern", pattern.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("guard", guard.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + pattern: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "pattern", "match_case")?, + )?, + guard: get_node_field_opt(vm, &object, "guard")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + body: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "body", "match_case")?, + )?, + range: Default::default(), + }) + } +} + +// sum +impl Node for ast::Pattern { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::MatchValue(cons) => cons.ast_to_object(vm, source_file), + Self::MatchSingleton(cons) => cons.ast_to_object(vm, source_file), + Self::MatchSequence(cons) => cons.ast_to_object(vm, source_file), + Self::MatchMapping(cons) => cons.ast_to_object(vm, source_file), + Self::MatchClass(cons) => cons.ast_to_object(vm, source_file), + Self::MatchStar(cons) => cons.ast_to_object(vm, source_file), + Self::MatchAs(cons) => cons.ast_to_object(vm, source_file), + Self::MatchOr(cons) => cons.ast_to_object(vm, source_file), + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodePatternMatchValue::static_type()) { + Self::MatchValue(ast::PatternMatchValue::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodePatternMatchSingleton::static_type()) { + Self::MatchSingleton(ast::PatternMatchSingleton::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodePatternMatchSequence::static_type()) { + Self::MatchSequence(ast::PatternMatchSequence::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodePatternMatchMapping::static_type()) { + Self::MatchMapping(ast::PatternMatchMapping::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodePatternMatchClass::static_type()) { + Self::MatchClass(ast::PatternMatchClass::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodePatternMatchStar::static_type()) { + Self::MatchStar(ast::PatternMatchStar::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodePatternMatchAs::static_type()) { + Self::MatchAs(ast::PatternMatchAs::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodePatternMatchOr::static_type()) { + Self::MatchOr(ast::PatternMatchOr::ast_from_object( + vm, + source_file, + object, + )?) + } else { + return Err(vm.new_type_error(format!( + "expected some sort of pattern, but got {}", + object.repr(vm)? + ))); + }) + } +} +// constructor +impl Node for ast::PatternMatchValue { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodePatternMatchValue::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "MatchValue")?, + )?, + range: range_from_object(vm, source_file, object, "MatchValue")?, + }) + } +} + +// constructor +impl Node for ast::PatternMatchSingleton { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + range, + } = self; + let node = NodeAst + .into_ref_with_type( + vm, + pyast::NodePatternMatchSingleton::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "MatchSingleton")?, + )?, + range: range_from_object(vm, source_file, object, "MatchSingleton")?, + }) + } +} + +impl Node for ast::Singleton { + fn ast_to_object(self, vm: &VirtualMachine, _source_file: &SourceFile) -> PyObjectRef { + match self { + ast::Singleton::None => vm.ctx.none(), + ast::Singleton::True => vm.ctx.new_bool(true).into(), + ast::Singleton::False => vm.ctx.new_bool(false).into(), + } + } + + fn ast_from_object( + vm: &VirtualMachine, + _source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + if vm.is_none(&object) { + Ok(ast::Singleton::None) + } else if object.is(&vm.ctx.true_value) { + Ok(ast::Singleton::True) + } else if object.is(&vm.ctx.false_value) { + Ok(ast::Singleton::False) + } else { + Err(vm.new_value_error(format!( + "Expected None, True, or False, got {:?}", + object.class().name() + ))) + } + } +} + +// constructor +impl Node for ast::PatternMatchSequence { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + patterns, + range, + } = self; + let node = NodeAst + .into_ref_with_type( + vm, + pyast::NodePatternMatchSequence::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("patterns", patterns.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + patterns: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "patterns", "MatchSequence")?, + )?, + range: range_from_object(vm, source_file, object, "MatchSequence")?, + }) + } +} + +// constructor +impl Node for ast::PatternMatchMapping { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + keys, + patterns, + rest, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodePatternMatchMapping::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("keys", keys.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("patterns", patterns.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("rest", rest.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + keys: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "keys", "MatchMapping")?, + )?, + patterns: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "patterns", "MatchMapping")?, + )?, + rest: get_node_field_opt(vm, &object, "rest")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "MatchMapping")?, + }) + } +} + +// constructor +impl Node for ast::PatternMatchClass { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + cls, + arguments, + range, + } = self; + let (patterns, kwd_attrs, kwd_patterns) = split_pattern_match_class(arguments); + let node = NodeAst + .into_ref_with_type(vm, pyast::NodePatternMatchClass::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("cls", cls.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("patterns", patterns.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("kwd_attrs", kwd_attrs.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item( + "kwd_patterns", + kwd_patterns.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let patterns = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "patterns", "MatchClass")?, + )?; + let kwd_attrs: PatternMatchClassKeywordAttributes = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "kwd_attrs", "MatchClass")?, + )?; + let kwd_patterns: PatternMatchClassKeywordPatterns = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "kwd_patterns", "MatchClass")?, + )?; + if kwd_attrs.0.len() != kwd_patterns.0.len() { + return Err(vm.new_value_error("MatchClass has mismatched kwd_attrs and kwd_patterns")); + } + let (patterns, keywords) = merge_pattern_match_class(patterns, kwd_attrs, kwd_patterns); + + Ok(Self { + node_index: Default::default(), + cls: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "cls", "MatchClass")?, + )?, + range: range_from_object(vm, source_file, object, "MatchClass")?, + arguments: ast::PatternArguments { + node_index: Default::default(), + range: Default::default(), + patterns, + keywords, + }, + }) + } +} + +struct PatternMatchClassPatterns(Vec<ast::Pattern>); + +impl Node for PatternMatchClassPatterns { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + self.0.ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self(Node::ast_from_object(vm, source_file, object)?)) + } +} + +struct PatternMatchClassKeywordAttributes(Vec<ast::Identifier>); + +impl Node for PatternMatchClassKeywordAttributes { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + self.0.ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self(Node::ast_from_object(vm, source_file, object)?)) + } +} + +struct PatternMatchClassKeywordPatterns(Vec<ast::Pattern>); + +impl Node for PatternMatchClassKeywordPatterns { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + self.0.ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self(Node::ast_from_object(vm, source_file, object)?)) + } +} +// constructor +impl Node for ast::PatternMatchStar { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodePatternMatchStar::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + name: get_node_field_opt(vm, &object, "name")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "MatchStar")?, + }) + } +} + +// constructor +impl Node for ast::PatternMatchAs { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + pattern, + name, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodePatternMatchAs::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("pattern", pattern.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + pattern: get_node_field_opt(vm, &object, "pattern")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + name: get_node_field_opt(vm, &object, "name")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "MatchAs")?, + }) + } +} + +// constructor +impl Node for ast::PatternMatchOr { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + patterns, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodePatternMatchOr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("patterns", patterns.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + patterns: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "patterns", "MatchOr")?, + )?, + range: range_from_object(vm, source_file, object, "MatchOr")?, + }) + } +} + +fn split_pattern_match_class( + arguments: ast::PatternArguments, +) -> ( + PatternMatchClassPatterns, + PatternMatchClassKeywordAttributes, + PatternMatchClassKeywordPatterns, +) { + let patterns = PatternMatchClassPatterns(arguments.patterns); + let kwd_attrs = PatternMatchClassKeywordAttributes( + arguments.keywords.iter().map(|k| k.attr.clone()).collect(), + ); + let kwd_patterns = PatternMatchClassKeywordPatterns( + arguments.keywords.into_iter().map(|k| k.pattern).collect(), + ); + (patterns, kwd_attrs, kwd_patterns) +} + +/// Merges the pattern match class attributes and patterns, opposite of [`split_pattern_match_class`]. +fn merge_pattern_match_class( + patterns: PatternMatchClassPatterns, + kwd_attrs: PatternMatchClassKeywordAttributes, + kwd_patterns: PatternMatchClassKeywordPatterns, +) -> (Vec<ast::Pattern>, Vec<ast::PatternKeyword>) { + let keywords = kwd_attrs + .0 + .into_iter() + .zip(kwd_patterns.0) + .map(|(attr, pattern)| ast::PatternKeyword { + range: Default::default(), + node_index: Default::default(), + attr, + pattern, + }) + .collect(); + (patterns.0, keywords) +} diff --git a/crates/vm/src/stdlib/_ast/pyast.rs b/crates/vm/src/stdlib/_ast/pyast.rs new file mode 100644 index 00000000000..4b6e97e13fc --- /dev/null +++ b/crates/vm/src/stdlib/_ast/pyast.rs @@ -0,0 +1,1835 @@ +#![allow(clippy::all)] + +use super::*; +use crate::builtins::{PyGenericAlias, PyTuple, PyTupleRef, PyTypeRef, make_union}; +use crate::common::ascii; +use crate::convert::ToPyObject; +use crate::function::FuncArgs; +use crate::types::Initializer; + +macro_rules! impl_node { + ( + #[pyclass(module = $_mod:literal, name = $_name:literal, base = $base:ty)] + $vis:vis struct $name:ident, + fields: [$($field:expr),* $(,)?], + attributes: [$($attr:expr),* $(,)?] $(,)? + ) => { + #[pyclass(module = $_mod, name = $_name, base = $base)] + #[repr(transparent)] + $vis struct $name($base); + + impl_base_node!($name, fields: [$($field),*], attributes: [$($attr),*]); + }; + // Without attributes + ( + #[pyclass(module = $_mod:literal, name = $_name:literal, base = $base:ty)] + $vis:vis struct $name:ident, + fields: [$($field:expr),* $(,)?] $(,)? + ) => { + impl_node!( + #[pyclass(module = $_mod, name = $_name, base = $base)] + $vis struct $name, + fields: [$($field),*], + attributes: [], + ); + }; + // Without fields + ( + #[pyclass(module = $_mod:literal, name = $_name:literal, base = $base:ty)] + $vis:vis struct $name:ident, + attributes: [$($attr:expr),* $(,)?] $(,)? + ) => { + impl_node!( + #[pyclass(module = $_mod, name = $_name, base = $base)] + $vis struct $name, + fields: [], + attributes: [$($attr),*], + ); + }; + // Without fields and attributes + ( + #[pyclass(module = $_mod:literal, name = $_name:literal, base = $base:ty)] + $vis:vis struct $name:ident $(,)? + ) => { + impl_node!( + #[pyclass(module = $_mod, name = $_name, base = $base)] + $vis struct $name, + fields: [], + attributes: [], + ); + }; +} + +macro_rules! impl_base_node { + // Base node without fields/attributes (e.g. NodeMod, NodeExpr) + ($name:ident) => { + #[pyclass(flags(HAS_DICT, BASETYPE))] + impl $name { + #[pymethod] + fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + super::python::_ast::ast_reduce(zelf, vm) + } + + #[pymethod] + fn __replace__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + super::python::_ast::ast_replace(zelf, args, vm) + } + + #[extend_class] + fn extend_class(ctx: &Context, class: &'static Py<PyType>) { + // AST types are mutable (heap types in CPython, not IMMUTABLETYPE) + // Safety: called during type initialization before any concurrent access + unsafe { + let flags = &class.slots.flags as *const crate::types::PyTypeFlags + as *mut crate::types::PyTypeFlags; + (*flags).remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + } + class.set_attr( + identifier!(ctx, _attributes), + ctx.empty_tuple.clone().into(), + ); + } + } + }; + // Leaf node with fields and attributes + ($name:ident, fields: [$($field:expr),*], attributes: [$($attr:expr),*]) => { + #[pyclass(flags(HAS_DICT, BASETYPE))] + impl $name { + #[pymethod] + fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + super::python::_ast::ast_reduce(zelf, vm) + } + + #[pymethod] + fn __replace__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + super::python::_ast::ast_replace(zelf, args, vm) + } + + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { + // AST types are mutable (heap types in CPython, not IMMUTABLETYPE) + // Safety: called during type initialization before any concurrent access + unsafe { + let flags = &class.slots.flags as *const crate::types::PyTypeFlags + as *mut crate::types::PyTypeFlags; + (*flags).remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + } + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + $( + ctx.new_str(ascii!($field)).into() + ),* + ]) + .into(), + ); + + class.set_str_attr( + "__match_args__", + ctx.new_tuple(vec![ + $( + ctx.new_str(ascii!($field)).into() + ),* + ]), + ctx, + ); + + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_tuple(vec![ + $( + ctx.new_str(ascii!($attr)).into() + ),* + ]) + .into(), + ); + + // Signal that this is a built-in AST node with field defaults + class.set_attr( + ctx.intern_str("_field_types"), + ctx.new_dict().into(), + ); + } + } + }; +} + +#[pyclass(module = "_ast", name = "mod", base = NodeAst)] +pub(crate) struct NodeMod(NodeAst); + +impl_base_node!(NodeMod); + +impl_node!( + #[pyclass(module = "_ast", name = "Module", base = NodeMod)] + pub(crate) struct NodeModModule, + fields: ["body", "type_ignores"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Interactive", base = NodeMod)] + pub(crate) struct NodeModInteractive, + fields: ["body"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Expression", base = NodeMod)] + pub(crate) struct NodeModExpression, + fields: ["body"], +); + +#[pyclass(module = "_ast", name = "stmt", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeStmt(NodeAst); + +impl_base_node!(NodeStmt); + +impl_node!( + #[pyclass(module = "_ast", name = "FunctionType", base = NodeMod)] + pub(crate) struct NodeModFunctionType, + fields: ["argtypes", "returns"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "FunctionDef", base = NodeStmt)] + pub(crate) struct NodeStmtFunctionDef, + fields: ["name", "args", "body", "decorator_list", "returns", "type_comment", "type_params"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "AsyncFunctionDef", base = NodeStmt)] + pub(crate) struct NodeStmtAsyncFunctionDef, + fields: ["name", "args", "body", "decorator_list", "returns", "type_comment", "type_params"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "ClassDef", base = NodeStmt)] + pub(crate) struct NodeStmtClassDef, + fields: ["name", "bases", "keywords", "body", "decorator_list", "type_params"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Return", base = NodeStmt)] + pub(crate) struct NodeStmtReturn, + fields: ["value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Delete", base = NodeStmt)] + pub(crate) struct NodeStmtDelete, + fields: ["targets"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Assign", base = NodeStmt)] + pub(crate) struct NodeStmtAssign, + fields: ["targets", "value", "type_comment"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "TypeAlias", base = NodeStmt)] + pub(crate) struct NodeStmtTypeAlias, + fields: ["name", "type_params", "value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "AugAssign", base = NodeStmt)] + pub(crate) struct NodeStmtAugAssign, + fields: ["target", "op", "value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "AnnAssign", base = NodeStmt)] + pub(crate) struct NodeStmtAnnAssign, + fields: ["target", "annotation", "value", "simple"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "For", base = NodeStmt)] + pub(crate) struct NodeStmtFor, + fields: ["target", "iter", "body", "orelse", "type_comment"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "AsyncFor", base = NodeStmt)] + pub(crate) struct NodeStmtAsyncFor, + fields: ["target", "iter", "body", "orelse", "type_comment"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "While", base = NodeStmt)] + pub(crate) struct NodeStmtWhile, + fields: ["test", "body", "orelse"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "If", base = NodeStmt)] + pub(crate) struct NodeStmtIf, + fields: ["test", "body", "orelse"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "With", base = NodeStmt)] + pub(crate) struct NodeStmtWith, + fields: ["items", "body", "type_comment"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "AsyncWith", base = NodeStmt)] + pub(crate) struct NodeStmtAsyncWith, + fields: ["items", "body", "type_comment"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Match", base = NodeStmt)] + pub(crate) struct NodeStmtMatch, + fields: ["subject", "cases"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Raise", base = NodeStmt)] + pub(crate) struct NodeStmtRaise, + fields: ["exc", "cause"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Try", base = NodeStmt)] + pub(crate) struct NodeStmtTry, + fields: ["body", "handlers", "orelse", "finalbody"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "TryStar", base = NodeStmt)] + pub(crate) struct NodeStmtTryStar, + fields: ["body", "handlers", "orelse", "finalbody"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Assert", base = NodeStmt)] + pub(crate) struct NodeStmtAssert, + fields: ["test", "msg"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Import", base = NodeStmt)] + pub(crate) struct NodeStmtImport, + fields: ["names"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "ImportFrom", base = NodeStmt)] + pub(crate) struct NodeStmtImportFrom, + fields: ["module", "names", "level"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Global", base = NodeStmt)] + pub(crate) struct NodeStmtGlobal, + fields: ["names"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Nonlocal", base = NodeStmt)] + pub(crate) struct NodeStmtNonlocal, + fields: ["names"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Expr", base = NodeStmt)] + pub(crate) struct NodeStmtExpr, + fields: ["value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Pass", base = NodeStmt)] + pub(crate) struct NodeStmtPass, + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Break", base = NodeStmt)] + pub(crate) struct NodeStmtBreak, + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +#[pyclass(module = "_ast", name = "expr", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeExpr(NodeAst); + +impl_base_node!(NodeExpr); + +impl_node!( + #[pyclass(module = "_ast", name = "Continue", base = NodeStmt)] + pub(crate) struct NodeStmtContinue, + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "BoolOp", base = NodeExpr)] + pub(crate) struct NodeExprBoolOp, + fields: ["op", "values"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "NamedExpr", base = NodeExpr)] + pub(crate) struct NodeExprNamedExpr, + fields: ["target", "value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "BinOp", base = NodeExpr)] + pub(crate) struct NodeExprBinOp, + fields: ["left", "op", "right"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "UnaryOp", base = NodeExpr)] + pub(crate) struct NodeExprUnaryOp, + fields: ["op", "operand"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Lambda", base = NodeExpr)] + pub(crate) struct NodeExprLambda, + fields: ["args", "body"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "IfExp", base = NodeExpr)] + pub(crate) struct NodeExprIfExp, + fields: ["test", "body", "orelse"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Dict", base = NodeExpr)] + pub(crate) struct NodeExprDict, + fields: ["keys", "values"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Set", base = NodeExpr)] + pub(crate) struct NodeExprSet, + fields: ["elts"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "ListComp", base = NodeExpr)] + pub(crate) struct NodeExprListComp, + fields: ["elt", "generators"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "SetComp", base = NodeExpr)] + pub(crate) struct NodeExprSetComp, + fields: ["elt", "generators"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "DictComp", base = NodeExpr)] + pub(crate) struct NodeExprDictComp, + fields: ["key", "value", "generators"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "GeneratorExp", base = NodeExpr)] + pub(crate) struct NodeExprGeneratorExp, + fields: ["elt", "generators"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Await", base = NodeExpr)] + pub(crate) struct NodeExprAwait, + fields: ["value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Yield", base = NodeExpr)] + pub(crate) struct NodeExprYield, + fields: ["value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "YieldFrom", base = NodeExpr)] + pub(crate) struct NodeExprYieldFrom, + fields: ["value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Compare", base = NodeExpr)] + pub(crate) struct NodeExprCompare, + fields: ["left", "ops", "comparators"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Call", base = NodeExpr)] + pub(crate) struct NodeExprCall, + fields: ["func", "args", "keywords"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "FormattedValue", base = NodeExpr)] + pub(crate) struct NodeExprFormattedValue, + fields: ["value", "conversion", "format_spec"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "JoinedStr", base = NodeExpr)] + pub(crate) struct NodeExprJoinedStr, + fields: ["values"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "TemplateStr", base = NodeExpr)] + pub(crate) struct NodeExprTemplateStr, + fields: ["values"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Interpolation", base = NodeExpr)] + pub(crate) struct NodeExprInterpolation, + fields: ["value", "str", "conversion", "format_spec"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +// NodeExprConstant needs custom Initializer to default kind to None +#[pyclass(module = "_ast", name = "Constant", base = NodeExpr)] +#[repr(transparent)] +pub(crate) struct NodeExprConstant(NodeExpr); + +#[pyclass(flags(HAS_DICT, BASETYPE), with(Initializer))] +impl NodeExprConstant { + #[extend_class] + fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { + // AST types are mutable (heap types, not IMMUTABLETYPE) + // Safety: called during type initialization before any concurrent access + unsafe { + let flags = &class.slots.flags as *const crate::types::PyTypeFlags + as *mut crate::types::PyTypeFlags; + (*flags).remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + } + class.set_attr( + identifier!(ctx, _fields), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("kind")).into(), + ]) + .into(), + ); + + class.set_str_attr( + "__match_args__", + ctx.new_tuple(vec![ + ctx.new_str(ascii!("value")).into(), + ctx.new_str(ascii!("kind")).into(), + ]), + ctx, + ); + + class.set_attr( + identifier!(ctx, _attributes), + ctx.new_tuple(vec![ + ctx.new_str(ascii!("lineno")).into(), + ctx.new_str(ascii!("col_offset")).into(), + ctx.new_str(ascii!("end_lineno")).into(), + ctx.new_str(ascii!("end_col_offset")).into(), + ]) + .into(), + ); + } +} + +impl Initializer for NodeExprConstant { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + <NodeAst as Initializer>::slot_init(zelf.clone(), args, vm)?; + // kind defaults to None if not provided + let dict = zelf.as_object().dict().unwrap(); + if !dict.contains_key("kind", vm) { + dict.set_item("kind", vm.ctx.none(), vm)?; + } + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } +} + +impl_node!( + #[pyclass(module = "_ast", name = "Attribute", base = NodeExpr)] + pub(crate) struct NodeExprAttribute, + fields: ["value", "attr", "ctx"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Subscript", base = NodeExpr)] + pub(crate) struct NodeExprSubscript, + fields: ["value", "slice", "ctx"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Starred", base = NodeExpr)] + pub(crate) struct NodeExprStarred, + fields: ["value", "ctx"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Name", base = NodeExpr)] + pub(crate) struct NodeExprName, + fields: ["id", "ctx"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "List", base = NodeExpr)] + pub(crate) struct NodeExprList, + fields: ["elts", "ctx"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Tuple", base = NodeExpr)] + pub(crate) struct NodeExprTuple, + fields: ["elts", "ctx"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +#[pyclass(module = "_ast", name = "expr_context", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeExprContext(NodeAst); + +impl_base_node!(NodeExprContext); + +impl_node!( + #[pyclass(module = "_ast", name = "Slice", base = NodeExpr)] + pub(crate) struct NodeExprSlice, + fields: ["lower", "upper", "step"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "Load", base = NodeExprContext)] + pub(crate) struct NodeExprContextLoad, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Store", base = NodeExprContext)] + pub(crate) struct NodeExprContextStore, +); + +#[pyclass(module = "_ast", name = "boolop", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeBoolOp(NodeAst); + +impl_base_node!(NodeBoolOp); + +impl_node!( + #[pyclass(module = "_ast", name = "Del", base = NodeExprContext)] + pub(crate) struct NodeExprContextDel, +); + +impl_node!( + #[pyclass(module = "_ast", name = "And", base = NodeBoolOp)] + pub(crate) struct NodeBoolOpAnd, +); + +#[pyclass(module = "_ast", name = "operator", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeOperator(NodeAst); + +impl_base_node!(NodeOperator); + +impl_node!( + #[pyclass(module = "_ast", name = "Or", base = NodeBoolOp)] + pub(crate) struct NodeBoolOpOr, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Add", base = NodeOperator)] + pub(crate) struct NodeOperatorAdd, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Sub", base = NodeOperator)] + pub(crate) struct NodeOperatorSub, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Mult", base = NodeOperator)] + pub(crate) struct NodeOperatorMult, +); + +impl_node!( + #[pyclass(module = "_ast", name = "MatMult", base = NodeOperator)] + pub(crate) struct NodeOperatorMatMult, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Div", base = NodeOperator)] + pub(crate) struct NodeOperatorDiv, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Mod", base = NodeOperator)] + pub(crate) struct NodeOperatorMod, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Pow", base = NodeOperator)] + pub(crate) struct NodeOperatorPow, +); + +impl_node!( + #[pyclass(module = "_ast", name = "LShift", base = NodeOperator)] + pub(crate) struct NodeOperatorLShift, +); + +impl_node!( + #[pyclass(module = "_ast", name = "RShift", base = NodeOperator)] + pub(crate) struct NodeOperatorRShift, +); + +impl_node!( + #[pyclass(module = "_ast", name = "BitOr", base = NodeOperator)] + pub(crate) struct NodeOperatorBitOr, +); + +impl_node!( + #[pyclass(module = "_ast", name = "BitXor", base = NodeOperator)] + pub(crate) struct NodeOperatorBitXor, +); + +impl_node!( + #[pyclass(module = "_ast", name = "BitAnd", base = NodeOperator)] + pub(crate) struct NodeOperatorBitAnd, +); + +#[pyclass(module = "_ast", name = "unaryop", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeUnaryOp(NodeAst); + +impl_base_node!(NodeUnaryOp); + +impl_node!( + #[pyclass(module = "_ast", name = "FloorDiv", base = NodeOperator)] + pub(crate) struct NodeOperatorFloorDiv, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Invert", base = NodeUnaryOp)] + pub(crate) struct NodeUnaryOpInvert, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Not", base = NodeUnaryOp)] + pub(crate) struct NodeUnaryOpNot, +); + +impl_node!( + #[pyclass(module = "_ast", name = "UAdd", base = NodeUnaryOp)] + pub(crate) struct NodeUnaryOpUAdd, +); + +#[pyclass(module = "_ast", name = "cmpop", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeCmpOp(NodeAst); + +impl_base_node!(NodeCmpOp); + +impl_node!( + #[pyclass(module = "_ast", name = "USub", base = NodeUnaryOp)] + pub(crate) struct NodeUnaryOpUSub, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Eq", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpEq, +); + +impl_node!( + #[pyclass(module = "_ast", name = "NotEq", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpNotEq, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Lt", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpLt, +); + +impl_node!( + #[pyclass(module = "_ast", name = "LtE", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpLtE, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Gt", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpGt, +); + +impl_node!( + #[pyclass(module = "_ast", name = "GtE", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpGtE, +); + +impl_node!( + #[pyclass(module = "_ast", name = "Is", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpIs, +); + +impl_node!( + #[pyclass(module = "_ast", name = "IsNot", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpIsNot, +); + +impl_node!( + #[pyclass(module = "_ast", name = "In", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpIn, +); + +impl_node!( + #[pyclass(module = "_ast", name = "NotIn", base = NodeCmpOp)] + pub(crate) struct NodeCmpOpNotIn, +); + +#[pyclass(module = "_ast", name = "excepthandler", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeExceptHandler(NodeAst); + +impl_base_node!(NodeExceptHandler); + +impl_node!( + #[pyclass(module = "_ast", name = "comprehension", base = NodeAst)] + pub(crate) struct NodeComprehension, + fields: ["target", "iter", "ifs", "is_async"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "ExceptHandler", base = NodeExceptHandler)] + pub(crate) struct NodeExceptHandlerExceptHandler, + fields: ["type", "name", "body"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "arguments", base = NodeAst)] + pub(crate) struct NodeArguments, + fields: ["posonlyargs", "args", "vararg", "kwonlyargs", "kw_defaults", "kwarg", "defaults"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "arg", base = NodeAst)] + pub(crate) struct NodeArg, + fields: ["arg", "annotation", "type_comment"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "keyword", base = NodeAst)] + pub(crate) struct NodeKeyword, + fields: ["arg", "value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "alias", base = NodeAst)] + pub(crate) struct NodeAlias, + fields: ["name", "asname"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "withitem", base = NodeAst)] + pub(crate) struct NodeWithItem, + fields: ["context_expr", "optional_vars"], +); + +#[pyclass(module = "_ast", name = "pattern", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodePattern(NodeAst); + +impl_base_node!(NodePattern); + +impl_node!( + #[pyclass(module = "_ast", name = "match_case", base = NodeAst)] + pub(crate) struct NodeMatchCase, + fields: ["pattern", "guard", "body"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "MatchValue", base = NodePattern)] + pub(crate) struct NodePatternMatchValue, + fields: ["value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "MatchSingleton", base = NodePattern)] + pub(crate) struct NodePatternMatchSingleton, + fields: ["value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "MatchSequence", base = NodePattern)] + pub(crate) struct NodePatternMatchSequence, + fields: ["patterns"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "MatchMapping", base = NodePattern)] + pub(crate) struct NodePatternMatchMapping, + fields: ["keys", "patterns", "rest"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "MatchClass", base = NodePattern)] + pub(crate) struct NodePatternMatchClass, + fields: ["cls", "patterns", "kwd_attrs", "kwd_patterns"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "MatchStar", base = NodePattern)] + pub(crate) struct NodePatternMatchStar, + fields: ["name"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "MatchAs", base = NodePattern)] + pub(crate) struct NodePatternMatchAs, + fields: ["pattern", "name"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +#[pyclass(module = "_ast", name = "type_ignore", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeTypeIgnore(NodeAst); + +impl_base_node!(NodeTypeIgnore); + +impl_node!( + #[pyclass(module = "_ast", name = "MatchOr", base = NodePattern)] + pub(crate) struct NodePatternMatchOr, + fields: ["patterns"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +#[pyclass(module = "_ast", name = "type_param", base = NodeAst)] +#[repr(transparent)] +pub(crate) struct NodeTypeParam(NodeAst); + +impl_base_node!(NodeTypeParam); + +impl_node!( + #[pyclass(module = "_ast", name = "TypeIgnore", base = NodeTypeIgnore)] + pub(crate) struct NodeTypeIgnoreTypeIgnore, + fields: ["lineno", "tag"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "TypeVar", base = NodeTypeParam)] + pub(crate) struct NodeTypeParamTypeVar, + fields: ["name", "bound", "default_value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "ParamSpec", base = NodeTypeParam)] + pub(crate) struct NodeTypeParamParamSpec, + fields: ["name", "default_value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +impl_node!( + #[pyclass(module = "_ast", name = "TypeVarTuple", base = NodeTypeParam)] + pub(crate) struct NodeTypeParamTypeVarTuple, + fields: ["name", "default_value"], + attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], +); + +/// Marker for how to resolve an ASDL field type into a Python type object. +#[derive(Clone, Copy)] +enum FieldType { + /// AST node type reference (e.g. "expr", "stmt") + Node(&'static str), + /// Built-in type reference (e.g. "str", "int", "object") + Builtin(&'static str), + /// list[NodeType] — Py_GenericAlias(list, node_type) + ListOf(&'static str), + /// list[BuiltinType] — Py_GenericAlias(list, builtin_type) + ListOfBuiltin(&'static str), + /// NodeType | None — Union[node_type, None] + Optional(&'static str), + /// BuiltinType | None — Union[builtin_type, None] + OptionalBuiltin(&'static str), +} + +/// Field type annotations for all concrete AST node classes. +/// Derived from add_ast_annotations() in Python-ast.c. +const FIELD_TYPES: &[(&str, &[(&str, FieldType)])] = &[ + // -- mod -- + ( + "Module", + &[ + ("body", FieldType::ListOf("stmt")), + ("type_ignores", FieldType::ListOf("type_ignore")), + ], + ), + ("Interactive", &[("body", FieldType::ListOf("stmt"))]), + ("Expression", &[("body", FieldType::Node("expr"))]), + ( + "FunctionType", + &[ + ("argtypes", FieldType::ListOf("expr")), + ("returns", FieldType::Node("expr")), + ], + ), + // -- stmt -- + ( + "FunctionDef", + &[ + ("name", FieldType::Builtin("str")), + ("args", FieldType::Node("arguments")), + ("body", FieldType::ListOf("stmt")), + ("decorator_list", FieldType::ListOf("expr")), + ("returns", FieldType::Optional("expr")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ("type_params", FieldType::ListOf("type_param")), + ], + ), + ( + "AsyncFunctionDef", + &[ + ("name", FieldType::Builtin("str")), + ("args", FieldType::Node("arguments")), + ("body", FieldType::ListOf("stmt")), + ("decorator_list", FieldType::ListOf("expr")), + ("returns", FieldType::Optional("expr")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ("type_params", FieldType::ListOf("type_param")), + ], + ), + ( + "ClassDef", + &[ + ("name", FieldType::Builtin("str")), + ("bases", FieldType::ListOf("expr")), + ("keywords", FieldType::ListOf("keyword")), + ("body", FieldType::ListOf("stmt")), + ("decorator_list", FieldType::ListOf("expr")), + ("type_params", FieldType::ListOf("type_param")), + ], + ), + ("Return", &[("value", FieldType::Optional("expr"))]), + ("Delete", &[("targets", FieldType::ListOf("expr"))]), + ( + "Assign", + &[ + ("targets", FieldType::ListOf("expr")), + ("value", FieldType::Node("expr")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "TypeAlias", + &[ + ("name", FieldType::Node("expr")), + ("type_params", FieldType::ListOf("type_param")), + ("value", FieldType::Node("expr")), + ], + ), + ( + "AugAssign", + &[ + ("target", FieldType::Node("expr")), + ("op", FieldType::Node("operator")), + ("value", FieldType::Node("expr")), + ], + ), + ( + "AnnAssign", + &[ + ("target", FieldType::Node("expr")), + ("annotation", FieldType::Node("expr")), + ("value", FieldType::Optional("expr")), + ("simple", FieldType::Builtin("int")), + ], + ), + ( + "For", + &[ + ("target", FieldType::Node("expr")), + ("iter", FieldType::Node("expr")), + ("body", FieldType::ListOf("stmt")), + ("orelse", FieldType::ListOf("stmt")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "AsyncFor", + &[ + ("target", FieldType::Node("expr")), + ("iter", FieldType::Node("expr")), + ("body", FieldType::ListOf("stmt")), + ("orelse", FieldType::ListOf("stmt")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "While", + &[ + ("test", FieldType::Node("expr")), + ("body", FieldType::ListOf("stmt")), + ("orelse", FieldType::ListOf("stmt")), + ], + ), + ( + "If", + &[ + ("test", FieldType::Node("expr")), + ("body", FieldType::ListOf("stmt")), + ("orelse", FieldType::ListOf("stmt")), + ], + ), + ( + "With", + &[ + ("items", FieldType::ListOf("withitem")), + ("body", FieldType::ListOf("stmt")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "AsyncWith", + &[ + ("items", FieldType::ListOf("withitem")), + ("body", FieldType::ListOf("stmt")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "Match", + &[ + ("subject", FieldType::Node("expr")), + ("cases", FieldType::ListOf("match_case")), + ], + ), + ( + "Raise", + &[ + ("exc", FieldType::Optional("expr")), + ("cause", FieldType::Optional("expr")), + ], + ), + ( + "Try", + &[ + ("body", FieldType::ListOf("stmt")), + ("handlers", FieldType::ListOf("excepthandler")), + ("orelse", FieldType::ListOf("stmt")), + ("finalbody", FieldType::ListOf("stmt")), + ], + ), + ( + "TryStar", + &[ + ("body", FieldType::ListOf("stmt")), + ("handlers", FieldType::ListOf("excepthandler")), + ("orelse", FieldType::ListOf("stmt")), + ("finalbody", FieldType::ListOf("stmt")), + ], + ), + ( + "Assert", + &[ + ("test", FieldType::Node("expr")), + ("msg", FieldType::Optional("expr")), + ], + ), + ("Import", &[("names", FieldType::ListOf("alias"))]), + ( + "ImportFrom", + &[ + ("module", FieldType::OptionalBuiltin("str")), + ("names", FieldType::ListOf("alias")), + ("level", FieldType::OptionalBuiltin("int")), + ], + ), + ("Global", &[("names", FieldType::ListOfBuiltin("str"))]), + ("Nonlocal", &[("names", FieldType::ListOfBuiltin("str"))]), + ("Expr", &[("value", FieldType::Node("expr"))]), + // -- expr -- + ( + "BoolOp", + &[ + ("op", FieldType::Node("boolop")), + ("values", FieldType::ListOf("expr")), + ], + ), + ( + "NamedExpr", + &[ + ("target", FieldType::Node("expr")), + ("value", FieldType::Node("expr")), + ], + ), + ( + "BinOp", + &[ + ("left", FieldType::Node("expr")), + ("op", FieldType::Node("operator")), + ("right", FieldType::Node("expr")), + ], + ), + ( + "UnaryOp", + &[ + ("op", FieldType::Node("unaryop")), + ("operand", FieldType::Node("expr")), + ], + ), + ( + "Lambda", + &[ + ("args", FieldType::Node("arguments")), + ("body", FieldType::Node("expr")), + ], + ), + ( + "IfExp", + &[ + ("test", FieldType::Node("expr")), + ("body", FieldType::Node("expr")), + ("orelse", FieldType::Node("expr")), + ], + ), + ( + "Dict", + &[ + ("keys", FieldType::ListOf("expr")), + ("values", FieldType::ListOf("expr")), + ], + ), + ("Set", &[("elts", FieldType::ListOf("expr"))]), + ( + "ListComp", + &[ + ("elt", FieldType::Node("expr")), + ("generators", FieldType::ListOf("comprehension")), + ], + ), + ( + "SetComp", + &[ + ("elt", FieldType::Node("expr")), + ("generators", FieldType::ListOf("comprehension")), + ], + ), + ( + "DictComp", + &[ + ("key", FieldType::Node("expr")), + ("value", FieldType::Node("expr")), + ("generators", FieldType::ListOf("comprehension")), + ], + ), + ( + "GeneratorExp", + &[ + ("elt", FieldType::Node("expr")), + ("generators", FieldType::ListOf("comprehension")), + ], + ), + ("Await", &[("value", FieldType::Node("expr"))]), + ("Yield", &[("value", FieldType::Optional("expr"))]), + ("YieldFrom", &[("value", FieldType::Node("expr"))]), + ( + "Compare", + &[ + ("left", FieldType::Node("expr")), + ("ops", FieldType::ListOf("cmpop")), + ("comparators", FieldType::ListOf("expr")), + ], + ), + ( + "Call", + &[ + ("func", FieldType::Node("expr")), + ("args", FieldType::ListOf("expr")), + ("keywords", FieldType::ListOf("keyword")), + ], + ), + ( + "FormattedValue", + &[ + ("value", FieldType::Node("expr")), + ("conversion", FieldType::Builtin("int")), + ("format_spec", FieldType::Optional("expr")), + ], + ), + ("JoinedStr", &[("values", FieldType::ListOf("expr"))]), + ("TemplateStr", &[("values", FieldType::ListOf("expr"))]), + ( + "Interpolation", + &[ + ("value", FieldType::Node("expr")), + ("str", FieldType::Builtin("object")), + ("conversion", FieldType::Builtin("int")), + ("format_spec", FieldType::Optional("expr")), + ], + ), + ( + "Constant", + &[ + ("value", FieldType::Builtin("object")), + ("kind", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "Attribute", + &[ + ("value", FieldType::Node("expr")), + ("attr", FieldType::Builtin("str")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Subscript", + &[ + ("value", FieldType::Node("expr")), + ("slice", FieldType::Node("expr")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Starred", + &[ + ("value", FieldType::Node("expr")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Name", + &[ + ("id", FieldType::Builtin("str")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "List", + &[ + ("elts", FieldType::ListOf("expr")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Tuple", + &[ + ("elts", FieldType::ListOf("expr")), + ("ctx", FieldType::Node("expr_context")), + ], + ), + ( + "Slice", + &[ + ("lower", FieldType::Optional("expr")), + ("upper", FieldType::Optional("expr")), + ("step", FieldType::Optional("expr")), + ], + ), + // -- misc -- + ( + "comprehension", + &[ + ("target", FieldType::Node("expr")), + ("iter", FieldType::Node("expr")), + ("ifs", FieldType::ListOf("expr")), + ("is_async", FieldType::Builtin("int")), + ], + ), + ( + "ExceptHandler", + &[ + ("type", FieldType::Optional("expr")), + ("name", FieldType::OptionalBuiltin("str")), + ("body", FieldType::ListOf("stmt")), + ], + ), + ( + "arguments", + &[ + ("posonlyargs", FieldType::ListOf("arg")), + ("args", FieldType::ListOf("arg")), + ("vararg", FieldType::Optional("arg")), + ("kwonlyargs", FieldType::ListOf("arg")), + ("kw_defaults", FieldType::ListOf("expr")), + ("kwarg", FieldType::Optional("arg")), + ("defaults", FieldType::ListOf("expr")), + ], + ), + ( + "arg", + &[ + ("arg", FieldType::Builtin("str")), + ("annotation", FieldType::Optional("expr")), + ("type_comment", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "keyword", + &[ + ("arg", FieldType::OptionalBuiltin("str")), + ("value", FieldType::Node("expr")), + ], + ), + ( + "alias", + &[ + ("name", FieldType::Builtin("str")), + ("asname", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "withitem", + &[ + ("context_expr", FieldType::Node("expr")), + ("optional_vars", FieldType::Optional("expr")), + ], + ), + ( + "match_case", + &[ + ("pattern", FieldType::Node("pattern")), + ("guard", FieldType::Optional("expr")), + ("body", FieldType::ListOf("stmt")), + ], + ), + // -- pattern -- + ("MatchValue", &[("value", FieldType::Node("expr"))]), + ("MatchSingleton", &[("value", FieldType::Builtin("object"))]), + ( + "MatchSequence", + &[("patterns", FieldType::ListOf("pattern"))], + ), + ( + "MatchMapping", + &[ + ("keys", FieldType::ListOf("expr")), + ("patterns", FieldType::ListOf("pattern")), + ("rest", FieldType::OptionalBuiltin("str")), + ], + ), + ( + "MatchClass", + &[ + ("cls", FieldType::Node("expr")), + ("patterns", FieldType::ListOf("pattern")), + ("kwd_attrs", FieldType::ListOfBuiltin("str")), + ("kwd_patterns", FieldType::ListOf("pattern")), + ], + ), + ("MatchStar", &[("name", FieldType::OptionalBuiltin("str"))]), + ( + "MatchAs", + &[ + ("pattern", FieldType::Optional("pattern")), + ("name", FieldType::OptionalBuiltin("str")), + ], + ), + ("MatchOr", &[("patterns", FieldType::ListOf("pattern"))]), + // -- type_ignore -- + ( + "TypeIgnore", + &[ + ("lineno", FieldType::Builtin("int")), + ("tag", FieldType::Builtin("str")), + ], + ), + // -- type_param -- + ( + "TypeVar", + &[ + ("name", FieldType::Builtin("str")), + ("bound", FieldType::Optional("expr")), + ("default_value", FieldType::Optional("expr")), + ], + ), + ( + "ParamSpec", + &[ + ("name", FieldType::Builtin("str")), + ("default_value", FieldType::Optional("expr")), + ], + ), + ( + "TypeVarTuple", + &[ + ("name", FieldType::Builtin("str")), + ("default_value", FieldType::Optional("expr")), + ], + ), +]; + +pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) { + extend_module!(vm, module, { + "AST" => NodeAst::make_static_type(), + "mod" => NodeMod::make_static_type(), + "Module" => NodeModModule::make_static_type(), + "Interactive" => NodeModInteractive::make_static_type(), + "Expression" => NodeModExpression::make_static_type(), + "FunctionType" => NodeModFunctionType::make_static_type(), + "stmt" => NodeStmt::make_static_type(), + "FunctionDef" => NodeStmtFunctionDef::make_static_type(), + "AsyncFunctionDef" => NodeStmtAsyncFunctionDef::make_static_type(), + "ClassDef" => NodeStmtClassDef::make_static_type(), + "Return" => NodeStmtReturn::make_static_type(), + "Delete" => NodeStmtDelete::make_static_type(), + "Assign" => NodeStmtAssign::make_static_type(), + "TypeAlias" => NodeStmtTypeAlias::make_static_type(), + "AugAssign" => NodeStmtAugAssign::make_static_type(), + "AnnAssign" => NodeStmtAnnAssign::make_static_type(), + "For" => NodeStmtFor::make_static_type(), + "AsyncFor" => NodeStmtAsyncFor::make_static_type(), + "While" => NodeStmtWhile::make_static_type(), + "If" => NodeStmtIf::make_static_type(), + "With" => NodeStmtWith::make_static_type(), + "AsyncWith" => NodeStmtAsyncWith::make_static_type(), + "Match" => NodeStmtMatch::make_static_type(), + "Raise" => NodeStmtRaise::make_static_type(), + "Try" => NodeStmtTry::make_static_type(), + "TryStar" => NodeStmtTryStar::make_static_type(), + "Assert" => NodeStmtAssert::make_static_type(), + "Import" => NodeStmtImport::make_static_type(), + "ImportFrom" => NodeStmtImportFrom::make_static_type(), + "Global" => NodeStmtGlobal::make_static_type(), + "Nonlocal" => NodeStmtNonlocal::make_static_type(), + "Expr" => NodeStmtExpr::make_static_type(), + "Pass" => NodeStmtPass::make_static_type(), + "Break" => NodeStmtBreak::make_static_type(), + "Continue" => NodeStmtContinue::make_static_type(), + "expr" => NodeExpr::make_static_type(), + "BoolOp" => NodeExprBoolOp::make_static_type(), + "NamedExpr" => NodeExprNamedExpr::make_static_type(), + "BinOp" => NodeExprBinOp::make_static_type(), + "UnaryOp" => NodeExprUnaryOp::make_static_type(), + "Lambda" => NodeExprLambda::make_static_type(), + "IfExp" => NodeExprIfExp::make_static_type(), + "Dict" => NodeExprDict::make_static_type(), + "Set" => NodeExprSet::make_static_type(), + "ListComp" => NodeExprListComp::make_static_type(), + "SetComp" => NodeExprSetComp::make_static_type(), + "DictComp" => NodeExprDictComp::make_static_type(), + "GeneratorExp" => NodeExprGeneratorExp::make_static_type(), + "Await" => NodeExprAwait::make_static_type(), + "Yield" => NodeExprYield::make_static_type(), + "YieldFrom" => NodeExprYieldFrom::make_static_type(), + "Compare" => NodeExprCompare::make_static_type(), + "Call" => NodeExprCall::make_static_type(), + "FormattedValue" => NodeExprFormattedValue::make_static_type(), + "JoinedStr" => NodeExprJoinedStr::make_static_type(), + "TemplateStr" => NodeExprTemplateStr::make_static_type(), + "Interpolation" => NodeExprInterpolation::make_static_type(), + "Constant" => NodeExprConstant::make_static_type(), + "Attribute" => NodeExprAttribute::make_static_type(), + "Subscript" => NodeExprSubscript::make_static_type(), + "Starred" => NodeExprStarred::make_static_type(), + "Name" => NodeExprName::make_static_type(), + "List" => NodeExprList::make_static_type(), + "Tuple" => NodeExprTuple::make_static_type(), + "Slice" => NodeExprSlice::make_static_type(), + "expr_context" => NodeExprContext::make_static_type(), + "Load" => NodeExprContextLoad::make_static_type(), + "Store" => NodeExprContextStore::make_static_type(), + "Del" => NodeExprContextDel::make_static_type(), + "boolop" => NodeBoolOp::make_static_type(), + "And" => NodeBoolOpAnd::make_static_type(), + "Or" => NodeBoolOpOr::make_static_type(), + "operator" => NodeOperator::make_static_type(), + "Add" => NodeOperatorAdd::make_static_type(), + "Sub" => NodeOperatorSub::make_static_type(), + "Mult" => NodeOperatorMult::make_static_type(), + "MatMult" => NodeOperatorMatMult::make_static_type(), + "Div" => NodeOperatorDiv::make_static_type(), + "Mod" => NodeOperatorMod::make_static_type(), + "Pow" => NodeOperatorPow::make_static_type(), + "LShift" => NodeOperatorLShift::make_static_type(), + "RShift" => NodeOperatorRShift::make_static_type(), + "BitOr" => NodeOperatorBitOr::make_static_type(), + "BitXor" => NodeOperatorBitXor::make_static_type(), + "BitAnd" => NodeOperatorBitAnd::make_static_type(), + "FloorDiv" => NodeOperatorFloorDiv::make_static_type(), + "unaryop" => NodeUnaryOp::make_static_type(), + "Invert" => NodeUnaryOpInvert::make_static_type(), + "Not" => NodeUnaryOpNot::make_static_type(), + "UAdd" => NodeUnaryOpUAdd::make_static_type(), + "USub" => NodeUnaryOpUSub::make_static_type(), + "cmpop" => NodeCmpOp::make_static_type(), + "Eq" => NodeCmpOpEq::make_static_type(), + "NotEq" => NodeCmpOpNotEq::make_static_type(), + "Lt" => NodeCmpOpLt::make_static_type(), + "LtE" => NodeCmpOpLtE::make_static_type(), + "Gt" => NodeCmpOpGt::make_static_type(), + "GtE" => NodeCmpOpGtE::make_static_type(), + "Is" => NodeCmpOpIs::make_static_type(), + "IsNot" => NodeCmpOpIsNot::make_static_type(), + "In" => NodeCmpOpIn::make_static_type(), + "NotIn" => NodeCmpOpNotIn::make_static_type(), + "comprehension" => NodeComprehension::make_static_type(), + "excepthandler" => NodeExceptHandler::make_static_type(), + "ExceptHandler" => NodeExceptHandlerExceptHandler::make_static_type(), + "arguments" => NodeArguments::make_static_type(), + "arg" => NodeArg::make_static_type(), + "keyword" => NodeKeyword::make_static_type(), + "alias" => NodeAlias::make_static_type(), + "withitem" => NodeWithItem::make_static_type(), + "match_case" => NodeMatchCase::make_static_type(), + "pattern" => NodePattern::make_static_type(), + "MatchValue" => NodePatternMatchValue::make_static_type(), + "MatchSingleton" => NodePatternMatchSingleton::make_static_type(), + "MatchSequence" => NodePatternMatchSequence::make_static_type(), + "MatchMapping" => NodePatternMatchMapping::make_static_type(), + "MatchClass" => NodePatternMatchClass::make_static_type(), + "MatchStar" => NodePatternMatchStar::make_static_type(), + "MatchAs" => NodePatternMatchAs::make_static_type(), + "MatchOr" => NodePatternMatchOr::make_static_type(), + "type_ignore" => NodeTypeIgnore::make_static_type(), + "TypeIgnore" => NodeTypeIgnoreTypeIgnore::make_static_type(), + "type_param" => NodeTypeParam::make_static_type(), + "TypeVar" => NodeTypeParamTypeVar::make_static_type(), + "ParamSpec" => NodeTypeParamParamSpec::make_static_type(), + "TypeVarTuple" => NodeTypeParamTypeVarTuple::make_static_type(), + }); + + // Populate _field_types with real Python type objects + populate_field_types(vm, module); + populate_singletons(vm, module); + force_ast_module_name(vm, module); + populate_repr(vm, module); +} + +fn populate_field_types(vm: &VirtualMachine, module: &Py<PyModule>) { + let list_type: PyTypeRef = vm.ctx.types.list_type.to_owned(); + let none_type: PyObjectRef = vm.ctx.types.none_type.to_owned().into(); + + // Resolve a builtin type name to a Python type object + let resolve_builtin = |name: &str| -> PyObjectRef { + let ty: &Py<PyType> = match name { + "str" => vm.ctx.types.str_type, + "int" => vm.ctx.types.int_type, + "object" => vm.ctx.types.object_type, + "bool" => vm.ctx.types.bool_type, + _ => unreachable!("unknown builtin type: {name}"), + }; + ty.to_owned().into() + }; + + // Resolve an AST node type name by looking it up from the module + let resolve_node = |name: &str| -> PyObjectRef { + module + .get_attr(vm.ctx.intern_str(name), vm) + .unwrap_or_else(|_| panic!("AST node type '{name}' not found in module")) + }; + + let field_types_attr = vm.ctx.intern_str("_field_types"); + let annotations_attr = vm.ctx.intern_str("__annotations__"); + let empty_dict: PyObjectRef = vm.ctx.new_dict().into(); + + for &(class_name, fields) in FIELD_TYPES { + if fields.is_empty() { + continue; + } + + let class = module + .get_attr(class_name, vm) + .unwrap_or_else(|_| panic!("AST class '{class_name}' not found in module")); + let dict = vm.ctx.new_dict(); + + for &(field_name, ref field_type) in fields { + let type_obj = match field_type { + FieldType::Node(name) => resolve_node(name), + FieldType::Builtin(name) => resolve_builtin(name), + FieldType::ListOf(name) => { + let elem = resolve_node(name); + let args = PyTuple::new_ref(vec![elem], &vm.ctx); + PyGenericAlias::new(list_type.clone(), args, false, vm).to_pyobject(vm) + } + FieldType::ListOfBuiltin(name) => { + let elem = resolve_builtin(name); + let args = PyTuple::new_ref(vec![elem], &vm.ctx); + PyGenericAlias::new(list_type.clone(), args, false, vm).to_pyobject(vm) + } + FieldType::Optional(name) => { + let base = resolve_node(name); + let union_args = PyTuple::new_ref(vec![base, none_type.clone()], &vm.ctx); + make_union(&union_args, vm).expect("failed to create union type") + } + FieldType::OptionalBuiltin(name) => { + let base = resolve_builtin(name); + let union_args = PyTuple::new_ref(vec![base, none_type.clone()], &vm.ctx); + make_union(&union_args, vm).expect("failed to create union type") + } + }; + dict.set_item(vm.ctx.intern_str(field_name), type_obj, vm) + .expect("failed to set field type"); + } + + let dict_obj: PyObjectRef = dict.into(); + if let Some(type_obj) = class.downcast_ref::<PyType>() { + type_obj.set_attr(field_types_attr, dict_obj.clone()); + type_obj.set_attr(annotations_attr, dict_obj); + + // Set None as class-level default for optional fields. + // When ast_type_init skips optional fields, the instance + // inherits None from the class (init_types in Python-ast.c). + let none = vm.ctx.none(); + for &(field_name, ref field_type) in fields { + if matches!( + field_type, + FieldType::Optional(_) | FieldType::OptionalBuiltin(_) + ) { + type_obj.set_attr(vm.ctx.intern_str(field_name), none.clone()); + } + } + } + } + + // CPython sets __annotations__ for all built-in AST node classes, even + // when _field_types is an empty dict (e.g., operators, Load/Store/Del). + for (_name, value) in &module.dict() { + let Some(type_obj) = value.downcast_ref::<PyType>() else { + continue; + }; + if let Some(field_types) = type_obj.get_attr(field_types_attr) { + type_obj.set_attr(annotations_attr, field_types); + } + } + + // Base AST classes (e.g., expr, stmt) should still expose __annotations__. + const BASE_AST_TYPES: &[&str] = &[ + "mod", + "stmt", + "expr", + "expr_context", + "boolop", + "operator", + "unaryop", + "cmpop", + "excepthandler", + "pattern", + "type_ignore", + "type_param", + ]; + for &class_name in BASE_AST_TYPES { + let class = module + .get_attr(class_name, vm) + .unwrap_or_else(|_| panic!("AST class '{class_name}' not found in module")); + let Some(type_obj) = class.downcast_ref::<PyType>() else { + continue; + }; + if type_obj.get_attr(field_types_attr).is_none() { + type_obj.set_attr(field_types_attr, empty_dict.clone()); + } + if type_obj.get_attr(annotations_attr).is_none() { + type_obj.set_attr(annotations_attr, empty_dict.clone()); + } + } +} + +fn populate_singletons(vm: &VirtualMachine, module: &Py<PyModule>) { + let instance_attr = vm.ctx.intern_str("_instance"); + const SINGLETON_TYPES: &[&str] = &[ + // expr_context + "Load", "Store", "Del", // boolop + "And", "Or", // operator + "Add", "Sub", "Mult", "MatMult", "Div", "Mod", "Pow", "LShift", "RShift", "BitOr", + "BitXor", "BitAnd", "FloorDiv", // unaryop + "Invert", "Not", "UAdd", "USub", // cmpop + "Eq", "NotEq", "Lt", "LtE", "Gt", "GtE", "Is", "IsNot", "In", "NotIn", + ]; + + for &class_name in SINGLETON_TYPES { + let class = module + .get_attr(class_name, vm) + .unwrap_or_else(|_| panic!("AST class '{class_name}' not found in module")); + let Some(type_obj) = class.downcast_ref::<PyType>() else { + continue; + }; + let instance = vm + .ctx + .new_base_object(type_obj.to_owned(), Some(vm.ctx.new_dict())); + type_obj.set_attr(instance_attr, instance); + } +} + +fn force_ast_module_name(vm: &VirtualMachine, module: &Py<PyModule>) { + let ast_name = vm.ctx.new_str("ast"); + for (_name, value) in &module.dict() { + let Some(type_obj) = value.downcast_ref::<PyType>() else { + continue; + }; + type_obj.set_attr(identifier!(vm, __module__), ast_name.clone().into()); + } +} + +fn populate_repr(_vm: &VirtualMachine, module: &Py<PyModule>) { + for (_name, value) in &module.dict() { + let Some(type_obj) = value.downcast_ref::<PyType>() else { + continue; + }; + type_obj + .slots + .repr + .store(Some(super::python::_ast::ast_repr)); + } +} diff --git a/crates/vm/src/stdlib/_ast/python.rs b/crates/vm/src/stdlib/_ast/python.rs new file mode 100644 index 00000000000..5c97d759934 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/python.rs @@ -0,0 +1,525 @@ +use super::{ + PY_CF_ALLOW_INCOMPLETE_INPUT, PY_CF_ALLOW_TOP_LEVEL_AWAIT, PY_CF_DONT_IMPLY_DEDENT, + PY_CF_IGNORE_COOKIE, PY_CF_ONLY_AST, PY_CF_OPTIMIZED_AST, PY_CF_SOURCE_IS_UTF8, + PY_CF_TYPE_COMMENTS, +}; + +#[pymodule] +pub(crate) mod _ast { + use crate::{ + AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, PyUtf8Str, PyUtf8StrRef}, + class::{PyClassImpl, StaticType}, + common::wtf8::Wtf8, + function::{FuncArgs, KwArgs, PyMethodDef, PyMethodFlags}, + stdlib::_ast::repr, + types::{Constructor, Initializer}, + warn, + }; + use indexmap::IndexMap; + #[pyattr] + #[pyclass(module = "_ast", name = "AST")] + #[derive(Debug, PyPayload)] + pub(crate) struct NodeAst; + + #[pyclass(with(Constructor, Initializer), flags(BASETYPE, HAS_DICT))] + impl NodeAst { + #[extend_class] + fn extend_class(ctx: &Context, class: &'static Py<PyType>) { + // AST types are mutable (heap types, not IMMUTABLETYPE) + // Safety: called during type initialization before any concurrent access + unsafe { + let flags = &class.slots.flags as *const crate::types::PyTypeFlags + as *mut crate::types::PyTypeFlags; + (*flags).remove(crate::types::PyTypeFlags::IMMUTABLETYPE); + } + let empty_tuple = ctx.empty_tuple.clone(); + class.set_str_attr("_fields", empty_tuple.clone(), ctx); + class.set_str_attr("_attributes", empty_tuple.clone(), ctx); + class.set_str_attr("__match_args__", empty_tuple.clone(), ctx); + + const AST_REDUCE: PyMethodDef = PyMethodDef::new_const( + "__reduce__", + |zelf: PyObjectRef, vm: &VirtualMachine| -> PyResult<PyTupleRef> { + ast_reduce(zelf, vm) + }, + PyMethodFlags::METHOD, + None, + ); + const AST_REPLACE: PyMethodDef = PyMethodDef::new_const( + "__replace__", + |zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine| -> PyResult { + ast_replace(zelf, args, vm) + }, + PyMethodFlags::METHOD, + None, + ); + + class.set_str_attr("__reduce__", AST_REDUCE.to_proper_method(class, ctx), ctx); + class.set_str_attr("__replace__", AST_REPLACE.to_proper_method(class, ctx), ctx); + class.slots.repr.store(Some(ast_repr)); + } + + #[pyattr] + fn _fields(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } + + #[pyattr] + fn _attributes(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } + + #[pyattr] + fn __match_args__(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } + + #[pymethod] + fn __reduce__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + ast_reduce(zelf, vm) + } + + #[pymethod] + fn __replace__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + ast_replace(zelf, args, vm) + } + } + + pub(crate) fn ast_reduce(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let dict = zelf.as_object().dict(); + let cls = zelf.class(); + let type_obj: PyObjectRef = cls.to_owned().into(); + + let Some(dict) = dict else { + return Ok(vm.ctx.new_tuple(vec![type_obj])); + }; + + let fields = cls.get_attr(vm.ctx.intern_str("_fields")); + if let Some(fields) = fields { + let fields: Vec<PyStrRef> = fields.try_to_value(vm)?; + let mut positional: Vec<PyObjectRef> = Vec::new(); + for field in fields { + if dict.get_item_opt::<Wtf8>(field.as_wtf8(), vm)?.is_some() { + positional.push(vm.ctx.none()); + } else { + break; + } + } + let args: PyObjectRef = vm.ctx.new_tuple(positional).into(); + let dict_obj: PyObjectRef = dict.into(); + return Ok(vm.ctx.new_tuple(vec![type_obj, args, dict_obj])); + } + + Ok(vm + .ctx + .new_tuple(vec![type_obj, vm.ctx.new_tuple(vec![]).into(), dict.into()])) + } + + pub(crate) fn ast_replace(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() { + return Err(vm.new_type_error("__replace__() takes no positional arguments")); + } + + let cls = zelf.class(); + let fields = cls.get_attr(vm.ctx.intern_str("_fields")); + let attributes = cls.get_attr(vm.ctx.intern_str("_attributes")); + let dict = zelf.as_object().dict(); + + let mut expecting: std::collections::HashSet<String> = std::collections::HashSet::new(); + if let Some(fields) = fields.clone() { + let fields: Vec<PyUtf8StrRef> = fields.try_to_value(vm)?; + for field in fields { + expecting.insert(field.as_str().to_owned()); + } + } + if let Some(attributes) = attributes.clone() { + let attributes: Vec<PyUtf8StrRef> = attributes.try_to_value(vm)?; + for attr in attributes { + expecting.insert(attr.as_str().to_owned()); + } + } + + for (key, _value) in &args.kwargs { + if !expecting.remove(key) { + return Err(vm.new_type_error(format!( + "{}.__replace__ got an unexpected keyword argument '{}'.", + cls.name(), + key + ))); + } + } + + if let Some(dict) = dict.as_ref() { + for (key, _value) in dict.items_vec() { + if let Ok(key) = key.downcast::<PyUtf8Str>() { + expecting.remove(key.as_str()); + } + } + if let Some(attributes) = attributes.clone() { + let attributes: Vec<PyUtf8StrRef> = attributes.try_to_value(vm)?; + for attr in attributes { + expecting.remove(attr.as_str()); + } + } + } + + // Discard optional fields (T | None). + if let Some(field_types) = cls.get_attr(vm.ctx.intern_str("_field_types")) + && let Ok(field_types) = field_types.downcast::<crate::builtins::PyDict>() + { + for (key, value) in field_types.items_vec() { + let Ok(key) = key.downcast::<PyUtf8Str>() else { + continue; + }; + if value.fast_isinstance(vm.ctx.types.union_type) { + expecting.remove(key.as_str()); + } + } + } + + if !expecting.is_empty() { + let mut names: Vec<String> = expecting + .into_iter() + .map(|name| format!("{name:?}")) + .collect(); + names.sort(); + let missing = names.join(", "); + let count = names.len(); + return Err(vm.new_type_error(format!( + "{}.__replace__ missing {} keyword argument{}: {}.", + cls.name(), + count, + if count == 1 { "" } else { "s" }, + missing + ))); + } + + let payload = vm.ctx.new_dict(); + if let Some(dict) = dict { + if let Some(fields) = fields.clone() { + let fields: Vec<PyStrRef> = fields.try_to_value(vm)?; + for field in fields { + if let Some(value) = dict.get_item_opt::<Wtf8>(field.as_wtf8(), vm)? { + payload.set_item(field.as_object(), value, vm)?; + } + } + } + if let Some(attributes) = attributes.clone() { + let attributes: Vec<PyStrRef> = attributes.try_to_value(vm)?; + for attr in attributes { + if let Some(value) = dict.get_item_opt::<Wtf8>(attr.as_wtf8(), vm)? { + payload.set_item(attr.as_object(), value, vm)?; + } + } + } + } + for (key, value) in args.kwargs { + payload.set_item(vm.ctx.intern_str(key), value, vm)?; + } + + let type_obj: PyObjectRef = cls.to_owned().into(); + let kwargs = payload + .items_vec() + .into_iter() + .map(|(key, value)| { + let key = key + .downcast::<PyUtf8Str>() + .map_err(|_| vm.new_type_error("keywords must be strings"))?; + Ok((key.as_str().to_owned(), value)) + }) + .collect::<PyResult<IndexMap<String, PyObjectRef>>>()?; + let result = type_obj.call(FuncArgs::new(vec![], KwArgs::new(kwargs)), vm)?; + Ok(result) + } + + pub(crate) fn ast_repr(zelf: &crate::PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let repr = repr::repr_ast_node(vm, &zelf.to_owned(), 3)?; + Ok(vm.ctx.new_str(repr)) + } + + impl Constructor for NodeAst { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if args.args.is_empty() + && args.kwargs.is_empty() + && let Some(instance) = cls.get_attr(vm.ctx.intern_str("_instance")) + { + return Ok(instance); + } + + // AST nodes accept extra arguments (unlike object.__new__) + // This matches CPython's behavior where AST has its own tp_new + let dict = if cls + .slots + .flags + .contains(crate::types::PyTypeFlags::HAS_DICT) + { + Some(vm.ctx.new_dict()) + } else { + None + }; + let zelf = vm.ctx.new_base_object(cls, dict); + + // type.__call__ does not invoke slot_init after slot_new + // for types with a custom slot_new, so we must call it here. + Self::slot_init(zelf.clone(), args, vm)?; + + Ok(zelf) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + impl Initializer for NodeAst { + type Args = FuncArgs; + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let fields = zelf + .class() + .get_attr(vm.ctx.intern_str("_fields")) + .ok_or_else(|| { + let module = zelf + .class() + .get_attr(vm.ctx.intern_str("__module__")) + .and_then(|obj| obj.try_to_value::<String>(vm).ok()) + .unwrap_or_else(|| "ast".to_owned()); + vm.new_attribute_error(format!( + "type object '{}.{}' has no attribute '_fields'", + module, + zelf.class().name() + )) + })?; + let fields: Vec<PyUtf8StrRef> = fields.try_to_value(vm)?; + let n_args = args.args.len(); + if n_args > fields.len() { + return Err(vm.new_type_error(format!( + "{} constructor takes at most {} positional argument{}", + zelf.class().name(), + fields.len(), + if fields.len() == 1 { "" } else { "s" }, + ))); + } + + // Track which fields were set + let mut set_fields = std::collections::HashSet::new(); + let mut attributes: Option<Vec<PyStrRef>> = None; + + for (name, arg) in fields.iter().zip(args.args) { + zelf.set_attr(name, arg, vm)?; + set_fields.insert(name.as_str().to_owned()); + } + for (key, value) in args.kwargs { + if let Some(pos) = fields.iter().position(|f| f.as_bytes() == key.as_bytes()) + && pos < n_args + { + return Err(vm.new_type_error(format!( + "{} got multiple values for argument '{}'", + zelf.class().name(), + key + ))); + } + + if fields + .iter() + .all(|field| field.as_bytes() != key.as_bytes()) + { + let attrs = if let Some(attrs) = &attributes { + attrs + } else { + let attrs = zelf + .class() + .get_attr(vm.ctx.intern_str("_attributes")) + .and_then(|attr| attr.try_to_value::<Vec<PyStrRef>>(vm).ok()) + .unwrap_or_default(); + attributes = Some(attrs); + attributes.as_ref().unwrap() + }; + if attrs.iter().all(|attr| attr.as_bytes() != key.as_bytes()) { + let message = vm.ctx.new_str(format!( + "{}.__init__ got an unexpected keyword argument '{}'. \ +Support for arbitrary keyword arguments is deprecated and will be removed in Python 3.15.", + zelf.class().name(), + key + )); + warn::warn( + message.into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + } + + set_fields.insert(key.clone()); + zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; + } + + // Use _field_types to determine defaults for unset fields. + // Only built-in AST node classes have _field_types populated. + let field_types = zelf.class().get_attr(vm.ctx.intern_str("_field_types")); + if let Some(Ok(ft_dict)) = + field_types.map(|ft| ft.downcast::<crate::builtins::PyDict>()) + { + let expr_ctx_type: PyObjectRef = + super::super::pyast::NodeExprContext::make_static_type().into(); + + for field in &fields { + if set_fields.contains(field.as_str()) { + continue; + } + if let Some(ftype) = ft_dict.get_item_opt::<Wtf8>(field.as_wtf8(), vm)? { + if ftype.fast_isinstance(vm.ctx.types.union_type) { + // Optional field (T | None) — no default + } else if ftype.fast_isinstance(vm.ctx.types.generic_alias_type) { + // List field (list[T]) — default to [] + let empty_list: PyObjectRef = vm.ctx.new_list(vec![]).into(); + zelf.set_attr(vm.ctx.intern_str(field.as_wtf8()), empty_list, vm)?; + } else if ftype.is(&expr_ctx_type) { + // expr_context — default to Load() + let load_type = + super::super::pyast::NodeExprContextLoad::make_static_type(); + let load_instance = load_type + .get_attr(vm.ctx.intern_str("_instance")) + .unwrap_or_else(|| { + vm.ctx.new_base_object(load_type, Some(vm.ctx.new_dict())) + }); + zelf.set_attr(vm.ctx.intern_str(field.as_wtf8()), load_instance, vm)?; + } else { + // Required field missing: emit DeprecationWarning. + let message = vm.ctx.new_str(format!( + "{}.__init__ missing 1 required positional argument: '{}'", + zelf.class().name(), + field.as_wtf8() + )); + warn::warn( + message.into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + } + } + } + + Ok(()) + } + + fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_init is defined") + } + } + + #[pyattr(name = "PyCF_SOURCE_IS_UTF8")] + use super::PY_CF_SOURCE_IS_UTF8; + + #[pyattr(name = "PyCF_DONT_IMPLY_DEDENT")] + use super::PY_CF_DONT_IMPLY_DEDENT; + + #[pyattr(name = "PyCF_ONLY_AST")] + use super::PY_CF_ONLY_AST; + + #[pyattr(name = "PyCF_IGNORE_COOKIE")] + use super::PY_CF_IGNORE_COOKIE; + + #[pyattr(name = "PyCF_TYPE_COMMENTS")] + use super::PY_CF_TYPE_COMMENTS; + + #[pyattr(name = "PyCF_ALLOW_TOP_LEVEL_AWAIT")] + use super::PY_CF_ALLOW_TOP_LEVEL_AWAIT; + + #[pyattr(name = "PyCF_ALLOW_INCOMPLETE_INPUT")] + use super::PY_CF_ALLOW_INCOMPLETE_INPUT; + + #[pyattr(name = "PyCF_OPTIMIZED_AST")] + use super::PY_CF_OPTIMIZED_AST; + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::pyast::extend_module_nodes(vm, module); + + let ast_type = module + .get_attr("AST", vm)? + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("AST is not a type"))?; + let ctx = &vm.ctx; + let empty_tuple = ctx.empty_tuple.clone(); + ast_type.set_str_attr("_fields", empty_tuple.clone(), ctx); + ast_type.set_str_attr("_attributes", empty_tuple.clone(), ctx); + ast_type.set_str_attr("__match_args__", empty_tuple.clone(), ctx); + + const AST_REDUCE: PyMethodDef = PyMethodDef::new_const( + "__reduce__", + |zelf: PyObjectRef, vm: &VirtualMachine| -> PyResult<PyTupleRef> { + ast_reduce(zelf, vm) + }, + PyMethodFlags::METHOD, + None, + ); + const AST_REPLACE: PyMethodDef = PyMethodDef::new_const( + "__replace__", + |zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine| -> PyResult { + ast_replace(zelf, args, vm) + }, + PyMethodFlags::METHOD, + None, + ); + let base_type = NodeAst::static_type(); + ast_type.set_str_attr( + "__reduce__", + AST_REDUCE.to_proper_method(base_type, ctx), + ctx, + ); + ast_type.set_str_attr( + "__replace__", + AST_REPLACE.to_proper_method(base_type, ctx), + ctx, + ); + ast_type.slots.repr.store(Some(ast_repr)); + + const EXPR_DOC: &str = "expr = BoolOp(boolop op, expr* values)\n\ + | NamedExpr(expr target, expr value)\n\ + | BinOp(expr left, operator op, expr right)\n\ + | UnaryOp(unaryop op, expr operand)\n\ + | Lambda(arguments args, expr body)\n\ + | IfExp(expr test, expr body, expr orelse)\n\ + | Dict(expr?* keys, expr* values)\n\ + | Set(expr* elts)\n\ + | ListComp(expr elt, comprehension* generators)\n\ + | SetComp(expr elt, comprehension* generators)\n\ + | DictComp(expr key, expr value, comprehension* generators)\n\ + | GeneratorExp(expr elt, comprehension* generators)\n\ + | Await(expr value)\n\ + | Yield(expr? value)\n\ + | YieldFrom(expr value)\n\ + | Compare(expr left, cmpop* ops, expr* comparators)\n\ + | Call(expr func, expr* args, keyword* keywords)\n\ + | FormattedValue(expr value, int conversion, expr? format_spec)\n\ + | Interpolation(expr value, constant str, int conversion, expr? format_spec)\n\ + | JoinedStr(expr* values)\n\ + | TemplateStr(expr* values)\n\ + | Constant(constant value, string? kind)\n\ + | Attribute(expr value, identifier attr, expr_context ctx)\n\ + | Subscript(expr value, expr slice, expr_context ctx)\n\ + | Starred(expr value, expr_context ctx)\n\ + | Name(identifier id, expr_context ctx)\n\ + | List(expr* elts, expr_context ctx)\n\ + | Tuple(expr* elts, expr_context ctx)\n\ + | Slice(expr? lower, expr? upper, expr? step)"; + let expr_type = super::super::pyast::NodeExpr::static_type(); + expr_type.set_attr( + identifier!(vm.ctx, __doc__), + vm.ctx.new_str(EXPR_DOC).into(), + ); + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/_ast/repr.rs b/crates/vm/src/stdlib/_ast/repr.rs new file mode 100644 index 00000000000..2897447fbec --- /dev/null +++ b/crates/vm/src/stdlib/_ast/repr.rs @@ -0,0 +1,155 @@ +use crate::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyList, PyTuple}, + class::PyClassImpl, + stdlib::_ast::NodeAst, +}; +use rustpython_common::wtf8::Wtf8Buf; + +fn repr_ast_list(vm: &VirtualMachine, items: Vec<PyObjectRef>, depth: usize) -> PyResult<Wtf8Buf> { + if items.is_empty() { + let empty_list: PyObjectRef = vm.ctx.new_list(vec![]).into(); + return Ok(empty_list.repr(vm)?.as_wtf8().to_owned()); + } + + let mut parts: Vec<Wtf8Buf> = Vec::new(); + let first = &items[0]; + let last = items.last().unwrap(); + + for (idx, item) in [first, last].iter().enumerate() { + if idx == 1 && items.len() == 1 { + break; + } + let repr = if item.fast_isinstance(&NodeAst::make_static_type()) { + repr_ast_node(vm, item, depth.saturating_sub(1))? + } else { + item.repr(vm)?.as_wtf8().to_owned() + }; + parts.push(repr); + } + + let mut rendered = Wtf8Buf::from("["); + if !parts.is_empty() { + rendered.push_wtf8(&parts[0]); + } + if items.len() > 2 { + if !parts[0].is_empty() { + rendered.push_wtf8(", ...".as_ref()); + } + if parts.len() > 1 { + rendered.push_wtf8(", ".as_ref()); + rendered.push_wtf8(&parts[1]); + } + } else if parts.len() > 1 { + rendered.push_wtf8(", ".as_ref()); + rendered.push_wtf8(&parts[1]); + } + rendered.push_wtf8("]".as_ref()); + Ok(rendered) +} + +fn repr_ast_tuple(vm: &VirtualMachine, items: Vec<PyObjectRef>, depth: usize) -> PyResult<Wtf8Buf> { + if items.is_empty() { + let empty_tuple: PyObjectRef = vm.ctx.empty_tuple.clone().into(); + return Ok(empty_tuple.repr(vm)?.as_wtf8().to_owned()); + } + + let mut parts: Vec<Wtf8Buf> = Vec::new(); + let first = &items[0]; + let last = items.last().unwrap(); + + for (idx, item) in [first, last].iter().enumerate() { + if idx == 1 && items.len() == 1 { + break; + } + let repr = if item.fast_isinstance(&NodeAst::make_static_type()) { + repr_ast_node(vm, item, depth.saturating_sub(1))? + } else { + item.repr(vm)?.as_wtf8().to_owned() + }; + parts.push(repr); + } + + let mut rendered = Wtf8Buf::from("("); + if !parts.is_empty() { + rendered.push_wtf8(&parts[0]); + } + if items.len() > 2 { + if !parts[0].is_empty() { + rendered.push_wtf8(", ...".as_ref()); + } + if parts.len() > 1 { + rendered.push_wtf8(", ".as_ref()); + rendered.push_wtf8(&parts[1]); + } + } else if parts.len() > 1 { + rendered.push_wtf8(", ".as_ref()); + rendered.push_wtf8(&parts[1]); + } + if items.len() == 1 { + rendered.push_wtf8(",".as_ref()); + } + rendered.push_wtf8(")".as_ref()); + Ok(rendered) +} + +pub(crate) fn repr_ast_node( + vm: &VirtualMachine, + obj: &PyObjectRef, + depth: usize, +) -> PyResult<Wtf8Buf> { + let cls = obj.class(); + if depth == 0 { + let mut s = Wtf8Buf::from(&*cls.name()); + s.push_wtf8("(...)".as_ref()); + return Ok(s); + } + + let fields = cls.get_attr(vm.ctx.intern_str("_fields")); + let fields = match fields { + Some(fields) => fields.try_to_value::<Vec<crate::builtins::PyStrRef>>(vm)?, + None => { + let mut s = Wtf8Buf::from(&*cls.name()); + s.push_wtf8("(...)".as_ref()); + return Ok(s); + } + }; + + if fields.is_empty() { + let mut s = Wtf8Buf::from(&*cls.name()); + s.push_wtf8("()".as_ref()); + return Ok(s); + } + + let mut rendered = Wtf8Buf::from(&*cls.name()); + rendered.push_wtf8("(".as_ref()); + + for (idx, field) in fields.iter().enumerate() { + let value = obj.get_attr(field, vm)?; + let value_repr = if value.fast_isinstance(vm.ctx.types.list_type) { + let list = value + .downcast::<PyList>() + .expect("list type should downcast"); + repr_ast_list(vm, list.borrow_vec().to_vec(), depth)? + } else if value.fast_isinstance(vm.ctx.types.tuple_type) { + let tuple = value + .downcast::<PyTuple>() + .expect("tuple type should downcast"); + repr_ast_tuple(vm, tuple.as_slice().to_vec(), depth)? + } else if value.fast_isinstance(&NodeAst::make_static_type()) { + repr_ast_node(vm, &value, depth.saturating_sub(1))? + } else { + value.repr(vm)?.as_wtf8().to_owned() + }; + + if idx > 0 { + rendered.push_wtf8(", ".as_ref()); + } + rendered.push_wtf8(field.as_wtf8()); + rendered.push_wtf8("=".as_ref()); + rendered.push_wtf8(&value_repr); + } + + rendered.push_wtf8(")".as_ref()); + Ok(rendered) +} diff --git a/crates/vm/src/stdlib/_ast/statement.rs b/crates/vm/src/stdlib/_ast/statement.rs new file mode 100644 index 00000000000..4c53bf01877 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/statement.rs @@ -0,0 +1,1328 @@ +use super::*; +use crate::stdlib::_ast::argument::{merge_class_def_args, split_class_def_args}; +use rustpython_compiler_core::SourceFile; + +// sum +impl Node for ast::Stmt { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::FunctionDef(cons) => cons.ast_to_object(vm, source_file), + Self::ClassDef(cons) => cons.ast_to_object(vm, source_file), + Self::Return(cons) => cons.ast_to_object(vm, source_file), + Self::Delete(cons) => cons.ast_to_object(vm, source_file), + Self::Assign(cons) => cons.ast_to_object(vm, source_file), + Self::TypeAlias(cons) => cons.ast_to_object(vm, source_file), + Self::AugAssign(cons) => cons.ast_to_object(vm, source_file), + Self::AnnAssign(cons) => cons.ast_to_object(vm, source_file), + Self::For(cons) => cons.ast_to_object(vm, source_file), + Self::While(cons) => cons.ast_to_object(vm, source_file), + Self::If(cons) => cons.ast_to_object(vm, source_file), + Self::With(cons) => cons.ast_to_object(vm, source_file), + Self::Match(cons) => cons.ast_to_object(vm, source_file), + Self::Raise(cons) => cons.ast_to_object(vm, source_file), + Self::Try(cons) => cons.ast_to_object(vm, source_file), + Self::Assert(cons) => cons.ast_to_object(vm, source_file), + Self::Import(cons) => cons.ast_to_object(vm, source_file), + Self::ImportFrom(cons) => cons.ast_to_object(vm, source_file), + Self::Global(cons) => cons.ast_to_object(vm, source_file), + Self::Nonlocal(cons) => cons.ast_to_object(vm, source_file), + Self::Expr(cons) => cons.ast_to_object(vm, source_file), + Self::Pass(cons) => cons.ast_to_object(vm, source_file), + Self::Break(cons) => cons.ast_to_object(vm, source_file), + Self::Continue(cons) => cons.ast_to_object(vm, source_file), + Self::IpyEscapeCommand(_) => { + unimplemented!("IPython escape command is not allowed in Python AST") + } + } + } + + #[allow(clippy::if_same_then_else)] + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + let _cls = _object.class(); + Ok(if _cls.is(pyast::NodeStmtFunctionDef::static_type()) { + Self::FunctionDef(ast::StmtFunctionDef::ast_from_object( + _vm, + source_file, + _object, + )?) + } else if _cls.is(pyast::NodeStmtAsyncFunctionDef::static_type()) { + Self::FunctionDef(ast::StmtFunctionDef::ast_from_object( + _vm, + source_file, + _object, + )?) + } else if _cls.is(pyast::NodeStmtClassDef::static_type()) { + Self::ClassDef(ast::StmtClassDef::ast_from_object( + _vm, + source_file, + _object, + )?) + } else if _cls.is(pyast::NodeStmtReturn::static_type()) { + Self::Return(ast::StmtReturn::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtDelete::static_type()) { + Self::Delete(ast::StmtDelete::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtAssign::static_type()) { + Self::Assign(ast::StmtAssign::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtTypeAlias::static_type()) { + Self::TypeAlias(ast::StmtTypeAlias::ast_from_object( + _vm, + source_file, + _object, + )?) + } else if _cls.is(pyast::NodeStmtAugAssign::static_type()) { + Self::AugAssign(ast::StmtAugAssign::ast_from_object( + _vm, + source_file, + _object, + )?) + } else if _cls.is(pyast::NodeStmtAnnAssign::static_type()) { + Self::AnnAssign(ast::StmtAnnAssign::ast_from_object( + _vm, + source_file, + _object, + )?) + } else if _cls.is(pyast::NodeStmtFor::static_type()) { + Self::For(ast::StmtFor::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtAsyncFor::static_type()) { + Self::For(ast::StmtFor::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtWhile::static_type()) { + Self::While(ast::StmtWhile::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtIf::static_type()) { + Self::If(ast::StmtIf::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtWith::static_type()) { + Self::With(ast::StmtWith::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtAsyncWith::static_type()) { + Self::With(ast::StmtWith::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtMatch::static_type()) { + Self::Match(ast::StmtMatch::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtRaise::static_type()) { + Self::Raise(ast::StmtRaise::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtTry::static_type()) { + Self::Try(ast::StmtTry::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtTryStar::static_type()) { + Self::Try(ast::StmtTry::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtAssert::static_type()) { + Self::Assert(ast::StmtAssert::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtImport::static_type()) { + Self::Import(ast::StmtImport::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtImportFrom::static_type()) { + Self::ImportFrom(ast::StmtImportFrom::ast_from_object( + _vm, + source_file, + _object, + )?) + } else if _cls.is(pyast::NodeStmtGlobal::static_type()) { + Self::Global(ast::StmtGlobal::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtNonlocal::static_type()) { + Self::Nonlocal(ast::StmtNonlocal::ast_from_object( + _vm, + source_file, + _object, + )?) + } else if _cls.is(pyast::NodeStmtExpr::static_type()) { + Self::Expr(ast::StmtExpr::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtPass::static_type()) { + Self::Pass(ast::StmtPass::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtBreak::static_type()) { + Self::Break(ast::StmtBreak::ast_from_object(_vm, source_file, _object)?) + } else if _cls.is(pyast::NodeStmtContinue::static_type()) { + Self::Continue(ast::StmtContinue::ast_from_object( + _vm, + source_file, + _object, + )?) + } else { + return Err(_vm.new_type_error(format!( + "expected some sort of stmt, but got {}", + _object.repr(_vm)? + ))); + }) + } +} + +// constructor +impl Node for ast::StmtFunctionDef { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + parameters, + body, + decorator_list, + returns, + // type_comment, + type_params, + is_async, + range: _range, + } = self; + let source_code = source_file.to_source_code(); + let def_line = source_code.line_index(name.range.start()); + let range = TextRange::new(source_code.line_start(def_line), _range.end()); + + let cls = if !is_async { + pyast::NodeStmtFunctionDef::static_type().to_owned() + } else { + pyast::NodeStmtAsyncFunctionDef::static_type().to_owned() + }; + + let node = NodeAst.into_ref_with_type(vm, cls).unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", vm.ctx.new_str(name.as_str()).to_pyobject(vm), vm) + .unwrap(); + dict.set_item("args", parameters.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item( + "decorator_list", + decorator_list.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + dict.set_item("returns", returns.ast_to_object(vm, source_file), vm) + .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", vm.ctx.none(), vm).unwrap(); + dict.set_item( + "type_params", + type_params + .map(|tp| tp.ast_to_object(vm, source_file)) + .unwrap_or_else(|| vm.ctx.new_list(vec![]).into()), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + let _cls = _object.class(); + let is_async = _cls.is(pyast::NodeStmtAsyncFunctionDef::static_type()); + let range = range_from_object(_vm, source_file, _object.clone(), "FunctionDef")?; + Ok(Self { + node_index: Default::default(), + name: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "name", "FunctionDef")?, + )?, + parameters: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "args", "FunctionDef")?, + )?, + body: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "body", "FunctionDef")?, + )?, + decorator_list: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "decorator_list", "FunctionDef")?, + )?, + returns: get_node_field_opt(_vm, &_object, "returns")? + .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + .transpose()?, + // TODO: Ruff ignores type_comment during parsing + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + type_params: Node::ast_from_object( + _vm, + source_file, + get_node_field_opt(_vm, &_object, "type_params")? + .unwrap_or_else(|| _vm.ctx.new_list(Vec::new()).into()), + )?, + range, + is_async, + }) + } +} + +// constructor +impl Node for ast::StmtClassDef { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + arguments, + body, + decorator_list, + type_params, + range: _range, + } = self; + let (bases, keywords) = split_class_def_args(arguments); + let source_code = source_file.to_source_code(); + let class_line = source_code.line_index(name.range.start()); + let range = TextRange::new(source_code.line_start(class_line), _range.end()); + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtClassDef::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item( + "bases", + bases + .map(|b| b.ast_to_object(_vm, source_file)) + .unwrap_or_else(|| _vm.ctx.new_list(vec![]).into()), + _vm, + ) + .unwrap(); + dict.set_item( + "keywords", + keywords + .map(|k| k.ast_to_object(_vm, source_file)) + .unwrap_or_else(|| _vm.ctx.new_list(vec![]).into()), + _vm, + ) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item( + "decorator_list", + decorator_list.ast_to_object(_vm, source_file), + _vm, + ) + .unwrap(); + dict.set_item( + "type_params", + type_params + .map(|tp| tp.ast_to_object(_vm, source_file)) + .unwrap_or_else(|| _vm.ctx.new_list(vec![]).into()), + _vm, + ) + .unwrap(); + node_add_location(&dict, range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + let bases = Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "bases", "ClassDef")?, + )?; + let keywords = Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "keywords", "ClassDef")?, + )?; + Ok(Self { + node_index: Default::default(), + name: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "name", "ClassDef")?, + )?, + arguments: merge_class_def_args(bases, keywords), + body: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "body", "ClassDef")?, + )?, + decorator_list: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "decorator_list", "ClassDef")?, + )?, + type_params: Node::ast_from_object( + _vm, + source_file, + get_node_field_opt(_vm, &_object, "type_params")? + .unwrap_or_else(|| _vm.ctx.new_list(Vec::new()).into()), + )?, + range: range_from_object(_vm, source_file, _object, "ClassDef")?, + }) + } +} +// constructor +impl Node for ast::StmtReturn { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtReturn::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: get_node_field_opt(_vm, &_object, "value")? + .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + .transpose()?, + range: range_from_object(_vm, source_file, _object, "Return")?, + }) + } +} +// constructor +impl Node for ast::StmtDelete { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + targets, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtDelete::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("targets", targets.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + targets: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "targets", "Delete")?, + )?, + range: range_from_object(_vm, source_file, _object, "Delete")?, + }) + } +} + +// constructor +impl Node for ast::StmtAssign { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + targets, + value, + // type_comment, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeStmtAssign::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("targets", targets.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + // TODO + dict.set_item("type_comment", vm.ctx.none(), vm).unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + targets: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "targets", "Assign")?, + )?, + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "Assign")?, + )?, + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(vm, source_file, object, "Assign")?, + }) + } +} + +// constructor +impl Node for ast::StmtTypeAlias { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + type_params, + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtTypeAlias::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item( + "type_params", + type_params + .map(|tp| tp.ast_to_object(_vm, source_file)) + .unwrap_or_else(|| _vm.ctx.new_list(Vec::new()).into()), + _vm, + ) + .unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + name: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "name", "TypeAlias")?, + )?, + type_params: Node::ast_from_object( + _vm, + source_file, + get_node_field_opt(_vm, &_object, "type_params")?.unwrap_or_else(|| _vm.ctx.none()), + )?, + value: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "value", "TypeAlias")?, + )?, + range: range_from_object(_vm, source_file, _object, "TypeAlias")?, + }) + } +} + +// constructor +impl Node for ast::StmtAugAssign { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + target, + op, + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtAugAssign::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("op", op.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + target: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "target", "AugAssign")?, + )?, + op: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "op", "AugAssign")?, + )?, + value: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "value", "AugAssign")?, + )?, + range: range_from_object(_vm, source_file, _object, "AugAssign")?, + }) + } +} + +// constructor +impl Node for ast::StmtAnnAssign { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + target, + annotation, + value, + simple, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtAnnAssign::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item( + "annotation", + annotation.ast_to_object(_vm, source_file), + _vm, + ) + .unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("simple", simple.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + target: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "target", "AnnAssign")?, + )?, + annotation: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "annotation", "AnnAssign")?, + )?, + value: get_node_field_opt(_vm, &_object, "value")? + .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + .transpose()?, + simple: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "simple", "AnnAssign")?, + )?, + range: range_from_object(_vm, source_file, _object, "AnnAssign")?, + }) + } +} + +// constructor +impl Node for ast::StmtFor { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + is_async, + target, + iter, + body, + orelse, + // type_comment, + range: _range, + } = self; + + let cls = if !is_async { + pyast::NodeStmtFor::static_type().to_owned() + } else { + pyast::NodeStmtAsyncFor::static_type().to_owned() + }; + + let node = NodeAst.into_ref_with_type(_vm, cls).unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("target", target.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("iter", iter.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("orelse", orelse.ast_to_object(_vm, source_file), _vm) + .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + let _cls = _object.class(); + debug_assert!( + _cls.is(pyast::NodeStmtFor::static_type()) + || _cls.is(pyast::NodeStmtAsyncFor::static_type()) + ); + let is_async = _cls.is(pyast::NodeStmtAsyncFor::static_type()); + Ok(Self { + node_index: Default::default(), + target: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "target", "For")?, + )?, + iter: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "iter", "For")?, + )?, + body: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "body", "For")?, + )?, + orelse: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "orelse", "For")?, + )?, + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(_vm, source_file, _object, "For")?, + is_async, + }) + } +} + +// constructor +impl Node for ast::StmtWhile { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + test, + body, + orelse, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtWhile::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("test", test.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("orelse", orelse.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + test: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "test", "While")?, + )?, + body: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "body", "While")?, + )?, + orelse: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "orelse", "While")?, + )?, + range: range_from_object(_vm, source_file, _object, "While")?, + }) + } +} +// constructor +impl Node for ast::StmtIf { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + test, + body, + range, + elif_else_clauses, + } = self; + elif_else_clause::ast_to_object( + ast::ElifElseClause { + node_index: Default::default(), + range, + test: Some(*test), + body, + }, + elif_else_clauses.into_iter(), + _vm, + source_file, + ) + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + elif_else_clause::ast_from_object(vm, source_file, object) + } +} +// constructor +impl Node for ast::StmtWith { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + is_async, + items, + body, + // type_comment, + range: _range, + } = self; + + let cls = if !is_async { + pyast::NodeStmtWith::static_type().to_owned() + } else { + pyast::NodeStmtAsyncWith::static_type().to_owned() + }; + + let node = NodeAst.into_ref_with_type(_vm, cls).unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("items", items.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) + .unwrap(); + // Ruff AST doesn't track type_comment, so always set to None + dict.set_item("type_comment", _vm.ctx.none(), _vm).unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + let _cls = _object.class(); + debug_assert!( + _cls.is(pyast::NodeStmtWith::static_type()) + || _cls.is(pyast::NodeStmtAsyncWith::static_type()) + ); + let is_async = _cls.is(pyast::NodeStmtAsyncWith::static_type()); + Ok(Self { + node_index: Default::default(), + items: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "items", "With")?, + )?, + body: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "body", "With")?, + )?, + // type_comment: get_node_field_opt(_vm, &_object, "type_comment")? + // .map(|obj| Node::ast_from_object(_vm, obj)) + // .transpose()?, + range: range_from_object(_vm, source_file, _object, "With")?, + is_async, + }) + } +} +// constructor +impl Node for ast::StmtMatch { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + subject, + cases, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtMatch::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("subject", subject.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("cases", cases.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + subject: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "subject", "Match")?, + )?, + cases: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "cases", "Match")?, + )?, + range: range_from_object(_vm, source_file, _object, "Match")?, + }) + } +} +// constructor +impl Node for ast::StmtRaise { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + exc, + cause, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtRaise::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("exc", exc.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("cause", cause.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + exc: get_node_field_opt(_vm, &_object, "exc")? + .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + .transpose()?, + cause: get_node_field_opt(_vm, &_object, "cause")? + .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + .transpose()?, + range: range_from_object(_vm, source_file, _object, "Raise")?, + }) + } +} +// constructor +impl Node for ast::StmtTry { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + body, + handlers, + orelse, + finalbody, + range: _range, + is_star, + } = self; + + // let cls = gen::NodeStmtTry::static_type().to_owned(); + let cls = if is_star { + pyast::NodeStmtTryStar::static_type() + } else { + pyast::NodeStmtTry::static_type() + } + .to_owned(); + + let node = NodeAst.into_ref_with_type(_vm, cls).unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("body", body.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("handlers", handlers.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("orelse", orelse.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("finalbody", finalbody.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + let _cls = _object.class(); + let is_star = _cls.is(pyast::NodeStmtTryStar::static_type()); + let _cls = _object.class(); + debug_assert!( + _cls.is(pyast::NodeStmtTry::static_type()) + || _cls.is(pyast::NodeStmtTryStar::static_type()) + ); + + Ok(Self { + node_index: Default::default(), + body: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "body", "Try")?, + )?, + handlers: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "handlers", "Try")?, + )?, + orelse: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "orelse", "Try")?, + )?, + finalbody: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "finalbody", "Try")?, + )?, + range: range_from_object(_vm, source_file, _object, "Try")?, + is_star, + }) + } +} +// constructor +impl Node for ast::StmtAssert { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + test, + msg, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtAssert::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("test", test.ast_to_object(_vm, source_file), _vm) + .unwrap(); + dict.set_item("msg", msg.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + test: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "test", "Assert")?, + )?, + msg: get_node_field_opt(_vm, &_object, "msg")? + .map(|obj| Node::ast_from_object(_vm, source_file, obj)) + .transpose()?, + range: range_from_object(_vm, source_file, _object, "Assert")?, + }) + } +} +// constructor +impl Node for ast::StmtImport { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + names, + range: _range, + is_lazy: _, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtImport::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("names", names.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + names: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "names", "Import")?, + )?, + range: range_from_object(_vm, source_file, _object, "Import")?, + is_lazy: false, // Placeholder + }) + } +} +// constructor +impl Node for ast::StmtImportFrom { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + module, + names, + level, + range, + is_lazy: _, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeStmtImportFrom::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("module", module.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("names", names.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("level", vm.ctx.new_int(level).to_pyobject(vm), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + module: get_node_field_opt(vm, &_object, "module")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + names: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &_object, "names", "ImportFrom")?, + )?, + level: get_node_field_opt(vm, &_object, "level")? + .map(|obj| -> PyResult<u32> { + let int: PyRef<PyInt> = obj.try_into_value(vm)?; + let value: i64 = int.try_to_primitive(vm)?; + if value < 0 { + return Err(vm.new_value_error("Negative ImportFrom level")); + } + u32::try_from(value) + .map_err(|_| vm.new_overflow_error("ImportFrom level out of range")) + }) + .transpose()? + .unwrap_or(0), + range: range_from_object(vm, source_file, _object, "ImportFrom")?, + is_lazy: false, // Placeholder + }) + } +} +// constructor +impl Node for ast::StmtGlobal { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + names, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtGlobal::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("names", names.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + names: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "names", "Global")?, + )?, + range: range_from_object(_vm, source_file, _object, "Global")?, + }) + } +} +// constructor +impl Node for ast::StmtNonlocal { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + names, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtNonlocal::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("names", names.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + names: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "names", "Nonlocal")?, + )?, + range: range_from_object(_vm, source_file, _object, "Nonlocal")?, + }) + } +} +// constructor +impl Node for ast::StmtExpr { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + value, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtExpr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(_vm, source_file), _vm) + .unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + value: Node::ast_from_object( + _vm, + source_file, + get_node_field(_vm, &_object, "value", "Expr")?, + )?, + range: range_from_object(_vm, source_file, _object, "Expr")?, + }) + } +} +// constructor +impl Node for ast::StmtPass { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtPass::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + let location = super::text_range_to_source_range(source_file, _range); + let start_row = location.start.row.get(); + let start_col = location.start.column.get(); + let mut end_row = location.end.row.get(); + let mut end_col = location.end.column.get(); + + // Align with CPython: when docstring optimization replaces a lone + // docstring with `pass`, the end position is on the same line even if + // it extends past the physical line length. + if end_row != start_row && _range.len() == TextSize::from(4) { + end_row = start_row; + end_col = start_col + 4; + } + + dict.set_item("lineno", _vm.ctx.new_int(start_row).into(), _vm) + .unwrap(); + dict.set_item("col_offset", _vm.ctx.new_int(start_col).into(), _vm) + .unwrap(); + dict.set_item("end_lineno", _vm.ctx.new_int(end_row).into(), _vm) + .unwrap(); + dict.set_item("end_col_offset", _vm.ctx.new_int(end_col).into(), _vm) + .unwrap(); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + range: range_from_object(_vm, source_file, _object, "Pass")?, + }) + } +} +// constructor +impl Node for ast::StmtBreak { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtBreak::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + range: range_from_object(_vm, source_file, _object, "Break")?, + }) + } +} + +// constructor +impl Node for ast::StmtContinue { + fn ast_to_object(self, _vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + range: _range, + } = self; + let node = NodeAst + .into_ref_with_type(_vm, pyast::NodeStmtContinue::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + node_add_location(&dict, _range, _vm, source_file); + node.into() + } + fn ast_from_object( + _vm: &VirtualMachine, + source_file: &SourceFile, + _object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + range: range_from_object(_vm, source_file, _object, "Continue")?, + }) + } +} diff --git a/crates/vm/src/stdlib/_ast/string.rs b/crates/vm/src/stdlib/_ast/string.rs new file mode 100644 index 00000000000..24cae476694 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/string.rs @@ -0,0 +1,957 @@ +use super::constant::{Constant, ConstantLiteral}; +use super::*; +use crate::warn; +use ast::str_prefix::StringLiteralPrefix; + +fn ruff_fstring_element_into_iter( + mut fstring_element: ast::InterpolatedStringElements, +) -> impl Iterator<Item = ast::InterpolatedStringElement> { + let default = ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { + node_index: Default::default(), + range: Default::default(), + value: Default::default(), + }); + fstring_element + .iter_mut() + .map(move |elem| core::mem::replace(elem, default.clone())) + .collect::<Vec<_>>() + .into_iter() +} + +fn ruff_fstring_element_to_joined_str_part( + element: ast::InterpolatedStringElement, +) -> JoinedStrPart { + match element { + ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { + range, + value, + node_index: _, + }) => JoinedStrPart::Constant(Constant::new_str( + value, + ast::str_prefix::StringLiteralPrefix::Empty, + range, + )), + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { + range, + expression, + debug_text: _, // TODO: What is this? + conversion, + format_spec, + node_index: _, + }) => JoinedStrPart::FormattedValue(FormattedValue { + value: expression, + conversion, + format_spec: ruff_format_spec_to_joined_str(format_spec), + range, + }), + } +} + +fn push_joined_str_literal( + output: &mut Vec<JoinedStrPart>, + pending: &mut Option<(String, StringLiteralPrefix, TextRange)>, +) { + if let Some((value, prefix, range)) = pending.take() + && !value.is_empty() + { + output.push(JoinedStrPart::Constant(Constant::new_str( + value, prefix, range, + ))); + } +} + +fn normalize_joined_str_parts(values: Vec<JoinedStrPart>) -> Vec<JoinedStrPart> { + let mut output = Vec::with_capacity(values.len()); + let mut pending: Option<(String, StringLiteralPrefix, TextRange)> = None; + + for part in values { + match part { + JoinedStrPart::Constant(constant) => { + let ConstantLiteral::Str { value, prefix } = constant.value else { + push_joined_str_literal(&mut output, &mut pending); + output.push(JoinedStrPart::Constant(constant)); + continue; + }; + let value: String = value.into(); + if let Some((pending_value, _, _)) = pending.as_mut() { + pending_value.push_str(&value); + } else { + pending = Some((value, prefix, constant.range)); + } + } + JoinedStrPart::FormattedValue(value) => { + push_joined_str_literal(&mut output, &mut pending); + output.push(JoinedStrPart::FormattedValue(value)); + } + } + } + + push_joined_str_literal(&mut output, &mut pending); + output +} + +fn push_template_str_literal( + output: &mut Vec<TemplateStrPart>, + pending: &mut Option<(String, StringLiteralPrefix, TextRange)>, +) { + if let Some((value, prefix, range)) = pending.take() + && !value.is_empty() + { + output.push(TemplateStrPart::Constant(Constant::new_str( + value, prefix, range, + ))); + } +} + +fn normalize_template_str_parts(values: Vec<TemplateStrPart>) -> Vec<TemplateStrPart> { + let mut output = Vec::with_capacity(values.len()); + let mut pending: Option<(String, StringLiteralPrefix, TextRange)> = None; + + for part in values { + match part { + TemplateStrPart::Constant(constant) => { + let ConstantLiteral::Str { value, prefix } = constant.value else { + push_template_str_literal(&mut output, &mut pending); + output.push(TemplateStrPart::Constant(constant)); + continue; + }; + let value: String = value.into(); + if let Some((pending_value, _, _)) = pending.as_mut() { + pending_value.push_str(&value); + } else { + pending = Some((value, prefix, constant.range)); + } + } + TemplateStrPart::Interpolation(value) => { + push_template_str_literal(&mut output, &mut pending); + output.push(TemplateStrPart::Interpolation(value)); + } + } + } + + push_template_str_literal(&mut output, &mut pending); + output +} + +fn warn_invalid_escape_sequences_in_format_spec( + vm: &VirtualMachine, + source_file: &SourceFile, + range: TextRange, +) { + let source = source_file.source_text(); + let start = range.start().to_usize(); + let end = range.end().to_usize(); + if start >= end || end > source.len() { + return; + } + let mut raw = &source[start..end]; + if raw.starts_with(':') { + raw = &raw[1..]; + } + + let mut chars = raw.chars().peekable(); + while let Some(ch) = chars.next() { + if ch != '\\' { + continue; + } + let Some(next) = chars.next() else { + break; + }; + let valid = match next { + '\\' | '\'' | '"' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' => true, + '\n' => true, + '\r' => { + if let Some('\n') = chars.peek().copied() { + chars.next(); + } + true + } + '0'..='7' => { + for _ in 0..2 { + if let Some('0'..='7') = chars.peek().copied() { + chars.next(); + } else { + break; + } + } + true + } + 'x' => { + for _ in 0..2 { + if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + chars.next(); + } else { + break; + } + } + true + } + 'u' => { + for _ in 0..4 { + if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + chars.next(); + } else { + break; + } + } + true + } + 'U' => { + for _ in 0..8 { + if chars.peek().is_some_and(|c| c.is_ascii_hexdigit()) { + chars.next(); + } else { + break; + } + } + true + } + 'N' => { + if let Some('{') = chars.peek().copied() { + chars.next(); + for c in chars.by_ref() { + if c == '}' { + break; + } + } + } + true + } + _ => false, + }; + if !valid { + let message = vm.ctx.new_str(format!( + "\"\\{next}\" is an invalid escape sequence. Such sequences will not work in the future. Did you mean \"\\\\{next}\"? A raw string is also an option." + )); + let _ = warn::warn( + message.into(), + Some(vm.ctx.exceptions.syntax_warning.to_owned()), + 1, + None, + vm, + ); + } + } +} + +fn ruff_format_spec_to_joined_str( + format_spec: Option<Box<ast::InterpolatedStringFormatSpec>>, +) -> Option<Box<JoinedStr>> { + match format_spec { + None => None, + Some(format_spec) => { + let ast::InterpolatedStringFormatSpec { + range, + elements, + node_index: _, + } = *format_spec; + let range = if range.start() > ruff_text_size::TextSize::from(0) { + TextRange::new( + range.start() - ruff_text_size::TextSize::from(1), + range.end(), + ) + } else { + range + }; + let values: Vec<_> = ruff_fstring_element_into_iter(elements) + .map(ruff_fstring_element_to_joined_str_part) + .collect(); + let values = normalize_joined_str_parts(values).into_boxed_slice(); + Some(Box::new(JoinedStr { range, values })) + } + } +} + +fn ruff_fstring_element_to_ruff_fstring_part( + element: ast::InterpolatedStringElement, +) -> ast::FStringPart { + match element { + ast::InterpolatedStringElement::Literal(value) => { + let ast::InterpolatedStringLiteralElement { + node_index, + range, + value, + } = value; + ast::FStringPart::Literal(ast::StringLiteral { + node_index, + range, + value, + flags: ast::StringLiteralFlags::empty(), + }) + } + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { + range, .. + }) => ast::FStringPart::FString(ast::FString { + node_index: Default::default(), + range, + elements: vec![element].into(), + flags: ast::FStringFlags::empty(), + }), + } +} + +fn joined_str_to_ruff_format_spec( + joined_str: Option<Box<JoinedStr>>, +) -> Option<Box<ast::InterpolatedStringFormatSpec>> { + match joined_str { + None => None, + Some(joined_str) => { + let JoinedStr { range, values } = *joined_str; + let elements: Vec<_> = Box::into_iter(values) + .map(joined_str_part_to_ruff_fstring_element) + .collect(); + let format_spec = ast::InterpolatedStringFormatSpec { + node_index: Default::default(), + range, + elements: elements.into(), + }; + Some(Box::new(format_spec)) + } + } +} + +#[derive(Debug)] +pub(super) struct JoinedStr { + pub(super) range: TextRange, + pub(super) values: Box<[JoinedStrPart]>, +} + +impl JoinedStr { + pub(super) fn into_expr(self) -> ast::Expr { + let Self { range, values } = self; + ast::Expr::FString(ast::ExprFString { + node_index: Default::default(), + range: Default::default(), + value: match values.len() { + // ruff represents an empty fstring like this: + 0 => ast::FStringValue::single(ast::FString { + node_index: Default::default(), + range, + elements: vec![].into(), + flags: ast::FStringFlags::empty(), + }), + 1 => ast::FStringValue::single( + Box::<[_]>::into_iter(values) + .map(joined_str_part_to_ruff_fstring_element) + .map(|element| ast::FString { + node_index: Default::default(), + range, + elements: vec![element].into(), + flags: ast::FStringFlags::empty(), + }) + .next() + .expect("FString has exactly one part"), + ), + _ => ast::FStringValue::concatenated( + Box::<[_]>::into_iter(values) + .map(joined_str_part_to_ruff_fstring_element) + .map(ruff_fstring_element_to_ruff_fstring_part) + .collect(), + ), + }, + }) + } +} + +fn joined_str_part_to_ruff_fstring_element(part: JoinedStrPart) -> ast::InterpolatedStringElement { + match part { + JoinedStrPart::FormattedValue(value) => { + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { + node_index: Default::default(), + range: value.range, + expression: value.value.clone(), + debug_text: None, // TODO: What is this? + conversion: value.conversion, + format_spec: joined_str_to_ruff_format_spec(value.format_spec), + }) + } + JoinedStrPart::Constant(value) => { + ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { + node_index: Default::default(), + range: value.range, + value: match value.value { + ConstantLiteral::Str { value, .. } => value, + _ => todo!(), + }, + }) + } + } +} + +// constructor +impl Node for JoinedStr { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { values, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprJoinedStr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "values", + BoxedSlice(values).ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let values: BoxedSlice<_> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "values", "JoinedStr")?, + )?; + Ok(Self { + values: values.0, + range: range_from_object(vm, source_file, object, "JoinedStr")?, + }) + } +} + +#[derive(Debug)] +pub(super) enum JoinedStrPart { + FormattedValue(FormattedValue), + Constant(Constant), +} + +// constructor +impl Node for JoinedStrPart { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::FormattedValue(value) => value.ast_to_object(vm, source_file), + Self::Constant(value) => value.ast_to_object(vm, source_file), + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + if cls.is(pyast::NodeExprFormattedValue::static_type()) { + Ok(Self::FormattedValue(Node::ast_from_object( + vm, + source_file, + object, + )?)) + } else { + Ok(Self::Constant(Node::ast_from_object( + vm, + source_file, + object, + )?)) + } + } +} + +#[derive(Debug)] +pub(super) struct FormattedValue { + value: Box<ast::Expr>, + conversion: ast::ConversionFlag, + format_spec: Option<Box<JoinedStr>>, + range: TextRange, +} + +// constructor +impl Node for FormattedValue { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + value, + conversion, + format_spec, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprFormattedValue::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("conversion", conversion.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item( + "format_spec", + format_spec.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "FormattedValue")?, + )?, + conversion: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "conversion", "FormattedValue")?, + )?, + format_spec: get_node_field_opt(vm, &object, "format_spec")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "FormattedValue")?, + }) + } +} + +pub(super) fn fstring_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + expression: ast::ExprFString, +) -> PyObjectRef { + let ast::ExprFString { + range, + mut value, + node_index: _, + } = expression; + let default_part = ast::FStringPart::FString(ast::FString { + node_index: Default::default(), + range: Default::default(), + elements: Default::default(), + flags: ast::FStringFlags::empty(), + }); + let mut values = Vec::new(); + for i in 0..value.as_slice().len() { + let part = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_part.clone()); + match part { + ast::FStringPart::Literal(ast::StringLiteral { + range, + value, + flags, + node_index: _, + }) => { + values.push(JoinedStrPart::Constant(Constant::new_str( + value, + flags.prefix(), + range, + ))); + } + ast::FStringPart::FString(ast::FString { + range: _, + elements, + flags: _, + node_index: _, + }) => { + for element in ruff_fstring_element_into_iter(elements) { + values.push(ruff_fstring_element_to_joined_str_part(element)); + } + } + } + } + let values = normalize_joined_str_parts(values); + for part in &values { + if let JoinedStrPart::FormattedValue(value) = part + && let Some(format_spec) = &value.format_spec + { + warn_invalid_escape_sequences_in_format_spec(vm, source_file, format_spec.range); + } + } + let c = JoinedStr { + range, + values: values.into_boxed_slice(), + }; + c.ast_to_object(vm, source_file) +} + +// ===== TString (Template String) Support ===== + +fn ruff_tstring_element_to_template_str_part( + element: ast::InterpolatedStringElement, + source_file: &SourceFile, +) -> TemplateStrPart { + match element { + ast::InterpolatedStringElement::Literal(ast::InterpolatedStringLiteralElement { + range, + value, + node_index: _, + }) => TemplateStrPart::Constant(Constant::new_str( + value, + ast::str_prefix::StringLiteralPrefix::Empty, + range, + )), + ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { + range, + expression, + debug_text, + conversion, + format_spec, + node_index: _, + }) => { + let expr_range = + extend_expr_range_with_wrapping_parens(source_file, range, expression.range()) + .unwrap_or_else(|| expression.range()); + let expr_str = if let Some(debug_text) = debug_text { + let expr_source = source_file.slice(expr_range); + let mut expr_with_debug = String::with_capacity( + debug_text.leading.len() + expr_source.len() + debug_text.trailing.len(), + ); + expr_with_debug.push_str(&debug_text.leading); + expr_with_debug.push_str(expr_source); + expr_with_debug.push_str(&debug_text.trailing); + strip_interpolation_expr(&expr_with_debug) + } else { + tstring_interpolation_expr_str(source_file, range, expr_range) + }; + TemplateStrPart::Interpolation(TStringInterpolation { + value: expression, + str: expr_str, + conversion, + format_spec: ruff_format_spec_to_joined_str(format_spec), + range, + }) + } + } +} + +fn tstring_interpolation_expr_str( + source_file: &SourceFile, + interpolation_range: TextRange, + expr_range: TextRange, +) -> String { + let expr_range = + extend_expr_range_with_wrapping_parens(source_file, interpolation_range, expr_range) + .unwrap_or(expr_range); + let start = interpolation_range.start() + TextSize::from(1); + let start = if start > expr_range.end() { + expr_range.start() + } else { + start + }; + let expr_source = source_file.slice(TextRange::new(start, expr_range.end())); + strip_interpolation_expr(expr_source) +} + +fn extend_expr_range_with_wrapping_parens( + source_file: &SourceFile, + interpolation_range: TextRange, + expr_range: TextRange, +) -> Option<TextRange> { + let left_slice = source_file.slice(TextRange::new( + interpolation_range.start(), + expr_range.start(), + )); + let mut left_char: Option<(usize, char)> = None; + for (idx, ch) in left_slice + .char_indices() + .collect::<Vec<_>>() + .into_iter() + .rev() + { + if !ch.is_whitespace() { + left_char = Some((idx, ch)); + break; + } + } + let (left_idx, left_ch) = left_char?; + if left_ch != '(' { + return None; + } + + let right_slice = + source_file.slice(TextRange::new(expr_range.end(), interpolation_range.end())); + let mut right_char: Option<(usize, char)> = None; + for (idx, ch) in right_slice.char_indices() { + if !ch.is_whitespace() { + right_char = Some((idx, ch)); + break; + } + } + let (right_idx, right_ch) = right_char?; + if right_ch != ')' { + return None; + } + + let left_pos = interpolation_range.start() + TextSize::from(left_idx as u32); + let right_pos = expr_range.end() + TextSize::from(right_idx as u32); + Some(TextRange::new(left_pos, right_pos + TextSize::from(1))) +} + +fn strip_interpolation_expr(expr_source: &str) -> String { + let mut end = expr_source.len(); + for (idx, ch) in expr_source.char_indices().rev() { + if ch.is_whitespace() || ch == '=' { + end = idx; + continue; + } + end = idx + ch.len_utf8(); + break; + } + expr_source[..end].to_owned() +} + +#[derive(Debug)] +pub(super) struct TemplateStr { + pub(super) range: TextRange, + pub(super) values: Box<[TemplateStrPart]>, +} + +pub(super) fn template_str_to_expr( + vm: &VirtualMachine, + template: TemplateStr, +) -> PyResult<ast::Expr> { + let TemplateStr { range, values } = template; + let elements = template_parts_to_elements(vm, values)?; + let tstring = ast::TString { + range, + node_index: Default::default(), + elements, + flags: ast::TStringFlags::empty(), + }; + Ok(ast::Expr::TString(ast::ExprTString { + node_index: Default::default(), + range, + value: ast::TStringValue::single(tstring), + })) +} + +pub(super) fn interpolation_to_expr( + vm: &VirtualMachine, + interpolation: TStringInterpolation, +) -> PyResult<ast::Expr> { + let part = TemplateStrPart::Interpolation(interpolation); + let elements = template_parts_to_elements(vm, vec![part].into_boxed_slice())?; + let range = TextRange::default(); + let tstring = ast::TString { + range, + node_index: Default::default(), + elements, + flags: ast::TStringFlags::empty(), + }; + Ok(ast::Expr::TString(ast::ExprTString { + node_index: Default::default(), + range, + value: ast::TStringValue::single(tstring), + })) +} + +fn template_parts_to_elements( + vm: &VirtualMachine, + values: Box<[TemplateStrPart]>, +) -> PyResult<ast::InterpolatedStringElements> { + let mut elements = Vec::with_capacity(values.len()); + for value in values.into_vec() { + elements.push(template_part_to_element(vm, value)?); + } + Ok(ast::InterpolatedStringElements::from(elements)) +} + +fn template_part_to_element( + vm: &VirtualMachine, + part: TemplateStrPart, +) -> PyResult<ast::InterpolatedStringElement> { + match part { + TemplateStrPart::Constant(constant) => { + let ConstantLiteral::Str { value, .. } = constant.value else { + return Err(vm.new_type_error("TemplateStr constant values must be strings")); + }; + Ok(ast::InterpolatedStringElement::Literal( + ast::InterpolatedStringLiteralElement { + range: constant.range, + node_index: Default::default(), + value, + }, + )) + } + TemplateStrPart::Interpolation(interpolation) => { + let TStringInterpolation { + value, + conversion, + format_spec, + range, + .. + } = interpolation; + let format_spec = joined_str_to_ruff_format_spec(format_spec); + Ok(ast::InterpolatedStringElement::Interpolation( + ast::InterpolatedElement { + range, + node_index: Default::default(), + expression: value, + debug_text: None, + conversion, + format_spec, + }, + )) + } + } +} + +// constructor +impl Node for TemplateStr { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { values, range } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprTemplateStr::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item( + "values", + BoxedSlice(values).ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let values: BoxedSlice<_> = Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "values", "TemplateStr")?, + )?; + Ok(Self { + values: values.0, + range: range_from_object(vm, source_file, object, "TemplateStr")?, + }) + } +} + +#[derive(Debug)] +pub(super) enum TemplateStrPart { + Interpolation(TStringInterpolation), + Constant(Constant), +} + +// constructor +impl Node for TemplateStrPart { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::Interpolation(value) => value.ast_to_object(vm, source_file), + Self::Constant(value) => value.ast_to_object(vm, source_file), + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + if cls.is(pyast::NodeExprInterpolation::static_type()) { + Ok(Self::Interpolation(Node::ast_from_object( + vm, + source_file, + object, + )?)) + } else { + Ok(Self::Constant(Node::ast_from_object( + vm, + source_file, + object, + )?)) + } + } +} + +#[derive(Debug)] +pub(super) struct TStringInterpolation { + value: Box<ast::Expr>, + str: String, + conversion: ast::ConversionFlag, + format_spec: Option<Box<JoinedStr>>, + range: TextRange, +} + +// constructor +impl Node for TStringInterpolation { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + value, + str, + conversion, + format_spec, + range, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeExprInterpolation::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("value", value.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("str", vm.ctx.new_str(str).into(), vm) + .unwrap(); + dict.set_item("conversion", conversion.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item( + "format_spec", + format_spec.ast_to_object(vm, source_file), + vm, + ) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let str_obj = get_node_field(vm, &object, "str", "Interpolation")?; + let str_val: String = str_obj.try_into_value(vm)?; + Ok(Self { + value: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "value", "Interpolation")?, + )?, + str: str_val, + conversion: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "conversion", "Interpolation")?, + )?, + format_spec: get_node_field_opt(vm, &object, "format_spec")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + range: range_from_object(vm, source_file, object, "Interpolation")?, + }) + } +} + +pub(super) fn tstring_to_object( + vm: &VirtualMachine, + source_file: &SourceFile, + expression: ast::ExprTString, +) -> PyObjectRef { + let ast::ExprTString { + range, + mut value, + node_index: _, + } = expression; + let default_tstring = ast::TString { + node_index: Default::default(), + range: Default::default(), + elements: Default::default(), + flags: ast::TStringFlags::empty(), + }; + let mut values = Vec::new(); + for i in 0..value.as_slice().len() { + let tstring = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_tstring.clone()); + for element in ruff_fstring_element_into_iter(tstring.elements) { + values.push(ruff_tstring_element_to_template_str_part( + element, + source_file, + )); + } + } + let values = normalize_template_str_parts(values); + let c = TemplateStr { + range, + values: values.into_boxed_slice(), + }; + c.ast_to_object(vm, source_file) +} diff --git a/crates/vm/src/stdlib/_ast/type_ignore.rs b/crates/vm/src/stdlib/_ast/type_ignore.rs new file mode 100644 index 00000000000..6e90ba9b80e --- /dev/null +++ b/crates/vm/src/stdlib/_ast/type_ignore.rs @@ -0,0 +1,74 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +pub(super) enum TypeIgnore { + TypeIgnore(TypeIgnoreTypeIgnore), +} + +// sum +impl Node for TypeIgnore { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::TypeIgnore(cons) => cons.ast_to_object(vm, source_file), + } + } + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeTypeIgnoreTypeIgnore::static_type()) { + Self::TypeIgnore(TypeIgnoreTypeIgnore::ast_from_object( + vm, + source_file, + object, + )?) + } else { + return Err(vm.new_type_error(format!( + "expected some sort of type_ignore, but got {}", + object.repr(vm)? + ))); + }) + } +} + +pub(super) struct TypeIgnoreTypeIgnore { + range: TextRange, + lineno: PyRefExact<PyInt>, + tag: PyRefExact<PyStr>, +} + +// constructor +impl Node for TypeIgnoreTypeIgnore { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { lineno, tag, range } = self; + let node = NodeAst + .into_ref_with_type( + vm, + pyast::NodeTypeIgnoreTypeIgnore::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("lineno", lineno.to_pyobject(vm), vm).unwrap(); + dict.set_item("tag", tag.to_pyobject(vm), vm).unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + lineno: get_node_field(vm, &object, "lineno", "TypeIgnore")? + .downcast_exact(vm) + .unwrap(), + tag: get_node_field(vm, &object, "tag", "TypeIgnore")? + .downcast_exact(vm) + .unwrap(), + range: range_from_object(vm, source_file, object, "TypeIgnore")?, + }) + } +} diff --git a/crates/vm/src/stdlib/_ast/type_parameters.rs b/crates/vm/src/stdlib/_ast/type_parameters.rs new file mode 100644 index 00000000000..0424ffbd768 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/type_parameters.rs @@ -0,0 +1,209 @@ +use super::*; +use rustpython_compiler_core::SourceFile; + +impl Node for ast::TypeParams { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + self.type_params.ast_to_object(vm, source_file) + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let type_params: Vec<ast::TypeParam> = Node::ast_from_object(vm, source_file, object)?; + let range = Option::zip(type_params.first(), type_params.last()) + .map(|(first, last)| first.range().cover(last.range())) + .unwrap_or_default(); + Ok(Self { + node_index: Default::default(), + type_params, + range, + }) + } + + fn is_none(&self) -> bool { + self.type_params.is_empty() + } +} + +// sum +impl Node for ast::TypeParam { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + match self { + Self::TypeVar(cons) => cons.ast_to_object(vm, source_file), + Self::ParamSpec(cons) => cons.ast_to_object(vm, source_file), + Self::TypeVarTuple(cons) => cons.ast_to_object(vm, source_file), + } + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + let cls = object.class(); + Ok(if cls.is(pyast::NodeTypeParamTypeVar::static_type()) { + Self::TypeVar(ast::TypeParamTypeVar::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodeTypeParamParamSpec::static_type()) { + Self::ParamSpec(ast::TypeParamParamSpec::ast_from_object( + vm, + source_file, + object, + )?) + } else if cls.is(pyast::NodeTypeParamTypeVarTuple::static_type()) { + Self::TypeVarTuple(ast::TypeParamTypeVarTuple::ast_from_object( + vm, + source_file, + object, + )?) + } else { + return Err(vm.new_type_error(format!( + "expected some sort of type_param, but got {}", + object.repr(vm)? + ))); + }) + } +} + +// constructor +impl Node for ast::TypeParamTypeVar { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + bound, + range, + default, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeTypeParamTypeVar::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("bound", bound.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("default_value", default.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + name: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "name", "TypeVar")?, + )?, + bound: get_node_field_opt(vm, &object, "bound")? + .map(|obj| Node::ast_from_object(vm, source_file, obj)) + .transpose()?, + default: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "default_value", "TypeVar")?, + )?, + range: range_from_object(vm, source_file, object, "TypeVar")?, + }) + } +} + +// constructor +impl Node for ast::TypeParamParamSpec { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + range, + default, + } = self; + let node = NodeAst + .into_ref_with_type(vm, pyast::NodeTypeParamParamSpec::static_type().to_owned()) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("default_value", default.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + name: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "name", "ParamSpec")?, + )?, + default: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "default_value", "ParamSpec")?, + )?, + range: range_from_object(vm, source_file, object, "ParamSpec")?, + }) + } +} + +// constructor +impl Node for ast::TypeParamTypeVarTuple { + fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef { + let Self { + node_index: _, + name, + range, + default, + } = self; + let node = NodeAst + .into_ref_with_type( + vm, + pyast::NodeTypeParamTypeVarTuple::static_type().to_owned(), + ) + .unwrap(); + let dict = node.as_object().dict().unwrap(); + dict.set_item("name", name.ast_to_object(vm, source_file), vm) + .unwrap(); + dict.set_item("default_value", default.ast_to_object(vm, source_file), vm) + .unwrap(); + node_add_location(&dict, range, vm, source_file); + node.into() + } + + fn ast_from_object( + vm: &VirtualMachine, + source_file: &SourceFile, + object: PyObjectRef, + ) -> PyResult<Self> { + Ok(Self { + node_index: Default::default(), + name: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "name", "TypeVarTuple")?, + )?, + default: Node::ast_from_object( + vm, + source_file, + get_node_field(vm, &object, "default_value", "TypeVarTuple")?, + )?, + range: range_from_object(vm, source_file, object, "TypeVarTuple")?, + }) + } +} diff --git a/crates/vm/src/stdlib/_ast/validate.rs b/crates/vm/src/stdlib/_ast/validate.rs new file mode 100644 index 00000000000..9957fe4ee39 --- /dev/null +++ b/crates/vm/src/stdlib/_ast/validate.rs @@ -0,0 +1,663 @@ +// spell-checker: ignore assignlist ifexp + +use super::module::Mod; +use crate::{PyResult, VirtualMachine}; +use ruff_python_ast as ast; + +fn expr_context_name(ctx: ast::ExprContext) -> &'static str { + match ctx { + ast::ExprContext::Load => "Load", + ast::ExprContext::Store => "Store", + ast::ExprContext::Del => "Del", + ast::ExprContext::Invalid => "Invalid", + } +} + +fn validate_name(vm: &VirtualMachine, name: &ast::name::Name) -> PyResult<()> { + match name.as_str() { + "None" | "True" | "False" => Err(vm.new_value_error(format!( + "identifier field can't represent '{}' constant", + name.as_str() + ))), + _ => Ok(()), + } +} + +fn validate_comprehension(vm: &VirtualMachine, gens: &[ast::Comprehension]) -> PyResult<()> { + if gens.is_empty() { + return Err(vm.new_value_error("comprehension with no generators")); + } + for comp in gens { + validate_expr(vm, &comp.target, ast::ExprContext::Store)?; + validate_expr(vm, &comp.iter, ast::ExprContext::Load)?; + validate_exprs(vm, &comp.ifs, ast::ExprContext::Load, false)?; + } + Ok(()) +} + +fn validate_keywords(vm: &VirtualMachine, keywords: &[ast::Keyword]) -> PyResult<()> { + for keyword in keywords { + validate_expr(vm, &keyword.value, ast::ExprContext::Load)?; + } + Ok(()) +} + +fn validate_parameters(vm: &VirtualMachine, params: &ast::Parameters) -> PyResult<()> { + for param in params + .posonlyargs + .iter() + .chain(&params.args) + .chain(&params.kwonlyargs) + { + if let Some(annotation) = &param.parameter.annotation { + validate_expr(vm, annotation, ast::ExprContext::Load)?; + } + if let Some(default) = &param.default { + validate_expr(vm, default, ast::ExprContext::Load)?; + } + } + if let Some(vararg) = &params.vararg + && let Some(annotation) = &vararg.annotation + { + validate_expr(vm, annotation, ast::ExprContext::Load)?; + } + if let Some(kwarg) = &params.kwarg + && let Some(annotation) = &kwarg.annotation + { + validate_expr(vm, annotation, ast::ExprContext::Load)?; + } + Ok(()) +} + +fn validate_nonempty_seq( + vm: &VirtualMachine, + len: usize, + what: &'static str, + owner: &'static str, +) -> PyResult<()> { + if len == 0 { + return Err(vm.new_value_error(format!("empty {what} on {owner}"))); + } + Ok(()) +} + +fn validate_assignlist( + vm: &VirtualMachine, + targets: &[ast::Expr], + ctx: ast::ExprContext, +) -> PyResult<()> { + validate_nonempty_seq( + vm, + targets.len(), + "targets", + if ctx == ast::ExprContext::Del { + "Delete" + } else { + "Assign" + }, + )?; + validate_exprs(vm, targets, ctx, false) +} + +fn validate_body(vm: &VirtualMachine, body: &[ast::Stmt], owner: &'static str) -> PyResult<()> { + validate_nonempty_seq(vm, body.len(), "body", owner)?; + validate_stmts(vm, body) +} + +fn validate_interpolated_elements<'a>( + vm: &VirtualMachine, + elements: impl IntoIterator<Item = ast::InterpolatedStringElementRef<'a>>, +) -> PyResult<()> { + for element in elements { + if let ast::InterpolatedStringElementRef::Interpolation(interpolation) = element { + validate_expr(vm, &interpolation.expression, ast::ExprContext::Load)?; + if let Some(format_spec) = &interpolation.format_spec { + for spec_element in &format_spec.elements { + if let ast::InterpolatedStringElement::Interpolation(spec_interp) = spec_element + { + validate_expr(vm, &spec_interp.expression, ast::ExprContext::Load)?; + } + } + } + } + } + Ok(()) +} + +fn validate_pattern_match_value(vm: &VirtualMachine, expr: &ast::Expr) -> PyResult<()> { + validate_expr(vm, expr, ast::ExprContext::Load)?; + match expr { + ast::Expr::NumberLiteral(_) | ast::Expr::StringLiteral(_) | ast::Expr::BytesLiteral(_) => { + Ok(()) + } + ast::Expr::Attribute(_) => Ok(()), + ast::Expr::UnaryOp(op) => match &*op.operand { + ast::Expr::NumberLiteral(_) => Ok(()), + _ => Err(vm.new_value_error("patterns may only match literals and attribute lookups")), + }, + ast::Expr::BinOp(bin) => match (&*bin.left, &*bin.right) { + (ast::Expr::NumberLiteral(_), ast::Expr::NumberLiteral(_)) => Ok(()), + _ => Err(vm.new_value_error("patterns may only match literals and attribute lookups")), + }, + ast::Expr::FString(_) | ast::Expr::TString(_) => Ok(()), + ast::Expr::BooleanLiteral(_) + | ast::Expr::NoneLiteral(_) + | ast::Expr::EllipsisLiteral(_) => { + Err(vm.new_value_error("unexpected constant inside of a literal pattern")) + } + _ => Err(vm.new_value_error("patterns may only match literals and attribute lookups")), + } +} + +fn validate_capture(vm: &VirtualMachine, name: &ast::Identifier) -> PyResult<()> { + if name.as_str() == "_" { + return Err(vm.new_value_error("can't capture name '_' in patterns")); + } + validate_name(vm, name.id()) +} + +fn validate_pattern(vm: &VirtualMachine, pattern: &ast::Pattern, star_ok: bool) -> PyResult<()> { + match pattern { + ast::Pattern::MatchValue(value) => validate_pattern_match_value(vm, &value.value), + ast::Pattern::MatchSingleton(singleton) => match singleton.value { + ast::Singleton::None | ast::Singleton::True | ast::Singleton::False => Ok(()), + }, + ast::Pattern::MatchSequence(seq) => validate_patterns(vm, &seq.patterns, true), + ast::Pattern::MatchMapping(mapping) => { + if mapping.keys.len() != mapping.patterns.len() { + return Err(vm.new_value_error( + "MatchMapping doesn't have the same number of keys as patterns", + )); + } + if let Some(rest) = &mapping.rest { + validate_capture(vm, rest)?; + } + for key in &mapping.keys { + if let ast::Expr::BooleanLiteral(_) | ast::Expr::NoneLiteral(_) = key { + continue; + } + validate_pattern_match_value(vm, key)?; + } + validate_patterns(vm, &mapping.patterns, false) + } + ast::Pattern::MatchClass(match_class) => { + validate_expr(vm, &match_class.cls, ast::ExprContext::Load)?; + let mut cls = match_class.cls.as_ref(); + loop { + match cls { + ast::Expr::Name(_) => break, + ast::Expr::Attribute(attr) => { + cls = &attr.value; + } + _ => { + return Err(vm.new_value_error( + "MatchClass cls field can only contain Name or Attribute nodes.", + )); + } + } + } + for keyword in &match_class.arguments.keywords { + validate_name(vm, keyword.attr.id())?; + } + validate_patterns(vm, &match_class.arguments.patterns, false)?; + for keyword in &match_class.arguments.keywords { + validate_pattern(vm, &keyword.pattern, false)?; + } + Ok(()) + } + ast::Pattern::MatchStar(star) => { + if !star_ok { + return Err(vm.new_value_error("can't use MatchStar here")); + } + if let Some(name) = &star.name { + validate_capture(vm, name)?; + } + Ok(()) + } + ast::Pattern::MatchAs(match_as) => { + if let Some(name) = &match_as.name { + validate_capture(vm, name)?; + } + match &match_as.pattern { + None => Ok(()), + Some(pattern) => { + if match_as.name.is_none() { + return Err(vm.new_value_error( + "MatchAs must specify a target name if a pattern is given", + )); + } + validate_pattern(vm, pattern, false) + } + } + } + ast::Pattern::MatchOr(match_or) => { + if match_or.patterns.len() < 2 { + return Err(vm.new_value_error("MatchOr requires at least 2 patterns")); + } + validate_patterns(vm, &match_or.patterns, false) + } + } +} + +fn validate_patterns( + vm: &VirtualMachine, + patterns: &[ast::Pattern], + star_ok: bool, +) -> PyResult<()> { + for pattern in patterns { + validate_pattern(vm, pattern, star_ok)?; + } + Ok(()) +} + +fn validate_typeparam(vm: &VirtualMachine, tp: &ast::TypeParam) -> PyResult<()> { + match tp { + ast::TypeParam::TypeVar(tp) => { + validate_name(vm, tp.name.id())?; + if let Some(bound) = &tp.bound { + validate_expr(vm, bound, ast::ExprContext::Load)?; + } + if let Some(default) = &tp.default { + validate_expr(vm, default, ast::ExprContext::Load)?; + } + } + ast::TypeParam::ParamSpec(tp) => { + validate_name(vm, tp.name.id())?; + if let Some(default) = &tp.default { + validate_expr(vm, default, ast::ExprContext::Load)?; + } + } + ast::TypeParam::TypeVarTuple(tp) => { + validate_name(vm, tp.name.id())?; + if let Some(default) = &tp.default { + validate_expr(vm, default, ast::ExprContext::Load)?; + } + } + } + Ok(()) +} + +fn validate_type_params( + vm: &VirtualMachine, + type_params: &Option<Box<ast::TypeParams>>, +) -> PyResult<()> { + if let Some(type_params) = type_params { + for tp in &type_params.type_params { + validate_typeparam(vm, tp)?; + } + } + Ok(()) +} + +fn validate_exprs( + vm: &VirtualMachine, + exprs: &[ast::Expr], + ctx: ast::ExprContext, + _null_ok: bool, +) -> PyResult<()> { + for expr in exprs { + validate_expr(vm, expr, ctx)?; + } + Ok(()) +} + +fn validate_expr(vm: &VirtualMachine, expr: &ast::Expr, ctx: ast::ExprContext) -> PyResult<()> { + let mut check_ctx = true; + let actual_ctx = match expr { + ast::Expr::Attribute(attr) => attr.ctx, + ast::Expr::Subscript(sub) => sub.ctx, + ast::Expr::Starred(star) => star.ctx, + ast::Expr::Name(name) => { + validate_name(vm, name.id())?; + name.ctx + } + ast::Expr::List(list) => list.ctx, + ast::Expr::Tuple(tuple) => tuple.ctx, + _ => { + if ctx != ast::ExprContext::Load { + return Err(vm.new_value_error(format!( + "expression which can't be assigned to in {} context", + expr_context_name(ctx) + ))); + } + check_ctx = false; + ast::ExprContext::Invalid + } + }; + if check_ctx && actual_ctx != ctx { + return Err(vm.new_value_error(format!( + "expression must have {} context but has {} instead", + expr_context_name(ctx), + expr_context_name(actual_ctx) + ))); + } + + match expr { + ast::Expr::BoolOp(op) => { + if op.values.len() < 2 { + return Err(vm.new_value_error("BoolOp with less than 2 values")); + } + validate_exprs(vm, &op.values, ast::ExprContext::Load, false) + } + ast::Expr::Named(named) => { + if !matches!(&*named.target, ast::Expr::Name(_)) { + return Err(vm.new_type_error("NamedExpr target must be a Name")); + } + validate_expr(vm, &named.value, ast::ExprContext::Load) + } + ast::Expr::BinOp(bin) => { + validate_expr(vm, &bin.left, ast::ExprContext::Load)?; + validate_expr(vm, &bin.right, ast::ExprContext::Load) + } + ast::Expr::UnaryOp(unary) => validate_expr(vm, &unary.operand, ast::ExprContext::Load), + ast::Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + validate_parameters(vm, parameters)?; + } + validate_expr(vm, &lambda.body, ast::ExprContext::Load) + } + ast::Expr::If(ifexp) => { + validate_expr(vm, &ifexp.test, ast::ExprContext::Load)?; + validate_expr(vm, &ifexp.body, ast::ExprContext::Load)?; + validate_expr(vm, &ifexp.orelse, ast::ExprContext::Load) + } + ast::Expr::Dict(dict) => { + for item in &dict.items { + if let Some(key) = &item.key { + validate_expr(vm, key, ast::ExprContext::Load)?; + } + validate_expr(vm, &item.value, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Expr::Set(set) => validate_exprs(vm, &set.elts, ast::ExprContext::Load, false), + ast::Expr::ListComp(list) => { + validate_comprehension(vm, &list.generators)?; + validate_expr(vm, &list.elt, ast::ExprContext::Load) + } + ast::Expr::SetComp(set) => { + validate_comprehension(vm, &set.generators)?; + validate_expr(vm, &set.elt, ast::ExprContext::Load) + } + ast::Expr::DictComp(dict) => { + validate_comprehension(vm, &dict.generators)?; + validate_expr(vm, &dict.key, ast::ExprContext::Load)?; + validate_expr(vm, &dict.value, ast::ExprContext::Load) + } + ast::Expr::Generator(generator) => { + validate_comprehension(vm, &generator.generators)?; + validate_expr(vm, &generator.elt, ast::ExprContext::Load) + } + ast::Expr::Yield(yield_expr) => { + if let Some(value) = &yield_expr.value { + validate_expr(vm, value, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Expr::YieldFrom(yield_expr) => { + validate_expr(vm, &yield_expr.value, ast::ExprContext::Load) + } + ast::Expr::Await(await_expr) => { + validate_expr(vm, &await_expr.value, ast::ExprContext::Load) + } + ast::Expr::Compare(compare) => { + if compare.comparators.is_empty() { + return Err(vm.new_value_error("Compare with no comparators")); + } + if compare.comparators.len() != compare.ops.len() { + return Err(vm.new_value_error( + "Compare has a different number of comparators and operands", + )); + } + validate_exprs(vm, &compare.comparators, ast::ExprContext::Load, false)?; + validate_expr(vm, &compare.left, ast::ExprContext::Load) + } + ast::Expr::Call(call) => { + validate_expr(vm, &call.func, ast::ExprContext::Load)?; + validate_exprs(vm, &call.arguments.args, ast::ExprContext::Load, false)?; + validate_keywords(vm, &call.arguments.keywords) + } + ast::Expr::FString(fstring) => validate_interpolated_elements( + vm, + fstring + .value + .elements() + .map(ast::InterpolatedStringElementRef::from), + ), + ast::Expr::TString(tstring) => validate_interpolated_elements( + vm, + tstring + .value + .elements() + .map(ast::InterpolatedStringElementRef::from), + ), + ast::Expr::StringLiteral(_) + | ast::Expr::BytesLiteral(_) + | ast::Expr::NumberLiteral(_) + | ast::Expr::BooleanLiteral(_) + | ast::Expr::NoneLiteral(_) + | ast::Expr::EllipsisLiteral(_) => Ok(()), + ast::Expr::Attribute(attr) => validate_expr(vm, &attr.value, ast::ExprContext::Load), + ast::Expr::Subscript(sub) => { + validate_expr(vm, &sub.slice, ast::ExprContext::Load)?; + validate_expr(vm, &sub.value, ast::ExprContext::Load) + } + ast::Expr::Starred(star) => validate_expr(vm, &star.value, ctx), + ast::Expr::Name(_) => Ok(()), + ast::Expr::List(list) => validate_exprs(vm, &list.elts, ctx, false), + ast::Expr::Tuple(tuple) => validate_exprs(vm, &tuple.elts, ctx, false), + ast::Expr::Slice(slice) => { + if let Some(lower) = &slice.lower { + validate_expr(vm, lower, ast::ExprContext::Load)?; + } + if let Some(upper) = &slice.upper { + validate_expr(vm, upper, ast::ExprContext::Load)?; + } + if let Some(step) = &slice.step { + validate_expr(vm, step, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Expr::IpyEscapeCommand(_) => Ok(()), + } +} + +fn validate_decorators(vm: &VirtualMachine, decorators: &[ast::Decorator]) -> PyResult<()> { + for decorator in decorators { + validate_expr(vm, &decorator.expression, ast::ExprContext::Load)?; + } + Ok(()) +} + +fn validate_stmt(vm: &VirtualMachine, stmt: &ast::Stmt) -> PyResult<()> { + match stmt { + ast::Stmt::FunctionDef(func) => { + let owner = if func.is_async { + "AsyncFunctionDef" + } else { + "FunctionDef" + }; + validate_body(vm, &func.body, owner)?; + validate_type_params(vm, &func.type_params)?; + validate_parameters(vm, &func.parameters)?; + validate_decorators(vm, &func.decorator_list)?; + if let Some(returns) = &func.returns { + validate_expr(vm, returns, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Stmt::ClassDef(class_def) => { + validate_body(vm, &class_def.body, "ClassDef")?; + validate_type_params(vm, &class_def.type_params)?; + if let Some(arguments) = &class_def.arguments { + validate_exprs(vm, &arguments.args, ast::ExprContext::Load, false)?; + validate_keywords(vm, &arguments.keywords)?; + } + validate_decorators(vm, &class_def.decorator_list) + } + ast::Stmt::Return(ret) => { + if let Some(value) = &ret.value { + validate_expr(vm, value, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Stmt::Delete(del) => validate_assignlist(vm, &del.targets, ast::ExprContext::Del), + ast::Stmt::Assign(assign) => { + validate_assignlist(vm, &assign.targets, ast::ExprContext::Store)?; + validate_expr(vm, &assign.value, ast::ExprContext::Load) + } + ast::Stmt::AugAssign(assign) => { + validate_expr(vm, &assign.target, ast::ExprContext::Store)?; + validate_expr(vm, &assign.value, ast::ExprContext::Load) + } + ast::Stmt::AnnAssign(assign) => { + if assign.simple && !matches!(&*assign.target, ast::Expr::Name(_)) { + return Err(vm.new_type_error("AnnAssign with simple non-Name target")); + } + validate_expr(vm, &assign.target, ast::ExprContext::Store)?; + if let Some(value) = &assign.value { + validate_expr(vm, value, ast::ExprContext::Load)?; + } + validate_expr(vm, &assign.annotation, ast::ExprContext::Load) + } + ast::Stmt::TypeAlias(alias) => { + if !matches!(&*alias.name, ast::Expr::Name(_)) { + return Err(vm.new_type_error("TypeAlias with non-Name name")); + } + validate_expr(vm, &alias.name, ast::ExprContext::Store)?; + validate_type_params(vm, &alias.type_params)?; + validate_expr(vm, &alias.value, ast::ExprContext::Load) + } + ast::Stmt::For(for_stmt) => { + let owner = if for_stmt.is_async { "AsyncFor" } else { "For" }; + validate_expr(vm, &for_stmt.target, ast::ExprContext::Store)?; + validate_expr(vm, &for_stmt.iter, ast::ExprContext::Load)?; + validate_body(vm, &for_stmt.body, owner)?; + validate_stmts(vm, &for_stmt.orelse) + } + ast::Stmt::While(while_stmt) => { + validate_expr(vm, &while_stmt.test, ast::ExprContext::Load)?; + validate_body(vm, &while_stmt.body, "While")?; + validate_stmts(vm, &while_stmt.orelse) + } + ast::Stmt::If(if_stmt) => { + validate_expr(vm, &if_stmt.test, ast::ExprContext::Load)?; + validate_body(vm, &if_stmt.body, "If")?; + for clause in &if_stmt.elif_else_clauses { + if let Some(test) = &clause.test { + validate_expr(vm, test, ast::ExprContext::Load)?; + } + validate_body(vm, &clause.body, "If")?; + } + Ok(()) + } + ast::Stmt::With(with_stmt) => { + let owner = if with_stmt.is_async { + "AsyncWith" + } else { + "With" + }; + validate_nonempty_seq(vm, with_stmt.items.len(), "items", owner)?; + for item in &with_stmt.items { + validate_expr(vm, &item.context_expr, ast::ExprContext::Load)?; + if let Some(optional_vars) = &item.optional_vars { + validate_expr(vm, optional_vars, ast::ExprContext::Store)?; + } + } + validate_body(vm, &with_stmt.body, owner) + } + ast::Stmt::Match(match_stmt) => { + validate_expr(vm, &match_stmt.subject, ast::ExprContext::Load)?; + validate_nonempty_seq(vm, match_stmt.cases.len(), "cases", "Match")?; + for case in &match_stmt.cases { + validate_pattern(vm, &case.pattern, false)?; + if let Some(guard) = &case.guard { + validate_expr(vm, guard, ast::ExprContext::Load)?; + } + validate_body(vm, &case.body, "match_case")?; + } + Ok(()) + } + ast::Stmt::Raise(raise) => { + if let Some(exc) = &raise.exc { + validate_expr(vm, exc, ast::ExprContext::Load)?; + if let Some(cause) = &raise.cause { + validate_expr(vm, cause, ast::ExprContext::Load)?; + } + } else if raise.cause.is_some() { + return Err(vm.new_value_error("Raise with cause but no exception")); + } + Ok(()) + } + ast::Stmt::Try(try_stmt) => { + let owner = if try_stmt.is_star { "TryStar" } else { "Try" }; + validate_body(vm, &try_stmt.body, owner)?; + if try_stmt.handlers.is_empty() && try_stmt.finalbody.is_empty() { + return Err(vm.new_value_error(format!( + "{owner} has neither except handlers nor finalbody" + ))); + } + if try_stmt.handlers.is_empty() && !try_stmt.orelse.is_empty() { + return Err( + vm.new_value_error(format!("{owner} has orelse but no except handlers")) + ); + } + for handler in &try_stmt.handlers { + let ast::ExceptHandler::ExceptHandler(handler) = handler; + if let Some(type_expr) = &handler.type_ { + validate_expr(vm, type_expr, ast::ExprContext::Load)?; + } + validate_body(vm, &handler.body, "ExceptHandler")?; + } + validate_stmts(vm, &try_stmt.finalbody)?; + validate_stmts(vm, &try_stmt.orelse) + } + ast::Stmt::Assert(assert_stmt) => { + validate_expr(vm, &assert_stmt.test, ast::ExprContext::Load)?; + if let Some(msg) = &assert_stmt.msg { + validate_expr(vm, msg, ast::ExprContext::Load)?; + } + Ok(()) + } + ast::Stmt::Import(import) => { + validate_nonempty_seq(vm, import.names.len(), "names", "Import")?; + Ok(()) + } + ast::Stmt::ImportFrom(import) => { + validate_nonempty_seq(vm, import.names.len(), "names", "ImportFrom")?; + Ok(()) + } + ast::Stmt::Global(global) => { + validate_nonempty_seq(vm, global.names.len(), "names", "Global")?; + Ok(()) + } + ast::Stmt::Nonlocal(nonlocal) => { + validate_nonempty_seq(vm, nonlocal.names.len(), "names", "Nonlocal")?; + Ok(()) + } + ast::Stmt::Expr(expr) => validate_expr(vm, &expr.value, ast::ExprContext::Load), + ast::Stmt::Pass(_) + | ast::Stmt::Break(_) + | ast::Stmt::Continue(_) + | ast::Stmt::IpyEscapeCommand(_) => Ok(()), + } +} + +fn validate_stmts(vm: &VirtualMachine, stmts: &[ast::Stmt]) -> PyResult<()> { + for stmt in stmts { + validate_stmt(vm, stmt)?; + } + Ok(()) +} + +pub(super) fn validate_mod(vm: &VirtualMachine, module: &Mod) -> PyResult<()> { + match module { + Mod::Module(module) => validate_stmts(vm, &module.body), + Mod::Interactive(module) => validate_stmts(vm, &module.body), + Mod::Expression(expr) => validate_expr(vm, &expr.body, ast::ExprContext::Load), + Mod::FunctionType(func_type) => { + validate_exprs(vm, &func_type.argtypes, ast::ExprContext::Load, false)?; + validate_expr(vm, &func_type.returns, ast::ExprContext::Load) + } + } +} diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs new file mode 100644 index 00000000000..39ebb3599bd --- /dev/null +++ b/crates/vm/src/stdlib/_codecs.rs @@ -0,0 +1,1395 @@ +// spell-checker: ignore unencodable pused + +pub(crate) use _codecs::module_def; + +use crate::common::static_cell::StaticCell; + +#[pymodule(with(#[cfg(windows)] _codecs_windows))] +mod _codecs { + use crate::codecs::{ErrorsHandler, PyDecodeContext, PyEncodeContext}; + use crate::common::encodings; + use crate::common::wtf8::Wtf8Buf; + use crate::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyStrRef, PyUtf8StrRef}, + codecs, + exceptions::cstring_error, + function::{ArgBytesLike, FuncArgs}, + }; + + #[pyfunction] + fn register(search_function: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + vm.state.codec_registry.register(search_function, vm) + } + + #[pyfunction] + fn unregister(search_function: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + vm.state.codec_registry.unregister(search_function) + } + + #[pyfunction] + fn lookup(encoding: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + if encoding.as_str().contains('\0') { + return Err(cstring_error(vm)); + } + vm.state + .codec_registry + .lookup(encoding.as_str(), vm) + .map(|codec| codec.into_tuple().into()) + } + + #[derive(FromArgs)] + struct CodeArgs { + obj: PyObjectRef, + #[pyarg(any, optional)] + encoding: Option<PyUtf8StrRef>, + #[pyarg(any, optional)] + errors: Option<PyUtf8StrRef>, + } + + impl CodeArgs { + fn apply( + self, + vm: &VirtualMachine, + f: fn( + &codecs::CodecsRegistry, + PyObjectRef, + &str, + Option<PyUtf8StrRef>, + &VirtualMachine, + ) -> PyResult, + ) -> PyResult { + let encoding = self + .encoding + .as_deref() + .map(|s| s.as_str()) + .unwrap_or(codecs::DEFAULT_ENCODING); + f( + &vm.state.codec_registry, + self.obj, + encoding, + self.errors, + vm, + ) + } + } + + #[pyfunction] + fn encode(args: CodeArgs, vm: &VirtualMachine) -> PyResult { + args.apply(vm, codecs::CodecsRegistry::encode) + } + + #[pyfunction] + fn decode(args: CodeArgs, vm: &VirtualMachine) -> PyResult { + args.apply(vm, codecs::CodecsRegistry::decode) + } + + #[pyfunction] + fn _forget_codec(encoding: PyUtf8StrRef, vm: &VirtualMachine) { + vm.state.codec_registry.forget(encoding.as_str()); + } + + #[pyfunction] + fn register_error( + name: PyUtf8StrRef, + handler: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if !handler.is_callable() { + return Err(vm.new_type_error("handler must be callable")); + } + vm.state + .codec_registry + .register_error(name.as_str().to_owned(), handler); + Ok(()) + } + + #[pyfunction] + fn lookup_error(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + if name.as_str().contains('\0') { + return Err(cstring_error(vm)); + } + vm.state.codec_registry.lookup_error(name.as_str(), vm) + } + + #[pyfunction] + fn _unregister_error(errors: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<bool> { + if errors.as_str().contains('\0') { + return Err(cstring_error(vm)); + } + vm.state + .codec_registry + .unregister_error(errors.as_str(), vm) + } + + type EncodeResult = PyResult<(Vec<u8>, usize)>; + + #[derive(FromArgs)] + struct EncodeArgs { + #[pyarg(positional)] + s: PyStrRef, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + } + + impl EncodeArgs { + #[inline] + fn encode<'a, F>(&'a self, name: &'a str, encode: F, vm: &'a VirtualMachine) -> EncodeResult + where + F: FnOnce(PyEncodeContext<'a>, &ErrorsHandler<'a>) -> PyResult<Vec<u8>>, + { + let ctx = PyEncodeContext::new(name, &self.s, vm); + let errors = ErrorsHandler::new(self.errors.as_deref(), vm); + let encoded = encode(ctx, &errors)?; + Ok((encoded, self.s.char_len())) + } + } + + type DecodeResult = PyResult<(Wtf8Buf, usize)>; + + #[derive(FromArgs)] + struct DecodeArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + #[pyarg(positional, default = false)] + final_decode: bool, + } + + impl DecodeArgs { + #[inline] + fn decode<'a, F>(&'a self, name: &'a str, decode: F, vm: &'a VirtualMachine) -> DecodeResult + where + F: FnOnce(PyDecodeContext<'a>, &ErrorsHandler<'a>, bool) -> DecodeResult, + { + let ctx = PyDecodeContext::new(name, &self.data, vm); + let errors = ErrorsHandler::new(self.errors.as_deref(), vm); + decode(ctx, &errors, self.final_decode) + } + } + + #[derive(FromArgs)] + struct DecodeArgsNoFinal { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + } + + impl DecodeArgsNoFinal { + #[inline] + fn decode<'a, F>(&'a self, name: &'a str, decode: F, vm: &'a VirtualMachine) -> DecodeResult + where + F: FnOnce(PyDecodeContext<'a>, &ErrorsHandler<'a>) -> DecodeResult, + { + let ctx = PyDecodeContext::new(name, &self.data, vm); + let errors = ErrorsHandler::new(self.errors.as_deref(), vm); + decode(ctx, &errors) + } + } + + macro_rules! do_codec { + ($module:ident :: $func:ident, $args: expr, $vm:expr) => {{ + use encodings::$module as codec; + $args.$func(codec::ENCODING_NAME, codec::$func, $vm) + }}; + } + + #[pyfunction] + fn utf_8_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { + if args.s.is_utf8() + || args + .errors + .as_ref() + .is_some_and(|s| s.is(identifier!(vm, surrogatepass))) + { + return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); + } + do_codec!(utf8::encode, args, vm) + } + + #[pyfunction] + fn utf_8_decode(args: DecodeArgs, vm: &VirtualMachine) -> DecodeResult { + do_codec!(utf8::decode, args, vm) + } + + #[pyfunction] + fn latin_1_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { + if args.s.isascii() { + return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); + } + do_codec!(latin_1::encode, args, vm) + } + + #[pyfunction] + fn latin_1_decode(args: DecodeArgsNoFinal, vm: &VirtualMachine) -> DecodeResult { + do_codec!(latin_1::decode, args, vm) + } + + #[pyfunction] + fn ascii_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { + if args.s.isascii() { + return Ok((args.s.as_bytes().to_vec(), args.s.byte_len())); + } + do_codec!(ascii::encode, args, vm) + } + + #[pyfunction] + fn ascii_decode(args: DecodeArgsNoFinal, vm: &VirtualMachine) -> DecodeResult { + do_codec!(ascii::decode, args, vm) + } + + // TODO: implement these codecs in Rust! + + macro_rules! delegate_pycodecs { + ($name:ident, $args:ident, $vm:ident) => {{ + rustpython_common::static_cell!( + static FUNC: PyObjectRef; + ); + super::delegate_pycodecs(&FUNC, stringify!($name), $args, $vm) + }}; + } + + #[pyfunction] + fn readbuffer_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(readbuffer_encode, args, vm) + } + #[pyfunction] + fn escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(escape_encode, args, vm) + } + #[pyfunction] + fn escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(escape_decode, args, vm) + } + #[pyfunction] + fn unicode_escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(unicode_escape_encode, args, vm) + } + #[pyfunction] + fn unicode_escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(unicode_escape_decode, args, vm) + } + #[pyfunction] + fn raw_unicode_escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(raw_unicode_escape_encode, args, vm) + } + #[pyfunction] + fn raw_unicode_escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(raw_unicode_escape_decode, args, vm) + } + #[pyfunction] + fn utf_7_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_7_encode, args, vm) + } + #[pyfunction] + fn utf_7_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_7_decode, args, vm) + } + #[pyfunction] + fn utf_16_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_encode, args, vm) + } + #[pyfunction] + fn utf_16_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_decode, args, vm) + } + #[pyfunction] + fn charmap_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(charmap_encode, args, vm) + } + #[pyfunction] + fn charmap_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(charmap_decode, args, vm) + } + #[pyfunction] + fn charmap_build(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(charmap_build, args, vm) + } + #[pyfunction] + fn utf_16_le_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_le_encode, args, vm) + } + #[pyfunction] + fn utf_16_le_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_le_decode, args, vm) + } + #[pyfunction] + fn utf_16_be_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_be_encode, args, vm) + } + #[pyfunction] + fn utf_16_be_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_be_decode, args, vm) + } + #[pyfunction] + fn utf_16_ex_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_16_ex_decode, args, vm) + } + #[pyfunction] + fn utf_32_ex_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_ex_decode, args, vm) + } + #[pyfunction] + fn utf_32_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_encode, args, vm) + } + #[pyfunction] + fn utf_32_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_decode, args, vm) + } + #[pyfunction] + fn utf_32_le_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_le_encode, args, vm) + } + #[pyfunction] + fn utf_32_le_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_le_decode, args, vm) + } + #[pyfunction] + fn utf_32_be_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_be_encode, args, vm) + } + #[pyfunction] + fn utf_32_be_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + delegate_pycodecs!(utf_32_be_decode, args, vm) + } +} + +#[inline] +fn delegate_pycodecs( + cell: &'static StaticCell<crate::PyObjectRef>, + name: &'static str, + args: crate::function::FuncArgs, + vm: &crate::VirtualMachine, +) -> crate::PyResult { + let f = cell.get_or_try_init(|| { + let module = vm.import("_pycodecs", 0)?; + module.get_attr(name, vm) + })?; + f.call(args, vm) +} + +#[cfg(windows)] +#[pymodule(sub)] +mod _codecs_windows { + use crate::{PyResult, VirtualMachine}; + use crate::{builtins::PyStrRef, builtins::PyUtf8StrRef, function::ArgBytesLike}; + + #[derive(FromArgs)] + struct MbcsEncodeArgs { + #[pyarg(positional)] + s: PyStrRef, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + } + + #[pyfunction] + fn mbcs_encode(args: MbcsEncodeArgs, vm: &VirtualMachine) -> PyResult<(Vec<u8>, usize)> { + use crate::common::windows::ToWideString; + use windows_sys::Win32::Globalization::{ + CP_ACP, WC_NO_BEST_FIT_CHARS, WideCharToMultiByte, + }; + + let errors = args.errors.as_ref().map(|s| s.as_str()).unwrap_or("strict"); + let s = match args.s.to_str() { + Some(s) => s, + None => { + // String contains surrogates - not encodable with mbcs + return Err(vm.new_unicode_encode_error( + "'mbcs' codec can't encode character: surrogates not allowed", + )); + } + }; + let char_len = args.s.char_len(); + + if s.is_empty() { + return Ok((Vec::new(), char_len)); + } + + // Convert UTF-8 string to UTF-16 + let wide: Vec<u16> = std::ffi::OsStr::new(s).to_wide(); + + // Get the required buffer size + let size = unsafe { + WideCharToMultiByte( + CP_ACP, + WC_NO_BEST_FIT_CHARS, + wide.as_ptr(), + wide.len() as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + + if size == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("mbcs_encode failed: {}", err))); + } + + let mut buffer = vec![0u8; size as usize]; + let mut used_default_char: i32 = 0; + + let result = unsafe { + WideCharToMultiByte( + CP_ACP, + WC_NO_BEST_FIT_CHARS, + wide.as_ptr(), + wide.len() as i32, + buffer.as_mut_ptr().cast(), + size, + core::ptr::null(), + if errors == "strict" { + &mut used_default_char + } else { + core::ptr::null_mut() + }, + ) + }; + + if result == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("mbcs_encode failed: {err}"))); + } + + if errors == "strict" && used_default_char != 0 { + return Err(vm.new_unicode_encode_error( + "'mbcs' codec can't encode characters: invalid character", + )); + } + + buffer.truncate(result as usize); + Ok((buffer, char_len)) + } + + #[derive(FromArgs)] + struct MbcsDecodeArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + #[pyarg(positional, default = false)] + #[allow(dead_code)] + r#final: bool, + } + + #[pyfunction] + fn mbcs_decode(args: MbcsDecodeArgs, vm: &VirtualMachine) -> PyResult<(String, usize)> { + use windows_sys::Win32::Globalization::{ + CP_ACP, MB_ERR_INVALID_CHARS, MultiByteToWideChar, + }; + + let _errors = args.errors.as_ref().map(|s| s.as_str()).unwrap_or("strict"); + let data = args.data.borrow_buf(); + let len = data.len(); + + if data.is_empty() { + return Ok((String::new(), 0)); + } + + // Get the required buffer size for UTF-16 + let size = unsafe { + MultiByteToWideChar( + CP_ACP, + MB_ERR_INVALID_CHARS, + data.as_ptr().cast(), + len as i32, + core::ptr::null_mut(), + 0, + ) + }; + + if size == 0 { + // Try without MB_ERR_INVALID_CHARS for non-strict mode (replacement behavior) + let size = unsafe { + MultiByteToWideChar( + CP_ACP, + 0, + data.as_ptr().cast(), + len as i32, + core::ptr::null_mut(), + 0, + ) + }; + if size == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("mbcs_decode failed: {}", err))); + } + + let mut buffer = vec![0u16; size as usize]; + let result = unsafe { + MultiByteToWideChar( + CP_ACP, + 0, + data.as_ptr().cast(), + len as i32, + buffer.as_mut_ptr(), + size, + ) + }; + if result == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("mbcs_decode failed: {}", err))); + } + buffer.truncate(result as usize); + let s = String::from_utf16(&buffer) + .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {}", e)))?; + return Ok((s, len)); + } + + // Strict mode succeeded - no invalid characters + let mut buffer = vec![0u16; size as usize]; + let result = unsafe { + MultiByteToWideChar( + CP_ACP, + MB_ERR_INVALID_CHARS, + data.as_ptr().cast(), + len as i32, + buffer.as_mut_ptr(), + size, + ) + }; + if result == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("mbcs_decode failed: {}", err))); + } + buffer.truncate(result as usize); + let s = String::from_utf16(&buffer) + .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {}", e)))?; + + Ok((s, len)) + } + + #[derive(FromArgs)] + struct OemEncodeArgs { + #[pyarg(positional)] + s: PyStrRef, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + } + + #[pyfunction] + fn oem_encode(args: OemEncodeArgs, vm: &VirtualMachine) -> PyResult<(Vec<u8>, usize)> { + use crate::common::windows::ToWideString; + use windows_sys::Win32::Globalization::{ + CP_OEMCP, WC_NO_BEST_FIT_CHARS, WideCharToMultiByte, + }; + + let errors = args.errors.as_ref().map(|s| s.as_str()).unwrap_or("strict"); + let s = match args.s.to_str() { + Some(s) => s, + None => { + // String contains surrogates - not encodable with oem + return Err(vm.new_unicode_encode_error( + "'oem' codec can't encode character: surrogates not allowed", + )); + } + }; + let char_len = args.s.char_len(); + + if s.is_empty() { + return Ok((Vec::new(), char_len)); + } + + // Convert UTF-8 string to UTF-16 + let wide: Vec<u16> = std::ffi::OsStr::new(s).to_wide(); + + // Get the required buffer size + let size = unsafe { + WideCharToMultiByte( + CP_OEMCP, + WC_NO_BEST_FIT_CHARS, + wide.as_ptr(), + wide.len() as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + + if size == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("oem_encode failed: {}", err))); + } + + let mut buffer = vec![0u8; size as usize]; + let mut used_default_char: i32 = 0; + + let result = unsafe { + WideCharToMultiByte( + CP_OEMCP, + WC_NO_BEST_FIT_CHARS, + wide.as_ptr(), + wide.len() as i32, + buffer.as_mut_ptr().cast(), + size, + core::ptr::null(), + if errors == "strict" { + &mut used_default_char + } else { + core::ptr::null_mut() + }, + ) + }; + + if result == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("oem_encode failed: {err}"))); + } + + if errors == "strict" && used_default_char != 0 { + return Err(vm.new_unicode_encode_error( + "'oem' codec can't encode characters: invalid character", + )); + } + + buffer.truncate(result as usize); + Ok((buffer, char_len)) + } + + #[derive(FromArgs)] + struct OemDecodeArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + #[pyarg(positional, default = false)] + #[allow(dead_code)] + r#final: bool, + } + + #[pyfunction] + fn oem_decode(args: OemDecodeArgs, vm: &VirtualMachine) -> PyResult<(String, usize)> { + use windows_sys::Win32::Globalization::{ + CP_OEMCP, MB_ERR_INVALID_CHARS, MultiByteToWideChar, + }; + + let _errors = args.errors.as_ref().map(|s| s.as_str()).unwrap_or("strict"); + let data = args.data.borrow_buf(); + let len = data.len(); + + if data.is_empty() { + return Ok((String::new(), 0)); + } + + // Get the required buffer size for UTF-16 + let size = unsafe { + MultiByteToWideChar( + CP_OEMCP, + MB_ERR_INVALID_CHARS, + data.as_ptr().cast(), + len as i32, + core::ptr::null_mut(), + 0, + ) + }; + + if size == 0 { + // Try without MB_ERR_INVALID_CHARS for non-strict mode (replacement behavior) + let size = unsafe { + MultiByteToWideChar( + CP_OEMCP, + 0, + data.as_ptr().cast(), + len as i32, + core::ptr::null_mut(), + 0, + ) + }; + if size == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("oem_decode failed: {}", err))); + } + + let mut buffer = vec![0u16; size as usize]; + let result = unsafe { + MultiByteToWideChar( + CP_OEMCP, + 0, + data.as_ptr().cast(), + len as i32, + buffer.as_mut_ptr(), + size, + ) + }; + if result == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("oem_decode failed: {}", err))); + } + buffer.truncate(result as usize); + let s = String::from_utf16(&buffer) + .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {}", e)))?; + return Ok((s, len)); + } + + // Strict mode succeeded - no invalid characters + let mut buffer = vec![0u16; size as usize]; + let result = unsafe { + MultiByteToWideChar( + CP_OEMCP, + MB_ERR_INVALID_CHARS, + data.as_ptr().cast(), + len as i32, + buffer.as_mut_ptr(), + size, + ) + }; + if result == 0 { + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("oem_decode failed: {}", err))); + } + buffer.truncate(result as usize); + let s = String::from_utf16(&buffer) + .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {}", e)))?; + + Ok((s, len)) + } + + #[derive(FromArgs)] + struct CodePageEncodeArgs { + #[pyarg(positional)] + code_page: i32, + #[pyarg(positional)] + s: PyStrRef, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + } + + fn code_page_encoding_name(code_page: u32) -> String { + match code_page { + 0 => "mbcs".to_string(), + cp => format!("cp{cp}"), + } + } + + /// Get WideCharToMultiByte flags for encoding. + /// Matches encode_code_page_flags() in CPython. + fn encode_code_page_flags(code_page: u32, errors: &str) -> u32 { + use windows_sys::Win32::Globalization::{WC_ERR_INVALID_CHARS, WC_NO_BEST_FIT_CHARS}; + if code_page == 65001 { + // CP_UTF8 + WC_ERR_INVALID_CHARS + } else if code_page == 65000 { + // CP_UTF7 only supports flags=0 + 0 + } else if errors == "replace" { + 0 + } else { + WC_NO_BEST_FIT_CHARS + } + } + + /// Try to encode the entire wide string at once (fast/strict path). + /// Returns Ok(Some(bytes)) on success, Ok(None) if there are unencodable chars, + /// or Err on OS error. + fn try_encode_code_page_strict( + code_page: u32, + wide: &[u16], + vm: &VirtualMachine, + ) -> PyResult<Option<Vec<u8>>> { + use windows_sys::Win32::Globalization::WideCharToMultiByte; + + let flags = encode_code_page_flags(code_page, "strict"); + + let use_default_char = code_page != 65001 && code_page != 65000; + let mut used_default_char: i32 = 0; + let pused = if use_default_char { + &mut used_default_char as *mut i32 + } else { + core::ptr::null_mut() + }; + + let size = unsafe { + WideCharToMultiByte( + code_page, + flags, + wide.as_ptr(), + wide.len() as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + pused, + ) + }; + + if size <= 0 { + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + if err_code == 1113 { + // ERROR_NO_UNICODE_TRANSLATION + return Ok(None); + } + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("code_page_encode: {err}"))); + } + + if use_default_char && used_default_char != 0 { + return Ok(None); + } + + let mut buffer = vec![0u8; size as usize]; + used_default_char = 0; + let pused = if use_default_char { + &mut used_default_char as *mut i32 + } else { + core::ptr::null_mut() + }; + + let result = unsafe { + WideCharToMultiByte( + code_page, + flags, + wide.as_ptr(), + wide.len() as i32, + buffer.as_mut_ptr().cast(), + size, + core::ptr::null(), + pused, + ) + }; + + if result <= 0 { + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + if err_code == 1113 { + return Ok(None); + } + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("code_page_encode: {err}"))); + } + + if use_default_char && used_default_char != 0 { + return Ok(None); + } + + buffer.truncate(result as usize); + Ok(Some(buffer)) + } + + /// Encode character by character with error handling. + fn encode_code_page_errors( + code_page: u32, + s: &PyStrRef, + errors: &str, + encoding_name: &str, + vm: &VirtualMachine, + ) -> PyResult<(Vec<u8>, usize)> { + use crate::builtins::{PyBytes, PyStr, PyTuple}; + use windows_sys::Win32::Globalization::WideCharToMultiByte; + + let char_len = s.char_len(); + let flags = encode_code_page_flags(code_page, errors); + let use_default_char = code_page != 65001 && code_page != 65000; + let encoding_str = vm.ctx.new_str(encoding_name); + let reason_str = vm.ctx.new_str("invalid character"); + + // For strict mode, find the first unencodable character and raise + if errors == "strict" { + // Find the failing position by trying each character + let mut fail_pos = 0; + for cp in s.as_wtf8().code_points() { + let ch = cp.to_u32(); + if (0xD800..=0xDFFF).contains(&ch) { + break; + } + let mut wchars = [0u16; 2]; + let wchar_len = if ch < 0x10000 { + wchars[0] = ch as u16; + 1 + } else { + wchars[0] = ((ch - 0x10000) >> 10) as u16 + 0xD800; + wchars[1] = ((ch - 0x10000) & 0x3FF) as u16 + 0xDC00; + 2 + }; + let mut used_default_char: i32 = 0; + let pused = if use_default_char { + &mut used_default_char as *mut i32 + } else { + core::ptr::null_mut() + }; + let outsize = unsafe { + WideCharToMultiByte( + code_page, + flags, + wchars.as_ptr(), + wchar_len, + core::ptr::null_mut(), + 0, + core::ptr::null(), + pused, + ) + }; + if outsize <= 0 || (use_default_char && used_default_char != 0) { + break; + } + fail_pos += 1; + } + return Err(vm.new_unicode_encode_error_real( + encoding_str, + s.clone(), + fail_pos, + fail_pos + 1, + reason_str, + )); + } + + let error_handler = vm.state.codec_registry.lookup_error(errors, vm)?; + let mut output = Vec::new(); + + // Collect code points for random access + let code_points: Vec<u32> = s.as_wtf8().code_points().map(|cp| cp.to_u32()).collect(); + + let mut pos = 0usize; + while pos < code_points.len() { + let ch = code_points[pos]; + + // Convert code point to UTF-16 + let mut wchars = [0u16; 2]; + let wchar_len; + let is_surrogate = (0xD800..=0xDFFF).contains(&ch); + + if is_surrogate { + wchar_len = 0; // Can't encode surrogates normally + } else if ch < 0x10000 { + wchars[0] = ch as u16; + wchar_len = 1; + } else { + wchars[0] = ((ch - 0x10000) >> 10) as u16 + 0xD800; + wchars[1] = ((ch - 0x10000) & 0x3FF) as u16 + 0xDC00; + wchar_len = 2; + } + + if !is_surrogate { + let mut used_default_char: i32 = 0; + let pused = if use_default_char { + &mut used_default_char as *mut i32 + } else { + core::ptr::null_mut() + }; + + let mut buf = [0u8; 8]; + let outsize = unsafe { + WideCharToMultiByte( + code_page, + flags, + wchars.as_ptr(), + wchar_len, + buf.as_mut_ptr().cast(), + buf.len() as i32, + core::ptr::null(), + pused, + ) + }; + + if outsize > 0 && (!use_default_char || used_default_char == 0) { + output.extend_from_slice(&buf[..outsize as usize]); + pos += 1; + continue; + } + } + + // Character can't be encoded - call error handler + let exc = vm.new_unicode_encode_error_real( + encoding_str.clone(), + s.clone(), + pos, + pos + 1, + reason_str.clone(), + ); + + let res = error_handler.call((exc,), vm)?; + let tuple_err = + || vm.new_type_error("encoding error handler must return (str/bytes, int) tuple"); + let tuple: &PyTuple = res.downcast_ref().ok_or_else(&tuple_err)?; + let tuple_slice = tuple.as_slice(); + if tuple_slice.len() != 2 { + return Err(tuple_err()); + } + + let replacement = &tuple_slice[0]; + let new_pos_obj = tuple_slice[1].clone(); + + if let Some(bytes) = replacement.downcast_ref::<PyBytes>() { + output.extend_from_slice(bytes); + } else if let Some(rep_str) = replacement.downcast_ref::<PyStr>() { + // Replacement string - try to encode each character + for rcp in rep_str.as_wtf8().code_points() { + let rch = rcp.to_u32(); + if rch > 127 { + return Err(vm.new_unicode_encode_error_real( + encoding_str.clone(), + s.clone(), + pos, + pos + 1, + vm.ctx + .new_str("unable to encode error handler result to ASCII"), + )); + } + output.push(rch as u8); + } + } else { + return Err(tuple_err()); + } + + let new_pos: isize = new_pos_obj.try_into_value(vm).map_err(|_| tuple_err())?; + pos = if new_pos < 0 { + (code_points.len() as isize + new_pos).max(0) as usize + } else { + new_pos as usize + }; + } + + Ok((output, char_len)) + } + + #[pyfunction] + fn code_page_encode( + args: CodePageEncodeArgs, + vm: &VirtualMachine, + ) -> PyResult<(Vec<u8>, usize)> { + use crate::common::windows::ToWideString; + + if args.code_page < 0 { + return Err(vm.new_value_error("invalid code page number")); + } + let errors = args.errors.as_ref().map(|s| s.as_str()).unwrap_or("strict"); + let code_page = args.code_page as u32; + let char_len = args.s.char_len(); + + if char_len == 0 { + return Ok((Vec::new(), 0)); + } + + let encoding_name = code_page_encoding_name(code_page); + + // Fast path: try encoding the whole string at once (only if no surrogates) + if let Some(str_data) = args.s.to_str() { + let wide: Vec<u16> = std::ffi::OsStr::new(str_data).to_wide(); + if let Some(result) = try_encode_code_page_strict(code_page, &wide, vm)? { + return Ok((result, char_len)); + } + } + + // Slow path: character by character with error handling + encode_code_page_errors(code_page, &args.s, errors, &encoding_name, vm) + } + + #[derive(FromArgs)] + struct CodePageDecodeArgs { + #[pyarg(positional)] + code_page: i32, + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(positional, optional)] + errors: Option<PyUtf8StrRef>, + #[pyarg(positional, default = false)] + r#final: bool, + } + + /// Try to decode the entire buffer with strict flags (fast path). + /// Returns Ok(Some(wide_chars)) on success, Ok(None) on decode error, + /// or Err on OS error. + fn try_decode_code_page_strict( + code_page: u32, + data: &[u8], + vm: &VirtualMachine, + ) -> PyResult<Option<Vec<u16>>> { + use windows_sys::Win32::Globalization::{MB_ERR_INVALID_CHARS, MultiByteToWideChar}; + + let mut flags = MB_ERR_INVALID_CHARS; + + loop { + let size = unsafe { + MultiByteToWideChar( + code_page, + flags, + data.as_ptr().cast(), + data.len() as i32, + core::ptr::null_mut(), + 0, + ) + }; + if size > 0 { + let mut buffer = vec![0u16; size as usize]; + let result = unsafe { + MultiByteToWideChar( + code_page, + flags, + data.as_ptr().cast(), + data.len() as i32, + buffer.as_mut_ptr(), + size, + ) + }; + if result > 0 { + buffer.truncate(result as usize); + return Ok(Some(buffer)); + } + } + + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + // ERROR_INVALID_FLAGS = 1004 + if flags != 0 && err_code == 1004 { + flags = 0; + continue; + } + // ERROR_NO_UNICODE_TRANSLATION = 1113 + if err_code == 1113 { + return Ok(None); + } + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("code_page_decode: {err}"))); + } + } + + /// Decode byte by byte with error handling (slow path). + fn decode_code_page_errors( + code_page: u32, + data: &[u8], + errors: &str, + is_final: bool, + encoding_name: &str, + vm: &VirtualMachine, + ) -> PyResult<(PyStrRef, usize)> { + use crate::builtins::PyTuple; + use crate::common::wtf8::Wtf8Buf; + use windows_sys::Win32::Globalization::{MB_ERR_INVALID_CHARS, MultiByteToWideChar}; + + let len = data.len(); + let encoding_str = vm.ctx.new_str(encoding_name); + let reason_str = vm + .ctx + .new_str("No mapping for the Unicode character exists in the target code page."); + + // For strict+final, find the failing position and raise + if errors == "strict" && is_final { + // Find the exact failing byte position by trying byte by byte + let mut fail_pos = 0; + let mut flags_s: u32 = MB_ERR_INVALID_CHARS; + let mut buf = [0u16; 2]; + while fail_pos < len { + let mut in_size = 1; + let mut found = false; + while in_size <= 4 && fail_pos + in_size <= len { + let outsize = unsafe { + MultiByteToWideChar( + code_page, + flags_s, + data[fail_pos..].as_ptr().cast(), + in_size as i32, + buf.as_mut_ptr(), + 2, + ) + }; + if outsize > 0 { + fail_pos += in_size; + found = true; + break; + } + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + if err_code == 1004 && flags_s != 0 { + flags_s = 0; + continue; + } + in_size += 1; + } + if !found { + break; + } + } + let object = vm.ctx.new_bytes(data.to_vec()); + return Err(vm.new_unicode_decode_error_real( + encoding_str, + object, + fail_pos, + fail_pos + 1, + reason_str, + )); + } + + let error_handler = if errors != "strict" + && errors != "ignore" + && errors != "replace" + && errors != "backslashreplace" + && errors != "surrogateescape" + { + Some(vm.state.codec_registry.lookup_error(errors, vm)?) + } else { + None + }; + + let mut wide_buf: Vec<u16> = Vec::new(); + let mut pos = 0usize; + let mut flags: u32 = MB_ERR_INVALID_CHARS; + + while pos < len { + // Try to decode with increasing byte counts (1, 2, 3, 4) + let mut in_size = 1; + let mut outsize; + let mut buffer = [0u16; 2]; + + loop { + outsize = unsafe { + MultiByteToWideChar( + code_page, + flags, + data[pos..].as_ptr().cast(), + in_size as i32, + buffer.as_mut_ptr(), + 2, + ) + }; + if outsize > 0 { + break; + } + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + if err_code == 1004 && flags != 0 { + // ERROR_INVALID_FLAGS - retry with flags=0 + flags = 0; + continue; + } + if err_code != 1113 && err_code != 122 { + // Not ERROR_NO_UNICODE_TRANSLATION and not ERROR_INSUFFICIENT_BUFFER + let err = std::io::Error::last_os_error(); + return Err(vm.new_os_error(format!("code_page_decode: {err}"))); + } + in_size += 1; + if in_size > 4 || pos + in_size > len { + break; + } + } + + if outsize <= 0 { + // Can't decode this byte sequence + if pos + in_size >= len && !is_final { + // Incomplete sequence at end, not final - stop here + break; + } + + // Handle the error based on error mode + match errors { + "ignore" => { + pos += 1; + } + "replace" => { + wide_buf.push(0xFFFD); + pos += 1; + } + "backslashreplace" => { + let byte = data[pos]; + for ch in format!("\\x{byte:02x}").encode_utf16() { + wide_buf.push(ch); + } + pos += 1; + } + "surrogateescape" => { + let byte = data[pos]; + wide_buf.push(0xDC00 + byte as u16); + pos += 1; + } + "strict" => { + let object = vm.ctx.new_bytes(data.to_vec()); + return Err(vm.new_unicode_decode_error_real( + encoding_str, + object, + pos, + pos + 1, + reason_str, + )); + } + _ => { + // Custom error handler + let object = vm.ctx.new_bytes(data.to_vec()); + let exc = vm.new_unicode_decode_error_real( + encoding_str.clone(), + object, + pos, + pos + 1, + reason_str.clone(), + ); + let handler = error_handler.as_ref().unwrap(); + let res = handler.call((exc,), vm)?; + let tuple_err = || { + vm.new_type_error("decoding error handler must return (str, int) tuple") + }; + let tuple: &PyTuple = res.downcast_ref().ok_or_else(&tuple_err)?; + let tuple_slice = tuple.as_slice(); + if tuple_slice.len() != 2 { + return Err(tuple_err()); + } + + let replacement: PyStrRef = tuple_slice[0] + .clone() + .try_into_value(vm) + .map_err(|_| tuple_err())?; + let new_pos: isize = tuple_slice[1] + .clone() + .try_into_value(vm) + .map_err(|_| tuple_err())?; + + for cp in replacement.as_wtf8().code_points() { + let u = cp.to_u32(); + if u < 0x10000 { + wide_buf.push(u as u16); + } else { + wide_buf.push(((u - 0x10000) >> 10) as u16 + 0xD800); + wide_buf.push(((u - 0x10000) & 0x3FF) as u16 + 0xDC00); + } + } + + pos = if new_pos < 0 { + (len as isize + new_pos).max(0) as usize + } else { + new_pos as usize + }; + } + } + } else { + // Successfully decoded + wide_buf.extend_from_slice(&buffer[..outsize as usize]); + pos += in_size; + } + } + + let s = Wtf8Buf::from_wide(&wide_buf); + Ok((vm.ctx.new_str(s), pos)) + } + + #[pyfunction] + fn code_page_decode( + args: CodePageDecodeArgs, + vm: &VirtualMachine, + ) -> PyResult<(PyStrRef, usize)> { + use crate::common::wtf8::Wtf8Buf; + + if args.code_page < 0 { + return Err(vm.new_value_error("invalid code page number")); + } + let errors = args.errors.as_ref().map(|s| s.as_str()).unwrap_or("strict"); + let code_page = args.code_page as u32; + let data = args.data.borrow_buf(); + let is_final = args.r#final; + + if data.is_empty() { + return Ok((vm.ctx.empty_str.to_owned(), 0)); + } + + let encoding_name = code_page_encoding_name(code_page); + + // Fast path: try to decode the whole buffer with strict flags + match try_decode_code_page_strict(code_page, &data, vm)? { + Some(wide) => { + let s = Wtf8Buf::from_wide(&wide); + return Ok((vm.ctx.new_str(s), data.len())); + } + None => { + // Decode error - fall through to slow path + } + } + + // Slow path: byte by byte with error handling + decode_code_page_errors(code_page, &data, errors, is_final, &encoding_name, vm) + } +} diff --git a/crates/vm/src/stdlib/_collections.rs b/crates/vm/src/stdlib/_collections.rs new file mode 100644 index 00000000000..2807e171777 --- /dev/null +++ b/crates/vm/src/stdlib/_collections.rs @@ -0,0 +1,739 @@ +pub(crate) use _collections::module_def; + +#[pymodule] +mod _collections { + use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, + builtins::{ + IterStatus::{Active, Exhausted}, + PositionIterInternal, PyGenericAlias, PyInt, PyStr, PyType, PyTypeRef, + }, + common::lock::{PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}, + function::{KwArgs, OptionalArg, PyComparisonValue}, + iter::PyExactSizeIterator, + protocol::{PyIterReturn, PyNumberMethods, PySequenceMethods}, + recursion::ReprGuard, + sequence::{MutObjectSequenceOp, OptionalRangeArgs}, + sliceable::SequenceIndexOp, + types::{ + AsNumber, AsSequence, Comparable, Constructor, DefaultConstructor, Initializer, + IterNext, Iterable, PyComparisonOp, Representable, SelfIter, + }, + utils::collection_repr, + }; + use alloc::collections::VecDeque; + use core::cmp::max; + use crossbeam_utils::atomic::AtomicCell; + + #[pyattr] + #[pyclass(module = "collections", name = "deque", unhashable = true)] + #[derive(Debug, Default, PyPayload)] + struct PyDeque { + deque: PyRwLock<VecDeque<PyObjectRef>>, + maxlen: Option<usize>, + state: AtomicCell<usize>, // incremented whenever the indices move + } + + type PyDequeRef = PyRef<PyDeque>; + + #[derive(FromArgs)] + struct PyDequeOptions { + #[pyarg(any, optional)] + iterable: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + maxlen: OptionalArg<PyObjectRef>, + } + + impl PyDeque { + fn borrow_deque(&self) -> PyRwLockReadGuard<'_, VecDeque<PyObjectRef>> { + self.deque.read() + } + + fn borrow_deque_mut(&self) -> PyRwLockWriteGuard<'_, VecDeque<PyObjectRef>> { + self.deque.write() + } + } + + #[pyclass( + flags(BASETYPE, HAS_WEAKREF), + with( + Constructor, + Initializer, + AsNumber, + AsSequence, + Comparable, + Iterable, + Representable + ) + )] + impl PyDeque { + #[pymethod] + fn append(&self, obj: PyObjectRef) { + self.state.fetch_add(1); + let mut deque = self.borrow_deque_mut(); + if self.maxlen == Some(deque.len()) { + deque.pop_front(); + } + deque.push_back(obj); + } + + #[pymethod] + fn appendleft(&self, obj: PyObjectRef) { + self.state.fetch_add(1); + let mut deque = self.borrow_deque_mut(); + if self.maxlen == Some(deque.len()) { + deque.pop_back(); + } + deque.push_front(obj); + } + + #[pymethod] + fn clear(&self) { + self.state.fetch_add(1); + self.borrow_deque_mut().clear() + } + + #[pymethod(name = "__copy__")] + #[pymethod] + fn copy(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + Self { + deque: PyRwLock::new(zelf.borrow_deque().clone()), + maxlen: zelf.maxlen, + state: AtomicCell::new(zelf.state.load()), + } + .into_ref_with_type(vm, zelf.class().to_owned()) + } + + #[pymethod] + fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + let start_state = self.state.load(); + let count = self.mut_count(vm, &obj)?; + + if start_state != self.state.load() { + return Err(vm.new_runtime_error("deque mutated during iteration")); + } + Ok(count) + } + + #[pymethod] + fn extend(&self, iter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self._extend(&iter, vm) + } + + fn _extend(&self, iter: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + self.state.fetch_add(1); + let max_len = self.maxlen; + let mut elements: Vec<PyObjectRef> = iter.try_to_value(vm)?; + if let Some(max_len) = max_len { + if max_len > elements.len() { + let mut deque = self.borrow_deque_mut(); + let drain_until = deque.len().saturating_sub(max_len - elements.len()); + deque.drain(..drain_until); + } else { + self.borrow_deque_mut().clear(); + elements.drain(..(elements.len() - max_len)); + } + } + self.borrow_deque_mut().extend(elements); + Ok(()) + } + + #[pymethod] + fn extendleft(&self, iter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let max_len = self.maxlen; + let mut elements: Vec<PyObjectRef> = iter.try_to_value(vm)?; + elements.reverse(); + + if let Some(max_len) = max_len { + if max_len > elements.len() { + let mut deque = self.borrow_deque_mut(); + let truncate_until = max_len - elements.len(); + deque.truncate(truncate_until); + } else { + self.borrow_deque_mut().clear(); + elements.truncate(max_len); + } + } + let mut created = VecDeque::from(elements); + let mut borrowed = self.borrow_deque_mut(); + created.append(&mut borrowed); + core::mem::swap(&mut created, &mut borrowed); + Ok(()) + } + + #[pymethod] + fn index( + &self, + needle: PyObjectRef, + range: OptionalRangeArgs, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let start_state = self.state.load(); + + let (start, stop) = range.saturate(self.__len__(), vm)?; + let index = self.mut_index_range(vm, &needle, start..stop)?; + if start_state != self.state.load() { + Err(vm.new_runtime_error("deque mutated during iteration")) + } else if let Some(index) = index.into() { + Ok(index) + } else { + Err(vm.new_value_error( + needle + .repr(vm) + .map(|repr| format!("{repr} is not in deque")) + .unwrap_or_else(|_| String::new()), + )) + } + } + + #[pymethod] + fn insert(&self, idx: i32, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.state.fetch_add(1); + let mut deque = self.borrow_deque_mut(); + + if self.maxlen == Some(deque.len()) { + return Err(vm.new_index_error("deque already at its maximum size")); + } + + let idx = if idx < 0 { + if -idx as usize > deque.len() { + 0 + } else { + deque.len() - ((-idx) as usize) + } + } else if idx as usize > deque.len() { + deque.len() + } else { + idx as usize + }; + + deque.insert(idx, obj); + + Ok(()) + } + + #[pymethod] + fn pop(&self, vm: &VirtualMachine) -> PyResult { + self.state.fetch_add(1); + self.borrow_deque_mut() + .pop_back() + .ok_or_else(|| vm.new_index_error("pop from an empty deque")) + } + + #[pymethod] + fn popleft(&self, vm: &VirtualMachine) -> PyResult { + self.state.fetch_add(1); + self.borrow_deque_mut() + .pop_front() + .ok_or_else(|| vm.new_index_error("pop from an empty deque")) + } + + #[pymethod] + fn remove(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let start_state = self.state.load(); + let index = self.mut_index(vm, &obj)?; + + if start_state != self.state.load() { + Err(vm.new_index_error("deque mutated during remove().")) + } else if let Some(index) = index.into() { + let mut deque = self.borrow_deque_mut(); + self.state.fetch_add(1); + Ok(deque.remove(index).unwrap()) + } else { + Err(vm.new_value_error("deque.remove(x): x not in deque")) + } + } + + #[pymethod] + fn reverse(&self) { + let rev: VecDeque<_> = self.borrow_deque().iter().cloned().rev().collect(); + *self.borrow_deque_mut() = rev; + } + + #[pymethod] + fn __reversed__(zelf: PyRef<Self>) -> PyResult<PyReverseDequeIterator> { + Ok(PyReverseDequeIterator { + state: zelf.state.load(), + internal: PyMutex::new(PositionIterInternal::new(zelf, 0)), + }) + } + + #[pymethod] + fn rotate(&self, mid: OptionalArg<isize>) { + self.state.fetch_add(1); + let mut deque = self.borrow_deque_mut(); + if !deque.is_empty() { + let mid = mid.unwrap_or(1) % deque.len() as isize; + if mid.is_negative() { + deque.rotate_left(-mid as usize); + } else { + deque.rotate_right(mid as usize); + } + } + } + + #[pygetset] + const fn maxlen(&self) -> Option<usize> { + self.maxlen + } + + fn __getitem__(&self, idx: isize, vm: &VirtualMachine) -> PyResult { + let deque = self.borrow_deque(); + idx.wrapped_at(deque.len()) + .and_then(|i| deque.get(i).cloned()) + .ok_or_else(|| vm.new_index_error("deque index out of range")) + } + + fn __setitem__(&self, idx: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mut deque = self.borrow_deque_mut(); + idx.wrapped_at(deque.len()) + .and_then(|i| deque.get_mut(i)) + .map(|x| *x = value) + .ok_or_else(|| vm.new_index_error("deque index out of range")) + } + + fn __delitem__(&self, idx: isize, vm: &VirtualMachine) -> PyResult<()> { + let mut deque = self.borrow_deque_mut(); + idx.wrapped_at(deque.len()) + .and_then(|i| deque.remove(i).map(drop)) + .ok_or_else(|| vm.new_index_error("deque index out of range")) + } + + fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + self._contains(&needle, vm) + } + + fn _contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + let start_state = self.state.load(); + let ret = self.mut_contains(vm, needle)?; + if start_state != self.state.load() { + Err(vm.new_runtime_error("deque mutated during iteration")) + } else { + Ok(ret) + } + } + + fn _mul(&self, n: isize, vm: &VirtualMachine) -> PyResult<VecDeque<PyObjectRef>> { + let deque = self.borrow_deque(); + let n = vm.check_repeat_or_overflow_error(deque.len(), n)?; + let mul_len = n * deque.len(); + let iter = deque.iter().cycle().take(mul_len); + let skipped = self + .maxlen + .and_then(|maxlen| mul_len.checked_sub(maxlen)) + .unwrap_or(0); + + let deque = iter.skip(skipped).cloned().collect(); + Ok(deque) + } + + fn __mul__(&self, n: isize, vm: &VirtualMachine) -> PyResult<Self> { + let deque = self._mul(n, vm)?; + Ok(Self { + deque: PyRwLock::new(deque), + maxlen: self.maxlen, + state: AtomicCell::new(0), + }) + } + + fn __imul__(zelf: PyRef<Self>, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + let mul_deque = zelf._mul(n, vm)?; + *zelf.borrow_deque_mut() = mul_deque; + Ok(zelf) + } + + fn __len__(&self) -> usize { + self.borrow_deque().len() + } + + fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<Self> { + if let Some(o) = other.downcast_ref::<Self>() { + let mut deque = self.borrow_deque().clone(); + let elements = o.borrow_deque().clone(); + deque.extend(elements); + + let skipped = self + .maxlen + .and_then(|maxlen| deque.len().checked_sub(maxlen)) + .unwrap_or(0); + deque.drain(..skipped); + + Ok(Self { + deque: PyRwLock::new(deque), + maxlen: self.maxlen, + state: AtomicCell::new(0), + }) + } else { + Err(vm.new_type_error(format!( + "can only concatenate deque (not \"{}\") to deque", + other.class().name() + ))) + } + } + + fn __iadd__( + zelf: PyRef<Self>, + other: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + zelf.extend(other, vm)?; + Ok(zelf) + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let cls = zelf.class().to_owned(); + let value = match zelf.maxlen { + Some(v) => vm.new_pyobj((vm.ctx.empty_tuple.clone(), v)), + None => vm.ctx.empty_tuple.clone().into(), + }; + Ok(vm.new_pyobj((cls, value, vm.ctx.none(), PyDequeIterator::new(zelf)))) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl MutObjectSequenceOp for PyDeque { + type Inner = VecDeque<PyObjectRef>; + + fn do_get(index: usize, inner: &Self::Inner) -> Option<&PyObject> { + inner.get(index).map(|r| r.as_ref()) + } + + fn do_lock(&self) -> impl core::ops::Deref<Target = Self::Inner> { + self.borrow_deque() + } + } + + impl DefaultConstructor for PyDeque {} + + impl Initializer for PyDeque { + type Args = PyDequeOptions; + + fn init( + zelf: PyRef<Self>, + PyDequeOptions { iterable, maxlen }: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<()> { + // TODO: This is _basically_ pyobject_to_opt_usize in itertools.rs + // need to move that function elsewhere and refactor usages. + let maxlen = if let Some(obj) = maxlen.into_option() { + if !vm.is_none(&obj) { + let maxlen: isize = obj + .downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error("an integer is required."))? + .try_to_primitive(vm)?; + + if maxlen.is_negative() { + return Err(vm.new_value_error("maxlen must be non-negative.")); + } + Some(maxlen as usize) + } else { + None + } + } else { + None + }; + + // retrieve elements first to not to make too huge lock + let elements = iterable + .into_option() + .map(|iter| { + let mut elements: Vec<PyObjectRef> = iter.try_to_value(vm)?; + if let Some(maxlen) = maxlen { + elements.drain(..elements.len().saturating_sub(maxlen)); + } + Ok(elements) + }) + .transpose()?; + + // SAFETY: This is hacky part for read-only field + // Because `maxlen` is only mutated from __init__. We can abuse the lock of deque to ensure this is locked enough. + // If we make a single lock of deque not only for extend but also for setting maxlen, it will be safe. + { + let mut deque = zelf.borrow_deque_mut(); + // Clear any previous data present. + deque.clear(); + unsafe { + // `maxlen` is better to be defined as UnsafeCell in common practice, + // but then more type works without any safety benefits + let unsafe_maxlen = + &zelf.maxlen as *const _ as *const core::cell::UnsafeCell<Option<usize>>; + *(*unsafe_maxlen).get() = maxlen; + } + if let Some(elements) = elements { + deque.extend(elements); + } + } + + Ok(()) + } + } + + impl AsNumber for PyDeque { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyDeque>().unwrap(); + Ok(!zelf.borrow_deque().is_empty()) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + + impl AsSequence for PyDeque { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyDeque::sequence_downcast(seq).__len__())), + concat: atomic_func!(|seq, other, vm| { + PyDeque::sequence_downcast(seq) + .concat(other, vm) + .map(|x| x.into_ref(&vm.ctx).into()) + }), + repeat: atomic_func!(|seq, n, vm| { + PyDeque::sequence_downcast(seq) + .__mul__(n, vm) + .map(|x| x.into_ref(&vm.ctx).into()) + }), + item: atomic_func!(|seq, i, vm| PyDeque::sequence_downcast(seq).__getitem__(i, vm)), + ass_item: atomic_func!(|seq, i, value, vm| { + let zelf = PyDeque::sequence_downcast(seq); + if let Some(value) = value { + zelf.__setitem__(i, value, vm) + } else { + zelf.__delitem__(i, vm) + } + }), + contains: atomic_func!( + |seq, needle, vm| PyDeque::sequence_downcast(seq)._contains(needle, vm) + ), + inplace_concat: atomic_func!(|seq, other, vm| { + let zelf = PyDeque::sequence_downcast(seq); + zelf._extend(other, vm)?; + Ok(zelf.to_owned().into()) + }), + inplace_repeat: atomic_func!(|seq, n, vm| { + let zelf = PyDeque::sequence_downcast(seq); + PyDeque::__imul__(zelf.to_owned(), n, vm).map(|x| x.into()) + }), + }; + + &AS_SEQUENCE + } + } + + impl Comparable for PyDeque { + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + if let Some(res) = op.identical_optimization(zelf, other) { + return Ok(res.into()); + } + let other = class_or_notimplemented!(Self, other); + let lhs = zelf.borrow_deque(); + let rhs = other.borrow_deque(); + lhs.iter() + .richcompare(rhs.iter(), op, vm) + .map(PyComparisonValue::Implemented) + } + } + + impl Iterable for PyDeque { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + Ok(PyDequeIterator::new(zelf).into_pyobject(vm)) + } + } + + impl Representable for PyDeque { + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let deque = zelf.borrow_deque().clone(); + let class = zelf.class(); + let class_name = class.name(); + let closing_part = zelf + .maxlen + .map(|maxlen| format!("], maxlen={maxlen}")) + .unwrap_or_else(|| "]".to_owned()); + + if zelf.__len__() == 0 { + return Ok(vm.ctx.new_str(format!("{class_name}([{closing_part})"))); + } + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + Ok(vm.ctx.new_str(collection_repr( + Some(&class_name), + "[", + &closing_part, + deque.iter(), + vm, + )?)) + } else { + Ok(vm.ctx.intern_str("[...]").to_owned()) + } + } + + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("repr() is overridden directly") + } + } + + #[pyattr] + #[pyclass(name = "_deque_iterator")] + #[derive(Debug, PyPayload)] + struct PyDequeIterator { + state: usize, + internal: PyMutex<PositionIterInternal<PyDequeRef>>, + } + + #[derive(FromArgs)] + struct DequeIterArgs { + #[pyarg(positional)] + deque: PyDequeRef, + + #[pyarg(positional, optional)] + index: OptionalArg<isize>, + } + + impl Constructor for PyDequeIterator { + type Args = (DequeIterArgs, KwArgs); + + fn py_new( + _cls: &Py<PyType>, + (DequeIterArgs { deque, index }, _kwargs): Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + let iter = Self::new(deque); + if let OptionalArg::Present(index) = index { + let index = max(index, 0) as usize; + iter.internal.lock().position = index; + } + Ok(iter) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyDequeIterator { + pub(crate) fn new(deque: PyDequeRef) -> Self { + Self { + state: deque.state.load(), + internal: PyMutex::new(PositionIterInternal::new(deque, 0)), + } + } + + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().length_hint(|obj| obj.__len__()) + } + + #[pymethod] + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> (PyTypeRef, (PyDequeRef, PyObjectRef)) { + let internal = zelf.internal.lock(); + let deque = match &internal.status { + Active(obj) => obj.clone(), + Exhausted => PyDeque::default().into_ref(&vm.ctx), + }; + ( + zelf.class().to_owned(), + (deque, vm.ctx.new_int(internal.position).into()), + ) + } + } + + impl SelfIter for PyDequeIterator {} + impl IterNext for PyDequeIterator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.internal.lock().next(|deque, pos| { + if zelf.state != deque.state.load() { + return Err(vm.new_runtime_error("Deque mutated during iteration")); + } + let deque = deque.borrow_deque(); + Ok(PyIterReturn::from_result( + deque.get(pos).cloned().ok_or(None), + )) + }) + } + } + + #[pyattr] + #[pyclass(name = "_deque_reverse_iterator")] + #[derive(Debug, PyPayload)] + struct PyReverseDequeIterator { + state: usize, + // position is counting from the tail + internal: PyMutex<PositionIterInternal<PyDequeRef>>, + } + + impl Constructor for PyReverseDequeIterator { + type Args = (DequeIterArgs, KwArgs); + + fn py_new( + _cls: &Py<PyType>, + (DequeIterArgs { deque, index }, _kwargs): Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + let iter = PyDeque::__reversed__(deque)?; + if let OptionalArg::Present(index) = index { + let index = max(index, 0) as usize; + iter.internal.lock().position = index; + } + Ok(iter) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyReverseDequeIterator { + #[pymethod] + fn __length_hint__(&self) -> usize { + self.internal.lock().length_hint(|obj| obj.__len__()) + } + + #[pymethod] + fn __reduce__( + zelf: PyRef<Self>, + vm: &VirtualMachine, + ) -> PyResult<(PyTypeRef, (PyDequeRef, PyObjectRef))> { + let internal = zelf.internal.lock(); + let deque = match &internal.status { + Active(obj) => obj.clone(), + Exhausted => PyDeque::default().into_ref(&vm.ctx), + }; + Ok(( + zelf.class().to_owned(), + (deque, vm.ctx.new_int(internal.position).into()), + )) + } + } + + impl SelfIter for PyReverseDequeIterator {} + impl IterNext for PyReverseDequeIterator { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + zelf.internal.lock().next(|deque, pos| { + if deque.state.load() != zelf.state { + return Err(vm.new_runtime_error("Deque mutated during iteration")); + } + let deque = deque.borrow_deque(); + let r = deque + .len() + .checked_sub(pos + 1) + .and_then(|pos| deque.get(pos)) + .cloned(); + Ok(PyIterReturn::from_result(r.ok_or(None))) + }) + } + } +} diff --git a/crates/vm/src/stdlib/_ctypes.rs b/crates/vm/src/stdlib/_ctypes.rs new file mode 100644 index 00000000000..2534f6128e8 --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes.rs @@ -0,0 +1,1327 @@ +// spell-checker:disable + +mod array; +mod base; +mod function; +mod library; +mod pointer; +mod simple; +mod structure; +mod union; + +use crate::{ + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyStr, PyType}, + class::PyClassImpl, + types::TypeDataRef, +}; +use core::ffi::{ + c_double, c_float, c_int, c_long, c_longlong, c_schar, c_short, c_uchar, c_uint, c_ulong, + c_ulonglong, c_ushort, +}; +use core::mem; +use widestring::WideChar; + +pub use array::PyCArray; +pub use base::{FfiArgValue, PyCData, PyCField, StgInfo, StgInfoFlags}; +pub use pointer::PyCPointer; +pub use simple::{PyCSimple, PyCSimpleType}; +pub use structure::PyCStructure; +pub use union::PyCUnion; + +/// Extension for PyType to get StgInfo +/// PyStgInfo_FromType +impl Py<PyType> { + /// Get StgInfo from a ctypes type object + /// + /// Returns a TypeDataRef to StgInfo if the type has one and is initialized, error otherwise. + /// Abstract classes (whose metaclass __init__ was not called) will have uninitialized StgInfo. + fn stg_info<'a>(&'a self, vm: &VirtualMachine) -> PyResult<TypeDataRef<'a, StgInfo>> { + self.stg_info_opt() + .ok_or_else(|| vm.new_type_error("abstract class")) + } + + /// Get StgInfo if initialized, None otherwise. + fn stg_info_opt(&self) -> Option<TypeDataRef<'_, StgInfo>> { + self.get_type_data::<StgInfo>() + .filter(|info| info.initialized) + } + + /// Get _type_ attribute as String (type code like "i", "d", etc.) + fn type_code(&self, vm: &VirtualMachine) -> Option<String> { + self.as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t: PyObjectRef| t.downcast_ref::<PyStr>().map(|s| s.to_string())) + } + + /// Mark all base classes as finalized + fn mark_bases_final(&self) { + for base in self.bases.read().iter() { + if let Some(mut stg) = base.get_type_data_mut::<StgInfo>() { + stg.flags |= StgInfoFlags::DICTFLAG_FINAL; + } else { + let mut stg = StgInfo::default(); + stg.flags |= StgInfoFlags::DICTFLAG_FINAL; + let _ = base.init_type_data(stg); + } + } + } +} + +impl PyType { + /// Check if StgInfo is already initialized. + /// Raises SystemError if already initialized. + pub(crate) fn check_not_initialized(&self, vm: &VirtualMachine) -> PyResult<()> { + if let Some(stg_info) = self.get_type_data::<StgInfo>() + && stg_info.initialized + { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.system_error.to_owned(), + format!("class \"{}\" already initialized", self.name()).into(), + )); + } + Ok(()) + } + + /// Check if StgInfo is already initialized, returning true if so. + /// Unlike check_not_initialized, does not raise an error. + pub(crate) fn is_initialized(&self) -> bool { + self.get_type_data::<StgInfo>() + .is_some_and(|stg_info| stg_info.initialized) + } +} + +// Dynamic type check helpers for PyCData +pub(crate) use _ctypes::module_def; + +// These check if an object's type's metaclass is a subclass of a specific metaclass + +/// Size of long double - platform dependent +/// x86_64 macOS/Linux: 16 bytes (80-bit extended + padding) +/// ARM64: 16 bytes (128-bit) +/// Windows: 8 bytes (same as double) +#[cfg(all( + any(target_arch = "x86_64", target_arch = "aarch64"), + not(target_os = "windows") +))] +const LONG_DOUBLE_SIZE: usize = 16; + +#[cfg(target_os = "windows")] +const LONG_DOUBLE_SIZE: usize = mem::size_of::<c_double>(); + +#[cfg(not(any( + all( + any(target_arch = "x86_64", target_arch = "aarch64"), + not(target_os = "windows") + ), + target_os = "windows" +)))] +const LONG_DOUBLE_SIZE: usize = mem::size_of::<c_double>(); + +/// Type information for ctypes simple types +struct TypeInfo { + pub size: usize, + pub ffi_type_fn: fn() -> libffi::middle::Type, +} + +/// Get type information (size and ffi_type) for a ctypes type code +fn type_info(ty: &str) -> Option<TypeInfo> { + use libffi::middle::Type; + match ty { + "c" => Some(TypeInfo { + size: mem::size_of::<c_schar>(), + ffi_type_fn: Type::u8, + }), + "u" => Some(TypeInfo { + size: mem::size_of::<WideChar>(), + ffi_type_fn: if mem::size_of::<WideChar>() == 2 { + Type::u16 + } else { + Type::u32 + }, + }), + "b" => Some(TypeInfo { + size: mem::size_of::<c_schar>(), + ffi_type_fn: Type::i8, + }), + "B" => Some(TypeInfo { + size: mem::size_of::<c_uchar>(), + ffi_type_fn: Type::u8, + }), + "h" | "v" => Some(TypeInfo { + size: mem::size_of::<c_short>(), + ffi_type_fn: Type::i16, + }), + "H" => Some(TypeInfo { + size: mem::size_of::<c_ushort>(), + ffi_type_fn: Type::u16, + }), + "i" => Some(TypeInfo { + size: mem::size_of::<c_int>(), + ffi_type_fn: Type::i32, + }), + "I" => Some(TypeInfo { + size: mem::size_of::<c_uint>(), + ffi_type_fn: Type::u32, + }), + "l" => Some(TypeInfo { + size: mem::size_of::<c_long>(), + ffi_type_fn: if mem::size_of::<c_long>() == 8 { + Type::i64 + } else { + Type::i32 + }, + }), + "L" => Some(TypeInfo { + size: mem::size_of::<c_ulong>(), + ffi_type_fn: if mem::size_of::<c_ulong>() == 8 { + Type::u64 + } else { + Type::u32 + }, + }), + "q" => Some(TypeInfo { + size: mem::size_of::<c_longlong>(), + ffi_type_fn: Type::i64, + }), + "Q" => Some(TypeInfo { + size: mem::size_of::<c_ulonglong>(), + ffi_type_fn: Type::u64, + }), + "f" => Some(TypeInfo { + size: mem::size_of::<c_float>(), + ffi_type_fn: Type::f32, + }), + "d" => Some(TypeInfo { + size: mem::size_of::<c_double>(), + ffi_type_fn: Type::f64, + }), + "g" => Some(TypeInfo { + // long double - platform dependent size + // x86_64 macOS/Linux: 16 bytes (80-bit extended + padding) + // ARM64: 16 bytes (128-bit) + // Windows: 8 bytes (same as double) + // Note: Use f64 as FFI type since Rust doesn't support long double natively + size: LONG_DOUBLE_SIZE, + ffi_type_fn: Type::f64, + }), + "?" => Some(TypeInfo { + size: mem::size_of::<c_uchar>(), + ffi_type_fn: Type::u8, + }), + "z" | "Z" | "P" | "X" | "O" => Some(TypeInfo { + size: mem::size_of::<usize>(), + ffi_type_fn: Type::pointer, + }), + "void" => Some(TypeInfo { + size: 0, + ffi_type_fn: Type::void, + }), + _ => None, + } +} + +/// Get size for a ctypes type code +fn get_size(ty: &str) -> usize { + type_info(ty).map(|t| t.size).expect("invalid type code") +} + +/// Get alignment for simple type codes from type_info(). +/// For primitive C types (c_int, c_long, etc.), alignment equals size. +fn get_align(ty: &str) -> usize { + get_size(ty) +} + +#[pymodule] +pub(crate) mod _ctypes { + use super::library; + use super::{PyCArray, PyCData, PyCPointer, PyCSimple, PyCStructure, PyCUnion}; + use crate::builtins::{PyType, PyTypeRef}; + use crate::class::StaticType; + use crate::convert::ToPyObject; + use crate::function::{Either, OptionalArg}; + use crate::types::Representable; + use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use num_traits::ToPrimitive; + + /// CArgObject - returned by byref() and paramfunc + /// tagPyCArgObject + #[pyclass(name = "CArgObject", module = "_ctypes", no_attr)] + #[derive(Debug, PyPayload)] + pub struct CArgObject { + /// Type tag ('P', 'V', 'i', 'd', etc.) + pub tag: u8, + /// The actual FFI value (mirrors union value) + pub value: super::FfiArgValue, + /// Reference to original object (for memory safety) + pub obj: PyObjectRef, + /// Size for struct/union ('V' tag) + #[allow(dead_code)] + pub size: usize, + /// Offset for byref() + pub offset: isize, + } + + /// is_literal_char - check if character is printable literal (not \\ or ') + fn is_literal_char(c: u8) -> bool { + c < 128 && c.is_ascii_graphic() && c != b'\\' && c != b'\'' + } + + impl Representable for CArgObject { + // PyCArg_repr - use tag and value fields directly + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + use super::base::FfiArgValue; + + let tag_char = zelf.tag as char; + + // Format value based on tag + match zelf.tag { + b'b' | b'h' | b'i' | b'l' | b'q' => { + // Signed integers + let n = match zelf.value { + FfiArgValue::I8(v) => v as i64, + FfiArgValue::I16(v) => v as i64, + FfiArgValue::I32(v) => v as i64, + FfiArgValue::I64(v) => v, + _ => 0, + }; + Ok(format!("<cparam '{}' ({})>", tag_char, n)) + } + b'B' | b'H' | b'I' | b'L' | b'Q' => { + // Unsigned integers + let n = match zelf.value { + FfiArgValue::U8(v) => v as u64, + FfiArgValue::U16(v) => v as u64, + FfiArgValue::U32(v) => v as u64, + FfiArgValue::U64(v) => v, + _ => 0, + }; + Ok(format!("<cparam '{}' ({})>", tag_char, n)) + } + b'f' => { + let v = match zelf.value { + FfiArgValue::F32(v) => v as f64, + _ => 0.0, + }; + Ok(format!("<cparam '{}' ({})>", tag_char, v)) + } + b'd' | b'g' => { + let v = match zelf.value { + FfiArgValue::F64(v) => v, + FfiArgValue::F32(v) => v as f64, + _ => 0.0, + }; + Ok(format!("<cparam '{}' ({})>", tag_char, v)) + } + b'c' => { + // c_char - single byte + let byte = match zelf.value { + FfiArgValue::I8(v) => v as u8, + FfiArgValue::U8(v) => v, + _ => 0, + }; + if is_literal_char(byte) { + Ok(format!("<cparam '{}' ('{}')>", tag_char, byte as char)) + } else { + Ok(format!("<cparam '{}' ('\\x{:02x}')>", tag_char, byte)) + } + } + b'z' | b'Z' | b'P' | b'V' => { + // Pointer types + let ptr = match zelf.value { + FfiArgValue::Pointer(v) => v, + _ => 0, + }; + if ptr == 0 { + Ok(format!("<cparam '{}' (nil)>", tag_char)) + } else { + Ok(format!("<cparam '{}' ({:#x})>", tag_char, ptr)) + } + } + _ => { + // Default fallback + let addr = zelf.get_id(); + if is_literal_char(zelf.tag) { + Ok(format!("<cparam '{}' at {:#x}>", tag_char, addr)) + } else { + Ok(format!("<cparam {:#04x} at {:#x}>", zelf.tag, addr)) + } + } + } + } + } + + #[pyclass(with(Representable))] + impl CArgObject { + #[pygetset] + fn _obj(&self) -> PyObjectRef { + self.obj.clone() + } + } + + #[pyattr(name = "__version__")] + const __VERSION__: &str = "1.1.0"; + + // TODO: get properly + #[pyattr] + const RTLD_LOCAL: i32 = 0; + + // TODO: get properly + #[pyattr] + const RTLD_GLOBAL: i32 = 0; + + #[pyattr] + const SIZEOF_TIME_T: usize = core::mem::size_of::<libc::time_t>(); + + #[pyattr] + const CTYPES_MAX_ARGCOUNT: usize = 1024; + + #[pyattr] + const FUNCFLAG_STDCALL: u32 = 0x0; + #[pyattr] + const FUNCFLAG_CDECL: u32 = 0x1; + #[pyattr] + const FUNCFLAG_HRESULT: u32 = 0x2; + #[pyattr] + const FUNCFLAG_PYTHONAPI: u32 = 0x4; + #[pyattr] + const FUNCFLAG_USE_ERRNO: u32 = 0x8; + #[pyattr] + const FUNCFLAG_USE_LASTERROR: u32 = 0x10; + + #[pyattr] + const TYPEFLAG_ISPOINTER: u32 = 0x100; + #[pyattr] + const TYPEFLAG_HASPOINTER: u32 = 0x200; + + #[pyattr] + const DICTFLAG_FINAL: u32 = 0x1000; + + #[pyattr(name = "ArgumentError", once)] + fn argument_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "_ctypes", + "ArgumentError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + #[cfg(target_os = "windows")] + #[pyattr(name = "COMError", once)] + fn com_error(vm: &VirtualMachine) -> PyTypeRef { + use crate::builtins::type_::PyAttributes; + use crate::function::FuncArgs; + use crate::types::{PyTypeFlags, PyTypeSlots}; + + // Sets hresult, text, details as instance attributes in __init__ + // This function has InitFunc signature for direct slots.init use + fn comerror_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let (hresult, text, details): ( + Option<PyObjectRef>, + Option<PyObjectRef>, + Option<PyObjectRef>, + ) = args.bind(vm)?; + let hresult = hresult.unwrap_or_else(|| vm.ctx.none()); + let text = text.unwrap_or_else(|| vm.ctx.none()); + let details = details.unwrap_or_else(|| vm.ctx.none()); + + // Set instance attributes + zelf.set_attr("hresult", hresult.clone(), vm)?; + zelf.set_attr("text", text.clone(), vm)?; + zelf.set_attr("details", details.clone(), vm)?; + + // self.args = args[1:] = (text, details) + // via: PyObject_SetAttrString(self, "args", PySequence_GetSlice(args, 1, size)) + let args_tuple: PyObjectRef = vm.ctx.new_tuple(vec![text, details]).into(); + zelf.set_attr("args", args_tuple, vm)?; + + Ok(()) + } + + // Create exception type with IMMUTABLETYPE flag + let mut attrs = PyAttributes::default(); + attrs.insert( + vm.ctx.intern_str("__module__"), + vm.ctx.new_str("_ctypes").into(), + ); + attrs.insert( + vm.ctx.intern_str("__doc__"), + vm.ctx + .new_str("Raised when a COM method call failed.") + .into(), + ); + + // Create slots with IMMUTABLETYPE flag + let slots = PyTypeSlots { + name: "COMError", + flags: PyTypeFlags::heap_type_flags() + | PyTypeFlags::HAS_DICT + | PyTypeFlags::IMMUTABLETYPE, + ..PyTypeSlots::default() + }; + + let exc_type = PyType::new_heap( + "COMError", + vec![vm.ctx.exceptions.exception_type.to_owned()], + attrs, + slots, + vm.ctx.types.type_type.to_owned(), + &vm.ctx, + ) + .unwrap(); + + // Set our custom init after new_heap, which runs init_slots that would + // otherwise overwrite slots.init with init_wrapper (due to __init__ in MRO). + exc_type.slots.init.store(Some(comerror_init)); + + exc_type + } + + /// Get the size of a ctypes type or instance + #[pyfunction] + pub fn sizeof(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + use super::structure::PyCStructType; + use super::union::PyCUnionType; + + // 1. Check if obj is a TYPE object (not instance) - PyStgInfo_FromType + if let Some(type_obj) = obj.downcast_ref::<PyType>() { + // Type object - return StgInfo.size + if let Some(stg_info) = type_obj.stg_info_opt() { + return Ok(stg_info.size); + } + // Fallback for type objects without StgInfo + // Array types + if type_obj + .class() + .fast_issubclass(super::array::PyCArrayType::static_type()) + && let Ok(stg) = type_obj.stg_info(vm) + { + return Ok(stg.size); + } + // Structure types + if type_obj + .class() + .fast_issubclass(PyCStructType::static_type()) + { + return super::structure::calculate_struct_size(type_obj, vm); + } + // Union types + if type_obj + .class() + .fast_issubclass(PyCUnionType::static_type()) + { + return super::union::calculate_union_size(type_obj, vm); + } + // Simple types + if type_obj.fast_issubclass(PyCSimple::static_type()) { + if let Ok(type_attr) = type_obj.as_object().get_attr("_type_", vm) + && let Ok(type_str) = type_attr.str(vm) + { + return Ok(super::get_size(type_str.as_ref())); + } + return Ok(core::mem::size_of::<usize>()); + } + // Pointer types + if type_obj.fast_issubclass(PyCPointer::static_type()) { + return Ok(core::mem::size_of::<usize>()); + } + return Err(vm.new_type_error("this type has no size")); + } + + // 2. Instance object - return actual buffer size (b_size) + // CDataObject_Check + return obj->b_size + if let Some(cdata) = obj.downcast_ref::<PyCData>() { + return Ok(cdata.size()); + } + if obj.fast_isinstance(PyCPointer::static_type()) { + return Ok(core::mem::size_of::<usize>()); + } + + Err(vm.new_type_error("this type has no size")) + } + + #[cfg(windows)] + #[pyfunction(name = "LoadLibrary")] + fn load_library_windows( + name: String, + _load_flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<usize> { + // TODO: audit functions first + // TODO: load_flags + let cache = library::libcache(); + let mut cache_write = cache.write(); + let (id, _) = cache_write.get_or_insert_lib(&name, vm).unwrap(); + Ok(id) + } + + #[cfg(not(windows))] + #[pyfunction(name = "dlopen")] + fn load_library_unix( + name: Option<crate::function::FsPath>, + load_flags: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<usize> { + // Default mode: RTLD_NOW | RTLD_LOCAL, always force RTLD_NOW + let mode = load_flags.unwrap_or(libc::RTLD_NOW | libc::RTLD_LOCAL) | libc::RTLD_NOW; + + match name { + Some(name) => { + let cache = library::libcache(); + let mut cache_write = cache.write(); + let os_str = name.as_os_str(vm)?; + let (id, _) = cache_write + .get_or_insert_lib_with_mode(&*os_str, mode, vm) + .map_err(|e| { + let name_str = os_str.to_string_lossy(); + vm.new_os_error(format!("{}: {}", name_str, e)) + })?; + Ok(id) + } + None => { + // dlopen(NULL, mode) to get the current process handle (for pythonapi) + let handle = unsafe { libc::dlopen(core::ptr::null(), mode) }; + if handle.is_null() { + let err = unsafe { libc::dlerror() }; + let msg = if err.is_null() { + "dlopen() error".to_string() + } else { + unsafe { + core::ffi::CStr::from_ptr(err) + .to_string_lossy() + .into_owned() + } + }; + return Err(vm.new_os_error(msg)); + } + // Add to library cache so symbol lookup works + let cache = library::libcache(); + let mut cache_write = cache.write(); + let id = cache_write.insert_raw_handle(handle); + Ok(id) + } + } + } + + #[pyfunction(name = "FreeLibrary")] + fn free_library(handle: usize) -> PyResult<()> { + let cache = library::libcache(); + let mut cache_write = cache.write(); + cache_write.drop_lib(handle); + Ok(()) + } + + #[cfg(not(windows))] + #[pyfunction] + fn dlclose(handle: usize, _vm: &VirtualMachine) -> PyResult<()> { + // Remove from cache, which triggers SharedLibrary drop. + // libloading::Library calls dlclose automatically on Drop. + let cache = library::libcache(); + let mut cache_write = cache.write(); + cache_write.drop_lib(handle); + Ok(()) + } + + #[cfg(not(windows))] + #[pyfunction] + fn dlsym( + handle: usize, + name: crate::builtins::PyUtf8StrRef, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let symbol_name = alloc::ffi::CString::new(name.as_str()) + .map_err(|_| vm.new_value_error("symbol name contains null byte"))?; + + // Clear previous error + unsafe { libc::dlerror() }; + + let ptr = unsafe { libc::dlsym(handle as *mut libc::c_void, symbol_name.as_ptr()) }; + + // Check for error via dlerror first + let err = unsafe { libc::dlerror() }; + if !err.is_null() { + let msg = unsafe { + core::ffi::CStr::from_ptr(err) + .to_string_lossy() + .into_owned() + }; + return Err(vm.new_os_error(msg)); + } + + // Treat NULL symbol address as error + // This handles cases like GNU IFUNCs that resolve to NULL + if ptr.is_null() { + return Err(vm.new_os_error(format!("symbol '{}' not found", name.as_str()))); + } + + Ok(ptr as usize) + } + + #[pyfunction(name = "POINTER")] + fn create_pointer_type(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyStr; + + // Get the _pointer_type_cache + let ctypes_module = vm.import("_ctypes", 0)?; + let cache = ctypes_module.get_attr("_pointer_type_cache", vm)?; + + // Check if already in cache using __getitem__ + if let Ok(cached) = vm.call_method(&cache, "__getitem__", (cls.clone(),)) + && !vm.is_none(&cached) + { + return Ok(cached); + } + + // Get the _Pointer base class + let pointer_base = ctypes_module.get_attr("_Pointer", vm)?; + + // Create a new type that inherits from _Pointer + let pointer_base_type = pointer_base + .clone() + .downcast::<crate::builtins::PyType>() + .map_err(|_| vm.new_type_error("_Pointer must be a type"))?; + let metaclass = pointer_base_type.class().to_owned(); + + let bases = vm.ctx.new_tuple(vec![pointer_base]); + let dict = vm.ctx.new_dict(); + + // PyUnicode_CheckExact(cls) - string creates incomplete pointer type + if let Some(s) = cls.downcast_ref::<PyStr>() { + // Incomplete pointer type: _type_ not set, cache key is id(result) + let name = format!("LP_{}", s.as_wtf8()); + + let new_type = metaclass + .as_object() + .call((vm.ctx.new_str(name), bases, dict), vm)?; + + // Store with id(result) as key for incomplete pointer types + let id_key: PyObjectRef = vm.ctx.new_int(new_type.get_id() as i64).into(); + vm.call_method(&cache, "__setitem__", (id_key, new_type.clone()))?; + + return Ok(new_type); + } + + // PyType_Check(cls) - type creates complete pointer type + if !cls.class().fast_issubclass(vm.ctx.types.type_type.as_ref()) { + return Err(vm.new_type_error("must be a ctypes type")); + } + + // Create the name for the pointer type + let name = if let Ok(type_obj) = cls.get_attr("__name__", vm) { + format!("LP_{}", type_obj.str(vm)?) + } else { + "LP_unknown".to_string() + }; + + // Complete pointer type: set _type_ attribute + dict.set_item("_type_", cls.clone(), vm)?; + + // Call the metaclass (PyCPointerType) to create the new type + let new_type = metaclass + .as_object() + .call((vm.ctx.new_str(name), bases, dict), vm)?; + + // Store in cache with cls as key + vm.call_method(&cache, "__setitem__", (cls, new_type.clone()))?; + + Ok(new_type) + } + + #[pyfunction] + fn pointer(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Get the type of the object + let obj_type = obj.class().to_owned(); + + // Create pointer type for this object's type + let ptr_type = create_pointer_type(obj_type.into(), vm)?; + + // Create an instance of the pointer type with the object + ptr_type.call((obj,), vm) + } + + #[pyfunction] + fn _pointer_type_cache() -> PyObjectRef { + todo!() + } + + #[cfg(target_os = "windows")] + #[pyfunction(name = "_check_HRESULT")] + fn check_hresult(_self: PyObjectRef, hr: i32, _vm: &VirtualMachine) -> PyResult<i32> { + // TODO: fixme + if hr < 0 { + // vm.ctx.new_windows_error(hr) + todo!(); + } else { + Ok(hr) + } + } + + #[pyfunction] + fn addressof(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + // All ctypes objects should return cdata buffer pointer + if let Some(cdata) = obj.downcast_ref::<PyCData>() { + Ok(cdata.buffer.read().as_ptr() as usize) + } else { + Err(vm.new_type_error("expected a ctypes instance")) + } + } + + #[pyfunction] + pub fn byref(obj: PyObjectRef, offset: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult { + use super::FfiArgValue; + + // Check if obj is a ctypes instance + if !obj.fast_isinstance(PyCData::static_type()) + && !obj.fast_isinstance(PyCSimple::static_type()) + { + return Err(vm.new_type_error(format!( + "byref() argument must be a ctypes instance, not '{}'", + obj.class().name() + ))); + } + + let offset_val = offset.unwrap_or(0); + + // Get buffer address: (char *)((CDataObject *)obj)->b_ptr + offset + let ptr_val = if let Some(simple) = obj.downcast_ref::<PyCSimple>() { + let buffer = simple.0.buffer.read(); + (buffer.as_ptr() as isize + offset_val) as usize + } else if let Some(cdata) = obj.downcast_ref::<PyCData>() { + let buffer = cdata.buffer.read(); + (buffer.as_ptr() as isize + offset_val) as usize + } else { + 0 + }; + + // Create CArgObject to hold the reference + Ok(CArgObject { + tag: b'P', + value: FfiArgValue::Pointer(ptr_val), + obj, + size: 0, + offset: offset_val, + } + .to_pyobject(vm)) + } + + #[pyfunction] + fn alignment(tp: Either<PyTypeRef, PyObjectRef>, vm: &VirtualMachine) -> PyResult<usize> { + use crate::builtins::PyType; + + let obj = match &tp { + Either::A(t) => t.as_object(), + Either::B(o) => o.as_ref(), + }; + + // 1. Check TypeDataSlot on class (for instances) + if let Some(stg_info) = obj.class().stg_info_opt() { + return Ok(stg_info.align); + } + + // 2. Check TypeDataSlot on type itself (for type objects) + if let Some(type_obj) = obj.downcast_ref::<PyType>() + && let Some(stg_info) = type_obj.stg_info_opt() + { + return Ok(stg_info.align); + } + + // 3. Fallback for simple types + if obj.fast_isinstance(PyCSimple::static_type()) + && let Ok(stg) = obj.class().stg_info(vm) + { + return Ok(stg.align); + } + if obj.fast_isinstance(PyCArray::static_type()) + && let Ok(stg) = obj.class().stg_info(vm) + { + return Ok(stg.align); + } + if obj.fast_isinstance(PyCStructure::static_type()) { + // Calculate alignment from _fields_ + let cls = obj.class(); + return alignment(Either::A(cls.to_owned()), vm); + } + if obj.fast_isinstance(PyCPointer::static_type()) { + // Pointer alignment is always pointer size + return Ok(core::mem::align_of::<usize>()); + } + if obj.fast_isinstance(PyCUnion::static_type()) { + // Calculate alignment from _fields_ + let cls = obj.class(); + return alignment(Either::A(cls.to_owned()), vm); + } + + // Get the type object to check + let type_obj: PyObjectRef = match &tp { + Either::A(t) => t.clone().into(), + Either::B(obj) => obj.class().to_owned().into(), + }; + + // For type objects, try to get alignment from _type_ attribute + if let Ok(type_attr) = type_obj.get_attr("_type_", vm) { + // Array/Pointer: _type_ is the element type (a PyType) + if let Ok(elem_type) = type_attr.clone().downcast::<crate::builtins::PyType>() { + return alignment(Either::A(elem_type), vm); + } + // Simple type: _type_ is a single character string + if let Ok(s) = type_attr.str(vm) { + let ty = s.to_string(); + if ty.len() == 1 && super::simple::SIMPLE_TYPE_CHARS.contains(ty.as_str()) { + return Ok(super::get_align(&ty)); + } + } + } + + // Structure/Union: max alignment of fields + if let Ok(fields_attr) = type_obj.get_attr("_fields_", vm) + && let Ok(fields) = fields_attr.try_to_value::<Vec<PyObjectRef>>(vm) + { + let mut max_align = 1usize; + for field in fields.iter() { + if let Some(tuple) = field.downcast_ref::<crate::builtins::PyTuple>() + && let Some(field_type) = tuple.get(1) + { + let align = + if let Ok(ft) = field_type.clone().downcast::<crate::builtins::PyType>() { + alignment(Either::A(ft), vm).unwrap_or(1) + } else { + 1 + }; + max_align = max_align.max(align); + } + } + return Ok(max_align); + } + + // For instances, delegate to their class + if let Either::B(obj) = &tp + && !obj.class().is(vm.ctx.types.type_type.as_ref()) + { + return alignment(Either::A(obj.class().to_owned()), vm); + } + + // No alignment info found + Err(vm.new_type_error("no alignment info")) + } + + #[pyfunction] + fn resize(obj: PyObjectRef, size: isize, vm: &VirtualMachine) -> PyResult<()> { + use alloc::borrow::Cow; + + // 1. Get StgInfo from object's class (validates ctypes instance) + let stg_info = obj + .class() + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("expected ctypes instance"))?; + + // 2. Validate size + if size < 0 || (size as usize) < stg_info.size { + return Err(vm.new_value_error(format!("minimum size is {}", stg_info.size))); + } + + // 3. Get PyCData via upcast (works for all ctypes types due to repr(transparent)) + let cdata = obj + .downcast_ref::<PyCData>() + .ok_or_else(|| vm.new_type_error("expected ctypes instance"))?; + + // 4. Check if buffer is owned (not borrowed from external memory) + { + let buffer = cdata.buffer.read(); + if matches!(&*buffer, Cow::Borrowed(_)) { + return Err(vm.new_value_error( + "Memory cannot be resized because this object doesn't own it", + )); + } + } + + // 5. Resize the buffer + let new_size = size as usize; + let mut buffer = cdata.buffer.write(); + let old_data = buffer.to_vec(); + let mut new_data = vec![0u8; new_size]; + let copy_len = old_data.len().min(new_size); + new_data[..copy_len].copy_from_slice(&old_data[..copy_len]); + *buffer = Cow::Owned(new_data); + + Ok(()) + } + + #[pyfunction] + fn get_errno() -> i32 { + super::function::get_errno_value() + } + + #[pyfunction] + fn set_errno(value: i32) -> i32 { + super::function::set_errno_value(value) + } + + #[cfg(windows)] + #[pyfunction] + fn get_last_error() -> PyResult<u32> { + Ok(super::function::get_last_error_value()) + } + + #[cfg(windows)] + #[pyfunction] + fn set_last_error(value: u32) -> u32 { + super::function::set_last_error_value(value) + } + + #[pyattr] + fn _memmove_addr(_vm: &VirtualMachine) -> usize { + let f = libc::memmove; + f as *const () as usize + } + + #[pyattr] + fn _memset_addr(_vm: &VirtualMachine) -> usize { + let f = libc::memset; + f as *const () as usize + } + + #[pyattr] + fn _string_at_addr(_vm: &VirtualMachine) -> usize { + super::function::INTERNAL_STRING_AT_ADDR + } + + #[pyattr] + fn _wstring_at_addr(_vm: &VirtualMachine) -> usize { + super::function::INTERNAL_WSTRING_AT_ADDR + } + + #[pyattr] + fn _cast_addr(_vm: &VirtualMachine) -> usize { + super::function::INTERNAL_CAST_ADDR + } + + #[pyattr] + fn _memoryview_at_addr(_vm: &VirtualMachine) -> usize { + super::function::INTERNAL_MEMORYVIEW_AT_ADDR + } + + #[pyfunction] + fn _cast( + obj: PyObjectRef, + src: PyObjectRef, + ctype: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + super::function::cast_impl(obj, src, ctype, vm) + } + + /// Python-level cast function (PYFUNCTYPE wrapper) + #[pyfunction] + fn cast(obj: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult { + super::function::cast_impl(obj.clone(), obj, typ, vm) + } + + /// Return buffer interface information for a ctypes type or object. + /// Returns a tuple (format, ndim, shape) where: + /// - format: PEP 3118 format string + /// - ndim: number of dimensions + /// - shape: tuple of dimension sizes + #[pyfunction] + fn buffer_info(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Determine if obj is a type or an instance + let is_type = obj.class().fast_issubclass(vm.ctx.types.type_type.as_ref()); + let cls = if is_type { + obj.clone() + } else { + obj.class().to_owned().into() + }; + + // Get format from type - try _type_ first (for simple types), then _stg_info_format_ + let format = if let Ok(type_attr) = cls.get_attr("_type_", vm) { + type_attr.str(vm)?.to_string() + } else if let Ok(format_attr) = cls.get_attr("_stg_info_format_", vm) { + format_attr.str(vm)?.to_string() + } else { + return Err(vm.new_type_error("not a ctypes type or object")); + }; + + // Non-array types have ndim=0 and empty shape + // TODO: Implement ndim/shape for arrays when StgInfo supports it + let ndim = 0; + let shape: Vec<PyObjectRef> = vec![]; + + let shape_tuple = vm.ctx.new_tuple(shape); + Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_str(format).into(), + vm.ctx.new_int(ndim).into(), + shape_tuple.into(), + ]) + .into()) + } + + /// Unpickle a ctypes object. + #[pyfunction] + fn _unpickle(typ: PyObjectRef, state: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if !state.class().is(vm.ctx.types.tuple_type.as_ref()) { + return Err(vm.new_type_error("state must be a tuple")); + } + let obj = vm.call_method(&typ, "__new__", (typ.clone(),))?; + vm.call_method(&obj, "__setstate__", (state,))?; + Ok(obj) + } + + /// Call a function at the given address with the given arguments. + #[pyfunction] + fn call_function( + func_addr: usize, + args: crate::builtins::PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { + call_function_internal(func_addr, args, 0, vm) + } + + /// Call a cdecl function at the given address with the given arguments. + #[pyfunction] + fn call_cdeclfunction( + func_addr: usize, + args: crate::builtins::PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult { + call_function_internal(func_addr, args, FUNCFLAG_CDECL, vm) + } + + fn call_function_internal( + func_addr: usize, + args: crate::builtins::PyTupleRef, + _flags: u32, + vm: &VirtualMachine, + ) -> PyResult { + use libffi::middle::{Arg, Cif, CodePtr, Type}; + + if func_addr == 0 { + return Err(vm.new_value_error("NULL function pointer")); + } + + let mut ffi_args: Vec<Arg<'_>> = Vec::with_capacity(args.len()); + let mut arg_values: Vec<isize> = Vec::with_capacity(args.len()); + let mut arg_types: Vec<Type> = Vec::with_capacity(args.len()); + + for arg in args.iter() { + if vm.is_none(arg) { + arg_values.push(0); + arg_types.push(Type::pointer()); + } else if let Ok(int_val) = arg.try_int(vm) { + let val = int_val.as_bigint().to_i64().unwrap_or(0) as isize; + arg_values.push(val); + arg_types.push(Type::isize()); + } else if let Some(bytes) = arg.downcast_ref::<crate::builtins::PyBytes>() { + let ptr = bytes.as_bytes().as_ptr() as isize; + arg_values.push(ptr); + arg_types.push(Type::pointer()); + } else if let Some(s) = arg.downcast_ref::<crate::builtins::PyStr>() { + let ptr = s.as_bytes().as_ptr() as isize; + arg_values.push(ptr); + arg_types.push(Type::pointer()); + } else { + return Err(vm.new_type_error(format!( + "Don't know how to convert parameter of type '{}'", + arg.class().name() + ))); + } + } + + for val in &arg_values { + ffi_args.push(Arg::new(val)); + } + + let cif = Cif::new(arg_types, Type::c_int()); + let code_ptr = CodePtr::from_ptr(func_addr as *const _); + let result: libc::c_int = unsafe { cif.call(code_ptr, &ffi_args) }; + Ok(vm.ctx.new_int(result).into()) + } + + /// Convert a pointer (as integer) to a Python object. + #[pyfunction(name = "PyObj_FromPtr")] + fn py_obj_from_ptr(ptr: usize, vm: &VirtualMachine) -> PyResult { + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + let raw_ptr = ptr as *mut crate::object::PyObject; + unsafe { + let obj = crate::PyObjectRef::from_raw(core::ptr::NonNull::new_unchecked(raw_ptr)); + let obj = core::mem::ManuallyDrop::new(obj); + Ok((*obj).clone()) + } + } + + #[pyfunction(name = "Py_INCREF")] + fn py_incref(obj: PyObjectRef, _vm: &VirtualMachine) -> PyObjectRef { + // TODO: + obj + } + + #[pyfunction(name = "Py_DECREF")] + fn py_decref(obj: PyObjectRef, _vm: &VirtualMachine) -> PyObjectRef { + // TODO: + obj + } + + #[cfg(target_os = "macos")] + #[pyfunction] + fn _dyld_shared_cache_contains_path( + path: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<bool> { + use alloc::ffi::CString; + + let path = match path { + Some(p) if !vm.is_none(&p) => p, + _ => return Ok(false), + }; + + let path_str = path.str(vm)?.to_string(); + let c_path = + CString::new(path_str).map_err(|_| vm.new_value_error("path contains null byte"))?; + + unsafe extern "C" { + fn _dyld_shared_cache_contains_path(path: *const libc::c_char) -> bool; + } + + let result = unsafe { _dyld_shared_cache_contains_path(c_path.as_ptr()) }; + Ok(result) + } + + #[cfg(windows)] + #[pyfunction(name = "FormatError")] + fn format_error_func(code: OptionalArg<u32>, _vm: &VirtualMachine) -> PyResult<String> { + use windows_sys::Win32::Foundation::{GetLastError, LocalFree}; + use windows_sys::Win32::System::Diagnostics::Debug::{ + FORMAT_MESSAGE_ALLOCATE_BUFFER, FORMAT_MESSAGE_FROM_SYSTEM, + FORMAT_MESSAGE_IGNORE_INSERTS, FormatMessageW, + }; + + let error_code = code.unwrap_or_else(|| unsafe { GetLastError() }); + + let mut buffer: *mut u16 = core::ptr::null_mut(); + let len = unsafe { + FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_IGNORE_INSERTS, + core::ptr::null(), + error_code, + 0, + &mut buffer as *mut *mut u16 as *mut u16, + 0, + core::ptr::null(), + ) + }; + + if len == 0 || buffer.is_null() { + return Ok("<no description>".to_string()); + } + + let message = unsafe { + let slice = core::slice::from_raw_parts(buffer, len as usize); + let msg = String::from_utf16_lossy(slice).trim_end().to_string(); + LocalFree(buffer as *mut _); + msg + }; + + Ok(message) + } + + #[cfg(windows)] + #[pyfunction(name = "CopyComPointer")] + fn copy_com_pointer(src: PyObjectRef, dst: PyObjectRef, vm: &VirtualMachine) -> PyResult<i32> { + use windows_sys::Win32::Foundation::{E_POINTER, S_OK}; + + // 1. Extract pointer-to-pointer address from dst (byref() result) + let pdst: usize = if let Some(carg) = dst.downcast_ref::<CArgObject>() { + // byref() result: object buffer address + offset + let base = if let Some(cdata) = carg.obj.downcast_ref::<PyCData>() { + cdata.buffer.read().as_ptr() as usize + } else { + return Ok(E_POINTER); + }; + (base as isize + carg.offset) as usize + } else { + return Ok(E_POINTER); + }; + + if pdst == 0 { + return Ok(E_POINTER); + } + + // 2. Extract COM pointer value from src + let src_ptr: usize = if vm.is_none(&src) { + 0 + } else if let Some(cdata) = src.downcast_ref::<PyCData>() { + // c_void_p etc: read pointer value from buffer + let buffer = cdata.buffer.read(); + if buffer.len() >= core::mem::size_of::<usize>() { + usize::from_ne_bytes( + buffer[..core::mem::size_of::<usize>()] + .try_into() + .unwrap_or([0; core::mem::size_of::<usize>()]), + ) + } else { + 0 + } + } else { + return Ok(E_POINTER); + }; + + // 3. Call IUnknown::AddRef if src is non-NULL + if src_ptr != 0 { + unsafe { + // IUnknown vtable: [QueryInterface, AddRef, Release, ...] + let iunknown = src_ptr as *mut *const usize; + let vtable = *iunknown; + debug_assert!(!vtable.is_null(), "IUnknown vtable is null"); + let addref_fn: extern "system" fn(*mut core::ffi::c_void) -> u32 = + core::mem::transmute(*vtable.add(1)); // AddRef is index 1 + addref_fn(src_ptr as *mut core::ffi::c_void); + } + } + + // 4. Copy pointer: *pdst = src + unsafe { + *(pdst as *mut usize) = src_ptr; + } + + Ok(S_OK) + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + use super::*; + + __module_exec(vm, module); + + let ctx = &vm.ctx; + PyCSimpleType::make_static_type(); + array::PyCArrayType::make_static_type(); + pointer::PyCPointerType::make_static_type(); + structure::PyCStructType::make_static_type(); + union::PyCUnionType::make_static_type(); + function::PyCFuncPtrType::make_static_type(); + function::RawMemoryBuffer::make_static_type(); + + extend_module!(vm, module, { + "_CData" => PyCData::make_static_type(), + "_SimpleCData" => PyCSimple::make_static_type(), + "Array" => PyCArray::make_static_type(), + "CField" => PyCField::make_static_type(), + "CFuncPtr" => function::PyCFuncPtr::make_static_type(), + "_Pointer" => PyCPointer::make_static_type(), + "_pointer_type_cache" => ctx.new_dict(), + "_array_type_cache" => ctx.new_dict(), + "Structure" => PyCStructure::make_static_type(), + "CThunkObject" => function::PyCThunk::make_static_type(), + "Union" => PyCUnion::make_static_type(), + }); + + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/_ctypes/array.rs b/crates/vm/src/stdlib/_ctypes/array.rs new file mode 100644 index 00000000000..568e2a4a0a9 --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes/array.rs @@ -0,0 +1,1369 @@ +use super::StgInfo; +use super::base::{CDATA_BUFFER_METHODS, PyCData}; +use super::type_info; +use crate::common::lock::LazyLock; +use crate::sliceable::SaturatedSliceIter; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + atomic_func, + builtins::{ + PyBytes, PyInt, PyList, PySlice, PyStr, PyType, PyTypeRef, genericalias::PyGenericAlias, + }, + class::StaticType, + function::{ArgBytesLike, FuncArgs, PySetterValue}, + protocol::{BufferDescriptor, PyBuffer, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + types::{AsBuffer, AsMapping, AsNumber, AsSequence, Constructor, Initializer}, +}; +use alloc::borrow::Cow; +use num_traits::{Signed, ToPrimitive}; + +/// Get itemsize from a PEP 3118 format string +/// Extracts the type code (last char after endianness prefix) and returns its size +fn get_size_from_format(fmt: &str) -> usize { + // Format is like "<f", ">q", etc. - strip endianness prefix and get type code + let code = fmt + .trim_start_matches(['<', '>', '@', '=', '!', '&']) + .chars() + .next() + .map(|c| c.to_string()); + code.map(|c| type_info(&c).map(|t| t.size).unwrap_or(1)) + .unwrap_or(1) +} + +/// Creates array type for (element_type, length) +/// Uses _array_type_cache to ensure identical calls return the same type object +pub(super) fn array_type_from_ctype( + itemtype: PyObjectRef, + length: usize, + vm: &VirtualMachine, +) -> PyResult { + // PyCArrayType_from_ctype + + // Get the _array_type_cache from _ctypes module + let ctypes_module = vm.import("_ctypes", 0)?; + let cache = ctypes_module.get_attr("_array_type_cache", vm)?; + + // Create cache key: (itemtype, length) tuple + let length_obj: PyObjectRef = vm.ctx.new_int(length).into(); + let cache_key = vm.ctx.new_tuple(vec![itemtype.clone(), length_obj]); + + // Check if already in cache + if let Ok(cached) = vm.call_method(&cache, "__getitem__", (cache_key.clone(),)) + && !vm.is_none(&cached) + { + return Ok(cached); + } + + // Cache miss - create new array type + let itemtype_ref = itemtype + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("Expected a type object"))?; + + let item_stg = itemtype_ref + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("_type_ must have storage info"))?; + + let element_size = item_stg.size; + let element_align = item_stg.align; + let item_format = item_stg.format.clone(); + let item_shape = item_stg.shape.clone(); + let item_flags = item_stg.flags; + + // Check overflow before multiplication + let total_size = element_size + .checked_mul(length) + .ok_or_else(|| vm.new_overflow_error("array too large"))?; + + // format name: "c_int_Array_5" + let type_name = format!("{}_Array_{}", itemtype_ref.name(), length); + + // Get item type code before moving itemtype + let item_type_code = itemtype_ref + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); + + let stg_info = StgInfo::new_array( + total_size, + element_align, + length, + itemtype_ref.clone(), + element_size, + item_format.as_deref(), + &item_shape, + item_flags, + ); + + let new_type = create_array_type_with_name(stg_info, &type_name, vm)?; + + // Special case for character arrays - add value/raw attributes + let new_type_ref: PyTypeRef = new_type + .clone() + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + match item_type_code.as_deref() { + Some("c") => add_char_array_getsets(&new_type_ref, vm), + Some("u") => add_wchar_array_getsets(&new_type_ref, vm), + _ => {} + } + + // Store in cache + vm.call_method(&cache, "__setitem__", (cache_key, new_type.clone()))?; + + Ok(new_type) +} + +/// create_array_type_with_name - create array type with specified name +fn create_array_type_with_name( + stg_info: StgInfo, + type_name: &str, + vm: &VirtualMachine, +) -> PyResult { + let metaclass = PyCArrayType::static_type().to_owned(); + let name = vm.ctx.new_str(type_name); + let bases = vm + .ctx + .new_tuple(vec![PyCArray::static_type().to_owned().into()]); + let dict = vm.ctx.new_dict(); + + let args = FuncArgs::new( + vec![name.into(), bases.into(), dict.into()], + crate::function::KwArgs::default(), + ); + + let new_type = crate::builtins::type_::PyType::slot_new(metaclass, args, vm)?; + + let type_ref: PyTypeRef = new_type + .clone() + .downcast() + .map_err(|_| vm.new_type_error("Failed to create array type"))?; + + // Set class attributes for _type_ and _length_ + if let Some(element_type) = stg_info.element_type.clone() { + new_type.set_attr("_type_", element_type, vm)?; + } + new_type.set_attr("_length_", vm.ctx.new_int(stg_info.length), vm)?; + + super::base::set_or_init_stginfo(&type_ref, stg_info); + + Ok(new_type) +} + +/// PyCArrayType - metatype for Array types +#[pyclass(name = "PyCArrayType", base = PyType, module = "_ctypes")] +#[derive(Debug)] +#[repr(transparent)] +pub(super) struct PyCArrayType(PyType); + +// PyCArrayType implements Initializer for slots.init (PyCArrayType_init) +impl Initializer for PyCArrayType { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // zelf is the newly created array type (e.g., T in "class T(Array)") + let new_type: &PyType = &zelf.0; + + new_type.check_not_initialized(vm)?; + + // 1. Get _length_ from class dict first + let direct_length = new_type + .attributes + .read() + .get(vm.ctx.intern_str("_length_")) + .cloned(); + + // 2. Get _type_ from class dict first + let direct_type = new_type + .attributes + .read() + .get(vm.ctx.intern_str("_type_")) + .cloned(); + + // 3. Find parent StgInfo from MRO (for inheritance) + // Note: PyType.mro does NOT include self, so no skip needed + let parent_stg_info = new_type + .mro + .read() + .iter() + .find_map(|base| base.stg_info_opt().map(|s| s.clone())); + + // 4. Resolve _length_ (direct or inherited) + let length = if let Some(length_attr) = direct_length { + // Direct _length_ defined - validate it (PyLong_Check) + let length_int = length_attr + .downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error("The '_length_' attribute must be an integer"))?; + let bigint = length_int.as_bigint(); + // Check sign first - negative values are ValueError + if bigint.is_negative() { + return Err(vm.new_value_error("The '_length_' attribute must not be negative")); + } + // Positive values that don't fit in usize are OverflowError + bigint + .to_usize() + .ok_or_else(|| vm.new_overflow_error("The '_length_' attribute is too large"))? + } else if let Some(ref parent_info) = parent_stg_info { + // Inherit from parent + parent_info.length + } else { + return Err(vm.new_attribute_error("class must define a '_length_' attribute")); + }; + + // 5. Resolve _type_ and get item_info (direct or inherited) + let (element_type, item_size, item_align, item_format, item_shape, item_flags) = + if let Some(type_attr) = direct_type { + // Direct _type_ defined - validate it (PyStgInfo_FromType) + let type_ref = type_attr + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("_type_ must be a type"))?; + let (size, align, format, shape, flags) = { + let item_info = type_ref + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("_type_ must have storage info"))?; + ( + item_info.size, + item_info.align, + item_info.format.clone(), + item_info.shape.clone(), + item_info.flags, + ) + }; + (type_ref, size, align, format, shape, flags) + } else if let Some(ref parent_info) = parent_stg_info { + // Inherit from parent + let parent_type = parent_info + .element_type + .clone() + .ok_or_else(|| vm.new_type_error("_type_ must have storage info"))?; + ( + parent_type, + parent_info.element_size, + parent_info.align, + parent_info.format.clone(), + parent_info.shape.clone(), + parent_info.flags, + ) + } else { + return Err(vm.new_attribute_error("class must define a '_type_' attribute")); + }; + + // 6. Check overflow (item_size != 0 && length > MAX / item_size) + if item_size != 0 && length > usize::MAX / item_size { + return Err(vm.new_overflow_error("array too large")); + } + + // 7. Initialize StgInfo (PyStgInfo_Init + field assignment) + let stg_info = StgInfo::new_array( + item_size * length, // size = item_size * length + item_align, // align = item_info->align + length, // length + element_type.clone(), + item_size, // element_size + item_format.as_deref(), + &item_shape, + item_flags, + ); + + // 8. Store StgInfo in type_data + super::base::set_or_init_stginfo(new_type, stg_info); + + // 9. Get type code before moving element_type + let item_type_code = element_type + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); + + // 10. Set class attributes for _type_ and _length_ + zelf.as_object().set_attr("_type_", element_type, vm)?; + zelf.as_object() + .set_attr("_length_", vm.ctx.new_int(length), vm)?; + + // 11. Special case for character arrays - add value/raw attributes + // if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) + // add_getset((PyTypeObject*)self, CharArray_getsets); + // else if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) + // add_getset((PyTypeObject*)self, WCharArray_getsets); + + // Get type ref for add_getset + let type_ref: PyTypeRef = zelf.as_object().to_owned().downcast().unwrap(); + match item_type_code.as_deref() { + Some("c") => add_char_array_getsets(&type_ref, vm), + Some("u") => add_wchar_array_getsets(&type_ref, vm), + _ => {} + } + + Ok(()) + } +} + +#[pyclass(flags(IMMUTABLETYPE), with(Initializer, AsNumber))] +impl PyCArrayType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the array type class that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. If already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } + + // 2. Check for CArgObject (PyCArg_CheckExact) + if let Some(carg) = value.downcast_ref::<super::_ctypes::CArgObject>() { + // Check if the wrapped object is an instance of the requested type + if carg.obj.is_instance(cls.as_object(), vm)? { + return Ok(value); // Return the CArgObject as-is + } + } + + // 3. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCArrayType::from_param(cls.as_object().to_owned(), as_parameter, vm); + } + + Err(vm.new_type_error(format!( + "expected {} instance instead of {}", + cls.name(), + value.class().name() + ))) + } +} + +impl AsNumber for PyCArrayType { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + multiply: Some(|a, b, vm| { + // a is a type object whose metaclass is PyCArrayType (e.g., Array_5) + let n = b + .try_index(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("array size too large"))?; + + if n < 0 { + return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); + } + + // Check for overflow before creating the new array type + let zelf_type = a + .downcast_ref::<PyType>() + .ok_or_else(|| vm.new_type_error("Expected type"))?; + + if let Some(stg_info) = zelf_type.stg_info_opt() { + let current_size = stg_info.size; + // Check if current_size * n would overflow + if current_size != 0 && (n as usize) > isize::MAX as usize / current_size { + return Err(vm.new_overflow_error("array too large")); + } + } + + // Use cached array type creation + // The element type of the new array is the current array type itself + array_type_from_ctype(a.to_owned(), n as usize, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +/// PyCArray - Array instance +/// All array metadata (element_type, length, element_size) is stored in the type's StgInfo +#[pyclass( + name = "Array", + base = PyCData, + metaclass = "PyCArrayType", + module = "_ctypes" +)] +#[derive(Debug)] +#[repr(transparent)] +pub struct PyCArray(pub PyCData); + +impl PyCArray { + /// Get the type code of array element type (e.g., "c" for c_char, "u" for c_wchar) + fn get_element_type_code(zelf: &Py<Self>, vm: &VirtualMachine) -> Option<String> { + zelf.class() + .stg_info_opt() + .and_then(|info| info.element_type.clone())? + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())) + } +} + +impl Constructor for PyCArray { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Check for abstract class - StgInfo must exist and be initialized + // Extract values in a block to drop the borrow before using cls + let (length, total_size) = { + let stg = cls.stg_info(vm)?; + (stg.length, stg.size) + }; + + // Check for too many initializers + if args.args.len() > length { + return Err(vm.new_index_error("too many initializers")); + } + + // Create array with zero-initialized buffer + let buffer = vec![0u8; total_size]; + let instance = PyCArray(PyCData::from_bytes_with_length(buffer, None, length)) + .into_ref_with_type(vm, cls)?; + + // Initialize elements using setitem_by_index (Array_init pattern) + for (i, value) in args.args.iter().enumerate() { + PyCArray::setitem_by_index(&instance, i as isize, value.clone(), vm)?; + } + + Ok(instance.into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyCArray { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Re-initialize array elements when __init__ is called + for (i, value) in args.args.iter().enumerate() { + PyCArray::setitem_by_index(&zelf, i as isize, value.clone(), vm)?; + } + Ok(()) + } +} + +impl AsSequence for PyCArray { + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| { + let zelf = PyCArray::sequence_downcast(seq); + Ok(zelf.class().stg_info_opt().map_or(0, |i| i.length)) + }), + item: atomic_func!(|seq, i, vm| { + let zelf = PyCArray::sequence_downcast(seq); + PyCArray::getitem_by_index(zelf, i, vm) + }), + ass_item: atomic_func!(|seq, i, value, vm| { + let zelf = PyCArray::sequence_downcast(seq); + match value { + Some(v) => PyCArray::setitem_by_index(zelf, i, v, vm), + None => Err(vm.new_type_error("cannot delete array elements")), + } + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } +} + +impl AsMapping for PyCArray { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + length: atomic_func!(|mapping, _vm| { + let zelf = PyCArray::mapping_downcast(mapping); + Ok(zelf.class().stg_info_opt().map_or(0, |i| i.length)) + }), + subscript: atomic_func!(|mapping, needle, vm| { + let zelf = PyCArray::mapping_downcast(mapping); + PyCArray::__getitem__(zelf, needle.to_owned(), vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyCArray::mapping_downcast(mapping); + match value { + Some(value) => PyCArray::__setitem__(zelf, needle.to_owned(), value, vm), + None => PyCArray::__delitem__(zelf, needle.to_owned(), vm), + } + }), + }); + &AS_MAPPING + } +} + +#[pyclass( + flags(BASETYPE, IMMUTABLETYPE), + with(Constructor, Initializer, AsSequence, AsMapping, AsBuffer) +)] +impl PyCArray { + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + fn int_to_bytes(i: &malachite_bigint::BigInt, size: usize) -> Vec<u8> { + // Try unsigned first (handles values like 0xFFFFFFFF that overflow signed) + // then fall back to signed (handles negative values) + match size { + 1 => { + if let Some(v) = i.to_u8() { + vec![v] + } else { + vec![i.to_i8().unwrap_or(0) as u8] + } + } + 2 => { + if let Some(v) = i.to_u16() { + v.to_ne_bytes().to_vec() + } else { + i.to_i16().unwrap_or(0).to_ne_bytes().to_vec() + } + } + 4 => { + if let Some(v) = i.to_u32() { + v.to_ne_bytes().to_vec() + } else { + i.to_i32().unwrap_or(0).to_ne_bytes().to_vec() + } + } + 8 => { + if let Some(v) = i.to_u64() { + v.to_ne_bytes().to_vec() + } else { + i.to_i64().unwrap_or(0).to_ne_bytes().to_vec() + } + } + _ => vec![0u8; size], + } + } + + fn bytes_to_int( + bytes: &[u8], + size: usize, + type_code: Option<&str>, + vm: &VirtualMachine, + ) -> PyObjectRef { + // Unsigned type codes: B (uchar), H (ushort), I (uint), L (ulong), Q (ulonglong) + let is_unsigned = matches!( + type_code, + Some("B") | Some("H") | Some("I") | Some("L") | Some("Q") + ); + + match (size, is_unsigned) { + (1, false) => vm.ctx.new_int(bytes[0] as i8).into(), + (1, true) => vm.ctx.new_int(bytes[0]).into(), + (2, false) => { + let val = i16::from_ne_bytes([bytes[0], bytes[1]]); + vm.ctx.new_int(val).into() + } + (2, true) => { + let val = u16::from_ne_bytes([bytes[0], bytes[1]]); + vm.ctx.new_int(val).into() + } + (4, false) => { + let val = i32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + vm.ctx.new_int(val).into() + } + (4, true) => { + let val = u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + vm.ctx.new_int(val).into() + } + (8, false) => { + let val = i64::from_ne_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]); + vm.ctx.new_int(val).into() + } + (8, true) => { + let val = u64::from_ne_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], + ]); + vm.ctx.new_int(val).into() + } + _ => vm.ctx.new_int(0).into(), + } + } + + fn getitem_by_index(zelf: &Py<PyCArray>, i: isize, vm: &VirtualMachine) -> PyResult { + let stg = zelf.class().stg_info_opt(); + let length = stg.as_ref().map_or(0, |i| i.length) as isize; + let index = if i < 0 { length + i } else { i }; + if index < 0 || index >= length { + return Err(vm.new_index_error("invalid index")); + } + let index = index as usize; + let element_size = stg.as_ref().map_or(0, |i| i.element_size); + let offset = index * element_size; + let type_code = Self::get_element_type_code(zelf, vm); + + // Get target buffer and offset (base's buffer if available, otherwise own) + let base_obj = zelf.0.base.read().clone(); + let (buffer_lock, final_offset) = if let Some(cdata) = base_obj + .as_ref() + .and_then(|b| b.downcast_ref::<super::PyCData>()) + { + (&cdata.buffer, zelf.0.base_offset.load() + offset) + } else { + (&zelf.0.buffer, offset) + }; + + let buffer = buffer_lock.read(); + Self::read_element_from_buffer( + &buffer, + final_offset, + element_size, + type_code.as_deref(), + vm, + ) + } + + /// Helper to read an element value from a buffer at given offset + fn read_element_from_buffer( + buffer: &[u8], + offset: usize, + element_size: usize, + type_code: Option<&str>, + vm: &VirtualMachine, + ) -> PyResult { + match type_code { + Some("c") => { + // Return single byte as bytes + if offset < buffer.len() { + Ok(vm.ctx.new_bytes(vec![buffer[offset]]).into()) + } else { + Ok(vm.ctx.new_bytes(vec![0]).into()) + } + } + Some("u") => { + // Return single wchar as str + if let Some(code) = wchar_from_bytes(&buffer[offset..]) { + let s = char::from_u32(code) + .map(|c| c.to_string()) + .unwrap_or_default(); + Ok(vm.ctx.new_str(s).into()) + } else { + Ok(vm.ctx.new_str("").into()) + } + } + Some("z") => { + // c_char_p: pointer to bytes - dereference to get string + if offset + element_size > buffer.len() { + return Ok(vm.ctx.none()); + } + let ptr_bytes = &buffer[offset..offset + element_size]; + let ptr_val = usize::from_ne_bytes( + ptr_bytes + .try_into() + .unwrap_or([0; core::mem::size_of::<usize>()]), + ); + if ptr_val == 0 { + return Ok(vm.ctx.none()); + } + // Read null-terminated string from pointer address + unsafe { + let ptr = ptr_val as *const u8; + let mut len = 0; + while *ptr.add(len) != 0 { + len += 1; + } + let bytes = core::slice::from_raw_parts(ptr, len); + Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) + } + } + Some("Z") => { + // c_wchar_p: pointer to wchar_t - dereference to get string + if offset + element_size > buffer.len() { + return Ok(vm.ctx.none()); + } + let ptr_bytes = &buffer[offset..offset + element_size]; + let ptr_val = usize::from_ne_bytes( + ptr_bytes + .try_into() + .unwrap_or([0; core::mem::size_of::<usize>()]), + ); + if ptr_val == 0 { + return Ok(vm.ctx.none()); + } + // Read null-terminated wide string using WCHAR_SIZE + unsafe { + let ptr = ptr_val as *const u8; + let mut chars = Vec::new(); + let mut pos = 0usize; + loop { + let code = if WCHAR_SIZE == 2 { + let bytes = core::slice::from_raw_parts(ptr.add(pos), 2); + u16::from_ne_bytes([bytes[0], bytes[1]]) as u32 + } else { + let bytes = core::slice::from_raw_parts(ptr.add(pos), 4); + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) + }; + if code == 0 { + break; + } + if let Some(ch) = char::from_u32(code) { + chars.push(ch); + } + pos += WCHAR_SIZE; + } + let s: String = chars.into_iter().collect(); + Ok(vm.ctx.new_str(s).into()) + } + } + Some("f") => { + // c_float + let val = buffer[offset..] + .first_chunk::<4>() + .copied() + .map_or(0.0, f32::from_ne_bytes); + Ok(vm.ctx.new_float(val as f64).into()) + } + Some("d") | Some("g") => { + // c_double / c_longdouble - read f64 from first 8 bytes + let val = buffer[offset..] + .first_chunk::<8>() + .copied() + .map_or(0.0, f64::from_ne_bytes); + Ok(vm.ctx.new_float(val).into()) + } + _ => { + if let Some(bytes) = buffer[offset..].get(..element_size) { + Ok(Self::bytes_to_int(bytes, element_size, type_code, vm)) + } else { + Ok(vm.ctx.new_int(0).into()) + } + } + } + } + + /// Helper to write an element value to a buffer at given offset + /// This is extracted to share code between direct write and base-buffer write + #[allow(clippy::too_many_arguments)] + fn write_element_to_buffer( + buffer: &mut [u8], + offset: usize, + element_size: usize, + type_code: Option<&str>, + value: &PyObject, + zelf: &Py<PyCArray>, + index: usize, + vm: &VirtualMachine, + ) -> PyResult<()> { + match type_code { + Some("c") => { + if let Some(b) = value.downcast_ref::<PyBytes>() { + if offset < buffer.len() { + buffer[offset] = b.as_bytes().first().copied().unwrap_or(0); + } + } else if let Ok(int_val) = value.try_int(vm) { + if offset < buffer.len() { + buffer[offset] = int_val.as_bigint().to_u8().unwrap_or(0); + } + } else { + return Err(vm.new_type_error("an integer or bytes of length 1 is required")); + } + } + Some("u") => { + if let Some(s) = value.downcast_ref::<PyStr>() { + let code = s + .as_wtf8() + .code_points() + .next() + .map(|c| c.to_u32()) + .unwrap_or(0); + if offset + WCHAR_SIZE <= buffer.len() { + wchar_to_bytes(code, &mut buffer[offset..]); + } + } else { + return Err(vm.new_type_error("unicode string expected")); + } + } + Some("z") => { + let (ptr_val, converted) = if value.is(&vm.ctx.none) { + (0usize, None) + } else if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + zelf.0.keep_alive(index, kept_alive); + (ptr, Some(value.to_owned())) + } else if let Ok(int_val) = value.try_index(vm) { + (int_val.as_bigint().to_usize().unwrap_or(0), None) + } else { + return Err(vm.new_type_error(format!( + "bytes or integer address expected instead of {} instance", + value.class().name() + ))); + }; + if offset + element_size <= buffer.len() { + buffer[offset..offset + element_size].copy_from_slice(&ptr_val.to_ne_bytes()); + } + if let Some(c) = converted { + return zelf.0.keep_ref(index, c, vm); + } + } + Some("Z") => { + let (ptr_val, converted) = if value.is(&vm.ctx.none) { + (0usize, None) + } else if let Some(s) = value.downcast_ref::<PyStr>() { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_wtf8(), vm); + (ptr, Some(holder)) + } else if let Ok(int_val) = value.try_index(vm) { + (int_val.as_bigint().to_usize().unwrap_or(0), None) + } else { + return Err(vm.new_type_error("unicode string or integer address expected")); + }; + if offset + element_size <= buffer.len() { + buffer[offset..offset + element_size].copy_from_slice(&ptr_val.to_ne_bytes()); + } + if let Some(c) = converted { + return zelf.0.keep_ref(index, c, vm); + } + } + Some("f") => { + // c_float: convert int/float to f32 bytes + let f32_val = if let Ok(float_val) = value.try_float(vm) { + float_val.to_f64() as f32 + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_f64().unwrap_or(0.0) as f32 + } else { + return Err(vm.new_type_error("a float is required")); + }; + if offset + 4 <= buffer.len() { + buffer[offset..offset + 4].copy_from_slice(&f32_val.to_ne_bytes()); + } + } + Some("d") | Some("g") => { + // c_double / c_longdouble: convert int/float to f64 bytes + let f64_val = if let Ok(float_val) = value.try_float(vm) { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_f64().unwrap_or(0.0) + } else { + return Err(vm.new_type_error("a float is required")); + }; + if offset + 8 <= buffer.len() { + buffer[offset..offset + 8].copy_from_slice(&f64_val.to_ne_bytes()); + } + // For "g" type, remaining bytes stay zero + } + _ => { + // Handle ctypes instances (copy their buffer) + if let Some(cdata) = value.downcast_ref::<PyCData>() { + let src_buffer = cdata.buffer.read(); + let copy_len = src_buffer.len().min(element_size); + if offset + copy_len <= buffer.len() { + buffer[offset..offset + copy_len].copy_from_slice(&src_buffer[..copy_len]); + } + // Other types: use int_to_bytes + } else if let Ok(int_val) = value.try_int(vm) { + let bytes = Self::int_to_bytes(int_val.as_bigint(), element_size); + if offset + element_size <= buffer.len() { + buffer[offset..offset + element_size].copy_from_slice(&bytes); + } + } else { + return Err(vm.new_type_error(format!( + "expected {} instance, not {}", + type_code.unwrap_or("value"), + value.class().name() + ))); + } + } + } + + // KeepRef + if super::base::PyCData::should_keep_ref(value) { + let to_keep = super::base::PyCData::get_kept_objects(value, vm); + zelf.0.keep_ref(index, to_keep, vm)?; + } + + Ok(()) + } + + fn setitem_by_index( + zelf: &Py<PyCArray>, + i: isize, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let stg = zelf.class().stg_info_opt(); + let length = stg.as_ref().map_or(0, |i| i.length) as isize; + let index = if i < 0 { length + i } else { i }; + if index < 0 || index >= length { + return Err(vm.new_index_error("invalid index")); + } + let index = index as usize; + let element_size = stg.as_ref().map_or(0, |i| i.element_size); + let offset = index * element_size; + let type_code = Self::get_element_type_code(zelf, vm); + + // Get target buffer and offset (base's buffer if available, otherwise own) + let base_obj = zelf.0.base.read().clone(); + let (buffer_lock, final_offset) = if let Some(cdata) = base_obj + .as_ref() + .and_then(|b| b.downcast_ref::<super::PyCData>()) + { + (&cdata.buffer, zelf.0.base_offset.load() + offset) + } else { + (&zelf.0.buffer, offset) + }; + + let mut buffer = buffer_lock.write(); + + // For shared memory (Cow::Borrowed), we need to write directly to the memory + // For owned memory (Cow::Owned), we can write to the owned buffer + match &mut *buffer { + Cow::Borrowed(slice) => { + // SAFETY: For from_buffer, the slice points to writable shared memory. + // Python's from_buffer requires writable buffer, so this is safe. + let ptr = slice.as_ptr() as *mut u8; + let len = slice.len(); + let owned_slice = unsafe { core::slice::from_raw_parts_mut(ptr, len) }; + Self::write_element_to_buffer( + owned_slice, + final_offset, + element_size, + type_code.as_deref(), + &value, + zelf, + index, + vm, + ) + } + Cow::Owned(vec) => Self::write_element_to_buffer( + vec, + final_offset, + element_size, + type_code.as_deref(), + &value, + zelf, + index, + vm, + ), + } + } + + // Array_subscript + fn __getitem__(zelf: &Py<Self>, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // PyIndex_Check + if let Some(i) = item.downcast_ref::<PyInt>() { + let i = i.as_bigint().to_isize().ok_or_else(|| { + vm.new_index_error("cannot fit index into an index-sized integer") + })?; + // getitem_by_index handles negative index normalization + Self::getitem_by_index(zelf, i, vm) + } + // PySlice_Check + else if let Some(slice) = item.downcast_ref::<PySlice>() { + Self::getitem_by_slice(zelf, slice, vm) + } else { + Err(vm.new_type_error("indices must be integers")) + } + } + + // Array_subscript slice handling + fn getitem_by_slice(zelf: &Py<Self>, slice: &PySlice, vm: &VirtualMachine) -> PyResult { + let stg = zelf.class().stg_info_opt(); + let length = stg.as_ref().map_or(0, |i| i.length); + + // PySlice_Unpack + PySlice_AdjustIndices + let sat_slice = slice.to_saturated(vm)?; + let (range, step, slice_len) = sat_slice.adjust_indices(length); + + let type_code = Self::get_element_type_code(zelf, vm); + let element_size = stg.as_ref().map_or(0, |i| i.element_size); + let start = range.start; + + match type_code.as_deref() { + // c_char → bytes (item_info->getfunc == "c") + Some("c") => { + if slice_len == 0 { + return Ok(vm.ctx.new_bytes(vec![]).into()); + } + let buffer = zelf.0.buffer.read(); + // step == 1 optimization: direct memcpy + if step == 1 { + let start_offset = start * element_size; + let end_offset = start_offset + slice_len; + if end_offset <= buffer.len() { + return Ok(vm + .ctx + .new_bytes(buffer[start_offset..end_offset].to_vec()) + .into()); + } + } + // Non-contiguous: iterate + let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); + let mut result = Vec::with_capacity(slice_len); + for idx in iter { + let offset = idx * element_size; + if offset < buffer.len() { + result.push(buffer[offset]); + } + } + Ok(vm.ctx.new_bytes(result).into()) + } + // c_wchar → str (item_info->getfunc == "u") + Some("u") => { + if slice_len == 0 { + return Ok(vm.ctx.new_str("").into()); + } + let buffer = zelf.0.buffer.read(); + // step == 1 optimization: direct conversion + if step == 1 { + let start_offset = start * WCHAR_SIZE; + let end_offset = start_offset + slice_len * WCHAR_SIZE; + if end_offset <= buffer.len() { + let wchar_bytes = &buffer[start_offset..end_offset]; + let result: String = wchar_bytes + .chunks(WCHAR_SIZE) + .filter_map(|chunk| wchar_from_bytes(chunk).and_then(char::from_u32)) + .collect(); + return Ok(vm.ctx.new_str(result).into()); + } + } + // Non-contiguous: iterate + let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); + let mut result = String::with_capacity(slice_len); + for idx in iter { + let offset = idx * WCHAR_SIZE; + if let Some(code_point) = wchar_from_bytes(&buffer[offset..]) + && let Some(c) = char::from_u32(code_point) + { + result.push(c); + } + } + Ok(vm.ctx.new_str(result).into()) + } + // Other types → list (PyList_New + Array_item for each) + _ => { + let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); + let mut result = Vec::with_capacity(slice_len); + for idx in iter { + result.push(Self::getitem_by_index(zelf, idx as isize, vm)?); + } + Ok(PyList::from(result).into_ref(&vm.ctx).into()) + } + } + } + + // Array_ass_subscript + fn __setitem__( + zelf: &Py<Self>, + item: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Array does not support item deletion + // (handled implicitly - value is always provided in __setitem__) + + // PyIndex_Check + if let Some(i) = item.downcast_ref::<PyInt>() { + let i = i.as_bigint().to_isize().ok_or_else(|| { + vm.new_index_error("cannot fit index into an index-sized integer") + })?; + // setitem_by_index handles negative index normalization + Self::setitem_by_index(zelf, i, value, vm) + } + // PySlice_Check + else if let Some(slice) = item.downcast_ref::<PySlice>() { + Self::setitem_by_slice(zelf, slice, value, vm) + } else { + Err(vm.new_type_error("indices must be integer")) + } + } + + // Array does not support item deletion + fn __delitem__(&self, _item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("Array does not support item deletion")) + } + + // Array_ass_subscript slice handling + fn setitem_by_slice( + zelf: &Py<Self>, + slice: &PySlice, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let length = zelf.class().stg_info_opt().map_or(0, |i| i.length); + + // PySlice_Unpack + PySlice_AdjustIndices + let sat_slice = slice.to_saturated(vm)?; + let (range, step, slice_len) = sat_slice.adjust_indices(length); + + // other_len = PySequence_Length(value); + let items: Vec<PyObjectRef> = vm.extract_elements_with(&value, Ok)?; + let other_len = items.len(); + + if other_len != slice_len { + return Err(vm.new_value_error("Can only assign sequence of same size")); + } + + // Use SaturatedSliceIter for correct index iteration (handles negative step) + let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); + + for (idx, item) in iter.zip(items) { + Self::setitem_by_index(zelf, idx as isize, item, vm)?; + } + Ok(()) + } + + fn __len__(zelf: &Py<Self>, _vm: &VirtualMachine) -> usize { + zelf.class().stg_info_opt().map_or(0, |i| i.length) + } +} + +impl AsBuffer for PyCArray { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let buffer_len = zelf.0.buffer.read().len(); + + // Get format and shape from type's StgInfo + let stg_info = zelf + .class() + .stg_info_opt() + .expect("PyCArray type must have StgInfo"); + let format = stg_info.format.clone(); + let shape = stg_info.shape.clone(); + + let desc = if let Some(fmt) = format + && !shape.is_empty() + { + // itemsize is the size of the base element type (item_info->size) + // For empty arrays, we still need the element size, not 0 + let total_elements: usize = shape.iter().product(); + let has_zero_dim = shape.contains(&0); + let itemsize = if total_elements > 0 && buffer_len > 0 { + buffer_len / total_elements + } else { + // For empty arrays, get itemsize from format type code + get_size_from_format(&fmt) + }; + + // Build dim_desc from shape (C-contiguous: row-major order) + // stride[i] = product(shape[i+1:]) * itemsize + // For empty arrays (any dimension is 0), all strides are 0 + let mut dim_desc = Vec::with_capacity(shape.len()); + let mut stride = itemsize as isize; + + for &dim_size in shape.iter().rev() { + let current_stride = if has_zero_dim { 0 } else { stride }; + dim_desc.push((dim_size, current_stride, 0)); + stride *= dim_size as isize; + } + dim_desc.reverse(); + + BufferDescriptor { + len: buffer_len, + readonly: false, + itemsize, + format: alloc::borrow::Cow::Owned(fmt), + dim_desc, + } + } else { + // Fallback to simple buffer if no format/shape info + BufferDescriptor::simple(buffer_len, false) + }; + + let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); + Ok(buf) + } +} + +// CharArray and WCharArray getsets - added dynamically via add_getset + +// CharArray_get_value +fn char_array_get_value(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let buffer = zelf.0.buffer.read(); + let len = buffer.iter().position(|&b| b == 0).unwrap_or(buffer.len()); + Ok(vm.ctx.new_bytes(buffer[..len].to_vec()).into()) +} + +// CharArray_set_value +fn char_array_set_value(obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let bytes = value + .downcast_ref::<PyBytes>() + .ok_or_else(|| vm.new_type_error("bytes expected"))?; + let mut buffer = zelf.0.buffer.write(); + let src = bytes.as_bytes(); + + if src.len() > buffer.len() { + return Err(vm.new_value_error("byte string too long")); + } + + buffer.to_mut()[..src.len()].copy_from_slice(src); + if src.len() < buffer.len() { + buffer.to_mut()[src.len()] = 0; + } + Ok(()) +} + +// CharArray_get_raw +fn char_array_get_raw(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let buffer = zelf.0.buffer.read(); + Ok(vm.ctx.new_bytes(buffer.to_vec()).into()) +} + +// CharArray_set_raw +fn char_array_set_raw( + obj: PyObjectRef, + value: PySetterValue<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + let value = value.ok_or_else(|| vm.new_attribute_error("cannot delete attribute"))?; + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let bytes_like = ArgBytesLike::try_from_object(vm, value)?; + let mut buffer = zelf.0.buffer.write(); + let src = bytes_like.borrow_buf(); + if src.len() > buffer.len() { + return Err(vm.new_value_error("byte string too long")); + } + buffer.to_mut()[..src.len()].copy_from_slice(&src); + Ok(()) +} + +// WCharArray_get_value +fn wchar_array_get_value(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let buffer = zelf.0.buffer.read(); + Ok(vm.ctx.new_str(wstring_from_bytes(&buffer)).into()) +} + +// WCharArray_set_value +fn wchar_array_set_value( + obj: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<()> { + let zelf = obj.downcast_ref::<PyCArray>().unwrap(); + let s = value + .downcast_ref::<PyStr>() + .ok_or_else(|| vm.new_type_error("unicode string expected"))?; + let mut buffer = zelf.0.buffer.write(); + let wchar_count = buffer.len() / WCHAR_SIZE; + let char_count = s.as_wtf8().code_points().count(); + + if char_count > wchar_count { + return Err(vm.new_value_error("string too long")); + } + + for (i, ch) in s.as_wtf8().code_points().enumerate() { + let offset = i * WCHAR_SIZE; + wchar_to_bytes(ch.to_u32(), &mut buffer.to_mut()[offset..]); + } + + let terminator_offset = char_count * WCHAR_SIZE; + if terminator_offset + WCHAR_SIZE <= buffer.len() { + wchar_to_bytes(0, &mut buffer.to_mut()[terminator_offset..]); + } + Ok(()) +} + +/// add_getset for c_char arrays - adds 'value' and 'raw' attributes +/// add_getset((PyTypeObject*)self, CharArray_getsets) +fn add_char_array_getsets(array_type: &Py<PyType>, vm: &VirtualMachine) { + // SAFETY: getset is owned by array_type which outlives the getset + let value_getset = unsafe { + vm.ctx.new_getset( + "value", + array_type, + char_array_get_value, + char_array_set_value, + ) + }; + let raw_getset = unsafe { + vm.ctx + .new_getset("raw", array_type, char_array_get_raw, char_array_set_raw) + }; + + array_type + .attributes + .write() + .insert(vm.ctx.intern_str("value"), value_getset.into()); + array_type + .attributes + .write() + .insert(vm.ctx.intern_str("raw"), raw_getset.into()); +} + +/// add_getset for c_wchar arrays - adds only 'value' attribute (no 'raw') +fn add_wchar_array_getsets(array_type: &Py<PyType>, vm: &VirtualMachine) { + // SAFETY: getset is owned by array_type which outlives the getset + let value_getset = unsafe { + vm.ctx.new_getset( + "value", + array_type, + wchar_array_get_value, + wchar_array_set_value, + ) + }; + + array_type + .attributes + .write() + .insert(vm.ctx.intern_str("value"), value_getset.into()); +} + +// wchar_t helpers - Platform-independent wide character handling +// Windows: sizeof(wchar_t) == 2 (UTF-16) +// Linux/macOS: sizeof(wchar_t) == 4 (UTF-32) + +/// Size of wchar_t on this platform +pub(super) const WCHAR_SIZE: usize = core::mem::size_of::<libc::wchar_t>(); + +/// Read a single wchar_t from bytes (platform-endian) +#[inline] +pub(super) fn wchar_from_bytes(bytes: &[u8]) -> Option<u32> { + if bytes.len() < WCHAR_SIZE { + return None; + } + Some(if WCHAR_SIZE == 2 { + u16::from_ne_bytes([bytes[0], bytes[1]]) as u32 + } else { + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) + }) +} + +/// Write a single wchar_t to bytes (platform-endian) +#[inline] +pub(super) fn wchar_to_bytes(ch: u32, buffer: &mut [u8]) { + if WCHAR_SIZE == 2 { + if buffer.len() >= 2 { + buffer[..2].copy_from_slice(&(ch as u16).to_ne_bytes()); + } + } else if buffer.len() >= 4 { + buffer[..4].copy_from_slice(&ch.to_ne_bytes()); + } +} + +/// Read a null-terminated wchar_t string from bytes, returns String +fn wstring_from_bytes(buffer: &[u8]) -> String { + let mut chars = Vec::new(); + for chunk in buffer.chunks(WCHAR_SIZE) { + if chunk.len() < WCHAR_SIZE { + break; + } + let code = if WCHAR_SIZE == 2 { + u16::from_ne_bytes([chunk[0], chunk[1]]) as u32 + } else { + u32::from_ne_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) + }; + if code == 0 { + break; // null terminator + } + if let Some(ch) = char::from_u32(code) { + chars.push(ch); + } + } + chars.into_iter().collect() +} diff --git a/crates/vm/src/stdlib/_ctypes/base.rs b/crates/vm/src/stdlib/_ctypes/base.rs new file mode 100644 index 00000000000..0bfbe57bb04 --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes/base.rs @@ -0,0 +1,2608 @@ +use super::array::{WCHAR_SIZE, wchar_from_bytes, wchar_to_bytes}; +use crate::builtins::{ + PyBytes, PyDict, PyList, PyMemoryView, PyStr, PyTuple, PyType, PyTypeRef, PyUtf8Str, +}; +use crate::class::StaticType; +use crate::convert::ToPyObject; +use crate::function::{ArgBytesLike, OptionalArg, PySetterValue}; +use crate::protocol::{BufferMethods, PyBuffer}; +use crate::types::{Constructor, GetDescriptor, Representable}; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, +}; +use alloc::borrow::Cow; +use core::ffi::{ + c_double, c_float, c_int, c_long, c_longlong, c_short, c_uint, c_ulong, c_ulonglong, c_ushort, +}; +use core::fmt::Debug; +use core::mem; +use crossbeam_utils::atomic::AtomicCell; +use num_traits::{Signed, ToPrimitive}; +use rustpython_common::lock::PyRwLock; +use rustpython_common::wtf8::Wtf8; +use widestring::WideChar; + +// StgInfo - Storage information for ctypes types +// Stored in TypeDataSlot of heap types (PyType::init_type_data/get_type_data) + +// Flag constants +bitflags::bitflags! { + #[derive(Default, Copy, Clone, Debug, PartialEq, Eq)] + pub struct StgInfoFlags: u32 { + // Function calling convention flags + /// Standard call convention (Windows) + const FUNCFLAG_STDCALL = 0x0; + /// C calling convention + const FUNCFLAG_CDECL = 0x1; + /// Function returns HRESULT + const FUNCFLAG_HRESULT = 0x2; + /// Use Python API calling convention + const FUNCFLAG_PYTHONAPI = 0x4; + /// Capture errno after call + const FUNCFLAG_USE_ERRNO = 0x8; + /// Capture last error after call (Windows) + const FUNCFLAG_USE_LASTERROR = 0x10; + + // Type flags + /// Type is a pointer type + const TYPEFLAG_ISPOINTER = 0x100; + /// Type contains pointer fields + const TYPEFLAG_HASPOINTER = 0x200; + /// Type is or contains a union + const TYPEFLAG_HASUNION = 0x400; + /// Type contains bitfield members + const TYPEFLAG_HASBITFIELD = 0x800; + + // Dict flags + /// Type is finalized (_fields_ has been set) + const DICTFLAG_FINAL = 0x1000; + } +} + +/// ParamFunc - determines how a type is passed to foreign functions +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) enum ParamFunc { + #[default] + None, + /// Array types are passed as pointers (tag = 'P') + Array, + /// Simple types use their specific conversion (tag = type code) + Simple, + /// Pointer types (tag = 'P') + Pointer, + /// Structure types (tag = 'V' for value) + Structure, + /// Union types (tag = 'V' for value) + Union, +} + +#[derive(Clone)] +pub struct StgInfo { + pub initialized: bool, + pub size: usize, // number of bytes + pub align: usize, // alignment requirements + pub length: usize, // number of fields (for arrays/structures) + pub proto: Option<PyTypeRef>, // Only for Pointer/ArrayObject + pub flags: StgInfoFlags, // type flags (TYPEFLAG_*, DICTFLAG_*) + + // Array-specific fields + pub element_type: Option<PyTypeRef>, // _type_ for arrays + pub element_size: usize, // size of each element + + // PEP 3118 buffer protocol fields + pub format: Option<String>, // struct format string (e.g., "i", "(5)i") + pub shape: Vec<usize>, // shape for multi-dimensional arrays + + // Function parameter conversion + pub(super) paramfunc: ParamFunc, // how to pass to foreign functions + + // Byte order (for _swappedbytes_) + pub big_endian: bool, // true if big endian, false if little endian + + // FFI field types for structure/union passing (inherited from base class) + pub ffi_field_types: Vec<libffi::middle::Type>, + + // Cached pointer type (non-inheritable via descriptor) + pub pointer_type: Option<PyObjectRef>, +} + +// StgInfo is stored in type_data which requires Send + Sync. +// The PyTypeRef in proto/element_type fields is protected by the type system's locking mechanism. +// ctypes objects are not thread-safe by design; users must synchronize access. +unsafe impl Send for StgInfo {} +unsafe impl Sync for StgInfo {} + +impl core::fmt::Debug for StgInfo { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("StgInfo") + .field("initialized", &self.initialized) + .field("size", &self.size) + .field("align", &self.align) + .field("length", &self.length) + .field("proto", &self.proto) + .field("flags", &self.flags) + .field("element_type", &self.element_type) + .field("element_size", &self.element_size) + .field("format", &self.format) + .field("shape", &self.shape) + .field("paramfunc", &self.paramfunc) + .field("big_endian", &self.big_endian) + .field("ffi_field_types", &self.ffi_field_types.len()) + .finish() + } +} + +impl Default for StgInfo { + fn default() -> Self { + StgInfo { + initialized: false, + size: 0, + align: 1, + length: 0, + proto: None, + flags: StgInfoFlags::empty(), + element_type: None, + element_size: 0, + format: None, + shape: Vec::new(), + paramfunc: ParamFunc::None, + big_endian: cfg!(target_endian = "big"), // native endian by default + ffi_field_types: Vec::new(), + pointer_type: None, + } + } +} + +impl StgInfo { + pub fn new(size: usize, align: usize) -> Self { + StgInfo { + initialized: true, + size, + align, + length: 0, + proto: None, + flags: StgInfoFlags::empty(), + element_type: None, + element_size: 0, + format: None, + shape: Vec::new(), + paramfunc: ParamFunc::None, + big_endian: cfg!(target_endian = "big"), // native endian by default + ffi_field_types: Vec::new(), + pointer_type: None, + } + } + + /// Create StgInfo for an array type + /// item_format: the innermost element's format string (kept as-is, e.g., "<i") + /// item_shape: the element's shape (will be prepended with length) + /// item_flags: the element type's flags (for HASPOINTER inheritance) + #[allow(clippy::too_many_arguments)] + pub fn new_array( + size: usize, + align: usize, + length: usize, + element_type: PyTypeRef, + element_size: usize, + item_format: Option<&str>, + item_shape: &[usize], + item_flags: StgInfoFlags, + ) -> Self { + // Format is kept from innermost element (e.g., "<i" for c_int arrays) + // The array dimensions go into shape only, not format + let format = item_format.map(|f| f.to_owned()); + + // Build shape: [length, ...item_shape] + let mut shape = vec![length]; + shape.extend_from_slice(item_shape); + + // Inherit HASPOINTER flag from element type + // if (iteminfo->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + // stginfo->flags |= TYPEFLAG_HASPOINTER; + let flags = if item_flags + .intersects(StgInfoFlags::TYPEFLAG_ISPOINTER | StgInfoFlags::TYPEFLAG_HASPOINTER) + { + StgInfoFlags::TYPEFLAG_HASPOINTER + } else { + StgInfoFlags::empty() + }; + + StgInfo { + initialized: true, + size, + align, + length, + proto: None, + flags, + element_type: Some(element_type), + element_size, + format, + shape, + paramfunc: ParamFunc::Array, + big_endian: cfg!(target_endian = "big"), // native endian by default + ffi_field_types: Vec::new(), + pointer_type: None, + } + } + + /// Get libffi type for this StgInfo + /// Note: For very large types, returns pointer type to avoid overflow + pub fn to_ffi_type(&self) -> libffi::middle::Type { + // Limit to avoid overflow in libffi (MAX_STRUCT_SIZE is platform-dependent) + const MAX_FFI_STRUCT_SIZE: usize = 1024 * 1024; // 1MB limit for safety + + match self.paramfunc { + ParamFunc::Structure | ParamFunc::Union => { + if !self.ffi_field_types.is_empty() { + libffi::middle::Type::structure(self.ffi_field_types.iter().cloned()) + } else if self.size <= MAX_FFI_STRUCT_SIZE { + // Small struct without field types: use bytes array + libffi::middle::Type::structure(core::iter::repeat_n( + libffi::middle::Type::u8(), + self.size, + )) + } else { + // Large struct: treat as pointer (passed by reference) + libffi::middle::Type::pointer() + } + } + ParamFunc::Array => { + if self.size > MAX_FFI_STRUCT_SIZE || self.length > MAX_FFI_STRUCT_SIZE { + // Large array: treat as pointer + libffi::middle::Type::pointer() + } else if let Some(ref fmt) = self.format { + let elem_type = Self::format_to_ffi_type(fmt); + libffi::middle::Type::structure(core::iter::repeat_n(elem_type, self.length)) + } else { + libffi::middle::Type::structure(core::iter::repeat_n( + libffi::middle::Type::u8(), + self.size, + )) + } + } + ParamFunc::Pointer => libffi::middle::Type::pointer(), + _ => { + // Simple type: derive from format + if let Some(ref fmt) = self.format { + Self::format_to_ffi_type(fmt) + } else { + libffi::middle::Type::u8() + } + } + } + } + + /// Convert format string to libffi type + fn format_to_ffi_type(fmt: &str) -> libffi::middle::Type { + // Strip endian prefix if present + let code = fmt.trim_start_matches(['<', '>', '!', '@', '=']); + match code { + "b" => libffi::middle::Type::i8(), + "B" => libffi::middle::Type::u8(), + "h" => libffi::middle::Type::i16(), + "H" => libffi::middle::Type::u16(), + "i" | "l" => libffi::middle::Type::i32(), + "I" | "L" => libffi::middle::Type::u32(), + "q" => libffi::middle::Type::i64(), + "Q" => libffi::middle::Type::u64(), + "f" => libffi::middle::Type::f32(), + "d" => libffi::middle::Type::f64(), + "P" | "z" | "Z" | "O" => libffi::middle::Type::pointer(), + _ => libffi::middle::Type::u8(), // default + } + } + + /// Check if this type is finalized (cannot set _fields_ again) + pub fn is_final(&self) -> bool { + self.flags.contains(StgInfoFlags::DICTFLAG_FINAL) + } + + /// Get proto type reference (for Pointer/Array types) + pub fn proto(&self) -> &Py<PyType> { + self.proto.as_deref().expect("type has proto") + } +} + +/// __pointer_type__ getter for ctypes metaclasses. +/// Reads from StgInfo.pointer_type (non-inheritable). +pub(super) fn pointer_type_get(zelf: &Py<PyType>, vm: &VirtualMachine) -> PyResult { + zelf.stg_info_opt() + .and_then(|info| info.pointer_type.clone()) + .ok_or_else(|| { + vm.new_attribute_error(format!( + "type {} has no attribute '__pointer_type__'", + zelf.name() + )) + }) +} + +/// __pointer_type__ setter for ctypes metaclasses. +/// Writes to StgInfo.pointer_type (non-inheritable). +pub(super) fn pointer_type_set( + zelf: &Py<PyType>, + value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<()> { + if let Some(mut info) = zelf.get_type_data_mut::<StgInfo>() { + info.pointer_type = Some(value); + Ok(()) + } else { + Err(vm.new_attribute_error(format!("cannot set __pointer_type__ on {}", zelf.name()))) + } +} + +/// Get PEP3118 format string for a field type +/// Returns the format string considering byte order +pub(super) fn get_field_format( + field_type: &PyObject, + big_endian: bool, + vm: &VirtualMachine, +) -> String { + let endian_prefix = if big_endian { ">" } else { "<" }; + + // 1. Check StgInfo for format + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(stg_info) = type_obj.stg_info_opt() + && let Some(fmt) = &stg_info.format + { + // For structures (T{...}), arrays ((n)...), and pointers (&...), return as-is + // These complex types have their own endianness markers inside + if fmt.starts_with('T') + || fmt.starts_with('(') + || fmt.starts_with('&') + || fmt.starts_with("X{") + { + return fmt.clone(); + } + + // For simple types, replace existing endian prefix with the correct one + let base_fmt = fmt.trim_start_matches(['<', '>', '@', '=', '!']); + if !base_fmt.is_empty() { + return format!("{}{}", endian_prefix, base_fmt); + } + return fmt.clone(); + } + + // 2. Try to get _type_ attribute for simple types + if let Ok(type_attr) = field_type.get_attr("_type_", vm) + && let Some(type_str) = type_attr.downcast_ref::<PyStr>() + { + let s = type_str + .to_str() + .expect("_type_ is validated as ASCII at type creation"); + return format!("{}{}", endian_prefix, s); + } + + // Default: single byte + "B".to_string() +} + +/// Compute byte order based on swapped flag +#[inline] +pub(super) fn is_big_endian(is_swapped: bool) -> bool { + if is_swapped { + !cfg!(target_endian = "big") + } else { + cfg!(target_endian = "big") + } +} + +/// Shared BufferMethods for all ctypes types (PyCArray, PyCSimple, PyCStructure, PyCUnion) +/// All these types are #[repr(transparent)] wrappers around PyCData +pub(super) static CDATA_BUFFER_METHODS: BufferMethods = BufferMethods { + obj_bytes: |buffer| { + rustpython_common::lock::PyRwLockReadGuard::map( + buffer.obj_as::<PyCData>().buffer.read(), + |x| &**x, + ) + .into() + }, + obj_bytes_mut: |buffer| { + rustpython_common::lock::PyRwLockWriteGuard::map( + buffer.obj_as::<PyCData>().buffer.write(), + |x| x.to_mut().as_mut_slice(), + ) + .into() + }, + release: |_| {}, + retain: |_| {}, +}; + +/// Convert Vec<T> to Vec<u8> by reinterpreting the memory (same allocation). +fn vec_to_bytes<T>(vec: Vec<T>) -> Vec<u8> { + let len = vec.len() * core::mem::size_of::<T>(); + let cap = vec.capacity() * core::mem::size_of::<T>(); + let ptr = vec.as_ptr() as *mut u8; + core::mem::forget(vec); + unsafe { Vec::from_raw_parts(ptr, len, cap) } +} + +/// Ensure PyBytes data is null-terminated. Returns (kept_alive_obj, pointer). +/// The caller must keep the returned object alive to keep the pointer valid. +pub(super) fn ensure_z_null_terminated( + bytes: &PyBytes, + vm: &VirtualMachine, +) -> (PyObjectRef, usize) { + let data = bytes.as_bytes(); + let mut buffer = data.to_vec(); + if !buffer.ends_with(&[0]) { + buffer.push(0); + } + let ptr = buffer.as_ptr() as usize; + let kept_alive: PyObjectRef = vm.ctx.new_bytes(buffer).into(); + (kept_alive, ptr) +} + +/// Convert str to null-terminated wchar_t buffer. Returns (PyBytes holder, pointer). +pub(super) fn str_to_wchar_bytes(s: &Wtf8, vm: &VirtualMachine) -> (PyObjectRef, usize) { + let wchars: Vec<libc::wchar_t> = s + .code_points() + .map(|cp| cp.to_u32() as libc::wchar_t) + .chain(core::iter::once(0)) + .collect(); + let ptr = wchars.as_ptr() as usize; + let bytes = vec_to_bytes(wchars); + let holder: PyObjectRef = vm.ctx.new_bytes(bytes).into(); + (holder, ptr) +} + +/// PyCData - base type for all ctypes data types +#[pyclass(name = "_CData", module = "_ctypes")] +#[derive(Debug, PyPayload)] +pub struct PyCData { + /// Memory buffer - Owned (self-owned) or Borrowed (external reference) + /// + /// SAFETY: Borrowed variant's 'static lifetime is not actually static. + /// When created via from_address or from_base_obj, only valid for the lifetime of the source memory. + /// Same behavior as CPython's b_ptr (user responsibility, kept alive via b_base). + pub buffer: PyRwLock<Cow<'static, [u8]>>, + /// pointer to base object or None (b_base) + pub base: PyRwLock<Option<PyObjectRef>>, + /// byte offset within base's buffer (for field access) + pub base_offset: AtomicCell<usize>, + /// index into base's b_objects list (b_index) + pub index: AtomicCell<usize>, + /// dictionary of references we need to keep (b_objects) + pub objects: PyRwLock<Option<PyObjectRef>>, + /// number of references we need (b_length) + pub length: AtomicCell<usize>, + /// References kept alive but not visible in _objects. + /// Used for null-terminated c_char_p buffer copies, since + /// RustPython's PyBytes lacks CPython's internal trailing null. + /// Keyed by unique_key (hierarchical) so nested fields don't collide. + pub(super) kept_refs: PyRwLock<std::collections::HashMap<String, PyObjectRef>>, +} + +impl PyCData { + /// Create from StgInfo (PyCData_MallocBuffer pattern) + pub fn from_stg_info(stg_info: &StgInfo) -> Self { + PyCData { + buffer: PyRwLock::new(Cow::Owned(vec![0u8; stg_info.size])), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(None), + length: AtomicCell::new(stg_info.length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } + } + + /// Create from existing bytes (copies data) + pub fn from_bytes(data: Vec<u8>, objects: Option<PyObjectRef>) -> Self { + PyCData { + buffer: PyRwLock::new(Cow::Owned(data)), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(objects), + length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } + } + + /// Create from bytes with specified length (for arrays) + pub fn from_bytes_with_length( + data: Vec<u8>, + objects: Option<PyObjectRef>, + length: usize, + ) -> Self { + PyCData { + buffer: PyRwLock::new(Cow::Owned(data)), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(objects), + length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } + } + + /// Create from external memory address + /// + /// # Safety + /// The returned slice's 'static lifetime is a lie. + /// Actually only valid for the lifetime of the memory pointed to by ptr. + /// PyCData_AtAddress + pub unsafe fn at_address(ptr: *const u8, size: usize) -> Self { + // = PyCData_AtAddress + // SAFETY: Caller must ensure ptr is valid for the lifetime of returned PyCData + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; + PyCData { + buffer: PyRwLock::new(Cow::Borrowed(slice)), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(None), + length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } + } + + /// Create from base object with offset and data copy + /// + /// Similar to from_base_with_offset, but also stores a copy of the data. + /// This is used for arrays where we need our own buffer for the buffer protocol, + /// but still maintain the base reference for KeepRef and tracking. + pub fn from_base_with_data( + base_obj: PyObjectRef, + offset: usize, + idx: usize, + length: usize, + data: Vec<u8>, + ) -> Self { + PyCData { + buffer: PyRwLock::new(Cow::Owned(data)), // Has its own buffer copy + base: PyRwLock::new(Some(base_obj)), // But still tracks base + base_offset: AtomicCell::new(offset), // And offset for writes + index: AtomicCell::new(idx), + objects: PyRwLock::new(None), + length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } + } + + /// Create from base object's buffer + /// + /// This creates a borrowed view into the base's buffer at the given address. + /// The base object is stored in b_base to keep the memory alive. + /// + /// # Safety + /// ptr must point into base_obj's buffer and remain valid as long as base_obj is alive. + pub unsafe fn from_base_obj( + ptr: *mut u8, + size: usize, + base_obj: PyObjectRef, + idx: usize, + ) -> Self { + // = PyCData_FromBaseObj + // SAFETY: ptr points into base_obj's buffer, kept alive via base reference + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; + PyCData { + buffer: PyRwLock::new(Cow::Borrowed(slice)), + base: PyRwLock::new(Some(base_obj)), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(idx), + objects: PyRwLock::new(None), + length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } + } + + /// Create from buffer protocol object (for from_buffer method) + /// + /// Unlike from_bytes, this shares memory with the source buffer. + /// The source object is stored in objects dict to keep the buffer alive. + /// Python stores with key -1 via KeepRef(result, -1, mv). + /// + /// # Safety + /// ptr must point to valid memory that remains valid as long as source is alive. + pub unsafe fn from_buffer_shared( + ptr: *const u8, + size: usize, + length: usize, + source: PyObjectRef, + vm: &VirtualMachine, + ) -> Self { + // SAFETY: Caller must ensure ptr is valid for the lifetime of source + let slice: &'static [u8] = unsafe { core::slice::from_raw_parts(ptr, size) }; + + // Python stores the reference in a dict with key "-1" (unique_key pattern) + let objects_dict = vm.ctx.new_dict(); + objects_dict + .set_item("-1", source, vm) + .expect("Failed to store buffer reference"); + + PyCData { + buffer: PyRwLock::new(Cow::Borrowed(slice)), + base: PyRwLock::new(None), + base_offset: AtomicCell::new(0), + index: AtomicCell::new(0), + objects: PyRwLock::new(Some(objects_dict.into())), + length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), + } + } + + /// Common implementation for from_buffer class method. + /// Validates buffer, creates memoryview, and returns PyCData sharing memory with source. + /// + /// CDataType_from_buffer_impl + pub fn from_buffer_impl( + cls: &Py<PyType>, + source: PyObjectRef, + offset: isize, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let (size, length) = { + let stg_info = cls + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("not a ctypes type"))?; + (stg_info.size, stg_info.length) + }; + + if offset < 0 { + return Err(vm.new_value_error("offset cannot be negative")); + } + let offset = offset as usize; + + // Get buffer from source (this exports the buffer) + let buffer = PyBuffer::try_from_object(vm, source)?; + + // Check if buffer is writable + if buffer.desc.readonly { + return Err(vm.new_type_error("underlying buffer is not writable")); + } + + // Check if buffer is C contiguous + if !buffer.desc.is_contiguous() { + return Err(vm.new_type_error("underlying buffer is not C contiguous")); + } + + // Check if buffer is large enough + let buffer_len = buffer.desc.len; + if offset + size > buffer_len { + return Err(vm.new_value_error(format!( + "Buffer size too small ({} instead of at least {} bytes)", + buffer_len, + offset + size + ))); + } + + // Get buffer pointer - the memory is owned by source + let ptr = { + let bytes = buffer.obj_bytes(); + bytes.as_ptr().wrapping_add(offset) + }; + + // Create memoryview to keep buffer exported (prevents source from being modified) + // mv = PyMemoryView_FromObject(obj); KeepRef(result, -1, mv); + let memoryview = PyMemoryView::from_buffer(buffer, vm)?; + let mv_obj = memoryview.into_pyobject(vm); + + // Create CData that shares memory with the buffer + Ok(unsafe { Self::from_buffer_shared(ptr, size, length, mv_obj, vm) }) + } + + /// Common implementation for from_buffer_copy class method. + /// Copies data from buffer and creates new independent instance. + /// + /// CDataType_from_buffer_copy_impl + pub fn from_buffer_copy_impl( + cls: &Py<PyType>, + source: &[u8], + offset: isize, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let (size, length) = { + let stg_info = cls + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("not a ctypes type"))?; + (stg_info.size, stg_info.length) + }; + + if offset < 0 { + return Err(vm.new_value_error("offset cannot be negative")); + } + let offset = offset as usize; + + // Check if buffer is large enough + if offset + size > source.len() { + return Err(vm.new_value_error(format!( + "Buffer size too small ({} instead of at least {} bytes)", + source.len(), + offset + size + ))); + } + + // Copy bytes from buffer at offset + let data = source[offset..offset + size].to_vec(); + + Ok(Self::from_bytes_with_length(data, None, length)) + } + + #[inline] + pub fn size(&self) -> usize { + self.buffer.read().len() + } + + /// Check if this buffer is borrowed (external memory reference) + #[inline] + pub fn is_borrowed(&self) -> bool { + matches!(&*self.buffer.read(), Cow::Borrowed(_)) + } + + /// Write bytes at offset - handles both borrowed and owned buffers + /// + /// For borrowed buffers (from from_address), writes directly to external memory. + /// For owned buffers, writes through to_mut() as normal. + /// + /// # Safety + /// For borrowed buffers, caller must ensure the memory is writable. + pub fn write_bytes_at_offset(&self, offset: usize, bytes: &[u8]) { + let buffer = self.buffer.read(); + if offset + bytes.len() > buffer.len() { + return; // Out of bounds + } + + match &*buffer { + Cow::Borrowed(slice) => { + // For borrowed memory, write directly + // SAFETY: We assume the caller knows this memory is writable + // (e.g., from from_address pointing to a ctypes buffer) + unsafe { + let ptr = slice.as_ptr() as *mut u8; + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr.add(offset), bytes.len()); + } + } + Cow::Owned(_) => { + // For owned memory, use to_mut() through write lock + drop(buffer); + let mut buffer = self.buffer.write(); + buffer.to_mut()[offset..offset + bytes.len()].copy_from_slice(bytes); + } + } + } + + /// Generate unique key for nested references (unique_key) + /// Creates a hierarchical key by walking up the b_base chain. + /// Format: "index:parent_index:grandparent_index:..." + pub fn unique_key(&self, index: usize) -> String { + let mut key = format!("{index:x}"); + // Walk up the base chain to build hierarchical key + if self.base.read().is_some() { + let parent_index = self.index.load(); + key.push_str(&format!(":{parent_index:x}")); + } + key + } + + /// Keep a reference in the objects dictionary (KeepRef) + /// + /// Stores 'keep' in this object's b_objects dict at key 'index'. + /// If keep is None, does nothing (optimization). + /// This function stores the value directly - caller should use get_kept_objects() + /// first if they want to store the _objects of a CData instead of the object itself. + /// + /// If this object has a base (is embedded in another structure/union/array), + /// the reference is stored in the root object's b_objects with a hierarchical key. + pub fn keep_ref(&self, index: usize, keep: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Optimization: no need to store None + if vm.is_none(&keep) { + return Ok(()); + } + + // Build hierarchical key + let key = self.unique_key(index); + + // If we have a base object, find root and store there + if let Some(base_obj) = self.base.read().clone() { + // Find root by walking up the base chain + let root_obj = Self::find_root_object(&base_obj); + Self::store_in_object(&root_obj, &key, keep, vm)?; + return Ok(()); + } + + // No base - store in own objects dict + let mut objects = self.objects.write(); + + // Initialize b_objects if needed + if objects.is_none() { + if self.length.load() > 0 { + // Need to store multiple references - create a dict + *objects = Some(vm.ctx.new_dict().into()); + } else { + // Only one reference needed - store directly + *objects = Some(keep); + return Ok(()); + } + } + + // If b_objects is not a dict, convert it to a dict first + // This preserves the existing reference (e.g., from cast) when adding new references + if let Some(obj) = objects.as_ref() + && obj.downcast_ref::<PyDict>().is_none() + { + // Convert existing single reference to a dict + let dict = vm.ctx.new_dict(); + // Store the original object with a special key (id-based) + let id_key: PyObjectRef = vm.ctx.new_int(obj.get_id() as i64).into(); + dict.set_item(&*id_key, obj.clone(), vm)?; + *objects = Some(dict.into()); + } + + // Store in dict with unique key + if let Some(dict_obj) = objects.as_ref() + && let Some(dict) = dict_obj.downcast_ref::<PyDict>() + { + let key_obj: PyObjectRef = vm.ctx.new_str(key).into(); + dict.set_item(&*key_obj, keep, vm)?; + } + + Ok(()) + } + + /// Keep a reference alive without exposing it in _objects. + /// Walks up to root object (same as keep_ref) so the reference + /// lives as long as the owning ctypes object. + /// Uses unique_key (hierarchical) so nested fields don't collide. + pub fn keep_alive(&self, index: usize, obj: PyObjectRef) { + let key = self.unique_key(index); + if let Some(base_obj) = self.base.read().clone() { + let root = Self::find_root_object(&base_obj); + if let Some(cdata) = root.downcast_ref::<PyCData>() { + cdata.kept_refs.write().insert(key, obj); + return; + } + } + self.kept_refs.write().insert(key, obj); + } + + /// Find the root object (one without a base) by walking up the base chain + fn find_root_object(obj: &PyObject) -> PyObjectRef { + // Try to get base from different ctypes types + let base = if let Some(cdata) = obj.downcast_ref::<PyCData>() { + cdata.base.read().clone() + } else { + None + }; + + // Recurse if there's a base, otherwise this is the root + if let Some(base_obj) = base { + Self::find_root_object(&base_obj) + } else { + obj.to_owned() + } + } + + /// Store a value in an object's _objects dict with the given key + fn store_in_object( + obj: &PyObject, + key: &str, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Get the objects dict from the object + let objects_lock = if let Some(cdata) = obj.downcast_ref::<PyCData>() { + &cdata.objects + } else { + return Ok(()); // Unknown type, skip + }; + + let mut objects = objects_lock.write(); + + // Initialize if needed + if objects.is_none() { + *objects = Some(vm.ctx.new_dict().into()); + } + + // If not a dict, convert to dict + if let Some(obj) = objects.as_ref() + && obj.downcast_ref::<PyDict>().is_none() + { + let dict = vm.ctx.new_dict(); + let id_key: PyObjectRef = vm.ctx.new_int(obj.get_id() as i64).into(); + dict.set_item(&*id_key, obj.clone(), vm)?; + *objects = Some(dict.into()); + } + + // Store in dict + if let Some(dict_obj) = objects.as_ref() + && let Some(dict) = dict_obj.downcast_ref::<PyDict>() + { + let key_obj: PyObjectRef = vm.ctx.new_str(key).into(); + dict.set_item(&*key_obj, value, vm)?; + } + + Ok(()) + } + + /// Get kept objects from a CData instance + /// Returns the _objects of the CData, or an empty dict if None. + pub fn get_kept_objects(value: &PyObject, vm: &VirtualMachine) -> PyObjectRef { + value + .downcast_ref::<PyCData>() + .and_then(|cdata| cdata.objects.read().clone()) + .unwrap_or_else(|| vm.ctx.new_dict().into()) + } + + /// Check if a value should be stored in _objects + /// Returns true for ctypes objects and bytes (for c_char_p) + pub fn should_keep_ref(value: &PyObject) -> bool { + value.downcast_ref::<PyCData>().is_some() || value.downcast_ref::<PyBytes>().is_some() + } + + /// PyCData_set + /// Sets a field value at the given offset, handling type conversion and KeepRef + #[allow(clippy::too_many_arguments)] + pub fn set_field( + &self, + proto: &PyObject, + value: PyObjectRef, + index: usize, + size: usize, + offset: usize, + needs_swap: bool, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if this is a c_char or c_wchar array field + let is_char_array = PyCField::is_char_array(proto, vm); + let is_wchar_array = PyCField::is_wchar_array(proto, vm); + + // For c_char arrays with bytes input, copy only up to first null + if is_char_array { + if let Some(bytes_val) = value.downcast_ref::<PyBytes>() { + let src = bytes_val.as_bytes(); + let to_copy = PyCField::bytes_for_char_array(src); + let copy_len = core::cmp::min(to_copy.len(), size); + self.write_bytes_at_offset(offset, &to_copy[..copy_len]); + self.keep_ref(index, value, vm)?; + return Ok(()); + } else { + return Err(vm.new_type_error("bytes expected instead of str instance")); + } + } + + // For c_wchar arrays with str input, convert to wchar_t + if is_wchar_array { + if let Some(str_val) = value.downcast_ref::<PyStr>() { + // Convert str to wchar_t bytes (platform-dependent size) + let mut wchar_bytes = Vec::with_capacity(size); + for cp in str_val.as_wtf8().code_points().take(size / WCHAR_SIZE) { + let mut bytes = [0u8; 4]; + wchar_to_bytes(cp.to_u32(), &mut bytes); + wchar_bytes.extend_from_slice(&bytes[..WCHAR_SIZE]); + } + // Pad with nulls to fill the array + while wchar_bytes.len() < size { + wchar_bytes.push(0); + } + self.write_bytes_at_offset(offset, &wchar_bytes); + self.keep_ref(index, value, vm)?; + return Ok(()); + } else if value.downcast_ref::<PyBytes>().is_some() { + return Err(vm.new_type_error("str expected instead of bytes instance")); + } + } + + // Special handling for Pointer fields with Array values + if let Some(proto_type) = proto.downcast_ref::<PyType>() + && proto_type + .class() + .fast_issubclass(super::pointer::PyCPointerType::static_type()) + && let Some(array) = value.downcast_ref::<super::array::PyCArray>() + { + let buffer_addr = { + let array_buffer = array.0.buffer.read(); + array_buffer.as_ptr() as usize + }; + let addr_bytes = buffer_addr.to_ne_bytes(); + let len = core::cmp::min(addr_bytes.len(), size); + self.write_bytes_at_offset(offset, &addr_bytes[..len]); + self.keep_ref(index, value, vm)?; + return Ok(()); + } + + // For array fields with tuple/list input, instantiate the array type + // and unpack elements as positional args (Array_init expects *args) + if let Some(proto_type) = proto.downcast_ref::<PyType>() + && let Some(stg) = proto_type.stg_info_opt() + && stg.element_type.is_some() + { + let items: Option<Vec<PyObjectRef>> = + if let Some(tuple) = value.downcast_ref::<PyTuple>() { + Some(tuple.to_vec()) + } else { + value + .downcast_ref::<crate::builtins::PyList>() + .map(|list| list.borrow_vec().to_vec()) + }; + if let Some(items) = items { + let array_obj = proto_type.as_object().call(items, vm).map_err(|e| { + // Wrap errors in RuntimeError with type name prefix + let type_name = proto_type.name().to_string(); + let exc_name = e.class().name().to_string(); + let exc_args = e.args(); + let exc_msg = exc_args + .first() + .and_then(|a| a.downcast_ref::<PyStr>().map(|s| s.to_string())) + .unwrap_or_default(); + vm.new_runtime_error(format!("({type_name}) {exc_name}: {exc_msg}")) + })?; + if let Some(arr) = array_obj.downcast_ref::<super::array::PyCArray>() { + let arr_buffer = arr.0.buffer.read(); + let len = core::cmp::min(arr_buffer.len(), size); + self.write_bytes_at_offset(offset, &arr_buffer[..len]); + drop(arr_buffer); + self.keep_ref(index, array_obj, vm)?; + return Ok(()); + } + } + } + + // Get field type code for special handling + let field_type_code = proto + .get_attr("_type_", vm) + .ok() + .and_then(|attr| attr.downcast_ref::<PyStr>().map(|s| s.to_string())); + + // c_char_p (z type) with bytes: store original in _objects, keep + // null-terminated copy alive separately for the pointer. + if field_type_code.as_deref() == Some("z") + && let Some(bytes_val) = value.downcast_ref::<PyBytes>() + { + let (kept_alive, ptr) = ensure_z_null_terminated(bytes_val, vm); + let mut result = vec![0u8; size]; + let addr_bytes = ptr.to_ne_bytes(); + let len = core::cmp::min(addr_bytes.len(), size); + result[..len].copy_from_slice(&addr_bytes[..len]); + if needs_swap { + result.reverse(); + } + self.write_bytes_at_offset(offset, &result); + self.keep_ref(index, value, vm)?; + self.keep_alive(index, kept_alive); + return Ok(()); + } + + let (mut bytes, converted_value) = if let Some(type_code) = &field_type_code { + PyCField::value_to_bytes_for_type(type_code, &value, size, vm)? + } else { + (PyCField::value_to_bytes(&value, size, vm)?, None) + }; + + // Swap bytes for opposite endianness + if needs_swap { + bytes.reverse(); + } + + self.write_bytes_at_offset(offset, &bytes); + + // KeepRef: for z/Z types use converted value, otherwise use original + if let Some(converted) = converted_value { + self.keep_ref(index, converted, vm)?; + } else if Self::should_keep_ref(&value) { + let to_keep = Self::get_kept_objects(&value, vm); + self.keep_ref(index, to_keep, vm)?; + } + + Ok(()) + } + + /// PyCData_get + /// Gets a field value at the given offset + pub fn get_field( + &self, + proto: &PyObject, + index: usize, + size: usize, + offset: usize, + base_obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // Get buffer data at offset + let buffer = self.buffer.read(); + if offset + size > buffer.len() { + return Ok(vm.ctx.new_int(0).into()); + } + + // Check if field type is an array type + if let Some(type_ref) = proto.downcast_ref::<PyType>() + && let Some(stg) = type_ref.stg_info_opt() + && stg.element_type.is_some() + { + // c_char array → return bytes + if PyCField::is_char_array(proto, vm) { + let data = &buffer[offset..offset + size]; + // Find first null terminator (or use full length) + let end = data.iter().position(|&b| b == 0).unwrap_or(data.len()); + return Ok(vm.ctx.new_bytes(data[..end].to_vec()).into()); + } + + // c_wchar array → return str + if PyCField::is_wchar_array(proto, vm) { + let data = &buffer[offset..offset + size]; + // wchar_t → char conversion, skip null + let chars: String = data + .chunks(WCHAR_SIZE) + .filter_map(|chunk| { + wchar_from_bytes(chunk) + .filter(|&wchar| wchar != 0) + .and_then(char::from_u32) + }) + .collect(); + return Ok(vm.ctx.new_str(chars).into()); + } + + // Other array types - create array with a copy of data from the base's buffer + // The array also keeps a reference to the base for keeping it alive and for writes + let array_data = buffer[offset..offset + size].to_vec(); + drop(buffer); + + let cdata_obj = + Self::from_base_with_data(base_obj, offset, index, stg.length, array_data); + let array_type: PyTypeRef = proto + .to_owned() + .downcast() + .map_err(|_| vm.new_type_error("expected array type"))?; + + return super::array::PyCArray(cdata_obj) + .into_ref_with_type(vm, array_type) + .map(Into::into); + } + + let buffer_data = buffer[offset..offset + size].to_vec(); + drop(buffer); + + // Get proto as type + let proto_type: PyTypeRef = proto + .to_owned() + .downcast() + .map_err(|_| vm.new_type_error("field proto is not a type"))?; + + let proto_metaclass = proto_type.class(); + + // Simple types: return primitive value + if proto_metaclass.fast_issubclass(super::simple::PyCSimpleType::static_type()) { + // Check for byte swapping + let needs_swap = base_obj + .class() + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok() + || proto_type + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok(); + + let data = if needs_swap && size > 1 { + let mut swapped = buffer_data.clone(); + swapped.reverse(); + swapped + } else { + buffer_data + }; + + return bytes_to_pyobject(&proto_type, &data, vm); + } + + // Complex types: create ctypes instance via PyCData_FromBaseObj + let ptr = self.buffer.read().as_ptr().wrapping_add(offset) as *mut u8; + let cdata_obj = unsafe { Self::from_base_obj(ptr, size, base_obj.clone(), index) }; + + if proto_metaclass.fast_issubclass(super::structure::PyCStructType::static_type()) + || proto_metaclass.fast_issubclass(super::union::PyCUnionType::static_type()) + || proto_metaclass.fast_issubclass(super::pointer::PyCPointerType::static_type()) + { + cdata_obj.into_ref_with_type(vm, proto_type).map(Into::into) + } else { + // Fallback + Ok(vm.ctx.new_int(0).into()) + } + } +} + +#[pyclass(flags(BASETYPE))] +impl PyCData { + #[pygetset] + fn _objects(&self) -> Option<PyObjectRef> { + self.objects.read().clone() + } + + #[pygetset] + fn _b_base_(&self) -> Option<PyObjectRef> { + self.base.read().clone() + } + + #[pygetset] + fn _b_needsfree_(&self) -> i32 { + // Borrowed (from_address) or has base object → 0 (don't free) + // Owned and no base → 1 (need to free) + if self.is_borrowed() || self.base.read().is_some() { + 0 + } else { + 1 + } + } + + // CDataType_methods - shared across all ctypes types + + #[pyclassmethod] + pub(super) fn from_buffer( + cls: PyTypeRef, + source: PyObjectRef, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cdata = Self::from_buffer_impl(&cls, source, offset.unwrap_or(0), vm)?; + cdata.into_ref_with_type(vm, cls).map(Into::into) + } + + #[pyclassmethod] + pub(super) fn from_buffer_copy( + cls: PyTypeRef, + source: ArgBytesLike, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cdata = + Self::from_buffer_copy_impl(&cls, &source.borrow_buf(), offset.unwrap_or(0), vm)?; + cdata.into_ref_with_type(vm, cls).map(Into::into) + } + + #[pyclassmethod] + pub(super) fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { + let size = { + let stg_info = cls.stg_info(vm)?; + stg_info.size + }; + + if size == 0 { + return Err(vm.new_type_error("abstract class")); + } + + // PyCData_AtAddress + let cdata = unsafe { Self::at_address(address as *const u8, size) }; + cdata.into_ref_with_type(vm, cls).map(Into::into) + } + + #[pyclassmethod] + pub(super) fn in_dll( + cls: PyTypeRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let size = { + let stg_info = cls.stg_info(vm)?; + stg_info.size + }; + + if size == 0 { + return Err(vm.new_type_error("abstract class")); + } + + // Get the library handle from dll object + let handle = if let Ok(int_handle) = dll.try_int(vm) { + int_handle + .as_bigint() + .to_usize() + .ok_or_else(|| vm.new_value_error("Invalid library handle"))? + } else { + dll.get_attr("_handle", vm)? + .try_int(vm)? + .as_bigint() + .to_usize() + .ok_or_else(|| vm.new_value_error("Invalid library handle"))? + }; + + // Look up the library in the cache and use lib.get() for symbol lookup + let library_cache = super::library::libcache().read(); + let library = library_cache + .get_lib(handle) + .ok_or_else(|| vm.new_value_error("Library not found"))?; + let inner_lib = library.lib.lock(); + + let symbol_name_with_nul = format!("{}\0", name.as_wtf8()); + let ptr: *const u8 = if let Some(lib) = &*inner_lib { + unsafe { + lib.get::<*const u8>(symbol_name_with_nul.as_bytes()) + .map(|sym| *sym) + .map_err(|_| { + vm.new_value_error(format!("symbol '{}' not found", name.as_wtf8())) + })? + } + } else { + return Err(vm.new_value_error("Library closed")); + }; + + // dlsym can return NULL for symbols that resolve to NULL (e.g., GNU IFUNC) + // Treat NULL addresses as errors + if ptr.is_null() { + return Err(vm.new_value_error(format!("symbol '{}' not found", name.as_wtf8()))); + } + + // PyCData_AtAddress + let cdata = unsafe { Self::at_address(ptr, size) }; + cdata.into_ref_with_type(vm, cls).map(Into::into) + } +} + +// PyCField - Field descriptor for Structure/Union types + +/// CField descriptor for Structure/Union field access +#[pyclass(name = "CField", module = "_ctypes")] +#[derive(Debug, PyPayload)] +pub struct PyCField { + /// Field name + pub(crate) name: String, + /// Byte offset of the field within the structure/union + pub(crate) offset: isize, + /// Byte size of the underlying type + pub(crate) byte_size_val: isize, + /// Index into PyCData's object array + pub(crate) index: usize, + /// The ctypes type for this field + pub(crate) proto: PyTypeRef, + /// Flag indicating if the field is anonymous (MakeAnonFields sets this) + pub(crate) anonymous: bool, + /// Bitfield size in bits (0 for non-bitfield) + pub(crate) bitfield_size: u16, + /// Bit offset within the storage unit (only meaningful for bitfields) + pub(crate) bit_offset_val: u16, +} + +impl PyCField { + /// Create a new CField descriptor (non-bitfield) + pub fn new( + name: String, + proto: PyTypeRef, + offset: isize, + byte_size: isize, + index: usize, + ) -> Self { + Self { + name, + offset, + byte_size_val: byte_size, + index, + proto, + anonymous: false, + bitfield_size: 0, + bit_offset_val: 0, + } + } + + /// Create a new CField descriptor for a bitfield + pub fn new_bitfield( + name: String, + proto: PyTypeRef, + offset: isize, + byte_size: isize, + bitfield_size: u16, + bit_offset: u16, + index: usize, + ) -> Self { + Self { + name, + offset, + byte_size_val: byte_size, + index, + proto, + anonymous: false, + bitfield_size, + bit_offset_val: bit_offset, + } + } + + /// Get the byte size of the field's underlying type + pub fn get_byte_size(&self) -> usize { + self.byte_size_val as usize + } + + /// Create a new CField from an existing field with adjusted offset and index + /// Used by MakeFields to promote anonymous fields + pub fn new_from_field(fdescr: &PyCField, index_offset: usize, offset_delta: isize) -> Self { + Self { + name: fdescr.name.clone(), + offset: fdescr.offset + offset_delta, + byte_size_val: fdescr.byte_size_val, + index: fdescr.index + index_offset, + proto: fdescr.proto.clone(), + anonymous: false, // promoted fields are not anonymous themselves + bitfield_size: fdescr.bitfield_size, + bit_offset_val: fdescr.bit_offset_val, + } + } + + /// Set anonymous flag + pub fn set_anonymous(&mut self, anonymous: bool) { + self.anonymous = anonymous; + } +} + +impl Constructor for PyCField { + type Args = crate::function::FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + // PyCField_new_impl: requires _internal_use=True + let internal_use = if let Some(v) = args.kwargs.get("_internal_use") { + v.clone().try_to_bool(vm)? + } else { + false + }; + + if !internal_use { + return Err(vm.new_type_error( + "CField is not intended to be used directly; use it via Structure or Union fields", + )); + } + + let name: String = args + .kwargs + .get("name") + .ok_or_else(|| vm.new_type_error("missing required argument: 'name'"))? + .try_to_value(vm)?; + + let field_type: PyTypeRef = args + .kwargs + .get("type") + .ok_or_else(|| vm.new_type_error("missing required argument: 'type'"))? + .clone() + .downcast() + .map_err(|_| vm.new_type_error("'type' must be a ctypes type"))?; + + let byte_size: isize = args + .kwargs + .get("byte_size") + .ok_or_else(|| vm.new_type_error("missing required argument: 'byte_size'"))? + .try_to_value(vm)?; + + let byte_offset: isize = args + .kwargs + .get("byte_offset") + .ok_or_else(|| vm.new_type_error("missing required argument: 'byte_offset'"))? + .try_to_value(vm)?; + + let index: usize = args + .kwargs + .get("index") + .ok_or_else(|| vm.new_type_error("missing required argument: 'index'"))? + .try_to_value(vm)?; + + // Validate byte_size matches the type + let type_size = super::base::get_field_size(field_type.as_object(), vm)? as isize; + if byte_size != type_size { + return Err(vm.new_value_error(format!( + "byte_size {} does not match type size {}", + byte_size, type_size + ))); + } + + let bit_size_val: Option<isize> = args + .kwargs + .get("bit_size") + .map(|v| v.try_to_value(vm)) + .transpose()?; + + let bit_offset_val: Option<isize> = args + .kwargs + .get("bit_offset") + .map(|v| v.try_to_value(vm)) + .transpose()?; + + if let Some(bs) = bit_size_val { + if bs < 0 { + return Err(vm.new_value_error("number of bits invalid for bit field")); + } + let bo = bit_offset_val.unwrap_or(0); + if bo < 0 { + return Err(vm.new_value_error("bit_offset must be >= 0")); + } + let type_bits = byte_size * 8; + if bo + bs > type_bits { + return Err(vm.new_value_error(format!( + "bit field '{}' overflows its type ({} + {} > {})", + name, bo, bs, type_bits + ))); + } + Ok(Self::new_bitfield( + name, + field_type, + byte_offset, + byte_size, + bs as u16, + bo as u16, + index, + )) + } else { + Ok(Self::new(name, field_type, byte_offset, byte_size, index)) + } + } +} + +impl Representable for PyCField { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + // Get type name from proto (which is always PyTypeRef) + let tp_name = zelf.proto.name().to_string(); + + // Bitfield: <Field type=TYPE, ofs=OFFSET:BIT_OFFSET, bits=NUM_BITS> + // Regular: <Field type=TYPE, ofs=OFFSET, size=SIZE> + if zelf.bitfield_size > 0 { + Ok(format!( + "<Field type={}, ofs={}:{}, bits={}>", + tp_name, zelf.offset, zelf.bit_offset_val, zelf.bitfield_size + )) + } else { + Ok(format!( + "<Field type={}, ofs={}, size={}>", + tp_name, zelf.offset, zelf.byte_size_val + )) + } + } +} + +/// PyCField_get +impl GetDescriptor for PyCField { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let zelf = zelf + .downcast::<PyCField>() + .map_err(|_| vm.new_type_error("expected CField"))?; + + // If obj is None, return the descriptor itself (class attribute access) + let obj = match obj { + Some(obj) if !vm.is_none(&obj) => obj, + _ => return Ok(zelf.into()), + }; + + let offset = zelf.offset as usize; + let size = zelf.get_byte_size(); + + // Get PyCData from obj (works for both Structure and Union) + let cdata = PyCField::get_cdata_from_obj(&obj, vm)?; + + // PyCData_get + cdata.get_field( + zelf.proto.as_object(), + zelf.index, + size, + offset, + obj.clone(), + vm, + ) + } +} + +impl PyCField { + /// Convert a Python value to bytes + fn value_to_bytes(value: &PyObject, size: usize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + // 1. Handle bytes objects + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let src = bytes.as_bytes(); + let mut result = vec![0u8; size]; + let len = core::cmp::min(src.len(), size); + result[..len].copy_from_slice(&src[..len]); + Ok(result) + } + // 2. Handle ctypes array instances (copy their buffer) + else if let Some(cdata) = value.downcast_ref::<super::PyCData>() { + let buffer = cdata.buffer.read(); + let mut result = vec![0u8; size]; + let len = core::cmp::min(buffer.len(), size); + result[..len].copy_from_slice(&buffer[..len]); + Ok(result) + } + // 4. Handle float values (check before int, since float.try_int would truncate) + else if let Some(float_val) = value.downcast_ref::<crate::builtins::PyFloat>() { + let f = float_val.to_f64(); + match size { + 4 => { + let val = f as f32; + Ok(val.to_ne_bytes().to_vec()) + } + 8 => Ok(f.to_ne_bytes().to_vec()), + _ => unreachable!("wrong payload size"), + } + } + // 4. Handle integer values + else if let Ok(int_val) = value.try_int(vm) { + let i = int_val.as_bigint(); + match size { + 1 => { + let val = i.to_i8().unwrap_or(0); + Ok(val.to_ne_bytes().to_vec()) + } + 2 => { + let val = i.to_i16().unwrap_or(0); + Ok(val.to_ne_bytes().to_vec()) + } + 4 => { + let val = i.to_i32().unwrap_or(0); + Ok(val.to_ne_bytes().to_vec()) + } + 8 => { + let val = i.to_i64().unwrap_or(0); + Ok(val.to_ne_bytes().to_vec()) + } + _ => Ok(vec![0u8; size]), + } + } else { + Ok(vec![0u8; size]) + } + } + + /// Convert a Python value to bytes with type-specific handling for pointer types. + /// Returns (bytes, optional holder for wchar buffer). + fn value_to_bytes_for_type( + type_code: &str, + value: &PyObject, + size: usize, + vm: &VirtualMachine, + ) -> PyResult<(Vec<u8>, Option<PyObjectRef>)> { + match type_code { + // c_float: always convert to float first (f_set) + "f" => { + let f = if let Some(float_val) = value.downcast_ref::<crate::builtins::PyFloat>() { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_i64().unwrap_or(0) as f64 + } else { + return Err(vm.new_type_error(format!( + "float expected instead of {}", + value.class().name() + ))); + }; + let val = f as f32; + Ok((val.to_ne_bytes().to_vec(), None)) + } + // c_double: always convert to float first (d_set) + "d" => { + let f = if let Some(float_val) = value.downcast_ref::<crate::builtins::PyFloat>() { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_i64().unwrap_or(0) as f64 + } else { + return Err(vm.new_type_error(format!( + "float expected instead of {}", + value.class().name() + ))); + }; + Ok((f.to_ne_bytes().to_vec(), None)) + } + // c_longdouble: convert to float (treated as f64 in RustPython) + "g" => { + let f = if let Some(float_val) = value.downcast_ref::<crate::builtins::PyFloat>() { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_i64().unwrap_or(0) as f64 + } else { + return Err(vm.new_type_error(format!( + "float expected instead of {}", + value.class().name() + ))); + }; + Ok((f.to_ne_bytes().to_vec(), None)) + } + "z" => { + // c_char_p with bytes is handled in set_field before this call. + // This handles integer address and None cases. + // Integer address + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_usize().unwrap_or(0); + let mut result = vec![0u8; size]; + let bytes = v.to_ne_bytes(); + let len = core::cmp::min(bytes.len(), size); + result[..len].copy_from_slice(&bytes[..len]); + return Ok((result, None)); + } + // None -> NULL pointer + if vm.is_none(value) { + return Ok((vec![0u8; size], None)); + } + Ok((PyCField::value_to_bytes(value, size, vm)?, None)) + } + "Z" => { + // c_wchar_p: store pointer to null-terminated wchar_t buffer + if let Some(s) = value.downcast_ref::<PyStr>() { + let (holder, ptr) = str_to_wchar_bytes(s.as_wtf8(), vm); + let mut result = vec![0u8; size]; + let addr_bytes = ptr.to_ne_bytes(); + let len = core::cmp::min(addr_bytes.len(), size); + result[..len].copy_from_slice(&addr_bytes[..len]); + return Ok((result, Some(holder))); + } + // Integer address + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_usize().unwrap_or(0); + let mut result = vec![0u8; size]; + let bytes = v.to_ne_bytes(); + let len = core::cmp::min(bytes.len(), size); + result[..len].copy_from_slice(&bytes[..len]); + return Ok((result, None)); + } + // None -> NULL pointer + if vm.is_none(value) { + return Ok((vec![0u8; size], None)); + } + Ok((PyCField::value_to_bytes(value, size, vm)?, None)) + } + "P" => { + // c_void_p: store integer as pointer + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_usize().unwrap_or(0); + let mut result = vec![0u8; size]; + let bytes = v.to_ne_bytes(); + let len = core::cmp::min(bytes.len(), size); + result[..len].copy_from_slice(&bytes[..len]); + return Ok((result, None)); + } + // None -> NULL pointer + if vm.is_none(value) { + return Ok((vec![0u8; size], None)); + } + Ok((PyCField::value_to_bytes(value, size, vm)?, None)) + } + _ => Ok((PyCField::value_to_bytes(value, size, vm)?, None)), + } + } + + /// Check if the field type is a c_char array (element type has _type_ == 'c') + fn is_char_array(proto: &PyObject, vm: &VirtualMachine) -> bool { + // Get element_type from StgInfo (for array types) + if let Some(proto_type) = proto.downcast_ref::<PyType>() + && let Some(stg) = proto_type.stg_info_opt() + && let Some(element_type) = &stg.element_type + { + // Check if element type has _type_ == "c" + if let Ok(type_code) = element_type.as_object().get_attr("_type_", vm) + && let Some(s) = type_code.downcast_ref::<PyStr>() + { + return s.as_bytes() == b"c"; + } + } + false + } + + /// Check if the field type is a c_wchar array (element type has _type_ == 'u') + fn is_wchar_array(proto: &PyObject, vm: &VirtualMachine) -> bool { + // Get element_type from StgInfo (for array types) + if let Some(proto_type) = proto.downcast_ref::<PyType>() + && let Some(stg) = proto_type.stg_info_opt() + && let Some(element_type) = &stg.element_type + { + // Check if element type has _type_ == "u" + if let Ok(type_code) = element_type.as_object().get_attr("_type_", vm) + && let Some(s) = type_code.downcast_ref::<PyStr>() + { + return s.as_bytes() == b"u"; + } + } + false + } + + /// Convert bytes for c_char array assignment (stops at first null terminator) + /// Returns (bytes_to_copy, copy_len) + fn bytes_for_char_array(src: &[u8]) -> &[u8] { + // Find first null terminator and include it + if let Some(null_pos) = src.iter().position(|&b| b == 0) { + &src[..=null_pos] + } else { + src + } + } +} + +#[pyclass(flags(IMMUTABLETYPE), with(Representable, GetDescriptor, Constructor))] +impl PyCField { + /// Get PyCData from object (works for both Structure and Union) + fn get_cdata_from_obj<'a>(obj: &'a PyObjectRef, vm: &VirtualMachine) -> PyResult<&'a PyCData> { + if let Some(s) = obj.downcast_ref::<super::structure::PyCStructure>() { + Ok(&s.0) + } else if let Some(u) = obj.downcast_ref::<super::union::PyCUnion>() { + Ok(&u.0) + } else { + Err(vm.new_type_error(format!( + "descriptor works only on Structure or Union instances, got {}", + obj.class().name() + ))) + } + } + + /// PyCField_set + #[pyslot] + fn descr_set( + zelf: &PyObject, + obj: PyObjectRef, + value: PySetterValue<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let zelf = zelf + .downcast_ref::<PyCField>() + .ok_or_else(|| vm.new_type_error("expected CField"))?; + + let offset = zelf.offset as usize; + let size = zelf.get_byte_size(); + + // Get PyCData from obj (works for both Structure and Union) + let cdata = Self::get_cdata_from_obj(&obj, vm)?; + + match value { + PySetterValue::Assign(value) => { + // Check if needs byte swapping + let needs_swap = (obj + .class() + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok() + || zelf + .proto + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok()) + && size > 1; + + // PyCData_set + cdata.set_field( + zelf.proto.as_object(), + value, + zelf.index, + size, + offset, + needs_swap, + vm, + ) + } + PySetterValue::Delete => Err(vm.new_type_error("cannot delete field")), + } + } + + #[pygetset] + fn name(&self) -> String { + self.name.clone() + } + + #[pygetset(name = "type")] + fn type_(&self) -> PyTypeRef { + self.proto.clone() + } + + #[pygetset] + fn offset(&self) -> isize { + self.offset + } + + #[pygetset] + fn byte_offset(&self) -> isize { + self.offset + } + + #[pygetset] + fn size(&self) -> isize { + // Legacy: encode as (bitfield_size << 16) | bit_offset for bitfields + if self.bitfield_size > 0 { + ((self.bitfield_size as isize) << 16) | (self.bit_offset_val as isize) + } else { + self.byte_size_val + } + } + + #[pygetset] + fn byte_size(&self) -> isize { + self.byte_size_val + } + + #[pygetset] + fn bit_offset(&self) -> isize { + self.bit_offset_val as isize + } + + #[pygetset] + fn bit_size(&self, vm: &VirtualMachine) -> PyObjectRef { + if self.bitfield_size > 0 { + vm.ctx.new_int(self.bitfield_size).into() + } else { + // Non-bitfield: bit_size = byte_size * 8 + let byte_size = self.byte_size_val as i128; + vm.ctx.new_int(byte_size * 8).into() + } + } + + #[pygetset] + fn is_bitfield(&self) -> bool { + self.bitfield_size > 0 + } + + #[pygetset] + fn is_anonymous(&self) -> bool { + self.anonymous + } +} + +// ParamFunc implementations (PyCArgObject creation) + +use super::_ctypes::CArgObject; + +/// Call the appropriate paramfunc based on StgInfo.paramfunc +/// info->paramfunc(st, obj) +pub(super) fn call_paramfunc(obj: &PyObject, vm: &VirtualMachine) -> PyResult<CArgObject> { + let cls = obj.class(); + let stg_info = cls + .stg_info_opt() + .ok_or_else(|| vm.new_type_error("not a ctypes type"))?; + + match stg_info.paramfunc { + ParamFunc::Simple => simple_paramfunc(obj, vm), + ParamFunc::Array => array_paramfunc(obj, vm), + ParamFunc::Pointer => pointer_paramfunc(obj, vm), + ParamFunc::Structure | ParamFunc::Union => struct_union_paramfunc(obj, &stg_info, vm), + ParamFunc::None => Err(vm.new_type_error("no paramfunc")), + } +} + +/// PyCSimpleType_paramfunc +fn simple_paramfunc(obj: &PyObject, vm: &VirtualMachine) -> PyResult<CArgObject> { + use super::simple::PyCSimple; + + let simple = obj + .downcast_ref::<PyCSimple>() + .ok_or_else(|| vm.new_type_error("expected simple type"))?; + + // Get type code from _type_ attribute + let cls = obj.class().to_owned(); + let type_code = cls + .type_code(vm) + .ok_or_else(|| vm.new_type_error("no _type_ attribute"))?; + let tag = type_code.as_bytes().first().copied().unwrap_or(b'?'); + + // Read value from buffer: memcpy(&parg->value, self->b_ptr, self->b_size) + let buffer = simple.0.buffer.read(); + let ffi_value = buffer_to_ffi_value(&type_code, &buffer); + + Ok(CArgObject { + tag, + value: ffi_value, + obj: obj.to_owned(), + size: 0, + offset: 0, + }) +} + +/// PyCArrayType_paramfunc +fn array_paramfunc(obj: &PyObject, vm: &VirtualMachine) -> PyResult<CArgObject> { + use super::array::PyCArray; + + let array = obj + .downcast_ref::<PyCArray>() + .ok_or_else(|| vm.new_type_error("expected array"))?; + + // p->value.p = (char *)self->b_ptr + let buffer = array.0.buffer.read(); + let ptr_val = buffer.as_ptr() as usize; + + Ok(CArgObject { + tag: b'P', + value: FfiArgValue::Pointer(ptr_val), + obj: obj.to_owned(), + size: 0, + offset: 0, + }) +} + +/// PyCPointerType_paramfunc +fn pointer_paramfunc(obj: &PyObject, vm: &VirtualMachine) -> PyResult<CArgObject> { + use super::pointer::PyCPointer; + + let ptr = obj + .downcast_ref::<PyCPointer>() + .ok_or_else(|| vm.new_type_error("expected pointer"))?; + + // parg->value.p = *(void **)self->b_ptr + let ptr_val = ptr.get_ptr_value(); + + Ok(CArgObject { + tag: b'P', + value: FfiArgValue::Pointer(ptr_val), + obj: obj.to_owned(), + size: 0, + offset: 0, + }) +} + +/// StructUnionType_paramfunc (for both Structure and Union) +fn struct_union_paramfunc( + obj: &PyObject, + stg_info: &StgInfo, + _vm: &VirtualMachine, +) -> PyResult<CArgObject> { + // Get buffer pointer + // For large structs (> sizeof(void*)), we'd need to allocate and copy. + // For now, just point to buffer directly and keep obj reference for memory safety. + let buffer = if let Some(cdata) = obj.downcast_ref::<PyCData>() { + cdata.buffer.read() + } else { + return Ok(CArgObject { + tag: b'V', + value: FfiArgValue::Pointer(0), + obj: obj.to_owned(), + size: stg_info.size, + offset: 0, + }); + }; + + let ptr_val = buffer.as_ptr() as usize; + let size = buffer.len(); + + Ok(CArgObject { + tag: b'V', + value: FfiArgValue::Pointer(ptr_val), + obj: obj.to_owned(), + size, + offset: 0, + }) +} + +// FfiArgValue - Owned FFI argument value + +/// Owned FFI argument value. Keeps the value alive for the duration of the FFI call. +#[derive(Debug, Clone)] +pub enum FfiArgValue { + U8(u8), + I8(i8), + U16(u16), + I16(i16), + U32(u32), + I32(i32), + U64(u64), + I64(i64), + F32(f32), + F64(f64), + Pointer(usize), + /// Pointer with owned data. The PyObjectRef keeps the pointed data alive. + OwnedPointer(usize, #[allow(dead_code)] PyObjectRef), +} + +impl FfiArgValue { + /// Create an Arg reference to this owned value + pub fn as_arg(&self) -> libffi::middle::Arg<'_> { + match self { + FfiArgValue::U8(v) => libffi::middle::Arg::new(v), + FfiArgValue::I8(v) => libffi::middle::Arg::new(v), + FfiArgValue::U16(v) => libffi::middle::Arg::new(v), + FfiArgValue::I16(v) => libffi::middle::Arg::new(v), + FfiArgValue::U32(v) => libffi::middle::Arg::new(v), + FfiArgValue::I32(v) => libffi::middle::Arg::new(v), + FfiArgValue::U64(v) => libffi::middle::Arg::new(v), + FfiArgValue::I64(v) => libffi::middle::Arg::new(v), + FfiArgValue::F32(v) => libffi::middle::Arg::new(v), + FfiArgValue::F64(v) => libffi::middle::Arg::new(v), + FfiArgValue::Pointer(v) => libffi::middle::Arg::new(v), + FfiArgValue::OwnedPointer(v, _) => libffi::middle::Arg::new(v), + } + } +} + +/// Convert buffer bytes to FfiArgValue based on type code +pub(super) fn buffer_to_ffi_value(type_code: &str, buffer: &[u8]) -> FfiArgValue { + match type_code { + "c" | "b" => { + let v = buffer.first().map(|&b| b as i8).unwrap_or(0); + FfiArgValue::I8(v) + } + "B" => { + let v = buffer.first().copied().unwrap_or(0); + FfiArgValue::U8(v) + } + "h" => { + let v = buffer.first_chunk().copied().map_or(0, i16::from_ne_bytes); + FfiArgValue::I16(v) + } + "H" => { + let v = buffer.first_chunk().copied().map_or(0, u16::from_ne_bytes); + FfiArgValue::U16(v) + } + "i" => { + let v = buffer.first_chunk().copied().map_or(0, i32::from_ne_bytes); + FfiArgValue::I32(v) + } + "I" => { + let v = buffer.first_chunk().copied().map_or(0, u32::from_ne_bytes); + FfiArgValue::U32(v) + } + "l" | "q" => { + let v = if let Some(&bytes) = buffer.first_chunk::<8>() { + i64::from_ne_bytes(bytes) + } else if let Some(&bytes) = buffer.first_chunk::<4>() { + i32::from_ne_bytes(bytes).into() + } else { + 0 + }; + FfiArgValue::I64(v) + } + "L" | "Q" => { + let v = if let Some(&bytes) = buffer.first_chunk::<8>() { + u64::from_ne_bytes(bytes) + } else if let Some(&bytes) = buffer.first_chunk::<4>() { + u32::from_ne_bytes(bytes).into() + } else { + 0 + }; + FfiArgValue::U64(v) + } + "f" => { + let v = buffer + .first_chunk::<4>() + .copied() + .map_or(0.0, f32::from_ne_bytes); + FfiArgValue::F32(v) + } + "d" | "g" => { + let v = buffer + .first_chunk::<8>() + .copied() + .map_or(0.0, f64::from_ne_bytes); + FfiArgValue::F64(v) + } + "z" | "Z" | "P" | "O" => FfiArgValue::Pointer(read_ptr_from_buffer(buffer)), + "?" => { + let v = buffer.first().map(|&b| b != 0).unwrap_or(false); + FfiArgValue::U8(if v { 1 } else { 0 }) + } + "u" => { + // wchar_t - 4 bytes on most platforms + let v = buffer.first_chunk().copied().map_or(0, u32::from_ne_bytes); + FfiArgValue::U32(v) + } + _ => FfiArgValue::Pointer(0), + } +} + +/// Convert bytes to appropriate Python object based on ctypes type +pub(super) fn bytes_to_pyobject( + cls: &Py<PyType>, + bytes: &[u8], + vm: &VirtualMachine, +) -> PyResult<PyObjectRef> { + // Try to get _type_ attribute + if let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) + && let Ok(s) = type_attr.str(vm) + { + let ty = s.to_string(); + return match ty.as_str() { + "c" => Ok(vm.ctx.new_bytes(bytes.to_vec()).into()), + "b" => { + let val = if !bytes.is_empty() { bytes[0] as i8 } else { 0 }; + Ok(vm.ctx.new_int(val).into()) + } + "B" => { + let val = if !bytes.is_empty() { bytes[0] } else { 0 }; + Ok(vm.ctx.new_int(val).into()) + } + "h" => { + const SIZE: usize = mem::size_of::<c_short>(); + let val = if bytes.len() >= SIZE { + c_short::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "H" => { + const SIZE: usize = mem::size_of::<c_ushort>(); + let val = if bytes.len() >= SIZE { + c_ushort::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "i" => { + const SIZE: usize = mem::size_of::<c_int>(); + let val = if bytes.len() >= SIZE { + c_int::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "I" => { + const SIZE: usize = mem::size_of::<c_uint>(); + let val = if bytes.len() >= SIZE { + c_uint::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "l" => { + const SIZE: usize = mem::size_of::<c_long>(); + let val = if bytes.len() >= SIZE { + c_long::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "L" => { + const SIZE: usize = mem::size_of::<c_ulong>(); + let val = if bytes.len() >= SIZE { + c_ulong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "q" => { + const SIZE: usize = mem::size_of::<c_longlong>(); + let val = if bytes.len() >= SIZE { + c_longlong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "Q" => { + const SIZE: usize = mem::size_of::<c_ulonglong>(); + let val = if bytes.len() >= SIZE { + c_ulonglong::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_int(val).into()) + } + "f" => { + const SIZE: usize = mem::size_of::<c_float>(); + let val = if bytes.len() >= SIZE { + c_float::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0.0 + }; + Ok(vm.ctx.new_float(val as f64).into()) + } + "d" => { + const SIZE: usize = mem::size_of::<c_double>(); + let val = if bytes.len() >= SIZE { + c_double::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0.0 + }; + Ok(vm.ctx.new_float(val).into()) + } + "g" => { + // long double - read as f64 for now since Rust doesn't have native long double + // This may lose precision on platforms where long double > 64 bits + const SIZE: usize = mem::size_of::<c_double>(); + let val = if bytes.len() >= SIZE { + c_double::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0.0 + }; + Ok(vm.ctx.new_float(val).into()) + } + "?" => { + let val = !bytes.is_empty() && bytes[0] != 0; + Ok(vm.ctx.new_bool(val).into()) + } + "v" => { + // VARIANT_BOOL: non-zero = True, zero = False + const SIZE: usize = mem::size_of::<c_short>(); + let val = if bytes.len() >= SIZE { + c_short::from_ne_bytes(bytes[..SIZE].try_into().expect("size checked")) + } else { + 0 + }; + Ok(vm.ctx.new_bool(val != 0).into()) + } + "z" => { + // c_char_p: read NULL-terminated string from pointer + let ptr = read_ptr_from_buffer(bytes); + if ptr == 0 { + return Ok(vm.ctx.none()); + } + let c_str = unsafe { core::ffi::CStr::from_ptr(ptr as _) }; + Ok(vm.ctx.new_bytes(c_str.to_bytes().to_vec()).into()) + } + "Z" => { + // c_wchar_p: read NULL-terminated wide string from pointer + let ptr = read_ptr_from_buffer(bytes); + if ptr == 0 { + return Ok(vm.ctx.none()); + } + let len = unsafe { libc::wcslen(ptr as *const libc::wchar_t) }; + let wchars = + unsafe { core::slice::from_raw_parts(ptr as *const libc::wchar_t, len) }; + // wchar_t is i32 on some platforms and u32 on others + #[allow( + clippy::unnecessary_cast, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + let s: String = wchars + .iter() + .filter_map(|&c| char::from_u32(c as u32)) + .collect(); + Ok(vm.ctx.new_str(s).into()) + } + "P" => { + // c_void_p: return pointer value as integer + let val = read_ptr_from_buffer(bytes); + if val == 0 { + return Ok(vm.ctx.none()); + } + Ok(vm.ctx.new_int(val).into()) + } + "u" => { + let val = if bytes.len() >= mem::size_of::<WideChar>() { + let wc = if mem::size_of::<WideChar>() == 2 { + u16::from_ne_bytes([bytes[0], bytes[1]]) as u32 + } else { + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) + }; + char::from_u32(wc).unwrap_or('\0') + } else { + '\0' + }; + Ok(vm.ctx.new_str(val).into()) + } + _ => Ok(vm.ctx.none()), + }; + } + // Default: return bytes as-is + Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) +} + +// Shared functions for Structure and Union types + +/// Parse a non-negative integer attribute, returning default if not present +pub(super) fn get_usize_attr( + obj: &PyObject, + attr: &str, + default: usize, + vm: &VirtualMachine, +) -> PyResult<usize> { + let Ok(attr_val) = obj.get_attr(vm.ctx.intern_str(attr), vm) else { + return Ok(default); + }; + let n = attr_val + .try_int(vm) + .map_err(|_| vm.new_value_error(format!("{attr} must be a non-negative integer")))?; + let val = n.as_bigint(); + if val.is_negative() { + return Err(vm.new_value_error(format!("{attr} must be a non-negative integer"))); + } + Ok(val.to_usize().unwrap_or(default)) +} + +/// Read a pointer value from buffer +#[inline] +pub(super) fn read_ptr_from_buffer(buffer: &[u8]) -> usize { + const PTR_SIZE: usize = core::mem::size_of::<usize>(); + buffer + .first_chunk::<PTR_SIZE>() + .copied() + .map_or(0, usize::from_ne_bytes) +} + +/// Check if a type is a "simple instance" (direct subclass of a simple type) +/// Returns TRUE for c_int, c_void_p, etc. (simple types with _type_ attribute) +/// Returns FALSE for Structure, Array, POINTER(T), etc. +pub(super) fn is_simple_instance(typ: &Py<PyType>) -> bool { + // _ctypes_simple_instance + // Check if the type's metaclass is PyCSimpleType + let metaclass = typ.class(); + metaclass.fast_issubclass(super::simple::PyCSimpleType::static_type()) +} + +/// Set or initialize StgInfo on a type +pub(super) fn set_or_init_stginfo(type_ref: &PyType, stg_info: StgInfo) { + if type_ref.init_type_data(stg_info.clone()).is_err() + && let Some(mut existing) = type_ref.get_type_data_mut::<StgInfo>() + { + // Preserve pointer_type cache across StgInfo replacement + let old_pointer_type = existing.pointer_type.take(); + *existing = stg_info; + if existing.pointer_type.is_none() { + existing.pointer_type = old_pointer_type; + } + } +} + +/// Check if a field type supports byte order swapping +pub(super) fn check_other_endian_support( + field_type: &PyObject, + vm: &VirtualMachine, +) -> PyResult<()> { + let other_endian_attr = if cfg!(target_endian = "little") { + "__ctype_be__" + } else { + "__ctype_le__" + }; + + if field_type.get_attr(other_endian_attr, vm).is_ok() { + return Ok(()); + } + + // Array type: recursively check element type + if let Ok(elem_type) = field_type.get_attr("_type_", vm) + && field_type.get_attr("_length_", vm).is_ok() + { + return check_other_endian_support(&elem_type, vm); + } + + // Structure/Union: has StgInfo but no _type_ attribute + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && type_obj.stg_info_opt().is_some() + && field_type.get_attr("_type_", vm).is_err() + { + return Ok(()); + } + + Err(vm.new_type_error(format!( + "This type does not support other endian: {}", + field_type.class().name() + ))) +} + +/// Get the size of a ctypes field type +pub(super) fn get_field_size(field_type: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(stg_info) = type_obj.stg_info_opt() + { + return Ok(stg_info.size); + } + + if let Some(size) = field_type + .get_attr("_type_", vm) + .ok() + .and_then(|type_attr| type_attr.str(vm).ok()) + .and_then(|type_str| { + let s = type_str.to_string(); + (s.len() == 1).then(|| super::get_size(&s)) + }) + { + return Ok(size); + } + + if let Some(s) = field_type + .get_attr("size_of_instances", vm) + .ok() + .and_then(|size_method| size_method.call((), vm).ok()) + .and_then(|size| size.try_int(vm).ok()) + .and_then(|n| n.as_bigint().to_usize()) + { + return Ok(s); + } + + Ok(core::mem::size_of::<usize>()) +} + +/// Get the alignment of a ctypes field type +pub(super) fn get_field_align(field_type: &PyObject, vm: &VirtualMachine) -> usize { + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(stg_info) = type_obj.stg_info_opt() + && stg_info.align > 0 + { + return stg_info.align; + } + + if let Some(align) = field_type + .get_attr("_type_", vm) + .ok() + .and_then(|type_attr| type_attr.str(vm).ok()) + .and_then(|type_str| { + let s = type_str.to_string(); + (s.len() == 1).then(|| super::get_size(&s)) + }) + { + return align; + } + + 1 +} + +/// Promote fields from anonymous struct/union to parent type +fn make_fields( + cls: &Py<PyType>, + descr: &super::PyCField, + index: usize, + offset: isize, + vm: &VirtualMachine, +) -> PyResult<()> { + let fields = descr.proto.as_object().get_attr("_fields_", vm)?; + let fieldlist: Vec<PyObjectRef> = if let Some(list) = fields.downcast_ref::<PyList>() { + list.borrow_vec().to_vec() + } else if let Some(tuple) = fields.downcast_ref::<PyTuple>() { + tuple.to_vec() + } else { + return Err(vm.new_type_error("_fields_ must be a sequence")); + }; + + for pair in fieldlist.iter() { + let field_tuple = pair + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("_fields_ must contain tuples"))?; + + if field_tuple.len() < 2 { + continue; + } + + let fname = field_tuple + .first() + .expect("len checked") + .downcast_ref::<PyUtf8Str>() + .ok_or_else(|| vm.new_type_error("field name must be a string"))?; + + let fdescr_obj = descr + .proto + .as_object() + .get_attr(vm.ctx.intern_str(fname.as_str()), vm)?; + let fdescr = fdescr_obj + .downcast_ref::<super::PyCField>() + .ok_or_else(|| vm.new_type_error("unexpected type"))?; + + if fdescr.anonymous { + make_fields( + cls, + fdescr, + index + fdescr.index, + offset + fdescr.offset, + vm, + )?; + continue; + } + + let new_descr = super::PyCField::new_from_field(fdescr, index, offset); + cls.set_attr( + vm.ctx.intern_str(fname.as_wtf8()), + new_descr.to_pyobject(vm), + ); + } + + Ok(()) +} + +/// Process _anonymous_ attribute for struct/union +pub(super) fn make_anon_fields(cls: &Py<PyType>, vm: &VirtualMachine) -> PyResult<()> { + let anon = match cls.as_object().get_attr("_anonymous_", vm) { + Ok(anon) => anon, + Err(_) => return Ok(()), + }; + + let anon_names: Vec<PyObjectRef> = if let Some(list) = anon.downcast_ref::<PyList>() { + list.borrow_vec().to_vec() + } else if let Some(tuple) = anon.downcast_ref::<PyTuple>() { + tuple.to_vec() + } else { + return Err(vm.new_type_error("_anonymous_ must be a sequence")); + }; + + for fname_obj in anon_names.iter() { + let fname = fname_obj + .downcast_ref::<PyStr>() + .ok_or_else(|| vm.new_type_error("_anonymous_ items must be strings"))?; + + let descr_obj = cls + .as_object() + .get_attr(vm.ctx.intern_str(fname.as_wtf8()), vm)?; + + let descr = descr_obj.downcast_ref::<super::PyCField>().ok_or_else(|| { + vm.new_attribute_error(format!( + "'{}' is specified in _anonymous_ but not in _fields_", + fname.as_wtf8() + )) + })?; + + let mut new_descr = super::PyCField::new_from_field(descr, 0, 0); + new_descr.set_anonymous(true); + cls.set_attr( + vm.ctx.intern_str(fname.as_wtf8()), + new_descr.to_pyobject(vm), + ); + + make_fields(cls, descr, descr.index, descr.offset, vm)?; + } + + Ok(()) +} diff --git a/crates/vm/src/stdlib/_ctypes/function.rs b/crates/vm/src/stdlib/_ctypes/function.rs new file mode 100644 index 00000000000..e28fd91abbc --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes/function.rs @@ -0,0 +1,2322 @@ +// spell-checker:disable + +use super::{ + _ctypes::CArgObject, + PyCArray, PyCData, PyCPointer, PyCStructure, StgInfo, + base::{CDATA_BUFFER_METHODS, FfiArgValue, ParamFunc, StgInfoFlags}, + simple::PyCSimple, + type_info, +}; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBytes, PyDict, PyNone, PyStr, PyTuple, PyType, PyTypeRef}, + class::StaticType, + function::FuncArgs, + protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}, + types::{AsBuffer, AsNumber, Callable, Constructor, Initializer, Representable}, + vm::thread::with_current_vm, +}; +use alloc::borrow::Cow; +use core::ffi::c_void; +use core::fmt::Debug; +use libffi::{ + low, + middle::{Arg, Cif, Closure, CodePtr, Type}, +}; +use libloading::Symbol; +use num_traits::{Signed, ToPrimitive}; +use rustpython_common::lock::PyRwLock; + +// Internal function addresses for special ctypes functions +pub(super) const INTERNAL_CAST_ADDR: usize = 1; +pub(super) const INTERNAL_STRING_AT_ADDR: usize = 2; +pub(super) const INTERNAL_WSTRING_AT_ADDR: usize = 3; +pub(super) const INTERNAL_MEMORYVIEW_AT_ADDR: usize = 4; + +// Thread-local errno storage for ctypes +std::thread_local! { + /// Thread-local storage for ctypes errno + /// This is separate from the system errno - ctypes swaps them during FFI calls + /// when use_errno=True is specified. + static CTYPES_LOCAL_ERRNO: core::cell::Cell<i32> = const { core::cell::Cell::new(0) }; +} + +/// Get ctypes thread-local errno value +pub(super) fn get_errno_value() -> i32 { + CTYPES_LOCAL_ERRNO.with(|e| e.get()) +} + +/// Set ctypes thread-local errno value, returns old value +pub(super) fn set_errno_value(value: i32) -> i32 { + CTYPES_LOCAL_ERRNO.with(|e| { + let old = e.get(); + e.set(value); + old + }) +} + +/// Save and restore errno around FFI call (called when use_errno=True) +/// Before: restore thread-local errno to system +/// After: save system errno to thread-local +#[cfg(not(windows))] +fn swap_errno<F, R>(f: F) -> R +where + F: FnOnce() -> R, +{ + // Before call: restore thread-local errno to system + let saved = CTYPES_LOCAL_ERRNO.with(|e| e.get()); + errno::set_errno(errno::Errno(saved)); + + // Call the function + let result = f(); + + // After call: save system errno to thread-local + let new_error = errno::errno().0; + CTYPES_LOCAL_ERRNO.with(|e| e.set(new_error)); + + result +} + +#[cfg(windows)] +std::thread_local! { + /// Thread-local storage for ctypes last_error (Windows only) + static CTYPES_LOCAL_LAST_ERROR: core::cell::Cell<u32> = const { core::cell::Cell::new(0) }; +} + +#[cfg(windows)] +pub(super) fn get_last_error_value() -> u32 { + CTYPES_LOCAL_LAST_ERROR.with(|e| e.get()) +} + +#[cfg(windows)] +pub(super) fn set_last_error_value(value: u32) -> u32 { + CTYPES_LOCAL_LAST_ERROR.with(|e| { + let old = e.get(); + e.set(value); + old + }) +} + +/// Save and restore last_error around FFI call (called when use_last_error=True) +#[cfg(windows)] +fn save_and_restore_last_error<F, R>(f: F) -> R +where + F: FnOnce() -> R, +{ + // Before call: restore thread-local last_error to Windows + let saved = CTYPES_LOCAL_LAST_ERROR.with(|e| e.get()); + unsafe { windows_sys::Win32::Foundation::SetLastError(saved) }; + + // Call the function + let result = f(); + + // After call: save Windows last_error to thread-local + let new_error = unsafe { windows_sys::Win32::Foundation::GetLastError() }; + CTYPES_LOCAL_LAST_ERROR.with(|e| e.set(new_error)); + + result +} + +type FP = unsafe extern "C" fn(); + +/// Get FFI type for a ctypes type code +fn get_ffi_type(ty: &str) -> Option<libffi::middle::Type> { + type_info(ty).map(|t| (t.ffi_type_fn)()) +} + +// PyCFuncPtr - Function pointer implementation + +/// Get FFI type from CArgObject tag character +fn ffi_type_from_tag(tag: u8) -> Type { + match tag { + b'c' | b'b' => Type::i8(), + b'B' => Type::u8(), + b'h' => Type::i16(), + b'H' => Type::u16(), + b'i' => Type::i32(), + b'I' => Type::u32(), + b'l' => { + if core::mem::size_of::<libc::c_long>() == 8 { + Type::i64() + } else { + Type::i32() + } + } + b'L' => { + if core::mem::size_of::<libc::c_ulong>() == 8 { + Type::u64() + } else { + Type::u32() + } + } + b'q' => Type::i64(), + b'Q' => Type::u64(), + b'f' => Type::f32(), + b'd' | b'g' => Type::f64(), + b'?' => Type::u8(), + b'u' => { + if core::mem::size_of::<super::WideChar>() == 2 { + Type::u16() + } else { + Type::u32() + } + } + _ => Type::pointer(), // 'P', 'V', 'z', 'Z', 'O', etc. + } +} + +/// Convert any object to a pointer value for c_void_p arguments +/// Follows ConvParam logic for pointer types +fn convert_to_pointer(value: &PyObject, vm: &VirtualMachine) -> PyResult<FfiArgValue> { + // 0. CArgObject (from byref()) -> buffer address + offset + if let Some(carg) = value.downcast_ref::<CArgObject>() { + // Get buffer address from the underlying object + let base_addr = if let Some(cdata) = carg.obj.downcast_ref::<PyCData>() { + cdata.buffer.read().as_ptr() as usize + } else { + return Err(vm.new_type_error(format!( + "byref() argument must be a ctypes instance, not '{}'", + carg.obj.class().name() + ))); + }; + let addr = (base_addr as isize + carg.offset) as usize; + return Ok(FfiArgValue::Pointer(addr)); + } + + // 1. None -> NULL + if value.is(&vm.ctx.none) { + return Ok(FfiArgValue::Pointer(0)); + } + + // 2. PyCArray -> buffer address (PyCArrayType_paramfunc) + if let Some(array) = value.downcast_ref::<PyCArray>() { + let addr = array.0.buffer.read().as_ptr() as usize; + return Ok(FfiArgValue::Pointer(addr)); + } + + // 3. PyCPointer -> stored pointer value + if let Some(ptr) = value.downcast_ref::<PyCPointer>() { + return Ok(FfiArgValue::Pointer(ptr.get_ptr_value())); + } + + // 4. PyCStructure -> buffer address + if let Some(struct_obj) = value.downcast_ref::<PyCStructure>() { + let addr = struct_obj.0.buffer.read().as_ptr() as usize; + return Ok(FfiArgValue::Pointer(addr)); + } + + // 5. PyCSimple (c_void_p, c_char_p, etc.) -> value from buffer + if let Some(simple) = value.downcast_ref::<PyCSimple>() { + let buffer = simple.0.buffer.read(); + if buffer.len() >= core::mem::size_of::<usize>() { + let addr = super::base::read_ptr_from_buffer(&buffer); + return Ok(FfiArgValue::Pointer(addr)); + } + } + + // 6. bytes -> buffer address (PyBytes_AsString) + if let Some(bytes) = value.downcast_ref::<crate::builtins::PyBytes>() { + let addr = bytes.as_bytes().as_ptr() as usize; + return Ok(FfiArgValue::Pointer(addr)); + } + + // 7. Integer -> direct value (PyLong_AsVoidPtr behavior) + if let Ok(int_val) = value.try_int(vm) { + let bigint = int_val.as_bigint(); + // Negative values: use signed conversion (allows -1 as 0xFFFF...) + if bigint.is_negative() { + if let Some(signed_val) = bigint.to_isize() { + return Ok(FfiArgValue::Pointer(signed_val as usize)); + } + } else if let Some(unsigned_val) = bigint.to_usize() { + return Ok(FfiArgValue::Pointer(unsigned_val)); + } + // Value out of range - raise OverflowError + return Err(vm.new_overflow_error("int too large to convert to pointer")); + } + + // 8. Check _as_parameter_ attribute ( recursive ConvParam) + if let Ok(as_param) = value.get_attr("_as_parameter_", vm) { + return convert_to_pointer(&as_param, vm); + } + + Err(vm.new_type_error(format!( + "cannot convert '{}' to c_void_p", + value.class().name() + ))) +} + +/// ConvParam-like conversion for when argtypes is None +/// Returns an Argument with FFI type, value, and optional keep object +fn conv_param(value: &PyObject, vm: &VirtualMachine) -> PyResult<Argument> { + // 1. CArgObject (from byref() or paramfunc) -> use stored type and value + if let Some(carg) = value.downcast_ref::<CArgObject>() { + let ffi_type = ffi_type_from_tag(carg.tag); + return Ok(Argument { + ffi_type, + keep: None, + value: carg.value.clone(), + }); + } + + // 2. None -> NULL pointer + if value.is(&vm.ctx.none) { + return Ok(Argument { + ffi_type: Type::pointer(), + keep: None, + value: FfiArgValue::Pointer(0), + }); + } + + // 3. ctypes objects -> use paramfunc + if let Ok(carg) = super::base::call_paramfunc(value, vm) { + let ffi_type = ffi_type_from_tag(carg.tag); + return Ok(Argument { + ffi_type, + keep: None, + value: carg.value.clone(), + }); + } + + // 4. Python str -> wide string pointer (like PyUnicode_AsWideCharString) + if let Some(s) = value.downcast_ref::<PyStr>() { + // Convert to null-terminated UTF-16, preserving lone surrogates + let wide: Vec<u16> = s + .as_wtf8() + .encode_wide() + .chain(core::iter::once(0)) + .collect(); + let wide_bytes: Vec<u8> = wide.iter().flat_map(|&x| x.to_ne_bytes()).collect(); + let keep = vm.ctx.new_bytes(wide_bytes); + let addr = keep.as_bytes().as_ptr() as usize; + return Ok(Argument { + ffi_type: Type::pointer(), + keep: Some(keep.into()), + value: FfiArgValue::Pointer(addr), + }); + } + + // 9. Python bytes -> null-terminated buffer pointer + // Need to ensure null termination like c_char_p + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let mut buffer = bytes.as_bytes().to_vec(); + buffer.push(0); // Add null terminator + let keep = vm.ctx.new_bytes(buffer); + let addr = keep.as_bytes().as_ptr() as usize; + return Ok(Argument { + ffi_type: Type::pointer(), + keep: Some(keep.into()), + value: FfiArgValue::Pointer(addr), + }); + } + + // 10. Python int -> i32 (default integer type) + if let Ok(int_val) = value.try_int(vm) { + let val = int_val.as_bigint().to_i32().unwrap_or(0); + return Ok(Argument { + ffi_type: Type::i32(), + keep: None, + value: FfiArgValue::I32(val), + }); + } + + // 11. Python float -> f64 + if let Ok(float_val) = value.try_float(vm) { + return Ok(Argument { + ffi_type: Type::f64(), + keep: None, + value: FfiArgValue::F64(float_val.to_f64()), + }); + } + + // 12. Check _as_parameter_ attribute + if let Ok(as_param) = value.get_attr("_as_parameter_", vm) { + return conv_param(&as_param, vm); + } + + Err(vm.new_type_error(format!( + "Don't know how to convert parameter {}", + value.class().name() + ))) +} + +trait ArgumentType { + fn to_ffi_type(&self, vm: &VirtualMachine) -> PyResult<Type>; + fn convert_object(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<FfiArgValue>; +} + +impl ArgumentType for PyTypeRef { + fn to_ffi_type(&self, vm: &VirtualMachine) -> PyResult<Type> { + use super::pointer::PyCPointer; + use super::structure::PyCStructure; + + // CArgObject (from byref()) should be treated as pointer + if self.fast_issubclass(CArgObject::static_type()) { + return Ok(Type::pointer()); + } + + // Pointer types (POINTER(T)) are always pointer FFI type + // Check if type is a subclass of _Pointer (PyCPointer) + if self.fast_issubclass(PyCPointer::static_type()) { + return Ok(Type::pointer()); + } + + // Structure types are passed as pointers + if self.fast_issubclass(PyCStructure::static_type()) { + return Ok(Type::pointer()); + } + + // Use get_attr to traverse MRO (for subclasses like MyInt(c_int)) + let typ = self + .as_object() + .get_attr(vm.ctx.intern_str("_type_"), vm) + .ok() + .ok_or(vm.new_type_error("Unsupported argument type"))?; + let typ = typ + .downcast_ref::<PyStr>() + .ok_or(vm.new_type_error("Unsupported argument type"))?; + let typ = typ.to_string(); + let typ = typ.as_str(); + get_ffi_type(typ) + .ok_or_else(|| vm.new_type_error(format!("Unsupported argument type: {}", typ))) + } + + fn convert_object(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<FfiArgValue> { + // Call from_param first to convert the value + // converter = PyTuple_GET_ITEM(argtypes, i); + // v = PyObject_CallOneArg(converter, arg); + let from_param = self + .as_object() + .get_attr(vm.ctx.intern_str("from_param"), vm)?; + let converted = from_param.call((value.clone(),), vm)?; + + // Then pass the converted value to ConvParam logic + // CArgObject (from from_param) -> use stored value directly + if let Some(carg) = converted.downcast_ref::<CArgObject>() { + return Ok(carg.value.clone()); + } + + // None -> NULL pointer + if vm.is_none(&converted) { + return Ok(FfiArgValue::Pointer(0)); + } + + // For pointer types (POINTER(T)), we need to pass the pointer VALUE stored in buffer + if self.fast_issubclass(PyCPointer::static_type()) { + if let Some(pointer) = converted.downcast_ref::<PyCPointer>() { + return Ok(FfiArgValue::Pointer(pointer.get_ptr_value())); + } + return convert_to_pointer(&converted, vm); + } + + // For structure types, convert to pointer to structure + if self.fast_issubclass(PyCStructure::static_type()) { + return convert_to_pointer(&converted, vm); + } + + // Get the type code for this argument type + let type_code = self + .as_object() + .get_attr(vm.ctx.intern_str("_type_"), vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); + + // For pointer types (c_void_p, c_char_p, c_wchar_p), handle as pointer + if matches!(type_code.as_deref(), Some("P") | Some("z") | Some("Z")) { + return convert_to_pointer(&converted, vm); + } + + // PyCSimple (already a ctypes instance from from_param) + if let Ok(simple) = converted.clone().downcast::<PyCSimple>() { + let typ = ArgumentType::to_ffi_type(self, vm)?; + let ffi_value = simple + .to_ffi_value(typ, vm) + .ok_or(vm.new_type_error("Unsupported argument type"))?; + return Ok(ffi_value); + } + + Err(vm.new_type_error("Unsupported argument type")) + } +} + +trait ReturnType { + fn to_ffi_type(&self, vm: &VirtualMachine) -> Option<Type>; +} + +impl ReturnType for PyTypeRef { + fn to_ffi_type(&self, vm: &VirtualMachine) -> Option<Type> { + // Try to get _type_ attribute first (for ctypes types like c_void_p) + if let Ok(type_attr) = self.as_object().get_attr(vm.ctx.intern_str("_type_"), vm) + && let Some(s) = type_attr.downcast_ref::<PyStr>() + && let Some(ffi_type) = s.to_str().and_then(get_ffi_type) + { + return Some(ffi_type); + } + + // Check for Structure/Array types (have StgInfo but no _type_) + // _ctypes_get_ffi_type: returns appropriately sized type for struct returns + if let Some(stg_info) = self.stg_info_opt() { + let size = stg_info.size; + // Small structs can be returned in registers + // Match can_return_struct_as_int/can_return_struct_as_sint64 + return Some(if size <= 4 { + Type::i32() + } else if size <= 8 { + Type::i64() + } else { + // Large structs: use pointer-sized return + // (ABI typically returns via hidden pointer parameter) + Type::pointer() + }); + } + + // Fallback to class name + get_ffi_type(self.name().to_string().as_str()) + } +} + +impl ReturnType for PyNone { + fn to_ffi_type(&self, _vm: &VirtualMachine) -> Option<Type> { + get_ffi_type("void") + } +} + +// PyCFuncPtrType - Metaclass for function pointer types +// PyCFuncPtrType_init + +#[pyclass(name = "PyCFuncPtrType", base = PyType, module = "_ctypes")] +#[derive(Debug)] +#[repr(transparent)] +pub(super) struct PyCFuncPtrType(PyType); + +impl Initializer for PyCFuncPtrType { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let obj: PyObjectRef = zelf.clone().into(); + let new_type: PyTypeRef = obj + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + new_type.check_not_initialized(vm)?; + + let ptr_size = core::mem::size_of::<usize>(); + let mut stg_info = StgInfo::new(ptr_size, ptr_size); + stg_info.format = Some("X{}".to_string()); + stg_info.length = 1; + stg_info.flags |= StgInfoFlags::TYPEFLAG_ISPOINTER; + stg_info.paramfunc = ParamFunc::Pointer; // CFuncPtr is passed as a pointer + + let _ = new_type.init_type_data(stg_info); + Ok(()) + } +} + +#[pyclass(flags(IMMUTABLETYPE), with(Initializer))] +impl PyCFuncPtrType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } +} + +/// PyCFuncPtr - Function pointer instance +/// Saved in _base.buffer +#[pyclass( + module = "_ctypes", + name = "CFuncPtr", + base = PyCData, + metaclass = "PyCFuncPtrType" +)] +#[repr(C)] +pub(super) struct PyCFuncPtr { + pub _base: PyCData, + /// Thunk for callbacks (keeps thunk alive) + pub thunk: PyRwLock<Option<PyRef<PyCThunk>>>, + /// Original Python callable (for callbacks) + pub callable: PyRwLock<Option<PyObjectRef>>, + /// Converters cache + pub converters: PyRwLock<Option<PyObjectRef>>, + /// Instance-level argtypes override + pub argtypes: PyRwLock<Option<PyObjectRef>>, + /// Instance-level restype override + pub restype: PyRwLock<Option<PyObjectRef>>, + /// Checker function + pub checker: PyRwLock<Option<PyObjectRef>>, + /// Error checking function + pub errcheck: PyRwLock<Option<PyObjectRef>>, + /// COM method vtable index + /// When set, the function reads the function pointer from the vtable at call time + #[cfg(windows)] + pub index: PyRwLock<Option<usize>>, + /// COM method IID (interface ID) for error handling + #[cfg(windows)] + pub iid: PyRwLock<Option<PyObjectRef>>, + /// Parameter flags for COM methods (direction: IN=1, OUT=2, IN|OUT=4) + /// Each element is (direction, name, default) tuple + pub paramflags: PyRwLock<Option<PyObjectRef>>, +} + +impl Debug for PyCFuncPtr { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyCFuncPtr") + .field("func_ptr", &self.get_func_ptr()) + .finish() + } +} + +/// Extract pointer value from a ctypes argument (c_void_p conversion) +fn extract_ptr_from_arg(arg: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { + // Try CArgObject first - extract the wrapped pointer value, applying offset + if let Some(carg) = arg.downcast_ref::<super::_ctypes::CArgObject>() { + if carg.offset != 0 + && let Some(cdata) = carg.obj.downcast_ref::<PyCData>() + { + let base = cdata.buffer.read().as_ptr() as isize; + return Ok((base + carg.offset) as usize); + } + return extract_ptr_from_arg(&carg.obj, vm); + } + // Try to get pointer value from various ctypes types + if let Some(ptr) = arg.downcast_ref::<PyCPointer>() { + return Ok(ptr.get_ptr_value()); + } + if let Some(simple) = arg.downcast_ref::<PyCSimple>() { + let buffer = simple.0.buffer.read(); + if let Some(&bytes) = buffer.first_chunk::<{ size_of::<usize>() }>() { + return Ok(usize::from_ne_bytes(bytes)); + } + } + if let Some(cdata) = arg.downcast_ref::<PyCData>() { + // For arrays/structures, return address of buffer + return Ok(cdata.buffer.read().as_ptr() as usize); + } + // PyStr: return internal buffer address + if let Some(s) = arg.downcast_ref::<PyStr>() { + return Ok(s.as_bytes().as_ptr() as usize); + } + // PyBytes: return internal buffer address + if let Some(bytes) = arg.downcast_ref::<PyBytes>() { + return Ok(bytes.as_bytes().as_ptr() as usize); + } + // Try as integer + if let Ok(int_val) = arg.try_int(vm) { + return Ok(int_val.as_bigint().to_usize().unwrap_or(0)); + } + Err(vm.new_type_error(format!( + "cannot convert '{}' to pointer", + arg.class().name() + ))) +} + +/// string_at implementation - read bytes from memory at ptr +fn string_at_impl(ptr: usize, size: isize, vm: &VirtualMachine) -> PyResult { + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + let ptr = ptr as *const u8; + let len = if size < 0 { + // size == -1 means use strlen + unsafe { libc::strlen(ptr as _) } + } else { + // Overflow check for huge size values + let size_usize = size as usize; + if size_usize > isize::MAX as usize / 2 { + return Err(vm.new_overflow_error("string too long")); + } + size_usize + }; + let bytes = unsafe { core::slice::from_raw_parts(ptr, len) }; + Ok(vm.ctx.new_bytes(bytes.to_vec()).into()) +} + +/// wstring_at implementation - read wide string from memory at ptr +fn wstring_at_impl(ptr: usize, size: isize, vm: &VirtualMachine) -> PyResult { + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + let w_ptr = ptr as *const libc::wchar_t; + let len = if size < 0 { + unsafe { libc::wcslen(w_ptr) } + } else { + // Overflow check for huge size values + let size_usize = size as usize; + if size_usize > isize::MAX as usize / core::mem::size_of::<libc::wchar_t>() { + return Err(vm.new_overflow_error("string too long")); + } + size_usize + }; + let wchars = unsafe { core::slice::from_raw_parts(w_ptr, len) }; + + // Windows: wchar_t = u16 (UTF-16) -> use Wtf8Buf::from_wide + // macOS/Linux: wchar_t = i32 (UTF-32) -> convert via char::from_u32 + #[cfg(windows)] + { + use rustpython_common::wtf8::Wtf8Buf; + let wide: Vec<u16> = wchars.to_vec(); + let wtf8 = Wtf8Buf::from_wide(&wide); + Ok(vm.ctx.new_str(wtf8).into()) + } + #[cfg(not(windows))] + { + #[allow( + clippy::useless_conversion, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + let s: String = wchars + .iter() + .filter_map(|&c| u32::try_from(c).ok().and_then(char::from_u32)) + .collect(); + Ok(vm.ctx.new_str(s).into()) + } +} + +/// A buffer wrapping raw memory at a given pointer, for zero-copy memoryview. +#[pyclass(name = "_RawMemoryBuffer", module = "_ctypes")] +#[derive(Debug, PyPayload)] +pub(super) struct RawMemoryBuffer { + ptr: *const u8, + size: usize, + readonly: bool, +} + +// SAFETY: The caller ensures the pointer remains valid +unsafe impl Send for RawMemoryBuffer {} +unsafe impl Sync for RawMemoryBuffer {} + +static RAW_MEMORY_BUFFER_METHODS: crate::protocol::BufferMethods = crate::protocol::BufferMethods { + obj_bytes: |buffer| { + let raw = buffer.obj_as::<RawMemoryBuffer>(); + let slice = unsafe { core::slice::from_raw_parts(raw.ptr, raw.size) }; + rustpython_common::borrow::BorrowedValue::Ref(slice) + }, + obj_bytes_mut: |buffer| { + let raw = buffer.obj_as::<RawMemoryBuffer>(); + let slice = unsafe { core::slice::from_raw_parts_mut(raw.ptr as *mut u8, raw.size) }; + rustpython_common::borrow::BorrowedValueMut::RefMut(slice) + }, + release: |_| {}, + retain: |_| {}, +}; + +#[pyclass(with(AsBuffer))] +impl RawMemoryBuffer {} + +impl AsBuffer for RawMemoryBuffer { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + Ok(PyBuffer::new( + zelf.to_owned().into(), + BufferDescriptor::simple(zelf.size, zelf.readonly), + &RAW_MEMORY_BUFFER_METHODS, + )) + } +} + +/// memoryview_at implementation - create a memoryview from memory at ptr +fn memoryview_at_impl(ptr: usize, size: isize, readonly: bool, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyMemoryView; + + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + if size < 0 { + return Err(vm.new_value_error("negative size")); + } + let len = size as usize; + let raw_buf = RawMemoryBuffer { + ptr: ptr as *const u8, + size: len, + readonly, + } + .into_pyobject(vm); + let mv = PyMemoryView::from_object(&raw_buf, vm)?; + Ok(mv.into_pyobject(vm)) +} + +// cast_check_pointertype +fn cast_check_pointertype(ctype: &PyObject, vm: &VirtualMachine) -> bool { + use super::pointer::PyCPointerType; + + // PyCPointerTypeObject_Check + if ctype.class().fast_issubclass(PyCPointerType::static_type()) { + return true; + } + + // PyCFuncPtrTypeObject_Check - TODO + + // simple pointer types via StgInfo.proto (c_void_p, c_char_p, etc.) + if let Ok(type_attr) = ctype.get_attr("_type_", vm) + && let Some(s) = type_attr.downcast_ref::<PyStr>() + { + let c = s + .to_str() + .expect("_type_ is validated as ASCII at type creation"); + if c.len() == 1 && "sPzUZXO".contains(c) { + return true; + } + } + + false +} + +/// cast implementation +/// _ctypes.c cast() +pub(super) fn cast_impl( + obj: PyObjectRef, + src: PyObjectRef, + ctype: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + // 1. cast_check_pointertype + if !cast_check_pointertype(&ctype, vm) { + return Err(vm.new_type_error(format!( + "cast() argument 2 must be a pointer type, not {}", + ctype.class().name() + ))); + } + + // 2. Extract pointer value - matches c_void_p_from_param_impl order + let ptr_value: usize = if vm.is_none(&obj) { + // None → NULL pointer + 0 + } else if let Ok(int_val) = obj.try_int(vm) { + // int/long → direct pointer value + int_val.as_bigint().to_usize().unwrap_or(0) + } else if let Some(bytes) = obj.downcast_ref::<PyBytes>() { + // bytes → buffer address (c_void_p_from_param: PyBytes_Check) + bytes.as_bytes().as_ptr() as usize + } else if let Some(s) = obj.downcast_ref::<PyStr>() { + // unicode/str → buffer address (c_void_p_from_param: PyUnicode_Check) + s.as_bytes().as_ptr() as usize + } else if let Some(ptr) = obj.downcast_ref::<PyCPointer>() { + // Pointer instance → contained pointer value + ptr.get_ptr_value() + } else if let Some(simple) = obj.downcast_ref::<PyCSimple>() { + // Simple type (c_void_p, c_char_p, etc.) → value from buffer + let buffer = simple.0.buffer.read(); + super::base::read_ptr_from_buffer(&buffer) + } else if let Some(cdata) = obj.downcast_ref::<PyCData>() { + // Array, Structure, Union → buffer address (b_ptr) + cdata.buffer.read().as_ptr() as usize + } else { + return Err(vm.new_type_error(format!( + "cast() argument 1 must be a ctypes instance, not {}", + obj.class().name() + ))); + }; + + // 3. Create result instance + let result = ctype.call((), vm)?; + + // 4. _objects reference tracking + // Share _objects dict between source and result, add id(src): src + if src.class().fast_issubclass(PyCData::static_type()) { + // Get the source's _objects, create dict if needed + let shared_objects: PyObjectRef = if let Some(src_cdata) = src.downcast_ref::<PyCData>() { + let mut src_objects = src_cdata.objects.write(); + if src_objects.is_none() { + // Create new dict + let dict = vm.ctx.new_dict(); + *src_objects = Some(dict.clone().into()); + dict.into() + } else if let Some(obj) = src_objects.as_ref() { + if obj.downcast_ref::<PyDict>().is_none() { + // Convert to dict (keep existing reference) + let dict = vm.ctx.new_dict(); + let id_key: PyObjectRef = vm.ctx.new_int(obj.get_id() as i64).into(); + let _ = dict.set_item(&*id_key, obj.clone(), vm); + *src_objects = Some(dict.clone().into()); + dict.into() + } else { + obj.clone() + } + } else { + vm.ctx.new_dict().into() + } + } else { + vm.ctx.new_dict().into() + }; + + // Add id(src): src to the shared dict + if let Some(dict) = shared_objects.downcast_ref::<PyDict>() { + let id_key: PyObjectRef = vm.ctx.new_int(src.get_id() as i64).into(); + let _ = dict.set_item(&*id_key, src.clone(), vm); + } + + // Set result's _objects to the shared dict + if let Some(result_cdata) = result.downcast_ref::<PyCData>() { + *result_cdata.objects.write() = Some(shared_objects); + } + } + + // 5. Store pointer value + if let Some(ptr) = result.downcast_ref::<PyCPointer>() { + ptr.set_ptr_value(ptr_value); + } else if let Some(cdata) = result.downcast_ref::<PyCData>() { + let bytes = ptr_value.to_ne_bytes(); + let mut buffer = cdata.buffer.write(); + let buf = buffer.to_mut(); + if buf.len() >= bytes.len() { + buf[..bytes.len()].copy_from_slice(&bytes); + } + } + + Ok(result) +} + +impl PyCFuncPtr { + /// Get function pointer address from buffer + fn get_func_ptr(&self) -> usize { + let buffer = self._base.buffer.read(); + super::base::read_ptr_from_buffer(&buffer) + } + + /// Get CodePtr from buffer for FFI calls + fn get_code_ptr(&self) -> Option<CodePtr> { + let addr = self.get_func_ptr(); + if addr != 0 { + Some(CodePtr(addr as *mut _)) + } else { + None + } + } + + /// Create buffer with function pointer address + fn make_ptr_buffer(addr: usize) -> Vec<u8> { + addr.to_ne_bytes().to_vec() + } +} + +impl Constructor for PyCFuncPtr { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Handle different argument forms: + // 1. Empty args: create uninitialized (NULL pointer) + // 2. One integer argument: function address + // 3. Tuple argument: (name, dll) form + // 4. Callable: callback creation + + let ptr_size = core::mem::size_of::<usize>(); + + if args.args.is_empty() { + return PyCFuncPtr { + _base: PyCData::from_bytes(vec![0u8; ptr_size], None), + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), + } + .into_ref_with_type(vm, cls) + .map(Into::into); + } + + let first_arg = &args.args[0]; + + // Check for COM method form: (index, name, [paramflags], [iid]) + // First arg is integer (vtable index), second arg is string (method name) + if args.args.len() >= 2 + && first_arg.try_int(vm).is_ok() + && args.args[1].downcast_ref::<PyStr>().is_some() + { + #[cfg(windows)] + let index = first_arg.try_int(vm)?.as_bigint().to_usize().unwrap_or(0); + + // args[3] is iid (GUID struct, optional) + // Also check if args[2] is a GUID (has Data1 attribute) when args[3] is not present + #[cfg(windows)] + let iid = args.args.get(3).cloned().or_else(|| { + args.args.get(2).and_then(|arg| { + // If it's a GUID struct (has Data1 attribute), use it as IID + if arg.get_attr("Data1", vm).is_ok() { + Some(arg.clone()) + } else { + None + } + }) + }); + + // args[2] is paramflags (tuple or None) + let paramflags = args.args.get(2).filter(|arg| !vm.is_none(arg)).cloned(); + + return PyCFuncPtr { + _base: PyCData::from_bytes(vec![0u8; ptr_size], None), + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(Some(index)), + #[cfg(windows)] + iid: PyRwLock::new(iid), + paramflags: PyRwLock::new(paramflags), + } + .into_ref_with_type(vm, cls) + .map(Into::into); + } + + // Check if first argument is an integer (function address) + if let Ok(addr) = first_arg.try_int(vm) { + let ptr_val = addr.as_bigint().to_usize().unwrap_or(0); + return PyCFuncPtr { + _base: PyCData::from_bytes(Self::make_ptr_buffer(ptr_val), None), + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), + } + .into_ref_with_type(vm, cls) + .map(Into::into); + } + + // Check if first argument is a tuple (name, dll) form + if let Some(tuple) = first_arg.downcast_ref::<PyTuple>() { + let name = tuple + .first() + .ok_or(vm.new_type_error("Expected a tuple with at least 2 elements"))? + .downcast_ref::<PyStr>() + .ok_or(vm.new_type_error("Expected a string"))? + .to_string(); + let dll = tuple + .iter() + .nth(1) + .ok_or(vm.new_type_error("Expected a tuple with at least 2 elements"))? + .clone(); + + // Get library handle and load function + let handle = dll.try_int(vm); + let handle = match handle { + Ok(handle) => handle.as_bigint().clone(), + Err(_) => dll + .get_attr("_handle", vm)? + .try_int(vm)? + .as_bigint() + .clone(), + }; + let library_cache = super::library::libcache().read(); + let library = library_cache + .get_lib( + handle + .to_usize() + .ok_or(vm.new_value_error("Invalid handle"))?, + ) + .ok_or_else(|| vm.new_value_error("Library not found"))?; + let inner_lib = library.lib.lock(); + + let terminated = format!("{}\0", &name); + let ptr_val = if let Some(lib) = &*inner_lib { + let pointer: Symbol<'_, FP> = unsafe { + lib.get(terminated.as_bytes()) + .map_err(|err| err.to_string()) + .map_err(|err| vm.new_attribute_error(err))? + }; + let addr = *pointer as usize; + // dlsym can return NULL for symbols that resolve to NULL (e.g., GNU IFUNC) + // Treat NULL addresses as errors + if addr == 0 { + return Err(vm.new_attribute_error(format!("function '{}' not found", name))); + } + addr + } else { + 0 + }; + + return PyCFuncPtr { + _base: PyCData::from_bytes(Self::make_ptr_buffer(ptr_val), None), + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), + } + .into_ref_with_type(vm, cls) + .map(Into::into); + } + + // Check if first argument is a Python callable (callback creation) + if first_arg.is_callable() { + // Get argument types and result type from the class + let class_argtypes = cls.get_attr(vm.ctx.intern_str("_argtypes_")); + let class_restype = cls.get_attr(vm.ctx.intern_str("_restype_")); + let class_flags = cls + .get_attr(vm.ctx.intern_str("_flags_")) + .and_then(|f| f.try_to_value::<u32>(vm).ok()) + .unwrap_or(0); + + // Create the thunk (C-callable wrapper for the Python function) + let thunk = PyCThunk::new( + first_arg.clone(), + class_argtypes.clone(), + class_restype.clone(), + class_flags, + vm, + )?; + let code_ptr = thunk.code_ptr(); + let ptr_val = code_ptr.0 as usize; + + // Store the thunk as a Python object to keep it alive + let thunk_ref: PyRef<PyCThunk> = thunk.into_ref(&vm.ctx); + + return PyCFuncPtr { + _base: PyCData::from_bytes(Self::make_ptr_buffer(ptr_val), None), + thunk: PyRwLock::new(Some(thunk_ref)), + callable: PyRwLock::new(Some(first_arg.clone())), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(class_argtypes), + restype: PyRwLock::new(class_restype), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), + } + .into_ref_with_type(vm, cls) + .map(Into::into); + } + + Err(vm.new_type_error("Expected an integer address or a tuple")) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +// PyCFuncPtr call helpers (similar to callproc.c flow) + +/// Handle internal function addresses (PYFUNCTYPE special cases) +/// Returns Some(result) if handled, None if should continue with normal call +fn handle_internal_func(addr: usize, args: &FuncArgs, vm: &VirtualMachine) -> Option<PyResult> { + if addr == INTERNAL_CAST_ADDR { + let result: PyResult<(PyObjectRef, PyObjectRef, PyObjectRef)> = args.clone().bind(vm); + return Some(result.and_then(|(obj, src, ctype)| cast_impl(obj, src, ctype, vm))); + } + + if addr == INTERNAL_STRING_AT_ADDR { + let result: PyResult<(PyObjectRef, Option<PyObjectRef>)> = args.clone().bind(vm); + return Some(result.and_then(|(ptr_arg, size_arg)| { + let ptr = extract_ptr_from_arg(&ptr_arg, vm)?; + let size = size_arg + .and_then(|s| s.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_isize()) + .unwrap_or(-1); + string_at_impl(ptr, size, vm) + })); + } + + if addr == INTERNAL_WSTRING_AT_ADDR { + let result: PyResult<(PyObjectRef, Option<PyObjectRef>)> = args.clone().bind(vm); + return Some(result.and_then(|(ptr_arg, size_arg)| { + let ptr = extract_ptr_from_arg(&ptr_arg, vm)?; + let size = size_arg + .and_then(|s| s.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_isize()) + .unwrap_or(-1); + wstring_at_impl(ptr, size, vm) + })); + } + + if addr == INTERNAL_MEMORYVIEW_AT_ADDR { + let result: PyResult<(PyObjectRef, PyObjectRef, Option<PyObjectRef>)> = + args.clone().bind(vm); + return Some(result.and_then(|(ptr_arg, size_arg, readonly_arg)| { + let ptr = extract_ptr_from_arg(&ptr_arg, vm)?; + let size_int = size_arg.try_int(vm)?; + let size = size_int + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("size too large"))?; + let readonly = readonly_arg + .and_then(|r| r.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_i32()) + .unwrap_or(0) + != 0; + memoryview_at_impl(ptr, size, readonly, vm) + })); + } + + None +} + +/// Call information extracted from PyCFuncPtr (argtypes, restype, etc.) +struct CallInfo { + explicit_arg_types: Option<Vec<PyTypeRef>>, + restype_obj: Option<PyObjectRef>, + restype_is_none: bool, + ffi_return_type: Type, + is_pointer_return: bool, +} + +/// Extract call information (argtypes, restype) from PyCFuncPtr +fn extract_call_info(zelf: &Py<PyCFuncPtr>, vm: &VirtualMachine) -> PyResult<CallInfo> { + // Get argtypes - first from instance, then from type's _argtypes_ + let explicit_arg_types: Option<Vec<PyTypeRef>> = + if let Some(argtypes_obj) = zelf.argtypes.read().as_ref() { + if !vm.is_none(argtypes_obj) { + Some( + argtypes_obj + .try_to_value::<Vec<PyObjectRef>>(vm)? + .into_iter() + .filter_map(|obj| obj.downcast::<PyType>().ok()) + .collect(), + ) + } else { + None // argtypes is None -> use ConvParam + } + } else if let Some(class_argtypes) = zelf + .as_object() + .class() + .get_attr(vm.ctx.intern_str("_argtypes_")) + && !vm.is_none(&class_argtypes) + { + Some( + class_argtypes + .try_to_value::<Vec<PyObjectRef>>(vm)? + .into_iter() + .filter_map(|obj| obj.downcast::<PyType>().ok()) + .collect(), + ) + } else { + None // No argtypes -> use ConvParam + }; + + // Get restype - first from instance, then from class's _restype_ + let restype_obj = zelf.restype.read().clone().or_else(|| { + zelf.as_object() + .class() + .get_attr(vm.ctx.intern_str("_restype_")) + }); + + // Check if restype is explicitly None (return void) + let restype_is_none = restype_obj.as_ref().is_some_and(|t| vm.is_none(t)); + let ffi_return_type = if restype_is_none { + Type::void() + } else { + restype_obj + .as_ref() + .and_then(|t| t.clone().downcast::<PyType>().ok()) + .and_then(|t| ReturnType::to_ffi_type(&t, vm)) + .unwrap_or_else(Type::i32) + }; + + // Check if return type is a pointer type via TYPEFLAG_ISPOINTER + // This handles c_void_p, c_char_p, c_wchar_p, and POINTER(T) types + let is_pointer_return = restype_obj + .as_ref() + .and_then(|t| t.clone().downcast::<PyType>().ok()) + .and_then(|t| { + t.stg_info_opt() + .map(|info| info.flags.contains(StgInfoFlags::TYPEFLAG_ISPOINTER)) + }) + .unwrap_or(false); + + Ok(CallInfo { + explicit_arg_types, + restype_obj, + restype_is_none, + ffi_return_type, + is_pointer_return, + }) +} + +/// Parsed paramflags: (direction, name, default) tuples +/// direction: 1=IN, 2=OUT, 4=IN|OUT (or 1|2=3) +type ParsedParamFlags = Vec<(u32, Option<String>, Option<PyObjectRef>)>; + +/// Parse paramflags from PyCFuncPtr +fn parse_paramflags( + zelf: &Py<PyCFuncPtr>, + vm: &VirtualMachine, +) -> PyResult<Option<ParsedParamFlags>> { + let Some(pf) = zelf.paramflags.read().as_ref().cloned() else { + return Ok(None); + }; + + let pf_vec = pf.try_to_value::<Vec<PyObjectRef>>(vm)?; + let parsed = pf_vec + .into_iter() + .map(|item| { + let Some(tuple) = item.downcast_ref::<PyTuple>() else { + // Single value means just the direction + let direction = item + .try_int(vm) + .ok() + .and_then(|i| i.as_bigint().to_u32()) + .unwrap_or(1); + return (direction, None, None); + }; + let direction = tuple + .first() + .and_then(|d| d.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_u32()) + .unwrap_or(1); + let name = tuple + .get(1) + .and_then(|n| n.downcast_ref::<PyStr>().map(|s| s.to_string())); + let default = tuple.get(2).cloned(); + (direction, name, default) + }) + .collect(); + Ok(Some(parsed)) +} + +/// Resolve COM method pointer from vtable (Windows only) +/// Returns (Some(CodePtr), true) if this is a COM method call, (None, false) otherwise +#[cfg(windows)] +fn resolve_com_method( + zelf: &Py<PyCFuncPtr>, + args: &FuncArgs, + vm: &VirtualMachine, +) -> PyResult<(Option<CodePtr>, bool)> { + let com_index = zelf.index.read(); + let Some(idx) = *com_index else { + return Ok((None, false)); + }; + + // First arg must be the COM object pointer + if args.args.is_empty() { + return Err(vm.new_type_error("COM method requires at least one argument (self)")); + } + + // Extract COM pointer value from first argument + let self_arg = &args.args[0]; + let com_ptr = if let Some(simple) = self_arg.downcast_ref::<PyCSimple>() { + let buffer = simple.0.buffer.read(); + if buffer.len() >= core::mem::size_of::<usize>() { + super::base::read_ptr_from_buffer(&buffer) + } else { + 0 + } + } else if let Ok(int_val) = self_arg.try_int(vm) { + int_val.as_bigint().to_usize().unwrap_or(0) + } else { + return Err(vm.new_type_error("COM method first argument must be a COM pointer")); + }; + + if com_ptr == 0 { + return Err(vm.new_value_error("NULL COM pointer access")); + } + + // Read vtable pointer from COM object: vtable = *(void**)com_ptr + let vtable_ptr = unsafe { *(com_ptr as *const usize) }; + if vtable_ptr == 0 { + return Err(vm.new_value_error("NULL vtable pointer")); + } + + // Read function pointer from vtable: func = vtable[index] + let fptr = unsafe { + let vtable = vtable_ptr as *const usize; + *vtable.add(idx) + }; + + if fptr == 0 { + return Err(vm.new_value_error("NULL function pointer in vtable")); + } + + Ok((Some(CodePtr(fptr as *mut _)), true)) +} + +/// Single argument for FFI call +// struct argument +struct Argument { + ffi_type: Type, + value: FfiArgValue, + #[allow(dead_code)] + keep: Option<PyObjectRef>, // Object to keep alive during call +} + +/// Out buffers for paramflags OUT parameters +type OutBuffers = Vec<(usize, PyObjectRef)>; + +/// Get buffer address from a ctypes object +fn get_buffer_addr(obj: &PyObjectRef) -> Option<usize> { + obj.downcast_ref::<PyCSimple>() + .map(|s| s.0.buffer.read().as_ptr() as usize) + .or_else(|| { + obj.downcast_ref::<super::structure::PyCStructure>() + .map(|s| s.0.buffer.read().as_ptr() as usize) + }) + .or_else(|| { + obj.downcast_ref::<PyCPointer>() + .map(|s| s.0.buffer.read().as_ptr() as usize) + }) +} + +/// Create OUT buffer for a parameter type +fn create_out_buffer(arg_type: &PyTypeRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // For POINTER(T) types, create T instance (the pointed-to type) + if arg_type.fast_issubclass(PyCPointer::static_type()) + && let Some(stg_info) = arg_type.stg_info_opt() + && let Some(ref proto) = stg_info.proto + { + return proto.as_object().call((), vm); + } + // Not a pointer type or no proto, create instance directly + arg_type.as_object().call((), vm) +} + +/// Build callargs when no argtypes specified (use ConvParam) +fn build_callargs_no_argtypes( + args: &FuncArgs, + vm: &VirtualMachine, +) -> PyResult<(Vec<Argument>, OutBuffers)> { + let arguments: Vec<Argument> = args + .args + .iter() + .map(|arg| conv_param(arg, vm)) + .collect::<PyResult<Vec<_>>>()?; + Ok((arguments, Vec::new())) +} + +/// Build callargs for regular function with argtypes (no paramflags) +fn build_callargs_simple( + args: &FuncArgs, + arg_types: &[PyTypeRef], + vm: &VirtualMachine, +) -> PyResult<(Vec<Argument>, OutBuffers)> { + let arguments: Vec<Argument> = args + .args + .iter() + .enumerate() + .map(|(n, arg)| { + let arg_type = arg_types + .get(n) + .ok_or_else(|| vm.new_type_error("argument amount mismatch"))?; + let ffi_type = ArgumentType::to_ffi_type(arg_type, vm)?; + let value = arg_type.convert_object(arg.clone(), vm)?; + Ok(Argument { + ffi_type, + keep: None, + value, + }) + }) + .collect::<PyResult<Vec<_>>>()?; + Ok((arguments, Vec::new())) +} + +/// Build callargs with paramflags (handles IN/OUT parameters) +fn build_callargs_with_paramflags( + args: &FuncArgs, + arg_types: &[PyTypeRef], + paramflags: &ParsedParamFlags, + skip_first_arg: bool, // true for COM methods + vm: &VirtualMachine, +) -> PyResult<(Vec<Argument>, OutBuffers)> { + let mut arguments = Vec::new(); + let mut out_buffers = Vec::new(); + + // For COM methods, first arg is self (pointer) + let mut caller_arg_idx = if skip_first_arg { + if !args.args.is_empty() { + let arg = conv_param(&args.args[0], vm)?; + arguments.push(arg); + } + 1usize + } else { + 0usize + }; + + // Process parameters based on paramflags + for (param_idx, (direction, _name, default)) in paramflags.iter().enumerate() { + let arg_type = arg_types + .get(param_idx) + .ok_or_else(|| vm.new_type_error("paramflags/argtypes mismatch"))?; + + let is_out = (*direction & 2) != 0; // OUT flag + let is_in = (*direction & 1) != 0 || *direction == 0; // IN flag or default + + let ffi_type = ArgumentType::to_ffi_type(arg_type, vm)?; + + if is_out && !is_in { + // Pure OUT parameter: create buffer, don't consume caller arg + let buffer = create_out_buffer(arg_type, vm)?; + let addr = get_buffer_addr(&buffer) + .ok_or_else(|| vm.new_type_error("Cannot create OUT buffer for this type"))?; + arguments.push(Argument { + ffi_type, + keep: None, + value: FfiArgValue::Pointer(addr), + }); + out_buffers.push((param_idx, buffer)); + } else { + // IN or IN|OUT: get from caller args or default + let arg = if caller_arg_idx < args.args.len() { + caller_arg_idx += 1; + args.args[caller_arg_idx - 1].clone() + } else if let Some(def) = default { + def.clone() + } else { + return Err(vm.new_type_error(format!("required argument {} missing", param_idx))); + }; + + if is_out { + // IN|OUT: track for return + out_buffers.push((param_idx, arg.clone())); + } + let value = arg_type.convert_object(arg, vm)?; + arguments.push(Argument { + ffi_type, + keep: None, + value, + }); + } + } + + Ok((arguments, out_buffers)) +} + +/// Build call arguments (main dispatcher) +fn build_callargs( + args: &FuncArgs, + call_info: &CallInfo, + paramflags: Option<&ParsedParamFlags>, + is_com_method: bool, + vm: &VirtualMachine, +) -> PyResult<(Vec<Argument>, OutBuffers)> { + let Some(ref arg_types) = call_info.explicit_arg_types else { + // No argtypes: use ConvParam + return build_callargs_no_argtypes(args, vm); + }; + + if let Some(pflags) = paramflags { + // Has paramflags: handle IN/OUT + build_callargs_with_paramflags(args, arg_types, pflags, is_com_method, vm) + } else if is_com_method { + // COM method without paramflags + let mut arguments = Vec::new(); + if !args.args.is_empty() { + arguments.push(conv_param(&args.args[0], vm)?); + } + for (n, arg) in args.args.iter().skip(1).enumerate() { + let arg_type = arg_types + .get(n) + .ok_or_else(|| vm.new_type_error("argument amount mismatch"))?; + let ffi_type = ArgumentType::to_ffi_type(arg_type, vm)?; + let value = arg_type.convert_object(arg.clone(), vm)?; + arguments.push(Argument { + ffi_type, + keep: None, + value, + }); + } + Ok((arguments, Vec::new())) + } else { + // Regular function + build_callargs_simple(args, arg_types, vm) + } +} + +/// Raw result from FFI call +enum RawResult { + Void, + Pointer(usize), + Value(libffi::low::ffi_arg), +} + +/// Execute FFI call +fn ctypes_callproc(code_ptr: CodePtr, arguments: &[Argument], call_info: &CallInfo) -> RawResult { + let ffi_arg_types: Vec<Type> = arguments.iter().map(|a| a.ffi_type.clone()).collect(); + let cif = Cif::new(ffi_arg_types, call_info.ffi_return_type.clone()); + let ffi_args: Vec<Arg<'_>> = arguments.iter().map(|a| a.value.as_arg()).collect(); + + if call_info.restype_is_none { + unsafe { cif.call::<()>(code_ptr, &ffi_args) }; + RawResult::Void + } else if call_info.is_pointer_return { + let result = unsafe { cif.call::<usize>(code_ptr, &ffi_args) }; + RawResult::Pointer(result) + } else { + let result = unsafe { cif.call::<libffi::low::ffi_arg>(code_ptr, &ffi_args) }; + RawResult::Value(result) + } +} + +/// Check and handle HRESULT errors (Windows) +#[cfg(windows)] +fn check_hresult(hresult: i32, zelf: &Py<PyCFuncPtr>, vm: &VirtualMachine) -> PyResult<()> { + if hresult >= 0 { + return Ok(()); + } + + if zelf.iid.read().is_some() { + // Raise COMError + let ctypes_module = vm.import("_ctypes", 0)?; + let com_error_type = ctypes_module.get_attr("COMError", vm)?; + let com_error_type = com_error_type + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("COMError is not a type"))?; + let hresult_obj: PyObjectRef = vm.ctx.new_int(hresult).into(); + let text: PyObjectRef = vm + .ctx + .new_str(format!("HRESULT: 0x{:08X}", hresult as u32)) + .into(); + let details: PyObjectRef = vm.ctx.none(); + let exc = vm.invoke_exception( + com_error_type.to_owned(), + vec![text.clone(), details.clone()], + )?; + let _ = exc.as_object().set_attr("hresult", hresult_obj, vm); + let _ = exc.as_object().set_attr("text", text, vm); + let _ = exc.as_object().set_attr("details", details, vm); + Err(exc) + } else { + // Raise OSError + let exc = vm.new_os_error(format!("HRESULT: 0x{:08X}", hresult as u32)); + let _ = exc + .as_object() + .set_attr("winerror", vm.ctx.new_int(hresult), vm); + Err(exc) + } +} + +/// Convert raw FFI result to Python object +// = GetResult +fn convert_raw_result( + raw_result: &mut RawResult, + call_info: &CallInfo, + vm: &VirtualMachine, +) -> Option<PyObjectRef> { + // Get result as bytes for type conversion + let (result_bytes, result_size) = match raw_result { + RawResult::Void => return None, + RawResult::Pointer(ptr) => { + let bytes = ptr.to_ne_bytes(); + (bytes.to_vec(), core::mem::size_of::<usize>()) + } + RawResult::Value(val) => { + let bytes = val.to_ne_bytes(); + (bytes.to_vec(), core::mem::size_of::<i64>()) + } + }; + + // 1. No restype → return as int + let restype = match &call_info.restype_obj { + None => { + // Default: return as int + let val = match raw_result { + RawResult::Pointer(p) => *p as isize, + RawResult::Value(v) => *v as isize, + RawResult::Void => return None, + }; + return Some(vm.ctx.new_int(val).into()); + } + Some(r) => r, + }; + + // 2. restype is None → return None + if restype.is(&vm.ctx.none()) { + return None; + } + + // 3. Get restype as PyType + let restype_type = match restype.clone().downcast::<PyType>() { + Ok(t) => t, + Err(_) => { + // Not a type, call it with int result + let val = match raw_result { + RawResult::Pointer(p) => *p as isize, + RawResult::Value(v) => *v as isize, + RawResult::Void => return None, + }; + return restype.call((val,), vm).ok(); + } + }; + + // 4. Get StgInfo + let stg_info = restype_type.stg_info_opt(); + + // No StgInfo → call restype with int + if stg_info.is_none() { + let val = match raw_result { + RawResult::Pointer(p) => *p as isize, + RawResult::Value(v) => *v as isize, + RawResult::Void => return None, + }; + return restype_type.as_object().call((val,), vm).ok(); + } + + let info = stg_info.unwrap(); + + // 5. Simple type with getfunc → use bytes_to_pyobject (info->getfunc) + // is_simple_instance returns TRUE for c_int, c_void_p, etc. + if super::base::is_simple_instance(&restype_type) { + return super::base::bytes_to_pyobject(&restype_type, &result_bytes, vm).ok(); + } + + // 6. Complex type → create ctypes instance (PyCData_FromBaseObj) + // This handles POINTER(T), Structure, Array, etc. + + // Special handling for POINTER(T) types - set pointer value directly + if info.flags.contains(StgInfoFlags::TYPEFLAG_ISPOINTER) + && info.proto.is_some() + && let RawResult::Pointer(ptr) = raw_result + && let Ok(instance) = restype_type.as_object().call((), vm) + { + if let Some(pointer) = instance.downcast_ref::<PyCPointer>() { + pointer.set_ptr_value(*ptr); + } + return Some(instance); + } + + // Create instance and copy result data + pycdata_from_ffi_result(&restype_type, &result_bytes, result_size, vm).ok() +} + +/// Create a ctypes instance from FFI result (PyCData_FromBaseObj equivalent) +fn pycdata_from_ffi_result( + typ: &PyTypeRef, + result_bytes: &[u8], + size: usize, + vm: &VirtualMachine, +) -> PyResult { + // Create instance + let instance = PyType::call(typ, ().into(), vm)?; + + // Copy result data into instance buffer + if let Some(cdata) = instance.downcast_ref::<PyCData>() { + let mut buffer = cdata.buffer.write(); + let copy_size = size.min(buffer.len()).min(result_bytes.len()); + if copy_size > 0 { + buffer.to_mut()[..copy_size].copy_from_slice(&result_bytes[..copy_size]); + } + } + + Ok(instance) +} + +/// Extract values from OUT buffers +fn extract_out_values( + out_buffers: Vec<(usize, PyObjectRef)>, + vm: &VirtualMachine, +) -> Vec<PyObjectRef> { + out_buffers + .into_iter() + .map(|(_, buffer)| buffer.get_attr("value", vm).unwrap_or(buffer)) + .collect() +} + +/// Build final result (main function) +fn build_result( + mut raw_result: RawResult, + call_info: &CallInfo, + out_buffers: OutBuffers, + zelf: &Py<PyCFuncPtr>, + args: &FuncArgs, + vm: &VirtualMachine, +) -> PyResult { + // Check HRESULT on Windows + #[cfg(windows)] + if let RawResult::Value(val) = raw_result { + let is_hresult = call_info + .restype_obj + .as_ref() + .and_then(|t| t.clone().downcast::<PyType>().ok()) + .is_some_and(|t| t.name().to_string() == "HRESULT"); + if is_hresult { + check_hresult(val as i32, zelf, vm)?; + } + } + + // Convert raw result to Python object + let mut result = convert_raw_result(&mut raw_result, call_info, vm); + + // Apply errcheck if set + if let Some(errcheck) = zelf.errcheck.read().as_ref() { + let args_tuple = PyTuple::new_ref(args.args.clone(), &vm.ctx); + let func_obj = zelf.as_object().to_owned(); + let result_obj = result.clone().unwrap_or_else(|| vm.ctx.none()); + result = Some(errcheck.call((result_obj, func_obj, args_tuple), vm)?); + } + + // Handle OUT parameter return values + if out_buffers.is_empty() { + return result.map(Ok).unwrap_or_else(|| Ok(vm.ctx.none())); + } + + let out_values = extract_out_values(out_buffers, vm); + Ok(match <[PyObjectRef; 1]>::try_from(out_values) { + Ok([single]) => single, + Err(v) => PyTuple::new_ref(v, &vm.ctx).into(), + }) +} + +impl Callable for PyCFuncPtr { + type Args = FuncArgs; + fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult { + // 1. Check for internal PYFUNCTYPE addresses + if let Some(result) = handle_internal_func(zelf.get_func_ptr(), &args, vm) { + return result; + } + + // 2. Resolve function pointer (COM or direct) + #[cfg(windows)] + let (func_ptr, is_com_method) = resolve_com_method(zelf, &args, vm)?; + #[cfg(not(windows))] + let (func_ptr, is_com_method) = (None::<CodePtr>, false); + + // 3. Extract call info (argtypes, restype) + let call_info = extract_call_info(zelf, vm)?; + + // 4. Parse paramflags + let paramflags = parse_paramflags(zelf, vm)?; + + // 5. Build call arguments + let (arguments, out_buffers) = + build_callargs(&args, &call_info, paramflags.as_ref(), is_com_method, vm)?; + + // 6. Get code pointer + let code_ptr = match func_ptr.or_else(|| zelf.get_code_ptr()) { + Some(cp) => cp, + None => { + debug_assert!(false, "NULL function pointer"); + // In release mode, this will crash + CodePtr(core::ptr::null_mut()) + } + }; + + // 7. Get flags to check for use_last_error/use_errno + let flags = PyCFuncPtr::_flags_(zelf, vm); + + // 8. Call the function (with use_last_error/use_errno handling) + #[cfg(not(windows))] + let raw_result = { + if flags & super::base::StgInfoFlags::FUNCFLAG_USE_ERRNO.bits() != 0 { + swap_errno(|| ctypes_callproc(code_ptr, &arguments, &call_info)) + } else { + ctypes_callproc(code_ptr, &arguments, &call_info) + } + }; + + #[cfg(windows)] + let raw_result = { + if flags & super::base::StgInfoFlags::FUNCFLAG_USE_LASTERROR.bits() != 0 { + save_and_restore_last_error(|| ctypes_callproc(code_ptr, &arguments, &call_info)) + } else { + ctypes_callproc(code_ptr, &arguments, &call_info) + } + }; + + // 9. Build result + build_result(raw_result, &call_info, out_buffers, zelf, &args, vm) + } +} + +// PyCFuncPtr_repr +impl Representable for PyCFuncPtr { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let type_name = zelf.class().name(); + // Use object id, not function pointer address + let addr = zelf.get_id(); + Ok(format!("<{} object at {:#x}>", type_name, addr)) + } +} + +// PyCData_NewGetBuffer +impl AsBuffer for PyCFuncPtr { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + // CFuncPtr types may not have StgInfo if PyCFuncPtrType metaclass is not used + // Use default values for function pointers: format="X{}", size=sizeof(pointer) + let (format, itemsize) = if let Some(stg_info) = zelf.class().stg_info_opt() { + ( + stg_info + .format + .clone() + .map(Cow::Owned) + .unwrap_or(Cow::Borrowed("X{}")), + stg_info.size, + ) + } else { + (Cow::Borrowed("X{}"), core::mem::size_of::<usize>()) + }; + let desc = BufferDescriptor { + len: itemsize, + readonly: false, + itemsize, + format, + dim_desc: vec![], + }; + let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); + Ok(buf) + } +} + +#[pyclass( + flags(BASETYPE), + with(Callable, Constructor, AsNumber, Representable, AsBuffer) +)] +impl PyCFuncPtr { + // restype getter/setter + #[pygetset] + fn restype(&self) -> Option<PyObjectRef> { + self.restype.read().clone() + } + + #[pygetset(setter)] + fn set_restype(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Must be type, callable, or None + if vm.is_none(&value) { + *self.restype.write() = None; + } else if value.downcast_ref::<PyType>().is_some() || value.is_callable() { + *self.restype.write() = Some(value); + } else { + return Err(vm.new_type_error("restype must be a type, a callable, or None")); + } + Ok(()) + } + + // argtypes getter/setter + #[pygetset] + fn argtypes(&self, vm: &VirtualMachine) -> PyObjectRef { + self.argtypes + .read() + .clone() + .unwrap_or_else(|| vm.ctx.empty_tuple.clone().into()) + } + + #[pygetset(name = "argtypes", setter)] + fn set_argtypes(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if vm.is_none(&value) { + *self.argtypes.write() = None; + } else { + // Store the argtypes object directly as it is + *self.argtypes.write() = Some(value); + } + Ok(()) + } + + // errcheck getter/setter + #[pygetset] + fn errcheck(&self) -> Option<PyObjectRef> { + self.errcheck.read().clone() + } + + #[pygetset(setter)] + fn set_errcheck(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if vm.is_none(&value) { + *self.errcheck.write() = None; + } else if value.is_callable() { + *self.errcheck.write() = Some(value); + } else { + return Err(vm.new_type_error("errcheck must be a callable or None")); + } + Ok(()) + } + + // _flags_ getter (read-only, from type's class attribute or StgInfo) + #[pygetset] + fn _flags_(zelf: &Py<Self>, vm: &VirtualMachine) -> u32 { + // First try to get _flags_ from type's class attribute (for dynamically created types) + // This is how CDLL sets use_errno: class _FuncPtr(_CFuncPtr): _flags_ = flags + if let Ok(flags_attr) = zelf.class().as_object().get_attr("_flags_", vm) + && let Ok(flags_int) = flags_attr.try_to_value::<u32>(vm) + { + return flags_int; + } + + // Fallback to StgInfo for native types + zelf.class() + .stg_info_opt() + .map(|stg| stg.flags.bits()) + .unwrap_or(StgInfoFlags::empty().bits()) + } +} + +impl AsNumber for PyCFuncPtr { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyCFuncPtr>().unwrap(); + Ok(zelf.get_func_ptr() != 0) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +// CThunkObject - FFI callback (thunk) implementation + +/// Userdata passed to the libffi callback. +struct ThunkUserData { + /// The Python callable to invoke + callable: PyObjectRef, + /// Argument types for conversion + arg_types: Vec<PyTypeRef>, + /// Result type for conversion (None means void) + pub res_type: Option<PyTypeRef>, + /// Function flags (FUNCFLAG_USE_ERRNO, etc.) + pub flags: u32, +} + +/// Check if ty is a subclass of a simple type (like MyInt(c_int)). +fn is_simple_subclass(ty: &Py<PyType>, vm: &VirtualMachine) -> bool { + let Ok(base) = ty.as_object().get_attr(vm.ctx.intern_str("__base__"), vm) else { + return false; + }; + base.get_attr(vm.ctx.intern_str("_type_"), vm).is_ok() +} + +/// Convert a C value to a Python object based on the type code. +fn ffi_to_python(ty: &Py<PyType>, ptr: *const c_void, vm: &VirtualMachine) -> PyObjectRef { + let type_code = ty.type_code(vm); + let raw_value: PyObjectRef = unsafe { + match type_code.as_deref() { + Some("b") => vm.ctx.new_int(*(ptr as *const i8) as i32).into(), + Some("B") => vm.ctx.new_int(*(ptr as *const u8) as i32).into(), + Some("c") => vm.ctx.new_bytes(vec![*(ptr as *const u8)]).into(), + Some("h") => vm.ctx.new_int(*(ptr as *const i16) as i32).into(), + Some("H") => vm.ctx.new_int(*(ptr as *const u16) as i32).into(), + Some("i") => vm.ctx.new_int(*(ptr as *const i32)).into(), + Some("I") => vm.ctx.new_int(*(ptr as *const u32)).into(), + Some("l") => vm.ctx.new_int(*(ptr as *const libc::c_long)).into(), + Some("L") => vm.ctx.new_int(*(ptr as *const libc::c_ulong)).into(), + Some("q") => vm.ctx.new_int(*(ptr as *const libc::c_longlong)).into(), + Some("Q") => vm.ctx.new_int(*(ptr as *const libc::c_ulonglong)).into(), + Some("f") => vm.ctx.new_float(*(ptr as *const f32) as f64).into(), + Some("d") => vm.ctx.new_float(*(ptr as *const f64)).into(), + Some("z") => { + // c_char_p: C string pointer → Python bytes + let cstr_ptr = *(ptr as *const *const libc::c_char); + if cstr_ptr.is_null() { + vm.ctx.none() + } else { + let cstr = core::ffi::CStr::from_ptr(cstr_ptr); + vm.ctx.new_bytes(cstr.to_bytes().to_vec()).into() + } + } + Some("Z") => { + // c_wchar_p: wchar_t* → Python str + let wstr_ptr = *(ptr as *const *const libc::wchar_t); + if wstr_ptr.is_null() { + vm.ctx.none() + } else { + let mut len = 0; + while *wstr_ptr.add(len) != 0 { + len += 1; + } + let slice = core::slice::from_raw_parts(wstr_ptr, len); + // Windows: wchar_t = u16 (UTF-16) -> use Wtf8Buf::from_wide + // Unix: wchar_t = i32 (UTF-32) -> convert via char::from_u32 + #[cfg(windows)] + { + use rustpython_common::wtf8::Wtf8Buf; + let wide: Vec<u16> = slice.to_vec(); + let wtf8 = Wtf8Buf::from_wide(&wide); + vm.ctx.new_str(wtf8).into() + } + #[cfg(not(windows))] + { + #[allow( + clippy::useless_conversion, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + let s: String = slice + .iter() + .filter_map(|&c| u32::try_from(c).ok().and_then(char::from_u32)) + .collect(); + vm.ctx.new_str(s).into() + } + } + } + Some("P") => vm.ctx.new_int(*(ptr as *const usize)).into(), + Some("?") => vm.ctx.new_bool(*(ptr as *const u8) != 0).into(), + _ => return vm.ctx.none(), + } + }; + + if !is_simple_subclass(ty, vm) { + return raw_value; + } + ty.as_object() + .call((raw_value.clone(),), vm) + .unwrap_or(raw_value) +} + +/// Convert a Python object to a C value and store it at the result pointer +fn python_to_ffi(obj: PyResult, ty: &Py<PyType>, result: *mut c_void, vm: &VirtualMachine) { + let Ok(obj) = obj else { return }; + + let type_code = ty.type_code(vm); + unsafe { + match type_code.as_deref() { + Some("b") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut i8) = i.as_bigint().to_i8().unwrap_or(0); + } + } + Some("B") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u8) = i.as_bigint().to_u8().unwrap_or(0); + } + } + Some("c") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u8) = i.as_bigint().to_u8().unwrap_or(0); + } + } + Some("h") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut i16) = i.as_bigint().to_i16().unwrap_or(0); + } + } + Some("H") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u16) = i.as_bigint().to_u16().unwrap_or(0); + } + } + Some("i") => { + if let Ok(i) = obj.try_int(vm) { + let val = i.as_bigint().to_i32().unwrap_or(0); + *(result as *mut libffi::low::ffi_arg) = val as libffi::low::ffi_arg; + } + } + Some("I") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u32) = i.as_bigint().to_u32().unwrap_or(0); + } + } + Some("l") | Some("q") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut i64) = i.as_bigint().to_i64().unwrap_or(0); + } + } + Some("L") | Some("Q") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut u64) = i.as_bigint().to_u64().unwrap_or(0); + } + } + Some("f") => { + if let Ok(f) = obj.try_float(vm) { + *(result as *mut f32) = f.to_f64() as f32; + } + } + Some("d") => { + if let Ok(f) = obj.try_float(vm) { + *(result as *mut f64) = f.to_f64(); + } + } + Some("P") | Some("z") | Some("Z") => { + if let Ok(i) = obj.try_int(vm) { + *(result as *mut usize) = i.as_bigint().to_usize().unwrap_or(0); + } + } + Some("?") => { + if let Ok(b) = obj.is_true(vm) { + *(result as *mut u8) = u8::from(b); + } + } + _ => {} + } + } +} + +/// The callback function that libffi calls when the closure is invoked. +unsafe extern "C" fn thunk_callback( + _cif: &low::ffi_cif, + result: &mut c_void, + args: *const *const c_void, + userdata: &ThunkUserData, +) { + with_current_vm(|vm| { + // Swap errno before call if FUNCFLAG_USE_ERRNO is set + let use_errno = userdata.flags & StgInfoFlags::FUNCFLAG_USE_ERRNO.bits() != 0; + let saved_errno = if use_errno { + let current = rustpython_common::os::get_errno(); + // TODO: swap with ctypes stored errno (thread-local) + Some(current) + } else { + None + }; + + let py_args: Vec<PyObjectRef> = userdata + .arg_types + .iter() + .enumerate() + .map(|(i, ty)| { + let arg_ptr = unsafe { *args.add(i) }; + ffi_to_python(ty, arg_ptr, vm) + }) + .collect(); + + let py_result = userdata.callable.call(py_args, vm); + + // Swap errno back after call + if use_errno { + let _current = rustpython_common::os::get_errno(); + // TODO: store current errno to ctypes storage + if let Some(saved) = saved_errno { + rustpython_common::os::set_errno(saved); + } + } + + // Call unraisable hook if exception occurred + if let Err(exc) = &py_result { + let repr = userdata + .callable + .repr(vm) + .map(|s| s.to_string()) + .unwrap_or_else(|_| "<unknown>".to_string()); + let msg = format!( + "Exception ignored while calling ctypes callback function {}", + repr + ); + vm.run_unraisable(exc.clone(), Some(msg), vm.ctx.none()); + } + + if let Some(ref res_type) = userdata.res_type { + python_to_ffi(py_result, res_type, result as *mut c_void, vm); + } + }); +} + +/// Holds the closure and userdata together to ensure proper lifetime. +struct ThunkData { + #[allow(dead_code)] + closure: Closure<'static>, + userdata_ptr: *mut ThunkUserData, +} + +impl Drop for ThunkData { + fn drop(&mut self) { + unsafe { + drop(Box::from_raw(self.userdata_ptr)); + } + } +} + +/// CThunkObject wraps a Python callable to make it callable from C code. +#[pyclass(name = "CThunkObject", module = "_ctypes")] +#[derive(PyPayload)] +pub(super) struct PyCThunk { + callable: PyObjectRef, + #[allow(dead_code)] + thunk_data: PyRwLock<Option<ThunkData>>, + code_ptr: CodePtr, +} + +impl Debug for PyCThunk { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyCThunk") + .field("callable", &self.callable) + .finish() + } +} + +impl PyCThunk { + pub fn new( + callable: PyObjectRef, + arg_types: Option<PyObjectRef>, + res_type: Option<PyObjectRef>, + flags: u32, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let arg_type_vec: Vec<PyTypeRef> = match arg_types { + Some(args) if !vm.is_none(&args) => args + .try_to_value::<Vec<PyObjectRef>>(vm)? + .into_iter() + .map(|item| { + item.downcast::<PyType>() + .map_err(|_| vm.new_type_error("_argtypes_ must be a sequence of types")) + }) + .collect::<PyResult<Vec<_>>>()?, + _ => Vec::new(), + }; + + let res_type_ref: Option<PyTypeRef> = match res_type { + Some(ref rt) if !vm.is_none(rt) => Some( + rt.clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("restype must be a ctypes type"))?, + ), + _ => None, + }; + + let ffi_arg_types: Vec<Type> = arg_type_vec + .iter() + .map(|ty| { + ty.type_code(vm) + .and_then(|code| get_ffi_type(&code)) + .unwrap_or(Type::pointer()) + }) + .collect(); + + let ffi_res_type = res_type_ref + .as_ref() + .and_then(|ty| ty.type_code(vm)) + .and_then(|code| get_ffi_type(&code)) + .unwrap_or(Type::void()); + + let cif = Cif::new(ffi_arg_types, ffi_res_type); + + let userdata = Box::new(ThunkUserData { + callable: callable.clone(), + arg_types: arg_type_vec, + res_type: res_type_ref, + flags, + }); + let userdata_ptr = Box::into_raw(userdata); + let userdata_ref: &'static ThunkUserData = unsafe { &*userdata_ptr }; + + let closure = Closure::new(cif, thunk_callback, userdata_ref); + let code_ptr = CodePtr(*closure.code_ptr() as *mut _); + + let thunk_data = ThunkData { + closure, + userdata_ptr, + }; + + Ok(Self { + callable, + thunk_data: PyRwLock::new(Some(thunk_data)), + code_ptr, + }) + } + + pub fn code_ptr(&self) -> CodePtr { + self.code_ptr + } +} + +unsafe impl Send for PyCThunk {} +unsafe impl Sync for PyCThunk {} + +#[pyclass] +impl PyCThunk { + #[pygetset] + fn callable(&self) -> PyObjectRef { + self.callable.clone() + } +} diff --git a/crates/vm/src/stdlib/_ctypes/library.rs b/crates/vm/src/stdlib/_ctypes/library.rs new file mode 100644 index 00000000000..35ccb433845 --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes/library.rs @@ -0,0 +1,150 @@ +use crate::VirtualMachine; +use alloc::fmt; +use libloading::Library; +use rustpython_common::lock::{PyMutex, PyRwLock}; +use std::collections::HashMap; +use std::ffi::OsStr; + +#[cfg(unix)] +use libloading::os::unix::Library as UnixLibrary; + +pub struct SharedLibrary { + pub(crate) lib: PyMutex<Option<Library>>, +} + +impl fmt::Debug for SharedLibrary { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SharedLibrary") + } +} + +impl SharedLibrary { + #[cfg(windows)] + pub fn new(name: impl AsRef<OsStr>) -> Result<SharedLibrary, libloading::Error> { + Ok(SharedLibrary { + lib: PyMutex::new(unsafe { Some(Library::new(name.as_ref())?) }), + }) + } + + #[cfg(unix)] + pub fn new_with_mode( + name: impl AsRef<OsStr>, + mode: i32, + ) -> Result<SharedLibrary, libloading::Error> { + Ok(SharedLibrary { + lib: PyMutex::new(Some(unsafe { + UnixLibrary::open(Some(name.as_ref()), mode)?.into() + })), + }) + } + + /// Create a SharedLibrary from a raw dlopen handle (for pythonapi / dlopen(NULL)) + #[cfg(unix)] + pub fn from_raw_handle(handle: *mut libc::c_void) -> SharedLibrary { + SharedLibrary { + lib: PyMutex::new(Some(unsafe { UnixLibrary::from_raw(handle).into() })), + } + } + + /// Get the underlying OS handle (HMODULE on Windows, dlopen handle on Unix) + pub fn get_pointer(&self) -> usize { + let lib_lock = self.lib.lock(); + if let Some(l) = &*lib_lock { + // libloading::Library internally stores the OS handle directly + // On Windows: HMODULE (*mut c_void) + // On Unix: *mut c_void from dlopen + // We use transmute_copy to read the handle without consuming the Library + unsafe { core::mem::transmute_copy::<Library, usize>(l) } + } else { + 0 + } + } + + fn is_closed(&self) -> bool { + let lib_lock = self.lib.lock(); + lib_lock.is_none() + } +} + +pub(super) struct ExternalLibs { + libraries: HashMap<usize, SharedLibrary>, +} + +impl ExternalLibs { + fn new() -> Self { + Self { + libraries: HashMap::new(), + } + } + + pub fn get_lib(&self, key: usize) -> Option<&SharedLibrary> { + self.libraries.get(&key) + } + + #[cfg(windows)] + pub fn get_or_insert_lib( + &mut self, + library_path: impl AsRef<OsStr>, + _vm: &VirtualMachine, + ) -> Result<(usize, &SharedLibrary), libloading::Error> { + let new_lib = SharedLibrary::new(library_path)?; + let key = new_lib.get_pointer(); + + // Check if library already exists and is not closed + let should_use_cached = self.libraries.get(&key).is_some_and(|l| !l.is_closed()); + + if should_use_cached { + // new_lib will be dropped, calling FreeLibrary (decrements refcount) + // But library stays loaded because cached version maintains refcount + drop(new_lib); + return Ok((key, self.libraries.get(&key).expect("just checked"))); + } + + self.libraries.insert(key, new_lib); + Ok((key, self.libraries.get(&key).expect("just inserted"))) + } + + #[cfg(unix)] + pub fn get_or_insert_lib_with_mode( + &mut self, + library_path: impl AsRef<OsStr>, + mode: i32, + _vm: &VirtualMachine, + ) -> Result<(usize, &SharedLibrary), libloading::Error> { + let new_lib = SharedLibrary::new_with_mode(library_path, mode)?; + let key = new_lib.get_pointer(); + + // Check if library already exists and is not closed + let should_use_cached = self.libraries.get(&key).is_some_and(|l| !l.is_closed()); + + if should_use_cached { + // new_lib will be dropped, calling dlclose (decrements refcount) + // But library stays loaded because cached version maintains refcount + drop(new_lib); + return Ok((key, self.libraries.get(&key).expect("just checked"))); + } + + self.libraries.insert(key, new_lib); + Ok((key, self.libraries.get(&key).expect("just inserted"))) + } + + /// Insert a raw dlopen handle into the cache (for pythonapi / dlopen(NULL)) + #[cfg(unix)] + pub fn insert_raw_handle(&mut self, handle: *mut libc::c_void) -> usize { + let shared_lib = SharedLibrary::from_raw_handle(handle); + let key = handle as usize; + self.libraries.insert(key, shared_lib); + key + } + + pub fn drop_lib(&mut self, key: usize) { + self.libraries.remove(&key); + } +} + +pub(super) fn libcache() -> &'static PyRwLock<ExternalLibs> { + rustpython_common::static_cell! { + static LIBCACHE: PyRwLock<ExternalLibs>; + } + LIBCACHE.get_or_init(|| PyRwLock::new(ExternalLibs::new())) +} diff --git a/crates/vm/src/stdlib/_ctypes/pointer.rs b/crates/vm/src/stdlib/_ctypes/pointer.rs new file mode 100644 index 00000000000..503af1f1d56 --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes/pointer.rs @@ -0,0 +1,866 @@ +use super::base::CDATA_BUFFER_METHODS; +use super::{PyCArray, PyCData, PyCSimple, PyCStructure, StgInfo, StgInfoFlags}; +use crate::atomic_func; +use crate::protocol::{BufferDescriptor, PyBuffer, PyMappingMethods, PyNumberMethods}; +use crate::types::{AsBuffer, AsMapping, AsNumber, Constructor, Initializer}; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBytes, PyInt, PyList, PySlice, PyStr, PyType, PyTypeRef}, + class::StaticType, + function::{FuncArgs, OptionalArg}, +}; +use alloc::borrow::Cow; +use num_traits::ToPrimitive; + +#[pyclass(name = "PyCPointerType", base = PyType, module = "_ctypes")] +#[derive(Debug)] +#[repr(transparent)] +pub(super) struct PyCPointerType(PyType); + +impl Initializer for PyCPointerType { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Get the type as PyTypeRef + let obj: PyObjectRef = zelf.clone().into(); + let new_type: PyTypeRef = obj + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + if new_type.is_initialized() { + return Ok(()); + } + + // Get the _type_ attribute (element type) + let proto = new_type + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|obj| obj.downcast::<PyType>().ok()); + + // Validate that _type_ has storage info (is a ctypes type) + if let Some(ref proto_type) = proto + && proto_type.stg_info_opt().is_none() + { + return Err(vm.new_type_error(format!("{} must have storage info", proto_type.name()))); + } + + // Initialize StgInfo for pointer type + let pointer_size = core::mem::size_of::<usize>(); + let mut stg_info = StgInfo::new(pointer_size, pointer_size); + stg_info.proto = proto; + stg_info.paramfunc = super::base::ParamFunc::Pointer; + stg_info.length = 1; + stg_info.flags |= StgInfoFlags::TYPEFLAG_ISPOINTER; + + // Set format string: "&<element_format>" or "&(shape)<element_format>" for arrays + if let Some(ref proto) = stg_info.proto + && let Some(item_info) = proto.stg_info_opt() + { + let current_format = item_info.format.as_deref().unwrap_or("B"); + // Include shape for array types in the pointer format + let shape_str = if !item_info.shape.is_empty() { + let dims: Vec<String> = item_info.shape.iter().map(|d| d.to_string()).collect(); + format!("({})", dims.join(",")) + } else { + String::new() + }; + stg_info.format = Some(format!("&{}{}", shape_str, current_format)); + } + + let _ = new_type.init_type_data(stg_info); + + // Cache: set target_type.__pointer_type__ = self (via StgInfo, not as inheritable attr) + if let Ok(type_attr) = new_type.as_object().get_attr("_type_", vm) + && let Ok(target_type) = type_attr.downcast::<PyType>() + && let Some(mut target_info) = target_type.get_type_data_mut::<StgInfo>() + { + let zelf_obj: PyObjectRef = zelf.into(); + target_info.pointer_type = Some(zelf_obj); + } + + Ok(()) + } +} + +#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(AsNumber, Initializer))] +impl PyCPointerType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the pointer type class that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. None is allowed for pointer types + if vm.is_none(&value) { + return Ok(value); + } + + // 1.5 CArgObject (from byref()) - check if underlying obj is instance of _type_ + if let Some(carg) = value.downcast_ref::<super::_ctypes::CArgObject>() + && let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) + && let Ok(type_ref) = type_attr.downcast::<PyType>() + && carg.obj.is_instance(type_ref.as_object(), vm)? + { + return Ok(value); + } + + // 2. If already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } + + // 3. If value is an instance of _type_ (the pointed-to type), wrap with byref + if let Ok(type_attr) = cls.as_object().get_attr("_type_", vm) + && let Ok(type_ref) = type_attr.downcast::<PyType>() + && value.is_instance(type_ref.as_object(), vm)? + { + // Return byref(value) + return super::_ctypes::byref(value, crate::function::OptionalArg::Missing, vm); + } + + // 4. Array/Pointer instances with compatible proto + // "Array instances are also pointers when the item types are the same." + let is_pointer_or_array = value.downcast_ref::<PyCPointer>().is_some() + || value.downcast_ref::<super::array::PyCArray>().is_some(); + + if is_pointer_or_array { + let is_compatible = { + if let Some(value_stginfo) = value.class().stg_info_opt() + && let Some(ref value_proto) = value_stginfo.proto + && let Some(cls_stginfo) = cls.stg_info_opt() + && let Some(ref cls_proto) = cls_stginfo.proto + { + // Check if value's proto is a subclass of target's proto + value_proto.fast_issubclass(cls_proto) + } else { + false + } + }; + if is_compatible { + return Ok(value); + } + } + + // 5. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCPointerType::from_param(cls.as_object().to_owned(), as_parameter, vm); + } + + Err(vm.new_type_error(format!( + "expected {} instance instead of {}", + cls.name(), + value.class().name() + ))) + } + + fn __mul__(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { + use super::array::array_type_from_ctype; + + if n < 0 { + return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); + } + // Use cached array type creation + array_type_from_ctype(cls.into(), n as usize, vm) + } + + // PyCPointerType_set_type: Complete an incomplete pointer type + #[pymethod] + fn set_type(zelf: PyTypeRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + use crate::AsObject; + + // 1. Validate that typ is a type + let typ_type = typ + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("_type_ must be a type"))?; + + // 2. Validate that typ has storage info + if typ_type.stg_info_opt().is_none() { + return Err(vm.new_type_error("_type_ must have storage info")); + } + + // 3. Update StgInfo.proto and format using mutable access + if let Some(mut stg_info) = zelf.get_type_data_mut::<StgInfo>() { + stg_info.proto = Some(typ_type.clone()); + + // Update format string: "&<element_format>" or "&(shape)<element_format>" for arrays + let item_info = typ_type.stg_info_opt().expect("proto has StgInfo"); + let current_format = item_info.format.as_deref().unwrap_or("B"); + // Include shape for array types in the pointer format + let shape_str = if !item_info.shape.is_empty() { + let dims: Vec<String> = item_info.shape.iter().map(|d| d.to_string()).collect(); + format!("({})", dims.join(",")) + } else { + String::new() + }; + stg_info.format = Some(format!("&{}{}", shape_str, current_format)); + } + + // 4. Set _type_ attribute on the pointer type + zelf.as_object().set_attr("_type_", typ_type.clone(), vm)?; + + // 5. Cache: set target_type.__pointer_type__ = self (via StgInfo) + if let Some(mut target_info) = typ_type.get_type_data_mut::<StgInfo>() { + target_info.pointer_type = Some(zelf.into()); + } + + Ok(()) + } +} + +impl AsNumber for PyCPointerType { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + multiply: Some(|a, b, vm| { + let cls = a + .downcast_ref::<PyType>() + .ok_or_else(|| vm.new_type_error("expected type"))?; + let n = b + .try_index(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("array size too large"))?; + PyCPointerType::__mul__(cls.to_owned(), n, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +/// PyCPointer - Pointer instance +/// `contents` is a computed property, not a stored field. +#[pyclass( + name = "_Pointer", + base = PyCData, + metaclass = "PyCPointerType", + module = "_ctypes" +)] +#[derive(Debug)] +#[repr(transparent)] +pub struct PyCPointer(pub PyCData); + +impl Constructor for PyCPointer { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Pointer_new: Check if _type_ is defined + let has_type = cls.stg_info_opt().is_some_and(|info| info.proto.is_some()); + if !has_type { + return Err(vm.new_type_error("Cannot create instance: has no _type_")); + } + + // Create a new PyCPointer instance with NULL pointer (all zeros) + // Initial contents is set via __init__ if provided + let cdata = PyCData::from_bytes(vec![0u8; core::mem::size_of::<usize>()], None); + // pointer instance has b_length set to 2 (for index 0 and 1) + cdata.length.store(2); + PyCPointer(cdata) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyCPointer { + type Args = (OptionalArg<PyObjectRef>,); + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let (value,) = args; + if let OptionalArg::Present(val) = value + && !vm.is_none(&val) + { + Self::set_contents(&zelf, val, vm)?; + } + Ok(()) + } +} + +#[pyclass( + flags(BASETYPE, IMMUTABLETYPE), + with(Constructor, Initializer, AsNumber, AsBuffer, AsMapping) +)] +impl PyCPointer { + /// Get the pointer value stored in buffer as usize + pub fn get_ptr_value(&self) -> usize { + let buffer = self.0.buffer.read(); + super::base::read_ptr_from_buffer(&buffer) + } + + /// Set the pointer value in buffer + pub fn set_ptr_value(&self, value: usize) { + let mut buffer = self.0.buffer.write(); + let bytes = value.to_ne_bytes(); + if buffer.len() >= bytes.len() { + buffer.to_mut()[..bytes.len()].copy_from_slice(&bytes); + } + } + + /// contents getter - reads address from b_ptr and creates an instance of the pointed-to type + #[pygetset] + fn contents(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Pointer_get_contents + let ptr_val = zelf.get_ptr_value(); + if ptr_val == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + + // Get element type from StgInfo.proto + let stg_info = zelf.class().stg_info(vm)?; + let proto_type = stg_info.proto(); + let element_size = proto_type + .stg_info_opt() + .map_or(core::mem::size_of::<usize>(), |info| info.size); + + // Create instance that references the memory directly + // PyCData.into_ref_with_type works for all ctypes (simple, structure, union, array, pointer) + let cdata = unsafe { super::base::PyCData::at_address(ptr_val as *const u8, element_size) }; + cdata + .into_ref_with_type(vm, proto_type.to_owned()) + .map(Into::into) + } + + /// contents setter - stores address in b_ptr and keeps reference + /// Pointer_set_contents + #[pygetset(setter)] + fn set_contents(zelf: &Py<Self>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Get stginfo and proto for type validation + let stg_info = zelf.class().stg_info(vm)?; + let proto = stg_info.proto(); + + // Check if value is CData, or isinstance(value, proto) + let cdata = if let Some(c) = value.downcast_ref::<PyCData>() { + c + } else if value.is_instance(proto.as_object(), vm)? { + value + .downcast_ref::<PyCData>() + .ok_or_else(|| vm.new_type_error("expected ctypes instance"))? + } else { + return Err(vm.new_type_error(format!( + "expected {} instead of {}", + proto.name(), + value.class().name() + ))); + }; + + // Set pointer value + { + let buffer = cdata.buffer.read(); + let addr = buffer.as_ptr() as usize; + drop(buffer); + zelf.set_ptr_value(addr); + } + + // KeepRef: store the object directly with index 1 + zelf.0.keep_ref(1, value.clone(), vm)?; + + // KeepRef: store GetKeepedObjects(dst) at index 0 + if let Some(kept) = cdata.objects.read().clone() { + zelf.0.keep_ref(0, kept, vm)?; + } + + Ok(()) + } + + // Pointer_subscript + fn __getitem__(zelf: &Py<Self>, item: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // PyIndex_Check + if let Some(i) = item.downcast_ref::<PyInt>() { + let i = i.as_bigint().to_isize().ok_or_else(|| { + vm.new_index_error("cannot fit index into an index-sized integer") + })?; + // Note: Pointer does NOT adjust negative indices (no length) + Self::getitem_by_index(zelf, i, vm) + } + // PySlice_Check + else if let Some(slice) = item.downcast_ref::<PySlice>() { + Self::getitem_by_slice(zelf, slice, vm) + } else { + Err(vm.new_type_error("Pointer indices must be integer")) + } + } + + // Pointer_item + fn getitem_by_index(zelf: &Py<Self>, index: isize, vm: &VirtualMachine) -> PyResult { + // if (*(void **)self->b_ptr == NULL) { PyErr_SetString(...); } + let ptr_value = zelf.get_ptr_value(); + if ptr_value == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + + // Get element type and size from StgInfo.proto + let stg_info = zelf.class().stg_info(vm)?; + let proto_type = stg_info.proto(); + let element_size = proto_type + .stg_info_opt() + .map_or(core::mem::size_of::<usize>(), |info| info.size); + + // offset = index * iteminfo->size + let offset = index * element_size as isize; + let addr = (ptr_value as isize + offset) as usize; + + // Check if it's a simple type (has _type_ attribute) + if let Ok(type_attr) = proto_type.as_object().get_attr("_type_", vm) + && let Ok(type_str) = type_attr.str(vm) + { + let type_code = type_str.to_string(); + return Self::read_value_at_address(addr, element_size, Some(&type_code), vm); + } + + // Complex type: create instance that references the memory directly (not a copy) + // This allows p[i].val = x to modify the original memory + // PyCData.into_ref_with_type works for all ctypes (array, structure, union, pointer) + let cdata = unsafe { super::base::PyCData::at_address(addr as *const u8, element_size) }; + cdata + .into_ref_with_type(vm, proto_type.to_owned()) + .map(Into::into) + } + + // Pointer_subscript slice handling (manual parsing, not PySlice_Unpack) + fn getitem_by_slice(zelf: &Py<Self>, slice: &PySlice, vm: &VirtualMachine) -> PyResult { + // Since pointers have no length, we have to dissect the slice ourselves + + // step: defaults to 1, step == 0 is error + let step: isize = if let Some(ref step_obj) = slice.step + && !vm.is_none(step_obj) + { + let s = step_obj + .try_int(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("slice step too large"))?; + if s == 0 { + return Err(vm.new_value_error("slice step cannot be zero")); + } + s + } else { + 1 + }; + + // start: defaults to 0, but required if step < 0 + let start: isize = if let Some(ref start_obj) = slice.start + && !vm.is_none(start_obj) + { + start_obj + .try_int(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("slice start too large"))? + } else { + if step < 0 { + return Err(vm.new_value_error("slice start is required for step < 0")); + } + 0 + }; + + // stop: ALWAYS required for pointers + if vm.is_none(&slice.stop) { + return Err(vm.new_value_error("slice stop is required")); + } + let stop: isize = slice + .stop + .try_int(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("slice stop too large"))?; + + // calculate length + let len: usize = if (step > 0 && start > stop) || (step < 0 && start < stop) { + 0 + } else if step > 0 { + ((stop - start - 1) / step + 1) as usize + } else { + ((stop - start + 1) / step + 1) as usize + }; + + // Get element info + let stg_info = zelf.class().stg_info(vm)?; + let element_size = if let Some(ref proto_type) = stg_info.proto { + proto_type.stg_info_opt().expect("proto has StgInfo").size + } else { + core::mem::size_of::<usize>() + }; + let type_code = stg_info + .proto + .as_ref() + .and_then(|p| p.as_object().get_attr("_type_", vm).ok()) + .and_then(|t| t.str(vm).ok()) + .map(|s| s.to_string()); + + let ptr_value = zelf.get_ptr_value(); + + // c_char → bytes + if type_code.as_deref() == Some("c") { + if len == 0 { + return Ok(vm.ctx.new_bytes(vec![]).into()); + } + let mut result = Vec::with_capacity(len); + if step == 1 { + // Optimized contiguous copy + let start_addr = (ptr_value as isize + start * element_size as isize) as *const u8; + unsafe { + result.extend_from_slice(core::slice::from_raw_parts(start_addr, len)); + } + } else { + let mut cur = start; + for _ in 0..len { + let addr = (ptr_value as isize + cur * element_size as isize) as *const u8; + unsafe { + result.push(*addr); + } + cur += step; + } + } + return Ok(vm.ctx.new_bytes(result).into()); + } + + // c_wchar → str + if type_code.as_deref() == Some("u") { + if len == 0 { + return Ok(vm.ctx.new_str("").into()); + } + let mut result = String::with_capacity(len); + let wchar_size = core::mem::size_of::<libc::wchar_t>(); + let mut cur = start; + for _ in 0..len { + let addr = (ptr_value as isize + cur * wchar_size as isize) as *const libc::wchar_t; + unsafe { + #[allow( + clippy::unnecessary_cast, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + if let Some(c) = char::from_u32(*addr as u32) { + result.push(c); + } + } + cur += step; + } + return Ok(vm.ctx.new_str(result).into()); + } + + // other types → list with Pointer_item for each + let mut items = Vec::with_capacity(len); + let mut cur = start; + for _ in 0..len { + items.push(Self::getitem_by_index(zelf, cur, vm)?); + cur += step; + } + Ok(PyList::from(items).into_ref(&vm.ctx).into()) + } + + // Pointer_ass_item + fn __setitem__( + zelf: &Py<Self>, + item: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Pointer does not support item deletion (value always provided) + // only integer indices supported for setitem + if let Some(i) = item.downcast_ref::<PyInt>() { + let i = i.as_bigint().to_isize().ok_or_else(|| { + vm.new_index_error("cannot fit index into an index-sized integer") + })?; + Self::setitem_by_index(zelf, i, value, vm) + } else { + Err(vm.new_type_error("Pointer indices must be integer")) + } + } + + fn setitem_by_index( + zelf: &Py<Self>, + index: isize, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let ptr_value = zelf.get_ptr_value(); + if ptr_value == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + + // Get element type, size and type_code from StgInfo.proto + let stg_info = zelf.class().stg_info(vm)?; + let proto_type = stg_info.proto(); + + // Get type code from proto's _type_ attribute + let type_code: Option<String> = proto_type + .as_object() + .get_attr("_type_", vm) + .ok() + .and_then(|t| t.downcast_ref::<PyStr>().map(|s| s.to_string())); + + let element_size = proto_type + .stg_info_opt() + .map_or(core::mem::size_of::<usize>(), |info| info.size); + + // Calculate address + let offset = index * element_size as isize; + let addr = (ptr_value as isize + offset) as usize; + + // Write value at address + // Handle Structure/Array types by copying their buffer + if let Some(cdata) = value.downcast_ref::<super::PyCData>() + && (cdata.fast_isinstance(PyCStructure::static_type()) + || cdata.fast_isinstance(PyCArray::static_type()) + || cdata.fast_isinstance(PyCSimple::static_type())) + { + let src_buffer = cdata.buffer.read(); + let copy_len = src_buffer.len().min(element_size); + unsafe { + let dest_ptr = addr as *mut u8; + core::ptr::copy_nonoverlapping(src_buffer.as_ptr(), dest_ptr, copy_len); + } + } else { + // Handle z/Z specially to store converted value + if type_code.as_deref() == Some("z") + && let Some(bytes) = value.downcast_ref::<PyBytes>() + { + let (kept_alive, ptr_val) = super::base::ensure_z_null_terminated(bytes, vm); + unsafe { + *(addr as *mut usize) = ptr_val; + } + zelf.0.keep_alive(index as usize, kept_alive); + return zelf.0.keep_ref(index as usize, value.clone(), vm); + } else if type_code.as_deref() == Some("Z") + && let Some(s) = value.downcast_ref::<PyStr>() + { + let (holder, ptr_val) = super::base::str_to_wchar_bytes(s.as_wtf8(), vm); + unsafe { + *(addr as *mut usize) = ptr_val; + } + return zelf.0.keep_ref(index as usize, holder, vm); + } else { + Self::write_value_at_address(addr, element_size, &value, type_code.as_deref(), vm)?; + } + } + + // KeepRef: store reference to keep value alive using actual index + zelf.0.keep_ref(index as usize, value, vm) + } + + /// Read a value from memory address + fn read_value_at_address( + addr: usize, + size: usize, + type_code: Option<&str>, + vm: &VirtualMachine, + ) -> PyResult { + unsafe { + let ptr = addr as *const u8; + match type_code { + // Single-byte types don't need read_unaligned + Some("c") => Ok(vm.ctx.new_bytes(vec![*ptr]).into()), + Some("b") => Ok(vm.ctx.new_int(*ptr as i8 as i32).into()), + Some("B") => Ok(vm.ctx.new_int(*ptr as i32).into()), + // Multi-byte types need read_unaligned for safety on strict-alignment architectures + Some("h") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const i16) as i32) + .into()), + Some("H") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const u16) as i32) + .into()), + Some("i") | Some("l") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const i32)) + .into()), + Some("I") | Some("L") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const u32)) + .into()), + Some("q") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const i64)) + .into()), + Some("Q") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const u64)) + .into()), + Some("f") => Ok(vm + .ctx + .new_float(core::ptr::read_unaligned(ptr as *const f32) as f64) + .into()), + Some("d") | Some("g") => Ok(vm + .ctx + .new_float(core::ptr::read_unaligned(ptr as *const f64)) + .into()), + Some("P") | Some("z") | Some("Z") => Ok(vm + .ctx + .new_int(core::ptr::read_unaligned(ptr as *const usize)) + .into()), + _ => { + // Default: read as bytes + let bytes = core::slice::from_raw_parts(ptr, size).to_vec(); + Ok(vm.ctx.new_bytes(bytes).into()) + } + } + } + } + + /// Write a value to memory address + fn write_value_at_address( + addr: usize, + size: usize, + value: &PyObject, + type_code: Option<&str>, + vm: &VirtualMachine, + ) -> PyResult<()> { + unsafe { + let ptr = addr as *mut u8; + + // Handle c_char_p (z) and c_wchar_p (Z) - store pointer address + // Note: PyBytes/PyStr cases are handled by caller (setitem_by_index) + match type_code { + Some("z") | Some("Z") => { + let ptr_val = if vm.is_none(value) { + 0usize + } else if let Ok(int_val) = value.try_index(vm) { + int_val.as_bigint().to_usize().unwrap_or(0) + } else { + return Err(vm.new_type_error("bytes/string or integer address expected")); + }; + core::ptr::write_unaligned(ptr as *mut usize, ptr_val); + return Ok(()); + } + _ => {} + } + + // Try to get value as integer + // Use write_unaligned for safety on strict-alignment architectures + if let Ok(int_val) = value.try_int(vm) { + let i = int_val.as_bigint(); + match size { + 1 => { + *ptr = i.to_u8().expect("int too large"); + } + 2 => { + core::ptr::write_unaligned( + ptr as *mut i16, + i.to_i16().expect("int too large"), + ); + } + 4 => { + core::ptr::write_unaligned( + ptr as *mut i32, + i.to_i32().expect("int too large"), + ); + } + 8 => { + core::ptr::write_unaligned( + ptr as *mut i64, + i.to_i64().expect("int too large"), + ); + } + _ => { + let bytes = i.to_signed_bytes_le(); + let copy_len = bytes.len().min(size); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, copy_len); + } + } + return Ok(()); + } + + // Try to get value as float + if let Ok(float_val) = value.try_float(vm) { + let f = float_val.to_f64(); + match size { + 4 => { + core::ptr::write_unaligned(ptr as *mut f32, f as f32); + } + 8 => { + core::ptr::write_unaligned(ptr as *mut f64, f); + } + _ => {} + } + return Ok(()); + } + + // Try bytes + if let Ok(bytes) = value.try_bytes_like(vm, |b| b.to_vec()) { + let copy_len = bytes.len().min(size); + core::ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, copy_len); + return Ok(()); + } + + Err(vm.new_type_error(format!( + "cannot convert {} to ctypes data", + value.class().name() + ))) + } + } +} + +impl AsNumber for PyCPointer { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|number, _vm| { + let zelf = number.obj.downcast_ref::<PyCPointer>().unwrap(); + Ok(zelf.get_ptr_value() != 0) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl AsMapping for PyCPointer { + fn as_mapping() -> &'static PyMappingMethods { + use crate::common::lock::LazyLock; + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + subscript: atomic_func!(|mapping, needle, vm| { + let zelf = PyCPointer::mapping_downcast(mapping); + PyCPointer::__getitem__(zelf, needle.to_owned(), vm) + }), + ass_subscript: atomic_func!(|mapping, needle, value, vm| { + let zelf = PyCPointer::mapping_downcast(mapping); + match value { + Some(value) => PyCPointer::__setitem__(zelf, needle.to_owned(), value, vm), + None => Err(vm.new_type_error("Pointer does not support item deletion")), + } + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } +} + +impl AsBuffer for PyCPointer { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let stg_info = zelf + .class() + .stg_info_opt() + .expect("PyCPointer type must have StgInfo"); + let format = stg_info + .format + .clone() + .map(Cow::Owned) + .unwrap_or(Cow::Borrowed("&B")); + let itemsize = stg_info.size; + // Pointer types are scalars with ndim=0, shape=() + let desc = BufferDescriptor { + len: itemsize, + readonly: false, + itemsize, + format, + dim_desc: vec![], + }; + let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); + Ok(buf) + } +} diff --git a/crates/vm/src/stdlib/_ctypes/simple.rs b/crates/vm/src/stdlib/_ctypes/simple.rs new file mode 100644 index 00000000000..3258d67d76f --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes/simple.rs @@ -0,0 +1,1461 @@ +use super::_ctypes::CArgObject; +use super::array::{PyCArray, WCHAR_SIZE, wchar_to_bytes}; +use super::base::{ + CDATA_BUFFER_METHODS, FfiArgValue, PyCData, StgInfo, StgInfoFlags, buffer_to_ffi_value, + bytes_to_pyobject, +}; +use super::function::PyCFuncPtr; +use super::get_size; +use super::pointer::PyCPointer; +use crate::builtins::{PyByteArray, PyBytes, PyInt, PyNone, PyStr, PyType, PyTypeRef}; +use crate::convert::ToPyObject; +use crate::function::{Either, FuncArgs, OptionalArg}; +use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; +use crate::types::{AsBuffer, AsNumber, Constructor, Initializer, Representable}; +use crate::{AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; +use alloc::borrow::Cow; +use core::fmt::Debug; +use num_traits::ToPrimitive; + +/// Valid type codes for ctypes simple types +#[cfg(windows)] +// spell-checker: disable-next-line +pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPXOv?g"; +#[cfg(not(windows))] +// spell-checker: disable-next-line +pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPOv?g"; + +/// Convert ctypes type code to PEP 3118 format code. +/// Some ctypes codes need to be mapped to standard-size codes based on platform. +/// _ctypes_alloc_format_string_for_type +fn ctypes_code_to_pep3118(code: char) -> char { + match code { + // c_int: map based on sizeof(int) + 'i' if core::mem::size_of::<core::ffi::c_int>() == 2 => 'h', + 'i' if core::mem::size_of::<core::ffi::c_int>() == 4 => 'i', + 'i' if core::mem::size_of::<core::ffi::c_int>() == 8 => 'q', + 'I' if core::mem::size_of::<core::ffi::c_int>() == 2 => 'H', + 'I' if core::mem::size_of::<core::ffi::c_int>() == 4 => 'I', + 'I' if core::mem::size_of::<core::ffi::c_int>() == 8 => 'Q', + // c_long: map based on sizeof(long) + 'l' if core::mem::size_of::<core::ffi::c_long>() == 4 => 'l', + 'l' if core::mem::size_of::<core::ffi::c_long>() == 8 => 'q', + 'L' if core::mem::size_of::<core::ffi::c_long>() == 4 => 'L', + 'L' if core::mem::size_of::<core::ffi::c_long>() == 8 => 'Q', + // c_bool: map based on sizeof(bool) - typically 1 byte on all platforms + '?' if core::mem::size_of::<bool>() == 1 => '?', + '?' if core::mem::size_of::<bool>() == 2 => 'H', + '?' if core::mem::size_of::<bool>() == 4 => 'L', + '?' if core::mem::size_of::<bool>() == 8 => 'Q', + // Default: use the same code + _ => code, + } +} + +/// _ctypes_alloc_format_string_for_type +fn alloc_format_string_for_type(code: char, big_endian: bool) -> String { + let prefix = if big_endian { ">" } else { "<" }; + let pep_code = ctypes_code_to_pep3118(code); + format!("{}{}", prefix, pep_code) +} + +/// Create a new simple type instance from a class +fn new_simple_type( + cls: Either<&PyObject, &Py<PyType>>, + vm: &VirtualMachine, +) -> PyResult<PyCSimple> { + let cls = match cls { + Either::A(obj) => obj, + Either::B(typ) => typ.as_object(), + }; + + let _type_ = cls + .get_attr("_type_", vm) + .map_err(|_| vm.new_attribute_error("class must define a '_type_' attribute"))?; + + if !_type_.is_instance((&vm.ctx.types.str_type).as_ref(), vm)? { + return Err(vm.new_type_error("class must define a '_type_' string attribute")); + } + + let tp_str = _type_.str(vm)?.to_string(); + + if tp_str.len() != 1 { + return Err(vm.new_value_error(format!( + "class must define a '_type_' attribute which must be a string of length 1, str: {tp_str}" + ))); + } + + if !SIMPLE_TYPE_CHARS.contains(tp_str.as_str()) { + return Err(vm.new_attribute_error(format!( + "class must define a '_type_' attribute which must be\n a single character string containing one of {SIMPLE_TYPE_CHARS}, currently it is {tp_str}." + ))); + } + + let size = get_size(&tp_str); + Ok(PyCSimple(PyCData::from_bytes(vec![0u8; size], None))) +} + +fn set_primitive(_type_: &str, value: &PyObject, vm: &VirtualMachine) -> PyResult { + match _type_ { + "c" => { + // c_set: accepts bytes(len=1), bytearray(len=1), or int(0-255) + if value + .downcast_ref_if_exact::<PyBytes>(vm) + .is_some_and(|v| v.len() == 1) + || value + .downcast_ref_if_exact::<PyByteArray>(vm) + .is_some_and(|v| v.borrow_buf().len() == 1) + || value.downcast_ref_if_exact::<PyInt>(vm).is_some_and(|v| { + v.as_bigint() + .to_i64() + .is_some_and(|n| (0..=255).contains(&n)) + }) + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error("one character bytes, bytearray or integer expected")) + } + } + "u" => { + if let Some(s) = value.downcast_ref::<PyStr>() { + if s.as_wtf8().code_points().count() == 1 { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error("one character unicode string expected")) + } + } else { + Err(vm.new_type_error(format!( + "unicode string expected instead of {} instance", + value.class().name() + ))) + } + } + "b" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { + // Support __index__ protocol + if value.try_index(vm).is_ok() { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!( + "an integer is required (got type {})", + value.class().name() + ))) + } + } + "f" | "d" | "g" => { + // Handle int specially to check overflow + if let Some(int_obj) = value.downcast_ref_if_exact::<PyInt>(vm) { + // Check if int can fit in f64 + if let Some(f) = int_obj.as_bigint().to_f64() + && f.is_finite() + { + return Ok(value.to_owned()); + } + return Err(vm.new_overflow_error("int too large to convert to float")); + } + // __float__ protocol + if value.try_float(vm).is_ok() { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!("must be real number, not {}", value.class().name()))) + } + } + "?" => Ok(PyObjectRef::from( + vm.ctx.new_bool(value.to_owned().try_to_bool(vm)?), + )), + "v" => { + // VARIANT_BOOL: any truthy → True + Ok(PyObjectRef::from( + vm.ctx.new_bool(value.to_owned().try_to_bool(vm)?), + )) + } + "B" => { + // Support __index__ protocol + if value.try_index(vm).is_ok() { + // Store as-is, conversion to unsigned happens in the getter + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!("int expected instead of {}", value.class().name()))) + } + } + "z" => { + if value.is(&vm.ctx.none) + || value.downcast_ref_if_exact::<PyInt>(vm).is_some() + || value.downcast_ref_if_exact::<PyBytes>(vm).is_some() + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!( + "bytes or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + "Z" => { + if value.is(&vm.ctx.none) + || value.downcast_ref_if_exact::<PyInt>(vm).is_some() + || value.downcast_ref_if_exact::<PyStr>(vm).is_some() + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!( + "unicode string or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + // O_set: py_object accepts any Python object + "O" => Ok(value.to_owned()), + // X_set: BSTR - same as Z (c_wchar_p), accepts None, int, or str + "X" => { + if value.is(&vm.ctx.none) + || value.downcast_ref_if_exact::<PyInt>(vm).is_some() + || value.downcast_ref_if_exact::<PyStr>(vm).is_some() + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error(format!( + "unicode string or integer address expected instead of {} instance", + value.class().name() + ))) + } + } + _ => { + // "P" + if value.downcast_ref_if_exact::<PyInt>(vm).is_some() + || value.downcast_ref_if_exact::<PyNone>(vm).is_some() + { + Ok(value.to_owned()) + } else { + Err(vm.new_type_error("cannot be converted to pointer")) + } + } + } +} + +#[pyclass(module = "_ctypes", name = "PyCSimpleType", base = PyType)] +#[derive(Debug)] +#[repr(transparent)] +pub struct PyCSimpleType(PyType); + +#[pyclass(flags(BASETYPE), with(AsNumber, Initializer))] +impl PyCSimpleType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + + #[allow(clippy::new_ret_no_self)] + #[pymethod] + fn new(cls: PyTypeRef, _: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(PyObjectRef::from( + new_simple_type(Either::B(&cls), vm)? + .into_ref_with_type(vm, cls)? + .clone(), + )) + } + + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the class (e.g., c_int) that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. If the value is already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } + + // 2. Get the type code to determine conversion rules + let type_code = cls.type_code(vm); + + // 3. Handle None for pointer types (c_char_p, c_wchar_p, c_void_p) + if vm.is_none(&value) && matches!(type_code.as_deref(), Some("z") | Some("Z") | Some("P")) { + return Ok(value); + } + + // Helper to create CArgObject wrapping a simple instance + let create_simple_with_value = |type_str: &str, val: &PyObject| -> PyResult { + let simple = new_simple_type(Either::B(&cls), vm)?; + let buffer_bytes = value_to_bytes_endian(type_str, val, false, vm); + *simple.0.buffer.write() = alloc::borrow::Cow::Owned(buffer_bytes.clone()); + let simple_obj: PyObjectRef = simple.into_ref_with_type(vm, cls.clone())?.into(); + // from_param returns CArgObject, not the simple type itself + let tag = type_str.as_bytes().first().copied().unwrap_or(b'?'); + let ffi_value = buffer_to_ffi_value(type_str, &buffer_bytes); + Ok(CArgObject { + tag, + value: ffi_value, + obj: simple_obj, + size: 0, + offset: 0, + } + .to_pyobject(vm)) + }; + + // 4. Try to convert value based on type code + match type_code.as_deref() { + // Integer types: accept integers + Some(tc @ ("b" | "B" | "h" | "H" | "i" | "I" | "l" | "L" | "q" | "Q")) => { + if value.try_int(vm).is_ok() { + return create_simple_with_value(tc, &value); + } + } + // Float types: accept numbers + Some(tc @ ("f" | "d" | "g")) => { + if value.try_float(vm).is_ok() || value.try_int(vm).is_ok() { + return create_simple_with_value(tc, &value); + } + } + // c_char: 1 byte character + Some("c") => { + if let Some(bytes) = value.downcast_ref::<PyBytes>() + && bytes.len() == 1 + { + return create_simple_with_value("c", &value); + } + if let Ok(int_val) = value.try_int(vm) + && int_val.as_bigint().to_u8().is_some() + { + return create_simple_with_value("c", &value); + } + return Err(vm.new_type_error("one character bytes, bytearray or integer expected")); + } + // c_wchar: 1 unicode character + Some("u") => { + if let Some(s) = value.downcast_ref::<PyStr>() + && s.as_wtf8().code_points().count() == 1 + { + return create_simple_with_value("u", &value); + } + return Err(vm.new_type_error("one character unicode string expected")); + } + // c_char_p: bytes pointer + Some("z") => { + // 1. bytes → create CArgObject with null-terminated buffer + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + return Ok(CArgObject { + tag: b'z', + value: FfiArgValue::OwnedPointer(ptr, kept_alive), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 2. Array/Pointer with c_char element type + if is_cchar_array_or_pointer(&value, vm) { + return Ok(value); + } + // 3. CArgObject (byref(c_char(...))) + if let Some(carg) = value.downcast_ref::<CArgObject>() + && carg.tag == b'c' + { + return Ok(value.clone()); + } + } + // c_wchar_p: unicode pointer + Some("Z") => { + // 1. str → create CArgObject with null-terminated wchar buffer + if let Some(s) = value.downcast_ref::<PyStr>() { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_wtf8(), vm); + return Ok(CArgObject { + tag: b'Z', + value: FfiArgValue::OwnedPointer(ptr, holder), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 2. Array/Pointer with c_wchar element type + if is_cwchar_array_or_pointer(&value, vm)? { + return Ok(value); + } + // 3. CArgObject (byref(c_wchar(...))) + if let Some(carg) = value.downcast_ref::<CArgObject>() + && carg.tag == b'u' + { + return Ok(value.clone()); + } + } + // c_void_p: most flexible - accepts int, bytes, str, any array/pointer, funcptr + Some("P") => { + // 1. int → create c_void_p with that address + if value.try_int(vm).is_ok() { + return create_simple_with_value("P", &value); + } + // 2. bytes → create CArgObject with null-terminated buffer + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + return Ok(CArgObject { + tag: b'z', + value: FfiArgValue::OwnedPointer(ptr, kept_alive), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 3. str → create CArgObject with null-terminated wchar buffer + if let Some(s) = value.downcast_ref::<PyStr>() { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_wtf8(), vm); + return Ok(CArgObject { + tag: b'Z', + value: FfiArgValue::OwnedPointer(ptr, holder), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 4. Any Array or Pointer → accept directly + if value.downcast_ref::<PyCArray>().is_some() + || value.downcast_ref::<PyCPointer>().is_some() + { + return Ok(value); + } + // 5. CArgObject with 'P' tag (byref(c_void_p(...))) + if let Some(carg) = value.downcast_ref::<CArgObject>() + && carg.tag == b'P' + { + return Ok(value.clone()); + } + // 6. PyCFuncPtr → extract function pointer address + if let Some(funcptr) = value.downcast_ref::<PyCFuncPtr>() { + let ptr_val = { + let buffer = funcptr._base.buffer.read(); + buffer + .first_chunk::<{ size_of::<usize>() }>() + .copied() + .map_or(0, usize::from_ne_bytes) + }; + return Ok(CArgObject { + tag: b'P', + value: FfiArgValue::Pointer(ptr_val), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + // 7. c_char_p or c_wchar_p instance → extract pointer value + if let Some(simple) = value.downcast_ref::<PyCSimple>() { + let value_type_code = value.class().type_code(vm); + if matches!(value_type_code.as_deref(), Some("z") | Some("Z")) { + let ptr_val = { + let buffer = simple.0.buffer.read(); + buffer + .first_chunk::<{ size_of::<usize>() }>() + .copied() + .map_or(0, usize::from_ne_bytes) + }; + return Ok(CArgObject { + tag: b'Z', + value: FfiArgValue::Pointer(ptr_val), + obj: value.clone(), + size: 0, + offset: 0, + } + .to_pyobject(vm)); + } + } + } + // c_bool + Some("?") => { + let bool_val = value.is_true(vm)?; + let bool_obj: PyObjectRef = vm.ctx.new_bool(bool_val).into(); + return create_simple_with_value("?", &bool_obj); + } + _ => {} + } + + // 5. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCSimpleType::from_param(cls.as_object().to_owned(), as_parameter, vm); + } + + // 6. Type-specific error messages + match type_code.as_deref() { + Some("z") => Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as ctypes.c_char_p", + value.class().name() + ))), + Some("Z") => Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as ctypes.c_wchar_p", + value.class().name() + ))), + _ => Err(vm.new_type_error("wrong type")), + } + } + + fn __mul__(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { + PyCSimple::repeat(cls, n, vm) + } +} + +impl AsNumber for PyCSimpleType { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + multiply: Some(|a, b, vm| { + // a is a PyCSimpleType instance (type object like c_char) + // b is int (array size) + let cls = a + .downcast_ref::<PyType>() + .ok_or_else(|| vm.new_type_error("expected type"))?; + let n = b + .try_index(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("array size too large"))?; + PyCSimple::repeat(cls.to_owned(), n, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl Initializer for PyCSimpleType { + type Args = FuncArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // type_init requires exactly 3 positional arguments: name, bases, dict + if args.args.len() != 3 { + return Err(vm.new_type_error(format!( + "type.__init__() takes 3 positional arguments but {} were given", + args.args.len() + ))); + } + + // Get the type from the metatype instance + let type_ref: PyTypeRef = zelf + .as_object() + .to_owned() + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + type_ref.check_not_initialized(vm)?; + + // Get _type_ attribute + let type_attr = match type_ref.as_object().get_attr("_type_", vm) { + Ok(attr) => attr, + Err(_) => { + return Err(vm.new_attribute_error("class must define a '_type_' attribute")); + } + }; + + // Validate _type_ is a string + let type_str = type_attr.str(vm)?.to_string(); + + // Validate _type_ is a single character + if type_str.len() != 1 { + return Err(vm.new_value_error( + "class must define a '_type_' attribute which must be a string of length 1", + )); + } + + // Validate _type_ is a valid type character + if !SIMPLE_TYPE_CHARS.contains(type_str.as_str()) { + return Err(vm.new_attribute_error(format!( + "class must define a '_type_' attribute which must be a single character string containing one of '{}', currently it is '{}'.", + SIMPLE_TYPE_CHARS, type_str + ))); + } + + // Initialize StgInfo + let size = super::get_size(&type_str); + let align = super::get_align(&type_str); + let mut stg_info = StgInfo::new(size, align); + + // Set format for PEP 3118 buffer protocol + stg_info.format = Some(alloc_format_string_for_type( + type_str.chars().next().unwrap_or('?'), + cfg!(target_endian = "big"), + )); + stg_info.paramfunc = super::base::ParamFunc::Simple; + + // Set TYPEFLAG_ISPOINTER for pointer types: z (c_char_p), Z (c_wchar_p), + // P (c_void_p), s (char array), X (BSTR), O (py_object) + if matches!(type_str.as_str(), "z" | "Z" | "P" | "s" | "X" | "O") { + stg_info.flags |= StgInfoFlags::TYPEFLAG_ISPOINTER; + } + + super::base::set_or_init_stginfo(&type_ref, stg_info); + + // Create __ctype_le__ and __ctype_be__ swapped types + create_swapped_types(&type_ref, &type_str, vm)?; + + Ok(()) + } +} + +/// Create __ctype_le__ and __ctype_be__ swapped byte order types +/// On little-endian systems: __ctype_le__ = self, __ctype_be__ = swapped type +/// On big-endian systems: __ctype_be__ = self, __ctype_le__ = swapped type +/// +/// - Single-byte types (c, b, B): __ctype_le__ = __ctype_be__ = self +/// - Pointer/unsupported types (z, Z, P, u, O): NO __ctype_le__/__ctype_be__ attributes +/// - Multi-byte numeric types (h, H, i, I, l, L, q, Q, f, d, g, ?): create swapped types +fn create_swapped_types( + type_ref: &Py<PyType>, + type_str: &str, + vm: &VirtualMachine, +) -> PyResult<()> { + use crate::builtins::PyDict; + + // Avoid infinite recursion - if __ctype_le__ already exists, skip + if type_ref.as_object().get_attr("__ctype_le__", vm).is_ok() { + return Ok(()); + } + + // Types that don't support byte order swapping - no __ctype_le__/__ctype_be__ + // c_void_p (P), c_char_p (z), c_wchar_p (Z), c_wchar (u), py_object (O) + let unsupported_types = ["P", "z", "Z", "u", "O"]; + if unsupported_types.contains(&type_str) { + return Ok(()); + } + + // Single-byte types - __ctype_le__ = __ctype_be__ = self (no swapping needed) + // c_char (c), c_byte (b), c_ubyte (B) + let single_byte_types = ["c", "b", "B"]; + if single_byte_types.contains(&type_str) { + type_ref + .as_object() + .set_attr("__ctype_le__", type_ref.as_object().to_owned(), vm)?; + type_ref + .as_object() + .set_attr("__ctype_be__", type_ref.as_object().to_owned(), vm)?; + return Ok(()); + } + + // Multi-byte types - create swapped type + // Check system byte order at compile time + let is_little_endian = cfg!(target_endian = "little"); + + // Create dict for the swapped (non-native) type + let swapped_dict: crate::PyRef<crate::builtins::PyDict> = PyDict::default().into_ref(&vm.ctx); + swapped_dict.set_item("_type_", vm.ctx.new_str(type_str).into(), vm)?; + + // Create the swapped type using the same metaclass + let metaclass = type_ref.class(); + let bases = vm.ctx.new_tuple(vec![type_ref.as_object().to_owned()]); + + // Set placeholder first to prevent recursion + type_ref + .as_object() + .set_attr("__ctype_le__", vm.ctx.none(), vm)?; + type_ref + .as_object() + .set_attr("__ctype_be__", vm.ctx.none(), vm)?; + + // Create only the non-native endian type + let suffix = if is_little_endian { "_be" } else { "_le" }; + let swapped_type = metaclass.as_object().call( + ( + vm.ctx.new_str(format!("{}{}", type_ref.name(), suffix)), + bases, + swapped_dict.as_object().to_owned(), + ), + vm, + )?; + + // Set _swappedbytes_ on the swapped type to indicate byte swapping is needed + swapped_type.set_attr("_swappedbytes_", vm.ctx.none(), vm)?; + + // Update swapped type's StgInfo format to use opposite endian prefix + if let Ok(swapped_type_ref) = swapped_type.clone().downcast::<PyType>() + && let Some(mut sw_stg) = swapped_type_ref.get_type_data_mut::<StgInfo>() + { + // Swapped: little-endian system uses big-endian prefix and vice versa + sw_stg.format = Some(alloc_format_string_for_type( + type_str.chars().next().unwrap_or('?'), + is_little_endian, + )); + } + + // Set attributes based on system byte order + // Native endian attribute points to self, non-native points to swapped type + if is_little_endian { + // Little-endian system: __ctype_le__ = self, __ctype_be__ = swapped + type_ref + .as_object() + .set_attr("__ctype_le__", type_ref.as_object().to_owned(), vm)?; + type_ref + .as_object() + .set_attr("__ctype_be__", swapped_type.clone(), vm)?; + swapped_type.set_attr("__ctype_le__", type_ref.as_object().to_owned(), vm)?; + swapped_type.set_attr("__ctype_be__", swapped_type.clone(), vm)?; + } else { + // Big-endian system: __ctype_be__ = self, __ctype_le__ = swapped + type_ref + .as_object() + .set_attr("__ctype_be__", type_ref.as_object().to_owned(), vm)?; + type_ref + .as_object() + .set_attr("__ctype_le__", swapped_type.clone(), vm)?; + swapped_type.set_attr("__ctype_be__", type_ref.as_object().to_owned(), vm)?; + swapped_type.set_attr("__ctype_le__", swapped_type.clone(), vm)?; + } + + Ok(()) +} + +#[pyclass( + module = "_ctypes", + name = "_SimpleCData", + base = PyCData, + metaclass = "PyCSimpleType" +)] +#[repr(transparent)] +pub struct PyCSimple(pub PyCData); + +impl Debug for PyCSimple { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyCSimple") + .field("size", &self.0.buffer.read().len()) + .finish() + } +} + +fn value_to_bytes_endian( + _type_: &str, + value: &PyObject, + swapped: bool, + vm: &VirtualMachine, +) -> Vec<u8> { + // Helper macro for endian conversion + macro_rules! to_bytes { + ($val:expr) => { + if swapped { + // Use opposite endianness + #[cfg(target_endian = "little")] + { + $val.to_be_bytes().to_vec() + } + #[cfg(target_endian = "big")] + { + $val.to_le_bytes().to_vec() + } + } else { + $val.to_ne_bytes().to_vec() + } + }; + } + + match _type_ { + "c" => { + // c_char - single byte (bytes, bytearray, or int 0-255) + if let Some(bytes) = value.downcast_ref::<PyBytes>() + && !bytes.is_empty() + { + return vec![bytes.as_bytes()[0]]; + } + if let Some(bytearray) = value.downcast_ref::<PyByteArray>() { + let buf = bytearray.borrow_buf(); + if !buf.is_empty() { + return vec![buf[0]]; + } + } + if let Ok(int_val) = value.try_int(vm) + && let Some(v) = int_val.as_bigint().to_u8() + { + return vec![v]; + } + vec![0] + } + "u" => { + // c_wchar - platform-dependent size (2 on Windows, 4 on Unix) + if let Some(s) = value.downcast_ref::<PyStr>() { + let mut cps = s.as_wtf8().code_points(); + if let (Some(c), None) = (cps.next(), cps.next()) { + let mut buffer = vec![0u8; WCHAR_SIZE]; + wchar_to_bytes(c.to_u32(), &mut buffer); + if swapped { + buffer.reverse(); + } + return buffer; + } + } + vec![0; WCHAR_SIZE] + } + "b" => { + // c_byte - signed char (1 byte) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as i8; + return vec![v as u8]; + } + vec![0] + } + "B" => { + // c_ubyte - unsigned char (1 byte) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as u8; + return vec![v]; + } + vec![0] + } + "h" => { + // c_short (2 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as i16; + return to_bytes!(v); + } + vec![0; 2] + } + "H" => { + // c_ushort (2 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as u16; + return to_bytes!(v); + } + vec![0; 2] + } + "i" => { + // c_int (4 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as i32; + return to_bytes!(v); + } + vec![0; 4] + } + "I" => { + // c_uint (4 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as u32; + return to_bytes!(v); + } + vec![0; 4] + } + "l" => { + // c_long (platform dependent) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as libc::c_long; + return to_bytes!(v); + } + const SIZE: usize = core::mem::size_of::<libc::c_long>(); + vec![0; SIZE] + } + "L" => { + // c_ulong (platform dependent) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as libc::c_ulong; + return to_bytes!(v); + } + const SIZE: usize = core::mem::size_of::<libc::c_ulong>(); + vec![0; SIZE] + } + "q" => { + // c_longlong (8 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as i64; + return to_bytes!(v); + } + vec![0; 8] + } + "Q" => { + // c_ulonglong (8 bytes) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val.as_bigint().to_i128().expect("int too large") as u64; + return to_bytes!(v); + } + vec![0; 8] + } + "f" => { + // c_float (4 bytes) - also accepts int + if let Ok(float_val) = value.try_float(vm) { + return to_bytes!(float_val.to_f64() as f32); + } + if let Ok(int_val) = value.try_int(vm) + && let Some(v) = int_val.as_bigint().to_f64() + { + return to_bytes!(v as f32); + } + vec![0; 4] + } + "d" => { + // c_double (8 bytes) - also accepts int + if let Ok(float_val) = value.try_float(vm) { + return to_bytes!(float_val.to_f64()); + } + if let Ok(int_val) = value.try_int(vm) + && let Some(v) = int_val.as_bigint().to_f64() + { + return to_bytes!(v); + } + vec![0; 8] + } + "g" => { + // long double - platform dependent size + // Store as f64, zero-pad to platform long double size + // Note: This may lose precision on platforms where long double > 64 bits + let f64_val = if let Ok(float_val) = value.try_float(vm) { + float_val.to_f64() + } else if let Ok(int_val) = value.try_int(vm) { + int_val.as_bigint().to_f64().unwrap_or(0.0) + } else { + 0.0 + }; + let f64_bytes = if swapped { + #[cfg(target_endian = "little")] + { + f64_val.to_be_bytes().to_vec() + } + #[cfg(target_endian = "big")] + { + f64_val.to_le_bytes().to_vec() + } + } else { + f64_val.to_ne_bytes().to_vec() + }; + // Pad to long double size + let long_double_size = super::get_size("g"); + let mut result = f64_bytes; + result.resize(long_double_size, 0); + result + } + "?" => { + // c_bool (1 byte) + if let Ok(b) = value.to_owned().try_to_bool(vm) { + return vec![if b { 1 } else { 0 }]; + } + vec![0] + } + "v" => { + // VARIANT_BOOL: True = 0xFFFF (-1 as i16), False = 0x0000 + if let Ok(b) = value.to_owned().try_to_bool(vm) { + let val: i16 = if b { -1 } else { 0 }; + return to_bytes!(val); + } + vec![0; 2] + } + "P" => { + // c_void_p - pointer type (platform pointer size) + if let Ok(int_val) = value.try_index(vm) { + let v = int_val + .as_bigint() + .to_usize() + .expect("int too large for pointer"); + return to_bytes!(v); + } + vec![0; core::mem::size_of::<usize>()] + } + "z" => { + // c_char_p - pointer to char (stores pointer value from int) + // PyBytes case is handled in slot_new/set_value with make_z_buffer() + if let Ok(int_val) = value.try_index(vm) { + let v = int_val + .as_bigint() + .to_usize() + .expect("int too large for pointer"); + return to_bytes!(v); + } + vec![0; core::mem::size_of::<usize>()] + } + "Z" => { + // c_wchar_p - pointer to wchar_t (stores pointer value from int) + // PyStr case is handled in slot_new/set_value with make_wchar_buffer() + if let Ok(int_val) = value.try_index(vm) { + let v = int_val + .as_bigint() + .to_usize() + .expect("int too large for pointer"); + return to_bytes!(v); + } + vec![0; core::mem::size_of::<usize>()] + } + "O" => { + // py_object - store object id as non-zero marker + // The actual object is stored in _objects + // Use object's id as a non-zero placeholder (indicates non-NULL) + let id = value.get_id(); + to_bytes!(id) + } + _ => vec![0], + } +} + +/// Check if value is a c_char array or pointer(c_char) +fn is_cchar_array_or_pointer(value: &PyObject, vm: &VirtualMachine) -> bool { + // Check Array with c_char element type + if let Some(arr) = value.downcast_ref::<PyCArray>() + && let Some(info) = arr.class().stg_info_opt() + && let Some(ref elem_type) = info.element_type + && let Some(elem_code) = elem_type.type_code(vm) + { + return elem_code == "c"; + } + // Check Pointer to c_char + if let Some(ptr) = value.downcast_ref::<PyCPointer>() + && let Some(info) = ptr.class().stg_info_opt() + && let Some(ref proto) = info.proto + && let Some(proto_code) = proto.type_code(vm) + { + return proto_code == "c"; + } + false +} + +/// Check if value is a c_wchar array or pointer(c_wchar) +fn is_cwchar_array_or_pointer(value: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + // Check Array with c_wchar element type + if let Some(arr) = value.downcast_ref::<PyCArray>() { + let info = arr.class().stg_info(vm)?; + let elem_type = info.element_type.as_ref().expect("array has element_type"); + if let Some(elem_code) = elem_type.type_code(vm) { + return Ok(elem_code == "u"); + } + } + // Check Pointer to c_wchar + if let Some(ptr) = value.downcast_ref::<PyCPointer>() { + let info = ptr.class().stg_info(vm)?; + if let Some(ref proto) = info.proto + && let Some(proto_code) = proto.type_code(vm) + { + return Ok(proto_code == "u"); + } + } + Ok(false) +} + +impl Constructor for PyCSimple { + type Args = (OptionalArg,); + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let args: Self::Args = args.bind(vm)?; + let _type_ = cls + .type_code(vm) + .ok_or_else(|| vm.new_type_error("abstract class"))?; + // Save the initial argument for c_char_p/c_wchar_p _objects + let init_arg = args.0.into_option(); + + // Handle z/Z types with PyBytes/PyStr separately to avoid memory leak + if let Some(ref v) = init_arg { + if _type_ == "z" { + if let Some(bytes) = v.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let buffer = ptr.to_ne_bytes().to_vec(); + let cdata = PyCData::from_bytes(buffer, Some(v.clone())); + *cdata.base.write() = Some(kept_alive); + return PyCSimple(cdata).into_ref_with_type(vm, cls).map(Into::into); + } + } else if _type_ == "Z" + && let Some(s) = v.downcast_ref::<PyStr>() + { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_wtf8(), vm); + let buffer = ptr.to_ne_bytes().to_vec(); + let cdata = PyCData::from_bytes(buffer, Some(holder)); + return PyCSimple(cdata).into_ref_with_type(vm, cls).map(Into::into); + } + } + + let value = if let Some(ref v) = init_arg { + set_primitive(_type_.as_str(), v, vm)? + } else { + match _type_.as_str() { + "c" | "u" => PyObjectRef::from(vm.ctx.new_bytes(vec![0])), + "b" | "B" | "h" | "H" | "i" | "I" | "l" | "q" | "L" | "Q" => { + PyObjectRef::from(vm.ctx.new_int(0)) + } + "f" | "d" | "g" => PyObjectRef::from(vm.ctx.new_float(0.0)), + "?" => PyObjectRef::from(vm.ctx.new_bool(false)), + _ => vm.ctx.none(), // "z" | "Z" | "P" + } + }; + + // Check if this is a swapped endian type (presence of attribute indicates swapping) + let swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + + let buffer = value_to_bytes_endian(&_type_, &value, swapped, vm); + + // For c_char_p (type "z"), c_wchar_p (type "Z"), and py_object (type "O"), + // store the initial value in _objects + let objects = if (_type_ == "z" || _type_ == "Z" || _type_ == "O") && init_arg.is_some() { + init_arg + } else { + None + }; + + PyCSimple(PyCData::from_bytes(buffer, objects)) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyCSimple { + type Args = (OptionalArg,); + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // If an argument is provided, update the value + if let Some(value) = args.0.into_option() { + PyCSimple::set_value(zelf.into(), value, vm)?; + } + Ok(()) + } +} + +// Simple_repr +impl Representable for PyCSimple { + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let cls = zelf.class(); + let type_name = cls.name(); + + // Check if base is _SimpleCData (direct simple type like c_int, c_char) + // vs subclass of simple type (like class X(c_int): pass) + let bases = cls.bases.read(); + let is_direct_simple = bases + .iter() + .any(|base| base.name().to_string() == "_SimpleCData"); + + if is_direct_simple { + // Direct SimpleCData: "typename(repr(value))" + let value = PyCSimple::value(zelf.to_owned().into(), vm)?; + let value_repr = value.repr(vm)?.to_string(); + Ok(format!("{}({})", type_name, value_repr)) + } else { + // Subclass: "<typename object at addr>" + let addr = zelf.get_id(); + Ok(format!("<{} object at {:#x}>", type_name, addr)) + } + } +} + +#[pyclass( + flags(BASETYPE), + with(Constructor, Initializer, AsBuffer, AsNumber, Representable) +)] +impl PyCSimple { + #[pygetset] + fn _b0_(&self) -> Option<PyObjectRef> { + self.0.base.read().clone() + } + + #[pygetset] + pub fn value(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let zelf: &Py<Self> = instance + .downcast_ref() + .ok_or_else(|| vm.new_type_error("cannot get value of instance"))?; + + // Get _type_ from class + let cls = zelf.class(); + let type_attr = cls + .as_object() + .get_attr("_type_", vm) + .map_err(|_| vm.new_type_error("no _type_ attribute"))?; + let type_code = type_attr.str(vm)?.to_string(); + + // Special handling for c_char_p (z) and c_wchar_p (Z) + // z_get, Z_get - dereference pointer to get string + if type_code == "z" { + // c_char_p: read pointer from buffer, dereference to get bytes string + let buffer = zelf.0.buffer.read(); + let ptr = super::base::read_ptr_from_buffer(&buffer); + if ptr == 0 { + return Ok(vm.ctx.none()); + } + // Read null-terminated string at the address + unsafe { + let cstr = core::ffi::CStr::from_ptr(ptr as _); + return Ok(vm.ctx.new_bytes(cstr.to_bytes().to_vec()).into()); + } + } + if type_code == "Z" { + // c_wchar_p: read pointer from buffer, dereference to get wide string + let buffer = zelf.0.buffer.read(); + let ptr = super::base::read_ptr_from_buffer(&buffer); + if ptr == 0 { + return Ok(vm.ctx.none()); + } + // Read null-terminated wide string at the address + // Windows: wchar_t = u16 (UTF-16) -> use Wtf8Buf::from_wide for surrogate pairs + // Unix: wchar_t = i32 (UTF-32) -> convert via char::from_u32 + unsafe { + let w_ptr = ptr as *const libc::wchar_t; + let len = libc::wcslen(w_ptr); + let wchars = core::slice::from_raw_parts(w_ptr, len); + #[cfg(windows)] + { + use rustpython_common::wtf8::Wtf8Buf; + let wide: Vec<u16> = wchars.to_vec(); + let wtf8 = Wtf8Buf::from_wide(&wide); + return Ok(vm.ctx.new_str(wtf8).into()); + } + #[cfg(not(windows))] + { + #[allow( + clippy::useless_conversion, + reason = "wchar_t is i32 on some platforms and u32 on others" + )] + let s: String = wchars + .iter() + .filter_map(|&c| u32::try_from(c).ok().and_then(char::from_u32)) + .collect(); + return Ok(vm.ctx.new_str(s).into()); + } + } + } + + // O_get: py_object - read PyObject pointer from buffer + if type_code == "O" { + let buffer = zelf.0.buffer.read(); + let ptr = super::base::read_ptr_from_buffer(&buffer); + if ptr == 0 { + return Err(vm.new_value_error("PyObject is NULL")); + } + // Non-NULL: return stored object from _objects if available + if let Some(obj) = zelf.0.objects.read().as_ref() { + return Ok(obj.clone()); + } + return Err(vm.new_value_error("PyObject is NULL")); + } + + // Check if this is a swapped endian type (presence of attribute indicates swapping) + let swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + + // Read value from buffer, swap bytes if needed + let buffer = zelf.0.buffer.read(); + let buffer_data: alloc::borrow::Cow<'_, [u8]> = if swapped { + // Reverse bytes for swapped endian types + let mut swapped_bytes = buffer.to_vec(); + swapped_bytes.reverse(); + alloc::borrow::Cow::Owned(swapped_bytes) + } else { + alloc::borrow::Cow::Borrowed(&*buffer) + }; + + let cls_ref = cls.to_owned(); + bytes_to_pyobject(&cls_ref, &buffer_data, vm).or_else(|_| { + // Fallback: return bytes as integer based on type + match type_code.as_str() { + "c" => { + if !buffer.is_empty() { + Ok(vm.ctx.new_bytes(vec![buffer[0]]).into()) + } else { + Ok(vm.ctx.new_bytes(vec![0]).into()) + } + } + "?" => { + let val = buffer.first().copied().unwrap_or(0); + Ok(vm.ctx.new_bool(val != 0).into()) + } + _ => Ok(vm.ctx.new_int(0).into()), + } + }) + } + + #[pygetset(setter)] + fn set_value(instance: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let zelf: PyRef<Self> = instance + .clone() + .downcast() + .map_err(|_| vm.new_type_error("cannot set value of instance"))?; + + // Get _type_ from class + let cls = zelf.class(); + let type_attr = cls + .as_object() + .get_attr("_type_", vm) + .map_err(|_| vm.new_type_error("no _type_ attribute"))?; + let type_code = type_attr.str(vm)?.to_string(); + + // Handle z/Z types with PyBytes/PyStr separately to avoid memory leak + if type_code == "z" { + if let Some(bytes) = value.downcast_ref::<PyBytes>() { + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + *zelf.0.buffer.write() = alloc::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); + *zelf.0.objects.write() = Some(value); + *zelf.0.base.write() = Some(kept_alive); + return Ok(()); + } + } else if type_code == "Z" + && let Some(s) = value.downcast_ref::<PyStr>() + { + let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_wtf8(), vm); + *zelf.0.buffer.write() = alloc::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); + *zelf.0.objects.write() = Some(holder); + return Ok(()); + } + + let content = set_primitive(&type_code, &value, vm)?; + + // Check if this is a swapped endian type (presence of attribute indicates swapping) + let swapped = instance + .class() + .as_object() + .get_attr("_swappedbytes_", vm) + .is_ok(); + + // Update buffer when value changes + let buffer_bytes = value_to_bytes_endian(&type_code, &content, swapped, vm); + + // If the buffer is borrowed (from shared memory), write in-place + // Otherwise replace with new owned buffer + let mut buffer = zelf.0.buffer.write(); + match &mut *buffer { + Cow::Borrowed(slice) => { + // SAFETY: For from_buffer, the slice points to writable shared memory. + // Python's from_buffer requires writable buffer, so this is safe. + let ptr = slice.as_ptr() as *mut u8; + let len = slice.len().min(buffer_bytes.len()); + unsafe { + core::ptr::copy_nonoverlapping(buffer_bytes.as_ptr(), ptr, len); + } + } + Cow::Owned(vec) => { + vec.copy_from_slice(&buffer_bytes); + } + } + + // For c_char_p (type "z"), c_wchar_p (type "Z"), and py_object (type "O"), + // keep the reference in _objects + if type_code == "z" || type_code == "Z" || type_code == "O" { + *zelf.0.objects.write() = Some(value); + } + Ok(()) + } + + #[pyclassmethod] + fn repeat(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { + use super::array::array_type_from_ctype; + + if n < 0 { + return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); + } + // Use cached array type creation + array_type_from_ctype(cls.into(), n as usize, vm) + } + + /// Simple_from_outparm - convert output parameter back to Python value + /// For direct subclasses of _SimpleCData (e.g., c_int), returns the value. + /// For subclasses of those (e.g., class MyInt(c_int)), returns self. + #[pymethod] + fn __ctypes_from_outparam__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // _ctypes_simple_instance: returns true if NOT a direct subclass of Simple_Type + // i.e., c_int (direct) -> false, MyInt(c_int) (subclass) -> true + let is_subclass_of_simple = { + let cls = zelf.class(); + let bases = cls.bases.read(); + // If base is NOT _SimpleCData, then it's a subclass of a subclass + !bases + .iter() + .any(|base| base.name().to_string() == "_SimpleCData") + }; + + if is_subclass_of_simple { + // Subclass of simple type (e.g., MyInt(c_int)): return self + Ok(zelf.into()) + } else { + // Direct simple type (e.g., c_int): return value + PyCSimple::value(zelf.into(), vm) + } + } +} + +impl PyCSimple { + /// Extract the value from this ctypes object as an owned FfiArgValue. + /// The value must be kept alive until after the FFI call completes. + pub fn to_ffi_value( + &self, + ty: libffi::middle::Type, + _vm: &VirtualMachine, + ) -> Option<FfiArgValue> { + let buffer = self.0.buffer.read(); + let bytes: &[u8] = &buffer; + + let ret = if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u8().as_raw_ptr()) { + let byte = *bytes.first()?; + FfiArgValue::U8(byte) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i8().as_raw_ptr()) { + let byte = *bytes.first()?; + FfiArgValue::I8(byte as i8) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u16().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<2>()?; + FfiArgValue::U16(u16::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i16().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<2>()?; + FfiArgValue::I16(i16::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u32().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<4>()?; + FfiArgValue::U32(u32::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i32().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<4>()?; + FfiArgValue::I32(i32::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::u64().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<8>()?; + FfiArgValue::U64(u64::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::i64().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<8>()?; + FfiArgValue::I64(i64::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::f32().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<4>()?; + FfiArgValue::F32(f32::from_ne_bytes(bytes)) + } else if core::ptr::eq(ty.as_raw_ptr(), libffi::middle::Type::f64().as_raw_ptr()) { + let bytes = *bytes.first_chunk::<8>()?; + FfiArgValue::F64(f64::from_ne_bytes(bytes)) + } else if core::ptr::eq( + ty.as_raw_ptr(), + libffi::middle::Type::pointer().as_raw_ptr(), + ) { + let bytes = *buffer.first_chunk::<{ size_of::<usize>() }>()?; + let val = usize::from_ne_bytes(bytes); + FfiArgValue::Pointer(val) + } else { + return None; + }; + Some(ret) + } +} + +impl AsBuffer for PyCSimple { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let stg_info = zelf + .class() + .stg_info_opt() + .expect("PyCSimple type must have StgInfo"); + let format = stg_info + .format + .clone() + .map(Cow::Owned) + .unwrap_or(Cow::Borrowed("B")); + let itemsize = stg_info.size; + // Simple types are scalars with ndim=0, shape=() + let desc = BufferDescriptor { + len: itemsize, + readonly: false, + itemsize, + format, + dim_desc: vec![], + }; + let buf = PyBuffer::new(zelf.to_owned().into(), desc, &CDATA_BUFFER_METHODS); + Ok(buf) + } +} + +/// Simple_bool: return non-zero if any byte in buffer is non-zero +impl AsNumber for PyCSimple { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + boolean: Some(|obj, _vm| { + let zelf = obj + .downcast_ref::<PyCSimple>() + .expect("PyCSimple::as_number called on non-PyCSimple"); + let buffer = zelf.0.buffer.read(); + // Simple_bool: memcmp(self->b_ptr, zeros, self->b_size) + // Returns true if any byte is non-zero + Ok(buffer.iter().any(|&b| b != 0)) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} diff --git a/crates/vm/src/stdlib/_ctypes/structure.rs b/crates/vm/src/stdlib/_ctypes/structure.rs new file mode 100644 index 00000000000..c9ab9205601 --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes/structure.rs @@ -0,0 +1,837 @@ +use super::base::{CDATA_BUFFER_METHODS, PyCData, PyCField, StgInfo, StgInfoFlags}; +use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef, PyUtf8Str}; +use crate::convert::ToPyObject; +use crate::function::{FuncArgs, OptionalArg, PySetterValue}; +use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; +use crate::stdlib::_warnings; +use crate::types::{AsBuffer, AsNumber, Constructor, Initializer, SetAttr}; +use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; +use alloc::borrow::Cow; +use core::fmt::Debug; +use num_traits::ToPrimitive; + +/// Calculate Structure type size from _fields_ (sum of field sizes) +pub(super) fn calculate_struct_size(cls: &Py<PyType>, vm: &VirtualMachine) -> PyResult<usize> { + if let Ok(fields_attr) = cls.as_object().get_attr("_fields_", vm) { + let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm)?; + let mut total_size = 0usize; + + for field in fields.iter() { + if let Some(tuple) = field.downcast_ref::<PyTuple>() + && let Some(field_type) = tuple.get(1) + { + total_size += super::_ctypes::sizeof(field_type.clone(), vm)?; + } + } + return Ok(total_size); + } + Ok(0) +} + +/// PyCStructType - metaclass for Structure +#[pyclass(name = "PyCStructType", base = PyType, module = "_ctypes")] +#[derive(Debug)] +#[repr(transparent)] +pub(super) struct PyCStructType(PyType); + +impl Constructor for PyCStructType { + type Args = FuncArgs; + + fn slot_new(metatype: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // 1. Create the new class using PyType::slot_new + let new_class = crate::builtins::type_::PyType::slot_new(metatype, args, vm)?; + + // 2. Get the new type + let new_type = new_class + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("expected type"))?; + + // 3. Mark base classes as finalized (subclassing finalizes the parent) + new_type.mark_bases_final(); + + // 4. Initialize StgInfo for the new type (initialized=false, to be set in init) + let stg_info = StgInfo::default(); + let _ = new_type.init_type_data(stg_info); + + // Note: _fields_ processing moved to Initializer::init() + Ok(new_class) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyCStructType { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Get the type as PyTypeRef by converting PyRef<Self> -> PyObjectRef -> PyRef<PyType> + let obj: PyObjectRef = zelf.clone().into(); + let new_type: PyTypeRef = obj + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + // Backward compatibility: skip initialization for abstract types + if new_type + .get_direct_attr(vm.ctx.intern_str("_abstract_")) + .is_some() + { + return Ok(()); + } + + new_type.check_not_initialized(vm)?; + + // Process _fields_ if defined directly on this class (not inherited) + if let Some(fields_attr) = new_type.get_direct_attr(vm.ctx.intern_str("_fields_")) { + Self::process_fields(&new_type, fields_attr, vm)?; + } else { + // No _fields_ defined - try to copy from base class (PyCStgInfo_clone) + let (has_base_info, base_clone) = { + let bases = new_type.bases.read(); + if let Some(base) = bases.first() { + (base.stg_info_opt().is_some(), Some(base.clone())) + } else { + (false, None) + } + }; + + if has_base_info && let Some(ref base) = base_clone { + // Clone base StgInfo (release guard before getting mutable reference) + let stg_info_opt = base.stg_info_opt().map(|baseinfo| { + let mut stg_info = baseinfo.clone(); + stg_info.flags &= !StgInfoFlags::DICTFLAG_FINAL; // Clear FINAL in subclass + stg_info.initialized = true; + stg_info.pointer_type = None; // Non-inheritable + stg_info + }); + + if let Some(stg_info) = stg_info_opt { + // Mark base as FINAL (now guard is released) + if let Some(mut base_stg) = base.get_type_data_mut::<StgInfo>() { + base_stg.flags |= StgInfoFlags::DICTFLAG_FINAL; + } + + super::base::set_or_init_stginfo(&new_type, stg_info); + return Ok(()); + } + } + + // No base StgInfo - create default + let mut stg_info = StgInfo::new(0, 1); + stg_info.paramfunc = super::base::ParamFunc::Structure; + stg_info.format = Some("B".to_string()); + super::base::set_or_init_stginfo(&new_type, stg_info); + } + + Ok(()) + } +} + +#[pyclass(flags(BASETYPE), with(AsNumber, Constructor, Initializer, SetAttr))] +impl PyCStructType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the structure type class that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. If already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } + + // 2. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCStructType::from_param(cls.as_object().to_owned(), as_parameter, vm); + } + + Err(vm.new_type_error(format!( + "expected {} instance instead of {}", + cls.name(), + value.class().name() + ))) + } + + // CDataType methods - delegated to PyCData implementations + + #[pymethod] + fn from_address(zelf: PyObjectRef, address: isize, vm: &VirtualMachine) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_address(cls, address, vm) + } + + #[pymethod] + fn from_buffer( + zelf: PyObjectRef, + source: PyObjectRef, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer(cls, source, offset, vm) + } + + #[pymethod] + fn from_buffer_copy( + zelf: PyObjectRef, + source: crate::function::ArgBytesLike, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer_copy(cls, source, offset, vm) + } + + #[pymethod] + fn in_dll( + zelf: PyObjectRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::in_dll(cls, dll, name, vm) + } + + /// Called when a new Structure subclass is created + #[pyclassmethod] + fn __init_subclass__(cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { + cls.mark_bases_final(); + + // Check if _fields_ is defined + if let Some(fields_attr) = cls.get_direct_attr(vm.ctx.intern_str("_fields_")) { + Self::process_fields(&cls, fields_attr, vm)?; + } + Ok(()) + } + + /// Process _fields_ and create CField descriptors + fn process_fields( + cls: &Py<PyType>, + fields_attr: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if this is a swapped byte order structure + let is_swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + + // Try to downcast to list or tuple + let fields: Vec<PyObjectRef> = if let Some(list) = fields_attr.downcast_ref::<PyList>() { + list.borrow_vec().to_vec() + } else if let Some(tuple) = fields_attr.downcast_ref::<PyTuple>() { + tuple.to_vec() + } else { + return Err(vm.new_type_error("_fields_ must be a list or tuple")); + }; + + let pack = super::base::get_usize_attr(cls.as_object(), "_pack_", 0, vm)?; + + // Emit DeprecationWarning on non-Windows when _pack_ is set without _layout_ + if pack > 0 && !cfg!(windows) { + let has_layout = cls.as_object().get_attr("_layout_", vm).is_ok(); + if !has_layout { + let base_type_name = "Structure"; + let msg = format!( + "Due to '_pack_', the '{}' {} will use memory layout compatible with \ + MSVC (Windows). If this is intended, set _layout_ to 'ms'. \ + The implicit default is deprecated and slated to become an error in \ + Python 3.19.", + cls.name(), + base_type_name, + ); + _warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm)?; + } + } + + let forced_alignment = + super::base::get_usize_attr(cls.as_object(), "_align_", 1, vm)?.max(1); + + // Determine byte order for format string + let big_endian = super::base::is_big_endian(is_swapped); + + // Initialize offset, alignment, type flags, and ffi_field_types from base class + let ( + mut offset, + mut max_align, + mut has_pointer, + mut has_union, + mut has_bitfield, + mut ffi_field_types, + ) = { + let bases = cls.bases.read(); + if let Some(base) = bases.first() + && let Some(baseinfo) = base.stg_info_opt() + { + ( + baseinfo.size, + core::cmp::max(baseinfo.align, forced_alignment), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASPOINTER), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASUNION), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD), + baseinfo.ffi_field_types.clone(), + ) + } else { + (0, forced_alignment, false, false, false, Vec::new()) + } + }; + + // Initialize PEP3118 format string + let mut format = String::from("T{"); + let mut last_end = 0usize; // Track end of last field for padding calculation + + // Bitfield layout tracking + let mut bitfield_bit_offset: u16 = 0; // Current bit position within bitfield group + let mut last_field_bit_size: u16 = 0; // For MSVC: bit size of previous storage unit + let use_msvc_bitfields = pack > 0; // MSVC layout when _pack_ is set + + for (index, field) in fields.iter().enumerate() { + let field_tuple = field + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("_fields_ must contain tuples"))?; + + if field_tuple.len() < 2 { + return Err( + vm.new_type_error("_fields_ tuple must have at least 2 elements (name, type)") + ); + } + + let name = field_tuple + .first() + .expect("len checked") + .downcast_ref::<PyUtf8Str>() + .ok_or_else(|| vm.new_type_error("field name must be a string"))? + .as_str() + .to_owned(); + + let field_type = field_tuple.get(1).expect("len checked").clone(); + + // For swapped byte order structures, validate field type supports byte swapping + if is_swapped { + super::base::check_other_endian_support(&field_type, vm)?; + } + + // Get size and alignment of the field type + let size = super::base::get_field_size(&field_type, vm)?; + let field_align = super::base::get_field_align(&field_type, vm); + + // Calculate effective alignment (PyCField_FromDesc) + let effective_align = if pack > 0 { + core::cmp::min(pack, field_align) + } else { + field_align + }; + + // Apply padding to align offset (cfield.c NO_BITFIELD case) + if effective_align > 0 && offset % effective_align != 0 { + let delta = effective_align - (offset % effective_align); + offset += delta; + } + + max_align = max_align.max(effective_align); + + // Propagate type flags from field type (HASPOINTER, HASUNION, HASBITFIELD) + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(field_stg) = type_obj.stg_info_opt() + { + // HASPOINTER: propagate if field is pointer or contains pointer + if field_stg.flags.intersects( + StgInfoFlags::TYPEFLAG_ISPOINTER | StgInfoFlags::TYPEFLAG_HASPOINTER, + ) { + has_pointer = true; + } + // HASUNION, HASBITFIELD: propagate directly + if field_stg.flags.contains(StgInfoFlags::TYPEFLAG_HASUNION) { + has_union = true; + } + if field_stg.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD) { + has_bitfield = true; + } + // Collect FFI type for this field + ffi_field_types.push(field_stg.to_ffi_type()); + } + + // Mark field type as finalized (using type as field finalizes it) + if let Some(type_obj) = field_type.downcast_ref::<PyType>() { + if let Some(mut stg_info) = type_obj.get_type_data_mut::<StgInfo>() { + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; + } else { + // Create StgInfo with FINAL flag if it doesn't exist + let mut stg_info = StgInfo::new(size, field_align); + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; + let _ = type_obj.init_type_data(stg_info); + } + } + + // Build format string: add padding before field + let padding = offset - last_end; + if padding > 0 { + if padding != 1 { + format.push_str(&padding.to_string()); + } + format.push('x'); + } + + // Get field format and add to format string + let field_format = super::base::get_field_format(&field_type, big_endian, vm); + + // Handle arrays: prepend shape + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(field_stg) = type_obj.stg_info_opt() + && !field_stg.shape.is_empty() + { + let shape_str = field_stg + .shape + .iter() + .map(|d| d.to_string()) + .collect::<Vec<_>>() + .join(","); + format.push_str(&std::format!("({}){}", shape_str, field_format)); + } else { + format.push_str(&field_format); + } + + // Add field name + format.push(':'); + format.push_str(&name); + format.push(':'); + + // Create CField descriptor with padding-adjusted offset + let field_type_ref = field_type + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("_fields_ type must be a ctypes type"))?; + + // Check for bitfield size (optional 3rd element in tuple) + let (c_field, field_advances_offset) = if field_tuple.len() > 2 { + let bit_size_obj = field_tuple.get(2).expect("len checked"); + let bit_size = bit_size_obj + .try_int(vm)? + .as_bigint() + .to_u16() + .ok_or_else(|| vm.new_value_error("number of bits invalid for bit field"))?; + has_bitfield = true; + + let type_bits = (size * 8) as u16; + let (advances, bit_offset); + + if use_msvc_bitfields { + // MSVC layout: different types start new storage unit + if bitfield_bit_offset + bit_size > type_bits + || type_bits != last_field_bit_size + { + // Close previous bitfield, start new allocation unit + bitfield_bit_offset = 0; + advances = true; + } else { + advances = false; + } + bit_offset = bitfield_bit_offset; + bitfield_bit_offset += bit_size; + last_field_bit_size = type_bits; + } else { + // GCC System V layout: pack within same type + let fits_in_current = bitfield_bit_offset + bit_size <= type_bits; + advances = if fits_in_current && bitfield_bit_offset > 0 { + false + } else if !fits_in_current { + bitfield_bit_offset = 0; + true + } else { + true + }; + bit_offset = bitfield_bit_offset; + bitfield_bit_offset += bit_size; + } + + // For packed bitfields that share offset, use the same offset as previous + let field_offset = if !advances { + offset - size // Reuse the previous field's offset + } else { + offset + }; + + ( + PyCField::new_bitfield( + name.clone(), + field_type_ref, + field_offset as isize, + size as isize, + bit_size, + bit_offset, + index, + ), + advances, + ) + } else { + bitfield_bit_offset = 0; // Reset on non-bitfield + last_field_bit_size = 0; + ( + PyCField::new( + name.clone(), + field_type_ref, + offset as isize, + size as isize, + index, + ), + true, + ) + }; + + // Set the CField as a class attribute + cls.set_attr(vm.ctx.intern_str(name.clone()), c_field.to_pyobject(vm)); + + // Update tracking - don't advance offset for packed bitfields + if field_advances_offset { + last_end = offset + size; + offset += size; + } + } + + // Calculate total_align = max(max_align, forced_alignment) + let total_align = core::cmp::max(max_align, forced_alignment); + + // Calculate aligned_size (PyCStructUnionType_update_stginfo) + let aligned_size = if total_align > 0 { + offset.div_ceil(total_align) * total_align + } else { + offset + }; + + // Complete format string: add final padding and close + let final_padding = aligned_size - last_end; + if final_padding > 0 { + if final_padding != 1 { + format.push_str(&final_padding.to_string()); + } + format.push('x'); + } + format.push('}'); + + // Check for circular self-reference: if a field of the same type as this + // structure was encountered, it would have marked this type's stginfo as FINAL. + if let Some(stg_info) = cls.get_type_data::<StgInfo>() + && stg_info.is_final() + { + return Err(vm.new_attribute_error("Structure or union cannot contain itself")); + } + + // Store StgInfo with aligned size and total alignment + let mut stg_info = StgInfo::new(aligned_size, total_align); + stg_info.length = fields.len(); + stg_info.format = Some(format); + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; // Mark as finalized + if has_pointer { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASPOINTER; + } + if has_union { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASUNION; + } + if has_bitfield { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASBITFIELD; + } + stg_info.paramfunc = super::base::ParamFunc::Structure; + // Set byte order: swap if _swappedbytes_ is defined + stg_info.big_endian = super::base::is_big_endian(is_swapped); + // Store FFI field types for structure passing + stg_info.ffi_field_types = ffi_field_types; + super::base::set_or_init_stginfo(cls, stg_info); + + // Process _anonymous_ fields + super::base::make_anon_fields(cls, vm)?; + + Ok(()) + } + + fn __mul__(cls: PyTypeRef, n: isize, vm: &VirtualMachine) -> PyResult { + use super::array::array_type_from_ctype; + + if n < 0 { + return Err(vm.new_value_error(format!("Array length must be >= 0, not {n}"))); + } + // Use cached array type creation + array_type_from_ctype(cls.into(), n as usize, vm) + } +} + +impl AsNumber for PyCStructType { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + multiply: Some(|a, b, vm| { + let cls = a + .downcast_ref::<PyType>() + .ok_or_else(|| vm.new_type_error("expected type"))?; + let n = b + .try_index(vm)? + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("array size too large"))?; + PyCStructType::__mul__(cls.to_owned(), n, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + +impl SetAttr for PyCStructType { + fn setattro( + zelf: &Py<Self>, + attr_name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if _fields_ is being set + if attr_name.as_bytes() == b"_fields_" { + let pytype: &Py<PyType> = zelf.to_base(); + + // Check finalization in separate scope to release read lock before process_fields + // This prevents deadlock: process_fields needs write lock on the same RwLock + let is_final = { + let Some(stg_info) = pytype.get_type_data::<StgInfo>() else { + return Err(vm.new_type_error("ctypes state is not initialized")); + }; + stg_info.is_final() + }; // Read lock released here + + if is_final { + return Err(vm.new_attribute_error("_fields_ is final")); + } + + // Process _fields_ and set attribute + let PySetterValue::Assign(fields_value) = value else { + return Err(vm.new_attribute_error("cannot delete _fields_")); + }; + // Process fields (this will also set DICTFLAG_FINAL) + PyCStructType::process_fields(pytype, fields_value.clone(), vm)?; + // Set the _fields_ attribute on the type + pytype + .attributes + .write() + .insert(vm.ctx.intern_str("_fields_"), fields_value); + return Ok(()); + } + // Delegate to PyType's setattro logic for type attributes + let attr_name_interned = vm.ctx.intern_str(attr_name.as_wtf8()); + let pytype: &Py<PyType> = zelf.to_base(); + + // Check for data descriptor first + if let Some(attr) = pytype.get_class_attr(attr_name_interned) { + let descr_set = attr.class().slots.descr_set.load(); + if let Some(descriptor) = descr_set { + return descriptor(&attr, pytype.to_owned().into(), value, vm); + } + } + + // Store in type's attributes dict + if let PySetterValue::Assign(value) = value { + pytype.attributes.write().insert(attr_name_interned, value); + } else { + let prev = pytype.attributes.write().shift_remove(attr_name_interned); + if prev.is_none() { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '{}'", + pytype.name(), + attr_name.as_wtf8(), + ))); + } + } + Ok(()) + } +} + +/// PyCStructure - base class for Structure instances +#[pyclass( + module = "_ctypes", + name = "Structure", + base = PyCData, + metaclass = "PyCStructType" +)] +#[repr(transparent)] +pub struct PyCStructure(pub PyCData); + +impl Debug for PyCStructure { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyCStructure") + .field("size", &self.0.size()) + .finish() + } +} + +impl Constructor for PyCStructure { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Check for abstract class and extract values in a block to drop the borrow + let (total_size, total_align, length) = { + let stg_info = cls.stg_info(vm)?; + (stg_info.size, stg_info.align, stg_info.length) + }; + + // Mark the class as finalized (instance creation finalizes the type) + if let Some(mut stg_info_mut) = cls.get_type_data_mut::<StgInfo>() { + stg_info_mut.flags |= StgInfoFlags::DICTFLAG_FINAL; + } + + // Initialize buffer with zeros using computed size + let mut new_stg_info = StgInfo::new(total_size, total_align); + new_stg_info.length = length; + PyCStructure(PyCData::from_stg_info(&new_stg_info)) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl PyCStructure { + /// Recursively initialize positional arguments through inheritance chain + /// Returns the number of arguments consumed + fn init_pos_args( + self_obj: &Py<Self>, + type_obj: &Py<PyType>, + args: &[PyObjectRef], + kwargs: &indexmap::IndexMap<String, PyObjectRef>, + index: usize, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let mut current_index = index; + + // 1. First process base class fields recursively + let base_clone = { + let bases = type_obj.bases.read(); + if let Some(base) = bases.first() + && base.stg_info_opt().is_some() + { + Some(base.clone()) + } else { + None + } + }; + + if let Some(ref base) = base_clone { + current_index = Self::init_pos_args(self_obj, base, args, kwargs, current_index, vm)?; + } + + // 2. Process this class's _fields_ + if let Some(fields_attr) = type_obj.get_direct_attr(vm.ctx.intern_str("_fields_")) { + let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm)?; + + for field in fields.iter() { + if current_index >= args.len() { + break; + } + if let Some(tuple) = field.downcast_ref::<PyTuple>() + && let Some(name) = tuple.first() + && let Some(name_str) = name.downcast_ref::<PyUtf8Str>() + { + let field_name = name_str.as_str().to_owned(); + // Check for duplicate in kwargs + if kwargs.contains_key(&field_name) { + return Err(vm.new_type_error(format!( + "duplicate values for field {:?}", + field_name + ))); + } + self_obj.as_object().set_attr( + vm.ctx.intern_str(field_name), + args[current_index].clone(), + vm, + )?; + current_index += 1; + } + } + } + + Ok(current_index) + } +} + +impl Initializer for PyCStructure { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Struct_init: handle positional and keyword arguments + let cls = zelf.class().to_owned(); + + // 1. Process positional arguments recursively through inheritance chain + if !args.args.is_empty() { + let consumed = + PyCStructure::init_pos_args(&zelf, &cls, &args.args, &args.kwargs, 0, vm)?; + + if consumed < args.args.len() { + return Err(vm.new_type_error("too many initializers")); + } + } + + // 2. Process keyword arguments + for (key, value) in args.kwargs.iter() { + zelf.as_object() + .set_attr(vm.ctx.intern_str(key.as_str()), value.clone(), vm)?; + } + + Ok(()) + } +} + +// Note: GetAttr and SetAttr are not implemented here. +// Field access is handled by CField descriptors registered on the class. + +#[pyclass( + flags(BASETYPE, IMMUTABLETYPE), + with(Constructor, Initializer, AsBuffer) +)] +impl PyCStructure { + #[pygetset] + fn _b0_(&self) -> Option<PyObjectRef> { + self.0.base.read().clone() + } +} + +impl AsBuffer for PyCStructure { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let buffer_len = zelf.0.buffer.read().len(); + + // PyCData_NewGetBuffer: use info->format if available, otherwise "B" + let format = zelf + .class() + .stg_info_opt() + .and_then(|info| info.format.clone()) + .unwrap_or_else(|| "B".to_string()); + + // Structure: ndim=0, shape=(), itemsize=struct_size + let buf = PyBuffer::new( + zelf.to_owned().into(), + BufferDescriptor { + len: buffer_len, + readonly: false, + itemsize: buffer_len, + format: Cow::Owned(format), + dim_desc: vec![], // ndim=0 means empty dim_desc + }, + &CDATA_BUFFER_METHODS, + ); + Ok(buf) + } +} diff --git a/crates/vm/src/stdlib/_ctypes/union.rs b/crates/vm/src/stdlib/_ctypes/union.rs new file mode 100644 index 00000000000..c1882141e98 --- /dev/null +++ b/crates/vm/src/stdlib/_ctypes/union.rs @@ -0,0 +1,699 @@ +use super::base::{CDATA_BUFFER_METHODS, StgInfoFlags}; +use super::{PyCData, PyCField, StgInfo}; +use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef, PyUtf8Str}; +use crate::convert::ToPyObject; +use crate::function::{ArgBytesLike, FuncArgs, OptionalArg, PySetterValue}; +use crate::protocol::{BufferDescriptor, PyBuffer}; +use crate::stdlib::_warnings; +use crate::types::{AsBuffer, Constructor, Initializer, SetAttr}; +use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; +use alloc::borrow::Cow; +use num_traits::ToPrimitive; + +/// Calculate Union type size from _fields_ (max field size) +pub(super) fn calculate_union_size(cls: &Py<PyType>, vm: &VirtualMachine) -> PyResult<usize> { + if let Ok(fields_attr) = cls.as_object().get_attr("_fields_", vm) { + let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm)?; + let mut max_size = 0usize; + + for field in fields.iter() { + if let Some(tuple) = field.downcast_ref::<PyTuple>() + && let Some(field_type) = tuple.get(1) + { + let field_size = super::_ctypes::sizeof(field_type.clone(), vm)?; + max_size = max_size.max(field_size); + } + } + return Ok(max_size); + } + Ok(0) +} + +/// PyCUnionType - metaclass for Union +#[pyclass(name = "UnionType", base = PyType, module = "_ctypes")] +#[derive(Debug)] +#[repr(transparent)] +pub(super) struct PyCUnionType(PyType); + +impl Constructor for PyCUnionType { + type Args = FuncArgs; + + fn slot_new(metatype: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // 1. Create the new class using PyType::slot_new + let new_class = crate::builtins::PyType::slot_new(metatype, args, vm)?; + + // 2. Get the new type + let new_type = new_class + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("expected type"))?; + + // 3. Mark base classes as finalized (subclassing finalizes the parent) + new_type.mark_bases_final(); + + // 4. Initialize StgInfo for the new type (initialized=false, to be set in init) + let stg_info = StgInfo::default(); + let _ = new_type.init_type_data(stg_info); + + // Note: _fields_ processing moved to Initializer::init() + Ok(new_class) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl Initializer for PyCUnionType { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Get the type as PyTypeRef by converting PyRef<Self> -> PyObjectRef -> PyRef<PyType> + let obj: PyObjectRef = zelf.clone().into(); + let new_type: PyTypeRef = obj + .downcast() + .map_err(|_| vm.new_type_error("expected type"))?; + + // Check for _abstract_ attribute - skip initialization if present + if new_type + .get_direct_attr(vm.ctx.intern_str("_abstract_")) + .is_some() + { + return Ok(()); + } + + new_type.check_not_initialized(vm)?; + + // Process _fields_ if defined directly on this class (not inherited) + // Use set_attr to trigger setattro + if let Some(fields_attr) = new_type.get_direct_attr(vm.ctx.intern_str("_fields_")) { + new_type + .as_object() + .set_attr(vm.ctx.intern_str("_fields_"), fields_attr, vm)?; + } else { + // No _fields_ defined - try to copy from base class + let (has_base_info, base_clone) = { + let bases = new_type.bases.read(); + if let Some(base) = bases.first() { + (base.stg_info_opt().is_some(), Some(base.clone())) + } else { + (false, None) + } + }; + + if has_base_info && let Some(ref base) = base_clone { + // Clone base StgInfo (release guard before getting mutable reference) + let stg_info_opt = base.stg_info_opt().map(|baseinfo| { + let mut stg_info = baseinfo.clone(); + stg_info.flags &= !StgInfoFlags::DICTFLAG_FINAL; // Clear FINAL flag in subclass + stg_info.initialized = true; + stg_info.pointer_type = None; // Non-inheritable + stg_info + }); + + if let Some(stg_info) = stg_info_opt { + // Mark base as FINAL (now guard is released) + if let Some(mut base_stg) = base.get_type_data_mut::<StgInfo>() { + base_stg.flags |= StgInfoFlags::DICTFLAG_FINAL; + } + + super::base::set_or_init_stginfo(&new_type, stg_info); + return Ok(()); + } + } + + // No base StgInfo - create default + let mut stg_info = StgInfo::new(0, 1); + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASUNION; + stg_info.paramfunc = super::base::ParamFunc::Union; + // PEP 3118 doesn't support union. Use 'B' for bytes. + stg_info.format = Some("B".to_string()); + super::base::set_or_init_stginfo(&new_type, stg_info); + } + + Ok(()) + } +} + +impl PyCUnionType { + /// Process _fields_ and create CField descriptors + /// For Union, all fields start at offset 0 + fn process_fields( + cls: &Py<PyType>, + fields_attr: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Check if already finalized + { + let Some(stg_info) = cls.get_type_data::<StgInfo>() else { + return Err(vm.new_type_error("ctypes state is not initialized")); + }; + if stg_info.is_final() { + return Err(vm.new_attribute_error("_fields_ is final")); + } + } // Read lock released here + + // Check if this is a swapped byte order union + let is_swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + + let fields: Vec<PyObjectRef> = if let Some(list) = fields_attr.downcast_ref::<PyList>() { + list.borrow_vec().to_vec() + } else if let Some(tuple) = fields_attr.downcast_ref::<PyTuple>() { + tuple.to_vec() + } else { + return Err(vm.new_type_error("_fields_ must be a list or tuple")); + }; + + let pack = super::base::get_usize_attr(cls.as_object(), "_pack_", 0, vm)?; + + // Emit DeprecationWarning on non-Windows when _pack_ is set without _layout_ + if pack > 0 && !cfg!(windows) { + let has_layout = cls.as_object().get_attr("_layout_", vm).is_ok(); + if !has_layout { + let msg = format!( + "Due to '_pack_', the '{}' Union will use memory layout compatible with \ + MSVC (Windows). If this is intended, set _layout_ to 'ms'. \ + The implicit default is deprecated and slated to become an error in \ + Python 3.19.", + cls.name(), + ); + _warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm)?; + } + } + + let forced_alignment = + super::base::get_usize_attr(cls.as_object(), "_align_", 1, vm)?.max(1); + + // Initialize size, alignment, type flags, and ffi_field_types from base class + // Note: Union fields always start at offset 0, but we inherit base size/align + let (mut max_size, mut max_align, mut has_pointer, mut has_bitfield, mut ffi_field_types) = { + let bases = cls.bases.read(); + if let Some(base) = bases.first() + && let Some(baseinfo) = base.stg_info_opt() + { + ( + baseinfo.size, + core::cmp::max(baseinfo.align, forced_alignment), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASPOINTER), + baseinfo.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD), + baseinfo.ffi_field_types.clone(), + ) + } else { + (0, forced_alignment, false, false, Vec::new()) + } + }; + + for (index, field) in fields.iter().enumerate() { + let field_tuple = field + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("_fields_ must contain tuples"))?; + + if field_tuple.len() < 2 { + return Err( + vm.new_type_error("_fields_ tuple must have at least 2 elements (name, type)") + ); + } + + let name = field_tuple + .first() + .expect("len checked") + .downcast_ref::<PyUtf8Str>() + .ok_or_else(|| vm.new_type_error("field name must be a string"))? + .as_str() + .to_owned(); + + let field_type = field_tuple.get(1).expect("len checked").clone(); + + // For swapped byte order unions, validate field type supports byte swapping + if is_swapped { + super::base::check_other_endian_support(&field_type, vm)?; + } + + let size = super::base::get_field_size(&field_type, vm)?; + let field_align = super::base::get_field_align(&field_type, vm); + + // Calculate effective alignment + let effective_align = if pack > 0 { + core::cmp::min(pack, field_align) + } else { + field_align + }; + + max_size = max_size.max(size); + max_align = max_align.max(effective_align); + + // Propagate type flags from field type (HASPOINTER, HASBITFIELD) + if let Some(type_obj) = field_type.downcast_ref::<PyType>() + && let Some(field_stg) = type_obj.stg_info_opt() + { + // HASPOINTER: propagate if field is pointer or contains pointer + if field_stg.flags.intersects( + StgInfoFlags::TYPEFLAG_ISPOINTER | StgInfoFlags::TYPEFLAG_HASPOINTER, + ) { + has_pointer = true; + } + // HASBITFIELD: propagate directly + if field_stg.flags.contains(StgInfoFlags::TYPEFLAG_HASBITFIELD) { + has_bitfield = true; + } + // Collect FFI type for this field + ffi_field_types.push(field_stg.to_ffi_type()); + } + + // Mark field type as finalized (using type as field finalizes it) + if let Some(type_obj) = field_type.downcast_ref::<PyType>() { + if let Some(mut stg_info) = type_obj.get_type_data_mut::<StgInfo>() { + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; + } else { + // Create StgInfo with FINAL flag if it doesn't exist + let mut stg_info = StgInfo::new(size, field_align); + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL; + let _ = type_obj.init_type_data(stg_info); + } + } + + // For Union, all fields start at offset 0 + let field_type_ref = field_type + .clone() + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("_fields_ type must be a ctypes type"))?; + + // Check for bitfield size (optional 3rd element in tuple) + // For unions, each field starts fresh (CPython: _layout.py) + let c_field = if field_tuple.len() > 2 { + let bit_size_obj = field_tuple.get(2).expect("len checked"); + let bit_size = bit_size_obj + .try_int(vm)? + .as_bigint() + .to_u16() + .ok_or_else(|| vm.new_value_error("number of bits invalid for bit field"))?; + has_bitfield = true; + + // Union fields all start at offset 0, so bit_offset = 0 + let mut bit_offset: u16 = 0; + let type_bits = (size * 8) as u16; + + // Big-endian: bit_offset = type_bits - bit_size + let big_endian = is_swapped != cfg!(target_endian = "big"); + if big_endian && type_bits >= bit_size { + bit_offset = type_bits - bit_size; + } + + PyCField::new_bitfield( + name.clone(), + field_type_ref, + 0, // Union fields always at offset 0 + size as isize, + bit_size, + bit_offset, + index, + ) + } else { + PyCField::new(name.clone(), field_type_ref, 0, size as isize, index) + }; + + cls.set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm)); + } + + // Calculate total_align and aligned_size + let total_align = core::cmp::max(max_align, forced_alignment); + let aligned_size = if total_align > 0 { + max_size.div_ceil(total_align) * total_align + } else { + max_size + }; + + // Check for circular self-reference + if let Some(stg_info) = cls.get_type_data::<StgInfo>() + && stg_info.is_final() + { + return Err(vm.new_attribute_error("Structure or union cannot contain itself")); + } + + // Store StgInfo with aligned size + let mut stg_info = StgInfo::new(aligned_size, total_align); + stg_info.length = fields.len(); + stg_info.flags |= StgInfoFlags::DICTFLAG_FINAL | StgInfoFlags::TYPEFLAG_HASUNION; + // PEP 3118 doesn't support union. Use 'B' for bytes. + stg_info.format = Some("B".to_string()); + if has_pointer { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASPOINTER; + } + if has_bitfield { + stg_info.flags |= StgInfoFlags::TYPEFLAG_HASBITFIELD; + } + stg_info.paramfunc = super::base::ParamFunc::Union; + // Set byte order: swap if _swappedbytes_ is defined + stg_info.big_endian = super::base::is_big_endian(is_swapped); + // Store FFI field types for union passing + stg_info.ffi_field_types = ffi_field_types; + super::base::set_or_init_stginfo(cls, stg_info); + + // Process _anonymous_ fields + super::base::make_anon_fields(cls, vm)?; + + Ok(()) + } +} + +#[pyclass(flags(BASETYPE), with(Constructor, Initializer, SetAttr))] +impl PyCUnionType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + + #[pymethod] + fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // zelf is the union type class that from_param was called on + let cls = zelf + .downcast::<PyType>() + .map_err(|_| vm.new_type_error("from_param: expected a type"))?; + + // 1. If already an instance of the requested type, return it + if value.is_instance(cls.as_object(), vm)? { + return Ok(value); + } + + // 2. Check for CArgObject (PyCArg_CheckExact) + if let Some(carg) = value.downcast_ref::<super::_ctypes::CArgObject>() { + // Check against proto (for pointer types) + if let Some(stg_info) = cls.stg_info_opt() + && let Some(ref proto) = stg_info.proto + && carg.obj.is_instance(proto.as_object(), vm)? + { + return Ok(value); + } + // Fallback: check if the wrapped object is an instance of the requested type + if carg.obj.is_instance(cls.as_object(), vm)? { + return Ok(value); // Return the CArgObject as-is + } + // CArgObject but wrong type + return Err(vm.new_type_error(format!( + "expected {} instance instead of pointer to {}", + cls.name(), + carg.obj.class().name() + ))); + } + + // 3. Check for _as_parameter_ attribute + if let Ok(as_parameter) = value.get_attr("_as_parameter_", vm) { + return PyCUnionType::from_param(cls.as_object().to_owned(), as_parameter, vm); + } + + Err(vm.new_type_error(format!( + "expected {} instance instead of {}", + cls.name(), + value.class().name() + ))) + } + + // CDataType methods - delegated to PyCData implementations + + #[pymethod] + fn from_address(zelf: PyObjectRef, address: isize, vm: &VirtualMachine) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_address(cls, address, vm) + } + + #[pymethod] + fn from_buffer( + zelf: PyObjectRef, + source: PyObjectRef, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer(cls, source, offset, vm) + } + + #[pymethod] + fn from_buffer_copy( + zelf: PyObjectRef, + source: ArgBytesLike, + offset: OptionalArg<isize>, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer_copy(cls, source, offset, vm) + } + + #[pymethod] + fn in_dll( + zelf: PyObjectRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::in_dll(cls, dll, name, vm) + } + + /// Called when a new Union subclass is created + #[pyclassmethod] + fn __init_subclass__(cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { + cls.mark_bases_final(); + + // Check if _fields_ is defined + if let Some(fields_attr) = cls.get_direct_attr(vm.ctx.intern_str("_fields_")) { + Self::process_fields(&cls, fields_attr, vm)?; + } + Ok(()) + } +} + +impl SetAttr for PyCUnionType { + fn setattro( + zelf: &Py<Self>, + attr_name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + let pytype: &Py<PyType> = zelf.to_base(); + let attr_name_interned = vm.ctx.intern_str(attr_name.as_wtf8()); + + // 1. First, do PyType's setattro (PyType_Type.tp_setattro first) + // Check for data descriptor first + if let Some(attr) = pytype.get_class_attr(attr_name_interned) { + let descr_set = attr.class().slots.descr_set.load(); + if let Some(descriptor) = descr_set { + descriptor(&attr, pytype.to_owned().into(), value.clone(), vm)?; + // After successful setattro, check if _fields_ and call process_fields + if attr_name.as_bytes() == b"_fields_" + && let PySetterValue::Assign(fields_value) = value + { + PyCUnionType::process_fields(pytype, fields_value, vm)?; + } + return Ok(()); + } + } + + // 2. If _fields_, call process_fields (which checks FINAL internally) + // Check BEFORE writing to dict to avoid storing _fields_ when FINAL + if attr_name.as_bytes() == b"_fields_" + && let PySetterValue::Assign(ref fields_value) = value + { + PyCUnionType::process_fields(pytype, fields_value.clone(), vm)?; + } + + // Store in type's attributes dict + match &value { + PySetterValue::Assign(v) => { + pytype + .attributes + .write() + .insert(attr_name_interned, v.clone()); + } + PySetterValue::Delete => { + let prev = pytype.attributes.write().shift_remove(attr_name_interned); + if prev.is_none() { + return Err(vm.new_attribute_error(format!( + "type object '{}' has no attribute '{}'", + pytype.name(), + attr_name.as_wtf8(), + ))); + } + } + } + + Ok(()) + } +} + +/// PyCUnion - base class for Union +#[pyclass(module = "_ctypes", name = "Union", base = PyCData, metaclass = "PyCUnionType")] +#[repr(transparent)] +pub struct PyCUnion(pub PyCData); + +impl core::fmt::Debug for PyCUnion { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyCUnion") + .field("size", &self.0.size()) + .finish() + } +} + +impl Constructor for PyCUnion { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Check for abstract class and extract values in a block to drop the borrow + let (total_size, total_align, length) = { + let stg_info = cls.stg_info(vm)?; + (stg_info.size, stg_info.align, stg_info.length) + }; + + // Mark the class as finalized (instance creation finalizes the type) + if let Some(mut stg_info_mut) = cls.get_type_data_mut::<StgInfo>() { + stg_info_mut.flags |= StgInfoFlags::DICTFLAG_FINAL; + } + + // Initialize buffer with zeros using computed size + let mut new_stg_info = StgInfo::new(total_size, total_align); + new_stg_info.length = length; + PyCUnion(PyCData::from_stg_info(&new_stg_info)) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } +} + +impl PyCUnion { + /// Recursively initialize positional arguments through inheritance chain + /// Returns the number of arguments consumed + fn init_pos_args( + self_obj: &Py<Self>, + type_obj: &Py<PyType>, + args: &[PyObjectRef], + kwargs: &indexmap::IndexMap<String, PyObjectRef>, + index: usize, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let mut current_index = index; + + // 1. First process base class fields recursively + // Recurse if base has StgInfo + let base_clone = { + let bases = type_obj.bases.read(); + if let Some(base) = bases.first() && + // Check if base has StgInfo + base.stg_info_opt().is_some() + { + Some(base.clone()) + } else { + None + } + }; + + if let Some(ref base) = base_clone { + current_index = Self::init_pos_args(self_obj, base, args, kwargs, current_index, vm)?; + } + + // 2. Process this class's _fields_ + if let Some(fields_attr) = type_obj.get_direct_attr(vm.ctx.intern_str("_fields_")) { + let fields: Vec<PyObjectRef> = fields_attr.try_to_value(vm)?; + + for field in fields.iter() { + if current_index >= args.len() { + break; + } + if let Some(tuple) = field.downcast_ref::<PyTuple>() + && let Some(name) = tuple.first() + && let Some(name_str) = name.downcast_ref::<PyUtf8Str>() + { + let field_name = name_str.as_str().to_owned(); + // Check for duplicate in kwargs + if kwargs.contains_key(&field_name) { + return Err(vm.new_type_error(format!( + "duplicate values for field {:?}", + field_name + ))); + } + self_obj.as_object().set_attr( + vm.ctx.intern_str(field_name), + args[current_index].clone(), + vm, + )?; + current_index += 1; + } + } + } + + Ok(current_index) + } +} + +impl Initializer for PyCUnion { + type Args = FuncArgs; + + fn init(zelf: crate::PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // Struct_init: handle positional and keyword arguments + let cls = zelf.class().to_owned(); + + // 1. Process positional arguments recursively through inheritance chain + if !args.args.is_empty() { + let consumed = PyCUnion::init_pos_args(&zelf, &cls, &args.args, &args.kwargs, 0, vm)?; + + if consumed < args.args.len() { + return Err(vm.new_type_error("too many initializers")); + } + } + + // 2. Process keyword arguments + for (key, value) in args.kwargs.iter() { + zelf.as_object() + .set_attr(vm.ctx.intern_str(key.as_str()), value.clone(), vm)?; + } + + Ok(()) + } +} + +#[pyclass( + flags(BASETYPE, IMMUTABLETYPE), + with(Constructor, Initializer, AsBuffer) +)] +impl PyCUnion {} + +impl AsBuffer for PyCUnion { + fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { + let buffer_len = zelf.0.buffer.read().len(); + + // PyCData_NewGetBuffer: use info->format if available, otherwise "B" + let format = zelf + .class() + .stg_info_opt() + .and_then(|info| info.format.clone()) + .unwrap_or_else(|| "B".to_string()); + + // Union: ndim=0, shape=(), itemsize=union_size + let buf = PyBuffer::new( + zelf.to_owned().into(), + BufferDescriptor { + len: buffer_len, + readonly: false, + itemsize: buffer_len, + format: Cow::Owned(format), + dim_desc: vec![], // ndim=0 means empty dim_desc + }, + &CDATA_BUFFER_METHODS, + ); + Ok(buf) + } +} diff --git a/crates/vm/src/stdlib/_functools.rs b/crates/vm/src/stdlib/_functools.rs new file mode 100644 index 00000000000..494f0e7fd83 --- /dev/null +++ b/crates/vm/src/stdlib/_functools.rs @@ -0,0 +1,516 @@ +pub(crate) use _functools::module_def; + +#[pymodule] +mod _functools { + use crate::{ + Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBoundMethod, PyDict, PyGenericAlias, PyTuple, PyType, PyTypeRef}, + common::lock::PyRwLock, + function::{FuncArgs, KwArgs, OptionalOption}, + object::AsObject, + protocol::PyIter, + pyclass, + recursion::ReprGuard, + types::{Callable, Constructor, GetDescriptor, Representable}, + }; + use indexmap::IndexMap; + use rustpython_common::wtf8::Wtf8Buf; + + #[derive(FromArgs)] + struct ReduceArgs { + function: PyObjectRef, + iterator: PyIter, + #[pyarg(any, optional, name = "initial")] + initial: OptionalOption<PyObjectRef>, + } + + #[pyfunction] + fn reduce(args: ReduceArgs, vm: &VirtualMachine) -> PyResult { + let ReduceArgs { + function, + iterator, + initial, + } = args; + let mut iter = iterator.iter_without_hint(vm)?; + // OptionalOption distinguishes between: + // - Missing: no argument provided → use first element from iterator + // - Present(None): explicitly passed None → use None as initial value + // - Present(Some(v)): passed a value → use that value + let start_value = if let Some(val) = initial.into_option() { + // initial was provided (could be None or Some value) + val.unwrap_or_else(|| vm.ctx.none()) + } else { + // initial was not provided at all + iter.next().transpose()?.ok_or_else(|| { + let exc_type = vm.ctx.exceptions.type_error.to_owned(); + vm.new_exception_msg( + exc_type, + "reduce() of empty sequence with no initial value".into(), + ) + })? + }; + + let mut accumulator = start_value; + for next_obj in iter { + accumulator = function.call((accumulator, next_obj?), vm)? + } + Ok(accumulator) + } + + // Placeholder singleton for partial arguments + // The singleton is stored as _instance on the type class + #[pyattr] + #[allow(non_snake_case)] + fn Placeholder(vm: &VirtualMachine) -> PyObjectRef { + let placeholder = PyPlaceholderType.into_pyobject(vm); + // Store the singleton on the type class for slot_new to find + let typ = placeholder.class(); + typ.set_attr(vm.ctx.intern_str("_instance"), placeholder.clone()); + placeholder + } + + #[pyattr] + #[pyclass(name = "_PlaceholderType", module = "functools")] + #[derive(Debug, PyPayload)] + pub struct PyPlaceholderType; + + impl Constructor for PyPlaceholderType { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() || !args.kwargs.is_empty() { + return Err(vm.new_type_error("_PlaceholderType takes no arguments")); + } + // Return the singleton stored on the type class + if let Some(instance) = cls.get_attr(vm.ctx.intern_str("_instance")) { + return Ok(instance); + } + // Fallback: create a new instance (shouldn't happen for base type after module init) + Ok(PyPlaceholderType.into_pyobject(vm)) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + // This is never called because we override slot_new + Ok(PyPlaceholderType) + } + } + + #[pyclass(with(Constructor, Representable))] + impl PyPlaceholderType { + #[pymethod] + fn __reduce__(&self) -> &'static str { + "Placeholder" + } + + #[pymethod] + fn __init_subclass__(_cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("cannot subclass '_PlaceholderType'")) + } + } + + impl Representable for PyPlaceholderType { + #[inline] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("Placeholder".to_owned()) + } + } + + fn is_placeholder(obj: &PyObjectRef) -> bool { + &*obj.class().name() == "_PlaceholderType" + } + + fn count_placeholders(args: &[PyObjectRef]) -> usize { + args.iter().filter(|a| is_placeholder(a)).count() + } + + #[pyattr] + #[pyclass(name = "partial", module = "functools")] + #[derive(Debug, PyPayload)] + pub struct PyPartial { + inner: PyRwLock<PyPartialInner>, + } + + #[derive(Debug)] + struct PyPartialInner { + func: PyObjectRef, + args: PyRef<PyTuple>, + keywords: PyRef<PyDict>, + phcount: usize, + } + + #[pyclass( + with(Constructor, Callable, GetDescriptor, Representable), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) + )] + impl PyPartial { + #[pygetset] + fn func(&self) -> PyObjectRef { + self.inner.read().func.clone() + } + + #[pygetset] + fn args(&self) -> PyRef<PyTuple> { + self.inner.read().args.clone() + } + + #[pygetset] + fn keywords(&self) -> PyRef<PyDict> { + self.inner.read().keywords.clone() + } + + #[pymethod] + fn __reduce__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult { + let inner = zelf.inner.read(); + let partial_type = zelf.class(); + + // Get __dict__ if it exists and is not empty + let dict_obj = match zelf.as_object().dict() { + Some(dict) if !dict.is_empty() => dict.into(), + _ => vm.ctx.none(), + }; + + let state = vm.ctx.new_tuple(vec![ + inner.func.clone(), + inner.args.clone().into(), + inner.keywords.clone().into(), + dict_obj, + ]); + Ok(vm + .ctx + .new_tuple(vec![ + partial_type.to_owned().into(), + vm.ctx.new_tuple(vec![inner.func.clone()]).into(), + state.into(), + ]) + .into()) + } + + #[pymethod] + fn __setstate__(zelf: &Py<Self>, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let state_tuple = state + .downcast::<PyTuple>() + .map_err(|_| vm.new_type_error("argument to __setstate__ must be a tuple"))?; + + if state_tuple.len() != 4 { + return Err(vm.new_type_error(format!( + "expected 4 items in state, got {}", + state_tuple.len() + ))); + } + + let func = &state_tuple[0]; + let args = &state_tuple[1]; + let kwds = &state_tuple[2]; + let dict = &state_tuple[3]; + + if !func.is_callable() { + return Err(vm.new_type_error("invalid partial state")); + } + + // Validate that args is a tuple (or subclass) + if !args.fast_isinstance(vm.ctx.types.tuple_type) { + return Err(vm.new_type_error("invalid partial state")); + } + // Always convert to base tuple, even if it's a subclass + let args_tuple = match args.clone().downcast::<PyTuple>() { + Ok(tuple) if tuple.class().is(vm.ctx.types.tuple_type) => tuple, + _ => { + // It's a tuple subclass, convert to base tuple + let elements: Vec<PyObjectRef> = args.try_to_value(vm)?; + vm.ctx.new_tuple(elements) + } + }; + + let keywords_dict = if kwds.is(&vm.ctx.none) { + vm.ctx.new_dict() + } else { + // Always convert to base dict, even if it's a subclass + let dict = kwds + .clone() + .downcast::<PyDict>() + .map_err(|_| vm.new_type_error("invalid partial state"))?; + if dict.class().is(vm.ctx.types.dict_type) { + // It's already a base dict + dict + } else { + // It's a dict subclass, convert to base dict + let new_dict = vm.ctx.new_dict(); + for (key, value) in dict { + new_dict.set_item(&*key, value, vm)?; + } + new_dict + } + }; + + // Validate no trailing placeholders + let args_slice = args_tuple.as_slice(); + if !args_slice.is_empty() && is_placeholder(args_slice.last().unwrap()) { + return Err(vm.new_type_error("trailing Placeholders are not allowed")); + } + let phcount = count_placeholders(args_slice); + + // Actually update the state + let mut inner = zelf.inner.write(); + inner.func = func.clone(); + // Handle args - use the already validated tuple + inner.args = args_tuple; + + // Handle keywords - keep the original type + inner.keywords = keywords_dict; + inner.phcount = phcount; + + // Update __dict__ if provided + let Some(instance_dict) = zelf.as_object().dict() else { + return Ok(()); + }; + + if dict.is(&vm.ctx.none) { + // If dict is None, clear the instance dict + instance_dict.clear(); + return Ok(()); + } + + let dict_obj = dict + .clone() + .downcast::<PyDict>() + .map_err(|_| vm.new_type_error("invalid partial state"))?; + + // Clear existing dict and update with new values + instance_dict.clear(); + for (key, value) in dict_obj { + instance_dict.set_item(&*key, value, vm)?; + } + + Ok(()) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl Constructor for PyPartial { + type Args = FuncArgs; + + fn py_new( + _cls: &crate::Py<crate::builtins::PyType>, + args: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let (func, args_slice) = args + .args + .split_first() + .ok_or_else(|| vm.new_type_error("partial expected at least 1 argument, got 0"))?; + + if !func.is_callable() { + return Err(vm.new_type_error("the first argument must be callable")); + } + + // Check for placeholders in kwargs + for (key, value) in &args.kwargs { + if is_placeholder(value) { + return Err(vm.new_type_error(format!( + "Placeholder cannot be passed as a keyword argument to partial(). \ + Did you mean partial(..., {}=Placeholder, ...)(value)?", + key + ))); + } + } + + // Handle nested partial objects + let (final_func, final_args, final_keywords) = + if let Some(partial) = func.downcast_ref::<Self>() { + let inner = partial.inner.read(); + let stored_args = inner.args.as_slice(); + + // Merge placeholders: replace placeholders in stored_args with new args + let mut merged_args = Vec::with_capacity(stored_args.len() + args_slice.len()); + let mut new_args_iter = args_slice.iter(); + + for stored_arg in stored_args { + if is_placeholder(stored_arg) { + // Replace placeholder with next new arg, or keep placeholder + if let Some(new_arg) = new_args_iter.next() { + merged_args.push(new_arg.clone()); + } else { + merged_args.push(stored_arg.clone()); + } + } else { + merged_args.push(stored_arg.clone()); + } + } + // Append remaining new args + merged_args.extend(new_args_iter.cloned()); + + (inner.func.clone(), merged_args, inner.keywords.clone()) + } else { + (func.clone(), args_slice.to_vec(), vm.ctx.new_dict()) + }; + + // Trailing placeholders are not allowed + if !final_args.is_empty() && is_placeholder(final_args.last().unwrap()) { + return Err(vm.new_type_error("trailing Placeholders are not allowed")); + } + + let phcount = count_placeholders(&final_args); + + // Add new keywords + for (key, value) in args.kwargs { + final_keywords.set_item(vm.ctx.intern_str(key.as_str()), value, vm)?; + } + + Ok(Self { + inner: PyRwLock::new(PyPartialInner { + func: final_func, + args: vm.ctx.new_tuple(final_args), + keywords: final_keywords, + phcount, + }), + }) + } + } + + impl Callable for PyPartial { + type Args = FuncArgs; + + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Clone and release lock before calling Python code to prevent deadlock + let (func, stored_args, keywords, phcount) = { + let inner = zelf.inner.read(); + ( + inner.func.clone(), + inner.args.clone(), + inner.keywords.clone(), + inner.phcount, + ) + }; + + // Check if we have enough args to fill placeholders + if phcount > 0 && args.args.len() < phcount { + return Err(vm.new_type_error(format!( + "missing positional arguments in 'partial' call; expected at least {}, got {}", + phcount, + args.args.len() + ))); + } + + // Build combined args, replacing placeholders + let mut combined_args = Vec::with_capacity(stored_args.len() + args.args.len()); + let mut new_args_iter = args.args.iter(); + + for stored_arg in stored_args.as_slice() { + if is_placeholder(stored_arg) { + // Replace placeholder with next new arg + if let Some(new_arg) = new_args_iter.next() { + combined_args.push(new_arg.clone()); + } else { + // This shouldn't happen if phcount check passed + combined_args.push(stored_arg.clone()); + } + } else { + combined_args.push(stored_arg.clone()); + } + } + // Append remaining new args + combined_args.extend(new_args_iter.cloned()); + + // Merge keywords from self.keywords and args.kwargs + let mut final_kwargs = IndexMap::new(); + + // Add keywords from self.keywords + for (key, value) in &*keywords { + let key_str = key + .downcast_ref::<crate::builtins::PyStr>() + .ok_or_else(|| vm.new_type_error("keywords must be strings"))?; + final_kwargs.insert(key_str.expect_str().to_owned(), value); + } + + // Add keywords from args.kwargs (these override self.keywords) + for (key, value) in args.kwargs { + final_kwargs.insert(key, value); + } + + func.call(FuncArgs::new(combined_args, KwArgs::new(final_kwargs)), vm) + } + } + + impl GetDescriptor for PyPartial { + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + _cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let obj = match obj { + Some(obj) if !vm.is_none(&obj) => obj, + _ => return Ok(zelf), + }; + Ok(PyBoundMethod::new(obj, zelf).into_ref(&vm.ctx).into()) + } + } + + impl Representable for PyPartial { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + // Check for recursive repr + let obj = zelf.as_object(); + if let Some(_guard) = ReprGuard::enter(vm, obj) { + // Clone and release lock before calling Python code to prevent deadlock + let (func, args, keywords) = { + let inner = zelf.inner.read(); + ( + inner.func.clone(), + inner.args.clone(), + inner.keywords.clone(), + ) + }; + + let qualname = zelf.class().__qualname__(vm); + let qualname_wtf8 = qualname + .downcast_ref::<crate::builtins::PyStr>() + .map(|s| s.as_wtf8().to_owned()) + .unwrap_or_else(|| Wtf8Buf::from(zelf.class().name().to_owned())); + let module = zelf.class().__module__(vm); + + let mut result = Wtf8Buf::new(); + if let Ok(module_str) = module.downcast::<crate::builtins::PyStr>() { + let module_name = module_str.as_wtf8(); + if module_name != "builtins" && !module_name.is_empty() { + result.push_wtf8(module_name); + result.push_char('.'); + } + } + result.push_wtf8(&qualname_wtf8); + result.push_char('('); + result.push_wtf8(func.repr(vm)?.as_wtf8()); + + for arg in args.as_slice() { + result.push_str(", "); + result.push_wtf8(arg.repr(vm)?.as_wtf8()); + } + + for (key, value) in &*keywords { + result.push_str(", "); + let key_str = if let Ok(s) = key.clone().downcast::<crate::builtins::PyStr>() { + s + } else { + key.str(vm)? + }; + result.push_wtf8(key_str.as_wtf8()); + result.push_char('='); + result.push_wtf8(value.repr(vm)?.as_wtf8()); + } + + result.push_char(')'); + Ok(result) + } else { + Ok(Wtf8Buf::from("...")) + } + } + } +} diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs new file mode 100644 index 00000000000..c0acb304a64 --- /dev/null +++ b/crates/vm/src/stdlib/_imp.rs @@ -0,0 +1,364 @@ +use crate::builtins::{PyCode, PyStrInterned}; +use crate::frozen::FrozenModule; +use crate::{VirtualMachine, builtins::PyBaseExceptionRef}; +use core::borrow::Borrow; + +pub(crate) use _imp::module_def; + +pub use crate::vm::resolve_frozen_alias; + +#[cfg(feature = "threading")] +#[pymodule(sub)] +mod lock { + use crate::{PyResult, VirtualMachine, stdlib::_thread::RawRMutex}; + + static IMP_LOCK: RawRMutex = RawRMutex::INIT; + + #[pyfunction] + fn acquire_lock(_vm: &VirtualMachine) { + acquire_lock_for_fork() + } + + #[pyfunction] + fn release_lock(vm: &VirtualMachine) -> PyResult<()> { + if !IMP_LOCK.is_locked() { + Err(vm.new_runtime_error("Global import lock not held")) + } else { + unsafe { IMP_LOCK.unlock() }; + Ok(()) + } + } + + #[pyfunction] + fn lock_held(_vm: &VirtualMachine) -> bool { + IMP_LOCK.is_locked() + } + + pub(super) fn acquire_lock_for_fork() { + IMP_LOCK.lock(); + } + + pub(super) fn release_lock_after_fork_parent() { + if IMP_LOCK.is_locked() && IMP_LOCK.is_owned_by_current_thread() { + unsafe { IMP_LOCK.unlock() }; + } + } + + /// Reset import lock after fork() — only if held by a dead thread. + /// + /// `IMP_LOCK` is a reentrant mutex. If the *current* (surviving) thread + /// held it at fork time, the child must be able to release it normally. + /// Only reset if a now-dead thread was the owner. + /// + /// # Safety + /// + /// Must only be called from single-threaded child after fork(). + #[cfg(unix)] + pub(crate) unsafe fn reinit_after_fork() { + if IMP_LOCK.is_locked() && !IMP_LOCK.is_owned_by_current_thread() { + // Held by a dead thread — reset to unlocked. + unsafe { rustpython_common::lock::zero_reinit_after_fork(&IMP_LOCK) }; + } + } + + /// Match CPython's `_PyImport_ReInitLock()` + `_PyImport_ReleaseLock()` + /// behavior in the post-fork child: + /// 1) if ownership metadata is stale (dead owner / changed tid), reset; + /// 2) if current thread owns the lock, release it. + #[cfg(unix)] + pub(super) unsafe fn after_fork_child_reinit_and_release() { + unsafe { reinit_after_fork() }; + if IMP_LOCK.is_locked() && IMP_LOCK.is_owned_by_current_thread() { + unsafe { IMP_LOCK.unlock() }; + } + } +} + +/// Re-export for fork safety code in posix.rs +#[cfg(feature = "threading")] +pub(crate) fn acquire_imp_lock_for_fork() { + lock::acquire_lock_for_fork(); +} + +#[cfg(feature = "threading")] +pub(crate) fn release_imp_lock_after_fork_parent() { + lock::release_lock_after_fork_parent(); +} + +#[cfg(all(unix, feature = "threading"))] +pub(crate) unsafe fn reinit_imp_lock_after_fork() { + unsafe { lock::reinit_after_fork() } +} + +#[cfg(all(unix, feature = "threading"))] +pub(crate) unsafe fn after_fork_child_imp_lock_release() { + unsafe { lock::after_fork_child_reinit_and_release() } +} + +#[cfg(not(feature = "threading"))] +#[pymodule(sub)] +mod lock { + use crate::vm::VirtualMachine; + #[pyfunction] + pub(super) const fn acquire_lock(_vm: &VirtualMachine) {} + #[pyfunction] + pub(super) const fn release_lock(_vm: &VirtualMachine) {} + #[pyfunction] + pub(super) const fn lock_held(_vm: &VirtualMachine) -> bool { + false + } +} + +#[allow(dead_code)] +enum FrozenError { + BadName, // The given module name wasn't valid. + NotFound, // It wasn't in PyImport_FrozenModules. + Disabled, // -X frozen_modules=off (and not essential) + Excluded, // The PyImport_FrozenModules entry has NULL "code" + // (module is present but marked as unimportable, stops search). + Invalid, // The PyImport_FrozenModules entry is bogus + // (eg. does not contain executable code). +} + +impl FrozenError { + fn to_pyexception(&self, mod_name: &str, vm: &VirtualMachine) -> PyBaseExceptionRef { + use FrozenError::*; + let msg = match self { + BadName | NotFound => format!("No such frozen object named {mod_name}"), + Disabled => format!( + "Frozen modules are disabled and the frozen object named {mod_name} is not essential" + ), + Excluded => format!("Excluded frozen object named {mod_name}"), + Invalid => format!("Frozen object named {mod_name} is invalid"), + }; + vm.new_import_error(msg, vm.ctx.new_utf8_str(mod_name)) + } +} + +// look_up_frozen + use_frozen in import.c +fn find_frozen(name: &str, vm: &VirtualMachine) -> Result<FrozenModule, FrozenError> { + let frozen = vm + .state + .frozen + .get(name) + .copied() + .ok_or(FrozenError::NotFound)?; + + // Bootstrap modules are always available regardless of override flag + if matches!( + name, + "_frozen_importlib" | "_frozen_importlib_external" | "zipimport" + ) { + return Ok(frozen); + } + + // use_frozen(): override > 0 → true, override < 0 → false, 0 → default (true) + // When disabled, non-bootstrap modules are simply not found (same as look_up_frozen) + let override_val = vm.state.override_frozen_modules.load(); + if override_val < 0 { + return Err(FrozenError::NotFound); + } + + Ok(frozen) +} + +#[pymodule(with(lock))] +mod _imp { + use crate::{ + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef, PyUtf8StrRef}, + convert::TryFromBorrowedObject, + function::OptionalArg, + import, version, + }; + + #[pyattr] + fn check_hash_based_pycs(vm: &VirtualMachine) -> PyStrRef { + vm.ctx + .new_str(vm.state.config.settings.check_hash_pycs_mode.to_string()) + } + + #[pyattr(name = "pyc_magic_number_token")] + use version::PYC_MAGIC_NUMBER_TOKEN; + + #[pyfunction] + const fn extension_suffixes() -> PyResult<Vec<PyObjectRef>> { + Ok(Vec::new()) + } + + #[pyfunction] + fn is_builtin(name: PyUtf8StrRef, vm: &VirtualMachine) -> bool { + vm.state.module_defs.contains_key(name.as_str()) + } + + #[pyfunction] + fn is_frozen(name: PyUtf8StrRef, vm: &VirtualMachine) -> bool { + super::find_frozen(name.as_str(), vm).is_ok() + } + + #[pyfunction] + fn create_builtin(spec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let sys_modules = vm.sys_module.get_attr("modules", vm).unwrap(); + let name: PyUtf8StrRef = spec.get_attr("name", vm)?.try_into_value(vm)?; + + // Check sys.modules first + if let Ok(module) = sys_modules.get_item(&*name, vm) { + return Ok(module); + } + + let name_str = name.as_str(); + if let Some(&def) = vm.state.module_defs.get(name_str) { + // Phase 1: Create module (use create slot if provided, else default creation) + let module = if let Some(create) = def.slots.create { + // Custom module creation + create(vm, &spec, def)? + } else { + // Default module creation + PyModule::from_def(def).into_ref(&vm.ctx) + }; + + // Initialize module dict and methods + // Corresponds to PyModule_FromDefAndSpec: md_def, _add_methods_to_object, PyModule_SetDocString + PyModule::__init_dict_from_def(vm, &module); + module.__init_methods(vm)?; + + // Add to sys.modules BEFORE exec (critical for circular import handling) + sys_modules.set_item(name.as_pystr(), module.clone().into(), vm)?; + + // Phase 2: Call exec slot (can safely import other modules now) + if let Some(exec) = def.slots.exec { + exec(vm, &module)?; + } + + return Ok(module.into()); + } + + Ok(vm.ctx.none()) + } + + #[pyfunction] + fn exec_builtin(_mod: PyRef<PyModule>) -> i32 { + // For multi-phase init modules, exec is already called in create_builtin + 0 + } + + #[pyfunction] + fn get_frozen_object( + name: PyUtf8StrRef, + data: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PyCode>> { + if let OptionalArg::Present(data) = data + && !vm.is_none(&data) + { + let buf = crate::protocol::PyBuffer::try_from_borrowed_object(vm, &data)?; + let contiguous = buf.as_contiguous().ok_or_else(|| { + vm.new_buffer_error("get_frozen_object() requires a contiguous buffer") + })?; + let invalid_err = || { + vm.new_import_error( + format!("Frozen object named '{}' is invalid", name.as_str()), + name.clone().into_wtf8(), + ) + }; + let bag = crate::builtins::code::PyObjBag(&vm.ctx); + let code = + rustpython_compiler_core::marshal::deserialize_code(&mut &contiguous[..], bag) + .map_err(|_| invalid_err())?; + return Ok(vm.ctx.new_code(code)); + } + import::make_frozen(vm, name.as_str()) + } + + #[pyfunction] + fn init_frozen(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + import::import_frozen(vm, name.as_str()) + } + + #[pyfunction] + fn is_frozen_package(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<bool> { + let name_str = name.as_str(); + super::find_frozen(name_str, vm) + .map(|frozen| frozen.package) + .map_err(|e| e.to_pyexception(name_str, vm)) + } + + #[pyfunction] + fn _override_frozen_modules_for_tests(value: isize, vm: &VirtualMachine) { + vm.state.override_frozen_modules.store(value); + } + + #[pyfunction] + fn _fix_co_filename(code: PyRef<PyCode>, path: PyStrRef, vm: &VirtualMachine) { + let old_name = code.source_path(); + let new_name = vm.ctx.intern_str(path.as_wtf8()); + super::update_code_filenames(&code, old_name, new_name); + } + + #[pyfunction] + fn _frozen_module_names(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let names = vm + .state + .frozen + .keys() + .map(|&name| vm.ctx.new_utf8_str(name).into()) + .collect(); + Ok(names) + } + + #[allow(clippy::type_complexity)] + #[pyfunction] + fn find_frozen( + name: PyUtf8StrRef, + withdata: OptionalArg<bool>, + vm: &VirtualMachine, + ) -> PyResult<Option<(Option<PyRef<PyMemoryView>>, bool, Option<PyStrRef>)>> { + use super::FrozenError::*; + + if withdata.into_option().is_some() { + // this is keyword-only argument in CPython + unimplemented!(); + } + + let name_str = name.as_str(); + let info = match super::find_frozen(name_str, vm) { + Ok(info) => info, + Err(NotFound | Disabled | BadName) => return Ok(None), + Err(e) => return Err(e.to_pyexception(name_str, vm)), + }; + + // When origname is empty (e.g. __hello_only__), return None. + // Otherwise return the resolved alias name. + let origname_str = super::resolve_frozen_alias(name_str); + let origname = if origname_str.is_empty() { + None + } else { + Some(vm.ctx.new_utf8_str(origname_str).into()) + }; + Ok(Some((None, info.package, origname))) + } + + #[pyfunction] + fn source_hash(key: u64, source: PyBytesRef) -> Vec<u8> { + let hash: u64 = crate::common::hash::keyed_hash(key, source.as_bytes()); + hash.to_le_bytes().to_vec() + } +} + +fn update_code_filenames( + code: &PyCode, + old_name: &'static PyStrInterned, + new_name: &'static PyStrInterned, +) { + let current = code.source_path(); + if !core::ptr::eq(current, old_name) && current.as_str() != old_name.as_str() { + return; + } + code.set_source_path(new_name); + for constant in code.code.constants.iter() { + let obj: &crate::PyObject = constant.borrow(); + if let Some(inner_code) = obj.downcast_ref::<PyCode>() { + update_code_filenames(inner_code, old_name, new_name); + } + } +} diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs new file mode 100644 index 00000000000..c238dda3725 --- /dev/null +++ b/crates/vm/src/stdlib/_io.rs @@ -0,0 +1,7035 @@ +/* + * I/O core tools. + */ +pub(crate) use _io::module_def; +#[cfg(all(unix, feature = "threading"))] +pub(crate) use _io::reinit_std_streams_after_fork; + +cfg_if::cfg_if! { + if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { + use crate::common::crt_fd::Offset; + } else { + type Offset = i64; + } +} + +// EAGAIN constant for BlockingIOError +cfg_if::cfg_if! { + if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { + const EAGAIN: i32 = libc::EAGAIN; + } else { + const EAGAIN: i32 = 11; // Standard POSIX value + } +} + +use crate::{ + AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyModule, +}; +pub use _io::{OpenArgs, io_open as open}; + +fn file_closed(file: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + file.get_attr("closed", vm)?.try_to_bool(vm) +} + +const DEFAULT_BUFFER_SIZE: usize = 128 * 1024; + +/// iobase_finalize in Modules/_io/iobase.c +fn iobase_finalize(zelf: &PyObject, vm: &VirtualMachine) { + // If `closed` doesn't exist or can't be evaluated as bool, then the + // object is probably in an unusable state, so ignore. + let closed = match vm.get_attribute_opt(zelf.to_owned(), "closed") { + Ok(Some(val)) => match val.try_to_bool(vm) { + Ok(b) => b, + Err(_) => return, + }, + _ => return, + }; + if !closed { + // Signal close() that it was called as part of the object + // finalization process. + let _ = zelf.set_attr("_finalizing", vm.ctx.true_value.clone(), vm); + if let Err(e) = vm.call_method(zelf, "close", ()) { + // BrokenPipeError during GC finalization is expected when pipe + // buffer objects are collected after the subprocess dies. The + // underlying fd is still properly closed by raw.close(). + // Popen.__del__ catches BrokenPipeError, but our tracing GC may + // finalize pipe buffers before Popen.__del__ runs. + if !e.fast_isinstance(vm.ctx.exceptions.broken_pipe_error) { + vm.run_unraisable(e, None, zelf.to_owned()); + } + } + } +} + +// not used on all platforms +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Fildes(pub i32); + +impl TryFromObject for Fildes { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + use crate::builtins::int; + let int = match obj.downcast::<int::PyInt>() { + Ok(i) => i, + Err(obj) => { + let fileno_meth = vm.get_attribute_opt(obj, "fileno")?.ok_or_else(|| { + vm.new_type_error("argument must be an int, or have a fileno() method.") + })?; + fileno_meth + .call((), vm)? + .downcast() + .map_err(|_| vm.new_type_error("fileno() returned a non-integer"))? + } + }; + let fd = int.try_to_primitive(vm)?; + if fd < 0 { + return Err(vm.new_value_error(format!( + "file descriptor cannot be a negative integer ({fd})" + ))); + } + Ok(Self(fd)) + } +} + +#[cfg(unix)] +impl std::os::fd::AsFd for Fildes { + fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> { + // SAFETY: none, really. but, python's os api of passing around file descriptors + // everywhere isn't really io-safe anyway, so, this is passed to the user. + unsafe { std::os::fd::BorrowedFd::borrow_raw(self.0) } + } +} +#[cfg(unix)] +impl std::os::fd::AsRawFd for Fildes { + fn as_raw_fd(&self) -> std::os::fd::RawFd { + self.0 + } +} + +#[pymodule] +mod _io { + use super::*; + use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + TryFromBorrowedObject, TryFromObject, + builtins::{ + PyBaseExceptionRef, PyBool, PyByteArray, PyBytes, PyBytesRef, PyDict, PyMemoryView, + PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, PyUtf8Str, PyUtf8StrRef, + }, + class::StaticType, + common::lock::{ + PyMappedThreadMutexGuard, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, + PyThreadMutex, PyThreadMutexGuard, + }, + common::wtf8::{Wtf8, Wtf8Buf}, + convert::ToPyObject, + exceptions::cstring_error, + function::{ + ArgBytesLike, ArgIterable, ArgMemoryBuffer, ArgSize, Either, FuncArgs, IntoFuncArgs, + OptionalArg, OptionalOption, PySetterValue, + }, + protocol::{ + BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, VecBuffer, + }, + recursion::ReprGuard, + types::{ + Callable, Constructor, DefaultConstructor, Destructor, Initializer, IterNext, Iterable, + Representable, + }, + vm::VirtualMachine, + }; + use alloc::borrow::Cow; + use bstr::ByteSlice; + use core::{ + ops::Range, + sync::atomic::{AtomicBool, Ordering}, + }; + use crossbeam_utils::atomic::AtomicCell; + use malachite_bigint::BigInt; + use num_traits::ToPrimitive; + use std::io::{self, Cursor, SeekFrom, prelude::*}; + + #[allow(clippy::let_and_return)] + fn validate_whence(whence: i32) -> bool { + let x = (0..=2).contains(&whence); + cfg_if::cfg_if! { + if #[cfg(any(target_os = "dragonfly", target_os = "freebsd", target_os = "linux"))] { + x || matches!(whence, libc::SEEK_DATA | libc::SEEK_HOLE) + } else { + x + } + } + } + + fn ensure_unclosed(file: &PyObject, msg: &str, vm: &VirtualMachine) -> PyResult<()> { + if file.get_attr("closed", vm)?.try_to_bool(vm)? { + Err(vm.new_value_error(msg)) + } else { + Ok(()) + } + } + + /// Check if an error is an OSError with errno == EINTR. + /// If so, call check_signals() and return Ok(None) to indicate retry. + /// Otherwise, return Ok(Some(val)) for success or Err for other errors. + /// This mirrors CPythons _PyIO_trap_eintr() pattern. + #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] + fn trap_eintr<T>(result: PyResult<T>, vm: &VirtualMachine) -> PyResult<Option<T>> { + match result { + Ok(val) => Ok(Some(val)), + Err(exc) => { + // Check if its an OSError with errno == EINTR + if exc.fast_isinstance(vm.ctx.exceptions.os_error) + && let Ok(errno_attr) = exc.as_object().get_attr("errno", vm) + && let Ok(errno_val) = i32::try_from_object(vm, errno_attr) + && errno_val == libc::EINTR + { + vm.check_signals()?; + return Ok(None); + } + Err(exc) + } + } + } + + /// WASM version: no EINTR handling needed + #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] + fn trap_eintr<T>(result: PyResult<T>, _vm: &VirtualMachine) -> PyResult<Option<T>> { + result.map(Some) + } + + pub fn new_unsupported_operation(vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { + vm.new_os_subtype_error(unsupported_operation().to_owned(), None, msg) + .upcast() + } + + fn _unsupported<T>(vm: &VirtualMachine, zelf: &PyObject, operation: &str) -> PyResult<T> { + Err(new_unsupported_operation( + vm, + format!("{}.{}() not supported", zelf.class().name(), operation), + )) + } + + #[derive(FromArgs)] + pub(super) struct OptionalSize { + // In a few functions, the default value is -1 rather than None. + // Make sure the default value doesn't affect compatibility. + #[pyarg(positional, default)] + size: Option<ArgSize>, + } + + impl OptionalSize { + #[allow(clippy::wrong_self_convention)] + pub fn to_usize(self) -> Option<usize> { + self.size?.to_usize() + } + + pub fn try_usize(self, vm: &VirtualMachine) -> PyResult<Option<usize>> { + self.size + .map(|v| { + let v = *v; + if v >= 0 { + Ok(v as usize) + } else { + Err(vm.new_value_error(format!("Negative size value {v}"))) + } + }) + .transpose() + } + } + + fn os_err(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { + use crate::convert::ToPyException; + err.to_pyexception(vm) + } + + pub(super) fn io_closed_error(vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_value_error("I/O operation on closed file") + } + + #[pyattr] + const DEFAULT_BUFFER_SIZE: usize = super::DEFAULT_BUFFER_SIZE; + + pub(super) fn seekfrom( + vm: &VirtualMachine, + offset: PyObjectRef, + how: OptionalArg<i32>, + ) -> PyResult<SeekFrom> { + let seek = match how { + OptionalArg::Present(0) | OptionalArg::Missing => { + SeekFrom::Start(offset.try_into_value(vm)?) + } + OptionalArg::Present(1) => SeekFrom::Current(offset.try_into_value(vm)?), + OptionalArg::Present(2) => SeekFrom::End(offset.try_into_value(vm)?), + _ => return Err(vm.new_value_error("invalid value for how")), + }; + Ok(seek) + } + + #[derive(Debug)] + struct BufferedIO { + cursor: Cursor<Vec<u8>>, + } + + impl BufferedIO { + const fn new(cursor: Cursor<Vec<u8>>) -> Self { + Self { cursor } + } + + fn write(&mut self, data: &[u8]) -> Option<u64> { + if data.is_empty() { + return Some(0); + } + let length = data.len(); + self.cursor.write_all(data).ok()?; + Some(length as u64) + } + + //return the entire contents of the underlying + fn getvalue(&self) -> Vec<u8> { + self.cursor.clone().into_inner() + } + + //skip to the jth position + fn seek(&mut self, seek: SeekFrom) -> io::Result<u64> { + self.cursor.seek(seek) + } + + //Read k bytes from the object and return. + fn read(&mut self, bytes: Option<usize>) -> Option<Vec<u8>> { + let pos = self.cursor.position().to_usize()?; + let avail_slice = self.cursor.get_ref().get(pos..)?; + // if we don't specify the number of bytes, or it's too big, give the whole rest of the slice + let n = bytes.map_or_else( + || avail_slice.len(), + |n| core::cmp::min(n, avail_slice.len()), + ); + let b = avail_slice[..n].to_vec(); + self.cursor.set_position((pos + n) as u64); + Some(b) + } + + const fn tell(&self) -> u64 { + self.cursor.position() + } + + fn readline(&mut self, size: Option<usize>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + self.read_until(size, b'\n', vm) + } + + fn read_until( + &mut self, + size: Option<usize>, + byte: u8, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + let size = match size { + None => { + let mut buf: Vec<u8> = Vec::new(); + self.cursor + .read_until(byte, &mut buf) + .map_err(|err| os_err(vm, err))?; + return Ok(buf); + } + Some(0) => { + return Ok(Vec::new()); + } + Some(size) => size, + }; + + let available = { + // For Cursor, fill_buf returns all of the remaining data unlike other BufReads which have outer reading source. + // Unless we add other data by write, there will be no more data. + let buf = self.cursor.fill_buf().map_err(|err| os_err(vm, err))?; + if size < buf.len() { &buf[..size] } else { buf } + }; + let buf = match available.find_byte(byte) { + Some(i) => available[..=i].to_vec(), + _ => available.to_vec(), + }; + self.cursor.consume(buf.len()); + Ok(buf) + } + + fn truncate(&mut self, pos: Option<usize>) -> usize { + let pos = pos.unwrap_or_else(|| self.tell() as usize); + self.cursor.get_mut().truncate(pos); + pos + } + } + + fn check_closed(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if file_closed(file, vm)? { + Err(io_closed_error(vm)) + } else { + Ok(()) + } + } + + fn check_readable(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if vm.call_method(file, "readable", ())?.try_to_bool(vm)? { + Ok(()) + } else { + Err(new_unsupported_operation( + vm, + "File or stream is not readable".to_owned(), + )) + } + } + + fn check_writable(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if vm.call_method(file, "writable", ())?.try_to_bool(vm)? { + Ok(()) + } else { + Err(new_unsupported_operation( + vm, + "File or stream is not writable.".to_owned(), + )) + } + } + + fn check_seekable(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if vm.call_method(file, "seekable", ())?.try_to_bool(vm)? { + Ok(()) + } else { + Err(new_unsupported_operation( + vm, + "File or stream is not seekable".to_owned(), + )) + } + } + + fn check_decoded(decoded: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { + decoded.downcast().map_err(|obj| { + vm.new_type_error(format!( + "decoder should return a string result, not '{}'", + obj.class().name() + )) + }) + } + + #[pyattr] + #[pyclass(name = "_IOBase")] + #[derive(Debug, Default, PyPayload)] + pub struct _IOBase; + + #[pyclass( + with(IterNext, Iterable, Destructor), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) + )] + impl _IOBase { + #[pymethod] + fn seek( + zelf: PyObjectRef, + _pos: PyObjectRef, + _whence: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + _unsupported(vm, &zelf, "seek") + } + + #[pymethod] + fn tell(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.call_method(&zelf, "seek", (0, 1)) + } + + #[pymethod] + fn truncate(zelf: PyObjectRef, _pos: OptionalArg, vm: &VirtualMachine) -> PyResult { + _unsupported(vm, &zelf, "truncate") + } + + #[pymethod] + fn fileno(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + _unsupported(vm, &zelf, "fileno") + } + + #[pyattr] + fn __closed(ctx: &Context) -> PyRef<PyBool> { + ctx.new_bool(false) + } + + #[pymethod] + fn __enter__(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + check_closed(&instance, vm)?; + Ok(instance) + } + + #[pymethod] + fn __exit__(instance: PyObjectRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + vm.call_method(&instance, "close", ())?; + Ok(()) + } + + #[pymethod] + fn flush(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // just check if this is closed; if it isn't, do nothing + check_closed(&instance, vm) + } + + #[pymethod] + fn seekable(_self: PyObjectRef) -> bool { + false + } + + #[pymethod] + fn readable(_self: PyObjectRef) -> bool { + false + } + + #[pymethod] + fn writable(_self: PyObjectRef) -> bool { + false + } + + #[pymethod] + fn isatty(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + check_closed(&instance, vm)?; + Ok(false) + } + + #[pygetset] + fn closed(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + instance.get_attr("__closed", vm) + } + + #[pymethod] + fn close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + iobase_close(&instance, vm) + } + + #[pymethod] + fn readline( + instance: PyObjectRef, + size: OptionalSize, + vm: &VirtualMachine, + ) -> PyResult<Vec<u8>> { + let size = size.to_usize(); + let read = instance.get_attr("read", vm)?; + let mut res = Vec::new(); + while size.is_none_or(|s| res.len() < s) { + let read_res = ArgBytesLike::try_from_object(vm, read.call((1,), vm)?)?; + if read_res.with_ref(|b| b.is_empty()) { + break; + } + read_res.with_ref(|b| res.extend_from_slice(b)); + if res.ends_with(b"\n") { + break; + } + } + Ok(res) + } + + #[pymethod] + fn readlines( + instance: PyObjectRef, + hint: OptionalOption<isize>, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + let hint = hint.flatten().unwrap_or(-1); + if hint <= 0 { + return instance.try_to_value(vm); + } + let hint = hint as usize; + let mut ret = Vec::new(); + let it = ArgIterable::<PyObjectRef>::try_from_object(vm, instance)?; + let mut full_len = 0; + for line in it.iter(vm)? { + let line = line?; + let line_len = line.length(vm)?; + ret.push(line.clone()); + full_len += line_len; + if full_len > hint { + break; + } + } + Ok(ret) + } + + #[pymethod] + fn writelines( + instance: PyObjectRef, + lines: ArgIterable, + vm: &VirtualMachine, + ) -> PyResult<()> { + check_closed(&instance, vm)?; + for line in lines.iter(vm)? { + vm.call_method(&instance, "write", (line?,))?; + } + Ok(()) + } + + #[pymethod(name = "_checkClosed")] + fn check_closed(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + check_closed(&instance, vm) + } + + #[pymethod(name = "_checkReadable")] + fn check_readable(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + check_readable(&instance, vm) + } + + #[pymethod(name = "_checkWritable")] + fn check_writable(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + check_writable(&instance, vm) + } + + #[pymethod(name = "_checkSeekable")] + fn check_seekable(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + check_seekable(&instance, vm) + } + } + + impl Destructor for _IOBase { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + // C-level IO types (FileIO, Buffered*, TextIOWrapper) have their own + // slot_del that calls iobase_finalize with proper _finalizing flag + // and _dealloc_warn chain. This base fallback is only reached by + // Python-level subclasses, where we silently discard close() errors + // to avoid surfacing unraisable from partially initialized objects. + let _ = vm.call_method(zelf, "close", ()); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } + + impl Iterable for _IOBase { + fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + check_closed(&zelf, vm)?; + Ok(zelf) + } + + fn iter(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + unreachable!("slot_iter is implemented") + } + } + + impl IterNext for _IOBase { + fn slot_iternext(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let line = vm.call_method(zelf, "readline", ())?; + Ok(if !line.clone().try_to_bool(vm)? { + PyIterReturn::StopIteration(None) + } else { + PyIterReturn::Return(line) + }) + } + + fn next(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { + unreachable!("slot_iternext is implemented") + } + } + + pub(super) fn iobase_close(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if !file_closed(file, vm)? { + let res = vm.call_method(file, "flush", ()); + file.set_attr("__closed", vm.new_pyobj(true), vm)?; + res?; + } + Ok(()) + } + + #[pyattr] + #[pyclass(name = "_RawIOBase", base = _IOBase)] + #[derive(Debug, Default)] + #[repr(transparent)] + pub(super) struct _RawIOBase(_IOBase); + + #[pyclass(flags(BASETYPE, HAS_DICT, HAS_WEAKREF))] + impl _RawIOBase { + #[pymethod] + fn read(instance: PyObjectRef, size: OptionalSize, vm: &VirtualMachine) -> PyResult { + if let Some(size) = size.to_usize() { + // FIXME: unnecessary zero-init + let b = PyByteArray::from(vec![0; size]).into_ref(&vm.ctx); + let n = <Option<isize>>::try_from_object( + vm, + vm.call_method(&instance, "readinto", (b.clone(),))?, + )?; + Ok(match n { + None => vm.ctx.none(), + Some(n) => { + // Validate the return value is within bounds + if n < 0 || (n as usize) > size { + return Err(vm.new_value_error(format!( + "readinto returned {n} outside buffer size {size}" + ))); + } + let n = n as usize; + let mut bytes = b.borrow_buf_mut(); + bytes.truncate(n); + // FIXME: try to use Arc::unwrap on the bytearray to get at the inner buffer + bytes.clone().to_pyobject(vm) + } + }) + } else { + vm.call_method(&instance, "readall", ()) + } + } + + #[pymethod] + fn readall(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<Option<Vec<u8>>> { + let mut chunks = Vec::new(); + let mut total_len = 0; + loop { + // Loop with EINTR handling (PEP 475) + let data = loop { + let res = vm.call_method(&instance, "read", (DEFAULT_BUFFER_SIZE,)); + match trap_eintr(res, vm)? { + Some(val) => break val, + None => continue, + } + }; + let data = <Option<PyBytesRef>>::try_from_object(vm, data)?; + match data { + None => { + if chunks.is_empty() { + return Ok(None); + } + break; + } + Some(b) => { + if b.as_bytes().is_empty() { + break; + } + total_len += b.as_bytes().len(); + chunks.push(b) + } + } + } + let mut ret = Vec::with_capacity(total_len); + for b in chunks { + ret.extend_from_slice(b.as_bytes()) + } + Ok(Some(ret)) + } + + #[pymethod] + fn readinto(_instance: PyObjectRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error(String::new())) + } + + #[pymethod] + fn write(_instance: PyObjectRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error(String::new())) + } + } + + #[pyattr] + #[pyclass(name = "_BufferedIOBase", base = _IOBase)] + #[derive(Debug, Default)] + #[repr(transparent)] + struct _BufferedIOBase(_IOBase); + + #[pyclass(flags(BASETYPE, HAS_WEAKREF))] + impl _BufferedIOBase { + #[pymethod] + fn read(zelf: PyObjectRef, _size: OptionalArg, vm: &VirtualMachine) -> PyResult { + _unsupported(vm, &zelf, "read") + } + + #[pymethod] + fn read1(zelf: PyObjectRef, _size: OptionalArg, vm: &VirtualMachine) -> PyResult { + _unsupported(vm, &zelf, "read1") + } + + fn _readinto( + zelf: PyObjectRef, + buf_obj: PyObjectRef, + method: &str, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let b = ArgMemoryBuffer::try_from_borrowed_object(vm, &buf_obj)?; + let l = b.len(); + let data = vm.call_method(&zelf, method, (l,))?; + if data.is(&buf_obj) { + return Ok(l); + } + let mut buf = b.borrow_buf_mut(); + let data = ArgBytesLike::try_from_object(vm, data)?; + let data = data.borrow_buf(); + match buf.get_mut(..data.len()) { + Some(slice) => { + slice.copy_from_slice(&data); + Ok(data.len()) + } + None => { + Err(vm.new_value_error("readinto: buffer and read data have different lengths")) + } + } + } + #[pymethod] + fn readinto(zelf: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + Self::_readinto(zelf, b, "read", vm) + } + + #[pymethod] + fn readinto1(zelf: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + Self::_readinto(zelf, b, "read1", vm) + } + + #[pymethod] + fn write(zelf: PyObjectRef, _b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + _unsupported(vm, &zelf, "write") + } + + #[pymethod] + fn detach(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + _unsupported(vm, &zelf, "detach") + } + } + + // TextIO Base has no public constructor + #[pyattr] + #[pyclass(name = "_TextIOBase", base = _IOBase)] + #[derive(Debug, Default)] + #[repr(transparent)] + struct _TextIOBase(_IOBase); + + #[pyclass(flags(BASETYPE, HAS_WEAKREF))] + impl _TextIOBase { + #[pygetset] + fn encoding(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } + + #[pygetset] + fn errors(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } + } + + #[derive(FromArgs, Clone)] + struct BufferSize { + #[pyarg(any, optional)] + buffer_size: OptionalArg<isize>, + } + + bitflags::bitflags! { + #[derive(Copy, Clone, Debug, PartialEq, Default)] + struct BufferedFlags: u8 { + const DETACHED = 1 << 0; + const WRITABLE = 1 << 1; + const READABLE = 1 << 2; + } + } + + #[derive(Debug, Default)] + struct BufferedData { + raw: Option<PyObjectRef>, + flags: BufferedFlags, + abs_pos: Offset, + buffer: Vec<u8>, + pos: Offset, + raw_pos: Offset, + read_end: Offset, + write_pos: Offset, + write_end: Offset, + } + + impl BufferedData { + fn check_init(&self, vm: &VirtualMachine) -> PyResult<&PyObject> { + if let Some(raw) = &self.raw { + Ok(raw) + } else { + let msg = if self.flags.contains(BufferedFlags::DETACHED) { + "raw stream has been detached" + } else { + "I/O operation on uninitialized object" + }; + Err(vm.new_value_error(msg)) + } + } + + #[inline] + const fn writable(&self) -> bool { + self.flags.contains(BufferedFlags::WRITABLE) + } + + #[inline] + const fn readable(&self) -> bool { + self.flags.contains(BufferedFlags::READABLE) + } + + #[inline] + const fn valid_read(&self) -> bool { + self.readable() && self.read_end != -1 + } + + #[inline] + const fn valid_write(&self) -> bool { + self.writable() && self.write_end != -1 + } + + #[inline] + const fn raw_offset(&self) -> Offset { + if (self.valid_read() || self.valid_write()) && self.raw_pos >= 0 { + self.raw_pos - self.pos + } else { + 0 + } + } + + #[inline] + const fn readahead(&self) -> Offset { + if self.valid_read() { + self.read_end - self.pos + } else { + 0 + } + } + + const fn reset_read(&mut self) { + self.read_end = -1; + } + + const fn reset_write(&mut self) { + self.write_pos = 0; + self.write_end = -1; + } + + fn flush(&mut self, vm: &VirtualMachine) -> PyResult<()> { + if !self.valid_write() || self.write_pos == self.write_end { + self.reset_write(); + return Ok(()); + } + + let rewind = self.raw_offset() + (self.pos - self.write_pos); + if rewind != 0 { + self.raw_seek(-rewind, 1, vm)?; + self.raw_pos -= rewind; + } + + while self.write_pos < self.write_end { + let n = + self.raw_write(None, self.write_pos as usize..self.write_end as usize, vm)?; + let n = match n { + Some(n) => n, + None => { + // BlockingIOError(errno, msg, characters_written=0) + return Err(vm.invoke_exception( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(0), + ], + )?); + } + }; + self.write_pos += n as Offset; + self.raw_pos = self.write_pos; + vm.check_signals()?; + } + + self.reset_write(); + + Ok(()) + } + + fn flush_rewind(&mut self, vm: &VirtualMachine) -> PyResult<()> { + self.flush(vm)?; + if self.readable() { + let res = self.raw_seek(-self.raw_offset(), 1, vm); + self.reset_read(); + res?; + } + Ok(()) + } + + fn raw_seek(&mut self, pos: Offset, whence: i32, vm: &VirtualMachine) -> PyResult<Offset> { + let ret = vm.call_method(self.check_init(vm)?, "seek", (pos, whence))?; + let offset = get_offset(ret, vm)?; + if offset < 0 { + return Err( + vm.new_os_error(format!("Raw stream returned invalid position {offset}")) + ); + } + self.abs_pos = offset; + Ok(offset) + } + + fn seek(&mut self, target: Offset, whence: i32, vm: &VirtualMachine) -> PyResult<Offset> { + if matches!(whence, 0 | 1) && self.readable() { + let current = self.raw_tell_cache(vm)?; + let available = self.readahead(); + if available > 0 { + let offset = if whence == 0 { + target - (current - self.raw_offset()) + } else { + target + }; + if offset >= -self.pos && offset <= available { + self.pos += offset; + // GH-95782: character devices may report raw position 0 + // even after reading, which would make this negative + let result = current - available + offset; + return Ok(if result < 0 { 0 } else { result }); + } + } + } + // raw.get_attr("seek", vm)?.call(args, vm) + if self.writable() { + self.flush(vm)?; + } + let target = if whence == 1 { + target - self.raw_offset() + } else { + target + }; + let res = self.raw_seek(target, whence, vm); + self.raw_pos = -1; + if res.is_ok() && self.readable() { + self.reset_read(); + } + res + } + + fn raw_tell(&mut self, vm: &VirtualMachine) -> PyResult<Offset> { + let raw = self.check_init(vm)?; + let ret = vm.call_method(raw, "tell", ())?; + let offset = get_offset(ret, vm)?; + if offset < 0 { + return Err( + vm.new_os_error(format!("Raw stream returned invalid position {offset}")) + ); + } + self.abs_pos = offset; + Ok(offset) + } + + fn raw_tell_cache(&mut self, vm: &VirtualMachine) -> PyResult<Offset> { + if self.abs_pos == -1 { + self.raw_tell(vm) + } else { + Ok(self.abs_pos) + } + } + + /// None means non-blocking failed + fn raw_write( + &mut self, + buf: Option<PyBuffer>, + buf_range: Range<usize>, + vm: &VirtualMachine, + ) -> PyResult<Option<usize>> { + let len = buf_range.len(); + + // Prepare the memoryview; if using the internal buffer, stash it + // in write_buf so we can restore it after the write. + let (mem_obj, write_buf) = if let Some(buf) = buf { + let mem_obj = + PyMemoryView::from_buffer_range(buf, buf_range, vm)?.into_ref(&vm.ctx); + (mem_obj, None) + } else { + let v = core::mem::take(&mut self.buffer); + let wb = VecBuffer::from(v).into_ref(&vm.ctx); + let mem_obj = + PyMemoryView::from_buffer_range(wb.clone().into_pybuffer(true), buf_range, vm)? + .into_ref(&vm.ctx); + (mem_obj, Some(wb)) + }; + + // Loop if write() raises EINTR (PEP 475) + let res = loop { + let res = vm.call_method(self.raw.as_ref().unwrap(), "write", (mem_obj.clone(),)); + match trap_eintr(res, vm) { + Ok(Some(val)) => break Ok(val), + Ok(None) => continue, + Err(e) => break Err(e), + } + }; + + // Restore internal buffer if we borrowed it + if let Some(wb) = write_buf { + mem_obj.release(); + self.buffer = wb.take(); + } + + let res = res?; + + if vm.is_none(&res) { + return Ok(None); + } + let n = isize::try_from_object(vm, res)?; + if n < 0 || n as usize > len { + return Err(vm.new_os_error(format!( + "raw write() returned invalid length {n} (should have been between 0 and {len})" + ))); + } + if self.abs_pos != -1 { + self.abs_pos += n as Offset + } + Ok(Some(n as usize)) + } + + fn write(&mut self, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { + if !self.valid_read() && !self.valid_write() { + self.pos = 0; + self.raw_pos = 0; + } + let avail = self.buffer.len() - self.pos as usize; + let buf_len; + { + let buf = obj.borrow_buf(); + buf_len = buf.len(); + if buf.len() <= avail { + self.buffer[self.pos as usize..][..buf.len()].copy_from_slice(&buf); + if !self.valid_write() || self.write_pos > self.pos { + self.write_pos = self.pos + } + self.adjust_position(self.pos + buf.len() as Offset); + if self.pos > self.write_end { + self.write_end = self.pos + } + return Ok(buf.len()); + } + } + + // if BlockingIOError, shift buffer + // and try to buffer the new data; otherwise propagate the error + match self.flush(vm) { + Ok(()) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.blocking_io_error) => { + if self.readable() { + self.reset_read(); + } + // Shift buffer and adjust positions + let shift = self.write_pos; + if shift > 0 { + self.buffer + .copy_within(shift as usize..self.write_end as usize, 0); + self.write_end -= shift; + self.raw_pos -= shift; + self.pos -= shift; + self.write_pos = 0; + } + let avail = self.buffer.len() - self.write_end as usize; + if buf_len <= avail { + // Everything can be buffered + let buf = obj.borrow_buf(); + self.buffer[self.write_end as usize..][..buf_len].copy_from_slice(&buf); + self.write_end += buf_len as Offset; + self.pos += buf_len as Offset; + return Ok(buf_len); + } + // Buffer as much as possible and return BlockingIOError + let buf = obj.borrow_buf(); + self.buffer[self.write_end as usize..][..avail].copy_from_slice(&buf[..avail]); + self.write_end += avail as Offset; + self.pos += avail as Offset; + return Err(vm.invoke_exception( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(avail), + ], + )?); + } + Err(e) => return Err(e), + } + + // Only reach here if flush succeeded + let offset = self.raw_offset(); + if offset != 0 { + self.raw_seek(-offset, 1, vm)?; + self.raw_pos -= offset; + } + + let mut remaining = buf_len; + let mut written = 0; + let buffer: PyBuffer = obj.into(); + while remaining > self.buffer.len() { + let res = self.raw_write(Some(buffer.clone()), written..buf_len, vm)?; + match res { + Some(n) => { + written += n; + if let Some(r) = remaining.checked_sub(n) { + remaining = r + } else { + break; + } + vm.check_signals()?; + } + None => { + // raw file is non-blocking + if remaining > self.buffer.len() { + // can't buffer everything, buffer what we can and error + let buf = buffer.as_contiguous().unwrap(); + let buffer_len = self.buffer.len(); + self.buffer.copy_from_slice(&buf[written..][..buffer_len]); + self.raw_pos = 0; + let buffer_size = self.buffer.len() as _; + self.adjust_position(buffer_size); + self.write_end = buffer_size; + // BlockingIOError(errno, msg, characters_written) + let chars_written = written + buffer_len; + return Err(vm.invoke_exception( + vm.ctx.exceptions.blocking_io_error.to_owned(), + vec![ + vm.new_pyobj(EAGAIN), + vm.new_pyobj("write could not complete without blocking"), + vm.new_pyobj(chars_written), + ], + )?); + } else { + break; + } + } + } + } + if self.readable() { + self.reset_read(); + } + if remaining > 0 { + let buf = buffer.as_contiguous().unwrap(); + self.buffer[..remaining].copy_from_slice(&buf[written..][..remaining]); + written += remaining; + } + self.write_pos = 0; + self.write_end = remaining as _; + self.adjust_position(remaining as _); + self.raw_pos = 0; + + Ok(written) + } + + fn active_read_slice(&self) -> &[u8] { + &self.buffer[self.pos as usize..][..self.readahead() as usize] + } + + fn read_fast(&mut self, n: usize) -> Option<Vec<u8>> { + let ret = self.active_read_slice().get(..n)?.to_vec(); + self.pos += n as Offset; + Some(ret) + } + + fn read_generic(&mut self, n: usize, vm: &VirtualMachine) -> PyResult<Option<Vec<u8>>> { + if let Some(fast) = self.read_fast(n) { + return Ok(Some(fast)); + } + + let current_size = self.readahead() as usize; + + let mut out = vec![0u8; n]; + let mut remaining = n; + let mut written = 0; + if current_size > 0 { + let slice = self.active_read_slice(); + out[..slice.len()].copy_from_slice(slice); + remaining -= current_size; + written += current_size; + self.pos += current_size as Offset; + } + if self.writable() { + self.flush_rewind(vm)?; + } + self.reset_read(); + macro_rules! handle_opt_read { + ($x:expr) => { + match ($x, written > 0) { + (Some(0), _) | (None, true) => { + out.truncate(written); + return Ok(Some(out)); + } + (Some(r), _) => r, + (None, _) => return Ok(None), + } + }; + } + while remaining > 0 && !self.buffer.is_empty() { + // MINUS_LAST_BLOCK() in CPython + let r = self.buffer.len() * (remaining / self.buffer.len()); + if r == 0 { + break; + } + let r = self.raw_read(Either::A(Some(&mut out)), written..written + r, vm)?; + let r = handle_opt_read!(r); + remaining -= r; + written += r; + } + self.pos = 0; + self.raw_pos = 0; + self.read_end = 0; + + while remaining > 0 && (self.read_end as usize) < self.buffer.len() { + let r = handle_opt_read!(self.fill_buffer(vm)?); + if remaining > r { + out[written..][..r].copy_from_slice(&self.buffer[self.pos as usize..][..r]); + written += r; + self.pos += r as Offset; + remaining -= r; + } else if remaining > 0 { + out[written..][..remaining] + .copy_from_slice(&self.buffer[self.pos as usize..][..remaining]); + written += remaining; + self.pos += remaining as Offset; + remaining = 0; + } + if remaining == 0 { + break; + } + } + + Ok(Some(out)) + } + + fn fill_buffer(&mut self, vm: &VirtualMachine) -> PyResult<Option<usize>> { + let start = if self.valid_read() { + self.read_end as usize + } else { + 0 + }; + let buf_end = self.buffer.len(); + let res = self.raw_read(Either::A(None), start..buf_end, vm)?; + if let Some(n) = res.filter(|n| *n > 0) { + let new_start = (start + n) as Offset; + self.read_end = new_start; + self.raw_pos = new_start; + } + Ok(res) + } + + fn raw_read( + &mut self, + v: Either<Option<&mut Vec<u8>>, PyBuffer>, + buf_range: Range<usize>, + vm: &VirtualMachine, + ) -> PyResult<Option<usize>> { + let len = buf_range.len(); + let res = match v { + Either::A(v) => { + let v = v.unwrap_or(&mut self.buffer); + let read_buf = VecBuffer::from(core::mem::take(v)).into_ref(&vm.ctx); + let mem_obj = PyMemoryView::from_buffer_range( + read_buf.clone().into_pybuffer(false), + buf_range, + vm, + )? + .into_ref(&vm.ctx); + + // Loop if readinto() raises EINTR (PEP 475) + let res = loop { + let res = vm.call_method( + self.raw.as_ref().unwrap(), + "readinto", + (mem_obj.clone(),), + ); + match trap_eintr(res, vm) { + Ok(Some(val)) => break Ok(val), + Ok(None) => continue, // EINTR, retry + Err(e) => break Err(e), + } + }; + + mem_obj.release(); + // Always restore the buffer, even if an error occurred + *v = read_buf.take(); + + res? + } + Either::B(buf) => { + let mem_obj = + PyMemoryView::from_buffer_range(buf, buf_range, vm)?.into_ref(&vm.ctx); + // Loop if readinto() raises EINTR (PEP 475) + loop { + let res = vm.call_method( + self.raw.as_ref().unwrap(), + "readinto", + (mem_obj.clone(),), + ); + match trap_eintr(res, vm)? { + Some(val) => break val, + None => continue, + } + } + } + }; + + if vm.is_none(&res) { + return Ok(None); + } + // Try to convert to int; if it fails, treat as -1 and chain the TypeError + let (n, type_error) = match isize::try_from_object(vm, res.clone()) { + Ok(n) => (n, None), + Err(e) => (-1, Some(e)), + }; + if n < 0 || n as usize > len { + let os_error = vm.new_os_error(format!( + "raw readinto() returned invalid length {n} (should have been between 0 and {len})" + )); + if let Some(cause) = type_error { + os_error.set___cause__(Some(cause)); + } + return Err(os_error); + } + if n > 0 && self.abs_pos != -1 { + self.abs_pos += n as Offset + } + Ok(Some(n as usize)) + } + + fn read_all(&mut self, vm: &VirtualMachine) -> PyResult<Option<PyBytesRef>> { + let buf = self.active_read_slice(); + let data = if buf.is_empty() { + None + } else { + let b = buf.to_vec(); + self.pos += buf.len() as Offset; + Some(b) + }; + + if self.writable() { + self.flush_rewind(vm)?; + } + + let readall = vm + .get_str_method(self.raw.clone().unwrap(), "readall") + .transpose()?; + if let Some(readall) = readall { + let res = readall.call((), vm)?; + let res = <Option<PyBytesRef>>::try_from_object(vm, res)?; + let ret = if let Some(mut data) = data { + if let Some(bytes) = res { + data.extend_from_slice(bytes.as_bytes()); + } + Some(PyBytes::from(data).into_ref(&vm.ctx)) + } else { + res + }; + return Ok(ret); + } + + let mut chunks = Vec::new(); + + let mut read_size = 0; + loop { + // Loop with EINTR handling (PEP 475) + let read_data = loop { + let res = vm.call_method(self.raw.as_ref().unwrap(), "read", ()); + match trap_eintr(res, vm)? { + Some(val) => break val, + None => continue, + } + }; + let read_data = <Option<PyBytesRef>>::try_from_object(vm, read_data)?; + + match read_data { + Some(b) if !b.as_bytes().is_empty() => { + let l = b.as_bytes().len(); + read_size += l; + if self.abs_pos != -1 { + self.abs_pos += l as Offset; + } + chunks.push(b); + } + read_data => { + let ret = if data.is_none() && read_size == 0 { + read_data + } else { + let mut data = data.unwrap_or_default(); + data.reserve(read_size); + for bytes in &chunks { + data.extend_from_slice(bytes.as_bytes()) + } + Some(PyBytes::from(data).into_ref(&vm.ctx)) + }; + break Ok(ret); + } + } + } + } + + const fn adjust_position(&mut self, new_pos: Offset) { + self.pos = new_pos; + if self.valid_read() && self.read_end < self.pos { + self.read_end = self.pos + } + } + + fn peek(&mut self, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let have = self.readahead(); + let slice = if have > 0 { + &self.buffer[self.pos as usize..][..have as usize] + } else { + self.reset_read(); + let r = self.fill_buffer(vm)?.unwrap_or(0); + self.pos = 0; + &self.buffer[..r] + }; + Ok(slice.to_vec()) + } + + fn readinto_generic( + &mut self, + buf: PyBuffer, + readinto1: bool, + vm: &VirtualMachine, + ) -> PyResult<Option<usize>> { + let mut written = 0; + let n = self.readahead(); + let buf_len; + { + let mut b = buf.as_contiguous_mut().unwrap(); + buf_len = b.len(); + if n > 0 { + if n as usize >= b.len() { + b.copy_from_slice(&self.buffer[self.pos as usize..][..buf_len]); + self.pos += buf_len as Offset; + return Ok(Some(buf_len)); + } + b[..n as usize] + .copy_from_slice(&self.buffer[self.pos as usize..][..n as usize]); + self.pos += n; + written = n as usize; + } + } + if self.writable() { + self.flush_rewind(vm)?; + } + self.reset_read(); + self.pos = 0; + + let mut remaining = buf_len - written; + while remaining > 0 { + let n = if remaining > self.buffer.len() { + self.raw_read(Either::B(buf.clone()), written..written + remaining, vm)? + } else if !(readinto1 && written != 0) { + let n = self.fill_buffer(vm)?; + if let Some(n) = n.filter(|&n| n > 0) { + let n = core::cmp::min(n, remaining); + buf.as_contiguous_mut().unwrap()[written..][..n] + .copy_from_slice(&self.buffer[self.pos as usize..][..n]); + self.pos += n as Offset; + written += n; + remaining -= n; + continue; + } + n + } else { + break; + }; + let n = match n { + Some(0) => break, + None if written > 0 => break, + None => return Ok(None), + Some(n) => n, + }; + + if readinto1 { + written += n; + break; + } + written += n; + remaining -= n; + } + + Ok(Some(written)) + } + } + + pub fn get_offset(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Offset> { + let int = obj.try_index(vm)?; + int.as_bigint().try_into().map_err(|_| { + vm.new_value_error(format!( + "cannot fit '{}' into an offset-sized integer", + obj.class().name() + )) + }) + } + + pub fn repr_file_obj_name(obj: &PyObject, vm: &VirtualMachine) -> PyResult<Option<PyStrRef>> { + let name = match obj.get_attr("name", vm) { + Ok(name) => Some(name), + Err(e) + if e.fast_isinstance(vm.ctx.exceptions.attribute_error) + || e.fast_isinstance(vm.ctx.exceptions.value_error) => + { + None + } + Err(e) => return Err(e), + }; + match name { + Some(name) => { + if let Some(_guard) = ReprGuard::enter(vm, obj) { + name.repr(vm).map(Some) + } else { + Err(vm.new_runtime_error(format!( + "reentrant call inside {}.__repr__", + obj.class().slot_name() + ))) + } + } + None => Ok(None), + } + } + + #[pyclass] + trait BufferedMixin: PyPayload + StaticType { + const CLASS_NAME: &'static str; + const READABLE: bool; + const WRITABLE: bool; + const SEEKABLE: bool = false; + + fn data(&self) -> &PyThreadMutex<BufferedData>; + fn closing(&self) -> &AtomicBool; + fn finalizing(&self) -> &AtomicBool; + + fn lock(&self, vm: &VirtualMachine) -> PyResult<PyThreadMutexGuard<'_, BufferedData>> { + self.data() + .lock_wrapped(|do_lock| vm.allow_threads(do_lock)) + .ok_or_else(|| vm.new_runtime_error("reentrant call inside buffered io")) + } + + #[pyslot] + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let zelf: PyRef<Self> = zelf.try_into_value(vm)?; + let (raw, BufferSize { buffer_size }): (PyObjectRef, _) = + args.bind(vm).map_err(|e| { + let str_repr = e + .__str__(vm) + .as_ref() + .map_or("<error getting exception str>".as_ref(), |s| s.as_wtf8()) + .to_owned(); + let msg = format!("{}() {}", Self::CLASS_NAME, str_repr); + vm.new_exception_msg(e.class().to_owned(), msg.into()) + })?; + zelf.init(raw, BufferSize { buffer_size }, vm) + } + + fn init( + &self, + raw: PyObjectRef, + BufferSize { buffer_size }: BufferSize, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut data = self.lock(vm)?; + data.raw = None; + data.flags.remove(BufferedFlags::DETACHED); + + let buffer_size = match buffer_size { + OptionalArg::Present(i) if i <= 0 => { + return Err(vm.new_value_error("buffer size must be strictly positive")); + } + OptionalArg::Present(i) => i as usize, + OptionalArg::Missing => DEFAULT_BUFFER_SIZE, + }; + + if Self::SEEKABLE { + check_seekable(&raw, vm)?; + } + if Self::READABLE { + data.flags.insert(BufferedFlags::READABLE); + check_readable(&raw, vm)?; + } + if Self::WRITABLE { + data.flags.insert(BufferedFlags::WRITABLE); + check_writable(&raw, vm)?; + } + + data.buffer = vec![0; buffer_size]; + + if Self::READABLE { + data.reset_read(); + } + if Self::WRITABLE { + data.reset_write(); + } + if Self::SEEKABLE { + data.pos = 0; + } + + data.raw = Some(raw); + + Ok(()) + } + + #[pymethod] + fn seek( + &self, + target: PyObjectRef, + whence: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<Offset> { + let whence = whence.unwrap_or(0); + if !validate_whence(whence) { + return Err(vm.new_value_error(format!("whence value {whence} unsupported"))); + } + let mut data = self.lock(vm)?; + let raw = data.check_init(vm)?; + ensure_unclosed(raw, "seek of closed file", vm)?; + check_seekable(raw, vm)?; + let target = get_offset(target, vm)?; + data.seek(target, whence, vm) + } + + #[pymethod] + fn tell(&self, vm: &VirtualMachine) -> PyResult<Offset> { + let mut data = self.lock(vm)?; + let raw_tell = data.raw_tell(vm)?; + let raw_offset = data.raw_offset(); + let mut pos = raw_tell - raw_offset; + // GH-95782 + if pos < 0 { + pos = 0; + } + Ok(pos) + } + + #[pymethod] + fn truncate( + zelf: PyRef<Self>, + pos: OptionalOption<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let pos = pos.flatten().to_pyobject(vm); + let mut data = zelf.lock(vm)?; + data.check_init(vm)?; + if !data.writable() { + return Err(new_unsupported_operation(vm, "truncate".to_owned())); + } + data.flush_rewind(vm)?; + let res = vm.call_method(data.raw.as_ref().unwrap(), "truncate", (pos,))?; + let _ = data.raw_tell(vm); + Ok(res) + } + #[pymethod] + fn detach(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + vm.call_method(zelf.as_object(), "flush", ())?; + let mut data = zelf.lock(vm)?; + data.flags.insert(BufferedFlags::DETACHED); + data.raw + .take() + .ok_or_else(|| vm.new_value_error("raw stream has been detached")) + } + + #[pymethod] + fn seekable(&self, vm: &VirtualMachine) -> PyResult { + vm.call_method(self.lock(vm)?.check_init(vm)?, "seekable", ()) + } + + #[pygetset] + fn raw(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + Ok(self.lock(vm)?.raw.clone()) + } + + /// Get raw stream without holding the lock (for calling Python code safely) + fn get_raw_unlocked(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let data = self.lock(vm)?; + Ok(data.check_init(vm)?.to_owned()) + } + + #[pygetset] + fn closed(&self, vm: &VirtualMachine) -> PyResult { + self.get_raw_unlocked(vm)?.get_attr("closed", vm) + } + + #[pygetset] + fn name(&self, vm: &VirtualMachine) -> PyResult { + self.get_raw_unlocked(vm)?.get_attr("name", vm) + } + + #[pygetset] + fn mode(&self, vm: &VirtualMachine) -> PyResult { + self.get_raw_unlocked(vm)?.get_attr("mode", vm) + } + + #[pymethod] + fn fileno(&self, vm: &VirtualMachine) -> PyResult { + vm.call_method(self.lock(vm)?.check_init(vm)?, "fileno", ()) + } + + #[pymethod] + fn isatty(&self, vm: &VirtualMachine) -> PyResult { + vm.call_method(self.lock(vm)?.check_init(vm)?, "isatty", ()) + } + + #[pyslot] + fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let name_repr = repr_file_obj_name(zelf, vm)?; + let cls = zelf.class(); + let slot_name = cls.slot_name(); + let repr = if let Some(name_repr) = name_repr { + format!("<{slot_name} name={name_repr}>") + } else { + format!("<{slot_name}>") + }; + Ok(vm.ctx.new_str(repr)) + } + + #[pymethod] + fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + Self::slot_repr(&zelf, vm) + } + + fn close_strict(&self, vm: &VirtualMachine) -> PyResult { + let mut data = self.lock(vm)?; + let raw = data.check_init(vm)?; + if file_closed(raw, vm)? { + return Ok(vm.ctx.none()); + } + let flush_res = data.flush(vm); + let close_res = vm.call_method(data.raw.as_ref().unwrap(), "close", ()); + exception_chain(flush_res, close_res) + } + + #[pymethod] + fn close(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + // Don't hold the lock while calling Python code to avoid reentrant lock issues + let raw = { + let data = zelf.lock(vm)?; + let raw = data.check_init(vm)?; + if file_closed(raw, vm)? { + return Ok(vm.ctx.none()); + } + raw.to_owned() + }; + if zelf.finalizing().load(Ordering::Relaxed) { + // _dealloc_warn: delegate to raw._dealloc_warn(source) + let _ = vm.call_method(&raw, "_dealloc_warn", (zelf.as_object().to_owned(),)); + } + // Set closing flag so that concurrent write() calls will fail + zelf.closing().store(true, Ordering::Release); + let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop); + let close_res = vm.call_method(&raw, "close", ()); + exception_chain(flush_res, close_res) + } + + #[pymethod] + fn readable(&self) -> bool { + Self::READABLE + } + + #[pymethod] + fn writable(&self) -> bool { + Self::WRITABLE + } + + #[pymethod] + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) + } + + #[pymethod] + fn __reduce_ex__(zelf: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { + if zelf.class().is(Self::static_type()) { + return Err( + vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())) + ); + } + let _ = proto; + reduce_ex_for_subclass(zelf, vm) + } + + #[pymethod] + fn _dealloc_warn( + zelf: PyRef<Self>, + source: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Get raw reference and release lock before calling downstream + let raw = { + let data = zelf.lock(vm)?; + data.raw.clone() + }; + if let Some(raw) = raw { + let _ = vm.call_method(&raw, "_dealloc_warn", (source,)); + } + Ok(()) + } + } + + #[pyclass] + trait BufferedReadable: PyPayload { + type Reader: BufferedMixin; + + fn reader(&self) -> &Self::Reader; + + #[pymethod] + fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Option<PyBytesRef>> { + let mut data = self.reader().lock(vm)?; + let raw = data.check_init(vm)?; + let n = size.size.map(|s| *s).unwrap_or(-1); + if n < -1 { + return Err(vm.new_value_error("read length must be non-negative or -1")); + } + ensure_unclosed(raw, "read of closed file", vm)?; + match n.to_usize() { + Some(n) => data + .read_generic(n, vm) + .map(|x| x.map(|b| PyBytes::from(b).into_ref(&vm.ctx))), + None => data.read_all(vm), + } + } + + #[pymethod] + fn peek(&self, _size: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut data = self.reader().lock(vm)?; + let raw = data.check_init(vm)?; + ensure_unclosed(raw, "peek of closed file", vm)?; + + if data.writable() { + let _ = data.flush_rewind(vm); + } + data.peek(vm) + } + + #[pymethod] + fn read1(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let mut data = self.reader().lock(vm)?; + let raw = data.check_init(vm)?; + ensure_unclosed(raw, "read of closed file", vm)?; + let n = size.to_usize().unwrap_or(data.buffer.len()); + if n == 0 { + return Ok(Vec::new()); + } + let have = data.readahead(); + if have > 0 { + let n = core::cmp::min(have as usize, n); + return Ok(data.read_fast(n).unwrap()); + } + // Flush write buffer before reading + if data.writable() { + data.flush_rewind(vm)?; + } + let mut v = vec![0; n]; + data.reset_read(); + let r = data + .raw_read(Either::A(Some(&mut v)), 0..n, vm)? + .unwrap_or(0); + v.truncate(r); + v.shrink_to_fit(); + Ok(v) + } + + #[pymethod] + fn readinto(&self, buf: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<Option<usize>> { + let mut data = self.reader().lock(vm)?; + let raw = data.check_init(vm)?; + ensure_unclosed(raw, "readinto of closed file", vm)?; + data.readinto_generic(buf.into(), false, vm) + } + + #[pymethod] + fn readinto1(&self, buf: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<Option<usize>> { + let mut data = self.reader().lock(vm)?; + let raw = data.check_init(vm)?; + ensure_unclosed(raw, "readinto of closed file", vm)?; + data.readinto_generic(buf.into(), true, vm) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { + // For read-only buffers, flush just calls raw.flush() + // Don't hold the lock while calling Python code to avoid reentrant lock issues + let raw = { + let data = self.reader().lock(vm)?; + data.check_init(vm)?.to_owned() + }; + ensure_unclosed(&raw, "flush of closed file", vm)?; + vm.call_method(&raw, "flush", ())?; + Ok(()) + } + } + + fn exception_chain<T>(e1: PyResult<()>, e2: PyResult<T>) -> PyResult<T> { + match (e1, e2) { + (Err(e1), Err(e)) => { + e.set___context__(Some(e1)); + Err(e) + } + (Err(e), Ok(_)) | (Ok(()), Err(e)) => Err(e), + (Ok(()), Ok(close_res)) => Ok(close_res), + } + } + + #[pyattr] + #[pyclass(name = "BufferedReader", base = _BufferedIOBase)] + #[derive(Debug, Default)] + struct BufferedReader { + _base: _BufferedIOBase, + data: PyThreadMutex<BufferedData>, + closing: AtomicBool, + finalizing: AtomicBool, + } + + impl BufferedMixin for BufferedReader { + const CLASS_NAME: &'static str = "BufferedReader"; + const READABLE: bool = true; + const WRITABLE: bool = false; + + fn data(&self) -> &PyThreadMutex<BufferedData> { + &self.data + } + + fn closing(&self) -> &AtomicBool { + &self.closing + } + + fn finalizing(&self) -> &AtomicBool { + &self.finalizing + } + } + + impl BufferedReadable for BufferedReader { + type Reader = Self; + + fn reader(&self) -> &Self::Reader { + self + } + } + + #[pyclass( + with(Constructor, BufferedMixin, BufferedReadable, Destructor), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) + )] + impl BufferedReader {} + + impl Destructor for BufferedReader { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if let Some(buf) = zelf.downcast_ref::<BufferedReader>() { + buf.finalizing.store(true, Ordering::Relaxed); + } + iobase_finalize(zelf, vm); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } + + impl DefaultConstructor for BufferedReader {} + + #[pyclass] + trait BufferedWritable: PyPayload { + type Writer: BufferedMixin; + + fn writer(&self) -> &Self::Writer; + + #[pymethod] + fn write(&self, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { + // Check if close() is in progress (Issue #31976) + // If closing, wait for close() to complete by spinning until raw is closed. + // Note: This spin-wait has no timeout because close() is expected to always + // complete (flush + fd close). + if self.writer().closing().load(Ordering::Acquire) { + loop { + let raw = { + let data = self.writer().lock(vm)?; + match &data.raw { + Some(raw) => raw.to_owned(), + None => break, // detached + } + }; + if file_closed(&raw, vm)? { + break; + } + // Yield to other threads + std::thread::yield_now(); + } + return Err(vm.new_value_error("write to closed file")); + } + let mut data = self.writer().lock(vm)?; + let raw = data.check_init(vm)?; + ensure_unclosed(raw, "write to closed file", vm)?; + + data.write(obj, vm) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { + let mut data = self.writer().lock(vm)?; + let raw = data.check_init(vm)?; + ensure_unclosed(raw, "flush of closed file", vm)?; + data.flush_rewind(vm) + } + } + + #[pyattr] + #[pyclass(name = "BufferedWriter", base = _BufferedIOBase)] + #[derive(Debug, Default)] + struct BufferedWriter { + _base: _BufferedIOBase, + data: PyThreadMutex<BufferedData>, + closing: AtomicBool, + finalizing: AtomicBool, + } + + impl BufferedMixin for BufferedWriter { + const CLASS_NAME: &'static str = "BufferedWriter"; + const READABLE: bool = false; + const WRITABLE: bool = true; + + fn data(&self) -> &PyThreadMutex<BufferedData> { + &self.data + } + + fn closing(&self) -> &AtomicBool { + &self.closing + } + + fn finalizing(&self) -> &AtomicBool { + &self.finalizing + } + } + + impl BufferedWritable for BufferedWriter { + type Writer = Self; + + fn writer(&self) -> &Self::Writer { + self + } + } + + #[pyclass( + with(Constructor, BufferedMixin, BufferedWritable, Destructor), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) + )] + impl BufferedWriter {} + + impl Destructor for BufferedWriter { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if let Some(buf) = zelf.downcast_ref::<BufferedWriter>() { + buf.finalizing.store(true, Ordering::Relaxed); + } + iobase_finalize(zelf, vm); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } + + impl DefaultConstructor for BufferedWriter {} + + #[pyattr] + #[pyclass(name = "BufferedRandom", base = _BufferedIOBase)] + #[derive(Debug, Default)] + struct BufferedRandom { + _base: _BufferedIOBase, + data: PyThreadMutex<BufferedData>, + closing: AtomicBool, + finalizing: AtomicBool, + } + + impl BufferedMixin for BufferedRandom { + const CLASS_NAME: &'static str = "BufferedRandom"; + const READABLE: bool = true; + const WRITABLE: bool = true; + const SEEKABLE: bool = true; + + fn data(&self) -> &PyThreadMutex<BufferedData> { + &self.data + } + + fn closing(&self) -> &AtomicBool { + &self.closing + } + + fn finalizing(&self) -> &AtomicBool { + &self.finalizing + } + } + + impl BufferedReadable for BufferedRandom { + type Reader = Self; + + fn reader(&self) -> &Self::Reader { + self + } + } + + impl BufferedWritable for BufferedRandom { + type Writer = Self; + + fn writer(&self) -> &Self::Writer { + self + } + } + + #[pyclass( + with( + Constructor, + BufferedMixin, + BufferedReadable, + BufferedWritable, + Destructor + ), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) + )] + impl BufferedRandom {} + + impl Destructor for BufferedRandom { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if let Some(buf) = zelf.downcast_ref::<BufferedRandom>() { + buf.finalizing.store(true, Ordering::Relaxed); + } + iobase_finalize(zelf, vm); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } + + impl DefaultConstructor for BufferedRandom {} + + #[pyattr] + #[pyclass(name = "BufferedRWPair", base = _BufferedIOBase)] + #[derive(Debug, Default)] + struct BufferedRWPair { + _base: _BufferedIOBase, + read: BufferedReader, + write: BufferedWriter, + } + + impl BufferedReadable for BufferedRWPair { + type Reader = BufferedReader; + + fn reader(&self) -> &Self::Reader { + &self.read + } + } + + impl BufferedWritable for BufferedRWPair { + type Writer = BufferedWriter; + + fn writer(&self) -> &Self::Writer { + &self.write + } + } + + impl DefaultConstructor for BufferedRWPair {} + + impl Initializer for BufferedRWPair { + type Args = (PyObjectRef, PyObjectRef, BufferSize); + + fn init( + zelf: PyRef<Self>, + (reader, writer, buffer_size): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<()> { + zelf.read.init(reader, buffer_size.clone(), vm)?; + zelf.write.init(writer, buffer_size, vm)?; + Ok(()) + } + } + + #[pyclass( + with( + Constructor, + Initializer, + BufferedReadable, + BufferedWritable, + Destructor + ), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) + )] + impl BufferedRWPair { + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { + self.write.flush(vm) + } + + #[pymethod] + const fn readable(&self) -> bool { + true + } + #[pymethod] + const fn writable(&self) -> bool { + true + } + + #[pygetset] + fn closed(&self, vm: &VirtualMachine) -> PyResult { + self.write.closed(vm) + } + + #[pymethod] + fn isatty(&self, vm: &VirtualMachine) -> PyResult { + // read.isatty() or write.isatty() + let res = self.read.isatty(vm)?; + if res.clone().try_to_bool(vm)? { + Ok(res) + } else { + self.write.isatty(vm) + } + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult { + let write_res = self.write.close_strict(vm).map(drop); + let read_res = self.read.close_strict(vm); + exception_chain(write_res, read_res) + } + } + + impl Destructor for BufferedRWPair { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + iobase_finalize(zelf, vm); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } + + #[derive(FromArgs)] + struct TextIOWrapperArgs { + #[pyarg(any, default)] + encoding: Option<PyUtf8StrRef>, + #[pyarg(any, default)] + errors: Option<PyUtf8StrRef>, + #[pyarg(any, default)] + newline: OptionalOption<Newlines>, + #[pyarg(any, default)] + line_buffering: OptionalOption<PyObjectRef>, + #[pyarg(any, default)] + write_through: OptionalOption<PyObjectRef>, + } + + #[derive(Debug, Copy, Clone, Default, PartialEq)] + enum Newlines { + #[default] + Universal, + Passthrough, + Lf, + Cr, + Crlf, + } + + impl Newlines { + /// returns position where the new line starts if found, otherwise position at which to + /// continue the search after more is read into the buffer + fn find_newline(&self, s: &Wtf8) -> Result<usize, usize> { + let len = s.len(); + match self { + Self::Universal | Self::Lf => s.find("\n".as_ref()).map(|p| p + 1).ok_or(len), + Self::Passthrough => { + let bytes = s.as_bytes(); + memchr::memchr2(b'\n', b'\r', bytes) + .map(|p| { + let nl_len = + if bytes[p] == b'\r' && bytes.get(p + 1).copied() == Some(b'\n') { + 2 + } else { + 1 + }; + p + nl_len + }) + .ok_or(len) + } + Self::Cr => s.find("\r".as_ref()).map(|p| p + 1).ok_or(len), + Self::Crlf => { + // s[searched..] == remaining + let mut searched = 0; + let mut remaining = s.as_bytes(); + loop { + match memchr::memchr(b'\r', remaining) { + Some(p) => match remaining.get(p + 1) { + Some(&ch_after_cr) => { + let pos_after = p + 2; + if ch_after_cr == b'\n' { + break Ok(searched + pos_after); + } else { + searched += pos_after; + remaining = &remaining[pos_after..]; + continue; + } + } + None => break Err(searched + p), + }, + None => break Err(len), + } + } + } + } + } + } + + impl TryFromObject for Newlines { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let nl = if vm.is_none(&obj) { + Self::Universal + } else { + let s = obj.downcast::<PyStr>().map_err(|obj| { + vm.new_type_error(format!( + "newline argument must be str or None, not {}", + obj.class().name() + )) + })?; + let wtf8 = s.as_wtf8(); + if !wtf8.is_utf8() { + let repr = s.repr(vm)?.as_str().to_owned(); + return Err(vm.new_value_error(format!("illegal newline value: {repr}"))); + } + let s_str = wtf8.as_str().expect("checked utf8"); + match s_str { + "" => Self::Passthrough, + "\n" => Self::Lf, + "\r" => Self::Cr, + "\r\n" => Self::Crlf, + _ => return Err(vm.new_value_error(format!("illegal newline value: {s}"))), + } + }; + Ok(nl) + } + } + + fn reduce_ex_for_subclass(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let cls = zelf.class(); + let new = vm + .get_attribute_opt(cls.to_owned().into(), "__new__")? + .ok_or_else(|| vm.new_attribute_error("type has no attribute '__new__'"))?; + let args = vm.ctx.new_tuple(vec![cls.to_owned().into()]); + let state = if let Some(getstate) = vm.get_attribute_opt(zelf.clone(), "__getstate__")? { + getstate.call((), vm)? + } else if let Ok(dict) = zelf.get_attr("__dict__", vm) { + dict + } else { + vm.ctx.none() + }; + Ok(vm.ctx.new_tuple(vec![new, args.into(), state]).into()) + } + + /// A length of or index into a UTF-8 string, measured in both chars and bytes + #[derive(Debug, Default, Copy, Clone)] + struct Utf8size { + bytes: usize, + chars: usize, + } + + impl Utf8size { + fn len_pystr(s: &PyStr) -> Self { + Self { + bytes: s.byte_len(), + chars: s.char_len(), + } + } + + fn len_str(s: &Wtf8) -> Self { + Self { + bytes: s.len(), + chars: s.code_points().count(), + } + } + } + + impl core::ops::Add for Utf8size { + type Output = Self; + + #[inline] + fn add(mut self, rhs: Self) -> Self { + self += rhs; + self + } + } + + impl core::ops::AddAssign for Utf8size { + #[inline] + fn add_assign(&mut self, rhs: Self) { + self.bytes += rhs.bytes; + self.chars += rhs.chars; + } + } + + impl core::ops::Sub for Utf8size { + type Output = Self; + + #[inline] + fn sub(mut self, rhs: Self) -> Self { + self -= rhs; + self + } + } + + impl core::ops::SubAssign for Utf8size { + #[inline] + fn sub_assign(&mut self, rhs: Self) { + self.bytes -= rhs.bytes; + self.chars -= rhs.chars; + } + } + + // TODO: implement legit fast-paths for other encodings + type EncodeFunc = fn(PyStrRef) -> PendingWrite; + const fn textio_encode_utf8(s: PyStrRef) -> PendingWrite { + PendingWrite::Utf8(s) + } + + #[derive(Debug)] + struct TextIOData { + buffer: PyObjectRef, + encoder: Option<(PyObjectRef, Option<EncodeFunc>)>, + decoder: Option<PyObjectRef>, + encoding: PyUtf8StrRef, + errors: PyUtf8StrRef, + newline: Newlines, + line_buffering: bool, + write_through: bool, + chunk_size: usize, + seekable: bool, + has_read1: bool, + // these are more state than configuration + pending: PendingWrites, + telling: bool, + snapshot: Option<(i32, PyBytesRef)>, + decoded_chars: Option<PyStrRef>, + // number of characters we've consumed from decoded_chars + decoded_chars_used: Utf8size, + b2cratio: f64, + } + + #[derive(Debug, Default)] + struct PendingWrites { + num_bytes: usize, + data: PendingWritesData, + } + + #[derive(Debug, Default)] + enum PendingWritesData { + #[default] + None, + One(PendingWrite), + Many(Vec<PendingWrite>), + } + + #[derive(Debug)] + enum PendingWrite { + Utf8(PyStrRef), + Bytes(PyBytesRef), + } + + impl PendingWrite { + fn as_bytes(&self) -> &[u8] { + match self { + Self::Utf8(s) => s.as_bytes(), + Self::Bytes(b) => b.as_bytes(), + } + } + } + + impl PendingWrites { + fn push(&mut self, write: PendingWrite) { + self.num_bytes += write.as_bytes().len(); + self.data = match core::mem::take(&mut self.data) { + PendingWritesData::None => PendingWritesData::One(write), + PendingWritesData::One(write1) => PendingWritesData::Many(vec![write1, write]), + PendingWritesData::Many(mut v) => { + v.push(write); + PendingWritesData::Many(v) + } + } + } + fn take(&mut self, vm: &VirtualMachine) -> PyBytesRef { + let Self { num_bytes, data } = core::mem::take(self); + if let PendingWritesData::One(PendingWrite::Bytes(b)) = data { + return b; + } + let writes_iter = match data { + PendingWritesData::None => itertools::Either::Left(vec![].into_iter()), + PendingWritesData::One(write) => itertools::Either::Right(core::iter::once(write)), + PendingWritesData::Many(writes) => itertools::Either::Left(writes.into_iter()), + }; + let mut buf = Vec::with_capacity(num_bytes); + writes_iter.for_each(|chunk| buf.extend_from_slice(chunk.as_bytes())); + PyBytes::from(buf).into_ref(&vm.ctx) + } + } + + #[derive(Default, Debug)] + struct TextIOCookie { + start_pos: Offset, + dec_flags: i32, + bytes_to_feed: i32, + chars_to_skip: i32, + need_eof: bool, + // chars_to_skip but utf8 bytes + bytes_to_skip: i32, + } + + impl TextIOCookie { + const START_POS_OFF: usize = 0; + const DEC_FLAGS_OFF: usize = Self::START_POS_OFF + core::mem::size_of::<Offset>(); + const BYTES_TO_FEED_OFF: usize = Self::DEC_FLAGS_OFF + 4; + const CHARS_TO_SKIP_OFF: usize = Self::BYTES_TO_FEED_OFF + 4; + const NEED_EOF_OFF: usize = Self::CHARS_TO_SKIP_OFF + 4; + const BYTES_TO_SKIP_OFF: usize = Self::NEED_EOF_OFF + 1; + const BYTE_LEN: usize = Self::BYTES_TO_SKIP_OFF + 4; + + fn parse(cookie: &BigInt) -> Option<Self> { + let (_, mut buf) = cookie.to_bytes_le(); + if buf.len() > Self::BYTE_LEN { + return None; + } + buf.resize(Self::BYTE_LEN, 0); + let buf: &[u8; Self::BYTE_LEN] = buf.as_array()?; + macro_rules! get_field { + ($t:ty, $off:ident) => { + <$t>::from_ne_bytes(*buf[Self::$off..].first_chunk().unwrap()) + }; + } + Some(Self { + start_pos: get_field!(Offset, START_POS_OFF), + dec_flags: get_field!(i32, DEC_FLAGS_OFF), + bytes_to_feed: get_field!(i32, BYTES_TO_FEED_OFF), + chars_to_skip: get_field!(i32, CHARS_TO_SKIP_OFF), + need_eof: get_field!(u8, NEED_EOF_OFF) != 0, + bytes_to_skip: get_field!(i32, BYTES_TO_SKIP_OFF), + }) + } + + fn build(&self) -> BigInt { + let mut buf = [0; Self::BYTE_LEN]; + macro_rules! set_field { + ($field:expr, $off:ident) => {{ + let field = $field; + buf[Self::$off..][..core::mem::size_of_val(&field)] + .copy_from_slice(&field.to_ne_bytes()) + }}; + } + set_field!(self.start_pos, START_POS_OFF); + set_field!(self.dec_flags, DEC_FLAGS_OFF); + set_field!(self.bytes_to_feed, BYTES_TO_FEED_OFF); + set_field!(self.chars_to_skip, CHARS_TO_SKIP_OFF); + set_field!(self.need_eof as u8, NEED_EOF_OFF); + set_field!(self.bytes_to_skip, BYTES_TO_SKIP_OFF); + BigInt::from_signed_bytes_le(&buf) + } + + fn set_decoder_state(&self, decoder: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if self.start_pos == 0 && self.dec_flags == 0 { + vm.call_method(decoder, "reset", ())?; + } else { + vm.call_method( + decoder, + "setstate", + ((vm.ctx.new_bytes(vec![]), self.dec_flags),), + )?; + } + Ok(()) + } + + const fn num_to_skip(&self) -> Utf8size { + Utf8size { + bytes: self.bytes_to_skip as usize, + chars: self.chars_to_skip as usize, + } + } + + const fn set_num_to_skip(&mut self, num: Utf8size) { + self.bytes_to_skip = num.bytes as i32; + self.chars_to_skip = num.chars as i32; + } + } + + #[pyclass(module = "_io", name, no_attr)] + #[derive(Debug, PyPayload)] + struct StatelessIncrementalEncoder { + encode: PyObjectRef, + errors: Option<PyStrRef>, + name: Option<PyStrRef>, + } + + #[pyclass] + impl StatelessIncrementalEncoder { + #[pymethod] + fn encode( + &self, + input: PyObjectRef, + _final: OptionalArg<bool>, + vm: &VirtualMachine, + ) -> PyResult { + let mut args: Vec<PyObjectRef> = vec![input]; + if let Some(errors) = &self.errors { + args.push(errors.to_owned().into()); + } + let res = self.encode.call(args, vm)?; + let tuple: PyTupleRef = res.try_into_value(vm)?; + if tuple.len() != 2 { + return Err(vm.new_type_error("encoder must return a tuple (object, integer)")); + } + Ok(tuple[0].clone()) + } + + #[pymethod] + fn reset(&self) {} + + #[pymethod] + fn setstate(&self, _state: PyObjectRef) {} + + #[pymethod] + fn getstate(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_int(0).into() + } + + #[pygetset] + fn name(&self) -> Option<PyStrRef> { + self.name.clone() + } + } + + #[pyclass(module = "_io", name, no_attr)] + #[derive(Debug, PyPayload)] + struct StatelessIncrementalDecoder { + decode: PyObjectRef, + errors: Option<PyStrRef>, + } + + #[pyclass] + impl StatelessIncrementalDecoder { + #[pymethod] + fn decode( + &self, + input: PyObjectRef, + _final: OptionalArg<bool>, + vm: &VirtualMachine, + ) -> PyResult { + let mut args: Vec<PyObjectRef> = vec![input]; + if let Some(errors) = &self.errors { + args.push(errors.to_owned().into()); + } + let res = self.decode.call(args, vm)?; + let tuple: PyTupleRef = res.try_into_value(vm)?; + if tuple.len() != 2 { + return Err(vm.new_type_error("decoder must return a tuple (object, integer)")); + } + Ok(tuple[0].clone()) + } + + #[pymethod] + fn getstate(&self, vm: &VirtualMachine) -> (PyBytesRef, u64) { + (vm.ctx.empty_bytes.to_owned(), 0) + } + + #[pymethod] + fn setstate(&self, _state: PyTupleRef, _vm: &VirtualMachine) {} + + #[pymethod] + fn reset(&self) {} + } + + #[pyattr] + #[pyclass(name = "TextIOWrapper", base = _TextIOBase)] + #[derive(Debug, Default)] + struct TextIOWrapper { + _base: _TextIOBase, + data: PyThreadMutex<Option<TextIOData>>, + finalizing: AtomicBool, + } + + impl DefaultConstructor for TextIOWrapper {} + + impl Initializer for TextIOWrapper { + type Args = (PyObjectRef, TextIOWrapperArgs); + + fn init( + zelf: PyRef<Self>, + (buffer, args): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut data = zelf.lock_opt(vm)?; + *data = None; + + let encoding = Self::resolve_encoding(args.encoding, vm)?; + + let errors = args.errors.unwrap_or_else(|| vm.ctx.new_utf8_str("strict")); + Self::validate_errors(&errors, vm)?; + + let has_read1 = vm.get_attribute_opt(buffer.clone(), "read1")?.is_some(); + let seekable = vm.call_method(&buffer, "seekable", ())?.try_to_bool(vm)?; + + let newline = match args.newline { + OptionalArg::Missing | OptionalArg::Present(None) => Newlines::default(), + OptionalArg::Present(Some(newline)) => newline, + }; + let (encoder, decoder) = + Self::find_coder(&buffer, encoding.as_str(), &errors, newline, vm)?; + if let Some((encoder, _)) = &encoder { + Self::adjust_encoder_state_for_bom(encoder, encoding.as_str(), &buffer, vm)?; + } + + let line_buffering = match args.line_buffering { + OptionalArg::Missing => false, + OptionalArg::Present(None) => false, + OptionalArg::Present(Some(value)) => value.try_to_bool(vm)?, + }; + let write_through = match args.write_through { + OptionalArg::Missing => false, + OptionalArg::Present(None) => false, + OptionalArg::Present(Some(value)) => value.try_to_bool(vm)?, + }; + + *data = Some(TextIOData { + buffer, + encoder, + decoder, + encoding, + errors, + newline, + line_buffering, + write_through, + chunk_size: 8192, + seekable, + has_read1, + + pending: PendingWrites::default(), + telling: seekable, + snapshot: None, + decoded_chars: None, + decoded_chars_used: Utf8size::default(), + b2cratio: 0.0, + }); + + Ok(()) + } + + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let zelf_ref: PyRef<Self> = zelf.try_into_value(vm)?; + { + let mut data = zelf_ref.lock_opt(vm)?; + *data = None; + } + let (buffer, text_args): (PyObjectRef, TextIOWrapperArgs) = args.bind(vm)?; + Self::init(zelf_ref, (buffer, text_args), vm) + } + } + + impl TextIOWrapper { + fn lock_opt( + &self, + vm: &VirtualMachine, + ) -> PyResult<PyThreadMutexGuard<'_, Option<TextIOData>>> { + self.data + .lock_wrapped(|do_lock| vm.allow_threads(do_lock)) + .ok_or_else(|| vm.new_runtime_error("reentrant call inside textio")) + } + + fn lock(&self, vm: &VirtualMachine) -> PyResult<PyMappedThreadMutexGuard<'_, TextIOData>> { + let lock = self.lock_opt(vm)?; + PyThreadMutexGuard::try_map(lock, |x| x.as_mut()) + .map_err(|_| vm.new_value_error("I/O operation on uninitialized object")) + } + + fn validate_errors(errors: &PyRef<PyUtf8Str>, vm: &VirtualMachine) -> PyResult<()> { + if errors.as_str().contains('\0') { + return Err(cstring_error(vm)); + } + vm.state + .codec_registry + .lookup_error(errors.as_str(), vm) + .map(drop) + } + + fn bool_from_index(value: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + let int = value.try_index(vm)?; + let value: i32 = int.try_to_primitive(vm)?; + Ok(value != 0) + } + + fn resolve_encoding( + encoding: Option<PyUtf8StrRef>, + vm: &VirtualMachine, + ) -> PyResult<PyUtf8StrRef> { + // Note: Do not issue EncodingWarning here. The warning should only + // be issued by io.text_encoding(), the public API. This function + // is used internally (e.g., for stdin/stdout/stderr initialization) + // where no warning should be emitted. + let encoding = match encoding { + None if vm.state.config.settings.utf8_mode > 0 => { + identifier_utf8!(vm, utf_8).to_owned() + } + Some(enc) if enc.as_str() == "locale" => match vm.import("locale", 0) { + Ok(locale) => locale + .get_attr("getencoding", vm)? + .call((), vm)? + .try_into_value(vm)?, + Err(err) + if err.fast_isinstance(vm.ctx.exceptions.import_error) + || err.fast_isinstance(vm.ctx.exceptions.module_not_found_error) => + { + identifier_utf8!(vm, utf_8).to_owned() + } + Err(err) => return Err(err), + }, + Some(enc) => { + if enc.as_str().contains('\0') { + return Err(cstring_error(vm)); + } + enc + } + _ => match vm.import("locale", 0) { + Ok(locale) => locale + .get_attr("getencoding", vm)? + .call((), vm)? + .try_into_value(vm)?, + Err(err) + if err.fast_isinstance(vm.ctx.exceptions.import_error) + || err.fast_isinstance(vm.ctx.exceptions.module_not_found_error) => + { + identifier_utf8!(vm, utf_8).to_owned() + } + Err(err) => return Err(err), + }, + }; + if encoding.as_str().contains('\0') { + return Err(cstring_error(vm)); + } + Ok(encoding) + } + + fn adjust_encoder_state_for_bom( + encoder: &PyObjectRef, + encoding: &str, + buffer: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<()> { + let needs_bom = matches!(encoding, "utf-8-sig" | "utf-16" | "utf-32"); + if !needs_bom { + return Ok(()); + } + let seekable = vm.call_method(buffer, "seekable", ())?.try_to_bool(vm)?; + if !seekable { + return Ok(()); + } + let pos = vm.call_method(buffer, "tell", ())?; + if vm.bool_eq(&pos, vm.ctx.new_int(0).as_ref())? { + return Ok(()); + } + if let Err(err) = vm.call_method(encoder, "setstate", (0,)) + && !err.fast_isinstance(vm.ctx.exceptions.attribute_error) + { + return Err(err); + } + Ok(()) + } + + #[allow(clippy::type_complexity)] + fn find_coder( + buffer: &PyObject, + encoding: &str, + errors: &Py<PyUtf8Str>, + newline: Newlines, + vm: &VirtualMachine, + ) -> PyResult<( + Option<(PyObjectRef, Option<EncodeFunc>)>, + Option<PyObjectRef>, + )> { + let codec = vm.state.codec_registry.lookup(encoding, vm)?; + if !codec.is_text_codec(vm)? { + return Err(vm.new_lookup_error(format!( + "'{encoding}' is not a text encoding; use codecs.open() to handle arbitrary codecs" + ))); + } + let errors = errors.to_owned().into_wtf8(); + + let encoder = if vm.call_method(buffer, "writable", ())?.try_to_bool(vm)? { + let incremental_encoder = + match codec.get_incremental_encoder(Some(errors.clone()), vm) { + Ok(encoder) => encoder, + Err(err) + if err.fast_isinstance(vm.ctx.exceptions.type_error) + || err.fast_isinstance(vm.ctx.exceptions.attribute_error) => + { + let name = vm + .get_attribute_opt(codec.as_tuple().to_owned().into(), "name")? + .and_then(|obj| obj.downcast::<PyStr>().ok()); + StatelessIncrementalEncoder { + encode: codec.get_encode_func().to_owned(), + errors: Some(errors.clone()), + name, + } + .into_ref(&vm.ctx) + .into() + } + Err(err) => return Err(err), + }; + let encoding_name = vm.get_attribute_opt(incremental_encoder.clone(), "name")?; + let encode_func = encoding_name.and_then(|name| { + let name = name.downcast_ref::<PyStr>()?; + match name.to_str()? { + "utf-8" => Some(textio_encode_utf8 as EncodeFunc), + _ => None, + } + }); + Some((incremental_encoder, encode_func)) + } else { + None + }; + + let decoder = if vm.call_method(buffer, "readable", ())?.try_to_bool(vm)? { + let decoder = match codec.get_incremental_decoder(Some(errors.clone()), vm) { + Ok(decoder) => decoder, + Err(err) + if err.fast_isinstance(vm.ctx.exceptions.type_error) + || err.fast_isinstance(vm.ctx.exceptions.attribute_error) => + { + StatelessIncrementalDecoder { + decode: codec.get_decode_func().to_owned(), + errors: Some(errors), + } + .into_ref(&vm.ctx) + .into() + } + Err(err) => return Err(err), + }; + if let Newlines::Universal | Newlines::Passthrough = newline { + let args = IncrementalNewlineDecoderArgs { + decoder, + translate: matches!(newline, Newlines::Universal), + errors: None, + }; + Some(IncrementalNewlineDecoder::construct_and_init(args, vm)?.into()) + } else { + Some(decoder) + } + } else { + None + }; + Ok((encoder, decoder)) + } + } + + #[inline] + fn flush_inner(textio: &mut TextIOData, vm: &VirtualMachine) -> PyResult { + textio.check_closed(vm)?; + textio.telling = textio.seekable; + textio.write_pending(vm)?; + vm.call_method(&textio.buffer, "flush", ()) + } + + #[pyclass( + with( + Constructor, + Initializer, + Destructor, + Iterable, + IterNext, + Representable + ), + flags(BASETYPE, HAS_WEAKREF) + )] + impl TextIOWrapper { + #[pymethod] + fn reconfigure(&self, args: TextIOWrapperArgs, vm: &VirtualMachine) -> PyResult<()> { + let mut data = self.lock(vm)?; + data.check_closed(vm)?; + + let mut encoding = data.encoding.clone(); + let mut errors = data.errors.clone(); + let mut newline = data.newline; + let mut encoding_changed = false; + let mut errors_changed = false; + let mut newline_changed = false; + let mut line_buffering = None; + let mut write_through = None; + + if let Some(enc) = args.encoding { + if enc.as_str().contains('\0') && enc.as_str().starts_with("locale") { + return Err(vm.new_lookup_error(format!("unknown encoding: {enc}"))); + } + let resolved = Self::resolve_encoding(Some(enc), vm)?; + encoding_changed = resolved.as_str() != encoding.as_str(); + encoding = resolved; + } + + if let Some(errs) = args.errors { + Self::validate_errors(&errs, vm)?; + errors_changed = errs.as_str() != errors.as_str(); + errors = errs; + } else if encoding_changed { + errors = identifier_utf8!(vm, strict).to_owned(); + errors_changed = true; + } + + if let OptionalArg::Present(nl) = args.newline { + let nl = nl.unwrap_or_default(); + newline_changed = nl != newline; + newline = nl; + } + + if let OptionalArg::Present(Some(value)) = args.line_buffering { + line_buffering = Some(Self::bool_from_index(value, vm)?); + } + if let OptionalArg::Present(Some(value)) = args.write_through { + write_through = Some(Self::bool_from_index(value, vm)?); + } + + if (encoding_changed || newline_changed) + && data.decoder.is_some() + && (data.decoded_chars.is_some() + || data.snapshot.is_some() + || data.decoded_chars_used.chars != 0) + { + return Err(new_unsupported_operation( + vm, + "cannot reconfigure encoding or newline after reading from the stream" + .to_owned(), + )); + } + + if data.pending.num_bytes > 0 { + data.write_pending(vm)?; + } + vm.call_method(&data.buffer, "flush", ())?; + + if encoding_changed || errors_changed || newline_changed { + if data.pending.num_bytes > 0 { + data.write_pending(vm)?; + } + let (encoder, decoder) = + Self::find_coder(&data.buffer, encoding.as_str(), &errors, newline, vm)?; + data.encoding = encoding; + data.errors = errors; + data.newline = newline; + data.encoder = encoder; + data.decoder = decoder; + data.set_decoded_chars(None); + data.snapshot = None; + data.decoded_chars_used = Utf8size::default(); + if let Some((encoder, _)) = &data.encoder { + Self::adjust_encoder_state_for_bom( + encoder, + data.encoding.as_str(), + &data.buffer, + vm, + )?; + } + } + + if let Some(line_buffering) = line_buffering { + data.line_buffering = line_buffering; + } + if let Some(write_through) = write_through { + data.write_through = write_through; + } + Ok(()) + } + + #[pymethod] + fn detach(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let mut textio = zelf.lock(vm)?; + + // Fail fast if already detached + if vm.is_none(&textio.buffer) { + return Err(vm.new_value_error("underlying buffer has been detached")); + } + + flush_inner(&mut textio, vm)?; + + let buffer = textio.buffer.clone(); + textio.buffer = vm.ctx.none(); + Ok(buffer) + } + + #[pymethod] + fn seekable(&self, vm: &VirtualMachine) -> PyResult { + let textio = self.lock(vm)?; + vm.call_method(&textio.buffer, "seekable", ()) + } + + #[pymethod] + fn readable(&self, vm: &VirtualMachine) -> PyResult { + let textio = self.lock(vm)?; + vm.call_method(&textio.buffer, "readable", ()) + } + + #[pymethod] + fn writable(&self, vm: &VirtualMachine) -> PyResult { + let textio = self.lock(vm)?; + vm.call_method(&textio.buffer, "writable", ()) + } + + #[pygetset] + fn line_buffering(&self, vm: &VirtualMachine) -> PyResult<bool> { + Ok(self.lock(vm)?.line_buffering) + } + + #[pygetset] + fn write_through(&self, vm: &VirtualMachine) -> PyResult<bool> { + Ok(self.lock(vm)?.write_through) + } + + #[pygetset] + fn newlines(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + let data = self.lock(vm)?; + let Some(decoder) = &data.decoder else { + return Ok(None); + }; + vm.get_attribute_opt(decoder.clone(), "newlines") + } + + #[pygetset(name = "_CHUNK_SIZE")] + fn chunksize(&self, vm: &VirtualMachine) -> PyResult<usize> { + Ok(self.lock(vm)?.chunk_size) + } + + #[pygetset(setter, name = "_CHUNK_SIZE")] + fn set_chunksize( + &self, + chunk_size: PySetterValue<usize>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut textio = self.lock(vm)?; + match chunk_size { + PySetterValue::Assign(chunk_size) => textio.chunk_size = chunk_size, + PySetterValue::Delete => Err(vm.new_attribute_error("cannot delete attribute"))?, + }; + // TODO: RUSTPYTHON + // Change chunk_size type, validate it manually and throws ValueError if invalid. + // https://github.com/python/cpython/blob/2e9da8e3522764d09f1d6054a2be567e91a30812/Modules/_io/textio.c#L3124-L3143 + Ok(()) + } + + #[pymethod] + fn seek( + zelf: PyRef<Self>, + cookie: PyObjectRef, + how: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult { + let how = how.unwrap_or(0); + + let reset_encoder = |encoder, start_of_stream| { + if start_of_stream { + vm.call_method(encoder, "reset", ()) + } else { + vm.call_method(encoder, "setstate", (0,)) + } + }; + + let textio = zelf.lock(vm)?; + + if !textio.seekable { + return Err(new_unsupported_operation( + vm, + "underlying stream is not seekable".to_owned(), + )); + } + + let cookie = match how { + // SEEK_SET + 0 => cookie, + // SEEK_CUR + 1 => { + if vm.bool_eq(&cookie, vm.ctx.new_int(0).as_ref())? { + vm.call_method(&textio.buffer, "tell", ())? + } else { + return Err(new_unsupported_operation( + vm, + "can't do nonzero cur-relative seeks".to_owned(), + )); + } + } + // SEEK_END + 2 => { + if vm.bool_eq(&cookie, vm.ctx.new_int(0).as_ref())? { + drop(textio); + vm.call_method(zelf.as_object(), "flush", ())?; + let mut textio = zelf.lock(vm)?; + textio.set_decoded_chars(None); + textio.snapshot = None; + if let Some(decoder) = &textio.decoder { + vm.call_method(decoder, "reset", ())?; + } + let res = vm.call_method(&textio.buffer, "seek", (0, 2))?; + if let Some((encoder, _)) = &textio.encoder { + let start_of_stream = vm.bool_eq(&res, vm.ctx.new_int(0).as_ref())?; + reset_encoder(encoder, start_of_stream)?; + } + return Ok(res); + } else { + return Err(new_unsupported_operation( + vm, + "can't do nonzero end-relative seeks".to_owned(), + )); + } + } + _ => { + return Err( + vm.new_value_error(format!("invalid whence ({how}, should be 0, 1 or 2)")) + ); + } + }; + use crate::types::PyComparisonOp; + if cookie.rich_compare_bool(vm.ctx.new_int(0).as_ref(), PyComparisonOp::Lt, vm)? { + return Err( + vm.new_value_error(format!("negative seek position {}", &cookie.repr(vm)?)) + ); + } + drop(textio); + vm.call_method(zelf.as_object(), "flush", ())?; + let cookie_obj = crate::builtins::PyIntRef::try_from_object(vm, cookie)?; + let cookie = TextIOCookie::parse(cookie_obj.as_bigint()) + .ok_or_else(|| vm.new_value_error("invalid cookie"))?; + let mut textio = zelf.lock(vm)?; + vm.call_method(&textio.buffer, "seek", (cookie.start_pos,))?; + textio.set_decoded_chars(None); + textio.snapshot = None; + if let Some(decoder) = &textio.decoder { + cookie.set_decoder_state(decoder, vm)?; + } + if cookie.chars_to_skip != 0 { + let TextIOData { + ref decoder, + ref buffer, + ref mut snapshot, + .. + } = *textio; + let decoder = decoder + .as_ref() + .ok_or_else(|| vm.new_value_error("invalid cookie"))?; + let input_chunk = vm.call_method(buffer, "read", (cookie.bytes_to_feed,))?; + let input_chunk: PyBytesRef = input_chunk.downcast().map_err(|obj| { + vm.new_type_error(format!( + "underlying read() should have returned a bytes object, not '{}'", + obj.class().name() + )) + })?; + *snapshot = Some((cookie.dec_flags, input_chunk.clone())); + let decoded = vm.call_method(decoder, "decode", (input_chunk, cookie.need_eof))?; + let decoded = check_decoded(decoded, vm)?; + let pos_is_valid = decoded + .as_wtf8() + .is_code_point_boundary(cookie.bytes_to_skip as usize); + textio.set_decoded_chars(Some(decoded)); + if !pos_is_valid { + return Err(vm.new_os_error("can't restore logical file position")); + } + textio.decoded_chars_used = cookie.num_to_skip(); + } else { + textio.snapshot = Some((cookie.dec_flags, PyBytes::from(vec![]).into_ref(&vm.ctx))) + } + if let Some((encoder, _)) = &textio.encoder { + let start_of_stream = cookie.start_pos == 0 && cookie.dec_flags == 0; + reset_encoder(encoder, start_of_stream)?; + } + Ok(cookie_obj.into()) + } + + #[pymethod] + fn tell(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let mut textio = zelf.lock(vm)?; + if !textio.seekable { + return Err(new_unsupported_operation( + vm, + "underlying stream is not seekable".to_owned(), + )); + } + if !textio.telling { + return Err(vm.new_os_error("telling position disabled by next() call")); + } + textio.write_pending(vm)?; + drop(textio); + vm.call_method(zelf.as_object(), "flush", ())?; + let textio = zelf.lock(vm)?; + let pos = vm.call_method(&textio.buffer, "tell", ())?; + let (decoder, (dec_flags, next_input)) = match (&textio.decoder, &textio.snapshot) { + (Some(d), Some(s)) => (d, s), + _ => return Ok(pos), + }; + let pos = Offset::try_from_object(vm, pos)?; + let mut cookie = TextIOCookie { + start_pos: pos - next_input.len() as Offset, + dec_flags: *dec_flags, + ..Default::default() + }; + if textio.decoded_chars_used.bytes == 0 { + return Ok(cookie.build().to_pyobject(vm)); + } + let decoder_getstate = || { + let state = vm.call_method(decoder, "getstate", ())?; + parse_decoder_state(state, vm) + }; + let decoder_decode = |b: &[u8]| { + let decoded = vm.call_method(decoder, "decode", (vm.ctx.new_bytes(b.to_vec()),))?; + let decoded = check_decoded(decoded, vm)?; + Ok(Utf8size::len_pystr(&decoded)) + }; + let saved_state = vm.call_method(decoder, "getstate", ())?; + let mut num_to_skip = textio.decoded_chars_used; + let mut skip_bytes = (textio.b2cratio * num_to_skip.chars as f64) as isize; + let mut skip_back = 1; + while skip_bytes > 0 { + cookie.set_decoder_state(decoder, vm)?; + let input = &next_input.as_bytes()[..skip_bytes as usize]; + let n_decoded = decoder_decode(input)?; + if n_decoded.chars <= num_to_skip.chars { + let (dec_buffer, dec_flags) = decoder_getstate()?; + if dec_buffer.is_empty() { + cookie.dec_flags = dec_flags; + num_to_skip -= n_decoded; + break; + } + skip_bytes -= dec_buffer.len() as isize; + skip_back = 1; + } else { + skip_bytes -= skip_back; + skip_back *= 2; + } + } + if skip_bytes <= 0 { + skip_bytes = 0; + cookie.set_decoder_state(decoder, vm)?; + } + let skip_bytes = skip_bytes as usize; + + cookie.start_pos += skip_bytes as Offset; + cookie.set_num_to_skip(num_to_skip); + + if num_to_skip.chars != 0 { + let mut n_decoded = Utf8size::default(); + let mut input = next_input.as_bytes(); + input = &input[skip_bytes..]; + while !input.is_empty() { + let (byte1, rest) = input.split_at(1); + let n = decoder_decode(byte1)?; + n_decoded += n; + cookie.bytes_to_feed += 1; + let (dec_buffer, dec_flags) = decoder_getstate()?; + if dec_buffer.is_empty() && n_decoded.chars <= num_to_skip.chars { + cookie.start_pos += cookie.bytes_to_feed as Offset; + num_to_skip -= n_decoded; + cookie.dec_flags = dec_flags; + cookie.bytes_to_feed = 0; + n_decoded = Utf8size::default(); + } + if n_decoded.chars >= num_to_skip.chars { + break; + } + input = rest; + } + if input.is_empty() { + let decoded = + vm.call_method(decoder, "decode", (vm.ctx.new_bytes(vec![]), true))?; + let decoded = check_decoded(decoded, vm)?; + let final_decoded_chars = n_decoded.chars + decoded.char_len(); + cookie.need_eof = true; + if final_decoded_chars < num_to_skip.chars { + return Err(vm.new_os_error("can't reconstruct logical file position")); + } + } + } + vm.call_method(decoder, "setstate", (saved_state,))?; + cookie.set_num_to_skip(num_to_skip); + Ok(cookie.build().to_pyobject(vm)) + } + + #[pygetset] + fn name(&self, vm: &VirtualMachine) -> PyResult { + let buffer = self.lock(vm)?.buffer.clone(); + buffer.get_attr("name", vm) + } + + #[pygetset] + fn encoding(&self, vm: &VirtualMachine) -> PyResult<PyUtf8StrRef> { + Ok(self.lock(vm)?.encoding.clone()) + } + + #[pygetset] + fn errors(&self, vm: &VirtualMachine) -> PyResult<PyUtf8StrRef> { + Ok(self.lock(vm)?.errors.clone()) + } + + #[pymethod] + fn fileno(&self, vm: &VirtualMachine) -> PyResult { + let buffer = self.lock(vm)?.buffer.clone(); + vm.call_method(&buffer, "fileno", ()) + } + + #[pymethod] + fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let mut textio = self.lock(vm)?; + textio.check_closed(vm)?; + let decoder = textio + .decoder + .clone() + .ok_or_else(|| new_unsupported_operation(vm, "not readable".to_owned()))?; + + textio.write_pending(vm)?; + + let s = if let Some(mut remaining) = size.to_usize() { + let mut chunks = Vec::new(); + let mut chunks_bytes = 0; + loop { + if let Some((s, char_len)) = textio.get_decoded_chars(remaining, vm) { + chunks_bytes += s.byte_len(); + chunks.push(s); + remaining = remaining.saturating_sub(char_len); + } + if remaining == 0 { + break; + } + let eof = textio.read_chunk(remaining, vm)?; + if eof { + break; + } + } + if chunks.is_empty() { + vm.ctx.empty_str.to_owned() + } else if chunks.len() == 1 { + chunks.pop().unwrap() + } else { + let mut ret = Wtf8Buf::with_capacity(chunks_bytes); + for chunk in chunks { + ret.push_wtf8(chunk.as_wtf8()) + } + PyStr::from(ret).into_ref(&vm.ctx) + } + } else { + let bytes = vm.call_method(&textio.buffer, "read", ())?; + let decoded = vm.call_method(&decoder, "decode", (bytes, true))?; + let decoded = check_decoded(decoded, vm)?; + let ret = textio.take_decoded_chars(Some(decoded), vm); + textio.snapshot = None; + ret + }; + Ok(s) + } + + #[pymethod] + fn write(&self, obj: PyStrRef, vm: &VirtualMachine) -> PyResult<usize> { + let mut textio = self.lock(vm)?; + textio.check_closed(vm)?; + + let (encoder, encode_func) = textio + .encoder + .as_ref() + .ok_or_else(|| new_unsupported_operation(vm, "not writable".to_owned()))?; + + let char_len = obj.char_len(); + + let data = obj.as_wtf8(); + + let replace_nl = match textio.newline { + Newlines::Lf => Some("\n"), + Newlines::Cr => Some("\r"), + Newlines::Crlf => Some("\r\n"), + Newlines::Universal if cfg!(windows) => Some("\r\n"), + _ => None, + }; + let has_lf = (replace_nl.is_some() || textio.line_buffering) + && data.contains_code_point('\n'.into()); + let flush = textio.line_buffering && (has_lf || data.contains_code_point('\r'.into())); + let chunk = if let Some(replace_nl) = replace_nl { + if has_lf { + PyStr::from(data.replace("\n".as_ref(), replace_nl.as_ref())).into_ref(&vm.ctx) + } else { + obj + } + } else { + obj + }; + let chunk = if let Some(encode_func) = *encode_func { + encode_func(chunk) + } else { + let b = vm.call_method(encoder, "encode", (chunk.clone(),))?; + b.downcast::<PyBytes>() + .map(PendingWrite::Bytes) + .or_else(|obj| { + // TODO: not sure if encode() returning the str it was passed is officially + // supported or just a quirk of how the CPython code is written + if obj.is(&chunk) { + Ok(PendingWrite::Utf8(chunk)) + } else { + Err(vm.new_type_error(format!( + "encoder should return a bytes object, not '{}'", + obj.class().name() + ))) + } + })? + }; + if textio.pending.num_bytes > 0 + && textio.pending.num_bytes + chunk.as_bytes().len() > textio.chunk_size + { + let buffer = textio.buffer.clone(); + let pending = textio.pending.take(vm); + drop(textio); + vm.call_method(&buffer, "write", (pending,))?; + textio = self.lock(vm)?; + textio.check_closed(vm)?; + if textio.pending.num_bytes > 0 { + let buffer = textio.buffer.clone(); + let pending = textio.pending.take(vm); + drop(textio); + vm.call_method(&buffer, "write", (pending,))?; + textio = self.lock(vm)?; + textio.check_closed(vm)?; + } + } + textio.pending.push(chunk); + if textio.pending.num_bytes > 0 + && (flush || textio.write_through || textio.pending.num_bytes >= textio.chunk_size) + { + let buffer = textio.buffer.clone(); + let pending = textio.pending.take(vm); + drop(textio); + vm.call_method(&buffer, "write", (pending,))?; + textio = self.lock(vm)?; + textio.check_closed(vm)?; + } + if flush { + let _ = vm.call_method(&textio.buffer, "flush", ()); + } + + Ok(char_len) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult { + let mut textio = self.lock(vm)?; + flush_inner(&mut textio, vm) + } + + #[pymethod] + fn truncate( + zelf: PyRef<Self>, + pos: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + // Implementation follows _pyio.py TextIOWrapper.truncate + let mut textio = zelf.lock(vm)?; + flush_inner(&mut textio, vm)?; + let buffer = textio.buffer.clone(); + drop(textio); + + let pos = match pos.into_option() { + Some(p) => p, + None => vm.call_method(zelf.as_object(), "tell", ())?, + }; + vm.call_method(&buffer, "truncate", (pos,)) + } + + #[pymethod] + fn isatty(&self, vm: &VirtualMachine) -> PyResult { + let textio = self.lock(vm)?; + textio.check_closed(vm)?; + vm.call_method(&textio.buffer, "isatty", ()) + } + + #[pymethod] + fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let limit = size.to_usize(); + + let mut textio = self.lock(vm)?; + check_closed(&textio.buffer, vm)?; + + textio.write_pending(vm)?; + + #[derive(Clone)] + struct SlicedStr(PyStrRef, Range<usize>); + + impl SlicedStr { + #[inline] + fn byte_len(&self) -> usize { + self.1.len() + } + + #[inline] + fn char_len(&self) -> usize { + if self.is_full_slice() { + self.0.char_len() + } else { + self.slice().code_points().count() + } + } + + #[inline] + fn is_full_slice(&self) -> bool { + self.1.len() >= self.0.byte_len() + } + + #[inline] + fn slice(&self) -> &Wtf8 { + &self.0.as_wtf8()[self.1.clone()] + } + + #[inline] + fn slice_pystr(self, vm: &VirtualMachine) -> PyStrRef { + if self.is_full_slice() { + self.0 + } else { + // TODO: try to use Arc::get_mut() on the str? + PyStr::from(self.slice()).into_ref(&vm.ctx) + } + } + + fn utf8_len(&self) -> Utf8size { + Utf8size { + bytes: self.byte_len(), + chars: self.char_len(), + } + } + } + + let mut start; + let mut end_pos; + let mut offset_to_buffer; + let mut chunked = Utf8size::default(); + let mut remaining: Option<SlicedStr> = None; + let mut chunks = Vec::new(); + + let cur_line = 'outer: loop { + let decoded_chars = loop { + match textio.decoded_chars.as_ref() { + Some(s) if !s.is_empty() => break s, + _ => {} + } + let eof = textio.read_chunk(0, vm)?; + if eof { + textio.set_decoded_chars(None); + textio.snapshot = None; + start = Utf8size::default(); + end_pos = Utf8size::default(); + offset_to_buffer = Utf8size::default(); + break 'outer None; + } + }; + let line = match remaining.take() { + None => { + start = textio.decoded_chars_used; + offset_to_buffer = Utf8size::default(); + decoded_chars.clone() + } + Some(remaining) => { + assert_eq!(textio.decoded_chars_used.bytes, 0); + offset_to_buffer = remaining.utf8_len(); + let decoded_chars = decoded_chars.as_wtf8(); + let line = if remaining.is_full_slice() { + let mut line = remaining.0; + line.concat_in_place(decoded_chars, vm); + line + } else { + let remaining = remaining.slice(); + let mut s = + Wtf8Buf::with_capacity(remaining.len() + decoded_chars.len()); + s.push_wtf8(remaining); + s.push_wtf8(decoded_chars); + PyStr::from(s).into_ref(&vm.ctx) + }; + start = Utf8size::default(); + line + } + }; + let line_from_start = &line.as_wtf8()[start.bytes..]; + let nl_res = textio.newline.find_newline(line_from_start); + match nl_res { + Ok(p) | Err(p) => { + end_pos = start + Utf8size::len_str(&line_from_start[..p]); + if let Some(limit) = limit { + // original CPython logic: end_pos = start + limit - chunked + if chunked.chars + end_pos.chars >= limit { + end_pos = start + + Utf8size { + chars: limit - chunked.chars, + bytes: crate::common::str::codepoint_range_end( + line_from_start, + limit - chunked.chars, + ) + .unwrap(), + }; + break Some(line); + } + } + } + } + if nl_res.is_ok() { + break Some(line); + } + if end_pos.bytes > start.bytes { + let chunk = SlicedStr(line.clone(), start.bytes..end_pos.bytes); + chunked += chunk.utf8_len(); + chunks.push(chunk); + } + let line_len = line.byte_len(); + if end_pos.bytes < line_len { + remaining = Some(SlicedStr(line, end_pos.bytes..line_len)); + } + textio.set_decoded_chars(None); + }; + + let cur_line = cur_line.map(|line| { + textio.decoded_chars_used = end_pos - offset_to_buffer; + SlicedStr(line, start.bytes..end_pos.bytes) + }); + // don't need to care about chunked.chars anymore + let mut chunked = chunked.bytes; + if let Some(remaining) = remaining { + chunked += remaining.byte_len(); + chunks.push(remaining); + } + let line = if !chunks.is_empty() { + if let Some(cur_line) = cur_line { + chunked += cur_line.byte_len(); + chunks.push(cur_line); + } + let mut s = Wtf8Buf::with_capacity(chunked); + for chunk in chunks { + s.push_wtf8(chunk.slice()) + } + PyStr::from(s).into_ref(&vm.ctx) + } else if let Some(cur_line) = cur_line { + cur_line.slice_pystr(vm) + } else { + vm.ctx.empty_str.to_owned() + }; + Ok(line) + } + + #[pymethod] + fn close(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { + let buffer = zelf.lock(vm)?.buffer.clone(); + if file_closed(&buffer, vm)? { + return Ok(()); + } + // https://github.com/python/cpython/issues/142594 + // The file_closed() check above may have triggered a reentrant + // call to detach() via a custom `closed` property. + // If so, the buffer is now detached and we should return early. + if vm.is_none(&zelf.lock(vm)?.buffer) { + return Ok(()); + } + if zelf.finalizing.load(Ordering::Relaxed) { + // _dealloc_warn: delegate to buffer._dealloc_warn(source) + let _ = vm.call_method(&buffer, "_dealloc_warn", (zelf.as_object().to_owned(),)); + } + let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop); + let close_res = vm.call_method(&buffer, "close", ()).map(drop); + exception_chain(flush_res, close_res) + } + + #[pygetset] + fn closed(&self, vm: &VirtualMachine) -> PyResult { + let buffer = self.lock(vm)?.buffer.clone(); + buffer.get_attr("closed", vm) + } + + #[pygetset] + fn buffer(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.lock(vm)?.buffer.clone()) + } + + #[pymethod] + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) + } + + #[pymethod] + fn __reduce_ex__(zelf: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { + if zelf.class().is(TextIOWrapper::static_type()) { + return Err( + vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name())) + ); + } + let _ = proto; + reduce_ex_for_subclass(zelf, vm) + } + } + + fn parse_decoder_state(state: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyBytesRef, i32)> { + use crate::builtins::{PyTuple, int}; + let state_err = || vm.new_type_error("illegal decoder state"); + let state = state.downcast::<PyTuple>().map_err(|_| state_err())?; + match state.as_slice() { + [buf, flags] => { + let buf = buf.clone().downcast::<PyBytes>().map_err(|obj| { + vm.new_type_error(format!( + "illegal decoder state: the first item should be a bytes object, not '{}'", + obj.class().name() + )) + })?; + let flags = flags.downcast_ref::<int::PyInt>().ok_or_else(state_err)?; + let flags = flags.try_to_primitive(vm)?; + Ok((buf, flags)) + } + _ => Err(state_err()), + } + } + + impl TextIOData { + fn write_pending(&mut self, vm: &VirtualMachine) -> PyResult<()> { + if self.pending.num_bytes == 0 { + return Ok(()); + } + let data = self.pending.take(vm); + vm.call_method(&self.buffer, "write", (data,))?; + Ok(()) + } + + /// returns true on EOF + fn read_chunk(&mut self, size_hint: usize, vm: &VirtualMachine) -> PyResult<bool> { + let decoder = self + .decoder + .as_ref() + .ok_or_else(|| new_unsupported_operation(vm, "not readable".to_owned()))?; + + let dec_state = if self.telling { + let state = vm.call_method(decoder, "getstate", ())?; + Some(parse_decoder_state(state, vm)?) + } else { + None + }; + + let method = if self.has_read1 { "read1" } else { "read" }; + let size_hint = if size_hint > 0 { + (self.b2cratio.max(1.0) * size_hint as f64) as usize + } else { + size_hint + }; + let chunk_size = core::cmp::max(self.chunk_size, size_hint); + let input_chunk = vm.call_method(&self.buffer, method, (chunk_size,))?; + + let buf = ArgBytesLike::try_from_borrowed_object(vm, &input_chunk).map_err(|_| { + vm.new_type_error(format!( + "underlying {}() should have returned a bytes-like object, not '{}'", + method, + input_chunk.class().name() + )) + })?; + let nbytes = buf.borrow_buf().len(); + let eof = nbytes == 0; + let decoded = vm.call_method(decoder, "decode", (input_chunk, eof))?; + let decoded = check_decoded(decoded, vm)?; + + let char_len = decoded.char_len(); + self.b2cratio = if char_len > 0 { + nbytes as f64 / char_len as f64 + } else { + 0.0 + }; + let eof = if char_len > 0 { false } else { eof }; + self.set_decoded_chars(Some(decoded)); + + if let Some((dec_buffer, dec_flags)) = dec_state { + // TODO: inplace append to bytes when refcount == 1 + let mut next_input = dec_buffer.as_bytes().to_vec(); + next_input.extend_from_slice(&buf.borrow_buf()); + self.snapshot = Some((dec_flags, PyBytes::from(next_input).into_ref(&vm.ctx))); + } + + Ok(eof) + } + + fn check_closed(&self, vm: &VirtualMachine) -> PyResult<()> { + check_closed(&self.buffer, vm) + } + + /// returns str, str.char_len() (it might not be cached in the str yet but we calculate it + /// anyway in this method) + fn get_decoded_chars( + &mut self, + n: usize, + vm: &VirtualMachine, + ) -> Option<(PyStrRef, usize)> { + if n == 0 { + return None; + } + let decoded_chars = self.decoded_chars.as_ref()?; + let avail = &decoded_chars.as_wtf8()[self.decoded_chars_used.bytes..]; + if avail.is_empty() { + return None; + } + let avail_chars = decoded_chars.char_len() - self.decoded_chars_used.chars; + let (chars, chars_used) = if n >= avail_chars { + if self.decoded_chars_used.bytes == 0 { + (decoded_chars.clone(), avail_chars) + } else { + (PyStr::from(avail).into_ref(&vm.ctx), avail_chars) + } + } else { + let s = crate::common::str::get_codepoints(avail, 0..n); + (PyStr::from(s).into_ref(&vm.ctx), n) + }; + self.decoded_chars_used += Utf8size { + bytes: chars.byte_len(), + chars: chars_used, + }; + Some((chars, chars_used)) + } + + fn set_decoded_chars(&mut self, s: Option<PyStrRef>) { + self.decoded_chars = s; + self.decoded_chars_used = Utf8size::default(); + } + + fn take_decoded_chars( + &mut self, + append: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyStrRef { + let empty_str = || vm.ctx.empty_str.to_owned(); + let chars_pos = core::mem::take(&mut self.decoded_chars_used).bytes; + let decoded_chars = match core::mem::take(&mut self.decoded_chars) { + None => return append.unwrap_or_else(empty_str), + Some(s) if s.is_empty() => return append.unwrap_or_else(empty_str), + Some(s) => s, + }; + let append_len = append.as_ref().map_or(0, |s| s.byte_len()); + if append_len == 0 && chars_pos == 0 { + return decoded_chars; + } + // TODO: in-place editing of `str` when refcount == 1 + let decoded_chars_unused = &decoded_chars.as_wtf8()[chars_pos..]; + let mut s = Wtf8Buf::with_capacity(decoded_chars_unused.len() + append_len); + s.push_wtf8(decoded_chars_unused); + if let Some(append) = append { + s.push_wtf8(append.as_wtf8()) + } + PyStr::from(s).into_ref(&vm.ctx) + } + } + + impl Destructor for TextIOWrapper { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if let Some(wrapper) = zelf.downcast_ref::<TextIOWrapper>() { + wrapper.finalizing.store(true, Ordering::Relaxed); + } + iobase_finalize(zelf, vm); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } + + impl Representable for TextIOWrapper { + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let type_name = zelf.class().slot_name(); + let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) else { + return Err( + vm.new_runtime_error(format!("reentrant call inside {type_name}.__repr__")) + ); + }; + let Some(data) = zelf.data.lock() else { + // Reentrant call + return Ok(vm.ctx.new_str(Wtf8Buf::from(format!("<{type_name}>")))); + }; + let Some(data) = data.as_ref() else { + return Err(vm.new_value_error("I/O operation on uninitialized object")); + }; + + let mut result = Wtf8Buf::from(format!("<{type_name}")); + + // Add name if present + if let Ok(Some(name)) = vm.get_attribute_opt(data.buffer.clone(), "name") { + let name_repr = name.repr(vm)?; + result.push_wtf8(" name=".as_ref()); + result.push_wtf8(name_repr.as_wtf8()); + } + + // Add mode if present (prefer the wrapper's attribute) + let mode_obj = match vm.get_attribute_opt(zelf.as_object().to_owned(), "mode") { + Ok(Some(mode)) => Some(mode), + Ok(None) | Err(_) => match vm.get_attribute_opt(data.buffer.clone(), "mode") { + Ok(Some(mode)) => Some(mode), + _ => None, + }, + }; + if let Some(mode) = mode_obj { + let mode_repr = mode.repr(vm)?; + result.push_wtf8(" mode=".as_ref()); + result.push_wtf8(mode_repr.as_wtf8()); + } + + // Add encoding (always valid UTF-8) + result.push_wtf8(" encoding='".as_ref()); + result.push_wtf8(data.encoding.as_str().as_ref()); + result.push_wtf8("'>".as_ref()); + + Ok(vm.ctx.new_str(result)) + } + + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("repr() is overridden directly") + } + } + + impl Iterable for TextIOWrapper { + fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + check_closed(&zelf, vm)?; + Ok(zelf) + } + + fn iter(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + unreachable!("slot_iter is implemented") + } + } + + impl IterNext for TextIOWrapper { + fn slot_iternext(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + // Set telling = false during iteration (matches CPython behavior) + let textio_ref: PyRef<TextIOWrapper> = + zelf.downcast_ref::<TextIOWrapper>().unwrap().to_owned(); + { + let mut textio = textio_ref.lock(vm)?; + textio.telling = false; + } + + let line = vm.call_method(zelf, "readline", ())?; + + if !line.clone().try_to_bool(vm)? { + // Restore telling on StopIteration + let mut textio = textio_ref.lock(vm)?; + textio.snapshot = None; + textio.telling = textio.seekable; + Ok(PyIterReturn::StopIteration(None)) + } else { + Ok(PyIterReturn::Return(line)) + } + } + + fn next(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { + unreachable!("slot_iternext is implemented") + } + } + + #[pyattr] + #[pyclass(name)] + #[derive(Debug, PyPayload, Default)] + struct IncrementalNewlineDecoder { + // TODO: Traverse + data: PyThreadMutex<Option<IncrementalNewlineDecoderData>>, + } + + #[derive(Debug)] + struct IncrementalNewlineDecoderData { + decoder: PyObjectRef, + // currently this is used for nothing + // errors: PyObjectRef, + pendingcr: bool, + translate: bool, + seennl: SeenNewline, + } + + bitflags! { + #[derive(Debug, PartialEq, Eq, Copy, Clone)] + struct SeenNewline: u8 { + const LF = 1; + const CR = 2; + const CRLF = 4; + } + } + + impl DefaultConstructor for IncrementalNewlineDecoder {} + + #[derive(FromArgs)] + struct IncrementalNewlineDecoderArgs { + #[pyarg(any)] + decoder: PyObjectRef, + #[pyarg(any)] + translate: bool, + #[pyarg(any, default)] + errors: Option<PyObjectRef>, + } + + impl Initializer for IncrementalNewlineDecoder { + type Args = IncrementalNewlineDecoderArgs; + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let _ = args.errors; + let mut data = zelf.lock_opt(vm)?; + *data = Some(IncrementalNewlineDecoderData { + decoder: args.decoder, + translate: args.translate, + pendingcr: false, + seennl: SeenNewline::empty(), + }); + Ok(()) + } + } + + #[pyclass(with(Constructor, Initializer))] + impl IncrementalNewlineDecoder { + fn lock_opt( + &self, + vm: &VirtualMachine, + ) -> PyResult<PyThreadMutexGuard<'_, Option<IncrementalNewlineDecoderData>>> { + self.data + .lock_wrapped(|do_lock| vm.allow_threads(do_lock)) + .ok_or_else(|| vm.new_runtime_error("reentrant call inside nldecoder")) + } + + fn lock( + &self, + vm: &VirtualMachine, + ) -> PyResult<PyMappedThreadMutexGuard<'_, IncrementalNewlineDecoderData>> { + let lock = self.lock_opt(vm)?; + PyThreadMutexGuard::try_map(lock, |x| x.as_mut()) + .map_err(|_| vm.new_value_error("I/O operation on uninitialized nldecoder")) + } + + #[pymethod] + fn decode(&self, args: NewlineDecodeArgs, vm: &VirtualMachine) -> PyResult<PyStrRef> { + self.lock(vm)?.decode(args.input, args.r#final, vm) + } + + #[pymethod] + fn getstate(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, u64)> { + let data = self.lock(vm)?; + let (buffer, flag) = if vm.is_none(&data.decoder) { + (vm.ctx.new_bytes(vec![]).into(), 0) + } else { + vm.call_method(&data.decoder, "getstate", ())? + .try_to_ref::<PyTuple>(vm)? + .extract_tuple::<(PyObjectRef, u64)>(vm)? + }; + let flag = (flag << 1) | (data.pendingcr as u64); + Ok((buffer, flag)) + } + + #[pymethod] + fn setstate(&self, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + let mut data = self.lock(vm)?; + let (buffer, flag) = state.extract_tuple::<(PyObjectRef, u64)>(vm)?; + data.pendingcr = flag & 1 != 0; + if !vm.is_none(&data.decoder) { + vm.call_method(&data.decoder, "setstate", ((buffer, flag >> 1),))?; + } + Ok(()) + } + + #[pymethod] + fn reset(&self, vm: &VirtualMachine) -> PyResult<()> { + let mut data = self.lock(vm)?; + data.seennl = SeenNewline::empty(); + data.pendingcr = false; + if !vm.is_none(&data.decoder) { + vm.call_method(&data.decoder, "reset", ())?; + } + Ok(()) + } + + #[pygetset] + fn newlines(&self, vm: &VirtualMachine) -> PyResult { + let data = self.lock(vm)?; + Ok(match data.seennl.bits() { + 1 => "\n".to_pyobject(vm), + 2 => "\r".to_pyobject(vm), + 3 => ("\r", "\n").to_pyobject(vm), + 4 => "\r\n".to_pyobject(vm), + 5 => ("\n", "\r\n").to_pyobject(vm), + 6 => ("\r", "\r\n").to_pyobject(vm), + 7 => ("\r", "\n", "\r\n").to_pyobject(vm), + _ => vm.ctx.none(), + }) + } + } + + #[derive(FromArgs)] + struct NewlineDecodeArgs { + #[pyarg(any)] + input: PyObjectRef, + #[pyarg(any, default)] + r#final: bool, + } + + impl IncrementalNewlineDecoderData { + fn decode( + &mut self, + input: PyObjectRef, + final_: bool, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + let output = if vm.is_none(&self.decoder) { + input + } else { + vm.call_method(&self.decoder, "decode", (input, final_))? + }; + let orig_output: PyStrRef = output.try_into_value(vm)?; + // this being Cow::Owned means we need to allocate a new string + let mut output = Cow::Borrowed(orig_output.as_wtf8()); + if self.pendingcr && (final_ || !output.is_empty()) { + output.to_mut().insert(0, '\r'.into()); + self.pendingcr = false; + } + if !final_ && let Some(s) = output.strip_suffix("\r") { + output = Cow::Owned(s.to_owned()); + self.pendingcr = true; + } + + if output.is_empty() { + return Ok(vm.ctx.empty_str.to_owned()); + } + + if (self.seennl == SeenNewline::LF || self.seennl.is_empty()) + && !output.contains_code_point('\r'.into()) + { + if self.seennl.is_empty() && output.contains_code_point('\n'.into()) { + self.seennl.insert(SeenNewline::LF); + } + } else if !self.translate { + let output = output.as_bytes(); + let mut matches = memchr::memchr2_iter(b'\r', b'\n', output); + while !self.seennl.is_all() { + let Some(i) = matches.next() else { break }; + match output[i] { + b'\n' => self.seennl.insert(SeenNewline::LF), + // if c isn't \n, it can only be \r + _ if output.get(i + 1) == Some(&b'\n') => { + matches.next(); + self.seennl.insert(SeenNewline::CRLF); + } + _ => self.seennl.insert(SeenNewline::CR), + } + } + } else { + let bytes = output.as_bytes(); + let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes); + let mut new_string = Wtf8Buf::with_capacity(output.len()); + let mut last_modification_index = 0; + while let Some(cr_index) = matches.next() { + if bytes[cr_index] == b'\r' { + // skip copying the CR + let mut next_chunk_index = cr_index + 1; + if bytes.get(cr_index + 1) == Some(&b'\n') { + matches.next(); + self.seennl.insert(SeenNewline::CRLF); + // skip the LF too + next_chunk_index += 1; + } else { + self.seennl.insert(SeenNewline::CR); + } + new_string.push_wtf8(&output[last_modification_index..cr_index]); + new_string.push_char('\n'); + last_modification_index = next_chunk_index; + } else { + self.seennl.insert(SeenNewline::LF); + } + } + new_string.push_wtf8(&output[last_modification_index..]); + output = Cow::Owned(new_string); + } + + Ok(match output { + Cow::Borrowed(_) => orig_output, + Cow::Owned(s) => vm.ctx.new_str(s), + }) + } + } + + #[pyattr] + #[pyclass(name = "StringIO", base = _TextIOBase)] + #[derive(Debug)] + struct StringIO { + _base: _TextIOBase, + buffer: PyRwLock<BufferedIO>, + closed: AtomicCell<bool>, + } + + #[derive(FromArgs)] + struct StringIONewArgs { + #[pyarg(positional, optional)] + object: OptionalOption<PyStrRef>, + + // TODO: use this + #[pyarg(any, default)] + #[allow(dead_code)] + newline: Newlines, + } + + impl Constructor for StringIO { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + _base: Default::default(), + buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), + closed: AtomicCell::new(false), + }) + } + } + + impl Initializer for StringIO { + type Args = StringIONewArgs; + + #[allow(unused_variables)] + fn init( + zelf: PyRef<Self>, + Self::Args { object, newline }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<()> { + let raw_bytes = object + .flatten() + .map_or_else(Vec::new, |v| v.as_bytes().to_vec()); + *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); + Ok(()) + } + } + + impl StringIO { + fn buffer(&self, vm: &VirtualMachine) -> PyResult<PyRwLockWriteGuard<'_, BufferedIO>> { + if !self.closed.load() { + Ok(self.buffer.write()) + } else { + Err(io_closed_error(vm)) + } + } + } + + #[pyclass(flags(BASETYPE, HAS_DICT, HAS_WEAKREF), with(Constructor, Initializer))] + impl StringIO { + #[pymethod] + const fn readable(&self) -> bool { + true + } + + #[pymethod] + const fn writable(&self) -> bool { + true + } + + #[pymethod] + const fn seekable(&self) -> bool { + true + } + + #[pygetset] + fn closed(&self) -> bool { + self.closed.load() + } + + #[pymethod] + fn close(&self) { + self.closed.store(true); + } + + // write string to underlying vector + #[pymethod] + fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult<u64> { + let bytes = data.as_bytes(); + self.buffer(vm)? + .write(bytes) + .ok_or_else(|| vm.new_type_error("Error Writing String")) + } + + // return the entire contents of the underlying + #[pymethod] + fn getvalue(&self, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let bytes = self.buffer(vm)?.getvalue(); + Wtf8Buf::from_bytes(bytes).map_err(|_| vm.new_value_error("Error Retrieving Value")) + } + + // skip to the jth position + #[pymethod] + fn seek( + &self, + offset: PyObjectRef, + how: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<u64> { + self.buffer(vm)? + .seek(seekfrom(vm, offset, how)?) + .map_err(|err| os_err(vm, err)) + } + + // Read k bytes from the object and return. + // If k is undefined || k == -1, then we read all bytes until the end of the file. + // This also increments the stream position by the value of k + #[pymethod] + fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let data = self.buffer(vm)?.read(size.to_usize()).unwrap_or_default(); + + let value = Wtf8Buf::from_bytes(data) + .map_err(|_| vm.new_value_error("Error Retrieving Value"))?; + Ok(value) + } + + #[pymethod] + fn tell(&self, vm: &VirtualMachine) -> PyResult<u64> { + Ok(self.buffer(vm)?.tell()) + } + + #[pymethod] + fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + // TODO size should correspond to the number of characters, at the moments its the number of + // bytes. + let input = self.buffer(vm)?.readline(size.to_usize(), vm)?; + Wtf8Buf::from_bytes(input).map_err(|_| vm.new_value_error("Error Retrieving Value")) + } + + #[pymethod] + fn truncate(&self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult<usize> { + let mut buffer = self.buffer(vm)?; + let pos = pos.try_usize(vm)?; + Ok(buffer.truncate(pos)) + } + + #[pygetset] + const fn line_buffering(&self) -> bool { + false + } + + #[pymethod] + fn __getstate__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let buffer = zelf.buffer(vm)?; + let content = Wtf8Buf::from_bytes(buffer.getvalue()) + .map_err(|_| vm.new_value_error("Error Retrieving Value"))?; + let pos = buffer.tell(); + drop(buffer); + + // Get __dict__ if it exists and is non-empty + let dict_obj: PyObjectRef = match zelf.as_object().dict() { + Some(d) if !d.is_empty() => d.into(), + _ => vm.ctx.none(), + }; + + // Return (content, newline, position, dict) + // TODO: store actual newline setting when it's implemented + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_str(content).into(), + vm.ctx.new_str("\n").into(), + vm.ctx.new_int(pos).into(), + dict_obj, + ])) + } + + #[pymethod] + fn __setstate__(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + // Check closed state first (like CHECK_CLOSED) + if zelf.closed.load() { + return Err(vm.new_value_error("__setstate__ on closed file")); + } + if state.len() != 4 { + return Err(vm.new_type_error(format!( + "__setstate__ argument should be 4-tuple, got {}", + state.len() + ))); + } + + let content: PyStrRef = state[0].clone().try_into_value(vm)?; + // state[1] is newline - TODO: use when newline handling is implemented + let pos: u64 = state[2].clone().try_into_value(vm)?; + let dict = &state[3]; + + // Set content and position + let raw_bytes = content.as_bytes().to_vec(); + let mut buffer = zelf.buffer.write(); + *buffer = BufferedIO::new(Cursor::new(raw_bytes)); + buffer + .seek(SeekFrom::Start(pos)) + .map_err(|err| os_err(vm, err))?; + drop(buffer); + + // Set __dict__ if provided + if !vm.is_none(dict) { + let dict_ref: PyRef<PyDict> = dict.clone().try_into_value(vm)?; + if let Some(obj_dict) = zelf.as_object().dict() { + obj_dict.clear(); + for (key, value) in dict_ref.into_iter() { + obj_dict.set_item(&*key, value, vm)?; + } + } + } + + Ok(()) + } + } + + #[derive(FromArgs)] + struct BytesIOArgs { + #[pyarg(any, optional)] + initial_bytes: OptionalArg<Option<ArgBytesLike>>, + } + + #[pyattr] + #[pyclass(name = "BytesIO", base = _BufferedIOBase)] + #[derive(Debug)] + struct BytesIO { + _base: _BufferedIOBase, + buffer: PyRwLock<BufferedIO>, + closed: AtomicCell<bool>, + exports: AtomicCell<usize>, + } + + impl Constructor for BytesIO { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + _base: Default::default(), + buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), + closed: AtomicCell::new(false), + exports: AtomicCell::new(0), + }) + } + } + + impl Initializer for BytesIO { + type Args = BytesIOArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + if zelf.exports.load() > 0 { + return Err( + vm.new_buffer_error("Existing exports of data: object cannot be re-sized") + ); + } + + let raw_bytes = args + .initial_bytes + .flatten() + .map_or_else(Vec::new, |input| input.borrow_buf().to_vec()); + *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); + Ok(()) + } + } + + impl BytesIO { + fn buffer(&self, vm: &VirtualMachine) -> PyResult<PyRwLockWriteGuard<'_, BufferedIO>> { + if !self.closed.load() { + Ok(self.buffer.write()) + } else { + Err(io_closed_error(vm)) + } + } + } + + #[pyclass( + flags(BASETYPE, HAS_DICT, HAS_WEAKREF), + with(PyRef, Constructor, Initializer) + )] + impl BytesIO { + #[pymethod] + const fn readable(&self) -> bool { + true + } + + #[pymethod] + const fn writable(&self) -> bool { + true + } + + #[pymethod] + const fn seekable(&self) -> bool { + true + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.closed.load() { + Err(io_closed_error(vm)) + } else { + Ok(()) + } + } + + #[pymethod] + fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<u64> { + let mut buffer = self.try_resizable(vm)?; + data.with_ref(|b| buffer.write(b)) + .ok_or_else(|| vm.new_type_error("Error Writing Bytes")) + } + + // Retrieves the entire bytes object value from the underlying buffer + #[pymethod] + fn getvalue(&self, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + let bytes = self.buffer(vm)?.getvalue(); + Ok(vm.ctx.new_bytes(bytes)) + } + + // Takes an integer k (bytes) and returns them from the underlying buffer + // If k is undefined || k == -1, then we read all bytes until the end of the file. + // This also increments the stream position by the value of k + #[pymethod] + #[pymethod(name = "read1")] + fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let buf = self.buffer(vm)?.read(size.to_usize()).unwrap_or_default(); + Ok(buf) + } + + #[pymethod] + fn readinto(&self, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<usize> { + let mut buf = self.buffer(vm)?; + let ret = buf + .cursor + .read(&mut obj.borrow_buf_mut()) + .map_err(|_| vm.new_value_error("Error readinto from Take"))?; + + Ok(ret) + } + + //skip to the jth position + #[pymethod] + fn seek( + &self, + offset: PyObjectRef, + how: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<u64> { + let seek_from = seekfrom(vm, offset, how)?; + let mut buffer = self.buffer(vm)?; + + // Handle negative positions by clamping to 0 + match seek_from { + SeekFrom::Current(offset) if offset < 0 => { + let current = buffer.tell(); + let new_pos = current.saturating_add_signed(offset); + buffer + .seek(SeekFrom::Start(new_pos)) + .map_err(|err| os_err(vm, err)) + } + _ => buffer.seek(seek_from).map_err(|err| os_err(vm, err)), + } + } + + #[pymethod] + fn tell(&self, vm: &VirtualMachine) -> PyResult<u64> { + Ok(self.buffer(vm)?.tell()) + } + + #[pymethod] + fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + self.buffer(vm)?.readline(size.to_usize(), vm) + } + + #[pymethod] + fn truncate(&self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult<usize> { + if self.closed.load() { + return Err(io_closed_error(vm)); + } + let mut buffer = self.try_resizable(vm)?; + let pos = pos.try_usize(vm)?; + Ok(buffer.truncate(pos)) + } + + #[pygetset] + fn closed(&self) -> bool { + self.closed.load() + } + + #[pymethod] + fn close(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.exports.load() > 0 { + return Err( + vm.new_buffer_error("Existing exports of data: object cannot be closed") + ); + } + self.closed.store(true); + Ok(()) + } + + #[pymethod] + fn __getstate__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let buffer = zelf.buffer(vm)?; + let content = buffer.getvalue(); + let pos = buffer.tell(); + drop(buffer); + + // Get __dict__ if it exists and is non-empty + let dict_obj: PyObjectRef = match zelf.as_object().dict() { + Some(d) if !d.is_empty() => d.into(), + _ => vm.ctx.none(), + }; + + // Return (content, position, dict) + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytes(content).into(), + vm.ctx.new_int(pos).into(), + dict_obj, + ])) + } + + #[pymethod] + fn __setstate__(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + if zelf.closed.load() { + return Err(vm.new_value_error("__setstate__ on closed file")); + } + if state.len() != 3 { + return Err(vm.new_type_error(format!( + "__setstate__ argument should be 3-tuple, got {}", + state.len() + ))); + } + + let content: PyBytesRef = state[0].clone().try_into_value(vm)?; + let pos: u64 = state[1].clone().try_into_value(vm)?; + let dict = &state[2]; + + // Check exports and set content (like CHECK_EXPORTS) + let mut buffer = zelf.try_resizable(vm)?; + *buffer = BufferedIO::new(Cursor::new(content.as_bytes().to_vec())); + buffer + .seek(SeekFrom::Start(pos)) + .map_err(|err| os_err(vm, err))?; + drop(buffer); + + // Set __dict__ if provided + if !vm.is_none(dict) { + let dict_ref: PyRef<PyDict> = dict.clone().try_into_value(vm)?; + if let Some(obj_dict) = zelf.as_object().dict() { + obj_dict.clear(); + for (key, value) in dict_ref.into_iter() { + obj_dict.set_item(&*key, value, vm)?; + } + } + } + + Ok(()) + } + + #[pymethod] + fn isatty(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.closed() { + return Err(io_closed_error(vm)); + } + + Ok(false) + } + } + + #[pyclass] + impl PyRef<BytesIO> { + #[pymethod] + fn getbuffer(self, vm: &VirtualMachine) -> PyResult<PyMemoryView> { + if self.closed.load() { + return Err(vm.new_value_error("I/O operation on closed file.")); + } + let len = self.buffer.read().cursor.get_ref().len(); + let buffer = PyBuffer::new( + self.into(), + BufferDescriptor::simple(len, false), + &BYTES_IO_BUFFER_METHODS, + ); + let view = PyMemoryView::from_buffer(buffer, vm)?; + Ok(view) + } + } + + static BYTES_IO_BUFFER_METHODS: BufferMethods = BufferMethods { + obj_bytes: |buffer| { + let zelf = buffer.obj_as::<BytesIO>(); + PyRwLockReadGuard::map(zelf.buffer.read(), |x| x.cursor.get_ref().as_slice()).into() + }, + obj_bytes_mut: |buffer| { + let zelf = buffer.obj_as::<BytesIO>(); + PyRwLockWriteGuard::map(zelf.buffer.write(), |x| x.cursor.get_mut().as_mut_slice()) + .into() + }, + + release: |buffer| { + buffer.obj_as::<BytesIO>().exports.fetch_sub(1); + }, + + retain: |buffer| { + buffer.obj_as::<BytesIO>().exports.fetch_add(1); + }, + }; + + impl BufferResizeGuard for BytesIO { + type Resizable<'a> = PyRwLockWriteGuard<'a, BufferedIO>; + + fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>> { + let w = self.buffer.write(); + (self.exports.load() == 0).then_some(w) + } + } + + #[repr(u8)] + #[derive(Debug)] + enum FileMode { + Read = b'r', + Write = b'w', + Exclusive = b'x', + Append = b'a', + } + + #[repr(u8)] + #[derive(Debug)] + enum EncodeMode { + Text = b't', + Bytes = b'b', + } + + #[derive(Debug)] + struct Mode { + file: FileMode, + encode: EncodeMode, + plus: bool, + } + + impl core::str::FromStr for Mode { + type Err = ParseModeError; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + let mut file = None; + let mut encode = None; + let mut plus = false; + macro_rules! set_mode { + ($var:ident, $mode:path, $err:ident) => {{ + match $var { + Some($mode) => return Err(ParseModeError::InvalidMode), + Some(_) => return Err(ParseModeError::$err), + None => $var = Some($mode), + } + }}; + } + + for ch in s.chars() { + match ch { + '+' => { + if plus { + return Err(ParseModeError::InvalidMode); + } + plus = true + } + 't' => set_mode!(encode, EncodeMode::Text, MultipleEncode), + 'b' => set_mode!(encode, EncodeMode::Bytes, MultipleEncode), + 'r' => set_mode!(file, FileMode::Read, MultipleFile), + 'a' => set_mode!(file, FileMode::Append, MultipleFile), + 'w' => set_mode!(file, FileMode::Write, MultipleFile), + 'x' => set_mode!(file, FileMode::Exclusive, MultipleFile), + _ => return Err(ParseModeError::InvalidMode), + } + } + + let file = file.ok_or(ParseModeError::NoFile)?; + let encode = encode.unwrap_or(EncodeMode::Text); + + Ok(Self { file, encode, plus }) + } + } + + impl Mode { + const fn rawmode(&self) -> &'static str { + match (&self.file, self.plus) { + (FileMode::Read, true) => "rb+", + (FileMode::Read, false) => "rb", + (FileMode::Write, true) => "wb+", + (FileMode::Write, false) => "wb", + (FileMode::Exclusive, true) => "xb+", + (FileMode::Exclusive, false) => "xb", + (FileMode::Append, true) => "ab+", + (FileMode::Append, false) => "ab", + } + } + } + + enum ParseModeError { + InvalidMode, + MultipleFile, + MultipleEncode, + NoFile, + } + + impl ParseModeError { + fn error_msg(&self, mode_string: &str) -> String { + match self { + Self::InvalidMode => format!("invalid mode: '{mode_string}'"), + Self::MultipleFile => { + "must have exactly one of create/read/write/append mode".to_owned() + } + Self::MultipleEncode => "can't have text and binary mode at once".to_owned(), + Self::NoFile => { + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_owned() + } + } + } + } + + #[derive(FromArgs)] + struct IoOpenArgs { + file: PyObjectRef, + #[pyarg(any, optional)] + mode: OptionalArg<PyUtf8StrRef>, + #[pyarg(flatten)] + opts: OpenArgs, + } + + #[pyfunction] + fn open(args: IoOpenArgs, vm: &VirtualMachine) -> PyResult { + io_open( + args.file, + args.mode.as_ref().into_option().map(|s| s.as_str()), + args.opts, + vm, + ) + } + + #[pyfunction] + fn open_code(file: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // TODO: lifecycle hooks or something? + io_open(file, Some("rb"), OpenArgs::default(), vm) + } + + #[derive(FromArgs)] + pub struct OpenArgs { + #[pyarg(any, default = -1)] + pub buffering: isize, + #[pyarg(any, default)] + pub encoding: Option<PyUtf8StrRef>, + #[pyarg(any, default)] + pub errors: Option<PyUtf8StrRef>, + #[pyarg(any, default)] + pub newline: Option<PyUtf8StrRef>, + #[pyarg(any, default = true)] + pub closefd: bool, + #[pyarg(any, default)] + pub opener: Option<PyObjectRef>, + } + + impl Default for OpenArgs { + fn default() -> Self { + Self { + buffering: -1, + encoding: None, + errors: None, + newline: None, + closefd: true, + opener: None, + } + } + } + + /// Reinit per-object IO buffer locks on std streams after `fork()`. + /// + /// # Safety + /// + /// Must only be called from the single-threaded child process immediately + /// after `fork()`, before any other thread is created. + #[cfg(all(unix, feature = "threading"))] + pub unsafe fn reinit_std_streams_after_fork(vm: &VirtualMachine) { + for name in ["stdin", "stdout", "stderr"] { + let Ok(stream) = vm.sys_module.get_attr(name, vm) else { + continue; + }; + reinit_io_locks(&stream); + } + } + + #[cfg(all(unix, feature = "threading"))] + fn reinit_io_locks(obj: &PyObject) { + use crate::common::lock::reinit_thread_mutex_after_fork; + + if let Some(tio) = obj.downcast_ref::<TextIOWrapper>() { + unsafe { reinit_thread_mutex_after_fork(&tio.data) }; + if let Some(guard) = tio.data.lock() + && let Some(ref data) = *guard + { + if let Some(ref decoder) = data.decoder { + reinit_io_locks(decoder); + } + reinit_io_locks(&data.buffer); + } + return; + } + if let Some(nl) = obj.downcast_ref::<IncrementalNewlineDecoder>() { + unsafe { reinit_thread_mutex_after_fork(&nl.data) }; + return; + } + if let Some(br) = obj.downcast_ref::<BufferedReader>() { + unsafe { reinit_thread_mutex_after_fork(&br.data) }; + return; + } + if let Some(bw) = obj.downcast_ref::<BufferedWriter>() { + unsafe { reinit_thread_mutex_after_fork(&bw.data) }; + return; + } + if let Some(brw) = obj.downcast_ref::<BufferedRandom>() { + unsafe { reinit_thread_mutex_after_fork(&brw.data) }; + return; + } + if let Some(brw) = obj.downcast_ref::<BufferedRWPair>() { + unsafe { reinit_thread_mutex_after_fork(&brw.read.data) }; + unsafe { reinit_thread_mutex_after_fork(&brw.write.data) }; + } + } + + pub fn io_open( + file: PyObjectRef, + mode: Option<&str>, + opts: OpenArgs, + vm: &VirtualMachine, + ) -> PyResult { + // mode is optional: 'rt' is the default mode (open from reading text) + let mode_string = mode.unwrap_or("r"); + let mode = mode_string + .parse::<Mode>() + .map_err(|e| vm.new_value_error(e.error_msg(mode_string)))?; + + if let EncodeMode::Bytes = mode.encode { + let msg = if opts.encoding.is_some() { + Some("binary mode doesn't take an encoding argument") + } else if opts.errors.is_some() { + Some("binary mode doesn't take an errors argument") + } else if opts.newline.is_some() { + Some("binary mode doesn't take a newline argument") + } else { + None + }; + if let Some(msg) = msg { + return Err(vm.new_value_error(msg)); + } + } + + // check file descriptor validity + #[cfg(all(unix, feature = "host_env"))] + if let Ok(crate::ospath::OsPathOrFd::Fd(fd)) = file.clone().try_into_value(vm) { + nix::fcntl::fcntl(fd, nix::fcntl::F_GETFD).map_err(|_| vm.new_last_errno_error())?; + } + + // Construct a RawIO (subclass of RawIOBase) + // On Windows, use _WindowsConsoleIO for console handles. + // This is subsequently consumed by a Buffered Class. + #[cfg(all(feature = "host_env", windows))] + let is_console = super::winconsoleio::pyio_get_console_type(&file, vm) != '\0'; + #[cfg(not(all(feature = "host_env", windows)))] + let is_console = false; + + let file_io_class: &Py<PyType> = { + cfg_if::cfg_if! { + if #[cfg(all(feature = "host_env", windows))] { + if is_console { + Some(super::winconsoleio::WindowsConsoleIO::static_type()) + } else { + Some(super::fileio::FileIO::static_type()) + } + } else if #[cfg(feature = "host_env")] { + Some(super::fileio::FileIO::static_type()) + } else { + None + } + } + } + .ok_or_else(|| { + new_unsupported_operation( + vm, + "Couldn't get FileIO, io.open likely isn't supported on your platform".to_owned(), + ) + })?; + let raw = PyType::call( + file_io_class, + (file, mode.rawmode(), opts.closefd, opts.opener).into_args(vm), + vm, + )?; + + let isatty = opts.buffering < 0 && { + let atty = vm.call_method(&raw, "isatty", ())?; + bool::try_from_object(vm, atty)? + }; + + // Warn if line buffering is requested in binary mode + if opts.buffering == 1 && matches!(mode.encode, EncodeMode::Bytes) { + crate::stdlib::_warnings::warn( + vm.ctx.exceptions.runtime_warning, + "line buffering (buffering=1) isn't supported in binary mode, the default buffer size will be used".to_owned(), + 1, + vm, + )?; + } + + let line_buffering = opts.buffering == 1 || isatty; + + let buffering = if opts.buffering < 0 || opts.buffering == 1 { + DEFAULT_BUFFER_SIZE + } else { + opts.buffering as usize + }; + + if buffering == 0 { + let ret = match mode.encode { + EncodeMode::Text => { + let _ = vm.call_method(&raw, "close", ()); + Err(vm.new_value_error("can't have unbuffered text I/O")) + } + EncodeMode::Bytes => Ok(raw), + }; + return ret; + } + + let cls = if mode.plus { + BufferedRandom::static_type() + } else if let FileMode::Read = mode.file { + BufferedReader::static_type() + } else { + BufferedWriter::static_type() + }; + let buffered = PyType::call(cls, (raw, buffering).into_args(vm), vm)?; + + match mode.encode { + EncodeMode::Text => { + let encoding = if is_console && opts.encoding.is_none() { + // Console IO always uses utf-8 + Some(PyUtf8Str::from("utf-8").into_ref(&vm.ctx)) + } else { + match opts.encoding { + Some(enc) => Some(enc), + None => { + let encoding = + text_encoding(vm.ctx.none(), OptionalArg::Present(2), vm)?; + Some(PyUtf8StrRef::try_from_object(vm, encoding.into())?) + } + } + }; + let tio = TextIOWrapper::static_type(); + let wrapper = PyType::call( + tio, + ( + buffered.clone(), + encoding, + opts.errors, + opts.newline, + line_buffering, + ) + .into_args(vm), + vm, + ) + .inspect_err(|_err| { + let _ = vm.call_method(&buffered, "close", ()); + })?; + wrapper.set_attr("mode", vm.new_pyobj(mode_string), vm)?; + Ok(wrapper) + } + EncodeMode::Bytes => Ok(buffered), + } + } + + fn create_unsupported_operation(ctx: &Context) -> PyTypeRef { + use crate::builtins::type_::PyAttributes; + use crate::types::PyTypeSlots; + + let mut attrs = PyAttributes::default(); + attrs.insert(identifier!(ctx, __module__), ctx.new_str("io").into()); + + PyType::new_heap( + "UnsupportedOperation", + vec![ + ctx.exceptions.os_error.to_owned(), + ctx.exceptions.value_error.to_owned(), + ], + attrs, + PyTypeSlots::heap_default(), + ctx.types.type_type.to_owned(), + ctx, + ) + .unwrap() + } + + pub fn unsupported_operation() -> &'static Py<PyType> { + rustpython_common::static_cell! { + static CELL: PyTypeRef; + } + CELL.get_or_init(|| create_unsupported_operation(Context::genesis())) + } + + #[pyfunction] + fn text_encoding( + encoding: PyObjectRef, + stacklevel: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + if vm.is_none(&encoding) { + let encoding = if vm.state.config.settings.utf8_mode > 0 { + "utf-8" + } else { + "locale" + }; + if vm.state.config.settings.warn_default_encoding { + let mut stacklevel = stacklevel.unwrap_or(2); + if stacklevel > 1 + && let Some(frame) = vm.current_frame() + && let Some(stdlib_dir) = vm.state.config.paths.stdlib_dir.as_deref() + { + let path = frame.code.source_path().as_str(); + if !path.starts_with(stdlib_dir) { + stacklevel = stacklevel.saturating_sub(1); + } + } + let stacklevel = usize::try_from(stacklevel).unwrap_or(0); + crate::stdlib::_warnings::warn( + vm.ctx.exceptions.encoding_warning, + "'encoding' argument not specified".to_owned(), + stacklevel, + vm, + )?; + } + return Ok(vm.ctx.new_str(encoding)); + } + encoding.try_into_value(vm) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn test_buffered_read() { + let data = vec![1, 2, 3, 4]; + let bytes = None; + let mut buffered = BufferedIO { + cursor: Cursor::new(data.clone()), + }; + + assert_eq!(buffered.read(bytes).unwrap(), data); + } + + #[test] + fn test_buffered_seek() { + let data = vec![1, 2, 3, 4]; + let count: u64 = 2; + let mut buffered = BufferedIO { + cursor: Cursor::new(data), + }; + + assert_eq!(buffered.seek(SeekFrom::Start(count)).unwrap(), count); + assert_eq!(buffered.read(Some(count as usize)).unwrap(), vec![3, 4]); + } + + #[test] + fn test_buffered_value() { + let data = vec![1, 2, 3, 4]; + let buffered = BufferedIO { + cursor: Cursor::new(data.clone()), + }; + + assert_eq!(buffered.getvalue(), data); + } + } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + // Call auto-generated initialization first + __module_exec(vm, module); + + // Initialize FileIO types (requires host_env for filesystem access) + #[cfg(feature = "host_env")] + super::fileio::module_exec(vm, module)?; + + // Initialize WindowsConsoleIO type (Windows only) + #[cfg(all(feature = "host_env", windows))] + super::winconsoleio::module_exec(vm, module)?; + + let unsupported_operation = unsupported_operation().to_owned(); + extend_module!(vm, module, { + "UnsupportedOperation" => unsupported_operation, + "BlockingIOError" => vm.ctx.exceptions.blocking_io_error.to_owned(), + }); + Ok(()) + } +} +// FileIO requires host environment for filesystem access +#[cfg(feature = "host_env")] +#[pymodule] +mod fileio { + use super::{_io::*, Offset, iobase_finalize}; + use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, + builtins::{PyBaseExceptionRef, PyUtf8Str, PyUtf8StrRef}, + common::{crt_fd, wtf8::Wtf8Buf}, + convert::{IntoPyException, ToPyException}, + exceptions::OSErrorBuilder, + function::{ArgBytesLike, ArgMemoryBuffer, OptionalArg, OptionalOption}, + ospath::{OsPath, OsPathOrFd}, + stdlib::os, + types::{Constructor, DefaultConstructor, Destructor, Initializer, Representable}, + }; + use crossbeam_utils::atomic::AtomicCell; + use std::io::Read; + + bitflags::bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + struct Mode: u8 { + const CREATED = 0b0001; + const READABLE = 0b0010; + const WRITABLE = 0b0100; + const APPENDING = 0b1000; + } + } + + enum ModeError { + Invalid, + BadRwa, + } + + impl ModeError { + fn error_msg(&self, mode_str: &str) -> String { + match self { + Self::Invalid => format!("invalid mode: {mode_str}"), + Self::BadRwa => { + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_owned() + } + } + } + } + + fn compute_mode(mode_str: &str) -> Result<(Mode, i32), ModeError> { + let mut flags = 0; + let mut plus = false; + let mut rwa = false; + let mut mode = Mode::empty(); + for c in mode_str.bytes() { + match c { + b'x' => { + if rwa { + return Err(ModeError::BadRwa); + } + rwa = true; + mode.insert(Mode::WRITABLE | Mode::CREATED); + flags |= libc::O_EXCL | libc::O_CREAT; + } + b'r' => { + if rwa { + return Err(ModeError::BadRwa); + } + rwa = true; + mode.insert(Mode::READABLE); + } + b'w' => { + if rwa { + return Err(ModeError::BadRwa); + } + rwa = true; + mode.insert(Mode::WRITABLE); + flags |= libc::O_CREAT | libc::O_TRUNC; + } + b'a' => { + if rwa { + return Err(ModeError::BadRwa); + } + rwa = true; + mode.insert(Mode::WRITABLE | Mode::APPENDING); + flags |= libc::O_APPEND | libc::O_CREAT; + } + b'+' => { + if plus { + return Err(ModeError::BadRwa); + } + plus = true; + mode.insert(Mode::READABLE | Mode::WRITABLE); + } + b'b' => {} + _ => return Err(ModeError::Invalid), + } + } + + if !rwa { + return Err(ModeError::BadRwa); + } + + if mode.contains(Mode::READABLE | Mode::WRITABLE) { + flags |= libc::O_RDWR + } else if mode.contains(Mode::READABLE) { + flags |= libc::O_RDONLY + } else { + flags |= libc::O_WRONLY + } + + #[cfg(windows)] + { + flags |= libc::O_BINARY | libc::O_NOINHERIT; + } + #[cfg(unix)] + { + flags |= libc::O_CLOEXEC + } + + Ok((mode, flags as _)) + } + + #[pyattr] + #[pyclass(module = "_io", name, base = _RawIOBase)] + #[derive(Debug)] + pub(super) struct FileIO { + _base: _RawIOBase, + fd: AtomicCell<i32>, + closefd: AtomicCell<bool>, + mode: AtomicCell<Mode>, + seekable: AtomicCell<Option<bool>>, + blksize: AtomicCell<i64>, + finalizing: AtomicCell<bool>, + } + + #[derive(FromArgs)] + pub struct FileIOArgs { + #[pyarg(positional)] + name: PyObjectRef, + #[pyarg(any, default)] + mode: Option<PyUtf8StrRef>, + #[pyarg(any, default = true)] + closefd: bool, + #[pyarg(any, default)] + opener: Option<PyObjectRef>, + } + + impl Default for FileIO { + fn default() -> Self { + Self { + _base: Default::default(), + fd: AtomicCell::new(-1), + closefd: AtomicCell::new(true), + mode: AtomicCell::new(Mode::empty()), + seekable: AtomicCell::new(None), + blksize: AtomicCell::new(super::DEFAULT_BUFFER_SIZE as _), + finalizing: AtomicCell::new(false), + } + } + } + + impl DefaultConstructor for FileIO {} + + impl Initializer for FileIO { + type Args = FileIOArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // TODO: let atomic_flag_works + let name = args.name; + // Check if bool is used as file descriptor + if name.class().is(vm.ctx.types.bool_type) { + crate::stdlib::_warnings::warn( + vm.ctx.exceptions.runtime_warning, + "bool is used as a file descriptor".to_owned(), + 1, + vm, + )?; + } + let arg_fd = if let Some(i) = name.downcast_ref::<crate::builtins::PyInt>() { + let fd = i.try_to_primitive(vm)?; + if fd < 0 { + return Err(vm.new_value_error("negative file descriptor")); + } + Some(fd) + } else { + None + }; + + let mode_obj = args + .mode + .unwrap_or_else(|| PyUtf8Str::from("rb").into_ref(&vm.ctx)); + let mode_str = mode_obj.as_str(); + let (mode, flags) = + compute_mode(mode_str).map_err(|e| vm.new_value_error(e.error_msg(mode_str)))?; + zelf.mode.store(mode); + + let (fd, filename) = if let Some(fd) = arg_fd { + zelf.closefd.store(args.closefd); + (fd, None) + } else { + zelf.closefd.store(true); + if !args.closefd { + return Err(vm.new_value_error("Cannot use closefd=False with file name")); + } + + if let Some(opener) = args.opener { + let fd = opener.call((name.clone(), flags), vm)?; + if !fd.fast_isinstance(vm.ctx.types.int_type) { + return Err(vm.new_type_error("expected integer from opener")); + } + let fd = i32::try_from_object(vm, fd)?; + if fd < 0 { + return Err(vm.new_value_error(format!("opener returned {fd}"))); + } + (fd, None) + } else { + let path = OsPath::try_from_fspath(name.clone(), vm)?; + #[cfg(any(unix, target_os = "wasi"))] + let fd = crt_fd::open(&path.clone().into_cstring(vm)?, flags, 0o666); + #[cfg(windows)] + let fd = crt_fd::wopen(&path.to_wide_cstring(vm)?, flags, 0o666); + let filename = OsPathOrFd::Path(path); + match fd { + Ok(fd) => (fd.into_raw(), Some(filename)), + Err(e) => { + return Err(OSErrorBuilder::with_filename_from_errno(&e, filename, vm)); + } + } + } + }; + let fd_is_own = arg_fd.is_none(); + zelf.fd.store(fd); + let fd = unsafe { crt_fd::Borrowed::borrow_raw(fd) }; + let filename = filename.unwrap_or(OsPathOrFd::Fd(fd)); + + // TODO: _Py_set_inheritable + + let fd_fstat = crate::common::fileutils::fstat(fd); + + #[cfg(windows)] + { + if let Err(err) = fd_fstat { + // If the fd is invalid, prevent destructor from trying to close it + if err.raw_os_error() + == Some(windows_sys::Win32::Foundation::ERROR_INVALID_HANDLE as i32) + { + zelf.fd.store(-1); + } + return Err(OSErrorBuilder::with_filename(&err, filename, vm)); + } + } + #[cfg(any(unix, target_os = "wasi"))] + { + match fd_fstat { + Ok(status) => { + if (status.st_mode & libc::S_IFMT) == libc::S_IFDIR { + // If fd was passed by user, don't close it on error + if !fd_is_own { + zelf.fd.store(-1); + } + let err = std::io::Error::from_raw_os_error(libc::EISDIR); + return Err(OSErrorBuilder::with_filename(&err, filename, vm)); + } + // Store st_blksize for _blksize property + if status.st_blksize > 1 { + #[allow( + clippy::useless_conversion, + reason = "needed for 32-bit platforms" + )] + zelf.blksize.store(i64::from(status.st_blksize)); + } + } + Err(err) => { + if err.raw_os_error() == Some(libc::EBADF) { + // fd is invalid, prevent destructor from trying to close it + zelf.fd.store(-1); + return Err(OSErrorBuilder::with_filename(&err, filename, vm)); + } + } + } + } + + #[cfg(windows)] + crate::stdlib::msvcrt::setmode_binary(fd); + if let Err(e) = zelf.as_object().set_attr("name", name, vm) { + // If fd was passed by user, don't close it on error + if !fd_is_own { + zelf.fd.store(-1); + } + return Err(e); + } + + if mode.contains(Mode::APPENDING) { + let _ = os::lseek(fd, 0, libc::SEEK_END, vm); + } + + Ok(()) + } + } + + impl Representable for FileIO { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let type_name = zelf.class().slot_name(); + let fd = zelf.fd.load(); + if fd < 0 { + return Ok(format!("<{type_name} [closed]>")); + } + let name_repr = repr_file_obj_name(zelf.as_object(), vm)?; + let mode = zelf.mode(); + let closefd = if zelf.closefd.load() { "True" } else { "False" }; + let repr = if let Some(name_repr) = name_repr { + format!("<{type_name} name={name_repr} mode='{mode}' closefd={closefd}>") + } else { + format!("<{type_name} fd={fd} mode='{mode}' closefd={closefd}>") + }; + Ok(repr) + } + } + + #[pyclass( + with(Constructor, Initializer, Representable, Destructor), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) + )] + impl FileIO { + fn io_error( + zelf: &Py<Self>, + error: std::io::Error, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + let exc = error.to_pyexception(vm); + if let Ok(name) = zelf.as_object().get_attr("name", vm) { + exc.as_object() + .set_attr("filename", name, vm) + .expect("OSError.filename set must success"); + } + exc + } + + #[pygetset] + fn closed(&self) -> bool { + self.fd.load() < 0 + } + + #[pygetset] + fn closefd(&self) -> bool { + self.closefd.load() + } + + #[pygetset(name = "_blksize")] + fn blksize(&self) -> i64 { + self.blksize.load() + } + + #[pymethod] + fn fileno(&self, vm: &VirtualMachine) -> PyResult<i32> { + let fd = self.fd.load(); + if fd >= 0 { + Ok(fd) + } else { + Err(io_closed_error(vm)) + } + } + + fn get_fd(&self, vm: &VirtualMachine) -> PyResult<crt_fd::Borrowed<'_>> { + self.fileno(vm) + .map(|fd| unsafe { crt_fd::Borrowed::borrow_raw(fd) }) + } + + #[pymethod] + fn readable(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(self.mode.load().contains(Mode::READABLE)) + } + + #[pymethod] + fn writable(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(self.mode.load().contains(Mode::WRITABLE)) + } + + #[pygetset] + fn mode(&self) -> &'static str { + let mode = self.mode.load(); + if mode.contains(Mode::CREATED) { + if mode.contains(Mode::READABLE) { + "xb+" + } else { + "xb" + } + } else if mode.contains(Mode::APPENDING) { + if mode.contains(Mode::READABLE) { + "ab+" + } else { + "ab" + } + } else if mode.contains(Mode::READABLE) { + if mode.contains(Mode::WRITABLE) { + "rb+" + } else { + "rb" + } + } else { + "wb" + } + } + + #[pymethod] + fn read( + zelf: &Py<Self>, + read_byte: OptionalSize, + vm: &VirtualMachine, + ) -> PyResult<Option<Vec<u8>>> { + if !zelf.mode.load().contains(Mode::READABLE) { + return Err(new_unsupported_operation( + vm, + "File or stream is not readable".to_owned(), + )); + } + let handle = zelf.get_fd(vm)?; + let bytes = if let Some(read_byte) = read_byte.to_usize() { + let mut bytes = vec![0; read_byte]; + // Loop on EINTR (PEP 475) + let n = loop { + match vm.allow_threads(|| crt_fd::read(handle, &mut bytes)) { + Ok(n) => break n, + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + return Ok(None); + } + Err(e) => return Err(Self::io_error(zelf, e, vm)), + } + }; + bytes.truncate(n); + bytes + } else { + let mut bytes = vec![]; + // Loop on EINTR (PEP 475) + loop { + match vm.allow_threads(|| { + let mut h = handle; + h.read_to_end(&mut bytes) + }) { + Ok(_) => break, + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + // Non-blocking mode: return None if EAGAIN (only if no data read yet) + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + if bytes.is_empty() { + return Ok(None); + } + break; + } + Err(e) => return Err(Self::io_error(zelf, e, vm)), + } + } + bytes + }; + + Ok(Some(bytes)) + } + + #[pymethod] + fn readinto( + zelf: &Py<Self>, + obj: ArgMemoryBuffer, + vm: &VirtualMachine, + ) -> PyResult<Option<usize>> { + if !zelf.mode.load().contains(Mode::READABLE) { + return Err(new_unsupported_operation( + vm, + "File or stream is not readable".to_owned(), + )); + } + + let handle = zelf.get_fd(vm)?; + + let mut buf = obj.borrow_buf_mut(); + // Loop on EINTR (PEP 475) + let ret = loop { + match vm.allow_threads(|| crt_fd::read(handle, &mut buf)) { + Ok(n) => break n, + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => { + return Ok(None); + } + Err(e) => return Err(Self::io_error(zelf, e, vm)), + } + }; + + Ok(Some(ret)) + } + + #[pymethod] + fn write( + zelf: &Py<Self>, + obj: ArgBytesLike, + vm: &VirtualMachine, + ) -> PyResult<Option<usize>> { + if !zelf.mode.load().contains(Mode::WRITABLE) { + return Err(new_unsupported_operation( + vm, + "File or stream is not writable".to_owned(), + )); + } + + let handle = zelf.get_fd(vm)?; + + // Loop on EINTR (PEP 475) + let len = loop { + match obj.with_ref(|b| vm.allow_threads(|| crt_fd::write(handle, b))) { + Ok(n) => break n, + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + // Non-blocking mode: return None if EAGAIN + Err(e) if e.raw_os_error() == Some(libc::EAGAIN) => return Ok(None), + Err(e) => return Err(Self::io_error(zelf, e, vm)), + } + }; + + //return number of bytes written + Ok(Some(len)) + } + + #[pymethod] + fn close(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + let res = iobase_close(zelf.as_object(), vm); + if !zelf.closefd.load() { + zelf.fd.store(-1); + return res; + } + let flush_exc = res.err(); + if zelf.finalizing.load() { + Self::dealloc_warn(zelf, zelf.as_object().to_owned(), vm); + } + let fd = zelf.fd.swap(-1); + let close_err = if fd >= 0 { + crt_fd::close(unsafe { crt_fd::Owned::from_raw(fd) }) + .map_err(|err| Self::io_error(zelf, err, vm)) + .err() + } else { + None + }; + match (flush_exc, close_err) { + (Some(fe), Some(ce)) => { + ce.set___context__(Some(fe)); + Err(ce) + } + (Some(e), None) | (None, Some(e)) => Err(e), + (None, None) => Ok(()), + } + } + + #[pymethod] + fn seekable(&self, vm: &VirtualMachine) -> PyResult<bool> { + let fd = self.get_fd(vm)?; + Ok(self.seekable.load().unwrap_or_else(|| { + let seekable = os::lseek(fd, 0, libc::SEEK_CUR, vm).is_ok(); + self.seekable.store(Some(seekable)); + seekable + })) + } + + #[pymethod] + fn seek( + &self, + offset: PyObjectRef, + how: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<Offset> { + let how = how.unwrap_or(0); + let fd = self.get_fd(vm)?; + let offset = get_offset(offset, vm)?; + + os::lseek(fd, offset, how, vm) + } + + #[pymethod] + fn tell(&self, vm: &VirtualMachine) -> PyResult<Offset> { + let fd = self.get_fd(vm)?; + os::lseek(fd, 0, libc::SEEK_CUR, vm) + } + + #[pymethod] + fn truncate(&self, len: OptionalOption, vm: &VirtualMachine) -> PyResult<Offset> { + let fd = self.get_fd(vm)?; + let len = match len.flatten() { + Some(l) => get_offset(l, vm)?, + None => os::lseek(fd, 0, libc::SEEK_CUR, vm)?, + }; + os::ftruncate(fd, len).map_err(|e| e.into_pyexception(vm))?; + Ok(len) + } + + #[pymethod] + fn isatty(&self, vm: &VirtualMachine) -> PyResult<bool> { + let fd = self.fileno(vm)?; + Ok(os::isatty(fd)) + } + + #[pymethod] + fn __getstate__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(format!("cannot pickle '{}' instances", zelf.class().name()))) + } + + /// fileio_dealloc_warn in Modules/_io/fileio.c + #[pymethod(name = "_dealloc_warn")] + fn _dealloc_warn_method( + zelf: &Py<Self>, + source: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + Self::dealloc_warn(zelf, source, vm); + Ok(()) + } + } + + impl FileIO { + /// Issue ResourceWarning if fd is still open and closefd is true. + fn dealloc_warn(zelf: &Py<Self>, source: PyObjectRef, vm: &VirtualMachine) { + if zelf.fd.load() >= 0 && zelf.closefd.load() { + let repr = source + .repr(vm) + .map(|s| s.as_wtf8().to_owned()) + .unwrap_or_else(|_| Wtf8Buf::from("<file>")); + if let Err(e) = crate::stdlib::_warnings::warn( + vm.ctx.exceptions.resource_warning, + format!("unclosed file {repr}"), + 1, + vm, + ) { + vm.run_unraisable(e, None, zelf.as_object().to_owned()); + } + } + } + } + + impl Destructor for FileIO { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if let Some(fileio) = zelf.downcast_ref::<FileIO>() { + fileio.finalizing.store(true); + } + iobase_finalize(zelf, vm); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } +} + +// WindowsConsoleIO requires host environment and Windows +#[cfg(all(feature = "host_env", windows))] +#[pymodule] +mod winconsoleio { + use super::{_io::*, iobase_finalize}; + use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{PyBaseExceptionRef, PyUtf8StrRef}, + common::{lock::PyMutex, wtf8::Wtf8Buf}, + convert::{IntoPyException, ToPyException}, + function::{ArgBytesLike, ArgMemoryBuffer, OptionalArg}, + types::{Constructor, DefaultConstructor, Destructor, Initializer, Representable}, + }; + use crossbeam_utils::atomic::AtomicCell; + use windows_sys::Win32::{ + Foundation::{self, GENERIC_READ, GENERIC_WRITE, INVALID_HANDLE_VALUE}, + Globalization::{CP_UTF8, MultiByteToWideChar, WideCharToMultiByte}, + Storage::FileSystem::{ + CreateFileW, FILE_SHARE_READ, FILE_SHARE_WRITE, GetFullPathNameW, OPEN_EXISTING, + }, + System::Console::{ + GetConsoleMode, GetNumberOfConsoleInputEvents, ReadConsoleW, WriteConsoleW, + }, + }; + + type HANDLE = Foundation::HANDLE; + + const SMALLBUF: usize = 4; + const BUFMAX: usize = 32 * 1024 * 1024; + + fn handle_from_fd(fd: i32) -> HANDLE { + unsafe { rustpython_common::suppress_iph!(libc::get_osfhandle(fd)) as HANDLE } + } + + fn is_invalid_handle(handle: HANDLE) -> bool { + handle == INVALID_HANDLE_VALUE || handle.is_null() + } + + /// Check if a HANDLE is a console and what type ('r', 'w', or '\0'). + fn get_console_type(handle: HANDLE) -> char { + if is_invalid_handle(handle) { + return '\0'; + } + let mut mode: u32 = 0; + if unsafe { GetConsoleMode(handle, &mut mode) } == 0 { + return '\0'; + } + let mut peek_count: u32 = 0; + if unsafe { GetNumberOfConsoleInputEvents(handle, &mut peek_count) } != 0 { + 'r' + } else { + 'w' + } + } + + /// Check if a Python object (fd or path string) refers to a console. + /// Returns 'r' (input), 'w' (output), 'x' (generic CON), or '\0' (not a console). + pub(super) fn pyio_get_console_type(path_or_fd: &PyObject, vm: &VirtualMachine) -> char { + // Try as integer fd first + if let Ok(fd) = i32::try_from_object(vm, path_or_fd.to_owned()) { + if fd >= 0 { + let handle = handle_from_fd(fd); + return get_console_type(handle); + } + return '\0'; + } + + // Try as string path + let Ok(name) = path_or_fd.str(vm) else { + return '\0'; + }; + let Some(name_str) = name.to_str() else { + // Surrogate strings can't be console device names + return '\0'; + }; + + if name_str.eq_ignore_ascii_case("CONIN$") { + return 'r'; + } + if name_str.eq_ignore_ascii_case("CONOUT$") { + return 'w'; + } + if name_str.eq_ignore_ascii_case("CON") { + return 'x'; + } + + // Resolve full path and check for console device names + let wide: Vec<u16> = name_str.encode_utf16().chain(core::iter::once(0)).collect(); + let mut buf = [0u16; 260]; // MAX_PATH + let length = unsafe { + GetFullPathNameW( + wide.as_ptr(), + buf.len() as u32, + buf.as_mut_ptr(), + core::ptr::null_mut(), + ) + }; + if length == 0 || length as usize > buf.len() { + return '\0'; + } + let full_path = &buf[..length as usize]; + // Skip \\?\ or \\.\ prefix + let path_part = if full_path.len() >= 4 + && full_path[0] == b'\\' as u16 + && full_path[1] == b'\\' as u16 + && (full_path[2] == b'.' as u16 || full_path[2] == b'?' as u16) + && full_path[3] == b'\\' as u16 + { + &full_path[4..] + } else { + full_path + }; + + let path_str = String::from_utf16_lossy(path_part); + if path_str.eq_ignore_ascii_case("CONIN$") { + 'r' + } else if path_str.eq_ignore_ascii_case("CONOUT$") { + 'w' + } else if path_str.eq_ignore_ascii_case("CON") { + 'x' + } else { + '\0' + } + } + + /// Find the last valid UTF-8 boundary in a byte slice. + fn find_last_utf8_boundary(buf: &[u8], len: usize) -> usize { + let len = len.min(buf.len()); + for count in 1..=4.min(len) { + let c = buf[len - count]; + if c < 0x80 { + return len; + } + if c >= 0xc0 { + let expected = if c < 0xe0 { + 2 + } else if c < 0xf0 { + 3 + } else { + 4 + }; + if count < expected { + // Incomplete multibyte sequence + return len - count; + } + return len; + } + } + len + } + + #[pyattr] + #[pyclass(module = "_io", name = "_WindowsConsoleIO", base = _RawIOBase)] + #[derive(Debug)] + pub(super) struct WindowsConsoleIO { + _base: _RawIOBase, + fd: AtomicCell<i32>, + readable: AtomicCell<bool>, + writable: AtomicCell<bool>, + closefd: AtomicCell<bool>, + finalizing: AtomicCell<bool>, + blksize: AtomicCell<i64>, + buf: PyMutex<[u8; SMALLBUF]>, + } + + impl Default for WindowsConsoleIO { + fn default() -> Self { + Self { + _base: Default::default(), + fd: AtomicCell::new(-1), + readable: AtomicCell::new(false), + writable: AtomicCell::new(false), + closefd: AtomicCell::new(false), + finalizing: AtomicCell::new(false), + blksize: AtomicCell::new(super::DEFAULT_BUFFER_SIZE as _), + buf: PyMutex::new([0u8; SMALLBUF]), + } + } + } + + impl DefaultConstructor for WindowsConsoleIO {} + + #[derive(FromArgs)] + pub struct WindowsConsoleIOArgs { + #[pyarg(positional)] + name: PyObjectRef, + #[pyarg(any, default)] + mode: Option<PyUtf8StrRef>, + #[pyarg(any, default = true)] + closefd: bool, + #[allow(dead_code)] + #[pyarg(any, default)] + opener: Option<PyObjectRef>, + } + + impl Initializer for WindowsConsoleIO { + type Args = WindowsConsoleIOArgs; + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + let nameobj = args.name; + + if zelf.fd.load() >= 0 { + if zelf.closefd.load() { + internal_close(&zelf); + } else { + zelf.fd.store(-1); + } + } + + // Warn if bool is used as file descriptor + if nameobj.class().is(vm.ctx.types.bool_type) { + crate::stdlib::_warnings::warn( + vm.ctx.exceptions.runtime_warning, + "bool is used as a file descriptor".to_owned(), + 1, + vm, + )?; + } + + // Try to get fd from integer + let mut fd: i32 = -1; + if let Some(i) = nameobj.downcast_ref::<crate::builtins::PyInt>() { + fd = i.try_to_primitive::<i32>(vm).unwrap_or(-1); + if fd < 0 { + return Err(vm.new_value_error("negative file descriptor")); + } + } + + // Parse mode + let mode_str: &str = args + .mode + .as_ref() + .map(|s: &PyUtf8StrRef| s.as_str()) + .unwrap_or("r"); + + let mut rwa = false; + let mut readable = false; + let mut writable = false; + let mut console_type = '\0'; + for c in mode_str.bytes() { + match c { + b'+' | b'a' | b'b' | b'x' => {} + b'r' => { + if rwa { + return Err( + vm.new_value_error("Must have exactly one of read or write mode") + ); + } + rwa = true; + readable = true; + } + b'w' => { + if rwa { + return Err( + vm.new_value_error("Must have exactly one of read or write mode") + ); + } + rwa = true; + writable = true; + } + _ => { + return Err(vm.new_value_error(format!("invalid mode: {mode_str}"))); + } + } + } + if !rwa { + return Err(vm.new_value_error("Must have exactly one of read or write mode")); + } + + zelf.readable.store(readable); + zelf.writable.store(writable); + + let mut _name_wide: Option<Vec<u16>> = None; + + if fd < 0 { + // Get console type from name + console_type = pyio_get_console_type(&nameobj, vm); + if console_type == 'x' { + if writable { + console_type = 'w'; + } else { + console_type = 'r'; + } + } + + // Opening by name + zelf.closefd.store(true); + if !args.closefd { + return Err(vm.new_value_error("Cannot use closefd=False with file name")); + } + + let name_str = nameobj.str(vm)?; + let wide: Vec<u16> = name_str + .as_wtf8() + .encode_wide() + .chain(core::iter::once(0)) + .collect(); + + let access = if writable { + GENERIC_WRITE + } else { + GENERIC_READ + }; + + // Try read/write first, fall back to specific access + let mut handle: HANDLE = unsafe { + CreateFileW( + wide.as_ptr(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + core::ptr::null(), + OPEN_EXISTING, + 0, + core::ptr::null_mut(), + ) + }; + if is_invalid_handle(handle) { + handle = unsafe { + CreateFileW( + wide.as_ptr(), + access, + FILE_SHARE_READ | FILE_SHARE_WRITE, + core::ptr::null(), + OPEN_EXISTING, + 0, + core::ptr::null_mut(), + ) + }; + } + + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + let osf_flags = if writable { + libc::O_WRONLY | libc::O_BINARY | 0x80 /* O_NOINHERIT */ + } else { + libc::O_RDONLY | libc::O_BINARY | 0x80 /* O_NOINHERIT */ + }; + + fd = unsafe { libc::open_osfhandle(handle as isize, osf_flags) }; + if fd < 0 { + unsafe { + Foundation::CloseHandle(handle); + } + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + _name_wide = Some(wide); + } else { + // When opened by fd, never close the fd (user owns it) + zelf.closefd.store(false); + } + + zelf.fd.store(fd); + + // Validate console type + if console_type == '\0' { + let handle = handle_from_fd(fd); + console_type = get_console_type(handle); + } + + if console_type == '\0' { + // Not a console at all + internal_close(&zelf); + return Err(vm.new_value_error("Cannot open non-console file")); + } + + if writable && console_type != 'w' { + internal_close(&zelf); + return Err(vm.new_value_error("Cannot open console input buffer for writing")); + } + if readable && console_type != 'r' { + internal_close(&zelf); + return Err(vm.new_value_error("Cannot open console output buffer for reading")); + } + + zelf.blksize.store(super::DEFAULT_BUFFER_SIZE as _); + *zelf.buf.lock() = [0u8; SMALLBUF]; + + zelf.as_object().set_attr("name", nameobj, vm)?; + + Ok(()) + } + } + + fn internal_close(zelf: &WindowsConsoleIO) { + let fd = zelf.fd.swap(-1); + if fd >= 0 && zelf.closefd.load() { + unsafe { + libc::close(fd); + } + } + } + + impl Representable for WindowsConsoleIO { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let type_name = zelf.class().slot_name(); + let fd = zelf.fd.load(); + if fd < 0 { + return Ok(format!("<{type_name} [closed]>")); + } + let mode = if zelf.readable.load() { "rb" } else { "wb" }; + let closefd = if zelf.closefd.load() { "True" } else { "False" }; + Ok(format!("<{type_name} mode='{mode}' closefd={closefd}>")) + } + } + + #[pyclass( + with(Constructor, Initializer, Representable, Destructor), + flags(BASETYPE, HAS_DICT, HAS_WEAKREF) + )] + impl WindowsConsoleIO { + #[allow(dead_code)] + fn io_error( + zelf: &Py<Self>, + error: std::io::Error, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + let exc = error.to_pyexception(vm); + if let Ok(name) = zelf.as_object().get_attr("name", vm) { + exc.as_object() + .set_attr("filename", name, vm) + .expect("OSError.filename set must succeed"); + } + exc + } + + #[pygetset] + fn closed(&self) -> bool { + self.fd.load() < 0 + } + + #[pygetset] + fn closefd(&self) -> bool { + self.closefd.load() + } + + #[pygetset(name = "_blksize")] + fn blksize(&self) -> i64 { + self.blksize.load() + } + + #[pygetset] + fn mode(&self) -> &'static str { + if self.readable.load() { "rb" } else { "wb" } + } + + #[pymethod] + fn fileno(&self, vm: &VirtualMachine) -> PyResult<i32> { + let fd = self.fd.load(); + if fd >= 0 { + Ok(fd) + } else { + Err(io_closed_error(vm)) + } + } + + fn get_fd(&self, vm: &VirtualMachine) -> PyResult<i32> { + self.fileno(vm) + } + + #[pymethod] + fn readable(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(self.readable.load()) + } + + #[pymethod] + fn writable(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(self.writable.load()) + } + + #[pymethod] + fn isatty(&self, vm: &VirtualMachine) -> PyResult<bool> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + Ok(true) + } + + #[pymethod] + fn close(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + let res = iobase_close(zelf.as_object(), vm); + if !zelf.closefd.load() { + zelf.fd.store(-1); + return res; + } + let flush_exc = res.err(); + if zelf.finalizing.load() { + Self::dealloc_warn(zelf, zelf.as_object().to_owned(), vm); + } + let fd = zelf.fd.swap(-1); + let close_err: Option<PyBaseExceptionRef> = if fd >= 0 { + let result = unsafe { libc::close(fd) }; + if result < 0 { + Some(std::io::Error::last_os_error().into_pyexception(vm)) + } else { + None + } + } else { + None + }; + match (flush_exc, close_err) { + (Some(fe), Some(ce)) => { + ce.set___context__(Some(fe)); + Err(ce) + } + (Some(e), None) | (None, Some(e)) => Err(e), + (None, None) => Ok(()), + } + } + + fn dealloc_warn(zelf: &Py<Self>, source: PyObjectRef, vm: &VirtualMachine) { + if zelf.fd.load() >= 0 && zelf.closefd.load() { + let repr = source + .repr(vm) + .map(|s| s.as_wtf8().to_owned()) + .unwrap_or_else(|_| Wtf8Buf::from("<file>")); + if let Err(e) = crate::stdlib::_warnings::warn( + vm.ctx.exceptions.resource_warning, + format!("unclosed file {repr}"), + 1, + vm, + ) { + vm.run_unraisable(e, None, zelf.as_object().to_owned()); + } + } + } + + fn copy_from_buf(buf: &mut [u8; SMALLBUF], dest: &mut [u8]) -> usize { + let mut n = 0; + while buf[0] != 0 && n < dest.len() { + dest[n] = buf[0]; + n += 1; + for i in 1..SMALLBUF { + buf[i - 1] = buf[i]; + } + buf[SMALLBUF - 1] = 0; + } + n + } + + #[pymethod] + fn readinto(&self, buffer: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<usize> { + let fd = self.get_fd(vm)?; + if !self.readable.load() { + return Err(new_unsupported_operation( + vm, + "Console buffer does not support reading".to_owned(), + )); + } + let mut buf_ref = buffer.borrow_buf_mut(); + let len = buf_ref.len(); + if len == 0 { + return Ok(0); + } + if len > BUFMAX { + return Err(vm.new_value_error(format!("cannot read more than {BUFMAX} bytes"))); + } + + let handle = handle_from_fd(fd); + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + // Each character may take up to 4 bytes in UTF-8. + let mut wlen = (len / 4) as u32; + if wlen == 0 { + wlen = 1; + } + + let dest = &mut *buf_ref; + + // Copy from internal buffer first + let mut read_len = { + let mut buf = self.buf.lock(); + Self::copy_from_buf(&mut buf, dest) + }; + if read_len > 0 { + wlen = wlen.saturating_sub(1); + } + if read_len >= len || wlen == 0 { + return Ok(read_len); + } + + // Read from console + let mut wbuf = vec![0u16; wlen as usize]; + let mut nread: u32 = 0; + let res = unsafe { + ReadConsoleW( + handle, + wbuf.as_mut_ptr() as _, + wlen, + &mut nread, + core::ptr::null(), + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + if nread == 0 { + return Ok(read_len); + } + + // Check for Ctrl+Z (EOF) + if nread > 0 && wbuf[0] == 0x1A { + return Ok(read_len); + } + + // Convert wchar to UTF-8 + let remaining = len - read_len; + let u8n; + if remaining < 4 { + // Buffer the result in the internal small buffer + let mut buf = self.buf.lock(); + let converted = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + buf.as_mut_ptr() as _, + SMALLBUF as i32, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if converted > 0 { + u8n = Self::copy_from_buf(&mut buf, &mut dest[read_len..]) as i32; + } else { + u8n = 0; + } + } else { + u8n = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + dest[read_len..].as_mut_ptr() as _, + remaining as i32, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + } + + if u8n > 0 { + read_len += u8n as usize; + } else { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(122) { + // ERROR_INSUFFICIENT_BUFFER + let needed = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if needed > 0 { + return Err(vm.new_system_error(format!( + "Buffer had room for {remaining} bytes but {needed} bytes required", + ))); + } + } + return Err(err.into_pyexception(vm)); + } + + Ok(read_len) + } + + #[pymethod] + fn readall(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + + let handle = handle_from_fd(self.fd.load()); + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + let mut result = Vec::new(); + + // Copy any buffered bytes first + { + let mut buf = self.buf.lock(); + let mut tmp = [0u8; SMALLBUF]; + let n = Self::copy_from_buf(&mut buf, &mut tmp); + result.extend_from_slice(&tmp[..n]); + } + + let mut wbuf = vec![0u16; 8192]; + loop { + let mut nread: u32 = 0; + let res = unsafe { + ReadConsoleW( + handle, + wbuf.as_mut_ptr() as _, + wbuf.len() as u32, + &mut nread, + core::ptr::null(), + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + if nread == 0 { + break; + } + // Ctrl+Z at start -> EOF + if wbuf[0] == 0x1A { + break; + } + // Convert to UTF-8 + let needed = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if needed == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + let offset = result.len(); + result.resize(offset + needed as usize, 0); + let written = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + result[offset..].as_mut_ptr() as _, + needed, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if written == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + // If we didn't fill the buffer, no more data + if nread < wbuf.len() as u32 { + break; + } + } + + Ok(vm.ctx.new_bytes(result).into()) + } + + #[pymethod] + fn read(&self, size: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + if !self.readable.load() { + return Err(new_unsupported_operation( + vm, + "Console buffer does not support reading".to_owned(), + )); + } + let size = size.unwrap_or(-1); + if size < 0 { + return self.readall(vm); + } + if size as usize > BUFMAX { + return Err(vm.new_value_error(format!("cannot read more than {BUFMAX} bytes"))); + } + let mut buf = vec![0u8; size as usize]; + let handle = handle_from_fd(self.fd.load()); + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + let len = size as usize; + + let mut wlen = (len / 4) as u32; + if wlen == 0 { + wlen = 1; + } + + let mut read_len = { + let mut ibuf = self.buf.lock(); + Self::copy_from_buf(&mut ibuf, &mut buf) + }; + if read_len > 0 { + wlen = wlen.saturating_sub(1); + } + if read_len >= len || wlen == 0 { + buf.truncate(read_len); + return Ok(vm.ctx.new_bytes(buf).into()); + } + + let mut wbuf = vec![0u16; wlen as usize]; + let mut nread: u32 = 0; + let res = unsafe { + ReadConsoleW( + handle, + wbuf.as_mut_ptr() as _, + wlen, + &mut nread, + core::ptr::null(), + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + if nread == 0 || wbuf[0] == 0x1A { + buf.truncate(read_len); + return Ok(vm.ctx.new_bytes(buf).into()); + } + + let remaining = len - read_len; + let u8n; + if remaining < 4 { + let mut ibuf = self.buf.lock(); + let converted = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + ibuf.as_mut_ptr() as _, + SMALLBUF as i32, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if converted > 0 { + u8n = Self::copy_from_buf(&mut ibuf, &mut buf[read_len..]) as i32; + } else { + u8n = 0; + } + } else { + u8n = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + buf[read_len..].as_mut_ptr() as _, + remaining as i32, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + } + + if u8n > 0 { + read_len += u8n as usize; + } else { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(122) { + // ERROR_INSUFFICIENT_BUFFER + let needed = unsafe { + WideCharToMultiByte( + CP_UTF8, + 0, + wbuf.as_ptr(), + nread as i32, + core::ptr::null_mut(), + 0, + core::ptr::null(), + core::ptr::null_mut(), + ) + }; + if needed > 0 { + return Err(vm.new_system_error(format!( + "Buffer had room for {remaining} bytes but {needed} bytes required", + ))); + } + } + return Err(err.into_pyexception(vm)); + } + + buf.truncate(read_len); + Ok(vm.ctx.new_bytes(buf).into()) + } + + #[pymethod] + fn write(&self, b: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { + if self.fd.load() < 0 { + return Err(io_closed_error(vm)); + } + if !self.writable.load() { + return Err(new_unsupported_operation( + vm, + "Console buffer does not support writing".to_owned(), + )); + } + + let handle = handle_from_fd(self.fd.load()); + if is_invalid_handle(handle) { + return Err(std::io::Error::last_os_error().to_pyexception(vm)); + } + + let data = b.borrow_buf(); + let data = &*data; + if data.is_empty() { + return Ok(0); + } + + let mut len = data.len().min(BUFMAX); + + // Cap at 32766/2 wchars * 3 bytes (UTF-8 to wchar ratio is at most 3:1) + let max_wlen: u32 = 32766 / 2; + len = len.min(max_wlen as usize * 3); + + // Reduce len until wlen fits within max_wlen + let wlen; + loop { + len = find_last_utf8_boundary(data, len); + let w = unsafe { + MultiByteToWideChar( + CP_UTF8, + 0, + data.as_ptr(), + len as i32, + core::ptr::null_mut(), + 0, + ) + }; + if w as u32 <= max_wlen { + wlen = w; + break; + } + len /= 2; + } + if wlen == 0 { + return Ok(0); + } + + let mut wbuf = vec![0u16; wlen as usize]; + let wlen = unsafe { + MultiByteToWideChar( + CP_UTF8, + 0, + data.as_ptr(), + len as i32, + wbuf.as_mut_ptr(), + wlen, + ) + }; + if wlen == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + + let mut n_written: u32 = 0; + let res = unsafe { + WriteConsoleW( + handle, + wbuf.as_ptr() as _, + wlen as u32, + &mut n_written, + core::ptr::null(), + ) + }; + if res == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + + // If we wrote fewer wchars than expected, recalculate bytes consumed + if n_written < wlen as u32 { + // Binary search to find how many input bytes correspond to n_written wchars + len = wchar_to_utf8_count(data, len, n_written); + } + + Ok(len) + } + + #[pymethod(name = "__reduce__")] + fn reduce(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle '_WindowsConsoleIO' instances")) + } + } + + /// Find how many UTF-8 bytes correspond to n wide chars. + fn wchar_to_utf8_count(data: &[u8], mut len: usize, mut n: u32) -> usize { + let mut start: usize = 0; + loop { + let mut mid = 0; + for i in (len / 2)..=len { + mid = find_last_utf8_boundary(data, i); + if mid != 0 { + break; + } + } + if mid == len { + return start + len; + } + if mid == 0 { + mid = if len > 1 { len - 1 } else { 1 }; + } + let wlen = unsafe { + MultiByteToWideChar( + CP_UTF8, + 0, + data[start..].as_ptr(), + mid as i32, + core::ptr::null_mut(), + 0, + ) + } as u32; + if wlen <= n { + start += mid; + len -= mid; + n -= wlen; + } else { + len = mid; + } + } + } + + impl Destructor for WindowsConsoleIO { + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + if let Some(cio) = zelf.downcast_ref::<WindowsConsoleIO>() { + cio.finalizing.store(true); + } + iobase_finalize(zelf, vm); + Ok(()) + } + + #[cold] + fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { + unreachable!("slot_del is implemented") + } + } +} diff --git a/crates/vm/src/stdlib/_operator.rs b/crates/vm/src/stdlib/_operator.rs new file mode 100644 index 00000000000..38b0f715d31 --- /dev/null +++ b/crates/vm/src/stdlib/_operator.rs @@ -0,0 +1,624 @@ +pub(crate) use _operator::module_def; + +#[pymodule] +mod _operator { + use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyInt, PyIntRef, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, PyUtf8StrRef}, + common::wtf8::{Wtf8, Wtf8Buf}, + function::{ArgBytesLike, Either, FuncArgs, KwArgs, OptionalArg}, + protocol::PyIter, + recursion::ReprGuard, + types::{Callable, Constructor, PyComparisonOp, Representable}, + }; + use constant_time_eq::constant_time_eq; + + #[pyfunction] + fn lt(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + a.rich_compare(b, PyComparisonOp::Lt, vm) + } + + #[pyfunction] + fn le(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + a.rich_compare(b, PyComparisonOp::Le, vm) + } + + #[pyfunction] + fn gt(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + a.rich_compare(b, PyComparisonOp::Gt, vm) + } + + #[pyfunction] + fn ge(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + a.rich_compare(b, PyComparisonOp::Ge, vm) + } + + #[pyfunction] + fn eq(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + a.rich_compare(b, PyComparisonOp::Eq, vm) + } + + #[pyfunction] + fn ne(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + a.rich_compare(b, PyComparisonOp::Ne, vm) + } + + #[pyfunction] + fn not_(a: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + a.try_to_bool(vm).map(|r| !r) + } + + #[pyfunction] + fn truth(a: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + a.try_to_bool(vm) + } + + #[pyfunction] + fn is_(a: PyObjectRef, b: PyObjectRef) -> PyResult<bool> { + Ok(a.is(&b)) + } + + #[pyfunction] + fn is_not(a: PyObjectRef, b: PyObjectRef) -> PyResult<bool> { + Ok(!a.is(&b)) + } + + #[pyfunction] + fn abs(a: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._abs(&a) + } + + #[pyfunction] + fn add(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._add(&a, &b) + } + + #[pyfunction] + fn and_(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._and(&a, &b) + } + + #[pyfunction] + fn floordiv(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._floordiv(&a, &b) + } + + // Note: Keep track of issue17567. Will need changes in order to strictly match behavior of + // a.__index__ as raised in the issue. Currently, we accept int subclasses. + #[pyfunction] + fn index(a: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyIntRef> { + a.try_index(vm) + } + + #[pyfunction] + fn invert(pos: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._invert(&pos) + } + + #[pyfunction] + fn lshift(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._lshift(&a, &b) + } + + #[pyfunction(name = "mod")] + fn mod_(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._mod(&a, &b) + } + + #[pyfunction] + fn mul(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._mul(&a, &b) + } + + #[pyfunction] + fn matmul(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._matmul(&a, &b) + } + + #[pyfunction] + fn neg(pos: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._neg(&pos) + } + + #[pyfunction] + fn or_(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._or(&a, &b) + } + + #[pyfunction] + fn pos(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._pos(&obj) + } + + #[pyfunction] + fn pow(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._pow(&a, &b, vm.ctx.none.as_object()) + } + + #[pyfunction] + fn rshift(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._rshift(&a, &b) + } + + #[pyfunction] + fn sub(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._sub(&a, &b) + } + + #[pyfunction] + fn truediv(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._truediv(&a, &b) + } + + #[pyfunction] + fn xor(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._xor(&a, &b) + } + + // Sequence based operators + + #[pyfunction] + fn concat(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Best attempt at checking that a is sequence-like. + if !a.class().has_attr(identifier!(vm, __getitem__)) + || a.fast_isinstance(vm.ctx.types.dict_type) + { + return Err( + vm.new_type_error(format!("{} object can't be concatenated", a.class().name())) + ); + } + vm._add(&a, &b) + } + + #[pyfunction] + fn contains(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + vm._contains(&a, &b) + } + + #[pyfunction(name = "countOf")] + fn count_of(a: PyIter, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + let mut count: usize = 0; + for element in a.iter_without_hint::<PyObjectRef>(vm)? { + let element = element?; + if element.is(&b) || vm.bool_eq(&b, &element)? { + count += 1; + } + } + Ok(count) + } + + #[pyfunction] + fn delitem(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + a.del_item(&*b, vm) + } + + #[pyfunction] + fn getitem(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + a.get_item(&*b, vm) + } + + #[pyfunction(name = "indexOf")] + fn index_of(a: PyIter, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + for (index, element) in a.iter_without_hint::<PyObjectRef>(vm)?.enumerate() { + let element = element?; + if element.is(&b) || vm.bool_eq(&b, &element)? { + return Ok(index); + } + } + Err(vm.new_value_error("sequence.index(x): x not in sequence")) + } + + #[pyfunction] + fn setitem( + a: PyObjectRef, + b: PyObjectRef, + c: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + a.set_item(&*b, c, vm) + } + + #[pyfunction] + fn length_hint(obj: PyObjectRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult<usize> { + let default: usize = default + .map(|v| { + if !v.fast_isinstance(vm.ctx.types.int_type) { + return Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + v.class().name() + ))); + } + v.downcast_ref::<PyInt>().unwrap().try_to_primitive(vm) + }) + .unwrap_or(Ok(0))?; + obj.length_hint(default, vm) + } + + // Inplace Operators + + #[pyfunction] + fn iadd(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._iadd(&a, &b) + } + + #[pyfunction] + fn iand(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._iand(&a, &b) + } + + #[pyfunction] + fn iconcat(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Best attempt at checking that a is sequence-like. + if !a.class().has_attr(identifier!(vm, __getitem__)) + || a.fast_isinstance(vm.ctx.types.dict_type) + { + return Err(vm.new_type_error(format!( + "'{}' object can't be concatenated", + a.class().name() + ))); + } + vm._iadd(&a, &b) + } + + #[pyfunction] + fn ifloordiv(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._ifloordiv(&a, &b) + } + + #[pyfunction] + fn ilshift(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._ilshift(&a, &b) + } + + #[pyfunction] + fn imod(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._imod(&a, &b) + } + + #[pyfunction] + fn imul(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._imul(&a, &b) + } + + #[pyfunction] + fn imatmul(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._imatmul(&a, &b) + } + + #[pyfunction] + fn ior(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._ior(&a, &b) + } + + #[pyfunction] + fn ipow(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._ipow(&a, &b, vm.ctx.none.as_object()) + } + + #[pyfunction] + fn irshift(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._irshift(&a, &b) + } + + #[pyfunction] + fn isub(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._isub(&a, &b) + } + + #[pyfunction] + fn itruediv(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._itruediv(&a, &b) + } + + #[pyfunction] + fn ixor(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._ixor(&a, &b) + } + + #[pyfunction] + fn _compare_digest( + a: Either<PyStrRef, ArgBytesLike>, + b: Either<PyStrRef, ArgBytesLike>, + vm: &VirtualMachine, + ) -> PyResult<bool> { + let res = match (a, b) { + (Either::A(a), Either::A(b)) => { + if !a.isascii() || !b.isascii() { + return Err(vm.new_type_error( + "comparing strings with non-ASCII characters is not supported", + )); + } + constant_time_eq(a.as_bytes(), b.as_bytes()) + } + (Either::B(a), Either::B(b)) => a.with_ref(|a| b.with_ref(|b| constant_time_eq(a, b))), + _ => { + return Err( + vm.new_type_error("unsupported operand types(s) or combination of types") + ); + } + }; + Ok(res) + } + + /// attrgetter(attr, /, *attrs) + /// -- + /// + /// Return a callable object that fetches the given attribute(s) from its operand. + /// After f = attrgetter('name'), the call f(r) returns r.name. + /// After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). + /// After h = attrgetter('name.first', 'name.last'), the call h(r) returns + /// (r.name.first, r.name.last). + #[pyattr] + #[pyclass(name = "attrgetter")] + #[derive(Debug, PyPayload)] + struct PyAttrGetter { + attrs: Vec<PyStrRef>, + } + + #[pyclass(with(Callable, Constructor, Representable))] + impl PyAttrGetter { + #[pygetset] + fn __text_signature__(&self) -> &'static str { + "(obj, /)" + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<(PyTypeRef, PyTupleRef)> { + let attrs = vm + .ctx + .new_tuple(zelf.attrs.iter().map(|v| v.clone().into()).collect()); + Ok((zelf.class().to_owned(), attrs)) + } + + // Go through dotted parts of string and call getattr on whatever is returned. + fn get_single_attr( + obj: PyObjectRef, + attr: &Py<PyStr>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + let attr_str = attr.as_bytes(); + let parts = attr_str.split(|&b| b == b'.').collect::<Vec<_>>(); + if parts.len() == 1 { + return obj.get_attr(attr, vm); + } + let mut obj = obj; + for part in parts { + let part = Wtf8::from_bytes(part).expect("originally valid WTF-8"); + obj = obj.get_attr(&vm.ctx.new_str(part), vm)?; + } + Ok(obj) + } + } + + impl Constructor for PyAttrGetter { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let n_attr = args.args.len(); + // Check we get no keyword and at least one positional. + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("attrgetter() takes no keyword arguments")); + } + if n_attr == 0 { + return Err(vm.new_type_error("attrgetter expected 1 argument, got 0.")); + } + let mut attrs = Vec::with_capacity(n_attr); + for o in args.args { + if let Ok(r) = o.try_into_value(vm) { + attrs.push(r); + } else { + return Err(vm.new_type_error("attribute name must be a string")); + } + } + Ok(Self { attrs }) + } + } + + impl Callable for PyAttrGetter { + type Args = PyObjectRef; + fn call(zelf: &Py<Self>, obj: Self::Args, vm: &VirtualMachine) -> PyResult { + // Handle case where we only have one attribute. + if zelf.attrs.len() == 1 { + return Self::get_single_attr(obj, &zelf.attrs[0], vm); + } + // Build tuple and call get_single on each element in attrs. + let mut results = Vec::with_capacity(zelf.attrs.len()); + for o in &zelf.attrs { + results.push(Self::get_single_attr(obj.clone(), o, vm)?); + } + Ok(vm.ctx.new_tuple(results).into()) + } + } + + impl Representable for PyAttrGetter { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let mut result = Wtf8Buf::from("operator.attrgetter("); + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let mut first = true; + for part in &zelf.attrs { + if !first { + result.push_str(", "); + } + first = false; + result.push_wtf8(part.as_object().repr(vm)?.as_wtf8()); + } + } else { + result.push_str("..."); + } + result.push_char(')'); + Ok(result) + } + } + + /// itemgetter(item, /, *items) + /// -- + /// + /// Return a callable object that fetches the given item(s) from its operand. + /// After f = itemgetter(2), the call f(r) returns r[2]. + /// After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) + #[pyattr] + #[pyclass(name = "itemgetter")] + #[derive(Debug, PyPayload)] + struct PyItemGetter { + items: Vec<PyObjectRef>, + } + + #[pyclass(with(Callable, Constructor, Representable))] + impl PyItemGetter { + #[pygetset] + fn __text_signature__(&self) -> &'static str { + "(obj, /)" + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyObjectRef { + let items = vm.ctx.new_tuple(zelf.items.to_vec()); + vm.new_pyobj((zelf.class().to_owned(), items)) + } + } + impl Constructor for PyItemGetter { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + // Check we get no keyword and at least one positional. + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("itemgetter() takes no keyword arguments")); + } + if args.args.is_empty() { + return Err(vm.new_type_error("itemgetter expected 1 argument, got 0.")); + } + Ok(Self { items: args.args }) + } + } + + impl Callable for PyItemGetter { + type Args = PyObjectRef; + fn call(zelf: &Py<Self>, obj: Self::Args, vm: &VirtualMachine) -> PyResult { + // Handle case where we only have one attribute. + if zelf.items.len() == 1 { + return obj.get_item(&*zelf.items[0], vm); + } + // Build tuple and call get_single on each element in attrs. + let mut results = Vec::with_capacity(zelf.items.len()); + for item in &zelf.items { + results.push(obj.get_item(&**item, vm)?); + } + Ok(vm.ctx.new_tuple(results).into()) + } + } + + impl Representable for PyItemGetter { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let mut result = Wtf8Buf::from("operator.itemgetter("); + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let mut first = true; + for item in &zelf.items { + if !first { + result.push_str(", "); + } + first = false; + result.push_wtf8(item.repr(vm)?.as_wtf8()); + } + } else { + result.push_str("..."); + } + result.push_char(')'); + Ok(result) + } + } + + /// methodcaller(name, /, *args, **kwargs) + /// -- + /// + /// Return a callable object that calls the given method on its operand. + /// After f = methodcaller('name'), the call f(r) returns r.name(). + /// After g = methodcaller('name', 'date', foo=1), the call g(r) returns + /// r.name('date', foo=1). + #[pyattr] + #[pyclass(name = "methodcaller")] + #[derive(Debug, PyPayload)] + struct PyMethodCaller { + name: PyUtf8StrRef, + args: FuncArgs, + } + + #[pyclass(with(Callable, Constructor, Representable))] + impl PyMethodCaller { + #[pygetset] + fn __text_signature__(&self) -> &'static str { + "(obj, /)" + } + + #[pymethod] + fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + // With no kwargs, return (type(obj), (name, *args)) tuple. + if zelf.args.kwargs.is_empty() { + let mut py_args = vec![zelf.name.as_object().to_owned()]; + py_args.append(&mut zelf.args.args.clone()); + Ok(vm.new_tuple((zelf.class().to_owned(), vm.ctx.new_tuple(py_args)))) + } else { + // If we have kwargs, create a partial function that contains them and pass back that + // along with the args. + let partial = vm.import("functools", 0)?.get_attr("partial", vm)?; + let args = FuncArgs::new( + vec![zelf.class().to_owned().into(), zelf.name.clone().into()], + KwArgs::new(zelf.args.kwargs.clone()), + ); + let callable = partial.call(args, vm)?; + Ok(vm.new_tuple((callable, vm.ctx.new_tuple(zelf.args.args.clone())))) + } + } + } + + impl Constructor for PyMethodCaller { + type Args = (PyObjectRef, FuncArgs); + + fn py_new( + _cls: &Py<PyType>, + (name, args): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let name = name + .try_into_value(vm) + .map_err(|_| vm.new_type_error("method name must be a string"))?; + Ok(Self { name, args }) + } + } + + impl Callable for PyMethodCaller { + type Args = PyObjectRef; + + #[inline] + fn call(zelf: &Py<Self>, obj: Self::Args, vm: &VirtualMachine) -> PyResult { + vm.call_method(&obj, zelf.name.as_str(), zelf.args.clone()) + } + } + + impl Representable for PyMethodCaller { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let mut result = Wtf8Buf::from("operator.methodcaller("); + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let args = &zelf.args.args; + let kwargs = &zelf.args.kwargs; + result.push_wtf8(zelf.name.as_object().repr(vm)?.as_wtf8()); + for v in args { + result.push_str(", "); + result.push_wtf8(v.repr(vm)?.as_wtf8()); + } + for (key, value) in kwargs { + result.push_str(", "); + result.push_str(key); + result.push_char('='); + result.push_wtf8(value.repr(vm)?.as_wtf8()); + } + } else { + result.push_str("..."); + } + result.push_char(')'); + Ok(result) + } + } +} diff --git a/crates/vm/src/stdlib/_signal.rs b/crates/vm/src/stdlib/_signal.rs new file mode 100644 index 00000000000..a69d766ce51 --- /dev/null +++ b/crates/vm/src/stdlib/_signal.rs @@ -0,0 +1,729 @@ +// spell-checker:disable + +pub(crate) use _signal::module_def; + +#[pymodule] +pub(crate) mod _signal { + #[cfg(any(unix, windows))] + use crate::convert::{IntoPyException, TryFromBorrowedObject}; + use crate::{Py, PyObjectRef, PyResult, VirtualMachine, signal}; + #[cfg(unix)] + use crate::{ + builtins::PyTypeRef, + function::{ArgIntoFloat, OptionalArg}, + }; + use core::sync::atomic::{self, Ordering}; + + #[cfg(any(unix, windows))] + use libc::sighandler_t; + #[allow(non_camel_case_types)] + #[cfg(not(any(unix, windows)))] + type sighandler_t = usize; + + cfg_if::cfg_if! { + if #[cfg(windows)] { + type WakeupFdRaw = libc::SOCKET; + struct WakeupFd(WakeupFdRaw); + const INVALID_WAKEUP: libc::SOCKET = windows_sys::Win32::Networking::WinSock::INVALID_SOCKET; + static WAKEUP: atomic::AtomicUsize = atomic::AtomicUsize::new(INVALID_WAKEUP); + // windows doesn't use the same fds for files and sockets like windows does, so we need + // this to know whether to send() or write() + static WAKEUP_IS_SOCKET: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); + + impl<'a> TryFromBorrowedObject<'a> for WakeupFd { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a crate::PyObject) -> PyResult<Self> { + use num_traits::One; + + let fd: &crate::Py<crate::builtins::PyInt> = obj.try_to_value(vm)?; + match fd.try_to_primitive::<usize>(vm) { + Ok(fd) => Ok(WakeupFd(fd as _)), + Err(e) => if (-fd.as_bigint()).is_one() { + Ok(WakeupFd(INVALID_WAKEUP)) + } else { + Err(e) + }, + } + } + } + } else { + type WakeupFdRaw = i32; + type WakeupFd = WakeupFdRaw; + const INVALID_WAKEUP: WakeupFd = -1; + static WAKEUP: atomic::AtomicI32 = atomic::AtomicI32::new(INVALID_WAKEUP); + } + } + + #[cfg(unix)] + pub use libc::SIG_ERR; + #[cfg(unix)] + pub use nix::unistd::alarm as sig_alarm; + + #[cfg(unix)] + #[pyattr] + pub use libc::{SIG_DFL, SIG_IGN}; + + // pthread_sigmask 'how' constants + #[cfg(unix)] + #[pyattr] + use libc::{SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK}; + + #[cfg(not(unix))] + #[pyattr] + pub const SIG_DFL: sighandler_t = 0; + #[cfg(not(unix))] + #[pyattr] + pub const SIG_IGN: sighandler_t = 1; + #[cfg(not(unix))] + #[allow(dead_code)] + pub const SIG_ERR: sighandler_t = -1 as _; + + #[cfg(all(unix, not(target_os = "redox")))] + unsafe extern "C" { + fn siginterrupt(sig: i32, flag: i32) -> i32; + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + mod ffi { + unsafe extern "C" { + pub fn getitimer(which: libc::c_int, curr_value: *mut libc::itimerval) -> libc::c_int; + pub fn setitimer( + which: libc::c_int, + new_value: *const libc::itimerval, + old_value: *mut libc::itimerval, + ) -> libc::c_int; + } + } + + #[pyattr] + use crate::signal::NSIG; + + #[cfg(any(unix, windows))] + #[pyattr] + pub use libc::{SIGABRT, SIGFPE, SIGILL, SIGINT, SIGSEGV, SIGTERM}; + + #[cfg(windows)] + #[pyattr] + const SIGBREAK: i32 = 21; // _SIGBREAK + + // Windows-specific control events for GenerateConsoleCtrlEvent + #[cfg(windows)] + #[pyattr] + const CTRL_C_EVENT: u32 = 0; + #[cfg(windows)] + #[pyattr] + const CTRL_BREAK_EVENT: u32 = 1; + + #[cfg(unix)] + #[pyattr] + use libc::{ + SIGALRM, SIGBUS, SIGCHLD, SIGCONT, SIGHUP, SIGIO, SIGKILL, SIGPIPE, SIGPROF, SIGQUIT, + SIGSTOP, SIGSYS, SIGTRAP, SIGTSTP, SIGTTIN, SIGTTOU, SIGURG, SIGUSR1, SIGUSR2, SIGVTALRM, + SIGWINCH, SIGXCPU, SIGXFSZ, + }; + + #[cfg(unix)] + #[cfg(not(any( + target_vendor = "apple", + target_os = "openbsd", + target_os = "freebsd", + target_os = "netbsd" + )))] + #[pyattr] + use libc::{SIGPWR, SIGSTKFLT}; + + // Interval timer constants + #[cfg(all(unix, not(target_os = "android")))] + #[pyattr] + use libc::{ITIMER_PROF, ITIMER_REAL, ITIMER_VIRTUAL}; + + #[cfg(target_os = "android")] + #[pyattr] + const ITIMER_REAL: libc::c_int = 0; + #[cfg(target_os = "android")] + #[pyattr] + const ITIMER_VIRTUAL: libc::c_int = 1; + #[cfg(target_os = "android")] + #[pyattr] + const ITIMER_PROF: libc::c_int = 2; + + #[cfg(unix)] + #[pyattr(name = "ItimerError", once)] + fn itimer_error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "signal", + "ItimerError", + Some(vec![vm.ctx.exceptions.os_error.to_owned()]), + ) + } + + #[cfg(any(unix, windows))] + pub(super) fn init_signal_handlers( + module: &Py<crate::builtins::PyModule>, + vm: &VirtualMachine, + ) { + if vm.state.config.settings.install_signal_handlers { + let sig_dfl = vm.new_pyobj(SIG_DFL as u8); + let sig_ign = vm.new_pyobj(SIG_IGN as u8); + + for signum in 1..NSIG { + let handler = unsafe { libc::signal(signum as i32, SIG_IGN) }; + if handler != SIG_ERR { + unsafe { libc::signal(signum as i32, handler) }; + } + let py_handler = if handler == SIG_DFL { + Some(sig_dfl.clone()) + } else if handler == SIG_IGN { + Some(sig_ign.clone()) + } else { + None + }; + vm.signal_handlers + .get_or_init(signal::new_signal_handlers) + .borrow_mut()[signum] = py_handler; + } + + let int_handler = module + .get_attr("default_int_handler", vm) + .expect("_signal does not have this attr?"); + signal(libc::SIGINT, int_handler, vm).expect("Failed to set sigint handler"); + } + } + + #[cfg(not(any(unix, windows)))] + #[pyfunction] + pub fn signal( + _signalnum: i32, + _handler: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + Err(vm.new_not_implemented_error("signal is not implemented on this platform")) + } + + #[cfg(any(unix, windows))] + #[pyfunction] + pub fn signal( + signalnum: i32, + handler: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + signal::assert_in_range(signalnum, vm)?; + #[cfg(windows)] + { + const VALID_SIGNALS: &[i32] = &[ + libc::SIGINT, + libc::SIGILL, + libc::SIGFPE, + libc::SIGSEGV, + libc::SIGTERM, + SIGBREAK, + libc::SIGABRT, + ]; + if !VALID_SIGNALS.contains(&signalnum) { + return Err(vm.new_value_error(format!("signal number {} out of range", signalnum))); + } + } + if !vm.is_main_thread() { + return Err(vm.new_value_error("signal only works in main thread")); + } + + let sig_handler = + match usize::try_from_borrowed_object(vm, &handler).ok() { + Some(SIG_DFL) => SIG_DFL, + Some(SIG_IGN) => SIG_IGN, + None if handler.is_callable() => run_signal as *const () as sighandler_t, + _ => return Err(vm.new_type_error( + "signal handler must be signal.SIG_IGN, signal.SIG_DFL, or a callable object", + )), + }; + signal::check_signals(vm)?; + + let old = unsafe { libc::signal(signalnum, sig_handler) }; + if old == SIG_ERR { + return Err(vm.new_os_error("Failed to set signal".to_owned())); + } + #[cfg(all(unix, not(target_os = "redox")))] + unsafe { + siginterrupt(signalnum, 1); + } + + let signal_handlers = vm.signal_handlers.get_or_init(signal::new_signal_handlers); + let old_handler = signal_handlers.borrow_mut()[signalnum as usize].replace(handler); + Ok(old_handler) + } + + #[pyfunction] + fn getsignal(signalnum: i32, vm: &VirtualMachine) -> PyResult { + signal::assert_in_range(signalnum, vm)?; + let signal_handlers = vm.signal_handlers.get_or_init(signal::new_signal_handlers); + let handler = signal_handlers.borrow()[signalnum as usize] + .clone() + .unwrap_or_else(|| vm.ctx.none()); + Ok(handler) + } + + #[cfg(unix)] + #[pyfunction] + fn alarm(time: u32) -> u32 { + let prev_time = if time == 0 { + sig_alarm::cancel() + } else { + sig_alarm::set(time) + }; + prev_time.unwrap_or(0) + } + + #[cfg(unix)] + #[pyfunction] + fn pause(vm: &VirtualMachine) -> PyResult<()> { + unsafe { libc::pause() }; + signal::check_signals(vm)?; + Ok(()) + } + + #[cfg(unix)] + fn timeval_to_double(tv: &libc::timeval) -> f64 { + tv.tv_sec as f64 + (tv.tv_usec as f64 / 1_000_000.0) + } + + #[cfg(unix)] + fn double_to_timeval(val: f64) -> libc::timeval { + libc::timeval { + tv_sec: val.trunc() as _, + tv_usec: ((val.fract()) * 1_000_000.0) as _, + } + } + + #[cfg(unix)] + fn itimerval_to_tuple(it: &libc::itimerval) -> (f64, f64) { + ( + timeval_to_double(&it.it_value), + timeval_to_double(&it.it_interval), + ) + } + + #[cfg(unix)] + #[pyfunction] + fn setitimer( + which: i32, + seconds: ArgIntoFloat, + interval: OptionalArg<ArgIntoFloat>, + vm: &VirtualMachine, + ) -> PyResult<(f64, f64)> { + let seconds: f64 = seconds.into(); + let interval: f64 = interval.map(|v| v.into()).unwrap_or(0.0); + let new = libc::itimerval { + it_value: double_to_timeval(seconds), + it_interval: double_to_timeval(interval), + }; + let mut old = core::mem::MaybeUninit::<libc::itimerval>::uninit(); + #[cfg(any(target_os = "linux", target_os = "android"))] + let ret = unsafe { ffi::setitimer(which, &new, old.as_mut_ptr()) }; + #[cfg(not(any(target_os = "linux", target_os = "android")))] + let ret = unsafe { libc::setitimer(which, &new, old.as_mut_ptr()) }; + if ret != 0 { + let err = std::io::Error::last_os_error(); + let itimer_error = itimer_error(vm); + return Err(vm.new_exception_msg(itimer_error, err.to_string().into())); + } + let old = unsafe { old.assume_init() }; + Ok(itimerval_to_tuple(&old)) + } + + #[cfg(unix)] + #[pyfunction] + fn getitimer(which: i32, vm: &VirtualMachine) -> PyResult<(f64, f64)> { + let mut old = core::mem::MaybeUninit::<libc::itimerval>::uninit(); + #[cfg(any(target_os = "linux", target_os = "android"))] + let ret = unsafe { ffi::getitimer(which, old.as_mut_ptr()) }; + #[cfg(not(any(target_os = "linux", target_os = "android")))] + let ret = unsafe { libc::getitimer(which, old.as_mut_ptr()) }; + if ret != 0 { + let err = std::io::Error::last_os_error(); + let itimer_error = itimer_error(vm); + return Err(vm.new_exception_msg(itimer_error, err.to_string().into())); + } + let old = unsafe { old.assume_init() }; + Ok(itimerval_to_tuple(&old)) + } + + #[pyfunction] + fn default_int_handler( + _signum: PyObjectRef, + _arg: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + Err(vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned())) + } + + #[derive(FromArgs)] + struct SetWakeupFdArgs { + fd: WakeupFd, + #[pyarg(named, default = true)] + warn_on_full_buffer: bool, + } + + #[pyfunction] + fn set_wakeup_fd(args: SetWakeupFdArgs, vm: &VirtualMachine) -> PyResult<i64> { + // TODO: implement warn_on_full_buffer + let _ = args.warn_on_full_buffer; + #[cfg(windows)] + let fd = args.fd.0; + #[cfg(not(windows))] + let fd = args.fd; + + if !vm.is_main_thread() { + return Err(vm.new_value_error("set_wakeup_fd only works in main thread")); + } + + #[cfg(windows)] + let is_socket = if fd != INVALID_WAKEUP { + use windows_sys::Win32::Networking::WinSock; + + crate::windows::init_winsock(); + let mut res = 0i32; + let mut res_size = core::mem::size_of::<i32>() as i32; + let res = unsafe { + WinSock::getsockopt( + fd, + WinSock::SOL_SOCKET, + WinSock::SO_ERROR, + &mut res as *mut i32 as *mut _, + &mut res_size, + ) + }; + // if getsockopt succeeded, fd is for sure a socket + let is_socket = res == 0; + if !is_socket { + let err = std::io::Error::last_os_error(); + // if getsockopt failed for some other reason, throw + if err.raw_os_error() != Some(WinSock::WSAENOTSOCK) { + return Err(err.into_pyexception(vm)); + } + // Validate that fd is a valid file descriptor using fstat + // First check if SOCKET can be safely cast to i32 (file descriptor) + let fd_i32 = i32::try_from(fd).map_err(|_| vm.new_value_error("invalid fd"))?; + // Verify the fd is valid by trying to fstat it + let borrowed_fd = + unsafe { crate::common::crt_fd::Borrowed::try_borrow_raw(fd_i32) } + .map_err(|e| e.into_pyexception(vm))?; + crate::common::fileutils::fstat(borrowed_fd).map_err(|e| e.into_pyexception(vm))?; + } + is_socket + } else { + false + }; + #[cfg(unix)] + if let Ok(fd) = unsafe { crate::common::crt_fd::Borrowed::try_borrow_raw(fd) } { + use nix::fcntl; + let oflags = fcntl::fcntl(fd, fcntl::F_GETFL).map_err(|e| e.into_pyexception(vm))?; + let nonblock = + fcntl::OFlag::from_bits_truncate(oflags).contains(fcntl::OFlag::O_NONBLOCK); + if !nonblock { + return Err(vm.new_value_error(format!( + "the fd {} must be in non-blocking mode", + fd.as_raw() + ))); + } + } + + let old_fd = WAKEUP.swap(fd, Ordering::Relaxed); + #[cfg(windows)] + WAKEUP_IS_SOCKET.store(is_socket, Ordering::Relaxed); + + #[cfg(windows)] + { + if old_fd == INVALID_WAKEUP { + Ok(-1) + } else { + Ok(old_fd as i64) + } + } + #[cfg(not(windows))] + { + Ok(old_fd as i64) + } + } + + #[cfg(target_os = "linux")] + #[pyfunction] + fn pidfd_send_signal( + pidfd: i32, + sig: i32, + siginfo: OptionalArg<PyObjectRef>, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult<()> { + signal::assert_in_range(sig, vm)?; + if let OptionalArg::Present(obj) = siginfo + && !vm.is_none(&obj) + { + return Err(vm.new_type_error("siginfo must be None")); + } + + let flags = flags.unwrap_or(0); + let ret = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + pidfd, + sig, + core::ptr::null::<libc::siginfo_t>(), + flags, + ) as libc::c_long + }; + + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyfunction(name = "siginterrupt")] + fn py_siginterrupt(signum: i32, flag: i32, vm: &VirtualMachine) -> PyResult<()> { + signal::assert_in_range(signum, vm)?; + let res = unsafe { siginterrupt(signum, flag) }; + if res < 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + /// CPython: signal_raise_signal (signalmodule.c) + #[cfg(any(unix, windows))] + #[pyfunction] + fn raise_signal(signalnum: i32, vm: &VirtualMachine) -> PyResult<()> { + signal::assert_in_range(signalnum, vm)?; + + // On Windows, only certain signals are supported + #[cfg(windows)] + { + // Windows supports: SIGINT(2), SIGILL(4), SIGFPE(8), SIGSEGV(11), SIGTERM(15), SIGBREAK(21), SIGABRT(22) + const VALID_SIGNALS: &[i32] = &[ + libc::SIGINT, + libc::SIGILL, + libc::SIGFPE, + libc::SIGSEGV, + libc::SIGTERM, + SIGBREAK, + libc::SIGABRT, + ]; + if !VALID_SIGNALS.contains(&signalnum) { + return Err(vm + .new_errno_error(libc::EINVAL, "Invalid argument") + .upcast()); + } + } + + let res = unsafe { libc::raise(signalnum) }; + if res != 0 { + return Err(vm.new_os_error(format!("raise_signal failed for signal {}", signalnum))); + } + + // Check if a signal was triggered and handle it + signal::check_signals(vm)?; + + Ok(()) + } + + /// CPython: signal_strsignal (signalmodule.c) + #[cfg(unix)] + #[pyfunction] + fn strsignal(signalnum: i32, vm: &VirtualMachine) -> PyResult<Option<String>> { + if signalnum < 1 || signalnum >= signal::NSIG as i32 { + return Err(vm.new_value_error(format!("signal number {} out of range", signalnum))); + } + let s = unsafe { libc::strsignal(signalnum) }; + if s.is_null() { + Ok(None) + } else { + let cstr = unsafe { core::ffi::CStr::from_ptr(s) }; + Ok(Some(cstr.to_string_lossy().into_owned())) + } + } + + #[cfg(windows)] + #[pyfunction] + fn strsignal(signalnum: i32, vm: &VirtualMachine) -> PyResult<Option<String>> { + if signalnum < 1 || signalnum >= signal::NSIG as i32 { + return Err(vm.new_value_error(format!("signal number {} out of range", signalnum))); + } + // Windows doesn't have strsignal(), provide our own mapping + let name = match signalnum { + libc::SIGINT => "Interrupt", + libc::SIGILL => "Illegal instruction", + libc::SIGFPE => "Floating-point exception", + libc::SIGSEGV => "Segmentation fault", + libc::SIGTERM => "Terminated", + SIGBREAK => "Break", + libc::SIGABRT => "Aborted", + _ => return Ok(None), + }; + Ok(Some(name.to_owned())) + } + + /// CPython: signal_valid_signals (signalmodule.c) + #[pyfunction] + fn valid_signals(vm: &VirtualMachine) -> PyResult { + use crate::PyPayload; + use crate::builtins::PySet; + let set = PySet::default().into_ref(&vm.ctx); + #[cfg(unix)] + { + // Use sigfillset to get all valid signals + let mut mask: libc::sigset_t = unsafe { core::mem::zeroed() }; + // SAFETY: mask is a valid pointer + if unsafe { libc::sigfillset(&mut mask) } != 0 { + return Err(vm.new_os_error("sigfillset failed".to_owned())); + } + // Convert the filled mask to a Python set + for signum in 1..signal::NSIG { + if unsafe { libc::sigismember(&mask, signum as i32) } == 1 { + set.add(vm.ctx.new_int(signum as i32).into(), vm)?; + } + } + } + #[cfg(windows)] + { + // Windows only supports a limited set of signals + for &signum in &[ + libc::SIGINT, + libc::SIGILL, + libc::SIGFPE, + libc::SIGSEGV, + libc::SIGTERM, + SIGBREAK, + libc::SIGABRT, + ] { + set.add(vm.ctx.new_int(signum).into(), vm)?; + } + } + #[cfg(not(any(unix, windows)))] + { + // Empty set for platforms without signal support (e.g., WASM) + let _ = &set; + } + Ok(set.into()) + } + + #[cfg(unix)] + fn sigset_to_pyset(mask: &libc::sigset_t, vm: &VirtualMachine) -> PyResult { + use crate::PyPayload; + use crate::builtins::PySet; + let set = PySet::default().into_ref(&vm.ctx); + for signum in 1..signal::NSIG { + // SAFETY: mask is a valid sigset_t + if unsafe { libc::sigismember(mask, signum as i32) } == 1 { + set.add(vm.ctx.new_int(signum as i32).into(), vm)?; + } + } + Ok(set.into()) + } + + #[cfg(unix)] + #[pyfunction] + fn pthread_sigmask( + how: i32, + mask: crate::function::ArgIterable, + vm: &VirtualMachine, + ) -> PyResult { + use crate::convert::IntoPyException; + + // Initialize sigset + let mut sigset: libc::sigset_t = unsafe { core::mem::zeroed() }; + // SAFETY: sigset is a valid pointer + if unsafe { libc::sigemptyset(&mut sigset) } != 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + + // Add signals to the set + for sig in mask.iter(vm)? { + let sig = sig?; + // Convert to i32, handling overflow by returning ValueError + let signum: i32 = sig.try_to_value(vm).map_err(|_| { + vm.new_value_error(format!( + "signal number out of range [1, {}]", + signal::NSIG - 1 + )) + })?; + // Validate signal number is in range [1, NSIG) + if signum < 1 || signum >= signal::NSIG as i32 { + return Err(vm.new_value_error(format!( + "signal number {} out of range [1, {}]", + signum, + signal::NSIG - 1 + ))); + } + // SAFETY: sigset is a valid pointer and signum is validated + if unsafe { libc::sigaddset(&mut sigset, signum) } != 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + } + + // Call pthread_sigmask + let mut old_mask: libc::sigset_t = unsafe { core::mem::zeroed() }; + // SAFETY: all pointers are valid + let err = unsafe { libc::pthread_sigmask(how, &sigset, &mut old_mask) }; + if err != 0 { + return Err(std::io::Error::from_raw_os_error(err).into_pyexception(vm)); + } + + // Check for pending signals + signal::check_signals(vm)?; + + // Convert old mask to Python set + sigset_to_pyset(&old_mask, vm) + } + + #[cfg(any(unix, windows))] + pub extern "C" fn run_signal(signum: i32) { + signal::TRIGGERS[signum as usize].store(true, Ordering::Relaxed); + signal::set_triggered(); + #[cfg(windows)] + if signum == libc::SIGINT + && let Some(handle) = signal::get_sigint_event() + { + unsafe { + windows_sys::Win32::System::Threading::SetEvent(handle as _); + } + } + let wakeup_fd = WAKEUP.load(Ordering::Relaxed); + if wakeup_fd != INVALID_WAKEUP { + let sigbyte = signum as u8; + #[cfg(windows)] + if WAKEUP_IS_SOCKET.load(Ordering::Relaxed) { + let _res = unsafe { + windows_sys::Win32::Networking::WinSock::send( + wakeup_fd, + &sigbyte as *const u8 as *const _, + 1, + 0, + ) + }; + return; + } + let _res = unsafe { libc::write(wakeup_fd as _, &sigbyte as *const u8 as *const _, 1) }; + // TODO: handle _res < 1, support warn_on_full_buffer + } + } + + /// Reset wakeup fd after fork in child process. + /// The child must not write to the parent's wakeup fd. + #[cfg(unix)] + pub(crate) fn clear_wakeup_fd_after_fork() { + WAKEUP.store(INVALID_WAKEUP, Ordering::Relaxed); + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + + #[cfg(any(unix, windows))] + init_signal_handlers(module, vm); + + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/_sre.rs b/crates/vm/src/stdlib/_sre.rs new file mode 100644 index 00000000000..ba7044fb5a9 --- /dev/null +++ b/crates/vm/src/stdlib/_sre.rs @@ -0,0 +1,930 @@ +pub(crate) use _sre::module_def; + +#[pymodule] +mod _sre { + use crate::{ + Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, + TryFromObject, VirtualMachine, atomic_func, + builtins::{ + PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyListRef, PyStr, + PyStrRef, PyTuple, PyTupleRef, PyTypeRef, + }, + common::wtf8::{Wtf8, Wtf8Buf, wtf8_concat}, + common::{ascii, hash::PyHash}, + convert::ToPyObject, + function::{ArgCallable, OptionalArg, PosArgs, PyComparisonValue}, + protocol::{PyBuffer, PyCallable, PyMappingMethods}, + stdlib::sys, + types::{AsMapping, Comparable, Hashable, Representable}, + }; + use core::str; + use crossbeam_utils::atomic::AtomicCell; + use itertools::Itertools; + use num_traits::ToPrimitive; + use rustpython_sre_engine::{ + Request, SearchIter, SreFlag, State, StrDrive, + string::{lower_ascii, lower_unicode, upper_unicode}, + }; + + #[pyattr] + pub use rustpython_sre_engine::{CODESIZE, MAXGROUPS, MAXREPEAT, SRE_MAGIC as MAGIC}; + + #[pyfunction] + const fn getcodesize() -> usize { + CODESIZE + } + + #[pyfunction] + fn ascii_iscased(ch: i32) -> bool { + (b'a' as i32..=b'z' as i32).contains(&ch) || (b'A' as i32..=b'Z' as i32).contains(&ch) + } + + #[pyfunction] + fn unicode_iscased(ch: i32) -> bool { + let ch = ch as u32; + ch != lower_unicode(ch) || ch != upper_unicode(ch) + } + + #[pyfunction] + fn ascii_tolower(ch: i32) -> i32 { + lower_ascii(ch as u32) as i32 + } + + #[pyfunction] + fn unicode_tolower(ch: i32) -> i32 { + lower_unicode(ch as u32) as i32 + } + + trait SreStr: StrDrive { + fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef; + + fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request<'_, Self> { + Request::new(self, start, end, &pattern.code, false) + } + } + + impl SreStr for &[u8] { + fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .new_bytes(self.iter().take(end).skip(start).cloned().collect()) + .into() + } + } + + impl SreStr for &Wtf8 { + fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .new_str( + self.code_points() + .take(end) + .skip(start) + .collect::<Wtf8Buf>(), + ) + .into() + } + } + + #[pyfunction] + fn compile( + pattern: PyObjectRef, + flags: u16, + code: PyObjectRef, + groups: usize, + groupindex: PyDictRef, + indexgroup: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<Pattern> { + // FIXME: + // pattern could only be None if called by re.Scanner + // re.Scanner has no official API and in CPython's implement + // isbytes will be hanging (-1) + // here is just a hack to let re.Scanner works only with str not bytes + let isbytes = !vm.is_none(&pattern) && !pattern.downcastable::<PyStr>(); + let code = code.try_to_value(vm)?; + Ok(Pattern { + pattern, + flags: SreFlag::from_bits_truncate(flags), + code, + groups, + groupindex, + indexgroup: indexgroup.try_to_value(vm)?, + isbytes, + }) + } + + #[pyattr] + #[pyclass(name = "SRE_Template")] + #[derive(Debug, PyPayload)] + struct Template { + literal: PyObjectRef, + items: Vec<(usize, PyObjectRef)>, + } + + #[pyclass] + impl Template { + fn compile( + pattern: PyRef<Pattern>, + repl: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + let re = vm.import("re", 0)?; + let func = re.get_attr("_compile_template", vm)?; + let result = func.call((pattern, repl.clone()), vm)?; + result + .downcast::<Self>() + .map_err(|_| vm.new_runtime_error("expected SRE_Template")) + } + } + + #[pyfunction] + fn template( + _pattern: PyObjectRef, + template: PyListRef, + vm: &VirtualMachine, + ) -> PyResult<Template> { + let err = || vm.new_type_error("invalid template"); + + let mut items = Vec::with_capacity(1); + let v = template.borrow_vec(); + let literal = v.first().ok_or_else(err)?.clone(); + let trunks = v[1..].chunks_exact(2); + + if !trunks.remainder().is_empty() { + return Err(err()); + } + + for trunk in trunks { + let index: usize = trunk[0] + .downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error("expected usize"))? + .try_to_primitive(vm)?; + items.push((index, trunk[1].clone())); + } + + Ok(Template { literal, items }) + } + + #[derive(FromArgs)] + struct StringArgs { + string: PyObjectRef, + #[pyarg(any, default = 0)] + pos: usize, + #[pyarg(any, default = sys::MAXSIZE as usize)] + endpos: usize, + } + + #[derive(FromArgs)] + struct SubArgs { + // repl: Either<ArgCallable, PyStrRef>, + repl: PyObjectRef, + string: PyObjectRef, + #[pyarg(any, default = 0)] + count: usize, + } + + #[derive(FromArgs)] + struct SplitArgs { + string: PyObjectRef, + #[pyarg(any, default = 0)] + maxsplit: isize, + } + + #[pyattr] + #[pyclass(module = "re", name = "Pattern")] + #[derive(Debug, PyPayload)] + pub(crate) struct Pattern { + pub pattern: PyObjectRef, + pub flags: SreFlag, + pub code: Vec<u32>, + pub groups: usize, + pub groupindex: PyDictRef, + pub indexgroup: Vec<Option<PyStrRef>>, + pub isbytes: bool, + } + + macro_rules! with_sre_str { + ($pattern:expr, $string:expr, $vm:expr, $f:expr) => { + if $pattern.isbytes { + Pattern::with_bytes($string, $vm, $f) + } else { + Pattern::with_str($string, $vm, $f) + } + }; + } + + #[pyclass(with(Hashable, Comparable, Representable), flags(HAS_WEAKREF))] + impl Pattern { + fn with_str<F, R>(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult<R> + where + F: FnOnce(&Wtf8) -> PyResult<R>, + { + let string = string.downcast_ref::<PyStr>().ok_or_else(|| { + vm.new_type_error(format!("expected string got '{}'", string.class())) + })?; + f(string.as_wtf8()) + } + + fn with_bytes<F, R>(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult<R> + where + F: FnOnce(&[u8]) -> PyResult<R>, + { + PyBuffer::try_from_borrowed_object(vm, string)?.contiguous_or_collect(f) + } + + #[pymethod(name = "match")] + fn py_match( + zelf: PyRef<Self>, + string_args: StringArgs, + vm: &VirtualMachine, + ) -> PyResult<Option<PyRef<Match>>> { + let StringArgs { + string, + pos, + endpos, + } = string_args; + with_sre_str!(zelf, &string.clone(), vm, |x| { + let req = x.create_request(&zelf, pos, endpos); + let mut state = State::default(); + Ok(state + .py_match(&req) + .then(|| Match::new(&mut state, zelf.clone(), string).into_ref(&vm.ctx))) + }) + } + + #[pymethod] + fn fullmatch( + zelf: PyRef<Self>, + string_args: StringArgs, + vm: &VirtualMachine, + ) -> PyResult<Option<PyRef<Match>>> { + with_sre_str!(zelf, &string_args.string.clone(), vm, |x| { + let mut req = x.create_request(&zelf, string_args.pos, string_args.endpos); + req.match_all = true; + let mut state = State::default(); + Ok(state.py_match(&req).then(|| { + Match::new(&mut state, zelf.clone(), string_args.string).into_ref(&vm.ctx) + })) + }) + } + + #[pymethod] + fn search( + zelf: PyRef<Self>, + string_args: StringArgs, + vm: &VirtualMachine, + ) -> PyResult<Option<PyRef<Match>>> { + with_sre_str!(zelf, &string_args.string.clone(), vm, |x| { + let req = x.create_request(&zelf, string_args.pos, string_args.endpos); + let mut state = State::default(); + Ok(state.search(req).then(|| { + Match::new(&mut state, zelf.clone(), string_args.string).into_ref(&vm.ctx) + })) + }) + } + + #[pymethod] + fn findall( + zelf: PyRef<Self>, + string_args: StringArgs, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + with_sre_str!(zelf, &string_args.string, vm, |s| { + let req = s.create_request(&zelf, string_args.pos, string_args.endpos); + let state = State::default(); + let mut match_list: Vec<PyObjectRef> = Vec::new(); + let mut iter = SearchIter { req, state }; + + while iter.next().is_some() { + let m = Match::new(&mut iter.state, zelf.clone(), string_args.string.clone()); + + let item = if zelf.groups == 0 || zelf.groups == 1 { + m.get_slice(zelf.groups, s, vm) + .unwrap_or_else(|| vm.ctx.none()) + } else { + m.groups(OptionalArg::Present(vm.ctx.new_str(ascii!("")).into()), vm)? + .into() + }; + + match_list.push(item); + } + + Ok(match_list) + }) + } + + #[pymethod] + fn finditer( + zelf: PyRef<Self>, + string_args: StringArgs, + vm: &VirtualMachine, + ) -> PyResult<PyCallableIterator> { + let scanner = SreScanner { + pattern: zelf, + string: string_args.string, + start: AtomicCell::new(string_args.pos), + end: string_args.endpos, + must_advance: AtomicCell::new(false), + } + .into_ref(&vm.ctx); + let search = vm.get_str_method(scanner.into(), "search").unwrap()?; + let search = ArgCallable::try_from_object(vm, search)?; + let iterator = PyCallableIterator::new(search, vm.ctx.none()); + Ok(iterator) + } + + #[pymethod] + fn scanner( + zelf: PyRef<Self>, + string_args: StringArgs, + vm: &VirtualMachine, + ) -> PyRef<SreScanner> { + SreScanner { + pattern: zelf, + string: string_args.string, + start: AtomicCell::new(string_args.pos), + end: string_args.endpos, + must_advance: AtomicCell::new(false), + } + .into_ref(&vm.ctx) + } + + #[pymethod] + fn sub(zelf: PyRef<Self>, sub_args: SubArgs, vm: &VirtualMachine) -> PyResult { + Self::sub_impl(zelf, sub_args, false, vm) + } + + #[pymethod] + fn subn(zelf: PyRef<Self>, sub_args: SubArgs, vm: &VirtualMachine) -> PyResult { + Self::sub_impl(zelf, sub_args, true, vm) + } + + #[pymethod] + fn split( + zelf: PyRef<Self>, + split_args: SplitArgs, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + with_sre_str!(zelf, &split_args.string, vm, |s| { + let req = s.create_request(&zelf, 0, usize::MAX); + let state = State::default(); + let mut split_list: Vec<PyObjectRef> = Vec::new(); + let mut iter = SearchIter { req, state }; + let mut n = 0; + let mut last = 0; + + while (split_args.maxsplit == 0 || n < split_args.maxsplit) && iter.next().is_some() + { + /* get segment before this match */ + split_list.push(s.slice(last, iter.state.start, vm)); + + let m = Match::new(&mut iter.state, zelf.clone(), split_args.string.clone()); + + // add groups (if any) + for i in 1..=zelf.groups { + split_list.push(m.get_slice(i, s, vm).unwrap_or_else(|| vm.ctx.none())); + } + + n += 1; + last = iter.state.cursor.position; + } + + // get segment following last match (even if empty) + split_list.push(req.string.slice(last, s.count(), vm)); + + Ok(split_list) + }) + } + + #[pygetset] + const fn flags(&self) -> u16 { + self.flags.bits() + } + + #[pygetset] + fn groupindex(&self) -> PyDictRef { + self.groupindex.clone() + } + + #[pygetset] + const fn groups(&self) -> usize { + self.groups + } + + #[pygetset] + fn pattern(&self) -> PyObjectRef { + self.pattern.clone() + } + + fn sub_impl( + zelf: PyRef<Self>, + sub_args: SubArgs, + subn: bool, + vm: &VirtualMachine, + ) -> PyResult { + let SubArgs { + repl, + string, + count, + } = sub_args; + + enum FilterType<'a> { + Literal(PyObjectRef), + Callable(PyCallable<'a>), + Template(PyRef<Template>), + } + + let filter = if let Some(callable) = repl.to_callable() { + FilterType::Callable(callable) + } else { + let is_template = if zelf.isbytes { + Self::with_bytes(&repl, vm, |x| Ok(x.contains(&b'\\')))? + } else { + Self::with_str(&repl, vm, |x| Ok(x.contains("\\".as_ref())))? + }; + + if is_template { + FilterType::Template(Template::compile(zelf.clone(), repl, vm)?) + } else { + FilterType::Literal(repl) + } + }; + + with_sre_str!(zelf, &string, vm, |s| { + let req = s.create_request(&zelf, 0, usize::MAX); + let state = State::default(); + let mut sub_list: Vec<PyObjectRef> = Vec::new(); + let mut iter = SearchIter { req, state }; + let mut n = 0; + let mut last_pos = 0; + + while (count == 0 || n < count) && iter.next().is_some() { + if last_pos < iter.state.start { + /* get segment before this match */ + sub_list.push(s.slice(last_pos, iter.state.start, vm)); + } + + match &filter { + FilterType::Literal(literal) => sub_list.push(literal.clone()), + FilterType::Callable(callable) => { + let m = Match::new(&mut iter.state, zelf.clone(), string.clone()) + .into_ref(&vm.ctx); + sub_list.push(callable.invoke((m,), vm)?); + } + FilterType::Template(template) => { + let m = Match::new(&mut iter.state, zelf.clone(), string.clone()); + // template.expand(m)? + // let mut list = vec![template.literal.clone()]; + sub_list.push(template.literal.clone()); + for (index, literal) in template.items.iter().cloned() { + if let Some(item) = m.get_slice(index, s, vm) { + sub_list.push(item); + } + sub_list.push(literal); + } + } + }; + + last_pos = iter.state.cursor.position; + n += 1; + } + + /* get segment following last match */ + sub_list.push(s.slice(last_pos, iter.req.end, vm)); + + let list = PyList::from(sub_list).into_pyobject(vm); + + let join_type: PyObjectRef = if zelf.isbytes { + vm.ctx.new_bytes(vec![]).into() + } else { + vm.ctx.new_str(ascii!("")).into() + }; + let ret = vm.call_method(&join_type, "join", (list,))?; + + Ok(if subn { (ret, n).to_pyobject(vm) } else { ret }) + }) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl Hashable for Pattern { + fn hash(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { + let hash = zelf.pattern.hash(vm)?; + let (_, code, _) = unsafe { zelf.code.align_to::<u8>() }; + let hash = hash ^ vm.state.hash_secret.hash_bytes(code); + let hash = hash ^ (zelf.flags.bits() as PyHash); + let hash = hash ^ (zelf.isbytes as i64); + Ok(hash) + } + } + + impl Comparable for Pattern { + fn cmp( + zelf: &crate::Py<Self>, + other: &PyObject, + op: crate::types::PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + if let Some(res) = op.identical_optimization(zelf, other) { + return Ok(res.into()); + } + op.eq_only(|| { + if let Some(other) = other.downcast_ref::<Self>() { + Ok(PyComparisonValue::Implemented( + zelf.flags == other.flags + && zelf.isbytes == other.isbytes + && zelf.code == other.code + && vm.bool_eq(&zelf.pattern, &other.pattern)?, + )) + } else { + Ok(PyComparisonValue::NotImplemented) + } + }) + } + } + + impl Representable for Pattern { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let flag_names = [ + ("re.IGNORECASE", SreFlag::IGNORECASE), + ("re.LOCALE", SreFlag::LOCALE), + ("re.MULTILINE", SreFlag::MULTILINE), + ("re.DOTALL", SreFlag::DOTALL), + ("re.UNICODE", SreFlag::UNICODE), + ("re.VERBOSE", SreFlag::VERBOSE), + ("re.DEBUG", SreFlag::DEBUG), + ("re.ASCII", SreFlag::ASCII), + ]; + + /* Omit re.UNICODE for valid string patterns. */ + let mut flags = zelf.flags; + if !zelf.isbytes + && (flags & (SreFlag::LOCALE | SreFlag::UNICODE | SreFlag::ASCII)) + == SreFlag::UNICODE + { + flags &= !SreFlag::UNICODE; + } + + let flags = flag_names + .iter() + .filter(|(_, flag)| flags.contains(*flag)) + .map(|(name, _)| name) + .join("|"); + + let pattern = zelf.pattern.repr(vm)?; + let mut result = Wtf8Buf::from("re.compile("); + let pat = if pattern.char_len() > 200 { + pattern.as_wtf8().code_points().take(200).collect() + } else { + pattern.as_wtf8().to_owned() + }; + result.push_wtf8(&pat); + if !flags.is_empty() { + result.push_str(", "); + result.push_str(&flags); + } + result.push_char(')'); + Ok(result) + } + } + + #[pyattr] + #[pyclass(module = "re", name = "Match")] + #[derive(Debug, PyPayload)] + pub(crate) struct Match { + string: PyObjectRef, + pattern: PyRef<Pattern>, + pos: usize, + endpos: usize, + lastindex: isize, + regs: Vec<(isize, isize)>, + } + + #[pyclass(with(AsMapping, Representable))] + impl Match { + pub(crate) fn new(state: &mut State, pattern: PyRef<Pattern>, string: PyObjectRef) -> Self { + let string_position = state.cursor.position; + let marks = &state.marks; + let mut regs = vec![(state.start as isize, string_position as isize)]; + for group in 0..pattern.groups { + let mark_index = 2 * group; + if mark_index + 1 < marks.raw().len() { + let start = marks.raw()[mark_index]; + let end = marks.raw()[mark_index + 1]; + if start.is_some() && end.is_some() { + regs.push((start.unpack() as isize, end.unpack() as isize)); + continue; + } + } + regs.push((-1, -1)); + } + Self { + string, + pattern, + pos: state.start, + endpos: string_position, + lastindex: marks.last_index(), + regs, + } + } + + #[pygetset] + const fn pos(&self) -> usize { + self.pos + } + + #[pygetset] + const fn endpos(&self) -> usize { + self.endpos + } + + #[pygetset] + const fn lastindex(&self) -> Option<isize> { + if self.lastindex >= 0 { + Some(self.lastindex) + } else { + None + } + } + + #[pygetset] + fn lastgroup(&self) -> Option<PyStrRef> { + let i = self.lastindex.to_usize()?; + self.pattern.indexgroup.get(i)?.clone() + } + + #[pygetset] + fn re(&self) -> PyRef<Pattern> { + self.pattern.clone() + } + + #[pygetset] + fn string(&self) -> PyObjectRef { + self.string.clone() + } + + #[pygetset] + fn regs(&self, vm: &VirtualMachine) -> PyTupleRef { + PyTuple::new_ref( + self.regs.iter().map(|&x| x.to_pyobject(vm)).collect(), + &vm.ctx, + ) + } + + #[pymethod] + fn start(&self, group: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<isize> { + self.span(group, vm).map(|x| x.0) + } + + #[pymethod] + fn end(&self, group: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<isize> { + self.span(group, vm).map(|x| x.1) + } + + #[pymethod] + fn span( + &self, + group: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<(isize, isize)> { + let index = group.map_or(Ok(0), |group| { + self.get_index(group, vm) + .ok_or_else(|| vm.new_index_error("no such group")) + })?; + Ok(self.regs[index]) + } + + #[pymethod] + fn expand(zelf: PyRef<Self>, template: PyStrRef, vm: &VirtualMachine) -> PyResult { + let re = vm.import("re", 0)?; + let func = re.get_attr("_expand", vm)?; + func.call((zelf.pattern.clone(), zelf, template), vm) + } + + #[pymethod] + fn group(&self, args: PosArgs<PyObjectRef>, vm: &VirtualMachine) -> PyResult { + with_sre_str!(self.pattern, &self.string, vm, |str_drive| { + let args = args.into_vec(); + if args.is_empty() { + return Ok(self.get_slice(0, str_drive, vm).unwrap().to_pyobject(vm)); + } + let mut v: Vec<PyObjectRef> = args + .into_iter() + .map(|x| { + self.get_index(x, vm) + .ok_or_else(|| vm.new_index_error("no such group")) + .map(|index| { + self.get_slice(index, str_drive, vm) + .map(|x| x.to_pyobject(vm)) + .unwrap_or_else(|| vm.ctx.none()) + }) + }) + .try_collect()?; + if v.len() == 1 { + Ok(v.pop().unwrap()) + } else { + Ok(vm.ctx.new_tuple(v).into()) + } + }) + } + + fn __getitem__( + &self, + group: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + with_sre_str!(self.pattern, &self.string, vm, |str_drive| { + let i = self + .get_index(group, vm) + .ok_or_else(|| vm.new_index_error("no such group"))?; + Ok(self.get_slice(i, str_drive, vm)) + }) + } + + #[pymethod] + fn groups( + &self, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyTupleRef> { + let default = default.unwrap_or_else(|| vm.ctx.none()); + + with_sre_str!(self.pattern, &self.string, vm, |str_drive| { + let v: Vec<PyObjectRef> = (1..self.regs.len()) + .map(|i| { + self.get_slice(i, str_drive, vm) + .map(|s| s.to_pyobject(vm)) + .unwrap_or_else(|| default.clone()) + }) + .collect(); + Ok(PyTuple::new_ref(v, &vm.ctx)) + }) + } + + #[pymethod] + fn groupdict( + &self, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyDictRef> { + let default = default.unwrap_or_else(|| vm.ctx.none()); + + with_sre_str!(self.pattern, &self.string, vm, |str_drive| { + let dict = vm.ctx.new_dict(); + + for (key, index) in self.pattern.groupindex.clone() { + let value = self + .get_index(index, vm) + .and_then(|x| self.get_slice(x, str_drive, vm)) + .map(|x| x.to_pyobject(vm)) + .unwrap_or_else(|| default.clone()); + dict.set_item(&*key, value, vm)?; + } + Ok(dict) + }) + } + + fn get_index(&self, group: PyObjectRef, vm: &VirtualMachine) -> Option<usize> { + let i = if let Ok(i) = group.try_index(vm) { + i + } else { + self.pattern + .groupindex + .get_item_opt(&*group, vm) + .ok()?? + .downcast::<PyInt>() + .ok()? + }; + let i = i.as_bigint().to_isize()?; + if i >= 0 && i as usize <= self.pattern.groups { + Some(i as usize) + } else { + None + } + } + + fn get_slice<S: SreStr>( + &self, + index: usize, + str_drive: S, + vm: &VirtualMachine, + ) -> Option<PyObjectRef> { + let (start, end) = self.regs[index]; + if start < 0 || end < 0 { + return None; + } + Some(str_drive.slice(start as usize, end as usize, vm)) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl AsMapping for Match { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: crate::common::lock::LazyLock<PyMappingMethods> = + crate::common::lock::LazyLock::new(|| PyMappingMethods { + subscript: atomic_func!(|mapping, needle, vm| { + Match::mapping_downcast(mapping) + .__getitem__(needle.to_owned(), vm) + .map(|x| x.to_pyobject(vm)) + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } + } + + impl Representable for Match { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + with_sre_str!(zelf.pattern, &zelf.string, vm, |str_drive| { + let match_repr = zelf.get_slice(0, str_drive, vm).unwrap().repr(vm)?; + Ok(wtf8_concat!( + "<re.Match object; span=(", + zelf.regs[0].0, + ", ", + zelf.regs[0].1, + "), match=", + match_repr.as_wtf8(), + '>', + )) + }) + } + } + + #[pyattr] + #[pyclass(name = "SRE_Scanner")] + #[derive(Debug, PyPayload)] + struct SreScanner { + pattern: PyRef<Pattern>, + string: PyObjectRef, + start: AtomicCell<usize>, + end: usize, + must_advance: AtomicCell<bool>, + } + + #[pyclass] + impl SreScanner { + #[pygetset] + fn pattern(&self) -> PyRef<Pattern> { + self.pattern.clone() + } + + #[pymethod(name = "match")] + fn py_match(&self, vm: &VirtualMachine) -> PyResult<Option<PyRef<Match>>> { + with_sre_str!(self.pattern, &self.string.clone(), vm, |s| { + let mut req = s.create_request(&self.pattern, self.start.load(), self.end); + let mut state = State::default(); + req.must_advance = self.must_advance.load(); + let has_matched = state.py_match(&req); + + self.must_advance + .store(state.cursor.position == state.start); + self.start.store(state.cursor.position); + + Ok(has_matched.then(|| { + Match::new(&mut state, self.pattern.clone(), self.string.clone()) + .into_ref(&vm.ctx) + })) + }) + } + + #[pymethod] + fn search(&self, vm: &VirtualMachine) -> PyResult<Option<PyRef<Match>>> { + if self.start.load() > self.end { + return Ok(None); + } + with_sre_str!(self.pattern, &self.string.clone(), vm, |s| { + let mut req = s.create_request(&self.pattern, self.start.load(), self.end); + let mut state = State::default(); + req.must_advance = self.must_advance.load(); + + let has_matched = state.search(req); + + self.must_advance + .store(state.cursor.position == state.start); + self.start.store(state.cursor.position); + + Ok(has_matched.then(|| { + Match::new(&mut state, self.pattern.clone(), self.string.clone()) + .into_ref(&vm.ctx) + })) + }) + } + } +} diff --git a/crates/vm/src/stdlib/_stat.rs b/crates/vm/src/stdlib/_stat.rs new file mode 100644 index 00000000000..44b55628d6f --- /dev/null +++ b/crates/vm/src/stdlib/_stat.rs @@ -0,0 +1,524 @@ +pub(crate) use _stat::module_def; + +#[pymodule] +mod _stat { + // Use libc::mode_t for Mode to match the system's definition + #[cfg(unix)] + type Mode = libc::mode_t; + #[cfg(windows)] + type Mode = u16; // Windows does not have mode_t, but stat constants are u16 + #[cfg(not(any(unix, windows)))] + type Mode = u32; // Fallback for unknown targets + + // libc_const macro for conditional compilation + macro_rules! libc_const { + (#[cfg($cfg:meta)] $name:ident, $fallback:expr) => {{ + #[cfg($cfg)] + { + libc::$name + } + #[cfg(not($cfg))] + { + $fallback + } + }}; + } + + #[pyattr] + pub const S_IFDIR: Mode = libc_const!( + #[cfg(unix)] + S_IFDIR, + 0o040000 + ); + + #[pyattr] + pub const S_IFCHR: Mode = libc_const!( + #[cfg(unix)] + S_IFCHR, + 0o020000 + ); + + #[pyattr] + pub const S_IFBLK: Mode = libc_const!( + #[cfg(unix)] + S_IFBLK, + 0o060000 + ); + + #[pyattr] + pub const S_IFREG: Mode = libc_const!( + #[cfg(unix)] + S_IFREG, + 0o100000 + ); + + #[pyattr] + pub const S_IFIFO: Mode = libc_const!( + #[cfg(unix)] + S_IFIFO, + 0o010000 + ); + + #[pyattr] + pub const S_IFLNK: Mode = libc_const!( + #[cfg(unix)] + S_IFLNK, + 0o120000 + ); + + #[pyattr] + pub const S_IFSOCK: Mode = libc_const!( + #[cfg(unix)] + S_IFSOCK, + 0o140000 + ); + + #[pyattr] + pub const S_IFDOOR: Mode = 0; // TODO: RUSTPYTHON Support Solaris + + #[pyattr] + pub const S_IFPORT: Mode = 0; // TODO: RUSTPYTHON Support Solaris + + // TODO: RUSTPYTHON Support BSD + // https://man.freebsd.org/cgi/man.cgi?stat(2) + + #[pyattr] + pub const S_IFWHT: Mode = if cfg!(target_os = "macos") { + 0o160000 + } else { + 0 + }; + + // Permission bits + + #[pyattr] + pub const S_ISUID: Mode = libc_const!( + #[cfg(unix)] + S_ISUID, + 0o4000 + ); + + #[pyattr] + pub const S_ISGID: Mode = libc_const!( + #[cfg(unix)] + S_ISGID, + 0o2000 + ); + + #[pyattr] + pub const S_ENFMT: Mode = libc_const!( + #[cfg(unix)] + S_ISGID, + 0o2000 + ); + + #[pyattr] + pub const S_ISVTX: Mode = libc_const!( + #[cfg(unix)] + S_ISVTX, + 0o1000 + ); + + #[pyattr] + pub const S_IRWXU: Mode = libc_const!( + #[cfg(unix)] + S_IRWXU, + 0o0700 + ); + + #[pyattr] + pub const S_IRUSR: Mode = libc_const!( + #[cfg(unix)] + S_IRUSR, + 0o0400 + ); + + #[pyattr] + pub const S_IREAD: Mode = libc_const!( + #[cfg(unix)] + S_IRUSR, + 0o0400 + ); + + #[pyattr] + pub const S_IWUSR: Mode = libc_const!( + #[cfg(unix)] + S_IWUSR, + 0o0200 + ); + + #[pyattr] + pub const S_IXUSR: Mode = libc_const!( + #[cfg(unix)] + S_IXUSR, + 0o0100 + ); + + #[pyattr] + pub const S_IRWXG: Mode = libc_const!( + #[cfg(unix)] + S_IRWXG, + 0o0070 + ); + + #[pyattr] + pub const S_IRGRP: Mode = libc_const!( + #[cfg(unix)] + S_IRGRP, + 0o0040 + ); + + #[pyattr] + pub const S_IWGRP: Mode = libc_const!( + #[cfg(unix)] + S_IWGRP, + 0o0020 + ); + + #[pyattr] + pub const S_IXGRP: Mode = libc_const!( + #[cfg(unix)] + S_IXGRP, + 0o0010 + ); + + #[pyattr] + pub const S_IRWXO: Mode = libc_const!( + #[cfg(unix)] + S_IRWXO, + 0o0007 + ); + + #[pyattr] + pub const S_IROTH: Mode = libc_const!( + #[cfg(unix)] + S_IROTH, + 0o0004 + ); + + #[pyattr] + pub const S_IWOTH: Mode = libc_const!( + #[cfg(unix)] + S_IWOTH, + 0o0002 + ); + + #[pyattr] + pub const S_IXOTH: Mode = libc_const!( + #[cfg(unix)] + S_IXOTH, + 0o0001 + ); + + #[pyattr] + pub const S_IWRITE: Mode = libc_const!( + #[cfg(all(unix, not(target_os = "android"), not(target_os = "redox")))] + S_IWRITE, + 0o0200 + ); + + #[pyattr] + pub const S_IEXEC: Mode = libc_const!( + #[cfg(all(unix, not(target_os = "android"), not(target_os = "redox")))] + S_IEXEC, + 0o0100 + ); + + // Windows file attributes (if on Windows) + + #[cfg(windows)] + #[pyattr] + pub use windows_sys::Win32::Storage::FileSystem::{ + FILE_ATTRIBUTE_ARCHIVE, FILE_ATTRIBUTE_COMPRESSED, FILE_ATTRIBUTE_DEVICE, + FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_ENCRYPTED, FILE_ATTRIBUTE_HIDDEN, + FILE_ATTRIBUTE_INTEGRITY_STREAM, FILE_ATTRIBUTE_NO_SCRUB_DATA, FILE_ATTRIBUTE_NORMAL, + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED, FILE_ATTRIBUTE_OFFLINE, FILE_ATTRIBUTE_READONLY, + FILE_ATTRIBUTE_REPARSE_POINT, FILE_ATTRIBUTE_SPARSE_FILE, FILE_ATTRIBUTE_SYSTEM, + FILE_ATTRIBUTE_TEMPORARY, FILE_ATTRIBUTE_VIRTUAL, + }; + + // Windows reparse point tags + #[cfg(windows)] + #[pyattr] + pub const IO_REPARSE_TAG_SYMLINK: u32 = 0xA000000C; + #[cfg(windows)] + #[pyattr] + pub const IO_REPARSE_TAG_MOUNT_POINT: u32 = 0xA0000003; + #[cfg(windows)] + #[pyattr] + pub const IO_REPARSE_TAG_APPEXECLINK: u32 = 0x8000001B; + + // Unix file flags (if on Unix) + + #[pyattr] + pub const UF_NODUMP: u32 = libc_const!( + #[cfg(target_os = "macos")] + UF_NODUMP, + 0x00000001 + ); + + #[pyattr] + pub const UF_IMMUTABLE: u32 = libc_const!( + #[cfg(target_os = "macos")] + UF_IMMUTABLE, + 0x00000002 + ); + + #[pyattr] + pub const UF_APPEND: u32 = libc_const!( + #[cfg(target_os = "macos")] + UF_APPEND, + 0x00000004 + ); + + #[pyattr] + pub const UF_OPAQUE: u32 = libc_const!( + #[cfg(target_os = "macos")] + UF_OPAQUE, + 0x00000008 + ); + + #[pyattr] + pub const UF_COMPRESSED: u32 = libc_const!( + #[cfg(target_os = "macos")] + UF_COMPRESSED, + 0x00000020 + ); + + #[pyattr] + pub const UF_HIDDEN: u32 = libc_const!( + #[cfg(target_os = "macos")] + UF_HIDDEN, + 0x00008000 + ); + + #[pyattr] + pub const SF_ARCHIVED: u32 = libc_const!( + #[cfg(target_os = "macos")] + SF_ARCHIVED, + 0x00010000 + ); + + #[pyattr] + pub const SF_IMMUTABLE: u32 = libc_const!( + #[cfg(target_os = "macos")] + SF_IMMUTABLE, + 0x00020000 + ); + + #[pyattr] + pub const SF_APPEND: u32 = libc_const!( + #[cfg(target_os = "macos")] + SF_APPEND, + 0x00040000 + ); + + #[pyattr] + pub const SF_SETTABLE: u32 = if cfg!(target_os = "macos") { + 0x3fff0000 + } else { + 0xffff0000 + }; + + #[pyattr] + pub const UF_NOUNLINK: u32 = 0x00000010; + + #[pyattr] + pub const SF_NOUNLINK: u32 = 0x00100000; + + #[pyattr] + pub const SF_SNAPSHOT: u32 = 0x00200000; + + #[pyattr] + pub const SF_FIRMLINK: u32 = 0x00800000; + + #[pyattr] + pub const SF_DATALESS: u32 = 0x40000000; + + // MacOS specific + + #[cfg(target_os = "macos")] + #[pyattr] + pub const SF_SUPPORTED: u32 = 0x009f0000; + + #[cfg(target_os = "macos")] + #[pyattr] + pub const SF_SYNTHETIC: u32 = 0xc0000000; + + // Stat result indices + + #[pyattr] + pub const ST_MODE: u32 = 0; + + #[pyattr] + pub const ST_INO: u32 = 1; + + #[pyattr] + pub const ST_DEV: u32 = 2; + + #[pyattr] + pub const ST_NLINK: u32 = 3; + + #[pyattr] + pub const ST_UID: u32 = 4; + + #[pyattr] + pub const ST_GID: u32 = 5; + + #[pyattr] + pub const ST_SIZE: u32 = 6; + + #[pyattr] + pub const ST_ATIME: u32 = 7; + + #[pyattr] + pub const ST_MTIME: u32 = 8; + + #[pyattr] + pub const ST_CTIME: u32 = 9; + + const S_IFMT: Mode = 0o170000; + + const S_IMODE: Mode = 0o7777; + + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISDIR(mode: Mode) -> bool { + (mode & S_IFMT) == S_IFDIR + } + + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISCHR(mode: Mode) -> bool { + (mode & S_IFMT) == S_IFCHR + } + + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISREG(mode: Mode) -> bool { + (mode & S_IFMT) == S_IFREG + } + + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISBLK(mode: Mode) -> bool { + (mode & S_IFMT) == S_IFBLK + } + + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISFIFO(mode: Mode) -> bool { + (mode & S_IFMT) == S_IFIFO + } + + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISLNK(mode: Mode) -> bool { + (mode & S_IFMT) == S_IFLNK + } + + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISSOCK(mode: Mode) -> bool { + (mode & S_IFMT) == S_IFSOCK + } + + // TODO: RUSTPYTHON Support Solaris + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISDOOR(_mode: Mode) -> bool { + false + } + + // TODO: RUSTPYTHON Support Solaris + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISPORT(_mode: Mode) -> bool { + false + } + + // TODO: RUSTPYTHON Support BSD + #[pyfunction] + #[allow(non_snake_case)] + const fn S_ISWHT(_mode: Mode) -> bool { + false + } + + #[pyfunction(name = "S_IMODE")] + #[allow(non_snake_case)] + const fn S_IMODE_method(mode: Mode) -> Mode { + mode & S_IMODE + } + + #[pyfunction(name = "S_IFMT")] + #[allow(non_snake_case)] + const fn S_IFMT_method(mode: Mode) -> Mode { + // 0o170000 is from the S_IFMT definition in CPython include/fileutils.h + mode & S_IFMT + } + + #[pyfunction] + const fn filetype(mode: Mode) -> char { + if S_ISREG(mode) { + '-' + } else if S_ISDIR(mode) { + 'd' + } else if S_ISLNK(mode) { + 'l' + } else if S_ISBLK(mode) { + 'b' + } else if S_ISCHR(mode) { + 'c' + } else if S_ISFIFO(mode) { + 'p' + } else if S_ISSOCK(mode) { + 's' + } else if S_ISDOOR(mode) { + 'D' // TODO: RUSTPYTHON Support Solaris + } else if S_ISPORT(mode) { + 'P' // TODO: RUSTPYTHON Support Solaris + } else if S_ISWHT(mode) { + 'w' // TODO: RUSTPYTHON Support BSD + } else { + '?' // Unknown file type + } + } + + // Convert file mode to string representation + #[pyfunction] + fn filemode(mode: Mode) -> String { + let mut result = String::with_capacity(10); + + // File type + result.push(filetype(mode)); + + // User permissions + result.push(if mode & S_IRUSR != 0 { 'r' } else { '-' }); + result.push(if mode & S_IWUSR != 0 { 'w' } else { '-' }); + if mode & S_ISUID != 0 { + result.push(if mode & S_IXUSR != 0 { 's' } else { 'S' }); + } else { + result.push(if mode & S_IXUSR != 0 { 'x' } else { '-' }); + } + + // Group permissions + result.push(if mode & S_IRGRP != 0 { 'r' } else { '-' }); + result.push(if mode & S_IWGRP != 0 { 'w' } else { '-' }); + if mode & S_ISGID != 0 { + result.push(if mode & S_IXGRP != 0 { 's' } else { 'S' }); + } else { + result.push(if mode & S_IXGRP != 0 { 'x' } else { '-' }); + } + + // Other permissions + result.push(if mode & S_IROTH != 0 { 'r' } else { '-' }); + result.push(if mode & S_IWOTH != 0 { 'w' } else { '-' }); + if mode & S_ISVTX != 0 { + result.push(if mode & S_IXOTH != 0 { 't' } else { 'T' }); + } else { + result.push(if mode & S_IXOTH != 0 { 'x' } else { '-' }); + } + + result + } +} diff --git a/crates/vm/src/stdlib/_string.rs b/crates/vm/src/stdlib/_string.rs new file mode 100644 index 00000000000..c620f2a40e2 --- /dev/null +++ b/crates/vm/src/stdlib/_string.rs @@ -0,0 +1,99 @@ +/* String builtin module + */ + +pub(crate) use _string::module_def; + +#[pymodule] +mod _string { + use crate::common::ascii; + use crate::common::format::{ + FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, + }; + use crate::common::wtf8::{CodePoint, Wtf8Buf}; + use crate::{ + PyObjectRef, PyResult, VirtualMachine, + builtins::{PyList, PyStrRef}, + convert::ToPyException, + convert::ToPyObject, + }; + use core::mem; + + fn create_format_part( + literal: Wtf8Buf, + field_name: Option<Wtf8Buf>, + format_spec: Option<Wtf8Buf>, + conversion_spec: Option<CodePoint>, + vm: &VirtualMachine, + ) -> PyObjectRef { + let tuple = ( + literal, + field_name, + format_spec, + conversion_spec.map(|c| c.to_string()), + ); + tuple.to_pyobject(vm) + } + + #[pyfunction] + fn formatter_parser(text: PyStrRef, vm: &VirtualMachine) -> PyResult<PyList> { + let format_string = + FormatString::from_str(text.as_wtf8()).map_err(|e| e.to_pyexception(vm))?; + + let mut result: Vec<PyObjectRef> = Vec::new(); + let mut literal = Wtf8Buf::new(); + for part in format_string.format_parts { + match part { + FormatPart::Field { + field_name, + conversion_spec, + format_spec, + } => { + result.push(create_format_part( + mem::take(&mut literal), + Some(field_name), + Some(format_spec), + conversion_spec, + vm, + )); + } + FormatPart::Literal(text) => literal.push_wtf8(&text), + } + } + if !literal.is_empty() { + result.push(create_format_part( + mem::take(&mut literal), + None, + None, + None, + vm, + )); + } + Ok(result.into()) + } + + #[pyfunction] + fn formatter_field_name_split( + text: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, PyList)> { + let field_name = FieldName::parse(text.as_wtf8()).map_err(|e| e.to_pyexception(vm))?; + + let first = match field_name.field_type { + FieldType::Auto => vm.ctx.new_str(ascii!("")).into(), + FieldType::Index(index) => index.to_pyobject(vm), + FieldType::Keyword(attribute) => attribute.to_pyobject(vm), + }; + + let rest = field_name + .parts + .iter() + .map(|p| match p { + FieldNamePart::Attribute(attribute) => (true, attribute).to_pyobject(vm), + FieldNamePart::StringIndex(index) => (false, index).to_pyobject(vm), + FieldNamePart::Index(index) => (false, *index).to_pyobject(vm), + }) + .collect(); + + Ok((first, rest)) + } +} diff --git a/crates/vm/src/stdlib/_symtable.rs b/crates/vm/src/stdlib/_symtable.rs new file mode 100644 index 00000000000..eede3d102c2 --- /dev/null +++ b/crates/vm/src/stdlib/_symtable.rs @@ -0,0 +1,330 @@ +pub(crate) use _symtable::module_def; + +#[pymodule] +mod _symtable { + use crate::{ + Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyUtf8StrRef}, + compiler, + types::Representable, + }; + use alloc::fmt; + use rustpython_codegen::symboltable::{ + CompilerScope, Symbol, SymbolFlags, SymbolScope, SymbolTable, + }; + + // Consts as defined at + // https://github.com/python/cpython/blob/6cb20a219a860eaf687b2d968b41c480c7461909/Include/internal/pycore_symtable.h#L156 + + #[pyattr] + pub const DEF_GLOBAL: i32 = 1; + + #[pyattr] + pub const DEF_LOCAL: i32 = 2; + + #[pyattr] + pub const DEF_PARAM: i32 = 2 << 1; + + #[pyattr] + pub const DEF_NONLOCAL: i32 = 2 << 2; + + #[pyattr] + pub const USE: i32 = 2 << 3; + + #[pyattr] + pub const DEF_FREE: i32 = 2 << 4; + + #[pyattr] + pub const DEF_FREE_CLASS: i32 = 2 << 5; + + #[pyattr] + pub const DEF_IMPORT: i32 = 2 << 6; + + #[pyattr] + pub const DEF_ANNOT: i32 = 2 << 7; + + #[pyattr] + pub const DEF_COMP_ITER: i32 = 2 << 8; + + #[pyattr] + pub const DEF_TYPE_PARAM: i32 = 2 << 9; + + #[pyattr] + pub const DEF_COMP_CELL: i32 = 2 << 10; + + #[pyattr] + pub const DEF_BOUND: i32 = DEF_LOCAL | DEF_PARAM | DEF_IMPORT; + + #[pyattr] + pub const SCOPE_OFFSET: i32 = 12; + + #[pyattr] + pub const SCOPE_MASK: i32 = DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL; + + #[pyattr] + pub const LOCAL: i32 = 1; + + #[pyattr] + pub const GLOBAL_EXPLICIT: i32 = 2; + + #[pyattr] + pub const GLOBAL_IMPLICIT: i32 = 3; + + #[pyattr] + pub const FREE: i32 = 4; + + #[pyattr] + pub const CELL: i32 = 5; + + #[pyattr] + pub const GENERATOR: i32 = 1; + + #[pyattr] + pub const GENERATOR_EXPRESSION: i32 = 2; + + #[pyattr] + pub const SCOPE_OFF: i32 = SCOPE_OFFSET; + + #[pyattr] + pub const TYPE_FUNCTION: i32 = 0; + + #[pyattr] + pub const TYPE_CLASS: i32 = 1; + + #[pyattr] + pub const TYPE_MODULE: i32 = 2; + + #[pyattr] + pub const TYPE_ANNOTATION: i32 = 3; + + #[pyattr] + pub const TYPE_TYPE_VAR_BOUND: i32 = 4; + + #[pyattr] + pub const TYPE_TYPE_ALIAS: i32 = 5; + + #[pyattr] + pub const TYPE_TYPE_PARAMETERS: i32 = 6; + + #[pyattr] + pub const TYPE_TYPE_VARIABLE: i32 = 7; + + #[pyfunction] + fn symtable( + source: PyUtf8StrRef, + filename: PyUtf8StrRef, + mode: PyUtf8StrRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<PySymbolTable>> { + let mode = mode + .as_str() + .parse::<compiler::Mode>() + .map_err(|err| vm.new_value_error(err.to_string()))?; + + let symtable = compiler::compile_symtable(source.as_str(), mode, filename.as_str()) + .map_err(|err| vm.new_syntax_error(&err, Some(source.as_str())))?; + + let py_symbol_table = to_py_symbol_table(symtable); + Ok(py_symbol_table.into_ref(&vm.ctx)) + } + + const fn to_py_symbol_table(symtable: SymbolTable) -> PySymbolTable { + PySymbolTable { symtable } + } + + #[pyattr] + #[pyclass(name = "symtable entry")] + #[derive(PyPayload)] + struct PySymbolTable { + symtable: SymbolTable, + } + + impl fmt::Debug for PySymbolTable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SymbolTable()") + } + } + + #[pyclass(with(Representable))] + impl PySymbolTable { + #[pygetset] + fn name(&self) -> String { + self.symtable.name.clone() + } + + #[pygetset(name = "type")] + fn typ(&self) -> i32 { + match self.symtable.typ { + CompilerScope::Function => TYPE_FUNCTION, + CompilerScope::Class => TYPE_CLASS, + CompilerScope::Module => TYPE_MODULE, + CompilerScope::TypeParams => TYPE_TYPE_PARAMETERS, + _ => -1, // TODO: missing types from the C implementation + } + } + + #[pygetset] + const fn lineno(&self) -> u32 { + self.symtable.line_number + } + + #[pygetset] + fn children(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let children = self + .symtable + .sub_tables + .iter() + .map(|t| to_py_symbol_table(t.clone()).into_pyobject(vm)) + .collect(); + Ok(children) + } + + #[pygetset] + fn id(&self) -> usize { + self as *const Self as *const core::ffi::c_void as usize + } + + #[pygetset] + fn identifiers(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let symbols = self + .symtable + .symbols + .keys() + .map(|s| vm.ctx.new_str(s.as_str()).into()) + .collect(); + Ok(symbols) + } + + #[pygetset] + fn symbols(&self, vm: &VirtualMachine) -> PyResult<PyDictRef> { + let dict = vm.ctx.new_dict(); + for (name, symbol) in &self.symtable.symbols { + dict.set_item(name, vm.new_pyobj(symbol.flags.bits()), vm) + .unwrap(); + } + Ok(dict) + } + + #[pygetset] + const fn nested(&self) -> bool { + self.symtable.is_nested + } + } + + impl Representable for PySymbolTable { + #[inline] + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + Ok(format!( + "<{} {}({}), line {}>", + Self::class(&vm.ctx).name(), + zelf.symtable.name, + zelf.id(), + zelf.symtable.line_number + )) + } + } + + #[pyattr] + #[pyclass(name = "Symbol")] + #[derive(PyPayload)] + struct PySymbol { + symbol: Symbol, + namespaces: Vec<SymbolTable>, + is_top_scope: bool, + } + + impl fmt::Debug for PySymbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Symbol()") + } + } + + #[pyclass] + impl PySymbol { + #[pymethod] + fn get_name(&self) -> String { + self.symbol.name.clone() + } + + #[pymethod] + const fn is_global(&self) -> bool { + self.symbol.is_global() || (self.is_top_scope && self.symbol.is_bound()) + } + + #[pymethod] + const fn is_declared_global(&self) -> bool { + matches!(self.symbol.scope, SymbolScope::GlobalExplicit) + } + + #[pymethod] + const fn is_local(&self) -> bool { + self.symbol.is_local() || (self.is_top_scope && self.symbol.is_bound()) + } + + #[pymethod] + const fn is_imported(&self) -> bool { + self.symbol.flags.contains(SymbolFlags::IMPORTED) + } + + #[pymethod] + const fn is_nested(&self) -> bool { + // TODO + false + } + + #[pymethod] + const fn is_nonlocal(&self) -> bool { + self.symbol.flags.contains(SymbolFlags::NONLOCAL) + } + + #[pymethod] + const fn is_referenced(&self) -> bool { + self.symbol.flags.contains(SymbolFlags::REFERENCED) + } + + #[pymethod] + const fn is_assigned(&self) -> bool { + self.symbol.flags.contains(SymbolFlags::ASSIGNED) + } + + #[pymethod] + const fn is_parameter(&self) -> bool { + self.symbol.flags.contains(SymbolFlags::PARAMETER) + } + + #[pymethod] + const fn is_free(&self) -> bool { + matches!(self.symbol.scope, SymbolScope::Free) + } + + #[pymethod] + const fn is_namespace(&self) -> bool { + !self.namespaces.is_empty() + } + + #[pymethod] + const fn is_annotated(&self) -> bool { + self.symbol.flags.contains(SymbolFlags::ANNOTATED) + } + + #[pymethod] + fn get_namespaces(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let namespaces = self + .namespaces + .iter() + .map(|table| to_py_symbol_table(table.clone()).into_pyobject(vm)) + .collect(); + Ok(namespaces) + } + + #[pymethod] + fn get_namespace(&self, vm: &VirtualMachine) -> PyResult { + if self.namespaces.len() != 1 { + return Err(vm.new_value_error("namespace is bound to multiple namespaces")); + } + Ok(to_py_symbol_table(self.namespaces.first().unwrap().clone()) + .into_ref(&vm.ctx) + .into()) + } + } +} diff --git a/crates/vm/src/stdlib/_sysconfig.rs b/crates/vm/src/stdlib/_sysconfig.rs new file mode 100644 index 00000000000..2de27d71ac9 --- /dev/null +++ b/crates/vm/src/stdlib/_sysconfig.rs @@ -0,0 +1,24 @@ +pub(crate) use _sysconfig::module_def; + +#[pymodule] +pub(crate) mod _sysconfig { + use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject}; + + #[pyfunction] + fn config_vars(vm: &VirtualMachine) -> PyDictRef { + let vars = vm.ctx.new_dict(); + + // FIXME: This is an entirely wrong implementation of EXT_SUFFIX. + // EXT_SUFFIX must be a string starting with "." for pip compatibility + // Using ".pyd" causes pip's _generic_abi() to fall back to _cpython_abis() + vars.set_item("EXT_SUFFIX", ".pyd".to_pyobject(vm), vm) + .unwrap(); + vars.set_item("SOABI", vm.ctx.none(), vm).unwrap(); + + vars.set_item("Py_GIL_DISABLED", (1).to_pyobject(vm), vm) + .unwrap(); + vars.set_item("Py_DEBUG", (0).to_pyobject(vm), vm).unwrap(); + + vars + } +} diff --git a/crates/vm/src/stdlib/_sysconfigdata.rs b/crates/vm/src/stdlib/_sysconfigdata.rs new file mode 100644 index 00000000000..99aab892e1c --- /dev/null +++ b/crates/vm/src/stdlib/_sysconfigdata.rs @@ -0,0 +1,75 @@ +// spell-checker: words LDSHARED ARFLAGS CPPFLAGS CCSHARED BASECFLAGS BLDSHARED + +pub(crate) use _sysconfigdata::module_def; + +#[pymodule] +mod _sysconfigdata { + use crate::stdlib::sys::{RUST_MULTIARCH, multiarch, sysconfigdata_name}; + use crate::{ + Py, PyResult, VirtualMachine, + builtins::{PyDictRef, PyModule}, + convert::ToPyObject, + }; + + fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + // Set build_time_vars attribute + let build_time_vars = build_time_vars(vm); + + // Add runtime-dependent values needed by sysconfig + let paths = &vm.state.config.paths; + build_time_vars.set_item("prefix", paths.prefix.clone().to_pyobject(vm), vm)?; + build_time_vars.set_item("exec_prefix", paths.exec_prefix.clone().to_pyobject(vm), vm)?; + let bindir = format!("{}/bin", &paths.exec_prefix); + build_time_vars.set_item("BINDIR", bindir.to_pyobject(vm), vm)?; + + module.set_attr("build_time_vars", build_time_vars, vm)?; + + // Ensure the module is registered under the platform-specific name + // (import_builtin() already handles this, but double-check for safety) + let sys_modules = vm.sys_module.get_attr("modules", vm)?; + let sysconfigdata_name = sysconfigdata_name(); + sys_modules.set_item(sysconfigdata_name.as_str(), module.to_owned().into(), vm)?; + + Ok(()) + } + + #[pyattr] + fn build_time_vars(vm: &VirtualMachine) -> PyDictRef { + let vars = vm.ctx.new_dict(); + let multiarch = multiarch(); + macro_rules! sysvars { + ($($key:literal => $value:expr),*$(,)?) => {{ + $(vars.set_item($key, $value.to_pyobject(vm), vm).unwrap();)* + }}; + } + sysvars! { + // Extension module suffix in CPython-compatible format + "EXT_SUFFIX" => format!(".rustpython313-{multiarch}.so"), + "MULTIARCH" => multiarch.clone(), + "RUST_MULTIARCH" => RUST_MULTIARCH, + // enough for tests to stop expecting urandom() to fail after restricting file resources + "HAVE_GETRANDOM" => 1, + // RustPython has no GIL (like free-threaded Python) + "Py_GIL_DISABLED" => 1, + "Py_DEBUG" => 0, + "ABIFLAGS" => "t", + // Compiler configuration for native extension builds + "CC" => "cc", + "CXX" => "c++", + "CFLAGS" => "", + "CPPFLAGS" => "", + "LDFLAGS" => "", + "LDSHARED" => "cc -shared", + "CCSHARED" => "", + "SHLIB_SUFFIX" => ".so", + "SO" => ".so", + "AR" => "ar", + "ARFLAGS" => "rcs", + "OPT" => "", + "BASECFLAGS" => "", + "BLDSHARED" => "cc -shared", + } + include!(concat!(env!("OUT_DIR"), "/env_vars.rs")); + vars + } +} diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs new file mode 100644 index 00000000000..765f2537440 --- /dev/null +++ b/crates/vm/src/stdlib/_thread.rs @@ -0,0 +1,1760 @@ +//! Implementation of the _thread module +#[cfg(unix)] +pub(crate) use _thread::after_fork_child; +pub use _thread::get_ident; +#[cfg_attr(target_arch = "wasm32", allow(unused_imports))] +pub(crate) use _thread::{ + CurrentFrameSlot, HandleEntry, RawRMutex, ShutdownEntry, get_all_current_frames, + init_main_thread_ident, module_def, +}; + +#[pymodule] +pub(crate) mod _thread { + use crate::{ + AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyStr, PyTupleRef, PyType, PyTypeRef, PyUtf8StrRef}, + common::wtf8::Wtf8Buf, + frame::FrameRef, + function::{ArgCallable, FuncArgs, KwArgs, OptionalArg, PySetterValue, TimeoutSeconds}, + types::{Constructor, GetAttr, Representable, SetAttr}, + }; + use alloc::{ + fmt, + sync::{Arc, Weak}, + }; + use core::{cell::RefCell, time::Duration}; + use parking_lot::{ + RawMutex, RawThreadId, + lock_api::{RawMutex as RawMutexT, RawMutexTimed, RawReentrantMutex}, + }; + use rustpython_common::str::levenshtein::{MOVE_COST, levenshtein_distance}; + use std::thread; + + // PYTHREAD_NAME: show current thread name + pub const PYTHREAD_NAME: Option<&str> = { + cfg_if::cfg_if! { + if #[cfg(windows)] { + Some("nt") + } else if #[cfg(unix)] { + Some("pthread") + } else if #[cfg(any(target_os = "solaris", target_os = "illumos"))] { + Some("solaris") + } else { + None + } + } + }; + + // TIMEOUT_MAX_IN_MICROSECONDS is a value in microseconds + #[cfg(not(target_os = "windows"))] + const TIMEOUT_MAX_IN_MICROSECONDS: i64 = i64::MAX / 1_000; + + #[cfg(target_os = "windows")] + const TIMEOUT_MAX_IN_MICROSECONDS: i64 = 0xffffffff * 1_000; + + // this is a value in seconds + #[pyattr] + const TIMEOUT_MAX: f64 = (TIMEOUT_MAX_IN_MICROSECONDS / 1_000_000) as f64; + + #[pyattr] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.exceptions.runtime_error.to_owned() + } + + #[derive(FromArgs)] + struct AcquireArgs { + #[pyarg(any, default = true)] + blocking: bool, + #[pyarg(any, default = TimeoutSeconds::new(-1.0))] + timeout: TimeoutSeconds, + } + + macro_rules! acquire_lock_impl { + ($mu:expr, $args:expr, $vm:expr) => {{ + let (mu, args, vm) = ($mu, $args, $vm); + let timeout = args.timeout.to_secs_f64(); + match args.blocking { + true if timeout == -1.0 => { + vm.allow_threads(|| mu.lock()); + Ok(true) + } + true if timeout < 0.0 => { + Err(vm + .new_value_error("timeout value must be a non-negative number".to_owned())) + } + true => { + if timeout > TIMEOUT_MAX { + return Err(vm.new_overflow_error("timeout value is too large".to_owned())); + } + + Ok(vm.allow_threads(|| mu.try_lock_for(Duration::from_secs_f64(timeout)))) + } + false if timeout != -1.0 => Err(vm + .new_value_error("can't specify a timeout for a non-blocking call".to_owned())), + false => Ok(mu.try_lock()), + } + }}; + } + macro_rules! repr_lock_impl { + ($zelf:expr) => {{ + let status = if $zelf.mu.is_locked() { + "locked" + } else { + "unlocked" + }; + Ok(format!( + "<{} {} object at {:#x}>", + status, + $zelf.class().name(), + $zelf.get_id() + )) + }}; + } + + #[pyattr(name = "LockType")] + #[pyattr(name = "lock")] + #[pyclass(module = "_thread", name = "lock")] + #[derive(PyPayload)] + struct Lock { + mu: RawMutex, + } + + impl fmt::Debug for Lock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("Lock") + } + } + + #[pyclass(with(Constructor, Representable), flags(HAS_WEAKREF))] + impl Lock { + #[pymethod] + #[pymethod(name = "acquire_lock")] + #[pymethod(name = "__enter__")] + fn acquire(&self, args: AcquireArgs, vm: &VirtualMachine) -> PyResult<bool> { + acquire_lock_impl!(&self.mu, args, vm) + } + #[pymethod] + #[pymethod(name = "release_lock")] + fn release(&self, vm: &VirtualMachine) -> PyResult<()> { + if !self.mu.is_locked() { + return Err(vm.new_runtime_error("release unlocked lock")); + } + unsafe { self.mu.unlock() }; + Ok(()) + } + + #[cfg(unix)] + #[pymethod] + fn _at_fork_reinit(&self, _vm: &VirtualMachine) -> PyResult<()> { + // Overwrite lock state to unlocked. Do NOT call unlock() here — + // after fork(), unlock_slow() would try to unpark stale waiters. + unsafe { rustpython_common::lock::zero_reinit_after_fork(&self.mu) }; + Ok(()) + } + + #[pymethod] + fn __exit__(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + self.release(vm) + } + + #[pymethod] + fn locked(&self) -> bool { + self.mu.is_locked() + } + } + + impl Constructor for Lock { + type Args = (); + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { mu: RawMutex::INIT }) + } + } + + impl Representable for Lock { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + repr_lock_impl!(zelf) + } + } + + pub type RawRMutex = RawReentrantMutex<RawMutex, RawThreadId>; + #[pyattr] + #[pyclass(module = "_thread", name = "RLock")] + #[derive(PyPayload)] + struct RLock { + mu: RawRMutex, + count: core::sync::atomic::AtomicUsize, + } + + impl fmt::Debug for RLock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad("RLock") + } + } + + #[pyclass(with(Representable), flags(BASETYPE, HAS_WEAKREF))] + impl RLock { + #[pyslot] + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Self { + mu: RawRMutex::INIT, + count: core::sync::atomic::AtomicUsize::new(0), + } + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + #[pymethod] + #[pymethod(name = "acquire_lock")] + #[pymethod(name = "__enter__")] + fn acquire(&self, args: AcquireArgs, vm: &VirtualMachine) -> PyResult<bool> { + if self.mu.is_owned_by_current_thread() { + // Re-entrant acquisition: just increment our count. + // parking_lot stays at 1 level; we track recursion ourselves. + self.count + .fetch_add(1, core::sync::atomic::Ordering::Relaxed); + return Ok(true); + } + let result = acquire_lock_impl!(&self.mu, args, vm)?; + if result { + self.count.store(1, core::sync::atomic::Ordering::Relaxed); + } + Ok(result) + } + #[pymethod] + #[pymethod(name = "release_lock")] + fn release(&self, vm: &VirtualMachine) -> PyResult<()> { + if !self.mu.is_owned_by_current_thread() { + return Err(vm.new_runtime_error("cannot release un-acquired lock")); + } + let prev = self + .count + .fetch_sub(1, core::sync::atomic::Ordering::Relaxed); + debug_assert!(prev > 0, "RLock count underflow"); + if prev == 1 { + unsafe { self.mu.unlock() }; + } + Ok(()) + } + + #[cfg(unix)] + #[pymethod] + fn _at_fork_reinit(&self, _vm: &VirtualMachine) -> PyResult<()> { + // Overwrite lock state to unlocked. Do NOT call unlock() here — + // after fork(), unlock_slow() would try to unpark stale waiters. + self.count.store(0, core::sync::atomic::Ordering::Relaxed); + unsafe { rustpython_common::lock::zero_reinit_after_fork(&self.mu) }; + Ok(()) + } + + #[pymethod] + fn locked(&self) -> bool { + self.mu.is_locked() + } + + #[pymethod] + fn _is_owned(&self) -> bool { + self.mu.is_owned_by_current_thread() + } + + #[pymethod] + fn _recursion_count(&self) -> usize { + if self.mu.is_owned_by_current_thread() { + self.count.load(core::sync::atomic::Ordering::Relaxed) + } else { + 0 + } + } + + #[pymethod] + fn _release_save(&self, vm: &VirtualMachine) -> PyResult<(usize, u64)> { + if !self.mu.is_owned_by_current_thread() { + return Err(vm.new_runtime_error("cannot release un-acquired lock")); + } + let count = self.count.swap(0, core::sync::atomic::Ordering::Relaxed); + debug_assert!(count > 0, "RLock count underflow"); + unsafe { self.mu.unlock() }; + Ok((count, current_thread_id())) + } + + #[pymethod] + fn _acquire_restore(&self, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { + let [count_obj, owner_obj] = state.as_slice() else { + return Err( + vm.new_type_error("_acquire_restore() argument 1 must be a 2-item tuple") + ); + }; + let count: usize = count_obj.clone().try_into_value(vm)?; + let _owner: u64 = owner_obj.clone().try_into_value(vm)?; + if count == 0 { + return Ok(()); + } + vm.allow_threads(|| self.mu.lock()); + self.count + .store(count, core::sync::atomic::Ordering::Relaxed); + Ok(()) + } + + #[pymethod] + fn __exit__(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + self.release(vm) + } + } + + impl Representable for RLock { + #[inline] + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let count = zelf.count.load(core::sync::atomic::Ordering::Relaxed); + let status = if zelf.mu.is_locked() { + "locked" + } else { + "unlocked" + }; + Ok(format!( + "<{} {} object count={} at {:#x}>", + status, + zelf.class().name(), + count, + zelf.get_id() + )) + } + } + + /// Get thread identity - uses pthread_self() on Unix for fork compatibility + #[pyfunction] + pub fn get_ident() -> u64 { + current_thread_id() + } + + #[cfg(all(unix, feature = "threading"))] + #[pyfunction] + fn _stop_the_world_stats(vm: &VirtualMachine) -> PyResult<PyDictRef> { + let stats = vm.state.stop_the_world.stats_snapshot(); + let d = vm.ctx.new_dict(); + d.set_item("stop_calls", vm.ctx.new_int(stats.stop_calls).into(), vm)?; + d.set_item( + "last_wait_ns", + vm.ctx.new_int(stats.last_wait_ns).into(), + vm, + )?; + d.set_item( + "total_wait_ns", + vm.ctx.new_int(stats.total_wait_ns).into(), + vm, + )?; + d.set_item("max_wait_ns", vm.ctx.new_int(stats.max_wait_ns).into(), vm)?; + d.set_item("poll_loops", vm.ctx.new_int(stats.poll_loops).into(), vm)?; + d.set_item( + "attached_seen", + vm.ctx.new_int(stats.attached_seen).into(), + vm, + )?; + d.set_item( + "forced_parks", + vm.ctx.new_int(stats.forced_parks).into(), + vm, + )?; + d.set_item( + "suspend_notifications", + vm.ctx.new_int(stats.suspend_notifications).into(), + vm, + )?; + d.set_item( + "attach_wait_yields", + vm.ctx.new_int(stats.attach_wait_yields).into(), + vm, + )?; + d.set_item( + "suspend_wait_yields", + vm.ctx.new_int(stats.suspend_wait_yields).into(), + vm, + )?; + d.set_item( + "world_stopped", + vm.ctx.new_bool(stats.world_stopped).into(), + vm, + )?; + Ok(d) + } + + #[cfg(all(unix, feature = "threading"))] + #[pyfunction] + fn _stop_the_world_reset_stats(vm: &VirtualMachine) { + vm.state.stop_the_world.reset_stats(); + } + + /// Set the name of the current thread + #[pyfunction] + fn set_name(name: PyUtf8StrRef) { + #[cfg(target_os = "linux")] + { + use alloc::ffi::CString; + if let Ok(c_name) = CString::new(name.as_str()) { + // pthread_setname_np on Linux has a 16-byte limit including null terminator + // TODO: Potential UTF-8 boundary issue when truncating thread name on Linux. + // https://github.com/RustPython/RustPython/pull/6726/changes#r2689379171 + let truncated = if c_name.as_bytes().len() > 15 { + CString::new(&c_name.as_bytes()[..15]).unwrap_or(c_name) + } else { + c_name + }; + unsafe { + libc::pthread_setname_np(libc::pthread_self(), truncated.as_ptr()); + } + } + } + #[cfg(target_os = "macos")] + { + use alloc::ffi::CString; + if let Ok(c_name) = CString::new(name.as_str()) { + unsafe { + libc::pthread_setname_np(c_name.as_ptr()); + } + } + } + #[cfg(windows)] + { + // Windows doesn't have a simple pthread_setname_np equivalent + // SetThreadDescription requires Windows 10+ + let _ = name; + } + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + { + let _ = name; + } + } + + /// Get OS-level thread ID (pthread_self on Unix) + /// This is important for fork compatibility - the ID must remain stable after fork + #[cfg(unix)] + fn current_thread_id() -> u64 { + // pthread_self() for fork compatibility + unsafe { libc::pthread_self() as u64 } + } + + #[cfg(not(unix))] + fn current_thread_id() -> u64 { + thread_to_rust_id(&thread::current()) + } + + /// Convert Rust thread to ID (used for non-unix platforms) + #[cfg(not(unix))] + fn thread_to_rust_id(t: &thread::Thread) -> u64 { + use core::hash::{Hash, Hasher}; + struct U64Hash { + v: Option<u64>, + } + impl Hasher for U64Hash { + fn write(&mut self, _: &[u8]) { + unreachable!() + } + fn write_u64(&mut self, i: u64) { + self.v = Some(i); + } + fn finish(&self) -> u64 { + self.v.expect("should have written a u64") + } + } + let mut h = U64Hash { v: None }; + t.id().hash(&mut h); + h.finish() + } + + /// Get thread ID for a given thread handle (used by start_new_thread) + fn thread_to_id(handle: &thread::JoinHandle<()>) -> u64 { + #[cfg(unix)] + { + // On Unix, use pthread ID from the handle + use std::os::unix::thread::JoinHandleExt; + handle.as_pthread_t() as u64 + } + #[cfg(not(unix))] + { + thread_to_rust_id(handle.thread()) + } + } + + #[pyfunction] + const fn allocate_lock() -> Lock { + Lock { mu: RawMutex::INIT } + } + + #[pyfunction] + fn start_new_thread(mut f_args: FuncArgs, vm: &VirtualMachine) -> PyResult<u64> { + if !f_args.kwargs.is_empty() { + return Err(vm.new_type_error("start_new_thread() takes no keyword arguments")); + } + let given = f_args.args.len(); + if given < 2 { + return Err(vm.new_type_error(format!( + "start_new_thread expected at least 2 arguments, got {given}" + ))); + } + if given > 3 { + return Err(vm.new_type_error(format!( + "start_new_thread expected at most 3 arguments, got {given}" + ))); + } + + let func_obj = f_args.take_positional().unwrap(); + let args_obj = f_args.take_positional().unwrap(); + let kwargs_obj = f_args.take_positional(); + + if func_obj.to_callable().is_none() { + return Err(vm.new_type_error("first arg must be callable")); + } + if !args_obj.fast_isinstance(vm.ctx.types.tuple_type) { + return Err(vm.new_type_error("2nd arg must be a tuple")); + } + if kwargs_obj + .as_ref() + .is_some_and(|obj| !obj.fast_isinstance(vm.ctx.types.dict_type)) + { + return Err(vm.new_type_error("optional 3rd arg must be a dictionary")); + } + + let func: ArgCallable = func_obj.clone().try_into_value(vm)?; + let args: PyTupleRef = args_obj.clone().try_into_value(vm)?; + let kwargs: Option<PyDictRef> = kwargs_obj.map(|obj| obj.try_into_value(vm)).transpose()?; + + vm.sys_module.get_attr("audit", vm)?.call( + ( + "_thread.start_new_thread", + func_obj, + args_obj, + kwargs + .as_ref() + .map_or_else(|| vm.ctx.none(), |k| k.clone().into()), + ), + vm, + )?; + + if vm + .state + .finalizing + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.python_finalization_error.to_owned(), + "can't create new thread at interpreter shutdown" + .to_owned() + .into(), + )); + } + + let args = FuncArgs::new( + args.to_vec(), + kwargs + .map_or_else(Default::default, |k| k.to_attributes(vm)) + .into_iter() + .map(|(k, v)| (k.as_str().to_owned(), v)) + .collect::<KwArgs>(), + ); + let mut thread_builder = thread::Builder::new(); + let stacksize = vm.state.stacksize.load(); + if stacksize != 0 { + thread_builder = thread_builder.stack_size(stacksize); + } + thread_builder + .spawn( + vm.new_thread() + .make_spawn_func(move |vm| run_thread(func, args, vm)), + ) + .map(|handle| thread_to_id(&handle)) + .map_err(|_err| vm.new_runtime_error("can't start new thread")) + } + + fn run_thread(func: ArgCallable, args: FuncArgs, vm: &VirtualMachine) { + // Increment thread count when thread actually starts executing + vm.state.thread_count.fetch_add(1); + + match func.invoke(args, vm) { + Ok(_obj) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.system_exit) => {} + Err(exc) => { + vm.run_unraisable( + exc, + Some("Exception ignored in thread started by".to_owned()), + func.into(), + ); + } + } + for lock in SENTINELS.take() { + if lock.mu.is_locked() { + unsafe { lock.mu.unlock() }; + } + } + // Clean up thread-local storage while VM context is still active + // This ensures __del__ methods are called properly + cleanup_thread_local_data(); + // Clean up frame tracking + crate::vm::thread::cleanup_current_thread_frames(vm); + vm.state.thread_count.fetch_sub(1); + } + + /// Clean up thread-local data for the current thread. + /// This triggers __del__ on objects stored in thread-local variables. + fn cleanup_thread_local_data() { + // Take all guards - this will trigger LocalGuard::drop for each, + // which removes the thread's dict from each Local instance + LOCAL_GUARDS.with(|guards| { + guards.borrow_mut().clear(); + }); + } + + #[cfg(all(not(target_arch = "wasm32"), feature = "host_env"))] + #[pyfunction] + fn interrupt_main(signum: OptionalArg<i32>, vm: &VirtualMachine) -> PyResult<()> { + crate::signal::set_interrupt_ex(signum.unwrap_or(libc::SIGINT), vm) + } + + #[pyfunction] + fn exit(vm: &VirtualMachine) -> PyResult { + Err(vm.new_exception_empty(vm.ctx.exceptions.system_exit.to_owned())) + } + + thread_local!(static SENTINELS: RefCell<Vec<PyRef<Lock>>> = const { RefCell::new(Vec::new()) }); + + #[pyfunction] + fn _set_sentinel(vm: &VirtualMachine) -> PyRef<Lock> { + let lock = Lock { mu: RawMutex::INIT }.into_ref(&vm.ctx); + SENTINELS.with_borrow_mut(|sentinels| sentinels.push(lock.clone())); + lock + } + + #[pyfunction] + fn stack_size(size: OptionalArg<usize>, vm: &VirtualMachine) -> usize { + let size = size.unwrap_or(0); + // TODO: do validation on this to make sure it's not too small + vm.state.stacksize.swap(size) + } + + #[pyfunction] + fn _count(vm: &VirtualMachine) -> usize { + vm.state.thread_count.load() + } + + #[pyfunction] + fn daemon_threads_allowed() -> bool { + // RustPython always allows daemon threads + true + } + + // Registry for non-daemon threads that need to be joined at shutdown + pub type ShutdownEntry = ( + Weak<parking_lot::Mutex<ThreadHandleInner>>, + Weak<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + ); + + #[pyfunction] + fn _shutdown(vm: &VirtualMachine) { + // Wait for all non-daemon threads to finish + let current_ident = get_ident(); + + loop { + // Find a thread that's not finished and not the current thread + let handle_to_join = { + let mut handles = vm.state.shutdown_handles.lock(); + // Clean up finished entries + handles.retain(|(inner_weak, _): &ShutdownEntry| { + inner_weak.upgrade().is_some_and(|inner| { + let guard = inner.lock(); + guard.state != ThreadHandleState::Done && guard.ident != current_ident + }) + }); + + // Find first unfinished handle + handles + .iter() + .find_map(|(inner_weak, done_event_weak): &ShutdownEntry| { + let inner = inner_weak.upgrade()?; + let done_event = done_event_weak.upgrade()?; + let guard = inner.lock(); + if guard.state != ThreadHandleState::Done && guard.ident != current_ident { + Some((inner.clone(), done_event.clone())) + } else { + None + } + }) + }; + + match handle_to_join { + Some((inner, done_event)) => { + if let Err(exc) = ThreadHandle::join_internal(&inner, &done_event, None, vm) { + vm.run_unraisable( + exc, + Some( + "Exception ignored while joining a thread in _thread._shutdown()" + .to_owned(), + ), + vm.ctx.none(), + ); + return; + } + } + None => break, // No more threads to wait on + } + } + } + + /// Add a non-daemon thread handle to the shutdown registry + fn add_to_shutdown_handles( + vm: &VirtualMachine, + inner: &Arc<parking_lot::Mutex<ThreadHandleInner>>, + done_event: &Arc<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + ) { + let mut handles = vm.state.shutdown_handles.lock(); + handles.push((Arc::downgrade(inner), Arc::downgrade(done_event))); + } + + fn remove_from_shutdown_handles( + vm: &VirtualMachine, + inner: &Arc<parking_lot::Mutex<ThreadHandleInner>>, + done_event: &Arc<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + ) { + let mut handles = vm.state.shutdown_handles.lock(); + handles.retain(|(inner_weak, done_event_weak): &ShutdownEntry| { + let Some(registered_inner) = inner_weak.upgrade() else { + return false; + }; + let Some(registered_done_event) = done_event_weak.upgrade() else { + return false; + }; + !(Arc::ptr_eq(&registered_inner, inner) + && Arc::ptr_eq(&registered_done_event, done_event)) + }); + } + + #[pyfunction] + fn _make_thread_handle(ident: u64, vm: &VirtualMachine) -> PyRef<ThreadHandle> { + let handle = ThreadHandle::new(vm); + { + let mut inner = handle.inner.lock(); + inner.ident = ident; + inner.state = ThreadHandleState::Running; + } + handle.into_ref(&vm.ctx) + } + + #[pyfunction] + fn _get_main_thread_ident(vm: &VirtualMachine) -> u64 { + vm.state.main_thread_ident.load() + } + + #[pyfunction] + fn _is_main_interpreter() -> bool { + // RustPython only has one interpreter + true + } + + /// Initialize the main thread ident. Should be called once at interpreter startup. + pub fn init_main_thread_ident(vm: &VirtualMachine) { + let ident = get_ident(); + vm.state.main_thread_ident.store(ident); + } + + /// ExceptHookArgs - simple class to hold exception hook arguments + /// This allows threading.py to import _excepthook and _ExceptHookArgs from _thread + #[pyattr] + #[pyclass(module = "_thread", name = "_ExceptHookArgs")] + #[derive(Debug, PyPayload)] + struct ExceptHookArgs { + exc_type: crate::PyObjectRef, + exc_value: crate::PyObjectRef, + exc_traceback: crate::PyObjectRef, + thread: crate::PyObjectRef, + } + + #[pyclass(with(Constructor))] + impl ExceptHookArgs { + #[pygetset] + fn exc_type(&self) -> crate::PyObjectRef { + self.exc_type.clone() + } + + #[pygetset] + fn exc_value(&self) -> crate::PyObjectRef { + self.exc_value.clone() + } + + #[pygetset] + fn exc_traceback(&self) -> crate::PyObjectRef { + self.exc_traceback.clone() + } + + #[pygetset] + fn thread(&self) -> crate::PyObjectRef { + self.thread.clone() + } + } + + impl Constructor for ExceptHookArgs { + // Takes a single iterable argument like namedtuple + type Args = (crate::PyObjectRef,); + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + // Convert the argument to a list/tuple and extract elements + let seq: Vec<crate::PyObjectRef> = args.0.try_to_value(vm)?; + if seq.len() != 4 { + return Err(vm.new_type_error(format!( + "_ExceptHookArgs expected 4 arguments, got {}", + seq.len() + ))); + } + Ok(Self { + exc_type: seq[0].clone(), + exc_value: seq[1].clone(), + exc_traceback: seq[2].clone(), + thread: seq[3].clone(), + }) + } + } + + /// Handle uncaught exception in Thread.run() + #[pyfunction] + fn _excepthook(args: crate::PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Type check: args must be _ExceptHookArgs + let args = args.downcast::<ExceptHookArgs>().map_err(|_| { + vm.new_type_error("_thread._excepthook argument type must be _ExceptHookArgs") + })?; + + let exc_type = args.exc_type.clone(); + let exc_value = args.exc_value.clone(); + let exc_traceback = args.exc_traceback.clone(); + let thread = args.thread.clone(); + + // Silently ignore SystemExit (identity check) + if exc_type.is(vm.ctx.exceptions.system_exit.as_ref()) { + return Ok(()); + } + + // Get stderr - fall back to thread._stderr if sys.stderr is None + let file = match vm.sys_module.get_attr("stderr", vm) { + Ok(stderr) if !vm.is_none(&stderr) => stderr, + _ => { + if vm.is_none(&thread) { + // do nothing if sys.stderr is None and thread is None + return Ok(()); + } + let thread_stderr = thread.get_attr("_stderr", vm)?; + if vm.is_none(&thread_stderr) { + // do nothing if sys.stderr is None and sys.stderr was None + // when the thread was created + return Ok(()); + } + thread_stderr + } + }; + + // Print "Exception in thread {thread.name}:" + let thread_name = if !vm.is_none(&thread) { + thread + .get_attr("name", vm) + .ok() + .and_then(|n| n.str(vm).ok()) + .map(|s| s.as_wtf8().to_owned()) + } else { + None + }; + let name = thread_name.unwrap_or_else(|| Wtf8Buf::from(format!("{}", get_ident()))); + + let _ = vm.call_method( + &file, + "write", + (format!("Exception in thread {}:\n", name),), + ); + + // Display the traceback + if let Ok(traceback_mod) = vm.import("traceback", 0) + && let Ok(print_exc) = traceback_mod.get_attr("print_exception", vm) + { + use crate::function::KwArgs; + let kwargs: KwArgs = vec![("file".to_owned(), file.clone())] + .into_iter() + .collect(); + let _ = print_exc.call_with_args( + crate::function::FuncArgs::new(vec![exc_type, exc_value, exc_traceback], kwargs), + vm, + ); + } + + // Flush file + let _ = vm.call_method(&file, "flush", ()); + Ok(()) + } + + // Thread-local storage for cleanup guards + // When a thread terminates, the guard is dropped, which triggers cleanup + thread_local! { + static LOCAL_GUARDS: RefCell<Vec<LocalGuard>> = const { RefCell::new(Vec::new()) }; + } + + // Guard that removes thread-local data when dropped + struct LocalGuard { + local: Weak<LocalData>, + thread_id: std::thread::ThreadId, + } + + impl Drop for LocalGuard { + fn drop(&mut self) { + if let Some(local_data) = self.local.upgrade() { + // Remove from map while holding the lock, but drop the value + // outside the lock to prevent deadlock if __del__ accesses _local + let removed = local_data.data.lock().remove(&self.thread_id); + drop(removed); + } + } + } + + // Shared data structure for Local + struct LocalData { + data: parking_lot::Mutex<std::collections::HashMap<std::thread::ThreadId, PyDictRef>>, + } + + impl fmt::Debug for LocalData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LocalData").finish_non_exhaustive() + } + } + + #[pyattr] + #[pyclass(module = "_thread", name = "_local")] + #[derive(Debug, PyPayload)] + struct Local { + inner: Arc<LocalData>, + } + + #[pyclass(with(GetAttr, SetAttr), flags(BASETYPE))] + impl Local { + fn l_dict(&self, vm: &VirtualMachine) -> PyDictRef { + let thread_id = std::thread::current().id(); + + // Fast path: check if dict exists under lock + if let Some(dict) = self.inner.data.lock().get(&thread_id).cloned() { + return dict; + } + + // Slow path: allocate dict outside lock to reduce lock hold time + let new_dict = vm.ctx.new_dict(); + + // Insert with double-check to handle races + let mut data = self.inner.data.lock(); + use std::collections::hash_map::Entry; + let (dict, need_guard) = match data.entry(thread_id) { + Entry::Occupied(e) => (e.get().clone(), false), + Entry::Vacant(e) => { + e.insert(new_dict.clone()); + (new_dict, true) + } + }; + drop(data); // Release lock before TLS access + + // Register cleanup guard only if we inserted a new entry + if need_guard { + let guard = LocalGuard { + local: Arc::downgrade(&self.inner), + thread_id, + }; + LOCAL_GUARDS.with(|guards| { + guards.borrow_mut().push(guard); + }); + } + + dict + } + + #[pyslot] + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Self { + inner: Arc::new(LocalData { + data: parking_lot::Mutex::new(std::collections::HashMap::new()), + }), + } + .into_ref_with_type(vm, cls) + .map(Into::into) + } + } + + impl GetAttr for Local { + fn getattro(zelf: &Py<Self>, attr: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + let l_dict = zelf.l_dict(vm); + if attr.as_bytes() == b"__dict__" { + Ok(l_dict.into()) + } else { + zelf.as_object() + .generic_getattr_opt(attr, Some(l_dict), vm)? + .ok_or_else(|| { + vm.new_attribute_error(format!( + "{} has no attribute '{}'", + zelf.class().name(), + attr + )) + }) + } + } + } + + impl SetAttr for Local { + fn setattro( + zelf: &Py<Self>, + attr: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + if attr.as_bytes() == b"__dict__" { + Err(vm.new_attribute_error(format!( + "{} attribute '__dict__' is read-only", + zelf.class().name() + ))) + } else { + let dict = zelf.l_dict(vm); + if let PySetterValue::Assign(value) = value { + dict.set_item(attr, value, vm)?; + } else { + dict.del_item(attr, vm)?; + } + Ok(()) + } + } + } + + // Registry of all ThreadHandles for fork cleanup + // Stores weak references so handles can be garbage collected normally + pub type HandleEntry = ( + Weak<parking_lot::Mutex<ThreadHandleInner>>, + Weak<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + ); + + // Re-export type from vm::thread for PyGlobalState + pub use crate::vm::thread::CurrentFrameSlot; + + /// Get all threads' current (top) frames. Used by sys._current_frames(). + pub fn get_all_current_frames(vm: &VirtualMachine) -> Vec<(u64, FrameRef)> { + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .filter_map(|(id, slot)| { + let frames = slot.frames.lock(); + // SAFETY: the owning thread can't pop while we hold the Mutex, + // so the FramePtr is valid for the duration of the lock. + frames + .last() + .map(|fp| (*id, unsafe { fp.as_ref() }.to_owned())) + }) + .collect() + } + + /// Called after fork() in child process to mark all other threads as done. + /// This prevents join() from hanging on threads that don't exist in the child. + /// + /// Precondition: `reinit_locks_after_fork()` has already been called, so all + /// parking_lot-based locks in VmState are in unlocked state. + #[cfg(unix)] + pub fn after_fork_child(vm: &VirtualMachine) { + let current_ident = get_ident(); + + // Update main thread ident - after fork, the current thread becomes the main thread + vm.state.main_thread_ident.store(current_ident); + + // Reinitialize frame slot for current thread. + // Locks are already reinit'd, so lock() is safe. + crate::vm::thread::reinit_frame_slot_after_fork(vm); + + // Clean up thread handles. All VmState locks were reinit'd to unlocked, + // so lock() won't deadlock. Per-thread Arc<Mutex<ThreadHandleInner>> + // locks are also reinit'd below before use. + { + let mut handles = vm.state.thread_handles.lock(); + handles.retain(|(inner_weak, done_event_weak): &HandleEntry| { + let Some(inner) = inner_weak.upgrade() else { + return false; + }; + let Some(done_event) = done_event_weak.upgrade() else { + return false; + }; + + // Reinit this per-handle lock in case a dead thread held it + reinit_parking_lot_mutex(&inner); + let mut inner_guard = inner.lock(); + + if inner_guard.ident == current_ident { + return true; + } + if inner_guard.state == ThreadHandleState::NotStarted { + return true; + } + + inner_guard.state = ThreadHandleState::Done; + inner_guard.join_handle = None; + drop(inner_guard); + + // Reinit and set the done event + let (lock, cvar) = &*done_event; + reinit_parking_lot_mutex(lock); + *lock.lock() = true; + cvar.notify_all(); + + true + }); + } + + // Clean up shutdown_handles. + { + let mut handles = vm.state.shutdown_handles.lock(); + handles.retain(|(inner_weak, done_event_weak): &ShutdownEntry| { + let Some(inner) = inner_weak.upgrade() else { + return false; + }; + let Some(done_event) = done_event_weak.upgrade() else { + return false; + }; + + reinit_parking_lot_mutex(&inner); + let mut inner_guard = inner.lock(); + + if inner_guard.ident == current_ident { + return true; + } + if inner_guard.state == ThreadHandleState::NotStarted { + return true; + } + + inner_guard.state = ThreadHandleState::Done; + drop(inner_guard); + + let (lock, cvar) = &*done_event; + reinit_parking_lot_mutex(lock); + *lock.lock() = true; + cvar.notify_all(); + + false + }); + } + } + + /// Reset a parking_lot::Mutex to unlocked state after fork. + #[cfg(unix)] + fn reinit_parking_lot_mutex<T: ?Sized>(mutex: &parking_lot::Mutex<T>) { + unsafe { rustpython_common::lock::zero_reinit_after_fork(mutex.raw()) }; + } + + // Thread handle state enum + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum ThreadHandleState { + NotStarted, + Starting, + Running, + Done, + } + + // Internal shared state for thread handle + pub struct ThreadHandleInner { + pub state: ThreadHandleState, + pub ident: u64, + pub join_handle: Option<thread::JoinHandle<()>>, + pub joining: bool, // True if a thread is currently joining + pub joined: bool, // Track if join has completed + } + + impl fmt::Debug for ThreadHandleInner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ThreadHandleInner") + .field("state", &self.state) + .field("ident", &self.ident) + .field("join_handle", &self.join_handle.is_some()) + .field("joining", &self.joining) + .field("joined", &self.joined) + .finish() + } + } + + /// _ThreadHandle - handle for joinable threads + #[pyattr] + #[pyclass(module = "_thread", name = "_ThreadHandle")] + #[derive(Debug, PyPayload)] + struct ThreadHandle { + inner: Arc<parking_lot::Mutex<ThreadHandleInner>>, + // Event to signal thread completion (for timed join support) + done_event: Arc<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + } + + #[pyclass(with(Representable))] + impl ThreadHandle { + fn new(vm: &VirtualMachine) -> Self { + let inner = Arc::new(parking_lot::Mutex::new(ThreadHandleInner { + state: ThreadHandleState::NotStarted, + ident: 0, + join_handle: None, + joining: false, + joined: false, + })); + let done_event = + Arc::new((parking_lot::Mutex::new(false), parking_lot::Condvar::new())); + + // Register in global registry for fork cleanup + vm.state + .thread_handles + .lock() + .push((Arc::downgrade(&inner), Arc::downgrade(&done_event))); + + Self { inner, done_event } + } + + fn join_internal( + inner: &Arc<parking_lot::Mutex<ThreadHandleInner>>, + done_event: &Arc<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + timeout_duration: Option<Duration>, + vm: &VirtualMachine, + ) -> PyResult<()> { + Self::check_started(inner, vm)?; + + let deadline = + timeout_duration.and_then(|timeout| std::time::Instant::now().checked_add(timeout)); + + // Wait for thread completion using Condvar (supports timeout) + // Loop to handle spurious wakeups + let (lock, cvar) = &**done_event; + let mut done = lock.lock(); + + // ThreadHandle_join semantics: self-join/finalizing checks + // apply only while target thread has not reported it is exiting yet. + if !*done { + let inner_guard = inner.lock(); + let current_ident = get_ident(); + if inner_guard.ident == current_ident + && inner_guard.state == ThreadHandleState::Running + { + return Err(vm.new_runtime_error("Cannot join current thread")); + } + if vm + .state + .finalizing + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.python_finalization_error.to_owned(), + "cannot join thread at interpreter shutdown" + .to_owned() + .into(), + )); + } + } + + while !*done { + if let Some(timeout) = timeout_duration { + let remaining = deadline.map_or(timeout, |deadline| { + deadline.saturating_duration_since(std::time::Instant::now()) + }); + if remaining.is_zero() { + return Ok(()); + } + let result = vm.allow_threads(|| cvar.wait_for(&mut done, remaining)); + if result.timed_out() && !*done { + // Timeout occurred and done is still false + return Ok(()); + } + } else { + // Infinite wait + vm.allow_threads(|| cvar.wait(&mut done)); + } + } + drop(done); + + // Thread is done, now perform cleanup + let join_handle = { + let mut inner_guard = inner.lock(); + + // If already joined, return immediately (idempotent) + if inner_guard.joined { + return Ok(()); + } + + // If another thread is already joining, wait for them to finish + if inner_guard.joining { + drop(inner_guard); + // Wait on done_event + let (lock, cvar) = &**done_event; + let mut done = lock.lock(); + while !*done { + vm.allow_threads(|| cvar.wait(&mut done)); + } + return Ok(()); + } + + // Mark that we're joining + inner_guard.joining = true; + + // Take the join handle if available + inner_guard.join_handle.take() + }; + + // Perform the actual join outside the lock + if let Some(handle) = join_handle { + // Ignore the result - panics in spawned threads are already handled + let _ = vm.allow_threads(|| handle.join()); + } + + // Mark as joined and clear joining flag + { + let mut inner_guard = inner.lock(); + inner_guard.joined = true; + inner_guard.joining = false; + } + + Ok(()) + } + + fn check_started( + inner: &Arc<parking_lot::Mutex<ThreadHandleInner>>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let state = inner.lock().state; + if matches!( + state, + ThreadHandleState::NotStarted | ThreadHandleState::Starting + ) { + return Err(vm.new_runtime_error("thread not started")); + } + Ok(()) + } + + fn set_done_internal( + inner: &Arc<parking_lot::Mutex<ThreadHandleInner>>, + done_event: &Arc<(parking_lot::Mutex<bool>, parking_lot::Condvar)>, + vm: &VirtualMachine, + ) -> PyResult<()> { + Self::check_started(inner, vm)?; + { + let mut inner_guard = inner.lock(); + inner_guard.state = ThreadHandleState::Done; + // _set_done() detach path. Dropping the JoinHandle + // detaches the underlying Rust thread. + inner_guard.join_handle = None; + inner_guard.joining = false; + inner_guard.joined = true; + } + remove_from_shutdown_handles(vm, inner, done_event); + + let (lock, cvar) = &**done_event; + *lock.lock() = true; + cvar.notify_all(); + Ok(()) + } + + fn parse_join_timeout( + timeout_obj: Option<crate::PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<Duration>> { + const JOIN_TIMEOUT_MAX_SECONDS: i64 = TIMEOUT_MAX_IN_MICROSECONDS / 1_000_000; + let Some(timeout_obj) = timeout_obj else { + return Ok(None); + }; + + if let Some(t) = timeout_obj.try_index_opt(vm) { + let t: i64 = t?.try_to_primitive(vm).map_err(|_| { + vm.new_overflow_error("timestamp too large to convert to C PyTime_t") + })?; + if !(-JOIN_TIMEOUT_MAX_SECONDS..=JOIN_TIMEOUT_MAX_SECONDS).contains(&t) { + return Err( + vm.new_overflow_error("timestamp too large to convert to C PyTime_t") + ); + } + if t < 0 { + return Ok(None); + } + return Ok(Some(Duration::from_secs(t as u64))); + } + + if let Some(t) = timeout_obj.try_float_opt(vm) { + let t = t?.to_f64(); + if t.is_nan() { + return Err(vm.new_value_error("Invalid value NaN (not a number)")); + } + if !t.is_finite() || !(-TIMEOUT_MAX..=TIMEOUT_MAX).contains(&t) { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + if t < 0.0 { + return Ok(None); + } + return Ok(Some(Duration::from_secs_f64(t))); + } + + Err(vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer or float", + timeout_obj.class().name() + ))) + } + + #[pygetset] + fn ident(&self) -> u64 { + self.inner.lock().ident + } + + #[pymethod] + fn is_done(&self, f_args: FuncArgs, vm: &VirtualMachine) -> PyResult<bool> { + if !f_args.kwargs.is_empty() { + return Err(vm.new_type_error("_ThreadHandle.is_done() takes no keyword arguments")); + } + let given = f_args.args.len(); + if given != 0 { + return Err(vm.new_type_error(format!( + "_ThreadHandle.is_done() takes no arguments ({given} given)" + ))); + } + + // If completion was observed, perform one-time join cleanup + // before returning True. + let done = { + let (lock, _) = &*self.done_event; + *lock.lock() + }; + if !done { + return Ok(false); + } + Self::join_internal(&self.inner, &self.done_event, Some(Duration::ZERO), vm)?; + Ok(true) + } + + #[pymethod] + fn _set_done(&self, f_args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + if !f_args.kwargs.is_empty() { + return Err( + vm.new_type_error("_ThreadHandle._set_done() takes no keyword arguments") + ); + } + let given = f_args.args.len(); + if given != 0 { + return Err(vm.new_type_error(format!( + "_ThreadHandle._set_done() takes no arguments ({given} given)" + ))); + } + + Self::set_done_internal(&self.inner, &self.done_event, vm) + } + + #[pymethod] + fn join(&self, mut f_args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + if !f_args.kwargs.is_empty() { + return Err(vm.new_type_error("_ThreadHandle.join() takes no keyword arguments")); + } + let given = f_args.args.len(); + if given > 1 { + return Err( + vm.new_type_error(format!("join() takes at most 1 argument ({given} given)")) + ); + } + let timeout = f_args.take_positional().filter(|obj| !vm.is_none(obj)); + let timeout_duration = Self::parse_join_timeout(timeout, vm)?; + Self::join_internal(&self.inner, &self.done_event, timeout_duration, vm) + } + + #[pyslot] + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + ThreadHandle::new(vm) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + } + + impl Representable for ThreadHandle { + fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + let ident = zelf.inner.lock().ident; + Ok(format!( + "<{} object: ident={ident}>", + zelf.class().slot_name() + )) + } + } + + #[pyfunction] + fn start_joinable_thread( + mut f_args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<PyRef<ThreadHandle>> { + let given = f_args.args.len() + f_args.kwargs.len(); + if given > 3 { + return Err(vm.new_type_error(format!( + "start_joinable_thread() takes at most 3 arguments ({given} given)" + ))); + } + + let function_pos = f_args.take_positional(); + let function_kw = f_args.take_keyword("function"); + if function_pos.is_some() && function_kw.is_some() { + return Err(vm.new_type_error( + "argument for start_joinable_thread() given by name ('function') and position (1)", + )); + } + let Some(function_obj) = function_pos.or(function_kw) else { + return Err(vm.new_type_error( + "start_joinable_thread() missing required argument 'function' (pos 1)", + )); + }; + + let handle_pos = f_args.take_positional(); + let handle_kw = f_args.take_keyword("handle"); + if handle_pos.is_some() && handle_kw.is_some() { + return Err(vm.new_type_error( + "argument for start_joinable_thread() given by name ('handle') and position (2)", + )); + } + let handle_obj = handle_pos.or(handle_kw); + + let daemon_pos = f_args.take_positional(); + let daemon_kw = f_args.take_keyword("daemon"); + if daemon_pos.is_some() && daemon_kw.is_some() { + return Err(vm.new_type_error( + "argument for start_joinable_thread() given by name ('daemon') and position (3)", + )); + } + let daemon = daemon_pos + .or(daemon_kw) + .map_or(Ok(true), |obj| obj.try_to_bool(vm))?; + + // Match CPython parser precedence: + // - required positional/keyword argument errors are raised before + // unknown keyword errors when `function` is missing. + if let Some(unexpected) = f_args.kwargs.keys().next() { + let suggestion = ["function", "handle", "daemon"] + .iter() + .filter_map(|candidate| { + let max_distance = (unexpected.len() + candidate.len() + 3) * MOVE_COST / 6; + let distance = levenshtein_distance( + unexpected.as_bytes(), + candidate.as_bytes(), + max_distance, + ); + (distance <= max_distance).then_some((distance, *candidate)) + }) + .min_by_key(|(distance, _)| *distance) + .map(|(_, candidate)| candidate); + let msg = if let Some(suggestion) = suggestion { + format!( + "start_joinable_thread() got an unexpected keyword argument '{unexpected}'. Did you mean '{suggestion}'?" + ) + } else { + format!("start_joinable_thread() got an unexpected keyword argument '{unexpected}'") + }; + return Err(vm.new_type_error(msg)); + } + + if function_obj.to_callable().is_none() { + return Err(vm.new_type_error("thread function must be callable")); + } + let function: ArgCallable = function_obj.clone().try_into_value(vm)?; + + let thread_handle_type = ThreadHandle::class(&vm.ctx); + let handle = if let Some(handle_obj) = handle_obj { + if vm.is_none(&handle_obj) { + None + } else if !handle_obj.class().is(thread_handle_type) { + return Err(vm.new_type_error("'handle' must be a _ThreadHandle")); + } else { + Some( + handle_obj + .downcast::<ThreadHandle>() + .map_err(|_| vm.new_type_error("'handle' must be a _ThreadHandle"))?, + ) + } + } else { + None + }; + + vm.sys_module.get_attr("audit", vm)?.call( + ( + "_thread.start_joinable_thread", + function_obj, + daemon, + handle + .as_ref() + .map_or_else(|| vm.ctx.none(), |h| h.clone().into()), + ), + vm, + )?; + + if vm + .state + .finalizing + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.python_finalization_error.to_owned(), + "can't create new thread at interpreter shutdown" + .to_owned() + .into(), + )); + } + + let handle = match handle { + Some(h) => h, + None => ThreadHandle::new(vm).into_ref(&vm.ctx), + }; + + // Must only start once (ThreadHandle_start). + { + let mut inner = handle.inner.lock(); + if inner.state != ThreadHandleState::NotStarted { + return Err(vm.new_runtime_error("thread already started")); + } + inner.state = ThreadHandleState::Starting; + inner.ident = 0; + inner.join_handle = None; + inner.joining = false; + inner.joined = false; + } + // Starting a handle always resets the completion event. + { + let (done_lock, _) = &*handle.done_event; + *done_lock.lock() = false; + } + + // Add non-daemon threads to shutdown registry so _shutdown() will wait for them + if !daemon { + add_to_shutdown_handles(vm, &handle.inner, &handle.done_event); + } + + let func = function; + let handle_clone = handle.clone(); + let inner_clone = handle.inner.clone(); + let done_event_clone = handle.done_event.clone(); + // Use std::sync (pthread-based) instead of parking_lot for these + // events so they remain fork-safe without the parking_lot_core patch. + let started_event = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); + let started_event_clone = Arc::clone(&started_event); + let handle_ready_event = + Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); + let handle_ready_event_clone = Arc::clone(&handle_ready_event); + + let mut thread_builder = thread::Builder::new(); + let stacksize = vm.state.stacksize.load(); + if stacksize != 0 { + thread_builder = thread_builder.stack_size(stacksize); + } + + let join_handle = thread_builder + .spawn(vm.new_thread().make_spawn_func(move |vm| { + // Publish ident for the parent starter thread. + { + inner_clone.lock().ident = get_ident(); + } + { + let (started_lock, started_cvar) = &*started_event_clone; + *started_lock.lock().unwrap() = true; + started_cvar.notify_all(); + } + // Don't execute the target function until parent marks the + // handle as running. + { + let (ready_lock, ready_cvar) = &*handle_ready_event_clone; + let mut ready = ready_lock.lock().unwrap(); + while !*ready { + // Short timeout so we stay responsive to STW requests. + let (guard, _) = ready_cvar + .wait_timeout(ready, core::time::Duration::from_millis(1)) + .unwrap(); + ready = guard; + } + } + + // Ensure cleanup happens even if the function panics + let inner_for_cleanup = inner_clone.clone(); + let done_event_for_cleanup = done_event_clone.clone(); + let vm_state = vm.state.clone(); + scopeguard::defer! { + // Mark as done + inner_for_cleanup.lock().state = ThreadHandleState::Done; + + // Handle sentinels + for lock in SENTINELS.take() { + if lock.mu.is_locked() { + unsafe { lock.mu.unlock() }; + } + } + + // Clean up thread-local data while VM context is still active + cleanup_thread_local_data(); + + // Clean up frame tracking + crate::vm::thread::cleanup_current_thread_frames(vm); + + vm_state.thread_count.fetch_sub(1); + + // The runtime no longer needs to wait for this thread. + remove_from_shutdown_handles(vm, &inner_for_cleanup, &done_event_for_cleanup); + + // Signal waiting threads that this thread is done + // This must be LAST to ensure all cleanup is complete before join() returns + { + let (lock, cvar) = &*done_event_for_cleanup; + *lock.lock() = true; + cvar.notify_all(); + } + } + + // Increment thread count when thread actually starts executing + vm_state.thread_count.fetch_add(1); + + // Run the function + match func.invoke((), vm) { + Ok(_) => {} + Err(e) if e.fast_isinstance(vm.ctx.exceptions.system_exit) => {} + Err(exc) => { + vm.run_unraisable( + exc, + Some("Exception ignored in thread started by".to_owned()), + func.into(), + ); + } + } + })) + .map_err(|_err| { + // force_done + remove_from_shutdown_handles on start failure. + { + let mut inner = handle.inner.lock(); + inner.state = ThreadHandleState::Done; + inner.join_handle = None; + inner.joining = false; + inner.joined = true; + } + { + let (done_lock, done_cvar) = &*handle.done_event; + *done_lock.lock() = true; + done_cvar.notify_all(); + } + if !daemon { + remove_from_shutdown_handles(vm, &handle.inner, &handle.done_event); + } + vm.new_runtime_error("can't start new thread") + })?; + + // Wait until the new thread has reported its ident. + { + let (started_lock, started_cvar) = &*started_event; + let mut started = started_lock.lock().unwrap(); + while !*started { + let (guard, _) = started_cvar + .wait_timeout(started, core::time::Duration::from_millis(1)) + .unwrap(); + started = guard; + } + } + + // Mark the handle running in the parent thread (like CPython's + // ThreadHandle_start sets THREAD_HANDLE_RUNNING after spawn succeeds). + { + let mut inner = handle.inner.lock(); + inner.join_handle = Some(join_handle); + inner.state = ThreadHandleState::Running; + } + + // Unblock the started thread once handle state is fully published. + { + let (ready_lock, ready_cvar) = &*handle_ready_event; + *ready_lock.lock().unwrap() = true; + ready_cvar.notify_all(); + } + + Ok(handle_clone) + } +} diff --git a/crates/vm/src/stdlib/_types.rs b/crates/vm/src/stdlib/_types.rs new file mode 100644 index 00000000000..385ecf8cf5a --- /dev/null +++ b/crates/vm/src/stdlib/_types.rs @@ -0,0 +1,157 @@ +//! Implementation of the `_types` module. +//! +//! This module exposes built-in types that are used by the `types` module. + +pub(crate) use _types::module_def; + +#[pymodule] +#[allow(non_snake_case)] +mod _types { + use crate::{PyObjectRef, VirtualMachine}; + + #[pyattr] + fn AsyncGeneratorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.async_generator.to_owned().into() + } + + #[pyattr] + fn BuiltinFunctionType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .types + .builtin_function_or_method_type + .to_owned() + .into() + } + + #[pyattr] + fn BuiltinMethodType(vm: &VirtualMachine) -> PyObjectRef { + // Same as BuiltinFunctionType in CPython + vm.ctx + .types + .builtin_function_or_method_type + .to_owned() + .into() + } + + #[pyattr] + fn CapsuleType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.capsule_type.to_owned().into() + } + + #[pyattr] + fn CellType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.cell_type.to_owned().into() + } + + #[pyattr] + fn CodeType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.code_type.to_owned().into() + } + + #[pyattr] + fn CoroutineType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.coroutine_type.to_owned().into() + } + + #[pyattr] + fn EllipsisType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.ellipsis_type.to_owned().into() + } + + #[pyattr] + fn FrameType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.frame_type.to_owned().into() + } + + #[pyattr] + fn FunctionType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.function_type.to_owned().into() + } + + #[pyattr] + fn GeneratorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.generator_type.to_owned().into() + } + + #[pyattr] + fn GenericAlias(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.generic_alias_type.to_owned().into() + } + + #[pyattr] + fn GetSetDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.getset_type.to_owned().into() + } + + #[pyattr] + fn LambdaType(vm: &VirtualMachine) -> PyObjectRef { + // Same as FunctionType in CPython + vm.ctx.types.function_type.to_owned().into() + } + + #[pyattr] + fn MappingProxyType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.mappingproxy_type.to_owned().into() + } + + #[pyattr] + fn MemberDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.member_descriptor_type.to_owned().into() + } + + #[pyattr] + fn MethodDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.method_descriptor_type.to_owned().into() + } + + #[pyattr] + fn ClassMethodDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + // TODO: implement as separate type + vm.ctx.types.method_descriptor_type.to_owned().into() + } + + #[pyattr] + fn MethodType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.bound_method_type.to_owned().into() + } + + #[pyattr] + fn MethodWrapperType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.method_wrapper_type.to_owned().into() + } + + #[pyattr] + fn ModuleType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.module_type.to_owned().into() + } + + #[pyattr] + fn NoneType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.none_type.to_owned().into() + } + + #[pyattr] + fn NotImplementedType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.not_implemented_type.to_owned().into() + } + + #[pyattr] + fn SimpleNamespace(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.namespace_type.to_owned().into() + } + + #[pyattr] + fn TracebackType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.traceback_type.to_owned().into() + } + + #[pyattr] + fn UnionType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.union_type.to_owned().into() + } + + #[pyattr] + fn WrapperDescriptorType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.wrapper_descriptor_type.to_owned().into() + } +} diff --git a/crates/vm/src/stdlib/_typing.rs b/crates/vm/src/stdlib/_typing.rs new file mode 100644 index 00000000000..7467a7f2574 --- /dev/null +++ b/crates/vm/src/stdlib/_typing.rs @@ -0,0 +1,505 @@ +// spell-checker:ignore typevarobject funcobj typevartuples +use crate::{ + Context, PyResult, VirtualMachine, builtins::pystr::AsPyStr, class::PyClassImpl, + function::IntoFuncArgs, +}; + +pub use crate::stdlib::typevar::{ + Generic, ParamSpec, ParamSpecArgs, ParamSpecKwargs, TypeVar, TypeVarTuple, + set_typeparam_default, +}; +pub(crate) use decl::module_def; +pub use decl::*; + +/// Initialize typing types (call extend_class) +pub fn init(ctx: &'static Context) { + NoDefault::extend_class(ctx, ctx.types.typing_no_default_type); +} + +pub fn call_typing_func_object<'a>( + vm: &VirtualMachine, + func_name: impl AsPyStr<'a>, + args: impl IntoFuncArgs, +) -> PyResult { + let module = vm.import("typing", 0)?; + let func = module.get_attr(func_name.as_pystr(&vm.ctx), vm)?; + func.call(args, vm) +} + +#[pymodule(name = "_typing", with(super::typevar::typevar))] +pub(crate) mod decl { + use crate::common::lock::LazyLock; + use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, + builtins::{PyGenericAlias, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, type_}, + common::wtf8::Wtf8Buf, + function::FuncArgs, + protocol::{PyMappingMethods, PyNumberMethods}, + types::{AsMapping, AsNumber, Callable, Constructor, Iterable, Representable}, + }; + + #[pyfunction] + pub(crate) fn _idfunc(args: FuncArgs, _vm: &VirtualMachine) -> PyObjectRef { + args.args[0].clone() + } + + #[pyfunction(name = "override")] + pub(crate) fn r#override(func: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Set __override__ attribute to True + // Skip the attribute silently if it is not writable. + // AttributeError happens if the object has __slots__ or a + // read-only property, TypeError if it's a builtin class. + let _ = func.set_attr("__override__", vm.ctx.true_value.clone(), vm); + Ok(func) + } + + #[pyclass(no_attr, name = "NoDefaultType", module = "typing")] + #[derive(Debug, PyPayload)] + pub struct NoDefault; + + #[pyclass(with(Constructor, Representable), flags(IMMUTABLETYPE))] + impl NoDefault { + #[pymethod] + fn __reduce__(&self, _vm: &VirtualMachine) -> String { + "NoDefault".to_owned() + } + } + + impl Constructor for NoDefault { + type Args = (); + + fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let _: () = args.bind(vm)?; + Ok(vm.ctx.typing_no_default.clone().into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unreachable!("NoDefault is a singleton, use slot_new") + } + } + + impl Representable for NoDefault { + #[inline(always)] + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + Ok("typing.NoDefault".to_owned()) + } + } + + #[pyattr] + #[pyclass(name = "_ConstEvaluator", module = "_typing")] + #[derive(Debug, PyPayload)] + pub(crate) struct ConstEvaluator { + value: PyObjectRef, + } + + #[pyclass(with(Constructor, Callable, Representable), flags(IMMUTABLETYPE))] + impl ConstEvaluator {} + + impl Constructor for ConstEvaluator { + type Args = FuncArgs; + + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create '_typing._ConstEvaluator' instances")) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unreachable!("ConstEvaluator cannot be instantiated from Python") + } + } + + /// annotationlib.Format.STRING = 4 + const ANNOTATE_FORMAT_STRING: i32 = 4; + + impl Callable for ConstEvaluator { + type Args = FuncArgs; + + fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let (format,): (i32,) = args.bind(vm)?; + let value = &zelf.value; + if format == ANNOTATE_FORMAT_STRING { + return typing_type_repr_value(value, vm); + } + Ok(value.clone()) + } + } + + /// String representation of a type for annotation purposes. + /// Equivalent of _Py_typing_type_repr. + fn typing_type_repr(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { + // Ellipsis + if obj.is(&vm.ctx.ellipsis) { + return Ok("...".to_owned()); + } + // NoneType -> "None" + if obj.is(&vm.ctx.types.none_type.as_object()) { + return Ok("None".to_owned()); + } + // Generic aliases (has __origin__ and __args__) -> repr + let has_origin = obj.get_attr("__origin__", vm).is_ok(); + let has_args = obj.get_attr("__args__", vm).is_ok(); + if has_origin && has_args { + return Ok(obj.repr(vm)?.to_string()); + } + // Has __qualname__ and __module__ + if let Ok(qualname) = obj.get_attr("__qualname__", vm) + && let Ok(module) = obj.get_attr("__module__", vm) + && !vm.is_none(&module) + && let Some(module_str) = module.downcast_ref::<crate::builtins::PyStr>() + { + if module_str.as_bytes() == b"builtins" { + return Ok(qualname.str_utf8(vm)?.as_str().to_owned()); + } + return Ok(format!( + "{}.{}", + module_str.as_wtf8(), + qualname.str_utf8(vm)?.as_str() + )); + } + // Fallback to repr + Ok(obj.repr(vm)?.to_string()) + } + + /// Format a value as a string for ANNOTATE_FORMAT_STRING. + /// Handles tuples specially by wrapping in parentheses. + fn typing_type_repr_value(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(tuple) = value.try_to_ref::<PyTuple>(vm) { + let mut parts = Vec::with_capacity(tuple.len()); + for item in tuple.iter() { + parts.push(typing_type_repr(item, vm)?); + } + let inner = if parts.len() == 1 { + format!("{},", parts[0]) + } else { + parts.join(", ") + }; + Ok(vm.ctx.new_str(format!("({})", inner)).into()) + } else { + Ok(vm.ctx.new_str(typing_type_repr(value, vm)?).into()) + } + } + + impl Representable for ConstEvaluator { + fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let value_repr = zelf.value.repr(vm)?; + Ok(format!("<constevaluator {}>", value_repr)) + } + } + + pub(crate) fn const_evaluator_alloc(value: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + ConstEvaluator { value }.into_ref(&vm.ctx).into() + } + + #[pyattr] + #[pyclass(name, module = "typing")] + #[derive(Debug, PyPayload)] + pub(crate) struct TypeAliasType { + name: PyStrRef, + type_params: PyTupleRef, + compute_value: PyObjectRef, + cached_value: crate::common::lock::PyMutex<Option<PyObjectRef>>, + module: Option<PyObjectRef>, + is_lazy: bool, + } + #[pyclass( + with(Constructor, Representable, AsMapping, AsNumber, Iterable), + flags(IMMUTABLETYPE) + )] + impl TypeAliasType { + /// Create from intrinsic: compute_value is a callable that returns the value + pub fn new(name: PyStrRef, type_params: PyTupleRef, compute_value: PyObjectRef) -> Self { + Self { + name, + type_params, + compute_value, + cached_value: crate::common::lock::PyMutex::new(None), + module: None, + is_lazy: true, + } + } + + /// Create with an eagerly evaluated value (used by constructor) + fn new_eager( + name: PyStrRef, + type_params: PyTupleRef, + value: PyObjectRef, + module: Option<PyObjectRef>, + ) -> Self { + Self { + name, + type_params, + compute_value: value.clone(), + cached_value: crate::common::lock::PyMutex::new(Some(value)), + module, + is_lazy: false, + } + } + + #[pygetset] + fn __name__(&self) -> PyObjectRef { + self.name.clone().into() + } + + #[pygetset] + fn __value__(&self, vm: &VirtualMachine) -> PyResult { + let cached = self.cached_value.lock().clone(); + if let Some(value) = cached { + return Ok(value); + } + // Call evaluator with format=1 (FORMAT_VALUE) + let value = self.compute_value.call((1i32,), vm)?; + *self.cached_value.lock() = Some(value.clone()); + Ok(value) + } + + #[pygetset] + fn __type_params__(&self) -> PyTupleRef { + self.type_params.clone() + } + + #[pygetset] + fn __parameters__(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // TypeVarTuples must be unpacked in __parameters__ + unpack_typevartuples(&self.type_params, vm).map(|t| t.into()) + } + + #[pygetset] + fn __module__(&self, vm: &VirtualMachine) -> PyObjectRef { + if let Some(ref module) = self.module { + return module.clone(); + } + // Fall back to compute_value's __module__ (like PyFunction_GetModule) + if let Ok(module) = self.compute_value.get_attr("__module__", vm) { + return module; + } + vm.ctx.none() + } + + fn __getitem__(zelf: PyRef<Self>, args: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if zelf.type_params.is_empty() { + return Err(vm.new_type_error("Only generic type aliases are subscriptable")); + } + let args_tuple = if let Ok(tuple) = args.try_to_ref::<PyTuple>(vm) { + tuple.to_owned() + } else { + PyTuple::new_ref(vec![args], &vm.ctx) + }; + let origin: PyObjectRef = zelf.as_object().to_owned(); + Ok(PyGenericAlias::new(origin, args_tuple, false, vm).into_pyobject(vm)) + } + + #[pymethod] + fn __reduce__(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyObjectRef { + zelf.name.clone().into() + } + + #[pymethod] + fn __typing_unpacked_tuple_args__(&self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } + + #[pygetset] + fn evaluate_value(&self, vm: &VirtualMachine) -> PyResult { + if self.is_lazy { + return Ok(self.compute_value.clone()); + } + Ok(const_evaluator_alloc(self.compute_value.clone(), vm)) + } + + /// Check type_params ordering: non-default params must precede default params. + /// Uses __default__ attribute to check if a type param has a default value, + /// comparing against typing.NoDefault sentinel (like get_type_param_default). + fn check_type_params( + type_params: &PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult<Option<PyTupleRef>> { + if type_params.is_empty() { + return Ok(None); + } + let no_default = &vm.ctx.typing_no_default; + let mut default_seen = false; + for param in type_params.iter() { + let dflt = param.get_attr("__default__", vm).map_err(|_| { + vm.new_type_error(format!( + "Expected a type param, got {}", + param + .repr(vm) + .map(|s| s.to_string()) + .unwrap_or_else(|_| "?".to_owned()) + )) + })?; + let is_no_default = dflt.is(no_default); + if is_no_default { + if default_seen { + return Err(vm.new_type_error(format!( + "non-default type parameter '{}' follows default type parameter", + param.repr(vm)? + ))); + } + } else { + default_seen = true; + } + } + Ok(Some(type_params.clone())) + } + } + + impl Constructor for TypeAliasType { + type Args = FuncArgs; + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + // typealias(name, value, *, type_params=()) + // name and value are positional-or-keyword; type_params is keyword-only. + + // Reject unexpected keyword arguments + for key in args.kwargs.keys() { + if key != "name" && key != "value" && key != "type_params" { + return Err(vm.new_type_error(format!( + "typealias() got an unexpected keyword argument '{key}'" + ))); + } + } + + // Reject too many positional arguments + if args.args.len() > 2 { + return Err(vm.new_type_error(format!( + "typealias() takes exactly 2 positional arguments ({} given)", + args.args.len() + ))); + } + + // Resolve name: positional[0] or kwarg + let name = if !args.args.is_empty() { + if args.kwargs.contains_key("name") { + return Err(vm.new_type_error( + "argument for typealias() given by name ('name') and position (1)", + )); + } + args.args[0].clone() + } else { + args.kwargs.get("name").cloned().ok_or_else(|| { + vm.new_type_error("typealias() missing required argument 'name' (pos 1)") + })? + }; + + // Resolve value: positional[1] or kwarg + let value = if args.args.len() >= 2 { + if args.kwargs.contains_key("value") { + return Err(vm.new_type_error( + "argument for typealias() given by name ('value') and position (2)", + )); + } + args.args[1].clone() + } else { + args.kwargs.get("value").cloned().ok_or_else(|| { + vm.new_type_error("typealias() missing required argument 'value' (pos 2)") + })? + }; + + let name = name.downcast::<crate::builtins::PyStr>().map_err(|obj| { + vm.new_type_error(format!( + "typealias() argument 'name' must be str, not {}", + obj.class().name() + )) + })?; + + let type_params = if let Some(tp) = args.kwargs.get("type_params") { + let tp = tp + .clone() + .downcast::<crate::builtins::PyTuple>() + .map_err(|_| vm.new_type_error("type_params must be a tuple"))?; + Self::check_type_params(&tp, vm)?; + tp + } else { + vm.ctx.empty_tuple.clone() + }; + + // Get caller's module name from frame globals, like typevar.rs caller() + let module = vm + .current_frame() + .and_then(|f| f.globals.get_item("__name__", vm).ok()); + + Ok(Self::new_eager(name, type_params, value, module)) + } + } + + impl Representable for TypeAliasType { + fn repr_wtf8(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + Ok(zelf.name.as_wtf8().to_owned()) + } + } + + impl AsMapping for TypeAliasType { + fn as_mapping() -> &'static PyMappingMethods { + static AS_MAPPING: LazyLock<PyMappingMethods> = LazyLock::new(|| PyMappingMethods { + subscript: atomic_func!(|mapping, needle, vm| { + let zelf = TypeAliasType::mapping_downcast(mapping); + TypeAliasType::__getitem__(zelf.to_owned(), needle.to_owned(), vm) + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + &AS_MAPPING + } + } + + impl AsNumber for TypeAliasType { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| type_::or_(a.to_owned(), b.to_owned(), vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + + impl Iterable for TypeAliasType { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + // Import typing.Unpack and return iter((Unpack[self],)) + let typing = vm.import("typing", 0)?; + let unpack = typing.get_attr("Unpack", vm)?; + let zelf_obj: PyObjectRef = zelf.into(); + let unpacked = vm.call_method(&unpack, "__getitem__", (zelf_obj,))?; + let tuple = PyTuple::new_ref(vec![unpacked], &vm.ctx); + Ok(tuple.as_object().get_iter(vm)?.into()) + } + } + + /// Wrap TypeVarTuples in Unpack[], matching unpack_typevartuples() + pub(crate) fn unpack_typevartuples( + type_params: &PyTupleRef, + vm: &VirtualMachine, + ) -> PyResult<PyTupleRef> { + let has_tvt = type_params + .iter() + .any(|p| p.downcastable::<crate::stdlib::typevar::TypeVarTuple>()); + if !has_tvt { + return Ok(type_params.clone()); + } + let typing = vm.import("typing", 0)?; + let unpack_cls = typing.get_attr("Unpack", vm)?; + let new_params: Vec<PyObjectRef> = type_params + .iter() + .map(|p| { + if p.downcastable::<crate::stdlib::typevar::TypeVarTuple>() { + vm.call_method(&unpack_cls, "__getitem__", (p.clone(),)) + } else { + Ok(p.clone()) + } + }) + .collect::<PyResult<_>>()?; + Ok(PyTuple::new_ref(new_params, &vm.ctx)) + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + + extend_module!(vm, module, { + "NoDefault" => vm.ctx.typing_no_default.clone(), + "Union" => vm.ctx.types.union_type.to_owned(), + }); + + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/_warnings.rs b/crates/vm/src/stdlib/_warnings.rs new file mode 100644 index 00000000000..a41ce2625c7 --- /dev/null +++ b/crates/vm/src/stdlib/_warnings.rs @@ -0,0 +1,213 @@ +pub(crate) use _warnings::module_def; + +use crate::{Py, PyResult, VirtualMachine, builtins::PyType}; + +pub fn warn( + category: &Py<PyType>, + message: String, + stack_level: usize, + vm: &VirtualMachine, +) -> PyResult<()> { + crate::warn::warn( + vm.new_pyobj(message), + Some(category.to_owned()), + isize::try_from(stack_level).unwrap_or(isize::MAX), + None, + vm, + ) +} + +#[pymodule] +mod _warnings { + use crate::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, + convert::TryFromObject, + function::OptionalArg, + }; + + #[pyattr] + fn filters(vm: &VirtualMachine) -> PyListRef { + vm.state.warnings.filters.clone() + } + + #[pyattr] + fn _defaultaction(vm: &VirtualMachine) -> PyStrRef { + vm.state.warnings.default_action.clone() + } + + #[pyattr] + fn _onceregistry(vm: &VirtualMachine) -> PyDictRef { + vm.state.warnings.once_registry.clone() + } + + #[pyattr] + fn _warnings_context(vm: &VirtualMachine) -> PyObjectRef { + vm.state + .warnings + .context_var + .get_or_init(|| { + // Try to create a real ContextVar if _contextvars is available. + // During early startup it may not be importable yet, in which + // case we fall back to None. This is safe because + // context_aware_warnings defaults to False. + if let Ok(contextvars) = vm.import("_contextvars", 0) + && let Ok(cv_cls) = contextvars.get_attr("ContextVar", vm) + && let Ok(cv) = cv_cls.call(("_warnings_context",), vm) + { + cv + } else { + vm.ctx.none() + } + }) + .clone() + } + + #[pyfunction] + fn _acquire_lock(vm: &VirtualMachine) { + vm.state.warnings.acquire_lock(); + } + + #[pyfunction] + fn _release_lock(vm: &VirtualMachine) -> PyResult<()> { + if !vm.state.warnings.release_lock() { + return Err(vm.new_runtime_error("cannot release un-acquired lock")); + } + Ok(()) + } + + #[pyfunction] + fn _filters_mutated_lock_held(vm: &VirtualMachine) { + vm.state.warnings.filters_mutated(); + } + + #[derive(FromArgs)] + struct WarnArgs { + #[pyarg(positional)] + message: PyObjectRef, + #[pyarg(any, optional)] + category: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + stacklevel: OptionalArg<i32>, + #[pyarg(named, optional)] + source: OptionalArg<PyObjectRef>, + #[pyarg(named, optional)] + skip_file_prefixes: OptionalArg<PyTupleRef>, + } + + /// Validate and resolve the category argument, matching get_category() in C. + fn get_category( + message: &PyObjectRef, + category: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<PyTypeRef>> { + let cat_obj = match category { + Some(c) if !vm.is_none(&c) => c, + _ => { + if message.fast_isinstance(vm.ctx.exceptions.warning) { + return Ok(Some(message.class().to_owned())); + } else { + return Ok(None); // will default to UserWarning in warn_explicit + } + } + }; + + let cat = PyTypeRef::try_from_object(vm, cat_obj.clone()).map_err(|_| { + vm.new_type_error(format!( + "category must be a Warning subclass, not '{}'", + cat_obj.class().name() + )) + })?; + + if !cat.fast_issubclass(vm.ctx.exceptions.warning) { + return Err(vm.new_type_error(format!( + "category must be a Warning subclass, not '{}'", + cat.class().name() + ))); + } + + Ok(Some(cat)) + } + + #[pyfunction] + fn warn(args: WarnArgs, vm: &VirtualMachine) -> PyResult<()> { + let level = args.stacklevel.unwrap_or(1) as isize; + + let category = get_category(&args.message, args.category.into_option(), vm)?; + + // Validate skip_file_prefixes: each element must be a str + let skip_prefixes = args.skip_file_prefixes.into_option(); + if let Some(ref prefixes) = skip_prefixes { + for item in prefixes.iter() { + if !item.class().is(vm.ctx.types.str_type) { + return Err(vm.new_type_error("skip_file_prefixes must be a tuple of strs")); + } + } + } + + crate::warn::warn_with_skip( + args.message, + category, + level, + args.source.into_option(), + skip_prefixes, + vm, + ) + } + + #[derive(FromArgs)] + struct WarnExplicitArgs { + #[pyarg(positional)] + message: PyObjectRef, + #[pyarg(positional)] + category: PyObjectRef, + #[pyarg(positional)] + filename: PyStrRef, + #[pyarg(positional)] + lineno: usize, + #[pyarg(any, optional)] + module: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + registry: OptionalArg<PyObjectRef>, + #[pyarg(any, optional)] + module_globals: OptionalArg<PyObjectRef>, + #[pyarg(named, optional)] + source: OptionalArg<PyObjectRef>, + } + + #[pyfunction] + fn warn_explicit(args: WarnExplicitArgs, vm: &VirtualMachine) -> PyResult<()> { + let registry = args.registry.into_option().unwrap_or_else(|| vm.ctx.none()); + + let module = args.module.into_option(); + + // Validate module_globals: must be None or a dict + if let Some(ref mg) = args.module_globals.into_option() + && !vm.is_none(mg) + && !mg.class().is(vm.ctx.types.dict_type) + { + return Err(vm.new_type_error("module_globals must be a dict")); + } + + let category = if vm.is_none(&args.category) { + None + } else { + Some( + PyTypeRef::try_from_object(vm, args.category) + .map_err(|_| vm.new_type_error("category must be a Warning subclass"))?, + ) + }; + + crate::warn::warn_explicit( + category, + args.message, + args.filename, + args.lineno, + module, + registry, + None, // source_line + args.source.into_option(), + vm, + ) + } +} diff --git a/crates/vm/src/stdlib/_weakref.rs b/crates/vm/src/stdlib/_weakref.rs new file mode 100644 index 00000000000..e7e030b2b01 --- /dev/null +++ b/crates/vm/src/stdlib/_weakref.rs @@ -0,0 +1,65 @@ +//! Implementation in line with the python `weakref` module. +//! +//! See also: +//! - [python weakref module](https://docs.python.org/3/library/weakref.html) +//! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) +//! +pub(crate) use _weakref::module_def; + +#[pymodule] +mod _weakref { + use crate::{ + PyObjectRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyTypeRef, PyWeak}, + }; + + #[pyattr(name = "ref")] + fn ref_(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.types.weakref_type.to_owned() + } + #[pyattr] + fn proxy(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.types.weakproxy_type.to_owned() + } + #[pyattr(name = "ReferenceType")] + fn reference_type(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.types.weakref_type.to_owned() + } + #[pyattr(name = "ProxyType")] + fn proxy_type(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.types.weakproxy_type.to_owned() + } + #[pyattr(name = "CallableProxyType")] + fn callable_proxy_type(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.types.weakproxy_type.to_owned() + } + + #[pyfunction] + fn getweakrefcount(obj: PyObjectRef) -> usize { + obj.weak_count().unwrap_or(0) + } + + #[pyfunction] + fn getweakrefs(obj: PyObjectRef) -> Vec<PyObjectRef> { + match obj.get_weak_references() { + Some(v) => v.into_iter().map(Into::into).collect(), + None => vec![], + } + } + + #[pyfunction] + fn _remove_dead_weakref( + dict: PyDictRef, + key: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + dict._as_dict_inner() + .delete_if(vm, &*key, |wr| { + let wr = wr + .downcast_ref::<PyWeak>() + .ok_or_else(|| vm.new_type_error("not a weakref"))?; + Ok(wr.is_dead()) + }) + .map(drop) + } +} diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs new file mode 100644 index 00000000000..36113f054da --- /dev/null +++ b/crates/vm/src/stdlib/_winapi.rs @@ -0,0 +1,2103 @@ +// spell-checker:disable + +#![allow(non_snake_case)] +pub(crate) use _winapi::module_def; + +#[pymodule] +mod _winapi { + use crate::{ + Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + builtins::PyStrRef, + common::{lock::PyMutex, windows::ToWideString}, + convert::{ToPyException, ToPyResult}, + function::{ArgMapping, ArgSequence, OptionalArg}, + types::Constructor, + windows::{WinHandle, WindowsSysResult}, + }; + use core::ptr::{null, null_mut}; + use rustpython_common::wtf8::Wtf8Buf; + use windows_sys::Win32::Foundation::{HANDLE, MAX_PATH}; + + #[pyattr] + use windows_sys::Win32::{ + Foundation::{ + DUPLICATE_CLOSE_SOURCE, DUPLICATE_SAME_ACCESS, ERROR_ACCESS_DENIED, + ERROR_ALREADY_EXISTS, ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, + ERROR_NETNAME_DELETED, ERROR_NO_DATA, ERROR_NO_SYSTEM_RESOURCES, + ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, ERROR_PIPE_CONNECTED, + ERROR_PRIVILEGE_NOT_HELD, ERROR_SEM_TIMEOUT, GENERIC_READ, GENERIC_WRITE, STILL_ACTIVE, + WAIT_ABANDONED_0, WAIT_OBJECT_0, WAIT_TIMEOUT, + }, + Globalization::{ + LCMAP_FULLWIDTH, LCMAP_HALFWIDTH, LCMAP_HIRAGANA, LCMAP_KATAKANA, + LCMAP_LINGUISTIC_CASING, LCMAP_LOWERCASE, LCMAP_SIMPLIFIED_CHINESE, LCMAP_TITLECASE, + LCMAP_TRADITIONAL_CHINESE, LCMAP_UPPERCASE, + }, + Storage::FileSystem::{ + COPY_FILE_ALLOW_DECRYPTED_DESTINATION, COPY_FILE_COPY_SYMLINK, + COPY_FILE_FAIL_IF_EXISTS, COPY_FILE_NO_BUFFERING, COPY_FILE_NO_OFFLOAD, + COPY_FILE_OPEN_SOURCE_FOR_WRITE, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC, + COPY_FILE_REQUEST_SECURITY_PRIVILEGES, COPY_FILE_RESTARTABLE, + COPY_FILE_RESUME_FROM_PAUSE, COPYFILE2_CALLBACK_CHUNK_FINISHED, + COPYFILE2_CALLBACK_CHUNK_STARTED, COPYFILE2_CALLBACK_ERROR, + COPYFILE2_CALLBACK_POLL_CONTINUE, COPYFILE2_CALLBACK_STREAM_FINISHED, + COPYFILE2_CALLBACK_STREAM_STARTED, COPYFILE2_PROGRESS_CANCEL, + COPYFILE2_PROGRESS_CONTINUE, COPYFILE2_PROGRESS_PAUSE, COPYFILE2_PROGRESS_QUIET, + COPYFILE2_PROGRESS_STOP, FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, + FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_TYPE_CHAR, FILE_TYPE_DISK, FILE_TYPE_PIPE, + FILE_TYPE_REMOTE, FILE_TYPE_UNKNOWN, OPEN_EXISTING, PIPE_ACCESS_DUPLEX, + PIPE_ACCESS_INBOUND, SYNCHRONIZE, + }, + System::{ + Console::{STD_ERROR_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE}, + Memory::{ + FILE_MAP_ALL_ACCESS, FILE_MAP_COPY, FILE_MAP_EXECUTE, FILE_MAP_READ, + FILE_MAP_WRITE, MEM_COMMIT, MEM_FREE, MEM_IMAGE, MEM_MAPPED, MEM_PRIVATE, + MEM_RESERVE, PAGE_EXECUTE, PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE, + PAGE_EXECUTE_WRITECOPY, PAGE_GUARD, PAGE_NOACCESS, PAGE_NOCACHE, PAGE_READONLY, + PAGE_READWRITE, PAGE_WRITECOMBINE, PAGE_WRITECOPY, SEC_COMMIT, SEC_IMAGE, + SEC_LARGE_PAGES, SEC_NOCACHE, SEC_RESERVE, SEC_WRITECOMBINE, + }, + Pipes::{ + NMPWAIT_WAIT_FOREVER, PIPE_READMODE_MESSAGE, PIPE_TYPE_MESSAGE, + PIPE_UNLIMITED_INSTANCES, PIPE_WAIT, + }, + SystemServices::LOCALE_NAME_MAX_LENGTH, + Threading::{ + ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, + CREATE_BREAKAWAY_FROM_JOB, CREATE_DEFAULT_ERROR_MODE, CREATE_NEW_CONSOLE, + CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, DETACHED_PROCESS, HIGH_PRIORITY_CLASS, + IDLE_PRIORITY_CLASS, INFINITE, NORMAL_PRIORITY_CLASS, PROCESS_ALL_ACCESS, + PROCESS_DUP_HANDLE, REALTIME_PRIORITY_CLASS, STARTF_FORCEOFFFEEDBACK, + STARTF_FORCEONFEEDBACK, STARTF_PREVENTPINNING, STARTF_RUNFULLSCREEN, + STARTF_TITLEISAPPID, STARTF_TITLEISLINKNAME, STARTF_UNTRUSTEDSOURCE, + STARTF_USECOUNTCHARS, STARTF_USEFILLATTRIBUTE, STARTF_USEHOTKEY, + STARTF_USEPOSITION, STARTF_USESHOWWINDOW, STARTF_USESIZE, STARTF_USESTDHANDLES, + }, + }, + UI::WindowsAndMessaging::SW_HIDE, + }; + + #[pyattr] + const NULL: isize = 0; + + #[pyattr] + const INVALID_HANDLE_VALUE: isize = -1; + + #[pyattr] + const COPY_FILE_DIRECTORY: u32 = 0x00000080; + + #[pyfunction] + fn CloseHandle(handle: WinHandle) -> WindowsSysResult<i32> { + WindowsSysResult(unsafe { windows_sys::Win32::Foundation::CloseHandle(handle.0) }) + } + + /// CreateFile - Create or open a file or I/O device. + #[pyfunction] + #[allow( + clippy::too_many_arguments, + reason = "matches Win32 CreateFile parameter structure" + )] + fn CreateFile( + file_name: PyStrRef, + desired_access: u32, + share_mode: u32, + _security_attributes: PyObjectRef, // Always NULL (0) + creation_disposition: u32, + flags_and_attributes: u32, + _template_file: PyObjectRef, // Always NULL (0) + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + use windows_sys::Win32::Storage::FileSystem::CreateFileW; + + let file_name_wide = file_name.as_wtf8().to_wide_with_nul(); + + let handle = unsafe { + CreateFileW( + file_name_wide.as_ptr(), + desired_access, + share_mode, + null(), + creation_disposition, + flags_and_attributes, + null_mut(), + ) + }; + + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + + Ok(WinHandle(handle)) + } + + #[pyfunction] + fn GetStdHandle( + std_handle: windows_sys::Win32::System::Console::STD_HANDLE, + vm: &VirtualMachine, + ) -> PyResult<Option<WinHandle>> { + let handle = unsafe { windows_sys::Win32::System::Console::GetStdHandle(std_handle) }; + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + Ok(if handle.is_null() { + // NULL handle - return None + None + } else { + Some(WinHandle(handle)) + }) + } + + #[pyfunction] + fn CreatePipe( + _pipe_attrs: PyObjectRef, + size: u32, + vm: &VirtualMachine, + ) -> PyResult<(WinHandle, WinHandle)> { + use windows_sys::Win32::Foundation::HANDLE; + let (read, write) = unsafe { + let mut read = core::mem::MaybeUninit::<HANDLE>::uninit(); + let mut write = core::mem::MaybeUninit::<HANDLE>::uninit(); + WindowsSysResult(windows_sys::Win32::System::Pipes::CreatePipe( + read.as_mut_ptr(), + write.as_mut_ptr(), + core::ptr::null(), + size, + )) + .to_pyresult(vm)?; + (read.assume_init(), write.assume_init()) + }; + Ok((WinHandle(read), WinHandle(write))) + } + + #[pyfunction] + fn DuplicateHandle( + src_process: WinHandle, + src: WinHandle, + target_process: WinHandle, + access: u32, + inherit: i32, + options: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + use windows_sys::Win32::Foundation::HANDLE; + let target = unsafe { + let mut target = core::mem::MaybeUninit::<HANDLE>::uninit(); + WindowsSysResult(windows_sys::Win32::Foundation::DuplicateHandle( + src_process.0, + src.0, + target_process.0, + target.as_mut_ptr(), + access, + inherit, + options.unwrap_or(0), + )) + .to_pyresult(vm)?; + target.assume_init() + }; + Ok(WinHandle(target)) + } + + #[pyfunction] + fn GetACP() -> u32 { + unsafe { windows_sys::Win32::Globalization::GetACP() } + } + + #[pyfunction] + fn GetCurrentProcess() -> WinHandle { + WinHandle(unsafe { windows_sys::Win32::System::Threading::GetCurrentProcess() }) + } + + #[pyfunction] + fn GetFileType( + h: WinHandle, + vm: &VirtualMachine, + ) -> PyResult<windows_sys::Win32::Storage::FileSystem::FILE_TYPE> { + let file_type = unsafe { windows_sys::Win32::Storage::FileSystem::GetFileType(h.0) }; + if file_type == 0 && unsafe { windows_sys::Win32::Foundation::GetLastError() } != 0 { + Err(vm.new_last_os_error()) + } else { + Ok(file_type) + } + } + + #[pyfunction] + fn GetLastError() -> u32 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } + + #[pyfunction] + fn GetVersion() -> u32 { + unsafe { windows_sys::Win32::System::SystemInformation::GetVersion() } + } + + #[derive(FromArgs)] + struct CreateProcessArgs { + #[pyarg(positional)] + name: Option<PyStrRef>, + #[pyarg(positional)] + command_line: Option<PyStrRef>, + #[pyarg(positional)] + _proc_attrs: PyObjectRef, + #[pyarg(positional)] + _thread_attrs: PyObjectRef, + #[pyarg(positional)] + inherit_handles: i32, + #[pyarg(positional)] + creation_flags: u32, + #[pyarg(positional)] + env_mapping: Option<ArgMapping>, + #[pyarg(positional)] + current_dir: Option<PyStrRef>, + #[pyarg(positional)] + startup_info: PyObjectRef, + } + + #[pyfunction] + fn CreateProcess( + args: CreateProcessArgs, + vm: &VirtualMachine, + ) -> PyResult<(WinHandle, WinHandle, u32, u32)> { + let mut si: windows_sys::Win32::System::Threading::STARTUPINFOEXW = + unsafe { core::mem::zeroed() }; + si.StartupInfo.cb = core::mem::size_of_val(&si) as _; + + macro_rules! si_attr { + ($attr:ident, $t:ty) => {{ + si.StartupInfo.$attr = <Option<$t>>::try_from_object( + vm, + args.startup_info.get_attr(stringify!($attr), vm)?, + )? + .unwrap_or(0) as _ + }}; + ($attr:ident) => {{ + si.StartupInfo.$attr = <Option<_>>::try_from_object( + vm, + args.startup_info.get_attr(stringify!($attr), vm)?, + )? + .unwrap_or(0) + }}; + } + si_attr!(dwFlags); + si_attr!(wShowWindow); + si_attr!(hStdInput, isize); + si_attr!(hStdOutput, isize); + si_attr!(hStdError, isize); + + let mut env = args + .env_mapping + .map(|m| getenvironment(m, vm)) + .transpose()?; + let env = env.as_mut().map_or_else(null_mut, |v| v.as_mut_ptr()); + + let mut attrlist = + getattributelist(args.startup_info.get_attr("lpAttributeList", vm)?, vm)?; + si.lpAttributeList = attrlist + .as_mut() + .map_or_else(null_mut, |l| l.attrlist.as_mut_ptr() as _); + + let wstr = |s: PyStrRef| { + let ws = widestring::WideCString::from_str(s.expect_str()) + .map_err(|err| err.to_pyexception(vm))?; + Ok(ws.into_vec_with_nul()) + }; + + // Validate no embedded null bytes in command name and command line + if let Some(ref name) = args.name + && name.as_bytes().contains(&0) + { + return Err(crate::exceptions::cstring_error(vm)); + } + if let Some(ref cmd) = args.command_line + && cmd.as_bytes().contains(&0) + { + return Err(crate::exceptions::cstring_error(vm)); + } + + let app_name = args.name.map(wstr).transpose()?; + let app_name = app_name.as_ref().map_or_else(null, |w| w.as_ptr()); + + let mut command_line = args.command_line.map(wstr).transpose()?; + let command_line = command_line + .as_mut() + .map_or_else(null_mut, |w| w.as_mut_ptr()); + + let mut current_dir = args.current_dir.map(wstr).transpose()?; + let current_dir = current_dir + .as_mut() + .map_or_else(null_mut, |w| w.as_mut_ptr()); + + let procinfo = unsafe { + let mut procinfo = core::mem::MaybeUninit::uninit(); + WindowsSysResult(windows_sys::Win32::System::Threading::CreateProcessW( + app_name, + command_line, + core::ptr::null(), + core::ptr::null(), + args.inherit_handles, + args.creation_flags + | windows_sys::Win32::System::Threading::EXTENDED_STARTUPINFO_PRESENT + | windows_sys::Win32::System::Threading::CREATE_UNICODE_ENVIRONMENT, + env as _, + current_dir, + &mut si as *mut _ as *mut _, + procinfo.as_mut_ptr(), + )) + .into_pyresult(vm)?; + procinfo.assume_init() + }; + + Ok(( + WinHandle(procinfo.hProcess), + WinHandle(procinfo.hThread), + procinfo.dwProcessId, + procinfo.dwThreadId, + )) + } + + #[pyfunction] + fn OpenProcess( + desired_access: u32, + inherit_handle: bool, + process_id: u32, + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + let handle = unsafe { + windows_sys::Win32::System::Threading::OpenProcess( + desired_access, + i32::from(inherit_handle), + process_id, + ) + }; + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + #[pyfunction] + fn ExitProcess(exit_code: u32) { + unsafe { windows_sys::Win32::System::Threading::ExitProcess(exit_code) } + } + + #[pyfunction] + fn NeedCurrentDirectoryForExePath(exe_name: PyStrRef) -> bool { + let exe_name = exe_name.as_wtf8().to_wide_with_nul(); + let return_value = unsafe { + windows_sys::Win32::System::Environment::NeedCurrentDirectoryForExePathW( + exe_name.as_ptr(), + ) + }; + return_value != 0 + } + + #[pyfunction] + fn CreateJunction( + src_path: PyStrRef, + dest_path: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let src_path = std::path::Path::new(src_path.expect_str()); + let dest_path = std::path::Path::new(dest_path.expect_str()); + + junction::create(src_path, dest_path).map_err(|e| e.to_pyexception(vm)) + } + + fn getenvironment(env: ArgMapping, vm: &VirtualMachine) -> PyResult<Vec<u16>> { + let keys = env.mapping().keys(vm)?; + let values = env.mapping().values(vm)?; + + let keys = ArgSequence::try_from_object(vm, keys)?.into_vec(); + let values = ArgSequence::try_from_object(vm, values)?.into_vec(); + + if keys.len() != values.len() { + return Err(vm.new_runtime_error("environment changed size during iteration")); + } + + // Deduplicate case-insensitive keys, keeping the last value + use std::collections::HashMap; + let mut last_entry: HashMap<String, widestring::WideString> = HashMap::new(); + for (k, v) in keys.into_iter().zip(values.into_iter()) { + let k = PyStrRef::try_from_object(vm, k)?; + let k = k.expect_str(); + let v = PyStrRef::try_from_object(vm, v)?; + let v = v.expect_str(); + if k.contains('\0') || v.contains('\0') { + return Err(crate::exceptions::cstring_error(vm)); + } + if k.is_empty() || k[1..].contains('=') { + return Err(vm.new_value_error("illegal environment variable name")); + } + let key_upper = k.to_uppercase(); + let mut entry = widestring::WideString::new(); + entry.push_str(k); + entry.push_str("="); + entry.push_str(v); + entry.push_str("\0"); + last_entry.insert(key_upper, entry); + } + + // Sort by uppercase key for case-insensitive ordering + let mut entries: Vec<(String, widestring::WideString)> = last_entry.into_iter().collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut out = widestring::WideString::new(); + for (_, entry) in entries { + out.push(entry); + } + out.push_str("\0"); + Ok(out.into_vec()) + } + + struct AttrList { + handlelist: Option<Vec<usize>>, + attrlist: Vec<u8>, + } + impl Drop for AttrList { + fn drop(&mut self) { + unsafe { + windows_sys::Win32::System::Threading::DeleteProcThreadAttributeList( + self.attrlist.as_mut_ptr() as *mut _, + ) + }; + } + } + + fn getattributelist(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Option<AttrList>> { + <Option<ArgMapping>>::try_from_object(vm, obj)? + .map(|mapping| { + let handlelist = mapping + .as_ref() + .get_item("handle_list", vm) + .ok() + .and_then(|obj| { + <Option<ArgSequence<usize>>>::try_from_object(vm, obj) + .map(|s| match s { + Some(s) if !s.is_empty() => Some(s.into_vec()), + _ => None, + }) + .transpose() + }) + .transpose()?; + + let attr_count = handlelist.is_some() as u32; + let (result, mut size) = unsafe { + let mut size = core::mem::MaybeUninit::uninit(); + let result = WindowsSysResult( + windows_sys::Win32::System::Threading::InitializeProcThreadAttributeList( + core::ptr::null_mut(), + attr_count, + 0, + size.as_mut_ptr(), + ), + ); + (result, size.assume_init()) + }; + if !result.is_err() + || unsafe { windows_sys::Win32::Foundation::GetLastError() } + != windows_sys::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER + { + return Err(vm.new_last_os_error()); + } + let mut attrlist = vec![0u8; size]; + WindowsSysResult(unsafe { + windows_sys::Win32::System::Threading::InitializeProcThreadAttributeList( + attrlist.as_mut_ptr() as *mut _, + attr_count, + 0, + &mut size, + ) + }) + .into_pyresult(vm)?; + let mut attrs = AttrList { + handlelist, + attrlist, + }; + if let Some(ref mut handlelist) = attrs.handlelist { + WindowsSysResult(unsafe { + windows_sys::Win32::System::Threading::UpdateProcThreadAttribute( + attrs.attrlist.as_mut_ptr() as _, + 0, + (2 & 0xffff) | 0x20000, // PROC_THREAD_ATTRIBUTE_HANDLE_LIST + handlelist.as_mut_ptr() as _, + (handlelist.len() * core::mem::size_of::<usize>()) as _, + core::ptr::null_mut(), + core::ptr::null(), + ) + }) + .into_pyresult(vm)?; + } + Ok(attrs) + }) + .transpose() + } + + #[pyfunction] + fn WaitForSingleObject(h: WinHandle, ms: i64, vm: &VirtualMachine) -> PyResult<u32> { + // Negative values (e.g., -1) map to INFINITE (0xFFFFFFFF) + let ms = if ms < 0 { + windows_sys::Win32::System::Threading::INFINITE + } else if ms > u32::MAX as i64 { + return Err(vm.new_overflow_error("timeout value is too large")); + } else { + ms as u32 + }; + let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0, ms) }; + if ret == windows_sys::Win32::Foundation::WAIT_FAILED { + Err(vm.new_last_os_error()) + } else { + Ok(ret) + } + } + + #[pyfunction] + fn WaitForMultipleObjects( + handle_seq: ArgSequence<isize>, + wait_all: bool, + milliseconds: u32, + vm: &VirtualMachine, + ) -> PyResult<u32> { + use windows_sys::Win32::Foundation::WAIT_FAILED; + use windows_sys::Win32::System::Threading::WaitForMultipleObjects as WinWaitForMultipleObjects; + + let handles: Vec<HANDLE> = handle_seq + .into_vec() + .into_iter() + .map(|h| h as HANDLE) + .collect(); + + if handles.is_empty() { + return Err(vm.new_value_error("handle_seq must not be empty")); + } + + if handles.len() > 64 { + return Err(vm.new_value_error("WaitForMultipleObjects supports at most 64 handles")); + } + + let ret = unsafe { + WinWaitForMultipleObjects( + handles.len() as u32, + handles.as_ptr(), + if wait_all { 1 } else { 0 }, + milliseconds, + ) + }; + + if ret == WAIT_FAILED { + Err(vm.new_last_os_error()) + } else { + Ok(ret) + } + } + + #[pyfunction] + fn GetExitCodeProcess(h: WinHandle, vm: &VirtualMachine) -> PyResult<u32> { + unsafe { + let mut ec = core::mem::MaybeUninit::uninit(); + WindowsSysResult(windows_sys::Win32::System::Threading::GetExitCodeProcess( + h.0, + ec.as_mut_ptr(), + )) + .to_pyresult(vm)?; + Ok(ec.assume_init()) + } + } + + #[pyfunction] + fn TerminateProcess(h: WinHandle, exit_code: u32) -> WindowsSysResult<i32> { + WindowsSysResult(unsafe { + windows_sys::Win32::System::Threading::TerminateProcess(h.0, exit_code) + }) + } + + #[pyfunction] + fn GetModuleFileName(handle: isize, vm: &VirtualMachine) -> PyResult<String> { + let mut path: Vec<u16> = vec![0; MAX_PATH as usize]; + + let length = unsafe { + windows_sys::Win32::System::LibraryLoader::GetModuleFileNameW( + handle as windows_sys::Win32::Foundation::HMODULE, + path.as_mut_ptr(), + path.len() as u32, + ) + }; + if length == 0 { + return Err(vm.new_runtime_error("GetModuleFileName failed")); + } + + let (path, _) = path.split_at(length as usize); + Ok(String::from_utf16(path).unwrap()) + } + + #[pyfunction] + fn OpenMutexW( + desired_access: u32, + inherit_handle: bool, + name: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + let name_wide = name.as_wtf8().to_wide_with_nul(); + let handle = unsafe { + windows_sys::Win32::System::Threading::OpenMutexW( + desired_access, + i32::from(inherit_handle), + name_wide.as_ptr(), + ) + }; + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + #[pyfunction] + fn ReleaseMutex(handle: WinHandle) -> WindowsSysResult<i32> { + WindowsSysResult(unsafe { windows_sys::Win32::System::Threading::ReleaseMutex(handle.0) }) + } + + // LOCALE_NAME_INVARIANT is an empty string in Windows API + #[pyattr] + const LOCALE_NAME_INVARIANT: &str = ""; + + #[pyattr] + const LOCALE_NAME_SYSTEM_DEFAULT: &str = "!x-sys-default-locale"; + + #[pyattr(name = "LOCALE_NAME_USER_DEFAULT")] + fn locale_name_user_default(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } + + /// LCMapStringEx - Map a string to another string using locale-specific rules + /// This is used by ntpath.normcase() for proper Windows case conversion + #[pyfunction] + fn LCMapStringEx( + locale: PyStrRef, + flags: u32, + src: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + use windows_sys::Win32::Globalization::{ + LCMAP_BYTEREV, LCMAP_HASH, LCMAP_SORTHANDLE, LCMAP_SORTKEY, + LCMapStringEx as WinLCMapStringEx, + }; + + // Reject unsupported flags + if flags & (LCMAP_SORTHANDLE | LCMAP_HASH | LCMAP_BYTEREV | LCMAP_SORTKEY) != 0 { + return Err(vm.new_value_error("unsupported flags")); + } + + // Use ToWideString which properly handles WTF-8 (including surrogates) + let locale_wide = locale.as_wtf8().to_wide_with_nul(); + let src_wide = src.as_wtf8().to_wide(); + + if src_wide.len() > i32::MAX as usize { + return Err(vm.new_overflow_error("input string is too long")); + } + + // First call to get required buffer size + let dest_size = unsafe { + WinLCMapStringEx( + locale_wide.as_ptr(), + flags, + src_wide.as_ptr(), + src_wide.len() as i32, + null_mut(), + 0, + null(), + null(), + 0, + ) + }; + + if dest_size <= 0 { + return Err(vm.new_last_os_error()); + } + + // Second call to perform the mapping + let mut dest = vec![0u16; dest_size as usize]; + let nmapped = unsafe { + WinLCMapStringEx( + locale_wide.as_ptr(), + flags, + src_wide.as_ptr(), + src_wide.len() as i32, + dest.as_mut_ptr(), + dest_size, + null(), + null(), + 0, + ) + }; + + if nmapped <= 0 { + return Err(vm.new_last_os_error()); + } + + dest.truncate(nmapped as usize); + + // Convert UTF-16 back to WTF-8 (handles surrogates properly) + let result = Wtf8Buf::from_wide(&dest); + Ok(vm.ctx.new_str(result)) + } + + #[derive(FromArgs)] + struct CreateNamedPipeArgs { + #[pyarg(positional)] + name: PyStrRef, + #[pyarg(positional)] + open_mode: u32, + #[pyarg(positional)] + pipe_mode: u32, + #[pyarg(positional)] + max_instances: u32, + #[pyarg(positional)] + out_buffer_size: u32, + #[pyarg(positional)] + in_buffer_size: u32, + #[pyarg(positional)] + default_timeout: u32, + #[pyarg(positional)] + _security_attributes: PyObjectRef, // Ignored, can be None + } + + /// CreateNamedPipe - Create a named pipe + #[pyfunction] + fn CreateNamedPipe(args: CreateNamedPipeArgs, vm: &VirtualMachine) -> PyResult<WinHandle> { + use windows_sys::Win32::System::Pipes::CreateNamedPipeW; + + let name_wide = args.name.as_wtf8().to_wide_with_nul(); + + let handle = unsafe { + CreateNamedPipeW( + name_wide.as_ptr(), + args.open_mode, + args.pipe_mode, + args.max_instances, + args.out_buffer_size, + args.in_buffer_size, + args.default_timeout, + null(), // security_attributes - NULL for now + ) + }; + + if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + + Ok(WinHandle(handle)) + } + + // ==================== Overlapped class ==================== + // Used for asynchronous I/O operations (ConnectNamedPipe, ReadFile, WriteFile) + + #[pyattr] + #[pyclass(name = "Overlapped", module = "_winapi")] + #[derive(Debug, PyPayload)] + struct Overlapped { + inner: PyMutex<OverlappedInner>, + } + + struct OverlappedInner { + // Box ensures the OVERLAPPED struct stays at a stable heap address + // even when the containing Overlapped Python object is moved during + // into_pyobject(). The OS holds a pointer to this struct for pending + // I/O operations, so it must not be relocated. + overlapped: Box<windows_sys::Win32::System::IO::OVERLAPPED>, + handle: HANDLE, + pending: bool, + completed: bool, + read_buffer: Option<Vec<u8>>, + write_buffer: Option<Vec<u8>>, + } + + impl core::fmt::Debug for OverlappedInner { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("OverlappedInner") + .field("handle", &self.handle) + .field("pending", &self.pending) + .field("completed", &self.completed) + .finish() + } + } + + unsafe impl Sync for OverlappedInner {} + unsafe impl Send for OverlappedInner {} + + #[pyclass(with(Constructor))] + impl Overlapped { + fn new_with_handle(handle: HANDLE) -> Self { + use windows_sys::Win32::System::Threading::CreateEventW; + + let event = unsafe { CreateEventW(null(), 1, 0, null()) }; + let mut overlapped: windows_sys::Win32::System::IO::OVERLAPPED = + unsafe { core::mem::zeroed() }; + overlapped.hEvent = event; + + Overlapped { + inner: PyMutex::new(OverlappedInner { + overlapped: Box::new(overlapped), + handle, + pending: false, + completed: false, + read_buffer: None, + write_buffer: None, + }), + } + } + + #[pymethod] + fn GetOverlappedResult(&self, wait: bool, vm: &VirtualMachine) -> PyResult<(u32, u32)> { + use windows_sys::Win32::Foundation::{ + ERROR_IO_INCOMPLETE, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED, ERROR_SUCCESS, + GetLastError, + }; + use windows_sys::Win32::System::IO::GetOverlappedResult; + + let mut inner = self.inner.lock(); + + let mut transferred: u32 = 0; + + let ret = unsafe { + GetOverlappedResult( + inner.handle, + &*inner.overlapped, + &mut transferred, + if wait { 1 } else { 0 }, + ) + }; + + let err = if ret == 0 { + unsafe { GetLastError() } + } else { + ERROR_SUCCESS + }; + + match err { + ERROR_SUCCESS | ERROR_MORE_DATA | ERROR_OPERATION_ABORTED => { + inner.completed = true; + inner.pending = false; + } + ERROR_IO_INCOMPLETE => {} + _ => { + inner.pending = false; + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + } + + if inner.completed + && let Some(read_buffer) = &mut inner.read_buffer + && transferred != read_buffer.len() as u32 + { + read_buffer.truncate(transferred as usize); + } + + Ok((transferred, err)) + } + + #[pymethod] + fn getbuffer(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + let inner = self.inner.lock(); + if !inner.completed { + return Err(vm.new_value_error( + "can't get read buffer before GetOverlappedResult() signals the operation completed", + )); + } + Ok(inner + .read_buffer + .as_ref() + .map(|buf| vm.ctx.new_bytes(buf.clone()).into())) + } + + #[pymethod] + fn cancel(&self, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::IO::CancelIoEx; + + let mut inner = self.inner.lock(); + let ret = if inner.pending { + unsafe { CancelIoEx(inner.handle, &*inner.overlapped) } + } else { + 1 + }; + if ret == 0 { + let err = unsafe { windows_sys::Win32::Foundation::GetLastError() }; + if err != windows_sys::Win32::Foundation::ERROR_NOT_FOUND { + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + } + inner.pending = false; + Ok(()) + } + + #[pygetset] + fn event(&self) -> isize { + let inner = self.inner.lock(); + inner.overlapped.hEvent as isize + } + } + + impl Constructor for Overlapped { + type Args = (); + + fn py_new( + _cls: &Py<crate::builtins::PyType>, + _args: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Overlapped::new_with_handle(null_mut())) + } + } + + impl Drop for OverlappedInner { + fn drop(&mut self) { + use windows_sys::Win32::Foundation::CloseHandle; + if !self.overlapped.hEvent.is_null() { + unsafe { CloseHandle(self.overlapped.hEvent) }; + } + } + } + + /// ConnectNamedPipe - Wait for a client to connect to a named pipe + #[derive(FromArgs)] + struct ConnectNamedPipeArgs { + #[pyarg(any)] + handle: WinHandle, + #[pyarg(any, optional)] + overlapped: OptionalArg<bool>, + } + + #[pyfunction] + fn ConnectNamedPipe(args: ConnectNamedPipeArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + use windows_sys::Win32::Foundation::{ + ERROR_IO_PENDING, ERROR_PIPE_CONNECTED, GetLastError, + }; + + let handle = args.handle; + let use_overlapped = args.overlapped.unwrap_or(false); + + if use_overlapped { + // Overlapped (async) mode + let ov = Overlapped::new_with_handle(handle.0); + + let _ret = { + let mut inner = ov.inner.lock(); + unsafe { + windows_sys::Win32::System::Pipes::ConnectNamedPipe( + handle.0, + &mut *inner.overlapped, + ) + } + }; + + let err = unsafe { GetLastError() }; + match err { + ERROR_IO_PENDING => { + let mut inner = ov.inner.lock(); + inner.pending = true; + } + ERROR_PIPE_CONNECTED => { + let inner = ov.inner.lock(); + unsafe { + windows_sys::Win32::System::Threading::SetEvent(inner.overlapped.hEvent); + } + } + _ => { + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + } + + Ok(ov.into_pyobject(vm)) + } else { + // Synchronous mode + let ret = unsafe { + windows_sys::Win32::System::Pipes::ConnectNamedPipe(handle.0, null_mut()) + }; + + if ret == 0 { + let err = unsafe { GetLastError() }; + if err != ERROR_PIPE_CONNECTED { + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + } + + Ok(vm.ctx.none()) + } + } + + /// Helper for GetShortPathName and GetLongPathName + fn get_path_name_impl( + path: &PyStrRef, + api_fn: unsafe extern "system" fn(*const u16, *mut u16, u32) -> u32, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + let path_wide = path.as_wtf8().to_wide_with_nul(); + + // First call to get required buffer size + let size = unsafe { api_fn(path_wide.as_ptr(), null_mut(), 0) }; + + if size == 0 { + return Err(vm.new_last_os_error()); + } + + // Second call to get the actual path + let mut buffer: Vec<u16> = vec![0; size as usize]; + let result = + unsafe { api_fn(path_wide.as_ptr(), buffer.as_mut_ptr(), buffer.len() as u32) }; + + if result == 0 { + return Err(vm.new_last_os_error()); + } + + // Truncate to actual length (excluding null terminator) + buffer.truncate(result as usize); + + // Convert UTF-16 back to WTF-8 (handles surrogates properly) + let result_str = Wtf8Buf::from_wide(&buffer); + Ok(vm.ctx.new_str(result_str)) + } + + /// GetShortPathName - Return the short version of the provided path. + #[pyfunction] + fn GetShortPathName(path: PyStrRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { + use windows_sys::Win32::Storage::FileSystem::GetShortPathNameW; + get_path_name_impl(&path, GetShortPathNameW, vm) + } + + /// GetLongPathName - Return the long version of the provided path. + #[pyfunction] + fn GetLongPathName(path: PyStrRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { + use windows_sys::Win32::Storage::FileSystem::GetLongPathNameW; + get_path_name_impl(&path, GetLongPathNameW, vm) + } + + /// WaitNamedPipe - Wait for an instance of a named pipe to become available. + #[pyfunction] + fn WaitNamedPipe(name: PyStrRef, timeout: u32, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Pipes::WaitNamedPipeW; + + let name_wide = name.as_wtf8().to_wide_with_nul(); + + let success = unsafe { WaitNamedPipeW(name_wide.as_ptr(), timeout) }; + + if success == 0 { + return Err(vm.new_last_os_error()); + } + + Ok(()) + } + + /// PeekNamedPipe - Peek at data in a named pipe without removing it. + #[pyfunction] + fn PeekNamedPipe( + handle: WinHandle, + size: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + use windows_sys::Win32::System::Pipes::PeekNamedPipe as WinPeekNamedPipe; + + let size = size.unwrap_or(0); + + if size < 0 { + return Err(vm.new_value_error("negative size")); + } + + let mut navail: u32 = 0; + let mut nleft: u32 = 0; + + if size > 0 { + let mut buf = vec![0u8; size as usize]; + let mut nread: u32 = 0; + + let ret = unsafe { + WinPeekNamedPipe( + handle.0, + buf.as_mut_ptr() as *mut _, + size as u32, + &mut nread, + &mut navail, + &mut nleft, + ) + }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + buf.truncate(nread as usize); + let bytes: PyObjectRef = vm.ctx.new_bytes(buf).into(); + Ok(vm + .ctx + .new_tuple(vec![ + bytes, + vm.ctx.new_int(navail).into(), + vm.ctx.new_int(nleft).into(), + ]) + .into()) + } else { + let ret = unsafe { + WinPeekNamedPipe(handle.0, null_mut(), 0, null_mut(), &mut navail, &mut nleft) + }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_int(navail).into(), + vm.ctx.new_int(nleft).into(), + ]) + .into()) + } + } + + /// CreateEventW - Create or open a named or unnamed event object. + #[pyfunction] + fn CreateEventW( + security_attributes: isize, // Always NULL (0) + manual_reset: bool, + initial_state: bool, + name: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + use windows_sys::Win32::System::Threading::CreateEventW as WinCreateEventW; + + let _ = security_attributes; // Ignored, always NULL + + let name_wide = name.map(|n| n.as_wtf8().to_wide_with_nul()); + let name_ptr = name_wide.as_ref().map_or(null(), |n| n.as_ptr()); + + let handle = unsafe { + WinCreateEventW( + null(), + i32::from(manual_reset), + i32::from(initial_state), + name_ptr, + ) + }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + + Ok(WinHandle(handle)) + } + + /// SetEvent - Set the specified event object to the signaled state. + #[pyfunction] + fn SetEvent(event: WinHandle, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Threading::SetEvent as WinSetEvent; + + let ret = unsafe { WinSetEvent(event.0) }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + Ok(()) + } + + #[derive(FromArgs)] + struct WriteFileArgs { + #[pyarg(any)] + handle: WinHandle, + #[pyarg(any)] + buffer: crate::function::ArgBytesLike, + #[pyarg(any, default = false)] + overlapped: bool, + } + + /// WriteFile - Write data to a file or I/O device. + #[pyfunction] + fn WriteFile(args: WriteFileArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + use windows_sys::Win32::Storage::FileSystem::WriteFile as WinWriteFile; + + let handle = args.handle; + let use_overlapped = args.overlapped; + let buf = args.buffer.borrow_buf(); + let len = core::cmp::min(buf.len(), u32::MAX as usize) as u32; + + if use_overlapped { + use windows_sys::Win32::Foundation::ERROR_IO_PENDING; + + let ov = Overlapped::new_with_handle(handle.0); + let err = { + let mut inner = ov.inner.lock(); + inner.write_buffer = Some(buf.to_vec()); + let write_buf = inner.write_buffer.as_ref().unwrap(); + let mut written: u32 = 0; + let ret = unsafe { + WinWriteFile( + handle.0, + write_buf.as_ptr() as *const _, + len, + &mut written, + &mut *inner.overlapped, + ) + }; + + let err = if ret == 0 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } else { + 0 + }; + + if ret == 0 && err != ERROR_IO_PENDING { + return Err(vm.new_last_os_error()); + } + if ret == 0 && err == ERROR_IO_PENDING { + inner.pending = true; + } + + err + }; + + // Without GIL, the Python-level PipeConnection._send_bytes has a + // race on _send_ov when the caller (SimpleQueue) skips locking on + // Windows. Wait for completion here so the caller never sees + // ERROR_IO_PENDING and never blocks in WaitForMultipleObjects, + // keeping the _send_ov window negligibly small. + if err == ERROR_IO_PENDING { + let event = ov.inner.lock().overlapped.hEvent; + vm.allow_threads(|| unsafe { + windows_sys::Win32::System::Threading::WaitForSingleObject( + event, + windows_sys::Win32::System::Threading::INFINITE, + ); + }); + let result = vm + .ctx + .new_tuple(vec![ov.into_pyobject(vm), vm.ctx.new_int(0u32).into()]); + return Ok(result.into()); + } + + let result = vm + .ctx + .new_tuple(vec![ov.into_pyobject(vm), vm.ctx.new_int(err).into()]); + return Ok(result.into()); + } + + let mut written: u32 = 0; + let ret = unsafe { + WinWriteFile( + handle.0, + buf.as_ptr() as *const _, + len, + &mut written, + null_mut(), + ) + }; + let err = if ret == 0 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } else { + 0 + }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_int(written).into(), + vm.ctx.new_int(err).into(), + ]) + .into()) + } + + #[derive(FromArgs)] + struct ReadFileArgs { + #[pyarg(any)] + handle: WinHandle, + #[pyarg(any)] + size: u32, + #[pyarg(any, default = false)] + overlapped: bool, + } + + /// ReadFile - Read data from a file or I/O device. + #[pyfunction] + fn ReadFile(args: ReadFileArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + use windows_sys::Win32::Storage::FileSystem::ReadFile as WinReadFile; + + let handle = args.handle; + let size = args.size; + let use_overlapped = args.overlapped; + + if use_overlapped { + use windows_sys::Win32::Foundation::ERROR_IO_PENDING; + + let ov = Overlapped::new_with_handle(handle.0); + let err = { + let mut inner = ov.inner.lock(); + inner.read_buffer = Some(vec![0u8; size as usize]); + let read_buf = inner.read_buffer.as_mut().unwrap(); + let mut nread: u32 = 0; + let ret = unsafe { + WinReadFile( + handle.0, + read_buf.as_mut_ptr() as *mut _, + size, + &mut nread, + &mut *inner.overlapped, + ) + }; + + let err = if ret == 0 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } else { + 0 + }; + + if ret == 0 && err != ERROR_IO_PENDING && err != ERROR_MORE_DATA { + return Err(vm.new_last_os_error()); + } + if ret == 0 && err == ERROR_IO_PENDING { + inner.pending = true; + } + + err + }; + let result = vm + .ctx + .new_tuple(vec![ov.into_pyobject(vm), vm.ctx.new_int(err).into()]); + return Ok(result.into()); + } + + let mut buf = vec![0u8; size as usize]; + let mut nread: u32 = 0; + let ret = unsafe { + WinReadFile( + handle.0, + buf.as_mut_ptr() as *mut _, + size, + &mut nread, + null_mut(), + ) + }; + let err = if ret == 0 { + unsafe { windows_sys::Win32::Foundation::GetLastError() } + } else { + 0 + }; + if ret == 0 && err != ERROR_MORE_DATA { + return Err(vm.new_last_os_error()); + } + buf.truncate(nread as usize); + Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_bytes(buf).into(), + vm.ctx.new_int(err).into(), + ]) + .into()) + } + + /// SetNamedPipeHandleState - Set the read mode and other options of a named pipe. + #[pyfunction] + fn SetNamedPipeHandleState( + named_pipe: WinHandle, + mode: PyObjectRef, + max_collection_count: PyObjectRef, + collect_data_timeout: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + use windows_sys::Win32::System::Pipes::SetNamedPipeHandleState as WinSetNamedPipeHandleState; + + let mut dw_args: [u32; 3] = [0; 3]; + let mut p_args: [*mut u32; 3] = [null_mut(); 3]; + + let objs = [&mode, &max_collection_count, &collect_data_timeout]; + for (i, obj) in objs.iter().enumerate() { + if !vm.is_none(obj) { + dw_args[i] = u32::try_from_object(vm, (*obj).clone())?; + p_args[i] = &mut dw_args[i]; + } + } + + let ret = + unsafe { WinSetNamedPipeHandleState(named_pipe.0, p_args[0], p_args[1], p_args[2]) }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(()) + } + + /// ResetEvent - Reset the specified event object to the nonsignaled state. + #[pyfunction] + fn ResetEvent(event: WinHandle, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Threading::ResetEvent as WinResetEvent; + + let ret = unsafe { WinResetEvent(event.0) }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(()) + } + + /// CreateMutexW - Create or open a named or unnamed mutex object. + #[pyfunction] + fn CreateMutexW( + security_attributes: isize, + initial_owner: bool, + name: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + use windows_sys::Win32::System::Threading::CreateMutexW as WinCreateMutexW; + + let _ = security_attributes; + let name_wide = name.map(|n| n.as_wtf8().to_wide_with_nul()); + let name_ptr = name_wide.as_ref().map_or(null(), |n| n.as_ptr()); + + let handle = unsafe { WinCreateMutexW(null(), i32::from(initial_owner), name_ptr) }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + /// OpenEventW - Open an existing named event object. + #[pyfunction] + fn OpenEventW( + desired_access: u32, + inherit_handle: bool, + name: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + use windows_sys::Win32::System::Threading::OpenEventW as WinOpenEventW; + + let name_wide = name.as_wtf8().to_wide_with_nul(); + let handle = unsafe { + WinOpenEventW( + desired_access, + i32::from(inherit_handle), + name_wide.as_ptr(), + ) + }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + const MAXIMUM_WAIT_OBJECTS: usize = 64; + + /// BatchedWaitForMultipleObjects - Wait for multiple handles, supporting more than 64. + #[pyfunction] + fn BatchedWaitForMultipleObjects( + handle_seq: PyObjectRef, + wait_all: bool, + milliseconds: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + use alloc::sync::Arc; + use core::sync::atomic::{AtomicU32, Ordering}; + use windows_sys::Win32::Foundation::{CloseHandle, WAIT_FAILED, WAIT_OBJECT_0}; + use windows_sys::Win32::System::SystemInformation::GetTickCount64; + use windows_sys::Win32::System::Threading::{ + CreateEventW as WinCreateEventW, CreateThread, GetExitCodeThread, + INFINITE as WIN_INFINITE, ResumeThread, SetEvent as WinSetEvent, TerminateThread, + WaitForMultipleObjects, + }; + + let milliseconds = milliseconds.unwrap_or(WIN_INFINITE); + + // Get handles from sequence + let seq = ArgSequence::<isize>::try_from_object(vm, handle_seq)?; + let handles: Vec<isize> = seq.into_vec(); + let nhandles = handles.len(); + + if nhandles == 0 { + return if wait_all { + Ok(vm.ctx.none()) + } else { + Ok(vm.ctx.new_list(vec![]).into()) + }; + } + + let max_total_objects = (MAXIMUM_WAIT_OBJECTS - 1) * (MAXIMUM_WAIT_OBJECTS - 1); + if nhandles > max_total_objects { + return Err(vm.new_value_error(format!( + "need at most {} handles, got a sequence of length {}", + max_total_objects, nhandles + ))); + } + + // Create batches of handles + let batch_size = MAXIMUM_WAIT_OBJECTS - 1; // Leave room for cancel_event + let mut batches: Vec<Vec<isize>> = Vec::new(); + let mut i = 0; + while i < nhandles { + let end = core::cmp::min(i + batch_size, nhandles); + batches.push(handles[i..end].to_vec()); + i = end; + } + + #[cfg(feature = "threading")] + let sigint_event = { + let is_main = crate::stdlib::_thread::get_ident() == vm.state.main_thread_ident.load(); + if is_main { + let handle = crate::signal::get_sigint_event().unwrap_or_else(|| { + let handle = unsafe { WinCreateEventW(null(), 1, 0, null()) }; + if !handle.is_null() { + crate::signal::set_sigint_event(handle as isize); + } + handle as isize + }); + if handle == 0 { None } else { Some(handle) } + } else { + None + } + }; + #[cfg(not(feature = "threading"))] + let sigint_event: Option<isize> = None; + + if wait_all { + // For wait_all, we wait sequentially for each batch + let mut err: Option<u32> = None; + let deadline = if milliseconds != WIN_INFINITE { + Some(unsafe { GetTickCount64() } + milliseconds as u64) + } else { + None + }; + + for batch in &batches { + let timeout = if let Some(deadline) = deadline { + let now = unsafe { GetTickCount64() }; + if now >= deadline { + err = Some(windows_sys::Win32::Foundation::WAIT_TIMEOUT); + break; + } + (deadline - now) as u32 + } else { + WIN_INFINITE + }; + + let batch_handles: Vec<_> = batch.iter().map(|&h| h as _).collect(); + let result = unsafe { + WaitForMultipleObjects( + batch_handles.len() as u32, + batch_handles.as_ptr(), + 1, // wait_all = TRUE + timeout, + ) + }; + + if result == WAIT_FAILED { + err = Some(unsafe { windows_sys::Win32::Foundation::GetLastError() }); + break; + } + if result == windows_sys::Win32::Foundation::WAIT_TIMEOUT { + err = Some(windows_sys::Win32::Foundation::WAIT_TIMEOUT); + break; + } + + if let Some(sigint_event) = sigint_event { + let sig_result = unsafe { + windows_sys::Win32::System::Threading::WaitForSingleObject( + sigint_event as _, + 0, + ) + }; + if sig_result == WAIT_OBJECT_0 { + err = Some(windows_sys::Win32::Foundation::ERROR_CONTROL_C_EXIT); + break; + } + if sig_result == WAIT_FAILED { + err = Some(unsafe { windows_sys::Win32::Foundation::GetLastError() }); + break; + } + } + } + + if let Some(err) = err { + if err == windows_sys::Win32::Foundation::WAIT_TIMEOUT { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.timeout_error.to_owned(), + None, + "timed out", + ) + .upcast()); + } + if err == windows_sys::Win32::Foundation::ERROR_CONTROL_C_EXIT { + return Err(vm + .new_errno_error(libc::EINTR, "Interrupted system call") + .upcast()); + } + return Err(vm.new_os_error(err as i32)); + } + + Ok(vm.ctx.none()) + } else { + // For wait_any, we use threads to wait on each batch in parallel + let cancel_event = unsafe { WinCreateEventW(null(), 1, 0, null()) }; // Manual reset, not signaled + if cancel_event.is_null() { + return Err(vm.new_last_os_error()); + } + + struct BatchData { + handles: Vec<isize>, + cancel_event: isize, + handle_base: usize, + result: AtomicU32, + thread: core::cell::UnsafeCell<isize>, + } + + unsafe impl Send for BatchData {} + unsafe impl Sync for BatchData {} + + let batch_data: Vec<Arc<BatchData>> = batches + .iter() + .enumerate() + .map(|(idx, batch)| { + let base = idx * batch_size; + let mut handles_with_cancel = batch.clone(); + handles_with_cancel.push(cancel_event as isize); + Arc::new(BatchData { + handles: handles_with_cancel, + cancel_event: cancel_event as isize, + handle_base: base, + result: AtomicU32::new(WAIT_FAILED), + thread: core::cell::UnsafeCell::new(0), + }) + }) + .collect(); + + // Thread function + extern "system" fn batch_wait_thread(param: *mut core::ffi::c_void) -> u32 { + let data = unsafe { &*(param as *const BatchData) }; + let handles: Vec<_> = data.handles.iter().map(|&h| h as _).collect(); + let result = unsafe { + WaitForMultipleObjects( + handles.len() as u32, + handles.as_ptr(), + 0, // wait_any + WIN_INFINITE, + ) + }; + data.result.store(result, Ordering::SeqCst); + + if result == WAIT_FAILED { + let err = unsafe { windows_sys::Win32::Foundation::GetLastError() }; + unsafe { WinSetEvent(data.cancel_event as _) }; + return err; + } else if result >= windows_sys::Win32::Foundation::WAIT_ABANDONED_0 + && result + < windows_sys::Win32::Foundation::WAIT_ABANDONED_0 + + MAXIMUM_WAIT_OBJECTS as u32 + { + data.result.store(WAIT_FAILED, Ordering::SeqCst); + unsafe { WinSetEvent(data.cancel_event as _) }; + return windows_sys::Win32::Foundation::ERROR_ABANDONED_WAIT_0; + } + 0 + } + + // Create threads + let mut thread_handles: Vec<isize> = Vec::new(); + for data in &batch_data { + let thread = unsafe { + CreateThread( + null(), + 1, // Smallest stack + Some(batch_wait_thread), + Arc::as_ptr(data) as *const _ as *mut _, + 4, // CREATE_SUSPENDED + null_mut(), + ) + }; + if thread.is_null() { + // Cleanup on error + for h in &thread_handles { + unsafe { TerminateThread(*h as _, 0) }; + unsafe { CloseHandle(*h as _) }; + } + unsafe { CloseHandle(cancel_event) }; + return Err(vm.new_last_os_error()); + } + unsafe { *data.thread.get() = thread as isize }; + thread_handles.push(thread as isize); + } + + // Resume all threads + for &thread in &thread_handles { + unsafe { ResumeThread(thread as _) }; + } + + // Wait for any thread to complete + let mut thread_handles_raw: Vec<_> = thread_handles.iter().map(|&h| h as _).collect(); + if let Some(sigint_event) = sigint_event { + thread_handles_raw.push(sigint_event as _); + } + let result = unsafe { + WaitForMultipleObjects( + thread_handles_raw.len() as u32, + thread_handles_raw.as_ptr(), + 0, // wait_any + milliseconds, + ) + }; + + let err = if result == WAIT_FAILED { + Some(unsafe { windows_sys::Win32::Foundation::GetLastError() }) + } else if result == windows_sys::Win32::Foundation::WAIT_TIMEOUT { + Some(windows_sys::Win32::Foundation::WAIT_TIMEOUT) + } else if sigint_event.is_some() + && result == WAIT_OBJECT_0 + thread_handles_raw.len() as u32 + { + Some(windows_sys::Win32::Foundation::ERROR_CONTROL_C_EXIT) + } else { + None + }; + + // Signal cancel event to stop other threads + unsafe { WinSetEvent(cancel_event) }; + + // Wait for all threads to finish + let thread_handles_only: Vec<_> = thread_handles.iter().map(|&h| h as _).collect(); + unsafe { + WaitForMultipleObjects( + thread_handles_only.len() as u32, + thread_handles_only.as_ptr(), + 1, // wait_all + WIN_INFINITE, + ) + }; + + // Check for errors from threads + let mut thread_err = err; + for data in &batch_data { + if thread_err.is_none() && data.result.load(Ordering::SeqCst) == WAIT_FAILED { + let mut exit_code: u32 = 0; + let thread = unsafe { *data.thread.get() }; + if unsafe { GetExitCodeThread(thread as _, &mut exit_code) } == 0 { + thread_err = + Some(unsafe { windows_sys::Win32::Foundation::GetLastError() }); + } else if exit_code != 0 { + thread_err = Some(exit_code); + } + } + let thread = unsafe { *data.thread.get() }; + unsafe { CloseHandle(thread as _) }; + } + + unsafe { CloseHandle(cancel_event) }; + + // Return result + if let Some(e) = thread_err { + if e == windows_sys::Win32::Foundation::WAIT_TIMEOUT { + return Err(vm + .new_os_subtype_error( + vm.ctx.exceptions.timeout_error.to_owned(), + None, + "timed out", + ) + .upcast()); + } + if e == windows_sys::Win32::Foundation::ERROR_CONTROL_C_EXIT { + return Err(vm + .new_errno_error(libc::EINTR, "Interrupted system call") + .upcast()); + } + return Err(vm.new_os_error(e as i32)); + } + + // Collect triggered indices + let mut triggered_indices: Vec<PyObjectRef> = Vec::new(); + for data in &batch_data { + let result = data.result.load(Ordering::SeqCst); + let triggered = result as i32 - WAIT_OBJECT_0 as i32; + // Check if it's a valid handle index (not the cancel_event which is last) + if triggered >= 0 && (triggered as usize) < data.handles.len() - 1 { + let index = data.handle_base + triggered as usize; + triggered_indices.push(vm.ctx.new_int(index).into()); + } + } + + Ok(vm.ctx.new_list(triggered_indices).into()) + } + } + + /// CreateFileMapping - Create or open a named or unnamed file mapping object. + #[pyfunction] + fn CreateFileMapping( + file_handle: WinHandle, + _security_attributes: PyObjectRef, + protect: u32, + max_size_high: u32, + max_size_low: u32, + name: Option<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + use windows_sys::Win32::System::Memory::CreateFileMappingW; + + if let Some(ref n) = name + && n.as_bytes().contains(&0) + { + return Err( + vm.new_value_error("CreateFileMapping: name must not contain null characters") + ); + } + let name_wide = name.as_ref().map(|n| n.as_wtf8().to_wide_with_nul()); + let name_ptr = name_wide.as_ref().map_or(null(), |n| n.as_ptr()); + + let handle = unsafe { + CreateFileMappingW( + file_handle.0, + null(), + protect, + max_size_high, + max_size_low, + name_ptr, + ) + }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + /// OpenFileMapping - Open a named file mapping object. + #[pyfunction] + fn OpenFileMapping( + desired_access: u32, + inherit_handle: bool, + name: PyStrRef, + vm: &VirtualMachine, + ) -> PyResult<WinHandle> { + use windows_sys::Win32::System::Memory::OpenFileMappingW; + + if name.as_bytes().contains(&0) { + return Err( + vm.new_value_error("OpenFileMapping: name must not contain null characters") + ); + } + let name_wide = name.as_wtf8().to_wide_with_nul(); + let handle = unsafe { + OpenFileMappingW( + desired_access, + i32::from(inherit_handle), + name_wide.as_ptr(), + ) + }; + + if handle.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(WinHandle(handle)) + } + + /// MapViewOfFile - Map a view of a file mapping into the address space. + #[pyfunction] + fn MapViewOfFile( + file_map: WinHandle, + desired_access: u32, + file_offset_high: u32, + file_offset_low: u32, + number_bytes: usize, + vm: &VirtualMachine, + ) -> PyResult<isize> { + let address = unsafe { + windows_sys::Win32::System::Memory::MapViewOfFile( + file_map.0, + desired_access, + file_offset_high, + file_offset_low, + number_bytes, + ) + }; + + let ptr = address.Value; + if ptr.is_null() { + return Err(vm.new_last_os_error()); + } + Ok(ptr as isize) + } + + /// UnmapViewOfFile - Unmap a mapped view of a file. + #[pyfunction] + fn UnmapViewOfFile(address: isize, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::System::Memory::MEMORY_MAPPED_VIEW_ADDRESS; + + let view = MEMORY_MAPPED_VIEW_ADDRESS { + Value: address as *mut core::ffi::c_void, + }; + let ret = unsafe { windows_sys::Win32::System::Memory::UnmapViewOfFile(view) }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(()) + } + + /// VirtualQuerySize - Return the size of a memory region. + #[pyfunction] + fn VirtualQuerySize(address: isize, vm: &VirtualMachine) -> PyResult<usize> { + use windows_sys::Win32::System::Memory::{MEMORY_BASIC_INFORMATION, VirtualQuery}; + + let mut mbi: MEMORY_BASIC_INFORMATION = unsafe { core::mem::zeroed() }; + let ret = unsafe { + VirtualQuery( + address as *const core::ffi::c_void, + &mut mbi, + core::mem::size_of::<MEMORY_BASIC_INFORMATION>(), + ) + }; + + if ret == 0 { + return Err(vm.new_last_os_error()); + } + Ok(mbi.RegionSize) + } + + /// CopyFile2 - Copy a file with extended parameters. + #[pyfunction] + fn CopyFile2( + existing_file_name: PyStrRef, + new_file_name: PyStrRef, + flags: u32, + _progress_routine: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + use windows_sys::Win32::Storage::FileSystem::{ + COPYFILE2_EXTENDED_PARAMETERS, CopyFile2 as WinCopyFile2, + }; + + let src_wide = existing_file_name.as_wtf8().to_wide_with_nul(); + let dst_wide = new_file_name.as_wtf8().to_wide_with_nul(); + + let mut params: COPYFILE2_EXTENDED_PARAMETERS = unsafe { core::mem::zeroed() }; + params.dwSize = core::mem::size_of::<COPYFILE2_EXTENDED_PARAMETERS>() as u32; + params.dwCopyFlags = flags; + + let hr = unsafe { WinCopyFile2(src_wide.as_ptr(), dst_wide.as_ptr(), &params) }; + + if hr < 0 { + // HRESULT failure - convert to Windows error code + let err = if (hr as u32 >> 16) == 0x8007 { + (hr as u32) & 0xFFFF + } else { + hr as u32 + }; + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + Ok(()) + } + + /// _mimetypes_read_windows_registry - Read MIME type associations from registry. + #[pyfunction] + fn _mimetypes_read_windows_registry( + on_type_read: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + use windows_sys::Win32::System::Registry::{ + HKEY, HKEY_CLASSES_ROOT, KEY_READ, REG_SZ, RegCloseKey, RegEnumKeyExW, RegOpenKeyExW, + RegQueryValueExW, + }; + + let mut hkcr: HKEY = null_mut() as HKEY; + let err = unsafe { RegOpenKeyExW(HKEY_CLASSES_ROOT, null(), 0, KEY_READ, &mut hkcr) }; + if err != 0 { + return Err(vm.new_os_error(err as i32)); + } + scopeguard::defer! { unsafe { RegCloseKey(hkcr) }; } + + let mut i: u32 = 0; + let mut entries: Vec<(String, String)> = Vec::new(); + + loop { + let mut ext_buf = [0u16; 128]; + let mut cch_ext: u32 = ext_buf.len() as u32; + + let err = unsafe { + RegEnumKeyExW( + hkcr, + i, + ext_buf.as_mut_ptr(), + &mut cch_ext, + null_mut(), + null_mut(), + null_mut(), + null_mut(), + ) + }; + i += 1; + + if err == windows_sys::Win32::Foundation::ERROR_NO_MORE_ITEMS { + break; + } + if err != 0 && err != windows_sys::Win32::Foundation::ERROR_MORE_DATA { + return Err(vm.new_os_error(err as i32)); + } + + // Only process keys starting with '.' + if cch_ext == 0 || ext_buf[0] != b'.' as u16 { + continue; + } + + let ext_wide = &ext_buf[..cch_ext as usize]; + + // Open subkey to read Content Type + let mut subkey: HKEY = null_mut() as HKEY; + let err = unsafe { RegOpenKeyExW(hkcr, ext_buf.as_ptr(), 0, KEY_READ, &mut subkey) }; + if err == windows_sys::Win32::Foundation::ERROR_FILE_NOT_FOUND + || err == windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED + { + continue; + } + if err != 0 { + return Err(vm.new_os_error(err as i32)); + } + + let content_type_key: Vec<u16> = "Content Type\0".encode_utf16().collect(); + let mut type_buf = [0u16; 256]; + let mut cb_type: u32 = (type_buf.len() * 2) as u32; + let mut reg_type: u32 = 0; + + let err = unsafe { + RegQueryValueExW( + subkey, + content_type_key.as_ptr(), + null_mut(), + &mut reg_type, + type_buf.as_mut_ptr() as *mut u8, + &mut cb_type, + ) + }; + unsafe { RegCloseKey(subkey) }; + + if err != 0 || reg_type != REG_SZ || cb_type == 0 { + continue; + } + + // Convert wide strings to Rust strings + let type_len = (cb_type as usize / 2).saturating_sub(1); // exclude null terminator + let type_str = String::from_utf16_lossy(&type_buf[..type_len]); + let ext_str = String::from_utf16_lossy(ext_wide); + + if type_str.is_empty() { + continue; + } + + entries.push((type_str, ext_str)); + + // Flush buffer periodically to call Python callback + if entries.len() >= 64 { + for (mime_type, ext) in entries.drain(..) { + on_type_read.call((vm.ctx.new_str(mime_type), vm.ctx.new_str(ext)), vm)?; + } + } + } + + // Process remaining entries + for (mime_type, ext) in entries { + on_type_read.call((vm.ctx.new_str(mime_type), vm.ctx.new_str(ext)), vm)?; + } + + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/_wmi.rs b/crates/vm/src/stdlib/_wmi.rs new file mode 100644 index 00000000000..96275e5ac4b --- /dev/null +++ b/crates/vm/src/stdlib/_wmi.rs @@ -0,0 +1,698 @@ +// spell-checker:disable +#![allow(non_snake_case)] + +pub(crate) use _wmi::module_def; + +// COM/WMI FFI declarations (not inside pymodule to avoid macro issues) +mod wmi_ffi { + #![allow(unsafe_op_in_unsafe_fn)] + use core::ffi::c_void; + + pub type HRESULT = i32; + + #[repr(C)] + pub struct GUID { + pub data1: u32, + pub data2: u16, + pub data3: u16, + pub data4: [u8; 8], + } + + // Opaque VARIANT type (24 bytes covers both 32-bit and 64-bit) + #[repr(C, align(8))] + pub struct VARIANT([u64; 3]); + + impl VARIANT { + pub fn zeroed() -> Self { + VARIANT([0u64; 3]) + } + } + + // CLSID_WbemLocator = {4590F811-1D3A-11D0-891F-00AA004B2E24} + pub const CLSID_WBEM_LOCATOR: GUID = GUID { + data1: 0x4590F811, + data2: 0x1D3A, + data3: 0x11D0, + data4: [0x89, 0x1F, 0x00, 0xAA, 0x00, 0x4B, 0x2E, 0x24], + }; + + // IID_IWbemLocator = {DC12A687-737F-11CF-884D-00AA004B2E24} + pub const IID_IWBEM_LOCATOR: GUID = GUID { + data1: 0xDC12A687, + data2: 0x737F, + data3: 0x11CF, + data4: [0x88, 0x4D, 0x00, 0xAA, 0x00, 0x4B, 0x2E, 0x24], + }; + + // COM constants + pub const COINIT_APARTMENTTHREADED: u32 = 0x2; + pub const CLSCTX_INPROC_SERVER: u32 = 0x1; + pub const RPC_C_AUTHN_LEVEL_DEFAULT: u32 = 0; + pub const RPC_C_IMP_LEVEL_IMPERSONATE: u32 = 3; + pub const RPC_C_AUTHN_LEVEL_CALL: u32 = 3; + pub const RPC_C_AUTHN_WINNT: u32 = 10; + pub const RPC_C_AUTHZ_NONE: u32 = 0; + pub const EOAC_NONE: u32 = 0; + pub const RPC_E_TOO_LATE: HRESULT = 0x80010119_u32 as i32; + + // WMI constants + pub const WBEM_FLAG_FORWARD_ONLY: i32 = 0x20; + pub const WBEM_FLAG_RETURN_IMMEDIATELY: i32 = 0x10; + pub const WBEM_S_FALSE: HRESULT = 1; + pub const WBEM_S_NO_MORE_DATA: HRESULT = 0x40005; + pub const WBEM_INFINITE: i32 = -1; + pub const WBEM_FLAVOR_MASK_ORIGIN: i32 = 0x60; + pub const WBEM_FLAVOR_ORIGIN_SYSTEM: i32 = 0x40; + + #[link(name = "ole32")] + unsafe extern "system" { + pub fn CoInitializeEx(pvReserved: *mut c_void, dwCoInit: u32) -> HRESULT; + pub fn CoUninitialize(); + pub fn CoInitializeSecurity( + pSecDesc: *const c_void, + cAuthSvc: i32, + asAuthSvc: *const c_void, + pReserved1: *const c_void, + dwAuthnLevel: u32, + dwImpLevel: u32, + pAuthList: *const c_void, + dwCapabilities: u32, + pReserved3: *const c_void, + ) -> HRESULT; + pub fn CoCreateInstance( + rclsid: *const GUID, + pUnkOuter: *mut c_void, + dwClsContext: u32, + riid: *const GUID, + ppv: *mut *mut c_void, + ) -> HRESULT; + pub fn CoSetProxyBlanket( + pProxy: *mut c_void, + dwAuthnSvc: u32, + dwAuthzSvc: u32, + pServerPrincName: *const u16, + dwAuthnLevel: u32, + dwImpLevel: u32, + pAuthInfo: *const c_void, + dwCapabilities: u32, + ) -> HRESULT; + } + + #[link(name = "oleaut32")] + unsafe extern "system" { + pub fn SysAllocString(psz: *const u16) -> *mut u16; + pub fn SysFreeString(bstrString: *mut u16); + pub fn VariantClear(pvarg: *mut VARIANT) -> HRESULT; + } + + #[link(name = "propsys")] + unsafe extern "system" { + pub fn VariantToString(varIn: *const VARIANT, pszBuf: *mut u16, cchBuf: u32) -> HRESULT; + } + + /// Release a COM object (IUnknown::Release, vtable index 2) + pub unsafe fn com_release(this: *mut c_void) { + if !this.is_null() { + let vtable = *(this as *const *const usize); + let release: unsafe extern "system" fn(*mut c_void) -> u32 = + core::mem::transmute(*vtable.add(2)); + release(this); + } + } + + /// IWbemLocator::ConnectServer (vtable index 3) + #[allow(clippy::too_many_arguments)] + pub unsafe fn locator_connect_server( + this: *mut c_void, + network_resource: *const u16, + user: *const u16, + password: *const u16, + locale: *const u16, + security_flags: i32, + authority: *const u16, + ctx: *mut c_void, + services: *mut *mut c_void, + ) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn( + *mut c_void, + *const u16, + *const u16, + *const u16, + *const u16, + i32, + *const u16, + *mut c_void, + *mut *mut c_void, + ) -> HRESULT = core::mem::transmute(*vtable.add(3)); + method( + this, + network_resource, + user, + password, + locale, + security_flags, + authority, + ctx, + services, + ) + } + + /// IWbemServices::ExecQuery (vtable index 20) + pub unsafe fn services_exec_query( + this: *mut c_void, + query_language: *const u16, + query: *const u16, + flags: i32, + ctx: *mut c_void, + enumerator: *mut *mut c_void, + ) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn( + *mut c_void, + *const u16, + *const u16, + i32, + *mut c_void, + *mut *mut c_void, + ) -> HRESULT = core::mem::transmute(*vtable.add(20)); + method(this, query_language, query, flags, ctx, enumerator) + } + + /// IEnumWbemClassObject::Next (vtable index 4) + pub unsafe fn enum_next( + this: *mut c_void, + timeout: i32, + count: u32, + objects: *mut *mut c_void, + returned: *mut u32, + ) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn( + *mut c_void, + i32, + u32, + *mut *mut c_void, + *mut u32, + ) -> HRESULT = core::mem::transmute(*vtable.add(4)); + method(this, timeout, count, objects, returned) + } + + /// IWbemClassObject::BeginEnumeration (vtable index 8) + pub unsafe fn object_begin_enumeration(this: *mut c_void, enum_flags: i32) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn(*mut c_void, i32) -> HRESULT = + core::mem::transmute(*vtable.add(8)); + method(this, enum_flags) + } + + /// IWbemClassObject::Next (vtable index 9) + pub unsafe fn object_next( + this: *mut c_void, + flags: i32, + name: *mut *mut u16, + val: *mut VARIANT, + cim_type: *mut i32, + flavor: *mut i32, + ) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn( + *mut c_void, + i32, + *mut *mut u16, + *mut VARIANT, + *mut i32, + *mut i32, + ) -> HRESULT = core::mem::transmute(*vtable.add(9)); + method(this, flags, name, val, cim_type, flavor) + } + + /// IWbemClassObject::EndEnumeration (vtable index 10) + pub unsafe fn object_end_enumeration(this: *mut c_void) -> HRESULT { + let vtable = *(this as *const *const usize); + let method: unsafe extern "system" fn(*mut c_void) -> HRESULT = + core::mem::transmute(*vtable.add(10)); + method(this) + } +} + +#[pymodule] +mod _wmi { + use super::wmi_ffi::*; + use crate::builtins::PyStrRef; + use crate::convert::ToPyException; + use crate::{PyResult, VirtualMachine}; + use core::ffi::c_void; + use core::ptr::{null, null_mut}; + use windows_sys::Win32::Foundation::{ + CloseHandle, ERROR_BROKEN_PIPE, ERROR_MORE_DATA, ERROR_NOT_ENOUGH_MEMORY, GetLastError, + HANDLE, WAIT_OBJECT_0, WAIT_TIMEOUT, + }; + use windows_sys::Win32::Storage::FileSystem::{ReadFile, WriteFile}; + use windows_sys::Win32::System::Pipes::CreatePipe; + use windows_sys::Win32::System::Threading::{ + CreateEventW, CreateThread, GetExitCodeThread, SetEvent, WaitForSingleObject, + }; + + const BUFFER_SIZE: usize = 8192; + + fn hresult_from_win32(err: u32) -> HRESULT { + if err == 0 { + 0 + } else { + ((err & 0xFFFF) | 0x80070000) as HRESULT + } + } + + fn succeeded(hr: HRESULT) -> bool { + hr >= 0 + } + + fn failed(hr: HRESULT) -> bool { + hr < 0 + } + + fn wide_str(s: &str) -> Vec<u16> { + s.encode_utf16().chain(core::iter::once(0)).collect() + } + + unsafe fn wcslen(s: *const u16) -> usize { + unsafe { + let mut len = 0; + while *s.add(len) != 0 { + len += 1; + } + len + } + } + + unsafe fn wait_event(event: HANDLE, timeout: u32) -> u32 { + unsafe { + match WaitForSingleObject(event, timeout) { + WAIT_OBJECT_0 => 0, + WAIT_TIMEOUT => WAIT_TIMEOUT, + _ => GetLastError(), + } + } + } + + struct QueryThreadData { + query: Vec<u16>, + write_pipe: HANDLE, + init_event: HANDLE, + connect_event: HANDLE, + } + + // SAFETY: QueryThreadData contains HANDLEs (isize) which are safe to send across threads + unsafe impl Send for QueryThreadData {} + + unsafe extern "system" fn query_thread(param: *mut c_void) -> u32 { + unsafe { query_thread_impl(param) } + } + + unsafe fn query_thread_impl(param: *mut c_void) -> u32 { + unsafe { + let data = Box::from_raw(param as *mut QueryThreadData); + let write_pipe = data.write_pipe; + let init_event = data.init_event; + let connect_event = data.connect_event; + + let mut locator: *mut c_void = null_mut(); + let mut services: *mut c_void = null_mut(); + let mut enumerator: *mut c_void = null_mut(); + let mut hr: HRESULT = 0; + + // gh-125315: Copy the query string first + let bstr_query = SysAllocString(data.query.as_ptr()); + if bstr_query.is_null() { + hr = hresult_from_win32(ERROR_NOT_ENOUGH_MEMORY); + } + + drop(data); + + if succeeded(hr) { + hr = CoInitializeEx(null_mut(), COINIT_APARTMENTTHREADED); + } + + if failed(hr) { + CloseHandle(write_pipe); + if !bstr_query.is_null() { + SysFreeString(bstr_query); + } + return hr as u32; + } + + hr = CoInitializeSecurity( + null(), + -1, + null(), + null(), + RPC_C_AUTHN_LEVEL_DEFAULT, + RPC_C_IMP_LEVEL_IMPERSONATE, + null(), + EOAC_NONE, + null(), + ); + // gh-96684: CoInitializeSecurity will fail if another part of the app has + // already called it. + if hr == RPC_E_TOO_LATE { + hr = 0; + } + + if succeeded(hr) { + hr = CoCreateInstance( + &CLSID_WBEM_LOCATOR, + null_mut(), + CLSCTX_INPROC_SERVER, + &IID_IWBEM_LOCATOR, + &mut locator, + ); + } + if succeeded(hr) && SetEvent(init_event) == 0 { + hr = hresult_from_win32(GetLastError()); + } + + if succeeded(hr) { + let root_cimv2 = wide_str("ROOT\\CIMV2"); + let bstr_root = SysAllocString(root_cimv2.as_ptr()); + hr = locator_connect_server( + locator, + bstr_root, + null(), + null(), + null(), + 0, + null(), + null_mut(), + &mut services, + ); + if !bstr_root.is_null() { + SysFreeString(bstr_root); + } + } + if succeeded(hr) && SetEvent(connect_event) == 0 { + hr = hresult_from_win32(GetLastError()); + } + + if succeeded(hr) { + hr = CoSetProxyBlanket( + services, + RPC_C_AUTHN_WINNT, + RPC_C_AUTHZ_NONE, + null(), + RPC_C_AUTHN_LEVEL_CALL, + RPC_C_IMP_LEVEL_IMPERSONATE, + null(), + EOAC_NONE, + ); + } + if succeeded(hr) { + let wql = wide_str("WQL"); + let bstr_wql = SysAllocString(wql.as_ptr()); + hr = services_exec_query( + services, + bstr_wql, + bstr_query, + WBEM_FLAG_FORWARD_ONLY | WBEM_FLAG_RETURN_IMMEDIATELY, + null_mut(), + &mut enumerator, + ); + if !bstr_wql.is_null() { + SysFreeString(bstr_wql); + } + } + + // Enumerate results and write to pipe + let mut value: *mut c_void; + let mut start_of_enum = true; + let null_sep: u16 = 0; + let eq_sign: u16 = b'=' as u16; + + while succeeded(hr) { + let mut got: u32 = 0; + let mut written: u32 = 0; + value = null_mut(); + hr = enum_next(enumerator, WBEM_INFINITE, 1, &mut value, &mut got); + + if hr == WBEM_S_FALSE { + hr = 0; + break; + } + if failed(hr) || got != 1 || value.is_null() { + continue; + } + + if !start_of_enum + && WriteFile( + write_pipe, + &null_sep as *const u16 as *const _, + 2, + &mut written, + null_mut(), + ) == 0 + { + hr = hresult_from_win32(GetLastError()); + com_release(value); + break; + } + start_of_enum = false; + + hr = object_begin_enumeration(value, 0); + if failed(hr) { + com_release(value); + break; + } + + while succeeded(hr) { + let mut prop_name: *mut u16 = null_mut(); + let mut prop_value = VARIANT::zeroed(); + let mut flavor: i32 = 0; + + hr = object_next( + value, + 0, + &mut prop_name, + &mut prop_value, + null_mut(), + &mut flavor, + ); + + if hr == WBEM_S_NO_MORE_DATA { + hr = 0; + break; + } + + if succeeded(hr) + && (flavor & WBEM_FLAVOR_MASK_ORIGIN) != WBEM_FLAVOR_ORIGIN_SYSTEM + { + let mut prop_str = [0u16; BUFFER_SIZE]; + hr = + VariantToString(&prop_value, prop_str.as_mut_ptr(), BUFFER_SIZE as u32); + + if succeeded(hr) { + let cb_str1 = (wcslen(prop_name) * 2) as u32; + let cb_str2 = (wcslen(prop_str.as_ptr()) * 2) as u32; + + if WriteFile( + write_pipe, + prop_name as *const _, + cb_str1, + &mut written, + null_mut(), + ) == 0 + || WriteFile( + write_pipe, + &eq_sign as *const u16 as *const _, + 2, + &mut written, + null_mut(), + ) == 0 + || WriteFile( + write_pipe, + prop_str.as_ptr() as *const _, + cb_str2, + &mut written, + null_mut(), + ) == 0 + || WriteFile( + write_pipe, + &null_sep as *const u16 as *const _, + 2, + &mut written, + null_mut(), + ) == 0 + { + hr = hresult_from_win32(GetLastError()); + } + } + + VariantClear(&mut prop_value); + SysFreeString(prop_name); + } + } + + object_end_enumeration(value); + com_release(value); + } + + // Cleanup + if !bstr_query.is_null() { + SysFreeString(bstr_query); + } + if !enumerator.is_null() { + com_release(enumerator); + } + if !services.is_null() { + com_release(services); + } + if !locator.is_null() { + com_release(locator); + } + CoUninitialize(); + CloseHandle(write_pipe); + + hr as u32 + } + } + + /// Runs a WMI query against the local machine. + /// + /// This returns a single string with 'name=value' pairs in a flat array separated + /// by null characters. + #[pyfunction] + fn exec_query(query: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { + let query_str = query.expect_str(); + + if !query_str + .get(..7) + .is_some_and(|s| s.eq_ignore_ascii_case("select ")) + { + return Err(vm.new_value_error("only SELECT queries are supported")); + } + + let query_wide = wide_str(query_str); + + let mut h_thread: HANDLE = null_mut(); + let mut err: u32 = 0; + let mut buffer = [0u16; BUFFER_SIZE]; + let mut offset: u32 = 0; + let mut bytes_read: u32 = 0; + + let mut read_pipe: HANDLE = null_mut(); + let mut write_pipe: HANDLE = null_mut(); + + unsafe { + let init_event = CreateEventW(null(), 1, 0, null()); + let connect_event = CreateEventW(null(), 1, 0, null()); + + if init_event.is_null() + || connect_event.is_null() + || CreatePipe(&mut read_pipe, &mut write_pipe, null(), 0) == 0 + { + err = GetLastError(); + } else { + let thread_data = Box::new(QueryThreadData { + query: query_wide, + write_pipe, + init_event, + connect_event, + }); + let thread_data_ptr = Box::into_raw(thread_data); + + h_thread = CreateThread( + null(), + 0, + Some(query_thread), + thread_data_ptr as *const _ as *mut _, + 0, + null_mut(), + ); + + if h_thread.is_null() { + err = GetLastError(); + // Thread didn't start, so recover data and close write pipe + let data = Box::from_raw(thread_data_ptr); + CloseHandle(data.write_pipe); + } + } + + // gh-112278: Timeout for COM init and WMI connection + if err == 0 { + err = wait_event(init_event, 1000); + if err == 0 { + err = wait_event(connect_event, 100); + } + } + + // Read results from pipe + while err == 0 { + let buf_ptr = (buffer.as_mut_ptr() as *mut u8).add(offset as usize); + let buf_remaining = (BUFFER_SIZE * 2) as u32 - offset; + + if ReadFile( + read_pipe, + buf_ptr as *mut _, + buf_remaining, + &mut bytes_read, + null_mut(), + ) != 0 + { + offset += bytes_read; + if offset >= (BUFFER_SIZE * 2) as u32 { + err = ERROR_MORE_DATA; + } + } else { + err = GetLastError(); + } + } + + if !read_pipe.is_null() { + CloseHandle(read_pipe); + } + + if !h_thread.is_null() { + let thread_err: u32; + match WaitForSingleObject(h_thread, 100) { + WAIT_OBJECT_0 => { + let mut exit_code: u32 = 0; + if GetExitCodeThread(h_thread, &mut exit_code) == 0 { + thread_err = GetLastError(); + } else { + thread_err = exit_code; + } + } + WAIT_TIMEOUT => { + thread_err = WAIT_TIMEOUT; + } + _ => { + thread_err = GetLastError(); + } + } + if err == 0 || err == ERROR_BROKEN_PIPE { + err = thread_err; + } + + CloseHandle(h_thread); + } + + CloseHandle(init_event); + CloseHandle(connect_event); + } + + if err == ERROR_MORE_DATA { + return Err(vm.new_os_error(format!( + "Query returns more than {} characters", + BUFFER_SIZE, + ))); + } else if err != 0 { + return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm)); + } + + if offset == 0 { + return Ok(String::new()); + } + + let char_count = (offset as usize) / 2 - 1; + Ok(String::from_utf16_lossy(&buffer[..char_count])) + } +} diff --git a/crates/vm/src/stdlib/atexit.rs b/crates/vm/src/stdlib/atexit.rs new file mode 100644 index 00000000000..638927fe90f --- /dev/null +++ b/crates/vm/src/stdlib/atexit.rs @@ -0,0 +1,92 @@ +pub use atexit::_run_exitfuncs; +pub(crate) use atexit::module_def; + +#[pymodule] +mod atexit { + use crate::{AsObject, PyObjectRef, PyResult, VirtualMachine, function::FuncArgs}; + + #[pyfunction] + fn register(func: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { + // Callbacks go in LIFO order (insert at front) + vm.state + .atexit_funcs + .lock() + .insert(0, Box::new((func.clone(), args))); + func + } + + #[pyfunction] + fn _clear(vm: &VirtualMachine) { + vm.state.atexit_funcs.lock().clear(); + } + + #[pyfunction] + fn unregister(func: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Iterate backward (oldest to newest in LIFO list). + // Release the lock during comparison so __eq__ can call atexit functions. + let mut i = { + let funcs = vm.state.atexit_funcs.lock(); + funcs.len() as isize - 1 + }; + while i >= 0 { + let (cb, entry_ptr) = { + let funcs = vm.state.atexit_funcs.lock(); + if i as usize >= funcs.len() { + i = funcs.len() as isize; + i -= 1; + continue; + } + let entry = &funcs[i as usize]; + (entry.0.clone(), &**entry as *const (PyObjectRef, FuncArgs)) + }; + // Lock released: __eq__ can safely call atexit functions + let eq = vm.bool_eq(&func, &cb)?; + if eq { + // The entry may have moved during __eq__. Search backward by identity. + let mut funcs = vm.state.atexit_funcs.lock(); + let mut j = (funcs.len() as isize - 1).min(i); + while j >= 0 { + if core::ptr::eq(&**funcs.get(j as usize).unwrap(), entry_ptr) { + funcs.remove(j as usize); + i = j; + break; + } + j -= 1; + } + } + { + let funcs = vm.state.atexit_funcs.lock(); + if i as usize >= funcs.len() { + i = funcs.len() as isize; + } + } + i -= 1; + } + Ok(()) + } + + #[pyfunction] + pub fn _run_exitfuncs(vm: &VirtualMachine) { + let funcs: Vec<_> = core::mem::take(&mut *vm.state.atexit_funcs.lock()); + // Callbacks stored in LIFO order, iterate forward + for entry in funcs.into_iter() { + let (func, args) = *entry; + if let Err(e) = func.call(args, vm) { + let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); + let msg = func + .repr(vm) + .ok() + .map(|r| format!("Exception ignored in atexit callback {}", r.as_wtf8())); + vm.run_unraisable(e, msg, vm.ctx.none()); + if exit { + break; + } + } + } + } + + #[pyfunction] + fn _ncallbacks(vm: &VirtualMachine) -> usize { + vm.state.atexit_funcs.lock().len() + } +} diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs new file mode 100644 index 00000000000..a506f774609 --- /dev/null +++ b/crates/vm/src/stdlib/builtins.rs @@ -0,0 +1,1474 @@ +//! Builtin function definitions. +//! +//! Implements the list of [builtin Python functions](https://docs.python.org/3/library/builtins.html). +use crate::{Py, VirtualMachine, builtins::PyModule, class::PyClassImpl}; +pub(crate) use builtins::{DOC, module_def}; +pub use builtins::{ascii, print, reversed}; + +#[pymodule] +mod builtins { + use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + builtins::{ + PyByteArray, PyBytes, PyDictRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, + PyUtf8StrRef, + enumerate::PyReverseSequenceIterator, + function::{PyCellRef, PyFunction}, + int::PyIntRef, + iter::PyCallableIterator, + list::{PyList, SortOptions}, + }, + common::{hash::PyHash, str::to_ascii}, + function::{ + ArgBytesLike, ArgCallable, ArgIndex, ArgIntoBool, ArgIterable, ArgMapping, + ArgStrOrBytesLike, Either, FsPath, FuncArgs, KwArgs, OptionalArg, OptionalOption, + PosArgs, + }, + protocol::{PyIter, PyIterReturn}, + py_io, + readline::{Readline, ReadlineResult}, + stdlib::sys, + types::PyComparisonOp, + }; + use itertools::Itertools; + use num_traits::{Signed, ToPrimitive, Zero}; + use rustpython_common::wtf8::CodePoint; + + #[cfg(not(feature = "rustpython-compiler"))] + const CODEGEN_NOT_SUPPORTED: &str = + "can't compile() to bytecode when the `codegen` feature of rustpython is disabled"; + + #[pyfunction] + fn abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._abs(&x) + } + + #[pyfunction] + fn all(iterable: ArgIterable<ArgIntoBool>, vm: &VirtualMachine) -> PyResult<bool> { + for item in iterable.iter(vm)? { + if !item?.into_bool() { + return Ok(false); + } + } + Ok(true) + } + + #[pyfunction] + fn any(iterable: ArgIterable<ArgIntoBool>, vm: &VirtualMachine) -> PyResult<bool> { + for item in iterable.iter(vm)? { + if item?.into_bool() { + return Ok(true); + } + } + Ok(false) + } + + #[pyfunction] + pub fn ascii(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<ascii::AsciiString> { + let repr = obj.repr(vm)?; + let ascii = to_ascii(repr.as_wtf8()); + Ok(ascii) + } + + #[pyfunction] + fn bin(x: PyIntRef) -> String { + let x = x.as_bigint(); + if x.is_negative() { + format!("-0b{:b}", x.abs()) + } else { + format!("0b{x:b}") + } + } + + #[pyfunction] + fn callable(obj: PyObjectRef) -> bool { + obj.is_callable() + } + + #[pyfunction] + fn chr(i: PyIntRef, vm: &VirtualMachine) -> PyResult<CodePoint> { + let value = i + .as_bigint() + .to_u32() + .and_then(CodePoint::from_u32) + .ok_or_else(|| vm.new_value_error("chr() arg not in range(0x110000)"))?; + Ok(value) + } + + #[derive(FromArgs)] + #[allow(dead_code)] + struct CompileArgs { + source: PyObjectRef, + filename: FsPath, + mode: PyUtf8StrRef, + #[pyarg(any, optional)] + flags: OptionalArg<PyIntRef>, + #[pyarg(any, optional)] + dont_inherit: OptionalArg<bool>, + #[pyarg(any, optional)] + optimize: OptionalArg<PyIntRef>, + #[pyarg(any, optional)] + _feature_version: OptionalArg<i32>, + } + + /// Detect PEP 263 encoding cookie from source bytes. + /// Checks first two lines for `# coding[:=] <encoding>` pattern. + /// Returns the encoding name if found, or None for default (UTF-8). + #[cfg(feature = "parser")] + fn detect_source_encoding(source: &[u8]) -> Option<String> { + fn find_encoding_in_line(line: &[u8]) -> Option<String> { + // PEP 263: '#' must be preceded only by whitespace/formfeed + let hash_pos = line.iter().position(|&b| b == b'#')?; + if !line[..hash_pos] + .iter() + .all(|&b| b == b' ' || b == b'\t' || b == b'\x0c' || b == b'\r') + { + return None; + } + let after_hash = &line[hash_pos..]; + + // Find "coding" after the # + let coding_pos = after_hash.windows(6).position(|w| w == b"coding")?; + let after_coding = &after_hash[coding_pos + 6..]; + + // Next char must be ':' or '=' + let rest = if after_coding.first() == Some(&b':') || after_coding.first() == Some(&b'=') + { + &after_coding[1..] + } else { + return None; + }; + + // Skip whitespace + let rest = rest + .iter() + .copied() + .skip_while(|&b| b == b' ' || b == b'\t') + .collect::<Vec<_>>(); + + // Read encoding name: [-\w.]+ + let name: String = rest + .iter() + .take_while(|&&b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_' || b == b'.') + .map(|&b| b as char) + .collect(); + + if name.is_empty() { None } else { Some(name) } + } + + // Split into lines (first two only) + let mut lines = source.splitn(3, |&b| b == b'\n'); + + if let Some(first) = lines.next() { + // Strip BOM if present + let first = first.strip_prefix(b"\xef\xbb\xbf").unwrap_or(first); + if let Some(enc) = find_encoding_in_line(first) { + return Some(enc); + } + // Only check second line if first line is blank or a comment + let trimmed = first + .iter() + .skip_while(|&&b| b == b' ' || b == b'\t' || b == b'\x0c' || b == b'\r') + .copied() + .collect::<Vec<_>>(); + if !trimmed.is_empty() && trimmed[0] != b'#' { + return None; + } + } + + lines.next().and_then(find_encoding_in_line) + } + + /// Decode source bytes to a string, handling PEP 263 encoding declarations + /// and BOM. Raises SyntaxError for invalid UTF-8 without an encoding + /// declaration. + /// Check if an encoding name is a UTF-8 variant after normalization. + /// Matches: utf-8, utf_8, utf8, UTF-8, etc. + #[cfg(feature = "parser")] + fn is_utf8_encoding(name: &str) -> bool { + let normalized: String = name.chars().filter(|&c| c != '-' && c != '_').collect(); + normalized.eq_ignore_ascii_case("utf8") + } + + #[cfg(feature = "parser")] + fn decode_source_bytes(source: &[u8], filename: &str, vm: &VirtualMachine) -> PyResult<String> { + let has_bom = source.starts_with(b"\xef\xbb\xbf"); + let encoding = detect_source_encoding(source); + + let is_utf8 = encoding.as_deref().is_none_or(is_utf8_encoding); + + // Validate BOM + encoding combination + if has_bom && !is_utf8 { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + format!("encoding problem for '{filename}': utf-8").into(), + )); + } + + if is_utf8 { + let src = if has_bom { &source[3..] } else { source }; + match core::str::from_utf8(src) { + Ok(s) => Ok(s.to_owned()), + Err(e) => { + let bad_byte = src[e.valid_up_to()]; + let line = src[..e.valid_up_to()] + .iter() + .filter(|&&b| b == b'\n') + .count() + + 1; + Err(vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + format!( + "Non-UTF-8 code starting with '\\x{bad_byte:02x}' \ + on line {line}, but no encoding declared; \ + see https://peps.python.org/pep-0263/ for details \ + ({filename}, line {line})" + ) + .into(), + )) + } + } + } else { + // Use codec registry for non-UTF-8 encodings + let enc = encoding.as_deref().unwrap(); + let bytes_obj = vm.ctx.new_bytes(source.to_vec()); + let decoded = vm + .state + .codec_registry + .decode_text(bytes_obj.into(), enc, None, vm) + .map_err(|exc| { + if exc.fast_isinstance(vm.ctx.exceptions.lookup_error) { + vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + format!("unknown encoding for '{filename}': {enc}").into(), + ) + } else { + exc + } + })?; + Ok(decoded.to_string_lossy().into_owned()) + } + } + + #[cfg(any(feature = "parser", feature = "compiler"))] + #[pyfunction] + fn compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult { + #[cfg(not(feature = "ast"))] + { + _ = args; // to disable unused warning + return Err(vm.new_type_error("AST Not Supported")); + } + #[cfg(feature = "ast")] + { + use crate::{class::PyClassImpl, stdlib::_ast}; + + let feature_version = feature_version_from_arg(args._feature_version, vm)?; + + let mode_str = args.mode.as_str(); + + let optimize: i32 = args.optimize.map_or(Ok(-1), |v| v.try_to_primitive(vm))?; + let optimize: u8 = if optimize == -1 { + vm.state.config.settings.optimize + } else { + optimize + .try_into() + .map_err(|_| vm.new_value_error("compile() optimize value invalid"))? + }; + + if args + .source + .fast_isinstance(&_ast::NodeAst::make_static_type()) + { + let flags: i32 = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; + let is_ast_only = !(flags & _ast::PY_CF_ONLY_AST).is_zero(); + + // func_type mode requires PyCF_ONLY_AST + if mode_str == "func_type" && !is_ast_only { + return Err(vm.new_value_error( + "compile() mode 'func_type' requires flag PyCF_ONLY_AST", + )); + } + + // compile(ast_node, ..., PyCF_ONLY_AST) returns the AST after validation + if is_ast_only { + let (expected_type, expected_name) = _ast::mode_type_and_name(mode_str) + .ok_or_else(|| { + vm.new_value_error( + "compile() mode must be 'exec', 'eval', 'single' or 'func_type'", + ) + })?; + if !args.source.fast_isinstance(&expected_type) { + return Err(vm.new_type_error(format!( + "expected {} node, got {}", + expected_name, + args.source.class().name() + ))); + } + _ast::validate_ast_object(vm, args.source.clone())?; + return Ok(args.source); + } + + #[cfg(not(feature = "rustpython-codegen"))] + { + return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())); + } + #[cfg(feature = "rustpython-codegen")] + { + let mode = mode_str + .parse::<crate::compiler::Mode>() + .map_err(|err| vm.new_value_error(err.to_string()))?; + return _ast::compile( + vm, + args.source, + &args.filename.to_string_lossy(), + mode, + Some(optimize), + ); + } + } + + #[cfg(not(feature = "parser"))] + { + const PARSER_NOT_SUPPORTED: &str = "can't compile() source code when the `parser` feature of rustpython is disabled"; + Err(vm.new_type_error(PARSER_NOT_SUPPORTED.to_owned())) + } + #[cfg(feature = "parser")] + { + use crate::convert::ToPyException; + + use ruff_python_parser as parser; + + let source = ArgStrOrBytesLike::try_from_object(vm, args.source)?; + let source = source.borrow_bytes(); + + let source = decode_source_bytes(&source, &args.filename.to_string_lossy(), vm)?; + let source = source.as_str(); + + let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; + + if !(flags & !_ast::PY_COMPILE_FLAGS_MASK).is_zero() { + return Err(vm.new_value_error("compile() unrecognized flags")); + } + + let allow_incomplete = !(flags & _ast::PY_CF_ALLOW_INCOMPLETE_INPUT).is_zero(); + let type_comments = !(flags & _ast::PY_CF_TYPE_COMMENTS).is_zero(); + + let optimize_level = optimize; + + if (flags & _ast::PY_CF_ONLY_AST).is_zero() { + #[cfg(not(feature = "compiler"))] + { + Err(vm.new_value_error(CODEGEN_NOT_SUPPORTED.to_owned())) + } + #[cfg(feature = "compiler")] + { + if let Some(feature_version) = feature_version { + let mode = mode_str + .parse::<parser::Mode>() + .map_err(|err| vm.new_value_error(err.to_string()))?; + let _ = _ast::parse( + vm, + source, + mode, + optimize_level, + Some(feature_version), + type_comments, + ) + .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm))?; + } + + let mode = mode_str + .parse::<crate::compiler::Mode>() + .map_err(|err| vm.new_value_error(err.to_string()))?; + + let mut opts = vm.compile_opts(); + opts.optimize = optimize; + + let code = vm + .compile_with_opts( + source, + mode, + args.filename.to_string_lossy().into_owned(), + opts, + ) + .map_err(|err| { + (err, Some(source), allow_incomplete).to_pyexception(vm) + })?; + Ok(code.into()) + } + } else { + if mode_str == "func_type" { + return _ast::parse_func_type(vm, source, optimize_level, feature_version) + .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm)); + } + + let mode = mode_str + .parse::<parser::Mode>() + .map_err(|err| vm.new_value_error(err.to_string()))?; + let parsed = _ast::parse( + vm, + source, + mode, + optimize_level, + feature_version, + type_comments, + ) + .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm))?; + + if mode_str == "single" { + return _ast::wrap_interactive(vm, parsed); + } + + Ok(parsed) + } + } + } + } + + #[cfg(feature = "ast")] + fn feature_version_from_arg( + feature_version: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<Option<ruff_python_ast::PythonVersion>> { + let minor = match feature_version.into_option() { + Some(minor) => minor, + None => return Ok(None), + }; + + if minor < 0 { + return Ok(None); + } + + let minor = u8::try_from(minor) + .map_err(|_| vm.new_value_error("compile() _feature_version out of range"))?; + Ok(Some(ruff_python_ast::PythonVersion { major: 3, minor })) + } + + #[pyfunction] + fn delattr(obj: PyObjectRef, attr: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { + vm.new_type_error(format!( + "attribute name must be string, not '{}'", + attr.class().name() + )) + })?; + obj.del_attr(attr, vm) + } + + #[pyfunction] + fn dir(obj: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<PyList> { + vm.dir(obj.into_option()) + } + + #[pyfunction] + fn divmod(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm._divmod(&a, &b) + } + + #[derive(FromArgs)] + struct ScopeArgs { + #[pyarg(any, default)] + globals: Option<PyObjectRef>, + #[pyarg(any, default)] + locals: Option<ArgMapping>, + } + + impl ScopeArgs { + fn make_scope( + self, + vm: &VirtualMachine, + func_name: &'static str, + ) -> PyResult<crate::scope::Scope> { + fn validate_globals_dict( + globals: &PyObject, + vm: &VirtualMachine, + func_name: &'static str, + ) -> PyResult<()> { + if !globals.fast_isinstance(vm.ctx.types.dict_type) { + return Err(match func_name { + "eval" => { + let is_mapping = globals.mapping_unchecked().check(); + vm.new_type_error(if is_mapping { + "globals must be a real dict; try eval(expr, {}, mapping)" + .to_owned() + } else { + "globals must be a dict".to_owned() + }) + } + "exec" => vm.new_type_error(format!( + "exec() globals must be a dict, not {}", + globals.class().name() + )), + _ => vm.new_type_error("globals must be a dict"), + }); + } + Ok(()) + } + + let (globals, locals) = match self.globals { + Some(globals) => { + validate_globals_dict(&globals, vm, func_name)?; + + let globals = PyDictRef::try_from_object(vm, globals)?; + if !globals.contains_key(identifier!(vm, __builtins__), vm) { + let builtins_dict = vm.builtins.dict().into(); + globals.set_item(identifier!(vm, __builtins__), builtins_dict, vm)?; + } + ( + globals.clone(), + self.locals + .unwrap_or_else(|| ArgMapping::from_dict_exact(globals.clone())), + ) + } + None => ( + vm.current_globals(), + if let Some(locals) = self.locals { + locals + } else { + vm.current_locals()? + }, + ), + }; + + let scope = crate::scope::Scope::with_builtins(Some(locals), globals, vm); + Ok(scope) + } + } + + #[pyfunction] + fn eval( + source: Either<ArgStrOrBytesLike, PyRef<crate::builtins::PyCode>>, + scope: ScopeArgs, + vm: &VirtualMachine, + ) -> PyResult { + let scope = scope.make_scope(vm, "eval")?; + + // source as string + let code = match source { + Either::A(either) => { + let source: &[u8] = &either.borrow_bytes(); + if source.contains(&0) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + "source code string cannot contain null bytes".into(), + )); + } + + let source = core::str::from_utf8(source).map_err(|err| { + let msg = format!( + "(unicode error) 'utf-8' codec can't decode byte 0x{:x?} in position {}: invalid start byte", + source[err.valid_up_to()], + err.valid_up_to() + ); + + vm.new_exception_msg(vm.ctx.exceptions.syntax_error.to_owned(), msg.into()) + })?; + Ok(Either::A(vm.ctx.new_utf8_str(source.trim_start()))) + } + Either::B(code) => Ok(Either::B(code)), + }?; + run_code(vm, code, scope, crate::compiler::Mode::Eval, "eval") + } + + #[pyfunction] + fn exec( + source: Either<PyUtf8StrRef, PyRef<crate::builtins::PyCode>>, + scope: ScopeArgs, + vm: &VirtualMachine, + ) -> PyResult { + let scope = scope.make_scope(vm, "exec")?; + run_code(vm, source, scope, crate::compiler::Mode::Exec, "exec") + } + + fn run_code( + vm: &VirtualMachine, + source: Either<PyUtf8StrRef, PyRef<crate::builtins::PyCode>>, + scope: crate::scope::Scope, + #[allow(unused_variables)] mode: crate::compiler::Mode, + func: &str, + ) -> PyResult { + // Determine code object: + let code_obj = match source { + #[cfg(feature = "rustpython-compiler")] + Either::A(string) => { + let source = string.as_str(); + vm.compile(source, mode, "<string>".to_owned()) + .map_err(|err| vm.new_syntax_error(&err, Some(source)))? + } + #[cfg(not(feature = "rustpython-compiler"))] + Either::A(_) => return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())), + Either::B(code_obj) => code_obj, + }; + + if !code_obj.freevars.is_empty() { + return Err(vm.new_type_error(format!( + "code object passed to {func}() may not contain free variables" + ))); + } + + // Run the code: + vm.run_code_obj(code_obj, scope) + } + + #[pyfunction] + fn format( + value: PyObjectRef, + format_spec: OptionalArg<PyStrRef>, + vm: &VirtualMachine, + ) -> PyResult<PyStrRef> { + vm.format(&value, format_spec.unwrap_or(vm.ctx.new_str(""))) + } + + #[pyfunction] + fn getattr( + obj: PyObjectRef, + attr: PyObjectRef, + default: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { + vm.new_type_error(format!( + "attribute name must be string, not '{}'", + attr.class().name() + )) + })?; + + if let OptionalArg::Present(default) = default { + Ok(vm.get_attribute_opt(obj, attr)?.unwrap_or(default)) + } else { + obj.get_attr(attr, vm) + } + } + + #[pyfunction] + fn globals(vm: &VirtualMachine) -> PyDictRef { + vm.current_globals() + } + + #[pyfunction] + fn hasattr(obj: PyObjectRef, attr: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { + vm.new_type_error(format!( + "attribute name must be string, not '{}'", + attr.class().name() + )) + })?; + Ok(vm.get_attribute_opt(obj, attr)?.is_some()) + } + + #[pyfunction] + fn hash(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyHash> { + obj.hash(vm) + } + + #[pyfunction] + fn breakpoint(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + match vm + .sys_module + .get_attr(vm.ctx.intern_str("breakpointhook"), vm) + { + Ok(hook) => hook.as_ref().call(args, vm), + Err(_) => Err(vm.new_runtime_error("lost sys.breakpointhook")), + } + } + + #[pyfunction] + fn hex(number: ArgIndex) -> String { + let number = number.into_int_ref(); + let n = number.as_bigint(); + format!("{n:#x}") + } + + #[pyfunction] + fn id(obj: PyObjectRef) -> usize { + obj.get_id() + } + + #[pyfunction] + fn input(prompt: OptionalArg<PyStrRef>, vm: &VirtualMachine) -> PyResult { + use std::io::IsTerminal; + + let stdin = sys::get_stdin(vm)?; + let stdout = sys::get_stdout(vm)?; + let stderr = sys::get_stderr(vm)?; + + let _ = vm.call_method(&stderr, "flush", ()); + + let fd_matches = |obj, expected| { + vm.call_method(obj, "fileno", ()) + .and_then(|o| i64::try_from_object(vm, o)) + .is_ok_and(|fd| fd == expected) + }; + + // Check if we should use rustyline (interactive terminal, not PTY child) + let use_rustyline = fd_matches(&stdin, 0) + && fd_matches(&stdout, 1) + && std::io::stdin().is_terminal() + && !is_pty_child(); + + // Disable rustyline if prompt contains surrogates (not valid UTF-8 for terminal) + let prompt_str = match &prompt { + OptionalArg::Present(s) => s.to_str(), + OptionalArg::Missing => Some(""), + }; + let use_rustyline = use_rustyline && prompt_str.is_some(); + + if use_rustyline { + let prompt = prompt_str.unwrap(); + let mut readline = Readline::new(()); + match readline.readline(prompt) { + ReadlineResult::Line(s) => Ok(vm.ctx.new_str(s).into()), + ReadlineResult::Eof => { + Err(vm.new_exception_empty(vm.ctx.exceptions.eof_error.to_owned())) + } + ReadlineResult::Interrupt => { + Err(vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned())) + } + ReadlineResult::Io(e) => Err(vm.new_os_error(e.to_string())), + #[cfg(unix)] + ReadlineResult::OsError(num) => Err(vm.new_os_error(num.to_string())), + ReadlineResult::Other(e) => Err(vm.new_runtime_error(e.to_string())), + } + } else { + if let OptionalArg::Present(prompt) = prompt { + vm.call_method(&stdout, "write", (prompt,))?; + } + let _ = vm.call_method(&stdout, "flush", ()); + py_io::file_readline(&stdin, None, vm) + } + } + + /// Check if we're running in a PTY child process (e.g., after pty.fork()). + /// pty.fork() calls setsid(), making the child a session leader. + /// In this case, rustyline may hang because it uses raw mode. + #[cfg(unix)] + fn is_pty_child() -> bool { + use nix::unistd::{getpid, getsid}; + // If this process is a session leader, we're likely in a PTY child + getsid(None) == Ok(getpid()) + } + + #[cfg(not(unix))] + fn is_pty_child() -> bool { + false + } + + #[pyfunction] + fn isinstance(obj: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + obj.is_instance(&typ, vm) + } + + #[pyfunction] + fn issubclass(subclass: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { + subclass.is_subclass(&typ, vm) + } + + #[pyfunction] + fn iter( + iter_target: PyObjectRef, + sentinel: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyIter> { + if let OptionalArg::Present(sentinel) = sentinel { + let callable = ArgCallable::try_from_object(vm, iter_target)?; + let iterator = PyCallableIterator::new(callable, sentinel) + .into_ref(&vm.ctx) + .into(); + Ok(PyIter::new(iterator)) + } else { + iter_target.get_iter(vm) + } + } + + #[pyfunction] + fn aiter(iter_target: PyObjectRef, vm: &VirtualMachine) -> PyResult { + iter_target.get_aiter(vm) + } + + #[pyfunction] + fn anext( + aiter: PyObjectRef, + default_value: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult { + use crate::builtins::asyncgenerator::PyAnextAwaitable; + + // Check if object is an async iterator (has __anext__ method) + if !aiter.class().has_attr(identifier!(vm, __anext__)) { + return Err(vm.new_type_error(format!( + "'{}' object is not an async iterator", + aiter.class().name() + ))); + } + + let awaitable = vm.call_method(&aiter, "__anext__", ())?; + + if let OptionalArg::Present(default) = default_value { + Ok(PyAnextAwaitable::new(awaitable, default) + .into_ref(&vm.ctx) + .into()) + } else { + Ok(awaitable) + } + } + + #[pyfunction] + fn len(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { + obj.length(vm) + } + + #[pyfunction] + fn locals(vm: &VirtualMachine) -> PyResult<ArgMapping> { + vm.current_locals() + } + + fn min_or_max( + mut args: FuncArgs, + vm: &VirtualMachine, + func_name: &str, + op: PyComparisonOp, + ) -> PyResult { + let default = args.take_keyword("default"); + let key_func = args.take_keyword("key"); + + if let Some(err) = args.check_kwargs_empty(vm) { + return Err(err); + } + + let candidates = match args.args.len().cmp(&1) { + core::cmp::Ordering::Greater => { + if default.is_some() { + return Err(vm.new_type_error(format!( + "Cannot specify a default for {func_name}() with multiple positional arguments" + ))); + } + args.args + } + core::cmp::Ordering::Equal => args.args[0].try_to_value(vm)?, + core::cmp::Ordering::Less => { + // zero arguments means type error: + return Err( + vm.new_type_error(format!("{func_name} expected at least 1 argument, got 0")) + ); + } + }; + + let mut candidates_iter = candidates.into_iter(); + let mut x = match candidates_iter.next() { + Some(x) => x, + None => { + return default.ok_or_else(|| { + vm.new_value_error(format!("{func_name}() iterable argument is empty")) + }); + } + }; + + let key_func = key_func.filter(|f| !vm.is_none(f)); + if let Some(ref key_func) = key_func { + let mut x_key = key_func.call((x.clone(),), vm)?; + for y in candidates_iter { + let y_key = key_func.call((y.clone(),), vm)?; + if y_key.rich_compare_bool(&x_key, op, vm)? { + x = y; + x_key = y_key; + } + } + } else { + for y in candidates_iter { + if y.rich_compare_bool(&x, op, vm)? { + x = y; + } + } + } + + Ok(x) + } + + #[pyfunction] + fn max(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + min_or_max(args, vm, "max", PyComparisonOp::Gt) + } + + #[pyfunction] + fn min(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + min_or_max(args, vm, "min", PyComparisonOp::Lt) + } + + #[pyfunction] + fn next( + iterator: PyObjectRef, + default_value: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn> { + if !PyIter::check(&iterator) { + return Err(vm.new_type_error(format!( + "{} object is not an iterator", + iterator.class().name() + ))); + } + PyIter::new(iterator) + .next(vm) + .map(|iter_ret| match iter_ret { + PyIterReturn::Return(obj) => PyIterReturn::Return(obj), + PyIterReturn::StopIteration(v) => { + default_value.map_or(PyIterReturn::StopIteration(v), PyIterReturn::Return) + } + }) + } + + #[pyfunction] + fn oct(number: ArgIndex, vm: &VirtualMachine) -> PyResult { + let number = number.into_int_ref(); + let n = number.as_bigint(); + let s = if n.is_negative() { + format!("-0o{:o}", n.abs()) + } else { + format!("0o{n:o}") + }; + + Ok(vm.ctx.new_str(s).into()) + } + + #[pyfunction] + fn ord(string: Either<ArgBytesLike, PyStrRef>, vm: &VirtualMachine) -> PyResult<u32> { + match string { + Either::A(bytes) => bytes.with_ref(|bytes| { + let bytes_len = bytes.len(); + if bytes_len != 1 { + return Err(vm.new_type_error(format!( + "ord() expected a character, but string of length {bytes_len} found" + ))); + } + Ok(u32::from(bytes[0])) + }), + Either::B(string) => match string.as_wtf8().code_points().exactly_one() { + Ok(character) => Ok(character.to_u32()), + Err(_) => { + let string_len = string.char_len(); + Err(vm.new_type_error(format!( + "ord() expected a character, but string of length {string_len} found" + ))) + } + }, + } + } + + #[derive(FromArgs)] + struct PowArgs { + base: PyObjectRef, + exp: PyObjectRef, + #[pyarg(any, optional, name = "mod")] + modulus: Option<PyObjectRef>, + } + + #[pyfunction] + fn pow(args: PowArgs, vm: &VirtualMachine) -> PyResult { + let PowArgs { + base: x, + exp: y, + modulus, + } = args; + let modulus = modulus.as_ref().map_or(vm.ctx.none.as_object(), |m| m); + vm._pow(&x, &y, modulus) + } + + #[pyfunction] + pub fn exit(exit_code_arg: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult { + let code = exit_code_arg.unwrap_or_else(|| vm.ctx.new_int(0).into()); + Err(vm.new_exception(vm.ctx.exceptions.system_exit.to_owned(), vec![code])) + } + + #[derive(Debug, Default, FromArgs)] + pub struct PrintOptions { + #[pyarg(named, default)] + sep: Option<PyStrRef>, + #[pyarg(named, default)] + end: Option<PyStrRef>, + #[pyarg(named, default = ArgIntoBool::FALSE)] + flush: ArgIntoBool, + #[pyarg(named, default)] + file: Option<PyObjectRef>, + } + + #[pyfunction] + pub fn print(objects: PosArgs, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { + let file = match options.file { + Some(f) => f, + None => sys::get_stdout(vm)?, + }; + let write = |obj: PyStrRef| vm.call_method(&file, "write", (obj,)); + + let sep = options.sep.unwrap_or_else(|| vm.ctx.new_str(" ")); + + let mut first = true; + for object in objects { + if first { + first = false; + } else { + write(sep.clone())?; + } + + write(object.str(vm)?)?; + } + + let end = options.end.unwrap_or_else(|| vm.ctx.new_str("\n")); + write(end)?; + + if options.flush.into() { + vm.call_method(&file, "flush", ())?; + } + + Ok(()) + } + + #[pyfunction] + fn repr(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { + obj.repr(vm) + } + + #[pyfunction] + pub fn reversed(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(reversed_method) = vm.get_method(obj.clone(), identifier!(vm, __reversed__)) { + reversed_method?.call((), vm) + } else { + vm.get_method_or_type_error(obj.clone(), identifier!(vm, __getitem__), || { + "argument to reversed() must be a sequence".to_owned() + })?; + let len = obj.length(vm)?; + let obj_iterator = PyReverseSequenceIterator::new(obj, len); + Ok(obj_iterator.into_pyobject(vm)) + } + } + + #[derive(FromArgs)] + pub struct RoundArgs { + number: PyObjectRef, + #[pyarg(any, optional)] + ndigits: OptionalOption<PyObjectRef>, + } + + #[pyfunction] + fn round(RoundArgs { number, ndigits }: RoundArgs, vm: &VirtualMachine) -> PyResult { + let meth = vm + .get_special_method(&number, identifier!(vm, __round__))? + .ok_or_else(|| { + vm.new_type_error(format!( + "type {} doesn't define __round__", + number.class().name() + )) + })?; + match ndigits.flatten() { + Some(obj) => { + let ndigits = obj.try_index(vm)?; + meth.invoke((ndigits,), vm) + } + None => { + // without a parameter, the result type is coerced to int + meth.invoke((), vm) + } + } + } + + #[pyfunction] + fn setattr( + obj: PyObjectRef, + attr: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { + vm.new_type_error(format!( + "attribute name must be string, not '{}'", + attr.class().name() + )) + })?; + obj.set_attr(attr, value, vm)?; + Ok(()) + } + + // builtin_slice + + #[pyfunction] + fn sorted(iterable: PyObjectRef, opts: SortOptions, vm: &VirtualMachine) -> PyResult<PyList> { + let items: Vec<_> = iterable.try_to_value(vm)?; + let lst = PyList::from(items); + lst.sort(opts, vm)?; + Ok(lst) + } + + #[derive(FromArgs)] + pub struct SumArgs { + #[pyarg(positional)] + iterable: ArgIterable, + #[pyarg(any, optional)] + start: OptionalArg<PyObjectRef>, + } + + #[pyfunction] + fn sum(SumArgs { iterable, start }: SumArgs, vm: &VirtualMachine) -> PyResult { + // Start with zero and add at will: + let mut sum = start + .into_option() + .unwrap_or_else(|| vm.ctx.new_int(0).into()); + + match_class!(match sum { + PyStr => + return Err(vm.new_type_error( + "sum() can't sum strings [use ''.join(seq) instead]".to_owned() + )), + PyBytes => + return Err(vm.new_type_error( + "sum() can't sum bytes [use b''.join(seq) instead]".to_owned() + )), + PyByteArray => + return Err(vm.new_type_error( + "sum() can't sum bytearray [use b''.join(seq) instead]".to_owned() + )), + _ => (), + }); + + for item in iterable.iter(vm)? { + sum = vm._add(&sum, &*item?)?; + } + Ok(sum) + } + + #[derive(FromArgs)] + struct ImportArgs { + #[pyarg(any)] + name: PyStrRef, + #[pyarg(any, default)] + globals: Option<PyObjectRef>, + #[allow(dead_code)] + #[pyarg(any, default)] + locals: Option<PyObjectRef>, + #[pyarg(any, default)] + fromlist: Option<PyObjectRef>, + #[pyarg(any, default)] + level: i32, + } + + #[pyfunction] + fn __import__(args: ImportArgs, vm: &VirtualMachine) -> PyResult { + crate::import::import_module_level(&args.name, args.globals, args.fromlist, args.level, vm) + } + + #[pyfunction] + fn vars(obj: OptionalArg, vm: &VirtualMachine) -> PyResult { + if let OptionalArg::Present(obj) = obj { + obj.get_attr(identifier!(vm, __dict__), vm) + .map_err(|_| vm.new_type_error("vars() argument must have __dict__ attribute")) + } else { + Ok(vm.current_locals()?.into()) + } + } + + #[pyfunction] + pub fn __build_class__( + function: PyRef<PyFunction>, + name: PyStrRef, + bases: PosArgs, + mut kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let name_obj: PyObjectRef = name.clone().into(); + + // Update bases. + let mut new_bases: Option<Vec<PyObjectRef>> = None; + let bases = PyTuple::new_ref(bases.into_vec(), &vm.ctx); + for (i, base) in bases.iter().enumerate() { + if base.fast_isinstance(vm.ctx.types.type_type) { + if let Some(bases) = &mut new_bases { + bases.push(base.clone()); + } + continue; + } + let mro_entries = + vm.get_attribute_opt(base.clone(), identifier!(vm, __mro_entries__))?; + let entries = match mro_entries { + Some(meth) => meth.call((bases.clone(),), vm)?, + None => { + if let Some(bases) = &mut new_bases { + bases.push(base.clone()); + } + continue; + } + }; + let entries: PyTupleRef = entries + .downcast() + .map_err(|_| vm.new_type_error("__mro_entries__ must return a tuple"))?; + let new_bases = new_bases.get_or_insert_with(|| bases[..i].to_vec()); + new_bases.extend_from_slice(&entries); + } + + let new_bases = new_bases.map(|v| PyTuple::new_ref(v, &vm.ctx)); + let (orig_bases, bases) = match new_bases { + Some(new) => (Some(bases), new), + None => (None, bases), + }; + + // Use downcast_exact to keep ref to old object on error. + let metaclass = kwargs + .pop_kwarg("metaclass") + .map(|metaclass| { + metaclass + .downcast_exact::<PyType>(vm) + .map(|m| m.into_pyref()) + }) + .unwrap_or_else(|| { + // if there are no bases, use type; else get the type of the first base + Ok(if bases.is_empty() { + vm.ctx.types.type_type.to_owned() + } else { + bases.first().unwrap().class().to_owned() + }) + }); + + let (metaclass, meta_name) = match metaclass { + Ok(mut metaclass) => { + for base in bases.iter() { + let base_class = base.class(); + // if winner is subtype of tmptype, continue (winner is more derived) + if metaclass.fast_issubclass(base_class) { + continue; + } + // if tmptype is subtype of winner, update (tmptype is more derived) + if base_class.fast_issubclass(&metaclass) { + metaclass = base_class.to_owned(); + continue; + } + // Metaclass conflict + return Err(vm.new_type_error( + "metaclass conflict: the metaclass of a derived class must be a (non-strict) \ + subclass of the metaclasses of all its bases", + )); + } + let meta_name = metaclass.slot_name(); + (metaclass.to_owned().into(), meta_name.to_owned()) + } + Err(obj) => (obj, "<metaclass>".to_owned()), + }; + + let bases: PyObjectRef = bases.into(); + + // Prepare uses full __getattribute__ resolution chain. + let namespace = vm + .get_attribute_opt(metaclass.clone(), identifier!(vm, __prepare__))? + .map_or(Ok(vm.ctx.new_dict().into()), |prepare| { + let args = FuncArgs::new(vec![name_obj.clone(), bases.clone()], kwargs.clone()); + prepare.call(args, vm) + })?; + + // Accept any PyMapping as namespace. + let namespace = ArgMapping::try_from_object(vm, namespace.clone()).map_err(|_| { + vm.new_type_error(format!( + "{}.__prepare__() must return a mapping, not {}", + meta_name, + namespace.class() + )) + })?; + + // For PEP 695 classes, set .type_params in namespace before calling the function + if let Ok(type_params) = function + .as_object() + .get_attr(identifier!(vm, __type_params__), vm) + && let Some(type_params_tuple) = type_params.downcast_ref::<PyTuple>() + && !type_params_tuple.is_empty() + { + // Set .type_params in namespace so the compiler-generated code can use it + namespace + .as_object() + .set_item(vm.ctx.intern_str(".type_params"), type_params, vm)?; + } + + let classcell = function.invoke_with_locals(().into(), Some(namespace.clone()), vm)?; + let classcell = <Option<PyCellRef>>::try_from_object(vm, classcell)?; + + if let Some(orig_bases) = orig_bases { + namespace.as_object().set_item( + identifier!(vm, __orig_bases__), + orig_bases.into(), + vm, + )?; + } + + // Remove .type_params from namespace before creating the class + namespace + .as_object() + .del_item(vm.ctx.intern_str(".type_params"), vm) + .ok(); + + let args = FuncArgs::new(vec![name_obj, bases, namespace.into()], kwargs); + let class = metaclass.call(args, vm)?; + + // For PEP 695 classes, set __type_params__ on the class from the function + if let Ok(type_params) = function + .as_object() + .get_attr(identifier!(vm, __type_params__), vm) + && let Some(type_params_tuple) = type_params.downcast_ref::<PyTuple>() + && !type_params_tuple.is_empty() + { + class.set_attr(identifier!(vm, __type_params__), type_params.clone(), vm)?; + // Also set __parameters__ for compatibility with typing module + class.set_attr(identifier!(vm, __parameters__), type_params, vm)?; + } + + // only check cell if cls is a type and cell is a cell object + if let Some(ref classcell) = classcell + && class.fast_isinstance(vm.ctx.types.type_type) + { + let cell_value = classcell.get().ok_or_else(|| { + vm.new_runtime_error(format!( + "__class__ not set defining {:?} as {:?}. Was __classcell__ propagated to type.__new__?", + name, class + )) + })?; + + if !cell_value.is(&class) { + return Err(vm.new_type_error(format!( + "__class__ set to {:?} defining {:?} as {:?}", + cell_value, name, class + ))); + } + } + + Ok(class) + } +} + +pub fn init_module(vm: &VirtualMachine, module: &Py<PyModule>) { + let ctx = &vm.ctx; + + crate::protocol::VecBuffer::make_static_type(); + + module.__init_methods(vm).unwrap(); + builtins::module_exec(vm, module).unwrap(); + + let debug_mode: bool = vm.state.config.settings.optimize == 0; + // Create dynamic ExceptionGroup with multiple inheritance (BaseExceptionGroup + Exception) + let exception_group = crate::exception_group::exception_group(); + + extend_module!(vm, module, { + "__debug__" => ctx.new_bool(debug_mode), + + "bool" => ctx.types.bool_type.to_owned(), + "bytearray" => ctx.types.bytearray_type.to_owned(), + "bytes" => ctx.types.bytes_type.to_owned(), + "classmethod" => ctx.types.classmethod_type.to_owned(), + "complex" => ctx.types.complex_type.to_owned(), + "dict" => ctx.types.dict_type.to_owned(), + "enumerate" => ctx.types.enumerate_type.to_owned(), + "float" => ctx.types.float_type.to_owned(), + "frozenset" => ctx.types.frozenset_type.to_owned(), + "filter" => ctx.types.filter_type.to_owned(), + "int" => ctx.types.int_type.to_owned(), + "list" => ctx.types.list_type.to_owned(), + "map" => ctx.types.map_type.to_owned(), + "memoryview" => ctx.types.memoryview_type.to_owned(), + "object" => ctx.types.object_type.to_owned(), + "property" => ctx.types.property_type.to_owned(), + "range" => ctx.types.range_type.to_owned(), + "set" => ctx.types.set_type.to_owned(), + "slice" => ctx.types.slice_type.to_owned(), + "staticmethod" => ctx.types.staticmethod_type.to_owned(), + "str" => ctx.types.str_type.to_owned(), + "super" => ctx.types.super_type.to_owned(), + "tuple" => ctx.types.tuple_type.to_owned(), + "type" => ctx.types.type_type.to_owned(), + "zip" => ctx.types.zip_type.to_owned(), + + // Constants + "None" => ctx.none(), + "True" => ctx.new_bool(true), + "False" => ctx.new_bool(false), + "NotImplemented" => ctx.not_implemented(), + "Ellipsis" => vm.ctx.ellipsis.clone(), + + // ordered by exception_hierarchy.txt + // Exceptions: + "BaseException" => ctx.exceptions.base_exception_type.to_owned(), + "BaseExceptionGroup" => ctx.exceptions.base_exception_group.to_owned(), + "ExceptionGroup" => exception_group.to_owned(), + "SystemExit" => ctx.exceptions.system_exit.to_owned(), + "KeyboardInterrupt" => ctx.exceptions.keyboard_interrupt.to_owned(), + "GeneratorExit" => ctx.exceptions.generator_exit.to_owned(), + "Exception" => ctx.exceptions.exception_type.to_owned(), + "StopIteration" => ctx.exceptions.stop_iteration.to_owned(), + "StopAsyncIteration" => ctx.exceptions.stop_async_iteration.to_owned(), + "ArithmeticError" => ctx.exceptions.arithmetic_error.to_owned(), + "FloatingPointError" => ctx.exceptions.floating_point_error.to_owned(), + "OverflowError" => ctx.exceptions.overflow_error.to_owned(), + "ZeroDivisionError" => ctx.exceptions.zero_division_error.to_owned(), + "AssertionError" => ctx.exceptions.assertion_error.to_owned(), + "AttributeError" => ctx.exceptions.attribute_error.to_owned(), + "BufferError" => ctx.exceptions.buffer_error.to_owned(), + "EOFError" => ctx.exceptions.eof_error.to_owned(), + "ImportError" => ctx.exceptions.import_error.to_owned(), + "ModuleNotFoundError" => ctx.exceptions.module_not_found_error.to_owned(), + "LookupError" => ctx.exceptions.lookup_error.to_owned(), + "IndexError" => ctx.exceptions.index_error.to_owned(), + "KeyError" => ctx.exceptions.key_error.to_owned(), + "MemoryError" => ctx.exceptions.memory_error.to_owned(), + "NameError" => ctx.exceptions.name_error.to_owned(), + "UnboundLocalError" => ctx.exceptions.unbound_local_error.to_owned(), + "OSError" => ctx.exceptions.os_error.to_owned(), + // OSError alias + "IOError" => ctx.exceptions.os_error.to_owned(), + "EnvironmentError" => ctx.exceptions.os_error.to_owned(), + "BlockingIOError" => ctx.exceptions.blocking_io_error.to_owned(), + "ChildProcessError" => ctx.exceptions.child_process_error.to_owned(), + "ConnectionError" => ctx.exceptions.connection_error.to_owned(), + "BrokenPipeError" => ctx.exceptions.broken_pipe_error.to_owned(), + "ConnectionAbortedError" => ctx.exceptions.connection_aborted_error.to_owned(), + "ConnectionRefusedError" => ctx.exceptions.connection_refused_error.to_owned(), + "ConnectionResetError" => ctx.exceptions.connection_reset_error.to_owned(), + "FileExistsError" => ctx.exceptions.file_exists_error.to_owned(), + "FileNotFoundError" => ctx.exceptions.file_not_found_error.to_owned(), + "InterruptedError" => ctx.exceptions.interrupted_error.to_owned(), + "IsADirectoryError" => ctx.exceptions.is_a_directory_error.to_owned(), + "NotADirectoryError" => ctx.exceptions.not_a_directory_error.to_owned(), + "PermissionError" => ctx.exceptions.permission_error.to_owned(), + "ProcessLookupError" => ctx.exceptions.process_lookup_error.to_owned(), + "TimeoutError" => ctx.exceptions.timeout_error.to_owned(), + "ReferenceError" => ctx.exceptions.reference_error.to_owned(), + "RuntimeError" => ctx.exceptions.runtime_error.to_owned(), + "NotImplementedError" => ctx.exceptions.not_implemented_error.to_owned(), + "RecursionError" => ctx.exceptions.recursion_error.to_owned(), + "SyntaxError" => ctx.exceptions.syntax_error.to_owned(), + "_IncompleteInputError" => ctx.exceptions.incomplete_input_error.to_owned(), + "IndentationError" => ctx.exceptions.indentation_error.to_owned(), + "TabError" => ctx.exceptions.tab_error.to_owned(), + "SystemError" => ctx.exceptions.system_error.to_owned(), + "TypeError" => ctx.exceptions.type_error.to_owned(), + "ValueError" => ctx.exceptions.value_error.to_owned(), + "UnicodeError" => ctx.exceptions.unicode_error.to_owned(), + "UnicodeDecodeError" => ctx.exceptions.unicode_decode_error.to_owned(), + "UnicodeEncodeError" => ctx.exceptions.unicode_encode_error.to_owned(), + "UnicodeTranslateError" => ctx.exceptions.unicode_translate_error.to_owned(), + + // Warnings + "Warning" => ctx.exceptions.warning.to_owned(), + "DeprecationWarning" => ctx.exceptions.deprecation_warning.to_owned(), + "PendingDeprecationWarning" => ctx.exceptions.pending_deprecation_warning.to_owned(), + "RuntimeWarning" => ctx.exceptions.runtime_warning.to_owned(), + "SyntaxWarning" => ctx.exceptions.syntax_warning.to_owned(), + "UserWarning" => ctx.exceptions.user_warning.to_owned(), + "FutureWarning" => ctx.exceptions.future_warning.to_owned(), + "ImportWarning" => ctx.exceptions.import_warning.to_owned(), + "UnicodeWarning" => ctx.exceptions.unicode_warning.to_owned(), + "BytesWarning" => ctx.exceptions.bytes_warning.to_owned(), + "ResourceWarning" => ctx.exceptions.resource_warning.to_owned(), + "EncodingWarning" => ctx.exceptions.encoding_warning.to_owned(), + }); + + #[cfg(feature = "jit")] + extend_module!(vm, module, { + "JitError" => ctx.exceptions.jit_error.to_owned(), + }); + + #[cfg(windows)] + extend_module!(vm, module, { + // OSError alias for Windows + "WindowsError" => ctx.exceptions.os_error.to_owned(), + }); +} diff --git a/vm/src/stdlib/errno.rs b/crates/vm/src/stdlib/errno.rs similarity index 95% rename from vm/src/stdlib/errno.rs rename to crates/vm/src/stdlib/errno.rs index a142d68a346..8e4efbaafe9 100644 --- a/vm/src/stdlib/errno.rs +++ b/crates/vm/src/stdlib/errno.rs @@ -1,23 +1,26 @@ -use crate::{builtins::PyModule, PyRef, VirtualMachine}; +// spell-checker:disable -#[pymodule] -mod errno {} +pub(crate) use errno_mod::module_def; -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = errno::make_module(vm); - let errorcode = vm.ctx.new_dict(); - extend_module!(vm, &module, { - "errorcode" => errorcode.clone(), - }); - for (name, code) in ERROR_CODES { - let name = vm.ctx.intern_str(*name); - let code = vm.new_pyobj(*code); - errorcode - .set_item(&*code, name.to_owned().into(), vm) - .unwrap(); - module.set_attr(name, code, vm).unwrap(); +#[pymodule(name = "errno")] +mod errno_mod { + use crate::{Py, PyResult, VirtualMachine, builtins::PyModule}; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + + let errorcode = vm.ctx.new_dict(); + extend_module!(vm, module, { + "errorcode" => errorcode.clone(), + }); + for (name, code) in super::ERROR_CODES { + let name = vm.ctx.intern_str(*name); + let code = vm.new_pyobj(*code); + errorcode.set_item(&*code, name.to_owned().into(), vm)?; + module.set_attr(name, code, vm)?; + } + Ok(()) } - module } #[cfg(any(unix, windows, target_os = "wasi"))] @@ -38,9 +41,9 @@ pub mod errors { WSAEPROVIDERFAILEDINIT, WSAEREFUSED, WSAEREMOTE, WSAESHUTDOWN, WSAESOCKTNOSUPPORT, WSAESTALE, WSAETIMEDOUT, WSAETOOMANYREFS, WSAEUSERS, WSAEWOULDBLOCK, WSAHOST_NOT_FOUND, WSAID_ACCEPTEX, WSAID_CONNECTEX, WSAID_DISCONNECTEX, WSAID_GETACCEPTEXSOCKADDRS, - WSAID_TRANSMITFILE, WSAID_TRANSMITPACKETS, WSAID_WSAPOLL, WSAID_WSARECVMSG, - WSANOTINITIALISED, WSANO_DATA, WSANO_RECOVERY, WSAPROTOCOL_LEN, WSASERVICE_NOT_FOUND, - WSASYSCALLFAILURE, WSASYSNOTREADY, WSASYS_STATUS_LEN, WSATRY_AGAIN, WSATYPE_NOT_FOUND, + WSAID_TRANSMITFILE, WSAID_TRANSMITPACKETS, WSAID_WSAPOLL, WSAID_WSARECVMSG, WSANO_DATA, + WSANO_RECOVERY, WSANOTINITIALISED, WSAPROTOCOL_LEN, WSASERVICE_NOT_FOUND, + WSASYS_STATUS_LEN, WSASYSCALLFAILURE, WSASYSNOTREADY, WSATRY_AGAIN, WSATYPE_NOT_FOUND, WSAVERNOTSUPPORTED, }, }; diff --git a/crates/vm/src/stdlib/gc.rs b/crates/vm/src/stdlib/gc.rs new file mode 100644 index 00000000000..3909186b5c0 --- /dev/null +++ b/crates/vm/src/stdlib/gc.rs @@ -0,0 +1,305 @@ +pub(crate) use gc::module_def; + +#[pymodule] +mod gc { + use crate::{ + PyObjectRef, PyResult, VirtualMachine, + builtins::PyListRef, + function::{FuncArgs, OptionalArg}, + gc_state, + }; + + // Debug flag constants + #[pyattr] + const DEBUG_STATS: u32 = gc_state::GcDebugFlags::STATS.bits(); + #[pyattr] + const DEBUG_COLLECTABLE: u32 = gc_state::GcDebugFlags::COLLECTABLE.bits(); + #[pyattr] + const DEBUG_UNCOLLECTABLE: u32 = gc_state::GcDebugFlags::UNCOLLECTABLE.bits(); + #[pyattr] + const DEBUG_SAVEALL: u32 = gc_state::GcDebugFlags::SAVEALL.bits(); + #[pyattr] + const DEBUG_LEAK: u32 = gc_state::GcDebugFlags::LEAK.bits(); + + /// Enable automatic garbage collection. + #[pyfunction] + fn enable() { + gc_state::gc_state().enable(); + } + + /// Disable automatic garbage collection. + #[pyfunction] + fn disable() { + gc_state::gc_state().disable(); + } + + /// Return true if automatic gc is enabled. + #[pyfunction] + fn isenabled() -> bool { + gc_state::gc_state().is_enabled() + } + + /// Run a garbage collection. Returns the number of unreachable objects found. + #[derive(FromArgs)] + struct CollectArgs { + #[pyarg(any, optional)] + generation: OptionalArg<i32>, + } + + #[pyfunction] + fn collect(args: CollectArgs, vm: &VirtualMachine) -> PyResult<i32> { + let generation = args.generation; + let generation_num = generation.unwrap_or(2); + if !(0..=2).contains(&generation_num) { + return Err(vm.new_value_error("invalid generation")); + } + + // Invoke callbacks with "start" phase + invoke_callbacks(vm, "start", generation_num as usize, &Default::default()); + + // Manual gc.collect() should run even if GC is disabled + let gc = gc_state::gc_state(); + let result = gc.collect_force(generation_num as usize); + + // Move objects from gc_state.garbage to vm.ctx.gc_garbage (for DEBUG_SAVEALL) + { + let mut state_garbage = gc.garbage.lock(); + if !state_garbage.is_empty() { + let py_garbage = &vm.ctx.gc_garbage; + let mut garbage_vec = py_garbage.borrow_vec_mut(); + for obj in state_garbage.drain(..) { + garbage_vec.push(obj); + } + } + } + + // Invoke callbacks with "stop" phase + invoke_callbacks(vm, "stop", generation_num as usize, &result); + + Ok((result.collected + result.uncollectable) as i32) + } + + /// Return the current collection thresholds as a tuple. + #[pyfunction] + fn get_threshold(vm: &VirtualMachine) -> PyObjectRef { + let (t0, t1, t2) = gc_state::gc_state().get_threshold(); + vm.ctx + .new_tuple(vec![ + vm.ctx.new_int(t0).into(), + vm.ctx.new_int(t1).into(), + vm.ctx.new_int(t2).into(), + ]) + .into() + } + + /// Set the collection thresholds. + #[pyfunction] + fn set_threshold(threshold0: u32, threshold1: OptionalArg<u32>, threshold2: OptionalArg<u32>) { + gc_state::gc_state().set_threshold( + threshold0, + threshold1.into_option(), + threshold2.into_option(), + ); + } + + /// Return the current collection counts as a tuple. + #[pyfunction] + fn get_count(vm: &VirtualMachine) -> PyObjectRef { + let (c0, c1, c2) = gc_state::gc_state().get_count(); + vm.ctx + .new_tuple(vec![ + vm.ctx.new_int(c0).into(), + vm.ctx.new_int(c1).into(), + vm.ctx.new_int(c2).into(), + ]) + .into() + } + + /// Return the current debugging flags. + #[pyfunction] + fn get_debug() -> u32 { + gc_state::gc_state().get_debug().bits() + } + + /// Set the debugging flags. + #[pyfunction] + fn set_debug(flags: u32) { + gc_state::gc_state().set_debug(gc_state::GcDebugFlags::from_bits_truncate(flags)); + } + + /// Return a list of per-generation gc stats. + #[pyfunction] + fn get_stats(vm: &VirtualMachine) -> PyResult<PyListRef> { + let stats = gc_state::gc_state().get_stats(); + let mut result = Vec::with_capacity(3); + + for stat in stats.iter() { + let dict = vm.ctx.new_dict(); + dict.set_item("collections", vm.ctx.new_int(stat.collections).into(), vm)?; + dict.set_item("collected", vm.ctx.new_int(stat.collected).into(), vm)?; + dict.set_item( + "uncollectable", + vm.ctx.new_int(stat.uncollectable).into(), + vm, + )?; + dict.set_item("candidates", vm.ctx.new_int(stat.candidates).into(), vm)?; + dict.set_item("duration", vm.ctx.new_float(stat.duration).into(), vm)?; + result.push(dict.into()); + } + + Ok(vm.ctx.new_list(result)) + } + + /// Return the list of objects tracked by the collector. + #[derive(FromArgs)] + struct GetObjectsArgs { + #[pyarg(any, optional)] + generation: OptionalArg<Option<i32>>, + } + + #[pyfunction] + fn get_objects(args: GetObjectsArgs, vm: &VirtualMachine) -> PyResult<PyListRef> { + let generation_opt = args.generation.flatten(); + if let Some(g) = generation_opt + && !(0..=2).contains(&g) + { + return Err(vm.new_value_error(format!("generation must be in range(0, 3), not {}", g))); + } + let objects = gc_state::gc_state().get_objects(generation_opt); + Ok(vm.ctx.new_list(objects)) + } + + /// Return the list of objects directly referred to by any of the arguments. + #[pyfunction] + fn get_referents(args: FuncArgs, vm: &VirtualMachine) -> PyListRef { + let mut result = Vec::new(); + + for obj in args.args { + // Use the gc_get_referents method to get references + result.extend(obj.gc_get_referents()); + } + + vm.ctx.new_list(result) + } + + /// Return the list of objects that directly refer to any of the arguments. + #[pyfunction] + fn get_referrers(args: FuncArgs, vm: &VirtualMachine) -> PyListRef { + use std::collections::HashSet; + + // Build a set of target object pointers for fast lookup + let targets: HashSet<usize> = args + .args + .iter() + .map(|obj| obj.as_ref() as *const crate::PyObject as usize) + .collect(); + + // Collect pointers of frames currently on the execution stack. + // In CPython, executing frames (_PyInterpreterFrame) are not GC-tracked + // PyObjects, so they never appear in get_referrers results. Since + // RustPython materializes every frame as a PyObject, we must exclude + // them manually to match the expected behavior. + let stack_frames: HashSet<usize> = vm + .frames + .borrow() + .iter() + .map(|fp| { + let frame: &crate::PyObject = unsafe { fp.as_ref() }.as_ref(); + frame as *const crate::PyObject as usize + }) + .collect(); + + let mut result = Vec::new(); + + // Scan all tracked objects across all generations + let all_objects = gc_state::gc_state().get_objects(None); + for obj in all_objects { + let obj_ptr = obj.as_ref() as *const crate::PyObject as usize; + if stack_frames.contains(&obj_ptr) { + continue; + } + let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; + for child_ptr in referent_ptrs { + if targets.contains(&(child_ptr.as_ptr() as usize)) { + result.push(obj.clone()); + break; + } + } + } + + vm.ctx.new_list(result) + } + + /// Return True if the object is tracked by the garbage collector. + #[pyfunction] + fn is_tracked(obj: PyObjectRef) -> bool { + // An object is tracked if it has IS_TRACE = true (has a trace function) + obj.is_gc_tracked() + } + + /// Return True if the object has been finalized by the garbage collector. + #[pyfunction] + fn is_finalized(obj: PyObjectRef) -> bool { + obj.gc_finalized() + } + + /// Freeze all objects tracked by gc. + #[pyfunction] + fn freeze() { + gc_state::gc_state().freeze(); + } + + /// Unfreeze all objects in the permanent generation. + #[pyfunction] + fn unfreeze() { + gc_state::gc_state().unfreeze(); + } + + /// Return the number of objects in the permanent generation. + #[pyfunction] + fn get_freeze_count() -> usize { + gc_state::gc_state().get_freeze_count() + } + + /// gc.garbage - list of uncollectable objects + #[pyattr] + fn garbage(vm: &VirtualMachine) -> PyListRef { + vm.ctx.gc_garbage.clone() + } + + /// gc.callbacks - list of callbacks to be invoked + #[pyattr] + fn callbacks(vm: &VirtualMachine) -> PyListRef { + vm.ctx.gc_callbacks.clone() + } + + /// Helper function to invoke GC callbacks + fn invoke_callbacks( + vm: &VirtualMachine, + phase: &str, + generation: usize, + result: &gc_state::CollectResult, + ) { + let callbacks_list = &vm.ctx.gc_callbacks; + let callbacks: Vec<PyObjectRef> = callbacks_list.borrow_vec().to_vec(); + if callbacks.is_empty() { + return; + } + + let phase_str: PyObjectRef = vm.ctx.new_str(phase).into(); + let info = vm.ctx.new_dict(); + let _ = info.set_item("generation", vm.ctx.new_int(generation).into(), vm); + let _ = info.set_item("collected", vm.ctx.new_int(result.collected).into(), vm); + let _ = info.set_item( + "uncollectable", + vm.ctx.new_int(result.uncollectable).into(), + vm, + ); + let _ = info.set_item("candidates", vm.ctx.new_int(result.candidates).into(), vm); + let _ = info.set_item("duration", vm.ctx.new_float(result.duration).into(), vm); + + for callback in callbacks { + let _ = callback.call((phase_str.clone(), info.clone()), vm); + } + } +} diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs new file mode 100644 index 00000000000..763c3ddce76 --- /dev/null +++ b/crates/vm/src/stdlib/itertools.rs @@ -0,0 +1,1699 @@ +pub(crate) use decl::module_def; + +#[pymodule(name = "itertools")] +mod decl { + use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, VirtualMachine, + builtins::{PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyType, PyTypeRef, int}, + common::{ + lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, + rc::PyRc, + }, + convert::ToPyObject, + function::{ArgCallable, FuncArgs, OptionalArg, OptionalOption, PosArgs}, + protocol::{PyIter, PyIterReturn, PyNumber}, + raise_if_stop, + stdlib::sys, + types::{Constructor, IterNext, Iterable, Representable, SelfIter}, + }; + use core::sync::atomic::{AtomicBool, Ordering}; + use crossbeam_utils::atomic::AtomicCell; + use malachite_bigint::BigInt; + use num_traits::One; + use rustpython_common::wtf8::Wtf8Buf; + + use alloc::fmt; + use num_traits::{Signed, ToPrimitive}; + + #[pyattr] + #[pyclass(name = "chain")] + #[derive(Debug, PyPayload)] + struct PyItertoolsChain { + source: PyRwLock<Option<PyIter>>, + active: PyRwLock<Option<PyIter>>, + } + + #[pyclass(with(IterNext, Iterable), flags(BASETYPE, HAS_DICT))] + impl PyItertoolsChain { + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let args_list = PyList::from(args.args); + Self { + source: PyRwLock::new(Some(args_list.to_pyobject(vm).get_iter(vm)?)), + active: PyRwLock::new(None), + } + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + #[pyclassmethod] + fn from_iterable( + cls: PyTypeRef, + source: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyRef<Self>> { + Self { + source: PyRwLock::new(Some(source.get_iter(vm)?)), + active: PyRwLock::new(None), + } + .into_ref_with_type(vm, cls) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + } + + impl SelfIter for PyItertoolsChain {} + + impl IterNext for PyItertoolsChain { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let Some(source) = zelf.source.read().clone() else { + return Ok(PyIterReturn::StopIteration(None)); + }; + let next = loop { + let maybe_active = zelf.active.read().clone(); + if let Some(active) = maybe_active { + match active.next(vm) { + Ok(PyIterReturn::Return(ok)) => { + break Ok(PyIterReturn::Return(ok)); + } + Ok(PyIterReturn::StopIteration(_)) => { + *zelf.active.write() = None; + } + Err(err) => { + break Err(err); + } + } + } else { + match source.next(vm) { + Ok(PyIterReturn::Return(ok)) => match ok.get_iter(vm) { + Ok(iter) => { + *zelf.active.write() = Some(iter); + } + Err(err) => { + break Err(err); + } + }, + Ok(PyIterReturn::StopIteration(_)) => { + break Ok(PyIterReturn::StopIteration(None)); + } + Err(err) => { + break Err(err); + } + } + } + }; + match next { + Err(_) | Ok(PyIterReturn::StopIteration(_)) => { + *zelf.source.write() = None; + } + _ => {} + }; + next + } + } + + #[pyattr] + #[pyclass(name = "compress")] + #[derive(Debug, PyPayload)] + struct PyItertoolsCompress { + data: PyIter, + selectors: PyIter, + } + + #[derive(FromArgs)] + struct CompressNewArgs { + #[pyarg(any)] + data: PyIter, + #[pyarg(any)] + selectors: PyIter, + } + + impl Constructor for PyItertoolsCompress { + type Args = CompressNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { data, selectors }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { data, selectors }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] + impl PyItertoolsCompress {} + + impl SelfIter for PyItertoolsCompress {} + + impl IterNext for PyItertoolsCompress { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + loop { + let sel_obj = raise_if_stop!(zelf.selectors.next(vm)?); + let verdict = sel_obj.clone().try_to_bool(vm)?; + let data_obj = zelf.data.next(vm)?; + + if verdict { + return Ok(data_obj); + } + } + } + } + + #[pyattr] + #[pyclass(name = "count")] + #[derive(Debug, PyPayload)] + struct PyItertoolsCount { + cur: PyRwLock<PyObjectRef>, + step: PyObjectRef, + } + + #[derive(FromArgs)] + struct CountNewArgs { + #[pyarg(any, optional)] + start: OptionalArg<PyObjectRef>, + + #[pyarg(any, optional)] + step: OptionalArg<PyObjectRef>, + } + + impl Constructor for PyItertoolsCount { + type Args = CountNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { start, step }: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let start = start.into_option().unwrap_or_else(|| vm.new_pyobj(0)); + let step = step.into_option().unwrap_or_else(|| vm.new_pyobj(1)); + if !PyNumber::check(&start) || !PyNumber::check(&step) { + return Err(vm.new_type_error("a number is required")); + } + + Ok(Self { + cur: PyRwLock::new(start), + step, + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor, Representable))] + impl PyItertoolsCount {} + + impl SelfIter for PyItertoolsCount {} + + impl IterNext for PyItertoolsCount { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let mut cur = zelf.cur.write(); + let step = zelf.step.clone(); + let result = cur.clone(); + *cur = vm._iadd(&cur, step.as_object())?; + Ok(PyIterReturn::Return(result.to_pyobject(vm))) + } + } + + impl Representable for PyItertoolsCount { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let cur_repr = zelf.cur.read().clone().repr(vm)?; + let step = &zelf.step; + let mut result = Wtf8Buf::from("count("); + result.push_wtf8(cur_repr.as_wtf8()); + if !vm.bool_eq(step, vm.ctx.new_int(1).as_object())? { + result.push_str(", "); + result.push_wtf8(step.repr(vm)?.as_wtf8()); + } + result.push_char(')'); + Ok(result) + } + } + + #[pyattr] + #[pyclass(name = "cycle")] + #[derive(Debug, PyPayload)] + struct PyItertoolsCycle { + iter: PyIter, + saved: PyRwLock<Vec<PyObjectRef>>, + index: AtomicCell<usize>, + } + + impl Constructor for PyItertoolsCycle { + type Args = PyIter; + + fn py_new(_cls: &Py<PyType>, iter: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + iter, + saved: PyRwLock::new(Vec::new()), + index: AtomicCell::new(0), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] + impl PyItertoolsCycle {} + + impl SelfIter for PyItertoolsCycle {} + + impl IterNext for PyItertoolsCycle { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let item = if let PyIterReturn::Return(item) = zelf.iter.next(vm)? { + zelf.saved.write().push(item.clone()); + item + } else { + let saved = zelf.saved.read(); + if saved.is_empty() { + return Ok(PyIterReturn::StopIteration(None)); + } + + let last_index = zelf.index.fetch_add(1); + + if last_index >= saved.len() - 1 { + zelf.index.store(0); + } + + saved[last_index].clone() + }; + + Ok(PyIterReturn::Return(item)) + } + } + + #[pyattr] + #[pyclass(name = "repeat")] + #[derive(Debug, PyPayload)] + struct PyItertoolsRepeat { + object: PyObjectRef, + times: Option<PyRwLock<usize>>, + } + + #[derive(FromArgs)] + struct PyRepeatNewArgs { + object: PyObjectRef, + #[pyarg(any, optional)] + times: OptionalArg<PyObjectRef>, + } + + impl Constructor for PyItertoolsRepeat { + type Args = PyRepeatNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { object, times }: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let times = match times.into_option() { + Some(obj) => { + let int = obj.try_index(vm)?; + let val: isize = int.try_to_primitive(vm)?; + // times always >= 0. + Some(PyRwLock::new(val.to_usize().unwrap_or(0))) + } + None => None, + }; + Ok(Self { object, times }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor, Representable), flags(BASETYPE))] + impl PyItertoolsRepeat { + #[pymethod] + fn __length_hint__(&self, vm: &VirtualMachine) -> PyResult<usize> { + // Return TypeError, length_hint picks this up and returns the default. + let times = self + .times + .as_ref() + .ok_or_else(|| vm.new_type_error("length of unsized object."))?; + Ok(*times.read()) + } + } + + impl SelfIter for PyItertoolsRepeat {} + + impl IterNext for PyItertoolsRepeat { + fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { + if let Some(ref times) = zelf.times { + let mut times = times.write(); + if *times == 0 { + return Ok(PyIterReturn::StopIteration(None)); + } + *times -= 1; + } + Ok(PyIterReturn::Return(zelf.object.clone())) + } + } + + impl Representable for PyItertoolsRepeat { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let mut result = Wtf8Buf::from("repeat("); + result.push_wtf8(zelf.object.repr(vm)?.as_wtf8()); + if let Some(ref times) = zelf.times { + result.push_str(", "); + result.push_str(&times.read().to_string()); + } + result.push_char(')'); + Ok(result) + } + } + + #[pyattr] + #[pyclass(name = "starmap")] + #[derive(Debug, PyPayload)] + struct PyItertoolsStarmap { + function: PyObjectRef, + iterable: PyIter, + } + + #[derive(FromArgs)] + struct StarmapNewArgs { + #[pyarg(positional)] + function: PyObjectRef, + #[pyarg(positional)] + iterable: PyIter, + } + + impl Constructor for PyItertoolsStarmap { + type Args = StarmapNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { function, iterable }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { function, iterable }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] + impl PyItertoolsStarmap {} + + impl SelfIter for PyItertoolsStarmap {} + + impl IterNext for PyItertoolsStarmap { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let obj = zelf.iterable.next(vm)?; + let function = &zelf.function; + match obj { + PyIterReturn::Return(obj) => { + let args: Vec<_> = obj.try_to_value(vm)?; + PyIterReturn::from_pyresult(function.call(args, vm), vm) + } + PyIterReturn::StopIteration(v) => Ok(PyIterReturn::StopIteration(v)), + } + } + } + + #[pyattr] + #[pyclass(name = "takewhile")] + #[derive(Debug, PyPayload)] + struct PyItertoolsTakewhile { + predicate: PyObjectRef, + iterable: PyIter, + stop_flag: AtomicCell<bool>, + } + + #[derive(FromArgs)] + struct TakewhileNewArgs { + #[pyarg(positional)] + predicate: PyObjectRef, + #[pyarg(positional)] + iterable: PyIter, + } + + impl Constructor for PyItertoolsTakewhile { + type Args = TakewhileNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { + predicate, + iterable, + }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + predicate, + iterable, + stop_flag: AtomicCell::new(false), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] + impl PyItertoolsTakewhile {} + + impl SelfIter for PyItertoolsTakewhile {} + + impl IterNext for PyItertoolsTakewhile { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + if zelf.stop_flag.load() { + return Ok(PyIterReturn::StopIteration(None)); + } + + // might be StopIteration or anything else, which is propagated upwards + let obj = raise_if_stop!(zelf.iterable.next(vm)?); + let predicate = &zelf.predicate; + + let verdict = predicate.call((obj.clone(),), vm)?; + let verdict = verdict.try_to_bool(vm)?; + if verdict { + Ok(PyIterReturn::Return(obj)) + } else { + zelf.stop_flag.store(true); + Ok(PyIterReturn::StopIteration(None)) + } + } + } + + #[pyattr] + #[pyclass(name = "dropwhile")] + #[derive(Debug, PyPayload)] + struct PyItertoolsDropwhile { + predicate: ArgCallable, + iterable: PyIter, + start_flag: AtomicCell<bool>, + } + + #[derive(FromArgs)] + struct DropwhileNewArgs { + #[pyarg(positional)] + predicate: ArgCallable, + #[pyarg(positional)] + iterable: PyIter, + } + + impl Constructor for PyItertoolsDropwhile { + type Args = DropwhileNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { + predicate, + iterable, + }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + predicate, + iterable, + start_flag: AtomicCell::new(false), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] + impl PyItertoolsDropwhile {} + + impl SelfIter for PyItertoolsDropwhile {} + + impl IterNext for PyItertoolsDropwhile { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let predicate = &zelf.predicate; + let iterable = &zelf.iterable; + + if !zelf.start_flag.load() { + loop { + let obj = raise_if_stop!(iterable.next(vm)?); + let pred = predicate.clone(); + let pred_value = pred.invoke((obj.clone(),), vm)?; + if !pred_value.try_to_bool(vm)? { + zelf.start_flag.store(true); + return Ok(PyIterReturn::Return(obj)); + } + } + } + iterable.next(vm) + } + } + + #[derive(Default)] + struct GroupByState { + current_value: Option<PyObjectRef>, + current_key: Option<PyObjectRef>, + next_group: bool, + grouper: Option<PyWeakRef<PyItertoolsGrouper>>, + } + + impl fmt::Debug for GroupByState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GroupByState") + .field("current_value", &self.current_value) + .field("current_key", &self.current_key) + .field("next_group", &self.next_group) + .finish() + } + } + + impl GroupByState { + fn is_current(&self, grouper: &Py<PyItertoolsGrouper>) -> bool { + self.grouper + .as_ref() + .and_then(|g| g.upgrade()) + .is_some_and(|current_grouper| grouper.is(&current_grouper)) + } + } + + #[pyattr] + #[pyclass(name = "groupby")] + #[derive(PyPayload)] + struct PyItertoolsGroupBy { + iterable: PyIter, + key_func: Option<PyObjectRef>, + state: PyMutex<GroupByState>, + } + + impl fmt::Debug for PyItertoolsGroupBy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PyItertoolsGroupBy") + .field("iterable", &self.iterable) + .field("key_func", &self.key_func) + .field("state", &self.state.lock()) + .finish() + } + } + + #[derive(FromArgs)] + struct GroupByArgs { + iterable: PyIter, + #[pyarg(any, optional)] + key: OptionalOption<PyObjectRef>, + } + + impl Constructor for PyItertoolsGroupBy { + type Args = GroupByArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { iterable, key }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + iterable, + key_func: key.flatten(), + state: PyMutex::new(GroupByState::default()), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsGroupBy { + pub(super) fn advance( + &self, + vm: &VirtualMachine, + ) -> PyResult<PyIterReturn<(PyObjectRef, PyObjectRef)>> { + let new_value = raise_if_stop!(self.iterable.next(vm)?); + let new_key = if let Some(ref kf) = self.key_func { + kf.call((new_value.clone(),), vm)? + } else { + new_value.clone() + }; + Ok(PyIterReturn::Return((new_value, new_key))) + } + } + + impl SelfIter for PyItertoolsGroupBy {} + + impl IterNext for PyItertoolsGroupBy { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let mut state = zelf.state.lock(); + state.grouper = None; + + if !state.next_group { + // FIXME: unnecessary clone. current_key always exist until assigning new + let current_key = state.current_key.clone(); + drop(state); + + let (value, key) = if let Some(old_key) = current_key { + loop { + let (value, new_key) = raise_if_stop!(zelf.advance(vm)?); + if !vm.bool_eq(&new_key, &old_key)? { + break (value, new_key); + } + } + } else { + raise_if_stop!(zelf.advance(vm)?) + }; + + state = zelf.state.lock(); + state.current_value = Some(value); + state.current_key = Some(key); + } + + state.next_group = false; + + let grouper = PyItertoolsGrouper { + groupby: zelf.to_owned(), + } + .into_ref(&vm.ctx); + + state.grouper = Some(grouper.downgrade(None, vm).unwrap()); + Ok(PyIterReturn::Return( + (state.current_key.as_ref().unwrap().clone(), grouper).to_pyobject(vm), + )) + } + } + + #[pyattr] + #[pyclass(name = "_grouper")] + #[derive(Debug, PyPayload)] + struct PyItertoolsGrouper { + groupby: PyRef<PyItertoolsGroupBy>, + } + + #[pyclass(with(IterNext, Iterable), flags(HAS_WEAKREF))] + impl PyItertoolsGrouper {} + + impl SelfIter for PyItertoolsGrouper {} + + impl IterNext for PyItertoolsGrouper { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let old_key = { + let mut state = zelf.groupby.state.lock(); + + if !state.is_current(zelf) { + return Ok(PyIterReturn::StopIteration(None)); + } + + // check to see if the value has already been retrieved from the iterator + if let Some(val) = state.current_value.take() { + return Ok(PyIterReturn::Return(val)); + } + + state.current_key.as_ref().unwrap().clone() + }; + let (value, key) = raise_if_stop!(zelf.groupby.advance(vm)?); + if vm.bool_eq(&key, &old_key)? { + Ok(PyIterReturn::Return(value)) + } else { + let mut state = zelf.groupby.state.lock(); + state.current_value = Some(value); + state.current_key = Some(key); + state.next_group = true; + state.grouper = None; + Ok(PyIterReturn::StopIteration(None)) + } + } + } + + #[pyattr] + #[pyclass(name = "islice")] + #[derive(Debug, PyPayload)] + struct PyItertoolsIslice { + iterable: PyIter, + cur: AtomicCell<usize>, + next: AtomicCell<usize>, + stop: Option<usize>, + step: usize, + } + + // Restrict obj to ints with value 0 <= val <= sys.maxsize + // On failure (out of range, non-int object) a ValueError is raised. + fn pyobject_to_opt_usize( + obj: PyObjectRef, + name: &'static str, + vm: &VirtualMachine, + ) -> PyResult<usize> { + let is_int = obj.fast_isinstance(vm.ctx.types.int_type); + if is_int { + let value = int::get_value(&obj).to_usize(); + if let Some(value) = value { + // Only succeeds for values for which 0 <= value <= sys.maxsize + if value <= sys::MAXSIZE as usize { + return Ok(value); + } + } + } + // We don't have an int or value was < 0 or > sys.maxsize + Err(vm.new_value_error(format!( + "{name} argument for islice() must be None or an integer: 0 <= x <= sys.maxsize." + ))) + } + + #[pyclass(with(IterNext, Iterable), flags(BASETYPE))] + impl PyItertoolsIslice { + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let (iter, start, stop, step) = match args.args.len() { + 0 | 1 => { + return Err(vm.new_type_error(format!( + "islice expected at least 2 arguments, got {}", + args.args.len() + ))); + } + 2 => { + let (iter, stop): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + (iter, 0usize, stop, 1usize) + } + _ => { + let (iter, start, stop, step) = if args.args.len() == 3 { + let (iter, start, stop): (PyObjectRef, PyObjectRef, PyObjectRef) = + args.bind(vm)?; + (iter, start, stop, 1usize) + } else { + let (iter, start, stop, step): ( + PyObjectRef, + PyObjectRef, + PyObjectRef, + PyObjectRef, + ) = args.bind(vm)?; + + let step = if !vm.is_none(&step) { + pyobject_to_opt_usize(step, "Step", vm)? + } else { + 1usize + }; + (iter, start, stop, step) + }; + let start = if !vm.is_none(&start) { + pyobject_to_opt_usize(start, "Start", vm)? + } else { + 0usize + }; + + (iter, start, stop, step) + } + }; + + let stop = if !vm.is_none(&stop) { + Some(pyobject_to_opt_usize(stop, "Stop", vm)?) + } else { + None + }; + + let iter = iter.get_iter(vm)?; + + Self { + iterable: iter, + cur: AtomicCell::new(0), + next: AtomicCell::new(start), + stop, + step, + } + .into_ref_with_type(vm, cls) + .map(Into::into) + } + } + + impl SelfIter for PyItertoolsIslice {} + + impl IterNext for PyItertoolsIslice { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + while zelf.cur.load() < zelf.next.load() { + zelf.iterable.next(vm)?; + zelf.cur.fetch_add(1); + } + + if let Some(stop) = zelf.stop + && zelf.cur.load() >= stop + { + return Ok(PyIterReturn::StopIteration(None)); + } + + let obj = raise_if_stop!(zelf.iterable.next(vm)?); + zelf.cur.fetch_add(1); + + // TODO is this overflow check required? attempts to copy CPython. + let (next, ovf) = zelf.next.load().overflowing_add(zelf.step); + zelf.next.store(if ovf { zelf.stop.unwrap() } else { next }); + + Ok(PyIterReturn::Return(obj)) + } + } + + #[pyattr] + #[pyclass(name = "filterfalse")] + #[derive(Debug, PyPayload)] + struct PyItertoolsFilterFalse { + predicate: PyObjectRef, + iterable: PyIter, + } + + #[derive(FromArgs)] + struct FilterFalseNewArgs { + #[pyarg(positional)] + predicate: PyObjectRef, + #[pyarg(positional)] + iterable: PyIter, + } + + impl Constructor for PyItertoolsFilterFalse { + type Args = FilterFalseNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { + predicate, + iterable, + }: Self::Args, + _vm: &VirtualMachine, + ) -> PyResult<Self> { + Ok(Self { + predicate, + iterable, + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] + impl PyItertoolsFilterFalse {} + + impl SelfIter for PyItertoolsFilterFalse {} + + impl IterNext for PyItertoolsFilterFalse { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let predicate = &zelf.predicate; + let iterable = &zelf.iterable; + + loop { + let obj = raise_if_stop!(iterable.next(vm)?); + let pred_value = if vm.is_none(predicate) { + obj.clone() + } else { + predicate.call((obj.clone(),), vm)? + }; + + if !pred_value.try_to_bool(vm)? { + return Ok(PyIterReturn::Return(obj)); + } + } + } + } + + #[pyattr] + #[pyclass(name = "accumulate")] + #[derive(Debug, PyPayload)] + struct PyItertoolsAccumulate { + iterable: PyIter, + bin_op: Option<PyObjectRef>, + initial: Option<PyObjectRef>, + acc_value: PyRwLock<Option<PyObjectRef>>, + } + + #[derive(FromArgs)] + struct AccumulateArgs { + iterable: PyIter, + #[pyarg(any, optional)] + func: OptionalOption<PyObjectRef>, + #[pyarg(named, optional)] + initial: OptionalOption<PyObjectRef>, + } + + impl Constructor for PyItertoolsAccumulate { + type Args = AccumulateArgs; + + fn py_new(_cls: &Py<PyType>, args: AccumulateArgs, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + iterable: args.iterable, + bin_op: args.func.flatten(), + initial: args.initial.flatten(), + acc_value: PyRwLock::new(None), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsAccumulate {} + + impl SelfIter for PyItertoolsAccumulate {} + + impl IterNext for PyItertoolsAccumulate { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let iterable = &zelf.iterable; + + let acc_value = zelf.acc_value.read().clone(); + + let next_acc_value = match acc_value { + None => match &zelf.initial { + None => raise_if_stop!(iterable.next(vm)?), + Some(obj) => obj.clone(), + }, + Some(value) => { + let obj = raise_if_stop!(iterable.next(vm)?); + match &zelf.bin_op { + None => vm._add(&value, &obj)?, + Some(op) => op.call((value, obj), vm)?, + } + } + }; + *zelf.acc_value.write() = Some(next_acc_value.clone()); + + Ok(PyIterReturn::Return(next_acc_value)) + } + } + + #[derive(Debug)] + struct PyItertoolsTeeData { + iterable: PyIter, + values: PyMutex<Vec<PyObjectRef>>, + running: AtomicBool, + } + + impl PyItertoolsTeeData { + fn new(iterable: PyIter, _vm: &VirtualMachine) -> PyResult<PyRc<Self>> { + Ok(PyRc::new(Self { + iterable, + values: PyMutex::new(vec![]), + running: AtomicBool::new(false), + })) + } + + fn get_item(&self, vm: &VirtualMachine, index: usize) -> PyResult<PyIterReturn> { + // Return cached value if available + { + let Some(values) = self.values.try_lock() else { + return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); + }; + if index < values.len() { + return Ok(PyIterReturn::Return(values[index].clone())); + } + } + // Prevent concurrent/reentrant calls to iterable.next() + if self.running.swap(true, Ordering::Acquire) { + return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); + } + let result = self.iterable.next(vm); + self.running.store(false, Ordering::Release); + let obj = raise_if_stop!(result?); + let Some(mut values) = self.values.try_lock() else { + return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); + }; + if values.len() == index { + values.push(obj); + } + Ok(PyIterReturn::Return(values[index].clone())) + } + } + + #[pyattr] + #[pyclass(name = "tee")] + #[derive(Debug, PyPayload)] + struct PyItertoolsTee { + tee_data: PyRc<PyItertoolsTeeData>, + index: AtomicCell<usize>, + } + + #[derive(FromArgs)] + struct TeeNewArgs { + #[pyarg(positional)] + iterable: PyIter, + #[pyarg(positional, optional)] + n: OptionalArg<usize>, + } + + impl Constructor for PyItertoolsTee { + type Args = TeeNewArgs; + + // TODO: make tee() a function, rename this class to itertools._tee and make + // teedata a python class + fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let TeeNewArgs { iterable, n } = args.bind(vm)?; + let n = n.unwrap_or(2); + + let copyable = if iterable.class().has_attr(identifier!(vm, __copy__)) { + vm.call_special_method(iterable.as_object(), identifier!(vm, __copy__), ())? + } else { + Self::from_iter(iterable, vm)? + }; + + let mut tee_vec: Vec<PyObjectRef> = Vec::with_capacity(n); + for _ in 0..n { + tee_vec.push(vm.call_special_method(&copyable, identifier!(vm, __copy__), ())?); + } + + Ok(PyTuple::new_ref(tee_vec, &vm.ctx).into()) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsTee { + fn from_iter(iterator: PyIter, vm: &VirtualMachine) -> PyResult { + let class = Self::class(&vm.ctx); + if iterator.class().is(Self::class(&vm.ctx)) { + return vm.call_special_method(&iterator, identifier!(vm, __copy__), ()); + } + Ok(Self { + tee_data: PyItertoolsTeeData::new(iterator, vm)?, + index: AtomicCell::new(0), + } + .into_ref_with_type(vm, class.to_owned())? + .into()) + } + + #[pymethod] + fn __copy__(&self) -> Self { + Self { + tee_data: PyRc::clone(&self.tee_data), + index: AtomicCell::new(self.index.load()), + } + } + } + impl SelfIter for PyItertoolsTee {} + impl IterNext for PyItertoolsTee { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let value = raise_if_stop!(zelf.tee_data.get_item(vm, zelf.index.load())?); + zelf.index.fetch_add(1); + Ok(PyIterReturn::Return(value)) + } + } + + #[pyattr] + #[pyclass(name = "product")] + #[derive(Debug, PyPayload)] + struct PyItertoolsProduct { + pools: Vec<Vec<PyObjectRef>>, + idxs: PyRwLock<Vec<usize>>, + cur: AtomicCell<usize>, + stop: AtomicCell<bool>, + } + + #[derive(FromArgs)] + struct ProductArgs { + #[pyarg(named, optional)] + repeat: OptionalArg<usize>, + } + + impl Constructor for PyItertoolsProduct { + type Args = (PosArgs<PyObjectRef>, ProductArgs); + + fn py_new( + _cls: &Py<PyType>, + (iterables, args): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let repeat = args.repeat.unwrap_or(1); + let mut pools = Vec::new(); + for arg in iterables.iter() { + pools.push(arg.try_to_value(vm)?); + } + let pools = core::iter::repeat_n(pools, repeat) + .flatten() + .collect::<Vec<Vec<PyObjectRef>>>(); + + let l = pools.len(); + + Ok(Self { + pools, + idxs: PyRwLock::new(vec![0; l]), + cur: AtomicCell::new(l.wrapping_sub(1)), + stop: AtomicCell::new(false), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsProduct { + fn update_idxs(&self, mut idxs: PyRwLockWriteGuard<'_, Vec<usize>>) { + if idxs.is_empty() { + self.stop.store(true); + return; + } + + let cur = self.cur.load(); + let lst_idx = &self.pools[cur].len() - 1; + + if idxs[cur] == lst_idx { + if cur == 0 { + self.stop.store(true); + return; + } + idxs[cur] = 0; + self.cur.fetch_sub(1); + self.update_idxs(idxs); + } else { + idxs[cur] += 1; + self.cur.store(idxs.len() - 1); + } + } + } + + impl SelfIter for PyItertoolsProduct {} + impl IterNext for PyItertoolsProduct { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + // stop signal + if zelf.stop.load() { + return Ok(PyIterReturn::StopIteration(None)); + } + + let pools = &zelf.pools; + + for p in pools { + if p.is_empty() { + return Ok(PyIterReturn::StopIteration(None)); + } + } + + let idxs = zelf.idxs.write(); + let res = vm.ctx.new_tuple( + pools + .iter() + .zip(idxs.iter()) + .map(|(pool, idx)| pool[*idx].clone()) + .collect(), + ); + + zelf.update_idxs(idxs); + + Ok(PyIterReturn::Return(res.into())) + } + } + + #[pyattr] + #[pyclass(name = "combinations")] + #[derive(Debug, PyPayload)] + struct PyItertoolsCombinations { + pool: Vec<PyObjectRef>, + indices: PyRwLock<Vec<usize>>, + result: PyRwLock<Option<Vec<PyObjectRef>>>, + r: AtomicCell<usize>, + exhausted: AtomicCell<bool>, + } + + #[derive(FromArgs)] + struct CombinationsNewArgs { + #[pyarg(any)] + iterable: PyObjectRef, + #[pyarg(any)] + r: PyIntRef, + } + + impl Constructor for PyItertoolsCombinations { + type Args = CombinationsNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { iterable, r }: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let pool: Vec<_> = iterable.try_to_value(vm)?; + + let r = r.as_bigint(); + if r.is_negative() { + return Err(vm.new_value_error("r must be non-negative")); + } + let r = r.to_usize().unwrap(); + + let n = pool.len(); + + Ok(Self { + pool, + indices: PyRwLock::new((0..r).collect()), + result: PyRwLock::new(None), + r: AtomicCell::new(r), + exhausted: AtomicCell::new(r > n), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsCombinations {} + + impl SelfIter for PyItertoolsCombinations {} + impl IterNext for PyItertoolsCombinations { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + // stop signal + if zelf.exhausted.load() { + return Ok(PyIterReturn::StopIteration(None)); + } + + let n = zelf.pool.len(); + let r = zelf.r.load(); + + if r == 0 { + zelf.exhausted.store(true); + return Ok(PyIterReturn::Return(vm.new_tuple(()).into())); + } + + let mut result_lock = zelf.result.write(); + let result = if let Some(ref mut result) = *result_lock { + let mut indices = zelf.indices.write(); + + // Scan indices right-to-left until finding one that is not at its maximum (i + n - r). + let mut idx = r as isize - 1; + while idx >= 0 && indices[idx as usize] == idx as usize + n - r { + idx -= 1; + } + + // If no suitable index is found, then the indices are all at + // their maximum value and we're done. + if idx < 0 { + zelf.exhausted.store(true); + return Ok(PyIterReturn::StopIteration(None)); + } else { + // Increment the current index which we know is not at its + // maximum. Then move back to the right setting each index + // to its lowest possible value (one higher than the index + // to its left -- this maintains the sort order invariant). + indices[idx as usize] += 1; + for j in idx as usize + 1..r { + indices[j] = indices[j - 1] + 1; + } + + // Update the result tuple for the new indices + // starting with i, the leftmost index that changed + for i in idx as usize..r { + let index = indices[i]; + let elem = &zelf.pool[index]; + elem.clone_into(&mut result[i]); + } + + result.to_vec() + } + } else { + let res = zelf.pool[0..r].to_vec(); + *result_lock = Some(res.clone()); + res + }; + + Ok(PyIterReturn::Return(vm.ctx.new_tuple(result).into())) + } + } + + #[pyattr] + #[pyclass(name = "combinations_with_replacement")] + #[derive(Debug, PyPayload)] + struct PyItertoolsCombinationsWithReplacement { + pool: Vec<PyObjectRef>, + indices: PyRwLock<Vec<usize>>, + r: AtomicCell<usize>, + exhausted: AtomicCell<bool>, + } + + impl Constructor for PyItertoolsCombinationsWithReplacement { + type Args = CombinationsNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { iterable, r }: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let pool: Vec<_> = iterable.try_to_value(vm)?; + let r = r.as_bigint(); + if r.is_negative() { + return Err(vm.new_value_error("r must be non-negative")); + } + let r = r.to_usize().unwrap(); + + let n = pool.len(); + + Ok(Self { + pool, + indices: PyRwLock::new(vec![0; r]), + r: AtomicCell::new(r), + exhausted: AtomicCell::new(n == 0 && r > 0), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsCombinationsWithReplacement {} + + impl SelfIter for PyItertoolsCombinationsWithReplacement {} + + impl IterNext for PyItertoolsCombinationsWithReplacement { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + // stop signal + if zelf.exhausted.load() { + return Ok(PyIterReturn::StopIteration(None)); + } + + let n = zelf.pool.len(); + let r = zelf.r.load(); + + if r == 0 { + zelf.exhausted.store(true); + return Ok(PyIterReturn::Return(vm.new_tuple(()).into())); + } + + let mut indices = zelf.indices.write(); + + let res = vm + .ctx + .new_tuple(indices.iter().map(|&i| zelf.pool[i].clone()).collect()); + + // Scan indices right-to-left until finding one that is not at its maximum (i + n - r). + let mut idx = r as isize - 1; + while idx >= 0 && indices[idx as usize] == n - 1 { + idx -= 1; + } + + // If no suitable index is found, then the indices are all at + // their maximum value and we're done. + if idx < 0 { + zelf.exhausted.store(true); + } else { + let index = indices[idx as usize] + 1; + + // Increment the current index which we know is not at its + // maximum. Then set all to the right to the same value. + for j in idx as usize..r { + indices[j] = index; + } + } + + Ok(PyIterReturn::Return(res.into())) + } + } + + #[pyattr] + #[pyclass(name = "permutations")] + #[derive(Debug, PyPayload)] + struct PyItertoolsPermutations { + pool: Vec<PyObjectRef>, // Collected input iterable + indices: PyRwLock<Vec<usize>>, // One index per element in pool + cycles: PyRwLock<Vec<usize>>, // One rollover counter per element in the result + result: PyRwLock<Option<Vec<usize>>>, // Indexes of the most recently returned result + r: AtomicCell<usize>, // Size of result tuple + exhausted: AtomicCell<bool>, // Set when the iterator is exhausted + } + + #[derive(FromArgs)] + struct PermutationsNewArgs { + #[pyarg(positional)] + iterable: PyObjectRef, + #[pyarg(positional, optional)] + r: OptionalOption<PyObjectRef>, + } + + impl Constructor for PyItertoolsPermutations { + type Args = PermutationsNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { iterable, r }: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let pool: Vec<_> = iterable.try_to_value(vm)?; + + let n = pool.len(); + // If r is not provided, r == n. If provided, r must be a positive integer, or None. + // If None, it behaves the same as if it was not provided. + let r = match r.flatten() { + Some(r) => { + let val = r + .downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error("Expected int as r"))? + .as_bigint(); + + if val.is_negative() { + return Err(vm.new_value_error("r must be non-negative")); + } + val.to_usize().unwrap() + } + None => n, + }; + + Ok(Self { + pool, + indices: PyRwLock::new((0..n).collect()), + cycles: PyRwLock::new((0..r.min(n)).map(|i| n - i).collect()), + result: PyRwLock::new(None), + r: AtomicCell::new(r), + exhausted: AtomicCell::new(r > n), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsPermutations {} + + impl SelfIter for PyItertoolsPermutations {} + + impl IterNext for PyItertoolsPermutations { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + // stop signal + if zelf.exhausted.load() { + return Ok(PyIterReturn::StopIteration(None)); + } + + let n = zelf.pool.len(); + let r = zelf.r.load(); + + if n == 0 { + zelf.exhausted.store(true); + return Ok(PyIterReturn::Return(vm.new_tuple(()).into())); + } + + let mut result = zelf.result.write(); + + if let Some(ref mut result) = *result { + let mut indices = zelf.indices.write(); + let mut cycles = zelf.cycles.write(); + let mut sentinel = false; + + // Decrement rightmost cycle, moving leftward upon zero rollover + for i in (0..r).rev() { + cycles[i] -= 1; + + if cycles[i] == 0 { + // rotation: indices[i:] = indices[i+1:] + indices[i:i+1] + let index = indices[i]; + for j in i..n - 1 { + indices[j] = indices[j + 1]; + } + indices[n - 1] = index; + cycles[i] = n - i; + } else { + let j = cycles[i]; + indices.swap(i, n - j); + + for k in i..r { + // start with i, the leftmost element that changed + // yield tuple(pool[k] for k in indices[:r]) + result[k] = indices[k]; + } + sentinel = true; + break; + } + } + if !sentinel { + zelf.exhausted.store(true); + return Ok(PyIterReturn::StopIteration(None)); + } + } else { + // On the first pass, initialize result tuple using the indices + *result = Some((0..r).collect()); + } + + Ok(PyIterReturn::Return( + vm.ctx + .new_tuple( + result + .as_ref() + .unwrap() + .iter() + .map(|&i| zelf.pool[i].clone()) + .collect(), + ) + .into(), + )) + } + } + + #[derive(FromArgs)] + struct ZipLongestArgs { + #[pyarg(named, optional)] + fillvalue: OptionalArg<PyObjectRef>, + } + + impl Constructor for PyItertoolsZipLongest { + type Args = (PosArgs<PyIter>, ZipLongestArgs); + + fn py_new( + _cls: &Py<PyType>, + (iterators, args): Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let fillvalue = args.fillvalue.unwrap_or_none(vm); + let iterators = iterators.into_vec(); + Ok(Self { + iterators, + fillvalue: PyRwLock::new(fillvalue), + }) + } + } + + #[pyattr] + #[pyclass(name = "zip_longest")] + #[derive(Debug, PyPayload)] + struct PyItertoolsZipLongest { + iterators: Vec<PyIter>, + fillvalue: PyRwLock<PyObjectRef>, + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsZipLongest {} + + impl SelfIter for PyItertoolsZipLongest {} + + impl IterNext for PyItertoolsZipLongest { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + if zelf.iterators.is_empty() { + return Ok(PyIterReturn::StopIteration(None)); + } + let mut result: Vec<PyObjectRef> = Vec::new(); + let mut num_active = zelf.iterators.len(); + + for idx in 0..zelf.iterators.len() { + let next_obj = match zelf.iterators[idx].next(vm)? { + PyIterReturn::Return(obj) => obj, + PyIterReturn::StopIteration(v) => { + num_active -= 1; + if num_active == 0 { + return Ok(PyIterReturn::StopIteration(v)); + } + zelf.fillvalue.read().clone() + } + }; + result.push(next_obj); + } + Ok(PyIterReturn::Return(vm.ctx.new_tuple(result).into())) + } + } + + #[pyattr] + #[pyclass(name = "pairwise")] + #[derive(Debug, PyPayload)] + struct PyItertoolsPairwise { + iterator: PyIter, + old: PyRwLock<Option<PyObjectRef>>, + } + + impl Constructor for PyItertoolsPairwise { + type Args = PyIter; + + fn py_new(_cls: &Py<PyType>, iterator: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + Ok(Self { + iterator, + old: PyRwLock::new(None), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor))] + impl PyItertoolsPairwise {} + + impl SelfIter for PyItertoolsPairwise {} + + impl IterNext for PyItertoolsPairwise { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let old_clone = { + let guard = zelf.old.read(); + guard.clone() + }; + let old = match old_clone { + None => match zelf.iterator.next(vm)? { + PyIterReturn::Return(obj) => { + // Needed for when we reenter + *zelf.old.write() = Some(obj.clone()); + obj + } + PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), + }, + Some(obj) => obj, + }; + + let new = raise_if_stop!(zelf.iterator.next(vm)?); + *zelf.old.write() = Some(new.clone()); + + Ok(PyIterReturn::Return(vm.new_tuple((old, new)).into())) + } + } + + #[pyattr] + #[pyclass(name = "batched")] + #[derive(Debug, PyPayload)] + struct PyItertoolsBatched { + exhausted: AtomicCell<bool>, + iterable: PyIter, + n: AtomicCell<usize>, + strict: AtomicCell<bool>, + } + + #[derive(FromArgs)] + struct BatchedNewArgs { + #[pyarg(positional)] + iterable_ref: PyObjectRef, + #[pyarg(positional)] + n: PyIntRef, + #[pyarg(named, default = false)] + strict: bool, + } + + impl Constructor for PyItertoolsBatched { + type Args = BatchedNewArgs; + + fn py_new( + _cls: &Py<PyType>, + Self::Args { + iterable_ref, + n, + strict, + }: Self::Args, + vm: &VirtualMachine, + ) -> PyResult<Self> { + let n = n.as_bigint(); + if n.lt(&BigInt::one()) { + return Err(vm.new_value_error("n must be at least one")); + } + let n = n + .to_usize() + .ok_or(vm.new_overflow_error("Python int too large to convert to usize"))?; + let iterable = iterable_ref.get_iter(vm)?; + + Ok(Self { + iterable, + n: AtomicCell::new(n), + exhausted: AtomicCell::new(false), + strict: AtomicCell::new(strict), + }) + } + } + + #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE, HAS_DICT))] + impl PyItertoolsBatched {} + + impl SelfIter for PyItertoolsBatched {} + + impl IterNext for PyItertoolsBatched { + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + if zelf.exhausted.load() { + return Ok(PyIterReturn::StopIteration(None)); + } + let mut result: Vec<PyObjectRef> = Vec::new(); + let n = zelf.n.load(); + for _ in 0..n { + match zelf.iterable.next(vm)? { + PyIterReturn::Return(obj) => { + result.push(obj); + } + PyIterReturn::StopIteration(_) => { + zelf.exhausted.store(true); + break; + } + } + } + let res_len = result.len(); + match res_len { + 0 => Ok(PyIterReturn::StopIteration(None)), + _ => { + if zelf.strict.load() && res_len != n { + Err(vm.new_value_error("batched(): incomplete batch")) + } else { + Ok(PyIterReturn::Return(vm.ctx.new_tuple(result).into())) + } + } + } + } + } +} diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs new file mode 100644 index 00000000000..412d71f49e2 --- /dev/null +++ b/crates/vm/src/stdlib/marshal.rs @@ -0,0 +1,252 @@ +// spell-checker:ignore pyfrozen pycomplex +pub(crate) use decl::module_def; + +#[pymodule(name = "marshal")] +mod decl { + use crate::builtins::code::{CodeObject, Literal, PyObjBag}; + use crate::class::StaticType; + use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, + builtins::{ + PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat, + PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple, + }, + common::wtf8::Wtf8, + convert::ToPyObject, + function::{ArgBytesLike, OptionalArg}, + object::{AsObject, PyPayload}, + protocol::PyBuffer, + }; + use malachite_bigint::BigInt; + use num_complex::Complex64; + use num_traits::Zero; + use rustpython_compiler_core::marshal; + + #[pyattr(name = "version")] + use marshal::FORMAT_VERSION; + + pub struct DumpError; + + impl marshal::Dumpable for PyObjectRef { + type Error = DumpError; + type Constant = Literal; + + fn with_dump<R>( + &self, + f: impl FnOnce(marshal::DumpableValue<'_, Self>) -> R, + ) -> Result<R, Self::Error> { + use marshal::DumpableValue::*; + if self.is(PyStopIteration::static_type()) { + return Ok(f(StopIter)); + } + let ret = match_class!(match self { + PyNone => f(None), + PyEllipsis => f(Ellipsis), + ref pyint @ PyInt => { + if self.class().is(PyBool::static_type()) { + f(Boolean(!pyint.as_bigint().is_zero())) + } else { + f(Integer(pyint.as_bigint())) + } + } + ref pyfloat @ PyFloat => { + f(Float(pyfloat.to_f64())) + } + ref pycomplex @ PyComplex => { + f(Complex(pycomplex.to_complex64())) + } + ref pystr @ PyStr => { + f(Str(pystr.as_wtf8())) + } + ref pylist @ PyList => { + f(List(&pylist.borrow_vec())) + } + ref pyset @ PySet => { + let elements = pyset.elements(); + f(Set(&elements)) + } + ref pyfrozen @ PyFrozenSet => { + let elements = pyfrozen.elements(); + f(Frozenset(&elements)) + } + ref pytuple @ PyTuple => { + f(Tuple(pytuple.as_slice())) + } + ref pydict @ PyDict => { + let entries = pydict.into_iter().collect::<Vec<_>>(); + f(Dict(&entries)) + } + ref bytes @ PyBytes => { + f(Bytes(bytes.as_bytes())) + } + ref bytes @ PyByteArray => { + f(Bytes(&bytes.borrow_buf())) + } + ref co @ PyCode => { + f(Code(co)) + } + _ => return Err(DumpError), + }); + Ok(ret) + } + } + + #[pyfunction] + fn dumps( + value: PyObjectRef, + _version: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<PyBytes> { + use marshal::Dumpable; + let mut buf = Vec::new(); + value + .with_dump(|val| marshal::serialize_value(&mut buf, val)) + .unwrap_or_else(Err) + .map_err(|DumpError| { + vm.new_not_implemented_error( + "TODO: not implemented yet or marshal unsupported type", + ) + })?; + Ok(PyBytes::from(buf)) + } + + #[pyfunction] + fn dump( + value: PyObjectRef, + f: PyObjectRef, + version: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let dumped = dumps(value, version, vm)?; + vm.call_method(&f, "write", (dumped,))?; + Ok(()) + } + + #[derive(Copy, Clone)] + struct PyMarshalBag<'a>(&'a VirtualMachine); + + impl<'a> marshal::MarshalBag for PyMarshalBag<'a> { + type Value = PyObjectRef; + type ConstantBag = PyObjBag<'a>; + + fn make_bool(&self, value: bool) -> Self::Value { + self.0.ctx.new_bool(value).into() + } + + fn make_none(&self) -> Self::Value { + self.0.ctx.none() + } + + fn make_ellipsis(&self) -> Self::Value { + self.0.ctx.ellipsis.clone().into() + } + + fn make_float(&self, value: f64) -> Self::Value { + self.0.ctx.new_float(value).into() + } + + fn make_complex(&self, value: Complex64) -> Self::Value { + self.0.ctx.new_complex(value).into() + } + + fn make_str(&self, value: &Wtf8) -> Self::Value { + self.0.ctx.new_str(value).into() + } + + fn make_bytes(&self, value: &[u8]) -> Self::Value { + self.0.ctx.new_bytes(value.to_vec()).into() + } + + fn make_int(&self, value: BigInt) -> Self::Value { + self.0.ctx.new_int(value).into() + } + + fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value { + let elements = elements.collect(); + self.0.ctx.new_tuple(elements).into() + } + + fn make_code(&self, code: CodeObject) -> Self::Value { + self.0.ctx.new_code(code).into() + } + + fn make_stop_iter(&self) -> Result<Self::Value, marshal::MarshalError> { + Ok(self.0.ctx.exceptions.stop_iteration.to_owned().into()) + } + + fn make_list( + &self, + it: impl Iterator<Item = Self::Value>, + ) -> Result<Self::Value, marshal::MarshalError> { + Ok(self.0.ctx.new_list(it.collect()).into()) + } + + fn make_set( + &self, + it: impl Iterator<Item = Self::Value>, + ) -> Result<Self::Value, marshal::MarshalError> { + let vm = self.0; + let set = PySet::default().into_ref(&vm.ctx); + for elem in it { + set.add(elem, vm).unwrap() + } + Ok(set.into()) + } + + fn make_frozenset( + &self, + it: impl Iterator<Item = Self::Value>, + ) -> Result<Self::Value, marshal::MarshalError> { + let vm = self.0; + Ok(PyFrozenSet::from_iter(vm, it).unwrap().to_pyobject(vm)) + } + + fn make_dict( + &self, + it: impl Iterator<Item = (Self::Value, Self::Value)>, + ) -> Result<Self::Value, marshal::MarshalError> { + let vm = self.0; + let dict = vm.ctx.new_dict(); + for (k, v) in it { + dict.set_item(&*k, v, vm).unwrap() + } + Ok(dict.into()) + } + + fn constant_bag(self) -> Self::ConstantBag { + PyObjBag(&self.0.ctx) + } + } + + #[pyfunction] + fn loads(pybuffer: PyBuffer, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let buf = pybuffer.as_contiguous().ok_or_else(|| { + vm.new_buffer_error("Buffer provided to marshal.loads() is not contiguous") + })?; + marshal::deserialize_value(&mut &buf[..], PyMarshalBag(vm)).map_err(|e| match e { + marshal::MarshalError::Eof => vm.new_exception_msg( + vm.ctx.exceptions.eof_error.to_owned(), + "marshal data too short".into(), + ), + marshal::MarshalError::InvalidBytecode => { + vm.new_value_error("Couldn't deserialize python bytecode") + } + marshal::MarshalError::InvalidUtf8 => { + vm.new_value_error("invalid utf8 in marshalled string") + } + marshal::MarshalError::InvalidLocation => { + vm.new_value_error("invalid location in marshalled object") + } + marshal::MarshalError::BadType => { + vm.new_value_error("bad marshal data (unknown type code)") + } + }) + } + + #[pyfunction] + fn load(f: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + let read_res = vm.call_method(&f, "read", ())?; + let bytes = ArgBytesLike::try_from_object(vm, read_res)?; + loads(PyBuffer::from(bytes), vm) + } +} diff --git a/crates/vm/src/stdlib/mod.rs b/crates/vm/src/stdlib/mod.rs new file mode 100644 index 00000000000..97fb4b372cf --- /dev/null +++ b/crates/vm/src/stdlib/mod.rs @@ -0,0 +1,150 @@ +mod _abc; +#[cfg(feature = "ast")] +pub(crate) mod _ast; +mod _codecs; +mod _collections; +mod _functools; +mod _imp; +pub mod _io; +mod _operator; +mod _sre; +mod _stat; +mod _string; +#[cfg(feature = "compiler")] +mod _symtable; +mod _sysconfig; +mod _sysconfigdata; +mod _types; +pub mod _typing; +pub mod _warnings; +mod _weakref; +pub mod atexit; +pub mod builtins; +pub mod errno; +mod gc; +mod itertools; +mod marshal; +pub mod time; +mod typevar; + +#[cfg(feature = "host_env")] +#[macro_use] +pub mod os; +#[cfg(all(feature = "host_env", windows))] +pub mod nt; +#[cfg(all(feature = "host_env", unix))] +pub mod posix; +#[cfg(all(feature = "host_env", not(any(unix, windows))))] +#[path = "posix_compat.rs"] +pub mod posix; + +#[cfg(all( + feature = "host_env", + any( + target_os = "linux", + target_os = "macos", + target_os = "windows", + target_os = "android" + ), + not(any(target_env = "musl", target_env = "sgx")) +))] +mod _ctypes; +#[cfg(all(feature = "host_env", windows))] +pub(crate) mod msvcrt; + +#[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "ios", target_os = "wasi", target_os = "redox")) +))] +mod pwd; + +#[cfg(feature = "host_env")] +pub(crate) mod _signal; +#[cfg(feature = "threading")] +pub mod _thread; +#[cfg(all(feature = "host_env", windows))] +mod _wmi; +pub mod sys; +#[cfg(all(feature = "host_env", windows))] +#[path = "_winapi.rs"] +mod winapi; +#[cfg(all(feature = "host_env", windows))] +mod winreg; +#[cfg(all(feature = "host_env", windows))] +mod winsound; + +use crate::{Context, builtins::PyModuleDef}; + +/// Returns module definitions for multi-phase init modules. +/// +/// These modules use multi-phase initialization pattern: +/// 1. Create module from def and add to sys.modules +/// 2. Call exec slot (can safely import other modules without circular import issues) +pub fn builtin_module_defs(ctx: &Context) -> Vec<&'static PyModuleDef> { + vec![ + _abc::module_def(ctx), + _types::module_def(ctx), + #[cfg(feature = "ast")] + _ast::module_def(ctx), + atexit::module_def(ctx), + _codecs::module_def(ctx), + _collections::module_def(ctx), + #[cfg(all( + feature = "host_env", + any( + target_os = "linux", + target_os = "macos", + target_os = "windows", + target_os = "android" + ), + not(any(target_env = "musl", target_env = "sgx")) + ))] + _ctypes::module_def(ctx), + errno::module_def(ctx), + _functools::module_def(ctx), + gc::module_def(ctx), + _imp::module_def(ctx), + _io::module_def(ctx), + itertools::module_def(ctx), + marshal::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + msvcrt::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + nt::module_def(ctx), + _operator::module_def(ctx), + #[cfg(all(feature = "host_env", any(unix, target_os = "wasi")))] + posix::module_def(ctx), + #[cfg(all(feature = "host_env", not(any(unix, windows, target_os = "wasi"))))] + posix::module_def(ctx), + #[cfg(all( + feature = "host_env", + unix, + not(any(target_os = "ios", target_os = "wasi", target_os = "redox")) + ))] + pwd::module_def(ctx), + #[cfg(feature = "host_env")] + _signal::module_def(ctx), + _sre::module_def(ctx), + _stat::module_def(ctx), + _string::module_def(ctx), + #[cfg(feature = "compiler")] + _symtable::module_def(ctx), + _sysconfigdata::module_def(ctx), + _sysconfig::module_def(ctx), + #[cfg(feature = "threading")] + _thread::module_def(ctx), + time::module_def(ctx), + _typing::module_def(ctx), + _warnings::module_def(ctx), + _weakref::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + winapi::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + winreg::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + winsound::module_def(ctx), + #[cfg(all(feature = "host_env", windows))] + _wmi::module_def(ctx), + ] +} diff --git a/crates/vm/src/stdlib/msvcrt.rs b/crates/vm/src/stdlib/msvcrt.rs new file mode 100644 index 00000000000..93364ea3596 --- /dev/null +++ b/crates/vm/src/stdlib/msvcrt.rs @@ -0,0 +1,189 @@ +// spell-checker:disable + +pub use msvcrt::*; + +#[pymodule] +mod msvcrt { + use crate::{ + PyRef, PyResult, VirtualMachine, + builtins::{PyBytes, PyStrRef}, + common::{crt_fd, suppress_iph}, + convert::IntoPyException, + }; + use itertools::Itertools; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::Diagnostics::Debug; + + #[pyattr] + use windows_sys::Win32::System::Diagnostics::Debug::{ + SEM_FAILCRITICALERRORS, SEM_NOALIGNMENTFAULTEXCEPT, SEM_NOGPFAULTERRORBOX, + SEM_NOOPENFILEERRORBOX, + }; + + pub fn setmode_binary(fd: crt_fd::Borrowed<'_>) { + unsafe { suppress_iph!(_setmode(fd, libc::O_BINARY)) }; + } + + unsafe extern "C" { + fn _getch() -> i32; + fn _getwch() -> u32; + fn _getche() -> i32; + fn _getwche() -> u32; + fn _putch(c: u32) -> i32; + fn _putwch(c: u16) -> u32; + fn _ungetch(c: i32) -> i32; + fn _ungetwch(c: u32) -> u32; + fn _locking(fd: i32, mode: i32, nbytes: i64) -> i32; + fn _heapmin() -> i32; + fn _kbhit() -> i32; + } + + // Locking mode constants + #[pyattr] + const LK_UNLCK: i32 = 0; // Unlock + #[pyattr] + const LK_LOCK: i32 = 1; // Lock (blocking) + #[pyattr] + const LK_NBLCK: i32 = 2; // Non-blocking lock + #[pyattr] + const LK_RLCK: i32 = 3; // Lock for reading (same as LK_LOCK) + #[pyattr] + const LK_NBRLCK: i32 = 4; // Non-blocking lock for reading (same as LK_NBLCK) + + #[pyfunction] + fn getch() -> Vec<u8> { + let c = unsafe { _getch() }; + vec![c as u8] + } + #[pyfunction] + fn getwch() -> String { + let c = unsafe { _getwch() }; + char::from_u32(c).unwrap().to_string() + } + #[pyfunction] + fn getche() -> Vec<u8> { + let c = unsafe { _getche() }; + vec![c as u8] + } + #[pyfunction] + fn getwche() -> String { + let c = unsafe { _getwche() }; + char::from_u32(c).unwrap().to_string() + } + #[pyfunction] + fn putch(b: PyRef<PyBytes>, vm: &VirtualMachine) -> PyResult<()> { + let &c = + b.as_bytes().iter().exactly_one().map_err(|_| { + vm.new_type_error("putch() argument must be a byte string of length 1") + })?; + unsafe { suppress_iph!(_putch(c.into())) }; + Ok(()) + } + #[pyfunction] + fn putwch(s: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let c = s + .expect_str() + .chars() + .exactly_one() + .map_err(|_| vm.new_type_error("putch() argument must be a string of length 1"))?; + unsafe { suppress_iph!(_putwch(c as u16)) }; + Ok(()) + } + + #[pyfunction] + fn ungetch(b: PyRef<PyBytes>, vm: &VirtualMachine) -> PyResult<()> { + let &c = b.as_bytes().iter().exactly_one().map_err(|_| { + vm.new_type_error("ungetch() argument must be a byte string of length 1") + })?; + let ret = unsafe { suppress_iph!(_ungetch(c as i32)) }; + if ret == -1 { + // EOF returned means the buffer is full + Err(vm.new_os_error(libc::ENOSPC)) + } else { + Ok(()) + } + } + + #[pyfunction] + fn ungetwch(s: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let c = + s.expect_str().chars().exactly_one().map_err(|_| { + vm.new_type_error("ungetwch() argument must be a string of length 1") + })?; + let ret = unsafe { suppress_iph!(_ungetwch(c as u32)) }; + if ret == 0xFFFF { + // WEOF returned means the buffer is full + Err(vm.new_os_error(libc::ENOSPC)) + } else { + Ok(()) + } + } + + #[pyfunction] + fn kbhit() -> i32 { + unsafe { _kbhit() } + } + + #[pyfunction] + fn locking(fd: i32, mode: i32, nbytes: i64, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { suppress_iph!(_locking(fd, mode, nbytes)) }; + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + #[pyfunction] + fn heapmin(vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { suppress_iph!(_heapmin()) }; + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + unsafe extern "C" { + fn _setmode(fd: crt_fd::Borrowed<'_>, flags: i32) -> i32; + } + + #[pyfunction] + fn setmode(fd: crt_fd::Borrowed<'_>, flags: i32, vm: &VirtualMachine) -> PyResult<i32> { + let flags = unsafe { suppress_iph!(_setmode(fd, flags)) }; + if flags == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(flags) + } + } + + #[pyfunction] + fn open_osfhandle(handle: isize, flags: i32, vm: &VirtualMachine) -> PyResult<i32> { + let ret = unsafe { suppress_iph!(libc::open_osfhandle(handle, flags)) }; + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(ret) + } + } + + #[pyfunction] + fn get_osfhandle(fd: crt_fd::Borrowed<'_>, vm: &VirtualMachine) -> PyResult<isize> { + crt_fd::as_handle(fd) + .map(|h| h.as_raw_handle() as _) + .map_err(|e| e.into_pyexception(vm)) + } + + #[allow(non_snake_case)] + #[pyfunction] + fn GetErrorMode() -> u32 { + unsafe { suppress_iph!(Debug::GetErrorMode()) } + } + + #[allow(non_snake_case)] + #[pyfunction] + fn SetErrorMode(mode: Debug::THREAD_ERROR_MODE, _: &VirtualMachine) -> u32 { + unsafe { suppress_iph!(Debug::SetErrorMode(mode)) } + } +} diff --git a/crates/vm/src/stdlib/nt.rs b/crates/vm/src/stdlib/nt.rs new file mode 100644 index 00000000000..5dd4cf4f001 --- /dev/null +++ b/crates/vm/src/stdlib/nt.rs @@ -0,0 +1,2301 @@ +// spell-checker:disable + +pub(crate) use module::module_def; +pub use module::raw_set_handle_inheritable; + +#[pymodule(name = "nt", with(super::os::_os))] +pub(crate) mod module { + use crate::{ + Py, PyResult, TryFromObject, VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyBytes, PyDictRef, PyListRef, PyStr, PyStrRef, PyTupleRef, + }, + common::{crt_fd, suppress_iph, windows::ToWideString}, + convert::ToPyException, + exceptions::OSErrorBuilder, + function::{ArgMapping, Either, OptionalArg}, + ospath::{OsPath, OsPathOrFd}, + stdlib::os::{_os, DirFd, SupportFunc, TargetIsDirectory}, + }; + use core::mem::MaybeUninit; + use libc::intptr_t; + use rustpython_common::wtf8::Wtf8Buf; + use std::os::windows::io::AsRawHandle; + use std::{env, io, os::windows::ffi::OsStringExt}; + use windows_sys::Win32::{ + Foundation::{self, INVALID_HANDLE_VALUE}, + Storage::FileSystem, + System::{Console, Threading}, + }; + + #[pyattr] + use libc::{O_BINARY, O_NOINHERIT, O_RANDOM, O_SEQUENTIAL, O_TEMPORARY, O_TEXT}; + + // Windows spawn mode constants + #[pyattr] + const P_WAIT: i32 = 0; + #[pyattr] + const P_NOWAIT: i32 = 1; + #[pyattr] + const P_OVERLAY: i32 = 2; + #[pyattr] + const P_NOWAITO: i32 = 3; + #[pyattr] + const P_DETACH: i32 = 4; + + // _O_SHORT_LIVED is not in libc, define manually + #[pyattr] + const O_SHORT_LIVED: i32 = 0x1000; + + // Exit code constant + #[pyattr] + const EX_OK: i32 = 0; + + // Maximum number of temporary files + #[pyattr] + const TMP_MAX: i32 = i32::MAX; + + #[pyattr] + use windows_sys::Win32::System::LibraryLoader::{ + LOAD_LIBRARY_SEARCH_APPLICATION_DIR as _LOAD_LIBRARY_SEARCH_APPLICATION_DIR, + LOAD_LIBRARY_SEARCH_DEFAULT_DIRS as _LOAD_LIBRARY_SEARCH_DEFAULT_DIRS, + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR as _LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR, + LOAD_LIBRARY_SEARCH_SYSTEM32 as _LOAD_LIBRARY_SEARCH_SYSTEM32, + LOAD_LIBRARY_SEARCH_USER_DIRS as _LOAD_LIBRARY_SEARCH_USER_DIRS, + }; + + #[pyfunction] + pub(super) fn access(path: OsPath, mode: u8, vm: &VirtualMachine) -> PyResult<bool> { + let attr = unsafe { FileSystem::GetFileAttributesW(path.to_wide_cstring(vm)?.as_ptr()) }; + Ok(attr != FileSystem::INVALID_FILE_ATTRIBUTES + && (mode & 2 == 0 + || attr & FileSystem::FILE_ATTRIBUTE_READONLY == 0 + || attr & FileSystem::FILE_ATTRIBUTE_DIRECTORY != 0)) + } + + #[pyfunction] + #[pyfunction(name = "unlink")] + pub(super) fn remove( + path: OsPath, + dir_fd: DirFd<'static, 0>, + vm: &VirtualMachine, + ) -> PyResult<()> { + // On Windows, use DeleteFileW directly. + // Rust's std::fs::remove_file may have different behavior for read-only files. + // See Py_DeleteFileW. + use windows_sys::Win32::Storage::FileSystem::{ + DeleteFileW, FindClose, FindFirstFileW, RemoveDirectoryW, WIN32_FIND_DATAW, + }; + use windows_sys::Win32::System::SystemServices::{ + IO_REPARSE_TAG_MOUNT_POINT, IO_REPARSE_TAG_SYMLINK, + }; + + let [] = dir_fd.0; + let wide_path = path.to_wide_cstring(vm)?; + let attrs = unsafe { FileSystem::GetFileAttributesW(wide_path.as_ptr()) }; + + let mut is_directory = false; + let mut is_link = false; + + if attrs != FileSystem::INVALID_FILE_ATTRIBUTES { + is_directory = (attrs & FileSystem::FILE_ATTRIBUTE_DIRECTORY) != 0; + + // Check if it's a symlink or junction point + if is_directory && (attrs & FileSystem::FILE_ATTRIBUTE_REPARSE_POINT) != 0 { + let mut find_data: WIN32_FIND_DATAW = unsafe { core::mem::zeroed() }; + let handle = unsafe { FindFirstFileW(wide_path.as_ptr(), &mut find_data) }; + if handle != INVALID_HANDLE_VALUE { + is_link = find_data.dwReserved0 == IO_REPARSE_TAG_SYMLINK + || find_data.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT; + unsafe { FindClose(handle) }; + } + } + } + + let result = if is_directory && is_link { + unsafe { RemoveDirectoryW(wide_path.as_ptr()) } + } else { + unsafe { DeleteFileW(wide_path.as_ptr()) } + }; + + if result == 0 { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + Ok(()) + } + + #[pyfunction] + pub(super) fn _supports_virtual_terminal() -> PyResult<bool> { + let mut mode = 0; + let handle = unsafe { Console::GetStdHandle(Console::STD_ERROR_HANDLE) }; + if unsafe { Console::GetConsoleMode(handle, &mut mode) } == 0 { + return Ok(false); + } + Ok(mode & Console::ENABLE_VIRTUAL_TERMINAL_PROCESSING != 0) + } + + #[derive(FromArgs)] + pub(super) struct SymlinkArgs<'fd> { + src: OsPath, + dst: OsPath, + #[pyarg(flatten)] + target_is_directory: TargetIsDirectory, + #[pyarg(flatten)] + _dir_fd: DirFd<'fd, { _os::SYMLINK_DIR_FD as usize }>, + } + + #[pyfunction] + pub(super) fn symlink(args: SymlinkArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { + use crate::exceptions::ToOSErrorBuilder; + use core::sync::atomic::{AtomicBool, Ordering}; + use windows_sys::Win32::Storage::FileSystem::WIN32_FILE_ATTRIBUTE_DATA; + use windows_sys::Win32::Storage::FileSystem::{ + CreateSymbolicLinkW, FILE_ATTRIBUTE_DIRECTORY, GetFileAttributesExW, + SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE, SYMBOLIC_LINK_FLAG_DIRECTORY, + }; + + static HAS_UNPRIVILEGED_FLAG: AtomicBool = AtomicBool::new(true); + + fn check_dir(src: &OsPath, dst: &OsPath) -> bool { + use windows_sys::Win32::Storage::FileSystem::GetFileExInfoStandard; + + let dst_parent = dst.as_path().parent(); + let Some(dst_parent) = dst_parent else { + return false; + }; + let resolved = if src.as_path().is_absolute() { + src.as_path().to_path_buf() + } else { + dst_parent.join(src.as_path()) + }; + let wide = match widestring::WideCString::from_os_str(&resolved) { + Ok(wide) => wide, + Err(_) => return false, + }; + let mut info: WIN32_FILE_ATTRIBUTE_DATA = unsafe { core::mem::zeroed() }; + let ok = unsafe { + GetFileAttributesExW( + wide.as_ptr(), + GetFileExInfoStandard, + &mut info as *mut _ as *mut _, + ) + }; + ok != 0 && (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0 + } + + let mut flags = 0u32; + if HAS_UNPRIVILEGED_FLAG.load(Ordering::Relaxed) { + flags |= SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; + } + if args.target_is_directory.target_is_directory || check_dir(&args.src, &args.dst) { + flags |= SYMBOLIC_LINK_FLAG_DIRECTORY; + } + + let src = args.src.to_wide_cstring(vm)?; + let dst = args.dst.to_wide_cstring(vm)?; + + let mut result = unsafe { CreateSymbolicLinkW(dst.as_ptr(), src.as_ptr(), flags) }; + if !result + && HAS_UNPRIVILEGED_FLAG.load(Ordering::Relaxed) + && unsafe { Foundation::GetLastError() } == Foundation::ERROR_INVALID_PARAMETER + { + let flags = flags & !SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE; + result = unsafe { CreateSymbolicLinkW(dst.as_ptr(), src.as_ptr(), flags) }; + if result + || unsafe { Foundation::GetLastError() } != Foundation::ERROR_INVALID_PARAMETER + { + HAS_UNPRIVILEGED_FLAG.store(false, Ordering::Relaxed); + } + } + + if !result { + let err = io::Error::last_os_error(); + let builder = err.to_os_error_builder(vm); + let builder = builder + .filename(args.src.filename(vm)) + .filename2(args.dst.filename(vm)); + return Err(builder.build(vm).upcast()); + } + + Ok(()) + } + + #[pyfunction] + fn set_inheritable( + fd: crt_fd::Borrowed<'_>, + inheritable: bool, + vm: &VirtualMachine, + ) -> PyResult<()> { + let handle = crt_fd::as_handle(fd).map_err(|e| e.to_pyexception(vm))?; + set_handle_inheritable(handle.as_raw_handle() as _, inheritable, vm) + } + + #[pyattr] + fn environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + + for (key, value) in env::vars() { + // Skip hidden Windows environment variables (e.g., =C:, =D:, =ExitCode) + // These are internal cmd.exe bookkeeping variables that store per-drive + // current directories and cannot be reliably modified via _wputenv(). + if key.starts_with('=') { + continue; + } + environ.set_item(&key, vm.new_pyobj(value), vm).unwrap(); + } + environ + } + + #[pyfunction] + fn _create_environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars() { + if key.starts_with('=') { + continue; + } + environ.set_item(&key, vm.new_pyobj(value), vm).unwrap(); + } + environ + } + + #[derive(FromArgs)] + struct ChmodArgs<'a> { + #[pyarg(any)] + path: OsPathOrFd<'a>, + #[pyarg(any)] + mode: u32, + #[pyarg(flatten)] + dir_fd: DirFd<'static, 0>, + #[pyarg(named, name = "follow_symlinks", optional)] + follow_symlinks: OptionalArg<bool>, + } + + const S_IWRITE: u32 = 128; + + fn win32_hchmod(handle: Foundation::HANDLE, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Storage::FileSystem::{ + FILE_BASIC_INFO, FileBasicInfo, GetFileInformationByHandleEx, + SetFileInformationByHandle, + }; + + // Get current file info + let mut info: FILE_BASIC_INFO = unsafe { core::mem::zeroed() }; + let ret = unsafe { + GetFileInformationByHandleEx( + handle, + FileBasicInfo, + &mut info as *mut _ as *mut _, + core::mem::size_of::<FILE_BASIC_INFO>() as u32, + ) + }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + // Modify readonly attribute based on S_IWRITE bit + if mode & S_IWRITE != 0 { + info.FileAttributes &= !FileSystem::FILE_ATTRIBUTE_READONLY; + } else { + info.FileAttributes |= FileSystem::FILE_ATTRIBUTE_READONLY; + } + + // Set the new attributes + let ret = unsafe { + SetFileInformationByHandle( + handle, + FileBasicInfo, + &info as *const _ as *const _, + core::mem::size_of::<FILE_BASIC_INFO>() as u32, + ) + }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + Ok(()) + } + + fn fchmod_impl(fd: i32, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + // Get Windows HANDLE from fd + let borrowed = unsafe { crt_fd::Borrowed::borrow_raw(fd) }; + let handle = crt_fd::as_handle(borrowed).map_err(|e| e.to_pyexception(vm))?; + let hfile = handle.as_raw_handle() as Foundation::HANDLE; + win32_hchmod(hfile, mode, vm) + } + + fn win32_lchmod(path: &OsPath, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Storage::FileSystem::{GetFileAttributesW, SetFileAttributesW}; + + let wide = path.to_wide_cstring(vm)?; + let attr = unsafe { GetFileAttributesW(wide.as_ptr()) }; + if attr == FileSystem::INVALID_FILE_ATTRIBUTES { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)); + } + let new_attr = if mode & S_IWRITE != 0 { + attr & !FileSystem::FILE_ATTRIBUTE_READONLY + } else { + attr | FileSystem::FILE_ATTRIBUTE_READONLY + }; + let ret = unsafe { SetFileAttributesW(wide.as_ptr(), new_attr) }; + if ret == 0 { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)); + } + Ok(()) + } + + #[pyfunction] + fn fchmod(fd: i32, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + fchmod_impl(fd, mode, vm) + } + + #[pyfunction] + fn chmod(args: ChmodArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { + let ChmodArgs { + path, + mode, + dir_fd, + follow_symlinks, + } = args; + let [] = dir_fd.0; + + // If path is a file descriptor, use fchmod + if let OsPathOrFd::Fd(fd) = path { + if follow_symlinks.into_option().is_some() { + return Err( + vm.new_value_error("chmod: follow_symlinks is not supported with fd argument") + ); + } + return fchmod_impl(fd.as_raw(), mode, vm); + } + + let OsPathOrFd::Path(path) = path else { + unreachable!() + }; + + let follow_symlinks = follow_symlinks.into_option().unwrap_or(false); + + if follow_symlinks { + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, + FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_ATTRIBUTES, OPEN_EXISTING, + }; + + let wide = path.to_wide_cstring(vm)?; + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + core::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + let result = win32_hchmod(handle, mode, vm); + unsafe { Foundation::CloseHandle(handle) }; + result + } else { + win32_lchmod(&path, mode, vm) + } + } + + /// Get the real file name (with correct case) without accessing the file. + /// Uses FindFirstFileW to get the name as stored on the filesystem. + #[pyfunction] + fn _findfirstfile(path: OsPath, vm: &VirtualMachine) -> PyResult<PyStrRef> { + use crate::common::windows::ToWideString; + use std::os::windows::ffi::OsStringExt; + use windows_sys::Win32::Storage::FileSystem::{ + FindClose, FindFirstFileW, WIN32_FIND_DATAW, + }; + + let wide_path = path.as_ref().to_wide_with_nul(); + let mut find_data: WIN32_FIND_DATAW = unsafe { core::mem::zeroed() }; + + let handle = unsafe { FindFirstFileW(wide_path.as_ptr(), &mut find_data) }; + if handle == INVALID_HANDLE_VALUE { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + + unsafe { FindClose(handle) }; + + // Convert the filename from the find data to a Rust string + // cFileName is a null-terminated wide string + let len = find_data + .cFileName + .iter() + .position(|&c| c == 0) + .unwrap_or(find_data.cFileName.len()); + let filename = std::ffi::OsString::from_wide(&find_data.cFileName[..len]); + let filename_str = filename + .to_str() + .ok_or_else(|| vm.new_unicode_decode_error("filename contains invalid UTF-8"))?; + + Ok(vm.ctx.new_str(filename_str).to_owned()) + } + + #[derive(FromArgs)] + struct PathArg { + #[pyarg(any)] + path: crate::PyObjectRef, + } + + impl PathArg { + fn to_path_or_fd(&self, vm: &VirtualMachine) -> Option<OsPathOrFd<'static>> { + OsPathOrFd::try_from_object(vm, self.path.clone()).ok() + } + } + + // File type test constants (PY_IF* constants - internal, not from Windows API) + const PY_IFREG: u32 = 1; // Regular file + const PY_IFDIR: u32 = 2; // Directory + const PY_IFLNK: u32 = 4; // Symlink + const PY_IFMNT: u32 = 8; // Mount point (junction) + const PY_IFLRP: u32 = 16; // Link Reparse Point (name-surrogate, symlink, junction) + const PY_IFRRP: u32 = 32; // Regular Reparse Point + + /// _testInfo - determine file type based on attributes and reparse tag + fn _test_info(attributes: u32, reparse_tag: u32, disk_device: bool, tested_type: u32) -> bool { + use windows_sys::Win32::Storage::FileSystem::{ + FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, + }; + use windows_sys::Win32::System::SystemServices::{ + IO_REPARSE_TAG_MOUNT_POINT, IO_REPARSE_TAG_SYMLINK, + }; + + match tested_type { + PY_IFREG => { + // diskDevice && attributes && !(attributes & FILE_ATTRIBUTE_DIRECTORY) + disk_device && attributes != 0 && (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0 + } + PY_IFDIR => (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0, + PY_IFLNK => { + (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 + && reparse_tag == IO_REPARSE_TAG_SYMLINK + } + PY_IFMNT => { + (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 + && reparse_tag == IO_REPARSE_TAG_MOUNT_POINT + } + PY_IFLRP => { + (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 + && is_reparse_tag_name_surrogate(reparse_tag) + } + PY_IFRRP => { + (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 + && reparse_tag != 0 + && !is_reparse_tag_name_surrogate(reparse_tag) + } + _ => false, + } + } + + fn is_reparse_tag_name_surrogate(tag: u32) -> bool { + (tag & 0x20000000) != 0 + } + + fn file_info_error_is_trustworthy(error: u32) -> bool { + use windows_sys::Win32::Foundation; + matches!( + error, + Foundation::ERROR_FILE_NOT_FOUND + | Foundation::ERROR_PATH_NOT_FOUND + | Foundation::ERROR_NOT_READY + | Foundation::ERROR_BAD_NET_NAME + | Foundation::ERROR_BAD_NETPATH + | Foundation::ERROR_BAD_PATHNAME + | Foundation::ERROR_INVALID_NAME + | Foundation::ERROR_FILENAME_EXCED_RANGE + ) + } + + /// _testFileTypeByHandle - test file type using an open handle + fn _test_file_type_by_handle( + handle: windows_sys::Win32::Foundation::HANDLE, + tested_type: u32, + disk_only: bool, + ) -> bool { + use windows_sys::Win32::Storage::FileSystem::{ + FILE_ATTRIBUTE_TAG_INFO, FILE_BASIC_INFO, FILE_TYPE_DISK, + FileAttributeTagInfo as FileAttributeTagInfoClass, FileBasicInfo, + GetFileInformationByHandleEx, GetFileType, + }; + + let disk_device = unsafe { GetFileType(handle) } == FILE_TYPE_DISK; + if disk_only && !disk_device { + return false; + } + + if tested_type != PY_IFREG && tested_type != PY_IFDIR { + // For symlinks/junctions, need FileAttributeTagInfo to get reparse tag + let mut info: FILE_ATTRIBUTE_TAG_INFO = unsafe { core::mem::zeroed() }; + let ret = unsafe { + GetFileInformationByHandleEx( + handle, + FileAttributeTagInfoClass, + &mut info as *mut _ as *mut _, + core::mem::size_of::<FILE_ATTRIBUTE_TAG_INFO>() as u32, + ) + }; + if ret == 0 { + return false; + } + _test_info( + info.FileAttributes, + info.ReparseTag, + disk_device, + tested_type, + ) + } else { + // For regular files/directories, FileBasicInfo is sufficient + let mut info: FILE_BASIC_INFO = unsafe { core::mem::zeroed() }; + let ret = unsafe { + GetFileInformationByHandleEx( + handle, + FileBasicInfo, + &mut info as *mut _ as *mut _, + core::mem::size_of::<FILE_BASIC_INFO>() as u32, + ) + }; + if ret == 0 { + return false; + } + _test_info(info.FileAttributes, 0, disk_device, tested_type) + } + } + + /// _testFileTypeByName - test file type by path name + fn _test_file_type_by_name(path: &std::path::Path, tested_type: u32) -> bool { + use crate::common::fileutils::windows::{ + FILE_INFO_BY_NAME_CLASS, get_file_information_by_name, + }; + use crate::common::windows::ToWideString; + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, FILE_READ_ATTRIBUTES, OPEN_EXISTING, + }; + use windows_sys::Win32::Storage::FileSystem::{FILE_DEVICE_CD_ROM, FILE_DEVICE_DISK}; + use windows_sys::Win32::System::Ioctl::FILE_DEVICE_VIRTUAL_DISK; + + match get_file_information_by_name( + path.as_os_str(), + FILE_INFO_BY_NAME_CLASS::FileStatBasicByNameInfo, + ) { + Ok(info) => { + let disk_device = matches!( + info.DeviceType, + FILE_DEVICE_DISK | FILE_DEVICE_VIRTUAL_DISK | FILE_DEVICE_CD_ROM + ); + let result = _test_info( + info.FileAttributes, + info.ReparseTag, + disk_device, + tested_type, + ); + if !result + || (tested_type != PY_IFREG && tested_type != PY_IFDIR) + || (info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0 + { + return result; + } + } + Err(err) => { + if let Some(code) = err.raw_os_error() + && file_info_error_is_trustworthy(code as u32) + { + return false; + } + } + } + + let wide_path = path.to_wide_with_nul(); + + let mut flags = FILE_FLAG_BACKUP_SEMANTICS; + if tested_type != PY_IFREG && tested_type != PY_IFDIR { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + let handle = unsafe { + CreateFileW( + wide_path.as_ptr(), + FILE_READ_ATTRIBUTES, + 0, + core::ptr::null(), + OPEN_EXISTING, + flags, + core::ptr::null_mut(), + ) + }; + + if handle != INVALID_HANDLE_VALUE { + let result = _test_file_type_by_handle(handle, tested_type, false); + unsafe { CloseHandle(handle) }; + return result; + } + + match unsafe { windows_sys::Win32::Foundation::GetLastError() } { + windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED + | windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION + | windows_sys::Win32::Foundation::ERROR_CANT_ACCESS_FILE + | windows_sys::Win32::Foundation::ERROR_INVALID_PARAMETER => { + let stat = if tested_type == PY_IFREG || tested_type == PY_IFDIR { + crate::windows::win32_xstat(path.as_os_str(), true) + } else { + crate::windows::win32_xstat(path.as_os_str(), false) + }; + if let Ok(st) = stat { + let disk_device = (st.st_mode & libc::S_IFREG as u16) != 0; + return _test_info( + st.st_file_attributes, + st.st_reparse_tag, + disk_device, + tested_type, + ); + } + } + _ => {} + } + + false + } + + /// _testFileExistsByName - test if path exists + fn _test_file_exists_by_name(path: &std::path::Path, follow_links: bool) -> bool { + use crate::common::fileutils::windows::{ + FILE_INFO_BY_NAME_CLASS, get_file_information_by_name, + }; + use crate::common::windows::ToWideString; + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, FILE_READ_ATTRIBUTES, OPEN_EXISTING, + }; + + match get_file_information_by_name( + path.as_os_str(), + FILE_INFO_BY_NAME_CLASS::FileStatBasicByNameInfo, + ) { + Ok(info) => { + if (info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) == 0 + || (!follow_links && is_reparse_tag_name_surrogate(info.ReparseTag)) + { + return true; + } + } + Err(err) => { + if let Some(code) = err.raw_os_error() + && file_info_error_is_trustworthy(code as u32) + { + return false; + } + } + } + + let wide_path = path.to_wide_with_nul(); + let mut flags = FILE_FLAG_BACKUP_SEMANTICS; + if !follow_links { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + let handle = unsafe { + CreateFileW( + wide_path.as_ptr(), + FILE_READ_ATTRIBUTES, + 0, + core::ptr::null(), + OPEN_EXISTING, + flags, + core::ptr::null_mut(), + ) + }; + if handle != INVALID_HANDLE_VALUE { + if follow_links { + unsafe { CloseHandle(handle) }; + return true; + } + let is_regular_reparse_point = _test_file_type_by_handle(handle, PY_IFRRP, false); + unsafe { CloseHandle(handle) }; + if !is_regular_reparse_point { + return true; + } + let handle = unsafe { + CreateFileW( + wide_path.as_ptr(), + FILE_READ_ATTRIBUTES, + 0, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + core::ptr::null_mut(), + ) + }; + if handle != INVALID_HANDLE_VALUE { + unsafe { CloseHandle(handle) }; + return true; + } + } + + match unsafe { windows_sys::Win32::Foundation::GetLastError() } { + windows_sys::Win32::Foundation::ERROR_ACCESS_DENIED + | windows_sys::Win32::Foundation::ERROR_SHARING_VIOLATION + | windows_sys::Win32::Foundation::ERROR_CANT_ACCESS_FILE + | windows_sys::Win32::Foundation::ERROR_INVALID_PARAMETER => { + let stat = crate::windows::win32_xstat(path.as_os_str(), follow_links); + return stat.is_ok(); + } + _ => {} + } + + false + } + + /// _testFileType wrapper - handles both fd and path + fn _test_file_type(path_or_fd: &OsPathOrFd<'_>, tested_type: u32) -> bool { + match path_or_fd { + OsPathOrFd::Fd(fd) => { + if let Ok(handle) = crate::common::crt_fd::as_handle(*fd) { + use std::os::windows::io::AsRawHandle; + _test_file_type_by_handle(handle.as_raw_handle() as _, tested_type, true) + } else { + false + } + } + OsPathOrFd::Path(path) => _test_file_type_by_name(path.as_ref(), tested_type), + } + } + + /// _testFileExists wrapper - handles both fd and path + fn _test_file_exists(path_or_fd: &OsPathOrFd<'_>, follow_links: bool) -> bool { + use windows_sys::Win32::Storage::FileSystem::{FILE_TYPE_UNKNOWN, GetFileType}; + + match path_or_fd { + OsPathOrFd::Fd(fd) => { + if let Ok(handle) = crate::common::crt_fd::as_handle(*fd) { + use std::os::windows::io::AsRawHandle; + let file_type = unsafe { GetFileType(handle.as_raw_handle() as _) }; + // GetFileType(hfile) != FILE_TYPE_UNKNOWN || !GetLastError() + if file_type != FILE_TYPE_UNKNOWN { + return true; + } + // Check if GetLastError is 0 (no error means valid handle) + unsafe { windows_sys::Win32::Foundation::GetLastError() == 0 } + } else { + false + } + } + OsPathOrFd::Path(path) => _test_file_exists_by_name(path.as_ref(), follow_links), + } + } + + /// Check if a path is a directory. + /// return _testFileType(path, PY_IFDIR) + #[pyfunction] + fn _path_isdir(args: PathArg, vm: &VirtualMachine) -> bool { + args.to_path_or_fd(vm) + .is_some_and(|p| _test_file_type(&p, PY_IFDIR)) + } + + /// Check if a path is a regular file. + /// return _testFileType(path, PY_IFREG) + #[pyfunction] + fn _path_isfile(args: PathArg, vm: &VirtualMachine) -> bool { + args.to_path_or_fd(vm) + .is_some_and(|p| _test_file_type(&p, PY_IFREG)) + } + + /// Check if a path is a symbolic link. + /// return _testFileType(path, PY_IFLNK) + #[pyfunction] + fn _path_islink(args: PathArg, vm: &VirtualMachine) -> bool { + args.to_path_or_fd(vm) + .is_some_and(|p| _test_file_type(&p, PY_IFLNK)) + } + + /// Check if a path is a junction (mount point). + /// return _testFileType(path, PY_IFMNT) + #[pyfunction] + fn _path_isjunction(args: PathArg, vm: &VirtualMachine) -> bool { + args.to_path_or_fd(vm) + .is_some_and(|p| _test_file_type(&p, PY_IFMNT)) + } + + /// Check if a path exists (follows symlinks). + /// return _testFileExists(path, TRUE) + #[pyfunction] + fn _path_exists(args: PathArg, vm: &VirtualMachine) -> bool { + args.to_path_or_fd(vm) + .is_some_and(|p| _test_file_exists(&p, true)) + } + + /// Check if a path exists (does not follow symlinks). + /// return _testFileExists(path, FALSE) + #[pyfunction] + fn _path_lexists(args: PathArg, vm: &VirtualMachine) -> bool { + args.to_path_or_fd(vm) + .is_some_and(|p| _test_file_exists(&p, false)) + } + + /// Check if a path is on a Windows Dev Drive. + #[pyfunction] + fn _path_isdevdrive(path: OsPath, vm: &VirtualMachine) -> PyResult<bool> { + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, + FILE_SHARE_WRITE, GetDriveTypeW, GetVolumePathNameW, OPEN_EXISTING, + }; + use windows_sys::Win32::System::IO::DeviceIoControl; + use windows_sys::Win32::System::Ioctl::FSCTL_QUERY_PERSISTENT_VOLUME_STATE; + use windows_sys::Win32::System::WindowsProgramming::DRIVE_FIXED; + + // PERSISTENT_VOLUME_STATE_DEV_VOLUME flag - not yet in windows-sys + const PERSISTENT_VOLUME_STATE_DEV_VOLUME: u32 = 0x00002000; + + // FILE_FS_PERSISTENT_VOLUME_INFORMATION structure + #[repr(C)] + struct FileFsPersistentVolumeInformation { + volume_flags: u32, + flag_mask: u32, + version: u32, + reserved: u32, + } + + let wide_path = path.to_wide_cstring(vm)?; + let mut volume = [0u16; Foundation::MAX_PATH as usize]; + + // Get volume path + let ret = unsafe { + GetVolumePathNameW(wide_path.as_ptr(), volume.as_mut_ptr(), volume.len() as _) + }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + + // Check if it's a fixed drive + if unsafe { GetDriveTypeW(volume.as_ptr()) } != DRIVE_FIXED { + return Ok(false); + } + + // Open the volume + let handle = unsafe { + CreateFileW( + volume.as_ptr(), + FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + core::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + + // Query persistent volume state + let mut volume_state = FileFsPersistentVolumeInformation { + volume_flags: 0, + flag_mask: PERSISTENT_VOLUME_STATE_DEV_VOLUME, + version: 1, + reserved: 0, + }; + + let ret = unsafe { + DeviceIoControl( + handle, + FSCTL_QUERY_PERSISTENT_VOLUME_STATE, + &volume_state as *const _ as *const core::ffi::c_void, + core::mem::size_of::<FileFsPersistentVolumeInformation>() as u32, + &mut volume_state as *mut _ as *mut core::ffi::c_void, + core::mem::size_of::<FileFsPersistentVolumeInformation>() as u32, + core::ptr::null_mut(), + core::ptr::null_mut(), + ) + }; + + unsafe { CloseHandle(handle) }; + + if ret == 0 { + let err = io::Error::last_os_error(); + // ERROR_INVALID_PARAMETER means not supported on this platform + if err.raw_os_error() == Some(Foundation::ERROR_INVALID_PARAMETER as i32) { + return Ok(false); + } + return Err(err.to_pyexception(vm)); + } + + Ok((volume_state.volume_flags & PERSISTENT_VOLUME_STATE_DEV_VOLUME) != 0) + } + + // cwait is available on MSVC only + #[cfg(target_env = "msvc")] + unsafe extern "C" { + fn _cwait(termstat: *mut i32, procHandle: intptr_t, action: i32) -> intptr_t; + } + + #[cfg(target_env = "msvc")] + #[pyfunction] + fn waitpid(pid: intptr_t, opt: i32, vm: &VirtualMachine) -> PyResult<(intptr_t, u64)> { + let mut status: i32 = 0; + let pid = unsafe { suppress_iph!(_cwait(&mut status, pid, opt)) }; + if pid == -1 { + Err(vm.new_last_errno_error()) + } else { + // Cast to unsigned to handle large exit codes (like 0xC000013A) + // then shift left by 8 to match POSIX waitpid format + let ustatus = (status as u32) as u64; + Ok((pid, ustatus << 8)) + } + } + + #[cfg(target_env = "msvc")] + #[pyfunction] + fn wait(vm: &VirtualMachine) -> PyResult<(intptr_t, u64)> { + waitpid(-1, 0, vm) + } + + #[pyfunction] + fn kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { + let sig = sig as u32; + let pid = pid as u32; + + if sig == Console::CTRL_C_EVENT || sig == Console::CTRL_BREAK_EVENT { + let ret = unsafe { Console::GenerateConsoleCtrlEvent(sig, pid) }; + let res = if ret == 0 { + Err(vm.new_last_os_error()) + } else { + Ok(()) + }; + return res; + } + + let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; + if h.is_null() { + return Err(vm.new_last_os_error()); + } + let ret = unsafe { Threading::TerminateProcess(h, sig) }; + let res = if ret == 0 { + Err(vm.new_last_os_error()) + } else { + Ok(()) + }; + unsafe { Foundation::CloseHandle(h) }; + res + } + + #[pyfunction] + fn get_terminal_size( + fd: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<_os::TerminalSizeData> { + let fd = fd.unwrap_or(1); // default to stdout + + // Use _get_osfhandle for all fds + let borrowed = unsafe { crt_fd::Borrowed::borrow_raw(fd) }; + let handle = crt_fd::as_handle(borrowed).map_err(|e| e.to_pyexception(vm))?; + let h = handle.as_raw_handle() as Foundation::HANDLE; + + let mut csbi = MaybeUninit::uninit(); + let ret = unsafe { Console::GetConsoleScreenBufferInfo(h, csbi.as_mut_ptr()) }; + if ret == 0 { + // Check if error is due to lack of read access on a console handle + // ERROR_ACCESS_DENIED (5) means it's a console but without read permission + // In that case, try opening CONOUT$ directly with read access + let err = unsafe { Foundation::GetLastError() }; + if err != Foundation::ERROR_ACCESS_DENIED { + return Err(vm.new_last_os_error()); + } + let conout: Vec<u16> = "CONOUT$\0".encode_utf16().collect(); + let console_handle = unsafe { + FileSystem::CreateFileW( + conout.as_ptr(), + Foundation::GENERIC_READ | Foundation::GENERIC_WRITE, + FileSystem::FILE_SHARE_READ | FileSystem::FILE_SHARE_WRITE, + core::ptr::null(), + FileSystem::OPEN_EXISTING, + 0, + core::ptr::null_mut(), + ) + }; + if console_handle == INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + let ret = + unsafe { Console::GetConsoleScreenBufferInfo(console_handle, csbi.as_mut_ptr()) }; + unsafe { Foundation::CloseHandle(console_handle) }; + if ret == 0 { + return Err(vm.new_last_os_error()); + } + } + let csbi = unsafe { csbi.assume_init() }; + let w = csbi.srWindow; + let columns = (w.Right - w.Left + 1) as usize; + let lines = (w.Bottom - w.Top + 1) as usize; + Ok(_os::TerminalSizeData { columns, lines }) + } + + #[cfg(target_env = "msvc")] + unsafe extern "C" { + fn _wexecv(cmdname: *const u16, argv: *const *const u16) -> intptr_t; + fn _wexecve( + cmdname: *const u16, + argv: *const *const u16, + envp: *const *const u16, + ) -> intptr_t; + fn _wspawnv(mode: i32, cmdname: *const u16, argv: *const *const u16) -> intptr_t; + fn _wspawnve( + mode: i32, + cmdname: *const u16, + argv: *const *const u16, + envp: *const *const u16, + ) -> intptr_t; + } + + #[cfg(target_env = "msvc")] + #[pyfunction] + fn spawnv( + mode: i32, + path: OsPath, + argv: Either<PyListRef, PyTupleRef>, + vm: &VirtualMachine, + ) -> PyResult<intptr_t> { + use crate::function::FsPath; + use core::iter::once; + + let path = path.to_wide_cstring(vm)?; + + let argv = vm.extract_elements_with(argv.as_ref(), |obj| { + let fspath = FsPath::try_from_path_like(obj, true, vm)?; + fspath.to_wide_cstring(vm) + })?; + + let first = argv + .first() + .ok_or_else(|| vm.new_value_error("spawnv() arg 3 must not be empty"))?; + + if first.is_empty() { + return Err(vm.new_value_error("spawnv() arg 3 first element cannot be empty")); + } + + let argv_spawn: Vec<*const u16> = argv + .iter() + .map(|v| v.as_ptr()) + .chain(once(core::ptr::null())) + .collect(); + + let result = unsafe { suppress_iph!(_wspawnv(mode, path.as_ptr(), argv_spawn.as_ptr())) }; + if result == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(result) + } + } + + #[cfg(target_env = "msvc")] + #[pyfunction] + fn spawnve( + mode: i32, + path: OsPath, + argv: Either<PyListRef, PyTupleRef>, + env: PyDictRef, + vm: &VirtualMachine, + ) -> PyResult<intptr_t> { + use crate::function::FsPath; + use core::iter::once; + + let path = path.to_wide_cstring(vm)?; + + let argv = vm.extract_elements_with(argv.as_ref(), |obj| { + let fspath = FsPath::try_from_path_like(obj, true, vm)?; + fspath.to_wide_cstring(vm) + })?; + + let first = argv + .first() + .ok_or_else(|| vm.new_value_error("spawnve() arg 2 cannot be empty"))?; + + if first.is_empty() { + return Err(vm.new_value_error("spawnve() arg 2 first element cannot be empty")); + } + + let argv_spawn: Vec<*const u16> = argv + .iter() + .map(|v| v.as_ptr()) + .chain(once(core::ptr::null())) + .collect(); + + // Build environment strings as "KEY=VALUE\0" wide strings + let mut env_strings: Vec<widestring::WideCString> = Vec::new(); + for (key, value) in env.into_iter() { + let key = FsPath::try_from_path_like(key, true, vm)?; + let value = FsPath::try_from_path_like(value, true, vm)?; + let key_str = key.to_string_lossy(); + let value_str = value.to_string_lossy(); + + // Validate: empty key or '=' in key after position 0 + // (search from index 1 because on Windows starting '=' is allowed + // for defining hidden environment variables) + if key_str.is_empty() || key_str.get(1..).is_some_and(|s| s.contains('=')) { + return Err(vm.new_value_error("illegal environment variable name")); + } + + let env_str = format!("{}={}", key_str, value_str); + env_strings.push( + widestring::WideCString::from_os_str(&*std::ffi::OsString::from(env_str)) + .map_err(|err| err.to_pyexception(vm))?, + ); + } + + let envp: Vec<*const u16> = env_strings + .iter() + .map(|s| s.as_ptr()) + .chain(once(core::ptr::null())) + .collect(); + + let result = unsafe { + suppress_iph!(_wspawnve( + mode, + path.as_ptr(), + argv_spawn.as_ptr(), + envp.as_ptr() + )) + }; + if result == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(result) + } + } + + #[cfg(target_env = "msvc")] + #[pyfunction] + fn execv( + path: OsPath, + argv: Either<PyListRef, PyTupleRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + use core::iter::once; + + let make_widestring = + |s: &str| widestring::WideCString::from_os_str(s).map_err(|err| err.to_pyexception(vm)); + + let path = path.to_wide_cstring(vm)?; + + let argv = vm.extract_elements_with(argv.as_ref(), |obj| { + let arg = PyStrRef::try_from_object(vm, obj)?; + make_widestring(arg.expect_str()) + })?; + + let first = argv + .first() + .ok_or_else(|| vm.new_value_error("execv() arg 2 must not be empty"))?; + + if first.is_empty() { + return Err(vm.new_value_error("execv() arg 2 first element cannot be empty")); + } + + let argv_execv: Vec<*const u16> = argv + .iter() + .map(|v| v.as_ptr()) + .chain(once(core::ptr::null())) + .collect(); + + if (unsafe { suppress_iph!(_wexecv(path.as_ptr(), argv_execv.as_ptr())) } == -1) { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + #[cfg(target_env = "msvc")] + #[pyfunction] + fn execve( + path: OsPath, + argv: Either<PyListRef, PyTupleRef>, + env: ArgMapping, + vm: &VirtualMachine, + ) -> PyResult<()> { + use core::iter::once; + + let make_widestring = + |s: &str| widestring::WideCString::from_os_str(s).map_err(|err| err.to_pyexception(vm)); + + let path = path.to_wide_cstring(vm)?; + + let argv = vm.extract_elements_with(argv.as_ref(), |obj| { + let arg = PyStrRef::try_from_object(vm, obj)?; + make_widestring(arg.expect_str()) + })?; + + let first = argv + .first() + .ok_or_else(|| vm.new_value_error("execve: argv must not be empty"))?; + + if first.is_empty() { + return Err(vm.new_value_error("execve: argv first element cannot be empty")); + } + + let argv_execve: Vec<*const u16> = argv + .iter() + .map(|v| v.as_ptr()) + .chain(once(core::ptr::null())) + .collect(); + + let env = crate::stdlib::os::envobj_to_dict(env, vm)?; + // Build environment strings as "KEY=VALUE\0" wide strings + let mut env_strings: Vec<widestring::WideCString> = Vec::new(); + for (key, value) in env.into_iter() { + let key = PyStrRef::try_from_object(vm, key)?; + let value = PyStrRef::try_from_object(vm, value)?; + let key_str = key.expect_str(); + let value_str = value.expect_str(); + + // Validate: no null characters in key or value + if key_str.contains('\0') || value_str.contains('\0') { + return Err(vm.new_value_error("embedded null character")); + } + // Validate: empty key or '=' in key after position 0 + // (search from index 1 because on Windows starting '=' is allowed + // for defining hidden environment variables) + if key_str.is_empty() || key_str.get(1..).is_some_and(|s| s.contains('=')) { + return Err(vm.new_value_error("illegal environment variable name")); + } + + let env_str = format!("{}={}", key_str, value_str); + env_strings.push(make_widestring(&env_str)?); + } + + let envp: Vec<*const u16> = env_strings + .iter() + .map(|s| s.as_ptr()) + .chain(once(core::ptr::null())) + .collect(); + + if (unsafe { suppress_iph!(_wexecve(path.as_ptr(), argv_execve.as_ptr(), envp.as_ptr())) } + == -1) + { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + #[pyfunction] + fn _getfinalpathname(path: OsPath, vm: &VirtualMachine) -> PyResult { + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, GetFinalPathNameByHandleW, OPEN_EXISTING, + VOLUME_NAME_DOS, + }; + + let wide = path.to_wide_cstring(vm)?; + let handle = unsafe { + CreateFileW( + wide.as_ptr(), + 0, + 0, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + core::ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + + let mut buffer: Vec<u16> = vec![0; Foundation::MAX_PATH as usize]; + let result = loop { + let ret = unsafe { + GetFinalPathNameByHandleW( + handle, + buffer.as_mut_ptr(), + buffer.len() as u32, + VOLUME_NAME_DOS, + ) + }; + if ret == 0 { + let err = io::Error::last_os_error(); + let _ = unsafe { Foundation::CloseHandle(handle) }; + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + if (ret as usize) < buffer.len() { + let final_path = std::ffi::OsString::from_wide(&buffer[..ret as usize]); + break Ok(path.mode().process_path(final_path, vm)); + } + buffer.resize(ret as usize, 0); + }; + + unsafe { Foundation::CloseHandle(handle) }; + result + } + + #[pyfunction] + fn _getfullpathname(path: OsPath, vm: &VirtualMachine) -> PyResult { + let wpath = path.to_wide_cstring(vm)?; + let mut buffer = vec![0u16; Foundation::MAX_PATH as usize]; + let ret = unsafe { + FileSystem::GetFullPathNameW( + wpath.as_ptr(), + buffer.len() as _, + buffer.as_mut_ptr(), + core::ptr::null_mut(), + ) + }; + if ret == 0 { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)); + } + if ret as usize > buffer.len() { + buffer.resize(ret as usize, 0); + let ret = unsafe { + FileSystem::GetFullPathNameW( + wpath.as_ptr(), + buffer.len() as _, + buffer.as_mut_ptr(), + core::ptr::null_mut(), + ) + }; + if ret == 0 { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)); + } + } + let buffer = widestring::WideCString::from_vec_truncate(buffer); + Ok(path.mode().process_path(buffer.to_os_string(), vm)) + } + + #[pyfunction] + fn _getvolumepathname(path: OsPath, vm: &VirtualMachine) -> PyResult { + let wide = path.to_wide_cstring(vm)?; + let buflen = core::cmp::max(wide.len(), Foundation::MAX_PATH as usize); + if buflen > u32::MAX as usize { + return Err(vm.new_overflow_error("path too long")); + } + let mut buffer = vec![0u16; buflen]; + let ret = unsafe { + FileSystem::GetVolumePathNameW(wide.as_ptr(), buffer.as_mut_ptr(), buflen as _) + }; + if ret == 0 { + let err = io::Error::last_os_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + let buffer = widestring::WideCString::from_vec_truncate(buffer); + Ok(path.mode().process_path(buffer.to_os_string(), vm)) + } + + /// Implements _Py_skiproot logic for Windows paths + /// Returns (drive_size, root_size) where: + /// - drive_size: length of the drive/UNC portion + /// - root_size: length of the root separator (0 or 1) + fn skiproot(path: &[u16]) -> (usize, usize) { + let len = path.len(); + if len == 0 { + return (0, 0); + } + + const SEP: u16 = b'\\' as u16; + const ALTSEP: u16 = b'/' as u16; + const COLON: u16 = b':' as u16; + + let is_sep = |c: u16| c == SEP || c == ALTSEP; + let get = |i: usize| path.get(i).copied().unwrap_or(0); + + if is_sep(get(0)) { + if is_sep(get(1)) { + // UNC or device path: \\server\share or \\?\device + // Check for \\?\UNC\server\share + let idx = if len >= 8 + && get(2) == b'?' as u16 + && is_sep(get(3)) + && (get(4) == b'U' as u16 || get(4) == b'u' as u16) + && (get(5) == b'N' as u16 || get(5) == b'n' as u16) + && (get(6) == b'C' as u16 || get(6) == b'c' as u16) + && is_sep(get(7)) + { + 8 + } else { + 2 + }; + + // Find the end of server name + let mut i = idx; + while i < len && !is_sep(get(i)) { + i += 1; + } + + if i >= len { + // No share part: \\server + return (i, 0); + } + + // Skip separator and find end of share name + i += 1; + while i < len && !is_sep(get(i)) { + i += 1; + } + + // drive = \\server\share, root = \ (if present) + if i >= len { (i, 0) } else { (i, 1) } + } else { + // Relative path with root: \Windows + (0, 1) + } + } else if len >= 2 && get(1) == COLON { + // Drive letter path + if len >= 3 && is_sep(get(2)) { + // Absolute: X:\Windows + (2, 1) + } else { + // Relative with drive: X:Windows + (2, 0) + } + } else { + // Relative path: Windows + (0, 0) + } + } + + #[pyfunction] + fn _path_splitroot_ex(path: crate::PyObjectRef, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + // Handle path-like objects via os.fspath, but without null check (non_strict=True) + let path = if let Some(fspath) = vm.get_method(path.clone(), identifier!(vm, __fspath__)) { + fspath?.call((), vm)? + } else { + path + }; + + // Convert to wide string, validating UTF-8 for bytes input + let (wide, is_bytes): (Vec<u16>, bool) = if let Some(s) = path.downcast_ref::<PyStr>() { + // Use encode_wide which handles WTF-8 (including surrogates) + let wide: Vec<u16> = s.as_wtf8().encode_wide().collect(); + (wide, false) + } else if let Some(b) = path.downcast_ref::<PyBytes>() { + // On Windows, bytes must be valid UTF-8 - this raises UnicodeDecodeError if not + let s = core::str::from_utf8(b.as_bytes()).map_err(|e| { + vm.new_exception_msg( + vm.ctx.exceptions.unicode_decode_error.to_owned(), + format!( + "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte", + b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0), + e.valid_up_to() + ) + .into(), + ) + })?; + let wide: Vec<u16> = s.encode_utf16().collect(); + (wide, true) + } else { + return Err(vm.new_type_error(format!( + "expected str or bytes, not {}", + path.class().name() + ))); + }; + + // Normalize slashes for parsing + let normalized: Vec<u16> = wide + .iter() + .map(|&c| if c == b'/' as u16 { b'\\' as u16 } else { c }) + .collect(); + + let (drv_size, root_size) = skiproot(&normalized); + + // Return as bytes if input was bytes, preserving the original content + if is_bytes { + // Convert UTF-16 back to UTF-8 for bytes output + let drv = String::from_utf16(&wide[..drv_size]) + .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; + let root = String::from_utf16(&wide[drv_size..drv_size + root_size]) + .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; + let tail = String::from_utf16(&wide[drv_size + root_size..]) + .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytes(drv.into_bytes()).into(), + vm.ctx.new_bytes(root.into_bytes()).into(), + vm.ctx.new_bytes(tail.into_bytes()).into(), + ])) + } else { + // For str output, use WTF-8 to handle surrogates + let drv = Wtf8Buf::from_wide(&wide[..drv_size]); + let root = Wtf8Buf::from_wide(&wide[drv_size..drv_size + root_size]); + let tail = Wtf8Buf::from_wide(&wide[drv_size + root_size..]); + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_str(drv).into(), + vm.ctx.new_str(root).into(), + vm.ctx.new_str(tail).into(), + ])) + } + } + + #[pyfunction] + fn _path_splitroot(path: OsPath, _vm: &VirtualMachine) -> (Wtf8Buf, Wtf8Buf) { + let orig: Vec<_> = path.path.to_wide(); + if orig.is_empty() { + return (Wtf8Buf::new(), Wtf8Buf::new()); + } + let backslashed: Vec<_> = orig + .iter() + .copied() + .map(|c| if c == b'/' as u16 { b'\\' as u16 } else { c }) + .chain(core::iter::once(0)) // null-terminated + .collect(); + + let mut end: *const u16 = core::ptr::null(); + let hr = unsafe { + windows_sys::Win32::UI::Shell::PathCchSkipRoot(backslashed.as_ptr(), &mut end) + }; + if hr >= 0 { + assert!(!end.is_null()); + let len: usize = unsafe { end.offset_from(backslashed.as_ptr()) } + .try_into() + .expect("len must be non-negative"); + assert!( + len < backslashed.len(), // backslashed is null-terminated + "path: {:?} {} < {}", + std::path::PathBuf::from(std::ffi::OsString::from_wide(&backslashed)), + len, + backslashed.len() + ); + if len != 0 { + ( + Wtf8Buf::from_wide(&orig[..len]), + Wtf8Buf::from_wide(&orig[len..]), + ) + } else { + (Wtf8Buf::from_wide(&orig), Wtf8Buf::new()) + } + } else { + (Wtf8Buf::new(), Wtf8Buf::from_wide(&orig)) + } + } + + /// Normalize a wide-char path (faithful port of _Py_normpath_and_size). + /// Uses lastC tracking like the C implementation. + fn normpath_wide(path: &[u16]) -> Vec<u16> { + if path.is_empty() { + return vec![b'.' as u16]; + } + + const SEP: u16 = b'\\' as u16; + const ALTSEP: u16 = b'/' as u16; + const DOT: u16 = b'.' as u16; + + let is_sep = |c: u16| c == SEP || c == ALTSEP; + let sep_or_end = |input: &[u16], idx: usize| idx >= input.len() || is_sep(input[idx]); + + // Work on a mutable copy with normalized separators + let mut buf: Vec<u16> = path + .iter() + .map(|&c| if c == ALTSEP { SEP } else { c }) + .collect(); + + let (drv_size, root_size) = skiproot(&buf); + let prefix_len = drv_size + root_size; + + // p1 = read cursor, p2 = write cursor + let mut p1 = prefix_len; + let mut p2 = prefix_len; + let mut min_p2 = if prefix_len > 0 { prefix_len } else { 0 }; + let mut last_c: u16 = if prefix_len > 0 { + min_p2 = prefix_len - 1; + let c = buf[min_p2]; + // On Windows, if last char of prefix is not SEP, advance min_p2 + if c != SEP { + min_p2 = prefix_len; + } + c + } else { + 0 + }; + + // Skip leading ".\" after prefix + if p1 < buf.len() && buf[p1] == DOT && sep_or_end(&buf, p1 + 1) { + p1 += 1; + last_c = SEP; // treat as if we consumed a separator + while p1 < buf.len() && buf[p1] == SEP { + p1 += 1; + } + } + + while p1 < buf.len() { + let c = buf[p1]; + + if last_c == SEP { + if c == DOT { + let sep_at_1 = sep_or_end(&buf, p1 + 1); + let sep_at_2 = !sep_at_1 && sep_or_end(&buf, p1 + 2); + if sep_at_2 && buf[p1 + 1] == DOT { + // ".." component + let mut p3 = p2; + while p3 != min_p2 && buf[p3 - 1] == SEP { + p3 -= 1; + } + while p3 != min_p2 && buf[p3 - 1] != SEP { + p3 -= 1; + } + if p2 == min_p2 + || (buf[p3] == DOT + && p3 + 1 < buf.len() + && buf[p3 + 1] == DOT + && (p3 + 2 >= buf.len() || buf[p3 + 2] == SEP)) + { + // Previous segment is also ../ or at minimum + buf[p2] = DOT; + p2 += 1; + buf[p2] = DOT; + p2 += 1; + last_c = DOT; + } else if buf[p3] == SEP { + // Absolute path - absorb segment + p2 = p3 + 1; + // last_c stays SEP + } else { + p2 = p3; + // last_c stays SEP + } + p1 += 1; // skip second dot (first dot is current p1) + } else if sep_at_1 { + // "." component - skip + } else { + buf[p2] = c; + p2 += 1; + last_c = c; + } + } else if c == SEP { + // Collapse multiple separators - skip + } else { + buf[p2] = c; + p2 += 1; + last_c = c; + } + } else { + buf[p2] = c; + p2 += 1; + last_c = c; + } + + p1 += 1; + } + + // Null-terminate style: trim trailing separators + if p2 != min_p2 { + while p2 > min_p2 + 1 && buf[p2 - 1] == SEP { + p2 -= 1; + } + } + + buf.truncate(p2); + + if buf.is_empty() { vec![DOT] } else { buf } + } + + #[pyfunction] + fn _path_normpath(path: crate::PyObjectRef, vm: &VirtualMachine) -> PyResult { + // Handle path-like objects via os.fspath + let path = if let Some(fspath) = vm.get_method(path.clone(), identifier!(vm, __fspath__)) { + fspath?.call((), vm)? + } else { + path + }; + + let (wide, is_bytes): (Vec<u16>, bool) = if let Some(s) = path.downcast_ref::<PyStr>() { + let wide: Vec<u16> = s.as_wtf8().encode_wide().collect(); + (wide, false) + } else if let Some(b) = path.downcast_ref::<PyBytes>() { + let s = core::str::from_utf8(b.as_bytes()).map_err(|e| { + vm.new_exception_msg( + vm.ctx.exceptions.unicode_decode_error.to_owned(), + format!( + "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte", + b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0), + e.valid_up_to() + ) + .into(), + ) + })?; + let wide: Vec<u16> = s.encode_utf16().collect(); + (wide, true) + } else { + return Err(vm.new_type_error(format!( + "expected str or bytes, not {}", + path.class().name() + ))); + }; + + let normalized = normpath_wide(&wide); + + if is_bytes { + let s = String::from_utf16(&normalized) + .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; + Ok(vm.ctx.new_bytes(s.into_bytes()).into()) + } else { + let s = Wtf8Buf::from_wide(&normalized); + Ok(vm.ctx.new_str(s).into()) + } + } + + #[pyfunction] + fn _getdiskusage(path: OsPath, vm: &VirtualMachine) -> PyResult<(u64, u64)> { + use FileSystem::GetDiskFreeSpaceExW; + + let wpath = path.to_wide_cstring(vm)?; + let mut _free_to_me: u64 = 0; + let mut total: u64 = 0; + let mut free: u64 = 0; + let ret = + unsafe { GetDiskFreeSpaceExW(wpath.as_ptr(), &mut _free_to_me, &mut total, &mut free) }; + if ret != 0 { + return Ok((total, free)); + } + let err = io::Error::last_os_error(); + if err.raw_os_error() == Some(Foundation::ERROR_DIRECTORY as i32) + && let Some(parent) = path.as_ref().parent() + { + let parent = widestring::WideCString::from_os_str(parent).unwrap(); + + let ret = unsafe { + GetDiskFreeSpaceExW(parent.as_ptr(), &mut _free_to_me, &mut total, &mut free) + }; + + return if ret == 0 { + Err(err.to_pyexception(vm)) + } else { + Ok((total, free)) + }; + } + Err(err.to_pyexception(vm)) + } + + #[pyfunction] + fn get_handle_inheritable(handle: intptr_t, vm: &VirtualMachine) -> PyResult<bool> { + let mut flags = 0; + if unsafe { Foundation::GetHandleInformation(handle as _, &mut flags) } == 0 { + return Err(vm.new_last_os_error()); + } + Ok(flags & Foundation::HANDLE_FLAG_INHERIT != 0) + } + + #[pyfunction] + fn get_inheritable(fd: i32, vm: &VirtualMachine) -> PyResult<bool> { + let borrowed = unsafe { crt_fd::Borrowed::borrow_raw(fd) }; + let handle = crt_fd::as_handle(borrowed).map_err(|e| e.to_pyexception(vm))?; + get_handle_inheritable(handle.as_raw_handle() as _, vm) + } + + #[pyfunction] + fn getlogin(vm: &VirtualMachine) -> PyResult<String> { + let mut buffer = [0u16; 257]; + let mut size = buffer.len() as u32; + + let success = unsafe { + windows_sys::Win32::System::WindowsProgramming::GetUserNameW( + buffer.as_mut_ptr(), + &mut size, + ) + }; + + if success != 0 { + // Convert the buffer (which is UTF-16) to a Rust String + let username = std::ffi::OsString::from_wide(&buffer[..(size - 1) as usize]); + Ok(username.to_str().unwrap().to_string()) + } else { + Err(vm.new_os_error(format!("Error code: {success}"))) + } + } + + pub fn raw_set_handle_inheritable(handle: intptr_t, inheritable: bool) -> std::io::Result<()> { + let flags = if inheritable { + Foundation::HANDLE_FLAG_INHERIT + } else { + 0 + }; + let res = unsafe { + Foundation::SetHandleInformation(handle as _, Foundation::HANDLE_FLAG_INHERIT, flags) + }; + if res == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } + + #[pyfunction] + fn listdrives(vm: &VirtualMachine) -> PyResult<PyListRef> { + use windows_sys::Win32::Foundation::ERROR_MORE_DATA; + + let mut buffer = [0u16; 256]; + let len = + unsafe { FileSystem::GetLogicalDriveStringsW(buffer.len() as _, buffer.as_mut_ptr()) }; + if len == 0 { + return Err(vm.new_last_os_error()); + } + if len as usize >= buffer.len() { + return Err(std::io::Error::from_raw_os_error(ERROR_MORE_DATA as _).to_pyexception(vm)); + } + let drives: Vec<_> = buffer[..(len - 1) as usize] + .split(|&c| c == 0) + .map(|drive| vm.new_pyobj(String::from_utf16_lossy(drive))) + .collect(); + Ok(vm.ctx.new_list(drives)) + } + + #[pyfunction] + fn listvolumes(vm: &VirtualMachine) -> PyResult<PyListRef> { + use windows_sys::Win32::Foundation::ERROR_NO_MORE_FILES; + + let mut result = Vec::new(); + let mut buffer = [0u16; Foundation::MAX_PATH as usize + 1]; + + let find = unsafe { FileSystem::FindFirstVolumeW(buffer.as_mut_ptr(), buffer.len() as _) }; + if find == INVALID_HANDLE_VALUE { + return Err(vm.new_last_os_error()); + } + + loop { + // Find the null terminator + let len = buffer.iter().position(|&c| c == 0).unwrap_or(buffer.len()); + let volume = String::from_utf16_lossy(&buffer[..len]); + result.push(vm.new_pyobj(volume)); + + let ret = unsafe { + FileSystem::FindNextVolumeW(find, buffer.as_mut_ptr(), buffer.len() as _) + }; + if ret == 0 { + let err = io::Error::last_os_error(); + unsafe { FileSystem::FindVolumeClose(find) }; + if err.raw_os_error() == Some(ERROR_NO_MORE_FILES as i32) { + break; + } + return Err(err.to_pyexception(vm)); + } + } + + Ok(vm.ctx.new_list(result)) + } + + #[pyfunction] + fn listmounts(volume: OsPath, vm: &VirtualMachine) -> PyResult<PyListRef> { + use windows_sys::Win32::Foundation::ERROR_MORE_DATA; + + let wide = volume.to_wide_cstring(vm)?; + let mut buflen: u32 = Foundation::MAX_PATH + 1; + let mut buffer: Vec<u16> = vec![0; buflen as usize]; + + loop { + let success = unsafe { + FileSystem::GetVolumePathNamesForVolumeNameW( + wide.as_ptr(), + buffer.as_mut_ptr(), + buflen, + &mut buflen, + ) + }; + if success != 0 { + break; + } + let err = io::Error::last_os_error(); + if err.raw_os_error() == Some(ERROR_MORE_DATA as i32) { + buffer.resize(buflen as usize, 0); + continue; + } + return Err(err.to_pyexception(vm)); + } + + // Parse null-separated strings + let mut result = Vec::new(); + let mut start = 0; + for (i, &c) in buffer.iter().enumerate() { + if c == 0 { + if i > start { + let mount = String::from_utf16_lossy(&buffer[start..i]); + result.push(vm.new_pyobj(mount)); + } + start = i + 1; + if start < buffer.len() && buffer[start] == 0 { + break; // Double null = end + } + } + } + + Ok(vm.ctx.new_list(result)) + } + + #[pyfunction] + fn set_handle_inheritable( + handle: intptr_t, + inheritable: bool, + vm: &VirtualMachine, + ) -> PyResult<()> { + raw_set_handle_inheritable(handle, inheritable).map_err(|e| e.to_pyexception(vm)) + } + + #[derive(FromArgs)] + struct MkdirArgs<'a> { + #[pyarg(any)] + path: OsPath, + #[pyarg(any, default = 0o777)] + mode: i32, + #[pyarg(flatten)] + dir_fd: DirFd<'a, { _os::MKDIR_DIR_FD as usize }>, + } + + #[pyfunction] + fn mkdir(args: MkdirArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { + use windows_sys::Win32::Foundation::LocalFree; + use windows_sys::Win32::Security::Authorization::{ + ConvertStringSecurityDescriptorToSecurityDescriptorW, SDDL_REVISION_1, + }; + use windows_sys::Win32::Security::SECURITY_ATTRIBUTES; + + let [] = args.dir_fd.0; + let wide = args.path.to_wide_cstring(vm)?; + + // special case: mode 0o700 sets a protected ACL + let res = if args.mode == 0o700 { + let mut sec_attr = SECURITY_ATTRIBUTES { + nLength: core::mem::size_of::<SECURITY_ATTRIBUTES>() as u32, + lpSecurityDescriptor: core::ptr::null_mut(), + bInheritHandle: 0, + }; + // Set a discretionary ACL (D) that is protected (P) and includes + // inheritable (OICI) entries that allow (A) full control (FA) to + // SYSTEM (SY), Administrators (BA), and the owner (OW). + let sddl: Vec<u16> = "D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)\0" + .encode_utf16() + .collect(); + let convert_result = unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl.as_ptr(), + SDDL_REVISION_1, + &mut sec_attr.lpSecurityDescriptor, + core::ptr::null_mut(), + ) + }; + if convert_result == 0 { + return Err(vm.new_last_os_error()); + } + let res = + unsafe { FileSystem::CreateDirectoryW(wide.as_ptr(), &sec_attr as *const _ as _) }; + unsafe { LocalFree(sec_attr.lpSecurityDescriptor) }; + res + } else { + unsafe { FileSystem::CreateDirectoryW(wide.as_ptr(), core::ptr::null_mut()) } + }; + + if res == 0 { + return Err(vm.new_last_os_error()); + } + Ok(()) + } + + unsafe extern "C" { + fn _umask(mask: i32) -> i32; + } + + /// Close fd and convert error to PyException (PEP 446 cleanup) + #[cold] + fn close_fd_and_raise(fd: i32, err: std::io::Error, vm: &VirtualMachine) -> PyBaseExceptionRef { + let _ = unsafe { crt_fd::Owned::from_raw(fd) }; + err.to_pyexception(vm) + } + + #[pyfunction] + fn umask(mask: i32, vm: &VirtualMachine) -> PyResult<i32> { + let result = unsafe { _umask(mask) }; + if result < 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(result) + } + } + + #[pyfunction] + fn pipe(vm: &VirtualMachine) -> PyResult<(i32, i32)> { + use windows_sys::Win32::Security::SECURITY_ATTRIBUTES; + use windows_sys::Win32::System::Pipes::CreatePipe; + + let mut attr = SECURITY_ATTRIBUTES { + nLength: core::mem::size_of::<SECURITY_ATTRIBUTES>() as u32, + lpSecurityDescriptor: core::ptr::null_mut(), + bInheritHandle: 0, + }; + + let (read_handle, write_handle) = unsafe { + let mut read = MaybeUninit::<isize>::uninit(); + let mut write = MaybeUninit::<isize>::uninit(); + let res = CreatePipe( + read.as_mut_ptr() as *mut _, + write.as_mut_ptr() as *mut _, + &mut attr as *mut _, + 0, + ); + if res == 0 { + return Err(vm.new_last_os_error()); + } + (read.assume_init(), write.assume_init()) + }; + + // Convert handles to file descriptors + // O_NOINHERIT = 0x80 (MSVC CRT) + const O_NOINHERIT: i32 = 0x80; + let read_fd = unsafe { libc::open_osfhandle(read_handle, O_NOINHERIT) }; + let write_fd = unsafe { libc::open_osfhandle(write_handle, libc::O_WRONLY | O_NOINHERIT) }; + + if read_fd == -1 || write_fd == -1 { + unsafe { + Foundation::CloseHandle(read_handle as _); + Foundation::CloseHandle(write_handle as _); + } + return Err(vm.new_last_os_error()); + } + + Ok((read_fd, write_fd)) + } + + #[pyfunction] + fn getppid() -> u32 { + use windows_sys::Win32::System::Threading::{GetCurrentProcess, PROCESS_BASIC_INFORMATION}; + + type NtQueryInformationProcessFn = unsafe extern "system" fn( + process_handle: isize, + process_information_class: u32, + process_information: *mut core::ffi::c_void, + process_information_length: u32, + return_length: *mut u32, + ) -> i32; + + let ntdll = unsafe { + windows_sys::Win32::System::LibraryLoader::GetModuleHandleW(windows_sys::w!( + "ntdll.dll" + )) + }; + if ntdll.is_null() { + return 0; + } + + let func = unsafe { + windows_sys::Win32::System::LibraryLoader::GetProcAddress( + ntdll, + c"NtQueryInformationProcess".as_ptr() as *const u8, + ) + }; + let Some(func) = func else { + return 0; + }; + let nt_query: NtQueryInformationProcessFn = unsafe { core::mem::transmute(func) }; + + let mut info: PROCESS_BASIC_INFORMATION = unsafe { core::mem::zeroed() }; + + let status = unsafe { + nt_query( + GetCurrentProcess() as isize, + 0, // ProcessBasicInformation + &mut info as *mut _ as *mut core::ffi::c_void, + core::mem::size_of::<PROCESS_BASIC_INFORMATION>() as u32, + core::ptr::null_mut(), + ) + }; + + if status >= 0 + && info.InheritedFromUniqueProcessId != 0 + && info.InheritedFromUniqueProcessId < u32::MAX as usize + { + info.InheritedFromUniqueProcessId as u32 + } else { + 0 + } + } + + #[pyfunction] + fn dup(fd: i32, vm: &VirtualMachine) -> PyResult<i32> { + let fd2 = unsafe { suppress_iph!(libc::dup(fd)) }; + if fd2 < 0 { + return Err(vm.new_last_errno_error()); + } + let borrowed = unsafe { crt_fd::Borrowed::borrow_raw(fd2) }; + let handle = crt_fd::as_handle(borrowed).map_err(|e| close_fd_and_raise(fd2, e, vm))?; + raw_set_handle_inheritable(handle.as_raw_handle() as _, false) + .map_err(|e| close_fd_and_raise(fd2, e, vm))?; + Ok(fd2) + } + + #[derive(FromArgs)] + struct Dup2Args { + #[pyarg(positional)] + fd: i32, + #[pyarg(positional)] + fd2: i32, + #[pyarg(any, default = true)] + inheritable: bool, + } + + #[pyfunction] + fn dup2(args: Dup2Args, vm: &VirtualMachine) -> PyResult<i32> { + let result = unsafe { suppress_iph!(libc::dup2(args.fd, args.fd2)) }; + if result < 0 { + return Err(vm.new_last_errno_error()); + } + if !args.inheritable { + let borrowed = unsafe { crt_fd::Borrowed::borrow_raw(args.fd2) }; + let handle = + crt_fd::as_handle(borrowed).map_err(|e| close_fd_and_raise(args.fd2, e, vm))?; + raw_set_handle_inheritable(handle.as_raw_handle() as _, false) + .map_err(|e| close_fd_and_raise(args.fd2, e, vm))?; + } + Ok(args.fd2) + } + + /// Windows-specific readlink that preserves \\?\ prefix for junctions + /// returns the substitute name from reparse data which includes the prefix + #[pyfunction] + fn readlink(path: OsPath, vm: &VirtualMachine) -> PyResult { + use crate::common::windows::ToWideString; + use windows_sys::Win32::Foundation::CloseHandle; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, + FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, OPEN_EXISTING, + }; + use windows_sys::Win32::System::IO::DeviceIoControl; + use windows_sys::Win32::System::Ioctl::FSCTL_GET_REPARSE_POINT; + + let mode = path.mode(); + let wide_path = path.as_ref().to_wide_with_nul(); + + // Open the file/directory with reparse point flag + let handle = unsafe { + CreateFileW( + wide_path.as_ptr(), + 0, // No access needed, just reading reparse data + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + core::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, + core::ptr::null_mut(), + ) + }; + + if handle == INVALID_HANDLE_VALUE { + return Err(OSErrorBuilder::with_filename( + &io::Error::last_os_error(), + path.clone(), + vm, + )); + } + + // Buffer for reparse data - MAXIMUM_REPARSE_DATA_BUFFER_SIZE is 16384 + const BUFFER_SIZE: usize = 16384; + let mut buffer = vec![0u8; BUFFER_SIZE]; + let mut bytes_returned: u32 = 0; + + let result = unsafe { + DeviceIoControl( + handle, + FSCTL_GET_REPARSE_POINT, + core::ptr::null(), + 0, + buffer.as_mut_ptr() as *mut _, + BUFFER_SIZE as u32, + &mut bytes_returned, + core::ptr::null_mut(), + ) + }; + + unsafe { CloseHandle(handle) }; + + if result == 0 { + return Err(OSErrorBuilder::with_filename( + &io::Error::last_os_error(), + path.clone(), + vm, + )); + } + + // Parse the reparse data buffer + // REPARSE_DATA_BUFFER structure: + // DWORD ReparseTag + // WORD ReparseDataLength + // WORD Reserved + // For symlinks/junctions (IO_REPARSE_TAG_SYMLINK/MOUNT_POINT): + // WORD SubstituteNameOffset + // WORD SubstituteNameLength + // WORD PrintNameOffset + // WORD PrintNameLength + // (For symlinks only: DWORD Flags) + // PathBuffer... + + let reparse_tag = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]); + + // Check if it's a symlink or mount point (junction) + use windows_sys::Win32::System::SystemServices::{ + IO_REPARSE_TAG_MOUNT_POINT, IO_REPARSE_TAG_SYMLINK, + }; + + let (substitute_offset, substitute_length, path_buffer_start) = + if reparse_tag == IO_REPARSE_TAG_SYMLINK { + // Symlink has Flags field (4 bytes) before PathBuffer + let sub_offset = u16::from_le_bytes([buffer[8], buffer[9]]) as usize; + let sub_length = u16::from_le_bytes([buffer[10], buffer[11]]) as usize; + // PathBuffer starts at offset 20 (after Flags at offset 16) + (sub_offset, sub_length, 20usize) + } else if reparse_tag == IO_REPARSE_TAG_MOUNT_POINT { + // Mount point (junction) has no Flags field + let sub_offset = u16::from_le_bytes([buffer[8], buffer[9]]) as usize; + let sub_length = u16::from_le_bytes([buffer[10], buffer[11]]) as usize; + // PathBuffer starts at offset 16 + (sub_offset, sub_length, 16usize) + } else { + return Err(vm.new_value_error("not a symbolic link")); + }; + + // Extract the substitute name + let path_start = path_buffer_start + substitute_offset; + let path_end = path_start + substitute_length; + + if path_end > buffer.len() { + return Err(vm.new_os_error("Invalid reparse data".to_owned())); + } + + // Convert from UTF-16LE + let path_slice = &buffer[path_start..path_end]; + let wide_chars: Vec<u16> = path_slice + .chunks_exact(2) + .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) + .collect(); + + let mut wide_chars = wide_chars; + // For mount points (junctions), the substitute name typically starts with \??\ + // Convert this to \\?\ + if wide_chars.len() > 4 + && wide_chars[0] == b'\\' as u16 + && wide_chars[1] == b'?' as u16 + && wide_chars[2] == b'?' as u16 + && wide_chars[3] == b'\\' as u16 + { + wide_chars[1] = b'\\' as u16; + } + + let result_path = std::ffi::OsString::from_wide(&wide_chars); + + Ok(mode.process_path(std::path::PathBuf::from(result_path), vm)) + } + + pub(crate) fn support_funcs() -> Vec<SupportFunc> { + Vec::new() + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs new file mode 100644 index 00000000000..03c5e33de76 --- /dev/null +++ b/crates/vm/src/stdlib/os.rs @@ -0,0 +1,2393 @@ +// spell-checker:disable + +use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + builtins::{PyModule, PySet}, + common::crt_fd, + convert::{IntoPyException, ToPyException, ToPyObject}, + function::{ArgumentError, FromArgs, FuncArgs}, +}; +use std::{fs, io, path::Path}; + +pub(crate) fn fs_metadata<P: AsRef<Path>>( + path: P, + follow_symlink: bool, +) -> io::Result<fs::Metadata> { + if follow_symlink { + fs::metadata(path.as_ref()) + } else { + fs::symlink_metadata(path.as_ref()) + } +} + +#[allow(dead_code)] +#[derive(FromArgs, Default)] +pub struct TargetIsDirectory { + #[pyarg(any, default = false)] + pub(crate) target_is_directory: bool, +} + +cfg_if::cfg_if! { + if #[cfg(all(any(unix, target_os = "wasi"), not(target_os = "redox")))] { + use libc::AT_FDCWD; + } else { + const AT_FDCWD: i32 = -100; + } +} +const DEFAULT_DIR_FD: crt_fd::Borrowed<'static> = unsafe { crt_fd::Borrowed::borrow_raw(AT_FDCWD) }; + +// XXX: AVAILABLE should be a bool, but we can't yet have it as a bool and just cast it to usize +#[derive(Copy, Clone, PartialEq, Eq)] +pub struct DirFd<'fd, const AVAILABLE: usize>(pub(crate) [crt_fd::Borrowed<'fd>; AVAILABLE]); + +impl<const AVAILABLE: usize> Default for DirFd<'_, AVAILABLE> { + fn default() -> Self { + Self([DEFAULT_DIR_FD; AVAILABLE]) + } +} + +// not used on all platforms +#[allow(unused)] +impl<'fd> DirFd<'fd, 1> { + #[inline(always)] + pub(crate) fn get_opt(self) -> Option<crt_fd::Borrowed<'fd>> { + let [fd] = self.0; + (fd != DEFAULT_DIR_FD).then_some(fd) + } + + #[inline] + pub(crate) fn raw_opt(self) -> Option<i32> { + self.get_opt().map(|fd| fd.as_raw()) + } + + #[inline(always)] + pub(crate) const fn get(self) -> crt_fd::Borrowed<'fd> { + let [fd] = self.0; + fd + } +} + +impl<const AVAILABLE: usize> FromArgs for DirFd<'_, AVAILABLE> { + fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { + let fd = match args.take_keyword("dir_fd") { + Some(o) if vm.is_none(&o) => Ok(DEFAULT_DIR_FD), + None => Ok(DEFAULT_DIR_FD), + Some(o) => { + warn_if_bool_fd(&o, vm).map_err(Into::<ArgumentError>::into)?; + let fd = o.try_index_opt(vm).unwrap_or_else(|| { + Err(vm.new_type_error(format!( + "argument should be integer or None, not {}", + o.class().name() + ))) + })?; + let fd = fd.try_to_primitive(vm)?; + unsafe { crt_fd::Borrowed::try_borrow_raw(fd) } + } + }; + if AVAILABLE == 0 && fd.as_ref().is_ok_and(|&fd| fd != DEFAULT_DIR_FD) { + return Err(vm + .new_not_implemented_error("dir_fd unavailable on this platform") + .into()); + } + let fd = fd.map_err(|e| e.to_pyexception(vm))?; + Ok(Self([fd; AVAILABLE])) + } +} + +#[derive(FromArgs)] +pub(super) struct FollowSymlinks( + #[pyarg(named, name = "follow_symlinks", default = true)] pub bool, +); + +#[cfg(not(windows))] +fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { + rustpython_common::os::bytes_as_os_str(b) + .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8")) +} + +pub(crate) fn warn_if_bool_fd(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + use crate::class::StaticType; + if obj + .class() + .is(crate::builtins::bool_::PyBool::static_type()) + { + crate::stdlib::_warnings::warn( + vm.ctx.exceptions.runtime_warning, + "bool is used as a file descriptor".to_owned(), + 1, + vm, + )?; + } + Ok(()) +} + +impl TryFromObject for crt_fd::Owned { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + warn_if_bool_fd(&obj, vm)?; + let fd = crt_fd::Raw::try_from_object(vm, obj)?; + unsafe { crt_fd::Owned::try_from_raw(fd) }.map_err(|e| e.into_pyexception(vm)) + } +} + +impl TryFromObject for crt_fd::Borrowed<'_> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + warn_if_bool_fd(&obj, vm)?; + let fd = crt_fd::Raw::try_from_object(vm, obj)?; + unsafe { crt_fd::Borrowed::try_borrow_raw(fd) }.map_err(|e| e.into_pyexception(vm)) + } +} + +impl ToPyObject for crt_fd::Owned { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + self.into_raw().to_pyobject(vm) + } +} + +impl ToPyObject for crt_fd::Borrowed<'_> { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + self.as_raw().to_pyobject(vm) + } +} + +#[pymodule(sub)] +pub(super) mod _os { + use super::{DirFd, FollowSymlinks, SupportFunc}; + #[cfg(windows)] + use crate::common::windows::ToWideString; + #[cfg(any(unix, windows))] + use crate::utils::ToCString; + use crate::{ + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + builtins::{ + PyBytesRef, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, + }, + common::{ + crt_fd, + fileutils::StatStruct, + lock::{OnceCell, PyRwLock}, + suppress_iph, + }, + convert::{IntoPyException, ToPyObject}, + exceptions::{OSErrorBuilder, ToOSErrorBuilder}, + function::{ArgBytesLike, ArgMemoryBuffer, FsPath, FuncArgs, OptionalArg}, + ospath::{OsPath, OsPathOrFd, OutputMode, PathConverter}, + protocol::PyIterReturn, + recursion::ReprGuard, + types::{Destructor, IterNext, Iterable, PyStructSequence, Representable, SelfIter}, + vm::VirtualMachine, + }; + use core::time::Duration; + use crossbeam_utils::atomic::AtomicCell; + use itertools::Itertools; + use rustpython_common::wtf8::Wtf8Buf; + use std::{env, fs, fs::OpenOptions, io, path::PathBuf, time::SystemTime}; + + const OPEN_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); + pub(crate) const MKDIR_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); + const STAT_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); + const UTIME_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); + pub(crate) const SYMLINK_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); + pub(crate) const UNLINK_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); + const RMDIR_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); + const SCANDIR_FD: bool = cfg!(all(unix, not(target_os = "redox"))); + + #[pyattr] + use libc::{O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY}; + + #[pyattr] + pub(crate) const F_OK: u8 = 0; + #[pyattr] + pub(crate) const R_OK: u8 = 1 << 2; + #[pyattr] + pub(crate) const W_OK: u8 = 1 << 1; + #[pyattr] + pub(crate) const X_OK: u8 = 1 << 0; + + // ST_RDONLY and ST_NOSUID flags for statvfs + #[cfg(all(unix, not(target_os = "redox")))] + #[pyattr] + const ST_RDONLY: libc::c_ulong = libc::ST_RDONLY; + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyattr] + const ST_NOSUID: libc::c_ulong = libc::ST_NOSUID; + + #[pyfunction] + fn close(fileno: crt_fd::Owned) -> io::Result<()> { + crt_fd::close(fileno) + } + + #[pyfunction] + fn closerange(fd_low: i32, fd_high: i32) { + for fileno in fd_low..fd_high { + if let Ok(fd) = unsafe { crt_fd::Owned::try_from_raw(fileno) } { + drop(fd); + } + } + } + + #[cfg(any(unix, windows, target_os = "wasi"))] + #[derive(FromArgs)] + struct OpenArgs<'fd> { + path: OsPath, + flags: i32, + #[pyarg(any, default)] + mode: Option<i32>, + #[pyarg(flatten)] + dir_fd: DirFd<'fd, { OPEN_DIR_FD as usize }>, + } + + #[pyfunction] + fn open(args: OpenArgs<'_>, vm: &VirtualMachine) -> PyResult<crt_fd::Owned> { + os_open(args.path, args.flags, args.mode, args.dir_fd, vm) + } + + #[cfg(any(unix, windows, target_os = "wasi"))] + pub(crate) fn os_open( + name: OsPath, + flags: i32, + mode: Option<i32>, + dir_fd: DirFd<'_, { OPEN_DIR_FD as usize }>, + vm: &VirtualMachine, + ) -> PyResult<crt_fd::Owned> { + let mode = mode.unwrap_or(0o777); + #[cfg(windows)] + let fd = { + let [] = dir_fd.0; + let name = name.to_wide_cstring(vm)?; + let flags = flags | libc::O_NOINHERIT; + crt_fd::wopen(&name, flags, mode) + }; + #[cfg(not(windows))] + let fd = { + let name = name.clone().into_cstring(vm)?; + #[cfg(not(target_os = "wasi"))] + let flags = flags | libc::O_CLOEXEC; + #[cfg(not(target_os = "redox"))] + if let Some(dir_fd) = dir_fd.get_opt() { + crt_fd::openat(dir_fd, &name, flags, mode) + } else { + crt_fd::open(&name, flags, mode) + } + #[cfg(target_os = "redox")] + { + let [] = dir_fd.0; + crt_fd::open(&name, flags, mode) + } + }; + fd.map_err(|err| OSErrorBuilder::with_filename_from_errno(&err, name, vm)) + } + + #[pyfunction] + fn fsync(fd: crt_fd::Borrowed<'_>) -> io::Result<()> { + crt_fd::fsync(fd) + } + + #[pyfunction] + fn read(fd: crt_fd::Borrowed<'_>, n: usize, vm: &VirtualMachine) -> PyResult<PyBytesRef> { + let mut buffer = vec![0u8; n]; + loop { + match vm.allow_threads(|| crt_fd::read(fd, &mut buffer)) { + Ok(n) => { + buffer.truncate(n); + return Ok(vm.ctx.new_bytes(buffer)); + } + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + Err(e) => return Err(e.into_pyexception(vm)), + } + } + } + + #[pyfunction] + fn readinto( + fd: crt_fd::Borrowed<'_>, + buffer: ArgMemoryBuffer, + vm: &VirtualMachine, + ) -> PyResult<usize> { + buffer.with_ref(|buf| { + loop { + match vm.allow_threads(|| crt_fd::read(fd, buf)) { + Ok(n) => return Ok(n), + Err(e) if e.raw_os_error() == Some(libc::EINTR) => { + vm.check_signals()?; + continue; + } + Err(e) => return Err(e.into_pyexception(vm)), + } + } + }) + } + + #[pyfunction] + fn write( + fd: crt_fd::Borrowed<'_>, + data: ArgBytesLike, + vm: &VirtualMachine, + ) -> io::Result<usize> { + data.with_ref(|b| vm.allow_threads(|| crt_fd::write(fd, b))) + } + + #[cfg(not(windows))] + #[pyfunction] + fn mkdir( + path: OsPath, + mode: OptionalArg<i32>, + dir_fd: DirFd<'_, { MKDIR_DIR_FD as usize }>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mode = mode.unwrap_or(0o777); + let c_path = path.clone().into_cstring(vm)?; + #[cfg(not(target_os = "redox"))] + if let Some(fd) = dir_fd.raw_opt() { + let res = unsafe { libc::mkdirat(fd, c_path.as_ptr(), mode as _) }; + return if res < 0 { + let err = crate::common::os::errno_io_error(); + Err(OSErrorBuilder::with_filename(&err, path, vm)) + } else { + Ok(()) + }; + } + #[cfg(target_os = "redox")] + let [] = dir_fd.0; + let res = unsafe { libc::mkdir(c_path.as_ptr(), mode as _) }; + if res < 0 { + let err = crate::common::os::errno_io_error(); + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + Ok(()) + } + + #[pyfunction] + fn mkdirs(path: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let os_path = vm.fsencode(&path)?; + fs::create_dir_all(&*os_path).map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(not(windows))] + #[pyfunction] + fn rmdir( + path: OsPath, + dir_fd: DirFd<'_, { RMDIR_DIR_FD as usize }>, + vm: &VirtualMachine, + ) -> PyResult<()> { + #[cfg(not(target_os = "redox"))] + if let Some(fd) = dir_fd.raw_opt() { + let c_path = path.clone().into_cstring(vm)?; + let res = unsafe { libc::unlinkat(fd, c_path.as_ptr(), libc::AT_REMOVEDIR) }; + return if res < 0 { + let err = crate::common::os::errno_io_error(); + Err(OSErrorBuilder::with_filename(&err, path, vm)) + } else { + Ok(()) + }; + } + #[cfg(target_os = "redox")] + let [] = dir_fd.0; + fs::remove_dir(&path).map_err(|err| OSErrorBuilder::with_filename(&err, path, vm)) + } + + #[cfg(windows)] + #[pyfunction] + fn rmdir(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult<()> { + let [] = dir_fd.0; + fs::remove_dir(&path).map_err(|err| OSErrorBuilder::with_filename(&err, path, vm)) + } + + const LISTDIR_FD: bool = cfg!(all(unix, not(target_os = "redox"))); + + #[pyfunction] + fn listdir( + path: OptionalArg<Option<OsPathOrFd<'_>>>, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + let path = path + .flatten() + .unwrap_or_else(|| OsPathOrFd::Path(OsPath::new_str("."))); + let list = match path { + OsPathOrFd::Path(path) => { + let dir_iter = match fs::read_dir(&path) { + Ok(iter) => iter, + Err(err) => { + return Err(OSErrorBuilder::with_filename(&err, path, vm)); + } + }; + let mode = path.mode(); + dir_iter + .map(|entry| match entry { + Ok(entry_path) => Ok(mode.process_path(entry_path.file_name(), vm)), + Err(err) => Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)), + }) + .collect::<PyResult<_>>()? + } + OsPathOrFd::Fd(fno) => { + #[cfg(not(all(unix, not(target_os = "redox"))))] + { + let _ = fno; + return Err( + vm.new_not_implemented_error("can't pass fd to listdir on this platform") + ); + } + #[cfg(all(unix, not(target_os = "redox")))] + { + use rustpython_common::os::ffi::OsStrExt; + use std::os::unix::io::IntoRawFd; + let new_fd = nix::unistd::dup(fno).map_err(|e| e.into_pyexception(vm))?; + let raw_fd = new_fd.into_raw_fd(); + let dir = OwnedDir::from_fd(raw_fd).map_err(|e| { + unsafe { libc::close(raw_fd) }; + e.into_pyexception(vm) + })?; + // OwnedDir::drop calls rewinddir (reset to start) then closedir. + let mut list = Vec::new(); + loop { + nix::errno::Errno::clear(); + let entry = unsafe { libc::readdir(dir.as_ptr()) }; + if entry.is_null() { + let err = nix::errno::Errno::last(); + if err != nix::errno::Errno::UnknownErrno { + return Err(io::Error::from(err).into_pyexception(vm)); + } + break; + } + let fname = unsafe { core::ffi::CStr::from_ptr((*entry).d_name.as_ptr()) } + .to_bytes(); + match fname { + b"." | b".." => continue, + _ => list.push( + OutputMode::String + .process_path(std::ffi::OsStr::from_bytes(fname), vm), + ), + } + } + list + } + } + }; + Ok(list) + } + + #[cfg(not(windows))] + fn env_bytes_as_bytes(obj: &crate::function::Either<PyStrRef, PyBytesRef>) -> &[u8] { + match obj { + crate::function::Either::A(s) => s.as_bytes(), + crate::function::Either::B(b) => b.as_bytes(), + } + } + + #[cfg(windows)] + unsafe extern "C" { + fn _wputenv(envstring: *const u16) -> libc::c_int; + } + + /// Check if wide string length exceeds Windows environment variable limit. + #[cfg(windows)] + fn check_env_var_len(wide_len: usize, vm: &VirtualMachine) -> PyResult<()> { + use crate::common::windows::_MAX_ENV; + if wide_len > _MAX_ENV + 1 { + return Err(vm.new_value_error(format!( + "the environment variable is longer than {_MAX_ENV} characters", + ))); + } + Ok(()) + } + + #[cfg(windows)] + #[pyfunction] + fn putenv(key: PyStrRef, value: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let key_str = key.expect_str(); + let value_str = value.expect_str(); + // Search from index 1 because on Windows starting '=' is allowed for + // defining hidden environment variables. + if key_str.is_empty() + || key_str.get(1..).is_some_and(|s| s.contains('=')) + || key_str.contains('\0') + || value_str.contains('\0') + { + return Err(vm.new_value_error("illegal environment variable name")); + } + let env_str = format!("{}={}", key_str, value_str); + let wide = env_str.to_wide_with_nul(); + check_env_var_len(wide.len(), vm)?; + + // Use _wputenv like CPython (not SetEnvironmentVariableW) to update CRT environ + let result = unsafe { suppress_iph!(_wputenv(wide.as_ptr())) }; + if result != 0 { + return Err(vm.new_last_errno_error()); + } + Ok(()) + } + + #[cfg(not(windows))] + #[pyfunction] + fn putenv( + key: crate::function::Either<PyStrRef, PyBytesRef>, + value: crate::function::Either<PyStrRef, PyBytesRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let key = env_bytes_as_bytes(&key); + let value = env_bytes_as_bytes(&value); + if key.contains(&b'\0') || value.contains(&b'\0') { + return Err(vm.new_value_error("embedded null byte")); + } + if key.is_empty() || key.contains(&b'=') { + return Err(vm.new_value_error("illegal environment variable name")); + } + let key = super::bytes_as_os_str(key, vm)?; + let value = super::bytes_as_os_str(value, vm)?; + // SAFETY: requirements forwarded from the caller + unsafe { env::set_var(key, value) }; + Ok(()) + } + + #[cfg(windows)] + #[pyfunction] + fn unsetenv(key: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + let key_str = key.expect_str(); + // Search from index 1 because on Windows starting '=' is allowed for + // defining hidden environment variables. + if key_str.is_empty() + || key_str.get(1..).is_some_and(|s| s.contains('=')) + || key_str.contains('\0') + { + return Err(vm.new_value_error("illegal environment variable name")); + } + // "key=" to unset (empty value removes the variable) + let env_str = format!("{}=", key_str); + let wide = env_str.to_wide_with_nul(); + check_env_var_len(wide.len(), vm)?; + + // Use _wputenv like CPython (not SetEnvironmentVariableW) to update CRT environ + let result = unsafe { suppress_iph!(_wputenv(wide.as_ptr())) }; + if result != 0 { + return Err(vm.new_last_errno_error()); + } + Ok(()) + } + + #[cfg(not(windows))] + #[pyfunction] + fn unsetenv( + key: crate::function::Either<PyStrRef, PyBytesRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let key = env_bytes_as_bytes(&key); + if key.contains(&b'\0') { + return Err(vm.new_value_error("embedded null byte")); + } + if key.is_empty() || key.contains(&b'=') { + let x = vm.new_errno_error( + 22, + format!( + "Invalid argument: {}", + core::str::from_utf8(key).unwrap_or("<bytes encoding failure>") + ), + ); + + return Err(x.upcast()); + } + let key = super::bytes_as_os_str(key, vm)?; + // SAFETY: requirements forwarded from the caller + unsafe { env::remove_var(key) }; + Ok(()) + } + + #[pyfunction] + fn readlink(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult { + let mode = path.mode(); + let [] = dir_fd.0; + let path = + fs::read_link(&path).map_err(|err| OSErrorBuilder::with_filename(&err, path, vm))?; + Ok(mode.process_path(path, vm)) + } + + #[pyattr] + #[pyclass(name)] + #[derive(Debug, PyPayload)] + struct DirEntry { + file_name: std::ffi::OsString, + pathval: PathBuf, + file_type: io::Result<fs::FileType>, + /// dirent d_type value, used when file_type is unavailable (fd-based scandir) + #[cfg(unix)] + d_type: Option<u8>, + /// Parent directory fd for fd-based scandir, used for fstatat + #[cfg(not(any(windows, target_os = "redox")))] + dir_fd: Option<crt_fd::Raw>, + mode: OutputMode, + stat: OnceCell<PyObjectRef>, + lstat: OnceCell<PyObjectRef>, + #[cfg(unix)] + ino: AtomicCell<u64>, + #[cfg(windows)] + ino: AtomicCell<Option<u128>>, + #[cfg(not(any(unix, windows)))] + ino: AtomicCell<Option<u64>>, + } + + #[pyclass(flags(DISALLOW_INSTANTIATION), with(Representable))] + impl DirEntry { + #[pygetset] + fn name(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.mode.process_path(&self.file_name, vm)) + } + + #[pygetset] + fn path(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.mode.process_path(&self.pathval, vm)) + } + + /// Build the DirFd to use for stat calls. + /// If this entry was produced by fd-based scandir, use the stored dir_fd + /// so that fstatat(dir_fd, name, ...) is used instead of stat(full_path). + fn stat_dir_fd(&self) -> DirFd<'_, { STAT_DIR_FD as usize }> { + #[cfg(not(any(windows, target_os = "redox")))] + if let Some(raw_fd) = self.dir_fd { + // Safety: the fd came from os.open() and is borrowed for + // the lifetime of this DirEntry reference. + let borrowed = unsafe { crt_fd::Borrowed::borrow_raw(raw_fd) }; + return DirFd([borrowed; STAT_DIR_FD as usize]); + } + DirFd::default() + } + + /// Stat-based mode test fallback. Uses fstatat when dir_fd is available. + #[cfg(unix)] + fn test_mode_via_stat( + &self, + follow_symlinks: bool, + mode_bits: u32, + vm: &VirtualMachine, + ) -> PyResult<bool> { + match self.stat(self.stat_dir_fd(), FollowSymlinks(follow_symlinks), vm) { + Ok(stat_obj) => { + let st_mode: i32 = stat_obj.get_attr("st_mode", vm)?.try_into_value(vm)?; + #[allow(clippy::unnecessary_cast)] + Ok((st_mode as u32 & libc::S_IFMT as u32) == mode_bits) + } + Err(e) => { + if e.fast_isinstance(vm.ctx.exceptions.file_not_found_error) { + Ok(false) + } else { + Err(e) + } + } + } + } + + #[pymethod] + fn is_dir(&self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult<bool> { + if let Ok(file_type) = &self.file_type + && (!follow_symlinks.0 || !file_type.is_symlink()) + { + return Ok(file_type.is_dir()); + } + #[cfg(unix)] + if let Some(dt) = self.d_type { + let is_symlink = dt == libc::DT_LNK; + let need_stat = dt == libc::DT_UNKNOWN || (follow_symlinks.0 && is_symlink); + if !need_stat { + return Ok(dt == libc::DT_DIR); + } + } + #[cfg(unix)] + return self.test_mode_via_stat(follow_symlinks.0, libc::S_IFDIR as _, vm); + #[cfg(not(unix))] + match super::fs_metadata(&self.pathval, follow_symlinks.0) { + Ok(meta) => Ok(meta.is_dir()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e.into_pyexception(vm)), + } + } + + #[pymethod] + fn is_file(&self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult<bool> { + if let Ok(file_type) = &self.file_type + && (!follow_symlinks.0 || !file_type.is_symlink()) + { + return Ok(file_type.is_file()); + } + #[cfg(unix)] + if let Some(dt) = self.d_type { + let is_symlink = dt == libc::DT_LNK; + let need_stat = dt == libc::DT_UNKNOWN || (follow_symlinks.0 && is_symlink); + if !need_stat { + return Ok(dt == libc::DT_REG); + } + } + #[cfg(unix)] + return self.test_mode_via_stat(follow_symlinks.0, libc::S_IFREG as _, vm); + #[cfg(not(unix))] + match super::fs_metadata(&self.pathval, follow_symlinks.0) { + Ok(meta) => Ok(meta.is_file()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e.into_pyexception(vm)), + } + } + + #[pymethod] + fn is_symlink(&self, vm: &VirtualMachine) -> PyResult<bool> { + if let Ok(file_type) = &self.file_type { + return Ok(file_type.is_symlink()); + } + #[cfg(unix)] + if let Some(dt) = self.d_type + && dt != libc::DT_UNKNOWN + { + return Ok(dt == libc::DT_LNK); + } + #[cfg(unix)] + return self.test_mode_via_stat(false, libc::S_IFLNK as _, vm); + #[cfg(not(unix))] + match &self.file_type { + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => { + use crate::convert::ToPyException; + Err(e.to_pyexception(vm)) + } + Ok(_) => Ok(false), + } + } + + #[pymethod] + fn stat( + &self, + dir_fd: DirFd<'_, { STAT_DIR_FD as usize }>, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + // Use stored dir_fd if the caller didn't provide one + let effective_dir_fd = if dir_fd == DirFd::default() { + self.stat_dir_fd() + } else { + dir_fd + }; + let do_stat = |follow_symlinks| { + stat( + OsPath { + path: self.pathval.as_os_str().to_owned(), + origin: None, + } + .into(), + effective_dir_fd, + FollowSymlinks(follow_symlinks), + vm, + ) + }; + let lstat = || match self.lstat.get() { + Some(val) => Ok(val), + None => { + let val = do_stat(false)?; + let _ = self.lstat.set(val); + Ok(self.lstat.get().unwrap()) + } + }; + let stat = if follow_symlinks.0 { + match self.stat.get() { + Some(val) => val, + None => { + let val = if self.is_symlink(vm)? { + do_stat(true)? + } else { + lstat()?.clone() + }; + let _ = self.stat.set(val); + self.stat.get().unwrap() + } + } + } else { + lstat()? + }; + Ok(stat.clone()) + } + + #[cfg(windows)] + #[pymethod] + fn inode(&self, vm: &VirtualMachine) -> PyResult<u128> { + match self.ino.load() { + Some(ino) => Ok(ino), + None => { + let stat = stat_inner( + OsPath::new_str(self.pathval.as_os_str()).into(), + DirFd::default(), + FollowSymlinks(false), + ) + .map_err(|e| e.into_pyexception(vm))? + .ok_or_else(|| crate::exceptions::cstring_error(vm))?; + // On Windows, combine st_ino and st_ino_high into 128-bit value + #[cfg(windows)] + let ino: u128 = stat.st_ino as u128 | ((stat.st_ino_high as u128) << 64); + #[cfg(not(windows))] + let ino: u128 = stat.st_ino as u128; + // Err(T) means other thread set `ino` at the mean time which is safe to ignore + let _ = self.ino.compare_exchange(None, Some(ino)); + Ok(ino) + } + } + } + + #[cfg(unix)] + #[pymethod] + fn inode(&self, _vm: &VirtualMachine) -> PyResult<u64> { + Ok(self.ino.load()) + } + + #[cfg(not(any(unix, windows)))] + #[pymethod] + fn inode(&self, _vm: &VirtualMachine) -> PyResult<Option<u64>> { + Ok(self.ino.load()) + } + + #[cfg(not(windows))] + #[pymethod] + const fn is_junction(&self, _vm: &VirtualMachine) -> PyResult<bool> { + Ok(false) + } + + #[cfg(windows)] + #[pymethod] + fn is_junction(&self, _vm: &VirtualMachine) -> PyResult<bool> { + Ok(junction::exists(self.pathval.clone()).unwrap_or(false)) + } + + #[pymethod] + fn __fspath__(&self, vm: &VirtualMachine) -> PyResult { + self.path(vm) + } + + #[pyclassmethod] + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyGenericAlias { + PyGenericAlias::from_args(cls, args, vm) + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'DirEntry' object")) + } + } + + impl Representable for DirEntry { + #[inline] + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + let name = match zelf.as_object().get_attr("name", vm) { + Ok(name) => Some(name), + Err(e) + if e.fast_isinstance(vm.ctx.exceptions.attribute_error) + || e.fast_isinstance(vm.ctx.exceptions.value_error) => + { + None + } + Err(e) => return Err(e), + }; + if let Some(name) = name { + if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { + let repr = name.repr(vm)?; + let mut result = Wtf8Buf::from(format!("<{} ", zelf.class())); + result.push_wtf8(repr.as_wtf8()); + result.push_char('>'); + Ok(result) + } else { + Err(vm.new_runtime_error(format!( + "reentrant call inside {}.__repr__", + zelf.class() + ))) + } + } else { + Ok(Wtf8Buf::from(format!("<{}>", zelf.class()))) + } + } + } + #[pyattr] + #[pyclass(name = "ScandirIter")] + #[derive(Debug, PyPayload)] + struct ScandirIterator { + entries: PyRwLock<Option<fs::ReadDir>>, + mode: OutputMode, + } + + #[pyclass(flags(DISALLOW_INSTANTIATION), with(Destructor, IterNext, Iterable))] + impl ScandirIterator { + #[pymethod] + fn close(&self) { + let entryref: &mut Option<fs::ReadDir> = &mut self.entries.write(); + let _dropped = entryref.take(); + } + + #[pymethod] + const fn __enter__(zelf: PyRef<Self>) -> PyRef<Self> { + zelf + } + + #[pymethod] + fn __exit__(zelf: PyRef<Self>, _args: FuncArgs) { + zelf.close() + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'ScandirIterator' object")) + } + } + impl Destructor for ScandirIterator { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + // Emit ResourceWarning if the iterator is not yet exhausted/closed + if zelf.entries.read().is_some() { + let _ = crate::stdlib::_warnings::warn( + vm.ctx.exceptions.resource_warning, + format!("unclosed scandir iterator {:?}", zelf.as_object()), + 1, + vm, + ); + zelf.close(); + } + Ok(()) + } + } + impl SelfIter for ScandirIterator {} + impl IterNext for ScandirIterator { + fn next(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let entryref: &mut Option<fs::ReadDir> = &mut zelf.entries.write(); + + match entryref { + None => Ok(PyIterReturn::StopIteration(None)), + Some(inner) => match inner.next() { + Some(entry) => match entry { + Ok(entry) => { + #[cfg(unix)] + let ino = { + use std::os::unix::fs::DirEntryExt; + entry.ino() + }; + // TODO: wasi is nightly + // #[cfg(target_os = "wasi")] + // let ino = { + // use std::os::wasi::fs::DirEntryExt; + // entry.ino() + // }; + #[cfg(not(unix))] + let ino = None; + + let pathval = entry.path(); + + // On Windows, pre-cache lstat from directory entry metadata + // This allows stat() to return cached data even if file is removed + #[cfg(windows)] + let lstat = { + let cell = OnceCell::new(); + if let Ok(stat_struct) = + crate::windows::win32_xstat(pathval.as_os_str(), false) + { + let stat_obj = + StatResultData::from_stat(&stat_struct, vm).to_pyobject(vm); + let _ = cell.set(stat_obj); + } + cell + }; + #[cfg(not(windows))] + let lstat = OnceCell::new(); + + Ok(PyIterReturn::Return( + DirEntry { + file_name: entry.file_name(), + pathval, + file_type: entry.file_type(), + #[cfg(unix)] + d_type: None, + #[cfg(not(any(windows, target_os = "redox")))] + dir_fd: None, + mode: zelf.mode, + lstat, + stat: OnceCell::new(), + ino: AtomicCell::new(ino), + } + .into_ref(&vm.ctx) + .into(), + )) + } + Err(err) => Err(err.into_pyexception(vm)), + }, + None => { + let _dropped = entryref.take(); + Ok(PyIterReturn::StopIteration(None)) + } + }, + } + } + } + + /// Wrapper around a raw `libc::DIR*` for fd-based scandir. + #[cfg(all(unix, not(target_os = "redox")))] + struct OwnedDir(core::ptr::NonNull<libc::DIR>); + + #[cfg(all(unix, not(target_os = "redox")))] + impl OwnedDir { + fn from_fd(fd: crt_fd::Raw) -> io::Result<Self> { + let ptr = unsafe { libc::fdopendir(fd) }; + core::ptr::NonNull::new(ptr) + .map(OwnedDir) + .ok_or_else(io::Error::last_os_error) + } + + fn as_ptr(&self) -> *mut libc::DIR { + self.0.as_ptr() + } + } + + #[cfg(all(unix, not(target_os = "redox")))] + impl Drop for OwnedDir { + fn drop(&mut self) { + unsafe { + libc::rewinddir(self.0.as_ptr()); + libc::closedir(self.0.as_ptr()); + } + } + } + + #[cfg(all(unix, not(target_os = "redox")))] + impl core::fmt::Debug for OwnedDir { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_tuple("OwnedDir").field(&self.0).finish() + } + } + + // Safety: OwnedDir wraps a *mut libc::DIR. All access is synchronized + // through the PyMutex in ScandirIteratorFd. + #[cfg(all(unix, not(target_os = "redox")))] + unsafe impl Send for OwnedDir {} + #[cfg(all(unix, not(target_os = "redox")))] + unsafe impl Sync for OwnedDir {} + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyattr] + #[pyclass(name = "ScandirIter")] + #[derive(Debug, PyPayload)] + struct ScandirIteratorFd { + dir: crate::common::lock::PyMutex<Option<OwnedDir>>, + /// The original fd passed to scandir(), stored in DirEntry for fstatat + orig_fd: crt_fd::Raw, + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyclass(flags(DISALLOW_INSTANTIATION), with(Destructor, IterNext, Iterable))] + impl ScandirIteratorFd { + #[pymethod] + fn close(&self) { + let _dropped = self.dir.lock().take(); + } + + #[pymethod] + const fn __enter__(zelf: PyRef<Self>) -> PyRef<Self> { + zelf + } + + #[pymethod] + fn __exit__(zelf: PyRef<Self>, _args: FuncArgs) { + zelf.close() + } + + #[pymethod] + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot pickle 'ScandirIterator' object")) + } + } + + #[cfg(all(unix, not(target_os = "redox")))] + impl Destructor for ScandirIteratorFd { + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { + if zelf.dir.lock().is_some() { + let _ = crate::stdlib::_warnings::warn( + vm.ctx.exceptions.resource_warning, + format!("unclosed scandir iterator {:?}", zelf.as_object()), + 1, + vm, + ); + zelf.close(); + } + Ok(()) + } + } + + #[cfg(all(unix, not(target_os = "redox")))] + impl SelfIter for ScandirIteratorFd {} + + #[cfg(all(unix, not(target_os = "redox")))] + impl IterNext for ScandirIteratorFd { + fn next(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + use rustpython_common::os::ffi::OsStrExt; + let mut guard = zelf.dir.lock(); + let dir = match guard.as_mut() { + None => return Ok(PyIterReturn::StopIteration(None)), + Some(dir) => dir, + }; + loop { + nix::errno::Errno::clear(); + let entry = unsafe { + let ptr = libc::readdir(dir.as_ptr()); + if ptr.is_null() { + let err = nix::errno::Errno::last(); + if err != nix::errno::Errno::UnknownErrno { + return Err(io::Error::from(err).into_pyexception(vm)); + } + drop(guard.take()); + return Ok(PyIterReturn::StopIteration(None)); + } + &*ptr + }; + let fname = unsafe { core::ffi::CStr::from_ptr(entry.d_name.as_ptr()) }.to_bytes(); + if fname == b"." || fname == b".." { + continue; + } + let file_name = std::ffi::OsString::from(std::ffi::OsStr::from_bytes(fname)); + let pathval = PathBuf::from(&file_name); + #[cfg(target_os = "freebsd")] + let ino = entry.d_fileno; + #[cfg(not(target_os = "freebsd"))] + let ino = entry.d_ino; + let d_type = entry.d_type; + return Ok(PyIterReturn::Return( + DirEntry { + file_name, + pathval, + file_type: Err(io::Error::other( + "file_type unavailable for fd-based scandir", + )), + d_type: if d_type == libc::DT_UNKNOWN { + None + } else { + Some(d_type) + }, + dir_fd: Some(zelf.orig_fd), + mode: OutputMode::String, + lstat: OnceCell::new(), + stat: OnceCell::new(), + ino: AtomicCell::new(ino as _), + } + .into_ref(&vm.ctx) + .into(), + )); + } + } + } + + #[pyfunction] + fn scandir(path: OptionalArg<Option<OsPathOrFd<'_>>>, vm: &VirtualMachine) -> PyResult { + let path = path + .flatten() + .unwrap_or_else(|| OsPathOrFd::Path(OsPath::new_str("."))); + match path { + OsPathOrFd::Path(path) => { + let entries = fs::read_dir(&path.path) + .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?; + Ok(ScandirIterator { + entries: PyRwLock::new(Some(entries)), + mode: path.mode(), + } + .into_ref(&vm.ctx) + .into()) + } + OsPathOrFd::Fd(fno) => { + #[cfg(not(all(unix, not(target_os = "redox"))))] + { + let _ = fno; + Err(vm.new_not_implemented_error("can't pass fd to scandir on this platform")) + } + #[cfg(all(unix, not(target_os = "redox")))] + { + use std::os::unix::io::IntoRawFd; + // closedir() closes the fd, so duplicate it first + let new_fd = nix::unistd::dup(fno).map_err(|e| e.into_pyexception(vm))?; + let raw_fd = new_fd.into_raw_fd(); + let dir = OwnedDir::from_fd(raw_fd).map_err(|e| { + // fdopendir failed, close the dup'd fd + unsafe { libc::close(raw_fd) }; + e.into_pyexception(vm) + })?; + Ok(ScandirIteratorFd { + dir: crate::common::lock::PyMutex::new(Some(dir)), + orig_fd: fno.as_raw(), + } + .into_ref(&vm.ctx) + .into()) + } + } + } + } + + #[derive(Debug, FromArgs)] + #[pystruct_sequence_data] + struct StatResultData { + pub st_mode: PyIntRef, + pub st_ino: PyIntRef, + pub st_dev: PyIntRef, + pub st_nlink: PyIntRef, + pub st_uid: PyIntRef, + pub st_gid: PyIntRef, + pub st_size: PyIntRef, + // Indices 7-9: integer seconds + #[cfg_attr(target_env = "musl", allow(deprecated))] + #[pyarg(positional, default)] + #[pystruct_sequence(unnamed)] + pub st_atime_int: libc::time_t, + #[cfg_attr(target_env = "musl", allow(deprecated))] + #[pyarg(positional, default)] + #[pystruct_sequence(unnamed)] + pub st_mtime_int: libc::time_t, + #[cfg_attr(target_env = "musl", allow(deprecated))] + #[pyarg(positional, default)] + #[pystruct_sequence(unnamed)] + pub st_ctime_int: libc::time_t, + // Float time attributes + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_atime: f64, + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_mtime: f64, + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_ctime: f64, + // Nanosecond attributes + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_atime_ns: i128, + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_mtime_ns: i128, + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_ctime_ns: i128, + // Unix-specific attributes + #[cfg(not(windows))] + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_blksize: i64, + #[cfg(not(windows))] + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_blocks: i64, + #[cfg(windows)] + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_reparse_tag: u32, + #[cfg(windows)] + #[pyarg(any, default)] + #[pystruct_sequence(skip)] + pub st_file_attributes: u32, + } + + impl StatResultData { + fn from_stat(stat: &StatStruct, vm: &VirtualMachine) -> Self { + let (atime, mtime, ctime); + #[cfg(any(unix, windows))] + #[cfg(not(any(target_os = "netbsd", target_os = "wasi")))] + { + atime = (stat.st_atime, stat.st_atime_nsec); + mtime = (stat.st_mtime, stat.st_mtime_nsec); + ctime = (stat.st_ctime, stat.st_ctime_nsec); + } + #[cfg(target_os = "netbsd")] + { + atime = (stat.st_atime, stat.st_atimensec); + mtime = (stat.st_mtime, stat.st_mtimensec); + ctime = (stat.st_ctime, stat.st_ctimensec); + } + #[cfg(target_os = "wasi")] + { + atime = (stat.st_atim.tv_sec, stat.st_atim.tv_nsec); + mtime = (stat.st_mtim.tv_sec, stat.st_mtim.tv_nsec); + ctime = (stat.st_ctim.tv_sec, stat.st_ctim.tv_nsec); + } + + const NANOS_PER_SEC: u32 = 1_000_000_000; + let to_f64 = |(s, ns)| (s as f64) + (ns as f64) / (NANOS_PER_SEC as f64); + let to_ns = |(s, ns)| s as i128 * NANOS_PER_SEC as i128 + ns as i128; + + #[cfg(windows)] + let st_reparse_tag = stat.st_reparse_tag; + #[cfg(windows)] + let st_file_attributes = stat.st_file_attributes; + + // On Windows, combine st_ino and st_ino_high into a 128-bit value + // like _pystat_l128_from_l64_l64 + #[cfg(windows)] + let st_ino: u128 = stat.st_ino as u128 | ((stat.st_ino_high as u128) << 64); + #[cfg(not(windows))] + let st_ino = stat.st_ino; + + #[cfg(not(windows))] + #[allow(clippy::useless_conversion, reason = "needed for 32-bit platforms")] + let st_blksize = i64::from(stat.st_blksize); + #[cfg(not(windows))] + #[allow(clippy::useless_conversion, reason = "needed for 32-bit platforms")] + let st_blocks = i64::from(stat.st_blocks); + + Self { + st_mode: vm.ctx.new_pyref(stat.st_mode), + st_ino: vm.ctx.new_pyref(st_ino), + st_dev: vm.ctx.new_pyref(stat.st_dev), + st_nlink: vm.ctx.new_pyref(stat.st_nlink), + st_uid: vm.ctx.new_pyref(stat.st_uid), + st_gid: vm.ctx.new_pyref(stat.st_gid), + st_size: vm.ctx.new_pyref(stat.st_size), + st_atime_int: atime.0, + st_mtime_int: mtime.0, + st_ctime_int: ctime.0, + st_atime: to_f64(atime), + st_mtime: to_f64(mtime), + st_ctime: to_f64(ctime), + st_atime_ns: to_ns(atime), + st_mtime_ns: to_ns(mtime), + st_ctime_ns: to_ns(ctime), + #[cfg(not(windows))] + st_blksize, + #[cfg(not(windows))] + st_blocks, + #[cfg(windows)] + st_reparse_tag, + #[cfg(windows)] + st_file_attributes, + } + } + } + + #[pyattr] + #[pystruct_sequence(name = "stat_result", module = "os", data = "StatResultData")] + struct PyStatResult; + + #[pyclass(with(PyStructSequence))] + impl PyStatResult { + #[pyslot] + fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let flatten_args = |r: &[PyObjectRef]| { + let mut vec_args = Vec::from(r); + loop { + if let Ok(obj) = vec_args.iter().exactly_one() { + match obj.downcast_ref::<PyTuple>() { + Some(t) => { + vec_args = Vec::from(t.as_slice()); + } + None => { + return vec_args; + } + } + } else { + return vec_args; + } + } + }; + + let args: FuncArgs = flatten_args(&args.args).into(); + + let stat: StatResultData = args.bind(vm)?; + Ok(stat.to_pyobject(vm)) + } + } + + #[cfg(windows)] + fn stat_inner( + file: OsPathOrFd<'_>, + dir_fd: DirFd<'_, { STAT_DIR_FD as usize }>, + follow_symlinks: FollowSymlinks, + ) -> io::Result<Option<StatStruct>> { + // TODO: replicate CPython's win32_xstat + let [] = dir_fd.0; + match file { + OsPathOrFd::Path(path) => crate::windows::win32_xstat(&path.path, follow_symlinks.0), + OsPathOrFd::Fd(fd) => crate::common::fileutils::fstat(fd), + } + .map(Some) + } + + #[cfg(not(windows))] + fn stat_inner( + file: OsPathOrFd<'_>, + dir_fd: DirFd<'_, { STAT_DIR_FD as usize }>, + follow_symlinks: FollowSymlinks, + ) -> io::Result<Option<StatStruct>> { + let mut stat = core::mem::MaybeUninit::uninit(); + let ret = match file { + OsPathOrFd::Path(path) => { + use rustpython_common::os::ffi::OsStrExt; + let path = path.as_ref().as_os_str().as_bytes(); + let path = match alloc::ffi::CString::new(path) { + Ok(x) => x, + Err(_) => return Ok(None), + }; + + #[cfg(not(target_os = "redox"))] + let fstatat_ret = dir_fd.raw_opt().map(|dir_fd| { + let flags = if follow_symlinks.0 { + 0 + } else { + libc::AT_SYMLINK_NOFOLLOW + }; + unsafe { libc::fstatat(dir_fd, path.as_ptr(), stat.as_mut_ptr(), flags) } + }); + #[cfg(target_os = "redox")] + let ([], fstatat_ret) = (dir_fd.0, None); + + fstatat_ret.unwrap_or_else(|| { + if follow_symlinks.0 { + unsafe { libc::stat(path.as_ptr(), stat.as_mut_ptr()) } + } else { + unsafe { libc::lstat(path.as_ptr(), stat.as_mut_ptr()) } + } + }) + } + OsPathOrFd::Fd(fd) => unsafe { libc::fstat(fd.as_raw(), stat.as_mut_ptr()) }, + }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + Ok(Some(unsafe { stat.assume_init() })) + } + + #[pyfunction] + #[pyfunction(name = "fstat")] + fn stat( + file: OsPathOrFd<'_>, + dir_fd: DirFd<'_, { STAT_DIR_FD as usize }>, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + let stat = stat_inner(file.clone(), dir_fd, follow_symlinks) + .map_err(|err| OSErrorBuilder::with_filename(&err, file, vm))? + .ok_or_else(|| crate::exceptions::cstring_error(vm))?; + Ok(StatResultData::from_stat(&stat, vm).to_pyobject(vm)) + } + + #[pyfunction] + fn lstat( + file: OsPath, + dir_fd: DirFd<'_, { STAT_DIR_FD as usize }>, + vm: &VirtualMachine, + ) -> PyResult { + stat(file.into(), dir_fd, FollowSymlinks(false), vm) + } + + fn curdir_inner(vm: &VirtualMachine) -> PyResult<PathBuf> { + env::current_dir().map_err(|err| err.into_pyexception(vm)) + } + + #[pyfunction] + fn getcwd(vm: &VirtualMachine) -> PyResult { + Ok(OutputMode::String.process_path(curdir_inner(vm)?, vm)) + } + + #[pyfunction] + fn getcwdb(vm: &VirtualMachine) -> PyResult { + Ok(OutputMode::Bytes.process_path(curdir_inner(vm)?, vm)) + } + + #[pyfunction] + fn chdir(path: OsPath, vm: &VirtualMachine) -> PyResult<()> { + env::set_current_dir(&path.path) + .map_err(|err| OSErrorBuilder::with_filename(&err, path, vm))?; + + #[cfg(windows)] + { + // win32_wchdir() + + // On Windows, set the per-drive CWD environment variable (=X:) + // This is required for GetFullPathNameW to work correctly with drive-relative paths + + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::System::Environment::SetEnvironmentVariableW; + + if let Ok(cwd) = env::current_dir() { + let cwd_str = cwd.as_os_str(); + let mut cwd_wide: Vec<u16> = cwd_str.encode_wide().collect(); + + // Check for UNC-like paths (\\server\share or //server/share) + // wcsncmp(new_path, L"\\\\", 2) == 0 || wcsncmp(new_path, L"//", 2) == 0 + let is_unc_like_path = cwd_wide.len() >= 2 + && ((cwd_wide[0] == b'\\' as u16 && cwd_wide[1] == b'\\' as u16) + || (cwd_wide[0] == b'/' as u16 && cwd_wide[1] == b'/' as u16)); + + if !is_unc_like_path { + // Create env var name "=X:" where X is the drive letter + let env_name: [u16; 4] = [b'=' as u16, cwd_wide[0], b':' as u16, 0]; + cwd_wide.push(0); // null-terminate the path + unsafe { + SetEnvironmentVariableW(env_name.as_ptr(), cwd_wide.as_ptr()); + } + } + } + } + + Ok(()) + } + + #[pyfunction] + fn fspath(path: PyObjectRef, vm: &VirtualMachine) -> PyResult<FsPath> { + FsPath::try_from_path_like(path, false, vm) + } + + #[pyfunction] + #[pyfunction(name = "replace")] + fn rename(src: PyObjectRef, dst: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let src = PathConverter::new() + .function("rename") + .argument("src") + .try_path(src, vm)?; + let dst = PathConverter::new() + .function("rename") + .argument("dst") + .try_path(dst, vm)?; + + fs::rename(&src.path, &dst.path).map_err(|err| { + let builder = err.to_os_error_builder(vm); + let builder = builder.filename(src.filename(vm)); + let builder = builder.filename2(dst.filename(vm)); + builder.build(vm).upcast() + }) + } + + #[pyfunction] + fn getpid(vm: &VirtualMachine) -> PyObjectRef { + let pid = if cfg!(target_arch = "wasm32") { + // Return an arbitrary value, greater than 1 which is special. + // The value 42 is picked from wasi-libc + // https://github.com/WebAssembly/wasi-libc/blob/wasi-sdk-21/libc-bottom-half/getpid/getpid.c + 42 + } else { + std::process::id() + }; + vm.ctx.new_int(pid).into() + } + + #[pyfunction] + fn cpu_count(vm: &VirtualMachine) -> PyObjectRef { + let cpu_count = num_cpus::get(); + vm.ctx.new_int(cpu_count).into() + } + + #[pyfunction] + fn _exit(code: i32) { + std::process::exit(code) + } + + #[pyfunction] + fn abort() { + unsafe extern "C" { + fn abort(); + } + unsafe { abort() } + } + + #[pyfunction] + fn urandom(size: isize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + if size < 0 { + return Err(vm.new_value_error("negative argument not allowed")); + } + let mut buf = vec![0u8; size as usize]; + getrandom::fill(&mut buf).map_err(|e| io::Error::from(e).into_pyexception(vm))?; + Ok(buf) + } + + #[pyfunction] + pub fn isatty(fd: i32) -> bool { + unsafe { suppress_iph!(libc::isatty(fd)) != 0 } + } + + #[pyfunction] + pub fn lseek( + fd: crt_fd::Borrowed<'_>, + position: crt_fd::Offset, + how: i32, + vm: &VirtualMachine, + ) -> PyResult<crt_fd::Offset> { + #[cfg(not(windows))] + let res = unsafe { suppress_iph!(libc::lseek(fd.as_raw(), position, how)) }; + #[cfg(windows)] + let res = unsafe { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem; + let handle = crt_fd::as_handle(fd).map_err(|e| e.into_pyexception(vm))?; + let mut distance_to_move: [i32; 2] = core::mem::transmute(position); + let ret = FileSystem::SetFilePointer( + handle.as_raw_handle(), + distance_to_move[0], + &mut distance_to_move[1], + how as _, + ); + if ret == FileSystem::INVALID_SET_FILE_POINTER { + -1 + } else { + distance_to_move[0] = ret as _; + core::mem::transmute::<[i32; 2], i64>(distance_to_move) + } + }; + if res < 0 { + Err(vm.new_last_os_error()) + } else { + Ok(res) + } + } + + #[derive(FromArgs)] + struct LinkArgs { + #[pyarg(any)] + src: OsPath, + #[pyarg(any)] + dst: OsPath, + #[pyarg(named, name = "follow_symlinks", optional)] + follow_symlinks: OptionalArg<bool>, + } + + #[pyfunction] + fn link(args: LinkArgs, vm: &VirtualMachine) -> PyResult<()> { + let LinkArgs { + src, + dst, + follow_symlinks, + } = args; + + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + let src_cstr = alloc::ffi::CString::new(src.path.as_os_str().as_bytes()) + .map_err(|_| vm.new_value_error("embedded null byte"))?; + let dst_cstr = alloc::ffi::CString::new(dst.path.as_os_str().as_bytes()) + .map_err(|_| vm.new_value_error("embedded null byte"))?; + + let follow = follow_symlinks.into_option().unwrap_or(true); + let flags = if follow { libc::AT_SYMLINK_FOLLOW } else { 0 }; + + let ret = unsafe { + libc::linkat( + libc::AT_FDCWD, + src_cstr.as_ptr(), + libc::AT_FDCWD, + dst_cstr.as_ptr(), + flags, + ) + }; + + if ret != 0 { + let err = std::io::Error::last_os_error(); + let builder = err.to_os_error_builder(vm); + let builder = builder.filename(src.filename(vm)); + let builder = builder.filename2(dst.filename(vm)); + return Err(builder.build(vm).upcast()); + } + + Ok(()) + } + + #[cfg(not(unix))] + { + let src_path = match follow_symlinks.into_option() { + Some(true) => { + // Explicit follow_symlinks=True: resolve symlinks + fs::canonicalize(&src.path).unwrap_or_else(|_| PathBuf::from(src.path.clone())) + } + Some(false) | None => { + // Default or explicit no-follow: native hard_link behavior + PathBuf::from(src.path.clone()) + } + }; + + fs::hard_link(&src_path, &dst.path).map_err(|err| { + let builder = err.to_os_error_builder(vm); + let builder = builder.filename(src.filename(vm)); + let builder = builder.filename2(dst.filename(vm)); + builder.build(vm).upcast() + }) + } + } + + #[cfg(any(unix, windows))] + #[pyfunction] + fn system(command: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> { + let cstr = command.to_cstring(vm)?; + let x = unsafe { libc::system(cstr.as_ptr()) }; + Ok(x) + } + + #[derive(FromArgs)] + struct UtimeArgs<'fd> { + path: OsPath, + #[pyarg(any, default)] + times: Option<PyTupleRef>, + #[pyarg(named, default)] + ns: Option<PyTupleRef>, + #[pyarg(flatten)] + dir_fd: DirFd<'fd, { UTIME_DIR_FD as usize }>, + #[pyarg(flatten)] + follow_symlinks: FollowSymlinks, + } + + #[pyfunction] + fn utime(args: UtimeArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { + let parse_tup = |tup: &Py<PyTuple>| -> Option<(PyObjectRef, PyObjectRef)> { + if tup.len() != 2 { + None + } else { + Some((tup[0].clone(), tup[1].clone())) + } + }; + let (acc, modif) = match (args.times, args.ns) { + (Some(t), None) => { + let (a, m) = parse_tup(&t).ok_or_else(|| { + vm.new_type_error("utime: 'times' must be either a tuple of two ints or None") + })?; + (a.try_into_value(vm)?, m.try_into_value(vm)?) + } + (None, Some(ns)) => { + let (a, m) = parse_tup(&ns) + .ok_or_else(|| vm.new_type_error("utime: 'ns' must be a tuple of two ints"))?; + let ns_in_sec: PyObjectRef = vm.ctx.new_int(1_000_000_000).into(); + let ns_to_dur = |obj: PyObjectRef| { + let divmod = vm._divmod(&obj, &ns_in_sec)?; + let (div, rem) = divmod + .downcast_ref::<PyTuple>() + .and_then(parse_tup) + .ok_or_else(|| { + vm.new_type_error(format!( + "{}.__divmod__() must return a 2-tuple, not {}", + obj.class().name(), + divmod.class().name() + )) + })?; + let secs = div.try_index(vm)?.try_to_primitive(vm)?; + let ns = rem.try_index(vm)?.try_to_primitive(vm)?; + Ok(Duration::new(secs, ns)) + }; + // TODO: do validation to make sure this doesn't.. underflow? + (ns_to_dur(a)?, ns_to_dur(m)?) + } + (None, None) => { + let now = SystemTime::now(); + let now = now.duration_since(SystemTime::UNIX_EPOCH).unwrap(); + (now, now) + } + (Some(_), Some(_)) => { + return Err(vm.new_value_error( + "utime: you may specify either 'times' or 'ns' but not both", + )); + } + }; + utime_impl(args.path, acc, modif, args.dir_fd, args.follow_symlinks, vm) + } + + fn utime_impl( + path: OsPath, + acc: Duration, + modif: Duration, + dir_fd: DirFd<'_, { UTIME_DIR_FD as usize }>, + _follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult<()> { + #[cfg(any(target_os = "wasi", unix))] + { + #[cfg(not(target_os = "redox"))] + { + let path_for_err = path.clone(); + let path = path.into_cstring(vm)?; + + let ts = |d: Duration| libc::timespec { + tv_sec: d.as_secs() as _, + tv_nsec: d.subsec_nanos() as _, + }; + let times = [ts(acc), ts(modif)]; + + let ret = unsafe { + libc::utimensat( + dir_fd.get().as_raw(), + path.as_ptr(), + times.as_ptr(), + if _follow_symlinks.0 { + 0 + } else { + libc::AT_SYMLINK_NOFOLLOW + }, + ) + }; + if ret < 0 { + Err(OSErrorBuilder::with_filename( + &io::Error::last_os_error(), + path_for_err, + vm, + )) + } else { + Ok(()) + } + } + #[cfg(target_os = "redox")] + { + let [] = dir_fd.0; + + let tv = |d: Duration| libc::timeval { + tv_sec: d.as_secs() as _, + tv_usec: d.as_micros() as _, + }; + nix::sys::stat::utimes(path.as_ref(), &tv(acc).into(), &tv(modif).into()) + .map_err(|err| err.into_pyexception(vm)) + } + } + #[cfg(windows)] + { + use std::{fs::OpenOptions, os::windows::prelude::*}; + type DWORD = u32; + use windows_sys::Win32::{Foundation::FILETIME, Storage::FileSystem}; + + let [] = dir_fd.0; + + if !_follow_symlinks.0 { + return Err(vm.new_not_implemented_error( + "utime: follow_symlinks unavailable on this platform", + )); + } + + let ft = |d: Duration| { + let intervals = ((d.as_secs() as i64 + 11644473600) * 10_000_000) + + (d.subsec_nanos() as i64 / 100); + FILETIME { + dwLowDateTime: intervals as DWORD, + dwHighDateTime: (intervals >> 32) as DWORD, + } + }; + + let acc = ft(acc); + let modif = ft(modif); + + let f = OpenOptions::new() + .write(true) + .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS) + .open(&path) + .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?; + + let ret = unsafe { + FileSystem::SetFileTime(f.as_raw_handle() as _, core::ptr::null(), &acc, &modif) + }; + + if ret == 0 { + Err(OSErrorBuilder::with_filename( + &io::Error::last_os_error(), + path, + vm, + )) + } else { + Ok(()) + } + } + } + + #[cfg(all(any(unix, windows), not(target_os = "redox")))] + #[derive(Debug)] + #[pystruct_sequence_data] + struct TimesResultData { + pub user: f64, + pub system: f64, + pub children_user: f64, + pub children_system: f64, + pub elapsed: f64, + } + + #[cfg(all(any(unix, windows), not(target_os = "redox")))] + #[pyattr] + #[pystruct_sequence(name = "times_result", module = "os", data = "TimesResultData")] + struct PyTimesResult; + + #[cfg(all(any(unix, windows), not(target_os = "redox")))] + #[pyclass(with(PyStructSequence))] + impl PyTimesResult {} + + #[cfg(all(any(unix, windows), not(target_os = "redox")))] + #[pyfunction] + fn times(vm: &VirtualMachine) -> PyResult { + #[cfg(windows)] + { + use core::mem::MaybeUninit; + use windows_sys::Win32::{Foundation::FILETIME, System::Threading}; + + let mut _create = MaybeUninit::<FILETIME>::uninit(); + let mut _exit = MaybeUninit::<FILETIME>::uninit(); + let mut kernel = MaybeUninit::<FILETIME>::uninit(); + let mut user = MaybeUninit::<FILETIME>::uninit(); + + unsafe { + let h_proc = Threading::GetCurrentProcess(); + Threading::GetProcessTimes( + h_proc, + _create.as_mut_ptr(), + _exit.as_mut_ptr(), + kernel.as_mut_ptr(), + user.as_mut_ptr(), + ); + } + + let kernel = unsafe { kernel.assume_init() }; + let user = unsafe { user.assume_init() }; + + let times_result = TimesResultData { + user: user.dwHighDateTime as f64 * 429.4967296 + user.dwLowDateTime as f64 * 1e-7, + system: kernel.dwHighDateTime as f64 * 429.4967296 + + kernel.dwLowDateTime as f64 * 1e-7, + children_user: 0.0, + children_system: 0.0, + elapsed: 0.0, + }; + + Ok(times_result.to_pyobject(vm)) + } + #[cfg(unix)] + { + let mut t = libc::tms { + tms_utime: 0, + tms_stime: 0, + tms_cutime: 0, + tms_cstime: 0, + }; + + let tick_for_second = unsafe { libc::sysconf(libc::_SC_CLK_TCK) } as f64; + let c = unsafe { libc::times(&mut t as *mut _) }; + + // XXX: The signedness of `clock_t` varies from platform to platform. + if c == (-1i8) as libc::clock_t { + return Err(vm.new_os_error("Fail to get times".to_string())); + } + + let times_result = TimesResultData { + user: t.tms_utime as f64 / tick_for_second, + system: t.tms_stime as f64 / tick_for_second, + children_user: t.tms_cutime as f64 / tick_for_second, + children_system: t.tms_cstime as f64 / tick_for_second, + elapsed: c as f64 / tick_for_second, + }; + + Ok(times_result.to_pyobject(vm)) + } + } + + #[cfg(target_os = "linux")] + #[derive(FromArgs)] + struct CopyFileRangeArgs<'fd> { + #[pyarg(positional)] + src: crt_fd::Borrowed<'fd>, + #[pyarg(positional)] + dst: crt_fd::Borrowed<'fd>, + #[pyarg(positional)] + count: i64, + #[pyarg(any, default)] + offset_src: Option<crt_fd::Offset>, + #[pyarg(any, default)] + offset_dst: Option<crt_fd::Offset>, + } + + #[cfg(target_os = "linux")] + #[pyfunction] + fn copy_file_range(args: CopyFileRangeArgs<'_>, vm: &VirtualMachine) -> PyResult<usize> { + #[allow(clippy::unnecessary_option_map_or_else)] + let p_offset_src = args.offset_src.as_ref().map_or_else(core::ptr::null, |x| x); + #[allow(clippy::unnecessary_option_map_or_else)] + let p_offset_dst = args.offset_dst.as_ref().map_or_else(core::ptr::null, |x| x); + let count: usize = args + .count + .try_into() + .map_err(|_| vm.new_value_error("count should >= 0"))?; + + // The flags argument is provided to allow + // for future extensions and currently must be to 0. + let flags = 0u32; + + // Safety: p_offset_src and p_offset_dst is a unique pointer for offset_src and offset_dst respectively, + // and will only be freed after this function ends. + // + // Why not use `libc::copy_file_range`: On `musl-libc`, `libc::copy_file_range` is not provided. Therefore + // we use syscalls directly instead. + let ret = unsafe { + libc::syscall( + libc::SYS_copy_file_range, + args.src, + p_offset_src as *mut i64, + args.dst, + p_offset_dst as *mut i64, + count, + flags, + ) + }; + + usize::try_from(ret).map_err(|_| vm.new_last_errno_error()) + } + + #[pyfunction] + fn strerror(e: i32) -> String { + unsafe { core::ffi::CStr::from_ptr(libc::strerror(e)) } + .to_string_lossy() + .into_owned() + } + + #[pyfunction] + pub fn ftruncate(fd: crt_fd::Borrowed<'_>, length: crt_fd::Offset) -> io::Result<()> { + crt_fd::ftruncate(fd, length) + } + + #[pyfunction] + fn truncate(path: PyObjectRef, length: crt_fd::Offset, vm: &VirtualMachine) -> PyResult<()> { + match path.clone().try_into_value::<crt_fd::Borrowed<'_>>(vm) { + Ok(fd) => return ftruncate(fd, length).map_err(|e| e.into_pyexception(vm)), + Err(e) if e.fast_isinstance(vm.ctx.exceptions.warning) => return Err(e), + Err(_) => {} + } + + #[cold] + fn error( + vm: &VirtualMachine, + error: std::io::Error, + path: OsPath, + ) -> crate::builtins::PyBaseExceptionRef { + OSErrorBuilder::with_filename(&error, path, vm) + } + + let path = OsPath::try_from_object(vm, path)?; + // TODO: just call libc::truncate() on POSIX + let f = match OpenOptions::new().write(true).open(&path) { + Ok(f) => f, + Err(e) => return Err(error(vm, e, path)), + }; + f.set_len(length as u64).map_err(|e| error(vm, e, path))?; + drop(f); + Ok(()) + } + + #[cfg(all(unix, not(any(target_os = "redox", target_os = "android"))))] + #[pyfunction] + fn getloadavg(vm: &VirtualMachine) -> PyResult<(f64, f64, f64)> { + let mut loadavg = [0f64; 3]; + + // Safety: loadavg is on stack and only write by `getloadavg` and are freed + // after this function ends. + unsafe { + if libc::getloadavg(&mut loadavg[0] as *mut f64, 3) != 3 { + return Err(vm.new_os_error("Load averages are unobtainable".to_string())); + } + } + + Ok((loadavg[0], loadavg[1], loadavg[2])) + } + + #[cfg(unix)] + #[pyfunction] + fn waitstatus_to_exitcode(status: i32, vm: &VirtualMachine) -> PyResult<i32> { + let status = u32::try_from(status) + .map_err(|_| vm.new_value_error(format!("invalid WEXITSTATUS: {status}")))?; + + let status = status as libc::c_int; + if libc::WIFEXITED(status) { + return Ok(libc::WEXITSTATUS(status)); + } + + if libc::WIFSIGNALED(status) { + return Ok(-libc::WTERMSIG(status)); + } + + Err(vm.new_value_error(format!("Invalid wait status: {status}"))) + } + + #[cfg(windows)] + #[pyfunction] + fn waitstatus_to_exitcode(status: u64, vm: &VirtualMachine) -> PyResult<u32> { + let exitcode = status >> 8; + // ExitProcess() accepts an UINT type: + // reject exit code which doesn't fit in an UINT + u32::try_from(exitcode) + .map_err(|_| vm.new_value_error(format!("Invalid exit code: {exitcode}"))) + } + + #[pyfunction] + fn device_encoding(fd: i32, _vm: &VirtualMachine) -> PyResult<Option<String>> { + if !isatty(fd) { + return Ok(None); + } + + cfg_if::cfg_if! { + if #[cfg(any(target_os = "android", target_os = "redox"))] { + Ok(Some("UTF-8".to_owned())) + } else if #[cfg(windows)] { + use windows_sys::Win32::System::Console; + let cp = match fd { + 0 => unsafe { Console::GetConsoleCP() }, + 1 | 2 => unsafe { Console::GetConsoleOutputCP() }, + _ => 0, + }; + + Ok(Some(format!("cp{cp}"))) + } else { + let encoding = unsafe { + let encoding = libc::nl_langinfo(libc::CODESET); + if encoding.is_null() || encoding.read() == b'\0' as libc::c_char { + "UTF-8".to_owned() + } else { + core::ffi::CStr::from_ptr(encoding).to_string_lossy().into_owned() + } + }; + + Ok(Some(encoding)) + } + } + } + + #[pystruct_sequence_data] + #[allow(dead_code)] + pub(crate) struct TerminalSizeData { + pub columns: usize, + pub lines: usize, + } + + #[pyattr] + #[pystruct_sequence(name = "terminal_size", module = "os", data = "TerminalSizeData")] + pub(crate) struct PyTerminalSize; + + #[pyclass(with(PyStructSequence))] + impl PyTerminalSize {} + + #[derive(Debug)] + #[pystruct_sequence_data] + pub(crate) struct UnameResultData { + pub sysname: String, + pub nodename: String, + pub release: String, + pub version: String, + pub machine: String, + } + + #[pyattr] + #[pystruct_sequence(name = "uname_result", module = "os", data = "UnameResultData")] + pub(crate) struct PyUnameResult; + + #[pyclass(with(PyStructSequence))] + impl PyUnameResult {} + + // statvfs_result: Result from statvfs or fstatvfs. + // = statvfs_result_fields + #[cfg(all(unix, not(target_os = "redox")))] + #[derive(Debug)] + #[pystruct_sequence_data] + pub(crate) struct StatvfsResultData { + pub f_bsize: libc::c_ulong, // filesystem block size + pub f_frsize: libc::c_ulong, // fragment size + pub f_blocks: libc::fsblkcnt_t, // size of fs in f_frsize units + pub f_bfree: libc::fsblkcnt_t, // free blocks + pub f_bavail: libc::fsblkcnt_t, // free blocks for unprivileged users + pub f_files: libc::fsfilcnt_t, // inodes + pub f_ffree: libc::fsfilcnt_t, // free inodes + pub f_favail: libc::fsfilcnt_t, // free inodes for unprivileged users + pub f_flag: libc::c_ulong, // mount flags + pub f_namemax: libc::c_ulong, // maximum filename length + #[pystruct_sequence(skip)] + pub f_fsid: libc::c_ulong, // filesystem ID (not in tuple but accessible as attribute) + } + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyattr] + #[pystruct_sequence(name = "statvfs_result", module = "os", data = "StatvfsResultData")] + pub(crate) struct PyStatvfsResult; + + #[cfg(all(unix, not(target_os = "redox")))] + #[pyclass(with(PyStructSequence))] + impl PyStatvfsResult { + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let seq: PyObjectRef = args.bind(vm)?; + crate::types::struct_sequence_new(cls, seq, vm) + } + } + + #[cfg(all(unix, not(target_os = "redox")))] + impl StatvfsResultData { + fn from_statvfs(st: libc::statvfs) -> Self { + // f_fsid is a struct on some platforms (e.g., Linux fsid_t) and a scalar on others. + // We extract raw bytes and interpret as a native-endian integer. + // Note: The value may differ across architectures due to endianness. + let f_fsid = { + let ptr = core::ptr::addr_of!(st.f_fsid) as *const u8; + let size = core::mem::size_of_val(&st.f_fsid); + if size >= 8 { + let bytes = unsafe { core::slice::from_raw_parts(ptr, 8) }; + u64::from_ne_bytes([ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], + bytes[7], + ]) as libc::c_ulong + } else if size >= 4 { + let bytes = unsafe { core::slice::from_raw_parts(ptr, 4) }; + u32::from_ne_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as libc::c_ulong + } else { + 0 + } + }; + + Self { + f_bsize: st.f_bsize, + f_frsize: st.f_frsize, + f_blocks: st.f_blocks, + f_bfree: st.f_bfree, + f_bavail: st.f_bavail, + f_files: st.f_files, + f_ffree: st.f_ffree, + f_favail: st.f_favail, + f_flag: st.f_flag, + f_namemax: st.f_namemax, + f_fsid, + } + } + } + + /// Perform a statvfs system call on the given path. + #[cfg(all(unix, not(target_os = "redox")))] + #[pyfunction] + #[pyfunction(name = "fstatvfs")] + fn statvfs(path: OsPathOrFd<'_>, vm: &VirtualMachine) -> PyResult { + let mut st: libc::statvfs = unsafe { core::mem::zeroed() }; + let ret = match &path { + OsPathOrFd::Path(p) => { + let cpath = p.clone().into_cstring(vm)?; + unsafe { libc::statvfs(cpath.as_ptr(), &mut st) } + } + OsPathOrFd::Fd(fd) => unsafe { libc::fstatvfs(fd.as_raw(), &mut st) }, + }; + if ret != 0 { + return Err(OSErrorBuilder::with_filename( + &io::Error::last_os_error(), + path, + vm, + )); + } + Ok(StatvfsResultData::from_statvfs(st).to_pyobject(vm)) + } + + pub(super) fn support_funcs() -> Vec<SupportFunc> { + let mut supports = super::platform::module::support_funcs(); + supports.extend(vec![ + SupportFunc::new("open", Some(false), Some(OPEN_DIR_FD), Some(false)), + SupportFunc::new("access", Some(false), Some(false), None), + SupportFunc::new("chdir", None, Some(false), Some(false)), + // chflags Some, None Some + SupportFunc::new("link", Some(false), Some(false), Some(cfg!(unix))), + SupportFunc::new("listdir", Some(LISTDIR_FD), Some(false), Some(false)), + SupportFunc::new("mkdir", Some(false), Some(MKDIR_DIR_FD), Some(false)), + // mkfifo Some Some None + // mknod Some Some None + SupportFunc::new("readlink", Some(false), None, Some(false)), + SupportFunc::new("remove", Some(false), Some(UNLINK_DIR_FD), Some(false)), + SupportFunc::new("unlink", Some(false), Some(UNLINK_DIR_FD), Some(false)), + SupportFunc::new("rename", Some(false), None, Some(false)), + SupportFunc::new("replace", Some(false), None, Some(false)), // TODO: Fix replace + SupportFunc::new("rmdir", Some(false), Some(RMDIR_DIR_FD), Some(false)), + SupportFunc::new("scandir", Some(SCANDIR_FD), Some(false), Some(false)), + SupportFunc::new("stat", Some(true), Some(STAT_DIR_FD), Some(true)), + SupportFunc::new("fstat", Some(true), Some(STAT_DIR_FD), Some(true)), + SupportFunc::new("symlink", Some(false), Some(SYMLINK_DIR_FD), Some(false)), + SupportFunc::new("truncate", Some(true), Some(false), Some(false)), + SupportFunc::new("ftruncate", Some(true), Some(false), Some(false)), + SupportFunc::new("fsync", Some(true), Some(false), Some(false)), + SupportFunc::new( + "utime", + Some(false), + Some(UTIME_DIR_FD), + Some(cfg!(all(unix, not(target_os = "redox")))), + ), + ]); + supports + } +} +pub(crate) use _os::{ftruncate, isatty, lseek}; + +pub(crate) struct SupportFunc { + name: &'static str, + // realistically, each of these is just a bool of "is this function in the supports_* set". + // However, None marks that the function maybe _should_ support fd/dir_fd/follow_symlinks, but + // we haven't implemented it yet. + fd: Option<bool>, + dir_fd: Option<bool>, + follow_symlinks: Option<bool>, +} + +impl SupportFunc { + pub(crate) const fn new( + name: &'static str, + fd: Option<bool>, + dir_fd: Option<bool>, + follow_symlinks: Option<bool>, + ) -> Self { + Self { + name, + fd, + dir_fd, + follow_symlinks, + } + } +} + +pub fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + let support_funcs = _os::support_funcs(); + let supports_fd = PySet::default().into_ref(&vm.ctx); + let supports_dir_fd = PySet::default().into_ref(&vm.ctx); + let supports_follow_symlinks = PySet::default().into_ref(&vm.ctx); + for support in support_funcs { + let func_obj = module.get_attr(support.name, vm)?; + if support.fd.unwrap_or(false) { + supports_fd.clone().add(func_obj.clone(), vm)?; + } + if support.dir_fd.unwrap_or(false) { + supports_dir_fd.clone().add(func_obj.clone(), vm)?; + } + if support.follow_symlinks.unwrap_or(false) { + supports_follow_symlinks.clone().add(func_obj, vm)?; + } + } + + extend_module!(vm, module, { + "supports_fd" => supports_fd, + "supports_dir_fd" => supports_dir_fd, + "supports_follow_symlinks" => supports_follow_symlinks, + "error" => vm.ctx.exceptions.os_error.to_owned(), + }); + + Ok(()) +} + +/// Convert a mapping (e.g. os._Environ) to a plain dict for use by execve/posix_spawn. +/// +/// For `os._Environ`, accesses the internal `_data` dict directly at the Rust level. +/// This avoids Python-level method calls that can deadlock after fork() when +/// parking_lot locks are held by threads that no longer exist. +#[cfg(any(unix, windows))] +pub(crate) fn envobj_to_dict( + env: crate::function::ArgMapping, + vm: &VirtualMachine, +) -> PyResult<crate::builtins::PyDictRef> { + let obj = env.obj(); + if let Some(dict) = obj.downcast_ref_if_exact::<crate::builtins::PyDict>(vm) { + return Ok(dict.to_owned()); + } + if let Some(inst_dict) = obj.dict() + && let Ok(Some(data)) = inst_dict.get_item_opt("_data", vm) + && let Some(dict) = data.downcast_ref_if_exact::<crate::builtins::PyDict>(vm) + { + return Ok(dict.to_owned()); + } + let keys = vm.call_method(obj, "keys", ())?; + let dict = vm.ctx.new_dict(); + for key in keys.get_iter(vm)?.into_iter::<PyObjectRef>(vm)? { + let key = key?; + let val = obj.get_item(&*key, vm)?; + dict.set_item(&*key, val, vm)?; + } + Ok(dict) +} + +#[cfg(not(windows))] +use super::posix as platform; + +#[cfg(windows)] +use super::nt as platform; + +pub(crate) use platform::module::MODULE_NAME; diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs new file mode 100644 index 00000000000..8cde18a47ba --- /dev/null +++ b/crates/vm/src/stdlib/posix.rs @@ -0,0 +1,2908 @@ +// spell-checker:disable + +use std::os::fd::BorrowedFd; + +pub(crate) use module::module_def; + +pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()> { + use nix::fcntl; + let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); + let mut new_flags = flags; + new_flags.set(fcntl::FdFlag::FD_CLOEXEC, !inheritable); + if flags != new_flags { + fcntl::fcntl(fd, fcntl::FcntlArg::F_SETFD(new_flags))?; + } + Ok(()) +} + +#[pymodule(name = "posix", with( + super::os::_os, + #[cfg(any( + target_os = "linux", + target_os = "netbsd", + target_os = "freebsd", + target_os = "android" + ))] + posix_sched +))] +pub mod module { + use crate::{ + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyInt, PyListRef, PyTupleRef, PyUtf8Str}, + convert::{IntoPyException, ToPyObject, TryFromObject}, + exceptions::OSErrorBuilder, + function::{ArgMapping, Either, KwArgs, OptionalArg}, + ospath::{OsPath, OsPathOrFd}, + stdlib::os::{ + _os, DirFd, FollowSymlinks, SupportFunc, TargetIsDirectory, fs_metadata, + warn_if_bool_fd, + }, + }; + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + use crate::{builtins::PyUtf8StrRef, utils::ToCString}; + use alloc::ffi::CString; + use bitflags::bitflags; + use core::ffi::CStr; + use nix::{ + errno::Errno, + fcntl, + unistd::{self, Gid, Pid, Uid}, + }; + use rustpython_common::os::ffi::OsStringExt; + use std::{ + env, fs, io, + os::fd::{AsFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd}, + }; + use strum::IntoEnumIterator; + use strum_macros::{EnumIter, EnumString}; + + #[cfg(target_os = "linux")] + #[pyattr] + use libc::PIDFD_NONBLOCK; + + #[cfg(target_os = "macos")] + #[pyattr] + use libc::{ + COPYFILE_DATA as _COPYFILE_DATA, O_EVTONLY, O_NOFOLLOW_ANY, PRIO_DARWIN_BG, + PRIO_DARWIN_NONUI, PRIO_DARWIN_PROCESS, PRIO_DARWIN_THREAD, + }; + + #[cfg(target_os = "freebsd")] + #[pyattr] + use libc::{SF_MNOWAIT, SF_NOCACHE, SF_NODISKIO, SF_SYNC}; + + #[cfg(any(target_os = "android", target_os = "linux"))] + #[pyattr] + use libc::{ + CLONE_FILES, CLONE_FS, CLONE_NEWCGROUP, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWNS, + CLONE_NEWPID, CLONE_NEWUSER, CLONE_NEWUTS, CLONE_SIGHAND, CLONE_SYSVSEM, CLONE_THREAD, + CLONE_VM, MFD_HUGE_SHIFT, O_NOATIME, O_TMPFILE, P_PIDFD, SCHED_BATCH, SCHED_DEADLINE, + SCHED_IDLE, SCHED_NORMAL, SCHED_RESET_ON_FORK, SPLICE_F_MORE, SPLICE_F_MOVE, + SPLICE_F_NONBLOCK, + }; + + #[cfg(any(target_os = "macos", target_os = "redox"))] + #[pyattr] + use libc::O_SYMLINK; + + #[cfg(any(target_os = "android", target_os = "redox", unix))] + #[pyattr] + use libc::{O_NOFOLLOW, PRIO_PGRP, PRIO_PROCESS, PRIO_USER}; + + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "netbsd"))] + #[pyattr] + use libc::{XATTR_CREATE, XATTR_REPLACE}; + + #[cfg(any(target_os = "android", target_os = "linux", target_os = "netbsd"))] + #[pyattr] + use libc::O_RSYNC; + + #[cfg(any(target_os = "android", target_os = "freebsd", target_os = "linux"))] + #[pyattr] + use libc::{ + MFD_ALLOW_SEALING, MFD_CLOEXEC, MFD_HUGE_MASK, MFD_HUGETLB, POSIX_FADV_DONTNEED, + POSIX_FADV_NOREUSE, POSIX_FADV_NORMAL, POSIX_FADV_RANDOM, POSIX_FADV_SEQUENTIAL, + POSIX_FADV_WILLNEED, + }; + + #[cfg(any(target_os = "android", target_os = "linux", target_os = "redox", unix))] + #[pyattr] + use libc::{RTLD_LAZY, RTLD_NOW, WNOHANG}; + + #[cfg(any(target_os = "android", target_os = "macos", target_os = "redox", unix))] + #[pyattr] + use libc::RTLD_GLOBAL; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "redox" + ))] + #[pyattr] + use libc::O_PATH; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd" + ))] + #[pyattr] + use libc::{ + EFD_CLOEXEC, EFD_NONBLOCK, EFD_SEMAPHORE, TFD_CLOEXEC, TFD_NONBLOCK, TFD_TIMER_ABSTIME, + TFD_TIMER_CANCEL_ON_SET, + }; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "linux", + target_os = "netbsd" + ))] + #[pyattr] + use libc::{GRND_NONBLOCK, GRND_RANDOM}; + + #[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "macos", + target_os = "redox", + unix + ))] + #[pyattr] + use libc::{F_OK, R_OK, W_OK, X_OK}; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "netbsd", + target_os = "redox", + unix + ))] + #[pyattr] + use libc::O_NONBLOCK; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd" + ))] + #[pyattr] + use libc::O_DSYNC; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd" + ))] + #[pyattr] + use libc::SCHED_OTHER; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos" + ))] + #[pyattr] + use libc::{RTLD_NODELETE, SEEK_DATA, SEEK_HOLE}; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd" + ))] + #[pyattr] + use libc::O_DIRECT; + + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "macos", + target_os = "netbsd", + target_os = "redox" + ))] + #[pyattr] + use libc::{O_EXLOCK, O_FSYNC, O_SHLOCK}; + + #[cfg(any( + target_os = "android", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "redox", + unix + ))] + #[pyattr] + use libc::RTLD_LOCAL; + + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "redox", + unix + ))] + #[pyattr] + use libc::WUNTRACED; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd" + ))] + #[pyattr] + use libc::{ + CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, CLD_TRAPPED, O_SYNC, P_ALL, + P_PGID, P_PID, RTLD_NOLOAD, SCHED_FIFO, SCHED_RR, + }; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "macos", + target_os = "netbsd", + target_os = "redox", + unix + ))] + #[pyattr] + use libc::O_DIRECTORY; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "redox" + ))] + #[pyattr] + use libc::{ + F_LOCK, F_TEST, F_TLOCK, F_ULOCK, O_ASYNC, O_NDELAY, O_NOCTTY, WEXITED, WNOWAIT, WSTOPPED, + }; + + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "macos", + target_os = "netbsd", + target_os = "redox", + unix + ))] + #[pyattr] + use libc::{O_CLOEXEC, WCONTINUED}; + + #[pyattr] + const EX_OK: i8 = exitcode::OK as i8; + + #[pyattr] + const EX_USAGE: i8 = exitcode::USAGE as i8; + + #[pyattr] + const EX_DATAERR: i8 = exitcode::DATAERR as i8; + + #[pyattr] + const EX_NOINPUT: i8 = exitcode::NOINPUT as i8; + + #[pyattr] + const EX_NOUSER: i8 = exitcode::NOUSER as i8; + + #[pyattr] + const EX_NOHOST: i8 = exitcode::NOHOST as i8; + + #[pyattr] + const EX_UNAVAILABLE: i8 = exitcode::UNAVAILABLE as i8; + + #[pyattr] + const EX_SOFTWARE: i8 = exitcode::SOFTWARE as i8; + + #[pyattr] + const EX_OSERR: i8 = exitcode::OSERR as i8; + + #[pyattr] + const EX_OSFILE: i8 = exitcode::OSFILE as i8; + + #[pyattr] + const EX_CANTCREAT: i8 = exitcode::CANTCREAT as i8; + + #[pyattr] + const EX_IOERR: i8 = exitcode::IOERR as i8; + + #[pyattr] + const EX_TEMPFAIL: i8 = exitcode::TEMPFAIL as i8; + + #[pyattr] + const EX_PROTOCOL: i8 = exitcode::PROTOCOL as i8; + + #[pyattr] + const EX_NOPERM: i8 = exitcode::NOPERM as i8; + + #[pyattr] + const EX_CONFIG: i8 = exitcode::CONFIG as i8; + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[pyattr] + const POSIX_SPAWN_OPEN: i32 = PosixSpawnFileActionIdentifier::Open as i32; + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[pyattr] + const POSIX_SPAWN_CLOSE: i32 = PosixSpawnFileActionIdentifier::Close as i32; + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[pyattr] + const POSIX_SPAWN_DUP2: i32 = PosixSpawnFileActionIdentifier::Dup2 as i32; + + impl TryFromObject for BorrowedFd<'_> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + crate::stdlib::os::warn_if_bool_fd(&obj, vm)?; + let fd = i32::try_from_object(vm, obj)?; + if fd == -1 { + return Err(io::Error::from_raw_os_error(libc::EBADF).into_pyexception(vm)); + } + // SAFETY: none, really. but, python's os api of passing around file descriptors + // everywhere isn't really io-safe anyway, so, this is passed to the user. + Ok(unsafe { BorrowedFd::borrow_raw(fd) }) + } + } + + impl TryFromObject for OwnedFd { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let fd = i32::try_from_object(vm, obj)?; + if fd == -1 { + return Err(io::Error::from_raw_os_error(libc::EBADF).into_pyexception(vm)); + } + // SAFETY: none, really. but, python's os api of passing around file descriptors + // everywhere isn't really io-safe anyway, so, this is passed to the user. + Ok(unsafe { Self::from_raw_fd(fd) }) + } + } + + impl ToPyObject for OwnedFd { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + self.into_raw_fd().to_pyobject(vm) + } + } + + // Flags for os_access + bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + pub struct AccessFlags: u8 { + const F_OK = _os::F_OK; + const R_OK = _os::R_OK; + const W_OK = _os::W_OK; + const X_OK = _os::X_OK; + } + } + + struct Permissions { + is_readable: bool, + is_writable: bool, + is_executable: bool, + } + + const fn get_permissions(mode: u32) -> Permissions { + Permissions { + is_readable: mode & 4 != 0, + is_writable: mode & 2 != 0, + is_executable: mode & 1 != 0, + } + } + + fn get_right_permission( + mode: u32, + file_owner: Uid, + file_group: Gid, + ) -> nix::Result<Permissions> { + let owner_mode = (mode & 0o700) >> 6; + let owner_permissions = get_permissions(owner_mode); + + let group_mode = (mode & 0o070) >> 3; + let group_permissions = get_permissions(group_mode); + + let others_mode = mode & 0o007; + let others_permissions = get_permissions(others_mode); + + let user_id = nix::unistd::getuid(); + let groups_ids = getgroups_impl()?; + + if file_owner == user_id { + Ok(owner_permissions) + } else if groups_ids.contains(&file_group) { + Ok(group_permissions) + } else { + Ok(others_permissions) + } + } + + #[cfg(any(target_os = "macos", target_os = "ios"))] + fn getgroups_impl() -> nix::Result<Vec<Gid>> { + use core::ptr; + use libc::{c_int, gid_t}; + + let ret = unsafe { libc::getgroups(0, ptr::null_mut()) }; + let mut groups = Vec::<Gid>::with_capacity(Errno::result(ret)? as usize); + let ret = unsafe { + libc::getgroups( + groups.capacity() as c_int, + groups.as_mut_ptr() as *mut gid_t, + ) + }; + + Errno::result(ret).map(|s| { + unsafe { groups.set_len(s as usize) }; + groups + }) + } + + #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "redox")))] + use nix::unistd::getgroups as getgroups_impl; + + #[cfg(target_os = "redox")] + fn getgroups_impl() -> nix::Result<Vec<Gid>> { + Err(nix::Error::EOPNOTSUPP) + } + + #[pyfunction] + fn getgroups(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + let group_ids = getgroups_impl().map_err(|e| e.into_pyexception(vm))?; + Ok(group_ids + .into_iter() + .map(|gid| vm.ctx.new_int(gid.as_raw()).into()) + .collect()) + } + + #[pyfunction] + pub(super) fn access(path: OsPath, mode: u8, vm: &VirtualMachine) -> PyResult<bool> { + use std::os::unix::fs::MetadataExt; + + let flags = AccessFlags::from_bits(mode).ok_or_else(|| { + vm.new_value_error( + "One of the flags is wrong, there are only 4 possibilities F_OK, R_OK, W_OK and X_OK", + ) + })?; + + let metadata = match fs::metadata(&path.path) { + Ok(m) => m, + // If the file doesn't exist, return False for any access check + Err(_) => return Ok(false), + }; + + // if it's only checking for F_OK + if flags == AccessFlags::F_OK { + return Ok(true); // File exists + } + + let user_id = metadata.uid(); + let group_id = metadata.gid(); + let mode = metadata.mode(); + + let perm = get_right_permission(mode, Uid::from_raw(user_id), Gid::from_raw(group_id)) + .map_err(|err| err.into_pyexception(vm))?; + + let r_ok = !flags.contains(AccessFlags::R_OK) || perm.is_readable; + let w_ok = !flags.contains(AccessFlags::W_OK) || perm.is_writable; + let x_ok = !flags.contains(AccessFlags::X_OK) || perm.is_executable; + + Ok(r_ok && w_ok && x_ok) + } + + #[pyattr] + fn environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars_os() { + let key: PyObjectRef = vm.ctx.new_bytes(key.into_vec()).into(); + let value: PyObjectRef = vm.ctx.new_bytes(value.into_vec()).into(); + environ.set_item(&*key, value, vm).unwrap(); + } + + environ + } + + #[pyfunction] + fn _create_environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars_os() { + let key: PyObjectRef = vm.ctx.new_bytes(key.into_vec()).into(); + let value: PyObjectRef = vm.ctx.new_bytes(value.into_vec()).into(); + environ.set_item(&*key, value, vm).unwrap(); + } + environ + } + + #[derive(FromArgs)] + pub(super) struct SymlinkArgs<'fd> { + src: OsPath, + dst: OsPath, + #[pyarg(flatten)] + _target_is_directory: TargetIsDirectory, + #[pyarg(flatten)] + dir_fd: DirFd<'fd, { _os::SYMLINK_DIR_FD as usize }>, + } + + #[pyfunction] + pub(super) fn symlink(args: SymlinkArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { + let src = args.src.into_cstring(vm)?; + let dst = args.dst.into_cstring(vm)?; + #[cfg(not(target_os = "redox"))] + { + nix::unistd::symlinkat(&*src, args.dir_fd.get(), &*dst) + .map_err(|err| err.into_pyexception(vm)) + } + #[cfg(target_os = "redox")] + { + let [] = args.dir_fd.0; + let res = unsafe { libc::symlink(src.as_ptr(), dst.as_ptr()) }; + if res < 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + } + + #[pyfunction] + #[pyfunction(name = "unlink")] + fn remove( + path: OsPath, + dir_fd: DirFd<'_, { _os::UNLINK_DIR_FD as usize }>, + vm: &VirtualMachine, + ) -> PyResult<()> { + #[cfg(not(target_os = "redox"))] + if let Some(fd) = dir_fd.raw_opt() { + let c_path = path.clone().into_cstring(vm)?; + let res = unsafe { libc::unlinkat(fd, c_path.as_ptr(), 0) }; + return if res < 0 { + let err = crate::common::os::errno_io_error(); + Err(OSErrorBuilder::with_filename(&err, path, vm)) + } else { + Ok(()) + }; + } + #[cfg(target_os = "redox")] + let [] = dir_fd.0; + fs::remove_file(&path).map_err(|err| OSErrorBuilder::with_filename(&err, path, vm)) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn fchdir(fd: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + warn_if_bool_fd(&fd, vm)?; + let fd = i32::try_from_object(vm, fd)?; + let ret = unsafe { libc::fchdir(fd) }; + if ret == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error().into_pyexception(vm)) + } + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn chroot(path: OsPath, vm: &VirtualMachine) -> PyResult<()> { + use crate::exceptions::OSErrorBuilder; + + nix::unistd::chroot(&*path.path).map_err(|err| { + // Use `From<nix::Error> for io::Error` when it is available + let io_err: io::Error = err.into(); + OSErrorBuilder::with_filename(&io_err, path, vm) + }) + } + + // As of now, redox does not seems to support chown command (cf. https://gitlab.redox-os.org/redox-os/coreutils , last checked on 05/07/2020) + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn chown( + path: OsPathOrFd<'_>, + uid: isize, + gid: isize, + dir_fd: DirFd<'_, 1>, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult<()> { + let uid = if uid >= 0 { + Some(nix::unistd::Uid::from_raw(uid as u32)) + } else if uid == -1 { + None + } else { + return Err(vm.new_os_error("Specified uid is not valid.")); + }; + + let gid = if gid >= 0 { + Some(nix::unistd::Gid::from_raw(gid as u32)) + } else if gid == -1 { + None + } else { + return Err(vm.new_os_error("Specified gid is not valid.")); + }; + + let flag = if follow_symlinks.0 { + nix::fcntl::AtFlags::empty() + } else { + nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW + }; + + match path { + OsPathOrFd::Path(ref p) => { + nix::unistd::fchownat(dir_fd.get(), p.path.as_os_str(), uid, gid, flag) + } + OsPathOrFd::Fd(fd) => nix::unistd::fchown(fd, uid, gid), + } + .map_err(|err| { + // Use `From<nix::Error> for io::Error` when it is available + let err = io::Error::from_raw_os_error(err as i32); + OSErrorBuilder::with_filename(&err, path, vm) + }) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn lchown(path: OsPath, uid: isize, gid: isize, vm: &VirtualMachine) -> PyResult<()> { + chown( + OsPathOrFd::Path(path), + uid, + gid, + DirFd::default(), + FollowSymlinks(false), + vm, + ) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn fchown(fd: BorrowedFd<'_>, uid: isize, gid: isize, vm: &VirtualMachine) -> PyResult<()> { + chown( + OsPathOrFd::Fd(fd.into()), + uid, + gid, + DirFd::default(), + FollowSymlinks(true), + vm, + ) + } + + #[derive(FromArgs)] + struct RegisterAtForkArgs { + #[pyarg(named, optional)] + before: OptionalArg<PyObjectRef>, + #[pyarg(named, optional)] + after_in_parent: OptionalArg<PyObjectRef>, + #[pyarg(named, optional)] + after_in_child: OptionalArg<PyObjectRef>, + } + + impl RegisterAtForkArgs { + fn into_validated( + self, + vm: &VirtualMachine, + ) -> PyResult<( + Option<PyObjectRef>, + Option<PyObjectRef>, + Option<PyObjectRef>, + )> { + fn into_option( + arg: OptionalArg<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<Option<PyObjectRef>> { + match arg { + OptionalArg::Present(obj) => { + if !obj.is_callable() { + return Err(vm.new_type_error("Args must be callable")); + } + Ok(Some(obj)) + } + OptionalArg::Missing => Ok(None), + } + } + let before = into_option(self.before, vm)?; + let after_in_parent = into_option(self.after_in_parent, vm)?; + let after_in_child = into_option(self.after_in_child, vm)?; + if before.is_none() && after_in_parent.is_none() && after_in_child.is_none() { + return Err(vm.new_type_error("At least one arg must be present")); + } + Ok((before, after_in_parent, after_in_child)) + } + } + + #[pyfunction] + fn register_at_fork( + args: RegisterAtForkArgs, + _ignored: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + let (before, after_in_parent, after_in_child) = args.into_validated(vm)?; + + if let Some(before) = before { + vm.state.before_forkers.lock().push(before); + } + if let Some(after_in_parent) = after_in_parent { + vm.state.after_forkers_parent.lock().push(after_in_parent); + } + if let Some(after_in_child) = after_in_child { + vm.state.after_forkers_child.lock().push(after_in_child); + } + Ok(()) + } + + fn run_at_forkers(mut funcs: Vec<PyObjectRef>, reversed: bool, vm: &VirtualMachine) { + if !funcs.is_empty() { + if reversed { + funcs.reverse(); + } + for func in funcs { + if let Err(e) = func.call((), vm) { + let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); + vm.run_unraisable(e, Some("Exception ignored in".to_owned()), func); + if exit { + // Do nothing! + } + } + } + } + } + + fn py_os_before_fork(vm: &VirtualMachine) { + let before_forkers: Vec<PyObjectRef> = vm.state.before_forkers.lock().clone(); + // functions must be executed in reversed order as they are registered + // only for before_forkers, refer: test_register_at_fork in test_posix + + run_at_forkers(before_forkers, true, vm); + + #[cfg(feature = "threading")] + crate::stdlib::_imp::acquire_imp_lock_for_fork(); + + #[cfg(feature = "threading")] + vm.state.stop_the_world.stop_the_world(vm); + } + + fn py_os_after_fork_child(vm: &VirtualMachine) { + #[cfg(feature = "threading")] + vm.state.stop_the_world.reset_after_fork(); + + // Phase 1: Reset all internal locks FIRST. + // After fork(), locks held by dead parent threads would deadlock + // if we try to acquire them. This must happen before anything else. + #[cfg(feature = "threading")] + reinit_locks_after_fork(vm); + + // Reinit per-object IO buffer locks on std streams. + // BufferedReader/Writer/TextIOWrapper use PyThreadMutex which can be + // held by dead parent threads, causing deadlocks on any IO in the child. + #[cfg(feature = "threading")] + unsafe { + crate::stdlib::_io::reinit_std_streams_after_fork(vm) + }; + + // Phase 2: Reset low-level atomic state (no locks needed). + crate::signal::clear_after_fork(); + crate::stdlib::_signal::_signal::clear_wakeup_fd_after_fork(); + + // Reset weakref stripe locks that may have been held during fork. + #[cfg(feature = "threading")] + crate::object::reset_weakref_locks_after_fork(); + + // Phase 3: Clean up thread state. Locks are now reinit'd so we can + // acquire them normally instead of using try_lock(). + #[cfg(feature = "threading")] + crate::stdlib::_thread::after_fork_child(vm); + + // CPython parity: reinit import lock ownership metadata in child + // and release the lock acquired by PyOS_BeforeFork(). + #[cfg(feature = "threading")] + unsafe { + crate::stdlib::_imp::after_fork_child_imp_lock_release() + }; + + // Initialize signal handlers for the child's main thread. + // When forked from a worker thread, the OnceCell is empty. + vm.signal_handlers + .get_or_init(crate::signal::new_signal_handlers); + + // Phase 4: Run Python-level at-fork callbacks. + let after_forkers_child: Vec<PyObjectRef> = vm.state.after_forkers_child.lock().clone(); + run_at_forkers(after_forkers_child, false, vm); + } + + /// Reset all parking_lot-based locks in the interpreter state after fork(). + /// + /// After fork(), only the calling thread survives. Any locks held by other + /// (now-dead) threads would cause deadlocks. We unconditionally reset them + /// to unlocked by zeroing the raw lock bytes. + #[cfg(all(unix, feature = "threading"))] + fn reinit_locks_after_fork(vm: &VirtualMachine) { + use rustpython_common::lock::reinit_mutex_after_fork; + + unsafe { + // PyGlobalState PyMutex locks + reinit_mutex_after_fork(&vm.state.before_forkers); + reinit_mutex_after_fork(&vm.state.after_forkers_child); + reinit_mutex_after_fork(&vm.state.after_forkers_parent); + reinit_mutex_after_fork(&vm.state.atexit_funcs); + reinit_mutex_after_fork(&vm.state.global_trace_func); + reinit_mutex_after_fork(&vm.state.global_profile_func); + reinit_mutex_after_fork(&vm.state.monitoring); + + // PyGlobalState parking_lot::Mutex locks + reinit_mutex_after_fork(&vm.state.thread_frames); + reinit_mutex_after_fork(&vm.state.thread_handles); + reinit_mutex_after_fork(&vm.state.shutdown_handles); + + // Context-level RwLock + vm.ctx.string_pool.reinit_after_fork(); + + // Codec registry RwLock + vm.state.codec_registry.reinit_after_fork(); + + // GC state (multiple Mutex + RwLock) + crate::gc_state::gc_state().reinit_after_fork(); + + // Import lock (RawReentrantMutex<RawMutex, RawThreadId>) + crate::stdlib::_imp::reinit_imp_lock_after_fork(); + } + } + + fn py_os_after_fork_parent(vm: &VirtualMachine) { + #[cfg(feature = "threading")] + vm.state.stop_the_world.start_the_world(vm); + + #[cfg(feature = "threading")] + crate::stdlib::_imp::release_imp_lock_after_fork_parent(); + + let after_forkers_parent: Vec<PyObjectRef> = vm.state.after_forkers_parent.lock().clone(); + run_at_forkers(after_forkers_parent, false, vm); + } + + /// Best-effort number of OS threads in this process. + /// Returns <= 0 when unavailable. + fn get_number_of_os_threads() -> isize { + #[cfg(target_os = "macos")] + { + type MachPortT = libc::c_uint; + type KernReturnT = libc::c_int; + type MachMsgTypeNumberT = libc::c_uint; + type ThreadActArrayT = *mut MachPortT; + const KERN_SUCCESS: KernReturnT = 0; + unsafe extern "C" { + fn mach_task_self() -> MachPortT; + fn task_for_pid( + task: MachPortT, + pid: libc::c_int, + target_task: *mut MachPortT, + ) -> KernReturnT; + fn task_threads( + target_task: MachPortT, + act_list: *mut ThreadActArrayT, + act_list_cnt: *mut MachMsgTypeNumberT, + ) -> KernReturnT; + fn vm_deallocate( + target_task: MachPortT, + address: libc::uintptr_t, + size: libc::uintptr_t, + ) -> KernReturnT; + } + + let self_task = unsafe { mach_task_self() }; + let mut proc_task: MachPortT = 0; + if unsafe { task_for_pid(self_task, libc::getpid(), &mut proc_task) } == KERN_SUCCESS { + let mut threads: ThreadActArrayT = core::ptr::null_mut(); + let mut n_threads: MachMsgTypeNumberT = 0; + if unsafe { task_threads(proc_task, &mut threads, &mut n_threads) } == KERN_SUCCESS + { + if !threads.is_null() { + let _ = unsafe { + vm_deallocate( + self_task, + threads as libc::uintptr_t, + (n_threads as usize * core::mem::size_of::<MachPortT>()) + as libc::uintptr_t, + ) + }; + } + return n_threads as isize; + } + } + 0 + } + #[cfg(target_os = "linux")] + { + use std::io::Read as _; + let mut file = match std::fs::File::open("/proc/self/stat") { + Ok(f) => f, + Err(_) => return 0, + }; + let mut buf = [0u8; 160]; + let n = match file.read(&mut buf) { + Ok(n) => n, + Err(_) => return 0, + }; + let line = match core::str::from_utf8(&buf[..n]) { + Ok(s) => s, + Err(_) => return 0, + }; + if let Some(field) = line.split_whitespace().nth(19) { + return field.parse::<isize>().unwrap_or(0); + } + 0 + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + 0 + } + } + + /// Warn if forking from a multi-threaded process. + /// `num_os_threads` should be captured before parent after-fork hooks run. + fn warn_if_multi_threaded(name: &str, num_os_threads: isize, vm: &VirtualMachine) { + let num_threads = if num_os_threads > 0 { + num_os_threads as usize + } else { + // CPython fallback: if OS-level count isn't available, use the + // threading module's active+limbo view. + // Only check threading if it was already imported. Avoid vm.import() + // which can execute arbitrary Python code in the fork path. + let threading = match vm + .sys_module + .get_attr("modules", vm) + .and_then(|m| m.get_item("threading", vm)) + { + Ok(m) => m, + Err(_) => return, + }; + let active = threading.get_attr("_active", vm).ok(); + let limbo = threading.get_attr("_limbo", vm).ok(); + + // Match threading module internals and avoid sequence overcounting: + // count only dict-backed _active/_limbo containers. + let count_dict = |obj: Option<crate::PyObjectRef>| -> usize { + obj.and_then(|o| { + o.downcast_ref::<crate::builtins::PyDict>() + .map(|d| d.__len__()) + }) + .unwrap_or(0) + }; + + count_dict(active) + count_dict(limbo) + }; + + if num_threads > 1 { + let pid = unsafe { libc::getpid() }; + let msg = format!( + "This process (pid={}) is multi-threaded, use of {}() may lead to deadlocks in the child.", + pid, name + ); + + // Match PyErr_WarnFormat(..., stacklevel=1) in CPython. + // Best effort: ignore failures like CPython does in this path. + let _ = + crate::stdlib::_warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm); + } + } + + #[pyfunction] + fn fork(vm: &VirtualMachine) -> PyResult<i32> { + if vm + .state + .finalizing + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.python_finalization_error.to_owned(), + "can't fork at interpreter shutdown".into(), + )); + } + + // RustPython does not yet have C-level audit hooks; call sys.audit() + // to preserve Python-visible behavior and failure semantics. + vm.sys_module + .get_attr("audit", vm)? + .call(("os.fork",), vm)?; + + py_os_before_fork(vm); + + let pid = unsafe { libc::fork() }; + // Save errno immediately — AfterFork callbacks may clobber it. + let saved_errno = nix::Error::last_raw(); + if pid == 0 { + py_os_after_fork_child(vm); + } else { + // Match CPython timing: capture this before parent after-fork hooks + // in case those hooks start threads. + let num_os_threads = get_number_of_os_threads(); + py_os_after_fork_parent(vm); + // Match CPython timing: warn only after parent callback path resumes world. + warn_if_multi_threaded("fork", num_os_threads, vm); + } + if pid == -1 { + Err(nix::Error::from_raw(saved_errno).into_pyexception(vm)) + } else { + Ok(pid) + } + } + + #[cfg(not(target_os = "redox"))] + const MKNOD_DIR_FD: bool = cfg!(not(target_vendor = "apple")); + + #[cfg(not(target_os = "redox"))] + #[derive(FromArgs)] + struct MknodArgs<'fd> { + #[pyarg(any)] + path: OsPath, + #[pyarg(any)] + mode: libc::mode_t, + #[pyarg(any)] + device: libc::dev_t, + #[pyarg(flatten)] + dir_fd: DirFd<'fd, { MKNOD_DIR_FD as usize }>, + } + + #[cfg(not(target_os = "redox"))] + impl MknodArgs<'_> { + fn _mknod(self, vm: &VirtualMachine) -> PyResult<i32> { + Ok(unsafe { + libc::mknod( + self.path.clone().into_cstring(vm)?.as_ptr(), + self.mode, + self.device, + ) + }) + } + + #[cfg(not(target_vendor = "apple"))] + fn mknod(self, vm: &VirtualMachine) -> PyResult<()> { + let ret = match self.dir_fd.raw_opt() { + None => self._mknod(vm)?, + Some(non_default_fd) => unsafe { + libc::mknodat( + non_default_fd, + self.path.clone().into_cstring(vm)?.as_ptr(), + self.mode, + self.device, + ) + }, + }; + if ret != 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + #[cfg(target_vendor = "apple")] + fn mknod(self, vm: &VirtualMachine) -> PyResult<()> { + let [] = self.dir_fd.0; + let ret = self._mknod(vm)?; + if ret != 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn mknod(args: MknodArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { + args.mknod(vm) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn nice(increment: i32, vm: &VirtualMachine) -> PyResult<i32> { + Errno::clear(); + let res = unsafe { libc::nice(increment) }; + if res == -1 && Errno::last_raw() != 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(res) + } + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn sched_get_priority_max(policy: i32, vm: &VirtualMachine) -> PyResult<i32> { + let max = unsafe { libc::sched_get_priority_max(policy) }; + if max == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(max) + } + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn sched_get_priority_min(policy: i32, vm: &VirtualMachine) -> PyResult<i32> { + let min = unsafe { libc::sched_get_priority_min(policy) }; + if min == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(min) + } + } + + #[pyfunction] + fn sched_yield(vm: &VirtualMachine) -> PyResult<()> { + nix::sched::sched_yield().map_err(|e| e.into_pyexception(vm)) + } + + #[pyfunction] + fn get_inheritable(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<bool> { + let flags = fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD); + match flags { + Ok(ret) => Ok((ret & libc::FD_CLOEXEC) == 0), + Err(err) => Err(err.into_pyexception(vm)), + } + } + + #[pyfunction] + fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { + super::set_inheritable(fd, inheritable).map_err(|err| err.into_pyexception(vm)) + } + + #[pyfunction] + fn get_blocking(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<bool> { + let flags = fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFL); + match flags { + Ok(ret) => Ok((ret & libc::O_NONBLOCK) == 0), + Err(err) => Err(err.into_pyexception(vm)), + } + } + + #[pyfunction] + fn set_blocking(fd: BorrowedFd<'_>, blocking: bool, vm: &VirtualMachine) -> PyResult<()> { + let _set_flag = || { + use nix::fcntl::{FcntlArg, OFlag, fcntl}; + + let flags = OFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFL)?); + let mut new_flags = flags; + new_flags.set(OFlag::from_bits_truncate(libc::O_NONBLOCK), !blocking); + if flags != new_flags { + fcntl(fd, FcntlArg::F_SETFL(new_flags))?; + } + Ok(()) + }; + _set_flag().map_err(|err: nix::Error| err.into_pyexception(vm)) + } + + #[pyfunction] + fn pipe(vm: &VirtualMachine) -> PyResult<(OwnedFd, OwnedFd)> { + use nix::unistd::pipe; + let (rfd, wfd) = pipe().map_err(|err| err.into_pyexception(vm))?; + set_inheritable(rfd.as_fd(), false, vm)?; + set_inheritable(wfd.as_fd(), false, vm)?; + Ok((rfd, wfd)) + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "emscripten", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd" + ))] + #[pyfunction] + fn pipe2(flags: libc::c_int, vm: &VirtualMachine) -> PyResult<(OwnedFd, OwnedFd)> { + let oflags = fcntl::OFlag::from_bits_truncate(flags); + nix::unistd::pipe2(oflags).map_err(|err| err.into_pyexception(vm)) + } + + fn _chmod( + path: OsPath, + dir_fd: DirFd<'_, 0>, + mode: u32, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult<()> { + let [] = dir_fd.0; + let err_path = path.clone(); + let body = move || { + use std::os::unix::fs::PermissionsExt; + let meta = fs_metadata(&path, follow_symlinks.0)?; + let mut permissions = meta.permissions(); + permissions.set_mode(mode); + fs::set_permissions(&path, permissions) + }; + body().map_err(|err| OSErrorBuilder::with_filename(&err, err_path, vm)) + } + + #[cfg(not(target_os = "redox"))] + fn _fchmod(fd: BorrowedFd<'_>, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + nix::sys::stat::fchmod( + fd, + nix::sys::stat::Mode::from_bits_truncate(mode as libc::mode_t), + ) + .map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn chmod( + path: OsPathOrFd<'_>, + dir_fd: DirFd<'_, 0>, + mode: u32, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult<()> { + match path { + OsPathOrFd::Path(path) => { + #[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd",))] + if !follow_symlinks.0 && dir_fd == Default::default() { + return lchmod(path, mode, vm); + } + _chmod(path, dir_fd, mode, follow_symlinks, vm) + } + OsPathOrFd::Fd(fd) => _fchmod(fd.into(), mode, vm), + } + } + + #[cfg(target_os = "redox")] + #[pyfunction] + fn chmod( + path: OsPath, + dir_fd: DirFd<0>, + mode: u32, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult<()> { + _chmod(path, dir_fd, mode, follow_symlinks, vm) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn fchmod(fd: BorrowedFd<'_>, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + _fchmod(fd, mode, vm) + } + + #[cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd",))] + #[pyfunction] + fn lchmod(path: OsPath, mode: u32, vm: &VirtualMachine) -> PyResult<()> { + unsafe extern "C" { + fn lchmod(path: *const libc::c_char, mode: libc::mode_t) -> libc::c_int; + } + let c_path = path.clone().into_cstring(vm)?; + if unsafe { lchmod(c_path.as_ptr(), mode as libc::mode_t) } == 0 { + Ok(()) + } else { + let err = std::io::Error::last_os_error(); + Err(OSErrorBuilder::with_filename(&err, path, vm)) + } + } + + #[pyfunction] + fn execv( + path: OsPath, + argv: Either<PyListRef, PyTupleRef>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let path = path.into_cstring(vm)?; + + let argv = vm.extract_elements_with(argv.as_ref(), |obj| { + OsPath::try_from_object(vm, obj)?.into_cstring(vm) + })?; + let argv: Vec<&CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); + + let first = argv + .first() + .ok_or_else(|| vm.new_value_error("execv() arg 2 must not be empty"))?; + if first.to_bytes().is_empty() { + return Err(vm.new_value_error("execv() arg 2 first element cannot be empty")); + } + + unistd::execv(&path, &argv) + .map(|_ok| ()) + .map_err(|err| err.into_pyexception(vm)) + } + + #[pyfunction] + fn execve( + path: OsPath, + argv: Either<PyListRef, PyTupleRef>, + env: ArgMapping, + vm: &VirtualMachine, + ) -> PyResult<()> { + let path = path.into_cstring(vm)?; + + let argv = vm.extract_elements_with(argv.as_ref(), |obj| { + OsPath::try_from_object(vm, obj)?.into_cstring(vm) + })?; + let argv: Vec<&CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); + + let first = argv + .first() + .ok_or_else(|| vm.new_value_error("execve() arg 2 must not be empty"))?; + + if first.to_bytes().is_empty() { + return Err(vm.new_value_error("execve() arg 2 first element cannot be empty")); + } + + let env = crate::stdlib::os::envobj_to_dict(env, vm)?; + let env = env + .into_iter() + .map(|(k, v)| -> PyResult<_> { + let (key, value) = ( + OsPath::try_from_object(vm, k)?.into_bytes(), + OsPath::try_from_object(vm, v)?.into_bytes(), + ); + + if key.is_empty() || memchr::memchr(b'=', &key).is_some() { + return Err(vm.new_value_error("illegal environment variable name")); + } + + let mut entry = key; + entry.push(b'='); + entry.extend_from_slice(&value); + + CString::new(entry).map_err(|err| err.into_pyexception(vm)) + }) + .collect::<Result<Vec<_>, _>>()?; + + let env: Vec<&CStr> = env.iter().map(|entry| entry.as_c_str()).collect(); + + unistd::execve(&path, &argv, &env).map_err(|err| err.into_pyexception(vm))?; + Ok(()) + } + + #[pyfunction] + fn getppid(vm: &VirtualMachine) -> PyObjectRef { + let ppid = unistd::getppid().as_raw(); + vm.ctx.new_int(ppid).into() + } + + #[pyfunction] + fn getgid(vm: &VirtualMachine) -> PyObjectRef { + let gid = unistd::getgid().as_raw(); + vm.ctx.new_int(gid).into() + } + + #[pyfunction] + fn getegid(vm: &VirtualMachine) -> PyObjectRef { + let egid = unistd::getegid().as_raw(); + vm.ctx.new_int(egid).into() + } + + #[pyfunction] + fn getpgid(pid: u32, vm: &VirtualMachine) -> PyResult { + let pgid = + unistd::getpgid(Some(Pid::from_raw(pid as i32))).map_err(|e| e.into_pyexception(vm))?; + Ok(vm.new_pyobj(pgid.as_raw())) + } + + #[pyfunction] + fn getpgrp(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.new_int(unistd::getpgrp().as_raw()).into() + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn getsid(pid: u32, vm: &VirtualMachine) -> PyResult { + let sid = + unistd::getsid(Some(Pid::from_raw(pid as i32))).map_err(|e| e.into_pyexception(vm))?; + Ok(vm.new_pyobj(sid.as_raw())) + } + + #[pyfunction] + fn getuid(vm: &VirtualMachine) -> PyObjectRef { + let uid = unistd::getuid().as_raw(); + vm.ctx.new_int(uid).into() + } + + #[pyfunction] + fn geteuid(vm: &VirtualMachine) -> PyObjectRef { + let euid = unistd::geteuid().as_raw(); + vm.ctx.new_int(euid).into() + } + + #[cfg(not(any(target_os = "wasi", target_os = "android")))] + #[pyfunction] + fn setgid(gid: Gid, vm: &VirtualMachine) -> PyResult<()> { + unistd::setgid(gid).map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(not(any(target_os = "wasi", target_os = "android", target_os = "redox")))] + #[pyfunction] + fn setegid(egid: Gid, vm: &VirtualMachine) -> PyResult<()> { + unistd::setegid(egid).map_err(|err| err.into_pyexception(vm)) + } + + #[pyfunction] + fn setpgid(pid: u32, pgid: u32, vm: &VirtualMachine) -> PyResult<()> { + unistd::setpgid(Pid::from_raw(pid as i32), Pid::from_raw(pgid as i32)) + .map_err(|err| err.into_pyexception(vm)) + } + + #[pyfunction] + fn setpgrp(vm: &VirtualMachine) -> PyResult<()> { + // setpgrp() is equivalent to setpgid(0, 0) + unistd::setpgid(Pid::from_raw(0), Pid::from_raw(0)).map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(not(any(target_os = "wasi", target_os = "redox")))] + #[pyfunction] + fn setsid(vm: &VirtualMachine) -> PyResult<()> { + unistd::setsid() + .map(|_ok| ()) + .map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(not(any(target_os = "wasi", target_os = "redox")))] + #[pyfunction] + fn tcgetpgrp(fd: i32, vm: &VirtualMachine) -> PyResult<libc::pid_t> { + use std::os::fd::BorrowedFd; + let fd = unsafe { BorrowedFd::borrow_raw(fd) }; + unistd::tcgetpgrp(fd) + .map(|pid| pid.as_raw()) + .map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(not(any(target_os = "wasi", target_os = "redox")))] + #[pyfunction] + fn tcsetpgrp(fd: i32, pgid: libc::pid_t, vm: &VirtualMachine) -> PyResult<()> { + use std::os::fd::BorrowedFd; + let fd = unsafe { BorrowedFd::borrow_raw(fd) }; + unistd::tcsetpgrp(fd, Pid::from_raw(pgid)).map_err(|err| err.into_pyexception(vm)) + } + + fn try_from_id(vm: &VirtualMachine, obj: PyObjectRef, typ_name: &str) -> PyResult<u32> { + use core::cmp::Ordering; + let i = obj + .try_to_ref::<PyInt>(vm) + .map_err(|_| { + vm.new_type_error(format!( + "an integer is required (got type {})", + obj.class().name() + )) + })? + .try_to_primitive::<i64>(vm)?; + + match i.cmp(&-1) { + Ordering::Greater => Ok(i.try_into().map_err(|_| { + vm.new_overflow_error(format!("{typ_name} is larger than maximum")) + })?), + Ordering::Less => { + Err(vm.new_overflow_error(format!("{typ_name} is less than minimum"))) + } + // -1 means does not change the value + // In CPython, this is `(uid_t) -1`, rustc gets mad when we try to declare + // a negative unsigned integer :). + Ordering::Equal => Ok(-1i32 as u32), + } + } + + impl TryFromObject for Uid { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + try_from_id(vm, obj, "uid").map(Self::from_raw) + } + } + + impl TryFromObject for Gid { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + try_from_id(vm, obj, "gid").map(Self::from_raw) + } + } + + #[cfg(not(any(target_os = "wasi", target_os = "android")))] + #[pyfunction] + fn setuid(uid: Uid) -> nix::Result<()> { + unistd::setuid(uid) + } + + #[cfg(not(any(target_os = "wasi", target_os = "android", target_os = "redox")))] + #[pyfunction] + fn seteuid(euid: Uid) -> nix::Result<()> { + unistd::seteuid(euid) + } + + #[cfg(not(any(target_os = "wasi", target_os = "android", target_os = "redox")))] + #[pyfunction] + fn setreuid(ruid: Uid, euid: Uid) -> nix::Result<()> { + let ret = unsafe { libc::setreuid(ruid.as_raw(), euid.as_raw()) }; + nix::Error::result(ret).map(drop) + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn setresuid(ruid: Uid, euid: Uid, suid: Uid) -> nix::Result<()> { + unistd::setresuid(ruid, euid, suid) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn openpty(vm: &VirtualMachine) -> PyResult<(OwnedFd, OwnedFd)> { + let r = nix::pty::openpty(None, None).map_err(|err| err.into_pyexception(vm))?; + for fd in [&r.master, &r.slave] { + super::set_inheritable(fd.as_fd(), false).map_err(|e| e.into_pyexception(vm))?; + } + Ok((r.master, r.slave)) + } + + #[pyfunction] + fn ttyname(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult { + let name = unistd::ttyname(fd).map_err(|e| e.into_pyexception(vm))?; + let name = name.into_os_string().into_string().unwrap(); + Ok(vm.ctx.new_str(name).into()) + } + + #[pyfunction] + fn umask(mask: libc::mode_t) -> libc::mode_t { + unsafe { libc::umask(mask) } + } + + #[pyfunction] + fn uname(vm: &VirtualMachine) -> PyResult<_os::UnameResultData> { + let info = uname::uname().map_err(|err| err.into_pyexception(vm))?; + Ok(_os::UnameResultData { + sysname: info.sysname, + nodename: info.nodename, + release: info.release, + version: info.version, + machine: info.machine, + }) + } + + #[pyfunction] + fn sync() { + #[cfg(not(any(target_os = "redox", target_os = "android")))] + unsafe { + libc::sync(); + } + } + + // cfg from nix + #[cfg(any(target_os = "android", target_os = "linux", target_os = "openbsd"))] + #[pyfunction] + fn getresuid() -> nix::Result<(u32, u32, u32)> { + let ret = unistd::getresuid()?; + Ok(( + ret.real.as_raw(), + ret.effective.as_raw(), + ret.saved.as_raw(), + )) + } + + // cfg from nix + #[cfg(any(target_os = "android", target_os = "linux", target_os = "openbsd"))] + #[pyfunction] + fn getresgid() -> nix::Result<(u32, u32, u32)> { + let ret = unistd::getresgid()?; + Ok(( + ret.real.as_raw(), + ret.effective.as_raw(), + ret.saved.as_raw(), + )) + } + + // cfg from nix + #[cfg(any(target_os = "freebsd", target_os = "linux", target_os = "openbsd"))] + #[pyfunction] + fn setresgid(rgid: Gid, egid: Gid, sgid: Gid, vm: &VirtualMachine) -> PyResult<()> { + unistd::setresgid(rgid, egid, sgid).map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(not(any(target_os = "wasi", target_os = "android", target_os = "redox")))] + #[pyfunction] + fn setregid(rgid: Gid, egid: Gid) -> nix::Result<()> { + let ret = unsafe { libc::setregid(rgid.as_raw(), egid.as_raw()) }; + nix::Error::result(ret).map(drop) + } + + // cfg from nix + #[cfg(any(target_os = "freebsd", target_os = "linux", target_os = "openbsd"))] + #[pyfunction] + fn initgroups(user_name: PyUtf8StrRef, gid: Gid, vm: &VirtualMachine) -> PyResult<()> { + let user = user_name.to_cstring(vm)?; + unistd::initgroups(&user, gid).map_err(|err| err.into_pyexception(vm)) + } + + // cfg from nix + #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "redox")))] + #[pyfunction] + fn setgroups( + group_ids: crate::function::ArgIterable<Gid>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let gids = group_ids.iter(vm)?.collect::<Result<Vec<_>, _>>()?; + unistd::setgroups(&gids).map_err(|err| err.into_pyexception(vm)) + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + fn envp_from_dict( + env: crate::function::ArgMapping, + vm: &VirtualMachine, + ) -> PyResult<Vec<CString>> { + let items = env.mapping().items(vm)?; + + // Convert items to list if it isn't already + let items = vm.ctx.new_list( + items + .get_iter(vm)? + .iter(vm)? + .collect::<PyResult<Vec<_>>>()?, + ); + + items + .borrow_vec() + .iter() + .map(|item| { + let tuple = item + .downcast_ref::<crate::builtins::PyTuple>() + .ok_or_else(|| vm.new_type_error("items() should return tuples"))?; + let tuple_items = tuple.as_slice(); + if tuple_items.len() != 2 { + return Err(vm.new_value_error("items() tuples should have exactly 2 elements")); + } + Ok((tuple_items[0].clone(), tuple_items[1].clone())) + }) + .collect::<PyResult<Vec<_>>>()? + .into_iter() + .map(|(k, v)| { + let k = OsPath::try_from_object(vm, k)?.into_bytes(); + let v = OsPath::try_from_object(vm, v)?.into_bytes(); + if k.contains(&0) { + return Err(vm.new_value_error("envp dict key cannot contain a nul byte")); + } + if k.contains(&b'=') { + return Err(vm.new_value_error("envp dict key cannot contain a '=' character")); + } + if v.contains(&0) { + return Err(vm.new_value_error("envp dict value cannot contain a nul byte")); + } + let mut env = k; + env.push(b'='); + env.extend(v); + Ok(unsafe { CString::from_vec_unchecked(env) }) + }) + .collect() + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[derive(FromArgs)] + pub(super) struct PosixSpawnArgs { + #[pyarg(positional)] + path: OsPath, + #[pyarg(positional)] + args: crate::function::ArgIterable<OsPath>, + #[pyarg(positional)] + env: Option<crate::function::ArgMapping>, + #[pyarg(named, default)] + file_actions: Option<crate::function::ArgIterable<PyTupleRef>>, + #[pyarg(named, default)] + setsigdef: Option<crate::function::ArgIterable<i32>>, + #[pyarg(named, default)] + setpgroup: Option<libc::pid_t>, + #[pyarg(named, default)] + resetids: bool, + #[pyarg(named, default)] + setsid: bool, + #[pyarg(named, default)] + setsigmask: Option<crate::function::ArgIterable<i32>>, + #[pyarg(named, default)] + scheduler: Option<PyTupleRef>, + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[derive(num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] + #[repr(i32)] + enum PosixSpawnFileActionIdentifier { + Open, + Close, + Dup2, + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + impl PosixSpawnArgs { + fn spawn(self, spawnp: bool, vm: &VirtualMachine) -> PyResult<libc::pid_t> { + use nix::sys::signal; + + use crate::TryFromBorrowedObject; + + let path = self + .path + .clone() + .into_cstring(vm) + .map_err(|_| vm.new_value_error("path should not have nul bytes"))?; + + let mut file_actions = + nix::spawn::PosixSpawnFileActions::init().map_err(|e| e.into_pyexception(vm))?; + if let Some(it) = self.file_actions { + for action in it.iter(vm)? { + let action = action?; + let (id, args) = action.split_first().ok_or_else(|| { + vm.new_type_error("Each file_actions element must be a non-empty tuple") + })?; + let id = i32::try_from_borrowed_object(vm, id)?; + let id = PosixSpawnFileActionIdentifier::try_from(id) + .map_err(|_| vm.new_type_error("Unknown file_actions identifier"))?; + let args: crate::function::FuncArgs = args.to_vec().into(); + let ret = match id { + PosixSpawnFileActionIdentifier::Open => { + let (fd, path, oflag, mode): (_, OsPath, _, _) = args.bind(vm)?; + let path = CString::new(path.into_bytes()).map_err(|_| { + vm.new_value_error( + "POSIX_SPAWN_OPEN path should not have nul bytes", + ) + })?; + let oflag = nix::fcntl::OFlag::from_bits_retain(oflag); + let mode = nix::sys::stat::Mode::from_bits_retain(mode); + file_actions.add_open(fd, &*path, oflag, mode) + } + PosixSpawnFileActionIdentifier::Close => { + let (fd,) = args.bind(vm)?; + file_actions.add_close(fd) + } + PosixSpawnFileActionIdentifier::Dup2 => { + let (fd, newfd) = args.bind(vm)?; + file_actions.add_dup2(fd, newfd) + } + }; + if let Err(err) = ret { + let err = err.into(); + return Err(OSErrorBuilder::with_filename(&err, self.path, vm)); + } + } + } + + let mut attrp = + nix::spawn::PosixSpawnAttr::init().map_err(|e| e.into_pyexception(vm))?; + let mut flags = nix::spawn::PosixSpawnFlags::empty(); + + if let Some(sigs) = self.setsigdef { + let mut set = signal::SigSet::empty(); + for sig in sigs.iter(vm)? { + let sig = sig?; + let sig = signal::Signal::try_from(sig).map_err(|_| { + vm.new_value_error(format!("signal number {sig} out of range")) + })?; + set.add(sig); + } + attrp + .set_sigdefault(&set) + .map_err(|e| e.into_pyexception(vm))?; + flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_SETSIGDEF); + } + + if let Some(pgid) = self.setpgroup { + attrp + .set_pgroup(nix::unistd::Pid::from_raw(pgid)) + .map_err(|e| e.into_pyexception(vm))?; + flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_SETPGROUP); + } + + if self.resetids { + flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_RESETIDS); + } + + if self.setsid { + // Note: POSIX_SPAWN_SETSID may not be available on all platforms + cfg_if::cfg_if! { + if #[cfg(any( + target_os = "linux", + target_os = "haiku", + target_os = "solaris", + target_os = "illumos", + target_os = "hurd", + ))] { + flags.insert(nix::spawn::PosixSpawnFlags::from_bits_retain(libc::POSIX_SPAWN_SETSID)); + } else { + return Err(vm.new_not_implemented_error( + "setsid parameter is not supported on this platform", + )); + } + } + } + + if let Some(sigs) = self.setsigmask { + let mut set = signal::SigSet::empty(); + for sig in sigs.iter(vm)? { + let sig = sig?; + let sig = signal::Signal::try_from(sig).map_err(|_| { + vm.new_value_error(format!("signal number {sig} out of range")) + })?; + set.add(sig); + } + attrp + .set_sigmask(&set) + .map_err(|e| e.into_pyexception(vm))?; + flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_SETSIGMASK); + } + + if let Some(_scheduler) = self.scheduler { + // TODO: Implement scheduler parameter handling + // This requires platform-specific sched_param struct handling + return Err( + vm.new_not_implemented_error("scheduler parameter is not yet implemented") + ); + } + + if !flags.is_empty() { + attrp.set_flags(flags).map_err(|e| e.into_pyexception(vm))?; + } + + let args: Vec<CString> = self + .args + .iter(vm)? + .map(|res| { + CString::new(res?.into_bytes()) + .map_err(|_| vm.new_value_error("path should not have nul bytes")) + }) + .collect::<Result<_, _>>()?; + let env = if let Some(env_dict) = self.env { + envp_from_dict(env_dict, vm)? + } else { + // env=None means use the current environment + + env::vars_os() + .map(|(k, v)| { + let mut entry = k.into_vec(); + entry.push(b'='); + entry.extend(v.into_vec()); + CString::new(entry).map_err(|_| { + vm.new_value_error("environment string contains null byte") + }) + }) + .collect::<PyResult<Vec<_>>>()? + }; + + let ret = if spawnp { + nix::spawn::posix_spawnp(&path, &file_actions, &attrp, &args, &env) + } else { + nix::spawn::posix_spawn(&*path, &file_actions, &attrp, &args, &env) + }; + ret.map(Into::into) + .map_err(|err| OSErrorBuilder::with_filename(&err.into(), self.path, vm)) + } + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[pyfunction] + fn posix_spawn(args: PosixSpawnArgs, vm: &VirtualMachine) -> PyResult<libc::pid_t> { + args.spawn(false, vm) + } + + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] + #[pyfunction] + fn posix_spawnp(args: PosixSpawnArgs, vm: &VirtualMachine) -> PyResult<libc::pid_t> { + args.spawn(true, vm) + } + + #[pyfunction(name = "WCOREDUMP")] + fn wcoredump(status: i32) -> bool { + libc::WCOREDUMP(status) + } + + #[pyfunction(name = "WIFCONTINUED")] + fn wifcontinued(status: i32) -> bool { + libc::WIFCONTINUED(status) + } + + #[pyfunction(name = "WIFSTOPPED")] + fn wifstopped(status: i32) -> bool { + libc::WIFSTOPPED(status) + } + + #[pyfunction(name = "WIFSIGNALED")] + fn wifsignaled(status: i32) -> bool { + libc::WIFSIGNALED(status) + } + + #[pyfunction(name = "WIFEXITED")] + fn wifexited(status: i32) -> bool { + libc::WIFEXITED(status) + } + + #[pyfunction(name = "WEXITSTATUS")] + fn wexitstatus(status: i32) -> i32 { + libc::WEXITSTATUS(status) + } + + #[pyfunction(name = "WSTOPSIG")] + fn wstopsig(status: i32) -> i32 { + libc::WSTOPSIG(status) + } + + #[pyfunction(name = "WTERMSIG")] + fn wtermsig(status: i32) -> i32 { + libc::WTERMSIG(status) + } + + #[cfg(target_os = "linux")] + #[pyfunction] + fn pidfd_open( + pid: libc::pid_t, + flags: OptionalArg<u32>, + vm: &VirtualMachine, + ) -> PyResult<OwnedFd> { + let flags = flags.unwrap_or(0); + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, flags) as libc::c_long }; + if fd == -1 { + Err(vm.new_last_errno_error()) + } else { + // Safety: syscall returns a new owned file descriptor. + Ok(unsafe { OwnedFd::from_raw_fd(fd as libc::c_int) }) + } + } + + #[pyfunction] + fn waitpid(pid: libc::pid_t, opt: i32, vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { + let mut status = 0; + loop { + // Capture errno inside the closure: attach_thread (called by + // allow_threads on return) can clobber errno via syscalls. + let (res, err) = vm.allow_threads(|| { + let r = unsafe { libc::waitpid(pid, &mut status, opt) }; + (r, nix::Error::last_raw()) + }); + if res == -1 { + if err == libc::EINTR { + vm.check_signals()?; + continue; + } + return Err(nix::Error::from_raw(err).into_pyexception(vm)); + } + return Ok((res, status)); + } + } + + #[pyfunction] + fn wait(vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { + waitpid(-1, 0, vm) + } + + #[pyfunction] + fn kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { + { + let ret = unsafe { libc::kill(pid, sig as i32) }; + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + } + + #[pyfunction] + fn get_terminal_size( + fd: OptionalArg<i32>, + vm: &VirtualMachine, + ) -> PyResult<_os::TerminalSizeData> { + let (columns, lines) = { + nix::ioctl_read_bad!(winsz, libc::TIOCGWINSZ, libc::winsize); + let mut w = libc::winsize { + ws_row: 0, + ws_col: 0, + ws_xpixel: 0, + ws_ypixel: 0, + }; + unsafe { winsz(fd.unwrap_or(libc::STDOUT_FILENO), &mut w) } + .map_err(|err| err.into_pyexception(vm))?; + (w.ws_col.into(), w.ws_row.into()) + }; + Ok(_os::TerminalSizeData { columns, lines }) + } + + // from libstd: + // https://github.com/rust-lang/rust/blob/daecab3a784f28082df90cebb204998051f3557d/src/libstd/sys/unix/fs.rs#L1251 + #[cfg(target_os = "macos")] + unsafe extern "C" { + fn fcopyfile( + in_fd: libc::c_int, + out_fd: libc::c_int, + state: *mut libc::c_void, // copyfile_state_t (unused) + flags: u32, // copyfile_flags_t + ) -> libc::c_int; + } + + #[cfg(target_os = "macos")] + #[pyfunction] + fn _fcopyfile(in_fd: i32, out_fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { fcopyfile(in_fd, out_fd, core::ptr::null_mut(), flags as u32) }; + if ret < 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + #[pyfunction] + fn dup(fd: BorrowedFd<'_>, vm: &VirtualMachine) -> PyResult<OwnedFd> { + let fd = nix::unistd::dup(fd).map_err(|e| e.into_pyexception(vm))?; + super::set_inheritable(fd.as_fd(), false) + .map(|()| fd) + .map_err(|e| e.into_pyexception(vm)) + } + + #[derive(FromArgs)] + struct Dup2Args<'fd> { + #[pyarg(positional)] + fd: BorrowedFd<'fd>, + #[pyarg(positional)] + fd2: OwnedFd, + #[pyarg(any, default = true)] + inheritable: bool, + } + + #[pyfunction] + fn dup2(args: Dup2Args<'_>, vm: &VirtualMachine) -> PyResult<OwnedFd> { + let mut fd2 = core::mem::ManuallyDrop::new(args.fd2); + nix::unistd::dup2(args.fd, &mut fd2).map_err(|e| e.into_pyexception(vm))?; + let fd2 = core::mem::ManuallyDrop::into_inner(fd2); + if !args.inheritable { + super::set_inheritable(fd2.as_fd(), false).map_err(|e| e.into_pyexception(vm))? + } + Ok(fd2) + } + + pub(crate) fn support_funcs() -> Vec<SupportFunc> { + vec![ + SupportFunc::new( + "chmod", + Some(false), + Some(false), + Some(cfg!(any( + target_os = "macos", + target_os = "freebsd", + target_os = "netbsd" + ))), + ), + #[cfg(not(target_os = "redox"))] + SupportFunc::new("chroot", Some(false), None, None), + #[cfg(not(target_os = "redox"))] + SupportFunc::new("chown", Some(true), Some(true), Some(true)), + #[cfg(not(target_os = "redox"))] + SupportFunc::new("lchown", None, None, None), + #[cfg(not(target_os = "redox"))] + SupportFunc::new("fchown", Some(true), None, Some(true)), + #[cfg(not(target_os = "redox"))] + SupportFunc::new("mknod", Some(true), Some(MKNOD_DIR_FD), Some(false)), + SupportFunc::new("umask", Some(false), Some(false), Some(false)), + SupportFunc::new("execv", None, None, None), + SupportFunc::new("pathconf", Some(true), None, None), + SupportFunc::new("fpathconf", Some(true), None, None), + SupportFunc::new("fchdir", Some(true), None, None), + ] + } + + #[pyfunction] + fn getlogin(vm: &VirtualMachine) -> PyResult<String> { + // Get a pointer to the login name string. The string is statically + // allocated and might be overwritten on subsequent calls to this + // function or to `cuserid()`. See man getlogin(3) for more information. + let ptr = unsafe { libc::getlogin() }; + if ptr.is_null() { + return Err(vm.new_os_error("unable to determine login name")); + } + let slice = unsafe { CStr::from_ptr(ptr) }; + slice + .to_str() + .map(|s| s.to_owned()) + .map_err(|e| vm.new_unicode_decode_error(format!("unable to decode login name: {e}"))) + } + + // cfg from nix + #[cfg(any( + target_os = "android", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + #[pyfunction] + fn getgrouplist( + user: PyUtf8StrRef, + group: u32, + vm: &VirtualMachine, + ) -> PyResult<Vec<PyObjectRef>> { + let user = user.to_cstring(vm)?; + let gid = Gid::from_raw(group); + let group_ids = unistd::getgrouplist(&user, gid).map_err(|err| err.into_pyexception(vm))?; + Ok(group_ids + .into_iter() + .map(|gid| vm.new_pyobj(gid.as_raw())) + .collect()) + } + + #[cfg(not(target_os = "redox"))] + cfg_if::cfg_if! { + if #[cfg(all(target_os = "linux", target_env = "gnu"))] { + type PriorityWhichType = libc::__priority_which_t; + } else { + type PriorityWhichType = libc::c_int; + } + } + #[cfg(not(target_os = "redox"))] + cfg_if::cfg_if! { + if #[cfg(target_os = "freebsd")] { + type PriorityWhoType = i32; + } else { + type PriorityWhoType = u32; + } + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn getpriority( + which: PriorityWhichType, + who: PriorityWhoType, + vm: &VirtualMachine, + ) -> PyResult { + Errno::clear(); + let retval = unsafe { libc::getpriority(which, who) }; + if Errno::last_raw() != 0 { + Err(vm.new_last_errno_error()) + } else { + Ok(vm.ctx.new_int(retval).into()) + } + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn setpriority( + which: PriorityWhichType, + who: PriorityWhoType, + priority: i32, + vm: &VirtualMachine, + ) -> PyResult<()> { + let retval = unsafe { libc::setpriority(which, who, priority) }; + if retval == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(()) + } + } + + struct PathconfName(i32); + + impl TryFromObject for PathconfName { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let i = match obj.downcast::<PyInt>() { + Ok(int) => int.try_to_primitive(vm)?, + Err(obj) => { + let s = obj.downcast::<PyUtf8Str>().map_err(|_| { + vm.new_type_error("configuration names must be strings or integers") + })?; + s.as_str() + .parse::<PathconfVar>() + .map_err(|_| vm.new_value_error("unrecognized configuration name"))? + as i32 + } + }; + Ok(Self(i)) + } + } + + // Copy from [nix::unistd::PathconfVar](https://docs.rs/nix/0.21.0/nix/unistd/enum.PathconfVar.html) + // Change enum name to fit python doc + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EnumIter, EnumString)] + #[repr(i32)] + #[allow(non_camel_case_types)] + pub enum PathconfVar { + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + ))] + /// Minimum number of bits needed to represent, as a signed integer value, + /// the maximum size of a regular file allowed in the specified directory. + PC_FILESIZEBITS = libc::_PC_FILESIZEBITS, + /// Maximum number of links to a single file. + PC_LINK_MAX = libc::_PC_LINK_MAX, + /// Maximum number of bytes in a terminal canonical input line. + PC_MAX_CANON = libc::_PC_MAX_CANON, + /// Minimum number of bytes for which space is available in a terminal input + /// queue; therefore, the maximum number of bytes a conforming application + /// may require to be typed as input before reading them. + PC_MAX_INPUT = libc::_PC_MAX_INPUT, + /// Maximum number of bytes in a filename (not including the terminating + /// null of a filename string). + PC_NAME_MAX = libc::_PC_NAME_MAX, + /// Maximum number of bytes the implementation will store as a pathname in a + /// user-supplied buffer of unspecified size, including the terminating null + /// character. Minimum number the implementation will accept as the maximum + /// number of bytes in a pathname. + PC_PATH_MAX = libc::_PC_PATH_MAX, + /// Maximum number of bytes that is guaranteed to be atomic when writing to + /// a pipe. + PC_PIPE_BUF = libc::_PC_PIPE_BUF, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox", + target_os = "solaris" + ))] + /// Symbolic links can be created. + PC_2_SYMLINKS = libc::_PC_2_SYMLINKS, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd", + target_os = "redox" + ))] + /// Minimum number of bytes of storage actually allocated for any portion of + /// a file. + PC_ALLOC_SIZE_MIN = libc::_PC_ALLOC_SIZE_MIN, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd" + ))] + /// Recommended increment for file transfer sizes between the + /// `POSIX_REC_MIN_XFER_SIZE` and `POSIX_REC_MAX_XFER_SIZE` values. + PC_REC_INCR_XFER_SIZE = libc::_PC_REC_INCR_XFER_SIZE, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd", + target_os = "redox" + ))] + /// Maximum recommended file transfer size. + PC_REC_MAX_XFER_SIZE = libc::_PC_REC_MAX_XFER_SIZE, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd", + target_os = "redox" + ))] + /// Minimum recommended file transfer size. + PC_REC_MIN_XFER_SIZE = libc::_PC_REC_MIN_XFER_SIZE, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "openbsd", + target_os = "redox" + ))] + /// Recommended file transfer buffer alignment. + PC_REC_XFER_ALIGN = libc::_PC_REC_XFER_ALIGN, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox", + target_os = "solaris" + ))] + /// Maximum number of bytes in a symbolic link. + PC_SYMLINK_MAX = libc::_PC_SYMLINK_MAX, + /// The use of `chown` and `fchown` is restricted to a process with + /// appropriate privileges, and to changing the group ID of a file only to + /// the effective group ID of the process or to one of its supplementary + /// group IDs. + PC_CHOWN_RESTRICTED = libc::_PC_CHOWN_RESTRICTED, + /// Pathname components longer than {NAME_MAX} generate an error. + PC_NO_TRUNC = libc::_PC_NO_TRUNC, + /// This symbol shall be defined to be the value of a character that shall + /// disable terminal special character handling. + PC_VDISABLE = libc::_PC_VDISABLE, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "illumos", + target_os = "linux", + target_os = "openbsd", + target_os = "redox", + target_os = "solaris" + ))] + /// Asynchronous input or output operations may be performed for the + /// associated file. + PC_ASYNC_IO = libc::_PC_ASYNC_IO, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "illumos", + target_os = "linux", + target_os = "openbsd", + target_os = "redox", + target_os = "solaris" + ))] + /// Prioritized input or output operations may be performed for the + /// associated file. + PC_PRIO_IO = libc::_PC_PRIO_IO, + #[cfg(any( + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "illumos", + target_os = "linux", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox", + target_os = "solaris" + ))] + /// Synchronized input or output operations may be performed for the + /// associated file. + PC_SYNC_IO = libc::_PC_SYNC_IO, + #[cfg(any(target_os = "dragonfly", target_os = "openbsd"))] + /// The resolution in nanoseconds for all file timestamps. + PC_TIMESTAMP_RESOLUTION = libc::_PC_TIMESTAMP_RESOLUTION, + } + + #[cfg(unix)] + #[pyfunction] + fn pathconf( + path: OsPathOrFd<'_>, + PathconfName(name): PathconfName, + vm: &VirtualMachine, + ) -> PyResult<Option<libc::c_long>> { + Errno::clear(); + debug_assert_eq!(Errno::last_raw(), 0); + let raw = match &path { + OsPathOrFd::Path(path) => { + let path = path.clone().into_cstring(vm)?; + unsafe { libc::pathconf(path.as_ptr(), name) } + } + OsPathOrFd::Fd(fd) => unsafe { libc::fpathconf(fd.as_raw(), name) }, + }; + + if raw == -1 { + if Errno::last_raw() == 0 { + Ok(None) + } else { + Err(OSErrorBuilder::with_filename( + &io::Error::from(Errno::last()), + path, + vm, + )) + } + } else { + Ok(Some(raw)) + } + } + + #[pyfunction] + fn fpathconf( + fd: BorrowedFd<'_>, + name: PathconfName, + vm: &VirtualMachine, + ) -> PyResult<Option<libc::c_long>> { + pathconf(OsPathOrFd::Fd(fd.into()), name, vm) + } + + #[pyattr] + fn pathconf_names(vm: &VirtualMachine) -> PyDictRef { + let pathname = vm.ctx.new_dict(); + for variant in PathconfVar::iter() { + // get the name of variant as a string to use as the dictionary key + let key = vm.ctx.new_str(format!("{variant:?}")); + // get the enum from the string and convert it to an integer for the dictionary value + let value = vm.ctx.new_int(variant as u8); + pathname + .set_item(&*key, value.into(), vm) + .expect("dict set_item unexpectedly failed"); + } + pathname + } + + #[cfg(not(target_os = "redox"))] + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EnumIter, EnumString)] + #[repr(i32)] + #[allow(non_camel_case_types)] + pub enum SysconfVar { + SC_2_CHAR_TERM = libc::_SC_2_CHAR_TERM, + SC_2_C_BIND = libc::_SC_2_C_BIND, + SC_2_C_DEV = libc::_SC_2_C_DEV, + SC_2_FORT_DEV = libc::_SC_2_FORT_DEV, + SC_2_FORT_RUN = libc::_SC_2_FORT_RUN, + SC_2_LOCALEDEF = libc::_SC_2_LOCALEDEF, + SC_2_SW_DEV = libc::_SC_2_SW_DEV, + SC_2_UPE = libc::_SC_2_UPE, + SC_2_VERSION = libc::_SC_2_VERSION, + SC_AIO_LISTIO_MAX = libc::_SC_AIO_LISTIO_MAX, + SC_AIO_MAX = libc::_SC_AIO_MAX, + SC_AIO_PRIO_DELTA_MAX = libc::_SC_AIO_PRIO_DELTA_MAX, + SC_ARG_MAX = libc::_SC_ARG_MAX, + SC_ASYNCHRONOUS_IO = libc::_SC_ASYNCHRONOUS_IO, + SC_ATEXIT_MAX = libc::_SC_ATEXIT_MAX, + SC_BC_BASE_MAX = libc::_SC_BC_BASE_MAX, + SC_BC_DIM_MAX = libc::_SC_BC_DIM_MAX, + SC_BC_SCALE_MAX = libc::_SC_BC_SCALE_MAX, + SC_BC_STRING_MAX = libc::_SC_BC_STRING_MAX, + SC_CHILD_MAX = libc::_SC_CHILD_MAX, + SC_CLK_TCK = libc::_SC_CLK_TCK, + SC_COLL_WEIGHTS_MAX = libc::_SC_COLL_WEIGHTS_MAX, + SC_DELAYTIMER_MAX = libc::_SC_DELAYTIMER_MAX, + SC_EXPR_NEST_MAX = libc::_SC_EXPR_NEST_MAX, + SC_FSYNC = libc::_SC_FSYNC, + SC_GETGR_R_SIZE_MAX = libc::_SC_GETGR_R_SIZE_MAX, + SC_GETPW_R_SIZE_MAX = libc::_SC_GETPW_R_SIZE_MAX, + SC_IOV_MAX = libc::_SC_IOV_MAX, + SC_JOB_CONTROL = libc::_SC_JOB_CONTROL, + SC_LINE_MAX = libc::_SC_LINE_MAX, + SC_LOGIN_NAME_MAX = libc::_SC_LOGIN_NAME_MAX, + SC_MAPPED_FILES = libc::_SC_MAPPED_FILES, + SC_MEMLOCK = libc::_SC_MEMLOCK, + SC_MEMLOCK_RANGE = libc::_SC_MEMLOCK_RANGE, + SC_MEMORY_PROTECTION = libc::_SC_MEMORY_PROTECTION, + SC_MESSAGE_PASSING = libc::_SC_MESSAGE_PASSING, + SC_MQ_OPEN_MAX = libc::_SC_MQ_OPEN_MAX, + SC_MQ_PRIO_MAX = libc::_SC_MQ_PRIO_MAX, + SC_NGROUPS_MAX = libc::_SC_NGROUPS_MAX, + SC_NPROCESSORS_CONF = libc::_SC_NPROCESSORS_CONF, + SC_NPROCESSORS_ONLN = libc::_SC_NPROCESSORS_ONLN, + SC_OPEN_MAX = libc::_SC_OPEN_MAX, + SC_PAGE_SIZE = libc::_SC_PAGE_SIZE, + #[cfg(any( + target_os = "linux", + target_vendor = "apple", + target_os = "netbsd", + target_os = "fuchsia" + ))] + SC_PASS_MAX = libc::_SC_PASS_MAX, + SC_PHYS_PAGES = libc::_SC_PHYS_PAGES, + SC_PRIORITIZED_IO = libc::_SC_PRIORITIZED_IO, + SC_PRIORITY_SCHEDULING = libc::_SC_PRIORITY_SCHEDULING, + SC_REALTIME_SIGNALS = libc::_SC_REALTIME_SIGNALS, + SC_RE_DUP_MAX = libc::_SC_RE_DUP_MAX, + SC_RTSIG_MAX = libc::_SC_RTSIG_MAX, + SC_SAVED_IDS = libc::_SC_SAVED_IDS, + SC_SEMAPHORES = libc::_SC_SEMAPHORES, + SC_SEM_NSEMS_MAX = libc::_SC_SEM_NSEMS_MAX, + SC_SEM_VALUE_MAX = libc::_SC_SEM_VALUE_MAX, + SC_SHARED_MEMORY_OBJECTS = libc::_SC_SHARED_MEMORY_OBJECTS, + SC_SIGQUEUE_MAX = libc::_SC_SIGQUEUE_MAX, + SC_STREAM_MAX = libc::_SC_STREAM_MAX, + SC_SYNCHRONIZED_IO = libc::_SC_SYNCHRONIZED_IO, + SC_THREADS = libc::_SC_THREADS, + SC_THREAD_ATTR_STACKADDR = libc::_SC_THREAD_ATTR_STACKADDR, + SC_THREAD_ATTR_STACKSIZE = libc::_SC_THREAD_ATTR_STACKSIZE, + SC_THREAD_DESTRUCTOR_ITERATIONS = libc::_SC_THREAD_DESTRUCTOR_ITERATIONS, + SC_THREAD_KEYS_MAX = libc::_SC_THREAD_KEYS_MAX, + SC_THREAD_PRIORITY_SCHEDULING = libc::_SC_THREAD_PRIORITY_SCHEDULING, + SC_THREAD_PRIO_INHERIT = libc::_SC_THREAD_PRIO_INHERIT, + SC_THREAD_PRIO_PROTECT = libc::_SC_THREAD_PRIO_PROTECT, + SC_THREAD_PROCESS_SHARED = libc::_SC_THREAD_PROCESS_SHARED, + SC_THREAD_SAFE_FUNCTIONS = libc::_SC_THREAD_SAFE_FUNCTIONS, + SC_THREAD_STACK_MIN = libc::_SC_THREAD_STACK_MIN, + SC_THREAD_THREADS_MAX = libc::_SC_THREAD_THREADS_MAX, + SC_TIMERS = libc::_SC_TIMERS, + SC_TIMER_MAX = libc::_SC_TIMER_MAX, + SC_TTY_NAME_MAX = libc::_SC_TTY_NAME_MAX, + SC_TZNAME_MAX = libc::_SC_TZNAME_MAX, + SC_VERSION = libc::_SC_VERSION, + SC_XOPEN_CRYPT = libc::_SC_XOPEN_CRYPT, + SC_XOPEN_ENH_I18N = libc::_SC_XOPEN_ENH_I18N, + SC_XOPEN_LEGACY = libc::_SC_XOPEN_LEGACY, + SC_XOPEN_REALTIME = libc::_SC_XOPEN_REALTIME, + SC_XOPEN_REALTIME_THREADS = libc::_SC_XOPEN_REALTIME_THREADS, + SC_XOPEN_SHM = libc::_SC_XOPEN_SHM, + SC_XOPEN_UNIX = libc::_SC_XOPEN_UNIX, + SC_XOPEN_VERSION = libc::_SC_XOPEN_VERSION, + SC_XOPEN_XCU_VERSION = libc::_SC_XOPEN_XCU_VERSION, + #[cfg(any( + target_os = "linux", + target_vendor = "apple", + target_os = "netbsd", + target_os = "fuchsia" + ))] + SC_XBS5_ILP32_OFF32 = libc::_SC_XBS5_ILP32_OFF32, + #[cfg(any( + target_os = "linux", + target_vendor = "apple", + target_os = "netbsd", + target_os = "fuchsia" + ))] + SC_XBS5_ILP32_OFFBIG = libc::_SC_XBS5_ILP32_OFFBIG, + #[cfg(any( + target_os = "linux", + target_vendor = "apple", + target_os = "netbsd", + target_os = "fuchsia" + ))] + SC_XBS5_LP64_OFF64 = libc::_SC_XBS5_LP64_OFF64, + #[cfg(any( + target_os = "linux", + target_vendor = "apple", + target_os = "netbsd", + target_os = "fuchsia" + ))] + SC_XBS5_LPBIG_OFFBIG = libc::_SC_XBS5_LPBIG_OFFBIG, + } + + #[cfg(target_os = "redox")] + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EnumIter, EnumString)] + #[repr(i32)] + #[allow(non_camel_case_types)] + pub enum SysconfVar { + SC_ARG_MAX = libc::_SC_ARG_MAX, + SC_CHILD_MAX = libc::_SC_CHILD_MAX, + SC_CLK_TCK = libc::_SC_CLK_TCK, + SC_NGROUPS_MAX = libc::_SC_NGROUPS_MAX, + SC_OPEN_MAX = libc::_SC_OPEN_MAX, + SC_STREAM_MAX = libc::_SC_STREAM_MAX, + SC_TZNAME_MAX = libc::_SC_TZNAME_MAX, + SC_VERSION = libc::_SC_VERSION, + SC_PAGE_SIZE = libc::_SC_PAGE_SIZE, + SC_RE_DUP_MAX = libc::_SC_RE_DUP_MAX, + SC_LOGIN_NAME_MAX = libc::_SC_LOGIN_NAME_MAX, + SC_TTY_NAME_MAX = libc::_SC_TTY_NAME_MAX, + SC_SYMLOOP_MAX = libc::_SC_SYMLOOP_MAX, + SC_HOST_NAME_MAX = libc::_SC_HOST_NAME_MAX, + } + + impl SysconfVar { + pub const SC_PAGESIZE: Self = Self::SC_PAGE_SIZE; + } + + struct SysconfName(i32); + + impl TryFromObject for SysconfName { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let i = match obj.downcast::<PyInt>() { + Ok(int) => int.try_to_primitive(vm)?, + Err(obj) => { + let s = obj.downcast::<PyUtf8Str>().map_err(|_| { + vm.new_type_error("configuration names must be strings or integers") + })?; + { + let name = s.as_str(); + name.parse::<SysconfVar>().or_else(|_| { + if name == "SC_PAGESIZE" { + Ok(SysconfVar::SC_PAGESIZE) + } else { + Err(vm.new_value_error("unrecognized configuration name")) + } + })? as i32 + } + } + }; + Ok(Self(i)) + } + } + + #[pyfunction] + fn sysconf(name: SysconfName, vm: &VirtualMachine) -> PyResult<libc::c_long> { + crate::common::os::set_errno(0); + let r = unsafe { libc::sysconf(name.0) }; + if r == -1 && crate::common::os::get_errno() != 0 { + return Err(vm.new_last_errno_error()); + } + Ok(r) + } + + #[pyattr] + fn sysconf_names(vm: &VirtualMachine) -> PyDictRef { + let names = vm.ctx.new_dict(); + for variant in SysconfVar::iter() { + // get the name of variant as a string to use as the dictionary key + let key = vm.ctx.new_str(format!("{variant:?}")); + // get the enum from the string and convert it to an integer for the dictionary value + let value = vm.ctx.new_int(variant as u8); + names + .set_item(&*key, value.into(), vm) + .expect("dict set_item unexpectedly failed"); + } + names + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[derive(FromArgs)] + struct SendFileArgs<'fd> { + out_fd: BorrowedFd<'fd>, + in_fd: BorrowedFd<'fd>, + offset: crate::common::crt_fd::Offset, + count: i64, + #[cfg(target_os = "macos")] + #[pyarg(any, optional)] + headers: OptionalArg<PyObjectRef>, + #[cfg(target_os = "macos")] + #[pyarg(any, optional)] + trailers: OptionalArg<PyObjectRef>, + #[cfg(target_os = "macos")] + #[allow(dead_code)] + #[pyarg(any, default)] + // TODO: not implemented + flags: OptionalArg<i32>, + } + + #[cfg(target_os = "linux")] + #[pyfunction] + fn sendfile(args: SendFileArgs<'_>, vm: &VirtualMachine) -> PyResult { + let mut file_offset = args.offset; + + let res = nix::sys::sendfile::sendfile( + args.out_fd, + args.in_fd, + Some(&mut file_offset), + args.count as usize, + ) + .map_err(|err| err.into_pyexception(vm))?; + Ok(vm.ctx.new_int(res as u64).into()) + } + + #[cfg(target_os = "macos")] + fn _extract_vec_bytes( + x: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<Option<Vec<crate::function::ArgBytesLike>>> { + x.into_option() + .map(|x| { + let v: Vec<crate::function::ArgBytesLike> = x.try_to_value(vm)?; + Ok(if v.is_empty() { None } else { Some(v) }) + }) + .transpose() + .map(Option::flatten) + } + + #[cfg(target_os = "macos")] + #[pyfunction] + fn sendfile(args: SendFileArgs<'_>, vm: &VirtualMachine) -> PyResult { + let headers = _extract_vec_bytes(args.headers, vm)?; + let count = headers + .as_ref() + .map(|v| v.iter().map(|s| s.len()).sum()) + .unwrap_or(0) as i64 + + args.count; + + let headers = headers + .as_ref() + .map(|v| v.iter().map(|b| b.borrow_buf()).collect::<Vec<_>>()); + let headers = headers + .as_ref() + .map(|v| v.iter().map(|borrowed| &**borrowed).collect::<Vec<_>>()); + let headers = headers.as_deref(); + + let trailers = _extract_vec_bytes(args.trailers, vm)?; + let trailers = trailers + .as_ref() + .map(|v| v.iter().map(|b| b.borrow_buf()).collect::<Vec<_>>()); + let trailers = trailers + .as_ref() + .map(|v| v.iter().map(|borrowed| &**borrowed).collect::<Vec<_>>()); + let trailers = trailers.as_deref(); + + let (res, written) = nix::sys::sendfile::sendfile( + args.in_fd, + args.out_fd, + args.offset, + Some(count), + headers, + trailers, + ); + // On macOS, sendfile can return EAGAIN even when some bytes were written. + // In that case, we should return the number of bytes written rather than + // raising an exception. Only raise an error if no bytes were written. + if let Err(err) = res + && written == 0 + { + return Err(err.into_pyexception(vm)); + } + Ok(vm.ctx.new_int(written as u64).into()) + } + + #[cfg(target_os = "linux")] + unsafe fn sys_getrandom(buf: *mut libc::c_void, buflen: usize, flags: u32) -> isize { + unsafe { libc::syscall(libc::SYS_getrandom, buf, buflen, flags as usize) as _ } + } + + #[cfg(target_os = "linux")] + #[pyfunction] + fn getrandom(size: isize, flags: OptionalArg<u32>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { + let size = usize::try_from(size) + .map_err(|_| vm.new_os_error(format!("Invalid argument for size: {size}")))?; + let mut buf = Vec::with_capacity(size); + unsafe { + let len = sys_getrandom( + buf.as_mut_ptr() as *mut libc::c_void, + size, + flags.unwrap_or(0), + ) + .try_into() + .map_err(|_| vm.new_last_os_error())?; + buf.set_len(len); + } + Ok(buf) + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } +} + +#[cfg(any( + target_os = "linux", + target_os = "netbsd", + target_os = "freebsd", + target_os = "android" +))] +#[pymodule(sub)] +mod posix_sched { + use crate::{ + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, builtins::PyTupleRef, + convert::ToPyObject, function::FuncArgs, types::PyStructSequence, + }; + + #[derive(FromArgs)] + struct SchedParamArgs { + #[pyarg(any)] + sched_priority: PyObjectRef, + } + + #[pystruct_sequence_data] + struct SchedParamData { + pub sched_priority: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence(name = "sched_param", module = "posix", data = "SchedParamData")] + struct PySchedParam; + + #[pyclass(with(PyStructSequence))] + impl PySchedParam { + #[pyslot] + fn slot_new( + cls: crate::builtins::PyTypeRef, + args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult { + use crate::PyPayload; + let SchedParamArgs { sched_priority } = args.bind(vm)?; + let items = vec![sched_priority]; + crate::builtins::PyTuple::new_unchecked(items.into_boxed_slice()) + .into_ref_with_type(vm, cls) + .map(Into::into) + } + + #[extend_class] + fn extend_pyclass(ctx: &crate::vm::Context, class: &'static Py<crate::builtins::PyType>) { + // Override __reduce__ to return (type, (sched_priority,)) + // instead of the generic structseq (type, ((sched_priority,),)). + // The trait's extend_class checks contains_key before setting default. + const SCHED_PARAM_REDUCE: crate::function::PyMethodDef = + crate::function::PyMethodDef::new_const( + "__reduce__", + |zelf: crate::PyRef<crate::builtins::PyTuple>, + vm: &VirtualMachine| + -> PyTupleRef { + vm.new_tuple((zelf.class().to_owned(), (zelf[0].clone(),))) + }, + crate::function::PyMethodFlags::METHOD, + None, + ); + class.set_attr( + ctx.intern_str("__reduce__"), + SCHED_PARAM_REDUCE.to_proper_method(class, ctx), + ); + } + } + + #[cfg(not(target_env = "musl"))] + fn convert_sched_param(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<libc::sched_param> { + use crate::{ + builtins::{PyInt, PyTuple}, + class::StaticType, + }; + if !obj.fast_isinstance(PySchedParam::static_type()) { + return Err(vm.new_type_error("must have a sched_param object")); + } + let tuple = obj.downcast_ref::<PyTuple>().unwrap(); + let priority = tuple[0].clone(); + let priority_type = priority.class().name().to_string(); + let value = priority.downcast::<PyInt>().map_err(|_| { + vm.new_type_error(format!("an integer is required (got type {priority_type})")) + })?; + let sched_priority = value.try_to_primitive(vm)?; + Ok(libc::sched_param { sched_priority }) + } + + #[pyfunction] + fn sched_getscheduler(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult<i32> { + let policy = unsafe { libc::sched_getscheduler(pid) }; + if policy == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(policy) + } + } + + #[cfg(not(target_env = "musl"))] + #[derive(FromArgs)] + struct SchedSetschedulerArgs { + #[pyarg(positional)] + pid: i32, + #[pyarg(positional)] + policy: i32, + #[pyarg(positional)] + sched_param: PyObjectRef, + } + + #[cfg(not(target_env = "musl"))] + #[pyfunction] + fn sched_setscheduler(args: SchedSetschedulerArgs, vm: &VirtualMachine) -> PyResult<i32> { + let libc_sched_param = convert_sched_param(&args.sched_param, vm)?; + let policy = unsafe { libc::sched_setscheduler(args.pid, args.policy, &libc_sched_param) }; + if policy == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(policy) + } + } + + #[pyfunction] + fn sched_getparam(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult<PyTupleRef> { + let param = unsafe { + let mut param = core::mem::MaybeUninit::uninit(); + if -1 == libc::sched_getparam(pid, param.as_mut_ptr()) { + return Err(vm.new_last_errno_error()); + } + param.assume_init() + }; + Ok(PySchedParam::from_data( + SchedParamData { + sched_priority: param.sched_priority.to_pyobject(vm), + }, + vm, + )) + } + + #[cfg(not(target_env = "musl"))] + #[derive(FromArgs)] + struct SchedSetParamArgs { + #[pyarg(positional)] + pid: i32, + #[pyarg(positional)] + sched_param: PyObjectRef, + } + + #[cfg(not(target_env = "musl"))] + #[pyfunction] + fn sched_setparam(args: SchedSetParamArgs, vm: &VirtualMachine) -> PyResult<i32> { + let libc_sched_param = convert_sched_param(&args.sched_param, vm)?; + let ret = unsafe { libc::sched_setparam(args.pid, &libc_sched_param) }; + if ret == -1 { + Err(vm.new_last_errno_error()) + } else { + Ok(ret) + } + } +} diff --git a/crates/vm/src/stdlib/posix_compat.rs b/crates/vm/src/stdlib/posix_compat.rs new file mode 100644 index 00000000000..89d3d94d7b2 --- /dev/null +++ b/crates/vm/src/stdlib/posix_compat.rs @@ -0,0 +1,78 @@ +// spell-checker:disable + +//! `posix` compatible module for `not(any(unix, windows))` + +pub(crate) use module::module_def; + +#[pymodule(name = "posix", with(super::os::_os))] +pub(crate) mod module { + use crate::{ + Py, PyObjectRef, PyResult, VirtualMachine, + builtins::PyStrRef, + convert::IntoPyException, + ospath::OsPath, + stdlib::os::{_os, DirFd, SupportFunc, TargetIsDirectory}, + }; + use std::{env, fs}; + + #[pyfunction] + pub(super) fn access(_path: PyStrRef, _mode: u8, vm: &VirtualMachine) -> PyResult<bool> { + os_unimpl("os.access", vm) + } + + #[pyfunction] + #[pyfunction(name = "unlink")] + fn remove(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult<()> { + let [] = dir_fd.0; + fs::remove_file(&path).map_err(|err| err.into_pyexception(vm)) + } + + #[derive(FromArgs)] + #[allow(unused)] + pub(super) struct SymlinkArgs<'a> { + src: OsPath, + dst: OsPath, + #[pyarg(flatten)] + _target_is_directory: TargetIsDirectory, + #[pyarg(flatten)] + _dir_fd: DirFd<'a, { _os::SYMLINK_DIR_FD as usize }>, + } + + #[pyfunction] + pub(super) fn symlink(_args: SymlinkArgs<'_>, vm: &VirtualMachine) -> PyResult<()> { + os_unimpl("os.symlink", vm) + } + + #[cfg(target_os = "wasi")] + #[pyattr] + fn environ(vm: &VirtualMachine) -> crate::builtins::PyDictRef { + use rustpython_common::os::ffi::OsStringExt; + + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars_os() { + let key: PyObjectRef = vm.ctx.new_bytes(key.into_vec()).into(); + let value: PyObjectRef = vm.ctx.new_bytes(value.into_vec()).into(); + environ.set_item(&*key, value, vm).unwrap(); + } + + environ + } + + #[allow(dead_code)] + fn os_unimpl<T>(func: &str, vm: &VirtualMachine) -> PyResult<T> { + Err(vm.new_os_error(format!("{} is not supported on this platform", func))) + } + + pub(crate) fn support_funcs() -> Vec<SupportFunc> { + Vec::new() + } + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + __module_exec(vm, module); + super::super::os::module_exec(vm, module)?; + Ok(()) + } +} diff --git a/crates/vm/src/stdlib/pwd.rs b/crates/vm/src/stdlib/pwd.rs new file mode 100644 index 00000000000..e8aee608cc7 --- /dev/null +++ b/crates/vm/src/stdlib/pwd.rs @@ -0,0 +1,117 @@ +// spell-checker:disable + +pub(crate) use pwd::module_def; + +#[pymodule] +mod pwd { + use crate::{ + PyResult, VirtualMachine, + builtins::{PyIntRef, PyUtf8StrRef}, + convert::IntoPyException, + exceptions, + types::PyStructSequence, + }; + use nix::unistd::{self, User}; + + #[cfg(not(target_os = "android"))] + use crate::{PyObjectRef, convert::ToPyObject}; + + #[pystruct_sequence_data] + struct PasswdData { + pw_name: String, + pw_passwd: String, + pw_uid: u32, + pw_gid: u32, + pw_gecos: String, + pw_dir: String, + pw_shell: String, + } + + #[pyattr] + #[pystruct_sequence(name = "struct_passwd", module = "pwd", data = "PasswdData")] + struct PyPasswd; + + #[pyclass(with(PyStructSequence))] + impl PyPasswd {} + + impl From<User> for PasswdData { + fn from(user: User) -> Self { + // this is just a pain... + let cstr_lossy = |s: alloc::ffi::CString| { + s.into_string() + .unwrap_or_else(|e| e.into_cstring().to_string_lossy().into_owned()) + }; + let pathbuf_lossy = |p: std::path::PathBuf| { + p.into_os_string() + .into_string() + .unwrap_or_else(|s| s.to_string_lossy().into_owned()) + }; + PasswdData { + pw_name: user.name, + pw_passwd: cstr_lossy(user.passwd), + pw_uid: user.uid.as_raw(), + pw_gid: user.gid.as_raw(), + pw_gecos: cstr_lossy(user.gecos), + pw_dir: pathbuf_lossy(user.dir), + pw_shell: pathbuf_lossy(user.shell), + } + } + } + + #[pyfunction] + fn getpwnam(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<PasswdData> { + let pw_name = name.as_str(); + if pw_name.contains('\0') { + return Err(exceptions::cstring_error(vm)); + } + let user = User::from_name(name.as_str()).ok().flatten(); + let user = user.ok_or_else(|| { + vm.new_key_error( + vm.ctx + .new_str(format!("getpwnam(): name not found: {pw_name}")) + .into(), + ) + })?; + Ok(PasswdData::from(user)) + } + + #[pyfunction] + fn getpwuid(uid: PyIntRef, vm: &VirtualMachine) -> PyResult<PasswdData> { + let uid_t = libc::uid_t::try_from(uid.as_bigint()) + .map(unistd::Uid::from_raw) + .ok(); + let user = uid_t + .map(User::from_uid) + .transpose() + .map_err(|err| err.into_pyexception(vm))? + .flatten(); + let user = user.ok_or_else(|| { + vm.new_key_error( + vm.ctx + .new_str(format!("getpwuid(): uid not found: {}", uid.as_bigint())) + .into(), + ) + })?; + Ok(PasswdData::from(user)) + } + + // TODO: maybe merge this functionality into nix? + #[cfg(not(target_os = "android"))] + #[pyfunction] + fn getpwall(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { + // setpwent, getpwent, etc are not thread safe. Could use fgetpwent_r, but this is easier + static GETPWALL: parking_lot::Mutex<()> = parking_lot::Mutex::new(()); + let _guard = GETPWALL.lock(); + let mut list = Vec::new(); + + unsafe { libc::setpwent() }; + while let Some(ptr) = core::ptr::NonNull::new(unsafe { libc::getpwent() }) { + let user = User::from(unsafe { ptr.as_ref() }); + let passwd = PasswdData::from(user).to_pyobject(vm); + list.push(passwd); + } + unsafe { libc::endpwent() }; + + Ok(list) + } +} diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs new file mode 100644 index 00000000000..33325c9dc60 --- /dev/null +++ b/crates/vm/src/stdlib/sys.rs @@ -0,0 +1,1911 @@ +pub(crate) mod monitoring; + +use crate::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule, convert::ToPyObject}; + +#[cfg(all(not(feature = "host_env"), feature = "stdio"))] +pub(crate) use sys::SandboxStdio; +pub(crate) use sys::{DOC, MAXSIZE, RUST_MULTIARCH, UnraisableHookArgsData, module_def, multiarch}; + +#[pymodule(name = "_jit")] +mod sys_jit { + /// Return True if the current Python executable supports JIT compilation, + /// and False otherwise. + #[pyfunction] + const fn is_available() -> bool { + false // RustPython has no JIT + } + + /// Return True if JIT compilation is enabled for the current Python process, + /// and False otherwise. + #[pyfunction] + const fn is_enabled() -> bool { + false // RustPython has no JIT + } + + /// Return True if the topmost Python frame is currently executing JIT code, + /// and False otherwise. + #[pyfunction] + const fn is_active() -> bool { + false // RustPython has no JIT + } +} + +#[pymodule] +mod sys { + use crate::{ + AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, + builtins::{ + PyBaseExceptionRef, PyDictRef, PyFrozenSet, PyNamespace, PyStr, PyStrRef, PyTuple, + PyTupleRef, PyTypeRef, PyUtf8StrRef, + }, + common::{ + ascii, + hash::{PyHash, PyUHash}, + }, + convert::ToPyObject, + frame::{Frame, FrameRef}, + function::{FuncArgs, KwArgs, OptionalArg, PosArgs}, + stdlib::{_warnings::warn, builtins}, + types::PyStructSequence, + version, + vm::{Settings, VirtualMachine}, + }; + use core::sync::atomic::Ordering; + use num_traits::ToPrimitive; + use std::{ + env::{self, VarError}, + io::{IsTerminal, Read, Write}, + }; + + #[cfg(windows)] + use windows_sys::Win32::{ + Foundation::MAX_PATH, + Storage::FileSystem::{ + GetFileVersionInfoSizeW, GetFileVersionInfoW, VS_FIXEDFILEINFO, VerQueryValueW, + }, + System::LibraryLoader::{GetModuleFileNameW, GetModuleHandleW}, + }; + + // Rust target triple (e.g., "x86_64-unknown-linux-gnu") + pub(crate) const RUST_MULTIARCH: &str = env!("RUSTPYTHON_TARGET_TRIPLE"); + + /// Convert Rust target triple to CPython-style multiarch + /// e.g., "x86_64-unknown-linux-gnu" -> "x86_64-linux-gnu" + pub(crate) fn multiarch() -> String { + RUST_MULTIARCH.replace("-unknown", "") + } + + #[pymodule(name = "monitoring", with(super::monitoring::sys_monitoring))] + pub(super) mod monitoring {} + + #[pyclass(no_attr, name = "_BootstrapStderr")] + #[derive(Debug, PyPayload)] + pub(super) struct BootstrapStderr; + + #[pyclass] + impl BootstrapStderr { + #[pymethod] + fn write(&self, s: PyStrRef) -> PyResult<usize> { + let bytes = s.as_bytes(); + let _ = std::io::stderr().write_all(bytes); + Ok(bytes.len()) + } + + #[pymethod] + fn flush(&self) -> PyResult<()> { + let _ = std::io::stderr().flush(); + Ok(()) + } + } + + /// Lightweight stdio wrapper for sandbox mode (no host_env). + /// Directly uses Rust's std::io for stdin/stdout/stderr without FileIO. + #[pyclass(no_attr, name = "_SandboxStdio")] + #[derive(Debug, PyPayload)] + pub struct SandboxStdio { + pub fd: i32, + pub name: String, + pub mode: String, + } + + #[pyclass] + impl SandboxStdio { + #[pymethod] + fn write(&self, s: PyStrRef, vm: &VirtualMachine) -> PyResult<usize> { + if self.fd == 0 { + return Err(vm.new_os_error("not writable".to_owned())); + } + let bytes = s.as_bytes(); + if self.fd == 2 { + std::io::stderr() + .write_all(bytes) + .map_err(|e| vm.new_os_error(e.to_string()))?; + } else { + std::io::stdout() + .write_all(bytes) + .map_err(|e| vm.new_os_error(e.to_string()))?; + } + Ok(bytes.len()) + } + + #[pymethod] + fn readline(&self, size: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult<String> { + if self.fd != 0 { + return Err(vm.new_os_error("not readable".to_owned())); + } + let size = size.unwrap_or(-1); + if size == 0 { + return Ok(String::new()); + } + let mut line = String::new(); + std::io::stdin() + .read_line(&mut line) + .map_err(|e| vm.new_os_error(e.to_string()))?; + if size > 0 { + line.truncate(size as usize); + } + Ok(line) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { + match self.fd { + 1 => { + std::io::stdout() + .flush() + .map_err(|e| vm.new_os_error(e.to_string()))?; + } + 2 => { + std::io::stderr() + .flush() + .map_err(|e| vm.new_os_error(e.to_string()))?; + } + _ => {} + } + Ok(()) + } + + #[pymethod] + fn fileno(&self) -> i32 { + self.fd + } + + #[pymethod] + fn isatty(&self) -> bool { + match self.fd { + 0 => std::io::stdin().is_terminal(), + 1 => std::io::stdout().is_terminal(), + 2 => std::io::stderr().is_terminal(), + _ => false, + } + } + + #[pymethod] + fn readable(&self) -> bool { + self.fd == 0 + } + + #[pymethod] + fn writable(&self) -> bool { + self.fd == 1 || self.fd == 2 + } + + #[pygetset] + fn closed(&self) -> bool { + false + } + + #[pygetset] + fn encoding(&self) -> String { + "utf-8".to_owned() + } + + #[pygetset] + fn errors(&self) -> String { + if self.fd == 2 { + "backslashreplace" + } else { + "strict" + } + .to_owned() + } + + #[pygetset(name = "name")] + fn name_prop(&self) -> String { + self.name.clone() + } + + #[pygetset(name = "mode")] + fn mode_prop(&self) -> String { + self.mode.clone() + } + } + + #[pyattr(name = "_rustpython_debugbuild")] + const RUSTPYTHON_DEBUGBUILD: bool = cfg!(debug_assertions); + + #[cfg(not(windows))] + #[pyattr(name = "abiflags")] + const ABIFLAGS_ATTR: &str = "t"; // 't' for free-threaded (no GIL) + // Internal constant used for sysconfigdata_name + pub const ABIFLAGS: &str = "t"; + #[pyattr(name = "api_version")] + const API_VERSION: u32 = 0x0; // what C api? + #[pyattr(name = "copyright")] + const COPYRIGHT: &str = "Copyright (c) 2019 RustPython Team"; + #[pyattr(name = "float_repr_style")] + const FLOAT_REPR_STYLE: &str = "short"; + #[pyattr(name = "_framework")] + const FRAMEWORK: &str = ""; + #[pyattr(name = "hexversion")] + const HEXVERSION: usize = version::VERSION_HEX; + #[pyattr(name = "maxsize")] + pub(crate) const MAXSIZE: isize = isize::MAX; + #[pyattr(name = "maxunicode")] + const MAXUNICODE: u32 = core::char::MAX as u32; + #[pyattr(name = "platform")] + pub const PLATFORM: &str = { + cfg_if::cfg_if! { + if #[cfg(target_os = "linux")] { + "linux" + } else if #[cfg(target_os = "android")] { + "android" + } else if #[cfg(target_os = "macos")] { + "darwin" + } else if #[cfg(target_os = "ios")] { + "ios" + } else if #[cfg(windows)] { + "win32" + } else if #[cfg(target_os = "wasi")] { + "wasi" + } else { + "unknown" + } + } + }; + #[pyattr(name = "ps1")] + const PS1: &str = ">>>>> "; + #[pyattr(name = "ps2")] + const PS2: &str = "..... "; + + #[cfg(windows)] + #[pyattr(name = "_vpath")] + const VPATH: Option<&'static str> = None; // TODO: actual VPATH value + + #[cfg(windows)] + #[pyattr(name = "dllhandle")] + const DLLHANDLE: usize = 0; + + #[pyattr] + fn prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.prefix.clone() + } + #[pyattr] + fn base_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_prefix.clone() + } + #[pyattr] + fn exec_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.exec_prefix.clone() + } + #[pyattr] + fn base_exec_prefix(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_exec_prefix.clone() + } + #[pyattr] + fn platlibdir(_vm: &VirtualMachine) -> &'static str { + option_env!("RUSTPYTHON_PLATLIBDIR").unwrap_or("lib") + } + #[pyattr] + fn _stdlib_dir(vm: &VirtualMachine) -> PyObjectRef { + vm.state.config.paths.stdlib_dir.clone().to_pyobject(vm) + } + + // alphabetical order with segments of pyattr and others + + #[pyattr] + fn argv(vm: &VirtualMachine) -> Vec<PyObjectRef> { + vm.state + .config + .settings + .argv + .iter() + .map(|arg| vm.ctx.new_str(arg.clone()).into()) + .collect() + } + + #[pyattr] + fn builtin_module_names(vm: &VirtualMachine) -> PyTupleRef { + let mut module_names: Vec<String> = + vm.state.module_defs.keys().map(|&s| s.to_owned()).collect(); + module_names.push("sys".to_owned()); + module_names.push("builtins".to_owned()); + + module_names.sort(); + vm.ctx.new_tuple( + module_names + .into_iter() + .map(|n| vm.ctx.new_str(n).into()) + .collect(), + ) + } + + // List from cpython/Python/stdlib_module_names.h + const STDLIB_MODULE_NAMES: &[&str] = &[ + "__future__", + "_abc", + "_aix_support", + "_android_support", + "_apple_support", + "_ast", + "_asyncio", + "_bisect", + "_blake2", + "_bz2", + "_codecs", + "_codecs_cn", + "_codecs_hk", + "_codecs_iso2022", + "_codecs_jp", + "_codecs_kr", + "_codecs_tw", + "_collections", + "_collections_abc", + "_colorize", + "_compat_pickle", + "_compression", + "_contextvars", + "_csv", + "_ctypes", + "_curses", + "_curses_panel", + "_datetime", + "_dbm", + "_decimal", + "_elementtree", + "_frozen_importlib", + "_frozen_importlib_external", + "_functools", + "_gdbm", + "_hashlib", + "_heapq", + "_imp", + "_interpchannels", + "_interpqueues", + "_interpreters", + "_io", + "_ios_support", + "_json", + "_locale", + "_lsprof", + "_lzma", + "_markupbase", + "_md5", + "_multibytecodec", + "_multiprocessing", + "_opcode", + "_opcode_metadata", + "_operator", + "_osx_support", + "_overlapped", + "_pickle", + "_posixshmem", + "_posixsubprocess", + "_py_abc", + "_pydatetime", + "_pydecimal", + "_pyio", + "_pylong", + "_pyrepl", + "_queue", + "_random", + "_scproxy", + "_sha1", + "_sha2", + "_sha3", + "_signal", + "_sitebuiltins", + "_socket", + "_sqlite3", + "_sre", + "_ssl", + "_stat", + "_statistics", + "_string", + "_strptime", + "_struct", + "_suggestions", + "_symtable", + "_sysconfig", + "_thread", + "_threading_local", + "_tkinter", + "_tokenize", + "_tracemalloc", + "_typing", + "_uuid", + "_warnings", + "_weakref", + "_weakrefset", + "_winapi", + "_wmi", + "_zoneinfo", + "abc", + "antigravity", + "argparse", + "array", + "ast", + "asyncio", + "atexit", + "base64", + "bdb", + "binascii", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "genericpath", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "idlelib", + "imaplib", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msvcrt", + "multiprocessing", + "netrc", + "nt", + "ntpath", + "nturl2path", + "numbers", + "opcode", + "operator", + "optparse", + "os", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "pydoc_data", + "pyexpat", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtplib", + "socket", + "socketserver", + "sqlite3", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "tempfile", + "termios", + "textwrap", + "this", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "tomllib", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", + ]; + + #[pyattr(once)] + fn stdlib_module_names(vm: &VirtualMachine) -> PyObjectRef { + let names = STDLIB_MODULE_NAMES + .iter() + .map(|&n| vm.ctx.new_str(n).into()); + PyFrozenSet::from_iter(vm, names) + .expect("Creating stdlib_module_names frozen set must succeed") + .to_pyobject(vm) + } + + #[pyattr] + fn byteorder(vm: &VirtualMachine) -> PyStrRef { + // https://doc.rust-lang.org/reference/conditional-compilation.html#target_endian + vm.ctx + .intern_str(if cfg!(target_endian = "little") { + "little" + } else if cfg!(target_endian = "big") { + "big" + } else { + "unknown" + }) + .to_owned() + } + + #[pyattr] + fn _base_executable(vm: &VirtualMachine) -> String { + vm.state.config.paths.base_executable.clone() + } + + #[pyattr] + fn dont_write_bytecode(vm: &VirtualMachine) -> bool { + !vm.state.config.settings.write_bytecode + } + + #[pyattr] + fn executable(vm: &VirtualMachine) -> String { + vm.state.config.paths.executable.clone() + } + + #[pyattr] + fn _git(vm: &VirtualMachine) -> PyTupleRef { + vm.new_tuple(( + ascii!("RustPython"), + version::get_git_identifier(), + version::get_git_revision(), + )) + } + + #[pyattr] + fn implementation(vm: &VirtualMachine) -> PyRef<PyNamespace> { + const NAME: &str = "rustpython"; + + let cache_tag = format!("{NAME}-{}{}", version::MAJOR, version::MINOR); + let ctx = &vm.ctx; + py_namespace!(vm, { + "name" => ctx.new_str(NAME), + "cache_tag" => ctx.new_str(cache_tag), + "_multiarch" => ctx.new_str(multiarch()), + "version" => version_info(vm), + "hexversion" => ctx.new_int(version::VERSION_HEX), + "supports_isolated_interpreters" => ctx.new_bool(false), + }) + } + + #[pyattr] + const fn meta_path(_vm: &VirtualMachine) -> Vec<PyObjectRef> { + Vec::new() + } + + #[pyattr] + fn orig_argv(vm: &VirtualMachine) -> Vec<PyObjectRef> { + env::args().map(|arg| vm.ctx.new_str(arg).into()).collect() + } + + #[pyattr] + fn path(vm: &VirtualMachine) -> Vec<PyObjectRef> { + vm.state + .config + .paths + .module_search_paths + .iter() + .map(|path| vm.ctx.new_str(path.clone()).into()) + .collect() + } + + #[pyattr] + const fn path_hooks(_vm: &VirtualMachine) -> Vec<PyObjectRef> { + Vec::new() + } + + #[pyattr] + fn path_importer_cache(vm: &VirtualMachine) -> PyDictRef { + vm.ctx.new_dict() + } + + #[pyattr] + fn pycache_prefix(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.none() + } + + #[pyattr] + fn version(_vm: &VirtualMachine) -> String { + version::get_version() + } + + #[cfg(windows)] + #[pyattr] + fn winver(_vm: &VirtualMachine) -> String { + // Note: This is Python DLL version in CPython, but we arbitrary fill it for compatibility + version::get_winver_number() + } + + #[pyattr] + fn _xoptions(vm: &VirtualMachine) -> PyDictRef { + let ctx = &vm.ctx; + let xopts = ctx.new_dict(); + for (key, value) in &vm.state.config.settings.xoptions { + let value = value.as_ref().map_or_else( + || ctx.new_bool(true).into(), + |s| ctx.new_str(s.clone()).into(), + ); + xopts.set_item(&**key, value, vm).unwrap(); + } + xopts + } + + #[pyattr] + fn warnoptions(vm: &VirtualMachine) -> Vec<PyObjectRef> { + vm.state + .config + .settings + .warnoptions + .iter() + .map(|s| vm.ctx.new_str(s.clone()).into()) + .collect() + } + + #[cfg(feature = "rustpython-compiler")] + #[pyfunction] + fn _baserepl(vm: &VirtualMachine) -> PyResult<()> { + // read stdin to end + let stdin = std::io::stdin(); + let mut handle = stdin.lock(); + let mut source = String::new(); + handle + .read_to_string(&mut source) + .map_err(|e| vm.new_os_error(format!("Error reading from stdin: {e}")))?; + vm.compile(&source, crate::compiler::Mode::Single, "<stdin>".to_owned()) + .map_err(|e| vm.new_os_error(format!("Error running stdin: {e}")))?; + Ok(()) + } + + #[pyfunction] + fn audit(_args: FuncArgs) { + // TODO: sys.audit implementation + } + + #[pyfunction] + const fn _is_gil_enabled() -> bool { + false // RustPython has no GIL (like free-threaded Python) + } + + /// Return True if remote debugging is enabled, False otherwise. + #[pyfunction] + const fn is_remote_debug_enabled() -> bool { + false // RustPython does not support remote debugging + } + + #[pyfunction] + fn exit(code: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult { + let status = code.unwrap_or_none(vm); + let args = if let Some(status_tuple) = status.downcast_ref::<PyTuple>() { + status_tuple.as_slice().to_vec() + } else { + vec![status] + }; + let exc = vm.invoke_exception(vm.ctx.exceptions.system_exit.to_owned(), args)?; + Err(exc) + } + + #[pyfunction] + fn call_tracing(func: PyObjectRef, args: PyTupleRef, vm: &VirtualMachine) -> PyResult { + // CPython temporarily enables tracing state around this call. + // RustPython does not currently model the full C-level tracing toggles, + // but call semantics (func(*args)) are matched. + func.call(PosArgs::new(args.as_slice().to_vec()), vm) + } + + #[pyfunction] + fn exception(vm: &VirtualMachine) -> Option<PyBaseExceptionRef> { + vm.topmost_exception() + } + + #[pyfunction(name = "__displayhook__")] + #[pyfunction] + fn displayhook(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + // Save non-None values as "_" + if vm.is_none(&obj) { + return Ok(()); + } + // set to none to avoid recursion while printing + vm.builtins.set_attr("_", vm.ctx.none(), vm)?; + // TODO: catch encoding errors + let repr = obj.repr(vm)?.into(); + builtins::print(PosArgs::new(vec![repr]), Default::default(), vm)?; + vm.builtins.set_attr("_", obj, vm)?; + Ok(()) + } + + #[pyfunction(name = "__excepthook__")] + #[pyfunction] + fn excepthook( + exc_type: PyObjectRef, + exc_val: PyObjectRef, + exc_tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let stderr = super::get_stderr(vm)?; + match vm.normalize_exception(exc_type.clone(), exc_val.clone(), exc_tb) { + Ok(exc) => { + // PyErr_Display: try traceback._print_exception_bltin first + if let Ok(tb_mod) = vm.import("traceback", 0) + && let Ok(print_exc_builtin) = tb_mod.get_attr("_print_exception_bltin", vm) + && print_exc_builtin + .call((exc.as_object().to_owned(),), vm) + .is_ok() + { + return Ok(()); + } + // Fallback to Rust-level exception printing + vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc) + } + Err(_) => { + let type_name = exc_val.class().name(); + let msg = format!( + "TypeError: print_exception(): Exception expected for value, {type_name} found\n" + ); + use crate::py_io::Write; + write!(&mut crate::py_io::PyWriter(stderr, vm), "{msg}")?; + Ok(()) + } + } + } + + #[pyfunction(name = "__breakpointhook__")] + #[pyfunction] + pub fn breakpointhook(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let env_var = std::env::var("PYTHONBREAKPOINT") + .and_then(|env_var| { + if env_var.is_empty() { + Err(VarError::NotPresent) + } else { + Ok(env_var) + } + }) + .unwrap_or_else(|_| "pdb.set_trace".to_owned()); + + if env_var.eq("0") { + return Ok(vm.ctx.none()); + }; + + let print_unimportable_module_warn = || { + warn( + vm.ctx.exceptions.runtime_warning, + format!("Ignoring unimportable $PYTHONBREAKPOINT: \"{env_var}\"",), + 0, + vm, + ) + .unwrap(); + Ok(vm.ctx.none()) + }; + + let last = match env_var.rsplit_once('.') { + Some((_, last)) => last, + None if !env_var.is_empty() => env_var.as_str(), + _ => return print_unimportable_module_warn(), + }; + + let (module_path, attr_name) = if last == env_var { + ("builtins", env_var.as_str()) + } else { + (&env_var[..(env_var.len() - last.len() - 1)], last) + }; + + let module = match vm.import(&vm.ctx.new_str(module_path), 0) { + Ok(module) => module, + Err(_) => { + return print_unimportable_module_warn(); + } + }; + + match vm.get_attribute_opt(module, &vm.ctx.new_str(attr_name)) { + Ok(Some(hook)) => hook.as_ref().call(args, vm), + _ => print_unimportable_module_warn(), + } + } + + #[pyfunction] + fn exc_info(vm: &VirtualMachine) -> (PyObjectRef, PyObjectRef, PyObjectRef) { + match vm.topmost_exception() { + Some(exception) => vm.split_exception(exception), + None => (vm.ctx.none(), vm.ctx.none(), vm.ctx.none()), + } + } + + #[pyattr] + fn flags(vm: &VirtualMachine) -> PyTupleRef { + PyFlags::from_data(FlagsData::from_settings(&vm.state.config.settings), vm) + } + + #[pyattr] + fn float_info(vm: &VirtualMachine) -> PyTupleRef { + PyFloatInfo::from_data(FloatInfoData::INFO, vm) + } + + #[pyfunction] + const fn getdefaultencoding() -> &'static str { + crate::codecs::DEFAULT_ENCODING + } + + #[pyfunction] + fn getrefcount(obj: PyObjectRef) -> usize { + obj.strong_count() + } + + #[pyfunction] + fn getrecursionlimit(vm: &VirtualMachine) -> usize { + vm.recursion_limit.get() + } + + #[derive(FromArgs)] + struct GetsizeofArgs { + obj: PyObjectRef, + #[pyarg(any, optional)] + default: Option<PyObjectRef>, + } + + #[pyfunction] + fn getsizeof(args: GetsizeofArgs, vm: &VirtualMachine) -> PyResult { + let sizeof = || -> PyResult<usize> { + let res = vm.call_special_method(&args.obj, identifier!(vm, __sizeof__), ())?; + let res = res.try_index(vm)?.try_to_primitive::<usize>(vm)?; + Ok(res + core::mem::size_of::<PyObject>()) + }; + sizeof() + .map(|x| vm.ctx.new_int(x).into()) + .or_else(|err| args.default.ok_or(err)) + } + + #[pyfunction] + fn getfilesystemencoding(vm: &VirtualMachine) -> PyStrRef { + vm.fs_encoding().to_owned() + } + + #[pyfunction] + fn getfilesystemencodeerrors(vm: &VirtualMachine) -> PyUtf8StrRef { + vm.fs_encode_errors().to_owned() + } + + #[pyfunction] + fn getprofile(vm: &VirtualMachine) -> PyObjectRef { + vm.profile_func.borrow().clone() + } + + #[pyfunction] + fn _getframe(offset: OptionalArg<usize>, vm: &VirtualMachine) -> PyResult<FrameRef> { + let offset = offset.into_option().unwrap_or(0); + let frames = vm.frames.borrow(); + if offset >= frames.len() { + return Err(vm.new_value_error("call stack is not deep enough")); + } + let idx = frames.len() - offset - 1; + // SAFETY: the FrameRef is alive on the call stack while it's in the Vec + let py: &crate::Py<Frame> = unsafe { frames[idx].as_ref() }; + Ok(py.to_owned()) + } + + #[pyfunction] + fn _getframemodulename(depth: OptionalArg<usize>, vm: &VirtualMachine) -> PyResult { + let depth = depth.into_option().unwrap_or(0); + + // Get the frame at the specified depth + let func_obj = { + let frames = vm.frames.borrow(); + if depth >= frames.len() { + return Ok(vm.ctx.none()); + } + let idx = frames.len() - depth - 1; + // SAFETY: the FrameRef is alive on the call stack while it's in the Vec + let frame: &crate::Py<Frame> = unsafe { frames[idx].as_ref() }; + frame.func_obj.clone() + }; + + // If the frame has a function object, return its __module__ attribute + if let Some(func_obj) = func_obj { + match func_obj.get_attr(identifier!(vm, __module__), vm) { + Ok(module) => Ok(module), + Err(_) => { + // CPython clears the error and returns None + Ok(vm.ctx.none()) + } + } + } else { + Ok(vm.ctx.none()) + } + } + + /// Return a dictionary mapping each thread's identifier to the topmost stack frame + /// currently active in that thread at the time the function is called. + #[cfg(feature = "threading")] + #[pyfunction] + fn _current_frames(vm: &VirtualMachine) -> PyResult<PyDictRef> { + use crate::AsObject; + use crate::stdlib::_thread::get_all_current_frames; + + let frames = get_all_current_frames(vm); + let dict = vm.ctx.new_dict(); + + for (thread_id, frame) in frames { + let key = vm.ctx.new_int(thread_id); + dict.set_item(key.as_object(), frame.into(), vm)?; + } + + Ok(dict) + } + + /// Return a dictionary mapping each thread's identifier to its currently + /// active exception, or None if no exception is active. + #[cfg(feature = "threading")] + #[pyfunction] + fn _current_exceptions(vm: &VirtualMachine) -> PyResult<PyDictRef> { + use crate::AsObject; + use crate::vm::thread::get_all_current_exceptions; + + let dict = vm.ctx.new_dict(); + for (thread_id, exc) in get_all_current_exceptions(vm) { + let key = vm.ctx.new_int(thread_id); + let value = exc.map_or_else(|| vm.ctx.none(), |e| e.into()); + dict.set_item(key.as_object(), value, vm)?; + } + + Ok(dict) + } + + #[cfg(not(feature = "threading"))] + #[pyfunction] + fn _current_exceptions(vm: &VirtualMachine) -> PyResult<PyDictRef> { + let dict = vm.ctx.new_dict(); + let key = vm.ctx.new_int(0); + dict.set_item(key.as_object(), vm.topmost_exception().to_pyobject(vm), vm)?; + Ok(dict) + } + + /// Stub for non-threading builds - returns empty dict + #[cfg(not(feature = "threading"))] + #[pyfunction] + fn _current_frames(vm: &VirtualMachine) -> PyResult<PyDictRef> { + Ok(vm.ctx.new_dict()) + } + + #[pyfunction] + fn gettrace(vm: &VirtualMachine) -> PyObjectRef { + vm.trace_func.borrow().clone() + } + + #[cfg(windows)] + fn get_kernel32_version() -> std::io::Result<(u32, u32, u32)> { + use crate::common::windows::ToWideString; + unsafe { + // Create a wide string for "kernel32.dll" + let module_name: Vec<u16> = std::ffi::OsStr::new("kernel32.dll").to_wide_with_nul(); + let h_kernel32 = GetModuleHandleW(module_name.as_ptr()); + if h_kernel32.is_null() { + return Err(std::io::Error::last_os_error()); + } + + // Prepare a buffer for the module file path + let mut kernel32_path = [0u16; MAX_PATH as usize]; + let len = GetModuleFileNameW( + h_kernel32, + kernel32_path.as_mut_ptr(), + kernel32_path.len() as u32, + ); + if len == 0 { + return Err(std::io::Error::last_os_error()); + } + + // Get the size of the version information block + let ver_block_size = + GetFileVersionInfoSizeW(kernel32_path.as_ptr(), core::ptr::null_mut()); + if ver_block_size == 0 { + return Err(std::io::Error::last_os_error()); + } + + // Allocate a buffer to hold the version information + let mut ver_block = vec![0u8; ver_block_size as usize]; + if GetFileVersionInfoW( + kernel32_path.as_ptr(), + 0, + ver_block_size, + ver_block.as_mut_ptr() as *mut _, + ) == 0 + { + return Err(std::io::Error::last_os_error()); + } + + // Prepare an empty sub-block string (L"") as required by VerQueryValueW + let sub_block: Vec<u16> = std::ffi::OsStr::new("").to_wide_with_nul(); + + let mut ffi_ptr: *mut VS_FIXEDFILEINFO = core::ptr::null_mut(); + let mut ffi_len: u32 = 0; + if VerQueryValueW( + ver_block.as_ptr() as *const _, + sub_block.as_ptr(), + &mut ffi_ptr as *mut *mut VS_FIXEDFILEINFO as *mut *mut _, + &mut ffi_len as *mut u32, + ) == 0 + || ffi_ptr.is_null() + { + return Err(std::io::Error::last_os_error()); + } + + // Extract the version numbers from the VS_FIXEDFILEINFO structure. + let ffi = *ffi_ptr; + let real_major = (ffi.dwProductVersionMS >> 16) & 0xFFFF; + let real_minor = ffi.dwProductVersionMS & 0xFFFF; + let real_build = (ffi.dwProductVersionLS >> 16) & 0xFFFF; + + Ok((real_major, real_minor, real_build)) + } + } + + #[cfg(windows)] + #[pyfunction] + fn getwindowsversion(vm: &VirtualMachine) -> PyResult<crate::builtins::tuple::PyTupleRef> { + use std::ffi::OsString; + use std::os::windows::ffi::OsStringExt; + use windows_sys::Win32::System::SystemInformation::{ + GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW, + }; + + let mut version: OSVERSIONINFOEXW = unsafe { core::mem::zeroed() }; + version.dwOSVersionInfoSize = core::mem::size_of::<OSVERSIONINFOEXW>() as u32; + let result = unsafe { + let os_vi = &mut version as *mut OSVERSIONINFOEXW as *mut OSVERSIONINFOW; + // SAFETY: GetVersionExW accepts a pointer of OSVERSIONINFOW, but windows-sys crate's type currently doesn't allow to do so. + // https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getversionexw#parameters + GetVersionExW(os_vi) + }; + + if result == 0 { + return Err(vm.new_os_error("failed to get windows version".to_owned())); + } + + let service_pack = { + let (last, _) = version + .szCSDVersion + .iter() + .take_while(|&x| x != &0) + .enumerate() + .last() + .unwrap_or((0, &0)); + let sp = OsString::from_wide(&version.szCSDVersion[..last]); + sp.into_string() + .map_err(|_| vm.new_os_error("service pack is not ASCII".to_owned()))? + }; + let real_version = get_kernel32_version().map_err(|e| vm.new_os_error(e.to_string()))?; + let winver = WindowsVersionData { + major: real_version.0, + minor: real_version.1, + build: real_version.2, + platform: version.dwPlatformId, + service_pack, + service_pack_major: version.wServicePackMajor, + service_pack_minor: version.wServicePackMinor, + suite_mask: version.wSuiteMask, + product_type: version.wProductType, + platform_version: (real_version.0, real_version.1, real_version.2), // TODO Provide accurate version, like CPython impl + }; + Ok(PyWindowsVersion::from_data(winver, vm)) + } + + fn _unraisablehook(unraisable: UnraisableHookArgsData, vm: &VirtualMachine) -> PyResult<()> { + use super::PyStderr; + + let stderr = PyStderr(vm); + if !vm.is_none(&unraisable.object) { + if !vm.is_none(&unraisable.err_msg) { + write!(stderr, "{}: ", unraisable.err_msg.str(vm)?); + } else { + write!(stderr, "Exception ignored in: "); + } + // exception in del will be ignored but printed + let repr = &unraisable.object.repr(vm); + let str = match repr { + Ok(v) => v.to_string(), + Err(_) => format!( + "<object {} repr() failed>", + unraisable.object.class().name() + ), + }; + writeln!(stderr, "{str}"); + } else if !vm.is_none(&unraisable.err_msg) { + writeln!(stderr, "{}:", unraisable.err_msg.str(vm)?); + } + + // Print traceback (using actual exc_traceback, not current stack) + if !vm.is_none(&unraisable.exc_traceback) { + let tb_module = vm.import("traceback", 0)?; + let print_tb = tb_module.get_attr("print_tb", vm)?; + let stderr_obj = super::get_stderr(vm)?; + let kwargs: KwArgs = [("file".to_string(), stderr_obj)].into_iter().collect(); + let _ = print_tb.call( + FuncArgs::new(vec![unraisable.exc_traceback.clone()], kwargs), + vm, + ); + } + + // Check exc_type + if vm.is_none(unraisable.exc_type.as_object()) { + return Ok(()); + } + assert!( + unraisable + .exc_type + .fast_issubclass(vm.ctx.exceptions.base_exception_type) + ); + + // Print module name (if not builtins or __main__) + let module_name = unraisable.exc_type.__module__(vm); + if let Ok(module_str) = module_name.downcast::<PyStr>() { + let module = module_str.as_wtf8(); + if module != "builtins" && module != "__main__" { + write!(stderr, "{}.", module); + } + } else { + write!(stderr, "<unknown>."); + } + + // Print qualname + let qualname = unraisable.exc_type.__qualname__(vm); + if let Ok(qualname_str) = qualname.downcast::<PyStr>() { + write!(stderr, "{}", qualname_str.as_wtf8()); + } else { + write!(stderr, "{}", unraisable.exc_type.name()); + } + + // Print exception value + if !vm.is_none(&unraisable.exc_value) { + write!(stderr, ": "); + if let Ok(str) = unraisable.exc_value.str(vm) { + write!(stderr, "{}", str.as_wtf8()); + } else { + write!(stderr, "<exception str() failed>"); + } + } + writeln!(stderr); + + // Flush stderr + if let Ok(stderr_obj) = super::get_stderr(vm) + && let Ok(flush) = stderr_obj.get_attr("flush", vm) + { + let _ = flush.call((), vm); + } + + Ok(()) + } + + #[pyattr] + #[pyfunction(name = "__unraisablehook__")] + fn unraisablehook(unraisable: UnraisableHookArgsData, vm: &VirtualMachine) { + if let Err(e) = _unraisablehook(unraisable, vm) { + let stderr = super::PyStderr(vm); + writeln!( + stderr, + "{}", + e.as_object() + .repr(vm) + .unwrap_or_else(|_| vm.ctx.empty_str.to_owned()) + ); + } + } + + #[pyattr] + fn hash_info(vm: &VirtualMachine) -> PyTupleRef { + PyHashInfo::from_data(HashInfoData::INFO, vm) + } + + #[pyfunction] + fn intern(s: PyRefExact<PyStr>, vm: &VirtualMachine) -> PyRef<PyStr> { + vm.ctx.intern_str(s).to_owned() + } + + #[pyattr] + fn int_info(vm: &VirtualMachine) -> PyTupleRef { + PyIntInfo::from_data(IntInfoData::INFO, vm) + } + + #[pyfunction] + fn get_int_max_str_digits(vm: &VirtualMachine) -> usize { + vm.state.int_max_str_digits.load() + } + + #[pyfunction] + fn set_int_max_str_digits(maxdigits: usize, vm: &VirtualMachine) -> PyResult<()> { + let threshold = IntInfoData::INFO.str_digits_check_threshold; + if maxdigits == 0 || maxdigits >= threshold { + vm.state.int_max_str_digits.store(maxdigits); + Ok(()) + } else { + let error = format!("maxdigits must be 0 or larger than {threshold:?}"); + Err(vm.new_value_error(error)) + } + } + + #[pyfunction] + fn is_finalizing(vm: &VirtualMachine) -> bool { + vm.state.finalizing.load(Ordering::Acquire) + } + + #[pyfunction] + fn setprofile(profilefunc: PyObjectRef, vm: &VirtualMachine) { + vm.profile_func.replace(profilefunc); + update_use_tracing(vm); + } + + #[pyfunction] + fn setrecursionlimit(recursion_limit: i32, vm: &VirtualMachine) -> PyResult<()> { + let recursion_limit = recursion_limit + .to_usize() + .filter(|&u| u >= 1) + .ok_or_else(|| { + vm.new_value_error("recursion limit must be greater than or equal to one") + })?; + let recursion_depth = vm.current_recursion_depth(); + + if recursion_limit > recursion_depth { + vm.recursion_limit.set(recursion_limit); + Ok(()) + } else { + Err(vm.new_recursion_error(format!( + "cannot set the recursion limit to {recursion_limit} at the recursion depth {recursion_depth}: the limit is too low" + ))) + } + } + + #[pyfunction] + fn settrace(tracefunc: PyObjectRef, vm: &VirtualMachine) { + vm.trace_func.replace(tracefunc); + update_use_tracing(vm); + } + + #[pyfunction] + fn _settraceallthreads(tracefunc: PyObjectRef, vm: &VirtualMachine) { + let func = (!vm.is_none(&tracefunc)).then(|| tracefunc.clone()); + *vm.state.global_trace_func.lock() = func; + vm.trace_func.replace(tracefunc); + update_use_tracing(vm); + } + + #[pyfunction] + fn _setprofileallthreads(profilefunc: PyObjectRef, vm: &VirtualMachine) { + let func = (!vm.is_none(&profilefunc)).then(|| profilefunc.clone()); + *vm.state.global_profile_func.lock() = func; + vm.profile_func.replace(profilefunc); + update_use_tracing(vm); + } + + #[cfg(feature = "threading")] + #[pyattr] + fn thread_info(vm: &VirtualMachine) -> PyTupleRef { + PyThreadInfo::from_data(ThreadInfoData::INFO, vm) + } + + #[pyattr] + fn version_info(vm: &VirtualMachine) -> PyTupleRef { + PyVersionInfo::from_data(VersionInfoData::VERSION, vm) + } + + fn update_use_tracing(vm: &VirtualMachine) { + let trace_is_none = vm.is_none(&vm.trace_func.borrow()); + let profile_is_none = vm.is_none(&vm.profile_func.borrow()); + let tracing = !(trace_is_none && profile_is_none); + vm.use_tracing.set(tracing); + } + + #[pyfunction] + fn set_coroutine_origin_tracking_depth(depth: i32, vm: &VirtualMachine) -> PyResult<()> { + if depth < 0 { + return Err(vm.new_value_error("depth must be >= 0")); + } + crate::vm::thread::COROUTINE_ORIGIN_TRACKING_DEPTH.set(depth as u32); + Ok(()) + } + + #[pyfunction] + fn get_coroutine_origin_tracking_depth() -> i32 { + crate::vm::thread::COROUTINE_ORIGIN_TRACKING_DEPTH.get() as i32 + } + + #[pyfunction] + fn _clear_type_descriptors(type_obj: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { + use crate::types::PyTypeFlags; + + // Check if type is immutable + if type_obj.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { + return Err(vm.new_type_error("argument is immutable")); + } + + let mut attributes = type_obj.attributes.write(); + + // Remove __dict__ descriptor if present + attributes.swap_remove(identifier!(vm, __dict__)); + + // Remove __weakref__ descriptor if present + attributes.swap_remove(identifier!(vm, __weakref__)); + + drop(attributes); + + // Update slots to notify subclasses and recalculate cached values + type_obj.update_slot::<true>(identifier!(vm, __dict__), &vm.ctx); + type_obj.update_slot::<true>(identifier!(vm, __weakref__), &vm.ctx); + + Ok(()) + } + + #[pyfunction] + fn getswitchinterval(vm: &VirtualMachine) -> f64 { + // Return the stored switch interval + vm.state.switch_interval.load() + } + + // TODO: vm.state.switch_interval is currently not used anywhere in the VM + #[pyfunction] + fn setswitchinterval(interval: f64, vm: &VirtualMachine) -> PyResult<()> { + // Validate the interval parameter like CPython does + if interval <= 0.0 { + return Err(vm.new_value_error("switch interval must be strictly positive")); + } + + // Store the switch interval value + vm.state.switch_interval.store(interval); + Ok(()) + } + + #[derive(FromArgs)] + struct SetAsyncgenHooksArgs { + #[pyarg(any, optional)] + firstiter: OptionalArg<Option<PyObjectRef>>, + #[pyarg(any, optional)] + finalizer: OptionalArg<Option<PyObjectRef>>, + } + + #[pyfunction] + fn set_asyncgen_hooks(args: SetAsyncgenHooksArgs, vm: &VirtualMachine) -> PyResult<()> { + if let Some(Some(finalizer)) = args.finalizer.as_option() + && !finalizer.is_callable() + { + return Err(vm.new_type_error(format!( + "callable finalizer expected, got {:.50}", + finalizer.class().name() + ))); + } + + if let Some(Some(firstiter)) = args.firstiter.as_option() + && !firstiter.is_callable() + { + return Err(vm.new_type_error(format!( + "callable firstiter expected, got {:.50}", + firstiter.class().name() + ))); + } + + if let Some(finalizer) = args.finalizer.into_option() { + *vm.async_gen_finalizer.borrow_mut() = finalizer; + } + if let Some(firstiter) = args.firstiter.into_option() { + *vm.async_gen_firstiter.borrow_mut() = firstiter; + } + + Ok(()) + } + + #[pystruct_sequence_data] + pub(super) struct AsyncgenHooksData { + firstiter: PyObjectRef, + finalizer: PyObjectRef, + } + + #[pyattr] + #[pystruct_sequence(name = "asyncgen_hooks", data = "AsyncgenHooksData")] + pub(super) struct PyAsyncgenHooks; + + #[pyclass(with(PyStructSequence))] + impl PyAsyncgenHooks {} + + #[pyfunction] + fn get_asyncgen_hooks(vm: &VirtualMachine) -> AsyncgenHooksData { + AsyncgenHooksData { + firstiter: vm.async_gen_firstiter.borrow().clone().to_pyobject(vm), + finalizer: vm.async_gen_finalizer.borrow().clone().to_pyobject(vm), + } + } + + /// sys.flags + /// + /// Flags provided through command line arguments or environment vars. + #[derive(Debug)] + #[pystruct_sequence_data] + pub(super) struct FlagsData { + /// -d + debug: u8, + /// -i + inspect: u8, + /// -i + interactive: u8, + /// -O or -OO + optimize: u8, + /// -B + dont_write_bytecode: u8, + /// -s + no_user_site: u8, + /// -S + no_site: u8, + /// -E + ignore_environment: u8, + /// -v + verbose: u8, + /// -b + bytes_warning: u64, + /// -q + quiet: u8, + /// -R + hash_randomization: u8, + /// -I + isolated: u8, + /// -X dev + dev_mode: bool, + /// -X utf8 + utf8_mode: u8, + /// -X int_max_str_digits=number + int_max_str_digits: i64, + /// -P, `PYTHONSAFEPATH` + safe_path: bool, + /// -X warn_default_encoding, PYTHONWARNDEFAULTENCODING + warn_default_encoding: u8, + } + + impl FlagsData { + const fn from_settings(settings: &Settings) -> Self { + Self { + debug: settings.debug, + inspect: settings.inspect as u8, + interactive: settings.interactive as u8, + optimize: settings.optimize, + dont_write_bytecode: (!settings.write_bytecode) as u8, + no_user_site: (!settings.user_site_directory) as u8, + no_site: (!settings.import_site) as u8, + ignore_environment: settings.ignore_environment as u8, + verbose: settings.verbose, + bytes_warning: settings.bytes_warning, + quiet: settings.quiet as u8, + hash_randomization: settings.hash_seed.is_none() as u8, + isolated: settings.isolated as u8, + dev_mode: settings.dev_mode, + utf8_mode: if settings.utf8_mode < 0 { + 1 + } else { + settings.utf8_mode as u8 + }, + int_max_str_digits: settings.int_max_str_digits, + safe_path: settings.safe_path, + warn_default_encoding: settings.warn_default_encoding as u8, + } + } + } + + #[pystruct_sequence(name = "flags", data = "FlagsData", no_attr)] + pub(super) struct PyFlags; + + #[pyclass(with(PyStructSequence))] + impl PyFlags { + #[pyslot] + fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create 'sys.flags' instances")) + } + + #[pygetset] + fn context_aware_warnings(&self, vm: &VirtualMachine) -> bool { + vm.state.config.settings.context_aware_warnings + } + + #[pygetset] + fn thread_inherit_context(&self, vm: &VirtualMachine) -> bool { + vm.state.config.settings.thread_inherit_context + } + } + + #[cfg(feature = "threading")] + #[pystruct_sequence_data] + pub(super) struct ThreadInfoData { + name: Option<&'static str>, + lock: Option<&'static str>, + version: Option<&'static str>, + } + + #[cfg(feature = "threading")] + impl ThreadInfoData { + const INFO: Self = Self { + name: crate::stdlib::_thread::_thread::PYTHREAD_NAME, + // As I know, there's only way to use lock as "Mutex" in Rust + // with satisfying python document spec. + lock: Some("mutex+cond"), + version: None, + }; + } + + #[cfg(feature = "threading")] + #[pystruct_sequence(name = "thread_info", data = "ThreadInfoData", no_attr)] + pub(super) struct PyThreadInfo; + + #[cfg(feature = "threading")] + #[pyclass(with(PyStructSequence))] + impl PyThreadInfo {} + + #[pystruct_sequence_data] + pub(super) struct FloatInfoData { + max: f64, + max_exp: i32, + max_10_exp: i32, + min: f64, + min_exp: i32, + min_10_exp: i32, + dig: u32, + mant_dig: u32, + epsilon: f64, + radix: u32, + rounds: i32, + } + + impl FloatInfoData { + const INFO: Self = Self { + max: f64::MAX, + max_exp: f64::MAX_EXP, + max_10_exp: f64::MAX_10_EXP, + min: f64::MIN_POSITIVE, + min_exp: f64::MIN_EXP, + min_10_exp: f64::MIN_10_EXP, + dig: f64::DIGITS, + mant_dig: f64::MANTISSA_DIGITS, + epsilon: f64::EPSILON, + radix: f64::RADIX, + rounds: 1, // FE_TONEAREST + }; + } + + #[pystruct_sequence(name = "float_info", data = "FloatInfoData", no_attr)] + pub(super) struct PyFloatInfo; + + #[pyclass(with(PyStructSequence))] + impl PyFloatInfo {} + + #[pystruct_sequence_data] + pub(super) struct HashInfoData { + width: usize, + modulus: PyUHash, + inf: PyHash, + nan: PyHash, + imag: PyHash, + algorithm: &'static str, + hash_bits: usize, + seed_bits: usize, + cutoff: usize, + } + + impl HashInfoData { + const INFO: Self = { + use rustpython_common::hash::*; + Self { + width: core::mem::size_of::<PyHash>() * 8, + modulus: MODULUS, + inf: INF, + nan: NAN, + imag: IMAG, + algorithm: ALGO, + hash_bits: HASH_BITS, + seed_bits: SEED_BITS, + cutoff: 0, // no small string optimizations + } + }; + } + + #[pystruct_sequence(name = "hash_info", data = "HashInfoData", no_attr)] + pub(super) struct PyHashInfo; + + #[pyclass(with(PyStructSequence))] + impl PyHashInfo {} + + #[pystruct_sequence_data] + pub(super) struct IntInfoData { + bits_per_digit: usize, + sizeof_digit: usize, + default_max_str_digits: usize, + str_digits_check_threshold: usize, + } + + impl IntInfoData { + const INFO: Self = Self { + bits_per_digit: 30, //? + sizeof_digit: core::mem::size_of::<u32>(), + default_max_str_digits: 4300, + str_digits_check_threshold: 640, + }; + } + + #[pystruct_sequence(name = "int_info", data = "IntInfoData", no_attr)] + pub(super) struct PyIntInfo; + + #[pyclass(with(PyStructSequence))] + impl PyIntInfo {} + + #[derive(Default, Debug)] + #[pystruct_sequence_data] + pub struct VersionInfoData { + major: usize, + minor: usize, + micro: usize, + releaselevel: &'static str, + serial: usize, + } + + impl VersionInfoData { + pub const VERSION: Self = Self { + major: version::MAJOR, + minor: version::MINOR, + micro: version::MICRO, + releaselevel: version::RELEASELEVEL, + serial: version::SERIAL, + }; + } + + #[pystruct_sequence(name = "version_info", data = "VersionInfoData", no_attr)] + pub struct PyVersionInfo; + + #[pyclass(with(PyStructSequence))] + impl PyVersionInfo { + #[pyslot] + fn slot_new( + _cls: crate::builtins::type_::PyTypeRef, + _args: crate::function::FuncArgs, + vm: &crate::VirtualMachine, + ) -> crate::PyResult { + Err(vm.new_type_error("cannot create 'sys.version_info' instances")) + } + } + + #[cfg(windows)] + #[derive(Default, Debug)] + #[pystruct_sequence_data] + pub(super) struct WindowsVersionData { + major: u32, + minor: u32, + build: u32, + platform: u32, + service_pack: String, + #[pystruct_sequence(skip)] + service_pack_major: u16, + #[pystruct_sequence(skip)] + service_pack_minor: u16, + #[pystruct_sequence(skip)] + suite_mask: u16, + #[pystruct_sequence(skip)] + product_type: u8, + #[pystruct_sequence(skip)] + platform_version: (u32, u32, u32), + } + + #[cfg(windows)] + #[pystruct_sequence(name = "getwindowsversion", data = "WindowsVersionData", no_attr)] + pub(super) struct PyWindowsVersion; + + #[cfg(windows)] + #[pyclass(with(PyStructSequence))] + impl PyWindowsVersion {} + + #[derive(Debug)] + #[pystruct_sequence_data(try_from_object)] + pub struct UnraisableHookArgsData { + pub exc_type: PyTypeRef, + pub exc_value: PyObjectRef, + pub exc_traceback: PyObjectRef, + pub err_msg: PyObjectRef, + pub object: PyObjectRef, + } + + #[pystruct_sequence(name = "UnraisableHookArgs", data = "UnraisableHookArgsData", no_attr)] + pub struct PyUnraisableHookArgs; + + #[pyclass(with(PyStructSequence))] + impl PyUnraisableHookArgs {} +} + +pub(crate) fn init_module(vm: &VirtualMachine, module: &Py<PyModule>, builtins: &Py<PyModule>) { + module.__init_methods(vm).unwrap(); + sys::module_exec(vm, module).unwrap(); + + let modules = vm.ctx.new_dict(); + modules + .set_item("sys", module.to_owned().into(), vm) + .unwrap(); + modules + .set_item("builtins", builtins.to_owned().into(), vm) + .unwrap(); + + // Create sys._jit submodule + let jit_def = sys_jit::module_def(&vm.ctx); + let jit_module = jit_def.create_module(vm).unwrap(); + + extend_module!(vm, module, { + "__doc__" => sys::DOC.to_owned().to_pyobject(vm), + "modules" => modules, + "_jit" => jit_module, + }); +} + +pub(crate) fn set_bootstrap_stderr(vm: &VirtualMachine) -> PyResult<()> { + let stderr = sys::BootstrapStderr.into_ref(&vm.ctx); + let stderr_obj: crate::PyObjectRef = stderr.into(); + vm.sys_module.set_attr("stderr", stderr_obj.clone(), vm)?; + vm.sys_module.set_attr("__stderr__", stderr_obj, vm)?; + Ok(()) +} + +/// Similar to PySys_WriteStderr in CPython. +/// +/// # Usage +/// +/// ```rust,ignore +/// writeln!(sys::PyStderr(vm), "foo bar baz :)"); +/// ``` +/// +/// Unlike writing to a `std::io::Write` with the `write[ln]!()` macro, there's no error condition here; +/// this is intended to be a replacement for the `eprint[ln]!()` macro, so `write!()`-ing to PyStderr just +/// returns `()`. +pub struct PyStderr<'vm>(pub &'vm VirtualMachine); + +impl PyStderr<'_> { + pub fn write_fmt(&self, args: core::fmt::Arguments<'_>) { + use crate::py_io::Write; + + let vm = self.0; + if let Ok(stderr) = get_stderr(vm) { + let mut stderr = crate::py_io::PyWriter(stderr, vm); + if let Ok(()) = stderr.write_fmt(args) { + return; + } + } + eprint!("{args}") + } +} + +pub fn get_stdin(vm: &VirtualMachine) -> PyResult { + vm.sys_module + .get_attr("stdin", vm) + .map_err(|_| vm.new_runtime_error("lost sys.stdin")) +} +pub fn get_stdout(vm: &VirtualMachine) -> PyResult { + vm.sys_module + .get_attr("stdout", vm) + .map_err(|_| vm.new_runtime_error("lost sys.stdout")) +} +pub fn get_stderr(vm: &VirtualMachine) -> PyResult { + vm.sys_module + .get_attr("stderr", vm) + .map_err(|_| vm.new_runtime_error("lost sys.stderr")) +} + +pub(crate) fn sysconfigdata_name() -> String { + format!( + "_sysconfigdata_{}_{}_{}", + sys::ABIFLAGS, + sys::PLATFORM, + sys::multiarch() + ) +} diff --git a/crates/vm/src/stdlib/sys/monitoring.rs b/crates/vm/src/stdlib/sys/monitoring.rs new file mode 100644 index 00000000000..739165073af --- /dev/null +++ b/crates/vm/src/stdlib/sys/monitoring.rs @@ -0,0 +1,1183 @@ +use crate::{ + AsObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyCode, PyDictRef, PyNamespace, PyUtf8StrRef, code::CoMonitoringData}, + function::FuncArgs, +}; +use core::sync::atomic::Ordering; +use crossbeam_utils::atomic::AtomicCell; +use std::collections::{HashMap, HashSet}; + +pub const TOOL_LIMIT: usize = 6; +const EVENTS_COUNT: usize = 19; +const LOCAL_EVENTS_COUNT: usize = 11; +const UNGROUPED_EVENTS_COUNT: usize = 18; + +// Event bit positions +bitflags::bitflags! { + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub struct MonitoringEvents: u32 { + const PY_START = 1 << 0; + const PY_RESUME = 1 << 1; + const PY_RETURN = 1 << 2; + const PY_YIELD = 1 << 3; + const CALL = 1 << 4; + const LINE = 1 << 5; + const INSTRUCTION = 1 << 6; + const JUMP = 1 << 7; + const BRANCH_LEFT = 1 << 8; + const BRANCH_RIGHT = 1 << 9; + const STOP_ITERATION = 1 << 10; + const RAISE = 1 << 11; + const EXCEPTION_HANDLED = 1 << 12; + const PY_UNWIND = 1 << 13; + const PY_THROW = 1 << 14; + const RERAISE = 1 << 15; + const C_RETURN = 1 << 16; + const C_RAISE = 1 << 17; + const BRANCH = 1 << 18; + } +} + +// Re-export as plain u32 constants for use in frame.rs hot-path checks +pub const EVENT_PY_START: u32 = MonitoringEvents::PY_START.bits(); +pub const EVENT_PY_RESUME: u32 = MonitoringEvents::PY_RESUME.bits(); +pub const EVENT_PY_RETURN: u32 = MonitoringEvents::PY_RETURN.bits(); +pub const EVENT_PY_YIELD: u32 = MonitoringEvents::PY_YIELD.bits(); +pub const EVENT_CALL: u32 = MonitoringEvents::CALL.bits(); +pub const EVENT_LINE: u32 = MonitoringEvents::LINE.bits(); +pub const EVENT_INSTRUCTION: u32 = MonitoringEvents::INSTRUCTION.bits(); +pub const EVENT_JUMP: u32 = MonitoringEvents::JUMP.bits(); +pub const EVENT_BRANCH_LEFT: u32 = MonitoringEvents::BRANCH_LEFT.bits(); +pub const EVENT_BRANCH_RIGHT: u32 = MonitoringEvents::BRANCH_RIGHT.bits(); +pub const EVENT_RAISE: u32 = MonitoringEvents::RAISE.bits(); +pub const EVENT_EXCEPTION_HANDLED: u32 = MonitoringEvents::EXCEPTION_HANDLED.bits(); +pub const EVENT_PY_UNWIND: u32 = MonitoringEvents::PY_UNWIND.bits(); +pub const EVENT_C_RETURN: u32 = MonitoringEvents::C_RETURN.bits(); +const EVENT_C_RAISE: u32 = MonitoringEvents::C_RAISE.bits(); +pub const EVENT_STOP_ITERATION: u32 = MonitoringEvents::STOP_ITERATION.bits(); +pub const EVENT_PY_THROW: u32 = MonitoringEvents::PY_THROW.bits(); +const EVENT_BRANCH: u32 = MonitoringEvents::BRANCH.bits(); +pub const EVENT_RERAISE: u32 = MonitoringEvents::RERAISE.bits(); +const EVENT_C_RETURN_MASK: u32 = EVENT_C_RETURN | EVENT_C_RAISE; + +const EVENT_NAMES: [&str; EVENTS_COUNT] = [ + "PY_START", + "PY_RESUME", + "PY_RETURN", + "PY_YIELD", + "CALL", + "LINE", + "INSTRUCTION", + "JUMP", + "BRANCH_LEFT", + "BRANCH_RIGHT", + "STOP_ITERATION", + "RAISE", + "EXCEPTION_HANDLED", + "PY_UNWIND", + "PY_THROW", + "RERAISE", + "C_RETURN", + "C_RAISE", + "BRANCH", +]; + +/// Interpreter-level monitoring state, shared by all threads. +pub struct MonitoringState { + pub tool_names: [Option<String>; TOOL_LIMIT], + pub global_events: [u32; TOOL_LIMIT], + pub local_events: HashMap<(usize, usize), u32>, + pub callbacks: HashMap<(usize, usize), PyObjectRef>, + /// Per-instruction disabled tools: (code_id, offset, tool) + pub disabled: HashSet<(usize, usize, usize)>, + /// Cached MISSING sentinel singleton + pub missing: Option<PyObjectRef>, + /// Cached DISABLE sentinel singleton + pub disable: Option<PyObjectRef>, +} + +impl Default for MonitoringState { + fn default() -> Self { + Self { + tool_names: Default::default(), + global_events: [0; TOOL_LIMIT], + local_events: HashMap::new(), + callbacks: HashMap::new(), + disabled: HashSet::new(), + missing: None, + disable: None, + } + } +} + +impl MonitoringState { + /// Compute the OR of all tools' global_events + local_events. + /// This is used for the fast-path atomic mask to skip monitoring + /// when no events are registered at all. + pub fn combined_events(&self) -> u32 { + let global = self.global_events.iter().fold(0, |acc, &e| acc | e); + let local = self.local_events.values().fold(0, |acc, &e| acc | e); + global | local + } + + /// Compute the events that apply to a specific code object: + /// global events OR'd with local events registered for that code. + /// This prevents events like INSTRUCTION that are local to one code + /// from being applied to unrelated code objects. + pub fn events_for_code(&self, code_id: usize) -> u32 { + let global = self.global_events.iter().fold(0, |acc, &e| acc | e); + let local = self + .local_events + .iter() + .filter(|((_, cid), _)| *cid == code_id) + .fold(0, |acc, (_, &e)| acc | e); + global | local + } +} + +/// Global atomic mask: OR of all tools' events. Checked in the hot path +/// to skip monitoring overhead when no events are registered. +/// Lives in PyGlobalState alongside the PyMutex<MonitoringState>. +pub type MonitoringEventsMask = AtomicCell<u32>; + +/// Get the MISSING sentinel, creating it if necessary. +pub fn get_missing(vm: &VirtualMachine) -> PyObjectRef { + let mut state = vm.state.monitoring.lock(); + if let Some(ref m) = state.missing { + m.clone() + } else { + let m: PyObjectRef = sys_monitoring::MonitoringSentinel.into_ref(&vm.ctx).into(); + state.missing = Some(m.clone()); + m + } +} + +/// Get the DISABLE sentinel, creating it if necessary. +pub fn get_disable(vm: &VirtualMachine) -> PyObjectRef { + let mut state = vm.state.monitoring.lock(); + if let Some(ref d) = state.disable { + d.clone() + } else { + let d: PyObjectRef = sys_monitoring::MonitoringSentinel.into_ref(&vm.ctx).into(); + state.disable = Some(d.clone()); + d + } +} + +fn check_valid_tool(tool_id: i32, vm: &VirtualMachine) -> PyResult<usize> { + if !(0..TOOL_LIMIT as i32).contains(&tool_id) { + return Err(vm.new_value_error(format!("invalid tool {tool_id} (must be between 0 and 5)"))); + } + Ok(tool_id as usize) +} + +fn check_tool_in_use(tool: usize, vm: &VirtualMachine) -> PyResult<()> { + let state = vm.state.monitoring.lock(); + if state.tool_names[tool].is_some() { + Ok(()) + } else { + Err(vm.new_value_error(format!("tool {tool} is not in use"))) + } +} + +fn parse_single_event(event: i32, vm: &VirtualMachine) -> PyResult<usize> { + let event = u32::try_from(event) + .map_err(|_| vm.new_value_error("The callback can only be set for one event at a time"))?; + if event.count_ones() != 1 { + return Err(vm.new_value_error("The callback can only be set for one event at a time")); + } + let event_id = event.trailing_zeros() as usize; + if event_id >= EVENTS_COUNT { + return Err(vm.new_value_error(format!("invalid event {event}"))); + } + Ok(event_id) +} + +fn normalize_event_set(event_set: i32, local: bool, vm: &VirtualMachine) -> PyResult<u32> { + let kind = if local { + "local event set" + } else { + "event set" + }; + if event_set < 0 { + return Err(vm.new_value_error(format!("invalid {kind} 0x{event_set:x}"))); + } + + let mut event_set = event_set as u32; + if event_set >= (1 << EVENTS_COUNT) { + return Err(vm.new_value_error(format!("invalid {kind} 0x{event_set:x}"))); + } + + if (event_set & EVENT_C_RETURN_MASK) != 0 && (event_set & EVENT_CALL) != EVENT_CALL { + return Err(vm.new_value_error("cannot set C_RETURN or C_RAISE events independently")); + } + + event_set &= !EVENT_C_RETURN_MASK; + + if (event_set & EVENT_BRANCH) != 0 { + event_set &= !EVENT_BRANCH; + event_set |= EVENT_BRANCH_LEFT | EVENT_BRANCH_RIGHT; + } + + if local && event_set >= (1 << LOCAL_EVENTS_COUNT) { + return Err(vm.new_value_error(format!("invalid local event set 0x{event_set:x}"))); + } + + Ok(event_set) +} + +/// Rewrite a code object's bytecode in-place with layered instrumentation. +/// +/// Three layers (outermost first): +/// 1. INSTRUMENTED_LINE — wraps line-start instructions (stores original in side-table) +/// 2. INSTRUMENTED_INSTRUCTION — wraps all traceable instructions (stores original in side-table) +/// 3. Regular INSTRUMENTED_* — direct 1:1 opcode swap (no side-table needed) +/// +/// De-instrumentation peels layers in reverse order. +pub fn instrument_code(code: &PyCode, events: u32) { + use rustpython_compiler_core::bytecode::{self, Instruction}; + + let len = code.code.instructions.len(); + let mut monitoring_data = code.monitoring_data.lock(); + + // === Phase 1-3: De-instrument all layers (outermost first) === + + // Phase 1: Remove INSTRUMENTED_LINE → restore from side-table + if let Some(data) = monitoring_data.as_mut() { + for i in 0..len { + if data.line_opcodes[i] != 0 { + let original = Instruction::try_from(data.line_opcodes[i]) + .expect("invalid opcode in line side-table"); + unsafe { + code.code.instructions.replace_op(i, original); + } + data.line_opcodes[i] = 0; + } + } + } + + // Phase 2: Remove INSTRUMENTED_INSTRUCTION → restore from side-table + if let Some(data) = monitoring_data.as_mut() { + for i in 0..len { + if data.per_instruction_opcodes[i] != 0 { + let original = Instruction::try_from(data.per_instruction_opcodes[i]) + .expect("invalid opcode in instruction side-table"); + unsafe { + code.code.instructions.replace_op(i, original); + } + data.per_instruction_opcodes[i] = 0; + } + } + } + + // Phase 3: Remove regular INSTRUMENTED_* and specialized opcodes → restore base opcodes. + // Also clear all CACHE entries so specialization starts fresh. + { + let mut i = 0; + while i < len { + let op = code.code.instructions[i].op; + let base_op = op.deoptimize(); + if u8::from(base_op) != u8::from(op) { + unsafe { + code.code.instructions.replace_op(i, base_op); + } + } + let caches = base_op.cache_entries(); + // Zero all CACHE entries (the op+arg bytes may have been overwritten + // by specialization with arbitrary data like pointers). + for c in 1..=caches { + if i + c < len { + unsafe { + code.code.instructions.write_cache_u16(i + c, 0); + } + } + } + i += 1 + caches; + } + } + + // All opcodes are now base opcodes. + + if events == 0 { + *monitoring_data = None; + return; + } + + // === Phase 4-6: Re-instrument (innermost first) === + + // Ensure monitoring data exists + if monitoring_data.is_none() { + *monitoring_data = Some(CoMonitoringData { + line_opcodes: vec![0u8; len], + per_instruction_opcodes: vec![0u8; len], + }); + } + let data = monitoring_data.as_mut().unwrap(); + // Resize if code length changed (shouldn't happen, but be safe) + data.line_opcodes.resize(len, 0); + data.per_instruction_opcodes.resize(len, 0); + + // Find _co_firsttraceable: index of first RESUME instruction + let first_traceable = code + .code + .instructions + .iter() + .position(|u| matches!(u.op, Instruction::Resume { .. } | Instruction::ResumeCheck)) + .unwrap_or(0); + + // Phase 4: Place regular INSTRUMENTED_* opcodes + for i in 0..len { + let op = code.code.instructions[i].op; + if let Some(instrumented) = op.to_instrumented() { + unsafe { + code.code.instructions.replace_op(i, instrumented); + } + } + } + + // Phase 5: Place INSTRUMENTED_INSTRUCTION (if EVENT_INSTRUCTION is active) + if events & EVENT_INSTRUCTION != 0 { + for i in first_traceable..len { + let op = code.code.instructions[i].op; + // Skip ExtendedArg + if matches!(op, Instruction::ExtendedArg) { + continue; + } + // Excluded: RESUME, END_FOR, CACHE (and their instrumented variants) + let base = op.to_base().map_or(op, |b| b); + if matches!( + base, + Instruction::Resume { .. } | Instruction::EndFor | Instruction::Cache + ) { + continue; + } + // Store current opcode (may already be INSTRUMENTED_*) and replace + data.per_instruction_opcodes[i] = u8::from(op); + unsafe { + code.code + .instructions + .replace_op(i, Instruction::InstrumentedInstruction); + } + } + } + + // Phase 6: Place INSTRUMENTED_LINE (if EVENT_LINE is active) + // Mirrors CPython's initialize_lines: first determine which positions + // are line starts, then mark branch/jump targets, then place opcodes. + if events & EVENT_LINE != 0 { + // is_line_start[i] = true if position i should have INSTRUMENTED_LINE + let mut is_line_start = vec![false; len]; + + // First pass: mark positions where the source line changes + let mut prev_line: Option<u32> = None; + for (i, unit) in code + .code + .instructions + .iter() + .enumerate() + .take(len) + .skip(first_traceable) + { + let op = unit.op; + let base = op.to_base().map_or(op, |b| b); + if matches!(base, Instruction::ExtendedArg) { + continue; + } + // Excluded opcodes + if matches!( + base, + Instruction::Resume { .. } + | Instruction::EndFor + | Instruction::EndSend + | Instruction::PopIter + | Instruction::EndAsyncFor + | Instruction::Cache + ) { + continue; + } + if let Some((loc, _)) = code.code.locations.get(i) { + let line = loc.line.get() as u32; + let is_new = prev_line != Some(line); + prev_line = Some(line); + if is_new && line > 0 { + is_line_start[i] = true; + } + } + } + + // Second pass: mark branch/jump targets as line starts. + // Every jump/branch target must be a line start, even if on the + // same source line as the preceding instruction. Critical for loops + // (JUMP_BACKWARD → FOR_ITER). + let mut arg_state = bytecode::OpArgState::default(); + let mut instr_idx = first_traceable; + for unit in code.code.instructions[first_traceable..len].iter().copied() { + let (op, arg) = arg_state.get(unit); + let base = op.to_base().map_or(op, |b| b); + + if matches!(base, Instruction::ExtendedArg) || matches!(base, Instruction::Cache) { + instr_idx += 1; + continue; + } + + let caches = base.cache_entries(); + let after_caches = instr_idx + 1 + caches; + let delta = u32::from(arg) as usize; + + let target: Option<usize> = match base { + // Forward relative jumps + Instruction::PopJumpIfFalse { .. } + | Instruction::PopJumpIfTrue { .. } + | Instruction::PopJumpIfNone { .. } + | Instruction::PopJumpIfNotNone { .. } + | Instruction::JumpForward { .. } => Some(after_caches + delta), + // Backward relative jumps + Instruction::JumpBackward { .. } | Instruction::JumpBackwardNoInterrupt { .. } => { + Some(after_caches.wrapping_sub(delta)) + } + Instruction::ForIter { .. } | Instruction::Send { .. } => { + // Skip over END_FOR/END_SEND + Some(after_caches + delta + 1) + } + _ => None, + }; + + if let Some(target_idx) = target + && target_idx < len + && !is_line_start[target_idx] + { + let target_op = code.code.instructions[target_idx].op; + let target_base = target_op.to_base().map_or(target_op, |b| b); + // Skip POP_ITER targets + if matches!(target_base, Instruction::PopIter) { + instr_idx += 1; + continue; + } + if let Some((loc, _)) = code.code.locations.get(target_idx) + && loc.line.get() > 0 + { + is_line_start[target_idx] = true; + } + } + instr_idx += 1; + } + + // Third pass: mark exception handler targets as line starts. + for entry in bytecode::decode_exception_table(&code.code.exceptiontable) { + let target_idx = entry.target as usize; + if target_idx < len && !is_line_start[target_idx] { + let target_op = code.code.instructions[target_idx].op; + let target_base = target_op.to_base().map_or(target_op, |b| b); + if !matches!(target_base, Instruction::PopIter) + && let Some((loc, _)) = code.code.locations.get(target_idx) + && loc.line.get() > 0 + { + is_line_start[target_idx] = true; + } + } + } + + // Fourth pass: actually place INSTRUMENTED_LINE at all marked positions + for (i, marked) in is_line_start + .iter() + .copied() + .enumerate() + .take(len) + .skip(first_traceable) + { + if marked { + let op = code.code.instructions[i].op; + data.line_opcodes[i] = u8::from(op); + unsafe { + code.code + .instructions + .replace_op(i, Instruction::InstrumentedLine); + } + } + } + } +} + +/// Update the global monitoring_events atomic mask from current state. +fn update_events_mask(vm: &VirtualMachine, state: &MonitoringState) { + let events = state.combined_events(); + vm.state.monitoring_events.store(events); + let new_ver = vm + .state + .instrumentation_version + .fetch_add(1, Ordering::Release) + + 1; + // Eagerly re-instrument all frames on the current thread's stack so that + // code objects already past their RESUME pick up the new event set. + // Each code object gets only the events that apply to it (global + its + // own local events), preventing e.g. INSTRUCTION from being applied to + // unrelated code objects. + for fp in vm.frames.borrow().iter() { + // SAFETY: frames in the Vec are alive while their FrameRef is on the call stack. + let frame = unsafe { fp.as_ref() }; + let code = &frame.code; + let code_ver = code.instrumentation_version.load(Ordering::Acquire); + if code_ver != new_ver { + let code_events = state.events_for_code(code.get_id()); + instrument_code(code, code_events); + code.instrumentation_version + .store(new_ver, Ordering::Release); + } + } +} + +fn use_tool_id(tool_id: i32, name: &str, vm: &VirtualMachine) -> PyResult<()> { + let tool = check_valid_tool(tool_id, vm)?; + let mut state = vm.state.monitoring.lock(); + if state.tool_names[tool].is_some() { + return Err(vm.new_value_error(format!("tool {tool_id} is already in use"))); + } + state.tool_names[tool] = Some(name.to_owned()); + Ok(()) +} + +fn clear_tool_id(tool_id: i32, vm: &VirtualMachine) -> PyResult<()> { + let tool = check_valid_tool(tool_id, vm)?; + let mut state = vm.state.monitoring.lock(); + if state.tool_names[tool].is_some() { + state.global_events[tool] = 0; + state + .local_events + .retain(|(local_tool, _), _| *local_tool != tool); + state.callbacks.retain(|(cb_tool, _), _| *cb_tool != tool); + state.disabled.retain(|&(_, _, t)| t != tool); + } + update_events_mask(vm, &state); + Ok(()) +} + +fn free_tool_id(tool_id: i32, vm: &VirtualMachine) -> PyResult<()> { + let tool = check_valid_tool(tool_id, vm)?; + let mut state = vm.state.monitoring.lock(); + if state.tool_names[tool].is_some() { + state.global_events[tool] = 0; + state + .local_events + .retain(|(local_tool, _), _| *local_tool != tool); + state.callbacks.retain(|(cb_tool, _), _| *cb_tool != tool); + state.disabled.retain(|&(_, _, t)| t != tool); + state.tool_names[tool] = None; + } + update_events_mask(vm, &state); + Ok(()) +} + +fn get_tool(tool_id: i32, vm: &VirtualMachine) -> PyResult<Option<String>> { + let tool = check_valid_tool(tool_id, vm)?; + let state = vm.state.monitoring.lock(); + Ok(state.tool_names[tool].clone()) +} + +fn register_callback( + tool_id: i32, + event: i32, + func: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<PyObjectRef> { + let tool = check_valid_tool(tool_id, vm)?; + let event_id = parse_single_event(event, vm)?; + + let mut state = vm.state.monitoring.lock(); + let prev = state + .callbacks + .remove(&(tool, event_id)) + .unwrap_or_else(|| vm.ctx.none()); + let branch_id = EVENT_BRANCH.trailing_zeros() as usize; + let branch_left_id = EVENT_BRANCH_LEFT.trailing_zeros() as usize; + let branch_right_id = EVENT_BRANCH_RIGHT.trailing_zeros() as usize; + if !vm.is_none(&func) { + state.callbacks.insert((tool, event_id), func.clone()); + // BRANCH is a composite event: also register for BRANCH_LEFT/RIGHT + if event_id == branch_id { + state.callbacks.insert((tool, branch_left_id), func.clone()); + state.callbacks.insert((tool, branch_right_id), func); + } + } else { + // Also clear BRANCH_LEFT/RIGHT when clearing BRANCH + if event_id == branch_id { + state.callbacks.remove(&(tool, branch_left_id)); + state.callbacks.remove(&(tool, branch_right_id)); + } + } + Ok(prev) +} + +fn get_events(tool_id: i32, vm: &VirtualMachine) -> PyResult<u32> { + let tool = check_valid_tool(tool_id, vm)?; + let state = vm.state.monitoring.lock(); + Ok(state.global_events[tool]) +} + +fn set_events(tool_id: i32, event_set: i32, vm: &VirtualMachine) -> PyResult<()> { + let tool = check_valid_tool(tool_id, vm)?; + check_tool_in_use(tool, vm)?; + let normalized = normalize_event_set(event_set, false, vm)?; + let mut state = vm.state.monitoring.lock(); + state.global_events[tool] = normalized; + update_events_mask(vm, &state); + Ok(()) +} + +fn get_local_events(tool_id: i32, code: PyObjectRef, vm: &VirtualMachine) -> PyResult<u32> { + if code.downcast_ref::<PyCode>().is_none() { + return Err(vm.new_type_error("code must be a code object")); + } + let tool = check_valid_tool(tool_id, vm)?; + let code_id = code.get_id(); + let state = vm.state.monitoring.lock(); + Ok(state + .local_events + .get(&(tool, code_id)) + .copied() + .unwrap_or(0)) +} + +fn set_local_events( + tool_id: i32, + code: PyObjectRef, + event_set: i32, + vm: &VirtualMachine, +) -> PyResult<()> { + if code.downcast_ref::<PyCode>().is_none() { + return Err(vm.new_type_error("code must be a code object")); + } + let tool = check_valid_tool(tool_id, vm)?; + check_tool_in_use(tool, vm)?; + let normalized = normalize_event_set(event_set, true, vm)?; + let code_id = code.get_id(); + let mut state = vm.state.monitoring.lock(); + if normalized == 0 { + state.local_events.remove(&(tool, code_id)); + } else { + state.local_events.insert((tool, code_id), normalized); + } + update_events_mask(vm, &state); + Ok(()) +} + +fn restart_events(vm: &VirtualMachine) { + let mut state = vm.state.monitoring.lock(); + state.disabled.clear(); +} + +fn all_events(vm: &VirtualMachine) -> PyResult<PyDictRef> { + // Collect data under the lock, then release before calling into Python VM. + let masks: Vec<(&str, u8)> = { + let state = vm.state.monitoring.lock(); + EVENT_NAMES + .iter() + .take(UNGROUPED_EVENTS_COUNT) + .enumerate() + .filter_map(|(event_id, event_name)| { + let event_bit = 1u32 << event_id; + let mut tools_mask = 0u8; + for tool in 0..TOOL_LIMIT { + if (state.global_events[tool] & event_bit) != 0 { + tools_mask |= 1 << tool; + } + } + if tools_mask != 0 { + Some((*event_name, tools_mask)) + } else { + None + } + }) + .collect() + }; + let all_events = vm.ctx.new_dict(); + for (name, mask) in masks { + all_events.set_item(name, vm.ctx.new_int(mask).into(), vm)?; + } + Ok(all_events) +} + +// Event dispatch + +use core::cell::Cell; + +thread_local! { + /// Re-entrancy guard: prevents monitoring callbacks from triggering + /// additional monitoring events (which would cause infinite recursion). + static FIRING: Cell<bool> = const { Cell::new(false) }; + + /// Tracks whether a RERAISE event has been fired since the last + /// EXCEPTION_HANDLED. Used to suppress duplicate RERAISE from + /// cleanup handlers that chain through multiple exception table entries. + static RERAISE_PENDING: Cell<bool> = const { Cell::new(false) }; +} + +/// Fire an event for all tools that have the event bit set. +/// `cb_extra` contains the callback arguments after the code object. +fn fire( + vm: &VirtualMachine, + event: u32, + code: &PyRef<PyCode>, + offset: u32, + cb_extra: &[PyObjectRef], +) -> PyResult<()> { + // Prevent recursive event firing + if FIRING.with(|f| f.get()) { + return Ok(()); + } + + let event_id = event.trailing_zeros() as usize; + let code_id = code.get_id(); + + // C_RETURN and C_RAISE are implicitly enabled when CALL is set. + let check_bit = if event & EVENT_C_RETURN_MASK != 0 { + event | EVENT_CALL + } else { + event + }; + + // Collect callbacks and snapshot the DISABLE sentinel under a single lock. + let (callbacks, disable_sentinel): (Vec<(usize, PyObjectRef)>, Option<PyObjectRef>) = { + let state = vm.state.monitoring.lock(); + let mut cbs = Vec::new(); + for tool in 0..TOOL_LIMIT { + let global = state.global_events[tool]; + let local = state + .local_events + .get(&(tool, code_id)) + .copied() + .unwrap_or(0); + if ((global | local) & check_bit) == 0 { + continue; + } + if state.disabled.contains(&(code_id, offset as usize, tool)) { + continue; + } + if let Some(cb) = state.callbacks.get(&(tool, event_id)) { + cbs.push((tool, cb.clone())); + } + } + (cbs, state.disable.clone()) + }; + + if callbacks.is_empty() { + return Ok(()); + } + + let mut args_vec = Vec::with_capacity(1 + cb_extra.len()); + args_vec.push(code.clone().into()); + args_vec.extend_from_slice(cb_extra); + let args = FuncArgs::from(args_vec); + + FIRING.with(|f| f.set(true)); + let result = (|| { + for (tool, cb) in callbacks { + let result = cb.call(args.clone(), vm)?; + if disable_sentinel.as_ref().is_some_and(|d| result.is(d)) { + // Only local events (event_id < LOCAL_EVENTS_COUNT) can be disabled. + // Non-local events (RAISE, EXCEPTION_HANDLED, PY_UNWIND, etc.) + // cannot be disabled per code object. + if event_id >= LOCAL_EVENTS_COUNT { + // Remove the callback. + let mut state = vm.state.monitoring.lock(); + state.callbacks.remove(&(tool, event_id)); + return Err(vm.new_value_error(format!( + "Cannot disable {} events. Callback removed.", + EVENT_NAMES[event_id] + ))); + } + let mut state = vm.state.monitoring.lock(); + state.disabled.insert((code_id, offset as usize, tool)); + } + } + Ok(()) + })(); + FIRING.with(|f| f.set(false)); + result +} + +// Public dispatch functions (called from frame.rs) + +pub fn fire_py_start(vm: &VirtualMachine, code: &PyRef<PyCode>, offset: u32) -> PyResult<()> { + fire( + vm, + EVENT_PY_START, + code, + offset, + &[vm.ctx.new_int(offset).into()], + ) +} + +pub fn fire_py_resume(vm: &VirtualMachine, code: &PyRef<PyCode>, offset: u32) -> PyResult<()> { + fire( + vm, + EVENT_PY_RESUME, + code, + offset, + &[vm.ctx.new_int(offset).into()], + ) +} + +pub fn fire_py_return( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + retval: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_PY_RETURN, + code, + offset, + &[vm.ctx.new_int(offset).into(), retval.clone()], + ) +} + +pub fn fire_py_yield( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + retval: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_PY_YIELD, + code, + offset, + &[vm.ctx.new_int(offset).into(), retval.clone()], + ) +} + +pub fn fire_call( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + callable: &PyObjectRef, + arg0: PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_CALL, + code, + offset, + &[vm.ctx.new_int(offset).into(), callable.clone(), arg0], + ) +} + +pub fn fire_c_return( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + callable: &PyObjectRef, + arg0: PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_C_RETURN, + code, + offset, + &[vm.ctx.new_int(offset).into(), callable.clone(), arg0], + ) +} + +pub fn fire_c_raise( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + callable: &PyObjectRef, + arg0: PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_C_RAISE, + code, + offset, + &[vm.ctx.new_int(offset).into(), callable.clone(), arg0], + ) +} + +pub fn fire_line( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + line: u32, +) -> PyResult<()> { + fire(vm, EVENT_LINE, code, offset, &[vm.ctx.new_int(line).into()]) +} + +pub fn fire_instruction(vm: &VirtualMachine, code: &PyRef<PyCode>, offset: u32) -> PyResult<()> { + fire( + vm, + EVENT_INSTRUCTION, + code, + offset, + &[vm.ctx.new_int(offset).into()], + ) +} + +pub fn fire_raise( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_RAISE, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +/// Only fires if no RERAISE has been fired since the last EXCEPTION_HANDLED, +/// preventing duplicate events from chained cleanup handlers. +pub fn fire_reraise( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + if RERAISE_PENDING.with(|f| f.get()) { + return Ok(()); + } + RERAISE_PENDING.with(|f| f.set(true)); + let result = fire( + vm, + EVENT_RERAISE, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ); + if result.is_err() { + RERAISE_PENDING.with(|f| f.set(false)); + } + result +} + +pub fn fire_exception_handled( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + RERAISE_PENDING.with(|f| f.set(false)); + fire( + vm, + EVENT_EXCEPTION_HANDLED, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +pub fn fire_py_unwind( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + RERAISE_PENDING.with(|f| f.set(false)); + fire( + vm, + EVENT_PY_UNWIND, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +pub fn fire_py_throw( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_PY_THROW, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +pub fn fire_stop_iteration( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + exception: &PyObjectRef, +) -> PyResult<()> { + fire( + vm, + EVENT_STOP_ITERATION, + code, + offset, + &[vm.ctx.new_int(offset).into(), exception.clone()], + ) +} + +pub fn fire_jump( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + destination: u32, +) -> PyResult<()> { + fire( + vm, + EVENT_JUMP, + code, + offset, + &[ + vm.ctx.new_int(offset).into(), + vm.ctx.new_int(destination).into(), + ], + ) +} + +pub fn fire_branch_left( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + destination: u32, +) -> PyResult<()> { + fire( + vm, + EVENT_BRANCH_LEFT, + code, + offset, + &[ + vm.ctx.new_int(offset).into(), + vm.ctx.new_int(destination).into(), + ], + ) +} + +pub fn fire_branch_right( + vm: &VirtualMachine, + code: &PyRef<PyCode>, + offset: u32, + destination: u32, +) -> PyResult<()> { + fire( + vm, + EVENT_BRANCH_RIGHT, + code, + offset, + &[ + vm.ctx.new_int(offset).into(), + vm.ctx.new_int(destination).into(), + ], + ) +} + +#[pymodule(sub)] +pub(super) mod sys_monitoring { + use super::*; + + #[pyclass(no_attr, module = "sys.monitoring", name = "_Sentinel")] + #[derive(Debug, PyPayload)] + pub(super) struct MonitoringSentinel; + + #[pyclass] + impl MonitoringSentinel {} + + #[pyattr(name = "DEBUGGER_ID")] + const DEBUGGER_ID: u8 = 0; + #[pyattr(name = "COVERAGE_ID")] + const COVERAGE_ID: u8 = 1; + #[pyattr(name = "PROFILER_ID")] + const PROFILER_ID: u8 = 2; + #[pyattr(name = "OPTIMIZER_ID")] + const OPTIMIZER_ID: u8 = 5; + + #[pyattr(once, name = "DISABLE")] + fn disable(vm: &VirtualMachine) -> PyObjectRef { + super::get_disable(vm) + } + + #[pyattr(once, name = "MISSING")] + fn missing(vm: &VirtualMachine) -> PyObjectRef { + super::get_missing(vm) + } + + #[pyattr(once)] + fn events(vm: &VirtualMachine) -> PyRef<PyNamespace> { + let events = PyNamespace::default().into_ref(&vm.ctx); + for (event_id, event_name) in EVENT_NAMES.iter().enumerate() { + events + .as_object() + .set_attr(*event_name, vm.ctx.new_int(1u32 << event_id), vm) + .expect("setting sys.monitoring.events attribute should not fail"); + } + events + .as_object() + .set_attr("NO_EVENTS", vm.ctx.new_int(0), vm) + .expect("setting sys.monitoring.events.NO_EVENTS should not fail"); + events + } + + #[pyfunction] + fn use_tool_id(tool_id: i32, name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<()> { + super::use_tool_id(tool_id, name.as_str(), vm) + } + + #[pyfunction] + fn clear_tool_id(tool_id: i32, vm: &VirtualMachine) -> PyResult<()> { + super::clear_tool_id(tool_id, vm) + } + + #[pyfunction] + fn free_tool_id(tool_id: i32, vm: &VirtualMachine) -> PyResult<()> { + super::free_tool_id(tool_id, vm) + } + + #[pyfunction] + fn get_tool(tool_id: i32, vm: &VirtualMachine) -> PyResult<Option<String>> { + super::get_tool(tool_id, vm) + } + + #[pyfunction] + fn register_callback( + tool_id: i32, + event: i32, + func: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<PyObjectRef> { + super::register_callback(tool_id, event, func, vm) + } + + #[pyfunction] + fn get_events(tool_id: i32, vm: &VirtualMachine) -> PyResult<u32> { + super::get_events(tool_id, vm) + } + + #[pyfunction] + fn set_events(tool_id: i32, event_set: i32, vm: &VirtualMachine) -> PyResult<()> { + super::set_events(tool_id, event_set, vm) + } + + #[pyfunction] + fn get_local_events(tool_id: i32, code: PyObjectRef, vm: &VirtualMachine) -> PyResult<u32> { + super::get_local_events(tool_id, code, vm) + } + + #[pyfunction] + fn set_local_events( + tool_id: i32, + code: PyObjectRef, + event_set: i32, + vm: &VirtualMachine, + ) -> PyResult<()> { + super::set_local_events(tool_id, code, event_set, vm) + } + + #[pyfunction] + fn restart_events(vm: &VirtualMachine) { + super::restart_events(vm) + } + + #[pyfunction] + fn _all_events(vm: &VirtualMachine) -> PyResult<PyDictRef> { + super::all_events(vm) + } +} diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs new file mode 100644 index 00000000000..d38152db84a --- /dev/null +++ b/crates/vm/src/stdlib/time.rs @@ -0,0 +1,1603 @@ +//cspell:ignore cfmt +//! The python `time` module. + +// See also: +// https://docs.python.org/3/library/time.html + +pub use decl::time; + +pub(crate) use decl::module_def; + +#[cfg(not(target_env = "msvc"))] +#[cfg(not(target_arch = "wasm32"))] +unsafe extern "C" { + #[cfg(not(target_os = "freebsd"))] + #[link_name = "daylight"] + static c_daylight: core::ffi::c_int; + // pub static dstbias: std::ffi::c_int; + #[link_name = "timezone"] + static c_timezone: core::ffi::c_long; + #[link_name = "tzname"] + static c_tzname: [*const core::ffi::c_char; 2]; + #[link_name = "tzset"] + fn c_tzset(); +} + +#[pymodule(name = "time", with(#[cfg(any(unix, windows))] platform))] +mod decl { + use crate::{ + AsObject, Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyStrRef, PyTypeRef}, + function::{Either, FuncArgs, OptionalArg}, + types::{PyStructSequence, struct_sequence_new}, + }; + #[cfg(any(unix, windows))] + use crate::{common::wtf8::Wtf8Buf, convert::ToPyObject}; + #[cfg(unix)] + use alloc::ffi::CString; + #[cfg(not(any(unix, windows)))] + use chrono::{ + DateTime, Datelike, TimeZone, Timelike, + naive::{NaiveDate, NaiveDateTime, NaiveTime}, + }; + use core::time::Duration; + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + use windows_sys::Win32::System::Time::{GetTimeZoneInformation, TIME_ZONE_INFORMATION}; + + #[cfg(windows)] + unsafe extern "C" { + fn wcsftime( + s: *mut libc::wchar_t, + max: libc::size_t, + format: *const libc::wchar_t, + tm: *const libc::tm, + ) -> libc::size_t; + } + + #[allow(dead_code)] + pub(super) const SEC_TO_MS: i64 = 1000; + #[allow(dead_code)] + pub(super) const MS_TO_US: i64 = 1000; + #[allow(dead_code)] + pub(super) const SEC_TO_US: i64 = SEC_TO_MS * MS_TO_US; + #[allow(dead_code)] + pub(super) const US_TO_NS: i64 = 1000; + #[allow(dead_code)] + pub(super) const MS_TO_NS: i64 = MS_TO_US * US_TO_NS; + #[allow(dead_code)] + pub(super) const SEC_TO_NS: i64 = SEC_TO_MS * MS_TO_NS; + #[allow(dead_code)] + pub(super) const NS_TO_MS: i64 = 1000 * 1000; + #[allow(dead_code)] + pub(super) const NS_TO_US: i64 = 1000; + + fn duration_since_system_now(vm: &VirtualMachine) -> PyResult<Duration> { + use std::time::{SystemTime, UNIX_EPOCH}; + + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| vm.new_value_error(format!("Time error: {e:?}"))) + } + + #[pyattr] + pub const _STRUCT_TM_ITEMS: usize = 11; + + // TODO: implement proper monotonic time for wasm/wasi. + #[cfg(not(any(unix, windows)))] + fn get_monotonic_time(vm: &VirtualMachine) -> PyResult<Duration> { + duration_since_system_now(vm) + } + + // TODO: implement proper perf time for wasm/wasi. + #[cfg(not(any(unix, windows)))] + fn get_perf_time(vm: &VirtualMachine) -> PyResult<Duration> { + duration_since_system_now(vm) + } + + #[pyfunction] + fn sleep(seconds: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let seconds_type_name = seconds.clone().class().name().to_owned(); + let dur = seconds.try_into_value::<Duration>(vm).map_err(|e| { + if e.class().is(vm.ctx.exceptions.value_error) + && let Some(s) = e.args().first().and_then(|arg| arg.str(vm).ok()) + && s.as_bytes() == b"negative duration" + { + return vm.new_value_error("sleep length must be non-negative"); + } + if e.class().is(vm.ctx.exceptions.type_error) { + return vm.new_type_error(format!( + "'{seconds_type_name}' object cannot be interpreted as an integer or float" + )); + } + e + })?; + + #[cfg(unix)] + { + // this is basically std::thread::sleep, but that catches interrupts and we don't want to; + let ts = nix::sys::time::TimeSpec::from(dur); + // Capture errno inside the closure: attach_thread (called by + // allow_threads on return) can clobber errno via syscalls. + let (res, err) = vm.allow_threads(|| { + let r = unsafe { libc::nanosleep(ts.as_ref(), core::ptr::null_mut()) }; + (r, nix::Error::last_raw()) + }); + let interrupted = res == -1 && err == libc::EINTR; + + if interrupted { + vm.check_signals()?; + } + } + + #[cfg(not(unix))] + { + vm.allow_threads(|| std::thread::sleep(dur)); + } + + Ok(()) + } + + #[pyfunction] + fn time_ns(vm: &VirtualMachine) -> PyResult<u64> { + Ok(duration_since_system_now(vm)?.as_nanos() as u64) + } + + #[pyfunction] + pub fn time(vm: &VirtualMachine) -> PyResult<f64> { + _time(vm) + } + + #[cfg(not(all( + target_arch = "wasm32", + not(any(target_os = "emscripten", target_os = "wasi")), + )))] + fn _time(vm: &VirtualMachine) -> PyResult<f64> { + Ok(duration_since_system_now(vm)?.as_secs_f64()) + } + + #[cfg(all( + target_arch = "wasm32", + feature = "wasmbind", + not(any(target_os = "emscripten", target_os = "wasi")) + ))] + fn _time(_vm: &VirtualMachine) -> PyResult<f64> { + use wasm_bindgen::prelude::*; + #[wasm_bindgen] + extern "C" { + type Date; + #[wasm_bindgen(static_method_of = Date)] + fn now() -> f64; + } + // Date.now returns unix time in milliseconds, we want it in seconds + Ok(Date::now() / 1000.0) + } + + #[cfg(all( + target_arch = "wasm32", + not(feature = "wasmbind"), + not(any(target_os = "emscripten", target_os = "wasi")) + ))] + fn _time(vm: &VirtualMachine) -> PyResult<f64> { + Err(vm.new_not_implemented_error("time.time")) + } + + #[pyfunction] + fn monotonic(vm: &VirtualMachine) -> PyResult<f64> { + Ok(get_monotonic_time(vm)?.as_secs_f64()) + } + + #[pyfunction] + fn monotonic_ns(vm: &VirtualMachine) -> PyResult<u128> { + Ok(get_monotonic_time(vm)?.as_nanos()) + } + + #[pyfunction] + fn perf_counter(vm: &VirtualMachine) -> PyResult<f64> { + Ok(get_perf_time(vm)?.as_secs_f64()) + } + + #[pyfunction] + fn perf_counter_ns(vm: &VirtualMachine) -> PyResult<u128> { + Ok(get_perf_time(vm)?.as_nanos()) + } + + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + pub(super) fn get_tz_info() -> TIME_ZONE_INFORMATION { + let mut info: TIME_ZONE_INFORMATION = unsafe { core::mem::zeroed() }; + unsafe { GetTimeZoneInformation(&mut info) }; + info + } + + // #[pyfunction] + // fn tzset() { + // unsafe { super::_tzset() }; + // } + + #[cfg(not(target_env = "msvc"))] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn altzone(_vm: &VirtualMachine) -> core::ffi::c_long { + // TODO: RUSTPYTHON; Add support for using the C altzone + unsafe { super::c_timezone - 3600 } + } + + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn altzone(_vm: &VirtualMachine) -> i32 { + let info = get_tz_info(); + // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 + (info.Bias + info.StandardBias) * 60 - 3600 + } + + #[cfg(not(target_env = "msvc"))] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn timezone(_vm: &VirtualMachine) -> core::ffi::c_long { + unsafe { super::c_timezone } + } + + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn timezone(_vm: &VirtualMachine) -> i32 { + let info = get_tz_info(); + // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 + (info.Bias + info.StandardBias) * 60 + } + + #[cfg(not(target_os = "freebsd"))] + #[cfg(not(target_env = "msvc"))] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn daylight(_vm: &VirtualMachine) -> core::ffi::c_int { + unsafe { super::c_daylight } + } + + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn daylight(_vm: &VirtualMachine) -> i32 { + let info = get_tz_info(); + // https://users.rust-lang.org/t/accessing-tzname-and-similar-constants-in-windows/125771/3 + (info.StandardBias != info.DaylightBias) as i32 + } + + #[cfg(not(target_env = "msvc"))] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn tzname(vm: &VirtualMachine) -> crate::builtins::PyTupleRef { + use crate::builtins::tuple::IntoPyTuple; + + unsafe fn to_str(s: *const core::ffi::c_char) -> String { + unsafe { core::ffi::CStr::from_ptr(s) } + .to_string_lossy() + .into_owned() + } + unsafe { (to_str(super::c_tzname[0]), to_str(super::c_tzname[1])) }.into_pytuple(vm) + } + + #[cfg(target_env = "msvc")] + #[cfg(not(target_arch = "wasm32"))] + #[pyattr] + fn tzname(vm: &VirtualMachine) -> crate::builtins::PyTupleRef { + use crate::builtins::tuple::IntoPyTuple; + let info = get_tz_info(); + let standard = widestring::decode_utf16_lossy(info.StandardName) + .take_while(|&c| c != '\0') + .collect::<String>(); + let daylight = widestring::decode_utf16_lossy(info.DaylightName) + .take_while(|&c| c != '\0') + .collect::<String>(); + let tz_name = (&*standard, &*daylight); + tz_name.into_pytuple(vm) + } + + #[cfg(not(any(unix, windows)))] + fn pyobj_to_date_time( + value: Either<f64, i64>, + vm: &VirtualMachine, + ) -> PyResult<DateTime<chrono::offset::Utc>> { + let secs = match value { + Either::A(float) => { + if !float.is_finite() { + return Err(vm.new_value_error("Invalid value for timestamp")); + } + float.floor() as i64 + } + Either::B(int) => int, + }; + DateTime::<chrono::offset::Utc>::from_timestamp(secs, 0) + .ok_or_else(|| vm.new_overflow_error("timestamp out of range for platform time_t")) + } + + #[cfg(not(any(unix, windows)))] + impl OptionalArg<Option<Either<f64, i64>>> { + /// Construct a localtime from the optional seconds, or get the current local time. + fn naive_or_local(self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { + Ok(match self { + Self::Present(Some(secs)) => pyobj_to_date_time(secs, vm)? + .with_timezone(&chrono::Local) + .naive_local(), + Self::Present(None) | Self::Missing => chrono::offset::Local::now().naive_local(), + }) + } + } + + #[cfg(any(unix, windows))] + struct CheckedTm { + tm: libc::tm, + #[cfg(unix)] + zone: Option<CString>, + } + + #[cfg(any(unix, windows))] + fn checked_tm_from_struct_time( + t: &StructTimeData, + vm: &VirtualMachine, + func_name: &'static str, + ) -> PyResult<CheckedTm> { + let invalid_tuple = + || vm.new_type_error(format!("{func_name}(): illegal time tuple argument")); + + let year: i64 = t.tm_year.clone().try_into_value(vm).map_err(|e| { + if e.class().is(vm.ctx.exceptions.overflow_error) { + vm.new_overflow_error("year out of range") + } else { + invalid_tuple() + } + })?; + if year < i64::from(i32::MIN) + 1900 || year > i64::from(i32::MAX) { + return Err(vm.new_overflow_error("year out of range")); + } + let year = year as i32; + let tm_mon = t + .tm_mon + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + - 1; + let tm_mday = t + .tm_mday + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + let tm_hour = t + .tm_hour + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + let tm_min = t + .tm_min + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + let tm_sec = t + .tm_sec + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + let tm_wday = (t + .tm_wday + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + + 1) + % 7; + let tm_yday = t + .tm_yday + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + - 1; + let tm_isdst = t + .tm_isdst + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + + let mut tm: libc::tm = unsafe { core::mem::zeroed() }; + tm.tm_year = year - 1900; + tm.tm_mon = tm_mon; + tm.tm_mday = tm_mday; + tm.tm_hour = tm_hour; + tm.tm_min = tm_min; + tm.tm_sec = tm_sec; + tm.tm_wday = tm_wday; + tm.tm_yday = tm_yday; + tm.tm_isdst = tm_isdst; + + if tm.tm_mon == -1 { + tm.tm_mon = 0; + } else if tm.tm_mon < 0 || tm.tm_mon > 11 { + return Err(vm.new_value_error("month out of range")); + } + if tm.tm_mday == 0 { + tm.tm_mday = 1; + } else if tm.tm_mday < 0 || tm.tm_mday > 31 { + return Err(vm.new_value_error("day of month out of range")); + } + if tm.tm_hour < 0 || tm.tm_hour > 23 { + return Err(vm.new_value_error("hour out of range")); + } + if tm.tm_min < 0 || tm.tm_min > 59 { + return Err(vm.new_value_error("minute out of range")); + } + if tm.tm_sec < 0 || tm.tm_sec > 61 { + return Err(vm.new_value_error("seconds out of range")); + } + if tm.tm_wday < 0 { + return Err(vm.new_value_error("day of week out of range")); + } + if tm.tm_yday == -1 { + tm.tm_yday = 0; + } else if tm.tm_yday < 0 || tm.tm_yday > 365 { + return Err(vm.new_value_error("day of year out of range")); + } + + #[cfg(unix)] + { + use crate::builtins::PyUtf8StrRef; + let zone = if t.tm_zone.is(&vm.ctx.none) { + None + } else { + let zone: PyUtf8StrRef = t + .tm_zone + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + Some( + CString::new(zone.as_str()) + .map_err(|_| vm.new_value_error("embedded null character"))?, + ) + }; + if let Some(zone) = &zone { + tm.tm_zone = zone.as_ptr().cast_mut(); + } + if !t.tm_gmtoff.is(&vm.ctx.none) { + let gmtoff: i64 = t + .tm_gmtoff + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_gmtoff = gmtoff as _; + } + + Ok(CheckedTm { tm, zone }) + } + #[cfg(windows)] + { + Ok(CheckedTm { tm }) + } + } + + #[cfg(any(unix, windows))] + fn asctime_from_tm(tm: &libc::tm) -> String { + const WDAY_NAME: [&str; 7] = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; + const MON_NAME: [&str; 12] = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + ]; + format!( + "{} {}{:>3} {:02}:{:02}:{:02} {}", + WDAY_NAME[tm.tm_wday as usize], + MON_NAME[tm.tm_mon as usize], + tm.tm_mday, + tm.tm_hour, + tm.tm_min, + tm.tm_sec, + tm.tm_year + 1900 + ) + } + + #[cfg(not(any(unix, windows)))] + impl OptionalArg<StructTimeData> { + fn naive_or_local(self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { + Ok(match self { + Self::Present(t) => t.to_date_time(vm)?, + Self::Missing => chrono::offset::Local::now().naive_local(), + }) + } + } + + /// https://docs.python.org/3/library/time.html?highlight=gmtime#time.gmtime + #[pyfunction] + fn gmtime( + secs: OptionalArg<Option<Either<f64, i64>>>, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + #[cfg(any(unix, windows))] + { + let ts = match secs { + OptionalArg::Present(Some(value)) => pyobj_to_time_t(value, vm)?, + OptionalArg::Present(None) | OptionalArg::Missing => current_time_t(), + }; + gmtime_from_timestamp(ts, vm) + } + + #[cfg(not(any(unix, windows)))] + { + let instant = match secs { + OptionalArg::Present(Some(secs)) => pyobj_to_date_time(secs, vm)?.naive_utc(), + OptionalArg::Present(None) | OptionalArg::Missing => { + chrono::offset::Utc::now().naive_utc() + } + }; + Ok(StructTimeData::new_utc(vm, instant)) + } + } + + #[pyfunction] + fn localtime( + secs: OptionalArg<Option<Either<f64, i64>>>, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + #[cfg(any(unix, windows))] + { + let ts = match secs { + OptionalArg::Present(Some(value)) => pyobj_to_time_t(value, vm)?, + OptionalArg::Present(None) | OptionalArg::Missing => current_time_t(), + }; + localtime_from_timestamp(ts, vm) + } + + #[cfg(not(any(unix, windows)))] + let instant = secs.naive_or_local(vm)?; + #[cfg(not(any(unix, windows)))] + { + Ok(StructTimeData::new_local(vm, instant, 0)) + } + } + + #[pyfunction] + fn mktime(t: StructTimeData, vm: &VirtualMachine) -> PyResult<f64> { + #[cfg(unix)] + { + unix_mktime(&t, vm) + } + + #[cfg(windows)] + { + win_mktime(&t, vm) + } + + #[cfg(not(any(unix, windows)))] + { + let datetime = t.to_date_time(vm)?; + // mktime interprets struct_time as local time + let local_dt = chrono::Local + .from_local_datetime(&datetime) + .single() + .ok_or_else(|| vm.new_overflow_error("mktime argument out of range"))?; + let seconds_since_epoch = local_dt.timestamp() as f64; + Ok(seconds_since_epoch) + } + } + + #[cfg(not(any(unix, windows)))] + const CFMT: &str = "%a %b %e %H:%M:%S %Y"; + + #[pyfunction] + fn asctime(t: OptionalArg<StructTimeData>, vm: &VirtualMachine) -> PyResult { + #[cfg(any(unix, windows))] + { + let tm = match t { + OptionalArg::Present(value) => { + checked_tm_from_struct_time(&value, vm, "asctime")?.tm + } + OptionalArg::Missing => { + let now = current_time_t(); + let local = localtime_from_timestamp(now, vm)?; + checked_tm_from_struct_time(&local, vm, "asctime")?.tm + } + }; + Ok(vm.ctx.new_str(asctime_from_tm(&tm)).into()) + } + + #[cfg(not(any(unix, windows)))] + { + let instant = t.naive_or_local(vm)?; + let formatted_time = instant.format(CFMT).to_string(); + Ok(vm.ctx.new_str(formatted_time).into()) + } + } + + #[pyfunction] + fn ctime(secs: OptionalArg<Option<Either<f64, i64>>>, vm: &VirtualMachine) -> PyResult<String> { + #[cfg(any(unix, windows))] + { + let ts = match secs { + OptionalArg::Present(Some(value)) => pyobj_to_time_t(value, vm)?, + OptionalArg::Present(None) | OptionalArg::Missing => current_time_t(), + }; + let local = localtime_from_timestamp(ts, vm)?; + let tm = checked_tm_from_struct_time(&local, vm, "asctime")?.tm; + Ok(asctime_from_tm(&tm)) + } + + #[cfg(not(any(unix, windows)))] + { + let instant = secs.naive_or_local(vm)?; + Ok(instant.format(CFMT).to_string()) + } + } + + #[cfg(any(unix, windows))] + fn strftime_crt(format: &PyStrRef, checked_tm: CheckedTm, vm: &VirtualMachine) -> PyResult { + #[cfg(unix)] + let _keep_zone_alive = &checked_tm.zone; + let mut tm = checked_tm.tm; + tm.tm_isdst = tm.tm_isdst.clamp(-1, 1); + + // MSVC strftime requires year in [1; 9999] + #[cfg(windows)] + { + let year = tm.tm_year + 1900; + if !(1..=9999).contains(&year) { + return Err(vm.new_value_error("strftime() requires year in [1; 9999]")); + } + } + + #[cfg(unix)] + fn strftime_ascii(fmt: &str, tm: &libc::tm, vm: &VirtualMachine) -> PyResult<String> { + let fmt_c = + CString::new(fmt).map_err(|_| vm.new_value_error("embedded null character"))?; + let mut size = 1024usize; + let max_scale = 256usize.saturating_mul(fmt.len().max(1)); + loop { + let mut out = vec![0u8; size]; + let written = unsafe { + libc::strftime( + out.as_mut_ptr().cast(), + out.len(), + fmt_c.as_ptr(), + tm as *const libc::tm, + ) + }; + if written > 0 || size >= max_scale { + return Ok(String::from_utf8_lossy(&out[..written]).into_owned()); + } + size = size.saturating_mul(2); + } + } + + #[cfg(windows)] + fn strftime_ascii(fmt: &str, tm: &libc::tm, vm: &VirtualMachine) -> PyResult<String> { + if fmt.contains('\0') { + return Err(vm.new_value_error("embedded null character")); + } + // Use wcsftime for proper Unicode output (e.g. %Z timezone names) + let fmt_wide: Vec<u16> = fmt.encode_utf16().chain(core::iter::once(0)).collect(); + let mut size = 1024usize; + let max_scale = 256usize.saturating_mul(fmt.len().max(1)); + loop { + let mut out = vec![0u16; size]; + let written = unsafe { + rustpython_common::suppress_iph!(wcsftime( + out.as_mut_ptr(), + out.len(), + fmt_wide.as_ptr(), + tm as *const libc::tm, + )) + }; + if written > 0 || size >= max_scale { + return Ok(String::from_utf16_lossy(&out[..written])); + } + size = size.saturating_mul(2); + } + } + + let mut out = Wtf8Buf::new(); + let mut ascii = String::new(); + + for codepoint in format.as_wtf8().code_points() { + if codepoint.to_u32() == 0 { + if !ascii.is_empty() { + let part = strftime_ascii(&ascii, &tm, vm)?; + out.extend(part.chars()); + ascii.clear(); + } + out.push(codepoint); + continue; + } + if let Some(ch) = codepoint.to_char() + && ch.is_ascii() + { + ascii.push(ch); + continue; + } + + if !ascii.is_empty() { + let part = strftime_ascii(&ascii, &tm, vm)?; + out.extend(part.chars()); + ascii.clear(); + } + out.push(codepoint); + } + if !ascii.is_empty() { + let part = strftime_ascii(&ascii, &tm, vm)?; + out.extend(part.chars()); + } + Ok(out.to_pyobject(vm)) + } + + #[pyfunction] + fn strftime(format: PyStrRef, t: OptionalArg<StructTimeData>, vm: &VirtualMachine) -> PyResult { + #[cfg(any(unix, windows))] + { + let checked_tm = match t { + OptionalArg::Present(value) => checked_tm_from_struct_time(&value, vm, "strftime")?, + OptionalArg::Missing => { + let now = current_time_t(); + let local = localtime_from_timestamp(now, vm)?; + checked_tm_from_struct_time(&local, vm, "strftime")? + } + }; + strftime_crt(&format, checked_tm, vm) + } + + #[cfg(not(any(unix, windows)))] + { + use core::fmt::Write; + + let fmt_lossy = format.to_string_lossy(); + + // If the struct_time can't be represented as NaiveDateTime + // (e.g. month=0), return the format string as-is, matching + // the fallback behavior for unsupported chrono formats. + let instant = match t.naive_or_local(vm) { + Ok(dt) => dt, + Err(_) => return Ok(vm.ctx.new_str(fmt_lossy.into_owned()).into()), + }; + + let mut formatted_time = String::new(); + write!(&mut formatted_time, "{}", instant.format(&fmt_lossy)) + .unwrap_or_else(|_| formatted_time = format.to_string()); + Ok(vm.ctx.new_str(formatted_time).into()) + } + } + + #[pyfunction] + fn strptime(string: PyStrRef, format: OptionalArg<PyStrRef>, vm: &VirtualMachine) -> PyResult { + // Call _strptime._strptime_time like CPython does + let strptime_module = vm.import("_strptime", 0)?; + let strptime_func = strptime_module.get_attr("_strptime_time", vm)?; + + // Call with positional arguments + match format.into_option() { + Some(fmt) => strptime_func.call((string, fmt), vm), + None => strptime_func.call((string,), vm), + } + } + + #[cfg(not(any( + windows, + target_vendor = "apple", + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "fuchsia", + target_os = "emscripten", + )))] + fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { + Err(vm.new_not_implemented_error("thread time unsupported in this system")) + } + + #[pyfunction] + fn thread_time(vm: &VirtualMachine) -> PyResult<f64> { + Ok(get_thread_time(vm)?.as_secs_f64()) + } + + #[pyfunction] + fn thread_time_ns(vm: &VirtualMachine) -> PyResult<u64> { + Ok(get_thread_time(vm)?.as_nanos() as u64) + } + + #[cfg(any(windows, all(target_arch = "wasm32", target_os = "emscripten")))] + pub(super) fn time_muldiv(ticks: i64, mul: i64, div: i64) -> u64 { + let int_part = ticks / div; + let ticks = ticks % div; + let remaining = (ticks * mul) / div; + (int_part * mul + remaining) as u64 + } + + #[cfg(all(target_arch = "wasm32", target_os = "emscripten"))] + fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { + let t: libc::tms = unsafe { + let mut t = core::mem::MaybeUninit::uninit(); + if libc::times(t.as_mut_ptr()) == -1 { + return Err(vm.new_os_error("Failed to get clock time".to_owned())); + } + t.assume_init() + }; + let freq = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; + + Ok(Duration::from_nanos( + time_muldiv(t.tms_utime, SEC_TO_NS, freq) + time_muldiv(t.tms_stime, SEC_TO_NS, freq), + )) + } + + #[cfg(not(any( + windows, + target_os = "macos", + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "linux", + target_os = "illumos", + target_os = "netbsd", + target_os = "solaris", + target_os = "openbsd", + target_os = "redox", + all(target_arch = "wasm32", target_os = "emscripten") + )))] + fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { + Err(vm.new_not_implemented_error("process time unsupported in this system")) + } + + #[pyfunction] + fn process_time(vm: &VirtualMachine) -> PyResult<f64> { + Ok(get_process_time(vm)?.as_secs_f64()) + } + + #[pyfunction] + fn process_time_ns(vm: &VirtualMachine) -> PyResult<u64> { + Ok(get_process_time(vm)?.as_nanos() as u64) + } + + /// Data struct for struct_time + #[pystruct_sequence_data(try_from_object)] + pub struct StructTimeData { + pub tm_year: PyObjectRef, + pub tm_mon: PyObjectRef, + pub tm_mday: PyObjectRef, + pub tm_hour: PyObjectRef, + pub tm_min: PyObjectRef, + pub tm_sec: PyObjectRef, + pub tm_wday: PyObjectRef, + pub tm_yday: PyObjectRef, + pub tm_isdst: PyObjectRef, + #[pystruct_sequence(skip)] + pub tm_zone: PyObjectRef, + #[pystruct_sequence(skip)] + pub tm_gmtoff: PyObjectRef, + } + + impl core::fmt::Debug for StructTimeData { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "struct_time()") + } + } + + impl StructTimeData { + #[cfg(not(any(unix, windows)))] + fn new_inner( + vm: &VirtualMachine, + tm: NaiveDateTime, + isdst: i32, + gmtoff: i32, + zone: &str, + ) -> Self { + Self { + tm_year: vm.ctx.new_int(tm.year()).into(), + tm_mon: vm.ctx.new_int(tm.month()).into(), + tm_mday: vm.ctx.new_int(tm.day()).into(), + tm_hour: vm.ctx.new_int(tm.hour()).into(), + tm_min: vm.ctx.new_int(tm.minute()).into(), + tm_sec: vm.ctx.new_int(tm.second()).into(), + tm_wday: vm.ctx.new_int(tm.weekday().num_days_from_monday()).into(), + tm_yday: vm.ctx.new_int(tm.ordinal()).into(), + tm_isdst: vm.ctx.new_int(isdst).into(), + tm_zone: vm.ctx.new_str(zone).into(), + tm_gmtoff: vm.ctx.new_int(gmtoff).into(), + } + } + + /// Create struct_time for UTC (gmtime) + #[cfg(not(any(unix, windows)))] + fn new_utc(vm: &VirtualMachine, tm: NaiveDateTime) -> Self { + Self::new_inner(vm, tm, 0, 0, "UTC") + } + + /// Create struct_time for local timezone (localtime) + #[cfg(not(any(unix, windows)))] + fn new_local(vm: &VirtualMachine, tm: NaiveDateTime, isdst: i32) -> Self { + let local_time = chrono::Local.from_local_datetime(&tm).unwrap(); + let offset_seconds = local_time.offset().local_minus_utc(); + let tz_abbr = local_time.format("%Z").to_string(); + Self::new_inner(vm, tm, isdst, offset_seconds, &tz_abbr) + } + + #[cfg(not(any(unix, windows)))] + fn to_date_time(&self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { + let invalid_overflow = || vm.new_overflow_error("mktime argument out of range"); + let invalid_value = || vm.new_value_error("invalid struct_time parameter"); + + macro_rules! field { + ($field:ident) => { + self.$field.clone().try_into_value(vm)? + }; + } + let dt = NaiveDateTime::new( + NaiveDate::from_ymd_opt(field!(tm_year), field!(tm_mon), field!(tm_mday)) + .ok_or_else(invalid_value)?, + NaiveTime::from_hms_opt(field!(tm_hour), field!(tm_min), field!(tm_sec)) + .ok_or_else(invalid_overflow)?, + ); + Ok(dt) + } + } + + #[pyattr] + #[pystruct_sequence(name = "struct_time", module = "time", data = "StructTimeData")] + pub struct PyStructTime; + + #[pyclass(with(PyStructSequence))] + impl PyStructTime { + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let (seq, _dict): (PyObjectRef, OptionalArg<PyObjectRef>) = args.bind(vm)?; + struct_sequence_new(cls, seq, vm) + } + } + + /// Extract fields from StructTimeData into a libc::tm for mktime. + #[cfg(any(unix, windows))] + pub(super) fn tm_from_struct_time( + t: &StructTimeData, + vm: &VirtualMachine, + ) -> PyResult<libc::tm> { + let invalid_tuple = || vm.new_type_error("mktime(): illegal time tuple argument"); + let year: i32 = t + .tm_year + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + if year < i32::MIN + 1900 { + return Err(vm.new_overflow_error("year out of range")); + } + + let mut tm: libc::tm = unsafe { core::mem::zeroed() }; + tm.tm_sec = t + .tm_sec + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_min = t + .tm_min + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_hour = t + .tm_hour + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_mday = t + .tm_mday + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + tm.tm_mon = t + .tm_mon + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + - 1; + tm.tm_year = year - 1900; + tm.tm_wday = -1; + tm.tm_yday = t + .tm_yday + .clone() + .try_into_value::<i32>(vm) + .map_err(|_| invalid_tuple())? + - 1; + tm.tm_isdst = t + .tm_isdst + .clone() + .try_into_value(vm) + .map_err(|_| invalid_tuple())?; + Ok(tm) + } + + #[cfg(any(unix, windows))] + #[cfg_attr(target_env = "musl", allow(deprecated))] + fn pyobj_to_time_t(value: Either<f64, i64>, vm: &VirtualMachine) -> PyResult<libc::time_t> { + match value { + Either::A(float) => { + if !float.is_finite() { + return Err(vm.new_value_error("Invalid value for timestamp")); + } + let secs = float.floor(); + #[cfg_attr(target_env = "musl", allow(deprecated))] + if secs < libc::time_t::MIN as f64 || secs > libc::time_t::MAX as f64 { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + #[cfg_attr(target_env = "musl", allow(deprecated))] + Ok(secs as libc::time_t) + } + Either::B(int) => { + // try_into is needed on 32-bit platforms where time_t != i64 + #[allow(clippy::useless_conversion)] + #[cfg_attr(target_env = "musl", allow(deprecated))] + let ts: libc::time_t = int.try_into().map_err(|_| { + vm.new_overflow_error("timestamp out of range for platform time_t") + })?; + Ok(ts) + } + } + } + + #[cfg(any(unix, windows))] + #[allow(unused_imports)] + use super::platform::*; + + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py<crate::builtins::PyModule>, + ) -> PyResult<()> { + #[cfg(not(target_env = "msvc"))] + #[cfg(not(target_arch = "wasm32"))] + unsafe { + super::c_tzset() + }; + + __module_exec(vm, module); + Ok(()) + } +} + +#[cfg(unix)] +#[pymodule(sub)] +mod platform { + #[allow(unused_imports)] + use super::decl::{SEC_TO_NS, StructTimeData, US_TO_NS}; + #[cfg_attr(target_os = "macos", allow(unused_imports))] + use crate::{ + PyObject, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, + builtins::{PyNamespace, PyUtf8StrRef}, + convert::IntoPyException, + }; + use core::time::Duration; + #[cfg_attr(target_env = "musl", allow(deprecated))] + use libc::time_t; + use nix::{sys::time::TimeSpec, time::ClockId}; + + #[cfg(target_os = "solaris")] + #[pyattr] + use libc::CLOCK_HIGHRES; + #[cfg(not(any( + target_os = "illumos", + target_os = "netbsd", + target_os = "solaris", + target_os = "openbsd", + target_os = "wasi", + )))] + #[pyattr] + use libc::CLOCK_PROCESS_CPUTIME_ID; + #[cfg(not(any( + target_os = "illumos", + target_os = "netbsd", + target_os = "solaris", + target_os = "openbsd", + target_os = "redox", + )))] + #[pyattr] + use libc::CLOCK_THREAD_CPUTIME_ID; + #[cfg(target_os = "linux")] + #[pyattr] + use libc::{CLOCK_BOOTTIME, CLOCK_MONOTONIC_RAW, CLOCK_TAI}; + #[pyattr] + use libc::{CLOCK_MONOTONIC, CLOCK_REALTIME}; + #[cfg(any(target_os = "freebsd", target_os = "openbsd", target_os = "dragonfly"))] + #[pyattr] + use libc::{CLOCK_PROF, CLOCK_UPTIME}; + + impl<'a> TryFromBorrowedObject<'a> for ClockId { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { + obj.try_to_value(vm).map(Self::from_raw) + } + } + + fn struct_time_from_tm(vm: &VirtualMachine, tm: libc::tm) -> StructTimeData { + let zone = unsafe { + if tm.tm_zone.is_null() { + String::new() + } else { + core::ffi::CStr::from_ptr(tm.tm_zone) + .to_string_lossy() + .into_owned() + } + }; + StructTimeData { + tm_year: vm.ctx.new_int(tm.tm_year + 1900).into(), + tm_mon: vm.ctx.new_int(tm.tm_mon + 1).into(), + tm_mday: vm.ctx.new_int(tm.tm_mday).into(), + tm_hour: vm.ctx.new_int(tm.tm_hour).into(), + tm_min: vm.ctx.new_int(tm.tm_min).into(), + tm_sec: vm.ctx.new_int(tm.tm_sec).into(), + tm_wday: vm.ctx.new_int((tm.tm_wday + 6) % 7).into(), + tm_yday: vm.ctx.new_int(tm.tm_yday + 1).into(), + tm_isdst: vm.ctx.new_int(tm.tm_isdst).into(), + tm_zone: vm.ctx.new_str(zone).into(), + tm_gmtoff: vm.ctx.new_int(tm.tm_gmtoff).into(), + } + } + + #[cfg_attr(target_env = "musl", allow(deprecated))] + pub(super) fn current_time_t() -> time_t { + unsafe { libc::time(core::ptr::null_mut()) } + } + + #[cfg_attr(target_env = "musl", allow(deprecated))] + pub(super) fn gmtime_from_timestamp( + when: time_t, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + let mut out = core::mem::MaybeUninit::<libc::tm>::uninit(); + let ret = unsafe { libc::gmtime_r(&when, out.as_mut_ptr()) }; + if ret.is_null() { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + Ok(struct_time_from_tm(vm, unsafe { out.assume_init() })) + } + + #[cfg_attr(target_env = "musl", allow(deprecated))] + pub(super) fn localtime_from_timestamp( + when: time_t, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + let mut out = core::mem::MaybeUninit::<libc::tm>::uninit(); + let ret = unsafe { libc::localtime_r(&when, out.as_mut_ptr()) }; + if ret.is_null() { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + Ok(struct_time_from_tm(vm, unsafe { out.assume_init() })) + } + + pub(super) fn unix_mktime(t: &StructTimeData, vm: &VirtualMachine) -> PyResult<f64> { + let mut tm = super::decl::tm_from_struct_time(t, vm)?; + let timestamp = unsafe { libc::mktime(&mut tm) }; + if timestamp == -1 && tm.tm_wday == -1 { + return Err(vm.new_overflow_error("mktime argument out of range")); + } + Ok(timestamp as f64) + } + + fn get_clock_time(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<Duration> { + let ts = nix::time::clock_gettime(clk_id).map_err(|e| e.into_pyexception(vm))?; + Ok(ts.into()) + } + + #[pyfunction] + fn clock_gettime(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<f64> { + get_clock_time(clk_id, vm).map(|d| d.as_secs_f64()) + } + + #[pyfunction] + fn clock_gettime_ns(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<u128> { + get_clock_time(clk_id, vm).map(|d| d.as_nanos()) + } + + #[cfg(not(target_os = "redox"))] + #[pyfunction] + fn clock_getres(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<f64> { + let ts = nix::time::clock_getres(clk_id).map_err(|e| e.into_pyexception(vm))?; + Ok(Duration::from(ts).as_secs_f64()) + } + + #[cfg(not(target_os = "redox"))] + #[cfg(not(target_vendor = "apple"))] + fn set_clock_time(clk_id: ClockId, timespec: TimeSpec, vm: &VirtualMachine) -> PyResult<()> { + nix::time::clock_settime(clk_id, timespec).map_err(|e| e.into_pyexception(vm)) + } + + #[cfg(not(target_os = "redox"))] + #[cfg(target_os = "macos")] + fn set_clock_time(clk_id: ClockId, timespec: TimeSpec, vm: &VirtualMachine) -> PyResult<()> { + // idk why nix disables clock_settime on macos + let ret = unsafe { libc::clock_settime(clk_id.as_raw(), timespec.as_ref()) }; + nix::Error::result(ret) + .map(drop) + .map_err(|e| e.into_pyexception(vm)) + } + + #[cfg(not(target_os = "redox"))] + #[cfg(any(not(target_vendor = "apple"), target_os = "macos"))] + #[pyfunction] + fn clock_settime(clk_id: ClockId, time: Duration, vm: &VirtualMachine) -> PyResult<()> { + set_clock_time(clk_id, time.into(), vm) + } + + #[cfg(not(target_os = "redox"))] + #[cfg(any(not(target_vendor = "apple"), target_os = "macos"))] + #[cfg_attr(target_env = "musl", allow(deprecated))] + #[pyfunction] + fn clock_settime_ns(clk_id: ClockId, time: libc::time_t, vm: &VirtualMachine) -> PyResult<()> { + let ts = Duration::from_nanos(time as _).into(); + set_clock_time(clk_id, ts, vm) + } + + // Requires all CLOCK constants available and clock_getres + #[cfg(any( + target_os = "macos", + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "emscripten", + target_os = "linux", + ))] + #[pyfunction] + fn get_clock_info(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<PyRef<PyNamespace>> { + let (adj, imp, mono, res) = match name.as_str() { + "monotonic" | "perf_counter" => ( + false, + "time.clock_gettime(CLOCK_MONOTONIC)", + true, + clock_getres(ClockId::CLOCK_MONOTONIC, vm)?, + ), + "process_time" => ( + false, + "time.clock_gettime(CLOCK_PROCESS_CPUTIME_ID)", + true, + clock_getres(ClockId::CLOCK_PROCESS_CPUTIME_ID, vm)?, + ), + "thread_time" => ( + false, + "time.clock_gettime(CLOCK_THREAD_CPUTIME_ID)", + true, + clock_getres(ClockId::CLOCK_THREAD_CPUTIME_ID, vm)?, + ), + "time" => ( + true, + "time.clock_gettime(CLOCK_REALTIME)", + false, + clock_getres(ClockId::CLOCK_REALTIME, vm)?, + ), + _ => return Err(vm.new_value_error("unknown clock")), + }; + + Ok(py_namespace!(vm, { + "implementation" => vm.new_pyobj(imp), + "monotonic" => vm.ctx.new_bool(mono), + "adjustable" => vm.ctx.new_bool(adj), + "resolution" => vm.ctx.new_float(res), + })) + } + + #[cfg(not(any( + target_os = "macos", + target_os = "android", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "fuchsia", + target_os = "emscripten", + target_os = "linux", + )))] + #[pyfunction] + fn get_clock_info(_name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<PyRef<PyNamespace>> { + Err(vm.new_not_implemented_error("get_clock_info unsupported on this system")) + } + + pub(super) fn get_monotonic_time(vm: &VirtualMachine) -> PyResult<Duration> { + get_clock_time(ClockId::CLOCK_MONOTONIC, vm) + } + + pub(super) fn get_perf_time(vm: &VirtualMachine) -> PyResult<Duration> { + get_clock_time(ClockId::CLOCK_MONOTONIC, vm) + } + + #[cfg(not(any( + target_os = "illumos", + target_os = "netbsd", + target_os = "openbsd", + target_os = "redox" + )))] + pub(super) fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { + get_clock_time(ClockId::CLOCK_THREAD_CPUTIME_ID, vm) + } + + #[cfg(target_os = "solaris")] + pub(super) fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { + Ok(Duration::from_nanos(unsafe { libc::gethrvtime() })) + } + + #[cfg(not(any( + target_os = "illumos", + target_os = "netbsd", + target_os = "solaris", + target_os = "openbsd", + )))] + pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { + get_clock_time(ClockId::CLOCK_PROCESS_CPUTIME_ID, vm) + } + + #[cfg(any( + target_os = "illumos", + target_os = "netbsd", + target_os = "solaris", + target_os = "openbsd", + ))] + pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { + use nix::sys::resource::{UsageWho, getrusage}; + fn from_timeval(tv: libc::timeval, vm: &VirtualMachine) -> PyResult<i64> { + (|tv: libc::timeval| { + let t = tv.tv_sec.checked_mul(SEC_TO_NS)?; + let u = (tv.tv_usec as i64).checked_mul(US_TO_NS)?; + t.checked_add(u) + })(tv) + .ok_or_else(|| vm.new_overflow_error("timestamp too large to convert to i64")) + } + let ru = getrusage(UsageWho::RUSAGE_SELF).map_err(|e| e.into_pyexception(vm))?; + let utime = from_timeval(ru.user_time().into(), vm)?; + let stime = from_timeval(ru.system_time().into(), vm)?; + + Ok(Duration::from_nanos((utime + stime) as u64)) + } +} + +#[cfg(windows)] +#[pymodule(sub)] +mod platform { + use super::decl::{MS_TO_NS, SEC_TO_NS, StructTimeData, get_tz_info, time_muldiv}; + use crate::{ + PyRef, PyResult, VirtualMachine, + builtins::{PyNamespace, PyUtf8StrRef}, + }; + use core::time::Duration; + use windows_sys::Win32::{ + Foundation::FILETIME, + System::Performance::{QueryPerformanceCounter, QueryPerformanceFrequency}, + System::SystemInformation::{GetSystemTimeAdjustment, GetTickCount64}, + System::Threading::{GetCurrentProcess, GetCurrentThread, GetProcessTimes, GetThreadTimes}, + }; + + unsafe extern "C" { + fn _gmtime64_s(tm: *mut libc::tm, time: *const libc::time_t) -> libc::c_int; + fn _localtime64_s(tm: *mut libc::tm, time: *const libc::time_t) -> libc::c_int; + #[link_name = "_mktime64"] + fn c_mktime(tm: *mut libc::tm) -> libc::time_t; + } + + fn struct_time_from_tm( + vm: &VirtualMachine, + tm: libc::tm, + zone: &str, + gmtoff: i32, + ) -> StructTimeData { + StructTimeData { + tm_year: vm.ctx.new_int(tm.tm_year + 1900).into(), + tm_mon: vm.ctx.new_int(tm.tm_mon + 1).into(), + tm_mday: vm.ctx.new_int(tm.tm_mday).into(), + tm_hour: vm.ctx.new_int(tm.tm_hour).into(), + tm_min: vm.ctx.new_int(tm.tm_min).into(), + tm_sec: vm.ctx.new_int(tm.tm_sec).into(), + tm_wday: vm.ctx.new_int((tm.tm_wday + 6) % 7).into(), + tm_yday: vm.ctx.new_int(tm.tm_yday + 1).into(), + tm_isdst: vm.ctx.new_int(tm.tm_isdst).into(), + tm_zone: vm.ctx.new_str(zone).into(), + tm_gmtoff: vm.ctx.new_int(gmtoff).into(), + } + } + + #[cfg_attr(target_env = "musl", allow(deprecated))] + pub(super) fn current_time_t() -> libc::time_t { + unsafe { libc::time(core::ptr::null_mut()) } + } + + #[cfg_attr(target_env = "musl", allow(deprecated))] + pub(super) fn gmtime_from_timestamp( + when: libc::time_t, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + let mut out = core::mem::MaybeUninit::<libc::tm>::uninit(); + let err = unsafe { _gmtime64_s(out.as_mut_ptr(), &when) }; + if err != 0 { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + Ok(struct_time_from_tm( + vm, + unsafe { out.assume_init() }, + "UTC", + 0, + )) + } + + #[cfg_attr(target_env = "musl", allow(deprecated))] + pub(super) fn localtime_from_timestamp( + when: libc::time_t, + vm: &VirtualMachine, + ) -> PyResult<StructTimeData> { + let mut out = core::mem::MaybeUninit::<libc::tm>::uninit(); + let err = unsafe { _localtime64_s(out.as_mut_ptr(), &when) }; + if err != 0 { + return Err(vm.new_overflow_error("timestamp out of range for platform time_t")); + } + let tm = unsafe { out.assume_init() }; + + // Get timezone info from Windows API + let info = get_tz_info(); + let (bias, name) = if tm.tm_isdst > 0 { + (info.DaylightBias, &info.DaylightName) + } else { + (info.StandardBias, &info.StandardName) + }; + let zone = widestring::decode_utf16_lossy(name.iter().copied()) + .take_while(|&c| c != '\0') + .collect::<String>(); + let gmtoff = -((info.Bias + bias) as i32) * 60; + + Ok(struct_time_from_tm(vm, tm, &zone, gmtoff)) + } + + pub(super) fn win_mktime(t: &StructTimeData, vm: &VirtualMachine) -> PyResult<f64> { + let mut tm = super::decl::tm_from_struct_time(t, vm)?; + let timestamp = unsafe { rustpython_common::suppress_iph!(c_mktime(&mut tm)) }; + if timestamp == -1 && tm.tm_wday == -1 { + return Err(vm.new_overflow_error("mktime argument out of range")); + } + Ok(timestamp as f64) + } + + fn u64_from_filetime(time: FILETIME) -> u64 { + let large: [u32; 2] = [time.dwLowDateTime, time.dwHighDateTime]; + unsafe { core::mem::transmute(large) } + } + + fn win_perf_counter_frequency(vm: &VirtualMachine) -> PyResult<i64> { + let frequency = unsafe { + let mut freq = core::mem::MaybeUninit::uninit(); + if QueryPerformanceFrequency(freq.as_mut_ptr()) == 0 { + return Err(vm.new_last_os_error()); + } + freq.assume_init() + }; + + if frequency < 1 { + Err(vm.new_runtime_error("invalid QueryPerformanceFrequency")) + } else if frequency > i64::MAX / SEC_TO_NS { + Err(vm.new_overflow_error("QueryPerformanceFrequency is too large")) + } else { + Ok(frequency) + } + } + + fn global_frequency(vm: &VirtualMachine) -> PyResult<i64> { + rustpython_common::static_cell! { + static FREQUENCY: PyResult<i64>; + }; + FREQUENCY + .get_or_init(|| win_perf_counter_frequency(vm)) + .clone() + } + + pub(super) fn get_perf_time(vm: &VirtualMachine) -> PyResult<Duration> { + let ticks = unsafe { + let mut performance_count = core::mem::MaybeUninit::uninit(); + QueryPerformanceCounter(performance_count.as_mut_ptr()); + performance_count.assume_init() + }; + + Ok(Duration::from_nanos(time_muldiv( + ticks, + SEC_TO_NS, + global_frequency(vm)?, + ))) + } + + fn get_system_time_adjustment(vm: &VirtualMachine) -> PyResult<u32> { + let mut _time_adjustment = core::mem::MaybeUninit::uninit(); + let mut time_increment = core::mem::MaybeUninit::uninit(); + let mut _is_time_adjustment_disabled = core::mem::MaybeUninit::uninit(); + let time_increment = unsafe { + if GetSystemTimeAdjustment( + _time_adjustment.as_mut_ptr(), + time_increment.as_mut_ptr(), + _is_time_adjustment_disabled.as_mut_ptr(), + ) == 0 + { + return Err(vm.new_last_os_error()); + } + time_increment.assume_init() + }; + Ok(time_increment) + } + + pub(super) fn get_monotonic_time(vm: &VirtualMachine) -> PyResult<Duration> { + let ticks = unsafe { GetTickCount64() }; + + Ok(Duration::from_nanos( + (ticks as i64) + .checked_mul(MS_TO_NS) + .ok_or_else(|| vm.new_overflow_error("timestamp too large to convert to i64"))? + as u64, + )) + } + + #[pyfunction] + fn get_clock_info(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<PyRef<PyNamespace>> { + let (adj, imp, mono, res) = match name.as_str() { + "monotonic" => ( + false, + "GetTickCount64()", + true, + get_system_time_adjustment(vm)? as f64 * 1e-7, + ), + "perf_counter" => ( + false, + "QueryPerformanceCounter()", + true, + 1.0 / (global_frequency(vm)? as f64), + ), + "process_time" => (false, "GetProcessTimes()", true, 1e-7), + "thread_time" => (false, "GetThreadTimes()", true, 1e-7), + "time" => ( + true, + "GetSystemTimeAsFileTime()", + false, + get_system_time_adjustment(vm)? as f64 * 1e-7, + ), + _ => return Err(vm.new_value_error("unknown clock")), + }; + + Ok(py_namespace!(vm, { + "implementation" => vm.new_pyobj(imp), + "monotonic" => vm.ctx.new_bool(mono), + "adjustable" => vm.ctx.new_bool(adj), + "resolution" => vm.ctx.new_float(res), + })) + } + + pub(super) fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { + let (kernel_time, user_time) = unsafe { + let mut _creation_time = core::mem::MaybeUninit::uninit(); + let mut _exit_time = core::mem::MaybeUninit::uninit(); + let mut kernel_time = core::mem::MaybeUninit::uninit(); + let mut user_time = core::mem::MaybeUninit::uninit(); + + let thread = GetCurrentThread(); + if GetThreadTimes( + thread, + _creation_time.as_mut_ptr(), + _exit_time.as_mut_ptr(), + kernel_time.as_mut_ptr(), + user_time.as_mut_ptr(), + ) == 0 + { + return Err(vm.new_os_error("Failed to get clock time".to_owned())); + } + (kernel_time.assume_init(), user_time.assume_init()) + }; + let k_time = u64_from_filetime(kernel_time); + let u_time = u64_from_filetime(user_time); + Ok(Duration::from_nanos((k_time + u_time) * 100)) + } + + pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { + let (kernel_time, user_time) = unsafe { + let mut _creation_time = core::mem::MaybeUninit::uninit(); + let mut _exit_time = core::mem::MaybeUninit::uninit(); + let mut kernel_time = core::mem::MaybeUninit::uninit(); + let mut user_time = core::mem::MaybeUninit::uninit(); + + let process = GetCurrentProcess(); + if GetProcessTimes( + process, + _creation_time.as_mut_ptr(), + _exit_time.as_mut_ptr(), + kernel_time.as_mut_ptr(), + user_time.as_mut_ptr(), + ) == 0 + { + return Err(vm.new_os_error("Failed to get clock time".to_owned())); + } + (kernel_time.assume_init(), user_time.assume_init()) + }; + let k_time = u64_from_filetime(kernel_time); + let u_time = u64_from_filetime(user_time); + Ok(Duration::from_nanos((k_time + u_time) * 100)) + } +} diff --git a/crates/vm/src/stdlib/typevar.rs b/crates/vm/src/stdlib/typevar.rs new file mode 100644 index 00000000000..b28fad21bd7 --- /dev/null +++ b/crates/vm/src/stdlib/typevar.rs @@ -0,0 +1,1103 @@ +// spell-checker:ignore typevarobject funcobj + +pub use typevar::*; + +#[pymodule(sub)] +pub(crate) mod typevar { + use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyTuple, PyTupleRef, PyType, PyTypeRef, make_union}, + common::lock::PyMutex, + function::{FuncArgs, PyComparisonValue}, + protocol::PyNumberMethods, + stdlib::_typing::{call_typing_func_object, decl::const_evaluator_alloc}, + types::{AsNumber, Comparable, Constructor, Iterable, PyComparisonOp, Representable}, + }; + + fn type_check(arg: PyObjectRef, msg: &str, vm: &VirtualMachine) -> PyResult<PyObjectRef> { + // Calling typing.py here leads to bootstrapping problems + if vm.is_none(&arg) { + return Ok(arg.class().to_owned().into()); + } + let message_str: PyObjectRef = vm.ctx.new_str(msg).into(); + call_typing_func_object(vm, "_type_check", (arg, message_str)) + } + + fn variance_repr( + name: &str, + infer_variance: bool, + covariant: bool, + contravariant: bool, + ) -> String { + if infer_variance { + return name.to_owned(); + } + let prefix = if covariant { + '+' + } else if contravariant { + '-' + } else { + '~' + }; + format!("{prefix}{name}") + } + + /// Get the module of the caller frame, similar to CPython's caller() function. + /// Returns the module name or None if not found. + /// + /// Note: CPython's implementation (in typevarobject.c) gets the module from the + /// frame's function object using PyFunction_GetModule(f->f_funcobj). However, + /// RustPython's Frame doesn't store a reference to the function object, so we + /// get the module name from the frame's globals dictionary instead. + fn caller(vm: &VirtualMachine) -> Option<PyObjectRef> { + let frame = vm.current_frame()?; + + // In RustPython, we get the module name from frame's globals + // This is similar to CPython's sys._getframe().f_globals.get('__name__') + frame.globals.get_item("__name__", vm).ok() + } + + /// Set __module__ attribute for an object based on the caller's module. + /// This follows CPython's behavior for TypeVar and similar objects. + fn set_module_from_caller(obj: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + // Note: CPython gets module from frame->f_funcobj, but RustPython's Frame + // architecture is different - we use globals['__name__'] instead + let module_value: PyObjectRef = if let Some(module_name) = caller(vm) { + // Special handling for certain module names + if let Ok(name_str) = module_name.str(vm) + && let Some(name) = name_str.to_str() + && (name == "builtins" || name.starts_with('<')) + { + return Ok(()); + } + module_name + } else { + vm.ctx.none() + }; + obj.set_attr("__module__", module_value, vm)?; + Ok(()) + } + + #[pyattr] + #[pyclass(name = "TypeVar", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct TypeVar { + name: PyObjectRef, // TODO PyStrRef? + bound: PyMutex<PyObjectRef>, + evaluate_bound: PyObjectRef, + constraints: PyMutex<PyObjectRef>, + evaluate_constraints: PyObjectRef, + default_value: PyMutex<PyObjectRef>, + evaluate_default: PyMutex<PyObjectRef>, + covariant: bool, + contravariant: bool, + infer_variance: bool, + } + #[pyclass( + flags(HAS_DICT, HAS_WEAKREF), + with(AsNumber, Constructor, Representable) + )] + impl TypeVar { + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of TypeVar")) + } + + #[pygetset] + fn __name__(&self) -> PyObjectRef { + self.name.clone() + } + + #[pygetset] + fn __constraints__(&self, vm: &VirtualMachine) -> PyResult { + let mut constraints = self.constraints.lock(); + if !vm.is_none(&constraints) { + return Ok(constraints.clone()); + } + let r = if !vm.is_none(&self.evaluate_constraints) { + *constraints = self.evaluate_constraints.call((1i32,), vm)?; + constraints.clone() + } else { + vm.ctx.empty_tuple.clone().into() + }; + Ok(r) + } + + #[pygetset] + fn __bound__(&self, vm: &VirtualMachine) -> PyResult { + let mut bound = self.bound.lock(); + if !vm.is_none(&bound) { + return Ok(bound.clone()); + } + let r = if !vm.is_none(&self.evaluate_bound) { + *bound = self.evaluate_bound.call((1i32,), vm)?; + bound.clone() + } else { + vm.ctx.none() + }; + Ok(r) + } + + #[pygetset] + const fn __covariant__(&self) -> bool { + self.covariant + } + + #[pygetset] + const fn __contravariant__(&self) -> bool { + self.contravariant + } + + #[pygetset] + const fn __infer_variance__(&self) -> bool { + self.infer_variance + } + + #[pygetset] + fn __default__(&self, vm: &VirtualMachine) -> PyResult { + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } + } + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) + } else { + Ok(vm.ctx.typing_no_default.clone().into()) + } + } + + #[pygetset] + fn evaluate_bound(&self, vm: &VirtualMachine) -> PyResult { + if !vm.is_none(&self.evaluate_bound) { + return Ok(self.evaluate_bound.clone()); + } + let bound = self.bound.lock(); + if !vm.is_none(&bound) { + return Ok(const_evaluator_alloc(bound.clone(), vm)); + } + Ok(vm.ctx.none()) + } + + #[pygetset] + fn evaluate_constraints(&self, vm: &VirtualMachine) -> PyResult { + if !vm.is_none(&self.evaluate_constraints) { + return Ok(self.evaluate_constraints.clone()); + } + let constraints = self.constraints.lock(); + if !vm.is_none(&constraints) { + return Ok(const_evaluator_alloc(constraints.clone(), vm)); + } + Ok(vm.ctx.none()) + } + + #[pygetset] + fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); + } + let default_value = self.default_value.lock().clone(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(const_evaluator_alloc(default_value, vm)); + } + Ok(vm.ctx.none()) + } + + #[pymethod] + fn __typing_subst__( + zelf: crate::PyRef<Self>, + arg: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + call_typing_func_object(vm, "_typevar_subst", (self_obj, arg)) + } + + #[pymethod] + fn __reduce__(&self) -> PyObjectRef { + self.name.clone() + } + + #[pymethod] + fn has_default(&self, vm: &VirtualMachine) -> bool { + if !vm.is_none(&self.evaluate_default.lock()) { + return true; + } + let default_value = self.default_value.lock(); + // Check if default_value is not NoDefault + !default_value.is(&vm.ctx.typing_no_default) + } + + #[pymethod] + fn __typing_prepare_subst__( + zelf: crate::PyRef<Self>, + alias: PyObjectRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // Convert args to tuple if needed + let args_tuple = + if let Ok(tuple) = args.try_to_ref::<rustpython_vm::builtins::PyTuple>(vm) { + tuple + } else { + return Ok(args); + }; + + // Get alias.__parameters__ + let parameters = alias.get_attr(identifier!(vm, __parameters__), vm)?; + let params_tuple: PyTupleRef = parameters.try_into_value(vm)?; + + // Find our index in parameters + let self_obj: PyObjectRef = zelf.to_owned().into(); + let param_index = params_tuple.iter().position(|p| p.is(&self_obj)); + + if let Some(index) = param_index { + // Check if we have enough arguments + if args_tuple.len() <= index && zelf.has_default(vm) { + // Need to add default value + let mut new_args: Vec<PyObjectRef> = args_tuple.iter().cloned().collect(); + + // Add default value at the correct position + while new_args.len() <= index { + // For the current parameter, add its default + if new_args.len() == index { + let default_val = zelf.__default__(vm)?; + new_args.push(default_val); + } else { + // This shouldn't happen in well-formed code + break; + } + } + + return Ok(rustpython_vm::builtins::PyTuple::new_ref(new_args, &vm.ctx).into()); + } + } + + // No changes needed + Ok(args) + } + } + + impl Representable for TypeVar { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let name = zelf.name.str_utf8(vm)?; + Ok(variance_repr( + name.as_str(), + zelf.infer_variance, + zelf.covariant, + zelf.contravariant, + )) + } + } + + impl AsNumber for TypeVar { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| { + let args = PyTuple::new_ref(vec![a.to_owned(), b.to_owned()], &vm.ctx); + make_union(&args, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + + impl Constructor for TypeVar { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let typevar = <Self as Constructor>::py_new(&cls, args, vm)?; + let obj = typevar.into_ref_with_type(vm, cls)?; + let obj_ref: PyObjectRef = obj.into(); + set_module_from_caller(&obj_ref, vm)?; + Ok(obj_ref) + } + + fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + let mut kwargs = args.kwargs; + // Parse arguments manually + let (name, constraints) = if args.args.is_empty() { + // Check if name is provided as keyword argument + if let Some(name) = kwargs.swap_remove("name") { + (name, vec![]) + } else { + return Err( + vm.new_type_error("TypeVar() missing required argument: 'name' (pos 1)") + ); + } + } else if args.args.len() == 1 { + (args.args[0].clone(), vec![]) + } else { + let name = args.args[0].clone(); + let constraints = args.args[1..].to_vec(); + (name, constraints) + }; + + let bound = kwargs.swap_remove("bound"); + let covariant = kwargs + .swap_remove("covariant") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let contravariant = kwargs + .swap_remove("contravariant") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let infer_variance = kwargs + .swap_remove("infer_variance") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let default = kwargs.swap_remove("default"); + + // Check for unexpected keyword arguments + if !kwargs.is_empty() { + let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); + return Err(vm.new_type_error(format!( + "TypeVar() got unexpected keyword argument(s): {}", + unexpected_keys.join(", ") + ))); + } + + // Check for invalid combinations + if covariant && contravariant { + return Err(vm.new_value_error("Bivariant type variables are not supported.")); + } + + if infer_variance && (covariant || contravariant) { + return Err(vm.new_value_error("Variance cannot be specified with infer_variance")); + } + + // Handle constraints and bound + let (constraints_obj, evaluate_constraints) = if !constraints.is_empty() { + // Check for single constraint + if constraints.len() == 1 { + return Err(vm.new_type_error("A single constraint is not allowed")); + } + if bound.is_some() { + return Err(vm.new_type_error("Constraints cannot be used with bound")); + } + let constraints_tuple = vm.ctx.new_tuple(constraints); + (constraints_tuple.into(), vm.ctx.none()) + } else { + (vm.ctx.none(), vm.ctx.none()) + }; + + // Handle bound + let (bound_obj, evaluate_bound) = if let Some(bound) = bound { + if vm.is_none(&bound) { + (vm.ctx.none(), vm.ctx.none()) + } else { + // Type check the bound + let bound = type_check(bound, "Bound must be a type.", vm)?; + (bound, vm.ctx.none()) + } + } else { + (vm.ctx.none(), vm.ctx.none()) + }; + + // Handle default value + let (default_value, evaluate_default) = if let Some(default) = default { + (default, vm.ctx.none()) + } else { + // If no default provided, use NoDefault singleton + (vm.ctx.typing_no_default.clone().into(), vm.ctx.none()) + }; + + Ok(Self { + name, + bound: PyMutex::new(bound_obj), + evaluate_bound, + constraints: PyMutex::new(constraints_obj), + evaluate_constraints, + default_value: PyMutex::new(default_value), + evaluate_default: PyMutex::new(evaluate_default), + covariant, + contravariant, + infer_variance, + }) + } + } + + impl TypeVar { + pub fn new( + vm: &VirtualMachine, + name: PyObjectRef, + evaluate_bound: PyObjectRef, + evaluate_constraints: PyObjectRef, + ) -> Self { + Self { + name, + bound: PyMutex::new(vm.ctx.none()), + evaluate_bound, + constraints: PyMutex::new(vm.ctx.none()), + evaluate_constraints, + default_value: PyMutex::new(vm.ctx.typing_no_default.clone().into()), + evaluate_default: PyMutex::new(vm.ctx.none()), + covariant: false, + contravariant: false, + infer_variance: true, + } + } + } + + #[pyattr] + #[pyclass(name = "ParamSpec", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct ParamSpec { + name: PyObjectRef, + bound: Option<PyObjectRef>, + default_value: PyMutex<PyObjectRef>, + evaluate_default: PyMutex<PyObjectRef>, + covariant: bool, + contravariant: bool, + infer_variance: bool, + } + + #[pyclass( + flags(HAS_DICT, HAS_WEAKREF), + with(AsNumber, Constructor, Representable) + )] + impl ParamSpec { + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of ParamSpec")) + } + + #[pygetset] + fn __name__(&self) -> PyObjectRef { + self.name.clone() + } + + #[pygetset] + fn args(zelf: crate::PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + let psa = ParamSpecArgs { + __origin__: self_obj, + }; + Ok(psa.into_ref(&vm.ctx).into()) + } + + #[pygetset] + fn kwargs(zelf: crate::PyRef<Self>, vm: &VirtualMachine) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + let psk = ParamSpecKwargs { + __origin__: self_obj, + }; + Ok(psk.into_ref(&vm.ctx).into()) + } + + #[pygetset] + fn __bound__(&self, vm: &VirtualMachine) -> PyObjectRef { + if let Some(bound) = self.bound.clone() { + return bound; + } + vm.ctx.none() + } + + #[pygetset] + const fn __covariant__(&self) -> bool { + self.covariant + } + + #[pygetset] + const fn __contravariant__(&self) -> bool { + self.contravariant + } + + #[pygetset] + const fn __infer_variance__(&self) -> bool { + self.infer_variance + } + + #[pygetset] + fn __default__(&self, vm: &VirtualMachine) -> PyResult { + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } + } + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) + } else { + Ok(vm.ctx.typing_no_default.clone().into()) + } + } + + #[pygetset] + fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); + } + let default_value = self.default_value.lock().clone(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(const_evaluator_alloc(default_value, vm)); + } + Ok(vm.ctx.none()) + } + + #[pymethod] + fn __reduce__(&self) -> PyResult { + Ok(self.name.clone()) + } + + #[pymethod] + fn has_default(&self, vm: &VirtualMachine) -> bool { + if !vm.is_none(&self.evaluate_default.lock()) { + return true; + } + !self.default_value.lock().is(&vm.ctx.typing_no_default) + } + + #[pymethod] + fn __typing_subst__( + zelf: crate::PyRef<Self>, + arg: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + call_typing_func_object(vm, "_paramspec_subst", (self_obj, arg)) + } + + #[pymethod] + fn __typing_prepare_subst__( + zelf: crate::PyRef<Self>, + alias: PyObjectRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + call_typing_func_object(vm, "_paramspec_prepare_subst", (self_obj, alias, args)) + } + } + + impl AsNumber for ParamSpec { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + or: Some(|a, b, vm| { + let args = PyTuple::new_ref(vec![a.to_owned(), b.to_owned()], &vm.ctx); + make_union(&args, vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + + impl Constructor for ParamSpec { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let mut kwargs = args.kwargs; + // Parse arguments manually + let name = if args.args.is_empty() { + // Check if name is provided as keyword argument + if let Some(name) = kwargs.swap_remove("name") { + name + } else { + return Err( + vm.new_type_error("ParamSpec() missing required argument: 'name' (pos 1)") + ); + } + } else if args.args.len() == 1 { + args.args[0].clone() + } else { + return Err(vm.new_type_error("ParamSpec() takes at most 1 positional argument")); + }; + + let bound = kwargs + .swap_remove("bound") + .map(|b| type_check(b, "Bound must be a type.", vm)) + .transpose()?; + let covariant = kwargs + .swap_remove("covariant") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let contravariant = kwargs + .swap_remove("contravariant") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let infer_variance = kwargs + .swap_remove("infer_variance") + .map(|v| v.try_to_bool(vm)) + .transpose()? + .unwrap_or(false); + let default = kwargs.swap_remove("default"); + + // Check for unexpected keyword arguments + if !kwargs.is_empty() { + let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); + return Err(vm.new_type_error(format!( + "ParamSpec() got unexpected keyword argument(s): {}", + unexpected_keys.join(", ") + ))); + } + + // Check for invalid combinations + if covariant && contravariant { + return Err(vm.new_value_error("Bivariant type variables are not supported.")); + } + + if infer_variance && (covariant || contravariant) { + return Err(vm.new_value_error("Variance cannot be specified with infer_variance")); + } + + // Handle default value + let default_value = default.unwrap_or_else(|| vm.ctx.typing_no_default.clone().into()); + + let paramspec = Self { + name, + bound, + default_value: PyMutex::new(default_value), + evaluate_default: PyMutex::new(vm.ctx.none()), + covariant, + contravariant, + infer_variance, + }; + + let obj = paramspec.into_ref_with_type(vm, cls)?; + let obj_ref: PyObjectRef = obj.into(); + set_module_from_caller(&obj_ref, vm)?; + Ok(obj_ref) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + impl Representable for ParamSpec { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let name = zelf.__name__().str_utf8(vm)?; + Ok(variance_repr( + name.as_str(), + zelf.infer_variance, + zelf.covariant, + zelf.contravariant, + )) + } + } + + impl ParamSpec { + pub fn new(name: PyObjectRef, vm: &VirtualMachine) -> Self { + Self { + name, + bound: None, + default_value: PyMutex::new(vm.ctx.typing_no_default.clone().into()), + evaluate_default: PyMutex::new(vm.ctx.none()), + covariant: false, + contravariant: false, + infer_variance: true, + } + } + } + + #[pyattr] + #[pyclass(name = "TypeVarTuple", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct TypeVarTuple { + name: PyObjectRef, + default_value: PyMutex<PyObjectRef>, + evaluate_default: PyMutex<PyObjectRef>, + } + #[pyclass( + flags(HAS_DICT, HAS_WEAKREF), + with(Constructor, Representable, Iterable) + )] + impl TypeVarTuple { + #[pygetset] + fn __name__(&self) -> PyObjectRef { + self.name.clone() + } + + #[pygetset] + fn __default__(&self, vm: &VirtualMachine) -> PyResult { + { + let default_value = self.default_value.lock(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(default_value.clone()); + } + } + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + let result = evaluator.call((1i32,), vm)?; + *self.default_value.lock() = result.clone(); + Ok(result) + } else { + Ok(vm.ctx.typing_no_default.clone().into()) + } + } + + #[pygetset] + fn evaluate_default(&self, vm: &VirtualMachine) -> PyResult { + let evaluator = self.evaluate_default.lock().clone(); + if !vm.is_none(&evaluator) { + return Ok(evaluator); + } + let default_value = self.default_value.lock().clone(); + if !default_value.is(&vm.ctx.typing_no_default) { + return Ok(const_evaluator_alloc(default_value, vm)); + } + Ok(vm.ctx.none()) + } + + #[pymethod] + fn has_default(&self, vm: &VirtualMachine) -> bool { + if !vm.is_none(&self.evaluate_default.lock()) { + return true; + } + let default_value = self.default_value.lock(); + !default_value.is(&vm.ctx.typing_no_default) + } + + #[pymethod] + fn __reduce__(&self) -> PyObjectRef { + self.name.clone() + } + + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of TypeVarTuple")) + } + + #[pymethod] + fn __typing_subst__(&self, _arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Substitution of bare TypeVarTuple is not supported")) + } + + #[pymethod] + fn __typing_prepare_subst__( + zelf: crate::PyRef<Self>, + alias: PyObjectRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let self_obj: PyObjectRef = zelf.into(); + call_typing_func_object(vm, "_typevartuple_prepare_subst", (self_obj, alias, args)) + } + } + + impl Iterable for TypeVarTuple { + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { + // When unpacking TypeVarTuple with *, return [Unpack[self]] + // This is how CPython handles Generic[*Ts] + let typing = vm.import("typing", 0)?; + let unpack = typing.get_attr("Unpack", vm)?; + let zelf_obj: PyObjectRef = zelf.into(); + let unpacked = vm.call_method(&unpack, "__getitem__", (zelf_obj,))?; + let list = vm.ctx.new_list(vec![unpacked]); + let list_obj: PyObjectRef = list.into(); + vm.call_method(&list_obj, "__iter__", ()) + } + } + + impl Constructor for TypeVarTuple { + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let mut kwargs = args.kwargs; + // Parse arguments manually + let name = if args.args.is_empty() { + // Check if name is provided as keyword argument + if let Some(name) = kwargs.swap_remove("name") { + name + } else { + return Err(vm.new_type_error( + "TypeVarTuple() missing required argument: 'name' (pos 1)", + )); + } + } else if args.args.len() == 1 { + args.args[0].clone() + } else { + return Err(vm.new_type_error("TypeVarTuple() takes at most 1 positional argument")); + }; + + let default = kwargs.swap_remove("default"); + + // Check for unexpected keyword arguments + if !kwargs.is_empty() { + let unexpected_keys: Vec<String> = kwargs.keys().map(|s| s.to_string()).collect(); + return Err(vm.new_type_error(format!( + "TypeVarTuple() got unexpected keyword argument(s): {}", + unexpected_keys.join(", ") + ))); + } + + // Handle default value + let (default_value, evaluate_default) = if let Some(default) = default { + (default, vm.ctx.none()) + } else { + // If no default provided, use NoDefault singleton + (vm.ctx.typing_no_default.clone().into(), vm.ctx.none()) + }; + + let typevartuple = Self { + name, + default_value: PyMutex::new(default_value), + evaluate_default: PyMutex::new(evaluate_default), + }; + + let obj = typevartuple.into_ref_with_type(vm, cls)?; + let obj_ref: PyObjectRef = obj.into(); + set_module_from_caller(&obj_ref, vm)?; + Ok(obj_ref) + } + + fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + unimplemented!("use slot_new") + } + } + + impl Representable for TypeVarTuple { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + let name = zelf.name.str(vm)?; + Ok(name.to_string()) + } + } + + impl TypeVarTuple { + pub fn new(name: PyObjectRef, vm: &VirtualMachine) -> Self { + Self { + name, + default_value: PyMutex::new(vm.ctx.typing_no_default.clone().into()), + evaluate_default: PyMutex::new(vm.ctx.none()), + } + } + } + + #[pyattr] + #[pyclass(name = "ParamSpecArgs", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct ParamSpecArgs { + __origin__: PyObjectRef, + } + #[pyclass(with(Constructor, Representable, Comparable), flags(HAS_WEAKREF))] + impl ParamSpecArgs { + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of ParamSpecArgs")) + } + + #[pygetset] + fn __origin__(&self) -> PyObjectRef { + self.__origin__.clone() + } + } + + impl Constructor for ParamSpecArgs { + type Args = (PyObjectRef,); + + fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + let origin = args.0; + Ok(Self { __origin__: origin }) + } + } + + impl Representable for ParamSpecArgs { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + // Check if origin is a ParamSpec + if let Ok(name) = zelf.__origin__.get_attr("__name__", vm) { + return Ok(format!("{name}.args", name = name.str(vm)?)); + } + Ok(format!("{:?}.args", zelf.__origin__)) + } + } + + impl Comparable for ParamSpecArgs { + fn cmp( + zelf: &crate::Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + if other.class().is(zelf.class()) + && let Some(other_args) = other.downcast_ref::<ParamSpecArgs>() + { + let eq = zelf.__origin__.rich_compare_bool( + &other_args.__origin__, + PyComparisonOp::Eq, + vm, + )?; + return Ok(PyComparisonValue::Implemented(eq)); + } + Ok(PyComparisonValue::NotImplemented) + }) + } + } + + #[pyattr] + #[pyclass(name = "ParamSpecKwargs", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct ParamSpecKwargs { + __origin__: PyObjectRef, + } + #[pyclass(with(Constructor, Representable, Comparable), flags(HAS_WEAKREF))] + impl ParamSpecKwargs { + #[pymethod] + fn __mro_entries__(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot subclass an instance of ParamSpecKwargs")) + } + + #[pygetset] + fn __origin__(&self) -> PyObjectRef { + self.__origin__.clone() + } + } + + impl Constructor for ParamSpecKwargs { + type Args = (PyObjectRef,); + + fn py_new(_cls: &Py<PyType>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> { + let origin = args.0; + Ok(Self { __origin__: origin }) + } + } + + impl Representable for ParamSpecKwargs { + #[inline(always)] + fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { + // Check if origin is a ParamSpec + if let Ok(name) = zelf.__origin__.get_attr("__name__", vm) { + return Ok(format!("{name}.kwargs", name = name.str(vm)?)); + } + Ok(format!("{:?}.kwargs", zelf.__origin__)) + } + } + + impl Comparable for ParamSpecKwargs { + fn cmp( + zelf: &crate::Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue> { + op.eq_only(|| { + if other.class().is(zelf.class()) + && let Some(other_kwargs) = other.downcast_ref::<ParamSpecKwargs>() + { + let eq = zelf.__origin__.rich_compare_bool( + &other_kwargs.__origin__, + PyComparisonOp::Eq, + vm, + )?; + return Ok(PyComparisonValue::Implemented(eq)); + } + Ok(PyComparisonValue::NotImplemented) + }) + } + } + + /// Helper function to call typing module functions with cls as first argument + /// Similar to CPython's call_typing_args_kwargs + fn call_typing_args_kwargs( + name: &'static str, + cls: PyTypeRef, + args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult { + let typing = vm.import("typing", 0)?; + let func = typing.get_attr(name, vm)?; + + // Prepare arguments: (cls, *args) + let mut call_args = vec![cls.into()]; + call_args.extend(args.args); + + // Call with prepared args and original kwargs + let func_args = FuncArgs { + args: call_args, + kwargs: args.kwargs, + }; + + func.call(func_args, vm) + } + + #[pyattr] + #[pyclass(name = "Generic", module = "typing")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub struct Generic; + + #[pyclass(flags(BASETYPE, HEAPTYPE))] + impl Generic { + #[pyattr] + fn __slots__(ctx: &Context) -> PyTupleRef { + ctx.empty_tuple.clone() + } + + #[pyclassmethod] + fn __class_getitem__(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + call_typing_args_kwargs("_generic_class_getitem", cls, args, vm) + } + + #[pyclassmethod] + fn __init_subclass__(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + call_typing_args_kwargs("_generic_init_subclass", cls, args, vm) + } + } + + /// Sets the default value for a type parameter, equivalent to CPython's _Py_set_typeparam_default + /// This is used by the CALL_INTRINSIC_2 SetTypeparamDefault instruction + pub fn set_typeparam_default( + type_param: PyObjectRef, + evaluate_default: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // Inner function to handle common pattern of setting evaluate_default + fn try_set_default<T>( + obj: &PyObject, + evaluate_default: &PyObject, + get_field: impl FnOnce(&T) -> &PyMutex<PyObjectRef>, + ) -> bool + where + T: PyPayload, + { + if let Some(typed_obj) = obj.downcast_ref::<T>() { + *get_field(typed_obj).lock() = evaluate_default.to_owned(); + true + } else { + false + } + } + + // Try each type parameter type + if try_set_default::<TypeVar>(&type_param, &evaluate_default, |tv| &tv.evaluate_default) + || try_set_default::<ParamSpec>(&type_param, &evaluate_default, |ps| { + &ps.evaluate_default + }) + || try_set_default::<TypeVarTuple>(&type_param, &evaluate_default, |tvt| { + &tvt.evaluate_default + }) + { + Ok(type_param) + } else { + Err(vm.new_type_error(format!( + "Expected a type param, got {}", + type_param.class().name() + ))) + } + } +} diff --git a/crates/vm/src/stdlib/winreg.rs b/crates/vm/src/stdlib/winreg.rs new file mode 100644 index 00000000000..264d14327da --- /dev/null +++ b/crates/vm/src/stdlib/winreg.rs @@ -0,0 +1,1129 @@ +// spell-checker:disable +#![allow(non_snake_case)] + +pub(crate) use winreg::module_def; + +#[pymodule] +mod winreg { + use crate::builtins::{PyInt, PyStr, PyTuple, PyTypeRef}; + use crate::common::hash::PyHash; + use crate::common::windows::ToWideString; + use crate::convert::TryFromObject; + use crate::function::FuncArgs; + use crate::object::AsObject; + use crate::protocol::PyNumberMethods; + use crate::types::{AsNumber, Hashable}; + use crate::{Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; + use core::ptr; + use crossbeam_utils::atomic::AtomicCell; + use malachite_bigint::Sign; + use num_traits::ToPrimitive; + use windows_sys::Win32::Foundation::{self, ERROR_MORE_DATA}; + use windows_sys::Win32::System::Registry; + + /// Atomic HKEY handle type for lock-free thread-safe access + type AtomicHKEY = AtomicCell<Registry::HKEY>; + + /// Convert byte slice to UTF-16 slice (zero-copy when aligned) + fn bytes_as_wide_slice(bytes: &[u8]) -> &[u16] { + // SAFETY: Windows Registry API returns properly aligned UTF-16 data. + // align_to handles any edge cases safely by returning empty prefix/suffix + // if alignment doesn't match. + let (prefix, u16_slice, suffix) = unsafe { bytes.align_to::<u16>() }; + debug_assert!( + prefix.is_empty() && suffix.is_empty(), + "Registry data should be u16-aligned" + ); + u16_slice + } + + fn os_error_from_windows_code( + vm: &VirtualMachine, + code: i32, + ) -> crate::PyRef<crate::builtins::PyBaseException> { + use crate::convert::ToPyException; + std::io::Error::from_raw_os_error(code).to_pyexception(vm) + } + + /// Wrapper type for HKEY that can be created from PyHkey or int + struct HKEYArg(Registry::HKEY); + + impl TryFromObject for HKEYArg { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + // Try PyHkey first + if let Some(hkey_obj) = obj.downcast_ref::<PyHkey>() { + return Ok(HKEYArg(hkey_obj.hkey.load())); + } + // Then try int + let handle = usize::try_from_object(vm, obj)?; + Ok(HKEYArg(handle as Registry::HKEY)) + } + } + + // access rights + #[pyattr] + pub use windows_sys::Win32::System::Registry::{ + KEY_ALL_ACCESS, KEY_CREATE_LINK, KEY_CREATE_SUB_KEY, KEY_ENUMERATE_SUB_KEYS, KEY_EXECUTE, + KEY_NOTIFY, KEY_QUERY_VALUE, KEY_READ, KEY_SET_VALUE, KEY_WOW64_32KEY, KEY_WOW64_64KEY, + KEY_WRITE, + }; + // value types + #[pyattr] + pub use windows_sys::Win32::System::Registry::{ + REG_BINARY, REG_CREATED_NEW_KEY, REG_DWORD, REG_DWORD_BIG_ENDIAN, REG_DWORD_LITTLE_ENDIAN, + REG_EXPAND_SZ, REG_FULL_RESOURCE_DESCRIPTOR, REG_LINK, REG_MULTI_SZ, REG_NONE, + REG_NOTIFY_CHANGE_ATTRIBUTES, REG_NOTIFY_CHANGE_LAST_SET, REG_NOTIFY_CHANGE_NAME, + REG_NOTIFY_CHANGE_SECURITY, REG_OPENED_EXISTING_KEY, REG_OPTION_BACKUP_RESTORE, + REG_OPTION_CREATE_LINK, REG_OPTION_NON_VOLATILE, REG_OPTION_OPEN_LINK, REG_OPTION_RESERVED, + REG_OPTION_VOLATILE, REG_QWORD, REG_QWORD_LITTLE_ENDIAN, REG_RESOURCE_LIST, + REG_RESOURCE_REQUIREMENTS_LIST, REG_SZ, REG_WHOLE_HIVE_VOLATILE, + }; + + // Additional constants not in windows-sys + #[pyattr] + const REG_REFRESH_HIVE: u32 = 0x00000002; + #[pyattr] + const REG_NO_LAZY_FLUSH: u32 = 0x00000004; + // REG_LEGAL_OPTION is a mask of all option flags + #[pyattr] + const REG_LEGAL_OPTION: u32 = Registry::REG_OPTION_RESERVED + | Registry::REG_OPTION_NON_VOLATILE + | Registry::REG_OPTION_VOLATILE + | Registry::REG_OPTION_CREATE_LINK + | Registry::REG_OPTION_BACKUP_RESTORE + | Registry::REG_OPTION_OPEN_LINK; + // REG_LEGAL_CHANGE_FILTER is a mask of all notify flags + #[pyattr] + const REG_LEGAL_CHANGE_FILTER: u32 = Registry::REG_NOTIFY_CHANGE_NAME + | Registry::REG_NOTIFY_CHANGE_ATTRIBUTES + | Registry::REG_NOTIFY_CHANGE_LAST_SET + | Registry::REG_NOTIFY_CHANGE_SECURITY; + + // error is an alias for OSError (for backwards compatibility) + #[pyattr] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.exceptions.os_error.to_owned() + } + + #[pyattr(once)] + fn HKEY_CLASSES_ROOT(vm: &VirtualMachine) -> PyRef<PyHkey> { + PyHkey::new(Registry::HKEY_CLASSES_ROOT).into_ref(&vm.ctx) + } + + #[pyattr(once)] + fn HKEY_CURRENT_USER(vm: &VirtualMachine) -> PyRef<PyHkey> { + PyHkey::new(Registry::HKEY_CURRENT_USER).into_ref(&vm.ctx) + } + + #[pyattr(once)] + fn HKEY_LOCAL_MACHINE(vm: &VirtualMachine) -> PyRef<PyHkey> { + PyHkey::new(Registry::HKEY_LOCAL_MACHINE).into_ref(&vm.ctx) + } + + #[pyattr(once)] + fn HKEY_USERS(vm: &VirtualMachine) -> PyRef<PyHkey> { + PyHkey::new(Registry::HKEY_USERS).into_ref(&vm.ctx) + } + + #[pyattr(once)] + fn HKEY_PERFORMANCE_DATA(vm: &VirtualMachine) -> PyRef<PyHkey> { + PyHkey::new(Registry::HKEY_PERFORMANCE_DATA).into_ref(&vm.ctx) + } + + #[pyattr(once)] + fn HKEY_CURRENT_CONFIG(vm: &VirtualMachine) -> PyRef<PyHkey> { + PyHkey::new(Registry::HKEY_CURRENT_CONFIG).into_ref(&vm.ctx) + } + + #[pyattr(once)] + fn HKEY_DYN_DATA(vm: &VirtualMachine) -> PyRef<PyHkey> { + PyHkey::new(Registry::HKEY_DYN_DATA).into_ref(&vm.ctx) + } + + #[pyattr] + #[pyclass(name = "HKEYType")] + #[derive(Debug, PyPayload)] + struct PyHkey { + hkey: AtomicHKEY, + } + + unsafe impl Send for PyHkey {} + unsafe impl Sync for PyHkey {} + + impl PyHkey { + fn new(hkey: Registry::HKEY) -> Self { + Self { + hkey: AtomicHKEY::new(hkey), + } + } + + fn unary_fail(vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(HKEY_ERR_MSG.to_owned())) + } + + fn binary_fail(vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(HKEY_ERR_MSG.to_owned())) + } + + fn ternary_fail(vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(HKEY_ERR_MSG.to_owned())) + } + } + + #[pyclass(with(AsNumber, Hashable))] + impl PyHkey { + #[pygetset] + fn handle(&self) -> usize { + self.hkey.load() as usize + } + + #[pymethod] + fn Close(&self, vm: &VirtualMachine) -> PyResult<()> { + // Atomically swap the handle with null and get the old value + let old_hkey = self.hkey.swap(core::ptr::null_mut()); + // Already closed - silently succeed + if old_hkey.is_null() { + return Ok(()); + } + let res = unsafe { Registry::RegCloseKey(old_hkey) }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("RegCloseKey failed with error code: {res}"))) + } + } + + #[pymethod] + fn Detach(&self) -> PyResult<usize> { + // Atomically swap the handle with null and return the old value + let old_hkey = self.hkey.swap(core::ptr::null_mut()); + Ok(old_hkey as usize) + } + + #[pymethod] + fn __enter__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyRef<Self>> { + Ok(zelf) + } + + #[pymethod] + fn __exit__(zelf: PyRef<Self>, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + zelf.Close(vm) + } + + fn __int__(&self) -> usize { + self.hkey.load() as usize + } + + #[pymethod] + fn __str__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + Ok(vm.ctx.new_str(format!("<PyHkey:{:p}>", zelf.hkey.load()))) + } + } + + impl Drop for PyHkey { + fn drop(&mut self) { + let hkey = self.hkey.swap(core::ptr::null_mut()); + if !hkey.is_null() { + unsafe { Registry::RegCloseKey(hkey) }; + } + } + } + + impl Hashable for PyHkey { + // CPython uses PyObject_GenericHash which hashes the object's address + fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyHash> { + Ok(zelf.get_id() as PyHash) + } + } + + pub const HKEY_ERR_MSG: &str = "bad operand type"; + + impl AsNumber for PyHkey { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + add: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + subtract: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + multiply: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + remainder: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + divmod: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + power: Some(|_a, _b, _c, vm| PyHkey::ternary_fail(vm)), + negative: Some(|_a, vm| PyHkey::unary_fail(vm)), + positive: Some(|_a, vm| PyHkey::unary_fail(vm)), + absolute: Some(|_a, vm| PyHkey::unary_fail(vm)), + boolean: Some(|a, _vm| { + let zelf = a.obj.downcast_ref::<PyHkey>().unwrap(); + Ok(!zelf.hkey.load().is_null()) + }), + invert: Some(|_a, vm| PyHkey::unary_fail(vm)), + lshift: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + rshift: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + and: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + xor: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + or: Some(|_a, _b, vm| PyHkey::binary_fail(vm)), + int: Some(|a, vm| { + if let Some(a) = a.downcast_ref::<PyHkey>() { + Ok(vm.new_pyobj(a.__int__())) + } else { + PyHkey::unary_fail(vm)?; + unreachable!() + } + }), + float: Some(|_a, vm| PyHkey::unary_fail(vm)), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + + #[pyfunction] + fn ConnectRegistry( + computer_name: Option<String>, + key: PyRef<PyHkey>, + vm: &VirtualMachine, + ) -> PyResult<PyHkey> { + if let Some(computer_name) = computer_name { + let mut ret_key = core::ptr::null_mut(); + let wide_computer_name = computer_name.to_wide_with_nul(); + let res = unsafe { + Registry::RegConnectRegistryW( + wide_computer_name.as_ptr(), + key.hkey.load(), + &mut ret_key, + ) + }; + if res == 0 { + Ok(PyHkey::new(ret_key)) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } else { + let mut ret_key = core::ptr::null_mut(); + let res = unsafe { + Registry::RegConnectRegistryW(core::ptr::null_mut(), key.hkey.load(), &mut ret_key) + }; + if res == 0 { + Ok(PyHkey::new(ret_key)) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + } + + #[pyfunction] + fn CreateKey(key: PyRef<PyHkey>, sub_key: String, vm: &VirtualMachine) -> PyResult<PyHkey> { + let wide_sub_key = sub_key.to_wide_with_nul(); + let mut out_key = core::ptr::null_mut(); + let res = unsafe { + Registry::RegCreateKeyW(key.hkey.load(), wide_sub_key.as_ptr(), &mut out_key) + }; + if res == 0 { + Ok(PyHkey::new(out_key)) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[derive(FromArgs, Debug)] + struct CreateKeyExArgs { + #[pyarg(any)] + key: PyRef<PyHkey>, + #[pyarg(any)] + sub_key: String, + #[pyarg(any, default = 0)] + reserved: u32, + #[pyarg(any, default = windows_sys::Win32::System::Registry::KEY_WRITE)] + access: u32, + } + + #[pyfunction] + fn CreateKeyEx(args: CreateKeyExArgs, vm: &VirtualMachine) -> PyResult<PyHkey> { + let wide_sub_key = args.sub_key.to_wide_with_nul(); + let mut res: Registry::HKEY = core::ptr::null_mut(); + let err = unsafe { + let key = args.key.hkey.load(); + Registry::RegCreateKeyExW( + key, + wide_sub_key.as_ptr(), + args.reserved, + core::ptr::null(), + Registry::REG_OPTION_NON_VOLATILE, + args.access, + core::ptr::null(), + &mut res, + core::ptr::null_mut(), + ) + }; + if err == 0 { + Ok(PyHkey { + #[allow(clippy::arc_with_non_send_sync)] + hkey: AtomicHKEY::new(res), + }) + } else { + Err(vm.new_os_error(format!("error code: {err}"))) + } + } + + #[pyfunction] + fn CloseKey(hkey: PyRef<PyHkey>, vm: &VirtualMachine) -> PyResult<()> { + hkey.Close(vm) + } + + #[pyfunction] + fn DeleteKey(key: PyRef<PyHkey>, sub_key: String, vm: &VirtualMachine) -> PyResult<()> { + let wide_sub_key = sub_key.to_wide_with_nul(); + let res = unsafe { Registry::RegDeleteKeyW(key.hkey.load(), wide_sub_key.as_ptr()) }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[pyfunction] + fn DeleteValue(key: PyRef<PyHkey>, value: Option<String>, vm: &VirtualMachine) -> PyResult<()> { + let wide_value = value.map(|v| v.to_wide_with_nul()); + let value_ptr = wide_value + .as_ref() + .map_or(core::ptr::null(), |v| v.as_ptr()); + let res = unsafe { Registry::RegDeleteValueW(key.hkey.load(), value_ptr) }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[derive(FromArgs, Debug)] + struct DeleteKeyExArgs { + #[pyarg(any)] + key: PyRef<PyHkey>, + #[pyarg(any)] + sub_key: String, + #[pyarg(any, default = windows_sys::Win32::System::Registry::KEY_WOW64_64KEY)] + access: u32, + #[pyarg(any, default = 0)] + reserved: u32, + } + + #[pyfunction] + fn DeleteKeyEx(args: DeleteKeyExArgs, vm: &VirtualMachine) -> PyResult<()> { + let wide_sub_key = args.sub_key.to_wide_with_nul(); + let res = unsafe { + Registry::RegDeleteKeyExW( + args.key.hkey.load(), + wide_sub_key.as_ptr(), + args.access, + args.reserved, + ) + }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[pyfunction] + fn EnumKey(key: PyRef<PyHkey>, index: i32, vm: &VirtualMachine) -> PyResult<String> { + // The Windows docs claim that the max key name length is 255 + // characters, plus a terminating nul character. However, + // empirical testing demonstrates that it is possible to + // create a 256 character key that is missing the terminating + // nul. RegEnumKeyEx requires a 257 character buffer to + // retrieve such a key name. + let mut tmpbuf = [0u16; 257]; + let mut len = tmpbuf.len() as u32; + let res = unsafe { + Registry::RegEnumKeyExW( + key.hkey.load(), + index as u32, + tmpbuf.as_mut_ptr(), + &mut len, + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + ) + }; + if res != 0 { + return Err(os_error_from_windows_code(vm, res as i32)); + } + String::from_utf16(&tmpbuf[..len as usize]) + .map_err(|e| vm.new_value_error(format!("UTF16 error: {e}"))) + } + + #[pyfunction] + fn EnumValue(hkey: PyRef<PyHkey>, index: u32, vm: &VirtualMachine) -> PyResult { + // Query registry for the required buffer sizes. + let mut ret_value_size: u32 = 0; + let mut ret_data_size: u32 = 0; + let hkey: Registry::HKEY = hkey.hkey.load(); + let rc = unsafe { + Registry::RegQueryInfoKeyW( + hkey, + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + &mut ret_value_size as *mut u32, + &mut ret_data_size as *mut u32, + ptr::null_mut(), + ptr::null_mut(), + ) + }; + if rc != 0 { + return Err(vm.new_os_error(format!("RegQueryInfoKeyW failed with error code {rc}"))); + } + + // Include room for null terminators. + ret_value_size += 1; + ret_data_size += 1; + let mut buf_value_size = ret_value_size; + let mut buf_data_size = ret_data_size; + + // Allocate buffers. + let mut ret_value_buf: Vec<u16> = vec![0; ret_value_size as usize]; + let mut ret_data_buf: Vec<u8> = vec![0; ret_data_size as usize]; + + // Loop to enumerate the registry value. + loop { + let mut current_value_size = ret_value_size; + let mut current_data_size = ret_data_size; + let mut reg_type: u32 = 0; + let rc = unsafe { + Registry::RegEnumValueW( + hkey, + index, + ret_value_buf.as_mut_ptr(), + &mut current_value_size as *mut u32, + ptr::null_mut(), + &mut reg_type as *mut u32, + ret_data_buf.as_mut_ptr(), + &mut current_data_size as *mut u32, + ) + }; + if rc == ERROR_MORE_DATA { + // Double the buffer sizes. + buf_data_size *= 2; + buf_value_size *= 2; + ret_data_buf.resize(buf_data_size as usize, 0); + ret_value_buf.resize(buf_value_size as usize, 0); + // Reset sizes for next iteration. + ret_value_size = buf_value_size; + ret_data_size = buf_data_size; + continue; + } + if rc != 0 { + return Err(vm.new_os_error(format!("RegEnumValueW failed with error code {rc}"))); + } + + // Convert the registry value name from UTF‑16. + let name_len = ret_value_buf + .iter() + .position(|&c| c == 0) + .unwrap_or(ret_value_buf.len()); + let name = String::from_utf16(&ret_value_buf[..name_len]) + .map_err(|e| vm.new_value_error(format!("UTF16 conversion error: {e}")))?; + + // Slice the data buffer to the actual size returned. + let data_slice = &ret_data_buf[..current_data_size as usize]; + let py_data = reg_to_py(vm, data_slice, reg_type)?; + + // Return tuple (value_name, data, type) + return Ok(vm + .ctx + .new_tuple(vec![ + vm.ctx.new_str(name).into(), + py_data, + vm.ctx.new_int(reg_type).into(), + ]) + .into()); + } + } + + #[pyfunction] + fn FlushKey(key: PyRef<PyHkey>, vm: &VirtualMachine) -> PyResult<()> { + let res = unsafe { Registry::RegFlushKey(key.hkey.load()) }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[pyfunction] + fn LoadKey( + key: PyRef<PyHkey>, + sub_key: String, + file_name: String, + vm: &VirtualMachine, + ) -> PyResult<()> { + let sub_key = sub_key.to_wide_with_nul(); + let file_name = file_name.to_wide_with_nul(); + let res = + unsafe { Registry::RegLoadKeyW(key.hkey.load(), sub_key.as_ptr(), file_name.as_ptr()) }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[derive(Debug, FromArgs)] + struct OpenKeyArgs { + #[pyarg(any)] + key: PyRef<PyHkey>, + #[pyarg(any)] + sub_key: String, + #[pyarg(any, default = 0)] + reserved: u32, + #[pyarg(any, default = windows_sys::Win32::System::Registry::KEY_READ)] + access: u32, + } + + #[pyfunction] + #[pyfunction(name = "OpenKeyEx")] + fn OpenKey(args: OpenKeyArgs, vm: &VirtualMachine) -> PyResult<PyHkey> { + let wide_sub_key = args.sub_key.to_wide_with_nul(); + let mut res: Registry::HKEY = core::ptr::null_mut(); + let err = unsafe { + let key = args.key.hkey.load(); + Registry::RegOpenKeyExW( + key, + wide_sub_key.as_ptr(), + args.reserved, + args.access, + &mut res, + ) + }; + if err == 0 { + Ok(PyHkey { + #[allow(clippy::arc_with_non_send_sync)] + hkey: AtomicHKEY::new(res), + }) + } else { + Err(os_error_from_windows_code(vm, err as i32)) + } + } + + #[pyfunction] + fn QueryInfoKey(key: HKEYArg, vm: &VirtualMachine) -> PyResult<PyRef<PyTuple>> { + let key = key.0; + let mut lpcsubkeys: u32 = 0; + let mut lpcvalues: u32 = 0; + let mut lpftlastwritetime: Foundation::FILETIME = unsafe { core::mem::zeroed() }; + let err = unsafe { + Registry::RegQueryInfoKeyW( + key, + core::ptr::null_mut(), + core::ptr::null_mut(), + 0 as _, + &mut lpcsubkeys, + core::ptr::null_mut(), + core::ptr::null_mut(), + &mut lpcvalues, + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + &mut lpftlastwritetime, + ) + }; + + if err != 0 { + return Err(vm.new_os_error(format!("error code: {err}"))); + } + let l: u64 = (lpftlastwritetime.dwHighDateTime as u64) << 32 + | lpftlastwritetime.dwLowDateTime as u64; + let tup: Vec<PyObjectRef> = vec![ + vm.ctx.new_int(lpcsubkeys).into(), + vm.ctx.new_int(lpcvalues).into(), + vm.ctx.new_int(l).into(), + ]; + Ok(vm.ctx.new_tuple(tup)) + } + + #[pyfunction] + fn QueryValue(key: HKEYArg, sub_key: Option<String>, vm: &VirtualMachine) -> PyResult<String> { + let hkey = key.0; + + if hkey == Registry::HKEY_PERFORMANCE_DATA { + return Err(os_error_from_windows_code( + vm, + Foundation::ERROR_INVALID_HANDLE as i32, + )); + } + + // Open subkey if provided and non-empty + let child_key = if let Some(ref sk) = sub_key { + if !sk.is_empty() { + let wide_sub_key = sk.to_wide_with_nul(); + let mut out_key = core::ptr::null_mut(); + let res = unsafe { + Registry::RegOpenKeyExW( + hkey, + wide_sub_key.as_ptr(), + 0, + Registry::KEY_QUERY_VALUE, + &mut out_key, + ) + }; + if res != 0 { + return Err(os_error_from_windows_code(vm, res as i32)); + } + Some(out_key) + } else { + None + } + } else { + None + }; + + let target_key = child_key.unwrap_or(hkey); + let mut buf_size: u32 = 256; + let mut buffer: Vec<u8> = vec![0; buf_size as usize]; + let mut reg_type: u32 = 0; + + // Loop to handle ERROR_MORE_DATA + let result = loop { + let mut size = buf_size; + let res = unsafe { + Registry::RegQueryValueExW( + target_key, + core::ptr::null(), // NULL value name for default value + core::ptr::null_mut(), + &mut reg_type, + buffer.as_mut_ptr(), + &mut size, + ) + }; + if res == ERROR_MORE_DATA { + buf_size *= 2; + buffer.resize(buf_size as usize, 0); + continue; + } + if res == Foundation::ERROR_FILE_NOT_FOUND { + // Return empty string if there's no default value + break Ok(String::new()); + } + if res != 0 { + break Err(os_error_from_windows_code(vm, res as i32)); + } + if reg_type != Registry::REG_SZ { + break Err(os_error_from_windows_code( + vm, + Foundation::ERROR_INVALID_DATA as i32, + )); + } + + // Convert UTF-16 to String + let u16_slice = bytes_as_wide_slice(&buffer[..size as usize]); + let len = u16_slice + .iter() + .position(|&c| c == 0) + .unwrap_or(u16_slice.len()); + break String::from_utf16(&u16_slice[..len]) + .map_err(|e| vm.new_value_error(format!("UTF16 error: {e}"))); + }; + + // Close child key if we opened one + if let Some(ck) = child_key { + unsafe { Registry::RegCloseKey(ck) }; + } + + result + } + + #[pyfunction] + fn QueryValueEx(key: HKEYArg, name: String, vm: &VirtualMachine) -> PyResult<PyRef<PyTuple>> { + let hkey = key.0; + let wide_name = name.to_wide_with_nul(); + let mut buf_size: u32 = 0; + let res = unsafe { + Registry::RegQueryValueExW( + hkey, + wide_name.as_ptr(), + core::ptr::null_mut(), + core::ptr::null_mut(), + core::ptr::null_mut(), + &mut buf_size, + ) + }; + // Handle ERROR_MORE_DATA by using a default buffer size + if res == ERROR_MORE_DATA || buf_size == 0 { + buf_size = 256; + } else if res != 0 { + return Err(os_error_from_windows_code(vm, res as i32)); + } + + let mut ret_buf = vec![0u8; buf_size as usize]; + let mut typ = 0; + let mut ret_size: u32; + + // Loop to handle ERROR_MORE_DATA + loop { + ret_size = buf_size; + let res = unsafe { + Registry::RegQueryValueExW( + hkey, + wide_name.as_ptr(), + core::ptr::null_mut(), + &mut typ, + ret_buf.as_mut_ptr(), + &mut ret_size, + ) + }; + + if res != ERROR_MORE_DATA { + if res != 0 { + return Err(os_error_from_windows_code(vm, res as i32)); + } + break; + } + + // Double buffer size and retry + buf_size *= 2; + ret_buf.resize(buf_size as usize, 0); + } + + // Only pass the bytes actually returned by the API + let obj = reg_to_py(vm, &ret_buf[..ret_size as usize], typ)?; + // Return tuple (value, type) + Ok(vm.ctx.new_tuple(vec![obj, vm.ctx.new_int(typ).into()])) + } + + #[pyfunction] + fn SaveKey(key: PyRef<PyHkey>, file_name: String, vm: &VirtualMachine) -> PyResult<()> { + let file_name = file_name.to_wide_with_nul(); + let res = unsafe { + Registry::RegSaveKeyW(key.hkey.load(), file_name.as_ptr(), core::ptr::null_mut()) + }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[pyfunction] + fn SetValue( + key: PyRef<PyHkey>, + sub_key: String, + typ: u32, + value: String, + vm: &VirtualMachine, + ) -> PyResult<()> { + if typ != Registry::REG_SZ { + return Err(vm.new_type_error("type must be winreg.REG_SZ")); + } + + let hkey = key.hkey.load(); + if hkey == Registry::HKEY_PERFORMANCE_DATA { + return Err(os_error_from_windows_code( + vm, + Foundation::ERROR_INVALID_HANDLE as i32, + )); + } + + // Create subkey if sub_key is non-empty + let child_key = if !sub_key.is_empty() { + let wide_sub_key = sub_key.to_wide_with_nul(); + let mut out_key = core::ptr::null_mut(); + let res = unsafe { + Registry::RegCreateKeyExW( + hkey, + wide_sub_key.as_ptr(), + 0, + core::ptr::null(), + 0, + Registry::KEY_SET_VALUE, + core::ptr::null(), + &mut out_key, + core::ptr::null_mut(), + ) + }; + if res != 0 { + return Err(os_error_from_windows_code(vm, res as i32)); + } + Some(out_key) + } else { + None + }; + + let target_key = child_key.unwrap_or(hkey); + // Convert value to UTF-16 for Wide API + let wide_value = value.to_wide_with_nul(); + let res = unsafe { + Registry::RegSetValueExW( + target_key, + core::ptr::null(), // value name is NULL + 0, + typ, + wide_value.as_ptr() as *const u8, + (wide_value.len() * 2) as u32, // byte count + ) + }; + + // Close child key if we created one + if let Some(ck) = child_key { + unsafe { Registry::RegCloseKey(ck) }; + } + + if res == 0 { + Ok(()) + } else { + Err(os_error_from_windows_code(vm, res as i32)) + } + } + + fn reg_to_py(vm: &VirtualMachine, ret_data: &[u8], typ: u32) -> PyResult { + match typ { + REG_DWORD => { + // If there isn’t enough data, return 0. + let val = ret_data + .first_chunk::<4>() + .copied() + .map_or(0, u32::from_ne_bytes); + Ok(vm.ctx.new_int(val).into()) + } + REG_QWORD => { + let val = ret_data + .first_chunk::<8>() + .copied() + .map_or(0, u64::from_ne_bytes); + Ok(vm.ctx.new_int(val).into()) + } + REG_SZ | REG_EXPAND_SZ => { + let u16_slice = bytes_as_wide_slice(ret_data); + // Only use characters up to the first NUL. + let len = u16_slice + .iter() + .position(|&c| c == 0) + .unwrap_or(u16_slice.len()); + let s = String::from_utf16(&u16_slice[..len]) + .map_err(|e| vm.new_value_error(format!("UTF16 error: {e}")))?; + Ok(vm.ctx.new_str(s).into()) + } + REG_MULTI_SZ => { + if ret_data.is_empty() { + Ok(vm.ctx.new_list(vec![]).into()) + } else { + let u16_slice = bytes_as_wide_slice(ret_data); + let u16_count = u16_slice.len(); + + // Remove trailing null if present (like countStrings) + let len = if u16_count > 0 && u16_slice[u16_count - 1] == 0 { + u16_count - 1 + } else { + u16_count + }; + + let mut strings: Vec<PyObjectRef> = Vec::new(); + let mut start = 0; + for i in 0..len { + if u16_slice[i] == 0 { + let s = String::from_utf16(&u16_slice[start..i]) + .map_err(|e| vm.new_value_error(format!("UTF16 error: {e}")))?; + strings.push(vm.ctx.new_str(s).into()); + start = i + 1; + } + } + // Handle last string if not null-terminated + if start < len { + let s = String::from_utf16(&u16_slice[start..len]) + .map_err(|e| vm.new_value_error(format!("UTF16 error: {e}")))?; + strings.push(vm.ctx.new_str(s).into()); + } + Ok(vm.ctx.new_list(strings).into()) + } + } + // For REG_BINARY and any other unknown types, return a bytes object if data exists. + _ => { + if ret_data.is_empty() { + Ok(vm.ctx.none()) + } else { + Ok(vm.ctx.new_bytes(ret_data.to_vec()).into()) + } + } + } + } + + fn py2reg(value: PyObjectRef, typ: u32, vm: &VirtualMachine) -> PyResult<Option<Vec<u8>>> { + match typ { + REG_DWORD => { + if vm.is_none(&value) { + return Ok(Some(0u32.to_le_bytes().to_vec())); + } + let val = value + .downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error("value must be an integer"))?; + let bigint = val.as_bigint(); + // Check for negative value - raise OverflowError + if bigint.sign() == Sign::Minus { + return Err(vm.new_overflow_error("int too big to convert")); + } + let val = bigint + .to_u32() + .ok_or_else(|| vm.new_overflow_error("int too big to convert"))?; + Ok(Some(val.to_le_bytes().to_vec())) + } + REG_QWORD => { + if vm.is_none(&value) { + return Ok(Some(0u64.to_le_bytes().to_vec())); + } + let val = value + .downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error("value must be an integer"))?; + let bigint = val.as_bigint(); + // Check for negative value - raise OverflowError + if bigint.sign() == Sign::Minus { + return Err(vm.new_overflow_error("int too big to convert")); + } + let val = bigint + .to_u64() + .ok_or_else(|| vm.new_overflow_error("int too big to convert"))?; + Ok(Some(val.to_le_bytes().to_vec())) + } + REG_SZ | REG_EXPAND_SZ => { + if vm.is_none(&value) { + // Return empty string as UTF-16 null terminator + return Ok(Some(vec![0u8, 0u8])); + } + let s = value + .downcast::<PyStr>() + .map_err(|_| vm.new_type_error("value must be a string"))?; + let wide = s.as_wtf8().to_wide_with_nul(); + // Convert Vec<u16> to Vec<u8> + let bytes: Vec<u8> = wide.iter().flat_map(|&c| c.to_le_bytes()).collect(); + Ok(Some(bytes)) + } + REG_MULTI_SZ => { + if vm.is_none(&value) { + // Empty list = double null terminator + return Ok(Some(vec![0u8, 0u8, 0u8, 0u8])); + } + let list = value + .downcast::<crate::builtins::PyList>() + .map_err(|_| vm.new_type_error("value must be a list of strings"))?; + + let mut bytes: Vec<u8> = Vec::new(); + for item in list.borrow_vec().iter() { + let s = item + .downcast_ref::<PyStr>() + .ok_or_else(|| vm.new_type_error("list items must be strings"))?; + let wide = s.as_wtf8().to_wide_with_nul(); + bytes.extend(wide.iter().flat_map(|&c| c.to_le_bytes())); + } + // Add final null terminator (double null at end) + bytes.extend([0u8, 0u8]); + Ok(Some(bytes)) + } + // REG_BINARY and other types + _ => { + if vm.is_none(&value) { + return Ok(None); + } + // Try to get bytes + if let Some(bytes) = value.downcast_ref::<crate::builtins::PyBytes>() { + return Ok(Some(bytes.as_bytes().to_vec())); + } + Err(vm.new_type_error(format!( + "Objects of type '{}' can not be used as binary registry values", + value.class().name() + ))) + } + } + } + + #[pyfunction] + fn SetValueEx( + key: PyRef<PyHkey>, + value_name: Option<String>, + _reserved: PyObjectRef, + typ: u32, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + let wide_value_name = value_name.as_deref().map(|s| s.to_wide_with_nul()); + let value_name_ptr = wide_value_name + .as_deref() + .map_or(core::ptr::null(), |s| s.as_ptr()); + let reg_value = py2reg(value, typ, vm)?; + let (ptr, len) = match &reg_value { + Some(v) => (v.as_ptr(), v.len() as u32), + None => (core::ptr::null(), 0), + }; + let res = + unsafe { Registry::RegSetValueExW(key.hkey.load(), value_name_ptr, 0, typ, ptr, len) }; + if res != 0 { + return Err(os_error_from_windows_code(vm, res as i32)); + } + Ok(()) + } + + #[pyfunction] + fn DisableReflectionKey(key: PyRef<PyHkey>, vm: &VirtualMachine) -> PyResult<()> { + let res = unsafe { Registry::RegDisableReflectionKey(key.hkey.load()) }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[pyfunction] + fn EnableReflectionKey(key: PyRef<PyHkey>, vm: &VirtualMachine) -> PyResult<()> { + let res = unsafe { Registry::RegEnableReflectionKey(key.hkey.load()) }; + if res == 0 { + Ok(()) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[pyfunction] + fn QueryReflectionKey(key: PyRef<PyHkey>, vm: &VirtualMachine) -> PyResult<bool> { + let mut result: i32 = 0; + let res = unsafe { Registry::RegQueryReflectionKey(key.hkey.load(), &mut result) }; + if res == 0 { + Ok(result != 0) + } else { + Err(vm.new_os_error(format!("error code: {res}"))) + } + } + + #[pyfunction] + fn ExpandEnvironmentStrings(i: String, vm: &VirtualMachine) -> PyResult<String> { + let wide_input = i.to_wide_with_nul(); + + // First call with size=0 to get required buffer size + let required_size = unsafe { + windows_sys::Win32::System::Environment::ExpandEnvironmentStringsW( + wide_input.as_ptr(), + core::ptr::null_mut(), + 0, + ) + }; + if required_size == 0 { + return Err(vm.new_os_error("ExpandEnvironmentStringsW failed".to_string())); + } + + // Allocate buffer with exact size and expand + let mut out = vec![0u16; required_size as usize]; + let r = unsafe { + windows_sys::Win32::System::Environment::ExpandEnvironmentStringsW( + wide_input.as_ptr(), + out.as_mut_ptr(), + required_size, + ) + }; + if r == 0 { + return Err(vm.new_os_error("ExpandEnvironmentStringsW failed".to_string())); + } + + let len = out.iter().position(|&c| c == 0).unwrap_or(out.len()); + String::from_utf16(&out[..len]).map_err(|e| vm.new_value_error(format!("UTF16 error: {e}"))) + } +} diff --git a/crates/vm/src/stdlib/winsound.rs b/crates/vm/src/stdlib/winsound.rs new file mode 100644 index 00000000000..0ca2e9a2258 --- /dev/null +++ b/crates/vm/src/stdlib/winsound.rs @@ -0,0 +1,202 @@ +// spell-checker:ignore pszSound fdwSound +#![allow(non_snake_case)] + +pub(crate) use winsound::module_def; + +mod win32 { + #[link(name = "winmm")] + unsafe extern "system" { + pub fn PlaySoundW(pszSound: *const u16, hmod: isize, fdwSound: u32) -> i32; + } + + unsafe extern "system" { + pub fn Beep(dwFreq: u32, dwDuration: u32) -> i32; + pub fn MessageBeep(uType: u32) -> i32; + } +} + +#[pymodule] +mod winsound { + use crate::builtins::{PyBytes, PyStr}; + use crate::common::windows::ToWideString; + use crate::convert::{IntoPyException, TryFromBorrowedObject}; + use crate::protocol::PyBuffer; + use crate::{AsObject, PyObjectRef, PyResult, VirtualMachine}; + + // PlaySound flags + #[pyattr] + const SND_SYNC: u32 = 0x0000; + #[pyattr] + const SND_ASYNC: u32 = 0x0001; + #[pyattr] + const SND_NODEFAULT: u32 = 0x0002; + #[pyattr] + const SND_MEMORY: u32 = 0x0004; + #[pyattr] + const SND_LOOP: u32 = 0x0008; + #[pyattr] + const SND_NOSTOP: u32 = 0x0010; + #[pyattr] + const SND_PURGE: u32 = 0x0040; + #[pyattr] + const SND_APPLICATION: u32 = 0x0080; + #[pyattr] + const SND_NOWAIT: u32 = 0x00002000; + #[pyattr] + const SND_ALIAS: u32 = 0x00010000; + #[pyattr] + const SND_FILENAME: u32 = 0x00020000; + #[pyattr] + const SND_SENTRY: u32 = 0x00080000; + #[pyattr] + const SND_SYSTEM: u32 = 0x00200000; + + // MessageBeep types + #[pyattr] + const MB_OK: u32 = 0x00000000; + #[pyattr] + const MB_ICONHAND: u32 = 0x00000010; + #[pyattr] + const MB_ICONQUESTION: u32 = 0x00000020; + #[pyattr] + const MB_ICONEXCLAMATION: u32 = 0x00000030; + #[pyattr] + const MB_ICONASTERISK: u32 = 0x00000040; + #[pyattr] + const MB_ICONERROR: u32 = MB_ICONHAND; + #[pyattr] + const MB_ICONSTOP: u32 = MB_ICONHAND; + #[pyattr] + const MB_ICONINFORMATION: u32 = MB_ICONASTERISK; + #[pyattr] + const MB_ICONWARNING: u32 = MB_ICONEXCLAMATION; + + #[derive(FromArgs)] + struct PlaySoundArgs { + #[pyarg(any)] + sound: PyObjectRef, + #[pyarg(any)] + flags: i32, + } + + #[pyfunction] + fn PlaySound(args: PlaySoundArgs, vm: &VirtualMachine) -> PyResult<()> { + let sound = args.sound; + let flags = args.flags as u32; + + if vm.is_none(&sound) { + let ok = unsafe { super::win32::PlaySoundW(core::ptr::null(), 0, flags) }; + if ok == 0 { + return Err(vm.new_runtime_error("Failed to play sound")); + } + return Ok(()); + } + + if flags & SND_MEMORY != 0 { + if flags & SND_ASYNC != 0 { + return Err(vm.new_runtime_error("Cannot play asynchronously from memory")); + } + let buffer = PyBuffer::try_from_borrowed_object(vm, &sound)?; + let buf = buffer + .as_contiguous() + .ok_or_else(|| vm.new_type_error("a bytes-like object is required, not 'str'"))?; + let ok = unsafe { super::win32::PlaySoundW(buf.as_ptr() as *const u16, 0, flags) }; + if ok == 0 { + return Err(vm.new_runtime_error("Failed to play sound")); + } + return Ok(()); + } + + if sound.downcastable::<PyBytes>() { + let type_name = sound.class().name().to_string(); + return Err(vm.new_type_error(format!( + "'sound' must be str, os.PathLike, or None, not {type_name}" + ))); + } + + // os.fspath(sound) + let path = match sound.downcast_ref::<PyStr>() { + Some(s) => s.as_wtf8().to_owned(), + None => { + let fspath = vm.get_method_or_type_error( + sound.clone(), + identifier!(vm, __fspath__), + || { + let type_name = sound.class().name().to_string(); + format!("'sound' must be str, os.PathLike, or None, not {type_name}") + }, + )?; + + if vm.is_none(&fspath) { + return Err(vm.new_type_error(format!( + "'sound' must be str, os.PathLike, or None, not {}", + sound.class().name() + ))); + } + let result = fspath.call((), vm)?; + + if result.downcastable::<PyBytes>() { + return Err(vm.new_type_error("'sound' must resolve to str, not bytes")); + } + + let s: &PyStr = result.downcast_ref().ok_or_else(|| { + vm.new_type_error(format!( + "expected {}.__fspath__() to return str or bytes, not {}", + sound.class().name(), + result.class().name() + )) + })?; + + s.as_wtf8().to_owned() + } + }; + + // Check for embedded null characters + if path.as_bytes().contains(&0) { + return Err(vm.new_value_error("embedded null character")); + } + + let wide = path.to_wide_with_nul(); + let ok = unsafe { super::win32::PlaySoundW(wide.as_ptr(), 0, flags) }; + if ok == 0 { + return Err(vm.new_runtime_error("Failed to play sound")); + } + Ok(()) + } + + #[derive(FromArgs)] + struct BeepArgs { + #[pyarg(any)] + frequency: i32, + #[pyarg(any)] + duration: i32, + } + + #[pyfunction] + fn Beep(args: BeepArgs, vm: &VirtualMachine) -> PyResult<()> { + if !(37..=32767).contains(&args.frequency) { + return Err(vm.new_value_error("frequency must be in 37 thru 32767")); + } + + let ok = unsafe { super::win32::Beep(args.frequency as u32, args.duration as u32) }; + if ok == 0 { + return Err(vm.new_runtime_error("Failed to beep")); + } + Ok(()) + } + + #[derive(FromArgs)] + struct MessageBeepArgs { + #[pyarg(any, default = 0)] + r#type: u32, + } + + #[pyfunction] + fn MessageBeep(args: MessageBeepArgs, vm: &VirtualMachine) -> PyResult<()> { + let ok = unsafe { super::win32::MessageBeep(args.r#type) }; + if ok == 0 { + return Err(std::io::Error::last_os_error().into_pyexception(vm)); + } + Ok(()) + } +} diff --git a/crates/vm/src/suggestion.rs b/crates/vm/src/suggestion.rs new file mode 100644 index 00000000000..b48b78af755 --- /dev/null +++ b/crates/vm/src/suggestion.rs @@ -0,0 +1,98 @@ +//! This module provides functionality to suggest similar names for attributes or variables. +//! This is used during tracebacks. + +use crate::{ + AsObject, Py, PyObject, PyObjectRef, VirtualMachine, + builtins::{PyStr, PyStrRef}, + exceptions::types::PyBaseException, + sliceable::SliceableSequenceOp, +}; +use core::iter::ExactSizeIterator; +use rustpython_common::str::levenshtein::{MOVE_COST, levenshtein_distance}; + +const MAX_CANDIDATE_ITEMS: usize = 750; + +pub fn calculate_suggestions<'a>( + dir_iter: impl ExactSizeIterator<Item = &'a PyObjectRef>, + name: &PyObject, +) -> Option<PyStrRef> { + if dir_iter.len() >= MAX_CANDIDATE_ITEMS { + return None; + } + + let mut suggestion: Option<&Py<PyStr>> = None; + let mut suggestion_distance = usize::MAX; + let name = name.downcast_ref::<PyStr>()?; + + for item in dir_iter { + let item_name = item.downcast_ref::<PyStr>()?; + if name.as_bytes() == item_name.as_bytes() { + continue; + } + // No more than 1/3 of the characters should need changed + let max_distance = usize::min( + (name.len() + item_name.len() + 3) * MOVE_COST / 6, + suggestion_distance - 1, + ); + let current_distance = + levenshtein_distance(name.as_bytes(), item_name.as_bytes(), max_distance); + if current_distance > max_distance { + continue; + } + if suggestion.is_none() || current_distance < suggestion_distance { + suggestion = Some(item_name); + suggestion_distance = current_distance; + } + } + suggestion.map(|r| r.to_owned()) +} + +pub fn offer_suggestions(exc: &Py<PyBaseException>, vm: &VirtualMachine) -> Option<PyStrRef> { + if exc + .class() + .fast_issubclass(vm.ctx.exceptions.attribute_error) + { + let name = exc.as_object().get_attr("name", vm).ok()?; + if vm.is_none(&name) { + return None; + } + let obj = exc.as_object().get_attr("obj", vm).ok()?; + if vm.is_none(&obj) { + return None; + } + + calculate_suggestions(vm.dir(Some(obj)).ok()?.borrow_vec().iter(), &name) + } else if exc.class().fast_issubclass(vm.ctx.exceptions.name_error) { + let name = exc.as_object().get_attr("name", vm).ok()?; + if vm.is_none(&name) { + return None; + } + let tb = exc.__traceback__()?; + let tb = tb.iter().last().unwrap_or(tb); + + let varnames = tb.frame.code.clone().co_varnames(vm); + if let Some(suggestions) = calculate_suggestions(varnames.iter(), &name) { + return Some(suggestions); + }; + + let globals: Vec<_> = tb.frame.globals.as_object().try_to_value(vm).ok()?; + if let Some(suggestions) = calculate_suggestions(globals.iter(), &name) { + return Some(suggestions); + }; + + let builtins: Vec<_> = tb.frame.builtins.try_to_value(vm).ok()?; + calculate_suggestions(builtins.iter(), &name) + } else if exc.class().fast_issubclass(vm.ctx.exceptions.import_error) { + let mod_name = exc.as_object().get_attr("name", vm).ok()?; + let wrong_name = exc.as_object().get_attr("name_from", vm).ok()?; + let mod_name_str = mod_name.downcast_ref::<PyStr>()?; + + // Look up the module in sys.modules + let sys_modules = vm.sys_module.get_attr("modules", vm).ok()?; + let module = sys_modules.get_item(mod_name_str, vm).ok()?; + + calculate_suggestions(vm.dir(Some(module)).ok()?.borrow_vec().iter(), &wrong_name) + } else { + None + } +} diff --git a/crates/vm/src/types/mod.rs b/crates/vm/src/types/mod.rs new file mode 100644 index 00000000000..b17a737545f --- /dev/null +++ b/crates/vm/src/types/mod.rs @@ -0,0 +1,9 @@ +mod slot; +pub mod slot_defs; +mod structseq; +mod zoo; + +pub use slot::*; +pub use slot_defs::{SLOT_DEFS, SlotAccessor, SlotDef}; +pub use structseq::{PyStructSequence, PyStructSequenceData, struct_sequence_new}; +pub(crate) use zoo::TypeZoo; diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs new file mode 100644 index 00000000000..8ffcfd0f3b6 --- /dev/null +++ b/crates/vm/src/types/slot.rs @@ -0,0 +1,2086 @@ +use crate::common::lock::{ + PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyRwLockReadGuard, PyRwLockWriteGuard, +}; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyInt, PyStr, PyStrInterned, PyType, PyTypeRef}, + bytecode::ComparisonOperator, + common::hash::{PyHash, fix_sentinel, hash_bigint}, + convert::ToPyObject, + function::{Either, FromArgs, FuncArgs, PyComparisonValue, PyMethodDef, PySetterValue}, + protocol::{ + PyBuffer, PyIterReturn, PyMapping, PyMappingMethods, PyMappingSlots, PyNumber, + PyNumberMethods, PyNumberSlots, PySequence, PySequenceMethods, PySequenceSlots, + }, + types::slot_defs::{SlotAccessor, find_slot_defs_by_name}, + vm::Context, +}; +use core::{any::Any, any::TypeId, borrow::Borrow, cmp::Ordering, ops::Deref}; +use crossbeam_utils::atomic::AtomicCell; +use num_traits::{Signed, ToPrimitive}; +use rustpython_common::wtf8::Wtf8Buf; + +/// Type-erased storage for extension module data attached to heap types. +pub struct TypeDataSlot { + // PyObject_GetTypeData + type_id: TypeId, + data: Box<dyn Any + Send + Sync>, +} + +impl TypeDataSlot { + /// Create a new type data slot with the given data. + pub fn new<T: Any + Send + Sync + 'static>(data: T) -> Self { + Self { + type_id: TypeId::of::<T>(), + data: Box::new(data), + } + } + + /// Get a reference to the data if the type matches. + pub fn get<T: Any + 'static>(&self) -> Option<&T> { + if self.type_id == TypeId::of::<T>() { + self.data.downcast_ref() + } else { + None + } + } + + /// Get a mutable reference to the data if the type matches. + pub fn get_mut<T: Any + 'static>(&mut self) -> Option<&mut T> { + if self.type_id == TypeId::of::<T>() { + self.data.downcast_mut() + } else { + None + } + } +} + +/// Read guard for type data access, using mapped guard for zero-cost deref. +pub struct TypeDataRef<'a, T: 'static> { + guard: PyMappedRwLockReadGuard<'a, T>, +} + +impl<'a, T: Any + 'static> TypeDataRef<'a, T> { + /// Try to create a TypeDataRef from a read guard. + /// Returns None if the slot is empty or contains a different type. + pub fn try_new(guard: PyRwLockReadGuard<'a, Option<TypeDataSlot>>) -> Option<Self> { + PyRwLockReadGuard::try_map(guard, |opt| opt.as_ref().and_then(|slot| slot.get::<T>())) + .ok() + .map(|guard| Self { guard }) + } +} + +impl<T: Any + 'static> core::ops::Deref for TypeDataRef<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +/// Write guard for type data access, using mapped guard for zero-cost deref. +pub struct TypeDataRefMut<'a, T: 'static> { + guard: PyMappedRwLockWriteGuard<'a, T>, +} + +impl<'a, T: Any + 'static> TypeDataRefMut<'a, T> { + /// Try to create a TypeDataRefMut from a write guard. + /// Returns None if the slot is empty or contains a different type. + pub fn try_new(guard: PyRwLockWriteGuard<'a, Option<TypeDataSlot>>) -> Option<Self> { + PyRwLockWriteGuard::try_map(guard, |opt| { + opt.as_mut().and_then(|slot| slot.get_mut::<T>()) + }) + .ok() + .map(|guard| Self { guard }) + } +} + +impl<T: Any + 'static> core::ops::Deref for TypeDataRefMut<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +impl<T: Any + 'static> core::ops::DerefMut for TypeDataRefMut<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.guard + } +} + +#[macro_export] +macro_rules! atomic_func { + ($x:expr) => { + Some($x) + }; +} + +// The corresponding field in CPython is `tp_` prefixed. +// e.g. name -> tp_name +#[derive(Default)] +#[non_exhaustive] +pub struct PyTypeSlots { + /// # Safety + /// For static types, always safe. + /// For heap types, `__name__` must alive + pub(crate) name: &'static str, // tp_name with <module>.<class> for print, not class name + + pub basicsize: usize, + pub itemsize: usize, // tp_itemsize + + // Methods to implement standard operations + + // Method suites for standard classes + pub as_number: PyNumberSlots, + pub as_sequence: PySequenceSlots, + pub as_mapping: PyMappingSlots, + + // More standard operations (here for binary compatibility) + pub hash: AtomicCell<Option<HashFunc>>, + pub call: AtomicCell<Option<GenericMethod>>, + pub vectorcall: AtomicCell<Option<VectorCallFunc>>, + pub str: AtomicCell<Option<StringifyFunc>>, + pub repr: AtomicCell<Option<StringifyFunc>>, + pub getattro: AtomicCell<Option<GetattroFunc>>, + pub setattro: AtomicCell<Option<SetattroFunc>>, + + // Functions to access object as input/output buffer + pub as_buffer: Option<AsBufferFunc>, + + // Assigned meaning in release 2.1 + // rich comparisons + pub richcompare: AtomicCell<Option<RichCompareFunc>>, + + // Iterators + pub iter: AtomicCell<Option<IterFunc>>, + pub iternext: AtomicCell<Option<IterNextFunc>>, + + pub methods: &'static [PyMethodDef], + + // Flags to define presence of optional/expanded features + pub flags: PyTypeFlags, + + // tp_doc + pub doc: Option<&'static str>, + + // Strong reference on a heap type, borrowed reference on a static type + // tp_base + // tp_dict + pub descr_get: AtomicCell<Option<DescrGetFunc>>, + pub descr_set: AtomicCell<Option<DescrSetFunc>>, + // tp_dictoffset + pub init: AtomicCell<Option<InitFunc>>, + // tp_alloc + pub alloc: AtomicCell<Option<AllocFunc>>, + pub new: AtomicCell<Option<NewFunc>>, + // tp_free + // tp_is_gc + // tp_bases + // tp_mro + // tp_cache + // tp_subclasses + // tp_weaklist + pub del: AtomicCell<Option<DelFunc>>, + + // The count of tp_members. + pub member_count: usize, +} + +impl PyTypeSlots { + pub fn new(name: &'static str, flags: PyTypeFlags) -> Self { + Self { + name, + flags, + ..Default::default() + } + } + + pub fn heap_default() -> Self { + Self { + // init: AtomicCell::new(Some(init_wrapper)), + ..Default::default() + } + } +} + +impl core::fmt::Debug for PyTypeSlots { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("PyTypeSlots") + } +} + +bitflags! { + #[derive(Copy, Clone, Debug, PartialEq)] + #[non_exhaustive] + pub struct PyTypeFlags: u64 { + const MANAGED_WEAKREF = 1 << 3; + const MANAGED_DICT = 1 << 4; + const SEQUENCE = 1 << 5; + const MAPPING = 1 << 6; + const DISALLOW_INSTANTIATION = 1 << 7; + const IMMUTABLETYPE = 1 << 8; + const HEAPTYPE = 1 << 9; + const BASETYPE = 1 << 10; + const METHOD_DESCRIPTOR = 1 << 17; + // For built-in types that match the subject itself in pattern matching + // (bool, int, float, str, bytes, bytearray, list, tuple, dict, set, frozenset) + // This is not a stable API + const _MATCH_SELF = 1 << 22; + const HAS_DICT = 1 << 40; + const HAS_WEAKREF = 1 << 41; + + #[cfg(debug_assertions)] + const _CREATED_WITH_FLAGS = 1 << 63; + } +} + +impl PyTypeFlags { + // Default used for both built-in and normal classes: empty, for now. + // CPython default: Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | Py_TPFLAGS_HAVE_VERSION_TAG + pub const DEFAULT: Self = Self::empty(); + + // CPython: See initialization of flags in type_new. + /// Used for types created in Python. Subclassable and are a + /// heaptype. + pub const fn heap_type_flags() -> Self { + match Self::from_bits(Self::DEFAULT.bits() | Self::HEAPTYPE.bits() | Self::BASETYPE.bits()) + { + Some(flags) => flags, + None => unreachable!(), + } + } + + pub const fn has_feature(self, flag: Self) -> bool { + self.contains(flag) + } + + #[cfg(debug_assertions)] + pub const fn is_created_with_flags(self) -> bool { + self.contains(Self::_CREATED_WITH_FLAGS) + } +} + +impl Default for PyTypeFlags { + fn default() -> Self { + Self::DEFAULT + } +} + +pub(crate) type GenericMethod = fn(&PyObject, FuncArgs, &VirtualMachine) -> PyResult; +/// Vectorcall function pointer (PEP 590). +/// args: owned positional args followed by kwarg values. +/// nargs: number of positional args (self prepended by caller if needed). +/// kwnames: keyword argument names (last kwnames.len() entries in args are kwarg values). +pub(crate) type VectorCallFunc = fn( + &PyObject, // callable + Vec<PyObjectRef>, // owned args (positional + kwarg values) + usize, // nargs (positional count) + Option<&[PyObjectRef]>, // kwnames (keyword argument names) + &VirtualMachine, +) -> PyResult; +pub(crate) type HashFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyHash>; +// CallFunc = GenericMethod +pub(crate) type StringifyFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyRef<PyStr>>; +pub(crate) type GetattroFunc = fn(&PyObject, &Py<PyStr>, &VirtualMachine) -> PyResult; +pub(crate) type SetattroFunc = + fn(&PyObject, &Py<PyStr>, PySetterValue, &VirtualMachine) -> PyResult<()>; +pub(crate) type AsBufferFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyBuffer>; +pub(crate) type RichCompareFunc = fn( + &PyObject, + &PyObject, + PyComparisonOp, + &VirtualMachine, +) -> PyResult<Either<PyObjectRef, PyComparisonValue>>; +pub(crate) type IterFunc = fn(PyObjectRef, &VirtualMachine) -> PyResult; +pub(crate) type IterNextFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyIterReturn>; +pub(crate) type DescrGetFunc = + fn(PyObjectRef, Option<PyObjectRef>, Option<PyObjectRef>, &VirtualMachine) -> PyResult; +pub(crate) type DescrSetFunc = + fn(&PyObject, PyObjectRef, PySetterValue, &VirtualMachine) -> PyResult<()>; +pub(crate) type AllocFunc = fn(PyTypeRef, usize, &VirtualMachine) -> PyResult; +pub(crate) type NewFunc = fn(PyTypeRef, FuncArgs, &VirtualMachine) -> PyResult; +pub(crate) type InitFunc = fn(PyObjectRef, FuncArgs, &VirtualMachine) -> PyResult<()>; +pub(crate) type DelFunc = fn(&PyObject, &VirtualMachine) -> PyResult<()>; + +// Sequence sub-slot function types +pub(crate) type SeqLenFunc = fn(PySequence<'_>, &VirtualMachine) -> PyResult<usize>; +pub(crate) type SeqConcatFunc = fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult; +pub(crate) type SeqRepeatFunc = fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult; +pub(crate) type SeqItemFunc = fn(PySequence<'_>, isize, &VirtualMachine) -> PyResult; +pub(crate) type SeqAssItemFunc = + fn(PySequence<'_>, isize, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>; +pub(crate) type SeqContainsFunc = fn(PySequence<'_>, &PyObject, &VirtualMachine) -> PyResult<bool>; + +// Mapping sub-slot function types +pub(crate) type MapLenFunc = fn(PyMapping<'_>, &VirtualMachine) -> PyResult<usize>; +pub(crate) type MapSubscriptFunc = fn(PyMapping<'_>, &PyObject, &VirtualMachine) -> PyResult; +pub(crate) type MapAssSubscriptFunc = + fn(PyMapping<'_>, &PyObject, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>; + +// slot_sq_length +pub(crate) fn len_wrapper(obj: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { + let ret = vm.call_special_method(obj, identifier!(vm, __len__), ())?; + let len = ret.downcast_ref::<PyInt>().ok_or_else(|| { + vm.new_type_error(format!( + "'{}' object cannot be interpreted as an integer", + ret.class() + )) + })?; + let len = len.as_bigint(); + if len.is_negative() { + return Err(vm.new_value_error("__len__() should return >= 0")); + } + let len = len + .to_isize() + .ok_or_else(|| vm.new_overflow_error("cannot fit 'int' into an index-sized integer"))?; + Ok(len as usize) +} + +pub(crate) fn contains_wrapper( + obj: &PyObject, + needle: &PyObject, + vm: &VirtualMachine, +) -> PyResult<bool> { + let ret = vm.call_special_method(obj, identifier!(vm, __contains__), (needle,))?; + ret.try_to_bool(vm) +} + +macro_rules! number_unary_op_wrapper { + ($name:ident) => { + |a, vm| vm.call_special_method(a.deref(), identifier!(vm, $name), ()) + }; +} +macro_rules! number_binary_op_wrapper { + ($name:ident) => { + |a, b, vm| vm.call_special_method(a, identifier!(vm, $name), (b.to_owned(),)) + }; +} +macro_rules! number_binary_right_op_wrapper { + ($name:ident) => { + |a, b, vm| vm.call_special_method(b, identifier!(vm, $name), (a.to_owned(),)) + }; +} +macro_rules! number_ternary_op_wrapper { + ($name:ident) => { + |a, b, c, vm: &VirtualMachine| { + if vm.is_none(c) { + vm.call_special_method(a, identifier!(vm, $name), (b.to_owned(),)) + } else { + vm.call_special_method(a, identifier!(vm, $name), (b.to_owned(), c.to_owned())) + } + } + }; +} +macro_rules! number_ternary_right_op_wrapper { + ($name:ident) => { + |a, b, c, vm: &VirtualMachine| { + if vm.is_none(c) { + vm.call_special_method(b, identifier!(vm, $name), (a.to_owned(),)) + } else { + vm.call_special_method(b, identifier!(vm, $name), (a.to_owned(), c.to_owned())) + } + } + }; +} +fn getitem_wrapper<K: ToPyObject>(obj: &PyObject, needle: K, vm: &VirtualMachine) -> PyResult { + vm.call_special_method(obj, identifier!(vm, __getitem__), (needle,)) +} + +fn setitem_wrapper<K: ToPyObject>( + obj: &PyObject, + needle: K, + value: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + match value { + Some(value) => vm.call_special_method(obj, identifier!(vm, __setitem__), (needle, value)), + None => vm.call_special_method(obj, identifier!(vm, __delitem__), (needle,)), + } + .map(drop) +} + +#[inline(never)] +fn mapping_setitem_wrapper( + mapping: PyMapping<'_>, + key: &PyObject, + value: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + setitem_wrapper(mapping.obj, key, value, vm) +} + +#[inline(never)] +fn mapping_getitem_wrapper( + mapping: PyMapping<'_>, + key: &PyObject, + vm: &VirtualMachine, +) -> PyResult { + getitem_wrapper(mapping.obj, key, vm) +} + +#[inline(never)] +fn mapping_len_wrapper(mapping: PyMapping<'_>, vm: &VirtualMachine) -> PyResult<usize> { + len_wrapper(mapping.obj, vm) +} + +#[inline(never)] +fn sequence_len_wrapper(seq: PySequence<'_>, vm: &VirtualMachine) -> PyResult<usize> { + len_wrapper(seq.obj, vm) +} + +#[inline(never)] +fn sequence_getitem_wrapper(seq: PySequence<'_>, i: isize, vm: &VirtualMachine) -> PyResult { + getitem_wrapper(seq.obj, i, vm) +} + +#[inline(never)] +fn sequence_setitem_wrapper( + seq: PySequence<'_>, + i: isize, + value: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + setitem_wrapper(seq.obj, i, value, vm) +} + +#[inline(never)] +fn sequence_contains_wrapper( + seq: PySequence<'_>, + needle: &PyObject, + vm: &VirtualMachine, +) -> PyResult<bool> { + contains_wrapper(seq.obj, needle, vm) +} + +#[inline(never)] +fn sequence_repeat_wrapper(seq: PySequence<'_>, n: isize, vm: &VirtualMachine) -> PyResult { + vm.call_special_method(seq.obj, identifier!(vm, __mul__), (n,)) +} + +#[inline(never)] +fn sequence_inplace_repeat_wrapper(seq: PySequence<'_>, n: isize, vm: &VirtualMachine) -> PyResult { + vm.call_special_method(seq.obj, identifier!(vm, __imul__), (n,)) +} + +fn repr_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let ret = vm.call_special_method(zelf, identifier!(vm, __repr__), ())?; + ret.downcast::<PyStr>().map_err(|obj| { + vm.new_type_error(format!( + "__repr__ returned non-string (type {})", + obj.class() + )) + }) +} + +fn str_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let ret = vm.call_special_method(zelf, identifier!(vm, __str__), ())?; + ret.downcast::<PyStr>().map_err(|obj| { + vm.new_type_error(format!( + "__str__ returned non-string (type {})", + obj.class() + )) + }) +} + +fn hash_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyHash> { + let hash_obj = vm.call_special_method(zelf, identifier!(vm, __hash__), ())?; + let py_int = hash_obj + .downcast_ref::<PyInt>() + .ok_or_else(|| vm.new_type_error("__hash__ method should return an integer"))?; + let big_int = py_int.as_bigint(); + let hash = big_int + .to_i64() + .map(fix_sentinel) + .unwrap_or_else(|| hash_bigint(big_int)); + Ok(hash) +} + +/// Marks a type as unhashable. Similar to PyObject_HashNotImplemented in CPython +pub fn hash_not_implemented(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyHash> { + Err(vm.new_type_error(format!("unhashable type: '{}'", zelf.class().name()))) +} + +fn call_wrapper(zelf: &PyObject, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + vm.call_special_method(zelf, identifier!(vm, __call__), args) +} + +fn getattro_wrapper(zelf: &PyObject, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + let __getattribute__ = identifier!(vm, __getattribute__); + let __getattr__ = identifier!(vm, __getattr__); + match vm.call_special_method(zelf, __getattribute__, (name.to_owned(),)) { + Ok(r) => Ok(r), + Err(e) + if e.fast_isinstance(vm.ctx.exceptions.attribute_error) + && zelf.class().has_attr(__getattr__) => + { + vm.call_special_method(zelf, __getattr__, (name.to_owned(),)) + } + Err(e) => Err(e), + } +} + +fn setattro_wrapper( + zelf: &PyObject, + name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, +) -> PyResult<()> { + let name = name.to_owned(); + match value { + PySetterValue::Assign(value) => { + vm.call_special_method(zelf, identifier!(vm, __setattr__), (name, value))?; + } + PySetterValue::Delete => { + vm.call_special_method(zelf, identifier!(vm, __delattr__), (name,))?; + } + }; + Ok(()) +} + +pub(crate) fn richcompare_wrapper( + zelf: &PyObject, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, +) -> PyResult<Either<PyObjectRef, PyComparisonValue>> { + vm.call_special_method(zelf, op.method_name(&vm.ctx), (other.to_owned(),)) + .map(Either::A) +} + +fn iter_wrapper(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // slot_tp_iter: if __iter__ is None, the type is explicitly not iterable + let cls = zelf.class(); + let iter_attr = cls.get_attr(identifier!(vm, __iter__)); + match iter_attr { + Some(attr) if vm.is_none(&attr) => { + Err(vm.new_type_error(format!("'{}' object is not iterable", cls.name()))) + } + _ => vm.call_special_method(&zelf, identifier!(vm, __iter__), ()), + } +} + +fn bool_wrapper(num: PyNumber<'_>, vm: &VirtualMachine) -> PyResult<bool> { + let result = vm.call_special_method(num.obj, identifier!(vm, __bool__), ())?; + // __bool__ must return exactly bool, not int subclass + if !result.class().is(vm.ctx.types.bool_type) { + return Err(vm.new_type_error(format!( + "__bool__ should return bool, returned {}", + result.class().name() + ))); + } + Ok(crate::builtins::bool_::get_value(&result)) +} + +// PyObject_SelfIter in CPython +const fn self_iter(zelf: PyObjectRef, _vm: &VirtualMachine) -> PyResult { + Ok(zelf) +} + +fn iternext_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + PyIterReturn::from_pyresult( + vm.call_special_method(zelf, identifier!(vm, __next__), ()), + vm, + ) +} + +fn descr_get_wrapper( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + cls: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult { + vm.call_special_method(&zelf, identifier!(vm, __get__), (obj, cls)) +} + +fn descr_set_wrapper( + zelf: &PyObject, + obj: PyObjectRef, + value: PySetterValue, + vm: &VirtualMachine, +) -> PyResult<()> { + match value { + PySetterValue::Assign(val) => { + vm.call_special_method(zelf, identifier!(vm, __set__), (obj, val)) + } + PySetterValue::Delete => vm.call_special_method(zelf, identifier!(vm, __delete__), (obj,)), + } + .map(drop) +} + +fn init_wrapper(obj: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let res = vm.call_special_method(&obj, identifier!(vm, __init__), args)?; + if !vm.is_none(&res) { + return Err(vm.new_type_error(format!( + "__init__() should return None, not '{:.200}'", + res.class().name() + ))); + } + Ok(()) +} + +pub(crate) fn new_wrapper(cls: PyTypeRef, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let new = cls.get_attr(identifier!(vm, __new__)).unwrap(); + args.prepend_arg(cls.into()); + new.call(args, vm) +} + +fn del_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + vm.call_special_method(zelf, identifier!(vm, __del__), ())?; + Ok(()) +} + +/// Result of looking up a slot function in MRO. +enum SlotLookupResult<T> { + /// Found a native slot function from a wrapper_descriptor. + NativeSlot(T), + /// Found a Python-level method (not a native slot). + /// The caller should use the wrapper function. + PythonMethod, + /// No method with this name found in MRO at all. + /// The caller should inherit the slot from MRO. + NotFound, +} + +impl PyType { + /// Update slots based on dunder method changes + /// + /// Iterates SLOT_DEFS to find all slots matching the given name and updates them. + /// Also recursively updates subclasses that don't have their own definition. + pub(crate) fn update_slot<const ADD: bool>(&self, name: &'static PyStrInterned, ctx: &Context) { + debug_assert!(name.as_str().starts_with("__")); + debug_assert!(name.as_str().ends_with("__")); + + // Find all slot_defs matching this name and update each + // NOTE: Collect into Vec first to avoid issues during iteration + let defs: Vec<_> = find_slot_defs_by_name(name.as_str()).collect(); + for def in defs { + self.update_one_slot::<ADD>(&def.accessor, name, ctx); + } + + // Recursively update subclasses that don't have their own definition + self.update_subclasses::<ADD>(name, ctx); + } + + /// Recursively update subclasses' slots + /// recurse_down_subclasses + fn update_subclasses<const ADD: bool>(&self, name: &'static PyStrInterned, ctx: &Context) { + let subclasses = self.subclasses.read(); + for weak_ref in subclasses.iter() { + let Some(subclass) = weak_ref.upgrade() else { + continue; + }; + let Some(subclass) = subclass.downcast_ref::<PyType>() else { + continue; + }; + + // Skip if subclass has its own definition for this attribute + if subclass.attributes.read().contains_key(name) { + continue; + } + + // Update subclass's slots + for def in find_slot_defs_by_name(name.as_str()) { + subclass.update_one_slot::<ADD>(&def.accessor, name, ctx); + } + + // Recurse into subclass's subclasses + subclass.update_subclasses::<ADD>(name, ctx); + } + } + + /// Update a single slot + fn update_one_slot<const ADD: bool>( + &self, + accessor: &SlotAccessor, + name: &'static PyStrInterned, + ctx: &Context, + ) { + use crate::builtins::descriptor::SlotFunc; + + // Helper macro for main slots + macro_rules! update_main_slot { + ($slot:ident, $wrapper:expr, $variant:ident) => {{ + if ADD { + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::$variant(f) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.$slot.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.$slot.store(Some($wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } else { + accessor.inherit_from_mro(self); + } + }}; + } + + // Helper macro for number/sequence/mapping sub-slots + macro_rules! update_sub_slot { + ($group:ident, $slot:ident, $wrapper:expr, $variant:ident) => {{ + if ADD { + // Check if this type defines any method that maps to this slot. + // Some slots like SqAssItem/MpAssSubscript are shared by multiple + // methods (__setitem__ and __delitem__). If any of those methods + // is defined, we must use the wrapper to ensure Python method calls. + let has_own = { + let guard = self.attributes.read(); + // Check the current method name + let mut result = guard.contains_key(name); + // For ass_item/ass_subscript slots, also check the paired method + // (__setitem__ and __delitem__ share the same slot) + if !result + && (stringify!($slot) == "ass_item" + || stringify!($slot) == "ass_subscript") + { + let setitem = ctx.intern_str("__setitem__"); + let delitem = ctx.intern_str("__delitem__"); + result = guard.contains_key(setitem) || guard.contains_key(delitem); + } + result + }; + if has_own { + self.slots.$group.$slot.store(Some($wrapper)); + } else { + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::$variant(f) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.$group.$slot.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.$group.$slot.store(Some($wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } + } else { + accessor.inherit_from_mro(self); + } + }}; + } + + match accessor { + // === Main slots === + SlotAccessor::TpRepr => update_main_slot!(repr, repr_wrapper, Repr), + SlotAccessor::TpStr => update_main_slot!(str, str_wrapper, Str), + SlotAccessor::TpHash => { + // Special handling for __hash__ = None + if ADD { + let method = self.attributes.read().get(name).cloned().or_else(|| { + self.mro + .read() + .iter() + .find_map(|cls| cls.attributes.read().get(name).cloned()) + }); + + if method.as_ref().is_some_and(|m| m.is(&ctx.none)) { + self.slots.hash.store(Some(hash_not_implemented)); + } else { + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::Hash(f) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.hash.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.hash.store(Some(hash_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } + } else { + accessor.inherit_from_mro(self); + } + } + SlotAccessor::TpCall => { + update_main_slot!(call, call_wrapper, Call); + // When __call__ is overridden in Python, clear vectorcall + // so the slow path through call_wrapper is used. + if ADD { + self.slots.vectorcall.store(None); + } + } + SlotAccessor::TpIter => update_main_slot!(iter, iter_wrapper, Iter), + SlotAccessor::TpIternext => update_main_slot!(iternext, iternext_wrapper, IterNext), + SlotAccessor::TpInit => update_main_slot!(init, init_wrapper, Init), + SlotAccessor::TpNew => { + // __new__ is not wrapped via PyWrapper + if ADD { + self.slots.new.store(Some(new_wrapper)); + } else { + accessor.inherit_from_mro(self); + } + } + SlotAccessor::TpDel => update_main_slot!(del, del_wrapper, Del), + SlotAccessor::TpGetattro => { + // __getattribute__ and __getattr__ both map to TpGetattro. + // If __getattr__ is defined anywhere in MRO, we must use the wrapper + // because the native slot won't call __getattr__. + let __getattr__ = identifier!(ctx, __getattr__); + let has_getattr = { + let attrs = self.attributes.read(); + let in_self = attrs.contains_key(__getattr__); + drop(attrs); + // mro[0] is self, so skip it + in_self + || self + .mro + .read() + .iter() + .skip(1) + .any(|cls| cls.attributes.read().contains_key(__getattr__)) + }; + + if has_getattr { + // Must use wrapper to handle __getattr__ + self.slots.getattro.store(Some(getattro_wrapper)); + } else if ADD { + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::GetAttro(f) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.getattro.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.getattro.store(Some(getattro_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } else { + accessor.inherit_from_mro(self); + } + } + SlotAccessor::TpSetattro => { + // __setattr__ and __delattr__ share the same slot + if ADD { + match self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::SetAttro(f) | SlotFunc::DelAttro(f) => Some(*f), + _ => None, + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.setattro.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.setattro.store(Some(setattro_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } else { + accessor.inherit_from_mro(self); + } + } + SlotAccessor::TpDescrGet => update_main_slot!(descr_get, descr_get_wrapper, DescrGet), + SlotAccessor::TpDescrSet => { + // __set__ and __delete__ share the same slot + if ADD { + match self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::DescrSet(f) | SlotFunc::DescrDel(f) => Some(*f), + _ => None, + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.descr_set.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.descr_set.store(Some(descr_set_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } else { + accessor.inherit_from_mro(self); + } + } + + // === Rich compare (__lt__, __le__, __eq__, __ne__, __gt__, __ge__) === + SlotAccessor::TpRichcompare => { + if ADD { + // Check if self or any class in MRO has a Python-defined comparison method + // All comparison ops share the same slot, so if any is overridden anywhere + // in the hierarchy with a Python function, we need to use the wrapper + let cmp_names = [ + identifier!(ctx, __eq__), + identifier!(ctx, __ne__), + identifier!(ctx, __lt__), + identifier!(ctx, __le__), + identifier!(ctx, __gt__), + identifier!(ctx, __ge__), + ]; + + let has_python_cmp = { + // Check self first + let attrs = self.attributes.read(); + let in_self = cmp_names.iter().any(|n| attrs.contains_key(*n)); + drop(attrs); + + // mro[0] is self, so skip it since we already checked self above + in_self + || self.mro.read()[1..].iter().any(|cls| { + let attrs = cls.attributes.read(); + cmp_names.iter().any(|n| { + if let Some(attr) = attrs.get(*n) { + // Check if it's a Python function (not a native descriptor) + !attr.class().is(ctx.types.wrapper_descriptor_type) + && !attr.class().is(ctx.types.method_descriptor_type) + } else { + false + } + }) + }) + }; + + if has_python_cmp { + // Use wrapper to call the Python method + self.slots.richcompare.store(Some(richcompare_wrapper)); + } else { + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::RichCompare(f, _) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.richcompare.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.richcompare.store(Some(richcompare_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } + } else { + accessor.inherit_from_mro(self); + } + } + + // === Number binary operations === + SlotAccessor::NbAdd => { + if name.as_str() == "__radd__" { + update_sub_slot!( + as_number, + right_add, + number_binary_right_op_wrapper!(__radd__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + add, + number_binary_op_wrapper!(__add__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceAdd => { + update_sub_slot!( + as_number, + inplace_add, + number_binary_op_wrapper!(__iadd__), + NumBinary + ) + } + SlotAccessor::NbSubtract => { + if name.as_str() == "__rsub__" { + update_sub_slot!( + as_number, + right_subtract, + number_binary_right_op_wrapper!(__rsub__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + subtract, + number_binary_op_wrapper!(__sub__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceSubtract => { + update_sub_slot!( + as_number, + inplace_subtract, + number_binary_op_wrapper!(__isub__), + NumBinary + ) + } + SlotAccessor::NbMultiply => { + if name.as_str() == "__rmul__" { + update_sub_slot!( + as_number, + right_multiply, + number_binary_right_op_wrapper!(__rmul__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + multiply, + number_binary_op_wrapper!(__mul__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceMultiply => { + update_sub_slot!( + as_number, + inplace_multiply, + number_binary_op_wrapper!(__imul__), + NumBinary + ) + } + SlotAccessor::NbRemainder => { + if name.as_str() == "__rmod__" { + update_sub_slot!( + as_number, + right_remainder, + number_binary_right_op_wrapper!(__rmod__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + remainder, + number_binary_op_wrapper!(__mod__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceRemainder => { + update_sub_slot!( + as_number, + inplace_remainder, + number_binary_op_wrapper!(__imod__), + NumBinary + ) + } + SlotAccessor::NbDivmod => { + if name.as_str() == "__rdivmod__" { + update_sub_slot!( + as_number, + right_divmod, + number_binary_right_op_wrapper!(__rdivmod__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + divmod, + number_binary_op_wrapper!(__divmod__), + NumBinary + ) + } + } + SlotAccessor::NbPower => { + if name.as_str() == "__rpow__" { + update_sub_slot!( + as_number, + right_power, + number_ternary_right_op_wrapper!(__rpow__), + NumTernary + ) + } else { + update_sub_slot!( + as_number, + power, + number_ternary_op_wrapper!(__pow__), + NumTernary + ) + } + } + SlotAccessor::NbInplacePower => { + update_sub_slot!( + as_number, + inplace_power, + number_ternary_op_wrapper!(__ipow__), + NumTernary + ) + } + SlotAccessor::NbFloorDivide => { + if name.as_str() == "__rfloordiv__" { + update_sub_slot!( + as_number, + right_floor_divide, + number_binary_right_op_wrapper!(__rfloordiv__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + floor_divide, + number_binary_op_wrapper!(__floordiv__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceFloorDivide => { + update_sub_slot!( + as_number, + inplace_floor_divide, + number_binary_op_wrapper!(__ifloordiv__), + NumBinary + ) + } + SlotAccessor::NbTrueDivide => { + if name.as_str() == "__rtruediv__" { + update_sub_slot!( + as_number, + right_true_divide, + number_binary_right_op_wrapper!(__rtruediv__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + true_divide, + number_binary_op_wrapper!(__truediv__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceTrueDivide => { + update_sub_slot!( + as_number, + inplace_true_divide, + number_binary_op_wrapper!(__itruediv__), + NumBinary + ) + } + SlotAccessor::NbMatrixMultiply => { + if name.as_str() == "__rmatmul__" { + update_sub_slot!( + as_number, + right_matrix_multiply, + number_binary_right_op_wrapper!(__rmatmul__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + matrix_multiply, + number_binary_op_wrapper!(__matmul__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceMatrixMultiply => { + update_sub_slot!( + as_number, + inplace_matrix_multiply, + number_binary_op_wrapper!(__imatmul__), + NumBinary + ) + } + + // === Number bitwise operations === + SlotAccessor::NbLshift => { + if name.as_str() == "__rlshift__" { + update_sub_slot!( + as_number, + right_lshift, + number_binary_right_op_wrapper!(__rlshift__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + lshift, + number_binary_op_wrapper!(__lshift__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceLshift => { + update_sub_slot!( + as_number, + inplace_lshift, + number_binary_op_wrapper!(__ilshift__), + NumBinary + ) + } + SlotAccessor::NbRshift => { + if name.as_str() == "__rrshift__" { + update_sub_slot!( + as_number, + right_rshift, + number_binary_right_op_wrapper!(__rrshift__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + rshift, + number_binary_op_wrapper!(__rshift__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceRshift => { + update_sub_slot!( + as_number, + inplace_rshift, + number_binary_op_wrapper!(__irshift__), + NumBinary + ) + } + SlotAccessor::NbAnd => { + if name.as_str() == "__rand__" { + update_sub_slot!( + as_number, + right_and, + number_binary_right_op_wrapper!(__rand__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + and, + number_binary_op_wrapper!(__and__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceAnd => { + update_sub_slot!( + as_number, + inplace_and, + number_binary_op_wrapper!(__iand__), + NumBinary + ) + } + SlotAccessor::NbXor => { + if name.as_str() == "__rxor__" { + update_sub_slot!( + as_number, + right_xor, + number_binary_right_op_wrapper!(__rxor__), + NumBinary + ) + } else { + update_sub_slot!( + as_number, + xor, + number_binary_op_wrapper!(__xor__), + NumBinary + ) + } + } + SlotAccessor::NbInplaceXor => { + update_sub_slot!( + as_number, + inplace_xor, + number_binary_op_wrapper!(__ixor__), + NumBinary + ) + } + SlotAccessor::NbOr => { + if name.as_str() == "__ror__" { + update_sub_slot!( + as_number, + right_or, + number_binary_right_op_wrapper!(__ror__), + NumBinary + ) + } else { + update_sub_slot!(as_number, or, number_binary_op_wrapper!(__or__), NumBinary) + } + } + SlotAccessor::NbInplaceOr => { + update_sub_slot!( + as_number, + inplace_or, + number_binary_op_wrapper!(__ior__), + NumBinary + ) + } + + // === Number unary operations === + SlotAccessor::NbNegative => { + update_sub_slot!( + as_number, + negative, + number_unary_op_wrapper!(__neg__), + NumUnary + ) + } + SlotAccessor::NbPositive => { + update_sub_slot!( + as_number, + positive, + number_unary_op_wrapper!(__pos__), + NumUnary + ) + } + SlotAccessor::NbAbsolute => { + update_sub_slot!( + as_number, + absolute, + number_unary_op_wrapper!(__abs__), + NumUnary + ) + } + SlotAccessor::NbInvert => { + update_sub_slot!( + as_number, + invert, + number_unary_op_wrapper!(__invert__), + NumUnary + ) + } + SlotAccessor::NbBool => { + update_sub_slot!(as_number, boolean, bool_wrapper, NumBoolean) + } + SlotAccessor::NbInt => { + update_sub_slot!(as_number, int, number_unary_op_wrapper!(__int__), NumUnary) + } + SlotAccessor::NbFloat => { + update_sub_slot!( + as_number, + float, + number_unary_op_wrapper!(__float__), + NumUnary + ) + } + SlotAccessor::NbIndex => { + update_sub_slot!( + as_number, + index, + number_unary_op_wrapper!(__index__), + NumUnary + ) + } + + // === Sequence slots === + SlotAccessor::SqLength => { + update_sub_slot!(as_sequence, length, sequence_len_wrapper, SeqLength) + } + SlotAccessor::SqConcat | SlotAccessor::SqInplaceConcat => { + // Sequence concat uses sq_concat slot - no generic wrapper needed + // (handled by number protocol fallback) + if !ADD { + accessor.inherit_from_mro(self); + } + } + SlotAccessor::SqRepeat => { + update_sub_slot!(as_sequence, repeat, sequence_repeat_wrapper, SeqRepeat) + } + SlotAccessor::SqInplaceRepeat => { + update_sub_slot!( + as_sequence, + inplace_repeat, + sequence_inplace_repeat_wrapper, + SeqRepeat + ) + } + SlotAccessor::SqItem => { + update_sub_slot!(as_sequence, item, sequence_getitem_wrapper, SeqItem) + } + SlotAccessor::SqAssItem => { + // SqAssItem is shared by __setitem__ (SeqSetItem) and __delitem__ (SeqDelItem) + if ADD { + let has_own = { + let guard = self.attributes.read(); + let setitem = ctx.intern_str("__setitem__"); + let delitem = ctx.intern_str("__delitem__"); + guard.contains_key(setitem) || guard.contains_key(delitem) + }; + if has_own { + self.slots + .as_sequence + .ass_item + .store(Some(sequence_setitem_wrapper)); + } else { + match self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::SeqSetItem(f) | SlotFunc::SeqDelItem(f) => Some(*f), + _ => None, + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.as_sequence.ass_item.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots + .as_sequence + .ass_item + .store(Some(sequence_setitem_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } + } else { + accessor.inherit_from_mro(self); + } + } + SlotAccessor::SqContains => { + update_sub_slot!( + as_sequence, + contains, + sequence_contains_wrapper, + SeqContains + ) + } + + // === Mapping slots === + SlotAccessor::MpLength => { + update_sub_slot!(as_mapping, length, mapping_len_wrapper, MapLength) + } + SlotAccessor::MpSubscript => { + update_sub_slot!(as_mapping, subscript, mapping_getitem_wrapper, MapSubscript) + } + SlotAccessor::MpAssSubscript => { + // MpAssSubscript is shared by __setitem__ (MapSetSubscript) and __delitem__ (MapDelSubscript) + if ADD { + let has_own = { + let guard = self.attributes.read(); + let setitem = ctx.intern_str("__setitem__"); + let delitem = ctx.intern_str("__delitem__"); + guard.contains_key(setitem) || guard.contains_key(delitem) + }; + if has_own { + self.slots + .as_mapping + .ass_subscript + .store(Some(mapping_setitem_wrapper)); + } else { + match self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::MapSetSubscript(f) | SlotFunc::MapDelSubscript(f) => Some(*f), + _ => None, + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.as_mapping.ass_subscript.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots + .as_mapping + .ass_subscript + .store(Some(mapping_setitem_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } + } else { + accessor.inherit_from_mro(self); + } + } + + // Reserved slots - no-op + _ => {} + } + } + + /// Look up a method in MRO and extract the slot function if it's a slot wrapper. + fn lookup_slot_in_mro<T: Copy>( + &self, + name: &'static PyStrInterned, + ctx: &Context, + extract: impl Fn(&crate::builtins::descriptor::SlotFunc) -> Option<T>, + ) -> SlotLookupResult<T> { + use crate::builtins::descriptor::PyWrapper; + + // Helper to check if a class is a subclass of another by checking MRO + let is_subclass_of = |subclass_mro: &[PyRef<PyType>], superclass: &Py<PyType>| -> bool { + subclass_mro.iter().any(|c| c.is(superclass)) + }; + + // Helper to extract slot from an attribute if it's a wrapper descriptor + // and the wrapper's type is compatible with the given class. + // bpo-37619: wrapper descriptor from wrong class should not be used directly. + let try_extract = |attr: &PyObjectRef, for_class_mro: &[PyRef<PyType>]| -> Option<T> { + if attr.class().is(ctx.types.wrapper_descriptor_type) { + attr.downcast_ref::<PyWrapper>().and_then(|wrapper| { + // Only extract slot if for_class is a subclass of wrapper.typ + if is_subclass_of(for_class_mro, wrapper.typ) { + extract(&wrapper.wrapped) + } else { + None + } + }) + } else { + None + } + }; + + let mro = self.mro.read(); + + // Look up in self's dict first + if let Some(attr) = self.attributes.read().get(name).cloned() { + if let Some(func) = try_extract(&attr, &mro) { + return SlotLookupResult::NativeSlot(func); + } + return SlotLookupResult::PythonMethod; + } + + // Look up in MRO (mro[0] is self, so skip it) + for (i, cls) in mro[1..].iter().enumerate() { + if let Some(attr) = cls.attributes.read().get(name).cloned() { + // Use the slice starting from this class in MRO + if let Some(func) = try_extract(&attr, &mro[i + 1..]) { + return SlotLookupResult::NativeSlot(func); + } + return SlotLookupResult::PythonMethod; + } + } + // No method found in MRO + SlotLookupResult::NotFound + } +} + +/// Trait for types that can be constructed via Python's `__new__` method. +/// +/// `slot_new` corresponds to the `__new__` type slot. +/// +/// In most cases, `__new__` simply initializes the payload and assigns a type, +/// so you only need to override `py_new`. The default `slot_new` implementation +/// will call `py_new` and then wrap the result with `into_ref_with_type`. +/// +/// However, if a subtype requires more than just payload initialization +/// (e.g., returning an existing object for optimization, setting attributes +/// after creation, or special handling of the class type), you should override +/// `slot_new` directly instead of `py_new`. +/// +/// # When to use `py_new` only (most common case): +/// - Simple payload initialization that just creates `Self` +/// - The type doesn't need special handling for subtypes +/// +/// # When to override `slot_new`: +/// - Returning existing objects (e.g., `PyInt`, `PyStr`, `PyBool` for optimization) +/// - Setting attributes or dict entries after object creation +/// - Special class type handling (e.g., `PyType` and its metaclasses) +/// - Post-creation mutations that require `PyRef` +#[pyclass] +pub trait Constructor: PyPayload + core::fmt::Debug { + type Args: FromArgs; + + /// The type slot for `__new__`. Override this only when you need special + /// behavior beyond simple payload creation. + #[inline] + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let args: Self::Args = args.bind(vm)?; + let payload = Self::py_new(&cls, args, vm)?; + payload.into_ref_with_type(vm, cls).map(Into::into) + } + + /// Creates the payload for this type. In most cases, just implement this method + /// and let the default `slot_new` handle wrapping with the correct type. + fn py_new(cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self>; +} + +pub trait DefaultConstructor: PyPayload + Default + core::fmt::Debug { + fn construct_and_init(args: Self::Args, vm: &VirtualMachine) -> PyResult<PyRef<Self>> + where + Self: Initializer, + { + let this = Self::default().into_ref(&vm.ctx); + Self::init(this.clone(), args, vm)?; + Ok(this) + } +} + +impl<T> Constructor for T +where + T: DefaultConstructor, +{ + type Args = FuncArgs; + + fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Self::default().into_ref_with_type(vm, cls).map(Into::into) + } + + fn py_new(cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> { + Err(vm.new_type_error(format!("cannot create {} instances", cls.slot_name()))) + } +} + +#[pyclass] +pub trait Initializer: PyPayload { + type Args: FromArgs; + + #[inline] + #[pyslot] + fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + #[cfg(debug_assertions)] + let class_name_for_debug = zelf.class().name().to_string(); + + let zelf = match zelf.try_into_value(vm) { + Ok(zelf) => zelf, + Err(err) => { + #[cfg(debug_assertions)] + { + if let Ok(msg) = err.as_object().repr(vm) { + let double_appearance = msg + .to_string_lossy() + .matches(&class_name_for_debug as &str) + .count() + == 2; + if double_appearance { + panic!( + "This type `{}` doesn't seem to support `init`. Override `slot_init` instead: {}", + class_name_for_debug, msg + ); + } + } + } + return Err(err); + } + }; + let args: Self::Args = args.bind(vm)?; + Self::init(zelf, args, vm) + } + + fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()>; +} + +#[pyclass] +pub trait Destructor: PyPayload { + #[inline] // for __del__ + #[pyslot] + fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { + let zelf = zelf + .downcast_ref() + .ok_or_else(|| vm.new_type_error("unexpected payload for __del__"))?; + Self::del(zelf, vm) + } + + fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()>; +} + +#[pyclass] +pub trait Callable: PyPayload { + type Args: FromArgs; + + #[inline] + #[pyslot] + fn slot_call(zelf: &PyObject, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + let zelf = zelf.downcast_ref().ok_or_else(|| { + let repr = zelf.repr(vm); + let help: Wtf8Buf = if let Ok(repr) = repr.as_ref() { + repr.as_wtf8().to_owned() + } else { + zelf.class().name().to_owned().into() + }; + let mut msg = Wtf8Buf::from("unexpected payload for __call__ of "); + msg.push_wtf8(&help); + vm.new_type_error(msg) + })?; + let args = args.bind(vm)?; + Self::call(zelf, args, vm) + } + + fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult; +} + +#[pyclass] +pub trait GetDescriptor: PyPayload { + #[pyslot] + fn descr_get( + zelf: PyObjectRef, + obj: Option<PyObjectRef>, + cls: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult; + + #[inline] + fn _as_pyref<'a>(zelf: &'a PyObject, vm: &VirtualMachine) -> PyResult<&'a Py<Self>> { + zelf.try_to_value(vm) + } + + #[inline] + fn _unwrap<'a>( + zelf: &'a PyObject, + obj: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> PyResult<(&'a Py<Self>, PyObjectRef)> { + let zelf = Self::_as_pyref(zelf, vm)?; + let obj = vm.unwrap_or_none(obj); + Ok((zelf, obj)) + } + + #[inline] + fn _check<'a>( + zelf: &'a PyObject, + obj: Option<PyObjectRef>, + vm: &VirtualMachine, + ) -> Option<(&'a Py<Self>, PyObjectRef)> { + // CPython descr_check + let obj = obj?; + // if (!PyObject_TypeCheck(obj, descr->d_type)) { + // PyErr_Format(PyExc_TypeError, + // "descriptor '%V' for '%.100s' objects " + // "doesn't apply to a '%.100s' object", + // descr_name((PyDescrObject *)descr), "?", + // descr->d_type->slot_name, + // obj->ob_type->slot_name); + // *pres = NULL; + // return 1; + // } else { + Some((Self::_as_pyref(zelf, vm).unwrap(), obj)) + } + + #[inline] + fn _cls_is(cls: &Option<PyObjectRef>, other: &impl Borrow<PyObject>) -> bool { + cls.as_ref().is_some_and(|cls| other.borrow().is(cls)) + } +} + +#[pyclass] +pub trait Hashable: PyPayload { + #[inline] + #[pyslot] + fn slot_hash(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyHash> { + let zelf = zelf + .downcast_ref() + .ok_or_else(|| vm.new_type_error("unexpected payload for __hash__"))?; + Self::hash(zelf, vm) + } + + fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash>; +} + +#[pyclass] +pub trait Representable: PyPayload { + #[inline] + #[pyslot] + fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let zelf = zelf + .downcast_ref() + .ok_or_else(|| vm.new_type_error("unexpected payload for __repr__"))?; + Self::repr(zelf, vm) + } + + #[inline] + fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> { + let repr = Self::repr_wtf8(zelf, vm)?; + Ok(vm.ctx.new_str(repr)) + } + + fn repr_wtf8(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Wtf8Buf> { + Self::repr_str(zelf, vm).map(|utf8| utf8.into()) + } + fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { + unreachable!("Representable requires overriding either repr_str or repr_wtf8") + } +} + +#[pyclass] +pub trait Comparable: PyPayload { + #[inline] + #[pyslot] + fn slot_richcompare( + zelf: &PyObject, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<Either<PyObjectRef, PyComparisonValue>> { + let zelf = zelf.downcast_ref().ok_or_else(|| { + vm.new_type_error(format!( + "unexpected payload for {}", + op.method_name(&vm.ctx).as_str() + )) + })?; + Self::cmp(zelf, other, op, vm).map(Either::B) + } + + fn cmp( + zelf: &Py<Self>, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, + ) -> PyResult<PyComparisonValue>; +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[repr(transparent)] +pub struct PyComparisonOp(ComparisonOperator); + +impl From<ComparisonOperator> for PyComparisonOp { + fn from(op: ComparisonOperator) -> Self { + Self(op) + } +} + +#[allow(non_upper_case_globals)] +impl PyComparisonOp { + pub const Lt: Self = Self(ComparisonOperator::Less); + pub const Gt: Self = Self(ComparisonOperator::Greater); + pub const Ne: Self = Self(ComparisonOperator::NotEqual); + pub const Eq: Self = Self(ComparisonOperator::Equal); + pub const Le: Self = Self(ComparisonOperator::LessOrEqual); + pub const Ge: Self = Self(ComparisonOperator::GreaterOrEqual); +} + +impl PyComparisonOp { + pub fn eq_only( + self, + f: impl FnOnce() -> PyResult<PyComparisonValue>, + ) -> PyResult<PyComparisonValue> { + match self { + Self::Eq => f(), + Self::Ne => f().map(|x| x.map(|eq| !eq)), + _ => Ok(PyComparisonValue::NotImplemented), + } + } + + pub fn eval_ord(self, ord: Ordering) -> bool { + let bit = match ord { + Ordering::Less => Self::Lt, + Ordering::Equal => Self::Eq, + Ordering::Greater => Self::Gt, + }; + u8::from(self.0) & u8::from(bit.0) != 0 + } + + pub const fn swapped(self) -> Self { + match self { + Self::Lt => Self::Gt, + Self::Le => Self::Ge, + Self::Eq => Self::Eq, + Self::Ne => Self::Ne, + Self::Ge => Self::Le, + Self::Gt => Self::Lt, + } + } + + pub fn method_name(self, ctx: &Context) -> &'static PyStrInterned { + match self { + Self::Lt => identifier!(ctx, __lt__), + Self::Le => identifier!(ctx, __le__), + Self::Eq => identifier!(ctx, __eq__), + Self::Ne => identifier!(ctx, __ne__), + Self::Ge => identifier!(ctx, __ge__), + Self::Gt => identifier!(ctx, __gt__), + } + } + + pub const fn operator_token(self) -> &'static str { + match self { + Self::Lt => "<", + Self::Le => "<=", + Self::Eq => "==", + Self::Ne => "!=", + Self::Ge => ">=", + Self::Gt => ">", + } + } + + /// Returns an appropriate return value for the comparison when a and b are the same object, if an + /// appropriate return value exists. + #[inline] + pub fn identical_optimization( + self, + a: &impl Borrow<PyObject>, + b: &impl Borrow<PyObject>, + ) -> Option<bool> { + self.map_eq(|| a.borrow().is(b.borrow())) + } + + /// Returns `Some(true)` when self is `Eq` and `f()` returns true. Returns `Some(false)` when self + /// is `Ne` and `f()` returns true. Otherwise returns `None`. + #[inline] + pub fn map_eq(self, f: impl FnOnce() -> bool) -> Option<bool> { + let eq = match self { + Self::Eq => true, + Self::Ne => false, + _ => return None, + }; + f().then_some(eq) + } +} + +#[pyclass] +pub trait GetAttr: PyPayload { + #[pyslot] + fn slot_getattro(obj: &PyObject, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { + let zelf = obj + .downcast_ref() + .ok_or_else(|| vm.new_type_error("unexpected payload for __getattribute__"))?; + Self::getattro(zelf, name, vm) + } + + fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult; +} + +#[pyclass] +pub trait SetAttr: PyPayload { + #[pyslot] + #[inline] + fn slot_setattro( + obj: &PyObject, + name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()> { + let zelf = obj + .downcast_ref::<Self>() + .ok_or_else(|| vm.new_type_error("unexpected payload for __setattr__"))?; + Self::setattro(zelf, name, value, vm) + } + + fn setattro( + zelf: &Py<Self>, + name: &Py<PyStr>, + value: PySetterValue, + vm: &VirtualMachine, + ) -> PyResult<()>; +} + +#[pyclass] +pub trait AsBuffer: PyPayload { + // TODO: `flags` parameter + #[inline] + #[pyslot] + fn slot_as_buffer(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyBuffer> { + let zelf = zelf + .downcast_ref() + .ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?; + Self::as_buffer(zelf, vm) + } + + fn as_buffer(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyBuffer>; +} + +#[pyclass] +pub trait AsMapping: PyPayload { + fn as_mapping() -> &'static PyMappingMethods; + + #[inline] + fn mapping_downcast(mapping: PyMapping<'_>) -> &Py<Self> { + unsafe { mapping.obj.downcast_unchecked_ref() } + } + + fn extend_slots(slots: &mut PyTypeSlots) { + slots.as_mapping.copy_from(Self::as_mapping()); + } +} + +#[pyclass] +pub trait AsSequence: PyPayload { + fn as_sequence() -> &'static PySequenceMethods; + + #[inline] + fn sequence_downcast(seq: PySequence<'_>) -> &Py<Self> { + unsafe { seq.obj.downcast_unchecked_ref() } + } + + fn extend_slots(slots: &mut PyTypeSlots) { + slots.as_sequence.copy_from(Self::as_sequence()); + } +} + +#[pyclass] +pub trait AsNumber: PyPayload { + #[pyslot] + fn as_number() -> &'static PyNumberMethods; + + fn extend_slots(slots: &mut PyTypeSlots) { + slots.as_number.copy_from(Self::as_number()); + } + + fn clone_exact(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyRef<Self> { + // not all AsNumber requires this implementation. + unimplemented!() + } + + #[inline] + fn number_downcast(num: PyNumber<'_>) -> &Py<Self> { + unsafe { num.obj.downcast_unchecked_ref() } + } + + #[inline] + fn number_downcast_exact(num: PyNumber<'_>, vm: &VirtualMachine) -> PyRef<Self> { + if let Some(zelf) = num.downcast_ref_if_exact::<Self>(vm) { + zelf.to_owned() + } else { + Self::clone_exact(Self::number_downcast(num), vm) + } + } +} + +#[pyclass] +pub trait Iterable: PyPayload { + #[pyslot] + fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = zelf + .downcast() + .map_err(|_| vm.new_type_error("unexpected payload for __iter__"))?; + Self::iter(zelf, vm) + } + + fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult; + + fn extend_slots(_slots: &mut PyTypeSlots) {} +} + +// `Iterator` fits better, but to avoid confusion with rust core::iter::Iterator +#[pyclass(with(Iterable))] +pub trait IterNext: PyPayload + Iterable { + #[pyslot] + fn slot_iternext(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyIterReturn> { + let zelf = zelf + .downcast_ref() + .ok_or_else(|| vm.new_type_error("unexpected payload for __next__"))?; + Self::next(zelf, vm) + } + + fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn>; +} + +pub trait SelfIter: PyPayload {} + +impl<T> Iterable for T +where + T: SelfIter, +{ + #[cold] + fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let repr = zelf.repr(vm)?; + unreachable!("slot must be overridden for {}", repr.as_wtf8()); + } + + #[cold] + fn iter(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { + unreachable!("slot_iter is implemented"); + } + + fn extend_slots(slots: &mut PyTypeSlots) { + let prev = slots.iter.swap(Some(self_iter)); + debug_assert!(prev.is_some()); // slot_iter would be set + } +} diff --git a/crates/vm/src/types/slot_defs.rs b/crates/vm/src/types/slot_defs.rs new file mode 100644 index 00000000000..efbcb9f55f6 --- /dev/null +++ b/crates/vm/src/types/slot_defs.rs @@ -0,0 +1,1503 @@ +//! Slot definitions array +//! +//! This module provides a centralized array of all slot definitions, + +use super::{PyComparisonOp, PyTypeSlots}; +use crate::builtins::descriptor::SlotFunc; + +/// Slot operation type +/// +/// Used to distinguish between different operations that share the same slot: +/// - RichCompare: Lt, Le, Eq, Ne, Gt, Ge +/// - Binary ops: Left (__add__) vs Right (__radd__) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SlotOp { + // RichCompare operations + Lt, + Le, + Eq, + Ne, + Gt, + Ge, + // Binary operation direction + Left, + Right, + // Setter vs Deleter + Delete, +} + +impl SlotOp { + /// Convert to PyComparisonOp if this is a comparison operation + pub fn as_compare_op(&self) -> Option<PyComparisonOp> { + match self { + Self::Lt => Some(PyComparisonOp::Lt), + Self::Le => Some(PyComparisonOp::Le), + Self::Eq => Some(PyComparisonOp::Eq), + Self::Ne => Some(PyComparisonOp::Ne), + Self::Gt => Some(PyComparisonOp::Gt), + Self::Ge => Some(PyComparisonOp::Ge), + _ => None, + } + } + + /// Check if this is a right operation (__radd__, __rsub__, etc.) + pub fn is_right(&self) -> bool { + matches!(self, Self::Right) + } +} + +/// Slot definition entry +#[derive(Clone, Copy)] +pub struct SlotDef { + /// Method name ("__init__", "__add__", etc.) + pub name: &'static str, + + /// Slot accessor (which slot field to access) + pub accessor: SlotAccessor, + + /// Operation type (for shared slots like RichCompare, binary ops) + pub op: Option<SlotOp>, + + /// Documentation string + pub doc: &'static str, +} + +/// Slot accessor +/// +/// Values match CPython's Py_* slot IDs from typeslots.h. +/// Unused slots are included for value reservation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum SlotAccessor { + // Buffer protocol (1-2) - Reserved, not used in RustPython + BfGetBuffer = 1, + BfReleaseBuffer = 2, + + // Mapping protocol (3-5) + MpAssSubscript = 3, + MpLength = 4, + MpSubscript = 5, + + // Number protocol (6-38) + NbAbsolute = 6, + NbAdd = 7, + NbAnd = 8, + NbBool = 9, + NbDivmod = 10, + NbFloat = 11, + NbFloorDivide = 12, + NbIndex = 13, + NbInplaceAdd = 14, + NbInplaceAnd = 15, + NbInplaceFloorDivide = 16, + NbInplaceLshift = 17, + NbInplaceMultiply = 18, + NbInplaceOr = 19, + NbInplacePower = 20, + NbInplaceRemainder = 21, + NbInplaceRshift = 22, + NbInplaceSubtract = 23, + NbInplaceTrueDivide = 24, + NbInplaceXor = 25, + NbInt = 26, + NbInvert = 27, + NbLshift = 28, + NbMultiply = 29, + NbNegative = 30, + NbOr = 31, + NbPositive = 32, + NbPower = 33, + NbRemainder = 34, + NbRshift = 35, + NbSubtract = 36, + NbTrueDivide = 37, + NbXor = 38, + + // Sequence protocol (39-46) + SqAssItem = 39, + SqConcat = 40, + SqContains = 41, + SqInplaceConcat = 42, + SqInplaceRepeat = 43, + SqItem = 44, + SqLength = 45, + SqRepeat = 46, + + // Type slots (47-74) + TpAlloc = 47, // Reserved + TpBase = 48, // Reserved + TpBases = 49, // Reserved + TpCall = 50, + TpClear = 51, // Reserved + TpDealloc = 52, // Reserved + TpDel = 53, + TpDescrGet = 54, + TpDescrSet = 55, + TpDoc = 56, // Reserved + TpGetattr = 57, // Reserved (use TpGetattro) + TpGetattro = 58, + TpHash = 59, + TpInit = 60, + TpIsGc = 61, // Reserved + TpIter = 62, + TpIternext = 63, + TpMethods = 64, // Reserved + TpNew = 65, + TpRepr = 66, + TpRichcompare = 67, + TpSetattr = 68, // Reserved (use TpSetattro) + TpSetattro = 69, + TpStr = 70, + TpTraverse = 71, // Reserved + TpMembers = 72, // Reserved + TpGetset = 73, // Reserved + TpFree = 74, // Reserved + + // Number protocol additions (75-76) + NbMatrixMultiply = 75, + NbInplaceMatrixMultiply = 76, + + // Async protocol (77-81) - Reserved for future + AmAwait = 77, + AmAiter = 78, + AmAnext = 79, + TpFinalize = 80, + AmSend = 81, +} + +impl SlotAccessor { + /// Check if this accessor is for a reserved/unused slot + pub fn is_reserved(&self) -> bool { + matches!( + self, + Self::BfGetBuffer + | Self::BfReleaseBuffer + | Self::TpAlloc + | Self::TpBase + | Self::TpBases + | Self::TpClear + | Self::TpDealloc + | Self::TpDoc + | Self::TpGetattr + | Self::TpIsGc + | Self::TpMethods + | Self::TpSetattr + | Self::TpTraverse + | Self::TpMembers + | Self::TpGetset + | Self::TpFree + | Self::TpFinalize + | Self::AmAwait + | Self::AmAiter + | Self::AmAnext + | Self::AmSend + ) + } + + /// Check if this is a number binary operation slot + pub fn is_number_binary(&self) -> bool { + matches!( + self, + Self::NbAdd + | Self::NbSubtract + | Self::NbMultiply + | Self::NbRemainder + | Self::NbDivmod + | Self::NbPower + | Self::NbLshift + | Self::NbRshift + | Self::NbAnd + | Self::NbXor + | Self::NbOr + | Self::NbFloorDivide + | Self::NbTrueDivide + | Self::NbMatrixMultiply + ) + } + + /// Check if this accessor refers to a shared slot + /// + /// Shared slots are used by multiple dunder methods: + /// - TpSetattro: __setattr__ and __delattr__ + /// - TpRichcompare: __lt__, __le__, __eq__, __ne__, __gt__, __ge__ + /// - TpDescrSet: __set__ and __delete__ + /// - SqAssItem/MpAssSubscript: __setitem__ and __delitem__ + /// - Number binaries: __add__ and __radd__, etc. + pub fn is_shared_slot(&self) -> bool { + matches!( + self, + Self::TpSetattro + | Self::TpRichcompare + | Self::TpDescrSet + | Self::SqAssItem + | Self::MpAssSubscript + ) || self.is_number_binary() + } + + /// Get underlying slot field name for debugging + pub fn slot_name(&self) -> &'static str { + match self { + Self::BfGetBuffer => "bf_getbuffer", + Self::BfReleaseBuffer => "bf_releasebuffer", + Self::MpAssSubscript => "mp_ass_subscript", + Self::MpLength => "mp_length", + Self::MpSubscript => "mp_subscript", + Self::NbAbsolute => "nb_absolute", + Self::NbAdd => "nb_add", + Self::NbAnd => "nb_and", + Self::NbBool => "nb_bool", + Self::NbDivmod => "nb_divmod", + Self::NbFloat => "nb_float", + Self::NbFloorDivide => "nb_floor_divide", + Self::NbIndex => "nb_index", + Self::NbInplaceAdd => "nb_inplace_add", + Self::NbInplaceAnd => "nb_inplace_and", + Self::NbInplaceFloorDivide => "nb_inplace_floor_divide", + Self::NbInplaceLshift => "nb_inplace_lshift", + Self::NbInplaceMultiply => "nb_inplace_multiply", + Self::NbInplaceOr => "nb_inplace_or", + Self::NbInplacePower => "nb_inplace_power", + Self::NbInplaceRemainder => "nb_inplace_remainder", + Self::NbInplaceRshift => "nb_inplace_rshift", + Self::NbInplaceSubtract => "nb_inplace_subtract", + Self::NbInplaceTrueDivide => "nb_inplace_true_divide", + Self::NbInplaceXor => "nb_inplace_xor", + Self::NbInt => "nb_int", + Self::NbInvert => "nb_invert", + Self::NbLshift => "nb_lshift", + Self::NbMultiply => "nb_multiply", + Self::NbNegative => "nb_negative", + Self::NbOr => "nb_or", + Self::NbPositive => "nb_positive", + Self::NbPower => "nb_power", + Self::NbRemainder => "nb_remainder", + Self::NbRshift => "nb_rshift", + Self::NbSubtract => "nb_subtract", + Self::NbTrueDivide => "nb_true_divide", + Self::NbXor => "nb_xor", + Self::SqAssItem => "sq_ass_item", + Self::SqConcat => "sq_concat", + Self::SqContains => "sq_contains", + Self::SqInplaceConcat => "sq_inplace_concat", + Self::SqInplaceRepeat => "sq_inplace_repeat", + Self::SqItem => "sq_item", + Self::SqLength => "sq_length", + Self::SqRepeat => "sq_repeat", + Self::TpAlloc => "tp_alloc", + Self::TpBase => "tp_base", + Self::TpBases => "tp_bases", + Self::TpCall => "tp_call", + Self::TpClear => "tp_clear", + Self::TpDealloc => "tp_dealloc", + Self::TpDel => "tp_del", + Self::TpDescrGet => "tp_descr_get", + Self::TpDescrSet => "tp_descr_set", + Self::TpDoc => "tp_doc", + Self::TpGetattr => "tp_getattr", + Self::TpGetattro => "tp_getattro", + Self::TpHash => "tp_hash", + Self::TpInit => "tp_init", + Self::TpIsGc => "tp_is_gc", + Self::TpIter => "tp_iter", + Self::TpIternext => "tp_iternext", + Self::TpMethods => "tp_methods", + Self::TpNew => "tp_new", + Self::TpRepr => "tp_repr", + Self::TpRichcompare => "tp_richcompare", + Self::TpSetattr => "tp_setattr", + Self::TpSetattro => "tp_setattro", + Self::TpStr => "tp_str", + Self::TpTraverse => "tp_traverse", + Self::TpMembers => "tp_members", + Self::TpGetset => "tp_getset", + Self::TpFree => "tp_free", + Self::NbMatrixMultiply => "nb_matrix_multiply", + Self::NbInplaceMatrixMultiply => "nb_inplace_matrix_multiply", + Self::AmAwait => "am_await", + Self::AmAiter => "am_aiter", + Self::AmAnext => "am_anext", + Self::TpFinalize => "tp_finalize", + Self::AmSend => "am_send", + } + } + + /// Extract the raw function pointer from a SlotFunc if it matches this accessor's type + pub fn extract_from_slot_func(&self, slot_func: &SlotFunc) -> bool { + match self { + // Type slots + Self::TpHash => matches!(slot_func, SlotFunc::Hash(_)), + Self::TpRepr => matches!(slot_func, SlotFunc::Repr(_)), + Self::TpStr => matches!(slot_func, SlotFunc::Str(_)), + Self::TpCall => matches!(slot_func, SlotFunc::Call(_)), + Self::TpIter => matches!(slot_func, SlotFunc::Iter(_)), + Self::TpIternext => matches!(slot_func, SlotFunc::IterNext(_)), + Self::TpInit => matches!(slot_func, SlotFunc::Init(_)), + Self::TpDel => matches!(slot_func, SlotFunc::Del(_)), + Self::TpGetattro => matches!(slot_func, SlotFunc::GetAttro(_)), + Self::TpSetattro => { + matches!(slot_func, SlotFunc::SetAttro(_) | SlotFunc::DelAttro(_)) + } + Self::TpDescrGet => matches!(slot_func, SlotFunc::DescrGet(_)), + Self::TpDescrSet => { + matches!(slot_func, SlotFunc::DescrSet(_) | SlotFunc::DescrDel(_)) + } + Self::TpRichcompare => matches!(slot_func, SlotFunc::RichCompare(_, _)), + + // Number - Power (ternary) + Self::NbPower | Self::NbInplacePower => { + matches!(slot_func, SlotFunc::NumTernary(_)) + } + // Number - Boolean + Self::NbBool => matches!(slot_func, SlotFunc::NumBoolean(_)), + // Number - Unary + Self::NbNegative + | Self::NbPositive + | Self::NbAbsolute + | Self::NbInvert + | Self::NbInt + | Self::NbFloat + | Self::NbIndex => matches!(slot_func, SlotFunc::NumUnary(_)), + // Number - Binary + Self::NbAdd + | Self::NbSubtract + | Self::NbMultiply + | Self::NbRemainder + | Self::NbDivmod + | Self::NbLshift + | Self::NbRshift + | Self::NbAnd + | Self::NbXor + | Self::NbOr + | Self::NbFloorDivide + | Self::NbTrueDivide + | Self::NbMatrixMultiply + | Self::NbInplaceAdd + | Self::NbInplaceSubtract + | Self::NbInplaceMultiply + | Self::NbInplaceRemainder + | Self::NbInplaceLshift + | Self::NbInplaceRshift + | Self::NbInplaceAnd + | Self::NbInplaceXor + | Self::NbInplaceOr + | Self::NbInplaceFloorDivide + | Self::NbInplaceTrueDivide + | Self::NbInplaceMatrixMultiply => matches!(slot_func, SlotFunc::NumBinary(_)), + + // Sequence + Self::SqLength => matches!(slot_func, SlotFunc::SeqLength(_)), + Self::SqConcat | Self::SqInplaceConcat => matches!(slot_func, SlotFunc::SeqConcat(_)), + Self::SqRepeat | Self::SqInplaceRepeat => matches!(slot_func, SlotFunc::SeqRepeat(_)), + Self::SqItem => matches!(slot_func, SlotFunc::SeqItem(_)), + Self::SqAssItem => { + matches!(slot_func, SlotFunc::SeqSetItem(_) | SlotFunc::SeqDelItem(_)) + } + Self::SqContains => matches!(slot_func, SlotFunc::SeqContains(_)), + + // Mapping + Self::MpLength => matches!(slot_func, SlotFunc::MapLength(_)), + Self::MpSubscript => matches!(slot_func, SlotFunc::MapSubscript(_)), + Self::MpAssSubscript => { + matches!( + slot_func, + SlotFunc::MapSetSubscript(_) | SlotFunc::MapDelSubscript(_) + ) + } + + // New and reserved slots + Self::TpNew => false, + _ => false, // Reserved slots + } + } + + /// Inherit slot value from MRO + pub fn inherit_from_mro(&self, typ: &crate::builtins::PyType) { + // mro[0] is self, so skip it + let mro_guard = typ.mro.read(); + let mro = &mro_guard[1..]; + + macro_rules! inherit_main { + ($slot:ident) => {{ + let inherited = mro.iter().find_map(|cls| cls.slots.$slot.load()); + typ.slots.$slot.store(inherited); + }}; + } + + macro_rules! inherit_number { + ($slot:ident) => {{ + let inherited = mro.iter().find_map(|cls| cls.slots.as_number.$slot.load()); + typ.slots.as_number.$slot.store(inherited); + }}; + } + + macro_rules! inherit_sequence { + ($slot:ident) => {{ + let inherited = mro + .iter() + .find_map(|cls| cls.slots.as_sequence.$slot.load()); + typ.slots.as_sequence.$slot.store(inherited); + }}; + } + + macro_rules! inherit_mapping { + ($slot:ident) => {{ + let inherited = mro.iter().find_map(|cls| cls.slots.as_mapping.$slot.load()); + typ.slots.as_mapping.$slot.store(inherited); + }}; + } + + match self { + // Type slots + Self::TpHash => inherit_main!(hash), + Self::TpRepr => inherit_main!(repr), + Self::TpStr => inherit_main!(str), + Self::TpCall => { + inherit_main!(call); + inherit_main!(vectorcall); + } + Self::TpIter => inherit_main!(iter), + Self::TpIternext => inherit_main!(iternext), + Self::TpInit => inherit_main!(init), + Self::TpNew => inherit_main!(new), + Self::TpDel => inherit_main!(del), + Self::TpGetattro => inherit_main!(getattro), + Self::TpSetattro => inherit_main!(setattro), + Self::TpDescrGet => inherit_main!(descr_get), + Self::TpDescrSet => inherit_main!(descr_set), + Self::TpRichcompare => inherit_main!(richcompare), + + // Number slots + Self::NbAdd => inherit_number!(add), + Self::NbSubtract => inherit_number!(subtract), + Self::NbMultiply => inherit_number!(multiply), + Self::NbRemainder => inherit_number!(remainder), + Self::NbDivmod => inherit_number!(divmod), + Self::NbPower => inherit_number!(power), + Self::NbLshift => inherit_number!(lshift), + Self::NbRshift => inherit_number!(rshift), + Self::NbAnd => inherit_number!(and), + Self::NbXor => inherit_number!(xor), + Self::NbOr => inherit_number!(or), + Self::NbFloorDivide => inherit_number!(floor_divide), + Self::NbTrueDivide => inherit_number!(true_divide), + Self::NbMatrixMultiply => inherit_number!(matrix_multiply), + Self::NbInplaceAdd => inherit_number!(inplace_add), + Self::NbInplaceSubtract => inherit_number!(inplace_subtract), + Self::NbInplaceMultiply => inherit_number!(inplace_multiply), + Self::NbInplaceRemainder => inherit_number!(inplace_remainder), + Self::NbInplacePower => inherit_number!(inplace_power), + Self::NbInplaceLshift => inherit_number!(inplace_lshift), + Self::NbInplaceRshift => inherit_number!(inplace_rshift), + Self::NbInplaceAnd => inherit_number!(inplace_and), + Self::NbInplaceXor => inherit_number!(inplace_xor), + Self::NbInplaceOr => inherit_number!(inplace_or), + Self::NbInplaceFloorDivide => inherit_number!(inplace_floor_divide), + Self::NbInplaceTrueDivide => inherit_number!(inplace_true_divide), + Self::NbInplaceMatrixMultiply => inherit_number!(inplace_matrix_multiply), + // Number unary + Self::NbNegative => inherit_number!(negative), + Self::NbPositive => inherit_number!(positive), + Self::NbAbsolute => inherit_number!(absolute), + Self::NbInvert => inherit_number!(invert), + Self::NbBool => inherit_number!(boolean), + Self::NbInt => inherit_number!(int), + Self::NbFloat => inherit_number!(float), + Self::NbIndex => inherit_number!(index), + + // Sequence slots + Self::SqLength => inherit_sequence!(length), + Self::SqConcat => inherit_sequence!(concat), + Self::SqRepeat => inherit_sequence!(repeat), + Self::SqItem => inherit_sequence!(item), + Self::SqAssItem => inherit_sequence!(ass_item), + Self::SqContains => inherit_sequence!(contains), + Self::SqInplaceConcat => inherit_sequence!(inplace_concat), + Self::SqInplaceRepeat => inherit_sequence!(inplace_repeat), + + // Mapping slots + Self::MpLength => inherit_mapping!(length), + Self::MpSubscript => inherit_mapping!(subscript), + Self::MpAssSubscript => inherit_mapping!(ass_subscript), + + // Reserved slots - no-op + _ => {} + } + } + + /// Copy slot from base type if self's slot is None + pub fn copyslot_if_none(&self, typ: &crate::builtins::PyType, base: &crate::builtins::PyType) { + macro_rules! copy_main { + ($slot:ident) => {{ + if typ.slots.$slot.load().is_none() { + if let Some(base_val) = base.slots.$slot.load() { + typ.slots.$slot.store(Some(base_val)); + } + } + }}; + } + + macro_rules! copy_number { + ($slot:ident) => {{ + if typ.slots.as_number.$slot.load().is_none() { + if let Some(base_val) = base.slots.as_number.$slot.load() { + typ.slots.as_number.$slot.store(Some(base_val)); + } + } + }}; + } + + macro_rules! copy_sequence { + ($slot:ident) => {{ + if typ.slots.as_sequence.$slot.load().is_none() { + if let Some(base_val) = base.slots.as_sequence.$slot.load() { + typ.slots.as_sequence.$slot.store(Some(base_val)); + } + } + }}; + } + + macro_rules! copy_mapping { + ($slot:ident) => {{ + if typ.slots.as_mapping.$slot.load().is_none() { + if let Some(base_val) = base.slots.as_mapping.$slot.load() { + typ.slots.as_mapping.$slot.store(Some(base_val)); + } + } + }}; + } + + match self { + // Type slots + Self::TpHash => copy_main!(hash), + Self::TpRepr => copy_main!(repr), + Self::TpStr => copy_main!(str), + Self::TpCall => { + copy_main!(call); + copy_main!(vectorcall); + } + Self::TpIter => copy_main!(iter), + Self::TpIternext => copy_main!(iternext), + Self::TpInit => { + // SLOTDEFINED check for multiple inheritance support + if typ.slots.init.load().is_none() + && let Some(base_val) = base.slots.init.load() + { + let slot_defined = base.base.as_ref().is_none_or(|bb| { + bb.slots.init.load().map(|v| v as usize) != Some(base_val as usize) + }); + if slot_defined { + typ.slots.init.store(Some(base_val)); + } + } + } + Self::TpNew => {} // handled by set_new() + Self::TpDel => copy_main!(del), + Self::TpGetattro => copy_main!(getattro), + Self::TpSetattro => copy_main!(setattro), + Self::TpDescrGet => copy_main!(descr_get), + Self::TpDescrSet => copy_main!(descr_set), + Self::TpRichcompare => copy_main!(richcompare), + + // Number slots + Self::NbAdd => copy_number!(add), + Self::NbSubtract => copy_number!(subtract), + Self::NbMultiply => copy_number!(multiply), + Self::NbRemainder => copy_number!(remainder), + Self::NbDivmod => copy_number!(divmod), + Self::NbPower => copy_number!(power), + Self::NbLshift => copy_number!(lshift), + Self::NbRshift => copy_number!(rshift), + Self::NbAnd => copy_number!(and), + Self::NbXor => copy_number!(xor), + Self::NbOr => copy_number!(or), + Self::NbFloorDivide => copy_number!(floor_divide), + Self::NbTrueDivide => copy_number!(true_divide), + Self::NbMatrixMultiply => copy_number!(matrix_multiply), + Self::NbInplaceAdd => copy_number!(inplace_add), + Self::NbInplaceSubtract => copy_number!(inplace_subtract), + Self::NbInplaceMultiply => copy_number!(inplace_multiply), + Self::NbInplaceRemainder => copy_number!(inplace_remainder), + Self::NbInplacePower => copy_number!(inplace_power), + Self::NbInplaceLshift => copy_number!(inplace_lshift), + Self::NbInplaceRshift => copy_number!(inplace_rshift), + Self::NbInplaceAnd => copy_number!(inplace_and), + Self::NbInplaceXor => copy_number!(inplace_xor), + Self::NbInplaceOr => copy_number!(inplace_or), + Self::NbInplaceFloorDivide => copy_number!(inplace_floor_divide), + Self::NbInplaceTrueDivide => copy_number!(inplace_true_divide), + Self::NbInplaceMatrixMultiply => copy_number!(inplace_matrix_multiply), + // Number unary + Self::NbNegative => copy_number!(negative), + Self::NbPositive => copy_number!(positive), + Self::NbAbsolute => copy_number!(absolute), + Self::NbInvert => copy_number!(invert), + Self::NbBool => copy_number!(boolean), + Self::NbInt => copy_number!(int), + Self::NbFloat => copy_number!(float), + Self::NbIndex => copy_number!(index), + + // Sequence slots + Self::SqLength => copy_sequence!(length), + Self::SqConcat => copy_sequence!(concat), + Self::SqRepeat => copy_sequence!(repeat), + Self::SqItem => copy_sequence!(item), + Self::SqAssItem => copy_sequence!(ass_item), + Self::SqContains => copy_sequence!(contains), + Self::SqInplaceConcat => copy_sequence!(inplace_concat), + Self::SqInplaceRepeat => copy_sequence!(inplace_repeat), + + // Mapping slots + Self::MpLength => copy_mapping!(length), + Self::MpSubscript => copy_mapping!(subscript), + Self::MpAssSubscript => copy_mapping!(ass_subscript), + + // Reserved slots - no-op + _ => {} + } + } + + /// Get the SlotFunc from type slots for this accessor + pub fn get_slot_func(&self, slots: &PyTypeSlots) -> Option<SlotFunc> { + match self { + // Type slots + Self::TpHash => slots.hash.load().map(SlotFunc::Hash), + Self::TpRepr => slots.repr.load().map(SlotFunc::Repr), + Self::TpStr => slots.str.load().map(SlotFunc::Str), + Self::TpCall => slots.call.load().map(SlotFunc::Call), + Self::TpIter => slots.iter.load().map(SlotFunc::Iter), + Self::TpIternext => slots.iternext.load().map(SlotFunc::IterNext), + Self::TpInit => slots.init.load().map(SlotFunc::Init), + Self::TpNew => None, // __new__ handled separately + Self::TpDel => slots.del.load().map(SlotFunc::Del), + Self::TpGetattro => slots.getattro.load().map(SlotFunc::GetAttro), + Self::TpSetattro => slots.setattro.load().map(SlotFunc::SetAttro), + Self::TpDescrGet => slots.descr_get.load().map(SlotFunc::DescrGet), + Self::TpDescrSet => slots.descr_set.load().map(SlotFunc::DescrSet), + Self::TpRichcompare => slots + .richcompare + .load() + .map(|f| SlotFunc::RichCompare(f, PyComparisonOp::Eq)), + + // Number binary slots + Self::NbAdd => slots.as_number.add.load().map(SlotFunc::NumBinary), + Self::NbSubtract => slots.as_number.subtract.load().map(SlotFunc::NumBinary), + Self::NbMultiply => slots.as_number.multiply.load().map(SlotFunc::NumBinary), + Self::NbRemainder => slots.as_number.remainder.load().map(SlotFunc::NumBinary), + Self::NbDivmod => slots.as_number.divmod.load().map(SlotFunc::NumBinary), + Self::NbPower => slots.as_number.power.load().map(SlotFunc::NumTernary), + Self::NbLshift => slots.as_number.lshift.load().map(SlotFunc::NumBinary), + Self::NbRshift => slots.as_number.rshift.load().map(SlotFunc::NumBinary), + Self::NbAnd => slots.as_number.and.load().map(SlotFunc::NumBinary), + Self::NbXor => slots.as_number.xor.load().map(SlotFunc::NumBinary), + Self::NbOr => slots.as_number.or.load().map(SlotFunc::NumBinary), + Self::NbFloorDivide => slots.as_number.floor_divide.load().map(SlotFunc::NumBinary), + Self::NbTrueDivide => slots.as_number.true_divide.load().map(SlotFunc::NumBinary), + Self::NbMatrixMultiply => slots + .as_number + .matrix_multiply + .load() + .map(SlotFunc::NumBinary), + + // Number inplace slots + Self::NbInplaceAdd => slots.as_number.inplace_add.load().map(SlotFunc::NumBinary), + Self::NbInplaceSubtract => slots + .as_number + .inplace_subtract + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceMultiply => slots + .as_number + .inplace_multiply + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceRemainder => slots + .as_number + .inplace_remainder + .load() + .map(SlotFunc::NumBinary), + Self::NbInplacePower => slots + .as_number + .inplace_power + .load() + .map(SlotFunc::NumTernary), + Self::NbInplaceLshift => slots + .as_number + .inplace_lshift + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceRshift => slots + .as_number + .inplace_rshift + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceAnd => slots.as_number.inplace_and.load().map(SlotFunc::NumBinary), + Self::NbInplaceXor => slots.as_number.inplace_xor.load().map(SlotFunc::NumBinary), + Self::NbInplaceOr => slots.as_number.inplace_or.load().map(SlotFunc::NumBinary), + Self::NbInplaceFloorDivide => slots + .as_number + .inplace_floor_divide + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceTrueDivide => slots + .as_number + .inplace_true_divide + .load() + .map(SlotFunc::NumBinary), + Self::NbInplaceMatrixMultiply => slots + .as_number + .inplace_matrix_multiply + .load() + .map(SlotFunc::NumBinary), + + // Number unary slots + Self::NbNegative => slots.as_number.negative.load().map(SlotFunc::NumUnary), + Self::NbPositive => slots.as_number.positive.load().map(SlotFunc::NumUnary), + Self::NbAbsolute => slots.as_number.absolute.load().map(SlotFunc::NumUnary), + Self::NbInvert => slots.as_number.invert.load().map(SlotFunc::NumUnary), + Self::NbBool => slots.as_number.boolean.load().map(SlotFunc::NumBoolean), + Self::NbInt => slots.as_number.int.load().map(SlotFunc::NumUnary), + Self::NbFloat => slots.as_number.float.load().map(SlotFunc::NumUnary), + Self::NbIndex => slots.as_number.index.load().map(SlotFunc::NumUnary), + + // Sequence slots + Self::SqLength => slots.as_sequence.length.load().map(SlotFunc::SeqLength), + Self::SqConcat => slots.as_sequence.concat.load().map(SlotFunc::SeqConcat), + Self::SqRepeat => slots.as_sequence.repeat.load().map(SlotFunc::SeqRepeat), + Self::SqItem => slots.as_sequence.item.load().map(SlotFunc::SeqItem), + Self::SqAssItem => slots.as_sequence.ass_item.load().map(SlotFunc::SeqSetItem), + Self::SqContains => slots.as_sequence.contains.load().map(SlotFunc::SeqContains), + Self::SqInplaceConcat => slots + .as_sequence + .inplace_concat + .load() + .map(SlotFunc::SeqConcat), + Self::SqInplaceRepeat => slots + .as_sequence + .inplace_repeat + .load() + .map(SlotFunc::SeqRepeat), + + // Mapping slots + Self::MpLength => slots.as_mapping.length.load().map(SlotFunc::MapLength), + Self::MpSubscript => slots + .as_mapping + .subscript + .load() + .map(SlotFunc::MapSubscript), + Self::MpAssSubscript => slots + .as_mapping + .ass_subscript + .load() + .map(SlotFunc::MapSetSubscript), + + // Reserved slots + _ => None, + } + } + + /// Get slot function considering SlotOp for right-hand and delete operations + pub fn get_slot_func_with_op( + &self, + slots: &PyTypeSlots, + op: Option<SlotOp>, + ) -> Option<SlotFunc> { + // For Delete operations, return the delete variant + if op == Some(SlotOp::Delete) { + match self { + Self::TpSetattro => return slots.setattro.load().map(SlotFunc::DelAttro), + Self::TpDescrSet => return slots.descr_set.load().map(SlotFunc::DescrDel), + Self::SqAssItem => { + return slots.as_sequence.ass_item.load().map(SlotFunc::SeqDelItem); + } + Self::MpAssSubscript => { + return slots + .as_mapping + .ass_subscript + .load() + .map(SlotFunc::MapDelSubscript); + } + _ => {} + } + } + // For Right operations on binary number slots, use right_* fields with swapped args + if op == Some(SlotOp::Right) { + match self { + Self::NbAdd => { + return slots + .as_number + .right_add + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbSubtract => { + return slots + .as_number + .right_subtract + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbMultiply => { + return slots + .as_number + .right_multiply + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbRemainder => { + return slots + .as_number + .right_remainder + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbDivmod => { + return slots + .as_number + .right_divmod + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbPower => { + return slots + .as_number + .right_power + .load() + .map(SlotFunc::NumTernaryRight); + } + Self::NbLshift => { + return slots + .as_number + .right_lshift + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbRshift => { + return slots + .as_number + .right_rshift + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbAnd => { + return slots + .as_number + .right_and + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbXor => { + return slots + .as_number + .right_xor + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbOr => { + return slots + .as_number + .right_or + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbFloorDivide => { + return slots + .as_number + .right_floor_divide + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbTrueDivide => { + return slots + .as_number + .right_true_divide + .load() + .map(SlotFunc::NumBinaryRight); + } + Self::NbMatrixMultiply => { + return slots + .as_number + .right_matrix_multiply + .load() + .map(SlotFunc::NumBinaryRight); + } + _ => {} + } + } + // For comparison operations, use the appropriate PyComparisonOp + if let Self::TpRichcompare = self + && let Some(cmp_op) = op.and_then(|o| o.as_compare_op()) + { + return slots + .richcompare + .load() + .map(|f| SlotFunc::RichCompare(f, cmp_op)); + } + // Fall back to existing get_slot_func for left/other operations + self.get_slot_func(slots) + } +} + +/// Find all slot definitions with a given name +pub fn find_slot_defs_by_name(name: &str) -> impl Iterator<Item = &'static SlotDef> { + SLOT_DEFS.iter().filter(move |def| def.name == name) +} + +/// Total number of slot definitions +pub const SLOT_DEFS_COUNT: usize = SLOT_DEFS.len(); + +/// All slot definitions +pub static SLOT_DEFS: &[SlotDef] = &[ + // Type slots (tp_*) + SlotDef { + name: "__init__", + accessor: SlotAccessor::TpInit, + op: None, + doc: "Initialize self. See help(type(self)) for accurate signature.", + }, + SlotDef { + name: "__new__", + accessor: SlotAccessor::TpNew, + op: None, + doc: "Create and return a new object. See help(type) for accurate signature.", + }, + SlotDef { + name: "__del__", + accessor: SlotAccessor::TpDel, + op: None, + doc: "Called when the instance is about to be destroyed.", + }, + SlotDef { + name: "__repr__", + accessor: SlotAccessor::TpRepr, + op: None, + doc: "Return repr(self).", + }, + SlotDef { + name: "__str__", + accessor: SlotAccessor::TpStr, + op: None, + doc: "Return str(self).", + }, + SlotDef { + name: "__hash__", + accessor: SlotAccessor::TpHash, + op: None, + doc: "Return hash(self).", + }, + SlotDef { + name: "__call__", + accessor: SlotAccessor::TpCall, + op: None, + doc: "Call self as a function.", + }, + SlotDef { + name: "__iter__", + accessor: SlotAccessor::TpIter, + op: None, + doc: "Implement iter(self).", + }, + SlotDef { + name: "__next__", + accessor: SlotAccessor::TpIternext, + op: None, + doc: "Implement next(self).", + }, + // Attribute access + SlotDef { + name: "__getattribute__", + accessor: SlotAccessor::TpGetattro, + op: None, + doc: "Return getattr(self, name).", + }, + SlotDef { + name: "__getattr__", + accessor: SlotAccessor::TpGetattro, + op: None, + doc: "Implement getattr(self, name).", + }, + SlotDef { + name: "__setattr__", + accessor: SlotAccessor::TpSetattro, + op: None, + doc: "Implement setattr(self, name, value).", + }, + SlotDef { + name: "__delattr__", + accessor: SlotAccessor::TpSetattro, + op: Some(SlotOp::Delete), + doc: "Implement delattr(self, name).", + }, + // Rich comparison - all map to TpRichcompare with different op + SlotDef { + name: "__eq__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Eq), + doc: "Return self==value.", + }, + SlotDef { + name: "__ne__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Ne), + doc: "Return self!=value.", + }, + SlotDef { + name: "__lt__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Lt), + doc: "Return self<value.", + }, + SlotDef { + name: "__le__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Le), + doc: "Return self<=value.", + }, + SlotDef { + name: "__gt__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Gt), + doc: "Return self>value.", + }, + SlotDef { + name: "__ge__", + accessor: SlotAccessor::TpRichcompare, + op: Some(SlotOp::Ge), + doc: "Return self>=value.", + }, + // Descriptor protocol + SlotDef { + name: "__get__", + accessor: SlotAccessor::TpDescrGet, + op: None, + doc: "Return an attribute of instance, which is of type owner.", + }, + SlotDef { + name: "__set__", + accessor: SlotAccessor::TpDescrSet, + op: None, + doc: "Set an attribute of instance to value.", + }, + SlotDef { + name: "__delete__", + accessor: SlotAccessor::TpDescrSet, + op: Some(SlotOp::Delete), + doc: "Delete an attribute of instance.", + }, + // Mapping protocol (mp_*) - must come before Sequence protocol + // so that mp_subscript wins over sq_item for __getitem__ + // (see CPython typeobject.c:10995-11006) + SlotDef { + name: "__len__", + accessor: SlotAccessor::MpLength, + op: None, + doc: "Return len(self).", + }, + SlotDef { + name: "__getitem__", + accessor: SlotAccessor::MpSubscript, + op: None, + doc: "Return self[key].", + }, + SlotDef { + name: "__setitem__", + accessor: SlotAccessor::MpAssSubscript, + op: None, + doc: "Set self[key] to value.", + }, + SlotDef { + name: "__delitem__", + accessor: SlotAccessor::MpAssSubscript, + op: Some(SlotOp::Delete), + doc: "Delete self[key].", + }, + // Sequence protocol (sq_*) + SlotDef { + name: "__len__", + accessor: SlotAccessor::SqLength, + op: None, + doc: "Return len(self).", + }, + SlotDef { + name: "__getitem__", + accessor: SlotAccessor::SqItem, + op: None, + doc: "Return self[key].", + }, + SlotDef { + name: "__setitem__", + accessor: SlotAccessor::SqAssItem, + op: None, + doc: "Set self[key] to value.", + }, + SlotDef { + name: "__delitem__", + accessor: SlotAccessor::SqAssItem, + op: Some(SlotOp::Delete), + doc: "Delete self[key].", + }, + SlotDef { + name: "__contains__", + accessor: SlotAccessor::SqContains, + op: None, + doc: "Return key in self.", + }, + // Number protocol - binary ops with left/right variants + SlotDef { + name: "__add__", + accessor: SlotAccessor::NbAdd, + op: Some(SlotOp::Left), + doc: "Return self+value.", + }, + SlotDef { + name: "__radd__", + accessor: SlotAccessor::NbAdd, + op: Some(SlotOp::Right), + doc: "Return value+self.", + }, + SlotDef { + name: "__iadd__", + accessor: SlotAccessor::NbInplaceAdd, + op: None, + doc: "Implement self+=value.", + }, + SlotDef { + name: "__sub__", + accessor: SlotAccessor::NbSubtract, + op: Some(SlotOp::Left), + doc: "Return self-value.", + }, + SlotDef { + name: "__rsub__", + accessor: SlotAccessor::NbSubtract, + op: Some(SlotOp::Right), + doc: "Return value-self.", + }, + SlotDef { + name: "__isub__", + accessor: SlotAccessor::NbInplaceSubtract, + op: None, + doc: "Implement self-=value.", + }, + SlotDef { + name: "__mul__", + accessor: SlotAccessor::NbMultiply, + op: Some(SlotOp::Left), + doc: "Return self*value.", + }, + SlotDef { + name: "__rmul__", + accessor: SlotAccessor::NbMultiply, + op: Some(SlotOp::Right), + doc: "Return value*self.", + }, + SlotDef { + name: "__imul__", + accessor: SlotAccessor::NbInplaceMultiply, + op: None, + doc: "Implement self*=value.", + }, + SlotDef { + name: "__mod__", + accessor: SlotAccessor::NbRemainder, + op: Some(SlotOp::Left), + doc: "Return self%value.", + }, + SlotDef { + name: "__rmod__", + accessor: SlotAccessor::NbRemainder, + op: Some(SlotOp::Right), + doc: "Return value%self.", + }, + SlotDef { + name: "__imod__", + accessor: SlotAccessor::NbInplaceRemainder, + op: None, + doc: "Implement self%=value.", + }, + SlotDef { + name: "__divmod__", + accessor: SlotAccessor::NbDivmod, + op: Some(SlotOp::Left), + doc: "Return divmod(self, value).", + }, + SlotDef { + name: "__rdivmod__", + accessor: SlotAccessor::NbDivmod, + op: Some(SlotOp::Right), + doc: "Return divmod(value, self).", + }, + SlotDef { + name: "__pow__", + accessor: SlotAccessor::NbPower, + op: Some(SlotOp::Left), + doc: "Return pow(self, value, mod).", + }, + SlotDef { + name: "__rpow__", + accessor: SlotAccessor::NbPower, + op: Some(SlotOp::Right), + doc: "Return pow(value, self, mod).", + }, + SlotDef { + name: "__ipow__", + accessor: SlotAccessor::NbInplacePower, + op: None, + doc: "Implement self**=value.", + }, + SlotDef { + name: "__lshift__", + accessor: SlotAccessor::NbLshift, + op: Some(SlotOp::Left), + doc: "Return self<<value.", + }, + SlotDef { + name: "__rlshift__", + accessor: SlotAccessor::NbLshift, + op: Some(SlotOp::Right), + doc: "Return value<<self.", + }, + SlotDef { + name: "__ilshift__", + accessor: SlotAccessor::NbInplaceLshift, + op: None, + doc: "Implement self<<=value.", + }, + SlotDef { + name: "__rshift__", + accessor: SlotAccessor::NbRshift, + op: Some(SlotOp::Left), + doc: "Return self>>value.", + }, + SlotDef { + name: "__rrshift__", + accessor: SlotAccessor::NbRshift, + op: Some(SlotOp::Right), + doc: "Return value>>self.", + }, + SlotDef { + name: "__irshift__", + accessor: SlotAccessor::NbInplaceRshift, + op: None, + doc: "Implement self>>=value.", + }, + SlotDef { + name: "__and__", + accessor: SlotAccessor::NbAnd, + op: Some(SlotOp::Left), + doc: "Return self&value.", + }, + SlotDef { + name: "__rand__", + accessor: SlotAccessor::NbAnd, + op: Some(SlotOp::Right), + doc: "Return value&self.", + }, + SlotDef { + name: "__iand__", + accessor: SlotAccessor::NbInplaceAnd, + op: None, + doc: "Implement self&=value.", + }, + SlotDef { + name: "__xor__", + accessor: SlotAccessor::NbXor, + op: Some(SlotOp::Left), + doc: "Return self^value.", + }, + SlotDef { + name: "__rxor__", + accessor: SlotAccessor::NbXor, + op: Some(SlotOp::Right), + doc: "Return value^self.", + }, + SlotDef { + name: "__ixor__", + accessor: SlotAccessor::NbInplaceXor, + op: None, + doc: "Implement self^=value.", + }, + SlotDef { + name: "__or__", + accessor: SlotAccessor::NbOr, + op: Some(SlotOp::Left), + doc: "Return self|value.", + }, + SlotDef { + name: "__ror__", + accessor: SlotAccessor::NbOr, + op: Some(SlotOp::Right), + doc: "Return value|self.", + }, + SlotDef { + name: "__ior__", + accessor: SlotAccessor::NbInplaceOr, + op: None, + doc: "Implement self|=value.", + }, + SlotDef { + name: "__floordiv__", + accessor: SlotAccessor::NbFloorDivide, + op: Some(SlotOp::Left), + doc: "Return self//value.", + }, + SlotDef { + name: "__rfloordiv__", + accessor: SlotAccessor::NbFloorDivide, + op: Some(SlotOp::Right), + doc: "Return value//self.", + }, + SlotDef { + name: "__ifloordiv__", + accessor: SlotAccessor::NbInplaceFloorDivide, + op: None, + doc: "Implement self//=value.", + }, + SlotDef { + name: "__truediv__", + accessor: SlotAccessor::NbTrueDivide, + op: Some(SlotOp::Left), + doc: "Return self/value.", + }, + SlotDef { + name: "__rtruediv__", + accessor: SlotAccessor::NbTrueDivide, + op: Some(SlotOp::Right), + doc: "Return value/self.", + }, + SlotDef { + name: "__itruediv__", + accessor: SlotAccessor::NbInplaceTrueDivide, + op: None, + doc: "Implement self/=value.", + }, + SlotDef { + name: "__matmul__", + accessor: SlotAccessor::NbMatrixMultiply, + op: Some(SlotOp::Left), + doc: "Return self@value.", + }, + SlotDef { + name: "__rmatmul__", + accessor: SlotAccessor::NbMatrixMultiply, + op: Some(SlotOp::Right), + doc: "Return value@self.", + }, + SlotDef { + name: "__imatmul__", + accessor: SlotAccessor::NbInplaceMatrixMultiply, + op: None, + doc: "Implement self@=value.", + }, + // Number unary operations + SlotDef { + name: "__neg__", + accessor: SlotAccessor::NbNegative, + op: None, + doc: "Return -self.", + }, + SlotDef { + name: "__pos__", + accessor: SlotAccessor::NbPositive, + op: None, + doc: "Return +self.", + }, + SlotDef { + name: "__abs__", + accessor: SlotAccessor::NbAbsolute, + op: None, + doc: "Return abs(self).", + }, + SlotDef { + name: "__invert__", + accessor: SlotAccessor::NbInvert, + op: None, + doc: "Return ~self.", + }, + SlotDef { + name: "__bool__", + accessor: SlotAccessor::NbBool, + op: None, + doc: "Return self != 0.", + }, + SlotDef { + name: "__int__", + accessor: SlotAccessor::NbInt, + op: None, + doc: "Return int(self).", + }, + SlotDef { + name: "__float__", + accessor: SlotAccessor::NbFloat, + op: None, + doc: "Return float(self).", + }, + SlotDef { + name: "__index__", + accessor: SlotAccessor::NbIndex, + op: None, + doc: "Return self converted to an integer, if self is suitable for use as an index into a list.", + }, + // Sequence inplace operations (also map to number slots for some types) + SlotDef { + name: "__add__", + accessor: SlotAccessor::SqConcat, + op: None, + doc: "Return self+value.", + }, + SlotDef { + name: "__mul__", + accessor: SlotAccessor::SqRepeat, + op: None, + doc: "Return self*value.", + }, + SlotDef { + name: "__rmul__", + accessor: SlotAccessor::SqRepeat, + op: None, + doc: "Return value*self.", + }, + SlotDef { + name: "__iadd__", + accessor: SlotAccessor::SqInplaceConcat, + op: None, + doc: "Implement self+=value.", + }, + SlotDef { + name: "__imul__", + accessor: SlotAccessor::SqInplaceRepeat, + op: None, + doc: "Implement self*=value.", + }, +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_find_by_name() { + // __len__ appears in both sequence and mapping + let len_defs: Vec<_> = find_slot_defs_by_name("__len__").collect(); + assert_eq!(len_defs.len(), 2); + + // __init__ appears once + let init_defs: Vec<_> = find_slot_defs_by_name("__init__").collect(); + assert_eq!(init_defs.len(), 1); + + // __add__ appears in number (left/right) and sequence + let add_defs: Vec<_> = find_slot_defs_by_name("__add__").collect(); + assert_eq!(add_defs.len(), 2); // NbAdd(Left) and SqConcat + } + + #[test] + fn test_slot_op() { + // Test comparison ops + assert_eq!(SlotOp::Lt.as_compare_op(), Some(PyComparisonOp::Lt)); + assert_eq!(SlotOp::Eq.as_compare_op(), Some(PyComparisonOp::Eq)); + assert_eq!(SlotOp::Left.as_compare_op(), None); + + // Test right check + assert!(SlotOp::Right.is_right()); + assert!(!SlotOp::Left.is_right()); + } +} diff --git a/crates/vm/src/types/structseq.rs b/crates/vm/src/types/structseq.rs new file mode 100644 index 00000000000..02d1a4d6349 --- /dev/null +++ b/crates/vm/src/types/structseq.rs @@ -0,0 +1,454 @@ +use crate::common::lock::LazyLock; +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, + builtins::{PyBaseExceptionRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef}, + class::{PyClassImpl, StaticType}, + function::{Either, FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags}, + iter::PyExactSizeIterator, + protocol::{PyMappingMethods, PySequenceMethods}, + sliceable::{SequenceIndex, SliceableSequenceOp}, + types::PyComparisonOp, + vm::Context, +}; + +const DEFAULT_STRUCTSEQ_REDUCE: PyMethodDef = PyMethodDef::new_const( + "__reduce__", + |zelf: PyRef<PyTuple>, vm: &VirtualMachine| -> PyTupleRef { + vm.new_tuple((zelf.class().to_owned(), (vm.ctx.new_tuple(zelf.to_vec()),))) + }, + PyMethodFlags::METHOD, + None, +); + +/// Create a new struct sequence instance from a sequence. +/// +/// The class must have `n_sequence_fields` and `n_fields` attributes set +/// (done automatically by `PyStructSequence::extend_pyclass`). +pub fn struct_sequence_new(cls: PyTypeRef, seq: PyObjectRef, vm: &VirtualMachine) -> PyResult { + // = structseq_new + + #[cold] + fn length_error( + tp_name: &str, + min_len: usize, + max_len: usize, + len: usize, + vm: &VirtualMachine, + ) -> PyBaseExceptionRef { + if min_len == max_len { + vm.new_type_error(format!( + "{tp_name}() takes a {min_len}-sequence ({len}-sequence given)" + )) + } else if len < min_len { + vm.new_type_error(format!( + "{tp_name}() takes an at least {min_len}-sequence ({len}-sequence given)" + )) + } else { + vm.new_type_error(format!( + "{tp_name}() takes an at most {max_len}-sequence ({len}-sequence given)" + )) + } + } + + let min_len: usize = cls + .get_attr(identifier!(vm.ctx, n_sequence_fields)) + .ok_or_else(|| vm.new_type_error("missing n_sequence_fields attribute"))? + .try_into_value(vm)?; + let max_len: usize = cls + .get_attr(identifier!(vm.ctx, n_fields)) + .ok_or_else(|| vm.new_type_error("missing n_fields attribute"))? + .try_into_value(vm)?; + + let seq: Vec<PyObjectRef> = seq.try_into_value(vm)?; + let len = seq.len(); + + if len < min_len || len > max_len { + return Err(length_error(&cls.slot_name(), min_len, max_len, len, vm)); + } + + // Copy items and pad with None + let mut items = seq; + items.resize_with(max_len, || vm.ctx.none()); + + PyTuple::new_unchecked(items.into_boxed_slice()) + .into_ref_with_type(vm, cls) + .map(Into::into) +} + +fn get_visible_len(obj: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { + obj.class() + .get_attr(identifier!(vm.ctx, n_sequence_fields)) + .ok_or_else(|| vm.new_type_error("missing n_sequence_fields"))? + .try_into_value(vm) +} + +/// Sequence methods for struct sequences. +/// Uses n_sequence_fields to determine visible length. +static STRUCT_SEQUENCE_AS_SEQUENCE: LazyLock<PySequenceMethods> = + LazyLock::new(|| PySequenceMethods { + length: atomic_func!(|seq, vm| get_visible_len(seq.obj, vm)), + concat: atomic_func!(|seq, other, vm| { + // Convert to visible-only tuple, then use regular tuple concat + let n_seq = get_visible_len(seq.obj, vm)?; + let tuple = seq.obj.downcast_ref::<PyTuple>().unwrap(); + let visible: Vec<_> = tuple.iter().take(n_seq).cloned().collect(); + let visible_tuple = PyTuple::new_ref(visible, &vm.ctx); + // Use tuple's concat implementation + visible_tuple + .as_object() + .sequence_unchecked() + .concat(other, vm) + }), + repeat: atomic_func!(|seq, n, vm| { + // Convert to visible-only tuple, then use regular tuple repeat + let n_seq = get_visible_len(seq.obj, vm)?; + let tuple = seq.obj.downcast_ref::<PyTuple>().unwrap(); + let visible: Vec<_> = tuple.iter().take(n_seq).cloned().collect(); + let visible_tuple = PyTuple::new_ref(visible, &vm.ctx); + // Use tuple's repeat implementation + visible_tuple.as_object().sequence_unchecked().repeat(n, vm) + }), + item: atomic_func!(|seq, i, vm| { + let n_seq = get_visible_len(seq.obj, vm)?; + let tuple = seq.obj.downcast_ref::<PyTuple>().unwrap(); + let idx = if i < 0 { + let pos_i = n_seq as isize + i; + if pos_i < 0 { + return Err(vm.new_index_error("tuple index out of range")); + } + pos_i as usize + } else { + i as usize + }; + if idx >= n_seq { + return Err(vm.new_index_error("tuple index out of range")); + } + Ok(tuple[idx].clone()) + }), + contains: atomic_func!(|seq, needle, vm| { + let n_seq = get_visible_len(seq.obj, vm)?; + let tuple = seq.obj.downcast_ref::<PyTuple>().unwrap(); + for item in tuple.iter().take(n_seq) { + if item.rich_compare_bool(needle, PyComparisonOp::Eq, vm)? { + return Ok(true); + } + } + Ok(false) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + +/// Mapping methods for struct sequences. +/// Handles subscript (indexing) with visible length bounds. +static STRUCT_SEQUENCE_AS_MAPPING: LazyLock<PyMappingMethods> = + LazyLock::new(|| PyMappingMethods { + length: atomic_func!(|mapping, vm| get_visible_len(mapping.obj, vm)), + subscript: atomic_func!(|mapping, needle, vm| { + let n_seq = get_visible_len(mapping.obj, vm)?; + let tuple = mapping.obj.downcast_ref::<PyTuple>().unwrap(); + let visible_elements = &tuple.as_slice()[..n_seq]; + + match SequenceIndex::try_from_borrowed_object(vm, needle, "tuple")? { + SequenceIndex::Int(i) => visible_elements.getitem_by_index(vm, i), + SequenceIndex::Slice(slice) => visible_elements + .getitem_by_slice(vm, slice) + .map(|x| vm.ctx.new_tuple(x).into()), + } + }), + ..PyMappingMethods::NOT_IMPLEMENTED + }); + +/// Trait for Data structs that back a PyStructSequence. +/// +/// This trait is implemented by `#[pystruct_sequence_data]` on the Data struct. +/// It provides field information, tuple conversion, and element parsing. +pub trait PyStructSequenceData: Sized { + /// Names of required fields (in order). Shown in repr. + const REQUIRED_FIELD_NAMES: &'static [&'static str]; + + /// Names of optional/skipped fields (in order, after required fields). + const OPTIONAL_FIELD_NAMES: &'static [&'static str]; + + /// Number of unnamed fields (visible but index-only access). + const UNNAMED_FIELDS_LEN: usize = 0; + + /// Convert this Data struct into a PyTuple. + fn into_tuple(self, vm: &VirtualMachine) -> PyTuple; + + /// Construct this Data struct from tuple elements. + /// Default implementation returns an error. + /// Override with `#[pystruct_sequence_data(try_from_object)]` to enable. + fn try_from_elements(_elements: Vec<PyObjectRef>, vm: &VirtualMachine) -> PyResult<Self> { + Err(vm.new_type_error("This struct sequence does not support construction from elements")) + } +} + +/// Trait for Python struct sequence types. +/// +/// This trait is implemented by the `#[pystruct_sequence]` macro on the Python type struct. +/// It connects to the Data struct and provides Python-level functionality. +#[pyclass] +pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { + /// The Data struct that provides field definitions. + type Data: PyStructSequenceData; + + /// Convert a Data struct into a PyStructSequence instance. + fn from_data(data: Self::Data, vm: &VirtualMachine) -> PyTupleRef { + let tuple = + <Self::Data as ::rustpython_vm::types::PyStructSequenceData>::into_tuple(data, vm); + let typ = Self::static_type(); + tuple + .into_ref_with_type(vm, typ.to_owned()) + .expect("Every PyStructSequence must be a valid tuple. This is a RustPython bug.") + } + + #[pyslot] + fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { + let zelf = zelf + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("unexpected payload for __repr__"))?; + + let field_names = Self::Data::REQUIRED_FIELD_NAMES; + let format_field = |(value, name): (&PyObject, _)| { + let s = value.repr(vm)?; + Ok(format!("{name}={s}")) + }; + let (body, suffix) = + if let Some(_guard) = rustpython_vm::recursion::ReprGuard::enter(vm, zelf.as_ref()) { + let fields: PyResult<Vec<_>> = zelf + .iter() + .map(|value| value.as_ref()) + .zip(field_names.iter().copied()) + .map(format_field) + .collect(); + (fields?.join(", "), "") + } else { + (String::new(), "...") + }; + // Build qualified name: if MODULE_NAME is already in TP_NAME, use it directly. + // Otherwise, check __module__ attribute (set by #[pymodule] at runtime). + let type_name = if Self::MODULE_NAME.is_some() { + alloc::borrow::Cow::Borrowed(Self::TP_NAME) + } else { + let typ = zelf.class(); + match typ.get_attr(identifier!(vm.ctx, __module__)) { + Some(module) if module.downcastable::<PyStr>() => { + let module_str = module.downcast_ref::<PyStr>().unwrap(); + alloc::borrow::Cow::Owned(format!("{}.{}", module_str.as_wtf8(), Self::NAME)) + } + _ => alloc::borrow::Cow::Borrowed(Self::TP_NAME), + } + }; + let repr_str = format!("{}({}{})", type_name, body, suffix); + Ok(vm.ctx.new_str(repr_str)) + } + + #[pymethod] + fn __replace__(zelf: PyRef<PyTuple>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.args.is_empty() { + return Err(vm.new_type_error("__replace__() takes no positional arguments")); + } + + if Self::Data::UNNAMED_FIELDS_LEN > 0 { + return Err(vm.new_type_error(format!( + "__replace__() is not supported for {} because it has unnamed field(s)", + zelf.class().slot_name() + ))); + } + + let n_fields = + Self::Data::REQUIRED_FIELD_NAMES.len() + Self::Data::OPTIONAL_FIELD_NAMES.len(); + let mut items: Vec<PyObjectRef> = zelf.as_slice()[..n_fields].to_vec(); + + let mut kwargs = args.kwargs.clone(); + + // Replace fields from kwargs + let all_field_names: Vec<&str> = Self::Data::REQUIRED_FIELD_NAMES + .iter() + .chain(Self::Data::OPTIONAL_FIELD_NAMES.iter()) + .copied() + .collect(); + for (i, &name) in all_field_names.iter().enumerate() { + if let Some(val) = kwargs.shift_remove(name) { + items[i] = val; + } + } + + // Check for unexpected keyword arguments + if !kwargs.is_empty() { + let names: Vec<&str> = kwargs.keys().map(|k| k.as_str()).collect(); + return Err(vm.new_type_error(format!("Got unexpected field name(s): {:?}", names))); + } + + PyTuple::new_unchecked(items.into_boxed_slice()) + .into_ref_with_type(vm, zelf.class().to_owned()) + .map(Into::into) + } + + #[pymethod] + fn __getitem__(zelf: PyRef<PyTuple>, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let n_seq = get_visible_len(zelf.as_ref(), vm)?; + let visible_elements = &zelf.as_slice()[..n_seq]; + + match SequenceIndex::try_from_borrowed_object(vm, &needle, "tuple")? { + SequenceIndex::Int(i) => visible_elements.getitem_by_index(vm, i), + SequenceIndex::Slice(slice) => visible_elements + .getitem_by_slice(vm, slice) + .map(|x| vm.ctx.new_tuple(x).into()), + } + } + + #[extend_class] + fn extend_pyclass(ctx: &Context, class: &'static Py<PyType>) { + // Getters for named visible fields (indices 0 to REQUIRED_FIELD_NAMES.len() - 1) + for (i, &name) in Self::Data::REQUIRED_FIELD_NAMES.iter().enumerate() { + // cast i to a u8 so there's less to store in the getter closure. + // Hopefully there's not struct sequences with >=256 elements :P + let i = i as u8; + class.set_attr( + ctx.intern_str(name), + ctx.new_readonly_getset(name, class, move |zelf: &PyTuple| { + zelf[i as usize].to_owned() + }) + .into(), + ); + } + + // Getters for hidden/skipped fields (indices after visible fields) + let visible_count = Self::Data::REQUIRED_FIELD_NAMES.len() + Self::Data::UNNAMED_FIELDS_LEN; + for (i, &name) in Self::Data::OPTIONAL_FIELD_NAMES.iter().enumerate() { + let idx = (visible_count + i) as u8; + class.set_attr( + ctx.intern_str(name), + ctx.new_readonly_getset(name, class, move |zelf: &PyTuple| { + zelf[idx as usize].to_owned() + }) + .into(), + ); + } + + class.set_attr( + identifier!(ctx, __match_args__), + ctx.new_tuple( + Self::Data::REQUIRED_FIELD_NAMES + .iter() + .map(|&name| ctx.new_str(name).into()) + .collect::<Vec<_>>(), + ) + .into(), + ); + + // special fields: + // n_sequence_fields = visible fields (named + unnamed) + // n_fields = all fields (visible + hidden/skipped) + // n_unnamed_fields + let n_unnamed_fields = Self::Data::UNNAMED_FIELDS_LEN; + let n_sequence_fields = Self::Data::REQUIRED_FIELD_NAMES.len() + n_unnamed_fields; + let n_fields = n_sequence_fields + Self::Data::OPTIONAL_FIELD_NAMES.len(); + class.set_attr( + identifier!(ctx, n_sequence_fields), + ctx.new_int(n_sequence_fields).into(), + ); + class.set_attr(identifier!(ctx, n_fields), ctx.new_int(n_fields).into()); + class.set_attr( + identifier!(ctx, n_unnamed_fields), + ctx.new_int(n_unnamed_fields).into(), + ); + + // Override as_sequence and as_mapping slots to use visible length + class + .slots + .as_sequence + .copy_from(&STRUCT_SEQUENCE_AS_SEQUENCE); + class + .slots + .as_mapping + .copy_from(&STRUCT_SEQUENCE_AS_MAPPING); + + // Override iter slot to return only visible elements + class.slots.iter.store(Some(struct_sequence_iter)); + + // Override hash slot to hash only visible elements + class.slots.hash.store(Some(struct_sequence_hash)); + + // Override richcompare slot to compare only visible elements + class + .slots + .richcompare + .store(Some(struct_sequence_richcompare)); + + // Default __reduce__: only set if not already overridden by the impl's extend_class. + // This allows struct sequences like sched_param to provide a custom __reduce__ + // (equivalent to METH_COEXIST in structseq.c). + if !class + .attributes + .read() + .contains_key(ctx.intern_str("__reduce__")) + { + class.set_attr( + ctx.intern_str("__reduce__"), + DEFAULT_STRUCTSEQ_REDUCE.to_proper_method(class, ctx), + ); + } + } +} + +/// Iterator function for struct sequences - returns only visible elements +fn struct_sequence_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let tuple = zelf + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("expected tuple"))?; + let n_seq = get_visible_len(&zelf, vm)?; + let visible: Vec<_> = tuple.iter().take(n_seq).cloned().collect(); + let visible_tuple = PyTuple::new_ref(visible, &vm.ctx); + visible_tuple + .as_object() + .to_owned() + .get_iter(vm) + .map(Into::into) +} + +/// Hash function for struct sequences - hashes only visible elements +fn struct_sequence_hash( + zelf: &PyObject, + vm: &VirtualMachine, +) -> PyResult<crate::common::hash::PyHash> { + let tuple = zelf + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("expected tuple"))?; + let n_seq = get_visible_len(zelf, vm)?; + // Create a visible-only tuple and hash it + let visible: Vec<_> = tuple.iter().take(n_seq).cloned().collect(); + let visible_tuple = PyTuple::new_ref(visible, &vm.ctx); + visible_tuple.as_object().hash(vm) +} + +/// Rich comparison for struct sequences - compares only visible elements +fn struct_sequence_richcompare( + zelf: &PyObject, + other: &PyObject, + op: PyComparisonOp, + vm: &VirtualMachine, +) -> PyResult<Either<PyObjectRef, PyComparisonValue>> { + let zelf_tuple = zelf + .downcast_ref::<PyTuple>() + .ok_or_else(|| vm.new_type_error("expected tuple"))?; + + // If other is not a tuple, return NotImplemented + let Some(other_tuple) = other.downcast_ref::<PyTuple>() else { + return Ok(Either::B(PyComparisonValue::NotImplemented)); + }; + + let zelf_len = get_visible_len(zelf, vm)?; + // For other, try to get visible len; if it fails (not a struct sequence), use full length + let other_len = get_visible_len(other, vm).unwrap_or(other_tuple.len()); + + let zelf_visible = &zelf_tuple.as_slice()[..zelf_len]; + let other_visible = &other_tuple.as_slice()[..other_len]; + + // Use the same comparison logic as regular tuples + zelf_visible + .iter() + .richcompare(other_visible.iter(), op, vm) + .map(|v| Either::B(PyComparisonValue::Implemented(v))) +} diff --git a/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs similarity index 81% rename from vm/src/types/zoo.rs rename to crates/vm/src/types/zoo.rs index 492b584e291..0394f672cf8 100644 --- a/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -1,16 +1,16 @@ use crate::{ + Py, builtins::{ - asyncgenerator, bool_, builtin_func, bytearray, bytes, classmethod, code, complex, + asyncgenerator, bool_, builtin_func, bytearray, bytes, capsule, classmethod, code, complex, coroutine, descriptor, dict, enumerate, filter, float, frame, function, generator, - genericalias, getset, int, iter, list, map, mappingproxy, memory, module, namespace, - object, property, pystr, range, set, singletons, slice, staticmethod, super_, traceback, - tuple, + genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, module, + namespace, object, property, pystr, range, set, singletons, slice, staticmethod, super_, + template, traceback, tuple, type_::{self, PyType}, union_, weakproxy, weakref, zip, }, class::StaticType, vm::Context, - Py, }; /// Holder of references to builtin types. @@ -21,12 +21,14 @@ pub struct TypeZoo { pub async_generator_asend: &'static Py<PyType>, pub async_generator_athrow: &'static Py<PyType>, pub async_generator_wrapped_value: &'static Py<PyType>, + pub anext_awaitable: &'static Py<PyType>, pub bytes_type: &'static Py<PyType>, pub bytes_iterator_type: &'static Py<PyType>, pub bytearray_type: &'static Py<PyType>, pub bytearray_iterator_type: &'static Py<PyType>, pub bool_type: &'static Py<PyType>, pub callable_iterator: &'static Py<PyType>, + pub capsule_type: &'static Py<PyType>, pub cell_type: &'static Py<PyType>, pub classmethod_type: &'static Py<PyType>, pub code_type: &'static Py<PyType>, @@ -88,10 +90,17 @@ pub struct TypeZoo { pub object_type: &'static Py<PyType>, pub ellipsis_type: &'static Py<PyType>, pub none_type: &'static Py<PyType>, + pub typing_no_default_type: &'static Py<PyType>, pub not_implemented_type: &'static Py<PyType>, pub generic_alias_type: &'static Py<PyType>, + pub generic_alias_iterator_type: &'static Py<PyType>, pub union_type: &'static Py<PyType>, + pub interpolation_type: &'static Py<PyType>, + pub template_type: &'static Py<PyType>, + pub template_iter_type: &'static Py<PyType>, pub member_descriptor_type: &'static Py<PyType>, + pub wrapper_descriptor_type: &'static Py<PyType>, + pub method_wrapper_type: &'static Py<PyType>, // RustPython-original types pub method_def: &'static Py<PyType>, @@ -101,12 +110,20 @@ impl TypeZoo { #[cold] pub(crate) fn init() -> Self { let (type_type, object_type, weakref_type) = crate::object::init_type_hierarchy(); + // the order matters for type, object, weakref, and int - must be initialized first + let type_type = type_::PyType::init_manually(type_type); + let object_type = object::PyBaseObject::init_manually(object_type); + let weakref_type = weakref::PyWeak::init_manually(weakref_type); + let int_type = int::PyInt::init_builtin_type(); + + // builtin_function_or_method and builtin_method share the same type (CPython behavior) + let builtin_function_or_method_type = builtin_func::PyNativeFunction::init_builtin_type(); + Self { - // the order matters for type, object, weakref, and int - type_type: type_::PyType::init_manually(type_type), - object_type: object::PyBaseObject::init_manually(object_type), - weakref_type: weakref::PyWeak::init_manually(weakref_type), - int_type: int::PyInt::init_builtin_type(), + type_type, + object_type, + weakref_type, + int_type, // types exposed as builtins bool_type: bool_::PyBool::init_builtin_type(), @@ -138,12 +155,14 @@ impl TypeZoo { async_generator_athrow: asyncgenerator::PyAsyncGenAThrow::init_builtin_type(), async_generator_wrapped_value: asyncgenerator::PyAsyncGenWrappedValue::init_builtin_type(), + anext_awaitable: asyncgenerator::PyAnextAwaitable::init_builtin_type(), bound_method_type: function::PyBoundMethod::init_builtin_type(), - builtin_function_or_method_type: builtin_func::PyNativeFunction::init_builtin_type(), - builtin_method_type: builtin_func::PyNativeMethod::init_builtin_type(), + builtin_function_or_method_type, + builtin_method_type: builtin_function_or_method_type, bytearray_iterator_type: bytearray::PyByteArrayIterator::init_builtin_type(), bytes_iterator_type: bytes::PyBytesIterator::init_builtin_type(), callable_iterator: iter::PyCallableIterator::init_builtin_type(), + capsule_type: capsule::PyCapsule::init_builtin_type(), cell_type: function::PyCell::init_builtin_type(), code_type: code::PyCode::init_builtin_type(), coroutine_type: coroutine::PyCoroutine::init_builtin_type(), @@ -179,10 +198,17 @@ impl TypeZoo { weakproxy_type: weakproxy::PyWeakProxy::init_builtin_type(), method_descriptor_type: descriptor::PyMethodDescriptor::init_builtin_type(), none_type: singletons::PyNone::init_builtin_type(), + typing_no_default_type: crate::stdlib::_typing::NoDefault::init_builtin_type(), not_implemented_type: singletons::PyNotImplemented::init_builtin_type(), generic_alias_type: genericalias::PyGenericAlias::init_builtin_type(), + generic_alias_iterator_type: genericalias::PyGenericAliasIterator::init_builtin_type(), union_type: union_::PyUnion::init_builtin_type(), + interpolation_type: interpolation::PyInterpolation::init_builtin_type(), + template_type: template::PyTemplate::init_builtin_type(), + template_iter_type: template::PyTemplateIter::init_builtin_type(), member_descriptor_type: descriptor::PyMemberDescriptor::init_builtin_type(), + wrapper_descriptor_type: descriptor::PyWrapper::init_builtin_type(), + method_wrapper_type: descriptor::PyMethodWrapper::init_builtin_type(), method_def: crate::function::HeapMethodDef::init_builtin_type(), } @@ -190,9 +216,11 @@ impl TypeZoo { /// Fill attributes of builtin types. #[cold] - pub(crate) fn extend(context: &Context) { - type_::init(context); + pub(crate) fn extend(context: &'static Context) { + // object must be initialized before type to set object.slots.init, + // which type will inherit via inherit_slots() object::init(context); + type_::init(context); list::init(context); set::init(context); tuple::init(context); @@ -209,6 +237,7 @@ impl TypeZoo { complex::init(context); bytes::init(context); bytearray::init(context); + capsule::init(context); property::init(context); getset::init(context); memory::init(context); @@ -233,6 +262,9 @@ impl TypeZoo { traceback::init(context); genericalias::init(context); union_::init(context); + interpolation::init(context); + template::init(context); descriptor::init(context); + crate::stdlib::_typing::init(context); } } diff --git a/crates/vm/src/utils.rs b/crates/vm/src/utils.rs new file mode 100644 index 00000000000..b5117ddd8d1 --- /dev/null +++ b/crates/vm/src/utils.rs @@ -0,0 +1,74 @@ +use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; + +use crate::{ + PyObjectRef, PyResult, VirtualMachine, + builtins::{PyStr, PyUtf8Str}, + convert::{ToPyException, ToPyObject}, + exceptions::cstring_error, +}; + +pub fn hash_iter<'a, I: IntoIterator<Item = &'a PyObjectRef>>( + iter: I, + vm: &VirtualMachine, +) -> PyResult<rustpython_common::hash::PyHash> { + vm.state.hash_secret.hash_iter(iter, |obj| obj.hash(vm)) +} + +impl ToPyObject for core::convert::Infallible { + fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { + match self {} + } +} + +pub trait ToCString: AsRef<Wtf8> { + fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<alloc::ffi::CString> { + alloc::ffi::CString::new(self.as_ref().as_bytes()).map_err(|err| err.to_pyexception(vm)) + } + fn ensure_no_nul(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.as_ref().as_bytes().contains(&b'\0') { + Err(cstring_error(vm)) + } else { + Ok(()) + } + } +} + +impl ToCString for &str {} +impl ToCString for PyStr {} +impl ToCString for PyUtf8Str {} + +pub(crate) fn collection_repr<'a, I>( + class_name: Option<&str>, + prefix: &str, + suffix: &str, + iter: I, + vm: &VirtualMachine, +) -> PyResult<Wtf8Buf> +where + I: core::iter::Iterator<Item = &'a PyObjectRef>, +{ + let mut repr = Wtf8Buf::new(); + if let Some(name) = class_name { + repr.push_str(name); + repr.push_char('('); + } + repr.push_str(prefix); + { + let mut parts_iter = iter.map(|o| o.repr(vm)); + let first = parts_iter + .next() + .transpose()? + .expect("this is not called for empty collection"); + repr.push_wtf8(first.as_wtf8()); + for part in parts_iter { + repr.push_str(", "); + repr.push_wtf8(part?.as_wtf8()); + } + } + repr.push_str(suffix); + if class_name.is_some() { + repr.push_char(')'); + } + + Ok(repr) +} diff --git a/crates/vm/src/version.rs b/crates/vm/src/version.rs new file mode 100644 index 00000000000..a75a6f47de6 --- /dev/null +++ b/crates/vm/src/version.rs @@ -0,0 +1,140 @@ +//! Several function to retrieve version information. + +use chrono::{Local, prelude::DateTime}; +use core::time::Duration; +use std::time::UNIX_EPOCH; + +// = 3.14.0alpha +pub const MAJOR: usize = 3; +pub const MINOR: usize = 14; +pub const MICRO: usize = 0; +pub const RELEASELEVEL: &str = "alpha"; +pub const RELEASELEVEL_N: usize = 0xA; +pub const SERIAL: usize = 0; + +pub const VERSION_HEX: usize = + (MAJOR << 24) | (MINOR << 16) | (MICRO << 8) | (RELEASELEVEL_N << 4) | SERIAL; + +pub fn get_version() -> String { + // Windows: include MSC v. for compatibility with ctypes.util.find_library + // MSC v.1929 = VS 2019, version 14+ makes find_msvcrt() return None + #[cfg(windows)] + let msc_info = { + let arch = if cfg!(target_pointer_width = "64") { + "64 bit (AMD64)" + } else { + "32 bit (Intel)" + }; + // Include both RustPython identifier and MSC v. for compatibility + format!(" MSC v.1929 {arch}",) + }; + + #[cfg(not(windows))] + let msc_info = String::new(); + + format!( + "{:.80} ({:.80}) \n[RustPython {} with {:.80}{}]", // \n is PyPy convention + get_version_number(), + get_build_info(), + env!("CARGO_PKG_VERSION"), + COMPILER, + msc_info, + ) +} + +pub fn get_version_number() -> String { + format!("{MAJOR}.{MINOR}.{MICRO}{RELEASELEVEL}") +} + +pub fn get_winver_number() -> String { + format!("{MAJOR}.{MINOR}") +} + +const COMPILER: &str = env!("RUSTC_VERSION"); + +pub fn get_build_info() -> String { + // See: https://reproducible-builds.org/docs/timestamps/ + let git_revision = get_git_revision(); + let separator = if git_revision.is_empty() { "" } else { ":" }; + + let git_identifier = get_git_identifier(); + + format!( + "{id}{sep}{revision}, {date:.20}, {time:.9}", + id = if git_identifier.is_empty() { + "default".to_owned() + } else { + git_identifier + }, + sep = separator, + revision = git_revision, + date = get_git_date(), + time = get_git_time(), + ) +} + +pub fn get_git_revision() -> String { + option_env!("RUSTPYTHON_GIT_HASH").unwrap_or("").to_owned() +} + +pub fn get_git_tag() -> String { + option_env!("RUSTPYTHON_GIT_TAG").unwrap_or("").to_owned() +} + +pub fn get_git_branch() -> String { + option_env!("RUSTPYTHON_GIT_BRANCH") + .unwrap_or("") + .to_owned() +} + +pub fn get_git_identifier() -> String { + let git_tag = get_git_tag(); + let git_branch = get_git_branch(); + + if git_tag.is_empty() || git_tag == "undefined" { + git_branch + } else { + git_tag + } +} + +fn get_git_timestamp_datetime() -> DateTime<Local> { + let timestamp = option_env!("RUSTPYTHON_GIT_TIMESTAMP") + .unwrap_or("") + .to_owned(); + let timestamp = timestamp.parse::<u64>().unwrap_or(0); + + let datetime = UNIX_EPOCH + Duration::from_secs(timestamp); + + datetime.into() +} + +pub fn get_git_date() -> String { + let datetime = get_git_timestamp_datetime(); + + datetime.format("%b %e %Y").to_string() +} + +pub fn get_git_time() -> String { + let datetime = get_git_timestamp_datetime(); + + datetime.format("%H:%M:%S").to_string() +} + +pub fn get_git_datetime() -> String { + let date = get_git_date(); + let time = get_git_time(); + + format!("{date} {time}") +} + +// Must be aligned to Lib/importlib/_bootstrap_external.py +pub const PYC_MAGIC_NUMBER: u16 = 2996; + +// CPython format: magic_number | ('\r' << 16) | ('\n' << 24) +// This protects against text-mode file reads +pub const PYC_MAGIC_NUMBER_TOKEN: u32 = + (PYC_MAGIC_NUMBER as u32) | ((b'\r' as u32) << 16) | ((b'\n' as u32) << 24); + +/// Magic number as little-endian bytes for .pyc files +pub const PYC_MAGIC_NUMBER_BYTES: [u8; 4] = PYC_MAGIC_NUMBER_TOKEN.to_le_bytes(); diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs new file mode 100644 index 00000000000..7294dc8f897 --- /dev/null +++ b/crates/vm/src/vm/compile.rs @@ -0,0 +1,361 @@ +//! Python code compilation functions. +//! +//! For code execution functions, see python_run.rs + +use crate::{ + PyRef, VirtualMachine, + builtins::PyCode, + compiler::{self, CompileError, CompileOpts}, +}; + +impl VirtualMachine { + pub fn compile( + &self, + source: &str, + mode: compiler::Mode, + source_path: String, + ) -> Result<PyRef<PyCode>, CompileError> { + self.compile_with_opts(source, mode, source_path, self.compile_opts()) + } + + pub fn compile_with_opts( + &self, + source: &str, + mode: compiler::Mode, + source_path: String, + opts: CompileOpts, + ) -> Result<PyRef<PyCode>, CompileError> { + let code = + compiler::compile(source, mode, &source_path, opts).map(|code| self.ctx.new_code(code)); + #[cfg(feature = "parser")] + if code.is_ok() { + self.emit_string_escape_warnings(source, &source_path); + } + code + } +} + +/// Scan source for invalid escape sequences in all string literals and emit +/// SyntaxWarning. +/// +/// Corresponds to: +/// - `warn_invalid_escape_sequence()` in `Parser/string_parser.c` +/// - `_PyTokenizer_warn_invalid_escape_sequence()` in `Parser/tokenizer/helpers.c` +#[cfg(feature = "parser")] +mod escape_warnings { + use super::*; + use crate::warn; + use ruff_python_ast::{self as ast, visitor::Visitor}; + use ruff_text_size::TextRange; + + /// Calculate 1-indexed line number at byte offset in source. + fn line_number_at(source: &str, offset: usize) -> usize { + source[..offset.min(source.len())] + .bytes() + .filter(|&b| b == b'\n') + .count() + + 1 + } + + /// Get content bounds (start, end byte offsets) of a quoted string literal, + /// excluding prefix characters and quote delimiters. + fn content_bounds(source: &str, range: TextRange) -> Option<(usize, usize)> { + let s = range.start().to_usize(); + let e = range.end().to_usize(); + if s >= e || e > source.len() { + return None; + } + let bytes = &source.as_bytes()[s..e]; + // Skip prefix (u, b, r, etc.) to find the first quote character. + let qi = bytes.iter().position(|&c| c == b'\'' || c == b'"')?; + let qc = bytes[qi]; + let ql = if bytes.get(qi + 1) == Some(&qc) && bytes.get(qi + 2) == Some(&qc) { + 3 + } else { + 1 + }; + let cs = s + qi + ql; + let ce = e.checked_sub(ql)?; + if cs <= ce { Some((cs, ce)) } else { None } + } + + /// Scan `source[start..end]` for the first invalid escape sequence. + /// Returns `Some((invalid_char, byte_offset_in_source))` for the first + /// invalid escape found, or `None` if all escapes are valid. + /// + /// When `is_bytes` is true, `\u`, `\U`, and `\N` are treated as invalid + /// (bytes literals only support byte-oriented escapes). + /// + /// Only reports the **first** invalid escape per string literal, matching + /// `_PyUnicode_DecodeUnicodeEscapeInternal2` which stores only the first + /// `first_invalid_escape_char`. + fn first_invalid_escape( + source: &str, + start: usize, + end: usize, + is_bytes: bool, + ) -> Option<(char, usize)> { + let raw = &source[start..end]; + let mut chars = raw.char_indices().peekable(); + while let Some((i, ch)) = chars.next() { + if ch != '\\' { + continue; + } + let Some((_, next)) = chars.next() else { + break; + }; + let valid = match next { + '\\' | '\'' | '"' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' => true, + '\n' => true, + '\r' => { + if matches!(chars.peek(), Some(&(_, '\n'))) { + chars.next(); + } + true + } + '0'..='7' => { + for _ in 0..2 { + if matches!(chars.peek(), Some(&(_, '0'..='7'))) { + chars.next(); + } else { + break; + } + } + true + } + 'x' | 'u' | 'U' => { + // \u and \U are only valid in string literals, not bytes + if is_bytes && next != 'x' { + false + } else { + let count = match next { + 'x' => 2, + 'u' => 4, + 'U' => 8, + _ => unreachable!(), + }; + for _ in 0..count { + if chars.peek().is_some_and(|&(_, c)| c.is_ascii_hexdigit()) { + chars.next(); + } else { + break; + } + } + true + } + } + 'N' => { + // \N{name} is only valid in string literals, not bytes + if is_bytes { + false + } else { + if matches!(chars.peek(), Some(&(_, '{'))) { + chars.next(); + for (_, c) in chars.by_ref() { + if c == '}' { + break; + } + } + } + true + } + } + _ => false, + }; + if !valid { + return Some((next, start + i)); + } + } + None + } + + /// Emit `SyntaxWarning` for an invalid escape sequence. + /// + /// `warn_invalid_escape_sequence()` in `Parser/string_parser.c` + fn warn_invalid_escape_sequence( + source: &str, + ch: char, + offset: usize, + filename: &str, + vm: &VirtualMachine, + ) { + let lineno = line_number_at(source, offset); + let message = vm.ctx.new_str(format!( + "\"\\{ch}\" is an invalid escape sequence. \ + Such sequences will not work in the future. \ + Did you mean \"\\\\{ch}\"? A raw string is also an option." + )); + let fname = vm.ctx.new_str(filename); + let _ = warn::warn_explicit( + Some(vm.ctx.exceptions.syntax_warning.to_owned()), + message.into(), + fname, + lineno, + None, + vm.ctx.none(), + None, + None, + vm, + ); + } + + struct EscapeWarningVisitor<'a> { + source: &'a str, + filename: &'a str, + vm: &'a VirtualMachine, + } + + impl<'a> EscapeWarningVisitor<'a> { + /// Check a quoted string/bytes literal for invalid escapes. + /// The range must include the prefix and quote delimiters. + fn check_quoted_literal(&self, range: TextRange, is_bytes: bool) { + if let Some((start, end)) = content_bounds(self.source, range) + && let Some((ch, offset)) = first_invalid_escape(self.source, start, end, is_bytes) + { + warn_invalid_escape_sequence(self.source, ch, offset, self.filename, self.vm); + } + } + + /// Check an f-string literal element for invalid escapes. + /// The range covers content only (no prefix/quotes). + /// + /// Also handles `\{` / `\}` at the literal–interpolation boundary, + /// equivalent to `_PyTokenizer_warn_invalid_escape_sequence` handling + /// `FSTRING_MIDDLE` / `FSTRING_END` tokens. + fn check_fstring_literal(&self, range: TextRange) { + let start = range.start().to_usize(); + let end = range.end().to_usize(); + if start >= end || end > self.source.len() { + return; + } + if let Some((ch, offset)) = first_invalid_escape(self.source, start, end, false) { + warn_invalid_escape_sequence(self.source, ch, offset, self.filename, self.vm); + return; + } + // In CPython, _PyTokenizer_warn_invalid_escape_sequence handles + // `\{` and `\}` for FSTRING_MIDDLE/FSTRING_END tokens. Ruff + // splits the literal element before the interpolation delimiter, + // so the `\` sits at the end of the literal range and the `{`/`}` + // sits just after it. Only warn when the number of trailing + // backslashes is odd (an even count means they are all escaped). + let trailing_bs = self.source.as_bytes()[start..end] + .iter() + .rev() + .take_while(|&&b| b == b'\\') + .count(); + if trailing_bs % 2 == 1 + && let Some(&after) = self.source.as_bytes().get(end) + && (after == b'{' || after == b'}') + { + warn_invalid_escape_sequence( + self.source, + after as char, + end - 1, + self.filename, + self.vm, + ); + } + } + + /// Visit f-string elements, checking literals and recursing into + /// interpolation expressions and format specs. + fn visit_fstring_elements(&mut self, elements: &'a ast::InterpolatedStringElements) { + for element in elements { + match element { + ast::InterpolatedStringElement::Literal(lit) => { + self.check_fstring_literal(lit.range); + } + ast::InterpolatedStringElement::Interpolation(interp) => { + self.visit_expr(&interp.expression); + if let Some(spec) = &interp.format_spec { + self.visit_fstring_elements(&spec.elements); + } + } + } + } + } + } + + impl<'a> Visitor<'a> for EscapeWarningVisitor<'a> { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + match expr { + // Regular string literals — decode_unicode_with_escapes path + ast::Expr::StringLiteral(string) => { + for part in string.value.as_slice() { + if !matches!( + part.flags.prefix(), + ast::str_prefix::StringLiteralPrefix::Raw { .. } + ) { + self.check_quoted_literal(part.range, false); + } + } + } + // Byte string literals — decode_bytes_with_escapes path + ast::Expr::BytesLiteral(bytes) => { + for part in bytes.value.as_slice() { + if !matches!( + part.flags.prefix(), + ast::str_prefix::ByteStringPrefix::Raw { .. } + ) { + self.check_quoted_literal(part.range, true); + } + } + } + // F-string literals — tokenizer + string_parser paths + ast::Expr::FString(fstring_expr) => { + for part in fstring_expr.value.as_slice() { + match part { + ast::FStringPart::Literal(string_lit) => { + // Plain string part in f-string concatenation + if !matches!( + string_lit.flags.prefix(), + ast::str_prefix::StringLiteralPrefix::Raw { .. } + ) { + self.check_quoted_literal(string_lit.range, false); + } + } + ast::FStringPart::FString(fstring) => { + if matches!( + fstring.flags.prefix(), + ast::str_prefix::FStringPrefix::Raw { .. } + ) { + continue; + } + self.visit_fstring_elements(&fstring.elements); + } + } + } + } + _ => ast::visitor::walk_expr(self, expr), + } + } + } + + impl VirtualMachine { + /// Walk all string literals in `source` and emit `SyntaxWarning` for + /// each that contains an invalid escape sequence. + pub(super) fn emit_string_escape_warnings(&self, source: &str, filename: &str) { + let Ok(parsed) = + ruff_python_parser::parse(source, ruff_python_parser::Mode::Module.into()) + else { + return; + }; + let ast = parsed.into_syntax(); + let mut visitor = EscapeWarningVisitor { + source, + filename, + vm: self, + }; + match ast { + ast::Mod::Module(module) => { + for stmt in &module.body { + visitor.visit_stmt(stmt); + } + } + ast::Mod::Expression(expr) => { + visitor.visit_expr(&expr.body); + } + } + } + } +} diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs new file mode 100644 index 00000000000..dc0af9386fe --- /dev/null +++ b/crates/vm/src/vm/context.rs @@ -0,0 +1,760 @@ +use crate::{ + PyResult, VirtualMachine, + builtins::{ + PyByteArray, PyBytes, PyComplex, PyDict, PyDictRef, PyEllipsis, PyFloat, PyFrozenSet, + PyInt, PyIntRef, PyList, PyListRef, PyNone, PyNotImplemented, PyStr, PyStrInterned, + PyTuple, PyTupleRef, PyType, PyTypeRef, PyUtf8Str, + bool_::PyBool, + code::{self, PyCode}, + descriptor::{ + MemberGetter, MemberKind, MemberSetter, MemberSetterFunc, PyDescriptorOwned, + PyMemberDef, PyMemberDescriptor, + }, + getset::PyGetSet, + object, pystr, + type_::PyAttributes, + }, + bytecode::{self, CodeFlags, CodeUnit, Instruction}, + class::StaticType, + common::rc::PyRc, + exceptions, + function::{ + HeapMethodDef, IntoPyGetterFunc, IntoPyNativeFn, IntoPySetterFunc, PyMethodDef, + PyMethodFlags, + }, + intern::{InternableString, MaybeInternedString, StringPool}, + object::{Py, PyObjectPayload, PyObjectRef, PyPayload, PyRef}, + types::{PyTypeFlags, PyTypeSlots, TypeZoo}, +}; +use malachite_bigint::BigInt; +use num_complex::Complex64; +use num_traits::ToPrimitive; +use rustpython_common::lock::PyRwLock; +use rustpython_compiler_core::{OneIndexed, SourceLocation}; + +#[derive(Debug)] +pub struct Context { + pub true_value: PyRef<PyBool>, + pub false_value: PyRef<PyBool>, + pub none: PyRef<PyNone>, + pub empty_tuple: PyTupleRef, + pub empty_frozenset: PyRef<PyFrozenSet>, + pub empty_str: &'static PyStrInterned, + pub empty_bytes: PyRef<PyBytes>, + pub ellipsis: PyRef<PyEllipsis>, + pub not_implemented: PyRef<PyNotImplemented>, + + pub typing_no_default: PyRef<crate::stdlib::_typing::NoDefault>, + + pub types: TypeZoo, + pub exceptions: exceptions::ExceptionZoo, + pub int_cache_pool: Vec<PyIntRef>, + pub(crate) latin1_char_cache: Vec<PyRef<PyStr>>, + pub(crate) ascii_char_cache: Vec<PyRef<PyStr>>, + pub(crate) init_cleanup_code: PyRef<PyCode>, + // there should only be exact objects of str in here, no non-str objects and no subclasses + pub(crate) string_pool: StringPool, + pub(crate) slot_new_wrapper: PyMethodDef, + pub names: ConstName, + + // GC module state (callbacks and garbage lists) + pub gc_callbacks: PyListRef, + pub gc_garbage: PyListRef, +} + +macro_rules! declare_const_name { + ($($name:ident$(: $s:literal)?,)*) => { + #[derive(Debug, Clone, Copy)] + #[allow(non_snake_case)] + pub struct ConstName { + $(pub $name: &'static PyStrInterned,)* + } + + impl ConstName { + unsafe fn new(pool: &StringPool, typ: &Py<PyType>) -> Self { + Self { + $($name: unsafe { pool.intern(declare_const_name!(@string $name $($s)?), typ.to_owned()) },)* + } + } + } + }; + (@string $name:ident) => { stringify!($name) }; + (@string $name:ident $string:literal) => { $string }; +} + +declare_const_name! { + True, + False, + None, + NotImplemented, + Ellipsis, + + // magic methods + __abs__, + __abstractmethods__, + __add__, + __aenter__, + __aexit__, + __aiter__, + __alloc__, + __all__, + __and__, + __anext__, + __annotate__, + __annotate_func__, + __annotations__, + __annotations_cache__, + __args__, + __await__, + __bases__, + __bool__, + __build_class__, + __builtins__, + __bytes__, + __cached__, + __call__, + __ceil__, + __cformat__, + __class__, + __class_getitem__, + __classcell__, + __complex__, + __contains__, + __copy__, + __deepcopy__, + __del__, + __delattr__, + __delete__, + __delitem__, + __dict__, + __dir__, + __div__, + __divmod__, + __doc__, + __enter__, + __eq__, + __exit__, + __file__, + __firstlineno__, + __float__, + __floor__, + __floordiv__, + __format__, + __fspath__, + __ge__, + __get__, + __getattr__, + __getattribute__, + __getformat__, + __getitem__, + __getnewargs__, + __getnewargs_ex__, + __getstate__, + __gt__, + __hash__, + __iadd__, + __iand__, + __idiv__, + __ifloordiv__, + __ilshift__, + __imatmul__, + __imod__, + __import__, + __imul__, + __index__, + __init__, + __init_subclass__, + __instancecheck__, + __int__, + __invert__, + __ior__, + __ipow__, + __irshift__, + __isub__, + __iter__, + __itruediv__, + __ixor__, + __jit__, // RustPython dialect + __le__, + __len__, + __length_hint__, + __lshift__, + __lt__, + __main__, + __match_args__, + __matmul__, + __missing__, + __mod__, + __module__, + __mro_entries__, + __mul__, + __name__, + __ne__, + __neg__, + __new__, + __next__, + __objclass__, + __or__, + __orig_bases__, + __orig_class__, + __origin__, + __parameters__, + __pos__, + __pow__, + __prepare__, + __qualname__, + __radd__, + __rand__, + __rdiv__, + __rdivmod__, + __reduce__, + __reduce_ex__, + __repr__, + __reversed__, + __rfloordiv__, + __rlshift__, + __rmatmul__, + __rmod__, + __rmul__, + __ror__, + __round__, + __rpow__, + __rrshift__, + __rshift__, + __rsub__, + __rtruediv__, + __rxor__, + __set__, + __setattr__, + __setitem__, + __setstate__, + __set_name__, + __slots__, + __slotnames__, + __str__, + __sub__, + __subclasscheck__, + __subclasshook__, + __subclasses__, + __sizeof__, + __truediv__, + __trunc__, + __type_params__, + __typing_subst__, + __typing_is_unpacked_typevartuple__, + __typing_prepare_subst__, + __typing_unpacked_tuple_args__, + __weakref__, + __xor__, + + // common names + _attributes, + _fields, + _defaultaction, + _onceregistry, + _showwarnmsg, + defaultaction, + onceregistry, + filters, + backslashreplace, + close, + copy, + decode, + encode, + flush, + ignore, + items, + keys, + modules, + n_fields, + n_sequence_fields, + n_unnamed_fields, + namereplace, + replace, + strict, + surrogateescape, + surrogatepass, + update, + utf_8: "utf-8", + values, + version, + WarningMessage, + xmlcharrefreplace, +} + +// Basic objects: +impl Context { + pub const INT_CACHE_POOL_RANGE: core::ops::RangeInclusive<i32> = (-5)..=256; + const INT_CACHE_POOL_MIN: i32 = *Self::INT_CACHE_POOL_RANGE.start(); + + pub fn genesis() -> &'static PyRc<Self> { + rustpython_common::static_cell! { + static CONTEXT: PyRc<Context>; + } + CONTEXT.get_or_init(|| { + let ctx = PyRc::new(Self::init_genesis()); + // SAFETY: ctx is heap-allocated via PyRc and will be stored in + // the CONTEXT static cell, so the Context lives for 'static. + let ctx_ref: &'static Context = unsafe { &*PyRc::as_ptr(&ctx) }; + crate::types::TypeZoo::extend(ctx_ref); + crate::exceptions::ExceptionZoo::extend(ctx_ref); + ctx + }) + } + + fn init_genesis() -> Self { + flame_guard!("init Context"); + let types = TypeZoo::init(); + let exceptions = exceptions::ExceptionZoo::init(); + + #[inline] + fn create_object<T: PyObjectPayload>(payload: T, cls: &'static Py<PyType>) -> PyRef<T> { + PyRef::new_ref(payload, cls.to_owned(), None) + } + + let none = create_object(PyNone, PyNone::static_type()); + let ellipsis = create_object(PyEllipsis, PyEllipsis::static_type()); + let not_implemented = create_object(PyNotImplemented, PyNotImplemented::static_type()); + + let typing_no_default = create_object( + crate::stdlib::_typing::NoDefault, + crate::stdlib::_typing::NoDefault::static_type(), + ); + + let int_cache_pool = Self::INT_CACHE_POOL_RANGE + .map(|v| { + PyRef::new_ref( + PyInt::from(BigInt::from(v)), + types.int_type.to_owned(), + None, + ) + }) + .collect(); + let latin1_char_cache: Vec<PyRef<PyStr>> = (0u8..=255) + .map(|b| create_object(PyStr::from(char::from(b)), types.str_type)) + .collect(); + let ascii_char_cache = latin1_char_cache[..128].to_vec(); + + let true_value = create_object(PyBool(PyInt::from(1)), types.bool_type); + let false_value = create_object(PyBool(PyInt::from(0)), types.bool_type); + + let empty_tuple = create_object( + PyTuple::new_unchecked(Vec::new().into_boxed_slice()), + types.tuple_type, + ); + let empty_frozenset = PyRef::new_ref( + PyFrozenSet::default(), + types.frozenset_type.to_owned(), + None, + ); + + let string_pool = StringPool::default(); + let names = unsafe { ConstName::new(&string_pool, types.str_type) }; + + let slot_new_wrapper = PyMethodDef::new_const( + names.__new__.as_str(), + PyType::__new__, + PyMethodFlags::METHOD, + None, + ); + let init_cleanup_code = Self::new_init_cleanup_code(&types, &names); + + let empty_str = unsafe { string_pool.intern("", types.str_type.to_owned()) }; + let empty_bytes = create_object(PyBytes::from(Vec::new()), types.bytes_type); + + // GC callbacks and garbage lists + let gc_callbacks = PyRef::new_ref(PyList::default(), types.list_type.to_owned(), None); + let gc_garbage = PyRef::new_ref(PyList::default(), types.list_type.to_owned(), None); + + Self { + true_value, + false_value, + none, + empty_tuple, + empty_frozenset, + empty_str, + empty_bytes, + ellipsis, + + not_implemented, + typing_no_default, + + types, + exceptions, + int_cache_pool, + latin1_char_cache, + ascii_char_cache, + init_cleanup_code, + string_pool, + slot_new_wrapper, + names, + + gc_callbacks, + gc_garbage, + } + } + + fn new_init_cleanup_code(types: &TypeZoo, names: &ConstName) -> PyRef<PyCode> { + let loc = SourceLocation { + line: OneIndexed::MIN, + character_offset: OneIndexed::from_zero_indexed(0), + }; + let instructions = [ + CodeUnit { + op: Instruction::ExitInitCheck, + arg: 0.into(), + }, + CodeUnit { + op: Instruction::ReturnValue, + arg: 0.into(), + }, + CodeUnit { + op: Instruction::Resume { + context: bytecode::Arg::marker(), + }, + arg: 0.into(), + }, + ]; + let code = bytecode::CodeObject { + instructions: instructions.into(), + locations: vec![(loc, loc); instructions.len()].into_boxed_slice(), + flags: CodeFlags::OPTIMIZED, + posonlyarg_count: 0, + arg_count: 0, + kwonlyarg_count: 0, + source_path: names.__init__, + first_line_number: None, + max_stackdepth: 2, + obj_name: names.__init__, + qualname: names.__init__, + cell2arg: None, + constants: core::iter::empty().collect(), + names: Vec::new().into_boxed_slice(), + varnames: Vec::new().into_boxed_slice(), + cellvars: Vec::new().into_boxed_slice(), + freevars: Vec::new().into_boxed_slice(), + linetable: Vec::new().into_boxed_slice(), + exceptiontable: Vec::new().into_boxed_slice(), + }; + PyRef::new_ref(PyCode::new(code), types.code_type.to_owned(), None) + } + + pub fn intern_str<S: InternableString>(&self, s: S) -> &'static PyStrInterned { + unsafe { self.string_pool.intern(s, self.types.str_type.to_owned()) } + } + + pub fn interned_str<S: MaybeInternedString + ?Sized>( + &self, + s: &S, + ) -> Option<&'static PyStrInterned> { + self.string_pool.interned(s) + } + + #[inline(always)] + pub fn none(&self) -> PyObjectRef { + self.none.clone().into() + } + + #[inline(always)] + pub fn not_implemented(&self) -> PyObjectRef { + self.not_implemented.clone().into() + } + + #[inline] + pub fn empty_tuple_typed<T>(&self) -> &Py<PyTuple<T>> { + let py: &Py<PyTuple> = &self.empty_tuple; + unsafe { core::mem::transmute(py) } + } + + // universal pyref constructor + pub fn new_pyref<T, P>(&self, value: T) -> PyRef<P> + where + T: Into<P>, + P: PyPayload + core::fmt::Debug, + { + value.into().into_ref(self) + } + + // shortcuts for common type + + #[inline] + pub fn new_int<T: Into<BigInt> + ToPrimitive>(&self, i: T) -> PyIntRef { + if let Some(i) = i.to_i32() + && Self::INT_CACHE_POOL_RANGE.contains(&i) + { + let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; + return self.int_cache_pool[inner_idx].clone(); + } + PyInt::from(i).into_ref(self) + } + + #[inline] + pub fn new_bigint(&self, i: &BigInt) -> PyIntRef { + if let Some(i) = i.to_i32() + && Self::INT_CACHE_POOL_RANGE.contains(&i) + { + let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; + return self.int_cache_pool[inner_idx].clone(); + } + PyInt::from(i.clone()).into_ref(self) + } + + #[inline] + pub fn new_float(&self, value: f64) -> PyRef<PyFloat> { + PyFloat::from(value).into_ref(self) + } + + #[inline] + pub fn new_complex(&self, value: Complex64) -> PyRef<PyComplex> { + PyComplex::from(value).into_ref(self) + } + + #[inline] + pub fn latin1_char(&self, ch: u8) -> PyRef<PyStr> { + self.latin1_char_cache[ch as usize].clone() + } + + #[inline] + fn latin1_singleton_index(s: &PyStr) -> Option<u8> { + let mut cps = s.as_wtf8().code_points(); + let cp = cps.next()?; + if cps.next().is_some() { + return None; + } + u8::try_from(cp.to_u32()).ok() + } + + #[inline] + pub fn new_str(&self, s: impl Into<pystr::PyStr>) -> PyRef<PyStr> { + let s = s.into(); + if let Some(ch) = Self::latin1_singleton_index(&s) { + return self.latin1_char(ch); + } + s.into_ref(self) + } + + #[inline] + pub fn new_utf8_str(&self, s: impl Into<PyUtf8Str>) -> PyRef<PyUtf8Str> { + s.into().into_ref(self) + } + + pub fn interned_or_new_str<S, M>(&self, s: S) -> PyRef<PyStr> + where + S: Into<PyStr> + AsRef<M>, + M: MaybeInternedString, + { + match self.interned_str(s.as_ref()) { + Some(s) => s.to_owned(), + None => self.new_str(s), + } + } + + #[inline] + pub fn new_bytes(&self, data: Vec<u8>) -> PyRef<PyBytes> { + if data.is_empty() { + self.empty_bytes.clone() + } else { + PyBytes::from(data).into_ref(self) + } + } + + #[inline] + pub fn new_bytearray(&self, data: Vec<u8>) -> PyRef<PyByteArray> { + PyByteArray::from(data).into_ref(self) + } + + #[inline(always)] + pub fn new_bool(&self, b: bool) -> PyRef<PyBool> { + let value = if b { + &self.true_value + } else { + &self.false_value + }; + value.to_owned() + } + + #[inline(always)] + pub fn new_tuple(&self, elements: Vec<PyObjectRef>) -> PyTupleRef { + PyTuple::new_ref(elements, self) + } + + #[inline(always)] + pub fn new_list(&self, elements: Vec<PyObjectRef>) -> PyListRef { + PyList::from(elements).into_ref(self) + } + + #[inline(always)] + pub fn new_dict(&self) -> PyDictRef { + PyDict::default().into_ref(self) + } + + pub fn new_class( + &self, + module: Option<&str>, + name: &str, + base: PyTypeRef, + slots: PyTypeSlots, + ) -> PyTypeRef { + let mut attrs = PyAttributes::default(); + if let Some(module) = module { + attrs.insert(identifier!(self, __module__), self.new_str(module).into()); + }; + PyType::new_heap( + name, + vec![base], + attrs, + slots, + self.types.type_type.to_owned(), + self, + ) + .unwrap() + } + + pub fn new_exception_type( + &self, + module: &str, + name: &str, + bases: Option<Vec<PyTypeRef>>, + ) -> PyTypeRef { + let bases = if let Some(bases) = bases { + bases + } else { + vec![self.exceptions.exception_type.to_owned()] + }; + let mut attrs = PyAttributes::default(); + attrs.insert(identifier!(self, __module__), self.new_str(module).into()); + + let interned_name = self.intern_str(name); + let slots = PyTypeSlots { + name: interned_name.as_str(), + basicsize: 0, + flags: PyTypeFlags::heap_type_flags() | PyTypeFlags::HAS_DICT, + ..PyTypeSlots::default() + }; + PyType::new_heap( + name, + bases, + attrs, + slots, + self.types.type_type.to_owned(), + self, + ) + .unwrap() + } + + pub fn new_method_def<F, FKind>( + &self, + name: &'static str, + f: F, + flags: PyMethodFlags, + doc: Option<&'static str>, + ) -> PyRef<HeapMethodDef> + where + F: IntoPyNativeFn<FKind>, + { + let def = PyMethodDef { + name, + func: Box::leak(Box::new(f.into_func())), + flags, + doc, + }; + let payload = HeapMethodDef::new(def); + PyRef::new_ref(payload, self.types.method_def.to_owned(), None) + } + + #[inline] + pub fn new_member( + &self, + name: &str, + member_kind: MemberKind, + getter: fn(&VirtualMachine, PyObjectRef) -> PyResult, + setter: MemberSetterFunc, + class: &'static Py<PyType>, + ) -> PyRef<PyMemberDescriptor> { + let member_def = PyMemberDef { + name: name.to_owned(), + kind: member_kind, + getter: MemberGetter::Getter(getter), + setter: MemberSetter::Setter(setter), + doc: None, + }; + let member_descriptor = PyMemberDescriptor { + common: PyDescriptorOwned { + typ: class.to_owned(), + name: self.intern_str(name), + qualname: PyRwLock::new(None), + }, + member: member_def, + }; + member_descriptor.into_ref(self) + } + + pub fn new_readonly_getset<F, T>( + &self, + name: impl Into<String>, + class: &'static Py<PyType>, + f: F, + ) -> PyRef<PyGetSet> + where + F: IntoPyGetterFunc<T>, + { + let name = name.into(); + let getset = PyGetSet::new(name, class).with_get(f); + PyRef::new_ref(getset, self.types.getset_type.to_owned(), None) + } + + pub fn new_static_getset<G, S, T, U>( + &self, + name: impl Into<String>, + class: &'static Py<PyType>, + g: G, + s: S, + ) -> PyRef<PyGetSet> + where + G: IntoPyGetterFunc<T>, + S: IntoPySetterFunc<U>, + { + let name = name.into(); + let getset = PyGetSet::new(name, class).with_get(g).with_set(s); + PyRef::new_ref(getset, self.types.getset_type.to_owned(), None) + } + + /// Creates a new `PyGetSet` with a heap type. + /// + /// # Safety + /// In practice, this constructor is safe because a getset is always owned by its `class` type. + /// However, it can be broken if used unconventionally. + pub unsafe fn new_getset<G, S, T, U>( + &self, + name: impl Into<String>, + class: &Py<PyType>, + g: G, + s: S, + ) -> PyRef<PyGetSet> + where + G: IntoPyGetterFunc<T>, + S: IntoPySetterFunc<U>, + { + let class = unsafe { &*(class as *const _) }; + self.new_static_getset(name, class, g, s) + } + + pub fn new_base_object(&self, class: PyTypeRef, dict: Option<PyDictRef>) -> PyObjectRef { + debug_assert_eq!( + class.slots.flags.contains(PyTypeFlags::HAS_DICT), + dict.is_some() + ); + PyRef::new_ref(object::PyBaseObject, class, dict).into() + } + + pub fn new_code(&self, code: impl code::IntoCodeObject) -> PyRef<PyCode> { + let code = code.into_code_object(self); + PyRef::new_ref(PyCode::new(code), self.types.code_type.to_owned(), None) + } +} + +impl AsRef<Self> for Context { + fn as_ref(&self) -> &Self { + self + } +} diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs new file mode 100644 index 00000000000..5bf7436e958 --- /dev/null +++ b/crates/vm/src/vm/interpreter.rs @@ -0,0 +1,574 @@ +#[cfg(all(unix, feature = "threading"))] +use super::StopTheWorldState; +use super::{Context, PyConfig, PyGlobalState, VirtualMachine, setting::Settings, thread}; +use crate::{ + PyResult, builtins, common::rc::PyRc, frozen::FrozenModule, getpath, py_freeze, stdlib::atexit, + vm::PyBaseExceptionRef, +}; +use alloc::collections::BTreeMap; +use core::sync::atomic::Ordering; + +type InitFunc = Box<dyn FnOnce(&mut VirtualMachine)>; + +/// Configuration builder for constructing an Interpreter. +/// +/// This is the preferred way to configure and create an interpreter with custom modules. +/// Modules must be registered before the interpreter is built, +/// similar to CPython's `PyImport_AppendInittab` which must be called before `Py_Initialize`. +/// +/// # Example +/// ``` +/// use rustpython_vm::Interpreter; +/// +/// let builder = Interpreter::builder(Default::default()); +/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx)) +/// let interp = builder.build(); +/// ``` +pub struct InterpreterBuilder { + settings: Settings, + pub ctx: PyRc<Context>, + module_defs: Vec<&'static builtins::PyModuleDef>, + frozen_modules: Vec<(&'static str, FrozenModule)>, + init_hooks: Vec<InitFunc>, +} + +/// Private helper to initialize a VM with settings, context, and custom initialization. +fn initialize_main_vm<F>( + settings: Settings, + ctx: PyRc<Context>, + module_defs: Vec<&'static builtins::PyModuleDef>, + frozen_modules: Vec<(&'static str, FrozenModule)>, + init_hooks: Vec<InitFunc>, + init: F, +) -> (VirtualMachine, PyRc<PyGlobalState>) +where + F: FnOnce(&mut VirtualMachine), +{ + use crate::codecs::CodecsRegistry; + use crate::common::hash::HashSecret; + use crate::common::lock::PyMutex; + use crate::warn::WarningsState; + use core::sync::atomic::{AtomicBool, AtomicU64}; + use crossbeam_utils::atomic::AtomicCell; + + let paths = getpath::init_path_config(&settings); + let config = PyConfig::new(settings, paths); + + // Build module_defs map from builtin modules + additional modules + let mut all_module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef> = + crate::stdlib::builtin_module_defs(&ctx) + .into_iter() + .chain(module_defs) + .map(|def| (def.name.as_str(), def)) + .collect(); + + // Register sysconfigdata under platform-specific name as well + if let Some(&sysconfigdata_def) = all_module_defs.get("_sysconfigdata") { + use std::sync::OnceLock; + static SYSCONFIGDATA_NAME: OnceLock<&'static str> = OnceLock::new(); + let leaked_name = *SYSCONFIGDATA_NAME.get_or_init(|| { + let name = crate::stdlib::sys::sysconfigdata_name(); + Box::leak(name.into_boxed_str()) + }); + all_module_defs.insert(leaked_name, sysconfigdata_def); + } + + // Create hash secret + let seed = match config.settings.hash_seed { + Some(seed) => seed, + None => super::process_hash_secret_seed(), + }; + let hash_secret = HashSecret::new(seed); + + // Create codec registry and warnings state + let codec_registry = CodecsRegistry::new(&ctx); + let warnings = WarningsState::init_state(&ctx); + + // Create int_max_str_digits + let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits { + -1 => 4300, + other => other, + } as usize); + + // Initialize frozen modules (core + user-provided) + let mut frozen: std::collections::HashMap<&'static str, FrozenModule, ahash::RandomState> = + core_frozen_inits().collect(); + frozen.extend(frozen_modules); + + // Create PyGlobalState + let global_state = PyRc::new(PyGlobalState { + config, + module_defs: all_module_defs, + frozen, + stacksize: AtomicCell::new(0), + thread_count: AtomicCell::new(0), + hash_secret, + atexit_funcs: PyMutex::default(), + codec_registry, + finalizing: AtomicBool::new(false), + warnings, + override_frozen_modules: AtomicCell::new(0), + before_forkers: PyMutex::default(), + after_forkers_child: PyMutex::default(), + after_forkers_parent: PyMutex::default(), + int_max_str_digits, + switch_interval: AtomicCell::new(0.005), + global_trace_func: PyMutex::default(), + global_profile_func: PyMutex::default(), + #[cfg(feature = "threading")] + main_thread_ident: AtomicCell::new(0), + #[cfg(feature = "threading")] + thread_frames: parking_lot::Mutex::new(std::collections::HashMap::new()), + #[cfg(feature = "threading")] + thread_handles: parking_lot::Mutex::new(Vec::new()), + #[cfg(feature = "threading")] + shutdown_handles: parking_lot::Mutex::new(Vec::new()), + monitoring: PyMutex::default(), + monitoring_events: AtomicCell::new(0), + instrumentation_version: AtomicU64::new(0), + #[cfg(all(unix, feature = "threading"))] + stop_the_world: StopTheWorldState::new(), + }); + + // Create VM with the global state + // Note: Don't clone here - init_hooks need exclusive access to mutate state + let mut vm = VirtualMachine::new(ctx, global_state); + + // Execute initialization hooks (can mutate vm.state) + for hook in init_hooks { + hook(&mut vm); + } + + // Call custom init function (can mutate vm.state) + init(&mut vm); + + vm.initialize(); + + // Clone global_state for Interpreter after all initialization is done + let global_state = vm.state.clone(); + (vm, global_state) +} + +impl InterpreterBuilder { + /// Create a new interpreter configuration with default settings. + pub fn new() -> Self { + Self { + settings: Settings::default(), + ctx: Context::genesis().clone(), + module_defs: Vec::new(), + frozen_modules: Vec::new(), + init_hooks: Vec::new(), + } + } + + /// Set custom settings for the interpreter. + /// + /// If called multiple times, only the last settings will be used. + pub fn settings(mut self, settings: Settings) -> Self { + self.settings = settings; + self + } + + /// Add a single native module definition. + /// + /// # Example + /// ``` + /// use rustpython_vm::{Interpreter, builtins::PyModuleDef}; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // Note: In practice, use module_def from your #[pymodule] + /// // let def = mymodule::module_def(&builder.ctx); + /// // let interp = builder.add_native_module(def).build(); + /// let interp = builder.build(); + /// ``` + pub fn add_native_module(self, def: &'static builtins::PyModuleDef) -> Self { + self.add_native_modules(&[def]) + } + + /// Add multiple native module definitions. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // In practice, use module_defs from rustpython_stdlib: + /// // let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + /// // let interp = builder.add_native_modules(&defs).build(); + /// let interp = builder.build(); + /// ``` + pub fn add_native_modules(mut self, defs: &[&'static builtins::PyModuleDef]) -> Self { + self.module_defs.extend_from_slice(defs); + self + } + + /// Add a custom initialization hook. + /// + /// Hooks are executed in the order they are added during interpreter creation. + /// This function will be called after modules are registered but before + /// the VM is initialized, allowing for additional customization. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let interp = Interpreter::builder(Default::default()) + /// .init_hook(|vm| { + /// // Custom initialization + /// }) + /// .build(); + /// ``` + pub fn init_hook<F>(mut self, init: F) -> Self + where + F: FnOnce(&mut VirtualMachine) + 'static, + { + self.init_hooks.push(Box::new(init)); + self + } + + /// Add frozen modules to the interpreter. + /// + /// Frozen modules are Python modules compiled into the binary. + /// This method accepts any iterator of (name, FrozenModule) pairs. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let interp = Interpreter::builder(Default::default()) + /// // In practice: .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + /// .build(); + /// ``` + pub fn add_frozen_modules<I>(mut self, frozen: I) -> Self + where + I: IntoIterator<Item = (&'static str, FrozenModule)>, + { + self.frozen_modules.extend(frozen); + self + } + + /// Build the interpreter. + /// + /// This consumes the configuration and returns a fully initialized Interpreter. + pub fn build(self) -> Interpreter { + let (vm, global_state) = initialize_main_vm( + self.settings, + self.ctx, + self.module_defs, + self.frozen_modules, + self.init_hooks, + |_| {}, // No additional init needed + ); + Interpreter { global_state, vm } + } + + /// Alias for `build()` for compatibility with the `interpreter()` pattern. + pub fn interpreter(self) -> Interpreter { + self.build() + } +} + +impl Default for InterpreterBuilder { + fn default() -> Self { + Self::new() + } +} + +/// The general interface for the VM +/// +/// # Examples +/// Runs a simple embedded hello world program. +/// ``` +/// use rustpython_vm::Interpreter; +/// use rustpython_vm::compiler::Mode; +/// Interpreter::without_stdlib(Default::default()).enter(|vm| { +/// let scope = vm.new_scope_with_builtins(); +/// let source = r#"print("Hello World!")"#; +/// let code_obj = vm.compile( +/// source, +/// Mode::Exec, +/// "<embedded>".to_owned(), +/// ).map_err(|err| vm.new_syntax_error(&err, Some(source))).unwrap(); +/// vm.run_code_obj(code_obj, scope).unwrap(); +/// }); +/// ``` +pub struct Interpreter { + pub global_state: PyRc<PyGlobalState>, + vm: VirtualMachine, +} + +impl Interpreter { + /// Create a new interpreter configuration builder. + /// + /// # Example + /// ``` + /// use rustpython_vm::Interpreter; + /// + /// let builder = Interpreter::builder(Default::default()); + /// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx)) + /// let interp = builder.build(); + /// ``` + pub fn builder(settings: Settings) -> InterpreterBuilder { + InterpreterBuilder::new().settings(settings) + } + + /// This is a bare unit to build up an interpreter without the standard library. + /// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterBuilder`. + /// To create an interpreter without the `rustpython` crate, but only with `rustpython-vm`, + /// try to build one from the source code of `InterpreterBuilder`. It will not be a one-liner but it also will not be too hard. + pub fn without_stdlib(settings: Settings) -> Self { + Self::with_init(settings, |_| {}) + } + + /// Create with initialize function taking mutable vm reference. + /// + /// Note: This is a legacy API. To add stdlib, use `Interpreter::builder()` instead. + pub fn with_init<F>(settings: Settings, init: F) -> Self + where + F: FnOnce(&mut VirtualMachine), + { + let (vm, global_state) = initialize_main_vm( + settings, + Context::genesis().clone(), + Vec::new(), // No module_defs + Vec::new(), // No frozen_modules + Vec::new(), // No init_hooks + init, + ); + Self { global_state, vm } + } + + /// Run a function with the main virtual machine and return a PyResult of the result. + /// + /// To enter vm context multiple times or to avoid buffer/exception management, this function is preferred. + /// `enter` is lightweight and it returns a python object in PyResult. + /// You can stop or continue the execution multiple times by calling `enter`. + /// + /// To finalize the vm once all desired `enter`s are called, calling `finalize` will be helpful. + /// + /// See also [`Interpreter::run`] for managed way to run the interpreter. + pub fn enter<F, R>(&self, f: F) -> R + where + F: FnOnce(&VirtualMachine) -> R, + { + thread::enter_vm(&self.vm, || f(&self.vm)) + } + + /// Run [`Interpreter::enter`] and call [`VirtualMachine::expect_pyresult`] for the result. + /// + /// This function is useful when you want to expect a result from the function, + /// but also print useful panic information when exception raised. + /// + /// See also [`Interpreter::enter`] and [`VirtualMachine::expect_pyresult`] for more information. + pub fn enter_and_expect<F, R>(&self, f: F, msg: &str) -> R + where + F: FnOnce(&VirtualMachine) -> PyResult<R>, + { + self.enter(|vm| { + let result = f(vm); + vm.expect_pyresult(result, msg) + }) + } + + /// Run a function with the main virtual machine and return exit code. + /// + /// To enter vm context only once and safely terminate the vm, this function is preferred. + /// Unlike [`Interpreter::enter`], `run` calls finalize and returns exit code. + /// You will not be able to obtain Python exception in this way. + /// + /// See [`Interpreter::finalize`] for the finalization steps. + /// See also [`Interpreter::enter`] for pure function call to obtain Python exception. + pub fn run<F>(self, f: F) -> u32 + where + F: FnOnce(&VirtualMachine) -> PyResult<()>, + { + let res = self.enter(|vm| f(vm)); + self.finalize(res.err()) + } + + /// Finalize vm and turns an exception to exit code. + /// + /// Finalization steps (matching Py_FinalizeEx): + /// 1. Flush stdout and stderr. + /// 1. Handle exit exception and turn it to exit code. + /// 1. Call threading._shutdown() to join non-daemon threads. + /// 1. Run atexit exit functions. + /// 1. Set finalizing flag (suppresses unraisable exceptions from __del__). + /// 1. Forced GC collection pass (collect cycles while builtins are available). + /// 1. Module finalization (finalize_modules). + /// 1. Final stdout/stderr flush. + /// + /// Note that calling `finalize` is not necessary by purpose though. + pub fn finalize(self, exc: Option<PyBaseExceptionRef>) -> u32 { + self.enter(|vm| { + vm.flush_std(); + + // See if any exception leaked out: + let exit_code = if let Some(exc) = exc { + vm.handle_exit_exception(exc) + } else { + 0 + }; + + // Wait for thread shutdown - call threading._shutdown() if available. + // This waits for all non-daemon threads to complete. + // threading module may not be imported, so ignore import errors. + if let Ok(threading) = vm.import("threading", 0) + && let Ok(shutdown) = threading.get_attr("_shutdown", vm) + && let Err(e) = shutdown.call((), vm) + { + vm.run_unraisable( + e, + Some("Exception ignored in threading shutdown".to_owned()), + threading, + ); + } + + // Run atexit handlers before setting finalizing flag. + // This allows unraisable exceptions from atexit handlers to be reported. + atexit::_run_exitfuncs(vm); + + // Now suppress unraisable exceptions from daemon threads and __del__ + // methods during the rest of shutdown. + vm.state.finalizing.store(true, Ordering::Release); + + // GC pass - collect cycles before module cleanup + crate::gc_state::gc_state().collect_force(2); + + // Module finalization: remove modules from sys.modules, GC collect + // (while builtins is still available for __del__), then clear module dicts. + vm.finalize_modules(); + + vm.flush_std(); + + exit_code + }) + } +} + +fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> { + let iter = core::iter::empty(); + macro_rules! ext_modules { + ($iter:ident, $($t:tt)*) => { + let $iter = $iter.chain(py_freeze!($($t)*)); + }; + } + + // Python modules that the vm calls into, but are not actually part of the stdlib. They could + // in theory be implemented in Rust, but are easiest to do in Python for one reason or another. + // Includes _importlib_bootstrap and _importlib_bootstrap_external + ext_modules!( + iter, + dir = "../../Lib/python_builtins", + crate_name = "rustpython_compiler_core" + ); + + // core stdlib Python modules that the vm calls into, but are still used in Python + // application code, e.g. copyreg + // FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules. + // We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterBuilder::new().init_stdlib().interpreter()` + // #[cfg(not(feature = "freeze-stdlib"))] + ext_modules!( + iter, + dir = "../../Lib/core_modules", + crate_name = "rustpython_compiler_core" + ); + + // Collect frozen module entries + let mut entries: Vec<_> = iter.collect(); + + // Add test module aliases + if let Some(hello_code) = entries + .iter() + .find(|(n, _)| *n == "__hello__") + .map(|(_, m)| m.code) + { + entries.push(( + "__hello_alias__", + FrozenModule { + code: hello_code, + package: false, + }, + )); + entries.push(( + "__phello_alias__", + FrozenModule { + code: hello_code, + package: true, + }, + )); + entries.push(( + "__phello_alias__.spam", + FrozenModule { + code: hello_code, + package: false, + }, + )); + entries.push(( + "__hello_only__", + FrozenModule { + code: hello_code, + package: false, + }, + )); + } + if let Some(code) = entries + .iter() + .find(|(n, _)| *n == "__phello__") + .map(|(_, m)| m.code) + { + entries.push(( + "__phello__.__init__", + FrozenModule { + code, + package: false, + }, + )); + } + if let Some(code) = entries + .iter() + .find(|(n, _)| *n == "__phello__.ham") + .map(|(_, m)| m.code) + { + entries.push(( + "__phello__.ham.__init__", + FrozenModule { + code, + package: false, + }, + )); + } + entries.into_iter() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + PyObjectRef, + builtins::{PyStr, int}, + }; + use malachite_bigint::ToBigInt; + + #[test] + fn test_add_py_integers() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let a: PyObjectRef = vm.ctx.new_int(33_i32).into(); + let b: PyObjectRef = vm.ctx.new_int(12_i32).into(); + let res = vm._add(&a, &b).unwrap(); + let value = int::get_value(&res); + assert_eq!(*value, 45_i32.to_bigint().unwrap()); + }) + } + + #[test] + fn test_multiply_str() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let a = vm.new_pyobj(crate::common::ascii!("Hello ")); + let b = vm.new_pyobj(4_i32); + let res = vm._mul(&a, &b).unwrap(); + let value = res.downcast_ref::<PyStr>().unwrap(); + assert_eq!(value.as_wtf8(), "Hello Hello Hello Hello ") + }) + } +} diff --git a/crates/vm/src/vm/method.rs b/crates/vm/src/vm/method.rs new file mode 100644 index 00000000000..5f47c8b8c5b --- /dev/null +++ b/crates/vm/src/vm/method.rs @@ -0,0 +1,143 @@ +//! This module will be replaced once #3100 is done +//! Do not expose this type to outside of this crate + +use super::VirtualMachine; +use crate::{ + builtins::{PyBaseObject, PyStr, PyStrInterned, descriptor::PyMethodDescriptor}, + function::{IntoFuncArgs, PyMethodFlags}, + object::{AsObject, Py, PyObject, PyObjectRef, PyResult}, + types::PyTypeFlags, +}; + +#[derive(Debug)] +pub enum PyMethod { + Function { + target: PyObjectRef, + func: PyObjectRef, + }, + Attribute(PyObjectRef), +} + +impl PyMethod { + pub fn get(obj: PyObjectRef, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult<Self> { + let cls = obj.class(); + let getattro = cls.slots.getattro.load().unwrap(); + if getattro as usize != PyBaseObject::getattro as *const () as usize { + return obj.get_attr(name, vm).map(Self::Attribute); + } + + // any correct method name is always interned already. + let interned_name = vm.ctx.interned_str(name); + let mut is_method = false; + + let cls_attr = match interned_name.and_then(|name| cls.get_attr(name)) { + Some(descr) => { + let descr_cls = descr.class(); + let descr_get = if descr_cls + .slots + .flags + .has_feature(PyTypeFlags::METHOD_DESCRIPTOR) + { + // For classmethods, we need descr_get to convert instance to class + if let Some(method_descr) = descr.downcast_ref::<PyMethodDescriptor>() + && method_descr.method.flags.contains(PyMethodFlags::CLASS) + { + descr_cls.slots.descr_get.load() + } else { + is_method = true; + None + } + } else { + let descr_get = descr_cls.slots.descr_get.load(); + if let Some(descr_get) = descr_get + && descr_cls.slots.descr_set.load().is_some() + { + let cls = cls.to_owned().into(); + return descr_get(descr, Some(obj), Some(cls), vm).map(Self::Attribute); + } + descr_get + }; + Some((descr, descr_get)) + } + None => None, + }; + + if let Some(dict) = obj.dict() + && let Some(attr) = dict.get_item_opt(name, vm)? + { + return Ok(Self::Attribute(attr)); + } + + if let Some((attr, descr_get)) = cls_attr { + match descr_get { + None if is_method => Ok(Self::Function { + target: obj, + func: attr, + }), + Some(descr_get) => { + let cls = cls.to_owned().into(); + descr_get(attr, Some(obj), Some(cls), vm).map(Self::Attribute) + } + None => Ok(Self::Attribute(attr)), + } + } else if let Some(getter) = cls.get_attr(identifier!(vm, __getattr__)) { + getter.call((obj, name.to_owned()), vm).map(Self::Attribute) + } else { + Err(vm.new_no_attribute_error(obj.clone(), name.to_owned())) + } + } + + pub(crate) fn get_special<const DIRECT: bool>( + obj: &PyObject, + name: &'static PyStrInterned, + vm: &VirtualMachine, + ) -> PyResult<Option<Self>> { + let obj_cls = obj.class(); + let attr = if DIRECT { + obj_cls.get_direct_attr(name) + } else { + obj_cls.get_attr(name) + }; + let func = match attr { + Some(f) => f, + None => { + return Ok(None); + } + }; + let meth = if func + .class() + .slots + .flags + .has_feature(PyTypeFlags::METHOD_DESCRIPTOR) + { + Self::Function { + target: obj.to_owned(), + func, + } + } else { + let obj_cls = obj_cls.to_owned().into(); + let attr = vm + .call_get_descriptor_specific(&func, Some(obj.to_owned()), Some(obj_cls)) + .unwrap_or(Ok(func))?; + Self::Attribute(attr) + }; + Ok(Some(meth)) + } + + pub fn invoke(self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { + let (func, args) = match self { + Self::Function { target, func } => (func, args.into_method_args(target, vm)), + Self::Attribute(func) => (func, args.into_args(vm)), + }; + func.call(args, vm) + } + + #[allow(dead_code)] + pub fn invoke_ref(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { + let (func, args) = match self { + Self::Function { target, func } => (func, args.into_method_args(target.clone(), vm)), + Self::Attribute(func) => (func, args.into_args(vm)), + }; + func.call(args, vm) + } +} diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs new file mode 100644 index 00000000000..05210eb09d7 --- /dev/null +++ b/crates/vm/src/vm/mod.rs @@ -0,0 +1,2318 @@ +//! Implement virtual machine to run instructions. +//! +//! See also: +//! <https://github.com/ProgVal/pythonvm-rust/blob/master/src/processor/mod.rs> + +#[cfg(feature = "rustpython-compiler")] +mod compile; +mod context; +mod interpreter; +mod method; +#[cfg(feature = "rustpython-compiler")] +mod python_run; +mod setting; +pub mod thread; +mod vm_new; +mod vm_object; +mod vm_ops; + +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + builtins::{ + self, PyBaseExceptionRef, PyDict, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, + PyStrRef, PyTypeRef, PyUtf8Str, PyUtf8StrInterned, PyWeak, + code::PyCode, + dict::{PyDictItems, PyDictValues}, + pystr::AsPyStr, + tuple::PyTuple, + }, + codecs::CodecsRegistry, + common::{hash::HashSecret, lock::PyMutex, rc::PyRc}, + convert::ToPyObject, + exceptions::types::PyBaseException, + frame::{ExecutionResult, Frame, FrameRef}, + frozen::FrozenModule, + function::{ArgMapping, FuncArgs, PySetterValue}, + import, + protocol::PyIterIter, + scope::Scope, + signal, stdlib, + warn::WarningsState, +}; +use alloc::{borrow::Cow, collections::BTreeMap}; +#[cfg(all(unix, feature = "threading"))] +use core::sync::atomic::AtomicI64; +use core::{ + cell::{Cell, OnceCell, RefCell}, + ptr::NonNull, + sync::atomic::{AtomicBool, AtomicU64, Ordering}, +}; +use crossbeam_utils::atomic::AtomicCell; +#[cfg(unix)] +use nix::{ + sys::signal::{SaFlags, SigAction, SigSet, Signal::SIGINT, kill, sigaction}, + unistd::getpid, +}; +use std::{ + collections::{HashMap, HashSet}, + ffi::{OsStr, OsString}, +}; + +pub use context::Context; +pub use interpreter::{Interpreter, InterpreterBuilder}; +pub(crate) use method::PyMethod; +pub use setting::{CheckHashPycsMode, Paths, PyConfig, Settings}; + +pub const MAX_MEMORY_SIZE: usize = isize::MAX as usize; + +// Objects are live when they are on stack, or referenced by a name (for now) + +/// Top level container of a python virtual machine. In theory you could +/// create more instances of this struct and have them operate fully isolated. +/// +/// To construct this, please refer to the [`Interpreter`] +pub struct VirtualMachine { + pub builtins: PyRef<PyModule>, + pub sys_module: PyRef<PyModule>, + pub ctx: PyRc<Context>, + pub frames: RefCell<Vec<FramePtr>>, + /// Thread-local data stack for bump-allocating frame-local data + /// (localsplus arrays for non-generator frames). + datastack: core::cell::UnsafeCell<crate::datastack::DataStack>, + pub wasm_id: Option<String>, + exceptions: RefCell<ExceptionStack>, + pub import_func: PyObjectRef, + pub(crate) importlib: PyObjectRef, + pub profile_func: RefCell<PyObjectRef>, + pub trace_func: RefCell<PyObjectRef>, + pub use_tracing: Cell<bool>, + pub recursion_limit: Cell<usize>, + pub(crate) signal_handlers: OnceCell<Box<RefCell<[Option<PyObjectRef>; signal::NSIG]>>>, + pub(crate) signal_rx: Option<signal::UserSignalReceiver>, + pub repr_guards: RefCell<HashSet<usize>>, + pub state: PyRc<PyGlobalState>, + pub initialized: bool, + recursion_depth: Cell<usize>, + /// C stack soft limit for detecting stack overflow (like c_stack_soft_limit) + #[cfg_attr(miri, allow(dead_code))] + c_stack_soft_limit: Cell<usize>, + /// Async generator firstiter hook (per-thread, set via sys.set_asyncgen_hooks) + pub async_gen_firstiter: RefCell<Option<PyObjectRef>>, + /// Async generator finalizer hook (per-thread, set via sys.set_asyncgen_hooks) + pub async_gen_finalizer: RefCell<Option<PyObjectRef>>, + /// Current running asyncio event loop for this thread + pub asyncio_running_loop: RefCell<Option<PyObjectRef>>, + /// Current running asyncio task for this thread + pub asyncio_running_task: RefCell<Option<PyObjectRef>>, + pub(crate) callable_cache: CallableCache, +} + +/// Non-owning frame pointer for the frames stack. +/// The pointed-to frame is kept alive by the caller of with_frame_exc/resume_gen_frame. +#[derive(Copy, Clone)] +pub struct FramePtr(NonNull<Py<Frame>>); + +impl FramePtr { + /// # Safety + /// The pointed-to frame must still be alive. + pub unsafe fn as_ref(&self) -> &Py<Frame> { + unsafe { self.0.as_ref() } + } +} + +// SAFETY: FramePtr is only stored in the VM's frames Vec while the corresponding +// FrameRef is alive on the call stack. The Vec is always empty when the VM moves between threads. +unsafe impl Send for FramePtr {} + +#[derive(Debug, Default)] +struct ExceptionStack { + stack: Vec<Option<PyBaseExceptionRef>>, +} + +/// Stop-the-world state for fork safety. Before `fork()`, the requester +/// stops all other Python threads so they are not holding internal locks. +#[cfg(all(unix, feature = "threading"))] +pub struct StopTheWorldState { + /// Fast-path flag checked in the bytecode loop (like `_PY_EVAL_PLEASE_STOP_BIT`) + pub(crate) requested: AtomicBool, + /// Whether the world is currently stopped (`stw->world_stopped`). + world_stopped: AtomicBool, + /// Ident of the thread that requested the stop (like `stw->requester`) + requester: AtomicU64, + /// Signaled by suspending threads when their state transitions to SUSPENDED + notify_mutex: std::sync::Mutex<()>, + notify_cv: std::sync::Condvar, + /// Number of non-requester threads still expected to park for current stop request. + thread_countdown: AtomicI64, + /// Number of stop-the-world attempts. + stats_stop_calls: AtomicU64, + /// Most recent stop-the-world wait duration in ns. + stats_last_wait_ns: AtomicU64, + /// Total accumulated stop-the-world wait duration in ns. + stats_total_wait_ns: AtomicU64, + /// Max observed stop-the-world wait duration in ns. + stats_max_wait_ns: AtomicU64, + /// Number of poll-loop iterations spent waiting. + stats_poll_loops: AtomicU64, + /// Number of ATTACHED threads observed while polling. + stats_attached_seen: AtomicU64, + /// Number of DETACHED->SUSPENDED parks requested by requester. + stats_forced_parks: AtomicU64, + /// Number of suspend notifications from worker threads. + stats_suspend_notifications: AtomicU64, + /// Number of yield loops while attach waited on SUSPENDED->DETACHED. + stats_attach_wait_yields: AtomicU64, + /// Number of yield loops while suspend waited on SUSPENDED->DETACHED. + stats_suspend_wait_yields: AtomicU64, +} + +#[cfg(all(unix, feature = "threading"))] +#[derive(Debug, Clone, Copy)] +pub struct StopTheWorldStats { + pub stop_calls: u64, + pub last_wait_ns: u64, + pub total_wait_ns: u64, + pub max_wait_ns: u64, + pub poll_loops: u64, + pub attached_seen: u64, + pub forced_parks: u64, + pub suspend_notifications: u64, + pub attach_wait_yields: u64, + pub suspend_wait_yields: u64, + pub world_stopped: bool, +} + +#[cfg(all(unix, feature = "threading"))] +impl Default for StopTheWorldState { + fn default() -> Self { + Self::new() + } +} + +#[cfg(all(unix, feature = "threading"))] +impl StopTheWorldState { + pub const fn new() -> Self { + Self { + requested: AtomicBool::new(false), + world_stopped: AtomicBool::new(false), + requester: AtomicU64::new(0), + notify_mutex: std::sync::Mutex::new(()), + notify_cv: std::sync::Condvar::new(), + thread_countdown: AtomicI64::new(0), + stats_stop_calls: AtomicU64::new(0), + stats_last_wait_ns: AtomicU64::new(0), + stats_total_wait_ns: AtomicU64::new(0), + stats_max_wait_ns: AtomicU64::new(0), + stats_poll_loops: AtomicU64::new(0), + stats_attached_seen: AtomicU64::new(0), + stats_forced_parks: AtomicU64::new(0), + stats_suspend_notifications: AtomicU64::new(0), + stats_attach_wait_yields: AtomicU64::new(0), + stats_suspend_wait_yields: AtomicU64::new(0), + } + } + + /// Wake the stop-the-world requester (called by each thread that suspends). + pub(crate) fn notify_suspended(&self) { + self.stats_suspend_notifications + .fetch_add(1, Ordering::Relaxed); + // Synchronize with requester wait loop to avoid lost wakeups. + let _guard = self.notify_mutex.lock().unwrap(); + self.decrement_thread_countdown(1); + self.notify_cv.notify_one(); + } + + #[inline] + fn init_thread_countdown(&self, vm: &VirtualMachine) -> i64 { + let requester = self.requester.load(Ordering::Relaxed); + let registry = vm.state.thread_frames.lock(); + // Keep requested/count initialization serialized with thread-slot + // registration (which also takes this lock), matching the + // HEAD_LOCK-guarded stop-the-world bookkeeping. + self.requested.store(true, Ordering::Release); + let count = registry + .keys() + .filter(|&&thread_id| thread_id != requester) + .count(); + let count = (count.min(i64::MAX as usize)) as i64; + self.thread_countdown.store(count, Ordering::Release); + count + } + + #[inline] + fn decrement_thread_countdown(&self, n: u64) { + if n == 0 { + return; + } + let n = (n.min(i64::MAX as u64)) as i64; + let prev = self.thread_countdown.fetch_sub(n, Ordering::AcqRel); + if prev <= n { + // Clamp at 0 for safety in case of duplicate notifications. + self.thread_countdown.store(0, Ordering::Release); + } + } + + /// Try to CAS detached threads directly to SUSPENDED and check whether + /// stop countdown reached zero after parking detached threads. + fn park_detached_threads(&self, vm: &VirtualMachine) -> bool { + use thread::{THREAD_ATTACHED, THREAD_DETACHED, THREAD_SUSPENDED}; + let requester = self.requester.load(Ordering::Relaxed); + let registry = vm.state.thread_frames.lock(); + let mut attached_seen = 0u64; + let mut forced_parks = 0u64; + for (&id, slot) in registry.iter() { + if id == requester { + continue; + } + let state = slot.state.load(Ordering::Relaxed); + if state == THREAD_DETACHED { + // CAS DETACHED → SUSPENDED (park without thread cooperation) + match slot.state.compare_exchange( + THREAD_DETACHED, + THREAD_SUSPENDED, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + slot.stop_requested.store(false, Ordering::Release); + forced_parks = forced_parks.saturating_add(1); + } + Err(THREAD_ATTACHED) => { + // Set per-thread stop bit (_PY_EVAL_PLEASE_STOP_BIT). + slot.stop_requested.store(true, Ordering::Release); + // Raced with a thread re-attaching; it will self-suspend. + attached_seen = attached_seen.saturating_add(1); + } + Err(THREAD_DETACHED) => { + // Extremely unlikely race; next poll will handle it. + } + Err(THREAD_SUSPENDED) => { + slot.stop_requested.store(false, Ordering::Release); + // Another path parked it first. + } + Err(other) => { + debug_assert!( + false, + "unexpected thread state in park_detached_threads: {other}" + ); + } + } + } else if state == THREAD_ATTACHED { + // Set per-thread stop bit (_PY_EVAL_PLEASE_STOP_BIT). + slot.stop_requested.store(true, Ordering::Release); + // Thread is in bytecode — it will see `requested` and self-suspend + attached_seen = attached_seen.saturating_add(1); + } + // THREAD_SUSPENDED → already parked + } + if attached_seen != 0 { + self.stats_attached_seen + .fetch_add(attached_seen, Ordering::Relaxed); + } + if forced_parks != 0 { + self.decrement_thread_countdown(forced_parks); + self.stats_forced_parks + .fetch_add(forced_parks, Ordering::Relaxed); + } + forced_parks != 0 && self.thread_countdown.load(Ordering::Acquire) == 0 + } + + /// Stop all non-requester threads (`stop_the_world`). + /// + /// 1. Sets `requested`, marking the requester thread. + /// 2. CAS detached threads to SUSPENDED. + /// 3. Waits (polling with 1 ms condvar timeout) for attached threads + /// to self-suspend in `check_signals`. + pub fn stop_the_world(&self, vm: &VirtualMachine) { + let start = std::time::Instant::now(); + let requester_ident = crate::stdlib::_thread::get_ident(); + self.requester.store(requester_ident, Ordering::Relaxed); + self.stats_stop_calls.fetch_add(1, Ordering::Relaxed); + let initial_countdown = self.init_thread_countdown(vm); + stw_trace(format_args!("stop begin requester={requester_ident}")); + if initial_countdown == 0 { + self.world_stopped.store(true, Ordering::Release); + #[cfg(debug_assertions)] + self.debug_assert_all_non_requester_suspended(vm); + stw_trace(format_args!( + "stop end requester={requester_ident} wait_ns=0 polls=0" + )); + return; + } + + let mut polls = 0u64; + loop { + if self.park_detached_threads(vm) { + break; + } + polls = polls.saturating_add(1); + // Wait up to 1 ms for a thread to notify us it suspended. + // Re-check under the wait mutex first to avoid a lost-wake race: + // a thread may have suspended and notified right before we enter wait. + let guard = self.notify_mutex.lock().unwrap(); + if self.thread_countdown.load(Ordering::Acquire) == 0 || self.park_detached_threads(vm) + { + drop(guard); + break; + } + let _ = self + .notify_cv + .wait_timeout(guard, core::time::Duration::from_millis(1)); + } + if polls != 0 { + self.stats_poll_loops.fetch_add(polls, Ordering::Relaxed); + } + let wait_ns = start.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; + self.stats_last_wait_ns.store(wait_ns, Ordering::Relaxed); + self.stats_total_wait_ns + .fetch_add(wait_ns, Ordering::Relaxed); + let mut prev_max = self.stats_max_wait_ns.load(Ordering::Relaxed); + while wait_ns > prev_max { + match self.stats_max_wait_ns.compare_exchange_weak( + prev_max, + wait_ns, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(observed) => prev_max = observed, + } + } + self.world_stopped.store(true, Ordering::Release); + #[cfg(debug_assertions)] + self.debug_assert_all_non_requester_suspended(vm); + stw_trace(format_args!( + "stop end requester={requester_ident} wait_ns={wait_ns} polls={polls}" + )); + } + + /// Resume all suspended threads (`start_the_world`). + pub fn start_the_world(&self, vm: &VirtualMachine) { + use thread::{THREAD_DETACHED, THREAD_SUSPENDED}; + let requester = self.requester.load(Ordering::Relaxed); + stw_trace(format_args!("start begin requester={requester}")); + let registry = vm.state.thread_frames.lock(); + // Clear the request flag BEFORE waking threads. Otherwise a thread + // returning from allow_threads → attach_thread could observe + // `requested == true`, re-suspend itself, and stay parked forever. + // Keep this write under the registry lock to serialize with new + // thread-slot initialization. + self.requested.store(false, Ordering::Release); + self.world_stopped.store(false, Ordering::Release); + for (&id, slot) in registry.iter() { + if id == requester { + continue; + } + slot.stop_requested.store(false, Ordering::Release); + let state = slot.state.load(Ordering::Relaxed); + debug_assert!( + state == THREAD_SUSPENDED, + "non-requester thread not suspended at start-the-world: id={id} state={state}" + ); + if state == THREAD_SUSPENDED { + slot.state.store(THREAD_DETACHED, Ordering::Release); + slot.thread.unpark(); + } + } + drop(registry); + self.thread_countdown.store(0, Ordering::Release); + self.requester.store(0, Ordering::Relaxed); + #[cfg(debug_assertions)] + self.debug_assert_all_non_requester_detached(vm); + stw_trace(format_args!("start end requester={requester}")); + } + + /// Reset after fork in the child (only one thread alive). + pub fn reset_after_fork(&self) { + self.requested.store(false, Ordering::Relaxed); + self.world_stopped.store(false, Ordering::Relaxed); + self.requester.store(0, Ordering::Relaxed); + self.thread_countdown.store(0, Ordering::Relaxed); + stw_trace(format_args!("reset-after-fork")); + } + + #[inline] + pub(crate) fn requester_ident(&self) -> u64 { + self.requester.load(Ordering::Relaxed) + } + + #[inline] + pub(crate) fn notify_thread_gone(&self) { + let _guard = self.notify_mutex.lock().unwrap(); + self.decrement_thread_countdown(1); + self.notify_cv.notify_one(); + } + + pub fn stats_snapshot(&self) -> StopTheWorldStats { + StopTheWorldStats { + stop_calls: self.stats_stop_calls.load(Ordering::Relaxed), + last_wait_ns: self.stats_last_wait_ns.load(Ordering::Relaxed), + total_wait_ns: self.stats_total_wait_ns.load(Ordering::Relaxed), + max_wait_ns: self.stats_max_wait_ns.load(Ordering::Relaxed), + poll_loops: self.stats_poll_loops.load(Ordering::Relaxed), + attached_seen: self.stats_attached_seen.load(Ordering::Relaxed), + forced_parks: self.stats_forced_parks.load(Ordering::Relaxed), + suspend_notifications: self.stats_suspend_notifications.load(Ordering::Relaxed), + attach_wait_yields: self.stats_attach_wait_yields.load(Ordering::Relaxed), + suspend_wait_yields: self.stats_suspend_wait_yields.load(Ordering::Relaxed), + world_stopped: self.world_stopped.load(Ordering::Relaxed), + } + } + + pub fn reset_stats(&self) { + self.stats_stop_calls.store(0, Ordering::Relaxed); + self.stats_last_wait_ns.store(0, Ordering::Relaxed); + self.stats_total_wait_ns.store(0, Ordering::Relaxed); + self.stats_max_wait_ns.store(0, Ordering::Relaxed); + self.stats_poll_loops.store(0, Ordering::Relaxed); + self.stats_attached_seen.store(0, Ordering::Relaxed); + self.stats_forced_parks.store(0, Ordering::Relaxed); + self.stats_suspend_notifications.store(0, Ordering::Relaxed); + self.stats_attach_wait_yields.store(0, Ordering::Relaxed); + self.stats_suspend_wait_yields.store(0, Ordering::Relaxed); + } + + #[inline] + pub(crate) fn add_attach_wait_yields(&self, n: u64) { + if n != 0 { + self.stats_attach_wait_yields + .fetch_add(n, Ordering::Relaxed); + } + } + + #[inline] + pub(crate) fn add_suspend_wait_yields(&self, n: u64) { + if n != 0 { + self.stats_suspend_wait_yields + .fetch_add(n, Ordering::Relaxed); + } + } + + #[cfg(debug_assertions)] + fn debug_assert_all_non_requester_suspended(&self, vm: &VirtualMachine) { + use thread::THREAD_SUSPENDED; + let requester = self.requester.load(Ordering::Relaxed); + let registry = vm.state.thread_frames.lock(); + for (&id, slot) in registry.iter() { + if id == requester { + continue; + } + let state = slot.state.load(Ordering::Relaxed); + debug_assert!( + state == THREAD_SUSPENDED, + "non-requester thread not suspended during stop-the-world: id={id} state={state}" + ); + } + } + + #[cfg(debug_assertions)] + fn debug_assert_all_non_requester_detached(&self, vm: &VirtualMachine) { + use thread::THREAD_SUSPENDED; + let requester = self.requester.load(Ordering::Relaxed); + let registry = vm.state.thread_frames.lock(); + for (&id, slot) in registry.iter() { + if id == requester { + continue; + } + let state = slot.state.load(Ordering::Relaxed); + debug_assert!( + state != THREAD_SUSPENDED, + "non-requester thread still suspended after start-the-world: id={id} state={state}" + ); + } + } +} + +#[cfg(all(unix, feature = "threading"))] +pub(super) fn stw_trace_enabled() -> bool { + static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var_os("RUSTPYTHON_STW_TRACE").is_some()) +} + +#[cfg(all(unix, feature = "threading"))] +pub(super) fn stw_trace(msg: core::fmt::Arguments<'_>) { + if stw_trace_enabled() { + use core::fmt::Write as _; + + // Avoid stdio locking here: this path runs around fork where a child + // may inherit a borrowed stderr lock and panic on eprintln!/stderr. + struct FixedBuf { + buf: [u8; 512], + len: usize, + } + + impl core::fmt::Write for FixedBuf { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + if self.len >= self.buf.len() { + return Ok(()); + } + let remain = self.buf.len() - self.len; + let src = s.as_bytes(); + let n = src.len().min(remain); + self.buf[self.len..self.len + n].copy_from_slice(&src[..n]); + self.len += n; + Ok(()) + } + } + + let mut out = FixedBuf { + buf: [0u8; 512], + len: 0, + }; + let _ = writeln!( + &mut out, + "[rp-stw tid={}] {}", + crate::stdlib::_thread::get_ident(), + msg + ); + unsafe { + let _ = libc::write(libc::STDERR_FILENO, out.buf.as_ptr().cast(), out.len); + } + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct CallableCache { + pub len: Option<PyObjectRef>, + pub isinstance: Option<PyObjectRef>, + pub list_append: Option<PyObjectRef>, +} + +pub struct PyGlobalState { + pub config: PyConfig, + pub module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef>, + pub frozen: HashMap<&'static str, FrozenModule, ahash::RandomState>, + pub stacksize: AtomicCell<usize>, + pub thread_count: AtomicCell<usize>, + pub hash_secret: HashSecret, + pub atexit_funcs: PyMutex<Vec<Box<(PyObjectRef, FuncArgs)>>>, + pub codec_registry: CodecsRegistry, + pub finalizing: AtomicBool, + pub warnings: WarningsState, + pub override_frozen_modules: AtomicCell<isize>, + pub before_forkers: PyMutex<Vec<PyObjectRef>>, + pub after_forkers_child: PyMutex<Vec<PyObjectRef>>, + pub after_forkers_parent: PyMutex<Vec<PyObjectRef>>, + pub int_max_str_digits: AtomicCell<usize>, + pub switch_interval: AtomicCell<f64>, + /// Global trace function for all threads (set by sys._settraceallthreads) + pub global_trace_func: PyMutex<Option<PyObjectRef>>, + /// Global profile function for all threads (set by sys._setprofileallthreads) + pub global_profile_func: PyMutex<Option<PyObjectRef>>, + /// Main thread identifier (pthread_self on Unix) + #[cfg(feature = "threading")] + pub main_thread_ident: AtomicCell<u64>, + /// Registry of all threads' slots for sys._current_frames() and sys._current_exceptions() + #[cfg(feature = "threading")] + pub thread_frames: parking_lot::Mutex<HashMap<u64, stdlib::_thread::CurrentFrameSlot>>, + /// Registry of all ThreadHandles for fork cleanup + #[cfg(feature = "threading")] + pub thread_handles: parking_lot::Mutex<Vec<stdlib::_thread::HandleEntry>>, + /// Registry for non-daemon threads that need to be joined at shutdown + #[cfg(feature = "threading")] + pub shutdown_handles: parking_lot::Mutex<Vec<stdlib::_thread::ShutdownEntry>>, + /// sys.monitoring state (tool names, events, callbacks) + pub monitoring: PyMutex<stdlib::sys::monitoring::MonitoringState>, + /// Fast-path mask: OR of all tools' events. 0 means no monitoring overhead. + pub monitoring_events: stdlib::sys::monitoring::MonitoringEventsMask, + /// Incremented on every monitoring state change. Code objects compare their + /// local version against this to decide whether re-instrumentation is needed. + pub instrumentation_version: AtomicU64, + /// Stop-the-world state for pre-fork thread suspension + #[cfg(all(unix, feature = "threading"))] + pub stop_the_world: StopTheWorldState, +} + +pub fn process_hash_secret_seed() -> u32 { + use std::sync::OnceLock; + static SEED: OnceLock<u32> = OnceLock::new(); + // os_random is expensive, but this is only ever called once + *SEED.get_or_init(|| u32::from_ne_bytes(rustpython_common::rand::os_random())) +} + +impl VirtualMachine { + fn init_callable_cache(&mut self) -> PyResult<()> { + self.callable_cache.len = Some(self.builtins.get_attr("len", self)?); + self.callable_cache.isinstance = Some(self.builtins.get_attr("isinstance", self)?); + let list_append = self + .ctx + .types + .list_type + .get_attr(self.ctx.intern_str("append")) + .ok_or_else(|| self.new_runtime_error("failed to cache list.append".to_owned()))?; + self.callable_cache.list_append = Some(list_append); + Ok(()) + } + + /// Bump-allocate `size` bytes from the thread data stack. + /// + /// # Safety + /// The returned pointer must be freed by calling `datastack_pop` in LIFO order. + #[inline(always)] + pub(crate) fn datastack_push(&self, size: usize) -> *mut u8 { + unsafe { (*self.datastack.get()).push(size) } + } + + /// Check whether the thread data stack currently has room for `size` bytes. + #[inline(always)] + pub(crate) fn datastack_has_space(&self, size: usize) -> bool { + unsafe { (*self.datastack.get()).has_space(size) } + } + + /// Pop a previous data stack allocation. + /// + /// # Safety + /// `base` must be a pointer returned by `datastack_push` on this VM, + /// and all allocations made after it must already have been popped. + #[inline(always)] + pub(crate) unsafe fn datastack_pop(&self, base: *mut u8) { + unsafe { (*self.datastack.get()).pop(base) } + } + + /// Temporarily detach the current thread (ATTACHED → DETACHED) while + /// running `f`, then re-attach afterwards. Allows `stop_the_world` to + /// park this thread during blocking syscalls. + /// + /// Equivalent to CPython's `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS`. + #[inline] + pub fn allow_threads<R>(&self, f: impl FnOnce() -> R) -> R { + thread::allow_threads(self, f) + } + + /// Check whether the current thread is the main thread. + /// Mirrors `_Py_ThreadCanHandleSignals`. + #[allow(dead_code)] + pub(crate) fn is_main_thread(&self) -> bool { + #[cfg(feature = "threading")] + { + crate::stdlib::_thread::get_ident() == self.state.main_thread_ident.load() + } + #[cfg(not(feature = "threading"))] + { + true + } + } + + /// Create a new `VirtualMachine` structure. + pub(crate) fn new(ctx: PyRc<Context>, state: PyRc<PyGlobalState>) -> Self { + flame_guard!("new VirtualMachine"); + + // make a new module without access to the vm; doesn't + // set __spec__, __loader__, etc. attributes + let new_module = |def| { + PyRef::new_ref( + PyModule::from_def(def), + ctx.types.module_type.to_owned(), + Some(ctx.new_dict()), + ) + }; + + // Hard-core modules: + let builtins = new_module(stdlib::builtins::module_def(&ctx)); + let sys_module = new_module(stdlib::sys::module_def(&ctx)); + + let import_func = ctx.none(); + let importlib = ctx.none(); + let profile_func = RefCell::new(ctx.none()); + let trace_func = RefCell::new(ctx.none()); + let signal_handlers = OnceCell::from(signal::new_signal_handlers()); + + let vm = Self { + builtins, + sys_module, + ctx, + frames: RefCell::new(vec![]), + datastack: core::cell::UnsafeCell::new(crate::datastack::DataStack::new()), + wasm_id: None, + exceptions: RefCell::default(), + import_func, + importlib, + profile_func, + trace_func, + use_tracing: Cell::new(false), + recursion_limit: Cell::new(if cfg!(debug_assertions) { 256 } else { 1000 }), + signal_handlers, + signal_rx: None, + repr_guards: RefCell::default(), + state, + initialized: false, + recursion_depth: Cell::new(0), + c_stack_soft_limit: Cell::new(Self::calculate_c_stack_soft_limit()), + async_gen_firstiter: RefCell::new(None), + async_gen_finalizer: RefCell::new(None), + asyncio_running_loop: RefCell::new(None), + asyncio_running_task: RefCell::new(None), + callable_cache: CallableCache::default(), + }; + + if vm.state.hash_secret.hash_str("") + != vm + .ctx + .interned_str("") + .expect("empty str must be interned") + .hash(&vm) + { + panic!("Interpreters in same process must share the hash seed"); + } + + vm.builtins.init_dict( + vm.ctx.intern_str("builtins"), + Some(vm.ctx.intern_str(stdlib::builtins::DOC.unwrap()).to_owned()), + &vm, + ); + vm.sys_module.init_dict( + vm.ctx.intern_str("sys"), + Some(vm.ctx.intern_str(stdlib::sys::DOC.unwrap()).to_owned()), + &vm, + ); + // let name = vm.sys_module.get_attr("__name__", &vm).unwrap(); + vm + } + + /// set up the encodings search function + /// init_importlib must be called before this call + #[cfg(feature = "encodings")] + fn import_encodings(&mut self) -> PyResult<()> { + self.import("encodings", 0).map_err(|import_err| { + let rustpythonpath_env = std::env::var("RUSTPYTHONPATH").ok(); + let pythonpath_env = std::env::var("PYTHONPATH").ok(); + let env_set = rustpythonpath_env.as_ref().is_some() || pythonpath_env.as_ref().is_some(); + let path_contains_env = self.state.config.paths.module_search_paths.iter().any(|s| { + Some(s.as_str()) == rustpythonpath_env.as_deref() || Some(s.as_str()) == pythonpath_env.as_deref() + }); + + let guide_message = if cfg!(feature = "freeze-stdlib") { + "`rustpython_pylib` may not be set while using `freeze-stdlib` feature. Try using `rustpython::InterpreterBuilder::init_stdlib` or manually call `builder.add_frozen_modules(rustpython_pylib::FROZEN_STDLIB)` in `rustpython_vm::Interpreter::builder()`." + } else if !env_set { + "Neither RUSTPYTHONPATH nor PYTHONPATH is set. Try setting one of them to the stdlib directory." + } else if path_contains_env { + "RUSTPYTHONPATH or PYTHONPATH is set, but it doesn't contain the encodings library. If you are customizing the RustPython vm/interpreter, try adding the stdlib directory to the path. If you are developing the RustPython interpreter, it might be a bug during development." + } else { + "RUSTPYTHONPATH or PYTHONPATH is set, but it wasn't loaded to `PyConfig::paths::module_search_paths`. If you are going to customize the RustPython vm/interpreter, those environment variables are not loaded in the Settings struct by default. Please try creating a customized instance of the Settings struct. If you are developing the RustPython interpreter, it might be a bug during development." + }; + + let mut msg = format!( + "RustPython could not import the encodings module. It usually means something went wrong. Please carefully read the following messages and follow the steps.\n\ + \n\ + {guide_message}"); + if !cfg!(feature = "freeze-stdlib") { + msg += "\n\ + If you don't have access to a consistent external environment (e.g. targeting wasm, embedding \ + rustpython in another application), try enabling the `freeze-stdlib` feature.\n\ + If this is intended and you want to exclude the encodings module from your interpreter, please remove the `encodings` feature from `rustpython-vm` crate."; + } + + let err = self.new_runtime_error(msg); + err.set___cause__(Some(import_err)); + err + })?; + Ok(()) + } + + fn import_ascii_utf8_encodings(&mut self) -> PyResult<()> { + // Use the Python import machinery (FrozenImporter) so modules get + // proper __spec__ and __loader__ attributes. + self.import("codecs", 0)?; + + // Use dotted names when freeze-stdlib is enabled (modules come from Lib/encodings/), + // otherwise use underscored names (modules come from core_modules/). + let (ascii_module_name, utf8_module_name) = if cfg!(feature = "freeze-stdlib") { + ("encodings.ascii", "encodings.utf_8") + } else { + ("encodings_ascii", "encodings_utf_8") + }; + + // Register ascii encoding + // __import__("encodings.ascii") returns top-level "encodings", so + // look up the actual submodule in sys.modules. + self.import(ascii_module_name, 0)?; + let sys_modules = self.sys_module.get_attr(identifier!(self, modules), self)?; + let ascii_module = sys_modules.get_item(ascii_module_name, self)?; + let getregentry = ascii_module.get_attr("getregentry", self)?; + let codec_info = getregentry.call((), self)?; + self.state + .codec_registry + .register_manual("ascii", codec_info.try_into_value(self)?)?; + + // Register utf-8 encoding (also as "utf8" alias since normalize_encoding_name + // maps "utf-8" → "utf_8" but leaves "utf8" as-is) + self.import(utf8_module_name, 0)?; + let utf8_module = sys_modules.get_item(utf8_module_name, self)?; + let getregentry = utf8_module.get_attr("getregentry", self)?; + let codec_info = getregentry.call((), self)?; + let utf8_codec: crate::codecs::PyCodec = codec_info.try_into_value(self)?; + self.state + .codec_registry + .register_manual("utf-8", utf8_codec.clone())?; + self.state + .codec_registry + .register_manual("utf8", utf8_codec)?; + + // Register latin-1 / iso8859-1 aliases needed very early for stdio + // bootstrap (e.g. PYTHONIOENCODING=latin-1). + if cfg!(feature = "freeze-stdlib") { + self.import("encodings.latin_1", 0)?; + let latin1_module = sys_modules.get_item("encodings.latin_1", self)?; + let getregentry = latin1_module.get_attr("getregentry", self)?; + let codec_info = getregentry.call((), self)?; + let latin1_codec: crate::codecs::PyCodec = codec_info.try_into_value(self)?; + for name in ["latin-1", "latin_1", "latin1", "iso8859-1", "iso8859_1"] { + self.state + .codec_registry + .register_manual(name, latin1_codec.clone())?; + } + } + Ok(()) + } + + fn initialize(&mut self) { + flame_guard!("init VirtualMachine"); + + if self.initialized { + panic!("Double Initialize Error"); + } + + // Initialize main thread ident before any threading operations + #[cfg(feature = "threading")] + stdlib::_thread::init_main_thread_ident(self); + + stdlib::builtins::init_module(self, &self.builtins); + let callable_cache_init = self.init_callable_cache(); + self.expect_pyresult(callable_cache_init, "failed to initialize callable cache"); + stdlib::sys::init_module(self, &self.sys_module, &self.builtins); + self.expect_pyresult( + stdlib::sys::set_bootstrap_stderr(self), + "failed to initialize bootstrap stderr", + ); + + let mut essential_init = || -> PyResult { + import::import_builtin(self, "_typing")?; + #[cfg(all(not(target_arch = "wasm32"), feature = "host_env"))] + import::import_builtin(self, "_signal")?; + #[cfg(any(feature = "parser", feature = "compiler"))] + import::import_builtin(self, "_ast")?; + #[cfg(not(feature = "threading"))] + import::import_frozen(self, "_thread")?; + let importlib = import::init_importlib_base(self)?; + self.import_ascii_utf8_encodings()?; + + { + let io = import::import_builtin(self, "_io")?; + + // Full stdio: FileIO → BufferedWriter → TextIOWrapper + #[cfg(all(feature = "host_env", feature = "stdio"))] + let make_stdio = |name: &str, fd: i32, write: bool| -> PyResult<PyObjectRef> { + let buffered_stdio = self.state.config.settings.buffered_stdio; + let unbuffered = write && !buffered_stdio; + let buf = crate::stdlib::_io::open( + self.ctx.new_int(fd).into(), + Some(if write { "wb" } else { "rb" }), + crate::stdlib::_io::OpenArgs { + buffering: if unbuffered { 0 } else { -1 }, + closefd: false, + ..Default::default() + }, + self, + )?; + let raw = if unbuffered { + buf.clone() + } else { + buf.get_attr("raw", self)? + }; + raw.set_attr("name", self.ctx.new_str(format!("<{name}>")), self)?; + let isatty = self.call_method(&raw, "isatty", ())?.is_true(self)?; + let write_through = !buffered_stdio; + let line_buffering = buffered_stdio && (isatty || fd == 2); + + let newline = if cfg!(windows) { None } else { Some("\n") }; + let encoding = self.state.config.settings.stdio_encoding.as_deref(); + // stderr always uses backslashreplace (ignores stdio_errors) + let errors = if fd == 2 { + Some("backslashreplace") + } else { + self.state.config.settings.stdio_errors.as_deref().or( + if self.state.config.settings.stdio_encoding.is_some() { + Some("strict") + } else { + Some("surrogateescape") + }, + ) + }; + + let stdio = self.call_method( + &io, + "TextIOWrapper", + ( + buf, + encoding, + errors, + newline, + line_buffering, + write_through, + ), + )?; + let mode = if write { "w" } else { "r" }; + stdio.set_attr("mode", self.ctx.new_str(mode), self)?; + Ok::<_, self::PyBaseExceptionRef>(stdio) + }; + + // Sandbox stdio: lightweight wrapper using Rust's std::io directly + #[cfg(all(not(feature = "host_env"), feature = "stdio"))] + let make_stdio = |name: &str, fd: i32, write: bool| { + let mode = if write { "w" } else { "r" }; + let stdio = stdlib::sys::SandboxStdio { + fd, + name: format!("<{name}>"), + mode: mode.to_owned(), + } + .into_ref(&self.ctx); + Ok(stdio.into()) + }; + + // No stdio: set to None (embedding use case) + #[cfg(not(feature = "stdio"))] + let make_stdio = |_name: &str, _fd: i32, _write: bool| { + Ok(crate::builtins::PyNone.into_pyobject(self)) + }; + + let set_stdio = |name, fd, write| { + let stdio: PyObjectRef = make_stdio(name, fd, write)?; + let dunder_name = self.ctx.intern_str(format!("__{name}__")); + self.sys_module.set_attr( + dunder_name, // e.g. __stdin__ + stdio.clone(), + self, + )?; + self.sys_module.set_attr(name, stdio, self)?; + Ok(()) + }; + set_stdio("stdin", 0, false)?; + set_stdio("stdout", 1, true)?; + set_stdio("stderr", 2, true)?; + + let io_open = io.get_attr("open", self)?; + self.builtins.set_attr("open", io_open, self)?; + } + + Ok(importlib) + }; + + let res = essential_init(); + let importlib = self.expect_pyresult(res, "essential initialization failed"); + + #[cfg(feature = "host_env")] + if self.state.config.settings.allow_external_library + && cfg!(feature = "rustpython-compiler") + && let Err(e) = import::init_importlib_package(self, importlib) + { + eprintln!( + "importlib initialization failed. This is critical for many complicated packages." + ); + self.print_exception(e); + } + + #[cfg(not(feature = "host_env"))] + let _ = importlib; + + let _expect_stdlib = cfg!(feature = "freeze-stdlib") + || !self.state.config.paths.module_search_paths.is_empty(); + + #[cfg(feature = "encodings")] + if _expect_stdlib { + if let Err(e) = self.import_encodings() { + eprintln!( + "encodings initialization failed. Only utf-8 encoding will be supported." + ); + self.print_exception(e); + } + } else { + // Here may not be the best place to give general `path_list` advice, + // but bare rustpython_vm::VirtualMachine users skipped proper settings must hit here while properly setup vm never enters here. + eprintln!( + "feature `encodings` is enabled but `paths.module_search_paths` is empty. \ + Please add the library path to `settings.path_list`. If you intended to disable the entire standard library (including the `encodings` feature), please also make sure to disable the `encodings` feature.\n\ + Tip: You may also want to add `\"\"` to `settings.path_list` in order to enable importing from the current working directory." + ); + } + + self.initialized = true; + } + + /// Set the custom signal channel for the interpreter + pub fn set_user_signal_channel(&mut self, signal_rx: signal::UserSignalReceiver) { + self.signal_rx = Some(signal_rx); + } + + /// Execute Python bytecode (`.pyc`) from an in-memory buffer. + /// + /// When the RustPython CLI is available, `.pyc` files are normally executed by + /// invoking `rustpython <input>.pyc`. This method provides an alternative for + /// environments where the binary is unavailable or file I/O is restricted + /// (e.g. WASM). + /// + /// ## Preparing a `.pyc` file + /// + /// First, compile a Python source file into bytecode: + /// + /// ```sh + /// # Generate a .pyc file + /// $ rustpython -m py_compile <input>.py + /// ``` + /// + /// ## Running the bytecode + /// + /// Load the resulting `.pyc` file into memory and execute it using the VM: + /// + /// ```no_run + /// use rustpython_vm::Interpreter; + /// Interpreter::without_stdlib(Default::default()).enter(|vm| { + /// let bytes = std::fs::read("__pycache__/<input>.rustpython-314.pyc").unwrap(); + /// let main_scope = vm.new_scope_with_main().unwrap(); + /// vm.run_pyc_bytes(&bytes, main_scope); + /// }); + /// ``` + pub fn run_pyc_bytes(&self, pyc_bytes: &[u8], scope: Scope) -> PyResult<()> { + let code = PyCode::from_pyc(pyc_bytes, Some("<pyc_bytes>"), None, None, self)?; + self.with_simple_run("<source>", |_module_dict| { + self.run_code_obj(code, scope)?; + Ok(()) + }) + } + + pub fn run_code_obj(&self, code: PyRef<PyCode>, scope: Scope) -> PyResult { + use crate::builtins::{PyFunction, PyModule}; + + // Create a function object for module code, similar to CPython's PyEval_EvalCode + let func = PyFunction::new(code.clone(), scope.globals.clone(), self)?; + let func_obj = func.into_ref(&self.ctx).into(); + + // Extract builtins from globals["__builtins__"], like PyEval_EvalCode + let builtins = match scope + .globals + .get_item_opt(identifier!(self, __builtins__), self)? + { + Some(b) => { + if let Some(module) = b.downcast_ref::<PyModule>() { + module.dict().into() + } else { + b + } + } + None => self.builtins.dict().into(), + }; + + let frame = + Frame::new(code, scope, builtins, &[], Some(func_obj), false, self).into_ref(&self.ctx); + self.run_frame(frame) + } + + #[cold] + pub fn run_unraisable(&self, e: PyBaseExceptionRef, msg: Option<String>, object: PyObjectRef) { + // During interpreter finalization, sys.unraisablehook may not be available, + // but we still need to report exceptions (especially from atexit callbacks). + // Write directly to stderr like PyErr_FormatUnraisable. + if self.state.finalizing.load(Ordering::Acquire) { + self.write_unraisable_to_stderr(&e, msg.as_deref(), &object); + return; + } + + let sys_module = self.import("sys", 0).unwrap(); + let unraisablehook = sys_module.get_attr("unraisablehook", self).unwrap(); + + let exc_type = e.class().to_owned(); + let exc_traceback = e.__traceback__().to_pyobject(self); // TODO: actual traceback + let exc_value = e.into(); + let args = stdlib::sys::UnraisableHookArgsData { + exc_type, + exc_value, + exc_traceback, + err_msg: self.new_pyobj(msg), + object, + }; + if let Err(e) = unraisablehook.call((args,), self) { + println!("{}", e.as_object().repr(self).unwrap()); + } + } + + /// Write unraisable exception to stderr during finalization. + /// Similar to _PyErr_WriteUnraisableDefaultHook in CPython. + fn write_unraisable_to_stderr( + &self, + e: &PyBaseExceptionRef, + msg: Option<&str>, + object: &PyObjectRef, + ) { + // Get stderr once and reuse it + let stderr = crate::stdlib::sys::get_stderr(self).ok(); + + let write_to_stderr = |s: &str, stderr: &Option<PyObjectRef>, vm: &VirtualMachine| { + if let Some(stderr) = stderr { + let _ = vm.call_method(stderr, "write", (s.to_owned(),)); + } else { + eprint!("{}", s); + } + }; + + let msg_str = if let Some(msg) = msg { + format!("{msg}: ") + } else { + "Exception ignored in: ".to_owned() + }; + write_to_stderr(&msg_str, &stderr, self); + + let repr_result = object.repr(self); + let repr_wtf8 = repr_result + .as_ref() + .map_or("<object repr failed>".as_ref(), |s| s.as_wtf8()); + write_to_stderr(&format!("{repr_wtf8}\n"), &stderr, self); + + // Write exception type and message + let exc_type_name = e.class().name(); + let msg = match e.as_object().str(self) { + Ok(exc_str) if !exc_str.as_wtf8().is_empty() => { + format!("{}: {}\n", exc_type_name, exc_str.as_wtf8()) + } + _ => format!("{}\n", exc_type_name), + }; + write_to_stderr(&msg, &stderr, self); + + // Flush stderr to ensure output is visible + if let Some(ref stderr) = stderr { + let _ = self.call_method(stderr, "flush", ()); + } + } + + #[inline(always)] + pub fn run_frame(&self, frame: FrameRef) -> PyResult { + match self.with_frame(frame, |f| f.run(self))? { + ExecutionResult::Return(value) => Ok(value), + _ => panic!("Got unexpected result from function"), + } + } + + /// Run `run` with main scope. + fn with_simple_run( + &self, + path: &str, + run: impl FnOnce(&Py<PyDict>) -> PyResult<()>, + ) -> PyResult<()> { + let sys_modules = self.sys_module.get_attr(identifier!(self, modules), self)?; + let main_module = sys_modules.get_item(identifier!(self, __main__), self)?; + let module_dict = main_module.dict().expect("main module must have __dict__"); + + // Track whether we set __file__ (for cleanup) + let set_file_name = !module_dict.contains_key(identifier!(self, __file__), self); + if set_file_name { + module_dict.set_item( + identifier!(self, __file__), + self.ctx.new_str(path).into(), + self, + )?; + module_dict.set_item(identifier!(self, __cached__), self.ctx.none(), self)?; + } + + let result = run(&module_dict); + + self.flush_io(); + + // Cleanup __file__ and __cached__ after execution + if set_file_name { + let _ = module_dict.del_item(identifier!(self, __file__), self); + let _ = module_dict.del_item(identifier!(self, __cached__), self); + } + + result + } + + /// flush_io + /// + /// Flush stdout and stderr. Errors are silently ignored. + fn flush_io(&self) { + if let Ok(stdout) = self.sys_module.get_attr("stdout", self) { + let _ = self.call_method(&stdout, identifier!(self, flush).as_str(), ()); + } + if let Ok(stderr) = self.sys_module.get_attr("stderr", self) { + let _ = self.call_method(&stderr, identifier!(self, flush).as_str(), ()); + } + } + + /// Clear module references during shutdown. + /// Follows the same phased algorithm as pylifecycle.c finalize_modules(): + /// no hardcoded module names, reverse import order, only builtins/sys last. + pub fn finalize_modules(&self) { + // Phase 1: Set special sys/builtins attributes to None, restore stdio + self.finalize_modules_delete_special(); + + // Phase 2: Remove all modules from sys.modules (set values to None), + // and collect weakrefs to modules preserving import order. + // No strong refs are kept — modules freed when their last ref drops. + let module_weakrefs = self.finalize_remove_modules(); + + // Phase 3: Clear sys.modules dict + self.finalize_clear_modules_dict(); + + // Phase 4: GC collect — modules removed from sys.modules are freed, + // exposing cycles (e.g., dict ↔ function.__globals__). GC collects + // these and calls __del__ while module dicts are still intact. + crate::gc_state::gc_state().collect_force(2); + + // Phase 5: Clear module dicts in reverse import order using 2-pass algorithm. + // Skip builtins and sys — those are cleared last. + self.finalize_clear_module_dicts(&module_weakrefs); + + // Phase 6: GC collect — pick up anything freed by dict clearing. + crate::gc_state::gc_state().collect_force(2); + + // Phase 7: Clear sys and builtins dicts last + self.finalize_clear_sys_builtins_dict(); + } + + /// Phase 1: Set special sys attributes to None and restore stdio. + fn finalize_modules_delete_special(&self) { + let none = self.ctx.none(); + let sys_dict = self.sys_module.dict(); + + // Set special sys attributes to None + for attr in &[ + "path", + "argv", + "ps1", + "ps2", + "last_exc", + "last_type", + "last_value", + "last_traceback", + "path_importer_cache", + "meta_path", + "path_hooks", + ] { + let _ = sys_dict.set_item(*attr, none.clone(), self); + } + + // Restore stdin/stdout/stderr from __stdin__/__stdout__/__stderr__ + for (std_name, dunder_name) in &[ + ("stdin", "__stdin__"), + ("stdout", "__stdout__"), + ("stderr", "__stderr__"), + ] { + let restored = sys_dict + .get_item_opt(*dunder_name, self) + .ok() + .flatten() + .unwrap_or_else(|| none.clone()); + let _ = sys_dict.set_item(*std_name, restored, self); + } + + // builtins._ = None + let _ = self.builtins.dict().set_item("_", none, self); + } + + /// Phase 2: Set all sys.modules values to None and collect weakrefs. + /// No strong refs are kept — modules are freed when removed from sys.modules + /// (if nothing else references them), allowing GC to collect their cycles. + fn finalize_remove_modules(&self) -> Vec<(String, PyRef<PyWeak>)> { + let mut module_weakrefs = Vec::new(); + + let Ok(modules) = self.sys_module.get_attr(identifier!(self, modules), self) else { + return module_weakrefs; + }; + let Some(modules_dict) = modules.downcast_ref::<PyDict>() else { + return module_weakrefs; + }; + + let none = self.ctx.none(); + let items: Vec<_> = modules_dict.into_iter().collect(); + + for (key, value) in items { + let name = key + .downcast_ref::<PyUtf8Str>() + .map(|s| s.as_str().to_owned()) + .unwrap_or_default(); + + // Save weakref to module (for later dict clearing) + if value.downcast_ref::<PyModule>().is_some() + && let Ok(weak) = value.downgrade(None, self) + { + module_weakrefs.push((name, weak)); + } + + // Set the value to None in sys.modules + let _ = modules_dict.set_item(&*key, none.clone(), self); + } + + module_weakrefs + } + + /// Phase 3: Clear sys.modules dict. + fn finalize_clear_modules_dict(&self) { + if let Ok(modules) = self.sys_module.get_attr(identifier!(self, modules), self) + && let Some(modules_dict) = modules.downcast_ref::<PyDict>() + { + modules_dict.clear(); + } + } + + /// Phase 5: Clear module dicts in reverse import order. + /// Skip builtins and sys — those are cleared last in Phase 7. + fn finalize_clear_module_dicts(&self, module_weakrefs: &[(String, PyRef<PyWeak>)]) { + let builtins_dict = self.builtins.dict(); + let sys_dict = self.sys_module.dict(); + + for (_name, weakref) in module_weakrefs.iter().rev() { + let Some(module_obj) = weakref.upgrade() else { + continue; + }; + let Some(module) = module_obj.downcast_ref::<PyModule>() else { + continue; + }; + + let dict = module.dict(); + // Skip builtins and sys — they are cleared last + if dict.is(&builtins_dict) || dict.is(&sys_dict) { + continue; + } + + Self::module_clear_dict(&dict, self); + } + } + + /// 2-pass module dict clearing (_PyModule_ClearDict algorithm). + /// Pass 1: Set names starting with '_' (except __builtins__) to None. + /// Pass 2: Set all remaining names (except __builtins__) to None. + pub(crate) fn module_clear_dict(dict: &Py<PyDict>, vm: &VirtualMachine) { + let none = vm.ctx.none(); + + // Pass 1: names starting with '_' (except __builtins__) + for (key, value) in dict.into_iter().collect::<Vec<_>>() { + if vm.is_none(&value) { + continue; + } + if let Some(key_str) = key.downcast_ref::<PyStr>() { + let name = key_str.as_wtf8(); + if name.starts_with("_") && name != "__builtins__" { + let _ = dict.set_item(key_str, none.clone(), vm); + } + } + } + + // Pass 2: all remaining (except __builtins__) + for (key, value) in dict.into_iter().collect::<Vec<_>>() { + if vm.is_none(&value) { + continue; + } + if let Some(key_str) = key.downcast_ref::<PyStr>() + && key_str.as_bytes() != b"__builtins__" + { + let _ = dict.set_item(key_str.as_wtf8(), none.clone(), vm); + } + } + } + + /// Phase 7: Clear sys and builtins dicts last. + fn finalize_clear_sys_builtins_dict(&self) { + Self::module_clear_dict(&self.sys_module.dict(), self); + Self::module_clear_dict(&self.builtins.dict(), self); + } + + pub fn current_recursion_depth(&self) -> usize { + self.recursion_depth.get() + } + + /// Stack margin bytes (like _PyOS_STACK_MARGIN_BYTES). + /// 2048 * sizeof(void*) = 16KB for 64-bit. + #[cfg_attr(miri, allow(dead_code))] + const STACK_MARGIN_BYTES: usize = 2048 * core::mem::size_of::<usize>(); + + /// Get the stack boundaries using platform-specific APIs. + /// Returns (base, top) where base is the lowest address and top is the highest. + #[cfg(all(not(miri), windows))] + fn get_stack_bounds() -> (usize, usize) { + use windows_sys::Win32::System::Threading::{ + GetCurrentThreadStackLimits, SetThreadStackGuarantee, + }; + let mut low: usize = 0; + let mut high: usize = 0; + unsafe { + GetCurrentThreadStackLimits(&mut low as *mut usize, &mut high as *mut usize); + // Add the guaranteed stack space (reserved for exception handling) + let mut guarantee: u32 = 0; + SetThreadStackGuarantee(&mut guarantee); + low += guarantee as usize; + } + (low, high) + } + + /// Get stack boundaries on non-Windows platforms. + /// Falls back to estimating based on current stack pointer. + #[cfg(all(not(miri), not(windows)))] + fn get_stack_bounds() -> (usize, usize) { + // Use pthread_attr_getstack on platforms that support it + #[cfg(any(target_os = "linux", target_os = "android"))] + { + use libc::{ + pthread_attr_destroy, pthread_attr_getstack, pthread_attr_t, pthread_getattr_np, + pthread_self, + }; + let mut attr: pthread_attr_t = unsafe { core::mem::zeroed() }; + unsafe { + if pthread_getattr_np(pthread_self(), &mut attr) == 0 { + let mut stack_addr: *mut libc::c_void = core::ptr::null_mut(); + let mut stack_size: libc::size_t = 0; + if pthread_attr_getstack(&attr, &mut stack_addr, &mut stack_size) == 0 { + pthread_attr_destroy(&mut attr); + let base = stack_addr as usize; + let top = base + stack_size; + return (base, top); + } + pthread_attr_destroy(&mut attr); + } + } + } + + #[cfg(target_os = "macos")] + { + use libc::{pthread_get_stackaddr_np, pthread_get_stacksize_np, pthread_self}; + unsafe { + let thread = pthread_self(); + let stack_top = pthread_get_stackaddr_np(thread) as usize; + let stack_size = pthread_get_stacksize_np(thread); + let stack_base = stack_top - stack_size; + return (stack_base, stack_top); + } + } + + // Fallback: estimate based on current SP and a default stack size + #[allow(unreachable_code)] + { + let current_sp = psm::stack_pointer() as usize; + // Assume 8MB stack, estimate base + let estimated_size = 8 * 1024 * 1024; + let base = current_sp.saturating_sub(estimated_size); + let top = current_sp + 1024 * 1024; // Assume we're not at the very top + (base, top) + } + } + + /// Calculate the C stack soft limit based on actual stack boundaries. + /// soft_limit = base + 2 * margin (for downward-growing stacks) + #[cfg(not(miri))] + fn calculate_c_stack_soft_limit() -> usize { + let (base, _top) = Self::get_stack_bounds(); + // Soft limit is 2 margins above the base + base + Self::STACK_MARGIN_BYTES * 2 + } + + /// Miri doesn't support inline assembly, so disable C stack checking. + #[cfg(miri)] + fn calculate_c_stack_soft_limit() -> usize { + 0 + } + + /// Check if we're near the C stack limit (like _Py_MakeRecCheck). + /// Returns true only when stack pointer is in the "danger zone" between + /// soft_limit and hard_limit (soft_limit - 2*margin). + #[cfg(not(miri))] + #[inline(always)] + fn check_c_stack_overflow(&self) -> bool { + let current_sp = psm::stack_pointer() as usize; + let soft_limit = self.c_stack_soft_limit.get(); + // Stack grows downward: check if we're below soft limit but above hard limit + // This matches CPython's _Py_MakeRecCheck behavior + current_sp < soft_limit + && current_sp >= soft_limit.saturating_sub(Self::STACK_MARGIN_BYTES * 2) + } + + /// Miri doesn't support inline assembly, so always return false. + #[cfg(miri)] + #[inline(always)] + fn check_c_stack_overflow(&self) -> bool { + false + } + + /// Used to run the body of a (possibly) recursive function. It will raise a + /// RecursionError if recursive functions are nested far too many times, + /// preventing a stack overflow. + pub fn with_recursion<R, F: FnOnce() -> PyResult<R>>(&self, _where: &str, f: F) -> PyResult<R> { + self.check_recursive_call(_where)?; + + // Native stack guard: check C stack like _Py_MakeRecCheck + if self.check_c_stack_overflow() { + return Err(self.new_recursion_error(_where.to_string())); + } + + self.recursion_depth.update(|d| d + 1); + scopeguard::defer! { self.recursion_depth.update(|d| d - 1) } + f() + } + + pub fn with_frame<R, F: FnOnce(FrameRef) -> PyResult<R>>( + &self, + frame: FrameRef, + f: F, + ) -> PyResult<R> { + self.with_frame_impl(frame, None, true, f) + } + + /// Like `with_frame` but allows specifying the initial exception state. + pub fn with_frame_exc<R, F: FnOnce(FrameRef) -> PyResult<R>>( + &self, + frame: FrameRef, + exc: Option<PyBaseExceptionRef>, + f: F, + ) -> PyResult<R> { + self.with_frame_impl(frame, exc, true, f) + } + + pub(crate) fn with_frame_untraced<R, F: FnOnce(FrameRef) -> PyResult<R>>( + &self, + frame: FrameRef, + f: F, + ) -> PyResult<R> { + self.with_frame_impl(frame, None, false, f) + } + + fn with_frame_impl<R, F: FnOnce(FrameRef) -> PyResult<R>>( + &self, + frame: FrameRef, + exc: Option<PyBaseExceptionRef>, + traced: bool, + f: F, + ) -> PyResult<R> { + self.with_recursion("", || { + // SAFETY: `frame` (FrameRef) stays alive for the entire closure scope, + // keeping the FramePtr valid. We pass a clone to `f` so that `f` + // consuming its FrameRef doesn't invalidate our pointer. + let fp = FramePtr(NonNull::from(&*frame)); + self.frames.borrow_mut().push(fp); + // Update the shared frame stack for sys._current_frames() and faulthandler + #[cfg(feature = "threading")] + crate::vm::thread::push_thread_frame(fp); + // Link frame into the signal-safe frame chain (previous pointer) + let old_frame = crate::vm::thread::set_current_frame((&**frame) as *const Frame); + frame.previous.store( + old_frame as *mut Frame, + core::sync::atomic::Ordering::Relaxed, + ); + // Push exception context for frame isolation. + // For normal calls: None (clean slate). + // For generators: the saved exception from last yield. + self.push_exception(exc); + let old_owner = frame.owner.swap( + crate::frame::FrameOwner::Thread as i8, + core::sync::atomic::Ordering::AcqRel, + ); + + // Ensure cleanup on panic: restore owner, pop exception, frame chain, and frames Vec. + scopeguard::defer! { + frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); + self.pop_exception(); + crate::vm::thread::set_current_frame(old_frame); + self.frames.borrow_mut().pop(); + #[cfg(feature = "threading")] + crate::vm::thread::pop_thread_frame(); + } + + if traced { + self.dispatch_traced_frame(&frame, |frame| f(frame.to_owned())) + } else { + f(frame.to_owned()) + } + }) + } + + /// Lightweight frame execution for generator/coroutine resume. + /// Pushes to the thread frame stack and fires trace/profile events, + /// but skips the thread exception update for performance. + pub fn resume_gen_frame<R, F: FnOnce(&Py<Frame>) -> PyResult<R>>( + &self, + frame: &FrameRef, + exc: Option<PyBaseExceptionRef>, + f: F, + ) -> PyResult<R> { + self.check_recursive_call("")?; + if self.check_c_stack_overflow() { + return Err(self.new_recursion_error(String::new())); + } + self.recursion_depth.update(|d| d + 1); + + // SAFETY: frame (&FrameRef) stays alive for the duration, so NonNull is valid until pop. + let fp = FramePtr(NonNull::from(&**frame)); + self.frames.borrow_mut().push(fp); + #[cfg(feature = "threading")] + crate::vm::thread::push_thread_frame(fp); + let old_frame = crate::vm::thread::set_current_frame((&***frame) as *const Frame); + frame.previous.store( + old_frame as *mut Frame, + core::sync::atomic::Ordering::Relaxed, + ); + // Inline exception push without thread exception update + self.exceptions.borrow_mut().stack.push(exc); + let old_owner = frame.owner.swap( + crate::frame::FrameOwner::Thread as i8, + core::sync::atomic::Ordering::AcqRel, + ); + + // Ensure cleanup on panic: restore owner, pop exception, frame chain, frames Vec, + // and recursion depth. + scopeguard::defer! { + frame.owner.store(old_owner, core::sync::atomic::Ordering::Release); + self.exceptions.borrow_mut().stack + .pop() + .expect("pop_exception() without nested exc stack"); + crate::vm::thread::set_current_frame(old_frame); + self.frames.borrow_mut().pop(); + #[cfg(feature = "threading")] + crate::vm::thread::pop_thread_frame(); + + self.recursion_depth.update(|d| d - 1); + } + + self.dispatch_traced_frame(frame, |frame| f(frame)) + } + + /// Fire trace/profile 'call' and 'return' events around a frame body. + /// + /// Matches `call_trace_protected` / `trace_trampoline` protocol: + /// - Fire `TraceEvent::Call`; if the trace function returns non-None, + /// install it as the per-frame `f_trace`. + /// - Execute the closure (the actual frame body). + /// - Fire `TraceEvent::Return` on both normal return **and** exception + /// unwind (`PY_UNWIND` → `PyTrace_RETURN` with `arg = None`). + /// Propagate any trace-function error, replacing the original exception. + fn dispatch_traced_frame<R, F: FnOnce(&Py<Frame>) -> PyResult<R>>( + &self, + frame: &Py<Frame>, + f: F, + ) -> PyResult<R> { + use crate::protocol::TraceEvent; + + // Fire 'call' trace event. current_frame() now returns the callee. + let trace_result = self.trace_event(TraceEvent::Call, None)?; + if let Some(local_trace) = trace_result { + *frame.trace.lock() = local_trace; + } + + let result = f(frame); + + // Fire 'return' event if frame is being traced or profiled. + // PY_UNWIND fires PyTrace_RETURN with arg=None — so we fire for + // both Ok and Err, matching `call_trace_protected` behavior. + if self.use_tracing.get() + && (!self.is_none(&frame.trace.lock()) || !self.is_none(&self.profile_func.borrow())) + { + let ret_result = self.trace_event(TraceEvent::Return, None); + // call_trace_protected: if trace function raises, its error + // replaces the original exception. + ret_result?; + } + + result + } + + /// Returns a basic CompileOpts instance with options accurate to the vm. Used + /// as the CompileOpts for `vm.compile()`. + #[cfg(feature = "rustpython-codegen")] + pub fn compile_opts(&self) -> crate::compiler::CompileOpts { + crate::compiler::CompileOpts { + optimize: self.state.config.settings.optimize, + debug_ranges: self.state.config.settings.code_debug_ranges, + } + } + + // To be called right before raising the recursion depth. + fn check_recursive_call(&self, _where: &str) -> PyResult<()> { + if self.recursion_depth.get() >= self.recursion_limit.get() { + Err(self.new_recursion_error(format!("maximum recursion depth exceeded {_where}"))) + } else { + Ok(()) + } + } + + pub fn current_frame(&self) -> Option<FrameRef> { + self.frames.borrow().last().map(|fp| { + // SAFETY: the caller keeps the FrameRef alive while it's in the Vec + unsafe { fp.as_ref() }.to_owned() + }) + } + + pub fn current_locals(&self) -> PyResult<ArgMapping> { + self.current_frame() + .expect("called current_locals but no frames on the stack") + .locals(self) + } + + pub fn current_globals(&self) -> PyDictRef { + self.current_frame() + .expect("called current_globals but no frames on the stack") + .globals + .clone() + } + + pub fn try_class(&self, module: &'static str, class: &'static str) -> PyResult<PyTypeRef> { + let class = self + .import(module, 0)? + .get_attr(class, self)? + .downcast() + .expect("not a class"); + Ok(class) + } + + pub fn class(&self, module: &'static str, class: &'static str) -> PyTypeRef { + let module = self + .import(module, 0) + .unwrap_or_else(|_| panic!("unable to import {module}")); + + let class = module + .get_attr(class, self) + .unwrap_or_else(|_| panic!("module {module:?} has no class {class}")); + class.downcast().expect("not a class") + } + + /// Call Python __import__ function without from_list. + /// Roughly equivalent to `import module_name` or `import top.submodule`. + /// + /// See also [`VirtualMachine::import_from`] for more advanced import. + /// See also [`rustpython_vm::import::import_source`] and other primitive import functions. + #[inline] + pub fn import<'a>(&self, module_name: impl AsPyStr<'a>, level: usize) -> PyResult { + let module_name = module_name.as_pystr(&self.ctx); + let from_list = self.ctx.empty_tuple_typed(); + self.import_inner(module_name, from_list, level) + } + + /// Call Python __import__ function caller with from_list. + /// Roughly equivalent to `from module_name import item1, item2` or `from top.submodule import item1, item2` + #[inline] + pub fn import_from<'a>( + &self, + module_name: impl AsPyStr<'a>, + from_list: &Py<PyTuple<PyStrRef>>, + level: usize, + ) -> PyResult { + let module_name = module_name.as_pystr(&self.ctx); + self.import_inner(module_name, from_list, level) + } + + fn import_inner( + &self, + module: &Py<PyStr>, + from_list: &Py<PyTuple<PyStrRef>>, + level: usize, + ) -> PyResult { + let import_func = self + .builtins + .get_attr(identifier!(self, __import__), self) + .map_err(|_| self.new_import_error("__import__ not found", module.to_owned()))?; + + let (locals, globals) = if let Some(frame) = self.current_frame() { + ( + Some(frame.locals.clone_mapping(self)), + Some(frame.globals.clone()), + ) + } else { + (None, None) + }; + let from_list: PyObjectRef = from_list.to_owned().into(); + import_func + .call((module.to_owned(), globals, locals, from_list, level), self) + .inspect_err(|exc| import::remove_importlib_frames(self, exc)) + } + + pub fn extract_elements_with<T, F>(&self, value: &PyObject, func: F) -> PyResult<Vec<T>> + where + F: Fn(PyObjectRef) -> PyResult<T>, + { + // Extract elements from item, if possible: + let cls = value.class(); + let list_borrow; + let slice = if cls.is(self.ctx.types.tuple_type) { + value.downcast_ref::<PyTuple>().unwrap().as_slice() + } else if cls.is(self.ctx.types.list_type) { + list_borrow = value.downcast_ref::<PyList>().unwrap().borrow_vec(); + &list_borrow + } else if cls.is(self.ctx.types.dict_values_type) { + // Atomic snapshot of dict values - prevents race condition during iteration + let values = value + .downcast_ref::<PyDictValues>() + .unwrap() + .dict + .values_vec(); + return values.into_iter().map(func).collect(); + } else if cls.is(self.ctx.types.dict_items_type) { + // Atomic snapshot of dict items - prevents race condition during iteration + let items = value + .downcast_ref::<PyDictItems>() + .unwrap() + .dict + .items_vec(); + return items + .into_iter() + .map(|(k, v)| func(self.ctx.new_tuple(vec![k, v]).into())) + .collect(); + } else { + return self.map_py_iter(value, func); + }; + slice.iter().map(|obj| func(obj.clone())).collect() + } + + pub fn map_iterable_object<F, R>(&self, obj: &PyObject, mut f: F) -> PyResult<PyResult<Vec<R>>> + where + F: FnMut(PyObjectRef) -> PyResult<R>, + { + match_class!(match obj { + ref l @ PyList => { + let mut i: usize = 0; + let mut results = Vec::with_capacity(l.borrow_vec().len()); + loop { + let elem = { + let elements = &*l.borrow_vec(); + if i >= elements.len() { + results.shrink_to_fit(); + return Ok(Ok(results)); + } else { + elements[i].clone() + } + // free the lock + }; + match f(elem) { + Ok(result) => results.push(result), + Err(err) => return Ok(Err(err)), + } + i += 1; + } + } + ref t @ PyTuple => Ok(t.iter().cloned().map(f).collect()), + // TODO: put internal iterable type + obj => { + Ok(self.map_py_iter(obj, f)) + } + }) + } + + fn map_py_iter<F, R>(&self, value: &PyObject, mut f: F) -> PyResult<Vec<R>> + where + F: FnMut(PyObjectRef) -> PyResult<R>, + { + let iter = value.to_owned().get_iter(self)?; + let cap = match self.length_hint_opt(value.to_owned()) { + Err(e) if e.class().is(self.ctx.exceptions.runtime_error) => return Err(e), + Ok(Some(value)) => Some(value), + // Use a power of 2 as a default capacity. + _ => None, + }; + // TODO: fix extend to do this check (?), see test_extend in Lib/test/list_tests.py, + // https://github.com/python/cpython/blob/v3.9.0/Objects/listobject.c#L922-L928 + if let Some(cap) = cap + && cap >= isize::MAX as usize + { + return Ok(Vec::new()); + } + + let mut results = PyIterIter::new(self, iter.as_ref(), cap) + .map(|element| f(element?)) + .collect::<PyResult<Vec<_>>>()?; + results.shrink_to_fit(); + Ok(results) + } + + pub fn get_attribute_opt<'a>( + &self, + obj: PyObjectRef, + attr_name: impl AsPyStr<'a>, + ) -> PyResult<Option<PyObjectRef>> { + let attr_name = attr_name.as_pystr(&self.ctx); + match obj.get_attr_inner(attr_name, self) { + Ok(attr) => Ok(Some(attr)), + Err(e) if e.fast_isinstance(self.ctx.exceptions.attribute_error) => Ok(None), + Err(e) => Err(e), + } + } + + pub fn set_attribute_error_context( + &self, + exc: &Py<PyBaseException>, + obj: PyObjectRef, + name: PyStrRef, + ) { + if exc.class().is(self.ctx.exceptions.attribute_error) { + let exc = exc.as_object(); + // Check if this exception was already augmented + let already_set = exc + .get_attr("name", self) + .ok() + .is_some_and(|v| !self.is_none(&v)); + if already_set { + return; + } + exc.set_attr("name", name, self).unwrap(); + exc.set_attr("obj", obj, self).unwrap(); + } + } + + // get_method should be used for internal access to magic methods (by-passing + // the full getattribute look-up. + pub fn get_method_or_type_error<F>( + &self, + obj: PyObjectRef, + method_name: &'static PyStrInterned, + err_msg: F, + ) -> PyResult + where + F: FnOnce() -> String, + { + let method = obj + .class() + .get_attr(method_name) + .ok_or_else(|| self.new_type_error(err_msg()))?; + self.call_if_get_descriptor(&method, obj) + } + + // TODO: remove + transfer over to get_special_method + pub(crate) fn get_method( + &self, + obj: PyObjectRef, + method_name: &'static PyStrInterned, + ) -> Option<PyResult> { + let method = obj.get_class_attr(method_name)?; + Some(self.call_if_get_descriptor(&method, obj)) + } + + pub(crate) fn get_str_method(&self, obj: PyObjectRef, method_name: &str) -> Option<PyResult> { + let method_name = self.ctx.interned_str(method_name)?; + self.get_method(obj, method_name) + } + + #[inline] + pub(crate) fn eval_breaker_tripped(&self) -> bool { + #[cfg(feature = "threading")] + if self.state.finalizing.load(Ordering::Relaxed) && !self.is_main_thread() { + return true; + } + + #[cfg(all(unix, feature = "threading"))] + if thread::stop_requested_for_current_thread() { + return true; + } + + #[cfg(not(target_arch = "wasm32"))] + if crate::signal::is_triggered() { + return true; + } + + false + } + + #[inline] + /// Checks for triggered signals and calls the appropriate handlers. A no-op on + /// platforms where signals are not supported. + pub fn check_signals(&self) -> PyResult<()> { + #[cfg(feature = "threading")] + if self.state.finalizing.load(Ordering::Acquire) && !self.is_main_thread() { + // once finalization starts, + // non-main Python threads should stop running bytecode. + return Err(self.new_exception(self.ctx.exceptions.system_exit.to_owned(), vec![])); + } + + // Suspend this thread if stop-the-world is in progress + #[cfg(all(unix, feature = "threading"))] + thread::suspend_if_needed(&self.state.stop_the_world); + + #[cfg(not(target_arch = "wasm32"))] + { + crate::signal::check_signals(self) + } + #[cfg(target_arch = "wasm32")] + { + Ok(()) + } + } + + pub(crate) fn push_exception(&self, exc: Option<PyBaseExceptionRef>) { + self.exceptions.borrow_mut().stack.push(exc); + #[cfg(feature = "threading")] + thread::update_thread_exception(self.topmost_exception()); + } + + pub(crate) fn pop_exception(&self) -> Option<PyBaseExceptionRef> { + let exc = self + .exceptions + .borrow_mut() + .stack + .pop() + .expect("pop_exception() without nested exc stack"); + #[cfg(feature = "threading")] + thread::update_thread_exception(self.topmost_exception()); + exc + } + + pub(crate) fn current_exception(&self) -> Option<PyBaseExceptionRef> { + self.exceptions.borrow().stack.last().cloned().flatten() + } + + pub(crate) fn set_exception(&self, exc: Option<PyBaseExceptionRef>) { + // don't be holding the RefCell guard while __del__ is called + let mut excs = self.exceptions.borrow_mut(); + debug_assert!( + !excs.stack.is_empty(), + "set_exception called with empty exception stack" + ); + if let Some(top) = excs.stack.last_mut() { + let prev = core::mem::replace(top, exc); + drop(excs); + drop(prev); + } else { + excs.stack.push(exc); + drop(excs); + } + #[cfg(feature = "threading")] + thread::update_thread_exception(self.topmost_exception()); + } + + pub(crate) fn contextualize_exception(&self, exception: &Py<PyBaseException>) { + if let Some(context_exc) = self.topmost_exception() + && !context_exc.is(exception) + { + // Traverse the context chain to find `exception` and break cycles + // Uses Floyd's cycle detection: o moves every step, slow_o every other step + let mut o = context_exc.clone(); + let mut slow_o = context_exc.clone(); + let mut slow_update_toggle = false; + while let Some(context) = o.__context__() { + if context.is(exception) { + o.set___context__(None); + break; + } + o = context; + if o.is(&slow_o) { + // Pre-existing cycle detected - all exceptions on the path were visited + break; + } + if slow_update_toggle && let Some(slow_context) = slow_o.__context__() { + slow_o = slow_context; + } + slow_update_toggle = !slow_update_toggle; + } + exception.set___context__(Some(context_exc)) + } + } + + pub(crate) fn topmost_exception(&self) -> Option<PyBaseExceptionRef> { + let excs = self.exceptions.borrow(); + excs.stack.iter().rev().find_map(|e| e.clone()) + } + + pub fn handle_exit_exception(&self, exc: PyBaseExceptionRef) -> u32 { + if exc.fast_isinstance(self.ctx.exceptions.system_exit) { + let args = exc.args(); + let msg = match args.as_slice() { + [] => return 0, + [arg] => match_class!(match arg { + ref i @ PyInt => { + use num_traits::cast::ToPrimitive; + // Try u32 first, then i32 (for negative values), else -1 for overflow + let code = i + .as_bigint() + .to_u32() + .or_else(|| i.as_bigint().to_i32().map(|v| v as u32)) + .unwrap_or(-1i32 as u32); + return code; + } + arg => { + if self.is_none(arg) { + return 0; + } else { + arg.str(self).ok() + } + } + }), + _ => args.as_object().repr(self).ok(), + }; + if let Some(msg) = msg { + // Write using Python's write() to use stderr's error handler (backslashreplace) + if let Ok(stderr) = stdlib::sys::get_stderr(self) { + let _ = self.call_method(&stderr, "write", (msg,)); + let _ = self.call_method(&stderr, "write", ("\n",)); + } + } + 1 + } else if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { + #[allow(clippy::if_same_then_else)] + { + self.print_exception(exc); + #[cfg(unix)] + { + let action = SigAction::new( + nix::sys::signal::SigHandler::SigDfl, + SaFlags::SA_ONSTACK, + SigSet::empty(), + ); + let result = unsafe { sigaction(SIGINT, &action) }; + if result.is_ok() { + self.flush_std(); + kill(getpid(), SIGINT).expect("Expect to be killed."); + } + + (libc::SIGINT as u32) + 128 + } + #[cfg(windows)] + { + // STATUS_CONTROL_C_EXIT - same as CPython + 0xC000013A + } + #[cfg(not(any(unix, windows)))] + { + 1 + } + } + } else { + self.print_exception(exc); + 1 + } + } + + #[doc(hidden)] + pub fn __module_set_attr( + &self, + module: &Py<PyModule>, + attr_name: &'static PyStrInterned, + attr_value: impl Into<PyObjectRef>, + ) -> PyResult<()> { + let val = attr_value.into(); + module + .as_object() + .generic_setattr(attr_name, PySetterValue::Assign(val), self) + } + + pub fn insert_sys_path(&self, obj: PyObjectRef) -> PyResult<()> { + let sys_path = self.sys_module.get_attr("path", self).unwrap(); + self.call_method(&sys_path, "insert", (0, obj))?; + Ok(()) + } + + pub fn run_module(&self, module: &str) -> PyResult<()> { + let runpy = self.import("runpy", 0)?; + let run_module_as_main = runpy.get_attr("_run_module_as_main", self)?; + run_module_as_main.call((module,), self)?; + Ok(()) + } + + pub fn fs_encoding(&self) -> &'static PyStrInterned { + identifier!(self, utf_8) + } + + pub fn fs_encode_errors(&self) -> &'static PyUtf8StrInterned { + if cfg!(windows) { + identifier_utf8!(self, surrogatepass) + } else { + identifier_utf8!(self, surrogateescape) + } + } + + pub fn fsdecode(&self, s: impl Into<OsString>) -> PyStrRef { + match s.into().into_string() { + Ok(s) => self.ctx.new_str(s), + Err(s) => { + let bytes = self.ctx.new_bytes(s.into_encoded_bytes()); + let errors = self.fs_encode_errors().to_owned(); + let res = self.state.codec_registry.decode_text( + bytes.into(), + "utf-8", + Some(errors), + self, + ); + self.expect_pyresult(res, "fsdecode should be lossless and never fail") + } + } + } + + pub fn fsencode<'a>(&self, s: &'a Py<PyStr>) -> PyResult<Cow<'a, OsStr>> { + if cfg!(windows) || s.is_utf8() { + // XXX: this is sketchy on windows; it's not guaranteed that the + // OsStr encoding will always be compatible with WTF-8. + let s = unsafe { OsStr::from_encoded_bytes_unchecked(s.as_bytes()) }; + return Ok(Cow::Borrowed(s)); + } + let errors = self.fs_encode_errors().to_owned(); + let bytes = self + .state + .codec_registry + .encode_text(s.to_owned(), "utf-8", Some(errors), self)? + .to_vec(); + // XXX: this is sketchy on windows; it's not guaranteed that the + // OsStr encoding will always be compatible with WTF-8. + let s = unsafe { OsString::from_encoded_bytes_unchecked(bytes) }; + Ok(Cow::Owned(s)) + } +} + +impl AsRef<Context> for VirtualMachine { + fn as_ref(&self) -> &Context { + &self.ctx + } +} + +/// Resolve frozen module alias to its original name. +/// Returns the original module name if an alias exists, otherwise returns the input name. +pub fn resolve_frozen_alias(name: &str) -> &str { + match name { + "_frozen_importlib" => "importlib._bootstrap", + "_frozen_importlib_external" => "importlib._bootstrap_external", + "encodings_ascii" => "encodings.ascii", + "encodings_utf_8" => "encodings.utf_8", + "__hello_alias__" | "__phello_alias__" | "__phello_alias__.spam" => "__hello__", + "__phello__.__init__" => "<__phello__", + "__phello__.ham.__init__" => "<__phello__.ham", + "__hello_only__" => "", + _ => name, + } +} + +#[test] +fn test_nested_frozen() { + use rustpython_vm as vm; + + vm::Interpreter::builder(Default::default()) + .add_frozen_modules(rustpython_vm::py_freeze!( + dir = "../../../../extra_tests/snippets" + )) + .build() + .enter(|vm| { + let scope = vm.new_scope_with_builtins(); + + let source = "from dir_module.dir_module_inner import value2"; + let code_obj = vm + .compile(source, vm::compiler::Mode::Exec, "<embedded>".to_owned()) + .map_err(|err| vm.new_syntax_error(&err, Some(source))) + .unwrap(); + + if let Err(e) = vm.run_code_obj(code_obj, scope) { + vm.print_exception(e); + panic!(); + } + }) +} + +#[test] +fn frozen_origname_matches() { + use rustpython_vm as vm; + + vm::Interpreter::builder(Default::default()) + .build() + .enter(|vm| { + let check = |name, expected| { + let module = import::import_frozen(vm, name).unwrap(); + let origname: PyStrRef = module + .get_attr("__origname__", vm) + .unwrap() + .try_into_value(vm) + .unwrap(); + assert_eq!(origname.as_wtf8(), expected); + }; + + check("_frozen_importlib", "importlib._bootstrap"); + check( + "_frozen_importlib_external", + "importlib._bootstrap_external", + ); + }); +} diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs new file mode 100644 index 00000000000..2f6f0bbee01 --- /dev/null +++ b/crates/vm/src/vm/python_run.rs @@ -0,0 +1,176 @@ +//! Python code execution functions. + +use crate::{ + AsObject, PyRef, PyResult, VirtualMachine, + builtins::PyCode, + compiler::{self}, + scope::Scope, +}; + +impl VirtualMachine { + /// PyRun_SimpleString + /// + /// Execute a string of Python code in a new scope with builtins. + pub fn run_simple_string(&self, source: &str) -> PyResult { + let scope = self.new_scope_with_builtins(); + self.run_string(scope, source, "<string>".to_owned()) + } + + /// PyRun_String + /// + /// Execute a string of Python code with explicit scope and source path. + pub fn run_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult { + let code_obj = self + .compile(source, compiler::Mode::Exec, source_path) + .map_err(|err| self.new_syntax_error(&err, Some(source)))?; + // linecache._register_code(code, source, filename) + let _ = self.register_code_in_linecache(&code_obj, source); + self.run_code_obj(code_obj, scope) + } + + /// Register a code object's source in linecache._interactive_cache + /// so that traceback can display source lines and caret indicators. + fn register_code_in_linecache(&self, code: &PyRef<PyCode>, source: &str) -> PyResult<()> { + let linecache = self.import("linecache", 0)?; + let register = linecache.get_attr("_register_code", self)?; + let source_str = self.ctx.new_str(source); + let filename = self.ctx.new_str(code.source_path().as_str()); + register.call((code.as_object().to_owned(), source_str, filename), self)?; + Ok(()) + } + + #[deprecated(note = "use run_string instead")] + pub fn run_code_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult { + self.run_string(scope, source, source_path) + } + + pub fn run_block_expr(&self, scope: Scope, source: &str) -> PyResult { + let code_obj = self + .compile(source, compiler::Mode::BlockExpr, "<embedded>".to_owned()) + .map_err(|err| self.new_syntax_error(&err, Some(source)))?; + self.run_code_obj(code_obj, scope) + } +} + +#[cfg(feature = "host_env")] +mod file_run { + use crate::{ + Py, PyResult, VirtualMachine, + builtins::{PyCode, PyDict}, + compiler::{self}, + scope::Scope, + }; + + impl VirtualMachine { + /// _PyRun_AnyFileObject (internal) + /// + /// Execute a Python file. Currently always delegates to run_simple_file + /// (interactive mode is handled separately in shell.rs). + /// + /// Note: This is an internal function. Use `run_file` for the public interface. + #[doc(hidden)] + pub fn run_any_file(&self, scope: Scope, path: &str) -> PyResult<()> { + let path = if path.is_empty() { "???" } else { path }; + self.run_simple_file(scope, path) + } + + /// _PyRun_SimpleFileObject + /// + /// Execute a Python file with __main__ module setup. + /// Sets __file__ and __cached__ before execution, removes them after. + fn run_simple_file(&self, scope: Scope, path: &str) -> PyResult<()> { + self.with_simple_run(path, |module_dict| { + self.run_simple_file_inner(module_dict, scope, path) + }) + } + + fn run_simple_file_inner( + &self, + module_dict: &Py<PyDict>, + scope: Scope, + path: &str, + ) -> PyResult<()> { + let pyc = maybe_pyc_file(path); + if pyc { + // pyc file execution + set_main_loader(module_dict, path, "SourcelessFileLoader", self)?; + let loader = module_dict.get_item("__loader__", self)?; + let get_code = loader.get_attr("get_code", self)?; + let code_obj = get_code.call((identifier!(self, __main__).to_owned(),), self)?; + let code = code_obj + .downcast::<PyCode>() + .map_err(|_| self.new_runtime_error("Bad code object in .pyc file"))?; + self.run_code_obj(code, scope)?; + } else { + if path != "<stdin>" { + set_main_loader(module_dict, path, "SourceFileLoader", self)?; + } + match std::fs::read_to_string(path) { + Ok(source) => { + let code_obj = self + .compile(&source, compiler::Mode::Exec, path.to_owned()) + .map_err(|err| self.new_syntax_error(&err, Some(&source)))?; + self.run_code_obj(code_obj, scope)?; + } + Err(err) => { + return Err(self.new_os_error(err.to_string())); + } + } + } + Ok(()) + } + + // #[deprecated(note = "use rustpython::run_file instead; if this changes causes problems, please report an issue.")] + pub fn run_script(&self, scope: Scope, path: &str) -> PyResult<()> { + self.run_any_file(scope, path) + } + } + + fn set_main_loader( + module_dict: &Py<PyDict>, + filename: &str, + loader_name: &str, + vm: &VirtualMachine, + ) -> PyResult<()> { + vm.import("importlib.machinery", 0)?; + let sys_modules = vm.sys_module.get_attr(identifier!(vm, modules), vm)?; + let machinery = sys_modules.get_item("importlib.machinery", vm)?; + let loader_name = vm.ctx.new_str(loader_name); + let loader_class = machinery.get_attr(&loader_name, vm)?; + let loader = loader_class.call((identifier!(vm, __main__).to_owned(), filename), vm)?; + module_dict.set_item("__loader__", loader, vm)?; + Ok(()) + } + + /// Check whether a file is maybe a pyc file. + /// + /// Detection is performed by: + /// 1. Checking if the filename ends with ".pyc" + /// 2. If not, reading the first 2 bytes and comparing with the magic number + fn maybe_pyc_file(path: &str) -> bool { + if path.ends_with(".pyc") { + return true; + } + maybe_pyc_file_with_magic(path).unwrap_or(false) + } + + fn maybe_pyc_file_with_magic(path: &str) -> std::io::Result<bool> { + let path_obj = std::path::Path::new(path); + if !path_obj.is_file() { + return Ok(false); + } + + let mut file = std::fs::File::open(path)?; + let mut buf = [0u8; 2]; + + use std::io::Read; + if file.read(&mut buf)? != 2 { + return Ok(false); + } + + // Read only two bytes of the magic. If the file was opened in + // text mode, the bytes 3 and 4 of the magic (\r\n) might not + // be read as they are on disk. + Ok(crate::import::check_pyc_magic_number_bytes(&buf)) + } +} diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs new file mode 100644 index 00000000000..0563e7131ac --- /dev/null +++ b/crates/vm/src/vm/setting.rs @@ -0,0 +1,222 @@ +#[cfg(feature = "flame-it")] +use std::ffi::OsString; + +/// Path configuration computed at runtime (like PyConfig path outputs) +#[derive(Debug, Clone, Default)] +pub struct Paths { + /// sys.executable + pub executable: String, + /// sys._base_executable (original interpreter in venv) + pub base_executable: String, + /// sys.prefix + pub prefix: String, + /// sys.base_prefix + pub base_prefix: String, + /// sys.exec_prefix + pub exec_prefix: String, + /// sys.base_exec_prefix + pub base_exec_prefix: String, + /// sys._stdlib_dir + pub stdlib_dir: Option<String>, + /// Computed module_search_paths (complete sys.path) + pub module_search_paths: Vec<String>, +} + +/// Combined configuration: user settings + computed paths +/// CPython directly exposes every fields under both of them. +/// We separate them to maintain better ownership discipline. +pub struct PyConfig { + pub settings: Settings, + pub paths: Paths, +} + +impl PyConfig { + pub fn new(settings: Settings, paths: Paths) -> Self { + Self { settings, paths } + } +} + +/// User-configurable settings for the python vm. +#[non_exhaustive] +pub struct Settings { + /// -I + pub isolated: bool, + + // int use_environment + /// -Xdev + pub dev_mode: bool, + + /// Not set SIGINT handler(i.e. for embedded mode) + pub install_signal_handlers: bool, + + /// PYTHONHASHSEED=x + /// None means use_hash_seed = 0 in CPython + pub hash_seed: Option<u32>, + + /// -X faulthandler, PYTHONFAULTHANDLER + pub faulthandler: bool, + + // int tracemalloc; + // int perf_profiling; + // int import_time; + /// -X no_debug_ranges: disable column info in bytecode + pub code_debug_ranges: bool, + // int show_ref_count; + // int dump_refs; + // wchar_t *dump_refs_file; + // int malloc_stats; + // wchar_t *filesystem_encoding; + // wchar_t *filesystem_errors; + // wchar_t *pycache_prefix; + // int parse_argv; + // PyWideStringList orig_argv; + /// sys.argv + pub argv: Vec<String>, + + // spell-checker:ignore Xfoo + /// -Xfoo[=bar] + pub xoptions: Vec<(String, Option<String>)>, + + // spell-checker:ignore Wfoo + /// -Wfoo + pub warnoptions: Vec<String>, + + /// -S + pub import_site: bool, + + /// -b + pub bytes_warning: u64, + + /// -X warn_default_encoding, PYTHONWARNDEFAULTENCODING + pub warn_default_encoding: bool, + + /// -X thread_inherit_context, whether new threads inherit context from parent + pub thread_inherit_context: bool, + + /// -X context_aware_warnings, whether warnings are context aware + pub context_aware_warnings: bool, + + /// -i + pub inspect: bool, + + /// -i, with no script + pub interactive: bool, + + // int optimization_level; + // int parser_debug; + /// -B + pub write_bytecode: bool, + + /// verbosity level (-v switch) + pub verbose: u8, + + /// -q + pub quiet: bool, + + /// -s + pub user_site_directory: bool, + + // int configure_c_stdio; + /// -u, PYTHONUNBUFFERED=x + pub buffered_stdio: bool, + + /// PYTHONIOENCODING - stdio encoding + pub stdio_encoding: Option<String>, + /// PYTHONIOENCODING - stdio error handler + pub stdio_errors: Option<String>, + pub utf8_mode: i8, + /// --check-hash-based-pycs + pub check_hash_pycs_mode: CheckHashPycsMode, + + // int use_frozen_modules; + /// -P + pub safe_path: bool, + + /// -X int_max_str_digits + pub int_max_str_digits: i64, + + // /* --- Path configuration inputs ------------ */ + // int pathconfig_warnings; + // wchar_t *program_name; + /// Environment PYTHONPATH (and RUSTPYTHONPATH) + pub path_list: Vec<String>, + + // wchar_t *home; + // wchar_t *platlibdir; + /// -d command line switch + pub debug: u8, + + /// -O optimization switch counter + pub optimize: u8, + + /// -E + pub ignore_environment: bool, + + /// false for wasm. Not a command-line option + pub allow_external_library: bool, + + #[cfg(feature = "flame-it")] + pub profile_output: Option<OsString>, + #[cfg(feature = "flame-it")] + pub profile_format: Option<String>, +} + +#[derive(Debug, Default, Copy, Clone, strum_macros::Display, strum_macros::EnumString)] +#[strum(serialize_all = "lowercase")] +pub enum CheckHashPycsMode { + #[default] + Default, + Always, + Never, +} + +impl Settings { + pub fn with_path(mut self, path: String) -> Self { + self.path_list.push(path); + self + } +} + +/// Sensible default settings. +impl Default for Settings { + fn default() -> Self { + Self { + debug: 0, + inspect: false, + interactive: false, + optimize: 0, + install_signal_handlers: true, + user_site_directory: true, + import_site: true, + ignore_environment: false, + verbose: 0, + quiet: false, + write_bytecode: true, + safe_path: false, + bytes_warning: 0, + xoptions: vec![], + isolated: false, + dev_mode: false, + warn_default_encoding: false, + thread_inherit_context: false, + context_aware_warnings: false, + warnoptions: vec![], + path_list: vec![], + argv: vec![], + hash_seed: None, + faulthandler: false, + code_debug_ranges: true, + buffered_stdio: true, + check_hash_pycs_mode: CheckHashPycsMode::Default, + allow_external_library: cfg!(feature = "importlib"), + stdio_encoding: None, + stdio_errors: None, + utf8_mode: -1, + int_max_str_digits: 4300, + #[cfg(feature = "flame-it")] + profile_output: None, + #[cfg(feature = "flame-it")] + profile_format: None, + } + } +} diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs new file mode 100644 index 00000000000..e7cc64f00b4 --- /dev/null +++ b/crates/vm/src/vm/thread.rs @@ -0,0 +1,641 @@ +#[cfg(feature = "threading")] +use super::FramePtr; +#[cfg(feature = "threading")] +use crate::builtins::PyBaseExceptionRef; +use crate::frame::Frame; +use crate::{AsObject, PyObject, VirtualMachine}; +#[cfg(feature = "threading")] +use alloc::sync::Arc; +use core::{ + cell::{Cell, RefCell}, + ptr::NonNull, + sync::atomic::{AtomicPtr, Ordering}, +}; +use itertools::Itertools; +use std::thread_local; + +// Thread states for stop-the-world support. +// DETACHED: not executing Python bytecode (in native code, or idle) +// ATTACHED: actively executing Python bytecode +// SUSPENDED: parked by a stop-the-world request +#[cfg(all(unix, feature = "threading"))] +pub const THREAD_DETACHED: i32 = 0; +#[cfg(all(unix, feature = "threading"))] +pub const THREAD_ATTACHED: i32 = 1; +#[cfg(all(unix, feature = "threading"))] +pub const THREAD_SUSPENDED: i32 = 2; + +/// Per-thread shared state for sys._current_frames() and sys._current_exceptions(). +/// The exception field uses atomic operations for lock-free cross-thread reads. +#[cfg(feature = "threading")] +pub struct ThreadSlot { + /// Raw frame pointers, valid while the owning thread's call stack is active. + /// Readers must hold the Mutex and convert to FrameRef inside the lock. + pub frames: parking_lot::Mutex<Vec<FramePtr>>, + pub exception: crate::PyAtomicRef<Option<crate::exceptions::types::PyBaseException>>, + /// Thread state for stop-the-world: DETACHED / ATTACHED / SUSPENDED + #[cfg(unix)] + pub state: core::sync::atomic::AtomicI32, + /// Per-thread stop request bit (eval breaker equivalent). + #[cfg(unix)] + pub stop_requested: core::sync::atomic::AtomicBool, + /// Handle for waking this thread from park in stop-the-world paths. + #[cfg(unix)] + pub thread: std::thread::Thread, +} + +#[cfg(feature = "threading")] +pub type CurrentFrameSlot = Arc<ThreadSlot>; + +thread_local! { + pub(super) static VM_STACK: RefCell<Vec<NonNull<VirtualMachine>>> = Vec::with_capacity(1).into(); + + pub(crate) static COROUTINE_ORIGIN_TRACKING_DEPTH: Cell<u32> = const { Cell::new(0) }; + + /// Current thread's slot for sys._current_frames() and sys._current_exceptions() + #[cfg(feature = "threading")] + static CURRENT_THREAD_SLOT: RefCell<Option<CurrentFrameSlot>> = const { RefCell::new(None) }; + + /// Current top frame for signal-safe traceback walking. + /// Mirrors `PyThreadState.current_frame`. Read by faulthandler's signal + /// handler to dump tracebacks without accessing RefCell or locks. + /// Uses AtomicPtr for async-signal-safety (signal handlers may read this + /// while the owning thread is writing). + pub(crate) static CURRENT_FRAME: AtomicPtr<Frame> = + const { AtomicPtr::new(core::ptr::null_mut()) }; + +} + +scoped_tls::scoped_thread_local!(static VM_CURRENT: VirtualMachine); + +pub fn with_current_vm<R>(f: impl FnOnce(&VirtualMachine) -> R) -> R { + if !VM_CURRENT.is_set() { + panic!("call with_current_vm() but VM_CURRENT is null"); + } + VM_CURRENT.with(f) +} + +pub fn enter_vm<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + VM_STACK.with(|vms| { + // Outermost enter_vm: transition DETACHED → ATTACHED + #[cfg(all(unix, feature = "threading"))] + let was_outermost = vms.borrow().is_empty(); + + vms.borrow_mut().push(vm.into()); + + // Initialize thread slot for this thread if not already done + #[cfg(feature = "threading")] + init_thread_slot_if_needed(vm); + + #[cfg(all(unix, feature = "threading"))] + if was_outermost { + attach_thread(vm); + } + + scopeguard::defer! { + // Outermost exit: transition ATTACHED → DETACHED + #[cfg(all(unix, feature = "threading"))] + if vms.borrow().len() == 1 { + detach_thread(); + } + vms.borrow_mut().pop(); + } + VM_CURRENT.set(vm, f) + }) +} + +/// Initialize thread slot for current thread if not already initialized. +/// Called automatically by enter_vm(). +#[cfg(feature = "threading")] +fn init_thread_slot_if_needed(vm: &VirtualMachine) { + CURRENT_THREAD_SLOT.with(|slot| { + if slot.borrow().is_none() { + let thread_id = crate::stdlib::_thread::get_ident(); + let mut registry = vm.state.thread_frames.lock(); + let new_slot = Arc::new(ThreadSlot { + frames: parking_lot::Mutex::new(Vec::new()), + exception: crate::PyAtomicRef::from(None::<PyBaseExceptionRef>), + #[cfg(unix)] + state: core::sync::atomic::AtomicI32::new( + if vm.state.stop_the_world.requested.load(Ordering::Acquire) { + // Match init_threadstate(): new thread-state starts + // suspended while stop-the-world is active. + THREAD_SUSPENDED + } else { + THREAD_DETACHED + }, + ), + #[cfg(unix)] + stop_requested: core::sync::atomic::AtomicBool::new(false), + #[cfg(unix)] + thread: std::thread::current(), + }); + registry.insert(thread_id, new_slot.clone()); + drop(registry); + *slot.borrow_mut() = Some(new_slot); + } + }); +} + +/// Transition DETACHED → ATTACHED. Blocks if the thread was SUSPENDED by +/// a stop-the-world request (like `_PyThreadState_Attach` + `tstate_wait_attach`). +#[cfg(all(unix, feature = "threading"))] +fn wait_while_suspended(slot: &ThreadSlot) -> u64 { + let mut wait_yields = 0u64; + while slot.state.load(Ordering::Acquire) == THREAD_SUSPENDED { + wait_yields = wait_yields.saturating_add(1); + std::thread::park(); + } + wait_yields +} + +#[cfg(all(unix, feature = "threading"))] +fn attach_thread(vm: &VirtualMachine) { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + super::stw_trace(format_args!("attach begin")); + loop { + match s.state.compare_exchange( + THREAD_DETACHED, + THREAD_ATTACHED, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + super::stw_trace(format_args!("attach DETACHED->ATTACHED")); + break; + } + Err(THREAD_SUSPENDED) => { + // Parked by stop-the-world — wait until released to DETACHED + super::stw_trace(format_args!("attach wait-suspended")); + let wait_yields = wait_while_suspended(s); + vm.state.stop_the_world.add_attach_wait_yields(wait_yields); + // Retry CAS + } + Err(state) => { + debug_assert!(false, "unexpected thread state in attach: {state}"); + break; + } + } + } + } + }); +} + +/// Transition ATTACHED → DETACHED (like `_PyThreadState_Detach`). +#[cfg(all(unix, feature = "threading"))] +fn detach_thread() { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + match s.state.compare_exchange( + THREAD_ATTACHED, + THREAD_DETACHED, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => {} + Err(THREAD_DETACHED) => { + debug_assert!(false, "detach called while already DETACHED"); + return; + } + Err(state) => { + debug_assert!(false, "unexpected thread state in detach: {state}"); + return; + } + } + super::stw_trace(format_args!("detach ATTACHED->DETACHED")); + } + }); +} + +/// Temporarily transition the current thread ATTACHED → DETACHED while +/// running `f`, then re-attach afterwards. This allows `stop_the_world` +/// to park this thread during blocking operations. +/// +/// `Py_BEGIN_ALLOW_THREADS` / `Py_END_ALLOW_THREADS` equivalent. +#[cfg(all(unix, feature = "threading"))] +pub fn allow_threads<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + // Preserve save/restore semantics: + // only detach if this call observed ATTACHED at entry, and always restore + // on unwind. + let should_transition = CURRENT_THREAD_SLOT.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|s| s.state.load(Ordering::Acquire) == THREAD_ATTACHED) + }); + if !should_transition { + return f(); + } + + detach_thread(); + let reattach_guard = scopeguard::guard(vm, attach_thread); + let result = f(); + drop(reattach_guard); + result +} + +/// No-op on non-unix or non-threading builds. +#[cfg(not(all(unix, feature = "threading")))] +pub fn allow_threads<R>(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + f() +} + +/// Called from check_signals when stop-the-world is requested. +/// Transitions ATTACHED → SUSPENDED and waits until released +/// (like `_PyThreadState_Suspend` + `_PyThreadState_Attach`). +#[cfg(all(unix, feature = "threading"))] +pub fn suspend_if_needed(stw: &super::StopTheWorldState) { + let should_suspend = CURRENT_THREAD_SLOT.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|s| s.stop_requested.load(Ordering::Relaxed)) + }); + if !should_suspend { + return; + } + + if !stw.requested.load(Ordering::Acquire) { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + s.stop_requested.store(false, Ordering::Release); + } + }); + return; + } + + do_suspend(stw); +} + +#[cfg(all(unix, feature = "threading"))] +#[cold] +fn do_suspend(stw: &super::StopTheWorldState) { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + // ATTACHED → SUSPENDED + match s.state.compare_exchange( + THREAD_ATTACHED, + THREAD_SUSPENDED, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + // Consumed this thread's stop request bit. + s.stop_requested.store(false, Ordering::Release); + } + Err(THREAD_DETACHED) => { + // Leaving VM; caller will re-check on next entry. + super::stw_trace(format_args!("suspend skip DETACHED")); + return; + } + Err(THREAD_SUSPENDED) => { + // Already parked by another path. + s.stop_requested.store(false, Ordering::Release); + super::stw_trace(format_args!("suspend skip already-suspended")); + return; + } + Err(state) => { + debug_assert!(false, "unexpected thread state in suspend: {state}"); + return; + } + } + super::stw_trace(format_args!("suspend ATTACHED->SUSPENDED")); + + // Re-check: if start_the_world already ran (cleared `requested`), + // no one will set us back to DETACHED — we must self-recover. + if !stw.requested.load(Ordering::Acquire) { + s.state.store(THREAD_ATTACHED, Ordering::Release); + s.stop_requested.store(false, Ordering::Release); + super::stw_trace(format_args!("suspend abort requested-cleared")); + return; + } + + // Notify the stop-the-world requester that we've parked + stw.notify_suspended(); + super::stw_trace(format_args!("suspend notified-requester")); + + // Wait until start_the_world sets us back to DETACHED + let wait_yields = wait_while_suspended(s); + stw.add_suspend_wait_yields(wait_yields); + + // Re-attach (DETACHED → ATTACHED), tstate_wait_attach CAS loop. + loop { + match s.state.compare_exchange( + THREAD_DETACHED, + THREAD_ATTACHED, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break, + Err(THREAD_SUSPENDED) => { + let extra_wait = wait_while_suspended(s); + stw.add_suspend_wait_yields(extra_wait); + } + Err(THREAD_ATTACHED) => break, + Err(state) => { + debug_assert!(false, "unexpected post-suspend state: {state}"); + break; + } + } + } + s.stop_requested.store(false, Ordering::Release); + super::stw_trace(format_args!("suspend resume -> ATTACHED")); + } + }); +} + +#[cfg(all(unix, feature = "threading"))] +#[inline] +pub fn stop_requested_for_current_thread() -> bool { + CURRENT_THREAD_SLOT.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|s| s.stop_requested.load(Ordering::Relaxed)) + }) +} + +/// Push a frame pointer onto the current thread's shared frame stack. +/// The pointed-to frame must remain alive until the matching pop. +#[cfg(feature = "threading")] +pub fn push_thread_frame(fp: FramePtr) { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + s.frames.lock().push(fp); + } else { + debug_assert!( + false, + "push_thread_frame called without initialized thread slot" + ); + } + }); +} + +/// Pop a frame from the current thread's shared frame stack. +/// Called when a frame is exited. +#[cfg(feature = "threading")] +pub fn pop_thread_frame() { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + s.frames.lock().pop(); + } else { + debug_assert!( + false, + "pop_thread_frame called without initialized thread slot" + ); + } + }); +} + +/// Set the current thread's top frame pointer for signal-safe traceback walking. +/// Returns the previous frame pointer so it can be restored on pop. +pub fn set_current_frame(frame: *const Frame) -> *const Frame { + CURRENT_FRAME.with(|c| c.swap(frame as *mut Frame, Ordering::Relaxed) as *const Frame) +} + +/// Get the current thread's top frame pointer. +/// Used by faulthandler's signal handler to start traceback walking. +pub fn get_current_frame() -> *const Frame { + CURRENT_FRAME.with(|c| c.load(Ordering::Relaxed) as *const Frame) +} + +/// Update the current thread's exception slot atomically (no locks). +/// Called from push_exception/pop_exception/set_exception. +#[cfg(feature = "threading")] +pub fn update_thread_exception(exc: Option<PyBaseExceptionRef>) { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + // SAFETY: Called only from the owning thread. The old ref is dropped + // here on the owning thread, which is safe. + let _old = unsafe { s.exception.swap(exc) }; + } + }); +} + +/// Collect all threads' current exceptions for sys._current_exceptions(). +/// Acquires the global registry lock briefly, then reads each slot's exception atomically. +#[cfg(feature = "threading")] +pub fn get_all_current_exceptions(vm: &VirtualMachine) -> Vec<(u64, Option<PyBaseExceptionRef>)> { + let registry = vm.state.thread_frames.lock(); + registry + .iter() + .map(|(id, slot)| (*id, slot.exception.to_owned())) + .collect() +} + +/// Cleanup thread slot for the current thread. Called at thread exit. +#[cfg(feature = "threading")] +pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { + let thread_id = crate::stdlib::_thread::get_ident(); + let current_slot = CURRENT_THREAD_SLOT.with(|slot| slot.borrow().as_ref().cloned()); + + // A dying thread should not remain logically ATTACHED while its + // thread-state slot is being removed. + #[cfg(all(unix, feature = "threading"))] + if let Some(slot) = &current_slot { + let _ = slot.state.compare_exchange( + THREAD_ATTACHED, + THREAD_DETACHED, + Ordering::AcqRel, + Ordering::Acquire, + ); + } + + // Guard against OS thread-id reuse races: only remove the registry entry + // if it still points at this thread's own slot. + let removed = if let Some(slot) = &current_slot { + let mut registry = vm.state.thread_frames.lock(); + match registry.get(&thread_id) { + Some(registered) if Arc::ptr_eq(registered, slot) => registry.remove(&thread_id), + _ => None, + } + } else { + None + }; + #[cfg(all(unix, feature = "threading"))] + if let Some(slot) = &removed + && vm.state.stop_the_world.requested.load(Ordering::Acquire) + && thread_id != vm.state.stop_the_world.requester_ident() + && slot.state.load(Ordering::Relaxed) != THREAD_SUSPENDED + { + // A non-requester thread disappeared while stop-the-world is pending. + // Unblock requester countdown progress. + vm.state.stop_the_world.notify_thread_gone(); + } + CURRENT_THREAD_SLOT.with(|s| { + *s.borrow_mut() = None; + }); +} + +/// Reinitialize thread slot after fork. Called in child process. +/// Creates a fresh slot and registers it for the current thread, +/// preserving the current thread's frames from `vm.frames`. +/// +/// Precondition: `reinit_locks_after_fork()` has already reset all +/// VmState locks to unlocked. +#[cfg(feature = "threading")] +pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { + let current_ident = crate::stdlib::_thread::get_ident(); + let current_frames: Vec<FramePtr> = vm.frames.borrow().clone(); + let new_slot = Arc::new(ThreadSlot { + frames: parking_lot::Mutex::new(current_frames), + exception: crate::PyAtomicRef::from(vm.topmost_exception()), + #[cfg(unix)] + state: core::sync::atomic::AtomicI32::new(THREAD_ATTACHED), + #[cfg(unix)] + stop_requested: core::sync::atomic::AtomicBool::new(false), + #[cfg(unix)] + thread: std::thread::current(), + }); + + // Lock is safe: reinit_locks_after_fork() already reset it to unlocked. + let mut registry = vm.state.thread_frames.lock(); + registry.clear(); + registry.insert(current_ident, new_slot.clone()); + drop(registry); + + CURRENT_THREAD_SLOT.with(|s| { + *s.borrow_mut() = Some(new_slot); + }); +} + +pub fn with_vm<F, R>(obj: &PyObject, f: F) -> Option<R> +where + F: Fn(&VirtualMachine) -> R, +{ + let vm_owns_obj = |interp: NonNull<VirtualMachine>| { + // SAFETY: all references in VM_STACK should be valid + let vm = unsafe { interp.as_ref() }; + obj.fast_isinstance(vm.ctx.types.object_type) + }; + VM_STACK.with(|vms| { + let interp = match vms.borrow().iter().copied().exactly_one() { + Ok(x) => { + debug_assert!(vm_owns_obj(x)); + x + } + Err(mut others) => others.find(|x| vm_owns_obj(*x))?, + }; + // SAFETY: all references in VM_STACK should be valid, and should not be changed or moved + // at least until this function returns and the stack unwinds to an enter_vm() call + let vm = unsafe { interp.as_ref() }; + Some(VM_CURRENT.set(vm, || f(vm))) + }) +} + +#[must_use = "ThreadedVirtualMachine does nothing unless you move it to another thread and call .run()"] +#[cfg(feature = "threading")] +pub struct ThreadedVirtualMachine { + pub(super) vm: VirtualMachine, +} + +#[cfg(feature = "threading")] +impl ThreadedVirtualMachine { + /// Create a `FnOnce()` that can easily be passed to a function like [`std::thread::Builder::spawn`] + /// + /// # Note + /// + /// If you return a `PyObjectRef` (or a type that contains one) from `F`, and don't `join()` + /// on the thread this `FnOnce` runs in, there is a possibility that that thread will panic + /// as `PyObjectRef`'s `Drop` implementation tries to run the `__del__` destructor of a + /// Python object but finds that it's not in the context of any vm. + pub fn make_spawn_func<F, R>(self, f: F) -> impl FnOnce() -> R + where + F: FnOnce(&VirtualMachine) -> R, + { + move || self.run(f) + } + + /// Run a function in this thread context + /// + /// # Note + /// + /// If you return a `PyObjectRef` (or a type that contains one) from `F`, and don't return the object + /// to the parent thread and then `join()` on the `JoinHandle` (or similar), there is a possibility that + /// the current thread will panic as `PyObjectRef`'s `Drop` implementation tries to run the `__del__` + /// destructor of a python object but finds that it's not in the context of any vm. + pub fn run<F, R>(&self, f: F) -> R + where + F: FnOnce(&VirtualMachine) -> R, + { + let vm = &self.vm; + enter_vm(vm, || f(vm)) + } +} + +impl VirtualMachine { + /// Start a new thread with access to the same interpreter. + /// + /// # Note + /// + /// If you return a `PyObjectRef` (or a type that contains one) from `F`, and don't `join()` + /// on the thread, there is a possibility that that thread will panic as `PyObjectRef`'s `Drop` + /// implementation tries to run the `__del__` destructor of a python object but finds that it's + /// not in the context of any vm. + #[cfg(feature = "threading")] + pub fn start_thread<F, R>(&self, f: F) -> std::thread::JoinHandle<R> + where + F: FnOnce(&Self) -> R, + F: Send + 'static, + R: Send + 'static, + { + let func = self.new_thread().make_spawn_func(f); + std::thread::spawn(func) + } + + /// Create a new VM thread that can be passed to a function like [`std::thread::spawn`] + /// to use the same interpreter on a different thread. Note that if you just want to + /// use this with `thread::spawn`, you can use + /// [`vm.start_thread()`](`VirtualMachine::start_thread`) as a convenience. + /// + /// # Usage + /// + /// ``` + /// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { + /// use std::thread::Builder; + /// let handle = Builder::new() + /// .name("my thread :)".into()) + /// .spawn(vm.new_thread().make_spawn_func(|vm| vm.ctx.none())) + /// .expect("couldn't spawn thread"); + /// let returned_obj = handle.join().expect("thread panicked"); + /// assert!(vm.is_none(&returned_obj)); + /// # }) + /// ``` + /// + /// Note: this function is safe, but running the returned ThreadedVirtualMachine in the same + /// thread context (i.e. with the same thread-local storage) doesn't have any + /// specific guaranteed behavior. + #[cfg(feature = "threading")] + pub fn new_thread(&self) -> ThreadedVirtualMachine { + let global_trace = self.state.global_trace_func.lock().clone(); + let global_profile = self.state.global_profile_func.lock().clone(); + let use_tracing = global_trace.is_some() || global_profile.is_some(); + + let vm = Self { + builtins: self.builtins.clone(), + sys_module: self.sys_module.clone(), + ctx: self.ctx.clone(), + frames: RefCell::new(vec![]), + datastack: core::cell::UnsafeCell::new(crate::datastack::DataStack::new()), + wasm_id: self.wasm_id.clone(), + exceptions: RefCell::default(), + import_func: self.import_func.clone(), + importlib: self.importlib.clone(), + profile_func: RefCell::new(global_profile.unwrap_or_else(|| self.ctx.none())), + trace_func: RefCell::new(global_trace.unwrap_or_else(|| self.ctx.none())), + use_tracing: Cell::new(use_tracing), + recursion_limit: self.recursion_limit.clone(), + signal_handlers: core::cell::OnceCell::new(), + signal_rx: None, + repr_guards: RefCell::default(), + state: self.state.clone(), + initialized: self.initialized, + recursion_depth: Cell::new(0), + c_stack_soft_limit: Cell::new(VirtualMachine::calculate_c_stack_soft_limit()), + async_gen_firstiter: RefCell::new(None), + async_gen_finalizer: RefCell::new(None), + asyncio_running_loop: RefCell::new(None), + asyncio_running_task: RefCell::new(None), + callable_cache: self.callable_cache.clone(), + }; + ThreadedVirtualMachine { vm } + } +} diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs new file mode 100644 index 00000000000..fb40268c569 --- /dev/null +++ b/crates/vm/src/vm/vm_new.rs @@ -0,0 +1,797 @@ +use crate::{ + AsObject, Py, PyObject, PyObjectRef, PyRef, PyResult, + builtins::{ + PyBaseException, PyBaseExceptionRef, PyBytesRef, PyDictRef, PyModule, PyOSError, PyStrRef, + PyType, PyTypeRef, + builtin_func::PyNativeFunction, + descriptor::PyMethodDescriptor, + tuple::{IntoPyTuple, PyTupleRef}, + }, + convert::{ToPyException, ToPyObject}, + exceptions::OSErrorBuilder, + function::{IntoPyNativeFn, PyMethodFlags}, + scope::Scope, + vm::VirtualMachine, +}; +use rustpython_common::wtf8::Wtf8Buf; +use rustpython_compiler_core::SourceLocation; + +macro_rules! define_exception_fn { + ( + fn $fn_name:ident, $attr:ident, $python_repr:ident + ) => { + #[doc = concat!( + "Create a new python ", + stringify!($python_repr), + " object.\nUseful for raising errors from python functions implemented in rust." + )] + pub fn $fn_name(&self, msg: impl Into<Wtf8Buf>) -> PyBaseExceptionRef + { + let err = self.ctx.exceptions.$attr.to_owned(); + self.new_exception_msg(err, msg.into()) + } + }; +} + +/// Collection of object creation helpers +impl VirtualMachine { + /// Create a new python object + pub fn new_pyobj(&self, value: impl ToPyObject) -> PyObjectRef { + value.to_pyobject(self) + } + + pub fn new_tuple(&self, value: impl IntoPyTuple) -> PyTupleRef { + value.into_pytuple(self) + } + + pub fn new_module( + &self, + name: &str, + dict: PyDictRef, + doc: Option<PyStrRef>, + ) -> PyRef<PyModule> { + let module = PyRef::new_ref( + PyModule::new(), + self.ctx.types.module_type.to_owned(), + Some(dict), + ); + module.init_dict(self.ctx.intern_str(name), doc, self); + module + } + + pub fn new_scope_with_builtins(&self) -> Scope { + Scope::with_builtins(None, self.ctx.new_dict(), self) + } + + pub fn new_scope_with_main(&self) -> PyResult<Scope> { + let scope = self.new_scope_with_builtins(); + let main_module = self.new_module("__main__", scope.globals.clone(), None); + + self.sys_module.get_attr("modules", self)?.set_item( + "__main__", + main_module.into(), + self, + )?; + + Ok(scope) + } + + pub fn new_function<F, FKind>(&self, name: &'static str, f: F) -> PyRef<PyNativeFunction> + where + F: IntoPyNativeFn<FKind>, + { + let def = self + .ctx + .new_method_def(name, f, PyMethodFlags::empty(), None); + def.build_function(self) + } + + pub fn new_method<F, FKind>( + &self, + name: &'static str, + class: &'static Py<PyType>, + f: F, + ) -> PyRef<PyMethodDescriptor> + where + F: IntoPyNativeFn<FKind>, + { + let def = self + .ctx + .new_method_def(name, f, PyMethodFlags::METHOD, None); + def.build_method(class, self) + } + + /// Instantiate an exception with arguments. + /// This function should only be used with builtin exception types; if a user-defined exception + /// type is passed in, it may not be fully initialized; try using + /// [`vm.invoke_exception()`][Self::invoke_exception] or + /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. + pub fn new_exception(&self, exc_type: PyTypeRef, args: Vec<PyObjectRef>) -> PyBaseExceptionRef { + debug_assert_eq!( + exc_type.slots.basicsize, + core::mem::size_of::<PyBaseException>(), + "vm.new_exception() is only for exception types without additional payload. The given type '{}' is not allowed. Use vm.new_os_subtype_error() for OSError subtypes.", + exc_type.name() + ); + + PyRef::new_ref( + PyBaseException::new(args, self), + exc_type, + Some(self.ctx.new_dict()), + ) + } + + pub fn new_os_error(&self, msg: impl ToPyObject) -> PyRef<PyBaseException> { + self.new_os_subtype_error(self.ctx.exceptions.os_error.to_owned(), None, msg) + .upcast() + } + + pub fn new_os_subtype_error( + &self, + exc_type: PyTypeRef, + errno: Option<i32>, + msg: impl ToPyObject, + ) -> PyRef<PyOSError> { + debug_assert_eq!(exc_type.slots.basicsize, core::mem::size_of::<PyOSError>()); + + OSErrorBuilder::with_subtype(exc_type, errno, msg, self).build(self) + } + + /// Instantiate an exception with no arguments. + /// This function should only be used with builtin exception types; if a user-defined exception + /// type is passed in, it may not be fully initialized; try using + /// [`vm.invoke_exception()`][Self::invoke_exception] or + /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. + pub fn new_exception_empty(&self, exc_type: PyTypeRef) -> PyBaseExceptionRef { + self.new_exception(exc_type, vec![]) + } + + /// Instantiate an exception with `msg` as the only argument. + /// This function should only be used with builtin exception types; if a user-defined exception + /// type is passed in, it may not be fully initialized; try using + /// [`vm.invoke_exception()`][Self::invoke_exception] or + /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. + pub fn new_exception_msg(&self, exc_type: PyTypeRef, msg: Wtf8Buf) -> PyBaseExceptionRef { + self.new_exception(exc_type, vec![self.ctx.new_str(msg).into()]) + } + + /// Instantiate an exception with `msg` as the only argument and `dict` for object + /// This function should only be used with builtin exception types; if a user-defined exception + /// type is passed in, it may not be fully initialized; try using + /// [`vm.invoke_exception()`][Self::invoke_exception] or + /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. + pub fn new_exception_msg_dict( + &self, + exc_type: PyTypeRef, + msg: Wtf8Buf, + dict: PyDictRef, + ) -> PyBaseExceptionRef { + PyRef::new_ref( + // TODO: this constructor might be invalid, because multiple + // exception (even builtin ones) are using custom constructors, + // see `OSError` as an example: + PyBaseException::new(vec![self.ctx.new_str(msg).into()], self), + exc_type, + Some(dict), + ) + } + + pub fn new_no_attribute_error(&self, obj: PyObjectRef, name: PyStrRef) -> PyBaseExceptionRef { + let msg = format!( + "'{}' object has no attribute '{}'", + obj.class().name(), + name + ); + let attribute_error = self.new_attribute_error(msg); + + // Use existing set_attribute_error_context function + self.set_attribute_error_context(&attribute_error, obj, name); + + attribute_error + } + + pub fn new_name_error(&self, msg: impl Into<Wtf8Buf>, name: PyStrRef) -> PyBaseExceptionRef { + let name_error_type = self.ctx.exceptions.name_error.to_owned(); + let name_error = self.new_exception_msg(name_error_type, msg.into()); + name_error.as_object().set_attr("name", name, self).unwrap(); + name_error + } + + pub fn new_unsupported_unary_error(&self, a: &PyObject, op: &str) -> PyBaseExceptionRef { + self.new_type_error(format!( + "bad operand type for {}: '{}'", + op, + a.class().name() + )) + } + + pub fn new_unsupported_bin_op_error( + &self, + a: &PyObject, + b: &PyObject, + op: &str, + ) -> PyBaseExceptionRef { + self.new_type_error(format!( + "unsupported operand type(s) for {}: '{}' and '{}'", + op, + a.class().name(), + b.class().name() + )) + } + + pub fn new_unsupported_ternary_op_error( + &self, + a: &PyObject, + b: &PyObject, + c: &PyObject, + op: &str, + ) -> PyBaseExceptionRef { + self.new_type_error(format!( + "Unsupported operand types for '{}': '{}', '{}', and '{}'", + op, + a.class().name(), + b.class().name(), + c.class().name() + )) + } + + /// Create a new OSError from the last OS error. + /// + /// On windows, windows-sys errors are expected to be handled by this function. + /// This is identical to `new_last_errno_error` on non-Windows platforms. + pub fn new_last_os_error(&self) -> PyBaseExceptionRef { + let err = std::io::Error::last_os_error(); + err.to_pyexception(self) + } + + /// Create a new OSError from the last POSIX errno. + /// + /// On windows, CRT errno are expected to be handled by this function. + /// This is identical to `new_last_os_error` on non-Windows platforms. + pub fn new_last_errno_error(&self) -> PyBaseExceptionRef { + let err = crate::common::os::errno_io_error(); + err.to_pyexception(self) + } + + pub fn new_errno_error(&self, errno: i32, msg: impl ToPyObject) -> PyRef<PyOSError> { + let exc_type = crate::exceptions::errno_to_exc_type(errno, self) + .unwrap_or(self.ctx.exceptions.os_error); + + self.new_os_subtype_error(exc_type.to_owned(), Some(errno), msg) + } + + pub fn new_unicode_decode_error_real( + &self, + encoding: PyStrRef, + object: PyBytesRef, + start: usize, + end: usize, + reason: PyStrRef, + ) -> PyBaseExceptionRef { + let start = self.ctx.new_int(start); + let end = self.ctx.new_int(end); + let exc = self.new_exception( + self.ctx.exceptions.unicode_decode_error.to_owned(), + vec![ + encoding.clone().into(), + object.clone().into(), + start.clone().into(), + end.clone().into(), + reason.clone().into(), + ], + ); + exc.as_object() + .set_attr("encoding", encoding, self) + .unwrap(); + exc.as_object().set_attr("object", object, self).unwrap(); + exc.as_object().set_attr("start", start, self).unwrap(); + exc.as_object().set_attr("end", end, self).unwrap(); + exc.as_object().set_attr("reason", reason, self).unwrap(); + exc + } + + pub fn new_unicode_encode_error_real( + &self, + encoding: PyStrRef, + object: PyStrRef, + start: usize, + end: usize, + reason: PyStrRef, + ) -> PyBaseExceptionRef { + let start = self.ctx.new_int(start); + let end = self.ctx.new_int(end); + let exc = self.new_exception( + self.ctx.exceptions.unicode_encode_error.to_owned(), + vec![ + encoding.clone().into(), + object.clone().into(), + start.clone().into(), + end.clone().into(), + reason.clone().into(), + ], + ); + exc.as_object() + .set_attr("encoding", encoding, self) + .unwrap(); + exc.as_object().set_attr("object", object, self).unwrap(); + exc.as_object().set_attr("start", start, self).unwrap(); + exc.as_object().set_attr("end", end, self).unwrap(); + exc.as_object().set_attr("reason", reason, self).unwrap(); + exc + } + + // TODO: don't take ownership should make the success path faster + pub fn new_key_error(&self, obj: PyObjectRef) -> PyBaseExceptionRef { + let key_error = self.ctx.exceptions.key_error.to_owned(); + self.new_exception(key_error, vec![obj]) + } + + #[cfg(any(feature = "parser", feature = "compiler"))] + pub fn new_syntax_error_maybe_incomplete( + &self, + error: &crate::compiler::CompileError, + source: Option<&str>, + allow_incomplete: bool, + ) -> PyBaseExceptionRef { + let incomplete_or_syntax = |allow| -> &'static Py<crate::builtins::PyType> { + if allow { + self.ctx.exceptions.incomplete_input_error + } else { + self.ctx.exceptions.syntax_error + } + }; + + let syntax_error_type = match &error { + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::IndentationError, + ), + .. + }) => { + // Detect tab/space mixing to raise TabError instead of IndentationError. + // This checks both within a single line and across different lines. + let is_tab_error = source.is_some_and(|source| { + let mut has_space_indent = false; + let mut has_tab_indent = false; + for line in source.lines() { + let indent: Vec<u8> = line + .bytes() + .take_while(|&b| b == b' ' || b == b'\t') + .collect(); + if indent.is_empty() { + continue; + } + if indent.contains(&b' ') && indent.contains(&b'\t') { + return true; + } + if indent.contains(&b' ') { + has_space_indent = true; + } + if indent.contains(&b'\t') { + has_tab_indent = true; + } + } + has_space_indent && has_tab_indent + }); + if is_tab_error { + self.ctx.exceptions.tab_error + } else { + self.ctx.exceptions.indentation_error + } + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::UnexpectedIndentation, + .. + }) => self.ctx.exceptions.indentation_error, + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::Eof, + ), + .. + }) => incomplete_or_syntax(allow_incomplete), + // Unclosed bracket errors (converted from Eof by from_ruff_parse_error) + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + is_unclosed_bracket: true, + .. + }) => incomplete_or_syntax(allow_incomplete), + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ), + ), + .. + }) => incomplete_or_syntax(allow_incomplete), + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::UnclosedStringError, + ), + raw_location, + .. + }) => { + if allow_incomplete { + let mut is_incomplete = false; + + if let Some(source) = source { + let loc = raw_location.start().to_usize(); + let mut iter = source.chars(); + if let Some(quote) = iter.nth(loc) + && iter.next() == Some(quote) + && iter.next() == Some(quote) + { + is_incomplete = true; + } + } + + incomplete_or_syntax(is_incomplete) + } else { + self.ctx.exceptions.syntax_error + } + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::OtherError(s), + raw_location, + .. + }) => { + if s.starts_with("Expected an indented block after") { + if allow_incomplete { + // Check that all chars in the error are whitespace, if so, the source is + // incomplete. Otherwise, we've found code that might violates + // indentation rules. + let mut is_incomplete = true; + if let Some(source) = source { + let start = raw_location.start().to_usize(); + let end = raw_location.end().to_usize(); + let mut iter = source.chars(); + iter.nth(start); + for _ in start..end { + if let Some(c) = iter.next() { + if !c.is_ascii_whitespace() { + is_incomplete = false; + } + } else { + break; + } + } + } + + if is_incomplete { + self.ctx.exceptions.incomplete_input_error + } else { + self.ctx.exceptions.indentation_error // not syntax_error + } + } else { + self.ctx.exceptions.indentation_error + } + } else { + self.ctx.exceptions.syntax_error + } + } + _ => self.ctx.exceptions.syntax_error, + } + .to_owned(); + + // TODO: replace to SourceCode + fn get_statement(source: &str, loc: Option<SourceLocation>) -> Option<String> { + let line = source + .split('\n') + .nth(loc?.line.to_zero_indexed())? + .trim_end_matches('\r') + .to_owned(); + Some(line + "\n") + } + + let statement = if let Some(source) = source { + get_statement(source, error.location()) + } else { + None + }; + + let mut msg = error.to_string(); + if let Some(msg) = msg.get_mut(..1) { + msg.make_ascii_lowercase(); + } + let mut narrow_caret = false; + match error { + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedString, + ) + | ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedString, + ), + ), + .. + }) => { + msg = "unterminated f-string literal".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ) + | ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FStringError( + ruff_python_parser::InterpolatedStringErrorType::UnterminatedTripleQuotedString, + ), + ), + .. + }) => { + msg = "unterminated triple-quoted f-string literal".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::FStringError(_) + | ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FStringError(_), + ), + .. + }) => { + // Replace backticks with single quotes to match CPython's error messages + msg = msg.replace('`', "'"); + msg.insert_str(0, "invalid syntax: "); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::UnexpectedExpressionToken, + .. + }) => msg.insert_str(0, "invalid syntax: "), + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::UnrecognizedToken { .. }, + ) + | ruff_python_parser::ParseErrorType::SimpleStatementsOnSameLine + | ruff_python_parser::ParseErrorType::SimpleAndCompoundStatementOnSameLine + | ruff_python_parser::ParseErrorType::ExpectedToken { .. } + | ruff_python_parser::ParseErrorType::ExpectedExpression, + .. + }) => { + msg = "invalid syntax".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::InvalidStarredExpressionUsage, + .. + }) => { + msg = "invalid syntax".to_owned(); + narrow_caret = true; + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::InvalidDeleteTarget, + .. + }) => { + msg = "invalid syntax".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::LineContinuationError, + ), + .. + }) => { + msg = "unexpected character after line continuation".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::UnclosedStringError, + ), + .. + }) => { + msg = "unterminated string".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::OtherError(s), + .. + }) if s.eq_ignore_ascii_case("bytes literal cannot be mixed with non-bytes literals") => { + msg = "cannot mix bytes and nonbytes literals".to_owned(); + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: ruff_python_parser::ParseErrorType::OtherError(s), + .. + }) if s.starts_with("Expected an identifier, but found a keyword") => { + msg = "invalid syntax".to_owned(); + } + _ => {} + } + if syntax_error_type.is(self.ctx.exceptions.tab_error) { + msg = "inconsistent use of tabs and spaces in indentation".to_owned(); + } + let syntax_error = self.new_exception_msg(syntax_error_type, msg.into()); + let (lineno, offset) = error.python_location(); + let lineno = self.ctx.new_int(lineno); + let offset = self.ctx.new_int(offset); + syntax_error + .as_object() + .set_attr("lineno", lineno, self) + .unwrap(); + syntax_error + .as_object() + .set_attr("offset", offset, self) + .unwrap(); + + // Set end_lineno and end_offset if available + if let Some((end_lineno, end_offset)) = error.python_end_location() { + let (end_lineno, end_offset) = if narrow_caret { + let (l, o) = error.python_location(); + (l, o + 1) + } else { + (end_lineno, end_offset) + }; + let end_lineno = self.ctx.new_int(end_lineno); + let end_offset = self.ctx.new_int(end_offset); + syntax_error + .as_object() + .set_attr("end_lineno", end_lineno, self) + .unwrap(); + syntax_error + .as_object() + .set_attr("end_offset", end_offset, self) + .unwrap(); + } + + syntax_error + .as_object() + .set_attr("text", statement.to_pyobject(self), self) + .unwrap(); + syntax_error + .as_object() + .set_attr("filename", self.ctx.new_str(error.source_path()), self) + .unwrap(); + + // Set _metadata for keyword typo suggestions in traceback module. + // Format: (start_line, col_offset, source_code) + // start_line=0 means "include all lines from beginning" which provides + // full context needed by _find_keyword_typos to compile-check suggestions. + if let Some(source) = source { + let metadata = self.ctx.new_tuple(vec![ + self.ctx.new_int(0).into(), + self.ctx.new_int(0).into(), + self.ctx.new_str(source).into(), + ]); + syntax_error + .as_object() + .set_attr("_metadata", metadata, self) + .unwrap(); + } + + syntax_error + } + + #[cfg(any(feature = "parser", feature = "compiler"))] + pub fn new_syntax_error( + &self, + error: &crate::compiler::CompileError, + source: Option<&str>, + ) -> PyBaseExceptionRef { + self.new_syntax_error_maybe_incomplete(error, source, false) + } + + pub fn new_import_error( + &self, + msg: impl Into<Wtf8Buf>, + name: impl Into<PyStrRef>, + ) -> PyBaseExceptionRef { + let import_error = self.ctx.exceptions.import_error.to_owned(); + let exc = self.new_exception_msg(import_error, msg.into()); + exc.as_object().set_attr("name", name.into(), self).unwrap(); + exc + } + + pub fn new_stop_iteration(&self, value: Option<PyObjectRef>) -> PyBaseExceptionRef { + let dict = self.ctx.new_dict(); + let args = if let Some(value) = value { + // manually set `value` attribute like StopIteration.__init__ + dict.set_item("value", value.clone(), self) + .expect("dict.__setitem__ never fails"); + vec![value] + } else { + Vec::new() + }; + + PyRef::new_ref( + PyBaseException::new(args, self), + self.ctx.exceptions.stop_iteration.to_owned(), + Some(dict), + ) + } + + fn new_downcast_error( + &self, + msg: &'static str, + error_type: &'static Py<PyType>, + class: &Py<PyType>, + obj: &PyObject, // the impl Borrow allows to pass PyObjectRef or &PyObject + ) -> PyBaseExceptionRef { + let actual_class = obj.class(); + let actual_type = &*actual_class.name(); + let expected_type = &*class.name(); + let msg = format!("Expected {msg} '{expected_type}' but '{actual_type}' found."); + #[cfg(debug_assertions)] + let msg = if class.get_id() == actual_class.get_id() { + let mut msg = msg; + msg += " It might mean this type doesn't support subclassing very well. e.g. Did you forget to add `#[pyclass(with(Constructor))]`?"; + msg + } else { + msg + }; + self.new_exception_msg(error_type.to_owned(), msg.into()) + } + + pub(crate) fn new_downcast_runtime_error( + &self, + class: &Py<PyType>, + obj: &impl AsObject, + ) -> PyBaseExceptionRef { + self.new_downcast_error( + "payload", + self.ctx.exceptions.runtime_error, + class, + obj.as_object(), + ) + } + + pub(crate) fn new_downcast_type_error( + &self, + class: &Py<PyType>, + obj: &impl AsObject, + ) -> PyBaseExceptionRef { + self.new_downcast_error( + "type", + self.ctx.exceptions.type_error, + class, + obj.as_object(), + ) + } + + define_exception_fn!(fn new_lookup_error, lookup_error, LookupError); + define_exception_fn!(fn new_eof_error, eof_error, EOFError); + define_exception_fn!(fn new_attribute_error, attribute_error, AttributeError); + define_exception_fn!(fn new_type_error, type_error, TypeError); + define_exception_fn!(fn new_system_error, system_error, SystemError); + + // TODO: remove & replace with new_unicode_decode_error_real + define_exception_fn!(fn new_unicode_decode_error, unicode_decode_error, UnicodeDecodeError); + + // TODO: remove & replace with new_unicode_encode_error_real + define_exception_fn!(fn new_unicode_encode_error, unicode_encode_error, UnicodeEncodeError); + + define_exception_fn!(fn new_value_error, value_error, ValueError); + + define_exception_fn!(fn new_buffer_error, buffer_error, BufferError); + define_exception_fn!(fn new_index_error, index_error, IndexError); + define_exception_fn!( + fn new_not_implemented_error, + not_implemented_error, + NotImplementedError + ); + define_exception_fn!(fn new_recursion_error, recursion_error, RecursionError); + define_exception_fn!(fn new_zero_division_error, zero_division_error, ZeroDivisionError); + define_exception_fn!(fn new_overflow_error, overflow_error, OverflowError); + define_exception_fn!(fn new_runtime_error, runtime_error, RuntimeError); + define_exception_fn!(fn new_python_finalization_error, python_finalization_error, PythonFinalizationError); + define_exception_fn!(fn new_memory_error, memory_error, MemoryError); +} diff --git a/crates/vm/src/vm/vm_object.rs b/crates/vm/src/vm/vm_object.rs new file mode 100644 index 00000000000..0d5b286148c --- /dev/null +++ b/crates/vm/src/vm/vm_object.rs @@ -0,0 +1,184 @@ +use super::PyMethod; +use crate::{ + builtins::{PyBaseExceptionRef, PyList, PyStrInterned, pystr::AsPyStr}, + function::IntoFuncArgs, + object::{AsObject, PyObject, PyObjectRef, PyResult}, + stdlib::sys, + vm::VirtualMachine, +}; + +/// PyObject support +impl VirtualMachine { + #[track_caller] + #[cold] + fn _py_panic_failed(&self, exc: PyBaseExceptionRef, msg: &str) -> ! { + #[cfg(not(all( + target_arch = "wasm32", + not(any(target_os = "emscripten", target_os = "wasi")), + )))] + { + self.print_exception(exc); + self.flush_std(); + panic!("{msg}") + } + #[cfg(all( + target_arch = "wasm32", + feature = "wasmbind", + not(any(target_os = "emscripten", target_os = "wasi")), + ))] + #[cfg(all(target_arch = "wasm32", not(target_os = "wasi"), feature = "wasmbind"))] + { + use wasm_bindgen::prelude::*; + #[wasm_bindgen] + extern "C" { + #[wasm_bindgen(js_namespace = console)] + fn error(s: &str); + } + let mut s = String::new(); + self.write_exception(&mut s, &exc).unwrap(); + error(&s); + panic!("{msg}; exception backtrace above") + } + #[cfg(all( + target_arch = "wasm32", + not(feature = "wasmbind"), + not(any(target_os = "emscripten", target_os = "wasi")), + ))] + { + use crate::convert::ToPyObject; + let err_string: String = exc.to_pyobject(self).repr(self).unwrap().to_string(); + eprintln!("{err_string}"); + panic!("{msg}; python exception not available") + } + } + + pub(crate) fn flush_std(&self) { + let vm = self; + if let Ok(stdout) = sys::get_stdout(vm) { + let _ = vm.call_method(&stdout, identifier!(vm, flush).as_str(), ()); + } + if let Ok(stderr) = sys::get_stderr(vm) { + let _ = vm.call_method(&stderr, identifier!(vm, flush).as_str(), ()); + } + } + + #[track_caller] + pub fn unwrap_pyresult<T>(&self, result: PyResult<T>) -> T { + match result { + Ok(x) => x, + Err(exc) => { + self._py_panic_failed(exc, "called `vm.unwrap_pyresult()` on an `Err` value") + } + } + } + #[track_caller] + pub fn expect_pyresult<T>(&self, result: PyResult<T>, msg: &str) -> T { + match result { + Ok(x) => x, + Err(exc) => self._py_panic_failed(exc, msg), + } + } + + /// Test whether a python object is `None`. + pub fn is_none(&self, obj: &PyObject) -> bool { + obj.is(&self.ctx.none) + } + pub fn option_if_none(&self, obj: PyObjectRef) -> Option<PyObjectRef> { + if self.is_none(&obj) { None } else { Some(obj) } + } + pub fn unwrap_or_none(&self, obj: Option<PyObjectRef>) -> PyObjectRef { + obj.unwrap_or_else(|| self.ctx.none()) + } + + pub fn call_get_descriptor_specific( + &self, + descr: &PyObject, + obj: Option<PyObjectRef>, + cls: Option<PyObjectRef>, + ) -> Option<PyResult> { + let descr_get = descr.class().slots.descr_get.load()?; + Some(descr_get(descr.to_owned(), obj, cls, self)) + } + + pub fn call_get_descriptor(&self, descr: &PyObject, obj: PyObjectRef) -> Option<PyResult> { + let cls = obj.class().to_owned().into(); + self.call_get_descriptor_specific(descr, Some(obj), Some(cls)) + } + + pub fn call_if_get_descriptor(&self, attr: &PyObject, obj: PyObjectRef) -> PyResult { + self.call_get_descriptor(attr, obj) + .unwrap_or_else(|| Ok(attr.to_owned())) + } + + #[inline] + pub fn call_method<T>(&self, obj: &PyObject, method_name: &str, args: T) -> PyResult + where + T: IntoFuncArgs, + { + flame_guard!(format!("call_method({:?})", method_name)); + + let dynamic_name; + let name = match self.ctx.interned_str(method_name) { + Some(name) => name.as_pystr(&self.ctx), + None => { + dynamic_name = self.ctx.new_str(method_name); + &dynamic_name + } + }; + PyMethod::get(obj.to_owned(), name, self)?.invoke(args, self) + } + + pub fn dir(&self, obj: Option<PyObjectRef>) -> PyResult<PyList> { + let seq = match obj { + Some(obj) => self + .get_special_method(&obj, identifier!(self, __dir__))? + .ok_or_else(|| self.new_type_error("object does not provide __dir__"))? + .invoke((), self)?, + None => self.call_method( + self.current_locals()?.as_object(), + identifier!(self, keys).as_str(), + (), + )?, + }; + let items: Vec<_> = seq.try_to_value(self)?; + let lst = PyList::from(items); + lst.sort(Default::default(), self)?; + Ok(lst) + } + + #[inline] + pub(crate) fn get_special_method( + &self, + obj: &PyObject, + method: &'static PyStrInterned, + ) -> PyResult<Option<PyMethod>> { + PyMethod::get_special::<false>(obj, method, self) + } + + /// NOT PUBLIC API + #[doc(hidden)] + pub fn call_special_method( + &self, + obj: &PyObject, + method: &'static PyStrInterned, + args: impl IntoFuncArgs, + ) -> PyResult { + self.get_special_method(obj, method)? + .ok_or_else(|| self.new_attribute_error(method.as_str().to_owned()))? + .invoke(args, self) + } + + /// Same as __builtins__.print in Python. + /// A convenience function to provide a simple way to print objects for debug purpose. + // NOTE: Keep the interface simple. + pub fn print(&self, args: impl IntoFuncArgs) -> PyResult<()> { + let ret = self.builtins.get_attr("print", self)?.call(args, self)?; + debug_assert!(self.is_none(&ret)); + Ok(()) + } + + #[deprecated(note = "in favor of `obj.call(args, vm)`")] + pub fn invoke(&self, obj: &impl AsObject, args: impl IntoFuncArgs) -> PyResult { + obj.as_object().call(args, self) + } +} diff --git a/vm/src/vm/vm_ops.rs b/crates/vm/src/vm/vm_ops.rs similarity index 77% rename from vm/src/vm/vm_ops.rs rename to crates/vm/src/vm/vm_ops.rs index 0fc9a1953ea..d5c70f87386 100644 --- a/vm/src/vm/vm_ops.rs +++ b/crates/vm/src/vm/vm_ops.rs @@ -1,8 +1,10 @@ -use super::{PyMethod, VirtualMachine}; +use super::VirtualMachine; +use crate::stdlib::_warnings; use crate::{ - builtins::{PyInt, PyIntRef, PyStr, PyStrRef}, + PyRef, + builtins::{PyInt, PyStr, PyStrRef, PyUtf8Str}, object::{AsObject, PyObject, PyObjectRef, PyResult}, - protocol::{PyIterReturn, PyNumberBinaryOp, PyNumberTernaryOp, PySequence}, + protocol::{PyNumberBinaryOp, PyNumberTernaryOp}, types::PyComparisonOp, }; use num_traits::ToPrimitive; @@ -114,11 +116,11 @@ impl VirtualMachine { Ok(None) } else { Err(e) - } + }; } }; let hint = result - .payload_if_subclass::<PyInt>(self) + .downcast_ref::<PyInt>() .ok_or_else(|| { self.new_type_error(format!( "'{}' object cannot be interpreted as an integer", @@ -127,7 +129,7 @@ impl VirtualMachine { })? .try_to_primitive::<isize>(self)?; if hint.is_negative() { - Err(self.new_value_error("__length_hint__() should return >= 0".to_owned())) + Err(self.new_value_error("__length_hint__() should return >= 0")) } else { Ok(Some(hint as usize)) } @@ -141,7 +143,7 @@ impl VirtualMachine { } else { let n = n as usize; if length > crate::stdlib::sys::MAXSIZE as usize / n { - Err(self.new_overflow_error("repeated value are too long".to_owned())) + Err(self.new_overflow_error("repeated value are too long")) } else { Ok(n) } @@ -151,13 +153,14 @@ impl VirtualMachine { /// Calling scheme used for binary operations: /// /// Order operations are tried until either a valid result or error: - /// b.rop(b,a)[*], a.op(a,b), b.rop(b,a) + /// `b.rop(b,a)[*], a.op(a,b), b.rop(b,a)` /// - /// [*] only when Py_TYPE(a) != Py_TYPE(b) && Py_TYPE(b) is a subclass of Py_TYPE(a) + /// `[*]` - only when Py_TYPE(a) != Py_TYPE(b) && Py_TYPE(b) is a subclass of Py_TYPE(a) pub fn binary_op1(&self, a: &PyObject, b: &PyObject, op_slot: PyNumberBinaryOp) -> PyResult { let class_a = a.class(); let class_b = b.class(); + // Number slots are inherited, direct access is O(1) let slot_a = class_a.slots.as_number.left_binary_op(op_slot); let mut slot_b = None; @@ -169,14 +172,14 @@ impl VirtualMachine { } if let Some(slot_a) = slot_a { - if let Some(slot_bb) = slot_b { - if class_b.fast_issubclass(class_a) { - let ret = slot_bb(a, b, self)?; - if !ret.is(&self.ctx.not_implemented) { - return Ok(ret); - } - slot_b = None; + if let Some(slot_bb) = slot_b + && class_b.fast_issubclass(class_a) + { + let ret = slot_bb(a, b, self)?; + if !ret.is(&self.ctx.not_implemented) { + return Ok(ret); } + slot_b = None; } let ret = slot_a(a, b, self)?; if !ret.is(&self.ctx.not_implemented) { @@ -205,7 +208,7 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - Err(self.new_unsupported_binop_error(a, b, op)) + Err(self.new_unsupported_bin_op_error(a, b, op)) } /// Binary in-place operators @@ -249,7 +252,7 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - Err(self.new_unsupported_binop_error(a, b, op)) + Err(self.new_unsupported_bin_op_error(a, b, op)) } fn ternary_op( @@ -264,6 +267,7 @@ impl VirtualMachine { let class_b = b.class(); let class_c = c.class(); + // Number slots are inherited, direct access is O(1) let slot_a = class_a.slots.as_number.left_ternary_op(op_slot); let mut slot_b = None; @@ -275,14 +279,14 @@ impl VirtualMachine { } if let Some(slot_a) = slot_a { - if let Some(slot_bb) = slot_b { - if class_b.fast_issubclass(class_a) { - let ret = slot_bb(a, b, c, self)?; - if !ret.is(&self.ctx.not_implemented) { - return Ok(ret); - } - slot_b = None; + if let Some(slot_bb) = slot_b + && class_b.fast_issubclass(class_a) + { + let ret = slot_bb(a, b, c, self)?; + if !ret.is(&self.ctx.not_implemented) { + return Ok(ret); } + slot_b = None; } let ret = slot_a(a, b, c, self)?; if !ret.is(&self.ctx.not_implemented) { @@ -297,14 +301,13 @@ impl VirtualMachine { } } - if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) { - if slot_a.map_or(false, |slot_a| (slot_a as usize) != (slot_c as usize)) - && slot_b.map_or(false, |slot_b| (slot_b as usize) != (slot_c as usize)) - { - let ret = slot_c(a, b, c, self)?; - if !ret.is(&self.ctx.not_implemented) { - return Ok(ret); - } + if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) + && slot_a.is_some_and(|slot_a| !core::ptr::fn_addr_eq(slot_a, slot_c)) + && slot_b.is_some_and(|slot_b| !core::ptr::fn_addr_eq(slot_b, slot_c)) + { + let ret = slot_c(a, b, c, self)?; + if !ret.is(&self.ctx.not_implemented) { + return Ok(ret); } } @@ -377,13 +380,15 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - if let Ok(seq_a) = PySequence::try_protocol(a, self) { - let result = seq_a.concat(b, self)?; + // Check if concat slot is available directly, matching PyNumber_Add behavior + let seq = a.sequence_unchecked(); + if let Some(f) = seq.slots().concat.load() { + let result = f(seq, b, self)?; if !result.is(&self.ctx.not_implemented) { return Ok(result); } } - Err(self.new_unsupported_binop_error(a, b, "+")) + Err(self.new_unsupported_bin_op_error(a, b, "+")) } pub fn _iadd(&self, a: &PyObject, b: &PyObject) -> PyResult { @@ -391,13 +396,16 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - if let Ok(seq_a) = PySequence::try_protocol(a, self) { - let result = seq_a.inplace_concat(b, self)?; + // Check inplace_concat or concat slot directly, matching PyNumber_InPlaceAdd behavior + let seq = a.sequence_unchecked(); + let slots = seq.slots(); + if let Some(f) = slots.inplace_concat.load().or_else(|| slots.concat.load()) { + let result = f(seq, b, self)?; if !result.is(&self.ctx.not_implemented) { return Ok(result); } } - Err(self.new_unsupported_binop_error(a, b, "+=")) + Err(self.new_unsupported_bin_op_error(a, b, "+=")) } pub fn _mul(&self, a: &PyObject, b: &PyObject) -> PyResult { @@ -405,20 +413,22 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - if let Ok(seq_a) = PySequence::try_protocol(a, self) { - let n = - b.try_index(self)?.as_bigint().to_isize().ok_or_else(|| { - self.new_overflow_error("repeated bytes are too long".to_owned()) - })?; + if let Ok(seq_a) = a.try_sequence(self) { + let n = b + .try_index(self)? + .as_bigint() + .to_isize() + .ok_or_else(|| self.new_overflow_error("repeated bytes are too long"))?; return seq_a.repeat(n, self); - } else if let Ok(seq_b) = PySequence::try_protocol(b, self) { - let n = - a.try_index(self)?.as_bigint().to_isize().ok_or_else(|| { - self.new_overflow_error("repeated bytes are too long".to_owned()) - })?; + } else if let Ok(seq_b) = b.try_sequence(self) { + let n = a + .try_index(self)? + .as_bigint() + .to_isize() + .ok_or_else(|| self.new_overflow_error("repeated bytes are too long"))?; return seq_b.repeat(n, self); } - Err(self.new_unsupported_binop_error(a, b, "*")) + Err(self.new_unsupported_bin_op_error(a, b, "*")) } pub fn _imul(&self, a: &PyObject, b: &PyObject) -> PyResult { @@ -431,23 +441,25 @@ impl VirtualMachine { if !result.is(&self.ctx.not_implemented) { return Ok(result); } - if let Ok(seq_a) = PySequence::try_protocol(a, self) { - let n = - b.try_index(self)?.as_bigint().to_isize().ok_or_else(|| { - self.new_overflow_error("repeated bytes are too long".to_owned()) - })?; + if let Ok(seq_a) = a.try_sequence(self) { + let n = b + .try_index(self)? + .as_bigint() + .to_isize() + .ok_or_else(|| self.new_overflow_error("repeated bytes are too long"))?; return seq_a.inplace_repeat(n, self); - } else if let Ok(seq_b) = PySequence::try_protocol(b, self) { - let n = - a.try_index(self)?.as_bigint().to_isize().ok_or_else(|| { - self.new_overflow_error("repeated bytes are too long".to_owned()) - })?; + } else if let Ok(seq_b) = b.try_sequence(self) { + let n = a + .try_index(self)? + .as_bigint() + .to_isize() + .ok_or_else(|| self.new_overflow_error("repeated bytes are too long"))?; /* Note that the right hand operand should not be * mutated in this case so inplace_repeat is not * used. */ return seq_b.repeat(n, self); } - Err(self.new_unsupported_binop_error(a, b, "*=")) + Err(self.new_unsupported_bin_op_error(a, b, "*=")) } pub fn _abs(&self, a: &PyObject) -> PyResult<PyObjectRef> { @@ -469,6 +481,17 @@ impl VirtualMachine { } pub fn _invert(&self, a: &PyObject) -> PyResult { + const STR: &str = "Bitwise inversion '~' on bool is deprecated and will be removed in Python 3.16. \ + This returns the bitwise inversion of the underlying int object and is usually not what you expect from negating a bool. \ + Use the 'not' operator for boolean negation or ~int(x) if you really want the bitwise inversion of the underlying int."; + if a.fast_isinstance(self.ctx.types.bool_type) { + _warnings::warn( + self.ctx.exceptions.deprecation_warning, + STR.to_owned(), + 1, + self, + )?; + } self.get_special_method(a, identifier!(self, __invert__))? .ok_or_else(|| self.new_unsupported_unary_error(a, "unary ~"))? .invoke((), self) @@ -501,33 +524,12 @@ impl VirtualMachine { )) }) } - - // https://docs.python.org/3/reference/expressions.html#membership-test-operations - fn _membership_iter_search( - &self, - haystack: &PyObject, - needle: PyObjectRef, - ) -> PyResult<PyIntRef> { - let iter = haystack.get_iter(self)?; - loop { - if let PyIterReturn::Return(element) = iter.next(self)? { - if self.bool_eq(&element, &needle)? { - return Ok(self.ctx.new_bool(true)); - } else { - continue; - } - } else { - return Ok(self.ctx.new_bool(false)); - } - } + pub fn format_utf8(&self, obj: &PyObject, format_spec: PyStrRef) -> PyResult<PyRef<PyUtf8Str>> { + self.format(obj, format_spec)?.try_into_utf8(self) } - pub fn _contains(&self, haystack: &PyObject, needle: PyObjectRef) -> PyResult { - match PyMethod::get_special::<false>(haystack, identifier!(self, __contains__), self)? { - Some(method) => method.invoke((needle,), self), - None => self - ._membership_iter_search(haystack, needle) - .map(Into::into), - } + pub fn _contains(&self, haystack: &PyObject, needle: &PyObject) -> PyResult<bool> { + let seq = haystack.sequence_unchecked(); + seq.contains(needle, self) } } diff --git a/crates/vm/src/warn.rs b/crates/vm/src/warn.rs new file mode 100644 index 00000000000..0bae7a9619a --- /dev/null +++ b/crates/vm/src/warn.rs @@ -0,0 +1,611 @@ +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyResult, VirtualMachine, + builtins::{ + PyBaseExceptionRef, PyDictRef, PyListRef, PyStr, PyStrInterned, PyStrRef, PyTuple, + PyTupleRef, PyTypeRef, + }, + convert::TryFromObject, +}; +use core::sync::atomic::{AtomicUsize, Ordering}; +use rustpython_common::lock::OnceCell; + +pub struct WarningsState { + pub filters: PyListRef, + pub once_registry: PyDictRef, + pub default_action: PyStrRef, + pub filters_version: AtomicUsize, + pub context_var: OnceCell<PyObjectRef>, + lock_count: AtomicUsize, +} + +impl WarningsState { + fn create_default_filters(ctx: &Context) -> PyListRef { + // init_filters(): non-debug default filter set. + ctx.new_list(vec![ + ctx.new_tuple(vec![ + ctx.new_str("default").into(), + ctx.none(), + ctx.exceptions.deprecation_warning.as_object().to_owned(), + ctx.new_str("__main__").into(), + ctx.new_int(0).into(), + ]) + .into(), + ctx.new_tuple(vec![ + ctx.new_str("ignore").into(), + ctx.none(), + ctx.exceptions.deprecation_warning.as_object().to_owned(), + ctx.none(), + ctx.new_int(0).into(), + ]) + .into(), + ctx.new_tuple(vec![ + ctx.new_str("ignore").into(), + ctx.none(), + ctx.exceptions + .pending_deprecation_warning + .as_object() + .to_owned(), + ctx.none(), + ctx.new_int(0).into(), + ]) + .into(), + ctx.new_tuple(vec![ + ctx.new_str("ignore").into(), + ctx.none(), + ctx.exceptions.import_warning.as_object().to_owned(), + ctx.none(), + ctx.new_int(0).into(), + ]) + .into(), + ctx.new_tuple(vec![ + ctx.new_str("ignore").into(), + ctx.none(), + ctx.exceptions.resource_warning.as_object().to_owned(), + ctx.none(), + ctx.new_int(0).into(), + ]) + .into(), + ]) + } + + pub fn init_state(ctx: &Context) -> Self { + Self { + filters: Self::create_default_filters(ctx), + once_registry: ctx.new_dict(), + default_action: ctx.new_str("default"), + filters_version: AtomicUsize::new(0), + context_var: OnceCell::new(), + lock_count: AtomicUsize::new(0), + } + } + + pub fn acquire_lock(&self) { + self.lock_count.fetch_add(1, Ordering::SeqCst); + } + + pub fn release_lock(&self) -> bool { + let prev = self.lock_count.load(Ordering::SeqCst); + if prev == 0 { + return false; + } + self.lock_count.fetch_sub(1, Ordering::SeqCst); + true + } + + pub fn filters_mutated(&self) { + self.filters_version.fetch_add(1, Ordering::SeqCst); + } +} + +/// None matches everything; plain strings do exact comparison; +/// regex objects use .match(). +fn check_matched(obj: &PyObject, arg: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { + if vm.is_none(obj) { + return Ok(true); + } + if obj.class().is(vm.ctx.types.str_type) { + return obj.rich_compare_bool(arg, crate::types::PyComparisonOp::Eq, vm); + } + let result = vm.call_method(obj, "match", (arg.to_owned(),))?; + result.is_true(vm) +} + +fn get_warnings_attr( + vm: &VirtualMachine, + attr_name: &'static PyStrInterned, + try_import: bool, +) -> PyResult<Option<PyObjectRef>> { + let module = if try_import + && !vm + .state + .finalizing + .load(core::sync::atomic::Ordering::SeqCst) + { + match vm.import("warnings", 0) { + Ok(module) => module, + Err(_) => return Ok(None), + } + } else { + match vm.sys_module.get_attr(identifier!(vm, modules), vm) { + Ok(modules) => match modules.get_item(vm.ctx.intern_str("warnings"), vm) { + Ok(module) => module, + Err(_) => return Ok(None), + }, + Err(_) => return Ok(None), + } + }; + match module.get_attr(attr_name, vm) { + Ok(attr) => Ok(Some(attr)), + Err(_) => Ok(None), + } +} + +/// Get the warnings filters list from sys.modules['warnings'].filters, +/// falling back to vm.state.warnings.filters. +fn get_warnings_filters(vm: &VirtualMachine) -> PyResult<PyListRef> { + if let Some(filters_obj) = get_warnings_attr(vm, identifier!(&vm.ctx, filters), false)? + && let Ok(filters) = filters_obj.try_into_value::<PyListRef>(vm) + { + return Ok(filters); + } + Ok(vm.state.warnings.filters.clone()) +} + +/// Get the default action from sys.modules['warnings']._defaultaction, +/// falling back to vm.state.warnings.default_action. +fn get_default_action(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if let Some(action) = get_warnings_attr(vm, identifier!(&vm.ctx, defaultaction), false)? { + if !action.class().is(vm.ctx.types.str_type) { + return Err(vm.new_type_error(format!( + "_warnings.defaultaction must be a string, not '{}'", + action.class().name() + ))); + } + return Ok(action); + } + Ok(vm.state.warnings.default_action.clone().into()) +} + +/// Get the once registry from sys.modules['warnings']._onceregistry, +/// falling back to vm.state.warnings.once_registry. +fn get_once_registry(vm: &VirtualMachine) -> PyResult<PyObjectRef> { + if let Some(registry) = get_warnings_attr(vm, identifier!(&vm.ctx, onceregistry), false)? { + if !registry.class().is(vm.ctx.types.dict_type) { + return Err(vm.new_type_error(format!( + "_warnings.onceregistry must be a dict, not '{}'", + registry.class().name() + ))); + } + return Ok(registry); + } + Ok(vm.state.warnings.once_registry.clone().into()) +} + +fn already_warned( + registry: &PyObject, + key: PyObjectRef, + should_set: bool, + vm: &VirtualMachine, +) -> PyResult<bool> { + if vm.is_none(registry) { + return Ok(false); + } + + let current_version = vm.state.warnings.filters_version.load(Ordering::SeqCst); + let version_obj = registry.get_item(identifier!(&vm.ctx, version), vm).ok(); + + let version_matches = version_obj.as_ref().is_some_and(|v| { + v.try_int(vm) + .map(|i| i.as_u32_mask() as usize == current_version) + .unwrap_or(false) + }); + + if version_matches { + if let Ok(val) = registry.get_item(key.as_ref(), vm) + && val.is_true(vm)? + { + return Ok(true); + } + } else if let Ok(dict) = PyDictRef::try_from_object(vm, registry.to_owned()) { + dict.clear(); + dict.set_item( + identifier!(&vm.ctx, version), + vm.ctx.new_int(current_version).into(), + vm, + )?; + } + + if should_set { + registry.set_item(key.as_ref(), vm.ctx.true_value.clone().into(), vm)?; + } + Ok(false) +} + +/// Create a `(text, category)` or `(text, category, 0)` key and record +/// it in the registry via `already_warned`. +fn update_registry( + registry: &PyObject, + text: &PyObject, + category: &PyObject, + add_zero: bool, + vm: &VirtualMachine, +) -> PyResult<bool> { + let altkey: PyObjectRef = if add_zero { + PyTuple::new_ref( + vec![ + text.to_owned(), + category.to_owned(), + vm.ctx.new_int(0).into(), + ], + &vm.ctx, + ) + .into() + } else { + PyTuple::new_ref(vec![text.to_owned(), category.to_owned()], &vm.ctx).into() + }; + already_warned(registry, altkey, true, vm) +} + +fn normalize_module(filename: &Py<PyStr>, vm: &VirtualMachine) -> PyObjectRef { + match filename.byte_len() { + 0 => vm.new_pyobj("<unknown>"), + len if len >= 3 && filename.as_bytes().ends_with(b".py") => { + vm.new_pyobj(&filename.as_wtf8()[..len - 3]) + } + _ => filename.as_object().to_owned(), + } +} + +/// Search the global filters list for a matching action. +// TODO: split into filter_search() + get_filter() and support +// context-aware filters (get_warnings_context_filters). +fn get_filter( + category: PyObjectRef, + text: PyObjectRef, + lineno: usize, + module: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + let filters = get_warnings_filters(vm)?; + + // filters could change while we are iterating over it. + // Re-check list length each iteration (matches C behavior). + let mut i = 0; + while i < filters.borrow_vec().len() { + let Some(tmp_item) = filters.borrow_vec().get(i).cloned() else { + break; + }; + let tmp_item = PyTupleRef::try_from_object(vm, tmp_item) + .ok() + .filter(|t| t.len() == 5) + .ok_or_else(|| { + vm.new_value_error(format!("_warnings.filters item {i} isn't a 5-tuple")) + })?; + + /* action, msg, cat, mod, ln = item */ + let action = &tmp_item[0]; + let good_msg = check_matched(&tmp_item[1], &text, vm)?; + let is_subclass = category.is_subclass(&tmp_item[2], vm)?; + let good_mod = check_matched(&tmp_item[3], &module, vm)?; + let ln: usize = tmp_item[4].try_int(vm).map_or(0, |v| v.as_u32_mask() as _); + + if good_msg && is_subclass && good_mod && (ln == 0 || lineno == ln) { + return Ok(action.to_owned()); + } + i += 1; + } + + get_default_action(vm) +} + +pub fn warn( + message: PyObjectRef, + category: Option<PyTypeRef>, + stack_level: isize, + source: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + warn_with_skip(message, category, stack_level, source, None, vm) +} + +/// do_warn: resolve context via setup_context, then call warn_explicit. +pub fn warn_with_skip( + message: PyObjectRef, + category: Option<PyTypeRef>, + mut stack_level: isize, + source: Option<PyObjectRef>, + skip_file_prefixes: Option<PyTupleRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + if let Some(ref prefixes) = skip_file_prefixes + && !prefixes.is_empty() + && stack_level < 2 + { + stack_level = 2; + } + let (filename, lineno, module, registry) = + setup_context(stack_level, skip_file_prefixes.as_ref(), vm)?; + warn_explicit( + category, message, filename, lineno, module, registry, None, source, vm, + ) +} + +/// Core warning logic matching `warn_explicit()` in `_warnings.c`. +#[allow(clippy::too_many_arguments)] +pub(crate) fn warn_explicit( + category: Option<PyTypeRef>, + message: PyObjectRef, + filename: PyStrRef, + lineno: usize, + module: Option<PyObjectRef>, + registry: PyObjectRef, + source_line: Option<PyObjectRef>, + source: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + // Normalize module. None → silent return (late-shutdown safety). + let module = module.unwrap_or_else(|| normalize_module(&filename, vm)); + if vm.is_none(&module) { + return Ok(()); + } + + // Normalize message. + let is_warning = message.fast_isinstance(vm.ctx.exceptions.warning); + let (text, category, message) = if is_warning { + let text = message.str(vm)?; + let cat = message.class().to_owned(); + (text, cat, message) + } else { + // For non-Warning messages, convert to string via str() + let text = message.str(vm)?; + let cat = category.unwrap_or_else(|| vm.ctx.exceptions.user_warning.to_owned()); + let instance = cat.as_object().call((text.clone(),), vm)?; + (text, cat, instance) + }; + + let lineno_obj: PyObjectRef = vm.ctx.new_int(lineno).into(); + + // key = (text, category, lineno) + let key: PyObjectRef = PyTuple::new_ref( + vec![ + text.clone().into(), + category.as_object().to_owned(), + lineno_obj.clone(), + ], + &vm.ctx, + ) + .into(); + + // Check if already warned + if !vm.is_none(&registry) && already_warned(&registry, key.clone(), false, vm)? { + return Ok(()); + } + + // Get filter action + let action = get_filter( + category.as_object().to_owned(), + text.clone().into(), + lineno, + module, + vm, + )?; + let action_str = PyStrRef::try_from_object(vm, action) + .map_err(|_| vm.new_type_error("action must be a string"))?; + + if action_str.as_bytes() == b"error" { + let exc = PyBaseExceptionRef::try_from_object(vm, message)?; + return Err(exc); + } + if action_str.as_bytes() == b"ignore" { + return Ok(()); + } + + // For everything except "always"/"all", record in registry then + // check per-action registries. + let already = if action_str.as_wtf8() != "always" && action_str.as_wtf8() != "all" { + if !vm.is_none(&registry) { + registry.set_item(&*key, vm.ctx.true_value.clone().into(), vm)?; + } + + let action_s = action_str.to_str(); + match action_s { + Some("once") => { + let reg = if vm.is_none(&registry) { + get_once_registry(vm)? + } else { + registry.clone() + }; + update_registry(&reg, text.as_ref(), category.as_object(), false, vm)? + } + Some("module") => { + if !vm.is_none(&registry) { + update_registry(&registry, text.as_ref(), category.as_object(), false, vm)? + } else { + false + } + } + Some("default") => false, + _ => { + return Err(vm.new_runtime_error(format!( + "Unrecognized action ({action_str}) in warnings.filters:\n {action_str}" + ))); + } + } + } else { + false + }; + + if already { + return Ok(()); + } + + call_show_warning( + category, + text, + message, + filename, + lineno, + lineno_obj, + source_line, + source, + vm, + ) +} + +#[allow(clippy::too_many_arguments)] +fn call_show_warning( + category: PyTypeRef, + text: PyStrRef, + message: PyObjectRef, + filename: PyStrRef, + lineno: usize, + lineno_obj: PyObjectRef, + source_line: Option<PyObjectRef>, + source: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + let Some(show_fn) = + get_warnings_attr(vm, identifier!(&vm.ctx, _showwarnmsg), source.is_some())? + else { + return show_warning(filename, lineno, text, category, source_line, vm); + }; + if !show_fn.is_callable() { + return Err(vm.new_type_error("warnings._showwarnmsg() must be set to a callable")); + } + let Some(warnmsg_cls) = get_warnings_attr(vm, identifier!(&vm.ctx, WarningMessage), false)? + else { + return Err(vm.new_runtime_error("unable to get warnings.WarningMessage")); + }; + + let msg = warnmsg_cls.call( + vec![ + message, + category.into(), + filename.into(), + lineno_obj, + vm.ctx.none(), + vm.ctx.none(), + vm.unwrap_or_none(source), + ], + vm, + )?; + show_fn.call((msg,), vm)?; + Ok(()) +} + +fn show_warning( + filename: PyStrRef, + lineno: usize, + text: PyStrRef, + category: PyTypeRef, + _source_line: Option<PyObjectRef>, + vm: &VirtualMachine, +) -> PyResult<()> { + let stderr = crate::stdlib::sys::PyStderr(vm); + writeln!( + stderr, + "{}:{}: {}: {}", + filename, + lineno, + category.name(), + text + ); + Ok(()) +} + +/// Check if a frame's filename starts with any of the given prefixes. +fn is_filename_to_skip(frame: &crate::frame::Frame, prefixes: &PyTupleRef) -> bool { + let filename = frame.f_code().co_filename(); + let filename_bytes = filename.as_bytes(); + prefixes.iter().any(|prefix| { + prefix + .downcast_ref::<PyStr>() + .is_some_and(|s| filename_bytes.starts_with(s.as_bytes())) + }) +} + +/// Like Frame::next_external_frame but also skips frames matching prefixes. +fn next_external_frame_with_skip( + frame: &crate::frame::FrameRef, + skip_file_prefixes: Option<&PyTupleRef>, + vm: &VirtualMachine, +) -> Option<crate::frame::FrameRef> { + let mut f = frame.f_back(vm); + loop { + let current: crate::frame::FrameRef = f.take()?; + if current.is_internal_frame() + || skip_file_prefixes.is_some_and(|p| is_filename_to_skip(&current, p)) + { + f = current.f_back(vm); + } else { + return Some(current); + } + } +} + +/// filename, module, and registry are new refs, globals is borrowed +/// Returns `Ok` on success, or `Err` on error (no new refs) +fn setup_context( + mut stack_level: isize, + skip_file_prefixes: Option<&PyTupleRef>, + vm: &VirtualMachine, +) -> PyResult<(PyStrRef, usize, Option<PyObjectRef>, PyObjectRef)> { + let mut f = vm.current_frame(); + + // Stack level comparisons to Python code is off by one as there is no + // warnings-related stack level to avoid. + if stack_level <= 0 || f.as_ref().is_some_and(|frame| frame.is_internal_frame()) { + while { + stack_level -= 1; + stack_level > 0 + } { + match f { + Some(tmp) => f = tmp.f_back(vm), + None => break, + } + } + } else { + while { + stack_level -= 1; + stack_level > 0 + } { + match f { + Some(tmp) => f = next_external_frame_with_skip(&tmp, skip_file_prefixes, vm), + None => break, + } + } + } + + let (globals, filename, lineno) = if let Some(f) = f { + (f.globals.clone(), f.code.source_path(), f.f_lineno()) + } else if let Some(frame) = vm.current_frame() { + // We have a frame but it wasn't found during stack walking + (frame.globals.clone(), vm.ctx.intern_str("<sys>"), 1) + } else { + // No frames on the stack - use sys.__dict__ (interp->sysdict) + let globals = vm + .sys_module + .as_object() + .get_attr(identifier!(vm, __dict__), vm) + .and_then(|d| { + d.downcast::<crate::builtins::PyDict>() + .map_err(|_| vm.new_type_error("sys.__dict__ is not a dictionary")) + })?; + (globals, vm.ctx.intern_str("<sys>"), 0) + }; + + let registry = match globals.get_item("__warningregistry__", vm) { + Ok(r) => r, + Err(_) => { + let r = vm.ctx.new_dict(); + globals.set_item("__warningregistry__", r.clone().into(), vm)?; + r.into() + } + }; + + // Setup module. + let module = globals + .get_item("__name__", vm) + .unwrap_or_else(|_| vm.new_pyobj("<string>")); + Ok((filename.to_owned(), lineno, Some(module), registry)) +} diff --git a/crates/vm/src/windows.rs b/crates/vm/src/windows.rs new file mode 100644 index 00000000000..ec9ac8f18fb --- /dev/null +++ b/crates/vm/src/windows.rs @@ -0,0 +1,541 @@ +use crate::common::fileutils::{ + StatStruct, + windows::{FILE_INFO_BY_NAME_CLASS, get_file_information_by_name}, +}; +use crate::{ + PyObjectRef, PyResult, TryFromObject, VirtualMachine, + convert::{ToPyObject, ToPyResult}, +}; +use rustpython_common::windows::ToWideString; +use std::ffi::OsStr; +use windows_sys::Win32::Foundation::{HANDLE, INVALID_HANDLE_VALUE}; + +/// Windows HANDLE wrapper for Python interop +#[derive(Clone, Copy)] +pub struct WinHandle(pub HANDLE); + +pub(crate) trait WindowsSysResultValue { + type Ok: ToPyObject; + fn is_err(&self) -> bool; + fn into_ok(self) -> Self::Ok; +} + +impl WindowsSysResultValue for HANDLE { + type Ok = WinHandle; + fn is_err(&self) -> bool { + *self == INVALID_HANDLE_VALUE + } + fn into_ok(self) -> Self::Ok { + WinHandle(self) + } +} + +// BOOL is i32 in windows-sys 0.61+ +impl WindowsSysResultValue for i32 { + type Ok = (); + fn is_err(&self) -> bool { + *self == 0 + } + fn into_ok(self) -> Self::Ok {} +} + +pub(crate) struct WindowsSysResult<T>(pub T); + +impl<T: WindowsSysResultValue> WindowsSysResult<T> { + pub fn is_err(&self) -> bool { + self.0.is_err() + } + pub fn into_pyresult(self, vm: &VirtualMachine) -> PyResult<T::Ok> { + if !self.is_err() { + Ok(self.0.into_ok()) + } else { + Err(vm.new_last_os_error()) + } + } +} + +impl<T: WindowsSysResultValue> ToPyResult for WindowsSysResult<T> { + fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { + let ok = self.into_pyresult(vm)?; + Ok(ok.to_pyobject(vm)) + } +} + +type HandleInt = isize; + +impl TryFromObject for WinHandle { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { + let handle = HandleInt::try_from_object(vm, obj)?; + Ok(WinHandle(handle as HANDLE)) + } +} + +impl ToPyObject for WinHandle { + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + (self.0 as HandleInt).to_pyobject(vm) + } +} + +pub fn init_winsock() { + static WSA_INIT: parking_lot::Once = parking_lot::Once::new(); + WSA_INIT.call_once(|| unsafe { + let mut wsa_data = core::mem::MaybeUninit::uninit(); + let _ = windows_sys::Win32::Networking::WinSock::WSAStartup(0x0101, wsa_data.as_mut_ptr()); + }) +} + +// win32_xstat in cpython +pub fn win32_xstat(path: &OsStr, traverse: bool) -> std::io::Result<StatStruct> { + let mut result = win32_xstat_impl(path, traverse)?; + // ctime is only deprecated from 3.12, so we copy birthtime across + result.st_ctime = result.st_birthtime; + result.st_ctime_nsec = result.st_birthtime_nsec; + Ok(result) +} + +fn is_reparse_tag_name_surrogate(tag: u32) -> bool { + (tag & 0x20000000) > 0 +} + +// Constants +const IO_REPARSE_TAG_SYMLINK: u32 = 0xA000000C; +const S_IFMT: u16 = libc::S_IFMT as u16; +const S_IFDIR: u16 = libc::S_IFDIR as u16; +const S_IFREG: u16 = libc::S_IFREG as u16; +const S_IFCHR: u16 = libc::S_IFCHR as u16; +const S_IFLNK: u16 = crate::common::fileutils::windows::S_IFLNK as u16; +const S_IFIFO: u16 = crate::common::fileutils::windows::S_IFIFO as u16; + +/// FILE_ATTRIBUTE_TAG_INFO structure for GetFileInformationByHandleEx +#[repr(C)] +#[derive(Default)] +struct FileAttributeTagInfo { + file_attributes: u32, + reparse_tag: u32, +} + +/// Ported from attributes_to_mode (fileutils.c) +fn attributes_to_mode(attr: u32) -> u16 { + use windows_sys::Win32::Storage::FileSystem::{ + FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_READONLY, + }; + let mut m: u16 = 0; + if attr & FILE_ATTRIBUTE_DIRECTORY != 0 { + m |= S_IFDIR | 0o111; // IFEXEC for user,group,other + } else { + m |= S_IFREG; + } + if attr & FILE_ATTRIBUTE_READONLY != 0 { + m |= 0o444; + } else { + m |= 0o666; + } + m +} + +/// Ported from _Py_attribute_data_to_stat (fileutils.c) +/// Converts BY_HANDLE_FILE_INFORMATION to StatStruct +fn attribute_data_to_stat( + info: &windows_sys::Win32::Storage::FileSystem::BY_HANDLE_FILE_INFORMATION, + reparse_tag: u32, + basic_info: Option<&windows_sys::Win32::Storage::FileSystem::FILE_BASIC_INFO>, + id_info: Option<&windows_sys::Win32::Storage::FileSystem::FILE_ID_INFO>, +) -> StatStruct { + use crate::common::fileutils::windows::SECS_BETWEEN_EPOCHS; + use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT; + + let mut st_mode = attributes_to_mode(info.dwFileAttributes); + let st_size = ((info.nFileSizeHigh as u64) << 32) | (info.nFileSizeLow as u64); + let st_dev = id_info + .map(|id| id.VolumeSerialNumber as u32) + .unwrap_or(info.dwVolumeSerialNumber); + let st_nlink = info.nNumberOfLinks as i32; + + // Convert FILETIME/LARGE_INTEGER to (time_t, nsec) + let filetime_to_time = |ft_low: u32, ft_high: u32| -> (libc::time_t, i32) { + let ticks = ((ft_high as i64) << 32) | (ft_low as i64); + let nsec = ((ticks % 10_000_000) * 100) as i32; + let sec = (ticks / 10_000_000 - SECS_BETWEEN_EPOCHS) as libc::time_t; + (sec, nsec) + }; + + let large_integer_to_time = |li: i64| -> (libc::time_t, i32) { + let nsec = ((li % 10_000_000) * 100) as i32; + let sec = (li / 10_000_000 - SECS_BETWEEN_EPOCHS) as libc::time_t; + (sec, nsec) + }; + + let (st_birthtime, st_birthtime_nsec); + let (st_mtime, st_mtime_nsec); + let (st_atime, st_atime_nsec); + + if let Some(bi) = basic_info { + (st_birthtime, st_birthtime_nsec) = large_integer_to_time(bi.CreationTime); + (st_mtime, st_mtime_nsec) = large_integer_to_time(bi.LastWriteTime); + (st_atime, st_atime_nsec) = large_integer_to_time(bi.LastAccessTime); + } else { + (st_birthtime, st_birthtime_nsec) = filetime_to_time( + info.ftCreationTime.dwLowDateTime, + info.ftCreationTime.dwHighDateTime, + ); + (st_mtime, st_mtime_nsec) = filetime_to_time( + info.ftLastWriteTime.dwLowDateTime, + info.ftLastWriteTime.dwHighDateTime, + ); + (st_atime, st_atime_nsec) = filetime_to_time( + info.ftLastAccessTime.dwLowDateTime, + info.ftLastAccessTime.dwHighDateTime, + ); + } + + // Get file ID from id_info or fallback to file index + let (st_ino, st_ino_high) = if let Some(id) = id_info { + // FILE_ID_INFO.FileId is FILE_ID_128 which is [u8; 16] + let bytes = id.FileId.Identifier; + let low = u64::from_le_bytes(bytes[0..8].try_into().unwrap()); + let high = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); + (low, high) + } else { + let ino = ((info.nFileIndexHigh as u64) << 32) | (info.nFileIndexLow as u64); + (ino, 0u64) + }; + + // Set symlink mode if applicable + if info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + && reparse_tag == IO_REPARSE_TAG_SYMLINK + { + st_mode = (st_mode & !S_IFMT) | S_IFLNK; + } + + StatStruct { + st_dev, + st_ino, + st_ino_high, + st_mode, + st_nlink, + st_uid: 0, + st_gid: 0, + st_rdev: 0, + st_size, + st_atime, + st_atime_nsec, + st_mtime, + st_mtime_nsec, + st_ctime: 0, // Will be set by caller + st_ctime_nsec: 0, + st_birthtime, + st_birthtime_nsec, + st_file_attributes: info.dwFileAttributes, + st_reparse_tag: reparse_tag, + } +} + +/// Get file info using FindFirstFileW (fallback when CreateFileW fails) +/// Ported from attributes_from_dir +fn attributes_from_dir( + path: &OsStr, +) -> std::io::Result<( + windows_sys::Win32::Storage::FileSystem::BY_HANDLE_FILE_INFORMATION, + u32, +)> { + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_REPARSE_POINT, FindClose, FindFirstFileW, + WIN32_FIND_DATAW, + }; + + let wide: Vec<u16> = path.to_wide_with_nul(); + let mut find_data: WIN32_FIND_DATAW = unsafe { core::mem::zeroed() }; + + let handle = unsafe { FindFirstFileW(wide.as_ptr(), &mut find_data) }; + if handle == INVALID_HANDLE_VALUE { + return Err(std::io::Error::last_os_error()); + } + unsafe { FindClose(handle) }; + + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { core::mem::zeroed() }; + info.dwFileAttributes = find_data.dwFileAttributes; + info.ftCreationTime = find_data.ftCreationTime; + info.ftLastAccessTime = find_data.ftLastAccessTime; + info.ftLastWriteTime = find_data.ftLastWriteTime; + info.nFileSizeHigh = find_data.nFileSizeHigh; + info.nFileSizeLow = find_data.nFileSizeLow; + + let reparse_tag = if find_data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + find_data.dwReserved0 + } else { + 0 + }; + + Ok((info, reparse_tag)) +} + +/// Ported from win32_xstat_slow_impl +fn win32_xstat_slow_impl(path: &OsStr, traverse: bool) -> std::io::Result<StatStruct> { + use windows_sys::Win32::{ + Foundation::{ + CloseHandle, ERROR_ACCESS_DENIED, ERROR_CANT_ACCESS_FILE, ERROR_INVALID_FUNCTION, + ERROR_INVALID_PARAMETER, ERROR_NOT_SUPPORTED, ERROR_SHARING_VIOLATION, GENERIC_READ, + INVALID_HANDLE_VALUE, + }, + Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, CreateFileW, FILE_ATTRIBUTE_DIRECTORY, + FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_REPARSE_POINT, FILE_BASIC_INFO, + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_ID_INFO, + FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_TYPE_CHAR, + FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_UNKNOWN, FileAttributeTagInfo, FileBasicInfo, + FileIdInfo, GetFileAttributesW, GetFileInformationByHandle, + GetFileInformationByHandleEx, GetFileType, INVALID_FILE_ATTRIBUTES, OPEN_EXISTING, + }, + }; + + let wide: Vec<u16> = path.to_wide_with_nul(); + + let access = FILE_READ_ATTRIBUTES; + let mut flags = FILE_FLAG_BACKUP_SEMANTICS; + if !traverse { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + + let mut h_file = unsafe { + CreateFileW( + wide.as_ptr(), + access, + 0, + core::ptr::null(), + OPEN_EXISTING, + flags, + core::ptr::null_mut(), + ) + }; + + let mut file_info: BY_HANDLE_FILE_INFORMATION = unsafe { core::mem::zeroed() }; + let mut tag_info = FileAttributeTagInfo::default(); + let mut is_unhandled_tag = false; + + if h_file == INVALID_HANDLE_VALUE { + let error = std::io::Error::last_os_error(); + let error_code = error.raw_os_error().unwrap_or(0) as u32; + + match error_code { + ERROR_ACCESS_DENIED | ERROR_SHARING_VIOLATION => { + // Try reading the parent directory using FindFirstFileW + let (info, reparse_tag) = attributes_from_dir(path)?; + file_info = info; + tag_info.reparse_tag = reparse_tag; + + if file_info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + && (traverse || !is_reparse_tag_name_surrogate(tag_info.reparse_tag)) + { + return Err(error); + } + // h_file remains INVALID_HANDLE_VALUE, we'll use file_info from FindFirstFileW + } + ERROR_INVALID_PARAMETER => { + // Retry with GENERIC_READ (needed for \\.\con) + h_file = unsafe { + CreateFileW( + wide.as_ptr(), + access | GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE, + core::ptr::null(), + OPEN_EXISTING, + flags, + core::ptr::null_mut(), + ) + }; + if h_file == INVALID_HANDLE_VALUE { + return Err(error); + } + } + ERROR_CANT_ACCESS_FILE if traverse => { + // bpo37834: open unhandled reparse points if traverse fails + is_unhandled_tag = true; + h_file = unsafe { + CreateFileW( + wide.as_ptr(), + access, + 0, + core::ptr::null(), + OPEN_EXISTING, + flags | FILE_FLAG_OPEN_REPARSE_POINT, + core::ptr::null_mut(), + ) + }; + if h_file == INVALID_HANDLE_VALUE { + return Err(error); + } + } + _ => return Err(error), + } + } + + // Scope for handle cleanup + let result = (|| -> std::io::Result<StatStruct> { + if h_file != INVALID_HANDLE_VALUE { + // Handle types other than files on disk + let file_type = unsafe { GetFileType(h_file) }; + if file_type != FILE_TYPE_DISK { + if file_type == FILE_TYPE_UNKNOWN { + let err = std::io::Error::last_os_error(); + if err.raw_os_error().unwrap_or(0) != 0 { + return Err(err); + } + } + let file_attributes = unsafe { GetFileAttributesW(wide.as_ptr()) }; + let mut st_mode: u16 = 0; + if file_attributes != INVALID_FILE_ATTRIBUTES + && file_attributes & FILE_ATTRIBUTE_DIRECTORY != 0 + { + st_mode = S_IFDIR; + } else if file_type == FILE_TYPE_CHAR { + st_mode = S_IFCHR; + } else if file_type == FILE_TYPE_PIPE { + st_mode = S_IFIFO; + } + return Ok(StatStruct { + st_mode, + ..Default::default() + }); + } + + // Query the reparse tag + if !traverse || is_unhandled_tag { + let mut local_tag_info: FileAttributeTagInfo = unsafe { core::mem::zeroed() }; + let ret = unsafe { + GetFileInformationByHandleEx( + h_file, + FileAttributeTagInfo, + &mut local_tag_info as *mut _ as *mut _, + core::mem::size_of::<FileAttributeTagInfo>() as u32, + ) + }; + if ret == 0 { + let err_code = + std::io::Error::last_os_error().raw_os_error().unwrap_or(0) as u32; + match err_code { + ERROR_INVALID_PARAMETER | ERROR_INVALID_FUNCTION | ERROR_NOT_SUPPORTED => { + local_tag_info.file_attributes = FILE_ATTRIBUTE_NORMAL; + local_tag_info.reparse_tag = 0; + } + _ => return Err(std::io::Error::last_os_error()), + } + } else if local_tag_info.file_attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + if is_reparse_tag_name_surrogate(local_tag_info.reparse_tag) { + if is_unhandled_tag { + return Err(std::io::Error::from_raw_os_error( + ERROR_CANT_ACCESS_FILE as i32, + )); + } + // This is a symlink, keep the tag info + } else if !is_unhandled_tag { + // Traverse a non-link reparse point + unsafe { CloseHandle(h_file) }; + return win32_xstat_slow_impl(path, true); + } + } + tag_info = local_tag_info; + } + + // Get file information + let ret = unsafe { GetFileInformationByHandle(h_file, &mut file_info) }; + if ret == 0 { + let err_code = std::io::Error::last_os_error().raw_os_error().unwrap_or(0) as u32; + match err_code { + ERROR_INVALID_PARAMETER | ERROR_INVALID_FUNCTION | ERROR_NOT_SUPPORTED => { + // Volumes and physical disks are block devices + return Ok(StatStruct { + st_mode: 0x6000, // S_IFBLK + ..Default::default() + }); + } + _ => return Err(std::io::Error::last_os_error()), + } + } + + // Get FILE_BASIC_INFO + let mut basic_info: FILE_BASIC_INFO = unsafe { core::mem::zeroed() }; + let has_basic_info = unsafe { + GetFileInformationByHandleEx( + h_file, + FileBasicInfo, + &mut basic_info as *mut _ as *mut _, + core::mem::size_of::<FILE_BASIC_INFO>() as u32, + ) + } != 0; + + // Get FILE_ID_INFO (optional) + let mut id_info: FILE_ID_INFO = unsafe { core::mem::zeroed() }; + let has_id_info = unsafe { + GetFileInformationByHandleEx( + h_file, + FileIdInfo, + &mut id_info as *mut _ as *mut _, + core::mem::size_of::<FILE_ID_INFO>() as u32, + ) + } != 0; + + let mut result = attribute_data_to_stat( + &file_info, + tag_info.reparse_tag, + if has_basic_info { + Some(&basic_info) + } else { + None + }, + if has_id_info { Some(&id_info) } else { None }, + ); + result.update_st_mode_from_path(path, file_info.dwFileAttributes); + Ok(result) + } else { + // We got file_info from attributes_from_dir + let mut result = attribute_data_to_stat(&file_info, tag_info.reparse_tag, None, None); + result.update_st_mode_from_path(path, file_info.dwFileAttributes); + Ok(result) + } + })(); + + // Cleanup + if h_file != INVALID_HANDLE_VALUE { + unsafe { CloseHandle(h_file) }; + } + + result +} + +fn win32_xstat_impl(path: &OsStr, traverse: bool) -> std::io::Result<StatStruct> { + use windows_sys::Win32::{Foundation, Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT}; + + let stat_info = + get_file_information_by_name(path, FILE_INFO_BY_NAME_CLASS::FileStatBasicByNameInfo); + match stat_info { + Ok(stat_info) => { + if (stat_info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT == 0) + || (!traverse && is_reparse_tag_name_surrogate(stat_info.ReparseTag)) + { + let mut result = + crate::common::fileutils::windows::stat_basic_info_to_stat(&stat_info); + // If st_ino is 0, fall through to slow path to get proper file ID + if result.st_ino != 0 || result.st_ino_high != 0 { + result.update_st_mode_from_path(path, stat_info.FileAttributes); + return Ok(result); + } + } + } + Err(e) => { + if let Some(errno) = e.raw_os_error() + && matches!( + errno as u32, + Foundation::ERROR_FILE_NOT_FOUND + | Foundation::ERROR_PATH_NOT_FOUND + | Foundation::ERROR_NOT_READY + | Foundation::ERROR_BAD_NET_NAME + ) + { + return Err(e); + } + } + } + + // Fallback to slow implementation + win32_xstat_slow_impl(path, traverse) +} diff --git a/crates/wasm/.cargo/config.toml b/crates/wasm/.cargo/config.toml new file mode 100644 index 00000000000..f4e8c002fc2 --- /dev/null +++ b/crates/wasm/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-unknown-unknown" diff --git a/crates/wasm/Cargo.toml b/crates/wasm/Cargo.toml new file mode 100644 index 00000000000..a2bb1a9f948 --- /dev/null +++ b/crates/wasm/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "rustpython_wasm" +description = "A Python-3 (CPython >= 3.5.0) Interpreter written in Rust, compiled to WASM" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[lib] +crate-type = ["cdylib", "rlib"] + +[features] +default = ["freeze-stdlib"] +freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib/freeze-stdlib", "rustpython-stdlib"] +no-start-func = [] + +[dependencies] +rustpython-common = { workspace = true } +rustpython-pylib = { workspace = true, optional = true } +rustpython-stdlib = { workspace = true, default-features = false, optional = true } +# make sure no threading! otherwise wasm build will fail +rustpython-vm = { workspace = true, features = ["compiler", "encodings", "serde", "wasmbind"] } + +ruff_python_parser = { workspace = true } + +serde = { workspace = true } +wasm-bindgen = { workspace = true } + +console_error_panic_hook = "0.1" +js-sys = "0.3" +serde-wasm-bindgen = "0.6.5" +wasm-bindgen-futures = "0.4" +web-sys = { version = "0.3", features = [ + "console", + "Document", + "Element", + "Window", + "Headers", + "Request", + "RequestInit", + "Response" +] } + +[package.metadata.wasm-pack.profile.release] +wasm-opt = false#["-O1"] + +[lints] +workspace = true diff --git a/wasm/lib/Lib/_microdistlib.py b/crates/wasm/Lib/_microdistlib.py similarity index 100% rename from wasm/lib/Lib/_microdistlib.py rename to crates/wasm/Lib/_microdistlib.py diff --git a/wasm/lib/Lib/asyncweb.py b/crates/wasm/Lib/asyncweb.py similarity index 100% rename from wasm/lib/Lib/asyncweb.py rename to crates/wasm/Lib/asyncweb.py diff --git a/wasm/lib/Lib/browser.py b/crates/wasm/Lib/browser.py similarity index 100% rename from wasm/lib/Lib/browser.py rename to crates/wasm/Lib/browser.py diff --git a/wasm/lib/Lib/whlimport.py b/crates/wasm/Lib/whlimport.py similarity index 100% rename from wasm/lib/Lib/whlimport.py rename to crates/wasm/Lib/whlimport.py diff --git a/crates/wasm/README.md b/crates/wasm/README.md new file mode 100644 index 00000000000..3a755009205 --- /dev/null +++ b/crates/wasm/README.md @@ -0,0 +1,40 @@ +# RustPython + +A Python-3 (CPython >= 3.8.0) Interpreter written in Rust. + +[![Build Status](https://travis-ci.org/RustPython/RustPython.svg?branch=main)](https://travis-ci.org/RustPython/RustPython) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) + +# WARNING: this project is still in a pre-alpha state! + +**Using this in a production project is inadvisable. Please only do so if you understand the risks.** + +## Usage + +#### Check out our [online demo](https://rustpython.github.io/demo/) running on WebAssembly. + +## Goals + +- Full Python-3 environment entirely in Rust (not CPython bindings) +- A clean implementation without compatibility hacks + +## Quick Documentation + +```js +pyEval(code, options?); +``` + +`code`: `string`: The Python code to run + +`options`: + +- `vars?`: `{ [key: string]: any }`: Variables passed to the VM that can be + accessed in Python with the variable `js_vars`. Functions do work, and + receive the Python kwargs as the `this` argument. +- `stdout?`: `"console" | ((out: string) => void) | null`: A function to replace the + native print function, and it will be `console.log` when giving `undefined` + or "console", and it will be a dumb function when giving null. + +## License + +This project is licensed under the MIT license. diff --git a/wasm/lib/src/browser_module.rs b/crates/wasm/src/browser_module.rs similarity index 84% rename from wasm/lib/src/browser_module.rs rename to crates/wasm/src/browser_module.rs index 32e7877a537..59ca55f45c7 100644 --- a/wasm/lib/src/browser_module.rs +++ b/crates/wasm/src/browser_module.rs @@ -1,20 +1,18 @@ -use rustpython_vm::VirtualMachine; - -pub(crate) use _browser::make_module; +pub(crate) use _browser::module_def; #[pymodule] mod _browser { use crate::{convert, js_module::PyPromise, vm_class::weak_vm, wasm_builtins::window}; use js_sys::Promise; use rustpython_vm::{ + PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyDictRef, PyStrRef}, class::PyClassImpl, convert::ToPyObject, function::{ArgCallable, OptionalArg}, - import::import_file, - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + import::import_source, }; - use wasm_bindgen::{prelude::*, JsCast}; + use wasm_bindgen::{JsCast, prelude::*}; use wasm_bindgen_futures::JsFuture; enum FetchResponseFormat { @@ -29,7 +27,7 @@ mod _browser { "json" => Ok(FetchResponseFormat::Json), "text" => Ok(FetchResponseFormat::Text), "array_buffer" => Ok(FetchResponseFormat::ArrayBuffer), - _ => Err(vm.new_type_error("Unknown fetch response_format".into())), + _ => Err(vm.new_type_error("Unknown fetch response_format")), } } fn get_response(&self, response: &web_sys::Response) -> Result<Promise, JsValue> { @@ -66,22 +64,19 @@ mod _browser { } = args; let response_format = match response_format { - Some(s) => FetchResponseFormat::from_str(vm, s.as_str())?, + Some(s) => FetchResponseFormat::from_str(vm, s.expect_str())?, None => FetchResponseFormat::Text, }; - let mut opts = web_sys::RequestInit::new(); + let opts = web_sys::RequestInit::new(); - match method { - Some(s) => opts.method(s.as_str()), - None => opts.method("GET"), - }; + opts.set_method(method.as_ref().map_or("GET", |s| s.expect_str())); if let Some(body) = body { - opts.body(Some(&convert::py_to_js(vm, body))); + opts.set_body(&convert::py_to_js(vm, body)); } - let request = web_sys::Request::new_with_str_and_init(url.as_str(), &opts) + let request = web_sys::Request::new_with_str_and_init(url.expect_str(), &opts) .map_err(|err| convert::js_py_typeerror(vm, err))?; if let Some(headers) = headers { @@ -89,7 +84,7 @@ mod _browser { for (key, value) in headers { let key = key.str(vm)?; let value = value.str(vm)?; - h.set(key.as_str(), value.as_str()) + h.set(key.expect_str(), value.expect_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } } @@ -97,7 +92,7 @@ mod _browser { if let Some(content_type) = content_type { request .headers() - .set("Content-Type", content_type.as_str()) + .set("Content-Type", content_type.expect_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } @@ -117,7 +112,8 @@ mod _browser { #[pyfunction] fn request_animation_frame(func: ArgCallable, vm: &VirtualMachine) -> PyResult { - use std::{cell::RefCell, rc::Rc}; + use alloc::rc::Rc; + use core::cell::RefCell; // this basic setup for request_animation_frame taken from: // https://rustwasm.github.io/wasm-bindgen/examples/request-animation-frame.html @@ -172,7 +168,7 @@ mod _browser { fn query(&self, query: PyStrRef, vm: &VirtualMachine) -> PyResult { let elem = self .doc - .query_selector(query.as_str()) + .query_selector(query.expect_str()) .map_err(|err| convert::js_py_typeerror(vm, err))? .map(|elem| Element { elem }) .to_pyobject(vm); @@ -181,12 +177,12 @@ mod _browser { } #[pyattr] - fn document(vm: &VirtualMachine) -> PyRef<Document> { + fn document(_vm: &VirtualMachine) -> PyRef<Document> { PyRef::new_ref( Document { doc: window().document().expect("Document missing from window"), }, - Document::make_class(&vm.ctx), + Document::make_static_type(), None, ) } @@ -207,7 +203,7 @@ mod _browser { default: OptionalArg<PyObjectRef>, vm: &VirtualMachine, ) -> PyObjectRef { - match self.elem.get_attribute(attr.as_str()) { + match self.elem.get_attribute(attr.expect_str()) { Some(s) => vm.ctx.new_str(s).into(), None => default.unwrap_or_none(vm), } @@ -216,7 +212,7 @@ mod _browser { #[pymethod] fn set_attr(&self, attr: PyStrRef, value: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { self.elem - .set_attribute(attr.as_str(), value.as_str()) + .set_attribute(attr.expect_str(), value.expect_str()) .map_err(|err| convert::js_py_typeerror(vm, err)) } } @@ -225,10 +221,10 @@ mod _browser { fn load_module(module: PyStrRef, path: PyStrRef, vm: &VirtualMachine) -> PyResult { let weak_vm = weak_vm(vm); - let mut opts = web_sys::RequestInit::new(); - opts.method("GET"); + let opts = web_sys::RequestInit::new(); + opts.set_method("GET"); - let request = web_sys::Request::new_with_str_and_init(path.as_str(), &opts) + let request = web_sys::Request::new_with_str_and_init(path.expect_str(), &opts) .map_err(|err| convert::js_py_typeerror(vm, err))?; let window = window(); @@ -245,7 +241,7 @@ mod _browser { .expect("that the vm is valid when the promise resolves"); stored_vm.interp.enter(move |vm| { let resp_text = text.as_string().unwrap(); - let res = import_file(vm, module.as_str(), "WEB".to_owned(), resp_text); + let res = import_source(vm, module.expect_str(), &resp_text); match res { Ok(_) => Ok(JsValue::null()), Err(err) => Err(convert::py_err_to_js_err(vm, &err)), @@ -256,8 +252,3 @@ mod _browser { Ok(PyPromise::from_future(future).into_pyobject(vm)) } } - -pub fn setup_browser_module(vm: &mut VirtualMachine) { - vm.add_native_module("_browser".to_owned(), Box::new(make_module)); - vm.add_frozen(py_freeze!(dir = "Lib")); -} diff --git a/crates/wasm/src/convert.rs b/crates/wasm/src/convert.rs new file mode 100644 index 00000000000..bbf263975f3 --- /dev/null +++ b/crates/wasm/src/convert.rs @@ -0,0 +1,289 @@ +#![allow(clippy::empty_docs)] // TODO: remove it later. false positive by wasm-bindgen generated code + +use crate::js_module; +use crate::vm_class::{WASMVirtualMachine, stored_vm_from_wasm}; +use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array}; +use rustpython_vm::{ + AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, + builtins::{PyBaseException, PyBaseExceptionRef}, + compiler::{CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType}, + exceptions, + function::{ArgBytesLike, FuncArgs}, + py_serde, +}; +use wasm_bindgen::{JsCast, closure::Closure, prelude::*}; + +#[wasm_bindgen(inline_js = r" +export class PyError extends Error { + constructor(info) { + const msg = info.args[0]; + if (typeof msg === 'string') super(msg); + else super(); + this.info = info; + } + get name() { return this.info.exc_type; } + get traceback() { return this.info.traceback; } + toString() { return this.info.rendered; } +} +")] +extern "C" { + pub type PyError; + #[wasm_bindgen(constructor)] + fn new(info: JsValue) -> PyError; +} + +pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &Py<PyBaseException>) -> JsValue { + let js_err = vm.try_class("_js", "JSError").ok(); + let js_arg = if js_err.is_some_and(|js_err| py_err.fast_isinstance(&js_err)) { + py_err.get_arg(0) + } else { + None + }; + let js_arg = js_arg + .as_ref() + .and_then(|x| x.downcast_ref::<js_module::PyJsValue>()); + match js_arg { + Some(val) => val.value.clone(), + None => { + let res = + serde_wasm_bindgen::to_value(&exceptions::SerializeException::new(vm, py_err)); + match res { + Ok(err_info) => PyError::new(err_info).into(), + Err(_) => { + // Fallback: create a basic JS Error with the exception type and message + let exc_type = py_err.class().name().to_string(); + let msg = match py_err.as_object().str(vm) { + Ok(s) => format!("{exc_type}: {s}"), + Err(_) => exc_type, + }; + js_sys::Error::new(&msg).into() + } + } + } + } +} + +pub fn js_py_typeerror(vm: &VirtualMachine, js_err: JsValue) -> PyBaseExceptionRef { + let msg: String = js_err.unchecked_into::<js_sys::Error>().to_string().into(); + vm.new_type_error(msg) +} + +pub fn js_err_to_py_err(vm: &VirtualMachine, js_err: &JsValue) -> PyBaseExceptionRef { + match js_err.dyn_ref::<js_sys::Error>() { + Some(err) => { + let exc_type = match String::from(err.name()).as_str() { + "TypeError" => vm.ctx.exceptions.type_error, + "ReferenceError" => vm.ctx.exceptions.name_error, + "SyntaxError" => vm.ctx.exceptions.syntax_error, + _ => vm.ctx.exceptions.exception_type, + } + .to_owned(); + vm.new_exception_msg(exc_type, String::from(err.message()).into()) + } + None => vm.new_exception_msg( + vm.ctx.exceptions.exception_type.to_owned(), + format!("{js_err:?}").into(), + ), + } +} + +pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { + if let Some(ref wasm_id) = vm.wasm_id + && py_obj.fast_isinstance(vm.ctx.types.function_type) + { + let wasm_vm = WASMVirtualMachine { + id: wasm_id.clone(), + }; + let weak_py_obj = wasm_vm.push_held_rc(py_obj).unwrap().unwrap(); + + let closure = move |args: Option<Box<[JsValue]>>, + kwargs: Option<Object>| + -> Result<JsValue, JsValue> { + let py_obj = match wasm_vm.assert_valid() { + Ok(_) => weak_py_obj + .upgrade() + .expect("weak_py_obj to be valid if VM is valid"), + Err(err) => { + return Err(err); + } + }; + stored_vm_from_wasm(&wasm_vm).interp.enter(move |vm| { + let args = match args { + Some(args) => Vec::from(args) + .into_iter() + .map(|arg| js_to_py(vm, arg)) + .collect::<Vec<_>>(), + None => Vec::new(), + }; + let mut py_func_args = FuncArgs::from(args); + if let Some(ref kwargs) = kwargs { + for pair in object_entries(kwargs) { + let (key, val) = pair?; + py_func_args + .kwargs + .insert(js_sys::JsString::from(key).into(), js_to_py(vm, val)); + } + } + let result = py_obj.call(py_func_args, vm); + pyresult_to_js_result(vm, result) + }) + }; + let closure = Closure::wrap(Box::new(closure) + as Box< + dyn FnMut(Option<Box<[JsValue]>>, Option<Object>) -> Result<JsValue, JsValue>, + >); + let func = closure.as_ref().clone(); + + // stores pretty much nothing, it's fine to leak this because if it gets dropped + // the error message is worse + closure.forget(); + + return func; + } + // the browser module might not be injected + if vm.try_class("_js", "Promise").is_ok() + && let Some(py_prom) = py_obj.downcast_ref::<js_module::PyPromise>() + { + return py_prom.as_js(vm).into(); + } + + if let Ok(bytes) = ArgBytesLike::try_from_borrowed_object(vm, &py_obj) { + bytes.with_ref(|bytes| unsafe { + // `Uint8Array::view` is an `unsafe fn` because it provides + // a direct view into the WASM linear memory; if you were to allocate + // something with Rust that view would probably become invalid. It's safe + // because we then copy the array using `Uint8Array::slice`. + let view = Uint8Array::view(bytes); + view.slice(0, bytes.len() as u32).into() + }) + } else { + py_serde::serialize(vm, &py_obj, &serde_wasm_bindgen::Serializer::new()) + .unwrap_or(JsValue::UNDEFINED) + } +} + +pub fn object_entries(obj: &Object) -> impl Iterator<Item = Result<(JsValue, JsValue), JsValue>> { + Object::entries(obj).values().into_iter().map(|pair| { + pair.map(|pair| { + let key = Reflect::get(&pair, &"0".into()).unwrap(); + let val = Reflect::get(&pair, &"1".into()).unwrap(); + (key, val) + }) + }) +} + +pub fn pyresult_to_js_result(vm: &VirtualMachine, result: PyResult) -> Result<JsValue, JsValue> { + result + .map(|value| py_to_js(vm, value)) + .map_err(|err| py_err_to_js_err(vm, &err)) +} + +pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { + if js_val.is_object() { + if let Some(promise) = js_val.dyn_ref::<Promise>() { + // the browser module might not be injected + if vm.try_class("browser", "Promise").is_ok() { + return js_module::PyPromise::new(promise.clone()) + .into_ref(&vm.ctx) + .into(); + } + } + if Array::is_array(&js_val) { + let js_arr: Array = js_val.into(); + let elems = js_arr + .values() + .into_iter() + .map(|val| js_to_py(vm, val.expect("Iteration over array failed"))) + .collect(); + vm.ctx.new_list(elems).into() + } else if ArrayBuffer::is_view(&js_val) || js_val.is_instance_of::<ArrayBuffer>() { + // unchecked_ref because if it's not an ArrayBuffer it could either be a TypedArray + // or a DataView, but they all have a `buffer` property + let u8_array = js_sys::Uint8Array::new( + &js_val + .dyn_ref::<ArrayBuffer>() + .cloned() + .unwrap_or_else(|| js_val.unchecked_ref::<Uint8Array>().buffer()), + ); + let mut vec = vec![0; u8_array.length() as usize]; + u8_array.copy_to(&mut vec); + vm.ctx.new_bytes(vec).into() + } else { + let dict = vm.ctx.new_dict(); + for pair in object_entries(&Object::from(js_val)) { + let (key, val) = pair.expect("iteration over object to not fail"); + let py_val = js_to_py(vm, val); + dict.set_item( + String::from(js_sys::JsString::from(key)).as_str(), + py_val, + vm, + ) + .unwrap(); + } + dict.into() + } + } else if js_val.is_function() { + let func = js_sys::Function::from(js_val); + vm.new_function( + vm.ctx.intern_str(String::from(func.name())).as_str(), + move |args: FuncArgs, vm: &VirtualMachine| -> PyResult { + let this = Object::new(); + for (k, v) in args.kwargs { + Reflect::set(&this, &k.into(), &py_to_js(vm, v)) + .expect("property to be settable"); + } + let js_args = args + .args + .into_iter() + .map(|v| py_to_js(vm, v)) + .collect::<Array>(); + func.apply(&this, &js_args) + .map(|val| js_to_py(vm, val)) + .map_err(|err| js_err_to_py_err(vm, &err)) + }, + ) + .into() + } else if let Some(err) = js_val.dyn_ref::<js_sys::Error>() { + js_err_to_py_err(vm, err).into() + } else if js_val.is_undefined() { + // Because `JSON.stringify(undefined)` returns undefined + vm.ctx.none() + } else { + py_serde::deserialize(vm, serde_wasm_bindgen::Deserializer::from(js_val)) + .unwrap_or_else(|_| vm.ctx.none()) + } +} + +pub fn syntax_err(err: CompileError) -> SyntaxError { + let js_err = SyntaxError::new(&format!("Error parsing Python code: {err}")); + let _ = Reflect::set( + &js_err, + &"row".into(), + &(err.location().unwrap().line.get()).into(), + ); + let _ = Reflect::set( + &js_err, + &"col".into(), + &(err.location().unwrap().character_offset.get()).into(), + ); + // | ParseErrorType::UnrecognizedToken(Token::Dedent, _) + let can_continue = matches!( + &err, + CompileError::Parse(ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::Eof) + | ParseErrorType::Lexical(LexicalErrorType::IndentationError), + .. + }) + ); + let _ = Reflect::set(&js_err, &"canContinue".into(), &can_continue.into()); + js_err +} + +pub trait PyResultExt<T> { + fn into_js(self, vm: &VirtualMachine) -> Result<T, JsValue>; +} +impl<T> PyResultExt<T> for PyResult<T> { + fn into_js(self, vm: &VirtualMachine) -> Result<T, JsValue> { + self.map_err(|err| py_err_to_js_err(vm, &err)) + } +} diff --git a/wasm/lib/src/js_module.rs b/crates/wasm/src/js_module.rs similarity index 92% rename from wasm/lib/src/js_module.rs rename to crates/wasm/src/js_module.rs index f0c5378c35a..314e2adbee0 100644 --- a/wasm/lib/src/js_module.rs +++ b/crates/wasm/src/js_module.rs @@ -1,25 +1,24 @@ pub(crate) use _js::{PyJsValue, PyPromise}; -use rustpython_vm::VirtualMachine; #[pymodule] mod _js { use crate::{ convert, - vm_class::{stored_vm_from_wasm, WASMVirtualMachine}, + vm_class::{WASMVirtualMachine, stored_vm_from_wasm}, weak_vm, }; + use core::{cell, fmt, future}; use js_sys::{Array, Object, Promise, Reflect}; use rustpython_vm::{ + Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyBaseExceptionRef, PyFloat, PyStrRef, PyType, PyTypeRef}, convert::{IntoObject, ToPyObject}, function::{ArgCallable, OptionalArg, OptionalOption, PosArgs}, protocol::PyIterReturn, types::{IterNext, Representable, SelfIter}, - Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, }; - use std::{cell, fmt, future}; - use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; - use wasm_bindgen_futures::{future_to_promise, JsFuture}; + use wasm_bindgen::{JsCast, closure::Closure, prelude::*}; + use wasm_bindgen_futures::{JsFuture, future_to_promise}; #[wasm_bindgen(inline_js = " export function has_prop(target, prop) { return prop in Object(target); } @@ -84,7 +83,7 @@ mod _js { impl JsProperty { fn into_js_value(self) -> JsValue { match self { - JsProperty::Str(s) => s.as_str().into(), + JsProperty::Str(s) => s.expect_str().into(), JsProperty::Js(value) => value.value.clone(), } } @@ -110,7 +109,7 @@ mod _js { #[pymethod] fn new_from_str(&self, s: PyStrRef) -> PyJsValue { - PyJsValue::new(s.as_str()) + PyJsValue::new(s.expect_str()) } #[pymethod] @@ -136,9 +135,7 @@ mod _js { } else if proto.value.is_null() { Object::create(proto.value.unchecked_ref()) } else { - return Err( - vm.new_value_error("prototype must be an Object or null".to_owned()) - ); + return Err(vm.new_value_error("prototype must be an Object or null")); } } else { Object::new() @@ -184,7 +181,7 @@ mod _js { let func = self .value .dyn_ref::<js_sys::Function>() - .ok_or_else(|| vm.new_type_error("JS value is not callable".to_owned()))?; + .ok_or_else(|| vm.new_type_error("JS value is not callable"))?; let js_args = args.iter().map(|x| -> &PyJsValue { x }).collect::<Array>(); let res = match opts.this { Some(this) => Reflect::apply(func, &this.value, &js_args), @@ -216,7 +213,7 @@ mod _js { let ctor = self .value .dyn_ref::<js_sys::Function>() - .ok_or_else(|| vm.new_type_error("JS value is not callable".to_owned()))?; + .ok_or_else(|| vm.new_type_error("JS value is not callable"))?; let proto = opts .prototype .as_ref() @@ -297,7 +294,7 @@ mod _js { } impl fmt::Debug for JsClosure { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.pad("JsClosure") } } @@ -326,7 +323,7 @@ mod _js { .map(|arg| PyJsValue::new(arg).into_pyobject(vm)), ); let res = py_obj.call(pyargs, vm); - convert::pyresult_to_jsresult(vm, res) + convert::pyresult_to_js_result(vm, res) }) }; let closure: ClosureType = if once { @@ -361,9 +358,7 @@ mod _js { #[pymethod] fn destroy(&self, vm: &VirtualMachine) -> PyResult<()> { let (closure, _) = self.closure.replace(None).ok_or_else(|| { - vm.new_value_error( - "can't destroy closure has already been destroyed or detached".to_owned(), - ) + vm.new_value_error("can't destroy closure has already been destroyed or detached") })?; drop(closure); self.destroyed.set(true); @@ -372,9 +367,7 @@ mod _js { #[pymethod] fn detach(&self, vm: &VirtualMachine) -> PyResult<PyJsValueRef> { let (closure, js_val) = self.closure.replace(None).ok_or_else(|| { - vm.new_value_error( - "can't detach closure has already been detached or destroyed".to_owned(), - ) + vm.new_value_error("can't detach closure has already been detached or destroyed") })?; closure.forget(); self.detached.set(true); @@ -500,7 +493,7 @@ mod _js { Some(on_fulfill) => stored_vm.interp.enter(move |vm| { let val = convert::js_to_py(vm, val); let res = on_fulfill.invoke((val,), vm); - convert::pyresult_to_jsresult(vm, res) + convert::pyresult_to_js_result(vm, res) }), None => Ok(val), }, @@ -508,7 +501,7 @@ mod _js { Some(on_reject) => stored_vm.interp.enter(move |vm| { let err = new_js_error(vm, err); let res = on_reject.invoke((err,), vm); - convert::pyresult_to_jsresult(vm, res) + convert::pyresult_to_js_result(vm, res) }), None => Err(err), }, @@ -574,9 +567,7 @@ mod _js { match self.obj.take() { Some(prom) => { if val.is_some() { - Err(vm.new_type_error( - "can't send non-None value to an awaitpromise".to_owned(), - )) + Err(vm.new_type_error("can't send non-None value to an AwaitPromise")) } else { Ok(PyIterReturn::Return(prom)) } @@ -620,8 +611,7 @@ mod _js { fn js_error(vm: &VirtualMachine) -> PyTypeRef { let ctx = &vm.ctx; let js_error = PyRef::leak( - PyType::new_simple_heap("JSError", &vm.ctx.exceptions.exception_type.to_owned(), ctx) - .unwrap(), + PyType::new_simple_heap("JSError", vm.ctx.exceptions.exception_type, ctx).unwrap(), ); extend_class!(ctx, js_error, { "value" => ctx.new_readonly_getset("value", js_error, |exc: PyBaseExceptionRef| exc.get_arg(0)), @@ -630,8 +620,4 @@ mod _js { } } -pub(crate) use _js::make_module; - -pub fn setup_js_module(vm: &mut VirtualMachine) { - vm.add_native_module("_js".to_owned(), Box::new(make_module)); -} +pub(crate) use _js::module_def; diff --git a/crates/wasm/src/lib.rs b/crates/wasm/src/lib.rs new file mode 100644 index 00000000000..99668df2855 --- /dev/null +++ b/crates/wasm/src/lib.rs @@ -0,0 +1,136 @@ +extern crate alloc; + +pub mod browser_module; +pub mod convert; +pub mod js_module; +pub mod vm_class; +pub mod wasm_builtins; + +#[macro_use] +extern crate rustpython_vm; + +use js_sys::{Reflect, WebAssembly::RuntimeError}; +use std::panic; +pub(crate) use vm_class::weak_vm; +use wasm_bindgen::prelude::*; + +/// Sets error info on the window object, and prints the backtrace to console +pub fn panic_hook(info: &panic::PanicHookInfo<'_>) { + // If something errors, just ignore it; we don't want to panic in the panic hook + let try_set_info = || { + let msg = &info.to_string(); + let window = match web_sys::window() { + Some(win) => win, + None => return, + }; + let _ = Reflect::set(&window, &"__RUSTPYTHON_ERROR_MSG".into(), &msg.into()); + let error = RuntimeError::new(msg); + let _ = Reflect::set(&window, &"__RUSTPYTHON_ERROR".into(), &error); + let stack = match Reflect::get(&error, &"stack".into()) { + Ok(stack) => stack, + Err(_) => return, + }; + let _ = Reflect::set(&window, &"__RUSTPYTHON_ERROR_STACK".into(), &stack); + }; + try_set_info(); + console_error_panic_hook::hook(info); +} + +#[doc(hidden)] +#[cfg(not(feature = "no-start-func"))] +#[wasm_bindgen(start)] +pub fn _setup_console_error() { + std::panic::set_hook(Box::new(panic_hook)); +} + +pub mod eval { + use crate::vm_class::VMStore; + use js_sys::{Object, Reflect, TypeError}; + use rustpython_vm::compiler::Mode; + use wasm_bindgen::prelude::*; + + const PY_EVAL_VM_ID: &str = "__py_eval_vm"; + + fn run_py(source: &str, options: Option<Object>, mode: Mode) -> Result<JsValue, JsValue> { + let vm = VMStore::init(PY_EVAL_VM_ID.into(), Some(true)); + let options = options.unwrap_or_default(); + let js_vars = { + let prop = Reflect::get(&options, &"vars".into())?; + if prop.is_undefined() { + None + } else if prop.is_object() { + Some(Object::from(prop)) + } else { + return Err(TypeError::new("vars must be an object").into()); + } + }; + + vm.set_stdout(Reflect::get(&options, &"stdout".into())?)?; + + if let Some(js_vars) = js_vars { + vm.add_to_scope("js_vars".into(), js_vars.into())?; + } + vm.run(source, mode, None) + } + + /// Evaluate Python code + /// + /// ```js + /// var result = pyEval(code, options?); + /// ``` + /// + /// `code`: `string`: The Python code to run in eval mode + /// + /// `options`: + /// + /// - `vars?`: `{ [key: string]: any }`: Variables passed to the VM that can be + /// accessed in Python with the variable `js_vars`. Functions do work, and + /// receive the Python kwargs as the `this` argument. + /// - `stdout?`: `"console" | ((out: string) => void) | null`: A function to replace the + /// native print native print function, and it will be `console.log` when giving + /// `undefined` or "console", and it will be a dumb function when giving null. + #[wasm_bindgen(js_name = pyEval)] + pub fn eval_py(source: &str, options: Option<Object>) -> Result<JsValue, JsValue> { + run_py(source, options, Mode::Eval) + } + + /// Evaluate Python code + /// + /// ```js + /// pyExec(code, options?); + /// ``` + /// + /// `code`: `string`: The Python code to run in exec mode + /// + /// `options`: The options are the same as eval mode + #[wasm_bindgen(js_name = pyExec)] + pub fn exec_py(source: &str, options: Option<Object>) -> Result<(), JsValue> { + run_py(source, options, Mode::Exec).map(drop) + } + + /// Evaluate Python code + /// + /// ```js + /// var result = pyExecSingle(code, options?); + /// ``` + /// + /// `code`: `string`: The Python code to run in exec single mode + /// + /// `options`: The options are the same as eval mode + #[wasm_bindgen(js_name = pyExecSingle)] + pub fn exec_single_py(source: &str, options: Option<Object>) -> Result<JsValue, JsValue> { + run_py(source, options, Mode::Single) + } +} + +/// A module containing all the wasm-bindgen exports that rustpython_wasm has +/// Re-export as `pub use rustpython_wasm::exports::*;` in the root of your crate if you want your +/// wasm module to mimic rustpython_wasm's API +pub mod exports { + pub use crate::convert::PyError; + pub use crate::eval::{eval_py, exec_py, exec_single_py}; + pub use crate::vm_class::{VMStore, WASMVirtualMachine}; +} + +#[doc(hidden)] +pub use exports::*; diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs new file mode 100644 index 00000000000..0657b0bc08e --- /dev/null +++ b/crates/wasm/src/vm_class.rs @@ -0,0 +1,348 @@ +use crate::{ + browser_module, + convert::{self, PyResultExt}, + js_module, wasm_builtins, +}; +use alloc::rc::{Rc, Weak}; +use core::cell::RefCell; +use js_sys::{Object, TypeError}; +use rustpython_vm::{ + Interpreter, PyObjectRef, PyRef, PyResult, Settings, VirtualMachine, builtins::PyWeak, + compiler::Mode, function::ArgMapping, scope::Scope, +}; +use std::collections::HashMap; +use wasm_bindgen::prelude::*; + +pub(crate) struct StoredVirtualMachine { + pub interp: Interpreter, + pub scope: Scope, + /// you can put a Rc in here, keep it as a Weak, and it'll be held only for + /// as long as the StoredVM is alive + held_objects: RefCell<Vec<PyObjectRef>>, +} + +#[pymodule] +mod _window { + use super::{js_module, wasm_builtins}; + use rustpython_vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> { + __module_exec(vm, module); + extend_module!(vm, module, { + "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(&vm.ctx), + }); + Ok(()) + } +} + +impl StoredVirtualMachine { + fn new(id: String, inject_browser_module: bool) -> StoredVirtualMachine { + let mut settings = Settings::default(); + settings.allow_external_library = false; + + let mut builder = Interpreter::builder(settings); + + #[cfg(feature = "freeze-stdlib")] + { + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + builder = builder + .add_native_modules(&defs) + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB); + } + + // Add wasm-specific modules + let js_def = js_module::module_def(&builder.ctx); + builder = builder.add_native_module(js_def); + + if inject_browser_module { + let window_def = _window::module_def(&builder.ctx); + let browser_def = browser_module::module_def(&builder.ctx); + builder = builder + .add_native_modules(&[window_def, browser_def]) + .add_frozen_modules(rustpython_vm::py_freeze!(dir = "../Lib")); + } + + let interp = builder + .init_hook(move |vm| { + vm.wasm_id = Some(id); + }) + .build(); + + let scope = interp.enter(|vm| vm.new_scope_with_builtins()); + + StoredVirtualMachine { + interp, + scope, + held_objects: RefCell::new(Vec::new()), + } + } +} + +// It's fine that it's thread local, since WASM doesn't even have threads yet. thread_local! +// probably gets compiled down to a normal-ish static variable, like Atomic* types do: +// https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions +thread_local! { + static STORED_VMS: RefCell<HashMap<String, Rc<StoredVirtualMachine>>> = RefCell::default(); +} + +pub fn get_vm_id(vm: &VirtualMachine) -> &str { + vm.wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set") +} +pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc<StoredVirtualMachine> { + STORED_VMS.with_borrow(|vms| { + vms.get(&wasm_vm.id) + .expect("VirtualMachine is not valid") + .clone() + }) +} +pub(crate) fn weak_vm(vm: &VirtualMachine) -> Weak<StoredVirtualMachine> { + let id = get_vm_id(vm); + STORED_VMS.with_borrow(|vms| Rc::downgrade(vms.get(id).expect("VirtualMachine is not valid"))) +} + +#[derive(Clone, Copy)] +#[wasm_bindgen(js_name = vmStore)] +pub struct VMStore; + +#[wasm_bindgen(js_class = vmStore)] +impl VMStore { + pub fn init(id: String, inject_browser_module: Option<bool>) -> WASMVirtualMachine { + STORED_VMS.with_borrow_mut(|vms| { + if !vms.contains_key(&id) { + let stored_vm = + StoredVirtualMachine::new(id.clone(), inject_browser_module.unwrap_or(true)); + vms.insert(id.clone(), Rc::new(stored_vm)); + } + }); + WASMVirtualMachine { id } + } + + pub(crate) fn _get(id: String) -> Option<WASMVirtualMachine> { + STORED_VMS.with_borrow(|vms| vms.contains_key(&id).then_some(WASMVirtualMachine { id })) + } + + pub fn get(id: String) -> JsValue { + match Self::_get(id) { + Some(wasm_vm) => wasm_vm.into(), + None => JsValue::UNDEFINED, + } + } + + pub fn destroy(id: String) { + STORED_VMS.with_borrow_mut(|vms| { + if let Some(stored_vm) = vms.remove(&id) { + // for f in stored_vm.drop_handlers.iter() { + // f(); + // } + // deallocate the VM + drop(stored_vm); + } + }); + } + + pub fn ids() -> Vec<JsValue> { + STORED_VMS.with_borrow(|vms| vms.keys().map(|k| k.into()).collect()) + } +} + +#[wasm_bindgen(js_name = VirtualMachine)] +#[derive(Clone)] +pub struct WASMVirtualMachine { + pub(crate) id: String, +} + +#[wasm_bindgen(js_class = VirtualMachine)] +impl WASMVirtualMachine { + pub(crate) fn with_unchecked<F, R>(&self, f: F) -> R + where + F: FnOnce(&StoredVirtualMachine) -> R, + { + let stored_vm = STORED_VMS.with_borrow_mut(|vms| vms.get_mut(&self.id).unwrap().clone()); + f(&stored_vm) + } + + pub(crate) fn with<F, R>(&self, f: F) -> Result<R, JsValue> + where + F: FnOnce(&StoredVirtualMachine) -> R, + { + self.assert_valid()?; + Ok(self.with_unchecked(f)) + } + + pub(crate) fn with_vm<F, R>(&self, f: F) -> Result<R, JsValue> + where + F: FnOnce(&VirtualMachine, &StoredVirtualMachine) -> R, + { + self.with(|stored| stored.interp.enter(|vm| f(vm, stored))) + } + + pub fn valid(&self) -> bool { + STORED_VMS.with_borrow(|vms| vms.contains_key(&self.id)) + } + + pub(crate) fn push_held_rc( + &self, + obj: PyObjectRef, + ) -> Result<PyResult<PyRef<PyWeak>>, JsValue> { + self.with_vm(|vm, stored_vm| { + let weak = obj.downgrade(None, vm)?; + stored_vm.held_objects.borrow_mut().push(obj); + Ok(weak) + }) + } + + pub fn assert_valid(&self) -> Result<(), JsValue> { + if self.valid() { + Ok(()) + } else { + Err(TypeError::new( + "Invalid VirtualMachine, this VM was destroyed while this reference was still held", + ) + .into()) + } + } + + pub fn destroy(&self) -> Result<(), JsValue> { + self.assert_valid()?; + VMStore::destroy(self.id.clone()); + Ok(()) + } + + #[wasm_bindgen(js_name = addToScope)] + pub fn add_to_scope(&self, name: String, value: JsValue) -> Result<(), JsValue> { + self.with_vm(move |vm, StoredVirtualMachine { scope, .. }| { + let value = convert::js_to_py(vm, value); + scope.globals.set_item(&name, value, vm).into_js(vm) + })? + } + + #[wasm_bindgen(js_name = setStdout)] + pub fn set_stdout(&self, stdout: JsValue) -> Result<(), JsValue> { + self.with_vm(|vm, _| { + fn error() -> JsValue { + TypeError::new("Unknown stdout option, please pass a function or 'console'").into() + } + use wasm_builtins::make_stdout_object; + let stdout: PyObjectRef = if let Some(s) = stdout.as_string() { + match s.as_str() { + "console" => make_stdout_object(vm, wasm_builtins::sys_stdout_write_console), + _ => return Err(error()), + } + } else if stdout.is_function() { + let func = js_sys::Function::from(stdout); + make_stdout_object(vm, move |data, vm| { + func.call1(&JsValue::UNDEFINED, &data.into()) + .map_err(|err| convert::js_py_typeerror(vm, err))?; + Ok(()) + }) + } else if stdout.is_null() { + make_stdout_object(vm, |_, _| Ok(())) + } else if stdout.is_undefined() { + make_stdout_object(vm, wasm_builtins::sys_stdout_write_console) + } else { + return Err(error()); + }; + vm.sys_module.set_attr("stdout", stdout, vm).unwrap(); + Ok(()) + })? + } + + #[wasm_bindgen(js_name = injectModule)] + pub fn inject_module( + &self, + name: String, + source: &str, + imports: Option<Object>, + ) -> Result<(), JsValue> { + self.with_vm(|vm, _| { + let code = vm + .compile(source, Mode::Exec, name.clone()) + .map_err(convert::syntax_err)?; + let attrs = vm.ctx.new_dict(); + attrs + .set_item("__name__", vm.new_pyobj(name.as_str()), vm) + .into_js(vm)?; + + if let Some(imports) = imports { + for entry in convert::object_entries(&imports) { + let (key, value) = entry?; + let key: String = Object::from(key).to_string().into(); + attrs + .set_item(key.as_str(), convert::js_to_py(vm, value), vm) + .into_js(vm)?; + } + } + + vm.run_code_obj( + code, + Scope::new( + Some(ArgMapping::from_dict_exact(attrs.clone())), + attrs.clone(), + ), + ) + .into_js(vm)?; + + let module = vm.new_module(&name, attrs, None); + + let sys_modules = vm.sys_module.get_attr("modules", vm).into_js(vm)?; + sys_modules.set_item(&name, module.into(), vm).into_js(vm)?; + + Ok(()) + })? + } + + #[wasm_bindgen(js_name = injectJSModule)] + pub fn inject_js_module(&self, name: String, module: Object) -> Result<(), JsValue> { + self.with_vm(|vm, _| { + let py_module = vm.new_module(&name, vm.ctx.new_dict(), None); + for entry in convert::object_entries(&module) { + let (key, value) = entry?; + let key = Object::from(key).to_string(); + extend_module!(vm, &py_module, { + String::from(key) => convert::js_to_py(vm, value), + }); + } + + let sys_modules = vm.sys_module.get_attr("modules", vm).into_js(vm)?; + sys_modules + .set_item(&name, py_module.into(), vm) + .into_js(vm)?; + + Ok(()) + })? + } + + pub(crate) fn run( + &self, + source: &str, + mode: Mode, + source_path: Option<String>, + ) -> Result<JsValue, JsValue> { + self.with_vm(|vm, StoredVirtualMachine { scope, .. }| { + let source_path = source_path.unwrap_or_else(|| "<wasm>".to_owned()); + let code = vm.compile(source, mode, source_path); + let code = code.map_err(convert::syntax_err)?; + let result = vm.run_code_obj(code, scope.clone()); + convert::pyresult_to_js_result(vm, result) + })? + } + + pub fn exec(&self, source: &str, source_path: Option<String>) -> Result<JsValue, JsValue> { + self.run(source, Mode::Exec, source_path) + } + + pub fn eval(&self, source: &str, source_path: Option<String>) -> Result<JsValue, JsValue> { + self.run(source, Mode::Eval, source_path) + } + + #[wasm_bindgen(js_name = execSingle)] + pub fn exec_single( + &self, + source: &str, + source_path: Option<String>, + ) -> Result<JsValue, JsValue> { + self.run(source, Mode::Single, source_path) + } +} diff --git a/wasm/lib/src/wasm_builtins.rs b/crates/wasm/src/wasm_builtins.rs similarity index 91% rename from wasm/lib/src/wasm_builtins.rs rename to crates/wasm/src/wasm_builtins.rs index 59d7880af40..efbc03c39ce 100644 --- a/wasm/lib/src/wasm_builtins.rs +++ b/crates/wasm/src/wasm_builtins.rs @@ -4,7 +4,7 @@ //! desktop. //! Implements functions listed here: https://docs.python.org/3/library/builtins.html. -use rustpython_vm::{builtins::PyStrRef, PyObjectRef, PyRef, PyResult, VirtualMachine}; +use rustpython_vm::{PyObjectRef, PyRef, PyResult, VirtualMachine, builtins::PyStrRef}; use web_sys::{self, console}; pub(crate) fn window() -> web_sys::Window { @@ -33,7 +33,7 @@ pub fn make_stdout_object( "write", cls, move |_self: PyObjectRef, data: PyStrRef, vm: &VirtualMachine| -> PyResult<()> { - write_f(data.as_str(), vm) + write_f(data.expect_str(), vm) }, ); let flush_method = vm.new_method("flush", cls, |_self: PyObjectRef| {}); diff --git a/crates/wtf8/Cargo.toml b/crates/wtf8/Cargo.toml new file mode 100644 index 00000000000..110b54ad0ca --- /dev/null +++ b/crates/wtf8/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rustpython-wtf8" +description = "An implementation of WTF-8 for use in RustPython" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +ascii = { workspace = true } +bstr = { workspace = true } +itertools = { workspace = true } +memchr = { workspace = true } diff --git a/crates/wtf8/src/concat.rs b/crates/wtf8/src/concat.rs new file mode 100644 index 00000000000..055a401d69f --- /dev/null +++ b/crates/wtf8/src/concat.rs @@ -0,0 +1,143 @@ +use alloc::borrow::{Cow, ToOwned}; +use alloc::boxed::Box; +use alloc::string::String; +use core::fmt; +use fmt::Write; + +use crate::{CodePoint, Wtf8, Wtf8Buf}; + +impl fmt::Write for Wtf8Buf { + #[inline] + fn write_str(&mut self, s: &str) -> fmt::Result { + self.push_str(s); + Ok(()) + } +} + +/// Trait for types that can be appended to a [`Wtf8Buf`], preserving surrogates. +pub trait Wtf8Concat { + fn fmt_wtf8(&self, buf: &mut Wtf8Buf); +} + +impl Wtf8Concat for Wtf8 { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + buf.push_wtf8(self); + } +} + +impl Wtf8Concat for Wtf8Buf { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + buf.push_wtf8(self); + } +} + +impl Wtf8Concat for str { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + buf.push_str(self); + } +} + +impl Wtf8Concat for String { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + buf.push_str(self); + } +} + +impl Wtf8Concat for char { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + buf.push_char(*self); + } +} + +impl Wtf8Concat for CodePoint { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + buf.push(*self); + } +} + +/// Wrapper that appends a [`fmt::Display`] value to a [`Wtf8Buf`]. +/// +/// Note: This goes through UTF-8 formatting, so lone surrogates in the +/// display output will be replaced with U+FFFD. Use direct [`Wtf8Concat`] +/// impls for surrogate-preserving concatenation. +#[allow(dead_code)] +pub struct DisplayAsWtf8<T>(pub T); + +impl<T: fmt::Display> Wtf8Concat for DisplayAsWtf8<T> { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + write!(buf, "{}", self.0).unwrap(); + } +} + +macro_rules! impl_wtf8_concat_for_int { + ($($t:ty),*) => { + $(impl Wtf8Concat for $t { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + write!(buf, "{}", self).unwrap(); + } + })* + }; +} + +impl_wtf8_concat_for_int!( + u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, f32, f64 +); + +impl<T: Wtf8Concat + ?Sized> Wtf8Concat for &T { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + (**self).fmt_wtf8(buf); + } +} + +impl<T: Wtf8Concat + ?Sized> Wtf8Concat for &mut T { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + (**self).fmt_wtf8(buf); + } +} + +impl<T: Wtf8Concat + ?Sized> Wtf8Concat for Box<T> { + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + (**self).fmt_wtf8(buf); + } +} + +impl<T: Wtf8Concat + ?Sized> Wtf8Concat for Cow<'_, T> +where + T: ToOwned, +{ + #[inline] + fn fmt_wtf8(&self, buf: &mut Wtf8Buf) { + (**self).fmt_wtf8(buf); + } +} + +/// Concatenate values into a [`Wtf8Buf`], preserving surrogates. +/// +/// Each argument must implement [`Wtf8Concat`]. String literals (`&str`), +/// [`Wtf8`], [`Wtf8Buf`], [`char`], and [`CodePoint`] are all supported. +/// +/// ``` +/// use rustpython_wtf8::Wtf8Buf; +/// let name = "world"; +/// let result = rustpython_wtf8::wtf8_concat!("hello, ", name, "!"); +/// assert_eq!(result, Wtf8Buf::from("hello, world!")); +/// ``` +#[macro_export] +macro_rules! wtf8_concat { + ($($arg:expr),* $(,)?) => {{ + let mut buf = $crate::Wtf8Buf::new(); + $($crate::Wtf8Concat::fmt_wtf8(&$arg, &mut buf);)* + buf + }}; +} diff --git a/crates/wtf8/src/core_char.rs b/crates/wtf8/src/core_char.rs new file mode 100644 index 00000000000..66f321d8aa7 --- /dev/null +++ b/crates/wtf8/src/core_char.rs @@ -0,0 +1,114 @@ +// spell-checker:disable +//! Unstable functions from [`core::char`] + +use core::slice; + +pub const MAX_LEN_UTF8: usize = 4; +pub const MAX_LEN_UTF16: usize = 2; + +// UTF-8 ranges and tags for encoding characters +const TAG_CONT: u8 = 0b1000_0000; +const TAG_TWO_B: u8 = 0b1100_0000; +const TAG_THREE_B: u8 = 0b1110_0000; +const TAG_FOUR_B: u8 = 0b1111_0000; +const MAX_ONE_B: u32 = 0x80; +const MAX_TWO_B: u32 = 0x800; +const MAX_THREE_B: u32 = 0x10000; + +#[inline] +#[must_use] +pub const fn len_utf8(code: u32) -> usize { + match code { + ..MAX_ONE_B => 1, + ..MAX_TWO_B => 2, + ..MAX_THREE_B => 3, + _ => 4, + } +} + +#[inline] +#[must_use] +const fn len_utf16(code: u32) -> usize { + if (code & 0xFFFF) == code { 1 } else { 2 } +} + +/// Encodes a raw `u32` value as UTF-8 into the provided byte buffer, +/// and then returns the subslice of the buffer that contains the encoded character. +/// +/// Unlike `char::encode_utf8`, this method also handles codepoints in the surrogate range. +/// (Creating a `char` in the surrogate range is UB.) +/// The result is valid [generalized UTF-8] but not valid UTF-8. +/// +/// [generalized UTF-8]: https://simonsapin.github.io/wtf-8/#generalized-utf8 +/// +/// # Panics +/// +/// Panics if the buffer is not large enough. +/// A buffer of length four is large enough to encode any `char`. +#[doc(hidden)] +#[inline] +pub fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> &mut [u8] { + let len = len_utf8(code); + match (len, &mut *dst) { + (1, [a, ..]) => { + *a = code as u8; + } + (2, [a, b, ..]) => { + *a = (code >> 6 & 0x1F) as u8 | TAG_TWO_B; + *b = (code & 0x3F) as u8 | TAG_CONT; + } + (3, [a, b, c, ..]) => { + *a = (code >> 12 & 0x0F) as u8 | TAG_THREE_B; + *b = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *c = (code & 0x3F) as u8 | TAG_CONT; + } + (4, [a, b, c, d, ..]) => { + *a = (code >> 18 & 0x07) as u8 | TAG_FOUR_B; + *b = (code >> 12 & 0x3F) as u8 | TAG_CONT; + *c = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *d = (code & 0x3F) as u8 | TAG_CONT; + } + _ => { + panic!( + "encode_utf8: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}", + dst_len = dst.len(), + ) + } + }; + // SAFETY: `<&mut [u8]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds. + unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) } +} + +/// Encodes a raw `u32` value as UTF-16 into the provided `u16` buffer, +/// and then returns the subslice of the buffer that contains the encoded character. +/// +/// Unlike `char::encode_utf16`, this method also handles codepoints in the surrogate range. +/// (Creating a `char` in the surrogate range is UB.) +/// +/// # Panics +/// +/// Panics if the buffer is not large enough. +/// A buffer of length 2 is large enough to encode any `char`. +#[doc(hidden)] +#[inline] +pub fn encode_utf16_raw(mut code: u32, dst: &mut [u16]) -> &mut [u16] { + let len = len_utf16(code); + match (len, &mut *dst) { + (1, [a, ..]) => { + *a = code as u16; + } + (2, [a, b, ..]) => { + code -= 0x1_0000; + *a = (code >> 10) as u16 | 0xD800; + *b = (code & 0x3FF) as u16 | 0xDC00; + } + _ => { + panic!( + "encode_utf16: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}", + dst_len = dst.len(), + ) + } + }; + // SAFETY: `<&mut [u16]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds. + unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) } +} diff --git a/crates/wtf8/src/core_str.rs b/crates/wtf8/src/core_str.rs new file mode 100644 index 00000000000..56f715cf756 --- /dev/null +++ b/crates/wtf8/src/core_str.rs @@ -0,0 +1,113 @@ +//! Operations related to UTF-8 validation. +//! +//! Copied from `core::str::validations` + +/// Returns the initial codepoint accumulator for the first byte. +/// The first byte is special, only want bottom 5 bits for width 2, 4 bits +/// for width 3, and 3 bits for width 4. +#[inline] +const fn utf8_first_byte(byte: u8, width: u32) -> u32 { + (byte & (0x7F >> width)) as u32 +} + +/// Returns the value of `ch` updated with continuation byte `byte`. +#[inline] +const fn utf8_acc_cont_byte(ch: u32, byte: u8) -> u32 { + (ch << 6) | (byte & CONT_MASK) as u32 +} + +/// Checks whether the byte is a UTF-8 continuation byte (i.e., starts with the +/// bits `10`). +#[inline] +pub(super) const fn utf8_is_cont_byte(byte: u8) -> bool { + (byte as i8) < -64 +} + +/// Reads the next code point out of a byte iterator (assuming a +/// UTF-8-like encoding). +/// +/// # Safety +/// +/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string +#[inline] +pub unsafe fn next_code_point<'a, I: Iterator<Item = &'a u8>>(bytes: &mut I) -> Option<u32> { + // Decode UTF-8 + let x = *bytes.next()?; + if x < 128 { + return Some(x as u32); + } + + // Multibyte case follows + // Decode from a byte combination out of: [[[x y] z] w] + // NOTE: Performance is sensitive to the exact formulation here + let init = utf8_first_byte(x, 2); + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let y = unsafe { *bytes.next().unwrap_unchecked() }; + let mut ch = utf8_acc_cont_byte(init, y); + if x >= 0xE0 { + // [[x y z] w] case + // 5th bit in 0xE0 .. 0xEF is always clear, so `init` is still valid + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let z = unsafe { *bytes.next().unwrap_unchecked() }; + let y_z = utf8_acc_cont_byte((y & CONT_MASK) as u32, z); + ch = init << 12 | y_z; + if x >= 0xF0 { + // [x y z w] case + // use only the lower 3 bits of `init` + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let w = unsafe { *bytes.next().unwrap_unchecked() }; + ch = (init & 7) << 18 | utf8_acc_cont_byte(y_z, w); + } + } + + Some(ch) +} + +/// Reads the last code point out of a byte iterator (assuming a +/// UTF-8-like encoding). +/// +/// # Safety +/// +/// `bytes` must produce a valid UTF-8-like (UTF-8 or WTF-8) string +#[inline] +pub unsafe fn next_code_point_reverse<'a, I>(bytes: &mut I) -> Option<u32> +where + I: DoubleEndedIterator<Item = &'a u8>, +{ + // Decode UTF-8 + let w = match *bytes.next_back()? { + next_byte if next_byte < 128 => return Some(next_byte as u32), + back_byte => back_byte, + }; + + // Multibyte case follows + // Decode from a byte combination out of: [x [y [z w]]] + let mut ch; + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let z = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(z, 2); + if utf8_is_cont_byte(z) { + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let y = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(y, 3); + if utf8_is_cont_byte(y) { + // SAFETY: `bytes` produces an UTF-8-like string, + // so the iterator must produce a value here. + let x = unsafe { *bytes.next_back().unwrap_unchecked() }; + ch = utf8_first_byte(x, 4); + ch = utf8_acc_cont_byte(ch, y); + } + ch = utf8_acc_cont_byte(ch, z); + } + ch = utf8_acc_cont_byte(ch, w); + + Some(ch) +} + +/// Mask of the value bits of a continuation byte. +const CONT_MASK: u8 = 0b0011_1111; diff --git a/crates/wtf8/src/core_str_count.rs b/crates/wtf8/src/core_str_count.rs new file mode 100644 index 00000000000..8f9d5585bc7 --- /dev/null +++ b/crates/wtf8/src/core_str_count.rs @@ -0,0 +1,142 @@ +// spell-checker:disable +//! Modified from core::str::count + +use super::Wtf8; + +const USIZE_SIZE: usize = core::mem::size_of::<usize>(); +const UNROLL_INNER: usize = 4; + +#[inline] +pub(super) fn count_chars(s: &Wtf8) -> usize { + if s.len() < USIZE_SIZE * UNROLL_INNER { + // Avoid entering the optimized implementation for strings where the + // difference is not likely to matter, or where it might even be slower. + // That said, a ton of thought was not spent on the particular threshold + // here, beyond "this value seems to make sense". + char_count_general_case(s.as_bytes()) + } else { + do_count_chars(s) + } +} + +fn do_count_chars(s: &Wtf8) -> usize { + // For correctness, `CHUNK_SIZE` must be: + // + // - Less than or equal to 255, otherwise we'll overflow bytes in `counts`. + // - A multiple of `UNROLL_INNER`, otherwise our `break` inside the + // `body.chunks(CHUNK_SIZE)` loop is incorrect. + // + // For performance, `CHUNK_SIZE` should be: + // - Relatively cheap to `/` against (so some simple sum of powers of two). + // - Large enough to avoid paying for the cost of the `sum_bytes_in_usize` + // too often. + const CHUNK_SIZE: usize = 192; + + // Check the properties of `CHUNK_SIZE` and `UNROLL_INNER` that are required + // for correctness. + const _: () = assert!(CHUNK_SIZE < 256); + const _: () = assert!(CHUNK_SIZE.is_multiple_of(UNROLL_INNER)); + + // SAFETY: transmuting `[u8]` to `[usize]` is safe except for size + // differences which are handled by `align_to`. + let (head, body, tail) = unsafe { s.as_bytes().align_to::<usize>() }; + + // This should be quite rare, and basically exists to handle the degenerate + // cases where align_to fails (as well as miri under symbolic alignment + // mode). + // + // The `unlikely` helps discourage LLVM from inlining the body, which is + // nice, as we would rather not mark the `char_count_general_case` function + // as cold. + if unlikely(body.is_empty() || head.len() > USIZE_SIZE || tail.len() > USIZE_SIZE) { + return char_count_general_case(s.as_bytes()); + } + + let mut total = char_count_general_case(head) + char_count_general_case(tail); + // Split `body` into `CHUNK_SIZE` chunks to reduce the frequency with which + // we call `sum_bytes_in_usize`. + for chunk in body.chunks(CHUNK_SIZE) { + // We accumulate intermediate sums in `counts`, where each byte contains + // a subset of the sum of this chunk, like a `[u8; size_of::<usize>()]`. + let mut counts = 0; + + let (unrolled_chunks, remainder) = chunk.as_chunks::<UNROLL_INNER>(); + for unrolled in unrolled_chunks { + for &word in unrolled { + // Because `CHUNK_SIZE` is < 256, this addition can't cause the + // count in any of the bytes to overflow into a subsequent byte. + counts += contains_non_continuation_byte(word); + } + } + + // Sum the values in `counts` (which, again, is conceptually a `[u8; + // size_of::<usize>()]`), and accumulate the result into `total`. + total += sum_bytes_in_usize(counts); + + // If there's any data in `remainder`, then handle it. This will only + // happen for the last `chunk` in `body.chunks()` (because `CHUNK_SIZE` + // is divisible by `UNROLL_INNER`), so we explicitly break at the end + // (which seems to help LLVM out). + if !remainder.is_empty() { + // Accumulate all the data in the remainder. + let mut counts = 0; + for &word in remainder { + counts += contains_non_continuation_byte(word); + } + total += sum_bytes_in_usize(counts); + break; + } + } + total +} + +// Checks each byte of `w` to see if it contains the first byte in a UTF-8 +// sequence. Bytes in `w` which are continuation bytes are left as `0x00` (e.g. +// false), and bytes which are non-continuation bytes are left as `0x01` (e.g. +// true) +#[inline] +fn contains_non_continuation_byte(w: usize) -> usize { + const LSB: usize = usize_repeat_u8(0x01); + ((!w >> 7) | (w >> 6)) & LSB +} + +// Morally equivalent to `values.to_ne_bytes().into_iter().sum::<usize>()`, but +// more efficient. +#[inline] +fn sum_bytes_in_usize(values: usize) -> usize { + const LSB_SHORTS: usize = usize_repeat_u16(0x0001); + const SKIP_BYTES: usize = usize_repeat_u16(0x00ff); + + let pair_sum: usize = (values & SKIP_BYTES) + ((values >> 8) & SKIP_BYTES); + pair_sum.wrapping_mul(LSB_SHORTS) >> ((USIZE_SIZE - 2) * 8) +} + +// This is the most direct implementation of the concept of "count the number of +// bytes in the string which are not continuation bytes", and is used for the +// head and tail of the input string (the first and last item in the tuple +// returned by `slice::align_to`). +fn char_count_general_case(s: &[u8]) -> usize { + s.iter() + .filter(|&&byte| !super::core_str::utf8_is_cont_byte(byte)) + .count() +} + +// polyfills of unstable library features + +const fn usize_repeat_u8(x: u8) -> usize { + usize::from_ne_bytes([x; size_of::<usize>()]) +} + +const fn usize_repeat_u16(x: u16) -> usize { + let mut r = 0usize; + let mut i = 0; + while i < size_of::<usize>() { + // Use `wrapping_shl` to make it work on targets with 16-bit `usize` + r = r.wrapping_shl(16) | (x as usize); + i += 2; + } + r +} +const fn unlikely(x: bool) -> bool { + x +} diff --git a/crates/wtf8/src/lib.rs b/crates/wtf8/src/lib.rs new file mode 100644 index 00000000000..772a2879944 --- /dev/null +++ b/crates/wtf8/src/lib.rs @@ -0,0 +1,1614 @@ +// spell-checker:disable + +//! An implementation of [WTF-8], a utf8-compatible encoding that allows for +//! unpaired surrogate codepoints. This implementation additionally allows for +//! paired surrogates that are nonetheless treated as two separate codepoints. +//! +//! +//! RustPython uses this because CPython internally uses a variant of UCS-1/2/4 +//! as its string storage, which treats each `u8`/`u16`/`u32` value (depending +//! on the highest codepoint value in the string) as simply integers, unlike +//! UTF-8 or UTF-16 where some characters are encoded using multi-byte +//! sequences. CPython additionally doesn't disallow the use of surrogates in +//! `str`s (which in UTF-16 pair together to represent codepoints with a value +//! higher than `u16::MAX`) and in fact takes quite extensive advantage of the +//! fact that they're allowed. The `surrogateescape` codec-error handler uses +//! them to represent byte sequences which are invalid in the given codec (e.g. +//! bytes with their high bit set in ASCII or UTF-8) by mapping them into the +//! surrogate range. `surrogateescape` is the default error handler in Python +//! for interacting with the filesystem, and thus if RustPython is to properly +//! support `surrogateescape`, its `str`s must be able to represent surrogates. +//! +//! We use WTF-8 over something more similar to CPython's string implementation +//! because of its compatibility with UTF-8, meaning that in the case where a +//! string has no surrogates, it can be viewed as a UTF-8 Rust [`prim@str`] without +//! needing any copies or re-encoding. +//! +//! This implementation is mostly copied from the WTF-8 implementation in the +//! Rust 1.85 standard library, which is used as the backing for [`OsStr`] on +//! Windows targets. As previously mentioned, however, it is modified to not +//! join two surrogates into one codepoint when concatenating strings, in order +//! to match CPython's behavior. +//! +//! [WTF-8]: https://simonsapin.github.io/wtf-8 +//! [`OsStr`]: std::ffi::OsStr + +#![no_std] +#![allow(clippy::precedence, clippy::match_overlapping_arm)] + +extern crate alloc; + +use alloc::borrow::{Cow, ToOwned}; +use alloc::boxed::Box; +use alloc::collections::TryReserveError; +use alloc::string::String; +use alloc::vec::Vec; +use core::borrow::Borrow; +use core::fmt; +use core::hash::{Hash, Hasher}; +use core::iter::FusedIterator; +use core::mem; +use core::ops; +use core::slice; +use core::str; +use core_char::MAX_LEN_UTF8; +use core_char::{MAX_LEN_UTF16, encode_utf8_raw, encode_utf16_raw, len_utf8}; +use core_str::{next_code_point, next_code_point_reverse}; +use itertools::{Either, Itertools}; + +use bstr::{ByteSlice, ByteVec}; + +mod core_char; +mod core_str; +mod core_str_count; + +const UTF8_REPLACEMENT_CHARACTER: &str = "\u{FFFD}"; + +/// A Unicode code point: from U+0000 to U+10FFFF. +/// +/// Compares with the `char` type, +/// which represents a Unicode scalar value: +/// a code point that is not a surrogate (U+D800 to U+DFFF). +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Copy)] +pub struct CodePoint { + value: u32, +} + +/// Format the code point as `U+` followed by four to six hexadecimal digits. +/// Example: `U+1F4A9` +impl fmt::Debug for CodePoint { + #[inline] + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "U+{:04X}", self.value) + } +} + +impl fmt::Display for CodePoint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.to_char_lossy().fmt(f) + } +} + +impl CodePoint { + /// Unsafely creates a new `CodePoint` without checking the value. + /// + /// # Safety + /// + /// `value` must be less than or equal to 0x10FFFF. + #[inline] + pub const unsafe fn from_u32_unchecked(value: u32) -> CodePoint { + CodePoint { value } + } + + /// Creates a new `CodePoint` if the value is a valid code point. + /// + /// Returns `None` if `value` is above 0x10FFFF. + #[inline] + pub const fn from_u32(value: u32) -> Option<CodePoint> { + match value { + 0..=0x10FFFF => Some(CodePoint { value }), + _ => None, + } + } + + /// Creates a new `CodePoint` from a `char`. + /// + /// Since all Unicode scalar values are code points, this always succeeds. + #[inline] + pub const fn from_char(value: char) -> CodePoint { + CodePoint { + value: value as u32, + } + } + + /// Returns the numeric value of the code point. + #[inline] + pub const fn to_u32(self) -> u32 { + self.value + } + + /// Returns the numeric value of the code point if it is a leading surrogate. + #[inline] + pub const fn to_lead_surrogate(self) -> Option<LeadSurrogate> { + match self.value { + lead @ 0xD800..=0xDBFF => Some(LeadSurrogate(lead as u16)), + _ => None, + } + } + + /// Returns the numeric value of the code point if it is a trailing surrogate. + #[inline] + pub const fn to_trail_surrogate(self) -> Option<TrailSurrogate> { + match self.value { + trail @ 0xDC00..=0xDFFF => Some(TrailSurrogate(trail as u16)), + _ => None, + } + } + + /// Optionally returns a Unicode scalar value for the code point. + /// + /// Returns `None` if the code point is a surrogate (from U+D800 to U+DFFF). + #[inline] + pub const fn to_char(self) -> Option<char> { + match self.value { + 0xD800..=0xDFFF => None, + _ => Some(unsafe { char::from_u32_unchecked(self.value) }), + } + } + + /// Returns a Unicode scalar value for the code point. + /// + /// Returns `'\u{FFFD}'` (the replacement character “�”) + /// if the code point is a surrogate (from U+D800 to U+DFFF). + #[inline] + pub fn to_char_lossy(self) -> char { + self.to_char().unwrap_or('\u{FFFD}') + } + + pub fn is_char_and(self, f: impl FnOnce(char) -> bool) -> bool { + self.to_char().is_some_and(f) + } + + pub fn encode_wtf8(self, dst: &mut [u8]) -> &mut Wtf8 { + unsafe { Wtf8::from_mut_bytes_unchecked(encode_utf8_raw(self.value, dst)) } + } + + pub const fn len_wtf8(&self) -> usize { + len_utf8(self.value) + } + + pub fn is_ascii(&self) -> bool { + self.is_char_and(|c| c.is_ascii()) + } +} + +impl From<u16> for CodePoint { + fn from(value: u16) -> Self { + unsafe { Self::from_u32_unchecked(value.into()) } + } +} + +impl From<u8> for CodePoint { + fn from(value: u8) -> Self { + char::from(value).into() + } +} + +impl From<char> for CodePoint { + fn from(value: char) -> Self { + Self::from_char(value) + } +} + +impl From<ascii::AsciiChar> for CodePoint { + fn from(value: ascii::AsciiChar) -> Self { + Self::from_char(value.into()) + } +} + +impl From<CodePoint> for Wtf8Buf { + fn from(ch: CodePoint) -> Self { + ch.encode_wtf8(&mut [0; MAX_LEN_UTF8]).to_owned() + } +} + +impl PartialEq<char> for CodePoint { + fn eq(&self, other: &char) -> bool { + self.to_u32() == *other as u32 + } +} +impl PartialEq<CodePoint> for char { + fn eq(&self, other: &CodePoint) -> bool { + *self as u32 == other.to_u32() + } +} + +#[derive(Clone, Copy)] +pub struct LeadSurrogate(u16); + +#[derive(Clone, Copy)] +pub struct TrailSurrogate(u16); + +impl LeadSurrogate { + pub const fn merge(self, trail: TrailSurrogate) -> char { + decode_surrogate_pair(self.0, trail.0) + } +} + +/// An owned, growable string of well-formed WTF-8 data. +/// +/// Similar to `String`, but can additionally contain surrogate code points +/// if they’re not in a surrogate pair. +#[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Default)] +pub struct Wtf8Buf { + bytes: Vec<u8>, +} + +impl ops::Deref for Wtf8Buf { + type Target = Wtf8; + + fn deref(&self) -> &Wtf8 { + self.as_slice() + } +} + +impl ops::DerefMut for Wtf8Buf { + fn deref_mut(&mut self) -> &mut Wtf8 { + self.as_mut_slice() + } +} + +impl Borrow<Wtf8> for Wtf8Buf { + fn borrow(&self) -> &Wtf8 { + self + } +} + +/// Formats the string in double quotes, with characters escaped according to +/// [`char::escape_debug`] and unpaired surrogates represented as `\u{xxxx}`, +/// where each `x` is a hexadecimal digit. +/// +/// For example, the code units [U+0061, U+D800, U+000A] are formatted as +/// `"a\u{D800}\n"`. +impl fmt::Debug for Wtf8Buf { + #[inline] + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, formatter) + } +} + +/// Formats the string with unpaired surrogates substituted with the replacement +/// character, U+FFFD. +impl fmt::Display for Wtf8Buf { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, formatter) + } +} + +impl Wtf8Buf { + /// Creates a new, empty WTF-8 string. + #[inline] + pub fn new() -> Wtf8Buf { + Wtf8Buf::default() + } + + /// Creates a new, empty WTF-8 string with pre-allocated capacity for `capacity` bytes. + #[inline] + pub fn with_capacity(capacity: usize) -> Wtf8Buf { + Wtf8Buf { + bytes: Vec::with_capacity(capacity), + } + } + + /// Creates a WTF-8 string from a WTF-8 byte vec. + /// + /// # Safety + /// + /// `value` must contain valid WTF-8. + #[inline] + pub const unsafe fn from_bytes_unchecked(value: Vec<u8>) -> Wtf8Buf { + Wtf8Buf { bytes: value } + } + + /// Create a WTF-8 string from a WTF-8 byte vec. + pub fn from_bytes(value: Vec<u8>) -> Result<Self, Vec<u8>> { + match Wtf8::from_bytes(&value) { + Some(_) => Ok(unsafe { Self::from_bytes_unchecked(value) }), + None => Err(value), + } + } + + /// Creates a WTF-8 string from a UTF-8 `String`. + /// + /// This takes ownership of the `String` and does not copy. + /// + /// Since WTF-8 is a superset of UTF-8, this always succeeds. + #[inline] + pub fn from_string(string: String) -> Wtf8Buf { + Wtf8Buf { + bytes: string.into_bytes(), + } + } + + pub fn join<I, S>(sep: impl AsRef<Wtf8>, iter: I) -> Wtf8Buf + where + I: IntoIterator<Item = S>, + S: AsRef<Wtf8>, + { + let sep = sep.as_ref(); + let mut iter = iter.into_iter(); + let mut buf = match iter.next() { + Some(first) => first.as_ref().to_owned(), + None => return Wtf8Buf::new(), + }; + for part in iter { + buf.push_wtf8(sep); + buf.push_wtf8(part.as_ref()); + } + buf + } + + pub fn clear(&mut self) { + self.bytes.clear(); + } + + /// Creates a WTF-8 string from a potentially ill-formed UTF-16 slice of 16-bit code units. + /// + /// This is lossless: calling `.encode_wide()` on the resulting string + /// will always return the original code units. + pub fn from_wide(v: &[u16]) -> Wtf8Buf { + let mut string = Wtf8Buf::with_capacity(v.len()); + for item in char::decode_utf16(v.iter().cloned()) { + match item { + Ok(ch) => string.push_char(ch), + Err(surrogate) => { + let surrogate = surrogate.unpaired_surrogate(); + // Surrogates are known to be in the code point range. + let code_point = CodePoint::from(surrogate); + // Skip the WTF-8 concatenation check, + // surrogate pairs are already decoded by decode_utf16 + string.push(code_point); + } + } + } + string + } + + #[inline] + pub fn as_slice(&self) -> &Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(&self.bytes) } + } + + #[inline] + pub fn as_mut_slice(&mut self) -> &mut Wtf8 { + // Safety: `Wtf8` doesn't expose any way to mutate the bytes that would + // cause them to change from well-formed UTF-8 to ill-formed UTF-8, + // which would break the assumptions of the `is_known_utf8` field. + unsafe { Wtf8::from_mut_bytes_unchecked(&mut self.bytes) } + } + + /// Reserves capacity for at least `additional` more bytes to be inserted + /// in the given `Wtf8Buf`. + /// The collection may reserve more space to avoid frequent reallocations. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + #[inline] + pub fn reserve(&mut self, additional: usize) { + self.bytes.reserve(additional) + } + + /// Tries to reserve capacity for at least `additional` more bytes to be + /// inserted in the given `Wtf8Buf`. The `Wtf8Buf` may reserve more space to + /// avoid frequent reallocations. After calling `try_reserve`, capacity will + /// be greater than or equal to `self.len() + additional`. Does nothing if + /// capacity is already sufficient. This method preserves the contents even + /// if an error occurs. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + #[inline] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.bytes.try_reserve(additional) + } + + #[inline] + pub fn reserve_exact(&mut self, additional: usize) { + self.bytes.reserve_exact(additional) + } + + /// Tries to reserve the minimum capacity for exactly `additional` more + /// bytes to be inserted in the given `Wtf8Buf`. After calling + /// `try_reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the `Wtf8Buf` more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: Wtf8Buf::try_reserve + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + #[inline] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.bytes.try_reserve_exact(additional) + } + + #[inline] + pub fn shrink_to_fit(&mut self) { + self.bytes.shrink_to_fit() + } + + #[inline] + pub fn shrink_to(&mut self, min_capacity: usize) { + self.bytes.shrink_to(min_capacity) + } + + #[inline] + pub fn leak<'a>(self) -> &'a mut Wtf8 { + unsafe { Wtf8::from_mut_bytes_unchecked(self.bytes.leak()) } + } + + /// Returns the number of bytes that this string buffer can hold without reallocating. + #[inline] + pub const fn capacity(&self) -> usize { + self.bytes.capacity() + } + + /// Append a UTF-8 slice at the end of the string. + #[inline] + pub fn push_str(&mut self, other: &str) { + self.bytes.extend_from_slice(other.as_bytes()) + } + + /// Append a WTF-8 slice at the end of the string. + #[inline] + pub fn push_wtf8(&mut self, other: &Wtf8) { + self.bytes.extend_from_slice(&other.bytes); + } + + /// Append a Unicode scalar value at the end of the string. + #[inline] + pub fn push_char(&mut self, c: char) { + self.push(CodePoint::from_char(c)) + } + + /// Append a code point at the end of the string. + #[inline] + pub fn push(&mut self, code_point: CodePoint) { + self.push_wtf8(code_point.encode_wtf8(&mut [0; MAX_LEN_UTF8])) + } + + pub fn pop(&mut self) -> Option<CodePoint> { + let ch = self.code_points().next_back()?; + let new_len = self.len() - ch.len_wtf8(); + self.bytes.truncate(new_len); + Some(ch) + } + + /// Shortens a string to the specified length. + /// + /// # Panics + /// + /// Panics if `new_len` > current length, + /// or if `new_len` is not a code point boundary. + #[inline] + pub fn truncate(&mut self, new_len: usize) { + assert!(is_code_point_boundary(self, new_len)); + self.bytes.truncate(new_len) + } + + /// Inserts a codepoint into this `Wtf8Buf` at a byte position. + #[inline] + pub fn insert(&mut self, idx: usize, c: CodePoint) { + self.insert_wtf8(idx, c.encode_wtf8(&mut [0; MAX_LEN_UTF8])) + } + + /// Inserts a WTF-8 slice into this `Wtf8Buf` at a byte position. + #[inline] + pub fn insert_wtf8(&mut self, idx: usize, w: &Wtf8) { + assert!(is_code_point_boundary(self, idx)); + + self.bytes.insert_str(idx, w) + } + + /// Consumes the WTF-8 string and tries to convert it to a vec of bytes. + #[inline] + pub fn into_bytes(self) -> Vec<u8> { + self.bytes + } + + /// Consumes the WTF-8 string and tries to convert it to UTF-8. + /// + /// This does not copy the data. + /// + /// If the contents are not well-formed UTF-8 + /// (that is, if the string contains surrogates), + /// the original WTF-8 string is returned instead. + pub fn into_string(self) -> Result<String, Wtf8Buf> { + if self.is_utf8() { + Ok(unsafe { String::from_utf8_unchecked(self.bytes) }) + } else { + Err(self) + } + } + + /// Consumes the WTF-8 string and converts it lossily to UTF-8. + /// + /// This does not copy the data (but may overwrite parts of it in place). + /// + /// Surrogates are replaced with `"\u{FFFD}"` (the replacement character “�”) + pub fn into_string_lossy(mut self) -> String { + let mut pos = 0; + while let Some((surrogate_pos, _)) = self.next_surrogate(pos) { + pos = surrogate_pos + 3; + // Surrogates and the replacement character are all 3 bytes, so + // they can substituted in-place. + self.bytes[surrogate_pos..pos].copy_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); + } + unsafe { String::from_utf8_unchecked(self.bytes) } + } + + /// Converts this `Wtf8Buf` into a boxed `Wtf8`. + #[inline] + pub fn into_box(self) -> Box<Wtf8> { + // SAFETY: relies on `Wtf8` being `repr(transparent)`. + unsafe { mem::transmute(self.bytes.into_boxed_slice()) } + } + + /// Converts a `Box<Wtf8>` into a `Wtf8Buf`. + pub fn from_box(boxed: Box<Wtf8>) -> Wtf8Buf { + let bytes: Box<[u8]> = unsafe { mem::transmute(boxed) }; + Wtf8Buf { + bytes: bytes.into_vec(), + } + } +} + +/// Creates a new WTF-8 string from an iterator of code points. +/// +/// This replaces surrogate code point pairs with supplementary code points, +/// like concatenating ill-formed UTF-16 strings effectively would. +impl FromIterator<CodePoint> for Wtf8Buf { + fn from_iter<T: IntoIterator<Item = CodePoint>>(iter: T) -> Wtf8Buf { + let mut string = Wtf8Buf::new(); + string.extend(iter); + string + } +} + +/// Append code points from an iterator to the string. +/// +/// This replaces surrogate code point pairs with supplementary code points, +/// like concatenating ill-formed UTF-16 strings effectively would. +impl Extend<CodePoint> for Wtf8Buf { + fn extend<T: IntoIterator<Item = CodePoint>>(&mut self, iter: T) { + let iterator = iter.into_iter(); + let (low, _high) = iterator.size_hint(); + // Lower bound of one byte per code point (ASCII only) + self.bytes.reserve(low); + iterator.for_each(move |code_point| self.push(code_point)); + } +} + +impl Extend<char> for Wtf8Buf { + fn extend<T: IntoIterator<Item = char>>(&mut self, iter: T) { + self.extend(iter.into_iter().map(CodePoint::from)) + } +} + +impl<W: AsRef<Wtf8>> Extend<W> for Wtf8Buf { + fn extend<T: IntoIterator<Item = W>>(&mut self, iter: T) { + iter.into_iter() + .for_each(move |w| self.push_wtf8(w.as_ref())); + } +} + +impl<W: AsRef<Wtf8>> FromIterator<W> for Wtf8Buf { + fn from_iter<T: IntoIterator<Item = W>>(iter: T) -> Self { + let mut buf = Wtf8Buf::new(); + iter.into_iter().for_each(|w| buf.push_wtf8(w.as_ref())); + buf + } +} + +impl Hash for Wtf8Buf { + fn hash<H: Hasher>(&self, state: &mut H) { + Wtf8::hash(self, state) + } +} + +impl AsRef<Wtf8> for Wtf8Buf { + fn as_ref(&self) -> &Wtf8 { + self + } +} + +impl From<String> for Wtf8Buf { + fn from(s: String) -> Self { + Wtf8Buf::from_string(s) + } +} + +impl From<&str> for Wtf8Buf { + fn from(s: &str) -> Self { + Wtf8Buf::from_string(s.to_owned()) + } +} + +impl From<ascii::AsciiString> for Wtf8Buf { + fn from(s: ascii::AsciiString) -> Self { + Wtf8Buf::from_string(s.into()) + } +} + +/// A borrowed slice of well-formed WTF-8 data. +/// +/// Similar to `&str`, but can additionally contain surrogate code points +/// if they’re not in a surrogate pair. +#[derive(PartialEq, Eq, PartialOrd, Ord)] +pub struct Wtf8 { + bytes: [u8], +} + +impl AsRef<Wtf8> for Wtf8 { + fn as_ref(&self) -> &Wtf8 { + self + } +} + +impl ToOwned for Wtf8 { + type Owned = Wtf8Buf; + + fn to_owned(&self) -> Self::Owned { + self.to_wtf8_buf() + } + + fn clone_into(&self, buf: &mut Self::Owned) { + self.bytes.clone_into(&mut buf.bytes); + } +} + +impl PartialEq<str> for Wtf8 { + fn eq(&self, other: &str) -> bool { + self.as_bytes().eq(other.as_bytes()) + } +} + +/// Formats the string in double quotes, with characters escaped according to +/// [`char::escape_debug`] and unpaired surrogates represented as `\u{xxxx}`, +/// where each `x` is a hexadecimal digit. +impl fmt::Debug for Wtf8 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fn write_str_escaped(f: &mut fmt::Formatter<'_>, s: &str) -> fmt::Result { + use core::fmt::Write; + for c in s.chars().flat_map(|c| c.escape_debug()) { + f.write_char(c)? + } + Ok(()) + } + + formatter.write_str("\"")?; + let mut pos = 0; + while let Some((surrogate_pos, surrogate)) = self.next_surrogate(pos) { + write_str_escaped(formatter, unsafe { + str::from_utf8_unchecked(&self.bytes[pos..surrogate_pos]) + })?; + write!(formatter, "\\u{{{surrogate:x}}}")?; + pos = surrogate_pos + 3; + } + write_str_escaped(formatter, unsafe { + str::from_utf8_unchecked(&self.bytes[pos..]) + })?; + formatter.write_str("\"") + } +} + +/// Formats the string with unpaired surrogates substituted with the replacement +/// character, U+FFFD. +impl fmt::Display for Wtf8 { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let wtf8_bytes = &self.bytes; + let mut pos = 0; + loop { + match self.next_surrogate(pos) { + Some((surrogate_pos, _)) => { + formatter.write_str(unsafe { + str::from_utf8_unchecked(&wtf8_bytes[pos..surrogate_pos]) + })?; + formatter.write_str(UTF8_REPLACEMENT_CHARACTER)?; + pos = surrogate_pos + 3; + } + None => { + let s = unsafe { str::from_utf8_unchecked(&wtf8_bytes[pos..]) }; + if pos == 0 { + return s.fmt(formatter); + } else { + return formatter.write_str(s); + } + } + } + } + } +} + +impl Default for &Wtf8 { + fn default() -> Self { + unsafe { Wtf8::from_bytes_unchecked(&[]) } + } +} + +impl Hash for Wtf8 { + fn hash<H: Hasher>(&self, state: &mut H) { + state.write(self.as_bytes()); + state.write_u8(0xff); + } +} + +impl Wtf8 { + /// Creates a WTF-8 slice from a UTF-8 `&str` slice. + /// + /// Since WTF-8 is a superset of UTF-8, this always succeeds. + #[inline] + pub fn new<S: AsRef<Wtf8> + ?Sized>(value: &S) -> &Wtf8 { + value.as_ref() + } + + /// Creates a WTF-8 slice from a WTF-8 byte slice. + /// + /// # Safety + /// + /// `value` must contain valid WTF-8. + #[inline] + pub const unsafe fn from_bytes_unchecked(value: &[u8]) -> &Wtf8 { + // SAFETY: start with &[u8], end with fancy &[u8] + unsafe { &*(value as *const [u8] as *const Wtf8) } + } + + /// Creates a mutable WTF-8 slice from a mutable WTF-8 byte slice. + /// + /// Since the byte slice is not checked for valid WTF-8, this functions is + /// marked unsafe. + #[inline] + const unsafe fn from_mut_bytes_unchecked(value: &mut [u8]) -> &mut Wtf8 { + // SAFETY: start with &mut [u8], end with fancy &mut [u8] + unsafe { &mut *(value as *mut [u8] as *mut Wtf8) } + } + + /// Create a WTF-8 slice from a WTF-8 byte slice. + // + // whoops! using WTF-8 for interchange! + #[inline] + pub fn from_bytes(b: &[u8]) -> Option<&Self> { + let mut rest = b; + while let Err(e) = core::str::from_utf8(rest) { + rest = &rest[e.valid_up_to()..]; + let _ = Self::decode_surrogate(rest)?; + rest = &rest[3..]; + } + Some(unsafe { Wtf8::from_bytes_unchecked(b) }) + } + + fn decode_surrogate(b: &[u8]) -> Option<CodePoint> { + let [0xed, b2 @ (0xa0..), b3, ..] = *b else { + return None; + }; + Some(decode_surrogate(b2, b3).into()) + } + + /// Returns the length, in WTF-8 bytes. + #[inline] + pub const fn len(&self) -> usize { + self.bytes.len() + } + + #[inline] + pub const fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// Returns the code point at `position` if it is in the ASCII range, + /// or `b'\xFF'` otherwise. + /// + /// # Panics + /// + /// Panics if `position` is beyond the end of the string. + #[inline] + pub const fn ascii_byte_at(&self, position: usize) -> u8 { + match self.bytes[position] { + ascii_byte @ 0x00..=0x7F => ascii_byte, + _ => 0xFF, + } + } + + /// Returns an iterator for the string’s code points. + #[inline] + pub fn code_points(&self) -> Wtf8CodePoints<'_> { + Wtf8CodePoints { + bytes: self.bytes.iter(), + } + } + + /// Returns an iterator for the string’s code points and their indices. + #[inline] + pub fn code_point_indices(&self) -> Wtf8CodePointIndices<'_> { + Wtf8CodePointIndices { + front_offset: 0, + iter: self.code_points(), + } + } + + /// Access raw bytes of WTF-8 data + #[inline] + pub const fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + /// Tries to convert the string to UTF-8 and return a `&str` slice. + /// + /// Returns `None` if the string contains surrogates. + /// + /// This does not copy the data. + #[inline] + pub const fn as_str(&self) -> Result<&str, str::Utf8Error> { + str::from_utf8(&self.bytes) + } + + /// Creates an owned `Wtf8Buf` from a borrowed `Wtf8`. + pub fn to_wtf8_buf(&self) -> Wtf8Buf { + Wtf8Buf { + bytes: self.bytes.to_vec(), + } + } + + /// Lossily converts the string to UTF-8. + /// Returns a UTF-8 `&str` slice if the contents are well-formed in UTF-8. + /// + /// Surrogates are replaced with `"\u{FFFD}"` (the replacement character “�”). + /// + /// This only copies the data if necessary (if it contains any surrogate). + pub fn to_string_lossy(&self) -> Cow<'_, str> { + let Some((surrogate_pos, _)) = self.next_surrogate(0) else { + return Cow::Borrowed(unsafe { str::from_utf8_unchecked(&self.bytes) }); + }; + let wtf8_bytes = &self.bytes; + let mut utf8_bytes = Vec::with_capacity(self.len()); + utf8_bytes.extend_from_slice(&wtf8_bytes[..surrogate_pos]); + utf8_bytes.extend_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); + let mut pos = surrogate_pos + 3; + loop { + match self.next_surrogate(pos) { + Some((surrogate_pos, _)) => { + utf8_bytes.extend_from_slice(&wtf8_bytes[pos..surrogate_pos]); + utf8_bytes.extend_from_slice(UTF8_REPLACEMENT_CHARACTER.as_bytes()); + pos = surrogate_pos + 3; + } + None => { + utf8_bytes.extend_from_slice(&wtf8_bytes[pos..]); + return Cow::Owned(unsafe { String::from_utf8_unchecked(utf8_bytes) }); + } + } + } + } + + /// Converts the WTF-8 string to potentially ill-formed UTF-16 + /// and return an iterator of 16-bit code units. + /// + /// This is lossless: + /// calling `Wtf8Buf::from_ill_formed_utf16` on the resulting code units + /// would always return the original WTF-8 string. + #[inline] + pub fn encode_wide(&self) -> EncodeWide<'_> { + EncodeWide { + code_points: self.code_points(), + extra: 0, + } + } + + pub const fn chunks(&self) -> Wtf8Chunks<'_> { + Wtf8Chunks { wtf8: self } + } + + pub fn map_utf8<'a, I>(&'a self, f: impl Fn(&'a str) -> I) -> impl Iterator<Item = CodePoint> + where + I: Iterator<Item = char>, + { + self.chunks().flat_map(move |chunk| match chunk { + Wtf8Chunk::Utf8(s) => Either::Left(f(s).map_into()), + Wtf8Chunk::Surrogate(c) => Either::Right(core::iter::once(c)), + }) + } + + #[inline] + fn next_surrogate(&self, mut pos: usize) -> Option<(usize, u16)> { + let mut iter = self.bytes[pos..].iter(); + loop { + let b = *iter.next()?; + if b < 0x80 { + pos += 1; + } else if b < 0xE0 { + iter.next(); + pos += 2; + } else if b == 0xED { + match (iter.next(), iter.next()) { + (Some(&b2), Some(&b3)) if b2 >= 0xA0 => { + return Some((pos, decode_surrogate(b2, b3))); + } + _ => pos += 3, + } + } else if b < 0xF0 { + iter.next(); + iter.next(); + pos += 3; + } else { + iter.next(); + iter.next(); + iter.next(); + pos += 4; + } + } + } + + pub fn is_code_point_boundary(&self, index: usize) -> bool { + is_code_point_boundary(self, index) + } + + /// Boxes this `Wtf8`. + #[inline] + pub fn into_box(&self) -> Box<Wtf8> { + let boxed: Box<[u8]> = self.bytes.into(); + unsafe { mem::transmute(boxed) } + } + + /// Creates a boxed, empty `Wtf8`. + pub fn empty_box() -> Box<Wtf8> { + let boxed: Box<[u8]> = Default::default(); + unsafe { mem::transmute(boxed) } + } + + #[inline] + pub fn make_ascii_lowercase(&mut self) { + self.bytes.make_ascii_lowercase() + } + + #[inline] + pub fn make_ascii_uppercase(&mut self) { + self.bytes.make_ascii_uppercase() + } + + #[inline] + pub fn to_ascii_lowercase(&self) -> Wtf8Buf { + Wtf8Buf { + bytes: self.bytes.to_ascii_lowercase(), + } + } + + #[inline] + pub fn to_ascii_uppercase(&self) -> Wtf8Buf { + Wtf8Buf { + bytes: self.bytes.to_ascii_uppercase(), + } + } + + pub fn to_lowercase(&self) -> Wtf8Buf { + let mut buf = Wtf8Buf::with_capacity(self.len()); + for chunk in self.chunks() { + match chunk { + Wtf8Chunk::Utf8(s) => buf.push_str(&s.to_lowercase()), + Wtf8Chunk::Surrogate(c) => buf.push(c), + } + } + buf + } + + pub fn to_uppercase(&self) -> Wtf8Buf { + let mut buf = Wtf8Buf::with_capacity(self.len()); + for chunk in self.chunks() { + match chunk { + Wtf8Chunk::Utf8(s) => buf.push_str(&s.to_uppercase()), + Wtf8Chunk::Surrogate(c) => buf.push(c), + } + } + buf + } + + #[inline] + pub const fn is_ascii(&self) -> bool { + self.bytes.is_ascii() + } + + #[inline] + pub fn is_utf8(&self) -> bool { + self.next_surrogate(0).is_none() + } + + #[inline] + pub fn eq_ignore_ascii_case(&self, other: &Self) -> bool { + self.bytes.eq_ignore_ascii_case(&other.bytes) + } + + pub fn split(&self, pat: &Wtf8) -> impl Iterator<Item = &Self> { + self.as_bytes() + .split_str(pat) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn splitn(&self, n: usize, pat: &Wtf8) -> impl Iterator<Item = &Self> { + self.as_bytes() + .splitn_str(n, pat) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn rsplit(&self, pat: &Wtf8) -> impl Iterator<Item = &Self> { + self.as_bytes() + .rsplit_str(pat) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn rsplitn(&self, n: usize, pat: &Wtf8) -> impl Iterator<Item = &Self> { + self.as_bytes() + .rsplitn_str(n, pat) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn trim(&self) -> &Self { + let w = self.bytes.trim(); + unsafe { Wtf8::from_bytes_unchecked(w) } + } + + pub fn trim_start(&self) -> &Self { + let w = self.bytes.trim_start(); + unsafe { Wtf8::from_bytes_unchecked(w) } + } + + pub fn trim_end(&self) -> &Self { + let w = self.bytes.trim_end(); + unsafe { Wtf8::from_bytes_unchecked(w) } + } + + pub fn trim_start_matches(&self, f: impl Fn(CodePoint) -> bool) -> &Self { + let mut iter = self.code_points(); + loop { + let old = iter.clone(); + match iter.next().map(&f) { + Some(true) => continue, + Some(false) => { + iter = old; + break; + } + None => return iter.as_wtf8(), + } + } + iter.as_wtf8() + } + + pub fn trim_end_matches(&self, f: impl Fn(CodePoint) -> bool) -> &Self { + let mut iter = self.code_points(); + loop { + let old = iter.clone(); + match iter.next_back().map(&f) { + Some(true) => continue, + Some(false) => { + iter = old; + break; + } + None => return iter.as_wtf8(), + } + } + iter.as_wtf8() + } + + pub fn trim_matches(&self, f: impl Fn(CodePoint) -> bool) -> &Self { + self.trim_start_matches(&f).trim_end_matches(&f) + } + + pub fn find(&self, pat: &Wtf8) -> Option<usize> { + memchr::memmem::find(self.as_bytes(), pat.as_bytes()) + } + + pub fn rfind(&self, pat: &Wtf8) -> Option<usize> { + memchr::memmem::rfind(self.as_bytes(), pat.as_bytes()) + } + + pub fn find_iter(&self, pat: &Wtf8) -> impl Iterator<Item = usize> { + memchr::memmem::find_iter(self.as_bytes(), pat.as_bytes()) + } + + pub fn rfind_iter(&self, pat: &Wtf8) -> impl Iterator<Item = usize> { + memchr::memmem::rfind_iter(self.as_bytes(), pat.as_bytes()) + } + + pub fn contains(&self, pat: &Wtf8) -> bool { + self.bytes.contains_str(pat) + } + + pub fn contains_code_point(&self, pat: CodePoint) -> bool { + self.bytes + .contains_str(pat.encode_wtf8(&mut [0; MAX_LEN_UTF8])) + } + + pub fn get(&self, range: impl ops::RangeBounds<usize>) -> Option<&Self> { + let start = match range.start_bound() { + ops::Bound::Included(&i) => i, + ops::Bound::Excluded(&i) => i.saturating_add(1), + ops::Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + ops::Bound::Included(&i) => i.saturating_add(1), + ops::Bound::Excluded(&i) => i, + ops::Bound::Unbounded => self.len(), + }; + // is_code_point_boundary checks that the index is in [0, .len()] + if start <= end && is_code_point_boundary(self, start) && is_code_point_boundary(self, end) + { + Some(unsafe { slice_unchecked(self, start, end) }) + } else { + None + } + } + + pub fn ends_with(&self, w: impl AsRef<Wtf8>) -> bool { + self.bytes.ends_with_str(w.as_ref()) + } + + pub fn starts_with(&self, w: impl AsRef<Wtf8>) -> bool { + self.bytes.starts_with_str(w.as_ref()) + } + + pub fn strip_prefix(&self, w: impl AsRef<Wtf8>) -> Option<&Self> { + self.bytes + .strip_prefix(w.as_ref().as_bytes()) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn strip_suffix(&self, w: impl AsRef<Wtf8>) -> Option<&Self> { + self.bytes + .strip_suffix(w.as_ref().as_bytes()) + .map(|w| unsafe { Wtf8::from_bytes_unchecked(w) }) + } + + pub fn replace(&self, from: &Wtf8, to: &Wtf8) -> Wtf8Buf { + let w = self.bytes.replace(from, to); + unsafe { Wtf8Buf::from_bytes_unchecked(w) } + } + + pub fn replacen(&self, from: &Wtf8, to: &Wtf8, n: usize) -> Wtf8Buf { + let w = self.bytes.replacen(from, to, n); + unsafe { Wtf8Buf::from_bytes_unchecked(w) } + } +} + +impl AsRef<Wtf8> for str { + fn as_ref(&self) -> &Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(self.as_bytes()) } + } +} + +impl AsRef<[u8]> for Wtf8 { + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +/// Returns a slice of the given string for the byte range \[`begin`..`end`). +/// +/// # Panics +/// +/// Panics when `begin` and `end` do not point to code point boundaries, +/// or point beyond the end of the string. +impl ops::Index<ops::Range<usize>> for Wtf8 { + type Output = Wtf8; + + #[inline] + #[track_caller] + fn index(&self, range: ops::Range<usize>) -> &Wtf8 { + // is_code_point_boundary checks that the index is in [0, .len()] + if range.start <= range.end + && is_code_point_boundary(self, range.start) + && is_code_point_boundary(self, range.end) + { + unsafe { slice_unchecked(self, range.start, range.end) } + } else { + slice_error_fail(self, range.start, range.end) + } + } +} + +/// Returns a slice of the given string from byte `begin` to its end. +/// +/// # Panics +/// +/// Panics when `begin` is not at a code point boundary, +/// or is beyond the end of the string. +impl ops::Index<ops::RangeFrom<usize>> for Wtf8 { + type Output = Wtf8; + + #[inline] + #[track_caller] + fn index(&self, range: ops::RangeFrom<usize>) -> &Wtf8 { + // is_code_point_boundary checks that the index is in [0, .len()] + if is_code_point_boundary(self, range.start) { + unsafe { slice_unchecked(self, range.start, self.len()) } + } else { + slice_error_fail(self, range.start, self.len()) + } + } +} + +/// Returns a slice of the given string from its beginning to byte `end`. +/// +/// # Panics +/// +/// Panics when `end` is not at a code point boundary, +/// or is beyond the end of the string. +impl ops::Index<ops::RangeTo<usize>> for Wtf8 { + type Output = Wtf8; + + #[inline] + #[track_caller] + fn index(&self, range: ops::RangeTo<usize>) -> &Wtf8 { + // is_code_point_boundary checks that the index is in [0, .len()] + if is_code_point_boundary(self, range.end) { + unsafe { slice_unchecked(self, 0, range.end) } + } else { + slice_error_fail(self, 0, range.end) + } + } +} + +impl ops::Index<ops::RangeFull> for Wtf8 { + type Output = Wtf8; + + #[inline] + fn index(&self, _range: ops::RangeFull) -> &Wtf8 { + self + } +} + +#[inline] +const fn decode_surrogate(second_byte: u8, third_byte: u8) -> u16 { + // The first byte is assumed to be 0xED + 0xD800 | (second_byte as u16 & 0x3F) << 6 | third_byte as u16 & 0x3F +} + +#[inline] +const fn decode_surrogate_pair(lead: u16, trail: u16) -> char { + let code_point = 0x10000 + ((((lead - 0xD800) as u32) << 10) | (trail - 0xDC00) as u32); + unsafe { char::from_u32_unchecked(code_point) } +} + +/// Copied from str::is_char_boundary +#[inline] +fn is_code_point_boundary(slice: &Wtf8, index: usize) -> bool { + if index == 0 { + return true; + } + match slice.bytes.get(index) { + None => index == slice.len(), + Some(&b) => (b as i8) >= -0x40, + } +} + +/// Verify that `index` is at the edge of either a valid UTF-8 codepoint +/// (i.e. a codepoint that's not a surrogate) or of the whole string. +/// +/// These are the cases currently permitted by `OsStr::slice_encoded_bytes`. +/// Splitting between surrogates is valid as far as WTF-8 is concerned, but +/// we do not permit it in the public API because WTF-8 is considered an +/// implementation detail. +#[track_caller] +#[inline] +pub fn check_utf8_boundary(slice: &Wtf8, index: usize) { + if index == 0 { + return; + } + match slice.bytes.get(index) { + Some(0xED) => (), // Might be a surrogate + Some(&b) if (b as i8) >= -0x40 => return, + Some(_) => panic!("byte index {index} is not a codepoint boundary"), + None if index == slice.len() => return, + None => panic!("byte index {index} is out of bounds"), + } + if slice.bytes[index + 1] >= 0xA0 { + // There's a surrogate after index. Now check before index. + if index >= 3 && slice.bytes[index - 3] == 0xED && slice.bytes[index - 2] >= 0xA0 { + panic!("byte index {index} lies between surrogate codepoints"); + } + } +} + +/// Copied from core::str::raw::slice_unchecked +/// +/// # Safety +/// +/// `begin` and `end` must be within bounds and on codepoint boundaries. +#[inline] +pub const unsafe fn slice_unchecked(s: &Wtf8, begin: usize, end: usize) -> &Wtf8 { + // SAFETY: memory layout of a &[u8] and &Wtf8 are the same + unsafe { + let len = end - begin; + let start = s.as_bytes().as_ptr().add(begin); + Wtf8::from_bytes_unchecked(slice::from_raw_parts(start, len)) + } +} + +/// Copied from core::str::raw::slice_error_fail +#[inline(never)] +#[track_caller] +pub fn slice_error_fail(s: &Wtf8, begin: usize, end: usize) -> ! { + assert!(begin <= end); + panic!("index {begin} and/or {end} in `{s:?}` do not lie on character boundary"); +} + +/// Iterator for the code points of a WTF-8 string. +/// +/// Created with the method `.code_points()`. +#[derive(Clone)] +pub struct Wtf8CodePoints<'a> { + bytes: slice::Iter<'a, u8>, +} + +impl Iterator for Wtf8CodePoints<'_> { + type Item = CodePoint; + + #[inline] + fn next(&mut self) -> Option<CodePoint> { + // SAFETY: `self.bytes` has been created from a WTF-8 string + unsafe { next_code_point(&mut self.bytes).map(|c| CodePoint { value: c }) } + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + let len = self.bytes.len(); + (len.saturating_add(3) / 4, Some(len)) + } + + fn last(mut self) -> Option<Self::Item> { + self.next_back() + } + + fn count(self) -> usize { + core_str_count::count_chars(self.as_wtf8()) + } +} + +impl DoubleEndedIterator for Wtf8CodePoints<'_> { + #[inline] + fn next_back(&mut self) -> Option<CodePoint> { + // SAFETY: `str` invariant says `self.iter` is a valid WTF-8 string and + // the resulting `ch` is a valid Unicode Code Point. + unsafe { + next_code_point_reverse(&mut self.bytes).map(|ch| CodePoint::from_u32_unchecked(ch)) + } + } +} + +impl<'a> Wtf8CodePoints<'a> { + pub fn as_wtf8(&self) -> &'a Wtf8 { + unsafe { Wtf8::from_bytes_unchecked(self.bytes.as_slice()) } + } +} + +#[derive(Clone)] +pub struct Wtf8CodePointIndices<'a> { + front_offset: usize, + iter: Wtf8CodePoints<'a>, +} + +impl Iterator for Wtf8CodePointIndices<'_> { + type Item = (usize, CodePoint); + + #[inline] + fn next(&mut self) -> Option<(usize, CodePoint)> { + let pre_len = self.iter.bytes.len(); + match self.iter.next() { + None => None, + Some(ch) => { + let index = self.front_offset; + let len = self.iter.bytes.len(); + self.front_offset += pre_len - len; + Some((index, ch)) + } + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + self.iter.size_hint() + } + + #[inline] + fn last(mut self) -> Option<(usize, CodePoint)> { + // No need to go through the entire string. + self.next_back() + } + + #[inline] + fn count(self) -> usize { + self.iter.count() + } +} + +impl DoubleEndedIterator for Wtf8CodePointIndices<'_> { + #[inline] + fn next_back(&mut self) -> Option<(usize, CodePoint)> { + self.iter.next_back().map(|ch| { + let index = self.front_offset + self.iter.bytes.len(); + (index, ch) + }) + } +} + +impl FusedIterator for Wtf8CodePointIndices<'_> {} + +/// Generates a wide character sequence for potentially ill-formed UTF-16. +#[derive(Clone)] +pub struct EncodeWide<'a> { + code_points: Wtf8CodePoints<'a>, + extra: u16, +} + +// Copied from libunicode/u_str.rs +impl Iterator for EncodeWide<'_> { + type Item = u16; + + #[inline] + fn next(&mut self) -> Option<u16> { + if self.extra != 0 { + let tmp = self.extra; + self.extra = 0; + return Some(tmp); + } + + let mut buf = [0; MAX_LEN_UTF16]; + self.code_points.next().map(|code_point| { + let n = encode_utf16_raw(code_point.value, &mut buf).len(); + if n == 2 { + self.extra = buf[1]; + } + buf[0] + }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + let (low, high) = self.code_points.size_hint(); + let ext = (self.extra != 0) as usize; + // every code point gets either one u16 or two u16, + // so this iterator is between 1 or 2 times as + // long as the underlying iterator. + ( + low + ext, + high.and_then(|n| n.checked_mul(2)) + .and_then(|n| n.checked_add(ext)), + ) + } +} + +impl FusedIterator for EncodeWide<'_> {} + +pub struct Wtf8Chunks<'a> { + wtf8: &'a Wtf8, +} + +impl<'a> Iterator for Wtf8Chunks<'a> { + type Item = Wtf8Chunk<'a>; + + fn next(&mut self) -> Option<Self::Item> { + match self.wtf8.next_surrogate(0) { + Some((0, surrogate)) => { + self.wtf8 = &self.wtf8[3..]; + Some(Wtf8Chunk::Surrogate(surrogate.into())) + } + Some((n, _)) => { + let s = unsafe { str::from_utf8_unchecked(&self.wtf8.as_bytes()[..n]) }; + self.wtf8 = &self.wtf8[n..]; + Some(Wtf8Chunk::Utf8(s)) + } + None => { + let s = + unsafe { str::from_utf8_unchecked(core::mem::take(&mut self.wtf8).as_bytes()) }; + (!s.is_empty()).then_some(Wtf8Chunk::Utf8(s)) + } + } + } +} + +pub enum Wtf8Chunk<'a> { + Utf8(&'a str), + Surrogate(CodePoint), +} + +impl Hash for CodePoint { + #[inline] + fn hash<H: Hasher>(&self, state: &mut H) { + self.value.hash(state) + } +} + +// == BOX IMPLS == + +/// # Safety +/// +/// `value` must be valid WTF-8. +pub unsafe fn from_boxed_wtf8_unchecked(value: Box<[u8]>) -> Box<Wtf8> { + unsafe { Box::from_raw(Box::into_raw(value) as *mut Wtf8) } +} + +impl Clone for Box<Wtf8> { + fn clone(&self) -> Self { + (&**self).into() + } +} + +impl Default for Box<Wtf8> { + fn default() -> Self { + unsafe { from_boxed_wtf8_unchecked(Box::default()) } + } +} + +impl From<&Wtf8> for Box<Wtf8> { + fn from(w: &Wtf8) -> Self { + w.into_box() + } +} + +impl<'a> From<&'a str> for &'a Wtf8 { + #[inline] + fn from(s: &'a str) -> &'a Wtf8 { + // Valid UTF-8 is always valid WTF-8 + unsafe { Wtf8::from_bytes_unchecked(s.as_bytes()) } + } +} + +impl From<&str> for Box<Wtf8> { + fn from(s: &str) -> Self { + Box::<str>::from(s).into() + } +} + +impl From<Box<str>> for Box<Wtf8> { + fn from(s: Box<str>) -> Self { + unsafe { from_boxed_wtf8_unchecked(s.into_boxed_bytes()) } + } +} + +impl From<Box<ascii::AsciiStr>> for Box<Wtf8> { + fn from(s: Box<ascii::AsciiStr>) -> Self { + <Box<str>>::from(s).into() + } +} + +impl From<Box<Wtf8>> for Box<[u8]> { + fn from(w: Box<Wtf8>) -> Self { + unsafe { Box::from_raw(Box::into_raw(w) as *mut [u8]) } + } +} + +impl From<Wtf8Buf> for Box<Wtf8> { + fn from(w: Wtf8Buf) -> Self { + w.into_box() + } +} + +impl From<Box<Wtf8>> for Wtf8Buf { + fn from(w: Box<Wtf8>) -> Self { + Wtf8Buf::from_box(w) + } +} + +impl From<String> for Box<Wtf8> { + fn from(s: String) -> Self { + s.into_boxed_str().into() + } +} + +mod concat; +pub use concat::Wtf8Concat; diff --git a/crawl_sourcecode.py b/crawl_sourcecode.py deleted file mode 100644 index 2daad4f682f..00000000000 --- a/crawl_sourcecode.py +++ /dev/null @@ -1,83 +0,0 @@ -""" This script can be used to test the equivalence in parsing between -rustpython and cpython. - -Usage example: - -$ python crawl_sourcecode.py crawl_sourcecode.py > cpython.txt -$ cargo run crawl_sourcecode.py crawl_sourcecode.py > rustpython.txt -$ diff cpython.txt rustpython.txt -""" - - -import ast -import sys -import symtable -import dis - -filename = sys.argv[1] -print('Crawling file:', filename) - - -with open(filename, 'r') as f: - source = f.read() - -t = ast.parse(source) -print(t) - -shift = 3 -def print_node(node, indent=0): - indents = ' ' * indent - if isinstance(node, ast.AST): - lineno = 'row={}'.format(node.lineno) if hasattr(node, 'lineno') else '' - print(indents, "NODE", node.__class__.__name__, lineno) - for field in node._fields: - print(indents,'-', field) - f = getattr(node, field) - if isinstance(f, list): - for f2 in f: - print_node(f2, indent=indent+shift) - else: - print_node(f, indent=indent+shift) - else: - print(indents, 'OBJ', node) - -print_node(t) - -# print(ast.dump(t)) -flag_names = [ - 'is_referenced', - 'is_assigned', - 'is_global', - 'is_local', - 'is_parameter', - 'is_free', -] - -def print_table(table, indent=0): - indents = ' ' * indent - print(indents, 'table:', table.get_name()) - print(indents, ' ', 'name:', table.get_name()) - print(indents, ' ', 'type:', table.get_type()) - print(indents, ' ', 'line:', table.get_lineno()) - print(indents, ' ', 'identifiers:', table.get_identifiers()) - print(indents, ' ', 'Syms:') - for sym in table.get_symbols(): - flags = [] - for flag_name in flag_names: - func = getattr(sym, flag_name) - if func(): - flags.append(flag_name) - print(indents, ' sym:', sym.get_name(), 'flags:', ' '.join(flags)) - if table.has_children(): - print(indents, ' ', 'Child tables:') - for child in table.get_children(): - print_table(child, indent=indent+shift) - -table = symtable.symtable(source, 'a', 'exec') -print_table(table) - -print() -print('======== dis.dis ========') -print() -co = compile(source, filename, 'exec') -dis.dis(co) diff --git a/demo_closures.py b/demo_closures.py deleted file mode 100644 index 00242407e67..00000000000 --- a/demo_closures.py +++ /dev/null @@ -1,13 +0,0 @@ - - -def foo(x): - def bar(z): - return z + x - return bar - -f = foo(9) -g = foo(10) - -print(f(2)) -print(g(2)) - diff --git a/deny.toml b/deny.toml new file mode 100644 index 00000000000..85513455fcc --- /dev/null +++ b/deny.toml @@ -0,0 +1,245 @@ +# This template contains all of the possible sections and their default values + +# Note that all fields that take a lint level have these possible values: +# * deny - An error will be produced and the check will fail +# * warn - A warning will be produced, but the check will not fail +# * allow - No warning or error will be produced, though in some cases a note +# will be + +# The values provided in this template are the default values that will be used +# when any section or field is not specified in your own configuration + +# Root options + +# The graph table configures how the dependency graph is constructed and thus +# which crates the checks are performed against +[graph] +# If 1 or more target triples (and optionally, target_features) are specified, +# only the specified targets will be checked when running `cargo deny check`. +# This means, if a particular package is only ever used as a target specific +# dependency, such as, for example, the `nix` crate only being used via the +# `target_family = "unix"` configuration, that only having windows targets in +# this list would mean the nix crate, as well as any of its exclusive +# dependencies not shared by any other crates, would be ignored, as the target +# list here is effectively saying which targets you are building for. +targets = [ + # The triple can be any string, but only the target triples built in to + # rustc (as of 1.40) can be checked against actual config expressions + #"x86_64-unknown-linux-musl", + # You can also specify which target_features you promise are enabled for a + # particular target. target_features are currently not validated against + # the actual valid features supported by the target architecture. + #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, +] +# When creating the dependency graph used as the source of truth when checks are +# executed, this field can be used to prune crates from the graph, removing them +# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate +# is pruned from the graph, all of its dependencies will also be pruned unless +# they are connected to another crate in the graph that hasn't been pruned, +# so it should be used with care. The identifiers are [Package ID Specifications] +# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) +#exclude = [] +# If true, metadata will be collected with `--all-features`. Note that this can't +# be toggled off if true, if you want to conditionally enable `--all-features` it +# is recommended to pass `--all-features` on the cmd line instead +all-features = false +# If true, metadata will be collected with `--no-default-features`. The same +# caveat with `all-features` applies +no-default-features = false +# If set, these feature will be enabled when collecting metadata. If `--features` +# is specified on the cmd line they will take precedence over this option. +#features = [] + +# The output table provides options for how/if diagnostics are outputted +[output] +# When outputting inclusion graphs in diagnostics that include features, this +# option can be used to specify the depth at which feature edges will be added. +# This option is included since the graphs can be quite large and the addition +# of features from the crate(s) to all of the graph roots can be far too verbose. +# This option can be overridden via `--feature-depth` on the cmd line +feature-depth = 1 + +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# The path where the advisory databases are cloned/fetched into +#db-path = "$CARGO_HOME/advisory-dbs" +# The url(s) of the advisory databases to use +#db-urls = ["https://github.com/rustsec/advisory-db"] +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + #"RUSTSEC-0000-0000", + #{ id = "RUSTSEC-0000-0000", reason = "you can specify a reason the advisory is ignored" }, + #"a-crate-that-is-yanked@0.1.1", # you can also ignore yanked crate versions if you wish + #{ crate = "a-crate-that-is-yanked@0.1.1", reason = "you can specify why you are ignoring the yanked crate" }, +] +# If this is true, then cargo deny will use the git executable to fetch advisory database. +# If this is false, then it uses a built-in git library. +# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. +# See Git Authentication for more information about setting up git authentication. +#git-fetch-with-cli = true + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSL-1.0", + "LGPL-3.0", + "Zlib", + "Unicode-DFS-2016", + "CC0-1.0", + "BSD-2-Clause", + "BSD-3-Clause", + "Python-2.0.1" +] +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.8 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # Each entry is the crate and version constraint, and its specific allow + # list + #{ allow = ["Zlib"], crate = "adler32" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +#[[licenses.clarify]] +# The package spec the clarification applies to +#crate = "ring" +# The SPDX expression for the license requirements of the crate +#expression = "MIT AND ISC AND OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +#license-files = [ +# Each entry is a crate relative path, and the (opaque) hash of its contents +#{ path = "LICENSE", hash = 0xbd0eed23 } +#] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries. +# To see how to mark a crate as unpublished (to the official registry), +# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. +ignore = false +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +registries = [ + #"https://sekretz.com/registry +] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# Lint level for when a crate version requirement is `*` +wildcards = "allow" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "all" +# The default lint level for `default` features for crates that are members of +# the workspace that is being checked. This can be overridden by allowing/denying +# `default` on a crate-by-crate basis if desired. +workspace-default-features = "allow" +# The default lint level for `default` features for external crates that are not +# members of the workspace. This can be overridden by allowing/denying `default` +# on a crate-by-crate basis if desired. +external-default-features = "allow" +# List of crates that are allowed. Use with care! +allow = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is allowed" }, +] +# List of crates to deny +deny = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is banned" }, + # Wrapper crates can optionally be specified to allow the crate when it + # is a direct dependency of the otherwise banned crate + #{ crate = "ansi_term@0.11.0", wrappers = ["this-crate-directly-depends-on-ansi_term"] }, +] + +# List of features to allow/deny +# Each entry the name of a crate and a version range. If version is +# not specified, all versions will be matched. +#[[bans.features]] +#crate = "reqwest" +# Features to not allow +#deny = ["json"] +# Features to allow +#allow = [ +# "rustls", +# "__rustls", +# "__tls", +# "hyper-rustls", +# "rustls", +# "rustls-pemfile", +# "rustls-tls-webpki-roots", +# "tokio-rustls", +# "webpki-roots", +#] +# If true, the allowed features must exactly match the enabled feature set. If +# this is set there is no point setting `deny` +#exact = true + +# Certain crates/versions that will be skipped when doing duplicate detection. +skip = [ + "windows_x86_64_msvc", + "windows_x86_64_gnullvm", + "windows_x86_64_gnu", + "windows_i686_msvc", + "windows_i686_gnu", + "windows_aarch64_msvc", + "windows_aarch64_gnullvm", + "windows-targets", + "windows-sys", + + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason why it can't be updated/removed" }, +] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite. +skip-tree = [ + #"ansi_term@0.11.0", # will be skipped along with _all_ of its direct and transitive dependencies + #{ crate = "ansi_term@0.11.0", depth = 20 }, +] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "warn" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "warn" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +allow-git = ["https://github.com/RustPython/__doc__"] diff --git a/derive-impl/Cargo.toml b/derive-impl/Cargo.toml deleted file mode 100644 index 2b2b4131b31..00000000000 --- a/derive-impl/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "rustpython-derive-impl" -version = "0.3.0" -description = "Rust language extensions and macros specific to rustpython." -authors = ["RustPython Team"] -repository = "https://github.com/RustPython/RustPython" -license = "MIT" -edition = "2021" - -[dependencies] -rustpython-compiler-core = { workspace = true } -rustpython-parser-core = { workspace = true } -rustpython-doc = { workspace = true } - -itertools = { workspace = true } -once_cell = { workspace = true } -syn = { workspace = true, features = ["full", "extra-traits"] } - -maplit = "1.0.2" -proc-macro2 = "1.0.60" -quote = "1.0.18" -syn-ext = { version = "0.4.0", features = ["full"] } -textwrap = { version = "0.15.0", default-features = false } diff --git a/derive-impl/src/compile_bytecode.rs b/derive-impl/src/compile_bytecode.rs deleted file mode 100644 index 6b5baef98c2..00000000000 --- a/derive-impl/src/compile_bytecode.rs +++ /dev/null @@ -1,411 +0,0 @@ -//! Parsing and processing for this form: -//! ```ignore -//! py_compile!( -//! // either: -//! source = "python_source_code", -//! // or -//! file = "file/path/relative/to/$CARGO_MANIFEST_DIR", -//! -//! // the mode to compile the code in -//! mode = "exec", // or "eval" or "single" -//! // the path put into the CodeObject, defaults to "frozen" -//! module_name = "frozen", -//! ) -//! ``` - -use crate::{extract_spans, Diagnostic}; -use once_cell::sync::Lazy; -use proc_macro2::{Span, TokenStream}; -use quote::quote; -use rustpython_compiler_core::{bytecode::CodeObject, frozen, Mode}; -use std::{ - collections::HashMap, - env, fs, - path::{Path, PathBuf}, -}; -use syn::{ - self, - parse::{Parse, ParseStream, Result as ParseResult}, - parse2, - spanned::Spanned, - Lit, LitByteStr, LitStr, Macro, Meta, MetaNameValue, Token, -}; - -static CARGO_MANIFEST_DIR: Lazy<PathBuf> = Lazy::new(|| { - PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present")) -}); - -enum CompilationSourceKind { - /// Source is a File (Path) - File(PathBuf), - /// Direct Raw source code - SourceCode(String), - /// Source is a directory - Dir(PathBuf), -} - -struct CompiledModule { - code: CodeObject, - package: bool, -} - -struct CompilationSource { - kind: CompilationSourceKind, - span: (Span, Span), -} - -pub trait Compiler { - fn compile( - &self, - source: &str, - mode: Mode, - module_name: String, - ) -> Result<CodeObject, Box<dyn std::error::Error>>; -} - -impl CompilationSource { - fn compile_string<D: std::fmt::Display, F: FnOnce() -> D>( - &self, - source: &str, - mode: Mode, - module_name: String, - compiler: &dyn Compiler, - origin: F, - ) -> Result<CodeObject, Diagnostic> { - compiler.compile(source, mode, module_name).map_err(|err| { - Diagnostic::spans_error( - self.span, - format!("Python compile error from {}: {}", origin(), err), - ) - }) - } - - fn compile( - &self, - mode: Mode, - module_name: String, - compiler: &dyn Compiler, - ) -> Result<HashMap<String, CompiledModule>, Diagnostic> { - match &self.kind { - CompilationSourceKind::Dir(rel_path) => self.compile_dir( - &CARGO_MANIFEST_DIR.join(rel_path), - String::new(), - mode, - compiler, - ), - _ => Ok(hashmap! { - module_name.clone() => CompiledModule { - code: self.compile_single(mode, module_name, compiler)?, - package: false, - }, - }), - } - } - - fn compile_single( - &self, - mode: Mode, - module_name: String, - compiler: &dyn Compiler, - ) -> Result<CodeObject, Diagnostic> { - match &self.kind { - CompilationSourceKind::File(rel_path) => { - let path = CARGO_MANIFEST_DIR.join(rel_path); - let source = fs::read_to_string(&path).map_err(|err| { - Diagnostic::spans_error( - self.span, - format!("Error reading file {path:?}: {err}"), - ) - })?; - self.compile_string(&source, mode, module_name, compiler, || rel_path.display()) - } - CompilationSourceKind::SourceCode(code) => { - self.compile_string(&textwrap::dedent(code), mode, module_name, compiler, || { - "string literal" - }) - } - CompilationSourceKind::Dir(_) => { - unreachable!("Can't use compile_single with directory source") - } - } - } - - fn compile_dir( - &self, - path: &Path, - parent: String, - mode: Mode, - compiler: &dyn Compiler, - ) -> Result<HashMap<String, CompiledModule>, Diagnostic> { - let mut code_map = HashMap::new(); - let paths = fs::read_dir(path) - .or_else(|e| { - if cfg!(windows) { - if let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) { - return fs::read_dir(real_path.trim()); - } - } - Err(e) - }) - .map_err(|err| { - Diagnostic::spans_error(self.span, format!("Error listing dir {path:?}: {err}")) - })?; - for path in paths { - let path = path.map_err(|err| { - Diagnostic::spans_error(self.span, format!("Failed to list file: {err}")) - })?; - let path = path.path(); - let file_name = path.file_name().unwrap().to_str().ok_or_else(|| { - Diagnostic::spans_error(self.span, format!("Invalid UTF-8 in file name {path:?}")) - })?; - if path.is_dir() { - code_map.extend(self.compile_dir( - &path, - if parent.is_empty() { - file_name.to_string() - } else { - format!("{parent}.{file_name}") - }, - mode, - compiler, - )?); - } else if file_name.ends_with(".py") { - let stem = path.file_stem().unwrap().to_str().unwrap(); - let is_init = stem == "__init__"; - let module_name = if is_init { - parent.clone() - } else if parent.is_empty() { - stem.to_owned() - } else { - format!("{parent}.{stem}") - }; - - let compile_path = |src_path: &Path| { - let source = fs::read_to_string(src_path).map_err(|err| { - Diagnostic::spans_error( - self.span, - format!("Error reading file {path:?}: {err}"), - ) - })?; - self.compile_string(&source, mode, module_name.clone(), compiler, || { - path.strip_prefix(&*CARGO_MANIFEST_DIR) - .ok() - .unwrap_or(&path) - .display() - }) - }; - let code = compile_path(&path).or_else(|e| { - if cfg!(windows) { - if let Ok(real_path) = fs::read_to_string(path.canonicalize().unwrap()) { - let joined = path.parent().unwrap().join(real_path.trim()); - if joined.exists() { - return compile_path(&joined); - } else { - return Err(e); - } - } - } - Err(e) - }); - - let code = match code { - Ok(code) => code, - Err(_) - if stem.starts_with("badsyntax_") - | parent.ends_with(".encoded_modules") => - { - // TODO: handle with macro arg rather than hard-coded path - continue; - } - Err(e) => return Err(e), - }; - - code_map.insert( - module_name, - CompiledModule { - code, - package: is_init, - }, - ); - } - } - Ok(code_map) - } -} - -/// This is essentially just a comma-separated list of Meta nodes, aka the inside of a MetaList. -struct PyCompileInput { - span: Span, - metas: Vec<Meta>, -} - -impl PyCompileInput { - fn parse(&self, allow_dir: bool) -> Result<PyCompileArgs, Diagnostic> { - let mut module_name = None; - let mut mode = None; - let mut source: Option<CompilationSource> = None; - let mut crate_name = None; - - fn assert_source_empty(source: &Option<CompilationSource>) -> Result<(), Diagnostic> { - if let Some(source) = source { - Err(Diagnostic::spans_error( - source.span, - "Cannot have more than one source", - )) - } else { - Ok(()) - } - } - - for meta in &self.metas { - if let Meta::NameValue(name_value) = meta { - let ident = match name_value.path.get_ident() { - Some(ident) => ident, - None => continue, - }; - let check_str = || match &name_value.lit { - Lit::Str(s) => Ok(s), - _ => Err(err_span!(name_value.lit, "{ident} must be a string")), - }; - if ident == "mode" { - let s = check_str()?; - match s.value().parse() { - Ok(mode_val) => mode = Some(mode_val), - Err(e) => bail_span!(s, "{}", e), - } - } else if ident == "module_name" { - module_name = Some(check_str()?.value()) - } else if ident == "source" { - assert_source_empty(&source)?; - let code = check_str()?.value(); - source = Some(CompilationSource { - kind: CompilationSourceKind::SourceCode(code), - span: extract_spans(&name_value).unwrap(), - }); - } else if ident == "file" { - assert_source_empty(&source)?; - let path = check_str()?.value().into(); - source = Some(CompilationSource { - kind: CompilationSourceKind::File(path), - span: extract_spans(&name_value).unwrap(), - }); - } else if ident == "dir" { - if !allow_dir { - bail_span!(ident, "py_compile doesn't accept dir") - } - - assert_source_empty(&source)?; - let path = check_str()?.value().into(); - source = Some(CompilationSource { - kind: CompilationSourceKind::Dir(path), - span: extract_spans(&name_value).unwrap(), - }); - } else if ident == "crate_name" { - let name = check_str()?.parse()?; - crate_name = Some(name); - } - } - } - - let source = source.ok_or_else(|| { - syn::Error::new( - self.span, - "Must have either file or source in py_compile!()/py_freeze!()", - ) - })?; - - Ok(PyCompileArgs { - source, - mode: mode.unwrap_or(Mode::Exec), - module_name: module_name.unwrap_or_else(|| "frozen".to_owned()), - crate_name: crate_name.unwrap_or_else(|| syn::parse_quote!(::rustpython_vm)), - }) - } -} - -fn parse_meta(input: ParseStream) -> ParseResult<Meta> { - let path = input.call(syn::Path::parse_mod_style)?; - let eq_token: Token![=] = input.parse()?; - let span = input.span(); - if input.peek(LitStr) { - Ok(Meta::NameValue(MetaNameValue { - path, - eq_token, - lit: Lit::Str(input.parse()?), - })) - } else if let Ok(mac) = input.parse::<Macro>() { - Ok(Meta::NameValue(MetaNameValue { - path, - eq_token, - lit: Lit::Str(LitStr::new(&mac.tokens.to_string(), mac.span())), - })) - } else { - Err(syn::Error::new(span, "Expected string or stringify macro")) - } -} - -impl Parse for PyCompileInput { - fn parse(input: ParseStream) -> ParseResult<Self> { - let span = input.cursor().span(); - let metas = input - .parse_terminated::<Meta, Token![,]>(parse_meta)? - .into_iter() - .collect(); - Ok(PyCompileInput { span, metas }) - } -} - -struct PyCompileArgs { - source: CompilationSource, - mode: Mode, - module_name: String, - crate_name: syn::Path, -} - -pub fn impl_py_compile( - input: TokenStream, - compiler: &dyn Compiler, -) -> Result<TokenStream, Diagnostic> { - let input: PyCompileInput = parse2(input)?; - let args = input.parse(false)?; - - let crate_name = args.crate_name; - let code = args - .source - .compile_single(args.mode, args.module_name, compiler)?; - - let frozen = frozen::FrozenCodeObject::encode(&code); - let bytes = LitByteStr::new(&frozen.bytes, Span::call_site()); - - let output = quote! { - #crate_name::frozen::FrozenCodeObject { bytes: &#bytes[..] } - }; - - Ok(output) -} - -pub fn impl_py_freeze( - input: TokenStream, - compiler: &dyn Compiler, -) -> Result<TokenStream, Diagnostic> { - let input: PyCompileInput = parse2(input)?; - let args = input.parse(true)?; - - let crate_name = args.crate_name; - let code_map = args.source.compile(args.mode, args.module_name, compiler)?; - - let data = frozen::FrozenLib::encode(code_map.iter().map(|(k, v)| { - let v = frozen::FrozenModule { - code: frozen::FrozenCodeObject::encode(&v.code), - package: v.package, - }; - (&**k, v) - })); - let bytes = LitByteStr::new(&data.bytes, Span::call_site()); - - let output = quote! { - #crate_name::frozen::FrozenLib::from_ref(#bytes) - }; - - Ok(output) -} diff --git a/derive-impl/src/error.rs b/derive-impl/src/error.rs deleted file mode 100644 index 2313e529a9f..00000000000 --- a/derive-impl/src/error.rs +++ /dev/null @@ -1,156 +0,0 @@ -// Taken from https://github.com/rustwasm/wasm-bindgen/blob/master/crates/backend/src/error.rs -// -// Copyright (c) 2014 Alex Crichton -// -// Permission is hereby granted, free of charge, to any -// person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the -// Software without restriction, including without -// limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of -// the Software, and to permit persons to whom the Software -// is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice -// shall be included in all copies or substantial portions -// of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -#![allow(dead_code)] - -use proc_macro2::*; -use quote::{ToTokens, TokenStreamExt}; -use syn::parse::Error; - -macro_rules! err_span { - ($span:expr, $($msg:tt)*) => ( - syn::Error::new_spanned(&$span, format_args!($($msg)*)) - ) -} - -macro_rules! bail_span { - ($($t:tt)*) => ( - return Err(err_span!($($t)*).into()) - ) -} - -// macro_rules! push_err_span { -// ($diagnostics:expr, $($t:tt)*) => { -// $diagnostics.push(err_span!($($t)*)) -// }; -// } - -// macro_rules! push_diag_result { -// ($diags:expr, $x:expr $(,)?) => { -// if let Err(e) = $x { -// $diags.push(e); -// } -// }; -// } - -#[derive(Debug)] -pub struct Diagnostic { - inner: Repr, -} - -#[derive(Debug)] -enum Repr { - Single { - text: String, - span: Option<(Span, Span)>, - }, - SynError(Error), - Multi { - diagnostics: Vec<Diagnostic>, - }, -} - -impl Diagnostic { - pub fn error<T: Into<String>>(text: T) -> Diagnostic { - Diagnostic { - inner: Repr::Single { - text: text.into(), - span: None, - }, - } - } - - pub(crate) fn spans_error<T: Into<String>>(spans: (Span, Span), text: T) -> Diagnostic { - Diagnostic { - inner: Repr::Single { - text: text.into(), - span: Some(spans), - }, - } - } - - pub fn from_vec(diagnostics: Vec<Diagnostic>) -> Result<(), Diagnostic> { - if diagnostics.is_empty() { - Ok(()) - } else { - Err(Diagnostic { - inner: Repr::Multi { diagnostics }, - }) - } - } - - pub fn panic(&self) -> ! { - match &self.inner { - Repr::Single { text, .. } => panic!("{}", text), - Repr::SynError(error) => panic!("{}", error), - Repr::Multi { diagnostics } => diagnostics[0].panic(), - } - } -} - -impl From<Error> for Diagnostic { - fn from(err: Error) -> Diagnostic { - Diagnostic { - inner: Repr::SynError(err), - } - } -} - -pub fn extract_spans(node: &dyn ToTokens) -> Option<(Span, Span)> { - let mut t = TokenStream::new(); - node.to_tokens(&mut t); - let mut tokens = t.into_iter(); - let start = tokens.next().map(|t| t.span()); - let end = tokens.last().map(|t| t.span()); - start.map(|start| (start, end.unwrap_or(start))) -} - -impl ToTokens for Diagnostic { - fn to_tokens(&self, dst: &mut TokenStream) { - match &self.inner { - Repr::Single { text, span } => { - let cs2 = (Span::call_site(), Span::call_site()); - let (start, end) = span.unwrap_or(cs2); - dst.append(Ident::new("compile_error", start)); - dst.append(Punct::new('!', Spacing::Alone)); - let mut message = TokenStream::new(); - message.append(Literal::string(text)); - let mut group = Group::new(Delimiter::Brace, message); - group.set_span(end); - dst.append(group); - } - Repr::Multi { diagnostics } => { - for diagnostic in diagnostics { - diagnostic.to_tokens(dst); - } - } - Repr::SynError(err) => { - err.to_compile_error().to_tokens(dst); - } - } - } -} diff --git a/derive-impl/src/from_args.rs b/derive-impl/src/from_args.rs deleted file mode 100644 index 48bcd4fe223..00000000000 --- a/derive-impl/src/from_args.rs +++ /dev/null @@ -1,228 +0,0 @@ -use proc_macro2::TokenStream; -use quote::{quote, ToTokens}; -use syn::{ - parse_quote, Attribute, Data, DeriveInput, Expr, Field, Ident, Lit, Meta, NestedMeta, Result, -}; - -/// The kind of the python parameter, this corresponds to the value of Parameter.kind -/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) -enum ParameterKind { - PositionalOnly, - PositionalOrKeyword, - KeywordOnly, - Flatten, -} - -impl ParameterKind { - fn from_ident(ident: &Ident) -> Option<ParameterKind> { - match ident.to_string().as_str() { - "positional" => Some(ParameterKind::PositionalOnly), - "any" => Some(ParameterKind::PositionalOrKeyword), - "named" => Some(ParameterKind::KeywordOnly), - "flatten" => Some(ParameterKind::Flatten), - _ => None, - } - } -} - -struct ArgAttribute { - name: Option<String>, - kind: ParameterKind, - default: Option<DefaultValue>, -} -// None == quote!(Default::default()) -type DefaultValue = Option<Expr>; - -impl ArgAttribute { - fn from_attribute(attr: &Attribute) -> Option<Result<ArgAttribute>> { - if !attr.path.is_ident("pyarg") { - return None; - } - let inner = move || { - let Meta::List(list) = attr.parse_meta()? else { - bail_span!(attr, "pyarg must be a list, like #[pyarg(...)]") - }; - let mut iter = list.nested.iter(); - let first_arg = iter.next().ok_or_else(|| { - err_span!(list, "There must be at least one argument to #[pyarg()]") - })?; - let kind = match first_arg { - NestedMeta::Meta(Meta::Path(path)) => { - path.get_ident().and_then(ParameterKind::from_ident) - } - _ => None, - }; - let kind = kind.ok_or_else(|| { - err_span!( - first_arg, - "The first argument to #[pyarg()] must be the parameter type, either \ - 'positional', 'any', 'named', or 'flatten'." - ) - })?; - - let mut attribute = ArgAttribute { - name: None, - kind, - default: None, - }; - - for arg in iter { - attribute.parse_argument(arg)?; - } - - Ok(attribute) - }; - Some(inner()) - } - - fn parse_argument(&mut self, arg: &NestedMeta) -> Result<()> { - if let ParameterKind::Flatten = self.kind { - bail_span!(arg, "can't put additional arguments on a flatten arg") - } - match arg { - NestedMeta::Meta(Meta::Path(path)) => { - if path.is_ident("default") || path.is_ident("optional") { - if self.default.is_none() { - self.default = Some(None); - } - } else { - bail_span!(path, "Unrecognized pyarg attribute"); - } - } - NestedMeta::Meta(Meta::NameValue(name_value)) => { - if name_value.path.is_ident("default") { - if matches!(self.default, Some(Some(_))) { - bail_span!(name_value, "Default already set"); - } - - match name_value.lit { - Lit::Str(ref val) => self.default = Some(Some(val.parse()?)), - _ => bail_span!(name_value, "Expected string value for default argument"), - } - } else if name_value.path.is_ident("name") { - if self.name.is_some() { - bail_span!(name_value, "already have a name") - } - - match &name_value.lit { - Lit::Str(val) => self.name = Some(val.value()), - _ => bail_span!(name_value, "Expected string value for name argument"), - } - } else { - bail_span!(name_value, "Unrecognized pyarg attribute"); - } - } - _ => bail_span!(arg, "Unrecognized pyarg attribute"), - } - - Ok(()) - } -} - -fn generate_field((i, field): (usize, &Field)) -> Result<TokenStream> { - let mut pyarg_attrs = field - .attrs - .iter() - .filter_map(ArgAttribute::from_attribute) - .collect::<std::result::Result<Vec<_>, _>>()?; - let attr = if pyarg_attrs.is_empty() { - ArgAttribute { - name: None, - kind: ParameterKind::PositionalOrKeyword, - default: None, - } - } else if pyarg_attrs.len() == 1 { - pyarg_attrs.remove(0) - } else { - bail_span!(field, "Multiple pyarg attributes on field"); - }; - - let name = field.ident.as_ref(); - let name_string = name.map(Ident::to_string); - if matches!(&name_string, Some(s) if s.starts_with("_phantom")) { - return Ok(quote! { - #name: ::std::marker::PhantomData, - }); - } - let field_name = match name { - Some(id) => id.to_token_stream(), - None => syn::Index::from(i).into_token_stream(), - }; - if let ParameterKind::Flatten = attr.kind { - return Ok(quote! { - #field_name: ::rustpython_vm::function::FromArgs::from_args(vm, args)?, - }); - } - let pyname = attr - .name - .or(name_string) - .ok_or_else(|| err_span!(field, "field in tuple struct must have name attribute"))?; - let middle = quote! { - .map(|x| ::rustpython_vm::convert::TryFromObject::try_from_object(vm, x)).transpose()? - }; - let ending = if let Some(default) = attr.default { - let default = default.unwrap_or_else(|| parse_quote!(::std::default::Default::default())); - quote! { - .map(::rustpython_vm::function::FromArgOptional::from_inner) - .unwrap_or_else(|| #default) - } - } else { - let err = match attr.kind { - ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { - ::rustpython_vm::function::ArgumentError::TooFewArgs - }, - ParameterKind::KeywordOnly => quote! { - ::rustpython_vm::function::ArgumentError::RequiredKeywordArgument(#pyname.to_owned()) - }, - ParameterKind::Flatten => unreachable!(), - }; - quote! { - .ok_or_else(|| #err)? - } - }; - - let file_output = match attr.kind { - ParameterKind::PositionalOnly => { - quote! { - #field_name: args.take_positional()#middle #ending, - } - } - ParameterKind::PositionalOrKeyword => { - quote! { - #field_name: args.take_positional_keyword(#pyname)#middle #ending, - } - } - ParameterKind::KeywordOnly => { - quote! { - #field_name: args.take_keyword(#pyname)#middle #ending, - } - } - ParameterKind::Flatten => unreachable!(), - }; - Ok(file_output) -} - -pub fn impl_from_args(input: DeriveInput) -> Result<TokenStream> { - let fields = match input.data { - Data::Struct(syn::DataStruct { fields, .. }) => fields - .iter() - .enumerate() - .map(generate_field) - .collect::<Result<TokenStream>>()?, - _ => bail_span!(input, "FromArgs input must be a struct"), - }; - - let name = input.ident; - let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); - let output = quote! { - impl #impl_generics ::rustpython_vm::function::FromArgs for #name #ty_generics #where_clause { - fn from_args( - vm: &::rustpython_vm::VirtualMachine, - args: &mut ::rustpython_vm::function::FuncArgs - ) -> ::std::result::Result<Self, ::rustpython_vm::function::ArgumentError> { - Ok(#name { #fields }) - } - } - }; - Ok(output) -} diff --git a/derive-impl/src/lib.rs b/derive-impl/src/lib.rs deleted file mode 100644 index 35292e7de09..00000000000 --- a/derive-impl/src/lib.rs +++ /dev/null @@ -1,83 +0,0 @@ -#![recursion_limit = "128"] -#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] -#![doc(html_root_url = "https://docs.rs/rustpython-derive/")] - -extern crate proc_macro; - -#[macro_use] -extern crate maplit; - -#[macro_use] -mod error; -#[macro_use] -mod util; - -mod compile_bytecode; -mod from_args; -mod pyclass; -mod pymodule; -mod pypayload; -mod pystructseq; -mod pytraverse; - -use error::{extract_spans, Diagnostic}; -use proc_macro2::TokenStream; -use quote::ToTokens; -use rustpython_doc as doc; -use syn::{AttributeArgs, DeriveInput, Item}; - -pub use compile_bytecode::Compiler; - -fn result_to_tokens(result: Result<TokenStream, impl Into<Diagnostic>>) -> TokenStream { - result - .map_err(|e| e.into()) - .unwrap_or_else(ToTokens::into_token_stream) -} - -pub fn derive_from_args(input: DeriveInput) -> TokenStream { - result_to_tokens(from_args::impl_from_args(input)) -} - -pub fn pyclass(attr: AttributeArgs, item: Item) -> TokenStream { - if matches!(item, syn::Item::Impl(_) | syn::Item::Trait(_)) { - result_to_tokens(pyclass::impl_pyclass_impl(attr, item)) - } else { - result_to_tokens(pyclass::impl_pyclass(attr, item)) - } -} - -pub fn pyexception(attr: AttributeArgs, item: Item) -> TokenStream { - if matches!(item, syn::Item::Impl(_)) { - result_to_tokens(pyclass::impl_pyexception_impl(attr, item)) - } else { - result_to_tokens(pyclass::impl_pyexception(attr, item)) - } -} - -pub fn pymodule(attr: AttributeArgs, item: Item) -> TokenStream { - result_to_tokens(pymodule::impl_pymodule(attr, item)) -} - -pub fn pystruct_sequence(input: DeriveInput) -> TokenStream { - result_to_tokens(pystructseq::impl_pystruct_sequence(input)) -} - -pub fn pystruct_sequence_try_from_object(input: DeriveInput) -> TokenStream { - result_to_tokens(pystructseq::impl_pystruct_sequence_try_from_object(input)) -} - -pub fn py_compile(input: TokenStream, compiler: &dyn Compiler) -> TokenStream { - result_to_tokens(compile_bytecode::impl_py_compile(input, compiler)) -} - -pub fn py_freeze(input: TokenStream, compiler: &dyn Compiler) -> TokenStream { - result_to_tokens(compile_bytecode::impl_py_freeze(input, compiler)) -} - -pub fn pypayload(input: DeriveInput) -> TokenStream { - result_to_tokens(pypayload::impl_pypayload(input)) -} - -pub fn pytraverse(item: DeriveInput) -> TokenStream { - result_to_tokens(pytraverse::impl_pytraverse(item)) -} diff --git a/derive-impl/src/pyclass.rs b/derive-impl/src/pyclass.rs deleted file mode 100644 index 8f688b366ab..00000000000 --- a/derive-impl/src/pyclass.rs +++ /dev/null @@ -1,1638 +0,0 @@ -use super::Diagnostic; -use crate::util::{ - format_doc, pyclass_ident_and_attrs, pyexception_ident_and_attrs, text_signature, - ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ExceptionItemMeta, ItemMeta, - ItemMetaInner, ItemNursery, SimpleItemMeta, ALL_ALLOWED_NAMES, -}; -use proc_macro2::{Delimiter, Group, Span, TokenStream, TokenTree}; -use quote::{quote, quote_spanned, ToTokens}; -use std::collections::{HashMap, HashSet}; -use std::str::FromStr; -use syn::{ - parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Item, Meta, NestedMeta, Result, -}; -use syn_ext::ext::*; - -#[derive(Copy, Clone, Debug)] -enum AttrName { - Method, - ClassMethod, - StaticMethod, - GetSet, - Slot, - Attr, - ExtendClass, - Member, -} - -impl std::fmt::Display for AttrName { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let s = match self { - Self::Method => "pymethod", - Self::ClassMethod => "pyclassmethod", - Self::StaticMethod => "pystaticmethod", - Self::GetSet => "pygetset", - Self::Slot => "pyslot", - Self::Attr => "pyattr", - Self::ExtendClass => "extend_class", - Self::Member => "pymember", - }; - s.fmt(f) - } -} - -impl FromStr for AttrName { - type Err = String; - fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { - Ok(match s { - "pymethod" => Self::Method, - "pyclassmethod" => Self::ClassMethod, - "pystaticmethod" => Self::StaticMethod, - "pygetset" => Self::GetSet, - "pyslot" => Self::Slot, - "pyattr" => Self::Attr, - "extend_class" => Self::ExtendClass, - "pymember" => Self::Member, - s => { - return Err(s.to_owned()); - } - }) - } -} - -#[derive(Default)] -struct ImplContext { - attribute_items: ItemNursery, - method_items: MethodNursery, - getset_items: GetSetNursery, - member_items: MemberNursery, - extend_slots_items: ItemNursery, - class_extensions: Vec<TokenStream>, - errors: Vec<syn::Error>, -} - -fn extract_items_into_context<'a, Item>( - context: &mut ImplContext, - items: impl Iterator<Item = &'a mut Item>, -) where - Item: ItemLike + ToTokens + GetIdent + syn_ext::ext::ItemAttrExt + 'a, -{ - for item in items { - let r = item.try_split_attr_mut(|attrs, item| { - let (py_items, cfgs) = attrs_to_content_items(attrs, impl_item_new::<Item>)?; - for py_item in py_items.iter().rev() { - let r = py_item.gen_impl_item(ImplItemArgs::<Item> { - item, - attrs, - context, - cfgs: cfgs.as_slice(), - }); - context.errors.ok_or_push(r); - } - Ok(()) - }); - context.errors.ok_or_push(r); - } - context.errors.ok_or_push(context.method_items.validate()); - context.errors.ok_or_push(context.getset_items.validate()); - context.errors.ok_or_push(context.member_items.validate()); -} - -pub(crate) fn impl_pyclass_impl(attr: AttributeArgs, item: Item) -> Result<TokenStream> { - let mut context = ImplContext::default(); - let mut tokens = match item { - Item::Impl(mut imp) => { - extract_items_into_context(&mut context, imp.items.iter_mut()); - - let (impl_ty, payload_guess) = match imp.self_ty.as_ref() { - syn::Type::Path(syn::TypePath { - path: syn::Path { segments, .. }, - .. - }) if segments.len() == 1 => { - let segment = &segments[0]; - let payload_ty = if segment.ident == "Py" || segment.ident == "PyRef" { - match &segment.arguments { - syn::PathArguments::AngleBracketed( - syn::AngleBracketedGenericArguments { args, .. }, - ) if args.len() == 1 => { - let arg = &args[0]; - match arg { - syn::GenericArgument::Type(syn::Type::Path( - syn::TypePath { - path: syn::Path { segments, .. }, - .. - }, - )) if segments.len() == 1 => segments[0].ident.clone(), - _ => { - return Err(syn::Error::new_spanned( - segment, - "Py{Ref}<T> is expected but Py{Ref}<?> is found", - )) - } - } - } - _ => { - return Err(syn::Error::new_spanned( - segment, - "Py{Ref}<T> is expected but Py{Ref}? is found", - )) - } - } - } else { - if !matches!(segment.arguments, syn::PathArguments::None) { - return Err(syn::Error::new_spanned( - segment, - "PyImpl can only be implemented for Py{Ref}<T> or T", - )); - } - segment.ident.clone() - }; - (segment.ident.clone(), payload_ty) - } - _ => { - return Err(syn::Error::new_spanned( - imp.self_ty, - "PyImpl can only be implemented for Py{Ref}<T> or T", - )) - } - }; - - let ExtractedImplAttrs { - payload: attr_payload, - flags, - with_impl, - with_method_defs, - with_slots, - } = extract_impl_attrs(attr, &impl_ty)?; - let payload_ty = attr_payload.unwrap_or(payload_guess); - let method_def = &context.method_items; - let getset_impl = &context.getset_items; - let member_impl = &context.member_items; - let extend_impl = context.attribute_items.validate()?; - let slots_impl = context.extend_slots_items.validate()?; - let class_extensions = &context.class_extensions; - - let extra_methods = [ - parse_quote! { - const __OWN_METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_def; - }, - parse_quote! { - fn __extend_py_class( - ctx: &::rustpython_vm::Context, - class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, - ) { - #getset_impl - #member_impl - #extend_impl - #(#class_extensions)* - } - }, - parse_quote! { - fn __extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { - #slots_impl - } - }, - ]; - imp.items.extend(extra_methods); - let is_main_impl = impl_ty == payload_ty; - if is_main_impl { - let method_defs = if with_method_defs.is_empty() { - quote!(#impl_ty::__OWN_METHOD_DEFS) - } else { - quote!( - rustpython_vm::function::PyMethodDef::__const_concat_arrays::< - { #impl_ty::__OWN_METHOD_DEFS.len() #(+ #with_method_defs.len())* }, - >(&[#impl_ty::__OWN_METHOD_DEFS, #(#with_method_defs,)*]) - ) - }; - quote! { - #imp - impl ::rustpython_vm::class::PyClassImpl for #payload_ty { - const TP_FLAGS: ::rustpython_vm::types::PyTypeFlags = #flags; - - fn impl_extend_class( - ctx: &::rustpython_vm::Context, - class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, - ) { - #impl_ty::__extend_py_class(ctx, class); - #with_impl - } - - const METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_defs; - - fn extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { - #impl_ty::__extend_slots(slots); - #with_slots - } - } - } - } else { - imp.into_token_stream() - } - } - Item::Trait(mut trai) => { - let mut context = ImplContext::default(); - let mut has_extend_slots = false; - for item in &trai.items { - let has = match item { - syn::TraitItem::Method(method) => { - &method.sig.ident.to_string() == "extend_slots" - } - _ => false, - }; - if has { - has_extend_slots = has; - break; - } - } - extract_items_into_context(&mut context, trai.items.iter_mut()); - - let ExtractedImplAttrs { - with_impl, - with_slots, - .. - } = extract_impl_attrs(attr, &trai.ident)?; - - let method_def = &context.method_items; - let getset_impl = &context.getset_items; - let member_impl = &context.member_items; - let extend_impl = &context.attribute_items.validate()?; - let slots_impl = &context.extend_slots_items.validate()?; - let class_extensions = &context.class_extensions; - let call_extend_slots = if has_extend_slots { - quote! { - Self::extend_slots(slots); - } - } else { - quote! {} - }; - let extra_methods = [ - parse_quote! { - const __OWN_METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_def; - }, - parse_quote! { - fn __extend_py_class( - ctx: &::rustpython_vm::Context, - class: &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType>, - ) { - #getset_impl - #member_impl - #extend_impl - #with_impl - #(#class_extensions)* - } - }, - parse_quote! { - fn __extend_slots(slots: &mut ::rustpython_vm::types::PyTypeSlots) { - #with_slots - #slots_impl - #call_extend_slots - } - }, - ]; - trai.items.extend(extra_methods); - - trai.into_token_stream() - } - item => item.into_token_stream(), - }; - if let Some(error) = context.errors.into_error() { - let error = Diagnostic::from(error); - tokens = quote! { - #tokens - #error - } - } - Ok(tokens) -} - -fn generate_class_def( - ident: &Ident, - name: &str, - module_name: Option<&str>, - base: Option<String>, - metaclass: Option<String>, - unhashable: bool, - attrs: &[Attribute], -) -> Result<TokenStream> { - let doc = attrs.doc().or_else(|| { - let module_name = module_name.unwrap_or("builtins"); - crate::doc::Database::shared() - .try_module_item(module_name, name) - .ok() - .flatten() - .map(str::to_owned) - }); - let doc = if let Some(doc) = doc { - quote!(Some(#doc)) - } else { - quote!(None) - }; - let module_class_name = if let Some(module_name) = module_name { - format!("{module_name}.{name}") - } else { - name.to_owned() - }; - let module_name = match module_name { - Some(v) => quote!(Some(#v) ), - None => quote!(None), - }; - let unhashable = if unhashable { - quote!(true) - } else { - quote!(false) - }; - let basicsize = quote!(std::mem::size_of::<#ident>()); - let is_pystruct = attrs.iter().any(|attr| { - attr.path.is_ident("derive") - && if let Ok(Meta::List(l)) = attr.parse_meta() { - l.nested - .into_iter() - .any(|n| n.get_ident().map_or(false, |p| p == "PyStructSequence")) - } else { - false - } - }); - if base.is_some() && is_pystruct { - bail_span!(ident, "PyStructSequence cannot have `base` class attr",); - } - let base_class = if is_pystruct { - Some(quote! { rustpython_vm::builtins::PyTuple }) - } else { - base.as_ref().map(|typ| { - let typ = Ident::new(typ, ident.span()); - quote_spanned! { ident.span() => #typ } - }) - } - .map(|typ| { - quote! { - fn static_baseclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { - use rustpython_vm::class::StaticType; - #typ::static_type() - } - } - }); - - let meta_class = metaclass.map(|typ| { - let typ = Ident::new(&typ, ident.span()); - quote! { - fn static_metaclass() -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { - use rustpython_vm::class::StaticType; - #typ::static_type() - } - } - }); - - let base_or_object = if let Some(base) = base { - let base = Ident::new(&base, ident.span()); - quote! { #base } - } else { - quote! { ::rustpython_vm::builtins::PyBaseObject } - }; - - let tokens = quote! { - impl ::rustpython_vm::class::PyClassDef for #ident { - const NAME: &'static str = #name; - const MODULE_NAME: Option<&'static str> = #module_name; - const TP_NAME: &'static str = #module_class_name; - const DOC: Option<&'static str> = #doc; - const BASICSIZE: usize = #basicsize; - const UNHASHABLE: bool = #unhashable; - - type Base = #base_or_object; - } - - impl ::rustpython_vm::class::StaticType for #ident { - fn static_cell() -> &'static ::rustpython_vm::common::static_cell::StaticCell<::rustpython_vm::builtins::PyTypeRef> { - ::rustpython_vm::common::static_cell! { - static CELL: ::rustpython_vm::builtins::PyTypeRef; - } - &CELL - } - - #meta_class - - #base_class - } - }; - Ok(tokens) -} - -pub(crate) fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result<TokenStream> { - if matches!(item, syn::Item::Use(_)) { - return Ok(quote!(#item)); - } - let (ident, attrs) = pyclass_ident_and_attrs(&item)?; - let fake_ident = Ident::new("pyclass", item.span()); - let class_meta = ClassItemMeta::from_nested(ident.clone(), fake_ident, attr.into_iter())?; - let class_name = class_meta.class_name()?; - let module_name = class_meta.module()?; - let base = class_meta.base()?; - let metaclass = class_meta.metaclass()?; - let unhashable = class_meta.unhashable()?; - - let class_def = generate_class_def( - ident, - &class_name, - module_name.as_deref(), - base, - metaclass, - unhashable, - attrs, - )?; - - const ALLOWED_TRAVERSE_OPTS: &[&str] = &["manual"]; - // try to know if it have a `#[pyclass(trace)]` exist on this struct - // TODO(discord9): rethink on auto detect `#[Derive(PyTrace)]` - - // 1. no `traverse` at all: generate a dummy try_traverse - // 2. `traverse = "manual"`: generate a try_traverse, but not #[derive(Traverse)] - // 3. `traverse`: generate a try_traverse, and #[derive(Traverse)] - let (maybe_trace_code, derive_trace) = { - if class_meta.inner()._has_key("traverse")? { - let maybe_trace_code = quote! { - impl ::rustpython_vm::object::MaybeTraverse for #ident { - const IS_TRACE: bool = true; - fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { - ::rustpython_vm::object::Traverse::traverse(self, tracer_fn); - } - } - }; - // if the key `traverse` exist but not as key-value, _optional_str return Err(...) - // so we need to check if it is Ok(Some(...)) - let value = class_meta.inner()._optional_str("traverse"); - let derive_trace = if let Ok(Some(s)) = value { - if !ALLOWED_TRAVERSE_OPTS.contains(&s.as_str()) { - bail_span!( - item, - "traverse attribute only accept {ALLOWED_TRAVERSE_OPTS:?} as value or no value at all", - ); - } - assert_eq!(s, "manual"); - quote! {} - } else { - quote! {#[derive(Traverse)]} - }; - (maybe_trace_code, derive_trace) - } else { - ( - // a dummy impl, which do nothing - // #attrs - quote! { - impl ::rustpython_vm::object::MaybeTraverse for #ident { - fn try_traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { - // do nothing - } - } - }, - quote! {}, - ) - } - }; - - let impl_payload = if let Some(ctx_type_name) = class_meta.ctx_name()? { - let ctx_type_ident = Ident::new(&ctx_type_name, ident.span()); // FIXME span - - // We need this to make extend mechanism work: - quote! { - impl ::rustpython_vm::PyPayload for #ident { - fn class(ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { - ctx.types.#ctx_type_ident - } - } - } - } else { - quote! {} - }; - - let empty_impl = if let Some(attrs) = class_meta.impl_attrs()? { - let attrs: Meta = parse_quote! (#attrs); - quote! { - #[pyclass(#attrs)] - impl #ident {} - } - } else { - quote! {} - }; - - let ret = quote! { - #derive_trace - #item - #maybe_trace_code - #class_def - #impl_payload - #empty_impl - }; - Ok(ret) -} - -/// Special macro to create exception types. -/// -/// Why do we need it and why can't we just use `pyclass` macro instead? -/// We generate exception types with a `macro_rules`, -/// similar to how CPython does it. -/// But, inside `macro_rules` we don't have an opportunity -/// to add non-literal attributes to `pyclass`. -/// That's why we have to use this proxy. -pub(crate) fn impl_pyexception(attr: AttributeArgs, item: Item) -> Result<TokenStream> { - let (ident, _attrs) = pyexception_ident_and_attrs(&item)?; - let fake_ident = Ident::new("pyclass", item.span()); - let class_meta = ExceptionItemMeta::from_nested(ident.clone(), fake_ident, attr.into_iter())?; - let class_name = class_meta.class_name()?; - - let base_class_name = class_meta.base()?; - let impl_payload = if let Some(ctx_type_name) = class_meta.ctx_name()? { - let ctx_type_ident = Ident::new(&ctx_type_name, ident.span()); // FIXME span - - // We need this to make extend mechanism work: - quote! { - impl ::rustpython_vm::PyPayload for #ident { - fn class(ctx: &::rustpython_vm::vm::Context) -> &'static ::rustpython_vm::Py<::rustpython_vm::builtins::PyType> { - ctx.exceptions.#ctx_type_ident - } - } - } - } else { - quote! {} - }; - let impl_pyclass = if class_meta.has_impl()? { - quote! { - #[pyexception] - impl #ident {} - } - } else { - quote! {} - }; - - let ret = quote! { - #[pyclass(module = false, name = #class_name, base = #base_class_name)] - #item - #impl_payload - #impl_pyclass - }; - Ok(ret) -} - -pub(crate) fn impl_pyexception_impl(attr: AttributeArgs, item: Item) -> Result<TokenStream> { - let Item::Impl(imp) = item else { - return Ok(item.into_token_stream()); - }; - - if !attr.is_empty() { - return Err(syn::Error::new_spanned( - &attr[0], - "#[pyexception] impl doesn't allow attrs. Use #[pyclass] instead.", - )); - } - - let mut has_slot_new = false; - let mut has_slot_init = false; - let syn::ItemImpl { - generics, - self_ty, - items, - .. - } = &imp; - for item in items { - // FIXME: better detection or correct wrapper implementation - let Some(ident) = item.get_ident() else { - continue; - }; - let item_name = ident.to_string(); - match item_name.as_str() { - "slot_new" => { - has_slot_new = true; - } - "slot_init" => { - has_slot_init = true; - } - _ => continue, - } - } - - let slot_new = if has_slot_new { - quote!() - } else { - quote! { - #[pyslot] - pub(crate) fn slot_new( - cls: ::rustpython_vm::builtins::PyTypeRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult { - <Self as ::rustpython_vm::class::PyClassDef>::Base::slot_new(cls, args, vm) - } - } - }; - - // We need this method, because of how `CPython` copies `__init__` - // from `BaseException` in `SimpleExtendsException` macro. - // See: `(initproc)BaseException_init` - // spell-checker:ignore initproc - let slot_init = if has_slot_init { - quote!() - } else { - // FIXME: this is a generic logic for types not only for exceptions - quote! { - #[pyslot] - #[pymethod(name="__init__")] - pub(crate) fn slot_init( - zelf: ::rustpython_vm::PyObjectRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult<()> { - <Self as ::rustpython_vm::class::PyClassDef>::Base::slot_init(zelf, args, vm) - } - } - }; - Ok(quote! { - #[pyclass(flags(BASETYPE, HAS_DICT))] - impl #generics #self_ty { - #(#items)* - - #slot_new - #slot_init - } - }) -} - -/// #[pymethod] and #[pyclassmethod] -struct MethodItem { - inner: ContentItemInner<AttrName>, -} - -/// #[pygetset] -struct GetSetItem { - inner: ContentItemInner<AttrName>, -} - -/// #[pyslot] -struct SlotItem { - inner: ContentItemInner<AttrName>, -} - -/// #[pyattr] -struct AttributeItem { - inner: ContentItemInner<AttrName>, -} - -/// #[extend_class] -struct ExtendClassItem { - inner: ContentItemInner<AttrName>, -} - -/// #[pymember] -struct MemberItem { - inner: ContentItemInner<AttrName>, -} - -impl ContentItem for MethodItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} -impl ContentItem for GetSetItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} -impl ContentItem for SlotItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} -impl ContentItem for AttributeItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} -impl ContentItem for ExtendClassItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} -impl ContentItem for MemberItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} - -struct ImplItemArgs<'a, Item: ItemLike> { - item: &'a Item, - attrs: &'a mut Vec<Attribute>, - context: &'a mut ImplContext, - cfgs: &'a [Attribute], -} - -trait ImplItem<Item>: ContentItem -where - Item: ItemLike + ToTokens + GetIdent, -{ - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()>; -} - -impl<Item> ImplItem<Item> for MethodItem -where - Item: ItemLike + ToTokens + GetIdent, -{ - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { - let func = args - .item - .function_or_method() - .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))?; - let ident = &func.sig().ident; - - let item_attr = args.attrs.remove(self.index()); - let item_meta = MethodItemMeta::from_attr(ident.clone(), &item_attr)?; - - let py_name = item_meta.method_name()?; - let sig_doc = text_signature(func.sig(), &py_name); - - let doc = args.attrs.doc().map(|doc| format_doc(&sig_doc, &doc)); - args.context.method_items.add_item(MethodNurseryItem { - py_name, - cfgs: args.cfgs.to_vec(), - ident: ident.to_owned(), - doc, - attr_name: self.inner.attr_name, - }); - Ok(()) - } -} - -impl<Item> ImplItem<Item> for GetSetItem -where - Item: ItemLike + ToTokens + GetIdent, -{ - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { - let func = args - .item - .function_or_method() - .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))?; - let ident = &func.sig().ident; - - let item_attr = args.attrs.remove(self.index()); - let item_meta = GetSetItemMeta::from_attr(ident.clone(), &item_attr)?; - - let (py_name, kind) = item_meta.getset_name()?; - args.context - .getset_items - .add_item(py_name, args.cfgs.to_vec(), kind, ident.clone())?; - Ok(()) - } -} - -impl<Item> ImplItem<Item> for SlotItem -where - Item: ItemLike + ToTokens + GetIdent, -{ - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { - let (ident, span) = if let Ok(c) = args.item.constant() { - (c.ident(), c.span()) - } else if let Ok(f) = args.item.function_or_method() { - (&f.sig().ident, f.span()) - } else { - return Err(self.new_syn_error(args.item.span(), "can only be on a method")); - }; - - let item_attr = args.attrs.remove(self.index()); - let item_meta = SlotItemMeta::from_attr(ident.clone(), &item_attr)?; - - let slot_ident = item_meta.slot_name()?; - let slot_ident = Ident::new(&slot_ident.to_string().to_lowercase(), slot_ident.span()); - let slot_name = slot_ident.to_string(); - let tokens = { - const NON_ATOMIC_SLOTS: &[&str] = &["as_buffer"]; - const POINTER_SLOTS: &[&str] = &["as_sequence", "as_mapping"]; - const STATIC_GEN_SLOTS: &[&str] = &["as_number"]; - - if NON_ATOMIC_SLOTS.contains(&slot_name.as_str()) { - quote_spanned! { span => - slots.#slot_ident = Some(Self::#ident as _); - } - } else if POINTER_SLOTS.contains(&slot_name.as_str()) { - quote_spanned! { span => - slots.#slot_ident.store(Some(PointerSlot::from(Self::#ident()))); - } - } else if STATIC_GEN_SLOTS.contains(&slot_name.as_str()) { - quote_spanned! { span => - slots.#slot_ident = Self::#ident().into(); - } - } else { - quote_spanned! { span => - slots.#slot_ident.store(Some(Self::#ident as _)); - } - } - }; - - let pyname = format!("(slot {slot_name})"); - args.context.extend_slots_items.add_item( - ident.clone(), - vec![pyname], - args.cfgs.to_vec(), - tokens, - 2, - )?; - - Ok(()) - } -} - -impl<Item> ImplItem<Item> for AttributeItem -where - Item: ItemLike + ToTokens + GetIdent, -{ - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { - let cfgs = args.cfgs.to_vec(); - let attr = args.attrs.remove(self.index()); - - let get_py_name = |attr: &Attribute, ident: &Ident| -> Result<_> { - let item_meta = SimpleItemMeta::from_attr(ident.clone(), attr)?; - let py_name = item_meta.simple_name()?; - Ok(py_name) - }; - let (ident, py_name, tokens) = - if args.item.function_or_method().is_ok() || args.item.constant().is_ok() { - let ident = args.item.get_ident().unwrap(); - let py_name = get_py_name(&attr, ident)?; - - let value = if args.item.constant().is_ok() { - // TODO: ctx.new_value - quote_spanned!(ident.span() => ctx.new_int(Self::#ident).into()) - } else { - quote_spanned!(ident.span() => Self::#ident(ctx)) - }; - ( - ident, - py_name.clone(), - quote! { - class.set_str_attr(#py_name, #value, ctx); - }, - ) - } else { - return Err(self.new_syn_error( - args.item.span(), - "can only be on a const or an associated method without argument", - )); - }; - - args.context - .attribute_items - .add_item(ident.clone(), vec![py_name], cfgs, tokens, 1)?; - - Ok(()) - } -} - -impl<Item> ImplItem<Item> for ExtendClassItem -where - Item: ItemLike + ToTokens + GetIdent, -{ - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { - args.attrs.remove(self.index()); - - let ident = &args - .item - .function_or_method() - .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))? - .sig() - .ident; - - args.context.class_extensions.push(quote! { - Self::#ident(ctx, class); - }); - - Ok(()) - } -} - -impl<Item> ImplItem<Item> for MemberItem -where - Item: ItemLike + ToTokens + GetIdent, -{ - fn gen_impl_item(&self, args: ImplItemArgs<'_, Item>) -> Result<()> { - let func = args - .item - .function_or_method() - .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a method"))?; - let ident = &func.sig().ident; - - let item_attr = args.attrs.remove(self.index()); - let item_meta = MemberItemMeta::from_attr(ident.clone(), &item_attr)?; - - let (py_name, member_item_kind) = item_meta.member_name()?; - let member_kind = match item_meta.member_kind()? { - Some(s) => match s.as_str() { - "bool" => MemberKind::Bool, - _ => unreachable!(), - }, - _ => MemberKind::ObjectEx, - }; - - args.context.member_items.add_item( - py_name, - member_item_kind, - member_kind, - ident.clone(), - )?; - Ok(()) - } -} - -#[derive(Default)] -struct MethodNursery { - items: Vec<MethodNurseryItem>, -} - -struct MethodNurseryItem { - py_name: String, - cfgs: Vec<Attribute>, - ident: Ident, - doc: Option<String>, - attr_name: AttrName, -} - -impl MethodNursery { - fn add_item(&mut self, item: MethodNurseryItem) { - self.items.push(item); - } - - fn validate(&mut self) -> Result<()> { - let mut name_set = HashSet::new(); - for item in &self.items { - if !name_set.insert((&item.py_name, &item.cfgs)) { - bail_span!(item.ident, "duplicate method name `{}`", item.py_name); - } - } - Ok(()) - } -} - -impl ToTokens for MethodNursery { - fn to_tokens(&self, tokens: &mut TokenStream) { - let mut inner_tokens = TokenStream::new(); - for item in &self.items { - let py_name = &item.py_name; - let ident = &item.ident; - let cfgs = &item.cfgs; - let doc = if let Some(doc) = item.doc.as_ref() { - quote! { Some(#doc) } - } else { - quote! { None } - }; - let flags = match &item.attr_name { - AttrName::Method => { - quote! { rustpython_vm::function::PyMethodFlags::METHOD } - } - AttrName::ClassMethod => { - quote! { rustpython_vm::function::PyMethodFlags::CLASS } - } - AttrName::StaticMethod => { - quote! { rustpython_vm::function::PyMethodFlags::STATIC } - } - _ => unreachable!(), - }; - // TODO: intern - // let py_name = if py_name.starts_with("__") && py_name.ends_with("__") { - // let name_ident = Ident::new(&py_name, ident.span()); - // quote_spanned! { ident.span() => ctx.names.#name_ident } - // } else { - // quote_spanned! { ident.span() => #py_name } - // }; - inner_tokens.extend(quote! [ - #(#cfgs)* - rustpython_vm::function::PyMethodDef::new_const( - #py_name, - Self::#ident, - #flags, - #doc, - ), - ]); - } - let array: TokenTree = Group::new(Delimiter::Bracket, inner_tokens).into(); - tokens.extend([array]); - } -} - -#[derive(Default)] -#[allow(clippy::type_complexity)] -struct GetSetNursery { - map: HashMap<(String, Vec<Attribute>), (Option<Ident>, Option<Ident>, Option<Ident>)>, - validated: bool, -} - -enum GetSetItemKind { - Get, - Set, - Delete, -} - -impl GetSetNursery { - fn add_item( - &mut self, - name: String, - cfgs: Vec<Attribute>, - kind: GetSetItemKind, - item_ident: Ident, - ) -> Result<()> { - assert!(!self.validated, "new item is not allowed after validation"); - if !matches!(kind, GetSetItemKind::Get) && !cfgs.is_empty() { - bail_span!(item_ident, "Only the getter can have #[cfg]",); - } - let entry = self.map.entry((name.clone(), cfgs)).or_default(); - let func = match kind { - GetSetItemKind::Get => &mut entry.0, - GetSetItemKind::Set => &mut entry.1, - GetSetItemKind::Delete => &mut entry.2, - }; - if func.is_some() { - bail_span!( - item_ident, - "Multiple property accessors with name '{}'", - name - ); - } - *func = Some(item_ident); - Ok(()) - } - - fn validate(&mut self) -> Result<()> { - let mut errors = Vec::new(); - for ((name, _cfgs), (getter, setter, deleter)) in &self.map { - if getter.is_none() { - errors.push(err_span!( - setter.as_ref().or(deleter.as_ref()).unwrap(), - "GetSet '{}' is missing a getter", - name - )); - }; - } - errors.into_result()?; - self.validated = true; - Ok(()) - } -} - -impl ToTokens for GetSetNursery { - fn to_tokens(&self, tokens: &mut TokenStream) { - assert!(self.validated, "Call `validate()` before token generation"); - let properties = self - .map - .iter() - .map(|((name, cfgs), (getter, setter, deleter))| { - let setter = match setter { - Some(setter) => quote_spanned! { setter.span() => .with_set(Self::#setter)}, - None => quote! {}, - }; - let deleter = match deleter { - Some(deleter) => { - quote_spanned! { deleter.span() => .with_delete(Self::#deleter)} - } - None => quote! {}, - }; - quote_spanned! { getter.span() => - #( #cfgs )* - class.set_str_attr( - #name, - ::rustpython_vm::PyRef::new_ref( - ::rustpython_vm::builtins::PyGetSet::new(#name.into(), class) - .with_get(Self::#getter) - #setter #deleter, - ctx.types.getset_type.to_owned(), None), - ctx - ); - } - }); - tokens.extend(properties); - } -} - -#[derive(Default)] -#[allow(clippy::type_complexity)] -struct MemberNursery { - map: HashMap<(String, MemberKind), (Option<Ident>, Option<Ident>)>, - validated: bool, -} - -enum MemberItemKind { - Get, - Set, -} - -#[derive(Eq, PartialEq, Hash)] -enum MemberKind { - Bool, - ObjectEx, -} - -impl MemberNursery { - fn add_item( - &mut self, - name: String, - kind: MemberItemKind, - member_kind: MemberKind, - item_ident: Ident, - ) -> Result<()> { - assert!(!self.validated, "new item is not allowed after validation"); - let entry = self.map.entry((name.clone(), member_kind)).or_default(); - let func = match kind { - MemberItemKind::Get => &mut entry.0, - MemberItemKind::Set => &mut entry.1, - }; - if func.is_some() { - bail_span!(item_ident, "Multiple member accessors with name '{}'", name); - } - *func = Some(item_ident); - Ok(()) - } - - fn validate(&mut self) -> Result<()> { - let mut errors = Vec::new(); - for ((name, _), (getter, setter)) in &self.map { - if getter.is_none() { - errors.push(err_span!( - setter.as_ref().unwrap(), - "Member '{}' is missing a getter", - name - )); - }; - } - errors.into_result()?; - self.validated = true; - Ok(()) - } -} - -impl ToTokens for MemberNursery { - fn to_tokens(&self, tokens: &mut TokenStream) { - assert!(self.validated, "Call `validate()` before token generation"); - let properties = self - .map - .iter() - .map(|((name, member_kind), (getter, setter))| { - let setter = match setter { - Some(setter) => quote_spanned! { setter.span() => Some(Self::#setter)}, - None => quote! { None }, - }; - let member_kind = match member_kind { - MemberKind::Bool => { - quote!(::rustpython_vm::builtins::descriptor::MemberKind::Bool) - } - MemberKind::ObjectEx => { - quote!(::rustpython_vm::builtins::descriptor::MemberKind::ObjectEx) - } - }; - quote_spanned! { getter.span() => - class.set_str_attr( - #name, - ctx.new_member(#name, #member_kind, Self::#getter, #setter, class), - ctx, - ); - } - }); - tokens.extend(properties); - } -} - -struct MethodItemMeta(ItemMetaInner); - -impl ItemMeta for MethodItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["name", "magic"]; - - fn from_inner(inner: ItemMetaInner) -> Self { - Self(inner) - } - fn inner(&self) -> &ItemMetaInner { - &self.0 - } -} - -impl MethodItemMeta { - fn method_name(&self) -> Result<String> { - let inner = self.inner(); - let name = inner._optional_str("name")?; - let magic = inner._bool("magic")?; - Ok(if let Some(name) = name { - name - } else { - let name = inner.item_name(); - if magic { - format!("__{name}__") - } else { - name - } - }) - } -} - -struct GetSetItemMeta(ItemMetaInner); - -impl ItemMeta for GetSetItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["name", "magic", "setter", "deleter"]; - - fn from_inner(inner: ItemMetaInner) -> Self { - Self(inner) - } - fn inner(&self) -> &ItemMetaInner { - &self.0 - } -} - -impl GetSetItemMeta { - fn getset_name(&self) -> Result<(String, GetSetItemKind)> { - let inner = self.inner(); - let magic = inner._bool("magic")?; - let kind = match (inner._bool("setter")?, inner._bool("deleter")?) { - (false, false) => GetSetItemKind::Get, - (true, false) => GetSetItemKind::Set, - (false, true) => GetSetItemKind::Delete, - (true, true) => { - bail_span!( - &inner.meta_ident, - "can't have both setter and deleter on a #[{}] fn", - inner.meta_name() - ) - } - }; - let name = inner._optional_str("name")?; - let py_name = if let Some(name) = name { - name - } else { - let sig_name = inner.item_name(); - let extract_prefix_name = |prefix, item_typ| { - if let Some(name) = sig_name.strip_prefix(prefix) { - if name.is_empty() { - Err(err_span!( - inner.meta_ident, - "A #[{}({typ})] fn with a {prefix}* name must \ - have something after \"{prefix}\"", - inner.meta_name(), - typ = item_typ, - prefix = prefix - )) - } else { - Ok(name.to_owned()) - } - } else { - Err(err_span!( - inner.meta_ident, - "A #[{}(setter)] fn must either have a `name` \ - parameter or a fn name along the lines of \"set_*\"", - inner.meta_name() - )) - } - }; - let name = match kind { - GetSetItemKind::Get => sig_name, - GetSetItemKind::Set => extract_prefix_name("set_", "setter")?, - GetSetItemKind::Delete => extract_prefix_name("del_", "deleter")?, - }; - if magic { - format!("__{name}__") - } else { - name - } - }; - Ok((py_name, kind)) - } -} - -struct SlotItemMeta(ItemMetaInner); - -impl ItemMeta for SlotItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &[]; // not used - - fn from_nested<I>(item_ident: Ident, meta_ident: Ident, mut nested: I) -> Result<Self> - where - I: std::iter::Iterator<Item = NestedMeta>, - { - let meta_map = if let Some(nested_meta) = nested.next() { - match nested_meta { - NestedMeta::Meta(meta) => { - Some([("name".to_owned(), (0, meta))].iter().cloned().collect()) - } - _ => None, - } - } else { - Some(HashMap::default()) - }; - let (Some(meta_map), None) = (meta_map, nested.next()) else { - bail_span!( - meta_ident, - "#[pyslot] must be of the form #[pyslot] or #[pyslot(slot_name)]" - ) - }; - Ok(Self::from_inner(ItemMetaInner { - item_ident, - meta_ident, - meta_map, - })) - } - - fn from_inner(inner: ItemMetaInner) -> Self { - Self(inner) - } - fn inner(&self) -> &ItemMetaInner { - &self.0 - } -} - -impl SlotItemMeta { - fn slot_name(&self) -> Result<Ident> { - let inner = self.inner(); - let slot_name = if let Some((_, meta)) = inner.meta_map.get("name") { - match meta { - Meta::Path(path) => path.get_ident().cloned(), - _ => None, - } - } else { - let ident_str = self.inner().item_name(); - let name = if let Some(stripped) = ident_str.strip_prefix("slot_") { - proc_macro2::Ident::new(stripped, inner.item_ident.span()) - } else { - inner.item_ident.clone() - }; - Some(name) - }; - slot_name.ok_or_else(|| { - err_span!( - inner.meta_ident, - "#[pyslot] must be of the form #[pyslot] or #[pyslot(slot_name)]", - ) - }) - } -} - -struct MemberItemMeta(ItemMetaInner); - -impl ItemMeta for MemberItemMeta { - const ALLOWED_NAMES: &'static [&'static str] = &["magic", "type", "setter"]; - - fn from_inner(inner: ItemMetaInner) -> Self { - Self(inner) - } - fn inner(&self) -> &ItemMetaInner { - &self.0 - } -} - -impl MemberItemMeta { - fn member_name(&self) -> Result<(String, MemberItemKind)> { - let inner = self.inner(); - let sig_name = inner.item_name(); - let extract_prefix_name = |prefix, item_typ| { - if let Some(name) = sig_name.strip_prefix(prefix) { - if name.is_empty() { - Err(err_span!( - inner.meta_ident, - "A #[{}({typ})] fn with a {prefix}* name must \ - have something after \"{prefix}\"", - inner.meta_name(), - typ = item_typ, - prefix = prefix - )) - } else { - Ok(name.to_owned()) - } - } else { - Err(err_span!( - inner.meta_ident, - "A #[{}(setter)] fn must either have a `name` \ - parameter or a fn name along the lines of \"set_*\"", - inner.meta_name() - )) - } - }; - let magic = inner._bool("magic")?; - let kind = if inner._bool("setter")? { - MemberItemKind::Set - } else { - MemberItemKind::Get - }; - let name = match kind { - MemberItemKind::Get => sig_name, - MemberItemKind::Set => extract_prefix_name("set_", "setter")?, - }; - Ok((if magic { format!("__{name}__") } else { name }, kind)) - } - - fn member_kind(&self) -> Result<Option<String>> { - let inner = self.inner(); - inner._optional_str("type") - } -} - -struct ExtractedImplAttrs { - payload: Option<Ident>, - flags: TokenStream, - with_impl: TokenStream, - with_method_defs: Vec<TokenStream>, - with_slots: TokenStream, -} - -fn extract_impl_attrs(attr: AttributeArgs, item: &Ident) -> Result<ExtractedImplAttrs> { - let mut withs = Vec::new(); - let mut with_method_defs = Vec::new(); - let mut with_slots = Vec::new(); - let mut flags = vec![quote! { - { - #[cfg(not(debug_assertions))] { - ::rustpython_vm::types::PyTypeFlags::DEFAULT - } - #[cfg(debug_assertions)] { - ::rustpython_vm::types::PyTypeFlags::DEFAULT - .union(::rustpython_vm::types::PyTypeFlags::_CREATED_WITH_FLAGS) - } - } - }]; - let mut payload = None; - - for attr in attr { - match attr { - NestedMeta::Meta(Meta::List(syn::MetaList { path, nested, .. })) => { - if path.is_ident("with") { - for meta in nested { - let NestedMeta::Meta(Meta::Path(path)) = meta else { - bail_span!(meta, "#[pyclass(with(...))] arguments should be paths") - }; - let (extend_class, method_defs, extend_slots) = - if path.is_ident("PyRef") || path.is_ident("Py") { - // special handling for PyRef - ( - quote!(#path::<Self>::__extend_py_class), - quote!(#path::<Self>::__OWN_METHOD_DEFS), - quote!(#path::<Self>::__extend_slots), - ) - } else { - ( - quote!(<Self as #path>::__extend_py_class), - quote!(<Self as #path>::__OWN_METHOD_DEFS), - quote!(<Self as #path>::__extend_slots), - ) - }; - let item_span = item.span().resolved_at(Span::call_site()); - withs.push(quote_spanned! { path.span() => - #extend_class(ctx, class); - }); - with_method_defs.push(method_defs); - with_slots.push(quote_spanned! { item_span => - #extend_slots(slots); - }); - } - } else if path.is_ident("flags") { - for meta in nested { - let NestedMeta::Meta(Meta::Path(path)) = meta else { - bail_span!(meta, "#[pyclass(flags(...))] arguments should be ident") - }; - let ident = path.get_ident().ok_or_else(|| { - err_span!(path, "#[pyclass(flags(...))] arguments should be ident") - })?; - flags.push(quote_spanned! { ident.span() => - .union(::rustpython_vm::types::PyTypeFlags::#ident) - }); - } - } else { - bail_span!(path, "Unknown pyimpl attribute") - } - } - NestedMeta::Meta(Meta::NameValue(syn::MetaNameValue { path, lit, .. })) => { - if path.is_ident("payload") { - if let syn::Lit::Str(lit) = lit { - payload = Some(Ident::new(&lit.value(), lit.span())); - } else { - bail_span!(lit, "payload must be a string literal") - } - } else { - bail_span!(path, "Unknown pyimpl attribute") - } - } - attr => bail_span!(attr, "Unknown pyimpl attribute"), - } - } - - Ok(ExtractedImplAttrs { - payload, - flags: quote! { - #(#flags)* - }, - with_impl: quote! { - #(#withs)* - }, - with_method_defs, - with_slots: quote! { - #(#with_slots)* - }, - }) -} - -fn impl_item_new<Item>( - index: usize, - attr_name: AttrName, -) -> Result<Box<dyn ImplItem<Item, AttrName = AttrName>>> -where - Item: ItemLike + ToTokens + GetIdent, -{ - use AttrName::*; - Ok(match attr_name { - attr_name @ Method | attr_name @ ClassMethod | attr_name @ StaticMethod => { - Box::new(MethodItem { - inner: ContentItemInner { index, attr_name }, - }) - } - GetSet => Box::new(GetSetItem { - inner: ContentItemInner { index, attr_name }, - }), - Slot => Box::new(SlotItem { - inner: ContentItemInner { index, attr_name }, - }), - Attr => Box::new(AttributeItem { - inner: ContentItemInner { index, attr_name }, - }), - ExtendClass => Box::new(ExtendClassItem { - inner: ContentItemInner { index, attr_name }, - }), - Member => Box::new(MemberItem { - inner: ContentItemInner { index, attr_name }, - }), - }) -} - -fn attrs_to_content_items<F, R>( - attrs: &[Attribute], - item_new: F, -) -> Result<(Vec<R>, Vec<Attribute>)> -where - F: Fn(usize, AttrName) -> Result<R>, -{ - let mut cfgs: Vec<Attribute> = Vec::new(); - let mut result = Vec::new(); - - let mut iter = attrs.iter().enumerate().peekable(); - while let Some((_, attr)) = iter.peek() { - // take all cfgs but no py items - let attr = *attr; - let attr_name = if let Some(ident) = attr.get_ident() { - ident.to_string() - } else { - continue; - }; - if attr_name == "cfg" { - cfgs.push(attr.clone()); - } else if ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { - break; - } - iter.next(); - } - - for (i, attr) in iter { - // take py items but no cfgs - let attr_name = if let Some(ident) = attr.get_ident() { - ident.to_string() - } else { - continue; - }; - if attr_name == "cfg" { - bail_span!(attr, "#[py*] items must be placed under `cfgs`",); - } - let attr_name = match AttrName::from_str(attr_name.as_str()) { - Ok(name) => name, - Err(wrong_name) => { - if ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { - bail_span!(attr, "#[pyclass] doesn't accept #[{}]", wrong_name) - } else { - continue; - } - } - }; - - result.push(item_new(i, attr_name)?); - } - Ok((result, cfgs)) -} - -#[allow(dead_code)] -fn parse_vec_ident( - attr: &[NestedMeta], - item: &Item, - index: usize, - message: &str, -) -> Result<String> { - Ok(attr - .get(index) - .ok_or_else(|| err_span!(item, "We require {} argument to be set", message))? - .get_ident() - .ok_or_else(|| { - err_span!( - item, - "We require {} argument to be ident or string", - message - ) - })? - .to_string()) -} diff --git a/derive-impl/src/pymodule.rs b/derive-impl/src/pymodule.rs deleted file mode 100644 index b9a59c8280a..00000000000 --- a/derive-impl/src/pymodule.rs +++ /dev/null @@ -1,744 +0,0 @@ -use crate::error::Diagnostic; -use crate::util::{ - format_doc, iter_use_idents, pyclass_ident_and_attrs, text_signature, AttrItemMeta, - AttributeExt, ClassItemMeta, ContentItem, ContentItemInner, ErrorVec, ItemMeta, ItemNursery, - ModuleItemMeta, SimpleItemMeta, ALL_ALLOWED_NAMES, -}; -use proc_macro2::{Delimiter, Group, TokenStream, TokenTree}; -use quote::{quote, quote_spanned, ToTokens}; -use std::{collections::HashSet, str::FromStr}; -use syn::{parse_quote, spanned::Spanned, Attribute, AttributeArgs, Ident, Item, Result}; -use syn_ext::ext::*; - -#[derive(Clone, Copy, Eq, PartialEq)] -enum AttrName { - Function, - Attr, - Class, -} - -impl std::fmt::Display for AttrName { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let s = match self { - Self::Function => "pyfunction", - Self::Attr => "pyattr", - Self::Class => "pyclass", - }; - s.fmt(f) - } -} - -impl FromStr for AttrName { - type Err = String; - fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { - Ok(match s { - "pyfunction" => Self::Function, - "pyattr" => Self::Attr, - "pyclass" => Self::Class, - s => { - return Err(s.to_owned()); - } - }) - } -} - -#[derive(Default)] -struct ModuleContext { - name: String, - function_items: FunctionNursery, - attribute_items: ItemNursery, - has_extend_module: bool, // TODO: check if `fn extend_module` exists - errors: Vec<syn::Error>, -} - -pub fn impl_pymodule(attr: AttributeArgs, module_item: Item) -> Result<TokenStream> { - let (doc, mut module_item) = match module_item { - Item::Mod(m) => (m.attrs.doc(), m), - other => bail_span!(other, "#[pymodule] can only be on a full module"), - }; - let fake_ident = Ident::new("pymodule", module_item.span()); - let module_meta = - ModuleItemMeta::from_nested(module_item.ident.clone(), fake_ident, attr.into_iter())?; - - // generation resources - let mut context = ModuleContext { - name: module_meta.simple_name()?, - ..Default::default() - }; - let items = module_item.items_mut().ok_or_else(|| { - module_meta.new_meta_error("requires actual module, not a module declaration") - })?; - - // collect to context - for item in items.iter_mut() { - if matches!(item, Item::Impl(_) | Item::Trait(_)) { - // #[pyclass] implementations - continue; - } - let r = item.try_split_attr_mut(|attrs, item| { - let (py_items, cfgs) = attrs_to_module_items(attrs, module_item_new)?; - for py_item in py_items.iter().rev() { - let r = py_item.gen_module_item(ModuleItemArgs { - item, - attrs, - context: &mut context, - cfgs: cfgs.as_slice(), - }); - context.errors.ok_or_push(r); - } - Ok(()) - }); - context.errors.ok_or_push(r); - } - - // append additional items - let module_name = context.name.as_str(); - let function_items = context.function_items.validate()?; - let attribute_items = context.attribute_items.validate()?; - let doc = doc.or_else(|| { - crate::doc::Database::shared() - .try_path(module_name) - .ok() - .flatten() - .map(str::to_owned) - }); - let doc = if let Some(doc) = doc { - quote!(Some(#doc)) - } else { - quote!(None) - }; - let is_submodule = module_meta.sub()?; - let withs = module_meta.with()?; - if !is_submodule { - items.extend([ - parse_quote! { - pub(crate) const MODULE_NAME: &'static str = #module_name; - }, - parse_quote! { - pub(crate) const DOC: Option<&'static str> = #doc; - }, - parse_quote! { - pub(crate) fn __module_def( - ctx: &::rustpython_vm::Context, - ) -> &'static ::rustpython_vm::builtins::PyModuleDef { - DEF.get_or_init(|| { - let mut def = ::rustpython_vm::builtins::PyModuleDef { - name: ctx.intern_str(MODULE_NAME), - doc: DOC.map(|doc| ctx.intern_str(doc)), - methods: METHOD_DEFS, - slots: Default::default(), - }; - def.slots.exec = Some(extend_module); - def - }) - } - }, - parse_quote! { - #[allow(dead_code)] - pub(crate) fn make_module( - vm: &::rustpython_vm::VirtualMachine - ) -> ::rustpython_vm::PyRef<::rustpython_vm::builtins::PyModule> { - use ::rustpython_vm::PyPayload; - let module = ::rustpython_vm::builtins::PyModule::from_def(__module_def(&vm.ctx)).into_ref(&vm.ctx); - __init_dict(vm, &module); - extend_module(vm, &module).unwrap(); - module - } - }, - ]); - } - if !is_submodule && !context.has_extend_module { - items.push(parse_quote! { - pub(crate) fn extend_module(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> { - __extend_module(vm, module); - Ok(()) - } - }); - } - let method_defs = if withs.is_empty() { - quote!(#function_items) - } else { - quote!({ - const OWN_METHODS: &'static [::rustpython_vm::function::PyMethodDef] = &#function_items; - rustpython_vm::function::PyMethodDef::__const_concat_arrays::< - { OWN_METHODS.len() #(+ super::#withs::METHOD_DEFS.len())* }, - >(&[#(super::#withs::METHOD_DEFS,)* OWN_METHODS]) - }) - }; - items.extend([ - parse_quote! { - ::rustpython_vm::common::static_cell! { - pub(crate) static DEF: ::rustpython_vm::builtins::PyModuleDef; - } - }, - parse_quote! { - pub(crate) const METHOD_DEFS: &'static [::rustpython_vm::function::PyMethodDef] = &#method_defs; - }, - parse_quote! { - pub(crate) fn __init_attributes( - vm: &::rustpython_vm::VirtualMachine, - module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>, - ) { - #( - super::#withs::__init_attributes(vm, module); - )* - let ctx = &vm.ctx; - #attribute_items - } - }, - parse_quote! { - pub(crate) fn __extend_module( - vm: &::rustpython_vm::VirtualMachine, - module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>, - ) { - module.__init_methods(vm).unwrap(); - __init_attributes(vm, module); - } - }, - parse_quote! { - pub(crate) fn __init_dict( - vm: &::rustpython_vm::VirtualMachine, - module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>, - ) { - ::rustpython_vm::builtins::PyModule::__init_dict_from_def(vm, module); - } - }, - ]); - - Ok(if let Some(error) = context.errors.into_error() { - let error = Diagnostic::from(error); - quote! { - #module_item - #error - } - } else { - module_item.into_token_stream() - }) -} - -fn module_item_new( - index: usize, - attr_name: AttrName, - py_attrs: Vec<usize>, -) -> Box<dyn ModuleItem<AttrName = AttrName>> { - match attr_name { - AttrName::Function => Box::new(FunctionItem { - inner: ContentItemInner { index, attr_name }, - py_attrs, - }), - AttrName::Attr => Box::new(AttributeItem { - inner: ContentItemInner { index, attr_name }, - py_attrs, - }), - AttrName::Class => Box::new(ClassItem { - inner: ContentItemInner { index, attr_name }, - py_attrs, - }), - } -} - -fn attrs_to_module_items<F, R>(attrs: &[Attribute], item_new: F) -> Result<(Vec<R>, Vec<Attribute>)> -where - F: Fn(usize, AttrName, Vec<usize>) -> R, -{ - let mut cfgs: Vec<Attribute> = Vec::new(); - let mut result = Vec::new(); - - let mut iter = attrs.iter().enumerate().peekable(); - while let Some((_, attr)) = iter.peek() { - // take all cfgs but no py items - let attr = *attr; - if let Some(ident) = attr.get_ident() { - let attr_name = ident.to_string(); - if attr_name == "cfg" { - cfgs.push(attr.clone()); - } else if ALL_ALLOWED_NAMES.contains(&attr_name.as_str()) { - break; - } - } - iter.next(); - } - - let mut closed = false; - let mut py_attrs = Vec::new(); - for (i, attr) in iter { - // take py items but no cfgs - let attr_name = if let Some(ident) = attr.get_ident() { - ident.to_string() - } else { - continue; - }; - if attr_name == "cfg" { - bail_span!(attr, "#[py*] items must be placed under `cfgs`") - } - - let attr_name = match AttrName::from_str(attr_name.as_str()) { - Ok(name) => name, - Err(wrong_name) => { - if !ALL_ALLOWED_NAMES.contains(&wrong_name.as_str()) { - continue; - } else if closed { - bail_span!(attr, "Only one #[pyattr] annotated #[py*] item can exist") - } else { - bail_span!(attr, "#[pymodule] doesn't accept #[{}]", wrong_name) - } - } - }; - - if attr_name == AttrName::Attr { - if !result.is_empty() { - bail_span!( - attr, - "#[pyattr] must be placed on top of other #[py*] items", - ) - } - py_attrs.push(i); - continue; - } - - if py_attrs.is_empty() { - result.push(item_new(i, attr_name, Vec::new())); - } else { - match attr_name { - AttrName::Class | AttrName::Function => { - result.push(item_new(i, attr_name, py_attrs.clone())); - } - _ => { - bail_span!( - attr, - "#[pyclass] or #[pyfunction] only can follow #[pyattr]", - ) - } - } - py_attrs.clear(); - closed = true; - } - } - - if let Some(last) = py_attrs.pop() { - assert!(!closed); - result.push(item_new(last, AttrName::Attr, py_attrs)); - } - Ok((result, cfgs)) -} - -#[derive(Default)] -struct FunctionNursery { - items: Vec<FunctionNurseryItem>, -} - -struct FunctionNurseryItem { - py_names: Vec<String>, - cfgs: Vec<Attribute>, - ident: Ident, - doc: String, -} - -impl FunctionNursery { - fn add_item(&mut self, item: FunctionNurseryItem) { - self.items.push(item); - } - - fn validate(self) -> Result<ValidatedFunctionNursery> { - let mut name_set = HashSet::new(); - for item in &self.items { - for py_name in &item.py_names { - if !name_set.insert((py_name.to_owned(), &item.cfgs)) { - bail_span!(item.ident, "duplicate method name `{}`", py_name); - } - } - } - Ok(ValidatedFunctionNursery(self)) - } -} - -struct ValidatedFunctionNursery(FunctionNursery); - -impl ToTokens for ValidatedFunctionNursery { - fn to_tokens(&self, tokens: &mut TokenStream) { - let mut inner_tokens = TokenStream::new(); - let flags = quote! { rustpython_vm::function::PyMethodFlags::empty() }; - for item in &self.0.items { - let ident = &item.ident; - let cfgs = &item.cfgs; - let cfgs = quote!(#(#cfgs)*); - let py_names = &item.py_names; - let doc = &item.doc; - let doc = quote!(Some(#doc)); - - inner_tokens.extend(quote![ - #( - #cfgs - rustpython_vm::function::PyMethodDef::new_const( - #py_names, - #ident, - #flags, - #doc, - ), - )* - ]); - } - let array: TokenTree = Group::new(Delimiter::Bracket, inner_tokens).into(); - tokens.extend([array]); - } -} - -/// #[pyfunction] -struct FunctionItem { - inner: ContentItemInner<AttrName>, - py_attrs: Vec<usize>, -} - -/// #[pyclass] -struct ClassItem { - inner: ContentItemInner<AttrName>, - py_attrs: Vec<usize>, -} - -/// #[pyattr] -struct AttributeItem { - inner: ContentItemInner<AttrName>, - py_attrs: Vec<usize>, -} - -impl ContentItem for FunctionItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} - -impl ContentItem for ClassItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} - -impl ContentItem for AttributeItem { - type AttrName = AttrName; - fn inner(&self) -> &ContentItemInner<AttrName> { - &self.inner - } -} - -struct ModuleItemArgs<'a> { - item: &'a mut Item, - attrs: &'a mut Vec<Attribute>, - context: &'a mut ModuleContext, - cfgs: &'a [Attribute], -} - -impl<'a> ModuleItemArgs<'a> { - fn module_name(&'a self) -> &'a str { - self.context.name.as_str() - } -} - -trait ModuleItem: ContentItem { - fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()>; -} - -impl ModuleItem for FunctionItem { - fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { - let func = args - .item - .function_or_method() - .map_err(|_| self.new_syn_error(args.item.span(), "can only be on a function"))?; - let ident = &func.sig().ident; - - let item_attr = args.attrs.remove(self.index()); - let item_meta = SimpleItemMeta::from_attr(ident.clone(), &item_attr)?; - - let py_name = item_meta.simple_name()?; - let sig_doc = text_signature(func.sig(), &py_name); - - let module = args.module_name(); - let doc = args.attrs.doc().or_else(|| { - crate::doc::Database::shared() - .try_module_item(module, &py_name) - .ok() // TODO: doc must exist at least one of code or CPython - .flatten() - .map(str::to_owned) - }); - let doc = if let Some(doc) = doc { - format_doc(&sig_doc, &doc) - } else { - sig_doc - }; - - let py_names = { - if self.py_attrs.is_empty() { - vec![py_name] - } else { - let mut py_names = HashSet::new(); - py_names.insert(py_name); - for attr_index in self.py_attrs.iter().rev() { - let mut loop_unit = || { - let attr_attr = args.attrs.remove(*attr_index); - let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr_attr)?; - - let py_name = item_meta.simple_name()?; - let inserted = py_names.insert(py_name.clone()); - if !inserted { - return Err(self.new_syn_error( - ident.span(), - &format!( - "`{py_name}` is duplicated name for multiple py* attribute" - ), - )); - } - Ok(()) - }; - let r = loop_unit(); - args.context.errors.ok_or_push(r); - } - let py_names: Vec<_> = py_names.into_iter().collect(); - py_names - } - }; - - args.context.function_items.add_item(FunctionNurseryItem { - ident: ident.to_owned(), - py_names, - cfgs: args.cfgs.to_vec(), - doc, - }); - Ok(()) - } -} - -impl ModuleItem for ClassItem { - fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { - let (ident, _) = pyclass_ident_and_attrs(args.item)?; - let (class_name, class_new) = { - let class_attr = &mut args.attrs[self.inner.index]; - let no_attr = class_attr.try_remove_name("no_attr")?; - if self.py_attrs.is_empty() { - // check no_attr before ClassItemMeta::from_attr - if no_attr.is_none() { - bail_span!( - ident, - "#[{name}] requires #[pyattr] to be a module attribute. \ - To keep it free type, try #[{name}(no_attr)]", - name = self.attr_name() - ) - } - } - let no_attr = no_attr.is_some(); - let is_use = matches!(&args.item, syn::Item::Use(_)); - - let class_meta = ClassItemMeta::from_attr(ident.clone(), class_attr)?; - let module_name = args.context.name.clone(); - let module_name = if let Some(class_module_name) = class_meta.module().ok().flatten() { - class_module_name - } else { - class_attr.fill_nested_meta("module", || { - parse_quote! {module = #module_name} - })?; - module_name - }; - let class_name = if no_attr && is_use { - "<NO ATTR>".to_owned() - } else { - class_meta.class_name()? - }; - let class_new = quote_spanned!(ident.span() => - let new_class = <#ident as ::rustpython_vm::class::PyClassImpl>::make_class(ctx); - new_class.set_attr(rustpython_vm::identifier!(ctx, __module__), vm.new_pyobj(#module_name)); - ); - (class_name, class_new) - }; - - let mut py_names = Vec::new(); - for attr_index in self.py_attrs.iter().rev() { - let mut loop_unit = || { - let attr_attr = args.attrs.remove(*attr_index); - let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr_attr)?; - - let py_name = item_meta - .optional_name() - .unwrap_or_else(|| class_name.clone()); - py_names.push(py_name); - - Ok(()) - }; - let r = loop_unit(); - args.context.errors.ok_or_push(r); - } - - let set_attr = match py_names.len() { - 0 => quote! { - let _ = new_class; // suppress warning - let _ = vm.ctx.intern_str(#class_name); - }, - 1 => { - let py_name = &py_names[0]; - quote! { - vm.__module_set_attr(&module, vm.ctx.intern_str(#py_name), new_class).unwrap(); - } - } - _ => quote! { - for name in [#(#py_names,)*] { - vm.__module_set_attr(&module, vm.ctx.intern_str(name), new_class.clone()).unwrap(); - } - }, - }; - - args.context.attribute_items.add_item( - ident.clone(), - py_names, - args.cfgs.to_vec(), - quote_spanned! { ident.span() => - #class_new - #set_attr - }, - 0, - )?; - Ok(()) - } -} - -impl ModuleItem for AttributeItem { - fn gen_module_item(&self, args: ModuleItemArgs<'_>) -> Result<()> { - let cfgs = args.cfgs.to_vec(); - let attr = args.attrs.remove(self.index()); - let (ident, py_name, let_obj) = match args.item { - Item::Fn(syn::ItemFn { sig, block, .. }) => { - let ident = &sig.ident; - // If `once` keyword is in #[pyattr], - // wrapping it with static_cell for preventing it from using it as function - let attr_meta = AttrItemMeta::from_attr(ident.clone(), &attr)?; - if attr_meta.inner()._bool("once")? { - let stmts = &block.stmts; - let return_type = match &sig.output { - syn::ReturnType::Default => { - unreachable!("#[pyattr] attached function must have return type.") - } - syn::ReturnType::Type(_, ty) => ty, - }; - let stmt: syn::Stmt = parse_quote! { - { - rustpython_common::static_cell! { - static ERROR: #return_type; - } - ERROR - .get_or_init(|| { - #(#stmts)* - }) - .clone() - } - }; - block.stmts = vec![stmt]; - } - - let py_name = attr_meta.simple_name()?; - ( - ident.clone(), - py_name, - quote_spanned! { ident.span() => - let obj = vm.new_pyobj(#ident(vm)); - }, - ) - } - Item::Const(syn::ItemConst { ident, .. }) => { - let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr)?; - let py_name = item_meta.simple_name()?; - ( - ident.clone(), - py_name, - quote_spanned! { ident.span() => - let obj = vm.new_pyobj(#ident); - }, - ) - } - Item::Use(item) => { - if !self.py_attrs.is_empty() { - return Err(self - .new_syn_error(item.span(), "Only single #[pyattr] is allowed for `use`")); - } - let _ = iter_use_idents(item, |ident, is_unique| { - let item_meta = SimpleItemMeta::from_attr(ident.clone(), &attr)?; - let py_name = if is_unique { - item_meta.simple_name()? - } else if item_meta.optional_name().is_some() { - // this check actually doesn't need to be placed in loop - return Err(self.new_syn_error( - ident.span(), - "`name` attribute is not allowed for multiple use items", - )); - } else { - ident.to_string() - }; - let tokens = quote_spanned! { ident.span() => - vm.__module_set_attr(module, vm.ctx.intern_str(#py_name), vm.new_pyobj(#ident)).unwrap(); - }; - args.context.attribute_items.add_item( - ident.clone(), - vec![py_name], - cfgs.clone(), - tokens, - 1, - )?; - Ok(()) - })?; - return Ok(()); - } - other => { - return Err( - self.new_syn_error(other.span(), "can only be on a function, const and use") - ) - } - }; - - let (tokens, py_names) = if self.py_attrs.is_empty() { - ( - quote_spanned! { ident.span() => { - #let_obj - vm.__module_set_attr(module, vm.ctx.intern_str(#py_name), obj).unwrap(); - }}, - vec![py_name], - ) - } else { - let mut names = vec![py_name]; - for attr_index in self.py_attrs.iter().rev() { - let mut loop_unit = || { - let attr_attr = args.attrs.remove(*attr_index); - let item_meta = AttrItemMeta::from_attr(ident.clone(), &attr_attr)?; - if item_meta.inner()._bool("once")? { - return Err(self.new_syn_error( - ident.span(), - "#[pyattr(once)] is only allowed for the bottom-most item", - )); - } - - let py_name = item_meta.optional_name().ok_or_else(|| { - self.new_syn_error( - ident.span(), - "#[pyattr(name = ...)] is mandatory except for the bottom-most item", - ) - })?; - names.push(py_name); - Ok(()) - }; - let r = loop_unit(); - args.context.errors.ok_or_push(r); - } - ( - quote_spanned! { ident.span() => { - #let_obj - for name in [(#(#names,)*)] { - vm.__module_set_attr(module, vm.ctx.intern_str(name), obj.clone()).unwrap(); - } - }}, - names, - ) - }; - - args.context - .attribute_items - .add_item(ident, py_names, cfgs, tokens, 1)?; - - Ok(()) - } -} diff --git a/derive-impl/src/pystructseq.rs b/derive-impl/src/pystructseq.rs deleted file mode 100644 index dce8c4768af..00000000000 --- a/derive-impl/src/pystructseq.rs +++ /dev/null @@ -1,70 +0,0 @@ -use proc_macro2::TokenStream; -use quote::quote; -use syn::{DeriveInput, Ident, Result}; - -fn field_names(input: &DeriveInput) -> Result<Vec<&Ident>> { - let fields = if let syn::Data::Struct(ref struc) = input.data { - &struc.fields - } else { - bail_span!( - input, - "#[pystruct_sequence] can only be on a struct declaration" - ) - }; - - let field_names: Vec<_> = match fields { - syn::Fields::Named(fields) => fields - .named - .iter() - .map(|field| field.ident.as_ref().unwrap()) - .collect(), - _ => bail_span!( - input, - "#[pystruct_sequence] can only be on a struct with named fields" - ), - }; - - Ok(field_names) -} - -pub(crate) fn impl_pystruct_sequence(input: DeriveInput) -> Result<TokenStream> { - let field_names = field_names(&input)?; - let ty = &input.ident; - let ret = quote! { - impl ::rustpython_vm::types::PyStructSequence for #ty { - const FIELD_NAMES: &'static [&'static str] = &[#(stringify!(#field_names)),*]; - fn into_tuple(self, vm: &::rustpython_vm::VirtualMachine) -> ::rustpython_vm::builtins::PyTuple { - let items = vec![#(::rustpython_vm::convert::ToPyObject::to_pyobject( - self.#field_names, - vm, - )),*]; - ::rustpython_vm::builtins::PyTuple::new_unchecked(items.into_boxed_slice()) - } - } - impl ::rustpython_vm::convert::ToPyObject for #ty { - fn to_pyobject(self, vm: &::rustpython_vm::VirtualMachine) -> ::rustpython_vm::PyObjectRef { - ::rustpython_vm::types::PyStructSequence::into_struct_sequence(self, vm).into() - } - } - }; - Ok(ret) -} - -pub(crate) fn impl_pystruct_sequence_try_from_object(input: DeriveInput) -> Result<TokenStream> { - let field_names = field_names(&input)?; - let ty = &input.ident; - let ret = quote! { - impl ::rustpython_vm::TryFromObject for #ty { - fn try_from_object(vm: &::rustpython_vm::VirtualMachine, seq: ::rustpython_vm::PyObjectRef) -> ::rustpython_vm::PyResult<Self> { - const LEN: usize = #ty::FIELD_NAMES.len(); - let seq = Self::try_elements_from::<LEN>(seq, vm)?; - // TODO: this is possible to be written without iterator - let mut iter = seq.into_iter(); - Ok(Self {#( - #field_names: iter.next().unwrap().clone().try_into_value(vm)? - ),*}) - } - } - }; - Ok(ret) -} diff --git a/derive-impl/src/pytraverse.rs b/derive-impl/src/pytraverse.rs deleted file mode 100644 index 93aa233a189..00000000000 --- a/derive-impl/src/pytraverse.rs +++ /dev/null @@ -1,138 +0,0 @@ -use proc_macro2::TokenStream; -use quote::quote; -use syn::{Attribute, DeriveInput, Field, Meta, MetaList, NestedMeta, Result}; - -struct TraverseAttr { - /// set to `true` if the attribute is `#[pytraverse(skip)]` - skip: bool, -} - -const ATTR_TRAVERSE: &str = "pytraverse"; - -/// get the `#[pytraverse(..)]` attribute from the struct -fn valid_get_traverse_attr_from_meta_list(list: &MetaList) -> Result<TraverseAttr> { - let find_skip_and_only_skip = || { - let len = list.nested.len(); - if len != 1 { - return None; - } - let mut iter = list.nested.iter(); - // we have checked the length, so unwrap is safe - let first_arg = iter.next().unwrap(); - let skip = match first_arg { - NestedMeta::Meta(Meta::Path(path)) => match path.is_ident("skip") { - true => true, - false => return None, - }, - _ => return None, - }; - Some(skip) - }; - let skip = find_skip_and_only_skip().ok_or_else(|| { - err_span!( - list, - "only support attr is #[pytraverse(skip)], got arguments: {:?}", - list.nested - ) - })?; - Ok(TraverseAttr { skip }) -} - -/// only accept `#[pytraverse(skip)]` for now -fn pytraverse_arg(attr: &Attribute) -> Option<Result<TraverseAttr>> { - if !attr.path.is_ident(ATTR_TRAVERSE) { - return None; - } - let ret = || { - let parsed = attr.parse_meta()?; - if let Meta::List(list) = parsed { - valid_get_traverse_attr_from_meta_list(&list) - } else { - bail_span!(attr, "pytraverse must be a list, like #[pytraverse(skip)]") - } - }; - Some(ret()) -} - -fn field_to_traverse_code(field: &Field) -> Result<TokenStream> { - let pytraverse_attrs = field - .attrs - .iter() - .filter_map(pytraverse_arg) - .collect::<std::result::Result<Vec<_>, _>>()?; - let do_trace = if pytraverse_attrs.len() > 1 { - bail_span!( - field, - "found multiple #[pytraverse] attributes on the same field, expect at most one" - ) - } else if pytraverse_attrs.is_empty() { - // default to always traverse every field - true - } else { - !pytraverse_attrs[0].skip - }; - let name = field.ident.as_ref().ok_or_else(|| { - syn::Error::new_spanned( - field.clone(), - "Field should have a name in non-tuple struct", - ) - })?; - if do_trace { - Ok(quote!( - ::rustpython_vm::object::Traverse::traverse(&self.#name, tracer_fn); - )) - } else { - Ok(quote!()) - } -} - -/// not trace corresponding field -fn gen_trace_code(item: &mut DeriveInput) -> Result<TokenStream> { - match &mut item.data { - syn::Data::Struct(s) => { - let fields = &mut s.fields; - match fields { - syn::Fields::Named(ref mut fields) => { - let res: Vec<TokenStream> = fields - .named - .iter_mut() - .map(|f| -> Result<TokenStream> { field_to_traverse_code(f) }) - .collect::<Result<_>>()?; - let res = res.into_iter().collect::<TokenStream>(); - Ok(res) - } - syn::Fields::Unnamed(fields) => { - let res: TokenStream = (0..fields.unnamed.len()) - .map(|i| { - let i = syn::Index::from(i); - quote!( - ::rustpython_vm::object::Traverse::traverse(&self.#i, tracer_fn); - ) - }) - .collect(); - Ok(res) - } - _ => Err(syn::Error::new_spanned( - fields, - "Only named and unnamed fields are supported", - )), - } - } - _ => Err(syn::Error::new_spanned(item, "Only structs are supported")), - } -} - -pub(crate) fn impl_pytraverse(mut item: DeriveInput) -> Result<TokenStream> { - let trace_code = gen_trace_code(&mut item)?; - - let ty = &item.ident; - - let ret = quote! { - unsafe impl ::rustpython_vm::object::Traverse for #ty { - fn traverse(&self, tracer_fn: &mut ::rustpython_vm::object::TraverseFn) { - #trace_code - } - } - }; - Ok(ret) -} diff --git a/derive/Cargo.toml b/derive/Cargo.toml deleted file mode 100644 index 5010877a6c9..00000000000 --- a/derive/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "rustpython-derive" -version = "0.3.0" -description = "Rust language extensions and macros specific to rustpython." -authors = ["RustPython Team"] -repository = "https://github.com/RustPython/RustPython" -license = "MIT" -edition = "2021" - -[lib] -proc-macro = true - -[dependencies] -rustpython-compiler = { workspace = true } -rustpython-derive-impl = { workspace = true } -syn = { workspace = true } diff --git a/derive/src/lib.rs b/derive/src/lib.rs deleted file mode 100644 index 76b7e234881..00000000000 --- a/derive/src/lib.rs +++ /dev/null @@ -1,107 +0,0 @@ -#![recursion_limit = "128"] -#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] -#![doc(html_root_url = "https://docs.rs/rustpython-derive/")] - -use proc_macro::TokenStream; -use rustpython_derive_impl as derive_impl; -use syn::parse_macro_input; - -#[proc_macro_derive(FromArgs, attributes(pyarg))] -pub fn derive_from_args(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input); - derive_impl::derive_from_args(input).into() -} - -#[proc_macro_attribute] -pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { - let attr = parse_macro_input!(attr); - let item = parse_macro_input!(item); - derive_impl::pyclass(attr, item).into() -} - -/// Helper macro to define `Exception` types. -/// More-or-less is an alias to `pyclass` macro. -/// -/// This macro serves a goal of generating multiple -/// `BaseException` / `Exception` -/// subtypes in a uniform and convenient manner. -/// It looks like `SimpleExtendsException` in `CPython`. -/// <https://github.com/python/cpython/blob/main/Objects/exceptions.c> -#[proc_macro_attribute] -pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { - let attr = parse_macro_input!(attr); - let item = parse_macro_input!(item); - derive_impl::pyexception(attr, item).into() -} - -#[proc_macro_attribute] -pub fn pymodule(attr: TokenStream, item: TokenStream) -> TokenStream { - let attr = parse_macro_input!(attr); - let item = parse_macro_input!(item); - derive_impl::pymodule(attr, item).into() -} - -#[proc_macro_derive(PyStructSequence)] -pub fn pystruct_sequence(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input); - derive_impl::pystruct_sequence(input).into() -} - -#[proc_macro_derive(TryIntoPyStructSequence)] -pub fn pystruct_sequence_try_from_object(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input); - derive_impl::pystruct_sequence_try_from_object(input).into() -} - -struct Compiler; -impl derive_impl::Compiler for Compiler { - fn compile( - &self, - source: &str, - mode: rustpython_compiler::Mode, - module_name: String, - ) -> Result<rustpython_compiler::CodeObject, Box<dyn std::error::Error>> { - use rustpython_compiler::{compile, CompileOpts}; - Ok(compile(source, mode, module_name, CompileOpts::default())?) - } -} - -#[proc_macro] -pub fn py_compile(input: TokenStream) -> TokenStream { - derive_impl::py_compile(input.into(), &Compiler).into() -} - -#[proc_macro] -pub fn py_freeze(input: TokenStream) -> TokenStream { - derive_impl::py_freeze(input.into(), &Compiler).into() -} - -#[proc_macro_derive(PyPayload)] -pub fn pypayload(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input); - derive_impl::pypayload(input).into() -} - -/// use on struct with named fields like `struct A{x:PyRef<B>, y:PyRef<C>}` to impl `Traverse` for datatype. -/// -/// use `#[pytraverse(skip)]` on fields you wish not to trace -/// -/// add `trace` attr to `#[pyclass]` to make it impl `MaybeTraverse` that will call `Traverse`'s `traverse` method so make it -/// traceable(Even from type-erased PyObject)(i.e. write `#[pyclass(trace)]`). -/// # Example -/// ```rust, ignore -/// #[pyclass(module = false, traverse)] -/// #[derive(Default, Traverse)] -/// pub struct PyList { -/// elements: PyRwLock<Vec<PyObjectRef>>, -/// #[pytraverse(skip)] -/// len: AtomicCell<usize>, -/// } -/// ``` -/// This create both `MaybeTraverse` that call `Traverse`'s `traverse` method and `Traverse` that impl `Traverse` -/// for `PyList` which call elements' `traverse` method and ignore `len` field. -#[proc_macro_derive(Traverse, attributes(pytraverse))] -pub fn pytraverse(item: proc_macro::TokenStream) -> proc_macro::TokenStream { - let item = parse_macro_input!(item); - derive_impl::pytraverse(item).into() -} diff --git a/example_projects/.gitignore b/example_projects/.gitignore new file mode 100644 index 00000000000..661ad40fd98 --- /dev/null +++ b/example_projects/.gitignore @@ -0,0 +1,2 @@ +*/target +*/Cargo.lock diff --git a/example_projects/aheui-rust.md b/example_projects/aheui-rust.md new file mode 100644 index 00000000000..d812ee244fb --- /dev/null +++ b/example_projects/aheui-rust.md @@ -0,0 +1,10 @@ +# aheui-rust + +- Crate link: https://github.com/youknowone/aheui-rust/tree/main/rpaheui +- Creating a frozenlib: https://github.com/youknowone/aheui-rust/blob/main/rpaheui/src/lib.rs + +This crate shows you how to embed an entire Python project into bytecode and ship it. Follow the `FROZEN` constant and see how it's made and how you can use it. + +If you'd like to learn more about how to initialize the Standard Library with the `freeze-stdlib` feature, check out the example project at `example_projects/frozen_stdlib/src/main.rs`. + +Just a heads-up, it doesn't automatically resolve dependencies. If you have more dependencies than the standard library, Don't forget to also freeze them. diff --git a/example_projects/barebone/Cargo.toml b/example_projects/barebone/Cargo.toml new file mode 100644 index 00000000000..f622b65f75e --- /dev/null +++ b/example_projects/barebone/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "example-barebone" +version = "0.1.0" +edition = "2021" + +[dependencies] +rustpython-vm = { path = "../../crates/vm", default-features = false, features = ["stdio"] } + +[workspace] + +[patch.crates-io] diff --git a/example_projects/barebone/src/main.rs b/example_projects/barebone/src/main.rs new file mode 100644 index 00000000000..0d9ffbefab1 --- /dev/null +++ b/example_projects/barebone/src/main.rs @@ -0,0 +1,17 @@ +use rustpython_vm::Interpreter; + +pub fn main() { + let interp = Interpreter::without_stdlib(Default::default()); + let value = interp.enter(|vm| { + let max = vm.builtins.get_attr("max", vm)?; + let value = max.call((vm.ctx.new_int(5), vm.ctx.new_int(10)), vm)?; + vm.print((vm.ctx.new_str("python print"), value.clone()))?; + Ok(value) + }); + match value { + Ok(value) => println!("Rust repr: {:?}", value), + Err(err) => { + interp.finalize(err); + } + } +} diff --git a/example_projects/frozen_stdlib/Cargo.toml b/example_projects/frozen_stdlib/Cargo.toml new file mode 100644 index 00000000000..91bb14a083b --- /dev/null +++ b/example_projects/frozen_stdlib/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "example_frozen_stdlib" +version = "0.1.0" +edition = "2021" + +[dependencies] +rustpython = { path = "../../", default-features = false, features = ["freeze-stdlib"] } +rustpython-vm = { path = "../../crates/vm", default-features = false, features = ["freeze-stdlib"] } +rustpython-pylib = { path = "../../crates/pylib", default-features = false, features = ["freeze-stdlib"] } + +[workspace] + +[patch.crates-io] diff --git a/example_projects/frozen_stdlib/src/main.rs b/example_projects/frozen_stdlib/src/main.rs new file mode 100644 index 00000000000..8ff316faed8 --- /dev/null +++ b/example_projects/frozen_stdlib/src/main.rs @@ -0,0 +1,39 @@ +// spell-checker:ignore aheui +//! Setting up a project with a frozen stdlib can be done *either* by using `rustpython::InterpreterBuilder` or `rustpython_vm::Interpreter::builder`. +//! See each function for example. +//! +//! See also: `aheui-rust.md` for freezing your own package. + +use rustpython::InterpreterBuilderExt; +use rustpython_vm::{PyResult, VirtualMachine}; + +fn run(keyword: &str, vm: &VirtualMachine) -> PyResult<()> { + let json = vm.import("json", 0)?; + let json_loads = json.get_attr("loads", vm)?; + let template = r#"{"key": "value"}"#; + let json_string = template.replace("value", keyword); + let dict = json_loads.call((vm.ctx.new_str(json_string),), vm)?; + vm.print((dict,))?; + Ok(()) +} + +fn interpreter_with_config() { + let interpreter = rustpython::InterpreterBuilder::new() + .init_stdlib() + .interpreter(); + // Use interpreter.enter to reuse the same interpreter later + interpreter.run(|vm| run("rustpython::InterpreterBuilder", vm)); +} + +fn interpreter_with_vm() { + let interpreter = rustpython_vm::Interpreter::builder(Default::default()) + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + .build(); + // Use interpreter.enter to reuse the same interpreter later + interpreter.run(|vm| run("rustpython_vm::Interpreter::builder", vm)); +} + +fn main() { + interpreter_with_config(); + interpreter_with_vm(); +} diff --git a/example_projects/wasm32_without_js/.gitignore b/example_projects/wasm32_without_js/.gitignore new file mode 100644 index 00000000000..50a623b5daa --- /dev/null +++ b/example_projects/wasm32_without_js/.gitignore @@ -0,0 +1,2 @@ +*/target/ +*/Cargo.lock diff --git a/example_projects/wasm32_without_js/README.md b/example_projects/wasm32_without_js/README.md new file mode 100644 index 00000000000..67fef3fba47 --- /dev/null +++ b/example_projects/wasm32_without_js/README.md @@ -0,0 +1,18 @@ +# RustPython wasm32 build without JS + +To test, build rustpython to wasm32-unknown-unknown target first. + +```shell +cd rustpython-without-js # due to `.cargo/config.toml` +cargo build +cd .. +``` + +Then there will be `rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm` file. + +Now we can run the wasm file with wasm runtime: + +```shell +cargo run --release --manifest-path wasm-runtime/Cargo.toml rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm +``` + diff --git a/example_projects/wasm32_without_js/rustpython-without-js/.cargo/config.toml b/example_projects/wasm32_without_js/rustpython-without-js/.cargo/config.toml new file mode 100644 index 00000000000..f86ad96761d --- /dev/null +++ b/example_projects/wasm32_without_js/rustpython-without-js/.cargo/config.toml @@ -0,0 +1,5 @@ +[build] +target = "wasm32-unknown-unknown" + +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"custom\""] diff --git a/example_projects/wasm32_without_js/rustpython-without-js/Cargo.toml b/example_projects/wasm32_without_js/rustpython-without-js/Cargo.toml new file mode 100644 index 00000000000..f987fe94a24 --- /dev/null +++ b/example_projects/wasm32_without_js/rustpython-without-js/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rustpython-without-js" +version = "0.1.0" +edition = "2024" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +getrandom = "0.3" +rustpython-vm = { path = "../../../crates/vm", default-features = false, features = ["compiler"] } + +[workspace] + +[patch.crates-io] diff --git a/example_projects/wasm32_without_js/rustpython-without-js/README.md b/example_projects/wasm32_without_js/rustpython-without-js/README.md new file mode 100644 index 00000000000..3c4abd59cf1 --- /dev/null +++ b/example_projects/wasm32_without_js/rustpython-without-js/README.md @@ -0,0 +1 @@ +A test crate to ensure that `rustpython-vm` compiles on `wasm32-unknown-unknown` without a JS host. diff --git a/example_projects/wasm32_without_js/rustpython-without-js/src/lib.rs b/example_projects/wasm32_without_js/rustpython-without-js/src/lib.rs new file mode 100644 index 00000000000..a152ebe1c4a --- /dev/null +++ b/example_projects/wasm32_without_js/rustpython-without-js/src/lib.rs @@ -0,0 +1,59 @@ +use rustpython_vm::{Interpreter}; + +unsafe extern "C" { + fn kv_get(kp: i32, kl: i32, vp: i32, vl: i32) -> i32; + + /// kp and kl are the key pointer and length in wasm memory, vp and vl are for the value + fn kv_put(kp: i32, kl: i32, vp: i32, vl: i32) -> i32; + + fn print(p: i32, l: i32) -> i32; +} + +#[unsafe(no_mangle)] +pub unsafe extern "C" fn eval(s: *const u8, l: usize) -> i32 { + // let src = unsafe { std::slice::from_raw_parts(s, l) }; + // let src = std::str::from_utf8(src).unwrap(); + // TODO: use src + let src = "1 + 3"; + + // 2. Execute Python code + let interpreter = Interpreter::without_stdlib(Default::default()); + let result = interpreter.enter(|vm| { + let scope = vm.new_scope_with_builtins(); + let res = match vm.run_block_expr(scope, src) { + Ok(val) => val, + Err(_) => return Err(-1), // Python execution error + }; + let repr_str = match res.repr(vm) { + Ok(repr) => repr.to_string(), + Err(_) => return Err(-1), // Failed to get string representation + }; + Ok(repr_str) + }); + let result = match result { + Ok(r) => r, + Err(code) => return code, + }; + + let msg = format!("eval result: {result}"); + + unsafe { + print( + msg.as_str().as_ptr() as usize as i32, + msg.len() as i32, + ) + }; + + 0 +} + +#[unsafe(no_mangle)] +unsafe extern "Rust" fn __getrandom_v03_custom( + _dest: *mut u8, + _len: usize, +) -> Result<(), getrandom::Error> { + // Err(getrandom::Error::UNSUPPORTED) + + // WARNING: This function **MUST** perform proper getrandom + Ok(()) +} diff --git a/example_projects/wasm32_without_js/wasm-runtime/.gitignore b/example_projects/wasm32_without_js/wasm-runtime/.gitignore new file mode 100644 index 00000000000..2e2101b5066 --- /dev/null +++ b/example_projects/wasm32_without_js/wasm-runtime/.gitignore @@ -0,0 +1,4 @@ +*.wasm +target +Cargo.lock +!wasm/rustpython.wasm diff --git a/example_projects/wasm32_without_js/wasm-runtime/Cargo.toml b/example_projects/wasm32_without_js/wasm-runtime/Cargo.toml new file mode 100644 index 00000000000..a1d0de51719 --- /dev/null +++ b/example_projects/wasm32_without_js/wasm-runtime/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "wasm-runtime" +version = "0.1.0" +edition = "2024" + +[dependencies] +wasmer = "6.1.0" + +[workspace] \ No newline at end of file diff --git a/example_projects/wasm32_without_js/wasm-runtime/README.md b/example_projects/wasm32_without_js/wasm-runtime/README.md new file mode 100644 index 00000000000..2fa2f9e119a --- /dev/null +++ b/example_projects/wasm32_without_js/wasm-runtime/README.md @@ -0,0 +1,19 @@ +# Simple WASM Runtime + +WebAssembly runtime POC with wasmer with HashMap-based KV store. +First make sure to install wat2wasm and rust. + +```bash +# following command installs rust +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +cargo run --release <wasm binary> +``` + +## WASM binary requirements + +Entry point is `eval(code_ptr: i32, code_len: i32) -> i32`, following are exported functions, on error return -1: + +- `kv_put(key_ptr: i32, key_len: i32, val_ptr: i32, val_len: i32) -> i32` +- `kv_get(key_ptr: i32, key_len: i32, val_ptr: i32, val_len: i32) -> i32` +- `print(msg_ptr: i32, msg_len: i32) -> i32` diff --git a/example_projects/wasm32_without_js/wasm-runtime/src/main.rs b/example_projects/wasm32_without_js/wasm-runtime/src/main.rs new file mode 100644 index 00000000000..8ca7d581ce4 --- /dev/null +++ b/example_projects/wasm32_without_js/wasm-runtime/src/main.rs @@ -0,0 +1,133 @@ +use std::collections::HashMap; +use wasmer::{ + Function, FunctionEnv, FunctionEnvMut, Instance, Memory, Module, Store, Value, imports, +}; + +struct Ctx { + kv: HashMap<Vec<u8>, Vec<u8>>, + mem: Option<Memory>, +} + +/// kp and kl are the key pointer and length in wasm memory, vp and vl are for the return value +/// if read value is bigger than vl then it will be truncated to vl, returns read bytes +fn kv_get(mut ctx: FunctionEnvMut<Ctx>, kp: i32, kl: i32, vp: i32, vl: i32) -> i32 { + let (c, s) = ctx.data_and_store_mut(); + let mut key = vec![0u8; kl as usize]; + if c.mem + .as_ref() + .unwrap() + .view(&s) + .read(kp as u64, &mut key) + .is_err() + { + return -1; + } + match c.kv.get(&key) { + Some(val) => { + let len = val.len().min(vl as usize); + if c.mem + .as_ref() + .unwrap() + .view(&s) + .write(vp as u64, &val[..len]) + .is_err() + { + return -1; + } + len as i32 + } + None => 0, + } +} + +/// kp and kl are the key pointer and length in wasm memory, vp and vl are for the value +fn kv_put(mut ctx: FunctionEnvMut<Ctx>, kp: i32, kl: i32, vp: i32, vl: i32) -> i32 { + let (c, s) = ctx.data_and_store_mut(); + let mut key = vec![0u8; kl as usize]; + let mut val = vec![0u8; vl as usize]; + let m = c.mem.as_ref().unwrap().view(&s); + if m.read(kp as u64, &mut key).is_err() || m.read(vp as u64, &mut val).is_err() { + return -1; + } + c.kv.insert(key, val); + 0 +} + +// // p and l are the buffer pointer and length in wasm memory. +// fn get_code(mut ctx:FunctionEnvMut<Ctx>, p: i32, l: i32) -> i32 { +// let file_name = std::env::args().nth(2).expect("file_name is not given"); +// let code : String = std::fs::read_to_string(file_name).expect("file read failed"); +// if code.len() > l as usize { +// eprintln!("code is too long"); +// return -1; +// } + +// let (c, s) = ctx.data_and_store_mut(); +// let m = c.mem.as_ref().unwrap().view(&s); +// if m.write(p as u64, code.as_bytes()).is_err() { +// return -2; +// } + +// 0 +// } + +// p and l are the message pointer and length in wasm memory. +fn print(mut ctx: FunctionEnvMut<Ctx>, p: i32, l: i32) -> i32 { + let (c, s) = ctx.data_and_store_mut(); + let mut msg = vec![0u8; l as usize]; + let m = c.mem.as_ref().unwrap().view(&s); + if m.read(p as u64, &mut msg).is_err() { + return -1; + } + let s = std::str::from_utf8(&msg).expect("print got non-utf8 str"); + println!("{s}"); + 0 +} + +fn main() { + let mut store = Store::default(); + let module = Module::new( + &store, + &std::fs::read(&std::env::args().nth(1).unwrap()).unwrap(), + ) + .unwrap(); + + // Prepare initial KV store with Python code + let mut initial_kv = HashMap::new(); + initial_kv.insert( + b"code".to_vec(), + b"a=10;b='str';f'{a}{b}'".to_vec(), // Python code to execute + ); + + let env = FunctionEnv::new( + &mut store, + Ctx { + kv: initial_kv, + mem: None, + }, + ); + let imports = imports! { + "env" => { + "kv_get" => Function::new_typed_with_env(&mut store, &env, kv_get), + "kv_put" => Function::new_typed_with_env(&mut store, &env, kv_put), + // "get_code" => Function::new_typed_with_env(&mut store, &env, get_code), + "print" => Function::new_typed_with_env(&mut store, &env, print), + } + }; + let inst = Instance::new(&mut store, &module, &imports).unwrap(); + env.as_mut(&mut store).mem = inst.exports.get_memory("memory").ok().cloned(); + let res = inst + .exports + .get_function("eval") + .unwrap() + // TODO: actually pass source code + .call(&mut store, &[wasmer::Value::I32(0), wasmer::Value::I32(0)]) + .unwrap(); + println!( + "Result: {}", + match res[0] { + Value::I32(v) => v, + _ => -1, + } + ); +} diff --git a/examples/atexit_example.py b/examples/atexit_example.py index 0324d5b50eb..c9c61ca567d 100644 --- a/examples/atexit_example.py +++ b/examples/atexit_example.py @@ -1,7 +1,9 @@ import atexit import sys + def myexit(): sys.exit(2) + atexit.register(myexit) diff --git a/examples/call_between_rust_and_python.py b/examples/call_between_rust_and_python.py index 60335d81e98..4a52e85eac2 100644 --- a/examples/call_between_rust_and_python.py +++ b/examples/call_between_rust_and_python.py @@ -1,14 +1,17 @@ from rust_py_module import RustStruct, rust_function + class PythonPerson: def __init__(self, name): self.name = name + def python_callback(): python_person = PythonPerson("Peter Python") rust_object = rust_function(42, "This is a python string", python_person) print("Printing member 'numbers' from rust struct: ", rust_object.numbers) rust_object.print_in_rust_from_python() + def take_string(string): print("Calling python function from rust with string: " + string) diff --git a/examples/call_between_rust_and_python.rs b/examples/call_between_rust_and_python.rs index 98fb434fa0f..3a7c0ce610e 100644 --- a/examples/call_between_rust_and_python.rs +++ b/examples/call_between_rust_and_python.rs @@ -1,23 +1,18 @@ +use rustpython::InterpreterBuilderExt; use rustpython::vm::{ - pyclass, pymodule, PyObject, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, + PyObject, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, pyclass, pymodule, }; pub fn main() { - let interp = rustpython::InterpreterConfig::new() - .init_stdlib() - .init_hook(Box::new(|vm| { - vm.add_native_module( - "rust_py_module".to_owned(), - Box::new(rust_py_module::make_module), - ); - })) - .interpreter(); + let builder = rustpython::Interpreter::builder(Default::default()); + let def = rust_py_module::module_def(&builder.ctx); + let interp = builder.init_stdlib().add_native_module(def).build(); interp.enter(|vm| { vm.insert_sys_path(vm.new_pyobj("examples")) .expect("add path"); - let module = vm.import("call_between_rust_and_python", None, 0).unwrap(); + let module = vm.import("call_between_rust_and_python", 0).unwrap(); let init_fn = module.get_attr("python_callback", vm).unwrap(); init_fn.call((), vm).unwrap(); @@ -31,7 +26,7 @@ pub fn main() { #[pymodule] mod rust_py_module { use super::*; - use rustpython::vm::{builtins::PyList, convert::ToPyObject, PyObjectRef}; + use rustpython::vm::{PyObjectRef, convert::ToPyObject}; #[pyfunction] fn rust_function( @@ -58,7 +53,7 @@ python_person.name: {}", impl ToPyObject for NumVec { fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { let list = self.0.into_iter().map(|e| vm.new_pyobj(e)).collect(); - PyList::new_ref(list, vm.as_ref()).to_pyobject(vm) + vm.ctx.new_list(list).to_pyobject(vm) } } @@ -89,7 +84,7 @@ python_person.name: {}", impl<'a> TryFromBorrowedObject<'a> for PythonPerson { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { let name = obj.get_attr("name", vm)?.try_into_value::<String>(vm)?; - Ok(PythonPerson { name }) + Ok(Self { name }) } } } diff --git a/examples/dis.rs b/examples/dis.rs index 0b1e7c0d3d1..0b6190dde3c 100644 --- a/examples/dis.rs +++ b/examples/dis.rs @@ -1,81 +1,75 @@ -/// This an example usage of the rustpython_compiler crate. -/// This program reads, parses, and compiles a file you provide -/// to RustPython bytecode, and then displays the output in the -/// `dis.dis` format. -/// -/// example usage: -/// $ cargo run --release --example dis demo*.py +//! This an example usage of the rustpython_compiler crate. +//! This program reads, parses, and compiles a file you provide +//! to RustPython bytecode, and then displays the output in the +//! `dis.dis` format. +//! +//! example usage: +//! $ cargo run --release --example dis demo*.py -#[macro_use] -extern crate clap; -extern crate env_logger; #[macro_use] extern crate log; -use clap::{App, Arg}; +use core::error::Error; +use lexopt::ValueExt; use rustpython_compiler as compiler; -use std::error::Error; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; -fn main() { +fn main() -> Result<(), lexopt::Error> { env_logger::init(); - let app = App::new("dis") - .version(crate_version!()) - .author(crate_authors!()) - .about("Compiles and disassembles python script files for viewing their bytecode.") - .arg( - Arg::with_name("scripts") - .help("Scripts to scan") - .multiple(true) - .required(true), - ) - .arg( - Arg::with_name("mode") - .help("The mode to compile the scripts in") - .long("mode") - .short("m") - .default_value("exec") - .possible_values(&["exec", "single", "eval"]) - .takes_value(true), - ) - .arg( - Arg::with_name("no_expand") - .help( - "Don't expand CodeObject LoadConst instructions to show \ - the instructions inside", - ) - .long("no-expand") - .short("x"), - ) - .arg( - Arg::with_name("optimize") - .help("The amount of optimization to apply to the compiled bytecode") - .short("O") - .multiple(true), - ); - let matches = app.get_matches(); - let mode = matches.value_of_lossy("mode").unwrap().parse().unwrap(); - let expand_code_objects = !matches.is_present("no_expand"); - let optimize = matches.occurrences_of("optimize") as u8; - let scripts = matches.values_of_os("scripts").unwrap(); + let mut scripts = vec![]; + let mut mode = compiler::Mode::Exec; + let mut expand_code_objects = true; + let mut optimize = 0; + + let mut parser = lexopt::Parser::from_env(); + while let Some(arg) = parser.next()? { + use lexopt::Arg::*; + match arg { + Long("help") | Short('h') => { + let bin_name = parser.bin_name().unwrap_or("dis"); + println!( + "usage: {bin_name} <scripts...> [-m,--mode=exec|single|eval] [-x,--no-expand] [-O]" + ); + println!( + "Compiles and disassembles python script files for viewing their bytecode." + ); + return Ok(()); + } + Value(x) => scripts.push(PathBuf::from(x)), + Long("mode") | Short('m') => { + mode = parser + .value()? + .parse_with(|s| s.parse::<compiler::Mode>().map_err(|e| e.to_string()))? + } + Long("no-expand") | Short('x') => expand_code_objects = false, + Short('O') => optimize += 1, + _ => return Err(arg.unexpected()), + } + } + + if scripts.is_empty() { + return Err("expected at least one argument".into()); + } let opts = compiler::CompileOpts { optimize, - ..Default::default() + debug_ranges: true, }; - for script in scripts.map(Path::new) { + for script in &scripts { if script.exists() && script.is_file() { - let res = display_script(script, mode, opts.clone(), expand_code_objects); + let res = display_script(script, mode, opts, expand_code_objects); if let Err(e) = res { - error!("Error while compiling {:?}: {}", script, e); + error!("Error while compiling {script:?}: {e}"); } } else { eprintln!("{script:?} is not a file."); } } + + Ok(()) } fn display_script( @@ -85,7 +79,7 @@ fn display_script( expand_code_objects: bool, ) -> Result<(), Box<dyn Error>> { let source = fs::read_to_string(path)?; - let code = compiler::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?; + let code = compiler::compile(&source, mode, &path.to_string_lossy(), opts)?; println!("{}:", path.display()); if expand_code_objects { println!("{}", code.display_expand_code_objects()); diff --git a/examples/freeze/freeze.py b/examples/freeze/freeze.py index c600ebf9592..9cd65519664 100644 --- a/examples/freeze/freeze.py +++ b/examples/freeze/freeze.py @@ -1,4 +1,3 @@ import time print("Hello world!!!", time.time()) - diff --git a/examples/freeze/main.rs b/examples/freeze/main.rs index 48991129073..ab26a2bc808 100644 --- a/examples/freeze/main.rs +++ b/examples/freeze/main.rs @@ -7,9 +7,8 @@ fn main() -> vm::PyResult<()> { fn run(vm: &vm::VirtualMachine) -> vm::PyResult<()> { let scope = vm.new_scope_with_builtins(); - // the file parameter is relative to the directory where the crate's Cargo.toml is located, see $CARGO_MANIFEST_DIR: - // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates - let module = vm::py_compile!(file = "examples/freeze/freeze.py"); + // the file parameter is relative to the current file. + let module = vm::py_compile!(file = "freeze.py"); let res = vm.run_code_obj(vm.ctx.new_code(module), scope); diff --git a/examples/generator.rs b/examples/generator.rs index 010ccd37976..55841c767a1 100644 --- a/examples/generator.rs +++ b/examples/generator.rs @@ -1,9 +1,9 @@ use rustpython_vm as vm; use std::process::ExitCode; use vm::{ + Interpreter, PyResult, builtins::PyIntRef, protocol::{PyIter, PyIterReturn}, - Interpreter, PyResult, }; fn py_main(interp: &Interpreter) -> vm::PyResult<()> { @@ -42,9 +42,9 @@ gen() } fn main() -> ExitCode { - let interp = vm::Interpreter::with_init(Default::default(), |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - }); + let builder = vm::Interpreter::builder(Default::default()); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); let result = py_main(&interp); - ExitCode::from(interp.run(|_vm| result)) + vm::common::os::exit_code(interp.run(|_vm| result)) } diff --git a/examples/mini_repl.rs b/examples/mini_repl.rs index fe3b13dcfda..d7baa4692c2 100644 --- a/examples/mini_repl.rs +++ b/examples/mini_repl.rs @@ -1,8 +1,9 @@ -///! This example show cases a very simple REPL. -///! While a much better REPL can be found in ../src/shell, -///! This much smaller REPL is still a useful example because it showcases inserting -///! values and functions into the Python runtime's scope, and showcases use -///! of the compilation mode "Single". +//! This example show cases a very simple REPL. +//! While a much better REPL can be found in ../src/shell, +//! This much smaller REPL is still a useful example because it showcases inserting +//! values and functions into the Python runtime's scope, and showcases use +//! of the compilation mode "Single". + use rustpython_vm as vm; // these are needed for special memory shenanigans to let us share a variable with Python and Rust use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/examples/package_embed.rs b/examples/package_embed.rs index b35d0639284..bb2f29e3f5f 100644 --- a/examples/package_embed.rs +++ b/examples/package_embed.rs @@ -1,13 +1,13 @@ use rustpython_vm as vm; use std::process::ExitCode; -use vm::{builtins::PyStrRef, Interpreter}; +use vm::{Interpreter, builtins::PyStrRef}; fn py_main(interp: &Interpreter) -> vm::PyResult<PyStrRef> { interp.enter(|vm| { // Add local library path vm.insert_sys_path(vm.new_pyobj("examples")) .expect("add examples to sys.path failed"); - let module = vm.import("package_embed", None, 0)?; + let module = vm.import("package_embed", 0)?; let name_func = module.get_attr("context", vm)?; let result = name_func.call((), vm)?; let result: PyStrRef = result.get_attr("name", vm)?.try_into_value(vm)?; @@ -19,12 +19,12 @@ fn main() -> ExitCode { // Add standard library path let mut settings = vm::Settings::default(); settings.path_list.push("Lib".to_owned()); - let interp = vm::Interpreter::with_init(settings, |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - }); + let builder = vm::Interpreter::builder(settings); + let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx); + let interp = builder.add_native_modules(&defs).build(); let result = py_main(&interp); let result = result.map(|result| { println!("name: {result}"); }); - ExitCode::from(interp.run(|_vm| result)) + vm::common::os::exit_code(interp.run(|_vm| result)) } diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs index 7774b8afbf1..440bcdb9b5f 100644 --- a/examples/parse_folder.rs +++ b/examples/parse_folder.rs @@ -4,38 +4,29 @@ /// /// example usage: /// $ RUST_LOG=info cargo run --release parse_folder /usr/lib/python3.7 - -#[macro_use] -extern crate clap; extern crate env_logger; #[macro_use] extern crate log; -use clap::{App, Arg}; -use rustpython_parser::{self as parser, ast}; +use ruff_python_parser::parse_module; +use rustpython_compiler::ast; use std::{ - path::Path, + path::{Path, PathBuf}, time::{Duration, Instant}, }; fn main() { env_logger::init(); - let app = App::new("parse_folders") - .version(crate_version!()) - .author(crate_authors!()) - .about("Walks over all .py files in a folder, and parses them.") - .arg( - Arg::with_name("folder") - .help("Folder to scan") - .required(true), - ); - let matches = app.get_matches(); - let folder = Path::new(matches.value_of("folder").unwrap()); + let folder: PathBuf = std::env::args_os() + .nth(1) + .expect("please pass a path argument") + .into(); + if folder.exists() && folder.is_dir() { println!("Parsing folder of python code: {folder:?}"); let t1 = Instant::now(); - let parsed_files = parse_folder(folder).unwrap(); + let parsed_files = parse_folder(&folder).unwrap(); let t2 = Instant::now(); let results = ScanResult { t1, @@ -50,9 +41,9 @@ fn main() { fn parse_folder(path: &Path) -> std::io::Result<Vec<ParsedFile>> { let mut res = vec![]; - info!("Parsing folder of python code: {:?}", path); + info!("Parsing folder of python code: {path:?}"); for entry in path.read_dir()? { - debug!("Entry: {:?}", entry); + debug!("Entry: {entry:?}"); let entry = entry?; let metadata = entry.metadata()?; @@ -65,7 +56,7 @@ fn parse_folder(path: &Path) -> std::io::Result<Vec<ParsedFile>> { let parsed_file = parse_python_file(&path); match &parsed_file.result { Ok(_) => {} - Err(y) => error!("Erreur in file {:?} {:?}", path, y), + Err(y) => error!("Error in file {path:?} {y:?}"), } res.push(parsed_file); @@ -75,24 +66,18 @@ fn parse_folder(path: &Path) -> std::io::Result<Vec<ParsedFile>> { } fn parse_python_file(filename: &Path) -> ParsedFile { - info!("Parsing file {:?}", filename); + info!("Parsing file {filename:?}"); match std::fs::read_to_string(filename) { Err(e) => ParsedFile { - // filename: Box::new(filename.to_path_buf()), - // code: "".to_owned(), num_lines: 0, result: Err(e.to_string()), }, Ok(source) => { let num_lines = source.lines().count(); - let result = parser::parse_program(&source, &filename.to_string_lossy()) + let result = parse_module(&source) + .map(|x| x.into_suite()) .map_err(|e| e.to_string()); - ParsedFile { - // filename: Box::new(filename.to_path_buf()), - // code: source.to_string(), - num_lines, - result, - } + ParsedFile { num_lines, result } } } } @@ -142,8 +127,6 @@ struct ScanResult { } struct ParsedFile { - // filename: Box<PathBuf>, - // code: String, num_lines: usize, result: ParseResult, } diff --git a/extra_tests/benchmarks/perf_fib.py b/extra_tests/benchmarks/perf_fib.py index 89ede851793..6b999f74436 100644 --- a/extra_tests/benchmarks/perf_fib.py +++ b/extra_tests/benchmarks/perf_fib.py @@ -2,14 +2,15 @@ def fib(n): a = 1 b = 1 for _ in range(n - 1): - temp = b - b = a + b - a = temp + temp = b + b = a + b + a = temp return b + print(fib(1)) print(fib(2)) print(fib(3)) print(fib(4)) -print(fib(5)) \ No newline at end of file +print(fib(5)) diff --git a/extra_tests/custom_text_test_runner.py b/extra_tests/custom_text_test_runner.py index cc3a8bbffff..afec493a66c 100644 --- a/extra_tests/custom_text_test_runner.py +++ b/extra_tests/custom_text_test_runner.py @@ -25,21 +25,24 @@ # SOFTWARE. -import unittest -import os, sys, traceback import inspect import json -import time -import re import operator -from unittest.runner import result -from unittest.runner import registerResult +import os +import re +import sys +import time +import traceback +import unittest from functools import reduce +from unittest.runner import registerResult, result + class TablePrinter(object): # Modified from https://github.com/agramian/table-printer, same license as above "Print a list of dicts as a table" - def __init__(self, fmt, sep='', ul=None, tl=None, bl=None): + + def __init__(self, fmt, sep="", ul=None, tl=None, bl=None): """ @param fmt: list of tuple(heading, key, width) heading: str, column label @@ -50,20 +53,44 @@ def __init__(self, fmt, sep='', ul=None, tl=None, bl=None): @param tl: string, character to draw as top line over table, or None @param bl: string, character to draw as bottom line under table, or None """ - super(TablePrinter,self).__init__() - fmt = [x + ('left',) if len(x) < 4 else x for x in fmt] - self.fmt = str(sep).join('{lb}{0}:{align}{1}{rb}'.format(key, width, lb='{', rb='}', align='<' if alignment == 'left' else '>') for heading,key,width,alignment in fmt) - self.head = {key:heading for heading,key,width,alignment in fmt} - self.ul = {key:str(ul)*width for heading,key,width,alignment in fmt} if ul else None - self.width = {key:width for heading,key,width,alignment in fmt} - self.tl = {key:str(tl)*width for heading,key,width,alignment in fmt} if tl else None - self.bl = {key:str(bl)*width for heading,key,width,alignment in fmt} if bl else None + super(TablePrinter, self).__init__() + fmt = [x + ("left",) if len(x) < 4 else x for x in fmt] + self.fmt = str(sep).join( + "{lb}{0}:{align}{1}{rb}".format( + key, width, lb="{", rb="}", align="<" if alignment == "left" else ">" + ) + for heading, key, width, alignment in fmt + ) + self.head = {key: heading for heading, key, width, alignment in fmt} + self.ul = ( + {key: str(ul) * width for heading, key, width, alignment in fmt} + if ul + else None + ) + self.width = {key: width for heading, key, width, alignment in fmt} + self.tl = ( + {key: str(tl) * width for heading, key, width, alignment in fmt} + if tl + else None + ) + self.bl = ( + {key: str(bl) * width for heading, key, width, alignment in fmt} + if bl + else None + ) def row(self, data, separation_character=False): if separation_character: - return self.fmt.format(**{ k:str(data.get(k,''))[:w] for k,w in self.width.items() }) + return self.fmt.format( + **{k: str(data.get(k, ""))[:w] for k, w in self.width.items()} + ) else: - data = { k:str(data.get(k,'')) if len(str(data.get(k,''))) <= w else '%s...' %str(data.get(k,''))[:(w-3)] for k,w in self.width.items() } + data = { + k: str(data.get(k, "")) + if len(str(data.get(k, ""))) <= w + else "%s..." % str(data.get(k, ""))[: (w - 3)] + for k, w in self.width.items() + } return self.fmt.format(**data) def __call__(self, data_list, totals=None): @@ -80,89 +107,111 @@ def __call__(self, data_list, totals=None): res.insert(len(res), _r(totals)) if self.bl: res.insert(len(res), _r(self.bl, True)) - return '\n'.join(res) + return "\n".join(res) def get_function_args(func_ref): try: - return [p for p in inspect.getargspec(func_ref).args if p != 'self'] + return [p for p in inspect.getfullargspec(func_ref).args if p != "self"] except: return None + def store_class_fields(class_ref, args_passed): - """ Store the passed in class fields in self - """ + """Store the passed in class fields in self""" params = get_function_args(class_ref.__init__) - for p in params: setattr(class_ref, p, args_passed[p]) + for p in params: + setattr(class_ref, p, args_passed[p]) + def sum_dict_key(d, key, cast_type=None): - """ Sum together all values matching a key given a passed dict - """ - return reduce( (lambda x, y: x + y), [eval("%s(x['%s'])" %(cast_type, key)) if cast_type else x[key] for x in d] ) + """Sum together all values matching a key given a passed dict""" + return reduce( + (lambda x, y: x + y), + [eval("%s(x['%s'])" % (cast_type, key)) if cast_type else x[key] for x in d], + ) + def case_name(name): - """ Test case name decorator to override function name. - """ + """Test case name decorator to override function name.""" + def decorator(function): - function.__dict__['test_case_name'] = name + function.__dict__["test_case_name"] = name return function + return decorator + def skip_device(name): - """ Decorator to mark a test to only run on certain devices - Takes single device name or list of names as argument + """Decorator to mark a test to only run on certain devices + Takes single device name or list of names as argument """ + def decorator(function): name_list = name if type(name) == list else [name] - function.__dict__['skip_device'] = name_list + function.__dict__["skip_device"] = name_list return function + return decorator + def _set_test_type(function, test_type): - """ Test type setter - """ - if 'test_type' in function.__dict__: - function.__dict__['test_type'].append(test_type) + """Test type setter""" + if "test_type" in function.__dict__: + function.__dict__["test_type"].append(test_type) else: - function.__dict__['test_type'] = [test_type] + function.__dict__["test_type"] = [test_type] return function + def smoke(function): - """ Test decorator to mark test as smoke type - """ - return _set_test_type(function, 'smoke') + """Test decorator to mark test as smoke type""" + return _set_test_type(function, "smoke") + def guide_discovery(function): - """ Test decorator to mark test as guide_discovery type - """ - return _set_test_type(function, 'guide_discovery') + """Test decorator to mark test as guide_discovery type""" + return _set_test_type(function, "guide_discovery") + def focus(function): - """ Test decorator to mark test as focus type to all rspec style debugging of cases - """ - return _set_test_type(function, 'focus') + """Test decorator to mark test as focus type to all rspec style debugging of cases""" + return _set_test_type(function, "focus") + class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" - def __init__(self,stream): + + def __init__(self, stream): self.stream = stream def __getattr__(self, attr): - if attr in ('stream', '__getstate__'): + if attr in ("stream", "__getstate__"): raise AttributeError(attr) - return getattr(self.stream,attr) + return getattr(self.stream, attr) def writeln(self, arg=None): if arg: self.write(arg) - self.write('\n') # text-mode streams translate to \r\n if needed + self.write("\n") # text-mode streams translate to \r\n if needed + class CustomTextTestResult(result.TestResult): _num_formatting_chars = 150 _execution_time_significant_digits = 4 _pass_percentage_significant_digits = 2 - def __init__(self, stream, descriptions, verbosity, results_file_path, result_screenshots_dir, show_previous_results, config, test_types): + def __init__( + self, + stream, + descriptions, + verbosity, + results_file_path, + result_screenshots_dir, + show_previous_results, + config, + test_types, + ): super(CustomTextTestResult, self).__init__(stream, descriptions, verbosity) store_class_fields(self, locals()) self.show_overall_results = verbosity > 0 @@ -178,12 +227,12 @@ def __init__(self, stream, descriptions, verbosity, results_file_path, result_sc self.separator3 = "_" * CustomTextTestResult._num_formatting_chars self.separator4 = "*" * CustomTextTestResult._num_formatting_chars self.separator_failure = "!" * CustomTextTestResult._num_formatting_chars - self.separator_pre_result = '.' * CustomTextTestResult._num_formatting_chars + self.separator_pre_result = "." * CustomTextTestResult._num_formatting_chars def getDescription(self, test): doc_first_line = test.shortDescription() if self.descriptions and doc_first_line: - return '\n'.join((str(test), doc_first_line)) + return "\n".join((str(test), doc_first_line)) else: return str(test) @@ -195,109 +244,170 @@ def startTestRun(self): self.results = None self.previous_suite_runs = [] if os.path.isfile(self.results_file_path): - with open(self.results_file_path, 'rb') as f: + with open(self.results_file_path, "rb") as f: try: self.results = json.load(f) # recreated results dict with int keys - self.results['suites'] = {int(k):v for (k,v) in list(self.results['suites'].items())} - self.suite_map = {v['name']:int(k) for (k,v) in list(self.results['suites'].items())} - self.previous_suite_runs = list(self.results['suites'].keys()) + self.results["suites"] = { + int(k): v for (k, v) in list(self.results["suites"].items()) + } + self.suite_map = { + v["name"]: int(k) + for (k, v) in list(self.results["suites"].items()) + } + self.previous_suite_runs = list(self.results["suites"].keys()) except: pass if not self.results: - self.results = {'suites': {}, - 'name': '', - 'num_passed': 0, - 'num_failed': 0, - 'num_skipped': 0, - 'num_expected_failures': 0, - 'execution_time': None} - self.suite_number = int(sorted(self.results['suites'].keys())[-1]) + 1 if len(self.results['suites']) else 0 + self.results = { + "suites": {}, + "name": "", + "num_passed": 0, + "num_failed": 0, + "num_skipped": 0, + "num_expected_failures": 0, + "execution_time": None, + } + self.suite_number = ( + int(sorted(self.results["suites"].keys())[-1]) + 1 + if len(self.results["suites"]) + else 0 + ) self.case_number = 0 self.suite_map = {} def stopTestRun(self): - # if no tests or some failure occured execution time may not have been set + # if no tests or some failure occurred execution time may not have been set try: - self.results['suites'][self.suite_map[self.suite]]['execution_time'] = format(self.suite_execution_time, '.%sf' %CustomTextTestResult._execution_time_significant_digits) + self.results["suites"][self.suite_map[self.suite]]["execution_time"] = ( + format( + self.suite_execution_time, + ".%sf" % CustomTextTestResult._execution_time_significant_digits, + ) + ) except: pass - self.results['execution_time'] = format(self.total_execution_time, '.%sf' %CustomTextTestResult._execution_time_significant_digits) + self.results["execution_time"] = format( + self.total_execution_time, + ".%sf" % CustomTextTestResult._execution_time_significant_digits, + ) self.stream.writeln(self.separator3) - with open(self.results_file_path, 'w') as f: + with open(self.results_file_path, "w") as f: json.dump(self.results, f) def startTest(self, test): - suite_base_category = test.__class__.base_test_category if hasattr(test.__class__, 'base_test_category') else '' - self.next_suite = os.path.join(suite_base_category, test.__class__.name if hasattr(test.__class__, 'name') else test.__class__.__name__) + suite_base_category = ( + test.__class__.base_test_category + if hasattr(test.__class__, "base_test_category") + else "" + ) + self.next_suite = os.path.join( + suite_base_category, + test.__class__.name + if hasattr(test.__class__, "name") + else test.__class__.__name__, + ) self.case = test._testMethodName super(CustomTextTestResult, self).startTest(test) if not self.suite or self.suite != self.next_suite: if self.suite: - self.results['suites'][self.suite_map[self.suite]]['execution_time'] = format(self.suite_execution_time, '.%sf' %CustomTextTestResult._execution_time_significant_digits) + self.results["suites"][self.suite_map[self.suite]]["execution_time"] = ( + format( + self.suite_execution_time, + ".%sf" + % CustomTextTestResult._execution_time_significant_digits, + ) + ) self.suite_execution_time = 0 self.suite = self.next_suite if self.show_test_info: self.stream.writeln(self.separator1) - self.stream.writeln("TEST SUITE: %s" %self.suite) - self.stream.writeln("Description: %s" %self.getSuiteDescription(test)) + self.stream.writeln("TEST SUITE: %s" % self.suite) + self.stream.writeln("Description: %s" % self.getSuiteDescription(test)) try: - name_override = getattr(test, test._testMethodName).__func__.__dict__['test_case_name'] + name_override = getattr(test, test._testMethodName).__func__.__dict__[ + "test_case_name" + ] except: name_override = None self.case = name_override if name_override else self.case if self.show_test_info: # self.stream.writeln(self.separator2) - self.stream.write("CASE: %s" %self.case) - if desc := test.shortDescription(): self.stream.write(" (Description: %s)" % desc) + self.stream.write("CASE: %s" % self.case) + if desc := test.shortDescription(): + self.stream.write(" (Description: %s)" % desc) self.stream.write("... ") # self.stream.writeln(self.separator2) self.stream.flush() self.current_case_number = self.case_number if self.suite not in self.suite_map: self.suite_map[self.suite] = self.suite_number - self.results['suites'][self.suite_number] = { - 'name': self.suite, - 'class': test.__class__.__name__, - 'module': re.compile('.* \((.*)\)').match(str(test)).group(1), - 'description': self.getSuiteDescription(test), - 'cases': {}, - 'used_case_names': {}, - 'num_passed': 0, - 'num_failed': 0, - 'num_skipped': 0, - 'num_expected_failures': 0, - 'execution_time': None} + self.results["suites"][self.suite_number] = { + "name": self.suite, + "class": test.__class__.__name__, + "module": re.compile(".* \((.*)\)").match(str(test)).group(1), + "description": self.getSuiteDescription(test), + "cases": {}, + "used_case_names": {}, + "num_passed": 0, + "num_failed": 0, + "num_skipped": 0, + "num_expected_failures": 0, + "execution_time": None, + } self.suite_number += 1 self.num_cases = 0 self.num_passed = 0 self.num_failed = 0 self.num_skipped = 0 self.num_expected_failures = 0 - self.results['suites'][self.suite_map[self.suite]]['cases'][self.case_number] = { - 'name': self.case, - 'method': test._testMethodName, - 'result': None, - 'description': test.shortDescription(), - 'note': None, - 'errors': None, - 'failures': None, - 'screenshots': [], - 'new_version': 'No', - 'execution_time': None} + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.case_number + ] = { + "name": self.case, + "method": test._testMethodName, + "result": None, + "description": test.shortDescription(), + "note": None, + "errors": None, + "failures": None, + "screenshots": [], + "new_version": "No", + "execution_time": None, + } self.start_time = time.time() if self.test_types: - if ('test_type' in getattr(test, test._testMethodName).__func__.__dict__ - and set([s.lower() for s in self.test_types]) == set([s.lower() for s in getattr(test, test._testMethodName).__func__.__dict__['test_type']])): + if "test_type" in getattr( + test, test._testMethodName + ).__func__.__dict__ and set([s.lower() for s in self.test_types]) == set( + [ + s.lower() + for s in getattr(test, test._testMethodName).__func__.__dict__[ + "test_type" + ] + ] + ): pass else: - getattr(test, test._testMethodName).__func__.__dict__['__unittest_skip_why__'] = 'Test run specified to only run tests of type "%s"' %','.join(self.test_types) - getattr(test, test._testMethodName).__func__.__dict__['__unittest_skip__'] = True - if 'skip_device' in getattr(test, test._testMethodName).__func__.__dict__: - for device in getattr(test, test._testMethodName).__func__.__dict__['skip_device']: - if self.config and device.lower() in self.config['device_name'].lower(): - getattr(test, test._testMethodName).__func__.__dict__['__unittest_skip_why__'] = 'Test is marked to be skipped on %s' %device - getattr(test, test._testMethodName).__func__.__dict__['__unittest_skip__'] = True + getattr(test, test._testMethodName).__func__.__dict__[ + "__unittest_skip_why__" + ] = 'Test run specified to only run tests of type "%s"' % ",".join( + self.test_types + ) + getattr(test, test._testMethodName).__func__.__dict__[ + "__unittest_skip__" + ] = True + if "skip_device" in getattr(test, test._testMethodName).__func__.__dict__: + for device in getattr(test, test._testMethodName).__func__.__dict__[ + "skip_device" + ]: + if self.config and device.lower() in self.config["device_name"].lower(): + getattr(test, test._testMethodName).__func__.__dict__[ + "__unittest_skip_why__" + ] = "Test is marked to be skipped on %s" % device + getattr(test, test._testMethodName).__func__.__dict__[ + "__unittest_skip__" + ] = True break def stopTest(self, test): @@ -307,19 +417,32 @@ def stopTest(self, test): self.total_execution_time += self.execution_time super(CustomTextTestResult, self).stopTest(test) self.num_cases += 1 - self.results['suites'][self.suite_map[self.suite]]['num_passed'] = self.num_passed - self.results['suites'][self.suite_map[self.suite]]['num_failed'] = self.num_failed - self.results['suites'][self.suite_map[self.suite]]['num_skipped'] = self.num_skipped - self.results['suites'][self.suite_map[self.suite]]['num_expected_failures'] = self.num_expected_failures - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['execution_time']= format(self.execution_time, '.%sf' %CustomTextTestResult._execution_time_significant_digits) - self.results['num_passed'] += self.num_passed - self.results['num_failed'] += self.num_failed - self.results['num_skipped'] += self.num_skipped - self.results['num_expected_failures'] += self.num_expected_failures + self.results["suites"][self.suite_map[self.suite]]["num_passed"] = ( + self.num_passed + ) + self.results["suites"][self.suite_map[self.suite]]["num_failed"] = ( + self.num_failed + ) + self.results["suites"][self.suite_map[self.suite]]["num_skipped"] = ( + self.num_skipped + ) + self.results["suites"][self.suite_map[self.suite]]["num_expected_failures"] = ( + self.num_expected_failures + ) + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["execution_time"] = format( + self.execution_time, + ".%sf" % CustomTextTestResult._execution_time_significant_digits, + ) + self.results["num_passed"] += self.num_passed + self.results["num_failed"] += self.num_failed + self.results["num_skipped"] += self.num_skipped + self.results["num_expected_failures"] += self.num_expected_failures self.case_number += 1 def print_error_string(self, err): - error_string = ''.join(traceback.format_exception(err[0], err[1], err[2])) + error_string = "".join(traceback.format_exception(err[0], err[1], err[2])) if self.show_errors: self.stream.writeln(self.separator_failure) self.stream.write(error_string) @@ -328,7 +451,9 @@ def print_error_string(self, err): def addScreenshots(self, test): for root, dirs, files in os.walk(self.result_screenshots_dir): for file in files: - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['screenshots'].append(os.path.join(root, file)) + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["screenshots"].append(os.path.join(root, file)) def addSuccess(self, test): super(CustomTextTestResult, self).addSuccess(test) @@ -336,7 +461,9 @@ def addSuccess(self, test): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("PASS") self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'passed' + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "passed" self.num_passed += 1 self.addScreenshots(test) @@ -347,8 +474,12 @@ def addError(self, test, err): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("ERROR") self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'error' - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['errors'] = error_string + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "error" + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["errors"] = error_string self.num_failed += 1 self.addScreenshots(test) @@ -359,8 +490,12 @@ def addFailure(self, test, err): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("FAIL") self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'failed' - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['failures'] = error_string + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "failed" + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["failures"] = error_string self.num_failed += 1 self.addScreenshots(test) @@ -370,8 +505,12 @@ def addSkip(self, test, reason): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("SKIPPED {0!r}".format(reason)) self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'skipped' - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['note'] = reason + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "skipped" + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["note"] = reason self.num_skipped += 1 def addExpectedFailure(self, test, err): @@ -380,7 +519,9 @@ def addExpectedFailure(self, test, err): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("EXPECTED FAILURE") self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'expected_failure' + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "expected_failure" self.num_expected_failures += 1 self.addScreenshots(test) @@ -396,103 +537,189 @@ def addUnexpectedSuccess(self, test): def printOverallSuiteResults(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('OVERALL SUITE RESULTS') + self.stream.writeln("OVERALL SUITE RESULTS") fmt = [ - ('SUITE', 'suite', 50, 'left'), - ('CASES', 'cases', 15, 'right'), - ('PASSED', 'passed', 15, 'right'), - ('FAILED', 'failed', 15, 'right'), - ('SKIPPED', 'skipped', 15, 'right'), - ('%', 'percentage', 20, 'right'), - ('TIME (s)', 'time', 20, 'right') + ("SUITE", "suite", 50, "left"), + ("CASES", "cases", 15, "right"), + ("PASSED", "passed", 15, "right"), + ("FAILED", "failed", 15, "right"), + ("SKIPPED", "skipped", 15, "right"), + ("%", "percentage", 20, "right"), + ("TIME (s)", "time", 20, "right"), ] data = [] - for x in r: data.append({'suite': r[x]['name'], - 'cases': r[x]['num_passed'] + r[x]['num_failed'], - 'passed': r[x]['num_passed'], - 'failed': r[x]['num_failed'], - 'skipped': r[x]['num_skipped'], - 'expected_failures': r[x]['num_expected_failures'], - 'percentage': float(r[x]['num_passed'])/(r[x]['num_passed'] + r[x]['num_failed']) * 100 if (r[x]['num_passed'] + r[x]['num_failed']) > 0 else 0, - 'time': r[x]['execution_time']}) - total_suites_passed = len([x for x in data if not x['failed']]) - total_suites_passed_percentage = format(float(total_suites_passed)/len(data) * 100, '.%sf' %CustomTextTestResult._pass_percentage_significant_digits) - totals = {'suite': 'TOTALS %s/%s (%s%%) suites passed' %(total_suites_passed, len(data), total_suites_passed_percentage), - 'cases': sum_dict_key(data, 'cases'), - 'passed': sum_dict_key(data, 'passed'), - 'failed': sum_dict_key(data, 'failed'), - 'skipped': sum_dict_key(data, 'skipped'), - 'percentage': sum_dict_key(data, 'percentage')/len(data), - 'time': sum_dict_key(data, 'time', 'float')} - for x in data: operator.setitem(x, 'percentage', format(x['percentage'], '.%sf' %CustomTextTestResult._pass_percentage_significant_digits)) - totals['percentage'] = format(totals['percentage'], '.%sf' %CustomTextTestResult._pass_percentage_significant_digits) - self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2, bl=self.separator3)(data, totals) ) + for x in r: + data.append( + { + "suite": r[x]["name"], + "cases": r[x]["num_passed"] + r[x]["num_failed"], + "passed": r[x]["num_passed"], + "failed": r[x]["num_failed"], + "skipped": r[x]["num_skipped"], + "expected_failures": r[x]["num_expected_failures"], + "percentage": float(r[x]["num_passed"]) + / (r[x]["num_passed"] + r[x]["num_failed"]) + * 100 + if (r[x]["num_passed"] + r[x]["num_failed"]) > 0 + else 0, + "time": r[x]["execution_time"], + } + ) + total_suites_passed = len([x for x in data if not x["failed"]]) + total_suites_passed_percentage = format( + float(total_suites_passed) / len(data) * 100, + ".%sf" % CustomTextTestResult._pass_percentage_significant_digits, + ) + totals = { + "suite": "TOTALS %s/%s (%s%%) suites passed" + % (total_suites_passed, len(data), total_suites_passed_percentage), + "cases": sum_dict_key(data, "cases"), + "passed": sum_dict_key(data, "passed"), + "failed": sum_dict_key(data, "failed"), + "skipped": sum_dict_key(data, "skipped"), + "percentage": sum_dict_key(data, "percentage") / len(data), + "time": sum_dict_key(data, "time", "float"), + } + for x in data: + operator.setitem( + x, + "percentage", + format( + x["percentage"], + ".%sf" % CustomTextTestResult._pass_percentage_significant_digits, + ), + ) + totals["percentage"] = format( + totals["percentage"], + ".%sf" % CustomTextTestResult._pass_percentage_significant_digits, + ) + self.stream.writeln( + TablePrinter( + fmt, tl=self.separator1, ul=self.separator2, bl=self.separator3 + )(data, totals) + ) self.stream.writeln() def printIndividualSuiteResults(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('INDIVIDUAL SUITE RESULTS') + self.stream.writeln("INDIVIDUAL SUITE RESULTS") fmt = [ - ('CASE', 'case', 50, 'left'), - ('DESCRIPTION', 'description', 50, 'right'), - ('RESULT', 'result', 25, 'right'), - ('TIME (s)', 'time', 25, 'right') + ("CASE", "case", 50, "left"), + ("DESCRIPTION", "description", 50, "right"), + ("RESULT", "result", 25, "right"), + ("TIME (s)", "time", 25, "right"), ] for suite in r: self.stream.writeln(self.separator1) - self.stream.write('{0: <50}'.format('SUITE: %s' %r[suite]['name'])) - self.stream.writeln('{0: <100}'.format('DESCRIPTION: %s' %(r[suite]['description'] if not r[suite]['description'] or len(r[suite]['description']) <= (100 - len('DESCRIPTION: ')) - else '%s...' %r[suite]['description'][:(97 - len('DESCRIPTION: '))]))) + self.stream.write("{0: <50}".format("SUITE: %s" % r[suite]["name"])) + self.stream.writeln( + "{0: <100}".format( + "DESCRIPTION: %s" + % ( + r[suite]["description"] + if not r[suite]["description"] + or len(r[suite]["description"]) <= (100 - len("DESCRIPTION: ")) + else "%s..." + % r[suite]["description"][: (97 - len("DESCRIPTION: "))] + ) + ) + ) data = [] - cases = r[suite]['cases'] - for x in cases: data.append({'case': cases[x]['name'], - 'description': cases[x]['description'], - 'result': cases[x]['result'].upper() if cases[x]['result'] else cases[x]['result'], - 'time': cases[x]['execution_time']}) - self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) ) + cases = r[suite]["cases"] + for x in cases: + data.append( + { + "case": cases[x]["name"], + "description": cases[x]["description"], + "result": cases[x]["result"].upper() + if cases[x]["result"] + else cases[x]["result"], + "time": cases[x]["execution_time"], + } + ) + self.stream.writeln( + TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) + ) self.stream.writeln(self.separator3) self.stream.writeln() def printErrorsOverview(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('FAILURES AND ERRORS OVERVIEW') + self.stream.writeln("FAILURES AND ERRORS OVERVIEW") fmt = [ - ('SUITE', 'suite', 50, 'left'), - ('CASE', 'case', 50, 'left'), - ('RESULT', 'result', 50, 'right') + ("SUITE", "suite", 50, "left"), + ("CASE", "case", 50, "left"), + ("RESULT", "result", 50, "right"), ] data = [] for suite in r: - cases = {k:v for (k,v) in list(r[suite]['cases'].items()) if v['failures'] or v['errors']} - for x in cases: data.append({'suite': '%s%s' %(r[suite]['name'], ' (%s)' %r[suite]['module'] if r[suite]['class'] != r[suite]['name'] else ''), - 'case': '%s%s' %(cases[x]['name'], ' (%s)' %cases[x]['method'] if cases[x]['name'] != cases[x]['method'] else ''), - 'result': cases[x]['result'].upper()}) - self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) ) + cases = { + k: v + for (k, v) in list(r[suite]["cases"].items()) + if v["failures"] or v["errors"] + } + for x in cases: + data.append( + { + "suite": "%s%s" + % ( + r[suite]["name"], + " (%s)" % r[suite]["module"] + if r[suite]["class"] != r[suite]["name"] + else "", + ), + "case": "%s%s" + % ( + cases[x]["name"], + " (%s)" % cases[x]["method"] + if cases[x]["name"] != cases[x]["method"] + else "", + ), + "result": cases[x]["result"].upper(), + } + ) + self.stream.writeln( + TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) + ) self.stream.writeln(self.separator3) self.stream.writeln() def printErrorsDetail(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('FAILURES AND ERRORS DETAIL') + self.stream.writeln("FAILURES AND ERRORS DETAIL") for suite in r: - failures_and_errors = [k for (k,v) in list(r[suite]['cases'].items()) if v['failures'] or v['errors']] - #print failures_and_errors - suite_str = '%s%s' %(r[suite]['name'], ' (%s)' %r[suite]['module'] if r[suite]['class'] != r[suite]['name'] else '') + failures_and_errors = [ + k + for (k, v) in list(r[suite]["cases"].items()) + if v["failures"] or v["errors"] + ] + # print failures_and_errors + suite_str = "%s%s" % ( + r[suite]["name"], + " (%s)" % r[suite]["module"] + if r[suite]["class"] != r[suite]["name"] + else "", + ) for case in failures_and_errors: - case_ref = r[suite]['cases'][case] - case_str = '%s%s' %(case_ref['name'], ' (%s)' %case_ref['method'] if case_ref['name'] != case_ref['method'] else '') - errors = case_ref['errors'] - failures = case_ref['failures'] + case_ref = r[suite]["cases"][case] + case_str = "%s%s" % ( + case_ref["name"], + " (%s)" % case_ref["method"] + if case_ref["name"] != case_ref["method"] + else "", + ) + errors = case_ref["errors"] + failures = case_ref["failures"] self.stream.writeln(self.separator1) if errors: - self.stream.writeln('ERROR: %s [%s]' %(case_str, suite_str)) + self.stream.writeln("ERROR: %s [%s]" % (case_str, suite_str)) self.stream.writeln(self.separator2) self.stream.writeln(errors) if failures: - self.stream.writeln('FAILURE: %s [%s]' %(case_str, suite_str)) + self.stream.writeln("FAILURE: %s [%s]" % (case_str, suite_str)) self.stream.writeln(self.separator2) self.stream.writeln(failures) self.stream.writeln(self.separator3) @@ -501,52 +728,85 @@ def printErrorsDetail(self, r): def printSkippedDetail(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('SKIPPED DETAIL') + self.stream.writeln("SKIPPED DETAIL") fmt = [ - ('SUITE', 'suite', 50, 'left'), - ('CASE', 'case', 50, 'left'), - ('REASON', 'reason', 50, 'right') + ("SUITE", "suite", 50, "left"), + ("CASE", "case", 50, "left"), + ("REASON", "reason", 50, "right"), ] data = [] for suite in r: - cases = {k:v for (k,v) in list(r[suite]['cases'].items()) if v['result'] == 'skipped'} - for x in cases: data.append({'suite': '%s%s' %(r[suite]['name'], ' (%s)' %r[suite]['module'] if r[suite]['class'] != r[suite]['name'] else ''), - 'case': '%s%s' %(cases[x]['name'], ' (%s)' %cases[x]['method'] if cases[x]['name'] != cases[x]['method'] else ''), - 'reason': cases[x]['note']}) - self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) ) + cases = { + k: v + for (k, v) in list(r[suite]["cases"].items()) + if v["result"] == "skipped" + } + for x in cases: + data.append( + { + "suite": "%s%s" + % ( + r[suite]["name"], + " (%s)" % r[suite]["module"] + if r[suite]["class"] != r[suite]["name"] + else "", + ), + "case": "%s%s" + % ( + cases[x]["name"], + " (%s)" % cases[x]["method"] + if cases[x]["name"] != cases[x]["method"] + else "", + ), + "reason": cases[x]["note"], + } + ) + self.stream.writeln( + TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) + ) self.stream.writeln(self.separator3) self.stream.writeln() def returnCode(self): return not self.wasSuccessful() + class CustomTextTestRunner(unittest.TextTestRunner): """A test runner class that displays results in textual form. It prints out the names of tests as they are run, errors as they occur, and a summary of the results at the end of the test run. """ - def __init__(self, - stream=sys.stderr, - descriptions=True, - verbosity=1, - failfast=False, - buffer=False, - resultclass=CustomTextTestResult, - results_file_path="results.json", - result_screenshots_dir='', - show_previous_results=False, - test_name=None, - test_description=None, - config=None, - test_types=None): + def __init__( + self, + stream=sys.stderr, + descriptions=True, + verbosity=1, + failfast=False, + buffer=False, + resultclass=CustomTextTestResult, + results_file_path="results.json", + result_screenshots_dir="", + show_previous_results=False, + test_name=None, + test_description=None, + config=None, + test_types=None, + ): store_class_fields(self, locals()) self.stream = _WritelnDecorator(stream) def _makeResult(self): - return self.resultclass(self.stream, self.descriptions, self.verbosity, - self.results_file_path, self.result_screenshots_dir, self.show_previous_results, - self.config, self.test_types) + return self.resultclass( + self.stream, + self.descriptions, + self.verbosity, + self.results_file_path, + self.result_screenshots_dir, + self.show_previous_results, + self.config, + self.test_types, + ) def run(self, test): output = "" @@ -556,22 +816,26 @@ def run(self, test): result.failfast = self.failfast result.buffer = self.buffer startTime = time.time() - startTestRun = getattr(result, 'startTestRun', None) + startTestRun = getattr(result, "startTestRun", None) if startTestRun is not None: startTestRun() try: test(result) finally: - stopTestRun = getattr(result, 'stopTestRun', None) + stopTestRun = getattr(result, "stopTestRun", None) if stopTestRun is not None: stopTestRun() stopTime = time.time() timeTaken = stopTime - startTime # filter results to output if result.show_previous_results: - r = result.results['suites'] + r = result.results["suites"] else: - r = {k:v for (k,v) in list(result.results['suites'].items()) if k not in result.previous_suite_runs} + r = { + k: v + for (k, v) in list(result.results["suites"].items()) + if k not in result.previous_suite_runs + } # print results based on verbosity if result.show_all: result.printSkippedDetail(r) @@ -584,15 +848,17 @@ def run(self, test): if result.show_overall_results: result.printOverallSuiteResults(r) run = result.testsRun - self.stream.writeln("Ran %d test case%s in %.4fs" % - (run, run != 1 and "s" or "", timeTaken)) + self.stream.writeln( + "Ran %d test case%s in %.4fs" % (run, run != 1 and "s" or "", timeTaken) + ) self.stream.writeln() expectedFails = unexpectedSuccesses = skipped = 0 try: - results = map(len, (result.expectedFailures, - result.unexpectedSuccesses, - result.skipped)) + results = map( + len, + (result.expectedFailures, result.unexpectedSuccesses, result.skipped), + ) except AttributeError: pass else: diff --git a/extra_tests/jsontests.py b/extra_tests/jsontests.py index 7bc743d8d35..f3213ac09d1 100644 --- a/extra_tests/jsontests.py +++ b/extra_tests/jsontests.py @@ -1,13 +1,14 @@ -import unittest -from custom_text_test_runner import CustomTextTestRunner as Runner -from test.libregrtest.runtest import findtests import os +import unittest +from custom_text_test_runner import CustomTextTestRunner as Runner +from test.libregrtest.findtests import findtests testnames = findtests() # idk why this fixes the hanging, if it does -testnames.remove('test_importlib') -testnames.insert(0, 'test_importlib') +testnames.remove("test_importlib") +testnames.insert(0, "test_importlib") + def loadTestsOrSkip(loader, name): try: @@ -17,12 +18,16 @@ def loadTestsOrSkip(loader, name): @unittest.skip(str(exc)) def testSkipped(self): pass + attrs = {name: testSkipped} TestClass = type("ModuleSkipped", (unittest.TestCase,), attrs) return loader.suiteClass((TestClass(name),)) + loader = unittest.defaultTestLoader -suite = loader.suiteClass([loadTestsOrSkip(loader, 'test.' + name) for name in testnames]) +suite = loader.suiteClass( + [loadTestsOrSkip(loader, "test." + name) for name in testnames] +) resultsfile = os.path.join(os.path.dirname(__file__), "cpython_tests_results.json") if os.path.exists(resultsfile): diff --git a/extra_tests/snippets/3.1.3.4.py b/extra_tests/snippets/3.1.3.4.py index 426c78b42d7..f254376329d 100644 --- a/extra_tests/snippets/3.1.3.4.py +++ b/extra_tests/snippets/3.1.3.4.py @@ -1,3 +1,2 @@ -l = [1,2,3] -assert [1,2,3,4,5] == (l + [4,5]) - +l = [1, 2, 3] +assert [1, 2, 3, 4, 5] == (l + [4, 5]) diff --git a/extra_tests/snippets/3.1.3.5.py b/extra_tests/snippets/3.1.3.5.py index c841430b1b2..11889d936c9 100644 --- a/extra_tests/snippets/3.1.3.5.py +++ b/extra_tests/snippets/3.1.3.5.py @@ -1,3 +1,3 @@ -x = [1,999,3] +x = [1, 999, 3] x[1] = 2 -assert [1,2,3] == x +assert [1, 2, 3] == x diff --git a/extra_tests/snippets/builtin___main__.py b/extra_tests/snippets/builtin___main__.py new file mode 100644 index 00000000000..97e76ce324a --- /dev/null +++ b/extra_tests/snippets/builtin___main__.py @@ -0,0 +1,5 @@ +import sys + +main_module = sys.modules["__main__"] +assert main_module.__file__.endswith("builtin___main__.py") +assert main_module.__cached__ is None diff --git a/extra_tests/snippets/builtin_abs.py b/extra_tests/snippets/builtin_abs.py index 1b744978e59..7add4f4bcf4 100644 --- a/extra_tests/snippets/builtin_abs.py +++ b/extra_tests/snippets/builtin_abs.py @@ -2,4 +2,3 @@ assert abs(7) == 7 assert abs(-3.21) == 3.21 assert abs(6.25) == 6.25 - diff --git a/extra_tests/snippets/builtin_all.py b/extra_tests/snippets/builtin_all.py index cf0eea39321..ba11ce0de92 100644 --- a/extra_tests/snippets/builtin_all.py +++ b/extra_tests/snippets/builtin_all.py @@ -1,5 +1,4 @@ -from testutils import assert_raises -from testutils import TestFailingBool, TestFailingIter +from testutils import TestFailingBool, TestFailingIter, assert_raises assert all([True]) assert not all([False]) diff --git a/extra_tests/snippets/builtin_any.py b/extra_tests/snippets/builtin_any.py index 59b4514f85c..11b47d1f084 100644 --- a/extra_tests/snippets/builtin_any.py +++ b/extra_tests/snippets/builtin_any.py @@ -1,5 +1,4 @@ -from testutils import assert_raises -from testutils import TestFailingBool, TestFailingIter +from testutils import TestFailingBool, TestFailingIter, assert_raises assert any([True]) assert not any([False]) diff --git a/extra_tests/snippets/builtin_ascii.py b/extra_tests/snippets/builtin_ascii.py index 2132723c6b2..5b5b45d9990 100644 --- a/extra_tests/snippets/builtin_ascii.py +++ b/extra_tests/snippets/builtin_ascii.py @@ -1,7 +1,7 @@ -assert ascii('hello world') == "'hello world'" -assert ascii('안녕 세상') == "'\\uc548\\ub155 \\uc138\\uc0c1'" -assert ascii('안녕 RustPython') == "'\\uc548\\ub155 RustPython'" -assert ascii(5) == '5' +assert ascii("hello world") == "'hello world'" +assert ascii("안녕 세상") == "'\\uc548\\ub155 \\uc138\\uc0c1'" +assert ascii("안녕 RustPython") == "'\\uc548\\ub155 RustPython'" +assert ascii(5) == "5" assert ascii(chr(0x10001)) == "'\\U00010001'" assert ascii(chr(0x9999)) == "'\\u9999'" -assert ascii(chr(0x0A)) == "'\\n'" \ No newline at end of file +assert ascii(chr(0x0A)) == "'\\n'" diff --git a/extra_tests/snippets/builtin_bin.py b/extra_tests/snippets/builtin_bin.py index 97f57b2f13e..4f7a54c1b26 100644 --- a/extra_tests/snippets/builtin_bin.py +++ b/extra_tests/snippets/builtin_bin.py @@ -1,13 +1,13 @@ -assert bin(0) == '0b0' -assert bin(1) == '0b1' -assert bin(-1) == '-0b1' -assert bin(2**24) == '0b1' + '0' * 24 -assert bin(2**24-1) == '0b' + '1' * 24 -assert bin(-(2**24)) == '-0b1' + '0' * 24 -assert bin(-(2**24-1)) == '-0b' + '1' * 24 +assert bin(0) == "0b0" +assert bin(1) == "0b1" +assert bin(-1) == "-0b1" +assert bin(2**24) == "0b1" + "0" * 24 +assert bin(2**24 - 1) == "0b" + "1" * 24 +assert bin(-(2**24)) == "-0b1" + "0" * 24 +assert bin(-(2**24 - 1)) == "-0b" + "1" * 24 -a = 2 ** 65 -assert bin(a) == '0b1' + '0' * 65 -assert bin(a-1) == '0b' + '1' * 65 -assert bin(-(a)) == '-0b1' + '0' * 65 -assert bin(-(a-1)) == '-0b' + '1' * 65 +a = 2**65 +assert bin(a) == "0b1" + "0" * 65 +assert bin(a - 1) == "0b" + "1" * 65 +assert bin(-(a)) == "-0b1" + "0" * 65 +assert bin(-(a - 1)) == "-0b" + "1" * 65 diff --git a/extra_tests/snippets/builtin_bool.py b/extra_tests/snippets/builtin_bool.py index a46dbaab936..902ed0cced0 100644 --- a/extra_tests/snippets/builtin_bool.py +++ b/extra_tests/snippets/builtin_bool.py @@ -18,7 +18,9 @@ assert bool(1) is True assert bool({}) is False -assert bool(NotImplemented) is True +# NotImplemented cannot be used in a boolean context (Python 3.14+) +with assert_raises(TypeError): + bool(NotImplemented) assert bool(...) is True if not 1: @@ -30,18 +32,20 @@ if not object(): raise BaseException + class Falsey: def __bool__(self): return False + assert not Falsey() -assert (True or fake) # noqa: F821 -assert (False or True) +assert True or fake # noqa: F821 +assert False or True assert not (False or False) assert ("thing" or 0) == "thing" -assert (True and True) +assert True and True assert not (False and fake) # noqa: F821 assert (True and 5) == 5 @@ -92,15 +96,17 @@ def __bool__(self): assert bool({"key": "value"}) is True assert bool([1]) is True -assert bool(set([1,2])) is True +assert bool(set([1, 2])) is True assert repr(True) == "True" + # Check __len__ work class TestMagicMethodLenZero: def __len__(self): return 0 + class TestMagicMethodLenOne: def __len__(self): return 1 @@ -118,6 +124,7 @@ def __bool__(self): def __len__(self): return 0 + class TestMagicMethodBoolFalseLenTrue: def __bool__(self): return False @@ -125,6 +132,7 @@ def __bool__(self): def __len__(self): return 1 + assert bool(TestMagicMethodBoolTrueLenFalse()) is True assert bool(TestMagicMethodBoolFalseLenTrue()) is False @@ -134,9 +142,11 @@ class TestBoolThrowError: def __bool__(self): return object() + with assert_raises(TypeError): bool(TestBoolThrowError()) + class TestLenThrowError: def __len__(self): return object() @@ -145,6 +155,7 @@ def __len__(self): with assert_raises(TypeError): bool(TestLenThrowError()) + # Verify that TypeError occurs when bad things are returned # from __bool__(). This isn't really a bool test, but # it's related. @@ -152,31 +163,45 @@ def check(o): with assert_raises(TypeError): bool(o) + class Foo(object): def __bool__(self): return self + + check(Foo()) + class Bar(object): def __bool__(self): return "Yes" + + check(Bar()) + class Baz(int): def __bool__(self): return self + + check(Baz()) + # __bool__() must return a bool not an int class Spam(int): def __bool__(self): return 1 + + check(Spam()) + class Eggs: def __len__(self): return -1 + with assert_raises(ValueError): bool(Eggs()) diff --git a/extra_tests/snippets/builtin_bytearray.py b/extra_tests/snippets/builtin_bytearray.py index 008d3d23ee4..ee11e913ff2 100644 --- a/extra_tests/snippets/builtin_bytearray.py +++ b/extra_tests/snippets/builtin_bytearray.py @@ -1,7 +1,8 @@ -from testutils import assert_raises import pickle import sys +from testutils import assert_raises + # new assert bytearray([1, 2, 3]) assert bytearray((1, 2, 3)) @@ -40,9 +41,11 @@ ) assert repr(bytearray(b"abcd")) == "bytearray(b'abcd')" + class B(bytearray): pass + assert repr(B()) == "B(b'')" assert ( repr(B([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255])) @@ -150,16 +153,41 @@ class B(bytearray): # # hex from hex assert bytearray([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" -bytearray.fromhex("62 6c7a 34350a ") == b"blz45\n" +# fromhex with str +assert bytearray.fromhex("62 6c7a 34350a ") == b"blz45\n" + +# fromhex with bytes +assert bytearray.fromhex(b"62 6c7a 34350a ") == b"blz45\n" +assert bytearray.fromhex(b"B9 01EF") == b"\xb9\x01\xef" + +# fromhex with bytearray (bytes-like object) +assert bytearray.fromhex(bytearray(b"4142")) == b"AB" + +# fromhex with memoryview (bytes-like object) +assert bytearray.fromhex(memoryview(b"4142")) == b"AB" + +# fromhex error: non-hexadecimal character try: bytearray.fromhex("62 a 21") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 4" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 4" try: bytearray.fromhex("6Z2") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + +# fromhex error: odd number of hex digits +try: + bytearray.fromhex("abc") +except ValueError as e: + assert str(e) == "fromhex() arg must contain an even number of hexadecimal digits" + +# fromhex error: wrong type with assert_raises(TypeError): + bytearray.fromhex(123) + +# fromhex with bytes containing invalid hex raises ValueError +with assert_raises(ValueError): bytearray.fromhex(b"hhjjk") # center assert [bytearray(b"koki").center(i, b"|") for i in range(3, 10)] == [ @@ -283,9 +311,9 @@ class B(bytearray): ) == bytearray(b"jiljlkmoomkaaaa") with assert_raises(TypeError): bytearray(b"").join((b"km", "kl")) -assert bytearray(b"abc").join(( - bytearray(b"123"), bytearray(b"xyz") -)) == bytearray(b"123abcxyz") +assert bytearray(b"abc").join((bytearray(b"123"), bytearray(b"xyz"))) == bytearray( + b"123abcxyz" +) # endswith startswith @@ -372,16 +400,45 @@ class B(bytearray): assert bytearray(b"mississippi").rstrip(b"ipz") == bytearray(b"mississ") - # split -assert bytearray(b"1,2,3").split(bytearray(b",")) == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] -assert bytearray(b"1,2,3").split(bytearray(b","), maxsplit=1) == [bytearray(b"1"), bytearray(b"2,3")] -assert bytearray(b"1,2,,3,").split(bytearray(b",")) == [bytearray(b"1"), bytearray(b"2"), bytearray(b""), bytearray(b"3"), bytearray(b"")] -assert bytearray(b"1 2 3").split() == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] +assert bytearray(b"1,2,3").split(bytearray(b",")) == [ + bytearray(b"1"), + bytearray(b"2"), + bytearray(b"3"), +] +assert bytearray(b"1,2,3").split(bytearray(b","), maxsplit=1) == [ + bytearray(b"1"), + bytearray(b"2,3"), +] +assert bytearray(b"1,2,,3,").split(bytearray(b",")) == [ + bytearray(b"1"), + bytearray(b"2"), + bytearray(b""), + bytearray(b"3"), + bytearray(b""), +] +assert bytearray(b"1 2 3").split() == [ + bytearray(b"1"), + bytearray(b"2"), + bytearray(b"3"), +] assert bytearray(b"1 2 3").split(maxsplit=1) == [bytearray(b"1"), bytearray(b"2 3")] -assert bytearray(b" 1 2 3 ").split() == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] -assert bytearray(b"k\ruh\nfz e f").split() == [bytearray(b"k"), bytearray(b"uh"), bytearray(b"fz"), bytearray(b"e"), bytearray(b"f")] -assert bytearray(b"Two lines\n").split(bytearray(b"\n")) == [bytearray(b"Two lines"), bytearray(b"")] +assert bytearray(b" 1 2 3 ").split() == [ + bytearray(b"1"), + bytearray(b"2"), + bytearray(b"3"), +] +assert bytearray(b"k\ruh\nfz e f").split() == [ + bytearray(b"k"), + bytearray(b"uh"), + bytearray(b"fz"), + bytearray(b"e"), + bytearray(b"f"), +] +assert bytearray(b"Two lines\n").split(bytearray(b"\n")) == [ + bytearray(b"Two lines"), + bytearray(b""), +] assert bytearray(b"").split() == [] assert bytearray(b"").split(bytearray(b"\n")) == [bytearray(b"")] assert bytearray(b"\n").split(bytearray(b"\n")) == [bytearray(b""), bytearray(b"")] @@ -534,16 +591,21 @@ class B(bytearray): i = SPLIT_FIXTURES[n_sp] sep = None if i[1] == None else bytearray(i[1]) try: - assert bytearray(i[0]).split(sep=sep, maxsplit=i[4]) == [bytearray(j) for j in i[2]] + assert bytearray(i[0]).split(sep=sep, maxsplit=i[4]) == [ + bytearray(j) for j in i[2] + ] except AssertionError: print(i[0], i[1], i[2]) print( - "Expected : ", [list(x) for x in bytearray(i[0]).split(sep=sep, maxsplit=i[4])] + "Expected : ", + [list(x) for x in bytearray(i[0]).split(sep=sep, maxsplit=i[4])], ) break try: - assert bytearray(i[0]).rsplit(sep=sep, maxsplit=i[4]) == [bytearray(j) for j in i[3]] + assert bytearray(i[0]).rsplit(sep=sep, maxsplit=i[4]) == [ + bytearray(j) for j in i[3] + ] except AssertionError: print(i[0], i[1], i[2]) print( @@ -557,34 +619,61 @@ class B(bytearray): # expandtabs a = bytearray(b"\x01\x03\r\x05\t8CYZ\t\x06CYZ\t\x17cba`\n\x12\x13\x14") -assert ( - a.expandtabs() == bytearray(b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14") +assert a.expandtabs() == bytearray( + b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14" +) +assert a.expandtabs(5) == bytearray( + b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14" +) +assert bytearray(b"01\t012\t0123\t01234").expandtabs() == bytearray( + b"01 012 0123 01234" +) +assert bytearray(b"01\t012\t0123\t01234").expandtabs(4) == bytearray( + b"01 012 0123 01234" ) -assert a.expandtabs(5) == bytearray(b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14") -assert bytearray(b"01\t012\t0123\t01234").expandtabs() == bytearray(b"01 012 0123 01234") -assert bytearray(b"01\t012\t0123\t01234").expandtabs(4) == bytearray(b"01 012 0123 01234") assert bytearray(b"123\t123").expandtabs(-5) == bytearray(b"123123") assert bytearray(b"123\t123").expandtabs(0) == bytearray(b"123123") # # partition -assert bytearray(b"123456789").partition(b"45") == ((b"123"), bytearray(b"45"), bytearray(b"6789")) -assert bytearray(b"14523456789").partition(b"45") == ((b"1"), bytearray(b"45"), bytearray(b"23456789")) +assert bytearray(b"123456789").partition(b"45") == ( + (b"123"), + bytearray(b"45"), + bytearray(b"6789"), +) +assert bytearray(b"14523456789").partition(b"45") == ( + (b"1"), + bytearray(b"45"), + bytearray(b"23456789"), +) a = bytearray(b"14523456789").partition(b"45") assert isinstance(a[1], bytearray) a = bytearray(b"14523456789").partition(memoryview(b"45")) assert isinstance(a[1], bytearray) # partition -assert bytearray(b"123456789").rpartition(bytearray(b"45")) == ((bytearray(b"123")), bytearray(b"45"), bytearray(b"6789")) -assert bytearray(b"14523456789").rpartition(bytearray(b"45")) == ((bytearray(b"14523")), bytearray(b"45"), bytearray(b"6789")) +assert bytearray(b"123456789").rpartition(bytearray(b"45")) == ( + (bytearray(b"123")), + bytearray(b"45"), + bytearray(b"6789"), +) +assert bytearray(b"14523456789").rpartition(bytearray(b"45")) == ( + (bytearray(b"14523")), + bytearray(b"45"), + bytearray(b"6789"), +) a = bytearray(b"14523456789").rpartition(b"45") assert isinstance(a[1], bytearray) a = bytearray(b"14523456789").rpartition(memoryview(b"45")) assert isinstance(a[1], bytearray) # splitlines -assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines() == [bytearray(b"ab c"), bytearray(b""), bytearray(b"de fg"), bytearray(b"kl")] +assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines() == [ + bytearray(b"ab c"), + bytearray(b""), + bytearray(b"de fg"), + bytearray(b"kl"), +] assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines(keepends=True) == [ bytearray(b"ab c\n"), bytearray(b"\n"), @@ -602,11 +691,15 @@ class B(bytearray): assert bytearray(b"42").zfill(-1) == bytearray(b"42") # replace -assert bytearray(b"123456789123").replace(b"23",b"XX") ==bytearray(b'1XX4567891XX') -assert bytearray(b"123456789123").replace(b"23",b"XX", 1) ==bytearray(b'1XX456789123') -assert bytearray(b"123456789123").replace(b"23",b"XX", 0) == bytearray(b"123456789123") -assert bytearray(b"123456789123").replace(b"23",b"XX", -1) ==bytearray(b'1XX4567891XX') -assert bytearray(b"123456789123").replace(b"23", bytearray(b"")) == bytearray(b"14567891") +assert bytearray(b"123456789123").replace(b"23", b"XX") == bytearray(b"1XX4567891XX") +assert bytearray(b"123456789123").replace(b"23", b"XX", 1) == bytearray(b"1XX456789123") +assert bytearray(b"123456789123").replace(b"23", b"XX", 0) == bytearray(b"123456789123") +assert bytearray(b"123456789123").replace(b"23", b"XX", -1) == bytearray( + b"1XX4567891XX" +) +assert bytearray(b"123456789123").replace(b"23", bytearray(b"")) == bytearray( + b"14567891" +) # clear @@ -642,25 +735,24 @@ class B(bytearray): # title assert bytearray(b"Hello world").title() == bytearray(b"Hello World") -assert ( - bytearray(b"they're bill's friends from the UK").title() - == bytearray(b"They'Re Bill'S Friends From The Uk") +assert bytearray(b"they're bill's friends from the UK").title() == bytearray( + b"They'Re Bill'S Friends From The Uk" ) # repeat by multiply -a = bytearray(b'abcd') -assert a * 0 == bytearray(b'') -assert a * -1 == bytearray(b'') -assert a * 1 == bytearray(b'abcd') -assert a * 3 == bytearray(b'abcdabcdabcd') -assert 3 * a == bytearray(b'abcdabcdabcd') - -a = bytearray(b'abcd') +a = bytearray(b"abcd") +assert a * 0 == bytearray(b"") +assert a * -1 == bytearray(b"") +assert a * 1 == bytearray(b"abcd") +assert a * 3 == bytearray(b"abcdabcdabcd") +assert 3 * a == bytearray(b"abcdabcdabcd") + +a = bytearray(b"abcd") a.__imul__(3) -assert a == bytearray(b'abcdabcdabcd') +assert a == bytearray(b"abcdabcdabcd") a.__imul__(0) -assert a == bytearray(b'') +assert a == bytearray(b"") # copy @@ -696,70 +788,89 @@ class B(bytearray): # remove -a = bytearray(b'abcdabcd') +a = bytearray(b"abcdabcd") a.remove(99) # the letter c # Only the first is removed -assert a == bytearray(b'abdabcd') +assert a == bytearray(b"abdabcd") # reverse -a = bytearray(b'hello, world') +a = bytearray(b"hello, world") a.reverse() -assert a == bytearray(b'dlrow ,olleh') +assert a == bytearray(b"dlrow ,olleh") # __setitem__ -a = bytearray(b'test') +a = bytearray(b"test") a[0] = 1 -assert a == bytearray(b'\x01est') +assert a == bytearray(b"\x01est") with assert_raises(TypeError): - a[0] = b'a' + a[0] = b"a" with assert_raises(TypeError): - a[0] = memoryview(b'a') + a[0] = memoryview(b"a") a[:2] = [0, 9] -assert a == bytearray(b'\x00\x09st') -a[1:3] = b'test' -assert a == bytearray(b'\x00testt') -a[:6] = memoryview(b'test') -assert a == bytearray(b'test') +assert a == bytearray(b"\x00\x09st") +a[1:3] = b"test" +assert a == bytearray(b"\x00testt") +a[:6] = memoryview(b"test") +assert a == bytearray(b"test") # mod -assert bytearray('rust%bpython%b', 'utf-8') % (b' ', b'!') == bytearray(b'rust python!') -assert bytearray('x=%i y=%f', 'utf-8') % (1, 2.5) == bytearray(b'x=1 y=2.500000') +assert bytearray("rust%bpython%b", "utf-8") % (b" ", b"!") == bytearray(b"rust python!") +assert bytearray("x=%i y=%f", "utf-8") % (1, 2.5) == bytearray(b"x=1 y=2.500000") # eq, ne -a = bytearray(b'hello, world') +a = bytearray(b"hello, world") b = a.copy() assert a.__ne__(b) is False -b = bytearray(b'my bytearray') +b = bytearray(b"my bytearray") assert a.__ne__(b) is True # pickle -a = bytearray(b'\xffab\x80\0\0\370\0\0') -assert pickle.dumps(a, 0) == b'c__builtin__\nbytearray\np0\n(c_codecs\nencode\np1\n(V\xffab\x80\\u0000\\u0000\xf8\\u0000\\u0000\np2\nVlatin1\np3\ntp4\nRp5\ntp6\nRp7\n.' -assert pickle.dumps(a, 1) == b'c__builtin__\nbytearray\nq\x00(c_codecs\nencode\nq\x01(X\x0c\x00\x00\x00\xc3\xbfab\xc2\x80\x00\x00\xc3\xb8\x00\x00q\x02X\x06\x00\x00\x00latin1q\x03tq\x04Rq\x05tq\x06Rq\x07.' -assert pickle.dumps(a, 2) == b'\x80\x02c__builtin__\nbytearray\nq\x00c_codecs\nencode\nq\x01X\x0c\x00\x00\x00\xc3\xbfab\xc2\x80\x00\x00\xc3\xb8\x00\x00q\x02X\x06\x00\x00\x00latin1q\x03\x86q\x04Rq\x05\x85q\x06Rq\x07.' -assert pickle.dumps(a, 3) == b'\x80\x03cbuiltins\nbytearray\nq\x00C\t\xffab\x80\x00\x00\xf8\x00\x00q\x01\x85q\x02Rq\x03.' -assert pickle.dumps(a, 4) == b'\x80\x04\x95*\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\tbytearray\x94\x93\x94C\t\xffab\x80\x00\x00\xf8\x00\x00\x94\x85\x94R\x94.' +a = bytearray(b"\xffab\x80\0\0\370\0\0") +assert ( + pickle.dumps(a, 0) + == b"c__builtin__\nbytearray\np0\n(c_codecs\nencode\np1\n(V\xffab\x80\\u0000\\u0000\xf8\\u0000\\u0000\np2\nVlatin1\np3\ntp4\nRp5\ntp6\nRp7\n." +) +assert ( + pickle.dumps(a, 1) + == b"c__builtin__\nbytearray\nq\x00(c_codecs\nencode\nq\x01(X\x0c\x00\x00\x00\xc3\xbfab\xc2\x80\x00\x00\xc3\xb8\x00\x00q\x02X\x06\x00\x00\x00latin1q\x03tq\x04Rq\x05tq\x06Rq\x07." +) +assert ( + pickle.dumps(a, 2) + == b"\x80\x02c__builtin__\nbytearray\nq\x00c_codecs\nencode\nq\x01X\x0c\x00\x00\x00\xc3\xbfab\xc2\x80\x00\x00\xc3\xb8\x00\x00q\x02X\x06\x00\x00\x00latin1q\x03\x86q\x04Rq\x05\x85q\x06Rq\x07." +) +assert ( + pickle.dumps(a, 3) + == b"\x80\x03cbuiltins\nbytearray\nq\x00C\t\xffab\x80\x00\x00\xf8\x00\x00q\x01\x85q\x02Rq\x03." +) +assert ( + pickle.dumps(a, 4) + == b"\x80\x04\x95*\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\tbytearray\x94\x93\x94C\t\xffab\x80\x00\x00\xf8\x00\x00\x94\x85\x94R\x94." +) + # pickle with subclass class A(bytes): pass + a = A() a.x = 10 -a.y = A(b'123') +a.y = A(b"123") b = pickle.loads(pickle.dumps(a, 4)) assert type(a) == type(b) assert a.x == b.x assert a.y == b.y assert a == b + class B(bytearray): pass + a = B() a.x = 10 -a.y = B(b'123') +a.y = B(b"123") b = pickle.loads(pickle.dumps(a, 4)) assert type(a) == type(b) assert a.x == b.x @@ -768,4 +879,6 @@ class B(bytearray): a = bytearray() for i in range(-1, 2, 1): - assert_raises(IndexError, lambda: a[-sys.maxsize - i], _msg='bytearray index out of range') \ No newline at end of file + assert_raises( + IndexError, lambda: a[-sys.maxsize - i], _msg="bytearray index out of range" + ) diff --git a/extra_tests/snippets/builtin_bytes.py b/extra_tests/snippets/builtin_bytes.py index 2a6d0f63ebf..2cb4c317f49 100644 --- a/extra_tests/snippets/builtin_bytes.py +++ b/extra_tests/snippets/builtin_bytes.py @@ -1,3 +1,5 @@ +import sys + from testutils import assert_raises, skip_if_unsupported # new @@ -135,16 +137,41 @@ # hex from hex assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" -bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" +# fromhex with str +assert bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" + +# fromhex with bytes +assert bytes.fromhex(b"62 6c7a 34350a ") == b"blz45\n" +assert bytes.fromhex(b"B9 01EF") == b"\xb9\x01\xef" + +# fromhex with bytearray (bytes-like object) +assert bytes.fromhex(bytearray(b"4142")) == b"AB" + +# fromhex with memoryview (bytes-like object) +assert bytes.fromhex(memoryview(b"4142")) == b"AB" + +# fromhex error: non-hexadecimal character try: bytes.fromhex("62 a 21") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 4" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 4" try: bytes.fromhex("6Z2") except ValueError as e: - str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + assert str(e) == "non-hexadecimal number found in fromhex() arg at position 1" + +# fromhex error: odd number of hex digits +try: + bytes.fromhex("abc") +except ValueError as e: + assert str(e) == "fromhex() arg must contain an even number of hexadecimal digits" + +# fromhex error: wrong type with assert_raises(TypeError): + bytes.fromhex(123) + +# fromhex with bytes containing invalid hex raises ValueError +with assert_raises(ValueError): bytes.fromhex(b"hhjjk") # center assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ @@ -596,51 +623,59 @@ # repeat by multiply -a = b'abcd' -assert a * 0 == b'' -assert a * -1 == b'' -assert a * 1 == b'abcd' -assert a * 3 == b'abcdabcdabcd' -assert 3 * a == b'abcdabcdabcd' +a = b"abcd" +assert a * 0 == b"" +assert a * -1 == b"" +assert a * 1 == b"abcd" +assert a * 3 == b"abcdabcdabcd" +assert 3 * a == b"abcdabcdabcd" # decode -assert b'\x72\x75\x73\x74'.decode('ascii') == 'rust' -assert b'\xc2\xae\x75\x73\x74'.decode('ascii', 'replace') == '��ust' -assert b'\xc2\xae\x75\x73\x74'.decode('ascii', 'ignore') == 'ust' -assert b'\xc2\xae\x75\x73\x74'.decode('utf-8') == '®ust' -assert b'\xc2\xae\x75\x73\x74'.decode() == '®ust' -assert b'\xe4\xb8\xad\xe6\x96\x87\xe5\xad\x97'.decode('utf-8') == '中文字' +assert b"\x72\x75\x73\x74".decode("ascii") == "rust" +assert b"\xc2\xae\x75\x73\x74".decode("ascii", "replace") == "��ust" +assert b"\xc2\xae\x75\x73\x74".decode("ascii", "ignore") == "ust" +assert b"\xc2\xae\x75\x73\x74".decode("utf-8") == "®ust" +assert b"\xc2\xae\x75\x73\x74".decode() == "®ust" +assert b"\xe4\xb8\xad\xe6\x96\x87\xe5\xad\x97".decode("utf-8") == "中文字" + +# gh-2391 +assert b"-\xff".decode(sys.getfilesystemencoding(), "surrogateescape") == "-\udcff" # mod -assert b'rust%bpython%b' % (b' ', b'!') == b'rust python!' -assert b'x=%i y=%f' % (1, 2.5) == b'x=1 y=2.500000' +assert b"rust%bpython%b" % (b" ", b"!") == b"rust python!" +assert b"x=%i y=%f" % (1, 2.5) == b"x=1 y=2.500000" + # __bytes__ def test__bytes__(): - foo = b'foo\x00bar' + foo = b"foo\x00bar" assert foo.__bytes__() == foo assert type(foo.__bytes__()) == bytes + class bytes_subclass(bytes): pass - bar = bytes_subclass(b'bar\x00foo') + + bar = bytes_subclass(b"bar\x00foo") assert bar.__bytes__() == bar assert type(bar.__bytes__()) == bytes + class A: def __bytes__(self): return b"bytess" + assert bytes(A()) == b"bytess" # Issue #2125 -b = b'abc' +b = b"abc" assert bytes(b) is b # Regression to # https://github.com/RustPython/RustPython/issues/2840 -a = b'123abc!?' +a = b"123abc!?" assert id(a) == id(a) assert id(a) != id(a * -1) assert id(a) != id(a * 0) @@ -652,20 +687,24 @@ def __bytes__(self): class SubBytes(bytes): pass -b = SubBytes(b'0123abc*&') + +b = SubBytes(b"0123abc*&") assert id(b) == id(b) assert id(b) != id(b * -1) assert id(b) != id(b * 0) assert id(b) != id(b * 1) assert id(b) != id(b * 2) + class B1(bytearray): def __new__(cls, value): assert type(value) == bytes me = super().__new__(cls, value) - me.foo = 'bar' + me.foo = "bar" return me -b = B1.fromhex('a0a1a2') -assert b.foo == 'bar' -skip_if_unsupported(3,11,test__bytes__) \ No newline at end of file + +b = B1.fromhex("a0a1a2") +assert b.foo == "bar" + +skip_if_unsupported(3, 11, test__bytes__) diff --git a/extra_tests/snippets/builtin_callable.py b/extra_tests/snippets/builtin_callable.py index db554df2452..52458bda94c 100644 --- a/extra_tests/snippets/builtin_callable.py +++ b/extra_tests/snippets/builtin_callable.py @@ -1,25 +1,48 @@ assert not callable(1) -def f(): pass + + +def f(): + pass + + assert callable(f) assert callable(len) assert callable(lambda: 1) assert callable(int) + class C: def __init__(self): # must be defined on class self.__call__ = lambda self: 1 - def f(self): pass + + def f(self): + pass + + assert callable(C) assert not callable(C()) assert callable(C().f) + class C: - def __call__(self): pass + def __call__(self): + pass + + assert callable(C()) -class C1(C): pass + + +class C1(C): + pass + + assert callable(C1()) + + class C: __call__ = 1 + + # CPython returns true here, but fails when actually calling it assert callable(C()) diff --git a/extra_tests/snippets/builtin_chr.py b/extra_tests/snippets/builtin_chr.py index 9b95452bdad..6fa56203d7b 100644 --- a/extra_tests/snippets/builtin_chr.py +++ b/extra_tests/snippets/builtin_chr.py @@ -4,5 +4,7 @@ assert "é" == chr(233) assert "🤡" == chr(129313) -assert_raises(TypeError, chr, _msg='chr() takes exactly one argument (0 given)') -assert_raises(ValueError, chr, 0x110005, _msg='ValueError: chr() arg not in range(0x110000)') +assert_raises(TypeError, chr, _msg="chr() takes exactly one argument (0 given)") +assert_raises( + ValueError, chr, 0x110005, _msg="ValueError: chr() arg not in range(0x110000)" +) diff --git a/extra_tests/snippets/builtin_dict.py b/extra_tests/snippets/builtin_dict.py index 3ccea972328..83a8c5f9945 100644 --- a/extra_tests/snippets/builtin_dict.py +++ b/extra_tests/snippets/builtin_dict.py @@ -5,49 +5,49 @@ assert len({}) == 0 assert len({"a": "b"}) == 1 assert len({"a": "b", "b": 1}) == 2 -assert len({"a": "b", "b": 1, "a" + "b": 2*2}) == 3 +assert len({"a": "b", "b": 1, "a" + "b": 2 * 2}) == 3 d = {} -d['a'] = d +d["a"] = d assert repr(d) == "{'a': {...}}" -assert {'a': 123}.get('a') == 123 -assert {'a': 123}.get('b') == None -assert {'a': 123}.get('b', 456) == 456 +assert {"a": 123}.get("a") == 123 +assert {"a": 123}.get("b") == None +assert {"a": 123}.get("b", 456) == 456 -d = {'a': 123, 'b': 456} -assert list(reversed(d)) == ['b', 'a'] -assert list(reversed(d.keys())) == ['b', 'a'] +d = {"a": 123, "b": 456} +assert list(reversed(d)) == ["b", "a"] +assert list(reversed(d.keys())) == ["b", "a"] assert list(reversed(d.values())) == [456, 123] -assert list(reversed(d.items())) == [('b', 456), ('a', 123)] +assert list(reversed(d.items())) == [("b", 456), ("a", 123)] with assert_raises(StopIteration): dict_reversed = reversed(d) for _ in range(len(d) + 1): next(dict_reversed) -assert 'dict' in dict().__doc__ +assert "dict" in dict().__doc__ -d = {'a': 123, 'b': 456} +d = {"a": 123, "b": 456} assert 1 not in d.items() -assert 'a' not in d.items() -assert 'a', 123 not in d.items() +assert "a" not in d.items() +assert "a", 123 not in d.items() assert () not in d.items() assert (1) not in d.items() -assert ('a') not in d.items() -assert ('a', 123) in d.items() -assert ('b', 456) in d.items() -assert ('a', 123, 3) not in d.items() -assert ('a', 123, 'b', 456) not in d.items() +assert ("a") not in d.items() +assert ("a", 123) in d.items() +assert ("b", 456) in d.items() +assert ("a", 123, 3) not in d.items() +assert ("a", 123, "b", 456) not in d.items() -d = {1: 10, "a": "ABC", (3,4): 5} +d = {1: 10, "a": "ABC", (3, 4): 5} assert 1 in d.keys() assert (1) in d.keys() assert "a" in d.keys() -assert (3,4) in d.keys() +assert (3, 4) in d.keys() assert () not in d.keys() assert 10 not in d.keys() assert (1, 10) not in d.keys() assert "abc" not in d.keys() -assert ((3,4),5) not in d.keys() +assert ((3, 4), 5) not in d.keys() d1 = {"a": 1, "b": 2} d2 = {"c": 3, "d": 4} @@ -64,55 +64,55 @@ assert not d1.keys().isdisjoint(d2.keys()) -assert dict(a=2, b=3) == {'a': 2, 'b': 3} -assert dict({'a': 2, 'b': 3}, b=4) == {'a': 2, 'b': 4} -assert dict([('a', 2), ('b', 3)]) == {'a': 2, 'b': 3} +assert dict(a=2, b=3) == {"a": 2, "b": 3} +assert dict({"a": 2, "b": 3}, b=4) == {"a": 2, "b": 4} +assert dict([("a", 2), ("b", 3)]) == {"a": 2, "b": 3} assert {} == {} -assert not {'a': 2} == {} -assert not {} == {'a': 2} -assert not {'b': 2} == {'a': 2} -assert not {'a': 4} == {'a': 2} -assert {'a': 2} == {'a': 2} +assert not {"a": 2} == {} +assert not {} == {"a": 2} +assert not {"b": 2} == {"a": 2} +assert not {"a": 4} == {"a": 2} +assert {"a": 2} == {"a": 2} -nan = float('nan') -assert {'a': nan} == {'a': nan} +nan = float("nan") +assert {"a": nan} == {"a": nan} -a = {'g': 5} -b = {'a': a, 'd': 9} +a = {"g": 5} +b = {"a": a, "d": 9} c = dict(b) -c['d'] = 3 -c['a']['g'] = 2 -assert a == {'g': 2} -assert b == {'a': a, 'd': 9} +c["d"] = 3 +c["a"]["g"] = 2 +assert a == {"g": 2} +assert b == {"a": a, "d": 9} a.clear() assert len(a) == 0 -a = {'a': 5, 'b': 6} +a = {"a": 5, "b": 6} res = set() for value in a.values(): - res.add(value) -assert res == set([5,6]) + res.add(value) +assert res == set([5, 6]) count = 0 -for (key, value) in a.items(): - assert a[key] == value - count += 1 +for key, value in a.items(): + assert a[key] == value + count += 1 assert count == len(a) res = set() for key in a.keys(): - res.add(key) -assert res == set(['a','b']) + res.add(key) +assert res == set(["a", "b"]) # Deleted values are correctly skipped over: -x = {'a': 1, 'b': 2, 'c': 3, 'd': 3} -del x['c'] +x = {"a": 1, "b": 2, "c": 3, "d": 3} +del x["c"] it = iter(x.items()) -assert ('a', 1) == next(it) -assert ('b', 2) == next(it) -assert ('d', 3) == next(it) +assert ("a", 1) == next(it) +assert ("b", 2) == next(it) +assert ("d", 3) == next(it) with assert_raises(StopIteration): next(it) @@ -121,7 +121,7 @@ assert cm.exception.args[0] == 10 # Iterating a dictionary is just its keys: -assert ['a', 'b', 'd'] == list(x) +assert ["a", "b", "d"] == list(x) # Iterating view captures dictionary when iterated. data = {1: 2, 3: 4} @@ -140,12 +140,12 @@ # But we can't add or delete items during iteration. d = {} a = iter(d.items()) -d['a'] = 2 +d["a"] = 2 b = iter(d.items()) -assert ('a', 2) == next(b) +assert ("a", 2) == next(b) with assert_raises(RuntimeError): next(a) -del d['a'] +del d["a"] with assert_raises(RuntimeError): next(b) @@ -164,21 +164,23 @@ x[(5, 6)] = 5 with assert_raises(TypeError): - x[[]] # Unhashable type. + x[[]] # Unhashable type. x["here"] = "here" assert x.get("not here", "default") == "default" assert x.get("here", "default") == "here" assert x.get("not here") == None + class LengthDict(dict): def __getitem__(self, k): return len(k) + x = LengthDict() assert type(x) == LengthDict -assert x['word'] == 4 -assert x.get('word') is None +assert x["word"] == 4 +assert x.get("word") is None assert 5 == eval("a + word", LengthDict()) @@ -189,15 +191,19 @@ def __missing__(self, k): self[k] = v return v + x = Squares() assert x[-5] == 25 + # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): def __init__(self, *args): self.values = args + def __hash__(self): return 1 + def __eq__(self, other): for x in self.values: for y in other.values: @@ -205,39 +211,40 @@ def __eq__(self, other): return True return False + x = {} -x[Hashable(1,2)] = 8 +x[Hashable(1, 2)] = 8 -assert x[Hashable(1,2)] == 8 -assert x[Hashable(3,1)] == 8 +assert x[Hashable(1, 2)] == 8 +assert x[Hashable(3, 1)] == 8 x[Hashable(8)] = 19 -x[Hashable(19,8)] = 1 +x[Hashable(19, 8)] = 1 assert x[Hashable(8)] == 1 assert len(x) == 2 -assert list({'a': 2, 'b': 10}) == ['a', 'b'] +assert list({"a": 2, "b": 10}) == ["a", "b"] x = {} -x['a'] = 2 -x['b'] = 10 -assert list(x) == ['a', 'b'] +x["a"] = 2 +x["b"] = 10 +assert list(x) == ["a", "b"] y = x.copy() -x['c'] = 12 -assert y == {'a': 2, 'b': 10} +x["c"] = 12 +assert y == {"a": 2, "b": 10} -y.update({'c': 19, "d": -1, 'b': 12}) -assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} +y.update({"c": 19, "d": -1, "b": 12}) +assert y == {"a": 2, "b": 12, "c": 19, "d": -1} y.update(y) -assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} # hasn't changed +assert y == {"a": 2, "b": 12, "c": 19, "d": -1} # hasn't changed # KeyError has object that used as key as an .args[0] with assert_raises(KeyError) as cm: - x['not here'] + x["not here"] assert cm.exception.args[0] == "not here" with assert_raises(KeyError) as cm: - x.pop('not here') + x.pop("not here") assert cm.exception.args[0] == "not here" with assert_raises(KeyError) as cm: @@ -247,7 +254,11 @@ def __eq__(self, other): x.pop(10) assert cm.exception.args[0] == 10 -class MyClass: pass + +class MyClass: + pass + + obj = MyClass() with assert_raises(KeyError) as cm: @@ -257,49 +268,65 @@ class MyClass: pass x.pop(obj) assert cm.exception.args[0] == obj -x = {1: 'a', '1': None} -assert x.pop(1) == 'a' -assert x.pop('1') is None +x = {1: "a", "1": None} +assert x.pop(1) == "a" +assert x.pop("1") is None assert x == {} -x = {1: 'a'} -assert (1, 'a') == x.popitem() +x = {1: "a"} +assert (1, "a") == x.popitem() assert x == {} with assert_raises(KeyError) as cm: x.popitem() -assert cm.exception.args == ('popitem(): dictionary is empty',) +assert cm.exception.args == ("popitem(): dictionary is empty",) -x = {'a': 4} -assert 4 == x.setdefault('a', 0) -assert x['a'] == 4 -assert 0 == x.setdefault('b', 0) -assert x['b'] == 0 -assert None == x.setdefault('c') -assert x['c'] is None +x = {"a": 4} +assert 4 == x.setdefault("a", 0) +assert x["a"] == 4 +assert 0 == x.setdefault("b", 0) +assert x["b"] == 0 +assert None == x.setdefault("c") +assert x["c"] is None assert {1: None, "b": None} == dict.fromkeys([1, "b"]) assert {1: 0, "b": 0} == dict.fromkeys([1, "b"], 0) -x = {'a': 1, 'b': 1, 'c': 1} -y = {'b': 2, 'c': 2, 'd': 2} -z = {'c': 3, 'd': 3, 'e': 3} +x = {"a": 1, "b": 1, "c": 1} +y = {"b": 2, "c": 2, "d": 2} +z = {"c": 3, "d": 3, "e": 3} w = {1: 1, **x, 2: 2, **y, 3: 3, **z, 4: 4} -assert w == {1: 1, 'a': 1, 'b': 2, 'c': 3, 2: 2, 'd': 3, 3: 3, 'e': 3, 4: 4} # not in cpython test suite +assert w == { + 1: 1, + "a": 1, + "b": 2, + "c": 3, + 2: 2, + "d": 3, + 3: 3, + "e": 3, + 4: 4, +} # not in cpython test suite assert str({True: True, 1.0: 1.0}) == str({True: 1.0}) + class A: def __hash__(self): return 1 + def __eq__(self, other): return isinstance(other, A) + + class B: def __hash__(self): return 1 + def __eq__(self, other): return isinstance(other, B) + s = {1: 0, A(): 1, B(): 2} assert len(s) == 3 assert s[1] == 0 @@ -307,19 +334,19 @@ def __eq__(self, other): assert s[B()] == 2 # Test dict usage in set with star expressions! -a = {'bla': 2} -b = {'c': 44, 'bla': 332, 'd': 6} -x = ['bla', 'c', 'd', 'f'] +a = {"bla": 2} +b = {"c": 44, "bla": 332, "d": 6} +x = ["bla", "c", "d", "f"] c = {*a, *b, *x} # print(c, type(c)) assert isinstance(c, set) -assert c == {'bla', 'c', 'd', 'f'} +assert c == {"bla", "c", "d", "f"} assert not {}.__ne__({}) -assert {}.__ne__({'a':'b'}) +assert {}.__ne__({"a": "b"}) assert {}.__ne__(1) == NotImplemented -it = iter({0: 1, 2: 3, 4:5, 6:7}) +it = iter({0: 1, 2: 3, 4: 5, 6: 7}) assert it.__length_hint__() == 4 next(it) assert it.__length_hint__() == 3 @@ -331,3 +358,53 @@ def __eq__(self, other): assert it.__length_hint__() == 0 assert_raises(StopIteration, next, it) assert it.__length_hint__() == 0 + +# Test dictionary unpacking with non-mapping objects +# This should raise TypeError for non-mapping objects +with assert_raises(TypeError) as cm: + {**[1, 2]} +assert "'list' object is not a mapping" in str(cm.exception) + +with assert_raises(TypeError) as cm: + {**[[1, 2], [3, 4]]} +assert "'list' object is not a mapping" in str(cm.exception) + +with assert_raises(TypeError) as cm: + {**"string"} +assert "'str' object is not a mapping" in str(cm.exception) + +with assert_raises(TypeError) as cm: + {**(1, 2, 3)} +assert "'tuple' object is not a mapping" in str(cm.exception) + +# Test that valid mappings still work +assert {**{"a": 1}, **{"b": 2}} == {"a": 1, "b": 2} + +# Test OrderedDict unpacking preserves order +import collections + +od = collections.OrderedDict([("a", 1), ("b", 2)]) +od.move_to_end("a") # Move 'a' to end: ['b', 'a'] +expected_order = list(od.items()) # [('b', 2), ('a', 1)] + + +def test_func(**kwargs): + return kwargs + + +result = test_func(**od) +assert list(result.items()) == expected_order, ( + f"Expected {expected_order}, got {list(result.items())}" +) + +# Test multiple OrderedDict unpacking +od1 = collections.OrderedDict([("x", 10), ("y", 20)]) +od2 = collections.OrderedDict([("z", 30), ("w", 40)]) +od2.move_to_end("z") # Move 'z' to end: ['w', 'z'] + +result = test_func(**od1, **od2) +# Should preserve order: x, y, w, z +expected_keys = ["x", "y", "w", "z"] +assert list(result.keys()) == expected_keys, ( + f"Expected {expected_keys}, got {list(result.keys())}" +) diff --git a/extra_tests/snippets/builtin_dict_union.py b/extra_tests/snippets/builtin_dict_union.py index f33f32a5e4e..ab3fa65d376 100644 --- a/extra_tests/snippets/builtin_dict_union.py +++ b/extra_tests/snippets/builtin_dict_union.py @@ -1,78 +1,91 @@ - from testutils import assert_raises, skip_if_unsupported + def test_dunion_ior0(): - a={1:2,2:3} - b={3:4,5:6} - a|=b + a = {1: 2, 2: 3} + b = {3: 4, 5: 6} + a |= b + + assert a == {1: 2, 2: 3, 3: 4, 5: 6}, f"wrong value assigned {a=}" + assert b == {3: 4, 5: 6}, f"right hand side modified, {b=}" - assert a == {1:2,2:3,3:4,5:6}, f"wrong value assigned {a=}" - assert b == {3:4,5:6}, f"right hand side modified, {b=}" def test_dunion_or0(): - a={1:2,2:3} - b={3:4,5:6} - c=a|b + a = {1: 2, 2: 3} + b = {3: 4, 5: 6} + c = a | b - assert a == {1:2,2:3}, f"left hand side of non-assignment operator modified {a=}" - assert b == {3:4,5:6}, f"right hand side of non-assignment operator modified, {b=}" - assert c == {1:2,2:3, 3:4, 5:6}, f"unexpected result of dict union {c=}" + assert a == {1: 2, 2: 3}, f"left hand side of non-assignment operator modified {a=}" + assert b == {3: 4, 5: 6}, ( + f"right hand side of non-assignment operator modified, {b=}" + ) + assert c == {1: 2, 2: 3, 3: 4, 5: 6}, f"unexpected result of dict union {c=}" def test_dunion_or1(): - a={1:2,2:3} - b={3:4,5:6} - c=a.__or__(b) + a = {1: 2, 2: 3} + b = {3: 4, 5: 6} + c = a.__or__(b) - assert a == {1:2,2:3}, f"left hand side of non-assignment operator modified {a=}" - assert b == {3:4,5:6}, f"right hand side of non-assignment operator modified, {b=}" - assert c == {1:2,2:3, 3:4, 5:6}, f"unexpected result of dict union {c=}" + assert a == {1: 2, 2: 3}, f"left hand side of non-assignment operator modified {a=}" + assert b == {3: 4, 5: 6}, ( + f"right hand side of non-assignment operator modified, {b=}" + ) + assert c == {1: 2, 2: 3, 3: 4, 5: 6}, f"unexpected result of dict union {c=}" def test_dunion_ror0(): - a={1:2,2:3} - b={3:4,5:6} - c=b.__ror__(a) + a = {1: 2, 2: 3} + b = {3: 4, 5: 6} + c = b.__ror__(a) - assert a == {1:2,2:3}, f"left hand side of non-assignment operator modified {a=}" - assert b == {3:4,5:6}, f"right hand side of non-assignment operator modified, {b=}" - assert c == {1:2,2:3, 3:4, 5:6}, f"unexpected result of dict union {c=}" + assert a == {1: 2, 2: 3}, f"left hand side of non-assignment operator modified {a=}" + assert b == {3: 4, 5: 6}, ( + f"right hand side of non-assignment operator modified, {b=}" + ) + assert c == {1: 2, 2: 3, 3: 4, 5: 6}, f"unexpected result of dict union {c=}" def test_dunion_other_types(): def perf_test_or(other_obj): - d={1:2} + d = {1: 2} return d.__or__(other_obj) is NotImplemented def perf_test_ror(other_obj): - d={1:2} + d = {1: 2} return d.__ror__(other_obj) is NotImplemented - test_fct={'__or__':perf_test_or, '__ror__':perf_test_ror} - others=['FooBar', 42, [36], set([19]), ['aa'], None] - for tfn,tf in test_fct.items(): + test_fct = {"__or__": perf_test_or, "__ror__": perf_test_ror} + others = ["FooBar", 42, [36], set([19]), ["aa"], None] + for tfn, tf in test_fct.items(): for other in others: assert tf(other), f"Failed: dict {tfn}, accepted {other}" # __ior__() has different behavior and needs to be tested separately d = {1: 2} - assert_raises(ValueError, - lambda: d.__ior__('FooBar'), - _msg='dictionary update sequence element #0 has length 1; 2 is required') - assert_raises(TypeError, - lambda: d.__ior__(42), - _msg='\'int\' object is not iterable') - assert_raises(TypeError, - lambda: d.__ior__([36]), - _msg='cannot convert dictionary update sequence element #0 to a sequence') - assert_raises(TypeError, - lambda: d.__ior__(set([36])), - _msg='cannot convert dictionary update sequence element #0 to a sequence') - res = d.__ior__(['aa']) - assert res == {1: 2, 'a': 'a'}, f"unexpected result of dict union {res=}" - assert_raises(TypeError, - lambda: d.__ior__(None), - _msg='TypeError: \'NoneType\' object is not iterable') + assert_raises( + ValueError, + lambda: d.__ior__("FooBar"), + _msg="dictionary update sequence element #0 has length 1; 2 is required", + ) + assert_raises(TypeError, lambda: d.__ior__(42), _msg="'int' object is not iterable") + assert_raises( + TypeError, + lambda: d.__ior__([36]), + _msg="cannot convert dictionary update sequence element #0 to a sequence", + ) + assert_raises( + TypeError, + lambda: d.__ior__(set([36])), + _msg="cannot convert dictionary update sequence element #0 to a sequence", + ) + res = d.__ior__(["aa"]) + assert res == {1: 2, "a": "a"}, f"unexpected result of dict union {res=}" + assert_raises( + TypeError, + lambda: d.__ior__(None), + _msg="TypeError: 'NoneType' object is not iterable", + ) skip_if_unsupported(3, 9, test_dunion_ior0) diff --git a/extra_tests/snippets/builtin_dir.py b/extra_tests/snippets/builtin_dir.py index 3e808597c15..cd2c8c33a21 100644 --- a/extra_tests/snippets/builtin_dir.py +++ b/extra_tests/snippets/builtin_dir.py @@ -1,9 +1,11 @@ assert isinstance(dir(), list) -assert '__builtins__' in dir() +assert "__builtins__" in dir() + class A: - def test(): - pass + def test(): + pass + a = A() @@ -13,24 +15,30 @@ def test(): a.x = 3 assert "x" in dir(a), "x not in a" + class B(A): - def __dir__(self): - return ('q', 'h') + def __dir__(self): + return ("q", "h") + # Gets sorted and turned into a list -assert ['h', 'q'] == dir(B()) +assert ["h", "q"] == dir(B()) # This calls type.__dir__ so isn't changed (but inheritance works)! -assert 'test' in dir(A) +assert "test" in dir(A) + # eval() takes any mapping-like type, so dir() must support them # TODO: eval() should take any mapping as locals, not just dict-derived types class A(dict): - def __getitem__(self, x): - return dir - def keys(self): - yield 6 - yield 5 + def __getitem__(self, x): + return dir + + def keys(self): + yield 6 + yield 5 + + assert eval("dir()", {}, A()) == [5, 6] import socket diff --git a/extra_tests/snippets/builtin_divmod.py b/extra_tests/snippets/builtin_divmod.py index 5a9443afe8b..f62d0f8eea8 100644 --- a/extra_tests/snippets/builtin_divmod.py +++ b/extra_tests/snippets/builtin_divmod.py @@ -1,9 +1,9 @@ from testutils import assert_raises assert divmod(11, 3) == (3, 2) -assert divmod(8,11) == (0, 8) +assert divmod(8, 11) == (0, 8) assert divmod(0.873, 0.252) == (3.0, 0.11699999999999999) assert divmod(-86340, 86400) == (-1, 60) -assert_raises(ZeroDivisionError, divmod, 5, 0, _msg='divmod by zero') -assert_raises(ZeroDivisionError, divmod, 5.0, 0.0, _msg='divmod by zero') +assert_raises(ZeroDivisionError, divmod, 5, 0, _msg="divmod by zero") +assert_raises(ZeroDivisionError, divmod, 5.0, 0.0, _msg="divmod by zero") diff --git a/extra_tests/snippets/builtin_ellipsis.py b/extra_tests/snippets/builtin_ellipsis.py index 5316b9f865c..cf99f3cc829 100644 --- a/extra_tests/snippets/builtin_ellipsis.py +++ b/extra_tests/snippets/builtin_ellipsis.py @@ -1,5 +1,3 @@ - - a = ... b = ... c = type(a)() # Test singleton behavior @@ -11,22 +9,22 @@ assert b is d assert d is e -assert Ellipsis.__repr__() == 'Ellipsis' -assert Ellipsis.__reduce__() == 'Ellipsis' +assert Ellipsis.__repr__() == "Ellipsis" +assert Ellipsis.__reduce__() == "Ellipsis" assert type(Ellipsis).__new__(type(Ellipsis)) == Ellipsis -assert type(Ellipsis).__reduce__(Ellipsis) == 'Ellipsis' +assert type(Ellipsis).__reduce__(Ellipsis) == "Ellipsis" try: type(Ellipsis).__new__(type(1)) except TypeError: pass else: - assert False, '`Ellipsis.__new__` should only accept `type(Ellipsis)` as argument' + assert False, "`Ellipsis.__new__` should only accept `type(Ellipsis)` as argument" try: type(Ellipsis).__reduce__(1) except TypeError: pass else: - assert False, '`Ellipsis.__reduce__` should only accept `Ellipsis` as argument' + assert False, "`Ellipsis.__reduce__` should only accept `Ellipsis` as argument" assert Ellipsis is ... Ellipsis = 2 diff --git a/extra_tests/snippets/builtin_enumerate.py b/extra_tests/snippets/builtin_enumerate.py index 35edadd1d73..0f107ea7aea 100644 --- a/extra_tests/snippets/builtin_enumerate.py +++ b/extra_tests/snippets/builtin_enumerate.py @@ -1,9 +1,14 @@ -assert list(enumerate(['a', 'b', 'c'])) == [(0, 'a'), (1, 'b'), (2, 'c')] +assert list(enumerate(["a", "b", "c"])) == [(0, "a"), (1, "b"), (2, "c")] assert type(enumerate([])) == enumerate -assert list(enumerate(['a', 'b', 'c'], -100)) == [(-100, 'a'), (-99, 'b'), (-98, 'c')] -assert list(enumerate(['a', 'b', 'c'], 2**200)) == [(2**200, 'a'), (2**200 + 1, 'b'), (2**200 + 2, 'c')] +assert list(enumerate(["a", "b", "c"], -100)) == [(-100, "a"), (-99, "b"), (-98, "c")] +assert list(enumerate(["a", "b", "c"], 2**200)) == [ + (2**200, "a"), + (2**200 + 1, "b"), + (2**200 + 2, "c"), +] + # test infinite iterator class Counter(object): diff --git a/extra_tests/snippets/builtin_eval.py b/extra_tests/snippets/builtin_eval.py index 6375bd0c1ae..2f2405c8d9e 100644 --- a/extra_tests/snippets/builtin_eval.py +++ b/extra_tests/snippets/builtin_eval.py @@ -1,4 +1,77 @@ -assert 3 == eval('1+2') +assert 3 == eval("1+2") -code = compile('5+3', 'x.py', 'eval') +code = compile("5+3", "x.py", "eval") assert eval(code) == 8 + +# Test that globals must be a dict +import collections + +# UserDict is a mapping but not a dict - should fail in eval +user_dict = collections.UserDict({"x": 5}) +try: + eval("x", user_dict) + assert False, "eval with UserDict globals should fail" +except TypeError as e: + # CPython: "globals must be a real dict; try eval(expr, {}, mapping)" + assert "globals must be a real dict" in str(e), e + +# Non-mapping should have different error message +try: + eval("x", 123) + assert False, "eval with int globals should fail" +except TypeError as e: + # CPython: "globals must be a dict" + assert "globals must be a dict" in str(e) + assert "real dict" not in str(e) + +# List is not a mapping +try: + eval("x", []) + assert False, "eval with list globals should fail" +except TypeError as e: + assert "globals must be a real dict" in str(e), e + +# Regular dict should work +assert eval("x", {"x": 42}) == 42 + +# None should use current globals +x = 100 +assert eval("x", None) == 100 + +# Test locals parameter +# Locals can be any mapping (unlike globals which must be dict) +assert eval("y", {"y": 1}, user_dict) == 1 # UserDict as locals is OK + +# But locals must still be a mapping +try: + eval("x", {"x": 1}, 123) + assert False, "eval with int locals should fail" +except TypeError as e: + # This error is handled by ArgMapping validation + assert "not a mapping" in str(e) or "locals must be a mapping" in str(e) + +# Test that __builtins__ is added if missing +globals_without_builtins = {"x": 5} +result = eval("x", globals_without_builtins) +assert result == 5 +assert "__builtins__" in globals_without_builtins + +# Test with both globals and locals +assert eval("x + y", {"x": 10}, {"y": 20}) == 30 + +# Test that when globals is None and locals is provided, it still works +assert eval("x + y", None, {"x": 1, "y": 2}) == 3 + + +# Test code object with free variables +def make_closure(): + z = 10 + return compile("x + z", "<string>", "eval") + + +closure_code = make_closure() +try: + eval(closure_code, {"x": 5}) + assert False, "eval with code containing free variables should fail" +except NameError as e: + pass diff --git a/extra_tests/snippets/builtin_exceptions.py b/extra_tests/snippets/builtin_exceptions.py index 4bff9c00964..8879e130bc2 100644 --- a/extra_tests/snippets/builtin_exceptions.py +++ b/extra_tests/snippets/builtin_exceptions.py @@ -1,43 +1,50 @@ import builtins -import platform import pickle +import platform import sys + def exceptions_eq(e1, e2): return type(e1) is type(e2) and e1.args == e2.args + def round_trip_repr(e): return exceptions_eq(e, eval(repr(e))) + # KeyError empty_exc = KeyError() -assert str(empty_exc) == '' +assert str(empty_exc) == "" assert round_trip_repr(empty_exc) assert len(empty_exc.args) == 0 assert type(empty_exc.args) == tuple -exc = KeyError('message') +exc = KeyError("message") assert str(exc) == "'message'" assert round_trip_repr(exc) assert LookupError.__str__(exc) == "message" -exc = KeyError('message', 'another message') +exc = KeyError("message", "another message") assert str(exc) == "('message', 'another message')" assert round_trip_repr(exc) -assert exc.args[0] == 'message' -assert exc.args[1] == 'another message' +assert exc.args[0] == "message" +assert exc.args[1] == "another message" + class A: def __repr__(self): - return 'A()' + return "A()" + def __str__(self): - return 'str' + return "str" + def __eq__(self, other): return type(other) is A + exc = KeyError(A()) -assert str(exc) == 'A()' +assert str(exc) == "A()" assert round_trip_repr(exc) # ImportError / ModuleNotFoundError @@ -47,46 +54,53 @@ def __eq__(self, other): assert exc.msg is None assert exc.args == () -exc = ImportError('hello') +exc = ImportError("hello") assert exc.name is None assert exc.path is None -assert exc.msg == 'hello' -assert exc.args == ('hello',) +assert exc.msg == "hello" +assert exc.args == ("hello",) -exc = ImportError('hello', name='name', path='path') -assert exc.name == 'name' -assert exc.path == 'path' -assert exc.msg == 'hello' -assert exc.args == ('hello',) +exc = ImportError("hello", name="name", path="path") +assert exc.name == "name" +assert exc.path == "path" +assert exc.msg == "hello" +assert exc.args == ("hello",) class NewException(Exception): - - def __init__(self, value): - self.value = value + def __init__(self, value): + self.value = value try: - raise NewException("test") + raise NewException("test") except NewException as e: - assert e.value == "test" + assert e.value == "test" -exc = SyntaxError('msg', 1, 2, 3, 4, 5) -assert exc.msg == 'msg' +exc = SyntaxError("msg", 1, 2, 3, 4, 5) +assert exc.msg == "msg" assert exc.filename is None assert exc.lineno is None assert exc.offset is None assert exc.text is None +err = SyntaxError("bad bad", ("bad.py", 1, 2, "abcdefg")) +err.msg = "changed" +assert err.msg == "changed" +assert str(err) == "changed (bad.py, line 1)" +del err.msg +assert err.msg is None + # Regression to: # https://github.com/RustPython/RustPython/issues/2779 + class MyError(Exception): pass -e = MyError('message') +e = MyError("message") try: raise e from e @@ -97,23 +111,23 @@ class MyError(Exception): assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" try: - raise ValueError('test') from e + raise ValueError("test") from e except ValueError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) # ok, will print two excs assert isinstance(exc, ValueError) assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" # New case: # potential recursion on `__context__` field -e = MyError('message') +e = MyError("message") try: try: @@ -121,15 +135,15 @@ class MyError(Exception): except MyError as exc: raise e else: - assert False, 'exception not raised' + assert False, "exception not raised" except MyError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) assert exc.__cause__ is None assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" -e = MyError('message') +e = MyError("message") try: try: @@ -137,15 +151,15 @@ class MyError(Exception): except MyError as exc: raise exc else: - assert False, 'exception not raised' + assert False, "exception not raised" except MyError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) assert exc.__cause__ is None assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" -e = MyError('message') +e = MyError("message") try: try: @@ -153,15 +167,15 @@ class MyError(Exception): except MyError as exc: raise e from e else: - assert False, 'exception not raised' + assert False, "exception not raised" except MyError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" -e = MyError('message') +e = MyError("message") try: try: @@ -169,23 +183,25 @@ class MyError(Exception): except MyError as exc: raise exc from e else: - assert False, 'exception not raised' + assert False, "exception not raised" except MyError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" # New case: # two exception in a recursion loop + class SubError(MyError): pass -e = MyError('message') -d = SubError('sub') + +e = MyError("message") +d = SubError("sub") try: @@ -197,9 +213,9 @@ class SubError(MyError): assert exc.__cause__ is d assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" -e = MyError('message') +e = MyError("message") try: raise d from e @@ -210,55 +226,75 @@ class SubError(MyError): assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" # New case: # explicit `__context__` manipulation. -e = MyError('message') +e = MyError("message") e.__context__ = e try: raise e except MyError as exc: # It was a segmentation fault before, will print info to stdout: - if platform.python_implementation() == 'RustPython': - # For some reason `CPython` hangs on this code: - sys.excepthook(type(exc), exc, exc.__traceback__) - assert isinstance(exc, MyError) - assert exc.__cause__ is None - assert exc.__context__ is e + sys.excepthook(type(exc), exc, exc.__traceback__) + assert isinstance(exc, MyError) + assert exc.__cause__ is None + assert exc.__context__ is e # Regression to # https://github.com/RustPython/RustPython/issues/2771 # `BaseException` and `Exception`: -assert BaseException.__new__.__qualname__ == 'BaseException.__new__' -assert BaseException.__init__.__qualname__ == 'BaseException.__init__' +assert BaseException.__new__.__qualname__ == "BaseException.__new__" +assert BaseException.__init__.__qualname__ == "BaseException.__init__" assert BaseException().__dict__ == {} -assert Exception.__new__.__qualname__ == 'Exception.__new__', Exception.__new__.__qualname__ -assert Exception.__init__.__qualname__ == 'Exception.__init__', Exception.__init__.__qualname__ +# Exception inherits __init__ from BaseException +assert Exception.__new__.__qualname__ == "Exception.__new__", ( + Exception.__new__.__qualname__ +) +assert Exception.__init__.__qualname__ == "BaseException.__init__", ( + Exception.__init__.__qualname__ +) assert Exception().__dict__ == {} -# Extends `BaseException`, simple: -assert KeyboardInterrupt.__new__.__qualname__ == 'KeyboardInterrupt.__new__', KeyboardInterrupt.__new__.__qualname__ -assert KeyboardInterrupt.__init__.__qualname__ == 'KeyboardInterrupt.__init__' +# Extends `BaseException`, simple - inherits __init__ from BaseException: +assert KeyboardInterrupt.__new__.__qualname__ == "KeyboardInterrupt.__new__", ( + KeyboardInterrupt.__new__.__qualname__ +) +assert KeyboardInterrupt.__init__.__qualname__ == "BaseException.__init__" assert KeyboardInterrupt().__dict__ == {} -# Extends `Exception`, simple: -assert TypeError.__new__.__qualname__ == 'TypeError.__new__' -assert TypeError.__init__.__qualname__ == 'TypeError.__init__' +# Extends `BaseException`, complex - has its own __init__: +# SystemExit_init sets self.code based on args length +assert SystemExit.__init__.__qualname__ == "SystemExit.__init__" +assert SystemExit.__dict__.get("__init__") is not None, ( + "SystemExit must have its own __init__" +) +assert SystemExit.__init__ is not BaseException.__init__ +assert SystemExit().__dict__ == {} +# SystemExit.code behavior: +assert SystemExit().code is None +assert SystemExit(1).code == 1 +assert SystemExit(1, 2).code == (1, 2) +assert SystemExit(1, 2, 3).code == (1, 2, 3) + + +# Extends `Exception`, simple - inherits __init__ from BaseException: +assert TypeError.__new__.__qualname__ == "TypeError.__new__" +assert TypeError.__init__.__qualname__ == "BaseException.__init__" assert TypeError().__dict__ == {} # Extends `Exception`, complex: -assert OSError.__new__.__qualname__ == 'OSError.__new__' -assert OSError.__init__.__qualname__ == 'OSError.__init__' +assert OSError.__new__.__qualname__ == "OSError.__new__" +assert OSError.__init__.__qualname__ == "OSError.__init__" assert OSError().__dict__ == {} assert OSError.errno assert OSError.strerror @@ -299,7 +335,7 @@ class SubError(MyError): assert x.filename2 == None assert str(x) == "0" -w = OSError('foo') +w = OSError("foo") assert w.errno == None assert not sys.platform.startswith("win") or w.winerror == None assert w.strerror == None @@ -315,7 +351,7 @@ class SubError(MyError): assert x.filename2 == None assert str(x) == "foo" -w = OSError('a', 'b', 'c', 'd', 'e', 'f') +w = OSError("a", "b", "c", "d", "e", "f") assert w.errno == None assert not sys.platform.startswith("win") or w.winerror == None assert w.strerror == None @@ -332,12 +368,11 @@ class SubError(MyError): assert str(x) == "('a', 'b', 'c', 'd', 'e', 'f')" # Custom `__new__` and `__init__`: -assert ImportError.__init__.__qualname__ == 'ImportError.__init__' -assert ImportError(name='a').name == 'a' -assert ( - ModuleNotFoundError.__init__.__qualname__ == 'ModuleNotFoundError.__init__' -) -assert ModuleNotFoundError(name='a').name == 'a' +assert ImportError.__init__.__qualname__ == "ImportError.__init__" +assert ImportError(name="a").name == "a" +# ModuleNotFoundError inherits __init__ from ImportError via MRO (MiddlingExtendsException) +assert ModuleNotFoundError.__init__.__qualname__ == "ImportError.__init__" +assert ModuleNotFoundError(name="a").name == "a" # Check that all exceptions have string `__doc__`: @@ -346,3 +381,15 @@ class SubError(MyError): vars(builtins).values(), ): assert isinstance(exc.__doc__, str) + + +# except* handling should normalize non-group exceptions +try: + raise ValueError("x") +except* ValueError as err: + assert isinstance(err, ExceptionGroup) + assert len(err.exceptions) == 1 + assert isinstance(err.exceptions[0], ValueError) + assert err.exceptions[0].args == ("x",) +else: + assert False, "except* handler did not run" diff --git a/extra_tests/snippets/builtin_exec.py b/extra_tests/snippets/builtin_exec.py index 289f878cc0b..2eae90e91c5 100644 --- a/extra_tests/snippets/builtin_exec.py +++ b/extra_tests/snippets/builtin_exec.py @@ -3,11 +3,11 @@ d = {} exec("def square(x):\n return x * x\n", {}, d) -assert 16 == d['square'](4) +assert 16 == d["square"](4) -exec("assert 2 == x", {}, {'x': 2}) -exec("assert 2 == x", {'x': 2}, {}) -exec("assert 4 == x", {'x': 2}, {'x': 4}) +exec("assert 2 == x", {}, {"x": 2}) +exec("assert 2 == x", {"x": 2}, {}) +exec("assert 4 == x", {"x": 2}, {"x": 4}) exec("assert max(1, 2) == 2", {}, {}) @@ -16,9 +16,11 @@ # Local environment shouldn't replace global environment: exec("assert max(1, 5, square(5)) == 25", None, {}) + # Closures aren't available if local scope is replaced: def g(): seven = "seven" + def f(): try: exec("seven", None, {}) @@ -26,7 +28,10 @@ def f(): pass else: raise NameError("seven shouldn't be in scope") + f() + + g() try: @@ -37,16 +42,16 @@ def f(): raise TypeError("exec should fail unless globals is a dict or None") g = globals() -g['x'] = 2 -exec('x += 2') +g["x"] = 2 +exec("x += 2") assert x == 4 # noqa: F821 -assert g['x'] == x # noqa: F821 +assert g["x"] == x # noqa: F821 exec("del x") -assert 'x' not in g +assert "x" not in g -assert 'g' in globals() -assert 'g' in locals() +assert "g" in globals() +assert "g" in locals() exec("assert 'g' in globals()") exec("assert 'g' in locals()") exec("assert 'g' not in globals()", {}) @@ -54,13 +59,15 @@ def f(): del g + def f(): g = 1 - assert 'g' not in globals() - assert 'g' in locals() + assert "g" not in globals() + assert "g" in locals() exec("assert 'g' not in globals()") exec("assert 'g' in locals()") exec("assert 'g' not in globals()", {}) exec("assert 'g' not in locals()", {}) + f() diff --git a/extra_tests/snippets/builtin_exit.py b/extra_tests/snippets/builtin_exit.py index f6dc387322f..a61ddbc6d8b 100644 --- a/extra_tests/snippets/builtin_exit.py +++ b/extra_tests/snippets/builtin_exit.py @@ -36,4 +36,4 @@ sys.exit(1) with assert_raises(SystemExit): - sys.exit("AB") \ No newline at end of file + sys.exit("AB") diff --git a/extra_tests/snippets/builtin_filter.py b/extra_tests/snippets/builtin_filter.py index 25c00803c41..f778fd0cfe9 100644 --- a/extra_tests/snippets/builtin_filter.py +++ b/extra_tests/snippets/builtin_filter.py @@ -1,4 +1,4 @@ -assert list(filter(lambda x: ((x % 2) == 0), [0, 1, 2])) == [0, 2] +assert list(filter(lambda x: (x % 2) == 0, [0, 1, 2])) == [0, 2] # None implies identity assert list(filter(None, [0, 1, 2])) == [1, 2] @@ -18,7 +18,7 @@ def __iter__(self): return self -it = filter(lambda x: ((x % 2) == 0), Counter()) +it = filter(lambda x: (x % 2) == 0, Counter()) assert next(it) == 2 assert next(it) == 4 diff --git a/extra_tests/snippets/builtin_format.py b/extra_tests/snippets/builtin_format.py index 6a8e6077eed..a5edcc89523 100644 --- a/extra_tests/snippets/builtin_format.py +++ b/extra_tests/snippets/builtin_format.py @@ -2,37 +2,52 @@ assert format(5, "b") == "101" -assert_raises(TypeError, format, 2, 3, _msg='format called with number') +assert_raises(TypeError, format, 2, 3, _msg="format called with number") assert format({}) == "{}" -assert_raises(TypeError, format, {}, 'b', _msg='format_spec not empty for dict') +assert_raises(TypeError, format, {}, "b", _msg="format_spec not empty for dict") + class BadFormat: def __format__(self, spec): return 42 + + assert_raises(TypeError, format, BadFormat()) + def test_zero_padding(): i = 1 - assert f'{i:04d}' == '0001' + assert f"{i:04d}" == "0001" + test_zero_padding() -assert '{:,}'.format(100) == '100' -assert '{:,}'.format(1024) == '1,024' -assert '{:_}'.format(65536) == '65_536' -assert '{:_}'.format(4294967296) == '4_294_967_296' -assert f'{100:_}' == '100' -assert f'{1024:_}' == '1_024' -assert f'{65536:,}' == '65,536' -assert f'{4294967296:,}' == '4,294,967,296' -assert 'F' == "{0:{base}}".format(15, base="X") -assert f'{255:#X}' == "0XFF" +assert "{:,}".format(100) == "100" +assert "{:,}".format(1024) == "1,024" +assert "{:_}".format(65536) == "65_536" +assert "{:_}".format(4294967296) == "4_294_967_296" +assert f"{100:_}" == "100" +assert f"{1024:_}" == "1_024" +assert f"{65536:,}" == "65,536" +assert f"{4294967296:,}" == "4,294,967,296" +assert "F" == "{0:{base}}".format(15, base="X") +assert f"{255:#X}" == "0XFF" assert f"{65:c}" == "A" -assert f"{0x1f5a5:c}" == "🖥" -assert_raises(ValueError, "{:+c}".format, 1, _msg="Sign not allowed with integer format specifier 'c'") -assert_raises(ValueError, "{:#c}".format, 1, _msg="Alternate form (#) not allowed with integer format specifier 'c'") +assert f"{0x1F5A5:c}" == "🖥" +assert_raises( + ValueError, + "{:+c}".format, + 1, + _msg="Sign not allowed with integer format specifier 'c'", +) +assert_raises( + ValueError, + "{:#c}".format, + 1, + _msg="Alternate form (#) not allowed with integer format specifier 'c'", +) assert f"{256:#010x}" == "0x00000100" assert f"{256:0=#10x}" == "0x00000100" assert f"{256:0>#10x}" == "000000x100" @@ -66,14 +81,34 @@ def test_zero_padding(): assert f"{123.456:+011,}" == "+00,123.456" assert f"{1234:.3g}" == "1.23e+03" assert f"{1234567:.6G}" == "1.23457E+06" -assert f'{"🐍":4}' == "🐍 " -assert_raises(ValueError, "{:,o}".format, 1, _msg="ValueError: Cannot specify ',' with 'o'.") -assert_raises(ValueError, "{:_n}".format, 1, _msg="ValueError: Cannot specify '_' with 'n'.") -assert_raises(ValueError, "{:,o}".format, 1.0, _msg="ValueError: Cannot specify ',' with 'o'.") -assert_raises(ValueError, "{:_n}".format, 1.0, _msg="ValueError: Cannot specify '_' with 'n'.") -assert_raises(ValueError, "{:,}".format, "abc", _msg="ValueError: Cannot specify ',' with 's'.") -assert_raises(ValueError, "{:,x}".format, "abc", _msg="ValueError: Cannot specify ',' with 'x'.") -assert_raises(OverflowError, "{:c}".format, 0x110000, _msg="OverflowError: %c arg not in range(0x110000)") +assert f"{1234:10}" == " 1234" +assert f"{1234:10,}" == " 1,234" +assert f"{1234:010,}" == "00,001,234" +assert f"{'🐍':4}" == "🐍 " +assert_raises( + ValueError, "{:,o}".format, 1, _msg="ValueError: Cannot specify ',' with 'o'." +) +assert_raises( + ValueError, "{:_n}".format, 1, _msg="ValueError: Cannot specify '_' with 'n'." +) +assert_raises( + ValueError, "{:,o}".format, 1.0, _msg="ValueError: Cannot specify ',' with 'o'." +) +assert_raises( + ValueError, "{:_n}".format, 1.0, _msg="ValueError: Cannot specify '_' with 'n'." +) +assert_raises( + ValueError, "{:,}".format, "abc", _msg="ValueError: Cannot specify ',' with 's'." +) +assert_raises( + ValueError, "{:,x}".format, "abc", _msg="ValueError: Cannot specify ',' with 'x'." +) +assert_raises( + OverflowError, + "{:c}".format, + 0x110000, + _msg="OverflowError: %c arg not in range(0x110000)", +) assert f"{3:f}" == "3.000000" assert f"{3.1415:.0f}" == "3" assert f"{3.1415:.1f}" == "3.1" @@ -115,14 +150,14 @@ def test_zero_padding(): assert f"{3.1415:#.4e}" == "3.1415e+00" assert f"{3.1415:#.5e}" == "3.14150e+00" assert f"{3.1415:#.5E}" == "3.14150E+00" -assert f"{3.1415:.0%}" == '314%' -assert f"{3.1415:.1%}" == '314.2%' -assert f"{3.1415:.2%}" == '314.15%' -assert f"{3.1415:.3%}" == '314.150%' -assert f"{3.1415:#.0%}" == '314.%' -assert f"{3.1415:#.1%}" == '314.2%' -assert f"{3.1415:#.2%}" == '314.15%' -assert f"{3.1415:#.3%}" == '314.150%' +assert f"{3.1415:.0%}" == "314%" +assert f"{3.1415:.1%}" == "314.2%" +assert f"{3.1415:.2%}" == "314.15%" +assert f"{3.1415:.3%}" == "314.150%" +assert f"{3.1415:#.0%}" == "314.%" +assert f"{3.1415:#.1%}" == "314.2%" +assert f"{3.1415:#.2%}" == "314.15%" +assert f"{3.1415:#.3%}" == "314.150%" assert f"{3.1415:.0}" == "3e+00" assert f"{3.1415:.1}" == "3e+00" assert f"{3.1415:.2}" == "3.1" @@ -133,9 +168,34 @@ def test_zero_padding(): assert f"{3.1415:#.2}" == "3.1" assert f"{3.1415:#.3}" == "3.14" assert f"{3.1415:#.4}" == "3.142" +assert f"{1234.5:10}" == " 1234.5" +assert f"{1234.5:10,}" == " 1,234.5" +assert f"{1234.5:010,}" == "0,001,234.5" +assert f"{12.34 + 5.6j}" == "(12.34+5.6j)" +assert f"{12.34 - 5.6j: }" == "( 12.34-5.6j)" +assert f"{12.34 + 5.6j:20}" == " (12.34+5.6j)" +assert f"{12.34 + 5.6j:<20}" == "(12.34+5.6j) " +assert f"{-12.34 + 5.6j:^20}" == " (-12.34+5.6j) " +assert f"{12.34 + 5.6j:^+20}" == " (+12.34+5.6j) " +assert f"{12.34 + 5.6j:_^+20}" == "___(+12.34+5.6j)____" +assert f"{-12.34 + 5.6j:f}" == "-12.340000+5.600000j" +assert f"{12.34 + 5.6j:.3f}" == "12.340+5.600j" +assert f"{12.34 + 5.6j:<30.8f}" == "12.34000000+5.60000000j " +assert f"{12.34 + 5.6j:g}" == "12.34+5.6j" +assert f"{12.34 + 5.6j:e}" == "1.234000e+01+5.600000e+00j" +assert f"{12.34 + 5.6j:E}" == "1.234000E+01+5.600000E+00j" +assert f"{12.34 + 5.6j:^30E}" == " 1.234000E+01+5.600000E+00j " +assert f"{12345.6 + 7890.1j:,}" == "(12,345.6+7,890.1j)" +assert f"{12345.6 + 7890.1j:_.3f}" == "12_345.600+7_890.100j" +assert f"{12345.6 + 7890.1j:>+30,f}" == " +12,345.600000+7,890.100000j" +assert f"{123456:,g}" == "123,456" +assert f"{123456:,G}" == "123,456" +assert f"{123456:,e}" == "1.234560e+05" +assert f"{123456:,E}" == "1.234560E+05" +assert f"{123456:,%}" == "12,345,600.000000%" # test issue 4558 x = 123456789012345678901234567890 for i in range(0, 30): - format(x, ',') + format(x, ",") x = x // 10 diff --git a/extra_tests/snippets/builtin_hash.py b/extra_tests/snippets/builtin_hash.py index bd98199db97..9b2c8388790 100644 --- a/extra_tests/snippets/builtin_hash.py +++ b/extra_tests/snippets/builtin_hash.py @@ -1,4 +1,3 @@ - from testutils import assert_raises @@ -13,6 +12,14 @@ class A: assert type(hash(1.1)) is int assert type(hash("")) is int + +class Evil: + def __hash__(self): + return 1 << 63 + + +assert hash(Evil()) == 4 + with assert_raises(TypeError): hash({}) diff --git a/extra_tests/snippets/builtin_hex.py b/extra_tests/snippets/builtin_hex.py index fac5e09c224..740817bc42f 100644 --- a/extra_tests/snippets/builtin_hex.py +++ b/extra_tests/snippets/builtin_hex.py @@ -1,6 +1,6 @@ from testutils import assert_raises -assert hex(16) == '0x10' -assert hex(-16) == '-0x10' +assert hex(16) == "0x10" +assert hex(-16) == "-0x10" -assert_raises(TypeError, hex, {}, _msg='ord() called with dict') +assert_raises(TypeError, hex, {}, _msg="ord() called with dict") diff --git a/extra_tests/snippets/builtin_int.py b/extra_tests/snippets/builtin_int.py index bc3cd5fd996..aab24cbb4cc 100644 --- a/extra_tests/snippets/builtin_int.py +++ b/extra_tests/snippets/builtin_int.py @@ -318,8 +318,9 @@ def __int__(self): assert isinstance((1).__round__(0), int) assert (0).__round__(0) == 0 assert (1).__round__(0) == 1 -assert_raises(TypeError, lambda: (0).__round__(None)) -assert_raises(TypeError, lambda: (1).__round__(None)) +# Python 3.14+: __round__(None) is now allowed, same as __round__() +assert (0).__round__(None) == 0 +assert (1).__round__(None) == 1 assert_raises(TypeError, lambda: (0).__round__(0.0)) assert_raises(TypeError, lambda: (1).__round__(0.0)) diff --git a/extra_tests/snippets/builtin_isinstance.py b/extra_tests/snippets/builtin_isinstance.py index c02f331d258..866c83f7cc7 100644 --- a/extra_tests/snippets/builtin_isinstance.py +++ b/extra_tests/snippets/builtin_isinstance.py @@ -1,4 +1,3 @@ - class Regular: pass @@ -41,14 +40,17 @@ class AlwaysInstanceOf(metaclass=MCAlwaysInstanceOf): assert isinstance(Regular(), AlwaysInstanceOf) assert isinstance(1, AlwaysInstanceOf) + class GenericInstance: def __instancecheck__(self, _): return True + assert isinstance(Regular(), GenericInstance()) assert isinstance([], GenericInstance()) assert isinstance(1, GenericInstance()) + class MCReturnInt(type): def __instancecheck__(self, instance): return 3 @@ -60,4 +62,13 @@ class ReturnInt(metaclass=MCReturnInt): assert isinstance("a", ReturnInt) is True -assert isinstance(1, ((int, float,), str)) +assert isinstance( + 1, + ( + ( + int, + float, + ), + str, + ), +) diff --git a/extra_tests/snippets/builtin_issubclass.py b/extra_tests/snippets/builtin_issubclass.py index 7f1d87abb1d..7c047515d46 100644 --- a/extra_tests/snippets/builtin_issubclass.py +++ b/extra_tests/snippets/builtin_issubclass.py @@ -1,4 +1,3 @@ - class A: pass @@ -49,14 +48,17 @@ class InheritedAlwaysSubClass(AlwaysSubClass): assert issubclass(InheritedAlwaysSubClass, AlwaysSubClass) assert issubclass(AlwaysSubClass, InheritedAlwaysSubClass) + class GenericInstance: def __subclasscheck__(self, _): return True + assert issubclass(A, GenericInstance()) assert issubclass(list, GenericInstance()) assert issubclass([], GenericInstance()) + class MCAVirtualSubClass(type): def __subclasscheck__(self, subclass): return subclass is A diff --git a/extra_tests/snippets/builtin_len.py b/extra_tests/snippets/builtin_len.py index 4872f20c427..4190e316980 100644 --- a/extra_tests/snippets/builtin_len.py +++ b/extra_tests/snippets/builtin_len.py @@ -1,2 +1,2 @@ -assert 3 == len([1,2,3]) -assert 2 == len((1,2)) +assert 3 == len([1, 2, 3]) +assert 2 == len((1, 2)) diff --git a/extra_tests/snippets/builtin_list.py b/extra_tests/snippets/builtin_list.py index 3261da100db..d4afbffa1cb 100644 --- a/extra_tests/snippets/builtin_list.py +++ b/extra_tests/snippets/builtin_list.py @@ -12,38 +12,85 @@ assert y == [2, 1, 2, 3, 1, 2, 3] a = [] -a.extend((1,2,3,4)) +a.extend((1, 2, 3, 4)) assert a == [1, 2, 3, 4] -a.extend('abcdefg') -assert a == [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f', 'g'] +a.extend("abcdefg") +assert a == [1, 2, 3, 4, "a", "b", "c", "d", "e", "f", "g"] a.extend(range(10)) -assert a == [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert a == [ + 1, + 2, + 3, + 4, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, +] a = [] -a.extend({1,2,3,4}) +a.extend({1, 2, 3, 4}) assert a == [1, 2, 3, 4] -a.extend({'a': 1, 'b': 2, 'z': 51}) -assert a == [1, 2, 3, 4, 'a', 'b', 'z'] +a.extend({"a": 1, "b": 2, "z": 51}) +assert a == [1, 2, 3, 4, "a", "b", "z"] + class Iter: def __iter__(self): yield 12 yield 28 + a.extend(Iter()) -assert a == [1, 2, 3, 4, 'a', 'b', 'z', 12, 28] +assert a == [1, 2, 3, 4, "a", "b", "z", 12, 28] + +a.extend(bytes(b"hello world")) +assert a == [ + 1, + 2, + 3, + 4, + "a", + "b", + "z", + 12, + 28, + 104, + 101, + 108, + 108, + 111, + 32, + 119, + 111, + 114, + 108, + 100, +] -a.extend(bytes(b'hello world')) -assert a == [1, 2, 3, 4, 'a', 'b', 'z', 12, 28, 104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100] class Next: def __next__(self): yield 12 yield 28 + assert_raises(TypeError, lambda: [].extend(3)) assert_raises(TypeError, lambda: [].extend(slice(0, 10, 1))) @@ -56,12 +103,12 @@ def __next__(self): assert x == [1, 2, 3] * 2 # index() -assert ['a', 'b', 'c'].index('b') == 1 +assert ["a", "b", "c"].index("b") == 1 assert [5, 6, 7].index(7) == 2 -assert_raises(ValueError, lambda: ['a', 'b', 'c'].index('z')) +assert_raises(ValueError, lambda: ["a", "b", "c"].index("z")) -x = [[1,0,-3], 'a', 1] -y = [[3,2,1], 'z', 2] +x = [[1, 0, -3], "a", 1] +y = [[3, 2, 1], "z", 2] assert x < y, "list __lt__ failed" x = [5, 13, 31] @@ -73,9 +120,12 @@ def __next__(self): assert x.pop() == 2 assert x == [0, 1] + def test_pop(lst, idx, value, new_lst): assert lst.pop(idx) == value assert lst == new_lst + + test_pop([0, 1, 2], -1, 2, [0, 1]) test_pop([0, 1, 2], 0, 0, [1, 2]) test_pop([0, 1, 2], 1, 1, [0, 2]) @@ -91,23 +141,23 @@ def test_pop(lst, idx, value, new_lst): assert repr(recursive) == "[[...]]" # insert() -x = ['a', 'b', 'c'] -x.insert(0, 'z') # insert is in-place, no return value -assert x == ['z', 'a', 'b', 'c'] +x = ["a", "b", "c"] +x.insert(0, "z") # insert is in-place, no return value +assert x == ["z", "a", "b", "c"] -x = ['a', 'b', 'c'] -x.insert(100, 'z') -assert x == ['a', 'b', 'c', 'z'] +x = ["a", "b", "c"] +x.insert(100, "z") +assert x == ["a", "b", "c", "z"] -x = ['a', 'b', 'c'] -x.insert(-1, 'z') -assert x == ['a', 'b', 'z', 'c'] +x = ["a", "b", "c"] +x.insert(-1, "z") +assert x == ["a", "b", "z", "c"] -x = ['a', 'b', 'c'] -x.insert(-100, 'z') -assert x == ['z', 'a', 'b', 'c'] +x = ["a", "b", "c"] +x.insert(-100, "z") +assert x == ["z", "a", "b", "c"] -assert_raises(OverflowError, lambda: x.insert(100000000000000000000, 'z')) +assert_raises(OverflowError, lambda: x.insert(100000000000000000000, "z")) x = [[], 2, {}] y = x.copy() @@ -123,7 +173,7 @@ def test_pop(lst, idx, value, new_lst): assert len(a) == 2 assert not 1 in a -assert_raises(ValueError, lambda: a.remove(10), _msg='Remove not exist element') +assert_raises(ValueError, lambda: a.remove(10), _msg="Remove not exist element") foo = bar = [1] foo += [2] @@ -138,10 +188,12 @@ def test_pop(lst, idx, value, new_lst): x.remove(x) assert x not in x + class Foo(object): def __eq__(self, x): return False + foo = Foo() foo1 = Foo() x = [1, foo, 2, foo, []] @@ -173,17 +225,17 @@ def __eq__(self, x): assert [foo] == [foo] for size in [1, 2, 3, 4, 5, 8, 10, 100, 1000]: - lst = list(range(size)) - orig = lst[:] - lst.sort() - assert lst == orig - assert sorted(lst) == orig - assert_raises(ZeroDivisionError, sorted, lst, key=lambda x: 1/x) - lst.reverse() - assert sorted(lst) == orig - assert sorted(lst, reverse=True) == lst - assert sorted(lst, key=lambda x: -x) == lst - assert sorted(lst, key=lambda x: -x, reverse=True) == orig + lst = list(range(size)) + orig = lst[:] + lst.sort() + assert lst == orig + assert sorted(lst) == orig + assert_raises(ZeroDivisionError, sorted, lst, key=lambda x: 1 / x) + lst.reverse() + assert sorted(lst) == orig + assert sorted(lst, reverse=True) == lst + assert sorted(lst, key=lambda x: -x) == lst + assert sorted(lst, key=lambda x: -x, reverse=True) == orig assert sorted([(1, 2, 3), (0, 3, 6)]) == [(0, 3, 6), (1, 2, 3)] assert sorted([(1, 2, 3), (0, 3, 6)], key=lambda x: x[0]) == [(0, 3, 6), (1, 2, 3)] @@ -191,34 +243,101 @@ def __eq__(self, x): assert sorted([(1, 2), (), (5,)], key=len) == [(), (5,), (1, 2)] lst = [3, 1, 5, 2, 4] + + class C: - def __init__(self, x): self.x = x - def __lt__(self, other): return self.x < other.x + def __init__(self, x): + self.x = x + + def __lt__(self, other): + return self.x < other.x + + lst.sort(key=C) assert lst == [1, 2, 3, 4, 5] lst = [3, 1, 5, 2, 4] + + class C: - def __init__(self, x): self.x = x - def __gt__(self, other): return self.x > other.x + def __init__(self, x): + self.x = x + + def __gt__(self, other): + return self.x > other.x + + lst.sort(key=C) assert lst == [1, 2, 3, 4, 5] + +# Test that sorted() uses __lt__ (not __gt__) for comparisons. +# Track which comparison method is actually called during sort. +class TrackComparison: + lt_calls = 0 + gt_calls = 0 + + def __init__(self, value): + self.value = value + + def __lt__(self, other): + TrackComparison.lt_calls += 1 + return self.value < other.value + + def __gt__(self, other): + TrackComparison.gt_calls += 1 + return self.value > other.value + + +# Reset and test sorted() +TrackComparison.lt_calls = 0 +TrackComparison.gt_calls = 0 +items = [TrackComparison(3), TrackComparison(1), TrackComparison(2)] +sorted(items) +assert TrackComparison.lt_calls > 0, "sorted() should call __lt__" +assert TrackComparison.gt_calls == 0, ( + f"sorted() should not call __gt__, but it was called {TrackComparison.gt_calls} times" +) + +# Reset and test list.sort() +TrackComparison.lt_calls = 0 +TrackComparison.gt_calls = 0 +items = [TrackComparison(3), TrackComparison(1), TrackComparison(2)] +items.sort() +assert TrackComparison.lt_calls > 0, "list.sort() should call __lt__" +assert TrackComparison.gt_calls == 0, ( + f"list.sort() should not call __gt__, but it was called {TrackComparison.gt_calls} times" +) + +# Reset and test sorted(reverse=True) - should still use __lt__, not __gt__ +TrackComparison.lt_calls = 0 +TrackComparison.gt_calls = 0 +items = [TrackComparison(3), TrackComparison(1), TrackComparison(2)] +sorted(items, reverse=True) +assert TrackComparison.lt_calls > 0, "sorted(reverse=True) should call __lt__" +assert TrackComparison.gt_calls == 0, ( + f"sorted(reverse=True) should not call __gt__, but it was called {TrackComparison.gt_calls} times" +) + lst = [5, 1, 2, 3, 4] + + def f(x): lst.append(1) return x -assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort" + + +assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort" assert lst == [1, 2, 3, 4, 5] # __delitem__ -x = ['a', 'b', 'c'] +x = ["a", "b", "c"] del x[0] -assert x == ['b', 'c'] +assert x == ["b", "c"] -x = ['a', 'b', 'c'] +x = ["a", "b", "c"] del x[-1] -assert x == ['a', 'b'] +assert x == ["a", "b"] x = y = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] del x[2:14:3] @@ -232,39 +351,48 @@ def f(x): x = list(range(12)) del x[10:2:-2] -assert x == [0,1,2,3,5,7,9,11] +assert x == [0, 1, 2, 3, 5, 7, 9, 11] + def bad_del_1(): - del ['a', 'b']['a'] + del ["a", "b"]["a"] + + assert_raises(TypeError, bad_del_1) + def bad_del_2(): - del ['a', 'b'][2] + del ["a", "b"][2] + + assert_raises(IndexError, bad_del_2) # __setitem__ # simple index x = [1, 2, 3, 4, 5] -x[0] = 'a' -assert x == ['a', 2, 3, 4, 5] -x[-1] = 'b' -assert x == ['a', 2, 3, 4, 'b'] -# make sure refrences are assigned correctly +x[0] = "a" +assert x == ["a", 2, 3, 4, 5] +x[-1] = "b" +assert x == ["a", 2, 3, 4, "b"] +# make sure references are assigned correctly y = [] x[1] = y y.append(100) assert x[1] == y assert x[1] == [100] -#index bounds + +# index bounds def set_index_out_of_bounds_high(): - x = [0, 1, 2, 3, 4] - x[5] = 'a' + x = [0, 1, 2, 3, 4] + x[5] = "a" + def set_index_out_of_bounds_low(): - x = [0, 1, 2, 3, 4] - x[-6] = 'a' + x = [0, 1, 2, 3, 4] + x[-6] = "a" + assert_raises(IndexError, set_index_out_of_bounds_high) assert_raises(IndexError, set_index_out_of_bounds_low) @@ -275,20 +403,20 @@ def set_index_out_of_bounds_low(): y = a[:] assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # replace whole list -x[:] = ['a', 'b', 'c'] -y[::1] = ['a', 'b', 'c'] -assert x == ['a', 'b', 'c'] +x[:] = ["a", "b", "c"] +y[::1] = ["a", "b", "c"] +assert x == ["a", "b", "c"] assert x == y # splice list start x = a[:] y = a[:] z = a[:] zz = a[:] -x[:1] = ['a', 'b', 'c'] -y[0:1] = ['a', 'b', 'c'] -z[:1:1] = ['a', 'b', 'c'] -zz[0:1:1] = ['a', 'b', 'c'] -assert x == ['a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x[:1] = ["a", "b", "c"] +y[0:1] = ["a", "b", "c"] +z[:1:1] = ["a", "b", "c"] +zz[0:1:1] = ["a", "b", "c"] +assert x == ["a", "b", "c", 1, 2, 3, 4, 5, 6, 7, 8, 9] assert x == y assert x == z assert x == zz @@ -297,11 +425,11 @@ def set_index_out_of_bounds_low(): y = a[:] z = a[:] zz = a[:] -x[5:] = ['a', 'b', 'c'] -y[5::1] = ['a', 'b', 'c'] -z[5:10] = ['a', 'b', 'c'] -zz[5:10:1] = ['a', 'b', 'c'] -assert x == [0, 1, 2, 3, 4, 'a', 'b', 'c'] +x[5:] = ["a", "b", "c"] +y[5::1] = ["a", "b", "c"] +z[5:10] = ["a", "b", "c"] +zz[5:10:1] = ["a", "b", "c"] +assert x == [0, 1, 2, 3, 4, "a", "b", "c"] assert x == y assert x == z assert x == zz @@ -310,11 +438,11 @@ def set_index_out_of_bounds_low(): y = a[:] z = a[:] zz = a[:] -x[1:1] = ['a', 'b', 'c'] -y[1:0] = ['a', 'b', 'c'] -z[1:1:1] = ['a', 'b', 'c'] -zz[1:0:1] = ['a', 'b', 'c'] -assert x == [0, 'a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x[1:1] = ["a", "b", "c"] +y[1:0] = ["a", "b", "c"] +z[1:1:1] = ["a", "b", "c"] +zz[1:0:1] = ["a", "b", "c"] +assert x == [0, "a", "b", "c", 1, 2, 3, 4, 5, 6, 7, 8, 9] assert x == y assert x == z assert x == zz @@ -323,24 +451,24 @@ def set_index_out_of_bounds_low(): y = a[:] z = a[:] zz = a[:] -x[-1:-1] = ['a', 'b', 'c'] -y[-1:9] = ['a', 'b', 'c'] -z[-1:-1:1] = ['a', 'b', 'c'] -zz[-1:9:1] = ['a', 'b', 'c'] -assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 'a', 'b', 'c', 9] +x[-1:-1] = ["a", "b", "c"] +y[-1:9] = ["a", "b", "c"] +z[-1:-1:1] = ["a", "b", "c"] +zz[-1:9:1] = ["a", "b", "c"] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, "a", "b", "c", 9] assert x == y assert x == z assert x == zz # splice mid x = a[:] y = a[:] -x[3:5] = ['a', 'b', 'c', 'd', 'e'] -y[3:5:1] = ['a', 'b', 'c', 'd', 'e'] -assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 5, 6, 7, 8, 9] +x[3:5] = ["a", "b", "c", "d", "e"] +y[3:5:1] = ["a", "b", "c", "d", "e"] +assert x == [0, 1, 2, "a", "b", "c", "d", "e", 5, 6, 7, 8, 9] assert x == y x = a[:] -x[3:5] = ['a'] -assert x == [0, 1, 2, 'a', 5, 6, 7, 8, 9] +x[3:5] = ["a"] +assert x == [0, 1, 2, "a", 5, 6, 7, 8, 9] # assign empty to non stepped empty slice does nothing x = a[:] y = a[:] @@ -355,88 +483,97 @@ def set_index_out_of_bounds_low(): y[2:8:1] = [] assert x == [0, 1, 8, 9] assert x == y -# make sure refrences are assigned correctly +# make sure references are assigned correctly yy = [] x = a[:] y = a[:] -x[3:5] = ['a', 'b', 'c', 'd', yy] -y[3:5:1] = ['a', 'b', 'c', 'd', yy] -assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [], 5, 6, 7, 8, 9] +x[3:5] = ["a", "b", "c", "d", yy] +y[3:5:1] = ["a", "b", "c", "d", yy] +assert x == [0, 1, 2, "a", "b", "c", "d", [], 5, 6, 7, 8, 9] assert x == y yy.append(100) -assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [100], 5, 6, 7, 8, 9] +assert x == [0, 1, 2, "a", "b", "c", "d", [100], 5, 6, 7, 8, 9] assert x == y assert x[7] == yy assert x[7] == [100] assert y[7] == yy assert y[7] == [100] + # no zero step def no_zero_step_set(): - x = [1, 2, 3, 4, 5] - x[0:4:0] = [11, 12, 13, 14, 15] + x = [1, 2, 3, 4, 5] + x[0:4:0] = [11, 12, 13, 14, 15] + + assert_raises(ValueError, no_zero_step_set) # stepped slice index # forward slice x = a[:] -x[2:8:2] = ['a', 'b', 'c'] -assert x == [0, 1, 'a', 3, 'b', 5, 'c', 7, 8, 9] +x[2:8:2] = ["a", "b", "c"] +assert x == [0, 1, "a", 3, "b", 5, "c", 7, 8, 9] x = a[:] y = a[:] z = a[:] zz = a[:] -c = ['a', 'b', 'c', 'd', 'e'] +c = ["a", "b", "c", "d", "e"] x[::2] = c y[-10::2] = c z[0:10:2] = c -zz[-13:13:2] = c # slice indexes will be truncated to bounds -assert x == ['a', 1, 'b', 3, 'c', 5, 'd', 7, 'e', 9] +zz[-13:13:2] = c # slice indexes will be truncated to bounds +assert x == ["a", 1, "b", 3, "c", 5, "d", 7, "e", 9] assert x == y assert x == z assert x == zz # backward slice x = a[:] -x[8:2:-2] = ['a', 'b', 'c'] -assert x == [0, 1, 2, 3, 'c', 5, 'b', 7, 'a', 9] +x[8:2:-2] = ["a", "b", "c"] +assert x == [0, 1, 2, 3, "c", 5, "b", 7, "a", 9] x = a[:] y = a[:] z = a[:] zz = a[:] -c = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'] +c = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] x[::-1] = c y[9:-11:-1] = c z[9::-1] = c -zz[11:-13:-1] = c # slice indexes will be truncated to bounds -assert x == ['j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a'] +zz[11:-13:-1] = c # slice indexes will be truncated to bounds +assert x == ["j", "i", "h", "g", "f", "e", "d", "c", "b", "a"] assert x == y assert x == z assert x == zz # step size bigger than len x = a[:] -x[::200] = ['a'] -assert x == ['a', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x[::200] = ["a"] +assert x == ["a", 1, 2, 3, 4, 5, 6, 7, 8, 9] x = a[:] -x[5::200] = ['a'] -assert x == [0, 1, 2, 3, 4, 'a', 6, 7, 8, 9] +x[5::200] = ["a"] +assert x == [0, 1, 2, 3, 4, "a", 6, 7, 8, 9] + # bad stepped slices def stepped_slice_assign_too_big(): - x = [0, 1, 2, 3, 4] - x[::2] = ['a', 'b', 'c', 'd'] + x = [0, 1, 2, 3, 4] + x[::2] = ["a", "b", "c", "d"] + assert_raises(ValueError, stepped_slice_assign_too_big) + def stepped_slice_assign_too_small(): - x = [0, 1, 2, 3, 4] - x[::2] = ['a', 'b'] + x = [0, 1, 2, 3, 4] + x[::2] = ["a", "b"] + assert_raises(ValueError, stepped_slice_assign_too_small) + # must assign iter t0 slice def must_assign_iter_to_slice(): - x = [0, 1, 2, 3, 4] - x[::2] = 42 + x = [0, 1, 2, 3, 4] + x[::2] = 42 + assert_raises(TypeError, must_assign_iter_to_slice) @@ -446,74 +583,87 @@ def must_assign_iter_to_slice(): # string x = a[:] x[3:8] = "abcdefghi" -assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 8, 9] +assert x == [0, 1, 2, "a", "b", "c", "d", "e", "f", "g", "h", "i", 8, 9] # tuple x = a[:] x[3:8] = (11, 12, 13, 14, 15) assert x == [0, 1, 2, 11, 12, 13, 14, 15, 8, 9] + # class # __next__ class CIterNext: - def __init__(self, sec=(1, 2, 3)): - self.sec = sec - self.index = 0 - def __iter__(self): - return self - def __next__(self): - if self.index >= len(self.sec): - raise StopIteration - v = self.sec[self.index] - self.index += 1 - return v + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + self.index = 0 + + def __iter__(self): + return self + + def __next__(self): + if self.index >= len(self.sec): + raise StopIteration + v = self.sec[self.index] + self.index += 1 + return v + x = list(range(10)) x[3:8] = CIterNext() assert x == [0, 1, 2, 1, 2, 3, 8, 9] + # __iter__ yield class CIter: - def __init__(self, sec=(1, 2, 3)): - self.sec = sec - def __iter__(self): - for n in self.sec: - yield n + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + + def __iter__(self): + for n in self.sec: + yield n + x = list(range(10)) x[3:8] = CIter() assert x == [0, 1, 2, 1, 2, 3, 8, 9] + # __getitem but no __iter__ sequence class CGetItem: - def __init__(self, sec=(1, 2, 3)): - self.sec = sec - def __getitem__(self, sub): - return self.sec[sub] + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + + def __getitem__(self, sub): + return self.sec[sub] + x = list(range(10)) x[3:8] = CGetItem() assert x == [0, 1, 2, 1, 2, 3, 8, 9] + # iter raises error class CIterError: - def __iter__(self): - for i in range(10): - if i > 5: - raise RuntimeError - yield i + def __iter__(self): + for i in range(10): + if i > 5: + raise RuntimeError + yield i + def bad_iter_assign(): - x = list(range(10)) - x[3:8] = CIterError() + x = list(range(10)) + x[3:8] = CIterError() + assert_raises(RuntimeError, bad_iter_assign) # slice assign when step or stop is -1 a = list(range(10)) x = a[:] -x[-1:-5:-1] = ['a', 'b', 'c', 'd'] -assert x == [0, 1, 2, 3, 4, 5, 'd', 'c', 'b', 'a'] +x[-1:-5:-1] = ["a", "b", "c", "d"] +assert x == [0, 1, 2, 3, 4, 5, "d", "c", "b", "a"] x = a[:] x[-5:-1:-1] = [] assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] @@ -557,17 +707,17 @@ def bad_iter_assign(): assert not [0, 0] > [0, 0] assert not [0, 0] < [0, 0] -assert not [float('nan'), float('nan')] <= [float('nan'), 1] -assert not [float('nan'), float('nan')] <= [float('nan'), float('nan')] -assert not [float('nan'), float('nan')] >= [float('nan'), float('nan')] -assert not [float('nan'), float('nan')] < [float('nan'), float('nan')] -assert not [float('nan'), float('nan')] > [float('nan'), float('nan')] +assert not [float("nan"), float("nan")] <= [float("nan"), 1] +assert not [float("nan"), float("nan")] <= [float("nan"), float("nan")] +assert not [float("nan"), float("nan")] >= [float("nan"), float("nan")] +assert not [float("nan"), float("nan")] < [float("nan"), float("nan")] +assert not [float("nan"), float("nan")] > [float("nan"), float("nan")] -assert [float('inf'), float('inf')] >= [float('inf'), 1] -assert [float('inf'), float('inf')] <= [float('inf'), float('inf')] -assert [float('inf'), float('inf')] >= [float('inf'), float('inf')] -assert not [float('inf'), float('inf')] < [float('inf'), float('inf')] -assert not [float('inf'), float('inf')] > [float('inf'), float('inf')] +assert [float("inf"), float("inf")] >= [float("inf"), 1] +assert [float("inf"), float("inf")] <= [float("inf"), float("inf")] +assert [float("inf"), float("inf")] >= [float("inf"), float("inf")] +assert not [float("inf"), float("inf")] < [float("inf"), float("inf")] +assert not [float("inf"), float("inf")] > [float("inf"), float("inf")] # list __iadd__ a = [] @@ -575,73 +725,147 @@ def bad_iter_assign(): assert a == [1, 2, 3] a = [] -a += (1,2,3,4) +a += (1, 2, 3, 4) assert a == [1, 2, 3, 4] -a += 'abcdefg' -assert a == [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f', 'g'] +a += "abcdefg" +assert a == [1, 2, 3, 4, "a", "b", "c", "d", "e", "f", "g"] a += range(10) -assert a == [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert a == [ + 1, + 2, + 3, + 4, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, +] a = [] -a += {1,2,3,4} +a += {1, 2, 3, 4} assert a == [1, 2, 3, 4] -a += {'a': 1, 'b': 2, 'z': 51} -assert a == [1, 2, 3, 4, 'a', 'b', 'z'] +a += {"a": 1, "b": 2, "z": 51} +assert a == [1, 2, 3, 4, "a", "b", "z"] + class Iter: def __iter__(self): yield 12 yield 28 + a += Iter() -assert a == [1, 2, 3, 4, 'a', 'b', 'z', 12, 28] +assert a == [1, 2, 3, 4, "a", "b", "z", 12, 28] + +a += bytes(b"hello world") +assert a == [ + 1, + 2, + 3, + 4, + "a", + "b", + "z", + 12, + 28, + 104, + 101, + 108, + 108, + 111, + 32, + 119, + 111, + 114, + 108, + 100, +] -a += bytes(b'hello world') -assert a == [1, 2, 3, 4, 'a', 'b', 'z', 12, 28, 104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100] class Next: def __next__(self): yield 12 yield 28 + def iadd_int(): a = [] a += 3 + def iadd_slice(): a = [] a += slice(0, 10, 1) + assert_raises(TypeError, iadd_int) assert_raises(TypeError, iadd_slice) -it = iter([1,2,3,4]) +it = iter([1, 2, 3, 4]) assert it.__length_hint__() == 4 assert next(it) == 1 assert it.__length_hint__() == 3 -assert list(it) == [2,3,4] +assert list(it) == [2, 3, 4] assert it.__length_hint__() == 0 -it = reversed([1,2,3,4]) +it = reversed([1, 2, 3, 4]) assert it.__length_hint__() == 4 assert next(it) == 4 assert it.__length_hint__() == 3 -assert list(it) == [3,2,1] +assert list(it) == [3, 2, 1] assert it.__length_hint__() == 0 a = [*[1, 2], 3, *[4, 5]] assert a == [1, 2, 3, 4, 5] +# Test for list unpacking evaluation order (https://github.com/RustPython/RustPython/issues/5566) +a = [1, 2] +b = [a.append(3), *a, a.append(4), *a] +assert a == [1, 2, 3, 4] +assert b == [None, 1, 2, 3, None, 1, 2, 3, 4] + +for base in object, list, tuple: + # do not assume that a type inherited from some sequence type behaves like + # that sequence type + class C(base): + def __iter__(self): + a.append(2) + + def inner(): + yield 3 + a.append(4) + + return inner() + + a = [1] + b = [*a, *C(), *a.copy()] + assert b == [1, 3, 1, 2, 4] + + # Test for list entering daedlock or not (https://github.com/RustPython/RustPython/pull/2933) class MutatingCompare: def __eq__(self, other): self.list.pop() return True + m = MutatingCompare() l = [1, 2, 3, m, 4] @@ -654,18 +878,21 @@ def __eq__(self, other): l = [1, 2, 3, m, 4] m.list = l -l.remove(4) -assert_raises(ValueError, lambda: l.index(4)) # element 4 must not be in the list +l.remove(4) +assert_raises(ValueError, lambda: l.index(4)) # element 4 must not be in the list + -# Test no panic occured when list elements was deleted in __eq__ +# Test no panic occurred when list elements was deleted in __eq__ class rewrite_list_eq(list): pass + class poc: def __eq__(self, other): list1.clear() return self + list1 = rewrite_list_eq([poc()]) list1.remove(list1) assert list1 == [] diff --git a/extra_tests/snippets/builtin_locals.py b/extra_tests/snippets/builtin_locals.py index 6f3fd847c4a..a10cfa389c1 100644 --- a/extra_tests/snippets/builtin_locals.py +++ b/extra_tests/snippets/builtin_locals.py @@ -1,19 +1,18 @@ - a = 5 b = 6 loc = locals() -assert loc['a'] == 5 -assert loc['b'] == 6 +assert loc["a"] == 5 +assert loc["b"] == 6 -def f(): - c = 4 - a = 7 - loc = locals() +def f(): + c = 4 + a = 7 - assert loc['a'] == 4 - assert loc['c'] == 7 - assert not 'b' in loc + loc = locals() + assert loc["a"] == 4 + assert loc["c"] == 7 + assert not "b" in loc diff --git a/extra_tests/snippets/builtin_map.py b/extra_tests/snippets/builtin_map.py index 0de8d2c597e..559d108e38b 100644 --- a/extra_tests/snippets/builtin_map.py +++ b/extra_tests/snippets/builtin_map.py @@ -1,5 +1,5 @@ a = list(map(str, [1, 2, 3])) -assert a == ['1', '2', '3'] +assert a == ["1", "2", "3"] b = list(map(lambda x, y: x + y, [1, 2, 4], [3, 5])) @@ -20,7 +20,7 @@ def __iter__(self): return self -it = map(lambda x: x+1, Counter()) +it = map(lambda x: x + 1, Counter()) assert next(it) == 2 assert next(it) == 3 diff --git a/extra_tests/snippets/builtin_mappingproxy.py b/extra_tests/snippets/builtin_mappingproxy.py index cfba56a8dfb..fdd653c408d 100644 --- a/extra_tests/snippets/builtin_mappingproxy.py +++ b/extra_tests/snippets/builtin_mappingproxy.py @@ -1,5 +1,6 @@ from testutils import assert_raises + class A(dict): def a(): pass @@ -8,16 +9,16 @@ def b(): pass -assert A.__dict__['a'] == A.a +assert A.__dict__["a"] == A.a with assert_raises(KeyError) as cm: - A.__dict__['not here'] + A.__dict__["not here"] assert cm.exception.args[0] == "not here" -assert 'b' in A.__dict__ -assert 'c' not in A.__dict__ +assert "b" in A.__dict__ +assert "c" not in A.__dict__ -assert '__dict__' in A.__dict__ +assert "__dict__" in A.__dict__ assert A.__dict__.get("not here", "default") == "default" assert A.__dict__.get("a", "default") is A.a diff --git a/extra_tests/snippets/builtin_max.py b/extra_tests/snippets/builtin_max.py index cb62123656e..fbb06267681 100644 --- a/extra_tests/snippets/builtin_max.py +++ b/extra_tests/snippets/builtin_max.py @@ -3,17 +3,22 @@ # simple values assert max(0, 0) == 0 assert max(1, 0) == 1 -assert max(1., 0.) == 1. +assert max(1.0, 0.0) == 1.0 assert max(-1, 0) == 0 assert max(1, 2, 3) == 3 # iterables assert max([1, 2, 3]) == 3 assert max((1, 2, 3)) == 3 -assert max({ - "a": 0, - "b": 1, -}) == "b" +assert ( + max( + { + "a": 0, + "b": 1, + } + ) + == "b" +) assert max([1, 2], default=0) == 2 assert max([], default=0) == 0 assert_raises(ValueError, max, []) @@ -30,7 +35,7 @@ # custom class -class MyComparable(): +class MyComparable: nb = 0 def __init__(self): @@ -47,7 +52,7 @@ def __gt__(self, other): assert max([first, second]) == second -class MyNotComparable(): +class MyNotComparable: pass diff --git a/extra_tests/snippets/builtin_memoryview.py b/extra_tests/snippets/builtin_memoryview.py index 81cd5015c17..f206056ebfd 100644 --- a/extra_tests/snippets/builtin_memoryview.py +++ b/extra_tests/snippets/builtin_memoryview.py @@ -10,48 +10,52 @@ assert hash(obj) == hash(a) -class A(array.array): - ... -class B(bytes): - ... +class A(array.array): ... -class C(): - ... -memoryview(bytearray('abcde', encoding='utf-8')) -memoryview(array.array('i', [1, 2, 3])) -memoryview(A('b', [0])) -memoryview(B('abcde', encoding='utf-8')) +class B(bytes): ... + + +class C: ... + + +memoryview(bytearray("abcde", encoding="utf-8")) +memoryview(array.array("i", [1, 2, 3])) +memoryview(A("b", [0])) +memoryview(B("abcde", encoding="utf-8")) assert_raises(TypeError, lambda: memoryview([1, 2, 3])) assert_raises(TypeError, lambda: memoryview((1, 2, 3))) assert_raises(TypeError, lambda: memoryview({})) -assert_raises(TypeError, lambda: memoryview('string')) +assert_raises(TypeError, lambda: memoryview("string")) assert_raises(TypeError, lambda: memoryview(C())) + def test_slice(): - b = b'123456789' + b = b"123456789" m = memoryview(b) m2 = memoryview(b) assert m == m assert m == m2 - assert m.tobytes() == b'123456789' + assert m.tobytes() == b"123456789" assert m == b - assert m[::2].tobytes() == b'13579' - assert m[::2] == b'13579' - assert m[1::2].tobytes() == b'2468' - assert m[::2][1:].tobytes() == b'3579' - assert m[::2][1:-1].tobytes() == b'357' - assert m[::2][::2].tobytes() == b'159' - assert m[::2][1::2].tobytes() == b'37' - assert m[::-1].tobytes() == b'987654321' - assert m[::-2].tobytes() == b'97531' + assert m[::2].tobytes() == b"13579" + assert m[::2] == b"13579" + assert m[1::2].tobytes() == b"2468" + assert m[::2][1:].tobytes() == b"3579" + assert m[::2][1:-1].tobytes() == b"357" + assert m[::2][::2].tobytes() == b"159" + assert m[::2][1::2].tobytes() == b"37" + assert m[::-1].tobytes() == b"987654321" + assert m[::-2].tobytes() == b"97531" + test_slice() + def test_resizable(): - b = bytearray(b'123') + b = bytearray(b"123") b.append(4) m = memoryview(b) assert_raises(BufferError, lambda: b.append(5)) @@ -68,18 +72,21 @@ def test_resizable(): m4.release() b.append(7) + test_resizable() + def test_delitem(): - a = b'abc' + a = b"abc" b = memoryview(a) - assert_raises(TypeError, lambda : b.__delitem__()) - assert_raises(TypeError, lambda : b.__delitem__(0)) - assert_raises(TypeError, lambda : b.__delitem__(10)) - a = bytearray(b'abc') + assert_raises(TypeError, lambda: b.__delitem__()) + assert_raises(TypeError, lambda: b.__delitem__(0)) + assert_raises(TypeError, lambda: b.__delitem__(10)) + a = bytearray(b"abc") b = memoryview(a) - assert_raises(TypeError, lambda : b.__delitem__()) - assert_raises(TypeError, lambda : b.__delitem__(1)) - assert_raises(TypeError, lambda : b.__delitem__(12)) + assert_raises(TypeError, lambda: b.__delitem__()) + assert_raises(TypeError, lambda: b.__delitem__(1)) + assert_raises(TypeError, lambda: b.__delitem__(12)) + -test_delitem() \ No newline at end of file +test_delitem() diff --git a/extra_tests/snippets/builtin_min.py b/extra_tests/snippets/builtin_min.py index 50ebc91f54b..fc8eebba2c8 100644 --- a/extra_tests/snippets/builtin_min.py +++ b/extra_tests/snippets/builtin_min.py @@ -3,17 +3,22 @@ # simple values assert min(0, 0) == 0 assert min(1, 0) == 0 -assert min(1., 0.) == 0. +assert min(1.0, 0.0) == 0.0 assert min(-1, 0) == -1 assert min(1, 2, 3) == 1 # iterables assert min([1, 2, 3]) == 1 assert min((1, 2, 3)) == 1 -assert min({ - "a": 0, - "b": 1, -}) == "a" +assert ( + min( + { + "a": 0, + "b": 1, + } + ) + == "a" +) assert min([1, 2], default=0) == 1 assert min([], default=0) == 0 @@ -31,7 +36,7 @@ # custom class -class MyComparable(): +class MyComparable: nb = 0 def __init__(self): @@ -48,7 +53,7 @@ def __gt__(self, other): assert min([first, second]) == first -class MyNotComparable(): +class MyNotComparable: pass diff --git a/extra_tests/snippets/builtin_none.py b/extra_tests/snippets/builtin_none.py index c75f04ea73f..061739153ff 100644 --- a/extra_tests/snippets/builtin_none.py +++ b/extra_tests/snippets/builtin_none.py @@ -4,22 +4,37 @@ x = None assert x is y + def none(): pass + def none2(): return None + assert none() is none() assert none() is x assert none() is none2() -assert str(None) == 'None' -assert repr(None) == 'None' +assert str(None) == "None" +assert repr(None) == "None" assert type(None)() is None assert None.__eq__(3) is NotImplemented assert None.__ne__(3) is NotImplemented assert None.__eq__(None) is True -# assert None.__ne__(None) is False # changed in 3.12 +assert None.__ne__(None) is False +assert None.__lt__(3) is NotImplemented +assert None.__le__(3) is NotImplemented +assert None.__gt__(3) is NotImplemented +assert None.__ge__(3) is NotImplemented + +none_type_dict = type(None).__dict__ +for name in ("__eq__", "__ne__", "__lt__", "__le__", "__gt__", "__ge__", "__hash__"): + assert name in none_type_dict + assert none_type_dict[name] is not object.__dict__[name] + assert type(none_type_dict[name]).__name__ == "wrapper_descriptor" + +assert hash(None) & 0xFFFFFFFF == 0xFCA86420 diff --git a/extra_tests/snippets/builtin_object.py b/extra_tests/snippets/builtin_object.py index ef83da83e28..64486e1673e 100644 --- a/extra_tests/snippets/builtin_object.py +++ b/extra_tests/snippets/builtin_object.py @@ -1,15 +1,14 @@ class MyObject: pass + assert not MyObject() == MyObject() assert MyObject() != MyObject() myobj = MyObject() assert myobj == myobj assert not myobj != myobj -object.__subclasshook__() == NotImplemented object.__subclasshook__(1) == NotImplemented -object.__subclasshook__(1, 2) == NotImplemented assert MyObject().__eq__(MyObject()) == NotImplemented assert MyObject().__ne__(MyObject()) == NotImplemented @@ -23,6 +22,13 @@ class MyObject: assert obj.__eq__(obj) is True assert obj.__ne__(obj) is False -assert not hasattr(obj, 'a') -obj.__dict__ = {'a': 1} +assert not hasattr(obj, "a") +obj.__dict__ = {"a": 1} assert obj.a == 1 + +# Value inside the formatter goes through a different path of resolution. +# Check that it still works all the same +d = { + 0: "ab", +} +assert "ab ab" == "{k[0]} {vv}".format(k=d, vv=d[0]) diff --git a/extra_tests/snippets/builtin_open.py b/extra_tests/snippets/builtin_open.py index f2c783f2a55..99dd337414a 100644 --- a/extra_tests/snippets/builtin_open.py +++ b/extra_tests/snippets/builtin_open.py @@ -1,19 +1,19 @@ from testutils import assert_raises -fd = open('README.md') -assert 'RustPython' in fd.read() +fd = open("README.md") +assert "RustPython" in fd.read() -assert_raises(FileNotFoundError, open, 'DoesNotExist') +assert_raises(FileNotFoundError, open, "DoesNotExist") # Use open as a context manager -with open('README.md', 'rt') as fp: +with open("README.md", "rt") as fp: contents = fp.read() assert type(contents) == str, "type is " + str(type(contents)) -with open('README.md', 'r') as fp: +with open("README.md", "r") as fp: contents = fp.read() assert type(contents) == str, "type is " + str(type(contents)) -with open('README.md', 'rb') as fp: +with open("README.md", "rb") as fp: contents = fp.read() assert type(contents) == bytes, "type is " + str(type(contents)) diff --git a/extra_tests/snippets/builtin_ord.py b/extra_tests/snippets/builtin_ord.py index 271728b84a7..e451e078c3c 100644 --- a/extra_tests/snippets/builtin_ord.py +++ b/extra_tests/snippets/builtin_ord.py @@ -3,11 +3,18 @@ assert ord("a") == 97 assert ord("é") == 233 assert ord("🤡") == 129313 -assert ord(b'a') == 97 -assert ord(bytearray(b'a')) == 97 +assert ord(b"a") == 97 +assert ord(bytearray(b"a")) == 97 -assert_raises(TypeError, ord, _msg='ord() is called with no argument') -assert_raises(TypeError, ord, "", _msg='ord() is called with an empty string') -assert_raises(TypeError, ord, "ab", _msg='ord() is called with more than one character') -assert_raises(TypeError, ord, b"ab", _msg='ord() expected a character, but string of length 2 found') -assert_raises(TypeError, ord, 1, _msg='ord() expected a string, bytes or bytearray, but found int') +assert_raises(TypeError, ord, _msg="ord() is called with no argument") +assert_raises(TypeError, ord, "", _msg="ord() is called with an empty string") +assert_raises(TypeError, ord, "ab", _msg="ord() is called with more than one character") +assert_raises( + TypeError, + ord, + b"ab", + _msg="ord() expected a character, but string of length 2 found", +) +assert_raises( + TypeError, ord, 1, _msg="ord() expected a string, bytes or bytearray, but found int" +) diff --git a/extra_tests/snippets/builtin_posixshmem.py b/extra_tests/snippets/builtin_posixshmem.py new file mode 100644 index 00000000000..38ace68d584 --- /dev/null +++ b/extra_tests/snippets/builtin_posixshmem.py @@ -0,0 +1,12 @@ +import os +import sys + +if os.name != "posix": + sys.exit(0) + +import _posixshmem + +name = f"/rp_posixshmem_{os.getpid()}" +fd = _posixshmem.shm_open(name, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) +os.close(fd) +_posixshmem.shm_unlink(name) diff --git a/extra_tests/snippets/builtin_pow.py b/extra_tests/snippets/builtin_pow.py index d31e5dd713a..d38b57c37a0 100644 --- a/extra_tests/snippets/builtin_pow.py +++ b/extra_tests/snippets/builtin_pow.py @@ -1,4 +1,4 @@ -from testutils import assert_raises, assert_equal +from testutils import assert_equal, assert_raises assert pow(3, 2) == 9 assert pow(5, 3, 100) == 25 @@ -12,7 +12,7 @@ assert pow(2.0, 1) == 2.0 assert pow(0, 10**1000) == 0 assert pow(1, 10**1000) == 1 -assert pow(-1, 10**1000+1) == -1 +assert pow(-1, 10**1000 + 1) == -1 assert pow(-1, 10**1000) == 1 assert pow(2, 4, 5) == 1 @@ -59,7 +59,7 @@ def powtest(type): assert_raises(ZeroDivisionError, pow, zero, exp) il, ih = -20, 20 - jl, jh = -5, 5 + jl, jh = -5, 5 kl, kh = -10, 10 asseq = assert_equal if type == float: @@ -76,10 +76,7 @@ def powtest(type): if type == float or j < 0: assert_raises(TypeError, pow, type(i), j, k) continue - asseq( - pow(type(i), j, k), - pow(type(i), j) % type(k) - ) + asseq(pow(type(i), j, k), pow(type(i), j) % type(k)) def test_powint(): @@ -92,40 +89,35 @@ def test_powfloat(): def test_other(): # Other tests-- not very systematic - assert_equal(pow(3,3) % 8, pow(3,3,8)) - assert_equal(pow(3,3) % -8, pow(3,3,-8)) - assert_equal(pow(3,2) % -2, pow(3,2,-2)) - assert_equal(pow(-3,3) % 8, pow(-3,3,8)) - assert_equal(pow(-3,3) % -8, pow(-3,3,-8)) - assert_equal(pow(5,2) % -8, pow(5,2,-8)) - - assert_equal(pow(3,3) % 8, pow(3,3,8)) - assert_equal(pow(3,3) % -8, pow(3,3,-8)) - assert_equal(pow(3,2) % -2, pow(3,2,-2)) - assert_equal(pow(-3,3) % 8, pow(-3,3,8)) - assert_equal(pow(-3,3) % -8, pow(-3,3,-8)) - assert_equal(pow(5,2) % -8, pow(5,2,-8)) + assert_equal(pow(3, 3) % 8, pow(3, 3, 8)) + assert_equal(pow(3, 3) % -8, pow(3, 3, -8)) + assert_equal(pow(3, 2) % -2, pow(3, 2, -2)) + assert_equal(pow(-3, 3) % 8, pow(-3, 3, 8)) + assert_equal(pow(-3, 3) % -8, pow(-3, 3, -8)) + assert_equal(pow(5, 2) % -8, pow(5, 2, -8)) + + assert_equal(pow(3, 3) % 8, pow(3, 3, 8)) + assert_equal(pow(3, 3) % -8, pow(3, 3, -8)) + assert_equal(pow(3, 2) % -2, pow(3, 2, -2)) + assert_equal(pow(-3, 3) % 8, pow(-3, 3, 8)) + assert_equal(pow(-3, 3) % -8, pow(-3, 3, -8)) + assert_equal(pow(5, 2) % -8, pow(5, 2, -8)) for i in range(-10, 11): for j in range(0, 6): for k in range(-7, 11): if j >= 0 and k != 0: - assert_equal( - pow(i,j) % k, - pow(i,j,k) - ) + assert_equal(pow(i, j) % k, pow(i, j, k)) if j >= 0 and k != 0: - assert_equal( - pow(int(i),j) % k, - pow(int(i),j,k) - ) + assert_equal(pow(int(i), j) % k, pow(int(i), j, k)) def test_bug643260(): class TestRpow: def __rpow__(self, other): return None - None ** TestRpow() # Won't fail when __rpow__ invoked. SF bug #643260. + + None ** TestRpow() # Won't fail when __rpow__ invoked. SF bug #643260. def test_bug705231(): @@ -141,15 +133,15 @@ def test_bug705231(): for b in range(-10, 11): eq(pow(a, float(b)), b & 1 and -1.0 or 1.0) for n in range(0, 100): - fiveto = float(5 ** n) + fiveto = float(5**n) # For small n, fiveto will be odd. Eventually we run out of # mantissa bits, though, and thereafer fiveto will be even. expected = fiveto % 2.0 and -1.0 or 1.0 eq(pow(a, fiveto), expected) eq(pow(a, -fiveto), expected) - eq(expected, 1.0) # else we didn't push fiveto to evenness + eq(expected, 1.0) # else we didn't push fiveto to evenness -tests = [f for name, f in locals().items() if name.startswith('test_')] +tests = [f for name, f in locals().items() if name.startswith("test_")] for f in tests: f() diff --git a/extra_tests/snippets/builtin_print.py b/extra_tests/snippets/builtin_print.py index db53c80edb2..2778cff65a6 100644 --- a/extra_tests/snippets/builtin_print.py +++ b/extra_tests/snippets/builtin_print.py @@ -1,16 +1,17 @@ -from testutils import assert_raises import io +from testutils import assert_raises + print(2 + 3) -assert_raises(TypeError, print, 'test', end=4, _msg='wrong type passed to end') -assert_raises(TypeError, print, 'test', sep=['a'], _msg='wrong type passed to sep') +assert_raises(TypeError, print, "test", end=4, _msg="wrong type passed to end") +assert_raises(TypeError, print, "test", sep=["a"], _msg="wrong type passed to sep") try: - print('test', end=None, sep=None, flush=None) + print("test", end=None, sep=None, flush=None) except: - assert False, 'Expected None passed to end, sep, and flush to not raise errors' + assert False, "Expected None passed to end, sep, and flush to not raise errors" buf = io.StringIO() -print('hello, world', file=buf) -assert buf.getvalue() == 'hello, world\n', buf.getvalue() +print("hello, world", file=buf) +assert buf.getvalue() == "hello, world\n", buf.getvalue() diff --git a/extra_tests/snippets/builtin_property.py b/extra_tests/snippets/builtin_property.py index 2a97c99b3bc..de64e526228 100644 --- a/extra_tests/snippets/builtin_property.py +++ b/extra_tests/snippets/builtin_property.py @@ -62,8 +62,8 @@ def foo(self): assert p.__doc__ is None # Test property instance __doc__ attribute: -p.__doc__ = '222' -assert p.__doc__ == '222' +p.__doc__ = "222" +assert p.__doc__ == "222" p1 = property("a", "b", "c") @@ -83,5 +83,5 @@ def foo(self): assert p1.__get__(None, object) is p1 # assert p1.__doc__ is 'a'.__doc__ -p2 = property('a', doc='pdoc') +p2 = property("a", doc="pdoc") # assert p2.__doc__ == 'pdoc' diff --git a/extra_tests/snippets/builtin_range.py b/extra_tests/snippets/builtin_range.py index 9f8f03b63c2..6bfb99f453d 100644 --- a/extra_tests/snippets/builtin_range.py +++ b/extra_tests/snippets/builtin_range.py @@ -1,11 +1,11 @@ from testutils import assert_raises -assert range(2**63+1)[2**63] == 9223372036854775808 +assert range(2**63 + 1)[2**63] == 9223372036854775808 # len tests -assert len(range(10, 5)) == 0, 'Range with no elements should have length = 0' -assert len(range(10, 5, -2)) == 3, 'Expected length 3, for elements: 10, 8, 6' -assert len(range(5, 10, 2)) == 3, 'Expected length 3, for elements: 5, 7, 9' +assert len(range(10, 5)) == 0, "Range with no elements should have length = 0" +assert len(range(10, 5, -2)) == 3, "Expected length 3, for elements: 10, 8, 6" +assert len(range(5, 10, 2)) == 3, "Expected length 3, for elements: 5, 7, 9" # index tests assert range(10).index(6) == 6 @@ -13,18 +13,18 @@ assert range(4, 10, 2).index(6) == 1 assert range(10, 4, -2).index(8) == 1 -assert_raises(ValueError, lambda: range(10).index(-1), _msg='out of bounds') -assert_raises(ValueError, lambda: range(10).index(10), _msg='out of bounds') -assert_raises(ValueError, lambda: range(4, 10, 2).index(5), _msg='out of step') -assert_raises(ValueError, lambda: range(10).index('foo'), _msg='not an int') -assert_raises(ValueError, lambda: range(1, 10, 0), _msg='step is zero') +assert_raises(ValueError, lambda: range(10).index(-1), _msg="out of bounds") +assert_raises(ValueError, lambda: range(10).index(10), _msg="out of bounds") +assert_raises(ValueError, lambda: range(4, 10, 2).index(5), _msg="out of step") +assert_raises(ValueError, lambda: range(10).index("foo"), _msg="not an int") +assert_raises(ValueError, lambda: range(1, 10, 0), _msg="step is zero") # get tests assert range(10)[0] == 0 assert range(10)[9] == 9 assert range(10, 0, -1)[0] == 10 assert range(10, 0, -1)[9] == 1 -assert_raises(IndexError, lambda: range(10)[10], _msg='out of bound') +assert_raises(IndexError, lambda: range(10)[10], _msg="out of bound") # slice tests assert range(10)[0:3] == range(3) @@ -34,13 +34,17 @@ assert range(10, 100, 3)[4:1000:5] == range(22, 100, 15) assert range(10)[:] == range(10) assert range(10, 0, -2)[0:5:2] == range(10, 0, -4) -assert range(10)[10:11] == range(10,10) +assert range(10)[10:11] == range(10, 10) assert range(0, 10, -1)[::-1] == range(1, 1) assert range(0, 10)[::-1] == range(9, -1, -1) assert range(0, -10)[::-1] == range(-1, -1, -1) assert range(0, -10)[::-1][::-1] == range(0, 0) -assert_raises(ValueError, lambda: range(0, 10)[::0], _msg='slice step cannot be zero') -assert_raises(TypeError, lambda: range(0, 10)['a':], _msg='slice indices must be integers or None or have an __index__ method') +assert_raises(ValueError, lambda: range(0, 10)[::0], _msg="slice step cannot be zero") +assert_raises( + TypeError, + lambda: range(0, 10)["a":], + _msg="slice indices must be integers or None or have an __index__ method", +) # count tests assert range(10).count(2) == 1 @@ -64,22 +68,22 @@ assert range(10).__ne__(range(0, 11, 1)) is True assert range(0, 10, 3).__eq__(range(0, 11, 3)) is True assert range(0, 10, 3).__ne__(range(0, 11, 3)) is False -#__lt__ +# __lt__ assert range(1, 2, 3).__lt__(range(1, 2, 3)) == NotImplemented assert range(1, 2, 1).__lt__(range(1, 2)) == NotImplemented assert range(2).__lt__(range(0, 2)) == NotImplemented -#__gt__ +# __gt__ assert range(1, 2, 3).__gt__(range(1, 2, 3)) == NotImplemented assert range(1, 2, 1).__gt__(range(1, 2)) == NotImplemented assert range(2).__gt__(range(0, 2)) == NotImplemented -#__le__ +# __le__ assert range(1, 2, 3).__le__(range(1, 2, 3)) == NotImplemented assert range(1, 2, 1).__le__(range(1, 2)) == NotImplemented assert range(2).__le__(range(0, 2)) == NotImplemented -#__ge__ +# __ge__ assert range(1, 2, 3).__ge__(range(1, 2, 3)) == NotImplemented assert range(1, 2, 1).__ge__(range(1, 2)) == NotImplemented assert range(2).__ge__(range(0, 2)) == NotImplemented @@ -101,12 +105,12 @@ assert -1 not in range(10) assert 9 not in range(10, 4, -2) assert 4 not in range(10, 4, -2) -assert 'foo' not in range(10) +assert "foo" not in range(10) # __reversed__ assert list(reversed(range(5))) == [4, 3, 2, 1, 0] assert list(reversed(range(5, 0, -1))) == [1, 2, 3, 4, 5] -assert list(reversed(range(1,10,5))) == [6, 1] +assert list(reversed(range(1, 10, 5))) == [6, 1] # __reduce__ assert range(10).__reduce__()[0] == range @@ -120,7 +124,7 @@ # negative index assert range(10)[-1] == 9 -assert_raises(IndexError, lambda: range(10)[-11], _msg='out of bound') +assert_raises(IndexError, lambda: range(10)[-11], _msg="out of bound") assert range(10)[-2:4] == range(8, 4) assert range(10)[-6:-2] == range(4, 8) assert range(50, 0, -2)[-5] == 10 diff --git a/extra_tests/snippets/builtin_reversed.py b/extra_tests/snippets/builtin_reversed.py index 261b5c3263d..0ec2f2828bf 100644 --- a/extra_tests/snippets/builtin_reversed.py +++ b/extra_tests/snippets/builtin_reversed.py @@ -1,4 +1,4 @@ assert list(reversed(range(5))) == [4, 3, 2, 1, 0] -l = [5,4,3,2,1] -assert list(reversed(l)) == [1,2,3,4,5] +l = [5, 4, 3, 2, 1] +assert list(reversed(l)) == [1, 2, 3, 4, 5] diff --git a/extra_tests/snippets/builtin_round.py b/extra_tests/snippets/builtin_round.py index b4b070c4cc3..99c4ed1d271 100644 --- a/extra_tests/snippets/builtin_round.py +++ b/extra_tests/snippets/builtin_round.py @@ -8,11 +8,11 @@ assert round(-1.5) == -2 # ValueError: cannot convert float NaN to integer -assert_raises(ValueError, round, float('nan')) +assert_raises(ValueError, round, float("nan")) # OverflowError: cannot convert float infinity to integer -assert_raises(OverflowError, round, float('inf')) +assert_raises(OverflowError, round, float("inf")) # OverflowError: cannot convert float infinity to integer -assert_raises(OverflowError, round, -float('inf')) +assert_raises(OverflowError, round, -float("inf")) assert round(0) == 0 assert isinstance(round(0), int) diff --git a/extra_tests/snippets/builtin_set.py b/extra_tests/snippets/builtin_set.py index 79f0602aea3..950875ea09a 100644 --- a/extra_tests/snippets/builtin_set.py +++ b/extra_tests/snippets/builtin_set.py @@ -1,38 +1,54 @@ from testutils import assert_raises -assert set([1,2]) == set([1,2]) -assert not set([1,2,3]) == set([1,2]) +assert set([1, 2]) == set([1, 2]) +assert not set([1, 2, 3]) == set([1, 2]) -assert set([1,2,3]) >= set([1,2]) -assert set([1,2]) >= set([1,2]) -assert not set([1,3]) >= set([1,2]) +assert set([1, 2, 3]) >= set([1, 2]) +assert set([1, 2]) >= set([1, 2]) +assert not set([1, 3]) >= set([1, 2]) -assert set([1,2,3]).issuperset(set([1,2])) -assert set([1,2]).issuperset(set([1,2])) -assert not set([1,3]).issuperset(set([1,2])) +assert set([1, 2, 3]).issuperset(set([1, 2])) +assert set([1, 2]).issuperset(set([1, 2])) +assert not set([1, 3]).issuperset(set([1, 2])) -assert set([1,2,3]) > set([1,2]) -assert not set([1,2]) > set([1,2]) -assert not set([1,3]) > set([1,2]) +assert set([1, 2, 3]) > set([1, 2]) +assert not set([1, 2]) > set([1, 2]) +assert not set([1, 3]) > set([1, 2]) -assert set([1,2]) <= set([1,2,3]) -assert set([1,2]) <= set([1,2]) -assert not set([1,3]) <= set([1,2]) +assert set([1, 2]) <= set([1, 2, 3]) +assert set([1, 2]) <= set([1, 2]) +assert not set([1, 3]) <= set([1, 2]) -assert set([1,2]).issubset(set([1,2,3])) -assert set([1,2]).issubset(set([1,2])) -assert not set([1,3]).issubset(set([1,2])) +assert set([1, 2]).issubset(set([1, 2, 3])) +assert set([1, 2]).issubset(set([1, 2])) +assert not set([1, 3]).issubset(set([1, 2])) -assert set([1,2]) < set([1,2,3]) -assert not set([1,2]) < set([1,2]) -assert not set([1,3]) < set([1,2]) +assert set([1, 2]) < set([1, 2, 3]) +assert not set([1, 2]) < set([1, 2]) +assert not set([1, 3]) < set([1, 2]) assert (set() == []) is False assert set().__eq__([]) == NotImplemented -assert_raises(TypeError, lambda: set() < [], _msg="'<' not supported between instances of 'set' and 'list'") -assert_raises(TypeError, lambda: set() <= [], _msg="'<=' not supported between instances of 'set' and 'list'") -assert_raises(TypeError, lambda: set() > [], _msg="'>' not supported between instances of 'set' and 'list'") -assert_raises(TypeError, lambda: set() >= [], _msg="'>=' not supported between instances of 'set' and 'list'") +assert_raises( + TypeError, + lambda: set() < [], + _msg="'<' not supported between instances of 'set' and 'list'", +) +assert_raises( + TypeError, + lambda: set() <= [], + _msg="'<=' not supported between instances of 'set' and 'list'", +) +assert_raises( + TypeError, + lambda: set() > [], + _msg="'>' not supported between instances of 'set' and 'list'", +) +assert_raises( + TypeError, + lambda: set() >= [], + _msg="'>=' not supported between instances of 'set' and 'list'", +) assert set().issuperset([]) assert set().issubset([]) assert not set().issuperset([1, 2, 3]) @@ -47,6 +63,7 @@ assert_raises(TypeError, set().issuperset, 3, _msg="'int' object is not iterable") assert_raises(TypeError, set().issubset, 3, _msg="'int' object is not iterable") + class Hashable(object): def __init__(self, obj): self.obj = obj @@ -57,6 +74,7 @@ def __repr__(self): def __hash__(self): return id(self) + assert repr(set()) == "set()" assert repr(set([1, 2, 3])) == "{1, 2, 3}" @@ -64,9 +82,11 @@ def __hash__(self): recursive.add(Hashable(recursive)) assert repr(recursive) == "{set(...)}" + class S(set): pass + assert repr(S()) == "S()" assert repr(S([1, 2, 3])) == "S({1, 2, 3})" @@ -79,44 +99,44 @@ class S(set): a.clear() assert len(a) == 0 -assert set([1,2,3]).union(set([4,5])) == set([1,2,3,4,5]) -assert set([1,2,3]).union(set([1,2,3,4,5])) == set([1,2,3,4,5]) -assert set([1,2,3]).union([1,2,3,4,5]) == set([1,2,3,4,5]) +assert set([1, 2, 3]).union(set([4, 5])) == set([1, 2, 3, 4, 5]) +assert set([1, 2, 3]).union(set([1, 2, 3, 4, 5])) == set([1, 2, 3, 4, 5]) +assert set([1, 2, 3]).union([1, 2, 3, 4, 5]) == set([1, 2, 3, 4, 5]) -assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5]) -assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5]) -assert_raises(TypeError, lambda: set([1,2,3]) | [1,2,3,4,5]) +assert set([1, 2, 3]) | set([4, 5]) == set([1, 2, 3, 4, 5]) +assert set([1, 2, 3]) | set([1, 2, 3, 4, 5]) == set([1, 2, 3, 4, 5]) +assert_raises(TypeError, lambda: set([1, 2, 3]) | [1, 2, 3, 4, 5]) -assert set([1,2,3]).intersection(set([1,2])) == set([1,2]) -assert set([1,2,3]).intersection(set([5,6])) == set([]) -assert set([1,2,3]).intersection([1,2]) == set([1,2]) +assert set([1, 2, 3]).intersection(set([1, 2])) == set([1, 2]) +assert set([1, 2, 3]).intersection(set([5, 6])) == set([]) +assert set([1, 2, 3]).intersection([1, 2]) == set([1, 2]) -assert set([1,2,3]) & set([4,5]) == set([]) -assert set([1,2,3]) & set([1,2,3,4,5]) == set([1,2,3]) -assert_raises(TypeError, lambda: set([1,2,3]) & [1,2,3,4,5]) +assert set([1, 2, 3]) & set([4, 5]) == set([]) +assert set([1, 2, 3]) & set([1, 2, 3, 4, 5]) == set([1, 2, 3]) +assert_raises(TypeError, lambda: set([1, 2, 3]) & [1, 2, 3, 4, 5]) -assert set([1,2,3]).difference(set([1,2])) == set([3]) -assert set([1,2,3]).difference(set([5,6])) == set([1,2,3]) -assert set([1,2,3]).difference([1,2]) == set([3]) +assert set([1, 2, 3]).difference(set([1, 2])) == set([3]) +assert set([1, 2, 3]).difference(set([5, 6])) == set([1, 2, 3]) +assert set([1, 2, 3]).difference([1, 2]) == set([3]) -assert set([1,2,3]) - set([4,5]) == set([1,2,3]) -assert set([1,2,3]) - set([1,2,3,4,5]) == set([]) -assert_raises(TypeError, lambda: set([1,2,3]) - [1,2,3,4,5]) +assert set([1, 2, 3]) - set([4, 5]) == set([1, 2, 3]) +assert set([1, 2, 3]) - set([1, 2, 3, 4, 5]) == set([]) +assert_raises(TypeError, lambda: set([1, 2, 3]) - [1, 2, 3, 4, 5]) -assert set([1,2]).__sub__(set([2,3])) == set([1]) -assert set([1,2]).__rsub__(set([2,3])) == set([3]) +assert set([1, 2]).__sub__(set([2, 3])) == set([1]) +assert set([1, 2]).__rsub__(set([2, 3])) == set([3]) -assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3]) -assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6]) -assert set([1,2,3]).symmetric_difference([1,2]) == set([3]) +assert set([1, 2, 3]).symmetric_difference(set([1, 2])) == set([3]) +assert set([1, 2, 3]).symmetric_difference(set([5, 6])) == set([1, 2, 3, 5, 6]) +assert set([1, 2, 3]).symmetric_difference([1, 2]) == set([3]) -assert set([1,2,3]) ^ set([4,5]) == set([1,2,3,4,5]) -assert set([1,2,3]) ^ set([1,2,3,4,5]) == set([4,5]) -assert_raises(TypeError, lambda: set([1,2,3]) ^ [1,2,3,4,5]) +assert set([1, 2, 3]) ^ set([4, 5]) == set([1, 2, 3, 4, 5]) +assert set([1, 2, 3]) ^ set([1, 2, 3, 4, 5]) == set([4, 5]) +assert_raises(TypeError, lambda: set([1, 2, 3]) ^ [1, 2, 3, 4, 5]) -assert set([1,2,3]).isdisjoint(set([5,6])) == True -assert set([1,2,3]).isdisjoint(set([2,5,6])) == False -assert set([1,2,3]).isdisjoint([5,6]) == True +assert set([1, 2, 3]).isdisjoint(set([5, 6])) == True +assert set([1, 2, 3]).isdisjoint(set([2, 5, 6])) == False +assert set([1, 2, 3]).isdisjoint([5, 6]) == True assert_raises(TypeError, lambda: set() & []) assert_raises(TypeError, lambda: set() | []) @@ -132,7 +152,7 @@ class S(set): assert not 1 in a assert a.discard(42) is None -a = set([1,2,3]) +a = set([1, 2, 3]) b = a.copy() assert len(a) == 3 assert len(b) == 3 @@ -140,71 +160,83 @@ class S(set): assert len(a) == 3 assert len(b) == 0 -a = set([1,2]) +a = set([1, 2]) b = a.pop() -assert b in [1,2] +assert b in [1, 2] c = a.pop() -assert (c in [1,2] and c != b) +assert c in [1, 2] and c != b assert_raises(KeyError, lambda: a.pop()) -a = set([1,2,3]) -a.update([3,4,5]) -assert a == set([1,2,3,4,5]) +a = set([1, 2, 3]) +a.update([3, 4, 5]) +assert a == set([1, 2, 3, 4, 5]) assert_raises(TypeError, lambda: a.update(1)) -a = set([1,2,3]) +a = set([1, 2, 3]) b = set() for e in a: - assert e == 1 or e == 2 or e == 3 - b.add(e) + assert e == 1 or e == 2 or e == 3 + b.add(e) assert a == b -a = set([1,2,3]) -a |= set([3,4,5]) -assert a == set([1,2,3,4,5]) +a = set([1, 2, 3]) +a |= set([3, 4, 5]) +assert a == set([1, 2, 3, 4, 5]) with assert_raises(TypeError): - a |= 1 + a |= 1 with assert_raises(TypeError): - a |= [1,2,3] + a |= [1, 2, 3] -a = set([1,2,3]) -a.intersection_update([2,3,4,5]) -assert a == set([2,3]) +a = set([1, 2, 3]) +a.intersection_update([2, 3, 4, 5]) +assert a == set([2, 3]) assert_raises(TypeError, lambda: a.intersection_update(1)) -a = set([1,2,3]) -a &= set([2,3,4,5]) -assert a == set([2,3]) +a = set([1, 2, 3]) +a &= set([2, 3, 4, 5]) +assert a == set([2, 3]) with assert_raises(TypeError): - a &= 1 + a &= 1 with assert_raises(TypeError): - a &= [1,2,3] + a &= [1, 2, 3] + +a = set([1, 2, 3]) +a &= a +assert a == set([1, 2, 3]) + +a = set([1, 2, 3]) +a -= a +assert a == set() + +a = set([1, 2, 3]) +a ^= a +assert a == set() -a = set([1,2,3]) -a.difference_update([3,4,5]) -assert a == set([1,2]) +a = set([1, 2, 3]) +a.difference_update([3, 4, 5]) +assert a == set([1, 2]) assert_raises(TypeError, lambda: a.difference_update(1)) -a = set([1,2,3]) -a -= set([3,4,5]) -assert a == set([1,2]) +a = set([1, 2, 3]) +a -= set([3, 4, 5]) +assert a == set([1, 2]) with assert_raises(TypeError): - a -= 1 + a -= 1 with assert_raises(TypeError): - a -= [1,2,3] + a -= [1, 2, 3] -a = set([1,2,3]) -a.symmetric_difference_update([3,4,5]) -assert a == set([1,2,4,5]) +a = set([1, 2, 3]) +a.symmetric_difference_update([3, 4, 5]) +assert a == set([1, 2, 4, 5]) assert_raises(TypeError, lambda: a.difference_update(1)) -a = set([1,2,3]) -a ^= set([3,4,5]) -assert a == set([1,2,4,5]) +a = set([1, 2, 3]) +a ^= set([3, 4, 5]) +assert a == set([1, 2, 4, 5]) with assert_raises(TypeError): - a ^= 1 + a ^= 1 with assert_raises(TypeError): - a ^= [1,2,3] + a ^= [1, 2, 3] a = set([1, 2, 3]) i = iter(a) @@ -218,142 +250,154 @@ class S(set): # frozen set -assert frozenset([1,2]) == frozenset([1,2]) -assert not frozenset([1,2,3]) == frozenset([1,2]) +assert frozenset([1, 2]) == frozenset([1, 2]) +assert not frozenset([1, 2, 3]) == frozenset([1, 2]) -assert frozenset([1,2,3]) >= frozenset([1,2]) -assert frozenset([1,2]) >= frozenset([1,2]) -assert not frozenset([1,3]) >= frozenset([1,2]) +assert frozenset([1, 2, 3]) >= frozenset([1, 2]) +assert frozenset([1, 2]) >= frozenset([1, 2]) +assert not frozenset([1, 3]) >= frozenset([1, 2]) -assert frozenset([1,2,3]).issuperset(frozenset([1,2])) -assert frozenset([1,2]).issuperset(frozenset([1,2])) -assert not frozenset([1,3]).issuperset(frozenset([1,2])) +assert frozenset([1, 2, 3]).issuperset(frozenset([1, 2])) +assert frozenset([1, 2]).issuperset(frozenset([1, 2])) +assert not frozenset([1, 3]).issuperset(frozenset([1, 2])) -assert frozenset([1,2,3]) > frozenset([1,2]) -assert not frozenset([1,2]) > frozenset([1,2]) -assert not frozenset([1,3]) > frozenset([1,2]) +assert frozenset([1, 2, 3]) > frozenset([1, 2]) +assert not frozenset([1, 2]) > frozenset([1, 2]) +assert not frozenset([1, 3]) > frozenset([1, 2]) -assert frozenset([1,2]) <= frozenset([1,2,3]) -assert frozenset([1,2]) <= frozenset([1,2]) -assert not frozenset([1,3]) <= frozenset([1,2]) +assert frozenset([1, 2]) <= frozenset([1, 2, 3]) +assert frozenset([1, 2]) <= frozenset([1, 2]) +assert not frozenset([1, 3]) <= frozenset([1, 2]) -assert frozenset([1,2]).issubset(frozenset([1,2,3])) -assert frozenset([1,2]).issubset(frozenset([1,2])) -assert not frozenset([1,3]).issubset(frozenset([1,2])) +assert frozenset([1, 2]).issubset(frozenset([1, 2, 3])) +assert frozenset([1, 2]).issubset(frozenset([1, 2])) +assert not frozenset([1, 3]).issubset(frozenset([1, 2])) -assert frozenset([1,2]) < frozenset([1,2,3]) -assert not frozenset([1,2]) < frozenset([1,2]) -assert not frozenset([1,3]) < frozenset([1,2]) +assert frozenset([1, 2]) < frozenset([1, 2, 3]) +assert not frozenset([1, 2]) < frozenset([1, 2]) +assert not frozenset([1, 3]) < frozenset([1, 2]) a = frozenset([1, 2, 3]) assert len(a) == 3 b = a.copy() assert b == a -assert frozenset([1,2,3]).union(frozenset([4,5])) == frozenset([1,2,3,4,5]) -assert frozenset([1,2,3]).union(frozenset([1,2,3,4,5])) == frozenset([1,2,3,4,5]) -assert frozenset([1,2,3]).union([1,2,3,4,5]) == frozenset([1,2,3,4,5]) +assert frozenset([1, 2, 3]).union(frozenset([4, 5])) == frozenset([1, 2, 3, 4, 5]) +assert frozenset([1, 2, 3]).union(frozenset([1, 2, 3, 4, 5])) == frozenset( + [1, 2, 3, 4, 5] +) +assert frozenset([1, 2, 3]).union([1, 2, 3, 4, 5]) == frozenset([1, 2, 3, 4, 5]) -assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) -assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) -assert_raises(TypeError, lambda: frozenset([1,2,3]) | [1,2,3,4,5]) +assert frozenset([1, 2, 3]) | frozenset([4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert frozenset([1, 2, 3]) | frozenset([1, 2, 3, 4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert_raises(TypeError, lambda: frozenset([1, 2, 3]) | [1, 2, 3, 4, 5]) -assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) -assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) -assert frozenset([1,2,3]).intersection([1,2]) == frozenset([1,2]) +assert frozenset([1, 2, 3]).intersection(frozenset([1, 2])) == frozenset([1, 2]) +assert frozenset([1, 2, 3]).intersection(frozenset([5, 6])) == frozenset([]) +assert frozenset([1, 2, 3]).intersection([1, 2]) == frozenset([1, 2]) -assert frozenset([1,2,3]) & frozenset([4,5]) == frozenset([]) -assert frozenset([1,2,3]) & frozenset([1,2,3,4,5]) == frozenset([1,2,3]) -assert_raises(TypeError, lambda: frozenset([1,2,3]) & [1,2,3,4,5]) +assert frozenset([1, 2, 3]) & frozenset([4, 5]) == frozenset([]) +assert frozenset([1, 2, 3]) & frozenset([1, 2, 3, 4, 5]) == frozenset([1, 2, 3]) +assert_raises(TypeError, lambda: frozenset([1, 2, 3]) & [1, 2, 3, 4, 5]) -assert frozenset([1,2,3]).difference(frozenset([1,2])) == frozenset([3]) -assert frozenset([1,2,3]).difference(frozenset([5,6])) == frozenset([1,2,3]) -assert frozenset([1,2,3]).difference([1,2]) == frozenset([3]) +assert frozenset([1, 2, 3]).difference(frozenset([1, 2])) == frozenset([3]) +assert frozenset([1, 2, 3]).difference(frozenset([5, 6])) == frozenset([1, 2, 3]) +assert frozenset([1, 2, 3]).difference([1, 2]) == frozenset([3]) -assert frozenset([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) -assert frozenset([1,2,3]) - frozenset([1,2,3,4,5]) == frozenset([]) -assert_raises(TypeError, lambda: frozenset([1,2,3]) - [1,2,3,4,5]) +assert frozenset([1, 2, 3]) - frozenset([4, 5]) == frozenset([1, 2, 3]) +assert frozenset([1, 2, 3]) - frozenset([1, 2, 3, 4, 5]) == frozenset([]) +assert_raises(TypeError, lambda: frozenset([1, 2, 3]) - [1, 2, 3, 4, 5]) -assert frozenset([1,2]).__sub__(frozenset([2,3])) == frozenset([1]) -assert frozenset([1,2]).__rsub__(frozenset([2,3])) == frozenset([3]) +assert frozenset([1, 2]).__sub__(frozenset([2, 3])) == frozenset([1]) +assert frozenset([1, 2]).__rsub__(frozenset([2, 3])) == frozenset([3]) -assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) -assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) -assert frozenset([1,2,3]).symmetric_difference([1,2]) == frozenset([3]) +assert frozenset([1, 2, 3]).symmetric_difference(frozenset([1, 2])) == frozenset([3]) +assert frozenset([1, 2, 3]).symmetric_difference(frozenset([5, 6])) == frozenset( + [1, 2, 3, 5, 6] +) +assert frozenset([1, 2, 3]).symmetric_difference([1, 2]) == frozenset([3]) -assert frozenset([1,2,3]) ^ frozenset([4,5]) == frozenset([1,2,3,4,5]) -assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) -assert_raises(TypeError, lambda: frozenset([1,2,3]) ^ [1,2,3,4,5]) +assert frozenset([1, 2, 3]) ^ frozenset([4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert frozenset([1, 2, 3]) ^ frozenset([1, 2, 3, 4, 5]) == frozenset([4, 5]) +assert_raises(TypeError, lambda: frozenset([1, 2, 3]) ^ [1, 2, 3, 4, 5]) -assert frozenset([1,2,3]).isdisjoint(frozenset([5,6])) == True -assert frozenset([1,2,3]).isdisjoint(frozenset([2,5,6])) == False -assert frozenset([1,2,3]).isdisjoint([5,6]) == True +assert frozenset([1, 2, 3]).isdisjoint(frozenset([5, 6])) == True +assert frozenset([1, 2, 3]).isdisjoint(frozenset([2, 5, 6])) == False +assert frozenset([1, 2, 3]).isdisjoint([5, 6]) == True assert_raises(TypeError, frozenset, [[]]) -a = frozenset([1,2,3]) +a = frozenset([1, 2, 3]) b = set() for e in a: - assert e == 1 or e == 2 or e == 3 - b.add(e) + assert e == 1 or e == 2 or e == 3 + b.add(e) assert a == b # set and frozen set -assert frozenset([1,2,3]).union(set([4,5])) == frozenset([1,2,3,4,5]) -assert set([1,2,3]).union(frozenset([4,5])) == set([1,2,3,4,5]) +assert frozenset([1, 2, 3]).union(set([4, 5])) == frozenset([1, 2, 3, 4, 5]) +assert set([1, 2, 3]).union(frozenset([4, 5])) == set([1, 2, 3, 4, 5]) + +assert frozenset([1, 2, 3]) | set([4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert set([1, 2, 3]) | frozenset([4, 5]) == set([1, 2, 3, 4, 5]) -assert frozenset([1,2,3]) | set([4,5]) == frozenset([1,2,3,4,5]) -assert set([1,2,3]) | frozenset([4,5]) == set([1,2,3,4,5]) +assert frozenset([1, 2, 3]).intersection(set([5, 6])) == frozenset([]) +assert set([1, 2, 3]).intersection(frozenset([5, 6])) == set([]) -assert frozenset([1,2,3]).intersection(set([5,6])) == frozenset([]) -assert set([1,2,3]).intersection(frozenset([5,6])) == set([]) +assert frozenset([1, 2, 3]) & set([1, 2, 3, 4, 5]) == frozenset([1, 2, 3]) +assert set([1, 2, 3]) & frozenset([1, 2, 3, 4, 5]) == set([1, 2, 3]) -assert frozenset([1,2,3]) & set([1,2,3,4,5]) == frozenset([1,2,3]) -assert set([1,2,3]) & frozenset([1,2,3,4,5]) == set([1,2,3]) +assert frozenset([1, 2, 3]).difference(set([5, 6])) == frozenset([1, 2, 3]) +assert set([1, 2, 3]).difference(frozenset([5, 6])) == set([1, 2, 3]) -assert frozenset([1,2,3]).difference(set([5,6])) == frozenset([1,2,3]) -assert set([1,2,3]).difference(frozenset([5,6])) == set([1,2,3]) +assert frozenset([1, 2, 3]) - set([4, 5]) == frozenset([1, 2, 3]) +assert set([1, 2, 3]) - frozenset([4, 5]) == frozenset([1, 2, 3]) -assert frozenset([1,2,3]) - set([4,5]) == frozenset([1,2,3]) -assert set([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) +assert frozenset([1, 2]).__sub__(set([2, 3])) == frozenset([1]) +assert frozenset([1, 2]).__rsub__(set([2, 3])) == set([3]) +assert set([1, 2]).__sub__(frozenset([2, 3])) == set([1]) +assert set([1, 2]).__rsub__(frozenset([2, 3])) == frozenset([3]) -assert frozenset([1,2]).__sub__(set([2,3])) == frozenset([1]) -assert frozenset([1,2]).__rsub__(set([2,3])) == set([3]) -assert set([1,2]).__sub__(frozenset([2,3])) == set([1]) -assert set([1,2]).__rsub__(frozenset([2,3])) == frozenset([3]) +assert frozenset([1, 2, 3]).symmetric_difference(set([1, 2])) == frozenset([3]) +assert set([1, 2, 3]).symmetric_difference(frozenset([1, 2])) == set([3]) -assert frozenset([1,2,3]).symmetric_difference(set([1,2])) == frozenset([3]) -assert set([1,2,3]).symmetric_difference(frozenset([1,2])) == set([3]) +assert frozenset([1, 2, 3]) ^ set([4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert set([1, 2, 3]) ^ frozenset([4, 5]) == set([1, 2, 3, 4, 5]) -assert frozenset([1,2,3]) ^ set([4,5]) == frozenset([1,2,3,4,5]) -assert set([1,2,3]) ^ frozenset([4,5]) == set([1,2,3,4,5]) class A: def __hash__(self): return 1 + + class B: def __hash__(self): return 1 + s = {1, A(), B()} assert len(s) == 3 s = {True} s.add(1.0) -assert str(s) == '{True}' +assert str(s) == "{True}" + class EqObject: def __init__(self, eq): self.eq = eq + def __eq__(self, other): return self.eq + def __hash__(self): return bool(self.eq) -assert 'x' == (EqObject('x') == EqObject('x')) -s = {EqObject('x')} -assert EqObject('x') in s -assert '[]' == (EqObject('[]') == EqObject('[]')) + +assert "x" == (EqObject("x") == EqObject("x")) +s = {EqObject("x")} +assert EqObject("x") in s +assert "[]" == (EqObject("[]") == EqObject("[]")) s = {EqObject([])} assert EqObject([]) not in s x = object() @@ -372,8 +416,8 @@ def __hash__(self): assert frozenset().__ne__(1) == NotImplemented empty_set = set() -non_empty_set = set([1,2,3]) -set_from_literal = {1,2,3} +non_empty_set = set([1, 2, 3]) +set_from_literal = {1, 2, 3} assert 1 in non_empty_set assert 4 not in non_empty_set @@ -382,8 +426,8 @@ def __hash__(self): assert 4 not in set_from_literal # TODO: Assert that empty aruguments raises exception. -non_empty_set.add('a') -assert 'a' in non_empty_set +non_empty_set.add("a") +assert "a" in non_empty_set # TODO: Assert that empty arguments, or item not in set raises exception. non_empty_set.remove(1) @@ -394,8 +438,10 @@ def __hash__(self): assert repr(frozenset()) == "frozenset()" assert repr(frozenset([1, 2, 3])) == "frozenset({1, 2, 3})" + class FS(frozenset): pass + assert repr(FS()) == "FS()" assert repr(FS([1, 2, 3])) == "FS({1, 2, 3})" diff --git a/extra_tests/snippets/builtin_slice.py b/extra_tests/snippets/builtin_slice.py index b5c3a8ceb47..9a5c1bc78d8 100644 --- a/extra_tests/snippets/builtin_slice.py +++ b/extra_tests/snippets/builtin_slice.py @@ -10,16 +10,16 @@ assert a.stop == 10 assert a.step == 1 -assert slice(10).__repr__() == 'slice(None, 10, None)' -assert slice(None).__repr__() == 'slice(None, None, None)' -assert slice(0, 10, 13).__repr__() == 'slice(0, 10, 13)' -assert slice('0', 1.1, 2+3j).__repr__() == "slice('0', 1.1, (2+3j))" +assert slice(10).__repr__() == "slice(None, 10, None)" +assert slice(None).__repr__() == "slice(None, None, None)" +assert slice(0, 10, 13).__repr__() == "slice(0, 10, 13)" +assert slice("0", 1.1, 2 + 3j).__repr__() == "slice('0', 1.1, (2+3j))" assert slice(10) == slice(10) assert slice(-1) != slice(1) assert slice(0, 10, 3) != slice(0, 11, 3) -assert slice(0, None, 3) != slice(0, 'a', 3) -assert slice(0, 'a', 3) == slice(0, 'a', 3) +assert slice(0, None, 3) != slice(0, "a", 3) +assert slice(0, "a", 3) == slice(0, "a", 3) assert slice(0, 0, 0).__eq__(slice(0, 0, 0)) assert not slice(0, 0, 1).__eq__(slice(0, 0, 0)) @@ -65,29 +65,29 @@ assert not slice(0, 0, 0) > slice(0, 0, 0) assert not slice(0, 0, 0) < slice(0, 0, 0) -assert not slice(0, float('nan'), float('nan')) <= slice(0, float('nan'), 1) -assert not slice(0, float('nan'), float('nan')) <= slice(0, float('nan'), float('nan')) -assert not slice(0, float('nan'), float('nan')) >= slice(0, float('nan'), float('nan')) -assert not slice(0, float('nan'), float('nan')) < slice(0, float('nan'), float('nan')) -assert not slice(0, float('nan'), float('nan')) > slice(0, float('nan'), float('nan')) +assert not slice(0, float("nan"), float("nan")) <= slice(0, float("nan"), 1) +assert not slice(0, float("nan"), float("nan")) <= slice(0, float("nan"), float("nan")) +assert not slice(0, float("nan"), float("nan")) >= slice(0, float("nan"), float("nan")) +assert not slice(0, float("nan"), float("nan")) < slice(0, float("nan"), float("nan")) +assert not slice(0, float("nan"), float("nan")) > slice(0, float("nan"), float("nan")) -assert slice(0, float('inf'), float('inf')) >= slice(0, float('inf'), 1) -assert slice(0, float('inf'), float('inf')) <= slice(0, float('inf'), float('inf')) -assert slice(0, float('inf'), float('inf')) >= slice(0, float('inf'), float('inf')) -assert not slice(0, float('inf'), float('inf')) < slice(0, float('inf'), float('inf')) -assert not slice(0, float('inf'), float('inf')) > slice(0, float('inf'), float('inf')) +assert slice(0, float("inf"), float("inf")) >= slice(0, float("inf"), 1) +assert slice(0, float("inf"), float("inf")) <= slice(0, float("inf"), float("inf")) +assert slice(0, float("inf"), float("inf")) >= slice(0, float("inf"), float("inf")) +assert not slice(0, float("inf"), float("inf")) < slice(0, float("inf"), float("inf")) +assert not slice(0, float("inf"), float("inf")) > slice(0, float("inf"), float("inf")) assert_raises(TypeError, lambda: slice(0) < 3) assert_raises(TypeError, lambda: slice(0) > 3) assert_raises(TypeError, lambda: slice(0) <= 3) assert_raises(TypeError, lambda: slice(0) >= 3) -assert slice(None ).indices(10) == (0, 10, 1) -assert slice(None, None, 2).indices(10) == (0, 10, 2) -assert slice(1, None, 2).indices(10) == (1, 10, 2) -assert slice(None, None, -1).indices(10) == (9, -1, -1) -assert slice(None, None, -2).indices(10) == (9, -1, -2) -assert slice(3, None, -2).indices(10) == (3, -1, -2) +assert slice(None).indices(10) == (0, 10, 1) +assert slice(None, None, 2).indices(10) == (0, 10, 2) +assert slice(1, None, 2).indices(10) == (1, 10, 2) +assert slice(None, None, -1).indices(10) == (9, -1, -1) +assert slice(None, None, -2).indices(10) == (9, -1, -2) +assert slice(3, None, -2).indices(10) == (3, -1, -2) # issue 3004 tests assert slice(None, -9).indices(10) == (0, 1, 1) @@ -103,21 +103,17 @@ assert slice(None, 9, -1).indices(10) == (9, 9, -1) assert slice(None, 10, -1).indices(10) == (9, 9, -1) -assert \ - slice(-100, 100).indices(10) == \ - slice(None ).indices(10) +assert slice(-100, 100).indices(10) == slice(None).indices(10) -assert \ - slice(100, -100, -1).indices(10) == \ - slice(None, None, -1).indices(10) +assert slice(100, -100, -1).indices(10) == slice(None, None, -1).indices(10) -assert slice(-100, 100, 2).indices(10) == (0, 10, 2) +assert slice(-100, 100, 2).indices(10) == (0, 10, 2) try: - slice(None, None, 0) - assert "zero step" == "throws an exception" + slice(None, None, 0) + assert "zero step" == "throws an exception" except: - pass + pass a = [] b = [1, 2] @@ -167,8 +163,8 @@ def __index__(self): return self.x -assert c[CustomIndex(1):CustomIndex(3)] == [1, 2] -assert d[CustomIndex(1):CustomIndex(3)] == "23" +assert c[CustomIndex(1) : CustomIndex(3)] == [1, 2] +assert d[CustomIndex(1) : CustomIndex(3)] == "23" def test_all_slices(): @@ -176,7 +172,7 @@ def test_all_slices(): test all possible slices except big number """ - mod = __import__('cpython_generated_slices') + mod = __import__("cpython_generated_slices") ll = mod.LL start = mod.START diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index cd7133e355c..415156184c4 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -1,14 +1,17 @@ -from testutils import assert_raises, AssertRaises, skip_if_unsupported +from testutils import AssertRaises, assert_raises, skip_if_unsupported assert "".__eq__(1) == NotImplemented -assert "a" == 'a' +assert "a" == "a" assert """a""" == "a" assert len(""" " "" " "" """) == 11 -assert "\"" == '"' -assert "\"" == """\"""" +assert '"' == '"' +assert '"' == """\"""" -assert "\n" == """ +assert ( + "\n" + == """ """ +) assert len(""" " \" """) == 5 assert len("é") == 1 @@ -30,7 +33,7 @@ assert repr("a") == "'a'" assert repr("can't") == '"can\'t"' assert repr('"won\'t"') == "'\"won\\'t\"'" -assert repr('\n\t') == "'\\n\\t'" +assert repr("\n\t") == "'\\n\\t'" assert str(["a", "b", "can't"]) == "['a', 'b', \"can't\"]" @@ -42,22 +45,22 @@ assert 0 * "x" == "" assert -1 * "x" == "" -assert_raises(OverflowError, lambda: 'xy' * 234234234234234234234234234234) - -a = 'Hallo' -assert a.lower() == 'hallo' -assert a.upper() == 'HALLO' -assert a.startswith('H') -assert a.startswith(('H', 1)) -assert a.startswith(('A', 'H')) -assert not a.startswith('f') -assert not a.startswith(('A', 'f')) -assert a.endswith('llo') -assert a.endswith(('lo', 1)) -assert a.endswith(('A', 'lo')) -assert not a.endswith('on') -assert not a.endswith(('A', 'll')) -assert a.zfill(8) == '000Hallo' +assert_raises(OverflowError, lambda: "xy" * 234234234234234234234234234234) + +a = "Hallo" +assert a.lower() == "hallo" +assert a.upper() == "HALLO" +assert a.startswith("H") +assert a.startswith(("H", 1)) +assert a.startswith(("A", "H")) +assert not a.startswith("f") +assert not a.startswith(("A", "f")) +assert a.endswith("llo") +assert a.endswith(("lo", 1)) +assert a.endswith(("A", "lo")) +assert not a.endswith("on") +assert not a.endswith(("A", "ll")) +assert a.zfill(8) == "000Hallo" assert a.isalnum() assert not a.isdigit() assert not a.isdecimal() @@ -65,34 +68,34 @@ assert a.istitle() assert a.isalpha() -s = '1 2 3' -assert s.split(' ', 1) == ['1', '2 3'] -assert s.rsplit(' ', 1) == ['1 2', '3'] - -b = ' hallo ' -assert b.strip() == 'hallo' -assert b.lstrip() == 'hallo ' -assert b.rstrip() == ' hallo' - -s = '^*RustPython*^' -assert s.strip('^*') == 'RustPython' -assert s.lstrip('^*') == 'RustPython*^' -assert s.rstrip('^*') == '^*RustPython' - -s = 'RustPython' -assert s.ljust(8) == 'RustPython' -assert s.rjust(8) == 'RustPython' -assert s.ljust(12) == 'RustPython ' -assert s.rjust(12) == ' RustPython' -assert s.ljust(12, '_') == 'RustPython__' -assert s.rjust(12, '_') == '__RustPython' +s = "1 2 3" +assert s.split(" ", 1) == ["1", "2 3"] +assert s.rsplit(" ", 1) == ["1 2", "3"] + +b = " hallo " +assert b.strip() == "hallo" +assert b.lstrip() == "hallo " +assert b.rstrip() == " hallo" + +s = "^*RustPython*^" +assert s.strip("^*") == "RustPython" +assert s.lstrip("^*") == "RustPython*^" +assert s.rstrip("^*") == "^*RustPython" + +s = "RustPython" +assert s.ljust(8) == "RustPython" +assert s.rjust(8) == "RustPython" +assert s.ljust(12) == "RustPython " +assert s.rjust(12) == " RustPython" +assert s.ljust(12, "_") == "RustPython__" +assert s.rjust(12, "_") == "__RustPython" # The fill character must be exactly one character long -assert_raises(TypeError, lambda: s.ljust(12, '__')) -assert_raises(TypeError, lambda: s.rjust(12, '__')) +assert_raises(TypeError, lambda: s.ljust(12, "__")) +assert_raises(TypeError, lambda: s.rjust(12, "__")) -c = 'hallo' -assert c.capitalize() == 'Hallo' -assert c.center(11, '-') == '---hallo---' +c = "hallo" +assert c.capitalize() == "Hallo" +assert c.center(11, "-") == "---hallo---" assert ["koki".center(i, "|") for i in range(3, 10)] == [ "koki", "koki", @@ -118,110 +121,157 @@ # requires CPython 3.7, and the CI currently runs with 3.6 # assert c.isascii() -assert c.index('a') == 1 -assert c.rindex('l') == 3 -assert c.find('h') == 0 -assert c.rfind('x') == -1 +assert c.index("a") == 1 +assert c.rindex("l") == 3 +assert c.find("h") == 0 +assert c.rfind("x") == -1 assert c.islower() -assert c.title() == 'Hallo' -assert c.count('l') == 2 - -assert 'aaa'.count('a') == 3 -assert 'aaa'.count('a', 1) == 2 -assert 'aaa'.count('a', 1, 2) == 1 -assert 'aaa'.count('a', 2, 2) == 0 -assert 'aaa'.count('a', 2, 1) == 0 - -assert '___a__'.find('a') == 3 -assert '___a__'.find('a', -10) == 3 -assert '___a__'.find('a', -3) == 3 -assert '___a__'.find('a', -2) == -1 -assert '___a__'.find('a', -1) == -1 -assert '___a__'.find('a', 0) == 3 -assert '___a__'.find('a', 3) == 3 -assert '___a__'.find('a', 4) == -1 -assert '___a__'.find('a', 10) == -1 -assert '___a__'.rfind('a', 3) == 3 -assert '___a__'.index('a', 3) == 3 - -assert '___a__'.find('a', 0, -10) == -1 -assert '___a__'.find('a', 0, -3) == -1 -assert '___a__'.find('a', 0, -2) == 3 -assert '___a__'.find('a', 0, -1) == 3 -assert '___a__'.find('a', 0, 0) == -1 -assert '___a__'.find('a', 0, 3) == -1 -assert '___a__'.find('a', 0, 4) == 3 -assert '___a__'.find('a', 0, 10) == 3 - -assert '___a__'.find('a', 3, 3) == -1 -assert '___a__'.find('a', 3, 4) == 3 -assert '___a__'.find('a', 4, 3) == -1 - -assert 'abcd'.startswith('b', 1) -assert 'abcd'.startswith(('b', 'z'), 1) -assert not 'abcd'.startswith('b', -4) -assert 'abcd'.startswith('b', -3) - -assert not 'abcd'.startswith('b', 3, 3) -assert 'abcd'.startswith('', 3, 3) -assert not 'abcd'.startswith('', 4, 3) - -assert ' '.isspace() -assert 'hello\nhallo\nHallo'.splitlines() == ['hello', 'hallo', 'Hallo'] -assert 'hello\nhallo\nHallo\n'.splitlines() == ['hello', 'hallo', 'Hallo'] -assert 'hello\nhallo\nHallo'.splitlines(keepends=True) == ['hello\n', 'hallo\n', 'Hallo'] -assert 'hello\nhallo\nHallo\n'.splitlines(keepends=True) == ['hello\n', 'hallo\n', 'Hallo\n'] -assert 'hello\vhallo\x0cHallo\x1cHELLO\x1dhoho\x1ehaha\x85another\u2028yetanother\u2029last\r\n.'.splitlines() == ['hello', 'hallo', 'Hallo', 'HELLO', 'hoho', 'haha', 'another', 'yetanother', 'last', '.'] -assert 'hello\vhallo\x0cHallo\x1cHELLO\x1dhoho\x1ehaha\x85another\u2028yetanother\u2029last\r\n.'.splitlines(keepends=True) == ['hello\x0b', 'hallo\x0c', 'Hallo\x1c', 'HELLO\x1d', 'hoho\x1e', 'haha\x85', 'another\u2028', 'yetanother\u2029', 'last\r\n', '.'] -assert 'abc\t12345\txyz'.expandtabs() == 'abc 12345 xyz' -assert '-'.join(['1', '2', '3']) == '1-2-3' -assert 'HALLO'.isupper() -assert "hello, my name is".partition("my ") == ('hello, ', 'my ', 'name is') -assert "hello".partition("is") == ('hello', '', '') -assert "hello, my name is".rpartition("is") == ('hello, my name ', 'is', '') -assert "hello".rpartition("is") == ('', '', 'hello') -assert not ''.isdecimal() -assert '123'.isdecimal() -assert not '\u00B2'.isdecimal() - -assert not ''.isidentifier() -assert 'python'.isidentifier() -assert '_'.isidentifier() -assert '유니코드'.isidentifier() -assert not '😂'.isidentifier() -assert not '123'.isidentifier() +assert c.title() == "Hallo" +assert c.count("l") == 2 + +assert "aaa".count("a") == 3 +assert "aaa".count("a", 1) == 2 +assert "aaa".count("a", 1, 2) == 1 +assert "aaa".count("a", 2, 2) == 0 +assert "aaa".count("a", 2, 1) == 0 + +assert "___a__".find("a") == 3 +assert "___a__".find("a", -10) == 3 +assert "___a__".find("a", -3) == 3 +assert "___a__".find("a", -2) == -1 +assert "___a__".find("a", -1) == -1 +assert "___a__".find("a", 0) == 3 +assert "___a__".find("a", 3) == 3 +assert "___a__".find("a", 4) == -1 +assert "___a__".find("a", 10) == -1 +assert "___a__".rfind("a", 3) == 3 +assert "___a__".index("a", 3) == 3 + +assert "___a__".find("a", 0, -10) == -1 +assert "___a__".find("a", 0, -3) == -1 +assert "___a__".find("a", 0, -2) == 3 +assert "___a__".find("a", 0, -1) == 3 +assert "___a__".find("a", 0, 0) == -1 +assert "___a__".find("a", 0, 3) == -1 +assert "___a__".find("a", 0, 4) == 3 +assert "___a__".find("a", 0, 10) == 3 + +assert "___a__".find("a", 3, 3) == -1 +assert "___a__".find("a", 3, 4) == 3 +assert "___a__".find("a", 4, 3) == -1 + +assert "abcd".startswith("b", 1) +assert "abcd".startswith(("b", "z"), 1) +assert not "abcd".startswith("b", -4) +assert "abcd".startswith("b", -3) + +assert not "abcd".startswith("b", 3, 3) +assert "abcd".startswith("", 3, 3) +assert not "abcd".startswith("", 4, 3) + +assert " ".isspace() +assert "hello\nhallo\nHallo".splitlines() == ["hello", "hallo", "Hallo"] +assert "hello\nhallo\nHallo\n".splitlines() == ["hello", "hallo", "Hallo"] +assert "hello\nhallo\nHallo".splitlines(keepends=True) == [ + "hello\n", + "hallo\n", + "Hallo", +] +assert "hello\nhallo\nHallo\n".splitlines(keepends=True) == [ + "hello\n", + "hallo\n", + "Hallo\n", +] +assert ( + "hello\vhallo\x0cHallo\x1cHELLO\x1dhoho\x1ehaha\x85another\u2028yetanother\u2029last\r\n.".splitlines() + == [ + "hello", + "hallo", + "Hallo", + "HELLO", + "hoho", + "haha", + "another", + "yetanother", + "last", + ".", + ] +) +assert ( + "hello\vhallo\x0cHallo\x1cHELLO\x1dhoho\x1ehaha\x85another\u2028yetanother\u2029last\r\n.".splitlines( + keepends=True + ) + == [ + "hello\x0b", + "hallo\x0c", + "Hallo\x1c", + "HELLO\x1d", + "hoho\x1e", + "haha\x85", + "another\u2028", + "yetanother\u2029", + "last\r\n", + ".", + ] +) +assert "abc\t12345\txyz".expandtabs() == "abc 12345 xyz" +assert "-".join(["1", "2", "3"]) == "1-2-3" +assert "HALLO".isupper() +assert "hello, my name is".partition("my ") == ("hello, ", "my ", "name is") +assert "hello".partition("is") == ("hello", "", "") +assert "hello, my name is".rpartition("is") == ("hello, my name ", "is", "") +assert "hello".rpartition("is") == ("", "", "hello") +assert not "".isdecimal() +assert "123".isdecimal() +assert not "\u00b2".isdecimal() + +assert not "".isidentifier() +assert "python".isidentifier() +assert "_".isidentifier() +assert "유니코드".isidentifier() +assert not "😂".isidentifier() +assert not "123".isidentifier() # String Formatting assert "{} {}".format(1, 2) == "1 2" assert "{0} {1}".format(2, 3) == "2 3" assert "--{:s>4}--".format(1) == "--sss1--" assert "{keyword} {0}".format(1, keyword=2) == "2 1" -assert "repr() shows quotes: {!r}; str() doesn't: {!s}".format( - 'test1', 'test2' -) == "repr() shows quotes: 'test1'; str() doesn't: test2", 'Output: {!r}, {!s}'.format('test1', 'test2') +assert ( + "repr() shows quotes: {!r}; str() doesn't: {!s}".format("test1", "test2") + == "repr() shows quotes: 'test1'; str() doesn't: test2" +), "Output: {!r}, {!s}".format("test1", "test2") class Foo: def __str__(self): - return 'str(Foo)' + return "str(Foo)" def __repr__(self): - return 'repr(Foo)' + return "repr(Foo)" f = Foo() -assert "{} {!s} {!r} {!a}".format(f, f, f, f) == 'str(Foo) str(Foo) repr(Foo) repr(Foo)' -assert "{foo} {foo!s} {foo!r} {foo!a}".format(foo=f) == 'str(Foo) str(Foo) repr(Foo) repr(Foo)' +assert "{} {!s} {!r} {!a}".format(f, f, f, f) == "str(Foo) str(Foo) repr(Foo) repr(Foo)" +assert ( + "{foo} {foo!s} {foo!r} {foo!a}".format(foo=f) + == "str(Foo) str(Foo) repr(Foo) repr(Foo)" +) # assert '{} {!r} {:10} {!r:10} {foo!r:10} {foo!r} {foo}'.format('txt1', 'txt2', 'txt3', 'txt4', 'txt5', foo='bar') # Printf-style String formatting assert "%d %d" % (1, 2) == "1 2" -assert "%*c " % (3, '❤') == " ❤ " -assert "%(first)s %(second)s" % {'second': 'World!', 'first': "Hello,"} == "Hello, World!" -assert "%(key())s" % {'key()': 'aaa'} +assert "%*c " % (3, "❤") == " ❤ " +assert ( + "%(first)s %(second)s" % {"second": "World!", "first": "Hello,"} == "Hello, World!" +) +assert "%(key())s" % {"key()": "aaa"} assert "%s %a %r" % (f, f, f) == "str(Foo) repr(Foo) repr(Foo)" -assert "repr() shows quotes: %r; str() doesn't: %s" % ("test1", "test2") == "repr() shows quotes: 'test1'; str() doesn't: test2" +assert ( + "repr() shows quotes: %r; str() doesn't: %s" % ("test1", "test2") + == "repr() shows quotes: 'test1'; str() doesn't: test2" +) assert "%f" % (1.2345) == "1.234500" assert "%+f" % (1.2345) == "+1.234500" assert "% f" % (1.2345) == " 1.234500" @@ -229,111 +279,132 @@ def __repr__(self): assert "%f" % (1.23456789012) == "1.234568" assert "%f" % (123) == "123.000000" assert "%f" % (-123) == "-123.000000" -assert "%e" % 1 == '1.000000e+00' -assert "%e" % 0 == '0.000000e+00' -assert "%e" % 0.1 == '1.000000e-01' -assert "%e" % 10 == '1.000000e+01' -assert "%.10e" % 1.2345678901234567890 == '1.2345678901e+00' -assert '%e' % float('nan') == 'nan' -assert '%e' % float('-nan') == 'nan' -assert '%E' % float('nan') == 'NAN' -assert '%e' % float('inf') == 'inf' -assert '%e' % float('-inf') == '-inf' -assert '%E' % float('inf') == 'INF' -assert "%g" % 123456.78901234567890 == '123457' -assert "%.0g" % 123456.78901234567890 == '1e+05' -assert "%.1g" % 123456.78901234567890 == '1e+05' -assert "%.2g" % 123456.78901234567890 == '1.2e+05' -assert "%g" % 1234567.8901234567890 == '1.23457e+06' -assert "%.0g" % 1234567.8901234567890 == '1e+06' -assert "%.1g" % 1234567.8901234567890 == '1e+06' -assert "%.2g" % 1234567.8901234567890 == '1.2e+06' -assert "%.3g" % 1234567.8901234567890 == '1.23e+06' -assert "%.5g" % 1234567.8901234567890 == '1.2346e+06' -assert "%.6g" % 1234567.8901234567890 == '1.23457e+06' -assert "%.7g" % 1234567.8901234567890 == '1234568' -assert "%.8g" % 1234567.8901234567890 == '1234567.9' -assert "%G" % 123456.78901234567890 == '123457' -assert "%.0G" % 123456.78901234567890 == '1E+05' -assert "%.1G" % 123456.78901234567890 == '1E+05' -assert "%.2G" % 123456.78901234567890 == '1.2E+05' -assert "%G" % 1234567.8901234567890 == '1.23457E+06' -assert "%.0G" % 1234567.8901234567890 == '1E+06' -assert "%.1G" % 1234567.8901234567890 == '1E+06' -assert "%.2G" % 1234567.8901234567890 == '1.2E+06' -assert "%.3G" % 1234567.8901234567890 == '1.23E+06' -assert "%.5G" % 1234567.8901234567890 == '1.2346E+06' -assert "%.6G" % 1234567.8901234567890 == '1.23457E+06' -assert "%.7G" % 1234567.8901234567890 == '1234568' -assert "%.8G" % 1234567.8901234567890 == '1234567.9' -assert '%g' % 0.12345678901234567890 == '0.123457' -assert '%g' % 0.12345678901234567890e-1 == '0.0123457' -assert '%g' % 0.12345678901234567890e-2 == '0.00123457' -assert '%g' % 0.12345678901234567890e-3 == '0.000123457' -assert '%g' % 0.12345678901234567890e-4 == '1.23457e-05' -assert '%g' % 0.12345678901234567890e-5 == '1.23457e-06' -assert '%.6g' % 0.12345678901234567890e-5 == '1.23457e-06' -assert '%.10g' % 0.12345678901234567890e-5 == '1.23456789e-06' -assert '%.20g' % 0.12345678901234567890e-5 == '1.2345678901234567384e-06' -assert '%G' % 0.12345678901234567890 == '0.123457' -assert '%G' % 0.12345678901234567890E-1 == '0.0123457' -assert '%G' % 0.12345678901234567890E-2 == '0.00123457' -assert '%G' % 0.12345678901234567890E-3 == '0.000123457' -assert '%G' % 0.12345678901234567890E-4 == '1.23457E-05' -assert '%G' % 0.12345678901234567890E-5 == '1.23457E-06' -assert '%.6G' % 0.12345678901234567890E-5 == '1.23457E-06' -assert '%.10G' % 0.12345678901234567890E-5 == '1.23456789E-06' -assert '%.20G' % 0.12345678901234567890E-5 == '1.2345678901234567384E-06' -assert '%g' % float('nan') == 'nan' -assert '%g' % float('-nan') == 'nan' -assert '%G' % float('nan') == 'NAN' -assert '%g' % float('inf') == 'inf' -assert '%g' % float('-inf') == '-inf' -assert '%G' % float('inf') == 'INF' -assert "%.0g" % 1.020e-13 == '1e-13' -assert "%.0g" % 1.020e-13 == '1e-13' -assert "%.1g" % 1.020e-13 == '1e-13' -assert "%.2g" % 1.020e-13 == '1e-13' -assert "%.3g" % 1.020e-13 == '1.02e-13' -assert "%.4g" % 1.020e-13 == '1.02e-13' -assert "%.5g" % 1.020e-13 == '1.02e-13' -assert "%.6g" % 1.020e-13 == '1.02e-13' -assert "%.7g" % 1.020e-13 == '1.02e-13' -assert "%g" % 1.020e-13 == '1.02e-13' -assert "%g" % 1.020e-4 == '0.000102' - -assert_raises(TypeError, lambda: "My name is %s and I'm %(age)d years old" % ("Foo", 25), _msg='format requires a mapping') -assert_raises(TypeError, lambda: "My name is %(name)s" % "Foo", _msg='format requires a mapping') -assert_raises(ValueError, lambda: "This %(food}s is great!" % {"food": "cookie"}, _msg='incomplete format key') -assert_raises(ValueError, lambda: "My name is %" % "Foo", _msg='incomplete format') - -assert 'a' < 'b' -assert 'a' <= 'b' -assert 'a' <= 'a' -assert 'z' > 'b' -assert 'z' >= 'b' -assert 'a' >= 'a' +assert "%e" % 1 == "1.000000e+00" +assert "%e" % 0 == "0.000000e+00" +assert "%e" % 0.1 == "1.000000e-01" +assert "%e" % 10 == "1.000000e+01" +assert "%.10e" % 1.2345678901234567890 == "1.2345678901e+00" +assert "%e" % float("nan") == "nan" +assert "%e" % float("-nan") == "nan" +assert "%E" % float("nan") == "NAN" +assert "%e" % float("inf") == "inf" +assert "%e" % float("-inf") == "-inf" +assert "%E" % float("inf") == "INF" +assert "%g" % 123456.78901234567890 == "123457" +assert "%.0g" % 123456.78901234567890 == "1e+05" +assert "%.1g" % 123456.78901234567890 == "1e+05" +assert "%.2g" % 123456.78901234567890 == "1.2e+05" +assert "%g" % 1234567.8901234567890 == "1.23457e+06" +assert "%.0g" % 1234567.8901234567890 == "1e+06" +assert "%.1g" % 1234567.8901234567890 == "1e+06" +assert "%.2g" % 1234567.8901234567890 == "1.2e+06" +assert "%.3g" % 1234567.8901234567890 == "1.23e+06" +assert "%.5g" % 1234567.8901234567890 == "1.2346e+06" +assert "%.6g" % 1234567.8901234567890 == "1.23457e+06" +assert "%.7g" % 1234567.8901234567890 == "1234568" +assert "%.8g" % 1234567.8901234567890 == "1234567.9" +assert "%G" % 123456.78901234567890 == "123457" +assert "%.0G" % 123456.78901234567890 == "1E+05" +assert "%.1G" % 123456.78901234567890 == "1E+05" +assert "%.2G" % 123456.78901234567890 == "1.2E+05" +assert "%G" % 1234567.8901234567890 == "1.23457E+06" +assert "%.0G" % 1234567.8901234567890 == "1E+06" +assert "%.1G" % 1234567.8901234567890 == "1E+06" +assert "%.2G" % 1234567.8901234567890 == "1.2E+06" +assert "%.3G" % 1234567.8901234567890 == "1.23E+06" +assert "%.5G" % 1234567.8901234567890 == "1.2346E+06" +assert "%.6G" % 1234567.8901234567890 == "1.23457E+06" +assert "%.7G" % 1234567.8901234567890 == "1234568" +assert "%.8G" % 1234567.8901234567890 == "1234567.9" +assert "%g" % 0.12345678901234567890 == "0.123457" +assert "%g" % 0.12345678901234567890e-1 == "0.0123457" +assert "%g" % 0.12345678901234567890e-2 == "0.00123457" +assert "%g" % 0.12345678901234567890e-3 == "0.000123457" +assert "%g" % 0.12345678901234567890e-4 == "1.23457e-05" +assert "%g" % 0.12345678901234567890e-5 == "1.23457e-06" +assert "%.6g" % 0.12345678901234567890e-5 == "1.23457e-06" +assert "%.10g" % 0.12345678901234567890e-5 == "1.23456789e-06" +assert "%.20g" % 0.12345678901234567890e-5 == "1.2345678901234567384e-06" +assert "%G" % 0.12345678901234567890 == "0.123457" +assert "%G" % 0.12345678901234567890e-1 == "0.0123457" +assert "%G" % 0.12345678901234567890e-2 == "0.00123457" +assert "%G" % 0.12345678901234567890e-3 == "0.000123457" +assert "%G" % 0.12345678901234567890e-4 == "1.23457E-05" +assert "%G" % 0.12345678901234567890e-5 == "1.23457E-06" +assert "%.6G" % 0.12345678901234567890e-5 == "1.23457E-06" +assert "%.10G" % 0.12345678901234567890e-5 == "1.23456789E-06" +assert "%.20G" % 0.12345678901234567890e-5 == "1.2345678901234567384E-06" +assert "%g" % float("nan") == "nan" +assert "%g" % float("-nan") == "nan" +assert "%G" % float("nan") == "NAN" +assert "%g" % float("inf") == "inf" +assert "%g" % float("-inf") == "-inf" +assert "%G" % float("inf") == "INF" +assert "%.0g" % 1.020e-13 == "1e-13" +assert "%.0g" % 1.020e-13 == "1e-13" +assert "%.1g" % 1.020e-13 == "1e-13" +assert "%.2g" % 1.020e-13 == "1e-13" +assert "%.3g" % 1.020e-13 == "1.02e-13" +assert "%.4g" % 1.020e-13 == "1.02e-13" +assert "%.5g" % 1.020e-13 == "1.02e-13" +assert "%.6g" % 1.020e-13 == "1.02e-13" +assert "%.7g" % 1.020e-13 == "1.02e-13" +assert "%g" % 1.020e-13 == "1.02e-13" +assert "%g" % 1.020e-4 == "0.000102" + +assert_raises( + TypeError, + lambda: "My name is %s and I'm %(age)d years old" % ("Foo", 25), + _msg="format requires a mapping", +) +assert_raises( + TypeError, lambda: "My name is %(name)s" % "Foo", _msg="format requires a mapping" +) +assert_raises( + ValueError, + lambda: "This %(food}s is great!" % {"food": "cookie"}, + _msg="incomplete format key", +) +assert_raises(ValueError, lambda: "My name is %" % "Foo", _msg="incomplete format") + +assert "a" < "b" +assert "a" <= "b" +assert "a" <= "a" +assert "z" > "b" +assert "z" >= "b" +assert "a" >= "a" # str.translate -assert "abc".translate({97: '🎅', 98: None, 99: "xd"}) == "🎅xd" +assert "abc".translate({97: "🎅", 98: None, 99: "xd"}) == "🎅xd" # str.maketrans assert str.maketrans({"a": "abc", "b": None, "c": 33}) == {97: "abc", 98: None, 99: 33} -assert str.maketrans("hello", "world", "rust") == {104: 119, 101: 111, 108: 108, 111: 100, 114: None, 117: None, 115: None, 116: None} +assert str.maketrans("hello", "world", "rust") == { + 104: 119, + 101: 111, + 108: 108, + 111: 100, + 114: None, + 117: None, + 115: None, + 116: None, +} + def try_mutate_str(): - word = "word" - word[0] = 'x' + word = "word" + word[0] = "x" + assert_raises(TypeError, try_mutate_str) -ss = ['Hello', '안녕', '👋'] -bs = [b'Hello', b'\xec\x95\x88\xeb\x85\x95', b'\xf0\x9f\x91\x8b'] +ss = ["Hello", "안녕", "👋"] +bs = [b"Hello", b"\xec\x95\x88\xeb\x85\x95", b"\xf0\x9f\x91\x8b"] for s, b in zip(ss, bs): assert s.encode() == b -for s, b, e in zip(ss, bs, ['u8', 'U8', 'utf-8', 'UTF-8', 'utf_8']): +for s, b, e in zip(ss, bs, ["u8", "U8", "utf-8", "UTF-8", "utf_8"]): assert s.encode(e) == b # assert s.encode(encoding=e) == b @@ -349,9 +420,9 @@ def try_mutate_str(): assert "\u0037" == "7" assert "\u0040" == "@" assert "\u0041" == "A" -assert "\u00BE" == "¾" +assert "\u00be" == "¾" assert "\u9487" == "钇" -assert "\U0001F609" == "😉" +assert "\U0001f609" == "😉" # test str iter iterable_str = "12345678😉" @@ -383,16 +454,16 @@ def try_mutate_str(): assert next(str_iter_reversed, None) == None assert_raises(StopIteration, next, str_iter_reversed) -assert str.__rmod__('%i', 30) == NotImplemented -assert_raises(TypeError, lambda: str.__rmod__(30, '%i')) +assert str.__rmod__("%i", 30) == NotImplemented +assert_raises(TypeError, lambda: str.__rmod__(30, "%i")) # test str index -index_str = 'Rust Python' +index_str = "Rust Python" -assert index_str[0] == 'R' -assert index_str[-1] == 'n' +assert index_str[0] == "R" +assert index_str[-1] == "n" -assert_raises(TypeError, lambda: index_str['a']) +assert_raises(TypeError, lambda: index_str["a"]) assert chr(9).__repr__() == "'\\t'" assert chr(99).__repr__() == "'c'" @@ -424,321 +495,330 @@ def try_mutate_str(): # >>> '{x} {y}'.format_map({'x': 1, 'y': 2}) # '1 2' -assert '{x} {y}'.format_map({'x': 1, 'y': 2}) == '1 2' +assert "{x} {y}".format_map({"x": 1, "y": 2}) == "1 2" # >>> '{x:04d}'.format_map({'x': 1}) # '0001' -assert '{x:04d}'.format_map({'x': 1}) == '0001' +assert "{x:04d}".format_map({"x": 1}) == "0001" # >>> '{x} {y}'.format_map('foo') # Traceback (most recent call last): # File "<stdin>", line 1, in <module> # TypeError: string indices must be integers with AssertRaises(TypeError, None): - '{x} {y}'.format_map('foo') + "{x} {y}".format_map("foo") # >>> '{x} {y}'.format_map(['foo']) # Traceback (most recent call last): # File "<stdin>", line 1, in <module> # TypeError: list indices must be integers or slices, not str with AssertRaises(TypeError, None): - '{x} {y}'.format_map(['foo']) + "{x} {y}".format_map(["foo"]) # >>> '{x} {y}'.format_map() # Traceback (most recent call last): # File "<stdin>", line 1, in <module> # TypeError: format_map() takes exactly one argument (0 given) -with AssertRaises(TypeError, msg='TypeError: format_map() takes exactly one argument (0 given)'): - '{x} {y}'.format_map(), +with AssertRaises( + TypeError, msg="TypeError: format_map() takes exactly one argument (0 given)" +): + ("{x} {y}".format_map(),) # >>> '{x} {y}'.format_map('foo', 'bar') # Traceback (most recent call last): # File "<stdin>", line 1, in <module> # TypeError: format_map() takes exactly one argument (2 given) -with AssertRaises(TypeError, msg='TypeError: format_map() takes exactly one argument (2 given)'): - '{x} {y}'.format_map('foo', 'bar') +with AssertRaises( + TypeError, msg="TypeError: format_map() takes exactly one argument (2 given)" +): + "{x} {y}".format_map("foo", "bar") # >>> '{x} {y}'.format_map({'x': 1}) # Traceback (most recent call last): # File "<stdin>", line 1, in <module> # KeyError: 'y' with AssertRaises(KeyError, msg="KeyError: 'y'"): - '{x} {y}'.format_map({'x': 1}) + "{x} {y}".format_map({"x": 1}) # >>> '{x} {y}'.format_map({'x': 1, 'z': 2}) # Traceback (most recent call last): # File "<stdin>", line 1, in <module> # KeyError: 'y' with AssertRaises(KeyError, msg="KeyError: 'y'"): - '{x} {y}'.format_map({'x': 1, 'z': 2}) + "{x} {y}".format_map({"x": 1, "z": 2}) # >>> '{{literal}}'.format_map('foo') # '{literal}' -assert '{{literal}}'.format_map('foo') == '{literal}' +assert "{{literal}}".format_map("foo") == "{literal}" # test formatting float values -assert f'{5:f}' == '5.000000' -assert f'{-5:f}' == '-5.000000' -assert f'{5.0:f}' == '5.000000' -assert f'{-5.0:f}' == '-5.000000' -assert f'{5:.2f}' == '5.00' -assert f'{5.0:.2f}' == '5.00' -assert f'{-5:.2f}' == '-5.00' -assert f'{-5.0:.2f}' == '-5.00' -assert f'{5.0:04f}' == '5.000000' -assert f'{5.1234:+f}' == '+5.123400' -assert f'{5.1234: f}' == ' 5.123400' -assert f'{5.1234:-f}' == '5.123400' -assert f'{-5.1234:-f}' == '-5.123400' -assert f'{1.0:+}' == '+1.0' -assert f'--{1.0:f>4}--' == '--f1.0--' -assert f'--{1.0:f<4}--' == '--1.0f--' -assert f'--{1.0:d^4}--' == '--1.0d--' -assert f'--{1.0:d^5}--' == '--d1.0d--' -assert f'--{1.1:f>6}--' == '--fff1.1--' -assert '{}'.format(float('nan')) == 'nan' -assert '{:f}'.format(float('nan')) == 'nan' -assert '{:f}'.format(float('-nan')) == 'nan' -assert '{:F}'.format(float('nan')) == 'NAN' -assert '{}'.format(float('inf')) == 'inf' -assert '{:f}'.format(float('inf')) == 'inf' -assert '{:f}'.format(float('-inf')) == '-inf' -assert '{:F}'.format(float('inf')) == 'INF' -assert f'{1234567890.1234:,.2f}' == '1,234,567,890.12' -assert f'{1234567890.1234:_.2f}' == '1_234_567_890.12' +assert f"{5:f}" == "5.000000" +assert f"{-5:f}" == "-5.000000" +assert f"{5.0:f}" == "5.000000" +assert f"{-5.0:f}" == "-5.000000" +assert f"{5:.2f}" == "5.00" +assert f"{5.0:.2f}" == "5.00" +assert f"{-5:.2f}" == "-5.00" +assert f"{-5.0:.2f}" == "-5.00" +assert f"{5.0:04f}" == "5.000000" +assert f"{5.1234:+f}" == "+5.123400" +assert f"{5.1234: f}" == " 5.123400" +assert f"{5.1234:-f}" == "5.123400" +assert f"{-5.1234:-f}" == "-5.123400" +assert f"{1.0:+}" == "+1.0" +assert f"--{1.0:f>4}--" == "--f1.0--" +assert f"--{1.0:f<4}--" == "--1.0f--" +assert f"--{1.0:d^4}--" == "--1.0d--" +assert f"--{1.0:d^5}--" == "--d1.0d--" +assert f"--{1.1:f>6}--" == "--fff1.1--" +assert "{}".format(float("nan")) == "nan" +assert "{:f}".format(float("nan")) == "nan" +assert "{:f}".format(float("-nan")) == "nan" +assert "{:F}".format(float("nan")) == "NAN" +assert "{}".format(float("inf")) == "inf" +assert "{:f}".format(float("inf")) == "inf" +assert "{:f}".format(float("-inf")) == "-inf" +assert "{:F}".format(float("inf")) == "INF" +assert f"{1234567890.1234:,.2f}" == "1,234,567,890.12" +assert f"{1234567890.1234:_.2f}" == "1_234_567_890.12" with AssertRaises(ValueError, msg="Unknown format code 'd' for object of type 'float'"): - f'{5.0:04d}' + f"{5.0:04d}" # Test % formatting -assert f'{10:%}' == '1000.000000%' -assert f'{10.0:%}' == '1000.000000%' -assert f'{10.0:.2%}' == '1000.00%' -assert f'{10.0:.8%}' == '1000.00000000%' -assert f'{-10:%}' == '-1000.000000%' -assert f'{-10.0:%}' == '-1000.000000%' -assert f'{-10.0:.2%}' == '-1000.00%' -assert f'{-10.0:.8%}' == '-1000.00000000%' -assert '{:%}'.format(float('nan')) == 'nan%' -assert '{:.2%}'.format(float('nan')) == 'nan%' -assert '{:%}'.format(float('inf')) == 'inf%' -assert '{:.2%}'.format(float('inf')) == 'inf%' -with AssertRaises(ValueError, msg='Invalid format specifier'): - f'{10.0:%3}' +assert f"{10:%}" == "1000.000000%" +assert f"{10.0:%}" == "1000.000000%" +assert f"{10.0:.2%}" == "1000.00%" +assert f"{10.0:.8%}" == "1000.00000000%" +assert f"{-10:%}" == "-1000.000000%" +assert f"{-10.0:%}" == "-1000.000000%" +assert f"{-10.0:.2%}" == "-1000.00%" +assert f"{-10.0:.8%}" == "-1000.00000000%" +assert "{:%}".format(float("nan")) == "nan%" +assert "{:.2%}".format(float("nan")) == "nan%" +assert "{:%}".format(float("inf")) == "inf%" +assert "{:.2%}".format(float("inf")) == "inf%" +with AssertRaises(ValueError, msg="Invalid format specifier"): + f"{10.0:%3}" # Test e & E formatting -assert '{:e}'.format(10) == '1.000000e+01' -assert '{:.2e}'.format(11) == '1.10e+01' -assert '{:e}'.format(10.0) == '1.000000e+01' -assert '{:e}'.format(-10.0) == '-1.000000e+01' -assert '{:.2e}'.format(10.0) == '1.00e+01' -assert '{:.2e}'.format(-10.0) == '-1.00e+01' -assert '{:.2e}'.format(10.1) == '1.01e+01' -assert '{:.2e}'.format(-10.1) == '-1.01e+01' -assert '{:.2e}'.format(10.001) == '1.00e+01' -assert '{:.4e}'.format(100.234) == '1.0023e+02' -assert '{:.5e}'.format(100.234) == '1.00234e+02' -assert '{:.2E}'.format(10.0) == '1.00E+01' -assert '{:.2E}'.format(-10.0) == '-1.00E+01' -assert '{:e}'.format(float('nan')) == 'nan' -assert '{:e}'.format(float('-nan')) == 'nan' -assert '{:E}'.format(float('nan')) == 'NAN' -assert '{:e}'.format(float('inf')) == 'inf' -assert '{:e}'.format(float('-inf')) == '-inf' -assert '{:E}'.format(float('inf')) == 'INF' +assert "{:e}".format(10) == "1.000000e+01" +assert "{:.2e}".format(11) == "1.10e+01" +assert "{:e}".format(10.0) == "1.000000e+01" +assert "{:e}".format(-10.0) == "-1.000000e+01" +assert "{:.2e}".format(10.0) == "1.00e+01" +assert "{:.2e}".format(-10.0) == "-1.00e+01" +assert "{:.2e}".format(10.1) == "1.01e+01" +assert "{:.2e}".format(-10.1) == "-1.01e+01" +assert "{:.2e}".format(10.001) == "1.00e+01" +assert "{:.4e}".format(100.234) == "1.0023e+02" +assert "{:.5e}".format(100.234) == "1.00234e+02" +assert "{:.2E}".format(10.0) == "1.00E+01" +assert "{:.2E}".format(-10.0) == "-1.00E+01" +assert "{:e}".format(float("nan")) == "nan" +assert "{:e}".format(float("-nan")) == "nan" +assert "{:E}".format(float("nan")) == "NAN" +assert "{:e}".format(float("inf")) == "inf" +assert "{:e}".format(float("-inf")) == "-inf" +assert "{:E}".format(float("inf")) == "INF" # Test g & G formatting -assert '{:g}'.format(10.0) == '10' -assert '{:g}'.format(100000.0) == '100000' -assert '{:g}'.format(123456.78901234567890) == '123457' -assert '{:.0g}'.format(123456.78901234567890) == '1e+05' -assert '{:.1g}'.format(123456.78901234567890) == '1e+05' -assert '{:.2g}'.format(123456.78901234567890) == '1.2e+05' -assert '{:g}'.format(1234567.8901234567890) == '1.23457e+06' -assert '{:.0g}'.format(1234567.8901234567890) == '1e+06' -assert '{:.1g}'.format(1234567.8901234567890) == '1e+06' -assert '{:.2g}'.format(1234567.8901234567890) == '1.2e+06' -assert '{:.3g}'.format(1234567.8901234567890) == '1.23e+06' -assert '{:.5g}'.format(1234567.8901234567890) == '1.2346e+06' -assert '{:.6g}'.format(1234567.8901234567890) == '1.23457e+06' -assert '{:.7g}'.format(1234567.8901234567890) == '1234568' -assert '{:.8g}'.format(1234567.8901234567890) == '1234567.9' -assert '{:G}'.format(123456.78901234567890) == '123457' -assert '{:.0G}'.format(123456.78901234567890) == '1E+05' -assert '{:.1G}'.format(123456.78901234567890) == '1E+05' -assert '{:.2G}'.format(123456.78901234567890) == '1.2E+05' -assert '{:G}'.format(1234567.8901234567890) == '1.23457E+06' -assert '{:.0G}'.format(1234567.8901234567890) == '1E+06' -assert '{:.1G}'.format(1234567.8901234567890) == '1E+06' -assert '{:.2G}'.format(1234567.8901234567890) == '1.2E+06' -assert '{:.3G}'.format(1234567.8901234567890) == '1.23E+06' -assert '{:.5G}'.format(1234567.8901234567890) == '1.2346E+06' -assert '{:.6G}'.format(1234567.8901234567890) == '1.23457E+06' -assert '{:.7G}'.format(1234567.8901234567890) == '1234568' -assert '{:.8G}'.format(1234567.8901234567890) == '1234567.9' -assert '{:g}'.format(0.12345678901234567890) == '0.123457' -assert '{:g}'.format(0.12345678901234567890e-1) == '0.0123457' -assert '{:g}'.format(0.12345678901234567890e-2) == '0.00123457' -assert '{:g}'.format(0.12345678901234567890e-3) == '0.000123457' -assert '{:g}'.format(0.12345678901234567890e-4) == '1.23457e-05' -assert '{:g}'.format(0.12345678901234567890e-5) == '1.23457e-06' -assert '{:.6g}'.format(0.12345678901234567890e-5) == '1.23457e-06' -assert '{:.10g}'.format(0.12345678901234567890e-5) == '1.23456789e-06' -assert '{:.20g}'.format(0.12345678901234567890e-5) == '1.2345678901234567384e-06' -assert '{:G}'.format(0.12345678901234567890) == '0.123457' -assert '{:G}'.format(0.12345678901234567890E-1) == '0.0123457' -assert '{:G}'.format(0.12345678901234567890E-2) == '0.00123457' -assert '{:G}'.format(0.12345678901234567890E-3) == '0.000123457' -assert '{:G}'.format(0.12345678901234567890E-4) == '1.23457E-05' -assert '{:G}'.format(0.12345678901234567890E-5) == '1.23457E-06' -assert '{:.6G}'.format(0.12345678901234567890E-5) == '1.23457E-06' -assert '{:.10G}'.format(0.12345678901234567890E-5) == '1.23456789E-06' -assert '{:.20G}'.format(0.12345678901234567890E-5) == '1.2345678901234567384E-06' -assert '{:g}'.format(float('nan')) == 'nan' -assert '{:g}'.format(float('-nan')) == 'nan' -assert '{:G}'.format(float('nan')) == 'NAN' -assert '{:g}'.format(float('inf')) == 'inf' -assert '{:g}'.format(float('-inf')) == '-inf' -assert '{:G}'.format(float('inf')) == 'INF' -assert '{:.0g}'.format(1.020e-13) == '1e-13' -assert '{:.0g}'.format(1.020e-13) == '1e-13' -assert '{:.1g}'.format(1.020e-13) == '1e-13' -assert '{:.2g}'.format(1.020e-13) == '1e-13' -assert '{:.3g}'.format(1.020e-13) == '1.02e-13' -assert '{:.4g}'.format(1.020e-13) == '1.02e-13' -assert '{:.5g}'.format(1.020e-13) == '1.02e-13' -assert '{:.6g}'.format(1.020e-13) == '1.02e-13' -assert '{:.7g}'.format(1.020e-13) == '1.02e-13' -assert '{:g}'.format(1.020e-13) == '1.02e-13' -assert "{:g}".format(1.020e-4) == '0.000102' +assert "{:g}".format(10.0) == "10" +assert "{:g}".format(100000.0) == "100000" +assert "{:g}".format(123456.78901234567890) == "123457" +assert "{:.0g}".format(123456.78901234567890) == "1e+05" +assert "{:.1g}".format(123456.78901234567890) == "1e+05" +assert "{:.2g}".format(123456.78901234567890) == "1.2e+05" +assert "{:g}".format(1234567.8901234567890) == "1.23457e+06" +assert "{:.0g}".format(1234567.8901234567890) == "1e+06" +assert "{:.1g}".format(1234567.8901234567890) == "1e+06" +assert "{:.2g}".format(1234567.8901234567890) == "1.2e+06" +assert "{:.3g}".format(1234567.8901234567890) == "1.23e+06" +assert "{:.5g}".format(1234567.8901234567890) == "1.2346e+06" +assert "{:.6g}".format(1234567.8901234567890) == "1.23457e+06" +assert "{:.7g}".format(1234567.8901234567890) == "1234568" +assert "{:.8g}".format(1234567.8901234567890) == "1234567.9" +assert "{:G}".format(123456.78901234567890) == "123457" +assert "{:.0G}".format(123456.78901234567890) == "1E+05" +assert "{:.1G}".format(123456.78901234567890) == "1E+05" +assert "{:.2G}".format(123456.78901234567890) == "1.2E+05" +assert "{:G}".format(1234567.8901234567890) == "1.23457E+06" +assert "{:.0G}".format(1234567.8901234567890) == "1E+06" +assert "{:.1G}".format(1234567.8901234567890) == "1E+06" +assert "{:.2G}".format(1234567.8901234567890) == "1.2E+06" +assert "{:.3G}".format(1234567.8901234567890) == "1.23E+06" +assert "{:.5G}".format(1234567.8901234567890) == "1.2346E+06" +assert "{:.6G}".format(1234567.8901234567890) == "1.23457E+06" +assert "{:.7G}".format(1234567.8901234567890) == "1234568" +assert "{:.8G}".format(1234567.8901234567890) == "1234567.9" +assert "{:g}".format(0.12345678901234567890) == "0.123457" +assert "{:g}".format(0.12345678901234567890e-1) == "0.0123457" +assert "{:g}".format(0.12345678901234567890e-2) == "0.00123457" +assert "{:g}".format(0.12345678901234567890e-3) == "0.000123457" +assert "{:g}".format(0.12345678901234567890e-4) == "1.23457e-05" +assert "{:g}".format(0.12345678901234567890e-5) == "1.23457e-06" +assert "{:.6g}".format(0.12345678901234567890e-5) == "1.23457e-06" +assert "{:.10g}".format(0.12345678901234567890e-5) == "1.23456789e-06" +assert "{:.20g}".format(0.12345678901234567890e-5) == "1.2345678901234567384e-06" +assert "{:G}".format(0.12345678901234567890) == "0.123457" +assert "{:G}".format(0.12345678901234567890e-1) == "0.0123457" +assert "{:G}".format(0.12345678901234567890e-2) == "0.00123457" +assert "{:G}".format(0.12345678901234567890e-3) == "0.000123457" +assert "{:G}".format(0.12345678901234567890e-4) == "1.23457E-05" +assert "{:G}".format(0.12345678901234567890e-5) == "1.23457E-06" +assert "{:.6G}".format(0.12345678901234567890e-5) == "1.23457E-06" +assert "{:.10G}".format(0.12345678901234567890e-5) == "1.23456789E-06" +assert "{:.20G}".format(0.12345678901234567890e-5) == "1.2345678901234567384E-06" +assert "{:g}".format(float("nan")) == "nan" +assert "{:g}".format(float("-nan")) == "nan" +assert "{:G}".format(float("nan")) == "NAN" +assert "{:g}".format(float("inf")) == "inf" +assert "{:g}".format(float("-inf")) == "-inf" +assert "{:G}".format(float("inf")) == "INF" +assert "{:.0g}".format(1.020e-13) == "1e-13" +assert "{:.0g}".format(1.020e-13) == "1e-13" +assert "{:.1g}".format(1.020e-13) == "1e-13" +assert "{:.2g}".format(1.020e-13) == "1e-13" +assert "{:.3g}".format(1.020e-13) == "1.02e-13" +assert "{:.4g}".format(1.020e-13) == "1.02e-13" +assert "{:.5g}".format(1.020e-13) == "1.02e-13" +assert "{:.6g}".format(1.020e-13) == "1.02e-13" +assert "{:.7g}".format(1.020e-13) == "1.02e-13" +assert "{:g}".format(1.020e-13) == "1.02e-13" +assert "{:g}".format(1.020e-4) == "0.000102" # Test n & N formatting -assert '{:n}'.format(999999.1234) == '999999' -assert '{:n}'.format(9999.1234) == '9999.12' -assert '{:n}'.format(-1000000.1234) == '-1e+06' -assert '{:n}'.format(1000000.1234) == '1e+06' -assert '{:.1n}'.format(1000000.1234) == '1e+06' -assert '{:.2n}'.format(1000000.1234) == '1e+06' -assert '{:.3n}'.format(1000000.1234) == '1e+06' -assert '{:.4n}'.format(1000000.1234) == '1e+06' -assert '{:.5n}'.format(1000000.1234) == '1e+06' -assert '{:.6n}'.format(1000000.1234) == '1e+06' -assert '{:.7n}'.format(1000000.1234) == '1000000' -assert '{:.8n}'.format(1000000.1234) == '1000000.1' -assert '{:.10n}'.format(1000000.1234) == '1000000.123' -assert '{:.11n}'.format(1000000.1234) == '1000000.1234' -assert '{:.11n}'.format(-1000000.1234) == '-1000000.1234' -assert '{:0n}'.format(-1000000.1234) == '-1e+06' -assert '{:n}'.format(-1000000.1234) == '-1e+06' -assert '{:-1n}'.format(-1000000.1234) == '-1e+06' +assert "{:n}".format(999999.1234) == "999999" +assert "{:n}".format(9999.1234) == "9999.12" +assert "{:n}".format(-1000000.1234) == "-1e+06" +assert "{:n}".format(1000000.1234) == "1e+06" +assert "{:.1n}".format(1000000.1234) == "1e+06" +assert "{:.2n}".format(1000000.1234) == "1e+06" +assert "{:.3n}".format(1000000.1234) == "1e+06" +assert "{:.4n}".format(1000000.1234) == "1e+06" +assert "{:.5n}".format(1000000.1234) == "1e+06" +assert "{:.6n}".format(1000000.1234) == "1e+06" +assert "{:.7n}".format(1000000.1234) == "1000000" +assert "{:.8n}".format(1000000.1234) == "1000000.1" +assert "{:.10n}".format(1000000.1234) == "1000000.123" +assert "{:.11n}".format(1000000.1234) == "1000000.1234" +assert "{:.11n}".format(-1000000.1234) == "-1000000.1234" +assert "{:0n}".format(-1000000.1234) == "-1e+06" +assert "{:n}".format(-1000000.1234) == "-1e+06" +assert "{:-1n}".format(-1000000.1234) == "-1e+06" with AssertRaises(ValueError, msg="Unknown format code 'N' for object of type 'float'"): - '{:N}'.format(999999.1234) + "{:N}".format(999999.1234) with AssertRaises(ValueError, msg="Unknown format code 'N' for object of type 'float'"): - '{:.1N}'.format(1000000.1234) + "{:.1N}".format(1000000.1234) with AssertRaises(ValueError, msg="Unknown format code 'N' for object of type 'float'"): - '{:0N}'.format(-1000000.1234) + "{:0N}".format(-1000000.1234) with AssertRaises(ValueError, msg="Unknown format code 'N' for object of type 'float'"): - '{:-1N}'.format(-1000000.1234) + "{:-1N}".format(-1000000.1234) + # remove*fix test def test_removeprefix(): - s = 'foobarfoo' - s_ref='foobarfoo' - assert s.removeprefix('f') == s_ref[1:] - assert s.removeprefix('fo') == s_ref[2:] - assert s.removeprefix('foo') == s_ref[3:] - - assert s.removeprefix('') == s_ref - assert s.removeprefix('bar') == s_ref - assert s.removeprefix('lol') == s_ref - assert s.removeprefix('_foo') == s_ref - assert s.removeprefix('-foo') == s_ref - assert s.removeprefix('afoo') == s_ref - assert s.removeprefix('*foo') == s_ref - - assert s==s_ref, 'undefined test fail' - - s_uc = '😱foobarfoo🖖' - s_ref_uc = '😱foobarfoo🖖' - assert s_uc.removeprefix('😱') == s_ref_uc[1:] - assert s_uc.removeprefix('😱fo') == s_ref_uc[3:] - assert s_uc.removeprefix('😱foo') == s_ref_uc[4:] - - assert s_uc.removeprefix('🖖') == s_ref_uc - assert s_uc.removeprefix('foo') == s_ref_uc - assert s_uc.removeprefix(' ') == s_ref_uc - assert s_uc.removeprefix('_😱') == s_ref_uc - assert s_uc.removeprefix(' 😱') == s_ref_uc - assert s_uc.removeprefix('-😱') == s_ref_uc - assert s_uc.removeprefix('#😱') == s_ref_uc + s = "foobarfoo" + s_ref = "foobarfoo" + assert s.removeprefix("f") == s_ref[1:] + assert s.removeprefix("fo") == s_ref[2:] + assert s.removeprefix("foo") == s_ref[3:] + + assert s.removeprefix("") == s_ref + assert s.removeprefix("bar") == s_ref + assert s.removeprefix("lol") == s_ref + assert s.removeprefix("_foo") == s_ref + assert s.removeprefix("-foo") == s_ref + assert s.removeprefix("afoo") == s_ref + assert s.removeprefix("*foo") == s_ref + + assert s == s_ref, "undefined test fail" + + s_uc = "😱foobarfoo🖖" + s_ref_uc = "😱foobarfoo🖖" + assert s_uc.removeprefix("😱") == s_ref_uc[1:] + assert s_uc.removeprefix("😱fo") == s_ref_uc[3:] + assert s_uc.removeprefix("😱foo") == s_ref_uc[4:] + + assert s_uc.removeprefix("🖖") == s_ref_uc + assert s_uc.removeprefix("foo") == s_ref_uc + assert s_uc.removeprefix(" ") == s_ref_uc + assert s_uc.removeprefix("_😱") == s_ref_uc + assert s_uc.removeprefix(" 😱") == s_ref_uc + assert s_uc.removeprefix("-😱") == s_ref_uc + assert s_uc.removeprefix("#😱") == s_ref_uc + def test_removeprefix_types(): - s='0123456' - s_ref='0123456' - others=[0,['012']] - found=False + s = "0123456" + s_ref = "0123456" + others = [0, ["012"]] + found = False for o in others: try: s.removeprefix(o) except: - found=True + found = True + + assert found, f"Removeprefix accepts other type: {type(o)}: {o=}" - assert found, f'Removeprefix accepts other type: {type(o)}: {o=}' def test_removesuffix(): - s='foobarfoo' - s_ref='foobarfoo' - assert s.removesuffix('o') == s_ref[:-1] - assert s.removesuffix('oo') == s_ref[:-2] - assert s.removesuffix('foo') == s_ref[:-3] - - assert s.removesuffix('') == s_ref - assert s.removesuffix('bar') == s_ref - assert s.removesuffix('lol') == s_ref - assert s.removesuffix('foo_') == s_ref - assert s.removesuffix('foo-') == s_ref - assert s.removesuffix('foo*') == s_ref - assert s.removesuffix('fooa') == s_ref - - assert s==s_ref, 'undefined test fail' - - s_uc = '😱foobarfoo🖖' - s_ref_uc = '😱foobarfoo🖖' - assert s_uc.removesuffix('🖖') == s_ref_uc[:-1] - assert s_uc.removesuffix('oo🖖') == s_ref_uc[:-3] - assert s_uc.removesuffix('foo🖖') == s_ref_uc[:-4] - - assert s_uc.removesuffix('😱') == s_ref_uc - assert s_uc.removesuffix('foo') == s_ref_uc - assert s_uc.removesuffix(' ') == s_ref_uc - assert s_uc.removesuffix('🖖_') == s_ref_uc - assert s_uc.removesuffix('🖖 ') == s_ref_uc - assert s_uc.removesuffix('🖖-') == s_ref_uc - assert s_uc.removesuffix('🖖#') == s_ref_uc + s = "foobarfoo" + s_ref = "foobarfoo" + assert s.removesuffix("o") == s_ref[:-1] + assert s.removesuffix("oo") == s_ref[:-2] + assert s.removesuffix("foo") == s_ref[:-3] + + assert s.removesuffix("") == s_ref + assert s.removesuffix("bar") == s_ref + assert s.removesuffix("lol") == s_ref + assert s.removesuffix("foo_") == s_ref + assert s.removesuffix("foo-") == s_ref + assert s.removesuffix("foo*") == s_ref + assert s.removesuffix("fooa") == s_ref + + assert s == s_ref, "undefined test fail" + + s_uc = "😱foobarfoo🖖" + s_ref_uc = "😱foobarfoo🖖" + assert s_uc.removesuffix("🖖") == s_ref_uc[:-1] + assert s_uc.removesuffix("oo🖖") == s_ref_uc[:-3] + assert s_uc.removesuffix("foo🖖") == s_ref_uc[:-4] + + assert s_uc.removesuffix("😱") == s_ref_uc + assert s_uc.removesuffix("foo") == s_ref_uc + assert s_uc.removesuffix(" ") == s_ref_uc + assert s_uc.removesuffix("🖖_") == s_ref_uc + assert s_uc.removesuffix("🖖 ") == s_ref_uc + assert s_uc.removesuffix("🖖-") == s_ref_uc + assert s_uc.removesuffix("🖖#") == s_ref_uc + def test_removesuffix_types(): - s='0123456' - s_ref='0123456' - others=[0,6,['6']] - found=False + s = "0123456" + s_ref = "0123456" + others = [0, 6, ["6"]] + found = False for o in others: try: s.removesuffix(o) except: - found=True + found = True - assert found, f'Removesuffix accepts other type: {type(o)}: {o=}' + assert found, f"Removesuffix accepts other type: {type(o)}: {o=}" -skip_if_unsupported(3,9,test_removeprefix) -skip_if_unsupported(3,9,test_removeprefix_types) -skip_if_unsupported(3,9,test_removesuffix) -skip_if_unsupported(3,9,test_removesuffix_types) + +skip_if_unsupported(3, 9, test_removeprefix) +skip_if_unsupported(3, 9, test_removeprefix_types) +skip_if_unsupported(3, 9, test_removesuffix) +skip_if_unsupported(3, 9, test_removesuffix_types) # Regression to # https://github.com/RustPython/RustPython/issues/2840 -a = 'abc123()' +a = "abc123()" assert id(a) == id(a) assert id(a) != id(a * -1) @@ -751,7 +831,8 @@ def test_removesuffix_types(): class MyString(str): pass -b = MyString('0123abc*&') + +b = MyString("0123abc*&") assert id(b) == id(b) assert id(b) != id(b * -1) assert id(b) != id(b * 0) diff --git a/extra_tests/snippets/builtin_str_encode.py b/extra_tests/snippets/builtin_str_encode.py index 790e156e6fc..156a83e205e 100644 --- a/extra_tests/snippets/builtin_str_encode.py +++ b/extra_tests/snippets/builtin_str_encode.py @@ -10,11 +10,13 @@ assert_raises(UnicodeEncodeError, "¿como estás?".encode, "ascii") + def round_trip(s, encoding="utf-8"): encoded = s.encode(encoding) decoded = encoded.decode(encoding) assert s == decoded + round_trip("👺♦ 𝐚Şđƒ ☆☝") round_trip("☢🐣 ᖇ𝓤𝕊тⓟ𝕐𝕥卄σ𝔫 ♬👣") round_trip("💀👌 ק𝔂tℍⓞ𝓷 ３ 🔥👤") diff --git a/extra_tests/snippets/builtin_str_subclass.py b/extra_tests/snippets/builtin_str_subclass.py index 3ec266d5c34..73e23615c4d 100644 --- a/extra_tests/snippets/builtin_str_subclass.py +++ b/extra_tests/snippets/builtin_str_subclass.py @@ -3,6 +3,7 @@ x = "An interesting piece of text" assert x is str(x) + class Stringy(str): def __new__(cls, value=""): return str.__new__(cls, value) @@ -10,6 +11,7 @@ def __new__(cls, value=""): def __init__(self, value): self.x = "substr" + y = Stringy(1) assert type(y) is Stringy, "Type of Stringy should be stringy" assert type(str(y)) is str, "Str of a str-subtype should be a str." diff --git a/extra_tests/snippets/builtin_str_unicode.py b/extra_tests/snippets/builtin_str_unicode.py index 8858cf9bfdf..ca4a99199c3 100644 --- a/extra_tests/snippets/builtin_str_unicode.py +++ b/extra_tests/snippets/builtin_str_unicode.py @@ -1,29 +1,29 @@ - # Test the unicode support! 👋 -ᚴ=2 +ᚴ = 2 -assert ᚴ*8 == 16 +assert ᚴ * 8 == 16 -ᚴ="👋" +ᚴ = "👋" -c = ᚴ*3 +c = ᚴ * 3 -assert c == '👋👋👋' +assert c == "👋👋👋" import unicodedata -assert unicodedata.category('a') == 'Ll' -assert unicodedata.category('A') == 'Lu' -assert unicodedata.name('a') == 'LATIN SMALL LETTER A' -assert unicodedata.lookup('LATIN SMALL LETTER A') == 'a' -assert unicodedata.bidirectional('a') == 'L' -assert unicodedata.east_asian_width('\u231a') == 'W' -assert unicodedata.normalize('NFC', 'bla') == 'bla' + +assert unicodedata.category("a") == "Ll" +assert unicodedata.category("A") == "Lu" +assert unicodedata.name("a") == "LATIN SMALL LETTER A" +assert unicodedata.lookup("LATIN SMALL LETTER A") == "a" +assert unicodedata.bidirectional("a") == "L" +assert unicodedata.east_asian_width("\u231a") == "W" +assert unicodedata.normalize("NFC", "bla") == "bla" # testing unicodedata.ucd_3_2_0 for idna -assert "abcСĤ".encode("idna") == b'xn--abc-7sa390b' -assert "abc䄣IJ".encode("idna") == b'xn--abcij-zb5f' +assert "abcСĤ".encode("idna") == b"xn--abc-7sa390b" +assert "abc䄣IJ".encode("idna") == b"xn--abcij-zb5f" # from CPython tests assert "python.org".encode("idna") == b"python.org" diff --git a/extra_tests/snippets/builtin_str_unicode_slice.py b/extra_tests/snippets/builtin_str_unicode_slice.py index c6ce88d549f..252f84b1c72 100644 --- a/extra_tests/snippets/builtin_str_unicode_slice.py +++ b/extra_tests/snippets/builtin_str_unicode_slice.py @@ -1,13 +1,14 @@ def test_slice_bounds(s): # End out of range assert s[0:100] == s - assert s[0:-100] == '' + assert s[0:-100] == "" # Start out of range - assert s[100:1] == '' + assert s[100:1] == "" # Out of range both sides # This is the behaviour in cpython # assert s[-100:100] == s + def expect_index_error(s, index): try: s[index] @@ -16,6 +17,7 @@ def expect_index_error(s, index): else: assert False + unicode_str = "∀∂" assert unicode_str[0] == "∀" assert unicode_str[1] == "∂" @@ -35,25 +37,25 @@ def expect_index_error(s, index): hebrew_text = "בְּרֵאשִׁית, בָּרָא אֱלֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָאָרֶץ" assert len(hebrew_text) == 60 assert len(hebrew_text[:]) == 60 -assert hebrew_text[0] == 'ב' -assert hebrew_text[1] == 'ְ' -assert hebrew_text[2] == 'ּ' -assert hebrew_text[3] == 'ר' -assert hebrew_text[4] == 'ֵ' -assert hebrew_text[5] == 'א' -assert hebrew_text[6] == 'ש' -assert hebrew_text[5:10] == 'אשִׁי' +assert hebrew_text[0] == "ב" +assert hebrew_text[1] == "ְ" +assert hebrew_text[2] == "ּ" +assert hebrew_text[3] == "ר" +assert hebrew_text[4] == "ֵ" +assert hebrew_text[5] == "א" +assert hebrew_text[6] == "ש" +assert hebrew_text[5:10] == "אשִׁי" assert len(hebrew_text[5:10]) == 5 -assert hebrew_text[-20:50] == 'מַיִם, וְא' +assert hebrew_text[-20:50] == "מַיִם, וְא" assert len(hebrew_text[-20:50]) == 10 -assert hebrew_text[:-30:1] == 'בְּרֵאשִׁית, בָּרָא אֱלֹהִים, ' +assert hebrew_text[:-30:1] == "בְּרֵאשִׁית, בָּרָא אֱלֹהִים, " assert len(hebrew_text[:-30:1]) == 30 -assert hebrew_text[10:-30] == 'ת, בָּרָא אֱלֹהִים, ' +assert hebrew_text[10:-30] == "ת, בָּרָא אֱלֹהִים, " assert len(hebrew_text[10:-30]) == 20 -assert hebrew_text[10:30:3] == 'תבר לִ,' +assert hebrew_text[10:30:3] == "תבר לִ," assert len(hebrew_text[10:30:3]) == 7 -assert hebrew_text[10:30:-3] == '' -assert hebrew_text[30:10:-3] == 'אםהֱאּ ' +assert hebrew_text[10:30:-3] == "" +assert hebrew_text[30:10:-3] == "אםהֱאּ " assert len(hebrew_text[30:10:-3]) == 7 -assert hebrew_text[30:10:-1] == 'א ,םיִהֹלֱא אָרָּב ,' +assert hebrew_text[30:10:-1] == "א ,םיִהֹלֱא אָרָּב ," assert len(hebrew_text[30:10:-1]) == 20 diff --git a/extra_tests/snippets/builtin_tuple.py b/extra_tests/snippets/builtin_tuple.py index fd59e906093..fc2f8d5bb75 100644 --- a/extra_tests/snippets/builtin_tuple.py +++ b/extra_tests/snippets/builtin_tuple.py @@ -1,8 +1,8 @@ from testutils import assert_raises -assert (1,2) == (1,2) +assert (1, 2) == (1, 2) -x = (1,2) +x = (1, 2) assert x[0] == 1 y = (1,) @@ -19,7 +19,7 @@ assert x > y, "tuple __gt__ failed" -b = (1,2,3) +b = (1, 2, 3) assert b.index(2) == 1 recursive_list = [] @@ -30,15 +30,17 @@ assert (None, "", 1).index(1) == 2 assert 1 in (None, "", 1) + class Foo(object): def __eq__(self, x): return False + foo = Foo() assert (foo,) == (foo,) a = (1, 2, 3) -a += 1, +a += (1,) assert a == (1, 2, 3, 1) b = (55, *a) @@ -80,14 +82,14 @@ def __eq__(self, x): assert not (0, 0) > (0, 0) assert not (0, 0) < (0, 0) -assert not (float('nan'), float('nan')) <= (float('nan'), 1) -assert not (float('nan'), float('nan')) <= (float('nan'), float('nan')) -assert not (float('nan'), float('nan')) >= (float('nan'), float('nan')) -assert not (float('nan'), float('nan')) < (float('nan'), float('nan')) -assert not (float('nan'), float('nan')) > (float('nan'), float('nan')) - -assert (float('inf'), float('inf')) >= (float('inf'), 1) -assert (float('inf'), float('inf')) <= (float('inf'), float('inf')) -assert (float('inf'), float('inf')) >= (float('inf'), float('inf')) -assert not (float('inf'), float('inf')) < (float('inf'), float('inf')) -assert not (float('inf'), float('inf')) > (float('inf'), float('inf')) +assert not (float("nan"), float("nan")) <= (float("nan"), 1) +assert not (float("nan"), float("nan")) <= (float("nan"), float("nan")) +assert not (float("nan"), float("nan")) >= (float("nan"), float("nan")) +assert not (float("nan"), float("nan")) < (float("nan"), float("nan")) +assert not (float("nan"), float("nan")) > (float("nan"), float("nan")) + +assert (float("inf"), float("inf")) >= (float("inf"), 1) +assert (float("inf"), float("inf")) <= (float("inf"), float("inf")) +assert (float("inf"), float("inf")) >= (float("inf"), float("inf")) +assert not (float("inf"), float("inf")) < (float("inf"), float("inf")) +assert not (float("inf"), float("inf")) > (float("inf"), float("inf")) diff --git a/extra_tests/snippets/builtin_type.py b/extra_tests/snippets/builtin_type.py index 9f30b9b0ed4..8cb0a09a215 100644 --- a/extra_tests/snippets/builtin_type.py +++ b/extra_tests/snippets/builtin_type.py @@ -1,6 +1,6 @@ import types -from testutils import assert_raises +from testutils import assert_raises # Spec: https://docs.python.org/2/library/types.html print(None) @@ -13,9 +13,9 @@ print("abc") # print(u"abc") # Structural below -print((1, 2)) # Tuple can be any length, but fixed after declared -x = (1,2) -print(x[0]) # Tuple can be any length, but fixed after declared +print((1, 2)) # Tuple can be any length, but fixed after declared +x = (1, 2) +print(x[0]) # Tuple can be any length, but fixed after declared print([1, 2, 3]) # print({"first":1,"second":2}) @@ -52,25 +52,34 @@ a = complex(2, 4) assert type(a) is complex assert type(a + a) is complex -assert repr(a) == '(2+4j)' +assert repr(a) == "(2+4j)" a = 10j -assert repr(a) == '10j' +assert repr(a) == "10j" a = 1 assert a.conjugate() == a a = 12345 -b = a*a*a*a*a*a*a*a +b = a * a * a * a * a * a * a * a assert b.bit_length() == 109 -assert type.__module__ == 'builtins' -assert type.__qualname__ == 'type' -assert type.__name__ == 'type' +assert type.__module__ == "builtins" +assert type.__qualname__ == "type" +assert type.__name__ == "type" assert isinstance(type.__doc__, str) -assert object.__qualname__ == 'object' -assert int.__qualname__ == 'int' +assert object.__qualname__ == "object" +assert int.__qualname__ == "int" + +with assert_raises(TypeError): + type.__module__ = "nope" + +with assert_raises(TypeError): + object.__module__ = "nope" + +with assert_raises(TypeError): + map.__module__ = "nope" class A(type): @@ -78,8 +87,8 @@ class A(type): class B(type): - __module__ = 'b' - __qualname__ = 'BB' + __module__ = "b" + __qualname__ = "BB" class C: @@ -87,23 +96,23 @@ class C: class D: - __module__ = 'd' - __qualname__ = 'DD' - - -assert A.__module__ == '__main__' -assert A.__qualname__ == 'A' -assert B.__module__ == 'b' -assert B.__qualname__ == 'BB' -assert C.__module__ == '__main__' -assert C.__qualname__ == 'C' -assert D.__module__ == 'd' -assert D.__qualname__ == 'DD' - -A.__qualname__ = 'AA' -B.__qualname__ = 'b' -assert A.__qualname__ == 'AA' -assert B.__qualname__ == 'b' + __module__ = "d" + __qualname__ = "DD" + + +assert A.__module__ == "__main__" +assert A.__qualname__ == "A" +assert B.__module__ == "b" +assert B.__qualname__ == "BB" +assert C.__module__ == "__main__" +assert C.__qualname__ == "C" +assert D.__module__ == "d" +assert D.__qualname__ == "DD" + +A.__qualname__ = "AA" +B.__qualname__ = "b" +assert A.__qualname__ == "AA" +assert B.__qualname__ == "b" with assert_raises(TypeError): del D.__qualname__ with assert_raises(TypeError): @@ -111,10 +120,9 @@ class D: with assert_raises(TypeError): del int.__qualname__ -from testutils import assert_raises - import platform -if platform.python_implementation() == 'RustPython': + +if platform.python_implementation() == "RustPython": gc = None else: import gc @@ -123,13 +131,13 @@ class D: assert type(object) is type assert type(object()) is object -new_type = type('New', (object,), {}) +new_type = type("New", (object,), {}) assert type(new_type) is type assert type(new_type()) is new_type -metaclass = type('MCl', (type,), {}) -cls = metaclass('Cls', (object,), {}) +metaclass = type("MCl", (type,), {}) +cls = metaclass("Cls", (object,), {}) inst = cls() assert type(inst) is cls @@ -154,10 +162,22 @@ class D: assert not issubclass(type, (int, float)) assert issubclass(type, (int, type)) -class A: pass -class B(A): pass -class C(A): pass -class D(B, C): pass + +class A: + pass + + +class B(A): + pass + + +class C(A): + pass + + +class D(B, C): + pass + assert A.__subclasses__() == [B, C] assert B.__subclasses__() == [D] @@ -174,7 +194,7 @@ class D(B, C): pass if gc: # gc sweep is needed here for CPython... gc.collect() - # ...while RustPython doesn't have `gc` yet. + # ...while RustPython doesn't have `gc` yet. if gc: # D.__new__ is a method bound to the D type, so just deleting D @@ -185,43 +205,103 @@ class D(B, C): pass assert type in object.__subclasses__() -assert cls.__name__ == 'Cls' +assert cls.__name__ == "Cls" # mro assert int.mro() == [int, object] assert bool.mro() == [bool, int, object] assert object.mro() == [object] + class A: pass + class B(A): pass + assert A.mro() == [A, object] assert B.mro() == [B, A, object] + class AA: pass + class BB(AA): pass + class C(B, BB): pass + assert C.mro() == [C, B, A, BB, AA, object] -assert type(Exception.args).__name__ == 'getset_descriptor' +class TypeA: + def __init__(self): + self.a = 1 + + +class TypeB: + __slots__ = "b" + + def __init__(self): + self.b = 2 + + +obj = TypeA() +with assert_raises(TypeError) as cm: + obj.__class__ = TypeB +assert "__class__ assignment: 'TypeB' object layout differs from 'TypeA'" in str( + cm.exception +) + + +# Test: same slot count but different slot names should fail +class SlotX: + __slots__ = ("x",) + + +class SlotY: + __slots__ = ("y",) + + +slot_obj = SlotX() +with assert_raises(TypeError) as cm: + slot_obj.__class__ = SlotY +assert "__class__ assignment: 'SlotY' object layout differs from 'SlotX'" in str( + cm.exception +) + + +# Test: same slots should succeed +class SlotA: + __slots__ = ("a",) + + +class SlotA2: + __slots__ = ("a",) + + +slot_a = SlotA() +slot_a.__class__ = SlotA2 # Should work + + +assert type(Exception.args).__name__ == "getset_descriptor" assert type(None).__bool__(None) is False + class A: pass + class B: pass + a = A() a.__class__ = B assert isinstance(a, B) @@ -234,30 +314,33 @@ class B: # Regression to # https://github.com/RustPython/RustPython/issues/2310 import builtins -assert builtins.iter.__class__.__module__ == 'builtins' -assert builtins.iter.__class__.__qualname__ == 'builtin_function_or_method' -assert iter.__class__.__module__ == 'builtins' -assert iter.__class__.__qualname__ == 'builtin_function_or_method' -assert type(iter).__module__ == 'builtins' -assert type(iter).__qualname__ == 'builtin_function_or_method' +assert builtins.iter.__class__.__module__ == "builtins" +assert builtins.iter.__class__.__qualname__ == "builtin_function_or_method" + +assert iter.__class__.__module__ == "builtins" +assert iter.__class__.__qualname__ == "builtin_function_or_method" +assert type(iter).__module__ == "builtins" +assert type(iter).__qualname__ == "builtin_function_or_method" # Regression to # https://github.com/RustPython/RustPython/issues/2767 # Marked as `#[pymethod]`: -assert str.replace.__qualname__ == 'str.replace' -assert str().replace.__qualname__ == 'str.replace' -assert int.to_bytes.__qualname__ == 'int.to_bytes' -assert int().to_bytes.__qualname__ == 'int.to_bytes' +assert str.replace.__qualname__ == "str.replace" +assert str().replace.__qualname__ == "str.replace" +assert int.to_bytes.__qualname__ == "int.to_bytes" +assert int().to_bytes.__qualname__ == "int.to_bytes" # Marked as `#[pyclassmethod]`: -assert dict.fromkeys.__qualname__ == 'dict.fromkeys' -assert object.__init_subclass__.__qualname__ == 'object.__init_subclass__' +assert dict.fromkeys.__qualname__ == "dict.fromkeys" +assert object.__init_subclass__.__qualname__ == "object.__init_subclass__" # Dynamic with `#[extend_class]`: -assert bytearray.maketrans.__qualname__ == 'bytearray.maketrans', bytearray.maketrans.__qualname__ +assert bytearray.maketrans.__qualname__ == "bytearray.maketrans", ( + bytearray.maketrans.__qualname__ +) # Third-party: @@ -285,47 +368,48 @@ def c(cls): def s(): pass -assert MyTypeWithMethod.method.__name__ == 'method' -assert MyTypeWithMethod().method.__name__ == 'method' -assert MyTypeWithMethod.clsmethod.__name__ == 'clsmethod' -assert MyTypeWithMethod().clsmethod.__name__ == 'clsmethod' -assert MyTypeWithMethod.stmethod.__name__ == 'stmethod' -assert MyTypeWithMethod().stmethod.__name__ == 'stmethod' - -assert MyTypeWithMethod.method.__qualname__ == 'MyTypeWithMethod.method' -assert MyTypeWithMethod().method.__qualname__ == 'MyTypeWithMethod.method' -assert MyTypeWithMethod.clsmethod.__qualname__ == 'MyTypeWithMethod.clsmethod' -assert MyTypeWithMethod().clsmethod.__qualname__ == 'MyTypeWithMethod.clsmethod' -assert MyTypeWithMethod.stmethod.__qualname__ == 'MyTypeWithMethod.stmethod' -assert MyTypeWithMethod().stmethod.__qualname__ == 'MyTypeWithMethod.stmethod' - -assert MyTypeWithMethod.N.m.__name__ == 'm' -assert MyTypeWithMethod().N.m.__name__ == 'm' -assert MyTypeWithMethod.N.c.__name__ == 'c' -assert MyTypeWithMethod().N.c.__name__ == 'c' -assert MyTypeWithMethod.N.s.__name__ == 's' -assert MyTypeWithMethod().N.s.__name__ == 's' - -assert MyTypeWithMethod.N.m.__qualname__ == 'MyTypeWithMethod.N.m' -assert MyTypeWithMethod().N.m.__qualname__ == 'MyTypeWithMethod.N.m' -assert MyTypeWithMethod.N.c.__qualname__ == 'MyTypeWithMethod.N.c' -assert MyTypeWithMethod().N.c.__qualname__ == 'MyTypeWithMethod.N.c' -assert MyTypeWithMethod.N.s.__qualname__ == 'MyTypeWithMethod.N.s' -assert MyTypeWithMethod().N.s.__qualname__ == 'MyTypeWithMethod.N.s' - -assert MyTypeWithMethod.N().m.__name__ == 'm' -assert MyTypeWithMethod().N().m.__name__ == 'm' -assert MyTypeWithMethod.N().c.__name__ == 'c' -assert MyTypeWithMethod().N().c.__name__ == 'c' -assert MyTypeWithMethod.N().s.__name__ == 's' -assert MyTypeWithMethod().N.s.__name__ == 's' - -assert MyTypeWithMethod.N().m.__qualname__ == 'MyTypeWithMethod.N.m' -assert MyTypeWithMethod().N().m.__qualname__ == 'MyTypeWithMethod.N.m' -assert MyTypeWithMethod.N().c.__qualname__ == 'MyTypeWithMethod.N.c' -assert MyTypeWithMethod().N().c.__qualname__ == 'MyTypeWithMethod.N.c' -assert MyTypeWithMethod.N().s.__qualname__ == 'MyTypeWithMethod.N.s' -assert MyTypeWithMethod().N().s.__qualname__ == 'MyTypeWithMethod.N.s' + +assert MyTypeWithMethod.method.__name__ == "method" +assert MyTypeWithMethod().method.__name__ == "method" +assert MyTypeWithMethod.clsmethod.__name__ == "clsmethod" +assert MyTypeWithMethod().clsmethod.__name__ == "clsmethod" +assert MyTypeWithMethod.stmethod.__name__ == "stmethod" +assert MyTypeWithMethod().stmethod.__name__ == "stmethod" + +assert MyTypeWithMethod.method.__qualname__ == "MyTypeWithMethod.method" +assert MyTypeWithMethod().method.__qualname__ == "MyTypeWithMethod.method" +assert MyTypeWithMethod.clsmethod.__qualname__ == "MyTypeWithMethod.clsmethod" +assert MyTypeWithMethod().clsmethod.__qualname__ == "MyTypeWithMethod.clsmethod" +assert MyTypeWithMethod.stmethod.__qualname__ == "MyTypeWithMethod.stmethod" +assert MyTypeWithMethod().stmethod.__qualname__ == "MyTypeWithMethod.stmethod" + +assert MyTypeWithMethod.N.m.__name__ == "m" +assert MyTypeWithMethod().N.m.__name__ == "m" +assert MyTypeWithMethod.N.c.__name__ == "c" +assert MyTypeWithMethod().N.c.__name__ == "c" +assert MyTypeWithMethod.N.s.__name__ == "s" +assert MyTypeWithMethod().N.s.__name__ == "s" + +assert MyTypeWithMethod.N.m.__qualname__ == "MyTypeWithMethod.N.m" +assert MyTypeWithMethod().N.m.__qualname__ == "MyTypeWithMethod.N.m" +assert MyTypeWithMethod.N.c.__qualname__ == "MyTypeWithMethod.N.c" +assert MyTypeWithMethod().N.c.__qualname__ == "MyTypeWithMethod.N.c" +assert MyTypeWithMethod.N.s.__qualname__ == "MyTypeWithMethod.N.s" +assert MyTypeWithMethod().N.s.__qualname__ == "MyTypeWithMethod.N.s" + +assert MyTypeWithMethod.N().m.__name__ == "m" +assert MyTypeWithMethod().N().m.__name__ == "m" +assert MyTypeWithMethod.N().c.__name__ == "c" +assert MyTypeWithMethod().N().c.__name__ == "c" +assert MyTypeWithMethod.N().s.__name__ == "s" +assert MyTypeWithMethod().N.s.__name__ == "s" + +assert MyTypeWithMethod.N().m.__qualname__ == "MyTypeWithMethod.N.m" +assert MyTypeWithMethod().N().m.__qualname__ == "MyTypeWithMethod.N.m" +assert MyTypeWithMethod.N().c.__qualname__ == "MyTypeWithMethod.N.c" +assert MyTypeWithMethod().N().c.__qualname__ == "MyTypeWithMethod.N.c" +assert MyTypeWithMethod.N().s.__qualname__ == "MyTypeWithMethod.N.s" +assert MyTypeWithMethod().N().s.__qualname__ == "MyTypeWithMethod.N.s" # Regresesion to @@ -339,26 +423,27 @@ def s(): # Regression to # https://github.com/RustPython/RustPython/issues/2788 -assert iter.__qualname__ == iter.__name__ == 'iter' -assert max.__qualname__ == max.__name__ == 'max' -assert min.__qualname__ == min.__name__ == 'min' +assert iter.__qualname__ == iter.__name__ == "iter" +assert max.__qualname__ == max.__name__ == "max" +assert min.__qualname__ == min.__name__ == "min" def custom_func(): pass -assert custom_func.__qualname__ == 'custom_func' + +assert custom_func.__qualname__ == "custom_func" # Regression to # https://github.com/RustPython/RustPython/issues/2786 -assert object.__new__.__name__ == '__new__' -assert object.__new__.__qualname__ == 'object.__new__' -assert object.__subclasshook__.__name__ == '__subclasshook__' -assert object.__subclasshook__.__qualname__ == 'object.__subclasshook__' -assert type.__new__.__name__ == '__new__' -assert type.__new__.__qualname__ == 'type.__new__' +assert object.__new__.__name__ == "__new__" +assert object.__new__.__qualname__ == "object.__new__" +assert object.__subclasshook__.__name__ == "__subclasshook__" +assert object.__subclasshook__.__qualname__ == "object.__subclasshook__" +assert type.__new__.__name__ == "__new__" +assert type.__new__.__qualname__ == "type.__new__" class AQ: @@ -414,75 +499,76 @@ def three_cls(cls): def three_st(): pass -assert AQ.one.__name__ == 'one' -assert AQ().one.__name__ == 'one' -assert AQ.one_cls.__name__ == 'one_cls' -assert AQ().one_cls.__name__ == 'one_cls' -assert AQ.one_st.__name__ == 'one_st' -assert AQ().one_st.__name__ == 'one_st' - -assert AQ.one.__qualname__ == 'AQ.one' -assert AQ().one.__qualname__ == 'AQ.one' -assert AQ.one_cls.__qualname__ == 'AQ.one_cls' -assert AQ().one_cls.__qualname__ == 'AQ.one_cls' -assert AQ.one_st.__qualname__ == 'AQ.one_st' -assert AQ().one_st.__qualname__ == 'AQ.one_st' - -assert AQ.two.__name__ == 'two' -assert AQ().two.__name__ == 'two' -assert AQ.two_cls.__name__ == 'two_cls' -assert AQ().two_cls.__name__ == 'two_cls' -assert AQ.two_st.__name__ == 'two_st' -assert AQ().two_st.__name__ == 'two_st' - -assert AQ.two.__qualname__ == 'AQ.two' -assert AQ().two.__qualname__ == 'AQ.two' -assert AQ.two_cls.__qualname__ == 'AQ.two_cls' -assert AQ().two_cls.__qualname__ == 'AQ.two_cls' -assert AQ.two_st.__qualname__ == 'AQ.two_st' -assert AQ().two_st.__qualname__ == 'AQ.two_st' - -assert BQ.one.__name__ == 'one' -assert BQ().one.__name__ == 'one' -assert BQ.one_cls.__name__ == 'one_cls' -assert BQ().one_cls.__name__ == 'one_cls' -assert BQ.one_st.__name__ == 'one_st' -assert BQ().one_st.__name__ == 'one_st' - -assert BQ.one.__qualname__ == 'BQ.one' -assert BQ().one.__qualname__ == 'BQ.one' -assert BQ.one_cls.__qualname__ == 'BQ.one_cls' -assert BQ().one_cls.__qualname__ == 'BQ.one_cls' -assert BQ.one_st.__qualname__ == 'BQ.one_st' -assert BQ().one_st.__qualname__ == 'BQ.one_st' - -assert BQ.two.__name__ == 'two' -assert BQ().two.__name__ == 'two' -assert BQ.two_cls.__name__ == 'two_cls' -assert BQ().two_cls.__name__ == 'two_cls' -assert BQ.two_st.__name__ == 'two_st' -assert BQ().two_st.__name__ == 'two_st' - -assert BQ.two.__qualname__ == 'AQ.two' -assert BQ().two.__qualname__ == 'AQ.two' -assert BQ.two_cls.__qualname__ == 'AQ.two_cls' -assert BQ().two_cls.__qualname__ == 'AQ.two_cls' -assert BQ.two_st.__qualname__ == 'AQ.two_st' -assert BQ().two_st.__qualname__ == 'AQ.two_st' - -assert BQ.three.__name__ == 'three' -assert BQ().three.__name__ == 'three' -assert BQ.three_cls.__name__ == 'three_cls' -assert BQ().three_cls.__name__ == 'three_cls' -assert BQ.three_st.__name__ == 'three_st' -assert BQ().three_st.__name__ == 'three_st' - -assert BQ.three.__qualname__ == 'BQ.three' -assert BQ().three.__qualname__ == 'BQ.three' -assert BQ.three_cls.__qualname__ == 'BQ.three_cls' -assert BQ().three_cls.__qualname__ == 'BQ.three_cls' -assert BQ.three_st.__qualname__ == 'BQ.three_st' -assert BQ().three_st.__qualname__ == 'BQ.three_st' + +assert AQ.one.__name__ == "one" +assert AQ().one.__name__ == "one" +assert AQ.one_cls.__name__ == "one_cls" +assert AQ().one_cls.__name__ == "one_cls" +assert AQ.one_st.__name__ == "one_st" +assert AQ().one_st.__name__ == "one_st" + +assert AQ.one.__qualname__ == "AQ.one" +assert AQ().one.__qualname__ == "AQ.one" +assert AQ.one_cls.__qualname__ == "AQ.one_cls" +assert AQ().one_cls.__qualname__ == "AQ.one_cls" +assert AQ.one_st.__qualname__ == "AQ.one_st" +assert AQ().one_st.__qualname__ == "AQ.one_st" + +assert AQ.two.__name__ == "two" +assert AQ().two.__name__ == "two" +assert AQ.two_cls.__name__ == "two_cls" +assert AQ().two_cls.__name__ == "two_cls" +assert AQ.two_st.__name__ == "two_st" +assert AQ().two_st.__name__ == "two_st" + +assert AQ.two.__qualname__ == "AQ.two" +assert AQ().two.__qualname__ == "AQ.two" +assert AQ.two_cls.__qualname__ == "AQ.two_cls" +assert AQ().two_cls.__qualname__ == "AQ.two_cls" +assert AQ.two_st.__qualname__ == "AQ.two_st" +assert AQ().two_st.__qualname__ == "AQ.two_st" + +assert BQ.one.__name__ == "one" +assert BQ().one.__name__ == "one" +assert BQ.one_cls.__name__ == "one_cls" +assert BQ().one_cls.__name__ == "one_cls" +assert BQ.one_st.__name__ == "one_st" +assert BQ().one_st.__name__ == "one_st" + +assert BQ.one.__qualname__ == "BQ.one" +assert BQ().one.__qualname__ == "BQ.one" +assert BQ.one_cls.__qualname__ == "BQ.one_cls" +assert BQ().one_cls.__qualname__ == "BQ.one_cls" +assert BQ.one_st.__qualname__ == "BQ.one_st" +assert BQ().one_st.__qualname__ == "BQ.one_st" + +assert BQ.two.__name__ == "two" +assert BQ().two.__name__ == "two" +assert BQ.two_cls.__name__ == "two_cls" +assert BQ().two_cls.__name__ == "two_cls" +assert BQ.two_st.__name__ == "two_st" +assert BQ().two_st.__name__ == "two_st" + +assert BQ.two.__qualname__ == "AQ.two" +assert BQ().two.__qualname__ == "AQ.two" +assert BQ.two_cls.__qualname__ == "AQ.two_cls" +assert BQ().two_cls.__qualname__ == "AQ.two_cls" +assert BQ.two_st.__qualname__ == "AQ.two_st" +assert BQ().two_st.__qualname__ == "AQ.two_st" + +assert BQ.three.__name__ == "three" +assert BQ().three.__name__ == "three" +assert BQ.three_cls.__name__ == "three_cls" +assert BQ().three_cls.__name__ == "three_cls" +assert BQ.three_st.__name__ == "three_st" +assert BQ().three_st.__name__ == "three_st" + +assert BQ.three.__qualname__ == "BQ.three" +assert BQ().three.__qualname__ == "BQ.three" +assert BQ.three_cls.__qualname__ == "BQ.three_cls" +assert BQ().three_cls.__qualname__ == "BQ.three_cls" +assert BQ.three_st.__qualname__ == "BQ.three_st" +assert BQ().three_st.__qualname__ == "BQ.three_st" class ClassWithNew: @@ -494,74 +580,110 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls, *args, **kwargs) -assert ClassWithNew.__new__.__qualname__ == 'ClassWithNew.__new__' -assert ClassWithNew().__new__.__qualname__ == 'ClassWithNew.__new__' -assert ClassWithNew.__new__.__name__ == '__new__' -assert ClassWithNew().__new__.__name__ == '__new__' +assert ClassWithNew.__new__.__qualname__ == "ClassWithNew.__new__" +assert ClassWithNew().__new__.__qualname__ == "ClassWithNew.__new__" +assert ClassWithNew.__new__.__name__ == "__new__" +assert ClassWithNew().__new__.__name__ == "__new__" +assert isinstance(ClassWithNew.__dict__.get("__new__"), staticmethod) -assert ClassWithNew.N.__new__.__qualname__ == 'ClassWithNew.N.__new__' -assert ClassWithNew().N.__new__.__qualname__ == 'ClassWithNew.N.__new__' -assert ClassWithNew.N.__new__.__name__ == '__new__' -assert ClassWithNew().N.__new__.__name__ == '__new__' -assert ClassWithNew.N().__new__.__qualname__ == 'ClassWithNew.N.__new__' -assert ClassWithNew().N().__new__.__qualname__ == 'ClassWithNew.N.__new__' -assert ClassWithNew.N().__new__.__name__ == '__new__' -assert ClassWithNew().N().__new__.__name__ == '__new__' +assert ClassWithNew.N.__new__.__qualname__ == "ClassWithNew.N.__new__" +assert ClassWithNew().N.__new__.__qualname__ == "ClassWithNew.N.__new__" +assert ClassWithNew.N.__new__.__name__ == "__new__" +assert ClassWithNew().N.__new__.__name__ == "__new__" +assert ClassWithNew.N().__new__.__qualname__ == "ClassWithNew.N.__new__" +assert ClassWithNew().N().__new__.__qualname__ == "ClassWithNew.N.__new__" +assert ClassWithNew.N().__new__.__name__ == "__new__" +assert ClassWithNew().N().__new__.__name__ == "__new__" +assert isinstance(ClassWithNew.N.__dict__.get("__new__"), staticmethod) # Regression to: # https://github.com/RustPython/RustPython/issues/2762 assert type.__prepare__() == {} -assert type.__prepare__('name') == {} -assert type.__prepare__('name', object) == {} -assert type.__prepare__('name', (bytes, str)) == {} +assert type.__prepare__("name") == {} +assert type.__prepare__("name", object) == {} +assert type.__prepare__("name", (bytes, str)) == {} assert type.__prepare__(a=1, b=2) == {} -assert type.__prepare__('name', (object, int), kw=True) == {} +assert type.__prepare__("name", (object, int), kw=True) == {} # Previously we needed `name` to be `str`: assert type.__prepare__(1) == {} assert int.__prepare__() == {} -assert int.__prepare__('name', (object, int), kw=True) == {} +assert int.__prepare__("name", (object, int), kw=True) == {} # Regression to # https://github.com/RustPython/RustPython/issues/2790 # `#[pyproperty]` -assert BaseException.args.__qualname__ == 'BaseException.args' +assert BaseException.args.__qualname__ == "BaseException.args" # class extension without `#[pyproperty]` override -assert Exception.args.__qualname__ == 'BaseException.args' +assert Exception.args.__qualname__ == "BaseException.args" # dynamic with `.new_readonly_getset` -assert SyntaxError.msg.__qualname__ == 'SyntaxError.msg' +assert SyntaxError.msg.__qualname__ == "SyntaxError.msg" # Regression to # https://github.com/RustPython/RustPython/issues/2794 -assert type.__subclasshook__.__qualname__ == 'type.__subclasshook__' -assert object.__subclasshook__.__qualname__ == 'object.__subclasshook__' +assert type.__subclasshook__.__qualname__ == "type.__subclasshook__" +assert object.__subclasshook__.__qualname__ == "object.__subclasshook__" # Regression to # https://github.com/RustPython/RustPython/issues/2776 -assert repr(BQ.one).startswith('<function BQ.one at 0x') -assert repr(BQ.one_st).startswith('<function BQ.one_st at 0x') +assert repr(BQ.one).startswith("<function BQ.one at 0x") +assert repr(BQ.one_st).startswith("<function BQ.one_st at 0x") -assert repr(BQ.two).startswith('<function AQ.two at 0x') -assert repr(BQ.two_st).startswith('<function AQ.two_st at 0x') +assert repr(BQ.two).startswith("<function AQ.two at 0x") +assert repr(BQ.two_st).startswith("<function AQ.two_st at 0x") -assert repr(BQ.three).startswith('<function BQ.three at 0x') -assert repr(BQ.three_st).startswith('<function BQ.three_st at 0x') +assert repr(BQ.three).startswith("<function BQ.three at 0x") +assert repr(BQ.three_st).startswith("<function BQ.three_st at 0x") def my_repr_func(): pass -assert repr(my_repr_func).startswith('<function my_repr_func at 0x') + +assert repr(my_repr_func).startswith("<function my_repr_func at 0x") # https://github.com/RustPython/RustPython/issues/3100 assert issubclass(types.BuiltinMethodType, types.BuiltinFunctionType) + +assert type.__dict__["__dict__"].__objclass__ is type +assert ( + type(type(type.__dict__["__dict__"]).__objclass__).__name__ == "member_descriptor" +) + + +class A(type): + pass + + +assert "__dict__" not in A.__dict__ + + +# regression tests for: https://github.com/RustPython/RustPython/issues/4505 + + +def foo(): + def inner(): + pass + + +assert foo.__code__.co_names == () + +stmts = """ +import blah + +def foo(): + pass +""" + +code = compile(stmts, "<test>", "exec") +assert code.co_names == ("blah", "foo") diff --git a/extra_tests/snippets/builtin_type_mro.py b/extra_tests/snippets/builtin_type_mro.py index 18acbb69160..5e7d5d35234 100644 --- a/extra_tests/snippets/builtin_type_mro.py +++ b/extra_tests/snippets/builtin_type_mro.py @@ -1,22 +1,29 @@ -class X(): +class X: pass -class Y(): + +class Y: pass + class A(X, Y): pass + assert (A, X, Y, object) == A.__mro__ + class B(X, Y): pass + assert (B, X, Y, object) == B.__mro__ + class C(A, B): pass + assert (C, A, B, X, Y, object) == C.__mro__ assert type.__mro__ == (type, object) diff --git a/extra_tests/snippets/builtin_zip.py b/extra_tests/snippets/builtin_zip.py index 3665c770212..e3e4c31aae3 100644 --- a/extra_tests/snippets/builtin_zip.py +++ b/extra_tests/snippets/builtin_zip.py @@ -1,9 +1,13 @@ -assert list(zip(['a', 'b', 'c'], range(3), [9, 8, 7, 99])) == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7)] +assert list(zip(["a", "b", "c"], range(3), [9, 8, 7, 99])) == [ + ("a", 0, 9), + ("b", 1, 8), + ("c", 2, 7), +] -assert list(zip(['a', 'b', 'c'])) == [('a',), ('b',), ('c',)] +assert list(zip(["a", "b", "c"])) == [("a",), ("b",), ("c",)] assert list(zip()) == [] -assert list(zip(*zip(['a', 'b', 'c'], range(1, 4)))) == [('a', 'b', 'c'), (1, 2, 3)] +assert list(zip(*zip(["a", "b", "c"], range(1, 4)))) == [("a", "b", "c"), (1, 2, 3)] # test infinite iterator diff --git a/extra_tests/snippets/builtins_module.py b/extra_tests/snippets/builtins_module.py index 1b5b6bdde54..bf762425c89 100644 --- a/extra_tests/snippets/builtins_module.py +++ b/extra_tests/snippets/builtins_module.py @@ -1,25 +1,37 @@ from testutils import assert_raises -assert '__builtins__' in globals() +assert "__builtins__" in globals() # assert type(__builtins__).__name__ == 'module' with assert_raises(AttributeError): __builtins__.__builtins__ assert __builtins__.__name__ == "builtins" import builtins + assert builtins.__name__ == "builtins" -__builtins__.x = 'new' -assert x == 'new' # noqa: F821 +__builtins__.x = "new" +assert x == "new" # noqa: F821 exec('assert "__builtins__" in globals()', dict()) -exec('assert __builtins__ == 7', {'__builtins__': 7}) -exec('assert not isinstance(__builtins__, dict)') -exec('assert isinstance(__builtins__, dict)', {}) +exec("assert __builtins__ == 7", {"__builtins__": 7}) +exec("assert not isinstance(__builtins__, dict)") +exec("assert isinstance(__builtins__, dict)", {}) namespace = {} -exec('', namespace) -assert namespace['__builtins__'] == __builtins__.__dict__ +exec("", namespace) +assert namespace["__builtins__"] == __builtins__.__dict__ + + +# function.__builtins__ should be a dict, not a module +# See: https://docs.python.org/3/reference/datamodel.html +def test_func(): + pass + + +assert isinstance(test_func.__builtins__, dict), ( + f"function.__builtins__ should be dict, got {type(test_func.__builtins__)}" +) # with assert_raises(NameError): # exec('print(__builtins__)', {'__builtins__': {}}) diff --git a/extra_tests/snippets/code_co_consts.py b/extra_tests/snippets/code_co_consts.py index 564a2ba448f..13f76a0d13e 100644 --- a/extra_tests/snippets/code_co_consts.py +++ b/extra_tests/snippets/code_co_consts.py @@ -1,31 +1,112 @@ +""" +Test co_consts behavior for Python 3.14+ + +In Python 3.14+: +- Functions with docstrings have the docstring as co_consts[0] +- CO_HAS_DOCSTRING flag (0x4000000) indicates docstring presence +- Functions without docstrings do NOT have None added as placeholder for docstring + +Note: Other constants (small integers, code objects, etc.) may still appear in co_consts +depending on optimization level. This test focuses on docstring behavior. +""" + + +# Test function with docstring - docstring should be co_consts[0] +def with_doc(): + """This is a docstring""" + return 1 + + +assert with_doc.__code__.co_consts[0] == "This is a docstring", ( + with_doc.__code__.co_consts +) +assert with_doc.__doc__ == "This is a docstring" +# Check CO_HAS_DOCSTRING flag (0x4000000) +assert with_doc.__code__.co_flags & 0x4000000, hex(with_doc.__code__.co_flags) + + +# Test function without docstring - should NOT have HAS_DOCSTRING flag +def no_doc(): + return 1 + + +assert not (no_doc.__code__.co_flags & 0x4000000), hex(no_doc.__code__.co_flags) +assert no_doc.__doc__ is None + + +# Test async function with docstring from asyncio import sleep -def f(): - def g(): - return 1 - assert g.__code__.co_consts[0] == None - return 2 +async def async_with_doc(): + """Async docstring""" + await sleep(1) + return 1 + + +assert async_with_doc.__code__.co_consts[0] == "Async docstring", ( + async_with_doc.__code__.co_consts +) +assert async_with_doc.__doc__ == "Async docstring" +assert async_with_doc.__code__.co_flags & 0x4000000 + -assert f.__code__.co_consts[0] == None +# Test async function without docstring +async def async_no_doc(): + await sleep(1) + return 1 -def generator(): - yield 1 - yield 2 -assert generator().gi_code.co_consts[0] == None +assert not (async_no_doc.__code__.co_flags & 0x4000000) +assert async_no_doc.__doc__ is None -async def async_f(): - await sleep(1) - return 1 -assert async_f.__code__.co_consts[0] == None +# Test generator with docstring +def gen_with_doc(): + """Generator docstring""" + yield 1 + yield 2 + +assert gen_with_doc.__code__.co_consts[0] == "Generator docstring" +assert gen_with_doc.__doc__ == "Generator docstring" +assert gen_with_doc.__code__.co_flags & 0x4000000 + + +# Test generator without docstring +def gen_no_doc(): + yield 1 + yield 2 + + +assert not (gen_no_doc.__code__.co_flags & 0x4000000) +assert gen_no_doc.__doc__ is None + + +# Test lambda - cannot have docstring lambda_f = lambda: 0 -assert lambda_f.__code__.co_consts[0] == None +assert not (lambda_f.__code__.co_flags & 0x4000000) +assert lambda_f.__doc__ is None + -class cls: - def f(): +# Test class method with docstring +class cls_with_doc: + def method(): + """Method docstring""" return 1 -assert cls().f.__code__.co_consts[0] == None + +assert cls_with_doc.method.__code__.co_consts[0] == "Method docstring" +assert cls_with_doc.method.__doc__ == "Method docstring" + + +# Test class method without docstring +class cls_no_doc: + def method(): + return 1 + + +assert not (cls_no_doc.method.__code__.co_flags & 0x4000000) +assert cls_no_doc.method.__doc__ is None + +print("All co_consts tests passed!") diff --git a/extra_tests/snippets/dir_main/__main__.py b/extra_tests/snippets/dir_main/__main__.py index c324c2e6e5e..2f9a147db12 100644 --- a/extra_tests/snippets/dir_main/__main__.py +++ b/extra_tests/snippets/dir_main/__main__.py @@ -1 +1 @@ -print('Hello') +print("Hello") diff --git a/extra_tests/snippets/dir_module/__init__.py b/extra_tests/snippets/dir_module/__init__.py index 5d9faff33d4..7408005f548 100644 --- a/extra_tests/snippets/dir_module/__init__.py +++ b/extra_tests/snippets/dir_module/__init__.py @@ -1,2 +1,2 @@ -from .relative import value from .dir_module_inner import value2 +from .relative import value diff --git a/extra_tests/snippets/example_fizzbuzz.py b/extra_tests/snippets/example_fizzbuzz.py index 7aafda0a8a1..cc6b76b0a67 100644 --- a/extra_tests/snippets/example_fizzbuzz.py +++ b/extra_tests/snippets/example_fizzbuzz.py @@ -8,6 +8,7 @@ def fizzbuzz(n): else: return str(n) + n = 1 while n < 10: print(fizzbuzz(n)) diff --git a/extra_tests/snippets/example_interactive.py b/extra_tests/snippets/example_interactive.py index 6e8b02c8620..5958dd11707 100644 --- a/extra_tests/snippets/example_interactive.py +++ b/extra_tests/snippets/example_interactive.py @@ -1,12 +1,14 @@ -c1 = compile("1 + 1", "", 'eval') +c1 = compile("1 + 1", "", "eval") code_class = type(c1) + def f(x, y, *args, power=1, **kwargs): - print("Constant String", 2, None, (2, 4)) + print("Constant String", 256, None, (2, 4)) assert code_class == type(c1) z = x * y - return z ** power + return z**power + c2 = f.__code__ # print(c2) @@ -17,14 +19,14 @@ def f(x, y, *args, power=1, **kwargs): # assert isinstance(c2.co_code, bytes) assert "Constant String" in c2.co_consts, c2.co_consts print(c2.co_consts) -assert 2 in c2.co_consts, c2.co_consts +assert 256 in c2.co_consts, c2.co_consts assert "example_interactive.py" in c2.co_filename -assert c2.co_firstlineno == 5, str(c2.co_firstlineno) +assert c2.co_firstlineno == 6, str(c2.co_firstlineno) # assert isinstance(c2.co_flags, int) # 'OPTIMIZED, NEWLOCALS, NOFREE' # assert c2.co_freevars == (), str(c2.co_freevars) -assert c2.co_kwonlyargcount == 1, (c2.co_kwonlyargcount) +assert c2.co_kwonlyargcount == 1, c2.co_kwonlyargcount # assert c2.co_lnotab == 0, c2.co_lnotab # b'\x00\x01' # Line number table -assert c2.co_name == 'f', c2.co_name +assert c2.co_name == "f", c2.co_name # assert c2.co_names == ('code_class', 'type', 'c1', 'AssertionError'), c2.co_names # , c2.co_names # assert c2.co_nlocals == 4, c2.co_nlocals # # assert c2.co_stacksize == 2, 'co_stacksize', diff --git a/extra_tests/snippets/forbidden_instantiation.py b/extra_tests/snippets/forbidden_instantiation.py index e5a178e8aeb..50b6f58f07f 100644 --- a/extra_tests/snippets/forbidden_instantiation.py +++ b/extra_tests/snippets/forbidden_instantiation.py @@ -1,19 +1,42 @@ -from typing import Type from types import ( - GeneratorType, CoroutineType, AsyncGeneratorType, BuiltinFunctionType, - BuiltinMethodType, WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, - ClassMethodDescriptorType, FrameType, GetSetDescriptorType, MemberDescriptorType + AsyncGeneratorType, + BuiltinFunctionType, + BuiltinMethodType, + ClassMethodDescriptorType, + CoroutineType, + FrameType, + GeneratorType, + GetSetDescriptorType, + MemberDescriptorType, + MethodDescriptorType, + MethodWrapperType, + WrapperDescriptorType, ) +from typing import Type + from testutils import assert_raises + def check_forbidden_instantiation(typ, reverse=False): f = reversed if reverse else iter with assert_raises(TypeError): type(f(typ()))() + dict_values, dict_items = lambda: {}.values(), lambda: {}.items() # types with custom forward iterators -iter_types = [list, set, str, bytearray, bytes, dict, tuple, lambda: range(0), dict_items, dict_values] +iter_types = [ + list, + set, + str, + bytearray, + bytes, + dict, + tuple, + lambda: range(0), + dict_items, + dict_values, +] # types with custom backwards iterators reviter_types = [list, dict, lambda: range(0), dict_values, dict_items] # internal types: @@ -22,14 +45,14 @@ def check_forbidden_instantiation(typ, reverse=False): CoroutineType, AsyncGeneratorType, BuiltinFunctionType, - BuiltinMethodType, # same as MethodWrapperType + BuiltinMethodType, # same as MethodWrapperType WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, ClassMethodDescriptorType, FrameType, - GetSetDescriptorType, # same as MemberDescriptorType - MemberDescriptorType + GetSetDescriptorType, # same as MemberDescriptorType + MemberDescriptorType, ] for typ in iter_types: @@ -38,4 +61,4 @@ def check_forbidden_instantiation(typ, reverse=False): check_forbidden_instantiation(typ, reverse=True) for typ in internal_types: with assert_raises(TypeError): - typ() \ No newline at end of file + typ() diff --git a/extra_tests/snippets/frozen.py b/extra_tests/snippets/frozen.py index d03658c1913..ccbe757319d 100644 --- a/extra_tests/snippets/frozen.py +++ b/extra_tests/snippets/frozen.py @@ -1,2 +1,3 @@ import __hello__ + assert __hello__.initialized == True diff --git a/extra_tests/snippets/import.py b/extra_tests/snippets/import.py index 309160d50f0..435e622418e 100644 --- a/extra_tests/snippets/import.py +++ b/extra_tests/snippets/import.py @@ -1,9 +1,11 @@ -import import_target, import_target as aliased -from import_target import func, other_func -from import_target import func as aliased_func, other_func as aliased_other_func +import import_mutual1 +import import_target +import import_target as aliased from import_star import * +from import_target import func, other_func +from import_target import func as aliased_func +from import_target import other_func as aliased_other_func -import import_mutual1 assert import_target.X == import_target.func() assert import_target.X == func() @@ -17,56 +19,60 @@ assert import_target.X == aliased_func() assert import_target.Y == aliased_other_func() -assert STAR_IMPORT == '123' +assert STAR_IMPORT == "123" try: from import_target import func, unknown_name - raise AssertionError('`unknown_name` does not cause an exception') + + raise AssertionError("`unknown_name` does not cause an exception") except ImportError: pass try: import mymodule except ModuleNotFoundError as exc: - assert exc.name == 'mymodule' + assert exc.name == "mymodule" test = __import__("import_target") assert test.X == import_target.X import builtins -class OverrideImportContext(): - def __enter__(self): - self.original_import = builtins.__import__ - def __exit__(self, exc_type, exc_val, exc_tb): - builtins.__import__ = self.original_import +class OverrideImportContext: + def __enter__(self): + self.original_import = builtins.__import__ + + def __exit__(self, exc_type, exc_val, exc_tb): + builtins.__import__ = self.original_import + with OverrideImportContext(): - def fake_import(name, globals=None, locals=None, fromlist=(), level=0): - return len(name) - builtins.__import__ = fake_import - import test - assert test == 4 + def fake_import(name, globals=None, locals=None, fromlist=(), level=0): + return len(name) + + builtins.__import__ = fake_import + import test + + assert test == 4 # TODO: Once we can determine current directory, use that to construct this # path: -#import sys -#sys.path.append("snippets/import_directory") -#import nested_target +# import sys +# sys.path.append("snippets/import_directory") +# import nested_target -#try: +# try: # X -#except NameError: +# except NameError: # pass -#else: +# else: # raise AssertionError('X should not be imported') from testutils import assert_raises with assert_raises(SyntaxError): - exec('import') - + exec("import") diff --git a/extra_tests/snippets/import_file.py b/extra_tests/snippets/import_file.py index 3f9eeed7048..17aae1b122c 100644 --- a/extra_tests/snippets/import_file.py +++ b/extra_tests/snippets/import_file.py @@ -1,4 +1,5 @@ import os + def import_file(): - assert os.path.basename(__file__) == "import_file.py" + assert os.path.basename(__file__) == "import_file.py" diff --git a/extra_tests/snippets/import_mutual1.py b/extra_tests/snippets/import_mutual1.py index 0dca4a34e0b..28239f977ed 100644 --- a/extra_tests/snippets/import_mutual1.py +++ b/extra_tests/snippets/import_mutual1.py @@ -1,4 +1,2 @@ - # Mutual recursive import: import import_mutual2 - diff --git a/extra_tests/snippets/import_mutual2.py b/extra_tests/snippets/import_mutual2.py index 388ce25217e..c9f0cbecb3c 100644 --- a/extra_tests/snippets/import_mutual2.py +++ b/extra_tests/snippets/import_mutual2.py @@ -1,3 +1,2 @@ - # Mutual recursive import: import import_mutual1 diff --git a/extra_tests/snippets/import_star.py b/extra_tests/snippets/import_star.py index efe23426fbc..c925972f739 100644 --- a/extra_tests/snippets/import_star.py +++ b/extra_tests/snippets/import_star.py @@ -1,3 +1,3 @@ # This is used by import.py; the two should be modified in concert -STAR_IMPORT = '123' +STAR_IMPORT = "123" diff --git a/extra_tests/snippets/import_target.py b/extra_tests/snippets/import_target.py index ba340ce3e02..da7b9214c55 100644 --- a/extra_tests/snippets/import_target.py +++ b/extra_tests/snippets/import_target.py @@ -1,10 +1,12 @@ # This is used by import.py; the two should be modified in concert -X = '123' -Y = 'abc' +X = "123" +Y = "abc" + def func(): return X + def other_func(): return Y diff --git a/extra_tests/snippets/intro/3.1.1.1.py b/extra_tests/snippets/intro/3.1.1.1.py index 8ef39bc8d97..0fe1dcc7e9d 100644 --- a/extra_tests/snippets/intro/3.1.1.1.py +++ b/extra_tests/snippets/intro/3.1.1.1.py @@ -1,8 +1,8 @@ assert 2 + 2 == 4 -assert 50 - 5*6 == 20 +assert 50 - 5 * 6 == 20 -assert (50 - 5*6) / 4 == 5 # This will crash -assert (50 - 5*6) / 4 == 5.0 +assert (50 - 5 * 6) / 4 == 5 # This will crash +assert (50 - 5 * 6) / 4 == 5.0 -assert 8 / 5 == 1.6 # division always returns a floating point number +assert 8 / 5 == 1.6 # division always returns a floating point number diff --git a/extra_tests/snippets/intro/3.1.1.3.py b/extra_tests/snippets/intro/3.1.1.3.py index 1972e615cde..e022c662726 100644 --- a/extra_tests/snippets/intro/3.1.1.3.py +++ b/extra_tests/snippets/intro/3.1.1.3.py @@ -1,3 +1,3 @@ -assert 25 == 5 ** 2 # 5 squared +assert 25 == 5**2 # 5 squared -assert 128 == 2 ** 7 # 2 to the power of 7 +assert 128 == 2**7 # 2 to the power of 7 diff --git a/extra_tests/snippets/intro/3.1.2.1.py b/extra_tests/snippets/intro/3.1.2.1.py index 9fd8ae6b062..ee823078ed5 100644 --- a/extra_tests/snippets/intro/3.1.2.1.py +++ b/extra_tests/snippets/intro/3.1.2.1.py @@ -1,7 +1,6 @@ -assert 'spam eggs' == 'spam eggs' # single quotes -assert "doesn't" == 'doesn\'t' # use \' to escape the single quote... +assert "spam eggs" == "spam eggs" # single quotes +assert "doesn't" == "doesn't" # use \' to escape the single quote... assert "doesn't" == "doesn't" # ...or use double quotes instead assert '"Yes," he said.' == '"Yes," he said.' -assert '"Yes," he said.' == "\"Yes,\" he said." +assert '"Yes," he said.' == '"Yes," he said.' assert '"Isn\'t," she said.' == '"Isn\'t," she said.' - diff --git a/extra_tests/snippets/intro/3.1.2.10.py b/extra_tests/snippets/intro/3.1.2.10.py index b69c2029c37..64603492895 100644 --- a/extra_tests/snippets/intro/3.1.2.10.py +++ b/extra_tests/snippets/intro/3.1.2.10.py @@ -1,6 +1,6 @@ -word = 'Python' -assert 'P' == word[0] # character in position 0 -assert 'n' == word[5] # character in position 5 -assert 'n' == word[-1] # last character -assert 'o' == word[-2] # second-last character -assert 'P' == word[-6] +word = "Python" +assert "P" == word[0] # character in position 0 +assert "n" == word[5] # character in position 5 +assert "n" == word[-1] # last character +assert "o" == word[-2] # second-last character +assert "P" == word[-6] diff --git a/extra_tests/snippets/intro/3.1.2.3.py b/extra_tests/snippets/intro/3.1.2.3.py index 9e63ae92b41..e509b535716 100644 --- a/extra_tests/snippets/intro/3.1.2.3.py +++ b/extra_tests/snippets/intro/3.1.2.3.py @@ -1 +1 @@ -print('C:\some\name') +print("C:\some\name") diff --git a/extra_tests/snippets/intro/3.1.2.5.py b/extra_tests/snippets/intro/3.1.2.5.py index 64ed5b98ab5..01489565dab 100644 --- a/extra_tests/snippets/intro/3.1.2.5.py +++ b/extra_tests/snippets/intro/3.1.2.5.py @@ -1 +1 @@ -assert 'unununium' == 3 * 'un' + 'ium' +assert "unununium" == 3 * "un" + "ium" diff --git a/extra_tests/snippets/intro/3.1.2.6.py b/extra_tests/snippets/intro/3.1.2.6.py index 92e434d6193..3f4580a99ac 100644 --- a/extra_tests/snippets/intro/3.1.2.6.py +++ b/extra_tests/snippets/intro/3.1.2.6.py @@ -1 +1 @@ -assert 'Python' == 'Py' 'thon' +assert "Python" == "Python" diff --git a/extra_tests/snippets/jit.py b/extra_tests/snippets/jit.py index 4b92fa235e6..887cbb50e7e 100644 --- a/extra_tests/snippets/jit.py +++ b/extra_tests/snippets/jit.py @@ -1,4 +1,3 @@ - def foo(): a = 5 return 10 + a diff --git a/extra_tests/snippets/name.py b/extra_tests/snippets/name.py index 97f9367ec01..fdf34db6985 100644 --- a/extra_tests/snippets/name.py +++ b/extra_tests/snippets/name.py @@ -1,9 +1,9 @@ -#when name.py is run __name__ should equal to __main__ +# when name.py is run __name__ should equal to __main__ assert __name__ == "__main__" from import_name import import_func -#__name__ should be set to import_func +# __name__ should be set to import_func import_func() assert __name__ == "__main__" diff --git a/extra_tests/snippets/operator_arithmetic.py b/extra_tests/snippets/operator_arithmetic.py index d44728d0ffd..c698997af12 100644 --- a/extra_tests/snippets/operator_arithmetic.py +++ b/extra_tests/snippets/operator_arithmetic.py @@ -5,7 +5,7 @@ a = 4 -assert a ** 3 == 64 +assert a**3 == 64 assert a * 3 == 12 assert a / 2 == 2 assert 2 == a / 2 @@ -32,3 +32,23 @@ # Right shift raises value error on negative assert_raises(ValueError, lambda: 1 >> -1) + +# Bitwise or, and, xor raises value error on incompatible types +assert_raises(TypeError, lambda: "abc" | True) +assert_raises(TypeError, lambda: "abc" & True) +assert_raises(TypeError, lambda: "abc" ^ True) +assert_raises(TypeError, lambda: True | "abc") +assert_raises(TypeError, lambda: True & "abc") +assert_raises(TypeError, lambda: True ^ "abc") +assert_raises(TypeError, lambda: "abc" | 1.5) +assert_raises(TypeError, lambda: "abc" & 1.5) +assert_raises(TypeError, lambda: "abc" ^ 1.5) +assert_raises(TypeError, lambda: 1.5 | "abc") +assert_raises(TypeError, lambda: 1.5 & "abc") +assert_raises(TypeError, lambda: 1.5 ^ "abc") +assert_raises(TypeError, lambda: True | 1.5) +assert_raises(TypeError, lambda: True & 1.5) +assert_raises(TypeError, lambda: True ^ 1.5) +assert_raises(TypeError, lambda: 1.5 | True) +assert_raises(TypeError, lambda: 1.5 & True) +assert_raises(TypeError, lambda: 1.5 ^ True) diff --git a/extra_tests/snippets/operator_cast.py b/extra_tests/snippets/operator_cast.py index fdf1613beb0..21a9237baf4 100644 --- a/extra_tests/snippets/operator_cast.py +++ b/extra_tests/snippets/operator_cast.py @@ -1,39 +1,39 @@ x = 1 y = 1.1 -assert x+y == 2.1 -#print(x+y) +assert x + y == 2.1 +# print(x+y) x = 1.1 y = 1 -assert x+y == 2.1 -#print(x+y) +assert x + y == 2.1 +# print(x+y) x = 1.1 y = 2.1 -assert x+y == 3.2 -#print(x+y) +assert x + y == 3.2 +# print(x+y) x = "ab" y = "cd" -assert x+y == "abcd" -#print(x+y) +assert x + y == "abcd" +# print(x+y) x = 2 y = 3 assert x**y == 8 -#print(x**y) +# print(x**y) x = 2.0 y = 3 assert x**y == 8.0 -#print(x**y) +# print(x**y) x = 2 y = 3.0 assert x**y == 8.0 -#print(x**y) +# print(x**y) x = 2.0 y = 3.0 assert x**y == 8.0 -#print(x**y) +# print(x**y) diff --git a/extra_tests/snippets/operator_comparison.py b/extra_tests/snippets/operator_comparison.py index 644a5ea6a34..71231f033dc 100644 --- a/extra_tests/snippets/operator_comparison.py +++ b/extra_tests/snippets/operator_comparison.py @@ -13,12 +13,14 @@ assert not 1 < 2 > 3 < 4 assert not 1 > 2 < 3 < 4 + def test_type_error(x, y): assert_raises(TypeError, lambda: x < y) assert_raises(TypeError, lambda: x <= y) assert_raises(TypeError, lambda: x > y) assert_raises(TypeError, lambda: x >= y) + test_type_error([], 0) test_type_error((), 0) @@ -34,6 +36,7 @@ def test_type_error(x, y): # floats that cannot be converted to big ints shouldn’t crash the vm import math + assert not (10**500 == math.inf) assert not (math.inf == 10**500) assert not (10**500 == math.nan) @@ -41,23 +44,23 @@ def test_type_error(x, y): # comparisons # floats with worse than integer precision -assert 2.**54 > 2**54 - 1 -assert 2.**54 < 2**54 + 1 -assert 2.**54 >= 2**54 - 1 -assert 2.**54 <= 2**54 + 1 -assert 2.**54 == 2**54 -assert not 2.**54 == 2**54 + 1 +assert 2.0**54 > 2**54 - 1 +assert 2.0**54 < 2**54 + 1 +assert 2.0**54 >= 2**54 - 1 +assert 2.0**54 <= 2**54 + 1 +assert 2.0**54 == 2**54 +assert not 2.0**54 == 2**54 + 1 # inverse operands -assert 2**54 - 1 < 2.**54 -assert 2**54 + 1 > 2.**54 -assert 2**54 - 1 <= 2.**54 -assert 2**54 + 1 >= 2.**54 -assert 2**54 == 2.**54 -assert not 2**54 + 1 == 2.**54 +assert 2**54 - 1 < 2.0**54 +assert 2**54 + 1 > 2.0**54 +assert 2**54 - 1 <= 2.0**54 +assert 2**54 + 1 >= 2.0**54 +assert 2**54 == 2.0**54 +assert not 2**54 + 1 == 2.0**54 -assert not 2.**54 < 2**54 - 1 -assert not 2.**54 > 2**54 + 1 +assert not 2.0**54 < 2**54 - 1 +assert not 2.0**54 > 2**54 + 1 # sub-int numbers assert 1.3 > 1 @@ -68,17 +71,17 @@ def test_type_error(x, y): assert -0.3 <= 0 # int out of float range comparisons -assert 10**500 > 2.**54 -assert -10**500 < -0.12 +assert 10**500 > 2.0**54 +assert -(10**500) < -0.12 # infinity and NaN comparisons assert math.inf > 10**500 assert math.inf >= 10**500 assert not math.inf < 10**500 -assert -math.inf < -10*500 -assert -math.inf <= -10*500 -assert not -math.inf > -10*500 +assert -math.inf < -10 * 500 +assert -math.inf <= -10 * 500 +assert not -math.inf > -10 * 500 assert not math.nan > 123 assert not math.nan < 123 diff --git a/extra_tests/snippets/operator_div.py b/extra_tests/snippets/operator_div.py index 8520a877c55..e99533cbd5e 100644 --- a/extra_tests/snippets/operator_div.py +++ b/extra_tests/snippets/operator_div.py @@ -2,7 +2,7 @@ assert_raises(ZeroDivisionError, lambda: 5 / 0) assert_raises(ZeroDivisionError, lambda: 5 / -0.0) -assert_raises(ZeroDivisionError, lambda: 5 / (2-2)) +assert_raises(ZeroDivisionError, lambda: 5 / (2 - 2)) assert_raises(ZeroDivisionError, lambda: 5 % 0) assert_raises(ZeroDivisionError, lambda: 5 // 0) assert_raises(ZeroDivisionError, lambda: 5.3 // (-0.0)) @@ -18,12 +18,16 @@ res = 10**3000 / (10**2998 + 5 * 10**2996) assert 95.238095 <= res <= 95.238096 -assert 10**500 / (2*10**(500-308)) == 5e307 -assert 10**500 / (10**(500-308)) == 1e308 -assert_raises(OverflowError, lambda: 10**500 / (10**(500-309)), _msg='too big result') +assert 10**500 / (2 * 10 ** (500 - 308)) == 5e307 +assert 10**500 / (10 ** (500 - 308)) == 1e308 +assert_raises( + OverflowError, lambda: 10**500 / (10 ** (500 - 309)), _msg="too big result" +) # a bit more than f64::MAX = 1.7976931348623157e+308_f64 assert (2 * 10**308) / 2 == 1e308 # when dividing too big int by a float, the operation should fail -assert_raises(OverflowError, lambda: (2 * 10**308) / 2.0, _msg='division of big int by float') +assert_raises( + OverflowError, lambda: (2 * 10**308) / 2.0, _msg="division of big int by float" +) diff --git a/extra_tests/snippets/operator_membership.py b/extra_tests/snippets/operator_membership.py index 2987c3c0fef..07065e2244f 100644 --- a/extra_tests/snippets/operator_membership.py +++ b/extra_tests/snippets/operator_membership.py @@ -46,15 +46,16 @@ assert 1 in range(0, 2) assert 3 not in range(0, 2) + # test __contains__ in user objects -class MyNotContainingClass(): +class MyNotContainingClass: pass assert_raises(TypeError, lambda: 1 in MyNotContainingClass()) -class MyContainingClass(): +class MyContainingClass: def __init__(self, value): self.value = value diff --git a/extra_tests/snippets/protocol_callable.py b/extra_tests/snippets/protocol_callable.py index c549ef468e2..1df0e717935 100644 --- a/extra_tests/snippets/protocol_callable.py +++ b/extra_tests/snippets/protocol_callable.py @@ -1,4 +1,4 @@ -class Callable(): +class Callable: def __init__(self): self.count = 0 @@ -6,13 +6,16 @@ def __call__(self): self.count += 1 return self.count + c = Callable() assert 1 == c() assert 2 == c() + class Inherited(Callable): pass + i = Inherited() assert 1 == i() diff --git a/extra_tests/snippets/protocol_index_bad.py b/extra_tests/snippets/protocol_index_bad.py index af71f2e689f..51d90426023 100644 --- a/extra_tests/snippets/protocol_index_bad.py +++ b/extra_tests/snippets/protocol_index_bad.py @@ -1,8 +1,10 @@ -""" Test that indexing ops don't hang when an object with a mutating +"""Test that indexing ops don't hang when an object with a mutating __index__ is used.""" -from testutils import assert_raises + from array import array +from testutils import assert_raises + class BadIndex: def __index__(self): @@ -15,18 +17,19 @@ def __index__(self): def run_setslice(): with assert_raises(IndexError): e[BadIndex()] = 42 - e[BadIndex():0:-1] = e - e[0:BadIndex():1] = e - e[0:10:BadIndex()] = e + e[BadIndex() : 0 : -1] = e + e[0 : BadIndex() : 1] = e + e[0 : 10 : BadIndex()] = e def run_delslice(): - del e[BadIndex():0:-1] - del e[0:BadIndex():1] - del e[0:10:BadIndex()] + del e[BadIndex() : 0 : -1] + del e[0 : BadIndex() : 1] + del e[0 : 10 : BadIndex()] + -# Check types -instances = [list(), bytearray(), array('b')] +# Check types +instances = [list(), bytearray(), array("b")] for e in instances: run_setslice() - run_delslice() \ No newline at end of file + run_delslice() diff --git a/extra_tests/snippets/protocol_iterable.py b/extra_tests/snippets/protocol_iterable.py index 7158296c389..7f32504d0a2 100644 --- a/extra_tests/snippets/protocol_iterable.py +++ b/extra_tests/snippets/protocol_iterable.py @@ -1,5 +1,6 @@ from testutils import assert_raises + def test_container(x): assert 3 in x assert 4 not in x @@ -10,27 +11,40 @@ def test_container(x): lst.extend(x) assert lst == [0, 1, 2, 3] + class C: def __iter__(self): return iter([0, 1, 2, 3]) + + test_container(C()) + class C: def __getitem__(self, x): - return (0, 1, 2, 3)[x] # raises IndexError on x==4 + return (0, 1, 2, 3)[x] # raises IndexError on x==4 + + test_container(C()) + class C: def __getitem__(self, x): if x > 3: raise StopIteration return x + + test_container(C()) -class C: pass + +class C: + pass + + assert_raises(TypeError, lambda: 5 in C()) assert_raises(TypeError, iter, C) -it = iter([1,2,3,4,5]) +it = iter([1, 2, 3, 4, 5]) call_it = iter(lambda: next(it), 4) -assert list(call_it) == [1,2,3] +assert list(call_it) == [1, 2, 3] diff --git a/extra_tests/snippets/protocol_iternext.py b/extra_tests/snippets/protocol_iternext.py index b2b30961c05..0eff9cff4ec 100644 --- a/extra_tests/snippets/protocol_iternext.py +++ b/extra_tests/snippets/protocol_iternext.py @@ -1,5 +1,3 @@ - - ls = [1, 2, 3] i = iter(ls) @@ -7,10 +5,10 @@ assert i.__next__() == 2 assert next(i) == 3 -assert next(i, 'w00t') == 'w00t' +assert next(i, "w00t") == "w00t" -s = '你好' +s = "你好" i = iter(s) i.__setstate__(1) -assert i.__next__() == '好' +assert i.__next__() == "好" assert i.__reduce__()[2] == 2 diff --git a/extra_tests/snippets/recursion.py b/extra_tests/snippets/recursion.py index f2a8d4e11d3..2d3b2205d68 100644 --- a/extra_tests/snippets/recursion.py +++ b/extra_tests/snippets/recursion.py @@ -1,8 +1,10 @@ from testutils import assert_raises + class Foo(object): pass + Foo.__repr__ = Foo.__str__ foo = Foo() diff --git a/extra_tests/snippets/sandbox_smoke.py b/extra_tests/snippets/sandbox_smoke.py new file mode 100644 index 00000000000..13fa0722742 --- /dev/null +++ b/extra_tests/snippets/sandbox_smoke.py @@ -0,0 +1,55 @@ +"""Sandbox mode smoke test. + +Verifies basic functionality that works in both sandbox and normal mode: +- stdio (print, sys.stdout/stdin/stderr) +- builtin modules (math, json) +- in-memory IO (BytesIO, StringIO) +- open() is properly blocked when FileIO is unavailable (sandbox) +""" + +import _io +import json +import math +import sys + +SANDBOX = not hasattr(_io, "FileIO") + +# stdio +print("1. print works") +assert sys.stdout.writable() +assert sys.stderr.writable() +assert sys.stdin.readable() +assert sys.stdout.fileno() == 1 + +# math +assert math.pi > 3.14 +print("2. math works:", math.pi) + +# json +d = json.loads('{"a": 1}') +assert d == {"a": 1} +print("3. json works:", d) + +# BytesIO / StringIO +buf = _io.BytesIO(b"hello") +assert buf.read() == b"hello" +sio = _io.StringIO("world") +assert sio.read() == "world" +print("4. BytesIO/StringIO work") + +# open() behavior depends on mode +if SANDBOX: + try: + open("/tmp/x", "w") + assert False, "should have raised" + except _io.UnsupportedOperation: + print("5. open() properly blocked (sandbox)") +else: + print("5. open() available (host_env)") + +# builtins +assert list(range(5)) == [0, 1, 2, 3, 4] +assert sorted([3, 1, 2]) == [1, 2, 3] +print("6. builtins work") + +print("All smoke tests passed!", "(sandbox)" if SANDBOX else "(host_env)") diff --git a/extra_tests/snippets/stdlib_abc_number.py b/extra_tests/snippets/stdlib_abc_number.py index c6aee97ec8b..2c1e81c1f86 100644 --- a/extra_tests/snippets/stdlib_abc_number.py +++ b/extra_tests/snippets/stdlib_abc_number.py @@ -71,4 +71,4 @@ class A(int): assert 1_2.3_4e0_0 == 12.34 with assert_raises(SyntaxError): - eval('1__2') + eval("1__2") diff --git a/extra_tests/snippets/stdlib_array.py b/extra_tests/snippets/stdlib_array.py index 9d9f99695de..ed2a8f22369 100644 --- a/extra_tests/snippets/stdlib_array.py +++ b/extra_tests/snippets/stdlib_array.py @@ -1,7 +1,8 @@ -from testutils import assert_raises from array import array from pickle import dumps, loads +from testutils import assert_raises + a1 = array("b", [0, 1, 2, 3]) assert a1.tobytes() == b"\x00\x01\x02\x03" @@ -23,6 +24,7 @@ b = array("B", [3, 2, 1, 0]) assert a.__ne__(b) is True + def test_float_with_integer_input(): f = array("f", [0, 1, 2.0, 3.0]) f.append(4) @@ -33,10 +35,11 @@ def test_float_with_integer_input(): f[0] = -2 assert f == array("f", [-2, 0, 2, 3, 4]) + test_float_with_integer_input() # slice assignment step overflow behaviour test -T = 'I' +T = "I" a = array(T, range(10)) b = array(T, [100]) a[::9999999999] = b @@ -57,9 +60,10 @@ def test_float_with_integer_input(): del a[0:0:-9999999999] assert a == array(T, [1, 2, 3, 4, 5, 6, 7, 8]) + def test_float_with_nan(): - f = float('nan') - a = array('f') + f = float("nan") + a = array("f") a.append(f) assert not (a == a) assert a != a @@ -68,30 +72,35 @@ def test_float_with_nan(): assert not (a > a) assert not (a >= a) + test_float_with_nan() + def test_different_type_cmp(): - a = array('i', [-1, -2, -3, -4]) - b = array('I', [1, 2, 3, 4]) - c = array('f', [1, 2, 3, 4]) + a = array("i", [-1, -2, -3, -4]) + b = array("I", [1, 2, 3, 4]) + c = array("f", [1, 2, 3, 4]) assert a < b assert b > a assert b == c assert a < c assert c > a + test_different_type_cmp() + def test_array_frombytes(): - a = array('b', [-1, -2]) + a = array("b", [-1, -2]) b = bytearray(a.tobytes()) - c = array('b', b) + c = array("b", b) assert a == c + test_array_frombytes() # test that indexing on an empty array doesn't panic -a = array('b') +a = array("b") with assert_raises(IndexError): a[0] with assert_raises(IndexError): @@ -99,21 +108,38 @@ def test_array_frombytes(): with assert_raises(IndexError): del a[42] -test_str = '🌉abc🌐def🌉🌐' -u = array('u', test_str) -# skip as 2 bytes character enviroment with CPython is failing the test +test_str = "🌉abc🌐def🌉🌐" +u = array("u", test_str) +# skip as 2 bytes character environment with CPython is failing the test if u.itemsize >= 4: assert u.__reduce_ex__(1)[1][1] == list(test_str) assert loads(dumps(u, 1)) == loads(dumps(u, 3)) # test array name -a = array('b', []) +a = array("b", []) assert str(a.__class__.__name__) == "array" # test arrayiterator name i = iter(a) assert str(i.__class__.__name__) == "arrayiterator" # teset array.__contains__ -a = array('B', [0]) +a = array("B", [0]) assert a.__contains__(0) assert not a.__contains__(1) + + +class _ReenteringWriter: + def __init__(self, arr): + self.arr = arr + self.reentered = False + + def write(self, chunk): + if not self.reentered: + self.reentered = True + self.arr.append(0) + return len(chunk) + + +arr = array("b", range(128)) +arr.tofile(_ReenteringWriter(arr)) +assert len(arr) == 129 diff --git a/extra_tests/snippets/stdlib_ast.py b/extra_tests/snippets/stdlib_ast.py index 08c1b3b76ee..dc626506fa9 100644 --- a/extra_tests/snippets/stdlib_ast.py +++ b/extra_tests/snippets/stdlib_ast.py @@ -1,5 +1,5 @@ - import ast + print(ast) source = """ @@ -11,30 +11,29 @@ def foo(): print(n) print(n.body) print(n.body[0].name) -assert n.body[0].name == 'foo' +assert n.body[0].name == "foo" foo = n.body[0] assert foo.lineno == 2 print(foo.body) assert len(foo.body) == 2 print(foo.body[0]) print(foo.body[0].value.func.id) -assert foo.body[0].value.func.id == 'print' +assert foo.body[0].value.func.id == "print" assert foo.body[0].lineno == 3 assert foo.body[1].lineno == 4 n = ast.parse("3 < 4 > 5\n") assert n.body[0].value.left.value == 3 -assert 'Lt' in str(n.body[0].value.ops[0]) -assert 'Gt' in str(n.body[0].value.ops[1]) +assert "Lt" in str(n.body[0].value.ops[0]) +assert "Gt" in str(n.body[0].value.ops[1]) assert n.body[0].value.comparators[0].value == 4 assert n.body[0].value.comparators[1].value == 5 -n = ast.parse('from ... import a\n') +n = ast.parse("from ... import a\n") print(n) i = n.body[0] assert i.level == 3 assert i.module is None -assert i.names[0].name == 'a' +assert i.names[0].name == "a" assert i.names[0].asname is None - diff --git a/extra_tests/snippets/stdlib_binascii.py b/extra_tests/snippets/stdlib_binascii.py index 5dcb5a03227..c4c2121fa46 100644 --- a/extra_tests/snippets/stdlib_binascii.py +++ b/extra_tests/snippets/stdlib_binascii.py @@ -1,6 +1,6 @@ import binascii -from testutils import assert_raises, assert_equal +from testutils import assert_equal, assert_raises # hexlify tests h = binascii.hexlify diff --git a/extra_tests/snippets/stdlib_collections.py b/extra_tests/snippets/stdlib_collections.py index 641a6e2a259..8a9bdb45336 100644 --- a/extra_tests/snippets/stdlib_collections.py +++ b/extra_tests/snippets/stdlib_collections.py @@ -1,6 +1,5 @@ from collections import deque - d = deque([0, 1, 2]) d.append(1) @@ -50,9 +49,10 @@ class BadRepr: def __repr__(self): self.d.pop() - return '' + return "" + b = BadRepr() d = deque([1, b, 2]) b.d = d -repr(d) \ No newline at end of file +repr(d) diff --git a/extra_tests/snippets/stdlib_collections_deque.py b/extra_tests/snippets/stdlib_collections_deque.py index 44498633bf3..25b2069f6e8 100644 --- a/extra_tests/snippets/stdlib_collections_deque.py +++ b/extra_tests/snippets/stdlib_collections_deque.py @@ -1,13 +1,14 @@ -from testutils import assert_raises from collections import deque from typing import Deque +from testutils import assert_raises + def test_deque_iterator__new__(): klass = type(iter(deque())) - s = 'abcd' + s = "abcd" d = klass(deque(s)) - assert (list(d) == list(s)) + assert list(d) == list(s) test_deque_iterator__new__() @@ -17,22 +18,22 @@ def test_deque_iterator__new__positional_index(): klass = type(iter(deque())) # index between 0 and len - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(len(s)): d = klass(deque(s), i) - assert (list(d) == list(s)[i:]) + assert list(d) == list(s)[i:] # negative index - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(-100, 0): d = klass(deque(s), i) - assert (list(d) == list(s)) + assert list(d) == list(s) # index ge len - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(len(s), 400): d = klass(deque(s), i) - assert (list(d) == list()) + assert list(d) == list() test_deque_iterator__new__positional_index() @@ -41,10 +42,10 @@ def test_deque_iterator__new__positional_index(): def test_deque_iterator__new__not_using_keyword_index(): klass = type(iter(deque())) - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(-100, 400): d = klass(deque(s), index=i) - assert (list(d) == list(s)) + assert list(d) == list(s) test_deque_iterator__new__not_using_keyword_index() @@ -54,22 +55,22 @@ def test_deque_reverse_iterator__new__positional_index(): klass = type(reversed(deque())) # index between 0 and len - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(len(s)): d = klass(deque(s), i) - assert (list(d) == list(reversed(s))[i:]) + assert list(d) == list(reversed(s))[i:] # negative index - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(-100, 0): d = klass(deque(s), i) - assert (list(d) == list(reversed(s))) + assert list(d) == list(reversed(s)) # index ge len - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(len(s), 400): d = klass(deque(s), i) - assert (list(d) == list()) + assert list(d) == list() test_deque_reverse_iterator__new__positional_index() @@ -78,10 +79,10 @@ def test_deque_reverse_iterator__new__positional_index(): def test_deque_reverse_iterator__new__not_using_keyword_index(): klass = type(reversed(deque())) - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(-100, 400): d = klass(deque(s), index=i) - assert (list(d) == list(reversed(s))) + assert list(d) == list(reversed(s)) test_deque_reverse_iterator__new__not_using_keyword_index() @@ -89,11 +90,13 @@ def test_deque_reverse_iterator__new__not_using_keyword_index(): assert repr(deque()) == "deque([])" assert repr(deque([1, 2, 3])) == "deque([1, 2, 3])" + class D(deque): pass + assert repr(D()) == "D([])" assert repr(D([1, 2, 3])) == "D([1, 2, 3])" -assert_raises(ValueError, lambda: deque().index(10,0,10000000000000000000000000)) \ No newline at end of file +assert_raises(ValueError, lambda: deque().index(10, 0, 10000000000000000000000000)) diff --git a/extra_tests/snippets/stdlib_csv.py b/extra_tests/snippets/stdlib_csv.py index 6ba66d30f74..eb3461e9082 100644 --- a/extra_tests/snippets/stdlib_csv.py +++ b/extra_tests/snippets/stdlib_csv.py @@ -1,45 +1,49 @@ +import csv + from testutils import assert_raises -import csv +for row in csv.reader(["one,two,three"]): + [one, two, three] = row + assert one == "one" + assert two == "two" + assert three == "three" -for row in csv.reader(['one,two,three']): - [one, two, three] = row - assert one == 'one' - assert two == 'two' - assert three == 'three' def f(): - iter = ['one,two,three', 'four,five,six'] - reader = csv.reader(iter) + iter = ["one,two,three", "four,five,six"] + reader = csv.reader(iter) - [one,two,three] = next(reader) - [four,five,six] = next(reader) + [one, two, three] = next(reader) + [four, five, six] = next(reader) + + assert one == "one" + assert two == "two" + assert three == "three" + assert four == "four" + assert five == "five" + assert six == "six" - assert one == 'one' - assert two == 'two' - assert three == 'three' - assert four == 'four' - assert five == 'five' - assert six == 'six' f() + def test_delim(): - iter = ['one|two|three', 'four|five|six'] - reader = csv.reader(iter, delimiter='|') - - [one,two,three] = next(reader) - [four,five,six] = next(reader) - - assert one == 'one' - assert two == 'two' - assert three == 'three' - assert four == 'four' - assert five == 'five' - assert six == 'six' - - with assert_raises(TypeError): - iter = ['one,,two,,three'] - csv.reader(iter, delimiter=',,') + iter = ["one|two|three", "four|five|six"] + reader = csv.reader(iter, delimiter="|") + + [one, two, three] = next(reader) + [four, five, six] = next(reader) + + assert one == "one" + assert two == "two" + assert three == "three" + assert four == "four" + assert five == "five" + assert six == "six" + + with assert_raises(TypeError): + iter = ["one,,two,,three"] + csv.reader(iter, delimiter=",,") + test_delim() diff --git a/extra_tests/snippets/stdlib_ctypes.py b/extra_tests/snippets/stdlib_ctypes.py new file mode 100644 index 00000000000..0a5d1387a8d --- /dev/null +++ b/extra_tests/snippets/stdlib_ctypes.py @@ -0,0 +1,401 @@ +import os as _os +import sys as _sys +import types as _types +from _ctypes import RTLD_GLOBAL, RTLD_LOCAL, Array, _SimpleCData, sizeof +from _ctypes import CFuncPtr as _CFuncPtr +from struct import calcsize as _calcsize + +assert Array.__class__.__name__ == "PyCArrayType" +assert Array.__base__.__name__ == "_CData" + +DEFAULT_MODE = RTLD_LOCAL +if _os.name == "posix" and _sys.platform == "darwin": + # On OS X 10.3, we use RTLD_GLOBAL as default mode + # because RTLD_LOCAL does not work at least on some + # libraries. OS X 10.3 is Darwin 7, so we check for + # that. + + if int(_os.uname().release.split(".")[0]) < 8: + DEFAULT_MODE = RTLD_GLOBAL + +from _ctypes import ( + FUNCFLAG_CDECL as _FUNCFLAG_CDECL, +) +from _ctypes import ( + FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI, +) +from _ctypes import ( + FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, +) +from _ctypes import ( + FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR, +) + + +def create_string_buffer(init, size=None): + """create_string_buffer(aBytes) -> character array + create_string_buffer(anInteger) -> character array + create_string_buffer(aBytes, anInteger) -> character array + """ + if isinstance(init, bytes): + if size is None: + size = len(init) + 1 + _sys.audit("ctypes.create_string_buffer", init, size) + buftype = c_char.__mul__(size) + # buftype = c_char * size + buf = buftype() + buf.value = init + return buf + elif isinstance(init, int): + _sys.audit("ctypes.create_string_buffer", None, init) + buftype = c_char.__mul__(init) + # buftype = c_char * init + buf = buftype() + return buf + raise TypeError(init) + + +def _check_size(typ, typecode=None): + # Check if sizeof(ctypes_type) against struct.calcsize. This + # should protect somewhat against a misconfigured libffi. + from struct import calcsize + + if typecode is None: + # Most _type_ codes are the same as used in struct + typecode = typ._type_ + actual, required = sizeof(typ), calcsize(typecode) + if actual != required: + raise SystemError( + "sizeof(%s) wrong: %d instead of %d" % (typ, actual, required) + ) + + +class c_short(_SimpleCData): + _type_ = "h" + + +_check_size(c_short) + + +class c_ushort(_SimpleCData): + _type_ = "H" + + +_check_size(c_ushort) + + +class c_long(_SimpleCData): + _type_ = "l" + + +_check_size(c_long) + + +class c_ulong(_SimpleCData): + _type_ = "L" + + +_check_size(c_ulong) + +if _calcsize("i") == _calcsize("l"): + # if int and long have the same size, make c_int an alias for c_long + c_int = c_long + c_uint = c_ulong +else: + + class c_int(_SimpleCData): + _type_ = "i" + + _check_size(c_int) + + class c_uint(_SimpleCData): + _type_ = "I" + + _check_size(c_uint) + + +class c_float(_SimpleCData): + _type_ = "f" + + +_check_size(c_float) + + +class c_double(_SimpleCData): + _type_ = "d" + + +_check_size(c_double) + + +class c_longdouble(_SimpleCData): + _type_ = "g" + + +if sizeof(c_longdouble) == sizeof(c_double): + c_longdouble = c_double + +if _calcsize("l") == _calcsize("q"): + # if long and long long have the same size, make c_longlong an alias for c_long + c_longlong = c_long + c_ulonglong = c_ulong +else: + + class c_longlong(_SimpleCData): + _type_ = "q" + + _check_size(c_longlong) + + class c_ulonglong(_SimpleCData): + _type_ = "Q" + + ## def from_param(cls, val): + ## return ('d', float(val), val) + ## from_param = classmethod(from_param) + _check_size(c_ulonglong) + + +class c_ubyte(_SimpleCData): + _type_ = "B" + + +c_ubyte.__ctype_le__ = c_ubyte.__ctype_be__ = c_ubyte +# backward compatibility: +##c_uchar = c_ubyte +_check_size(c_ubyte) + + +class c_byte(_SimpleCData): + _type_ = "b" + + +c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte +_check_size(c_byte) + + +class c_char(_SimpleCData): + _type_ = "c" + + +c_char.__ctype_le__ = c_char.__ctype_be__ = c_char +_check_size(c_char) + + +class c_char_p(_SimpleCData): + _type_ = "z" + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value) + + +_check_size(c_char_p, "P") + + +class c_void_p(_SimpleCData): + _type_ = "P" + + +c_voidp = c_void_p # backwards compatibility (to a bug) +_check_size(c_void_p) + + +class c_bool(_SimpleCData): + _type_ = "?" + + +_check_size(c_bool) + +i = c_int(42) +f = c_float(3.14) +# s = create_string_buffer(b'\000' * 32) +assert i.value == 42 +assert abs(f.value - 3.14) < 1e-06 + +if _os.name == "nt": + from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL + from _ctypes import LoadLibrary as _dlopen +elif _os.name == "posix": + from _ctypes import dlopen as _dlopen + + +class CDLL(object): + """An instance of this class represents a loaded dll/shared + library, exporting functions using the standard C calling + convention (named 'cdecl' on Windows). + + The exported functions can be accessed as attributes, or by + indexing with the function name. Examples: + + <obj>.qsort -> callable object + <obj>['qsort'] -> callable object + + Calling the functions releases the Python GIL during the call and + reacquires it afterwards. + """ + + _func_flags_ = _FUNCFLAG_CDECL + _func_restype_ = c_int + # default values for repr + _name = "<uninitialized>" + _handle = 0 + _FuncPtr = None + + def __init__( + self, + name, + mode=DEFAULT_MODE, + handle=None, + use_errno=False, + use_last_error=False, + winmode=None, + ): + self._name = name + flags = self._func_flags_ + if use_errno: + flags |= _FUNCFLAG_USE_ERRNO + if use_last_error: + flags |= _FUNCFLAG_USE_LASTERROR + if _sys.platform.startswith("aix"): + """When the name contains ".a(" and ends with ")", + e.g., "libFOO.a(libFOO.so)" - this is taken to be an + archive(member) syntax for dlopen(), and the mode is adjusted. + Otherwise, name is presented to dlopen() as a file argument. + """ + if name and name.endswith(")") and ".a(" in name: + mode |= _os.RTLD_MEMBER | _os.RTLD_NOW + if _os.name == "nt": + if winmode is not None: + mode = winmode + else: + import nt + + mode = 4096 + if "/" in name or "\\" in name: + self._name = nt._getfullpathname(self._name) + mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR + + class _FuncPtr(_CFuncPtr): + _flags_ = flags + _restype_ = self._func_restype_ + + self._FuncPtr = _FuncPtr + + if handle is None: + self._handle = _dlopen(self._name, mode) + else: + self._handle = handle + + def __repr__(self): + return "<%s '%s', handle %x at %#x>" % ( + self.__class__.__name__, + self._name, + (self._handle & (_sys.maxsize * 2 + 1)), + id(self) & (_sys.maxsize * 2 + 1), + ) + + def __getattr__(self, name): + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + func = self.__getitem__(name) + setattr(self, name, func) + return func + + def __getitem__(self, name_or_ordinal): + func = self._FuncPtr((name_or_ordinal, self)) + if not isinstance(name_or_ordinal, int): + func.__name__ = name_or_ordinal + return func + + +class LibraryLoader(object): + def __init__(self, dlltype): + self._dlltype = dlltype + + def __getattr__(self, name): + if name[0] == "_": + raise AttributeError(name) + try: + dll = self._dlltype(name) + except OSError: + raise AttributeError(name) + setattr(self, name, dll) + return dll + + def __getitem__(self, name): + return getattr(self, name) + + def LoadLibrary(self, name): + return self._dlltype(name) + + __class_getitem__ = classmethod(_types.GenericAlias) + + +cdll = LibraryLoader(CDLL) + +test_byte_array = create_string_buffer(b"Hello, World!\n") +assert test_byte_array._length_ == 15 + +if _os.name == "posix": + if _sys.platform == "darwin": + libc = cdll.LoadLibrary("libc.dylib") + libc.rand() + i = c_int(1) + # print("start srand") + # print(libc.srand(i)) + # print(test_byte_array) +else: + import os + + libc = cdll.msvcrt + libc.rand() + i = c_int(1) + print("start srand") + # print(libc.srand(i)) + # print(test_byte_array) + # print(test_byte_array._type_) + # print("start printf") + # libc.printf(test_byte_array) + + # windows pip support + + def get_win_folder_via_ctypes(csidl_name: str) -> str: + """Get folder with ctypes.""" + # There is no 'CSIDL_DOWNLOADS'. + # Use 'CSIDL_PROFILE' (40) and append the default folder 'Downloads' instead. + # https://learn.microsoft.com/en-us/windows/win32/shell/knownfolderid + + import ctypes # noqa: PLC0415 + + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + "CSIDL_PERSONAL": 5, + "CSIDL_MYPICTURES": 39, + "CSIDL_MYVIDEO": 14, + "CSIDL_MYMUSIC": 13, + "CSIDL_DOWNLOADS": 40, + "CSIDL_DESKTOPDIRECTORY": 16, + }.get(csidl_name) + if csidl_const is None: + msg = f"Unknown CSIDL name: {csidl_name}" + raise ValueError(msg) + + buf = ctypes.create_unicode_buffer(1024) + windll = getattr(ctypes, "windll") # noqa: B009 # using getattr to avoid false positive with mypy type checker + windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if it has high-bit chars. + if any(ord(c) > 255 for c in buf): # noqa: PLR2004 + buf2 = ctypes.create_unicode_buffer(1024) + if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + if csidl_name == "CSIDL_DOWNLOADS": + return os.path.join(buf.value, "Downloads") # noqa: PTH118 + + return buf.value + + # print(get_win_folder_via_ctypes("CSIDL_DOWNLOADS")) + +print("done") diff --git a/extra_tests/snippets/stdlib_datetime.py b/extra_tests/snippets/stdlib_datetime.py index cd1f27733bd..57ffd3a4413 100644 --- a/extra_tests/snippets/stdlib_datetime.py +++ b/extra_tests/snippets/stdlib_datetime.py @@ -4,23 +4,21 @@ """ # import copy -import sys -import random - -from operator import lt, le, gt, ge, eq, ne, truediv, floordiv, mod - import datetime as datetime_module -from datetime import MINYEAR, MAXYEAR -from datetime import timedelta -from datetime import tzinfo -from datetime import time -from datetime import timezone -from datetime import date, datetime +import random +import sys import time as _time +from datetime import MAXYEAR, MINYEAR, date, datetime, time, timedelta, timezone, tzinfo +from operator import eq, floordiv, ge, gt, le, lt, mod, ne, truediv from testutils import ( - assert_raises, assert_equal, assert_true, assert_false, assert_isinstance, - assert_in) + assert_equal, + assert_false, + assert_in, + assert_isinstance, + assert_raises, + assert_true, +) # An arbitrary collection of objects of non-datetime types, for testing # mixed-type comparisons. @@ -40,7 +38,7 @@ assert_equal(datetime_module.MINYEAR, 1) assert_equal(datetime_module.MAXYEAR, 9999) -if hasattr(datetime_module, '_divide_and_round'): +if hasattr(datetime_module, "_divide_and_round"): # def test_divide_and_round(self): dar = datetime_module._divide_and_round @@ -68,8 +66,8 @@ ############################################################################# # tzinfo tests -class FixedOffset(tzinfo): +class FixedOffset(tzinfo): def __init__(self, offset, name, dstoffset=42): if isinstance(offset, int): offset = timedelta(minutes=offset) @@ -78,24 +76,30 @@ def __init__(self, offset, name, dstoffset=42): self.__offset = offset self.__name = name self.__dstoffset = dstoffset + def __repr__(self): return self.__name.lower() + def utcoffset(self, dt): return self.__offset + def tzname(self, dt): return self.__name + def dst(self, dt): return self.__dstoffset -class PicklableFixedOffset(FixedOffset): +class PicklableFixedOffset(FixedOffset): def __init__(self, offset=None, name=None, dstoffset=None): FixedOffset.__init__(self, offset, name, dstoffset) + class _TZInfo(tzinfo): def utcoffset(self, datetime_module): return random.random() + # class TestTZInfo(unittest.TestCase): # def test_refcnt_crash_bug_22044(self): @@ -117,11 +121,14 @@ def utcoffset(self, datetime_module): with assert_raises(NotImplementedError): useless.dst(dt) + # def test_subclass_must_override(self): class NotEnough(tzinfo): def __init__(self, offset, name): self.__offset = offset self.__name = name + + assert_true(issubclass(NotEnough, tzinfo)) ne = NotEnough(3, "NotByALongShot") assert_isinstance(ne, tzinfo) @@ -138,14 +145,14 @@ def __init__(self, offset, name): # XXX: bug #1302 # def test_normal(self): -#fo = FixedOffset(3, "Three") -#assert_isinstance(fo, tzinfo) -#for dt in datetime.now(), None: +# fo = FixedOffset(3, "Three") +# assert_isinstance(fo, tzinfo) +# for dt in datetime.now(), None: # assert_equal(fo.utcoffset(dt), timedelta(minutes=3)) # assert_equal(fo.tzname(dt), "Three") # assert_equal(fo.dst(dt), timedelta(minutes=42)) -''' +""" class TestTimeZone(unittest.TestCase): def setUp(self): @@ -277,17 +284,17 @@ def test_deepcopy(self): tz = timezone.utc tz_copy = copy.deepcopy(tz) self.assertIs(tz_copy, tz) -''' +""" ############################################################################# # Base class for testing a particular aspect of timedelta, time, date and # datetime comparisons. # class HarmlessMixedComparison: - # Test that __eq__ and __ne__ don't complain for mixed-type comparisons. +# Test that __eq__ and __ne__ don't complain for mixed-type comparisons. - # Subclasses must define 'theclass', and theclass(1, 1, 1) must be a - # legit constructor. +# Subclasses must define 'theclass', and theclass(1, 1, 1) must be a +# legit constructor. for theclass in timedelta, date, time: # def test_harmless_mixed_comparison(self): diff --git a/extra_tests/snippets/stdlib_dir_module.py b/extra_tests/snippets/stdlib_dir_module.py index 560fb02bf6a..db00055c7cc 100644 --- a/extra_tests/snippets/stdlib_dir_module.py +++ b/extra_tests/snippets/stdlib_dir_module.py @@ -1,13 +1,13 @@ +import dir_module from testutils import assert_equal -import dir_module assert dir_module.value == 5 assert dir_module.value2 == 7 try: dir_module.unknown_attr except AttributeError as e: - assert 'dir_module' in str(e) + assert "dir_module" in str(e) else: assert False @@ -15,7 +15,7 @@ try: dir_module.unknown_attr except AttributeError as e: - assert 'dir_module' not in str(e) + assert "dir_module" not in str(e) else: assert False @@ -23,9 +23,10 @@ try: dir_module.unknown_attr except AttributeError as e: - assert 'dir_module' not in str(e) + assert "dir_module" not in str(e) else: assert False from dir_module import dir_module_inner -assert dir_module_inner.__name__ == 'dir_module.dir_module_inner' + +assert dir_module_inner.__name__ == "dir_module.dir_module_inner" diff --git a/extra_tests/snippets/stdlib_dis.py b/extra_tests/snippets/stdlib_dis.py index e9951ef4025..42296168f88 100644 --- a/extra_tests/snippets/stdlib_dis.py +++ b/extra_tests/snippets/stdlib_dis.py @@ -2,37 +2,53 @@ dis.dis(compile("5 + x + 5 or 2", "", "eval")) print("\n") -dis.dis(compile(""" +dis.dis( + compile( + """ def f(x): return 1 -""", "", "exec")) +""", + "", + "exec", + ) +) print("\n") -dis.dis(compile(""" +dis.dis( + compile( + """ if a: 1 or 2 elif x == 'hello': 3 else: 4 -""", "", "exec")) +""", + "", + "exec", + ) +) print("\n") dis.dis(compile("f(x=1, y=2)", "", "eval")) print("\n") + def f(): with g(): # noqa: F821 try: for a in {1: 4, 2: 5}: yield [True and False or True, []] except Exception: - raise not ValueError({1 for i in [1,2,3]}) + raise not ValueError({1 for i in [1, 2, 3]}) + dis.dis(f) + class A(object): def f(): x += 1 # noqa: F821 pass + def g(): for i in range(5): if i: @@ -40,5 +56,6 @@ def g(): else: break + print("A.f\n") dis.dis(A.f) diff --git a/extra_tests/snippets/stdlib_functools.py b/extra_tests/snippets/stdlib_functools.py index 0bdafcb3b82..d3e2495e8a2 100644 --- a/extra_tests/snippets/stdlib_functools.py +++ b/extra_tests/snippets/stdlib_functools.py @@ -1,6 +1,8 @@ from functools import reduce + from testutils import assert_raises + class Squares: def __init__(self, max): self.max = max @@ -10,21 +12,24 @@ def __len__(self): return len(self.sofar) def __getitem__(self, i): - if not 0 <= i < self.max: raise IndexError + if not 0 <= i < self.max: + raise IndexError n = len(self.sofar) while n <= i: - self.sofar.append(n*n) + self.sofar.append(n * n) n += 1 return self.sofar[i] + def add(a, b): return a + b -assert reduce(add, ['a', 'b', 'c']) == 'abc' -assert reduce(add, ['a', 'b', 'c'], str(42)) == '42abc' -assert reduce(add, [['a', 'c'], [], ['d', 'w']], []) == ['a','c','d','w'] -assert reduce(add, [['a', 'c'], [], ['d', 'w']], []) == ['a','c','d','w'] -assert reduce(lambda x, y: x*y, range(2, 21), 1) == 2432902008176640000 + +assert reduce(add, ["a", "b", "c"]) == "abc" +assert reduce(add, ["a", "b", "c"], str(42)) == "42abc" +assert reduce(add, [["a", "c"], [], ["d", "w"]], []) == ["a", "c", "d", "w"] +assert reduce(add, [["a", "c"], [], ["d", "w"]], []) == ["a", "c", "d", "w"] +assert reduce(lambda x, y: x * y, range(2, 21), 1) == 2432902008176640000 assert reduce(add, Squares(10)) == 285 assert reduce(add, Squares(10), 0) == 285 assert reduce(add, Squares(0), 0) == 0 @@ -40,32 +45,40 @@ def add(a, b): with assert_raises(TypeError): reduce(42, 42, 42) + class TestFailingIter: def __iter__(self): raise RuntimeError + with assert_raises(RuntimeError): reduce(add, TestFailingIter()) assert reduce(add, [], None) == None assert reduce(add, [], 42) == 42 + class BadSeq: def __getitem__(self, index): raise ValueError + + with assert_raises(ValueError): reduce(42, BadSeq()) + # Test reduce()'s use of iterators. class SequenceClass: def __init__(self, n): self.n = n + def __getitem__(self, i): if 0 <= i < self.n: return i else: raise IndexError + assert reduce(add, SequenceClass(5)) == 10 assert reduce(add, SequenceClass(5), 42) == 52 with assert_raises(TypeError): diff --git a/extra_tests/snippets/stdlib_hashlib.py b/extra_tests/snippets/stdlib_hashlib.py index 811e3d27b66..c5feb709e17 100644 --- a/extra_tests/snippets/stdlib_hashlib.py +++ b/extra_tests/snippets/stdlib_hashlib.py @@ -1,39 +1,50 @@ - import hashlib # print(hashlib.md5) h = hashlib.md5() -h.update(b'a') -g = hashlib.md5(b'a') -assert h.name == g.name == 'md5' +h.update(b"a") +g = hashlib.md5(b"a") +assert h.name == g.name == "md5" print(h.hexdigest()) print(g.hexdigest()) -assert h.hexdigest() == g.hexdigest() == '0cc175b9c0f1b6a831c399e269772661' +assert h.hexdigest() == g.hexdigest() == "0cc175b9c0f1b6a831c399e269772661" assert h.digest_size == g.digest_size == 16 h = hashlib.sha256() -h.update(b'a') -g = hashlib.sha256(b'a') -assert h.name == g.name == 'sha256' +h.update(b"a") +g = hashlib.sha256(b"a") +assert h.name == g.name == "sha256" assert h.digest_size == g.digest_size == 32 print(h.hexdigest()) print(g.hexdigest()) -assert h.hexdigest() == g.hexdigest() == 'ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb' +assert ( + h.hexdigest() + == g.hexdigest() + == "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb" +) h = hashlib.sha512() -g = hashlib.sha512(b'a') -assert h.name == g.name == 'sha512' -h.update(b'a') +g = hashlib.sha512(b"a") +assert h.name == g.name == "sha512" +h.update(b"a") print(h.hexdigest()) print(g.hexdigest()) -assert h.hexdigest() == g.hexdigest() == '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75' +assert ( + h.hexdigest() + == g.hexdigest() + == "1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75" +) h = hashlib.new("blake2s", b"fubar") print(h.hexdigest()) -assert h.hexdigest() == 'a0e1ad0c123c9c65e8ef850db2ce4b5cef2c35b06527c615b0154353574d0415' -h.update(b'bla') +assert ( + h.hexdigest() == "a0e1ad0c123c9c65e8ef850db2ce4b5cef2c35b06527c615b0154353574d0415" +) +h.update(b"bla") print(h.hexdigest()) -assert h.hexdigest() == '25738bfe4cc104131e1b45bece4dfd4e7e1d6f0dffda1211e996e9d5d3b66e81' +assert ( + h.hexdigest() == "25738bfe4cc104131e1b45bece4dfd4e7e1d6f0dffda1211e996e9d5d3b66e81" +) diff --git a/extra_tests/snippets/stdlib_imghdr.py b/extra_tests/snippets/stdlib_imghdr.py deleted file mode 100644 index 5ca524e269b..00000000000 --- a/extra_tests/snippets/stdlib_imghdr.py +++ /dev/null @@ -1,27 +0,0 @@ -# unittest for modified imghdr.py -# Should be replace it into https://github.com/python/cpython/blob/main/Lib/test/test_imghdr.py -import os -import imghdr - - -TEST_FILES = ( - #('python.png', 'png'), - ('python.gif', 'gif'), - ('python.bmp', 'bmp'), - ('python.ppm', 'ppm'), - ('python.pgm', 'pgm'), - ('python.pbm', 'pbm'), - ('python.jpg', 'jpeg'), - ('python.ras', 'rast'), - #('python.sgi', 'rgb'), - ('python.tiff', 'tiff'), - ('python.xbm', 'xbm'), - ('python.webp', 'webp'), - ('python.exr', 'exr'), -) - -resource_dir = os.path.join(os.path.dirname(__file__), 'imghdrdata') - -for fname, expected in TEST_FILES: - res = imghdr.what(os.path.join(resource_dir, fname)) - assert res == expected \ No newline at end of file diff --git a/extra_tests/snippets/stdlib_imp.py b/extra_tests/snippets/stdlib_imp.py index bd28e95f3d5..835b50d6171 100644 --- a/extra_tests/snippets/stdlib_imp.py +++ b/extra_tests/snippets/stdlib_imp.py @@ -8,9 +8,11 @@ assert _imp.is_frozen("__hello__") == True assert _imp.is_frozen("math") == False + class FakeSpec: - def __init__(self, name): - self.name = name + def __init__(self, name): + self.name = name + A = FakeSpec("time") diff --git a/extra_tests/snippets/stdlib_io.py b/extra_tests/snippets/stdlib_io.py index 3dda7278c07..93c083a90c1 100644 --- a/extra_tests/snippets/stdlib_io.py +++ b/extra_tests/snippets/stdlib_io.py @@ -1,31 +1,32 @@ -from io import BufferedReader, FileIO, StringIO, BytesIO import os +from io import BufferedReader, BytesIO, FileIO, RawIOBase, StringIO, TextIOWrapper + from testutils import assert_raises -fi = FileIO('README.md') +fi = FileIO("README.md") assert fi.seekable() bb = BufferedReader(fi) assert bb.seekable() result = bb.read() -assert len(result) <= 8*1024 +assert len(result) <= 16 * 1024 assert len(result) >= 0 assert isinstance(result, bytes) -with FileIO('README.md') as fio: - res = fio.read() - assert len(result) <= 8*1024 - assert len(result) >= 0 - assert isinstance(result, bytes) +with FileIO("README.md") as fio: + res = fio.read() + assert len(res) <= 16 * 1024 + assert len(res) >= 0 + assert isinstance(res, bytes) -fd = os.open('README.md', os.O_RDONLY) +fd = os.open("README.md", os.O_RDONLY) with FileIO(fd) as fio: - res2 = fio.read() - assert res == res2 + res2 = fio.read() + assert res == res2 -fi = FileIO('README.md') +fi = FileIO("README.md") fi.read() fi.close() assert fi.closefd @@ -34,8 +35,52 @@ with assert_raises(ValueError): fi.read() -with FileIO('README.md') as fio: - nres = fio.read(1) - assert len(nres) == 1 - nres = fio.read(2) - assert len(nres) == 2 +with FileIO("README.md") as fio: + nres = fio.read(1) + assert len(nres) == 1 + nres = fio.read(2) + assert len(nres) == 2 + + +# Test that IOBase.isatty() raises ValueError when called on a closed file. +# Minimal subclass that inherits IOBase.isatty() without overriding it. +class MinimalRaw(RawIOBase): + def readinto(self, b): + return 0 + + +f = MinimalRaw() +assert not f.closed +assert not f.isatty() # open file: should return False + +f.close() +assert f.closed + +with assert_raises(ValueError): + f.isatty() + + +class Gh6588: + def __init__(self): + self.textio = None + self.closed = False + + def writable(self): + return True + + def readable(self): + return False + + def seekable(self): + return False + + def write(self, data): + self.textio.reconfigure(encoding="utf-8") + return len(data) + + +raw = Gh6588() +textio = TextIOWrapper(raw, encoding="utf-8", write_through=True) +raw.textio = textio +with assert_raises(AttributeError): + textio.writelines(["x"]) diff --git a/extra_tests/snippets/stdlib_io_bytesio.py b/extra_tests/snippets/stdlib_io_bytesio.py index 57144487640..ba8ae20015e 100644 --- a/extra_tests/snippets/stdlib_io_bytesio.py +++ b/extra_tests/snippets/stdlib_io_bytesio.py @@ -1,8 +1,8 @@ - from io import BytesIO + def test_01(): - bytes_string = b'Test String 1' + bytes_string = b"Test String 1" f = BytesIO() f.write(bytes_string) @@ -10,45 +10,49 @@ def test_01(): assert f.tell() == len(bytes_string) assert f.getvalue() == bytes_string + def test_02(): - bytes_string = b'Test String 2' + bytes_string = b"Test String 2" f = BytesIO(bytes_string) assert f.read() == bytes_string - assert f.read() == b'' + assert f.read() == b"" + def test_03(): """ - Tests that the read method (integer arg) - returns the expected value + Tests that the read method (integer arg) + returns the expected value """ - string = b'Test String 3' + string = b"Test String 3" f = BytesIO(string) - assert f.read(1) == b'T' - assert f.read(1) == b'e' - assert f.read(1) == b's' - assert f.read(1) == b't' + assert f.read(1) == b"T" + assert f.read(1) == b"e" + assert f.read(1) == b"s" + assert f.read(1) == b"t" + def test_04(): """ - Tests that the read method increments the - cursor position and the seek method moves - the cursor to the appropriate position + Tests that the read method increments the + cursor position and the seek method moves + the cursor to the appropriate position """ - string = b'Test String 4' + string = b"Test String 4" f = BytesIO(string) - assert f.read(4) == b'Test' + assert f.read(4) == b"Test" assert f.tell() == 4 assert f.seek(0) == 0 - assert f.read(4) == b'Test' + assert f.read(4) == b"Test" + def test_05(): """ - Tests that the write method accpets bytearray + Tests that the write method accepts bytearray """ - bytes_string = b'Test String 5' + bytes_string = b"Test String 5" f = BytesIO() f.write(bytearray(bytes_string)) @@ -58,16 +62,40 @@ def test_05(): def test_06(): """ - Tests readline + Tests readline """ - bytes_string = b'Test String 6\nnew line is here\nfinished' + bytes_string = b"Test String 6\nnew line is here\nfinished" f = BytesIO(bytes_string) - assert f.readline() == b'Test String 6\n' - assert f.readline() == b'new line is here\n' - assert f.readline() == b'finished' - assert f.readline() == b'' + assert f.readline() == b"Test String 6\n" + assert f.readline() == b"new line is here\n" + assert f.readline() == b"finished" + assert f.readline() == b"" + + +def test_07(): + """ + Tests that flush() returns None when the file is open, + and raises ValueError when the file is closed. + CPython reference: Modules/_io/bytesio.c:325-335 + """ + f = BytesIO(b"Test String 7") + + # flush() on an open BytesIO returns None + assert f.flush() is None + + # flush() is defined directly on BytesIO (not just inherited) + assert "flush" in BytesIO.__dict__ + + f.close() + + # flush() on a closed BytesIO raises ValueError + try: + f.flush() + assert False, "Expected ValueError not raised" + except ValueError: + pass if __name__ == "__main__": @@ -77,4 +105,4 @@ def test_06(): test_04() test_05() test_06() - + test_07() diff --git a/extra_tests/snippets/stdlib_io_stringio.py b/extra_tests/snippets/stdlib_io_stringio.py index 828f0506ed0..5419eef2bb2 100644 --- a/extra_tests/snippets/stdlib_io_stringio.py +++ b/extra_tests/snippets/stdlib_io_stringio.py @@ -1,68 +1,73 @@ - from io import StringIO + def test_01(): """ - Test that the constructor and getvalue - method return expected values + Test that the constructor and getvalue + method return expected values """ - string = 'Test String 1' + string = "Test String 1" f = StringIO() f.write(string) assert f.tell() == len(string) assert f.getvalue() == string + def test_02(): """ - Test that the read method (no arg) - results the expected value + Test that the read method (no arg) + results the expected value """ - string = 'Test String 2' + string = "Test String 2" f = StringIO(string) assert f.read() == string - assert f.read() == '' + assert f.read() == "" + def test_03(): """ - Tests that the read method (integer arg) - returns the expected value + Tests that the read method (integer arg) + returns the expected value """ - string = 'Test String 3' + string = "Test String 3" f = StringIO(string) - assert f.read(1) == 'T' - assert f.read(1) == 'e' - assert f.read(1) == 's' - assert f.read(1) == 't' + assert f.read(1) == "T" + assert f.read(1) == "e" + assert f.read(1) == "s" + assert f.read(1) == "t" + def test_04(): """ - Tests that the read method increments the - cursor position and the seek method moves - the cursor to the appropriate position + Tests that the read method increments the + cursor position and the seek method moves + the cursor to the appropriate position """ - string = 'Test String 4' + string = "Test String 4" f = StringIO(string) - assert f.read(4) == 'Test' + assert f.read(4) == "Test" assert f.tell() == 4 assert f.seek(0) == 0 - assert f.read(4) == 'Test' + assert f.read(4) == "Test" + def test_05(): """ - Tests readline + Tests readline """ - string = 'Test String 6\nnew line is here\nfinished' + string = "Test String 6\nnew line is here\nfinished" f = StringIO(string) - assert f.readline() == 'Test String 6\n' - assert f.readline() == 'new line is here\n' - assert f.readline() == 'finished' - assert f.readline() == '' + assert f.readline() == "Test String 6\n" + assert f.readline() == "new line is here\n" + assert f.readline() == "finished" + assert f.readline() == "" + if __name__ == "__main__": test_01() diff --git a/extra_tests/snippets/stdlib_itertools.py b/extra_tests/snippets/stdlib_itertools.py index 58684f611d8..ce7a494713a 100644 --- a/extra_tests/snippets/stdlib_itertools.py +++ b/extra_tests/snippets/stdlib_itertools.py @@ -2,8 +2,6 @@ from testutils import assert_raises -import pickle - # itertools.chain tests chain = itertools.chain @@ -12,13 +10,13 @@ assert list(chain([], "", b"", ())) == [] assert list(chain([1, 2, 3, 4])) == [1, 2, 3, 4] -assert list(chain("ab", "cd", (), 'e')) == ['a', 'b', 'c', 'd', 'e'] +assert list(chain("ab", "cd", (), "e")) == ["a", "b", "c", "d", "e"] with assert_raises(TypeError): list(chain(1)) x = chain("ab", 1) -assert next(x) == 'a' -assert next(x) == 'b' +assert next(x) == "a" +assert next(x) == "b" with assert_raises(TypeError): next(x) @@ -37,21 +35,21 @@ list(chain(1)) args = ["abc", "def"] -assert list(chain.from_iterable(args)) == ['a', 'b', 'c', 'd', 'e', 'f'] +assert list(chain.from_iterable(args)) == ["a", "b", "c", "d", "e", "f"] args = [[], "", b"", ()] assert list(chain.from_iterable(args)) == [] -args = ["ab", "cd", (), 'e'] -assert list(chain.from_iterable(args)) == ['a', 'b', 'c', 'd', 'e'] +args = ["ab", "cd", (), "e"] +assert list(chain.from_iterable(args)) == ["a", "b", "c", "d", "e"] x = chain.from_iterable(["ab", 1]) -assert next(x) == 'a' -assert next(x) == 'b' +assert next(x) == "a" +assert next(x) == "b" with assert_raises(TypeError): next(x) -# iterables are lazily evaluted +# iterables are lazily evaluated x = chain.from_iterable(itertools.repeat(range(2))) assert next(x) == 0 assert next(x) == 1 @@ -174,17 +172,14 @@ # itertools.starmap tests starmap = itertools.starmap -assert list(starmap(pow, zip(range(3), range(1,7)))) == [0**1, 1**2, 2**3] +assert list(starmap(pow, zip(range(3), range(1, 7)))) == [0**1, 1**2, 2**3] assert list(starmap(pow, [])) == [] -assert list(starmap(pow, [iter([4,5])])) == [4**5] +assert list(starmap(pow, [iter([4, 5])])) == [4**5] with assert_raises(TypeError): starmap(pow) # itertools.takewhile tests -def underten(x): - return x<10 - from itertools import takewhile as tw t = tw(lambda n: n < 5, [1, 2, 5, 1, 3]) @@ -226,76 +221,38 @@ def underten(x): with assert_raises(StopIteration): next(t) -it = tw(underten, [1, 3, 5, 20, 2, 4, 6, 8]) -assert pickle.dumps(it, 0) == b'citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb.' -assert pickle.dumps(it, 1) == b'citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 2) == b'\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 3) == b'\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 4) == b'\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b.' -assert pickle.dumps(it, 5) == b'\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b.' -next(it) -next(it) -next(it) -try: - next(it) -except StopIteration: - pass -assert pickle.dumps(it, 0) == b'citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb.' -assert pickle.dumps(it, 1) == b'citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 2) == b'\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 3) == b'\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 4) == b'\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b.' -assert pickle.dumps(it, 5) == b'\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b.' -for proto in range(pickle.HIGHEST_PROTOCOL + 1): - try: - next(pickle.loads(pickle.dumps(it, proto))) - assert False - except StopIteration: - pass - - - # itertools.islice tests + def assert_matches_seq(it, seq): assert list(it) == list(seq) -def test_islice_pickle(it): - for p in range(pickle.HIGHEST_PROTOCOL + 1): - it == pickle.loads(pickle.dumps(it, p)) i = itertools.islice it = i([1, 2, 3, 4, 5], 3) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([0.5, 1, 1.5, 2, 2.5, 3, 4, 5], 1, 6, 2) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([1, 2], None) assert_matches_seq(it, [1, 2]) -test_islice_pickle(it) it = i([1, 2, 3], None, None, None) assert_matches_seq(it, [1, 2, 3]) -test_islice_pickle(it) it = i([1, 2, 3], 1, None, None) assert_matches_seq(it, [2, 3]) -test_islice_pickle(it) it = i([1, 2, 3], None, 2, None) assert_matches_seq(it, [1, 2]) -test_islice_pickle(it) it = i([1, 2, 3], None, None, 3) assert_matches_seq(it, [1]) -test_islice_pickle(it) # itertools.filterfalse -it = itertools.filterfalse(lambda x: x%2, range(10)) +it = itertools.filterfalse(lambda x: x % 2, range(10)) assert 0 == next(it) assert 2 == next(it) assert 4 == next(it) @@ -314,30 +271,13 @@ def test_islice_pickle(it): # itertools.dropwhile -it = itertools.dropwhile(lambda x: x<5, [1,4,6,4,1]) +it = itertools.dropwhile(lambda x: x < 5, [1, 4, 6, 4, 1]) assert 6 == next(it) assert 4 == next(it) assert 1 == next(it) with assert_raises(StopIteration): next(it) -it = itertools.dropwhile(underten, [1, 3, 5, 20, 2, 4, 6, 8]) -assert pickle.dumps(it, 0) == b'citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb.' -assert pickle.dumps(it, 1) == b'citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 2) == b'\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 3) == b'\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 4) == b'\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b.' -assert pickle.dumps(it, 5) == b'\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b.' -next(it) -assert pickle.dumps(it, 0) == b'citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb.' -assert pickle.dumps(it, 1) == b'citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 2) == b'\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 3) == b'\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 4) == b'\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b.' -assert pickle.dumps(it, 5) == b'\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b.' -for proto in range(pickle.HIGHEST_PROTOCOL + 1): - assert next(pickle.loads(pickle.dumps(it, proto))) == 2 - # itertools.accumulate it = itertools.accumulate([6, 3, 7, 1, 0, 9, 8, 8]) @@ -352,7 +292,7 @@ def test_islice_pickle(it): with assert_raises(StopIteration): next(it) -it = itertools.accumulate([3, 2, 4, 1, 0, 5, 8], lambda a, v: a*v) +it = itertools.accumulate([3, 2, 4, 1, 0, 5, 8], lambda a, v: a * v) assert 3 == next(it) assert 6 == next(it) assert 24 == next(it) @@ -364,12 +304,12 @@ def test_islice_pickle(it): next(it) # itertools.compress -assert list(itertools.compress("ABCDEF", [1,0,1,0,1,1])) == list("ACEF") -assert list(itertools.compress("ABCDEF", [0,0,0,0,0,0])) == list("") -assert list(itertools.compress("ABCDEF", [1,1,1,1,1,1])) == list("ABCDEF") -assert list(itertools.compress("ABCDEF", [1,0,1])) == list("AC") -assert list(itertools.compress("ABC", [0,1,1,1,1,1])) == list("BC") -assert list(itertools.compress("ABCDEF", [True,False,"t","",1,9])) == list("ACEF") +assert list(itertools.compress("ABCDEF", [1, 0, 1, 0, 1, 1])) == list("ACEF") +assert list(itertools.compress("ABCDEF", [0, 0, 0, 0, 0, 0])) == list("") +assert list(itertools.compress("ABCDEF", [1, 1, 1, 1, 1, 1])) == list("ABCDEF") +assert list(itertools.compress("ABCDEF", [1, 0, 1])) == list("AC") +assert list(itertools.compress("ABC", [0, 1, 1, 1, 1, 1])) == list("BC") +assert list(itertools.compress("ABCDEF", [True, False, "t", "", 1, 9])) == list("ACEF") # itertools.tee @@ -384,7 +324,7 @@ def test_islice_pickle(it): t = itertools.tee(range(1000)) assert list(t[0]) == list(t[1]) == list(range(1000)) -t = itertools.tee([1,22,333], 3) +t = itertools.tee([1, 22, 333], 3) assert len(t) == 3 assert 1 == next(t[0]) assert 1 == next(t[1]) @@ -402,29 +342,29 @@ def test_islice_pickle(it): with assert_raises(StopIteration): next(t[1]) -t0, t1 = itertools.tee([1,2,3]) +t0, t1 = itertools.tee([1, 2, 3]) tc = t0.__copy__() -assert list(t0) == [1,2,3] -assert list(t1) == [1,2,3] -assert list(tc) == [1,2,3] +assert list(t0) == [1, 2, 3] +assert list(t1) == [1, 2, 3] +assert list(tc) == [1, 2, 3] -t0, t1 = itertools.tee([1,2,3]) +t0, t1 = itertools.tee([1, 2, 3]) assert 1 == next(t0) # advance index of t0 by 1 before __copy__() t0c = t0.__copy__() t1c = t1.__copy__() -assert list(t0) == [2,3] -assert list(t0c) == [2,3] -assert list(t1) == [1,2,3] -assert list(t1c) == [1,2,3] +assert list(t0) == [2, 3] +assert list(t0c) == [2, 3] +assert list(t1) == [1, 2, 3] +assert list(t1c) == [1, 2, 3] -t0, t1 = itertools.tee([1,2,3]) +t0, t1 = itertools.tee([1, 2, 3]) t2, t3 = itertools.tee(t0) -assert list(t1) == [1,2,3] -assert list(t2) == [1,2,3] -assert list(t3) == [1,2,3] +assert list(t1) == [1, 2, 3] +assert list(t2) == [1, 2, 3] +assert list(t3) == [1, 2, 3] -t = itertools.tee([1,2,3]) -assert list(t[0]) == [1,2,3] +t = itertools.tee([1, 2, 3]) +assert list(t[0]) == [1, 2, 3] assert list(t[0]) == [] # itertools.product @@ -530,22 +470,36 @@ def test_islice_pickle(it): # itertools.zip_longest tests zl = itertools.zip_longest -assert list(zl(['a', 'b', 'c'], range(3), [9, 8, 7])) \ - == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7)] -assert list(zl(['a', 'b', 'c'], range(3), [9, 8, 7, 99])) \ - == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7), (None, None, 99)] -assert list(zl(['a', 'b', 'c'], range(3), [9, 8, 7, 99], fillvalue='d')) \ - == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7), ('d', 'd', 99)] - -assert list(zl(['a', 'b', 'c'])) == [('a',), ('b',), ('c',)] +assert list(zl(["a", "b", "c"], range(3), [9, 8, 7])) == [ + ("a", 0, 9), + ("b", 1, 8), + ("c", 2, 7), +] +assert list(zl(["a", "b", "c"], range(3), [9, 8, 7, 99])) == [ + ("a", 0, 9), + ("b", 1, 8), + ("c", 2, 7), + (None, None, 99), +] +assert list(zl(["a", "b", "c"], range(3), [9, 8, 7, 99], fillvalue="d")) == [ + ("a", 0, 9), + ("b", 1, 8), + ("c", 2, 7), + ("d", "d", 99), +] + +assert list(zl(["a", "b", "c"])) == [("a",), ("b",), ("c",)] assert list(zl()) == [] -assert list(zl(*zl(['a', 'b', 'c'], range(1, 4)))) \ - == [('a', 'b', 'c'), (1, 2, 3)] -assert list(zl(*zl(['a', 'b', 'c'], range(1, 5)))) \ - == [('a', 'b', 'c', None), (1, 2, 3, 4)] -assert list(zl(*zl(['a', 'b', 'c'], range(1, 5), fillvalue=100))) \ - == [('a', 'b', 'c', 100), (1, 2, 3, 4)] +assert list(zl(*zl(["a", "b", "c"], range(1, 4)))) == [("a", "b", "c"), (1, 2, 3)] +assert list(zl(*zl(["a", "b", "c"], range(1, 5)))) == [ + ("a", "b", "c", None), + (1, 2, 3, 4), +] +assert list(zl(*zl(["a", "b", "c"], range(1, 5), fillvalue=100))) == [ + ("a", "b", "c", 100), + (1, 2, 3, 4), +] # test infinite iterator @@ -565,7 +519,7 @@ def __iter__(self): assert next(it) == (1, 4) assert next(it) == (2, 5) -it = zl([1,2], [3]) +it = zl([1, 2], [3]) assert next(it) == (1, 3) assert next(it) == (2, None) with assert_raises(StopIteration): diff --git a/extra_tests/snippets/stdlib_json.py b/extra_tests/snippets/stdlib_json.py index a91f3bd817d..758cd129cd0 100644 --- a/extra_tests/snippets/stdlib_json.py +++ b/extra_tests/snippets/stdlib_json.py @@ -1,6 +1,8 @@ -from testutils import assert_raises import json -from io import StringIO, BytesIO +from io import BytesIO, StringIO + +from testutils import assert_raises + def round_trip_test(obj): # serde_json and Python's json module produce slightly differently spaced @@ -8,16 +10,19 @@ def round_trip_test(obj): # proxy return obj == json.loads(json.dumps(obj)) + def json_dump(obj): f = StringIO() json.dump(obj, f) f.seek(0) return f.getvalue() + def json_load(obj): f = StringIO(obj) if isinstance(obj, str) else BytesIO(bytes(obj)) return json.load(f) + assert '"string"' == json.dumps("string") assert '"string"' == json_dump("string") @@ -33,41 +38,41 @@ def json_load(obj): assert "false" == json.dumps(False) assert "false" == json_dump(False) -assert 'null' == json.dumps(None) -assert 'null' == json_dump(None) +assert "null" == json.dumps(None) +assert "null" == json_dump(None) -assert '[]' == json.dumps([]) -assert '[]' == json_dump([]) +assert "[]" == json.dumps([]) +assert "[]" == json_dump([]) -assert '[1]' == json.dumps([1]) -assert '[1]' == json_dump([1]) +assert "[1]" == json.dumps([1]) +assert "[1]" == json_dump([1]) -assert '[[1]]' == json.dumps([[1]]) -assert '[[1]]' == json_dump([[1]]) +assert "[[1]]" == json.dumps([[1]]) +assert "[[1]]" == json_dump([[1]]) assert round_trip_test([1, "string", 1.0, True]) -assert '[]' == json.dumps(()) -assert '[]' == json_dump(()) +assert "[]" == json.dumps(()) +assert "[]" == json_dump(()) -assert '[1]' == json.dumps((1,)) -assert '[1]' == json_dump((1,)) +assert "[1]" == json.dumps((1,)) +assert "[1]" == json_dump((1,)) -assert '[[1]]' == json.dumps(((1,),)) -assert '[[1]]' == json_dump(((1,),)) +assert "[[1]]" == json.dumps(((1,),)) +assert "[[1]]" == json_dump(((1,),)) # tuples don't round-trip through json assert [1, "string", 1.0, True] == json.loads(json.dumps((1, "string", 1.0, True))) -assert '{}' == json.dumps({}) -assert '{}' == json_dump({}) -assert round_trip_test({'a': 'b'}) +assert "{}" == json.dumps({}) +assert "{}" == json_dump({}) +assert round_trip_test({"a": "b"}) # should reject non-str keys in jsons assert_raises(json.JSONDecodeError, lambda: json.loads('{3: "abc"}')) assert_raises(json.JSONDecodeError, lambda: json_load('{3: "abc"}')) # should serialize non-str keys as strings -assert json.dumps({'3': 'abc'}) == json.dumps({3: 'abc'}) +assert json.dumps({"3": "abc"}) == json.dumps({3: "abc"}) assert 1 == json.loads("1") assert 1 == json.loads(b"1") @@ -104,51 +109,60 @@ def json_load(obj): assert "str" == json_load(b'"str"') assert "str" == json_load(bytearray(b'"str"')) -assert True is json.loads('true') -assert True is json.loads(b'true') -assert True is json.loads(bytearray(b'true')) -assert True is json_load('true') -assert True is json_load(b'true') -assert True is json_load(bytearray(b'true')) - -assert False is json.loads('false') -assert False is json.loads(b'false') -assert False is json.loads(bytearray(b'false')) -assert False is json_load('false') -assert False is json_load(b'false') -assert False is json_load(bytearray(b'false')) - -assert None is json.loads('null') -assert None is json.loads(b'null') -assert None is json.loads(bytearray(b'null')) -assert None is json_load('null') -assert None is json_load(b'null') -assert None is json_load(bytearray(b'null')) - -assert [] == json.loads('[]') -assert [] == json.loads(b'[]') -assert [] == json.loads(bytearray(b'[]')) -assert [] == json_load('[]') -assert [] == json_load(b'[]') -assert [] == json_load(bytearray(b'[]')) - -assert ['a'] == json.loads('["a"]') -assert ['a'] == json.loads(b'["a"]') -assert ['a'] == json.loads(bytearray(b'["a"]')) -assert ['a'] == json_load('["a"]') -assert ['a'] == json_load(b'["a"]') -assert ['a'] == json_load(bytearray(b'["a"]')) - -assert [['a'], 'b'] == json.loads('[["a"], "b"]') -assert [['a'], 'b'] == json.loads(b'[["a"], "b"]') -assert [['a'], 'b'] == json.loads(bytearray(b'[["a"], "b"]')) -assert [['a'], 'b'] == json_load('[["a"], "b"]') -assert [['a'], 'b'] == json_load(b'[["a"], "b"]') -assert [['a'], 'b'] == json_load(bytearray(b'[["a"], "b"]')) - -class String(str): pass -class Bytes(bytes): pass -class ByteArray(bytearray): pass +assert True is json.loads("true") +assert True is json.loads(b"true") +assert True is json.loads(bytearray(b"true")) +assert True is json_load("true") +assert True is json_load(b"true") +assert True is json_load(bytearray(b"true")) + +assert False is json.loads("false") +assert False is json.loads(b"false") +assert False is json.loads(bytearray(b"false")) +assert False is json_load("false") +assert False is json_load(b"false") +assert False is json_load(bytearray(b"false")) + +assert None is json.loads("null") +assert None is json.loads(b"null") +assert None is json.loads(bytearray(b"null")) +assert None is json_load("null") +assert None is json_load(b"null") +assert None is json_load(bytearray(b"null")) + +assert [] == json.loads("[]") +assert [] == json.loads(b"[]") +assert [] == json.loads(bytearray(b"[]")) +assert [] == json_load("[]") +assert [] == json_load(b"[]") +assert [] == json_load(bytearray(b"[]")) + +assert ["a"] == json.loads('["a"]') +assert ["a"] == json.loads(b'["a"]') +assert ["a"] == json.loads(bytearray(b'["a"]')) +assert ["a"] == json_load('["a"]') +assert ["a"] == json_load(b'["a"]') +assert ["a"] == json_load(bytearray(b'["a"]')) + +assert [["a"], "b"] == json.loads('[["a"], "b"]') +assert [["a"], "b"] == json.loads(b'[["a"], "b"]') +assert [["a"], "b"] == json.loads(bytearray(b'[["a"], "b"]')) +assert [["a"], "b"] == json_load('[["a"], "b"]') +assert [["a"], "b"] == json_load(b'[["a"], "b"]') +assert [["a"], "b"] == json_load(bytearray(b'[["a"], "b"]')) + + +class String(str): + pass + + +class Bytes(bytes): + pass + + +class ByteArray(bytearray): + pass + assert "string" == json.loads(String('"string"')) assert "string" == json.loads(Bytes(b'"string"')) @@ -160,29 +174,46 @@ class ByteArray(bytearray): pass assert '"string"' == json.dumps(String("string")) assert '"string"' == json_dump(String("string")) -class Int(int): pass -class Float(float): pass -assert '1' == json.dumps(Int(1)) -assert '1' == json_dump(Int(1)) +class Int(int): + pass + + +class Float(float): + pass + + +assert "1" == json.dumps(Int(1)) +assert "1" == json_dump(Int(1)) + +assert "0.5" == json.dumps(Float(0.5)) +assert "0.5" == json_dump(Float(0.5)) + + +class List(list): + pass + + +class Tuple(tuple): + pass + -assert '0.5' == json.dumps(Float(0.5)) -assert '0.5' == json_dump(Float(0.5)) +class Dict(dict): + pass -class List(list): pass -class Tuple(tuple): pass -class Dict(dict): pass -assert '[1]' == json.dumps(List([1])) -assert '[1]' == json_dump(List([1])) +assert "[1]" == json.dumps(List([1])) +assert "[1]" == json_dump(List([1])) -assert json.dumps((1, "string", 1.0, True)) == json.dumps(Tuple((1, "string", 1.0, True))) +assert json.dumps((1, "string", 1.0, True)) == json.dumps( + Tuple((1, "string", 1.0, True)) +) assert json_dump((1, "string", 1.0, True)) == json_dump(Tuple((1, "string", 1.0, True))) -assert json.dumps({'a': 'b'}) == json.dumps(Dict({'a': 'b'})) -assert json_dump({'a': 'b'}) == json_dump(Dict({'a': 'b'})) +assert json.dumps({"a": "b"}) == json.dumps(Dict({"a": "b"})) +assert json_dump({"a": "b"}) == json_dump(Dict({"a": "b"})) i = 7**500 assert json.dumps(i) == str(i) -assert json.decoder.scanstring('✨x"', 1) == ('x', 3) +assert json.decoder.scanstring('✨x"', 1) == ("x", 3) diff --git a/extra_tests/snippets/stdlib_logging.py b/extra_tests/snippets/stdlib_logging.py index 0356404624a..18c26f0fb6f 100644 --- a/extra_tests/snippets/stdlib_logging.py +++ b/extra_tests/snippets/stdlib_logging.py @@ -1,4 +1,3 @@ - import io import sys @@ -7,12 +6,11 @@ import logging -logging.error('WOOT') -logging.warning('WARN') +logging.error("WOOT") +logging.warning("WARN") res = f.getvalue() -assert 'WOOT' in res -assert 'WARN' in res +assert "WOOT" in res +assert "WARN" in res print(res) - diff --git a/extra_tests/snippets/stdlib_marshal.py b/extra_tests/snippets/stdlib_marshal.py index 8ad11c3cc65..db843ff65d5 100644 --- a/extra_tests/snippets/stdlib_marshal.py +++ b/extra_tests/snippets/stdlib_marshal.py @@ -1,11 +1,12 @@ -import unittest import marshal +import unittest + class MarshalTests(unittest.TestCase): """ Testing the (incomplete) marshal module. """ - + def dump_then_load(self, data): return marshal.loads(marshal.dumps(data)) @@ -34,7 +35,7 @@ def test_marshal_str(self): def test_marshal_list(self): self._test_marshal([]) self._test_marshal([1, "hello", 1.0]) - self._test_marshal([[0], ['a','b']]) + self._test_marshal([[0], ["a", "b"]]) def test_marshal_tuple(self): self._test_marshal(()) @@ -42,31 +43,31 @@ def test_marshal_tuple(self): def test_marshal_dict(self): self._test_marshal({}) - self._test_marshal({'a':1, 1:'a'}) - self._test_marshal({'a':{'b':2}, 'c':[0.0, 4.0, 6, 9]}) - + self._test_marshal({"a": 1, 1: "a"}) + self._test_marshal({"a": {"b": 2}, "c": [0.0, 4.0, 6, 9]}) + def test_marshal_set(self): self._test_marshal(set()) self._test_marshal({1, 2, 3}) - self._test_marshal({1, 'a', 'b'}) + self._test_marshal({1, "a", "b"}) def test_marshal_frozen_set(self): self._test_marshal(frozenset()) self._test_marshal(frozenset({1, 2, 3})) - self._test_marshal(frozenset({1, 'a', 'b'})) + self._test_marshal(frozenset({1, "a", "b"})) def test_marshal_bytearray(self): self.assertEqual( self.dump_then_load(bytearray([])), - bytearray(b''), + bytearray(b""), ) self.assertEqual( self.dump_then_load(bytearray([1, 2])), - bytearray(b'\x01\x02'), + bytearray(b"\x01\x02"), ) def test_roundtrip(self): - orig = compile("1 + 1", "", 'eval') + orig = compile("1 + 1", "", "eval") dumped = marshal.dumps(orig) loaded = marshal.loads(dumped) diff --git a/extra_tests/snippets/stdlib_math.py b/extra_tests/snippets/stdlib_math.py index 442bbc97a5d..a6bb0099c05 100644 --- a/extra_tests/snippets/stdlib_math.py +++ b/extra_tests/snippets/stdlib_math.py @@ -1,9 +1,10 @@ import math + from testutils import assert_raises, skip_if_unsupported -NAN = float('nan') -INF = float('inf') -NINF = float('-inf') +NAN = float("nan") +INF = float("inf") +NINF = float("-inf") # assert(math.exp(2) == math.exp(2.0)) # assert(math.exp(True) == math.exp(1.0)) @@ -46,6 +47,7 @@ def float_ceil_exists(): assert isinstance(math.ceil(3.3), int) assert isinstance(math.floor(4.4), int) + class A(object): def __trunc__(self): return 2 @@ -56,10 +58,12 @@ def __ceil__(self): def __floor__(self): return 4 + assert math.trunc(A()) == 2 assert math.ceil(A()) == 3 assert math.floor(A()) == 4 + class A(object): def __trunc__(self): return 2.2 @@ -70,23 +74,26 @@ def __ceil__(self): def __floor__(self): return 4.4 + assert math.trunc(A()) == 2.2 assert math.ceil(A()) == 3.3 assert math.floor(A()) == 4.4 + class A(object): def __trunc__(self): - return 'trunc' + return "trunc" def __ceil__(self): - return 'ceil' + return "ceil" def __floor__(self): - return 'floor' + return "floor" + -assert math.trunc(A()) == 'trunc' -assert math.ceil(A()) == 'ceil' -assert math.floor(A()) == 'floor' +assert math.trunc(A()) == "trunc" +assert math.ceil(A()) == "ceil" +assert math.floor(A()) == "floor" with assert_raises(TypeError): math.trunc(object()) @@ -97,44 +104,54 @@ def __floor__(self): isclose = math.isclose + def assertIsClose(a, b, *args, **kwargs): assert isclose(a, b, *args, **kwargs) == True, "%s and %s should be close!" % (a, b) + def assertIsNotClose(a, b, *args, **kwargs): - assert isclose(a, b, *args, **kwargs) == False, "%s and %s should not be close!" % (a, b) + assert isclose(a, b, *args, **kwargs) == False, "%s and %s should not be close!" % ( + a, + b, + ) + def assertAllClose(examples, *args, **kwargs): for a, b in examples: assertIsClose(a, b, *args, **kwargs) + def assertAllNotClose(examples, *args, **kwargs): for a, b in examples: assertIsNotClose(a, b, *args, **kwargs) + # test_negative_tolerances: ValueError should be raised if either tolerance is less than zero assert_raises(ValueError, lambda: isclose(1, 1, rel_tol=-1e-100)) assert_raises(ValueError, lambda: isclose(1, 1, rel_tol=1e-100, abs_tol=-1e10)) # test_identical: identical values must test as close -identical_examples = [(2.0, 2.0), - (0.1e200, 0.1e200), - (1.123e-300, 1.123e-300), - (12345, 12345.0), - (0.0, -0.0), - (345678, 345678)] +identical_examples = [ + (2.0, 2.0), + (0.1e200, 0.1e200), + (1.123e-300, 1.123e-300), + (12345, 12345.0), + (0.0, -0.0), + (345678, 345678), +] assertAllClose(identical_examples, rel_tol=0.0, abs_tol=0.0) # test_eight_decimal_places: examples that are close to 1e-8, but not 1e-9 -eight_decimal_places_examples = [(1e8, 1e8 + 1), - (-1e-8, -1.000000009e-8), - (1.12345678, 1.12345679)] +eight_decimal_places_examples = [ + (1e8, 1e8 + 1), + (-1e-8, -1.000000009e-8), + (1.12345678, 1.12345679), +] assertAllClose(eight_decimal_places_examples, rel_tol=1e-08) assertAllNotClose(eight_decimal_places_examples, rel_tol=1e-09) # test_near_zero: values close to zero -near_zero_examples = [(1e-9, 0.0), - (-1e-9, 0.0), - (-1e-150, 0.0)] +near_zero_examples = [(1e-9, 0.0), (-1e-9, 0.0), (-1e-150, 0.0)] # these should not be close to any rel_tol assertAllNotClose(near_zero_examples, rel_tol=0.9) # these should be close to abs_tol=1e-8 @@ -147,35 +164,36 @@ def assertAllNotClose(examples, *args, **kwargs): assertIsClose(NINF, NINF, abs_tol=0.0) # test_inf_ninf_nan(self): these should never be close (following IEEE 754 rules for equality) -not_close_examples = [(NAN, NAN), - (NAN, 1e-100), - (1e-100, NAN), - (INF, NAN), - (NAN, INF), - (INF, NINF), - (INF, 1.0), - (1.0, INF), - (INF, 1e308), - (1e308, INF)] +not_close_examples = [ + (NAN, NAN), + (NAN, 1e-100), + (1e-100, NAN), + (INF, NAN), + (NAN, INF), + (INF, NINF), + (INF, 1.0), + (1.0, INF), + (INF, 1e308), + (1e308, INF), +] # use largest reasonable tolerance assertAllNotClose(not_close_examples, abs_tol=0.999999999999999) # test_zero_tolerance: test with zero tolerance -zero_tolerance_close_examples = [(1.0, 1.0), - (-3.4, -3.4), - (-1e-300, -1e-300)] +zero_tolerance_close_examples = [(1.0, 1.0), (-3.4, -3.4), (-1e-300, -1e-300)] assertAllClose(zero_tolerance_close_examples, rel_tol=0.0) -zero_tolerance_not_close_examples = [(1.0, 1.000000000000001), - (0.99999999999999, 1.0), - (1.0e200, .999999999999999e200)] +zero_tolerance_not_close_examples = [ + (1.0, 1.000000000000001), + (0.99999999999999, 1.0), + (1.0e200, 0.999999999999999e200), +] assertAllNotClose(zero_tolerance_not_close_examples, rel_tol=0.0) # test_asymmetry: test the asymmetry example from PEP 485 assertAllClose([(9, 10), (10, 9)], rel_tol=0.1) # test_integers: test with integer values -integer_examples = [(100000001, 100000000), - (123456789, 123456788)] +integer_examples = [(100000001, 100000000), (123456789, 123456788)] assertAllClose(integer_examples, rel_tol=1e-8) assertAllNotClose(integer_examples, rel_tol=1e-9) @@ -184,26 +202,26 @@ def assertAllNotClose(examples, *args, **kwargs): # test_fractions: test with Fraction values assert math.copysign(1, 42) == 1.0 -assert math.copysign(0., 42) == 0.0 -assert math.copysign(1., -42) == -1.0 -assert math.copysign(3, 0.) == 3.0 -assert math.copysign(4., -0.) == -4.0 +assert math.copysign(0.0, 42) == 0.0 +assert math.copysign(1.0, -42) == -1.0 +assert math.copysign(3, 0.0) == 3.0 +assert math.copysign(4.0, -0.0) == -4.0 assert_raises(TypeError, math.copysign) # copysign should let us distinguish signs of zeros -assert math.copysign(1., 0.) == 1. -assert math.copysign(1., -0.) == -1. -assert math.copysign(INF, 0.) == INF -assert math.copysign(INF, -0.) == NINF -assert math.copysign(NINF, 0.) == INF -assert math.copysign(NINF, -0.) == NINF +assert math.copysign(1.0, 0.0) == 1.0 +assert math.copysign(1.0, -0.0) == -1.0 +assert math.copysign(INF, 0.0) == INF +assert math.copysign(INF, -0.0) == NINF +assert math.copysign(NINF, 0.0) == INF +assert math.copysign(NINF, -0.0) == NINF # and of infinities -assert math.copysign(1., INF) == 1. -assert math.copysign(1., NINF) == -1. +assert math.copysign(1.0, INF) == 1.0 +assert math.copysign(1.0, NINF) == -1.0 assert math.copysign(INF, INF) == INF assert math.copysign(INF, NINF) == NINF assert math.copysign(NINF, INF) == INF assert math.copysign(NINF, NINF) == NINF -assert math.isnan(math.copysign(NAN, 1.)) +assert math.isnan(math.copysign(NAN, 1.0)) assert math.isnan(math.copysign(NAN, INF)) assert math.isnan(math.copysign(NAN, NINF)) assert math.isnan(math.copysign(NAN, NAN)) @@ -212,7 +230,7 @@ def assertAllNotClose(examples, *args, **kwargs): # given platform. assert math.isinf(math.copysign(INF, NAN)) # similarly, copysign(2., NAN) could be 2. or -2. -assert abs(math.copysign(2., NAN)) == 2. +assert abs(math.copysign(2.0, NAN)) == 2.0 assert str(math.frexp(0.0)) == str((+0.0, 0)) assert str(math.frexp(-0.0)) == str((-0.0, 0)) @@ -248,7 +266,7 @@ def assertAllNotClose(examples, *args, **kwargs): assert math.factorial(20) == 2432902008176640000 assert_raises(ValueError, lambda: math.factorial(-1)) -if hasattr(math, 'nextafter'): +if hasattr(math, "nextafter"): try: assert math.nextafter(4503599627370496.0, -INF) == 4503599627370495.5 assert math.nextafter(4503599627370496.0, INF) == 4503599627370497.0 @@ -278,16 +296,18 @@ def assertAllNotClose(examples, *args, **kwargs): assert math.fmod(-10, 1) == -0.0 assert math.fmod(-10, 0.5) == -0.0 assert math.fmod(-10, 1.5) == -1.0 -assert math.isnan(math.fmod(NAN, 1.)) == True -assert math.isnan(math.fmod(1., NAN)) == True +assert math.isnan(math.fmod(NAN, 1.0)) == True +assert math.isnan(math.fmod(1.0, NAN)) == True assert math.isnan(math.fmod(NAN, NAN)) == True -assert_raises(ValueError, lambda: math.fmod(1., 0.)) -assert_raises(ValueError, lambda: math.fmod(INF, 1.)) -assert_raises(ValueError, lambda: math.fmod(NINF, 1.)) -assert_raises(ValueError, lambda: math.fmod(INF, 0.)) +assert_raises(ValueError, lambda: math.fmod(1.0, 0.0)) +assert_raises(ValueError, lambda: math.fmod(INF, 1.0)) +assert_raises(ValueError, lambda: math.fmod(NINF, 1.0)) +assert_raises(ValueError, lambda: math.fmod(INF, 0.0)) assert math.fmod(3.0, INF) == 3.0 assert math.fmod(-3.0, INF) == -3.0 assert math.fmod(3.0, NINF) == 3.0 assert math.fmod(-3.0, NINF) == -3.0 assert math.fmod(0.0, 3.0) == 0.0 assert math.fmod(0.0, NINF) == 0.0 + +assert math.gamma(1) == 1.0 diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index 849fc238a84..d00924e10f2 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -1,29 +1,28 @@ import os -import time import stat import sys +import time from testutils import assert_raises -assert os.name == 'posix' or os.name == 'nt' +assert os.name == "posix" or os.name == "nt" -fd = os.open('README.md', os.O_RDONLY) +fd = os.open("README.md", os.O_RDONLY) assert fd > 0 os.close(fd) assert_raises(OSError, lambda: os.read(fd, 10)) -assert_raises(FileNotFoundError, - lambda: os.open('DOES_NOT_EXIST', os.O_RDONLY)) -assert_raises(FileNotFoundError, - lambda: os.open('DOES_NOT_EXIST', os.O_WRONLY)) -assert_raises(FileNotFoundError, - lambda: os.rename('DOES_NOT_EXIST', 'DOES_NOT_EXIST 2')) +assert_raises(FileNotFoundError, lambda: os.open("DOES_NOT_EXIST", os.O_RDONLY)) +assert_raises(FileNotFoundError, lambda: os.open("DOES_NOT_EXIST", os.O_WRONLY)) +assert_raises( + FileNotFoundError, lambda: os.rename("DOES_NOT_EXIST", "DOES_NOT_EXIST 2") +) # sendfile only supports in_fd as non-socket on linux and solaris if hasattr(os, "sendfile") and sys.platform.startswith("linux"): - src_fd = os.open('README.md', os.O_RDONLY) - dest_fd = os.open('destination.md', os.O_RDWR | os.O_CREAT) - src_len = os.stat('README.md').st_size + src_fd = os.open("README.md", os.O_RDONLY) + dest_fd = os.open("destination.md", os.O_RDWR | os.O_CREAT) + src_len = os.stat("README.md").st_size bytes_sent = os.sendfile(dest_fd, src_fd, 0, src_len) assert src_len == bytes_sent @@ -32,10 +31,10 @@ assert os.read(src_fd, src_len) == os.read(dest_fd, bytes_sent) os.close(src_fd) os.close(dest_fd) - os.remove('destination.md') + os.remove("destination.md") try: - os.open('DOES_NOT_EXIST', 0) + os.open("DOES_NOT_EXIST", 0) except OSError as err: assert err.errno == 2 @@ -81,15 +80,14 @@ assert_raises(TypeError, lambda: os.fspath([1, 2, 3])) -class TestWithTempDir(): +class TestWithTempDir: def __enter__(self): if os.name == "nt": base_folder = os.environ["TEMP"] else: base_folder = "/tmp" - name = os.path.join(base_folder, - "rustpython_test_os_" + str(int(time.time()))) + name = os.path.join(base_folder, "rustpython_test_os_" + str(int(time.time()))) while os.path.isdir(name): name = name + "_" @@ -102,7 +100,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): pass -class TestWithTempCurrentDir(): +class TestWithTempCurrentDir: def __enter__(self): self.prev_cwd = os.getcwd() @@ -130,8 +128,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert os.write(fd, CONTENT3) == len(CONTENT3) os.close(fd) - assert_raises(FileExistsError, - lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL)) + assert_raises( + FileExistsError, lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + ) fd = os.open(fname, os.O_RDONLY) assert os.read(fd, len(CONTENT2)) == CONTENT2 @@ -150,7 +149,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert not os.isatty(fd) # TODO: get os.lseek working on windows - if os.name != 'nt': + if os.name != "nt": fd = os.open(fname3, 0) assert os.read(fd, len(CONTENT2)) == CONTENT2 assert os.read(fd, len(CONTENT3)) == CONTENT3 @@ -201,8 +200,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): if dir_entry.is_symlink(): symlinks.add(dir_entry.name) - assert names == set( - [FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) + assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) assert paths == set([fname, fname2, folder, symlink_file, symlink_folder]) assert dirs == set([FOLDER, SYMLINK_FOLDER]) assert dirs_no_symlink == set([FOLDER]) @@ -270,23 +268,26 @@ def __exit__(self, exc_type, exc_val, exc_tb): os.stat(fname).st_mode == os.stat(symlink_file).st_mode os.stat(fname, follow_symlinks=False).st_ino == os.stat( - symlink_file, follow_symlinks=False).st_ino + symlink_file, follow_symlinks=False + ).st_ino os.stat(fname, follow_symlinks=False).st_mode == os.stat( - symlink_file, follow_symlinks=False).st_mode + symlink_file, follow_symlinks=False + ).st_mode # os.chmod if os.name != "nt": os.chmod(fname, 0o666) - assert oct(os.stat(fname).st_mode) == '0o100666' + assert oct(os.stat(fname).st_mode) == "0o100666" -# os.chown + # os.chown if os.name != "nt": # setup root_in_posix = False - if hasattr(os, 'geteuid'): - root_in_posix = (os.geteuid() == 0) + if hasattr(os, "geteuid"): + root_in_posix = os.geteuid() == 0 try: import pwd + all_users = [u.pw_uid for u in pwd.getpwall()] except (ImportError, AttributeError): all_users = [] @@ -299,10 +300,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): if not root_in_posix and len(all_users) > 1: uid_1, uid_2 = all_users[:2] gid = os.stat(fname1).st_gid - assert_raises(PermissionError, - lambda: os.chown(fname1, uid_1, gid)) - assert_raises(PermissionError, - lambda: os.chown(fname1, uid_2, gid)) + assert_raises(PermissionError, lambda: os.chown(fname1, uid_1, gid)) + assert_raises(PermissionError, lambda: os.chown(fname1, uid_2, gid)) # test chown with root perm and file name if root_in_posix and len(all_users) > 1: @@ -327,7 +326,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert uid == uid_2 # test gid change - if hasattr(os, 'getgroups'): + if hasattr(os, "getgroups"): groups = os.getgroups() if len(groups) > 1: gid_1, gid_2 = groups[:2] @@ -434,7 +433,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): os.close(wfd) # os.pipe2 -if sys.platform.startswith('linux') or sys.platform.startswith('freebsd'): +if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): rfd, wfd = os.pipe2(0) try: os.write(wfd, CONTENT2) @@ -460,11 +459,11 @@ def __exit__(self, exc_type, exc_val, exc_tb): with TestWithTempDir() as tmpdir: for i in range(0, 4): - file_name = os.path.join(tmpdir, 'file' + str(i)) - with open(file_name, 'w') as f: - f.write('test') + file_name = os.path.join(tmpdir, "file" + str(i)) + with open(file_name, "w") as f: + f.write("test") - expected_files = ['file0', 'file1', 'file2', 'file3'] + expected_files = ["file0", "file1", "file2", "file3"] dir_iter = os.scandir(tmpdir) collected_files = [dir_entry.name for dir_entry in dir_iter] @@ -476,13 +475,14 @@ def __exit__(self, exc_type, exc_val, exc_tb): dir_iter.close() - expected_files_bytes = [(file.encode(), os.path.join(tmpdir, - file).encode()) - for file in expected_files] + expected_files_bytes = [ + (file.encode(), os.path.join(tmpdir, file).encode()) for file in expected_files + ] dir_iter_bytes = os.scandir(tmpdir.encode()) - collected_files_bytes = [(dir_entry.name, dir_entry.path) - for dir_entry in dir_iter_bytes] + collected_files_bytes = [ + (dir_entry.name, dir_entry.path) for dir_entry in dir_iter_bytes + ] assert set(collected_files_bytes) == set(expected_files_bytes) @@ -492,8 +492,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert set(collected_files) == set(expected_files) collected_files = os.listdir(tmpdir.encode()) - assert set(collected_files) == set( - [file.encode() for file in expected_files]) + assert set(collected_files) == set([file.encode() for file in expected_files]) with TestWithTempCurrentDir(): os.chdir(tmpdir) @@ -502,20 +501,30 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert set(collected_files) == set(expected_files) # system() -if "win" not in sys.platform: - assert os.system('ls') == 0 - assert os.system('{') != 0 +if os.name in ("posix", "nt"): + assert os.system("echo test") == 0 + assert os.system("&") != 0 for arg in [None, 1, 1.0, TabError]: assert_raises(TypeError, os.system, arg) # Testing for os.pathconf_names -if not sys.platform.startswith('win'): +if not sys.platform.startswith("win"): assert len(os.pathconf_names) > 0 - assert 'PC_NAME_MAX' in os.pathconf_names + assert "PC_NAME_MAX" in os.pathconf_names for option, index in os.pathconf_names.items(): if sys.platform == "darwin": # TODO: check why it fails if option in ["PC_MAX_CANON", "PC_MAX_INPUT", "PC_VDISABLE"]: continue - assert os.pathconf('/', index) == os.pathconf('/', option) + assert os.pathconf("/", index) == os.pathconf("/", option) + +# os.access - test with empty path and nonexistent files +assert os.access("", os.F_OK) is False +assert os.access("", os.R_OK) is False +assert os.access("", os.W_OK) is False +assert os.access("", os.X_OK) is False +assert os.access("nonexistent_file_12345", os.F_OK) is False +assert os.access("nonexistent_file_12345", os.W_OK) is False +assert os.access("README.md", os.F_OK) is True +assert os.access("README.md", os.R_OK) is True diff --git a/extra_tests/snippets/stdlib_pwd.py b/extra_tests/snippets/stdlib_pwd.py new file mode 100644 index 00000000000..c3aeb7c8703 --- /dev/null +++ b/extra_tests/snippets/stdlib_pwd.py @@ -0,0 +1,14 @@ +import sys + +# windows doesn't support pwd +if sys.platform.startswith("win"): + exit(0) + +import pwd + +from testutils import assert_raises + +with assert_raises(KeyError): + fake_name = "fake_user" + while pwd.getpwnam(fake_name): + fake_name += "1" diff --git a/extra_tests/snippets/stdlib_random.py b/extra_tests/snippets/stdlib_random.py index 969b09d339b..60fc9a30973 100644 --- a/extra_tests/snippets/stdlib_random.py +++ b/extra_tests/snippets/stdlib_random.py @@ -15,8 +15,8 @@ assert random.choice(left) == 5 # random.choices -expected = ['red', 'green', 'red', 'black', 'black', 'red'] -result = random.choices(['red', 'black', 'green'], [18, 18, 2], k=6) +expected = ["red", "green", "red", "black", "black", "red"] +result = random.choices(["red", "black", "green"], [18, 18, 2], k=6) assert expected == result # random.sample @@ -30,7 +30,7 @@ assert len(zero_size_buf) == 0 non_zero_buf = random.randbytes(4) assert type(non_zero_buf) is bytes -assert list(non_zero_buf) == list(b'\xb9\x7fi\xf7') +assert list(non_zero_buf) == list(b"\xb9\x7fi\xf7") # TODO : random.random(), random.uniform(), random.triangular(), # random.betavariate, random.expovariate, random.gammavariate, diff --git a/extra_tests/snippets/stdlib_re.py b/extra_tests/snippets/stdlib_re.py index 17ecdba7f68..53f21f91734 100644 --- a/extra_tests/snippets/stdlib_re.py +++ b/extra_tests/snippets/stdlib_re.py @@ -1,8 +1,7 @@ - import re haystack = "Hello world" -needle = 'ello' +needle = "ello" mo = re.search(needle, haystack) print(mo) @@ -12,62 +11,71 @@ assert mo.start() == 1 assert mo.end() == 5 -assert re.escape('python.exe') == 'python\\.exe' +assert re.escape("python.exe") == "python\\.exe" -p = re.compile('ab') -s = p.sub('x', 'abcabca') +p = re.compile("ab") +s = p.sub("x", "abcabca") # print(s) -assert s == 'xcxca' +assert s == "xcxca" -idpattern = r'([_a-z][_a-z0-9]*)' +idpattern = r"([_a-z][_a-z0-9]*)" -mo = re.search(idpattern, '7382 _boe0+2') -assert mo.group(0) == '_boe0' +mo = re.search(idpattern, "7382 _boe0+2") +assert mo.group(0) == "_boe0" # tes op range -assert re.compile('[a-z]').match('a').span() == (0, 1) -assert re.compile('[a-z]').fullmatch('z').span() == (0, 1) +assert re.compile("[a-z]").match("a").span() == (0, 1) +assert re.compile("[a-z]").fullmatch("z").span() == (0, 1) # test op charset -assert re.compile('[_a-z0-9]*').match('_09az').group() == '_09az' +assert re.compile("[_a-z0-9]*").match("_09az").group() == "_09az" # test op bigcharset -assert re.compile('[你好a-z]*').match('a好z你?').group() == 'a好z你' -assert re.compile('[你好a-z]+').search('1232321 a好z你 !!?').group() == 'a好z你' +assert re.compile("[你好a-z]*").match("a好z你?").group() == "a好z你" +assert re.compile("[你好a-z]+").search("1232321 a好z你 !!?").group() == "a好z你" # test op repeat one -assert re.compile('a*').match('aaa').span() == (0, 3) -assert re.compile('abcd*').match('abcdddd').group() == 'abcdddd' -assert re.compile('abcd*').match('abc').group() == 'abc' -assert re.compile('abcd*e').match('abce').group() == 'abce' -assert re.compile('abcd*e+').match('abcddeee').group() == 'abcddeee' -assert re.compile('abcd+').match('abcddd').group() == 'abcddd' +assert re.compile("a*").match("aaa").span() == (0, 3) +assert re.compile("abcd*").match("abcdddd").group() == "abcdddd" +assert re.compile("abcd*").match("abc").group() == "abc" +assert re.compile("abcd*e").match("abce").group() == "abce" +assert re.compile("abcd*e+").match("abcddeee").group() == "abcddeee" +assert re.compile("abcd+").match("abcddd").group() == "abcddd" # test op mark -assert re.compile('(a)b').match('ab').group(0, 1) == ('ab', 'a') -assert re.compile('a(b)(cd)').match('abcd').group(0, 1, 2) == ('abcd', 'b', 'cd') +assert re.compile("(a)b").match("ab").group(0, 1) == ("ab", "a") +assert re.compile("a(b)(cd)").match("abcd").group(0, 1, 2) == ("abcd", "b", "cd") # test op repeat -assert re.compile('(ab)+').match('abab') -assert re.compile('(a)(b)(cd)*').match('abcdcdcd').group(0, 1, 2, 3) == ('abcdcdcd', 'a', 'b', 'cd') -assert re.compile('ab()+cd').match('abcd').group() == 'abcd' -assert re.compile('(a)+').match('aaa').groups() == ('a',) -assert re.compile('(a+)').match('aaa').groups() == ('aaa',) +assert re.compile("(ab)+").match("abab") +assert re.compile("(a)(b)(cd)*").match("abcdcdcd").group(0, 1, 2, 3) == ( + "abcdcdcd", + "a", + "b", + "cd", +) +assert re.compile("ab()+cd").match("abcd").group() == "abcd" +assert re.compile("(a)+").match("aaa").groups() == ("a",) +assert re.compile("(a+)").match("aaa").groups() == ("aaa",) # test Match object method -assert re.compile('(a)(bc)').match('abc')[1] == 'a' -assert re.compile('a(b)(?P<a>c)d').match('abcd').groupdict() == {'a': 'c'} +assert re.compile("(a)(bc)").match("abc")[1] == "a" +assert re.compile("a(b)(?P<a>c)d").match("abcd").groupdict() == {"a": "c"} # test op branch -assert re.compile(r'((?=\d|\.\d)(?P<int>\d*)|a)').match('123.2132').group() == '123' +assert re.compile(r"((?=\d|\.\d)(?P<int>\d*)|a)").match("123.2132").group() == "123" + +assert re.sub(r"^\s*", "X", "test") == "Xtest" -assert re.sub(r'^\s*', 'X', 'test') == 'Xtest' +assert re.match(r"\babc\b", "abc").group() == "abc" -assert re.match(r'\babc\b', 'abc').group() == 'abc' +urlpattern = re.compile("//([^/#?]*)(.*)", re.DOTALL) +url = "//www.example.org:80/foo/bar/baz.html" +assert urlpattern.match(url).group(1) == "www.example.org:80" -urlpattern = re.compile('//([^/#?]*)(.*)', re.DOTALL) -url = '//www.example.org:80/foo/bar/baz.html' -assert urlpattern.match(url).group(1) == 'www.example.org:80' +assert re.compile("(?:\w+(?:\s|/(?!>))*)*").match("a /bb />ccc").group() == "a /bb " +assert re.compile("(?:(1)?)*").match("111").group() == "111" -assert re.compile('(?:\w+(?:\s|/(?!>))*)*').match('a /bb />ccc').group() == 'a /bb ' -assert re.compile('(?:(1)?)*').match('111').group() == '111' \ No newline at end of file +# Test of fix re.fullmatch POSSESSIVE_REPEAT, issue #7183 +assert re.fullmatch(r"([0-9]++(?:\.[0-9]+)*+)", "1.25.38") +assert re.fullmatch(r"([0-9]++(?:\.[0-9]+)*+)", "1.25.38").group(0) == "1.25.38" diff --git a/extra_tests/snippets/stdlib_select.py b/extra_tests/snippets/stdlib_select.py index fe246db99b9..d27bb82b1c3 100644 --- a/extra_tests/snippets/stdlib_select.py +++ b/extra_tests/snippets/stdlib_select.py @@ -1,8 +1,8 @@ -from testutils import assert_raises - import select -import sys import socket +import sys + +from testutils import assert_raises class Nope: diff --git a/extra_tests/snippets/stdlib_signal.py b/extra_tests/snippets/stdlib_signal.py index eb4a25f90db..9d0b13bbd67 100644 --- a/extra_tests/snippets/stdlib_signal.py +++ b/extra_tests/snippets/stdlib_signal.py @@ -1,17 +1,19 @@ import signal -import time import sys +import time + from testutils import assert_raises assert_raises(TypeError, lambda: signal.signal(signal.SIGINT, 2)) signals = [] + def handler(signum, frame): - signals.append(signum) + signals.append(signum) -signal.signal(signal.SIGILL, signal.SIG_IGN); +signal.signal(signal.SIGILL, signal.SIG_IGN) assert signal.getsignal(signal.SIGILL) is signal.SIG_IGN old_signal = signal.signal(signal.SIGILL, signal.SIG_DFL) @@ -21,24 +23,21 @@ def handler(signum, frame): # unix if "win" not in sys.platform: - signal.signal(signal.SIGALRM, handler) - assert signal.getsignal(signal.SIGALRM) is handler - - signal.alarm(1) - time.sleep(2.0) - assert signals == [signal.SIGALRM] - - signal.signal(signal.SIGALRM, signal.SIG_IGN) - signal.alarm(1) - time.sleep(2.0) - - assert signals == [signal.SIGALRM] + signal.signal(signal.SIGALRM, handler) + assert signal.getsignal(signal.SIGALRM) is handler - signal.signal(signal.SIGALRM, handler) - signal.alarm(1) - time.sleep(2.0) + signal.alarm(1) + time.sleep(2.0) + assert signals == [signal.SIGALRM] - assert signals == [signal.SIGALRM, signal.SIGALRM] + signal.signal(signal.SIGALRM, signal.SIG_IGN) + signal.alarm(1) + time.sleep(2.0) + assert signals == [signal.SIGALRM] + signal.signal(signal.SIGALRM, handler) + signal.alarm(1) + time.sleep(2.0) + assert signals == [signal.SIGALRM, signal.SIGALRM] diff --git a/extra_tests/snippets/stdlib_socket.py b/extra_tests/snippets/stdlib_socket.py index bbedb794ba5..3f56d2b926e 100644 --- a/extra_tests/snippets/stdlib_socket.py +++ b/extra_tests/snippets/stdlib_socket.py @@ -1,12 +1,13 @@ import _socket -import socket import os +import socket + from testutils import assert_raises assert _socket.socket == _socket.SocketType -MESSAGE_A = b'aaaa' -MESSAGE_B= b'bbbbb' +MESSAGE_A = b"aaaa" +MESSAGE_B = b"bbbbb" # TCP @@ -26,9 +27,9 @@ assert recv_a == MESSAGE_A assert recv_b == MESSAGE_B -fd = open('README.md', 'rb') +fd = open("README.md", "rb") connector.sendfile(fd) -recv_readme = connection.recv(os.stat('README.md').st_size) +recv_readme = connection.recv(os.stat("README.md").st_size) # need this because sendfile leaves the cursor at the end of the file fd.seek(0) assert recv_readme == fd.read() @@ -36,14 +37,14 @@ # fileno if os.name == "posix": - connector_fd = connector.fileno() - connection_fd = connection.fileno() - os.write(connector_fd, MESSAGE_A) - connection.send(MESSAGE_B) - recv_a = connection.recv(len(MESSAGE_A)) - recv_b = os.read(connector_fd, (len(MESSAGE_B))) - assert recv_a == MESSAGE_A - assert recv_b == MESSAGE_B + connector_fd = connector.fileno() + connection_fd = connection.fileno() + os.write(connector_fd, MESSAGE_A) + connection.send(MESSAGE_B) + recv_a = connection.recv(len(MESSAGE_A)) + recv_b = os.read(connector_fd, (len(MESSAGE_B))) + assert recv_a == MESSAGE_A + assert recv_b == MESSAGE_B connection.close() connector.close() @@ -51,30 +52,30 @@ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) with assert_raises(TypeError): - s.connect(("127.0.0.1", 8888, 8888)) + s.connect(("127.0.0.1", 8888, 8888)) with assert_raises(OSError): - # Lets hope nobody is listening on port 1 - s.connect(("127.0.0.1", 1)) + # Lets hope nobody is listening on port 1 + s.connect(("127.0.0.1", 1)) with assert_raises(TypeError): - s.bind(("127.0.0.1", 8888, 8888)) + s.bind(("127.0.0.1", 8888, 8888)) with assert_raises(OSError): - # Lets hope nobody run this test on machine with ip 1.2.3.4 - s.bind(("1.2.3.4", 8888)) + # Lets hope nobody run this test on machine with ip 1.2.3.4 + s.bind(("1.2.3.4", 8888)) with assert_raises(TypeError): - s.bind((888, 8888)) + s.bind((888, 8888)) s.close() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("127.0.0.1", 0)) with assert_raises(OSError): - s.recv(100) + s.recv(100) with assert_raises(OSError): - s.send(MESSAGE_A) + s.send(MESSAGE_A) s.close() @@ -117,48 +118,49 @@ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) with assert_raises(OSError): - s.bind(("1.2.3.4", 888)) + s.bind(("1.2.3.4", 888)) s.close() ### Errors with assert_raises(OSError): - socket.socket(100, socket.SOCK_STREAM) + socket.socket(100, socket.SOCK_STREAM) with assert_raises(OSError): - socket.socket(socket.AF_INET, 1000) + socket.socket(socket.AF_INET, 1000) with assert_raises(OSError): - socket.inet_aton("test") + socket.inet_aton("test") -with assert_raises(OverflowError): - socket.htonl(-1) +# TODO: RUSTPYTHON +# with assert_raises(ValueError): +# socket.htonl(-1) -assert socket.htonl(0)==0 -assert socket.htonl(10)==167772160 +assert socket.htonl(0) == 0 +assert socket.htonl(10) == 167772160 -assert socket.inet_aton("127.0.0.1")==b"\x7f\x00\x00\x01" -assert socket.inet_aton("255.255.255.255")==b"\xff\xff\xff\xff" +assert socket.inet_aton("127.0.0.1") == b"\x7f\x00\x00\x01" +assert socket.inet_aton("255.255.255.255") == b"\xff\xff\xff\xff" -assert socket.inet_ntoa(b"\x7f\x00\x00\x01")=="127.0.0.1" -assert socket.inet_ntoa(b"\xff\xff\xff\xff")=="255.255.255.255" +assert socket.inet_ntoa(b"\x7f\x00\x00\x01") == "127.0.0.1" +assert socket.inet_ntoa(b"\xff\xff\xff\xff") == "255.255.255.255" with assert_raises(OSError): - socket.inet_ntoa(b"\xff\xff\xff\xff\xff") + socket.inet_ntoa(b"\xff\xff\xff\xff\xff") with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - pass + pass with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - connector = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - connector.connect(("127.0.0.1", listener.getsockname()[1])) - (connection, addr) = listener.accept() - connection.settimeout(1.0) - with assert_raises(OSError): # TODO: check that it raises a socket.timeout - # testing that it doesn't work with the timeout; that it stops blocking eventually - connection.recv(len(MESSAGE_A)) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + connector = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + connector.connect(("127.0.0.1", listener.getsockname()[1])) + (connection, addr) = listener.accept() + connection.settimeout(1.0) + with assert_raises(OSError): # TODO: check that it raises a socket.timeout + # testing that it doesn't work with the timeout; that it stops blocking eventually + connection.recv(len(MESSAGE_A)) for exc, expected_name in [ (socket.gaierror, "gaierror"), diff --git a/extra_tests/snippets/stdlib_sqlite.py b/extra_tests/snippets/stdlib_sqlite.py index 8ec5416fe29..f2e02b48cf1 100644 --- a/extra_tests/snippets/stdlib_sqlite.py +++ b/extra_tests/snippets/stdlib_sqlite.py @@ -18,6 +18,7 @@ INSERT INTO foo(key) VALUES (11); """) + class AggrSum: def __init__(self): self.val = 0.0 @@ -28,6 +29,7 @@ def step(self, val): def finalize(self): return self.val + cx.create_aggregate("mysum", 1, AggrSum) cur.execute("select mysum(key) from foo") assert cur.fetchone()[0] == 28.0 @@ -35,15 +37,19 @@ def finalize(self): # toobig = 2**64 # cur.execute("insert into foo(key) values (?)", (toobig,)) + class AggrText: def __init__(self): self.txt = "" + def step(self, txt): txt = str(txt) self.txt = self.txt + txt + def finalize(self): return self.txt + cx.create_aggregate("aggtxt", 1, AggrText) cur.execute("select aggtxt(key) from foo") -assert cur.fetchone()[0] == '341011' \ No newline at end of file +assert cur.fetchone()[0] == "341011" diff --git a/extra_tests/snippets/stdlib_string.py b/extra_tests/snippets/stdlib_string.py index 9151d2f593d..02bfc95c2db 100644 --- a/extra_tests/snippets/stdlib_string.py +++ b/extra_tests/snippets/stdlib_string.py @@ -1,22 +1,25 @@ import string +assert string.ascii_letters == "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +assert string.ascii_lowercase == "abcdefghijklmnopqrstuvwxyz" +assert string.ascii_uppercase == "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +assert string.digits == "0123456789" +assert string.hexdigits == "0123456789abcdefABCDEF" +assert string.octdigits == "01234567" +assert string.punctuation == "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" +assert string.whitespace == " \t\n\r\x0b\x0c", string.whitespace +assert ( + string.printable + == "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c" +) -assert string.ascii_letters == 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' -assert string.ascii_lowercase == 'abcdefghijklmnopqrstuvwxyz' -assert string.ascii_uppercase == 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' -assert string.digits == '0123456789' -assert string.hexdigits == '0123456789abcdefABCDEF' -assert string.octdigits == '01234567' -assert string.punctuation == '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~' -assert string.whitespace == ' \t\n\r\x0b\x0c', string.whitespace -assert string.printable == '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c' - -assert string.capwords('bla bla', ' ') == 'Bla Bla' +assert string.capwords("bla bla", " ") == "Bla Bla" from string import Template -s = Template('$who likes $what') -r = s.substitute(who='tim', what='kung pow') -assert r == 'tim likes kung pow' + +s = Template("$who likes $what") +r = s.substitute(who="tim", what="kung pow") +assert r == "tim likes kung pow" from string import Formatter diff --git a/extra_tests/snippets/stdlib_struct.py b/extra_tests/snippets/stdlib_struct.py index 83154c8100b..3f97f2519cc 100644 --- a/extra_tests/snippets/stdlib_struct.py +++ b/extra_tests/snippets/stdlib_struct.py @@ -1,51 +1,52 @@ +import struct from testutils import assert_raises -import struct -data = struct.pack('IH', 14, 12) +data = struct.pack("IH", 14, 12) assert data == bytes([14, 0, 0, 0, 12, 0]) -v1, v2 = struct.unpack('IH', data) +v1, v2 = struct.unpack("IH", data) assert v1 == 14 assert v2 == 12 -data = struct.pack('<IH', 14, 12) +data = struct.pack("<IH", 14, 12) assert data == bytes([14, 0, 0, 0, 12, 0]) -v1, v2 = struct.unpack('<IH', data) +v1, v2 = struct.unpack("<IH", data) assert v1 == 14 assert v2 == 12 -data = struct.pack('>IH', 14, 12) +data = struct.pack(">IH", 14, 12) assert data == bytes([0, 0, 0, 14, 0, 12]) -v1, v2 = struct.unpack('>IH', data) +v1, v2 = struct.unpack(">IH", data) assert v1 == 14 assert v2 == 12 -data = struct.pack('3B', 65, 66, 67) +data = struct.pack("3B", 65, 66, 67) assert data == bytes([65, 66, 67]) -v1, v2, v3 = struct.unpack('3B', data) +v1, v2, v3 = struct.unpack("3B", data) assert v1 == 65 assert v2 == 66 assert v3 == 67 with assert_raises(Exception): - data = struct.pack('B0B', 65, 66) + data = struct.pack("B0B", 65, 66) with assert_raises(Exception): - data = struct.pack('B2B', 65, 66) + data = struct.pack("B2B", 65, 66) -data = struct.pack('B1B', 65, 66) +data = struct.pack("B1B", 65, 66) with assert_raises(Exception): - struct.pack('<IH', "14", 12) + struct.pack("<IH", "14", 12) assert struct.calcsize("B") == 1 # assert struct.calcsize("<L4B") == 12 -assert struct.Struct('3B').pack(65, 66, 67) == bytes([65, 66, 67]) +assert struct.Struct("3B").pack(65, 66, 67) == bytes([65, 66, 67]) + class Indexable(object): def __init__(self, value): @@ -54,23 +55,24 @@ def __init__(self, value): def __index__(self): return self._value -data = struct.pack('B', Indexable(65)) + +data = struct.pack("B", Indexable(65)) assert data == bytes([65]) -data = struct.pack('5s', b"test1") +data = struct.pack("5s", b"test1") assert data == b"test1" -data = struct.pack('3s', b"test2") +data = struct.pack("3s", b"test2") assert data == b"tes" -data = struct.pack('7s', b"test3") +data = struct.pack("7s", b"test3") assert data == b"test3\0\0" -data = struct.pack('?', True) -assert data == b'\1' +data = struct.pack("?", True) +assert data == b"\1" -data = struct.pack('?', []) -assert data == b'\0' +data = struct.pack("?", []) +assert data == b"\0" assert struct.error.__module__ == "struct" assert struct.error.__name__ == "error" diff --git a/extra_tests/snippets/stdlib_subprocess.py b/extra_tests/snippets/stdlib_subprocess.py index 96ead765836..014011ea145 100644 --- a/extra_tests/snippets/stdlib_subprocess.py +++ b/extra_tests/snippets/stdlib_subprocess.py @@ -1,22 +1,27 @@ +import signal import subprocess -import time import sys -import signal +import time from testutils import assert_raises is_unix = not sys.platform.startswith("win") if is_unix: + def echo(text): return ["echo", text] + def sleep(secs): return ["sleep", str(secs)] else: + def echo(text): return ["cmd", "/C", f"echo {text}"] + def sleep(secs): # TODO: make work in a non-unixy environment (something with timeout.exe?) - return ["sleep", str(secs)] + return ["powershell", "/C", "sleep", str(secs)] + p = subprocess.Popen(echo("test")) @@ -32,7 +37,7 @@ def sleep(secs): assert p.poll() is None with assert_raises(subprocess.TimeoutExpired): - assert p.wait(1) + assert p.wait(1) p.wait() @@ -48,17 +53,17 @@ def sleep(secs): p.terminate() p.wait() if is_unix: - assert p.returncode == -signal.SIGTERM + assert p.returncode == -signal.SIGTERM else: - assert p.returncode == 1 + assert p.returncode == 1 p = subprocess.Popen(sleep(2)) p.kill() p.wait() if is_unix: - assert p.returncode == -signal.SIGKILL + assert p.returncode == -signal.SIGKILL else: - assert p.returncode == 1 + assert p.returncode == 1 p = subprocess.Popen(echo("test"), stdout=subprocess.PIPE) (stdout, stderr) = p.communicate() @@ -66,4 +71,4 @@ def sleep(secs): p = subprocess.Popen(sleep(5), stdout=subprocess.PIPE) with assert_raises(subprocess.TimeoutExpired): - p.communicate(timeout=1) + p.communicate(timeout=1) diff --git a/extra_tests/snippets/stdlib_sys.py b/extra_tests/snippets/stdlib_sys.py index 5d8859ac8e8..155fc905a73 100644 --- a/extra_tests/snippets/stdlib_sys.py +++ b/extra_tests/snippets/stdlib_sys.py @@ -1,26 +1,31 @@ -import sys import os import subprocess +import sys from testutils import assert_raises -print('python executable:', sys.executable) +print("python executable:", sys.executable) print(sys.argv) -assert sys.argv[0].endswith('.py') +assert sys.argv[0].endswith(".py") -assert sys.platform == "linux" or sys.platform == "darwin" or sys.platform == "win32" or sys.platform == "unknown" +assert ( + sys.platform == "linux" + or sys.platform == "darwin" + or sys.platform == "win32" + or sys.platform == "unknown" +) if hasattr(sys, "_framework"): assert type(sys._framework) is str assert isinstance(sys.builtin_module_names, tuple) -assert 'sys' in sys.builtin_module_names +assert "sys" in sys.builtin_module_names assert isinstance(sys.implementation.name, str) assert isinstance(sys.implementation.cache_tag, str) -assert sys.getfilesystemencoding() == 'utf-8' -assert sys.getfilesystemencodeerrors().startswith('surrogate') +assert sys.getfilesystemencoding() == "utf-8" +assert sys.getfilesystemencodeerrors().startswith("surrogate") assert sys.byteorder == "little" or sys.byteorder == "big" @@ -35,15 +40,18 @@ events = [] + def trc(frame, event, arg): fn_name = frame.f_code.co_name events.append((fn_name, event, arg)) - print('trace event:', fn_name, event, arg) + print("trace event:", fn_name, event, arg) + def demo(x): if x > 0: demo(x - 1) + sys.settrace(trc) demo(5) sys.settrace(None) @@ -53,7 +61,7 @@ def demo(x): assert sys.exc_info() == (None, None, None) try: - 1/0 + 1 / 0 except ZeroDivisionError as exc: exc_info = sys.exc_info() assert exc_info[0] == type(exc) == ZeroDivisionError @@ -62,10 +70,12 @@ def demo(x): # Recursion: + def recursive_call(n): if n > 0: recursive_call(n - 1) + sys.setrecursionlimit(200) assert sys.getrecursionlimit() == 200 @@ -74,14 +84,28 @@ def recursive_call(n): if sys.platform.startswith("win"): winver = sys.getwindowsversion() - print(f'winver: {winver} {winver.platform_version}') + print(f"winver: {winver} {winver.platform_version}") # the biggest value of wSuiteMask (https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa#members). - all_masks = 0x00000004 | 0x00000400 | 0x00004000 | 0x00000080 | 0x00000002 | 0x00000040 | 0x00000200 | \ - 0x00000100 | 0x00000001 | 0x00000020 | 0x00002000 | 0x00000010 | 0x00008000 | 0x00020000 + all_masks = ( + 0x00000004 + | 0x00000400 + | 0x00004000 + | 0x00000080 + | 0x00000002 + | 0x00000040 + | 0x00000200 + | 0x00000100 + | 0x00000001 + | 0x00000020 + | 0x00002000 + | 0x00000010 + | 0x00008000 + | 0x00020000 + ) # We really can't test if the results are correct, so it just checks for meaningful value - assert winver.major > 0 + assert winver.major > 6 assert winver.minor >= 0 assert winver.build > 0 assert winver.platform == 2 @@ -89,10 +113,10 @@ def recursive_call(n): assert 0 <= winver.suite_mask <= all_masks assert 1 <= winver.product_type <= 3 - # XXX if platform_version is implemented correctly, this'll break on compatiblity mode or a build without manifest + # XXX if platform_version is implemented correctly, this'll break on compatibility mode or a build without manifest # these fields can mismatch in CPython - # assert winver.major == winver.platform_version[0] - # assert winver.minor == winver.platform_version[1] + assert winver.major == winver.platform_version[0] + assert winver.minor == winver.platform_version[1] # assert winver.build == winver.platform_version[2] # test int_max_str_digits getter and setter @@ -112,18 +136,25 @@ def recursive_call(n): # Test the PYTHONSAFEPATH environment variable code = "import sys; print(sys.flags.safe_path)" env = dict(os.environ) -env.pop('PYTHONSAFEPATH', None) -args = (sys.executable, '-P', '-c', code) +env.pop("PYTHONSAFEPATH", None) +args = (sys.executable, "-P", "-c", code) -proc = subprocess.run( - args, stdout=subprocess.PIPE, - universal_newlines=True, env=env) -assert proc.stdout.rstrip() == 'True', proc +proc = subprocess.run(args, stdout=subprocess.PIPE, universal_newlines=True, env=env) +assert proc.stdout.rstrip() == "True", proc assert proc.returncode == 0, proc -env['PYTHONSAFEPATH'] = '1' -proc = subprocess.run( - args, stdout=subprocess.PIPE, - universal_newlines=True, env=env) -assert proc.stdout.rstrip() == 'True' +env["PYTHONSAFEPATH"] = "1" +proc = subprocess.run(args, stdout=subprocess.PIPE, universal_newlines=True, env=env) +assert proc.stdout.rstrip() == "True" assert proc.returncode == 0, proc + +assert sys._getframemodulename() == "__main__", sys._getframemodulename() + + +def test_getframemodulename(): + return sys._getframemodulename() + + +test_getframemodulename.__module__ = "awesome_module" + +assert test_getframemodulename() == "awesome_module" diff --git a/extra_tests/snippets/stdlib_sys_getframe.py b/extra_tests/snippets/stdlib_sys_getframe.py index d4328286aad..50447ce8822 100644 --- a/extra_tests/snippets/stdlib_sys_getframe.py +++ b/extra_tests/snippets/stdlib_sys_getframe.py @@ -2,20 +2,24 @@ value = 189 locals_dict = sys._getframe().f_locals -assert locals_dict['value'] == 189 -foo = 'bar' -assert locals_dict['foo'] == foo +assert locals_dict["value"] == 189 +foo = "bar" +assert locals_dict["foo"] == foo + def test_function(): x = 17 assert sys._getframe().f_locals is not locals_dict - assert sys._getframe().f_locals['x'] == 17 - assert sys._getframe(1).f_locals['foo'] == 'bar' + assert sys._getframe().f_locals["x"] == 17 + assert sys._getframe(1).f_locals["foo"] == "bar" + test_function() -class TestClass(): + +class TestClass: def __init__(self): - assert sys._getframe().f_locals['self'] == self + assert sys._getframe().f_locals["self"] == self + TestClass() diff --git a/extra_tests/snippets/stdlib_time.py b/extra_tests/snippets/stdlib_time.py index baf6755306b..9a92969f5f9 100644 --- a/extra_tests/snippets/stdlib_time.py +++ b/extra_tests/snippets/stdlib_time.py @@ -1,5 +1,3 @@ - - import time x = time.gmtime(1000) @@ -9,14 +7,13 @@ assert x.tm_sec == 40 assert x.tm_isdst == 0 -s = time.strftime('%Y-%m-%d-%H-%M-%S', x) +s = time.strftime("%Y-%m-%d-%H-%M-%S", x) # print(s) -assert s == '1970-01-01-00-16-40' +assert s == "1970-01-01-00-16-40" -x2 = time.strptime(s, '%Y-%m-%d-%H-%M-%S') +x2 = time.strptime(s, "%Y-%m-%d-%H-%M-%S") assert x2.tm_min == 16 s = time.asctime(x) # print(s) -assert s == 'Thu Jan 1 00:16:40 1970' - +assert s == "Thu Jan 1 00:16:40 1970" diff --git a/extra_tests/snippets/stdlib_traceback.py b/extra_tests/snippets/stdlib_traceback.py index 689f36e0276..c2cc5773dbc 100644 --- a/extra_tests/snippets/stdlib_traceback.py +++ b/extra_tests/snippets/stdlib_traceback.py @@ -1,27 +1,27 @@ import traceback try: - 1/0 + 1 / 0 except ZeroDivisionError as ex: - tb = traceback.extract_tb(ex.__traceback__) - assert len(tb) == 1 + tb = traceback.extract_tb(ex.__traceback__) + assert len(tb) == 1 try: - try: - 1/0 - except ZeroDivisionError as ex: - raise KeyError().with_traceback(ex.__traceback__) + try: + 1 / 0 + except ZeroDivisionError as ex: + raise KeyError().with_traceback(ex.__traceback__) except KeyError as ex2: - tb = traceback.extract_tb(ex2.__traceback__) - assert tb[1].line == "1/0" + tb = traceback.extract_tb(ex2.__traceback__) + assert tb[1].line == "1 / 0" try: - try: - 1/0 - except ZeroDivisionError as ex: - raise ex.with_traceback(None) + try: + 1 / 0 + except ZeroDivisionError as ex: + raise ex.with_traceback(None) except ZeroDivisionError as ex2: - tb = traceback.extract_tb(ex2.__traceback__) - assert len(tb) == 1 + tb = traceback.extract_tb(ex2.__traceback__) + assert len(tb) == 1 diff --git a/extra_tests/snippets/stdlib_types.py b/extra_tests/snippets/stdlib_types.py index 479004b6cfb..cdecf12dd2b 100644 --- a/extra_tests/snippets/stdlib_types.py +++ b/extra_tests/snippets/stdlib_types.py @@ -1,10 +1,36 @@ +import _ast +import platform import types from testutils import assert_raises -ns = types.SimpleNamespace(a=2, b='Rust') +ns = types.SimpleNamespace(a=2, b="Rust") assert ns.a == 2 assert ns.b == "Rust" with assert_raises(AttributeError): _ = ns.c + + +def _run_missing_type_params_regression(): + args = _ast.arguments( + posonlyargs=[], + args=[], + vararg=None, + kwonlyargs=[], + kw_defaults=[], + kwarg=None, + defaults=[], + ) + pass_stmt = _ast.Pass(lineno=1, col_offset=4, end_lineno=1, end_col_offset=8) + fn = _ast.FunctionDef("f", args, [pass_stmt], [], None, None) + fn.lineno = 1 + fn.col_offset = 0 + fn.end_lineno = 1 + fn.end_col_offset = 8 + mod = _ast.Module([fn], []) + compiled = compile(mod, "<stdlib_types_missing_type_params>", "exec") + exec(compiled, {}) + + +_run_missing_type_params_regression() diff --git a/extra_tests/snippets/stdlib_typing.py b/extra_tests/snippets/stdlib_typing.py new file mode 100644 index 00000000000..07348945842 --- /dev/null +++ b/extra_tests/snippets/stdlib_typing.py @@ -0,0 +1,37 @@ +from collections.abc import Awaitable, Callable +from typing import TypeVar + +T = TypeVar("T") + + +def abort_signal_handler( + fn: Callable[[], Awaitable[T]], on_abort: Callable[[], None] | None = None +) -> T: + pass + + +# Ensure PEP 604 unions work with typing.Callable aliases. +TracebackFilter = bool | Callable[[int], int] + + +# Test that Union/Optional in function parameter annotations work correctly. +# This tests that annotation scopes can access global implicit symbols (like Union) +# that are imported at module level but not explicitly bound in the function scope. +# Regression test for: rich +from typing import Optional, Union + + +def function_with_union_param(x: Optional[Union[int, str]] = None) -> None: + pass + + +class ClassWithUnionParams: + def __init__( + self, + color: Optional[Union[str, int]] = None, + bold: Optional[bool] = None, + ) -> None: + pass + + def method(self, value: Union[int, float]) -> Union[str, bytes]: + return str(value) diff --git a/extra_tests/snippets/stdlib_weakref.py b/extra_tests/snippets/stdlib_weakref.py index 059ddd00009..17762addca7 100644 --- a/extra_tests/snippets/stdlib_weakref.py +++ b/extra_tests/snippets/stdlib_weakref.py @@ -1,4 +1,5 @@ -from _weakref import ref, proxy +from _weakref import proxy, ref + from testutils import assert_raises diff --git a/extra_tests/snippets/stdlib_xdrlib.py b/extra_tests/snippets/stdlib_xdrlib.py deleted file mode 100644 index 681cd774674..00000000000 --- a/extra_tests/snippets/stdlib_xdrlib.py +++ /dev/null @@ -1,12 +0,0 @@ -# This probably will be superceeded by the python unittests when that works. - -import xdrlib - -p = xdrlib.Packer() -p.pack_int(1337) - -d = p.get_buffer() - -print(d) - -# assert d == b'\x00\x00\x059' diff --git a/extra_tests/snippets/stdlib_zlib.py b/extra_tests/snippets/stdlib_zlib.py index 308a8d23bfb..ac2b525b0a7 100644 --- a/extra_tests/snippets/stdlib_zlib.py +++ b/extra_tests/snippets/stdlib_zlib.py @@ -1,4 +1,5 @@ import zlib + from testutils import assert_raises # checksum functions diff --git a/extra_tests/snippets/syntax_annotations.py b/extra_tests/snippets/syntax_annotations.py new file mode 100644 index 00000000000..0f78bdc2be1 --- /dev/null +++ b/extra_tests/snippets/syntax_annotations.py @@ -0,0 +1,10 @@ +from typing import get_type_hints + +def func(s: str) -> int: + return int(s) + +hints = get_type_hints(func) + +# The order of type hints matters for certain functions +# e.g. functools.singledispatch +assert list(hints.items()) == [('s', str), ('return', int)] diff --git a/extra_tests/snippets/syntax_assignment.py b/extra_tests/snippets/syntax_assignment.py index 8635dc5d795..851558a9db0 100644 --- a/extra_tests/snippets/syntax_assignment.py +++ b/extra_tests/snippets/syntax_assignment.py @@ -59,7 +59,18 @@ def g(): assert a == 1337 assert b == False -assert __annotations__['a'] == bool +# PEP 649: In Python 3.14, __annotations__ is not automatically defined at module level +# Accessing it raises NameError +from testutils import assert_raises + +with assert_raises(NameError): + __annotations__ + +# Use __annotate__ to get annotations (PEP 649) +assert callable(__annotate__) +annotations = __annotate__(1) # 1 = FORMAT_VALUE +assert annotations['a'] == bool +assert annotations['b'] == bool n = 0 diff --git a/extra_tests/snippets/syntax_async_comprehension.py b/extra_tests/snippets/syntax_async_comprehension.py new file mode 100644 index 00000000000..7d1d02672df --- /dev/null +++ b/extra_tests/snippets/syntax_async_comprehension.py @@ -0,0 +1,171 @@ +import asyncio +from types import GeneratorType, AsyncGeneratorType + + +async def f_async(x): + await asyncio.sleep(0.001) + return x + + +def f_iter(): + for i in range(5): + yield i + + +async def f_aiter(): + for i in range(5): + await asyncio.sleep(0.001) + yield i + + +async def run_async(): + # list + x = [i for i in range(5)] + assert isinstance(x, list) + for i, e in enumerate(x): + assert e == i + + x = [await f_async(i) for i in range(5)] + assert isinstance(x, list) + for i, e in enumerate(x): + assert e == i + + x = [e async for e in f_aiter()] + assert isinstance(x, list) + for i, e in enumerate(x): + assert e == i + + x = [await f_async(i) async for i in f_aiter()] + assert isinstance(x, list) + for i, e in enumerate(x): + assert e == i + + # set + x = {i for i in range(5)} + assert isinstance(x, set) + for e in x: + assert e in range(5) + assert x == {0, 1, 2, 3, 4} + + x = {await f_async(i) for i in range(5)} + assert isinstance(x, set) + for e in x: + assert e in range(5) + assert x == {0, 1, 2, 3, 4} + + x = {e async for e in f_aiter()} + assert isinstance(x, set) + for e in x: + assert e in range(5) + assert x == {0, 1, 2, 3, 4} + + x = {await f_async(i) async for i in f_aiter()} + assert isinstance(x, set) + for e in x: + assert e in range(5) + assert x == {0, 1, 2, 3, 4} + + # dict + x = {i: i for i in range(5)} + assert isinstance(x, dict) + for k, v in x.items(): + assert k == v + assert x == {0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + + x = {await f_async(i): i for i in range(5)} + assert isinstance(x, dict) + for k, v in x.items(): + assert k == v + assert x == {0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + + x = {i: await f_async(i) for i in range(5)} + assert isinstance(x, dict) + for k, v in x.items(): + assert k == v + assert x == {0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + + x = {await f_async(i): await f_async(i) for i in range(5)} + assert isinstance(x, dict) + for k, v in x.items(): + assert k == v + assert x == {0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + + x = {i: i async for i in f_aiter()} + assert isinstance(x, dict) + for k, v in x.items(): + assert k == v + assert x == {0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + + x = {await f_async(i): i async for i in f_aiter()} + assert isinstance(x, dict) + for k, v in x.items(): + assert k == v + assert x == {0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + + x = {i: await f_async(i) async for i in f_aiter()} + assert isinstance(x, dict) + for k, v in x.items(): + assert k == v + assert x == {0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + + x = {await f_async(i): await f_async(i) async for i in f_aiter()} + assert isinstance(x, dict) + for k, v in x.items(): + assert k == v + assert x == {0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + + # generator + x = (i for i in range(5)) + assert isinstance(x, GeneratorType) + for i, e in enumerate(x): + assert e == i + + x = (await f_async(i) for i in range(5)) + assert isinstance(x, AsyncGeneratorType) + i = 0 + async for e in x: + assert e == i + i += 1 + + x = (e async for e in f_aiter()) + assert isinstance(x, AsyncGeneratorType) + i = 0 + async for e in x: + assert i == e + i += 1 + + x = (await f_async(i) async for i in f_aiter()) + assert isinstance(x, AsyncGeneratorType) + i = 0 + async for e in x: + assert i == e + i += 1 + + +def run_sync(): + async def test_async_for(x): + i = 0 + async for e in x: + assert e == i + i += 1 + + x = (i for i in range(5)) + assert isinstance(x, GeneratorType) + for i, e in enumerate(x): + assert e == i + + x = (await f_async(i) for i in range(5)) + assert isinstance(x, AsyncGeneratorType) + asyncio.run(test_async_for(x), debug=True) + + x = (e async for e in f_aiter()) + assert isinstance(x, AsyncGeneratorType) + asyncio.run(test_async_for(x), debug=True) + + x = (await f_async(i) async for i in f_aiter()) + assert isinstance(x, AsyncGeneratorType) + asyncio.run(test_async_for(x), debug=True) + + +asyncio.run(run_async(), debug=True) +run_sync() diff --git a/extra_tests/snippets/syntax_class.py b/extra_tests/snippets/syntax_class.py index 28d066a9e97..4e80e7edf8c 100644 --- a/extra_tests/snippets/syntax_class.py +++ b/extra_tests/snippets/syntax_class.py @@ -50,7 +50,7 @@ def kungfu(x): assert x == 3 -assert Bar.__doc__ == " W00t " +assert Bar.__doc__ == "W00t " bar = Bar(42) assert bar.get_x.__doc__ == None @@ -147,7 +147,7 @@ class T3: test3 """ -assert T3.__doc__ == "\n test3\n " +assert T3.__doc__ == "\ntest3\n" class T4: @@ -186,7 +186,7 @@ def b(): class A: pass -assert A.__doc__ == None +assert A.__doc__ is None, A.__doc__ class B: "Docstring" diff --git a/extra_tests/snippets/syntax_comprehension.py b/extra_tests/snippets/syntax_comprehension.py index 41bd1ec8512..6445c655306 100644 --- a/extra_tests/snippets/syntax_comprehension.py +++ b/extra_tests/snippets/syntax_comprehension.py @@ -43,5 +43,5 @@ def f(): - # Test no panic occured. + # Test no panic occurred. [[x := 1 for j in range(5)] for i in range(5)] diff --git a/extra_tests/snippets/syntax_doc.py b/extra_tests/snippets/syntax_doc.py new file mode 100644 index 00000000000..bdfc1fe7788 --- /dev/null +++ b/extra_tests/snippets/syntax_doc.py @@ -0,0 +1,15 @@ + +def f1(): + """ + x + \ty + """ +assert f1.__doc__ == '\nx\ny\n' + +def f2(): + """ +\t x +\t\ty + """ + +assert f2.__doc__ == '\nx\n y\n' diff --git a/extra_tests/snippets/syntax_forbidden_name.py b/extra_tests/snippets/syntax_forbidden_name.py index 2e114fe8800..3bd8148436e 100644 --- a/extra_tests/snippets/syntax_forbidden_name.py +++ b/extra_tests/snippets/syntax_forbidden_name.py @@ -21,6 +21,12 @@ def raisesSyntaxError(parse_stmt, exec_stmt=None): raisesSyntaxError("", "del __debug__") raisesSyntaxError("", "(a, __debug__, c) = (1, 2, 3)") raisesSyntaxError("", "(a, *__debug__, c) = (1, 2, 3)") +raisesSyntaxError("", "__debug__ : int") +raisesSyntaxError("", "__debug__ : int = 1") -# TODO: -# raisesSyntaxError("", "__debug__ : int") +# Import statements +raisesSyntaxError("import sys as __debug__") +raisesSyntaxError("from sys import path as __debug__") + +# Comprehension iteration targets +raisesSyntaxError("[x for __debug__ in range(5)]") diff --git a/extra_tests/snippets/syntax_fstring.py b/extra_tests/snippets/syntax_fstring.py index 5d274478f15..ec198a8125d 100644 --- a/extra_tests/snippets/syntax_fstring.py +++ b/extra_tests/snippets/syntax_fstring.py @@ -54,12 +54,6 @@ part_spec = ">+#10x" assert f"{16:0{part_spec}}{foo}" == '00000+0x10bar' -# TODO: RUSTPYTHON, delete the next block once `test_fstring.py` can successfully parse -assert f'{10:#{1}0x}' == ' 0xa' -assert f'{10:{"#"}1{0}{"x"}}' == ' 0xa' -assert f'{-10:-{"#"}1{0}x}' == ' -0xa' -assert f'{-10:{"-"}#{1}0{"x"}}' == ' -0xa' - spec = "bla" assert_raises(ValueError, lambda: f"{16:{spec}}") diff --git a/extra_tests/snippets/syntax_function2.py b/extra_tests/snippets/syntax_function2.py index ebea34fe580..4a04acd51c1 100644 --- a/extra_tests/snippets/syntax_function2.py +++ b/extra_tests/snippets/syntax_function2.py @@ -44,7 +44,7 @@ def f3(): """ pass -assert f3.__doc__ == "\n test3\n " +assert f3.__doc__ == "\ntest3\n" def f4(): "test4" @@ -52,6 +52,8 @@ def f4(): assert f4.__doc__ == "test4" +assert type(lambda: None).__doc__.startswith("Create a function object."), type(f4).__doc__ + def revdocstr(f): d = f.__doc__ @@ -78,6 +80,7 @@ def nested(): def f7(): + # PEP 649: annotations are deferred, so void is not evaluated at definition time try: def t() -> void: # noqa: F821 pass @@ -85,7 +88,7 @@ def t() -> void: # noqa: F821 return True return False -assert f7() +assert not f7() # PEP 649: no NameError because annotation is deferred def f8() -> int: diff --git a/extra_tests/snippets/syntax_match.py b/extra_tests/snippets/syntax_match.py new file mode 100644 index 00000000000..8868cb93c38 --- /dev/null +++ b/extra_tests/snippets/syntax_match.py @@ -0,0 +1,151 @@ +i = 0 +z = 1 +match i: + case 0: + z = 0 + case 1: + z = 2 + case _: + z = 3 + +assert z == 0 +# Test enum +from enum import Enum + + +class Color(Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + + +def test_color(color): + z = -1 + match color: + case Color.RED: + z = 1 + case Color.GREEN: + z = 2 + case Color.BLUE: + z = 3 + assert z == color.value + + +for color in Color: + test_color(color) + + +# test or +def test_or(i): + z = -1 + match i: + case 0 | 1: + z = 0 + case 2 | 3: + z = 1 + case _: + z = 2 + return z + + +assert test_or(0) == 0 +assert test_or(1) == 0 +assert test_or(2) == 1 +assert test_or(3) == 1 +assert test_or(4) == 2 + +# test mapping +data = {"a": 1, "b": 2} +match data: + case {"a": x}: + assert x == 1 + case _: + assert False + +match data: + case {"a": x, "b": y}: + assert x == 1, x + assert y == 2, y + case _: + assert False + +# test mapping with rest +match data: + case {"a": x, **rest}: + assert x == 1 + assert rest == {"b": 2} + case _: + assert False + +# test empty rest +data2 = {"a": 1} +match data2: + case {"a": x, **rest}: + assert x == 1 + assert rest == {} + case _: + assert False + +# test rest with multiple keys +data3 = {"a": 1, "b": 2, "c": 3, "d": 4} +match data3: + case {"a": x, "b": y, **rest}: + assert x == 1 + assert y == 2 + assert rest == {"c": 3, "d": 4} + case _: + assert False + +match data3: + case {"a": x, "b": y, "c": z, **rest}: + assert x == 1 + assert y == 2 + assert z == 3 + assert rest == {"d": 4} + case _: + assert False + +# test mapping pattern with wildcard fallback (reproduces wheelinfo.py issue) +test_dict = {"sha256": "abc123"} +result = None +match test_dict: + case {"sha256": checksum}: + result = checksum + case _: + result = "no checksum" +assert result == "abc123" + +# test with no match +test_dict2 = {"md5": "xyz789"} +match test_dict2: + case {"sha256": checksum}: + result = checksum + case _: + result = "no checksum" +assert result == "no checksum" + + +# test mapping patterns - comprehensive tests +def test_mapping_comprehensive(): + # Single key capture + data = {"a": 1} + match data: + case {"a": x}: + captured = x + case _: + captured = None + assert captured == 1, f"Expected 1, got {captured}" + + # Multiple keys + data = {"a": 1, "b": 2} + match data: + case {"a": x, "b": y}: + cap_x = x + cap_y = y + case _: + cap_x = cap_y = None + assert cap_x == 1, f"Expected x=1, got {cap_x}" + assert cap_y == 2, f"Expected y=2, got {cap_y}" + + +test_mapping_comprehensive() diff --git a/extra_tests/snippets/syntax_non_utf8.py b/extra_tests/snippets/syntax_non_utf8.py index ad93d5b33c1..8f11078bc07 100644 --- a/extra_tests/snippets/syntax_non_utf8.py +++ b/extra_tests/snippets/syntax_non_utf8.py @@ -5,5 +5,5 @@ dir_path = os.path.dirname(os.path.realpath(__file__)) with assert_raises(SyntaxError): - with open(os.path.join(dir_path , "non_utf8.txt")) as f: + with open(os.path.join(dir_path , "non_utf8.txt"), encoding="latin-1") as f: eval(f.read()) diff --git a/extra_tests/snippets/syntax_short_circuit_bool.py b/extra_tests/snippets/syntax_short_circuit_bool.py index 76d89352cbb..6cbae190cae 100644 --- a/extra_tests/snippets/syntax_short_circuit_bool.py +++ b/extra_tests/snippets/syntax_short_circuit_bool.py @@ -31,3 +31,6 @@ def __bool__(self): # if ExplodingBool(False) and False and True and False: # pass + +# Issue #3567: nested BoolOps should not call __bool__ redundantly +assert (ExplodingBool(False) and False or False) == False diff --git a/extra_tests/snippets/test_threading.py b/extra_tests/snippets/test_threading.py index 41024b360e7..4d7c29f5093 100644 --- a/extra_tests/snippets/test_threading.py +++ b/extra_tests/snippets/test_threading.py @@ -11,7 +11,7 @@ def thread_function(name): output.append((0, 0)) -x = threading.Thread(target=thread_function, args=(1, )) +x = threading.Thread(target=thread_function, args=(1,)) output.append((0, 1)) x.start() output.append((0, 2)) diff --git a/extra_tests/snippets/testutils.py b/extra_tests/snippets/testutils.py index 437fa06ae3a..aac153441e5 100644 --- a/extra_tests/snippets/testutils.py +++ b/extra_tests/snippets/testutils.py @@ -1,6 +1,7 @@ import platform import sys + def assert_raises(expected, *args, _msg=None, **kw): if args: f, f_args = args[0], args[1:] @@ -22,8 +23,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: - failmsg = self.failmsg or \ - '{} was not raised'.format(self.expected.__name__) + failmsg = self.failmsg or "{} was not raised".format(self.expected.__name__) assert False, failmsg if not issubclass(exc_type, self.expected): return False @@ -36,6 +36,7 @@ class TestFailingBool: def __bool__(self): raise RuntimeError + class TestFailingIter: def __iter__(self): raise RuntimeError @@ -48,47 +49,64 @@ def _assert_print(f, args): raised = False finally: if raised: - print('Assertion Failure:', *args) + print("Assertion Failure:", *args) + def _typed(obj): - return '{}({})'.format(type(obj), obj) + return "{}({})".format(type(obj), obj) def assert_equal(a, b): - _assert_print(lambda: a == b, [_typed(a), '==', _typed(b)]) + _assert_print(lambda: a == b, [_typed(a), "==", _typed(b)]) def assert_true(e): - _assert_print(lambda: e is True, [_typed(e), 'is True']) + _assert_print(lambda: e is True, [_typed(e), "is True"]) def assert_false(e): - _assert_print(lambda: e is False, [_typed(e), 'is False']) + _assert_print(lambda: e is False, [_typed(e), "is False"]) + def assert_isinstance(obj, klass): - _assert_print(lambda: isinstance(obj, klass), ['isisntance(', _typed(obj), ',', klass, ')']) + _assert_print( + lambda: isinstance(obj, klass), ["isisntance(", _typed(obj), ",", klass, ")"] + ) + def assert_in(a, b): - _assert_print(lambda: a in b, [a, 'in', b]) + _assert_print(lambda: a in b, [a, "in", b]) + def skip_if_unsupported(req_maj_vers, req_min_vers, test_fct): def exec(): test_fct() - if platform.python_implementation() == 'RustPython': + if platform.python_implementation() == "RustPython": exec() - elif sys.version_info.major>=req_maj_vers and sys.version_info.minor>=req_min_vers: + elif ( + sys.version_info.major >= req_maj_vers + and sys.version_info.minor >= req_min_vers + ): exec() else: - print(f'Skipping test as a higher python version is required. Using {platform.python_implementation()} {platform.python_version()}') + print( + f"Skipping test as a higher python version is required. Using {platform.python_implementation()} {platform.python_version()}" + ) + def fail_if_unsupported(req_maj_vers, req_min_vers, test_fct): def exec(): test_fct() - if platform.python_implementation() == 'RustPython': + if platform.python_implementation() == "RustPython": exec() - elif sys.version_info.major>=req_maj_vers and sys.version_info.minor>=req_min_vers: + elif ( + sys.version_info.major >= req_maj_vers + and sys.version_info.minor >= req_min_vers + ): exec() else: - assert False, f'Test cannot performed on this python version. {platform.python_implementation()} {platform.python_version()}' + assert False, ( + f"Test cannot performed on this python version. {platform.python_implementation()} {platform.python_version()}" + ) diff --git a/extra_tests/snippets/vm_specialization.py b/extra_tests/snippets/vm_specialization.py new file mode 100644 index 00000000000..2c884cc2f6d --- /dev/null +++ b/extra_tests/snippets/vm_specialization.py @@ -0,0 +1,71 @@ +## BinaryOp inplace-add unicode: deopt falls back to __add__/__iadd__ + + +class S(str): + def __add__(self, other): + return "ADD" + + def __iadd__(self, other): + return "IADD" + + +def add_path_fallback_uses_add(): + x = "a" + y = "b" + for i in range(1200): + if i == 600: + x = S("s") + y = "t" + x = x + y + return x + + +def iadd_path_fallback_uses_iadd(): + x = "a" + y = "b" + for i in range(1200): + if i == 600: + x = S("s") + y = "t" + x += y + return x + + +assert add_path_fallback_uses_add().startswith("ADD") +assert iadd_path_fallback_uses_iadd().startswith("IADD") + + +## BINARY_SUBSCR_STR_INT: ASCII singleton identity + + +def check_ascii_subscr_singleton_after_warmup(): + s = "abc" + first = None + for i in range(4000): + c = s[0] + if i >= 3500: + if first is None: + first = c + else: + assert c is first + + +check_ascii_subscr_singleton_after_warmup() + + +## BINARY_SUBSCR_STR_INT: Latin-1 singleton identity + + +def check_latin1_subscr_singleton_after_warmup(): + for s in ("abc", "éx"): + first = None + for i in range(5000): + c = s[0] + if i >= 4500: + if first is None: + first = c + else: + assert c is first + + +check_latin1_subscr_singleton_after_warmup() diff --git a/extra_tests/test_manager_fork_debug.py b/extra_tests/test_manager_fork_debug.py new file mode 100644 index 00000000000..6110f7e3699 --- /dev/null +++ b/extra_tests/test_manager_fork_debug.py @@ -0,0 +1,149 @@ +"""Minimal reproduction of multiprocessing Manager + fork failure.""" + +import multiprocessing +import os +import sys +import time +import traceback + +import pytest + +pytestmark = pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork") + + +def test_basic_manager(): + """Test Manager without fork - does it work at all?""" + print("=== Test 1: Basic Manager (no fork) ===") + ctx = multiprocessing.get_context("fork") + manager = ctx.Manager() + try: + ev = manager.Event() + print(f" Event created: {ev}") + ev.set() + print(f" Event set, is_set={ev.is_set()}") + assert ev.is_set() + print(" PASS") + finally: + manager.shutdown() + + +def test_manager_with_process(): + """Test Manager shared between parent and child process.""" + print("\n=== Test 2: Manager with forked child ===") + ctx = multiprocessing.get_context("fork") + manager = ctx.Manager() + try: + result = manager.Value("i", 0) + ev = manager.Event() + + def child_fn(): + try: + ev.set() + result.value = 42 + except Exception as e: + print(f" CHILD ERROR: {e}", file=sys.stderr) + traceback.print_exc() + sys.exit(1) + + print(f" Starting child process...") + process = ctx.Process(target=child_fn) + process.start() + print(f" Waiting for child (pid={process.pid})...") + process.join(timeout=10) + + if process.exitcode != 0: + print(f" FAIL: child exited with code {process.exitcode}") + return False + + print(f" Child done. result={result.value}, event={ev.is_set()}") + assert result.value == 42 + assert ev.is_set() + print(" PASS") + return True + finally: + manager.shutdown() + + +def test_manager_server_alive_after_fork(): + """Test that Manager server survives after forking a child.""" + print("\n=== Test 3: Manager server alive after fork ===") + ctx = multiprocessing.get_context("fork") + manager = ctx.Manager() + try: + ev = manager.Event() + + # Fork a child that does nothing with the manager + pid = os.fork() + if pid == 0: + # Child - exit immediately + os._exit(0) + + # Parent - wait for child + os.waitpid(pid, 0) + + # Now try to use the manager in the parent + print(f" After fork, trying to use Manager in parent...") + ev.set() + print(f" ev.is_set() = {ev.is_set()}") + assert ev.is_set() + print(" PASS") + return True + finally: + manager.shutdown() + + +def test_manager_server_alive_after_fork_with_child_usage(): + """Test that Manager server survives when child also uses it.""" + print("\n=== Test 4: Manager server alive after fork + child usage ===") + ctx = multiprocessing.get_context("fork") + manager = ctx.Manager() + try: + child_ev = manager.Event() + parent_ev = manager.Event() + + def child_fn(): + try: + child_ev.set() + except Exception as e: + print(f" CHILD ERROR: {e}", file=sys.stderr) + traceback.print_exc() + sys.exit(1) + + process = ctx.Process(target=child_fn) + process.start() + process.join(timeout=10) + + if process.exitcode != 0: + print(f" FAIL: child exited with code {process.exitcode}") + return False + + # Now use manager in parent AFTER child is done + print(f" Child done. Trying parent usage...") + parent_ev.set() + print(f" child_ev={child_ev.is_set()}, parent_ev={parent_ev.is_set()}") + assert child_ev.is_set() + assert parent_ev.is_set() + print(" PASS") + return True + finally: + manager.shutdown() + + +if __name__ == "__main__": + test_basic_manager() + + passed = 0 + total = 10 + for i in range(total): + print(f"\n--- Iteration {i + 1}/{total} ---") + ok = True + ok = ok and test_manager_with_process() + ok = ok and test_manager_server_alive_after_fork() + ok = ok and test_manager_server_alive_after_fork_with_child_usage() + if ok: + passed += 1 + else: + print(f" FAILED on iteration {i + 1}") + + print(f"\n=== Results: {passed}/{total} passed ===") + sys.exit(0 if passed == total else 1) diff --git a/extra_tests/test_snippets.py b/extra_tests/test_snippets.py index c191c1e6380..d58b86f66b2 100644 --- a/extra_tests/test_snippets.py +++ b/extra_tests/test_snippets.py @@ -2,14 +2,14 @@ # in the snippets folder. -import sys -import os -import unittest +import contextlib +import enum import glob import logging +import os import subprocess -import contextlib -import enum +import sys +import unittest from pathlib import Path @@ -42,23 +42,27 @@ def perform_test(filename, method, test_type): def run_via_cpython(filename): - """ Simply invoke python itself on the script """ + """Simply invoke python itself on the script""" env = os.environ.copy() subprocess.check_call([sys.executable, filename], env=env) -RUSTPYTHON_BINARY = os.environ.get("RUSTPYTHON") or os.path.join(ROOT_DIR, "target/release/rustpython") + +RUSTPYTHON_BINARY = os.environ.get("RUSTPYTHON") or os.path.join( + ROOT_DIR, "target/release/rustpython" +) RUSTPYTHON_BINARY = os.path.abspath(RUSTPYTHON_BINARY) + def run_via_rustpython(filename, test_type): env = os.environ.copy() - env['RUST_LOG'] = 'info,cargo=error,jobserver=error' - env['RUST_BACKTRACE'] = '1' + env["RUST_LOG"] = "info,cargo=error,jobserver=error" + env["RUST_BACKTRACE"] = "1" subprocess.check_call([RUSTPYTHON_BINARY, filename], env=env) def create_test_function(cls, filename, method, test_type): - """ Create a test function for a single snippet """ + """Create a test function for a single snippet""" core_test_directory, snippet_filename = os.path.split(filename) test_function_name = "test_{}_".format(method) + os.path.splitext(snippet_filename)[ 0 @@ -74,7 +78,7 @@ def test_function(self): def populate(method): def wrapper(cls): - """ Decorator function which can populate a unittest.TestCase class """ + """Decorator function which can populate a unittest.TestCase class""" for test_type, filename in get_test_files(): create_test_function(cls, filename, method, test_type) return cls @@ -83,7 +87,7 @@ def wrapper(cls): def get_test_files(): - """ Retrieve test files """ + """Retrieve test files""" for test_type, test_dir in TEST_DIRS.items(): for filepath in sorted(glob.iglob(os.path.join(test_dir, "*.py"))): filename = os.path.split(filepath)[1] @@ -122,7 +126,9 @@ class SampleTestCase(unittest.TestCase): @classmethod def setUpClass(cls): # Here add resource files - cls.slices_resource_path = Path(TEST_DIRS[_TestType.functional]) / "cpython_generated_slices.py" + cls.slices_resource_path = ( + Path(TEST_DIRS[_TestType.functional]) / "cpython_generated_slices.py" + ) if cls.slices_resource_path.exists(): cls.slices_resource_path.unlink() diff --git a/installer-config/installer.nsi b/installer-config/installer.nsi new file mode 100644 index 00000000000..b68a9fbb593 --- /dev/null +++ b/installer-config/installer.nsi @@ -0,0 +1,671 @@ +; Set the compression algorithm. +!if "{{compression}}" == "" + SetCompressor /SOLID lzma +!else + SetCompressor /SOLID "{{compression}}" +!endif + +Unicode true + +!include MUI2.nsh +!include FileFunc.nsh +!include x64.nsh +!include WordFunc.nsh +!include "FileAssociation.nsh" +!include "StrFunc.nsh" +!include "StrFunc.nsh" +${StrCase} +${StrLoc} + +!define MANUFACTURER "{{manufacturer}}" +!define PRODUCTNAME "{{product_name}}" +!define VERSION "{{version}}" +!define VERSIONWITHBUILD "{{version_with_build}}" +!define SHORTDESCRIPTION "{{short_description}}" +!define INSTALLMODE "{{install_mode}}" +!define LICENSE "{{license}}" +!define INSTALLERICON "{{installer_icon}}" +!define SIDEBARIMAGE "{{sidebar_image}}" +!define HEADERIMAGE "{{header_image}}" +!define MAINBINARYNAME "{{main_binary_name}}" +!define MAINBINARYSRCPATH "{{main_binary_path}}" +!define IDENTIFIER "{{identifier}}" +!define COPYRIGHT "{{copyright}}" +!define OUTFILE "{{out_file}}" +!define ARCH "{{arch}}" +!define PLUGINSPATH "{{additional_plugins_path}}" +!define ALLOWDOWNGRADES "{{allow_downgrades}}" +!define DISPLAYLANGUAGESELECTOR "{{display_language_selector}}" +!define UNINSTKEY "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCTNAME}" +!define MANUPRODUCTKEY "Software\${MANUFACTURER}\${PRODUCTNAME}" +!define UNINSTALLERSIGNCOMMAND "{{uninstaller_sign_cmd}}" +!define ESTIMATEDSIZE "{{estimated_size}}" + +Name "${PRODUCTNAME}" +BrandingText "${COPYRIGHT}" +OutFile "${OUTFILE}" + +VIProductVersion "${VERSIONWITHBUILD}" +VIAddVersionKey "ProductName" "${PRODUCTNAME}" +VIAddVersionKey "FileDescription" "${SHORTDESCRIPTION}" +VIAddVersionKey "LegalCopyright" "${COPYRIGHT}" +VIAddVersionKey "FileVersion" "${VERSION}" +VIAddVersionKey "ProductVersion" "${VERSION}" + +; Plugins path, currently exists for linux only +!if "${PLUGINSPATH}" != "" + !addplugindir "${PLUGINSPATH}" +!endif + +!if "${UNINSTALLERSIGNCOMMAND}" != "" + !uninstfinalize '${UNINSTALLERSIGNCOMMAND}' +!endif + +; Handle install mode, `perUser`, `perMachine` or `both` +!if "${INSTALLMODE}" == "perMachine" + RequestExecutionLevel highest +!endif + +!if "${INSTALLMODE}" == "currentUser" + RequestExecutionLevel user +!endif + +!if "${INSTALLMODE}" == "both" + !define MULTIUSER_MUI + !define MULTIUSER_INSTALLMODE_INSTDIR "${PRODUCTNAME}" + !define MULTIUSER_INSTALLMODE_COMMANDLINE + !if "${ARCH}" == "x64" + !define MULTIUSER_USE_PROGRAMFILES64 + !else if "${ARCH}" == "arm64" + !define MULTIUSER_USE_PROGRAMFILES64 + !endif + !define MULTIUSER_INSTALLMODE_DEFAULT_REGISTRY_KEY "${UNINSTKEY}" + !define MULTIUSER_INSTALLMODE_DEFAULT_REGISTRY_VALUENAME "CurrentUser" + !define MULTIUSER_INSTALLMODEPAGE_SHOWUSERNAME + !define MULTIUSER_INSTALLMODE_FUNCTION RestorePreviousInstallLocation + !define MULTIUSER_EXECUTIONLEVEL Highest + !include MultiUser.nsh +!endif + +; installer icon +!if "${INSTALLERICON}" != "" + !define MUI_ICON "${INSTALLERICON}" +!endif + +; installer sidebar image +!if "${SIDEBARIMAGE}" != "" + !define MUI_WELCOMEFINISHPAGE_BITMAP "${SIDEBARIMAGE}" +!endif + +; installer header image +!if "${HEADERIMAGE}" != "" + !define MUI_HEADERIMAGE + !define MUI_HEADERIMAGE_BITMAP "${HEADERIMAGE}" +!endif + +; Define registry key to store installer language +!define MUI_LANGDLL_REGISTRY_ROOT "HKCU" +!define MUI_LANGDLL_REGISTRY_KEY "${MANUPRODUCTKEY}" +!define MUI_LANGDLL_REGISTRY_VALUENAME "Installer Language" + +; Installer pages, must be ordered as they appear +; 1. Welcome Page +!define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive +!insertmacro MUI_PAGE_WELCOME + +; 2. License Page (if defined) +!if "${LICENSE}" != "" + !define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive + !insertmacro MUI_PAGE_LICENSE "${LICENSE}" +!endif + +; 3. Install mode (if it is set to `both`) +!if "${INSTALLMODE}" == "both" + !define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive + !insertmacro MULTIUSER_PAGE_INSTALLMODE +!endif + + +; 4. Custom page to ask user if he wants to reinstall/uninstall +; only if a previous installtion was detected +Var ReinstallPageCheck +Page custom PageReinstall PageLeaveReinstall +Function PageReinstall + ; Uninstall previous WiX installation if exists. + ; + ; A WiX installer stores the isntallation info in registry + ; using a UUID and so we have to loop through all keys under + ; `HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall` + ; and check if `DisplayName` and `Publisher` keys match ${PRODUCTNAME} and ${MANUFACTURER} + ; + ; This has a potentional issue that there maybe another installation that matches + ; our ${PRODUCTNAME} and ${MANUFACTURER} but wasn't installed by our WiX installer, + ; however, this should be fine since the user will have to confirm the uninstallation + ; and they can chose to abort it if doesn't make sense. + StrCpy $0 0 + wix_loop: + EnumRegKey $1 HKLM "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall" $0 + StrCmp $1 "" wix_done ; Exit loop if there is no more keys to loop on + IntOp $0 $0 + 1 + ReadRegStr $R0 HKLM "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$1" "DisplayName" + ReadRegStr $R1 HKLM "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$1" "Publisher" + StrCmp "$R0$R1" "${PRODUCTNAME}${MANUFACTURER}" 0 wix_loop + ReadRegStr $R0 HKLM "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$1" "UninstallString" + ${StrCase} $R1 $R0 "L" + ${StrLoc} $R0 $R1 "msiexec" ">" + StrCmp $R0 0 0 wix_done + StrCpy $R7 "wix" + StrCpy $R6 "SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\$1" + Goto compare_version + wix_done: + + ; Check if there is an existing installation, if not, abort the reinstall page + ReadRegStr $R0 SHCTX "${UNINSTKEY}" "" + ReadRegStr $R1 SHCTX "${UNINSTKEY}" "UninstallString" + ${IfThen} "$R0$R1" == "" ${|} Abort ${|} + + ; Compare this installar version with the existing installation + ; and modify the messages presented to the user accordingly + compare_version: + StrCpy $R4 "$(older)" + ${If} $R7 == "wix" + ReadRegStr $R0 HKLM "$R6" "DisplayVersion" + ${Else} + ReadRegStr $R0 SHCTX "${UNINSTKEY}" "DisplayVersion" + ${EndIf} + ${IfThen} $R0 == "" ${|} StrCpy $R4 "$(unknown)" ${|} + + nsis_tauri_utils::SemverCompare "${VERSION}" $R0 + Pop $R0 + ; Reinstalling the same version + ${If} $R0 == 0 + StrCpy $R1 "$(alreadyInstalledLong)" + StrCpy $R2 "$(addOrReinstall)" + StrCpy $R3 "$(uninstallApp)" + !insertmacro MUI_HEADER_TEXT "$(alreadyInstalled)" "$(chooseMaintenanceOption)" + StrCpy $R5 "2" + ; Upgrading + ${ElseIf} $R0 == 1 + StrCpy $R1 "$(olderOrUnknownVersionInstalled)" + StrCpy $R2 "$(uninstallBeforeInstalling)" + StrCpy $R3 "$(dontUninstall)" + !insertmacro MUI_HEADER_TEXT "$(alreadyInstalled)" "$(choowHowToInstall)" + StrCpy $R5 "1" + ; Downgrading + ${ElseIf} $R0 == -1 + StrCpy $R1 "$(newerVersionInstalled)" + StrCpy $R2 "$(uninstallBeforeInstalling)" + !if "${ALLOWDOWNGRADES}" == "true" + StrCpy $R3 "$(dontUninstall)" + !else + StrCpy $R3 "$(dontUninstallDowngrade)" + !endif + !insertmacro MUI_HEADER_TEXT "$(alreadyInstalled)" "$(choowHowToInstall)" + StrCpy $R5 "1" + ${Else} + Abort + ${EndIf} + + Call SkipIfPassive + + nsDialogs::Create 1018 + Pop $R4 + ${IfThen} $(^RTL) == 1 ${|} nsDialogs::SetRTL $(^RTL) ${|} + + ${NSD_CreateLabel} 0 0 100% 24u $R1 + Pop $R1 + + ${NSD_CreateRadioButton} 30u 50u -30u 8u $R2 + Pop $R2 + ${NSD_OnClick} $R2 PageReinstallUpdateSelection + + ${NSD_CreateRadioButton} 30u 70u -30u 8u $R3 + Pop $R3 + ; disable this radio button if downgrading and downgrades are disabled + !if "${ALLOWDOWNGRADES}" == "false" + ${IfThen} $R0 == -1 ${|} EnableWindow $R3 0 ${|} + !endif + ${NSD_OnClick} $R3 PageReinstallUpdateSelection + + ; Check the first radio button if this the first time + ; we enter this page or if the second button wasn't + ; selected the last time we were on this page + ${If} $ReinstallPageCheck != 2 + SendMessage $R2 ${BM_SETCHECK} ${BST_CHECKED} 0 + ${Else} + SendMessage $R3 ${BM_SETCHECK} ${BST_CHECKED} 0 + ${EndIf} + + ${NSD_SetFocus} $R2 + nsDialogs::Show +FunctionEnd +Function PageReinstallUpdateSelection + ${NSD_GetState} $R2 $R1 + ${If} $R1 == ${BST_CHECKED} + StrCpy $ReinstallPageCheck 1 + ${Else} + StrCpy $ReinstallPageCheck 2 + ${EndIf} +FunctionEnd +Function PageLeaveReinstall + ${NSD_GetState} $R2 $R1 + + ; $R5 holds whether we are reinstalling the same version or not + ; $R5 == "1" -> different versions + ; $R5 == "2" -> same version + ; + ; $R1 holds the radio buttons state. its meaning is dependant on the context + StrCmp $R5 "1" 0 +2 ; Existing install is not the same version? + StrCmp $R1 "1" reinst_uninstall reinst_done ; $R1 == "1", then user chose to uninstall existing version, otherwise skip uninstalling + StrCmp $R1 "1" reinst_done ; Same version? skip uninstalling + + reinst_uninstall: + HideWindow + ClearErrors + + ${If} $R7 == "wix" + ReadRegStr $R1 HKLM "$R6" "UninstallString" + ExecWait '$R1' $0 + ${Else} + ReadRegStr $4 SHCTX "${MANUPRODUCTKEY}" "" + ReadRegStr $R1 SHCTX "${UNINSTKEY}" "UninstallString" + ExecWait '$R1 /P _?=$4' $0 + ${EndIf} + + BringToFront + + ${IfThen} ${Errors} ${|} StrCpy $0 2 ${|} ; ExecWait failed, set fake exit code + + ${If} $0 <> 0 + ${OrIf} ${FileExists} "$INSTDIR\${MAINBINARYNAME}.exe" + ${If} $0 = 1 ; User aborted uninstaller? + StrCmp $R5 "2" 0 +2 ; Is the existing install the same version? + Quit ; ...yes, already installed, we are done + Abort + ${EndIf} + MessageBox MB_ICONEXCLAMATION "$(unableToUninstall)" + Abort + ${Else} + StrCpy $0 $R1 1 + ${IfThen} $0 == '"' ${|} StrCpy $R1 $R1 -1 1 ${|} ; Strip quotes from UninstallString + Delete $R1 + RMDir $INSTDIR + ${EndIf} + reinst_done: +FunctionEnd + +; 5. Choose install directoy page +!define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive +!insertmacro MUI_PAGE_DIRECTORY + +; 6. Start menu shortcut page +!define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive +Var AppStartMenuFolder +!insertmacro MUI_PAGE_STARTMENU Application $AppStartMenuFolder + +; 7. Installation page +!insertmacro MUI_PAGE_INSTFILES + +; 8. Finish page +; +; Don't auto jump to finish page after installation page, +; because the installation page has useful info that can be used debug any issues with the installer. +!define MUI_FINISHPAGE_NOAUTOCLOSE +; Use show readme button in the finish page as a button create a desktop shortcut +!define MUI_FINISHPAGE_SHOWREADME +!define MUI_FINISHPAGE_SHOWREADME_TEXT "$(createDesktop)" +!define MUI_FINISHPAGE_SHOWREADME_FUNCTION CreateDesktopShortcut +; Show run app after installation. +!define MUI_FINISHPAGE_RUN "$INSTDIR\${MAINBINARYNAME}.exe" +!define MUI_PAGE_CUSTOMFUNCTION_PRE SkipIfPassive +!insertmacro MUI_PAGE_FINISH + +; Uninstaller Pages +; 1. Confirm uninstall page +{{#if appdata_paths}} +Var DeleteAppDataCheckbox +Var DeleteAppDataCheckboxState +!define /ifndef WS_EX_LAYOUTRTL 0x00400000 +!define MUI_PAGE_CUSTOMFUNCTION_SHOW un.ConfirmShow +Function un.ConfirmShow + FindWindow $1 "#32770" "" $HWNDPARENT ; Find inner dialog + ${If} $(^RTL) == 1 + System::Call 'USER32::CreateWindowEx(i${__NSD_CheckBox_EXSTYLE}|${WS_EX_LAYOUTRTL},t"${__NSD_CheckBox_CLASS}",t "$(deleteAppData)",i${__NSD_CheckBox_STYLE},i 50,i 100,i 400, i 25,i$1,i0,i0,i0)i.s' + ${Else} + System::Call 'USER32::CreateWindowEx(i${__NSD_CheckBox_EXSTYLE},t"${__NSD_CheckBox_CLASS}",t "$(deleteAppData)",i${__NSD_CheckBox_STYLE},i 0,i 100,i 400, i 25,i$1,i0,i0,i0)i.s' + ${EndIf} + Pop $DeleteAppDataCheckbox + SendMessage $HWNDPARENT ${WM_GETFONT} 0 0 $1 + SendMessage $DeleteAppDataCheckbox ${WM_SETFONT} $1 1 +FunctionEnd +!define MUI_PAGE_CUSTOMFUNCTION_LEAVE un.ConfirmLeave +Function un.ConfirmLeave + SendMessage $DeleteAppDataCheckbox ${BM_GETCHECK} 0 0 $DeleteAppDataCheckboxState +FunctionEnd +{{/if}} +!insertmacro MUI_UNPAGE_CONFIRM + +; 2. Uninstalling Page +!insertmacro MUI_UNPAGE_INSTFILES + +;Languages +{{#each languages}} +!insertmacro MUI_LANGUAGE "{{this}}" +{{/each}} +!insertmacro MUI_RESERVEFILE_LANGDLL +{{#each language_files}} + !include "{{this}}" +{{/each}} + +!macro SetContext + !if "${INSTALLMODE}" == "currentUser" + SetShellVarContext current + !else if "${INSTALLMODE}" == "perMachine" + SetShellVarContext all + !endif + + ${If} ${RunningX64} + !if "${ARCH}" == "x64" + SetRegView 64 + !else if "${ARCH}" == "arm64" + SetRegView 64 + !else + SetRegView 32 + !endif + ${EndIf} +!macroend + +Var PassiveMode +Function .onInit + ${GetOptions} $CMDLINE "/P" $PassiveMode + IfErrors +2 0 + StrCpy $PassiveMode 1 + + !if "${DISPLAYLANGUAGESELECTOR}" == "true" + !insertmacro MUI_LANGDLL_DISPLAY + !endif + + !insertmacro SetContext + + ${If} $INSTDIR == "" + ; Set default install location + !if "${INSTALLMODE}" == "perMachine" + ${If} ${RunningX64} + !if "${ARCH}" == "x64" + StrCpy $INSTDIR "$PROGRAMFILES64\${PRODUCTNAME}" + !else if "${ARCH}" == "arm64" + StrCpy $INSTDIR "$PROGRAMFILES64\${PRODUCTNAME}" + !else + StrCpy $INSTDIR "$PROGRAMFILES\${PRODUCTNAME}" + !endif + ${Else} + StrCpy $INSTDIR "$PROGRAMFILES\${PRODUCTNAME}" + ${EndIf} + !else if "${INSTALLMODE}" == "currentUser" + StrCpy $INSTDIR "$LOCALAPPDATA\${PRODUCTNAME}" + !endif + + Call RestorePreviousInstallLocation + ${EndIf} + + + !if "${INSTALLMODE}" == "both" + !insertmacro MULTIUSER_INIT + !endif +FunctionEnd + + +Section EarlyChecks + ; Abort silent installer if downgrades is disabled + !if "${ALLOWDOWNGRADES}" == "false" + IfSilent 0 silent_downgrades_done + ; If downgrading + ${If} $R0 == -1 + System::Call 'kernel32::AttachConsole(i -1)i.r0' + ${If} $0 != 0 + System::Call 'kernel32::GetStdHandle(i -11)i.r0' + System::call 'kernel32::SetConsoleTextAttribute(i r0, i 0x0004)' ; set red color + FileWrite $0 "$(silentDowngrades)" + ${EndIf} + Abort + ${EndIf} + silent_downgrades_done: + !endif + +SectionEnd + +{{#if preinstall_section}} +{{unescape_newlines preinstall_section}} +{{/if}} + +!macro CheckIfAppIsRunning + nsis_tauri_utils::FindProcess "${MAINBINARYNAME}.exe" + Pop $R0 + ${If} $R0 = 0 + IfSilent kill 0 + ${IfThen} $PassiveMode != 1 ${|} MessageBox MB_OKCANCEL "$(appRunningOkKill)" IDOK kill IDCANCEL cancel ${|} + kill: + nsis_tauri_utils::KillProcess "${MAINBINARYNAME}.exe" + Pop $R0 + Sleep 500 + ${If} $R0 = 0 + Goto app_check_done + ${Else} + IfSilent silent ui + silent: + System::Call 'kernel32::AttachConsole(i -1)i.r0' + ${If} $0 != 0 + System::Call 'kernel32::GetStdHandle(i -11)i.r0' + System::call 'kernel32::SetConsoleTextAttribute(i r0, i 0x0004)' ; set red color + FileWrite $0 "$(appRunning)$\n" + ${EndIf} + Abort + ui: + Abort "$(failedToKillApp)" + ${EndIf} + cancel: + Abort "$(appRunning)" + ${EndIf} + app_check_done: +!macroend + +Section Install + SetOutPath $INSTDIR + + !insertmacro CheckIfAppIsRunning + + ; Copy main executable + File "${MAINBINARYSRCPATH}" + + ; Create resources directory structure + {{#each resources_dirs}} + CreateDirectory "$INSTDIR\\{{this}}" + {{/each}} + + ; Copy resources + {{#each resources}} + File /a "/oname={{this}}" "{{@key}}" + {{/each}} + + ; Copy external binaries + {{#each binaries}} + File /a "/oname={{this}}" "{{@key}}" + {{/each}} + + ; Create file associations + {{#each file_associations as |association| ~}} + {{#each association.ext as |ext| ~}} + !insertmacro APP_ASSOCIATE "{{ext}}" "{{or association.name ext}}" "{{association-description association.description ext}}" "$INSTDIR\${MAINBINARYNAME}.exe,0" "Open with ${PRODUCTNAME}" "$INSTDIR\${MAINBINARYNAME}.exe $\"%1$\"" + {{/each}} + {{/each}} + + ; Register deep links + {{#each deep_link_protocols as |protocol| ~}} + WriteRegStr SHCTX "Software\Classes\\{{protocol}}" "URL Protocol" "" + WriteRegStr SHCTX "Software\Classes\\{{protocol}}" "" "URL:${BUNDLEID} protocol" + WriteRegStr SHCTX "Software\Classes\\{{protocol}}\DefaultIcon" "" "$\"$INSTDIR\${MAINBINARYNAME}.exe$\",0" + WriteRegStr SHCTX "Software\Classes\\{{protocol}}\shell\open\command" "" "$\"$INSTDIR\${MAINBINARYNAME}.exe$\" $\"%1$\"" + {{/each}} + + ; Create uninstaller + WriteUninstaller "$INSTDIR\uninstall.exe" + + ; Save $INSTDIR in registry for future installations + WriteRegStr SHCTX "${MANUPRODUCTKEY}" "" $INSTDIR + + !if "${INSTALLMODE}" == "both" + ; Save install mode to be selected by default for the next installation such as updating + ; or when uninstalling + WriteRegStr SHCTX "${UNINSTKEY}" $MultiUser.InstallMode 1 + !endif + + ; Registry information for add/remove programs + WriteRegStr SHCTX "${UNINSTKEY}" "DisplayName" "${PRODUCTNAME}" + WriteRegStr SHCTX "${UNINSTKEY}" "DisplayIcon" "$\"$INSTDIR\${MAINBINARYNAME}.exe$\"" + WriteRegStr SHCTX "${UNINSTKEY}" "DisplayVersion" "${VERSION}" + WriteRegStr SHCTX "${UNINSTKEY}" "Publisher" "${MANUFACTURER}" + WriteRegStr SHCTX "${UNINSTKEY}" "InstallLocation" "$\"$INSTDIR$\"" + WriteRegStr SHCTX "${UNINSTKEY}" "UninstallString" "$\"$INSTDIR\uninstall.exe$\"" + WriteRegDWORD SHCTX "${UNINSTKEY}" "NoModify" "1" + WriteRegDWORD SHCTX "${UNINSTKEY}" "NoRepair" "1" + WriteRegDWORD SHCTX "${UNINSTKEY}" "EstimatedSize" "${ESTIMATEDSIZE}" + + ; Create start menu shortcut (GUI) + !insertmacro MUI_STARTMENU_WRITE_BEGIN Application + Call CreateStartMenuShortcut + !insertmacro MUI_STARTMENU_WRITE_END + + ; Create shortcuts for silent and passive installers, which + ; can be disabled by passing `/NS` flag + ; GUI installer has buttons for users to control creating them + IfSilent check_ns_flag 0 + ${IfThen} $PassiveMode == 1 ${|} Goto check_ns_flag ${|} + Goto shortcuts_done + check_ns_flag: + ${GetOptions} $CMDLINE "/NS" $R0 + IfErrors 0 shortcuts_done + Call CreateDesktopShortcut + Call CreateStartMenuShortcut + shortcuts_done: + + ; Auto close this page for passive mode + ${IfThen} $PassiveMode == 1 ${|} SetAutoClose true ${|} +SectionEnd + +Function .onInstSuccess + ; Check for `/R` flag only in silent and passive installers because + ; GUI installer has a toggle for the user to (re)start the app + IfSilent check_r_flag 0 + ${IfThen} $PassiveMode == 1 ${|} Goto check_r_flag ${|} + Goto run_done + check_r_flag: + ${GetOptions} $CMDLINE "/R" $R0 + IfErrors run_done 0 + Exec '"$INSTDIR\${MAINBINARYNAME}.exe"' + run_done: +FunctionEnd + +Function un.onInit + !insertmacro SetContext + + !if "${INSTALLMODE}" == "both" + !insertmacro MULTIUSER_UNINIT + !endif + + !insertmacro MUI_UNGETLANGUAGE +FunctionEnd + +Section Uninstall + !insertmacro CheckIfAppIsRunning + + ; Delete the app directory and its content from disk + ; Copy main executable + Delete "$INSTDIR\${MAINBINARYNAME}.exe" + + ; Delete resources + {{#each resources}} + Delete "$INSTDIR\\{{this}}" + {{/each}} + + ; Delete external binaries + {{#each binaries}} + Delete "$INSTDIR\\{{this}}" + {{/each}} + + ; Delete app associations + {{#each file_associations as |association| ~}} + {{#each association.ext as |ext| ~}} + !insertmacro APP_UNASSOCIATE "{{ext}}" "{{or association.name ext}}" + {{/each}} + {{/each}} + + ; Delete deep links + {{#each deep_link_protocols as |protocol| ~}} + ReadRegStr $R7 SHCTX "Software\Classes\\{{protocol}}\shell\open\command" "" + !if $R7 == "$\"$INSTDIR\${MAINBINARYNAME}.exe$\" $\"%1$\"" + DeleteRegKey SHCTX "Software\Classes\\{{protocol}}" + !endif + {{/each}} + + ; Delete uninstaller + Delete "$INSTDIR\uninstall.exe" + + {{#each resources_dirs}} + RMDir /REBOOTOK "$INSTDIR\\{{this}}" + {{/each}} + RMDir "$INSTDIR" + + ; Remove start menu shortcut + !insertmacro MUI_STARTMENU_GETFOLDER Application $AppStartMenuFolder + Delete "$SMPROGRAMS\$AppStartMenuFolder\${PRODUCTNAME}.lnk" + RMDir "$SMPROGRAMS\$AppStartMenuFolder" + + ; Remove desktop shortcuts + Delete "$DESKTOP\${PRODUCTNAME}.lnk" + + ; Remove registry information for add/remove programs + !if "${INSTALLMODE}" == "both" + DeleteRegKey SHCTX "${UNINSTKEY}" + !else if "${INSTALLMODE}" == "perMachine" + DeleteRegKey HKLM "${UNINSTKEY}" + !else + DeleteRegKey HKCU "${UNINSTKEY}" + !endif + + DeleteRegValue HKCU "${MANUPRODUCTKEY}" "Installer Language" + + ; Delete app data + {{#if appdata_paths}} + ${If} $DeleteAppDataCheckboxState == 1 + SetShellVarContext current + {{#each appdata_paths}} + RmDir /r "{{unescape_dollar_sign this}}" + {{/each}} + ${EndIf} + {{/if}} + + ${GetOptions} $CMDLINE "/P" $R0 + IfErrors +2 0 + SetAutoClose true +SectionEnd + +Function RestorePreviousInstallLocation + ReadRegStr $4 SHCTX "${MANUPRODUCTKEY}" "" + StrCmp $4 "" +2 0 + StrCpy $INSTDIR $4 +FunctionEnd + +Function SkipIfPassive + ${IfThen} $PassiveMode == 1 ${|} Abort ${|} +FunctionEnd + +Function CreateDesktopShortcut + CreateShortcut "$DESKTOP\${PRODUCTNAME}.lnk" "$INSTDIR\${MAINBINARYNAME}.exe" + ApplicationID::Set "$DESKTOP\${PRODUCTNAME}.lnk" "${IDENTIFIER}" +FunctionEnd + +Function CreateStartMenuShortcut + CreateDirectory "$SMPROGRAMS\$AppStartMenuFolder" + CreateShortcut "$SMPROGRAMS\$AppStartMenuFolder\${PRODUCTNAME}.lnk" "$INSTDIR\${MAINBINARYNAME}.exe" + ApplicationID::Set "$SMPROGRAMS\$AppStartMenuFolder\${PRODUCTNAME}.lnk" "${IDENTIFIER}" +FunctionEnd \ No newline at end of file diff --git a/installer-config/installer.wxs b/installer-config/installer.wxs new file mode 100644 index 00000000000..1fdc77e01ae --- /dev/null +++ b/installer-config/installer.wxs @@ -0,0 +1,317 @@ +<?if $(sys.BUILDARCH)="x86"?> + <?define Win64 = "no" ?> + <?define PlatformProgramFilesFolder = "ProgramFilesFolder" ?> +<?elseif $(sys.BUILDARCH)="x64"?> + <?define Win64 = "yes" ?> + <?define PlatformProgramFilesFolder = "ProgramFiles64Folder" ?> +<?else?> + <?error Unsupported value of sys.BUILDARCH=$(sys.BUILDARCH)?> +<?endif?> + +<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"> + <Product + Id="*" + Name="{{product_name}}" + UpgradeCode="{{upgrade_code}}" + Language="!(loc.CargoPackagerLanguage)" + Manufacturer="{{manufacturer}}" + Version="{{version}}"> + + <Package Id="*" + Keywords="Installer" + InstallerVersion="450" + Languages="0" + Compressed="yes" + InstallScope="perMachine" + SummaryCodepage="!(loc.CargoPackagerCodepage)"/> + + <!-- https://docs.microsoft.com/en-us/windows/win32/msi/reinstallmode --> + <!-- reinstall all files; rewrite all registry entries; reinstall all shortcuts --> + <Property Id="REINSTALLMODE" Value="amus" /> + + {{#if allow_downgrades}} + <MajorUpgrade Schedule="afterInstallInitialize" AllowDowngrades="yes" /> + {{else}} + <MajorUpgrade Schedule="afterInstallInitialize" DowngradeErrorMessage="!(loc.DowngradeErrorMessage)" AllowSameVersionUpgrades="yes" /> + {{/if}} + + <InstallExecuteSequence> + <RemoveShortcuts>Installed AND NOT UPGRADINGPRODUCTCODE</RemoveShortcuts> + </InstallExecuteSequence> + + <Media Id="1" Cabinet="app.cab" EmbedCab="yes" /> + + {{#if banner_path}} + <WixVariable Id="WixUIBannerBmp" Value="{{banner_path}}" /> + {{/if}} + {{#if dialog_image_path}} + <WixVariable Id="WixUIDialogBmp" Value="{{dialog_image_path}}" /> + {{/if}} + {{#if license}} + <WixVariable Id="WixUILicenseRtf" Value="{{license}}" /> + {{/if}} + + {{#if icon_path}} + <Icon Id="ProductIcon" SourceFile="{{icon_path}}"/> + <Property Id="ARPPRODUCTICON" Value="ProductIcon" /> + {{/if}} + <Property Id="ARPNOREPAIR" Value="yes" Secure="yes" /> <!-- Remove repair --> + <SetProperty Id="ARPNOMODIFY" Value="1" After="InstallValidate" Sequence="execute"/> + + <!-- initialize with previous InstallDir --> + <Property Id="INSTALLDIR"> + <RegistrySearch Id="PrevInstallDirReg" Root="HKCU" Key="Software\\{{manufacturer}}\\{{product_name}}" Name="InstallDir" Type="raw"/> + </Property> + + <!-- launch app checkbox --> + <Property Id="WIXUI_EXITDIALOGOPTIONALCHECKBOXTEXT" Value="!(loc.LaunchApp)" /> + <Property Id="WIXUI_EXITDIALOGOPTIONALCHECKBOX" Value="1"/> + <Property Id="WixShellExecTarget" Value="[!Path]" /> + <CustomAction Id="LaunchApplication" BinaryKey="WixCA" DllEntry="WixShellExec" Impersonate="yes" /> + + <UI> + <!-- launch app checkbox --> + <Publish Dialog="ExitDialog" Control="Finish" Event="DoAction" Value="LaunchApplication">WIXUI_EXITDIALOGOPTIONALCHECKBOX = 1 and NOT Installed</Publish> + + <Property Id="WIXUI_INSTALLDIR" Value="INSTALLDIR" /> + + {{#unless license}} + <!-- Skip license dialog --> + <Publish Dialog="WelcomeDlg" + Control="Next" + Event="NewDialog" + Value="InstallDirDlg" + Order="2">1</Publish> + <Publish Dialog="InstallDirDlg" + Control="Back" + Event="NewDialog" + Value="WelcomeDlg" + Order="2">1</Publish> + {{/unless}} + </UI> + + <UIRef Id="WixUI_InstallDir" /> + + <Directory Id="TARGETDIR" Name="SourceDir"> + <Directory Id="DesktopFolder" Name="Desktop"> + <Component Id="ApplicationShortcutDesktop" Guid="*"> + <Shortcut Id="ApplicationDesktopShortcut" Name="{{product_name}}" Description="Runs {{product_name}}" Target="[!Path]" WorkingDirectory="INSTALLDIR" /> + <RemoveFolder Id="DesktopFolder" On="uninstall" /> + <RegistryValue Root="HKCU" Key="Software\\{{manufacturer}}\\{{product_name}}" Name="Desktop Shortcut" Type="integer" Value="1" KeyPath="yes" /> + </Component> + </Directory> + <Directory Id="$(var.PlatformProgramFilesFolder)" Name="PFiles"> + <Directory Id="INSTALLDIR" Name="{{product_name}}"/> + </Directory> + <Directory Id="ProgramMenuFolder"> + <Directory Id="ApplicationProgramsFolder" Name="{{product_name}}"/> + </Directory> + + <Component Id="PathEnvPerMachine" Guid="*"> + <Condition>ALLUSERS=1 OR (ALLUSERS=2 AND Privileged)</Condition> + <RegistryValue Root="HKLM" Key="Software\[Manufacturer]\[ProductName]" Name="InstallDir" Type="string" Value="[APPLICATIONFOLDER]" KeyPath="yes" /> + <!-- [APPLICATIONFOLDER] contains trailing backslash --> + <Environment Id="PathPerMachine" Name="PATH" Value="[APPLICATIONFOLDER]bin" Permanent="no" Part="last" Action="set" System="yes" /> + </Component> + <Component Id="PathEnvPerUser" Guid="*"> + <Condition>ALLUSERS="" OR (ALLUSERS=2 AND (NOT Privileged))</Condition> + <RegistryValue Root="HKCU" Key="Software\[Manufacturer]\[ProductName]" Name="InstallDir" Type="string" Value="[APPLICATIONFOLDER]" KeyPath="yes" /> + <Environment Id="PathPerUser" Name="PATH" Value="[APPLICATIONFOLDER]bin" Permanent="no" Part="last" Action="set" System="no" /> + </Component> + </Directory> + + <DirectoryRef Id="INSTALLDIR"> + <Component Id="RegistryEntries" Guid="*"> + <RegistryKey Root="HKCU" Key="Software\\{{manufacturer}}\\{{product_name}}"> + <RegistryValue Name="InstallDir" Type="string" Value="[INSTALLDIR]" KeyPath="yes" /> + </RegistryKey> + <!-- Change the Root to HKCU for perUser installations --> + {{#each deep_link_protocols as |protocol| ~}} + <RegistryKey Root="HKLM" Key="Software\Classes\\{{protocol}}"> + <RegistryValue Type="string" Name="URL Protocol" Value=""/> + <RegistryValue Type="string" Value="URL:{{bundle_id}} protocol"/> + <RegistryKey Key="DefaultIcon"> + <RegistryValue Type="string" Value="&quot;[!Path]&quot;,0" /> + </RegistryKey> + <RegistryKey Key="shell\open\command"> + <RegistryValue Type="string" Value="&quot;[!Path]&quot; &quot;%1&quot;" /> + </RegistryKey> + </RegistryKey> + {{/each~}} + </Component> + <Component Id="Path" Guid="{{path_component_guid}}" Win64="$(var.Win64)"> + <File Id="Path" Source="{{app_exe_source}}" KeyPath="yes" Checksum="yes"/> + {{#each file_associations as |association| ~}} + {{#each association.ext as |ext| ~}} + <ProgId Id="{{../../product_name}}.{{ext}}" Advertise="yes" Description="{{association.description}}"> + <Extension Id="{{ext}}" Advertise="yes"> + <Verb Id="open" Command="Open with {{../../product_name}}" Argument="&quot;%1&quot;" /> + </Extension> + </ProgId> + {{/each~}} + {{/each~}} + </Component> + {{#each binaries as |bin| ~}} + <Component Id="{{ bin.id }}" Guid="{{bin.guid}}" Win64="$(var.Win64)"> + <File Id="Bin_{{ bin.id }}" Source="{{bin.path}}" KeyPath="yes"/> + </Component> + {{/each~}} + {{resources}} + <Component Id="CMP_UninstallShortcut" Guid="*"> + + <Shortcut Id="UninstallShortcut" + Name="Uninstall {{product_name}}" + Description="Uninstalls {{product_name}}" + Target="[System64Folder]msiexec.exe" + Arguments="/x [ProductCode]" /> + + <RemoveFolder Id="INSTALLDIR" + On="uninstall" /> + + <RegistryValue Root="HKCU" + Key="Software\\{{manufacturer}}\\{{product_name}}" + Name="Uninstaller Shortcut" + Type="integer" + Value="1" + KeyPath="yes" /> + </Component> + </DirectoryRef> + + <DirectoryRef Id="ApplicationProgramsFolder"> + <Component Id="ApplicationShortcut" Guid="*"> + <Shortcut Id="ApplicationStartMenuShortcut" + Name="{{product_name}}" + Description="Runs {{product_name}}" + Target="[!Path]" + {{#if icon_path}} + Icon="ProductIcon" + {{/if}} + WorkingDirectory="INSTALLDIR"> + <ShortcutProperty Key="System.AppUserModel.ID" Value="{{identifier}}"/> + </Shortcut> + <RemoveFolder Id="ApplicationProgramsFolder" On="uninstall"/> + <RegistryValue Root="HKCU" Key="Software\\{{manufacturer}}\\{{product_name}}" Name="Start Menu Shortcut" Type="integer" Value="1" KeyPath="yes"/> + </Component> + </DirectoryRef> + + {{#each merge_modules as |msm| ~}} + <DirectoryRef Id="TARGETDIR"> + <Merge Id="{{ msm.name }}" SourceFile="{{ msm.path }}" DiskId="1" Language="!(loc.CargoPackagerLanguage)" /> + </DirectoryRef> + + <Feature Id="{{ msm.name }}" Title="{{ msm.name }}" AllowAdvertise="no" Display="hidden" Level="1"> + <MergeRef Id="{{ msm.name }}"/> + </Feature> + {{/each~}} + + <Feature + Id="MainProgram" + Title="Application" + Description="!(loc.InstallAppFeature)" + Level="1" + ConfigurableDirectory="INSTALLDIR" + AllowAdvertise="no" + Display="expand" + Absent="disallow"> + + <ComponentRef Id="RegistryEntries"/> + + {{#each resource_file_ids as |resource_file_id| ~}} + <ComponentRef Id="{{ resource_file_id }}"/> + {{/each~}} + <Feature Id="ShortcutsFeature" + Title="Shortcuts" + Level="1"> + <ComponentRef Id="Path"/> + <ComponentRef Id="CMP_UninstallShortcut" /> + <ComponentRef Id="ApplicationShortcut" /> + <ComponentRef Id="ApplicationShortcutDesktop" /> + </Feature> + + <Feature + Id="Environment" + Title="PATH Environment Variable" + Description="!(loc.PathEnvVarFeature)" + Level="1" + Absent="allow"> + <ComponentRef Id="Path"/> + {{#each binaries as |bin| ~}} + <ComponentRef Id="{{ bin.id }}"/> + {{/each~}} + </Feature> + </Feature> + + <Feature Id="Path" + Title="Add to PATH" + Description="Add to PATH environment variable" + Level="1" + AllowAdvertise="no"> + <ComponentRef Id="PathEnvPerMachine" /> + <ComponentRef Id="PathEnvPerUser" /> + </Feature> + + <Feature Id="External" AllowAdvertise="no" Absent="disallow"> + {{#each component_group_refs as |id| ~}} + <ComponentGroupRef Id="{{ id }}"/> + {{/each~}} + {{#each component_refs as |id| ~}} + <ComponentRef Id="{{ id }}"/> + {{/each~}} + {{#each feature_group_refs as |id| ~}} + <FeatureGroupRef Id="{{ id }}"/> + {{/each~}} + {{#each feature_refs as |id| ~}} + <FeatureRef Id="{{ id }}"/> + {{/each~}} + {{#each merge_refs as |id| ~}} + <MergeRef Id="{{ id }}"/> + {{/each~}} + </Feature> + + {{#each custom_action_refs as |id| ~}} + <CustomActionRef Id="{{ id }}"/> + {{/each~}} + + {{#if install_webview}} + <!-- WebView2 --> + <Property Id="WVRTINSTALLED"> + <RegistrySearch Id="WVRTInstalledSystem" Root="HKLM" Key="SOFTWARE\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}" Name="pv" Type="raw" Win64="no" /> + <RegistrySearch Id="WVRTInstalledUser" Root="HKCU" Key="SOFTWARE\Microsoft\EdgeUpdate\Clients\{F3017226-FE2A-4295-8BDF-00C3A9A7E4C5}" Name="pv" Type="raw"/> + </Property> + + {{#if download_bootstrapper}} + <CustomAction Id='DownloadAndInvokeBootstrapper' Directory="INSTALLDIR" Execute="deferred" ExeCommand='powershell.exe -NoProfile -windowstyle hidden try [\{] [\[]Net.ServicePointManager[\]]::SecurityProtocol = [\[]Net.SecurityProtocolType[\]]::Tls12 [\}] catch [\{][\}]; Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?LinkId=2124703" -OutFile "$env:TEMP\MicrosoftEdgeWebview2Setup.exe" ; Start-Process -FilePath "$env:TEMP\MicrosoftEdgeWebview2Setup.exe" -ArgumentList ({{webview_installer_args}} &apos;/install&apos;) -Wait' Return='check'/> + <InstallExecuteSequence> + <Custom Action='DownloadAndInvokeBootstrapper' Before='InstallFinalize'> + <![CDATA[NOT(REMOVE OR WVRTINSTALLED)]]> + </Custom> + </InstallExecuteSequence> + {{/if}} + + <!-- Embedded webview bootstrapper mode --> + {{#if webview2_bootstrapper_path}} + <Binary Id="MicrosoftEdgeWebview2Setup.exe" SourceFile="{{webview2_bootstrapper_path}}"/> + <CustomAction Id='InvokeBootstrapper' BinaryKey='MicrosoftEdgeWebview2Setup.exe' Execute="deferred" ExeCommand='{{webview_installer_args}} /install' Return='check' /> + <InstallExecuteSequence> + <Custom Action='InvokeBootstrapper' Before='InstallFinalize'> + <![CDATA[NOT(REMOVE OR WVRTINSTALLED)]]> + </Custom> + </InstallExecuteSequence> + {{/if}} + + <!-- Embedded offline installer --> + {{#if webview2_installer_path}} + <Binary Id="MicrosoftEdgeWebView2RuntimeInstaller.exe" SourceFile="{{webview2_installer_path}}"/> + <CustomAction Id='InvokeStandalone' BinaryKey='MicrosoftEdgeWebView2RuntimeInstaller.exe' Execute="deferred" ExeCommand='{{webview_installer_args}} /install' Return='check' /> + <InstallExecuteSequence> + <Custom Action='InvokeStandalone' Before='InstallFinalize'> + <![CDATA[NOT(REMOVE OR WVRTINSTALLED)]]> + </Custom> + </InstallExecuteSequence> + {{/if}} + + {{/if}} + + <SetProperty Id="ARPINSTALLLOCATION" Value="[INSTALLDIR]" After="CostFinalize"/> + </Product> +</Wix> \ No newline at end of file diff --git a/jit/Cargo.toml b/jit/Cargo.toml deleted file mode 100644 index 71f8822cc9c..00000000000 --- a/jit/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "rustpython-jit" -version = "0.3.0" -description = "Experimental JIT(just in time) compiler for python code." -authors = ["RustPython Team"] -repository = "https://github.com/RustPython/RustPython" -license = "MIT" -edition = "2021" - -autotests = false - -[dependencies] -rustpython-compiler-core = { workspace = true } - -num-traits = { workspace = true } -thiserror = { workspace = true } - -cranelift = "0.88.0" -cranelift-jit = "0.88.0" -cranelift-module = "0.88.0" -libffi = "3.1.0" - -[dev-dependencies] -rustpython-derive = { path = "../derive", version = "0.3.0" } - -approx = "0.5.1" - -[[test]] -name = "integration" -path = "tests/lib.rs" diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs deleted file mode 100644 index ee1cc04875f..00000000000 --- a/jit/src/instructions.rs +++ /dev/null @@ -1,524 +0,0 @@ -use cranelift::prelude::*; -use num_traits::cast::ToPrimitive; -use rustpython_compiler_core::bytecode::{ - self, BinaryOperator, BorrowedConstant, CodeObject, ComparisonOperator, Instruction, Label, - OpArg, OpArgState, UnaryOperator, -}; -use std::collections::HashMap; - -use super::{JitCompileError, JitSig, JitType}; - -#[repr(u16)] -enum CustomTrapCode { - /// Raised when shifting by a negative number - NegativeShiftCount = 0, -} - -#[derive(Clone)] -struct Local { - var: Variable, - ty: JitType, -} - -#[derive(Debug)] -enum JitValue { - Int(Value), - Float(Value), - Bool(Value), - None, - Tuple(Vec<JitValue>), -} - -impl JitValue { - fn from_type_and_value(ty: JitType, val: Value) -> JitValue { - match ty { - JitType::Int => JitValue::Int(val), - JitType::Float => JitValue::Float(val), - JitType::Bool => JitValue::Bool(val), - } - } - - fn to_jit_type(&self) -> Option<JitType> { - match self { - JitValue::Int(_) => Some(JitType::Int), - JitValue::Float(_) => Some(JitType::Float), - JitValue::Bool(_) => Some(JitType::Bool), - JitValue::None | JitValue::Tuple(_) => None, - } - } - - fn into_value(self) -> Option<Value> { - match self { - JitValue::Int(val) | JitValue::Float(val) | JitValue::Bool(val) => Some(val), - JitValue::None | JitValue::Tuple(_) => None, - } - } -} - -pub struct FunctionCompiler<'a, 'b> { - builder: &'a mut FunctionBuilder<'b>, - stack: Vec<JitValue>, - variables: Box<[Option<Local>]>, - label_to_block: HashMap<Label, Block>, - pub(crate) sig: JitSig, -} - -impl<'a, 'b> FunctionCompiler<'a, 'b> { - pub fn new( - builder: &'a mut FunctionBuilder<'b>, - num_variables: usize, - arg_types: &[JitType], - entry_block: Block, - ) -> FunctionCompiler<'a, 'b> { - let mut compiler = FunctionCompiler { - builder, - stack: Vec::new(), - variables: vec![None; num_variables].into_boxed_slice(), - label_to_block: HashMap::new(), - sig: JitSig { - args: arg_types.to_vec(), - ret: None, - }, - }; - let params = compiler.builder.func.dfg.block_params(entry_block).to_vec(); - for (i, (ty, val)) in arg_types.iter().zip(params).enumerate() { - compiler - .store_variable(i as u32, JitValue::from_type_and_value(ty.clone(), val)) - .unwrap(); - } - compiler - } - - fn pop_multiple(&mut self, count: usize) -> Vec<JitValue> { - let stack_len = self.stack.len(); - self.stack.drain(stack_len - count..).collect() - } - - fn store_variable( - &mut self, - idx: bytecode::NameIdx, - val: JitValue, - ) -> Result<(), JitCompileError> { - let builder = &mut self.builder; - let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; - let local = self.variables[idx as usize].get_or_insert_with(|| { - let var = Variable::new(idx as usize); - let local = Local { - var, - ty: ty.clone(), - }; - builder.declare_var(var, ty.to_cranelift()); - local - }); - if ty != local.ty { - Err(JitCompileError::NotSupported) - } else { - self.builder.def_var(local.var, val.into_value().unwrap()); - Ok(()) - } - } - - fn boolean_val(&mut self, val: JitValue) -> Result<Value, JitCompileError> { - match val { - JitValue::Float(val) => { - let zero = self.builder.ins().f64const(0); - let val = self.builder.ins().fcmp(FloatCC::NotEqual, val, zero); - Ok(self.builder.ins().bint(types::I8, val)) - } - JitValue::Int(val) => { - let zero = self.builder.ins().iconst(types::I64, 0); - let val = self.builder.ins().icmp(IntCC::NotEqual, val, zero); - Ok(self.builder.ins().bint(types::I8, val)) - } - JitValue::Bool(val) => Ok(val), - JitValue::None => Ok(self.builder.ins().iconst(types::I8, 0)), - JitValue::Tuple(_) => Err(JitCompileError::NotSupported), - } - } - - fn get_or_create_block(&mut self, label: Label) -> Block { - let builder = &mut self.builder; - *self - .label_to_block - .entry(label) - .or_insert_with(|| builder.create_block()) - } - - pub fn compile<C: bytecode::Constant>( - &mut self, - bytecode: &CodeObject<C>, - ) -> Result<(), JitCompileError> { - // TODO: figure out if this is sufficient -- previously individual labels were associated - // pretty much per-bytecode that uses them, or at least per "type" of block -- in theory an - // if block and a with block might jump to the same place. Now it's all "flattened", so - // there might be less distinction between different types of blocks going off - // label_targets alone - let label_targets = bytecode.label_targets(); - - let mut arg_state = OpArgState::default(); - for (offset, instruction) in bytecode.instructions.iter().enumerate() { - let (instruction, arg) = arg_state.get(*instruction); - let label = Label(offset as u32); - if label_targets.contains(&label) { - let block = self.get_or_create_block(label); - - // If the current block is not terminated/filled just jump - // into the new block. - if !self.builder.is_filled() { - self.builder.ins().jump(block, &[]); - } - - self.builder.switch_to_block(block); - } - - // Sometimes the bytecode contains instructions after a return - // just ignore those until we are at the next label - if self.builder.is_filled() { - continue; - } - - self.add_instruction(instruction, arg, &bytecode.constants)?; - } - - Ok(()) - } - - fn load_const<C: bytecode::Constant>( - &mut self, - constant: BorrowedConstant<C>, - ) -> Result<(), JitCompileError> { - match constant { - BorrowedConstant::Integer { value } => { - let val = self.builder.ins().iconst( - types::I64, - value.to_i64().ok_or(JitCompileError::NotSupported)?, - ); - self.stack.push(JitValue::Int(val)); - Ok(()) - } - BorrowedConstant::Float { value } => { - let val = self.builder.ins().f64const(value); - self.stack.push(JitValue::Float(val)); - Ok(()) - } - BorrowedConstant::Boolean { value } => { - let val = self.builder.ins().iconst(types::I8, value as i64); - self.stack.push(JitValue::Bool(val)); - Ok(()) - } - BorrowedConstant::None => { - self.stack.push(JitValue::None); - Ok(()) - } - _ => Err(JitCompileError::NotSupported), - } - } - - pub fn add_instruction<C: bytecode::Constant>( - &mut self, - instruction: Instruction, - arg: OpArg, - constants: &[C], - ) -> Result<(), JitCompileError> { - match instruction { - Instruction::ExtendedArg => Ok(()), - Instruction::JumpIfFalse { target } => { - let cond = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - - let val = self.boolean_val(cond)?; - let then_block = self.get_or_create_block(target.get(arg)); - self.builder.ins().brz(val, then_block, &[]); - - let block = self.builder.create_block(); - self.builder.ins().jump(block, &[]); - self.builder.switch_to_block(block); - - Ok(()) - } - Instruction::JumpIfTrue { target } => { - let cond = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - - let val = self.boolean_val(cond)?; - let then_block = self.get_or_create_block(target.get(arg)); - self.builder.ins().brnz(val, then_block, &[]); - - let block = self.builder.create_block(); - self.builder.ins().jump(block, &[]); - self.builder.switch_to_block(block); - - Ok(()) - } - Instruction::Jump { target } => { - let target_block = self.get_or_create_block(target.get(arg)); - self.builder.ins().jump(target_block, &[]); - - Ok(()) - } - Instruction::LoadFast(idx) => { - let local = self.variables[idx.get(arg) as usize] - .as_ref() - .ok_or(JitCompileError::BadBytecode)?; - self.stack.push(JitValue::from_type_and_value( - local.ty.clone(), - self.builder.use_var(local.var), - )); - Ok(()) - } - Instruction::StoreFast(idx) => { - let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - self.store_variable(idx.get(arg), val) - } - Instruction::LoadConst { idx } => { - self.load_const(constants[idx.get(arg) as usize].borrow_constant()) - } - Instruction::BuildTuple { size } => { - let elements = self.pop_multiple(size.get(arg) as usize); - self.stack.push(JitValue::Tuple(elements)); - Ok(()) - } - Instruction::UnpackSequence { size } => { - let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - - let elements = match val { - JitValue::Tuple(elements) => elements, - _ => return Err(JitCompileError::NotSupported), - }; - - if elements.len() != size.get(arg) as usize { - return Err(JitCompileError::NotSupported); - } - - self.stack.extend(elements.into_iter().rev()); - Ok(()) - } - Instruction::ReturnValue => { - let val = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - if let Some(ref ty) = self.sig.ret { - if val.to_jit_type().as_ref() != Some(ty) { - return Err(JitCompileError::NotSupported); - } - } else { - let ty = val.to_jit_type().ok_or(JitCompileError::NotSupported)?; - self.sig.ret = Some(ty.clone()); - self.builder - .func - .signature - .returns - .push(AbiParam::new(ty.to_cranelift())); - } - self.builder.ins().return_(&[val.into_value().unwrap()]); - Ok(()) - } - Instruction::CompareOperation { op, .. } => { - let op = op.get(arg); - // the rhs is popped off first - let b = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - let a = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - - let a_type: Option<JitType> = a.to_jit_type(); - let b_type: Option<JitType> = b.to_jit_type(); - - match (a, b) { - (JitValue::Int(a), JitValue::Int(b)) - | (JitValue::Bool(a), JitValue::Bool(b)) - | (JitValue::Bool(a), JitValue::Int(b)) - | (JitValue::Int(a), JitValue::Bool(b)) => { - let operand_one = match a_type.unwrap() { - JitType::Bool => self.builder.ins().uextend(types::I64, a), - _ => a, - }; - - let operand_two = match b_type.unwrap() { - JitType::Bool => self.builder.ins().uextend(types::I64, b), - _ => b, - }; - - let cond = match op { - ComparisonOperator::Equal => IntCC::Equal, - ComparisonOperator::NotEqual => IntCC::NotEqual, - ComparisonOperator::Less => IntCC::SignedLessThan, - ComparisonOperator::LessOrEqual => IntCC::SignedLessThanOrEqual, - ComparisonOperator::Greater => IntCC::SignedGreaterThan, - ComparisonOperator::GreaterOrEqual => IntCC::SignedGreaterThanOrEqual, - }; - - let val = self.builder.ins().icmp(cond, operand_one, operand_two); - // TODO: Remove this `bint` in cranelift 0.90 as icmp now returns i8 - self.stack - .push(JitValue::Bool(self.builder.ins().bint(types::I8, val))); - Ok(()) - } - (JitValue::Float(a), JitValue::Float(b)) => { - let cond = match op { - ComparisonOperator::Equal => FloatCC::Equal, - ComparisonOperator::NotEqual => FloatCC::NotEqual, - ComparisonOperator::Less => FloatCC::LessThan, - ComparisonOperator::LessOrEqual => FloatCC::LessThanOrEqual, - ComparisonOperator::Greater => FloatCC::GreaterThan, - ComparisonOperator::GreaterOrEqual => FloatCC::GreaterThanOrEqual, - }; - - let val = self.builder.ins().fcmp(cond, a, b); - // TODO: Remove this `bint` in cranelift 0.90 as fcmp now returns i8 - self.stack - .push(JitValue::Bool(self.builder.ins().bint(types::I8, val))); - Ok(()) - } - _ => Err(JitCompileError::NotSupported), - } - } - Instruction::UnaryOperation { op, .. } => { - let op = op.get(arg); - let a = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - match (op, a) { - (UnaryOperator::Minus, JitValue::Int(val)) => { - // Compile minus as 0 - a. - let zero = self.builder.ins().iconst(types::I64, 0); - let out = self.compile_sub(zero, val); - self.stack.push(JitValue::Int(out)); - Ok(()) - } - (UnaryOperator::Plus, JitValue::Int(val)) => { - // Nothing to do - self.stack.push(JitValue::Int(val)); - Ok(()) - } - (UnaryOperator::Not, a) => { - let boolean = self.boolean_val(a)?; - let not_boolean = self.builder.ins().bxor_imm(boolean, 1); - self.stack.push(JitValue::Bool(not_boolean)); - Ok(()) - } - _ => Err(JitCompileError::NotSupported), - } - } - Instruction::BinaryOperation { op } | Instruction::BinaryOperationInplace { op } => { - let op = op.get(arg); - // the rhs is popped off first - let b = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - let a = self.stack.pop().ok_or(JitCompileError::BadBytecode)?; - - let a_type = a.to_jit_type(); - let b_type = b.to_jit_type(); - - let val = match (op, a, b) { - (BinaryOperator::Add, JitValue::Int(a), JitValue::Int(b)) => { - let (out, carry) = self.builder.ins().iadd_ifcout(a, b); - self.builder.ins().trapif( - IntCC::Overflow, - carry, - TrapCode::IntegerOverflow, - ); - JitValue::Int(out) - } - (BinaryOperator::Subtract, JitValue::Int(a), JitValue::Int(b)) => { - JitValue::Int(self.compile_sub(a, b)) - } - (BinaryOperator::FloorDivide, JitValue::Int(a), JitValue::Int(b)) => { - JitValue::Int(self.builder.ins().sdiv(a, b)) - } - (BinaryOperator::Modulo, JitValue::Int(a), JitValue::Int(b)) => { - JitValue::Int(self.builder.ins().srem(a, b)) - } - ( - BinaryOperator::Lshift | BinaryOperator::Rshift, - JitValue::Int(a), - JitValue::Int(b), - ) => { - // Shifts throw an exception if we have a negative shift count - // Remove all bits except the sign bit, and trap if its 1 (i.e. negative). - let sign = self.builder.ins().ushr_imm(b, 63); - self.builder.ins().trapnz( - sign, - TrapCode::User(CustomTrapCode::NegativeShiftCount as u16), - ); - - let out = if op == BinaryOperator::Lshift { - self.builder.ins().ishl(a, b) - } else { - self.builder.ins().sshr(a, b) - }; - JitValue::Int(out) - } - (BinaryOperator::And, JitValue::Int(a), JitValue::Int(b)) => { - JitValue::Int(self.builder.ins().band(a, b)) - } - (BinaryOperator::Or, JitValue::Int(a), JitValue::Int(b)) => { - JitValue::Int(self.builder.ins().bor(a, b)) - } - (BinaryOperator::Xor, JitValue::Int(a), JitValue::Int(b)) => { - JitValue::Int(self.builder.ins().bxor(a, b)) - } - - // Floats - (BinaryOperator::Add, JitValue::Float(a), JitValue::Float(b)) => { - JitValue::Float(self.builder.ins().fadd(a, b)) - } - (BinaryOperator::Subtract, JitValue::Float(a), JitValue::Float(b)) => { - JitValue::Float(self.builder.ins().fsub(a, b)) - } - (BinaryOperator::Multiply, JitValue::Float(a), JitValue::Float(b)) => { - JitValue::Float(self.builder.ins().fmul(a, b)) - } - (BinaryOperator::Divide, JitValue::Float(a), JitValue::Float(b)) => { - JitValue::Float(self.builder.ins().fdiv(a, b)) - } - - // Floats and Integers - (_, JitValue::Int(a), JitValue::Float(b)) - | (_, JitValue::Float(a), JitValue::Int(b)) => { - let operand_one = match a_type.unwrap() { - JitType::Int => self.builder.ins().fcvt_from_sint(types::F64, a), - _ => a, - }; - - let operand_two = match b_type.unwrap() { - JitType::Int => self.builder.ins().fcvt_from_sint(types::F64, b), - _ => b, - }; - - match op { - BinaryOperator::Add => { - JitValue::Float(self.builder.ins().fadd(operand_one, operand_two)) - } - BinaryOperator::Subtract => { - JitValue::Float(self.builder.ins().fsub(operand_one, operand_two)) - } - BinaryOperator::Multiply => { - JitValue::Float(self.builder.ins().fmul(operand_one, operand_two)) - } - BinaryOperator::Divide => { - JitValue::Float(self.builder.ins().fdiv(operand_one, operand_two)) - } - _ => return Err(JitCompileError::NotSupported), - } - } - _ => return Err(JitCompileError::NotSupported), - }; - self.stack.push(val); - - Ok(()) - } - Instruction::SetupLoop { .. } | Instruction::PopBlock => { - // TODO: block support - Ok(()) - } - _ => Err(JitCompileError::NotSupported), - } - } - - fn compile_sub(&mut self, a: Value, b: Value) -> Value { - // TODO: this should be fine, but cranelift doesn't special-case isub_ifbout - // let (out, carry) = self.builder.ins().isub_ifbout(a, b); - // self.builder - // .ins() - // .trapif(IntCC::Overflow, carry, TrapCode::IntegerOverflow); - // TODO: this shouldn't wrap - let neg_b = self.builder.ins().ineg(b); - let (out, carry) = self.builder.ins().iadd_ifcout(a, neg_b); - self.builder - .ins() - .trapif(IntCC::Overflow, carry, TrapCode::IntegerOverflow); - out - } -} diff --git a/jit/src/lib.rs b/jit/src/lib.rs deleted file mode 100644 index d982ce4391b..00000000000 --- a/jit/src/lib.rs +++ /dev/null @@ -1,348 +0,0 @@ -mod instructions; - -use cranelift::prelude::*; -use cranelift_jit::{JITBuilder, JITModule}; -use cranelift_module::{FuncId, Linkage, Module, ModuleError}; -use instructions::FunctionCompiler; -use rustpython_compiler_core::bytecode; -use std::{fmt, mem::ManuallyDrop}; - -#[derive(Debug, thiserror::Error)] -#[non_exhaustive] -pub enum JitCompileError { - #[error("function can't be jitted")] - NotSupported, - #[error("bad bytecode")] - BadBytecode, - #[error("error while compiling to machine code: {0}")] - CraneliftError(#[from] ModuleError), -} - -#[derive(Debug, thiserror::Error, Eq, PartialEq)] -#[non_exhaustive] -pub enum JitArgumentError { - #[error("argument is of wrong type")] - ArgumentTypeMismatch, - #[error("wrong number of arguments")] - WrongNumberOfArguments, -} - -struct Jit { - builder_context: FunctionBuilderContext, - ctx: codegen::Context, - module: JITModule, -} - -impl Jit { - fn new() -> Self { - let builder = JITBuilder::new(cranelift_module::default_libcall_names()) - .expect("Failed to build JITBuilder"); - let module = JITModule::new(builder); - Self { - builder_context: FunctionBuilderContext::new(), - ctx: module.make_context(), - module, - } - } - - fn build_function<C: bytecode::Constant>( - &mut self, - bytecode: &bytecode::CodeObject<C>, - args: &[JitType], - ) -> Result<(FuncId, JitSig), JitCompileError> { - for arg in args { - self.ctx - .func - .signature - .params - .push(AbiParam::new(arg.to_cranelift())); - } - - let mut builder = FunctionBuilder::new(&mut self.ctx.func, &mut self.builder_context); - let entry_block = builder.create_block(); - builder.append_block_params_for_function_params(entry_block); - builder.switch_to_block(entry_block); - - let sig = { - let mut compiler = - FunctionCompiler::new(&mut builder, bytecode.varnames.len(), args, entry_block); - - compiler.compile(bytecode)?; - - compiler.sig - }; - - builder.seal_all_blocks(); - builder.finalize(); - - let id = self.module.declare_function( - &format!("jit_{}", bytecode.obj_name.as_ref()), - Linkage::Export, - &self.ctx.func.signature, - )?; - - self.module.define_function(id, &mut self.ctx)?; - - self.module.clear_context(&mut self.ctx); - - Ok((id, sig)) - } -} - -pub fn compile<C: bytecode::Constant>( - bytecode: &bytecode::CodeObject<C>, - args: &[JitType], -) -> Result<CompiledCode, JitCompileError> { - let mut jit = Jit::new(); - - let (id, sig) = jit.build_function(bytecode, args)?; - - jit.module.finalize_definitions(); - - let code = jit.module.get_finalized_function(id); - Ok(CompiledCode { - sig, - code, - module: ManuallyDrop::new(jit.module), - }) -} - -pub struct CompiledCode { - sig: JitSig, - code: *const u8, - module: ManuallyDrop<JITModule>, -} - -impl CompiledCode { - pub fn args_builder(&self) -> ArgsBuilder<'_> { - ArgsBuilder::new(self) - } - - pub fn invoke(&self, args: &[AbiValue]) -> Result<Option<AbiValue>, JitArgumentError> { - if self.sig.args.len() != args.len() { - return Err(JitArgumentError::WrongNumberOfArguments); - } - - let cif_args = self - .sig - .args - .iter() - .zip(args.iter()) - .map(|(ty, val)| type_check(ty, val).map(|_| val)) - .map(|v| v.map(AbiValue::to_libffi_arg)) - .collect::<Result<Vec<_>, _>>()?; - Ok(unsafe { self.invoke_raw(&cif_args) }) - } - - unsafe fn invoke_raw(&self, cif_args: &[libffi::middle::Arg]) -> Option<AbiValue> { - let cif = self.sig.to_cif(); - let value = cif.call::<UnTypedAbiValue>( - libffi::middle::CodePtr::from_ptr(self.code as *const _), - cif_args, - ); - self.sig.ret.as_ref().map(|ty| value.to_typed(ty)) - } -} - -struct JitSig { - args: Vec<JitType>, - ret: Option<JitType>, -} - -impl JitSig { - fn to_cif(&self) -> libffi::middle::Cif { - let ret = match self.ret { - Some(ref ty) => ty.to_libffi(), - None => libffi::middle::Type::void(), - }; - libffi::middle::Cif::new(self.args.iter().map(JitType::to_libffi), ret) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum JitType { - Int, - Float, - Bool, -} - -impl JitType { - fn to_cranelift(&self) -> types::Type { - match self { - Self::Int => types::I64, - Self::Float => types::F64, - Self::Bool => types::I8, - } - } - - fn to_libffi(&self) -> libffi::middle::Type { - match self { - Self::Int => libffi::middle::Type::i64(), - Self::Float => libffi::middle::Type::f64(), - Self::Bool => libffi::middle::Type::u8(), - } - } -} - -#[derive(Debug, Clone, PartialEq)] -#[non_exhaustive] -pub enum AbiValue { - Float(f64), - Int(i64), - Bool(bool), -} - -impl AbiValue { - fn to_libffi_arg(&self) -> libffi::middle::Arg { - match self { - AbiValue::Int(ref i) => libffi::middle::Arg::new(i), - AbiValue::Float(ref f) => libffi::middle::Arg::new(f), - AbiValue::Bool(ref b) => libffi::middle::Arg::new(b), - } - } -} - -impl From<i64> for AbiValue { - fn from(i: i64) -> Self { - AbiValue::Int(i) - } -} - -impl From<f64> for AbiValue { - fn from(f: f64) -> Self { - AbiValue::Float(f) - } -} - -impl From<bool> for AbiValue { - fn from(b: bool) -> Self { - AbiValue::Bool(b) - } -} - -impl TryFrom<AbiValue> for i64 { - type Error = (); - - fn try_from(value: AbiValue) -> Result<Self, Self::Error> { - match value { - AbiValue::Int(i) => Ok(i), - _ => Err(()), - } - } -} - -impl TryFrom<AbiValue> for f64 { - type Error = (); - - fn try_from(value: AbiValue) -> Result<Self, Self::Error> { - match value { - AbiValue::Float(f) => Ok(f), - _ => Err(()), - } - } -} - -impl TryFrom<AbiValue> for bool { - type Error = (); - - fn try_from(value: AbiValue) -> Result<Self, Self::Error> { - match value { - AbiValue::Bool(b) => Ok(b), - _ => Err(()), - } - } -} - -fn type_check(ty: &JitType, val: &AbiValue) -> Result<(), JitArgumentError> { - match (ty, val) { - (JitType::Int, AbiValue::Int(_)) - | (JitType::Float, AbiValue::Float(_)) - | (JitType::Bool, AbiValue::Bool(_)) => Ok(()), - _ => Err(JitArgumentError::ArgumentTypeMismatch), - } -} - -#[derive(Copy, Clone)] -union UnTypedAbiValue { - float: f64, - int: i64, - boolean: u8, - _void: (), -} - -impl UnTypedAbiValue { - unsafe fn to_typed(self, ty: &JitType) -> AbiValue { - match ty { - JitType::Int => AbiValue::Int(self.int), - JitType::Float => AbiValue::Float(self.float), - JitType::Bool => AbiValue::Bool(self.boolean != 0), - } - } -} - -// we don't actually ever touch CompiledCode til we drop it, it should be safe. -// TODO: confirm with wasmtime ppl that it's not unsound? -unsafe impl Send for CompiledCode {} -unsafe impl Sync for CompiledCode {} - -impl Drop for CompiledCode { - fn drop(&mut self) { - // SAFETY: The only pointer that this memory will also be dropped now - unsafe { ManuallyDrop::take(&mut self.module).free_memory() } - } -} - -impl fmt::Debug for CompiledCode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("[compiled code]") - } -} - -pub struct ArgsBuilder<'a> { - values: Vec<Option<AbiValue>>, - code: &'a CompiledCode, -} - -impl<'a> ArgsBuilder<'a> { - fn new(code: &'a CompiledCode) -> ArgsBuilder<'a> { - ArgsBuilder { - values: vec![None; code.sig.args.len()], - code, - } - } - - pub fn set(&mut self, idx: usize, value: AbiValue) -> Result<(), JitArgumentError> { - type_check(&self.code.sig.args[idx], &value).map(|_| { - self.values[idx] = Some(value); - }) - } - - pub fn is_set(&self, idx: usize) -> bool { - self.values[idx].is_some() - } - - pub fn into_args(self) -> Option<Args<'a>> { - self.values - .iter() - .map(|v| v.as_ref().map(AbiValue::to_libffi_arg)) - .collect::<Option<_>>() - .map(|cif_args| Args { - _values: self.values, - cif_args, - code: self.code, - }) - } -} - -pub struct Args<'a> { - _values: Vec<Option<AbiValue>>, - cif_args: Vec<libffi::middle::Arg>, - code: &'a CompiledCode, -} - -impl<'a> Args<'a> { - pub fn invoke(&self) -> Option<AbiValue> { - unsafe { self.code.invoke_raw(&self.cif_args) } - } -} diff --git a/jit/tests/common.rs b/jit/tests/common.rs deleted file mode 100644 index 2f56e5db16f..00000000000 --- a/jit/tests/common.rs +++ /dev/null @@ -1,204 +0,0 @@ -use rustpython_compiler_core::bytecode::{ - CodeObject, ConstantData, Instruction, OpArg, OpArgState, -}; -use rustpython_jit::{CompiledCode, JitType}; -use std::collections::HashMap; -use std::ops::ControlFlow; - -#[derive(Debug, Clone)] -pub struct Function { - code: Box<CodeObject>, - annotations: HashMap<String, StackValue>, -} - -impl Function { - pub fn compile(self) -> CompiledCode { - let mut arg_types = Vec::new(); - for arg in self.code.arg_names().args { - let arg_type = match self.annotations.get(arg) { - Some(StackValue::String(annotation)) => match annotation.as_str() { - "int" => JitType::Int, - "float" => JitType::Float, - "bool" => JitType::Bool, - _ => panic!("Unrecognised jit type"), - }, - _ => panic!("Argument have annotation"), - }; - arg_types.push(arg_type); - } - - rustpython_jit::compile(&self.code, &arg_types).expect("Compile failure") - } -} - -#[derive(Debug, Clone)] -enum StackValue { - String(String), - None, - Map(HashMap<String, StackValue>), - Code(Box<CodeObject>), - Function(Function), -} - -impl From<ConstantData> for StackValue { - fn from(value: ConstantData) -> Self { - match value { - ConstantData::Str { value } => StackValue::String(value), - ConstantData::None => StackValue::None, - ConstantData::Code { code } => StackValue::Code(code), - c => unimplemented!("constant {:?} isn't yet supported in py_function!", c), - } - } -} - -pub struct StackMachine { - stack: Vec<StackValue>, - locals: HashMap<String, StackValue>, -} - -impl StackMachine { - pub fn new() -> StackMachine { - StackMachine { - stack: Vec::new(), - locals: HashMap::new(), - } - } - - pub fn run(&mut self, code: CodeObject) { - let mut oparg_state = OpArgState::default(); - code.instructions.iter().try_for_each(|&word| { - let (instruction, arg) = oparg_state.get(word); - self.process_instruction(instruction, arg, &code.constants, &code.names) - }); - } - - fn process_instruction( - &mut self, - instruction: Instruction, - arg: OpArg, - constants: &[ConstantData], - names: &[String], - ) -> ControlFlow<()> { - match instruction { - Instruction::LoadConst { idx } => { - let idx = idx.get(arg); - self.stack.push(constants[idx as usize].clone().into()) - } - Instruction::LoadNameAny(idx) => self - .stack - .push(StackValue::String(names[idx.get(arg) as usize].clone())), - Instruction::StoreLocal(idx) => { - let idx = idx.get(arg); - self.locals - .insert(names[idx as usize].clone(), self.stack.pop().unwrap()); - } - Instruction::StoreAttr { .. } => { - // Do nothing except throw away the stack values - self.stack.pop().unwrap(); - self.stack.pop().unwrap(); - } - Instruction::BuildMap { size, .. } => { - let mut map = HashMap::new(); - for _ in 0..size.get(arg) { - let value = self.stack.pop().unwrap(); - let name = if let Some(StackValue::String(name)) = self.stack.pop() { - name - } else { - unimplemented!("no string keys isn't yet supported in py_function!") - }; - map.insert(name, value); - } - self.stack.push(StackValue::Map(map)); - } - Instruction::MakeFunction(_flags) => { - let _name = if let Some(StackValue::String(name)) = self.stack.pop() { - name - } else { - panic!("Expected function name") - }; - let code = if let Some(StackValue::Code(code)) = self.stack.pop() { - code - } else { - panic!("Expected function code") - }; - let annotations = if let Some(StackValue::Map(map)) = self.stack.pop() { - map - } else { - panic!("Expected function annotations") - }; - self.stack - .push(StackValue::Function(Function { code, annotations })); - } - Instruction::Duplicate => { - let value = self.stack.last().unwrap().clone(); - self.stack.push(value); - } - Instruction::Rotate2 => { - let i = self.stack.len() - 2; - self.stack[i..].rotate_right(1); - } - Instruction::Rotate3 => { - let i = self.stack.len() - 3; - self.stack[i..].rotate_right(1); - } - Instruction::ReturnValue => return ControlFlow::Break(()), - Instruction::ExtendedArg => {} - _ => unimplemented!( - "instruction {:?} isn't yet supported in py_function!", - instruction - ), - } - ControlFlow::Continue(()) - } - - pub fn get_function(&self, name: &str) -> Function { - if let Some(StackValue::Function(function)) = self.locals.get(name) { - function.clone() - } else { - panic!("There was no function named {}", name) - } - } -} - -macro_rules! jit_function { - ($func_name:ident => $($t:tt)*) => { - { - let code = rustpython_derive::py_compile!( - crate_name = "rustpython_compiler_core", - source = $($t)* - ); - let code = code.decode(rustpython_compiler_core::bytecode::BasicBag); - let mut machine = $crate::common::StackMachine::new(); - machine.run(code); - machine.get_function(stringify!($func_name)).compile() - } - }; - ($func_name:ident($($arg_name:ident:$arg_type:ty),*) -> $ret_type:ty => $($t:tt)*) => { - { - let jit_code = jit_function!($func_name => $($t)*); - - move |$($arg_name:$arg_type),*| -> Result<$ret_type, rustpython_jit::JitArgumentError> { - jit_code - .invoke(&[$($arg_name.into()),*]) - .map(|ret| match ret { - Some(ret) => ret.try_into().expect("jit function returned unexpected type"), - None => panic!("jit function unexpectedly returned None") - }) - } - } - }; - ($func_name:ident($($arg_name:ident:$arg_type:ty),*) => $($t:tt)*) => { - { - let jit_code = jit_function!($func_name => $($t)*); - - move |$($arg_name:$arg_type),*| -> Result<(), rustpython_jit::JitArgumentError> { - jit_code - .invoke(&[$($arg_name.into()),*]) - .map(|ret| match ret { - Some(ret) => panic!("jit function unexpectedly returned a value {:?}", ret), - None => () - }) - } - } - }; -} diff --git a/jit/tests/float_tests.rs b/jit/tests/float_tests.rs deleted file mode 100644 index 2ba7dec8229..00000000000 --- a/jit/tests/float_tests.rs +++ /dev/null @@ -1,268 +0,0 @@ -macro_rules! assert_approx_eq { - ($left:expr, $right:expr) => { - match ($left, $right) { - (Ok(lhs), Ok(rhs)) => approx::assert_relative_eq!(lhs, rhs), - (lhs, rhs) => assert_eq!(lhs, rhs), - } - }; -} - -macro_rules! assert_bits_eq { - ($left:expr, $right:expr) => { - match ($left, $right) { - (Ok(lhs), Ok(rhs)) => assert!(lhs.to_bits() == rhs.to_bits()), - (lhs, rhs) => assert_eq!(lhs, rhs), - } - }; -} - -#[test] -fn test_add() { - let add = jit_function! { add(a:f64, b:f64) -> f64 => r##" - def add(a: float, b: float): - return a + b - "## }; - - assert_approx_eq!(add(5.5, 10.2), Ok(15.7)); - assert_approx_eq!(add(-4.5, 7.6), Ok(3.1)); - assert_approx_eq!(add(-5.2, -3.9), Ok(-9.1)); - assert_bits_eq!(add(-5.2, f64::NAN), Ok(f64::NAN)); - assert_eq!(add(2.0, f64::INFINITY), Ok(f64::INFINITY)); - assert_eq!(add(-2.0, f64::NEG_INFINITY), Ok(f64::NEG_INFINITY)); - assert_eq!(add(1.0, f64::NEG_INFINITY), Ok(f64::NEG_INFINITY)); -} - -#[test] -fn test_add_with_integer() { - let add = jit_function! { add(a:f64, b:i64) -> f64 => r##" - def add(a: float, b: int): - return a + b - "## }; - - assert_approx_eq!(add(5.5, 10), Ok(15.5)); - assert_approx_eq!(add(-4.6, 7), Ok(2.4)); - assert_approx_eq!(add(-5.2, -3), Ok(-8.2)); -} - -#[test] -fn test_sub() { - let sub = jit_function! { sub(a:f64, b:f64) -> f64 => r##" - def sub(a: float, b: float): - return a - b - "## }; - - assert_approx_eq!(sub(5.2, 3.6), Ok(1.6)); - assert_approx_eq!(sub(3.4, 4.2), Ok(-0.8)); - assert_approx_eq!(sub(-2.1, 1.3), Ok(-3.4)); - assert_approx_eq!(sub(3.1, -1.3), Ok(4.4)); - assert_bits_eq!(sub(-5.2, f64::NAN), Ok(f64::NAN)); - assert_eq!(sub(f64::INFINITY, 2.0), Ok(f64::INFINITY)); - assert_eq!(sub(-2.0, f64::NEG_INFINITY), Ok(f64::INFINITY)); - assert_eq!(sub(1.0, f64::INFINITY), Ok(f64::NEG_INFINITY)); -} - -#[test] -fn test_sub_with_integer() { - let sub = jit_function! { sub(a:i64, b:f64) -> f64 => r##" - def sub(a: int, b: float): - return a - b - "## }; - - assert_approx_eq!(sub(5, 3.6), Ok(1.4)); - assert_approx_eq!(sub(3, -4.2), Ok(7.2)); - assert_approx_eq!(sub(-2, 1.3), Ok(-3.3)); - assert_approx_eq!(sub(-3, -1.3), Ok(-1.7)); -} - -#[test] -fn test_mul() { - let mul = jit_function! { mul(a:f64, b:f64) -> f64 => r##" - def mul(a: float, b: float): - return a * b - "## }; - - assert_approx_eq!(mul(5.2, 2.0), Ok(10.4)); - assert_approx_eq!(mul(3.4, -1.7), Ok(-5.779999999999999)); - assert_bits_eq!(mul(1.0, 0.0), Ok(0.0f64)); - assert_bits_eq!(mul(1.0, -0.0), Ok(-0.0f64)); - assert_bits_eq!(mul(-1.0, 0.0), Ok(-0.0f64)); - assert_bits_eq!(mul(-1.0, -0.0), Ok(0.0f64)); - assert_bits_eq!(mul(-5.2, f64::NAN), Ok(f64::NAN)); - assert_eq!(mul(1.0, f64::INFINITY), Ok(f64::INFINITY)); - assert_eq!(mul(1.0, f64::NEG_INFINITY), Ok(f64::NEG_INFINITY)); - assert_eq!(mul(-1.0, f64::INFINITY), Ok(f64::NEG_INFINITY)); - assert!(mul(0.0, f64::INFINITY).unwrap().is_nan()); - assert_eq!(mul(f64::NEG_INFINITY, f64::INFINITY), Ok(f64::NEG_INFINITY)); -} - -#[test] -fn test_mul_with_integer() { - let mul = jit_function! { mul(a:f64, b:i64) -> f64 => r##" - def mul(a: float, b: int): - return a * b - "## }; - - assert_approx_eq!(mul(5.2, 2), Ok(10.4)); - assert_approx_eq!(mul(3.4, -1), Ok(-3.4)); - assert_bits_eq!(mul(1.0, 0), Ok(0.0f64)); - assert_bits_eq!(mul(-0.0, 1), Ok(-0.0f64)); - assert_bits_eq!(mul(0.0, -1), Ok(-0.0f64)); - assert_bits_eq!(mul(-0.0, -1), Ok(0.0f64)); -} - -#[test] -fn test_div() { - let div = jit_function! { div(a:f64, b:f64) -> f64 => r##" - def div(a: float, b: float): - return a / b - "## }; - - assert_approx_eq!(div(5.2, 2.0), Ok(2.6)); - assert_approx_eq!(div(3.4, -1.7), Ok(-2.0)); - assert_eq!(div(1.0, 0.0), Ok(f64::INFINITY)); - assert_eq!(div(1.0, -0.0), Ok(f64::NEG_INFINITY)); - assert_eq!(div(-1.0, 0.0), Ok(f64::NEG_INFINITY)); - assert_eq!(div(-1.0, -0.0), Ok(f64::INFINITY)); - assert_bits_eq!(div(-5.2, f64::NAN), Ok(f64::NAN)); - assert_eq!(div(f64::INFINITY, 2.0), Ok(f64::INFINITY)); - assert_bits_eq!(div(-2.0, f64::NEG_INFINITY), Ok(0.0f64)); - assert_bits_eq!(div(1.0, f64::INFINITY), Ok(0.0f64)); - assert_bits_eq!(div(2.0, f64::NEG_INFINITY), Ok(-0.0f64)); - assert_bits_eq!(div(-1.0, f64::INFINITY), Ok(-0.0f64)); -} - -#[test] -fn test_div_with_integer() { - let div = jit_function! { div(a:f64, b:i64) -> f64 => r##" - def div(a: float, b: int): - return a / b - "## }; - - assert_approx_eq!(div(5.2, 2), Ok(2.6)); - assert_approx_eq!(div(3.4, -1), Ok(-3.4)); - assert_eq!(div(1.0, 0), Ok(f64::INFINITY)); - assert_eq!(div(1.0, -0), Ok(f64::INFINITY)); - assert_eq!(div(-1.0, 0), Ok(f64::NEG_INFINITY)); - assert_eq!(div(-1.0, -0), Ok(f64::NEG_INFINITY)); - assert_eq!(div(f64::INFINITY, 2), Ok(f64::INFINITY)); - assert_eq!(div(f64::NEG_INFINITY, 3), Ok(f64::NEG_INFINITY)); -} - -#[test] -fn test_if_bool() { - let if_bool = jit_function! { if_bool(a:f64) -> i64 => r##" - def if_bool(a: float): - if a: - return 1 - return 0 - "## }; - - assert_eq!(if_bool(5.2), Ok(1)); - assert_eq!(if_bool(-3.4), Ok(1)); - assert_eq!(if_bool(f64::NAN), Ok(1)); - assert_eq!(if_bool(f64::INFINITY), Ok(1)); - - assert_eq!(if_bool(0.0), Ok(0)); -} - -#[test] -fn test_float_eq() { - let float_eq = jit_function! { float_eq(a: f64, b: f64) -> bool => r##" - def float_eq(a: float, b: float): - return a == b - "## }; - - assert_eq!(float_eq(2.0, 2.0), Ok(true)); - assert_eq!(float_eq(3.4, -1.7), Ok(false)); - assert_eq!(float_eq(0.0, 0.0), Ok(true)); - assert_eq!(float_eq(-0.0, -0.0), Ok(true)); - assert_eq!(float_eq(-0.0, 0.0), Ok(true)); - assert_eq!(float_eq(-5.2, f64::NAN), Ok(false)); - assert_eq!(float_eq(f64::NAN, f64::NAN), Ok(false)); - assert_eq!(float_eq(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); -} - -#[test] -fn test_float_ne() { - let float_ne = jit_function! { float_ne(a: f64, b: f64) -> bool => r##" - def float_ne(a: float, b: float): - return a != b - "## }; - - assert_eq!(float_ne(2.0, 2.0), Ok(false)); - assert_eq!(float_ne(3.4, -1.7), Ok(true)); - assert_eq!(float_ne(0.0, 0.0), Ok(false)); - assert_eq!(float_ne(-0.0, -0.0), Ok(false)); - assert_eq!(float_ne(-0.0, 0.0), Ok(false)); - assert_eq!(float_ne(-5.2, f64::NAN), Ok(true)); - assert_eq!(float_ne(f64::NAN, f64::NAN), Ok(true)); - assert_eq!(float_ne(f64::INFINITY, f64::NEG_INFINITY), Ok(true)); -} - -#[test] -fn test_float_gt() { - let float_gt = jit_function! { float_gt(a: f64, b: f64) -> bool => r##" - def float_gt(a: float, b: float): - return a > b - "## }; - - assert_eq!(float_gt(2.0, 2.0), Ok(false)); - assert_eq!(float_gt(3.4, -1.7), Ok(true)); - assert_eq!(float_gt(0.0, 0.0), Ok(false)); - assert_eq!(float_gt(-0.0, -0.0), Ok(false)); - assert_eq!(float_gt(-0.0, 0.0), Ok(false)); - assert_eq!(float_gt(-5.2, f64::NAN), Ok(false)); - assert_eq!(float_gt(f64::NAN, f64::NAN), Ok(false)); - assert_eq!(float_gt(f64::INFINITY, f64::NEG_INFINITY), Ok(true)); -} - -#[test] -fn test_float_gte() { - let float_gte = jit_function! { float_gte(a: f64, b: f64) -> bool => r##" - def float_gte(a: float, b: float): - return a >= b - "## }; - - assert_eq!(float_gte(2.0, 2.0), Ok(true)); - assert_eq!(float_gte(3.4, -1.7), Ok(true)); - assert_eq!(float_gte(0.0, 0.0), Ok(true)); - assert_eq!(float_gte(-0.0, -0.0), Ok(true)); - assert_eq!(float_gte(-0.0, 0.0), Ok(true)); - assert_eq!(float_gte(-5.2, f64::NAN), Ok(false)); - assert_eq!(float_gte(f64::NAN, f64::NAN), Ok(false)); - assert_eq!(float_gte(f64::INFINITY, f64::NEG_INFINITY), Ok(true)); -} - -#[test] -fn test_float_lt() { - let float_lt = jit_function! { float_lt(a: f64, b: f64) -> bool => r##" - def float_lt(a: float, b: float): - return a < b - "## }; - - assert_eq!(float_lt(2.0, 2.0), Ok(false)); - assert_eq!(float_lt(3.4, -1.7), Ok(false)); - assert_eq!(float_lt(0.0, 0.0), Ok(false)); - assert_eq!(float_lt(-0.0, -0.0), Ok(false)); - assert_eq!(float_lt(-0.0, 0.0), Ok(false)); - assert_eq!(float_lt(-5.2, f64::NAN), Ok(false)); - assert_eq!(float_lt(f64::NAN, f64::NAN), Ok(false)); - assert_eq!(float_lt(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); -} - -#[test] -fn test_float_lte() { - let float_lte = jit_function! { float_lte(a: f64, b: f64) -> bool => r##" - def float_lte(a: float, b: float): - return a <= b - "## }; - - assert_eq!(float_lte(2.0, 2.0), Ok(true)); - assert_eq!(float_lte(3.4, -1.7), Ok(false)); - assert_eq!(float_lte(0.0, 0.0), Ok(true)); - assert_eq!(float_lte(-0.0, -0.0), Ok(true)); - assert_eq!(float_lte(-0.0, 0.0), Ok(true)); - assert_eq!(float_lte(-5.2, f64::NAN), Ok(false)); - assert_eq!(float_lte(f64::NAN, f64::NAN), Ok(false)); - assert_eq!(float_lte(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); -} diff --git a/jit/tests/int_tests.rs b/jit/tests/int_tests.rs deleted file mode 100644 index 9ce3f3b4a63..00000000000 --- a/jit/tests/int_tests.rs +++ /dev/null @@ -1,247 +0,0 @@ -#[test] -fn test_add() { - let add = jit_function! { add(a:i64, b:i64) -> i64 => r##" - def add(a: int, b: int): - return a + b - "## }; - - assert_eq!(add(5, 10), Ok(15)); - assert_eq!(add(-5, 12), Ok(7)); - assert_eq!(add(-5, -3), Ok(-8)); -} - -#[test] -fn test_sub() { - let sub = jit_function! { sub(a:i64, b:i64) -> i64 => r##" - def sub(a: int, b: int): - return a - b - "## }; - - assert_eq!(sub(5, 10), Ok(-5)); - assert_eq!(sub(12, 10), Ok(2)); - assert_eq!(sub(7, 10), Ok(-3)); - assert_eq!(sub(-3, -10), Ok(7)); -} - -#[test] -fn test_floor_div() { - let floor_div = jit_function! { floor_div(a:i64, b:i64) -> i64 => r##" - def floor_div(a: int, b: int): - return a // b - "## }; - - assert_eq!(floor_div(5, 10), Ok(0)); - assert_eq!(floor_div(5, 2), Ok(2)); - assert_eq!(floor_div(12, 10), Ok(1)); - assert_eq!(floor_div(7, 10), Ok(0)); - assert_eq!(floor_div(-3, -1), Ok(3)); - assert_eq!(floor_div(-3, 1), Ok(-3)); -} - -#[test] -fn test_mod() { - let modulo = jit_function! { modulo(a:i64, b:i64) -> i64 => r##" - def modulo(a: int, b: int): - return a % b - "## }; - - assert_eq!(modulo(5, 10), Ok(5)); - assert_eq!(modulo(5, 2), Ok(1)); - assert_eq!(modulo(12, 10), Ok(2)); - assert_eq!(modulo(7, 10), Ok(7)); - assert_eq!(modulo(-3, 1), Ok(0)); - assert_eq!(modulo(-5, 10), Ok(-5)); -} - -#[test] -fn test_lshift() { - let lshift = jit_function! { lshift(a:i64, b:i64) -> i64 => r##" - def lshift(a: int, b: int): - return a << b - "## }; - - assert_eq!(lshift(5, 10), Ok(5120)); - assert_eq!(lshift(5, 2), Ok(20)); - assert_eq!(lshift(12, 10), Ok(12288)); - assert_eq!(lshift(7, 10), Ok(7168)); - assert_eq!(lshift(-3, 1), Ok(-6)); - assert_eq!(lshift(-10, 2), Ok(-40)); -} - -#[test] -fn test_rshift() { - let rshift = jit_function! { rshift(a:i64, b:i64) -> i64 => r##" - def rshift(a: int, b: int): - return a >> b - "## }; - - assert_eq!(rshift(5120, 10), Ok(5)); - assert_eq!(rshift(20, 2), Ok(5)); - assert_eq!(rshift(12288, 10), Ok(12)); - assert_eq!(rshift(7168, 10), Ok(7)); - assert_eq!(rshift(-3, 1), Ok(-2)); - assert_eq!(rshift(-10, 2), Ok(-3)); -} - -#[test] -fn test_and() { - let bitand = jit_function! { bitand(a:i64, b:i64) -> i64 => r##" - def bitand(a: int, b: int): - return a & b - "## }; - - assert_eq!(bitand(5120, 10), Ok(0)); - assert_eq!(bitand(20, 16), Ok(16)); - assert_eq!(bitand(12488, 4249), Ok(4232)); - assert_eq!(bitand(7168, 2), Ok(0)); - assert_eq!(bitand(-3, 1), Ok(1)); - assert_eq!(bitand(-10, 2), Ok(2)); -} - -#[test] -fn test_or() { - let bitor = jit_function! { bitor(a:i64, b:i64) -> i64 => r##" - def bitor(a: int, b: int): - return a | b - "## }; - - assert_eq!(bitor(5120, 10), Ok(5130)); - assert_eq!(bitor(20, 16), Ok(20)); - assert_eq!(bitor(12488, 4249), Ok(12505)); - assert_eq!(bitor(7168, 2), Ok(7170)); - assert_eq!(bitor(-3, 1), Ok(-3)); - assert_eq!(bitor(-10, 2), Ok(-10)); -} - -#[test] -fn test_xor() { - let bitxor = jit_function! { bitxor(a:i64, b:i64) -> i64 => r##" - def bitxor(a: int, b: int): - return a ^ b - "## }; - - assert_eq!(bitxor(5120, 10), Ok(5130)); - assert_eq!(bitxor(20, 16), Ok(4)); - assert_eq!(bitxor(12488, 4249), Ok(8273)); - assert_eq!(bitxor(7168, 2), Ok(7170)); - assert_eq!(bitxor(-3, 1), Ok(-4)); - assert_eq!(bitxor(-10, 2), Ok(-12)); -} - -#[test] -fn test_eq() { - let eq = jit_function! { eq(a:i64, b:i64) -> i64 => r##" - def eq(a: int, b: int): - if a == b: - return 1 - return 0 - "## }; - - assert_eq!(eq(0, 0), Ok(1)); - assert_eq!(eq(1, 1), Ok(1)); - assert_eq!(eq(0, 1), Ok(0)); - assert_eq!(eq(-200, 200), Ok(0)); -} - -#[test] -fn test_gt() { - let gt = jit_function! { gt(a:i64, b:i64) -> i64 => r##" - def gt(a: int, b: int): - if a > b: - return 1 - return 0 - "## }; - - assert_eq!(gt(5, 2), Ok(1)); - assert_eq!(gt(2, 5), Ok(0)); - assert_eq!(gt(2, 2), Ok(0)); - assert_eq!(gt(5, 5), Ok(0)); - assert_eq!(gt(-1, -10), Ok(1)); - assert_eq!(gt(1, -1), Ok(1)); -} - -#[test] -fn test_lt() { - let lt = jit_function! { lt(a:i64, b:i64) -> i64 => r##" - def lt(a: int, b: int): - if a < b: - return 1 - return 0 - "## }; - - assert_eq!(lt(-1, -5), Ok(0)); - assert_eq!(lt(10, 0), Ok(0)); - assert_eq!(lt(0, 1), Ok(1)); - assert_eq!(lt(-10, -1), Ok(1)); - assert_eq!(lt(100, 100), Ok(0)); -} - -#[test] -fn test_gte() { - let gte = jit_function! { gte(a:i64, b:i64) -> i64 => r##" - def gte(a: int, b: int): - if a >= b: - return 1 - return 0 - "## }; - - assert_eq!(gte(-64, -64), Ok(1)); - assert_eq!(gte(100, -1), Ok(1)); - assert_eq!(gte(1, 2), Ok(0)); - assert_eq!(gte(1, 0), Ok(1)); -} - -#[test] -fn test_lte() { - let lte = jit_function! { lte(a:i64, b:i64) -> i64 => r##" - def lte(a: int, b: int): - if a <= b: - return 1 - return 0 - "## }; - - assert_eq!(lte(-100, -100), Ok(1)); - assert_eq!(lte(-100, 100), Ok(1)); - assert_eq!(lte(10, 1), Ok(0)); - assert_eq!(lte(0, -2), Ok(0)); -} - -#[test] -fn test_minus() { - let minus = jit_function! { minus(a:i64) -> i64 => r##" - def minus(a: int): - return -a - "## }; - - assert_eq!(minus(5), Ok(-5)); - assert_eq!(minus(12), Ok(-12)); - assert_eq!(minus(-7), Ok(7)); - assert_eq!(minus(-3), Ok(3)); - assert_eq!(minus(0), Ok(0)); -} - -#[test] -fn test_plus() { - let plus = jit_function! { plus(a:i64) -> i64 => r##" - def plus(a: int): - return +a - "## }; - - assert_eq!(plus(5), Ok(5)); - assert_eq!(plus(12), Ok(12)); - assert_eq!(plus(-7), Ok(-7)); - assert_eq!(plus(-3), Ok(-3)); - assert_eq!(plus(0), Ok(0)); -} - -#[test] -fn test_not() { - let not_ = jit_function! { not_(a: i64) -> bool => r##" - def not_(a: int): - return not a - "## }; - - assert_eq!(not_(0), Ok(true)); - assert_eq!(not_(1), Ok(false)); - assert_eq!(not_(-1), Ok(false)); -} diff --git a/logo.ico b/logo.ico new file mode 100644 index 00000000000..24cbb8c3789 Binary files /dev/null and b/logo.ico differ diff --git a/pylib/Cargo.toml b/pylib/Cargo.toml deleted file mode 100644 index 256dbe9d61c..00000000000 --- a/pylib/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "rustpython-pylib" -version = "0.3.0" -authors = ["RustPython Team"] -description = "A subset of the Python standard library for use with RustPython" -repository = "https://github.com/RustPython/RustPython" -license-file = "Lib/PSF-LICENSE" -edition = "2021" -include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"] - -[features] -freeze-stdlib = [] - -[dependencies] -rustpython-compiler-core = { workspace = true } -rustpython-derive = { version = "0.3.0", path = "../derive" } - -[build-dependencies] -glob = { workspace = true } diff --git a/pylib/Lib b/pylib/Lib deleted file mode 120000 index 47f928ff4d5..00000000000 --- a/pylib/Lib +++ /dev/null @@ -1 +0,0 @@ -../Lib \ No newline at end of file diff --git a/pylib/build.rs b/pylib/build.rs deleted file mode 100644 index a541bc56d5b..00000000000 --- a/pylib/build.rs +++ /dev/null @@ -1,41 +0,0 @@ -fn main() { - process_python_libs("../vm/Lib/python_builtins/*"); - - #[cfg(not(feature = "stdlib"))] - process_python_libs("../vm/Lib/core_modules/*"); - #[cfg(feature = "freeze-stdlib")] - if cfg!(windows) { - process_python_libs("../Lib/**/*"); - } else { - process_python_libs("./Lib/**/*"); - } - - if cfg!(windows) { - if let Ok(real_path) = std::fs::read_to_string("Lib") { - let canonicalized_path = std::fs::canonicalize(real_path) - .expect("failed to resolve RUSTPYTHONPATH during build time"); - println!( - "cargo:rustc-env=win_lib_path={}", - canonicalized_path.to_str().unwrap() - ); - } - } -} - -// remove *.pyc files and add *.py to watch list -fn process_python_libs(pattern: &str) { - let glob = glob::glob(pattern).unwrap_or_else(|e| panic!("failed to glob {pattern:?}: {e}")); - for entry in glob.flatten() { - if entry.is_dir() { - continue; - } - let display = entry.display(); - if display.to_string().ends_with(".pyc") { - if std::fs::remove_file(&entry).is_err() { - println!("cargo:warning=failed to remove {display}") - } - continue; - } - println!("cargo:rerun-if-changed={display}"); - } -} diff --git a/pylib/src/lib.rs b/pylib/src/lib.rs deleted file mode 100644 index f8a47ba67da..00000000000 --- a/pylib/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! This crate includes the compiled python bytecode of the RustPython standard library. The most -//! common way to use this crate is to just add the `"freeze-stdlib"` feature to `rustpython-vm`, -//! in order to automatically include the python part of the standard library into the binary. - -// windows needs to read the symlink out of `Lib` as git turns it into a text file, -// so build.rs sets this env var -pub const LIB_PATH: &str = match option_env!("win_lib_path") { - Some(s) => s, - None => concat!(env!("CARGO_MANIFEST_DIR"), "/Lib"), -}; - -#[cfg(feature = "freeze-stdlib")] -pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib = - rustpython_derive::py_freeze!(dir = "./Lib", crate_name = "rustpython_compiler_core"); diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000000..2ed67851f0a --- /dev/null +++ b/ruff.toml @@ -0,0 +1,15 @@ +exclude = [ + "Lib", + "vm/Lib", + "benches", + "syntax_*.py", # Do not format files that are specifically testing for syntax + "badsyntax_*.py", +] + +[lint] +select = [ + "E9", # pycodestyle (error) + "F63", # pyflakes + "F7", + "F82", +] diff --git a/rustfmt.toml b/rustfmt.toml index 3a26366d4da..f216078d96f 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1 +1 @@ -edition = "2021" +edition = "2024" diff --git a/scripts/bench.sh b/scripts/bench.sh new file mode 100644 index 00000000000..084970b0fcc --- /dev/null +++ b/scripts/bench.sh @@ -0,0 +1,14 @@ +for BENCH in "int" "nbody" "fannkuch" "scimark"; do + for CPYTHON in "3.9" "3.10"; do + CMD="python${CPYTHON} ${BENCH}.py -o reports/${BENCH}_cpython${CPYTHON}.pyperf" + echo "${CMD}" + ${CMD} + sleep 1 + done + for RUSTPYTHON in "3819" "main"; do + CMD="./target/release/rustpython_${RUSTPYTHON} ${BENCH}.py -o reports/${BENCH}_rustpython_${RUSTPYTHON}.pyperf" + echo "${CMD}" + ${CMD} + sleep 1 + done +done diff --git a/scripts/cargo-llvm-cov.py b/scripts/cargo-llvm-cov.py index a77d56a87c6..9a7b24dd04c 100644 --- a/scripts/cargo-llvm-cov.py +++ b/scripts/cargo-llvm-cov.py @@ -3,18 +3,21 @@ TARGET = "extra_tests/snippets" + def run_llvm_cov(file_path: str): - """ Run cargo llvm-cov on a file. """ + """Run cargo llvm-cov on a file.""" if file_path.endswith(".py"): command = ["cargo", "llvm-cov", "--no-report", "run", "--", file_path] subprocess.call(command) + def iterate_files(folder: str): - """ Iterate over all files in a folder. """ + """Iterate over all files in a folder.""" for root, _, files in os.walk(folder): for file in files: file_path = os.path.join(root, file) run_llvm_cov(file_path) + if __name__ == "__main__": - iterate_files(TARGET) \ No newline at end of file + iterate_files(TARGET) diff --git a/scripts/check_redundant_patches.py b/scripts/check_redundant_patches.py new file mode 100644 index 00000000000..25cd2e1229e --- /dev/null +++ b/scripts/check_redundant_patches.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +import ast +import pathlib +import sys + +ROOT = pathlib.Path(__file__).parents[1] +TEST_DIR = ROOT / "Lib" / "test" + + +def main(): + exit_status = 0 + for file in TEST_DIR.rglob("**/*.py"): + try: + contents = file.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + + try: + tree = ast.parse(contents) + except SyntaxError: + continue + + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + + name = node.name + if not name.startswith("test"): + continue + + if node.decorator_list: + continue + + func_code = ast.unparse(node.body) + if func_code in ( + f"await super().{name}()", + f"return await super().{name}()", + f"return super().{name}()", + f"super().{name}()", + ): + exit_status += 1 + rel = file.relative_to(ROOT) + lineno = node.lineno + print( + f"{rel}:{name}:{lineno} is a test patch that can be safely removed", + file=sys.stderr, + ) + return exit_status + + +if __name__ == "__main__": + exit(main()) diff --git a/scripts/checklist_template.md b/scripts/checklist_template.md new file mode 100644 index 00000000000..ab80209ca02 --- /dev/null +++ b/scripts/checklist_template.md @@ -0,0 +1,21 @@ +{% macro display_line(i) %}- {% if i.completed == True %}[x] {% elif i.completed == False %}[ ] {% endif %}{{ i.name }}{% if i.pr != None %} {{ i.pr }}{% endif %}{% endmacro %} +# List of libraries + +{% for lib in update_libs %}{{ display_line(lib) }} +{% endfor %} + +# List of un-added libraries +These libraries are not added yet. Pure python one will be possible while others are not. + +{% for lib in add_libs %}{{ display_line(lib) }} +{% endfor %} + +# List of tests without python libraries + +{% for lib in update_tests %}{{ display_line(lib) }} +{% endfor %} + +# List of un-added tests without python libraries + +{% for lib in add_tests %}{{ display_line(lib) }} +{% endfor %} diff --git a/scripts/crawl_sourcecode.py b/scripts/crawl_sourcecode.py new file mode 100644 index 00000000000..30e01d04450 --- /dev/null +++ b/scripts/crawl_sourcecode.py @@ -0,0 +1,87 @@ +"""This script can be used to test the equivalence in parsing between +rustpython and cpython. + +Usage example: + +$ python crawl_sourcecode.py crawl_sourcecode.py > cpython.txt +$ cargo run crawl_sourcecode.py crawl_sourcecode.py > rustpython.txt +$ diff cpython.txt rustpython.txt +""" + +import ast +import dis +import symtable +import sys + +filename = sys.argv[1] +print("Crawling file:", filename) + + +with open(filename, "r") as f: + source = f.read() + +t = ast.parse(source) +print(t) + +shift = 3 + + +def print_node(node, indent=0): + indents = " " * indent + if isinstance(node, ast.AST): + lineno = "row={}".format(node.lineno) if hasattr(node, "lineno") else "" + print(indents, "NODE", node.__class__.__name__, lineno) + for field in node._fields: + print(indents, "-", field) + f = getattr(node, field) + if isinstance(f, list): + for f2 in f: + print_node(f2, indent=indent + shift) + else: + print_node(f, indent=indent + shift) + else: + print(indents, "OBJ", node) + + +print_node(t) + +# print(ast.dump(t)) +flag_names = [ + "is_referenced", + "is_assigned", + "is_global", + "is_local", + "is_parameter", + "is_free", +] + + +def print_table(table, indent=0): + indents = " " * indent + print(indents, "table:", table.get_name()) + print(indents, " ", "name:", table.get_name()) + print(indents, " ", "type:", table.get_type()) + print(indents, " ", "line:", table.get_lineno()) + print(indents, " ", "identifiers:", table.get_identifiers()) + print(indents, " ", "Syms:") + for sym in table.get_symbols(): + flags = [] + for flag_name in flag_names: + func = getattr(sym, flag_name) + if func(): + flags.append(flag_name) + print(indents, " sym:", sym.get_name(), "flags:", " ".join(flags)) + if table.has_children(): + print(indents, " ", "Child tables:") + for child in table.get_children(): + print_table(child, indent=indent + shift) + + +table = symtable.symtable(source, "a", "exec") +print_table(table) + +print() +print("======== dis.dis ========") +print() +co = compile(source, filename, "exec") +dis.dis(co) diff --git a/scripts/find_eq.py b/scripts/find_eq.py new file mode 100644 index 00000000000..b79982807b4 --- /dev/null +++ b/scripts/find_eq.py @@ -0,0 +1,95 @@ +# Run differential queries to find equivalent files in cpython and rustpython +# Arguments +# --cpython: Path to cpython source code +# --print-diff: Print the diff between the files +# --color: Output color +# --files: Optional globbing pattern to match files in cpython source code +# --checklist: output as checklist + +import argparse +import difflib +import pathlib + +parser = argparse.ArgumentParser( + description="Find equivalent files in cpython and rustpython" +) +parser.add_argument( + "--cpython", type=pathlib.Path, required=True, help="Path to cpython source code" +) +parser.add_argument( + "--print-diff", action="store_true", help="Print the diff between the files" +) +parser.add_argument("--color", action="store_true", help="Output color") +parser.add_argument( + "--files", + type=str, + default="*.py", + help="Optional globbing pattern to match files in cpython source code", +) + +args = parser.parse_args() + +if not args.cpython.exists(): + raise FileNotFoundError(f"Path {args.cpython} does not exist") +if not args.cpython.is_dir(): + raise NotADirectoryError(f"Path {args.cpython} is not a directory") +if not args.cpython.is_absolute(): + args.cpython = args.cpython.resolve() + +cpython_lib = args.cpython / "Lib" +rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib" +assert rustpython_lib.exists(), ( + "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place" +) + +# walk through the cpython lib directory +cpython_files = [] +for path in cpython_lib.rglob(args.files): + if path.is_file(): + # remove the cpython lib path from the file path + path = path.relative_to(cpython_lib) + cpython_files.append(path) + +for path in cpython_files: + # check if the file exists in the rustpython lib directory + rustpython_path = rustpython_lib / path + if rustpython_path.exists(): + # open both files and compare them + try: + with open(cpython_lib / path, "r") as cpython_file: + cpython_code = cpython_file.read() + with open(rustpython_lib / path, "r") as rustpython_file: + rustpython_code = rustpython_file.read() + # compare the files + diff = difflib.unified_diff( + cpython_code.splitlines(), + rustpython_code.splitlines(), + lineterm="", + fromfile=str(path), + tofile=str(path), + ) + # print the diff if there are differences + diff = list(diff) + if len(diff) > 0: + if args.print_diff: + print("Differences:") + for line in diff: + print(line) + else: + print(f"File is not identical: {path}") + else: + print(f"File is identical: {path}") + except Exception as e: + print(f"Unable to check file {path}: {e}") + else: + print(f"File not found in RustPython: {path}") + +# check for files in rustpython lib directory that are not in cpython lib directory +rustpython_files = [] +for path in rustpython_lib.rglob(args.files): + if path.is_file(): + # remove the rustpython lib path from the file path + path = path.relative_to(rustpython_lib) + rustpython_files.append(path) + if path not in cpython_files: + print(f"File not found in CPython: {path}") diff --git a/scripts/generate_checklist.py b/scripts/generate_checklist.py new file mode 100644 index 00000000000..759e4bd2a24 --- /dev/null +++ b/scripts/generate_checklist.py @@ -0,0 +1,268 @@ +# Arguments +# --cpython: Path to cpython source code +# --updated-libs: Libraries that have been updated in RustPython + + +import argparse +import dataclasses +import difflib +import pathlib +import warnings +from typing import Optional + +import requests +from jinja2 import Environment, FileSystemLoader + +parser = argparse.ArgumentParser( + description="Find equivalent files in cpython and rustpython" +) +parser.add_argument( + "--cpython", type=pathlib.Path, required=True, help="Path to cpython source code" +) +parser.add_argument( + "--notes", type=pathlib.Path, required=False, help="Path to notes file" +) + +args = parser.parse_args() + + +def check_pr(pr_id: str) -> bool: + if pr_id.startswith("#"): + pr_id = pr_id[1:] + int_pr_id = int(pr_id) + req = f"https://api.github.com/repos/RustPython/RustPython/pulls/{int_pr_id}" + response = requests.get(req).json() + return response["merged_at"] is not None + + +@dataclasses.dataclass +class LibUpdate: + pr: Optional[str] = None + done: bool = True + + +def parse_updated_lib_issue(issue_body: str) -> dict[str, LibUpdate]: + lines = issue_body.splitlines() + updated_libs = {} + for line in lines: + if line.strip().startswith("- "): + line = line.strip()[2:] + out = line.split(" ") + out = [x for x in out if x] + assert len(out) < 3 + if len(out) == 1: + updated_libs[out[0]] = LibUpdate() + elif len(out) == 2: + updated_libs[out[0]] = LibUpdate(out[1], check_pr(out[1])) + return updated_libs + + +def get_updated_libs() -> dict[str, LibUpdate]: + issue_id = "5736" + req = f"https://api.github.com/repos/RustPython/RustPython/issues/{issue_id}" + response = requests.get(req).json() + return parse_updated_lib_issue(response["body"]) + + +updated_libs = get_updated_libs() + +if not args.cpython.exists(): + raise FileNotFoundError(f"Path {args.cpython} does not exist") +if not args.cpython.is_dir(): + raise NotADirectoryError(f"Path {args.cpython} is not a directory") +if not args.cpython.is_absolute(): + args.cpython = args.cpython.resolve() + +notes: dict = {} +if args.notes: + # check if the file exists in the rustpython lib directory + notes_path = args.notes + if notes_path.exists(): + with open(notes_path) as f: + for line in f: + line = line.strip() + if not line.startswith("//") and line: + line_split = line.split(" ") + if len(line_split) > 1: + rest = " ".join(line_split[1:]) + if line_split[0] in notes: + notes[line_split[0]].append(rest) + else: + notes[line_split[0]] = [rest] + else: + raise ValueError(f"Invalid note: {line}") + + else: + raise FileNotFoundError(f"Path {notes_path} does not exist") + +cpython_lib = args.cpython / "Lib" +rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib" +assert rustpython_lib.exists(), ( + "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place" +) + +ignored_objs = ["__pycache__", "test"] +# loop through the top-level directories in the cpython lib directory +libs = [] +for path in cpython_lib.iterdir(): + if path.is_dir() and path.name not in ignored_objs: + # add the directory name to the list of libraries + libs.append(path.name) + elif path.is_file() and path.name.endswith(".py") and path.name not in ignored_objs: + # add the file name to the list of libraries + libs.append(path.name) + +tests = [] +cpython_lib_test = cpython_lib / "test" +for path in cpython_lib_test.iterdir(): + if ( + path.is_dir() + and path.name not in ignored_objs + and path.name.startswith("test_") + ): + # add the directory name to the list of libraries + tests.append(path.name) + elif ( + path.is_file() + and path.name.endswith(".py") + and path.name not in ignored_objs + and path.name.startswith("test_") + ): + # add the file name to the list of libraries + file_name = path.name.replace("test_", "") + if file_name not in libs and file_name.replace(".py", "") not in libs: + tests.append(path.name) + + +def check_diff(file1, file2): + try: + with open(file1, "r") as f1, open(file2, "r") as f2: + f1_lines = f1.readlines() + f2_lines = f2.readlines() + diff = difflib.unified_diff(f1_lines, f2_lines, lineterm="") + diff_lines = list(diff) + return len(diff_lines) + except UnicodeDecodeError: + return False + + +def check_completion_pr(display_name): + for lib in updated_libs: + if lib == str(display_name): + return updated_libs[lib].done, updated_libs[lib].pr + return False, None + + +def check_test_completion(rustpython_path, cpython_path): + if rustpython_path.exists() and rustpython_path.is_file(): + if cpython_path.exists() and cpython_path.is_file(): + if not rustpython_path.exists() or not rustpython_path.is_file(): + return False + elif check_diff(rustpython_path, cpython_path) > 0: + return False + return True + return False + + +def check_lib_completion(rustpython_path, cpython_path): + test_name = "test_" + rustpython_path.name + rustpython_test_path = rustpython_lib / "test" / test_name + cpython_test_path = cpython_lib / "test" / test_name + if cpython_test_path.exists() and not check_test_completion( + rustpython_test_path, cpython_test_path + ): + return False + if rustpython_path.exists() and rustpython_path.is_file(): + if check_diff(rustpython_path, cpython_path) > 0: + return False + return True + return False + + +def handle_notes(display_path) -> list[str]: + if str(display_path) in notes: + res = notes[str(display_path)] + # remove the note from the notes list + del notes[str(display_path)] + return res + return [] + + +@dataclasses.dataclass +class Output: + name: str + pr: Optional[str] + completed: Optional[bool] + notes: list[str] + + +update_libs_output = [] +add_libs_output = [] +for path in libs: + # check if the file exists in the rustpython lib directory + rustpython_path = rustpython_lib / path + # remove the file extension if it exists + display_path = pathlib.Path(path).with_suffix("") + (completed, pr) = check_completion_pr(display_path) + if rustpython_path.exists(): + if not completed: + # check if the file exists in the cpython lib directory + cpython_path = cpython_lib / path + # check if the file exists in the rustpython lib directory + if rustpython_path.exists() and rustpython_path.is_file(): + completed = check_lib_completion(rustpython_path, cpython_path) + update_libs_output.append( + Output(str(display_path), pr, completed, handle_notes(display_path)) + ) + else: + if pr is not None and completed: + update_libs_output.append( + Output(str(display_path), pr, None, handle_notes(display_path)) + ) + else: + add_libs_output.append( + Output(str(display_path), pr, None, handle_notes(display_path)) + ) + +update_tests_output = [] +add_tests_output = [] +for path in tests: + # check if the file exists in the rustpython lib directory + rustpython_path = rustpython_lib / "test" / path + # remove the file extension if it exists + display_path = pathlib.Path(path).with_suffix("") + (completed, pr) = check_completion_pr(display_path) + if rustpython_path.exists(): + if not completed: + # check if the file exists in the cpython lib directory + cpython_path = cpython_lib / "test" / path + # check if the file exists in the rustpython lib directory + if rustpython_path.exists() and rustpython_path.is_file(): + completed = check_lib_completion(rustpython_path, cpython_path) + update_tests_output.append( + Output(str(display_path), pr, completed, handle_notes(display_path)) + ) + else: + if pr is not None and completed: + update_tests_output.append( + Output(str(display_path), pr, None, handle_notes(display_path)) + ) + else: + add_tests_output.append( + Output(str(display_path), pr, None, handle_notes(display_path)) + ) + +for note in notes: + # add a warning for each note that is not attached to a file + for n in notes[note]: + warnings.warn(f"Unattached Note: {note} - {n}") + +env = Environment(loader=FileSystemLoader(".")) +template = env.get_template("checklist_template.md") +output = template.render( + update_libs=update_libs_output, + add_libs=add_libs_output, + update_tests=update_tests_output, + add_tests=add_tests_output, +) +print(output) diff --git a/scripts/generate_opcode_metadata.py b/scripts/generate_opcode_metadata.py new file mode 100644 index 00000000000..42fb55a7c01 --- /dev/null +++ b/scripts/generate_opcode_metadata.py @@ -0,0 +1,81 @@ +""" +Generate Lib/_opcode_metadata.py for RustPython bytecode. + +This file generates opcode metadata that is compatible with CPython 3.13. +""" + +import itertools +import pathlib +import re +import typing + +ROOT = pathlib.Path(__file__).parents[1] +BYTECODE_FILE = ( + ROOT / "crates" / "compiler-core" / "src" / "bytecode" / "instruction.rs" +) +OPCODE_METADATA_FILE = ROOT / "Lib" / "_opcode_metadata.py" + + +class Opcode(typing.NamedTuple): + rust_name: str + id: int + + @property + def cpython_name(self) -> str: + name = re.sub(r"(?<=[a-z0-9])([A-Z])", r"_\1", self.rust_name) + return re.sub(r"(\D)(\d+)$", r"\1_\2", name).upper() + + @classmethod + def from_str(cls, body: str): + raw_variants = re.split(r"(\d+),", body.strip()) + raw_variants.remove("") + for raw_name, raw_id in itertools.batched(raw_variants, 2): + name = re.findall(r"\b[A-Z][A-Za-z]*\d*\b(?=\s*[\({=])", raw_name)[0] + yield cls(rust_name=name.strip(), id=int(raw_id)) + + def __lt__(self, other: typing.Self) -> bool: + return self.id < other.id + + +def extract_enum_body(contents: str, enum_name: str) -> str: + res = re.search(f"pub enum {enum_name} " + r"\{(.+?)\n\}", contents, re.DOTALL) + if not res: + raise ValueError(f"Could not find {enum_name} enum") + + return "\n".join( + line.split("//")[0].strip() # Remove any comment. i.e. "foo // some comment" + for line in res.group(1).splitlines() + if not line.strip().startswith("//") # Ignore comment lines + ) + + +contents = BYTECODE_FILE.read_text(encoding="utf-8") +enum_body = "\n".join( + extract_enum_body(contents, enum_name) + for enum_name in ("Instruction", "PseudoInstruction") +) +opcodes = list(Opcode.from_str(enum_body)) + +# Generate the output file +output = """# This file is generated by scripts/generate_opcode_metadata.py +# for RustPython bytecode format (CPython 3.13 compatible opcode numbers). +# Do not edit! + +_specializations = {} + +_specialized_opmap = {} + +opmap = { +""" + +for opcode in sorted(opcodes): + output += f" '{opcode.cpython_name}': {opcode.id},\n" + +output += """} + +# CPython 3.13 compatible: opcodes < 44 have no argument +HAVE_ARGUMENT = 44 +MIN_INSTRUMENTED_OPCODE = 236 +""" + +OPCODE_METADATA_FILE.write_text(output, encoding="utf-8") diff --git a/scripts/generate_sre_constants.py b/scripts/generate_sre_constants.py new file mode 100644 index 00000000000..8e4091d2eb9 --- /dev/null +++ b/scripts/generate_sre_constants.py @@ -0,0 +1,139 @@ +#! /usr/bin/env python3 +# This script generates crates/sre_engine/src/constants.rs from Lib/re/_constants.py. + +SCRIPT_NAME = "scripts/generate_sre_constants.py" + + +def update_file(file, content): + try: + with open(file, "r") as fobj: + if fobj.read() == content: + return False + except (OSError, ValueError): + pass + with open(file, "w") as fobj: + fobj.write(content) + return True + + +sre_constants_header = f"""\ +/* + * Secret Labs' Regular Expression Engine + * + * regular expression matching engine + * + * Auto-generated by {SCRIPT_NAME} from + * Lib/re/_constants.py. + * + * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. + * + * See the sre.c file for information on usage and redistribution. + */ + +""" + + +def dump_enum(d, enum_name, derives, strip_prefix=""): + """Generate Rust enum definitions from a Python dictionary. + + Args: + d (list): The list containing the enum variants. + enum_name (str): The name of the enum to generate. + derives (str): The derive attributes to include. + strip_prefix (str, optional): A prefix to strip from the variant names. Defaults to "". + + Returns: + list: A list of strings representing the enum definition. + """ + items = sorted(d) + print(f"items is {items}") + content = [f"{derives}\n"] + content.append("#[repr(u32)]\n") + content.append("#[allow(non_camel_case_types, clippy::upper_case_acronyms)]\n") + content.append(f"pub enum {enum_name} {{\n") + for i, item in enumerate(items): + name = str(item).removeprefix(strip_prefix) + content.append(f" {name} = {i},\n") + content.append("}\n\n") + return content + + +def dump_bitflags(d, prefix, derives, struct_name, int_t): + """Generate Rust bitflags definitions from a Python dictionary. + + Args: + d (dict): The dictionary containing the bitflag variants. + prefix (str): The prefix to strip from the variant names. + derives (str): The derive attributes to include. + struct_name (str): The name of the struct to generate. + int_t (str): The integer type to use for the bitflags. + + Returns: + list: A list of strings representing the bitflags definition. + """ + items = [(value, name) for name, value in d.items() if name.startswith(prefix)] + content = ["bitflags! {\n"] + content.append(f"{derives}\n") if derives else None + content.append(f" pub struct {struct_name}: {int_t} {{\n") + for value, name in sorted(items): + name = str(name).removeprefix(prefix) + content.append(f" const {name} = {value};\n") + content.append(" }\n") + content.append("}\n\n") + return content + + +def main( + infile="Lib/re/_constants.py", + outfile_constants="crates/sre_engine/src/constants.rs", +): + ns = {} + with open(infile) as fp: + code = fp.read() + exec(code, ns) + + content = [sre_constants_header] + content.append("use bitflags::bitflags;\n\n") + content.append(f"pub const SRE_MAGIC: usize = {ns['MAGIC']};\n") + content.extend( + dump_enum( + ns["OPCODES"], + "SreOpcode", + "#[derive(num_enum::TryFromPrimitive, Debug, PartialEq, Eq)]", + ) + ) + content.extend( + dump_enum( + ns["ATCODES"], + "SreAtCode", + "#[derive(num_enum::TryFromPrimitive, Debug, PartialEq, Eq)]", + "AT_", + ) + ) + content.extend( + dump_enum( + ns["CHCODES"], + "SreCatCode", + "#[derive(num_enum::TryFromPrimitive, Debug)]", + "CATEGORY_", + ) + ) + + content.extend( + dump_bitflags( + ns, + "SRE_FLAG_", + "#[derive(Debug, PartialEq, Eq, Clone, Copy)]", + "SreFlag", + "u16", + ) + ) + content.extend(dump_bitflags(ns, "SRE_INFO_", "", "SreInfo", "u32")) + + update_file(outfile_constants, "".join(content)) + + +if __name__ == "__main__": + import sys + + main(*sys.argv[1:]) diff --git a/scripts/libc_posix.py b/scripts/libc_posix.py new file mode 100644 index 00000000000..82f3eb96d83 --- /dev/null +++ b/scripts/libc_posix.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python +import collections +import dataclasses +import pathlib +import re +import subprocess +import urllib.request + +import tomllib + +CPYTHON_VERSION = "3.14" + +CONSTS_PATTERN = re.compile(r"\b_*[A-Z]+(?:_+[A-Z]+)*_*\b") + +CARGO_TOML_FILE = pathlib.Path(__file__).parents[1] / "Cargo.toml" +CARGO_TOML = tomllib.loads(CARGO_TOML_FILE.read_text()) +LIBC_DATA = CARGO_TOML["workspace"]["dependencies"]["libc"] + + +LIBC_VERSION = LIBC_DATA["version"] if isinstance(LIBC_DATA, dict) else LIBC_DATA +BASE_URL = f"https://raw.githubusercontent.com/rust-lang/libc/refs/tags/{LIBC_VERSION}" + +EXCLUDE = frozenset( + { + # Defined at `vm/src/stdlib/os.rs` + "O_APPEND", + "O_CREAT", + "O_EXCL", + "O_RDONLY", + "O_RDWR", + "O_TRUNC", + "O_WRONLY", + "SEEK_CUR", + "SEEK_END", + "SEEK_SET", + # Functions, not consts + "WCOREDUMP", + "WIFCONTINUED", + "WIFSTOPPED", + "WIFSIGNALED", + "WIFEXITED", + "WEXITSTATUS", + "WSTOPSIG", + "WTERMSIG", + # False positive + # "EOF", + } +) + + +def rustfmt(code: str) -> str: + return subprocess.check_output(["rustfmt", "--emit=stdout"], input=code, text=True) + + +@dataclasses.dataclass(eq=True, frozen=True, slots=True) +class Cfg: + inner: str + + def __str__(self) -> str: + return self.inner + + def __lt__(self, other) -> bool: + si, oi = map(str, (self.inner, other.inner)) + + # Smaller length cfgs are smaller, regardless of value. + return (len(si), si) < (len(oi), oi) + + +@dataclasses.dataclass(frozen=True, kw_only=True, slots=True) +class Target: + cfgs: set[Cfg] + sources: set[str] = dataclasses.field(default_factory=set) + extras: set[str] = dataclasses.field(default_factory=set) + + +TARGETS = ( + Target( + cfgs={Cfg('target_os = "android"')}, + sources={ + f"{BASE_URL}/src/unix/linux_like/android/mod.rs", + f"{BASE_URL}/libc-test/semver/android.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "dragonfly"')}, + sources={ + f"{BASE_URL}/src/unix/bsd/freebsdlike/dragonfly/mod.rs", + f"{BASE_URL}/libc-test/semver/dragonfly.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "freebsd"')}, + sources={ + f"{BASE_URL}/src/unix/bsd/freebsdlike/freebsd/mod.rs", + f"{BASE_URL}/libc-test/semver/freebsd.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "linux"')}, + sources={ + f"{BASE_URL}/src/unix/linux_like/mod.rs", + f"{BASE_URL}/src/unix/linux_like/linux_l4re_shared.rs", + f"{BASE_URL}/libc-test/semver/linux.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "macos"')}, + sources={ + f"{BASE_URL}/src/unix/bsd/apple/mod.rs", + f"{BASE_URL}/libc-test/semver/apple.txt", + }, + extras={"COPYFILE_DATA as _COPYFILE_DATA"}, + ), + Target( + cfgs={Cfg('target_os = "netbsd"')}, + sources={ + f"{BASE_URL}/src/unix/bsd/netbsdlike/netbsd/mod.rs", + f"{BASE_URL}/libc-test/semver/netbsd.txt", + }, + ), + Target( + cfgs={Cfg('target_os = "redox"')}, + sources={ + f"{BASE_URL}/src/unix/redox/mod.rs", + f"{BASE_URL}/libc-test/semver/redox.txt", + }, + ), + Target(cfgs={Cfg("unix")}, sources={f"{BASE_URL}/libc-test/semver/unix.txt"}), +) + + +def extract_consts( + contents: str, + *, + pattern: re.Pattern = CONSTS_PATTERN, + exclude: frozenset[str] = EXCLUDE, +) -> frozenset[str]: + """ + Extract all words that are comprised from only uppercase letters + underscores. + + Parameters + ---------- + contents : str + Contents to extract the constants from. + pattern : re.Pattern, Optional + RE compiled pattern for extracting the consts. + exclude : frozenset[str], Optional + Items to exclude from the returned value. + + Returns + ------- + frozenset[str] + All constant names. + """ + result = frozenset(pattern.findall(contents)) + return result - exclude + + +def consts_from_url( + url: str, *, pattern: re.Pattern = CONSTS_PATTERN, exclude: frozenset[str] = EXCLUDE +) -> frozenset[str]: + """ + Extract all consts from the contents found at the given URL. + + Parameters + ---------- + url : str + URL to fetch the contents from. + pattern : re.Pattern, Optional + RE compiled pattern for extracting the consts. + exclude : frozenset[str], Optional + Items to exclude from the returned value. + + Returns + ------- + frozenset[str] + All constant names at the URL. + """ + try: + with urllib.request.urlopen(url) as f: + contents = f.read().decode() + except urllib.error.HTTPError as err: + err.add_note(url) + raise + + return extract_consts(contents, pattern=pattern, exclude=exclude) + + +def main(): + # Step 1: Get all OS contants that we do want from upstream + wanted_consts = consts_from_url( + f"https://docs.python.org/{CPYTHON_VERSION}/library/os.html", + # TODO: Exclude matches if they have `(` after (those are functions) + pattern=re.compile(r"\bos\.(_*[A-Z]+(?:_+[A-Z]+)*_*)"), + ) + + # Step 2: build dict of what consts are available per cfg. `cfg -> {consts}` + available = collections.defaultdict(set) + for target in TARGETS: + consts = set() + for source in target.sources: + consts |= consts_from_url(source) + + for cfg in target.cfgs: + available[cfg] |= consts + + # Step 3: Keep only the "wanted" consts. Build a groupped mapping of `{cfgs} -> {consts}' + groups = collections.defaultdict(set) + available_items = available.items() + for wanted_const in wanted_consts: + cfgs = frozenset( + cfg for cfg, consts in available_items if wanted_const in consts + ) + if not cfgs: + # We have no cfgs for a wanted const :/ + continue + + groups[cfgs].add(wanted_const) + + # Step 4: Build output + output = "" + for cfgs, consts in sorted(groups.items(), key=lambda t: (len(t[0]), sorted(t[0]))): + target = next((target for target in TARGETS if target.cfgs == cfgs), None) + if target: + # If we found an exact target. Add its "extras" as-is + consts |= target.extras + + cfgs_inner = ",".join(sorted(map(str, cfgs))) + + if len(cfgs) >= 2: + cfgs_rust = f"#[cfg(any({cfgs_inner}))]" + else: + cfgs_rust = f"#[cfg({cfgs_inner})]" + + imports = ",".join(consts) + entry = f""" +{cfgs_rust} +#[pyattr] +use libc::{{{imports}}}; +""".strip() + + output += f"{entry}\n\n" + + print(rustfmt(output)) + + +if __name__ == "__main__": + main() diff --git a/scripts/make_ssl_data_rs.py b/scripts/make_ssl_data_rs.py new file mode 100755 index 00000000000..a089fe6ee0c --- /dev/null +++ b/scripts/make_ssl_data_rs.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +""" +Generate Rust SSL error mapping code from OpenSSL sources. + +This is based on CPython's Tools/ssl/make_ssl_data.py but generates +Rust code instead of C headers. + +It takes two arguments: +- the path to the OpenSSL source tree (e.g. git checkout) +- the path to the Rust file to be generated (e.g. stdlib/src/ssl/ssl_data.rs) +- error codes are version specific +""" + +import argparse +import datetime +import operator +import os +import re +import sys + +parser = argparse.ArgumentParser( + description="Generate ssl_data.rs from OpenSSL sources" +) +parser.add_argument("srcdir", help="OpenSSL source directory") +parser.add_argument("output", nargs="?", default=None) + + +def _file_search(fname, pat): + with open(fname, encoding="utf-8") as f: + for line in f: + match = pat.search(line) + if match is not None: + yield match + + +def parse_err_h(args): + """Parse err codes, e.g. ERR_LIB_X509: 11""" + pat = re.compile(r"#\s*define\W+ERR_LIB_(\w+)\s+(\d+)") + lib2errnum = {} + for match in _file_search(args.err_h, pat): + libname, num = match.groups() + lib2errnum[libname] = int(num) + + return lib2errnum + + +def parse_openssl_error_text(args): + """Parse error reasons, X509_R_AKID_MISMATCH""" + # ignore backslash line continuation for now + pat = re.compile(r"^((\w+?)_R_(\w+)):(\d+):") + for match in _file_search(args.errtxt, pat): + reason, libname, errname, num = match.groups() + if "_F_" in reason: + # ignore function codes + continue + num = int(num) + yield reason, libname, errname, num + + +def parse_extra_reasons(args): + """Parse extra reasons from openssl.ec""" + pat = re.compile(r"^R\s+((\w+)_R_(\w+))\s+(\d+)") + for match in _file_search(args.errcodes, pat): + reason, libname, errname, num = match.groups() + num = int(num) + yield reason, libname, errname, num + + +def gen_library_codes_rust(args): + """Generate Rust phf map for library codes""" + yield "// Maps lib_code -> library name" + yield '// Example: 20 -> "SSL"' + yield "pub static LIBRARY_CODES: phf::Map<u32, &'static str> = phf_map! {" + + # Deduplicate: keep the last one if there are duplicates + seen = {} + for libname in sorted(args.lib2errnum): + lib_num = args.lib2errnum[libname] + seen[lib_num] = libname + + for lib_num in sorted(seen.keys()): + libname = seen[lib_num] + yield f' {lib_num}u32 => "{libname}",' + yield "};" + yield "" + + +def gen_error_codes_rust(args): + """Generate Rust phf map for error codes""" + yield "// Maps encoded (lib, reason) -> error mnemonic" + yield '// Example: encode_error_key(20, 134) -> "CERTIFICATE_VERIFY_FAILED"' + yield "// Key encoding: (lib << 32) | reason" + yield "pub static ERROR_CODES: phf::Map<u64, &'static str> = phf_map! {" + for reason, libname, errname, num in args.reasons: + if libname not in args.lib2errnum: + continue + lib_num = args.lib2errnum[libname] + # Encode (lib, reason) as single u64 + key = (lib_num << 32) | num + yield f' {key}u64 => "{errname}",' + yield "};" + yield "" + + +def main(): + args = parser.parse_args() + + args.err_h = os.path.join(args.srcdir, "include", "openssl", "err.h") + if not os.path.isfile(args.err_h): + # Fall back to infile for OpenSSL 3.0.0 + args.err_h += ".in" + args.errcodes = os.path.join(args.srcdir, "crypto", "err", "openssl.ec") + args.errtxt = os.path.join(args.srcdir, "crypto", "err", "openssl.txt") + + if not os.path.isfile(args.errtxt): + parser.error(f"File {args.errtxt} not found in srcdir\n.") + + # {X509: 11, ...} + args.lib2errnum = parse_err_h(args) + + # [('X509_R_AKID_MISMATCH', 'X509', 'AKID_MISMATCH', 110), ...] + reasons = [] + reasons.extend(parse_openssl_error_text(args)) + reasons.extend(parse_extra_reasons(args)) + # sort by libname, numeric error code + args.reasons = sorted(reasons, key=operator.itemgetter(0, 3)) + + lines = [ + "// File generated by tools/make_ssl_data_rs.py", + f"// Generated on {datetime.datetime.now(datetime.timezone.utc).isoformat()}", + f"// Source: OpenSSL from {args.srcdir}", + "// spell-checker: disable", + "", + "use phf::phf_map;", + "", + ] + lines.extend(gen_library_codes_rust(args)) + lines.extend(gen_error_codes_rust(args)) + + # Add helper function + lines.extend( + [ + "/// Helper function to create encoded key from (lib, reason) pair", + "#[inline]", + "pub fn encode_error_key(lib: i32, reason: i32) -> u64 {", + " ((lib as u64) << 32) | (reason as u64 & 0xFFFFFFFF)", + "}", + "", + ] + ) + + if args.output is None: + for line in lines: + print(line) + else: + with open(args.output, "w") as output: + for line in lines: + print(line, file=output) + + print(f"Generated {args.output}") + print(f"Found {len(args.lib2errnum)} library codes") + print(f"Found {len(args.reasons)} error codes") + + +if __name__ == "__main__": + main() diff --git a/scripts/notes.txt b/scripts/notes.txt new file mode 100644 index 00000000000..d781f7c7bf2 --- /dev/null +++ b/scripts/notes.txt @@ -0,0 +1,48 @@ +__future__ Related test is `test_future_stmt` +abc `_collections_abc.py` +abc `_py_abc.py` +code Related test is `test_code_module` +codecs `_pycodecs.py` +collections See also #3418 +ctypes #5572 +datetime `_pydatetime.py` +decimal `_pydecimal.py` +dis See also #3846 +importlib #4565 +io `_pyio.py` +io #3960 +io #4702 +locale #3850 +mailbox #4072 +multiprocessing #3965 +os Blocker: Some tests requires async comprehension +os #3960 +os #4053 +pickle #3876 +pickle `_compat_pickle.py` +pickle `test/pickletester.py` supports `test_pickle.py` +pickle `test/test_picklebuffer.py` +pydoc `pydoc_data` +queue See also #3608 +re Don't forget sre files `sre_compile.py`, `sre_constants.py`, `sre_parse.py` +shutil #3960 +site Don't forget `_sitebuiltins.py` +venv #3960 +warnings #4013 + +// test + +test_array #3876 +test_gc #4158 +test_marshal #3458 +test_mmap #3847 +test_posix #4496 +test_property #3430 +test_set #3992 +test_structseq #4063 +test_super #3865 +test_support #4538 +test_syntax #4469 +test_sys #4541 +test_time #3850 +test_time #4157 \ No newline at end of file diff --git a/scripts/release-wapm.sh b/scripts/release-wapm.sh index 1b8e2ecfe2b..62171fbe61c 100644 --- a/scripts/release-wapm.sh +++ b/scripts/release-wapm.sh @@ -7,6 +7,6 @@ FEATURES_FOR_WAPM=(stdlib zlib) export BUILDTIME_RUSTPYTHONPATH="/lib/rustpython" -cargo build --release --target wasm32-wasi --no-default-features --features="${FEATURES_FOR_WAPM[*]}" +cargo build --release --target wasm32-wasip1 --no-default-features --features="${FEATURES_FOR_WAPM[*]}" wapm publish diff --git a/scripts/update_lib/.gitignore b/scripts/update_lib/.gitignore new file mode 100644 index 00000000000..ceddaa37f12 --- /dev/null +++ b/scripts/update_lib/.gitignore @@ -0,0 +1 @@ +.cache/ diff --git a/scripts/update_lib/__init__.py b/scripts/update_lib/__init__.py new file mode 100644 index 00000000000..ccb2628d6a4 --- /dev/null +++ b/scripts/update_lib/__init__.py @@ -0,0 +1,37 @@ +""" +Library for updating Python test files with RustPython-specific patches. +""" + +from .patch_spec import ( + COMMENT, + DEFAULT_INDENT, + UT, + PatchEntry, + Patches, + PatchSpec, + UtMethod, + apply_patches, + build_patch_dict, + extract_patches, + iter_patches, + iter_tests, + patches_from_json, + patches_to_json, +) + +__all__ = [ + "COMMENT", + "DEFAULT_INDENT", + "UT", + "Patches", + "PatchEntry", + "PatchSpec", + "UtMethod", + "apply_patches", + "build_patch_dict", + "extract_patches", + "iter_patches", + "iter_tests", + "patches_from_json", + "patches_to_json", +] diff --git a/scripts/update_lib/__main__.py b/scripts/update_lib/__main__.py new file mode 100644 index 00000000000..49399db6f43 --- /dev/null +++ b/scripts/update_lib/__main__.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +""" +Update library tools for RustPython. + +Usage: + python scripts/update_lib quick cpython/Lib/test/test_foo.py + python scripts/update_lib copy-lib cpython/Lib/dataclasses.py + python scripts/update_lib migrate cpython/Lib/test/test_foo.py + python scripts/update_lib patches --from Lib/test/foo.py --to cpython/Lib/test/foo.py + python scripts/update_lib auto-mark Lib/test/test_foo.py +""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Update library tools for RustPython", + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser( + "quick", + help="Quick update: patch + auto-mark (recommended)", + add_help=False, + ) + subparsers.add_parser( + "migrate", + help="Migrate test file(s) from CPython, preserving RustPython markers", + add_help=False, + ) + subparsers.add_parser( + "patches", + help="Patch management (extract/apply patches between files)", + add_help=False, + ) + subparsers.add_parser( + "auto-mark", + help="Run tests and auto-mark failures with @expectedFailure", + add_help=False, + ) + subparsers.add_parser( + "copy-lib", + help="Copy library file/directory from CPython (delete existing first)", + add_help=False, + ) + subparsers.add_parser( + "deps", + help="Show dependency information for a module", + add_help=False, + ) + subparsers.add_parser( + "todo", + help="Show prioritized list of modules to update", + add_help=False, + ) + + args, remaining = parser.parse_known_args(argv) + + if args.command == "quick": + from update_lib.cmd_quick import main as quick_main + + return quick_main(remaining) + + if args.command == "copy-lib": + from update_lib.cmd_copy_lib import main as copy_lib_main + + return copy_lib_main(remaining) + + if args.command == "migrate": + from update_lib.cmd_migrate import main as migrate_main + + return migrate_main(remaining) + + if args.command == "patches": + from update_lib.cmd_patches import main as patches_main + + return patches_main(remaining) + + if args.command == "auto-mark": + from update_lib.cmd_auto_mark import main as cmd_auto_mark_main + + return cmd_auto_mark_main(remaining) + + if args.command == "deps": + from update_lib.cmd_deps import main as cmd_deps_main + + return cmd_deps_main(remaining) + + if args.command == "todo": + from update_lib.cmd_todo import main as cmd_todo_main + + return cmd_todo_main(remaining) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_auto_mark.py b/scripts/update_lib/cmd_auto_mark.py new file mode 100644 index 00000000000..c77cbf300f1 --- /dev/null +++ b/scripts/update_lib/cmd_auto_mark.py @@ -0,0 +1,1044 @@ +#!/usr/bin/env python +""" +Auto-mark test failures in Python test suite. + +This module provides functions to: +- Run tests with RustPython and parse results +- Extract test names from test file paths +- Mark failing tests with @unittest.expectedFailure +- Remove expectedFailure from tests that now pass +""" + +import ast +import pathlib +import re +import subprocess +import sys +from dataclasses import dataclass, field + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from update_lib import COMMENT, PatchSpec, UtMethod, apply_patches +from update_lib.file_utils import get_test_module_name + + +class TestRunError(Exception): + """Raised when test run fails entirely (e.g., import error, crash).""" + + pass + + +@dataclass +class Test: + name: str = "" + path: str = "" + result: str = "" + error_message: str = "" + + +@dataclass +class TestResult: + tests_result: str = "" + tests: list[Test] = field(default_factory=list) + unexpected_successes: list[Test] = field(default_factory=list) + stdout: str = "" + + +def run_test(test_name: str, skip_build: bool = False) -> TestResult: + """ + Run a test with RustPython and return parsed results. + + Args: + test_name: Test module name (e.g., "test_foo" or "test_ctypes.test_bar") + skip_build: If True, use pre-built binary instead of cargo run + + Returns: + TestResult with parsed test results + """ + if skip_build: + cmd = ["./target/release/rustpython"] + if sys.platform == "win32": + cmd = ["./target/release/rustpython.exe"] + else: + cmd = ["cargo", "run", "--release", "--"] + + result = subprocess.run( + cmd + ["-m", "test", "-v", "-u", "all", "--slowest", test_name], + stdout=subprocess.PIPE, # Capture stdout for parsing + stderr=None, # Let stderr pass through to terminal + text=True, + ) + return parse_results(result) + + +def _try_parse_test_info(test_info: str) -> tuple[str, str] | None: + """Try to extract (name, path) from 'test_name (path)' or 'test_name (path) [subtest]'.""" + first_space = test_info.find(" ") + if first_space > 0: + name = test_info[:first_space] + rest = test_info[first_space:].strip() + if rest.startswith("("): + end_paren = rest.find(")") + if end_paren > 0: + return name, rest[1:end_paren] + return None + + +def parse_results(result: subprocess.CompletedProcess) -> TestResult: + """Parse subprocess result into TestResult.""" + lines = result.stdout.splitlines() + test_results = TestResult() + test_results.stdout = result.stdout + in_test_results = False + # For multiline format: "test_name (path)\ndocstring ... RESULT" + pending_test_info = None + + for line in lines: + if re.search(r"Run \d+ tests? sequentially", line): + in_test_results = True + elif "== Tests result: " in line: + in_test_results = False + + if in_test_results and " ... " in line: + stripped = line.strip() + # Skip lines that don't look like test results + if stripped.startswith("tests") or stripped.startswith("["): + pending_test_info = None + continue + # Parse: "test_name (path) [subtest] ... RESULT" + parts = stripped.split(" ... ") + if len(parts) >= 2: + test_info = parts[0] + result_str = parts[-1].lower() + # Only process FAIL or ERROR + if result_str not in ("fail", "error"): + pending_test_info = None + continue + # Try parsing from this line (single-line format) + parsed = _try_parse_test_info(test_info) + if not parsed and pending_test_info: + # Multiline format: previous line had test_name (path) + parsed = _try_parse_test_info(pending_test_info) + if parsed: + test = Test() + test.name, test.path = parsed + test.result = result_str + test_results.tests.append(test) + pending_test_info = None + + elif in_test_results: + # Track test info for multiline format: + # test_name (path) + # docstring ... RESULT + stripped = line.strip() + if ( + stripped + and "(" in stripped + and stripped.endswith(")") + and ":" not in stripped.split("(")[0] + ): + pending_test_info = stripped + else: + pending_test_info = None + + # Also check for Tests result on non-" ... " lines + if "== Tests result: " in line: + res = line.split("== Tests result: ")[1] + res = res.split(" ")[0] + test_results.tests_result = res + + elif "== Tests result: " in line: + res = line.split("== Tests result: ")[1] + res = res.split(" ")[0] + test_results.tests_result = res + + # Parse: "UNEXPECTED SUCCESS: test_name (path)" + if line.startswith("UNEXPECTED SUCCESS: "): + rest = line[len("UNEXPECTED SUCCESS: ") :] + # Format: "test_name (path)" + first_space = rest.find(" ") + if first_space > 0: + test = Test() + test.name = rest[:first_space] + path_part = rest[first_space:].strip() + if path_part.startswith("(") and path_part.endswith(")"): + test.path = path_part[1:-1] + test.result = "unexpected_success" + test_results.unexpected_successes.append(test) + + # Parse error details to extract error messages + _parse_error_details(test_results, lines) + + return test_results + + +def _parse_error_details(test_results: TestResult, lines: list[str]) -> None: + """Parse error details section to extract error messages for each test.""" + # Build a lookup dict for tests by (name, path) + test_lookup: dict[tuple[str, str], Test] = {} + for test in test_results.tests: + test_lookup[(test.name, test.path)] = test + + # Parse error detail blocks + # Format: + # ====================================================================== + # FAIL: test_name (path) + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # ... + # AssertionError: message + # + # ====================================================================== + i = 0 + while i < len(lines): + line = lines[i] + # Look for FAIL: or ERROR: header + if line.startswith(("FAIL: ", "ERROR: ")): + # Parse: "FAIL: test_name (path)" or "ERROR: test_name (path)" + header = line.split(": ", 1)[1] if ": " in line else "" + first_space = header.find(" ") + if first_space > 0: + test_name = header[:first_space] + path_part = header[first_space:].strip() + if path_part.startswith("(") and path_part.endswith(")"): + test_path = path_part[1:-1] + + # Find the last non-empty line before the next separator or end + error_lines = [] + i += 1 + # Skip the separator line + if i < len(lines) and lines[i].startswith("-----"): + i += 1 + + # Collect lines until the next separator or end + while i < len(lines): + current = lines[i] + if current.startswith("=====") or current.startswith("-----"): + break + error_lines.append(current) + i += 1 + + # Find the last non-empty line (the error message) + error_message = "" + for err_line in reversed(error_lines): + stripped = err_line.strip() + if stripped: + error_message = stripped + break + + # Update the test with the error message + if (test_name, test_path) in test_lookup: + test_lookup[ + (test_name, test_path) + ].error_message = error_message + + continue + i += 1 + + +def path_to_test_parts(path: str) -> list[str]: + """ + Extract [ClassName, method_name] from test path. + + Args: + path: Test path like "test.module_name.ClassName.test_method" + + Returns: + [ClassName, method_name] - last 2 elements + """ + parts = path.split(".") + return parts[-2:] + + +def _expand_stripped_to_children( + contents: str, + stripped_tests: set[tuple[str, str]], + all_failing_tests: set[tuple[str, str]], +) -> set[tuple[str, str]]: + """Find child-class failures that correspond to stripped parent-class markers. + + When ``strip_reasonless_expected_failures`` removes a marker from a parent + (mixin) class, test failures are reported against the concrete subclasses, + not the parent itself. This function maps those child failures back so + they get re-marked (and later consolidated to the parent by + ``_consolidate_to_parent``). + + Returns the set of ``(class, method)`` pairs from *all_failing_tests* that + should be re-marked. + """ + # Direct matches (stripped test itself is a concrete TestCase) + result = stripped_tests & all_failing_tests + + unmatched = stripped_tests - all_failing_tests + if not unmatched: + return result + + tree = ast.parse(contents) + class_bases, class_methods = _build_inheritance_info(tree) + + for parent_cls, method_name in unmatched: + if method_name not in class_methods.get(parent_cls, set()): + continue + for cls in _find_all_inheritors( + parent_cls, method_name, class_bases, class_methods + ): + if (cls, method_name) in all_failing_tests: + result.add((cls, method_name)) + + return result + + +def _consolidate_to_parent( + contents: str, + failing_tests: set[tuple[str, str]], + error_messages: dict[tuple[str, str], str] | None = None, +) -> tuple[set[tuple[str, str]], dict[tuple[str, str], str] | None]: + """Move failures to the parent class when ALL inheritors fail. + + If every concrete subclass that inherits a method from a parent class + appears in *failing_tests*, replace those per-subclass entries with a + single entry on the parent. This avoids creating redundant super-call + overrides in every child. + + Returns: + (consolidated_failing_tests, consolidated_error_messages) + """ + tree = ast.parse(contents) + class_bases, class_methods = _build_inheritance_info(tree) + + # Group by (defining_parent, method) → set of failing children + from collections import defaultdict + + groups: dict[tuple[str, str], set[str]] = defaultdict(set) + for class_name, method_name in failing_tests: + defining = _find_method_definition( + class_name, method_name, class_bases, class_methods + ) + if defining and defining != class_name: + groups[(defining, method_name)].add(class_name) + + if not groups: + return failing_tests, error_messages + + result = set(failing_tests) + new_error_messages = dict(error_messages) if error_messages else {} + + for (parent, method_name), failing_children in groups.items(): + all_inheritors = _find_all_inheritors( + parent, method_name, class_bases, class_methods + ) + + if all_inheritors and failing_children >= all_inheritors: + # All inheritors fail → mark on parent instead + children_keys = {(child, method_name) for child in failing_children} + result -= children_keys + result.add((parent, method_name)) + # Pick any child's error message for the parent + if new_error_messages: + for child in failing_children: + msg = new_error_messages.pop((child, method_name), "") + if msg: + new_error_messages[(parent, method_name)] = msg + + return result, new_error_messages or error_messages + + +def build_patches( + test_parts_set: set[tuple[str, str]], + error_messages: dict[tuple[str, str], str] | None = None, +) -> dict: + """Convert failing tests to patch format.""" + patches = {} + error_messages = error_messages or {} + for class_name, method_name in sorted(test_parts_set): + if class_name not in patches: + patches[class_name] = {} + reason = error_messages.get((class_name, method_name), "") + patches[class_name][method_name] = [ + PatchSpec(UtMethod.ExpectedFailure, None, reason) + ] + return patches + + +def _is_super_call_only(func_node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: + """Check if the method body is just 'return super().method_name()' or 'return await super().method_name()'.""" + if len(func_node.body) != 1: + return False + stmt = func_node.body[0] + if not isinstance(stmt, ast.Return) or stmt.value is None: + return False + call = stmt.value + # Unwrap await for async methods + if isinstance(call, ast.Await): + call = call.value + if not isinstance(call, ast.Call): + return False + if not isinstance(call.func, ast.Attribute): + return False + # Verify the method name matches + if call.func.attr != func_node.name: + return False + super_call = call.func.value + if not isinstance(super_call, ast.Call): + return False + if not isinstance(super_call.func, ast.Name) or super_call.func.id != "super": + return False + return True + + +def _method_removal_range( + func_node: ast.FunctionDef | ast.AsyncFunctionDef, lines: list[str] +) -> range: + """Line range covering an entire method including decorators and a preceding COMMENT line.""" + first = ( + func_node.decorator_list[0].lineno - 1 + if func_node.decorator_list + else func_node.lineno - 1 + ) + if ( + first > 0 + and lines[first - 1].strip().startswith("#") + and COMMENT in lines[first - 1] + ): + first -= 1 + # Also remove a preceding blank line to avoid double-blanks after removal + if first > 0 and not lines[first - 1].strip(): + first -= 1 + return range(first, func_node.end_lineno) + + +def _build_inheritance_info(tree: ast.Module) -> tuple[dict, dict]: + """ + Build inheritance information from AST. + + Returns: + class_bases: dict[str, list[str]] - parent classes for each class + class_methods: dict[str, set[str]] - methods directly defined in each class + """ + all_classes = { + node.name for node in ast.walk(tree) if isinstance(node, ast.ClassDef) + } + class_bases = {} + class_methods = {} + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + bases = [ + base.id + for base in node.bases + if isinstance(base, ast.Name) and base.id in all_classes + ] + class_bases[node.name] = bases + methods = { + item.name + for item in node.body + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + class_methods[node.name] = methods + + return class_bases, class_methods + + +def _find_method_definition( + class_name: str, method_name: str, class_bases: dict, class_methods: dict +) -> str | None: + """Find the class where a method is actually defined (BFS).""" + if method_name in class_methods.get(class_name, set()): + return class_name + + visited = set() + queue = list(class_bases.get(class_name, [])) + + while queue: + current = queue.pop(0) + if current in visited: + continue + visited.add(current) + + if method_name in class_methods.get(current, set()): + return current + queue.extend(class_bases.get(current, [])) + + return None + + +def _find_all_inheritors( + parent: str, method_name: str, class_bases: dict, class_methods: dict +) -> set[str]: + """Find all classes that inherit *method_name* from *parent* (not overriding it).""" + return { + cls + for cls in class_bases + if cls != parent + and method_name not in class_methods.get(cls, set()) + and _find_method_definition(cls, method_name, class_bases, class_methods) + == parent + } + + +def remove_expected_failures( + contents: str, tests_to_remove: set[tuple[str, str]] +) -> str: + """Remove @unittest.expectedFailure decorators from tests that now pass.""" + if not tests_to_remove: + return contents + + tree = ast.parse(contents) + lines = contents.splitlines() + lines_to_remove = set() + + class_bases, class_methods = _build_inheritance_info(tree) + + resolved_tests = set() + for class_name, method_name in tests_to_remove: + defining_class = _find_method_definition( + class_name, method_name, class_bases, class_methods + ) + if defining_class: + resolved_tests.add((defining_class, method_name)) + + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + class_name = node.name + for item in node.body: + if not isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + method_name = item.name + if (class_name, method_name) not in resolved_tests: + continue + + remove_entire_method = _is_super_call_only(item) + + if remove_entire_method: + lines_to_remove.update(_method_removal_range(item, lines)) + else: + for dec in item.decorator_list: + dec_line = dec.lineno - 1 + line_content = lines[dec_line] + + if "expectedFailure" not in line_content: + continue + + has_comment_on_line = COMMENT in line_content + has_comment_before = ( + dec_line > 0 + and lines[dec_line - 1].strip().startswith("#") + and COMMENT in lines[dec_line - 1] + ) + has_comment_after = ( + dec_line + 1 < len(lines) + and lines[dec_line + 1].strip().startswith("#") + and COMMENT not in lines[dec_line + 1] + ) + + if has_comment_on_line or has_comment_before: + lines_to_remove.add(dec_line) + if has_comment_before: + lines_to_remove.add(dec_line - 1) + if has_comment_after and has_comment_on_line: + lines_to_remove.add(dec_line + 1) + + for line_idx in sorted(lines_to_remove, reverse=True): + del lines[line_idx] + + return "\n".join(lines) + "\n" if lines else "" + + +def collect_test_changes( + results: TestResult, + module_prefix: str | None = None, +) -> tuple[set[tuple[str, str]], set[tuple[str, str]], dict[tuple[str, str], str]]: + """ + Collect failing tests and unexpected successes from test results. + + Args: + results: TestResult from run_test() + module_prefix: If set, only collect tests whose path starts with this prefix + + Returns: + (failing_tests, unexpected_successes, error_messages) + - failing_tests: set of (class_name, method_name) tuples + - unexpected_successes: set of (class_name, method_name) tuples + - error_messages: dict mapping (class_name, method_name) to error message + """ + failing_tests = set() + error_messages: dict[tuple[str, str], str] = {} + for test in results.tests: + if test.result in ("fail", "error"): + if module_prefix and not test.path.startswith(module_prefix): + continue + test_parts = path_to_test_parts(test.path) + if len(test_parts) == 2: + key = tuple(test_parts) + failing_tests.add(key) + if test.error_message: + error_messages[key] = test.error_message + + unexpected_successes = set() + for test in results.unexpected_successes: + if module_prefix and not test.path.startswith(module_prefix): + continue + test_parts = path_to_test_parts(test.path) + if len(test_parts) == 2: + unexpected_successes.add(tuple(test_parts)) + + return failing_tests, unexpected_successes, error_messages + + +def apply_test_changes( + contents: str, + failing_tests: set[tuple[str, str]], + unexpected_successes: set[tuple[str, str]], + error_messages: dict[tuple[str, str], str] | None = None, +) -> str: + """ + Apply test changes to content. + + Args: + contents: File content + failing_tests: Set of (class_name, method_name) to mark as expectedFailure + unexpected_successes: Set of (class_name, method_name) to remove expectedFailure + error_messages: Dict mapping (class_name, method_name) to error message + + Returns: + Modified content + """ + if unexpected_successes: + contents = remove_expected_failures(contents, unexpected_successes) + + if failing_tests: + failing_tests, error_messages = _consolidate_to_parent( + contents, failing_tests, error_messages + ) + patches = build_patches(failing_tests, error_messages) + contents = apply_patches(contents, patches) + + return contents + + +def strip_reasonless_expected_failures( + contents: str, +) -> tuple[str, set[tuple[str, str]]]: + """Strip @expectedFailure decorators that have no failure reason. + + Markers like ``@unittest.expectedFailure # TODO: RUSTPYTHON`` (without a + reason after the semicolon) are removed so the tests fail normally during + the next test run and error messages can be captured. + + Returns: + (modified_contents, stripped_tests) where stripped_tests is a set of + (class_name, method_name) tuples whose markers were removed. + """ + tree = ast.parse(contents) + lines = contents.splitlines() + stripped_tests: set[tuple[str, str]] = set() + lines_to_remove: set[int] = set() + + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + for item in node.body: + if not isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for dec in item.decorator_list: + dec_line = dec.lineno - 1 + line_content = lines[dec_line] + + if "expectedFailure" not in line_content: + continue + + has_comment_on_line = COMMENT in line_content + has_comment_before = ( + dec_line > 0 + and lines[dec_line - 1].strip().startswith("#") + and COMMENT in lines[dec_line - 1] + ) + + if not has_comment_on_line and not has_comment_before: + continue # not our marker + + # Check if there's a reason (on either the decorator or before) + for check_line in ( + line_content, + lines[dec_line - 1] if has_comment_before else "", + ): + match = re.search(rf"{COMMENT}(.*)", check_line) + if match and match.group(1).strip(";:, "): + break # has a reason, keep it + else: + # No reason found — strip this decorator + stripped_tests.add((node.name, item.name)) + + if _is_super_call_only(item): + # Remove entire super-call override (the method + # exists only to apply the decorator; without it + # the override is pointless and blocks parent + # consolidation) + lines_to_remove.update(_method_removal_range(item, lines)) + else: + lines_to_remove.add(dec_line) + + if has_comment_before: + lines_to_remove.add(dec_line - 1) + + # Also remove a reason-comment on the line after (old format) + if ( + has_comment_on_line + and dec_line + 1 < len(lines) + and lines[dec_line + 1].strip().startswith("#") + and COMMENT not in lines[dec_line + 1] + ): + lines_to_remove.add(dec_line + 1) + + if not lines_to_remove: + return contents, stripped_tests + + for idx in sorted(lines_to_remove, reverse=True): + del lines[idx] + + return "\n".join(lines) + "\n" if lines else "", stripped_tests + + +def extract_test_methods(contents: str) -> set[tuple[str, str]]: + """ + Extract all test method names from file contents. + + Returns: + Set of (class_name, method_name) tuples + """ + from update_lib.file_utils import safe_parse_ast + from update_lib.patch_spec import iter_tests + + tree = safe_parse_ast(contents) + if tree is None: + return set() + + return {(cls_node.name, fn_node.name) for cls_node, fn_node in iter_tests(tree)} + + +def auto_mark_file( + test_path: pathlib.Path, + mark_failure: bool = False, + verbose: bool = True, + original_methods: set[tuple[str, str]] | None = None, + skip_build: bool = False, +) -> tuple[int, int, int]: + """ + Run tests and auto-mark failures in a test file. + + Args: + test_path: Path to the test file + mark_failure: If True, add @expectedFailure to ALL failing tests + verbose: Print progress messages + original_methods: If provided, only auto-mark failures for NEW methods + (methods not in original_methods) even without mark_failure. + Failures in existing methods are treated as regressions. + + Returns: + (num_failures_added, num_successes_removed, num_regressions) + """ + test_path = pathlib.Path(test_path).resolve() + if not test_path.exists(): + raise FileNotFoundError(f"File not found: {test_path}") + + # Strip reason-less markers so those tests fail normally and we capture + # their error messages during the test run. + contents = test_path.read_text(encoding="utf-8") + original_contents = contents + contents, stripped_tests = strip_reasonless_expected_failures(contents) + if stripped_tests: + test_path.write_text(contents, encoding="utf-8") + + test_name = get_test_module_name(test_path) + if verbose: + print(f"Running test: {test_name}") + + results = run_test(test_name, skip_build=skip_build) + + # Check if test run failed entirely (e.g., import error, crash) + if ( + not results.tests_result + and not results.tests + and not results.unexpected_successes + ): + # Restore original contents before raising + if stripped_tests: + test_path.write_text(original_contents, encoding="utf-8") + raise TestRunError( + f"Test run failed for {test_name}. " + f"Output: {results.stdout[-500:] if results.stdout else '(no output)'}" + ) + + # If the run crashed (incomplete), restore original file so that markers + # for tests that never ran are preserved. Only observed results will be + # re-applied below. + if not results.tests_result and stripped_tests: + test_path.write_text(original_contents, encoding="utf-8") + stripped_tests = set() + + contents = test_path.read_text(encoding="utf-8") + + all_failing_tests, unexpected_successes, error_messages = collect_test_changes( + results + ) + + # Determine which failures to mark + if mark_failure: + failing_tests = all_failing_tests + elif original_methods is not None: + # Smart mode: only mark NEW test failures (not regressions) + current_methods = extract_test_methods(contents) + new_methods = current_methods - original_methods + failing_tests = {t for t in all_failing_tests if t in new_methods} + else: + failing_tests = set() + + # Re-mark stripped tests that still fail (to restore markers with reasons). + # Uses inheritance expansion: if a parent marker was stripped, child + # failures are included so _consolidate_to_parent can re-mark the parent. + failing_tests |= _expand_stripped_to_children( + contents, stripped_tests, all_failing_tests + ) + + regressions = all_failing_tests - failing_tests + + if verbose: + for class_name, method_name in failing_tests: + label = "(new test)" if original_methods is not None else "" + err_msg = error_messages.get((class_name, method_name), "") + err_hint = f" - {err_msg}" if err_msg else "" + print( + f"Marking as failing {label}: {class_name}.{method_name}{err_hint}".replace( + " ", " " + ) + ) + for class_name, method_name in unexpected_successes: + print(f"Removing expectedFailure: {class_name}.{method_name}") + + contents = apply_test_changes( + contents, failing_tests, unexpected_successes, error_messages + ) + + if failing_tests or unexpected_successes: + test_path.write_text(contents, encoding="utf-8") + + # Show hints about unmarked failures + if verbose: + unmarked_failures = all_failing_tests - failing_tests + if unmarked_failures: + print( + f"Hint: {len(unmarked_failures)} failing tests can be marked with --mark-failure; " + "but review first and do not blindly mark them all" + ) + for class_name, method_name in sorted(unmarked_failures): + err_msg = error_messages.get((class_name, method_name), "") + err_hint = f" - {err_msg}" if err_msg else "" + print(f" {class_name}.{method_name}{err_hint}") + + return len(failing_tests), len(unexpected_successes), len(regressions) + + +def auto_mark_directory( + test_dir: pathlib.Path, + mark_failure: bool = False, + verbose: bool = True, + original_methods_per_file: dict[pathlib.Path, set[tuple[str, str]]] | None = None, + skip_build: bool = False, +) -> tuple[int, int, int]: + """ + Run tests and auto-mark failures in a test directory. + + Runs the test once for the whole directory, then applies results to each file. + + Args: + test_dir: Path to the test directory + mark_failure: If True, add @expectedFailure to ALL failing tests + verbose: Print progress messages + original_methods_per_file: If provided, only auto-mark failures for NEW methods + even without mark_failure. Dict maps file path to + set of (class_name, method_name) tuples. + + Returns: + (num_failures_added, num_successes_removed, num_regressions) + """ + test_dir = pathlib.Path(test_dir).resolve() + if not test_dir.exists(): + raise FileNotFoundError(f"Directory not found: {test_dir}") + if not test_dir.is_dir(): + raise ValueError(f"Not a directory: {test_dir}") + + # Get all .py files in directory + test_files = sorted(test_dir.glob("**/*.py")) + + # Strip reason-less markers from ALL files before running tests so those + # tests fail normally and we capture their error messages. + stripped_per_file: dict[pathlib.Path, set[tuple[str, str]]] = {} + original_per_file: dict[pathlib.Path, str] = {} + for test_file in test_files: + contents = test_file.read_text(encoding="utf-8") + stripped_contents, stripped = strip_reasonless_expected_failures(contents) + if stripped: + original_per_file[test_file] = contents + test_file.write_text(stripped_contents, encoding="utf-8") + stripped_per_file[test_file] = stripped + + test_name = get_test_module_name(test_dir) + if verbose: + print(f"Running test: {test_name}") + + results = run_test(test_name, skip_build=skip_build) + + # Check if test run failed entirely (e.g., import error, crash) + if ( + not results.tests_result + and not results.tests + and not results.unexpected_successes + ): + # Restore original contents before raising + for fpath, original in original_per_file.items(): + fpath.write_text(original, encoding="utf-8") + raise TestRunError( + f"Test run failed for {test_name}. " + f"Output: {results.stdout[-500:] if results.stdout else '(no output)'}" + ) + + # If the run crashed (incomplete), restore original files so that markers + # for tests that never ran are preserved. + if not results.tests_result and original_per_file: + for fpath, original in original_per_file.items(): + fpath.write_text(original, encoding="utf-8") + stripped_per_file.clear() + + total_added = 0 + total_removed = 0 + total_regressions = 0 + all_regressions: list[tuple[str, str, str, str]] = [] + + for test_file in test_files: + # Get module prefix for this file (e.g., "test_inspect.test_inspect") + module_prefix = get_test_module_name(test_file) + # For __init__.py, the test path doesn't include "__init__" + if module_prefix.endswith(".__init__"): + module_prefix = module_prefix[:-9] # Remove ".__init__" + + all_failing_tests, unexpected_successes, error_messages = collect_test_changes( + results, module_prefix="test." + module_prefix + "." + ) + + # Determine which failures to mark + if mark_failure: + failing_tests = all_failing_tests + elif original_methods_per_file is not None: + # Smart mode: only mark NEW test failures + contents = test_file.read_text(encoding="utf-8") + current_methods = extract_test_methods(contents) + original_methods = original_methods_per_file.get(test_file, set()) + new_methods = current_methods - original_methods + failing_tests = {t for t in all_failing_tests if t in new_methods} + else: + failing_tests = set() + + # Re-mark stripped tests that still fail (restore markers with reasons). + # Uses inheritance expansion for parent→child mapping. + stripped = stripped_per_file.get(test_file, set()) + if stripped: + file_contents = test_file.read_text(encoding="utf-8") + failing_tests |= _expand_stripped_to_children( + file_contents, stripped, all_failing_tests + ) + + regressions = all_failing_tests - failing_tests + + if failing_tests or unexpected_successes: + if verbose: + for class_name, method_name in failing_tests: + label = ( + "(new test)" if original_methods_per_file is not None else "" + ) + err_msg = error_messages.get((class_name, method_name), "") + err_hint = f" - {err_msg}" if err_msg else "" + print( + f" {test_file.name}: Marking as failing {label}: {class_name}.{method_name}{err_hint}".replace( + " :", ":" + ) + ) + for class_name, method_name in unexpected_successes: + print( + f" {test_file.name}: Removing expectedFailure: {class_name}.{method_name}" + ) + + contents = test_file.read_text(encoding="utf-8") + contents = apply_test_changes( + contents, failing_tests, unexpected_successes, error_messages + ) + test_file.write_text(contents, encoding="utf-8") + + # Collect regressions with error messages for later reporting + for class_name, method_name in regressions: + err_msg = error_messages.get((class_name, method_name), "") + all_regressions.append((test_file.name, class_name, method_name, err_msg)) + + total_added += len(failing_tests) + total_removed += len(unexpected_successes) + total_regressions += len(regressions) + + # Show hints about unmarked failures + if verbose and total_regressions > 0: + print( + f"Hint: {total_regressions} failing tests can be marked with --mark-failure; " + "but review first and do not blindly mark them all" + ) + for file_name, class_name, method_name, err_msg in sorted(all_regressions): + err_hint = f" - {err_msg}" if err_msg else "" + print(f" {file_name}: {class_name}.{method_name}{err_hint}") + + return total_added, total_removed, total_regressions + + +def main(argv: list[str] | None = None) -> int: + import argparse + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "path", + type=pathlib.Path, + help="Path to test file or directory (e.g., Lib/test/test_foo.py or Lib/test/test_foo/)", + ) + parser.add_argument( + "--mark-failure", + action="store_true", + help="Also add @expectedFailure to failing tests (default: only remove unexpected successes)", + ) + parser.add_argument( + "--build", + action=argparse.BooleanOptionalAction, + default=True, + help="Build with cargo (default: enabled)", + ) + + args = parser.parse_args(argv) + + try: + if args.path.is_dir(): + num_added, num_removed, _ = auto_mark_directory( + args.path, mark_failure=args.mark_failure, skip_build=not args.build + ) + else: + num_added, num_removed, _ = auto_mark_file( + args.path, mark_failure=args.mark_failure, skip_build=not args.build + ) + if args.mark_failure: + print(f"Added expectedFailure to {num_added} tests") + print(f"Removed expectedFailure from {num_removed} tests") + return 0 + except (FileNotFoundError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_copy_lib.py b/scripts/update_lib/cmd_copy_lib.py new file mode 100644 index 00000000000..1b16497fc83 --- /dev/null +++ b/scripts/update_lib/cmd_copy_lib.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +""" +Copy library files from CPython. + +Usage: + # Single file + python scripts/update_lib copy-lib cpython/Lib/dataclasses.py + + # Directory + python scripts/update_lib copy-lib cpython/Lib/json +""" + +import argparse +import pathlib +import shutil +import sys + + +def _copy_single( + src_path: pathlib.Path, + lib_path: pathlib.Path, + verbose: bool = True, +) -> None: + """Copy a single file or directory.""" + # Remove existing file/directory + if lib_path.exists(): + if lib_path.is_dir(): + if verbose: + print(f"Removing directory: {lib_path}") + shutil.rmtree(lib_path) + else: + if verbose: + print(f"Removing file: {lib_path}") + lib_path.unlink() + + # Copy + if src_path.is_dir(): + if verbose: + print(f"Copying directory: {src_path} -> {lib_path}") + lib_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(src_path, lib_path) + else: + if verbose: + print(f"Copying file: {src_path} -> {lib_path}") + lib_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_path, lib_path) + + +def copy_lib( + src_path: pathlib.Path, + verbose: bool = True, +) -> None: + """ + Copy library file or directory from CPython. + + Also copies additional files if defined in DEPENDENCIES table. + + Args: + src_path: Source path (e.g., cpython/Lib/dataclasses.py or cpython/Lib/json) + verbose: Print progress messages + """ + from update_lib.deps import get_lib_paths + from update_lib.file_utils import parse_lib_path + + # Extract module name and cpython prefix from path + path_str = str(src_path).replace("\\", "/") + if "/Lib/" not in path_str: + raise ValueError(f"Path must contain '/Lib/' (got: {src_path})") + + cpython_prefix, after_lib = path_str.split("/Lib/", 1) + # Get module name (first component, without .py) + name = after_lib.split("/")[0] + if name.endswith(".py"): + name = name[:-3] + + # Get all paths to copy from DEPENDENCIES table + all_src_paths = get_lib_paths(name, cpython_prefix) + + # Copy each file + for src in all_src_paths: + if src.exists(): + lib_path = parse_lib_path(src) + _copy_single(src, lib_path, verbose) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "path", + type=pathlib.Path, + help="Source path containing /Lib/ (e.g., cpython/Lib/dataclasses.py)", + ) + + args = parser.parse_args(argv) + + try: + copy_lib(args.path) + return 0 + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_deps.py b/scripts/update_lib/cmd_deps.py new file mode 100644 index 00000000000..affb4b3609c --- /dev/null +++ b/scripts/update_lib/cmd_deps.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python +""" +Show dependency information for a module. + +Usage: + python scripts/update_lib deps dis + python scripts/update_lib deps dataclasses + python scripts/update_lib deps dis --depth 2 + python scripts/update_lib deps all # Show all modules' dependencies +""" + +import argparse +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + + +def get_all_modules(cpython_prefix: str) -> list[str]: + """Get all top-level module names from cpython/Lib/. + + Includes private modules (_*) that are not hard_deps of other modules. + + Returns: + Sorted list of module names (without .py extension) + """ + from update_lib.deps import resolve_hard_dep_parent + + lib_dir = pathlib.Path(cpython_prefix) / "Lib" + if not lib_dir.exists(): + return [] + + modules = set() + for entry in lib_dir.iterdir(): + # Skip hidden files + if entry.name.startswith("."): + continue + # Skip test directory + if entry.name == "test": + continue + + if entry.is_file() and entry.suffix == ".py": + name = entry.stem + elif entry.is_dir() and (entry / "__init__.py").exists(): + name = entry.name + else: + continue + + # Skip modules that are hard_deps of other modules + # e.g., _pydatetime is a hard_dep of datetime, pydoc_data is a hard_dep of pydoc + if resolve_hard_dep_parent(name, cpython_prefix) is not None: + continue + + modules.add(name) + + return sorted(modules) + + +def format_deps_tree( + cpython_prefix: str, + lib_prefix: str, + max_depth: int, + *, + name: str | None = None, + soft_deps: set[str] | None = None, + hard_deps: set[str] | None = None, + _depth: int = 0, + _visited: set[str] | None = None, + _indent: str = "", +) -> list[str]: + """Format soft dependencies as a tree with up-to-date status. + + Args: + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + max_depth: Maximum recursion depth + name: Module name (used to compute deps if soft_deps not provided) + soft_deps: Pre-computed soft dependencies (optional) + hard_deps: Hard dependencies to show under the module (root level only) + _depth: Current depth (internal) + _visited: Already visited modules (internal) + _indent: Current indentation (internal) + + Returns: + List of formatted lines + """ + from update_lib.deps import ( + get_lib_paths, + get_rust_deps, + get_soft_deps, + is_up_to_date, + ) + + lines = [] + + if _visited is None: + _visited = set() + + # Compute deps from name if not provided + if soft_deps is None: + soft_deps = get_soft_deps(name, cpython_prefix) if name else set() + + soft_deps = sorted(soft_deps) + + if not soft_deps and not hard_deps: + return lines + + # Separate up-to-date and outdated modules + up_to_date_deps = [] + outdated_deps = [] + dup_deps = [] + + for dep in soft_deps: + # Skip if library doesn't exist in cpython + lib_paths = get_lib_paths(dep, cpython_prefix) + if not any(p.exists() for p in lib_paths): + continue + + up_to_date = is_up_to_date(dep, cpython_prefix, lib_prefix) + if up_to_date: + # Up-to-date modules collected compactly, no dup tracking needed + up_to_date_deps.append(dep) + elif dep in _visited: + # Only track dup for outdated modules + dup_deps.append(dep) + else: + outdated_deps.append(dep) + + # Show outdated modules with expansion + for dep in outdated_deps: + dep_native = get_rust_deps(dep, cpython_prefix) + native_suffix = ( + f" (native: {', '.join(sorted(dep_native))})" if dep_native else "" + ) + lines.append(f"{_indent}- [ ] {dep}{native_suffix}") + _visited.add(dep) + + # Show hard_deps under this module (only at root level, i.e., when hard_deps is provided) + if hard_deps and dep in soft_deps: + for hd in sorted(hard_deps): + hd_up_to_date = is_up_to_date(hd, cpython_prefix, lib_prefix) + hd_marker = "[x]" if hd_up_to_date else "[ ]" + lines.append(f"{_indent} - {hd_marker} {hd}") + hard_deps = None # Only show once + + # Recurse if within depth limit + if _depth < max_depth - 1: + lines.extend( + format_deps_tree( + cpython_prefix, + lib_prefix, + max_depth, + name=dep, + _depth=_depth + 1, + _visited=_visited, + _indent=_indent + " ", + ) + ) + + # Show duplicates compactly (only for outdated) + if dup_deps: + lines.append(f"{_indent}- [ ] {', '.join(dup_deps)}") + + # Show up-to-date modules compactly on one line + if up_to_date_deps: + lines.append(f"{_indent}- [x] {', '.join(up_to_date_deps)}") + + return lines + + +def format_deps( + name: str, + cpython_prefix: str, + lib_prefix: str, + max_depth: int = 10, + _visited: set[str] | None = None, +) -> list[str]: + """Format all dependency information for a module. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + max_depth: Maximum recursion depth + _visited: Shared visited set for deduplication across modules + + Returns: + List of formatted lines + """ + from update_lib.deps import ( + DEPENDENCIES, + count_test_todos, + find_dependent_tests_tree, + get_lib_paths, + get_test_paths, + is_path_synced, + is_test_up_to_date, + resolve_hard_dep_parent, + ) + + if _visited is None: + _visited = set() + + lines = [] + + # Resolve test_ prefix to module (e.g., test_pydoc -> pydoc) + if name.startswith("test_"): + module_name = name[5:] # strip "test_" + lines.append(f"(redirecting {name} -> {module_name})") + name = module_name + + # Resolve hard_dep to parent module (e.g., pydoc_data -> pydoc) + parent = resolve_hard_dep_parent(name, cpython_prefix) + if parent: + lines.append(f"(redirecting {name} -> {parent})") + name = parent + + # lib paths (only show existing) + lib_paths = get_lib_paths(name, cpython_prefix) + existing_lib_paths = [p for p in lib_paths if p.exists()] + for p in existing_lib_paths: + synced = is_path_synced(p, cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + lines.append(f"{marker} lib: {p}") + + # test paths (only show existing) + test_paths = get_test_paths(name, cpython_prefix) + existing_test_paths = [p for p in test_paths if p.exists()] + for p in existing_test_paths: + test_name = p.stem if p.is_file() else p.name + synced = is_test_up_to_date(test_name, cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + todo_count = count_test_todos(test_name, lib_prefix) + todo_suffix = f" (TODO: {todo_count})" if todo_count > 0 else "" + lines.append(f"{marker} test: {p}{todo_suffix}") + + # If no lib or test paths exist, module doesn't exist + if not existing_lib_paths and not existing_test_paths: + lines.append(f"(module '{name}' not found)") + return lines + + # Collect all hard_deps (explicit from DEPENDENCIES + implicit from lib_paths) + dep_info = DEPENDENCIES.get(name, {}) + explicit_hard_deps = dep_info.get("hard_deps", []) + + # Get implicit hard_deps from lib_paths (e.g., _pydecimal.py for decimal) + all_hard_deps = set() + for hd in explicit_hard_deps: + # Remove .py extension if present + all_hard_deps.add(hd[:-3] if hd.endswith(".py") else hd) + + for p in existing_lib_paths: + dep_name = p.stem if p.is_file() else p.name + if dep_name != name: # Skip the main module itself + all_hard_deps.add(dep_name) + + lines.append("\ndependencies:") + lines.extend( + format_deps_tree( + cpython_prefix, + lib_prefix, + max_depth, + soft_deps={name}, + _visited=_visited, + hard_deps=all_hard_deps, + ) + ) + + # Show dependent tests as tree (depth 2: module + direct importers + their importers) + tree = find_dependent_tests_tree(name, lib_prefix=lib_prefix, max_depth=2) + lines.extend(_format_dependent_tests_tree(tree, cpython_prefix, lib_prefix)) + + return lines + + +def _format_dependent_tests_tree( + tree: dict, + cpython_prefix: str, + lib_prefix: str, + indent: str = "", +) -> list[str]: + """Format dependent tests tree for display.""" + from update_lib.deps import is_up_to_date + + lines = [] + module = tree["module"] + tests = tree["tests"] + children = tree["children"] + + if indent == "": + # Root level + # Count total tests in tree + def count_tests(t: dict) -> int: + total = len(t.get("tests", [])) + for c in t.get("children", []): + total += count_tests(c) + return total + + total = count_tests(tree) + if total == 0 and not children: + lines.append(f"\ndependent tests: (no tests depend on {module})") + return lines + lines.append(f"\ndependent tests: ({total} tests)") + + # Check if module is up-to-date + synced = is_up_to_date(module.split(".")[0], cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + + # Format this node + if tests: + test_str = " ".join(tests) + if indent == "": + lines.append(f"- {marker} {module}: {test_str}") + else: + lines.append(f"{indent}- {marker} {module}: {test_str}") + elif indent != "" and children: + # Has children but no direct tests + lines.append(f"{indent}- {marker} {module}:") + + # Format children + child_indent = indent + " " if indent else " " + for child in children: + lines.extend( + _format_dependent_tests_tree( + child, cpython_prefix, lib_prefix, child_indent + ) + ) + + return lines + + +def _resolve_module_name( + name: str, + cpython_prefix: str, + lib_prefix: str, +) -> list[str]: + """Resolve module name through redirects. + + Returns a list of module names (usually 1, but test support files may expand to multiple). + """ + import pathlib + + from update_lib.deps import ( + _build_test_import_graph, + get_lib_paths, + get_test_paths, + resolve_hard_dep_parent, + resolve_test_to_lib, + ) + + # Resolve test to library group (e.g., test_urllib2 -> urllib) + if name.startswith("test_"): + lib_group = resolve_test_to_lib(name) + if lib_group: + return [lib_group] + name = name[5:] + + # Resolve hard_dep to parent + parent = resolve_hard_dep_parent(name, cpython_prefix) + if parent: + return [parent] + + # Check if it's a valid module + lib_paths = get_lib_paths(name, cpython_prefix) + test_paths = get_test_paths(name, cpython_prefix) + if any(p.exists() for p in lib_paths) or any(p.exists() for p in test_paths): + return [name] + + # Check for test support files (e.g., string_tests -> bytes, str, userstring) + test_support_path = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{name}.py" + if test_support_path.exists(): + test_dir = pathlib.Path(lib_prefix) / "test" + if test_dir.exists(): + import_graph, _ = _build_test_import_graph(test_dir) + importing_tests = [] + for file_key, imports in import_graph.items(): + if name in imports and file_key.startswith("test_"): + importing_tests.append(file_key) + if importing_tests: + # Resolve test names to module names (test_bytes -> bytes) + return sorted(set(t[5:] for t in importing_tests)) + + return [name] + + +def show_deps( + names: list[str], + cpython_prefix: str, + lib_prefix: str, + max_depth: int = 10, +) -> None: + """Show all dependency information for modules.""" + # Expand "all" to all module names + expanded_names = [] + for name in names: + if name == "all": + expanded_names.extend(get_all_modules(cpython_prefix)) + else: + expanded_names.append(name) + + # Resolve and deduplicate names (preserving order) + seen: set[str] = set() + resolved_names: list[str] = [] + for name in expanded_names: + for resolved in _resolve_module_name(name, cpython_prefix, lib_prefix): + if resolved not in seen: + seen.add(resolved) + resolved_names.append(resolved) + + # Shared visited set across all modules + visited: set[str] = set() + + for i, name in enumerate(resolved_names): + if i > 0: + print() # blank line between modules + for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited): + print(line) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "names", + nargs="+", + help="Module names (e.g., dis, dataclasses) or 'all' for all modules", + ) + parser.add_argument( + "--cpython", + default="cpython", + help="CPython directory prefix (default: cpython)", + ) + parser.add_argument( + "--lib", + default="Lib", + help="Local Lib directory prefix (default: Lib)", + ) + parser.add_argument( + "--depth", + type=int, + default=10, + help="Maximum recursion depth for soft_deps tree (default: 10)", + ) + + args = parser.parse_args(argv) + + try: + show_deps(args.names, args.cpython, args.lib, args.depth) + return 0 + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_migrate.py b/scripts/update_lib/cmd_migrate.py new file mode 100644 index 00000000000..97cdf7b141b --- /dev/null +++ b/scripts/update_lib/cmd_migrate.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +""" +Migrate test file(s) from CPython, preserving RustPython markers. + +Usage: + python scripts/update_lib migrate cpython/Lib/test/test_foo.py + +This will: + 1. Extract patches from Lib/test/test_foo.py (if exists) + 2. Apply them to cpython/Lib/test/test_foo.py + 3. Write result to Lib/test/test_foo.py +""" + +import argparse +import pathlib +import shutil +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from update_lib.file_utils import parse_lib_path + + +def patch_single_content( + src_path: pathlib.Path, + lib_path: pathlib.Path, +) -> str: + """ + Patch content without writing to disk. + + Args: + src_path: Source file path (e.g., cpython/Lib/test/foo.py) + lib_path: Lib path to extract patches from (e.g., Lib/test/foo.py) + + Returns: + The patched content. + """ + from update_lib import apply_patches, extract_patches + + # Extract patches from existing file (if exists) + if lib_path.exists(): + patches = extract_patches(lib_path.read_text(encoding="utf-8")) + else: + patches = {} + + # Apply patches to source content + src_content = src_path.read_text(encoding="utf-8") + return apply_patches(src_content, patches) + + +def patch_file( + src_path: pathlib.Path, + lib_path: pathlib.Path | None = None, + verbose: bool = True, +) -> None: + """ + Patch a single file from source to lib. + + Args: + src_path: Source file path (e.g., cpython/Lib/test/foo.py) + lib_path: Target lib path. If None, derived from src_path. + verbose: Print progress messages + """ + if lib_path is None: + lib_path = parse_lib_path(src_path) + + if lib_path.exists(): + if verbose: + print(f"Patching: {src_path} -> {lib_path}") + content = patch_single_content(src_path, lib_path) + else: + if verbose: + print(f"Copying: {src_path} -> {lib_path}") + content = src_path.read_text(encoding="utf-8") + + lib_path.parent.mkdir(parents=True, exist_ok=True) + lib_path.write_text(content, encoding="utf-8") + + +def patch_directory( + src_dir: pathlib.Path, + lib_dir: pathlib.Path | None = None, + verbose: bool = True, +) -> None: + """ + Patch all files in a directory from source to lib. + + Args: + src_dir: Source directory path (e.g., cpython/Lib/test/test_foo/) + lib_dir: Target lib directory. If None, derived from src_dir. + verbose: Print progress messages + """ + if lib_dir is None: + lib_dir = parse_lib_path(src_dir) + + src_files = sorted(f for f in src_dir.glob("**/*") if f.is_file()) + + for src_file in src_files: + rel_path = src_file.relative_to(src_dir) + lib_file = lib_dir / rel_path + + if src_file.suffix == ".py": + if lib_file.exists(): + if verbose: + print(f"Patching: {src_file} -> {lib_file}") + content = patch_single_content(src_file, lib_file) + else: + if verbose: + print(f"Copying: {src_file} -> {lib_file}") + content = src_file.read_text(encoding="utf-8") + + lib_file.parent.mkdir(parents=True, exist_ok=True) + lib_file.write_text(content, encoding="utf-8") + else: + if verbose: + print(f"Copying: {src_file} -> {lib_file}") + lib_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_file, lib_file) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "path", + type=pathlib.Path, + help="Source path containing /Lib/ (file or directory)", + ) + + args = parser.parse_args(argv) + + try: + if args.path.is_dir(): + patch_directory(args.path) + else: + patch_file(args.path) + return 0 + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_patches.py b/scripts/update_lib/cmd_patches.py new file mode 100644 index 00000000000..67ebf1822b7 --- /dev/null +++ b/scripts/update_lib/cmd_patches.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +""" +Patch management for test files. + +Usage: + # Extract patches from one file and apply to another + python scripts/update_lib patches --from Lib/test/foo.py --to cpython/Lib/test/foo.py + + # Show patches as JSON + python scripts/update_lib patches --from Lib/test/foo.py --show-patches + + # Apply patches from JSON file + python scripts/update_lib patches -p patches.json --to Lib/test/foo.py +""" + +import argparse +import json +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + + +def write_output(data: str, dest: str) -> None: + if dest == "-": + print(data, end="") + return + + with open(dest, "w") as fd: + fd.write(data) + + +def main(argv: list[str] | None = None) -> int: + from update_lib import ( + apply_patches, + extract_patches, + patches_from_json, + patches_to_json, + ) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + patches_group = parser.add_mutually_exclusive_group(required=True) + patches_group.add_argument( + "-p", + "--patches", + type=pathlib.Path, + help="File path to file containing patches in a JSON format", + ) + patches_group.add_argument( + "--from", + dest="gather_from", + type=pathlib.Path, + help="File to gather patches from", + ) + + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument( + "--to", + type=pathlib.Path, + help="File to apply patches to", + ) + group.add_argument( + "--show-patches", + action="store_true", + help="Show the patches and exit", + ) + + parser.add_argument( + "-o", + "--output", + default="-", + help="Output file. Set to '-' for stdout", + ) + + args = parser.parse_args(argv) + + # Validate required arguments + if args.to is None and not args.show_patches: + parser.error("--to or --show-patches is required") + + try: + if args.patches: + patches = patches_from_json(json.loads(args.patches.read_text())) + else: + patches = extract_patches(args.gather_from.read_text()) + + if args.show_patches: + output = json.dumps(patches_to_json(patches), indent=4) + "\n" + write_output(output, args.output) + return 0 + + patched = apply_patches(args.to.read_text(), patches) + write_output(patched, args.output) + return 0 + + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_quick.py b/scripts/update_lib/cmd_quick.py new file mode 100644 index 00000000000..c43e0761518 --- /dev/null +++ b/scripts/update_lib/cmd_quick.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python +""" +Quick update for test files from CPython. + +Usage: + # Library + test: copy lib, then patch + auto-mark test + commit + python scripts/update_lib quick cpython/Lib/dataclasses.py + + # Shortcut: just the module name + python scripts/update_lib quick dataclasses + + # Test file: patch + auto-mark + python scripts/update_lib quick cpython/Lib/test/test_foo.py + + # Test file: migrate only + python scripts/update_lib quick cpython/Lib/test/test_foo.py --no-auto-mark + + # Test file: auto-mark only (Lib/ path implies --no-migrate) + python scripts/update_lib quick Lib/test/test_foo.py + + # Directory: patch all + auto-mark all + python scripts/update_lib quick cpython/Lib/test/test_dataclasses/ + + # Skip git commit + python scripts/update_lib quick dataclasses --no-commit +""" + +import argparse +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from update_lib.deps import DEPENDENCIES, get_test_paths +from update_lib.file_utils import ( + construct_lib_path, + get_cpython_dir, + get_module_name, + get_test_files, + is_lib_path, + is_test_path, + lib_to_test_path, + parse_lib_path, + resolve_module_path, + safe_read_text, +) + + +def collect_original_methods( + lib_path: pathlib.Path, +) -> set[tuple[str, str]] | dict[pathlib.Path, set[tuple[str, str]]] | None: + """ + Collect original test methods from lib path before patching. + + Returns: + - For file: set of (class_name, method_name) or None if file doesn't exist + - For directory: dict mapping file path to set of methods, or None if dir doesn't exist + """ + from update_lib.cmd_auto_mark import extract_test_methods + + if not lib_path.exists(): + return None + + if lib_path.is_file(): + content = safe_read_text(lib_path) + return extract_test_methods(content) if content else set() + else: + result = {} + for lib_file in get_test_files(lib_path): + content = safe_read_text(lib_file) + if content: + result[lib_file.resolve()] = extract_test_methods(content) + return result + + +def quick( + src_path: pathlib.Path, + no_migrate: bool = False, + no_auto_mark: bool = False, + mark_failure: bool = False, + verbose: bool = True, + skip_build: bool = False, +) -> list[pathlib.Path]: + """ + Process a file or directory: migrate + auto-mark. + + Args: + src_path: Source path (file or directory) + no_migrate: Skip migration step + no_auto_mark: Skip auto-mark step + mark_failure: Add @expectedFailure to ALL failing tests + verbose: Print progress messages + skip_build: Skip cargo build, use pre-built binary + + Returns: + List of extra paths (data dirs, hard deps) that were copied/migrated. + """ + from update_lib.cmd_auto_mark import auto_mark_directory, auto_mark_file + from update_lib.cmd_migrate import patch_directory, patch_file + + extra_paths: list[pathlib.Path] = [] + + # Determine lib_path and whether to migrate + if is_lib_path(src_path): + no_migrate = True + lib_path = src_path + else: + lib_path = parse_lib_path(src_path) + + is_dir = src_path.is_dir() + + # Capture original test methods before migration (for smart auto-mark) + original_methods = collect_original_methods(lib_path) + + # Step 1: Migrate + if not no_migrate: + if is_dir: + patch_directory(src_path, lib_path, verbose=verbose) + else: + patch_file(src_path, lib_path, verbose=verbose) + + # Step 1.5: Handle test dependencies + from update_lib.deps import get_test_dependencies + + test_deps = get_test_dependencies(src_path) + + # Migrate dependency files + for dep_src in test_deps["hard_deps"]: + dep_lib = parse_lib_path(dep_src) + if verbose: + print(f"Migrating dependency: {dep_src.name}") + if dep_src.is_dir(): + patch_directory(dep_src, dep_lib, verbose=False) + else: + patch_file(dep_src, dep_lib, verbose=False) + extra_paths.append(dep_lib) + + # Copy data directories (no migration) + import shutil + + for data_src in test_deps["data"]: + data_lib = parse_lib_path(data_src) + if verbose: + print(f"Copying data: {data_src.name}") + if data_lib.exists(): + if data_lib.is_dir(): + shutil.rmtree(data_lib) + else: + data_lib.unlink() + if data_src.is_dir(): + shutil.copytree(data_src, data_lib) + else: + data_lib.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(data_src, data_lib) + extra_paths.append(data_lib) + + # Step 2: Auto-mark + if not no_auto_mark: + if not lib_path.exists(): + raise FileNotFoundError(f"Path not found: {lib_path}") + + if is_dir: + num_added, num_removed, _ = auto_mark_directory( + lib_path, + mark_failure=mark_failure, + verbose=verbose, + original_methods_per_file=original_methods, + skip_build=skip_build, + ) + else: + num_added, num_removed, _ = auto_mark_file( + lib_path, + mark_failure=mark_failure, + verbose=verbose, + original_methods=original_methods, + skip_build=skip_build, + ) + + if verbose: + if num_added: + print(f"Added expectedFailure to {num_added} tests") + print(f"Removed expectedFailure from {num_removed} tests") + + return extra_paths + + +def get_cpython_version(cpython_dir: pathlib.Path) -> str: + """Get CPython version from git tag.""" + import subprocess + + result = subprocess.run( + ["git", "describe", "--tags"], + cwd=cpython_dir, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def git_commit( + name: str, + lib_path: pathlib.Path | None, + test_paths: list[pathlib.Path] | pathlib.Path | None, + cpython_dir: pathlib.Path, + hard_deps: list[pathlib.Path] | None = None, + verbose: bool = True, +) -> bool: + """Commit changes with CPython author. + + Args: + name: Module name (e.g., "dataclasses") + lib_path: Path to library file/directory (or None) + test_paths: Path(s) to test file/directory (or None) + cpython_dir: Path to cpython directory + hard_deps: Path(s) to hard dependency files (or None) + verbose: Print progress messages + + Returns: + True if commit was created, False otherwise + """ + import subprocess + + # Normalize test_paths to list + if test_paths is None: + test_paths = [] + elif isinstance(test_paths, pathlib.Path): + test_paths = [test_paths] + + # Normalize hard_deps to list + if hard_deps is None: + hard_deps = [] + + # Stage changes + paths_to_add = [] + if lib_path and lib_path.exists(): + paths_to_add.append(str(lib_path)) + for test_path in test_paths: + if test_path and test_path.exists(): + paths_to_add.append(str(test_path)) + for dep_path in hard_deps: + if dep_path and dep_path.exists(): + paths_to_add.append(str(dep_path)) + + if not paths_to_add: + return False + + version = get_cpython_version(cpython_dir) + subprocess.run(["git", "add"] + paths_to_add, check=True) + + # Check if there are staged changes + result = subprocess.run( + ["git", "diff", "--cached", "--quiet"], + capture_output=True, + ) + if result.returncode == 0: + if verbose: + print("No changes to commit") + return False + + # Commit with CPython author + message = f"Update {name} from {version}" + subprocess.run( + [ + "git", + "commit", + "--author", + "CPython Developers <>", + "-m", + message, + ], + check=True, + ) + if verbose: + print(f"Committed: {message}") + return True + + +def _expand_shortcut(path: pathlib.Path) -> pathlib.Path: + """Expand simple name to cpython/Lib path if it exists. + + Examples: + dataclasses -> cpython/Lib/dataclasses.py (if exists) + json -> cpython/Lib/json/ (if exists) + test_types -> cpython/Lib/test/test_types.py (if exists) + regrtest -> cpython/Lib/test/libregrtest (from DEPENDENCIES) + """ + # Only expand if it's a simple name (no path separators) and doesn't exist + if "/" in str(path) or path.exists(): + return path + + name = str(path) + + # Check DEPENDENCIES table for path overrides (e.g., regrtest) + from update_lib.deps import DEPENDENCIES + + if name in DEPENDENCIES and "lib" in DEPENDENCIES[name]: + lib_paths = DEPENDENCIES[name]["lib"] + if lib_paths: + override_path = construct_lib_path("cpython", lib_paths[0]) + if override_path.exists(): + return override_path + + # Test shortcut: test_foo -> cpython/Lib/test/test_foo + if name.startswith("test_"): + resolved = resolve_module_path(f"test/{name}", "cpython", prefer="dir") + if resolved.exists(): + return resolved + + # Library shortcut: foo -> cpython/Lib/foo + resolved = resolve_module_path(name, "cpython", prefer="file") + if resolved.exists(): + return resolved + + # Extension module shortcut: winreg -> cpython/Lib/test/test_winreg + # For C/Rust extension modules that have no Python source but have tests + resolved = resolve_module_path(f"test/test_{name}", "cpython", prefer="dir") + if resolved.exists(): + return resolved + + # Return original (will likely fail later with a clear error) + return path + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "path", + type=pathlib.Path, + help="Source path (file or directory)", + ) + parser.add_argument( + "--copy", + action=argparse.BooleanOptionalAction, + default=True, + help="Copy library file (default: enabled, implied disabled if test path)", + ) + parser.add_argument( + "--migrate", + action=argparse.BooleanOptionalAction, + default=True, + help="Migrate test file (default: enabled, implied disabled if Lib/ path)", + ) + parser.add_argument( + "--auto-mark", + action=argparse.BooleanOptionalAction, + default=True, + help="Auto-mark test failures (default: enabled)", + ) + parser.add_argument( + "--mark-failure", + action="store_true", + help="Add @expectedFailure to failing tests", + ) + parser.add_argument( + "--commit", + action=argparse.BooleanOptionalAction, + default=True, + help="Create git commit (default: enabled)", + ) + parser.add_argument( + "--build", + action=argparse.BooleanOptionalAction, + default=True, + help="Build with cargo (default: enabled)", + ) + + args = parser.parse_args(argv) + + try: + src_path = args.path + + # Shortcut: expand simple name to cpython/Lib path + src_path = _expand_shortcut(src_path) + original_src = src_path # Keep for commit + + # Track library path for commit + lib_file_path = None + test_path = None + hard_deps_for_commit = [] + + # If it's a library path (not test path), do copy_lib first + if not is_test_path(src_path): + # Get library destination path for commit + lib_file_path = parse_lib_path(src_path) + + if args.copy: + from update_lib.cmd_copy_lib import copy_lib + + copy_lib(src_path) + + # Get all test paths from DEPENDENCIES (or fall back to default) + module_name = get_module_name(original_src) + cpython_dir = get_cpython_dir(original_src) + test_src_paths = get_test_paths(module_name, str(cpython_dir)) + + # Fall back to default test path if DEPENDENCIES has no entry + if not test_src_paths: + default_test = lib_to_test_path(original_src) + if default_test.exists(): + test_src_paths = (default_test,) + + # Collect hard dependencies for commit + lib_deps = DEPENDENCIES.get(module_name, {}) + for dep_name in lib_deps.get("hard_deps", []): + dep_lib_path = pathlib.Path("Lib") / dep_name + if dep_lib_path.exists(): + hard_deps_for_commit.append(dep_lib_path) + + # Process all test paths + test_paths_for_commit = [] + for test_src in test_src_paths: + if not test_src.exists(): + print(f"Warning: Test path does not exist: {test_src}") + continue + + test_lib_path = parse_lib_path(test_src) + test_paths_for_commit.append(test_lib_path) + + extra = quick( + test_src, + no_migrate=not args.migrate, + no_auto_mark=not args.auto_mark, + mark_failure=args.mark_failure, + skip_build=not args.build, + ) + hard_deps_for_commit.extend(extra) + + test_paths = test_paths_for_commit + else: + # It's a test path - process single test + test_path = ( + parse_lib_path(src_path) if not is_lib_path(src_path) else src_path + ) + + extra = quick( + src_path, + no_migrate=not args.migrate, + no_auto_mark=not args.auto_mark, + mark_failure=args.mark_failure, + skip_build=not args.build, + ) + hard_deps_for_commit.extend(extra) + test_paths = [test_path] + + # Step 3: Git commit + if args.commit: + cpython_dir = get_cpython_dir(original_src) + git_commit( + get_module_name(original_src), + lib_file_path, + test_paths, + cpython_dir, + hard_deps=hard_deps_for_commit, + ) + + return 0 + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + except Exception as e: + # Handle TestRunError with a clean message + from update_lib.cmd_auto_mark import TestRunError + + if isinstance(e, TestRunError): + print(f"Error: {e}", file=sys.stderr) + return 1 + raise + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/cmd_todo.py b/scripts/update_lib/cmd_todo.py new file mode 100644 index 00000000000..a7cf83f7b9e --- /dev/null +++ b/scripts/update_lib/cmd_todo.py @@ -0,0 +1,703 @@ +#!/usr/bin/env python +""" +Show prioritized list of modules to update. + +Usage: + python scripts/update_lib todo + python scripts/update_lib todo --limit 20 +""" + +import argparse +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from update_lib.deps import ( + count_test_todos, + get_module_diff_stat, + get_module_last_updated, + get_test_last_updated, + is_test_tracked, + is_test_up_to_date, +) + + +def compute_todo_list( + cpython_prefix: str, + lib_prefix: str, + include_done: bool = False, +) -> list[dict]: + """Compute prioritized list of modules to update. + + Scoring: + - Modules with no pylib dependencies: score = -1 + - Modules with pylib dependencies: score = count of NOT up-to-date deps + + Sorting (ascending by score): + 1. More reverse dependencies (modules depending on this) = higher priority + 2. Fewer native dependencies = higher priority + + Returns: + List of dicts with module info, sorted by priority + """ + from update_lib.cmd_deps import get_all_modules + from update_lib.deps import ( + get_all_hard_deps, + get_rust_deps, + get_soft_deps, + is_up_to_date, + ) + + all_modules = get_all_modules(cpython_prefix) + + # Build dependency data for all modules + module_data = {} + for name in all_modules: + soft_deps = get_soft_deps(name, cpython_prefix) + native_deps = get_rust_deps(name, cpython_prefix) + up_to_date = is_up_to_date(name, cpython_prefix, lib_prefix) + + # Get hard_deps and check their status + hard_deps = get_all_hard_deps(name, cpython_prefix) + hard_deps_status = { + hd: is_up_to_date(hd, cpython_prefix, lib_prefix) for hd in hard_deps + } + + module_data[name] = { + "name": name, + "soft_deps": soft_deps, + "native_deps": native_deps, + "up_to_date": up_to_date, + "hard_deps_status": hard_deps_status, + } + + # Build reverse dependency map: who depends on this module + reverse_deps: dict[str, set[str]] = {name: set() for name in all_modules} + for name, data in module_data.items(): + for dep in data["soft_deps"]: + if dep in reverse_deps: + reverse_deps[dep].add(name) + + # Compute scores and filter + result = [] + for name, data in module_data.items(): + hard_deps_status = data["hard_deps_status"] + has_outdated_hard_deps = any(not ok for ok in hard_deps_status.values()) + + # Include if: not up-to-date, or has outdated hard_deps, or --done + if data["up_to_date"] and not has_outdated_hard_deps and not include_done: + continue + + soft_deps = data["soft_deps"] + if not soft_deps: + # No pylib dependencies + score = -1 + total_deps = 0 + else: + # Count NOT up-to-date dependencies + score = sum( + 1 + for dep in soft_deps + if dep in module_data and not module_data[dep]["up_to_date"] + ) + total_deps = len(soft_deps) + + result.append( + { + "name": name, + "score": score, + "total_deps": total_deps, + "reverse_deps": reverse_deps[name], + "reverse_deps_count": len(reverse_deps[name]), + "native_deps_count": len(data["native_deps"]), + "native_deps": data["native_deps"], + "soft_deps": soft_deps, + "up_to_date": data["up_to_date"], + "hard_deps_status": hard_deps_status, + } + ) + + # Sort by: + # 1. score (ascending) - fewer outstanding deps first + # 2. reverse_deps_count (descending) - more dependents first + # 3. native_deps_count (ascending) - fewer native deps first + result.sort( + key=lambda x: ( + x["score"], + -x["reverse_deps_count"], + x["native_deps_count"], + ) + ) + + return result + + +def get_all_tests(cpython_prefix: str) -> list[str]: + """Get all test module names from cpython/Lib/test/. + + Returns: + Sorted list of test names (e.g., ["test_abc", "test_dis", ...]) + """ + test_dir = pathlib.Path(cpython_prefix) / "Lib" / "test" + if not test_dir.exists(): + return [] + + tests = set() + for entry in test_dir.iterdir(): + # Skip non-test items + if "test" not in entry.name: + continue + + # Exclude special cases + if "regrtest" in entry.name: + continue + + if entry.is_file() and entry.suffix == ".py": + tests.add(entry.stem) + elif entry.is_dir() and (entry / "__init__.py").exists(): + tests.add(entry.name) + + return sorted(tests) + + +def get_untracked_files( + cpython_prefix: str, + lib_prefix: str, +) -> list[str]: + """Get files that exist in cpython/Lib but not in our Lib. + + Excludes files that belong to tracked modules (shown in library todo) + and hard_deps of those modules. + Includes all file types (.py, .txt, .pem, .json, etc.) + + Returns: + Sorted list of relative paths (e.g., ["foo.py", "data/file.txt"]) + """ + from update_lib.cmd_deps import get_all_modules + from update_lib.deps import resolve_hard_dep_parent + + cpython_lib = pathlib.Path(cpython_prefix) / "Lib" + local_lib = pathlib.Path(lib_prefix) + + if not cpython_lib.exists(): + return [] + + # Get tracked modules (shown in library todo) + tracked_modules = set(get_all_modules(cpython_prefix)) + + untracked = [] + + for cpython_file in cpython_lib.rglob("*"): + # Skip directories + if cpython_file.is_dir(): + continue + + # Get relative path from Lib/ + rel_path = cpython_file.relative_to(cpython_lib) + + # Skip test/ directory (handled separately by test todo) + if rel_path.parts and rel_path.parts[0] == "test": + continue + + # Check if file belongs to a tracked module + # e.g., idlelib/Icons/idle.gif -> module "idlelib" + # e.g., foo.py -> module "foo" + first_part = rel_path.parts[0] + if first_part.endswith(".py"): + module_name = first_part[:-3] # Remove .py + else: + module_name = first_part + + if module_name in tracked_modules: + continue + + # Check if this is a hard_dep of a tracked module + if resolve_hard_dep_parent(module_name, cpython_prefix) is not None: + continue + + # Check if exists in local lib + local_file = local_lib / rel_path + if not local_file.exists(): + untracked.append(str(rel_path)) + + return sorted(untracked) + + +def get_original_files( + cpython_prefix: str, + lib_prefix: str, +) -> list[str]: + """Get top-level files/modules that exist in our Lib but not in cpython/Lib. + + These are RustPython-original files that don't come from CPython. + Modules that exist in cpython are handled by the library todo (even if + they have additional local files), so they are excluded here. + Excludes test/ directory (handled separately). + + Returns: + Sorted list of top-level names (e.g., ["_dummy_thread.py"]) + """ + cpython_lib = pathlib.Path(cpython_prefix) / "Lib" + local_lib = pathlib.Path(lib_prefix) + + if not local_lib.exists(): + return [] + + original = [] + + # Only check top-level entries + for entry in local_lib.iterdir(): + name = entry.name + + # Skip hidden files and __pycache__ + if name.startswith(".") or name == "__pycache__": + continue + + # Skip test/ directory (handled separately) + if name == "test": + continue + + # Skip site-packages (not a module) + if name == "site-packages": + continue + + # Only include if it doesn't exist in cpython at all + cpython_entry = cpython_lib / name + if not cpython_entry.exists(): + original.append(name) + + return sorted(original) + + +def _build_test_to_lib_map( + cpython_prefix: str, +) -> tuple[dict[str, str], dict[str, list[str]]]: + """Build reverse mapping from test name to library name using DEPENDENCIES. + + Returns: + Tuple of: + - Dict mapping test_name -> lib_name (e.g., "test_htmlparser" -> "html") + - Dict mapping lib_name -> ordered list of test_names + """ + import pathlib + + from update_lib.deps import DEPENDENCIES + + test_to_lib = {} + lib_test_order: dict[str, list[str]] = {} + for lib_name, dep_info in DEPENDENCIES.items(): + if "test" not in dep_info: + continue + lib_test_order[lib_name] = [] + for test_path in dep_info["test"]: + # test_path is like "test_htmlparser.py" or "test_multiprocessing_fork" + path = pathlib.Path(test_path) + if path.suffix == ".py": + test_name = path.stem + else: + test_name = path.name + test_to_lib[test_name] = lib_name + lib_test_order[lib_name].append(test_name) + + return test_to_lib, lib_test_order + + +def compute_test_todo_list( + cpython_prefix: str, + lib_prefix: str, + include_done: bool = False, + lib_status: dict[str, bool] | None = None, +) -> list[dict]: + """Compute prioritized list of tests to update. + + Scoring: + - If corresponding lib is up-to-date: score = 0 (ready) + - If no corresponding lib: score = 1 (independent) + - If corresponding lib is NOT up-to-date: score = 2 (wait for lib) + + Returns: + List of dicts with test info, sorted by priority + """ + all_tests = get_all_tests(cpython_prefix) + test_to_lib, lib_test_order = _build_test_to_lib_map(cpython_prefix) + + result = [] + for test_name in all_tests: + up_to_date = is_test_up_to_date(test_name, cpython_prefix, lib_prefix) + + if up_to_date and not include_done: + continue + + tracked = is_test_tracked(test_name, cpython_prefix, lib_prefix) + + # Check DEPENDENCIES mapping first, then fall back to simple extraction + if test_name in test_to_lib: + lib_name = test_to_lib[test_name] + # Get order from DEPENDENCIES + test_order = lib_test_order[lib_name].index(test_name) + else: + # Extract lib name from test name: + # - test_foo -> foo + # - datetimetester -> datetime + # - xmltests -> xml + lib_name = ( + test_name.removeprefix("test_") + .removeprefix("_test") + .removesuffix("tester") + .removesuffix("tests") + ) + test_order = 0 # Default order for tests not in DEPENDENCIES + + # Check if corresponding lib is up-to-date + # Scoring: 0 = lib ready (highest priority), 1 = no lib, 2 = lib pending + if lib_status and lib_name in lib_status: + lib_up_to_date = lib_status[lib_name] + if lib_up_to_date: + score = 0 # Lib is ready, can update test + else: + score = 2 # Wait for lib first + else: + score = 1 # No corresponding lib (independent test) + + todo_count = count_test_todos(test_name, lib_prefix) if tracked else 0 + + result.append( + { + "name": test_name, + "lib_name": lib_name, + "score": score, + "up_to_date": up_to_date, + "tracked": tracked, + "todo_count": todo_count, + "test_order": test_order, + } + ) + + # Sort by score (ascending) + result.sort(key=lambda x: x["score"]) + + return result + + +def _format_meta_suffix(item: dict) -> str: + """Format metadata suffix (last updated date and diff count).""" + parts = [] + last_updated = item.get("last_updated") + diff_lines = item.get("diff_lines", 0) + if last_updated: + parts.append(last_updated) + if diff_lines > 0: + parts.append(f"Δ{diff_lines}") + return f" | {' '.join(parts)}" if parts else "" + + +def _format_test_suffix(item: dict) -> str: + """Format suffix for test item (TODO count or untracked).""" + tracked = item.get("tracked", True) + if not tracked: + return " (untracked)" + todo_count = item.get("todo_count", 0) + if todo_count > 0: + return f" ({todo_count} TODO)" + return "" + + +def format_test_todo_list( + todo_list: list[dict], + limit: int | None = None, +) -> list[str]: + """Format test todo list for display. + + Groups tests by lib_name. If multiple tests share the same lib_name, + the first test is shown as the primary and others are indented below it. + """ + lines = [] + + if limit: + todo_list = todo_list[:limit] + + # Group by lib_name + grouped: dict[str, list[dict]] = {} + for item in todo_list: + lib_name = item.get("lib_name", item["name"]) + if lib_name not in grouped: + grouped[lib_name] = [] + grouped[lib_name].append(item) + + # Sort each group by test_order (from DEPENDENCIES) + for tests in grouped.values(): + tests.sort(key=lambda x: x.get("test_order", 0)) + + for lib_name, tests in grouped.items(): + # First test is the primary + primary = tests[0] + done_mark = "[x]" if primary["up_to_date"] else "[ ]" + suffix = _format_test_suffix(primary) + meta = _format_meta_suffix(primary) + lines.append(f"- {done_mark} `{primary['name']}`{suffix}{meta}") + + # Rest are indented + for item in tests[1:]: + done_mark = "[x]" if item["up_to_date"] else "[ ]" + suffix = _format_test_suffix(item) + meta = _format_meta_suffix(item) + lines.append(f" - {done_mark} `{item['name']}`{suffix}{meta}") + + return lines + + +def format_todo_list( + todo_list: list[dict], + test_by_lib: dict[str, list[dict]] | None = None, + limit: int | None = None, + verbose: bool = False, +) -> list[str]: + """Format todo list for display. + + Args: + todo_list: List from compute_todo_list() + test_by_lib: Dict mapping lib_name -> list of test infos (optional) + limit: Maximum number of items to show + verbose: Show detailed dependency information + + Returns: + List of formatted lines + """ + lines = [] + + if limit: + todo_list = todo_list[:limit] + + for item in todo_list: + name = item["name"] + score = item["score"] + total_deps = item["total_deps"] + rev_count = item["reverse_deps_count"] + + done_mark = "[x]" if item["up_to_date"] else "[ ]" + + if score == -1: + score_str = "no deps" + else: + score_str = f"{score}/{total_deps} deps" + + rev_str = f"{rev_count} dependents" if rev_count else "" + + parts = ["-", done_mark, f"[{score_str}]", f"`{name}`"] + if rev_str: + parts.append(f"({rev_str})") + + line = " ".join(parts) + _format_meta_suffix(item) + lines.append(line) + + # Show hard_deps: + # - Normal mode: only show if lib is up-to-date but hard_deps are not + # - Verbose mode: always show all hard_deps with their status + hard_deps_status = item.get("hard_deps_status", {}) + if verbose and hard_deps_status: + for hd in sorted(hard_deps_status.keys()): + hd_mark = "[x]" if hard_deps_status[hd] else "[ ]" + lines.append(f" - {hd_mark} {hd} (hard_dep)") + elif item["up_to_date"]: + for hd, ok in sorted(hard_deps_status.items()): + if not ok: + lines.append(f" - [ ] {hd} (hard_dep)") + + # Show corresponding tests if exist + if test_by_lib and name in test_by_lib: + for test_info in test_by_lib[name]: + test_done_mark = "[x]" if test_info["up_to_date"] else "[ ]" + suffix = _format_test_suffix(test_info) + meta = _format_meta_suffix(test_info) + lines.append( + f" - {test_done_mark} `{test_info['name']}`{suffix}{meta}" + ) + + # Verbose mode: show detailed dependency info + if verbose: + if item["reverse_deps"]: + lines.append(f" dependents: {', '.join(sorted(item['reverse_deps']))}") + if item["soft_deps"]: + lines.append(f" python: {', '.join(sorted(item['soft_deps']))}") + if item["native_deps"]: + lines.append(f" native: {', '.join(sorted(item['native_deps']))}") + + return lines + + +def format_all_todo( + cpython_prefix: str, + lib_prefix: str, + limit: int | None = None, + include_done: bool = False, + verbose: bool = False, +) -> list[str]: + """Format prioritized list of modules and tests to update. + + Returns: + List of formatted lines + """ + from update_lib.cmd_deps import get_all_modules + from update_lib.deps import is_up_to_date + + lines = [] + + # Build lib status map for test scoring + lib_status = {} + for name in get_all_modules(cpython_prefix): + lib_status[name] = is_up_to_date(name, cpython_prefix, lib_prefix) + + # Compute test todo (always include all to find libs with pending tests) + test_todo = compute_test_todo_list( + cpython_prefix, lib_prefix, include_done=True, lib_status=lib_status + ) + + # Build test_by_lib map (only for tests with corresponding lib) + test_by_lib: dict[str, list[dict]] = {} + no_lib_tests = [] + # Set of libs that have pending tests + libs_with_pending_tests = set() + for test in test_todo: + if test["score"] == 1: # no lib + if not test["up_to_date"] or include_done: + no_lib_tests.append(test) + else: + lib_name = test["lib_name"] + if lib_name not in test_by_lib: + test_by_lib[lib_name] = [] + test_by_lib[lib_name].append(test) + if not test["up_to_date"]: + libs_with_pending_tests.add(lib_name) + + # Sort each lib's tests by test_order (from DEPENDENCIES) + for tests in test_by_lib.values(): + tests.sort(key=lambda x: x.get("test_order", 0)) + + # Compute lib todo - include libs with pending tests even if lib is done + lib_todo_base = compute_todo_list(cpython_prefix, lib_prefix, include_done=True) + + # Filter lib todo: include if lib is not done OR has pending test + lib_todo = [] + for item in lib_todo_base: + lib_not_done = not item["up_to_date"] + has_pending_test = item["name"] in libs_with_pending_tests + + if include_done or lib_not_done or has_pending_test: + lib_todo.append(item) + + # Add metadata (last updated date and diff stat) to lib items + for item in lib_todo: + item["last_updated"] = get_module_last_updated( + item["name"], cpython_prefix, lib_prefix + ) + item["diff_lines"] = ( + 0 + if item["up_to_date"] + else get_module_diff_stat(item["name"], cpython_prefix, lib_prefix) + ) + + # Add last_updated to displayed test items + for tests in test_by_lib.values(): + for test in tests: + test["last_updated"] = get_test_last_updated( + test["name"], cpython_prefix, lib_prefix + ) + for test in no_lib_tests: + test["last_updated"] = get_test_last_updated( + test["name"], cpython_prefix, lib_prefix + ) + + # Format lib todo with embedded tests + lines.extend(format_todo_list(lib_todo, test_by_lib, limit, verbose)) + + # Format "no lib" tests separately if any + if no_lib_tests: + lines.append("") + lines.append("## Standalone Tests") + lines.extend(format_test_todo_list(no_lib_tests, limit)) + + # Format untracked files (in cpython but not in our Lib) + untracked = get_untracked_files(cpython_prefix, lib_prefix) + if untracked: + lines.append("") + lines.append("## Untracked Files") + display_untracked = untracked[:limit] if limit else untracked + for path in display_untracked: + lines.append(f"- {path}") + if limit and len(untracked) > limit: + lines.append(f" ... and {len(untracked) - limit} more") + + # Format original files (in our Lib but not in cpython) + original = get_original_files(cpython_prefix, lib_prefix) + if original: + lines.append("") + lines.append("## Original Files") + display_original = original[:limit] if limit else original + for path in display_original: + lines.append(f"- {path}") + if limit and len(original) > limit: + lines.append(f" ... and {len(original) - limit} more") + + return lines + + +def show_todo( + cpython_prefix: str, + lib_prefix: str, + limit: int | None = None, + include_done: bool = False, + verbose: bool = False, +) -> None: + """Show prioritized list of modules and tests to update.""" + for line in format_all_todo( + cpython_prefix, lib_prefix, limit, include_done, verbose + ): + print(line) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--cpython", + default="cpython", + help="CPython directory prefix (default: cpython)", + ) + parser.add_argument( + "--lib", + default="Lib", + help="Local Lib directory prefix (default: Lib)", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Maximum number of items to show", + ) + parser.add_argument( + "--done", + action="store_true", + help="Include already up-to-date modules", + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Show detailed dependency information", + ) + + args = parser.parse_args(argv) + + try: + show_todo(args.cpython, args.lib, args.limit, args.done, args.verbose) + return 0 + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py new file mode 100644 index 00000000000..49ddf9a8730 --- /dev/null +++ b/scripts/update_lib/deps.py @@ -0,0 +1,1721 @@ +""" +Dependency resolution for library updates. + +Handles: +- Irregular library paths (e.g., libregrtest at Lib/test/libregrtest/) +- Library dependencies (e.g., datetime requires _pydatetime) +- Test dependencies (auto-detected from 'from test import ...') +""" + +import ast +import difflib +import functools +import pathlib +import shelve +import subprocess + +from update_lib.file_utils import ( + _dircmp_is_same, + compare_dir_contents, + compare_file_contents, + compare_paths, + construct_lib_path, + cpython_to_local_path, + read_python_files, + resolve_module_path, + resolve_test_path, + safe_parse_ast, + safe_read_text, +) + +# === Import parsing utilities === + + +class ImportVisitor(ast.NodeVisitor): + def __init__(self) -> None: + self.__imports = set() + + @property + def test_imports(self) -> set[str]: + imports = set() + for module in self.__imports: + if not module.startswith("test."): + continue + name = module.removeprefix("test.") + + if name == "support" or name.startswith("support."): + continue + + imports.add(name) + + return imports + + @property + def lib_imports(self) -> set[str]: + return {module for module in self.__imports if not module.startswith("test.")} + + def visit_Import(self, node): + for alias in node.names: + self.__imports.add(alias.name) + + def visit_ImportFrom(self, node): + try: + module = node.module + except AttributeError: + # Ignore `from . import my_internal_module` + return + + if module is None: # Ignore `from . import my_internal_module` + return + + for alias in node.names: + # We only care about what we import if it was from the "test" module + if module == "test": + name = f"{module}.{alias.name}" + else: + name = module + + self.__imports.add(name) + + def visit_Call(self, node) -> None: + """ + In test files, there's sometimes use of: + + ```python + import test.support + from test.support import script_helper + + script = support.findfile("_test_atexit.py") + script_helper.run_test_script(script) + ``` + + This imports "_test_atexit.py" but does not show as an import node. + """ + func = node.func + if not isinstance(func, ast.Attribute): + return + + value = func.value + if not isinstance(value, ast.Name): + return + + if (value.id != "support") or (func.attr != "findfile"): + return + + arg = node.args[0] + if not isinstance(arg, ast.Constant): + return + + target = arg.value + if not target.endswith(".py"): + return + + target = target.removesuffix(".py") + self.__imports.add(f"test.{target}") + + +def parse_test_imports(content: str) -> frozenset[str]: + """Parse test file content and extract test package dependencies.""" + if not (tree := safe_parse_ast(content)): + return set() + + visitor = ImportVisitor() + visitor.visit(tree) + return visitor.test_imports + + +def parse_lib_imports(content: str) -> frozenset[str]: + """Parse library file and extract all imported module names.""" + if not (tree := safe_parse_ast(content)): + return set() + + visitor = ImportVisitor() + visitor.visit(tree) + return visitor.lib_imports + + +# === TODO marker utilities === + +TODO_MARKER = "TODO: RUSTPYTHON" + + +def filter_rustpython_todo(content: str) -> str: + """Remove lines containing RustPython TODO markers.""" + lines = content.splitlines(keepends=True) + return "".join(line for line in lines if TODO_MARKER not in line) + + +def count_rustpython_todo(content: str) -> int: + """Count lines containing RustPython TODO markers.""" + return content.count(TODO_MARKER) + + +def count_todo_in_path(path: pathlib.Path) -> int: + """Count RustPython TODO markers in a file or directory of .py files.""" + if path.is_file(): + content = safe_read_text(path) + return count_rustpython_todo(content) if content else 0 + + return sum(count_rustpython_todo(content) for _, content in read_python_files(path)) + + +# === Test utilities === + + +def _get_cpython_test_path(test_name: str, cpython_prefix: str) -> pathlib.Path | None: + """Return the CPython test path for a test name, or None if missing.""" + cpython_path = resolve_test_path(test_name, cpython_prefix, prefer="dir") + return cpython_path if cpython_path.exists() else None + + +def _get_local_test_path( + cpython_test_path: pathlib.Path, lib_prefix: str +) -> pathlib.Path: + """Return the local Lib/test path matching a CPython test path.""" + return pathlib.Path(lib_prefix) / "test" / cpython_test_path.name + + +def is_test_tracked(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool: + """Check if a test exists in the local Lib/test.""" + cpython_path = _get_cpython_test_path(test_name, cpython_prefix) + if cpython_path is None: + return True + local_path = _get_local_test_path(cpython_path, lib_prefix) + return local_path.exists() + + +def is_test_up_to_date(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool: + """Check if a test is up-to-date, ignoring RustPython TODO markers.""" + cpython_path = _get_cpython_test_path(test_name, cpython_prefix) + if cpython_path is None: + return True + + local_path = _get_local_test_path(cpython_path, lib_prefix) + if not local_path.exists(): + return False + + if cpython_path.is_file(): + return compare_file_contents( + cpython_path, local_path, local_filter=filter_rustpython_todo + ) + + return compare_dir_contents( + cpython_path, local_path, local_filter=filter_rustpython_todo + ) + + +def count_test_todos(test_name: str, lib_prefix: str) -> int: + """Count RustPython TODO markers in a test file/directory.""" + local_dir = pathlib.Path(lib_prefix) / "test" / test_name + local_file = pathlib.Path(lib_prefix) / "test" / f"{test_name}.py" + + if local_dir.exists(): + return count_todo_in_path(local_dir) + if local_file.exists(): + return count_todo_in_path(local_file) + return 0 + + +# === Cross-process cache using shelve === + + +def _get_cpython_version(cpython_prefix: str) -> str: + """Get CPython version from git tag for cache namespace.""" + try: + result = subprocess.run( + ["git", "describe", "--tags", "--abbrev=0"], + cwd=cpython_prefix, + capture_output=True, + text=True, + ) + if result.returncode == 0: + return result.stdout.strip() + except Exception: + pass + return "unknown" + + +def _get_cache_path() -> str: + """Get cache file path (without extension - shelve adds its own).""" + cache_dir = pathlib.Path(__file__).parent / ".cache" + cache_dir.mkdir(parents=True, exist_ok=True) + return str(cache_dir / "import_graph_cache") + + +def clear_import_graph_caches() -> None: + """Clear in-process import graph caches (for testing).""" + if "_test_import_graph_cache" in globals(): + globals()["_test_import_graph_cache"].clear() + if "_lib_import_graph_cache" in globals(): + globals()["_lib_import_graph_cache"].clear() + + +# Manual dependency table for irregular cases +# Format: "name" -> {"lib": [...], "test": [...], "data": [...], "hard_deps": [...]} +# - lib: override default path (default: name.py or name/) +# - hard_deps: additional files to copy alongside the main module +DEPENDENCIES = { + # regrtest is in Lib/test/libregrtest/, not Lib/libregrtest/ + "regrtest": { + "lib": ["test/libregrtest"], + "test": ["test_regrtest"], + "data": ["test/regrtestdata"], + }, + # Rust-implemented modules (no lib file, only test) + "int": { + "lib": [], + "hard_deps": ["_pylong.py"], + "test": [ + "test_int.py", + "test_long.py", + "test_int_literal.py", + ], + }, + "exception": { + "lib": [], + "test": [ + "test_exceptions.py", + "test_baseexception.py", + "test_except_star.py", + "test_exception_group.py", + "test_exception_hierarchy.py", + "test_exception_variations.py", + ], + }, + "dict": { + "lib": [], + "test": [ + "test_dict.py", + "test_dictcomps.py", + "test_dictviews.py", + "test_userdict.py", + "mapping_tests.py", + ], + }, + "list": { + "lib": [], + "test": [ + "test_list.py", + "test_listcomps.py", + "test_userlist.py", + ], + }, + "__future__": { + "test": [ + "test___future__.py", + "test_future_stmt.py", + ], + }, + "site": { + "hard_deps": ["_sitebuiltins.py"], + }, + "opcode": { + "hard_deps": ["_opcode_metadata.py"], + "test": [ + "test_opcode.py", + "test__opcode.py", + "test_opcodes.py", + ], + }, + "pickle": { + "hard_deps": ["_compat_pickle.py"], + "test": [ + "test_pickle.py", + "test_picklebuffer.py", + "test_pickletools.py", + ], + }, + "re": { + "hard_deps": ["sre_compile.py", "sre_constants.py", "sre_parse.py"], + "test": [ + "test_re.py", + "re_tests.py", + ], + }, + "weakref": { + "hard_deps": ["_weakrefset.py"], + "test": [ + "test_weakref.py", + "test_weakset.py", + ], + }, + "codecs": { + "test": [ + "test_charmapcodec.py", + "test_codeccallbacks.py", + "test_codecencodings_cn.py", + "test_codecencodings_hk.py", + "test_codecencodings_iso2022.py", + "test_codecencodings_jp.py", + "test_codecencodings_kr.py", + "test_codecencodings_tw.py", + "test_codecmaps_cn.py", + "test_codecmaps_hk.py", + "test_codecmaps_jp.py", + "test_codecmaps_kr.py", + "test_codecmaps_tw.py", + "test_codecs.py", + "test_multibytecodec.py", + "testcodec.py", + ], + }, + # Non-pattern hard_deps (can't be auto-detected) + "ast": { + "hard_deps": ["_ast_unparse.py"], + "test": [ + "test_ast.py", + "test_unparse.py", + "test_type_comments.py", + ], + }, + # Data directories + "pydoc": { + "hard_deps": ["pydoc_data"], + }, + "turtle": { + "hard_deps": ["turtledemo"], + }, + "sysconfig": { + "hard_deps": ["_aix_support.py", "_osx_support.py"], + "test": [ + "test_sysconfig.py", + "test__osx_support.py", + ], + }, + "tkinter": { + "test": [ + "test_tkinter", + "test_ttk", + "test_ttk_textonly.py", + "test_tcl.py", + "test_idle", + ], + }, + # Test support library (like regrtest) + "support": { + "lib": ["test/support"], + "data": ["test/wheeldata"], + "test": [ + "test_support.py", + "test_script_helper.py", + ], + }, + # test_htmlparser tests html.parser + "html": { + "hard_deps": ["_markupbase.py"], + "test": ["test_html.py", "test_htmlparser.py"], + }, + "xml": { + "test": [ + "test_xml_etree.py", + "test_xml_etree_c.py", + "test_minidom.py", + "test_pulldom.py", + "test_pyexpat.py", + "test_sax.py", + "test_xml_dom_minicompat.py", + "test_xml_dom_xmlbuilder.py", + ], + }, + "multiprocessing": { + "test": [ + "test_multiprocessing_fork", + "test_multiprocessing_forkserver", + "test_multiprocessing_spawn", + "test_multiprocessing_main_handling.py", + "_test_multiprocessing.py", + ], + }, + "urllib": { + "test": [ + "test_urllib.py", + "test_urllib2.py", + "test_urllib2_localnet.py", + "test_urllib2net.py", + "test_urllibnet.py", + "test_urlparse.py", + "test_urllib_response.py", + "test_robotparser.py", + ], + }, + "collections": { + "hard_deps": ["_collections_abc.py"], + "test": [ + "test_collections.py", + "test_deque.py", + "test_defaultdict.py", + "test_ordered_dict.py", + ], + }, + "http": { + "test": [ + "test_httplib.py", + "test_http_cookiejar.py", + "test_http_cookies.py", + "test_httpservers.py", + ], + }, + "unicode": { + "lib": [], + "test": [ + "test_unicodedata.py", + "test_unicode_file.py", + "test_unicode_file_functions.py", + "test_unicode_identifiers.py", + "test_ucn.py", + ], + }, + "typing": { + "test": [ + "test_typing.py", + "test_type_aliases.py", + "test_type_annotations.py", + "test_type_params.py", + "test_genericalias.py", + ], + }, + "unpack": { + "lib": [], + "test": [ + "test_unpack.py", + "test_unpack_ex.py", + ], + }, + "zipimport": { + "test": [ + "test_zipimport.py", + "test_zipimport_support.py", + ], + }, + "time": { + "lib": [], + "test": [ + "test_time.py", + "test_strftime.py", + ], + }, + "sys": { + "lib": [], + "test": [ + "test_sys.py", + "test_syslog.py", + "test_sys_setprofile.py", + "test_sys_settrace.py", + "test_audit.py", + "audit-tests.py", + ], + }, + "str": { + "lib": [], + "test": [ + "test_str.py", + "test_fstring.py", + "test_string_literals.py", + ], + }, + "thread": { + "lib": [], + "test": [ + "test_thread.py", + "test_thread_local_bytecode.py", + "test_threadsignals.py", + ], + }, + "threading": { + "hard_deps": ["_threading_local.py"], + "test": [ + "test_threading.py", + "test_threadedtempfile.py", + "test_threading_local.py", + ], + }, + "class": { + "lib": [], + "test": [ + "test_class.py", + "test_genericclass.py", + "test_subclassinit.py", + ], + }, + "generator": { + "lib": [], + "test": [ + "test_generators.py", + "test_genexps.py", + "test_generator_stop.py", + "test_yield_from.py", + ], + }, + "descr": { + "lib": [], + "test": [ + "test_descr.py", + "test_descrtut.py", + ], + }, + "code": { + "test": [ + "test_code_module.py", + ], + }, + "contextlib": { + "test": [ + "test_contextlib.py", + "test_contextlib_async.py", + ], + }, + "io": { + "hard_deps": ["_pyio.py"], + "test": [ + "test_io.py", + "test_bufio.py", + "test_fileio.py", + "test_memoryio.py", + ], + }, + "dbm": { + "test": [ + "test_dbm.py", + "test_dbm_dumb.py", + "test_dbm_gnu.py", + "test_dbm_ndbm.py", + "test_dbm_sqlite3.py", + ], + }, + "datetime": { + "hard_deps": ["_strptime.py"], + "test": [ + "test_datetime.py", + "test_strptime.py", + ], + }, + "locale": { + "test": [ + "test_locale.py", + "test__locale.py", + ], + }, + "numbers": { + "test": [ + "test_numbers.py", + "test_abstract_numbers.py", + ], + }, + "file": { + "lib": [], + "test": [ + "test_file.py", + "test_largefile.py", + ], + }, + "fcntl": { + "lib": [], + "test": [ + "test_fcntl.py", + "test_ioctl.py", + ], + }, + "select": { + "lib": [], + "test": [ + "test_select.py", + "test_poll.py", + ], + }, + "xmlrpc": { + "test": [ + "test_xmlrpc.py", + "test_docxmlrpc.py", + ], + }, + "ctypes": { + "test": [ + "test_ctypes", + "test_stable_abi_ctypes.py", + ], + }, + # Grouped tests for modules without custom lib paths + "compile": { + "lib": [], + "test": [ + "test_compile.py", + "test_compiler_assemble.py", + "test_compiler_codegen.py", + "test_peepholer.py", + ], + }, + "math": { + "lib": [], + "test": [ + "test_math.py", + "test_math_property.py", + ], + }, + "float": { + "lib": [], + "test": [ + "test_float.py", + "test_strtod.py", + ], + }, + "zipfile": { + "test": [ + "test_zipfile.py", + "test_zipfile64.py", + ], + }, + "smtplib": { + "test": [ + "test_smtplib.py", + "test_smtpnet.py", + ], + }, + "profile": { + "test": [ + "test_profile.py", + "test_cprofile.py", + ], + }, + "string": { + "test": [ + "test_string.py", + "test_userstring.py", + ], + }, + "os": { + "test": [ + "test_os.py", + "test_popen.py", + ], + }, + "pyrepl": { + "test": [ + "test_pyrepl", + "test_repl.py", + ], + }, + "concurrent": { + "test": [ + "test_concurrent_futures", + "test_interpreters", + "test__interpreters.py", + "test__interpchannels.py", + "test_crossinterp.py", + ], + }, + "atexit": { + "test": [ + "test_atexit.py", + "_test_atexit.py", + ], + }, + "eintr": { + "test": [ + "test_eintr.py", + "_test_eintr.py", + ] + }, + "curses": { + "test": [ + "test_curses.py", + "curses_tests.py", + ], + }, +} + + +def resolve_hard_dep_parent(name: str, cpython_prefix: str) -> str | None: + """Resolve a hard_dep name to its parent module. + + Only returns a parent if the file is actually tracked: + - Explicitly listed in DEPENDENCIES as a hard_dep + - Or auto-detected _py{module}.py pattern where the parent module exists + + Args: + name: Module or file name (with or without .py extension) + cpython_prefix: CPython directory prefix + + Returns: + Parent module name if found and tracked, None otherwise + """ + # Normalize: remove .py extension if present + if name.endswith(".py"): + name = name[:-3] + + # Check DEPENDENCIES table first (explicit hard_deps) + for module_name, dep_info in DEPENDENCIES.items(): + hard_deps = dep_info.get("hard_deps", []) + for dep in hard_deps: + # Normalize dep: remove .py extension + dep_normalized = dep[:-3] if dep.endswith(".py") else dep + if dep_normalized == name: + return module_name + + # Auto-detect _py{module} or _py_{module} patterns + # Only if the parent module actually exists + if name.startswith("_py"): + # _py_abc -> abc + # _pydatetime -> datetime + parent = name.removeprefix("_py_").removeprefix("_py") + + # Verify the parent module exists + lib_dir = pathlib.Path(cpython_prefix) / "Lib" + parent_file = lib_dir / f"{parent}.py" + parent_dir = lib_dir / parent + if parent_file.exists() or ( + parent_dir.exists() and (parent_dir / "__init__.py").exists() + ): + return parent + + return None + + +def resolve_test_to_lib(test_name: str) -> str | None: + """Resolve a test name to its library group from DEPENDENCIES. + + Args: + test_name: Test name with or without test_ prefix (e.g., "test_urllib2" or "urllib2") + + Returns: + Library name if test belongs to a group, None otherwise + """ + # Normalize: add test_ prefix if not present + if not test_name.startswith("test_"): + test_name = f"test_{test_name}" + + for lib_name, dep_info in DEPENDENCIES.items(): + tests = dep_info.get("test", []) + for test_path in tests: + # test_path is like "test_urllib2.py" or "test_multiprocessing_fork" + path_stem = test_path.removesuffix(".py") + if path_stem == test_name: + return lib_name + + return None + + +# Test-specific dependencies (only when auto-detection isn't enough) +# - hard_deps: files to migrate (tightly coupled, must be migrated together) +# - data: directories to copy without migration +TEST_DEPENDENCIES = { + # Audio tests + "test_winsound": { + "data": ["audiodata"], + }, + "test_wave": { + "data": ["audiodata"], + }, + "audiotests": { + "data": ["audiodata"], + }, + # Archive tests + "test_tarfile": { + "data": ["archivetestdata"], + }, + "test_zipfile": { + "data": ["archivetestdata"], + }, + # Config tests + "test_configparser": { + "data": ["configdata"], + }, + "test_config": { + "data": ["configdata"], + }, + # Other data directories + "test_decimal": { + "data": ["decimaltestdata"], + }, + "test_dtrace": { + "data": ["dtracedata"], + }, + "test_math": { + "data": ["mathdata"], + }, + "test_ssl": { + "data": ["certdata"], + }, + "test_subprocess": { + "data": ["subprocessdata"], + }, + "test_tkinter": { + "data": ["tkinterdata"], + }, + "test_tokenize": { + "data": ["tokenizedata"], + }, + "test_type_annotations": { + "data": ["typinganndata"], + }, + "test_zipimport": { + "data": ["zipimport_data"], + }, + # XML tests share xmltestdata + "test_xml_etree": { + "data": ["xmltestdata"], + }, + "test_pulldom": { + "data": ["xmltestdata"], + }, + "test_sax": { + "data": ["xmltestdata"], + }, + "test_minidom": { + "data": ["xmltestdata"], + }, + # Multibytecodec support needs cjkencodings + "multibytecodec_support": { + "data": ["cjkencodings"], + }, + # i18n + "i18n_helper": { + "data": ["translationdata"], + }, + # wheeldata is used by test_makefile and support + "test_makefile": { + "data": ["wheeldata"], + }, + # profilee is used by test_monitoring + "test_monitoring": { + "hard_deps": ["profilee"], + }, +} + + +@functools.cache +def get_lib_paths(name: str, cpython_prefix: str) -> tuple[pathlib.Path, ...]: + """Get all library paths for a module. + + Args: + name: Module name (e.g., "datetime", "libregrtest") + cpython_prefix: CPython directory prefix + + Returns: + Tuple of paths to copy + """ + dep_info = DEPENDENCIES.get(name, {}) + + # Get main lib path (override or default) + if "lib" in dep_info: + paths = [construct_lib_path(cpython_prefix, p) for p in dep_info["lib"]] + else: + # Default: try file first, then directory + paths = [resolve_module_path(name, cpython_prefix, prefer="file")] + + # Add hard_deps from DEPENDENCIES + for dep in dep_info.get("hard_deps", []): + paths.append(construct_lib_path(cpython_prefix, dep)) + + # Auto-detect _py{module}.py or _py_{module}.py patterns + for pattern in [f"_py{name}.py", f"_py_{name}.py"]: + auto_path = construct_lib_path(cpython_prefix, pattern) + if auto_path.exists() and auto_path not in paths: + paths.append(auto_path) + + return tuple(paths) + + +def get_all_hard_deps(name: str, cpython_prefix: str) -> list[str]: + """Get all hard_deps for a module (explicit + auto-detected). + + Args: + name: Module name (e.g., "decimal", "datetime") + cpython_prefix: CPython directory prefix + + Returns: + List of hard_dep names (without .py extension) + """ + dep_info = DEPENDENCIES.get(name, {}) + hard_deps = set() + + # Explicit hard_deps from DEPENDENCIES + for hd in dep_info.get("hard_deps", []): + # Remove .py extension if present + hard_deps.add(hd[:-3] if hd.endswith(".py") else hd) + + # Auto-detect _py{module}.py or _py_{module}.py patterns + for pattern in [f"_py{name}.py", f"_py_{name}.py"]: + auto_path = construct_lib_path(cpython_prefix, pattern) + if auto_path.exists(): + hard_deps.add(auto_path.stem) + + return sorted(hard_deps) + + +@functools.cache +def get_test_paths(name: str, cpython_prefix: str) -> tuple[pathlib.Path, ...]: + """Get all test paths for a module. + + Args: + name: Module name (e.g., "datetime", "libregrtest") + cpython_prefix: CPython directory prefix + + Returns: + Tuple of test paths + """ + if name in DEPENDENCIES and "test" in DEPENDENCIES[name]: + return tuple( + construct_lib_path(cpython_prefix, f"test/{p}") + for p in DEPENDENCIES[name]["test"] + ) + + # Default: try directory first, then file + return (resolve_module_path(f"test/test_{name}", cpython_prefix, prefer="dir"),) + + +@functools.cache +def get_all_imports(name: str, cpython_prefix: str) -> frozenset[str]: + """Get all imports from a library file. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + + Returns: + Frozenset of all imported module names + """ + all_imports = set() + for lib_path in get_lib_paths(name, cpython_prefix): + if lib_path.exists(): + for _, content in read_python_files(lib_path): + all_imports.update(parse_lib_imports(content)) + + # Remove self + all_imports.discard(name) + return frozenset(all_imports) + + +@functools.cache +def get_soft_deps(name: str, cpython_prefix: str) -> frozenset[str]: + """Get soft dependencies by parsing imports from library file. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + + Returns: + Frozenset of imported stdlib module names (those that exist in cpython/Lib/) + """ + all_imports = get_all_imports(name, cpython_prefix) + + # Filter: only include modules that exist in cpython/Lib/ + stdlib_deps = set() + for imp in all_imports: + module_path = resolve_module_path(imp, cpython_prefix) + if module_path.exists(): + stdlib_deps.add(imp) + + return frozenset(stdlib_deps) + + +@functools.cache +def get_rust_deps(name: str, cpython_prefix: str) -> frozenset[str]: + """Get Rust/C dependencies (imports that don't exist in cpython/Lib/). + + Args: + name: Module name + cpython_prefix: CPython directory prefix + + Returns: + Frozenset of imported module names that are built-in or C extensions + """ + all_imports = get_all_imports(name, cpython_prefix) + soft_deps = get_soft_deps(name, cpython_prefix) + return frozenset(all_imports - soft_deps) + + +def is_path_synced( + cpython_path: pathlib.Path, + cpython_prefix: str, + lib_prefix: str, +) -> bool: + """Check if a CPython path is synced with local. + + Args: + cpython_path: Path in CPython directory + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + + Returns: + True if synced, False otherwise + """ + local_path = cpython_to_local_path(cpython_path, cpython_prefix, lib_prefix) + if local_path is None: + return False + return compare_paths(cpython_path, local_path) + + +@functools.cache +def is_up_to_date(name: str, cpython_prefix: str, lib_prefix: str) -> bool: + """Check if a module is up-to-date by comparing files. + + Args: + name: Module name + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + + Returns: + True if all files match, False otherwise + """ + lib_paths = get_lib_paths(name, cpython_prefix) + + found_any = False + for cpython_path in lib_paths: + if not cpython_path.exists(): + continue + + found_any = True + + # Convert cpython path to local path + # cpython/Lib/foo.py -> Lib/foo.py + rel_path = cpython_path.relative_to(cpython_prefix) + local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + + if not compare_paths(cpython_path, local_path): + return False + + if not found_any: + dep_info = DEPENDENCIES.get(name, {}) + if dep_info.get("lib") == []: + return True + return found_any + + +def _count_file_diff(file_a: pathlib.Path, file_b: pathlib.Path) -> int: + """Count changed lines between two text files using difflib.""" + a_content = safe_read_text(file_a) + b_content = safe_read_text(file_b) + if a_content is None or b_content is None: + return 0 + if a_content == b_content: + return 0 + a_lines = a_content.splitlines() + b_lines = b_content.splitlines() + count = 0 + for line in difflib.unified_diff(a_lines, b_lines, lineterm=""): + if (line.startswith("+") and not line.startswith("+++")) or ( + line.startswith("-") and not line.startswith("---") + ): + count += 1 + return count + + +def _count_path_diff(path_a: pathlib.Path, path_b: pathlib.Path) -> int: + """Count changed lines between two paths (file or directory, *.py only).""" + if path_a.is_file() and path_b.is_file(): + return _count_file_diff(path_a, path_b) + if path_a.is_dir() and path_b.is_dir(): + total = 0 + a_files = {f.relative_to(path_a) for f in path_a.rglob("*.py")} + b_files = {f.relative_to(path_b) for f in path_b.rglob("*.py")} + for rel in a_files & b_files: + total += _count_file_diff(path_a / rel, path_b / rel) + for rel in a_files - b_files: + content = safe_read_text(path_a / rel) + if content: + total += len(content.splitlines()) + for rel in b_files - a_files: + content = safe_read_text(path_b / rel) + if content: + total += len(content.splitlines()) + return total + return 0 + + +@functools.cache +def _bulk_last_updated() -> dict[str, str]: + """Get last git commit dates for all paths under Lib/ in one git call. + + Keys are Lib/-relative paths (e.g. "re/__init__.py", "test/test_os.py", + "os.py"), plus directory rollups (e.g. "re", "test/test_zoneinfo"). + + Returns: + Dict mapping Lib/-relative path to date string. + """ + file_map: dict[str, str] = {} + try: + result = subprocess.run( + ["git", "log", "--format=%cd", "--date=short", "--name-only", "--", "Lib/"], + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode != 0: + return file_map + except Exception: + return file_map + + current_date = None + for line in result.stdout.splitlines(): + line = line.strip() + if not line: + continue + # Date lines are YYYY-MM-DD format + if len(line) == 10 and line[4] == "-" and line[7] == "-": + current_date = line + elif current_date and line.startswith("Lib/"): + # Strip "Lib/" prefix to get Lib-relative key + rel = line[4:] + if rel and rel not in file_map: + file_map[rel] = current_date + + # Pre-compute directory rollups + dir_map: dict[str, str] = {} + for filepath, date in file_map.items(): + parts = filepath.split("/") + for i in range(1, len(parts)): + dirpath = "/".join(parts[:i]) + if dirpath not in dir_map or date > dir_map[dirpath]: + dir_map[dirpath] = date + + dir_map.update(file_map) + return dir_map + + +@functools.cache +def _lib_prefix_stripped(lib_prefix: str) -> str: + """Get the normalized prefix to strip from paths, with trailing /.""" + # e.g. "Lib" -> "Lib/", "./Lib" -> "Lib/", "../Lib" -> "../Lib/" + return pathlib.Path(lib_prefix).as_posix().rstrip("/") + "/" + + +def _lookup_last_updated(paths: list[str], lib_prefix: str) -> str | None: + """Look up the most recent date among paths from the bulk cache.""" + cache = _bulk_last_updated() + prefix = _lib_prefix_stripped(lib_prefix) + latest = None + for p in paths: + p_norm = pathlib.Path(p).as_posix() + # Strip lib_prefix to get Lib-relative key + # e.g. "Lib/test/test_os.py" -> "test/test_os.py" + # "../Lib/re" -> "re" + if p_norm.startswith(prefix): + key = p_norm[len(prefix) :] + else: + key = p_norm + date = cache.get(key) + if date and (latest is None or date > latest): + latest = date + return latest + + +def get_module_last_updated( + name: str, cpython_prefix: str, lib_prefix: str +) -> str | None: + """Get the last git commit date for a module's Lib files.""" + local_paths = [] + for cpython_path in get_lib_paths(name, cpython_prefix): + if not cpython_path.exists(): + continue + try: + rel_path = cpython_path.relative_to(cpython_prefix) + local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + if local_path.exists(): + local_paths.append(str(local_path)) + except ValueError: + continue + if not local_paths: + return None + return _lookup_last_updated(local_paths, lib_prefix) + + +def get_module_diff_stat(name: str, cpython_prefix: str, lib_prefix: str) -> int: + """Count differing lines between cpython and local Lib for a module.""" + total = 0 + for cpython_path in get_lib_paths(name, cpython_prefix): + if not cpython_path.exists(): + continue + try: + rel_path = cpython_path.relative_to(cpython_prefix) + local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + except ValueError: + continue + if not local_path.exists(): + continue + total += _count_path_diff(cpython_path, local_path) + return total + + +def get_test_last_updated( + test_name: str, cpython_prefix: str, lib_prefix: str +) -> str | None: + """Get the last git commit date for a test's files.""" + cpython_path = _get_cpython_test_path(test_name, cpython_prefix) + if cpython_path is None: + return None + local_path = _get_local_test_path(cpython_path, lib_prefix) + if not local_path.exists(): + return None + return _lookup_last_updated([str(local_path)], lib_prefix) + + +def get_test_dependencies( + test_path: pathlib.Path, +) -> dict[str, list[pathlib.Path]]: + """Get test dependencies by parsing imports. + + Args: + test_path: Path to test file or directory + + Returns: + Dict with "hard_deps" (files to migrate) and "data" (dirs to copy) + """ + result = {"hard_deps": [], "data": []} + + if not test_path.exists(): + return result + + # Parse all files for imports (auto-detect deps) + all_imports = set() + for _, content in read_python_files(test_path): + all_imports.update(parse_test_imports(content)) + + # Also add manual dependencies from TEST_DEPENDENCIES + test_name = test_path.stem if test_path.is_file() else test_path.name + manual_deps = TEST_DEPENDENCIES.get(test_name, {}) + if "hard_deps" in manual_deps: + all_imports.update(manual_deps["hard_deps"]) + + # Convert imports to paths (deps) + for imp in all_imports: + # Skip other test modules (test_*) - they are independently managed + # via their own update_lib entry. Only support/helper modules + # (e.g., string_tests, mapping_tests) should be treated as hard deps. + if imp.startswith("test_"): + continue + + dep_path = test_path.parent / f"{imp}.py" + if not dep_path.exists(): + dep_path = test_path.parent / imp + + if dep_path.exists() and dep_path not in result["hard_deps"]: + result["hard_deps"].append(dep_path) + + # Add data paths from manual table (for the test file itself) + if "data" in manual_deps: + for data_name in manual_deps["data"]: + data_path = test_path.parent / data_name + if data_path.exists() and data_path not in result["data"]: + result["data"].append(data_path) + + # Also add data from auto-detected deps' TEST_DEPENDENCIES + # e.g., test_codecencodings_kr -> multibytecodec_support -> cjkencodings + for imp in all_imports: + dep_info = TEST_DEPENDENCIES.get(imp, {}) + if "data" in dep_info: + for data_name in dep_info["data"]: + data_path = test_path.parent / data_name + if data_path.exists() and data_path not in result["data"]: + result["data"].append(data_path) + + return result + + +def _parse_test_submodule_imports(content: str) -> dict[str, set[str]]: + """Parse 'from test.X import Y' to get submodule imports. + + Args: + content: Python file content + + Returns: + Dict mapping submodule (e.g., "test_bar") -> set of imported names (e.g., {"helper"}) + """ + tree = safe_parse_ast(content) + if tree is None: + return {} + + result: dict[str, set[str]] = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.module and node.module.startswith("test."): + # from test.test_bar import helper -> test_bar: {helper} + parts = node.module.split(".") + if len(parts) >= 2: + submodule = parts[1] + if submodule not in ("support", "__init__"): + if submodule not in result: + result[submodule] = set() + for alias in node.names: + result[submodule].add(alias.name) + + return result + + +_test_import_graph_cache: dict[ + str, tuple[dict[str, set[str]], dict[str, set[str]]] +] = {} + + +def _is_standard_lib_path(path: str) -> bool: + """Check if path is the standard Lib directory (not a temp dir).""" + if "/tmp" in path.lower() or "/var/folders" in path.lower(): + return False + return ( + path == "Lib/test" + or path.endswith("/Lib/test") + or path == "Lib" + or path.endswith("/Lib") + ) + + +def _build_test_import_graph( + test_dir: pathlib.Path, +) -> tuple[dict[str, set[str]], dict[str, set[str]]]: + """Build import graphs for files within test directory (recursive). + + Uses cross-process shelve cache based on CPython version. + + Args: + test_dir: Path to Lib/test/ directory + + Returns: + Tuple of: + - Dict mapping relative path (without .py) -> set of test modules it imports + - Dict mapping relative path (without .py) -> set of all lib imports + """ + # In-process cache + cache_key = str(test_dir) + if cache_key in _test_import_graph_cache: + return _test_import_graph_cache[cache_key] + + # Cross-process cache (only for standard Lib/test directory) + use_file_cache = _is_standard_lib_path(cache_key) + if use_file_cache: + version = _get_cpython_version("cpython") + shelve_key = f"test_import_graph:{version}" + try: + with shelve.open(_get_cache_path()) as db: + if shelve_key in db: + import_graph, lib_imports_graph = db[shelve_key] + _test_import_graph_cache[cache_key] = ( + import_graph, + lib_imports_graph, + ) + return import_graph, lib_imports_graph + except Exception: + pass + + # Build from scratch + import_graph: dict[str, set[str]] = {} + lib_imports_graph: dict[str, set[str]] = {} + + for py_file in test_dir.glob("**/*.py"): + content = safe_read_text(py_file) + if content is None: + continue + + imports = set() + imports.update(parse_test_imports(content)) + all_imports = parse_lib_imports(content) + + for imp in all_imports: + if (py_file.parent / f"{imp}.py").exists(): + imports.add(imp) + if (test_dir / f"{imp}.py").exists(): + imports.add(imp) + + submodule_imports = _parse_test_submodule_imports(content) + for submodule, imported_names in submodule_imports.items(): + submodule_dir = test_dir / submodule + if submodule_dir.is_dir(): + for name in imported_names: + if (submodule_dir / f"{name}.py").exists(): + imports.add(name) + + rel_path = py_file.relative_to(test_dir) + key = str(rel_path.with_suffix("")) + import_graph[key] = imports + lib_imports_graph[key] = all_imports + + # Save to cross-process cache + if use_file_cache: + try: + with shelve.open(_get_cache_path()) as db: + db[shelve_key] = (import_graph, lib_imports_graph) + except Exception: + pass + _test_import_graph_cache[cache_key] = (import_graph, lib_imports_graph) + + return import_graph, lib_imports_graph + + +_lib_import_graph_cache: dict[str, dict[str, set[str]]] = {} + + +def _build_lib_import_graph(lib_prefix: str) -> dict[str, set[str]]: + """Build import graph for Lib modules (full module paths like urllib.request). + + Uses cross-process shelve cache based on CPython version. + + Args: + lib_prefix: RustPython Lib directory + + Returns: + Dict mapping full_module_path -> set of modules it imports + """ + # In-process cache + if lib_prefix in _lib_import_graph_cache: + return _lib_import_graph_cache[lib_prefix] + + # Cross-process cache (only for standard Lib directory) + use_file_cache = _is_standard_lib_path(lib_prefix) + if use_file_cache: + version = _get_cpython_version("cpython") + shelve_key = f"lib_import_graph:{version}" + try: + with shelve.open(_get_cache_path()) as db: + if shelve_key in db: + import_graph = db[shelve_key] + _lib_import_graph_cache[lib_prefix] = import_graph + return import_graph + except Exception: + pass + + # Build from scratch + lib_dir = pathlib.Path(lib_prefix) + if not lib_dir.exists(): + return {} + + import_graph: dict[str, set[str]] = {} + + for entry in lib_dir.iterdir(): + if entry.name.startswith(("_", ".")): + continue + if entry.name == "test": + continue + + if entry.is_file() and entry.suffix == ".py": + content = safe_read_text(entry) + if content: + imports = parse_lib_imports(content) + imports.discard(entry.stem) + import_graph[entry.stem] = imports + elif entry.is_dir() and (entry / "__init__.py").exists(): + for py_file in entry.glob("**/*.py"): + content = safe_read_text(py_file) + if content: + imports = parse_lib_imports(content) + rel_path = py_file.relative_to(lib_dir) + if rel_path.name == "__init__.py": + full_name = str(rel_path.parent).replace("/", ".") + else: + full_name = str(rel_path.with_suffix("")).replace("/", ".") + imports.discard(full_name.split(".")[0]) + import_graph[full_name] = imports + + # Save to cross-process cache + if use_file_cache: + try: + with shelve.open(_get_cache_path()) as db: + db[shelve_key] = import_graph + except Exception: + pass + _lib_import_graph_cache[lib_prefix] = import_graph + + return import_graph + + +def _get_lib_modules_importing( + module_name: str, lib_import_graph: dict[str, set[str]] +) -> set[str]: + """Find Lib modules (full paths) that import module_name or any of its submodules.""" + importers: set[str] = set() + target_top = module_name.split(".")[0] + + for full_path, imports in lib_import_graph.items(): + if full_path.split(".")[0] == target_top: + continue # Skip same package + # Match if module imports target OR any submodule of target + # e.g., for "xml": match imports of "xml", "xml.parsers", "xml.etree.ElementTree" + matches = any( + imp == module_name or imp.startswith(module_name + ".") for imp in imports + ) + if matches: + importers.add(full_path) + + return importers + + +def _consolidate_submodules( + modules: set[str], threshold: int = 3 +) -> dict[str, set[str]]: + """Consolidate submodules if count exceeds threshold. + + Args: + modules: Set of full module paths (e.g., {"urllib.request", "urllib.parse", "xml.dom", "xml.sax"}) + threshold: If submodules > threshold, consolidate to parent + + Returns: + Dict mapping display_name -> set of original module paths + e.g., {"urllib.request": {"urllib.request"}, "xml": {"xml.dom", "xml.sax", "xml.etree", "xml.parsers"}} + """ + # Group by top-level package + by_package: dict[str, set[str]] = {} + for mod in modules: + parts = mod.split(".") + top = parts[0] + if top not in by_package: + by_package[top] = set() + by_package[top].add(mod) + + result: dict[str, set[str]] = {} + for top, submods in by_package.items(): + if len(submods) > threshold: + # Consolidate to top-level + result[top] = submods + else: + # Keep individual + for mod in submods: + result[mod] = {mod} + + return result + + +# Modules that are used everywhere - show but don't expand their dependents +_BLOCKLIST_MODULES = frozenset( + { + "unittest", + "test.support", + "support", + "doctest", + "typing", + "abc", + "collections.abc", + "functools", + "itertools", + "operator", + "contextlib", + "warnings", + "types", + "enum", + "re", + "io", + "os", + "sys", + } +) + + +def find_dependent_tests_tree( + module_name: str, + lib_prefix: str, + max_depth: int = 1, + _depth: int = 0, + _visited_tests: set[str] | None = None, + _visited_modules: set[str] | None = None, +) -> dict: + """Find dependent tests in a tree structure. + + Args: + module_name: Module to search for (e.g., "ftplib") + lib_prefix: RustPython Lib directory + max_depth: Maximum depth to recurse (default 1 = show direct + 1 level of Lib deps) + + Returns: + Dict with structure: + { + "module": "ftplib", + "tests": ["test_ftplib", "test_urllib2"], # Direct importers + "children": [ + {"module": "urllib.request", "tests": [...], "children": []}, + ... + ] + } + """ + lib_dir = pathlib.Path(lib_prefix) + test_dir = lib_dir / "test" + + if _visited_tests is None: + _visited_tests = set() + if _visited_modules is None: + _visited_modules = set() + + # Build graphs + test_import_graph, test_lib_imports = _build_test_import_graph(test_dir) + lib_import_graph = _build_lib_import_graph(lib_prefix) + + # Find tests that directly import this module + target_top = module_name.split(".")[0] + direct_tests: set[str] = set() + for file_key, imports in test_lib_imports.items(): + if file_key in _visited_tests: + continue + # Match exact module OR any child submodule + # e.g., "xml" matches imports of "xml", "xml.parsers", "xml.etree.ElementTree" + # but "collections._defaultdict" only matches "collections._defaultdict" (no children) + matches = any( + imp == module_name or imp.startswith(module_name + ".") for imp in imports + ) + if matches: + # Check if it's a test file + if pathlib.Path(file_key).name.startswith("test_"): + direct_tests.add(file_key) + _visited_tests.add(file_key) + + # Consolidate test names (test_sqlite3/test_dbapi -> test_sqlite3) + consolidated_tests = {_consolidate_file_key(t) for t in direct_tests} + + # Mark this module as visited (cycle detection) + _visited_modules.add(module_name) + _visited_modules.add(target_top) + + children = [] + # Check blocklist and depth limit + should_expand = ( + _depth < max_depth + and module_name not in _BLOCKLIST_MODULES + and target_top not in _BLOCKLIST_MODULES + ) + + if should_expand: + # Find Lib modules that import this module + lib_importers = _get_lib_modules_importing(module_name, lib_import_graph) + + # Skip already visited modules (cycle detection) and blocklisted modules + lib_importers = { + m + for m in lib_importers + if m not in _visited_modules + and m.split(".")[0] not in _visited_modules + and m not in _BLOCKLIST_MODULES + and m.split(".")[0] not in _BLOCKLIST_MODULES + } + + # Consolidate submodules (xml.dom, xml.sax, xml.etree -> xml if > 3) + consolidated_libs = _consolidate_submodules(lib_importers, threshold=3) + + # Build children + for display_name, original_mods in sorted(consolidated_libs.items()): + child = find_dependent_tests_tree( + display_name, + lib_prefix, + max_depth, + _depth + 1, + _visited_tests, + _visited_modules, + ) + if child["tests"] or child["children"]: + children.append(child) + + return { + "module": module_name, + "tests": sorted(consolidated_tests), + "children": children, + } + + +def _consolidate_file_key(file_key: str) -> str: + """Consolidate file_key to test name. + + Args: + file_key: Relative path without .py (e.g., "test_foo", "test_bar/test_sub") + + Returns: + Consolidated test name: + - "test_foo" for "test_foo" + - "test_sqlite3" for "test_sqlite3/test_dbapi" + """ + parts = pathlib.Path(file_key).parts + if len(parts) == 1: + return parts[0] + return parts[0] diff --git a/scripts/update_lib/file_utils.py b/scripts/update_lib/file_utils.py new file mode 100644 index 00000000000..cb86ee2e664 --- /dev/null +++ b/scripts/update_lib/file_utils.py @@ -0,0 +1,289 @@ +""" +File utilities for update_lib. + +This module provides functions for: +- Safe file reading with error handling +- Safe AST parsing with error handling +- Iterating over Python files +- Parsing and converting library paths +- Detecting test paths vs library paths +- Comparing files or directories for equality +""" + +from __future__ import annotations + +import ast +import filecmp +import pathlib +from collections.abc import Callable, Iterator + +# === I/O utilities === + + +def safe_read_text(path: pathlib.Path) -> str | None: + """Read file content with UTF-8 encoding, returning None on error.""" + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return None + + +def safe_parse_ast(content: str) -> ast.Module | None: + """Parse Python content into AST, returning None on syntax error.""" + try: + return ast.parse(content) + except SyntaxError: + return None + + +def iter_python_files(path: pathlib.Path) -> Iterator[pathlib.Path]: + """Yield Python files from a file or directory.""" + if path.is_file(): + yield path + else: + yield from path.glob("**/*.py") + + +def read_python_files(path: pathlib.Path) -> Iterator[tuple[pathlib.Path, str]]: + """Read all Python files from a path, yielding (path, content) pairs.""" + for py_file in iter_python_files(path): + content = safe_read_text(py_file) + if content is not None: + yield py_file, content + + +# === Path utilities === + + +def parse_lib_path(path: pathlib.Path | str) -> pathlib.Path: + """ + Extract the Lib/... portion from a path containing /Lib/. + + Example: + parse_lib_path("cpython/Lib/test/foo.py") -> Path("Lib/test/foo.py") + """ + path_str = str(path).replace("\\", "/") + lib_marker = "/Lib/" + + if lib_marker not in path_str: + raise ValueError(f"Path must contain '/Lib/' or '\\Lib\\' (got: {path})") + + idx = path_str.index(lib_marker) + return pathlib.Path(path_str[idx + 1 :]) + + +def is_lib_path(path: pathlib.Path) -> bool: + """Check if path starts with Lib/""" + path_str = str(path).replace("\\", "/") + return path_str.startswith("Lib/") or path_str.startswith("./Lib/") + + +def is_test_path(path: pathlib.Path) -> bool: + """Check if path is a test path (contains /Lib/test/ or starts with Lib/test/)""" + path_str = str(path).replace("\\", "/") + return "/Lib/test/" in path_str or path_str.startswith("Lib/test/") + + +def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path: + """ + Convert library path to test path. + + Examples: + cpython/Lib/dataclasses.py -> cpython/Lib/test/test_dataclasses/ + cpython/Lib/json/__init__.py -> cpython/Lib/test/test_json/ + """ + path_str = str(src_path).replace("\\", "/") + lib_marker = "/Lib/" + + if lib_marker in path_str: + lib_path = parse_lib_path(src_path) + lib_name = lib_path.stem if lib_path.suffix == ".py" else lib_path.name + if lib_name == "__init__": + lib_name = lib_path.parent.name + prefix = path_str[: path_str.index(lib_marker)] + dir_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}/") + if dir_path.exists(): + return dir_path + file_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}.py") + if file_path.exists(): + return file_path + return dir_path + else: + lib_name = src_path.stem if src_path.suffix == ".py" else src_path.name + if lib_name == "__init__": + lib_name = src_path.parent.name + dir_path = pathlib.Path(f"Lib/test/test_{lib_name}/") + if dir_path.exists(): + return dir_path + file_path = pathlib.Path(f"Lib/test/test_{lib_name}.py") + if file_path.exists(): + return file_path + return dir_path + + +def get_test_files(path: pathlib.Path) -> list[pathlib.Path]: + """Get all .py test files in a path (file or directory).""" + if path.is_file(): + return [path] + return sorted(path.glob("**/*.py")) + + +def get_test_module_name(test_path: pathlib.Path) -> str: + """ + Extract test module name from a test file path. + + Examples: + Lib/test/test_foo.py -> test_foo + Lib/test/test_ctypes/test_bar.py -> test_ctypes.test_bar + """ + test_path = pathlib.Path(test_path) + if test_path.parent.name.startswith("test_"): + return f"{test_path.parent.name}.{test_path.stem}" + return test_path.stem + + +def resolve_module_path( + name: str, prefix: str = "cpython", prefer: str = "file" +) -> pathlib.Path: + """ + Resolve module path, trying file or directory. + + Args: + name: Module name (e.g., "dataclasses", "json") + prefix: CPython directory prefix + prefer: "file" to try .py first, "dir" to try directory first + """ + file_path = pathlib.Path(f"{prefix}/Lib/{name}.py") + dir_path = pathlib.Path(f"{prefix}/Lib/{name}") + + if prefer == "file": + if file_path.exists(): + return file_path + if dir_path.exists(): + return dir_path + return file_path + else: + if dir_path.exists(): + return dir_path + if file_path.exists(): + return file_path + return dir_path + + +def construct_lib_path(prefix: str, *parts: str) -> pathlib.Path: + """Build a path under prefix/Lib/.""" + return pathlib.Path(prefix) / "Lib" / pathlib.Path(*parts) + + +def resolve_test_path( + test_name: str, prefix: str = "cpython", prefer: str = "dir" +) -> pathlib.Path: + """Resolve a test module path under Lib/test/.""" + return resolve_module_path(f"test/{test_name}", prefix, prefer=prefer) + + +def cpython_to_local_path( + cpython_path: pathlib.Path, + cpython_prefix: str, + lib_prefix: str, +) -> pathlib.Path | None: + """Convert CPython path to local Lib path.""" + try: + rel_path = cpython_path.relative_to(cpython_prefix) + return pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + except ValueError: + return None + + +def get_module_name(path: pathlib.Path) -> str: + """Extract module name from path, handling __init__.py.""" + if path.suffix == ".py": + name = path.stem + if name == "__init__": + return path.parent.name + return name + return path.name + + +def get_cpython_dir(src_path: pathlib.Path) -> pathlib.Path: + """Extract CPython directory from a path containing /Lib/.""" + path_str = str(src_path).replace("\\", "/") + lib_marker = "/Lib/" + if lib_marker in path_str: + idx = path_str.index(lib_marker) + return pathlib.Path(path_str[:idx]) + return pathlib.Path("cpython") + + +# === Comparison utilities === + + +def _dircmp_is_same(dcmp: filecmp.dircmp) -> bool: + """Recursively check if two directories are identical.""" + if dcmp.diff_files or dcmp.left_only or dcmp.right_only: + return False + + for subdir in dcmp.subdirs.values(): + if not _dircmp_is_same(subdir): + return False + + return True + + +def compare_paths(cpython_path: pathlib.Path, local_path: pathlib.Path) -> bool: + """Compare a CPython path with a local path (file or directory).""" + if not local_path.exists(): + return False + + if cpython_path.is_file(): + return filecmp.cmp(cpython_path, local_path, shallow=False) + + dcmp = filecmp.dircmp(cpython_path, local_path) + return _dircmp_is_same(dcmp) + + +def compare_file_contents( + cpython_path: pathlib.Path, + local_path: pathlib.Path, + *, + local_filter: Callable[[str], str] | None = None, + encoding: str = "utf-8", +) -> bool: + """Compare two files as text, optionally filtering local content.""" + try: + cpython_content = cpython_path.read_text(encoding=encoding) + local_content = local_path.read_text(encoding=encoding) + except (OSError, UnicodeDecodeError): + return False + + if local_filter is not None: + local_content = local_filter(local_content) + + return cpython_content == local_content + + +def compare_dir_contents( + cpython_dir: pathlib.Path, + local_dir: pathlib.Path, + *, + pattern: str = "*.py", + local_filter: Callable[[str], str] | None = None, + encoding: str = "utf-8", +) -> bool: + """Compare directory contents for matching files and text.""" + cpython_files = {f.relative_to(cpython_dir) for f in cpython_dir.rglob(pattern)} + local_files = {f.relative_to(local_dir) for f in local_dir.rglob(pattern)} + + if cpython_files != local_files: + return False + + for rel_path in cpython_files: + if not compare_file_contents( + cpython_dir / rel_path, + local_dir / rel_path, + local_filter=local_filter, + encoding=encoding, + ): + return False + + return True diff --git a/scripts/update_lib/patch_spec.py b/scripts/update_lib/patch_spec.py new file mode 100644 index 00000000000..d27d2e22fa7 --- /dev/null +++ b/scripts/update_lib/patch_spec.py @@ -0,0 +1,397 @@ +""" +Low-level module for converting between test files and JSON patches. + +This module handles: +- Extracting patches from test files (file -> JSON) +- Applying patches to test files (JSON -> file) +""" + +import ast +import collections +import enum +import textwrap +import typing + +if typing.TYPE_CHECKING: + from collections.abc import Iterator + +type Patches = dict[str, dict[str, list["PatchSpec"]]] + +DEFAULT_INDENT = " " * 4 +COMMENT = "TODO: RUSTPYTHON" +UT = "unittest" + + +@enum.unique +class UtMethod(enum.StrEnum): + """ + UnitTest Method. + """ + + def _generate_next_value_(name, start, count, last_values) -> str: + return name[0].lower() + name[1:] + + def has_args(self) -> bool: + return self != self.ExpectedFailure + + def has_cond(self) -> bool: + return self.endswith(("If", "Unless")) + + ExpectedFailure = enum.auto() + ExpectedFailureIf = enum.auto() + ExpectedFailureIfWindows = enum.auto() + Skip = enum.auto() + SkipIf = enum.auto() + SkipUnless = enum.auto() + + +class PatchSpec(typing.NamedTuple): + """ + Attributes + ---------- + ut_method : UtMethod + unittest method. + cond : str, optional + `ut_method` condition. Relevant only for some of `ut_method` types. + reason : str, optional + Reason for why the test is patched in this way. + """ + + ut_method: UtMethod + cond: str | None = None + reason: str = "" + + @property + def _reason(self) -> str: + return f"{COMMENT}; {self.reason}".strip(" ;") + + @property + def _attr_node(self) -> ast.Attribute: + return ast.Attribute(value=ast.Name(id=UT), attr=self.ut_method) + + def as_ast_node(self) -> ast.Attribute | ast.Call: + if not self.ut_method.has_args(): + return self._attr_node + + args = [] + if self.cond: + args.append(ast.parse(self.cond).body[0].value) + args.append(ast.Constant(value=self._reason)) + + return ast.Call(func=self._attr_node, args=args, keywords=[]) + + def as_decorator(self) -> str: + unparsed = ast.unparse(self.as_ast_node()) + # ast.unparse uses single quotes; convert to double quotes for ruff compatibility + unparsed = _single_to_double_quotes(unparsed) + + if not self.ut_method.has_args(): + unparsed = f"{unparsed} # {self._reason}" + + return f"@{unparsed}" + + +def _single_to_double_quotes(s: str) -> str: + """Convert single-quoted strings to double-quoted strings. + + Falls back to original if conversion breaks the AST equivalence. + """ + import re + + def replace_string(match: re.Match) -> str: + content = match.group(1) + # Unescape single quotes and escape double quotes + content = content.replace("\\'", "'").replace('"', '\\"') + return f'"{content}"' + + # Match single-quoted strings (handles escaped single quotes inside) + converted = re.sub(r"'((?:[^'\\]|\\.)*)'", replace_string, s) + + # Verify: parse converted and unparse should equal original + try: + converted_ast = ast.parse(converted, mode="eval") + if ast.unparse(converted_ast) == s: + return converted + except SyntaxError: + pass + + # Fall back to original if conversion failed + return s + + +class PatchEntry(typing.NamedTuple): + """ + Stores patch metadata. + + Attributes + ---------- + parent_class : str + Parent class of test. + test_name : str + Test name. + spec : PatchSpec + Patch spec. + """ + + parent_class: str + test_name: str + spec: PatchSpec + + @classmethod + def iter_patch_entries( + cls, tree: ast.Module, lines: list[str] + ) -> "Iterator[typing.Self]": + import re + import sys + + for cls_node, fn_node in iter_tests(tree): + parent_class = cls_node.name + for dec_node in fn_node.decorator_list: + if not isinstance(dec_node, (ast.Attribute, ast.Call)): + continue + + attr_node = ( + dec_node if isinstance(dec_node, ast.Attribute) else dec_node.func + ) + + if ( + isinstance(attr_node, ast.Name) + or getattr(attr_node.value, "id", None) != UT + ): + continue + + cond = None + try: + ut_method = UtMethod(attr_node.attr) + except ValueError: + continue + + # If our ut_method has args then, + # we need to search for a constant that contains our `COMMENT`. + # Otherwise we need to search it in the raw source code :/ + if ut_method.has_args(): + reason = next( + ( + node.value + for node in ast.walk(dec_node) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and COMMENT in node.value + ), + None, + ) + + # If we didn't find a constant containing <COMMENT>, + # then we didn't put this decorator + if not reason: + continue + + if ut_method.has_cond(): + cond = ast.unparse(dec_node.args[0]) + else: + # Search first on decorator line, then in the line before + for line in lines[dec_node.lineno - 1 : dec_node.lineno - 3 : -1]: + if found := re.search(rf"{COMMENT}.?(.*)", line): + reason = found.group() + break + else: + # Didn't find our `COMMENT` :) + continue + + reason = reason.removeprefix(COMMENT).strip(";:, ") + spec = PatchSpec(ut_method, cond, reason) + yield cls(parent_class, fn_node.name, spec) + + +def iter_tests( + tree: ast.Module, +) -> "Iterator[tuple[ast.ClassDef, ast.FunctionDef | ast.AsyncFunctionDef]]": + for key, nodes in ast.iter_fields(tree): + if key != "body": + continue + + for cls_node in nodes: + if not isinstance(cls_node, ast.ClassDef): + continue + + for fn_node in cls_node.body: + if not isinstance(fn_node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + + yield (cls_node, fn_node) + + +def iter_patches(contents: str) -> "Iterator[PatchEntry]": + lines = contents.splitlines() + tree = ast.parse(contents) + yield from PatchEntry.iter_patch_entries(tree, lines) + + +def build_patch_dict(it: "Iterator[PatchEntry]") -> Patches: + patches = collections.defaultdict(lambda: collections.defaultdict(list)) + for entry in it: + patches[entry.parent_class][entry.test_name].append(entry.spec) + + return {k: dict(v) for k, v in patches.items()} + + +def extract_patches(contents: str) -> Patches: + """Extract patches from file contents and return as dict.""" + return build_patch_dict(iter_patches(contents)) + + +def _iter_patch_lines( + tree: ast.Module, patches: Patches +) -> "Iterator[tuple[int, str]]": + import sys + + # Build cache of all classes (for Phase 2 to find classes without methods) + cache = {} + # Build per-class set of async method names (for Phase 2 to generate correct override) + async_methods: dict[str, set[str]] = {} + # Track class bases for inherited async method lookup + class_bases: dict[str, list[str]] = {} + all_classes = {node.name for node in tree.body if isinstance(node, ast.ClassDef)} + for node in tree.body: + if isinstance(node, ast.ClassDef): + cache[node.name] = node.end_lineno + class_bases[node.name] = [ + base.id + for base in node.bases + if isinstance(base, ast.Name) and base.id in all_classes + ] + cls_async: set[str] = set() + for item in node.body: + if isinstance(item, ast.AsyncFunctionDef): + cls_async.add(item.name) + if cls_async: + async_methods[node.name] = cls_async + + # Phase 1: Iterate and mark existing tests + for cls_node, fn_node in iter_tests(tree): + specs = patches.get(cls_node.name, {}).pop(fn_node.name, None) + if not specs: + continue + + lineno = min( + (dec_node.lineno for dec_node in fn_node.decorator_list), + default=fn_node.lineno, + ) + indent = " " * fn_node.col_offset + patch_lines = "\n".join(spec.as_decorator() for spec in specs) + yield (lineno - 1, textwrap.indent(patch_lines, indent)) + + # Phase 2: Iterate and mark inherited tests + for cls_name, tests in sorted(patches.items()): + lineno = cache.get(cls_name) + if not lineno: + print(f"WARNING: {cls_name} does not exist in remote file", file=sys.stderr) + continue + + for test_name, specs in sorted(tests.items()): + decorators = "\n".join(spec.as_decorator() for spec in specs) + # Check current class and ancestors for async method + is_async = False + queue = [cls_name] + visited: set[str] = set() + while queue: + cur = queue.pop(0) + if cur in visited: + continue + visited.add(cur) + if test_name in async_methods.get(cur, set()): + is_async = True + break + queue.extend(class_bases.get(cur, [])) + if is_async: + patch_lines = f""" +{decorators} +async def {test_name}(self): +{DEFAULT_INDENT}return await super().{test_name}() +""".rstrip() + else: + patch_lines = f""" +{decorators} +def {test_name}(self): +{DEFAULT_INDENT}return super().{test_name}() +""".rstrip() + yield (lineno, textwrap.indent(patch_lines, DEFAULT_INDENT)) + + +def _has_unittest_import(tree: ast.Module) -> bool: + """Check if 'import unittest' is already present in the file.""" + for node in tree.body: + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == UT and alias.asname is None: + return True + return False + + +def _find_import_insert_line(tree: ast.Module) -> int: + """Find the line number after the last import statement.""" + last_import_line = None + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + last_import_line = node.end_lineno or node.lineno + if last_import_line is not None: + return last_import_line + # No imports found - insert after module docstring if present, else at top + if ( + tree.body + and isinstance(tree.body[0], ast.Expr) + and isinstance(tree.body[0].value, ast.Constant) + and isinstance(tree.body[0].value.value, str) + ): + return tree.body[0].end_lineno or tree.body[0].lineno + return 0 + + +def apply_patches(contents: str, patches: Patches) -> str: + """Apply patches to file contents and return modified contents.""" + tree = ast.parse(contents) + lines = contents.splitlines() + + modifications = list(_iter_patch_lines(tree, patches)) + + # If we have modifications and unittest is not imported, add it + if modifications and not _has_unittest_import(tree): + import_line = _find_import_insert_line(tree) + modifications.append( + ( + import_line, + "\nimport unittest # XXX: RUSTPYTHON; importing to be able to skip tests", + ) + ) + + # Going in reverse to not disrupt the line offset + for lineno, patch in sorted(modifications, reverse=True): + lines.insert(lineno, patch) + + joined = "\n".join(lines) + return f"{joined}\n" + + +def patches_to_json(patches: Patches) -> dict: + """Convert patches to JSON-serializable dict.""" + return { + cls_name: { + test_name: [spec._asdict() for spec in specs] + for test_name, specs in tests.items() + } + for cls_name, tests in patches.items() + } + + +def patches_from_json(data: dict) -> Patches: + """Convert JSON dict back to Patches.""" + return { + cls_name: { + test_name: [ + PatchSpec(**spec)._replace(ut_method=UtMethod(spec["ut_method"])) + for spec in specs + ] + for test_name, specs in tests.items() + } + for cls_name, tests in data.items() + } diff --git a/scripts/update_lib/tests/__init__.py b/scripts/update_lib/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/update_lib/tests/test_auto_mark.py b/scripts/update_lib/tests/test_auto_mark.py new file mode 100644 index 00000000000..ce89b0f9918 --- /dev/null +++ b/scripts/update_lib/tests/test_auto_mark.py @@ -0,0 +1,1085 @@ +"""Tests for auto_mark.py - test result parsing and auto-marking.""" + +import ast +import pathlib +import subprocess +import tempfile +import unittest +from unittest import mock + +from update_lib.cmd_auto_mark import ( + Test, + TestResult, + TestRunError, + _expand_stripped_to_children, + _is_super_call_only, + apply_test_changes, + auto_mark_directory, + auto_mark_file, + collect_test_changes, + extract_test_methods, + parse_results, + path_to_test_parts, + remove_expected_failures, + strip_reasonless_expected_failures, +) +from update_lib.patch_spec import COMMENT + + +def _make_result(stdout: str) -> subprocess.CompletedProcess: + return subprocess.CompletedProcess( + args=["test"], returncode=0, stdout=stdout, stderr="" + ) + + +# -- fixtures shared across inheritance-aware tests -- + +BASE_TWO_CHILDREN = """import unittest + +class Base: + def test_foo(self): + pass + +class ChildA(Base, unittest.TestCase): + pass + +class ChildB(Base, unittest.TestCase): + pass +""" + +BASE_TWO_CHILDREN_ONE_OVERRIDE = """import unittest + +class Base: + def test_foo(self): + pass + +class ChildA(Base, unittest.TestCase): + pass + +class ChildB(Base, unittest.TestCase): + def test_foo(self): + # own implementation + pass +""" + + +class TestParseResults(unittest.TestCase): + """Tests for parse_results function.""" + + def test_parse_fail_and_error(self): + """FAIL and ERROR are collected; ok is ignored.""" + stdout = """\ +Run 3 tests sequentially +test_one (test.test_example.TestA.test_one) ... FAIL +test_two (test.test_example.TestA.test_two) ... ok +test_three (test.test_example.TestB.test_three) ... ERROR +----------- +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(len(result.tests), 2) + by_name = {t.name: t for t in result.tests} + self.assertEqual(by_name["test_one"].path, "test.test_example.TestA.test_one") + self.assertEqual(by_name["test_one"].result, "fail") + self.assertEqual(by_name["test_three"].result, "error") + + def test_parse_unexpected_success(self): + stdout = """\ +Run 1 tests sequentially +test_foo (test.test_example.TestClass.test_foo) ... unexpected success +----------- +UNEXPECTED SUCCESS: test_foo (test.test_example.TestClass.test_foo) +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(len(result.unexpected_successes), 1) + self.assertEqual(result.unexpected_successes[0].name, "test_foo") + self.assertEqual( + result.unexpected_successes[0].path, "test.test_example.TestClass.test_foo" + ) + + def test_parse_tests_result(self): + result = parse_results(_make_result("== Tests result: FAILURE ==\n")) + self.assertEqual(result.tests_result, "FAILURE") + + def test_parse_crashed_run_no_tests_result(self): + """Test results are still parsed when the runner crashes (no Tests result line).""" + stdout = """\ +Run 1 test sequentially in a single process +0:00:00 [1/1] test_ast +test_foo (test.test_ast.test_ast.TestA.test_foo) ... FAIL +test_bar (test.test_ast.test_ast.TestA.test_bar) ... ok +test_baz (test.test_ast.test_ast.TestB.test_baz) ... ERROR +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(result.tests_result, "") + self.assertEqual(len(result.tests), 2) + names = {t.name for t in result.tests} + self.assertIn("test_foo", names) + self.assertIn("test_baz", names) + + def test_parse_crashed_run_has_unexpected_success(self): + """Unexpected successes are parsed even without Tests result line.""" + stdout = """\ +Run 1 test sequentially in a single process +0:00:00 [1/1] test_ast +test_foo (test.test_ast.test_ast.TestA.test_foo) ... unexpected success +UNEXPECTED SUCCESS: test_foo (test.test_ast.test_ast.TestA.test_foo) +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(result.tests_result, "") + self.assertEqual(len(result.unexpected_successes), 1) + + def test_parse_error_messages(self): + """Single and multiple error messages are parsed from tracebacks.""" + stdout = """\ +Run 2 tests sequentially +test_foo (test.test_example.TestClass.test_foo) ... FAIL +test_bar (test.test_example.TestClass.test_bar) ... ERROR +----------- +====================================================================== +FAIL: test_foo (test.test_example.TestClass.test_foo) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "test.py", line 10, in test_foo + self.assertEqual(1, 2) +AssertionError: 1 != 2 + +====================================================================== +ERROR: test_bar (test.test_example.TestClass.test_bar) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "test.py", line 20, in test_bar + raise ValueError("oops") +ValueError: oops + +====================================================================== +""" + result = parse_results(_make_result(stdout)) + by_name = {t.name: t for t in result.tests} + self.assertEqual(by_name["test_foo"].error_message, "AssertionError: 1 != 2") + self.assertEqual(by_name["test_bar"].error_message, "ValueError: oops") + + def test_parse_directory_test_multiple_submodules(self): + """Failures across submodule boundaries are all detected.""" + stdout = """\ +Run 3 tests sequentially +0:00:00 [ 1/3] test_asyncio.test_buffered_proto +test_ok (test.test_asyncio.test_buffered_proto.TestProto.test_ok) ... ok + +---------------------------------------------------------------------- +Ran 1 tests in 0.1s + +OK + +0:00:01 [ 2/3] test_asyncio.test_events +test_create (test.test_asyncio.test_events.TestEvents.test_create) ... FAIL + +---------------------------------------------------------------------- +Ran 1 tests in 0.2s + +FAILED (failures=1) + +0:00:02 [ 3/3] test_asyncio.test_tasks +test_gather (test.test_asyncio.test_tasks.TestTasks.test_gather) ... ERROR + +---------------------------------------------------------------------- +Ran 1 tests in 0.3s + +FAILED (errors=1) + +== Tests result: FAILURE == +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(len(result.tests), 2) + names = {t.name for t in result.tests} + self.assertIn("test_create", names) + self.assertIn("test_gather", names) + self.assertEqual(result.tests_result, "FAILURE") + + def test_parse_multiline_test_with_docstring(self): + """Two-line output (test_name + docstring ... RESULT) is handled.""" + stdout = """\ +Run 3 tests sequentially +test_ok (test.test_example.TestClass.test_ok) ... ok +test_with_doc (test.test_example.TestClass.test_with_doc) +Test that something works ... ERROR +test_normal_fail (test.test_example.TestClass.test_normal_fail) ... FAIL +""" + result = parse_results(_make_result(stdout)) + self.assertEqual(len(result.tests), 2) + names = {t.name for t in result.tests} + self.assertIn("test_with_doc", names) + self.assertIn("test_normal_fail", names) + test_doc = next(t for t in result.tests if t.name == "test_with_doc") + self.assertEqual(test_doc.path, "test.test_example.TestClass.test_with_doc") + self.assertEqual(test_doc.result, "error") + + +class TestPathToTestParts(unittest.TestCase): + def test_simple_path(self): + self.assertEqual( + path_to_test_parts("test.test_foo.TestClass.test_method"), + ["TestClass", "test_method"], + ) + + def test_nested_path(self): + self.assertEqual( + path_to_test_parts("test.test_foo.test_bar.TestClass.test_method"), + ["TestClass", "test_method"], + ) + + +class TestCollectTestChanges(unittest.TestCase): + def test_collect_failures_and_error_messages(self): + """Failures and error messages are collected; empty messages are omitted.""" + results = TestResult() + results.tests = [ + Test( + name="test_foo", + path="test.test_example.TestClass.test_foo", + result="fail", + error_message="AssertionError: 1 != 2", + ), + Test( + name="test_bar", + path="test.test_example.TestClass.test_bar", + result="error", + error_message="", + ), + ] + failing, successes, error_messages = collect_test_changes(results) + + self.assertEqual( + failing, {("TestClass", "test_foo"), ("TestClass", "test_bar")} + ) + self.assertEqual(successes, set()) + self.assertEqual(len(error_messages), 1) + self.assertEqual( + error_messages[("TestClass", "test_foo")], "AssertionError: 1 != 2" + ) + + def test_collect_unexpected_successes(self): + results = TestResult() + results.unexpected_successes = [ + Test( + name="test_foo", + path="test.test_example.TestClass.test_foo", + result="unexpected_success", + ), + ] + _, successes, _ = collect_test_changes(results) + self.assertEqual(successes, {("TestClass", "test_foo")}) + + def test_module_prefix_filtering(self): + """Prefix filters with both short and 'test.' prefix formats.""" + results = TestResult() + results.tests = [ + Test(name="test_foo", path="test_a.TestClass.test_foo", result="fail"), + Test( + name="test_bar", + path="test.test_dataclasses.TestCase.test_bar", + result="fail", + ), + Test( + name="test_baz", + path="test.test_other.TestOther.test_baz", + result="fail", + ), + ] + failing_a, _, _ = collect_test_changes(results, module_prefix="test_a.") + self.assertEqual(failing_a, {("TestClass", "test_foo")}) + + failing_dc, _, _ = collect_test_changes( + results, module_prefix="test.test_dataclasses." + ) + self.assertEqual(failing_dc, {("TestCase", "test_bar")}) + + def test_collect_init_module_matching(self): + """__init__.py tests match after stripping .__init__ from the prefix.""" + results = TestResult() + results.tests = [ + Test( + name="test_field_repr", + path="test.test_dataclasses.TestCase.test_field_repr", + result="fail", + ), + ] + module_prefix = "test_dataclasses.__init__" + if module_prefix.endswith(".__init__"): + module_prefix = module_prefix[:-9] + module_prefix = "test." + module_prefix + "." + + failing, _, _ = collect_test_changes(results, module_prefix=module_prefix) + self.assertEqual(failing, {("TestCase", "test_field_repr")}) + + +class TestExtractTestMethods(unittest.TestCase): + def test_extract_methods(self): + """Extracts from single and multiple classes.""" + code = """ +class TestA(unittest.TestCase): + def test_a(self): + pass + +class TestB(unittest.TestCase): + def test_b(self): + pass +""" + methods = extract_test_methods(code) + self.assertEqual(methods, {("TestA", "test_a"), ("TestB", "test_b")}) + + def test_extract_syntax_error_returns_empty(self): + self.assertEqual(extract_test_methods("this is not valid python {"), set()) + + +class TestRemoveExpectedFailures(unittest.TestCase): + def test_remove_comment_before(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertIn("def test_one(self):", result) + + def test_remove_inline_comment(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_one(self): + pass +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertNotIn("@unittest.expectedFailure", result) + + def test_remove_super_call_method(self): + """Super-call-only override is removed entirely (sync).""" + code = f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + return super().test_one() +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertNotIn("def test_one", result) + + def test_remove_async_super_call_override(self): + """Super-call-only override is removed entirely (async).""" + code = f"""import unittest + +class BaseTest: + async def test_async_one(self): + pass + +class TestChild(BaseTest, unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + async def test_async_one(self): + return await super().test_async_one() +""" + result = remove_expected_failures(code, {("TestChild", "test_async_one")}) + self.assertNotIn("return await super().test_async_one()", result) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertIn("class TestChild", result) + self.assertIn("async def test_async_one(self):", result) + + def test_remove_with_comment_after(self): + """Reason comment on the line after the decorator is also removed.""" + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + # RuntimeError: something went wrong + def test_one(self): + pass +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertNotIn("RuntimeError: something went wrong", result) + self.assertIn("def test_one(self):", result) + + def test_no_removal_without_comment(self): + """Decorators without our COMMENT marker are left untouched.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure + def test_one(self): + pass +""" + result = remove_expected_failures(code, {("TestFoo", "test_one")}) + self.assertIn("@unittest.expectedFailure", result) + + +class TestStripReasonlessExpectedFailures(unittest.TestCase): + def test_strip_reason_formats(self): + """Strips both inline-comment and comment-before formats when no reason.""" + for label, code in [ + ( + "inline", + f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_one(self): + pass +""", + ), + ( + "comment-before", + f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""", + ), + ]: + with self.subTest(label): + result, stripped = strip_reasonless_expected_failures(code) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertIn("def test_one(self):", result) + self.assertEqual(stripped, {("TestFoo", "test_one")}) + + def test_keep_with_reason(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT}; AssertionError: 1 != 2 + def test_one(self): + pass +""" + result, stripped = strip_reasonless_expected_failures(code) + self.assertIn("@unittest.expectedFailure", result) + self.assertEqual(stripped, set()) + + def test_strip_with_comment_after(self): + """Old-format reason comment on the next line is also removed.""" + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + # RuntimeError: something went wrong + def test_one(self): + pass +""" + result, stripped = strip_reasonless_expected_failures(code) + self.assertNotIn("RuntimeError", result) + self.assertIn("def test_one(self):", result) + self.assertEqual(stripped, {("TestFoo", "test_one")}) + + def test_strip_super_call_override(self): + """Super-call overrides are removed entirely (both comment formats).""" + for label, code in [ + ( + "comment-before", + f"""import unittest + +class _BaseTests: + def test_foo(self): + pass + +class TestChild(_BaseTests, unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_foo(self): + return super().test_foo() +""", + ), + ( + "inline", + f"""import unittest + +class _BaseTests: + def test_foo(self): + pass + +class TestChild(_BaseTests, unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + return super().test_foo() +""", + ), + ]: + with self.subTest(label): + result, stripped = strip_reasonless_expected_failures(code) + self.assertNotIn("return super().test_foo()", result) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertEqual(stripped, {("TestChild", "test_foo")}) + self.assertIn("class _BaseTests:", result) + + def test_no_strip_without_comment(self): + """Markers without our COMMENT are NOT stripped.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure + def test_one(self): + pass +""" + result, stripped = strip_reasonless_expected_failures(code) + self.assertIn("@unittest.expectedFailure", result) + self.assertEqual(stripped, set()) + + def test_mixed_with_and_without_reason(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_no_reason(self): + pass + + @unittest.expectedFailure # {COMMENT}; has a reason + def test_has_reason(self): + pass +""" + result, stripped = strip_reasonless_expected_failures(code) + self.assertEqual(stripped, {("TestFoo", "test_no_reason")}) + self.assertIn("has a reason", result) + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + + +class TestExpandStrippedToChildren(unittest.TestCase): + def test_parent_to_children(self): + """Parent stripped → all/partial failing children returned.""" + stripped = {("Base", "test_foo")} + all_children = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + + # All children fail + result = _expand_stripped_to_children(BASE_TWO_CHILDREN, stripped, all_children) + self.assertEqual(result, all_children) + + # Only one child fails + partial = {("ChildA", "test_foo")} + result = _expand_stripped_to_children(BASE_TWO_CHILDREN, stripped, partial) + self.assertEqual(result, partial) + + def test_direct_match(self): + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + s = {("TestFoo", "test_one")} + self.assertEqual(_expand_stripped_to_children(code, s, s), s) + + def test_child_with_own_override_excluded(self): + stripped = {("Base", "test_foo")} + all_failing = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + result = _expand_stripped_to_children( + BASE_TWO_CHILDREN_ONE_OVERRIDE, stripped, all_failing + ) + # ChildA inherits → included; ChildB has own method → excluded + self.assertEqual(result, {("ChildA", "test_foo")}) + + +class TestApplyTestChanges(unittest.TestCase): + def test_apply_failing_tests(self): + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + result = apply_test_changes(code, {("TestFoo", "test_one")}, set()) + self.assertIn("@unittest.expectedFailure", result) + self.assertIn(COMMENT, result) + + def test_apply_removes_unexpected_success(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""" + result = apply_test_changes(code, set(), {("TestFoo", "test_one")}) + self.assertNotIn("@unittest.expectedFailure", result) + self.assertIn("def test_one(self):", result) + + def test_apply_both_changes(self): + code = f"""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass + + # {COMMENT} + @unittest.expectedFailure + def test_two(self): + pass +""" + result = apply_test_changes( + code, {("TestFoo", "test_one")}, {("TestFoo", "test_two")} + ) + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + + def test_apply_with_error_message(self): + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + result = apply_test_changes( + code, + {("TestFoo", "test_one")}, + set(), + {("TestFoo", "test_one"): "AssertionError: 1 != 2"}, + ) + self.assertIn("AssertionError: 1 != 2", result) + self.assertIn(COMMENT, result) + + +class TestConsolidateToParent(unittest.TestCase): + def test_all_children_fail_marks_parent_with_message(self): + """All subclasses fail → marks parent; error message is transferred.""" + failing = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + error_messages = {("ChildA", "test_foo"): "RuntimeError: boom"} + result = apply_test_changes(BASE_TWO_CHILDREN, failing, set(), error_messages) + + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + self.assertNotIn("return super()", result) + self.assertIn("RuntimeError: boom", result) + + def test_partial_children_fail_marks_children(self): + result = apply_test_changes(BASE_TWO_CHILDREN, {("ChildA", "test_foo")}, set()) + self.assertIn("return super().test_foo()", result) + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + + def test_child_with_own_override_not_consolidated(self): + failing = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + result = apply_test_changes(BASE_TWO_CHILDREN_ONE_OVERRIDE, failing, set()) + self.assertEqual(result.count("@unittest.expectedFailure"), 2) + + def test_strip_then_consolidate_restores_parent_marker(self): + """End-to-end: strip parent marker → child failures → re-mark on parent.""" + code = f"""import unittest + +class _BaseTests: + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + pass + +class ChildA(_BaseTests, unittest.TestCase): + pass + +class ChildB(_BaseTests, unittest.TestCase): + pass +""" + stripped_code, stripped_tests = strip_reasonless_expected_failures(code) + self.assertEqual(stripped_tests, {("_BaseTests", "test_foo")}) + + all_failing = {("ChildA", "test_foo"), ("ChildB", "test_foo")} + error_messages = {("ChildA", "test_foo"): "RuntimeError: boom"} + + to_remark = _expand_stripped_to_children( + stripped_code, stripped_tests, all_failing + ) + self.assertEqual(to_remark, all_failing) + + result = apply_test_changes(stripped_code, to_remark, set(), error_messages) + self.assertIn("RuntimeError: boom", result) + self.assertEqual(result.count("@unittest.expectedFailure"), 1) + self.assertNotIn("return super()", result) + + +class TestSmartAutoMarkFiltering(unittest.TestCase): + """Tests for smart auto-mark filtering (new tests vs regressions).""" + + @staticmethod + def _filter(all_failing, original, current): + new = current - original + to_mark = {t for t in all_failing if t in new} + return to_mark, all_failing - to_mark + + def test_new_vs_regression(self): + """New failures are marked; existing (regression) failures are not.""" + original = {("TestFoo", "test_old1"), ("TestFoo", "test_old2")} + current = original | {("TestFoo", "test_new1"), ("TestFoo", "test_new2")} + all_failing = {("TestFoo", "test_old1"), ("TestFoo", "test_new1")} + + to_mark, regressions = self._filter(all_failing, original, current) + self.assertEqual(to_mark, {("TestFoo", "test_new1")}) + self.assertEqual(regressions, {("TestFoo", "test_old1")}) + + # Edge: all new → all marked + to_mark, regressions = self._filter(all_failing, set(), current) + self.assertEqual(to_mark, all_failing) + self.assertEqual(regressions, set()) + + # Edge: all old → nothing marked + to_mark, regressions = self._filter(all_failing, current, current) + self.assertEqual(to_mark, set()) + self.assertEqual(regressions, all_failing) + + def test_filters_across_classes(self): + original = {("TestA", "test_a"), ("TestB", "test_b")} + current = original | {("TestA", "test_new_a"), ("TestC", "test_c")} + all_failing = { + ("TestA", "test_a"), # regression + ("TestA", "test_new_a"), # new + ("TestC", "test_c"), # new (new class) + } + to_mark, regressions = self._filter(all_failing, original, current) + self.assertEqual(to_mark, {("TestA", "test_new_a"), ("TestC", "test_c")}) + self.assertEqual(regressions, {("TestA", "test_a")}) + + +class TestIsSuperCallOnly(unittest.TestCase): + @staticmethod + def _parse_method(code): + tree = ast.parse(code) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return node + return None + + def test_sync(self): + cases = [ + ("return super().test_one()", True), + ("return super().test_two()", False), # mismatched name + ("pass", False), # regular body + ("x = 1\n return super().test_one()", False), # multiple stmts + ] + for body, expected in cases: + with self.subTest(body=body): + code = f""" +class Foo: + def test_one(self): + {body} +""" + self.assertEqual( + _is_super_call_only(self._parse_method(code)), expected + ) + + def test_async(self): + cases = [ + ("return await super().test_one()", True), + ("return await super().test_two()", False), + ("return super().test_one()", True), # sync call in async method + ] + for body, expected in cases: + with self.subTest(body=body): + code = f""" +class Foo: + async def test_one(self): + {body} +""" + self.assertEqual( + _is_super_call_only(self._parse_method(code)), expected + ) + + +class TestAutoMarkFileWithCrashedRun(unittest.TestCase): + """auto_mark_file should process partial results when test runner crashes.""" + + CRASHED_STDOUT = """\ +Run 1 test sequentially in a single process +0:00:00 [1/1] test_example +test_foo (test.test_example.TestA.test_foo) ... FAIL +test_bar (test.test_example.TestA.test_bar) ... ok +====================================================================== +FAIL: test_foo (test.test_example.TestA.test_foo) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "test.py", line 10, in test_foo + self.assertEqual(1, 2) +AssertionError: 1 != 2 +""" + + def test_auto_mark_file_crashed_run(self): + """auto_mark_file processes results even when tests_result is empty (crash).""" + test_code = f"""import unittest + +class TestA(unittest.TestCase): + def test_foo(self): + pass + + def test_bar(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test_example.py" + test_file.write_text(test_code) + + mock_result = TestResult() + mock_result.tests_result = "" + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.TestA.test_foo", + result="fail", + error_message="AssertionError: 1 != 2", + ), + ] + + with mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ): + added, removed, regressions = auto_mark_file( + test_file, mark_failure=True, verbose=False + ) + + self.assertEqual(added, 1) + contents = test_file.read_text() + self.assertIn("expectedFailure", contents) + + def test_auto_mark_file_no_results_at_all_raises(self): + """auto_mark_file raises TestRunError when there are zero parsed results.""" + test_code = """import unittest + +class TestA(unittest.TestCase): + def test_foo(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test_example.py" + test_file.write_text(test_code) + + mock_result = TestResult() + mock_result.tests_result = "" + mock_result.tests = [] + mock_result.stdout = "some crash output" + + with mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ): + with self.assertRaises(TestRunError): + auto_mark_file(test_file, verbose=False) + + +class TestAutoMarkDirectoryWithCrashedRun(unittest.TestCase): + """auto_mark_directory should process partial results when test runner crashes.""" + + def test_auto_mark_directory_crashed_run(self): + """auto_mark_directory processes results even when tests_result is empty.""" + test_code = f"""import unittest + +class TestA(unittest.TestCase): + def test_foo(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) / "test_example" + test_dir.mkdir() + test_file = test_dir / "test_sub.py" + test_file.write_text(test_code) + + mock_result = TestResult() + mock_result.tests_result = "" + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.test_sub.TestA.test_foo", + result="fail", + error_message="AssertionError: oops", + ), + ] + + with ( + mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ), + mock.patch( + "update_lib.cmd_auto_mark.get_test_module_name", + side_effect=lambda p: ( + "test_example" if p == test_dir else "test_example.test_sub" + ), + ), + ): + added, removed, regressions = auto_mark_directory( + test_dir, mark_failure=True, verbose=False + ) + + self.assertEqual(added, 1) + contents = test_file.read_text() + self.assertIn("expectedFailure", contents) + + def test_auto_mark_directory_no_results_raises(self): + """auto_mark_directory raises TestRunError when zero results.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) / "test_example" + test_dir.mkdir() + test_file = test_dir / "test_sub.py" + test_file.write_text("import unittest\n") + + mock_result = TestResult() + mock_result.tests_result = "" + mock_result.tests = [] + mock_result.stdout = "crash" + + with ( + mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ), + mock.patch( + "update_lib.cmd_auto_mark.get_test_module_name", + return_value="test_example", + ), + ): + with self.assertRaises(TestRunError): + auto_mark_directory(test_dir, verbose=False) + + +class TestAutoMarkFileRestoresOnCrash(unittest.TestCase): + """Stripped markers must be restored when the test runner crashes.""" + + def test_stripped_markers_restored_when_crash(self): + """Markers stripped before run must be restored for unobserved tests on crash.""" + test_code = f"""\ +import unittest + +class TestA(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + pass + + @unittest.expectedFailure # {COMMENT} + def test_bar(self): + pass + + @unittest.expectedFailure # {COMMENT} + def test_baz(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test_example.py" + test_file.write_text(test_code) + + # Simulate a crashed run that only observed test_foo (failed) + # test_bar and test_baz never ran due to crash + mock_result = TestResult() + mock_result.tests_result = "" # no Tests result line (crash) + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.TestA.test_foo", + result="fail", + error_message="AssertionError: 1 != 2", + ), + ] + + with mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ): + auto_mark_file(test_file, verbose=False) + + contents = test_file.read_text() + # test_bar and test_baz were not observed — their markers must be restored + self.assertIn("def test_bar", contents) + self.assertIn("def test_baz", contents) + # Count expectedFailure markers: all 3 should be present + self.assertEqual(contents.count("expectedFailure"), 3, contents) + + def test_stripped_markers_removed_when_complete_run(self): + """Markers are properly removed when the run completes normally.""" + test_code = f"""\ +import unittest + +class TestA(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + pass + + @unittest.expectedFailure # {COMMENT} + def test_bar(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test_example.py" + test_file.write_text(test_code) + + # Simulate a complete run where test_foo fails but test_bar passes + mock_result = TestResult() + mock_result.tests_result = "FAILURE" # normal completion + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.TestA.test_foo", + result="fail", + error_message="AssertionError", + ), + ] + # test_bar passes → shows as unexpected success + mock_result.unexpected_successes = [ + Test( + name="test_bar", + path="test.test_example.TestA.test_bar", + result="unexpected success", + ), + ] + + with mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ): + auto_mark_file(test_file, verbose=False) + + contents = test_file.read_text() + # test_foo should still have marker (re-added) + self.assertEqual(contents.count("expectedFailure"), 1, contents) + self.assertIn("def test_foo", contents) + + +class TestAutoMarkDirectoryRestoresOnCrash(unittest.TestCase): + """Stripped markers must be restored for directory runs that crash.""" + + def test_stripped_markers_restored_when_crash(self): + test_code = f"""\ +import unittest + +class TestA(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_foo(self): + pass + + @unittest.expectedFailure # {COMMENT} + def test_bar(self): + pass +""" + with tempfile.TemporaryDirectory() as tmpdir: + test_dir = pathlib.Path(tmpdir) / "test_example" + test_dir.mkdir() + test_file = test_dir / "test_sub.py" + test_file.write_text(test_code) + + mock_result = TestResult() + mock_result.tests_result = "" # crash + mock_result.tests = [ + Test( + name="test_foo", + path="test.test_example.test_sub.TestA.test_foo", + result="fail", + ), + ] + + with ( + mock.patch( + "update_lib.cmd_auto_mark.run_test", return_value=mock_result + ), + mock.patch( + "update_lib.cmd_auto_mark.get_test_module_name", + side_effect=lambda p: ( + "test_example" if p == test_dir else "test_example.test_sub" + ), + ), + ): + auto_mark_directory(test_dir, verbose=False) + + contents = test_file.read_text() + # Both markers must be present (unobserved test_bar restored) + self.assertEqual(contents.count("expectedFailure"), 2, contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_copy_lib.py b/scripts/update_lib/tests/test_copy_lib.py new file mode 100644 index 00000000000..aca00cb18f3 --- /dev/null +++ b/scripts/update_lib/tests/test_copy_lib.py @@ -0,0 +1,75 @@ +"""Tests for copy_lib.py - library copying with dependencies.""" + +import pathlib +import tempfile +import unittest + + +class TestCopySingle(unittest.TestCase): + """Tests for _copy_single helper function.""" + + def test_copies_file(self): + """Test copying a single file.""" + from update_lib.cmd_copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source.py" + src.write_text("content") + dst = tmpdir / "dest.py" + + _copy_single(src, dst, verbose=False) + + self.assertTrue(dst.exists()) + self.assertEqual(dst.read_text(), "content") + + def test_copies_directory(self): + """Test copying a directory.""" + from update_lib.cmd_copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source_dir" + src.mkdir() + (src / "file.py").write_text("content") + dst = tmpdir / "dest_dir" + + _copy_single(src, dst, verbose=False) + + self.assertTrue(dst.exists()) + self.assertTrue((dst / "file.py").exists()) + + def test_removes_existing_before_copy(self): + """Test that existing destination is removed before copy.""" + from update_lib.cmd_copy_lib import _copy_single + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + src = tmpdir / "source.py" + src.write_text("new content") + dst = tmpdir / "dest.py" + dst.write_text("old content") + + _copy_single(src, dst, verbose=False) + + self.assertEqual(dst.read_text(), "new content") + + +class TestCopyLib(unittest.TestCase): + """Tests for copy_lib function.""" + + def test_raises_on_path_without_lib(self): + """Test that copy_lib raises ValueError when path doesn't contain /Lib/.""" + from update_lib.cmd_copy_lib import copy_lib + + with self.assertRaises(ValueError) as ctx: + copy_lib(pathlib.Path("some/path/without/lib.py")) + + self.assertIn("/Lib/", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py new file mode 100644 index 00000000000..d97af2867aa --- /dev/null +++ b/scripts/update_lib/tests/test_deps.py @@ -0,0 +1,394 @@ +"""Tests for deps.py - dependency resolution.""" + +import pathlib +import tempfile +import unittest + +from update_lib.deps import ( + get_lib_paths, + get_soft_deps, + get_test_dependencies, + get_test_paths, + parse_lib_imports, + parse_test_imports, +) + + +class TestParseTestImports(unittest.TestCase): + """Tests for parse_test_imports function.""" + + def test_from_test_import(self): + """Test parsing 'from test import foo'.""" + code = """ +from test import string_tests +from test import lock_tests, other_tests +""" + imports = parse_test_imports(code) + self.assertEqual(imports, {"string_tests", "lock_tests", "other_tests"}) + + def test_from_test_dot_module(self): + """Test parsing 'from test.foo import bar'.""" + code = """ +from test.string_tests import CommonTest +from test.support import verbose +""" + imports = parse_test_imports(code) + self.assertEqual(imports, {"string_tests"}) # support is excluded + + def test_excludes_support(self): + """Test that 'support' is excluded.""" + code = """ +from test import support +from test.support import verbose +""" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + def test_regular_imports_ignored(self): + """Test that regular imports are ignored.""" + code = """ +import os +from collections import defaultdict +from . import helper +""" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + def test_syntax_error_returns_empty(self): + """Test that syntax errors return empty set.""" + code = "this is not valid python {" + imports = parse_test_imports(code) + self.assertEqual(imports, set()) + + +class TestGetLibPaths(unittest.TestCase): + """Tests for get_lib_paths function.""" + + def test_auto_detect_py_module(self): + """Test auto-detection of _py{module}.py pattern.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "mymodule.py").write_text("# mymodule") + (lib_dir / "_pymymodule.py").write_text("# _pymymodule") + + paths = get_lib_paths("mymodule", str(tmpdir)) + self.assertEqual(len(paths), 2) + self.assertIn(tmpdir / "Lib" / "mymodule.py", paths) + self.assertIn(tmpdir / "Lib" / "_pymymodule.py", paths) + + def test_default_file(self): + """Test default to .py file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# foo") + + paths = get_lib_paths("foo", str(tmpdir)) + self.assertEqual(paths, (tmpdir / "Lib" / "foo.py",)) + + def test_default_directory(self): + """Test default to directory when file doesn't exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo").mkdir() + + paths = get_lib_paths("foo", str(tmpdir)) + self.assertEqual(paths, (tmpdir / "Lib" / "foo",)) + + +class TestGetTestPaths(unittest.TestCase): + """Tests for get_test_paths function.""" + + def test_known_dependency(self): + """Test test with known path override.""" + paths = get_test_paths("regrtest", "cpython") + self.assertEqual(len(paths), 1) + self.assertEqual(paths[0], pathlib.Path("cpython/Lib/test/test_regrtest")) + + def test_default_directory(self): + """Test default to test_name/ directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "Lib" / "test" + test_dir.mkdir(parents=True) + (test_dir / "test_foo").mkdir() + + paths = get_test_paths("foo", str(tmpdir)) + self.assertEqual(paths, (tmpdir / "Lib" / "test" / "test_foo",)) + + def test_default_file(self): + """Test fallback to test_name.py file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "Lib" / "test" + test_dir.mkdir(parents=True) + (test_dir / "test_foo.py").write_text("# test") + + paths = get_test_paths("foo", str(tmpdir)) + self.assertEqual(paths, (tmpdir / "Lib" / "test" / "test_foo.py",)) + + +class TestGetTestDependencies(unittest.TestCase): + """Tests for get_test_dependencies function.""" + + def test_parse_file_imports(self): + """Test parsing imports from test file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "test" + test_dir.mkdir() + + # Create test file with import + test_file = test_dir / "test_foo.py" + test_file.write_text(""" +from test import string_tests + +class TestFoo: + pass +""") + # Create the dependency file + (test_dir / "string_tests.py").write_text("# string tests") + + result = get_test_dependencies(test_file) + self.assertEqual(len(result["hard_deps"]), 1) + self.assertEqual(result["hard_deps"][0], test_dir / "string_tests.py") + self.assertEqual(result["data"], []) + + def test_nonexistent_path(self): + """Test nonexistent path returns empty.""" + result = get_test_dependencies(pathlib.Path("/nonexistent/path")) + self.assertEqual(result, {"hard_deps": [], "data": []}) + + def test_transitive_data_dependency(self): + """Test that data deps are resolved transitively. + + Chain: test_codecencodings_kr -> multibytecodec_support -> cjkencodings + """ + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_dir = tmpdir / "test" + test_dir.mkdir() + + # Create test_codecencodings_kr.py that imports multibytecodec_support + test_file = test_dir / "test_codecencodings_kr.py" + test_file.write_text(""" +from test import multibytecodec_support + +class TestKR: + pass +""") + # Create multibytecodec_support.py (the intermediate dependency) + (test_dir / "multibytecodec_support.py").write_text("# support module") + + # Create cjkencodings directory (the data dependency of multibytecodec_support) + (test_dir / "cjkencodings").mkdir() + + result = get_test_dependencies(test_file) + + # Should find multibytecodec_support.py as a hard_dep + self.assertEqual(len(result["hard_deps"]), 1) + self.assertEqual( + result["hard_deps"][0], test_dir / "multibytecodec_support.py" + ) + + # Should find cjkencodings as data (from multibytecodec_support's TEST_DEPENDENCIES) + self.assertEqual(len(result["data"]), 1) + self.assertEqual(result["data"][0], test_dir / "cjkencodings") + + +class TestParseLibImports(unittest.TestCase): + """Tests for parse_lib_imports function.""" + + def test_import_statement(self): + """Test parsing 'import foo'.""" + code = """ +import os +import sys +import collections.abc +""" + imports = parse_lib_imports(code) + self.assertEqual(imports, {"os", "sys", "collections.abc"}) + + def test_from_import(self): + """Test parsing 'from foo import bar'.""" + code = """ +from os import path +from collections.abc import Mapping +from typing import Optional +""" + imports = parse_lib_imports(code) + self.assertEqual(imports, {"os", "collections.abc", "typing"}) + + def test_mixed_imports(self): + """Test mixed import styles.""" + code = """ +import sys +from os import path +from collections import defaultdict +import functools +""" + imports = parse_lib_imports(code) + self.assertEqual(imports, {"sys", "os", "collections", "functools"}) + + def test_syntax_error_returns_empty(self): + """Test that syntax errors return empty set.""" + code = "this is not valid python {" + imports = parse_lib_imports(code) + self.assertEqual(imports, set()) + + def test_relative_import_skipped(self): + """Test that relative imports (no module) are skipped.""" + code = """ +from . import foo +from .. import bar +""" + imports = parse_lib_imports(code) + self.assertEqual(imports, set()) + + +class TestGetSoftDeps(unittest.TestCase): + """Tests for get_soft_deps function.""" + + def test_with_temp_files(self): + """Test soft deps detection with temp files.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + # Create a module that imports another module + (lib_dir / "foo.py").write_text(""" +import bar +from baz import something +""") + # Create the imported modules + (lib_dir / "bar.py").write_text("# bar module") + (lib_dir / "baz.py").write_text("# baz module") + + soft_deps = get_soft_deps("foo", str(tmpdir)) + self.assertEqual(soft_deps, {"bar", "baz"}) + + def test_skips_self(self): + """Test that module doesn't include itself in soft_deps.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + # Create a module that imports itself (circular) + (lib_dir / "foo.py").write_text(""" +import foo +import bar +""") + (lib_dir / "bar.py").write_text("# bar module") + + soft_deps = get_soft_deps("foo", str(tmpdir)) + self.assertNotIn("foo", soft_deps) + self.assertIn("bar", soft_deps) + + def test_filters_nonexistent(self): + """Test that nonexistent modules are filtered out.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + + # Create a module that imports nonexistent module + (lib_dir / "foo.py").write_text(""" +import bar +import nonexistent +""") + (lib_dir / "bar.py").write_text("# bar module") + # nonexistent.py is NOT created + + soft_deps = get_soft_deps("foo", str(tmpdir)) + self.assertEqual(soft_deps, {"bar"}) + + +class TestDircmpIsSame(unittest.TestCase): + """Tests for _dircmp_is_same function.""" + + def test_identical_directories(self): + """Test that identical directories return True.""" + import filecmp + + from update_lib.deps import _dircmp_is_same + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + dir1 = tmpdir / "dir1" + dir2 = tmpdir / "dir2" + dir1.mkdir() + dir2.mkdir() + + (dir1 / "file.py").write_text("content") + (dir2 / "file.py").write_text("content") + + dcmp = filecmp.dircmp(dir1, dir2) + self.assertTrue(_dircmp_is_same(dcmp)) + + def test_different_files(self): + """Test that directories with different files return False.""" + import filecmp + + from update_lib.deps import _dircmp_is_same + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + dir1 = tmpdir / "dir1" + dir2 = tmpdir / "dir2" + dir1.mkdir() + dir2.mkdir() + + (dir1 / "file.py").write_text("content1") + (dir2 / "file.py").write_text("content2") + + dcmp = filecmp.dircmp(dir1, dir2) + self.assertFalse(_dircmp_is_same(dcmp)) + + def test_nested_identical(self): + """Test that nested identical directories return True.""" + import filecmp + + from update_lib.deps import _dircmp_is_same + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + dir1 = tmpdir / "dir1" + dir2 = tmpdir / "dir2" + (dir1 / "sub").mkdir(parents=True) + (dir2 / "sub").mkdir(parents=True) + + (dir1 / "sub" / "file.py").write_text("content") + (dir2 / "sub" / "file.py").write_text("content") + + dcmp = filecmp.dircmp(dir1, dir2) + self.assertTrue(_dircmp_is_same(dcmp)) + + def test_nested_different(self): + """Test that nested directories with differences return False.""" + import filecmp + + from update_lib.deps import _dircmp_is_same + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + dir1 = tmpdir / "dir1" + dir2 = tmpdir / "dir2" + (dir1 / "sub").mkdir(parents=True) + (dir2 / "sub").mkdir(parents=True) + + (dir1 / "sub" / "file.py").write_text("content1") + (dir2 / "sub" / "file.py").write_text("content2") + + dcmp = filecmp.dircmp(dir1, dir2) + self.assertFalse(_dircmp_is_same(dcmp)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_migrate.py b/scripts/update_lib/tests/test_migrate.py new file mode 100644 index 00000000000..0cc247ba841 --- /dev/null +++ b/scripts/update_lib/tests/test_migrate.py @@ -0,0 +1,196 @@ +"""Tests for migrate.py - file migration operations.""" + +import pathlib +import tempfile +import unittest + +from update_lib.cmd_migrate import ( + patch_directory, + patch_file, + patch_single_content, +) +from update_lib.patch_spec import COMMENT + + +class TestPatchSingleContent(unittest.TestCase): + """Tests for patch_single_content function.""" + + def test_patch_with_no_existing_file(self): + """Test patching when lib file doesn't exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source file + src_path = tmpdir / "src.py" + src_path.write_text("""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""") + + # Non-existent lib path + lib_path = tmpdir / "lib.py" + + result = patch_single_content(src_path, lib_path) + + # Should return source content unchanged + self.assertIn("def test_one(self):", result) + self.assertNotIn(COMMENT, result) + + def test_patch_with_existing_patches(self): + """Test patching preserves existing patches.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source file (new version) + src_path = tmpdir / "src.py" + src_path.write_text("""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass + + def test_two(self): + pass +""") + + # Create lib file with existing patch + lib_path = tmpdir / "lib.py" + lib_path.write_text(f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""") + + result = patch_single_content(src_path, lib_path) + + # Should have patch on test_one + self.assertIn("@unittest.expectedFailure", result) + self.assertIn(COMMENT, result) + # Should have test_two from source + self.assertIn("def test_two(self):", result) + + +class TestPatchFile(unittest.TestCase): + """Tests for patch_file function.""" + + def test_patch_file_creates_output(self): + """Test that patch_file writes output file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source file + src_path = tmpdir / "src.py" + src_path.write_text("""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""") + + # Output path + lib_path = tmpdir / "Lib" / "test.py" + + patch_file(src_path, lib_path, verbose=False) + + # File should exist + self.assertTrue(lib_path.exists()) + content = lib_path.read_text() + self.assertIn("def test_one(self):", content) + + def test_patch_file_preserves_patches(self): + """Test that patch_file preserves existing patches.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source file + src_path = tmpdir / "src.py" + src_path.write_text("""import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""") + + # Create existing lib file with patch + lib_path = tmpdir / "lib.py" + lib_path.write_text(f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""") + + patch_file(src_path, lib_path, verbose=False) + + content = lib_path.read_text() + self.assertIn("@unittest.expectedFailure", content) + + +class TestPatchDirectory(unittest.TestCase): + """Tests for patch_directory function.""" + + def test_patch_directory_all_files(self): + """Test that patch_directory processes all .py files.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source directory with files + src_dir = tmpdir / "src" + src_dir.mkdir() + (src_dir / "test_a.py").write_text("# test_a") + (src_dir / "test_b.py").write_text("# test_b") + (src_dir / "subdir").mkdir() + (src_dir / "subdir" / "test_c.py").write_text("# test_c") + + # Output directory + lib_dir = tmpdir / "lib" + + patch_directory(src_dir, lib_dir, verbose=False) + + # All files should exist + self.assertTrue((lib_dir / "test_a.py").exists()) + self.assertTrue((lib_dir / "test_b.py").exists()) + self.assertTrue((lib_dir / "subdir" / "test_c.py").exists()) + + def test_patch_directory_preserves_patches(self): + """Test that patch_directory preserves patches in existing files.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + + # Create source directory + src_dir = tmpdir / "src" + src_dir.mkdir() + (src_dir / "test_a.py").write_text("""import unittest + +class TestA(unittest.TestCase): + def test_one(self): + pass +""") + + # Create lib directory with patched file + lib_dir = tmpdir / "lib" + lib_dir.mkdir() + (lib_dir / "test_a.py").write_text(f"""import unittest + +class TestA(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""") + + patch_directory(src_dir, lib_dir, verbose=False) + + content = (lib_dir / "test_a.py").read_text() + self.assertIn("@unittest.expectedFailure", content) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_patch_spec.py b/scripts/update_lib/tests/test_patch_spec.py new file mode 100644 index 00000000000..798bd851b3c --- /dev/null +++ b/scripts/update_lib/tests/test_patch_spec.py @@ -0,0 +1,362 @@ +"""Tests for patch_spec.py - core patch extraction and application.""" + +import ast +import unittest + +from update_lib.patch_spec import ( + COMMENT, + PatchSpec, + UtMethod, + _find_import_insert_line, + apply_patches, + extract_patches, + iter_tests, +) + + +class TestIterTests(unittest.TestCase): + """Tests for iter_tests function.""" + + def test_iter_tests_simple(self): + """Test iterating over test methods in a class.""" + code = """ +class TestFoo(unittest.TestCase): + def test_one(self): + pass + + def test_two(self): + pass +""" + tree = ast.parse(code) + results = list(iter_tests(tree)) + self.assertEqual(len(results), 2) + self.assertEqual(results[0][0].name, "TestFoo") + self.assertEqual(results[0][1].name, "test_one") + self.assertEqual(results[1][1].name, "test_two") + + def test_iter_tests_multiple_classes(self): + """Test iterating over multiple test classes.""" + code = """ +class TestFoo(unittest.TestCase): + def test_foo(self): + pass + +class TestBar(unittest.TestCase): + def test_bar(self): + pass +""" + tree = ast.parse(code) + results = list(iter_tests(tree)) + self.assertEqual(len(results), 2) + self.assertEqual(results[0][0].name, "TestFoo") + self.assertEqual(results[1][0].name, "TestBar") + + def test_iter_tests_async(self): + """Test iterating over async test methods.""" + code = """ +class TestAsync(unittest.TestCase): + async def test_async(self): + pass +""" + tree = ast.parse(code) + results = list(iter_tests(tree)) + self.assertEqual(len(results), 1) + self.assertEqual(results[0][1].name, "test_async") + + +class TestExtractPatches(unittest.TestCase): + """Tests for extract_patches function.""" + + def test_extract_expected_failure(self): + """Test extracting @unittest.expectedFailure decorator.""" + code = f""" +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""" + patches = extract_patches(code) + self.assertIn("TestFoo", patches) + self.assertIn("test_one", patches["TestFoo"]) + specs = patches["TestFoo"]["test_one"] + self.assertEqual(len(specs), 1) + self.assertEqual(specs[0].ut_method, UtMethod.ExpectedFailure) + + def test_extract_expected_failure_inline_comment(self): + """Test extracting expectedFailure with inline comment.""" + code = f""" +class TestFoo(unittest.TestCase): + @unittest.expectedFailure # {COMMENT} + def test_one(self): + pass +""" + patches = extract_patches(code) + self.assertIn("TestFoo", patches) + self.assertIn("test_one", patches["TestFoo"]) + + def test_extract_skip_with_reason(self): + """Test extracting @unittest.skip with reason.""" + code = f''' +class TestFoo(unittest.TestCase): + @unittest.skip("{COMMENT}; not implemented") + def test_one(self): + pass +''' + patches = extract_patches(code) + self.assertIn("TestFoo", patches) + specs = patches["TestFoo"]["test_one"] + self.assertEqual(specs[0].ut_method, UtMethod.Skip) + self.assertIn("not implemented", specs[0].reason) + + def test_extract_skip_if(self): + """Test extracting @unittest.skipIf decorator.""" + code = f''' +class TestFoo(unittest.TestCase): + @unittest.skipIf(sys.platform == "win32", "{COMMENT}; windows issue") + def test_one(self): + pass +''' + patches = extract_patches(code) + specs = patches["TestFoo"]["test_one"] + self.assertEqual(specs[0].ut_method, UtMethod.SkipIf) + # ast.unparse normalizes quotes to single quotes + self.assertIn("sys.platform", specs[0].cond) + self.assertIn("win32", specs[0].cond) + + def test_no_patches_without_comment(self): + """Test that decorators without COMMENT are not extracted.""" + code = """ +class TestFoo(unittest.TestCase): + @unittest.expectedFailure + def test_one(self): + pass +""" + patches = extract_patches(code) + self.assertEqual(patches, {}) + + def test_multiple_patches_same_method(self): + """Test extracting multiple decorators on same method.""" + code = f''' +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + @unittest.skip("{COMMENT}; reason") + def test_one(self): + pass +''' + patches = extract_patches(code) + specs = patches["TestFoo"]["test_one"] + self.assertEqual(len(specs), 2) + + +class TestApplyPatches(unittest.TestCase): + """Tests for apply_patches function.""" + + def test_apply_expected_failure(self): + """Test applying @unittest.expectedFailure.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.ExpectedFailure, None, "")]} + } + result = apply_patches(code, patches) + self.assertIn("@unittest.expectedFailure", result) + self.assertIn(COMMENT, result) + + def test_apply_skip_with_reason(self): + """Test applying @unittest.skip with reason.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.Skip, None, "not ready")]} + } + result = apply_patches(code, patches) + self.assertIn("@unittest.skip", result) + self.assertIn("not ready", result) + + def test_apply_skip_if(self): + """Test applying @unittest.skipIf.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + patches = { + "TestFoo": { + "test_one": [ + PatchSpec(UtMethod.SkipIf, "sys.platform == 'win32'", "windows") + ] + } + } + result = apply_patches(code, patches) + self.assertIn("@unittest.skipIf", result) + self.assertIn('sys.platform == "win32"', result) + + def test_apply_preserves_existing_decorators(self): + """Test that existing decorators are preserved.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + @some_decorator + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.ExpectedFailure, None, "")]} + } + result = apply_patches(code, patches) + self.assertIn("@some_decorator", result) + self.assertIn("@unittest.expectedFailure", result) + + def test_apply_inherited_method(self): + """Test applying patch to inherited method (creates override).""" + code = """import unittest + +class TestFoo(unittest.TestCase): + pass +""" + patches = { + "TestFoo": { + "test_inherited": [PatchSpec(UtMethod.ExpectedFailure, None, "")] + } + } + result = apply_patches(code, patches) + self.assertIn("def test_inherited(self):", result) + self.assertIn("return super().test_inherited()", result) + + def test_apply_adds_unittest_import(self): + """Test that unittest import is added if missing.""" + code = """import sys + +class TestFoo: + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.ExpectedFailure, None, "")]} + } + result = apply_patches(code, patches) + # Should add unittest import after existing imports + self.assertIn("import unittest", result) + + def test_apply_no_duplicate_import(self): + """Test that unittest import is not duplicated.""" + code = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + patches = { + "TestFoo": {"test_one": [PatchSpec(UtMethod.ExpectedFailure, None, "")]} + } + result = apply_patches(code, patches) + # Count occurrences of 'import unittest' + count = result.count("import unittest") + self.assertEqual(count, 1) + + +class TestPatchSpec(unittest.TestCase): + """Tests for PatchSpec class.""" + + def test_as_decorator_expected_failure(self): + """Test generating expectedFailure decorator string.""" + spec = PatchSpec(UtMethod.ExpectedFailure, None, "reason") + decorator = spec.as_decorator() + self.assertIn("@unittest.expectedFailure", decorator) + self.assertIn(COMMENT, decorator) + self.assertIn("reason", decorator) + + def test_as_decorator_skip(self): + """Test generating skip decorator string.""" + spec = PatchSpec(UtMethod.Skip, None, "not ready") + decorator = spec.as_decorator() + self.assertIn("@unittest.skip", decorator) + self.assertIn("not ready", decorator) + + def test_as_decorator_skip_if(self): + """Test generating skipIf decorator string.""" + spec = PatchSpec(UtMethod.SkipIf, "condition", "reason") + decorator = spec.as_decorator() + self.assertIn("@unittest.skipIf", decorator) + self.assertIn("condition", decorator) + + +class TestRoundTrip(unittest.TestCase): + """Tests for extract -> apply round trip.""" + + def test_round_trip_expected_failure(self): + """Test that extracted patches can be re-applied.""" + original = f"""import unittest + +class TestFoo(unittest.TestCase): + # {COMMENT} + @unittest.expectedFailure + def test_one(self): + pass +""" + # Extract patches + patches = extract_patches(original) + + # Apply to clean code + clean = """import unittest + +class TestFoo(unittest.TestCase): + def test_one(self): + pass +""" + result = apply_patches(clean, patches) + + # Should have the decorator + self.assertIn("@unittest.expectedFailure", result) + self.assertIn(COMMENT, result) + + +class TestFindImportInsertLine(unittest.TestCase): + """Tests for _find_import_insert_line function.""" + + def test_with_imports(self): + """Test finding line after imports.""" + code = """import os +import sys + +class Foo: + pass +""" + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 2) + + def test_no_imports_with_docstring(self): + """Test fallback to after docstring when no imports.""" + code = '''"""Module docstring.""" + +class Foo: + pass +''' + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 1) + + def test_no_imports_no_docstring(self): + """Test fallback to line 0 when no imports and no docstring.""" + code = """class Foo: + pass +""" + tree = ast.parse(code) + line = _find_import_insert_line(tree) + self.assertEqual(line, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_path.py b/scripts/update_lib/tests/test_path.py new file mode 100644 index 00000000000..f2dcdcf8f05 --- /dev/null +++ b/scripts/update_lib/tests/test_path.py @@ -0,0 +1,224 @@ +"""Tests for path.py - path utilities.""" + +import pathlib +import tempfile +import unittest + +from update_lib.file_utils import ( + get_test_files, + get_test_module_name, + is_lib_path, + is_test_path, + lib_to_test_path, + parse_lib_path, +) + + +class TestParseLibPath(unittest.TestCase): + """Tests for parse_lib_path function.""" + + def test_parse_cpython_path(self): + """Test parsing cpython/Lib/... path.""" + result = parse_lib_path("cpython/Lib/test/test_foo.py") + self.assertEqual(result, pathlib.Path("Lib/test/test_foo.py")) + + def test_parse_nested_path(self): + """Test parsing deeply nested path.""" + result = parse_lib_path("/home/user/cpython/Lib/test/test_foo/test_bar.py") + self.assertEqual(result, pathlib.Path("Lib/test/test_foo/test_bar.py")) + + def test_parse_windows_path(self): + """Test parsing Windows-style path.""" + result = parse_lib_path("C:\\cpython\\Lib\\test\\test_foo.py") + self.assertEqual(result, pathlib.Path("Lib/test/test_foo.py")) + + def test_parse_directory(self): + """Test parsing directory path.""" + result = parse_lib_path("cpython/Lib/test/test_json/") + self.assertEqual(result, pathlib.Path("Lib/test/test_json/")) + + def test_parse_no_lib_raises(self): + """Test that path without /Lib/ raises ValueError.""" + with self.assertRaises(ValueError) as ctx: + parse_lib_path("some/random/path.py") + self.assertIn("/Lib/", str(ctx.exception)) + + +class TestIsLibPath(unittest.TestCase): + """Tests for is_lib_path function.""" + + def test_lib_path(self): + """Test detecting Lib/ path.""" + self.assertTrue(is_lib_path(pathlib.Path("Lib/test/test_foo.py"))) + self.assertTrue(is_lib_path(pathlib.Path("./Lib/test/test_foo.py"))) + + def test_cpython_path_not_lib(self): + """Test that cpython/Lib/ is not detected as lib path.""" + self.assertFalse(is_lib_path(pathlib.Path("cpython/Lib/test/test_foo.py"))) + + def test_random_path_not_lib(self): + """Test that random path is not lib path.""" + self.assertFalse(is_lib_path(pathlib.Path("some/other/path.py"))) + + +class TestIsTestPath(unittest.TestCase): + """Tests for is_test_path function.""" + + def test_cpython_test_path(self): + """Test detecting cpython test path.""" + self.assertTrue(is_test_path(pathlib.Path("cpython/Lib/test/test_foo.py"))) + + def test_lib_test_path(self): + """Test detecting Lib/test path.""" + self.assertTrue(is_test_path(pathlib.Path("Lib/test/test_foo.py"))) + + def test_library_path_not_test(self): + """Test that library path (not test) is not test path.""" + self.assertFalse(is_test_path(pathlib.Path("cpython/Lib/dataclasses.py"))) + self.assertFalse(is_test_path(pathlib.Path("Lib/dataclasses.py"))) + + +class TestLibToTestPath(unittest.TestCase): + """Tests for lib_to_test_path function.""" + + def test_prefers_directory_over_file(self): + """Test that directory is preferred when both exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # Create structure: tmpdir/Lib/foo.py, tmpdir/Lib/test/test_foo/, tmpdir/Lib/test/test_foo.py + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# lib") + test_dir = lib_dir / "test" + test_dir.mkdir() + (test_dir / "test_foo").mkdir() + (test_dir / "test_foo.py").write_text("# test file") + + result = lib_to_test_path(tmpdir / "Lib" / "foo.py") + # Should prefer directory + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_foo/") + + def test_falls_back_to_file(self): + """Test that file is used when directory doesn't exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # Create structure: tmpdir/Lib/foo.py, tmpdir/Lib/test/test_foo.py (no directory) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# lib") + test_dir = lib_dir / "test" + test_dir.mkdir() + (test_dir / "test_foo.py").write_text("# test file") + + result = lib_to_test_path(tmpdir / "Lib" / "foo.py") + # Should fall back to file + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_foo.py") + + def test_defaults_to_directory_when_neither_exists(self): + """Test that directory path is returned when neither exists.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + (lib_dir / "foo.py").write_text("# lib") + test_dir = lib_dir / "test" + test_dir.mkdir() + # Neither test_foo/ nor test_foo.py exists + + result = lib_to_test_path(tmpdir / "Lib" / "foo.py") + # Should default to directory + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_foo/") + + def test_lib_path_prefers_directory(self): + """Test Lib/ path prefers directory when it exists.""" + # This test uses actual Lib/ paths, checking current behavior + # When neither exists, defaults to directory + result = lib_to_test_path(pathlib.Path("Lib/nonexistent_module.py")) + self.assertEqual(result, pathlib.Path("Lib/test/test_nonexistent_module/")) + + def test_init_py_uses_parent_name(self): + """Test __init__.py uses parent directory name.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # Create structure: tmpdir/Lib/json/__init__.py + lib_dir = tmpdir / "Lib" + lib_dir.mkdir() + json_dir = lib_dir / "json" + json_dir.mkdir() + (json_dir / "__init__.py").write_text("# json init") + test_dir = lib_dir / "test" + test_dir.mkdir() + + result = lib_to_test_path(tmpdir / "Lib" / "json" / "__init__.py") + # Should use "json" not "__init__" + self.assertEqual(result, tmpdir / "Lib" / "test" / "test_json/") + + def test_init_py_lib_path_uses_parent_name(self): + """Test __init__.py with Lib/ path uses parent directory name.""" + result = lib_to_test_path(pathlib.Path("Lib/json/__init__.py")) + # Should use "json" not "__init__" + self.assertEqual(result, pathlib.Path("Lib/test/test_json/")) + + +class TestGetTestFiles(unittest.TestCase): + """Tests for get_test_files function.""" + + def test_single_file(self): + """Test getting single file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_file = tmpdir / "test.py" + test_file.write_text("# test") + + files = get_test_files(test_file) + self.assertEqual(len(files), 1) + self.assertEqual(files[0], test_file) + + def test_directory(self): + """Test getting all .py files from directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + (tmpdir / "test_a.py").write_text("# a") + (tmpdir / "test_b.py").write_text("# b") + (tmpdir / "not_python.txt").write_text("# not python") + + files = get_test_files(tmpdir) + self.assertEqual(len(files), 2) + names = [f.name for f in files] + self.assertIn("test_a.py", names) + self.assertIn("test_b.py", names) + + def test_nested_directory(self): + """Test getting .py files from nested directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + (tmpdir / "test_a.py").write_text("# a") + subdir = tmpdir / "subdir" + subdir.mkdir() + (subdir / "test_b.py").write_text("# b") + + files = get_test_files(tmpdir) + self.assertEqual(len(files), 2) + + +class TestTestNameFromPath(unittest.TestCase): + """Tests for get_test_module_name function.""" + + def test_simple_test_file(self): + """Test extracting name from simple test file.""" + path = pathlib.Path("Lib/test/test_foo.py") + self.assertEqual(get_test_module_name(path), "test_foo") + + def test_nested_test_file(self): + """Test extracting name from nested test directory.""" + path = pathlib.Path("Lib/test/test_ctypes/test_bar.py") + self.assertEqual(get_test_module_name(path), "test_ctypes.test_bar") + + def test_test_directory(self): + """Test extracting name from test directory.""" + path = pathlib.Path("Lib/test/test_json") + self.assertEqual(get_test_module_name(path), "test_json") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_lib/tests/test_quick.py b/scripts/update_lib/tests/test_quick.py new file mode 100644 index 00000000000..f0262eebd04 --- /dev/null +++ b/scripts/update_lib/tests/test_quick.py @@ -0,0 +1,287 @@ +"""Tests for quick.py - quick update functionality.""" + +import pathlib +import tempfile +import unittest +from unittest.mock import patch + +from update_lib.cmd_quick import ( + _expand_shortcut, + collect_original_methods, + get_cpython_dir, + git_commit, +) +from update_lib.file_utils import lib_to_test_path + + +class TestGetCpythonDir(unittest.TestCase): + """Tests for get_cpython_dir function.""" + + def test_extract_from_full_path(self): + """Test extracting cpython dir from full path.""" + path = pathlib.Path("cpython/Lib/dataclasses.py") + result = get_cpython_dir(path) + self.assertEqual(result, pathlib.Path("cpython")) + + def test_extract_from_absolute_path(self): + """Test extracting cpython dir from absolute path.""" + path = pathlib.Path("/some/path/cpython/Lib/test/test_foo.py") + result = get_cpython_dir(path) + self.assertEqual(result, pathlib.Path("/some/path/cpython")) + + def test_shortcut_defaults_to_cpython(self): + """Test that shortcut (no /Lib/) defaults to 'cpython'.""" + path = pathlib.Path("dataclasses") + result = get_cpython_dir(path) + self.assertEqual(result, pathlib.Path("cpython")) + + +class TestExpandShortcut(unittest.TestCase): + """Tests for _expand_shortcut function.""" + + def test_expand_shortcut_to_test_path_integration(self): + """Test that expanded shortcut works with lib_to_test_path. + + This tests the fix for the bug where args.path was used instead of + the expanded src_path when calling lib_to_test_path. + """ + # Simulate the flow in main(): + # 1. User provides "dataclasses" + # 2. _expand_shortcut converts to "cpython/Lib/dataclasses.py" + # 3. lib_to_test_path should receive the expanded path, not original + + original_path = pathlib.Path("dataclasses") + expanded_path = _expand_shortcut(original_path) + + # If cpython/Lib/dataclasses.py exists, it should be expanded + if expanded_path != original_path: + # The expanded path should work with lib_to_test_path + test_path = lib_to_test_path(expanded_path) + # Should return a valid test path, not raise an error + self.assertTrue(str(test_path).startswith("cpython/Lib/test/")) + + # The original unexpanded path would fail or give wrong result + # This is what the bug was - using args.path instead of src_path + + def test_expand_shortcut_file(self): + """Test expanding a simple name to file path.""" + # This test checks the shortcut works when file exists + path = pathlib.Path("dataclasses") + result = _expand_shortcut(path) + + expected_file = pathlib.Path("cpython/Lib/dataclasses.py") + expected_dir = pathlib.Path("cpython/Lib/dataclasses") + + if expected_file.exists(): + self.assertEqual(result, expected_file) + elif expected_dir.exists(): + self.assertEqual(result, expected_dir) + else: + # If neither exists, should return original + self.assertEqual(result, path) + + def test_expand_shortcut_already_full_path(self): + """Test that full paths are not modified.""" + path = pathlib.Path("cpython/Lib/dataclasses.py") + result = _expand_shortcut(path) + self.assertEqual(result, path) + + def test_expand_shortcut_nonexistent(self): + """Test that nonexistent names are returned as-is.""" + path = pathlib.Path("nonexistent_module_xyz") + result = _expand_shortcut(path) + self.assertEqual(result, path) + + def test_expand_shortcut_uses_dependencies_table(self): + """Test that _expand_shortcut uses DEPENDENCIES table for overrides.""" + from update_lib.deps import DEPENDENCIES + + # regrtest has lib override in DEPENDENCIES + self.assertIn("regrtest", DEPENDENCIES) + self.assertIn("lib", DEPENDENCIES["regrtest"]) + + # _expand_shortcut should use this override when path exists + path = pathlib.Path("regrtest") + expected = pathlib.Path("cpython/Lib/test/libregrtest") + + # Only test expansion if cpython checkout exists + if expected.exists(): + result = _expand_shortcut(path) + self.assertEqual( + result, expected, "_expand_shortcut should expand 'regrtest'" + ) + + +class TestCollectOriginalMethods(unittest.TestCase): + """Tests for collect_original_methods function.""" + + def test_collect_from_file(self): + """Test collecting methods from single file.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + test_file = tmpdir / "test.py" + test_file.write_text(""" +class TestFoo: + def test_one(self): + pass + + def test_two(self): + pass +""") + + methods = collect_original_methods(test_file) + self.assertIsInstance(methods, set) + self.assertEqual(len(methods), 2) + self.assertIn(("TestFoo", "test_one"), methods) + self.assertIn(("TestFoo", "test_two"), methods) + + def test_collect_from_directory(self): + """Test collecting methods from directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + (tmpdir / "test_a.py").write_text(""" +class TestA: + def test_a(self): + pass +""") + (tmpdir / "test_b.py").write_text(""" +class TestB: + def test_b(self): + pass +""") + + methods = collect_original_methods(tmpdir) + self.assertIsInstance(methods, dict) + self.assertEqual(len(methods), 2) + + +class TestGitCommit(unittest.TestCase): + """Tests for git_commit function.""" + + @patch("subprocess.run") + @patch("update_lib.cmd_quick.get_cpython_version") + def test_none_lib_path_not_added(self, mock_version, mock_run): + """Test that None lib_path doesn't add '.' to git.""" + mock_version.return_value = "v3.14.0" + mock_run.return_value.returncode = 1 # Has changes + + with tempfile.TemporaryDirectory() as tmpdir: + test_file = pathlib.Path(tmpdir) / "test.py" + test_file.write_text("# test") + + git_commit("test", None, test_file, pathlib.Path("cpython"), verbose=False) + + # Check git add was called with only test_file, not "." + add_call = mock_run.call_args_list[0] + self.assertIn(str(test_file), add_call[0][0]) + self.assertNotIn(".", add_call[0][0][2:]) # Skip "git" and "add" + + @patch("subprocess.run") + @patch("update_lib.cmd_quick.get_cpython_version") + def test_none_test_path_not_added(self, mock_version, mock_run): + """Test that None test_path doesn't add '.' to git.""" + mock_version.return_value = "v3.14.0" + mock_run.return_value.returncode = 1 + + with tempfile.TemporaryDirectory() as tmpdir: + lib_file = pathlib.Path(tmpdir) / "lib.py" + lib_file.write_text("# lib") + + git_commit("lib", lib_file, None, pathlib.Path("cpython"), verbose=False) + + add_call = mock_run.call_args_list[0] + self.assertIn(str(lib_file), add_call[0][0]) + self.assertNotIn(".", add_call[0][0][2:]) + + def test_both_none_returns_false(self): + """Test that both paths None returns False without git operations.""" + # No mocking needed - should return early before any subprocess calls + result = git_commit("test", None, None, pathlib.Path("cpython"), verbose=False) + self.assertFalse(result) + + @patch("subprocess.run") + @patch("update_lib.cmd_quick.get_cpython_version") + def test_hard_deps_are_added(self, mock_version, mock_run): + """Test that hard_deps are included in git commit.""" + mock_version.return_value = "v3.14.0" + mock_run.return_value.returncode = 1 # Has changes + + with tempfile.TemporaryDirectory() as tmpdir: + lib_file = pathlib.Path(tmpdir) / "lib.py" + lib_file.write_text("# lib") + test_file = pathlib.Path(tmpdir) / "test.py" + test_file.write_text("# test") + dep_file = pathlib.Path(tmpdir) / "_dep.py" + dep_file.write_text("# dep") + + git_commit( + "test", + lib_file, + test_file, + pathlib.Path("cpython"), + hard_deps=[dep_file], + verbose=False, + ) + + # Check git add was called with all three files + add_call = mock_run.call_args_list[0] + add_args = add_call[0][0] + self.assertIn(str(lib_file), add_args) + self.assertIn(str(test_file), add_args) + self.assertIn(str(dep_file), add_args) + + @patch("subprocess.run") + @patch("update_lib.cmd_quick.get_cpython_version") + def test_nonexistent_hard_deps_not_added(self, mock_version, mock_run): + """Test that nonexistent hard_deps don't cause errors.""" + mock_version.return_value = "v3.14.0" + mock_run.return_value.returncode = 1 # Has changes + + with tempfile.TemporaryDirectory() as tmpdir: + lib_file = pathlib.Path(tmpdir) / "lib.py" + lib_file.write_text("# lib") + nonexistent_dep = pathlib.Path(tmpdir) / "nonexistent.py" + + git_commit( + "test", + lib_file, + None, + pathlib.Path("cpython"), + hard_deps=[nonexistent_dep], + verbose=False, + ) + + # Check git add was called with only lib_file + add_call = mock_run.call_args_list[0] + add_args = add_call[0][0] + self.assertIn(str(lib_file), add_args) + self.assertNotIn(str(nonexistent_dep), add_args) + + +class TestQuickTestRunFailure(unittest.TestCase): + """Tests for quick() behavior when test run fails.""" + + @patch("update_lib.cmd_auto_mark.run_test") + def test_auto_mark_raises_on_test_run_failure(self, mock_run_test): + """Test that auto_mark_file raises when test run fails entirely.""" + from update_lib.cmd_auto_mark import TestResult, TestRunError, auto_mark_file + + # Simulate test runner crash (empty tests_result) + mock_run_test.return_value = TestResult( + tests_result="", tests=[], stdout="crash" + ) + + with tempfile.TemporaryDirectory() as tmpdir: + # Create a fake test file with Lib/test structure + lib_test_dir = pathlib.Path(tmpdir) / "Lib" / "test" + lib_test_dir.mkdir(parents=True) + test_file = lib_test_dir / "test_foo.py" + test_file.write_text("import unittest\nclass Test(unittest.TestCase): pass") + + # auto_mark_file should raise TestRunError + with self.assertRaises(TestRunError): + auto_mark_file(test_file) + + +if __name__ == "__main__": + unittest.main() diff --git a/whats_left.py b/scripts/whats_left.py similarity index 86% rename from whats_left.py rename to scripts/whats_left.py index 4f087f89af3..9a4d57df6ae 100755 --- a/whats_left.py +++ b/scripts/whats_left.py @@ -1,4 +1,7 @@ #!/usr/bin/env -S python3 -I +# /// script +# requires-python = ">=3.14" +# /// # This script generates Lib/snippets/whats_left_data.py with these variables defined: # expected_methods - a dictionary mapping builtin objects to their methods @@ -14,20 +17,19 @@ # We then run this second generated script with RustPython. import argparse -import re +import inspect +import json import os +import platform import re +import subprocess import sys -import json import warnings -import inspect -import subprocess -import platform from pydoc import ModuleScanner if not sys.flags.isolated: print("running without -I option.") - print("python -I whats_left.py") + print("python -I scripts/whats_left.py") exit(1) GENERATED_FILE = "extra_tests/not_impl.py" @@ -35,8 +37,11 @@ implementation = platform.python_implementation() if implementation != "CPython": sys.exit(f"whats_left.py must be run under CPython, got {implementation} instead") -if sys.version_info[:2] < (3, 12): - sys.exit(f"whats_left.py must be run under CPython 3.12 or newer, got {implementation} {sys.version} instead") +if sys.version_info[:2] < (3, 14): + sys.exit( + f"whats_left.py must be run under CPython 3.14 or newer, got {implementation} {sys.version} instead. If you have uv, try `uv run python -I scripts/whats_left.py` to select a proper Python interpreter easier." + ) + def parse_args(): parser = argparse.ArgumentParser(description="Process some integers.") @@ -55,6 +60,17 @@ def parse_args(): action="store_true", help="print output as JSON (instead of line by line)", ) + parser.add_argument( + "--no-default-features", + action="store_true", + help="disable default features when building RustPython", + ) + parser.add_argument( + "--features", + action="append", + help="which features to enable when building RustPython (default: [])", + default=[], + ) args = parser.parse_args() return args @@ -62,46 +78,21 @@ def parse_args(): args = parse_args() - -# modules suggested for deprecation by PEP 594 (www.python.org/dev/peps/pep-0594/) -# some of these might be implemented, but they are not a priority -PEP_594_MODULES = { - "aifc", - "asynchat", - "asyncore", - "audioop", - "binhex", - "cgi", - "cgitb", - "chunk", - "crypt", - "formatter", - "fpectl", - "imghdr", - "imp", - "macpath", - "msilib", - "nntplib", - "nis", - "ossaudiodev", - "parser", - "pipes", - "smtpd", - "sndhdr", - "spwd", - "sunau", - "telnetlib", - "uu", - "xdrlib", -} - # CPython specific modules (mostly consisting of templates/tests) CPYTHON_SPECIFIC_MODS = { - 'xxmodule', 'xxsubtype', 'xxlimited', '_xxtestfuzz' - '_testbuffer', '_testcapi', '_testimportmultiple', '_testinternalcapi', '_testmultiphase', + "xxmodule", + "xxsubtype", + "xxlimited", + "_xxtestfuzz", + "_testbuffer", + "_testcapi", + "_testimportmultiple", + "_testinternalcapi", + "_testmultiphase", + "_testlimitedcapi", } -IGNORED_MODULES = {"this", "antigravity"} | PEP_594_MODULES | CPYTHON_SPECIFIC_MODS +IGNORED_MODULES = {"this", "antigravity"} | CPYTHON_SPECIFIC_MODS sys.path = [ path @@ -209,6 +200,9 @@ def gen_methods(): typ = eval(typ_code) attrs = [] for attr in dir(typ): + # Skip attributes in dir() but not actually accessible (e.g., descriptor that raises) + if not hasattr(typ, attr): + continue if attr_is_not_inherited(typ, attr): attrs.append((attr, extra_info(getattr(typ, attr)))) methods[typ.__name__] = (typ_code, extra_info(typ), attrs) @@ -315,7 +309,7 @@ def gen_modules(): output += gen_methods() output += f""" cpymods = {gen_modules()!r} -libdir = {os.path.abspath("Lib/").encode('utf8')!r} +libdir = {os.path.abspath("Lib/").encode("utf8")!r} """ @@ -334,17 +328,19 @@ def gen_modules(): expected_methods = {} cpymods = {} libdir = "" + + # This function holds the source code that will be run under RustPython def compare(): import inspect import io + import json import os + import platform import re import sys import warnings from contextlib import redirect_stdout - import json - import platform def method_incompatibility_reason(typ, method_name, real_method_value): has_method = hasattr(typ, method_name) @@ -373,7 +369,9 @@ def method_incompatibility_reason(typ, method_name, real_method_value): if platform.python_implementation() == "CPython": if not_implementeds: - sys.exit("ERROR: CPython should have all the methods") + sys.exit( + f"ERROR: CPython should have all the methods but missing: {not_implementeds}" + ) mod_names = [ name.decode() @@ -400,7 +398,9 @@ def method_incompatibility_reason(typ, method_name, real_method_value): if rustpymod is None: result["not_implemented"][modname] = None elif isinstance(rustpymod, Exception): - result["failed_to_import"][modname] = rustpymod.__class__.__name__ + str(rustpymod) + result["failed_to_import"][modname] = rustpymod.__class__.__name__ + str( + rustpymod + ) else: implemented_items = sorted(set(cpymod) & set(rustpymod)) mod_missing_items = set(cpymod) - set(rustpymod) @@ -442,19 +442,38 @@ def remove_one_indent(s): compare_src = inspect.getsourcelines(compare)[0][1:] output += "".join(remove_one_indent(line) for line in compare_src) -with open(GENERATED_FILE, "w", encoding='utf-8') as f: +with open(GENERATED_FILE, "w", encoding="utf-8") as f: f.write(output + "\n") -subprocess.run(["cargo", "build", "--release", "--features=ssl"], check=True) +cargo_build_command = ["cargo", "build", "--release"] +if args.no_default_features: + cargo_build_command.append("--no-default-features") + +joined_features = ",".join(args.features) +if args.features: + cargo_build_command.extend(["--features", joined_features]) + +subprocess.run(cargo_build_command, check=True) + +cargo_run_command = ["cargo", "run", "--release"] +if args.no_default_features: + cargo_run_command.append("--no-default-features") + +if args.features: + cargo_run_command.extend(["--features", joined_features]) + +cargo_run_command.extend(["-q", "--", GENERATED_FILE]) + result = subprocess.run( - ["cargo", "run", "--release", "--features=ssl", "-q", "--", GENERATED_FILE], + cargo_run_command, env={**os.environ.copy(), "RUSTPYTHONPATH": "Lib"}, text=True, capture_output=True, ) # The last line should be json output, the rest of the lines can contain noise # because importing certain modules can print stuff to stdout/stderr +print(result.stderr, file=sys.stderr) result = json.loads(result.stdout.splitlines()[-1]) if args.json: @@ -499,7 +518,7 @@ def remove_one_indent(s): if args.doc: print("\n# mismatching `__doc__`s (warnings)") for modname, mismatched in result["mismatched_doc_items"].items(): - for (item, rustpy_doc, cpython_doc) in mismatched: + for item, rustpy_doc, cpython_doc in mismatched: print(f"{item} {repr(rustpy_doc)} != {repr(cpython_doc)}") diff --git a/src/interpreter.rs b/src/interpreter.rs index b84f167ae47..b9ee2dbbc44 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -1,104 +1,76 @@ -use rustpython_vm::{Interpreter, Settings, VirtualMachine}; +use rustpython_vm::InterpreterBuilder; -pub type InitHook = Box<dyn FnOnce(&mut VirtualMachine)>; - -/// The convenient way to create [rustpython_vm::Interpreter] with stdlib and other stuffs. -/// -/// Basic usage: -/// ``` -/// let interpreter = rustpython::InterpreterConfig::new() -/// .init_stdlib() -/// .interpreter(); -/// ``` -/// -/// To override [rustpython_vm::Settings]: -/// ``` -/// use rustpython_vm::Settings; -/// // Override your settings here. -/// let mut settings = Settings::default(); -/// settings.debug = true; -/// // You may want to add paths to `rustpython_vm::Settings::path_list` to allow import python libraries. -/// settings.path_list.push("".to_owned()); // add current working directory -/// let interpreter = rustpython::InterpreterConfig::new() -/// .settings(settings) -/// .interpreter(); -/// ``` -/// -/// To add native modules: -/// ```compile_fail -/// let interpreter = rustpython::InterpreterConfig::new() -/// .init_stdlib() -/// .init_hook(Box::new(|vm| { -/// vm.add_native_module( -/// "your_module_name".to_owned(), -/// Box::new(your_module::make_module), -/// ); -/// })) -/// .interpreter(); -/// ``` -#[derive(Default)] -pub struct InterpreterConfig { - settings: Option<Settings>, - init_hooks: Vec<InitHook>, +/// Extension trait for InterpreterBuilder to add rustpython-specific functionality. +pub trait InterpreterBuilderExt { + /// Initialize the Python standard library. + /// + /// Requires the `stdlib` feature to be enabled. + #[cfg(feature = "stdlib")] + fn init_stdlib(self) -> Self; } -impl InterpreterConfig { - pub fn new() -> Self { - Self::default() - } - pub fn interpreter(self) -> Interpreter { - let settings = self.settings.unwrap_or_default(); - Interpreter::with_init(settings, |vm| { - for hook in self.init_hooks { - hook(vm); - } - }) - } - - pub fn settings(mut self, settings: Settings) -> Self { - self.settings = Some(settings); - self - } - pub fn init_hook(mut self, hook: InitHook) -> Self { - self.init_hooks.push(hook); - self - } +impl InterpreterBuilderExt for InterpreterBuilder { #[cfg(feature = "stdlib")] - pub fn init_stdlib(self) -> Self { - self.init_hook(Box::new(init_stdlib)) + fn init_stdlib(self) -> Self { + let defs = rustpython_stdlib::stdlib_module_defs(&self.ctx); + let builder = self.add_native_modules(&defs); + + #[cfg(feature = "freeze-stdlib")] + let builder = builder + .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB) + .init_hook(set_frozen_stdlib_dir); + + #[cfg(not(feature = "freeze-stdlib"))] + let builder = builder.init_hook(setup_dynamic_stdlib); + + builder } } -#[cfg(feature = "stdlib")] -pub fn init_stdlib(vm: &mut VirtualMachine) { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); +/// Set stdlib_dir for frozen standard library +#[cfg(all(feature = "stdlib", feature = "freeze-stdlib"))] +fn set_frozen_stdlib_dir(vm: &mut crate::VirtualMachine) { + use rustpython_vm::common::rc::PyRc; - // if we're on freeze-stdlib, the core stdlib modules will be included anyway - #[cfg(feature = "freeze-stdlib")] - vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); + let state = PyRc::get_mut(&mut vm.state).unwrap(); + state.config.paths.stdlib_dir = Some(rustpython_pylib::LIB_PATH.to_owned()); +} - #[cfg(not(feature = "freeze-stdlib"))] - { - use rustpython_vm::common::rc::PyRc; +/// Setup dynamic standard library loading from filesystem +#[cfg(all(feature = "stdlib", not(feature = "freeze-stdlib")))] +fn setup_dynamic_stdlib(vm: &mut crate::VirtualMachine) { + use rustpython_vm::common::rc::PyRc; - let state = PyRc::get_mut(&mut vm.state).unwrap(); - let settings = &mut state.settings; + let state = PyRc::get_mut(&mut vm.state).unwrap(); + let paths = collect_stdlib_paths(); - let path_list = std::mem::take(&mut settings.path_list); + // Set stdlib_dir to the first stdlib path if available + if let Some(first_path) = paths.first() { + state.config.paths.stdlib_dir = Some(first_path.clone()); + } - // BUILDTIME_RUSTPYTHONPATH should be set when distributing - if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") { - settings.path_list.extend( - crate::settings::split_paths(paths) - .map(|path| path.into_os_string().into_string().unwrap()), - ) - } else { - #[cfg(feature = "rustpython-pylib")] - settings - .path_list - .push(rustpython_pylib::LIB_PATH.to_owned()) - } + // Insert at the beginning so stdlib comes before user paths + for path in paths.into_iter().rev() { + state.config.paths.module_search_paths.insert(0, path); + } +} - settings.path_list.extend(path_list); +/// Collect standard library paths from build-time configuration +#[cfg(all(feature = "stdlib", not(feature = "freeze-stdlib")))] +fn collect_stdlib_paths() -> Vec<String> { + // BUILDTIME_RUSTPYTHONPATH should be set when distributing + if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") { + crate::settings::split_paths(paths) + .map(|path| path.into_os_string().into_string().unwrap()) + .collect() + } else { + #[cfg(feature = "rustpython-pylib")] + { + vec![rustpython_pylib::LIB_PATH.to_owned()] + } + #[cfg(not(feature = "rustpython-pylib"))] + { + vec![] + } } } diff --git a/src/lib.rs b/src/lib.rs index f916dd6828c..07bfc45e59f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,23 +1,29 @@ //! This is the `rustpython` binary. If you're looking to embed RustPython into your application, -//! you're likely looking for the [`rustpython-vm`](https://docs.rs/rustpython-vm) crate. +//! you're likely looking for the [`rustpython_vm`] crate. //! -//! You can install `rustpython` with `cargo install rustpython`, or if you'd like to inject your -//! own native modules you can make a binary crate that depends on the `rustpython` crate (and -//! probably `rustpython-vm`, too), and make a `main.rs` that looks like: +//! You can install `rustpython` with `cargo install rustpython`. If you'd like to inject your +//! own native modules, you can make a binary crate that depends on the `rustpython` crate (and +//! probably [`rustpython_vm`], too), and make a `main.rs` that looks like: //! //! ```no_run +//! use rustpython::{InterpreterBuilder, InterpreterBuilderExt}; //! use rustpython_vm::{pymodule, py_freeze}; -//! fn main() { -//! rustpython::run(|vm| { -//! vm.add_native_module("mymod".to_owned(), Box::new(mymod::make_module)); -//! vm.add_frozen(py_freeze!(source = "def foo(): pass", module_name = "otherthing")); -//! }); +//! +//! fn main() -> std::process::ExitCode { +//! let builder = InterpreterBuilder::new().init_stdlib(); +//! // Add a native module using builder.ctx +//! let my_mod_def = my_mod::module_def(&builder.ctx); +//! let builder = builder +//! .add_native_module(my_mod_def) +//! // Add a frozen module +//! .add_frozen_modules(py_freeze!(source = "def foo(): pass", module_name = "other_thing")); +//! +//! rustpython::run(builder) //! } //! //! #[pymodule] -//! mod mymod { +//! mod my_mod { //! use rustpython_vm::builtins::PyStrRef; -//TODO: use rustpython_vm::prelude::*; //! //! #[pyfunction] //! fn do_thing(x: i32) -> i32 { @@ -27,7 +33,7 @@ //! #[pyfunction] //! fn other_thing(s: PyStrRef) -> (String, usize) { //! let new_string = format!("hello from rust, {}!", s); -//! let prev_len = s.as_str().len(); +//! let prev_len = s.byte_len(); //! (new_string, prev_len) //! } //! } @@ -35,11 +41,11 @@ //! //! The binary will have all the standard arguments of a python interpreter (including a REPL!) but //! it will have your modules loaded into the vm. +//! +//! See [`rustpython_derive`](../rustpython_derive/index.html) crate for documentation on macros used in the example above. + #![allow(clippy::needless_doctest_main)] -#[macro_use] -extern crate clap; -extern crate env_logger; #[macro_use] extern crate log; @@ -50,17 +56,31 @@ mod interpreter; mod settings; mod shell; -use atty::Stream; -use rustpython_vm::{scope::Scope, PyResult, VirtualMachine}; -use std::{env, process::ExitCode}; - -pub use interpreter::InterpreterConfig; -pub use rustpython_vm as vm; -pub use settings::{opts_with_clap, RunMode}; - -/// The main cli of the `rustpython` interpreter. This function will return with `std::process::ExitCode` +use rustpython_vm::{AsObject, PyObjectRef, PyResult, VirtualMachine, scope::Scope}; +use std::env; +use std::io::IsTerminal; +use std::process::ExitCode; + +pub use interpreter::InterpreterBuilderExt; +pub use rustpython_vm::{self as vm, Interpreter, InterpreterBuilder}; +pub use settings::{InstallPipMode, RunMode, parse_opts}; +pub use shell::run_shell; + +#[cfg(all( + feature = "ssl", + not(any(feature = "ssl-rustls", feature = "ssl-openssl")) +))] +compile_error!( + "Feature \"ssl\" is now enabled by either \"ssl-rustls\" or \"ssl-openssl\" to be enabled. Do not manually pass \"ssl\" feature. To enable ssl-openssl, use --no-default-features to disable ssl-rustls" +); + +/// The main cli of the `rustpython` interpreter. This function will return `std::process::ExitCode` /// based on the return code of the python code ran through the cli. -pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { +/// +/// **Note**: This function provides no way to further initialize the VM after the builder is applied. +/// All VM initialization (adding native modules, init hooks, etc.) must be done through the +/// [`InterpreterBuilder`] parameter before calling this function. +pub fn run(mut builder: InterpreterBuilder) -> ExitCode { env_logger::init(); // NOTE: This is not a WASI convention. But it will be convenient since POSIX shell always defines it. @@ -71,15 +91,18 @@ pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { }; } - let (settings, run_mode) = opts_with_clap(); - - // Be quiet if "quiet" arg is set OR stdin is not connected to a terminal - let quiet_var = settings.quiet || !atty::is(Stream::Stdin); + let (settings, run_mode) = match parse_opts() { + Ok(x) => x, + Err(e) => { + println!("{e}"); + return ExitCode::FAILURE; + } + }; // don't translate newlines (\r\n <=> \n) #[cfg(windows)] { - extern "C" { + unsafe extern "C" { fn _setmode(fd: i32, flags: i32) -> i32; } unsafe { @@ -89,35 +112,14 @@ pub fn run(init: impl FnOnce(&mut VirtualMachine) + 'static) -> ExitCode { } } - let mut config = InterpreterConfig::new().settings(settings); - #[cfg(feature = "stdlib")] - { - config = config.init_stdlib(); - } - config = config.init_hook(Box::new(init)); - - let interp = config.interpreter(); - let exitcode = interp.run(move |vm| run_rustpython(vm, run_mode, quiet_var)); - - ExitCode::from(exitcode) -} - -fn setup_main_module(vm: &VirtualMachine) -> PyResult<Scope> { - let scope = vm.new_scope_with_builtins(); - let main_module = vm.new_module("__main__", scope.globals.clone(), None); - main_module - .dict() - .set_item("__annotations__", vm.ctx.new_dict().into(), vm) - .expect("Failed to initialize __main__.__annotations__"); + builder = builder.settings(settings); - vm.sys_module - .get_attr("modules", vm)? - .set_item("__main__", main_module.into(), vm)?; + let interp = builder.interpreter(); + let exitcode = interp.run(move |vm| run_rustpython(vm, run_mode)); - Ok(scope) + rustpython_vm::common::os::exit_code(exitcode) } -#[cfg(feature = "ssl")] fn get_pip(scope: Scope, vm: &VirtualMachine) -> PyResult<()> { let get_getpip = rustpython_vm::py_compile!( source = r#"\ @@ -128,54 +130,113 @@ __import__("io").TextIOWrapper( mode = "eval" ); eprintln!("downloading get-pip.py..."); - let getpip_code = vm.run_code_obj(vm.ctx.new_code(get_getpip), scope.clone())?; + let getpip_code = vm.run_code_obj(vm.ctx.new_code(get_getpip), vm.new_scope_with_builtins())?; let getpip_code: rustpython_vm::builtins::PyStrRef = getpip_code .downcast() .expect("TextIOWrapper.read() should return str"); eprintln!("running get-pip.py..."); - vm.run_code_string(scope, getpip_code.as_str(), "get-pip.py".to_owned())?; + vm.run_string(scope, getpip_code.expect_str(), "get-pip.py".to_owned())?; Ok(()) } -#[cfg(feature = "ssl")] -fn ensurepip(_: Scope, vm: &VirtualMachine) -> PyResult<()> { - vm.run_module("ensurepip") +fn install_pip(installer: InstallPipMode, scope: Scope, vm: &VirtualMachine) -> PyResult<()> { + if !cfg!(feature = "ssl") { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.system_error.to_owned(), + "install-pip requires rustpython be build with '--features=ssl'".into(), + )); + } + + match installer { + InstallPipMode::Ensurepip => vm.run_module("ensurepip"), + InstallPipMode::GetPip => get_pip(scope, vm), + } } -fn install_pip(_installer: &str, _scope: Scope, vm: &VirtualMachine) -> PyResult<()> { - #[cfg(feature = "ssl")] +// pymain_run_file_obj in Modules/main.c +fn run_file(vm: &VirtualMachine, scope: Scope, path: &str) -> PyResult<()> { + // Check if path is a package/directory with __main__.py + if let Some(_importer) = get_importer(path, vm)? { + vm.insert_sys_path(vm.new_pyobj(path))?; + let runpy = vm.import("runpy", 0)?; + let run_module_as_main = runpy.get_attr("_run_module_as_main", vm)?; + run_module_as_main.call((vm::identifier!(vm, __main__).to_owned(), false), vm)?; + return Ok(()); + } + + // Add script directory to sys.path[0] + if !vm.state.config.settings.safe_path { + let dir = std::path::Path::new(path) + .parent() + .and_then(|p| p.to_str()) + .unwrap_or(""); + vm.insert_sys_path(vm.new_pyobj(dir))?; + } + + #[cfg(feature = "host_env")] + { + vm.run_any_file(scope, path) + } + #[cfg(not(feature = "host_env"))] { - match _installer { - "ensurepip" => ensurepip(_scope, vm), - "get-pip" => get_pip(_scope, vm), - _ => unreachable!(), + // In sandbox mode, the binary reads the file and feeds source to the VM. + // The VM itself has no filesystem access. + let path = if path.is_empty() { "???" } else { path }; + match std::fs::read_to_string(path) { + Ok(source) => vm.run_string(scope, &source, path.to_owned()).map(drop), + Err(err) => Err(vm.new_os_error(err.to_string())), } } +} + +fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { + use rustpython_vm::builtins::PyDictRef; + use rustpython_vm::convert::TryFromObject; - #[cfg(not(feature = "ssl"))] - Err(vm.new_exception_msg( - vm.ctx.exceptions.system_error.to_owned(), - "install-pip requires rustpython be build with '--features=ssl'".to_owned(), - )) + let path_importer_cache = vm.sys_module.get_attr("path_importer_cache", vm)?; + let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?; + if let Some(importer) = path_importer_cache.get_item_opt(path, vm)? { + return Ok(Some(importer)); + } + let path_obj = vm.ctx.new_str(path); + let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?; + let mut importer = None; + let path_hooks: Vec<PyObjectRef> = path_hooks.try_into_value(vm)?; + for path_hook in path_hooks { + match path_hook.call((path_obj.clone(),), vm) { + Ok(imp) => { + importer = Some(imp); + break; + } + Err(e) if e.fast_isinstance(vm.ctx.exceptions.import_error) => continue, + Err(e) => return Err(e), + } + } + Ok(if let Some(imp) = importer { + let imp = path_importer_cache.get_or_insert(vm, path_obj.into(), || imp.clone())?; + Some(imp) + } else { + None + }) } -fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode, quiet: bool) -> PyResult<()> { +// pymain_run_python +fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { #[cfg(feature = "flame-it")] let main_guard = flame::start_guard("RustPython main"); - let scope = setup_main_module(vm)?; + let scope = vm.new_scope_with_main()?; - if !vm.state.settings.safe_path { - // TODO: The prepending path depends on running mode - // See https://docs.python.org/3/using/cmdline.html#cmdoption-P - vm.run_code_string( - vm.new_scope_with_builtins(), - "import sys; sys.path.insert(0, '')", - "<embedded>".to_owned(), - )?; + // Initialize warnings module to process sys.warnoptions + // _PyWarnings_Init() + if vm.import("warnings", 0).is_err() { + warn!("Failed to import warnings module"); } - let site_result = vm.import("site", None, 0); + // Import site first, before setting sys.path[0] + // This matches CPython's behavior where site.removeduppaths() runs + // before sys.path[0] is set, preventing '' from being converted to cwd + let site_result = vm.import("site", 0); if site_result.is_err() { warn!( "Failed to import site, consider adding the Lib directory to your RUSTPYTHONPATH \ @@ -183,58 +244,96 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode, quiet: bool) -> PyResu ); } - match run_mode { + // _PyPathConfig_ComputeSysPath0 - set sys.path[0] after site import + if !vm.state.config.settings.safe_path { + let path0: Option<String> = match &run_mode { + RunMode::Command(_) => Some(String::new()), + RunMode::Module(_) => env::current_dir() + .ok() + .and_then(|p| p.to_str().map(|s| s.to_owned())), + RunMode::Script(_) | RunMode::InstallPip(_) => None, // handled by run_script + RunMode::Repl => Some(String::new()), + }; + + if let Some(path) = path0 { + vm.insert_sys_path(vm.new_pyobj(path))?; + } + } + + // Enable faulthandler if -X faulthandler, PYTHONFAULTHANDLER or -X dev is set + // _PyFaulthandler_Init() + if vm.state.config.settings.faulthandler { + let _ = vm.run_simple_string("import faulthandler; faulthandler.enable()"); + } + + let is_repl = matches!(run_mode, RunMode::Repl); + if !vm.state.config.settings.quiet + && (vm.state.config.settings.verbose > 0 || (is_repl && std::io::stdin().is_terminal())) + { + eprintln!( + "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", + env!("CARGO_PKG_VERSION") + ); + eprintln!( + "RustPython {}.{}.{}", + vm::version::MAJOR, + vm::version::MINOR, + vm::version::MICRO, + ); + + eprintln!("Type \"help\", \"copyright\", \"credits\" or \"license\" for more information."); + } + let res = match run_mode { RunMode::Command(command) => { - debug!("Running command {}", command); - vm.run_code_string(scope, &command, "<stdin>".to_owned())?; + debug!("Running command {command}"); + vm.run_string(scope.clone(), &command, "<string>".to_owned()) + .map(drop) } RunMode::Module(module) => { - debug!("Running module {}", module); - vm.run_module(&module)?; + debug!("Running module {module}"); + vm.run_module(&module) } - RunMode::InstallPip(installer) => { - install_pip(&installer, scope, vm)?; + RunMode::InstallPip(installer) => install_pip(installer, scope.clone(), vm), + RunMode::Script(script_path) => { + // pymain_run_file_obj + debug!("Running script {}", &script_path); + run_file(vm, scope.clone(), &script_path) } - RunMode::ScriptInteractive(script, interactive) => { - if let Some(script) = script { - debug!("Running script {}", &script); - vm.run_script(scope.clone(), &script)?; - } else if !quiet { - println!( - "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", - crate_version!() - ); - } - if interactive { - shell::run_shell(vm, scope)?; - } - } - } + RunMode::Repl => Ok(()), + }; + let result = if is_repl || vm.state.config.settings.inspect { + shell::run_shell(vm, scope) + } else { + res + }; + #[cfg(feature = "flame-it")] { main_guard.end(); - if let Err(e) = write_profile(&vm.state.as_ref().settings) { + if let Err(e) = write_profile(&vm.state.as_ref().config.settings) { error!("Error writing profile information: {}", e); } } - Ok(()) + + result } #[cfg(feature = "flame-it")] -fn write_profile(settings: &Settings) -> Result<(), Box<dyn std::error::Error>> { +fn write_profile(settings: &Settings) -> Result<(), Box<dyn core::error::Error>> { use std::{fs, io}; enum ProfileFormat { Html, Text, - Speedscope, + SpeedScope, } let profile_output = settings.profile_output.as_deref(); let profile_format = match settings.profile_format.as_deref() { Some("html") => ProfileFormat::Html, Some("text") => ProfileFormat::Text, None if profile_output == Some("-".as_ref()) => ProfileFormat::Text, - Some("speedscope") | None => ProfileFormat::Speedscope, + // spell-checker:ignore speedscope + Some("speedscope") | None => ProfileFormat::SpeedScope, Some(other) => { error!("Unknown profile format {}", other); // TODO: Need to change to ExitCode or Termination @@ -245,7 +344,7 @@ fn write_profile(settings: &Settings) -> Result<(), Box<dyn std::error::Error>> let profile_output = profile_output.unwrap_or_else(|| match profile_format { ProfileFormat::Html => "flame-graph.html".as_ref(), ProfileFormat::Text => "flame.txt".as_ref(), - ProfileFormat::Speedscope => "flamescope.json".as_ref(), + ProfileFormat::SpeedScope => "flamescope.json".as_ref(), }); let profile_output: Box<dyn io::Write> = if profile_output == "-" { @@ -259,7 +358,7 @@ fn write_profile(settings: &Settings) -> Result<(), Box<dyn std::error::Error>> match profile_format { ProfileFormat::Html => flame::dump_html(profile_output)?, ProfileFormat::Text => flame::dump_text_to_writer(profile_output)?, - ProfileFormat::Speedscope => flamescope::dump(profile_output)?, + ProfileFormat::SpeedScope => flamescope::dump(profile_output)?, } Ok(()) @@ -271,20 +370,23 @@ mod tests { use rustpython_vm::Interpreter; fn interpreter() -> Interpreter { - InterpreterConfig::new().init_stdlib().interpreter() + InterpreterBuilder::new().init_stdlib().interpreter() } #[test] fn test_run_script() { interpreter().enter(|vm| { vm.unwrap_pyresult((|| { - let scope = setup_main_module(vm)?; + let scope = vm.new_scope_with_main()?; // test file run - vm.run_script(scope, "extra_tests/snippets/dir_main/__main__.py")?; - - let scope = setup_main_module(vm)?; - // test module run - vm.run_script(scope, "extra_tests/snippets/dir_main")?; + run_file(vm, scope, "extra_tests/snippets/dir_main/__main__.py")?; + + #[cfg(feature = "host_env")] + { + let scope = vm.new_scope_with_main()?; + // test module run (directory with __main__.py) + run_file(vm, scope, "extra_tests/snippets/dir_main")?; + } Ok(()) })()); diff --git a/src/main.rs b/src/main.rs index e88ea40f3df..3953b9dacfe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,10 @@ +use rustpython::{InterpreterBuilder, InterpreterBuilderExt}; + pub fn main() -> std::process::ExitCode { - rustpython::run(|_vm| {}) + let mut config = InterpreterBuilder::new(); + #[cfg(feature = "stdlib")] + { + config = config.init_stdlib(); + } + rustpython::run(config) } diff --git a/src/settings.rs b/src/settings.rs index 494d594de48..5233cf98d49 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -1,187 +1,220 @@ -use clap::{App, AppSettings, Arg, ArgMatches}; -use rustpython_vm::Settings; -use std::{env, str::FromStr}; +use lexopt::Arg::*; +use lexopt::ValueExt; +use rustpython_vm::{Settings, vm::CheckHashPycsMode}; +use std::str::FromStr; +use std::{cmp, env}; pub enum RunMode { - ScriptInteractive(Option<String>, bool), + Script(String), Command(String), Module(String), - InstallPip(String), + InstallPip(InstallPipMode), + Repl, } -pub fn opts_with_clap() -> (Settings, RunMode) { - let app = App::new("RustPython"); - let matches = parse_arguments(app); - settings_from(&matches) +pub enum InstallPipMode { + /// Install pip using the ensurepip pip module. This has a higher chance of + /// success, but may not install the latest version of pip. + Ensurepip, + /// Install pip using the get-pip.py script, which retrieves the latest pip version. + /// This can be broken due to incompatibilities with cpython. + GetPip, } -fn parse_arguments<'a>(app: App<'a, '_>) -> ArgMatches<'a> { - let app = app - .setting(AppSettings::TrailingVarArg) - .version(crate_version!()) - .author(crate_authors!()) - .about("Rust implementation of the Python language") - .usage("rustpython [OPTIONS] [-c CMD | -m MODULE | FILE] [PYARGS]...") - .arg( - Arg::with_name("script") - .required(false) - .allow_hyphen_values(true) - .multiple(true) - .value_name("script, args") - .min_values(1), - ) - .arg( - Arg::with_name("c") - .short("c") - .takes_value(true) - .allow_hyphen_values(true) - .multiple(true) - .value_name("cmd, args") - .min_values(1) - .help("run the given string as a program"), - ) - .arg( - Arg::with_name("m") - .short("m") - .takes_value(true) - .allow_hyphen_values(true) - .multiple(true) - .value_name("module, args") - .min_values(1) - .help("run library module as script"), - ) - .arg( - Arg::with_name("install_pip") - .long("install-pip") - .takes_value(true) - .allow_hyphen_values(true) - .multiple(true) - .value_name("get-pip args") - .min_values(0) - .help("install the pip package manager for rustpython; \ - requires rustpython be build with the ssl feature enabled." - ), - ) - .arg( - Arg::with_name("optimize") - .short("O") - .multiple(true) - .help("Optimize. Set __debug__ to false. Remove debug statements."), - ) - .arg( - Arg::with_name("verbose") - .short("v") - .multiple(true) - .help("Give the verbosity (can be applied multiple times)"), - ) - .arg(Arg::with_name("debug").short("d").help("Debug the parser.")) - .arg( - Arg::with_name("quiet") - .short("q") - .help("Be quiet at startup."), - ) - .arg( - Arg::with_name("inspect") - .short("i") - .help("Inspect interactively after running the script."), - ) - .arg( - Arg::with_name("no-user-site") - .short("s") - .help("don't add user site directory to sys.path."), - ) - .arg( - Arg::with_name("no-site") - .short("S") - .help("don't imply 'import site' on initialization"), - ) - .arg( - Arg::with_name("dont-write-bytecode") - .short("B") - .help("don't write .pyc files on import"), - ) - .arg( - Arg::with_name("safe-path") - .short("P") - .help("don’t prepend a potentially unsafe path to sys.path"), - ) - .arg( - Arg::with_name("ignore-environment") - .short("E") - .help("Ignore environment variables PYTHON* such as PYTHONPATH"), - ) - .arg( - Arg::with_name("isolate") - .short("I") - .help("isolate Python from the user's environment (implies -E and -s)"), - ) - .arg( - Arg::with_name("implementation-option") - .short("X") - .takes_value(true) - .multiple(true) - .number_of_values(1) - .help("set implementation-specific option"), - ) - .arg( - Arg::with_name("warning-control") - .short("W") - .takes_value(true) - .multiple(true) - .number_of_values(1) - .help("warning control; arg is action:message:category:module:lineno"), - ) - .arg( - Arg::with_name("check-hash-based-pycs") - .long("check-hash-based-pycs") - .takes_value(true) - .number_of_values(1) - .default_value("default") - .help("always|default|never\ncontrol how Python invalidates hash-based .pyc files"), - ) - .arg( - Arg::with_name("bytes-warning") - .short("b") - .multiple(true) - .help("issue warnings about using bytes where strings are usually expected (-bb: issue errors)"), - ).arg( - Arg::with_name("unbuffered") - .short("u") - .help( - "force the stdout and stderr streams to be unbuffered; \ - this option has no effect on stdin; also PYTHONUNBUFFERED=x", - ), - ); +impl FromStr for InstallPipMode { + type Err = &'static str; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "ensurepip" => Ok(Self::Ensurepip), + "get-pip" => Ok(Self::GetPip), + _ => Err("--install-pip takes ensurepip or get-pip as first argument"), + } + } +} + +#[derive(Default)] +struct CliArgs { + bytes_warning: u8, + dont_write_bytecode: bool, + debug: u8, + ignore_environment: bool, + inspect: bool, + isolate: bool, + optimize: u8, + safe_path: bool, + quiet: bool, + random_hash_seed: bool, + no_user_site: bool, + no_site: bool, + unbuffered: bool, + verbose: u8, + warning_control: Vec<String>, + implementation_option: Vec<String>, + check_hash_based_pycs: CheckHashPycsMode, + #[cfg(feature = "flame-it")] - let app = app - .arg( - Arg::with_name("profile_output") - .long("profile-output") - .takes_value(true) - .help("the file to output the profiling information to"), - ) - .arg( - Arg::with_name("profile_format") - .long("profile-format") - .takes_value(true) - .help("the profile format to output the profiling information in"), - ); - app.get_matches() + profile_output: Option<std::ffi::OsString>, + #[cfg(feature = "flame-it")] + profile_format: Option<String>, +} + +const USAGE_STRING: &str = "\ +usage: {PROG} [option] ... [-c cmd | -m mod | file | -] [arg] ... +Options (and corresponding environment variables): +-b : issue warnings about converting bytes/bytearray to str and comparing + bytes/bytearray with str or bytes with int. (-bb: issue errors) +-B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x +-c cmd : program passed in as string (terminates option list) +-d : turn on parser debugging output (for experts only, only works on + debug builds); also PYTHONDEBUG=x +-E : ignore PYTHON* environment variables (such as PYTHONPATH) +-h : print this help message and exit (also -? or --help) +-i : inspect interactively after running script; forces a prompt even + if stdin does not appear to be a terminal; also PYTHONINSPECT=x +-I : isolate Python from the user's environment (implies -E and -s) +-m mod : run library module as a script (terminates option list) +-O : remove assert and __debug__-dependent statements; add .opt-1 before + .pyc extension; also PYTHONOPTIMIZE=x +-OO : do -O changes and also discard docstrings; add .opt-2 before + .pyc extension +-P : don't prepend a potentially unsafe path to sys.path; also + PYTHONSAFEPATH +-q : don't print version and copyright messages on interactive startup +-s : don't add user site directory to sys.path; also PYTHONNOUSERSITE=x +-S : don't imply 'import site' on initialization +-u : force the stdout and stderr streams to be unbuffered; + this option has no effect on stdin; also PYTHONUNBUFFERED=x +-v : verbose (trace import statements); also PYTHONVERBOSE=x + can be supplied multiple times to increase verbosity +-V : print the Python version number and exit (also --version) + when given twice, print more information about the build +-W arg : warning control; arg is action:message:category:module:lineno + also PYTHONWARNINGS=arg +-x : skip first line of source, allowing use of non-Unix forms of #!cmd +-X opt : set implementation-specific option +--check-hash-based-pycs always|default|never: + control how Python invalidates hash-based .pyc files +--help-env: print help about Python environment variables and exit +--help-xoptions: print help about implementation-specific -X options and exit +--help-all: print complete help information and exit + +RustPython extensions: + + +Arguments: +file : program read from script file +- : program read from stdin (default; interactive mode if a tty) +arg ...: arguments passed to program in sys.argv[1:] +"; + +fn parse_args() -> Result<(CliArgs, RunMode, Vec<String>), lexopt::Error> { + let mut args = CliArgs::default(); + let mut parser = lexopt::Parser::from_env(); + fn argv(argv0: String, mut parser: lexopt::Parser) -> Result<Vec<String>, lexopt::Error> { + std::iter::once(Ok(argv0)) + .chain(parser.raw_args()?.map(|arg| arg.string())) + .collect() + } + while let Some(arg) = parser.next()? { + match arg { + Short('b') => args.bytes_warning += 1, + Short('B') => args.dont_write_bytecode = true, + Short('c') => { + let cmd = parser.value()?.string()?; + return Ok((args, RunMode::Command(cmd), argv("-c".to_owned(), parser)?)); + } + Short('d') => args.debug += 1, + Short('E') => args.ignore_environment = true, + Short('h' | '?') | Long("help") => help(parser), + Short('i') => args.inspect = true, + Short('I') => args.isolate = true, + Short('m') => { + let module = parser.value()?.string()?; + let argv = argv("PLACEHOLDER".to_owned(), parser)?; + return Ok((args, RunMode::Module(module), argv)); + } + Short('O') => args.optimize += 1, + Short('P') => args.safe_path = true, + Short('q') => args.quiet = true, + Short('R') => args.random_hash_seed = true, + Short('S') => args.no_site = true, + Short('s') => args.no_user_site = true, + Short('u') => args.unbuffered = true, + Short('v') => args.verbose += 1, + Short('V') | Long("version") => version(), + Short('W') => args.warning_control.push(parser.value()?.string()?), + // TODO: Short('x') => + Short('X') => args.implementation_option.push(parser.value()?.string()?), + + Long("check-hash-based-pycs") => { + args.check_hash_based_pycs = parser.value()?.parse()? + } + + // TODO: make these more specific + Long("help-env") => help(parser), + Long("help-xoptions") => help(parser), + Long("help-all") => help(parser), + + #[cfg(feature = "flame-it")] + Long("profile-output") => args.profile_output = Some(parser.value()?), + #[cfg(feature = "flame-it")] + Long("profile-format") => args.profile_format = Some(parser.value()?.string()?), + + Long("install-pip") => { + let (mode, argv) = if let Some(val) = parser.optional_value() { + (val.parse()?, vec![val.string()?]) + } else if let Ok(argv0) = parser.value() { + let mode = argv0.parse()?; + (mode, argv(argv0.string()?, parser)?) + } else { + ( + InstallPipMode::Ensurepip, + ["ensurepip", "--upgrade", "--default-pip"] + .map(str::to_owned) + .into(), + ) + }; + return Ok((args, RunMode::InstallPip(mode), argv)); + } + Value(script_name) => { + let script_name = script_name.string()?; + let mode = if script_name == "-" { + RunMode::Repl + } else { + RunMode::Script(script_name.clone()) + }; + return Ok((args, mode, argv(script_name, parser)?)); + } + _ => return Err(arg.unexpected()), + } + } + Ok((args, RunMode::Repl, vec![])) +} + +fn help(parser: lexopt::Parser) -> ! { + let usage = USAGE_STRING.replace("{PROG}", parser.bin_name().unwrap_or("rustpython")); + print!("{usage}"); + std::process::exit(0); +} + +fn version() -> ! { + println!("Python {}", rustpython_vm::version::get_version()); + std::process::exit(0); } /// Create settings by examining command line arguments and environment /// variables. -fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { +pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { + let (args, mode, argv) = parse_args()?; + let mut settings = Settings::default(); - settings.isolated = matches.is_present("isolate"); - settings.ignore_environment = matches.is_present("ignore-environment"); - settings.interactive = !matches.is_present("c") - && !matches.is_present("m") - && (!matches.is_present("script") || matches.is_present("inspect")); - settings.bytes_warning = matches.occurrences_of("bytes-warning"); - settings.no_site = matches.is_present("no-site"); + settings.isolated = args.isolate; + settings.ignore_environment = settings.isolated || args.ignore_environment; + settings.bytes_warning = args.bytes_warning.into(); + settings.import_site = !args.no_site; - let ignore_environment = settings.ignore_environment || settings.isolated; + let ignore_environment = settings.ignore_environment; if !ignore_environment { settings.path_list.extend(get_paths("RUSTPYTHONPATH")); @@ -189,108 +222,170 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { } // Now process command line flags: - if matches.is_present("debug") || (!ignore_environment && env::var_os("PYTHONDEBUG").is_some()) - { - settings.debug = true; - } - if matches.is_present("inspect") - || (!ignore_environment && env::var_os("PYTHONINSPECT").is_some()) - { - settings.inspect = true; - } + let get_env = |env| (!ignore_environment).then(|| env::var_os(env)).flatten(); - if matches.is_present("optimize") { - settings.optimize = matches.occurrences_of("optimize").try_into().unwrap(); - } else if !ignore_environment { - if let Ok(value) = get_env_var_value("PYTHONOPTIMIZE") { - settings.optimize = value; - } - } + let env_count = |env| { + get_env(env).filter(|v| !v.is_empty()).map_or(0, |val| { + val.to_str().and_then(|v| v.parse::<u8>().ok()).unwrap_or(1) + }) + }; - if matches.is_present("verbose") { - settings.verbose = matches.occurrences_of("verbose").try_into().unwrap(); - } else if !ignore_environment { - if let Ok(value) = get_env_var_value("PYTHONVERBOSE") { - settings.verbose = value; - } - } + settings.optimize = cmp::max(args.optimize, env_count("PYTHONOPTIMIZE")); + settings.verbose = cmp::max(args.verbose, env_count("PYTHONVERBOSE")); + settings.debug = cmp::max(args.debug, env_count("PYTHONDEBUG")); - if matches.is_present("no-user-site") - || matches.is_present("isolate") - || (!ignore_environment && env::var_os("PYTHONNOUSERSITE").is_some()) - { - settings.no_user_site = true; - } + let env_bool = |env| get_env(env).is_some_and(|v| !v.is_empty()); - if matches.is_present("quiet") { - settings.quiet = true; - } + settings.user_site_directory = + !(settings.isolated || args.no_user_site || env_bool("PYTHONNOUSERSITE")); + settings.quiet = args.quiet; + settings.write_bytecode = !(args.dont_write_bytecode || env_bool("PYTHONDONTWRITEBYTECODE")); + settings.safe_path = settings.isolated || args.safe_path || env_bool("PYTHONSAFEPATH"); + settings.inspect = args.inspect || env_bool("PYTHONINSPECT"); + settings.interactive = args.inspect; + settings.buffered_stdio = !args.unbuffered; - if matches.is_present("dont-write-bytecode") - || (!ignore_environment && env::var_os("PYTHONDONTWRITEBYTECODE").is_some()) - { - settings.dont_write_bytecode = true; - } - if !ignore_environment && env::var_os("PYTHONINTMAXSTRDIGITS").is_some() { - settings.int_max_str_digits = match env::var("PYTHONINTMAXSTRDIGITS").unwrap().parse() { - Ok(digits) if digits == 0 || digits >= 640 => digits, + if let Some(val) = get_env("PYTHONINTMAXSTRDIGITS") { + settings.int_max_str_digits = match val.to_str().and_then(|s| s.parse().ok()) { + Some(digits @ (0 | 640..)) => digits, _ => { - error!("Fatal Python error: config_init_int_max_str_digits: PYTHONINTMAXSTRDIGITS: invalid limit; must be >= 640 or 0 for unlimited.\nPython runtime state: preinitialized"); + error!( + "Fatal Python error: config_init_int_max_str_digits: PYTHONINTMAXSTRDIGITS: invalid limit; must be >= 640 or 0 for unlimited.\nPython runtime state: preinitialized" + ); std::process::exit(1); } }; } - if matches.is_present("safe-path") - || (!ignore_environment && env::var_os("PYTHONSAFEPATH").is_some()) + settings.check_hash_pycs_mode = args.check_hash_based_pycs; + + if let Some(val) = get_env("PYTHONUTF8") + && let Some(val_str) = val.to_str() + && !val_str.is_empty() { - settings.safe_path = true; + settings.utf8_mode = match val_str { + "1" => 1, + "0" => 0, + _ => { + error!( + "Fatal Python error: config_init_utf8_mode: \ + PYTHONUTF8=N: N is missing or invalid\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; } - settings.check_hash_based_pycs = matches - .value_of("check-hash-based-pycs") - .unwrap_or("default") - .to_owned(); - - let mut dev_mode = false; - let mut warn_default_encoding = false; - if let Some(xopts) = matches.values_of("implementation-option") { - settings.xopts.extend(xopts.map(|s| { - let mut parts = s.splitn(2, '='); - let name = parts.next().unwrap().to_owned(); - let value = parts.next().map(ToOwned::to_owned); - if name == "dev" { - dev_mode = true - } - if name == "warn_default_encoding" { - warn_default_encoding = true - } - if name == "no_sig_int" { - settings.no_sig_int = true; + let xopts = args.implementation_option.into_iter().map(|s| { + let (name, value) = match s.split_once('=') { + Some((name, value)) => (name.to_owned(), Some(value)), + None => (s, None), + }; + match &*name { + "dev" => settings.dev_mode = true, + "faulthandler" => settings.faulthandler = true, + "warn_default_encoding" => settings.warn_default_encoding = true, + "utf8" => { + settings.utf8_mode = match value { + None => 1, + Some("1") => 1, + Some("0") => 0, + _ => { + error!( + "Fatal Python error: config_init_utf8_mode: \ + -X utf8=n: n is missing or invalid\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; } - if name == "int_max_str_digits" { - settings.int_max_str_digits = match value.as_ref().unwrap().parse() { + "no_sig_int" => settings.install_signal_handlers = false, + "no_debug_ranges" => settings.code_debug_ranges = false, + "int_max_str_digits" => { + settings.int_max_str_digits = match value.unwrap().parse() { Ok(digits) if digits == 0 || digits >= 640 => digits, _ => { - - error!("Fatal Python error: config_init_int_max_str_digits: -X int_max_str_digits: invalid limit; must be >= 640 or 0 for unlimited.\nPython runtime state: preinitialized"); - std::process::exit(1); - }, + error!( + "Fatal Python error: config_init_int_max_str_digits: \ + -X int_max_str_digits: \ + invalid limit; must be >= 640 or 0 for unlimited.\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } }; } - (name, value) - })); + "thread_inherit_context" => { + settings.thread_inherit_context = match value { + Some("1") => true, + Some("0") => false, + _ => { + error!( + "Fatal Python error: config_init_thread_inherit_context: \ + -X thread_inherit_context=n: n is missing or invalid\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; + } + _ => {} + } + (name, value.map(str::to_owned)) + }); + settings.xoptions.extend(xopts); + + // Resolve utf8_mode if not explicitly set by PYTHONUTF8 or -X utf8. + // Default to UTF-8 mode since RustPython's locale encoding detection + // is incomplete. Users can set PYTHONUTF8=0 or -X utf8=0 to disable. + if settings.utf8_mode < 0 { + settings.utf8_mode = 1; + } + + settings.warn_default_encoding = + settings.warn_default_encoding || env_bool("PYTHONWARNDEFAULTENCODING"); + settings.faulthandler = settings.faulthandler || env_bool("PYTHONFAULTHANDLER"); + if env_bool("PYTHONNODEBUGRANGES") { + settings.code_debug_ranges = false; + } + if let Some(val) = get_env("PYTHON_THREAD_INHERIT_CONTEXT") { + settings.thread_inherit_context = match val.to_str() { + Some("1") => true, + Some("0") => false, + _ => { + error!( + "Fatal Python error: config_init_thread_inherit_context: \ + PYTHON_THREAD_INHERIT_CONTEXT=N: N is missing or invalid\n\ + Python runtime state: preinitialized" + ); + std::process::exit(1); + } + }; } - settings.dev_mode = dev_mode; - if warn_default_encoding - || (!ignore_environment && env::var_os("PYTHONWARNDEFAULTENCODING").is_some()) + + // Parse PYTHONIOENCODING=encoding[:errors] + if let Some(val) = get_env("PYTHONIOENCODING") + && let Some(val_str) = val.to_str() + && !val_str.is_empty() { - settings.warn_default_encoding = true; + if let Some((enc, err)) = val_str.split_once(':') { + if !enc.is_empty() { + settings.stdio_encoding = Some(enc.to_owned()); + } + if !err.is_empty() { + settings.stdio_errors = Some(err.to_owned()); + } + } else { + settings.stdio_encoding = Some(val_str.to_owned()); + } } - if dev_mode { - settings.warnopts.push("default".to_owned()) + if settings.dev_mode { + settings.warnoptions.push("default".to_owned()); + settings.faulthandler = true; } if settings.bytes_warning > 0 { let warn = if settings.bytes_warning > 1 { @@ -298,79 +393,53 @@ fn settings_from(matches: &ArgMatches) -> (Settings, RunMode) { } else { "default::BytesWarning" }; - settings.warnopts.push(warn.to_owned()); + settings.warnoptions.push(warn.to_owned()); } - if let Some(warnings) = matches.values_of("warning-control") { - settings.warnopts.extend(warnings.map(ToOwned::to_owned)); + if let Some(val) = get_env("PYTHONWARNINGS") + && let Some(val_str) = val.to_str() + && !val_str.is_empty() + { + for warning in val_str.split(',') { + let warning = warning.trim(); + if !warning.is_empty() { + settings.warnoptions.push(warning.to_owned()); + } + } } + settings.warnoptions.extend(args.warning_control); - let (mode, argv) = if let Some(mut cmd) = matches.values_of("c") { - let command = cmd.next().expect("clap ensure this exists"); - let argv = std::iter::once("-c".to_owned()) - .chain(cmd.map(ToOwned::to_owned)) - .collect(); - (RunMode::Command(command.to_owned()), argv) - } else if let Some(mut cmd) = matches.values_of("m") { - let module = cmd.next().expect("clap ensure this exists"); - let argv = std::iter::once("PLACEHOLDER".to_owned()) - .chain(cmd.map(ToOwned::to_owned)) - .collect(); - (RunMode::Module(module.to_owned()), argv) - } else if let Some(get_pip_args) = matches.values_of("install_pip") { - settings.isolated = true; - let mut args: Vec<_> = get_pip_args.map(ToOwned::to_owned).collect(); - if args.is_empty() { - args.push("ensurepip".to_owned()); - args.push("--upgrade".to_owned()); - args.push("--default-pip".to_owned()); + settings.hash_seed = match (!args.random_hash_seed) + .then(|| get_env("PYTHONHASHSEED")) + .flatten() + { + Some(s) if s == "random" || s.is_empty() => None, + Some(s) => { + let seed = s.parse_with(|s| { + s.parse::<u32>().map_err(|_| { + "Fatal Python init error: PYTHONHASHSEED must be \ + \"random\" or an integer in range [0; 4294967295]" + }) + })?; + Some(seed) } - let installer = args[0].clone(); - let mode = match installer.as_str() { - "ensurepip" | "get-pip" => RunMode::InstallPip(installer), - _ => panic!("--install-pip takes ensurepip or get-pip as first argument"), - }; - (mode, args) - } else if let Some(argv) = matches.values_of("script") { - let argv: Vec<_> = argv.map(ToOwned::to_owned).collect(); - let script = argv[0].clone(); - ( - RunMode::ScriptInteractive(Some(script), matches.is_present("inspect")), - argv, - ) - } else { - (RunMode::ScriptInteractive(None, true), vec!["".to_owned()]) + None => None, }; - let hash_seed = match env::var("PYTHONHASHSEED") { - Ok(s) if s == "random" => Some(None), - Ok(s) => s.parse::<u32>().ok().map(Some), - Err(_) => Some(None), - }; - settings.hash_seed = hash_seed.unwrap_or_else(|| { - error!("Fatal Python init error: PYTHONHASHSEED must be \"random\" or an integer in range [0; 4294967295]"); - // TODO: Need to change to ExitCode or Termination - std::process::exit(1) - }); - settings.argv = argv; - (settings, mode) -} + #[cfg(feature = "flame-it")] + { + settings.profile_output = args.profile_output; + settings.profile_format = args.profile_format; + } -/// Get environment variable and turn it into integer. -fn get_env_var_value(name: &str) -> Result<u8, std::env::VarError> { - env::var(name).map(|value| { - if let Ok(value) = u8::from_str(&value) { - value - } else { - 1 - } - }) + Ok((settings, mode)) } /// Helper function to retrieve a sequence of paths from an environment variable. fn get_paths(env_variable_name: &str) -> impl Iterator<Item = String> + '_ { env::var_os(env_variable_name) + .filter(|v| !v.is_empty()) .into_iter() .flat_map(move |paths| { split_paths(&paths) @@ -389,8 +458,10 @@ pub(crate) use env::split_paths; pub(crate) fn split_paths<T: AsRef<std::ffi::OsStr> + ?Sized>( s: &T, ) -> impl Iterator<Item = std::path::PathBuf> + '_ { - use std::os::wasi::ffi::OsStrExt; - let s = s.as_ref().as_bytes(); - s.split(|b| *b == b':') - .map(|x| std::ffi::OsStr::from_bytes(x).to_owned().into()) + let s = s.as_ref().as_encoded_bytes(); + s.split(|b| *b == b':').map(|x| { + unsafe { std::ffi::OsStr::from_encoded_bytes_unchecked(x) } + .to_owned() + .into() + }) } diff --git a/src/shell.rs b/src/shell.rs index 75d980b44c9..6c75e94572c 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,18 +1,22 @@ mod helper; -use rustpython_parser::{lexer::LexicalErrorType, ParseErrorType, Tok}; +use rustpython_compiler::{ + CompileError, ParseError, parser::InterpolatedStringErrorType, parser::LexicalErrorType, + parser::ParseErrorType, +}; use rustpython_vm::{ + AsObject, PyResult, VirtualMachine, builtins::PyBaseExceptionRef, - compiler::{self, CompileError, CompileErrorType}, + compiler::{self}, readline::{Readline, ReadlineResult}, scope::Scope, - version, AsObject, PyResult, VirtualMachine, }; enum ShellExecResult { Ok, PyErr(PyBaseExceptionRef), - Continue, + ContinueBlock, + ContinueLine, } fn shell_exec( @@ -20,11 +24,17 @@ fn shell_exec( source: &str, scope: Scope, empty_line_given: bool, - continuing: bool, + continuing_block: bool, ) -> ShellExecResult { + // compiling expects only UNIX style line endings, and will replace windows line endings + // internally. Since we might need to analyze the source to determine if an error could be + // resolved by future input, we need the location from the error to match the source code that + // was actually compiled. + #[cfg(windows)] + let source = &source.replace("\r\n", "\n"); match vm.compile(source, compiler::Mode::Single, "<stdin>".to_owned()) { Ok(code) => { - if empty_line_given || !continuing { + if empty_line_given || !continuing_block { // We want to execute the full code match vm.run_code_obj(code, scope) { Ok(_val) => ShellExecResult::Ok, @@ -35,29 +45,51 @@ fn shell_exec( ShellExecResult::Ok } } - Err(CompileError { - error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)), + Err(CompileError::Parse(ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::Eof), .. - }) - | Err(CompileError { - error: CompileErrorType::Parse(ParseErrorType::Eof), + })) => ShellExecResult::ContinueLine, + Err(CompileError::Parse(ParseError { + error: + ParseErrorType::Lexical(LexicalErrorType::FStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString, + )), .. - }) => ShellExecResult::Continue, + })) => ShellExecResult::ContinueLine, Err(err) => { + // Check if the error is from an unclosed triple quoted string (which should always + // continue) + if let CompileError::Parse(ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError), + raw_location, + .. + }) = err + { + let loc = raw_location.start().to_usize(); + let mut iter = source.chars(); + if let Some(quote) = iter.nth(loc) + && iter.next() == Some(quote) + && iter.next() == Some(quote) + { + return ShellExecResult::ContinueLine; + } + }; + // bad_error == true if we are handling an error that should be thrown even if we are continuing // if its an indentation error, set to true if we are continuing and the error is on column 0, // since indentations errors on columns other than 0 should be ignored. // if its an unrecognized token for dedent, set to false - let bad_error = match err.error { - CompileErrorType::Parse(ref p) => { - if matches!( - p, - ParseErrorType::Lexical(LexicalErrorType::IndentationError) - ) { - continuing && err.location.is_some() - } else { - !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _)) + let bad_error = match err { + CompileError::Parse(ref p) => { + match &p.error { + ParseErrorType::Lexical(LexicalErrorType::IndentationError) => { + continuing_block + } // && p.location.is_some() + ParseErrorType::OtherError(msg) => { + !msg.starts_with("Expected an indented block") + } + _ => true, // !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _)) } } _ => true, // It is a bad error for everything else @@ -67,12 +99,13 @@ fn shell_exec( if empty_line_given || bad_error { ShellExecResult::PyErr(vm.new_syntax_error(&err, Some(source))) } else { - ShellExecResult::Continue + ShellExecResult::ContinueBlock } } } } +/// Enter a repl loop pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { let mut repl = Readline::new(helper::ShellHelper::new(vm, scope.globals.clone())); let mut full_input = String::new(); @@ -91,30 +124,33 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { println!("No previous history."); } - let mut continuing = false; - - println!( - "RustPython {}.{}.{}", - version::MAJOR, - version::MINOR, - version::MICRO, - ); - - println!("Type \"help\", \"copyright\", \"credits\" or \"license\" for more information."); + // We might either be waiting to know if a block is complete, or waiting to know if a multiline + // statement is complete. In the former case, we need to ensure that we read one extra new line + // to know that the block is complete. In the latter, we can execute as soon as the statement is + // valid. + let mut continuing_block = false; + let mut continuing_line = false; loop { - let prompt_name = if continuing { "ps2" } else { "ps1" }; + let prompt_name = if continuing_block || continuing_line { + "ps2" + } else { + "ps1" + }; let prompt = vm .sys_module .get_attr(prompt_name, vm) .and_then(|prompt| prompt.str(vm)); let prompt = match prompt { - Ok(ref s) => s.as_str(), + Ok(ref s) => s.expect_str(), Err(_) => "", }; + + continuing_line = false; let result = match repl.readline(prompt) { ReadlineResult::Line(line) => { - debug!("You entered {:?}", line); + #[cfg(debug_assertions)] + debug!("You entered {line:?}"); repl.add_history_entry(line.trim_end()).unwrap(); @@ -127,39 +163,44 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { } full_input.push('\n'); - match shell_exec(vm, &full_input, scope.clone(), empty_line_given, continuing) { + match shell_exec( + vm, + &full_input, + scope.clone(), + empty_line_given, + continuing_block, + ) { ShellExecResult::Ok => { - if continuing { + if continuing_block { if empty_line_given { - // We should be exiting continue mode - continuing = false; + // We should exit continue mode since the block successfully executed + continuing_block = false; full_input.clear(); - Ok(()) - } else { - // We should stay in continue mode - continuing = true; - Ok(()) } } else { // We aren't in continue mode so proceed normally - continuing = false; full_input.clear(); - Ok(()) } + Ok(()) } - ShellExecResult::Continue => { - continuing = true; + // Continue, but don't change the mode + ShellExecResult::ContinueLine => { + continuing_line = true; + Ok(()) + } + ShellExecResult::ContinueBlock => { + continuing_block = true; Ok(()) } ShellExecResult::PyErr(err) => { - continuing = false; + continuing_block = false; full_input.clear(); Err(err) } } } ReadlineResult::Interrupt => { - continuing = false; + continuing_block = false; full_input.clear(); let keyboard_interrupt = vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned()); @@ -168,6 +209,15 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { ReadlineResult::Eof => { break; } + #[cfg(unix)] + ReadlineResult::OsError(num) => { + let os_error = vm.new_exception_msg( + vm.ctx.exceptions.os_error.to_owned(), + format!("{num:?}").into(), + ); + vm.print_exception(os_error); + break; + } ReadlineResult::Other(err) => { eprintln!("Readline error: {err:?}"); break; diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 34691e7995a..944e936397d 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,8 +1,9 @@ #![cfg_attr(target_arch = "wasm32", allow(dead_code))] use rustpython_vm::{ + AsObject, PyResult, TryFromObject, VirtualMachine, builtins::{PyDictRef, PyStrRef}, function::ArgIterable, - identifier, AsObject, PyResult, TryFromObject, VirtualMachine, + identifier, }; pub struct ShellHelper<'vm> { @@ -22,7 +23,7 @@ fn split_idents_on_dot(line: &str) -> Option<(usize, Vec<String>)> { match c { '.' => { // check for a double dot - if i != 0 && words.last().map_or(false, |s| s.is_empty()) { + if i != 0 && words.last().is_some_and(|s| s.is_empty()) { return None; } reverse_string(words.last_mut().unwrap()); @@ -53,7 +54,7 @@ fn split_idents_on_dot(line: &str) -> Option<(usize, Vec<String>)> { } impl<'vm> ShellHelper<'vm> { - pub fn new(vm: &'vm VirtualMachine, globals: PyDictRef) -> Self { + pub const fn new(vm: &'vm VirtualMachine, globals: PyDictRef) -> Self { ShellHelper { vm, globals } } @@ -107,7 +108,7 @@ impl<'vm> ShellHelper<'vm> { .filter(|res| { res.as_ref() .ok() - .map_or(true, |s| s.as_str().starts_with(word_start)) + .is_none_or(|s| s.as_bytes().starts_with(word_start.as_bytes())) }) .collect::<Result<Vec<_>, _>>() .ok()?; @@ -119,7 +120,7 @@ impl<'vm> ShellHelper<'vm> { // only the completions that don't start with a '_' let no_underscore = all_completions .iter() - .filter(|&s| !s.as_str().starts_with('_')) + .filter(|&s| !s.as_bytes().starts_with(b"_")) .cloned() .collect::<Vec<_>>(); @@ -133,13 +134,13 @@ impl<'vm> ShellHelper<'vm> { }; // sort the completions alphabetically - completions.sort_by(|a, b| std::cmp::Ord::cmp(a.as_str(), b.as_str())); + completions.sort_by(|a, b| a.as_wtf8().cmp(b.as_wtf8())); Some(( startpos, completions .into_iter() - .map(|s| s.as_str().to_owned()) + .map(|s| s.expect_str().to_owned()) .collect(), )) } diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml deleted file mode 100644 index 7c394804b18..00000000000 --- a/stdlib/Cargo.toml +++ /dev/null @@ -1,128 +0,0 @@ -[package] -name = "rustpython-stdlib" -version = "0.3.0" -description = "RustPython standard libraries in Rust." -authors = ["RustPython Team"] -repository = "https://github.com/RustPython/RustPython" -license = "MIT" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[features] -threading = ["rustpython-common/threading", "rustpython-vm/threading"] -zlib = ["libz-sys", "flate2/zlib"] -bz2 = ["bzip2"] -ssl = ["openssl", "openssl-sys", "foreign-types-shared"] -ssl-vendor = ["ssl", "openssl/vendored", "openssl-probe"] - -[dependencies] -# rustpython crates -rustpython-derive = { workspace = true } -rustpython-vm = { workspace = true } -rustpython-common = { workspace = true } - -ahash = { workspace = true } -ascii = { workspace = true } -cfg-if = { workspace = true } -crossbeam-utils = { workspace = true } -hex = { workspace = true } -itertools = { workspace = true } -libc = { workspace = true } -nix = { workspace = true } -num-complex = { workspace = true } -malachite-bigint = { workspace = true } -num-integer = { workspace = true } -num-traits = { workspace = true } -num_enum = { workspace = true } -once_cell = { workspace = true } -parking_lot = { workspace = true } - -memchr = "2.4.1" -base64 = "0.13.0" -csv-core = "0.1.10" -dyn-clone = "1.0.10" -libz-sys = { version = "1.1.5", optional = true } -puruspe = "0.2.0" -xml-rs = "0.8.14" - -# random -rand = { workspace = true } -rand_core = "0.6.3" -mt19937 = "2.0.1" - -# Crypto: -digest = "0.10.3" -md-5 = "0.10.1" -sha-1 = "0.10.0" -sha2 = "0.10.2" -sha3 = "0.10.1" -blake2 = "0.10.4" - -## unicode stuff -unicode_names2 = { workspace = true } -# TODO: use unic for this; needed for title case: -# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 -unicode-casing = "0.1.0" -# update version all at the same time -unic-char-property = "0.9.0" -unic-normal = "0.9.0" -unic-ucd-bidi = "0.9.0" -unic-ucd-category = "0.9.0" -unic-ucd-age = "0.9.0" -unic-ucd-ident = "0.9.0" -ucd = "0.1.1" - -# compression -adler32 = "1.2.0" -crc32fast = "1.3.2" -flate2 = "1.0.23" -bzip2 = { version = "0.4", optional = true } - -# uuid -[target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies] -mac_address = "1.1.3" -uuid = { version = "1.1.2", features = ["v1", "fast-rng", "macro-diagnostics"] } - -# mmap -[target.'cfg(all(unix, not(target_arch = "wasm32")))'.dependencies] -memmap2 = "0.5.4" -page_size = "0.4" - -[target.'cfg(all(unix, not(target_os = "redox"), not(target_os = "ios")))'.dependencies] -termios = "0.3.3" - -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -gethostname = "0.2.3" -socket2 = { version = "0.4.4", features = ["all"] } -dns-lookup = "1.0.8" -openssl = { version = "0.10.62", optional = true } -openssl-sys = { version = "0.9.80", optional = true } -openssl-probe = { version = "0.1.5", optional = true } -foreign-types-shared = { version = "0.1.1", optional = true } - -[target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies] -libsqlite3-sys = { version = "0.25", features = ["min_sqlite_version_3_7_16", "bundled"] } - -[target.'cfg(windows)'.dependencies] -paste = { workspace = true } -schannel = { workspace = true } -widestring = { workspace = true } - -[target.'cfg(windows)'.dependencies.winapi] -version = "0.3.9" -features = [ - "winsock2", "ifdef", "netioapi", -] - -[target.'cfg(windows)'.dependencies.windows-sys] -version = "0.52.0" -features = [ - "Win32_Networking_WinSock", - "Win32_NetworkManagement_IpHelper", - "Win32_NetworkManagement_Ndis", - "Win32_Security_Cryptography", -] - -[target.'cfg(target_os = "macos")'.dependencies] -system-configuration = "0.5.0" diff --git a/stdlib/build.rs b/stdlib/build.rs deleted file mode 100644 index baaa99c4c84..00000000000 --- a/stdlib/build.rs +++ /dev/null @@ -1,28 +0,0 @@ -fn main() { - #[allow(clippy::unusual_byte_groupings)] - if let Ok(v) = std::env::var("DEP_OPENSSL_VERSION_NUMBER") { - println!("cargo:rustc-env=OPENSSL_API_VERSION={v}"); - // cfg setup from openssl crate's build script - let version = u64::from_str_radix(&v, 16).unwrap(); - if version >= 0x1_00_01_00_0 { - println!("cargo:rustc-cfg=ossl101"); - } - if version >= 0x1_00_02_00_0 { - println!("cargo:rustc-cfg=ossl102"); - } - if version >= 0x1_01_00_00_0 { - println!("cargo:rustc-cfg=ossl110"); - } - if version >= 0x1_01_00_07_0 { - println!("cargo:rustc-cfg=ossl110g"); - } - if version >= 0x1_01_01_00_0 { - println!("cargo:rustc-cfg=ossl111"); - } - } - if let Ok(v) = std::env::var("DEP_OPENSSL_CONF") { - for conf in v.split(',') { - println!("cargo:rustc-cfg=osslconf=\"{conf}\""); - } - } -} diff --git a/stdlib/src/array.rs b/stdlib/src/array.rs deleted file mode 100644 index a0beb5d1ee8..00000000000 --- a/stdlib/src/array.rs +++ /dev/null @@ -1,1673 +0,0 @@ -// spell-checker:ignore typecode tofile tolist fromfile - -use rustpython_vm::{builtins::PyModule, PyRef, VirtualMachine}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = array::make_module(vm); - - let array = module - .get_attr("array", vm) - .expect("Expect array has array type."); - - let collections_abc = vm - .import("collections.abc", None, 0) - .expect("Expect collections exist."); - let abc = collections_abc - .get_attr("abc", vm) - .expect("Expect collections has abc submodule."); - let mutable_sequence = abc - .get_attr("MutableSequence", vm) - .expect("Expect collections.abc has MutableSequence type."); - - let register = &mutable_sequence - .get_attr("register", vm) - .expect("Expect collections.abc.MutableSequence has register method."); - register - .call((array,), vm) - .expect("Expect collections.abc.MutableSequence.register(array.array) not fail."); - - module -} - -#[pymodule(name = "array")] -mod array { - use crate::{ - common::{ - atomic::{self, AtomicUsize}, - lock::{ - PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutex, PyRwLock, - PyRwLockReadGuard, PyRwLockWriteGuard, - }, - str::wchar_t, - }, - vm::{ - atomic_func, - builtins::{ - PositionIterInternal, PyByteArray, PyBytes, PyBytesRef, PyDictRef, PyFloat, PyInt, - PyList, PyListRef, PyStr, PyStrRef, PyTupleRef, PyTypeRef, - }, - class_or_notimplemented, - convert::{ToPyObject, ToPyResult, TryFromBorrowedObject, TryFromObject}, - function::{ - ArgBytesLike, ArgIntoFloat, ArgIterable, KwArgs, OptionalArg, PyComparisonValue, - }, - protocol::{ - BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, - PyMappingMethods, PySequenceMethods, - }, - sequence::{OptionalRangeArgs, SequenceExt, SequenceMutExt}, - sliceable::{ - SaturatedSlice, SequenceIndex, SequenceIndexOp, SliceableSequenceMutOp, - SliceableSequenceOp, - }, - types::{ - AsBuffer, AsMapping, AsSequence, Comparable, Constructor, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, - }, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - }, - }; - use itertools::Itertools; - use num_traits::ToPrimitive; - use std::{cmp::Ordering, fmt, os::raw}; - - macro_rules! def_array_enum { - ($(($n:ident, $t:ty, $c:literal, $scode:literal)),*$(,)?) => { - #[derive(Debug, Clone)] - pub enum ArrayContentType { - $($n(Vec<$t>),)* - } - - impl ArrayContentType { - fn from_char(c: char) -> Result<Self, String> { - match c { - $($c => Ok(ArrayContentType::$n(Vec::new())),)* - _ => Err( - "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d)".into() - ), - } - } - - fn typecode(&self) -> char { - match self { - $(ArrayContentType::$n(_) => $c,)* - } - } - - fn typecode_str(&self) -> &'static str { - match self { - $(ArrayContentType::$n(_) => $scode,)* - } - } - - fn itemsize_of_typecode(c: char) -> Option<usize> { - match c { - $($c => Some(std::mem::size_of::<$t>()),)* - _ => None, - } - } - - fn itemsize(&self) -> usize { - match self { - $(ArrayContentType::$n(_) => std::mem::size_of::<$t>(),)* - } - } - - fn addr(&self) -> usize { - match self { - $(ArrayContentType::$n(v) => v.as_ptr() as usize,)* - } - } - - fn len(&self) -> usize { - match self { - $(ArrayContentType::$n(v) => v.len(),)* - } - } - - fn reserve(&mut self, len: usize) { - match self { - $(ArrayContentType::$n(v) => v.reserve(len),)* - } - } - - fn push(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => { - let val = <$t>::try_into_from_object(vm, obj)?; - v.push(val); - })* - } - Ok(()) - } - - fn pop(&mut self, i: isize, vm: &VirtualMachine) -> PyResult { - match self { - $(ArrayContentType::$n(v) => { - let i = v.wrap_index(i).ok_or_else(|| { - vm.new_index_error("pop index out of range".to_owned()) - })?; - v.remove(i).to_pyresult(vm) - })* - } - } - - fn insert( - &mut self, - i: isize, - obj: PyObjectRef, - vm: &VirtualMachine - ) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => { - let val = <$t>::try_into_from_object(vm, obj)?; - v.insert(i.saturated_at(v.len()), val); - })* - } - Ok(()) - } - - fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> usize { - match self { - $(ArrayContentType::$n(v) => { - if let Ok(val) = <$t>::try_into_from_object(vm, obj) { - v.iter().filter(|&&a| a == val).count() - } else { - 0 - } - })* - } - } - - fn remove(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()>{ - match self { - $(ArrayContentType::$n(v) => { - if let Ok(val) = <$t>::try_into_from_object(vm, obj) { - if let Some(pos) = v.iter().position(|&a| a == val) { - v.remove(pos); - return Ok(()); - } - } - Err(vm.new_value_error("array.remove(x): x not in array".to_owned())) - })* - } - } - - fn frombytes_move(&mut self, b: Vec<u8>) { - match self { - $(ArrayContentType::$n(v) => { - if v.is_empty() { - // safe because every configuration of bytes for the types we - // support are valid - let b = std::mem::ManuallyDrop::new(b); - let ptr = b.as_ptr() as *mut $t; - let len = b.len() / std::mem::size_of::<$t>(); - let capacity = b.capacity() / std::mem::size_of::<$t>(); - *v = unsafe { Vec::from_raw_parts(ptr, len, capacity) }; - } else { - self.frombytes(&b); - } - })* - } - } - - fn frombytes(&mut self, b: &[u8]) { - match self { - $(ArrayContentType::$n(v) => { - // safe because every configuration of bytes for the types we - // support are valid - if b.len() > 0 { - let ptr = b.as_ptr() as *const $t; - let ptr_len = b.len() / std::mem::size_of::<$t>(); - let slice = unsafe { std::slice::from_raw_parts(ptr, ptr_len) }; - v.extend_from_slice(slice); - } - })* - } - } - - fn fromlist(&mut self, list: &PyList, vm: &VirtualMachine) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => { - // convert list before modify self - let mut list: Vec<$t> = list - .borrow_vec() - .iter() - .cloned() - .map(|value| <$t>::try_into_from_object(vm, value)) - .try_collect()?; - v.append(&mut list); - Ok(()) - })* - } - } - - fn get_bytes(&self) -> &[u8] { - match self { - $(ArrayContentType::$n(v) => { - // safe because we're just reading memory as bytes - let ptr = v.as_ptr() as *const u8; - let ptr_len = v.len() * std::mem::size_of::<$t>(); - unsafe { std::slice::from_raw_parts(ptr, ptr_len) } - })* - } - } - - fn get_bytes_mut(&mut self) -> &mut [u8] { - match self { - $(ArrayContentType::$n(v) => { - // safe because we're just reading memory as bytes - let ptr = v.as_ptr() as *mut u8; - let ptr_len = v.len() * std::mem::size_of::<$t>(); - unsafe { std::slice::from_raw_parts_mut(ptr, ptr_len) } - })* - } - } - - fn index( - &self, - obj: PyObjectRef, - start: usize, - stop: usize, - vm: &VirtualMachine - ) -> PyResult<usize> { - match self { - $(ArrayContentType::$n(v) => { - if let Ok(val) = <$t>::try_into_from_object(vm, obj) { - if let Some(pos) = v.iter().take(stop as _).skip(start as _).position(|&elem| elem == val) { - return Ok(pos + start); - } - } - Err(vm.new_value_error("array.index(x): x not in array".to_owned())) - })* - } - } - - fn reverse(&mut self) { - match self { - $(ArrayContentType::$n(v) => v.reverse(),)* - } - } - - fn get( - &self, - i: usize, - vm: &VirtualMachine - ) -> Option<PyResult> { - match self { - $(ArrayContentType::$n(v) => { - v.get(i).map(|x| x.to_pyresult(vm)) - })* - } - } - - fn getitem_by_index(&self, i: isize, vm: &VirtualMachine) -> PyResult { - match self { - $(ArrayContentType::$n(v) => { - v.getitem_by_index(vm, i).map(|x| x.to_pyresult(vm))? - })* - } - } - - fn getitem_by_slice(&self, slice: SaturatedSlice, vm: &VirtualMachine) -> PyResult { - match self { - $(ArrayContentType::$n(v) => { - let r = v.getitem_by_slice(vm, slice)?; - let array = PyArray::from(ArrayContentType::$n(r)); - array.to_pyresult(vm) - })* - } - } - - fn setitem_by_index( - &mut self, - i: isize, - value: PyObjectRef, - vm: &VirtualMachine - ) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => { - let value = <$t>::try_into_from_object(vm, value)?; - v.setitem_by_index(vm, i, value) - })* - } - } - - fn setitem_by_slice( - &mut self, - slice: SaturatedSlice, - items: &ArrayContentType, - vm: &VirtualMachine - ) -> PyResult<()> { - match self { - $(Self::$n(elements) => if let ArrayContentType::$n(items) = items { - elements.setitem_by_slice(vm, slice, items) - } else { - Err(vm.new_type_error( - "bad argument type for built-in operation".to_owned() - )) - },)* - } - } - - fn setitem_by_slice_no_resize( - &mut self, - slice: SaturatedSlice, - items: &ArrayContentType, - vm: &VirtualMachine - ) -> PyResult<()> { - match self { - $(Self::$n(elements) => if let ArrayContentType::$n(items) = items { - elements.setitem_by_slice_no_resize(vm, slice, items) - } else { - Err(vm.new_type_error( - "bad argument type for built-in operation".to_owned() - )) - },)* - } - } - - fn delitem_by_index(&mut self, i: isize, vm: &VirtualMachine) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => { - v.delitem_by_index(vm, i) - })* - } - } - - fn delitem_by_slice(&mut self, slice: SaturatedSlice, vm: &VirtualMachine) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => { - v.delitem_by_slice(vm, slice) - })* - } - } - - fn add(&self, other: &ArrayContentType, vm: &VirtualMachine) -> PyResult<Self> { - match self { - $(ArrayContentType::$n(v) => if let ArrayContentType::$n(other) = other { - let elements = v.iter().chain(other.iter()).cloned().collect(); - Ok(ArrayContentType::$n(elements)) - } else { - Err(vm.new_type_error( - "bad argument type for built-in operation".to_owned() - )) - },)* - } - } - - fn iadd(&mut self, other: &ArrayContentType, vm: &VirtualMachine) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => if let ArrayContentType::$n(other) = other { - v.extend(other); - Ok(()) - } else { - Err(vm.new_type_error( - "can only extend with array of same kind".to_owned() - )) - },)* - } - } - - fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult<Self> { - match self { - $(ArrayContentType::$n(v) => { - // MemoryError instead Overflow Error, hard to says it is right - // but it is how cpython doing right now - let elements = v.mul(vm, value).map_err(|_| vm.new_memory_error("".to_owned()))?; - Ok(ArrayContentType::$n(elements)) - })* - } - } - - fn imul(&mut self, value: isize, vm: &VirtualMachine) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => { - // MemoryError instead Overflow Error, hard to says it is right - // but it is how cpython doing right now - v.imul(vm, value).map_err(|_| vm.new_memory_error("".to_owned())) - })* - } - } - - fn byteswap(&mut self) { - match self { - $(ArrayContentType::$n(v) => { - for element in v.iter_mut() { - let x = element.byteswap(); - *element = x; - } - })* - } - } - - fn repr(&self, class_name: &str, _vm: &VirtualMachine) -> PyResult<String> { - // we don't need ReprGuard here - let s = match self { - $(ArrayContentType::$n(v) => { - if v.is_empty() { - format!("{}('{}')", class_name, $c) - } else { - format!("{}('{}', [{}])", class_name, $c, v.iter().format(", ")) - } - })* - }; - Ok(s) - } - - fn iter<'a, 'vm: 'a>( - &'a self, - vm: &'vm VirtualMachine - ) -> impl Iterator<Item = PyResult> + 'a { - (0..self.len()).map(move |i| self.get(i, vm).unwrap()) - } - - fn cmp(&self, other: &ArrayContentType) -> Result<Option<Ordering>, ()> { - match self { - $(ArrayContentType::$n(v) => { - if let ArrayContentType::$n(other) = other { - Ok(PartialOrd::partial_cmp(v, other)) - } else { - Err(()) - } - })* - } - } - - fn get_objects(&self, vm: &VirtualMachine) -> Vec<PyObjectRef> { - match self { - $(ArrayContentType::$n(v) => { - v.iter().map(|&x| x.to_object(vm)).collect() - })* - } - } - } - }; - } - - def_array_enum!( - (SignedByte, i8, 'b', "b"), - (UnsignedByte, u8, 'B', "B"), - (PyUnicode, WideChar, 'u', "u"), - (SignedShort, raw::c_short, 'h', "h"), - (UnsignedShort, raw::c_ushort, 'H', "H"), - (SignedInt, raw::c_int, 'i', "i"), - (UnsignedInt, raw::c_uint, 'I', "I"), - (SignedLong, raw::c_long, 'l', "l"), - (UnsignedLong, raw::c_ulong, 'L', "L"), - (SignedLongLong, raw::c_longlong, 'q', "q"), - (UnsignedLongLong, raw::c_ulonglong, 'Q', "Q"), - (Float, f32, 'f', "f"), - (Double, f64, 'd', "d"), - ); - - #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] - pub struct WideChar(wchar_t); - - trait ArrayElement: Sized { - fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self>; - fn byteswap(self) -> Self; - fn to_object(self, vm: &VirtualMachine) -> PyObjectRef; - } - - macro_rules! impl_int_element { - ($($t:ty,)*) => {$( - impl ArrayElement for $t { - fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - obj.try_index(vm)?.try_to_primitive(vm) - } - fn byteswap(self) -> Self { - <$t>::swap_bytes(self) - } - fn to_object(self, vm: &VirtualMachine) -> PyObjectRef { - self.to_pyobject(vm) - } - } - )*}; - } - - macro_rules! impl_float_element { - ($(($t:ty, $f_from:path, $f_swap:path, $f_to:path),)*) => {$( - impl ArrayElement for $t { - fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - $f_from(vm, obj) - } - fn byteswap(self) -> Self { - $f_swap(self) - } - fn to_object(self, vm: &VirtualMachine) -> PyObjectRef { - $f_to(self).into_pyobject(vm) - } - } - )*}; - } - - impl_int_element!(i8, u8, i16, u16, i32, u32, i64, u64,); - impl_float_element!( - ( - f32, - f32_try_into_from_object, - f32_swap_bytes, - pyfloat_from_f32 - ), - (f64, f64_try_into_from_object, f64_swap_bytes, PyFloat::from), - ); - - fn f32_swap_bytes(x: f32) -> f32 { - f32::from_bits(x.to_bits().swap_bytes()) - } - - fn f64_swap_bytes(x: f64) -> f64 { - f64::from_bits(x.to_bits().swap_bytes()) - } - - fn f32_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<f32> { - ArgIntoFloat::try_from_object(vm, obj).map(|x| *x as f32) - } - - fn f64_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<f64> { - ArgIntoFloat::try_from_object(vm, obj).map(Into::into) - } - - fn pyfloat_from_f32(value: f32) -> PyFloat { - PyFloat::from(value as f64) - } - - impl ArrayElement for WideChar { - fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - PyStrRef::try_from_object(vm, obj)? - .as_str() - .chars() - .exactly_one() - .map(|ch| Self(ch as _)) - .map_err(|_| vm.new_type_error("array item must be unicode character".into())) - } - fn byteswap(self) -> Self { - Self(self.0.swap_bytes()) - } - fn to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - unreachable!() - } - } - - fn u32_to_char(ch: u32) -> Result<char, String> { - if ch > 0x10ffff { - return Err(format!( - "character U+{ch:4x} is not in range [U+0000; U+10ffff]" - )); - }; - char::from_u32(ch).ok_or_else(|| { - format!( - "'utf-8' codec can't encode character '\\u{ch:x}' \ - in position 0: surrogates not allowed" - ) - }) - } - - impl TryFrom<WideChar> for char { - type Error = String; - - fn try_from(ch: WideChar) -> Result<Self, Self::Error> { - // safe because every configuration of bytes for the types we support are valid - u32_to_char(ch.0 as u32) - } - } - - impl ToPyResult for WideChar { - fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { - Ok( - String::from(char::try_from(self).map_err(|e| vm.new_unicode_encode_error(e))?) - .to_pyobject(vm), - ) - } - } - - impl fmt::Display for WideChar { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - unreachable!("`repr(array('u'))` calls `PyStr::repr`") - } - } - - #[pyattr] - #[pyattr(name = "ArrayType")] - #[pyclass(name = "array")] - #[derive(Debug, PyPayload)] - pub struct PyArray { - array: PyRwLock<ArrayContentType>, - exports: AtomicUsize, - } - - pub type PyArrayRef = PyRef<PyArray>; - - impl From<ArrayContentType> for PyArray { - fn from(array: ArrayContentType) -> Self { - PyArray { - array: PyRwLock::new(array), - exports: AtomicUsize::new(0), - } - } - } - - #[derive(FromArgs)] - pub struct ArrayNewArgs { - #[pyarg(positional)] - spec: PyStrRef, - #[pyarg(positional, optional)] - init: OptionalArg<PyObjectRef>, - } - - impl Constructor for PyArray { - type Args = (ArrayNewArgs, KwArgs); - - fn py_new( - cls: PyTypeRef, - (ArrayNewArgs { spec, init }, kwargs): Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let spec = spec.as_str().chars().exactly_one().map_err(|_| { - vm.new_type_error( - "array() argument 1 must be a unicode character, not str".to_owned(), - ) - })?; - - if cls.is(PyArray::class(&vm.ctx)) && !kwargs.is_empty() { - return Err( - vm.new_type_error("array.array() takes no keyword arguments".to_owned()) - ); - } - - let mut array = - ArrayContentType::from_char(spec).map_err(|err| vm.new_value_error(err))?; - - if let OptionalArg::Present(init) = init { - if let Some(init) = init.payload::<PyArray>() { - match (spec, init.read().typecode()) { - (spec, ch) if spec == ch => array.frombytes(&init.get_bytes()), - (spec, 'u') => { - return Err(vm.new_type_error(format!( - "cannot use a unicode array to initialize an array with typecode '{spec}'" - ))) - } - _ => { - for obj in init.read().iter(vm) { - array.push(obj?, vm)?; - } - } - } - } else if let Some(utf8) = init.payload::<PyStr>() { - if spec == 'u' { - let bytes = Self::_unicode_to_wchar_bytes(utf8.as_str(), array.itemsize()); - array.frombytes_move(bytes); - } else { - return Err(vm.new_type_error(format!( - "cannot use a str to initialize an array with typecode '{spec}'" - ))); - } - } else if init.payload_is::<PyBytes>() || init.payload_is::<PyByteArray>() { - init.try_bytes_like(vm, |x| array.frombytes(x))?; - } else if let Ok(iter) = ArgIterable::try_from_object(vm, init.clone()) { - for obj in iter.iter(vm)? { - array.push(obj?, vm)?; - } - } else { - init.try_bytes_like(vm, |x| array.frombytes(x))?; - } - } - - let zelf = Self::from(array).into_ref_with_type(vm, cls)?; - Ok(zelf.into()) - } - } - - #[pyclass( - flags(BASETYPE), - with( - Comparable, - AsBuffer, - AsMapping, - AsSequence, - Iterable, - Constructor, - Representable - ) - )] - impl PyArray { - fn read(&self) -> PyRwLockReadGuard<'_, ArrayContentType> { - self.array.read() - } - - fn write(&self) -> PyRwLockWriteGuard<'_, ArrayContentType> { - self.array.write() - } - - #[pygetset] - fn typecode(&self, vm: &VirtualMachine) -> PyStrRef { - vm.ctx - .intern_str(self.read().typecode().to_string()) - .to_owned() - } - - #[pygetset] - fn itemsize(&self) -> usize { - self.read().itemsize() - } - - #[pymethod] - fn append(zelf: &Py<Self>, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - zelf.try_resizable(vm)?.push(x, vm) - } - - #[pymethod] - fn buffer_info(&self) -> (usize, usize) { - let array = self.read(); - (array.addr(), array.len()) - } - - #[pymethod] - fn count(&self, x: PyObjectRef, vm: &VirtualMachine) -> usize { - self.read().count(x, vm) - } - - #[pymethod] - fn remove(zelf: &Py<Self>, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - zelf.try_resizable(vm)?.remove(x, vm) - } - - #[pymethod] - fn extend(zelf: &Py<Self>, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut w = zelf.try_resizable(vm)?; - if zelf.is(&obj) { - w.imul(2, vm) - } else if let Some(array) = obj.payload::<PyArray>() { - w.iadd(&array.read(), vm) - } else { - let iter = ArgIterable::try_from_object(vm, obj)?; - // zelf.extend_from_iterable(iter, vm) - for obj in iter.iter(vm)? { - w.push(obj?, vm)?; - } - Ok(()) - } - } - - fn _wchar_bytes_to_string( - bytes: &[u8], - item_size: usize, - vm: &VirtualMachine, - ) -> PyResult<String> { - if item_size == 2 { - // safe because every configuration of bytes for the types we support are valid - let utf16 = unsafe { - std::slice::from_raw_parts( - bytes.as_ptr() as *const u16, - bytes.len() / std::mem::size_of::<u16>(), - ) - }; - Ok(String::from_utf16_lossy(utf16)) - } else { - // safe because every configuration of bytes for the types we support are valid - let chars = unsafe { - std::slice::from_raw_parts( - bytes.as_ptr() as *const u32, - bytes.len() / std::mem::size_of::<u32>(), - ) - }; - chars - .iter() - .map(|&ch| { - // cpython issue 17223 - u32_to_char(ch).map_err(|msg| vm.new_value_error(msg)) - }) - .try_collect() - } - } - - fn _unicode_to_wchar_bytes(utf8: &str, item_size: usize) -> Vec<u8> { - if item_size == 2 { - utf8.encode_utf16() - .flat_map(|ch| ch.to_ne_bytes()) - .collect() - } else { - utf8.chars() - .flat_map(|ch| (ch as u32).to_ne_bytes()) - .collect() - } - } - - #[pymethod] - fn fromunicode(zelf: &Py<Self>, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let utf8: &str = obj.try_to_value(vm).map_err(|_| { - vm.new_type_error(format!( - "fromunicode() argument must be str, not {}", - obj.class().name() - )) - })?; - if zelf.read().typecode() != 'u' { - return Err(vm.new_value_error( - "fromunicode() may only be called on unicode type arrays".into(), - )); - } - let mut w = zelf.try_resizable(vm)?; - let bytes = Self::_unicode_to_wchar_bytes(utf8, w.itemsize()); - w.frombytes_move(bytes); - Ok(()) - } - - #[pymethod] - fn tounicode(&self, vm: &VirtualMachine) -> PyResult<String> { - let array = self.array.read(); - if array.typecode() != 'u' { - return Err(vm.new_value_error( - "tounicode() may only be called on unicode type arrays".into(), - )); - } - let bytes = array.get_bytes(); - Self::_wchar_bytes_to_string(bytes, self.itemsize(), vm) - } - - fn _from_bytes(&self, b: &[u8], itemsize: usize, vm: &VirtualMachine) -> PyResult<()> { - if b.len() % itemsize != 0 { - return Err( - vm.new_value_error("bytes length not a multiple of item size".to_owned()) - ); - } - if b.len() / itemsize > 0 { - self.try_resizable(vm)?.frombytes(b); - } - Ok(()) - } - - #[pymethod] - fn frombytes(&self, b: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { - let b = b.borrow_buf(); - let itemsize = self.read().itemsize(); - self._from_bytes(&b, itemsize, vm) - } - - #[pymethod] - fn fromfile(&self, f: PyObjectRef, n: isize, vm: &VirtualMachine) -> PyResult<()> { - let itemsize = self.itemsize(); - if n < 0 { - return Err(vm.new_value_error("negative count".to_owned())); - } - let n = vm.check_repeat_or_overflow_error(itemsize, n)?; - let nbytes = n * itemsize; - - let b = vm.call_method(&f, "read", (nbytes,))?; - let b = b - .downcast::<PyBytes>() - .map_err(|_| vm.new_type_error("read() didn't return bytes".to_owned()))?; - - let not_enough_bytes = b.len() != nbytes; - - self._from_bytes(b.as_bytes(), itemsize, vm)?; - - if not_enough_bytes { - Err(vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "read() didn't return enough bytes".to_owned(), - )) - } else { - Ok(()) - } - } - - #[pymethod] - fn byteswap(&self) { - self.write().byteswap(); - } - - #[pymethod] - fn index( - &self, - x: PyObjectRef, - range: OptionalRangeArgs, - vm: &VirtualMachine, - ) -> PyResult<usize> { - let (start, stop) = range.saturate(self.len(), vm)?; - self.read().index(x, start, stop, vm) - } - - #[pymethod] - fn insert(zelf: &Py<Self>, i: isize, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut w = zelf.try_resizable(vm)?; - w.insert(i, x, vm) - } - - #[pymethod] - fn pop(zelf: &Py<Self>, i: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult { - let mut w = zelf.try_resizable(vm)?; - if w.len() == 0 { - Err(vm.new_index_error("pop from empty array".to_owned())) - } else { - w.pop(i.unwrap_or(-1), vm) - } - } - - #[pymethod] - pub(crate) fn tobytes(&self) -> Vec<u8> { - self.read().get_bytes().to_vec() - } - - #[pymethod] - fn tofile(&self, f: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - /* Write 64K blocks at a time */ - /* XXX Make the block size settable */ - const BLOCKSIZE: usize = 64 * 1024; - - let bytes = self.read(); - let bytes = bytes.get_bytes(); - - for b in bytes.chunks(BLOCKSIZE) { - let b = PyBytes::from(b.to_vec()).into_ref(&vm.ctx); - vm.call_method(&f, "write", (b,))?; - } - Ok(()) - } - - pub(crate) fn get_bytes(&self) -> PyMappedRwLockReadGuard<'_, [u8]> { - PyRwLockReadGuard::map(self.read(), |a| a.get_bytes()) - } - - pub(crate) fn get_bytes_mut(&self) -> PyMappedRwLockWriteGuard<'_, [u8]> { - PyRwLockWriteGuard::map(self.write(), |a| a.get_bytes_mut()) - } - - #[pymethod] - fn tolist(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let array = self.read(); - let mut v = Vec::with_capacity(array.len()); - for obj in array.iter(vm) { - v.push(obj?); - } - Ok(v) - } - - #[pymethod] - fn fromlist(zelf: &Py<Self>, list: PyListRef, vm: &VirtualMachine) -> PyResult<()> { - zelf.try_resizable(vm)?.fromlist(&list, vm) - } - - #[pymethod] - fn reverse(&self) { - self.write().reverse() - } - - #[pymethod(magic)] - fn copy(&self) -> PyArray { - self.array.read().clone().into() - } - - #[pymethod(magic)] - fn deepcopy(&self, _memo: PyObjectRef) -> PyArray { - self.copy() - } - - fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { - SequenceIndex::Int(i) => self.read().getitem_by_index(i, vm), - SequenceIndex::Slice(slice) => self.read().getitem_by_slice(slice, vm), - } - } - - #[pymethod(magic)] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self._getitem(&needle, vm) - } - - fn _setitem( - zelf: &Py<Self>, - needle: &PyObject, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { - SequenceIndex::Int(i) => zelf.write().setitem_by_index(i, value, vm), - SequenceIndex::Slice(slice) => { - let cloned; - let guard; - let items = if zelf.is(&value) { - cloned = zelf.read().clone(); - &cloned - } else { - match value.payload::<PyArray>() { - Some(array) => { - guard = array.read(); - &*guard - } - None => { - return Err(vm.new_type_error(format!( - "can only assign array (not \"{}\") to array slice", - value.class() - ))); - } - } - }; - if let Ok(mut w) = zelf.try_resizable(vm) { - w.setitem_by_slice(slice, items, vm) - } else { - zelf.write().setitem_by_slice_no_resize(slice, items, vm) - } - } - } - } - - #[pymethod(magic)] - fn setitem( - zelf: &Py<Self>, - needle: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::_setitem(zelf, &needle, value, vm) - } - - fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { - SequenceIndex::Int(i) => self.try_resizable(vm)?.delitem_by_index(i, vm), - SequenceIndex::Slice(slice) => self.try_resizable(vm)?.delitem_by_slice(slice, vm), - } - } - - #[pymethod(magic)] - fn delitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self._delitem(&needle, vm) - } - - #[pymethod(magic)] - fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - if let Some(other) = other.payload::<PyArray>() { - self.read() - .add(&other.read(), vm) - .map(|array| PyArray::from(array).into_ref(&vm.ctx)) - } else { - Err(vm.new_type_error(format!( - "can only append array (not \"{}\") to array", - other.class().name() - ))) - } - } - - #[pymethod(magic)] - fn iadd( - zelf: PyRef<Self>, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - if zelf.is(&other) { - zelf.try_resizable(vm)?.imul(2, vm)?; - } else if let Some(other) = other.payload::<PyArray>() { - zelf.try_resizable(vm)?.iadd(&other.read(), vm)?; - } else { - return Err(vm.new_type_error(format!( - "can only extend array with array (not \"{}\")", - other.class().name() - ))); - } - Ok(zelf) - } - - #[pymethod(name = "__rmul__")] - #[pymethod(magic)] - fn mul(&self, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - self.read() - .mul(value, vm) - .map(|x| Self::from(x).into_ref(&vm.ctx)) - } - - #[pymethod(magic)] - fn imul(zelf: PyRef<Self>, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.try_resizable(vm)?.imul(value, vm)?; - Ok(zelf) - } - - #[pymethod(magic)] - pub(crate) fn len(&self) -> usize { - self.read().len() - } - - fn array_eq(&self, other: &Self, vm: &VirtualMachine) -> PyResult<bool> { - // we cannot use zelf.is(other) for shortcut because if we contenting a - // float value NaN we always return False even they are the same object. - if self.len() != other.len() { - return Ok(false); - } - let array_a = self.read(); - let array_b = other.read(); - - // fast path for same ArrayContentType type - if let Ok(ord) = array_a.cmp(&array_b) { - return Ok(ord == Some(Ordering::Equal)); - } - - let iter = Iterator::zip(array_a.iter(vm), array_b.iter(vm)); - - for (a, b) in iter { - if !vm.bool_eq(&*a?, &*b?)? { - return Ok(false); - } - } - Ok(true) - } - - #[pymethod(magic)] - fn reduce_ex( - zelf: &Py<Self>, - proto: usize, - vm: &VirtualMachine, - ) -> PyResult<(PyObjectRef, PyTupleRef, Option<PyDictRef>)> { - if proto < 3 { - return Self::reduce(zelf, vm); - } - let array = zelf.read(); - let cls = zelf.class().to_owned(); - let typecode = vm.ctx.new_str(array.typecode_str()); - let bytes = vm.ctx.new_bytes(array.get_bytes().to_vec()); - let code = MachineFormatCode::from_typecode(array.typecode()).unwrap(); - let code = PyInt::from(u8::from(code)).into_pyobject(vm); - let module = vm.import("array", None, 0)?; - let func = module.get_attr("_array_reconstructor", vm)?; - Ok(( - func, - vm.new_tuple((cls, typecode, code, bytes)), - zelf.as_object().dict(), - )) - } - - #[pymethod(magic)] - fn reduce( - zelf: &Py<Self>, - vm: &VirtualMachine, - ) -> PyResult<(PyObjectRef, PyTupleRef, Option<PyDictRef>)> { - let array = zelf.read(); - let cls = zelf.class().to_owned(); - let typecode = vm.ctx.new_str(array.typecode_str()); - let values = if array.typecode() == 'u' { - let s = Self::_wchar_bytes_to_string(array.get_bytes(), array.itemsize(), vm)?; - s.chars().map(|x| x.to_pyobject(vm)).collect() - } else { - array.get_objects(vm) - }; - let values = vm.ctx.new_list(values); - Ok(( - cls.into(), - vm.new_tuple((typecode, values)), - zelf.as_object().dict(), - )) - } - - #[pymethod(magic)] - fn contains(&self, value: PyObjectRef, vm: &VirtualMachine) -> bool { - let array = self.array.read(); - for element in array - .iter(vm) - .map(|x| x.expect("Expected to be checked by array.len() and read lock.")) - { - if let Ok(true) = - element.rich_compare_bool(value.as_object(), PyComparisonOp::Eq, vm) - { - return true; - } - } - - false - } - } - - impl Comparable for PyArray { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - // TODO: deduplicate this logic with sequence::cmp in sequence.rs. Maybe make it generic? - - // we cannot use zelf.is(other) for shortcut because if we contenting a - // float value NaN we always return False even they are the same object. - let other = class_or_notimplemented!(Self, other); - - if let PyComparisonValue::Implemented(x) = - op.eq_only(|| Ok(zelf.array_eq(other, vm)?.into()))? - { - return Ok(x.into()); - } - - let array_a = zelf.read(); - let array_b = other.read(); - - let res = match array_a.cmp(&array_b) { - // fast path for same ArrayContentType type - Ok(partial_ord) => partial_ord.map_or(false, |ord| op.eval_ord(ord)), - Err(()) => { - let iter = Iterator::zip(array_a.iter(vm), array_b.iter(vm)); - - for (a, b) in iter { - let ret = match op { - PyComparisonOp::Lt | PyComparisonOp::Le => { - vm.bool_seq_lt(&*a?, &*b?)? - } - PyComparisonOp::Gt | PyComparisonOp::Ge => { - vm.bool_seq_gt(&*a?, &*b?)? - } - _ => unreachable!(), - }; - if let Some(v) = ret { - return Ok(PyComparisonValue::Implemented(v)); - } - } - - // fallback: - op.eval_ord(array_a.len().cmp(&array_b.len())) - } - }; - - Ok(res.into()) - } - } - - impl AsBuffer for PyArray { - fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { - let array = zelf.read(); - let buf = PyBuffer::new( - zelf.to_owned().into(), - BufferDescriptor::format( - array.len() * array.itemsize(), - false, - array.itemsize(), - array.typecode_str().into(), - ), - &BUFFER_METHODS, - ); - Ok(buf) - } - } - - impl Representable for PyArray { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let class = zelf.class(); - let class_name = class.name(); - if zelf.read().typecode() == 'u' { - if zelf.len() == 0 { - return Ok(format!("{class_name}('u')")); - } - let to_unicode = zelf.tounicode(vm)?; - let escape = crate::vm::literal::escape::UnicodeEscape::new_repr(&to_unicode); - return Ok(format!("{}('u', {})", class_name, escape.str_repr())); - } - zelf.read().repr(&class_name, vm) - } - } - - static BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| buffer.obj_as::<PyArray>().get_bytes().into(), - obj_bytes_mut: |buffer| buffer.obj_as::<PyArray>().get_bytes_mut().into(), - release: |buffer| { - buffer - .obj_as::<PyArray>() - .exports - .fetch_sub(1, atomic::Ordering::Release); - }, - retain: |buffer| { - buffer - .obj_as::<PyArray>() - .exports - .fetch_add(1, atomic::Ordering::Release); - }, - }; - - impl AsMapping for PyArray { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: PyMappingMethods = PyMappingMethods { - length: atomic_func!(|mapping, _vm| Ok(PyArray::mapping_downcast(mapping).len())), - subscript: atomic_func!(|mapping, needle, vm| { - PyArray::mapping_downcast(mapping)._getitem(needle, vm) - }), - ass_subscript: atomic_func!(|mapping, needle, value, vm| { - let zelf = PyArray::mapping_downcast(mapping); - if let Some(value) = value { - PyArray::_setitem(zelf, needle, value, vm) - } else { - zelf._delitem(needle, vm) - } - }), - }; - &AS_MAPPING - } - } - - impl AsSequence for PyArray { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyArray::sequence_downcast(seq).len())), - concat: atomic_func!(|seq, other, vm| { - let zelf = PyArray::sequence_downcast(seq); - PyArray::add(zelf, other.to_owned(), vm).map(|x| x.into()) - }), - repeat: atomic_func!(|seq, n, vm| { - PyArray::sequence_downcast(seq).mul(n, vm).map(|x| x.into()) - }), - item: atomic_func!(|seq, i, vm| { - PyArray::sequence_downcast(seq) - .read() - .getitem_by_index(i, vm) - }), - ass_item: atomic_func!(|seq, i, value, vm| { - let zelf = PyArray::sequence_downcast(seq); - if let Some(value) = value { - zelf.write().setitem_by_index(i, value, vm) - } else { - zelf.write().delitem_by_index(i, vm) - } - }), - contains: atomic_func!(|seq, target, vm| { - let zelf = PyArray::sequence_downcast(seq); - Ok(zelf.contains(target.to_owned(), vm)) - }), - inplace_concat: atomic_func!(|seq, other, vm| { - let zelf = PyArray::sequence_downcast(seq).to_owned(); - PyArray::iadd(zelf, other.to_owned(), vm).map(|x| x.into()) - }), - inplace_repeat: atomic_func!(|seq, n, vm| { - let zelf = PyArray::sequence_downcast(seq).to_owned(); - PyArray::imul(zelf, n, vm).map(|x| x.into()) - }), - }; - &AS_SEQUENCE - } - } - - impl Iterable for PyArray { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok(PyArrayIter { - internal: PyMutex::new(PositionIterInternal::new(zelf, 0)), - } - .into_pyobject(vm)) - } - } - - impl BufferResizeGuard for PyArray { - type Resizable<'a> = PyRwLockWriteGuard<'a, ArrayContentType>; - - fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>> { - let w = self.write(); - (self.exports.load(atomic::Ordering::SeqCst) == 0).then_some(w) - } - } - - #[pyattr] - #[pyclass(name = "arrayiterator", traverse)] - #[derive(Debug, PyPayload)] - pub struct PyArrayIter { - internal: PyMutex<PositionIterInternal<PyArrayRef>>, - } - - #[pyclass(with(IterNext, Iterable), flags(HAS_DICT))] - impl PyArrayIter { - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.internal - .lock() - .set_state(state, |obj, pos| pos.min(obj.len()), vm) - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) - } - } - - impl SelfIter for PyArrayIter {} - impl IterNext for PyArrayIter { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.internal.lock().next(|array, pos| { - Ok(match array.read().get(pos, vm) { - Some(item) => PyIterReturn::Return(item?), - None => PyIterReturn::StopIteration(None), - }) - }) - } - } - - #[derive(FromArgs)] - struct ReconstructorArgs { - #[pyarg(positional)] - arraytype: PyTypeRef, - #[pyarg(positional)] - typecode: PyStrRef, - #[pyarg(positional)] - mformat_code: MachineFormatCode, - #[pyarg(positional)] - items: PyBytesRef, - } - - #[derive(Debug, Copy, Clone, Eq, PartialEq)] - #[repr(u8)] - enum MachineFormatCode { - Int8 { signed: bool }, // 0, 1 - Int16 { signed: bool, big_endian: bool }, // 2, 3, 4, 5 - Int32 { signed: bool, big_endian: bool }, // 6, 7, 8, 9 - Int64 { signed: bool, big_endian: bool }, // 10, 11, 12, 13 - Ieee754Float { big_endian: bool }, // 14, 15 - Ieee754Double { big_endian: bool }, // 16, 17 - Utf16 { big_endian: bool }, // 18, 19 - Utf32 { big_endian: bool }, // 20, 21 - } - - impl From<MachineFormatCode> for u8 { - fn from(code: MachineFormatCode) -> u8 { - use MachineFormatCode::*; - match code { - Int8 { signed } => signed as u8, - Int16 { signed, big_endian } => 2 + signed as u8 * 2 + big_endian as u8, - Int32 { signed, big_endian } => 6 + signed as u8 * 2 + big_endian as u8, - Int64 { signed, big_endian } => 10 + signed as u8 * 2 + big_endian as u8, - Ieee754Float { big_endian } => 14 + big_endian as u8, - Ieee754Double { big_endian } => 16 + big_endian as u8, - Utf16 { big_endian } => 18 + big_endian as u8, - Utf32 { big_endian } => 20 + big_endian as u8, - } - } - } - - impl TryFrom<u8> for MachineFormatCode { - type Error = u8; - - fn try_from(code: u8) -> Result<Self, Self::Error> { - let big_endian = code % 2 != 0; - let signed = match code { - 0 | 1 => code != 0, - 2..=13 => (code - 2) % 4 >= 2, - _ => false, - }; - match code { - 0..=1 => Ok(Self::Int8 { signed }), - 2..=5 => Ok(Self::Int16 { signed, big_endian }), - 6..=9 => Ok(Self::Int32 { signed, big_endian }), - 10..=13 => Ok(Self::Int64 { signed, big_endian }), - 14..=15 => Ok(Self::Ieee754Float { big_endian }), - 16..=17 => Ok(Self::Ieee754Double { big_endian }), - 18..=19 => Ok(Self::Utf16 { big_endian }), - 20..=21 => Ok(Self::Utf32 { big_endian }), - _ => Err(code), - } - } - } - - impl<'a> TryFromBorrowedObject<'a> for MachineFormatCode { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - obj.try_to_ref::<PyInt>(vm) - .map_err(|_| { - vm.new_type_error(format!( - "an integer is required (got type {})", - obj.class().name() - )) - })? - .try_to_primitive::<i32>(vm)? - .to_u8() - .unwrap_or(u8::MAX) - .try_into() - .map_err(|_| { - vm.new_value_error("third argument must be a valid machine format code.".into()) - }) - } - } - - impl MachineFormatCode { - fn from_typecode(code: char) -> Option<Self> { - use std::mem::size_of; - let signed = code.is_ascii_uppercase(); - let big_endian = cfg!(target_endian = "big"); - let int_size = match code { - 'b' | 'B' => return Some(Self::Int8 { signed }), - 'u' => { - return match size_of::<wchar_t>() { - 2 => Some(Self::Utf16 { big_endian }), - 4 => Some(Self::Utf32 { big_endian }), - _ => None, - } - } - 'f' => { - // Copied from CPython - const Y: f32 = 16711938.0; - return match &Y.to_ne_bytes() { - b"\x4b\x7f\x01\x02" => Some(Self::Ieee754Float { big_endian: true }), - b"\x02\x01\x7f\x4b" => Some(Self::Ieee754Float { big_endian: false }), - _ => None, - }; - } - 'd' => { - // Copied from CPython - const Y: f64 = 9006104071832581.0; - return match &Y.to_ne_bytes() { - b"\x43\x3f\xff\x01\x02\x03\x04\x05" => { - Some(Self::Ieee754Double { big_endian: true }) - } - b"\x05\x04\x03\x02\x01\xff\x3f\x43" => { - Some(Self::Ieee754Double { big_endian: false }) - } - _ => None, - }; - } - _ => ArrayContentType::itemsize_of_typecode(code)? as u8, - }; - match int_size { - 2 => Some(Self::Int16 { signed, big_endian }), - 4 => Some(Self::Int32 { signed, big_endian }), - 8 => Some(Self::Int64 { signed, big_endian }), - _ => None, - } - } - fn item_size(self) -> usize { - match self { - Self::Int8 { .. } => 1, - Self::Int16 { .. } | Self::Utf16 { .. } => 2, - Self::Int32 { .. } | Self::Utf32 { .. } | Self::Ieee754Float { .. } => 4, - Self::Int64 { .. } | Self::Ieee754Double { .. } => 8, - } - } - } - - fn check_array_type(typ: PyTypeRef, vm: &VirtualMachine) -> PyResult<PyTypeRef> { - if !typ.fast_issubclass(PyArray::class(&vm.ctx)) { - return Err( - vm.new_type_error(format!("{} is not a subtype of array.array", typ.name())) - ); - } - Ok(typ) - } - - fn check_type_code(spec: PyStrRef, vm: &VirtualMachine) -> PyResult<ArrayContentType> { - let spec = spec.as_str().chars().exactly_one().map_err(|_| { - vm.new_type_error( - "_array_reconstructor() argument 2 must be a unicode character, not str".into(), - ) - })?; - ArrayContentType::from_char(spec) - .map_err(|_| vm.new_value_error("second argument must be a valid type code".into())) - } - - macro_rules! chunk_to_obj { - ($BYTE:ident, $TY:ty, $BIG_ENDIAN:ident) => {{ - let b = <[u8; ::std::mem::size_of::<$TY>()]>::try_from($BYTE).unwrap(); - if $BIG_ENDIAN { - <$TY>::from_be_bytes(b) - } else { - <$TY>::from_le_bytes(b) - } - }}; - ($VM:ident, $BYTE:ident, $TY:ty, $BIG_ENDIAN:ident) => { - chunk_to_obj!($BYTE, $TY, $BIG_ENDIAN).to_pyobject($VM) - }; - ($VM:ident, $BYTE:ident, $SIGNED_TY:ty, $UNSIGNED_TY:ty, $SIGNED:ident, $BIG_ENDIAN:ident) => {{ - let b = <[u8; ::std::mem::size_of::<$SIGNED_TY>()]>::try_from($BYTE).unwrap(); - match ($SIGNED, $BIG_ENDIAN) { - (false, false) => <$UNSIGNED_TY>::from_le_bytes(b).to_pyobject($VM), - (false, true) => <$UNSIGNED_TY>::from_be_bytes(b).to_pyobject($VM), - (true, false) => <$SIGNED_TY>::from_le_bytes(b).to_pyobject($VM), - (true, true) => <$SIGNED_TY>::from_be_bytes(b).to_pyobject($VM), - } - }}; - } - - #[pyfunction] - fn _array_reconstructor(args: ReconstructorArgs, vm: &VirtualMachine) -> PyResult<PyArrayRef> { - let cls = check_array_type(args.arraytype, vm)?; - let mut array = check_type_code(args.typecode, vm)?; - let format = args.mformat_code; - let bytes = args.items.as_bytes(); - if bytes.len() % format.item_size() != 0 { - return Err(vm.new_value_error("bytes length not a multiple of item size".into())); - } - if MachineFormatCode::from_typecode(array.typecode()) == Some(format) { - array.frombytes(bytes); - return PyArray::from(array).into_ref_with_type(vm, cls); - } - if !matches!( - format, - MachineFormatCode::Utf16 { .. } | MachineFormatCode::Utf32 { .. } - ) { - array.reserve(bytes.len() / format.item_size()); - } - let mut chunks = bytes.chunks(format.item_size()); - match format { - MachineFormatCode::Ieee754Float { big_endian } => { - chunks.try_for_each(|b| array.push(chunk_to_obj!(vm, b, f32, big_endian), vm))? - } - MachineFormatCode::Ieee754Double { big_endian } => { - chunks.try_for_each(|b| array.push(chunk_to_obj!(vm, b, f64, big_endian), vm))? - } - MachineFormatCode::Int8 { signed } => chunks - .try_for_each(|b| array.push(chunk_to_obj!(vm, b, i8, u8, signed, false), vm))?, - MachineFormatCode::Int16 { signed, big_endian } => chunks.try_for_each(|b| { - array.push(chunk_to_obj!(vm, b, i16, u16, signed, big_endian), vm) - })?, - MachineFormatCode::Int32 { signed, big_endian } => chunks.try_for_each(|b| { - array.push(chunk_to_obj!(vm, b, i32, u32, signed, big_endian), vm) - })?, - MachineFormatCode::Int64 { signed, big_endian } => chunks.try_for_each(|b| { - array.push(chunk_to_obj!(vm, b, i64, u64, signed, big_endian), vm) - })?, - MachineFormatCode::Utf16 { big_endian } => { - let utf16: Vec<_> = chunks.map(|b| chunk_to_obj!(b, u16, big_endian)).collect(); - let s = String::from_utf16(&utf16).map_err(|_| { - vm.new_unicode_encode_error("items cannot decode as utf16".into()) - })?; - let bytes = PyArray::_unicode_to_wchar_bytes(&s, array.itemsize()); - array.frombytes_move(bytes); - } - MachineFormatCode::Utf32 { big_endian } => { - let s: String = chunks - .map(|b| chunk_to_obj!(b, u32, big_endian)) - .map(|ch| u32_to_char(ch).map_err(|msg| vm.new_value_error(msg))) - .try_collect()?; - let bytes = PyArray::_unicode_to_wchar_bytes(&s, array.itemsize()); - array.frombytes_move(bytes); - } - }; - PyArray::from(array).into_ref_with_type(vm, cls) - } -} diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs deleted file mode 100644 index d7467fc8a5f..00000000000 --- a/stdlib/src/binascii.rs +++ /dev/null @@ -1,756 +0,0 @@ -// spell-checker:ignore hexlify unhexlify uuencodes - -pub(super) use decl::crc32; -pub(crate) use decl::make_module; -use rustpython_vm::{builtins::PyBaseExceptionRef, convert::ToPyException, VirtualMachine}; - -const PAD: u8 = 61u8; -const MAXLINESIZE: usize = 76; // Excluding the CRLF - -#[pymodule(name = "binascii")] -mod decl { - use super::{MAXLINESIZE, PAD}; - use crate::vm::{ - builtins::{PyIntRef, PyTypeRef}, - convert::ToPyException, - function::{ArgAsciiBuffer, ArgBytesLike, OptionalArg}, - PyResult, VirtualMachine, - }; - use itertools::Itertools; - - #[pyattr(name = "Error", once)] - pub(super) fn error_type(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "binascii", - "Error", - Some(vec![vm.ctx.exceptions.value_error.to_owned()]), - ) - } - - #[pyattr(name = "Incomplete", once)] - fn incomplete_type(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type("binascii", "Incomplete", None) - } - - fn hex_nibble(n: u8) -> u8 { - match n { - 0..=9 => b'0' + n, - 10..=15 => b'a' + (n - 10), - _ => unreachable!(), - } - } - - #[pyfunction(name = "b2a_hex")] - #[pyfunction] - fn hexlify(data: ArgBytesLike) -> Vec<u8> { - data.with_ref(|bytes| { - let mut hex = Vec::<u8>::with_capacity(bytes.len() * 2); - for b in bytes { - hex.push(hex_nibble(b >> 4)); - hex.push(hex_nibble(b & 0xf)); - } - hex - }) - } - - fn unhex_nibble(c: u8) -> Option<u8> { - match c { - b'0'..=b'9' => Some(c - b'0'), - b'a'..=b'f' => Some(c - b'a' + 10), - b'A'..=b'F' => Some(c - b'A' + 10), - _ => None, - } - } - - #[pyfunction(name = "a2b_hex")] - #[pyfunction] - fn unhexlify(data: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - data.with_ref(|hex_bytes| { - if hex_bytes.len() % 2 != 0 { - return Err(super::new_binascii_error( - "Odd-length string".to_owned(), - vm, - )); - } - - let mut unhex = Vec::<u8>::with_capacity(hex_bytes.len() / 2); - for (n1, n2) in hex_bytes.iter().tuples() { - if let (Some(n1), Some(n2)) = (unhex_nibble(*n1), unhex_nibble(*n2)) { - unhex.push(n1 << 4 | n2); - } else { - return Err(super::new_binascii_error( - "Non-hexadecimal digit found".to_owned(), - vm, - )); - } - } - - Ok(unhex) - }) - } - - #[pyfunction] - pub(crate) fn crc32(data: ArgBytesLike, init: OptionalArg<PyIntRef>) -> u32 { - let init = init.map_or(0, |i| i.as_u32_mask()); - - let mut hasher = crc32fast::Hasher::new_with_initial(init); - data.with_ref(|bytes| { - hasher.update(bytes); - hasher.finalize() - }) - } - - #[pyfunction] - pub(crate) fn crc_hqx(data: ArgBytesLike, init: PyIntRef) -> u32 { - const CRCTAB_HQX: [u16; 256] = [ - 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, - 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, - 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, - 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, - 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, - 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5, - 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, - 0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, - 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4, - 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, - 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 0xff9f, - 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, - 0xd10c, 0xc12d, 0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, - 0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, 0x1290, - 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, - 0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, - 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, - 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, - 0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d, - 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37, 0x7a16, - 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, - 0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, - 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, - 0x3eb2, 0x0ed1, 0x1ef0, - ]; - - let mut crc = init.as_u32_mask() & 0xffff; - - data.with_ref(|buf| { - for byte in buf { - crc = - ((crc << 8) & 0xFF00) ^ CRCTAB_HQX[((crc >> 8) as u8 ^ (byte)) as usize] as u32; - } - }); - - crc - } - - #[derive(FromArgs)] - struct NewlineArg { - #[pyarg(named, default = "true")] - newline: bool, - } - - #[derive(FromArgs)] - struct A2bBase64Args { - #[pyarg(any)] - s: ArgAsciiBuffer, - #[pyarg(named, default = "false")] - strict_mode: bool, - } - - #[pyfunction] - fn a2b_base64(args: A2bBase64Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - #[rustfmt::skip] - // Converts between ASCII and base-64 characters. The index of a given number yields the - // number in ASCII while the value of said index yields the number in base-64. For example - // "=" is 61 in ASCII but 0 (since it's the pad character) in base-64, so BASE64_TABLE[61] == 0 - const BASE64_TABLE: [i8; 256] = [ - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63, - 52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, /* Note PAD->0 */ - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, - 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1, - -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, - 41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1, - - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - ]; - - let A2bBase64Args { s, strict_mode } = args; - s.with_ref(|b| { - if b.is_empty() { - return Ok(vec![]); - } - - if strict_mode && b[0] == PAD { - return Err(base64::DecodeError::InvalidByte(0, 61)); - } - - let mut decoded: Vec<u8> = vec![]; - - let mut quad_pos = 0; // position in the nibble - let mut pads = 0; - let mut left_char: u8 = 0; - let mut padding_started = false; - for (i, &el) in b.iter().enumerate() { - if el == PAD { - padding_started = true; - - pads += 1; - if quad_pos >= 2 && quad_pos + pads >= 4 { - if strict_mode && i + 1 < b.len() { - // Represents excess data after padding error - return Err(base64::DecodeError::InvalidLastSymbol(i, PAD)); - } - - return Ok(decoded); - } - - continue; - } - - let binary_char = BASE64_TABLE[el as usize]; - if binary_char >= 64 || binary_char == -1 { - if strict_mode { - // Represents non-base64 data error - return Err(base64::DecodeError::InvalidByte(i, el)); - } - continue; - } - - if strict_mode && padding_started { - // Represents discontinuous padding error - return Err(base64::DecodeError::InvalidByte(i, PAD)); - } - pads = 0; - - // Decode individual ASCII character - match quad_pos { - 0 => { - quad_pos = 1; - left_char = binary_char as u8; - } - 1 => { - quad_pos = 2; - decoded.push((left_char << 2) | (binary_char >> 4) as u8); - left_char = (binary_char & 0x0f) as u8; - } - 2 => { - quad_pos = 3; - decoded.push((left_char << 4) | (binary_char >> 2) as u8); - left_char = (binary_char & 0x03) as u8; - } - 3 => { - quad_pos = 0; - decoded.push((left_char << 6) | binary_char as u8); - left_char = 0; - } - _ => unsafe { - // quad_pos is only assigned in this match statement to constants - std::hint::unreachable_unchecked() - }, - } - } - - match quad_pos { - 0 => Ok(decoded), - 1 => Err(base64::DecodeError::InvalidLastSymbol( - decoded.len() / 3 * 4 + 1, - 0, - )), - _ => Err(base64::DecodeError::InvalidLength), - } - }) - .map_err(|err| super::Base64DecodeError(err).to_pyexception(vm)) - } - - #[pyfunction] - fn b2a_base64(data: ArgBytesLike, NewlineArg { newline }: NewlineArg) -> Vec<u8> { - // https://stackoverflow.com/questions/63916821 - let mut encoded = data.with_ref(|b| base64::encode(b)).into_bytes(); - if newline { - encoded.push(b'\n'); - } - encoded - } - - #[inline] - fn uu_a2b_read(c: &u8, vm: &VirtualMachine) -> PyResult<u8> { - // Check the character for legality - // The 64 instead of the expected 63 is because - // there are a few uuencodes out there that use - // '`' as zero instead of space. - if !(b' '..=(b' ' + 64)).contains(c) { - if [b'\r', b'\n'].contains(c) { - return Ok(0); - } - return Err(vm.new_value_error("Illegal char".to_string())); - } - Ok((*c - b' ') & 0x3f) - } - - #[derive(FromArgs)] - struct A2bQpArgs { - #[pyarg(any)] - data: ArgAsciiBuffer, - #[pyarg(named, default = "false")] - header: bool, - } - #[pyfunction] - fn a2b_qp(args: A2bQpArgs) -> PyResult<Vec<u8>> { - let s = args.data; - let header = args.header; - s.with_ref(|buffer| { - let len = buffer.len(); - let mut out_data = Vec::with_capacity(len); - - let mut idx = 0; - - while idx < len { - if buffer[idx] == b'=' { - idx += 1; - if idx >= len { - break; - } - // Soft line breaks - if (buffer[idx] == b'\n') || (buffer[idx] == b'\r') { - if buffer[idx] != b'\n' { - while idx < len && buffer[idx] != b'\n' { - idx += 1; - } - } - if idx < len { - idx += 1; - } - } else if buffer[idx] == b'=' { - // roken case from broken python qp - out_data.push(b'='); - idx += 1; - } else if idx + 1 < len - && ((buffer[idx] >= b'A' && buffer[idx] <= b'F') - || (buffer[idx] >= b'a' && buffer[idx] <= b'f') - || (buffer[idx] >= b'0' && buffer[idx] <= b'9')) - && ((buffer[idx + 1] >= b'A' && buffer[idx + 1] <= b'F') - || (buffer[idx + 1] >= b'a' && buffer[idx + 1] <= b'f') - || (buffer[idx + 1] >= b'0' && buffer[idx + 1] <= b'9')) - { - // hexval - if let (Some(ch1), Some(ch2)) = - (unhex_nibble(buffer[idx]), unhex_nibble(buffer[idx + 1])) - { - out_data.push(ch1 << 4 | ch2); - } - idx += 2; - } else { - out_data.push(b'='); - } - } else if header && buffer[idx] == b'_' { - out_data.push(b' '); - idx += 1; - } else { - out_data.push(buffer[idx]); - idx += 1; - } - } - - Ok(out_data) - }) - } - - #[derive(FromArgs)] - struct B2aQpArgs { - #[pyarg(any)] - data: ArgAsciiBuffer, - #[pyarg(named, default = "false")] - quotetabs: bool, - #[pyarg(named, default = "true")] - istext: bool, - #[pyarg(named, default = "false")] - header: bool, - } - - #[pyfunction] - fn b2a_qp(args: B2aQpArgs) -> PyResult<Vec<u8>> { - let s = args.data; - let quotetabs = args.quotetabs; - let istext = args.istext; - let header = args.header; - s.with_ref(|buf| { - let buflen = buf.len(); - let mut linelen = 0; - let mut odatalen = 0; - let mut crlf = false; - let mut ch; - - let mut inidx; - let mut outidx; - - inidx = 0; - while inidx < buflen { - if buf[inidx] == b'\n' { - break; - } - inidx += 1; - } - if buflen > 0 && inidx < buflen && buf[inidx - 1] == b'\r' { - crlf = true; - } - - inidx = 0; - while inidx < buflen { - let mut delta = 0; - if (buf[inidx] > 126) - || (buf[inidx] == b'=') - || (header && buf[inidx] == b'_') - || (buf[inidx] == b'.' - && linelen == 0 - && (inidx + 1 == buflen - || buf[inidx + 1] == b'\n' - || buf[inidx + 1] == b'\r' - || buf[inidx + 1] == 0)) - || (!istext && ((buf[inidx] == b'\r') || (buf[inidx] == b'\n'))) - || ((buf[inidx] == b'\t' || buf[inidx] == b' ') && (inidx + 1 == buflen)) - || ((buf[inidx] < 33) - && (buf[inidx] != b'\r') - && (buf[inidx] != b'\n') - && (quotetabs || ((buf[inidx] != b'\t') && (buf[inidx] != b' ')))) - { - if (linelen + 3) >= MAXLINESIZE { - linelen = 0; - delta += if crlf { 3 } else { 2 }; - } - linelen += 3; - delta += 3; - inidx += 1; - } else if istext - && ((buf[inidx] == b'\n') - || ((inidx + 1 < buflen) - && (buf[inidx] == b'\r') - && (buf[inidx + 1] == b'\n'))) - { - linelen = 0; - // Protect against whitespace on end of line - if (inidx != 0) && ((buf[inidx - 1] == b' ') || (buf[inidx - 1] == b'\t')) { - delta += 2; - } - delta += if crlf { 2 } else { 1 }; - inidx += if buf[inidx] == b'\r' { 2 } else { 1 }; - } else { - if (inidx + 1 != buflen) - && (buf[inidx + 1] != b'\n') - && (linelen + 1) >= MAXLINESIZE - { - linelen = 0; - delta += if crlf { 3 } else { 2 }; - } - linelen += 1; - delta += 1; - inidx += 1; - } - odatalen += delta; - } - - let mut out_data = Vec::with_capacity(odatalen); - inidx = 0; - outidx = 0; - linelen = 0; - - while inidx < buflen { - if (buf[inidx] > 126) - || (buf[inidx] == b'=') - || (header && buf[inidx] == b'_') - || ((buf[inidx] == b'.') - && (linelen == 0) - && (inidx + 1 == buflen - || buf[inidx + 1] == b'\n' - || buf[inidx + 1] == b'\r' - || buf[inidx + 1] == 0)) - || (!istext && ((buf[inidx] == b'\r') || (buf[inidx] == b'\n'))) - || ((buf[inidx] == b'\t' || buf[inidx] == b' ') && (inidx + 1 == buflen)) - || ((buf[inidx] < 33) - && (buf[inidx] != b'\r') - && (buf[inidx] != b'\n') - && (quotetabs || ((buf[inidx] != b'\t') && (buf[inidx] != b' ')))) - { - if (linelen + 3) >= MAXLINESIZE { - // MAXLINESIZE = 76 - out_data.push(b'='); - outidx += 1; - if crlf { - out_data.push(b'\r'); - outidx += 1; - } - out_data.push(b'\n'); - outidx += 1; - linelen = 0; - } - out_data.push(b'='); - outidx += 1; - - ch = hex_nibble(buf[inidx] >> 4); - if (b'a'..=b'f').contains(&ch) { - ch -= b' '; - } - out_data.push(ch); - ch = hex_nibble(buf[inidx] & 0xf); - if (b'a'..=b'f').contains(&ch) { - ch -= b' '; - } - out_data.push(ch); - - outidx += 2; - inidx += 1; - linelen += 3; - } else if istext - && ((buf[inidx] == b'\n') - || ((inidx + 1 < buflen) - && (buf[inidx] == b'\r') - && (buf[inidx + 1] == b'\n'))) - { - linelen = 0; - if (outidx != 0) - && ((out_data[outidx - 1] == b' ') || (out_data[outidx - 1] == b'\t')) - { - ch = hex_nibble(out_data[outidx - 1] >> 4); - if (b'a'..=b'f').contains(&ch) { - ch -= b' '; - } - out_data.push(ch); - ch = hex_nibble(out_data[outidx - 1] & 0xf); - if (b'a'..=b'f').contains(&ch) { - ch -= b' '; - } - out_data.push(ch); - out_data[outidx - 1] = b'='; - outidx += 2; - } - - if crlf { - out_data.push(b'\r'); - outidx += 1; - } - out_data.push(b'\n'); - outidx += 1; - inidx += if buf[inidx] == b'\r' { 2 } else { 1 }; - } else { - if (inidx + 1 != buflen) && (buf[inidx + 1] != b'\n') && (linelen + 1) >= 76 { - // MAXLINESIZE = 76 - out_data.push(b'='); - outidx += 1; - if crlf { - out_data.push(b'\r'); - outidx += 1; - } - out_data.push(b'\n'); - outidx += 1; - linelen = 0; - } - linelen += 1; - if header && buf[inidx] == b' ' { - out_data.push(b'_'); - outidx += 1; - inidx += 1; - } else { - out_data.push(buf[inidx]); - outidx += 1; - inidx += 1; - } - } - } - Ok(out_data) - }) - } - - #[pyfunction] - fn rlecode_hqx(s: ArgAsciiBuffer) -> PyResult<Vec<u8>> { - const RUNCHAR: u8 = 0x90; // b'\x90' - s.with_ref(|buffer| { - let len = buffer.len(); - let mut out_data = Vec::<u8>::with_capacity((len * 2) + 2); - - let mut idx = 0; - while idx < len { - let ch = buffer[idx]; - - if ch == RUNCHAR { - out_data.push(RUNCHAR); - out_data.push(0); - return Ok(out_data); - } else { - let mut inend = idx + 1; - while inend < len && buffer[inend] == ch && inend < idx + 255 { - inend += 1; - } - if inend - idx > 3 { - out_data.push(ch); - out_data.push(RUNCHAR); - out_data.push(((inend - idx) % 256) as u8); - idx = inend - 1; - } else { - out_data.push(ch); - } - } - idx += 1; - } - Ok(out_data) - }) - } - - #[pyfunction] - fn rledecode_hqx(s: ArgAsciiBuffer) -> PyResult<Vec<u8>> { - const RUNCHAR: u8 = 0x90; //b'\x90' - s.with_ref(|buffer| { - let len = buffer.len(); - let mut out_data = Vec::<u8>::with_capacity(len); - let mut idx = 0; - - out_data.push(buffer[idx]); - idx += 1; - - while idx < len { - if buffer[idx] == RUNCHAR { - if buffer[idx + 1] == 0 { - out_data.push(RUNCHAR); - } else { - let ch = buffer[idx - 1]; - let range = buffer[idx + 1]; - idx += 1; - for _ in 1..range { - out_data.push(ch); - } - } - } else { - out_data.push(buffer[idx]); - } - idx += 1; - } - Ok(out_data) - }) - } - - #[pyfunction] - fn a2b_uu(s: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - s.with_ref(|b| { - // First byte: binary data length (in bytes) - let length = if b.is_empty() { - ((-0x20i32) & 0x3fi32) as usize - } else { - ((b[0] - b' ') & 0x3f) as usize - }; - - // Allocate the buffer - let mut res = Vec::<u8>::with_capacity(length); - let trailing_garbage_error = || Err(vm.new_value_error("Trailing garbage".to_string())); - - for chunk in b.get(1..).unwrap_or_default().chunks(4) { - let (char_a, char_b, char_c, char_d) = { - let mut chunk = chunk - .iter() - .map(|x| uu_a2b_read(x, vm)) - .collect::<Result<Vec<_>, _>>()?; - while chunk.len() < 4 { - chunk.push(0); - } - (chunk[0], chunk[1], chunk[2], chunk[3]) - }; - - if res.len() < length { - res.push(char_a << 2 | char_b >> 4); - } else if char_a != 0 || char_b != 0 { - return trailing_garbage_error(); - } - - if res.len() < length { - res.push((char_b & 0xf) | char_c >> 2); - } else if char_c != 0 { - return trailing_garbage_error(); - } - - if res.len() < length { - res.push((char_c & 0x3) << 6 | char_d); - } else if char_d != 0 { - return trailing_garbage_error(); - } - } - - let remaining_length = length - res.len(); - if remaining_length > 0 { - res.extend(vec![0; remaining_length]); - } - Ok(res) - }) - } - - #[derive(FromArgs)] - struct BacktickArg { - #[pyarg(named, default = "true")] - backtick: bool, - } - - #[pyfunction] - fn b2a_uu( - data: ArgBytesLike, - BacktickArg { backtick }: BacktickArg, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - #[inline] - fn uu_b2a(num: u8, backtick: bool) -> u8 { - if backtick && num != 0 { - 0x60 - } else { - b' ' + num - } - } - - data.with_ref(|b| { - let length = b.len(); - if length > 45 { - return Err(vm.new_value_error("At most 45 bytes at once".to_string())); - } - let mut res = Vec::<u8>::with_capacity(2 + ((length + 2) / 3) * 4); - res.push(uu_b2a(length as u8, backtick)); - - for chunk in b.chunks(3) { - let char_a = *chunk.first().unwrap(); - let char_b = *chunk.get(1).unwrap_or(&0); - let char_c = *chunk.get(2).unwrap_or(&0); - - res.push(uu_b2a(char_a >> 2, backtick)); - res.push(uu_b2a((char_a & 0x3) << 4 | char_b >> 4, backtick)); - res.push(uu_b2a((char_b & 0xf) << 2 | char_c >> 6, backtick)); - res.push(uu_b2a(char_c & 0x3f, backtick)); - } - - res.push(0xau8); - Ok(res) - }) - } -} - -struct Base64DecodeError(base64::DecodeError); - -fn new_binascii_error(msg: String, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception_msg(decl::error_type(vm), msg) -} - -impl ToPyException for Base64DecodeError { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - use base64::DecodeError::*; - let message = match self.0 { - InvalidByte(0, PAD) => "Leading padding not allowed".to_owned(), - InvalidByte(_, PAD) => "Discontinuous padding not allowed".to_owned(), - InvalidByte(_, _) => "Only base64 data is allowed".to_owned(), - InvalidLastSymbol(_, PAD) => "Excess data after padding".to_owned(), - InvalidLastSymbol(length, _) => { - format!("Invalid base64-encoded string: number of data characters {} cannot be 1 more than a multiple of 4", length) - } - InvalidLength => "Incorrect padding".to_owned(), - }; - new_binascii_error(format!("error decoding base64: {message}"), vm) - } -} diff --git a/stdlib/src/blake2.rs b/stdlib/src/blake2.rs deleted file mode 100644 index 9b7da3327cc..00000000000 --- a/stdlib/src/blake2.rs +++ /dev/null @@ -1,19 +0,0 @@ -// spell-checker:ignore usedforsecurity HASHXOF - -pub(crate) use _blake2::make_module; - -#[pymodule] -mod _blake2 { - use crate::hashlib::_hashlib::{local_blake2b, local_blake2s, BlakeHashArgs}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; - - #[pyfunction] - fn blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_blake2b(args).into_pyobject(vm)) - } - - #[pyfunction] - fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_blake2s(args).into_pyobject(vm)) - } -} diff --git a/stdlib/src/bz2.rs b/stdlib/src/bz2.rs deleted file mode 100644 index f150b06eb80..00000000000 --- a/stdlib/src/bz2.rs +++ /dev/null @@ -1,248 +0,0 @@ -// spell-checker:ignore compresslevel - -pub(crate) use _bz2::make_module; - -#[pymodule] -mod _bz2 { - use crate::common::lock::PyMutex; - use crate::vm::{ - builtins::{PyBytesRef, PyTypeRef}, - function::{ArgBytesLike, OptionalArg}, - object::{PyPayload, PyResult}, - types::Constructor, - VirtualMachine, - }; - use bzip2::{write::BzEncoder, Decompress, Status}; - use std::{fmt, io::Write}; - - // const BUFSIZ: i32 = 8192; - - struct DecompressorState { - decoder: Decompress, - eof: bool, - needs_input: bool, - // input_buffer: Vec<u8>, - // output_buffer: Vec<u8>, - } - - #[pyattr] - #[pyclass(name = "BZ2Decompressor")] - #[derive(PyPayload)] - struct BZ2Decompressor { - state: PyMutex<DecompressorState>, - } - - impl fmt::Debug for BZ2Decompressor { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "_bz2.BZ2Decompressor") - } - } - - impl Constructor for BZ2Decompressor { - type Args = (); - - fn py_new(cls: PyTypeRef, _: Self::Args, vm: &VirtualMachine) -> PyResult { - Self { - state: PyMutex::new(DecompressorState { - decoder: Decompress::new(false), - eof: false, - needs_input: true, - // input_buffer: Vec::new(), - // output_buffer: Vec::new(), - }), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(Constructor))] - impl BZ2Decompressor { - #[pymethod] - fn decompress( - &self, - data: ArgBytesLike, - // TODO: PyIntRef - max_length: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<PyBytesRef> { - let max_length = max_length.unwrap_or(-1); - if max_length >= 0 { - return Err(vm.new_not_implemented_error( - "the max_value argument is not implemented yet".to_owned(), - )); - } - // let max_length = if max_length < 0 || max_length >= BUFSIZ { - // BUFSIZ - // } else { - // max_length - // }; - - let mut state = self.state.lock(); - let DecompressorState { - decoder, - eof, - .. - // needs_input, - // input_buffer, - // output_buffer, - } = &mut *state; - - if *eof { - return Err(vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "End of stream already reached".to_owned(), - )); - } - - // data.with_ref(|data| input_buffer.extend(data)); - - // If max_length is negative: - // read the input X bytes at a time, compress it and append it to output. - // Once you're out of input, setting needs_input to true and return the - // output as bytes. - // - // TODO: - // If max_length is non-negative: - // Read the input X bytes at a time, compress it and append it to - // the output. If output reaches `max_length` in size, return - // it (up to max_length), and store the rest of the output - // for later. - - // TODO: arbitrary choice, not the right way to do it. - let mut buf = Vec::with_capacity(data.len() * 32); - - let before = decoder.total_in(); - let res = data.with_ref(|data| decoder.decompress_vec(data, &mut buf)); - let _written = (decoder.total_in() - before) as usize; - - let res = match res { - Ok(x) => x, - // TODO: error message - _ => return Err(vm.new_os_error("Invalid data stream".to_owned())), - }; - - if res == Status::StreamEnd { - *eof = true; - } - Ok(vm.ctx.new_bytes(buf.to_vec())) - } - - #[pygetset] - fn eof(&self) -> bool { - let state = self.state.lock(); - state.eof - } - - #[pygetset] - fn unused_data(&self, vm: &VirtualMachine) -> PyBytesRef { - // Data found after the end of the compressed stream. - // If this attribute is accessed before the end of the stream - // has been reached, its value will be b''. - vm.ctx.new_bytes(b"".to_vec()) - // alternatively, be more honest: - // Err(vm.new_not_implemented_error( - // "unused_data isn't implemented yet".to_owned(), - // )) - // - // TODO - // let state = self.state.lock(); - // if state.eof { - // vm.ctx.new_bytes(state.input_buffer.to_vec()) - // else { - // vm.ctx.new_bytes(b"".to_vec()) - // } - } - - #[pygetset] - fn needs_input(&self) -> bool { - // False if the decompress() method can provide more - // decompressed data before requiring new uncompressed input. - let state = self.state.lock(); - state.needs_input - } - - // TODO: mro()? - } - - struct CompressorState { - flushed: bool, - encoder: Option<BzEncoder<Vec<u8>>>, - } - - #[pyattr] - #[pyclass(name = "BZ2Compressor")] - #[derive(PyPayload)] - struct BZ2Compressor { - state: PyMutex<CompressorState>, - } - - impl fmt::Debug for BZ2Compressor { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "_bz2.BZ2Compressor") - } - } - - impl Constructor for BZ2Compressor { - type Args = (OptionalArg<i32>,); - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let (compresslevel,) = args; - // TODO: seriously? - // compresslevel.unwrap_or(bzip2::Compression::best().level().try_into().unwrap()); - let compresslevel = compresslevel.unwrap_or(9); - let level = match compresslevel { - valid_level @ 1..=9 => bzip2::Compression::new(valid_level as u32), - _ => { - return Err( - vm.new_value_error("compresslevel must be between 1 and 9".to_owned()) - ) - } - }; - - Self { - state: PyMutex::new(CompressorState { - flushed: false, - encoder: Some(BzEncoder::new(Vec::new(), level)), - }), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - // TODO: return partial results from compress() instead of returning everything in flush() - #[pyclass(with(Constructor))] - impl BZ2Compressor { - #[pymethod] - fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - let mut state = self.state.lock(); - if state.flushed { - return Err(vm.new_value_error("Compressor has been flushed".to_owned())); - } - - // let CompressorState { flushed, encoder } = &mut *state; - let CompressorState { encoder, .. } = &mut *state; - - // TODO: handle Err - data.with_ref(|input_bytes| encoder.as_mut().unwrap().write_all(input_bytes).unwrap()); - Ok(vm.ctx.new_bytes(Vec::new())) - } - - #[pymethod] - fn flush(&self, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - let mut state = self.state.lock(); - if state.flushed { - return Err(vm.new_value_error("Repeated call to flush()".to_owned())); - } - - // let CompressorState { flushed, encoder } = &mut *state; - let CompressorState { encoder, .. } = &mut *state; - - // TODO: handle Err - let out = encoder.take().unwrap().finish().unwrap(); - state.flushed = true; - Ok(vm.ctx.new_bytes(out.to_vec())) - } - } -} diff --git a/stdlib/src/cmath.rs b/stdlib/src/cmath.rs deleted file mode 100644 index 4a9590c5f41..00000000000 --- a/stdlib/src/cmath.rs +++ /dev/null @@ -1,207 +0,0 @@ -// TODO: Keep track of rust-num/num-complex/issues/2. A common trait could help with duplication -// that exists between cmath and math. -pub(crate) use cmath::make_module; -#[pymodule] -mod cmath { - use crate::vm::{ - function::{ArgIntoComplex, ArgIntoFloat, OptionalArg}, - PyResult, VirtualMachine, - }; - use num_complex::Complex64; - - // Constants - #[pyattr] - use std::f64::consts::{E as e, PI as pi, TAU as tau}; - #[pyattr] - use std::f64::{INFINITY as inf, NAN as nan}; - #[pyattr(name = "infj")] - const INFJ: Complex64 = Complex64::new(0., std::f64::INFINITY); - #[pyattr(name = "nanj")] - const NANJ: Complex64 = Complex64::new(0., std::f64::NAN); - - #[pyfunction] - fn phase(z: ArgIntoComplex) -> f64 { - z.arg() - } - - #[pyfunction] - fn polar(x: ArgIntoComplex) -> (f64, f64) { - x.to_polar() - } - - #[pyfunction] - fn rect(r: ArgIntoFloat, phi: ArgIntoFloat) -> Complex64 { - Complex64::from_polar(*r, *phi) - } - - #[pyfunction] - fn isinf(z: ArgIntoComplex) -> bool { - let Complex64 { re, im } = *z; - re.is_infinite() || im.is_infinite() - } - - #[pyfunction] - fn isfinite(z: ArgIntoComplex) -> bool { - z.is_finite() - } - - #[pyfunction] - fn isnan(z: ArgIntoComplex) -> bool { - z.is_nan() - } - - #[pyfunction] - fn exp(z: ArgIntoComplex, vm: &VirtualMachine) -> PyResult<Complex64> { - let z = *z; - result_or_overflow(z, z.exp(), vm) - } - - #[pyfunction] - fn sqrt(z: ArgIntoComplex) -> Complex64 { - z.sqrt() - } - - #[pyfunction] - fn sin(z: ArgIntoComplex) -> Complex64 { - z.sin() - } - - #[pyfunction] - fn asin(z: ArgIntoComplex) -> Complex64 { - z.asin() - } - - #[pyfunction] - fn cos(z: ArgIntoComplex) -> Complex64 { - z.cos() - } - - #[pyfunction] - fn acos(z: ArgIntoComplex) -> Complex64 { - z.acos() - } - - #[pyfunction] - fn log(z: ArgIntoComplex, base: OptionalArg<ArgIntoComplex>) -> Complex64 { - // TODO: Complex64.log with a negative base yields wrong results. - // Issue is with num_complex::Complex64 implementation of log - // which returns NaN when base is negative. - // log10(z) / log10(base) yields correct results but division - // doesn't handle pos/neg zero nicely. (i.e log(1, 0.5)) - z.log( - base.into_option() - .map(|base| base.re) - .unwrap_or(std::f64::consts::E), - ) - } - - #[pyfunction] - fn log10(z: ArgIntoComplex) -> Complex64 { - z.log(10.0) - } - - #[pyfunction] - fn acosh(z: ArgIntoComplex) -> Complex64 { - z.acosh() - } - - #[pyfunction] - fn atan(z: ArgIntoComplex) -> Complex64 { - z.atan() - } - - #[pyfunction] - fn atanh(z: ArgIntoComplex) -> Complex64 { - z.atanh() - } - - #[pyfunction] - fn tan(z: ArgIntoComplex) -> Complex64 { - z.tan() - } - - #[pyfunction] - fn tanh(z: ArgIntoComplex) -> Complex64 { - z.tanh() - } - - #[pyfunction] - fn sinh(z: ArgIntoComplex) -> Complex64 { - z.sinh() - } - - #[pyfunction] - fn cosh(z: ArgIntoComplex) -> Complex64 { - z.cosh() - } - - #[pyfunction] - fn asinh(z: ArgIntoComplex) -> Complex64 { - z.asinh() - } - - #[derive(FromArgs)] - struct IsCloseArgs { - #[pyarg(positional)] - a: ArgIntoComplex, - #[pyarg(positional)] - b: ArgIntoComplex, - #[pyarg(named, optional)] - rel_tol: OptionalArg<ArgIntoFloat>, - #[pyarg(named, optional)] - abs_tol: OptionalArg<ArgIntoFloat>, - } - - #[pyfunction] - fn isclose(args: IsCloseArgs, vm: &VirtualMachine) -> PyResult<bool> { - let a = *args.a; - let b = *args.b; - let rel_tol = args.rel_tol.map_or(1e-09, Into::into); - let abs_tol = args.abs_tol.map_or(0.0, Into::into); - - if rel_tol < 0.0 || abs_tol < 0.0 { - return Err(vm.new_value_error("tolerances must be non-negative".to_owned())); - } - - if a == b { - /* short circuit exact equality -- needed to catch two infinities of - the same sign. And perhaps speeds things up a bit sometimes. - */ - return Ok(true); - } - - /* This catches the case of two infinities of opposite sign, or - one infinity and one finite number. Two infinities of opposite - sign would otherwise have an infinite relative tolerance. - Two infinities of the same sign are caught by the equality check - above. - */ - if a.is_infinite() || b.is_infinite() { - return Ok(false); - } - - let diff = c_abs(b - a); - - Ok(diff <= (rel_tol * c_abs(b)) || (diff <= (rel_tol * c_abs(a))) || diff <= abs_tol) - } - - #[inline] - fn c_abs(Complex64 { re, im }: Complex64) -> f64 { - re.hypot(im) - } - - #[inline] - fn result_or_overflow( - value: Complex64, - result: Complex64, - vm: &VirtualMachine, - ) -> PyResult<Complex64> { - if !result.is_finite() && value.is_finite() { - // CPython doesn't return `inf` when called with finite - // values, it raises OverflowError instead. - Err(vm.new_overflow_error("math range error".to_owned())) - } else { - Ok(result) - } - } -} diff --git a/stdlib/src/contextvars.rs b/stdlib/src/contextvars.rs deleted file mode 100644 index d808d1d08e8..00000000000 --- a/stdlib/src/contextvars.rs +++ /dev/null @@ -1,208 +0,0 @@ -pub(crate) use _contextvars::make_module; - -#[pymodule] -mod _contextvars { - use crate::vm::{ - builtins::{PyFunction, PyStrRef, PyTypeRef}, - function::{ArgCallable, FuncArgs, OptionalArg}, - types::{Initializer, Representable}, - Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - }; - - #[pyattr] - #[pyclass(name = "Context")] - #[derive(Debug, PyPayload)] - struct PyContext {} // not to confuse with vm::Context - - #[pyclass(with(Initializer))] - impl PyContext { - #[pymethod] - fn run( - &self, - _callable: ArgCallable, - _args: FuncArgs, - _vm: &VirtualMachine, - ) -> PyResult<PyFunction> { - unimplemented!("Context.run is currently under construction") - } - - #[pymethod] - fn copy(&self, _vm: &VirtualMachine) -> PyResult<Self> { - unimplemented!("Context.copy is currently under construction") - } - - #[pymethod(magic)] - fn getitem(&self, _var: PyObjectRef) -> PyResult<PyObjectRef> { - unimplemented!("Context.__getitem__ is currently under construction") - } - - #[pymethod(magic)] - fn contains(&self, _var: PyObjectRef) -> PyResult<bool> { - unimplemented!("Context.__contains__ is currently under construction") - } - - #[pymethod(magic)] - fn len(&self) -> usize { - unimplemented!("Context.__len__ is currently under construction") - } - - #[pymethod(magic)] - fn iter(&self) -> PyResult { - unimplemented!("Context.__iter__ is currently under construction") - } - - #[pymethod] - fn get( - &self, - _key: PyObjectRef, - _default: OptionalArg<PyObjectRef>, - ) -> PyResult<PyObjectRef> { - unimplemented!("Context.get is currently under construction") - } - - #[pymethod] - fn keys(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> Vec<PyObjectRef> { - unimplemented!("Context.keys is currently under construction") - } - - #[pymethod] - fn values(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> Vec<PyObjectRef> { - unimplemented!("Context.values is currently under construction") - } - } - - impl Initializer for PyContext { - type Args = FuncArgs; - - fn init(_obj: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { - unimplemented!("Context.__init__ is currently under construction") - } - } - - #[pyattr] - #[pyclass(name, traverse)] - #[derive(Debug, PyPayload)] - struct ContextVar { - #[pytraverse(skip)] - #[allow(dead_code)] // TODO: RUSTPYTHON - name: String, - #[allow(dead_code)] // TODO: RUSTPYTHON - default: Option<PyObjectRef>, - } - - #[derive(FromArgs)] - struct ContextVarOptions { - #[pyarg(positional)] - #[allow(dead_code)] // TODO: RUSTPYTHON - name: PyStrRef, - #[pyarg(any, optional)] - #[allow(dead_code)] // TODO: RUSTPYTHON - default: OptionalArg<PyObjectRef>, - } - - #[pyclass(with(Initializer, Representable))] - impl ContextVar { - #[pygetset] - fn name(&self) -> String { - self.name.clone() - } - - #[pymethod] - fn get( - &self, - _default: OptionalArg<PyObjectRef>, - _vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { - unimplemented!("ContextVar.get() is currently under construction") - } - - #[pymethod] - fn set(&self, _value: PyObjectRef, _vm: &VirtualMachine) -> PyResult<()> { - unimplemented!("ContextVar.set() is currently under construction") - } - - #[pymethod] - fn reset( - _zelf: PyRef<Self>, - _token: PyRef<ContextToken>, - _vm: &VirtualMachine, - ) -> PyResult<()> { - unimplemented!("ContextVar.reset() is currently under construction") - } - - #[pyclassmethod(magic)] - fn class_getitem(_cls: PyTypeRef, _key: PyStrRef, _vm: &VirtualMachine) -> PyResult<()> { - unimplemented!("ContextVar.__class_getitem__() is currently under construction") - } - } - - impl Initializer for ContextVar { - type Args = ContextVarOptions; - - fn init(_obj: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { - unimplemented!("ContextVar.__init__() is currently under construction") - } - } - - impl Representable for ContextVar { - #[inline] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unimplemented!("<ContextVar name={{}} default={{}} at {{}}") - // format!( - // "<ContextVar name={} default={:?} at {:#x}>", - // zelf.name.as_str(), - // zelf.default.map_or("", |x| PyStr::from(*x).as_str()), - // zelf.get_id() - // ) - } - } - - #[pyattr] - #[pyclass(name = "Token")] - #[derive(Debug, PyPayload)] - struct ContextToken {} - - #[derive(FromArgs)] - struct ContextTokenOptions { - #[pyarg(positional)] - #[allow(dead_code)] // TODO: RUSTPYTHON - context: PyObjectRef, - #[pyarg(positional)] - #[allow(dead_code)] // TODO: RUSTPYTHON - var: PyObjectRef, - #[pyarg(positional)] - #[allow(dead_code)] // TODO: RUSTPYTHON - old_value: PyObjectRef, - } - - #[pyclass(with(Initializer, Representable))] - impl ContextToken { - #[pygetset] - fn var(&self, _vm: &VirtualMachine) -> PyObjectRef { - unimplemented!("Token.var() is currently under construction") - } - - #[pygetset] - fn old_value(&self, _vm: &VirtualMachine) -> PyObjectRef { - unimplemented!("Token.old_value() is currently under construction") - } - } - - impl Initializer for ContextToken { - type Args = ContextTokenOptions; - - fn init(_obj: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { - unimplemented!("Token.__init__() is currently under construction") - } - } - - impl Representable for ContextToken { - #[inline] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unimplemented!("<Token {{}}var={{}} at {{}}>") - } - } - - #[pyfunction] - fn copy_context() {} -} diff --git a/stdlib/src/csv.rs b/stdlib/src/csv.rs deleted file mode 100644 index bee3fd5faa5..00000000000 --- a/stdlib/src/csv.rs +++ /dev/null @@ -1,325 +0,0 @@ -pub(crate) use _csv::make_module; - -#[pymodule] -mod _csv { - use crate::common::lock::PyMutex; - use crate::vm::{ - builtins::{PyStr, PyTypeRef}, - function::{ArgIterable, ArgumentError, FromArgs, FuncArgs}, - match_class, - protocol::{PyIter, PyIterReturn}, - types::{IterNext, Iterable, SelfIter}, - AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, - }; - use itertools::{self, Itertools}; - use std::fmt; - - #[pyattr] - const QUOTE_MINIMAL: i32 = QuoteStyle::Minimal as i32; - #[pyattr] - const QUOTE_ALL: i32 = QuoteStyle::All as i32; - #[pyattr] - const QUOTE_NONNUMERIC: i32 = QuoteStyle::Nonnumeric as i32; - #[pyattr] - const QUOTE_NONE: i32 = QuoteStyle::None as i32; - - #[pyattr(name = "Error", once)] - fn error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "_csv", - "Error", - Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), - ) - } - - #[pyfunction] - fn reader( - iter: PyIter, - options: FormatOptions, - // TODO: handle quote style, etc - _rest: FuncArgs, - _vm: &VirtualMachine, - ) -> PyResult<Reader> { - Ok(Reader { - iter, - state: PyMutex::new(ReadState { - buffer: vec![0; 1024], - output_ends: vec![0; 16], - reader: options.to_reader(), - }), - }) - } - - #[pyfunction] - fn writer( - file: PyObjectRef, - options: FormatOptions, - // TODO: handle quote style, etc - _rest: FuncArgs, - vm: &VirtualMachine, - ) -> PyResult<Writer> { - let write = match vm.get_attribute_opt(file.clone(), "write")? { - Some(write_meth) => write_meth, - None if file.is_callable() => file, - None => { - return Err(vm.new_type_error("argument 1 must have a \"write\" method".to_owned())) - } - }; - - Ok(Writer { - write, - state: PyMutex::new(WriteState { - buffer: vec![0; 1024], - writer: options.to_writer(), - }), - }) - } - - #[inline] - fn resize_buf<T: num_traits::PrimInt>(buf: &mut Vec<T>) { - let new_size = buf.len() * 2; - buf.resize(new_size, T::zero()); - } - - #[repr(i32)] - pub enum QuoteStyle { - Minimal = 0, - All = 1, - Nonnumeric = 2, - None = 3, - } - - struct FormatOptions { - delimiter: u8, - quotechar: u8, - } - - impl FromArgs for FormatOptions { - fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - let delimiter = if let Some(delimiter) = args.kwargs.remove("delimiter") { - delimiter - .try_to_value::<&str>(vm)? - .bytes() - .exactly_one() - .map_err(|_| { - let msg = r#""delimiter" must be a 1-character string"#; - vm.new_type_error(msg.to_owned()) - })? - } else { - b',' - }; - - let quotechar = if let Some(quotechar) = args.kwargs.remove("quotechar") { - quotechar - .try_to_value::<&str>(vm)? - .bytes() - .exactly_one() - .map_err(|_| { - let msg = r#""quotechar" must be a 1-character string"#; - vm.new_type_error(msg.to_owned()) - })? - } else { - b'"' - }; - - Ok(FormatOptions { - delimiter, - quotechar, - }) - } - } - - impl FormatOptions { - fn to_reader(&self) -> csv_core::Reader { - csv_core::ReaderBuilder::new() - .delimiter(self.delimiter) - .quote(self.quotechar) - .terminator(csv_core::Terminator::CRLF) - .build() - } - fn to_writer(&self) -> csv_core::Writer { - csv_core::WriterBuilder::new() - .delimiter(self.delimiter) - .quote(self.quotechar) - .terminator(csv_core::Terminator::CRLF) - .build() - } - } - - struct ReadState { - buffer: Vec<u8>, - output_ends: Vec<usize>, - reader: csv_core::Reader, - } - - #[pyclass(no_attr, module = "_csv", name = "reader", traverse)] - #[derive(PyPayload)] - pub(super) struct Reader { - iter: PyIter, - #[pytraverse(skip)] - state: PyMutex<ReadState>, - } - - impl fmt::Debug for Reader { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "_csv.reader") - } - } - - #[pyclass(with(IterNext, Iterable))] - impl Reader {} - impl SelfIter for Reader {} - impl IterNext for Reader { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let string = match zelf.iter.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - let string = string.downcast::<PyStr>().map_err(|obj| { - vm.new_type_error(format!( - "iterator should return strings, not {} (the file should be opened in text mode)", - obj.class().name() - )) - })?; - let input = string.as_str().as_bytes(); - - let mut state = zelf.state.lock(); - let ReadState { - buffer, - output_ends, - reader, - } = &mut *state; - - let mut input_offset = 0; - let mut output_offset = 0; - let mut output_ends_offset = 0; - - loop { - let (res, nread, nwritten, nends) = reader.read_record( - &input[input_offset..], - &mut buffer[output_offset..], - &mut output_ends[output_ends_offset..], - ); - input_offset += nread; - output_offset += nwritten; - output_ends_offset += nends; - match res { - csv_core::ReadRecordResult::InputEmpty => {} - csv_core::ReadRecordResult::OutputFull => resize_buf(buffer), - csv_core::ReadRecordResult::OutputEndsFull => resize_buf(output_ends), - csv_core::ReadRecordResult::Record => break, - csv_core::ReadRecordResult::End => { - return Ok(PyIterReturn::StopIteration(None)) - } - } - } - let rest = &input[input_offset..]; - if !rest.iter().all(|&c| matches!(c, b'\r' | b'\n')) { - return Err(vm.new_value_error( - "new-line character seen in unquoted field - \ - do you need to open the file in universal-newline mode?" - .to_owned(), - )); - } - - let mut prev_end = 0; - let out = output_ends[..output_ends_offset] - .iter() - .map(|&end| { - let range = prev_end..end; - prev_end = end; - let s = std::str::from_utf8(&buffer[range]) - // not sure if this is possible - the input was all strings - .map_err(|_e| vm.new_unicode_decode_error("csv not utf8".to_owned()))?; - Ok(vm.ctx.new_str(s).into()) - }) - .collect::<Result<_, _>>()?; - Ok(PyIterReturn::Return(vm.ctx.new_list(out).into())) - } - } - - struct WriteState { - buffer: Vec<u8>, - writer: csv_core::Writer, - } - - #[pyclass(no_attr, module = "_csv", name = "writer", traverse)] - #[derive(PyPayload)] - pub(super) struct Writer { - write: PyObjectRef, - #[pytraverse(skip)] - state: PyMutex<WriteState>, - } - - impl fmt::Debug for Writer { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "_csv.writer") - } - } - - #[pyclass] - impl Writer { - #[pymethod] - fn writerow(&self, row: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut state = self.state.lock(); - let WriteState { buffer, writer } = &mut *state; - - let mut buffer_offset = 0; - - macro_rules! handle_res { - ($x:expr) => {{ - let (res, nwritten) = $x; - buffer_offset += nwritten; - match res { - csv_core::WriteResult::InputEmpty => break, - csv_core::WriteResult::OutputFull => resize_buf(buffer), - } - }}; - } - - let row = ArgIterable::try_from_object(vm, row)?; - for field in row.iter(vm)? { - let field: PyObjectRef = field?; - let stringified; - let data: &[u8] = match_class!(match field { - ref s @ PyStr => s.as_str().as_bytes(), - crate::builtins::PyNone => b"", - ref obj => { - stringified = obj.str(vm)?; - stringified.as_str().as_bytes() - } - }); - - let mut input_offset = 0; - - loop { - let (res, nread, nwritten) = - writer.field(&data[input_offset..], &mut buffer[buffer_offset..]); - input_offset += nread; - handle_res!((res, nwritten)); - } - - loop { - handle_res!(writer.delimiter(&mut buffer[buffer_offset..])); - } - } - - loop { - handle_res!(writer.terminator(&mut buffer[buffer_offset..])); - } - - let s = std::str::from_utf8(&buffer[..buffer_offset]) - .map_err(|_| vm.new_unicode_decode_error("csv not utf8".to_owned()))?; - - self.write.call((s,), vm) - } - - #[pymethod] - fn writerows(&self, rows: ArgIterable, vm: &VirtualMachine) -> PyResult<()> { - for row in rows.iter(vm)? { - self.writerow(row?, vm)?; - } - Ok(()) - } - } -} diff --git a/stdlib/src/dis.rs b/stdlib/src/dis.rs deleted file mode 100644 index 9ac6245a240..00000000000 --- a/stdlib/src/dis.rs +++ /dev/null @@ -1,45 +0,0 @@ -pub(crate) use decl::make_module; - -#[pymodule(name = "dis")] -mod decl { - use crate::vm::{ - builtins::{PyCode, PyDictRef, PyStrRef}, - bytecode::CodeFlags, - compiler, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, - }; - - #[pyfunction] - fn dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let co = if let Ok(co) = obj.get_attr("__code__", vm) { - // Method or function: - PyRef::try_from_object(vm, co)? - } else if let Ok(co_str) = PyStrRef::try_from_object(vm, obj.clone()) { - // String: - vm.compile(co_str.as_str(), compiler::Mode::Exec, "<dis>".to_owned()) - .map_err(|err| vm.new_syntax_error(&err, Some(co_str.as_str())))? - } else { - PyRef::try_from_object(vm, obj)? - }; - disassemble(co) - } - - #[pyfunction] - fn disassemble(co: PyRef<PyCode>) -> PyResult<()> { - print!("{}", &co.code); - Ok(()) - } - - #[pyattr(name = "COMPILER_FLAG_NAMES")] - fn compiler_flag_names(vm: &VirtualMachine) -> PyDictRef { - let dict = vm.ctx.new_dict(); - for (name, flag) in CodeFlags::NAME_MAPPING { - dict.set_item( - &*vm.new_pyobj(flag.bits()), - vm.ctx.new_str(*name).into(), - vm, - ) - .unwrap(); - } - dict - } -} diff --git a/stdlib/src/faulthandler.rs b/stdlib/src/faulthandler.rs deleted file mode 100644 index 2fb93ecc884..00000000000 --- a/stdlib/src/faulthandler.rs +++ /dev/null @@ -1,63 +0,0 @@ -pub(crate) use decl::make_module; - -#[pymodule(name = "faulthandler")] -mod decl { - use crate::vm::{frame::Frame, function::OptionalArg, stdlib::sys::PyStderr, VirtualMachine}; - - fn dump_frame(frame: &Frame, vm: &VirtualMachine) { - let stderr = PyStderr(vm); - writeln!( - stderr, - " File \"{}\", line {} in {}", - frame.code.source_path, - frame.current_location().row.to_usize(), - frame.code.obj_name - ) - } - - #[pyfunction] - fn dump_traceback( - _file: OptionalArg<i64>, - _all_threads: OptionalArg<bool>, - vm: &VirtualMachine, - ) { - let stderr = PyStderr(vm); - writeln!(stderr, "Stack (most recent call first):"); - - for frame in vm.frames.borrow().iter() { - dump_frame(frame, vm); - } - } - - #[derive(FromArgs)] - #[allow(unused)] - struct EnableArgs { - #[pyarg(any, default)] - file: Option<i64>, - #[pyarg(any, default = "true")] - all_threads: bool, - } - - #[pyfunction] - fn enable(_args: EnableArgs) { - // TODO - } - - #[derive(FromArgs)] - #[allow(unused)] - struct RegisterArgs { - #[pyarg(positional)] - signum: i64, - #[pyarg(any, default)] - file: Option<i64>, - #[pyarg(any, default = "true")] - all_threads: bool, - #[pyarg(any, default = "false")] - chain: bool, - } - - #[pyfunction] - fn register(_args: RegisterArgs) { - // TODO - } -} diff --git a/stdlib/src/gc.rs b/stdlib/src/gc.rs deleted file mode 100644 index c78eea9c299..00000000000 --- a/stdlib/src/gc.rs +++ /dev/null @@ -1,76 +0,0 @@ -pub(crate) use gc::make_module; - -#[pymodule] -mod gc { - use crate::vm::{function::FuncArgs, PyResult, VirtualMachine}; - - #[pyfunction] - fn collect(_args: FuncArgs, _vm: &VirtualMachine) -> i32 { - 0 - } - - #[pyfunction] - fn isenabled(_args: FuncArgs, _vm: &VirtualMachine) -> bool { - false - } - - #[pyfunction] - fn enable(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn disable(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn get_count(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn get_debug(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn get_objects(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn get_refererts(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn get_stats(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn get_threshold(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn is_tracked(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn set_debug(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } - - #[pyfunction] - fn set_threshold(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_not_implemented_error("".to_owned())) - } -} diff --git a/stdlib/src/hashlib.rs b/stdlib/src/hashlib.rs deleted file mode 100644 index 6944c37f9d2..00000000000 --- a/stdlib/src/hashlib.rs +++ /dev/null @@ -1,432 +0,0 @@ -// spell-checker:ignore usedforsecurity HASHXOF - -pub(crate) use _hashlib::make_module; - -#[pymodule] -pub mod _hashlib { - use crate::common::lock::PyRwLock; - use crate::vm::{ - builtins::{PyBytes, PyStrRef, PyTypeRef}, - convert::ToPyObject, - function::{ArgBytesLike, ArgStrOrBytesLike, FuncArgs, OptionalArg}, - protocol::PyBuffer, - PyObjectRef, PyPayload, PyResult, VirtualMachine, - }; - use blake2::{Blake2b512, Blake2s256}; - use digest::{core_api::BlockSizeUser, DynDigest}; - use digest::{ExtendableOutput, Update}; - use dyn_clone::{clone_trait_object, DynClone}; - use md5::Md5; - use sha1::Sha1; - use sha2::{Sha224, Sha256, Sha384, Sha512}; - use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; - - #[derive(FromArgs, Debug)] - #[allow(unused)] - struct NewHashArgs { - #[pyarg(positional)] - name: PyStrRef, - #[pyarg(any, optional)] - data: OptionalArg<ArgBytesLike>, - #[pyarg(named, default = "true")] - usedforsecurity: bool, - } - - #[derive(FromArgs)] - #[allow(unused)] - pub struct BlakeHashArgs { - #[pyarg(positional, optional)] - pub data: OptionalArg<ArgBytesLike>, - #[pyarg(named, default = "true")] - usedforsecurity: bool, - } - - impl From<NewHashArgs> for BlakeHashArgs { - fn from(args: NewHashArgs) -> Self { - Self { - data: args.data, - usedforsecurity: args.usedforsecurity, - } - } - } - - #[derive(FromArgs, Debug)] - #[allow(unused)] - pub struct HashArgs { - #[pyarg(any, optional)] - pub string: OptionalArg<ArgBytesLike>, - #[pyarg(named, default = "true")] - usedforsecurity: bool, - } - - impl From<NewHashArgs> for HashArgs { - fn from(args: NewHashArgs) -> Self { - Self { - string: args.data, - usedforsecurity: args.usedforsecurity, - } - } - } - - #[derive(FromArgs)] - #[allow(unused)] - struct XofDigestArgs { - #[pyarg(positional)] - length: isize, - } - - impl XofDigestArgs { - fn length(&self, vm: &VirtualMachine) -> PyResult<usize> { - usize::try_from(self.length) - .map_err(|_| vm.new_value_error("length must be non-negative".to_owned())) - } - } - - #[pyattr] - #[pyclass(module = "_hashlib", name = "HASH")] - #[derive(PyPayload)] - pub struct PyHasher { - pub name: String, - pub ctx: PyRwLock<HashWrapper>, - } - - impl std::fmt::Debug for PyHasher { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "HASH {}", self.name) - } - } - - #[pyclass] - impl PyHasher { - fn new(name: &str, d: HashWrapper) -> Self { - PyHasher { - name: name.to_owned(), - ctx: PyRwLock::new(d), - } - } - - #[pyslot] - fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("cannot create '_hashlib.HASH' instances".into())) - } - - #[pygetset] - fn name(&self) -> String { - self.name.clone() - } - - #[pygetset] - fn digest_size(&self) -> usize { - self.ctx.read().digest_size() - } - - #[pygetset] - fn block_size(&self) -> usize { - self.ctx.read().block_size() - } - - #[pymethod] - fn update(&self, data: ArgBytesLike) { - data.with_ref(|bytes| self.ctx.write().update(bytes)); - } - - #[pymethod] - fn digest(&self) -> PyBytes { - self.ctx.read().finalize().into() - } - - #[pymethod] - fn hexdigest(&self) -> String { - hex::encode(self.ctx.read().finalize()) - } - - #[pymethod] - fn copy(&self) -> Self { - PyHasher::new(&self.name, self.ctx.read().clone()) - } - } - - #[pyattr] - #[pyclass(module = "_hashlib", name = "HASHXOF")] - #[derive(PyPayload)] - pub struct PyHasherXof { - name: String, - ctx: PyRwLock<HashXofWrapper>, - } - - impl std::fmt::Debug for PyHasherXof { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "HASHXOF {}", self.name) - } - } - - #[pyclass] - impl PyHasherXof { - fn new(name: &str, d: HashXofWrapper) -> Self { - PyHasherXof { - name: name.to_owned(), - ctx: PyRwLock::new(d), - } - } - - #[pyslot] - fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("cannot create '_hashlib.HASHXOF' instances".into())) - } - - #[pygetset] - fn name(&self) -> String { - self.name.clone() - } - - #[pygetset] - fn digest_size(&self) -> usize { - 0 - } - - #[pygetset] - fn block_size(&self) -> usize { - self.ctx.read().block_size() - } - - #[pymethod] - fn update(&self, data: ArgBytesLike) { - data.with_ref(|bytes| self.ctx.write().update(bytes)); - } - - #[pymethod] - fn digest(&self, args: XofDigestArgs, vm: &VirtualMachine) -> PyResult<PyBytes> { - Ok(self.ctx.read().finalize_xof(args.length(vm)?).into()) - } - - #[pymethod] - fn hexdigest(&self, args: XofDigestArgs, vm: &VirtualMachine) -> PyResult<String> { - Ok(hex::encode(self.ctx.read().finalize_xof(args.length(vm)?))) - } - - #[pymethod] - fn copy(&self) -> Self { - PyHasherXof::new(&self.name, self.ctx.read().clone()) - } - } - - #[pyfunction(name = "new")] - fn hashlib_new(args: NewHashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - match args.name.as_str().to_lowercase().as_str() { - "md5" => Ok(local_md5(args.into()).into_pyobject(vm)), - "sha1" => Ok(local_sha1(args.into()).into_pyobject(vm)), - "sha224" => Ok(local_sha224(args.into()).into_pyobject(vm)), - "sha256" => Ok(local_sha256(args.into()).into_pyobject(vm)), - "sha384" => Ok(local_sha384(args.into()).into_pyobject(vm)), - "sha512" => Ok(local_sha512(args.into()).into_pyobject(vm)), - "sha3_224" => Ok(local_sha3_224(args.into()).into_pyobject(vm)), - "sha3_256" => Ok(local_sha3_256(args.into()).into_pyobject(vm)), - "sha3_384" => Ok(local_sha3_384(args.into()).into_pyobject(vm)), - "sha3_512" => Ok(local_sha3_512(args.into()).into_pyobject(vm)), - "shake_128" => Ok(local_shake_128(args.into()).into_pyobject(vm)), - "shake_256" => Ok(local_shake_256(args.into()).into_pyobject(vm)), - "blake2b" => Ok(local_blake2b(args.into()).into_pyobject(vm)), - "blake2s" => Ok(local_blake2s(args.into()).into_pyobject(vm)), - other => Err(vm.new_value_error(format!("Unknown hashing algorithm: {other}"))), - } - } - - #[pyfunction(name = "openssl_md5")] - pub fn local_md5(args: HashArgs) -> PyHasher { - PyHasher::new("md5", HashWrapper::new::<Md5>(args.string)) - } - - #[pyfunction(name = "openssl_sha1")] - pub fn local_sha1(args: HashArgs) -> PyHasher { - PyHasher::new("sha1", HashWrapper::new::<Sha1>(args.string)) - } - - #[pyfunction(name = "openssl_sha224")] - pub fn local_sha224(args: HashArgs) -> PyHasher { - PyHasher::new("sha224", HashWrapper::new::<Sha224>(args.string)) - } - - #[pyfunction(name = "openssl_sha256")] - pub fn local_sha256(args: HashArgs) -> PyHasher { - PyHasher::new("sha256", HashWrapper::new::<Sha256>(args.string)) - } - - #[pyfunction(name = "openssl_sha384")] - pub fn local_sha384(args: HashArgs) -> PyHasher { - PyHasher::new("sha384", HashWrapper::new::<Sha384>(args.string)) - } - - #[pyfunction(name = "openssl_sha512")] - pub fn local_sha512(args: HashArgs) -> PyHasher { - PyHasher::new("sha512", HashWrapper::new::<Sha512>(args.string)) - } - - #[pyfunction(name = "openssl_sha3_224")] - pub fn local_sha3_224(args: HashArgs) -> PyHasher { - PyHasher::new("sha3_224", HashWrapper::new::<Sha3_224>(args.string)) - } - - #[pyfunction(name = "openssl_sha3_256")] - pub fn local_sha3_256(args: HashArgs) -> PyHasher { - PyHasher::new("sha3_256", HashWrapper::new::<Sha3_256>(args.string)) - } - - #[pyfunction(name = "openssl_sha3_384")] - pub fn local_sha3_384(args: HashArgs) -> PyHasher { - PyHasher::new("sha3_384", HashWrapper::new::<Sha3_384>(args.string)) - } - - #[pyfunction(name = "openssl_sha3_512")] - pub fn local_sha3_512(args: HashArgs) -> PyHasher { - PyHasher::new("sha3_512", HashWrapper::new::<Sha3_512>(args.string)) - } - - #[pyfunction(name = "openssl_shake_128")] - pub fn local_shake_128(args: HashArgs) -> PyHasherXof { - PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(args.string)) - } - - #[pyfunction(name = "openssl_shake_256")] - pub fn local_shake_256(args: HashArgs) -> PyHasherXof { - PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(args.string)) - } - - #[pyfunction(name = "openssl_blake2b")] - pub fn local_blake2b(args: BlakeHashArgs) -> PyHasher { - PyHasher::new("blake2b", HashWrapper::new::<Blake2b512>(args.data)) - } - - #[pyfunction(name = "openssl_blake2s")] - pub fn local_blake2s(args: BlakeHashArgs) -> PyHasher { - PyHasher::new("blake2s", HashWrapper::new::<Blake2s256>(args.data)) - } - - #[pyfunction] - fn compare_digest( - a: ArgStrOrBytesLike, - b: ArgStrOrBytesLike, - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { - fn is_str(arg: &ArgStrOrBytesLike) -> bool { - matches!(arg, ArgStrOrBytesLike::Str(_)) - } - - if is_str(&a) != is_str(&b) { - return Err(vm.new_type_error(format!( - "a bytes-like object is required, not '{}'", - b.as_object().class().name() - ))); - } - - let a_hash = a.borrow_bytes().to_vec(); - let b_hash = b.borrow_bytes().to_vec(); - - Ok((a_hash == b_hash).to_pyobject(vm)) - } - - #[derive(FromArgs, Debug)] - #[allow(unused)] - pub struct NewHMACHashArgs { - #[pyarg(positional)] - name: PyBuffer, - #[pyarg(any, optional)] - data: OptionalArg<ArgBytesLike>, - #[pyarg(named, default = "true")] - digestmod: bool, // TODO: RUSTPYTHON support functions & name functions - } - - #[pyfunction] - fn hmac_new(_args: NewHMACHashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - Err(vm.new_type_error("cannot create 'hmac' instances".into())) // TODO: RUSTPYTHON support hmac - } - - pub trait ThreadSafeDynDigest: DynClone + DynDigest + Sync + Send {} - impl<T> ThreadSafeDynDigest for T where T: DynClone + DynDigest + Sync + Send {} - - clone_trait_object!(ThreadSafeDynDigest); - - /// Generic wrapper patching around the hashing libraries. - #[derive(Clone)] - pub struct HashWrapper { - block_size: usize, - inner: Box<dyn ThreadSafeDynDigest>, - } - - impl HashWrapper { - pub fn new<D>(data: OptionalArg<ArgBytesLike>) -> Self - where - D: ThreadSafeDynDigest + BlockSizeUser + Default + 'static, - { - let mut h = HashWrapper { - block_size: D::block_size(), - inner: Box::<D>::default(), - }; - if let OptionalArg::Present(d) = data { - d.with_ref(|bytes| h.update(bytes)); - } - h - } - - fn update(&mut self, data: &[u8]) { - self.inner.update(data); - } - - fn block_size(&self) -> usize { - self.block_size - } - - fn digest_size(&self) -> usize { - self.inner.output_size() - } - - fn finalize(&self) -> Vec<u8> { - let cloned = self.inner.box_clone(); - cloned.finalize().into_vec() - } - } - - #[derive(Clone)] - pub enum HashXofWrapper { - Shake128(Shake128), - Shake256(Shake256), - } - - impl HashXofWrapper { - pub fn new_shake_128(data: OptionalArg<ArgBytesLike>) -> Self { - let mut h = HashXofWrapper::Shake128(Shake128::default()); - if let OptionalArg::Present(d) = data { - d.with_ref(|bytes| h.update(bytes)); - } - h - } - - pub fn new_shake_256(data: OptionalArg<ArgBytesLike>) -> Self { - let mut h = HashXofWrapper::Shake256(Shake256::default()); - if let OptionalArg::Present(d) = data { - d.with_ref(|bytes| h.update(bytes)); - } - h - } - - fn update(&mut self, data: &[u8]) { - match self { - HashXofWrapper::Shake128(h) => h.update(data), - HashXofWrapper::Shake256(h) => h.update(data), - } - } - - fn block_size(&self) -> usize { - match self { - HashXofWrapper::Shake128(_) => Shake128::block_size(), - HashXofWrapper::Shake256(_) => Shake256::block_size(), - } - } - - fn finalize_xof(&self, length: usize) -> Vec<u8> { - match self { - HashXofWrapper::Shake128(h) => h.clone().finalize_boxed(length).into_vec(), - HashXofWrapper::Shake256(h) => h.clone().finalize_boxed(length).into_vec(), - } - } - } -} diff --git a/stdlib/src/json.rs b/stdlib/src/json.rs deleted file mode 100644 index 921e545e5df..00000000000 --- a/stdlib/src/json.rs +++ /dev/null @@ -1,260 +0,0 @@ -pub(crate) use _json::make_module; -mod machinery; - -#[pymodule] -mod _json { - use super::machinery; - use crate::vm::{ - builtins::{PyBaseExceptionRef, PyStrRef, PyType, PyTypeRef}, - convert::{ToPyObject, ToPyResult}, - function::{IntoFuncArgs, OptionalArg}, - protocol::PyIterReturn, - types::{Callable, Constructor}, - AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, - }; - use malachite_bigint::BigInt; - use std::str::FromStr; - - #[pyattr(name = "make_scanner")] - #[pyclass(name = "Scanner", traverse)] - #[derive(Debug, PyPayload)] - struct JsonScanner { - #[pytraverse(skip)] - strict: bool, - object_hook: Option<PyObjectRef>, - object_pairs_hook: Option<PyObjectRef>, - parse_float: Option<PyObjectRef>, - parse_int: Option<PyObjectRef>, - parse_constant: PyObjectRef, - ctx: PyObjectRef, - } - - impl Constructor for JsonScanner { - type Args = PyObjectRef; - - fn py_new(cls: PyTypeRef, ctx: Self::Args, vm: &VirtualMachine) -> PyResult { - let strict = ctx.get_attr("strict", vm)?.try_to_bool(vm)?; - let object_hook = vm.option_if_none(ctx.get_attr("object_hook", vm)?); - let object_pairs_hook = vm.option_if_none(ctx.get_attr("object_pairs_hook", vm)?); - let parse_float = ctx.get_attr("parse_float", vm)?; - let parse_float = if vm.is_none(&parse_float) || parse_float.is(vm.ctx.types.float_type) - { - None - } else { - Some(parse_float) - }; - let parse_int = ctx.get_attr("parse_int", vm)?; - let parse_int = if vm.is_none(&parse_int) || parse_int.is(vm.ctx.types.int_type) { - None - } else { - Some(parse_int) - }; - let parse_constant = ctx.get_attr("parse_constant", vm)?; - - Self { - strict, - object_hook, - object_pairs_hook, - parse_float, - parse_int, - parse_constant, - ctx, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(Callable, Constructor))] - impl JsonScanner { - fn parse( - &self, - s: &str, - pystr: PyStrRef, - idx: usize, - scan_once: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn> { - let c = match s.chars().next() { - Some(c) => c, - None => { - return Ok(PyIterReturn::StopIteration(Some( - vm.ctx.new_int(idx).into(), - ))) - } - }; - let next_idx = idx + c.len_utf8(); - match c { - '"' => { - return scanstring(pystr, next_idx, OptionalArg::Present(self.strict), vm) - .map(|x| PyIterReturn::Return(x.to_pyobject(vm))) - } - '{' => { - // TODO: parse the object in rust - let parse_obj = self.ctx.get_attr("parse_object", vm)?; - let result = parse_obj.call( - ( - (pystr, next_idx), - self.strict, - scan_once, - self.object_hook.clone(), - self.object_pairs_hook.clone(), - ), - vm, - ); - return PyIterReturn::from_pyresult(result, vm); - } - '[' => { - // TODO: parse the array in rust - let parse_array = self.ctx.get_attr("parse_array", vm)?; - return PyIterReturn::from_pyresult( - parse_array.call(((pystr, next_idx), scan_once), vm), - vm, - ); - } - _ => {} - } - - macro_rules! parse_const { - ($s:literal, $val:expr) => { - if s.starts_with($s) { - return Ok(PyIterReturn::Return( - vm.new_tuple(($val, idx + $s.len())).into(), - )); - } - }; - } - - parse_const!("null", vm.ctx.none()); - parse_const!("true", true); - parse_const!("false", false); - - if let Some((res, len)) = self.parse_number(s, vm) { - return Ok(PyIterReturn::Return(vm.new_tuple((res?, idx + len)).into())); - } - - macro_rules! parse_constant { - ($s:literal) => { - if s.starts_with($s) { - return Ok(PyIterReturn::Return( - vm.new_tuple((self.parse_constant.call(($s,), vm)?, idx + $s.len())) - .into(), - )); - } - }; - } - - parse_constant!("NaN"); - parse_constant!("Infinity"); - parse_constant!("-Infinity"); - - Ok(PyIterReturn::StopIteration(Some( - vm.ctx.new_int(idx).into(), - ))) - } - - fn parse_number(&self, s: &str, vm: &VirtualMachine) -> Option<(PyResult, usize)> { - let mut has_neg = false; - let mut has_decimal = false; - let mut has_exponent = false; - let mut has_e_sign = false; - let mut i = 0; - for c in s.chars() { - match c { - '-' if i == 0 => has_neg = true, - n if n.is_ascii_digit() => {} - '.' if !has_decimal => has_decimal = true, - 'e' | 'E' if !has_exponent => has_exponent = true, - '+' | '-' if !has_e_sign => has_e_sign = true, - _ => break, - } - i += 1; - } - if i == 0 || (i == 1 && has_neg) { - return None; - } - let buf = &s[..i]; - let ret = if has_decimal || has_exponent { - // float - if let Some(ref parse_float) = self.parse_float { - parse_float.call((buf,), vm) - } else { - Ok(vm.ctx.new_float(f64::from_str(buf).unwrap()).into()) - } - } else if let Some(ref parse_int) = self.parse_int { - parse_int.call((buf,), vm) - } else { - Ok(vm.new_pyobj(BigInt::from_str(buf).unwrap())) - }; - Some((ret, buf.len())) - } - } - - impl Callable for JsonScanner { - type Args = (PyStrRef, isize); - fn call(zelf: &Py<Self>, (pystr, idx): Self::Args, vm: &VirtualMachine) -> PyResult { - if idx < 0 { - return Err(vm.new_value_error("idx cannot be negative".to_owned())); - } - let idx = idx as usize; - let mut chars = pystr.as_str().chars(); - if idx > 0 && chars.nth(idx - 1).is_none() { - PyIterReturn::StopIteration(Some(vm.ctx.new_int(idx).into())).to_pyresult(vm) - } else { - zelf.parse( - chars.as_str(), - pystr.clone(), - idx, - zelf.to_owned().into(), - vm, - ) - .and_then(|x| x.to_pyresult(vm)) - } - } - } - - fn encode_string(s: &str, ascii_only: bool) -> String { - let mut buf = Vec::<u8>::with_capacity(s.len() + 2); - machinery::write_json_string(s, ascii_only, &mut buf) - // SAFETY: writing to a vec can't fail - .unwrap_or_else(|_| unsafe { std::hint::unreachable_unchecked() }); - // SAFETY: we only output valid utf8 from write_json_string - unsafe { String::from_utf8_unchecked(buf) } - } - - #[pyfunction] - fn encode_basestring(s: PyStrRef) -> String { - encode_string(s.as_str(), false) - } - - #[pyfunction] - fn encode_basestring_ascii(s: PyStrRef) -> String { - encode_string(s.as_str(), true) - } - - fn py_decode_error( - e: machinery::DecodeError, - s: PyStrRef, - vm: &VirtualMachine, - ) -> PyBaseExceptionRef { - let get_error = || -> PyResult<_> { - let cls = vm.try_class("json", "JSONDecodeError")?; - let exc = PyType::call(&cls, (e.msg, s, e.pos).into_args(vm), vm)?; - exc.try_into_value(vm) - }; - match get_error() { - Ok(x) | Err(x) => x, - } - } - - #[pyfunction] - fn scanstring( - s: PyStrRef, - end: usize, - strict: OptionalArg<bool>, - vm: &VirtualMachine, - ) -> PyResult<(String, usize)> { - machinery::scanstring(s.as_str(), end, strict.unwrap_or(true)) - .map_err(|e| py_decode_error(e, s, vm)) - } -} diff --git a/stdlib/src/json/machinery.rs b/stdlib/src/json/machinery.rs deleted file mode 100644 index fc6b5308664..00000000000 --- a/stdlib/src/json/machinery.rs +++ /dev/null @@ -1,226 +0,0 @@ -// derived from https://github.com/lovasoa/json_in_type - -// BSD 2-Clause License -// -// Copyright (c) 2018, Ophir LOJKINE -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// * Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use std::io; - -static ESCAPE_CHARS: [&str; 0x20] = [ - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", "\\u0007", "\\b", - "\\t", "\\n", "\\u000", "\\f", "\\r", "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", - "\\u0013", "\\u0014", "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", - "\\u001", "\\u001c", "\\u001d", "\\u001e", "\\u001f", -]; - -// This bitset represents which bytes can be copied as-is to a JSON string (0) -// And which one need to be escaped (1) -// The characters that need escaping are 0x00 to 0x1F, 0x22 ("), 0x5C (\), 0x7F (DEL) -// Non-ASCII unicode characters can be safely included in a JSON string -#[allow(clippy::unusual_byte_groupings)] // it's groups of 16, come on clippy -static NEEDS_ESCAPING_BITSET: [u64; 4] = [ - //fedcba9876543210_fedcba9876543210_fedcba9876543210_fedcba9876543210 - 0b0000000000000000_0000000000000100_1111111111111111_1111111111111111, // 3_2_1_0 - 0b1000000000000000_0000000000000000_0001000000000000_0000000000000000, // 7_6_5_4 - 0b0000000000000000_0000000000000000_0000000000000000_0000000000000000, // B_A_9_8 - 0b0000000000000000_0000000000000000_0000000000000000_0000000000000000, // F_E_D_C -]; - -#[inline(always)] -fn json_escaped_char(c: u8) -> Option<&'static str> { - let bitset_value = NEEDS_ESCAPING_BITSET[(c / 64) as usize] & (1 << (c % 64)); - if bitset_value == 0 { - None - } else { - Some(match c { - x if x < 0x20 => ESCAPE_CHARS[c as usize], - b'\\' => "\\\\", - b'\"' => "\\\"", - 0x7F => "\\u007f", - _ => unreachable!(), - }) - } -} - -pub fn write_json_string<W: io::Write>(s: &str, ascii_only: bool, w: &mut W) -> io::Result<()> { - w.write_all(b"\"")?; - let mut write_start_idx = 0; - let bytes = s.as_bytes(); - if ascii_only { - for (idx, c) in s.char_indices() { - if c.is_ascii() { - if let Some(escaped) = json_escaped_char(c as u8) { - w.write_all(&bytes[write_start_idx..idx])?; - w.write_all(escaped.as_bytes())?; - write_start_idx = idx + 1; - } - } else { - w.write_all(&bytes[write_start_idx..idx])?; - write_start_idx = idx + c.len_utf8(); - // codepoints outside the BMP get 2 '\uxxxx' sequences to represent them - for point in c.encode_utf16(&mut [0; 2]) { - write!(w, "\\u{point:04x}")?; - } - } - } - } else { - for (idx, c) in s.bytes().enumerate() { - if let Some(escaped) = json_escaped_char(c) { - w.write_all(&bytes[write_start_idx..idx])?; - w.write_all(escaped.as_bytes())?; - write_start_idx = idx + 1; - } - } - } - w.write_all(&bytes[write_start_idx..])?; - w.write_all(b"\"") -} - -#[derive(Debug)] -pub struct DecodeError { - pub msg: String, - pub pos: usize, -} -impl DecodeError { - fn new(msg: impl Into<String>, pos: usize) -> Self { - let msg = msg.into(); - Self { msg, pos } - } -} - -enum StrOrChar<'a> { - Str(&'a str), - Char(char), -} -impl StrOrChar<'_> { - fn len(&self) -> usize { - match self { - StrOrChar::Str(s) => s.len(), - StrOrChar::Char(c) => c.len_utf8(), - } - } -} -pub fn scanstring<'a>( - s: &'a str, - end: usize, - strict: bool, -) -> Result<(String, usize), DecodeError> { - let mut chunks: Vec<StrOrChar<'a>> = Vec::new(); - let mut output_len = 0usize; - let mut push_chunk = |chunk: StrOrChar<'a>| { - output_len += chunk.len(); - chunks.push(chunk); - }; - let unterminated_err = || DecodeError::new("Unterminated string starting at", end - 1); - let mut chars = s.char_indices().enumerate().skip(end).peekable(); - let (_, (mut chunk_start, _)) = chars.peek().ok_or_else(unterminated_err)?; - while let Some((char_i, (byte_i, c))) = chars.next() { - match c { - '"' => { - push_chunk(StrOrChar::Str(&s[chunk_start..byte_i])); - let mut out = String::with_capacity(output_len); - for x in chunks { - match x { - StrOrChar::Str(s) => out.push_str(s), - StrOrChar::Char(c) => out.push(c), - } - } - return Ok((out, char_i + 1)); - } - '\\' => { - push_chunk(StrOrChar::Str(&s[chunk_start..byte_i])); - let (_, (_, c)) = chars.next().ok_or_else(unterminated_err)?; - let esc = match c { - '"' => "\"", - '\\' => "\\", - '/' => "/", - 'b' => "\x08", - 'f' => "\x0c", - 'n' => "\n", - 'r' => "\r", - 't' => "\t", - 'u' => { - let surrogate_err = || DecodeError::new("unpaired surrogate", char_i); - let mut uni = decode_unicode(&mut chars, char_i)?; - chunk_start = byte_i + 6; - if (0xd800..=0xdbff).contains(&uni) { - // uni is a surrogate -- try to find its pair - if let Some(&(pos2, (_, '\\'))) = chars.peek() { - // ok, the next char starts an escape - chars.next(); - if let Some((_, (_, 'u'))) = chars.peek() { - // ok, it's a unicode escape - chars.next(); - let uni2 = decode_unicode(&mut chars, pos2)?; - chunk_start = pos2 + 6; - if (0xdc00..=0xdfff).contains(&uni2) { - // ok, we found what we were looking for -- \uXXXX\uXXXX, both surrogates - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)); - } else { - // if we don't find a matching surrogate, error -- until str - // isn't utf8 internally, we can't parse surrogates - return Err(surrogate_err()); - } - } else { - return Err(surrogate_err()); - } - } - } - push_chunk(StrOrChar::Char( - std::char::from_u32(uni).ok_or_else(surrogate_err)?, - )); - continue; - } - _ => return Err(DecodeError::new(format!("Invalid \\escape: {c:?}"), char_i)), - }; - chunk_start = byte_i + 2; - push_chunk(StrOrChar::Str(esc)); - } - '\x00'..='\x1f' if strict => { - return Err(DecodeError::new( - format!("Invalid control character {c:?} at"), - char_i, - )); - } - _ => {} - } - } - Err(unterminated_err()) -} - -#[inline] -fn decode_unicode<I>(it: &mut I, pos: usize) -> Result<u32, DecodeError> -where - I: Iterator<Item = (usize, (usize, char))>, -{ - let err = || DecodeError::new("Invalid \\uXXXX escape", pos); - let mut uni = 0; - for x in (0..4).rev() { - let (_, (_, c)) = it.next().ok_or_else(err)?; - let d = c.to_digit(16).ok_or_else(err)?; - uni += d * 16u32.pow(x); - } - Ok(uni) -} diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs deleted file mode 100644 index 2a1a7d5ce08..00000000000 --- a/stdlib/src/lib.rs +++ /dev/null @@ -1,183 +0,0 @@ -// to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of -// how `mod` works, but we want this sometimes for pymodule declarations -#![allow(clippy::module_inception)] - -#[macro_use] -extern crate rustpython_derive; - -pub mod array; -mod binascii; -mod bisect; -mod cmath; -mod contextvars; -mod csv; -mod dis; -mod gc; - -mod blake2; -mod hashlib; -mod md5; -mod sha1; -mod sha256; -mod sha3; - -mod json; -#[cfg(not(any(target_os = "ios", target_os = "android", target_arch = "wasm32")))] -mod locale; -mod math; -#[cfg(unix)] -mod mmap; -mod pyexpat; -mod pystruct; -mod random; -mod statistics; -// TODO: maybe make this an extension module, if we ever get those -// mod re; -#[cfg(feature = "bz2")] -mod bz2; -#[cfg(not(target_arch = "wasm32"))] -pub mod socket; -#[cfg(all(unix, not(target_os = "redox")))] -mod syslog; -mod unicodedata; -mod zlib; - -#[cfg(not(target_arch = "wasm32"))] -mod faulthandler; -#[cfg(any(unix, target_os = "wasi"))] -mod fcntl; -#[cfg(not(target_arch = "wasm32"))] -mod multiprocessing; -#[cfg(unix)] -mod posixsubprocess; -// libc is missing constants on redox -#[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] -mod grp; -#[cfg(all(unix, not(target_os = "redox")))] -mod resource; -#[cfg(target_os = "macos")] -mod scproxy; -#[cfg(not(target_arch = "wasm32"))] -mod select; -#[cfg(not(any(target_os = "android", target_arch = "wasm32")))] -mod sqlite; -#[cfg(all(not(target_arch = "wasm32"), feature = "ssl"))] -mod ssl; -#[cfg(all(unix, not(target_os = "redox"), not(target_os = "ios")))] -mod termios; -#[cfg(not(any( - target_os = "android", - target_os = "ios", - target_os = "windows", - target_arch = "wasm32", - target_os = "redox", -)))] -mod uuid; - -use rustpython_common as common; -use rustpython_vm as vm; - -use crate::vm::{builtins, stdlib::StdlibInitFunc}; -use std::borrow::Cow; - -pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInitFunc)> { - macro_rules! modules { - { - $( - #[cfg($cfg:meta)] - { $( $key:expr => $val:expr),* $(,)? } - )* - } => {{ - [ - $( - $(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)* - )* - ] - .into_iter() - }}; - } - modules! { - #[cfg(all())] - { - "array" => array::make_module, - "binascii" => binascii::make_module, - "_bisect" => bisect::make_module, - "cmath" => cmath::make_module, - "_contextvars" => contextvars::make_module, - "_csv" => csv::make_module, - "_dis" => dis::make_module, - "gc" => gc::make_module, - "_hashlib" => hashlib::make_module, - "_sha1" => sha1::make_module, - "_sha3" => sha3::make_module, - "_sha256" => sha256::make_module, - // "_sha512" => sha512::make_module, // TODO: RUSPYTHON fix strange fail on vm: 'static type has not been initialized' - "_md5" => md5::make_module, - "_blake2" => blake2::make_module, - "_json" => json::make_module, - "math" => math::make_module, - "pyexpat" => pyexpat::make_module, - "_random" => random::make_module, - "_statistics" => statistics::make_module, - "_struct" => pystruct::make_module, - "unicodedata" => unicodedata::make_module, - "zlib" => zlib::make_module, - "_statistics" => statistics::make_module, - // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, - } - #[cfg(any(unix, target_os = "wasi"))] - { - "fcntl" => fcntl::make_module, - } - #[cfg(not(target_arch = "wasm32"))] - { - "_multiprocessing" => multiprocessing::make_module, - "select" => select::make_module, - "_socket" => socket::make_module, - "faulthandler" => faulthandler::make_module, - } - #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] - { - "_sqlite3" => sqlite::make_module, - } - #[cfg(feature = "ssl")] - { - "_ssl" => ssl::make_module, - } - #[cfg(feature = "bz2")] - { - "_bz2" => bz2::make_module, - } - // Unix-only - #[cfg(unix)] - { - "_posixsubprocess" => posixsubprocess::make_module, - "mmap" => mmap::make_module, - } - #[cfg(all(unix, not(target_os = "redox")))] - { - "syslog" => syslog::make_module, - "resource" => resource::make_module, - } - #[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))] - { - "termios" => termios::make_module, - } - #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] - { - "grp" => grp::make_module, - } - #[cfg(target_os = "macos")] - { - "_scproxy" => scproxy::make_module, - } - #[cfg(not(any(target_os = "android", target_os = "ios", target_os = "windows", target_arch = "wasm32", target_os = "redox")))] - { - "_uuid" => uuid::make_module, - } - #[cfg(not(any(target_os = "ios", target_os = "android", target_arch = "wasm32")))] - { - "_locale" => locale::make_module, - } - } -} diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs deleted file mode 100644 index bbe1008e53d..00000000000 --- a/stdlib/src/locale.rs +++ /dev/null @@ -1,216 +0,0 @@ -pub(crate) use _locale::make_module; - -#[cfg(windows)] -#[repr(C)] -struct lconv { - decimal_point: *mut libc::c_char, - thousands_sep: *mut libc::c_char, - grouping: *mut libc::c_char, - int_curr_symbol: *mut libc::c_char, - currency_symbol: *mut libc::c_char, - mon_decimal_point: *mut libc::c_char, - mon_thousands_sep: *mut libc::c_char, - mon_grouping: *mut libc::c_char, - positive_sign: *mut libc::c_char, - negative_sign: *mut libc::c_char, - int_frac_digits: libc::c_char, - frac_digits: libc::c_char, - p_cs_precedes: libc::c_char, - p_sep_by_space: libc::c_char, - n_cs_precedes: libc::c_char, - n_sep_by_space: libc::c_char, - p_sign_posn: libc::c_char, - n_sign_posn: libc::c_char, - int_p_cs_precedes: libc::c_char, - int_n_cs_precedes: libc::c_char, - int_p_sep_by_space: libc::c_char, - int_n_sep_by_space: libc::c_char, - int_p_sign_posn: libc::c_char, - int_n_sign_posn: libc::c_char, -} - -#[cfg(windows)] -extern "C" { - fn localeconv() -> *mut lconv; -} - -#[cfg(unix)] -use libc::localeconv; - -#[pymodule] -mod _locale { - use rustpython_vm::{ - builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef}, - convert::ToPyException, - function::OptionalArg, - PyObjectRef, PyResult, VirtualMachine, - }; - use std::{ - ffi::{CStr, CString}, - ptr, - }; - - #[cfg(all( - unix, - not(any(target_os = "ios", target_os = "android", target_os = "redox")) - ))] - #[pyattr] - use libc::{ - ABDAY_1, ABDAY_2, ABDAY_3, ABDAY_4, ABDAY_5, ABDAY_6, ABDAY_7, ABMON_1, ABMON_10, ABMON_11, - ABMON_12, ABMON_2, ABMON_3, ABMON_4, ABMON_5, ABMON_6, ABMON_7, ABMON_8, ABMON_9, - ALT_DIGITS, AM_STR, CODESET, CRNCYSTR, DAY_1, DAY_2, DAY_3, DAY_4, DAY_5, DAY_6, DAY_7, - D_FMT, D_T_FMT, ERA, ERA_D_FMT, ERA_D_T_FMT, ERA_T_FMT, LC_MESSAGES, MON_1, MON_10, MON_11, - MON_12, MON_2, MON_3, MON_4, MON_5, MON_6, MON_7, MON_8, MON_9, NOEXPR, PM_STR, RADIXCHAR, - THOUSEP, T_FMT, T_FMT_AMPM, YESEXPR, - }; - - #[pyattr] - use libc::{LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, LC_NUMERIC, LC_TIME}; - - #[pyattr(name = "CHAR_MAX")] - fn char_max(vm: &VirtualMachine) -> PyIntRef { - vm.ctx.new_int(libc::c_char::MAX) - } - - unsafe fn copy_grouping(group: *const libc::c_char, vm: &VirtualMachine) -> PyListRef { - let mut group_vec: Vec<PyObjectRef> = Vec::new(); - if group.is_null() { - return vm.ctx.new_list(group_vec); - } - - let mut ptr = group; - while ![0, libc::c_char::MAX].contains(&*ptr) { - let val = vm.ctx.new_int(*ptr); - group_vec.push(val.into()); - ptr = ptr.add(1); - } - // https://github.com/python/cpython/blob/677320348728ce058fa3579017e985af74a236d4/Modules/_localemodule.c#L80 - if !group_vec.is_empty() { - group_vec.push(vm.ctx.new_int(0).into()); - } - vm.ctx.new_list(group_vec) - } - - unsafe fn pystr_from_raw_cstr(vm: &VirtualMachine, raw_ptr: *const libc::c_char) -> PyResult { - let slice = unsafe { CStr::from_ptr(raw_ptr) }; - let string = slice - .to_str() - .expect("localeconv always return decodable string"); - Ok(vm.new_pyobj(string)) - } - - #[pyattr(name = "Error", once)] - fn error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "locale", - "Error", - Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), - ) - } - - #[pyfunction] - fn strcoll(string1: PyStrRef, string2: PyStrRef, vm: &VirtualMachine) -> PyResult { - let cstr1 = CString::new(string1.as_str()).map_err(|e| e.to_pyexception(vm))?; - let cstr2 = CString::new(string2.as_str()).map_err(|e| e.to_pyexception(vm))?; - Ok(vm.new_pyobj(unsafe { libc::strcoll(cstr1.as_ptr(), cstr2.as_ptr()) })) - } - - #[pyfunction] - fn strxfrm(string: PyStrRef, vm: &VirtualMachine) -> PyResult { - // https://github.com/python/cpython/blob/eaae563b6878aa050b4ad406b67728b6b066220e/Modules/_localemodule.c#L390-L442 - let n1 = string.byte_len() + 1; - let mut buff = vec![0u8; n1]; - - let cstr = CString::new(string.as_str()).map_err(|e| e.to_pyexception(vm))?; - let n2 = unsafe { libc::strxfrm(buff.as_mut_ptr() as _, cstr.as_ptr(), n1) }; - buff = vec![0u8; n2 + 1]; - unsafe { - libc::strxfrm(buff.as_mut_ptr() as _, cstr.as_ptr(), n2 + 1); - } - Ok(vm.new_pyobj(String::from_utf8(buff).expect("strxfrm returned invalid utf-8 string"))) - } - - #[pyfunction] - fn localeconv(vm: &VirtualMachine) -> PyResult<PyDictRef> { - let result = vm.ctx.new_dict(); - - unsafe { - macro_rules! set_string_field { - ($lc:expr, $field:ident) => {{ - result.set_item( - stringify!($field), - pystr_from_raw_cstr(vm, (*$lc).$field)?, - vm, - )? - }}; - } - - macro_rules! set_int_field { - ($lc:expr, $field:ident) => {{ - result.set_item(stringify!($field), vm.new_pyobj((*$lc).$field), vm)? - }}; - } - - macro_rules! set_group_field { - ($lc:expr, $field:ident) => {{ - result.set_item( - stringify!($field), - copy_grouping((*$lc).$field, vm).into(), - vm, - )? - }}; - } - - let lc = super::localeconv(); - set_group_field!(lc, mon_grouping); - set_group_field!(lc, grouping); - set_int_field!(lc, int_frac_digits); - set_int_field!(lc, frac_digits); - set_int_field!(lc, p_cs_precedes); - set_int_field!(lc, p_sep_by_space); - set_int_field!(lc, n_cs_precedes); - set_int_field!(lc, p_sign_posn); - set_int_field!(lc, n_sign_posn); - set_string_field!(lc, decimal_point); - set_string_field!(lc, thousands_sep); - set_string_field!(lc, int_curr_symbol); - set_string_field!(lc, currency_symbol); - set_string_field!(lc, mon_decimal_point); - set_string_field!(lc, mon_thousands_sep); - set_int_field!(lc, n_sep_by_space); - set_string_field!(lc, positive_sign); - set_string_field!(lc, negative_sign); - } - Ok(result) - } - - #[derive(FromArgs)] - struct LocaleArgs { - #[pyarg(any)] - category: i32, - #[pyarg(any, optional)] - locale: OptionalArg<Option<PyStrRef>>, - } - - #[pyfunction] - fn setlocale(args: LocaleArgs, vm: &VirtualMachine) -> PyResult { - let error = error(vm); - if cfg!(windows) && (args.category < LC_ALL || args.category > LC_TIME) { - return Err(vm.new_exception_msg(error, String::from("unsupported locale setting"))); - } - unsafe { - let result = match args.locale.flatten() { - None => libc::setlocale(args.category, ptr::null()), - Some(locale) => { - let c_locale: CString = - CString::new(locale.as_str()).map_err(|e| e.to_pyexception(vm))?; - libc::setlocale(args.category, c_locale.as_ptr()) - } - }; - if result.is_null() { - return Err(vm.new_exception_msg(error, String::from("unsupported locale setting"))); - } - pystr_from_raw_cstr(vm, result) - } - } -} diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs deleted file mode 100644 index bc977e2a531..00000000000 --- a/stdlib/src/math.rs +++ /dev/null @@ -1,913 +0,0 @@ -pub(crate) use math::make_module; - -#[pymodule] -mod math { - use crate::vm::{ - builtins::{try_bigint_to_f64, try_f64_to_bigint, PyFloat, PyInt, PyIntRef, PyStrInterned}, - function::{ArgIndex, ArgIntoFloat, ArgIterable, Either, OptionalArg, PosArgs}, - identifier, PyObject, PyObjectRef, PyRef, PyResult, VirtualMachine, - }; - use itertools::Itertools; - use malachite_bigint::BigInt; - use num_traits::{One, Signed, Zero}; - use rustpython_common::{float_ops, int::true_div}; - use std::cmp::Ordering; - - // Constants - #[pyattr] - use std::f64::consts::{E as e, PI as pi, TAU as tau}; - #[pyattr] - use std::f64::{INFINITY as inf, NAN as nan}; - - // Helper macro: - macro_rules! call_math_func { - ( $fun:ident, $name:ident, $vm:ident ) => {{ - let value = *$name; - let result = value.$fun(); - result_or_overflow(value, result, $vm) - }}; - } - - #[inline] - fn result_or_overflow(value: f64, result: f64, vm: &VirtualMachine) -> PyResult<f64> { - if !result.is_finite() && value.is_finite() { - // CPython doesn't return `inf` when called with finite - // values, it raises OverflowError instead. - Err(vm.new_overflow_error("math range error".to_owned())) - } else { - Ok(result) - } - } - - // Number theory functions: - #[pyfunction] - fn fabs(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(abs, x, vm) - } - - #[pyfunction] - fn isfinite(x: ArgIntoFloat) -> bool { - x.is_finite() - } - - #[pyfunction] - fn isinf(x: ArgIntoFloat) -> bool { - x.is_infinite() - } - - #[pyfunction] - fn isnan(x: ArgIntoFloat) -> bool { - x.is_nan() - } - - #[derive(FromArgs)] - struct IsCloseArgs { - #[pyarg(positional)] - a: ArgIntoFloat, - #[pyarg(positional)] - b: ArgIntoFloat, - #[pyarg(named, optional)] - rel_tol: OptionalArg<ArgIntoFloat>, - #[pyarg(named, optional)] - abs_tol: OptionalArg<ArgIntoFloat>, - } - - #[pyfunction] - fn isclose(args: IsCloseArgs, vm: &VirtualMachine) -> PyResult<bool> { - let a = *args.a; - let b = *args.b; - let rel_tol = args.rel_tol.map_or(1e-09, |value| value.into()); - let abs_tol = args.abs_tol.map_or(0.0, |value| value.into()); - - if rel_tol < 0.0 || abs_tol < 0.0 { - return Err(vm.new_value_error("tolerances must be non-negative".to_owned())); - } - - if a == b { - /* short circuit exact equality -- needed to catch two infinities of - the same sign. And perhaps speeds things up a bit sometimes. - */ - return Ok(true); - } - - /* This catches the case of two infinities of opposite sign, or - one infinity and one finite number. Two infinities of opposite - sign would otherwise have an infinite relative tolerance. - Two infinities of the same sign are caught by the equality check - above. - */ - - if a.is_infinite() || b.is_infinite() { - return Ok(false); - } - - let diff = (b - a).abs(); - - Ok((diff <= (rel_tol * b).abs()) || (diff <= (rel_tol * a).abs()) || (diff <= abs_tol)) - } - - #[pyfunction] - fn copysign(x: ArgIntoFloat, y: ArgIntoFloat) -> f64 { - if x.is_nan() || y.is_nan() { - x.into() - } else { - x.copysign(*y) - } - } - - // Power and logarithmic functions: - #[pyfunction] - fn exp(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(exp, x, vm) - } - - #[pyfunction] - fn exp2(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(exp2, x, vm) - } - - #[pyfunction] - fn expm1(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(exp_m1, x, vm) - } - - #[pyfunction] - fn log(x: PyObjectRef, base: OptionalArg<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { - let base = base.map(|b| *b).unwrap_or(std::f64::consts::E); - log2(x, vm).map(|logx| logx / base.log2()) - } - - #[pyfunction] - fn log1p(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x.is_nan() || x > -1.0_f64 { - Ok((x + 1.0_f64).ln()) - } else { - Err(vm.new_value_error("math domain error".to_owned())) - } - } - - /// Generates the base-2 logarithm of a BigInt `x` - fn int_log2(x: &BigInt) -> f64 { - // log2(x) = log2(2^n * 2^-n * x) = n + log2(x/2^n) - // If we set 2^n to be the greatest power of 2 below x, then x/2^n is in [1, 2), and can - // thus be converted into a float. - let n = x.bits() as u32 - 1; - let frac = true_div(x, &BigInt::from(2).pow(n)); - f64::from(n) + frac.log2() - } - - #[pyfunction] - fn log2(x: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { - match x.try_float(vm) { - Ok(x) => { - let x = x.to_f64(); - if x.is_nan() || x > 0.0_f64 { - Ok(x.log2()) - } else { - Err(vm.new_value_error("math domain error".to_owned())) - } - } - Err(float_err) => { - if let Ok(x) = x.try_int(vm) { - let x = x.as_bigint(); - if x.is_positive() { - Ok(int_log2(x)) - } else { - Err(vm.new_value_error("math domain error".to_owned())) - } - } else { - // Return the float error, as it will be more intuitive to users - Err(float_err) - } - } - } - } - - #[pyfunction] - fn log10(x: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { - log2(x, vm).map(|logx| logx / 10f64.log2()) - } - - #[pyfunction] - fn pow(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - let y = *y; - - if x < 0.0 && x.is_finite() && y.fract() != 0.0 && y.is_finite() { - return Err(vm.new_value_error("math domain error".to_owned())); - } - - if x == 0.0 && y < 0.0 && y != f64::NEG_INFINITY { - return Err(vm.new_value_error("math domain error".to_owned())); - } - - let value = x.powf(y); - - Ok(value) - } - - #[pyfunction] - fn sqrt(value: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let value = *value; - if value.is_nan() { - return Ok(value); - } - if value.is_sign_negative() { - return Err(vm.new_value_error("math domain error".to_owned())); - } - Ok(value.sqrt()) - } - - #[pyfunction] - fn isqrt(x: ArgIndex, vm: &VirtualMachine) -> PyResult<BigInt> { - let value = x.as_bigint(); - - if value.is_negative() { - return Err(vm.new_value_error("isqrt() argument must be nonnegative".to_owned())); - } - Ok(value.sqrt()) - } - - // Trigonometric functions: - #[pyfunction] - fn acos(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x.is_nan() || (-1.0_f64..=1.0_f64).contains(&x) { - Ok(x.acos()) - } else { - Err(vm.new_value_error("math domain error".to_owned())) - } - } - - #[pyfunction] - fn asin(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x.is_nan() || (-1.0_f64..=1.0_f64).contains(&x) { - Ok(x.asin()) - } else { - Err(vm.new_value_error("math domain error".to_owned())) - } - } - - #[pyfunction] - fn atan(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(atan, x, vm) - } - - #[pyfunction] - fn atan2(y: ArgIntoFloat, x: ArgIntoFloat) -> f64 { - y.atan2(*x) - } - - #[pyfunction] - fn cos(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(cos, x, vm) - } - - #[pyfunction] - fn hypot(coordinates: PosArgs<ArgIntoFloat>) -> f64 { - let mut coordinates = ArgIntoFloat::vec_into_f64(coordinates.into_vec()); - let mut max = 0.0; - let mut has_nan = false; - for f in &mut coordinates { - *f = f.abs(); - if f.is_nan() { - has_nan = true; - } else if *f > max { - max = *f - } - } - // inf takes precedence over nan - if max.is_infinite() { - return max; - } - if has_nan { - return f64::NAN; - } - coordinates.sort_unstable_by(|x, y| x.total_cmp(y).reverse()); - vector_norm(&coordinates) - } - - /// Implementation of accurate hypotenuse algorithm from Borges 2019. - /// See https://arxiv.org/abs/1904.09481. - /// This assumes that its arguments are positive finite and have been scaled to avoid overflow - /// and underflow. - fn accurate_hypot(max: f64, min: f64) -> f64 { - if min <= max * (f64::EPSILON / 2.0).sqrt() { - return max; - } - let hypot = max.mul_add(max, min * min).sqrt(); - let hypot_sq = hypot * hypot; - let max_sq = max * max; - let correction = (-min).mul_add(min, hypot_sq - max_sq) + hypot.mul_add(hypot, -hypot_sq) - - max.mul_add(max, -max_sq); - hypot - correction / (2.0 * hypot) - } - - /// Calculates the norm of the vector given by `v`. - /// `v` is assumed to be a list of non-negative finite floats, sorted in descending order. - fn vector_norm(v: &[f64]) -> f64 { - // Drop zeros from the vector. - let zero_count = v.iter().rev().cloned().take_while(|x| *x == 0.0).count(); - let v = &v[..v.len() - zero_count]; - if v.is_empty() { - return 0.0; - } - if v.len() == 1 { - return v[0]; - } - // Calculate scaling to avoid overflow / underflow. - let max = *v.first().unwrap(); - let min = *v.last().unwrap(); - let scale = if max > (f64::MAX / v.len() as f64).sqrt() { - max - } else if min < f64::MIN_POSITIVE.sqrt() { - // ^ This can be an `else if`, because if the max is near f64::MAX and the min is near - // f64::MIN_POSITIVE, then the min is relatively unimportant and will be effectively - // ignored. - min - } else { - 1.0 - }; - let mut norm = v - .iter() - .copied() - .map(|x| x / scale) - .reduce(accurate_hypot) - .unwrap_or_default(); - if v.len() > 2 { - // For larger lists of numbers, we can accumulate a rounding error, so a correction is - // needed, similar to that in `accurate_hypot()`. - // First, we estimate [sum of squares - norm^2], then we add the first-order - // approximation of the square root of that to `norm`. - let correction = v - .iter() - .copied() - .map(|x| (x / scale).powi(2)) - .chain(std::iter::once(-norm * norm)) - // Pairwise summation of floats gives less rounding error than a naive sum. - .tree_fold1(std::ops::Add::add) - .expect("expected at least 1 element"); - norm = norm + correction / (2.0 * norm); - } - norm * scale - } - - #[pyfunction] - fn dist(p: Vec<ArgIntoFloat>, q: Vec<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { - let mut max = 0.0; - let mut has_nan = false; - - let p = ArgIntoFloat::vec_into_f64(p); - let q = ArgIntoFloat::vec_into_f64(q); - let mut diffs = vec![]; - - if p.len() != q.len() { - return Err(vm.new_value_error( - "both points must have the same number of dimensions".to_owned(), - )); - } - - for i in 0..p.len() { - let px = p[i]; - let qx = q[i]; - - let x = (px - qx).abs(); - if x.is_nan() { - has_nan = true; - } - - diffs.push(x); - if x > max { - max = x; - } - } - - if max.is_infinite() { - return Ok(max); - } - if has_nan { - return Ok(f64::NAN); - } - diffs.sort_unstable_by(|x, y| x.total_cmp(y).reverse()); - Ok(vector_norm(&diffs)) - } - - #[pyfunction] - fn sin(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(sin, x, vm) - } - - #[pyfunction] - fn tan(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(tan, x, vm) - } - - #[pyfunction] - fn degrees(x: ArgIntoFloat) -> f64 { - *x * (180.0 / std::f64::consts::PI) - } - - #[pyfunction] - fn radians(x: ArgIntoFloat) -> f64 { - *x * (std::f64::consts::PI / 180.0) - } - - // Hyperbolic functions: - - #[pyfunction] - fn acosh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x.is_sign_negative() || x.is_zero() { - Err(vm.new_value_error("math domain error".to_owned())) - } else { - Ok(x.acosh()) - } - } - - #[pyfunction] - fn asinh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(asinh, x, vm) - } - - #[pyfunction] - fn atanh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - if x >= 1.0_f64 || x <= -1.0_f64 { - Err(vm.new_value_error("math domain error".to_owned())) - } else { - Ok(x.atanh()) - } - } - - #[pyfunction] - fn cosh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(cosh, x, vm) - } - - #[pyfunction] - fn sinh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(sinh, x, vm) - } - - #[pyfunction] - fn tanh(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - call_math_func!(tanh, x, vm) - } - - // Special functions: - #[pyfunction] - fn erf(x: ArgIntoFloat) -> f64 { - let x = *x; - if x.is_nan() { - x - } else { - puruspe::erf(x) - } - } - - #[pyfunction] - fn erfc(x: ArgIntoFloat) -> f64 { - let x = *x; - if x.is_nan() { - x - } else { - puruspe::erfc(x) - } - } - - #[pyfunction] - fn gamma(x: ArgIntoFloat) -> f64 { - let x = *x; - if x.is_finite() { - puruspe::gamma(x) - } else if x.is_nan() || x.is_sign_positive() { - x - } else { - f64::NAN - } - } - - #[pyfunction] - fn lgamma(x: ArgIntoFloat) -> f64 { - let x = *x; - if x.is_finite() { - puruspe::ln_gamma(x) - } else if x.is_nan() { - x - } else { - f64::INFINITY - } - } - - fn try_magic_method( - func_name: &'static PyStrInterned, - vm: &VirtualMachine, - value: &PyObject, - ) -> PyResult { - let method = vm.get_method_or_type_error(value.to_owned(), func_name, || { - format!( - "type '{}' doesn't define '{}' method", - value.class().name(), - func_name.as_str(), - ) - })?; - method.call((), vm) - } - - #[pyfunction] - fn trunc(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - try_magic_method(identifier!(vm, __trunc__), vm, &x) - } - - #[pyfunction] - fn ceil(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let result_or_err = try_magic_method(identifier!(vm, __ceil__), vm, &x); - if result_or_err.is_err() { - if let Some(v) = x.try_float_opt(vm) { - let v = try_f64_to_bigint(v?.to_f64().ceil(), vm)?; - return Ok(vm.ctx.new_int(v).into()); - } - } - result_or_err - } - - #[pyfunction] - fn floor(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let result_or_err = try_magic_method(identifier!(vm, __floor__), vm, &x); - if result_or_err.is_err() { - if let Some(v) = x.try_float_opt(vm) { - let v = try_f64_to_bigint(v?.to_f64().floor(), vm)?; - return Ok(vm.ctx.new_int(v).into()); - } - } - result_or_err - } - - #[pyfunction] - fn frexp(x: ArgIntoFloat) -> (f64, i32) { - let value = *x; - if value.is_finite() { - let (m, exp) = float_ops::ufrexp(value); - (m * value.signum(), exp) - } else { - (value, 0) - } - } - - #[pyfunction] - fn ldexp( - x: Either<PyRef<PyFloat>, PyIntRef>, - i: PyIntRef, - vm: &VirtualMachine, - ) -> PyResult<f64> { - let value = match x { - Either::A(f) => f.to_f64(), - Either::B(z) => try_bigint_to_f64(z.as_bigint(), vm)?, - }; - - if value == 0_f64 || !value.is_finite() { - // NaNs, zeros and infinities are returned unchanged - Ok(value) - } else { - let result = value * (2_f64).powf(try_bigint_to_f64(i.as_bigint(), vm)?); - result_or_overflow(value, result, vm) - } - } - - fn math_perf_arb_len_int_op<F>(args: PosArgs<ArgIndex>, op: F, default: BigInt) -> BigInt - where - F: Fn(&BigInt, &PyInt) -> BigInt, - { - let argvec = args.into_vec(); - - if argvec.is_empty() { - return default; - } else if argvec.len() == 1 { - return op(argvec[0].as_bigint(), &argvec[0]); - } - - let mut res = argvec[0].as_bigint().clone(); - for num in &argvec[1..] { - res = op(&res, num) - } - res - } - - #[pyfunction] - fn gcd(args: PosArgs<ArgIndex>) -> BigInt { - use num_integer::Integer; - math_perf_arb_len_int_op(args, |x, y| x.gcd(y.as_bigint()), BigInt::zero()) - } - - #[pyfunction] - fn lcm(args: PosArgs<ArgIndex>) -> BigInt { - use num_integer::Integer; - math_perf_arb_len_int_op(args, |x, y| x.lcm(y.as_bigint()), BigInt::one()) - } - - #[pyfunction] - fn cbrt(x: ArgIntoFloat) -> f64 { - x.cbrt() - } - - #[pyfunction] - fn fsum(seq: ArgIterable<ArgIntoFloat>, vm: &VirtualMachine) -> PyResult<f64> { - let mut partials = vec![]; - let mut special_sum = 0.0; - let mut inf_sum = 0.0; - - for obj in seq.iter(vm)? { - let mut x = *obj?; - - let xsave = x; - let mut j = 0; - // This inner loop applies `hi`/`lo` summation to each - // partial so that the list of partial sums remains exact. - for i in 0..partials.len() { - let mut y: f64 = partials[i]; - if x.abs() < y.abs() { - std::mem::swap(&mut x, &mut y); - } - // Rounded `x+y` is stored in `hi` with round-off stored in - // `lo`. Together `hi+lo` are exactly equal to `x+y`. - let hi = x + y; - let lo = y - (hi - x); - if lo != 0.0 { - partials[j] = lo; - j += 1; - } - x = hi; - } - - if !x.is_finite() { - // a nonfinite x could arise either as - // a result of intermediate overflow, or - // as a result of a nan or inf in the - // summands - if xsave.is_finite() { - return Err(vm.new_overflow_error("intermediate overflow in fsum".to_owned())); - } - if xsave.is_infinite() { - inf_sum += xsave; - } - special_sum += xsave; - // reset partials - partials.clear(); - } - - if j >= partials.len() { - partials.push(x); - } else { - partials[j] = x; - partials.truncate(j + 1); - } - } - if special_sum != 0.0 { - return if inf_sum.is_nan() { - Err(vm.new_value_error("-inf + inf in fsum".to_owned())) - } else { - Ok(special_sum) - }; - } - - let mut n = partials.len(); - if n > 0 { - n -= 1; - let mut hi = partials[n]; - - let mut lo = 0.0; - while n > 0 { - let x = hi; - - n -= 1; - let y = partials[n]; - - hi = x + y; - lo = y - (hi - x); - if lo != 0.0 { - break; - } - } - if n > 0 && ((lo < 0.0 && partials[n - 1] < 0.0) || (lo > 0.0 && partials[n - 1] > 0.0)) - { - let y = lo + lo; - let x = hi + y; - - // Make half-even rounding work across multiple partials. - // Needed so that sum([1e-16, 1, 1e16]) will round-up the last - // digit to two instead of down to zero (the 1e-16 makes the 1 - // slightly closer to two). With a potential 1 ULP rounding - // error fixed-up, math.fsum() can guarantee commutativity. - if y == x - hi { - hi = x; - } - } - - Ok(hi) - } else { - Ok(0.0) - } - } - - #[pyfunction] - fn factorial(x: PyIntRef, vm: &VirtualMachine) -> PyResult<BigInt> { - let value = x.as_bigint(); - let one = BigInt::one(); - if value.is_negative() { - return Err( - vm.new_value_error("factorial() not defined for negative values".to_owned()) - ); - } else if *value <= one { - return Ok(one); - } - // start from 2, since we know that value > 1 and 1*2=2 - let mut current = one + 1; - let mut product = BigInt::from(2u8); - while current < *value { - current += 1; - product *= &current; - } - Ok(product) - } - - #[pyfunction] - fn perm( - n: ArgIndex, - k: OptionalArg<Option<ArgIndex>>, - vm: &VirtualMachine, - ) -> PyResult<BigInt> { - let n = n.as_bigint(); - let k_ref; - let v = match k.flatten() { - Some(k) => { - k_ref = k; - k_ref.as_bigint() - } - None => n, - }; - - if n.is_negative() || v.is_negative() { - return Err(vm.new_value_error("perm() not defined for negative values".to_owned())); - } - if v > n { - return Ok(BigInt::zero()); - } - let mut result = BigInt::one(); - let mut current = n.clone(); - let tmp = n - v; - while current > tmp { - result *= &current; - current -= 1; - } - Ok(result) - } - - #[pyfunction] - fn comb(n: ArgIndex, k: ArgIndex, vm: &VirtualMachine) -> PyResult<BigInt> { - let mut k = k.as_bigint(); - let n = n.as_bigint(); - let one = BigInt::one(); - let zero = BigInt::zero(); - - if n.is_negative() || k.is_negative() { - return Err(vm.new_value_error("comb() not defined for negative values".to_owned())); - } - - let temp = n - k; - if temp.is_negative() { - return Ok(zero); - } - - if temp < *k { - k = &temp - } - - if k.is_zero() { - return Ok(one); - } - - let mut result = n.clone(); - let mut factor = n.clone(); - let mut current = one; - while current < *k { - factor -= 1; - current += 1; - - result *= &factor; - result /= &current; - } - - Ok(result) - } - - #[pyfunction] - fn modf(x: ArgIntoFloat) -> (f64, f64) { - let x = *x; - if !x.is_finite() { - if x.is_infinite() { - return (0.0_f64.copysign(x), x); - } else if x.is_nan() { - return (x, x); - } - } - - (x.fract(), x.trunc()) - } - - #[pyfunction] - fn nextafter(x: ArgIntoFloat, y: ArgIntoFloat) -> f64 { - float_ops::nextafter(*x, *y) - } - - #[pyfunction] - fn ulp(x: ArgIntoFloat) -> f64 { - float_ops::ulp(*x) - } - - fn fmod(x: f64, y: f64) -> f64 { - if y.is_infinite() && x.is_finite() { - return x; - } - - x % y - } - - #[pyfunction(name = "fmod")] - fn py_fmod(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - let y = *y; - - let r = fmod(x, y); - - if r.is_nan() && !x.is_nan() && !y.is_nan() { - return Err(vm.new_value_error("math domain error".to_owned())); - } - - Ok(r) - } - - #[pyfunction] - fn remainder(x: ArgIntoFloat, y: ArgIntoFloat, vm: &VirtualMachine) -> PyResult<f64> { - let x = *x; - let y = *y; - - if x.is_finite() && y.is_finite() { - if y == 0.0 { - return Err(vm.new_value_error("math domain error".to_owned())); - } - - let absx = x.abs(); - let absy = y.abs(); - let modulus = absx % absy; - - let c = absy - modulus; - let r = match modulus.partial_cmp(&c) { - Some(Ordering::Less) => modulus, - Some(Ordering::Greater) => -c, - _ => modulus - 2.0 * fmod(0.5 * (absx - modulus), absy), - }; - - return Ok(1.0_f64.copysign(x) * r); - } - if x.is_infinite() && !y.is_nan() { - return Err(vm.new_value_error("math domain error".to_owned())); - } - if x.is_nan() || y.is_nan() { - return Ok(f64::NAN); - } - if y.is_infinite() { - Ok(x) - } else { - Err(vm.new_value_error("math domain error".to_owned())) - } - } - - #[derive(FromArgs)] - struct ProdArgs { - #[pyarg(positional)] - iterable: ArgIterable<PyObjectRef>, - #[pyarg(named, optional)] - start: OptionalArg<PyObjectRef>, - } - - #[pyfunction] - fn prod(args: ProdArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - let iter = args.iterable; - - let mut result = args.start.unwrap_or_else(|| vm.new_pyobj(1)); - - // TODO: CPython has optimized implementation for this - // refer: https://github.com/python/cpython/blob/main/Modules/mathmodule.c#L3093-L3193 - for obj in iter.iter(vm)? { - let obj = obj?; - - result = vm - ._mul(&result, &obj) - .map_err(|_| vm.new_type_error("math type error".to_owned()))?; - } - - Ok(result) - } -} diff --git a/stdlib/src/md5.rs b/stdlib/src/md5.rs deleted file mode 100644 index 833d217f5be..00000000000 --- a/stdlib/src/md5.rs +++ /dev/null @@ -1,12 +0,0 @@ -pub(crate) use _md5::make_module; - -#[pymodule] -mod _md5 { - use crate::hashlib::_hashlib::{local_md5, HashArgs}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; - - #[pyfunction] - fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_md5(args).into_pyobject(vm)) - } -} diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs deleted file mode 100644 index d3207e60a17..00000000000 --- a/stdlib/src/mmap.rs +++ /dev/null @@ -1,1121 +0,0 @@ -//! mmap module -pub(crate) use mmap::make_module; - -#[pymodule] -mod mmap { - use crate::common::{ - borrow::{BorrowedValue, BorrowedValueMut}, - lock::{MapImmutable, PyMutex, PyMutexGuard}, - }; - use crate::vm::{ - atomic_func, - builtins::{PyBytes, PyBytesRef, PyInt, PyIntRef, PyTypeRef}, - byte::{bytes_from_object, value_from_object}, - function::{ArgBytesLike, FuncArgs, OptionalArg}, - protocol::{ - BufferDescriptor, BufferMethods, PyBuffer, PyMappingMethods, PySequenceMethods, - }, - sliceable::{SaturatedSlice, SequenceIndex, SequenceIndexOp}, - types::{AsBuffer, AsMapping, AsSequence, Constructor, Representable}, - AsObject, FromArgs, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, VirtualMachine, - }; - use crossbeam_utils::atomic::AtomicCell; - use memmap2::{Advice, Mmap, MmapMut, MmapOptions}; - use nix::unistd; - use num_traits::Signed; - use std::fs::File; - use std::io::Write; - use std::ops::{Deref, DerefMut}; - #[cfg(unix)] - use std::os::unix::io::{FromRawFd, IntoRawFd, RawFd}; - - fn advice_try_from_i32(vm: &VirtualMachine, i: i32) -> PyResult<Advice> { - Ok(match i { - libc::MADV_NORMAL => Advice::Normal, - libc::MADV_RANDOM => Advice::Random, - libc::MADV_SEQUENTIAL => Advice::Sequential, - libc::MADV_WILLNEED => Advice::WillNeed, - libc::MADV_DONTNEED => Advice::DontNeed, - #[cfg(any(target_os = "linux", target_os = "macos", target_os = "ios"))] - libc::MADV_FREE => Advice::Free, - #[cfg(target_os = "linux")] - libc::MADV_DONTFORK => Advice::DontFork, - #[cfg(target_os = "linux")] - libc::MADV_DOFORK => Advice::DoFork, - #[cfg(target_os = "linux")] - libc::MADV_MERGEABLE => Advice::Mergeable, - #[cfg(target_os = "linux")] - libc::MADV_UNMERGEABLE => Advice::Unmergeable, - #[cfg(target_os = "linux")] - libc::MADV_HUGEPAGE => Advice::HugePage, - #[cfg(target_os = "linux")] - libc::MADV_NOHUGEPAGE => Advice::NoHugePage, - #[cfg(target_os = "linux")] - libc::MADV_REMOVE => Advice::Remove, - #[cfg(target_os = "linux")] - libc::MADV_DONTDUMP => Advice::DontDump, - #[cfg(target_os = "linux")] - libc::MADV_DODUMP => Advice::DoDump, - #[cfg(target_os = "linux")] - libc::MADV_HWPOISON => Advice::HwPoison, - _ => return Err(vm.new_value_error("Not a valid Advice value".to_owned())), - }) - } - - #[repr(C)] - #[derive(PartialEq, Eq, Debug)] - enum AccessMode { - Default = 0, - Read = 1, - Write = 2, - Copy = 3, - } - - impl<'a> TryFromBorrowedObject<'a> for AccessMode { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - let i = u32::try_from_borrowed_object(vm, obj)?; - Ok(match i { - 0 => Self::Default, - 1 => Self::Read, - 2 => Self::Write, - 3 => Self::Copy, - _ => return Err(vm.new_value_error("Not a valid AccessMode value".to_owned())), - }) - } - } - - #[pyattr] - use libc::{ - MADV_DONTNEED, MADV_NORMAL, MADV_RANDOM, MADV_SEQUENTIAL, MADV_WILLNEED, MAP_ANON, - MAP_ANONYMOUS, MAP_PRIVATE, MAP_SHARED, PROT_READ, PROT_WRITE, - }; - - #[cfg(target_os = "macos")] - #[pyattr] - use libc::{MADV_FREE_REUSABLE, MADV_FREE_REUSE}; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "fuchsia", - target_os = "freebsd", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple" - ))] - #[pyattr] - use libc::MADV_FREE; - - #[cfg(target_os = "linux")] - #[pyattr] - use libc::{ - MADV_DODUMP, MADV_DOFORK, MADV_DONTDUMP, MADV_DONTFORK, MADV_HUGEPAGE, MADV_HWPOISON, - MADV_MERGEABLE, MADV_NOHUGEPAGE, MADV_REMOVE, MADV_UNMERGEABLE, - }; - - #[cfg(any( - target_os = "android", - all( - target_os = "linux", - any( - target_arch = "aarch64", - target_arch = "arm", - target_arch = "powerpc", - target_arch = "powerpc64", - target_arch = "s390x", - target_arch = "x86", - target_arch = "x86_64", - target_arch = "sparc64" - ) - ) - ))] - #[pyattr] - use libc::MADV_SOFT_OFFLINE; - - #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))] - #[pyattr] - use libc::{MAP_DENYWRITE, MAP_EXECUTABLE, MAP_POPULATE}; - - #[pyattr] - const ACCESS_DEFAULT: u32 = AccessMode::Default as u32; - #[pyattr] - const ACCESS_READ: u32 = AccessMode::Read as u32; - #[pyattr] - const ACCESS_WRITE: u32 = AccessMode::Write as u32; - #[pyattr] - const ACCESS_COPY: u32 = AccessMode::Copy as u32; - - #[cfg(all(unix, not(target_arch = "wasm32")))] - #[pyattr(name = "PAGESIZE", once)] - fn page_size(_vm: &VirtualMachine) -> usize { - page_size::get() - } - - #[cfg(all(unix, not(target_arch = "wasm32")))] - #[pyattr(name = "ALLOCATIONGRANULARITY", once)] - fn granularity(_vm: &VirtualMachine) -> usize { - page_size::get_granularity() - } - - #[pyattr(name = "error", once)] - fn error_type(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.exceptions.os_error.to_owned() - } - - #[derive(Debug)] - enum MmapObj { - Write(MmapMut), - Read(Mmap), - } - - #[pyattr] - #[pyclass(name = "mmap")] - #[derive(Debug, PyPayload)] - struct PyMmap { - closed: AtomicCell<bool>, - mmap: PyMutex<Option<MmapObj>>, - fd: RawFd, - offset: libc::off_t, - size: AtomicCell<usize>, - pos: AtomicCell<usize>, // relative to offset - exports: AtomicCell<usize>, - access: AccessMode, - } - - #[derive(FromArgs)] - struct MmapNewArgs { - #[pyarg(any)] - fileno: RawFd, - #[pyarg(any)] - length: isize, - #[pyarg(any, default = "MAP_SHARED")] - flags: libc::c_int, - #[pyarg(any, default = "PROT_WRITE|PROT_READ")] - prot: libc::c_int, - #[pyarg(any, default = "AccessMode::Default")] - access: AccessMode, - #[pyarg(any, default = "0")] - offset: libc::off_t, - } - - #[derive(FromArgs)] - pub struct FlushOptions { - #[pyarg(positional, default)] - offset: Option<isize>, - #[pyarg(positional, default)] - size: Option<isize>, - } - - impl FlushOptions { - fn values(self, len: usize) -> Option<(usize, usize)> { - let offset = if let Some(offset) = self.offset { - if offset < 0 { - return None; - } - offset as usize - } else { - 0 - }; - let size = if let Some(size) = self.size { - if size < 0 { - return None; - } - size as usize - } else { - len - }; - if len.checked_sub(offset)? < size { - return None; - } - Some((offset, size)) - } - } - - #[derive(FromArgs, Clone)] - pub struct FindOptions { - #[pyarg(positional)] - sub: Vec<u8>, - #[pyarg(positional, default)] - start: Option<isize>, - #[pyarg(positional, default)] - end: Option<isize>, - } - - #[cfg(not(target_os = "redox"))] - #[derive(FromArgs)] - pub struct AdviseOptions { - #[pyarg(positional)] - option: libc::c_int, - #[pyarg(positional, default)] - start: Option<PyIntRef>, - #[pyarg(positional, default)] - length: Option<PyIntRef>, - } - - #[cfg(not(target_os = "redox"))] - impl AdviseOptions { - fn values(self, len: usize, vm: &VirtualMachine) -> PyResult<(libc::c_int, usize, usize)> { - let start = self - .start - .map(|s| { - s.try_to_primitive::<usize>(vm) - .ok() - .filter(|s| *s < len) - .ok_or_else(|| vm.new_value_error("madvise start out of bounds".to_owned())) - }) - .transpose()? - .unwrap_or(0); - let length = self - .length - .map(|s| { - s.try_to_primitive::<usize>(vm) - .map_err(|_| vm.new_value_error("madvise length invalid".to_owned())) - }) - .transpose()? - .unwrap_or(len); - - if isize::MAX as usize - start < length { - return Err(vm.new_overflow_error("madvise length too large".to_owned())); - } - - let length = if start + length > len { - len - start - } else { - length - }; - - Ok((self.option, start, length)) - } - } - - impl Constructor for PyMmap { - type Args = MmapNewArgs; - - // TODO: Windows is not supported right now. - #[cfg(unix)] - fn py_new( - cls: PyTypeRef, - MmapNewArgs { - fileno: mut fd, - length, - flags, - prot, - access, - offset, - }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let map_size = length; - if map_size < 0 { - return Err( - vm.new_overflow_error("memory mapped length must be positive".to_owned()) - ); - } - let mut map_size = map_size as usize; - - if offset < 0 { - return Err( - vm.new_overflow_error("memory mapped offset must be positive".to_owned()) - ); - } - - if (access != AccessMode::Default) - && ((flags != MAP_SHARED) || (prot != (PROT_WRITE | PROT_READ))) - { - return Err(vm.new_value_error( - "mmap can't specify both access and flags, prot.".to_owned(), - )); - } - - // TODO: memmap2 doesn't support mapping with pro and flags right now - let (_flags, _prot, access) = match access { - AccessMode::Read => (MAP_SHARED, PROT_READ, access), - AccessMode::Write => (MAP_SHARED, PROT_READ | PROT_WRITE, access), - AccessMode::Copy => (MAP_PRIVATE, PROT_READ | PROT_WRITE, access), - AccessMode::Default => { - let access = if (prot & PROT_READ) != 0 && (prot & PROT_WRITE) != 0 { - access - } else if (prot & PROT_WRITE) != 0 { - AccessMode::Write - } else { - AccessMode::Read - }; - (flags, prot, access) - } - }; - - if fd != -1 { - let file = unsafe { File::from_raw_fd(fd) }; - let metadata = file - .metadata() - .map_err(|e| vm.new_os_error(e.to_string()))?; - let file_len: libc::off_t = metadata.len().try_into().expect("file size overflow"); - // File::from_raw_fd will consume the fd, so we - // have to get it again. - fd = file.into_raw_fd(); - if map_size == 0 { - if file_len == 0 { - return Err(vm.new_value_error("cannot mmap an empty file".to_owned())); - } - - if offset > file_len { - return Err( - vm.new_value_error("mmap offset is greater than file size".to_owned()) - ); - } - - map_size = (file_len - offset) - .try_into() - .map_err(|_| vm.new_value_error("mmap length is too large".to_owned()))?; - } else if offset > file_len || file_len - offset < map_size as libc::off_t { - return Err( - vm.new_value_error("mmap length is greater than file size".to_owned()) - ); - } - } - - let mut mmap_opt = MmapOptions::new(); - let mmap_opt = mmap_opt.offset(offset.try_into().unwrap()).len(map_size); - - let (fd, mmap) = if fd == -1 { - ( - fd, - MmapObj::Write( - mmap_opt - .map_anon() - .map_err(|e| vm.new_os_error(e.to_string()))?, - ), - ) - } else { - let new_fd = unistd::dup(fd).map_err(|e| vm.new_os_error(e.to_string()))?; - let mmap = match access { - AccessMode::Default | AccessMode::Write => MmapObj::Write( - unsafe { mmap_opt.map_mut(fd) } - .map_err(|e| vm.new_os_error(e.to_string()))?, - ), - AccessMode::Read => MmapObj::Read( - unsafe { mmap_opt.map(fd) }.map_err(|e| vm.new_os_error(e.to_string()))?, - ), - AccessMode::Copy => MmapObj::Write( - unsafe { mmap_opt.map_copy(fd) } - .map_err(|e| vm.new_os_error(e.to_string()))?, - ), - }; - (new_fd, mmap) - }; - - let m_obj = Self { - closed: AtomicCell::new(false), - mmap: PyMutex::new(Some(mmap)), - fd, - offset, - size: AtomicCell::new(map_size), - pos: AtomicCell::new(0), - exports: AtomicCell::new(0), - access, - }; - - m_obj.into_ref_with_type(vm, cls).map(Into::into) - } - } - - static BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| buffer.obj_as::<PyMmap>().as_bytes(), - obj_bytes_mut: |buffer| buffer.obj_as::<PyMmap>().as_bytes_mut(), - release: |buffer| { - buffer.obj_as::<PyMmap>().exports.fetch_sub(1); - }, - retain: |buffer| { - buffer.obj_as::<PyMmap>().exports.fetch_add(1); - }, - }; - - impl AsBuffer for PyMmap { - fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { - let buf = PyBuffer::new( - zelf.to_owned().into(), - BufferDescriptor::simple(zelf.len(), true), - &BUFFER_METHODS, - ); - - Ok(buf) - } - } - - impl AsMapping for PyMmap { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: PyMappingMethods = PyMappingMethods { - length: atomic_func!(|mapping, _vm| Ok(PyMmap::mapping_downcast(mapping).len())), - subscript: atomic_func!(|mapping, needle, vm| { - PyMmap::mapping_downcast(mapping)._getitem(needle, vm) - }), - ass_subscript: atomic_func!(|mapping, needle, value, vm| { - let zelf = PyMmap::mapping_downcast(mapping); - if let Some(value) = value { - PyMmap::_setitem(zelf, needle, value, vm) - } else { - Err(vm - .new_type_error("mmap object doesn't support item deletion".to_owned())) - } - }), - }; - &AS_MAPPING - } - } - - impl AsSequence for PyMmap { - fn as_sequence() -> &'static PySequenceMethods { - use once_cell::sync::Lazy; - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyMmap::sequence_downcast(seq).len())), - item: atomic_func!(|seq, i, vm| { - let zelf = PyMmap::sequence_downcast(seq); - zelf.getitem_by_index(i, vm) - }), - ass_item: atomic_func!(|seq, i, value, vm| { - let zelf = PyMmap::sequence_downcast(seq); - if let Some(value) = value { - PyMmap::setitem_by_index(zelf, i, value, vm) - } else { - Err(vm - .new_type_error("mmap object doesn't support item deletion".to_owned())) - } - }), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } - } - - #[pyclass( - with(Constructor, AsMapping, AsSequence, AsBuffer, Representable), - flags(BASETYPE) - )] - impl PyMmap { - fn as_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - PyMutexGuard::map(self.mmap.lock(), |m| { - match m.as_mut().expect("mmap closed or invalid") { - MmapObj::Read(_) => panic!("mmap can't modify a readonly memory map."), - MmapObj::Write(mmap) => &mut mmap[..], - } - }) - .into() - } - - fn as_bytes(&self) -> BorrowedValue<[u8]> { - PyMutexGuard::map_immutable(self.mmap.lock(), |m| { - match m.as_ref().expect("mmap closed or invalid") { - MmapObj::Read(ref mmap) => &mmap[..], - MmapObj::Write(ref mmap) => &mmap[..], - } - }) - .into() - } - - #[pymethod(magic)] - fn len(&self) -> usize { - self.size.load() - } - - #[inline] - fn pos(&self) -> usize { - self.pos.load() - } - - #[inline] - fn advance_pos(&self, step: usize) { - self.pos.store(self.pos() + step); - } - - #[inline] - fn try_writable<R>( - &self, - vm: &VirtualMachine, - f: impl FnOnce(&mut MmapMut) -> R, - ) -> PyResult<R> { - if matches!(self.access, AccessMode::Read) { - return Err( - vm.new_type_error("mmap can't modify a readonly memory map.".to_owned()) - ); - } - - match self.check_valid(vm)?.deref_mut().as_mut().unwrap() { - MmapObj::Write(mmap) => Ok(f(mmap)), - _ => unreachable!("already check"), - } - } - - fn check_valid(&self, vm: &VirtualMachine) -> PyResult<PyMutexGuard<Option<MmapObj>>> { - let m = self.mmap.lock(); - - if m.is_none() { - return Err(vm.new_value_error("mmap closed or invalid".to_owned())); - } - - Ok(m) - } - - /// TODO: impl resize - #[allow(dead_code)] - fn check_resizeable(&self, vm: &VirtualMachine) -> PyResult<()> { - if self.exports.load() > 0 { - return Err(vm.new_buffer_error( - "mmap can't resize with extant buffers exported.".to_owned(), - )); - } - - if self.access == AccessMode::Write || self.access == AccessMode::Default { - return Ok(()); - } - - Err(vm.new_type_error( - "mmap can't resize a readonly or copy-on-write memory map.".to_owned(), - )) - } - - #[pygetset] - fn closed(&self) -> bool { - self.closed.load() - } - - #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult<()> { - if self.closed() { - return Ok(()); - } - - if self.exports.load() > 0 { - return Err(vm.new_buffer_error("cannot close exported pointers exist.".to_owned())); - } - let mut mmap = self.mmap.lock(); - self.closed.store(true); - *mmap = None; - - Ok(()) - } - - fn get_find_range(&self, options: FindOptions) -> (usize, usize) { - let size = self.len(); - let start = options - .start - .map(|start| start.saturated_at(size)) - .unwrap_or_else(|| self.pos()); - let end = options - .end - .map(|end| end.saturated_at(size)) - .unwrap_or(size); - (start, end) - } - - #[pymethod] - fn find(&self, options: FindOptions, vm: &VirtualMachine) -> PyResult<PyInt> { - let (start, end) = self.get_find_range(options.clone()); - - let sub = &options.sub; - - if sub.is_empty() { - return Ok(PyInt::from(0isize)); - } - - let mmap = self.check_valid(vm)?; - let buf = match mmap.as_ref().unwrap() { - MmapObj::Read(mmap) => &mmap[start..end], - MmapObj::Write(mmap) => &mmap[start..end], - }; - let pos = buf.windows(sub.len()).position(|window| window == sub); - - Ok(pos.map_or(PyInt::from(-1isize), |i| PyInt::from(start + i))) - } - - #[pymethod] - fn rfind(&self, options: FindOptions, vm: &VirtualMachine) -> PyResult<PyInt> { - let (start, end) = self.get_find_range(options.clone()); - - let sub = &options.sub; - if sub.is_empty() { - return Ok(PyInt::from(0isize)); - } - - let mmap = self.check_valid(vm)?; - let buf = match mmap.as_ref().unwrap() { - MmapObj::Read(mmap) => &mmap[start..end], - MmapObj::Write(mmap) => &mmap[start..end], - }; - let pos = buf.windows(sub.len()).rposition(|window| window == sub); - - Ok(pos.map_or(PyInt::from(-1isize), |i| PyInt::from(start + i))) - } - - #[pymethod] - fn flush(&self, options: FlushOptions, vm: &VirtualMachine) -> PyResult<()> { - let (offset, size) = options - .values(self.len()) - .ok_or_else(|| vm.new_value_error("flush values out of range".to_owned()))?; - - if self.access == AccessMode::Read || self.access == AccessMode::Copy { - return Ok(()); - } - - match self.check_valid(vm)?.deref().as_ref().unwrap() { - MmapObj::Read(_mmap) => {} - MmapObj::Write(mmap) => { - mmap.flush_range(offset, size) - .map_err(|e| vm.new_os_error(e.to_string()))?; - } - } - - Ok(()) - } - - #[cfg(not(target_os = "redox"))] - #[allow(unused_assignments)] - #[pymethod] - fn madvise(&self, options: AdviseOptions, vm: &VirtualMachine) -> PyResult<()> { - let (option, _start, _length) = options.values(self.len(), vm)?; - let advice = advice_try_from_i32(vm, option)?; - - //TODO: memmap2 doesn't support madvise range right now. - match self.check_valid(vm)?.deref().as_ref().unwrap() { - MmapObj::Read(mmap) => mmap.advise(advice), - MmapObj::Write(mmap) => mmap.advise(advice), - } - .map_err(|e| vm.new_os_error(e.to_string()))?; - - Ok(()) - } - - #[pymethod(name = "move")] - fn move_( - &self, - dest: PyIntRef, - src: PyIntRef, - cnt: PyIntRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - fn args( - dest: PyIntRef, - src: PyIntRef, - cnt: PyIntRef, - size: usize, - vm: &VirtualMachine, - ) -> Option<(usize, usize, usize)> { - if dest.as_bigint().is_negative() - || src.as_bigint().is_negative() - || cnt.as_bigint().is_negative() - { - return None; - } - let dest = dest.try_to_primitive(vm).ok()?; - let src = src.try_to_primitive(vm).ok()?; - let cnt = cnt.try_to_primitive(vm).ok()?; - if size - dest < cnt || size - src < cnt { - return None; - } - Some((dest, src, cnt)) - } - - let size = self.len(); - let (dest, src, cnt) = args(dest, src, cnt, size, vm).ok_or_else(|| { - vm.new_value_error("source, destination, or count out of range".to_owned()) - })?; - - let dest_end = dest + cnt; - let src_end = src + cnt; - - self.try_writable(vm, |mmap| { - let src_buf = mmap[src..src_end].to_vec(); - (&mut mmap[dest..dest_end]) - .write(&src_buf) - .map_err(|e| vm.new_os_error(e.to_string()))?; - Ok(()) - })? - } - - #[pymethod] - fn read(&self, n: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - let num_bytes = n - .map(|obj| { - let class = obj.class().to_owned(); - obj.try_into_value::<Option<isize>>(vm).map_err(|_| { - vm.new_type_error(format!( - "read argument must be int or None, not {}", - class.name() - )) - }) - }) - .transpose()? - .flatten(); - let mmap = self.check_valid(vm)?; - let pos = self.pos(); - let remaining = self.len().saturating_sub(pos); - let num_bytes = num_bytes - .filter(|&n| n >= 0 && (n as usize) <= remaining) - .map(|n| n as usize) - .unwrap_or(remaining); - - let end_pos = pos + num_bytes; - let bytes = match mmap.deref().as_ref().unwrap() { - MmapObj::Read(mmap) => mmap[pos..end_pos].to_vec(), - MmapObj::Write(mmap) => mmap[pos..end_pos].to_vec(), - }; - - let result = PyBytes::from(bytes).into_ref(&vm.ctx); - - self.advance_pos(num_bytes); - - Ok(result) - } - - #[pymethod] - fn read_byte(&self, vm: &VirtualMachine) -> PyResult<PyIntRef> { - let pos = self.pos(); - if pos >= self.len() { - return Err(vm.new_value_error("read byte out of range".to_owned())); - } - - let b = match self.check_valid(vm)?.deref().as_ref().unwrap() { - MmapObj::Read(mmap) => mmap[pos], - MmapObj::Write(mmap) => mmap[pos], - }; - - self.advance_pos(1); - - Ok(PyInt::from(b).into_ref(&vm.ctx)) - } - - #[pymethod] - fn readline(&self, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - let pos = self.pos(); - let mmap = self.check_valid(vm)?; - - let remaining = self.len().saturating_sub(pos); - if remaining == 0 { - return Ok(PyBytes::from(vec![]).into_ref(&vm.ctx)); - } - - let eof = match mmap.as_ref().unwrap() { - MmapObj::Read(mmap) => &mmap[pos..], - MmapObj::Write(mmap) => &mmap[pos..], - } - .iter() - .position(|&x| x == b'\n'); - - let end_pos = if let Some(i) = eof { - pos + i + 1 - } else { - self.len() - }; - - let bytes = match mmap.deref().as_ref().unwrap() { - MmapObj::Read(mmap) => mmap[pos..end_pos].to_vec(), - MmapObj::Write(mmap) => mmap[pos..end_pos].to_vec(), - }; - - let result = PyBytes::from(bytes).into_ref(&vm.ctx); - - self.advance_pos(end_pos - pos); - - Ok(result) - } - - // TODO: supports resize - #[pymethod] - fn resize(&self, _newsize: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { - self.check_resizeable(vm)?; - Err(vm.new_system_error("mmap: resizing not available--no mremap()".to_owned())) - } - - #[pymethod] - fn seek( - &self, - dist: isize, - whence: OptionalArg<libc::c_int>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let how = whence.unwrap_or(0); - let size = self.len(); - - let new_pos = match how { - 0 => dist, // relative to start - 1 => { - // relative to current position - let pos = self.pos(); - if (((isize::MAX as usize) - pos) as isize) < dist { - return Err(vm.new_value_error("seek out of range".to_owned())); - } - pos as isize + dist - } - 2 => { - // relative to end - if (((isize::MAX as usize) - size) as isize) < dist { - return Err(vm.new_value_error("seek out of range".to_owned())); - } - size as isize + dist - } - _ => return Err(vm.new_value_error("unknown seek type".to_owned())), - }; - - if new_pos < 0 || (new_pos as usize) > size { - return Err(vm.new_value_error("seek out of range".to_owned())); - } - - self.pos.store(new_pos as usize); - - Ok(()) - } - - #[pymethod] - fn size(&self, vm: &VirtualMachine) -> PyResult<PyIntRef> { - let new_fd = unistd::dup(self.fd).map_err(|e| vm.new_os_error(e.to_string()))?; - let file = unsafe { File::from_raw_fd(new_fd) }; - let file_len = match file.metadata() { - Ok(m) => m.len(), - Err(e) => return Err(vm.new_os_error(e.to_string())), - }; - - Ok(PyInt::from(file_len).into_ref(&vm.ctx)) - } - - #[pymethod] - fn tell(&self) -> PyResult<usize> { - Ok(self.pos()) - } - - #[pymethod] - fn write(&self, bytes: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyIntRef> { - let pos = self.pos(); - let size = self.len(); - - let data = bytes.borrow_buf(); - - if pos > size || size - pos < data.len() { - return Err(vm.new_value_error("data out of range".to_owned())); - } - - let len = self.try_writable(vm, |mmap| { - (&mut mmap[pos..(pos + data.len())]) - .write(&data) - .map_err(|e| vm.new_os_error(e.to_string()))?; - Ok(data.len()) - })??; - - self.advance_pos(len); - - Ok(PyInt::from(len).into_ref(&vm.ctx)) - } - - #[pymethod] - fn write_byte(&self, byte: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let b = value_from_object(vm, &byte)?; - - let pos = self.pos(); - let size = self.len(); - - if pos >= size { - return Err(vm.new_value_error("write byte out of range".to_owned())); - } - - self.try_writable(vm, |mmap| { - mmap[pos] = b; - })?; - - self.advance_pos(1); - - Ok(()) - } - - #[pymethod(magic)] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - self._getitem(&needle, vm) - } - - #[pymethod(magic)] - fn setitem( - zelf: &Py<Self>, - needle: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::_setitem(zelf, &needle, value, vm) - } - - #[pymethod(magic)] - fn enter(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - let _m = zelf.check_valid(vm)?; - Ok(zelf.to_owned()) - } - - #[pymethod(magic)] - fn exit(zelf: &Py<Self>, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - zelf.close(vm) - } - } - - impl PyMmap { - fn getitem_by_index(&self, i: isize, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - let i = i - .wrapped_at(self.len()) - .ok_or_else(|| vm.new_index_error("mmap index out of range".to_owned()))?; - - let b = match self.check_valid(vm)?.deref().as_ref().unwrap() { - MmapObj::Read(mmap) => mmap[i], - MmapObj::Write(mmap) => mmap[i], - }; - - Ok(PyInt::from(b).into_ref(&vm.ctx).into()) - } - - fn getitem_by_slice( - &self, - slice: &SaturatedSlice, - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { - let (range, step, slice_len) = slice.adjust_indices(self.len()); - - let mmap = self.check_valid(vm)?; - - if slice_len == 0 { - return Ok(PyBytes::from(vec![]).into_ref(&vm.ctx).into()); - } else if step == 1 { - let bytes = match mmap.deref().as_ref().unwrap() { - MmapObj::Read(mmap) => &mmap[range], - MmapObj::Write(mmap) => &mmap[range], - }; - return Ok(PyBytes::from(bytes.to_vec()).into_ref(&vm.ctx).into()); - } - - let mut result_buf = Vec::with_capacity(slice_len); - if step.is_negative() { - for i in range.rev().step_by(step.unsigned_abs()) { - let b = match mmap.deref().as_ref().unwrap() { - MmapObj::Read(mmap) => mmap[i], - MmapObj::Write(mmap) => mmap[i], - }; - result_buf.push(b); - } - } else { - for i in range.step_by(step.unsigned_abs()) { - let b = match mmap.deref().as_ref().unwrap() { - MmapObj::Read(mmap) => mmap[i], - MmapObj::Write(mmap) => mmap[i], - }; - result_buf.push(b); - } - } - Ok(PyBytes::from(result_buf).into_ref(&vm.ctx).into()) - } - - fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "mmap")? { - SequenceIndex::Int(i) => self.getitem_by_index(i, vm), - SequenceIndex::Slice(slice) => self.getitem_by_slice(&slice, vm), - } - } - - fn _setitem( - zelf: &Py<Self>, - needle: &PyObject, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "mmap")? { - SequenceIndex::Int(i) => Self::setitem_by_index(zelf, i, value, vm), - SequenceIndex::Slice(slice) => Self::setitem_by_slice(zelf, &slice, value, vm), - } - } - - fn setitem_by_index( - &self, - i: isize, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let i: usize = i - .wrapped_at(self.len()) - .ok_or_else(|| vm.new_index_error("mmap index out of range".to_owned()))?; - - let b = value_from_object(vm, &value)?; - - self.try_writable(vm, |mmap| { - mmap[i] = b; - })?; - - Ok(()) - } - - fn setitem_by_slice( - &self, - slice: &SaturatedSlice, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let (range, step, slice_len) = slice.adjust_indices(self.len()); - - let bytes = bytes_from_object(vm, &value)?; - - if bytes.len() != slice_len { - return Err(vm.new_index_error("mmap slice assignment is wrong size".to_owned())); - } - - if slice_len == 0 { - // do nothing - Ok(()) - } else if step == 1 { - self.try_writable(vm, |mmap| { - (&mut mmap[range]) - .write(&bytes) - .map_err(|e| vm.new_os_error(e.to_string()))?; - Ok(()) - })? - } else { - let mut bi = 0; // bytes index - if step.is_negative() { - for i in range.rev().step_by(step.unsigned_abs()) { - self.try_writable(vm, |mmap| { - mmap[i] = bytes[bi]; - })?; - bi += 1; - } - } else { - for i in range.step_by(step.unsigned_abs()) { - self.try_writable(vm, |mmap| { - mmap[i] = bytes[bi]; - })?; - bi += 1; - } - } - Ok(()) - } - } - } - - impl Representable for PyMmap { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - let mmap = zelf.mmap.lock(); - - if mmap.is_none() { - return Ok("<mmap.mmap closed=True>".to_owned()); - } - - let access_str = match zelf.access { - AccessMode::Default => "ACCESS_DEFAULT", - AccessMode::Read => "ACCESS_READ", - AccessMode::Write => "ACCESS_WRITE", - AccessMode::Copy => "ACCESS_COPY", - }; - - let repr = format!( - "<mmap.mmap closed=False, access={}, length={}, pos={}, offset={}>", - access_str, - zelf.len(), - zelf.pos(), - zelf.offset - ); - - Ok(repr) - } - } -} diff --git a/stdlib/src/multiprocessing.rs b/stdlib/src/multiprocessing.rs deleted file mode 100644 index a6d902eb63a..00000000000 --- a/stdlib/src/multiprocessing.rs +++ /dev/null @@ -1,46 +0,0 @@ -pub(crate) use _multiprocessing::make_module; - -#[cfg(windows)] -#[pymodule] -mod _multiprocessing { - use crate::vm::{function::ArgBytesLike, stdlib::os, PyResult, VirtualMachine}; - use windows_sys::Win32::Networking::WinSock::{self, SOCKET}; - - #[pyfunction] - fn closesocket(socket: usize, vm: &VirtualMachine) -> PyResult<()> { - let res = unsafe { WinSock::closesocket(socket as SOCKET) }; - if res == 0 { - Err(os::errno_err(vm)) - } else { - Ok(()) - } - } - - #[pyfunction] - fn recv(socket: usize, size: usize, vm: &VirtualMachine) -> PyResult<libc::c_int> { - let mut buf = vec![0; size]; - let nread = - unsafe { WinSock::recv(socket as SOCKET, buf.as_mut_ptr() as *mut _, size as i32, 0) }; - if nread < 0 { - Err(os::errno_err(vm)) - } else { - Ok(nread) - } - } - - #[pyfunction] - fn send(socket: usize, buf: ArgBytesLike, vm: &VirtualMachine) -> PyResult<libc::c_int> { - let ret = buf.with_ref(|b| unsafe { - WinSock::send(socket as SOCKET, b.as_ptr() as *const _, b.len() as i32, 0) - }); - if ret < 0 { - Err(os::errno_err(vm)) - } else { - Ok(ret) - } - } -} - -#[cfg(not(windows))] -#[pymodule] -mod _multiprocessing {} diff --git a/stdlib/src/posixsubprocess.rs b/stdlib/src/posixsubprocess.rs deleted file mode 100644 index 5f8cb3f5b92..00000000000 --- a/stdlib/src/posixsubprocess.rs +++ /dev/null @@ -1,256 +0,0 @@ -use crate::vm::{ - builtins::PyListRef, - function::ArgSequence, - stdlib::{os::OsPath, posix}, - {PyObjectRef, PyResult, TryFromObject, VirtualMachine}, -}; -use nix::{errno::Errno, unistd}; -#[cfg(not(target_os = "redox"))] -use std::ffi::CStr; -#[cfg(not(target_os = "redox"))] -use std::os::unix::io::AsRawFd; -use std::{ - convert::Infallible as Never, - ffi::CString, - io::{self, prelude::*}, -}; -use unistd::{Gid, Uid}; - -pub(crate) use _posixsubprocess::make_module; - -#[pymodule] -mod _posixsubprocess { - use super::{exec, CStrPathLike, ForkExecArgs, ProcArgs}; - use crate::vm::{convert::IntoPyException, PyResult, VirtualMachine}; - - #[pyfunction] - fn fork_exec(args: ForkExecArgs, vm: &VirtualMachine) -> PyResult<libc::pid_t> { - if args.preexec_fn.is_some() { - return Err(vm.new_not_implemented_error("preexec_fn not supported yet".to_owned())); - } - let cstrs_to_ptrs = |cstrs: &[CStrPathLike]| { - cstrs - .iter() - .map(|s| s.s.as_ptr()) - .chain(std::iter::once(std::ptr::null())) - .collect::<Vec<_>>() - }; - let argv = cstrs_to_ptrs(&args.args); - let argv = &argv; - let envp = args.env_list.as_ref().map(|s| cstrs_to_ptrs(s)); - let envp = envp.as_deref(); - match unsafe { nix::unistd::fork() }.map_err(|err| err.into_pyexception(vm))? { - nix::unistd::ForkResult::Child => exec(&args, ProcArgs { argv, envp }), - nix::unistd::ForkResult::Parent { child } => Ok(child.as_raw()), - } - } -} - -macro_rules! gen_args { - ($($field:ident: $t:ty),*$(,)?) => { - #[allow(dead_code)] - #[derive(FromArgs)] - struct ForkExecArgs { - $(#[pyarg(positional)] $field: $t,)* - } - }; -} - -struct CStrPathLike { - s: CString, -} -impl TryFromObject for CStrPathLike { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let s = OsPath::try_from_object(vm, obj)?.into_cstring(vm)?; - Ok(CStrPathLike { s }) - } -} - -gen_args! { - args: ArgSequence<CStrPathLike> /* list */, - exec_list: ArgSequence<CStrPathLike> /* list */, - close_fds: bool, - fds_to_keep: ArgSequence<i32>, - cwd: Option<CStrPathLike>, - env_list: Option<ArgSequence<CStrPathLike>>, - p2cread: i32, - p2cwrite: i32, - c2pread: i32, - c2pwrite: i32, - errread: i32, - errwrite: i32, - errpipe_read: i32, - errpipe_write: i32, - restore_signals: bool, - call_setsid: bool, - // TODO: Difference between gid_to_set and gid_object. - // One is a `gid_t` and the other is a `PyObject` in CPython. - gid_to_set: Option<Option<Gid>>, - gid_object: PyObjectRef, - groups_list: Option<PyListRef>, - uid: Option<Option<Uid>>, - child_umask: i32, - preexec_fn: Option<PyObjectRef>, - use_vfork: bool, -} - -// can't reallocate inside of exec(), so we reallocate prior to fork() and pass this along -struct ProcArgs<'a> { - argv: &'a [*const libc::c_char], - envp: Option<&'a [*const libc::c_char]>, -} - -fn exec(args: &ForkExecArgs, procargs: ProcArgs) -> ! { - match exec_inner(args, procargs) { - Ok(x) => match x {}, - Err(e) => { - let buf: &mut [u8] = &mut [0; 256]; - let mut cur = io::Cursor::new(&mut *buf); - // TODO: check if reached preexec, if not then have "noexec" after - let _ = write!(cur, "OSError:{}:", e as i32); - let pos = cur.position(); - let _ = unistd::write(args.errpipe_write, &buf[..pos as usize]); - std::process::exit(255) - } - } -} - -fn exec_inner(args: &ForkExecArgs, procargs: ProcArgs) -> nix::Result<Never> { - for &fd in args.fds_to_keep.as_slice() { - if fd != args.errpipe_write { - posix::raw_set_inheritable(fd, true)? - } - } - - for &fd in &[args.p2cwrite, args.c2pread, args.errread] { - if fd != -1 { - unistd::close(fd)?; - } - } - unistd::close(args.errpipe_read)?; - - let c2pwrite = if args.c2pwrite == 0 { - let fd = unistd::dup(args.c2pwrite)?; - posix::raw_set_inheritable(fd, true)?; - fd - } else { - args.c2pwrite - }; - - let mut errwrite = args.errwrite; - while errwrite == 0 || errwrite == 1 { - errwrite = unistd::dup(errwrite)?; - posix::raw_set_inheritable(errwrite, true)?; - } - - let dup_into_stdio = |fd, io_fd| { - if fd == io_fd { - posix::raw_set_inheritable(fd, true) - } else if fd != -1 { - unistd::dup2(fd, io_fd).map(drop) - } else { - Ok(()) - } - }; - dup_into_stdio(args.p2cread, 0)?; - dup_into_stdio(c2pwrite, 1)?; - dup_into_stdio(errwrite, 2)?; - - if let Some(ref cwd) = args.cwd { - unistd::chdir(cwd.s.as_c_str())? - } - - if args.child_umask >= 0 { - // TODO: umask(child_umask); - } - - if args.restore_signals { - // TODO: restore signals SIGPIPE, SIGXFZ, SIGXFSZ to SIG_DFL - } - - if args.call_setsid { - #[cfg(not(target_os = "redox"))] - unistd::setsid()?; - } - - if let Some(_groups_list) = args.groups_list.as_ref() { - // TODO: setgroups - // unistd::setgroups(groups_size, groups); - } - - if let Some(_gid) = args.gid_to_set.as_ref() { - // TODO: setgid - // unistd::setregid(gid, gid)?; - } - - if let Some(_uid) = args.uid.as_ref() { - // TODO: setuid - // unistd::setreuid(uid, uid)?; - } - - if args.close_fds { - #[cfg(not(target_os = "redox"))] - close_fds(3, &args.fds_to_keep)?; - } - - let mut first_err = None; - for exec in args.exec_list.as_slice() { - // not using nix's versions of these functions because those allocate the char-ptr array, - // and we can't allocate - if let Some(envp) = procargs.envp { - unsafe { libc::execve(exec.s.as_ptr(), procargs.argv.as_ptr(), envp.as_ptr()) }; - } else { - unsafe { libc::execv(exec.s.as_ptr(), procargs.argv.as_ptr()) }; - } - let e = Errno::last(); - if e != Errno::ENOENT && e != Errno::ENOTDIR && first_err.is_none() { - first_err = Some(e) - } - } - Err(first_err.unwrap_or_else(Errno::last)) -} - -#[cfg(not(target_os = "redox"))] -fn close_fds(above: i32, keep: &[i32]) -> nix::Result<()> { - use nix::{dir::Dir, fcntl::OFlag}; - // TODO: close fds by brute force if readdir doesn't work: - // https://github.com/python/cpython/blob/3.8/Modules/_posixsubprocess.c#L220 - let mut dir = Dir::open( - FD_DIR_NAME, - OFlag::O_RDONLY | OFlag::O_DIRECTORY, - nix::sys::stat::Mode::empty(), - )?; - let dirfd = dir.as_raw_fd(); - for e in dir.iter() { - if let Some(fd) = pos_int_from_ascii(e?.file_name()) { - if fd != dirfd && fd > above && !keep.contains(&fd) { - unistd::close(fd)? - } - } - } - Ok(()) -} - -#[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple", -))] -const FD_DIR_NAME: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"/dev/fd\0") }; - -#[cfg(any(target_os = "linux", target_os = "android"))] -const FD_DIR_NAME: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"/proc/self/fd\0") }; - -#[cfg(not(target_os = "redox"))] -fn pos_int_from_ascii(name: &CStr) -> Option<i32> { - let mut num = 0; - for c in name.to_bytes() { - if !c.is_ascii_digit() { - return None; - } - num = num * 10 + i32::from(c - b'0') - } - Some(num) -} diff --git a/stdlib/src/pyexpat.rs b/stdlib/src/pyexpat.rs deleted file mode 100644 index 89267d3f7ee..00000000000 --- a/stdlib/src/pyexpat.rs +++ /dev/null @@ -1,183 +0,0 @@ -/* Pyexpat builtin module -* -* -*/ - -use crate::vm::{builtins::PyModule, extend_module, PyRef, VirtualMachine}; - -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _pyexpat::make_module(vm); - - extend_module!(vm, &module, { - "errors" => _errors::make_module(vm), - "model" => _model::make_module(vm), - }); - - module -} - -macro_rules! create_property { - ($ctx: expr, $attributes: expr, $name: expr, $class: expr, $element: ident) => { - let attr = $ctx.new_getset( - $name, - $class, - move |this: &PyExpatLikeXmlParser| this.$element.read().clone(), - move |this: &PyExpatLikeXmlParser, func: PyObjectRef| *this.$element.write() = func, - ); - - $attributes.insert($ctx.intern_str($name), attr.into()); - }; -} - -#[pymodule(name = "pyexpat")] -mod _pyexpat { - use crate::vm::{ - builtins::{PyStr, PyStrRef, PyType}, - function::ArgBytesLike, - function::{IntoFuncArgs, OptionalArg}, - Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - }; - use rustpython_common::lock::PyRwLock; - use std::io::Cursor; - use xml::reader::XmlEvent; - type MutableObject = PyRwLock<PyObjectRef>; - - #[pyattr] - #[pyclass(name = "xmlparser", module = false, traverse)] - #[derive(Debug, PyPayload)] - pub struct PyExpatLikeXmlParser { - start_element: MutableObject, - end_element: MutableObject, - character_data: MutableObject, - entity_decl: MutableObject, - buffer_text: MutableObject, - } - type PyExpatLikeXmlParserRef = PyRef<PyExpatLikeXmlParser>; - - #[inline] - fn invoke_handler<T>(vm: &VirtualMachine, handler: &MutableObject, args: T) - where - T: IntoFuncArgs, - { - handler.read().call(args, vm).ok(); - } - - #[pyclass] - impl PyExpatLikeXmlParser { - fn new(vm: &VirtualMachine) -> PyResult<PyExpatLikeXmlParserRef> { - Ok(PyExpatLikeXmlParser { - start_element: MutableObject::new(vm.ctx.none()), - end_element: MutableObject::new(vm.ctx.none()), - character_data: MutableObject::new(vm.ctx.none()), - entity_decl: MutableObject::new(vm.ctx.none()), - buffer_text: MutableObject::new(vm.ctx.new_bool(false).into()), - } - .into_ref(&vm.ctx)) - } - - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - let mut attributes = class.attributes.write(); - - create_property!(ctx, attributes, "StartElementHandler", class, start_element); - create_property!(ctx, attributes, "EndElementHandler", class, end_element); - create_property!( - ctx, - attributes, - "CharacterDataHandler", - class, - character_data - ); - create_property!(ctx, attributes, "EntityDeclHandler", class, entity_decl); - create_property!(ctx, attributes, "buffer_text", class, buffer_text); - } - - fn create_config(&self) -> xml::ParserConfig { - xml::ParserConfig::new() - .cdata_to_characters(true) - .coalesce_characters(false) - .whitespace_to_characters(true) - } - - fn do_parse<T>(&self, vm: &VirtualMachine, parser: xml::EventReader<T>) - where - T: std::io::Read, - { - for e in parser { - match e { - Ok(XmlEvent::StartElement { - name, attributes, .. - }) => { - let dict = vm.ctx.new_dict(); - for attribute in attributes { - dict.set_item( - attribute.name.local_name.as_str(), - vm.ctx.new_str(attribute.value).into(), - vm, - ) - .unwrap(); - } - - let name_str = PyStr::from(name.local_name).into_ref(&vm.ctx); - invoke_handler(vm, &self.start_element, (name_str, dict)); - } - Ok(XmlEvent::EndElement { name, .. }) => { - let name_str = PyStr::from(name.local_name).into_ref(&vm.ctx); - invoke_handler(vm, &self.end_element, (name_str,)); - } - Ok(XmlEvent::Characters(chars)) => { - let str = PyStr::from(chars).into_ref(&vm.ctx); - invoke_handler(vm, &self.character_data, (str,)); - } - _ => {} - } - } - } - - #[pymethod(name = "Parse")] - fn parse(&self, data: PyStrRef, _isfinal: OptionalArg<bool>, vm: &VirtualMachine) { - let reader = Cursor::<Vec<u8>>::new(data.as_str().as_bytes().to_vec()); - let parser = self.create_config().create_reader(reader); - self.do_parse(vm, parser); - } - - #[pymethod(name = "ParseFile")] - fn parse_file(&self, file: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // todo: read chunks at a time - let read_res = vm.call_method(&file, "read", ())?; - let bytes_like = ArgBytesLike::try_from_object(vm, read_res)?; - let buf = bytes_like.borrow_buf().to_vec(); - let reader = Cursor::new(buf); - let parser = self.create_config().create_reader(reader); - self.do_parse(vm, parser); - - // todo: return value - Ok(()) - } - } - - #[derive(FromArgs)] - #[allow(dead_code)] - struct ParserCreateArgs { - #[pyarg(any, optional)] - encoding: OptionalArg<PyStrRef>, - #[pyarg(any, optional)] - namespace_separator: OptionalArg<PyStrRef>, - #[pyarg(any, optional)] - intern: OptionalArg<PyStrRef>, - } - - #[pyfunction(name = "ParserCreate")] - fn parser_create( - _args: ParserCreateArgs, - vm: &VirtualMachine, - ) -> PyResult<PyExpatLikeXmlParserRef> { - PyExpatLikeXmlParser::new(vm) - } -} - -#[pymodule(name = "model")] -mod _model {} - -#[pymodule(name = "errors")] -mod _errors {} diff --git a/stdlib/src/random.rs b/stdlib/src/random.rs deleted file mode 100644 index 1dfc4fcc30f..00000000000 --- a/stdlib/src/random.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! Random module. - -pub(crate) use _random::make_module; - -#[pymodule] -mod _random { - use crate::common::lock::PyMutex; - use crate::vm::{ - builtins::{PyInt, PyTypeRef}, - function::OptionalOption, - types::Constructor, - PyObjectRef, PyPayload, PyResult, VirtualMachine, - }; - use malachite_bigint::{BigInt, BigUint, Sign}; - use num_traits::{Signed, Zero}; - use rand::{rngs::StdRng, RngCore, SeedableRng}; - - #[derive(Debug)] - enum PyRng { - Std(Box<StdRng>), - MT(Box<mt19937::MT19937>), - } - - impl Default for PyRng { - fn default() -> Self { - PyRng::Std(Box::new(StdRng::from_entropy())) - } - } - - impl RngCore for PyRng { - fn next_u32(&mut self) -> u32 { - match self { - Self::Std(s) => s.next_u32(), - Self::MT(m) => m.next_u32(), - } - } - fn next_u64(&mut self) -> u64 { - match self { - Self::Std(s) => s.next_u64(), - Self::MT(m) => m.next_u64(), - } - } - fn fill_bytes(&mut self, dest: &mut [u8]) { - match self { - Self::Std(s) => s.fill_bytes(dest), - Self::MT(m) => m.fill_bytes(dest), - } - } - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand::Error> { - match self { - Self::Std(s) => s.try_fill_bytes(dest), - Self::MT(m) => m.try_fill_bytes(dest), - } - } - } - - #[pyattr] - #[pyclass(name = "Random")] - #[derive(Debug, PyPayload)] - struct PyRandom { - rng: PyMutex<PyRng>, - } - - impl Constructor for PyRandom { - type Args = OptionalOption<PyObjectRef>; - - fn py_new( - cls: PyTypeRef, - // TODO: use x as the seed. - _x: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyRandom { - rng: PyMutex::default(), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(flags(BASETYPE), with(Constructor))] - impl PyRandom { - #[pymethod] - fn random(&self) -> f64 { - let mut rng = self.rng.lock(); - mt19937::gen_res53(&mut *rng) - } - - #[pymethod] - fn seed(&self, n: OptionalOption<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { - let new_rng = n - .flatten() - .map(|n| { - // Fallback to using hash if object isn't Int-like. - let (_, mut key) = match n.downcast::<PyInt>() { - Ok(n) => n.as_bigint().abs(), - Err(obj) => BigInt::from(obj.hash(vm)?).abs(), - } - .to_u32_digits(); - if cfg!(target_endian = "big") { - key.reverse(); - } - let key = if key.is_empty() { &[0] } else { key.as_slice() }; - Ok(PyRng::MT(Box::new(mt19937::MT19937::new_with_slice_seed( - key, - )))) - }) - .transpose()? - .unwrap_or_default(); - - *self.rng.lock() = new_rng; - Ok(()) - } - - #[pymethod] - fn getrandbits(&self, k: isize, vm: &VirtualMachine) -> PyResult<BigInt> { - match k { - k if k < 0 => { - Err(vm.new_value_error("number of bits must be non-negative".to_owned())) - } - 0 => Ok(BigInt::zero()), - _ => { - let mut rng = self.rng.lock(); - let mut k = k; - let mut gen_u32 = |k| { - let r = rng.next_u32(); - if k < 32 { - r >> (32 - k) - } else { - r - } - }; - - let words = (k - 1) / 32 + 1; - let wordarray = (0..words) - .map(|_| { - let word = gen_u32(k); - k = k.wrapping_sub(32); - word - }) - .collect::<Vec<_>>(); - - let uint = BigUint::new(wordarray); - // very unlikely but might as well check - let sign = if uint.is_zero() { - Sign::NoSign - } else { - Sign::Plus - }; - Ok(BigInt::from_biguint(sign, uint)) - } - } - } - } -} diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs deleted file mode 100644 index 48705f4c9b7..00000000000 --- a/stdlib/src/select.rs +++ /dev/null @@ -1,399 +0,0 @@ -use crate::vm::{ - builtins::PyListRef, builtins::PyModule, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, - VirtualMachine, -}; -use std::{io, mem}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - #[cfg(windows)] - crate::vm::stdlib::nt::init_winsock(); - - #[cfg(unix)] - { - use crate::vm::class::PyClassImpl; - decl::poll::PyPoll::make_class(&vm.ctx); - } - - decl::make_module(vm) -} - -#[cfg(unix)] -mod platform { - pub use libc::{fd_set, select, timeval, FD_ISSET, FD_SET, FD_SETSIZE, FD_ZERO}; - pub use std::os::unix::io::RawFd; - - pub fn check_err(x: i32) -> bool { - x < 0 - } -} - -#[allow(non_snake_case)] -#[cfg(windows)] -mod platform { - use windows_sys::Win32::Networking::WinSock; - pub use WinSock::{select, FD_SET as fd_set, FD_SETSIZE, SOCKET as RawFd, TIMEVAL as timeval}; - - // based off winsock2.h: https://gist.github.com/piscisaureus/906386#file-winsock2-h-L128-L141 - - pub unsafe fn FD_SET(fd: RawFd, set: *mut fd_set) { - let mut slot = std::ptr::addr_of_mut!((*set).fd_array).cast::<RawFd>(); - let fd_count = (*set).fd_count; - for _ in 0..fd_count { - if *slot == fd { - return; - } - slot = slot.add(1); - } - // slot == &fd_array[fd_count] at this point - if fd_count < FD_SETSIZE { - *slot = fd as RawFd; - (*set).fd_count += 1; - } - } - - pub unsafe fn FD_ZERO(set: *mut fd_set) { - (*set).fd_count = 0; - } - - pub unsafe fn FD_ISSET(fd: RawFd, set: *mut fd_set) -> bool { - use WinSock::__WSAFDIsSet; - __WSAFDIsSet(fd as _, set) != 0 - } - - pub fn check_err(x: i32) -> bool { - x == WinSock::SOCKET_ERROR - } -} - -pub use platform::timeval; -use platform::RawFd; - -#[derive(Traverse)] -struct Selectable { - obj: PyObjectRef, - #[pytraverse(skip)] - fno: RawFd, -} - -impl TryFromObject for Selectable { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let fno = obj.try_to_value(vm).or_else(|_| { - let meth = vm.get_method_or_type_error( - obj.clone(), - vm.ctx.interned_str("fileno").unwrap(), - || "select arg must be an int or object with a fileno() method".to_owned(), - )?; - meth.call((), vm)?.try_into_value(vm) - })?; - Ok(Selectable { obj, fno }) - } -} - -// Keep it in a MaybeUninit, since on windows FD_ZERO doesn't actually zero the whole thing -#[repr(transparent)] -pub struct FdSet(mem::MaybeUninit<platform::fd_set>); - -impl FdSet { - pub fn new() -> FdSet { - // it's just ints, and all the code that's actually - // interacting with it is in C, so it's safe to zero - let mut fdset = std::mem::MaybeUninit::zeroed(); - unsafe { platform::FD_ZERO(fdset.as_mut_ptr()) }; - FdSet(fdset) - } - - pub fn insert(&mut self, fd: RawFd) { - unsafe { platform::FD_SET(fd, self.0.as_mut_ptr()) }; - } - - pub fn contains(&mut self, fd: RawFd) -> bool { - unsafe { platform::FD_ISSET(fd, self.0.as_mut_ptr()) } - } - - pub fn clear(&mut self) { - unsafe { platform::FD_ZERO(self.0.as_mut_ptr()) }; - } - - pub fn highest(&mut self) -> Option<RawFd> { - (0..platform::FD_SETSIZE as RawFd) - .rev() - .find(|&i| self.contains(i)) - } -} - -pub fn select( - nfds: libc::c_int, - readfds: &mut FdSet, - writefds: &mut FdSet, - errfds: &mut FdSet, - timeout: Option<&mut timeval>, -) -> io::Result<i32> { - let timeout = match timeout { - Some(tv) => tv as *mut timeval, - None => std::ptr::null_mut(), - }; - let ret = unsafe { - platform::select( - nfds, - readfds.0.as_mut_ptr(), - writefds.0.as_mut_ptr(), - errfds.0.as_mut_ptr(), - timeout, - ) - }; - if platform::check_err(ret) { - Err(io::Error::last_os_error()) - } else { - Ok(ret) - } -} - -fn sec_to_timeval(sec: f64) -> timeval { - timeval { - tv_sec: sec.trunc() as _, - tv_usec: (sec.fract() * 1e6) as _, - } -} - -#[pymodule(name = "select")] -mod decl { - use super::*; - use crate::vm::{ - builtins::PyTypeRef, - convert::ToPyException, - function::{Either, OptionalOption}, - stdlib::time, - PyObjectRef, PyResult, VirtualMachine, - }; - - #[pyattr] - fn error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.exceptions.os_error.to_owned() - } - - #[pyfunction] - fn select( - rlist: PyObjectRef, - wlist: PyObjectRef, - xlist: PyObjectRef, - timeout: OptionalOption<Either<f64, isize>>, - vm: &VirtualMachine, - ) -> PyResult<(PyListRef, PyListRef, PyListRef)> { - let mut timeout = timeout.flatten().map(|e| match e { - Either::A(f) => f, - Either::B(i) => i as f64, - }); - if let Some(timeout) = timeout { - if timeout < 0.0 { - return Err(vm.new_value_error("timeout must be positive".to_owned())); - } - } - let deadline = timeout.map(|s| time::time(vm).unwrap() + s); - - let seq2set = |list: &PyObject| -> PyResult<(Vec<Selectable>, FdSet)> { - let v: Vec<Selectable> = list.try_to_value(vm)?; - let mut fds = FdSet::new(); - for fd in &v { - fds.insert(fd.fno); - } - Ok((v, fds)) - }; - - let (rlist, mut r) = seq2set(&rlist)?; - let (wlist, mut w) = seq2set(&wlist)?; - let (xlist, mut x) = seq2set(&xlist)?; - - if rlist.is_empty() && wlist.is_empty() && xlist.is_empty() { - let empty = vm.ctx.new_list(vec![]); - return Ok((empty.clone(), empty.clone(), empty)); - } - - let nfds: i32 = [&mut r, &mut w, &mut x] - .iter_mut() - .filter_map(|set| set.highest()) - .max() - .map_or(0, |n| n + 1) as _; - - loop { - let mut tv = timeout.map(sec_to_timeval); - let res = super::select(nfds, &mut r, &mut w, &mut x, tv.as_mut()); - - match res { - Ok(_) => break, - Err(err) if err.kind() == io::ErrorKind::Interrupted => {} - Err(err) => return Err(err.to_pyexception(vm)), - } - - vm.check_signals()?; - - if let Some(ref mut timeout) = timeout { - *timeout = deadline.unwrap() - time::time(vm).unwrap(); - if *timeout < 0.0 { - r.clear(); - w.clear(); - x.clear(); - break; - } - // retry select() if we haven't reached the deadline yet - } - } - - let set2list = |list: Vec<Selectable>, mut set: FdSet| { - vm.ctx.new_list( - list.into_iter() - .filter(|fd| set.contains(fd.fno)) - .map(|fd| fd.obj) - .collect(), - ) - }; - - let rlist = set2list(rlist, r); - let wlist = set2list(wlist, w); - let xlist = set2list(xlist, x); - - Ok((rlist, wlist, xlist)) - } - - #[cfg(unix)] - #[pyfunction] - fn poll() -> poll::PyPoll { - poll::PyPoll::default() - } - - #[cfg(unix)] - #[pyattr] - use libc::{POLLERR, POLLHUP, POLLIN, POLLNVAL, POLLOUT, POLLPRI}; - - #[cfg(unix)] - pub(super) mod poll { - use super::*; - use crate::vm::{ - builtins::PyFloat, common::lock::PyMutex, convert::ToPyObject, function::OptionalArg, - stdlib::io::Fildes, AsObject, PyPayload, - }; - use libc::pollfd; - use num_traits::ToPrimitive; - use std::time; - - #[pyclass(module = "select", name = "poll")] - #[derive(Default, Debug, PyPayload)] - pub struct PyPoll { - // keep sorted - fds: PyMutex<Vec<pollfd>>, - } - - #[inline] - fn search(fds: &[pollfd], fd: i32) -> Result<usize, usize> { - fds.binary_search_by_key(&fd, |pfd| pfd.fd) - } - - fn insert_fd(fds: &mut Vec<pollfd>, fd: i32, events: i16) { - match search(fds, fd) { - Ok(i) => fds[i].events = events, - Err(i) => fds.insert( - i, - pollfd { - fd, - events, - revents: 0, - }, - ), - } - } - - fn get_fd_mut(fds: &mut [pollfd], fd: i32) -> Option<&mut pollfd> { - search(fds, fd).ok().map(move |i| &mut fds[i]) - } - - fn remove_fd(fds: &mut Vec<pollfd>, fd: i32) -> Option<pollfd> { - search(fds, fd).ok().map(|i| fds.remove(i)) - } - - const DEFAULT_EVENTS: i16 = libc::POLLIN | libc::POLLPRI | libc::POLLOUT; - - #[pyclass] - impl PyPoll { - #[pymethod] - fn register(&self, Fildes(fd): Fildes, eventmask: OptionalArg<u16>) { - insert_fd( - &mut self.fds.lock(), - fd, - eventmask.map_or(DEFAULT_EVENTS, |e| e as i16), - ) - } - - #[pymethod] - fn modify(&self, Fildes(fd): Fildes, eventmask: u16) -> io::Result<()> { - let mut fds = self.fds.lock(); - let pfd = get_fd_mut(&mut fds, fd) - .ok_or_else(|| io::Error::from_raw_os_error(libc::ENOENT))?; - pfd.events = eventmask as i16; - Ok(()) - } - - #[pymethod] - fn unregister(&self, Fildes(fd): Fildes, vm: &VirtualMachine) -> PyResult<()> { - let removed = remove_fd(&mut self.fds.lock(), fd); - removed - .map(drop) - .ok_or_else(|| vm.new_key_error(vm.ctx.new_int(fd).into())) - } - - #[pymethod] - fn poll( - &self, - timeout: OptionalOption, - vm: &VirtualMachine, - ) -> PyResult<Vec<PyObjectRef>> { - let mut fds = self.fds.lock(); - let timeout_ms = match timeout.flatten() { - Some(ms) => { - let ms = if let Some(float) = ms.payload::<PyFloat>() { - float.to_f64().to_i32() - } else if let Some(int) = ms.try_index_opt(vm) { - int?.as_bigint().to_i32() - } else { - return Err(vm.new_type_error(format!( - "expected an int or float for duration, got {}", - ms.class() - ))); - }; - ms.ok_or_else(|| vm.new_value_error("value out of range".to_owned()))? - } - None => -1, - }; - let timeout_ms = if timeout_ms < 0 { -1 } else { timeout_ms }; - let deadline = (timeout_ms >= 0) - .then(|| time::Instant::now() + time::Duration::from_millis(timeout_ms as u64)); - let mut poll_timeout = timeout_ms; - loop { - let res = unsafe { libc::poll(fds.as_mut_ptr(), fds.len() as _, poll_timeout) }; - let res = if res < 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - }; - match res { - Ok(()) => break, - Err(e) if e.kind() == io::ErrorKind::Interrupted => { - vm.check_signals()?; - if let Some(d) = deadline { - match d.checked_duration_since(time::Instant::now()) { - Some(remaining) => poll_timeout = remaining.as_millis() as i32, - // we've timed out - None => break, - } - } - } - Err(e) => return Err(e.to_pyexception(vm)), - } - } - Ok(fds - .iter() - .filter(|pfd| pfd.revents != 0) - .map(|pfd| (pfd.fd, pfd.revents & 0xfff).to_pyobject(vm)) - .collect()) - } - } - } -} diff --git a/stdlib/src/sha1.rs b/stdlib/src/sha1.rs deleted file mode 100644 index 3820e7d96a4..00000000000 --- a/stdlib/src/sha1.rs +++ /dev/null @@ -1,12 +0,0 @@ -pub(crate) use _sha1::make_module; - -#[pymodule] -mod _sha1 { - use crate::hashlib::_hashlib::{local_sha1, HashArgs}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; - - #[pyfunction] - fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha1(args).into_pyobject(vm)) - } -} diff --git a/stdlib/src/sha256.rs b/stdlib/src/sha256.rs deleted file mode 100644 index bae22fa4cc3..00000000000 --- a/stdlib/src/sha256.rs +++ /dev/null @@ -1,17 +0,0 @@ -pub(crate) use _sha256::make_module; - -#[pymodule] -mod _sha256 { - use crate::hashlib::_hashlib::{local_sha224, local_sha256, HashArgs}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; - - #[pyfunction] - fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha224(args).into_pyobject(vm)) - } - - #[pyfunction] - fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha256(args).into_pyobject(vm)) - } -} diff --git a/stdlib/src/sha3.rs b/stdlib/src/sha3.rs deleted file mode 100644 index f0c1c5ef69c..00000000000 --- a/stdlib/src/sha3.rs +++ /dev/null @@ -1,40 +0,0 @@ -pub(crate) use _sha3::make_module; - -#[pymodule] -mod _sha3 { - use crate::hashlib::_hashlib::{ - local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128, - local_shake_256, HashArgs, - }; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; - - #[pyfunction] - fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha3_224(args).into_pyobject(vm)) - } - - #[pyfunction] - fn sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha3_256(args).into_pyobject(vm)) - } - - #[pyfunction] - fn sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha3_384(args).into_pyobject(vm)) - } - - #[pyfunction] - fn sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_sha3_512(args).into_pyobject(vm)) - } - - #[pyfunction] - fn shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_shake_128(args).into_pyobject(vm)) - } - - #[pyfunction] - fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(local_shake_256(args).into_pyobject(vm)) - } -} diff --git a/stdlib/src/sha512.rs b/stdlib/src/sha512.rs deleted file mode 100644 index bb93fc9dba4..00000000000 --- a/stdlib/src/sha512.rs +++ /dev/null @@ -1,20 +0,0 @@ -// spell-checker:ignore usedforsecurity HASHXOF - -pub(crate) use _sha512::make_module; - -#[pymodule] -mod _sha512 { - use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher}; - use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use sha2::{Sha384, Sha512}; - - #[pyfunction(name = "sha384")] - fn sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - Ok(PyHasher::new("sha384", HashWrapper::new::<Sha384>(args.string)).into_pyobject(vm)) - } - - #[pyfunction(name = "sha512")] - fn sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - Ok(PyHasher::new("sha512", HashWrapper::new::<Sha512>(args.string)).into_pyobject(vm)) - } -} diff --git a/stdlib/src/socket.rs b/stdlib/src/socket.rs deleted file mode 100644 index c6624df5464..00000000000 --- a/stdlib/src/socket.rs +++ /dev/null @@ -1,2375 +0,0 @@ -use crate::vm::{builtins::PyModule, PyRef, VirtualMachine}; -#[cfg(feature = "ssl")] -pub(super) use _socket::{sock_select, timeout_error_msg, PySocket, SelectKind}; - -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - #[cfg(windows)] - crate::vm::stdlib::nt::init_winsock(); - _socket::make_module(vm) -} - -#[pymodule] -mod _socket { - use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard}; - use crate::vm::{ - builtins::{PyBaseExceptionRef, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, - convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject}, - function::{ArgBytesLike, ArgMemoryBuffer, Either, FsPath, OptionalArg, OptionalOption}, - types::{DefaultConstructor, Initializer, Representable}, - utils::ToCString, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - }; - use crossbeam_utils::atomic::AtomicCell; - use num_traits::ToPrimitive; - use socket2::{Domain, Protocol, Socket, Type as SocketType}; - use std::{ - ffi, - io::{self, Read, Write}, - mem::MaybeUninit, - net::{self, Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, ToSocketAddrs}, - time::{Duration, Instant}, - }; - - #[cfg(unix)] - use libc as c; - #[cfg(windows)] - mod c { - pub use winapi::shared::netioapi::{if_indextoname, if_nametoindex}; - pub use winapi::shared::ws2def::{ - INADDR_ANY, INADDR_BROADCAST, INADDR_LOOPBACK, INADDR_NONE, - }; - pub use winapi::um::winsock2::{ - getprotobyname, getservbyname, getservbyport, getsockopt, setsockopt, - SO_EXCLUSIVEADDRUSE, - }; - pub use winapi::um::ws2tcpip::{ - EAI_AGAIN, EAI_BADFLAGS, EAI_FAIL, EAI_FAMILY, EAI_MEMORY, EAI_NODATA, EAI_NONAME, - EAI_SERVICE, EAI_SOCKTYPE, - }; - pub use windows_sys::Win32::Networking::WinSock::{ - AF_DECnet, AF_APPLETALK, AF_IPX, AF_LINK, AI_ADDRCONFIG, AI_ALL, AI_CANONNAME, - AI_NUMERICSERV, AI_V4MAPPED, IPPORT_RESERVED, IPPROTO_AH, IPPROTO_DSTOPTS, IPPROTO_EGP, - IPPROTO_ESP, IPPROTO_FRAGMENT, IPPROTO_GGP, IPPROTO_HOPOPTS, IPPROTO_ICMP, - IPPROTO_ICMPV6, IPPROTO_IDP, IPPROTO_IGMP, IPPROTO_IP, IPPROTO_IP as IPPROTO_IPIP, - IPPROTO_IPV4, IPPROTO_IPV6, IPPROTO_ND, IPPROTO_NONE, IPPROTO_PIM, IPPROTO_PUP, - IPPROTO_RAW, IPPROTO_ROUTING, IPPROTO_TCP, IPPROTO_UDP, IPV6_CHECKSUM, IPV6_DONTFRAG, - IPV6_HOPLIMIT, IPV6_HOPOPTS, IPV6_JOIN_GROUP, IPV6_LEAVE_GROUP, IPV6_MULTICAST_HOPS, - IPV6_MULTICAST_IF, IPV6_MULTICAST_LOOP, IPV6_PKTINFO, IPV6_RECVRTHDR, IPV6_RECVTCLASS, - IPV6_RTHDR, IPV6_TCLASS, IPV6_UNICAST_HOPS, IPV6_V6ONLY, IP_ADD_MEMBERSHIP, - IP_DROP_MEMBERSHIP, IP_HDRINCL, IP_MULTICAST_IF, IP_MULTICAST_LOOP, IP_MULTICAST_TTL, - IP_OPTIONS, IP_RECVDSTADDR, IP_TOS, IP_TTL, MSG_BCAST, MSG_CTRUNC, MSG_DONTROUTE, - MSG_MCAST, MSG_OOB, MSG_PEEK, MSG_TRUNC, MSG_WAITALL, NI_DGRAM, NI_MAXHOST, NI_MAXSERV, - NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, RCVALL_IPLEVEL, RCVALL_OFF, - RCVALL_ON, RCVALL_SOCKETLEVELONLY, SD_BOTH as SHUT_RDWR, SD_RECEIVE as SHUT_RD, - SD_SEND as SHUT_WR, SIO_KEEPALIVE_VALS, SIO_LOOPBACK_FAST_PATH, SIO_RCVALL, SOCK_DGRAM, - SOCK_RAW, SOCK_RDM, SOCK_SEQPACKET, SOCK_STREAM, SOL_SOCKET, SOMAXCONN, SO_BROADCAST, - SO_ERROR, SO_LINGER, SO_OOBINLINE, SO_REUSEADDR, SO_TYPE, SO_USELOOPBACK, TCP_NODELAY, - WSAEBADF, WSAECONNRESET, WSAENOTSOCK, WSAEWOULDBLOCK, - }; - pub const IF_NAMESIZE: usize = - windows_sys::Win32::NetworkManagement::Ndis::IF_MAX_STRING_SIZE as _; - pub const AF_UNSPEC: i32 = windows_sys::Win32::Networking::WinSock::AF_UNSPEC as _; - pub const AF_INET: i32 = windows_sys::Win32::Networking::WinSock::AF_INET as _; - pub const AF_INET6: i32 = windows_sys::Win32::Networking::WinSock::AF_INET6 as _; - pub const AI_PASSIVE: i32 = windows_sys::Win32::Networking::WinSock::AI_PASSIVE as _; - pub const AI_NUMERICHOST: i32 = - windows_sys::Win32::Networking::WinSock::AI_NUMERICHOST as _; - } - // constants - #[pyattr(name = "has_ipv6")] - const HAS_IPV6: bool = true; - #[pyattr] - // put IPPROTO_MAX later - use c::{ - AF_INET, AF_INET6, AF_UNSPEC, INADDR_ANY, INADDR_LOOPBACK, INADDR_NONE, IPPROTO_ICMP, - IPPROTO_ICMPV6, IPPROTO_IP, IPPROTO_IPV6, IPPROTO_TCP, IPPROTO_TCP as SOL_TCP, IPPROTO_UDP, - MSG_CTRUNC, MSG_DONTROUTE, MSG_OOB, MSG_PEEK, MSG_TRUNC, MSG_WAITALL, NI_DGRAM, NI_MAXHOST, - NI_NAMEREQD, NI_NOFQDN, NI_NUMERICHOST, NI_NUMERICSERV, SHUT_RD, SHUT_RDWR, SHUT_WR, - SOCK_DGRAM, SOCK_STREAM, SOL_SOCKET, SO_BROADCAST, SO_ERROR, SO_LINGER, SO_OOBINLINE, - SO_REUSEADDR, SO_TYPE, TCP_NODELAY, - }; - - #[cfg(not(target_os = "redox"))] - #[pyattr] - use c::{ - AF_DECnet, AF_APPLETALK, AF_IPX, IPPROTO_AH, IPPROTO_DSTOPTS, IPPROTO_EGP, IPPROTO_ESP, - IPPROTO_FRAGMENT, IPPROTO_HOPOPTS, IPPROTO_IDP, IPPROTO_IGMP, IPPROTO_IPIP, IPPROTO_NONE, - IPPROTO_PIM, IPPROTO_PUP, IPPROTO_RAW, IPPROTO_ROUTING, - }; - - #[cfg(unix)] - #[pyattr] - use c::{AF_UNIX, SO_REUSEPORT}; - - #[pyattr] - use c::{AI_ADDRCONFIG, AI_NUMERICHOST, AI_NUMERICSERV, AI_PASSIVE}; - - #[cfg(not(target_os = "redox"))] - #[pyattr] - use c::{SOCK_RAW, SOCK_RDM, SOCK_SEQPACKET}; - - #[cfg(target_os = "android")] - #[pyattr] - use c::{SOL_ATALK, SOL_AX25, SOL_IPX, SOL_NETROM, SOL_ROSE}; - - #[cfg(target_os = "freebsd")] - #[pyattr] - use c::SO_SETFIB; - - #[cfg(target_os = "linux")] - #[pyattr] - use c::{ - CAN_BCM, CAN_EFF_FLAG, CAN_EFF_MASK, CAN_ERR_FLAG, CAN_ERR_MASK, CAN_ISOTP, CAN_J1939, - CAN_RAW, CAN_RAW_ERR_FILTER, CAN_RAW_FD_FRAMES, CAN_RAW_FILTER, CAN_RAW_JOIN_FILTERS, - CAN_RAW_LOOPBACK, CAN_RAW_RECV_OWN_MSGS, CAN_RTR_FLAG, CAN_SFF_MASK, IPPROTO_MPTCP, - J1939_IDLE_ADDR, J1939_MAX_UNICAST_ADDR, J1939_NLA_BYTES_ACKED, J1939_NLA_PAD, - J1939_NO_ADDR, J1939_NO_NAME, J1939_NO_PGN, J1939_PGN_ADDRESS_CLAIMED, - J1939_PGN_ADDRESS_COMMANDED, J1939_PGN_MAX, J1939_PGN_PDU1_MAX, J1939_PGN_REQUEST, - SCM_J1939_DEST_ADDR, SCM_J1939_DEST_NAME, SCM_J1939_ERRQUEUE, SCM_J1939_PRIO, SOL_CAN_BASE, - SOL_CAN_RAW, SO_J1939_ERRQUEUE, SO_J1939_FILTER, SO_J1939_PROMISC, SO_J1939_SEND_PRIO, - }; - - #[cfg(all(target_os = "linux", target_env = "gnu"))] - #[pyattr] - use c::SOL_RDS; - - #[cfg(target_os = "netbsd")] - #[pyattr] - use c::IPPROTO_VRRP; - - #[cfg(target_vendor = "apple")] - #[pyattr] - use c::{AF_SYSTEM, PF_SYSTEM, SYSPROTO_CONTROL, TCP_KEEPALIVE}; - - #[cfg(windows)] - #[pyattr] - use c::{ - IPPORT_RESERVED, IPPROTO_IPV4, RCVALL_IPLEVEL, RCVALL_OFF, RCVALL_ON, - RCVALL_SOCKETLEVELONLY, SIO_KEEPALIVE_VALS, SIO_LOOPBACK_FAST_PATH, SIO_RCVALL, - SO_EXCLUSIVEADDRUSE, - }; - - #[cfg(not(windows))] - #[pyattr] - const IPPORT_RESERVED: i32 = 1024; - - #[pyattr] - const IPPORT_USERRESERVED: i32 = 5000; - - #[cfg(any(unix, target_os = "android"))] - #[pyattr] - use c::{ - EAI_SYSTEM, MSG_EOR, SO_ACCEPTCONN, SO_DEBUG, SO_DONTROUTE, SO_KEEPALIVE, SO_RCVBUF, - SO_RCVLOWAT, SO_RCVTIMEO, SO_SNDBUF, SO_SNDLOWAT, SO_SNDTIMEO, - }; - - #[cfg(any(target_os = "android", target_os = "linux"))] - #[pyattr] - use c::{ - ALG_OP_DECRYPT, ALG_OP_ENCRYPT, ALG_SET_AEAD_ASSOCLEN, ALG_SET_AEAD_AUTHSIZE, ALG_SET_IV, - ALG_SET_KEY, ALG_SET_OP, IPV6_DSTOPTS, IPV6_NEXTHOP, IPV6_PATHMTU, IPV6_RECVDSTOPTS, - IPV6_RECVHOPLIMIT, IPV6_RECVHOPOPTS, IPV6_RECVPATHMTU, IPV6_RTHDRDSTOPTS, - IP_DEFAULT_MULTICAST_LOOP, IP_RECVOPTS, IP_RETOPTS, NETLINK_CRYPTO, NETLINK_DNRTMSG, - NETLINK_FIREWALL, NETLINK_IP6_FW, NETLINK_NFLOG, NETLINK_ROUTE, NETLINK_USERSOCK, - NETLINK_XFRM, SOL_ALG, SO_PASSSEC, SO_PEERSEC, - }; - - #[cfg(any(target_os = "android", target_vendor = "apple"))] - #[pyattr] - use c::{AI_DEFAULT, AI_MASK, AI_V4MAPPED_CFG}; - - #[cfg(any(target_os = "freebsd", target_os = "netbsd"))] - #[pyattr] - use c::MSG_NOTIFICATION; - - #[cfg(any(target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - use c::TCP_USER_TIMEOUT; - - #[cfg(any(unix, target_os = "android", windows))] - #[pyattr] - use c::{ - INADDR_BROADCAST, IPV6_MULTICAST_HOPS, IPV6_MULTICAST_IF, IPV6_MULTICAST_LOOP, - IPV6_UNICAST_HOPS, IPV6_V6ONLY, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_MULTICAST_IF, - IP_MULTICAST_LOOP, IP_MULTICAST_TTL, IP_TTL, - }; - - #[cfg(any(unix, target_os = "android", windows))] - #[pyattr] - const INADDR_UNSPEC_GROUP: u32 = 0xe0000000; - - #[cfg(any(unix, target_os = "android", windows))] - #[pyattr] - const INADDR_ALLHOSTS_GROUP: u32 = 0xe0000001; - - #[cfg(any(unix, target_os = "android", windows))] - #[pyattr] - const INADDR_MAX_LOCAL_GROUP: u32 = 0xe00000ff; - - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - use c::{ - AF_ALG, AF_ASH, AF_ATMPVC, AF_ATMSVC, AF_AX25, AF_BRIDGE, AF_CAN, AF_ECONET, AF_IRDA, - AF_LLC, AF_NETBEUI, AF_NETLINK, AF_NETROM, AF_PACKET, AF_PPPOX, AF_RDS, AF_SECURITY, - AF_TIPC, AF_VSOCK, AF_WANPIPE, AF_X25, IP_TRANSPARENT, MSG_CONFIRM, MSG_ERRQUEUE, - MSG_FASTOPEN, MSG_MORE, PF_CAN, PF_PACKET, PF_RDS, SCM_CREDENTIALS, SOL_IP, SOL_TIPC, - SOL_UDP, SO_BINDTODEVICE, SO_MARK, TCP_CORK, TCP_DEFER_ACCEPT, TCP_LINGER2, TCP_QUICKACK, - TCP_SYNCNT, TCP_WINDOW_CLAMP, - }; - - // gated on presence of AF_VSOCK: - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - const SO_VM_SOCKETS_BUFFER_SIZE: u32 = 0; - - // gated on presence of AF_VSOCK: - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - const SO_VM_SOCKETS_BUFFER_MIN_SIZE: u32 = 1; - - // gated on presence of AF_VSOCK: - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - const SO_VM_SOCKETS_BUFFER_MAX_SIZE: u32 = 2; - - // gated on presence of AF_VSOCK: - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - const VMADDR_CID_ANY: u32 = 0xffffffff; // 0xffffffff - - // gated on presence of AF_VSOCK: - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - const VMADDR_PORT_ANY: u32 = 0xffffffff; // 0xffffffff - - // gated on presence of AF_VSOCK: - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - const VMADDR_CID_HOST: u32 = 2; - - // gated on presence of AF_VSOCK: - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - #[pyattr] - const VM_SOCKETS_INVALID_VERSION: u32 = 0xffffffff; // 0xffffffff - - // TODO: gated on https://github.com/rust-lang/libc/pull/1662 - // // gated on presence of AF_VSOCK: - // #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - // #[pyattr(name = "IOCTL_VM_SOCKETS_GET_LOCAL_CID", once)] - // fn ioctl_vm_sockets_get_local_cid(_vm: &VirtualMachine) -> i32 { - // c::_IO(7, 0xb9) - // } - - #[cfg(not(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] - #[pyattr] - const SOL_IP: i32 = 0; - - #[cfg(not(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] - #[pyattr] - const SOL_UDP: i32 = 17; - - #[cfg(any(target_os = "android", target_os = "linux", windows))] - #[pyattr] - use c::{IPV6_HOPOPTS, IPV6_RECVRTHDR, IPV6_RTHDR, IP_OPTIONS}; - - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_vendor = "apple" - ))] - #[pyattr] - use c::{IPPROTO_HELLO, IPPROTO_XTP, LOCAL_PEERCRED, MSG_EOF}; - - #[cfg(any(target_os = "netbsd", target_os = "openbsd", windows))] - #[pyattr] - use c::{MSG_BCAST, MSG_MCAST}; - - #[cfg(any( - target_os = "android", - target_os = "fuchsia", - target_os = "freebsd", - target_os = "linux" - ))] - #[pyattr] - use c::{IPPROTO_UDPLITE, TCP_CONGESTION}; - - #[cfg(any( - target_os = "android", - target_os = "fuchsia", - target_os = "freebsd", - target_os = "linux" - ))] - #[pyattr] - const UDPLITE_SEND_CSCOV: i32 = 10; - - #[cfg(any( - target_os = "android", - target_os = "fuchsia", - target_os = "freebsd", - target_os = "linux" - ))] - #[pyattr] - const UDPLITE_RECV_CSCOV: i32 = 11; - - #[cfg(any( - target_os = "android", - target_os = "fuchsia", - target_os = "linux", - target_os = "openbsd" - ))] - #[pyattr] - use c::AF_KEY; - - #[cfg(any( - target_os = "android", - target_os = "fuchsia", - target_os = "linux", - target_os = "redox" - ))] - #[pyattr] - use c::SO_DOMAIN; - - #[cfg(any( - target_os = "android", - target_os = "fuchsia", - all( - target_os = "linux", - any( - target_arch = "aarch64", - target_arch = "i686", - target_arch = "loongarch64", - target_arch = "mips", - target_arch = "powerpc", - target_arch = "powerpc64", - target_arch = "powerpc64le", - target_arch = "riscv64gc", - target_arch = "s390x", - target_arch = "x86_64" - ) - ), - target_os = "redox" - ))] - #[pyattr] - use c::SO_PRIORITY; - - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "netbsd", - target_os = "openbsd" - ))] - #[pyattr] - use c::IPPROTO_MOBILE; - - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "netbsd", - target_vendor = "apple" - ))] - #[pyattr] - use c::SCM_CREDS; - - #[cfg(any( - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_vendor = "apple" - ))] - #[pyattr] - use c::TCP_FASTOPEN; - - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - all( - target_os = "linux", - any( - target_arch = "aarch64", - target_arch = "i686", - target_arch = "loongarch64", - target_arch = "mips", - target_arch = "powerpc", - target_arch = "powerpc64", - target_arch = "powerpc64le", - target_arch = "riscv64gc", - target_arch = "s390x", - target_arch = "x86_64" - ) - ), - target_os = "redox" - ))] - #[pyattr] - use c::SO_PROTOCOL; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - windows - ))] - #[pyattr] - use c::IPV6_DONTFRAG; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "fuchsia", - target_os = "linux", - target_os = "redox" - ))] - #[pyattr] - use c::{SO_PASSCRED, SO_PEERCRED}; - - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd" - ))] - #[pyattr] - use c::TCP_INFO; - - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_vendor = "apple" - ))] - #[pyattr] - use c::IP_RECVTOS; - - #[cfg(any( - target_os = "android", - target_os = "netbsd", - target_os = "redox", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::NI_MAXSERV; - - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple" - ))] - #[pyattr] - use c::{IPPROTO_EON, IPPROTO_IPCOMP}; - - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "netbsd", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::IPPROTO_ND; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::{IPV6_CHECKSUM, IPV6_HOPLIMIT}; - - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd" - ))] - #[pyattr] - use c::IPPROTO_SCTP; // also in windows - - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::{AI_ALL, AI_V4MAPPED}; - - #[cfg(any( - target_os = "android", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::EAI_NODATA; - - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::{ - AF_LINK, IPPROTO_GGP, IPV6_JOIN_GROUP, IPV6_LEAVE_GROUP, IP_RECVDSTADDR, SO_USELOOPBACK, - }; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd" - ))] - #[pyattr] - use c::{MSG_CMSG_CLOEXEC, MSG_NOSIGNAL}; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd", - target_os = "redox" - ))] - #[pyattr] - use c::TCP_KEEPIDLE; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd", - target_vendor = "apple" - ))] - #[pyattr] - use c::{TCP_KEEPCNT, TCP_KEEPINTVL}; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_os = "redox" - ))] - #[pyattr] - use c::{SOCK_CLOEXEC, SOCK_NONBLOCK}; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple" - ))] - #[pyattr] - use c::{ - AF_ROUTE, AF_SNA, EAI_OVERFLOW, IPPROTO_GRE, IPPROTO_RSVP, IPPROTO_TP, IPV6_RECVPKTINFO, - MSG_DONTWAIT, SCM_RIGHTS, TCP_MAXSEG, - }; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::IPV6_PKTINFO; - - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::AI_CANONNAME; - - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple", - windows - ))] - #[pyattr] - use c::{ - EAI_AGAIN, EAI_BADFLAGS, EAI_FAIL, EAI_FAMILY, EAI_MEMORY, EAI_NONAME, EAI_SERVICE, - EAI_SOCKTYPE, IPV6_RECVTCLASS, IPV6_TCLASS, IP_HDRINCL, IP_TOS, SOMAXCONN, - }; - - #[cfg(not(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_vendor = "apple", - windows - )))] - #[pyattr] - const SOMAXCONN: i32 = 5; // Common value - - // HERE IS WHERE THE BLUETOOTH CONSTANTS START - // TODO: there should be a more intelligent way of detecting bluetooth on a platform. - // CPython uses header-detection, but blocks NetBSD and DragonFly BSD - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "openbsd" - ))] - #[pyattr] - use c::AF_BLUETOOTH; - - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "openbsd" - ))] - #[pyattr] - const BDADDR_ANY: &str = "00:00:00:00:00:00"; - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - target_os = "openbsd" - ))] - #[pyattr] - const BDADDR_LOCAL: &str = "00:00:00:FF:FF:FF"; - // HERE IS WHERE THE BLUETOOTH CONSTANTS END - - #[cfg(windows)] - #[pyattr] - use windows_sys::Win32::Networking::WinSock::{ - IPPROTO_CBT, IPPROTO_ICLFXBM, IPPROTO_IGP, IPPROTO_L2TP, IPPROTO_PGM, IPPROTO_RDP, - IPPROTO_SCTP, IPPROTO_ST, - }; - - #[pyattr] - fn error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.exceptions.os_error.to_owned() - } - - #[pyattr] - fn timeout(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.exceptions.timeout_error.to_owned() - } - - #[pyattr(once)] - fn herror(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "socket", - "herror", - Some(vec![vm.ctx.exceptions.os_error.to_owned()]), - ) - } - #[pyattr(once)] - fn gaierror(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "socket", - "gaierror", - Some(vec![vm.ctx.exceptions.os_error.to_owned()]), - ) - } - - #[pyfunction] - fn htonl(x: u32) -> u32 { - u32::to_be(x) - } - #[pyfunction] - fn htons(x: u16) -> u16 { - u16::to_be(x) - } - #[pyfunction] - fn ntohl(x: u32) -> u32 { - u32::from_be(x) - } - #[pyfunction] - fn ntohs(x: u16) -> u16 { - u16::from_be(x) - } - - #[cfg(unix)] - type RawSocket = std::os::unix::io::RawFd; - #[cfg(windows)] - type RawSocket = std::os::windows::raw::SOCKET; - - #[cfg(unix)] - macro_rules! errcode { - ($e:ident) => { - c::$e - }; - } - #[cfg(windows)] - macro_rules! errcode { - ($e:ident) => { - paste::paste!(c::[<WSA $e>]) - }; -} - - #[cfg(windows)] - use winapi::shared::netioapi; - - fn get_raw_sock(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<RawSocket> { - #[cfg(unix)] - type CastFrom = libc::c_long; - #[cfg(windows)] - type CastFrom = libc::c_longlong; - - // should really just be to_index() but test_socket tests the error messages explicitly - if obj.fast_isinstance(vm.ctx.types.float_type) { - return Err(vm.new_type_error("integer argument expected, got float".to_owned())); - } - let int = obj - .try_index_opt(vm) - .unwrap_or_else(|| Err(vm.new_type_error("an integer is required".to_owned())))?; - int.try_to_primitive::<CastFrom>(vm) - .map(|sock| sock as RawSocket) - } - - #[pyattr(name = "socket")] - #[pyattr(name = "SocketType")] - #[pyclass(name = "socket")] - #[derive(Debug, PyPayload)] - pub struct PySocket { - kind: AtomicCell<i32>, - family: AtomicCell<i32>, - proto: AtomicCell<i32>, - pub(crate) timeout: AtomicCell<f64>, - sock: PyRwLock<Option<Socket>>, - } - - const _: () = assert!(std::mem::size_of::<Option<Socket>>() == std::mem::size_of::<Socket>()); - - impl Default for PySocket { - fn default() -> Self { - PySocket { - kind: AtomicCell::default(), - family: AtomicCell::default(), - proto: AtomicCell::default(), - timeout: AtomicCell::new(-1.0), - sock: PyRwLock::new(None), - } - } - } - - #[cfg(windows)] - const CLOSED_ERR: i32 = c::WSAENOTSOCK; - #[cfg(unix)] - const CLOSED_ERR: i32 = c::EBADF; - - impl Read for &PySocket { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { - (&mut &*self.sock()?).read(buf) - } - } - impl Write for &PySocket { - fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { - (&mut &*self.sock()?).write(buf) - } - - fn flush(&mut self) -> std::io::Result<()> { - (&mut &*self.sock()?).flush() - } - } - - impl PySocket { - pub fn sock_opt(&self) -> Option<PyMappedRwLockReadGuard<'_, Socket>> { - PyRwLockReadGuard::try_map(self.sock.read(), |sock| sock.as_ref()).ok() - } - - pub fn sock(&self) -> io::Result<PyMappedRwLockReadGuard<'_, Socket>> { - self.sock_opt() - .ok_or_else(|| io::Error::from_raw_os_error(CLOSED_ERR)) - } - - fn init_inner( - &self, - family: i32, - socket_kind: i32, - proto: i32, - sock: Socket, - ) -> io::Result<()> { - self.family.store(family); - self.kind.store(socket_kind); - self.proto.store(proto); - let mut s = self.sock.write(); - let sock = s.insert(sock); - let timeout = DEFAULT_TIMEOUT.load(); - self.timeout.store(timeout); - if timeout >= 0.0 { - sock.set_nonblocking(true)?; - } - Ok(()) - } - - /// returns Err(blocking) - pub fn get_timeout(&self) -> Result<Duration, bool> { - let timeout = self.timeout.load(); - if timeout > 0.0 { - Ok(Duration::from_secs_f64(timeout)) - } else { - Err(timeout != 0.0) - } - } - - fn sock_op<F, R>( - &self, - vm: &VirtualMachine, - select: SelectKind, - f: F, - ) -> Result<R, IoOrPyException> - where - F: FnMut() -> io::Result<R>, - { - self.sock_op_timeout_err(vm, select, self.get_timeout().ok(), f) - } - - fn sock_op_timeout_err<F, R>( - &self, - vm: &VirtualMachine, - select: SelectKind, - timeout: Option<Duration>, - mut f: F, - ) -> Result<R, IoOrPyException> - where - F: FnMut() -> io::Result<R>, - { - let deadline = timeout.map(Deadline::new); - - loop { - if deadline.is_some() || matches!(select, SelectKind::Connect) { - let interval = deadline.as_ref().map(|d| d.time_until()).transpose()?; - let res = sock_select(&*self.sock()?, select, interval); - match res { - Ok(true) => return Err(IoOrPyException::Timeout), - Err(e) if e.kind() == io::ErrorKind::Interrupted => { - vm.check_signals()?; - continue; - } - Err(e) => return Err(e.into()), - Ok(false) => {} // no timeout, continue as normal - } - } - - let err = loop { - // loop on interrupt - match f() { - Ok(x) => return Ok(x), - Err(e) if e.kind() == io::ErrorKind::Interrupted => vm.check_signals()?, - Err(e) => break e, - } - }; - if timeout.is_some() && err.kind() == io::ErrorKind::WouldBlock { - continue; - } - return Err(err.into()); - } - } - - fn extract_address( - &self, - addr: PyObjectRef, - caller: &str, - vm: &VirtualMachine, - ) -> Result<socket2::SockAddr, IoOrPyException> { - let family = self.family.load(); - match family { - #[cfg(unix)] - c::AF_UNIX => { - use std::os::unix::ffi::OsStrExt; - let buf = crate::vm::function::ArgStrOrBytesLike::try_from_object(vm, addr)?; - let path = &*buf.borrow_bytes(); - socket2::SockAddr::unix(ffi::OsStr::from_bytes(path)) - .map_err(|_| vm.new_os_error("AF_UNIX path too long".to_owned()).into()) - } - c::AF_INET => { - let tuple: PyTupleRef = addr.downcast().map_err(|obj| { - vm.new_type_error(format!( - "{}(): AF_INET address must be tuple, not {}", - caller, - obj.class().name() - )) - })?; - if tuple.len() != 2 { - return Err(vm - .new_type_error( - "AF_INET address must be a pair (host, post)".to_owned(), - ) - .into()); - } - let addr = Address::from_tuple(&tuple, vm)?; - let mut addr4 = get_addr(vm, addr.host, c::AF_INET)?; - match &mut addr4 { - SocketAddr::V4(addr4) => { - addr4.set_port(addr.port); - } - SocketAddr::V6(_) => unreachable!(), - } - Ok(addr4.into()) - } - c::AF_INET6 => { - let tuple: PyTupleRef = addr.downcast().map_err(|obj| { - vm.new_type_error(format!( - "{}(): AF_INET6 address must be tuple, not {}", - caller, - obj.class().name() - )) - })?; - match tuple.len() { - 2..=4 => {} - _ => return Err(vm.new_type_error( - "AF_INET6 address must be a tuple (host, port[, flowinfo[, scopeid]])" - .to_owned(), - ).into()), - } - let (addr, flowinfo, scopeid) = Address::from_tuple_ipv6(&tuple, vm)?; - let mut addr6 = get_addr(vm, addr.host, c::AF_INET6)?; - match &mut addr6 { - SocketAddr::V6(addr6) => { - addr6.set_port(addr.port); - addr6.set_flowinfo(flowinfo); - addr6.set_scope_id(scopeid); - } - SocketAddr::V4(_) => unreachable!(), - } - Ok(addr6.into()) - } - _ => Err(vm.new_os_error(format!("{caller}(): bad family")).into()), - } - } - - fn connect_inner( - &self, - address: PyObjectRef, - caller: &str, - vm: &VirtualMachine, - ) -> Result<(), IoOrPyException> { - let sock_addr = self.extract_address(address, caller, vm)?; - - let err = match self.sock()?.connect(&sock_addr) { - Ok(()) => return Ok(()), - Err(e) => e, - }; - - let wait_connect = if err.kind() == io::ErrorKind::Interrupted { - vm.check_signals()?; - self.timeout.load() != 0.0 - } else { - #[cfg(unix)] - use c::EINPROGRESS; - #[cfg(windows)] - use c::WSAEWOULDBLOCK as EINPROGRESS; - - self.timeout.load() > 0.0 && err.raw_os_error() == Some(EINPROGRESS) - }; - - if wait_connect { - // basically, connect() is async, and it registers an "error" on the socket when it's - // done connecting. SelectKind::Connect fills the errorfds fd_set, so if we wake up - // from poll and the error is EISCONN then we know that the connect is done - self.sock_op(vm, SelectKind::Connect, || { - let sock = self.sock()?; - let err = sock.take_error()?; - match err { - Some(e) if e.raw_os_error() == Some(libc::EISCONN) => Ok(()), - Some(e) => Err(e), - // TODO: is this accurate? - None => Ok(()), - } - }) - } else { - Err(err.into()) - } - } - } - - impl DefaultConstructor for PySocket {} - - impl Initializer for PySocket { - type Args = ( - OptionalArg<i32>, - OptionalArg<i32>, - OptionalArg<i32>, - OptionalOption<PyObjectRef>, - ); - - fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - Self::_init(zelf, args, vm).map_err(|e| e.into_pyexception(vm)) - } - } - - impl Representable for PySocket { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(format!( - "<socket object, fd={}, family={}, type={}, proto={}>", - // cast because INVALID_SOCKET is unsigned, so would show usize::MAX instead of -1 - zelf.fileno() as i64, - zelf.family.load(), - zelf.kind.load(), - zelf.proto.load(), - )) - } - } - - #[pyclass(with(DefaultConstructor, Initializer, Representable), flags(BASETYPE))] - impl PySocket { - fn _init( - zelf: PyRef<Self>, - (family, socket_kind, proto, fileno): <Self as Initializer>::Args, - vm: &VirtualMachine, - ) -> Result<(), IoOrPyException> { - let mut family = family.unwrap_or(-1); - let mut socket_kind = socket_kind.unwrap_or(-1); - let mut proto = proto.unwrap_or(-1); - let fileno = fileno - .flatten() - .map(|obj| get_raw_sock(obj, vm)) - .transpose()?; - let sock; - if let Some(fileno) = fileno { - sock = sock_from_raw(fileno, vm)?; - match sock.local_addr() { - Ok(addr) if family == -1 => family = addr.family() as i32, - Err(e) - if family == -1 - || matches!( - e.raw_os_error(), - Some(errcode!(ENOTSOCK)) | Some(errcode!(EBADF)) - ) => - { - std::mem::forget(sock); - return Err(e.into()); - } - _ => {} - } - if socket_kind == -1 { - socket_kind = sock.r#type().map_err(|e| e.into_pyexception(vm))?.into(); - } - cfg_if::cfg_if! { - if #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "linux", - ))] { - if proto == -1 { - proto = sock.protocol()?.map_or(0, Into::into); - } - } else { - proto = 0; - } - } - } else { - if family == -1 { - family = c::AF_INET as _ - } - if socket_kind == -1 { - socket_kind = c::SOCK_STREAM - } - if proto == -1 { - proto = 0 - } - sock = Socket::new( - Domain::from(family), - SocketType::from(socket_kind), - Some(Protocol::from(proto)), - )?; - }; - Ok(zelf.init_inner(family, socket_kind, proto, sock)?) - } - - #[pymethod] - fn connect( - &self, - address: PyObjectRef, - vm: &VirtualMachine, - ) -> Result<(), IoOrPyException> { - self.connect_inner(address, "connect", vm) - } - - #[pymethod] - fn connect_ex(&self, address: PyObjectRef, vm: &VirtualMachine) -> PyResult<i32> { - match self.connect_inner(address, "connect_ex", vm) { - Ok(()) => Ok(0), - Err(err) => err.errno(), - } - } - - #[pymethod] - fn bind(&self, address: PyObjectRef, vm: &VirtualMachine) -> Result<(), IoOrPyException> { - let sock_addr = self.extract_address(address, "bind", vm)?; - Ok(self.sock()?.bind(&sock_addr)?) - } - - #[pymethod] - fn listen(&self, backlog: OptionalArg<i32>) -> io::Result<()> { - let backlog = backlog.unwrap_or(128); - let backlog = if backlog < 0 { 0 } else { backlog }; - self.sock()?.listen(backlog) - } - - #[pymethod] - fn _accept( - &self, - vm: &VirtualMachine, - ) -> Result<(RawSocket, PyObjectRef), IoOrPyException> { - let (sock, addr) = self.sock_op(vm, SelectKind::Read, || self.sock()?.accept())?; - let fd = into_sock_fileno(sock); - Ok((fd, get_addr_tuple(&addr, vm))) - } - - #[pymethod] - fn recv( - &self, - bufsize: usize, - flags: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> Result<Vec<u8>, IoOrPyException> { - let flags = flags.unwrap_or(0); - let mut buffer = Vec::with_capacity(bufsize); - let sock = self.sock()?; - let n = self.sock_op(vm, SelectKind::Read, || { - sock.recv_with_flags(buffer.spare_capacity_mut(), flags) - })?; - unsafe { buffer.set_len(n) }; - Ok(buffer) - } - - #[pymethod] - fn recv_into( - &self, - buf: ArgMemoryBuffer, - flags: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> Result<usize, IoOrPyException> { - let flags = flags.unwrap_or(0); - let sock = self.sock()?; - let mut buf = buf.borrow_buf_mut(); - let buf = &mut *buf; - self.sock_op(vm, SelectKind::Read, || { - sock.recv_with_flags(slice_as_uninit(buf), flags) - }) - } - - #[pymethod] - fn recvfrom( - &self, - bufsize: isize, - flags: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> Result<(Vec<u8>, PyObjectRef), IoOrPyException> { - let flags = flags.unwrap_or(0); - let bufsize = bufsize - .to_usize() - .ok_or_else(|| vm.new_value_error("negative buffersize in recvfrom".to_owned()))?; - let mut buffer = Vec::with_capacity(bufsize); - let (n, addr) = self.sock_op(vm, SelectKind::Read, || { - self.sock()? - .recv_from_with_flags(buffer.spare_capacity_mut(), flags) - })?; - unsafe { buffer.set_len(n) }; - Ok((buffer, get_addr_tuple(&addr, vm))) - } - - #[pymethod] - fn recvfrom_into( - &self, - buf: ArgMemoryBuffer, - nbytes: OptionalArg<isize>, - flags: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> Result<(usize, PyObjectRef), IoOrPyException> { - let mut buf = buf.borrow_buf_mut(); - let buf = &mut *buf; - let buf = match nbytes { - OptionalArg::Present(i) => { - let i = i.to_usize().ok_or_else(|| { - vm.new_value_error("negative buffersize in recvfrom_into".to_owned()) - })?; - buf.get_mut(..i).ok_or_else(|| { - vm.new_value_error( - "nbytes is greater than the length of the buffer".to_owned(), - ) - })? - } - OptionalArg::Missing => buf, - }; - let flags = flags.unwrap_or(0); - let sock = self.sock()?; - let (n, addr) = self.sock_op(vm, SelectKind::Read, || { - sock.recv_from_with_flags(slice_as_uninit(buf), flags) - })?; - Ok((n, get_addr_tuple(&addr, vm))) - } - - #[pymethod] - fn send( - &self, - bytes: ArgBytesLike, - flags: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> Result<usize, IoOrPyException> { - let flags = flags.unwrap_or(0); - let buf = bytes.borrow_buf(); - let buf = &*buf; - self.sock_op(vm, SelectKind::Write, || { - self.sock()?.send_with_flags(buf, flags) - }) - } - - #[pymethod] - fn sendall( - &self, - bytes: ArgBytesLike, - flags: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> Result<(), IoOrPyException> { - let flags = flags.unwrap_or(0); - - let timeout = self.get_timeout().ok(); - - let deadline = timeout.map(Deadline::new); - - let buf = bytes.borrow_buf(); - let buf = &*buf; - let mut buf_offset = 0; - // now we have like 3 layers of interrupt loop :) - while buf_offset < buf.len() { - let interval = deadline.as_ref().map(|d| d.time_until()).transpose()?; - self.sock_op_timeout_err(vm, SelectKind::Write, interval, || { - let subbuf = &buf[buf_offset..]; - buf_offset += self.sock()?.send_with_flags(subbuf, flags)?; - Ok(()) - })?; - vm.check_signals()?; - } - Ok(()) - } - - #[pymethod] - fn sendto( - &self, - bytes: ArgBytesLike, - arg2: PyObjectRef, - arg3: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> Result<usize, IoOrPyException> { - // signature is bytes[, flags], address - let (flags, address) = match arg3 { - OptionalArg::Present(arg3) => { - // should just be i32::try_from_obj but tests check for error message - let int = arg2.try_index_opt(vm).unwrap_or_else(|| { - Err(vm.new_type_error("an integer is required".to_owned())) - })?; - let flags = int.try_to_primitive::<i32>(vm)?; - (flags, arg3) - } - OptionalArg::Missing => (0, arg2), - }; - let addr = self.extract_address(address, "sendto", vm)?; - let buf = bytes.borrow_buf(); - let buf = &*buf; - self.sock_op(vm, SelectKind::Write, || { - self.sock()?.send_to_with_flags(buf, &addr, flags) - }) - } - - #[pymethod] - fn close(&self) -> io::Result<()> { - let sock = self.detach(); - if sock != INVALID_SOCKET { - close_inner(sock)?; - } - Ok(()) - } - #[pymethod] - #[inline] - fn detach(&self) -> RawSocket { - let sock = self.sock.write().take(); - sock.map_or(INVALID_SOCKET, into_sock_fileno) - } - - #[pymethod] - fn fileno(&self) -> RawSocket { - self.sock - .read() - .as_ref() - .map_or(INVALID_SOCKET, sock_fileno) - } - - #[pymethod] - fn getsockname(&self, vm: &VirtualMachine) -> std::io::Result<PyObjectRef> { - let addr = self.sock()?.local_addr()?; - - Ok(get_addr_tuple(&addr, vm)) - } - #[pymethod] - fn getpeername(&self, vm: &VirtualMachine) -> std::io::Result<PyObjectRef> { - let addr = self.sock()?.peer_addr()?; - - Ok(get_addr_tuple(&addr, vm)) - } - - #[pymethod] - fn gettimeout(&self) -> Option<f64> { - let timeout = self.timeout.load(); - if timeout >= 0.0 { - Some(timeout) - } else { - None - } - } - - #[pymethod] - fn setblocking(&self, block: bool) -> io::Result<()> { - self.timeout.store(if block { -1.0 } else { 0.0 }); - self.sock()?.set_nonblocking(!block) - } - - #[pymethod] - fn getblocking(&self) -> bool { - self.timeout.load() != 0.0 - } - - #[pymethod] - fn settimeout(&self, timeout: Option<Duration>) -> io::Result<()> { - self.timeout - .store(timeout.map_or(-1.0, |d| d.as_secs_f64())); - // even if timeout is > 0 the socket needs to be nonblocking in order for us to select() on - // it - self.sock()?.set_nonblocking(timeout.is_some()) - } - - #[pymethod] - fn getsockopt( - &self, - level: i32, - name: i32, - buflen: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> Result<PyObjectRef, IoOrPyException> { - let sock = self.sock()?; - let fd = sock_fileno(&sock); - let buflen = buflen.unwrap_or(0); - if buflen == 0 { - let mut flag: libc::c_int = 0; - let mut flagsize = std::mem::size_of::<libc::c_int>() as _; - let ret = unsafe { - c::getsockopt( - fd as _, - level, - name, - &mut flag as *mut libc::c_int as *mut _, - &mut flagsize, - ) - }; - if ret < 0 { - return Err(crate::common::os::errno().into()); - } - Ok(vm.ctx.new_int(flag).into()) - } else { - if buflen <= 0 || buflen > 1024 { - return Err(vm - .new_os_error("getsockopt buflen out of range".to_owned()) - .into()); - } - let mut buf = vec![0u8; buflen as usize]; - let mut buflen = buflen as _; - let ret = unsafe { - c::getsockopt( - fd as _, - level, - name, - buf.as_mut_ptr() as *mut _, - &mut buflen, - ) - }; - if ret < 0 { - return Err(crate::common::os::errno().into()); - } - buf.truncate(buflen as usize); - Ok(vm.ctx.new_bytes(buf).into()) - } - } - - #[pymethod] - fn setsockopt( - &self, - level: i32, - name: i32, - value: Option<Either<ArgBytesLike, i32>>, - optlen: OptionalArg<u32>, - vm: &VirtualMachine, - ) -> Result<(), IoOrPyException> { - let sock = self.sock()?; - let fd = sock_fileno(&sock); - let ret = match (value, optlen) { - (Some(Either::A(b)), OptionalArg::Missing) => b.with_ref(|b| unsafe { - c::setsockopt(fd as _, level, name, b.as_ptr() as *const _, b.len() as _) - }), - (Some(Either::B(ref val)), OptionalArg::Missing) => unsafe { - c::setsockopt( - fd as _, - level, - name, - val as *const i32 as *const _, - std::mem::size_of::<i32>() as _, - ) - }, - (None, OptionalArg::Present(optlen)) => unsafe { - c::setsockopt(fd as _, level, name, std::ptr::null(), optlen as _) - }, - _ => { - return Err(vm - .new_type_error("expected the value arg xor the optlen arg".to_owned()) - .into()); - } - }; - if ret < 0 { - Err(crate::common::os::errno().into()) - } else { - Ok(()) - } - } - - #[pymethod] - fn shutdown(&self, how: i32, vm: &VirtualMachine) -> Result<(), IoOrPyException> { - let how = match how { - c::SHUT_RD => Shutdown::Read, - c::SHUT_WR => Shutdown::Write, - c::SHUT_RDWR => Shutdown::Both, - _ => { - return Err(vm - .new_value_error("`how` must be SHUT_RD, SHUT_WR, or SHUT_RDWR".to_owned()) - .into()) - } - }; - Ok(self.sock()?.shutdown(how)?) - } - - #[pygetset(name = "type")] - fn kind(&self) -> i32 { - self.kind.load() - } - #[pygetset] - fn family(&self) -> i32 { - self.family.load() - } - #[pygetset] - fn proto(&self) -> i32 { - self.proto.load() - } - } - - struct Address { - host: PyStrRef, - port: u16, - } - - impl ToSocketAddrs for Address { - type Iter = std::vec::IntoIter<SocketAddr>; - fn to_socket_addrs(&self) -> io::Result<Self::Iter> { - (self.host.as_str(), self.port).to_socket_addrs() - } - } - - impl TryFromObject for Address { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let tuple = PyTupleRef::try_from_object(vm, obj)?; - if tuple.len() != 2 { - Err(vm.new_type_error("Address tuple should have only 2 values".to_owned())) - } else { - Self::from_tuple(&tuple, vm) - } - } - } - - impl Address { - fn from_tuple(tuple: &[PyObjectRef], vm: &VirtualMachine) -> PyResult<Self> { - let host = PyStrRef::try_from_object(vm, tuple[0].clone())?; - let port = i32::try_from_borrowed_object(vm, &tuple[1])?; - let port = port - .to_u16() - .ok_or_else(|| vm.new_overflow_error("port must be 0-65535.".to_owned()))?; - Ok(Address { host, port }) - } - fn from_tuple_ipv6( - tuple: &[PyObjectRef], - vm: &VirtualMachine, - ) -> PyResult<(Self, u32, u32)> { - let addr = Address::from_tuple(tuple, vm)?; - let flowinfo = tuple - .get(2) - .map(|obj| u32::try_from_borrowed_object(vm, obj)) - .transpose()? - .unwrap_or(0); - let scopeid = tuple - .get(3) - .map(|obj| u32::try_from_borrowed_object(vm, obj)) - .transpose()? - .unwrap_or(0); - if flowinfo > 0xfffff { - return Err(vm.new_overflow_error("flowinfo must be 0-1048575.".to_owned())); - } - Ok((addr, flowinfo, scopeid)) - } - } - - fn get_ip_addr_tuple(addr: &SocketAddr, vm: &VirtualMachine) -> PyObjectRef { - match addr { - SocketAddr::V4(addr) => (addr.ip().to_string(), addr.port()).to_pyobject(vm), - SocketAddr::V6(addr) => ( - addr.ip().to_string(), - addr.port(), - addr.flowinfo(), - addr.scope_id(), - ) - .to_pyobject(vm), - } - } - - fn get_addr_tuple(addr: &socket2::SockAddr, vm: &VirtualMachine) -> PyObjectRef { - if let Some(addr) = addr.as_socket() { - return get_ip_addr_tuple(&addr, vm); - } - #[cfg(unix)] - use nix::sys::socket::{SockaddrLike, UnixAddr}; - #[cfg(unix)] - if let Some(unix_addr) = unsafe { UnixAddr::from_raw(addr.as_ptr(), Some(addr.len())) } { - use std::os::unix::ffi::OsStrExt; - #[cfg(any(target_os = "android", target_os = "linux"))] - if let Some(abstractpath) = unix_addr.as_abstract() { - return vm.ctx.new_bytes([b"\0", abstractpath].concat()).into(); - } - // necessary on macos - let path = ffi::OsStr::as_bytes(unix_addr.path().unwrap_or("".as_ref()).as_ref()); - let nul_pos = memchr::memchr(b'\0', path).unwrap_or(path.len()); - let path = ffi::OsStr::from_bytes(&path[..nul_pos]); - return vm.ctx.new_str(path.to_string_lossy()).into(); - } - // TODO: support more address families - (String::new(), 0).to_pyobject(vm) - } - - #[pyfunction] - fn gethostname(vm: &VirtualMachine) -> PyResult<PyStrRef> { - gethostname::gethostname() - .into_string() - .map(|hostname| vm.ctx.new_str(hostname)) - .map_err(|err| vm.new_os_error(err.into_string().unwrap())) - } - - #[cfg(all(unix, not(target_os = "redox")))] - #[pyfunction] - fn sethostname(hostname: PyStrRef) -> nix::Result<()> { - nix::unistd::sethostname(hostname.as_str()) - } - - #[pyfunction] - fn inet_aton(ip_string: PyStrRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - ip_string - .as_str() - .parse::<Ipv4Addr>() - .map(|ip_addr| Vec::<u8>::from(ip_addr.octets())) - .map_err(|_| { - vm.new_os_error("illegal IP address string passed to inet_aton".to_owned()) - }) - } - - #[pyfunction] - fn inet_ntoa(packed_ip: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let packed_ip = packed_ip.borrow_buf(); - let packed_ip = <&[u8; 4]>::try_from(&*packed_ip) - .map_err(|_| vm.new_os_error("packed IP wrong length for inet_ntoa".to_owned()))?; - Ok(vm.ctx.new_str(Ipv4Addr::from(*packed_ip).to_string())) - } - - fn cstr_opt_as_ptr(x: &OptionalArg<ffi::CString>) -> *const libc::c_char { - x.as_ref().map_or_else(std::ptr::null, |s| s.as_ptr()) - } - - #[pyfunction] - fn getservbyname( - servicename: PyStrRef, - protocolname: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<u16> { - let cstr_name = servicename.to_cstring(vm)?; - let cstr_proto = protocolname - .as_ref() - .map(|s| s.to_cstring(vm)) - .transpose()?; - let cstr_proto = cstr_opt_as_ptr(&cstr_proto); - let serv = unsafe { c::getservbyname(cstr_name.as_ptr(), cstr_proto) }; - if serv.is_null() { - return Err(vm.new_os_error("service/proto not found".to_owned())); - } - let port = unsafe { (*serv).s_port }; - Ok(u16::from_be(port as u16)) - } - - #[pyfunction] - fn getservbyport( - port: i32, - protocolname: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<String> { - let port = port.to_u16().ok_or_else(|| { - vm.new_overflow_error("getservbyport: port must be 0-65535.".to_owned()) - })?; - let cstr_proto = protocolname - .as_ref() - .map(|s| s.to_cstring(vm)) - .transpose()?; - let cstr_proto = cstr_opt_as_ptr(&cstr_proto); - let serv = unsafe { c::getservbyport(port.to_be() as _, cstr_proto) }; - if serv.is_null() { - return Err(vm.new_os_error("port/proto not found".to_owned())); - } - let s = unsafe { ffi::CStr::from_ptr((*serv).s_name) }; - Ok(s.to_string_lossy().into_owned()) - } - - fn slice_as_uninit<T>(v: &mut [T]) -> &mut [MaybeUninit<T>] { - unsafe { &mut *(v as *mut [T] as *mut [MaybeUninit<T>]) } - } - - enum IoOrPyException { - Timeout, - Py(PyBaseExceptionRef), - Io(io::Error), - } - impl From<PyBaseExceptionRef> for IoOrPyException { - fn from(exc: PyBaseExceptionRef) -> Self { - Self::Py(exc) - } - } - impl From<io::Error> for IoOrPyException { - fn from(err: io::Error) -> Self { - Self::Io(err) - } - } - impl IoOrPyException { - fn errno(self) -> PyResult<i32> { - match self { - Self::Timeout => Ok(errcode!(EWOULDBLOCK)), - Self::Io(err) => { - // TODO: just unwrap()? - Ok(err.raw_os_error().unwrap_or(1)) - } - Self::Py(exc) => Err(exc), - } - } - } - impl IntoPyException for IoOrPyException { - #[inline] - fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { - match self { - Self::Timeout => timeout_error(vm), - Self::Py(exc) => exc, - Self::Io(err) => err.into_pyexception(vm), - } - } - } - - #[derive(Copy, Clone)] - pub(crate) enum SelectKind { - Read, - Write, - Connect, - } - - /// returns true if timed out - pub(crate) fn sock_select( - sock: &Socket, - kind: SelectKind, - interval: Option<Duration>, - ) -> io::Result<bool> { - let fd = sock_fileno(sock); - #[cfg(unix)] - { - use nix::poll::*; - let events = match kind { - SelectKind::Read => PollFlags::POLLIN, - SelectKind::Write => PollFlags::POLLOUT, - SelectKind::Connect => PollFlags::POLLOUT | PollFlags::POLLERR, - }; - let mut pollfd = [PollFd::new(fd, events)]; - let timeout = match interval { - Some(d) => d.as_millis() as _, - None => -1, - }; - let ret = poll(&mut pollfd, timeout)?; - Ok(ret == 0) - } - #[cfg(windows)] - { - use crate::select; - - let mut reads = select::FdSet::new(); - let mut writes = select::FdSet::new(); - let mut errs = select::FdSet::new(); - - let fd = fd as usize; - match kind { - SelectKind::Read => reads.insert(fd), - SelectKind::Write => writes.insert(fd), - SelectKind::Connect => { - writes.insert(fd); - errs.insert(fd); - } - } - - let mut interval = interval.map(|dur| select::timeval { - tv_sec: dur.as_secs() as _, - tv_usec: dur.subsec_micros() as _, - }); - - select::select( - fd as i32 + 1, - &mut reads, - &mut writes, - &mut errs, - interval.as_mut(), - ) - .map(|ret| ret == 0) - } - } - - #[derive(FromArgs)] - struct GAIOptions { - #[pyarg(positional)] - host: Option<PyStrRef>, - #[pyarg(positional)] - port: Option<Either<PyStrRef, i32>>, - - #[pyarg(positional, default = "c::AF_UNSPEC")] - family: i32, - #[pyarg(positional, default = "0")] - ty: i32, - #[pyarg(positional, default = "0")] - proto: i32, - #[pyarg(positional, default = "0")] - flags: i32, - } - - #[pyfunction] - fn getaddrinfo( - opts: GAIOptions, - vm: &VirtualMachine, - ) -> Result<Vec<PyObjectRef>, IoOrPyException> { - let hints = dns_lookup::AddrInfoHints { - socktype: opts.ty, - protocol: opts.proto, - address: opts.family, - flags: opts.flags, - }; - - let host = opts.host.as_ref().map(|s| s.as_str()); - let port = opts.port.as_ref().map(|p| -> std::borrow::Cow<str> { - match p { - Either::A(ref s) => s.as_str().into(), - Either::B(i) => i.to_string().into(), - } - }); - let port = port.as_ref().map(|p| p.as_ref()); - - let addrs = dns_lookup::getaddrinfo(host, port, Some(hints)) - .map_err(|err| convert_socket_error(vm, err, SocketError::GaiError))?; - - let list = addrs - .map(|ai| { - ai.map(|ai| { - vm.new_tuple(( - ai.address, - ai.socktype, - ai.protocol, - ai.canonname, - get_ip_addr_tuple(&ai.sockaddr, vm), - )) - .into() - }) - }) - .collect::<io::Result<Vec<_>>>()?; - Ok(list) - } - - #[pyfunction] - fn gethostbyaddr( - addr: PyStrRef, - vm: &VirtualMachine, - ) -> Result<(String, PyListRef, PyListRef), IoOrPyException> { - let addr = get_addr(vm, addr, c::AF_UNSPEC)?; - let (hostname, _) = dns_lookup::getnameinfo(&addr, 0) - .map_err(|e| convert_socket_error(vm, e, SocketError::HError))?; - Ok(( - hostname, - vm.ctx.new_list(vec![]), - vm.ctx - .new_list(vec![vm.ctx.new_str(addr.ip().to_string()).into()]), - )) - } - - #[pyfunction] - fn gethostbyname(name: PyStrRef, vm: &VirtualMachine) -> Result<String, IoOrPyException> { - let addr = get_addr(vm, name, c::AF_INET)?; - match addr { - SocketAddr::V4(ip) => Ok(ip.ip().to_string()), - _ => unreachable!(), - } - } - - #[pyfunction] - fn gethostbyname_ex( - name: PyStrRef, - vm: &VirtualMachine, - ) -> Result<(String, PyListRef, PyListRef), IoOrPyException> { - let addr = get_addr(vm, name, c::AF_INET)?; - let (hostname, _) = dns_lookup::getnameinfo(&addr, 0) - .map_err(|e| convert_socket_error(vm, e, SocketError::HError))?; - Ok(( - hostname, - vm.ctx.new_list(vec![]), - vm.ctx - .new_list(vec![vm.ctx.new_str(addr.ip().to_string()).into()]), - )) - } - - #[pyfunction] - fn inet_pton(af_inet: i32, ip_string: PyStrRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - static ERROR_MSG: &str = "illegal IP address string passed to inet_pton"; - let ip_addr = match af_inet { - c::AF_INET => ip_string - .as_str() - .parse::<Ipv4Addr>() - .map_err(|_| vm.new_os_error(ERROR_MSG.to_owned()))? - .octets() - .to_vec(), - c::AF_INET6 => ip_string - .as_str() - .parse::<Ipv6Addr>() - .map_err(|_| vm.new_os_error(ERROR_MSG.to_owned()))? - .octets() - .to_vec(), - _ => return Err(vm.new_os_error("Address family not supported by protocol".to_owned())), - }; - Ok(ip_addr) - } - - #[pyfunction] - fn inet_ntop(af_inet: i32, packed_ip: ArgBytesLike, vm: &VirtualMachine) -> PyResult<String> { - let packed_ip = packed_ip.borrow_buf(); - match af_inet { - c::AF_INET => { - let packed_ip = <&[u8; 4]>::try_from(&*packed_ip).map_err(|_| { - vm.new_value_error("invalid length of packed IP address string".to_owned()) - })?; - Ok(Ipv4Addr::from(*packed_ip).to_string()) - } - c::AF_INET6 => { - let packed_ip = <&[u8; 16]>::try_from(&*packed_ip).map_err(|_| { - vm.new_value_error("invalid length of packed IP address string".to_owned()) - })?; - Ok(get_ipv6_addr_str(Ipv6Addr::from(*packed_ip))) - } - _ => Err(vm.new_value_error(format!("unknown address family {af_inet}"))), - } - } - - #[pyfunction] - fn getprotobyname(name: PyStrRef, vm: &VirtualMachine) -> PyResult { - let cstr = name.to_cstring(vm)?; - let proto = unsafe { c::getprotobyname(cstr.as_ptr()) }; - if proto.is_null() { - return Err(vm.new_os_error("protocol not found".to_owned())); - } - let num = unsafe { (*proto).p_proto }; - Ok(vm.ctx.new_int(num).into()) - } - - #[pyfunction] - fn getnameinfo( - address: PyTupleRef, - flags: i32, - vm: &VirtualMachine, - ) -> Result<(String, String), IoOrPyException> { - match address.len() { - 2..=4 => {} - _ => { - return Err(vm - .new_type_error("illegal sockaddr argument".to_owned()) - .into()) - } - } - let (addr, flowinfo, scopeid) = Address::from_tuple_ipv6(&address, vm)?; - let hints = dns_lookup::AddrInfoHints { - address: c::AF_UNSPEC, - socktype: c::SOCK_DGRAM, - flags: c::AI_NUMERICHOST, - protocol: 0, - }; - let service = addr.port.to_string(); - let mut res = - dns_lookup::getaddrinfo(Some(addr.host.as_str()), Some(&service), Some(hints)) - .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError))? - .filter_map(Result::ok); - let mut ainfo = res.next().unwrap(); - if res.next().is_some() { - return Err(vm - .new_os_error("sockaddr resolved to multiple addresses".to_owned()) - .into()); - } - match &mut ainfo.sockaddr { - SocketAddr::V4(_) => { - if address.len() != 2 { - return Err(vm - .new_os_error("IPv4 sockaddr must be 2 tuple".to_owned()) - .into()); - } - } - SocketAddr::V6(addr) => { - addr.set_flowinfo(flowinfo); - addr.set_scope_id(scopeid); - } - } - dns_lookup::getnameinfo(&ainfo.sockaddr, flags) - .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError)) - } - - #[cfg(unix)] - #[pyfunction] - fn socketpair( - family: OptionalArg<i32>, - socket_kind: OptionalArg<i32>, - proto: OptionalArg<i32>, - ) -> Result<(PySocket, PySocket), IoOrPyException> { - let family = family.unwrap_or(libc::AF_UNIX); - let socket_kind = socket_kind.unwrap_or(libc::SOCK_STREAM); - let proto = proto.unwrap_or(0); - let (a, b) = Socket::pair(family.into(), socket_kind.into(), Some(proto.into()))?; - let py_a = PySocket::default(); - py_a.init_inner(family, socket_kind, proto, a)?; - let py_b = PySocket::default(); - py_b.init_inner(family, socket_kind, proto, b)?; - Ok((py_a, py_b)) - } - - #[cfg(all(unix, not(target_os = "redox")))] - type IfIndex = c::c_uint; - #[cfg(windows)] - type IfIndex = winapi::shared::ifdef::NET_IFINDEX; - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn if_nametoindex(name: FsPath, vm: &VirtualMachine) -> PyResult<IfIndex> { - let name = name.to_cstring(vm)?; - - let ret = unsafe { c::if_nametoindex(name.as_ptr()) }; - if ret == 0 { - Err(vm.new_os_error("no interface with this name".to_owned())) - } else { - Ok(ret) - } - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn if_indextoname(index: IfIndex, vm: &VirtualMachine) -> PyResult<String> { - let mut buf = [0; c::IF_NAMESIZE + 1]; - let ret = unsafe { c::if_indextoname(index, buf.as_mut_ptr()) }; - if ret.is_null() { - Err(crate::vm::stdlib::os::errno_err(vm)) - } else { - let buf = unsafe { ffi::CStr::from_ptr(buf.as_ptr()) }; - Ok(buf.to_string_lossy().into_owned()) - } - } - - #[cfg(any( - windows, - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "ios", - target_os = "linux", - target_os = "macos", - target_os = "netbsd", - target_os = "openbsd", - ))] - #[pyfunction] - fn if_nameindex(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - #[cfg(not(windows))] - { - let list = nix::net::if_::if_nameindex() - .map_err(|err| err.into_pyexception(vm))? - .to_slice() - .iter() - .map(|iface| { - let tup: (u32, String) = - (iface.index(), iface.name().to_string_lossy().into_owned()); - tup.to_pyobject(vm) - }) - .collect(); - - Ok(list) - } - #[cfg(windows)] - { - use std::ptr; - - let table = MibTable::get_raw().map_err(|err| err.into_pyexception(vm))?; - let list = table.as_slice().iter().map(|entry| { - let name = - get_name(&entry.InterfaceLuid).map_err(|err| err.into_pyexception(vm))?; - let tup = (entry.InterfaceIndex, name.to_string_lossy()); - Ok(tup.to_pyobject(vm)) - }); - let list = list.collect::<PyResult<_>>()?; - return Ok(list); - - fn get_name( - luid: &winapi::shared::ifdef::NET_LUID, - ) -> io::Result<widestring::WideCString> { - let mut buf = [0; c::IF_NAMESIZE + 1]; - let ret = unsafe { - netioapi::ConvertInterfaceLuidToNameW(luid, buf.as_mut_ptr(), buf.len()) - }; - if ret == 0 { - Ok(widestring::WideCString::from_ustr_truncate( - widestring::WideStr::from_slice(&buf[..]), - )) - } else { - Err(io::Error::from_raw_os_error(ret as i32)) - } - } - struct MibTable { - ptr: ptr::NonNull<netioapi::MIB_IF_TABLE2>, - } - impl MibTable { - fn get_raw() -> io::Result<Self> { - let mut ptr = ptr::null_mut(); - let ret = unsafe { netioapi::GetIfTable2Ex(netioapi::MibIfTableRaw, &mut ptr) }; - if ret == 0 { - let ptr = unsafe { ptr::NonNull::new_unchecked(ptr) }; - Ok(Self { ptr }) - } else { - Err(io::Error::from_raw_os_error(ret as i32)) - } - } - } - impl MibTable { - fn as_slice(&self) -> &[netioapi::MIB_IF_ROW2] { - unsafe { - let p = self.ptr.as_ptr(); - let ptr = ptr::addr_of!((*p).Table) as *const netioapi::MIB_IF_ROW2; - std::slice::from_raw_parts(ptr, (*p).NumEntries as usize) - } - } - } - impl Drop for MibTable { - fn drop(&mut self) { - unsafe { netioapi::FreeMibTable(self.ptr.as_ptr() as *mut _) } - } - } - } - } - - fn get_addr( - vm: &VirtualMachine, - pyname: PyStrRef, - af: i32, - ) -> Result<SocketAddr, IoOrPyException> { - let name = pyname.as_str(); - if name.is_empty() { - let hints = dns_lookup::AddrInfoHints { - address: af, - socktype: c::SOCK_DGRAM, - flags: c::AI_PASSIVE, - protocol: 0, - }; - let mut res = dns_lookup::getaddrinfo(None, Some("0"), Some(hints)) - .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError))?; - let ainfo = res.next().unwrap()?; - if res.next().is_some() { - return Err(vm - .new_os_error("wildcard resolved to multiple address".to_owned()) - .into()); - } - return Ok(ainfo.sockaddr); - } - if name == "255.255.255.255" || name == "<broadcast>" { - match af { - c::AF_INET | c::AF_UNSPEC => {} - _ => { - return Err(vm - .new_os_error("address family mismatched".to_owned()) - .into()) - } - } - return Ok(SocketAddr::V4(net::SocketAddrV4::new( - c::INADDR_BROADCAST.into(), - 0, - ))); - } - if let c::AF_INET | c::AF_UNSPEC = af { - if let Ok(addr) = name.parse::<Ipv4Addr>() { - return Ok(SocketAddr::V4(net::SocketAddrV4::new(addr, 0))); - } - } - if matches!(af, c::AF_INET | c::AF_UNSPEC) && !name.contains('%') { - if let Ok(addr) = name.parse::<Ipv6Addr>() { - return Ok(SocketAddr::V6(net::SocketAddrV6::new(addr, 0, 0, 0))); - } - } - let hints = dns_lookup::AddrInfoHints { - address: af, - ..Default::default() - }; - let name = vm - .state - .codec_registry - .encode_text(pyname, "idna", None, vm)?; - let name = std::str::from_utf8(name.as_bytes()) - .map_err(|_| vm.new_runtime_error("idna output is not utf8".to_owned()))?; - let mut res = dns_lookup::getaddrinfo(Some(name), None, Some(hints)) - .map_err(|e| convert_socket_error(vm, e, SocketError::GaiError))?; - Ok(res.next().unwrap().map(|ainfo| ainfo.sockaddr)?) - } - - fn sock_from_raw(fileno: RawSocket, vm: &VirtualMachine) -> PyResult<Socket> { - let invalid = { - cfg_if::cfg_if! { - if #[cfg(windows)] { - fileno == INVALID_SOCKET - } else { - fileno < 0 - } - } - }; - if invalid { - return Err(vm.new_value_error("negative file descriptor".to_owned())); - } - Ok(unsafe { sock_from_raw_unchecked(fileno) }) - } - /// SAFETY: fileno must not be equal to INVALID_SOCKET - unsafe fn sock_from_raw_unchecked(fileno: RawSocket) -> Socket { - #[cfg(unix)] - { - use std::os::unix::io::FromRawFd; - Socket::from_raw_fd(fileno) - } - #[cfg(windows)] - { - use std::os::windows::io::FromRawSocket; - Socket::from_raw_socket(fileno) - } - } - pub(super) fn sock_fileno(sock: &Socket) -> RawSocket { - #[cfg(unix)] - { - use std::os::unix::io::AsRawFd; - sock.as_raw_fd() - } - #[cfg(windows)] - { - use std::os::windows::io::AsRawSocket; - sock.as_raw_socket() - } - } - fn into_sock_fileno(sock: Socket) -> RawSocket { - #[cfg(unix)] - { - use std::os::unix::io::IntoRawFd; - sock.into_raw_fd() - } - #[cfg(windows)] - { - use std::os::windows::io::IntoRawSocket; - sock.into_raw_socket() - } - } - - pub(super) const INVALID_SOCKET: RawSocket = { - #[cfg(unix)] - { - -1 - } - #[cfg(windows)] - { - windows_sys::Win32::Networking::WinSock::INVALID_SOCKET as RawSocket - } - }; - - fn convert_socket_error( - vm: &VirtualMachine, - err: dns_lookup::LookupError, - err_kind: SocketError, - ) -> IoOrPyException { - if let dns_lookup::LookupErrorKind::System = err.kind() { - return io::Error::from(err).into(); - } - let strerr = { - #[cfg(unix)] - { - let s = match err_kind { - SocketError::GaiError => unsafe { - ffi::CStr::from_ptr(libc::gai_strerror(err.error_num())) - }, - SocketError::HError => unsafe { - ffi::CStr::from_ptr(libc::hstrerror(err.error_num())) - }, - }; - s.to_str().unwrap() - } - #[cfg(windows)] - { - "getaddrinfo failed" - } - }; - let exception_cls = match err_kind { - SocketError::GaiError => gaierror(vm), - SocketError::HError => herror(vm), - }; - vm.new_exception( - exception_cls, - vec![vm.new_pyobj(err.error_num()), vm.ctx.new_str(strerr).into()], - ) - .into() - } - - fn timeout_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - timeout_error_msg(vm, "timed out".to_owned()) - } - pub(crate) fn timeout_error_msg(vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { - vm.new_exception_msg(timeout(vm), msg) - } - - fn get_ipv6_addr_str(ipv6: Ipv6Addr) -> String { - match ipv6.to_ipv4() { - // instead of "::0.0.ddd.ddd" it's "::xxxx" - Some(v4) if !ipv6.is_unspecified() && matches!(v4.octets(), [0, 0, _, _]) => { - format!("::{:x}", u32::from(v4)) - } - _ => ipv6.to_string(), - } - } - - pub(crate) struct Deadline { - deadline: Instant, - } - - impl Deadline { - fn new(timeout: Duration) -> Self { - Self { - deadline: Instant::now() + timeout, - } - } - fn time_until(&self) -> Result<Duration, IoOrPyException> { - self.deadline - .checked_duration_since(Instant::now()) - // past the deadline already - .ok_or(IoOrPyException::Timeout) - } - } - - static DEFAULT_TIMEOUT: AtomicCell<f64> = AtomicCell::new(-1.0); - - #[pyfunction] - fn getdefaulttimeout() -> Option<f64> { - let timeout = DEFAULT_TIMEOUT.load(); - if timeout >= 0.0 { - Some(timeout) - } else { - None - } - } - - #[pyfunction] - fn setdefaulttimeout(timeout: Option<Duration>) { - DEFAULT_TIMEOUT.store(timeout.map_or(-1.0, |d| d.as_secs_f64())); - } - - #[pyfunction] - fn dup(x: PyObjectRef, vm: &VirtualMachine) -> Result<RawSocket, IoOrPyException> { - let sock = get_raw_sock(x, vm)?; - let sock = std::mem::ManuallyDrop::new(sock_from_raw(sock, vm)?); - let newsock = sock.try_clone()?; - let fd = into_sock_fileno(newsock); - #[cfg(windows)] - crate::vm::stdlib::nt::raw_set_handle_inheritable(fd as _, false)?; - Ok(fd) - } - - #[pyfunction] - fn close(x: PyObjectRef, vm: &VirtualMachine) -> Result<(), IoOrPyException> { - Ok(close_inner(get_raw_sock(x, vm)?)?) - } - - fn close_inner(x: RawSocket) -> io::Result<()> { - #[cfg(unix)] - use libc::close; - #[cfg(windows)] - use windows_sys::Win32::Networking::WinSock::closesocket as close; - let ret = unsafe { close(x as _) }; - if ret < 0 { - let err = crate::common::os::errno(); - if err.raw_os_error() != Some(errcode!(ECONNRESET)) { - return Err(err); - } - } - Ok(()) - } - - enum SocketError { - HError, - GaiError, - } -} diff --git a/stdlib/src/sqlite.rs b/stdlib/src/sqlite.rs deleted file mode 100644 index ecc5fea71c7..00000000000 --- a/stdlib/src/sqlite.rs +++ /dev/null @@ -1,3045 +0,0 @@ -// spell-checker:ignore libsqlite3 threadsafety PYSQLITE decltypes colnames collseq cantinit dirtywal -// spell-checker:ignore corruptfs narg setinputsizes setoutputsize lastrowid arraysize executemany -// spell-checker:ignore blobopen executescript iterdump getlimit setlimit errorcode errorname -// spell-checker:ignore rowid rowcount fetchone fetchmany fetchall errcode errname vtable pagecount -// spell-checker:ignore autocommit libversion toobig errmsg nomem threadsafe longlong vdbe reindex -// spell-checker:ignore savepoint cantopen ioerr nolfs nomem notadb notfound fullpath notempdir vtab -// spell-checker:ignore checkreservedlock noent fstat rdlock shmlock shmmap shmopen shmsize sharedcache -// spell-checker:ignore cantlock commithook foreignkey notnull primarykey gettemppath autoindex convpath -// spell-checker:ignore dbmoved vnode nbytes - -use rustpython_vm::{builtins::PyModule, AsObject, PyRef, VirtualMachine}; - -// pub(crate) use _sqlite::make_module; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - // TODO: sqlite version check - let module = _sqlite::make_module(vm); - _sqlite::setup_module(module.as_object(), vm); - module -} - -#[pymodule] -mod _sqlite { - use libsqlite3_sys::{ - sqlite3, sqlite3_aggregate_context, sqlite3_backup_finish, sqlite3_backup_init, - sqlite3_backup_pagecount, sqlite3_backup_remaining, sqlite3_backup_step, sqlite3_bind_blob, - sqlite3_bind_double, sqlite3_bind_int64, sqlite3_bind_null, sqlite3_bind_parameter_count, - sqlite3_bind_parameter_name, sqlite3_bind_text, sqlite3_blob, sqlite3_blob_bytes, - sqlite3_blob_close, sqlite3_blob_open, sqlite3_blob_read, sqlite3_blob_write, - sqlite3_busy_timeout, sqlite3_changes, sqlite3_close_v2, sqlite3_column_blob, - sqlite3_column_bytes, sqlite3_column_count, sqlite3_column_decltype, sqlite3_column_double, - sqlite3_column_int64, sqlite3_column_name, sqlite3_column_text, sqlite3_column_type, - sqlite3_complete, sqlite3_context, sqlite3_context_db_handle, sqlite3_create_collation_v2, - sqlite3_create_function_v2, sqlite3_create_window_function, sqlite3_data_count, - sqlite3_db_handle, sqlite3_errcode, sqlite3_errmsg, sqlite3_exec, sqlite3_expanded_sql, - sqlite3_extended_errcode, sqlite3_finalize, sqlite3_get_autocommit, sqlite3_interrupt, - sqlite3_last_insert_rowid, sqlite3_libversion, sqlite3_limit, sqlite3_open_v2, - sqlite3_prepare_v2, sqlite3_progress_handler, sqlite3_reset, sqlite3_result_blob, - sqlite3_result_double, sqlite3_result_error, sqlite3_result_error_nomem, - sqlite3_result_error_toobig, sqlite3_result_int64, sqlite3_result_null, - sqlite3_result_text, sqlite3_set_authorizer, sqlite3_sleep, sqlite3_step, sqlite3_stmt, - sqlite3_stmt_busy, sqlite3_stmt_readonly, sqlite3_threadsafe, sqlite3_total_changes, - sqlite3_trace_v2, sqlite3_user_data, sqlite3_value, sqlite3_value_blob, - sqlite3_value_bytes, sqlite3_value_double, sqlite3_value_int64, sqlite3_value_text, - sqlite3_value_type, SQLITE_BLOB, SQLITE_DETERMINISTIC, SQLITE_FLOAT, SQLITE_INTEGER, - SQLITE_NULL, SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE, SQLITE_OPEN_URI, SQLITE_TEXT, - SQLITE_TRACE_STMT, SQLITE_TRANSIENT, SQLITE_UTF8, - }; - use malachite_bigint::Sign; - use rustpython_common::{ - atomic::{Ordering, PyAtomic, Radium}, - hash::PyHash, - lock::{PyMappedMutexGuard, PyMutex, PyMutexGuard}, - static_cell, - }; - use rustpython_vm::{ - atomic_func, - builtins::{ - PyBaseException, PyBaseExceptionRef, PyByteArray, PyBytes, PyDict, PyDictRef, PyFloat, - PyInt, PyIntRef, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, - }, - convert::IntoObject, - function::{ArgCallable, ArgIterable, FsPath, FuncArgs, OptionalArg, PyComparisonValue}, - protocol::{PyBuffer, PyIterReturn, PyMappingMethods, PySequence, PySequenceMethods}, - sliceable::{SaturatedSliceIter, SliceableSequenceOp}, - types::{ - AsMapping, AsSequence, Callable, Comparable, Constructor, Hashable, IterNext, Iterable, - PyComparisonOp, SelfIter, - }, - utils::ToCString, - AsObject, Py, PyAtomicRef, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, VirtualMachine, - __exports::paste, - object::{Traverse, TraverseFn}, - }; - use std::{ - ffi::{c_int, c_longlong, c_uint, c_void, CStr}, - fmt::Debug, - ops::Deref, - ptr::{addr_of_mut, null, null_mut}, - thread::ThreadId, - }; - - macro_rules! exceptions { - ($(($x:ident, $base:expr)),*) => { - paste::paste! { - static_cell! { - $( - static [<$x:snake:upper>]: PyTypeRef; - )* - } - $( - #[allow(dead_code)] - fn [<new_ $x:snake>](vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { - vm.new_exception_msg([<$x:snake _type>]().to_owned(), msg) - } - fn [<$x:snake _type>]() -> &'static Py<PyType> { - [<$x:snake:upper>].get().expect("exception type not initialize") - } - )* - fn setup_module_exceptions(module: &PyObject, vm: &VirtualMachine) { - $( - #[allow(clippy::redundant_closure_call)] - let exception = [<$x:snake:upper>].get_or_init(|| { - let base = $base(vm); - vm.ctx.new_exception_type("_sqlite3", stringify!($x), Some(vec![base.to_owned()])) - }); - module.set_attr(stringify!($x), exception.clone().into_object(), vm).unwrap(); - )* - } - } - }; - } - - exceptions!( - (Warning, |vm: &VirtualMachine| vm - .ctx - .exceptions - .exception_type), - (Error, |vm: &VirtualMachine| vm - .ctx - .exceptions - .exception_type), - (InterfaceError, |_| error_type()), - (DatabaseError, |_| error_type()), - (DataError, |_| database_error_type()), - (OperationalError, |_| database_error_type()), - (IntegrityError, |_| database_error_type()), - (InternalError, |_| database_error_type()), - (ProgrammingError, |_| database_error_type()), - (NotSupportedError, |_| database_error_type()) - ); - - #[pyattr] - fn sqlite_version(vm: &VirtualMachine) -> String { - let s = unsafe { sqlite3_libversion() }; - ptr_to_str(s, vm).unwrap().to_owned() - } - - #[pyattr] - fn threadsafety(_: &VirtualMachine) -> c_int { - let mode = unsafe { sqlite3_threadsafe() }; - match mode { - 0 => 0, - 1 => 3, - 2 => 1, - _ => panic!("Unable to interpret SQLite threadsafety mode"), - } - } - - #[pyattr(name = "_deprecated_version")] - const PYSQLITE_VERSION: &str = "2.6.0"; - - #[pyattr] - const PARSE_DECLTYPES: c_int = 1; - #[pyattr] - const PARSE_COLNAMES: c_int = 2; - - #[pyattr] - use libsqlite3_sys::{ - SQLITE_ALTER_TABLE, SQLITE_ANALYZE, SQLITE_ATTACH, SQLITE_CREATE_INDEX, - SQLITE_CREATE_TABLE, SQLITE_CREATE_TEMP_INDEX, SQLITE_CREATE_TEMP_TABLE, - SQLITE_CREATE_TEMP_TRIGGER, SQLITE_CREATE_TEMP_VIEW, SQLITE_CREATE_TRIGGER, - SQLITE_CREATE_VIEW, SQLITE_CREATE_VTABLE, SQLITE_DELETE, SQLITE_DENY, SQLITE_DETACH, - SQLITE_DROP_INDEX, SQLITE_DROP_TABLE, SQLITE_DROP_TEMP_INDEX, SQLITE_DROP_TEMP_TABLE, - SQLITE_DROP_TEMP_TRIGGER, SQLITE_DROP_TEMP_VIEW, SQLITE_DROP_TRIGGER, SQLITE_DROP_VIEW, - SQLITE_DROP_VTABLE, SQLITE_FUNCTION, SQLITE_IGNORE, SQLITE_INSERT, SQLITE_LIMIT_ATTACHED, - SQLITE_LIMIT_COLUMN, SQLITE_LIMIT_COMPOUND_SELECT, SQLITE_LIMIT_EXPR_DEPTH, - SQLITE_LIMIT_FUNCTION_ARG, SQLITE_LIMIT_LENGTH, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, - SQLITE_LIMIT_SQL_LENGTH, SQLITE_LIMIT_TRIGGER_DEPTH, SQLITE_LIMIT_VARIABLE_NUMBER, - SQLITE_LIMIT_VDBE_OP, SQLITE_LIMIT_WORKER_THREADS, SQLITE_PRAGMA, SQLITE_READ, - SQLITE_RECURSIVE, SQLITE_REINDEX, SQLITE_SAVEPOINT, SQLITE_SELECT, SQLITE_TRANSACTION, - SQLITE_UPDATE, - }; - - macro_rules! error_codes { - ($($x:ident),*) => { - $( - #[allow(unused_imports)] - use libsqlite3_sys::$x; - )* - static ERROR_CODES: &[(&str, c_int)] = &[ - $( - (stringify!($x), libsqlite3_sys::$x), - )* - ]; - }; - } - - error_codes!( - SQLITE_ABORT, - SQLITE_AUTH, - SQLITE_BUSY, - SQLITE_CANTOPEN, - SQLITE_CONSTRAINT, - SQLITE_CORRUPT, - SQLITE_DONE, - SQLITE_EMPTY, - SQLITE_ERROR, - SQLITE_FORMAT, - SQLITE_FULL, - SQLITE_INTERNAL, - SQLITE_INTERRUPT, - SQLITE_IOERR, - SQLITE_LOCKED, - SQLITE_MISMATCH, - SQLITE_MISUSE, - SQLITE_NOLFS, - SQLITE_NOMEM, - SQLITE_NOTADB, - SQLITE_NOTFOUND, - SQLITE_OK, - SQLITE_PERM, - SQLITE_PROTOCOL, - SQLITE_RANGE, - SQLITE_READONLY, - SQLITE_ROW, - SQLITE_SCHEMA, - SQLITE_TOOBIG, - SQLITE_NOTICE, - SQLITE_WARNING, - SQLITE_ABORT_ROLLBACK, - SQLITE_BUSY_RECOVERY, - SQLITE_CANTOPEN_FULLPATH, - SQLITE_CANTOPEN_ISDIR, - SQLITE_CANTOPEN_NOTEMPDIR, - SQLITE_CORRUPT_VTAB, - SQLITE_IOERR_ACCESS, - SQLITE_IOERR_BLOCKED, - SQLITE_IOERR_CHECKRESERVEDLOCK, - SQLITE_IOERR_CLOSE, - SQLITE_IOERR_DELETE, - SQLITE_IOERR_DELETE_NOENT, - SQLITE_IOERR_DIR_CLOSE, - SQLITE_IOERR_DIR_FSYNC, - SQLITE_IOERR_FSTAT, - SQLITE_IOERR_FSYNC, - SQLITE_IOERR_LOCK, - SQLITE_IOERR_NOMEM, - SQLITE_IOERR_RDLOCK, - SQLITE_IOERR_READ, - SQLITE_IOERR_SEEK, - SQLITE_IOERR_SHMLOCK, - SQLITE_IOERR_SHMMAP, - SQLITE_IOERR_SHMOPEN, - SQLITE_IOERR_SHMSIZE, - SQLITE_IOERR_SHORT_READ, - SQLITE_IOERR_TRUNCATE, - SQLITE_IOERR_UNLOCK, - SQLITE_IOERR_WRITE, - SQLITE_LOCKED_SHAREDCACHE, - SQLITE_READONLY_CANTLOCK, - SQLITE_READONLY_RECOVERY, - SQLITE_CONSTRAINT_CHECK, - SQLITE_CONSTRAINT_COMMITHOOK, - SQLITE_CONSTRAINT_FOREIGNKEY, - SQLITE_CONSTRAINT_FUNCTION, - SQLITE_CONSTRAINT_NOTNULL, - SQLITE_CONSTRAINT_PRIMARYKEY, - SQLITE_CONSTRAINT_TRIGGER, - SQLITE_CONSTRAINT_UNIQUE, - SQLITE_CONSTRAINT_VTAB, - SQLITE_READONLY_ROLLBACK, - SQLITE_IOERR_MMAP, - SQLITE_NOTICE_RECOVER_ROLLBACK, - SQLITE_NOTICE_RECOVER_WAL, - SQLITE_BUSY_SNAPSHOT, - SQLITE_IOERR_GETTEMPPATH, - SQLITE_WARNING_AUTOINDEX, - SQLITE_CANTOPEN_CONVPATH, - SQLITE_IOERR_CONVPATH, - SQLITE_CONSTRAINT_ROWID, - SQLITE_READONLY_DBMOVED, - SQLITE_AUTH_USER, - SQLITE_OK_LOAD_PERMANENTLY, - SQLITE_IOERR_VNODE, - SQLITE_IOERR_AUTH, - SQLITE_IOERR_BEGIN_ATOMIC, - SQLITE_IOERR_COMMIT_ATOMIC, - SQLITE_IOERR_ROLLBACK_ATOMIC, - SQLITE_ERROR_MISSING_COLLSEQ, - SQLITE_ERROR_RETRY, - SQLITE_READONLY_CANTINIT, - SQLITE_READONLY_DIRECTORY, - SQLITE_CORRUPT_SEQUENCE, - SQLITE_LOCKED_VTAB, - SQLITE_CANTOPEN_DIRTYWAL, - SQLITE_ERROR_SNAPSHOT, - SQLITE_CANTOPEN_SYMLINK, - SQLITE_CONSTRAINT_PINNED, - SQLITE_OK_SYMLINK, - SQLITE_BUSY_TIMEOUT, - SQLITE_CORRUPT_INDEX, - SQLITE_IOERR_DATA, - SQLITE_IOERR_CORRUPTFS - ); - - #[derive(FromArgs)] - struct ConnectArgs { - #[pyarg(any)] - database: FsPath, - #[pyarg(any, default = "5.0")] - timeout: f64, - #[pyarg(any, default = "0")] - detect_types: c_int, - #[pyarg(any, default = "Some(vm.ctx.empty_str.to_owned())")] - isolation_level: Option<PyStrRef>, - #[pyarg(any, default = "true")] - check_same_thread: bool, - #[pyarg(any, default = "Connection::class(&vm.ctx).to_owned()")] - factory: PyTypeRef, - // TODO: cache statements - #[allow(dead_code)] - #[pyarg(any, default = "0")] - cached_statements: c_int, - #[pyarg(any, default = "false")] - uri: bool, - } - - unsafe impl Traverse for ConnectArgs { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.isolation_level.traverse(tracer_fn); - self.factory.traverse(tracer_fn); - } - } - - #[derive(FromArgs)] - struct BackupArgs { - #[pyarg(any)] - target: PyRef<Connection>, - #[pyarg(named, default = "-1")] - pages: c_int, - #[pyarg(named, optional)] - progress: Option<ArgCallable>, - #[pyarg(named, optional)] - name: Option<PyStrRef>, - #[pyarg(named, default = "0.250")] - sleep: f64, - } - - unsafe impl Traverse for BackupArgs { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.progress.traverse(tracer_fn); - self.name.traverse(tracer_fn); - } - } - - #[derive(FromArgs)] - struct CreateFunctionArgs { - #[pyarg(any)] - name: PyStrRef, - #[pyarg(any)] - narg: c_int, - #[pyarg(any)] - func: PyObjectRef, - #[pyarg(named, default)] - deterministic: bool, - } - - #[derive(FromArgs)] - struct CreateAggregateArgs { - #[pyarg(any)] - name: PyStrRef, - #[pyarg(positional)] - narg: c_int, - #[pyarg(positional)] - aggregate_class: PyObjectRef, - } - - #[derive(FromArgs)] - struct BlobOpenArgs { - #[pyarg(positional)] - table: PyStrRef, - #[pyarg(positional)] - column: PyStrRef, - #[pyarg(positional)] - row: i64, - #[pyarg(named, default)] - readonly: bool, - #[pyarg(named, default = "vm.ctx.new_str(stringify!(main))")] - name: PyStrRef, - } - - struct CallbackData { - obj: *const PyObject, - vm: *const VirtualMachine, - } - - impl CallbackData { - fn new(obj: PyObjectRef, vm: &VirtualMachine) -> Option<Self> { - (!vm.is_none(&obj)).then_some(Self { - obj: obj.into_raw(), - vm, - }) - } - - fn retrieve(&self) -> (&PyObject, &VirtualMachine) { - unsafe { (&*self.obj, &*self.vm) } - } - - unsafe extern "C" fn destructor(data: *mut c_void) { - drop(Box::from_raw(data.cast::<Self>())); - } - - unsafe extern "C" fn func_callback( - context: *mut sqlite3_context, - argc: c_int, - argv: *mut *mut sqlite3_value, - ) { - let context = SqliteContext::from(context); - let (func, vm) = (*context.user_data::<Self>()).retrieve(); - let args = std::slice::from_raw_parts(argv, argc as usize); - - let f = || -> PyResult<()> { - let db = context.db_handle(); - let args = args - .iter() - .cloned() - .map(|val| value_to_object(val, db, vm)) - .collect::<PyResult<Vec<PyObjectRef>>>()?; - - let val = func.call(args, vm)?; - - context.result_from_object(&val, vm) - }; - - if let Err(exc) = f() { - context.result_exception(vm, exc, "user-defined function raised exception\0") - } - } - - unsafe extern "C" fn step_callback( - context: *mut sqlite3_context, - argc: c_int, - argv: *mut *mut sqlite3_value, - ) { - let context = SqliteContext::from(context); - let (cls, vm) = (*context.user_data::<Self>()).retrieve(); - let args = std::slice::from_raw_parts(argv, argc as usize); - let instance = context.aggregate_context::<*const PyObject>(); - if (*instance).is_null() { - match cls.call((), vm) { - Ok(obj) => *instance = obj.into_raw(), - Err(exc) => { - return context.result_exception( - vm, - exc, - "user-defined aggregate's '__init__' method raised error\0", - ) - } - } - } - let instance = &**instance; - - Self::call_method_with_args(context, instance, "step", args, vm); - } - - unsafe extern "C" fn finalize_callback(context: *mut sqlite3_context) { - let context = SqliteContext::from(context); - let (_, vm) = (*context.user_data::<Self>()).retrieve(); - let instance = context.aggregate_context::<*const PyObject>(); - let Some(instance) = (*instance).as_ref() else { - return; - }; - - Self::callback_result_from_method(context, instance, "finalize", vm); - } - - unsafe extern "C" fn collation_callback( - data: *mut c_void, - a_len: c_int, - a_ptr: *const c_void, - b_len: c_int, - b_ptr: *const c_void, - ) -> c_int { - let (callable, vm) = (*data.cast::<Self>()).retrieve(); - - let f = || -> PyResult<c_int> { - let text1 = ptr_to_string(a_ptr.cast(), a_len, null_mut(), vm)?; - let text1 = vm.ctx.new_str(text1); - let text2 = ptr_to_string(b_ptr.cast(), b_len, null_mut(), vm)?; - let text2 = vm.ctx.new_str(text2); - - let val = callable.call((text1, text2), vm)?; - let Some(val) = val.to_number().index(vm) else { - return Ok(0); - }; - - let val = match val?.as_bigint().sign() { - Sign::Plus => 1, - Sign::Minus => -1, - Sign::NoSign => 0, - }; - - Ok(val) - }; - - f().unwrap_or(0) - } - - unsafe extern "C" fn value_callback(context: *mut sqlite3_context) { - let context = SqliteContext::from(context); - let (_, vm) = (*context.user_data::<Self>()).retrieve(); - let instance = context.aggregate_context::<*const PyObject>(); - let instance = &**instance; - - Self::callback_result_from_method(context, instance, "value", vm); - } - - unsafe extern "C" fn inverse_callback( - context: *mut sqlite3_context, - argc: c_int, - argv: *mut *mut sqlite3_value, - ) { - let context = SqliteContext::from(context); - let (_, vm) = (*context.user_data::<Self>()).retrieve(); - let args = std::slice::from_raw_parts(argv, argc as usize); - let instance = context.aggregate_context::<*const PyObject>(); - let instance = &**instance; - - Self::call_method_with_args(context, instance, "inverse", args, vm); - } - - unsafe extern "C" fn authorizer_callback( - data: *mut c_void, - action: c_int, - arg1: *const libc::c_char, - arg2: *const libc::c_char, - db_name: *const libc::c_char, - access: *const libc::c_char, - ) -> c_int { - let (callable, vm) = (*data.cast::<Self>()).retrieve(); - let f = || -> PyResult<c_int> { - let arg1 = ptr_to_str(arg1, vm)?; - let arg2 = ptr_to_str(arg2, vm)?; - let db_name = ptr_to_str(db_name, vm)?; - let access = ptr_to_str(access, vm)?; - - let val = callable.call((action, arg1, arg2, db_name, access), vm)?; - let Some(val) = val.payload::<PyInt>() else { - return Ok(SQLITE_DENY); - }; - val.try_to_primitive::<c_int>(vm) - }; - - f().unwrap_or(SQLITE_DENY) - } - - unsafe extern "C" fn trace_callback( - _typ: c_uint, - data: *mut c_void, - stmt: *mut c_void, - sql: *mut c_void, - ) -> c_int { - let (callable, vm) = (*data.cast::<Self>()).retrieve(); - let expanded = sqlite3_expanded_sql(stmt.cast()); - let f = || -> PyResult<()> { - let stmt = ptr_to_str(expanded, vm).or_else(|_| ptr_to_str(sql.cast(), vm))?; - callable.call((stmt,), vm)?; - Ok(()) - }; - let _ = f(); - 0 - } - - unsafe extern "C" fn progress_callback(data: *mut c_void) -> c_int { - let (callable, vm) = (*data.cast::<Self>()).retrieve(); - if let Ok(val) = callable.call((), vm) { - if let Ok(val) = val.is_true(vm) { - return val as c_int; - } - } - -1 - } - - fn callback_result_from_method( - context: SqliteContext, - instance: &PyObject, - name: &str, - vm: &VirtualMachine, - ) { - let f = || -> PyResult<()> { - let val = vm.call_method(instance, name, ())?; - context.result_from_object(&val, vm) - }; - - if let Err(exc) = f() { - if exc.fast_isinstance(vm.ctx.exceptions.attribute_error) { - context.result_exception( - vm, - exc, - &format!("user-defined aggregate's '{name}' method not defined\0"), - ) - } else { - context.result_exception( - vm, - exc, - &format!("user-defined aggregate's '{name}' method raised error\0"), - ) - } - } - } - - fn call_method_with_args( - context: SqliteContext, - instance: &PyObject, - name: &str, - args: &[*mut sqlite3_value], - vm: &VirtualMachine, - ) { - let f = || -> PyResult<()> { - let db = context.db_handle(); - let args = args - .iter() - .cloned() - .map(|val| value_to_object(val, db, vm)) - .collect::<PyResult<Vec<PyObjectRef>>>()?; - vm.call_method(instance, name, args).map(drop) - }; - - if let Err(exc) = f() { - if exc.fast_isinstance(vm.ctx.exceptions.attribute_error) { - context.result_exception( - vm, - exc, - &format!("user-defined aggregate's '{name}' method not defined\0"), - ) - } else { - context.result_exception( - vm, - exc, - &format!("user-defined aggregate's '{name}' method raised error\0"), - ) - } - } - } - } - - impl Drop for CallbackData { - fn drop(&mut self) { - unsafe { PyObjectRef::from_raw(self.obj) }; - } - } - - #[pyfunction] - fn connect(args: ConnectArgs, vm: &VirtualMachine) -> PyResult { - Connection::py_new(args.factory.clone(), args, vm) - } - - #[pyfunction] - fn complete_statement(statement: PyStrRef, vm: &VirtualMachine) -> PyResult<bool> { - let s = statement.to_cstring(vm)?; - let ret = unsafe { sqlite3_complete(s.as_ptr()) }; - Ok(ret == 1) - } - - #[pyfunction] - fn enable_callback_tracebacks(flag: bool) { - enable_traceback().store(flag, Ordering::Relaxed); - } - - #[pyfunction] - fn register_adapter(typ: PyTypeRef, adapter: ArgCallable, vm: &VirtualMachine) -> PyResult<()> { - if typ.is(PyInt::class(&vm.ctx)) - || typ.is(PyFloat::class(&vm.ctx)) - || typ.is(PyStr::class(&vm.ctx)) - || typ.is(PyByteArray::class(&vm.ctx)) - { - let _ = BASE_TYPE_ADAPTED.set(()); - } - let protocol = PrepareProtocol::class(&vm.ctx).to_owned(); - let key = vm.ctx.new_tuple(vec![typ.into(), protocol.into()]); - adapters().set_item(key.as_object(), adapter.into(), vm) - } - - #[pyfunction] - fn register_converter( - typename: PyStrRef, - converter: ArgCallable, - vm: &VirtualMachine, - ) -> PyResult<()> { - let name = typename.as_str().to_uppercase(); - converters().set_item(&name, converter.into(), vm) - } - - fn _adapt<F>(obj: &PyObject, proto: PyTypeRef, alt: F, vm: &VirtualMachine) -> PyResult - where - F: FnOnce(&PyObject) -> PyResult, - { - let proto = proto.into_object(); - let key = vm - .ctx - .new_tuple(vec![obj.class().to_owned().into(), proto.clone()]); - - if let Some(adapter) = adapters().get_item_opt(key.as_object(), vm)? { - return adapter.call((obj,), vm); - } - if let Ok(adapter) = proto.get_attr("__adapt__", vm) { - match adapter.call((obj,), vm) { - Ok(val) => return Ok(val), - Err(exc) => { - if !exc.fast_isinstance(vm.ctx.exceptions.type_error) { - return Err(exc); - } - } - } - } - if let Ok(adapter) = obj.get_attr("__conform__", vm) { - match adapter.call((proto,), vm) { - Ok(val) => return Ok(val), - Err(exc) => { - if !exc.fast_isinstance(vm.ctx.exceptions.type_error) { - return Err(exc); - } - } - } - } - - alt(obj) - } - - #[pyfunction] - fn adapt( - obj: PyObjectRef, - proto: OptionalArg<Option<PyTypeRef>>, - alt: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - // TODO: None proto - let proto = proto - .flatten() - .unwrap_or_else(|| PrepareProtocol::class(&vm.ctx).to_owned()); - - _adapt( - &obj, - proto, - |_| { - if let OptionalArg::Present(alt) = alt { - Ok(alt) - } else { - Err(new_programming_error(vm, "can't adapt".to_owned())) - } - }, - vm, - ) - } - - fn need_adapt(obj: &PyObject, vm: &VirtualMachine) -> bool { - if BASE_TYPE_ADAPTED.get().is_some() { - true - } else { - let cls = obj.class(); - !(cls.is(vm.ctx.types.int_type) - || cls.is(vm.ctx.types.float_type) - || cls.is(vm.ctx.types.str_type) - || cls.is(vm.ctx.types.bytearray_type)) - } - } - - static_cell! { - static CONVERTERS: PyDictRef; - static ADAPTERS: PyDictRef; - static BASE_TYPE_ADAPTED: (); - static USER_FUNCTION_EXCEPTION: PyAtomicRef<Option<PyBaseException>>; - static ENABLE_TRACEBACK: PyAtomic<bool>; - } - - fn converters() -> &'static Py<PyDict> { - CONVERTERS.get().expect("converters not initialize") - } - - fn adapters() -> &'static Py<PyDict> { - ADAPTERS.get().expect("adapters not initialize") - } - - fn user_function_exception() -> &'static PyAtomicRef<Option<PyBaseException>> { - USER_FUNCTION_EXCEPTION - .get() - .expect("user function exception not initialize") - } - - fn enable_traceback() -> &'static PyAtomic<bool> { - ENABLE_TRACEBACK - .get() - .expect("enable traceback not initialize") - } - - pub(super) fn setup_module(module: &PyObject, vm: &VirtualMachine) { - for (name, code) in ERROR_CODES { - let name = vm.ctx.intern_str(*name); - let code = vm.new_pyobj(*code); - module.set_attr(name, code, vm).unwrap(); - } - - setup_module_exceptions(module, vm); - - let _ = CONVERTERS.set(vm.ctx.new_dict()); - let _ = ADAPTERS.set(vm.ctx.new_dict()); - let _ = USER_FUNCTION_EXCEPTION.set(PyAtomicRef::from(None)); - let _ = ENABLE_TRACEBACK.set(Radium::new(false)); - - module - .set_attr("converters", converters().to_owned(), vm) - .unwrap(); - module - .set_attr("adapters", adapters().to_owned(), vm) - .unwrap(); - } - - #[pyattr] - #[pyclass(name)] - #[derive(PyPayload)] - struct Connection { - db: PyMutex<Option<Sqlite>>, - detect_types: c_int, - isolation_level: PyAtomicRef<Option<PyStr>>, - check_same_thread: bool, - thread_ident: ThreadId, - row_factory: PyAtomicRef<Option<PyObject>>, - text_factory: PyAtomicRef<PyObject>, - } - - impl Debug for Connection { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - write!(f, "Sqlite3 Connection") - } - } - - impl Constructor for Connection { - type Args = ConnectArgs; - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - Ok(Self::new(args, vm)?.into_ref_with_type(vm, cls)?.into()) - } - } - - impl Callable for Connection { - type Args = (PyStrRef,); - - fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult { - if let Some(stmt) = Statement::new(zelf, &args.0, vm)? { - Ok(stmt.into_ref(&vm.ctx).into()) - } else { - Ok(vm.ctx.none()) - } - } - } - - #[pyclass(with(Constructor, Callable), flags(BASETYPE))] - impl Connection { - fn new(args: ConnectArgs, vm: &VirtualMachine) -> PyResult<Self> { - let path = args.database.to_cstring(vm)?; - let db = Sqlite::from(SqliteRaw::open(path.as_ptr(), args.uri, vm)?); - let timeout = (args.timeout * 1000.0) as c_int; - db.busy_timeout(timeout); - if let Some(isolation_level) = &args.isolation_level { - begin_statement_ptr_from_isolation_level(isolation_level, vm)?; - } - let text_factory = PyStr::class(&vm.ctx).to_owned().into_object(); - - Ok(Self { - db: PyMutex::new(Some(db)), - detect_types: args.detect_types, - isolation_level: PyAtomicRef::from(args.isolation_level), - check_same_thread: args.check_same_thread, - thread_ident: std::thread::current().id(), - row_factory: PyAtomicRef::from(None), - text_factory: PyAtomicRef::from(text_factory), - }) - } - - fn db_lock(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<Sqlite>> { - self.check_thread(vm)?; - self._db_lock(vm) - } - - fn _db_lock(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<Sqlite>> { - let guard = self.db.lock(); - if guard.is_some() { - Ok(PyMutexGuard::map(guard, |x| unsafe { - x.as_mut().unwrap_unchecked() - })) - } else { - Err(new_programming_error( - vm, - "Cannot operate on a closed database.".to_owned(), - )) - } - } - - #[pymethod] - fn cursor( - zelf: PyRef<Self>, - factory: OptionalArg<ArgCallable>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Cursor>> { - zelf.db_lock(vm).map(drop)?; - - let cursor = if let OptionalArg::Present(factory) = factory { - let cursor = factory.invoke((zelf.clone(),), vm)?; - let cursor = cursor.downcast::<Cursor>().map_err(|x| { - vm.new_type_error(format!("factory must return a cursor, not {}", x.class())) - })?; - let _ = unsafe { cursor.row_factory.swap(zelf.row_factory.to_owned()) }; - cursor - } else { - let row_factory = zelf.row_factory.to_owned(); - Cursor::new(zelf, row_factory, vm).into_ref(&vm.ctx) - }; - Ok(cursor) - } - - #[pymethod] - fn blobopen( - zelf: PyRef<Self>, - args: BlobOpenArgs, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Blob>> { - let table = args.table.to_cstring(vm)?; - let column = args.column.to_cstring(vm)?; - let name = args.name.to_cstring(vm)?; - - let db = zelf.db_lock(vm)?; - - let mut blob = null_mut(); - let ret = unsafe { - sqlite3_blob_open( - db.db, - name.as_ptr(), - table.as_ptr(), - column.as_ptr(), - args.row, - (!args.readonly) as c_int, - &mut blob, - ) - }; - db.check(ret, vm)?; - drop(db); - - let blob = SqliteBlob { blob }; - let blob = Blob { - connection: zelf, - inner: PyMutex::new(Some(BlobInner { blob, offset: 0 })), - }; - Ok(blob.into_ref(&vm.ctx)) - } - - #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult<()> { - self.check_thread(vm)?; - self.db.lock().take(); - Ok(()) - } - - #[pymethod] - fn commit(&self, vm: &VirtualMachine) -> PyResult<()> { - self.db_lock(vm)?.implicit_commit(vm) - } - - #[pymethod] - fn rollback(&self, vm: &VirtualMachine) -> PyResult<()> { - let db = self.db_lock(vm)?; - if !db.is_autocommit() { - db._exec(b"ROLLBACK\0", vm) - } else { - Ok(()) - } - } - - #[pymethod] - fn execute( - zelf: PyRef<Self>, - sql: PyStrRef, - parameters: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Cursor>> { - let row_factory = zelf.row_factory.to_owned(); - let cursor = Cursor::new(zelf, row_factory, vm).into_ref(&vm.ctx); - Cursor::execute(cursor, sql, parameters, vm) - } - - #[pymethod] - fn executemany( - zelf: PyRef<Self>, - sql: PyStrRef, - seq_of_params: ArgIterable, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Cursor>> { - let row_factory = zelf.row_factory.to_owned(); - let cursor = Cursor::new(zelf, row_factory, vm).into_ref(&vm.ctx); - Cursor::executemany(cursor, sql, seq_of_params, vm) - } - - #[pymethod] - fn executescript( - zelf: PyRef<Self>, - script: PyStrRef, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Cursor>> { - let row_factory = zelf.row_factory.to_owned(); - Cursor::executescript( - Cursor::new(zelf, row_factory, vm).into_ref(&vm.ctx), - script, - vm, - ) - } - - #[pymethod] - fn backup(zelf: &Py<Self>, args: BackupArgs, vm: &VirtualMachine) -> PyResult<()> { - let BackupArgs { - target, - pages, - progress, - name, - sleep, - } = args; - if zelf.is(&target) { - return Err( - vm.new_value_error("target cannot be the same connection instance".to_owned()) - ); - } - - let pages = if pages == 0 { -1 } else { pages }; - - let name_cstring; - let name_ptr = if let Some(name) = &name { - name_cstring = name.to_cstring(vm)?; - name_cstring.as_ptr() - } else { - b"main\0".as_ptr().cast() - }; - - let sleep_ms = (sleep * 1000.0) as c_int; - - let db = zelf.db_lock(vm)?; - let target_db = target.db_lock(vm)?; - - let handle = unsafe { - sqlite3_backup_init(target_db.db, b"main\0".as_ptr().cast(), db.db, name_ptr) - }; - - if handle.is_null() { - return Err(target_db.error_extended(vm)); - } - - drop(db); - drop(target_db); - - loop { - let ret = unsafe { sqlite3_backup_step(handle, pages) }; - - if let Some(progress) = &progress { - let remaining = unsafe { sqlite3_backup_remaining(handle) }; - let pagecount = unsafe { sqlite3_backup_pagecount(handle) }; - if let Err(err) = progress.invoke((ret, remaining, pagecount), vm) { - unsafe { sqlite3_backup_finish(handle) }; - return Err(err); - } - } - - if ret == SQLITE_BUSY || ret == SQLITE_LOCKED { - unsafe { sqlite3_sleep(sleep_ms) }; - } else if ret != SQLITE_OK { - break; - } - } - - let ret = unsafe { sqlite3_backup_finish(handle) }; - if ret == SQLITE_OK { - Ok(()) - } else { - Err(target.db_lock(vm)?.error_extended(vm)) - } - } - - #[pymethod] - fn create_function(&self, args: CreateFunctionArgs, vm: &VirtualMachine) -> PyResult<()> { - let name = args.name.to_cstring(vm)?; - let flags = if args.deterministic { - SQLITE_UTF8 | SQLITE_DETERMINISTIC - } else { - SQLITE_UTF8 - }; - let db = self.db_lock(vm)?; - let Some(data) = CallbackData::new(args.func, vm) else { - return db.create_function( - name.as_ptr(), - args.narg, - flags, - null_mut(), - None, - None, - None, - None, - vm, - ); - }; - - db.create_function( - name.as_ptr(), - args.narg, - flags, - Box::into_raw(Box::new(data)).cast(), - Some(CallbackData::func_callback), - None, - None, - Some(CallbackData::destructor), - vm, - ) - } - - #[pymethod] - fn create_aggregate(&self, args: CreateAggregateArgs, vm: &VirtualMachine) -> PyResult<()> { - let name = args.name.to_cstring(vm)?; - let db = self.db_lock(vm)?; - let Some(data) = CallbackData::new(args.aggregate_class, vm) else { - return db.create_function( - name.as_ptr(), - args.narg, - SQLITE_UTF8, - null_mut(), - None, - None, - None, - None, - vm, - ); - }; - - db.create_function( - name.as_ptr(), - args.narg, - SQLITE_UTF8, - Box::into_raw(Box::new(data)).cast(), - None, - Some(CallbackData::step_callback), - Some(CallbackData::finalize_callback), - Some(CallbackData::destructor), - vm, - ) - } - - #[pymethod] - fn create_collation( - &self, - name: PyStrRef, - callable: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let name = name.to_cstring(vm)?; - let db = self.db_lock(vm)?; - let Some(data) = CallbackData::new(callable.clone(), vm) else { - unsafe { - sqlite3_create_collation_v2( - db.db, - name.as_ptr(), - SQLITE_UTF8, - null_mut(), - None, - None, - ); - } - return Ok(()); - }; - let data = Box::into_raw(Box::new(data)); - - if !callable.is_callable() { - return Err(vm.new_type_error("parameter must be callable".to_owned())); - } - - let ret = unsafe { - sqlite3_create_collation_v2( - db.db, - name.as_ptr(), - SQLITE_UTF8, - data.cast(), - Some(CallbackData::collation_callback), - Some(CallbackData::destructor), - ) - }; - - // TODO: replace with Result.inspect_err when stable - if let Err(exc) = db.check(ret, vm) { - // create_collation do not call destructor if error occur - let _ = unsafe { Box::from_raw(data) }; - Err(exc) - } else { - Ok(()) - } - } - - #[pymethod] - fn create_window_function( - &self, - name: PyStrRef, - narg: c_int, - aggregate_class: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let name = name.to_cstring(vm)?; - let db = self.db_lock(vm)?; - let Some(data) = CallbackData::new(aggregate_class, vm) else { - unsafe { - sqlite3_create_window_function( - db.db, - name.as_ptr(), - narg, - SQLITE_UTF8, - null_mut(), - None, - None, - None, - None, - None, - ) - }; - return Ok(()); - }; - - let ret = unsafe { - sqlite3_create_window_function( - db.db, - name.as_ptr(), - narg, - SQLITE_UTF8, - Box::into_raw(Box::new(data)).cast(), - Some(CallbackData::step_callback), - Some(CallbackData::finalize_callback), - Some(CallbackData::value_callback), - Some(CallbackData::inverse_callback), - Some(CallbackData::destructor), - ) - }; - db.check(ret, vm) - .map_err(|_| new_programming_error(vm, "Error creating window function".to_owned())) - } - - #[pymethod] - fn set_authorizer(&self, callable: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let db = self.db_lock(vm)?; - let Some(data) = CallbackData::new(callable, vm) else { - unsafe { sqlite3_set_authorizer(db.db, None, null_mut()) }; - return Ok(()); - }; - - let ret = unsafe { - sqlite3_set_authorizer( - db.db, - Some(CallbackData::authorizer_callback), - Box::into_raw(Box::new(data)).cast(), - ) - }; - db.check(ret, vm).map_err(|_| { - new_operational_error(vm, "Error setting authorizer callback".to_owned()) - }) - } - - #[pymethod] - fn set_trace_callback(&self, callable: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let db = self.db_lock(vm)?; - let Some(data) = CallbackData::new(callable, vm) else { - unsafe { sqlite3_trace_v2(db.db, SQLITE_TRACE_STMT as u32, None, null_mut()) }; - return Ok(()); - }; - - let ret = unsafe { - sqlite3_trace_v2( - db.db, - SQLITE_TRACE_STMT as u32, - Some(CallbackData::trace_callback), - Box::into_raw(Box::new(data)).cast(), - ) - }; - - db.check(ret, vm) - } - - #[pymethod] - fn set_progress_handler( - &self, - callable: PyObjectRef, - n: c_int, - vm: &VirtualMachine, - ) -> PyResult<()> { - let db = self.db_lock(vm)?; - let Some(data) = CallbackData::new(callable, vm) else { - unsafe { sqlite3_progress_handler(db.db, n, None, null_mut()) }; - return Ok(()); - }; - - unsafe { - sqlite3_progress_handler( - db.db, - n, - Some(CallbackData::progress_callback), - Box::into_raw(Box::new(data)).cast(), - ) - }; - - Ok(()) - } - - #[pymethod] - fn iterdump(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - let module = vm.import("sqlite3.dump", None, 0)?; - let func = module.get_attr("_iterdump", vm)?; - func.call((zelf,), vm) - } - - #[pymethod] - fn interrupt(&self, vm: &VirtualMachine) -> PyResult<()> { - // DO NOT check thread safety - self._db_lock(vm).map(|x| x.interrupt()) - } - - #[pymethod] - fn getlimit(&self, category: c_int, vm: &VirtualMachine) -> PyResult<c_int> { - self.db_lock(vm)?.limit(category, -1, vm) - } - - #[pymethod] - fn setlimit(&self, category: c_int, limit: c_int, vm: &VirtualMachine) -> PyResult<c_int> { - self.db_lock(vm)?.limit(category, limit, vm) - } - - #[pymethod(magic)] - fn enter(zelf: PyRef<Self>) -> PyRef<Self> { - zelf - } - - #[pymethod(magic)] - fn exit( - &self, - cls: PyObjectRef, - exc: PyObjectRef, - tb: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if vm.is_none(&cls) && vm.is_none(&exc) && vm.is_none(&tb) { - self.commit(vm) - } else { - self.rollback(vm) - } - } - - #[pygetset] - fn isolation_level(&self) -> Option<PyStrRef> { - self.isolation_level.deref().map(|x| x.to_owned()) - } - #[pygetset(setter)] - fn set_isolation_level(&self, val: Option<PyStrRef>, vm: &VirtualMachine) -> PyResult<()> { - if let Some(val) = &val { - begin_statement_ptr_from_isolation_level(val, vm)?; - } - let _ = unsafe { self.isolation_level.swap(val) }; - Ok(()) - } - - #[pygetset] - fn text_factory(&self) -> PyObjectRef { - self.text_factory.to_owned() - } - #[pygetset(setter)] - fn set_text_factory(&self, val: PyObjectRef) { - let _ = unsafe { self.text_factory.swap(val) }; - } - - #[pygetset] - fn row_factory(&self) -> Option<PyObjectRef> { - self.row_factory.to_owned() - } - #[pygetset(setter)] - fn set_row_factory(&self, val: Option<PyObjectRef>) { - let _ = unsafe { self.row_factory.swap(val) }; - } - - fn check_thread(&self, vm: &VirtualMachine) -> PyResult<()> { - if self.check_same_thread && (std::thread::current().id() != self.thread_ident) { - Err(new_programming_error( - vm, - "SQLite objects created in a thread can only be used in that same thread." - .to_owned(), - )) - } else { - Ok(()) - } - } - - #[pygetset] - fn in_transaction(&self, vm: &VirtualMachine) -> PyResult<bool> { - self._db_lock(vm).map(|x| !x.is_autocommit()) - } - - #[pygetset] - fn total_changes(&self, vm: &VirtualMachine) -> PyResult<c_int> { - self._db_lock(vm).map(|x| x.total_changes()) - } - } - - #[pyattr] - #[pyclass(name, traverse)] - #[derive(Debug, PyPayload)] - struct Cursor { - connection: PyRef<Connection>, - #[pytraverse(skip)] - arraysize: PyAtomic<c_int>, - #[pytraverse(skip)] - row_factory: PyAtomicRef<Option<PyObject>>, - inner: PyMutex<Option<CursorInner>>, - } - - #[derive(Debug, Traverse)] - struct CursorInner { - description: Option<PyTupleRef>, - row_cast_map: Vec<Option<PyObjectRef>>, - #[pytraverse(skip)] - lastrowid: i64, - #[pytraverse(skip)] - rowcount: i64, - statement: Option<PyRef<Statement>>, - } - - #[pyclass(with(Constructor, IterNext, Iterable), flags(BASETYPE))] - impl Cursor { - fn new( - connection: PyRef<Connection>, - row_factory: Option<PyObjectRef>, - _vm: &VirtualMachine, - ) -> Self { - Self { - connection, - arraysize: Radium::new(1), - row_factory: PyAtomicRef::from(row_factory), - inner: PyMutex::from(Some(CursorInner { - description: None, - row_cast_map: vec![], - lastrowid: -1, - rowcount: -1, - statement: None, - })), - } - } - - fn inner(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<CursorInner>> { - let guard = self.inner.lock(); - if guard.is_some() { - Ok(PyMutexGuard::map(guard, |x| unsafe { - x.as_mut().unwrap_unchecked() - })) - } else { - Err(new_programming_error( - vm, - "Cannot operate on a closed cursor.".to_owned(), - )) - } - } - - #[pymethod] - fn execute( - zelf: PyRef<Self>, - sql: PyStrRef, - parameters: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - let mut inner = zelf.inner(vm)?; - - if let Some(stmt) = inner.statement.take() { - stmt.lock().reset(); - } - - let Some(stmt) = Statement::new(&zelf.connection, &sql, vm)? else { - drop(inner); - return Ok(zelf); - }; - let stmt = stmt.into_ref(&vm.ctx); - - inner.rowcount = if stmt.is_dml { 0 } else { -1 }; - - let db = zelf.connection.db_lock(vm)?; - - if stmt.is_dml && db.is_autocommit() { - db.begin_transaction( - zelf.connection - .isolation_level - .deref() - .map(|x| x.to_owned()), - vm, - )?; - } - - let st = stmt.lock(); - if let OptionalArg::Present(parameters) = parameters { - st.bind_parameters(&parameters, vm)?; - } - - let ret = st.step(); - - if ret != SQLITE_DONE && ret != SQLITE_ROW { - if let Some(exc) = unsafe { user_function_exception().swap(None) } { - return Err(exc); - } - return Err(db.error_extended(vm)); - } - - inner.row_cast_map = zelf.build_row_cast_map(&st, vm)?; - - inner.description = st.columns_description(vm)?; - - if ret == SQLITE_ROW { - drop(st); - inner.statement = Some(stmt); - } else { - st.reset(); - drop(st); - if stmt.is_dml { - inner.rowcount += db.changes() as i64; - } - } - - inner.lastrowid = db.lastrowid(); - - drop(inner); - drop(db); - Ok(zelf) - } - - #[pymethod] - fn executemany( - zelf: PyRef<Self>, - sql: PyStrRef, - seq_of_params: ArgIterable, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - let mut inner = zelf.inner(vm)?; - - if let Some(stmt) = inner.statement.take() { - stmt.lock().reset(); - } - - let Some(stmt) = Statement::new(&zelf.connection, &sql, vm)? else { - drop(inner); - return Ok(zelf); - }; - let stmt = stmt.into_ref(&vm.ctx); - - let st = stmt.lock(); - - if st.readonly() { - return Err(new_programming_error( - vm, - "executemany() can only execute DML statements.".to_owned(), - )); - } - - inner.description = st.columns_description(vm)?; - - inner.rowcount = if stmt.is_dml { 0 } else { -1 }; - - let db = zelf.connection.db_lock(vm)?; - - if stmt.is_dml && db.is_autocommit() { - db.begin_transaction( - zelf.connection - .isolation_level - .deref() - .map(|x| x.to_owned()), - vm, - )?; - } - - let iter = seq_of_params.iter(vm)?; - for params in iter { - let params = params?; - st.bind_parameters(&params, vm)?; - - if !st.step_row_else_done(vm)? { - if stmt.is_dml { - inner.rowcount += db.changes() as i64; - } - st.reset(); - } - - // if let Some(exc) = unsafe { user_function_exception().swap(None) } { - // return Err(exc); - // } - } - - if st.busy() { - drop(st); - inner.statement = Some(stmt); - } - - drop(inner); - drop(db); - Ok(zelf) - } - - #[pymethod] - fn executescript( - zelf: PyRef<Self>, - script: PyStrRef, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - let db = zelf.connection.db_lock(vm)?; - - db.sql_limit(script.byte_len(), vm)?; - - db.implicit_commit(vm)?; - - let script = script.to_cstring(vm)?; - let mut ptr = script.as_ptr(); - - while let Some(st) = db.prepare(ptr, &mut ptr, vm)? { - while st.step_row_else_done(vm)? {} - } - - drop(db); - Ok(zelf) - } - - #[pymethod] - fn fetchone(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult { - Self::next(zelf, vm).map(|x| match x { - PyIterReturn::Return(row) => row, - PyIterReturn::StopIteration(_) => vm.ctx.none(), - }) - } - - #[pymethod] - fn fetchmany( - zelf: &Py<Self>, - max_rows: OptionalArg<c_int>, - vm: &VirtualMachine, - ) -> PyResult<Vec<PyObjectRef>> { - let max_rows = max_rows.unwrap_or_else(|| zelf.arraysize.load(Ordering::Relaxed)); - let mut list = vec![]; - while let PyIterReturn::Return(row) = Self::next(zelf, vm)? { - list.push(row); - if list.len() as c_int >= max_rows { - break; - } - } - Ok(list) - } - - #[pymethod] - fn fetchall(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let mut list = vec![]; - while let PyIterReturn::Return(row) = Self::next(zelf, vm)? { - list.push(row); - } - Ok(list) - } - - #[pymethod] - fn close(&self) { - if let Some(inner) = self.inner.lock().take() { - if let Some(stmt) = inner.statement { - stmt.lock().reset(); - } - } - } - - #[pymethod] - fn setinputsizes(&self, _sizes: PyObjectRef) {} - #[pymethod] - fn setoutputsize(&self, _size: PyObjectRef, _column: OptionalArg<PyObjectRef>) {} - - #[pygetset] - fn connection(&self) -> PyRef<Connection> { - self.connection.clone() - } - - #[pygetset] - fn lastrowid(&self, vm: &VirtualMachine) -> PyResult<i64> { - self.inner(vm).map(|x| x.lastrowid) - } - - #[pygetset] - fn rowcount(&self, vm: &VirtualMachine) -> PyResult<i64> { - self.inner(vm).map(|x| x.rowcount) - } - - #[pygetset] - fn description(&self, vm: &VirtualMachine) -> PyResult<Option<PyTupleRef>> { - self.inner(vm).map(|x| x.description.clone()) - } - - #[pygetset] - fn arraysize(&self) -> c_int { - self.arraysize.load(Ordering::Relaxed) - } - #[pygetset(setter)] - fn set_arraysize(&self, val: c_int) { - self.arraysize.store(val, Ordering::Relaxed); - } - - fn build_row_cast_map( - &self, - st: &SqliteStatementRaw, - vm: &VirtualMachine, - ) -> PyResult<Vec<Option<PyObjectRef>>> { - if self.connection.detect_types == 0 { - return Ok(vec![]); - } - - let mut cast_map = vec![]; - let num_cols = st.column_count(); - - for i in 0..num_cols { - if self.connection.detect_types & PARSE_COLNAMES != 0 { - let col_name = st.column_name(i); - let col_name = ptr_to_str(col_name, vm)?; - let col_name = col_name - .chars() - .skip_while(|&x| x != '[') - .skip(1) - .take_while(|&x| x != ']') - .flat_map(|x| x.to_uppercase()) - .collect::<String>(); - if let Some(converter) = converters().get_item_opt(&col_name, vm)? { - cast_map.push(Some(converter.clone())); - continue; - } - } - if self.connection.detect_types & PARSE_DECLTYPES != 0 { - let decltype = st.column_decltype(i); - let decltype = ptr_to_str(decltype, vm)?; - if let Some(decltype) = decltype.split_terminator(&[' ', '(']).next() { - let decltype = decltype.to_uppercase(); - if let Some(converter) = converters().get_item_opt(&decltype, vm)? { - cast_map.push(Some(converter.clone())); - continue; - } - } - } - cast_map.push(None); - } - - Ok(cast_map) - } - } - - impl Constructor for Cursor { - type Args = (PyRef<Connection>,); - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - Self::new(args.0, None, vm) - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl SelfIter for Cursor {} - impl IterNext for Cursor { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let mut inner = zelf.inner(vm)?; - let Some(stmt) = &inner.statement else { - return Ok(PyIterReturn::StopIteration(None)); - }; - let st = stmt.lock(); - let db = zelf.connection.db_lock(vm)?; - // fetch_one_row - - let num_cols = st.data_count(); - - let mut row = Vec::with_capacity(num_cols as usize); - - for i in 0..num_cols { - let val = if let Some(converter) = - inner.row_cast_map.get(i as usize).cloned().flatten() - { - let blob = st.column_blob(i); - if blob.is_null() { - vm.ctx.none() - } else { - let nbytes = st.column_bytes(i); - let blob = unsafe { - std::slice::from_raw_parts(blob.cast::<u8>(), nbytes as usize) - }; - let blob = vm.ctx.new_bytes(blob.to_vec()); - converter.call((blob,), vm)? - } - } else { - let col_type = st.column_type(i); - match col_type { - SQLITE_NULL => vm.ctx.none(), - SQLITE_INTEGER => vm.ctx.new_int(st.column_int(i)).into(), - SQLITE_FLOAT => vm.ctx.new_float(st.column_double(i)).into(), - SQLITE_TEXT => { - let text = - ptr_to_vec(st.column_text(i), st.column_bytes(i), db.db, vm)?; - - let text_factory = zelf.connection.text_factory.to_owned(); - - if text_factory.is(PyStr::class(&vm.ctx)) { - let text = String::from_utf8(text).map_err(|_| { - new_operational_error(vm, "not valid UTF-8".to_owned()) - })?; - vm.ctx.new_str(text).into() - } else if text_factory.is(PyBytes::class(&vm.ctx)) { - vm.ctx.new_bytes(text).into() - } else if text_factory.is(PyByteArray::class(&vm.ctx)) { - PyByteArray::from(text).into_ref(&vm.ctx).into() - } else { - let bytes = vm.ctx.new_bytes(text); - text_factory.call((bytes,), vm)? - } - } - SQLITE_BLOB => { - let blob = ptr_to_vec( - st.column_blob(i).cast(), - st.column_bytes(i), - db.db, - vm, - )?; - - vm.ctx.new_bytes(blob).into() - } - _ => { - return Err(vm.new_not_implemented_error(format!( - "unknown column type: {col_type}" - ))); - } - } - }; - - row.push(val); - } - - if !st.step_row_else_done(vm)? { - st.reset(); - drop(st); - if stmt.is_dml { - inner.rowcount = db.changes() as i64; - } - inner.statement = None; - } else { - drop(st); - } - - drop(db); - drop(inner); - - let row = vm.ctx.new_tuple(row); - - if let Some(row_factory) = zelf.row_factory.to_owned() { - row_factory - .call((zelf.to_owned(), row), vm) - .map(PyIterReturn::Return) - } else { - Ok(PyIterReturn::Return(row.into())) - } - } - } - - #[pyattr] - #[pyclass(name, traverse)] - #[derive(Debug, PyPayload)] - struct Row { - data: PyTupleRef, - description: PyTupleRef, - } - - #[pyclass( - with(Constructor, Hashable, Comparable, Iterable, AsMapping, AsSequence), - flags(BASETYPE) - )] - impl Row { - #[pymethod] - fn keys(&self, _vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - Ok(self - .description - .iter() - .map(|x| x.payload::<PyTuple>().unwrap().as_slice()[0].clone()) - .collect()) - } - - fn subscript(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - if let Some(i) = needle.payload::<PyInt>() { - let i = i.try_to_primitive::<isize>(vm)?; - self.data.getitem_by_index(vm, i) - } else if let Some(name) = needle.payload::<PyStr>() { - for (obj, i) in self.description.iter().zip(0..) { - let obj = &obj.payload::<PyTuple>().unwrap().as_slice()[0]; - let Some(obj) = obj.payload::<PyStr>() else { - break; - }; - let a_iter = name.as_str().chars().flat_map(|x| x.to_uppercase()); - let b_iter = obj.as_str().chars().flat_map(|x| x.to_uppercase()); - - if a_iter.eq(b_iter) { - return self.data.getitem_by_index(vm, i); - } - } - Err(vm.new_index_error("No item with that key".to_owned())) - } else if let Some(slice) = needle.payload::<PySlice>() { - let list = self.data.getitem_by_slice(vm, slice.to_saturated(vm)?)?; - Ok(vm.ctx.new_tuple(list).into()) - } else { - Err(vm.new_index_error("Index must be int or string".to_owned())) - } - } - } - - impl Constructor for Row { - type Args = (PyRef<Cursor>, PyTupleRef); - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let description = args - .0 - .inner(vm)? - .description - .clone() - .ok_or_else(|| vm.new_value_error("no description in Cursor".to_owned()))?; - - Self { - data: args.1, - description, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl Hashable for Row { - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - Ok(zelf.description.as_object().hash(vm)? | zelf.data.as_object().hash(vm)?) - } - } - - impl Comparable for Row { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - op.eq_only(|| { - if let Some(other) = other.payload::<Self>() { - let eq = vm - .bool_eq(zelf.description.as_object(), other.description.as_object())? - && vm.bool_eq(zelf.data.as_object(), other.data.as_object())?; - Ok(eq.into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } - }) - } - } - - impl Iterable for Row { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Iterable::iter(zelf.data.clone(), vm) - } - } - - impl AsMapping for Row { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: once_cell::sync::Lazy<PyMappingMethods> = - once_cell::sync::Lazy::new(|| PyMappingMethods { - length: atomic_func!(|mapping, _vm| Ok(Row::mapping_downcast(mapping) - .data - .len())), - subscript: atomic_func!(|mapping, needle, vm| { - Row::mapping_downcast(mapping).subscript(needle, vm) - }), - ..PyMappingMethods::NOT_IMPLEMENTED - }); - &AS_MAPPING - } - } - - impl AsSequence for Row { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: once_cell::sync::Lazy<PySequenceMethods> = - once_cell::sync::Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(Row::sequence_downcast(seq).data.len())), - item: atomic_func!(|seq, i, vm| Row::sequence_downcast(seq) - .data - .getitem_by_index(vm, i)), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } - } - - #[pyattr] - #[pyclass(name, traverse)] - #[derive(Debug, PyPayload)] - struct Blob { - connection: PyRef<Connection>, - #[pytraverse(skip)] - inner: PyMutex<Option<BlobInner>>, - } - - #[derive(Debug)] - struct BlobInner { - blob: SqliteBlob, - offset: c_int, - } - - impl Drop for BlobInner { - fn drop(&mut self) { - unsafe { sqlite3_blob_close(self.blob.blob) }; - } - } - - #[pyclass(with(AsMapping))] - impl Blob { - #[pymethod] - fn close(&self) { - self.inner.lock().take(); - } - - #[pymethod] - fn read( - &self, - length: OptionalArg<c_int>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<PyBytes>> { - let mut length = length.unwrap_or(-1); - let mut inner = self.inner(vm)?; - let blob_len = inner.blob.bytes(); - let max_read = blob_len - inner.offset; - - if length < 0 || length > max_read { - length = max_read; - } - - if length == 0 { - Ok(vm.ctx.empty_bytes.clone()) - } else { - let mut buf = Vec::<u8>::with_capacity(length as usize); - let ret = inner - .blob - .read(buf.as_mut_ptr().cast(), length, inner.offset); - self.check(ret, vm)?; - unsafe { buf.set_len(length as usize) }; - inner.offset += length; - Ok(vm.ctx.new_bytes(buf)) - } - } - - #[pymethod] - fn write(&self, data: PyBuffer, vm: &VirtualMachine) -> PyResult<()> { - let mut inner = self.inner(vm)?; - let blob_len = inner.blob.bytes(); - let length = Self::expect_write(blob_len, data.desc.len, inner.offset, vm)?; - - let ret = data.contiguous_or_collect(|buf| { - inner.blob.write(buf.as_ptr().cast(), length, inner.offset) - }); - - self.check(ret, vm)?; - inner.offset += length; - Ok(()) - } - - #[pymethod] - fn tell(&self, vm: &VirtualMachine) -> PyResult<c_int> { - self.inner(vm).map(|x| x.offset) - } - - #[pymethod] - fn seek( - &self, - mut offset: c_int, - origin: OptionalArg<c_int>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let origin = origin.unwrap_or(libc::SEEK_SET); - let mut inner = self.inner(vm)?; - let blob_len = inner.blob.bytes(); - - let overflow_err = - || vm.new_overflow_error("seek offset results in overflow".to_owned()); - - match origin { - libc::SEEK_SET => {} - libc::SEEK_CUR => { - offset = offset.checked_add(inner.offset).ok_or_else(overflow_err)? - } - libc::SEEK_END => offset = offset.checked_add(blob_len).ok_or_else(overflow_err)?, - _ => { - return Err(vm.new_value_error( - "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END".to_owned(), - )) - } - } - - if offset < 0 || offset > blob_len { - Err(vm.new_value_error("offset out of blob range".to_owned())) - } else { - inner.offset = offset; - Ok(()) - } - } - - #[pymethod(magic)] - fn enter(zelf: PyRef<Self>) -> PyRef<Self> { - zelf - } - - #[pymethod(magic)] - fn exit(&self, _args: FuncArgs) { - self.close() - } - - fn inner(&self, vm: &VirtualMachine) -> PyResult<PyMappedMutexGuard<BlobInner>> { - let guard = self.inner.lock(); - if guard.is_some() { - Ok(PyMutexGuard::map(guard, |x| unsafe { - x.as_mut().unwrap_unchecked() - })) - } else { - Err(new_programming_error( - vm, - "Cannot operate on a closed blob.".to_owned(), - )) - } - } - - fn wrapped_index(index: PyIntRef, length: c_int, vm: &VirtualMachine) -> PyResult<c_int> { - let mut index = index.try_to_primitive::<c_int>(vm)?; - if index < 0 { - index += length; - } - if index < 0 || index >= length { - Err(vm.new_index_error("Blob index out of range".to_owned())) - } else { - Ok(index) - } - } - - fn expect_write( - blob_len: c_int, - length: usize, - offset: c_int, - vm: &VirtualMachine, - ) -> PyResult<c_int> { - let max_write = blob_len - offset; - if length <= max_write as usize { - Ok(length as c_int) - } else { - Err(vm.new_value_error("data longer than blob length".to_owned())) - } - } - - fn subscript(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - let inner = self.inner(vm)?; - if let Some(index) = needle.try_index_opt(vm) { - let blob_len = inner.blob.bytes(); - let index = Self::wrapped_index(index?, blob_len, vm)?; - let mut byte: u8 = 0; - let ret = inner.blob.read_single(&mut byte, index); - self.check(ret, vm).map(|_| vm.ctx.new_int(byte).into()) - } else if let Some(slice) = needle.payload::<PySlice>() { - let blob_len = inner.blob.bytes(); - let slice = slice.to_saturated(vm)?; - let (range, step, length) = slice.adjust_indices(blob_len as usize); - let mut buf = Vec::<u8>::with_capacity(length); - - if step == 1 { - let ret = inner.blob.read( - buf.as_mut_ptr().cast(), - length as c_int, - range.start as c_int, - ); - self.check(ret, vm)?; - unsafe { buf.set_len(length) }; - } else { - let iter = SaturatedSliceIter::from_adjust_indices(range, step, length); - let mut byte: u8 = 0; - for index in iter { - let ret = inner.blob.read_single(&mut byte, index as c_int); - self.check(ret, vm)?; - buf.push(byte); - } - } - Ok(vm.ctx.new_bytes(buf).into()) - } else { - Err(vm.new_type_error("Blob indices must be integers".to_owned())) - } - } - - fn ass_subscript( - &self, - needle: &PyObject, - value: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let Some(value) = value else { - return Err(vm.new_type_error("Blob doesn't support deletion".to_owned())); - }; - let inner = self.inner(vm)?; - - if let Some(index) = needle.try_index_opt(vm) { - let Some(value) = value.payload::<PyInt>() else { - return Err(vm.new_type_error(format!( - "'{}' object cannot be interpreted as an integer", - value.class() - ))); - }; - let value = value.try_to_primitive::<u8>(vm)?; - let blob_len = inner.blob.bytes(); - let index = Self::wrapped_index(index?, blob_len, vm)?; - Self::expect_write(blob_len, 1, index, vm)?; - let ret = inner.blob.write_single(value, index); - self.check(ret, vm) - } else if let Some(_slice) = needle.payload::<PySlice>() { - Err(vm.new_not_implemented_error( - "Blob slice assignment is not implemented".to_owned(), - )) - // let blob_len = inner.blob.bytes(); - // let slice = slice.to_saturated(vm)?; - // let (range, step, length) = slice.adjust_indices(blob_len as usize); - } else { - Err(vm.new_type_error("Blob indices must be integers".to_owned())) - } - } - - fn check(&self, ret: c_int, vm: &VirtualMachine) -> PyResult<()> { - if ret == SQLITE_OK { - Ok(()) - } else { - Err(self.connection.db_lock(vm)?.error_extended(vm)) - } - } - } - - impl AsMapping for Blob { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: PyMappingMethods = PyMappingMethods { - length: atomic_func!(|mapping, vm| Blob::mapping_downcast(mapping) - .inner(vm) - .map(|x| x.blob.bytes() as usize)), - subscript: atomic_func!(|mapping, needle, vm| { - Blob::mapping_downcast(mapping).subscript(needle, vm) - }), - ass_subscript: atomic_func!(|mapping, needle, value, vm| { - Blob::mapping_downcast(mapping).ass_subscript(needle, value, vm) - }), - }; - &AS_MAPPING - } - } - - #[pyattr] - #[pyclass(name)] - #[derive(Debug, PyPayload)] - struct PrepareProtocol {} - - #[pyclass()] - impl PrepareProtocol {} - - #[pyattr] - #[pyclass(name)] - #[derive(PyPayload)] - struct Statement { - st: PyMutex<SqliteStatement>, - pub is_dml: bool, - } - - impl Debug for Statement { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - write!( - f, - "{} Statement", - if self.is_dml { "DML" } else { "Non-DML" } - ) - } - } - - #[pyclass()] - impl Statement { - fn new( - connection: &Connection, - sql: &PyStr, - vm: &VirtualMachine, - ) -> PyResult<Option<Self>> { - let sql_cstr = sql.to_cstring(vm)?; - let sql_len = sql.byte_len() + 1; - - let db = connection.db_lock(vm)?; - - db.sql_limit(sql_len, vm)?; - - let mut tail = null(); - let st = db.prepare(sql_cstr.as_ptr(), &mut tail, vm)?; - - let Some(st) = st else { - return Ok(None); - }; - - let tail = unsafe { CStr::from_ptr(tail) }; - let tail = tail.to_bytes(); - if lstrip_sql(tail).is_some() { - return Err(new_programming_error( - vm, - "You can only execute one statement at a time.".to_owned(), - )); - } - - let is_dml = if let Some(head) = lstrip_sql(sql_cstr.as_bytes()) { - head.len() >= 6 - && (head[..6].eq_ignore_ascii_case(b"insert") - || head[..6].eq_ignore_ascii_case(b"update") - || head[..6].eq_ignore_ascii_case(b"delete") - || (head.len() >= 7 && head[..7].eq_ignore_ascii_case(b"replace"))) - } else { - false - }; - - Ok(Some(Self { - st: PyMutex::from(st), - is_dml, - })) - } - - fn lock(&self) -> PyMutexGuard<SqliteStatement> { - self.st.lock() - } - } - - struct Sqlite { - raw: SqliteRaw, - } - - impl From<SqliteRaw> for Sqlite { - fn from(raw: SqliteRaw) -> Self { - Self { raw } - } - } - - impl Drop for Sqlite { - fn drop(&mut self) { - unsafe { sqlite3_close_v2(self.raw.db) }; - } - } - - impl Deref for Sqlite { - type Target = SqliteRaw; - - fn deref(&self) -> &Self::Target { - &self.raw - } - } - - #[derive(Copy, Clone)] - struct SqliteRaw { - db: *mut sqlite3, - } - - cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - unsafe impl Send for SqliteStatement {} - // unsafe impl Sync for SqliteStatement {} - unsafe impl Send for Sqlite {} - // unsafe impl Sync for Sqlite {} - unsafe impl Send for SqliteBlob {} - } - } - - impl From<SqliteStatementRaw> for SqliteRaw { - fn from(stmt: SqliteStatementRaw) -> Self { - unsafe { - Self { - db: sqlite3_db_handle(stmt.st), - } - } - } - } - - impl SqliteRaw { - fn check(self, ret: c_int, vm: &VirtualMachine) -> PyResult<()> { - if ret == SQLITE_OK { - Ok(()) - } else { - Err(self.error_extended(vm)) - } - } - - fn error_extended(self, vm: &VirtualMachine) -> PyBaseExceptionRef { - let errcode = unsafe { sqlite3_errcode(self.db) }; - let typ = exception_type_from_errcode(errcode, vm); - let extended_errcode = unsafe { sqlite3_extended_errcode(self.db) }; - let errmsg = unsafe { sqlite3_errmsg(self.db) }; - let errmsg = unsafe { CStr::from_ptr(errmsg) }; - let errmsg = errmsg.to_str().unwrap().to_owned(); - - raise_exception(typ.to_owned(), extended_errcode, errmsg, vm) - } - - fn open(path: *const libc::c_char, uri: bool, vm: &VirtualMachine) -> PyResult<Self> { - let mut db = null_mut(); - let ret = unsafe { - sqlite3_open_v2( - path, - addr_of_mut!(db), - SQLITE_OPEN_READWRITE - | SQLITE_OPEN_CREATE - | if uri { SQLITE_OPEN_URI } else { 0 }, - null(), - ) - }; - let zelf = Self { db }; - zelf.check(ret, vm).map(|_| zelf) - } - - fn _exec(self, sql: &[u8], vm: &VirtualMachine) -> PyResult<()> { - let ret = - unsafe { sqlite3_exec(self.db, sql.as_ptr().cast(), None, null_mut(), null_mut()) }; - self.check(ret, vm) - } - - fn prepare( - self, - sql: *const libc::c_char, - tail: *mut *const libc::c_char, - vm: &VirtualMachine, - ) -> PyResult<Option<SqliteStatement>> { - let mut st = null_mut(); - let ret = unsafe { sqlite3_prepare_v2(self.db, sql, -1, &mut st, tail) }; - self.check(ret, vm)?; - if st.is_null() { - Ok(None) - } else { - Ok(Some(SqliteStatement::from(SqliteStatementRaw::from(st)))) - } - } - - fn limit(self, category: c_int, limit: c_int, vm: &VirtualMachine) -> PyResult<c_int> { - let old_limit = unsafe { sqlite3_limit(self.db, category, limit) }; - if old_limit >= 0 { - Ok(old_limit) - } else { - Err(new_programming_error( - vm, - "'category' is out of bounds".to_owned(), - )) - } - } - - fn sql_limit(self, len: usize, vm: &VirtualMachine) -> PyResult<()> { - if len <= unsafe { sqlite3_limit(self.db, SQLITE_LIMIT_SQL_LENGTH, -1) } as usize { - Ok(()) - } else { - Err(new_data_error(vm, "query string is too large".to_owned())) - } - } - - fn is_autocommit(self) -> bool { - unsafe { sqlite3_get_autocommit(self.db) != 0 } - } - - fn changes(self) -> c_int { - unsafe { sqlite3_changes(self.db) } - } - - fn total_changes(self) -> c_int { - unsafe { sqlite3_total_changes(self.db) } - } - - fn lastrowid(self) -> c_longlong { - unsafe { sqlite3_last_insert_rowid(self.db) } - } - - fn implicit_commit(self, vm: &VirtualMachine) -> PyResult<()> { - if self.is_autocommit() { - Ok(()) - } else { - self._exec(b"COMMIT\0", vm) - } - } - - fn begin_transaction( - self, - isolation_level: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let Some(isolation_level) = isolation_level else { - return Ok(()); - }; - let mut s = Vec::with_capacity(16); - s.extend(b"BEGIN "); - s.extend(isolation_level.as_str().bytes()); - s.push(b'\0'); - self._exec(&s, vm) - } - - fn interrupt(self) { - unsafe { sqlite3_interrupt(self.db) } - } - - fn busy_timeout(self, timeout: i32) { - unsafe { sqlite3_busy_timeout(self.db, timeout) }; - } - - #[allow(clippy::too_many_arguments)] - fn create_function( - self, - name: *const libc::c_char, - narg: c_int, - flags: c_int, - data: *mut c_void, - func: Option< - unsafe extern "C" fn( - arg1: *mut sqlite3_context, - arg2: c_int, - arg3: *mut *mut sqlite3_value, - ), - >, - step: Option< - unsafe extern "C" fn( - arg1: *mut sqlite3_context, - arg2: c_int, - arg3: *mut *mut sqlite3_value, - ), - >, - finalize: Option<unsafe extern "C" fn(arg1: *mut sqlite3_context)>, - destroy: Option<unsafe extern "C" fn(arg1: *mut c_void)>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let ret = unsafe { - sqlite3_create_function_v2( - self.db, name, narg, flags, data, func, step, finalize, destroy, - ) - }; - self.check(ret, vm) - .map_err(|_| new_operational_error(vm, "Error creating function".to_owned())) - } - } - - struct SqliteStatement { - raw: SqliteStatementRaw, - } - - impl From<SqliteStatementRaw> for SqliteStatement { - fn from(raw: SqliteStatementRaw) -> Self { - Self { raw } - } - } - - impl Drop for SqliteStatement { - fn drop(&mut self) { - unsafe { - sqlite3_finalize(self.raw.st); - } - } - } - - impl Deref for SqliteStatement { - type Target = SqliteStatementRaw; - - fn deref(&self) -> &Self::Target { - &self.raw - } - } - - #[derive(Copy, Clone)] - struct SqliteStatementRaw { - st: *mut sqlite3_stmt, - } - - impl From<*mut sqlite3_stmt> for SqliteStatementRaw { - fn from(st: *mut sqlite3_stmt) -> Self { - SqliteStatementRaw { st } - } - } - - impl SqliteStatementRaw { - fn step(self) -> c_int { - unsafe { sqlite3_step(self.st) } - } - - fn step_row_else_done(self, vm: &VirtualMachine) -> PyResult<bool> { - let ret = self.step(); - - if let Some(exc) = unsafe { user_function_exception().swap(None) } { - Err(exc) - } else if ret == SQLITE_ROW { - Ok(true) - } else if ret == SQLITE_DONE { - Ok(false) - } else { - Err(SqliteRaw::from(self).error_extended(vm)) - } - } - - fn reset(self) { - unsafe { sqlite3_reset(self.st) }; - } - - fn data_count(self) -> c_int { - unsafe { sqlite3_data_count(self.st) } - } - - fn bind_parameter( - self, - pos: c_int, - parameter: &PyObject, - vm: &VirtualMachine, - ) -> PyResult<()> { - let adapted; - let obj = if need_adapt(parameter, vm) { - adapted = _adapt( - parameter, - PrepareProtocol::class(&vm.ctx).to_owned(), - |x| Ok(x.to_owned()), - vm, - )?; - &adapted - } else { - parameter - }; - - let ret = if vm.is_none(obj) { - unsafe { sqlite3_bind_null(self.st, pos) } - } else if let Some(val) = obj.payload::<PyInt>() { - let val = val.try_to_primitive::<i64>(vm)?; - unsafe { sqlite3_bind_int64(self.st, pos, val) } - } else if let Some(val) = obj.payload::<PyFloat>() { - let val = val.to_f64(); - unsafe { sqlite3_bind_double(self.st, pos, val) } - } else if let Some(val) = obj.payload::<PyStr>() { - let (ptr, len) = str_to_ptr_len(val, vm)?; - unsafe { sqlite3_bind_text(self.st, pos, ptr, len, SQLITE_TRANSIENT()) } - } else if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, obj) { - let (ptr, len) = buffer_to_ptr_len(&buffer, vm)?; - unsafe { sqlite3_bind_blob(self.st, pos, ptr, len, SQLITE_TRANSIENT()) } - } else { - return Err(new_programming_error( - vm, - format!( - "Error binding parameter {}: type '{}' is not supported", - pos, - obj.class() - ), - )); - }; - - if ret == SQLITE_OK { - Ok(()) - } else { - let db = SqliteRaw::from(self); - db.check(ret, vm) - } - } - - fn bind_parameters(self, parameters: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if let Some(dict) = parameters.downcast_ref::<PyDict>() { - self.bind_parameters_name(dict, vm) - } else if let Ok(seq) = PySequence::try_protocol(parameters, vm) { - self.bind_parameters_sequence(seq, vm) - } else { - Err(new_programming_error( - vm, - "parameters are of unsupported type".to_owned(), - )) - } - } - - fn bind_parameters_name(self, dict: &Py<PyDict>, vm: &VirtualMachine) -> PyResult<()> { - let num_needed = unsafe { sqlite3_bind_parameter_count(self.st) }; - - for i in 1..=num_needed { - let name = unsafe { sqlite3_bind_parameter_name(self.st, i) }; - if name.is_null() { - return Err(new_programming_error(vm, "Binding {} has no name, but you supplied a dictionary (which has only names).".to_owned())); - } - let name = unsafe { name.add(1) }; - let name = ptr_to_str(name, vm)?; - - let val = dict.get_item(name, vm)?; - - self.bind_parameter(i, &val, vm)?; - } - Ok(()) - } - - fn bind_parameters_sequence(self, seq: PySequence, vm: &VirtualMachine) -> PyResult<()> { - let num_needed = unsafe { sqlite3_bind_parameter_count(self.st) }; - if seq.length(vm)? != num_needed as usize { - return Err(new_programming_error( - vm, - "Incorrect number of binding supplied".to_owned(), - )); - } - - for i in 1..=num_needed { - let val = seq.get_item(i as isize - 1, vm)?; - self.bind_parameter(i, &val, vm)?; - } - Ok(()) - } - - fn column_count(self) -> c_int { - unsafe { sqlite3_column_count(self.st) } - } - - fn column_type(self, pos: c_int) -> c_int { - unsafe { sqlite3_column_type(self.st, pos) } - } - - fn column_int(self, pos: c_int) -> i64 { - unsafe { sqlite3_column_int64(self.st, pos) } - } - - fn column_double(self, pos: c_int) -> f64 { - unsafe { sqlite3_column_double(self.st, pos) } - } - - fn column_blob(self, pos: c_int) -> *const c_void { - unsafe { sqlite3_column_blob(self.st, pos) } - } - - fn column_text(self, pos: c_int) -> *const u8 { - unsafe { sqlite3_column_text(self.st, pos) } - } - - fn column_decltype(self, pos: c_int) -> *const libc::c_char { - unsafe { sqlite3_column_decltype(self.st, pos) } - } - - fn column_bytes(self, pos: c_int) -> c_int { - unsafe { sqlite3_column_bytes(self.st, pos) } - } - - fn column_name(self, pos: c_int) -> *const libc::c_char { - unsafe { sqlite3_column_name(self.st, pos) } - } - - fn columns_name(self, vm: &VirtualMachine) -> PyResult<Vec<PyStrRef>> { - let count = self.column_count(); - (0..count) - .map(|i| { - let name = self.column_name(i); - ptr_to_str(name, vm).map(|x| vm.ctx.new_str(x)) - }) - .collect() - } - - fn columns_description(self, vm: &VirtualMachine) -> PyResult<Option<PyTupleRef>> { - if self.column_count() == 0 { - return Ok(None); - } - let columns = self - .columns_name(vm)? - .into_iter() - .map(|s| { - vm.ctx - .new_tuple(vec![ - s.into(), - vm.ctx.none(), - vm.ctx.none(), - vm.ctx.none(), - vm.ctx.none(), - vm.ctx.none(), - vm.ctx.none(), - ]) - .into() - }) - .collect(); - Ok(Some(vm.ctx.new_tuple(columns))) - } - - fn busy(self) -> bool { - unsafe { sqlite3_stmt_busy(self.st) != 0 } - } - - fn readonly(self) -> bool { - unsafe { sqlite3_stmt_readonly(self.st) != 0 } - } - } - - #[derive(Debug, Copy, Clone)] - struct SqliteBlob { - blob: *mut sqlite3_blob, - } - - impl SqliteBlob { - fn bytes(self) -> c_int { - unsafe { sqlite3_blob_bytes(self.blob) } - } - - fn write(self, buf: *const c_void, length: c_int, offset: c_int) -> c_int { - unsafe { sqlite3_blob_write(self.blob, buf, length, offset) } - } - - fn read(self, buf: *mut c_void, length: c_int, offset: c_int) -> c_int { - unsafe { sqlite3_blob_read(self.blob, buf, length, offset) } - } - - fn read_single(self, byte: &mut u8, offset: c_int) -> c_int { - self.read(byte as *mut u8 as *mut _, 1, offset) - } - - fn write_single(self, byte: u8, offset: c_int) -> c_int { - self.write(&byte as *const u8 as *const _, 1, offset) - } - } - - #[derive(Copy, Clone)] - struct SqliteContext { - ctx: *mut sqlite3_context, - } - - impl From<*mut sqlite3_context> for SqliteContext { - fn from(ctx: *mut sqlite3_context) -> Self { - Self { ctx } - } - } - - impl SqliteContext { - fn user_data<T>(self) -> *mut T { - unsafe { sqlite3_user_data(self.ctx).cast() } - } - - fn aggregate_context<T>(self) -> *mut T { - unsafe { sqlite3_aggregate_context(self.ctx, std::mem::size_of::<T>() as c_int).cast() } - } - - fn result_exception(self, vm: &VirtualMachine, exc: PyBaseExceptionRef, msg: &str) { - if exc.fast_isinstance(vm.ctx.exceptions.memory_error) { - unsafe { sqlite3_result_error_nomem(self.ctx) } - } else if exc.fast_isinstance(vm.ctx.exceptions.overflow_error) { - unsafe { sqlite3_result_error_toobig(self.ctx) } - } else { - unsafe { sqlite3_result_error(self.ctx, msg.as_ptr().cast(), -1) } - } - if enable_traceback().load(Ordering::Relaxed) { - vm.print_exception(exc); - } - } - - fn db_handle(self) -> *mut sqlite3 { - unsafe { sqlite3_context_db_handle(self.ctx) } - } - - fn result_from_object(self, val: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - unsafe { - if vm.is_none(val) { - sqlite3_result_null(self.ctx) - } else if let Some(val) = val.payload::<PyInt>() { - sqlite3_result_int64(self.ctx, val.try_to_primitive(vm)?) - } else if let Some(val) = val.payload::<PyFloat>() { - sqlite3_result_double(self.ctx, val.to_f64()) - } else if let Some(val) = val.payload::<PyStr>() { - let (ptr, len) = str_to_ptr_len(val, vm)?; - sqlite3_result_text(self.ctx, ptr, len, SQLITE_TRANSIENT()) - } else if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, val) { - let (ptr, len) = buffer_to_ptr_len(&buffer, vm)?; - sqlite3_result_blob(self.ctx, ptr, len, SQLITE_TRANSIENT()) - } else { - return Err(new_programming_error( - vm, - "result type not support".to_owned(), - )); - } - } - Ok(()) - } - } - - fn value_to_object(val: *mut sqlite3_value, db: *mut sqlite3, vm: &VirtualMachine) -> PyResult { - let obj = unsafe { - match sqlite3_value_type(val) { - SQLITE_INTEGER => vm.ctx.new_int(sqlite3_value_int64(val)).into(), - SQLITE_FLOAT => vm.ctx.new_float(sqlite3_value_double(val)).into(), - SQLITE_TEXT => { - let text = - ptr_to_vec(sqlite3_value_text(val), sqlite3_value_bytes(val), db, vm)?; - let text = String::from_utf8(text).map_err(|_| { - vm.new_value_error("invalid utf-8 with SQLITE_TEXT".to_owned()) - })?; - vm.ctx.new_str(text).into() - } - SQLITE_BLOB => { - let blob = ptr_to_vec( - sqlite3_value_blob(val).cast(), - sqlite3_value_bytes(val), - db, - vm, - )?; - vm.ctx.new_bytes(blob).into() - } - _ => vm.ctx.none(), - } - }; - Ok(obj) - } - - fn ptr_to_str<'a>(p: *const libc::c_char, vm: &VirtualMachine) -> PyResult<&'a str> { - if p.is_null() { - return Err(vm.new_memory_error("string pointer is null".to_owned())); - } - unsafe { CStr::from_ptr(p).to_str() } - .map_err(|_| vm.new_value_error("Invalid UIF-8 codepoint".to_owned())) - } - - fn ptr_to_string( - p: *const u8, - nbytes: c_int, - db: *mut sqlite3, - vm: &VirtualMachine, - ) -> PyResult<String> { - let s = ptr_to_vec(p, nbytes, db, vm)?; - String::from_utf8(s).map_err(|_| vm.new_value_error("invalid utf-8".to_owned())) - } - - fn ptr_to_vec( - p: *const u8, - nbytes: c_int, - db: *mut sqlite3, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - if p.is_null() { - if !db.is_null() && unsafe { sqlite3_errcode(db) } == SQLITE_NOMEM { - Err(vm.new_memory_error("sqlite out of memory".to_owned())) - } else { - Ok(vec![]) - } - } else if nbytes < 0 { - Err(vm.new_system_error("negative size with ptr".to_owned())) - } else { - Ok(unsafe { std::slice::from_raw_parts(p.cast(), nbytes as usize) }.to_vec()) - } - } - - fn str_to_ptr_len(s: &PyStr, vm: &VirtualMachine) -> PyResult<(*const libc::c_char, i32)> { - let len = c_int::try_from(s.byte_len()) - .map_err(|_| vm.new_overflow_error("TEXT longer than INT_MAX bytes".to_owned()))?; - let ptr = s.as_str().as_ptr().cast(); - Ok((ptr, len)) - } - - fn buffer_to_ptr_len(buffer: &PyBuffer, vm: &VirtualMachine) -> PyResult<(*const c_void, i32)> { - let bytes = buffer.as_contiguous().ok_or_else(|| { - vm.new_buffer_error("underlying buffer is not C-contiguous".to_owned()) - })?; - let len = c_int::try_from(bytes.len()) - .map_err(|_| vm.new_overflow_error("BLOB longer than INT_MAX bytes".to_owned()))?; - let ptr = bytes.as_ptr().cast(); - Ok((ptr, len)) - } - - fn exception_type_from_errcode(errcode: c_int, vm: &VirtualMachine) -> &'static Py<PyType> { - match errcode { - SQLITE_INTERNAL | SQLITE_NOTFOUND => internal_error_type(), - SQLITE_NOMEM => vm.ctx.exceptions.memory_error, - SQLITE_ERROR | SQLITE_PERM | SQLITE_ABORT | SQLITE_BUSY | SQLITE_LOCKED - | SQLITE_READONLY | SQLITE_INTERRUPT | SQLITE_IOERR | SQLITE_FULL | SQLITE_CANTOPEN - | SQLITE_PROTOCOL | SQLITE_EMPTY | SQLITE_SCHEMA => operational_error_type(), - SQLITE_CORRUPT => database_error_type(), - SQLITE_TOOBIG => data_error_type(), - SQLITE_CONSTRAINT | SQLITE_MISMATCH => integrity_error_type(), - SQLITE_MISUSE | SQLITE_RANGE => interface_error_type(), - _ => database_error_type(), - } - } - - fn name_from_errcode(errcode: c_int) -> &'static str { - for (name, code) in ERROR_CODES { - if *code == errcode { - return name; - } - } - "unknown error code" - } - - fn raise_exception( - typ: PyTypeRef, - errcode: c_int, - msg: String, - vm: &VirtualMachine, - ) -> PyBaseExceptionRef { - let dict = vm.ctx.new_dict(); - if let Err(e) = dict.set_item("sqlite_errorcode", vm.ctx.new_int(errcode).into(), vm) { - return e; - } - let errname = name_from_errcode(errcode); - if let Err(e) = dict.set_item("sqlite_errorname", vm.ctx.new_str(errname).into(), vm) { - return e; - } - - vm.new_exception_msg_dict(typ, msg, dict) - } - - static BEGIN_STATEMENTS: &[&[u8]] = &[ - b"BEGIN ", - b"BEGIN DEFERRED", - b"BEGIN IMMEDIATE", - b"BEGIN EXCLUSIVE", - ]; - - fn begin_statement_ptr_from_isolation_level( - s: &PyStr, - vm: &VirtualMachine, - ) -> PyResult<*const libc::c_char> { - BEGIN_STATEMENTS - .iter() - .find(|&&x| x[6..].eq_ignore_ascii_case(s.as_str().as_bytes())) - .map(|&x| x.as_ptr().cast()) - .ok_or_else(|| { - vm.new_value_error( - "isolation_level string must be '', 'DEFERRED', 'IMMEDIATE', or 'EXCLUSIVE'" - .to_owned(), - ) - }) - } - - fn lstrip_sql(sql: &[u8]) -> Option<&[u8]> { - let mut pos = sql; - loop { - match pos.first()? { - b' ' | b'\t' | b'\x0c' | b'\n' | b'\r' => { - pos = &pos[1..]; - } - b'-' => { - if *pos.get(1)? == b'-' { - // line comments - pos = &pos[2..]; - while *pos.first()? != b'\n' { - pos = &pos[1..]; - } - } else { - return Some(pos); - } - } - b'/' => { - if *pos.get(1)? == b'*' { - // c style comments - pos = &pos[2..]; - while *pos.first()? != b'*' || *pos.get(1)? != b'/' { - pos = &pos[1..]; - } - } else { - return Some(pos); - } - } - _ => return Some(pos), - } - } - } -} diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs deleted file mode 100644 index cf3a8020380..00000000000 --- a/stdlib/src/ssl.rs +++ /dev/null @@ -1,1529 +0,0 @@ -use crate::vm::{builtins::PyModule, PyRef, VirtualMachine}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - // if openssl is vendored, it doesn't know the locations of system certificates - #[cfg(feature = "ssl-vendor")] - if let None | Some("0") = option_env!("OPENSSL_NO_VENDOR") { - openssl_probe::init_ssl_cert_env_vars(); - } - openssl::init(); - _ssl::make_module(vm) -} - -#[allow(non_upper_case_globals)] -#[pymodule(with(ossl101, windows))] -mod _ssl { - use super::bio; - use crate::{ - common::{ - ascii, - lock::{ - PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, - }, - }, - socket::{self, PySocket}, - vm::{ - builtins::{PyBaseExceptionRef, PyStrRef, PyType, PyTypeRef, PyWeak}, - convert::{ToPyException, ToPyObject}, - exceptions, - function::{ - ArgBytesLike, ArgCallable, ArgMemoryBuffer, ArgStrOrBytesLike, Either, FsPath, - OptionalArg, - }, - types::Constructor, - utils::ToCString, - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - }, - }; - use crossbeam_utils::atomic::AtomicCell; - use foreign_types_shared::{ForeignType, ForeignTypeRef}; - use openssl::{ - asn1::{Asn1Object, Asn1ObjectRef}, - error::ErrorStack, - nid::Nid, - ssl::{self, SslContextBuilder, SslOptions, SslVerifyMode}, - x509::{self, X509Ref, X509}, - }; - use openssl_sys as sys; - use std::{ - ffi::CStr, - fmt, - io::{Read, Write}, - time::Instant, - }; - - // Constants - #[pyattr] - use sys::{ - SSL_OP_NO_SSLv2 as OP_NO_SSLv2, - SSL_OP_NO_SSLv3 as OP_NO_SSLv3, - SSL_OP_NO_TLSv1 as OP_NO_TLSv1, - // TODO: so many more of these - SSL_AD_DECODE_ERROR as ALERT_DESCRIPTION_DECODE_ERROR, - SSL_AD_ILLEGAL_PARAMETER as ALERT_DESCRIPTION_ILLEGAL_PARAMETER, - SSL_AD_UNRECOGNIZED_NAME as ALERT_DESCRIPTION_UNRECOGNIZED_NAME, - // SSL_ERROR_INVALID_ERROR_CODE, - SSL_ERROR_SSL, - // SSL_ERROR_WANT_X509_LOOKUP, - SSL_ERROR_SYSCALL, - SSL_ERROR_WANT_CONNECT, - SSL_ERROR_WANT_READ, - SSL_ERROR_WANT_WRITE, - // #ifdef SSL_OP_SINGLE_ECDH_USE - // SSL_OP_SINGLE_ECDH_USE as OP_SINGLE_ECDH_USE - // #endif - // X509_V_FLAG_CRL_CHECK as VERIFY_CRL_CHECK_LEAF, - // sys::X509_V_FLAG_CRL_CHECK|sys::X509_V_FLAG_CRL_CHECK_ALL as VERIFY_CRL_CHECK_CHAIN - // X509_V_FLAG_X509_STRICT as VERIFY_X509_STRICT, - SSL_ERROR_ZERO_RETURN, - SSL_OP_CIPHER_SERVER_PREFERENCE as OP_CIPHER_SERVER_PREFERENCE, - SSL_OP_NO_TICKET as OP_NO_TICKET, - SSL_OP_SINGLE_DH_USE as OP_SINGLE_DH_USE, - }; - - // taken from CPython, should probably be kept up to date with their version if it ever changes - #[pyattr] - const _DEFAULT_CIPHERS: &str = - "DEFAULT:!aNULL:!eNULL:!MD5:!3DES:!DES:!RC4:!IDEA:!SEED:!aDSS:!SRP:!PSK"; - // #[pyattr] PROTOCOL_SSLv2: u32 = SslVersion::Ssl2 as u32; // unsupported - // #[pyattr] PROTOCOL_SSLv3: u32 = SslVersion::Ssl3 as u32; - #[pyattr] - const PROTOCOL_SSLv23: u32 = SslVersion::Tls as u32; - #[pyattr] - const PROTOCOL_TLS: u32 = SslVersion::Tls as u32; - #[pyattr] - const PROTOCOL_TLS_CLIENT: u32 = SslVersion::TlsClient as u32; - #[pyattr] - const PROTOCOL_TLS_SERVER: u32 = SslVersion::TlsServer as u32; - #[pyattr] - const PROTOCOL_TLSv1: u32 = SslVersion::Tls1 as u32; - #[pyattr] - const PROTO_MINIMUM_SUPPORTED: i32 = ProtoVersion::MinSupported as i32; - #[pyattr] - const PROTO_SSLv3: i32 = ProtoVersion::Ssl3 as i32; - #[pyattr] - const PROTO_TLSv1: i32 = ProtoVersion::Tls1 as i32; - #[pyattr] - const PROTO_TLSv1_1: i32 = ProtoVersion::Tls1_1 as i32; - #[pyattr] - const PROTO_TLSv1_2: i32 = ProtoVersion::Tls1_2 as i32; - #[pyattr] - const PROTO_TLSv1_3: i32 = ProtoVersion::Tls1_3 as i32; - #[pyattr] - const PROTO_MAXIMUM_SUPPORTED: i32 = ProtoVersion::MaxSupported as i32; - #[pyattr] - const OP_ALL: libc::c_ulong = (sys::SSL_OP_ALL & !sys::SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) as _; - #[pyattr] - const HAS_TLS_UNIQUE: bool = true; - #[pyattr] - const CERT_NONE: u32 = CertRequirements::None as u32; - #[pyattr] - const CERT_OPTIONAL: u32 = CertRequirements::Optional as u32; - #[pyattr] - const CERT_REQUIRED: u32 = CertRequirements::Required as u32; - #[pyattr] - const VERIFY_DEFAULT: u32 = 0; - #[pyattr] - const SSL_ERROR_EOF: u32 = 8; // custom for python - #[pyattr] - const HAS_SNI: bool = true; - #[pyattr] - const HAS_ECDH: bool = false; - #[pyattr] - const HAS_NPN: bool = false; - #[pyattr] - const HAS_ALPN: bool = true; - #[pyattr] - const HAS_SSLv2: bool = true; - #[pyattr] - const HAS_SSLv3: bool = true; - #[pyattr] - const HAS_TLSv1: bool = true; - #[pyattr] - const HAS_TLSv1_1: bool = true; - #[pyattr] - const HAS_TLSv1_2: bool = true; - #[pyattr] - const HAS_TLSv1_3: bool = cfg!(ossl111); - - // the openssl version from the API headers - - #[pyattr(name = "OPENSSL_VERSION")] - fn openssl_version(_vm: &VirtualMachine) -> &str { - openssl::version::version() - } - #[pyattr(name = "OPENSSL_VERSION_NUMBER")] - fn openssl_version_number(_vm: &VirtualMachine) -> i64 { - openssl::version::number() - } - #[pyattr(name = "OPENSSL_VERSION_INFO")] - fn openssl_version_info(_vm: &VirtualMachine) -> OpensslVersionInfo { - parse_version_info(openssl::version::number()) - } - - #[pyattr(name = "_OPENSSL_API_VERSION")] - fn _openssl_api_version(_vm: &VirtualMachine) -> OpensslVersionInfo { - let openssl_api_version = i64::from_str_radix(env!("OPENSSL_API_VERSION"), 16).unwrap(); - parse_version_info(openssl_api_version) - } - - /// An error occurred in the SSL implementation. - #[pyattr(name = "SSLError", once)] - fn ssl_error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "ssl", - "SSLError", - Some(vec![vm.ctx.exceptions.os_error.to_owned()]), - ) - } - - /// A certificate could not be verified. - #[pyattr(name = "SSLCertVerificationError", once)] - fn ssl_cert_verification_error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "ssl", - "SSLCertVerificationError", - Some(vec![ - ssl_error(vm), - vm.ctx.exceptions.value_error.to_owned(), - ]), - ) - } - - /// SSL/TLS session closed cleanly. - #[pyattr(name = "SSLZeroReturnError", once)] - fn ssl_zero_return_error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx - .new_exception_type("ssl", "SSLZeroReturnError", Some(vec![ssl_error(vm)])) - } - - /// Non-blocking SSL socket needs to read more data before the requested operation can be completed. - #[pyattr(name = "SSLWantReadError", once)] - fn ssl_want_read_error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx - .new_exception_type("ssl", "SSLWantReadError", Some(vec![ssl_error(vm)])) - } - - /// Non-blocking SSL socket needs to write more data before the requested operation can be completed. - #[pyattr(name = "SSLWantWriteError", once)] - fn ssl_want_write_error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx - .new_exception_type("ssl", "SSLWantWriteError", Some(vec![ssl_error(vm)])) - } - - /// System error when attempting SSL operation. - #[pyattr(name = "SSLSyscallError", once)] - fn ssl_syscall_error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx - .new_exception_type("ssl", "SSLSyscallError", Some(vec![ssl_error(vm)])) - } - - /// SSL/TLS connection terminated abruptly. - #[pyattr(name = "SSLEOFError", once)] - fn ssl_eof_error(vm: &VirtualMachine) -> PyTypeRef { - PyType::new_simple_heap("ssl.SSLEOFError", &ssl_error(vm), &vm.ctx).unwrap() - } - - type OpensslVersionInfo = (u8, u8, u8, u8, u8); - const fn parse_version_info(mut n: i64) -> OpensslVersionInfo { - let status = (n & 0xF) as u8; - n >>= 4; - let patch = (n & 0xFF) as u8; - n >>= 8; - let fix = (n & 0xFF) as u8; - n >>= 8; - let minor = (n & 0xFF) as u8; - n >>= 8; - let major = (n & 0xFF) as u8; - (major, minor, fix, patch, status) - } - - #[derive(Copy, Clone, num_enum::IntoPrimitive, num_enum::TryFromPrimitive, PartialEq)] - #[repr(i32)] - enum SslVersion { - Ssl2, - Ssl3 = 1, - Tls, - Tls1, - // TODO: Tls1_1, Tls1_2 ? - TlsClient = 0x10, - TlsServer, - } - - #[derive(Copy, Clone, num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] - #[repr(i32)] - enum ProtoVersion { - MinSupported = -2, - Ssl3 = sys::SSL3_VERSION, - Tls1 = sys::TLS1_VERSION, - Tls1_1 = sys::TLS1_1_VERSION, - Tls1_2 = sys::TLS1_2_VERSION, - #[cfg(ossl111)] - Tls1_3 = sys::TLS1_3_VERSION, - #[cfg(not(ossl111))] - Tls1_3 = 0x304, - MaxSupported = -1, - } - - #[derive(num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] - #[repr(i32)] - enum CertRequirements { - None, - Optional, - Required, - } - - #[derive(Debug, PartialEq)] - enum SslServerOrClient { - Client, - Server, - } - - unsafe fn ptr2obj(ptr: *mut sys::ASN1_OBJECT) -> Option<Asn1Object> { - if ptr.is_null() { - None - } else { - Some(Asn1Object::from_ptr(ptr)) - } - } - - fn _txt2obj(s: &CStr, no_name: bool) -> Option<Asn1Object> { - unsafe { ptr2obj(sys::OBJ_txt2obj(s.as_ptr(), i32::from(no_name))) } - } - fn _nid2obj(nid: Nid) -> Option<Asn1Object> { - unsafe { ptr2obj(sys::OBJ_nid2obj(nid.as_raw())) } - } - fn obj2txt(obj: &Asn1ObjectRef, no_name: bool) -> Option<String> { - let no_name = i32::from(no_name); - let ptr = obj.as_ptr(); - let b = unsafe { - let buflen = sys::OBJ_obj2txt(std::ptr::null_mut(), 0, ptr, no_name); - assert!(buflen >= 0); - if buflen == 0 { - return None; - } - let buflen = buflen as usize; - let mut buf = Vec::<u8>::with_capacity(buflen + 1); - let ret = sys::OBJ_obj2txt( - buf.as_mut_ptr() as *mut libc::c_char, - buf.capacity() as _, - ptr, - no_name, - ); - assert!(ret >= 0); - // SAFETY: OBJ_obj2txt initialized the buffer successfully - buf.set_len(buflen); - buf - }; - let s = String::from_utf8(b) - .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()); - Some(s) - } - - type PyNid = (libc::c_int, String, String, Option<String>); - fn obj2py(obj: &Asn1ObjectRef) -> PyNid { - let nid = obj.nid(); - ( - nid.as_raw(), - nid.short_name().unwrap().to_owned(), - nid.long_name().unwrap().to_owned(), - obj2txt(obj, true), - ) - } - - #[derive(FromArgs)] - struct Txt2ObjArgs { - txt: PyStrRef, - #[pyarg(any, default = "false")] - name: bool, - } - - #[pyfunction] - fn txt2obj(args: Txt2ObjArgs, vm: &VirtualMachine) -> PyResult<PyNid> { - _txt2obj(&args.txt.to_cstring(vm)?, !args.name) - .as_deref() - .map(obj2py) - .ok_or_else(|| vm.new_value_error(format!("unknown object '{}'", args.txt))) - } - - #[pyfunction] - fn nid2obj(nid: libc::c_int, vm: &VirtualMachine) -> PyResult<PyNid> { - _nid2obj(Nid::from_raw(nid)) - .as_deref() - .map(obj2py) - .ok_or_else(|| vm.new_value_error(format!("unknown NID {nid}"))) - } - - #[pyfunction] - fn get_default_verify_paths() -> (String, String, String, String) { - macro_rules! convert { - ($f:ident) => { - CStr::from_ptr(sys::$f()).to_string_lossy().into_owned() - }; - } - unsafe { - ( - convert!(X509_get_default_cert_file_env), - convert!(X509_get_default_cert_file), - convert!(X509_get_default_cert_dir_env), - convert!(X509_get_default_cert_dir), - ) - } - } - - #[pyfunction(name = "RAND_status")] - fn rand_status() -> i32 { - unsafe { sys::RAND_status() } - } - - #[pyfunction(name = "RAND_add")] - fn rand_add(string: ArgStrOrBytesLike, entropy: f64) { - let f = |b: &[u8]| { - for buf in b.chunks(libc::c_int::max_value() as usize) { - unsafe { sys::RAND_add(buf.as_ptr() as *const _, buf.len() as _, entropy) } - } - }; - f(&string.borrow_bytes()) - } - - #[pyfunction(name = "RAND_bytes")] - fn rand_bytes(n: i32, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - if n < 0 { - return Err(vm.new_value_error("num must be positive".to_owned())); - } - let mut buf = vec![0; n as usize]; - openssl::rand::rand_bytes(&mut buf).map_err(|e| convert_openssl_error(vm, e))?; - Ok(buf) - } - - #[pyfunction(name = "RAND_pseudo_bytes")] - fn rand_pseudo_bytes(n: i32, vm: &VirtualMachine) -> PyResult<(Vec<u8>, bool)> { - if n < 0 { - return Err(vm.new_value_error("num must be positive".to_owned())); - } - let mut buf = vec![0; n as usize]; - let ret = unsafe { sys::RAND_bytes(buf.as_mut_ptr(), n) }; - match ret { - 0 | 1 => Ok((buf, ret == 1)), - _ => Err(convert_openssl_error(vm, ErrorStack::get())), - } - } - - #[pyattr] - #[pyclass(module = "ssl", name = "_SSLContext")] - #[derive(PyPayload)] - struct PySslContext { - ctx: PyRwLock<SslContextBuilder>, - check_hostname: AtomicCell<bool>, - protocol: SslVersion, - post_handshake_auth: PyMutex<bool>, - } - - impl fmt::Debug for PySslContext { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("_SSLContext") - } - } - - fn builder_as_ctx(x: &SslContextBuilder) -> &ssl::SslContextRef { - unsafe { ssl::SslContextRef::from_ptr(x.as_ptr()) } - } - - impl Constructor for PySslContext { - type Args = i32; - - fn py_new(cls: PyTypeRef, proto_version: Self::Args, vm: &VirtualMachine) -> PyResult { - let proto = SslVersion::try_from(proto_version) - .map_err(|_| vm.new_value_error("invalid protocol version".to_owned()))?; - let method = match proto { - // SslVersion::Ssl3 => unsafe { ssl::SslMethod::from_ptr(sys::SSLv3_method()) }, - SslVersion::Tls => ssl::SslMethod::tls(), - // TODO: Tls1_1, Tls1_2 ? - SslVersion::TlsClient => ssl::SslMethod::tls_client(), - SslVersion::TlsServer => ssl::SslMethod::tls_server(), - _ => return Err(vm.new_value_error("invalid protocol version".to_owned())), - }; - let mut builder = - SslContextBuilder::new(method).map_err(|e| convert_openssl_error(vm, e))?; - - #[cfg(target_os = "android")] - android::load_client_ca_list(vm, &mut builder)?; - - let check_hostname = proto == SslVersion::TlsClient; - builder.set_verify(if check_hostname { - SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT - } else { - SslVerifyMode::NONE - }); - - let mut options = SslOptions::ALL & !SslOptions::DONT_INSERT_EMPTY_FRAGMENTS; - if proto != SslVersion::Ssl2 { - options |= SslOptions::NO_SSLV2; - } - if proto != SslVersion::Ssl3 { - options |= SslOptions::NO_SSLV3; - } - options |= SslOptions::NO_COMPRESSION; - options |= SslOptions::CIPHER_SERVER_PREFERENCE; - options |= SslOptions::SINGLE_DH_USE; - options |= SslOptions::SINGLE_ECDH_USE; - builder.set_options(options); - - let mode = ssl::SslMode::ACCEPT_MOVING_WRITE_BUFFER | ssl::SslMode::AUTO_RETRY; - builder.set_mode(mode); - - #[cfg(ossl111)] - unsafe { - sys::SSL_CTX_set_post_handshake_auth(builder.as_ptr(), 0); - } - - builder - .set_session_id_context(b"Python") - .map_err(|e| convert_openssl_error(vm, e))?; - - PySslContext { - ctx: PyRwLock::new(builder), - check_hostname: AtomicCell::new(check_hostname), - protocol: proto, - post_handshake_auth: PyMutex::new(false), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(flags(BASETYPE), with(Constructor))] - impl PySslContext { - fn builder(&self) -> PyRwLockWriteGuard<'_, SslContextBuilder> { - self.ctx.write() - } - fn ctx(&self) -> PyMappedRwLockReadGuard<'_, ssl::SslContextRef> { - PyRwLockReadGuard::map(self.ctx.read(), builder_as_ctx) - } - - #[pygetset] - fn post_handshake_auth(&self) -> bool { - *self.post_handshake_auth.lock() - } - #[pygetset(setter)] - fn set_post_handshake_auth( - &self, - value: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let value = value - .ok_or_else(|| vm.new_attribute_error("cannot delete attribute".to_owned()))?; - *self.post_handshake_auth.lock() = value.is_true(vm)?; - Ok(()) - } - - #[pymethod] - fn set_ciphers(&self, cipherlist: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - let ciphers = cipherlist.as_str(); - if ciphers.contains('\0') { - return Err(exceptions::cstring_error(vm)); - } - self.builder().set_cipher_list(ciphers).map_err(|_| { - vm.new_exception_msg(ssl_error(vm), "No cipher can be selected.".to_owned()) - }) - } - - #[pygetset] - fn options(&self) -> libc::c_ulong { - self.ctx.read().options().bits() as _ - } - #[pygetset(setter)] - fn set_options(&self, opts: libc::c_ulong) { - self.builder() - .set_options(SslOptions::from_bits_truncate(opts as _)); - } - #[pygetset] - fn protocol(&self) -> i32 { - self.protocol as i32 - } - #[pygetset] - fn verify_mode(&self) -> i32 { - let mode = self.ctx().verify_mode(); - if mode == SslVerifyMode::NONE { - CertRequirements::None.into() - } else if mode == SslVerifyMode::PEER { - CertRequirements::Optional.into() - } else if mode == SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT { - CertRequirements::Required.into() - } else { - unreachable!() - } - } - #[pygetset(setter)] - fn set_verify_mode(&self, cert: i32, vm: &VirtualMachine) -> PyResult<()> { - let mut ctx = self.builder(); - let cert_req = CertRequirements::try_from(cert) - .map_err(|_| vm.new_value_error("invalid value for verify_mode".to_owned()))?; - let mode = match cert_req { - CertRequirements::None if self.check_hostname.load() => { - return Err(vm.new_value_error( - "Cannot set verify_mode to CERT_NONE when check_hostname is enabled." - .to_owned(), - )) - } - CertRequirements::None => SslVerifyMode::NONE, - CertRequirements::Optional => SslVerifyMode::PEER, - CertRequirements::Required => { - SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT - } - }; - ctx.set_verify(mode); - Ok(()) - } - #[pygetset] - fn check_hostname(&self) -> bool { - self.check_hostname.load() - } - #[pygetset(setter)] - fn set_check_hostname(&self, ch: bool) { - let mut ctx = self.builder(); - if ch && builder_as_ctx(&ctx).verify_mode() == SslVerifyMode::NONE { - ctx.set_verify(SslVerifyMode::PEER | SslVerifyMode::FAIL_IF_NO_PEER_CERT); - } - self.check_hostname.store(ch); - } - - #[pymethod] - fn set_default_verify_paths(&self, vm: &VirtualMachine) -> PyResult<()> { - self.builder() - .set_default_verify_paths() - .map_err(|e| convert_openssl_error(vm, e)) - } - - #[pymethod] - fn _set_alpn_protocols(&self, protos: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { - #[cfg(ossl102)] - { - let mut ctx = self.builder(); - let server = protos.with_ref(|pbuf| { - if pbuf.len() > libc::c_uint::MAX as usize { - return Err(vm.new_overflow_error(format!( - "protocols longer than {} bytes", - libc::c_uint::MAX - ))); - } - ctx.set_alpn_protos(pbuf) - .map_err(|e| convert_openssl_error(vm, e))?; - Ok(pbuf.to_vec()) - })?; - ctx.set_alpn_select_callback(move |_, client| { - ssl::select_next_proto(&server, client).ok_or(ssl::AlpnError::NOACK) - }); - Ok(()) - } - #[cfg(not(ossl102))] - { - Err(vm.new_not_implemented_error( - "The NPN extension requires OpenSSL 1.0.1 or later.".to_owned(), - )) - } - } - - #[pymethod] - fn load_verify_locations( - &self, - args: LoadVerifyLocationsArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { - if let (None, None, None) = (&args.cafile, &args.capath, &args.cadata) { - return Err( - vm.new_type_error("cafile, capath and cadata cannot be all omitted".to_owned()) - ); - } - - #[cold] - fn invalid_cadata(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_type_error( - "cadata should be an ASCII string or a bytes-like object".to_owned(), - ) - } - - // validate cadata type and load cadata - if let Some(cadata) = args.cadata { - let certs = match cadata { - Either::A(s) => { - if !s.is_ascii() { - return Err(invalid_cadata(vm)); - } - X509::stack_from_pem(s.as_str().as_bytes()) - } - Either::B(b) => b.with_ref(x509_stack_from_der), - }; - let certs = certs.map_err(|e| convert_openssl_error(vm, e))?; - let mut ctx = self.builder(); - let store = ctx.cert_store_mut(); - for cert in certs { - store - .add_cert(cert) - .map_err(|e| convert_openssl_error(vm, e))?; - } - } - - if args.cafile.is_some() || args.capath.is_some() { - let cafile = args.cafile.map(|s| s.to_cstring(vm)).transpose()?; - let capath = args.capath.map(|s| s.to_cstring(vm)).transpose()?; - let ret = unsafe { - let ctx = self.ctx.write(); - sys::SSL_CTX_load_verify_locations( - ctx.as_ptr(), - cafile - .as_ref() - .map_or_else(std::ptr::null, |cs| cs.as_ptr()), - capath - .as_ref() - .map_or_else(std::ptr::null, |cs| cs.as_ptr()), - ) - }; - if ret != 1 { - let errno = std::io::Error::last_os_error().raw_os_error().unwrap(); - let err = if errno != 0 { - crate::vm::stdlib::os::errno_err(vm) - } else { - convert_openssl_error(vm, ErrorStack::get()) - }; - return Err(err); - } - } - - Ok(()) - } - - #[pymethod] - fn get_ca_certs( - &self, - binary_form: OptionalArg<bool>, - vm: &VirtualMachine, - ) -> PyResult<Vec<PyObjectRef>> { - let binary_form = binary_form.unwrap_or(false); - let certs = self - .ctx() - .cert_store() - .all_certificates() - .iter() - .map(|cert| cert_to_py(vm, cert, binary_form)) - .collect::<Result<Vec<_>, _>>()?; - Ok(certs) - } - - #[pymethod] - fn load_cert_chain(&self, args: LoadCertChainArgs, vm: &VirtualMachine) -> PyResult<()> { - let LoadCertChainArgs { - certfile, - keyfile, - password, - } = args; - // TODO: requires passing a callback to C - if password.is_some() { - return Err( - vm.new_not_implemented_error("password arg not yet supported".to_owned()) - ); - } - let mut ctx = self.builder(); - let key_path = keyfile.map(|path| path.to_path_buf(vm)).transpose()?; - let cert_path = certfile.to_path_buf(vm)?; - ctx.set_certificate_chain_file(&cert_path) - .and_then(|()| { - ctx.set_private_key_file( - key_path.as_ref().unwrap_or(&cert_path), - ssl::SslFiletype::PEM, - ) - }) - .and_then(|()| ctx.check_private_key()) - .map_err(|e| convert_openssl_error(vm, e)) - } - - #[pymethod] - fn _wrap_socket( - zelf: PyRef<Self>, - args: WrapSocketArgs, - vm: &VirtualMachine, - ) -> PyResult<PySslSocket> { - let mut ssl = ssl::Ssl::new(&zelf.ctx()).map_err(|e| convert_openssl_error(vm, e))?; - - let socket_type = if args.server_side { - ssl.set_accept_state(); - SslServerOrClient::Server - } else { - ssl.set_connect_state(); - SslServerOrClient::Client - }; - - if let Some(hostname) = &args.server_hostname { - let hostname = hostname.as_str(); - if hostname.is_empty() || hostname.starts_with('.') { - return Err(vm.new_value_error( - "server_hostname cannot be an empty string or start with a leading dot." - .to_owned(), - )); - } - let ip = hostname.parse::<std::net::IpAddr>(); - if ip.is_err() { - ssl.set_hostname(hostname) - .map_err(|e| convert_openssl_error(vm, e))?; - } - if zelf.check_hostname.load() { - if let Ok(ip) = ip { - ssl.param_mut() - .set_ip(ip) - .map_err(|e| convert_openssl_error(vm, e))?; - } else { - ssl.param_mut() - .set_host(hostname) - .map_err(|e| convert_openssl_error(vm, e))?; - } - } - } - - let stream = ssl::SslStream::new(ssl, SocketStream(args.sock.clone())) - .map_err(|e| convert_openssl_error(vm, e))?; - - // TODO: use this - let _ = args.session; - - Ok(PySslSocket { - ctx: zelf, - stream: PyRwLock::new(stream), - socket_type, - server_hostname: args.server_hostname, - owner: PyRwLock::new(args.owner.map(|o| o.downgrade(None, vm)).transpose()?), - }) - } - } - - #[derive(FromArgs)] - struct WrapSocketArgs { - sock: PyRef<PySocket>, - server_side: bool, - #[pyarg(any, default)] - server_hostname: Option<PyStrRef>, - #[pyarg(named, default)] - owner: Option<PyObjectRef>, - #[pyarg(named, default)] - session: Option<PyObjectRef>, - } - - #[derive(FromArgs)] - struct LoadVerifyLocationsArgs { - #[pyarg(any, default)] - cafile: Option<PyStrRef>, - #[pyarg(any, default)] - capath: Option<PyStrRef>, - #[pyarg(any, default)] - cadata: Option<Either<PyStrRef, ArgBytesLike>>, - } - - #[derive(FromArgs)] - struct LoadCertChainArgs { - certfile: FsPath, - #[pyarg(any, optional)] - keyfile: Option<FsPath>, - #[pyarg(any, optional)] - password: Option<Either<PyStrRef, ArgCallable>>, - } - - // Err is true if the socket is blocking - type SocketDeadline = Result<Instant, bool>; - - enum SelectRet { - Nonblocking, - TimedOut, - IsBlocking, - Closed, - Ok, - } - - #[derive(Clone, Copy)] - enum SslNeeds { - Read, - Write, - } - - struct SocketStream(PyRef<PySocket>); - - impl SocketStream { - fn timeout_deadline(&self) -> SocketDeadline { - self.0.get_timeout().map(|d| Instant::now() + d) - } - - fn select(&self, needs: SslNeeds, deadline: &SocketDeadline) -> SelectRet { - let sock = match self.0.sock_opt() { - Some(s) => s, - None => return SelectRet::Closed, - }; - let deadline = match &deadline { - Ok(deadline) => match deadline.checked_duration_since(Instant::now()) { - Some(deadline) => deadline, - None => return SelectRet::TimedOut, - }, - Err(true) => return SelectRet::IsBlocking, - Err(false) => return SelectRet::Nonblocking, - }; - let res = socket::sock_select( - &sock, - match needs { - SslNeeds::Read => socket::SelectKind::Read, - SslNeeds::Write => socket::SelectKind::Write, - }, - Some(deadline), - ); - match res { - Ok(true) => SelectRet::TimedOut, - _ => SelectRet::Ok, - } - } - - fn socket_needs( - &self, - err: &ssl::Error, - deadline: &SocketDeadline, - ) -> (Option<SslNeeds>, SelectRet) { - let needs = match err.code() { - ssl::ErrorCode::WANT_READ => Some(SslNeeds::Read), - ssl::ErrorCode::WANT_WRITE => Some(SslNeeds::Write), - _ => None, - }; - let state = needs.map_or(SelectRet::Ok, |needs| self.select(needs, deadline)); - (needs, state) - } - } - - fn socket_closed_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception_msg( - ssl_error(vm), - "Underlying socket has been closed.".to_owned(), - ) - } - - #[pyattr] - #[pyclass(module = "ssl", name = "_SSLSocket", traverse)] - #[derive(PyPayload)] - struct PySslSocket { - ctx: PyRef<PySslContext>, - #[pytraverse(skip)] - stream: PyRwLock<ssl::SslStream<SocketStream>>, - #[pytraverse(skip)] - socket_type: SslServerOrClient, - server_hostname: Option<PyStrRef>, - owner: PyRwLock<Option<PyRef<PyWeak>>>, - } - - impl fmt::Debug for PySslSocket { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad("_SSLSocket") - } - } - - #[pyclass] - impl PySslSocket { - #[pygetset] - fn owner(&self) -> Option<PyObjectRef> { - self.owner.read().as_ref().and_then(|weak| weak.upgrade()) - } - #[pygetset(setter)] - fn set_owner(&self, owner: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut lock = self.owner.write(); - lock.take(); - *lock = Some(owner.downgrade(None, vm)?); - Ok(()) - } - #[pygetset] - fn server_side(&self) -> bool { - self.socket_type == SslServerOrClient::Server - } - #[pygetset] - fn context(&self) -> PyRef<PySslContext> { - self.ctx.clone() - } - #[pygetset] - fn server_hostname(&self) -> Option<PyStrRef> { - self.server_hostname.clone() - } - - #[pymethod] - fn getpeercert( - &self, - binary: OptionalArg<bool>, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - let binary = binary.unwrap_or(false); - let stream = self.stream.read(); - if !stream.ssl().is_init_finished() { - return Err(vm.new_value_error("handshake not done yet".to_owned())); - } - stream - .ssl() - .peer_certificate() - .map(|cert| cert_to_py(vm, &cert, binary)) - .transpose() - } - - #[pymethod] - fn version(&self) -> Option<&'static str> { - let v = self.stream.read().ssl().version_str(); - if v == "unknown" { - None - } else { - Some(v) - } - } - - #[pymethod] - fn cipher(&self) -> Option<CipherTuple> { - self.stream - .read() - .ssl() - .current_cipher() - .map(cipher_to_tuple) - } - - #[cfg(osslconf = "OPENSSL_NO_COMP")] - #[pymethod] - fn compression(&self) -> Option<&'static str> { - None - } - #[cfg(not(osslconf = "OPENSSL_NO_COMP"))] - #[pymethod] - fn compression(&self) -> Option<&'static str> { - let stream = self.stream.read(); - let comp_method = unsafe { sys::SSL_get_current_compression(stream.ssl().as_ptr()) }; - if comp_method.is_null() { - return None; - } - let typ = unsafe { sys::COMP_get_type(comp_method) }; - let nid = Nid::from_raw(typ); - if nid == Nid::UNDEF { - return None; - } - nid.short_name().ok() - } - - #[pymethod] - fn do_handshake(&self, vm: &VirtualMachine) -> PyResult<()> { - let mut stream = self.stream.write(); - let timeout = stream.get_ref().timeout_deadline(); - loop { - let err = match stream.do_handshake() { - Ok(()) => return Ok(()), - Err(e) => e, - }; - let (needs, state) = stream.get_ref().socket_needs(&err, &timeout); - match state { - SelectRet::TimedOut => { - return Err(socket::timeout_error_msg( - vm, - "The handshake operation timed out".to_owned(), - )) - } - SelectRet::Closed => return Err(socket_closed_error(vm)), - SelectRet::Nonblocking => {} - _ => { - if needs.is_some() { - continue; - } - } - } - return Err(convert_ssl_error(vm, err)); - } - } - - #[pymethod] - fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { - let mut stream = self.stream.write(); - let data = data.borrow_buf(); - let data = &*data; - let timeout = stream.get_ref().timeout_deadline(); - let state = stream.get_ref().select(SslNeeds::Write, &timeout); - match state { - SelectRet::TimedOut => { - return Err(socket::timeout_error_msg( - vm, - "The write operation timed out".to_owned(), - )) - } - SelectRet::Closed => return Err(socket_closed_error(vm)), - _ => {} - } - loop { - let err = match stream.ssl_write(data) { - Ok(len) => return Ok(len), - Err(e) => e, - }; - let (needs, state) = stream.get_ref().socket_needs(&err, &timeout); - match state { - SelectRet::TimedOut => { - return Err(socket::timeout_error_msg( - vm, - "The write operation timed out".to_owned(), - )) - } - SelectRet::Closed => return Err(socket_closed_error(vm)), - SelectRet::Nonblocking => {} - _ => { - if needs.is_some() { - continue; - } - } - } - return Err(convert_ssl_error(vm, err)); - } - } - - #[pymethod] - fn read( - &self, - n: usize, - buffer: OptionalArg<ArgMemoryBuffer>, - vm: &VirtualMachine, - ) -> PyResult { - let mut stream = self.stream.write(); - let mut inner_buffer = if let OptionalArg::Present(buffer) = &buffer { - Either::A(buffer.borrow_buf_mut()) - } else { - Either::B(vec![0u8; n]) - }; - let buf = match &mut inner_buffer { - Either::A(b) => &mut **b, - Either::B(b) => b.as_mut_slice(), - }; - let buf = match buf.get_mut(..n) { - Some(b) => b, - None => buf, - }; - let timeout = stream.get_ref().timeout_deadline(); - let count = loop { - let err = match stream.ssl_read(buf) { - Ok(count) => break count, - Err(e) => e, - }; - if err.code() == ssl::ErrorCode::ZERO_RETURN - && stream.get_shutdown() == ssl::ShutdownState::RECEIVED - { - break 0; - } - let (needs, state) = stream.get_ref().socket_needs(&err, &timeout); - match state { - SelectRet::TimedOut => { - return Err(socket::timeout_error_msg( - vm, - "The read operation timed out".to_owned(), - )) - } - SelectRet::Nonblocking => {} - _ => { - if needs.is_some() { - continue; - } - } - } - return Err(convert_ssl_error(vm, err)); - }; - let ret = match inner_buffer { - Either::A(_buf) => vm.ctx.new_int(count).into(), - Either::B(mut buf) => { - buf.truncate(n); - buf.shrink_to_fit(); - vm.ctx.new_bytes(buf).into() - } - }; - Ok(ret) - } - } - - #[track_caller] - fn convert_openssl_error(vm: &VirtualMachine, err: ErrorStack) -> PyBaseExceptionRef { - let cls = ssl_error(vm); - match err.errors().last() { - Some(e) => { - let caller = std::panic::Location::caller(); - let (file, line) = (caller.file(), caller.line()); - let file = file - .rsplit_once(&['/', '\\'][..]) - .map_or(file, |(_, basename)| basename); - // TODO: map the error codes to code names, e.g. "CERTIFICATE_VERIFY_FAILED", just requires a big hashmap/dict - let errstr = e.reason().unwrap_or("unknown error"); - let msg = if let Some(lib) = e.library() { - // add `library` attribute - let attr_name = vm.ctx.as_ref().intern_str("library"); - cls.set_attr(attr_name, vm.ctx.new_str(lib).into()); - format!("[{lib}] {errstr} ({file}:{line})") - } else { - format!("{errstr} ({file}:{line})") - }; - // add `reason` attribute - let attr_name = vm.ctx.as_ref().intern_str("reason"); - cls.set_attr(attr_name, vm.ctx.new_str(errstr).into()); - - let reason = sys::ERR_GET_REASON(e.code()); - vm.new_exception( - cls, - vec![vm.ctx.new_int(reason).into(), vm.ctx.new_str(msg).into()], - ) - } - None => vm.new_exception_empty(cls), - } - } - #[track_caller] - fn convert_ssl_error( - vm: &VirtualMachine, - e: impl std::borrow::Borrow<ssl::Error>, - ) -> PyBaseExceptionRef { - let e = e.borrow(); - let (cls, msg) = match e.code() { - ssl::ErrorCode::WANT_READ => ( - vm.class("_ssl", "SSLWantReadError"), - "The operation did not complete (read)", - ), - ssl::ErrorCode::WANT_WRITE => ( - vm.class("_ssl", "SSLWantWriteError"), - "The operation did not complete (write)", - ), - ssl::ErrorCode::SYSCALL => match e.io_error() { - Some(io_err) => return io_err.to_pyexception(vm), - None => ( - vm.class("_ssl", "SSLSyscallError"), - "EOF occurred in violation of protocol", - ), - }, - ssl::ErrorCode::SSL => match e.ssl_error() { - Some(e) => return convert_openssl_error(vm, e.clone()), - None => (ssl_error(vm), "A failure in the SSL library occurred"), - }, - _ => (ssl_error(vm), "A failure in the SSL library occurred"), - }; - vm.new_exception_msg(cls, msg.to_owned()) - } - - // SSL_FILETYPE_ASN1 part of _add_ca_certs in CPython - fn x509_stack_from_der(der: &[u8]) -> Result<Vec<X509>, ErrorStack> { - unsafe { - openssl::init(); - let bio = bio::MemBioSlice::new(der)?; - - let mut certs = vec![]; - loop { - let cert = sys::d2i_X509_bio(bio.as_ptr(), std::ptr::null_mut()); - if cert.is_null() { - break; - } - certs.push(X509::from_ptr(cert)); - } - - let err = sys::ERR_peek_last_error(); - - if certs.is_empty() { - // let msg = if filetype == sys::SSL_FILETYPE_PEM { - // "no start line: cadata does not contain a certificate" - // } else { - // "not enough data: cadata does not contain a certificate" - // }; - return Err(ErrorStack::get()); - } - if err != 0 { - return Err(ErrorStack::get()); - } - - Ok(certs) - } - } - - type CipherTuple = (&'static str, &'static str, i32); - - fn cipher_to_tuple(cipher: &ssl::SslCipherRef) -> CipherTuple { - (cipher.name(), cipher.version(), cipher.bits().secret) - } - - fn cert_to_py(vm: &VirtualMachine, cert: &X509Ref, binary: bool) -> PyResult { - let r = if binary { - let b = cert.to_der().map_err(|e| convert_openssl_error(vm, e))?; - vm.ctx.new_bytes(b).into() - } else { - let dict = vm.ctx.new_dict(); - - let name_to_py = |name: &x509::X509NameRef| -> PyResult { - let list = name - .entries() - .map(|entry| { - let txt = obj2txt(entry.object(), false).to_pyobject(vm); - let data = vm.ctx.new_str(entry.data().as_utf8()?.to_owned()); - Ok(vm.new_tuple(((txt, data),)).into()) - }) - .collect::<Result<_, _>>() - .map_err(|e| convert_openssl_error(vm, e))?; - Ok(vm.ctx.new_tuple(list).into()) - }; - - dict.set_item("subject", name_to_py(cert.subject_name())?, vm)?; - dict.set_item("issuer", name_to_py(cert.issuer_name())?, vm)?; - dict.set_item("version", vm.new_pyobj(cert.version()), vm)?; - - let serial_num = cert - .serial_number() - .to_bn() - .and_then(|bn| bn.to_hex_str()) - .map_err(|e| convert_openssl_error(vm, e))?; - dict.set_item( - "serialNumber", - vm.ctx.new_str(serial_num.to_owned()).into(), - vm, - )?; - - dict.set_item( - "notBefore", - vm.ctx.new_str(cert.not_before().to_string()).into(), - vm, - )?; - dict.set_item( - "notAfter", - vm.ctx.new_str(cert.not_after().to_string()).into(), - vm, - )?; - - #[allow(clippy::manual_map)] - if let Some(names) = cert.subject_alt_names() { - let san = names - .iter() - .filter_map(|gen_name| { - if let Some(email) = gen_name.email() { - Some(vm.new_tuple((ascii!("email"), email)).into()) - } else if let Some(dnsname) = gen_name.dnsname() { - Some(vm.new_tuple((ascii!("DNS"), dnsname)).into()) - } else if let Some(ip) = gen_name.ipaddress() { - Some( - vm.new_tuple(( - ascii!("IP Address"), - String::from_utf8_lossy(ip).into_owned(), - )) - .into(), - ) - } else { - // TODO: convert every type of general name: - // https://github.com/python/cpython/blob/3.6/Modules/_ssl.c#L1092-L1231 - None - } - }) - .collect(); - dict.set_item("subjectAltName", vm.ctx.new_tuple(san).into(), vm)?; - }; - - dict.into() - }; - Ok(r) - } - - #[pyfunction] - fn _test_decode_cert(path: FsPath, vm: &VirtualMachine) -> PyResult { - let path = path.to_path_buf(vm)?; - let pem = std::fs::read(path).map_err(|e| e.to_pyexception(vm))?; - let x509 = X509::from_pem(&pem).map_err(|e| convert_openssl_error(vm, e))?; - cert_to_py(vm, &x509, false) - } - - impl Read for SocketStream { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { - let mut socket: &PySocket = &self.0; - socket.read(buf) - } - } - - impl Write for SocketStream { - fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { - let mut socket: &PySocket = &self.0; - socket.write(buf) - } - fn flush(&mut self) -> std::io::Result<()> { - let mut socket: &PySocket = &self.0; - socket.flush() - } - } - - #[cfg(target_os = "android")] - mod android { - use super::convert_openssl_error; - use crate::vm::{builtins::PyBaseExceptionRef, VirtualMachine}; - use openssl::{ - ssl::SslContextBuilder, - x509::{store::X509StoreBuilder, X509}, - }; - use std::{ - fs::{read_dir, File}, - io::Read, - path::Path, - }; - - static CERT_DIR: &'static str = "/system/etc/security/cacerts"; - - pub(super) fn load_client_ca_list( - vm: &VirtualMachine, - b: &mut SslContextBuilder, - ) -> Result<(), PyBaseExceptionRef> { - let root = Path::new(CERT_DIR); - if !root.is_dir() { - return Err(vm.new_exception_msg( - vm.ctx.exceptions.file_not_found_error.to_owned(), - CERT_DIR.to_string(), - )); - } - - let mut combined_pem = String::new(); - let entries = read_dir(root) - .map_err(|err| vm.new_os_error(format!("read cert root: {}", err)))?; - for entry in entries { - let entry = - entry.map_err(|err| vm.new_os_error(format!("iter cert root: {}", err)))?; - - let path = entry.path(); - if !path.is_file() { - continue; - } - - File::open(&path) - .and_then(|mut file| file.read_to_string(&mut combined_pem)) - .map_err(|err| { - vm.new_os_error(format!("open cert file {}: {}", path.display(), err)) - })?; - - combined_pem.push('\n'); - } - - let mut store_b = - X509StoreBuilder::new().map_err(|err| convert_openssl_error(vm, err))?; - let x509_vec = X509::stack_from_pem(combined_pem.as_bytes()) - .map_err(|err| convert_openssl_error(vm, err))?; - for x509 in x509_vec { - store_b - .add_cert(x509) - .map_err(|err| convert_openssl_error(vm, err))?; - } - b.set_cert_store(store_b.build()); - - Ok(()) - } - } -} - -#[cfg(not(ossl101))] -#[pymodule(sub)] -mod ossl101 {} - -#[cfg(not(ossl111))] -#[pymodule(sub)] -mod ossl111 {} - -#[cfg(not(windows))] -#[pymodule(sub)] -mod windows {} - -#[allow(non_upper_case_globals)] -#[cfg(ossl101)] -#[pymodule(sub)] -mod ossl101 { - #[pyattr] - use openssl_sys::{ - SSL_OP_NO_TLSv1_1 as OP_NO_TLSv1_1, SSL_OP_NO_TLSv1_2 as OP_NO_TLSv1_2, - SSL_OP_NO_COMPRESSION as OP_NO_COMPRESSION, - }; -} - -#[allow(non_upper_case_globals)] -#[cfg(ossl111)] -#[pymodule(sub)] -mod ossl111 { - #[pyattr] - use openssl_sys::SSL_OP_NO_TLSv1_3 as OP_NO_TLSv1_3; -} - -#[cfg(windows)] -#[pymodule(sub)] -mod windows { - use crate::{ - common::ascii, - vm::{ - builtins::{PyFrozenSet, PyStrRef}, - convert::ToPyException, - PyObjectRef, PyPayload, PyResult, VirtualMachine, - }, - }; - - #[pyfunction] - fn enum_certificates(store_name: PyStrRef, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - use schannel::{cert_context::ValidUses, cert_store::CertStore, RawPointer}; - use windows_sys::Win32::Security::Cryptography; - - // TODO: check every store for it, not just 2 of them: - // https://github.com/python/cpython/blob/3.8/Modules/_ssl.c#L5603-L5610 - let open_fns = [CertStore::open_current_user, CertStore::open_local_machine]; - let stores = open_fns - .iter() - .filter_map(|open| open(store_name.as_str()).ok()) - .collect::<Vec<_>>(); - let certs = stores.iter().flat_map(|s| s.certs()).map(|c| { - let cert = vm.ctx.new_bytes(c.to_der().to_owned()); - let enc_type = unsafe { - let ptr = c.as_ptr() as *const Cryptography::CERT_CONTEXT; - (*ptr).dwCertEncodingType - }; - let enc_type = match enc_type { - Cryptography::X509_ASN_ENCODING => vm.new_pyobj(ascii!("x509_asn")), - Cryptography::PKCS_7_ASN_ENCODING => vm.new_pyobj(ascii!("pkcs_7_asn")), - other => vm.new_pyobj(other), - }; - let usage: PyObjectRef = match c.valid_uses()? { - ValidUses::All => vm.ctx.new_bool(true).into(), - ValidUses::Oids(oids) => PyFrozenSet::from_iter( - vm, - oids.into_iter().map(|oid| vm.ctx.new_str(oid).into()), - ) - .unwrap() - .into_ref(&vm.ctx) - .into(), - }; - Ok(vm.new_tuple((cert, enc_type, usage)).into()) - }); - let certs = certs - .collect::<Result<Vec<_>, _>>() - .map_err(|e: std::io::Error| e.to_pyexception(vm))?; - Ok(certs) - } -} - -mod bio { - //! based off rust-openssl's private `bio` module - - use libc::c_int; - use openssl::error::ErrorStack; - use openssl_sys as sys; - use std::marker::PhantomData; - - pub struct MemBioSlice<'a>(*mut sys::BIO, PhantomData<&'a [u8]>); - - impl<'a> Drop for MemBioSlice<'a> { - fn drop(&mut self) { - unsafe { - sys::BIO_free_all(self.0); - } - } - } - - impl<'a> MemBioSlice<'a> { - pub fn new(buf: &'a [u8]) -> Result<MemBioSlice<'a>, ErrorStack> { - openssl::init(); - - assert!(buf.len() <= c_int::max_value() as usize); - let bio = unsafe { sys::BIO_new_mem_buf(buf.as_ptr() as *const _, buf.len() as c_int) }; - if bio.is_null() { - return Err(ErrorStack::get()); - } - - Ok(MemBioSlice(bio, PhantomData)) - } - - pub fn as_ptr(&self) -> *mut sys::BIO { - self.0 - } - } -} diff --git a/stdlib/src/unicodedata.rs b/stdlib/src/unicodedata.rs deleted file mode 100644 index 56105ceca66..00000000000 --- a/stdlib/src/unicodedata.rs +++ /dev/null @@ -1,218 +0,0 @@ -/* Access to the unicode database. - See also: https://docs.python.org/3/library/unicodedata.html -*/ - -// spell-checker:ignore nfkc unistr unidata - -use crate::vm::{ - builtins::PyModule, builtins::PyStr, convert::TryFromBorrowedObject, PyObject, PyObjectRef, - PyPayload, PyRef, PyResult, VirtualMachine, -}; - -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = unicodedata::make_module(vm); - - let ucd: PyObjectRef = unicodedata::Ucd::new(unic_ucd_age::UNICODE_VERSION) - .into_ref(&vm.ctx) - .into(); - - for attr in [ - "category", - "lookup", - "name", - "bidirectional", - "east_asian_width", - "normalize", - ] - .into_iter() - { - crate::vm::extend_module!(vm, &module, { - attr => ucd.get_attr(attr, vm).unwrap(), - }); - } - - module -} - -enum NormalizeForm { - Nfc, - Nfkc, - Nfd, - Nfkd, -} - -impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - obj.try_value_with( - |form: &PyStr| { - Ok(match form.as_str() { - "NFC" => NormalizeForm::Nfc, - "NFKC" => NormalizeForm::Nfkc, - "NFD" => NormalizeForm::Nfd, - "NFKD" => NormalizeForm::Nfkd, - _ => return Err(vm.new_value_error("invalid normalization form".to_owned())), - }) - }, - vm, - ) - } -} - -#[pymodule] -mod unicodedata { - use crate::vm::{ - builtins::PyStrRef, function::OptionalArg, PyObjectRef, PyPayload, PyRef, PyResult, - VirtualMachine, - }; - use itertools::Itertools; - use ucd::{Codepoint, EastAsianWidth}; - use unic_char_property::EnumeratedCharProperty; - use unic_normal::StrNormalForm; - use unic_ucd_age::{Age, UnicodeVersion, UNICODE_VERSION}; - use unic_ucd_bidi::BidiClass; - use unic_ucd_category::GeneralCategory; - - #[pyattr] - #[pyclass(name = "UCD")] - #[derive(Debug, PyPayload)] - pub(super) struct Ucd { - unic_version: UnicodeVersion, - } - - impl Ucd { - pub fn new(unic_version: UnicodeVersion) -> Self { - Self { unic_version } - } - - fn check_age(&self, c: char) -> bool { - Age::of(c).map_or(false, |age| age.actual() <= self.unic_version) - } - - fn extract_char(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<Option<char>> { - let c = character.as_str().chars().exactly_one().map_err(|_| { - vm.new_type_error("argument must be an unicode character, not str".to_owned()) - })?; - - Ok(self.check_age(c).then_some(c)) - } - } - - #[pyclass] - impl Ucd { - #[pymethod] - fn category(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { - Ok(self - .extract_char(character, vm)? - .map_or(GeneralCategory::Unassigned, GeneralCategory::of) - .abbr_name() - .to_owned()) - } - - #[pymethod] - fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { - if let Some(character) = unicode_names2::character(name.as_str()) { - if self.check_age(character) { - return Ok(character.to_string()); - } - } - Err(vm.new_lookup_error(format!("undefined character name '{name}'"))) - } - - #[pymethod] - fn name( - &self, - character: PyStrRef, - default: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let c = self.extract_char(character, vm)?; - - if let Some(c) = c { - if self.check_age(c) { - if let Some(name) = unicode_names2::name(c) { - return Ok(vm.ctx.new_str(name.to_string()).into()); - } - } - } - default.ok_or_else(|| vm.new_value_error("character name not found!".to_owned())) - } - - #[pymethod] - fn bidirectional( - &self, - character: PyStrRef, - vm: &VirtualMachine, - ) -> PyResult<&'static str> { - let bidi = match self.extract_char(character, vm)? { - Some(c) => BidiClass::of(c).abbr_name(), - None => "", - }; - Ok(bidi) - } - - /// NOTE: This function uses 9.0.0 database instead of 3.2.0 - #[pymethod] - fn east_asian_width( - &self, - character: PyStrRef, - vm: &VirtualMachine, - ) -> PyResult<&'static str> { - Ok(self - .extract_char(character, vm)? - .map_or(EastAsianWidth::Neutral, |c| c.east_asian_width()) - .abbr_name()) - } - - #[pymethod] - fn normalize(&self, form: super::NormalizeForm, unistr: PyStrRef) -> PyResult<String> { - use super::NormalizeForm::*; - let text = unistr.as_str(); - let normalized_text = match form { - Nfc => text.nfc().collect::<String>(), - Nfkc => text.nfkc().collect::<String>(), - Nfd => text.nfd().collect::<String>(), - Nfkd => text.nfkd().collect::<String>(), - }; - Ok(normalized_text) - } - - #[pygetset] - fn unidata_version(&self) -> String { - self.unic_version.to_string() - } - } - - trait EastAsianWidthAbbrName { - fn abbr_name(&self) -> &'static str; - } - - impl EastAsianWidthAbbrName for EastAsianWidth { - fn abbr_name(&self) -> &'static str { - match self { - EastAsianWidth::Narrow => "Na", - EastAsianWidth::Wide => "W", - EastAsianWidth::Neutral => "N", - EastAsianWidth::Ambiguous => "A", - EastAsianWidth::FullWidth => "F", - EastAsianWidth::HalfWidth => "H", - } - } - } - - #[pyattr] - fn ucd_3_2_0(vm: &VirtualMachine) -> PyRef<Ucd> { - Ucd { - unic_version: UnicodeVersion { - major: 3, - minor: 2, - micro: 0, - }, - } - .into_ref(&vm.ctx) - } - - #[pyattr] - fn unidata_version(_vm: &VirtualMachine) -> String { - UNICODE_VERSION.to_string() - } -} diff --git a/stdlib/src/uuid.rs b/stdlib/src/uuid.rs deleted file mode 100644 index e5434da0e1b..00000000000 --- a/stdlib/src/uuid.rs +++ /dev/null @@ -1,46 +0,0 @@ -pub(crate) use _uuid::make_module; - -#[pymodule] -mod _uuid { - use crate::{builtins::PyNone, vm::VirtualMachine}; - use mac_address::get_mac_address; - use once_cell::sync::OnceCell; - use rand::Rng; - use std::time::{Duration, SystemTime}; - use uuid::{ - v1::{Context, Timestamp}, - Uuid, - }; - - fn get_node_id() -> [u8; 6] { - match get_mac_address() { - Ok(Some(_ma)) => get_mac_address().unwrap().unwrap().bytes(), - _ => rand::thread_rng().gen::<[u8; 6]>(), - } - } - - pub fn now_unix_duration() -> Duration { - use std::time::UNIX_EPOCH; - - let now = SystemTime::now(); - now.duration_since(UNIX_EPOCH) - .expect("SystemTime before UNIX EPOCH!") - } - - #[pyfunction] - fn generate_time_safe() -> (Vec<u8>, PyNone) { - static CONTEXT: Context = Context::new(0); - let now = now_unix_duration(); - let ts = Timestamp::from_unix(&CONTEXT, now.as_secs(), now.subsec_nanos()); - - static NODE_ID: OnceCell<[u8; 6]> = OnceCell::new(); - let unique_node_id = NODE_ID.get_or_init(get_node_id); - - (Uuid::new_v1(ts, unique_node_id).as_bytes().to_vec(), PyNone) - } - - #[pyattr] - fn has_uuid_generate_time_safe(_vm: &VirtualMachine) -> u32 { - 0 - } -} diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs deleted file mode 100644 index 37ee1c83f7f..00000000000 --- a/stdlib/src/zlib.rs +++ /dev/null @@ -1,590 +0,0 @@ -// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz - -pub(crate) use zlib::make_module; - -#[pymodule] -mod zlib { - use crate::vm::{ - builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyIntRef, PyTypeRef}, - common::lock::PyMutex, - convert::TryFromBorrowedObject, - function::{ArgBytesLike, ArgPrimitiveIndex, ArgSize, OptionalArg}, - PyObject, PyPayload, PyResult, VirtualMachine, - }; - use adler32::RollingAdler32 as Adler32; - use crossbeam_utils::atomic::AtomicCell; - use flate2::{ - write::ZlibEncoder, Compress, Compression, Decompress, FlushCompress, FlushDecompress, - Status, - }; - use std::io::Write; - - #[cfg(not(feature = "zlib"))] - mod constants { - pub const Z_NO_COMPRESSION: i32 = 0; - pub const Z_BEST_COMPRESSION: i32 = 9; - pub const Z_BEST_SPEED: i32 = 1; - pub const Z_DEFAULT_COMPRESSION: i32 = -1; - pub const Z_NO_FLUSH: i32 = 0; - pub const Z_PARTIAL_FLUSH: i32 = 1; - pub const Z_SYNC_FLUSH: i32 = 2; - pub const Z_FULL_FLUSH: i32 = 3; - // not sure what the value here means, but it's the only compression method zlibmodule - // supports, so it doesn't really matter - pub const Z_DEFLATED: i32 = 8; - } - #[cfg(feature = "zlib")] - use libz_sys as constants; - - #[pyattr] - use constants::{ - Z_BEST_COMPRESSION, Z_BEST_SPEED, Z_DEFAULT_COMPRESSION, Z_DEFLATED as DEFLATED, - Z_FULL_FLUSH, Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, - }; - - #[cfg(feature = "zlib")] - #[pyattr] - use libz_sys::{ - Z_BLOCK, Z_DEFAULT_STRATEGY, Z_FILTERED, Z_FINISH, Z_FIXED, Z_HUFFMAN_ONLY, Z_RLE, Z_TREES, - }; - - // copied from zlibmodule.c (commit 530f506ac91338) - #[pyattr] - const MAX_WBITS: i8 = 15; - #[pyattr] - const DEF_BUF_SIZE: usize = 16 * 1024; - #[pyattr] - const DEF_MEM_LEVEL: u8 = 8; - - #[pyattr(once)] - fn error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.new_exception_type( - "zlib", - "error", - Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), - ) - } - - #[pyfunction] - fn adler32(data: ArgBytesLike, begin_state: OptionalArg<PyIntRef>) -> u32 { - data.with_ref(|data| { - let begin_state = begin_state.map_or(1, |i| i.as_u32_mask()); - - let mut hasher = Adler32::from_value(begin_state); - hasher.update_buffer(data); - hasher.hash() - }) - } - - #[pyfunction] - fn crc32(data: ArgBytesLike, begin_state: OptionalArg<PyIntRef>) -> u32 { - crate::binascii::crc32(data, begin_state) - } - - #[derive(FromArgs)] - struct PyFuncCompressArgs { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg(any, default = "Level::new(Z_DEFAULT_COMPRESSION)")] - level: Level, - #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] - wbits: ArgPrimitiveIndex<i8>, - } - - /// Returns a bytes object containing compressed data. - #[pyfunction] - fn compress(args: PyFuncCompressArgs, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - let PyFuncCompressArgs { - data, - level, - ref wbits, - } = args; - let level = level.ok_or_else(|| new_zlib_error("Bad compression level", vm))?; - - let encoded_bytes = if args.wbits.value == MAX_WBITS { - let mut encoder = ZlibEncoder::new(Vec::new(), level); - data.with_ref(|input_bytes| encoder.write_all(input_bytes).unwrap()); - encoder.finish().unwrap() - } else { - let mut inner = CompressInner::new(InitOptions::new(wbits.value, vm)?.compress(level)); - data.with_ref(|input_bytes| inner.compress(input_bytes, vm))?; - inner.flush(vm)? - }; - Ok(vm.ctx.new_bytes(encoded_bytes)) - } - - enum InitOptions { - Standard { - header: bool, - // [De]Compress::new_with_window_bits is only enabled for zlib; miniz_oxide doesn't - // support wbits (yet?) - #[cfg(feature = "zlib")] - wbits: u8, - }, - #[cfg(feature = "zlib")] - Gzip { wbits: u8 }, - } - - impl InitOptions { - fn new(wbits: i8, vm: &VirtualMachine) -> PyResult<InitOptions> { - let header = wbits > 0; - let wbits = wbits.unsigned_abs(); - match wbits { - 9..=15 => Ok(InitOptions::Standard { - header, - #[cfg(feature = "zlib")] - wbits, - }), - #[cfg(feature = "zlib")] - 25..=31 => Ok(InitOptions::Gzip { wbits: wbits - 16 }), - _ => Err(vm.new_value_error("Invalid initialization option".to_owned())), - } - } - - fn decompress(self) -> Decompress { - match self { - #[cfg(not(feature = "zlib"))] - Self::Standard { header } => Decompress::new(header), - #[cfg(feature = "zlib")] - Self::Standard { header, wbits } => Decompress::new_with_window_bits(header, wbits), - #[cfg(feature = "zlib")] - Self::Gzip { wbits } => Decompress::new_gzip(wbits), - } - } - fn compress(self, level: Compression) -> Compress { - match self { - #[cfg(not(feature = "zlib"))] - Self::Standard { header } => Compress::new(level, header), - #[cfg(feature = "zlib")] - Self::Standard { header, wbits } => { - Compress::new_with_window_bits(level, header, wbits) - } - #[cfg(feature = "zlib")] - Self::Gzip { wbits } => Compress::new_gzip(level, wbits), - } - } - } - - fn _decompress( - mut data: &[u8], - d: &mut Decompress, - bufsize: usize, - max_length: Option<usize>, - is_flush: bool, - vm: &VirtualMachine, - ) -> PyResult<(Vec<u8>, bool)> { - if data.is_empty() { - return Ok((Vec::new(), true)); - } - let mut buf = Vec::new(); - - loop { - let final_chunk = data.len() <= CHUNKSIZE; - let chunk = if final_chunk { - data - } else { - &data[..CHUNKSIZE] - }; - // if this is the final chunk, finish it - let flush = if is_flush { - if final_chunk { - FlushDecompress::Finish - } else { - FlushDecompress::None - } - } else { - FlushDecompress::Sync - }; - loop { - let additional = if let Some(max_length) = max_length { - std::cmp::min(bufsize, max_length - buf.capacity()) - } else { - bufsize - }; - if additional == 0 { - return Ok((buf, false)); - } - - buf.reserve_exact(additional); - let prev_in = d.total_in(); - let status = d - .decompress_vec(chunk, &mut buf, flush) - .map_err(|_| new_zlib_error("invalid input data", vm))?; - let consumed = d.total_in() - prev_in; - data = &data[consumed as usize..]; - let stream_end = status == Status::StreamEnd; - if stream_end || data.is_empty() { - // we've reached the end of the stream, we're done - buf.shrink_to_fit(); - return Ok((buf, stream_end)); - } else if !chunk.is_empty() && consumed == 0 { - // we're gonna need a bigger buffer - continue; - } else { - // next chunk - break; - } - } - } - } - - #[derive(FromArgs)] - struct PyFuncDecompressArgs { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] - wbits: ArgPrimitiveIndex<i8>, - #[pyarg(any, default = "ArgPrimitiveIndex { value: DEF_BUF_SIZE }")] - bufsize: ArgPrimitiveIndex<usize>, - } - - /// Returns a bytes object containing the uncompressed data. - #[pyfunction] - fn decompress(args: PyFuncDecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let PyFuncDecompressArgs { - data, - wbits, - bufsize, - } = args; - data.with_ref(|data| { - let mut d = InitOptions::new(wbits.value, vm)?.decompress(); - let (buf, stream_end) = _decompress(data, &mut d, bufsize.value, None, false, vm)?; - if !stream_end { - return Err(new_zlib_error( - "Error -5 while decompressing data: incomplete or truncated stream", - vm, - )); - } - Ok(buf) - }) - } - - #[derive(FromArgs)] - struct DecompressobjArgs { - #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] - wbits: ArgPrimitiveIndex<i8>, - #[cfg(feature = "zlib")] - #[pyarg(any, optional)] - _zdict: OptionalArg<ArgBytesLike>, - } - - #[pyfunction] - fn decompressobj(args: DecompressobjArgs, vm: &VirtualMachine) -> PyResult<PyDecompress> { - #[allow(unused_mut)] - let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); - #[cfg(feature = "zlib")] - if let OptionalArg::Present(_dict) = args._zdict { - // FIXME: always fails - // dict.with_ref(|d| decompress.set_dictionary(d)); - } - Ok(PyDecompress { - decompress: PyMutex::new(decompress), - eof: AtomicCell::new(false), - unused_data: PyMutex::new(PyBytes::from(vec![]).into_ref(&vm.ctx)), - unconsumed_tail: PyMutex::new(PyBytes::from(vec![]).into_ref(&vm.ctx)), - }) - } - #[pyattr] - #[pyclass(name = "Decompress")] - #[derive(Debug, PyPayload)] - struct PyDecompress { - decompress: PyMutex<Decompress>, - eof: AtomicCell<bool>, - unused_data: PyMutex<PyBytesRef>, - unconsumed_tail: PyMutex<PyBytesRef>, - } - #[pyclass] - impl PyDecompress { - #[pygetset] - fn eof(&self) -> bool { - self.eof.load() - } - #[pygetset] - fn unused_data(&self) -> PyBytesRef { - self.unused_data.lock().clone() - } - #[pygetset] - fn unconsumed_tail(&self) -> PyBytesRef { - self.unconsumed_tail.lock().clone() - } - - fn save_unused_input( - &self, - d: &Decompress, - data: &[u8], - stream_end: bool, - orig_in: u64, - vm: &VirtualMachine, - ) { - let leftover = &data[(d.total_in() - orig_in) as usize..]; - - if stream_end && !leftover.is_empty() { - let mut unused_data = self.unused_data.lock(); - let unused: Vec<_> = unused_data - .as_bytes() - .iter() - .chain(leftover) - .copied() - .collect(); - *unused_data = vm.ctx.new_pyref(unused); - } - } - - #[pymethod] - fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let max_length = args.max_length.value; - let max_length = (max_length != 0).then_some(max_length); - let data = args.data.borrow_buf(); - let data = &*data; - - let mut d = self.decompress.lock(); - let orig_in = d.total_in(); - - let (ret, stream_end) = - match _decompress(data, &mut d, DEF_BUF_SIZE, max_length, false, vm) { - Ok((buf, true)) => { - self.eof.store(true); - (Ok(buf), true) - } - Ok((buf, false)) => (Ok(buf), false), - Err(err) => (Err(err), false), - }; - self.save_unused_input(&d, data, stream_end, orig_in, vm); - - let leftover = if stream_end { - b"" - } else { - &data[(d.total_in() - orig_in) as usize..] - }; - - let mut unconsumed_tail = self.unconsumed_tail.lock(); - if !leftover.is_empty() || !unconsumed_tail.is_empty() { - *unconsumed_tail = PyBytes::from(leftover.to_owned()).into_ref(&vm.ctx); - } - - ret - } - - #[pymethod] - fn flush(&self, length: OptionalArg<ArgSize>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let length = match length { - OptionalArg::Present(l) => { - let l: isize = l.into(); - if l <= 0 { - return Err( - vm.new_value_error("length must be greater than zero".to_owned()) - ); - } else { - l as usize - } - } - OptionalArg::Missing => DEF_BUF_SIZE, - }; - - let mut data = self.unconsumed_tail.lock(); - let mut d = self.decompress.lock(); - - let orig_in = d.total_in(); - - let (ret, stream_end) = match _decompress(&data, &mut d, length, None, true, vm) { - Ok((buf, stream_end)) => (Ok(buf), stream_end), - Err(err) => (Err(err), false), - }; - self.save_unused_input(&d, &data, stream_end, orig_in, vm); - - *data = PyBytes::from(Vec::new()).into_ref(&vm.ctx); - - // TODO: drop the inner decompressor, somehow - // if stream_end { - // - // } - ret - } - } - - #[derive(FromArgs)] - struct DecompressArgs { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg( - any, - default = "rustpython_vm::function::ArgPrimitiveIndex { value: 0 }" - )] - max_length: ArgPrimitiveIndex<usize>, - } - - #[derive(FromArgs)] - #[allow(dead_code)] // FIXME: use args - struct CompressobjArgs { - #[pyarg(any, default = "Level::new(Z_DEFAULT_COMPRESSION)")] - level: Level, - // only DEFLATED is valid right now, it's w/e - #[pyarg(any, default = "DEFLATED")] - _method: i32, - #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] - wbits: ArgPrimitiveIndex<i8>, - #[pyarg(any, name = "_memLevel", default = "DEF_MEM_LEVEL")] - _mem_level: u8, - #[cfg(feature = "zlib")] - #[pyarg(any, default = "Z_DEFAULT_STRATEGY")] - _strategy: i32, - #[cfg(feature = "zlib")] - #[pyarg(any, optional)] - zdict: Option<ArgBytesLike>, - } - - #[pyfunction] - fn compressobj(args: CompressobjArgs, vm: &VirtualMachine) -> PyResult<PyCompress> { - let CompressobjArgs { - level, - wbits, - #[cfg(feature = "zlib")] - zdict, - .. - } = args; - let level = - level.ok_or_else(|| vm.new_value_error("invalid initialization option".to_owned()))?; - #[allow(unused_mut)] - let mut compress = InitOptions::new(wbits.value, vm)?.compress(level); - #[cfg(feature = "zlib")] - if let Some(zdict) = zdict { - zdict.with_ref(|zdict| compress.set_dictionary(zdict).unwrap()); - } - Ok(PyCompress { - inner: PyMutex::new(CompressInner::new(compress)), - }) - } - - #[derive(Debug)] - struct CompressInner { - compress: Compress, - unconsumed: Vec<u8>, - } - - #[pyattr] - #[pyclass(name = "Compress")] - #[derive(Debug, PyPayload)] - struct PyCompress { - inner: PyMutex<CompressInner>, - } - - #[pyclass] - impl PyCompress { - #[pymethod] - fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let mut inner = self.inner.lock(); - data.with_ref(|b| inner.compress(b, vm)) - } - - // TODO: mode argument isn't used - #[pymethod] - fn flush(&self, _mode: OptionalArg<i32>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - self.inner.lock().flush(vm) - } - - // TODO: This is an optional feature of Compress - // #[pymethod] - // #[pymethod(magic)] - // #[pymethod(name = "__deepcopy__")] - // fn copy(&self) -> Self { - // todo!("<flate2::Compress as Clone>") - // } - } - - const CHUNKSIZE: usize = u32::MAX as usize; - - impl CompressInner { - fn new(compress: Compress) -> Self { - Self { - compress, - unconsumed: Vec::new(), - } - } - fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let orig_in = self.compress.total_in() as usize; - let mut cur_in = 0; - let unconsumed = std::mem::take(&mut self.unconsumed); - let mut buf = Vec::new(); - - 'outer: for chunk in unconsumed.chunks(CHUNKSIZE).chain(data.chunks(CHUNKSIZE)) { - while cur_in < chunk.len() { - buf.reserve(DEF_BUF_SIZE); - let status = self - .compress - .compress_vec(&chunk[cur_in..], &mut buf, FlushCompress::None) - .map_err(|_| { - self.unconsumed.extend_from_slice(&data[cur_in..]); - new_zlib_error("error while compressing", vm) - })?; - cur_in = (self.compress.total_in() as usize) - orig_in; - match status { - Status::Ok => continue, - Status::StreamEnd => break 'outer, - _ => break, - } - } - } - self.unconsumed.extend_from_slice(&data[cur_in..]); - - buf.shrink_to_fit(); - Ok(buf) - } - - // TODO: flush mode (FlushDecompress) parameter - fn flush(&mut self, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let data = std::mem::take(&mut self.unconsumed); - let mut data_it = data.chunks(CHUNKSIZE); - let mut buf = Vec::new(); - - loop { - let chunk = data_it.next().unwrap_or(&[]); - if buf.len() == buf.capacity() { - buf.reserve(DEF_BUF_SIZE); - } - let status = self - .compress - .compress_vec(chunk, &mut buf, FlushCompress::Finish) - .map_err(|_| new_zlib_error("error while compressing", vm))?; - match status { - Status::StreamEnd => break, - _ => continue, - } - } - - buf.shrink_to_fit(); - Ok(buf) - } - } - - fn new_zlib_error(message: &str, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception_msg(vm.class("zlib", "error"), message.to_owned()) - } - - struct Level(Option<flate2::Compression>); - - impl Level { - fn new(level: i32) -> Self { - let compression = match level { - Z_DEFAULT_COMPRESSION => Compression::default(), - valid_level @ Z_NO_COMPRESSION..=Z_BEST_COMPRESSION => { - Compression::new(valid_level as u32) - } - _ => return Self(None), - }; - Self(Some(compression)) - } - fn ok_or_else( - self, - f: impl FnOnce() -> PyBaseExceptionRef, - ) -> PyResult<flate2::Compression> { - self.0.ok_or_else(f) - } - } - - impl<'a> TryFromBorrowedObject<'a> for Level { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - let int: i32 = obj.try_index(vm)?.try_to_primitive(vm)?; - Ok(Self::new(int)) - } - } -} diff --git a/vm/.gitignore b/vm/.gitignore deleted file mode 100644 index 4931874f785..00000000000 --- a/vm/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -venv/ -tests/*.bytecode -Cargo.lock -python_compiler/target -target/ diff --git a/vm/Cargo.toml b/vm/Cargo.toml deleted file mode 100644 index f061f54a85e..00000000000 --- a/vm/Cargo.toml +++ /dev/null @@ -1,151 +0,0 @@ -[package] -name = "rustpython-vm" -version = "0.3.0" -description = "RustPython virtual machine." -authors = ["RustPython Team"] -repository = "https://github.com/RustPython/RustPython" -license = "MIT" -edition = "2021" -include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"] - -[features] -default = ["compiler"] -importlib = [] -encodings = ["importlib"] -vm-tracing-logging = [] -flame-it = ["flame", "flamer"] -freeze-stdlib = ["encodings"] -jit = ["rustpython-jit"] -threading = ["rustpython-common/threading"] -compiler = ["parser", "codegen", "rustpython-compiler"] -ast = ["rustpython-ast"] -codegen = ["rustpython-codegen", "ast"] -parser = ["rustpython-parser", "ast"] -serde = ["dep:serde"] - -[dependencies] -rustpython-compiler = { workspace = true, optional = true } -rustpython-codegen = { workspace = true, optional = true } -rustpython-common = { workspace = true } -rustpython-derive = { workspace = true } -rustpython-jit = { workspace = true, optional = true } - -rustpython-ast = { workspace = true, optional = true } -rustpython-parser = { workspace = true, optional = true } -rustpython-compiler-core = { workspace = true } -rustpython-parser-core = { workspace = true } -rustpython-literal = { workspace = true } -rustpython-format = { workspace = true } - -ascii = { workspace = true } -ahash = { workspace = true } -atty = { workspace = true } -bitflags = { workspace = true } -bstr = { workspace = true } -cfg-if = { workspace = true } -crossbeam-utils = { workspace = true } -chrono = { workspace = true, features = ["wasmbind"] } -flame = { workspace = true, optional = true } -hex = { workspace = true } -indexmap = { workspace = true } -itertools = { workspace = true } -is-macro = { workspace = true } -libc = { workspace = true } -log = { workspace = true } -nix = { workspace = true } -malachite-bigint = { workspace = true } -num-complex = { workspace = true } -num-integer = { workspace = true } -num-traits = { workspace = true } -num_enum = { workspace = true } -once_cell = { workspace = true } -parking_lot = { workspace = true } -paste = { workspace = true } -rand = { workspace = true } -serde = { workspace = true, optional = true } -static_assertions = { workspace = true } -thiserror = { workspace = true } -thread_local = { workspace = true } - -caseless = "0.2.1" -getrandom = { version = "0.2.6", features = ["js"] } -flamer = { version = "0.4", optional = true } -half = "1.8.2" -memchr = "2.4.1" -memoffset = "0.6.5" -optional = "0.5.0" -result-like = "0.4.5" -timsort = "0.1.2" - -# RustPython crates implementing functionality based on CPython -sre-engine = "0.4.1" -# to work on sre-engine locally or git version -# sre-engine = { git = "https://github.com/RustPython/sre-engine", rev = "refs/pull/14/head" } -# sre-engine = { git = "https://github.com/RustPython/sre-engine" } -# sre-engine = { path = "../../sre-engine" } - -## unicode stuff -unicode_names2 = { workspace = true } -# TODO: use unic for this; needed for title case: -# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 -unicode-casing = "0.1.0" -# update version all at the same time -unic-ucd-bidi = "0.9.0" -unic-ucd-category = "0.9.0" -unic-ucd-ident = "0.9.0" - -[target.'cfg(unix)'.dependencies] -exitcode = "1.1.2" -uname = "0.1.1" -strum = "0.24.0" -strum_macros = "0.24.0" - -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -rustyline = { workspace = true } -which = "4.2.5" - -[target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] -num_cpus = "1.13.1" - -[target.'cfg(windows)'.dependencies] -schannel = { workspace = true } -widestring = { workspace = true } -winreg = "0.10.1" - -[target.'cfg(windows)'.dependencies.windows] -version = "0.52.0" -features = [ - "Win32_Foundation", - "Win32_System_LibraryLoader", - "Win32_System_Threading", - "Win32_UI_Shell", -] - -[target.'cfg(windows)'.dependencies.windows-sys] -version = "0.52.0" -features = [ - "Win32_Foundation", - "Win32_Networking_WinSock", - "Win32_Security", - "Win32_Storage_FileSystem", - "Win32_System_Console", - "Win32_System_Diagnostics_Debug", - "Win32_System_LibraryLoader", - "Win32_System_Memory", - "Win32_System_Performance", - "Win32_System_Pipes", - "Win32_System_Registry", - "Win32_System_SystemInformation", - "Win32_System_SystemServices", - "Win32_System_Threading", - "Win32_UI_Shell", - "Win32_UI_WindowsAndMessaging", -] - -[target.'cfg(target_arch = "wasm32")'.dependencies] -wasm-bindgen = "0.2.80" - -[build-dependencies] -glob = { workspace = true } -itertools = { workspace = true } -rustc_version = "0.4.0" diff --git a/vm/Lib/core_modules/codecs.py b/vm/Lib/core_modules/codecs.py deleted file mode 120000 index db3231198d9..00000000000 --- a/vm/Lib/core_modules/codecs.py +++ /dev/null @@ -1 +0,0 @@ -../../../Lib/codecs.py \ No newline at end of file diff --git a/vm/Lib/core_modules/copyreg.py b/vm/Lib/core_modules/copyreg.py deleted file mode 120000 index 4ac7f40c43d..00000000000 --- a/vm/Lib/core_modules/copyreg.py +++ /dev/null @@ -1 +0,0 @@ -../../../Lib/copyreg.py \ No newline at end of file diff --git a/vm/Lib/core_modules/encodings_utf_8.py b/vm/Lib/core_modules/encodings_utf_8.py deleted file mode 120000 index a8f34066264..00000000000 --- a/vm/Lib/core_modules/encodings_utf_8.py +++ /dev/null @@ -1 +0,0 @@ -../../../Lib/encodings/utf_8.py \ No newline at end of file diff --git a/vm/Lib/python_builtins/__hello__.py b/vm/Lib/python_builtins/__hello__.py deleted file mode 120000 index f6cae8932f0..00000000000 --- a/vm/Lib/python_builtins/__hello__.py +++ /dev/null @@ -1 +0,0 @@ -../../../Lib/__hello__.py \ No newline at end of file diff --git a/vm/Lib/python_builtins/__phello__ b/vm/Lib/python_builtins/__phello__ deleted file mode 120000 index 113aeb15040..00000000000 --- a/vm/Lib/python_builtins/__phello__ +++ /dev/null @@ -1 +0,0 @@ -../../../Lib/__phello__ \ No newline at end of file diff --git a/vm/Lib/python_builtins/__reducelib.py b/vm/Lib/python_builtins/__reducelib.py deleted file mode 100644 index 0067cd0a818..00000000000 --- a/vm/Lib/python_builtins/__reducelib.py +++ /dev/null @@ -1,86 +0,0 @@ -# Modified from code from the PyPy project: -# https://bitbucket.org/pypy/pypy/src/default/pypy/objspace/std/objectobject.py - -# The MIT License - -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import copyreg - - -def _abstract_method_error(typ): - methods = ", ".join(sorted(typ.__abstractmethods__)) - err = "Can't instantiate abstract class %s with abstract methods %s" - raise TypeError(err % (typ.__name__, methods)) - - -def reduce_2(obj): - cls = obj.__class__ - - try: - getnewargs = obj.__getnewargs__ - except AttributeError: - args = () - else: - args = getnewargs() - if not isinstance(args, tuple): - raise TypeError("__getnewargs__ should return a tuple") - - try: - getstate = obj.__getstate__ - except AttributeError: - state = getattr(obj, "__dict__", None) - names = slotnames(cls) # not checking for list - if names is not None: - slots = {} - for name in names: - try: - value = getattr(obj, name) - except AttributeError: - pass - else: - slots[name] = value - if slots: - state = state, slots - else: - state = getstate() - - listitems = iter(obj) if isinstance(obj, list) else None - dictitems = iter(obj.items()) if isinstance(obj, dict) else None - - newobj = copyreg.__newobj__ - - args2 = (cls,) + args - return newobj, args2, state, listitems, dictitems - - -def slotnames(cls): - if not isinstance(cls, type): - return None - - try: - return cls.__dict__["__slotnames__"] - except KeyError: - pass - - slotnames = copyreg._slotnames(cls) - if not isinstance(slotnames, list) and slotnames is not None: - raise TypeError("copyreg._slotnames didn't return a list or None") - return slotnames diff --git a/vm/Lib/python_builtins/_frozen_importlib.py b/vm/Lib/python_builtins/_frozen_importlib.py deleted file mode 120000 index d1d4364abac..00000000000 --- a/vm/Lib/python_builtins/_frozen_importlib.py +++ /dev/null @@ -1 +0,0 @@ -../../../Lib/importlib/_bootstrap.py \ No newline at end of file diff --git a/vm/Lib/python_builtins/_frozen_importlib_external.py b/vm/Lib/python_builtins/_frozen_importlib_external.py deleted file mode 120000 index ba8aff58051..00000000000 --- a/vm/Lib/python_builtins/_frozen_importlib_external.py +++ /dev/null @@ -1 +0,0 @@ -../../../Lib/importlib/_bootstrap_external.py \ No newline at end of file diff --git a/vm/Lib/python_builtins/_thread.py b/vm/Lib/python_builtins/_thread.py deleted file mode 120000 index fa4a34c4fb9..00000000000 --- a/vm/Lib/python_builtins/_thread.py +++ /dev/null @@ -1 +0,0 @@ -../../../Lib/_dummy_thread.py \ No newline at end of file diff --git a/vm/build.rs b/vm/build.rs deleted file mode 100644 index 063153f6e7a..00000000000 --- a/vm/build.rs +++ /dev/null @@ -1,72 +0,0 @@ -use itertools::Itertools; -use std::{env, io::prelude::*, path::PathBuf, process::Command}; - -fn main() { - let frozen_libs = if cfg!(feature = "freeze-stdlib") { - "Lib/*/*.py" - } else { - "Lib/python_builtins/*.py" - }; - for entry in glob::glob(frozen_libs).expect("Lib/ exists?").flatten() { - let display = entry.display(); - println!("cargo:rerun-if-changed={display}"); - } - println!("cargo:rerun-if-changed=../Lib/importlib/_bootstrap.py"); - - println!("cargo:rustc-env=RUSTPYTHON_GIT_HASH={}", git_hash()); - println!( - "cargo:rustc-env=RUSTPYTHON_GIT_TIMESTAMP={}", - git_timestamp() - ); - println!("cargo:rustc-env=RUSTPYTHON_GIT_TAG={}", git_tag()); - println!("cargo:rustc-env=RUSTPYTHON_GIT_BRANCH={}", git_branch()); - println!( - "cargo:rustc-env=RUSTC_VERSION={}", - rustc_version::version().unwrap() - ); - - println!( - "cargo:rustc-env=RUSTPYTHON_TARGET_TRIPLE={}", - env::var("TARGET").unwrap() - ); - - let mut env_path = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - env_path.push("env_vars.rs"); - let mut f = std::fs::File::create(env_path).unwrap(); - write!( - f, - "sysvars! {{ {} }}", - std::env::vars_os().format_with(", ", |(k, v), f| f(&format_args!("{k:?} => {v:?}"))) - ) - .unwrap(); -} - -fn git_hash() -> String { - git(&["rev-parse", "--short", "HEAD"]) -} - -fn git_timestamp() -> String { - git(&["log", "-1", "--format=%ct"]) -} - -fn git_tag() -> String { - git(&["describe", "--all", "--always", "--dirty"]) -} - -fn git_branch() -> String { - git(&["name-rev", "--name-only", "HEAD"]) -} - -fn git(args: &[&str]) -> String { - command("git", args) -} - -fn command(cmd: &str, args: &[&str]) -> String { - match Command::new(cmd).args(args).output() { - Ok(output) => match String::from_utf8(output.stdout) { - Ok(s) => s, - Err(err) => format!("(output error: {err})"), - }, - Err(err) => format!("(command error: {err})"), - } -} diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs deleted file mode 100644 index e1cd43dff96..00000000000 --- a/vm/src/anystr.rs +++ /dev/null @@ -1,477 +0,0 @@ -use crate::{ - builtins::{PyIntRef, PyTuple}, - cformat::cformat_string, - convert::TryFromBorrowedObject, - function::OptionalOption, - Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, -}; -use num_traits::{cast::ToPrimitive, sign::Signed}; - -#[derive(FromArgs)] -pub struct SplitArgs<T: TryFromObject + AnyStrWrapper> { - #[pyarg(any, default)] - sep: Option<T>, - #[pyarg(any, default = "-1")] - maxsplit: isize, -} - -impl<T: TryFromObject + AnyStrWrapper> SplitArgs<T> { - pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Option<T>, isize)> { - let sep = if let Some(s) = self.sep { - let sep = s.as_ref(); - if sep.is_empty() { - return Err(vm.new_value_error("empty separator".to_owned())); - } - Some(s) - } else { - None - }; - Ok((sep, self.maxsplit)) - } -} - -#[derive(FromArgs)] -pub struct SplitLinesArgs { - #[pyarg(any, default = "false")] - pub keepends: bool, -} - -#[derive(FromArgs)] -pub struct ExpandTabsArgs { - #[pyarg(any, default = "8")] - tabsize: isize, -} - -impl ExpandTabsArgs { - pub fn tabsize(&self) -> usize { - self.tabsize.to_usize().unwrap_or(0) - } -} - -#[derive(FromArgs)] -pub struct StartsEndsWithArgs { - #[pyarg(positional)] - affix: PyObjectRef, - #[pyarg(positional, default)] - start: Option<PyIntRef>, - #[pyarg(positional, default)] - end: Option<PyIntRef>, -} - -impl StartsEndsWithArgs { - pub fn get_value(self, len: usize) -> (PyObjectRef, Option<std::ops::Range<usize>>) { - let range = if self.start.is_some() || self.end.is_some() { - Some(adjust_indices(self.start, self.end, len)) - } else { - None - }; - (self.affix, range) - } - - #[inline] - pub fn prepare<S, F>(self, s: &S, len: usize, substr: F) -> Option<(PyObjectRef, &S)> - where - S: ?Sized + AnyStr, - F: Fn(&S, std::ops::Range<usize>) -> &S, - { - let (affix, range) = self.get_value(len); - let substr = if let Some(range) = range { - if !range.is_normal() { - return None; - } - substr(s, range) - } else { - s - }; - Some((affix, substr)) - } -} - -fn saturate_to_isize(py_int: PyIntRef) -> isize { - let big = py_int.as_bigint(); - big.to_isize().unwrap_or_else(|| { - if big.is_negative() { - isize::MIN - } else { - isize::MAX - } - }) -} - -// help get optional string indices -pub fn adjust_indices( - start: Option<PyIntRef>, - end: Option<PyIntRef>, - len: usize, -) -> std::ops::Range<usize> { - let mut start = start.map_or(0, saturate_to_isize); - let mut end = end.map_or(len as isize, saturate_to_isize); - if end > len as isize { - end = len as isize; - } else if end < 0 { - end += len as isize; - if end < 0 { - end = 0; - } - } - if start < 0 { - start += len as isize; - if start < 0 { - start = 0; - } - } - start as usize..end as usize -} - -pub trait StringRange { - fn is_normal(&self) -> bool; -} - -impl StringRange for std::ops::Range<usize> { - fn is_normal(&self) -> bool { - self.start <= self.end - } -} - -pub trait AnyStrWrapper { - type Str: ?Sized + AnyStr; - fn as_ref(&self) -> &Self::Str; -} - -pub trait AnyStrContainer<S> -where - S: ?Sized, -{ - fn new() -> Self; - fn with_capacity(capacity: usize) -> Self; - fn push_str(&mut self, s: &S); -} - -pub trait AnyStr { - type Char: Copy; - type Container: AnyStrContainer<Self> + Extend<Self::Char>; - type CharIter<'a>: Iterator<Item = char> + 'a - where - Self: 'a; - type ElementIter<'a>: Iterator<Item = Self::Char> + 'a - where - Self: 'a; - - fn element_bytes_len(c: Self::Char) -> usize; - - fn to_container(&self) -> Self::Container; - fn as_bytes(&self) -> &[u8]; - fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error>; - fn chars(&self) -> Self::CharIter<'_>; - fn elements(&self) -> Self::ElementIter<'_>; - fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self; - // FIXME: get_chars is expensive for str - fn get_chars(&self, range: std::ops::Range<usize>) -> &Self; - fn bytes_len(&self) -> usize; - // NOTE: str::chars().count() consumes the O(n) time. But pystr::char_len does cache. - // So using chars_len directly is too expensive and the below method shouldn't be implemented. - // fn chars_len(&self) -> usize; - fn is_empty(&self) -> bool; - - fn py_add(&self, other: &Self) -> Self::Container { - let mut new = Self::Container::with_capacity(self.bytes_len() + other.bytes_len()); - new.push_str(self); - new.push_str(other); - new - } - - fn py_split<T, SP, SN, SW, R>( - &self, - args: SplitArgs<T>, - vm: &VirtualMachine, - split: SP, - splitn: SN, - splitw: SW, - ) -> PyResult<Vec<R>> - where - T: TryFromObject + AnyStrWrapper<Str = Self>, - SP: Fn(&Self, &Self, &VirtualMachine) -> Vec<R>, - SN: Fn(&Self, &Self, usize, &VirtualMachine) -> Vec<R>, - SW: Fn(&Self, isize, &VirtualMachine) -> Vec<R>, - { - let (sep, maxsplit) = args.get_value(vm)?; - let splits = if let Some(pattern) = sep { - if maxsplit < 0 { - split(self, pattern.as_ref(), vm) - } else { - splitn(self, pattern.as_ref(), (maxsplit + 1) as usize, vm) - } - } else { - splitw(self, maxsplit, vm) - }; - Ok(splits) - } - fn py_split_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> - where - F: Fn(&Self) -> PyObjectRef; - fn py_rsplit_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> - where - F: Fn(&Self) -> PyObjectRef; - - #[inline] - fn py_startsendswith<'a, T, F>( - &self, - affix: &'a PyObject, - func_name: &str, - py_type_name: &str, - func: F, - vm: &VirtualMachine, - ) -> PyResult<bool> - where - T: TryFromBorrowedObject<'a>, - F: Fn(&Self, T) -> bool, - { - single_or_tuple_any( - affix, - &|s: T| Ok(func(self, s)), - &|o| { - format!( - "{} first arg must be {} or a tuple of {}, not {}", - func_name, - py_type_name, - py_type_name, - o.class(), - ) - }, - vm, - ) - } - - #[inline] - fn py_strip<'a, S, FC, FD>( - &'a self, - chars: OptionalOption<S>, - func_chars: FC, - func_default: FD, - ) -> &'a Self - where - S: AnyStrWrapper<Str = Self>, - FC: Fn(&'a Self, &Self) -> &'a Self, - FD: Fn(&'a Self) -> &'a Self, - { - let chars = chars.flatten(); - match chars { - Some(chars) => func_chars(self, chars.as_ref()), - None => func_default(self), - } - } - - #[inline] - fn py_find<F>(&self, needle: &Self, range: std::ops::Range<usize>, find: F) -> Option<usize> - where - F: Fn(&Self, &Self) -> Option<usize>, - { - if range.is_normal() { - let start = range.start; - let index = find(self.get_chars(range), needle)?; - Some(start + index) - } else { - None - } - } - - #[inline] - fn py_count<F>(&self, needle: &Self, range: std::ops::Range<usize>, count: F) -> usize - where - F: Fn(&Self, &Self) -> usize, - { - if range.is_normal() { - count(self.get_chars(range), needle) - } else { - 0 - } - } - - fn py_pad(&self, left: usize, right: usize, fillchar: Self::Char) -> Self::Container { - let mut u = Self::Container::with_capacity( - (left + right) * Self::element_bytes_len(fillchar) + self.bytes_len(), - ); - u.extend(std::iter::repeat(fillchar).take(left)); - u.push_str(self); - u.extend(std::iter::repeat(fillchar).take(right)); - u - } - - fn py_center(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { - let marg = width - len; - let left = marg / 2 + (marg & width & 1); - self.py_pad(left, marg - left, fillchar) - } - - fn py_ljust(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { - self.py_pad(0, width - len, fillchar) - } - - fn py_rjust(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { - self.py_pad(width - len, 0, fillchar) - } - - fn py_join( - &self, - mut iter: impl std::iter::Iterator< - Item = PyResult<impl AnyStrWrapper<Str = Self> + TryFromObject>, - >, - ) -> PyResult<Self::Container> { - let mut joined = if let Some(elem) = iter.next() { - elem?.as_ref().to_container() - } else { - return Ok(Self::Container::new()); - }; - for elem in iter { - let elem = elem?; - joined.push_str(self); - joined.push_str(elem.as_ref()); - } - Ok(joined) - } - - fn py_partition<'a, F, S>( - &'a self, - sub: &Self, - split: F, - vm: &VirtualMachine, - ) -> PyResult<(Self::Container, bool, Self::Container)> - where - F: Fn() -> S, - S: std::iter::Iterator<Item = &'a Self>, - { - if sub.is_empty() { - return Err(vm.new_value_error("empty separator".to_owned())); - } - - let mut sp = split(); - let front = sp.next().unwrap().to_container(); - let (has_mid, back) = if let Some(back) = sp.next() { - (true, back.to_container()) - } else { - (false, Self::Container::new()) - }; - Ok((front, has_mid, back)) - } - - fn py_removeprefix<FC>(&self, prefix: &Self, prefix_len: usize, is_prefix: FC) -> &Self - where - FC: Fn(&Self, &Self) -> bool, - { - //if self.py_starts_with(prefix) { - if is_prefix(self, prefix) { - self.get_bytes(prefix_len..self.bytes_len()) - } else { - self - } - } - - fn py_removesuffix<FC>(&self, suffix: &Self, suffix_len: usize, is_suffix: FC) -> &Self - where - FC: Fn(&Self, &Self) -> bool, - { - if is_suffix(self, suffix) { - self.get_bytes(0..self.bytes_len() - suffix_len) - } else { - self - } - } - - // TODO: remove this function from anystr. - // See https://github.com/RustPython/RustPython/pull/4709/files#r1141013993 - fn py_bytes_splitlines<FW, W>(&self, options: SplitLinesArgs, into_wrapper: FW) -> Vec<W> - where - FW: Fn(&Self) -> W, - { - let keep = options.keepends as usize; - let mut elements = Vec::new(); - let mut last_i = 0; - let mut enumerated = self.as_bytes().iter().enumerate().peekable(); - while let Some((i, ch)) = enumerated.next() { - let (end_len, i_diff) = match *ch { - b'\n' => (keep, 1), - b'\r' => { - let is_rn = enumerated.peek().map_or(false, |(_, ch)| **ch == b'\n'); - if is_rn { - let _ = enumerated.next(); - (keep + keep, 2) - } else { - (keep, 1) - } - } - _ => { - continue; - } - }; - let range = last_i..i + end_len; - last_i = i + i_diff; - elements.push(into_wrapper(self.get_bytes(range))); - } - if last_i != self.bytes_len() { - elements.push(into_wrapper(self.get_bytes(last_i..self.bytes_len()))); - } - elements - } - - fn py_zfill(&self, width: isize) -> Vec<u8> { - let width = width.to_usize().unwrap_or(0); - rustpython_common::str::zfill(self.as_bytes(), width) - } - - fn py_iscase<F, G>(&self, is_case: F, is_opposite: G) -> bool - where - F: Fn(char) -> bool, - G: Fn(char) -> bool, - { - // Unified form of CPython functions: - // _Py_bytes_islower - // Py_bytes_isupper - // unicode_islower_impl - // unicode_isupper_impl - let mut cased = false; - for c in self.chars() { - if is_opposite(c) { - return false; - } else if !cased && is_case(c) { - cased = true - } - } - cased - } - - fn py_cformat(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { - let format_string = self.as_utf8_str().unwrap(); - cformat_string(vm, format_string, values) - } -} - -/// Tests that the predicate is True on a single value, or if the value is a tuple a tuple, then -/// test that any of the values contained within the tuples satisfies the predicate. Type parameter -/// T specifies the type that is expected, if the input value is not of that type or a tuple of -/// values of that type, then a TypeError is raised. -pub fn single_or_tuple_any<'a, T, F, M>( - obj: &'a PyObject, - predicate: &F, - message: &M, - vm: &VirtualMachine, -) -> PyResult<bool> -where - T: TryFromBorrowedObject<'a>, - F: Fn(T) -> PyResult<bool>, - M: Fn(&PyObject) -> String, -{ - match obj.try_to_value::<T>(vm) { - Ok(single) => (predicate)(single), - Err(_) => { - let tuple: &Py<PyTuple> = obj - .try_to_value(vm) - .map_err(|_| vm.new_type_error((message)(obj)))?; - for obj in tuple { - if single_or_tuple_any(obj, predicate, message, vm)? { - return Ok(true); - } - } - Ok(false) - } - } -} diff --git a/vm/src/buffer.rs b/vm/src/buffer.rs deleted file mode 100644 index 2b61ee629d1..00000000000 --- a/vm/src/buffer.rs +++ /dev/null @@ -1,647 +0,0 @@ -use crate::{ - builtins::{PyBaseExceptionRef, PyBytesRef, PyTuple, PyTupleRef, PyTypeRef}, - common::{static_cell, str::wchar_t}, - convert::ToPyObject, - function::{ArgBytesLike, ArgIntoBool, ArgIntoFloat}, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, -}; -use half::f16; -use itertools::Itertools; -use malachite_bigint::BigInt; -use num_traits::{PrimInt, ToPrimitive}; -use std::{fmt, iter::Peekable, mem, os::raw}; - -type PackFunc = fn(&VirtualMachine, PyObjectRef, &mut [u8]) -> PyResult<()>; -type UnpackFunc = fn(&VirtualMachine, &[u8]) -> PyObjectRef; - -static OVERFLOW_MSG: &str = "total struct size too long"; // not a const to reduce code size - -#[derive(Debug, Copy, Clone, PartialEq)] -pub(crate) enum Endianness { - Native, - Little, - Big, - Host, -} - -impl Endianness { - /// Parse endianness - /// See also: https://docs.python.org/3/library/struct.html?highlight=struct#byte-order-size-and-alignment - fn parse<I>(chars: &mut Peekable<I>) -> Endianness - where - I: Sized + Iterator<Item = u8>, - { - let e = match chars.peek() { - Some(b'@') => Endianness::Native, - Some(b'=') => Endianness::Host, - Some(b'<') => Endianness::Little, - Some(b'>') | Some(b'!') => Endianness::Big, - _ => return Endianness::Native, - }; - chars.next().unwrap(); - e - } -} - -trait ByteOrder { - fn convert<I: PrimInt>(i: I) -> I; -} -enum BigEndian {} -impl ByteOrder for BigEndian { - fn convert<I: PrimInt>(i: I) -> I { - i.to_be() - } -} -enum LittleEndian {} -impl ByteOrder for LittleEndian { - fn convert<I: PrimInt>(i: I) -> I { - i.to_le() - } -} - -#[cfg(target_endian = "big")] -type NativeEndian = BigEndian; -#[cfg(target_endian = "little")] -type NativeEndian = LittleEndian; - -#[derive(Copy, Clone, num_enum::TryFromPrimitive)] -#[repr(u8)] -pub(crate) enum FormatType { - Pad = b'x', - SByte = b'b', - UByte = b'B', - Char = b'c', - WideChar = b'u', - Str = b's', - Pascal = b'p', - Short = b'h', - UShort = b'H', - Int = b'i', - UInt = b'I', - Long = b'l', - ULong = b'L', - SSizeT = b'n', - SizeT = b'N', - LongLong = b'q', - ULongLong = b'Q', - Bool = b'?', - Half = b'e', - Float = b'f', - Double = b'd', - VoidP = b'P', -} - -impl fmt::Debug for FormatType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&(*self as u8 as char), f) - } -} - -impl FormatType { - fn info(self, e: Endianness) -> &'static FormatInfo { - use mem::{align_of, size_of}; - use FormatType::*; - macro_rules! native_info { - ($t:ty) => {{ - &FormatInfo { - size: size_of::<$t>(), - align: align_of::<$t>(), - pack: Some(<$t as Packable>::pack::<NativeEndian>), - unpack: Some(<$t as Packable>::unpack::<NativeEndian>), - } - }}; - } - macro_rules! nonnative_info { - ($t:ty, $end:ty) => {{ - &FormatInfo { - size: size_of::<$t>(), - align: 0, - pack: Some(<$t as Packable>::pack::<$end>), - unpack: Some(<$t as Packable>::unpack::<$end>), - } - }}; - } - macro_rules! match_nonnative { - ($zelf:expr, $end:ty) => {{ - match $zelf { - Pad | Str | Pascal => &FormatInfo { - size: size_of::<u8>(), - align: 0, - pack: None, - unpack: None, - }, - SByte => nonnative_info!(i8, $end), - UByte => nonnative_info!(u8, $end), - Char => &FormatInfo { - size: size_of::<u8>(), - align: 0, - pack: Some(pack_char), - unpack: Some(unpack_char), - }, - Short => nonnative_info!(i16, $end), - UShort => nonnative_info!(u16, $end), - Int | Long => nonnative_info!(i32, $end), - UInt | ULong => nonnative_info!(u32, $end), - LongLong => nonnative_info!(i64, $end), - ULongLong => nonnative_info!(u64, $end), - Bool => nonnative_info!(bool, $end), - Half => nonnative_info!(f16, $end), - Float => nonnative_info!(f32, $end), - Double => nonnative_info!(f64, $end), - _ => unreachable!(), // size_t or void* - } - }}; - } - match e { - Endianness::Native => match self { - Pad | Str | Pascal => &FormatInfo { - size: size_of::<raw::c_char>(), - align: 0, - pack: None, - unpack: None, - }, - SByte => native_info!(raw::c_schar), - UByte => native_info!(raw::c_uchar), - Char => &FormatInfo { - size: size_of::<raw::c_char>(), - align: 0, - pack: Some(pack_char), - unpack: Some(unpack_char), - }, - WideChar => native_info!(wchar_t), - Short => native_info!(raw::c_short), - UShort => native_info!(raw::c_ushort), - Int => native_info!(raw::c_int), - UInt => native_info!(raw::c_uint), - Long => native_info!(raw::c_long), - ULong => native_info!(raw::c_ulong), - SSizeT => native_info!(isize), // ssize_t == isize - SizeT => native_info!(usize), // size_t == usize - LongLong => native_info!(raw::c_longlong), - ULongLong => native_info!(raw::c_ulonglong), - Bool => native_info!(bool), - Half => native_info!(f16), - Float => native_info!(raw::c_float), - Double => native_info!(raw::c_double), - VoidP => native_info!(*mut raw::c_void), - }, - Endianness::Big => match_nonnative!(self, BigEndian), - Endianness::Little => match_nonnative!(self, LittleEndian), - Endianness::Host => match_nonnative!(self, NativeEndian), - } - } -} - -#[derive(Debug, Clone)] -pub(crate) struct FormatCode { - pub repeat: usize, - pub code: FormatType, - pub info: &'static FormatInfo, - pub pre_padding: usize, -} - -impl FormatCode { - pub fn arg_count(&self) -> usize { - match self.code { - FormatType::Pad => 0, - FormatType::Str | FormatType::Pascal => 1, - _ => self.repeat, - } - } - - pub fn parse<I>( - chars: &mut Peekable<I>, - endianness: Endianness, - ) -> Result<(Vec<Self>, usize, usize), String> - where - I: Sized + Iterator<Item = u8>, - { - let mut offset = 0isize; - let mut arg_count = 0usize; - let mut codes = vec![]; - while chars.peek().is_some() { - // determine repeat operator: - let repeat = match chars.peek() { - Some(b'0'..=b'9') => { - let mut repeat = 0isize; - while let Some(b'0'..=b'9') = chars.peek() { - if let Some(c) = chars.next() { - let current_digit = c - b'0'; - repeat = repeat - .checked_mul(10) - .and_then(|r| r.checked_add(current_digit as _)) - .ok_or_else(|| OVERFLOW_MSG.to_owned())?; - } - } - repeat - } - _ => 1, - }; - - // determine format char: - let c = chars - .next() - .ok_or_else(|| "repeat count given without format specifier".to_owned())?; - let code = FormatType::try_from(c) - .ok() - .filter(|c| match c { - FormatType::SSizeT | FormatType::SizeT | FormatType::VoidP => { - endianness == Endianness::Native - } - _ => true, - }) - .ok_or_else(|| "bad char in struct format".to_owned())?; - - let info = code.info(endianness); - - let padding = compensate_alignment(offset as usize, info.align) - .ok_or_else(|| OVERFLOW_MSG.to_owned())?; - offset = padding - .to_isize() - .and_then(|extra| offset.checked_add(extra)) - .ok_or_else(|| OVERFLOW_MSG.to_owned())?; - - let code = FormatCode { - repeat: repeat as usize, - code, - info, - pre_padding: padding, - }; - arg_count += code.arg_count(); - codes.push(code); - - offset = (info.size as isize) - .checked_mul(repeat) - .and_then(|item_size| offset.checked_add(item_size)) - .ok_or_else(|| OVERFLOW_MSG.to_owned())?; - } - - Ok((codes, offset as usize, arg_count)) - } -} - -fn compensate_alignment(offset: usize, align: usize) -> Option<usize> { - if align != 0 && offset != 0 { - // a % b == a & (b-1) if b is a power of 2 - (align - 1).checked_sub((offset - 1) & (align - 1)) - } else { - // alignment is already all good - Some(0) - } -} - -pub(crate) struct FormatInfo { - pub size: usize, - pub align: usize, - pub pack: Option<PackFunc>, - pub unpack: Option<UnpackFunc>, -} -impl fmt::Debug for FormatInfo { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("FormatInfo") - .field("size", &self.size) - .field("align", &self.align) - .finish() - } -} - -#[derive(Debug, Clone)] -pub struct FormatSpec { - #[allow(dead_code)] - pub(crate) endianness: Endianness, - pub(crate) codes: Vec<FormatCode>, - pub size: usize, - pub arg_count: usize, -} - -impl FormatSpec { - pub fn parse(fmt: &[u8], vm: &VirtualMachine) -> PyResult<FormatSpec> { - let mut chars = fmt.iter().copied().peekable(); - - // First determine "@", "<", ">","!" or "=" - let endianness = Endianness::parse(&mut chars); - - // Now, analyze struct string further: - let (codes, size, arg_count) = - FormatCode::parse(&mut chars, endianness).map_err(|err| new_struct_error(vm, err))?; - - Ok(FormatSpec { - endianness, - codes, - size, - arg_count, - }) - } - - pub fn pack(&self, args: Vec<PyObjectRef>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - // Create data vector: - let mut data = vec![0; self.size]; - - self.pack_into(&mut data, args, vm)?; - - Ok(data) - } - - pub fn pack_into( - &self, - mut buffer: &mut [u8], - args: Vec<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - if self.arg_count != args.len() { - return Err(new_struct_error( - vm, - format!( - "pack expected {} items for packing (got {})", - self.codes.len(), - args.len() - ), - )); - } - - let mut args = args.into_iter(); - // Loop over all opcodes: - for code in &self.codes { - buffer = &mut buffer[code.pre_padding..]; - debug!("code: {:?}", code); - match code.code { - FormatType::Str => { - let (buf, rest) = buffer.split_at_mut(code.repeat); - pack_string(vm, args.next().unwrap(), buf)?; - buffer = rest; - } - FormatType::Pascal => { - let (buf, rest) = buffer.split_at_mut(code.repeat); - pack_pascal(vm, args.next().unwrap(), buf)?; - buffer = rest; - } - FormatType::Pad => { - let (pad_buf, rest) = buffer.split_at_mut(code.repeat); - for el in pad_buf { - *el = 0 - } - buffer = rest; - } - _ => { - let pack = code.info.pack.unwrap(); - for arg in args.by_ref().take(code.repeat) { - let (item_buf, rest) = buffer.split_at_mut(code.info.size); - pack(vm, arg, item_buf)?; - buffer = rest; - } - } - } - } - - Ok(()) - } - - pub fn unpack(&self, mut data: &[u8], vm: &VirtualMachine) -> PyResult<PyTupleRef> { - if self.size != data.len() { - return Err(new_struct_error( - vm, - format!("unpack requires a buffer of {} bytes", self.size), - )); - } - - let mut items = Vec::with_capacity(self.arg_count); - for code in &self.codes { - data = &data[code.pre_padding..]; - debug!("unpack code: {:?}", code); - match code.code { - FormatType::Pad => { - data = &data[code.repeat..]; - } - FormatType::Str => { - let (str_data, rest) = data.split_at(code.repeat); - // string is just stored inline - items.push(vm.ctx.new_bytes(str_data.to_vec()).into()); - data = rest; - } - FormatType::Pascal => { - let (str_data, rest) = data.split_at(code.repeat); - items.push(unpack_pascal(vm, str_data)); - data = rest; - } - _ => { - let unpack = code.info.unpack.unwrap(); - for _ in 0..code.repeat { - let (item_data, rest) = data.split_at(code.info.size); - items.push(unpack(vm, item_data)); - data = rest; - } - } - }; - } - - Ok(PyTuple::new_ref(items, &vm.ctx)) - } - - #[inline] - pub fn size(&self) -> usize { - self.size - } -} - -trait Packable { - fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()>; - fn unpack<E: ByteOrder>(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef; -} - -trait PackInt: PrimInt { - fn pack_int<E: ByteOrder>(self, data: &mut [u8]); - fn unpack_int<E: ByteOrder>(data: &[u8]) -> Self; -} - -macro_rules! make_pack_primint { - ($T:ty) => { - impl PackInt for $T { - fn pack_int<E: ByteOrder>(self, data: &mut [u8]) { - let i = E::convert(self); - data.copy_from_slice(&i.to_ne_bytes()); - } - #[inline] - fn unpack_int<E: ByteOrder>(data: &[u8]) -> Self { - let mut x = [0; std::mem::size_of::<$T>()]; - x.copy_from_slice(data); - E::convert(<$T>::from_ne_bytes(x)) - } - } - - impl Packable for $T { - fn pack<E: ByteOrder>( - vm: &VirtualMachine, - arg: PyObjectRef, - data: &mut [u8], - ) -> PyResult<()> { - let i: $T = get_int_or_index(vm, arg)?; - i.pack_int::<E>(data); - Ok(()) - } - - fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { - let i = <$T>::unpack_int::<E>(rdr); - vm.ctx.new_int(i).into() - } - } - }; -} - -fn get_int_or_index<T>(vm: &VirtualMachine, arg: PyObjectRef) -> PyResult<T> -where - T: PrimInt + for<'a> TryFrom<&'a BigInt>, -{ - let index = arg.try_index_opt(vm).unwrap_or_else(|| { - Err(new_struct_error( - vm, - "required argument is not an integer".to_owned(), - )) - })?; - index - .try_to_primitive(vm) - .map_err(|_| new_struct_error(vm, "argument out of range".to_owned())) -} - -make_pack_primint!(i8); -make_pack_primint!(u8); -make_pack_primint!(i16); -make_pack_primint!(u16); -make_pack_primint!(i32); -make_pack_primint!(u32); -make_pack_primint!(i64); -make_pack_primint!(u64); -make_pack_primint!(usize); -make_pack_primint!(isize); - -macro_rules! make_pack_float { - ($T:ty) => { - impl Packable for $T { - fn pack<E: ByteOrder>( - vm: &VirtualMachine, - arg: PyObjectRef, - data: &mut [u8], - ) -> PyResult<()> { - let f = *ArgIntoFloat::try_from_object(vm, arg)? as $T; - f.to_bits().pack_int::<E>(data); - Ok(()) - } - - fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { - let i = PackInt::unpack_int::<E>(rdr); - <$T>::from_bits(i).to_pyobject(vm) - } - } - }; -} - -make_pack_float!(f32); -make_pack_float!(f64); - -impl Packable for f16 { - fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { - let f_64 = *ArgIntoFloat::try_from_object(vm, arg)?; - let f_16 = f16::from_f64(f_64); - if f_16.is_infinite() != f_64.is_infinite() { - return Err(vm.new_overflow_error("float too large to pack with e format".to_owned())); - } - f_16.to_bits().pack_int::<E>(data); - Ok(()) - } - - fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { - let i = PackInt::unpack_int::<E>(rdr); - f16::from_bits(i).to_f64().to_pyobject(vm) - } -} - -impl Packable for *mut raw::c_void { - fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { - usize::pack::<E>(vm, arg, data) - } - - fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { - usize::unpack::<E>(vm, rdr) - } -} - -impl Packable for bool { - fn pack<E: ByteOrder>(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { - let v = *ArgIntoBool::try_from_object(vm, arg)? as u8; - v.pack_int::<E>(data); - Ok(()) - } - - fn unpack<E: ByteOrder>(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { - let i = u8::unpack_int::<E>(rdr); - vm.ctx.new_bool(i != 0).into() - } -} - -fn pack_char(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { - let v = PyBytesRef::try_from_object(vm, arg)?; - let ch = *v.as_bytes().iter().exactly_one().map_err(|_| { - new_struct_error( - vm, - "char format requires a bytes object of length 1".to_owned(), - ) - })?; - data[0] = ch; - Ok(()) -} - -fn pack_string(vm: &VirtualMachine, arg: PyObjectRef, buf: &mut [u8]) -> PyResult<()> { - let b = ArgBytesLike::try_from_object(vm, arg)?; - b.with_ref(|data| write_string(buf, data)); - Ok(()) -} - -fn pack_pascal(vm: &VirtualMachine, arg: PyObjectRef, buf: &mut [u8]) -> PyResult<()> { - if buf.is_empty() { - return Ok(()); - } - let b = ArgBytesLike::try_from_object(vm, arg)?; - b.with_ref(|data| { - let string_length = std::cmp::min(std::cmp::min(data.len(), 255), buf.len() - 1); - buf[0] = string_length as u8; - write_string(&mut buf[1..], data); - }); - Ok(()) -} - -fn write_string(buf: &mut [u8], data: &[u8]) { - let len_from_data = std::cmp::min(data.len(), buf.len()); - buf[..len_from_data].copy_from_slice(&data[..len_from_data]); - for byte in &mut buf[len_from_data..] { - *byte = 0 - } -} - -fn unpack_char(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef { - vm.ctx.new_bytes(vec![data[0]]).into() -} - -fn unpack_pascal(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef { - let (&len, data) = match data.split_first() { - Some(x) => x, - None => { - // cpython throws an internal SystemError here - return vm.ctx.new_bytes(vec![]).into(); - } - }; - let len = std::cmp::min(len as usize, data.len()); - vm.ctx.new_bytes(data[..len].to_vec()).into() -} - -// XXX: are those functions expected to be placed here? -pub fn struct_error_type(vm: &VirtualMachine) -> &'static PyTypeRef { - static_cell! { - static INSTANCE: PyTypeRef; - } - INSTANCE.get_or_init(|| vm.ctx.new_exception_type("struct", "error", None)) -} - -pub fn new_struct_error(vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { - // can't just STRUCT_ERROR.get().unwrap() cause this could be called before from buffer - // machinery, independent of whether _struct was ever imported - vm.new_exception_msg(struct_error_type(vm).clone(), msg) -} diff --git a/vm/src/builtins/asyncgenerator.rs b/vm/src/builtins/asyncgenerator.rs deleted file mode 100644 index d58744da0ad..00000000000 --- a/vm/src/builtins/asyncgenerator.rs +++ /dev/null @@ -1,428 +0,0 @@ -use super::{PyCode, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; -use crate::{ - builtins::PyBaseExceptionRef, - class::PyClassImpl, - coroutine::Coro, - frame::FrameRef, - function::OptionalArg, - protocol::PyIterReturn, - types::{Constructor, IterNext, Iterable, Representable, SelfIter, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -use crossbeam_utils::atomic::AtomicCell; - -#[pyclass(name = "async_generator", module = false)] -#[derive(Debug)] -pub struct PyAsyncGen { - inner: Coro, - running_async: AtomicCell<bool>, -} -type PyAsyncGenRef = PyRef<PyAsyncGen>; - -impl PyPayload for PyAsyncGen { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.async_generator - } -} - -#[pyclass(with(PyRef, Constructor, Representable))] -impl PyAsyncGen { - pub fn as_coro(&self) -> &Coro { - &self.inner - } - - pub fn new(frame: FrameRef, name: PyStrRef) -> Self { - PyAsyncGen { - inner: Coro::new(frame, name), - running_async: AtomicCell::new(false), - } - } - - #[pygetset(magic)] - fn name(&self) -> PyStrRef { - self.inner.name() - } - - #[pygetset(magic, setter)] - fn set_name(&self, name: PyStrRef) { - self.inner.set_name(name) - } - - #[pygetset] - fn ag_await(&self, _vm: &VirtualMachine) -> Option<PyObjectRef> { - self.inner.frame().yield_from_target() - } - #[pygetset] - fn ag_frame(&self, _vm: &VirtualMachine) -> FrameRef { - self.inner.frame() - } - #[pygetset] - fn ag_running(&self, _vm: &VirtualMachine) -> bool { - self.inner.running() - } - #[pygetset] - fn ag_code(&self, _vm: &VirtualMachine) -> PyRef<PyCode> { - self.inner.frame().code.clone() - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } -} - -#[pyclass] -impl PyRef<PyAsyncGen> { - #[pymethod(magic)] - fn aiter(self, _vm: &VirtualMachine) -> PyRef<PyAsyncGen> { - self - } - - #[pymethod(magic)] - fn anext(self, vm: &VirtualMachine) -> PyAsyncGenASend { - Self::asend(self, vm.ctx.none(), vm) - } - - #[pymethod] - fn asend(self, value: PyObjectRef, _vm: &VirtualMachine) -> PyAsyncGenASend { - PyAsyncGenASend { - ag: self, - state: AtomicCell::new(AwaitableState::Init), - value, - } - } - - #[pymethod] - fn athrow( - self, - exc_type: PyObjectRef, - exc_val: OptionalArg, - exc_tb: OptionalArg, - vm: &VirtualMachine, - ) -> PyAsyncGenAThrow { - PyAsyncGenAThrow { - ag: self, - aclose: false, - state: AtomicCell::new(AwaitableState::Init), - value: ( - exc_type, - exc_val.unwrap_or_none(vm), - exc_tb.unwrap_or_none(vm), - ), - } - } - - #[pymethod] - fn aclose(self, vm: &VirtualMachine) -> PyAsyncGenAThrow { - PyAsyncGenAThrow { - ag: self, - aclose: true, - state: AtomicCell::new(AwaitableState::Init), - value: ( - vm.ctx.exceptions.generator_exit.to_owned().into(), - vm.ctx.none(), - vm.ctx.none(), - ), - } - } -} - -impl Representable for PyAsyncGen { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - Ok(zelf.inner.repr(zelf.as_object(), zelf.get_id(), vm)) - } -} - -impl Unconstructible for PyAsyncGen {} - -#[pyclass(module = false, name = "async_generator_wrapped_value")] -#[derive(Debug)] -pub(crate) struct PyAsyncGenWrappedValue(pub PyObjectRef); -impl PyPayload for PyAsyncGenWrappedValue { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.async_generator_wrapped_value - } -} - -#[pyclass] -impl PyAsyncGenWrappedValue {} - -impl PyAsyncGenWrappedValue { - fn unbox(ag: &PyAsyncGen, val: PyResult<PyIterReturn>, vm: &VirtualMachine) -> PyResult { - let (closed, async_done) = match &val { - Ok(PyIterReturn::StopIteration(_)) => (true, true), - Err(e) if e.fast_isinstance(vm.ctx.exceptions.generator_exit) => (true, true), - Err(_) => (false, true), - _ => (false, false), - }; - if closed { - ag.inner.closed.store(true); - } - if async_done { - ag.running_async.store(false); - } - let val = val?.into_async_pyresult(vm)?; - match_class!(match val { - val @ Self => { - ag.running_async.store(false); - Err(vm.new_stop_iteration(Some(val.0.clone()))) - } - val => Ok(val), - }) - } -} - -#[derive(Debug, Clone, Copy)] -enum AwaitableState { - Init, - Iter, - Closed, -} - -#[pyclass(module = false, name = "async_generator_asend")] -#[derive(Debug)] -pub(crate) struct PyAsyncGenASend { - ag: PyAsyncGenRef, - state: AtomicCell<AwaitableState>, - value: PyObjectRef, -} - -impl PyPayload for PyAsyncGenASend { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.async_generator_asend - } -} - -#[pyclass(with(IterNext, Iterable))] -impl PyAsyncGenASend { - #[pymethod(name = "__await__")] - fn r#await(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyRef<Self> { - zelf - } - - #[pymethod] - fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let val = match self.state.load() { - AwaitableState::Closed => { - return Err(vm.new_runtime_error( - "cannot reuse already awaited __anext__()/asend()".to_owned(), - )) - } - AwaitableState::Iter => val, // already running, all good - AwaitableState::Init => { - if self.ag.running_async.load() { - return Err(vm.new_runtime_error( - "anext(): asynchronous generator is already running".to_owned(), - )); - } - self.ag.running_async.store(true); - self.state.store(AwaitableState::Iter); - if vm.is_none(&val) { - self.value.clone() - } else { - val - } - } - }; - let res = self.ag.inner.send(self.ag.as_object(), val, vm); - let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); - if res.is_err() { - self.close(); - } - res - } - - #[pymethod] - fn throw( - &self, - exc_type: PyObjectRef, - exc_val: OptionalArg, - exc_tb: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - if let AwaitableState::Closed = self.state.load() { - return Err( - vm.new_runtime_error("cannot reuse already awaited __anext__()/asend()".to_owned()) - ); - } - - let res = self.ag.inner.throw( - self.ag.as_object(), - exc_type, - exc_val.unwrap_or_none(vm), - exc_tb.unwrap_or_none(vm), - vm, - ); - let res = PyAsyncGenWrappedValue::unbox(&self.ag, res, vm); - if res.is_err() { - self.close(); - } - res - } - - #[pymethod] - fn close(&self) { - self.state.store(AwaitableState::Closed); - } -} - -impl SelfIter for PyAsyncGenASend {} -impl IterNext for PyAsyncGenASend { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - PyIterReturn::from_pyresult(zelf.send(vm.ctx.none(), vm), vm) - } -} - -#[pyclass(module = false, name = "async_generator_athrow")] -#[derive(Debug)] -pub(crate) struct PyAsyncGenAThrow { - ag: PyAsyncGenRef, - aclose: bool, - state: AtomicCell<AwaitableState>, - value: (PyObjectRef, PyObjectRef, PyObjectRef), -} - -impl PyPayload for PyAsyncGenAThrow { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.async_generator_athrow - } -} - -#[pyclass(with(IterNext, Iterable))] -impl PyAsyncGenAThrow { - #[pymethod(name = "__await__")] - fn r#await(zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyRef<Self> { - zelf - } - - #[pymethod] - fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match self.state.load() { - AwaitableState::Closed => { - Err(vm - .new_runtime_error("cannot reuse already awaited aclose()/athrow()".to_owned())) - } - AwaitableState::Init => { - if self.ag.running_async.load() { - self.state.store(AwaitableState::Closed); - let msg = if self.aclose { - "aclose(): asynchronous generator is already running" - } else { - "athrow(): asynchronous generator is already running" - }; - return Err(vm.new_runtime_error(msg.to_owned())); - } - if self.ag.inner.closed() { - self.state.store(AwaitableState::Closed); - return Err(vm.new_stop_iteration(None)); - } - if !vm.is_none(&val) { - return Err(vm.new_runtime_error( - "can't send non-None value to a just-started async generator".to_owned(), - )); - } - self.state.store(AwaitableState::Iter); - self.ag.running_async.store(true); - - let (ty, val, tb) = self.value.clone(); - let ret = self.ag.inner.throw(self.ag.as_object(), ty, val, tb, vm); - let ret = if self.aclose { - if self.ignored_close(&ret) { - Err(self.yield_close(vm)) - } else { - ret.and_then(|o| o.into_async_pyresult(vm)) - } - } else { - PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) - }; - ret.map_err(|e| self.check_error(e, vm)) - } - AwaitableState::Iter => { - let ret = self.ag.inner.send(self.ag.as_object(), val, vm); - if self.aclose { - match ret { - Ok(PyIterReturn::Return(v)) if v.payload_is::<PyAsyncGenWrappedValue>() => { - Err(self.yield_close(vm)) - } - other => other - .and_then(|o| o.into_async_pyresult(vm)) - .map_err(|e| self.check_error(e, vm)), - } - } else { - PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) - } - } - } - } - - #[pymethod] - fn throw( - &self, - exc_type: PyObjectRef, - exc_val: OptionalArg, - exc_tb: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let ret = self.ag.inner.throw( - self.ag.as_object(), - exc_type, - exc_val.unwrap_or_none(vm), - exc_tb.unwrap_or_none(vm), - vm, - ); - let res = if self.aclose { - if self.ignored_close(&ret) { - Err(self.yield_close(vm)) - } else { - ret.and_then(|o| o.into_async_pyresult(vm)) - } - } else { - PyAsyncGenWrappedValue::unbox(&self.ag, ret, vm) - }; - res.map_err(|e| self.check_error(e, vm)) - } - - #[pymethod] - fn close(&self) { - self.state.store(AwaitableState::Closed); - } - - fn ignored_close(&self, res: &PyResult<PyIterReturn>) -> bool { - res.as_ref().map_or(false, |v| match v { - PyIterReturn::Return(obj) => obj.payload_is::<PyAsyncGenWrappedValue>(), - PyIterReturn::StopIteration(_) => false, - }) - } - fn yield_close(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - self.ag.running_async.store(false); - self.state.store(AwaitableState::Closed); - vm.new_runtime_error("async generator ignored GeneratorExit".to_owned()) - } - fn check_error(&self, exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyBaseExceptionRef { - self.ag.running_async.store(false); - self.state.store(AwaitableState::Closed); - if self.aclose - && (exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) - || exc.fast_isinstance(vm.ctx.exceptions.generator_exit)) - { - vm.new_stop_iteration(None) - } else { - exc - } - } -} - -impl SelfIter for PyAsyncGenAThrow {} -impl IterNext for PyAsyncGenAThrow { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - PyIterReturn::from_pyresult(zelf.send(vm.ctx.none(), vm), vm) - } -} - -pub fn init(ctx: &Context) { - PyAsyncGen::extend_class(ctx, ctx.types.async_generator); - PyAsyncGenASend::extend_class(ctx, ctx.types.async_generator_asend); - PyAsyncGenAThrow::extend_class(ctx, ctx.types.async_generator_athrow); -} diff --git a/vm/src/builtins/bool.rs b/vm/src/builtins/bool.rs deleted file mode 100644 index b3cb20238e0..00000000000 --- a/vm/src/builtins/bool.rs +++ /dev/null @@ -1,213 +0,0 @@ -use super::{PyInt, PyStrRef, PyType, PyTypeRef}; -use crate::{ - class::PyClassImpl, - convert::{IntoPyException, ToPyObject, ToPyResult}, - function::OptionalArg, - identifier, - protocol::PyNumberMethods, - types::{AsNumber, Constructor, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, - VirtualMachine, -}; -use malachite_bigint::Sign; -use num_traits::Zero; -use rustpython_format::FormatSpec; -use std::fmt::{Debug, Formatter}; - -impl ToPyObject for bool { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_bool(self).into() - } -} - -impl<'a> TryFromBorrowedObject<'a> for bool { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<bool> { - if obj.fast_isinstance(vm.ctx.types.int_type) { - Ok(get_value(obj)) - } else { - Err(vm.new_type_error(format!("Expected type bool, not {}", obj.class().name()))) - } - } -} - -impl PyObjectRef { - /// Convert Python bool into Rust bool. - pub fn try_to_bool(self, vm: &VirtualMachine) -> PyResult<bool> { - if self.is(&vm.ctx.true_value) { - return Ok(true); - } - if self.is(&vm.ctx.false_value) { - return Ok(false); - } - let rs_bool = match vm.get_method(self.clone(), identifier!(vm, __bool__)) { - Some(method_or_err) => { - // If descriptor returns Error, propagate it further - let method = method_or_err?; - let bool_obj = method.call((), vm)?; - if !bool_obj.fast_isinstance(vm.ctx.types.bool_type) { - return Err(vm.new_type_error(format!( - "__bool__ should return bool, returned type {}", - bool_obj.class().name() - ))); - } - - get_value(&bool_obj) - } - None => match vm.get_method(self, identifier!(vm, __len__)) { - Some(method_or_err) => { - let method = method_or_err?; - let bool_obj = method.call((), vm)?; - let int_obj = bool_obj.payload::<PyInt>().ok_or_else(|| { - vm.new_type_error(format!( - "'{}' object cannot be interpreted as an integer", - bool_obj.class().name() - )) - })?; - - let len_val = int_obj.as_bigint(); - if len_val.sign() == Sign::Minus { - return Err(vm.new_value_error("__len__() should return >= 0".to_owned())); - } - !len_val.is_zero() - } - None => true, - }, - }; - Ok(rs_bool) - } -} - -#[pyclass(name = "bool", module = false, base = "PyInt")] -pub struct PyBool; - -impl PyPayload for PyBool { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.bool_type - } -} - -impl Debug for PyBool { - fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result { - todo!() - } -} - -impl Constructor for PyBool { - type Args = OptionalArg<PyObjectRef>; - - fn py_new(zelf: PyTypeRef, x: Self::Args, vm: &VirtualMachine) -> PyResult { - if !zelf.fast_isinstance(vm.ctx.types.type_type) { - let actual_class = zelf.class(); - let actual_type = &actual_class.name(); - return Err(vm.new_type_error(format!( - "requires a 'type' object but received a '{actual_type}'" - ))); - } - let val = x.map_or(Ok(false), |val| val.try_to_bool(vm))?; - Ok(vm.ctx.new_bool(val).into()) - } -} - -#[pyclass(with(Constructor, AsNumber, Representable))] -impl PyBool { - #[pymethod(magic)] - fn format(obj: PyObjectRef, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { - let new_bool = obj.try_to_bool(vm)?; - FormatSpec::parse(spec.as_str()) - .and_then(|format_spec| format_spec.format_bool(new_bool)) - .map_err(|err| err.into_pyexception(vm)) - } - - #[pymethod(name = "__ror__")] - #[pymethod(magic)] - fn or(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if lhs.fast_isinstance(vm.ctx.types.bool_type) - && rhs.fast_isinstance(vm.ctx.types.bool_type) - { - let lhs = get_value(&lhs); - let rhs = get_value(&rhs); - (lhs || rhs).to_pyobject(vm) - } else { - get_py_int(&lhs).or(rhs, vm).to_pyobject(vm) - } - } - - #[pymethod(name = "__rand__")] - #[pymethod(magic)] - fn and(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if lhs.fast_isinstance(vm.ctx.types.bool_type) - && rhs.fast_isinstance(vm.ctx.types.bool_type) - { - let lhs = get_value(&lhs); - let rhs = get_value(&rhs); - (lhs && rhs).to_pyobject(vm) - } else { - get_py_int(&lhs).and(rhs, vm).to_pyobject(vm) - } - } - - #[pymethod(name = "__rxor__")] - #[pymethod(magic)] - fn xor(lhs: PyObjectRef, rhs: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if lhs.fast_isinstance(vm.ctx.types.bool_type) - && rhs.fast_isinstance(vm.ctx.types.bool_type) - { - let lhs = get_value(&lhs); - let rhs = get_value(&rhs); - (lhs ^ rhs).to_pyobject(vm) - } else { - get_py_int(&lhs).xor(rhs, vm).to_pyobject(vm) - } - } -} - -impl AsNumber for PyBool { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - and: Some(|a, b, vm| PyBool::and(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), - xor: Some(|a, b, vm| PyBool::xor(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), - or: Some(|a, b, vm| PyBool::or(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), - ..PyInt::AS_NUMBER - }; - &AS_NUMBER - } -} - -impl Representable for PyBool { - #[inline] - fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let name = if get_value(zelf.as_object()) { - vm.ctx.names.True - } else { - vm.ctx.names.False - }; - Ok(name.to_owned()) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use slot_repr instead") - } -} - -pub(crate) fn init(context: &Context) { - PyBool::extend_class(context, context.types.bool_type); -} - -// pub fn not(vm: &VirtualMachine, obj: &PyObject) -> PyResult<bool> { -// if obj.fast_isinstance(vm.ctx.types.bool_type) { -// let value = get_value(obj); -// Ok(!value) -// } else { -// Err(vm.new_type_error(format!("Can only invert a bool, on {:?}", obj))) -// } -// } - -// Retrieve inner int value: -pub(crate) fn get_value(obj: &PyObject) -> bool { - !obj.payload::<PyInt>().unwrap().as_bigint().is_zero() -} - -fn get_py_int(obj: &PyObject) -> &PyInt { - obj.payload::<PyInt>().unwrap() -} diff --git a/vm/src/builtins/builtin_func.rs b/vm/src/builtins/builtin_func.rs deleted file mode 100644 index 515ae7d726d..00000000000 --- a/vm/src/builtins/builtin_func.rs +++ /dev/null @@ -1,250 +0,0 @@ -use super::{type_, PyStrInterned, PyStrRef, PyType}; -use crate::{ - class::PyClassImpl, - convert::TryFromObject, - function::{FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags, PyNativeFn}, - types::{Callable, Comparable, Constructor, PyComparisonOp, Representable, Unconstructible}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use std::fmt; - -// PyCFunctionObject in CPython -#[pyclass(name = "builtin_function_or_method", module = false)] -pub struct PyNativeFunction { - pub(crate) value: &'static PyMethodDef, - pub(crate) zelf: Option<PyObjectRef>, - pub(crate) module: Option<&'static PyStrInterned>, // None for bound method -} - -impl PyPayload for PyNativeFunction { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.builtin_function_or_method_type - } -} - -impl fmt::Debug for PyNativeFunction { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "builtin function {}.{} ({:?}) self as instance of {:?}", - self.module.map_or("<unknown>", |m| m.as_str()), - self.value.name, - self.value.flags, - self.zelf.as_ref().map(|z| z.class().name().to_owned()) - ) - } -} - -impl PyNativeFunction { - pub fn with_module(mut self, module: &'static PyStrInterned) -> Self { - self.module = Some(module); - self - } - - pub fn into_ref(self, ctx: &Context) -> PyRef<Self> { - PyRef::new_ref( - self, - ctx.types.builtin_function_or_method_type.to_owned(), - None, - ) - } - - pub fn as_func(&self) -> &'static PyNativeFn { - self.value.func - } -} - -impl Callable for PyNativeFunction { - type Args = FuncArgs; - #[inline] - fn call(zelf: &Py<Self>, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { - if let Some(z) = &zelf.zelf { - args.prepend_arg(z.clone()); - } - (zelf.value.func)(vm, args) - } -} - -#[pyclass(with(Callable, Constructor), flags(HAS_DICT))] -impl PyNativeFunction { - #[pygetset(magic)] - fn module(zelf: NativeFunctionOrMethod) -> Option<&'static PyStrInterned> { - zelf.0.module - } - #[pygetset(magic)] - fn name(zelf: NativeFunctionOrMethod) -> &'static str { - zelf.0.value.name - } - #[pygetset(magic)] - fn qualname(zelf: NativeFunctionOrMethod, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let zelf = zelf.0; - let flags = zelf.value.flags; - // if flags.contains(PyMethodFlags::CLASS) || flags.contains(PyMethodFlags::STATIC) { - let qualname = if let Some(bound) = &zelf.zelf { - let prefix = if flags.contains(PyMethodFlags::CLASS) { - bound - .get_attr("__qualname__", vm) - .unwrap() - .str(vm) - .unwrap() - .to_string() - } else { - bound.class().name().to_string() - }; - vm.ctx.new_str(format!("{}.{}", prefix, &zelf.value.name)) - } else { - vm.ctx.intern_str(zelf.value.name).to_owned() - }; - Ok(qualname) - } - #[pygetset(magic)] - fn doc(zelf: NativeFunctionOrMethod) -> Option<&'static str> { - zelf.0.value.doc - } - #[pygetset(name = "__self__")] - fn __self__(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.none() - } - #[pymethod(magic)] - fn reduce(&self) -> &'static str { - // TODO: return (getattr, (self.object, self.name)) if this is a method - self.value.name - } - #[pymethod(magic)] - fn reduce_ex(zelf: PyObjectRef, _ver: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm.call_special_method(&zelf, identifier!(vm, __reduce__), ()) - } - #[pygetset(magic)] - fn text_signature(zelf: NativeFunctionOrMethod) -> Option<&'static str> { - let doc = zelf.0.value.doc?; - let signature = type_::get_text_signature_from_internal_doc(zelf.0.value.name, doc)?; - Some(signature) - } -} - -impl Representable for PyNativeFunction { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(format!("<built-in function {}>", zelf.value.name)) - } -} - -impl Unconstructible for PyNativeFunction {} - -// `PyCMethodObject` in CPython -#[pyclass(name = "builtin_method", module = false, base = "PyNativeFunction")] -pub struct PyNativeMethod { - pub(crate) func: PyNativeFunction, - pub(crate) class: &'static Py<PyType>, // TODO: the actual life is &'self -} - -#[pyclass(with(Callable, Comparable, Representable), flags(HAS_DICT))] -impl PyNativeMethod { - #[pygetset(magic)] - fn qualname(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let prefix = zelf.class.name().to_string(); - Ok(vm - .ctx - .new_str(format!("{}.{}", prefix, &zelf.func.value.name))) - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, (PyObjectRef, &'static str))> { - // TODO: return (getattr, (self.object, self.name)) if this is a method - let getattr = vm.builtins.get_attr("getattr", vm)?; - let target = self - .func - .zelf - .clone() - .unwrap_or_else(|| self.class.to_owned().into()); - let name = self.func.value.name; - Ok((getattr, (target, name))) - } - - #[pygetset(name = "__self__")] - fn __self__(zelf: PyRef<Self>, _vm: &VirtualMachine) -> Option<PyObjectRef> { - zelf.func.zelf.clone() - } -} - -impl PyPayload for PyNativeMethod { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.builtin_method_type - } -} - -impl fmt::Debug for PyNativeMethod { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "builtin method of {:?} with {:?}", - &*self.class.name(), - &self.func - ) - } -} - -impl Comparable for PyNativeMethod { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - _vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - op.eq_only(|| { - if let Some(other) = other.payload::<Self>() { - let eq = match (zelf.func.zelf.as_ref(), other.func.zelf.as_ref()) { - (Some(z), Some(o)) => z.is(o), - (None, None) => true, - _ => false, - }; - let eq = eq && std::ptr::eq(zelf.func.value, other.func.value); - Ok(eq.into()) - } else { - Ok(PyComparisonValue::NotImplemented) - } - }) - } -} - -impl Callable for PyNativeMethod { - type Args = FuncArgs; - #[inline] - fn call(zelf: &Py<Self>, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { - if let Some(zelf) = &zelf.func.zelf { - args.prepend_arg(zelf.clone()); - } - (zelf.func.value.func)(vm, args) - } -} - -impl Representable for PyNativeMethod { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(format!( - "<built-in method {} of {} object at ...>", - &zelf.func.value.name, - zelf.class.name() - )) - } -} - -impl Unconstructible for PyNativeMethod {} - -pub fn init(context: &Context) { - PyNativeFunction::extend_class(context, context.types.builtin_function_or_method_type); - PyNativeMethod::extend_class(context, context.types.builtin_method_type); -} - -struct NativeFunctionOrMethod(PyRef<PyNativeFunction>); - -impl TryFromObject for NativeFunctionOrMethod { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let class = vm.ctx.types.builtin_function_or_method_type; - if obj.fast_isinstance(class) { - Ok(NativeFunctionOrMethod(unsafe { obj.downcast_unchecked() })) - } else { - Err(vm.new_downcast_type_error(class, &obj)) - } - } -} diff --git a/vm/src/builtins/classmethod.rs b/vm/src/builtins/classmethod.rs deleted file mode 100644 index 02f836199eb..00000000000 --- a/vm/src/builtins/classmethod.rs +++ /dev/null @@ -1,184 +0,0 @@ -use super::{PyBoundMethod, PyStr, PyType, PyTypeRef}; -use crate::{ - class::PyClassImpl, - common::lock::PyMutex, - types::{Constructor, GetDescriptor, Initializer, Representable}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -/// classmethod(function) -> method -/// -/// Convert a function to be a class method. -/// -/// A class method receives the class as implicit first argument, -/// just like an instance method receives the instance. -/// To declare a class method, use this idiom: -/// -/// class C: -/// @classmethod -/// def f(cls, arg1, arg2, ...): -/// ... -/// -/// It can be called either on the class (e.g. C.f()) or on an instance -/// (e.g. C().f()). The instance is ignored except for its class. -/// If a class method is called for a derived class, the derived class -/// object is passed as the implied first argument. -/// -/// Class methods are different than C++ or Java static methods. -/// If you want those, see the staticmethod builtin. -#[pyclass(module = false, name = "classmethod")] -#[derive(Debug)] -pub struct PyClassMethod { - callable: PyMutex<PyObjectRef>, -} - -impl From<PyObjectRef> for PyClassMethod { - fn from(callable: PyObjectRef) -> Self { - Self { - callable: PyMutex::new(callable), - } - } -} - -impl PyPayload for PyClassMethod { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.classmethod_type - } -} - -impl GetDescriptor for PyClassMethod { - fn descr_get( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let (zelf, _obj) = Self::_unwrap(&zelf, obj, vm)?; - let cls = cls.unwrap_or_else(|| _obj.class().to_owned().into()); - let call_descr_get: PyResult<PyObjectRef> = zelf.callable.lock().get_attr("__get__", vm); - match call_descr_get { - Err(_) => Ok(PyBoundMethod::new_ref(cls, zelf.callable.lock().clone(), &vm.ctx).into()), - Ok(call_descr_get) => call_descr_get.call((cls.clone(), cls), vm), - } - } -} - -impl Constructor for PyClassMethod { - type Args = PyObjectRef; - - fn py_new(cls: PyTypeRef, callable: Self::Args, vm: &VirtualMachine) -> PyResult { - let doc = callable.get_attr("__doc__", vm); - - let result = PyClassMethod { - callable: PyMutex::new(callable), - } - .into_ref_with_type(vm, cls)?; - let obj = PyObjectRef::from(result); - - if let Ok(doc) = doc { - obj.set_attr("__doc__", doc, vm)?; - } - - Ok(obj) - } -} - -impl Initializer for PyClassMethod { - type Args = PyObjectRef; - - fn init(zelf: PyRef<Self>, callable: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { - *zelf.callable.lock() = callable; - Ok(()) - } -} - -impl PyClassMethod { - pub fn new_ref(callable: PyObjectRef, ctx: &Context) -> PyRef<Self> { - PyRef::new_ref( - Self { - callable: PyMutex::new(callable), - }, - ctx.types.classmethod_type.to_owned(), - None, - ) - } -} - -#[pyclass( - with(GetDescriptor, Constructor, Representable), - flags(BASETYPE, HAS_DICT) -)] -impl PyClassMethod { - #[pygetset(magic)] - fn func(&self) -> PyObjectRef { - self.callable.lock().clone() - } - - #[pygetset(magic)] - fn wrapped(&self) -> PyObjectRef { - self.callable.lock().clone() - } - - #[pygetset(magic)] - fn module(&self, vm: &VirtualMachine) -> PyResult { - self.callable.lock().get_attr("__module__", vm) - } - - #[pygetset(magic)] - fn qualname(&self, vm: &VirtualMachine) -> PyResult { - self.callable.lock().get_attr("__qualname__", vm) - } - - #[pygetset(magic)] - fn name(&self, vm: &VirtualMachine) -> PyResult { - self.callable.lock().get_attr("__name__", vm) - } - - #[pygetset(magic)] - fn annotations(&self, vm: &VirtualMachine) -> PyResult { - self.callable.lock().get_attr("__annotations__", vm) - } - - #[pygetset(magic)] - fn isabstractmethod(&self, vm: &VirtualMachine) -> PyObjectRef { - match vm.get_attribute_opt(self.callable.lock().clone(), "__isabstractmethod__") { - Ok(Some(is_abstract)) => is_abstract, - _ => vm.ctx.new_bool(false).into(), - } - } - - #[pygetset(magic, setter)] - fn set_isabstractmethod(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.callable - .lock() - .set_attr("__isabstractmethod__", value, vm)?; - Ok(()) - } -} - -impl Representable for PyClassMethod { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let callable = zelf.callable.lock().repr(vm).unwrap(); - let class = Self::class(&vm.ctx); - - let repr = match ( - class - .qualname(vm) - .downcast_ref::<PyStr>() - .map(|n| n.as_str()), - class.module(vm).downcast_ref::<PyStr>().map(|m| m.as_str()), - ) { - (None, _) => return Err(vm.new_type_error("Unknown qualified name".into())), - (Some(qualname), Some(module)) if module != "builtins" => { - format!("<{module}.{qualname}({callable})>") - } - _ => format!("<{}({})>", class.slot_name(), callable), - }; - Ok(repr) - } -} - -pub(crate) fn init(context: &Context) { - PyClassMethod::extend_class(context, context.types.classmethod_type); -} diff --git a/vm/src/builtins/code.rs b/vm/src/builtins/code.rs deleted file mode 100644 index bedd71f241b..00000000000 --- a/vm/src/builtins/code.rs +++ /dev/null @@ -1,444 +0,0 @@ -/*! Infamous code object. The python class `code` - -*/ - -use super::{PyStrRef, PyTupleRef, PyType, PyTypeRef}; -use crate::{ - builtins::PyStrInterned, - bytecode::{self, AsBag, BorrowedConstant, CodeFlags, Constant, ConstantBag}, - class::{PyClassImpl, StaticType}, - convert::ToPyObject, - frozen, - function::{FuncArgs, OptionalArg}, - source_code::OneIndexed, - types::Representable, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; -use malachite_bigint::BigInt; -use num_traits::Zero; -use std::{borrow::Borrow, fmt, ops::Deref}; - -#[derive(FromArgs)] -pub struct ReplaceArgs { - #[pyarg(named, optional)] - co_posonlyargcount: OptionalArg<u32>, - #[pyarg(named, optional)] - co_argcount: OptionalArg<u32>, - #[pyarg(named, optional)] - co_kwonlyargcount: OptionalArg<u32>, - #[pyarg(named, optional)] - co_filename: OptionalArg<PyStrRef>, - #[pyarg(named, optional)] - co_firstlineno: OptionalArg<u32>, - #[pyarg(named, optional)] - co_consts: OptionalArg<Vec<PyObjectRef>>, - #[pyarg(named, optional)] - co_name: OptionalArg<PyStrRef>, - #[pyarg(named, optional)] - co_names: OptionalArg<Vec<PyObjectRef>>, - #[pyarg(named, optional)] - co_flags: OptionalArg<u16>, - #[pyarg(named, optional)] - co_varnames: OptionalArg<Vec<PyObjectRef>>, -} - -#[derive(Clone)] -#[repr(transparent)] -pub struct Literal(PyObjectRef); - -impl Borrow<PyObject> for Literal { - fn borrow(&self) -> &PyObject { - &self.0 - } -} - -impl From<Literal> for PyObjectRef { - fn from(obj: Literal) -> Self { - obj.0 - } -} - -fn borrow_obj_constant(obj: &PyObject) -> BorrowedConstant<Literal> { - match_class!(match obj { - ref i @ super::int::PyInt => { - let value = i.as_bigint(); - if obj.class().is(super::bool_::PyBool::static_type()) { - BorrowedConstant::Boolean { - value: !value.is_zero(), - } - } else { - BorrowedConstant::Integer { value } - } - } - ref f @ super::float::PyFloat => BorrowedConstant::Float { value: f.to_f64() }, - ref c @ super::complex::PyComplex => BorrowedConstant::Complex { - value: c.to_complex() - }, - ref s @ super::pystr::PyStr => BorrowedConstant::Str { value: s.as_str() }, - ref b @ super::bytes::PyBytes => BorrowedConstant::Bytes { - value: b.as_bytes() - }, - ref c @ PyCode => { - BorrowedConstant::Code { code: &c.code } - } - ref t @ super::tuple::PyTuple => { - let elements = t.as_slice(); - // SAFETY: Literal is repr(transparent) over PyObjectRef, and a Literal tuple only ever - // has other literals as elements - let elements = unsafe { &*(elements as *const [PyObjectRef] as *const [Literal]) }; - BorrowedConstant::Tuple { elements } - } - super::singletons::PyNone => BorrowedConstant::None, - super::slice::PyEllipsis => BorrowedConstant::Ellipsis, - _ => panic!("unexpected payload for constant python value"), - }) -} - -impl Constant for Literal { - type Name = &'static PyStrInterned; - fn borrow_constant(&self) -> BorrowedConstant<Self> { - borrow_obj_constant(&self.0) - } -} - -impl<'a> AsBag for &'a Context { - type Bag = PyObjBag<'a>; - fn as_bag(self) -> PyObjBag<'a> { - PyObjBag(self) - } -} -impl<'a> AsBag for &'a VirtualMachine { - type Bag = PyObjBag<'a>; - fn as_bag(self) -> PyObjBag<'a> { - PyObjBag(&self.ctx) - } -} - -#[derive(Clone, Copy)] -pub struct PyObjBag<'a>(pub &'a Context); - -impl ConstantBag for PyObjBag<'_> { - type Constant = Literal; - - fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant { - let ctx = self.0; - let obj = match constant { - bytecode::BorrowedConstant::Integer { value } => ctx.new_bigint(value).into(), - bytecode::BorrowedConstant::Float { value } => ctx.new_float(value).into(), - bytecode::BorrowedConstant::Complex { value } => ctx.new_complex(value).into(), - bytecode::BorrowedConstant::Str { value } if value.len() <= 20 => { - ctx.intern_str(value).to_object() - } - bytecode::BorrowedConstant::Str { value } => ctx.new_str(value).into(), - bytecode::BorrowedConstant::Bytes { value } => ctx.new_bytes(value.to_vec()).into(), - bytecode::BorrowedConstant::Boolean { value } => ctx.new_bool(value).into(), - bytecode::BorrowedConstant::Code { code } => { - ctx.new_code(code.map_clone_bag(self)).into() - } - bytecode::BorrowedConstant::Tuple { elements } => { - let elements = elements - .iter() - .map(|constant| self.make_constant(constant.borrow_constant()).0) - .collect(); - ctx.new_tuple(elements).into() - } - bytecode::BorrowedConstant::None => ctx.none(), - bytecode::BorrowedConstant::Ellipsis => ctx.ellipsis(), - }; - Literal(obj) - } - - fn make_name(&self, name: &str) -> &'static PyStrInterned { - self.0.intern_str(name) - } - - fn make_int(&self, value: BigInt) -> Self::Constant { - Literal(self.0.new_int(value).into()) - } - - fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant { - Literal(self.0.new_tuple(elements.map(|lit| lit.0).collect()).into()) - } - - fn make_code(&self, code: CodeObject) -> Self::Constant { - Literal(self.0.new_code(code).into()) - } -} - -pub type CodeObject = bytecode::CodeObject<Literal>; - -pub trait IntoCodeObject { - fn into_code_object(self, ctx: &Context) -> CodeObject; -} - -impl IntoCodeObject for CodeObject { - fn into_code_object(self, _ctx: &Context) -> CodeObject { - self - } -} - -impl IntoCodeObject for bytecode::CodeObject { - fn into_code_object(self, ctx: &Context) -> CodeObject { - self.map_bag(PyObjBag(ctx)) - } -} - -impl<B: AsRef<[u8]>> IntoCodeObject for frozen::FrozenCodeObject<B> { - fn into_code_object(self, ctx: &Context) -> CodeObject { - self.decode(ctx) - } -} - -#[pyclass(module = false, name = "code")] -pub struct PyCode { - pub code: CodeObject, -} - -impl Deref for PyCode { - type Target = CodeObject; - fn deref(&self) -> &Self::Target { - &self.code - } -} - -impl PyCode { - pub fn new(code: CodeObject) -> PyCode { - PyCode { code } - } -} - -impl fmt::Debug for PyCode { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "code: {:?}", self.code) - } -} - -impl PyPayload for PyCode { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.code_type - } -} - -impl Representable for PyCode { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - let code = &zelf.code; - Ok(format!( - "<code object {} at {:#x} file {:?}, line {}>", - code.obj_name, - zelf.get_id(), - code.source_path.as_str(), - code.first_line_number.map_or(-1, |n| n.get() as i32) - )) - } -} - -#[pyclass(with(Representable))] -impl PyCode { - #[pyslot] - fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("Cannot directly create code object".to_owned())) - } - - #[pygetset] - fn co_posonlyargcount(&self) -> usize { - self.code.posonlyarg_count as usize - } - - #[pygetset] - fn co_argcount(&self) -> usize { - self.code.arg_count as usize - } - - #[pygetset] - fn co_stacksize(&self) -> u32 { - self.code.max_stackdepth - } - - #[pygetset] - pub fn co_filename(&self) -> PyStrRef { - self.code.source_path.to_owned() - } - - #[pygetset] - pub fn co_cellvars(&self, vm: &VirtualMachine) -> PyTupleRef { - let cellvars = self - .code - .cellvars - .deref() - .iter() - .map(|name| name.to_pyobject(vm)) - .collect(); - vm.ctx.new_tuple(cellvars) - } - - #[pygetset] - fn co_nlocals(&self) -> usize { - self.varnames.len() - } - - #[pygetset] - fn co_firstlineno(&self) -> u32 { - self.code.first_line_number.map_or(0, |n| n.get()) - } - - #[pygetset] - fn co_kwonlyargcount(&self) -> usize { - self.code.kwonlyarg_count as usize - } - - #[pygetset] - fn co_consts(&self, vm: &VirtualMachine) -> PyTupleRef { - let consts = self.code.constants.iter().map(|x| x.0.clone()).collect(); - vm.ctx.new_tuple(consts) - } - - #[pygetset] - fn co_name(&self) -> PyStrRef { - self.code.obj_name.to_owned() - } - - #[pygetset] - fn co_names(&self, vm: &VirtualMachine) -> PyTupleRef { - let names = self - .code - .names - .deref() - .iter() - .map(|name| name.to_pyobject(vm)) - .collect(); - vm.ctx.new_tuple(names) - } - - #[pygetset] - fn co_flags(&self) -> u16 { - self.code.flags.bits() - } - - #[pygetset] - pub fn co_varnames(&self, vm: &VirtualMachine) -> PyTupleRef { - let varnames = self.code.varnames.iter().map(|s| s.to_object()).collect(); - vm.ctx.new_tuple(varnames) - } - - #[pygetset] - pub fn co_freevars(&self, vm: &VirtualMachine) -> PyTupleRef { - let names = self - .code - .freevars - .deref() - .iter() - .map(|name| name.to_pyobject(vm)) - .collect(); - vm.ctx.new_tuple(names) - } - - #[pymethod] - pub fn replace(&self, args: ReplaceArgs, vm: &VirtualMachine) -> PyResult<PyCode> { - let posonlyarg_count = match args.co_posonlyargcount { - OptionalArg::Present(posonlyarg_count) => posonlyarg_count, - OptionalArg::Missing => self.code.posonlyarg_count, - }; - - let arg_count = match args.co_argcount { - OptionalArg::Present(arg_count) => arg_count, - OptionalArg::Missing => self.code.arg_count, - }; - - let source_path = match args.co_filename { - OptionalArg::Present(source_path) => source_path, - OptionalArg::Missing => self.code.source_path.to_owned(), - }; - - let first_line_number = match args.co_firstlineno { - OptionalArg::Present(first_line_number) => OneIndexed::new(first_line_number), - OptionalArg::Missing => self.code.first_line_number, - }; - - let kwonlyarg_count = match args.co_kwonlyargcount { - OptionalArg::Present(kwonlyarg_count) => kwonlyarg_count, - OptionalArg::Missing => self.code.kwonlyarg_count, - }; - - let constants = match args.co_consts { - OptionalArg::Present(constants) => constants, - OptionalArg::Missing => self.code.constants.iter().map(|x| x.0.clone()).collect(), - }; - - let obj_name = match args.co_name { - OptionalArg::Present(obj_name) => obj_name, - OptionalArg::Missing => self.code.obj_name.to_owned(), - }; - - let names = match args.co_names { - OptionalArg::Present(names) => names, - OptionalArg::Missing => self - .code - .names - .deref() - .iter() - .map(|name| name.to_pyobject(vm)) - .collect(), - }; - - let flags = match args.co_flags { - OptionalArg::Present(flags) => flags, - OptionalArg::Missing => self.code.flags.bits(), - }; - - let varnames = match args.co_varnames { - OptionalArg::Present(varnames) => varnames, - OptionalArg::Missing => self.code.varnames.iter().map(|s| s.to_object()).collect(), - }; - - Ok(PyCode { - code: CodeObject { - flags: CodeFlags::from_bits_truncate(flags), - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path: source_path.as_object().as_interned_str(vm).unwrap(), - first_line_number, - obj_name: obj_name.as_object().as_interned_str(vm).unwrap(), - - max_stackdepth: self.code.max_stackdepth, - instructions: self.code.instructions.clone(), - locations: self.code.locations.clone(), - constants: constants.into_iter().map(Literal).collect(), - names: names - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), - varnames: varnames - .into_iter() - .map(|o| o.as_interned_str(vm).unwrap()) - .collect(), - cellvars: self.code.cellvars.clone(), - freevars: self.code.freevars.clone(), - cell2arg: self.code.cell2arg.clone(), - }, - }) - } -} - -impl fmt::Display for PyCode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - (**self).fmt(f) - } -} - -impl ToPyObject for CodeObject { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_code(self).into() - } -} - -impl ToPyObject for bytecode::CodeObject { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_code(self).into() - } -} - -pub fn init(ctx: &Context) { - PyCode::extend_class(ctx, ctx.types.code_type); -} diff --git a/vm/src/builtins/complex.rs b/vm/src/builtins/complex.rs deleted file mode 100644 index 609dcb9b6cc..00000000000 --- a/vm/src/builtins/complex.rs +++ /dev/null @@ -1,551 +0,0 @@ -use super::{float, PyStr, PyType, PyTypeRef}; -use crate::{ - class::PyClassImpl, - convert::{ToPyObject, ToPyResult}, - function::{ - OptionalArg, OptionalOption, - PyArithmeticValue::{self, *}, - PyComparisonValue, - }, - identifier, - protocol::PyNumberMethods, - types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use num_complex::Complex64; -use num_traits::Zero; -use rustpython_common::hash; -use std::num::Wrapping; - -/// Create a complex number from a real part and an optional imaginary part. -/// -/// This is equivalent to (real + imag*1j) where imag defaults to 0. -#[pyclass(module = false, name = "complex")] -#[derive(Debug, Copy, Clone, PartialEq)] -pub struct PyComplex { - value: Complex64, -} - -impl PyComplex { - pub fn to_complex64(self) -> Complex64 { - self.value - } -} - -impl PyPayload for PyComplex { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.complex_type - } -} - -impl ToPyObject for Complex64 { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - PyComplex::new_ref(self, &vm.ctx).into() - } -} - -impl From<Complex64> for PyComplex { - fn from(value: Complex64) -> Self { - PyComplex { value } - } -} - -impl PyObjectRef { - /// Tries converting a python object into a complex, returns an option of whether the complex - /// and whether the object was a complex originally or coereced into one - pub fn try_complex(&self, vm: &VirtualMachine) -> PyResult<Option<(Complex64, bool)>> { - if let Some(complex) = self.payload_if_exact::<PyComplex>(vm) { - return Ok(Some((complex.value, true))); - } - if let Some(method) = vm.get_method(self.clone(), identifier!(vm, __complex__)) { - let result = method?.call((), vm)?; - // TODO: returning strict subclasses of complex in __complex__ is deprecated - return match result.payload::<PyComplex>() { - Some(complex_obj) => Ok(Some((complex_obj.value, true))), - None => Err(vm.new_type_error(format!( - "__complex__ returned non-complex (type '{}')", - result.class().name() - ))), - }; - } - // `complex` does not have a `__complex__` by default, so subclasses might not either, - // use the actual stored value in this case - if let Some(complex) = self.payload_if_subclass::<PyComplex>(vm) { - return Ok(Some((complex.value, true))); - } - if let Some(float) = self.try_float_opt(vm) { - return Ok(Some((Complex64::new(float?.to_f64(), 0.0), false))); - } - Ok(None) - } -} - -pub fn init(context: &Context) { - PyComplex::extend_class(context, context.types.complex_type); -} - -fn to_op_complex(value: &PyObject, vm: &VirtualMachine) -> PyResult<Option<Complex64>> { - let r = if let Some(complex) = value.payload_if_subclass::<PyComplex>(vm) { - Some(complex.value) - } else { - float::to_op_float(value, vm)?.map(|float| Complex64::new(float, 0.0)) - }; - Ok(r) -} - -fn inner_div(v1: Complex64, v2: Complex64, vm: &VirtualMachine) -> PyResult<Complex64> { - if v2.is_zero() { - return Err(vm.new_zero_division_error("complex division by zero".to_owned())); - } - - Ok(v1.fdiv(v2)) -} - -fn inner_pow(v1: Complex64, v2: Complex64, vm: &VirtualMachine) -> PyResult<Complex64> { - if v1.is_zero() { - return if v2.im != 0.0 { - let msg = format!("{v1} cannot be raised to a negative or complex power"); - Err(vm.new_zero_division_error(msg)) - } else if v2.is_zero() { - Ok(Complex64::new(1.0, 0.0)) - } else { - Ok(Complex64::new(0.0, 0.0)) - }; - } - - let ans = v1.powc(v2); - if ans.is_infinite() && !(v1.is_infinite() || v2.is_infinite()) { - Err(vm.new_overflow_error("complex exponentiation overflow".to_owned())) - } else { - Ok(ans) - } -} - -impl Constructor for PyComplex { - type Args = ComplexArgs; - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let imag_missing = args.imag.is_missing(); - let (real, real_was_complex) = match args.real { - OptionalArg::Missing => (Complex64::new(0.0, 0.0), false), - OptionalArg::Present(val) => { - let val = if cls.is(vm.ctx.types.complex_type) && imag_missing { - match val.downcast_exact::<PyComplex>(vm) { - Ok(c) => { - return Ok(c.into_pyref().into()); - } - Err(val) => val, - } - } else { - val - }; - - if let Some(c) = val.try_complex(vm)? { - c - } else if let Some(s) = val.payload_if_subclass::<PyStr>(vm) { - if args.imag.is_present() { - return Err(vm.new_type_error( - "complex() can't take second arg if first is a string".to_owned(), - )); - } - let value = parse_str(s.as_str().trim()).ok_or_else(|| { - vm.new_value_error("complex() arg is a malformed string".to_owned()) - })?; - return Self::from(value) - .into_ref_with_type(vm, cls) - .map(Into::into); - } else { - return Err(vm.new_type_error(format!( - "complex() first argument must be a string or a number, not '{}'", - val.class().name() - ))); - } - } - }; - - let (imag, imag_was_complex) = match args.imag { - // Copy the imaginary from the real to the real of the imaginary - // if an imaginary argument is not passed in - OptionalArg::Missing => (Complex64::new(real.im, 0.0), false), - OptionalArg::Present(obj) => { - if let Some(c) = obj.try_complex(vm)? { - c - } else if obj.class().fast_issubclass(vm.ctx.types.str_type) { - return Err( - vm.new_type_error("complex() second arg can't be a string".to_owned()) - ); - } else { - return Err(vm.new_type_error(format!( - "complex() second argument must be a number, not '{}'", - obj.class().name() - ))); - } - } - }; - - let final_real = if imag_was_complex { - real.re - imag.im - } else { - real.re - }; - - let final_imag = if real_was_complex && !imag_missing { - imag.re + real.im - } else { - imag.re - }; - let value = Complex64::new(final_real, final_imag); - Self::from(value) - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -impl PyComplex { - pub fn new_ref(value: Complex64, ctx: &Context) -> PyRef<Self> { - PyRef::new_ref(Self::from(value), ctx.types.complex_type.to_owned(), None) - } - - pub fn to_complex(&self) -> Complex64 { - self.value - } -} - -#[pyclass( - flags(BASETYPE), - with(Comparable, Hashable, Constructor, AsNumber, Representable) -)] -impl PyComplex { - #[pymethod(magic)] - fn complex(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRef<PyComplex> { - if zelf.is(vm.ctx.types.complex_type) { - zelf - } else { - PyComplex::from(zelf.value).into_ref(&vm.ctx) - } - } - - #[pygetset] - fn real(&self) -> f64 { - self.value.re - } - - #[pygetset] - fn imag(&self) -> f64 { - self.value.im - } - - #[pymethod(magic)] - fn abs(&self, vm: &VirtualMachine) -> PyResult<f64> { - let Complex64 { im, re } = self.value; - let is_finite = im.is_finite() && re.is_finite(); - let abs_result = re.hypot(im); - if is_finite && abs_result.is_infinite() { - Err(vm.new_overflow_error("absolute value too large".to_string())) - } else { - Ok(abs_result) - } - } - - #[inline] - fn op<F>( - &self, - other: PyObjectRef, - op: F, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> - where - F: Fn(Complex64, Complex64) -> PyResult<Complex64>, - { - to_op_complex(&other, vm)?.map_or_else( - || Ok(NotImplemented), - |other| Ok(Implemented(op(self.value, other)?)), - ) - } - - #[pymethod(name = "__radd__")] - #[pymethod(magic)] - fn add( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| Ok(a + b), vm) - } - - #[pymethod(magic)] - fn sub( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| Ok(a - b), vm) - } - - #[pymethod(magic)] - fn rsub( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| Ok(b - a), vm) - } - - #[pymethod] - fn conjugate(&self) -> Complex64 { - self.value.conj() - } - - #[pymethod(name = "__rmul__")] - #[pymethod(magic)] - fn mul( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| Ok(a * b), vm) - } - - #[pymethod(magic)] - fn truediv( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| inner_div(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rtruediv( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| inner_div(b, a, vm), vm) - } - - #[pymethod(magic)] - fn pos(&self) -> Complex64 { - self.value - } - - #[pymethod(magic)] - fn neg(&self) -> Complex64 { - -self.value - } - - #[pymethod(magic)] - fn pow( - &self, - other: PyObjectRef, - mod_val: OptionalOption<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - if mod_val.flatten().is_some() { - Err(vm.new_value_error("complex modulo not allowed".to_owned())) - } else { - self.op(other, |a, b| inner_pow(a, b, vm), vm) - } - } - - #[pymethod(magic)] - fn rpow( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Complex64>> { - self.op(other, |a, b| inner_pow(b, a, vm), vm) - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - !Complex64::is_zero(&self.value) - } - - #[pymethod(magic)] - fn getnewargs(&self) -> (f64, f64) { - let Complex64 { re, im } = self.value; - (re, im) - } -} - -impl Comparable for PyComplex { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - op.eq_only(|| { - let result = if let Some(other) = other.payload_if_subclass::<PyComplex>(vm) { - if zelf.value.re.is_nan() - && zelf.value.im.is_nan() - && other.value.re.is_nan() - && other.value.im.is_nan() - { - true - } else { - zelf.value == other.value - } - } else { - match float::to_op_float(other, vm) { - Ok(Some(other)) => zelf.value == other.into(), - Err(_) => false, - Ok(None) => return Ok(PyComparisonValue::NotImplemented), - } - }; - Ok(PyComparisonValue::Implemented(result)) - }) - } -} - -impl Hashable for PyComplex { - #[inline] - fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<hash::PyHash> { - let value = zelf.value; - - let re_hash = - hash::hash_float(value.re).unwrap_or_else(|| hash::hash_object_id(zelf.get_id())); - - let im_hash = - hash::hash_float(value.im).unwrap_or_else(|| hash::hash_object_id(zelf.get_id())); - - let Wrapping(ret) = Wrapping(re_hash) + Wrapping(im_hash) * Wrapping(hash::IMAG); - Ok(hash::fix_sentinel(ret)) - } -} - -impl AsNumber for PyComplex { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - add: Some(|a, b, vm| PyComplex::number_op(a, b, |a, b, _vm| a + b, vm)), - subtract: Some(|a, b, vm| PyComplex::number_op(a, b, |a, b, _vm| a - b, vm)), - multiply: Some(|a, b, vm| PyComplex::number_op(a, b, |a, b, _vm| a * b, vm)), - power: Some(|a, b, c, vm| { - if vm.is_none(c) { - PyComplex::number_op(a, b, inner_pow, vm) - } else { - Err(vm.new_value_error(String::from("complex modulo"))) - } - }), - negative: Some(|number, vm| { - let value = PyComplex::number_downcast(number).value; - (-value).to_pyresult(vm) - }), - positive: Some(|number, vm| { - PyComplex::number_downcast_exact(number, vm).to_pyresult(vm) - }), - absolute: Some(|number, vm| { - let value = PyComplex::number_downcast(number).value; - value.norm().to_pyresult(vm) - }), - boolean: Some(|number, _vm| Ok(PyComplex::number_downcast(number).value.is_zero())), - true_divide: Some(|a, b, vm| PyComplex::number_op(a, b, inner_div, vm)), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } - - fn clone_exact(zelf: &Py<Self>, vm: &VirtualMachine) -> PyRef<Self> { - vm.ctx.new_complex(zelf.value) - } -} - -impl Representable for PyComplex { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - // TODO: when you fix this, move it to rustpython_common::complex::repr and update - // ast/src/unparse.rs + impl Display for Constant in ast/src/constant.rs - let Complex64 { re, im } = zelf.value; - // integer => drop ., fractional => float_ops - let mut im_part = if im.fract() == 0.0 { - im.to_string() - } else { - crate::literal::float::to_string(im) - }; - im_part.push('j'); - - // positive empty => return im_part, integer => drop ., fractional => float_ops - let re_part = if re == 0.0 { - if re.is_sign_positive() { - return Ok(im_part); - } else { - re.to_string() - } - } else if re.fract() == 0.0 { - re.to_string() - } else { - crate::literal::float::to_string(re) - }; - let mut result = String::with_capacity( - re_part.len() + im_part.len() + 2 + im.is_sign_positive() as usize, - ); - result.push('('); - result.push_str(&re_part); - if im.is_sign_positive() || im.is_nan() { - result.push('+'); - } - result.push_str(&im_part); - result.push(')'); - Ok(result) - } -} - -impl PyComplex { - fn number_op<F, R>(a: &PyObject, b: &PyObject, op: F, vm: &VirtualMachine) -> PyResult - where - F: FnOnce(Complex64, Complex64, &VirtualMachine) -> R, - R: ToPyResult, - { - if let (Some(a), Some(b)) = (to_op_complex(a, vm)?, to_op_complex(b, vm)?) { - op(a, b, vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - } -} - -#[derive(FromArgs)] -pub struct ComplexArgs { - #[pyarg(any, optional)] - real: OptionalArg<PyObjectRef>, - #[pyarg(any, optional)] - imag: OptionalArg<PyObjectRef>, -} - -fn parse_str(s: &str) -> Option<Complex64> { - // Handle parentheses - let s = match s.strip_prefix('(') { - None => s, - Some(s) => match s.strip_suffix(')') { - None => return None, - Some(s) => s.trim(), - }, - }; - - let value = match s.strip_suffix(|c| c == 'j' || c == 'J') { - None => Complex64::new(crate::literal::float::parse_str(s)?, 0.0), - Some(mut s) => { - let mut real = 0.0; - // Find the central +/- operator. If it exists, parse the real part. - for (i, w) in s.as_bytes().windows(2).enumerate() { - if (w[1] == b'+' || w[1] == b'-') && !(w[0] == b'e' || w[0] == b'E') { - real = crate::literal::float::parse_str(&s[..=i])?; - s = &s[i + 1..]; - break; - } - } - - let imag = match s { - // "j", "+j" - "" | "+" => 1.0, - // "-j" - "-" => -1.0, - s => crate::literal::float::parse_str(s)?, - }; - - Complex64::new(real, imag) - } - }; - Some(value) -} diff --git a/vm/src/builtins/coroutine.rs b/vm/src/builtins/coroutine.rs deleted file mode 100644 index 2454e27e2c2..00000000000 --- a/vm/src/builtins/coroutine.rs +++ /dev/null @@ -1,160 +0,0 @@ -use super::{PyCode, PyStrRef, PyType}; -use crate::{ - class::PyClassImpl, - coroutine::Coro, - frame::FrameRef, - function::OptionalArg, - protocol::PyIterReturn, - types::{Constructor, IterNext, Iterable, Representable, SelfIter, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "coroutine")] -#[derive(Debug)] -// PyCoro_Type in CPython -pub struct PyCoroutine { - inner: Coro, -} - -impl PyPayload for PyCoroutine { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.coroutine_type - } -} - -#[pyclass(with(Constructor, IterNext, Representable))] -impl PyCoroutine { - pub fn as_coro(&self) -> &Coro { - &self.inner - } - - pub fn new(frame: FrameRef, name: PyStrRef) -> Self { - PyCoroutine { - inner: Coro::new(frame, name), - } - } - - #[pygetset(magic)] - fn name(&self) -> PyStrRef { - self.inner.name() - } - - #[pygetset(magic, setter)] - fn set_name(&self, name: PyStrRef) { - self.inner.set_name(name) - } - - #[pymethod] - fn send(zelf: &Py<Self>, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.inner.send(zelf.as_object(), value, vm) - } - - #[pymethod] - fn throw( - zelf: &Py<Self>, - exc_type: PyObjectRef, - exc_val: OptionalArg, - exc_tb: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn> { - zelf.inner.throw( - zelf.as_object(), - exc_type, - exc_val.unwrap_or_none(vm), - exc_tb.unwrap_or_none(vm), - vm, - ) - } - - #[pymethod] - fn close(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()> { - zelf.inner.close(zelf.as_object(), vm) - } - - #[pymethod(name = "__await__")] - fn r#await(zelf: PyRef<Self>) -> PyCoroutineWrapper { - PyCoroutineWrapper { coro: zelf } - } - - #[pygetset] - fn cr_await(&self, _vm: &VirtualMachine) -> Option<PyObjectRef> { - self.inner.frame().yield_from_target() - } - #[pygetset] - fn cr_frame(&self, _vm: &VirtualMachine) -> FrameRef { - self.inner.frame() - } - #[pygetset] - fn cr_running(&self, _vm: &VirtualMachine) -> bool { - self.inner.running() - } - #[pygetset] - fn cr_code(&self, _vm: &VirtualMachine) -> PyRef<PyCode> { - self.inner.frame().code.clone() - } - // TODO: coroutine origin tracking: - // https://docs.python.org/3/library/sys.html#sys.set_coroutine_origin_tracking_depth - #[pygetset] - fn cr_origin(&self, _vm: &VirtualMachine) -> Option<(PyStrRef, usize, PyStrRef)> { - None - } -} -impl Unconstructible for PyCoroutine {} - -impl Representable for PyCoroutine { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - Ok(zelf.inner.repr(zelf.as_object(), zelf.get_id(), vm)) - } -} - -impl SelfIter for PyCoroutine {} -impl IterNext for PyCoroutine { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - Self::send(zelf, vm.ctx.none(), vm) - } -} - -#[pyclass(module = false, name = "coroutine_wrapper")] -#[derive(Debug)] -// PyCoroWrapper_Type in CPython -pub struct PyCoroutineWrapper { - coro: PyRef<PyCoroutine>, -} - -impl PyPayload for PyCoroutineWrapper { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.coroutine_wrapper_type - } -} - -#[pyclass(with(IterNext, Iterable))] -impl PyCoroutineWrapper { - #[pymethod] - fn send(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - PyCoroutine::send(&self.coro, val, vm) - } - - #[pymethod] - fn throw( - &self, - exc_type: PyObjectRef, - exc_val: OptionalArg, - exc_tb: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn> { - PyCoroutine::throw(&self.coro, exc_type, exc_val, exc_tb, vm) - } -} - -impl SelfIter for PyCoroutineWrapper {} -impl IterNext for PyCoroutineWrapper { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - Self::send(zelf, vm.ctx.none(), vm) - } -} - -pub fn init(ctx: &Context) { - PyCoroutine::extend_class(ctx, ctx.types.coroutine_type); - PyCoroutineWrapper::extend_class(ctx, ctx.types.coroutine_wrapper_type); -} diff --git a/vm/src/builtins/descriptor.rs b/vm/src/builtins/descriptor.rs deleted file mode 100644 index 6b92e4c36ba..00000000000 --- a/vm/src/builtins/descriptor.rs +++ /dev/null @@ -1,359 +0,0 @@ -use super::{PyStr, PyStrInterned, PyType}; -use crate::{ - builtins::{builtin_func::PyNativeMethod, type_}, - class::PyClassImpl, - function::{FuncArgs, PyMethodDef, PyMethodFlags, PySetterValue}, - types::{Callable, Constructor, GetDescriptor, Representable, Unconstructible}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use rustpython_common::lock::PyRwLock; - -#[derive(Debug)] -pub struct PyDescriptor { - pub typ: &'static Py<PyType>, - pub name: &'static PyStrInterned, - pub qualname: PyRwLock<Option<String>>, -} - -#[derive(Debug)] -pub struct PyDescriptorOwned { - pub typ: PyRef<PyType>, - pub name: &'static PyStrInterned, - pub qualname: PyRwLock<Option<String>>, -} - -#[pyclass(name = "method_descriptor", module = false)] -pub struct PyMethodDescriptor { - pub common: PyDescriptor, - pub method: &'static PyMethodDef, - // vectorcall: vectorcallfunc, -} - -impl PyMethodDescriptor { - pub fn new(method: &'static PyMethodDef, typ: &'static Py<PyType>, ctx: &Context) -> Self { - Self { - common: PyDescriptor { - typ, - name: ctx.intern_str(method.name), - qualname: PyRwLock::new(None), - }, - method, - } - } -} - -impl PyPayload for PyMethodDescriptor { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.method_descriptor_type - } -} - -impl std::fmt::Debug for PyMethodDescriptor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "method descriptor for '{}'", self.common.name) - } -} - -impl GetDescriptor for PyMethodDescriptor { - fn descr_get( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let descr = Self::_as_pyref(&zelf, vm).unwrap(); - let bound = match obj { - Some(obj) => { - if descr.method.flags.contains(PyMethodFlags::METHOD) { - if cls.map_or(false, |c| c.fast_isinstance(vm.ctx.types.type_type)) { - obj - } else { - return Err(vm.new_type_error(format!( - "descriptor '{}' needs a type, not '{}', as arg 2", - descr.common.name.as_str(), - obj.class().name() - ))); - } - } else if descr.method.flags.contains(PyMethodFlags::CLASS) { - obj.class().to_owned().into() - } else { - unimplemented!() - } - } - None if descr.method.flags.contains(PyMethodFlags::CLASS) => cls.unwrap(), - None => return Ok(zelf), - }; - // Ok(descr.method.build_bound_method(&vm.ctx, bound, class).into()) - Ok(descr.bind(bound, &vm.ctx).into()) - } -} - -impl Callable for PyMethodDescriptor { - type Args = FuncArgs; - #[inline] - fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - (zelf.method.func)(vm, args) - } -} - -impl PyMethodDescriptor { - pub fn bind(&self, obj: PyObjectRef, ctx: &Context) -> PyRef<PyNativeMethod> { - self.method.build_bound_method(ctx, obj, self.common.typ) - } -} - -#[pyclass( - with(GetDescriptor, Callable, Constructor, Representable), - flags(METHOD_DESCRIPTOR) -)] -impl PyMethodDescriptor { - #[pygetset(magic)] - fn name(&self) -> &'static PyStrInterned { - self.common.name - } - #[pygetset(magic)] - fn qualname(&self) -> String { - format!("{}.{}", self.common.typ.name(), &self.common.name) - } - #[pygetset(magic)] - fn doc(&self) -> Option<&'static str> { - self.method.doc - } - #[pygetset(magic)] - fn text_signature(&self) -> Option<String> { - self.method.doc.and_then(|doc| { - type_::get_text_signature_from_internal_doc(self.method.name, doc) - .map(|signature| signature.to_string()) - }) - } - #[pymethod(magic)] - fn reduce( - &self, - vm: &VirtualMachine, - ) -> (Option<PyObjectRef>, (Option<PyObjectRef>, &'static str)) { - let builtins_getattr = vm.builtins.get_attr("getattr", vm).ok(); - let classname = vm.builtins.get_attr(&self.common.typ.__name__(vm), vm).ok(); - (builtins_getattr, (classname, self.method.name)) - } -} - -impl Representable for PyMethodDescriptor { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(format!( - "<method '{}' of '{}' objects>", - &zelf.method.name, - zelf.common.typ.name() - )) - } -} - -impl Unconstructible for PyMethodDescriptor {} - -#[derive(Debug)] -pub enum MemberKind { - Bool = 14, - ObjectEx = 16, -} - -pub type MemberSetterFunc = Option<fn(&VirtualMachine, PyObjectRef, PySetterValue) -> PyResult<()>>; - -pub enum MemberGetter { - Getter(fn(&VirtualMachine, PyObjectRef) -> PyResult), - Offset(usize), -} - -pub enum MemberSetter { - Setter(MemberSetterFunc), - Offset(usize), -} - -pub struct PyMemberDef { - pub name: String, - pub kind: MemberKind, - pub getter: MemberGetter, - pub setter: MemberSetter, - pub doc: Option<String>, -} - -impl PyMemberDef { - fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - match self.getter { - MemberGetter::Getter(getter) => (getter)(vm, obj), - MemberGetter::Offset(offset) => get_slot_from_object(obj, offset, self, vm), - } - } - - fn set( - &self, - obj: PyObjectRef, - value: PySetterValue<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - match self.setter { - MemberSetter::Setter(setter) => match setter { - Some(setter) => (setter)(vm, obj, value), - None => Err(vm.new_attribute_error("readonly attribute".to_string())), - }, - MemberSetter::Offset(offset) => set_slot_at_object(obj, offset, self, value, vm), - } - } -} - -impl std::fmt::Debug for PyMemberDef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PyMemberDef") - .field("name", &self.name) - .field("kind", &self.kind) - .field("doc", &self.doc) - .finish() - } -} - -// PyMemberDescrObject in CPython -#[pyclass(name = "member_descriptor", module = false)] -#[derive(Debug)] -pub struct PyMemberDescriptor { - pub common: PyDescriptorOwned, - pub member: PyMemberDef, -} - -impl PyPayload for PyMemberDescriptor { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.member_descriptor_type - } -} - -fn calculate_qualname(descr: &PyDescriptorOwned, vm: &VirtualMachine) -> PyResult<Option<String>> { - if let Some(qualname) = vm.get_attribute_opt(descr.typ.clone().into(), "__qualname__")? { - let str = qualname.downcast::<PyStr>().map_err(|_| { - vm.new_type_error( - "<descriptor>.__objclass__.__qualname__ is not a unicode object".to_owned(), - ) - })?; - Ok(Some(format!("{}.{}", str, descr.name))) - } else { - Ok(None) - } -} - -#[pyclass(with(GetDescriptor, Constructor, Representable), flags(BASETYPE))] -impl PyMemberDescriptor { - #[pygetset(magic)] - fn doc(&self) -> Option<String> { - self.member.doc.to_owned() - } - - #[pygetset(magic)] - fn qualname(&self, vm: &VirtualMachine) -> PyResult<Option<String>> { - let qualname = self.common.qualname.read(); - Ok(if qualname.is_none() { - drop(qualname); - let calculated = calculate_qualname(&self.common, vm)?; - *self.common.qualname.write() = calculated.to_owned(); - calculated - } else { - qualname.to_owned() - }) - } - - #[pyslot] - fn descr_set( - zelf: &PyObject, - obj: PyObjectRef, - value: PySetterValue<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let zelf = Self::_as_pyref(zelf, vm)?; - zelf.member.set(obj, value, vm) - } -} - -// PyMember_GetOne -fn get_slot_from_object( - obj: PyObjectRef, - offset: usize, - member: &PyMemberDef, - vm: &VirtualMachine, -) -> PyResult { - let slot = match member.kind { - MemberKind::Bool => obj - .get_slot(offset) - .unwrap_or_else(|| vm.ctx.new_bool(false).into()), - MemberKind::ObjectEx => obj.get_slot(offset).ok_or_else(|| { - vm.new_attribute_error(format!( - "'{}' object has no attribute '{}'", - obj.class().name(), - member.name - )) - })?, - }; - Ok(slot) -} - -// PyMember_SetOne -fn set_slot_at_object( - obj: PyObjectRef, - offset: usize, - member: &PyMemberDef, - value: PySetterValue, - vm: &VirtualMachine, -) -> PyResult<()> { - match member.kind { - MemberKind::Bool => { - match value { - PySetterValue::Assign(v) => { - if !v.class().is(vm.ctx.types.bool_type) { - return Err( - vm.new_type_error("attribute value type must be bool".to_owned()) - ); - } - - obj.set_slot(offset, Some(v)) - } - PySetterValue::Delete => obj.set_slot(offset, None), - }; - } - MemberKind::ObjectEx => match value { - PySetterValue::Assign(v) => obj.set_slot(offset, Some(v)), - PySetterValue::Delete => obj.set_slot(offset, None), - }, - } - - Ok(()) -} - -impl Unconstructible for PyMemberDescriptor {} - -impl Representable for PyMemberDescriptor { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(format!( - "<member '{}' of '{}' objects>", - zelf.common.name, - zelf.common.typ.name(), - )) - } -} - -impl GetDescriptor for PyMemberDescriptor { - fn descr_get( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - _cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - match obj { - Some(x) => { - let zelf = Self::_as_pyref(&zelf, vm)?; - zelf.member.get(x, vm) - } - None => Ok(zelf), - } - } -} - -pub fn init(ctx: &Context) { - PyMemberDescriptor::extend_class(ctx, ctx.types.member_descriptor_type); - PyMethodDescriptor::extend_class(ctx, ctx.types.method_descriptor_type); -} diff --git a/vm/src/builtins/dict.rs b/vm/src/builtins/dict.rs deleted file mode 100644 index 1a323b4c478..00000000000 --- a/vm/src/builtins/dict.rs +++ /dev/null @@ -1,1319 +0,0 @@ -use super::{ - set::PySetInner, IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, - PyMappingProxy, PySet, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, -}; -use crate::{ - atomic_func, - builtins::{ - iter::{builtins_iter, builtins_reversed}, - type_::PyAttributes, - PyTuple, - }, - class::{PyClassDef, PyClassImpl}, - common::ascii, - dictdatatype::{self, DictKey}, - function::{ - ArgIterable, FuncArgs, KwArgs, OptionalArg, PyArithmeticValue::*, PyComparisonValue, - }, - iter::PyExactSizeIterator, - protocol::{PyIterIter, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, - recursion::ReprGuard, - types::{ - AsMapping, AsNumber, AsSequence, Callable, Comparable, Constructor, Initializer, IterNext, - Iterable, PyComparisonOp, Representable, SelfIter, Unconstructible, - }, - vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, - TryFromObject, -}; -use once_cell::sync::Lazy; -use rustpython_common::lock::PyMutex; -use std::fmt; - -pub type DictContentType = dictdatatype::Dict; - -#[pyclass(module = false, name = "dict", unhashable = true, traverse)] -#[derive(Default)] -pub struct PyDict { - entries: DictContentType, -} -pub type PyDictRef = PyRef<PyDict>; - -impl fmt::Debug for PyDict { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // TODO: implement more detailed, non-recursive Debug formatter - f.write_str("dict") - } -} - -impl PyPayload for PyDict { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.dict_type - } -} - -impl PyDict { - pub fn new_ref(ctx: &Context) -> PyRef<Self> { - PyRef::new_ref(Self::default(), ctx.types.dict_type.to_owned(), None) - } - - /// escape hatch to access the underlying data structure directly. prefer adding a method on - /// PyDict instead of using this - pub(crate) fn _as_dict_inner(&self) -> &DictContentType { - &self.entries - } - - // Used in update and ior. - pub(crate) fn merge_object(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let casted: Result<PyRefExact<PyDict>, _> = other.downcast_exact(vm); - let other = match casted { - Ok(dict_other) => return self.merge_dict(dict_other.into_pyref(), vm), - Err(other) => other, - }; - let dict = &self.entries; - if let Some(keys) = vm.get_method(other.clone(), vm.ctx.intern_str("keys")) { - let keys = keys?.call((), vm)?.get_iter(vm)?; - while let PyIterReturn::Return(key) = keys.next(vm)? { - let val = other.get_item(&*key, vm)?; - dict.insert(vm, &*key, val)?; - } - } else { - let iter = other.get_iter(vm)?; - loop { - fn err(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_value_error("Iterator must have exactly two elements".to_owned()) - } - let element = match iter.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(_) => break, - }; - let elem_iter = element.get_iter(vm)?; - let key = elem_iter.next(vm)?.into_result().map_err(|_| err(vm))?; - let value = elem_iter.next(vm)?.into_result().map_err(|_| err(vm))?; - if matches!(elem_iter.next(vm)?, PyIterReturn::Return(_)) { - return Err(err(vm)); - } - dict.insert(vm, &*key, value)?; - } - } - Ok(()) - } - - fn merge_dict(&self, dict_other: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { - let dict = &self.entries; - let dict_size = &dict_other.size(); - for (key, value) in &dict_other { - dict.insert(vm, &*key, value)?; - } - if dict_other.entries.has_changed_size(dict_size) { - return Err(vm.new_runtime_error("dict mutated during update".to_owned())); - } - Ok(()) - } - - fn inner_cmp( - zelf: &Py<Self>, - other: &Py<PyDict>, - op: PyComparisonOp, - item: bool, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - if op == PyComparisonOp::Ne { - return Self::inner_cmp(zelf, other, PyComparisonOp::Eq, item, vm) - .map(|x| x.map(|eq| !eq)); - } - if !op.eval_ord(zelf.len().cmp(&other.len())) { - return Ok(Implemented(false)); - } - let (superset, subset) = if zelf.len() < other.len() { - (other, zelf) - } else { - (zelf, other) - }; - for (k, v1) in subset { - match superset.get_item_opt(&*k, vm)? { - Some(v2) => { - if v1.is(&v2) { - continue; - } - if item && !vm.bool_eq(&v1, &v2)? { - return Ok(Implemented(false)); - } - } - None => { - return Ok(Implemented(false)); - } - } - } - Ok(Implemented(true)) - } - - pub fn is_empty(&self) -> bool { - self.entries.len() == 0 - } - - /// Set item variant which can be called with multiple - /// key types, such as str to name a notable one. - pub(crate) fn inner_setitem<K: DictKey + ?Sized>( - &self, - key: &K, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - self.entries.insert(vm, key, value) - } - - pub(crate) fn inner_delitem<K: DictKey + ?Sized>( - &self, - key: &K, - vm: &VirtualMachine, - ) -> PyResult<()> { - self.entries.delete(vm, key) - } - - pub fn get_or_insert( - &self, - vm: &VirtualMachine, - key: PyObjectRef, - default: impl FnOnce() -> PyObjectRef, - ) -> PyResult { - self.entries.setdefault(vm, &*key, default) - } - - pub fn from_attributes(attrs: PyAttributes, vm: &VirtualMachine) -> PyResult<Self> { - let entries = DictContentType::default(); - - for (key, value) in attrs { - entries.insert(vm, key, value)?; - } - - Ok(Self { entries }) - } - - pub fn contains_key<K: DictKey + ?Sized>(&self, key: &K, vm: &VirtualMachine) -> bool { - self.entries.contains(vm, key).unwrap() - } - - pub fn size(&self) -> dictdatatype::DictSize { - self.entries.size() - } -} - -// Python dict methods: -#[allow(clippy::len_without_is_empty)] -#[pyclass( - with( - Py, - PyRef, - Constructor, - Initializer, - AsMapping, - Comparable, - Iterable, - AsSequence, - AsNumber, - Representable - ), - flags(BASETYPE) -)] -impl PyDict { - #[pyclassmethod] - fn fromkeys( - class: PyTypeRef, - iterable: ArgIterable, - value: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let value = value.unwrap_or_none(vm); - let d = PyType::call(&class, ().into(), vm)?; - match d.downcast_exact::<PyDict>(vm) { - Ok(pydict) => { - for key in iterable.iter(vm)? { - pydict.setitem(key?, value.clone(), vm)?; - } - Ok(pydict.into_pyref().into()) - } - Err(pyobj) => { - for key in iterable.iter(vm)? { - pyobj.set_item(&*key?, value.clone(), vm)?; - } - Ok(pyobj) - } - } - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - !self.entries.is_empty() - } - - #[pymethod(magic)] - pub fn len(&self) -> usize { - self.entries.len() - } - - #[pymethod(magic)] - fn sizeof(&self) -> usize { - std::mem::size_of::<Self>() + self.entries.sizeof() - } - - #[pymethod(magic)] - fn contains(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self.entries.contains(vm, &*key) - } - - #[pymethod(magic)] - fn delitem(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.inner_delitem(&*key, vm) - } - - #[pymethod] - pub fn clear(&self) { - self.entries.clear() - } - - #[pymethod(magic)] - fn setitem(&self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.inner_setitem(&*key, value, vm) - } - - #[pymethod] - fn get( - &self, - key: PyObjectRef, - default: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - match self.entries.get(vm, &*key)? { - Some(value) => Ok(value), - None => Ok(default.unwrap_or_none(vm)), - } - } - - #[pymethod] - fn setdefault( - &self, - key: PyObjectRef, - default: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - self.entries - .setdefault(vm, &*key, || default.unwrap_or_none(vm)) - } - - #[pymethod] - pub fn copy(&self) -> PyDict { - PyDict { - entries: self.entries.clone(), - } - } - - #[pymethod] - fn update( - &self, - dict_obj: OptionalArg<PyObjectRef>, - kwargs: KwArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { - if let OptionalArg::Present(dict_obj) = dict_obj { - self.merge_object(dict_obj, vm)?; - } - for (key, value) in kwargs.into_iter() { - self.entries.insert(vm, &key, value)?; - } - Ok(()) - } - - #[pymethod(magic)] - fn ior(zelf: PyRef<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.merge_object(other, vm)?; - Ok(zelf) - } - - #[pymethod(magic)] - fn ror(zelf: PyRef<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let dicted: Result<PyDictRef, _> = other.downcast(); - if let Ok(other) = dicted { - let other_cp = other.copy(); - other_cp.merge_dict(zelf, vm)?; - return Ok(other_cp.into_pyobject(vm)); - } - Ok(vm.ctx.not_implemented()) - } - - #[pymethod(magic)] - fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let dicted: Result<PyDictRef, _> = other.downcast(); - if let Ok(other) = dicted { - let self_cp = self.copy(); - self_cp.merge_dict(other, vm)?; - return Ok(self_cp.into_pyobject(vm)); - } - Ok(vm.ctx.not_implemented()) - } - - #[pymethod] - fn pop( - &self, - key: PyObjectRef, - default: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - match self.entries.pop(vm, &*key)? { - Some(value) => Ok(value), - None => default.ok_or_else(|| vm.new_key_error(key)), - } - } - - #[pymethod] - fn popitem(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { - let (key, value) = self.entries.pop_back().ok_or_else(|| { - let err_msg = vm - .ctx - .new_str(ascii!("popitem(): dictionary is empty")) - .into(); - vm.new_key_error(err_msg) - })?; - Ok((key, value)) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } -} - -#[pyclass] -impl Py<PyDict> { - #[pymethod(magic)] - #[cfg_attr(feature = "flame-it", flame("PyDictRef"))] - fn getitem(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.inner_getitem(&*key, vm) - } -} - -#[pyclass] -impl PyRef<PyDict> { - #[pymethod] - fn keys(self) -> PyDictKeys { - PyDictKeys::new(self) - } - - #[pymethod] - fn values(self) -> PyDictValues { - PyDictValues::new(self) - } - - #[pymethod] - fn items(self) -> PyDictItems { - PyDictItems::new(self) - } - - #[pymethod(magic)] - fn reversed(self) -> PyDictReverseKeyIterator { - PyDictReverseKeyIterator::new(self) - } -} - -impl Constructor for PyDict { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyDict::default() - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -impl Initializer for PyDict { - type Args = (OptionalArg<PyObjectRef>, KwArgs); - - fn init( - zelf: PyRef<Self>, - (dict_obj, kwargs): Self::Args, - vm: &VirtualMachine, - ) -> PyResult<()> { - zelf.update(dict_obj, kwargs, vm) - } -} - -impl AsMapping for PyDict { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: PyMappingMethods = PyMappingMethods { - length: atomic_func!(|mapping, _vm| Ok(PyDict::mapping_downcast(mapping).len())), - subscript: atomic_func!(|mapping, needle, vm| { - PyDict::mapping_downcast(mapping).inner_getitem(needle, vm) - }), - ass_subscript: atomic_func!(|mapping, needle, value, vm| { - let zelf = PyDict::mapping_downcast(mapping); - if let Some(value) = value { - zelf.inner_setitem(needle, value, vm) - } else { - zelf.inner_delitem(needle, vm) - } - }), - }; - &AS_MAPPING - } -} - -impl AsSequence for PyDict { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - contains: atomic_func!(|seq, target, vm| PyDict::sequence_downcast(seq) - .entries - .contains(vm, target)), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -impl AsNumber for PyDict { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyDict>() { - PyDict::or(a, b.to_pyobject(vm), vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - inplace_or: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyDict>() { - PyDict::ior(a.to_owned(), b.to_pyobject(vm), vm).map(|d| d.into()) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl Comparable for PyDict { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - op.eq_only(|| { - let other = class_or_notimplemented!(Self, other); - Self::inner_cmp(zelf, other, PyComparisonOp::Eq, true, vm) - }) - } -} - -impl Iterable for PyDict { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok(PyDictKeyIterator::new(zelf).into_pyobject(vm)) - } -} - -impl Representable for PyDict { - #[inline] - fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let s = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let mut str_parts = Vec::with_capacity(zelf.len()); - for (key, value) in zelf { - let key_repr = &key.repr(vm)?; - let value_repr = value.repr(vm)?; - str_parts.push(format!("{key_repr}: {value_repr}")); - } - - vm.ctx.new_str(format!("{{{}}}", str_parts.join(", "))) - } else { - vm.ctx.intern_str("{...}").to_owned() - }; - Ok(s) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } -} - -impl Py<PyDict> { - #[inline] - fn exact_dict(&self, vm: &VirtualMachine) -> bool { - self.class().is(vm.ctx.types.dict_type) - } - - fn missing_opt<K: DictKey + ?Sized>( - &self, - key: &K, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - vm.get_method(self.to_owned().into(), identifier!(vm, __missing__)) - .map(|methods| methods?.call((key.to_pyobject(vm),), vm)) - .transpose() - } - - #[inline] - fn inner_getitem<K: DictKey + ?Sized>( - &self, - key: &K, - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { - if let Some(value) = self.entries.get(vm, key)? { - Ok(value) - } else if let Some(value) = self.missing_opt(key, vm)? { - Ok(value) - } else { - Err(vm.new_key_error(key.to_pyobject(vm))) - } - } - - /// Take a python dictionary and convert it to attributes. - pub fn to_attributes(&self, vm: &VirtualMachine) -> PyAttributes { - let mut attrs = PyAttributes::default(); - for (key, value) in self { - let key: PyRefExact<PyStr> = key.downcast_exact(vm).expect("dict has non-string keys"); - attrs.insert(vm.ctx.intern_str(key), value); - } - attrs - } - - pub fn get_item_opt<K: DictKey + ?Sized>( - &self, - key: &K, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - if self.exact_dict(vm) { - self.entries.get(vm, key) - // FIXME: check __missing__? - } else { - match self.as_object().get_item(key, vm) { - Ok(value) => Ok(Some(value)), - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - self.missing_opt(key, vm) - } - Err(e) => Err(e), - } - } - } - - pub fn get_item<K: DictKey + ?Sized>(&self, key: &K, vm: &VirtualMachine) -> PyResult { - if self.exact_dict(vm) { - self.inner_getitem(key, vm) - } else { - self.as_object().get_item(key, vm) - } - } - - pub fn set_item<K: DictKey + ?Sized>( - &self, - key: &K, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if self.exact_dict(vm) { - self.inner_setitem(key, value, vm) - } else { - self.as_object().set_item(key, value, vm) - } - } - - pub fn del_item<K: DictKey + ?Sized>(&self, key: &K, vm: &VirtualMachine) -> PyResult<()> { - if self.exact_dict(vm) { - self.inner_delitem(key, vm) - } else { - self.as_object().del_item(key, vm) - } - } - - pub fn get_chain<K: DictKey + ?Sized>( - &self, - other: &Self, - key: &K, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - let self_exact = self.exact_dict(vm); - let other_exact = other.exact_dict(vm); - if self_exact && other_exact { - self.entries.get_chain(&other.entries, vm, key) - } else if let Some(value) = self.get_item_opt(key, vm)? { - Ok(Some(value)) - } else { - other.get_item_opt(key, vm) - } - } -} - -// Implement IntoIterator so that we can easily iterate dictionaries from rust code. -impl IntoIterator for PyDictRef { - type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIntoIter; - - fn into_iter(self) -> Self::IntoIter { - DictIntoIter::new(self) - } -} - -impl<'a> IntoIterator for &'a PyDictRef { - type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIter<'a>; - - fn into_iter(self) -> Self::IntoIter { - DictIter::new(self) - } -} - -impl<'a> IntoIterator for &'a Py<PyDict> { - type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIter<'a>; - - fn into_iter(self) -> Self::IntoIter { - DictIter::new(self) - } -} - -impl<'a> IntoIterator for &'a PyDict { - type Item = (PyObjectRef, PyObjectRef); - type IntoIter = DictIter<'a>; - - fn into_iter(self) -> Self::IntoIter { - DictIter::new(self) - } -} - -pub struct DictIntoIter { - dict: PyDictRef, - position: usize, -} - -impl DictIntoIter { - pub fn new(dict: PyDictRef) -> DictIntoIter { - DictIntoIter { dict, position: 0 } - } -} - -impl Iterator for DictIntoIter { - type Item = (PyObjectRef, PyObjectRef); - - fn next(&mut self) -> Option<Self::Item> { - let (position, key, value) = self.dict.entries.next_entry(self.position)?; - self.position = position; - Some((key, value)) - } - - fn size_hint(&self) -> (usize, Option<usize>) { - let l = self.len(); - (l, Some(l)) - } -} -impl ExactSizeIterator for DictIntoIter { - fn len(&self) -> usize { - self.dict.entries.len_from_entry_index(self.position) - } -} - -pub struct DictIter<'a> { - dict: &'a PyDict, - position: usize, -} - -impl<'a> DictIter<'a> { - pub fn new(dict: &'a PyDict) -> Self { - DictIter { dict, position: 0 } - } -} - -impl Iterator for DictIter<'_> { - type Item = (PyObjectRef, PyObjectRef); - - fn next(&mut self) -> Option<Self::Item> { - let (position, key, value) = self.dict.entries.next_entry(self.position)?; - self.position = position; - Some((key, value)) - } - - fn size_hint(&self) -> (usize, Option<usize>) { - let l = self.len(); - (l, Some(l)) - } -} -impl ExactSizeIterator for DictIter<'_> { - fn len(&self) -> usize { - self.dict.entries.len_from_entry_index(self.position) - } -} - -#[pyclass] -trait DictView: PyPayload + PyClassDef + Iterable + Representable -where - Self::ReverseIter: PyPayload, -{ - type ReverseIter; - - fn dict(&self) -> &PyDictRef; - fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef; - - #[pymethod(magic)] - fn len(&self) -> usize { - self.dict().len() - } - - #[pymethod(magic)] - fn reversed(&self) -> Self::ReverseIter; -} - -macro_rules! dict_view { - ( $name: ident, $iter_name: ident, $reverse_iter_name: ident, - $class: ident, $iter_class: ident, $reverse_iter_class: ident, - $class_name: literal, $iter_class_name: literal, $reverse_iter_class_name: literal, - $result_fn: expr) => { - #[pyclass(module = false, name = $class_name)] - #[derive(Debug)] - pub(crate) struct $name { - pub dict: PyDictRef, - } - - impl $name { - pub fn new(dict: PyDictRef) -> Self { - $name { dict } - } - } - - impl DictView for $name { - type ReverseIter = $reverse_iter_name; - fn dict(&self) -> &PyDictRef { - &self.dict - } - fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef { - #[allow(clippy::redundant_closure_call)] - $result_fn(vm, key, value) - } - fn reversed(&self) -> Self::ReverseIter { - $reverse_iter_name::new(self.dict.clone()) - } - } - - impl Iterable for $name { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok($iter_name::new(zelf.dict.clone()).into_pyobject(vm)) - } - } - - impl PyPayload for $name { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.$class - } - } - - impl Representable for $name { - #[inline] - fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let s = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let mut str_parts = Vec::with_capacity(zelf.len()); - for (key, value) in zelf.dict().clone() { - let s = &Self::item(vm, key, value).repr(vm)?; - str_parts.push(s.as_str().to_owned()); - } - vm.ctx - .new_str(format!("{}([{}])", Self::NAME, str_parts.join(", "))) - } else { - vm.ctx.intern_str("{...}").to_owned() - }; - Ok(s) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } - } - - #[pyclass(module = false, name = $iter_class_name)] - #[derive(Debug)] - pub(crate) struct $iter_name { - pub size: dictdatatype::DictSize, - pub internal: PyMutex<PositionIterInternal<PyDictRef>>, - } - - impl PyPayload for $iter_name { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.$iter_class - } - } - - #[pyclass(with(Constructor, IterNext, Iterable))] - impl $iter_name { - fn new(dict: PyDictRef) -> Self { - $iter_name { - size: dict.size(), - internal: PyMutex::new(PositionIterInternal::new(dict, 0)), - } - } - - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().length_hint(|_| self.size.entries_size) - } - - #[allow(clippy::redundant_closure_call)] - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - let iter = builtins_iter(vm).to_owned(); - let internal = self.internal.lock(); - let entries = match &internal.status { - IterStatus::Active(dict) => dict - .into_iter() - .map(|(key, value)| ($result_fn)(vm, key, value)) - .collect::<Vec<_>>(), - IterStatus::Exhausted => vec![], - }; - vm.new_tuple((iter, (vm.ctx.new_list(entries),))) - } - } - impl Unconstructible for $iter_name {} - - impl SelfIter for $iter_name {} - impl IterNext for $iter_name { - #[allow(clippy::redundant_closure_call)] - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let mut internal = zelf.internal.lock(); - let next = if let IterStatus::Active(dict) = &internal.status { - if dict.entries.has_changed_size(&zelf.size) { - internal.status = IterStatus::Exhausted; - return Err(vm.new_runtime_error( - "dictionary changed size during iteration".to_owned(), - )); - } - match dict.entries.next_entry(internal.position) { - Some((position, key, value)) => { - internal.position = position; - PyIterReturn::Return(($result_fn)(vm, key, value)) - } - None => { - internal.status = IterStatus::Exhausted; - PyIterReturn::StopIteration(None) - } - } - } else { - PyIterReturn::StopIteration(None) - }; - Ok(next) - } - } - - #[pyclass(module = false, name = $reverse_iter_class_name)] - #[derive(Debug)] - pub(crate) struct $reverse_iter_name { - pub size: dictdatatype::DictSize, - internal: PyMutex<PositionIterInternal<PyDictRef>>, - } - - impl PyPayload for $reverse_iter_name { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.$reverse_iter_class - } - } - - #[pyclass(with(Constructor, IterNext, Iterable))] - impl $reverse_iter_name { - fn new(dict: PyDictRef) -> Self { - let size = dict.size(); - let position = size.entries_size.saturating_sub(1); - $reverse_iter_name { - size, - internal: PyMutex::new(PositionIterInternal::new(dict, position)), - } - } - - #[allow(clippy::redundant_closure_call)] - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - let iter = builtins_reversed(vm).to_owned(); - let internal = self.internal.lock(); - // TODO: entries must be reversed too - let entries = match &internal.status { - IterStatus::Active(dict) => dict - .into_iter() - .map(|(key, value)| ($result_fn)(vm, key, value)) - .collect::<Vec<_>>(), - IterStatus::Exhausted => vec![], - }; - vm.new_tuple((iter, (vm.ctx.new_list(entries),))) - } - - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal - .lock() - .rev_length_hint(|_| self.size.entries_size) - } - } - impl Unconstructible for $reverse_iter_name {} - - impl SelfIter for $reverse_iter_name {} - impl IterNext for $reverse_iter_name { - #[allow(clippy::redundant_closure_call)] - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let mut internal = zelf.internal.lock(); - let next = if let IterStatus::Active(dict) = &internal.status { - if dict.entries.has_changed_size(&zelf.size) { - internal.status = IterStatus::Exhausted; - return Err(vm.new_runtime_error( - "dictionary changed size during iteration".to_owned(), - )); - } - match dict.entries.prev_entry(internal.position) { - Some((position, key, value)) => { - if internal.position == position { - internal.status = IterStatus::Exhausted; - } else { - internal.position = position; - } - PyIterReturn::Return(($result_fn)(vm, key, value)) - } - None => { - internal.status = IterStatus::Exhausted; - PyIterReturn::StopIteration(None) - } - } - } else { - PyIterReturn::StopIteration(None) - }; - Ok(next) - } - } - }; -} - -dict_view! { - PyDictKeys, - PyDictKeyIterator, - PyDictReverseKeyIterator, - dict_keys_type, - dict_keyiterator_type, - dict_reversekeyiterator_type, - "dict_keys", - "dict_keyiterator", - "dict_reversekeyiterator", - |_vm: &VirtualMachine, key: PyObjectRef, _value: PyObjectRef| key -} - -dict_view! { - PyDictValues, - PyDictValueIterator, - PyDictReverseValueIterator, - dict_values_type, - dict_valueiterator_type, - dict_reversevalueiterator_type, - "dict_values", - "dict_valueiterator", - "dict_reversevalueiterator", - |_vm: &VirtualMachine, _key: PyObjectRef, value: PyObjectRef| value -} - -dict_view! { - PyDictItems, - PyDictItemIterator, - PyDictReverseItemIterator, - dict_items_type, - dict_itemiterator_type, - dict_reverseitemiterator_type, - "dict_items", - "dict_itemiterator", - "dict_reverseitemiterator", - |vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef| - vm.new_tuple((key, value)).into() -} - -// Set operations defined on set-like views of the dictionary. -#[pyclass] -trait ViewSetOps: DictView { - fn to_set(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PySetInner> { - let len = zelf.dict().len(); - let zelf: PyObjectRef = Self::iter(zelf, vm)?; - let iter = PyIterIter::new(vm, zelf, Some(len)); - PySetInner::from_iter(iter, vm) - } - - #[pymethod(name = "__rxor__")] - #[pymethod(magic)] - fn xor(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { - let zelf = Self::to_set(zelf, vm)?; - let inner = zelf.symmetric_difference(other, vm)?; - Ok(PySet { inner }) - } - - #[pymethod(name = "__rand__")] - #[pymethod(magic)] - fn and(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { - let zelf = Self::to_set(zelf, vm)?; - let inner = zelf.intersection(other, vm)?; - Ok(PySet { inner }) - } - - #[pymethod(name = "__ror__")] - #[pymethod(magic)] - fn or(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { - let zelf = Self::to_set(zelf, vm)?; - let inner = zelf.union(other, vm)?; - Ok(PySet { inner }) - } - - #[pymethod(magic)] - fn sub(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { - let zelf = Self::to_set(zelf, vm)?; - let inner = zelf.difference(other, vm)?; - Ok(PySet { inner }) - } - - #[pymethod(magic)] - fn rsub(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySet> { - let left = PySetInner::from_iter(other.iter(vm)?, vm)?; - let right = ArgIterable::try_from_object(vm, Self::iter(zelf, vm)?)?; - let inner = left.difference(right, vm)?; - Ok(PySet { inner }) - } - - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - match_class!(match other { - ref dictview @ Self => { - return PyDict::inner_cmp( - zelf.dict(), - dictview.dict(), - op, - !zelf.class().is(vm.ctx.types.dict_keys_type), - vm, - ); - } - ref _set @ PySet => { - let inner = Self::to_set(zelf.to_owned(), vm)?; - let zelf_set = PySet { inner }.into_pyobject(vm); - return PySet::cmp(zelf_set.downcast_ref().unwrap(), other, op, vm); - } - ref _dictitems @ PyDictItems => {} - ref _dictkeys @ PyDictKeys => {} - _ => { - return Ok(NotImplemented); - } - }); - let lhs: Vec<PyObjectRef> = zelf.as_object().to_owned().try_into_value(vm)?; - let rhs: Vec<PyObjectRef> = other.to_owned().try_into_value(vm)?; - lhs.iter() - .richcompare(rhs.iter(), op, vm) - .map(PyComparisonValue::Implemented) - } - - #[pymethod] - fn isdisjoint(zelf: PyRef<Self>, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - // TODO: to_set is an expensive operation. After merging #3316 rewrite implementation using PySequence_Contains. - let zelf = Self::to_set(zelf, vm)?; - let result = zelf.isdisjoint(other, vm)?; - Ok(result) - } -} - -impl ViewSetOps for PyDictKeys {} -#[pyclass(with( - DictView, - Constructor, - Comparable, - Iterable, - ViewSetOps, - AsSequence, - AsNumber, - Representable -))] -impl PyDictKeys { - #[pymethod(magic)] - fn contains(zelf: PyRef<Self>, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - zelf.dict().contains(key, vm) - } - - #[pygetset] - fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().clone()) - } -} -impl Unconstructible for PyDictKeys {} - -impl Comparable for PyDictKeys { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - ViewSetOps::cmp(zelf, other, op, vm) - } -} - -impl AsSequence for PyDictKeys { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyDictKeys::sequence_downcast(seq).len())), - contains: atomic_func!(|seq, target, vm| { - PyDictKeys::sequence_downcast(seq) - .dict - .entries - .contains(vm, target) - }), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -impl AsNumber for PyDictKeys { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - subtract: Some(set_inner_number_subtract), - and: Some(set_inner_number_and), - xor: Some(set_inner_number_xor), - or: Some(set_inner_number_or), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl ViewSetOps for PyDictItems {} -#[pyclass(with( - DictView, - Constructor, - Comparable, - Iterable, - ViewSetOps, - AsSequence, - AsNumber, - Representable -))] -impl PyDictItems { - #[pymethod(magic)] - fn contains(zelf: PyRef<Self>, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - let needle = match_class! { - match needle { - tuple @ PyTuple => tuple, - _ => return Ok(false), - } - }; - if needle.len() != 2 { - return Ok(false); - } - let key = needle.fast_getitem(0); - if !zelf.dict().contains(key.clone(), vm)? { - return Ok(false); - } - let value = needle.fast_getitem(1); - let found = zelf.dict().getitem(key, vm)?; - vm.identical_or_equal(&found, &value) - } - #[pygetset] - fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().clone()) - } -} -impl Unconstructible for PyDictItems {} - -impl Comparable for PyDictItems { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - ViewSetOps::cmp(zelf, other, op, vm) - } -} - -impl AsSequence for PyDictItems { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyDictItems::sequence_downcast(seq).len())), - contains: atomic_func!(|seq, target, vm| { - PyDictItems::sequence_downcast(seq) - .dict - .entries - .contains(vm, target) - }), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -impl AsNumber for PyDictItems { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - subtract: Some(set_inner_number_subtract), - and: Some(set_inner_number_and), - xor: Some(set_inner_number_xor), - or: Some(set_inner_number_or), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -#[pyclass(with(DictView, Constructor, Iterable, AsSequence, Representable))] -impl PyDictValues { - #[pygetset] - fn mapping(zelf: PyRef<Self>) -> PyMappingProxy { - PyMappingProxy::from(zelf.dict().clone()) - } -} -impl Unconstructible for PyDictValues {} - -impl AsSequence for PyDictValues { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyDictValues::sequence_downcast(seq).len())), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -fn set_inner_number_op<F>(a: &PyObject, b: &PyObject, f: F, vm: &VirtualMachine) -> PyResult -where - F: FnOnce(PySetInner, ArgIterable) -> PyResult<PySetInner>, -{ - let a = PySetInner::from_iter( - ArgIterable::try_from_object(vm, a.to_owned())?.iter(vm)?, - vm, - )?; - let b = ArgIterable::try_from_object(vm, b.to_owned())?; - Ok(PySet { inner: f(a, b)? }.into_pyobject(vm)) -} - -fn set_inner_number_subtract(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { - set_inner_number_op(a, b, |a, b| a.difference(b, vm), vm) -} - -fn set_inner_number_and(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { - set_inner_number_op(a, b, |a, b| a.intersection(b, vm), vm) -} - -fn set_inner_number_xor(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { - set_inner_number_op(a, b, |a, b| a.symmetric_difference(b, vm), vm) -} - -fn set_inner_number_or(a: &PyObject, b: &PyObject, vm: &VirtualMachine) -> PyResult { - set_inner_number_op(a, b, |a, b| a.union(b, vm), vm) -} - -pub(crate) fn init(context: &Context) { - PyDict::extend_class(context, context.types.dict_type); - PyDictKeys::extend_class(context, context.types.dict_keys_type); - PyDictKeyIterator::extend_class(context, context.types.dict_keyiterator_type); - PyDictReverseKeyIterator::extend_class(context, context.types.dict_reversekeyiterator_type); - PyDictValues::extend_class(context, context.types.dict_values_type); - PyDictValueIterator::extend_class(context, context.types.dict_valueiterator_type); - PyDictReverseValueIterator::extend_class(context, context.types.dict_reversevalueiterator_type); - PyDictItems::extend_class(context, context.types.dict_items_type); - PyDictItemIterator::extend_class(context, context.types.dict_itemiterator_type); - PyDictReverseItemIterator::extend_class(context, context.types.dict_reverseitemiterator_type); -} diff --git a/vm/src/builtins/enumerate.rs b/vm/src/builtins/enumerate.rs deleted file mode 100644 index 64d7c1ed367..00000000000 --- a/vm/src/builtins/enumerate.rs +++ /dev/null @@ -1,145 +0,0 @@ -use super::{ - IterStatus, PositionIterInternal, PyGenericAlias, PyIntRef, PyTupleRef, PyType, PyTypeRef, -}; -use crate::common::lock::{PyMutex, PyRwLock}; -use crate::{ - class::PyClassImpl, - convert::ToPyObject, - function::OptionalArg, - protocol::{PyIter, PyIterReturn}, - types::{Constructor, IterNext, Iterable, SelfIter}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; -use malachite_bigint::BigInt; -use num_traits::Zero; - -#[pyclass(module = false, name = "enumerate", traverse)] -#[derive(Debug)] -pub struct PyEnumerate { - #[pytraverse(skip)] - counter: PyRwLock<BigInt>, - iterator: PyIter, -} - -impl PyPayload for PyEnumerate { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.enumerate_type - } -} - -#[derive(FromArgs)] -pub struct EnumerateArgs { - iterator: PyIter, - #[pyarg(any, optional)] - start: OptionalArg<PyIntRef>, -} - -impl Constructor for PyEnumerate { - type Args = EnumerateArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { iterator, start }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let counter = start.map_or_else(BigInt::zero, |start| start.as_bigint().clone()); - PyEnumerate { - counter: PyRwLock::new(counter), - iterator, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -#[pyclass(with(Py, IterNext, Iterable, Constructor), flags(BASETYPE))] -impl PyEnumerate { - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } -} - -#[pyclass] -impl Py<PyEnumerate> { - #[pymethod(magic)] - fn reduce(&self) -> (PyTypeRef, (PyIter, BigInt)) { - ( - self.class().to_owned(), - (self.iterator.clone(), self.counter.read().clone()), - ) - } -} - -impl SelfIter for PyEnumerate {} -impl IterNext for PyEnumerate { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let next_obj = match zelf.iterator.next(vm)? { - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - PyIterReturn::Return(obj) => obj, - }; - let mut counter = zelf.counter.write(); - let position = counter.clone(); - *counter += 1; - Ok(PyIterReturn::Return((position, next_obj).to_pyobject(vm))) - } -} - -#[pyclass(module = false, name = "reversed", traverse)] -#[derive(Debug)] -pub struct PyReverseSequenceIterator { - internal: PyMutex<PositionIterInternal<PyObjectRef>>, -} - -impl PyPayload for PyReverseSequenceIterator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.reverse_iter_type - } -} - -#[pyclass(with(IterNext, Iterable))] -impl PyReverseSequenceIterator { - pub fn new(obj: PyObjectRef, len: usize) -> Self { - let position = len.saturating_sub(1); - Self { - internal: PyMutex::new(PositionIterInternal::new(obj, position)), - } - } - - #[pymethod(magic)] - fn length_hint(&self, vm: &VirtualMachine) -> PyResult<usize> { - let internal = self.internal.lock(); - if let IterStatus::Active(obj) = &internal.status { - if internal.position <= obj.length(vm)? { - return Ok(internal.position + 1); - } - } - Ok(0) - } - - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.internal.lock().set_state(state, |_, pos| pos, vm) - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_reversed_reduce(|x| x.clone(), vm) - } -} - -impl SelfIter for PyReverseSequenceIterator {} -impl IterNext for PyReverseSequenceIterator { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.internal - .lock() - .rev_next(|obj, pos| PyIterReturn::from_getitem_result(obj.get_item(&pos, vm), vm)) - } -} - -pub fn init(context: &Context) { - PyEnumerate::extend_class(context, context.types.enumerate_type); - PyReverseSequenceIterator::extend_class(context, context.types.reverse_iter_type); -} diff --git a/vm/src/builtins/filter.rs b/vm/src/builtins/filter.rs deleted file mode 100644 index 3b33ff766fe..00000000000 --- a/vm/src/builtins/filter.rs +++ /dev/null @@ -1,74 +0,0 @@ -use super::{PyType, PyTypeRef}; -use crate::{ - class::PyClassImpl, - protocol::{PyIter, PyIterReturn}, - types::{Constructor, IterNext, Iterable, SelfIter}, - Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "filter", traverse)] -#[derive(Debug)] -pub struct PyFilter { - predicate: PyObjectRef, - iterator: PyIter, -} - -impl PyPayload for PyFilter { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.filter_type - } -} - -impl Constructor for PyFilter { - type Args = (PyObjectRef, PyIter); - - fn py_new(cls: PyTypeRef, (function, iterator): Self::Args, vm: &VirtualMachine) -> PyResult { - Self { - predicate: function, - iterator, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -#[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] -impl PyFilter { - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> (PyTypeRef, (PyObjectRef, PyIter)) { - ( - vm.ctx.types.filter_type.to_owned(), - (self.predicate.clone(), self.iterator.clone()), - ) - } -} - -impl SelfIter for PyFilter {} -impl IterNext for PyFilter { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let predicate = &zelf.predicate; - loop { - let next_obj = match zelf.iterator.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - let predicate_value = if vm.is_none(predicate) { - next_obj.clone() - } else { - // the predicate itself can raise StopIteration which does stop the filter - // iteration - match PyIterReturn::from_pyresult(predicate.call((next_obj.clone(),), vm), vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - } - }; - if predicate_value.try_to_bool(vm)? { - return Ok(PyIterReturn::Return(next_obj)); - } - } - } -} - -pub fn init(context: &Context) { - PyFilter::extend_class(context, context.types.filter_type); -} diff --git a/vm/src/builtins/float.rs b/vm/src/builtins/float.rs deleted file mode 100644 index 1cd041b7b9d..00000000000 --- a/vm/src/builtins/float.rs +++ /dev/null @@ -1,617 +0,0 @@ -// spell-checker:ignore numer denom - -use super::{ - try_bigint_to_f64, PyByteArray, PyBytes, PyInt, PyIntRef, PyStr, PyStrRef, PyType, PyTypeRef, -}; -use crate::{ - class::PyClassImpl, - common::{float_ops, hash}, - convert::{IntoPyException, ToPyObject, ToPyResult}, - function::{ - ArgBytesLike, OptionalArg, OptionalOption, - PyArithmeticValue::{self, *}, - PyComparisonValue, - }, - protocol::PyNumberMethods, - types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, VirtualMachine, -}; -use malachite_bigint::{BigInt, ToBigInt}; -use num_complex::Complex64; -use num_traits::{Signed, ToPrimitive, Zero}; -use rustpython_common::int::float_to_ratio; -use rustpython_format::FormatSpec; - -#[pyclass(module = false, name = "float")] -#[derive(Debug, Copy, Clone, PartialEq)] -pub struct PyFloat { - value: f64, -} - -impl PyFloat { - pub fn to_f64(&self) -> f64 { - self.value - } -} - -impl PyPayload for PyFloat { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.float_type - } -} - -impl ToPyObject for f64 { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_float(self).into() - } -} -impl ToPyObject for f32 { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_float(f64::from(self)).into() - } -} - -impl From<f64> for PyFloat { - fn from(value: f64) -> Self { - PyFloat { value } - } -} - -pub(crate) fn to_op_float(obj: &PyObject, vm: &VirtualMachine) -> PyResult<Option<f64>> { - let v = if let Some(float) = obj.payload_if_subclass::<PyFloat>(vm) { - Some(float.value) - } else if let Some(int) = obj.payload_if_subclass::<PyInt>(vm) { - Some(try_bigint_to_f64(int.as_bigint(), vm)?) - } else { - None - }; - Ok(v) -} - -macro_rules! impl_try_from_object_float { - ($($t:ty),*) => { - $(impl TryFromObject for $t { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - PyRef::<PyFloat>::try_from_object(vm, obj).map(|f| f.to_f64() as $t) - } - })* - }; -} - -impl_try_from_object_float!(f32, f64); - -fn inner_div(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<f64> { - float_ops::div(v1, v2) - .ok_or_else(|| vm.new_zero_division_error("float division by zero".to_owned())) -} - -fn inner_mod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<f64> { - float_ops::mod_(v1, v2) - .ok_or_else(|| vm.new_zero_division_error("float mod by zero".to_owned())) -} - -pub fn try_to_bigint(value: f64, vm: &VirtualMachine) -> PyResult<BigInt> { - match value.to_bigint() { - Some(int) => Ok(int), - None => { - if value.is_infinite() { - Err(vm.new_overflow_error( - "OverflowError: cannot convert float infinity to integer".to_owned(), - )) - } else if value.is_nan() { - Err(vm - .new_value_error("ValueError: cannot convert float NaN to integer".to_owned())) - } else { - // unreachable unless BigInt has a bug - unreachable!( - "A finite float value failed to be converted to bigint: {}", - value - ) - } - } - } -} - -fn inner_floordiv(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<f64> { - float_ops::floordiv(v1, v2) - .ok_or_else(|| vm.new_zero_division_error("float floordiv by zero".to_owned())) -} - -fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { - float_ops::divmod(v1, v2).ok_or_else(|| vm.new_zero_division_error("float divmod()".to_owned())) -} - -pub fn float_pow(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { - if v1.is_zero() && v2.is_sign_negative() { - let msg = format!("{v1} cannot be raised to a negative power"); - Err(vm.new_zero_division_error(msg)) - } else if v1.is_sign_negative() && (v2.floor() - v2).abs() > f64::EPSILON { - let v1 = Complex64::new(v1, 0.); - let v2 = Complex64::new(v2, 0.); - Ok(v1.powc(v2).to_pyobject(vm)) - } else { - Ok(v1.powf(v2).to_pyobject(vm)) - } -} - -impl Constructor for PyFloat { - type Args = OptionalArg<PyObjectRef>; - - fn py_new(cls: PyTypeRef, arg: Self::Args, vm: &VirtualMachine) -> PyResult { - let float_val = match arg { - OptionalArg::Missing => 0.0, - OptionalArg::Present(val) => { - if cls.is(vm.ctx.types.float_type) && val.class().is(vm.ctx.types.float_type) { - return Ok(val); - } - - if let Some(f) = val.try_float_opt(vm) { - f?.value - } else { - float_from_string(val, vm)? - } - } - }; - PyFloat::from(float_val) - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -fn float_from_string(val: PyObjectRef, vm: &VirtualMachine) -> PyResult<f64> { - let (bytearray, buffer, buffer_lock); - let b = if let Some(s) = val.payload_if_subclass::<PyStr>(vm) { - s.as_str().trim().as_bytes() - } else if let Some(bytes) = val.payload_if_subclass::<PyBytes>(vm) { - bytes.as_bytes() - } else if let Some(buf) = val.payload_if_subclass::<PyByteArray>(vm) { - bytearray = buf.borrow_buf(); - &*bytearray - } else if let Ok(b) = ArgBytesLike::try_from_borrowed_object(vm, &val) { - buffer = b; - buffer_lock = buffer.borrow_buf(); - &*buffer_lock - } else { - return Err(vm.new_type_error(format!( - "float() argument must be a string or a number, not '{}'", - val.class().name() - ))); - }; - crate::literal::float::parse_bytes(b).ok_or_else(|| { - val.repr(vm) - .map(|repr| vm.new_value_error(format!("could not convert string to float: {repr}"))) - .unwrap_or_else(|e| e) - }) -} - -#[pyclass( - flags(BASETYPE), - with(Comparable, Hashable, Constructor, AsNumber, Representable) -)] -impl PyFloat { - #[pymethod(magic)] - fn format(&self, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { - FormatSpec::parse(spec.as_str()) - .and_then(|format_spec| format_spec.format_float(self.value)) - .map_err(|err| err.into_pyexception(vm)) - } - - #[pystaticmethod(magic)] - fn getformat(spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { - if !matches!(spec.as_str(), "double" | "float") { - return Err(vm.new_value_error( - "__getformat__() argument 1 must be 'double' or 'float'".to_owned(), - )); - } - - const BIG_ENDIAN: bool = cfg!(target_endian = "big"); - - Ok(if BIG_ENDIAN { - "IEEE, big-endian" - } else { - "IEEE, little-endian" - } - .to_owned()) - } - - #[pymethod(magic)] - fn abs(&self) -> f64 { - self.value.abs() - } - - #[inline] - fn simple_op<F>( - &self, - other: PyObjectRef, - op: F, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> - where - F: Fn(f64, f64) -> PyResult<f64>, - { - to_op_float(&other, vm)?.map_or_else( - || Ok(NotImplemented), - |other| Ok(Implemented(op(self.value, other)?)), - ) - } - - #[inline] - fn complex_op<F>(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyResult - where - F: Fn(f64, f64) -> PyResult, - { - to_op_float(&other, vm)?.map_or_else( - || Ok(vm.ctx.not_implemented()), - |other| op(self.value, other), - ) - } - - #[inline] - fn tuple_op<F>( - &self, - other: PyObjectRef, - op: F, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<(f64, f64)>> - where - F: Fn(f64, f64) -> PyResult<(f64, f64)>, - { - to_op_float(&other, vm)?.map_or_else( - || Ok(NotImplemented), - |other| Ok(Implemented(op(self.value, other)?)), - ) - } - - #[pymethod(name = "__radd__")] - #[pymethod(magic)] - fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| Ok(a + b), vm) - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - self.value != 0.0 - } - - #[pymethod(magic)] - fn divmod( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<(f64, f64)>> { - self.tuple_op(other, |a, b| inner_divmod(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rdivmod( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<(f64, f64)>> { - self.tuple_op(other, |a, b| inner_divmod(b, a, vm), vm) - } - - #[pymethod(magic)] - fn floordiv( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_floordiv(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rfloordiv( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_floordiv(b, a, vm), vm) - } - - #[pymethod(name = "__mod__")] - fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_mod(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_mod(b, a, vm), vm) - } - - #[pymethod(magic)] - fn pos(&self) -> f64 { - self.value - } - - #[pymethod(magic)] - fn neg(&self) -> f64 { - -self.value - } - - #[pymethod(magic)] - fn pow( - &self, - other: PyObjectRef, - mod_val: OptionalOption<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - if mod_val.flatten().is_some() { - Err(vm.new_type_error("floating point pow() does not accept a 3rd argument".to_owned())) - } else { - self.complex_op(other, |a, b| float_pow(a, b, vm), vm) - } - } - - #[pymethod(magic)] - fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.complex_op(other, |a, b| float_pow(b, a, vm), vm) - } - - #[pymethod(magic)] - fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| Ok(a - b), vm) - } - - #[pymethod(magic)] - fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| Ok(b - a), vm) - } - - #[pymethod(magic)] - fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_div(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rtruediv( - &self, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| inner_div(b, a, vm), vm) - } - - #[pymethod(name = "__rmul__")] - #[pymethod(magic)] - fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<f64>> { - self.simple_op(other, |a, b| Ok(a * b), vm) - } - - #[pymethod(magic)] - fn trunc(&self, vm: &VirtualMachine) -> PyResult<BigInt> { - try_to_bigint(self.value, vm) - } - - #[pymethod(magic)] - fn floor(&self, vm: &VirtualMachine) -> PyResult<BigInt> { - try_to_bigint(self.value.floor(), vm) - } - - #[pymethod(magic)] - fn ceil(&self, vm: &VirtualMachine) -> PyResult<BigInt> { - try_to_bigint(self.value.ceil(), vm) - } - - #[pymethod(magic)] - fn round(&self, ndigits: OptionalOption<PyIntRef>, vm: &VirtualMachine) -> PyResult { - let ndigits = ndigits.flatten(); - let value = if let Some(ndigits) = ndigits { - let ndigits = ndigits.as_bigint(); - let ndigits = match ndigits.to_i32() { - Some(n) => n, - None if ndigits.is_positive() => i32::MAX, - None => i32::MIN, - }; - let float = float_ops::round_float_digits(self.value, ndigits).ok_or_else(|| { - vm.new_overflow_error("overflow occurred during round".to_owned()) - })?; - vm.ctx.new_float(float).into() - } else { - let fract = self.value.fract(); - let value = if (fract.abs() - 0.5).abs() < f64::EPSILON { - if self.value.trunc() % 2.0 == 0.0 { - self.value - fract - } else { - self.value + fract - } - } else { - self.value.round() - }; - let int = try_to_bigint(value, vm)?; - vm.ctx.new_int(int).into() - }; - Ok(value) - } - - #[pymethod(magic)] - fn int(&self, vm: &VirtualMachine) -> PyResult<BigInt> { - self.trunc(vm) - } - - #[pymethod(magic)] - fn float(zelf: PyRef<Self>) -> PyRef<Self> { - zelf - } - - #[pygetset] - fn real(zelf: PyRef<Self>) -> PyRef<Self> { - zelf - } - - #[pygetset] - fn imag(&self) -> f64 { - 0.0f64 - } - - #[pymethod] - fn conjugate(zelf: PyRef<Self>) -> PyRef<Self> { - zelf - } - - #[pymethod] - fn is_integer(&self) -> bool { - crate::literal::float::is_integer(self.value) - } - - #[pymethod] - fn as_integer_ratio(&self, vm: &VirtualMachine) -> PyResult<(PyIntRef, PyIntRef)> { - let value = self.value; - - float_to_ratio(value) - .map(|(numer, denom)| (vm.ctx.new_bigint(&numer), vm.ctx.new_bigint(&denom))) - .ok_or_else(|| { - if value.is_infinite() { - vm.new_overflow_error("cannot convert Infinity to integer ratio".to_owned()) - } else if value.is_nan() { - vm.new_value_error("cannot convert NaN to integer ratio".to_owned()) - } else { - unreachable!("finite float must able to convert to integer ratio") - } - }) - } - - #[pyclassmethod] - fn fromhex(cls: PyTypeRef, string: PyStrRef, vm: &VirtualMachine) -> PyResult { - let result = crate::literal::float::from_hex(string.as_str().trim()).ok_or_else(|| { - vm.new_value_error("invalid hexadecimal floating-point string".to_owned()) - })?; - PyType::call(&cls, vec![vm.ctx.new_float(result).into()].into(), vm) - } - - #[pymethod] - fn hex(&self) -> String { - crate::literal::float::to_hex(self.value) - } - - #[pymethod(magic)] - fn getnewargs(&self, vm: &VirtualMachine) -> PyObjectRef { - (self.value,).to_pyobject(vm) - } -} - -impl Comparable for PyFloat { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - let ret = if let Some(other) = other.payload_if_subclass::<PyFloat>(vm) { - zelf.value - .partial_cmp(&other.value) - .map_or_else(|| op == PyComparisonOp::Ne, |ord| op.eval_ord(ord)) - } else if let Some(other) = other.payload_if_subclass::<PyInt>(vm) { - let a = zelf.to_f64(); - let b = other.as_bigint(); - match op { - PyComparisonOp::Lt => float_ops::lt_int(a, b), - PyComparisonOp::Le => { - if let (Some(a_int), Some(b_float)) = (a.to_bigint(), b.to_f64()) { - a <= b_float && a_int <= *b - } else { - float_ops::lt_int(a, b) - } - } - PyComparisonOp::Eq => float_ops::eq_int(a, b), - PyComparisonOp::Ne => !float_ops::eq_int(a, b), - PyComparisonOp::Ge => { - if let (Some(a_int), Some(b_float)) = (a.to_bigint(), b.to_f64()) { - a >= b_float && a_int >= *b - } else { - float_ops::gt_int(a, b) - } - } - PyComparisonOp::Gt => float_ops::gt_int(a, b), - } - } else { - return Ok(NotImplemented); - }; - Ok(Implemented(ret)) - } -} - -impl Hashable for PyFloat { - #[inline] - fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<hash::PyHash> { - Ok(hash::hash_float(zelf.to_f64()).unwrap_or_else(|| hash::hash_object_id(zelf.get_id()))) - } -} - -impl AsNumber for PyFloat { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - add: Some(|a, b, vm| PyFloat::number_op(a, b, |a, b, _vm| a + b, vm)), - subtract: Some(|a, b, vm| PyFloat::number_op(a, b, |a, b, _vm| a - b, vm)), - multiply: Some(|a, b, vm| PyFloat::number_op(a, b, |a, b, _vm| a * b, vm)), - remainder: Some(|a, b, vm| PyFloat::number_op(a, b, inner_mod, vm)), - divmod: Some(|a, b, vm| PyFloat::number_op(a, b, inner_divmod, vm)), - power: Some(|a, b, c, vm| { - if vm.is_none(c) { - PyFloat::number_op(a, b, float_pow, vm) - } else { - Err(vm.new_type_error(String::from( - "pow() 3rd argument not allowed unless all arguments are integers", - ))) - } - }), - negative: Some(|num, vm| { - let value = PyFloat::number_downcast(num).value; - (-value).to_pyresult(vm) - }), - positive: Some(|num, vm| PyFloat::number_downcast_exact(num, vm).to_pyresult(vm)), - absolute: Some(|num, vm| { - let value = PyFloat::number_downcast(num).value; - value.abs().to_pyresult(vm) - }), - boolean: Some(|num, _vm| Ok(PyFloat::number_downcast(num).value.is_zero())), - int: Some(|num, vm| { - let value = PyFloat::number_downcast(num).value; - try_to_bigint(value, vm).map(|x| PyInt::from(x).into_pyobject(vm)) - }), - float: Some(|num, vm| Ok(PyFloat::number_downcast_exact(num, vm).into())), - floor_divide: Some(|a, b, vm| PyFloat::number_op(a, b, inner_floordiv, vm)), - true_divide: Some(|a, b, vm| PyFloat::number_op(a, b, inner_div, vm)), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } - - #[inline] - fn clone_exact(zelf: &Py<Self>, vm: &VirtualMachine) -> PyRef<Self> { - vm.ctx.new_float(zelf.value) - } -} - -impl Representable for PyFloat { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(crate::literal::float::to_string(zelf.value)) - } -} - -impl PyFloat { - fn number_op<F, R>(a: &PyObject, b: &PyObject, op: F, vm: &VirtualMachine) -> PyResult - where - F: FnOnce(f64, f64, &VirtualMachine) -> R, - R: ToPyResult, - { - if let (Some(a), Some(b)) = (to_op_float(a, vm)?, to_op_float(b, vm)?) { - op(a, b, vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - } -} - -// Retrieve inner float value: -#[cfg(feature = "serde")] -pub(crate) fn get_value(obj: &PyObject) -> f64 { - obj.payload::<PyFloat>().unwrap().value -} - -#[rustfmt::skip] // to avoid line splitting -pub fn init(context: &Context) { - PyFloat::extend_class(context, context.types.float_type); -} diff --git a/vm/src/builtins/frame.rs b/vm/src/builtins/frame.rs deleted file mode 100644 index d93799db92f..00000000000 --- a/vm/src/builtins/frame.rs +++ /dev/null @@ -1,128 +0,0 @@ -/*! The python `frame` type. - -*/ - -use super::{PyCode, PyDictRef, PyIntRef, PyStrRef}; -use crate::{ - class::PyClassImpl, - frame::{Frame, FrameRef}, - function::PySetterValue, - types::{Constructor, Representable, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyRef, PyResult, VirtualMachine, -}; -use num_traits::Zero; - -pub fn init(context: &Context) { - Frame::extend_class(context, context.types.frame_type); -} - -impl Unconstructible for Frame {} - -impl Representable for Frame { - #[inline] - fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - const REPR: &str = "<frame object at .. >"; - Ok(vm.ctx.intern_str(REPR).to_owned()) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } -} - -#[pyclass(with(Constructor, Py))] -impl Frame { - #[pymethod] - fn clear(&self) { - // TODO - } - - #[pygetset] - fn f_globals(&self) -> PyDictRef { - self.globals.clone() - } - - #[pygetset] - fn f_locals(&self, vm: &VirtualMachine) -> PyResult { - self.locals(vm).map(Into::into) - } - - #[pygetset] - pub fn f_code(&self) -> PyRef<PyCode> { - self.code.clone() - } - - #[pygetset] - fn f_lasti(&self) -> u32 { - self.lasti() - } - - #[pygetset] - pub fn f_lineno(&self) -> usize { - self.current_location().row.to_usize() - } - - #[pygetset] - fn f_trace(&self) -> PyObjectRef { - let boxed = self.trace.lock(); - boxed.clone() - } - - #[pygetset(setter)] - fn set_f_trace(&self, value: PySetterValue, vm: &VirtualMachine) { - let mut storage = self.trace.lock(); - *storage = value.unwrap_or_none(vm); - } - - #[pymember(type = "bool")] - fn f_trace_lines(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { - let zelf: FrameRef = zelf.downcast().unwrap_or_else(|_| unreachable!()); - - let boxed = zelf.trace_lines.lock(); - Ok(vm.ctx.new_bool(*boxed).into()) - } - - #[pymember(type = "bool", setter)] - fn set_f_trace_lines( - vm: &VirtualMachine, - zelf: PyObjectRef, - value: PySetterValue, - ) -> PyResult<()> { - match value { - PySetterValue::Assign(value) => { - let zelf: FrameRef = zelf.downcast().unwrap_or_else(|_| unreachable!()); - - let value: PyIntRef = value.downcast().map_err(|_| { - vm.new_type_error("attribute value type must be bool".to_owned()) - })?; - - let mut trace_lines = zelf.trace_lines.lock(); - *trace_lines = !value.as_bigint().is_zero(); - - Ok(()) - } - PySetterValue::Delete => { - Err(vm.new_type_error("can't delete numeric/char attribute".to_owned())) - } - } - } -} - -#[pyclass] -impl Py<Frame> { - #[pygetset] - pub fn f_back(&self, vm: &VirtualMachine) -> Option<PyRef<Frame>> { - // TODO: actually store f_back inside Frame struct - - // get the frame in the frame stack that appears before this one. - // won't work if this frame isn't in the frame stack, hence the todo above - vm.frames - .borrow() - .iter() - .rev() - .skip_while(|p| !p.is(self.as_object())) - .nth(1) - .cloned() - } -} diff --git a/vm/src/builtins/function.rs b/vm/src/builtins/function.rs deleted file mode 100644 index ab069e0e73f..00000000000 --- a/vm/src/builtins/function.rs +++ /dev/null @@ -1,725 +0,0 @@ -#[cfg(feature = "jit")] -mod jitfunc; - -use super::{ - tuple::PyTupleTyped, PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyStr, PyStrRef, - PyTupleRef, PyType, PyTypeRef, -}; -#[cfg(feature = "jit")] -use crate::common::lock::OnceCell; -use crate::common::lock::PyMutex; -use crate::convert::ToPyObject; -use crate::function::ArgMapping; -use crate::object::{Traverse, TraverseFn}; -use crate::{ - bytecode, - class::PyClassImpl, - frame::Frame, - function::{FuncArgs, OptionalArg, PyComparisonValue, PySetterValue}, - scope::Scope, - types::{ - Callable, Comparable, Constructor, GetAttr, GetDescriptor, PyComparisonOp, Representable, - }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use itertools::Itertools; -#[cfg(feature = "jit")] -use rustpython_jit::CompiledCode; - -#[pyclass(module = false, name = "function", traverse = "manual")] -#[derive(Debug)] -pub struct PyFunction { - code: PyRef<PyCode>, - globals: PyDictRef, - closure: Option<PyTupleTyped<PyCellRef>>, - defaults_and_kwdefaults: PyMutex<(Option<PyTupleRef>, Option<PyDictRef>)>, - name: PyMutex<PyStrRef>, - qualname: PyMutex<PyStrRef>, - type_params: PyMutex<PyTupleRef>, - #[cfg(feature = "jit")] - jitted_code: OnceCell<CompiledCode>, -} - -unsafe impl Traverse for PyFunction { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.globals.traverse(tracer_fn); - self.closure.traverse(tracer_fn); - self.defaults_and_kwdefaults.traverse(tracer_fn); - } -} - -impl PyFunction { - pub(crate) fn new( - code: PyRef<PyCode>, - globals: PyDictRef, - closure: Option<PyTupleTyped<PyCellRef>>, - defaults: Option<PyTupleRef>, - kw_only_defaults: Option<PyDictRef>, - qualname: PyStrRef, - type_params: PyTupleRef, - ) -> Self { - let name = PyMutex::new(code.obj_name.to_owned()); - PyFunction { - code, - globals, - closure, - defaults_and_kwdefaults: PyMutex::new((defaults, kw_only_defaults)), - name, - qualname: PyMutex::new(qualname), - type_params: PyMutex::new(type_params), - #[cfg(feature = "jit")] - jitted_code: OnceCell::new(), - } - } - - fn fill_locals_from_args( - &self, - frame: &Frame, - func_args: FuncArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { - let code = &*self.code; - let nargs = func_args.args.len(); - let nexpected_args = code.arg_count as usize; - let total_args = code.arg_count as usize + code.kwonlyarg_count as usize; - // let arg_names = self.code.arg_names(); - - // This parses the arguments from args and kwargs into - // the proper variables keeping into account default values - // and starargs and kwargs. - // See also: PyEval_EvalCodeWithName in cpython: - // https://github.com/python/cpython/blob/main/Python/ceval.c#L3681 - - let mut fastlocals = frame.fastlocals.lock(); - - let mut args_iter = func_args.args.into_iter(); - - // Copy positional arguments into local variables - // zip short-circuits if either iterator returns None, which is the behavior we want -- - // only fill as much as there is to fill with as much as we have - for (local, arg) in Iterator::zip( - fastlocals.iter_mut().take(nexpected_args), - args_iter.by_ref().take(nargs), - ) { - *local = Some(arg); - } - - let mut vararg_offset = total_args; - // Pack other positional arguments in to *args: - if code.flags.contains(bytecode::CodeFlags::HAS_VARARGS) { - let vararg_value = vm.ctx.new_tuple(args_iter.collect()); - fastlocals[vararg_offset] = Some(vararg_value.into()); - vararg_offset += 1; - } else { - // Check the number of positional arguments - if nargs > nexpected_args { - return Err(vm.new_type_error(format!( - "{}() takes {} positional arguments but {} were given", - self.qualname(), - nexpected_args, - nargs - ))); - } - } - - // Do we support `**kwargs` ? - let kwargs = if code.flags.contains(bytecode::CodeFlags::HAS_VARKEYWORDS) { - let d = vm.ctx.new_dict(); - fastlocals[vararg_offset] = Some(d.clone().into()); - Some(d) - } else { - None - }; - - let argpos = |range: std::ops::Range<_>, name: &str| { - code.varnames - .iter() - .enumerate() - .skip(range.start) - .take(range.end - range.start) - .find(|(_, s)| s.as_str() == name) - .map(|(p, _)| p) - }; - - let mut posonly_passed_as_kwarg = Vec::new(); - // Handle keyword arguments - for (name, value) in func_args.kwargs { - // Check if we have a parameter with this name: - if let Some(pos) = argpos(code.posonlyarg_count as usize..total_args, &name) { - let slot = &mut fastlocals[pos]; - if slot.is_some() { - return Err(vm.new_type_error(format!( - "{}() got multiple values for argument '{}'", - self.qualname(), - name - ))); - } - *slot = Some(value); - } else if let Some(kwargs) = kwargs.as_ref() { - kwargs.set_item(&name, value, vm)?; - } else if argpos(0..code.posonlyarg_count as usize, &name).is_some() { - posonly_passed_as_kwarg.push(name); - } else { - return Err(vm.new_type_error(format!( - "{}() got an unexpected keyword argument '{}'", - self.qualname(), - name - ))); - } - } - if !posonly_passed_as_kwarg.is_empty() { - return Err(vm.new_type_error(format!( - "{}() got some positional-only arguments passed as keyword arguments: '{}'", - self.qualname(), - posonly_passed_as_kwarg.into_iter().format(", "), - ))); - } - - let mut defaults_and_kwdefaults = None; - // can't be a closure cause it returns a reference to a captured variable :/ - macro_rules! get_defaults { - () => {{ - defaults_and_kwdefaults - .get_or_insert_with(|| self.defaults_and_kwdefaults.lock().clone()) - }}; - } - - // Add missing positional arguments, if we have fewer positional arguments than the - // function definition calls for - if nargs < nexpected_args { - let defaults = get_defaults!().0.as_ref().map(|tup| tup.as_slice()); - let ndefs = defaults.map_or(0, |d| d.len()); - - let nrequired = code.arg_count as usize - ndefs; - - // Given the number of defaults available, check all the arguments for which we - // _don't_ have defaults; if any are missing, raise an exception - let mut missing: Vec<_> = (nargs..nrequired) - .filter_map(|i| { - if fastlocals[i].is_none() { - Some(&code.varnames[i]) - } else { - None - } - }) - .collect(); - let missing_args_len = missing.len(); - - if !missing.is_empty() { - let last = if missing.len() > 1 { - missing.pop() - } else { - None - }; - - let (and, right) = if let Some(last) = last { - ( - if missing.len() == 1 { - "' and '" - } else { - "', and '" - }, - last.as_str(), - ) - } else { - ("", "") - }; - - return Err(vm.new_type_error(format!( - "{}() missing {} required positional argument{}: '{}{}{}'", - self.qualname(), - missing_args_len, - if missing_args_len == 1 { "" } else { "s" }, - missing.iter().join("', '"), - and, - right, - ))); - } - - if let Some(defaults) = defaults { - let n = std::cmp::min(nargs, nexpected_args); - let i = n.saturating_sub(nrequired); - - // We have sufficient defaults, so iterate over the corresponding names and use - // the default if we don't already have a value - for i in i..defaults.len() { - let slot = &mut fastlocals[nrequired + i]; - if slot.is_none() { - *slot = Some(defaults[i].clone()); - } - } - } - }; - - if code.kwonlyarg_count > 0 { - // TODO: compile a list of missing arguments - // let mut missing = vec![]; - // Check if kw only arguments are all present: - for (slot, kwarg) in fastlocals - .iter_mut() - .zip(&*code.varnames) - .skip(code.arg_count as usize) - .take(code.kwonlyarg_count as usize) - .filter(|(slot, _)| slot.is_none()) - { - if let Some(defaults) = &get_defaults!().1 { - if let Some(default) = defaults.get_item_opt(&**kwarg, vm)? { - *slot = Some(default); - continue; - } - } - - // No default value and not specified. - return Err( - vm.new_type_error(format!("Missing required kw only argument: '{kwarg}'")) - ); - } - } - - if let Some(cell2arg) = code.cell2arg.as_deref() { - for (cell_idx, arg_idx) in cell2arg.iter().enumerate().filter(|(_, i)| **i != -1) { - let x = fastlocals[*arg_idx as usize].take(); - frame.cells_frees[cell_idx].set(x); - } - } - - Ok(()) - } - - pub fn invoke_with_locals( - &self, - func_args: FuncArgs, - locals: Option<ArgMapping>, - vm: &VirtualMachine, - ) -> PyResult { - #[cfg(feature = "jit")] - if let Some(jitted_code) = self.jitted_code.get() { - match jitfunc::get_jit_args(self, &func_args, jitted_code, vm) { - Ok(args) => { - return Ok(args.invoke().to_pyobject(vm)); - } - Err(err) => info!( - "jit: function `{}` is falling back to being interpreted because of the \ - error: {}", - self.code.obj_name, err - ), - } - } - - let code = &self.code; - - let locals = if self.code.flags.contains(bytecode::CodeFlags::NEW_LOCALS) { - ArgMapping::from_dict_exact(vm.ctx.new_dict()) - } else if let Some(locals) = locals { - locals - } else { - ArgMapping::from_dict_exact(self.globals.clone()) - }; - - // Construct frame: - let frame = Frame::new( - code.clone(), - Scope::new(Some(locals), self.globals.clone()), - vm.builtins.dict(), - self.closure.as_ref().map_or(&[], |c| c.as_slice()), - vm, - ) - .into_ref(&vm.ctx); - - self.fill_locals_from_args(&frame, func_args, vm)?; - - // If we have a generator, create a new generator - let is_gen = code.flags.contains(bytecode::CodeFlags::IS_GENERATOR); - let is_coro = code.flags.contains(bytecode::CodeFlags::IS_COROUTINE); - match (is_gen, is_coro) { - (true, false) => Ok(PyGenerator::new(frame, self.name()).into_pyobject(vm)), - (false, true) => Ok(PyCoroutine::new(frame, self.name()).into_pyobject(vm)), - (true, true) => Ok(PyAsyncGen::new(frame, self.name()).into_pyobject(vm)), - (false, false) => vm.run_frame(frame), - } - } - - #[inline(always)] - pub fn invoke(&self, func_args: FuncArgs, vm: &VirtualMachine) -> PyResult { - self.invoke_with_locals(func_args, None, vm) - } -} - -impl PyPayload for PyFunction { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.function_type - } -} - -#[pyclass( - with(GetDescriptor, Callable, Representable), - flags(HAS_DICT, METHOD_DESCRIPTOR) -)] -impl PyFunction { - #[pygetset(magic)] - fn code(&self) -> PyRef<PyCode> { - self.code.clone() - } - - #[pygetset(magic)] - fn defaults(&self) -> Option<PyTupleRef> { - self.defaults_and_kwdefaults.lock().0.clone() - } - #[pygetset(magic, setter)] - fn set_defaults(&self, defaults: Option<PyTupleRef>) { - self.defaults_and_kwdefaults.lock().0 = defaults - } - - #[pygetset(magic)] - fn kwdefaults(&self) -> Option<PyDictRef> { - self.defaults_and_kwdefaults.lock().1.clone() - } - #[pygetset(magic, setter)] - fn set_kwdefaults(&self, kwdefaults: Option<PyDictRef>) { - self.defaults_and_kwdefaults.lock().1 = kwdefaults - } - - // {"__closure__", T_OBJECT, OFF(func_closure), READONLY}, - // {"__doc__", T_OBJECT, OFF(func_doc), 0}, - // {"__globals__", T_OBJECT, OFF(func_globals), READONLY}, - // {"__module__", T_OBJECT, OFF(func_module), 0}, - // {"__builtins__", T_OBJECT, OFF(func_builtins), READONLY}, - #[pymember(magic)] - fn globals(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { - let zelf = Self::_as_pyref(&zelf, vm)?; - Ok(zelf.globals.clone().into()) - } - - #[pymember(magic)] - fn closure(vm: &VirtualMachine, zelf: PyObjectRef) -> PyResult { - let zelf = Self::_as_pyref(&zelf, vm)?; - Ok(vm.unwrap_or_none(zelf.closure.clone().map(|x| x.to_pyobject(vm)))) - } - - #[pygetset(magic)] - fn name(&self) -> PyStrRef { - self.name.lock().clone() - } - - #[pygetset(magic, setter)] - fn set_name(&self, name: PyStrRef) { - *self.name.lock() = name; - } - - #[pygetset(magic)] - fn qualname(&self) -> PyStrRef { - self.qualname.lock().clone() - } - - #[pygetset(magic, setter)] - fn set_qualname(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - match value { - PySetterValue::Assign(value) => { - let Ok(qualname) = value.downcast::<PyStr>() else { - return Err(vm.new_type_error( - "__qualname__ must be set to a string object".to_string(), - )); - }; - *self.qualname.lock() = qualname; - } - PySetterValue::Delete => { - return Err( - vm.new_type_error("__qualname__ must be set to a string object".to_string()) - ); - } - } - Ok(()) - } - - #[pygetset(magic)] - fn type_params(&self) -> PyTupleRef { - self.type_params.lock().clone() - } - - #[pygetset(magic, setter)] - fn set_type_params( - &self, - value: PySetterValue<PyTupleRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - match value { - PySetterValue::Assign(value) => { - *self.type_params.lock() = value; - } - PySetterValue::Delete => { - return Err( - vm.new_type_error("__type_params__ must be set to a tuple object".to_string()) - ); - } - } - Ok(()) - } - - #[cfg(feature = "jit")] - #[pymethod(magic)] - fn jit(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { - zelf.jitted_code - .get_or_try_init(|| { - let arg_types = jitfunc::get_jit_arg_types(&zelf, vm)?; - rustpython_jit::compile(&zelf.code.code, &arg_types) - .map_err(|err| jitfunc::new_jit_error(err.to_string(), vm)) - }) - .map(drop) - } -} - -impl GetDescriptor for PyFunction { - fn descr_get( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let (_zelf, obj) = Self::_unwrap(&zelf, obj, vm)?; - let obj = if vm.is_none(&obj) && !Self::_cls_is(&cls, obj.class()) { - zelf - } else { - PyBoundMethod::new_ref(obj, zelf, &vm.ctx).into() - }; - Ok(obj) - } -} - -impl Callable for PyFunction { - type Args = FuncArgs; - #[inline] - fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - zelf.invoke(args, vm) - } -} - -impl Representable for PyFunction { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(format!( - "<function {} at {:#x}>", - zelf.qualname(), - zelf.get_id() - )) - } -} - -#[pyclass(module = false, name = "method", traverse)] -#[derive(Debug)] -pub struct PyBoundMethod { - object: PyObjectRef, - function: PyObjectRef, -} - -impl Callable for PyBoundMethod { - type Args = FuncArgs; - #[inline] - fn call(zelf: &Py<Self>, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { - args.prepend_arg(zelf.object.clone()); - zelf.function.call(args, vm) - } -} - -impl Comparable for PyBoundMethod { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - _vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - op.eq_only(|| { - let other = class_or_notimplemented!(Self, other); - Ok(PyComparisonValue::Implemented( - zelf.function.is(&other.function) && zelf.object.is(&other.object), - )) - }) - } -} - -impl GetAttr for PyBoundMethod { - fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - let class_attr = vm - .ctx - .interned_str(name) - .and_then(|attr_name| zelf.get_class_attr(attr_name)); - if let Some(obj) = class_attr { - return vm.call_if_get_descriptor(&obj, zelf.to_owned().into()); - } - zelf.function.get_attr(name, vm) - } -} - -#[derive(FromArgs)] -pub struct PyBoundMethodNewArgs { - #[pyarg(positional)] - function: PyObjectRef, - #[pyarg(positional)] - object: PyObjectRef, -} - -impl Constructor for PyBoundMethod { - type Args = PyBoundMethodNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { function, object }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyBoundMethod::new(object, function) - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -impl PyBoundMethod { - fn new(object: PyObjectRef, function: PyObjectRef) -> Self { - PyBoundMethod { object, function } - } - - pub fn new_ref(object: PyObjectRef, function: PyObjectRef, ctx: &Context) -> PyRef<Self> { - PyRef::new_ref( - Self::new(object, function), - ctx.types.bound_method_type.to_owned(), - None, - ) - } -} - -#[pyclass( - with(Callable, Comparable, GetAttr, Constructor, Representable), - flags(HAS_DICT) -)] -impl PyBoundMethod { - #[pymethod(magic)] - fn reduce( - &self, - vm: &VirtualMachine, - ) -> (Option<PyObjectRef>, (PyObjectRef, Option<PyObjectRef>)) { - let builtins_getattr = vm.builtins.get_attr("getattr", vm).ok(); - let funcself = self.object.clone(); - let funcname = self.function.get_attr("__name__", vm).ok(); - (builtins_getattr, (funcself, funcname)) - } - - #[pygetset(magic)] - fn doc(&self, vm: &VirtualMachine) -> PyResult { - self.function.get_attr("__doc__", vm) - } - - #[pygetset(magic)] - fn func(&self) -> PyObjectRef { - self.function.clone() - } - - #[pygetset(name = "__self__")] - fn get_self(&self) -> PyObjectRef { - self.object.clone() - } - - #[pygetset(magic)] - fn module(&self, vm: &VirtualMachine) -> Option<PyObjectRef> { - self.function.get_attr("__module__", vm).ok() - } - - #[pygetset(magic)] - fn qualname(&self, vm: &VirtualMachine) -> PyResult { - if self - .function - .fast_isinstance(vm.ctx.types.builtin_function_or_method_type) - { - // Special case: we work with `__new__`, which is not really a method. - // It is a function, so its `__qualname__` is just `__new__`. - // We need to add object's part manually. - let obj_name = vm.get_attribute_opt(self.object.clone(), "__qualname__")?; - let obj_name: Option<PyStrRef> = obj_name.and_then(|o| o.downcast().ok()); - return Ok(vm - .ctx - .new_str(format!( - "{}.__new__", - obj_name.as_ref().map_or("?", |s| s.as_str()) - )) - .into()); - } - self.function.get_attr("__qualname__", vm) - } -} - -impl PyPayload for PyBoundMethod { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.bound_method_type - } -} - -impl Representable for PyBoundMethod { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - #[allow(clippy::needless_match)] // False positive on nightly - let funcname = - if let Some(qname) = vm.get_attribute_opt(zelf.function.clone(), "__qualname__")? { - Some(qname) - } else { - vm.get_attribute_opt(zelf.function.clone(), "__name__")? - }; - let funcname: Option<PyStrRef> = funcname.and_then(|o| o.downcast().ok()); - Ok(format!( - "<bound method {} of {}>", - funcname.as_ref().map_or("?", |s| s.as_str()), - &zelf.object.repr(vm)?.as_str(), - )) - } -} - -#[pyclass(module = false, name = "cell", traverse)] -#[derive(Debug, Default)] -pub(crate) struct PyCell { - contents: PyMutex<Option<PyObjectRef>>, -} -pub(crate) type PyCellRef = PyRef<PyCell>; - -impl PyPayload for PyCell { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.cell_type - } -} - -impl Constructor for PyCell { - type Args = OptionalArg; - - fn py_new(cls: PyTypeRef, value: Self::Args, vm: &VirtualMachine) -> PyResult { - Self::new(value.into_option()) - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -#[pyclass(with(Constructor))] -impl PyCell { - pub fn new(contents: Option<PyObjectRef>) -> Self { - Self { - contents: PyMutex::new(contents), - } - } - - pub fn get(&self) -> Option<PyObjectRef> { - self.contents.lock().clone() - } - pub fn set(&self, x: Option<PyObjectRef>) { - *self.contents.lock() = x; - } - - #[pygetset] - fn cell_contents(&self, vm: &VirtualMachine) -> PyResult { - self.get() - .ok_or_else(|| vm.new_value_error("Cell is empty".to_owned())) - } - #[pygetset(setter)] - fn set_cell_contents(&self, x: PyObjectRef) { - self.set(Some(x)) - } -} - -pub fn init(context: &Context) { - PyFunction::extend_class(context, context.types.function_type); - PyBoundMethod::extend_class(context, context.types.bound_method_type); - PyCell::extend_class(context, context.types.cell_type); -} diff --git a/vm/src/builtins/function/jitfunc.rs b/vm/src/builtins/function/jitfunc.rs deleted file mode 100644 index fe73c3afc06..00000000000 --- a/vm/src/builtins/function/jitfunc.rs +++ /dev/null @@ -1,199 +0,0 @@ -use crate::{ - builtins::{bool_, float, int, PyBaseExceptionRef, PyDictRef, PyFunction, PyStrInterned}, - bytecode::CodeFlags, - convert::ToPyObject, - function::FuncArgs, - AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, -}; -use num_traits::ToPrimitive; -use rustpython_jit::{AbiValue, Args, CompiledCode, JitArgumentError, JitType}; - -#[derive(Debug, thiserror::Error)] -pub enum ArgsError { - #[error("wrong number of arguments passed")] - WrongNumberOfArgs, - #[error("argument passed multiple times")] - ArgPassedMultipleTimes, - #[error("not a keyword argument")] - NotAKeywordArg, - #[error("not all arguments passed")] - NotAllArgsPassed, - #[error("integer can't fit into a machine integer")] - IntOverflow, - #[error("type can't be used in a jit function")] - NonJitType, - #[error("{0}")] - JitError(#[from] JitArgumentError), -} - -impl ToPyObject for AbiValue { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - AbiValue::Int(i) => i.to_pyobject(vm), - AbiValue::Float(f) => f.to_pyobject(vm), - AbiValue::Bool(b) => b.to_pyobject(vm), - _ => unimplemented!(), - } - } -} - -pub fn new_jit_error(msg: String, vm: &VirtualMachine) -> PyBaseExceptionRef { - let jit_error = vm.ctx.exceptions.jit_error.to_owned(); - vm.new_exception_msg(jit_error, msg) -} - -fn get_jit_arg_type(dict: &PyDictRef, name: &str, vm: &VirtualMachine) -> PyResult<JitType> { - if let Some(value) = dict.get_item_opt(name, vm)? { - if value.is(vm.ctx.types.int_type) { - Ok(JitType::Int) - } else if value.is(vm.ctx.types.float_type) { - Ok(JitType::Float) - } else if value.is(vm.ctx.types.bool_type) { - Ok(JitType::Bool) - } else { - Err(new_jit_error( - "Jit requires argument to be either int or float".to_owned(), - vm, - )) - } - } else { - Err(new_jit_error( - format!("argument {name} needs annotation"), - vm, - )) - } -} - -pub fn get_jit_arg_types(func: &Py<PyFunction>, vm: &VirtualMachine) -> PyResult<Vec<JitType>> { - let arg_names = func.code.arg_names(); - - if func - .code - .flags - .intersects(CodeFlags::HAS_VARARGS | CodeFlags::HAS_VARKEYWORDS) - { - return Err(new_jit_error( - "Can't jit functions with variable number of arguments".to_owned(), - vm, - )); - } - - if arg_names.args.is_empty() && arg_names.kwonlyargs.is_empty() { - return Ok(Vec::new()); - } - - let func_obj: PyObjectRef = func.as_ref().to_owned(); - let annotations = func_obj.get_attr("__annotations__", vm)?; - if vm.is_none(&annotations) { - Err(new_jit_error( - "Jitting function requires arguments to have annotations".to_owned(), - vm, - )) - } else if let Ok(dict) = PyDictRef::try_from_object(vm, annotations) { - let mut arg_types = Vec::new(); - - for arg in arg_names.args { - arg_types.push(get_jit_arg_type(&dict, arg.as_str(), vm)?); - } - - for arg in arg_names.kwonlyargs { - arg_types.push(get_jit_arg_type(&dict, arg.as_str(), vm)?); - } - - Ok(arg_types) - } else { - Err(vm.new_type_error("Function annotations aren't a dict".to_owned())) - } -} - -fn get_jit_value(vm: &VirtualMachine, obj: &PyObject) -> Result<AbiValue, ArgsError> { - // This does exact type checks as subclasses of int/float can't be passed to jitted functions - let cls = obj.class(); - if cls.is(vm.ctx.types.int_type) { - int::get_value(obj) - .to_i64() - .map(AbiValue::Int) - .ok_or(ArgsError::IntOverflow) - } else if cls.is(vm.ctx.types.float_type) { - Ok(AbiValue::Float( - obj.downcast_ref::<float::PyFloat>().unwrap().to_f64(), - )) - } else if cls.is(vm.ctx.types.bool_type) { - Ok(AbiValue::Bool(bool_::get_value(obj))) - } else { - Err(ArgsError::NonJitType) - } -} - -/// Like `fill_locals_from_args` but to populate arguments for calling a jit function. -/// This also doesn't do full error handling but instead return None if anything is wrong. In -/// that case it falls back to the executing the bytecode version which will call -/// `fill_locals_from_args` which will raise the actual exception if needed. -#[cfg(feature = "jit")] -pub(crate) fn get_jit_args<'a>( - func: &PyFunction, - func_args: &FuncArgs, - jitted_code: &'a CompiledCode, - vm: &VirtualMachine, -) -> Result<Args<'a>, ArgsError> { - let mut jit_args = jitted_code.args_builder(); - let nargs = func_args.args.len(); - let arg_names = func.code.arg_names(); - - if nargs > func.code.arg_count as usize || nargs < func.code.posonlyarg_count as usize { - return Err(ArgsError::WrongNumberOfArgs); - } - - // Add positional arguments - for i in 0..nargs { - jit_args.set(i, get_jit_value(vm, &func_args.args[i])?)?; - } - - // Handle keyword arguments - for (name, value) in &func_args.kwargs { - let arg_pos = - |args: &[&PyStrInterned], name: &str| args.iter().position(|arg| arg.as_str() == name); - if let Some(arg_idx) = arg_pos(arg_names.args, name) { - if jit_args.is_set(arg_idx) { - return Err(ArgsError::ArgPassedMultipleTimes); - } - jit_args.set(arg_idx, get_jit_value(vm, value)?)?; - } else if let Some(kwarg_idx) = arg_pos(arg_names.kwonlyargs, name) { - let arg_idx = kwarg_idx + func.code.arg_count as usize; - if jit_args.is_set(arg_idx) { - return Err(ArgsError::ArgPassedMultipleTimes); - } - jit_args.set(arg_idx, get_jit_value(vm, value)?)?; - } else { - return Err(ArgsError::NotAKeywordArg); - } - } - - let (defaults, kwdefaults) = func.defaults_and_kwdefaults.lock().clone(); - - // fill in positional defaults - if let Some(defaults) = defaults { - for (i, default) in defaults.iter().enumerate() { - let arg_idx = i + func.code.arg_count as usize - defaults.len(); - if !jit_args.is_set(arg_idx) { - jit_args.set(arg_idx, get_jit_value(vm, default)?)?; - } - } - } - - // fill in keyword only defaults - if let Some(kw_only_defaults) = kwdefaults { - for (i, name) in arg_names.kwonlyargs.iter().enumerate() { - let arg_idx = i + func.code.arg_count as usize; - if !jit_args.is_set(arg_idx) { - let default = kw_only_defaults - .get_item(&**name, vm) - .map_err(|_| ArgsError::NotAllArgsPassed) - .and_then(|obj| get_jit_value(vm, &obj))?; - jit_args.set(arg_idx, default)?; - } - } - } - - jit_args.into_args().ok_or(ArgsError::NotAllArgsPassed) -} diff --git a/vm/src/builtins/generator.rs b/vm/src/builtins/generator.rs deleted file mode 100644 index eceac5ba935..00000000000 --- a/vm/src/builtins/generator.rs +++ /dev/null @@ -1,116 +0,0 @@ -/* - * The mythical generator. - */ - -use super::{PyCode, PyStrRef, PyType}; -use crate::{ - class::PyClassImpl, - coroutine::Coro, - frame::FrameRef, - function::OptionalArg, - protocol::PyIterReturn, - types::{Constructor, IterNext, Iterable, Representable, SelfIter, Unconstructible}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "generator")] -#[derive(Debug)] -pub struct PyGenerator { - inner: Coro, -} - -impl PyPayload for PyGenerator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.generator_type - } -} - -#[pyclass(with(Py, Constructor, IterNext, Iterable))] -impl PyGenerator { - pub fn as_coro(&self) -> &Coro { - &self.inner - } - - pub fn new(frame: FrameRef, name: PyStrRef) -> Self { - PyGenerator { - inner: Coro::new(frame, name), - } - } - - #[pygetset(magic)] - fn name(&self) -> PyStrRef { - self.inner.name() - } - - #[pygetset(magic, setter)] - fn set_name(&self, name: PyStrRef) { - self.inner.set_name(name) - } - - #[pygetset] - fn gi_frame(&self, _vm: &VirtualMachine) -> FrameRef { - self.inner.frame() - } - #[pygetset] - fn gi_running(&self, _vm: &VirtualMachine) -> bool { - self.inner.running() - } - #[pygetset] - fn gi_code(&self, _vm: &VirtualMachine) -> PyRef<PyCode> { - self.inner.frame().code.clone() - } - #[pygetset] - fn gi_yieldfrom(&self, _vm: &VirtualMachine) -> Option<PyObjectRef> { - self.inner.frame().yield_from_target() - } -} - -#[pyclass] -impl Py<PyGenerator> { - #[pymethod] - fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - self.inner.send(self.as_object(), value, vm) - } - - #[pymethod] - fn throw( - &self, - exc_type: PyObjectRef, - exc_val: OptionalArg, - exc_tb: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn> { - self.inner.throw( - self.as_object(), - exc_type, - exc_val.unwrap_or_none(vm), - exc_tb.unwrap_or_none(vm), - vm, - ) - } - - #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult<()> { - self.inner.close(self.as_object(), vm) - } -} - -impl Unconstructible for PyGenerator {} - -impl Representable for PyGenerator { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - Ok(zelf.inner.repr(zelf.as_object(), zelf.get_id(), vm)) - } -} - -impl SelfIter for PyGenerator {} -impl IterNext for PyGenerator { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.send(vm.ctx.none(), vm) - } -} - -pub fn init(ctx: &Context) { - PyGenerator::extend_class(ctx, ctx.types.generator_type); -} diff --git a/vm/src/builtins/genericalias.rs b/vm/src/builtins/genericalias.rs deleted file mode 100644 index 2746b031283..00000000000 --- a/vm/src/builtins/genericalias.rs +++ /dev/null @@ -1,420 +0,0 @@ -use once_cell::sync::Lazy; - -use super::type_; -use crate::{ - atomic_func, - builtins::{PyList, PyStr, PyTuple, PyTupleRef, PyType, PyTypeRef}, - class::PyClassImpl, - common::hash, - convert::ToPyObject, - function::{FuncArgs, PyComparisonValue}, - protocol::{PyMappingMethods, PyNumberMethods}, - types::{ - AsMapping, AsNumber, Callable, Comparable, Constructor, GetAttr, Hashable, PyComparisonOp, - Representable, - }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, -}; -use std::fmt; - -static ATTR_EXCEPTIONS: [&str; 8] = [ - "__origin__", - "__args__", - "__parameters__", - "__mro_entries__", - "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ - "__reduce__", - "__copy__", - "__deepcopy__", -]; - -#[pyclass(module = "types", name = "GenericAlias")] -pub struct PyGenericAlias { - origin: PyTypeRef, - args: PyTupleRef, - parameters: PyTupleRef, -} - -impl fmt::Debug for PyGenericAlias { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("GenericAlias") - } -} - -impl PyPayload for PyGenericAlias { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.generic_alias_type - } -} - -impl Constructor for PyGenericAlias { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - if !args.kwargs.is_empty() { - return Err(vm.new_type_error("GenericAlias() takes no keyword arguments".to_owned())); - } - let (origin, arguments): (_, PyObjectRef) = args.bind(vm)?; - PyGenericAlias::new(origin, arguments, vm) - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -#[pyclass( - with( - AsNumber, - AsMapping, - Callable, - Comparable, - Constructor, - GetAttr, - Hashable, - Representable - ), - flags(BASETYPE) -)] -impl PyGenericAlias { - pub fn new(origin: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> Self { - let args = if let Ok(tuple) = args.try_to_ref::<PyTuple>(vm) { - tuple.to_owned() - } else { - PyTuple::new_ref(vec![args], &vm.ctx) - }; - - let parameters = make_parameters(&args, vm); - Self { - origin, - args, - parameters, - } - } - - fn repr(&self, vm: &VirtualMachine) -> PyResult<String> { - fn repr_item(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { - if obj.is(&vm.ctx.ellipsis) { - return Ok("...".to_string()); - } - - if vm - .get_attribute_opt(obj.clone(), identifier!(vm, __origin__))? - .is_some() - && vm - .get_attribute_opt(obj.clone(), identifier!(vm, __args__))? - .is_some() - { - return Ok(obj.repr(vm)?.as_str().to_string()); - } - - match ( - vm.get_attribute_opt(obj.clone(), identifier!(vm, __qualname__))? - .and_then(|o| o.downcast_ref::<PyStr>().map(|n| n.as_str().to_string())), - vm.get_attribute_opt(obj.clone(), identifier!(vm, __module__))? - .and_then(|o| o.downcast_ref::<PyStr>().map(|m| m.as_str().to_string())), - ) { - (None, _) | (_, None) => Ok(obj.repr(vm)?.as_str().to_string()), - (Some(qualname), Some(module)) => Ok(if module == "builtins" { - qualname - } else { - format!("{module}.{qualname}") - }), - } - } - - Ok(format!( - "{}[{}]", - repr_item(self.origin.clone().into(), vm)?, - if self.args.len() == 0 { - "()".to_owned() - } else { - self.args - .iter() - .map(|o| repr_item(o.clone(), vm)) - .collect::<PyResult<Vec<_>>>()? - .join(", ") - } - )) - } - - #[pygetset(magic)] - fn parameters(&self) -> PyObjectRef { - self.parameters.clone().into() - } - - #[pygetset(magic)] - fn args(&self) -> PyObjectRef { - self.args.clone().into() - } - - #[pygetset(magic)] - fn origin(&self) -> PyObjectRef { - self.origin.clone().into() - } - - #[pymethod(magic)] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let new_args = subs_parameters( - |vm| self.repr(vm), - self.args.clone(), - self.parameters.clone(), - needle, - vm, - )?; - - Ok( - PyGenericAlias::new(self.origin.clone(), new_args.to_pyobject(vm), vm) - .into_pyobject(vm), - ) - } - - #[pymethod(magic)] - fn dir(&self, vm: &VirtualMachine) -> PyResult<PyList> { - let dir = vm.dir(Some(self.origin()))?; - for exc in &ATTR_EXCEPTIONS { - if !dir.contains((*exc).to_pyobject(vm), vm)? { - dir.append((*exc).to_pyobject(vm)); - } - } - Ok(dir) - } - - #[pymethod(magic)] - fn reduce(zelf: &Py<Self>, vm: &VirtualMachine) -> (PyTypeRef, (PyTypeRef, PyTupleRef)) { - ( - vm.ctx.types.generic_alias_type.to_owned(), - (zelf.origin.clone(), zelf.args.clone()), - ) - } - - #[pymethod(magic)] - fn mro_entries(&self, _bases: PyObjectRef, vm: &VirtualMachine) -> PyTupleRef { - PyTuple::new_ref(vec![self.origin()], &vm.ctx) - } - - #[pymethod(magic)] - fn instancecheck(_zelf: PyRef<Self>, _obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm - .new_type_error("isinstance() argument 2 cannot be a parameterized generic".to_owned())) - } - - #[pymethod(magic)] - fn subclasscheck(_zelf: PyRef<Self>, _obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm - .new_type_error("issubclass() argument 2 cannot be a parameterized generic".to_owned())) - } - - #[pymethod(magic)] - fn ror(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - type_::or_(other, zelf, vm) - } - - #[pymethod(magic)] - fn or(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - type_::or_(zelf, other, vm) - } -} - -pub(crate) fn is_typevar(obj: &PyObjectRef, vm: &VirtualMachine) -> bool { - let class = obj.class(); - "TypeVar" == &*class.slot_name() - && class - .get_attr(identifier!(vm, __module__)) - .and_then(|o| o.downcast_ref::<PyStr>().map(|s| s.as_str() == "typing")) - .unwrap_or(false) -} - -pub(crate) fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { - let mut parameters: Vec<PyObjectRef> = Vec::with_capacity(args.len()); - for arg in args { - if is_typevar(arg, vm) { - if !parameters.iter().any(|param| param.is(arg)) { - parameters.push(arg.clone()); - } - } else if let Ok(obj) = arg.get_attr(identifier!(vm, __parameters__), vm) { - if let Ok(sub_params) = obj.try_to_ref::<PyTuple>(vm) { - for sub_param in sub_params { - if !parameters.iter().any(|param| param.is(sub_param)) { - parameters.push(sub_param.clone()); - } - } - } - } - } - parameters.shrink_to_fit(); - - PyTuple::new_ref(parameters, &vm.ctx) -} - -#[inline] -fn tuple_index(tuple: &PyTupleRef, item: &PyObjectRef) -> Option<usize> { - tuple.iter().position(|element| element.is(item)) -} - -fn subs_tvars( - obj: PyObjectRef, - params: &PyTupleRef, - argitems: &[PyObjectRef], - vm: &VirtualMachine, -) -> PyResult { - obj.get_attr(identifier!(vm, __parameters__), vm) - .ok() - .and_then(|sub_params| { - PyTupleRef::try_from_object(vm, sub_params) - .ok() - .and_then(|sub_params| { - if sub_params.len() > 0 { - let sub_args = sub_params - .iter() - .map(|arg| { - if let Some(idx) = tuple_index(params, arg) { - argitems[idx].clone() - } else { - arg.clone() - } - }) - .collect::<Vec<_>>(); - let sub_args: PyObjectRef = PyTuple::new_ref(sub_args, &vm.ctx).into(); - Some(obj.get_item(&*sub_args, vm)) - } else { - None - } - }) - }) - .unwrap_or(Ok(obj)) -} - -pub fn subs_parameters<F: Fn(&VirtualMachine) -> PyResult<String>>( - repr: F, - args: PyTupleRef, - parameters: PyTupleRef, - needle: PyObjectRef, - vm: &VirtualMachine, -) -> PyResult<PyTupleRef> { - let num_params = parameters.len(); - if num_params == 0 { - return Err(vm.new_type_error(format!("There are no type variables left in {}", repr(vm)?))); - } - - let items = needle.try_to_ref::<PyTuple>(vm); - let arg_items = match items { - Ok(tuple) => tuple.as_slice(), - Err(_) => std::slice::from_ref(&needle), - }; - - let num_items = arg_items.len(); - if num_params != num_items { - let plural = if num_items > num_params { - "many" - } else { - "few" - }; - return Err(vm.new_type_error(format!("Too {} arguments for {}", plural, repr(vm)?))); - } - - let new_args = args - .iter() - .map(|arg| { - if is_typevar(arg, vm) { - let idx = tuple_index(&parameters, arg).unwrap(); - Ok(arg_items[idx].clone()) - } else { - subs_tvars(arg.clone(), &parameters, arg_items, vm) - } - }) - .collect::<PyResult<Vec<_>>>()?; - - Ok(PyTuple::new_ref(new_args, &vm.ctx)) -} - -impl AsMapping for PyGenericAlias { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy<PyMappingMethods> = Lazy::new(|| PyMappingMethods { - subscript: atomic_func!(|mapping, needle, vm| { - PyGenericAlias::mapping_downcast(mapping).getitem(needle.to_owned(), vm) - }), - ..PyMappingMethods::NOT_IMPLEMENTED - }); - &AS_MAPPING - } -} - -impl AsNumber for PyGenericAlias { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| Ok(PyGenericAlias::or(a.to_owned(), b.to_owned(), vm))), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl Callable for PyGenericAlias { - type Args = FuncArgs; - fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyType::call(&zelf.origin, args, vm).map(|obj| { - if let Err(exc) = obj.set_attr(identifier!(vm, __orig_class__), zelf.to_owned(), vm) { - if !exc.fast_isinstance(vm.ctx.exceptions.attribute_error) - && !exc.fast_isinstance(vm.ctx.exceptions.type_error) - { - return Err(exc); - } - } - Ok(obj) - })? - } -} - -impl Comparable for PyGenericAlias { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - op.eq_only(|| { - let other = class_or_notimplemented!(Self, other); - Ok(PyComparisonValue::Implemented( - if !zelf - .origin() - .rich_compare_bool(&other.origin(), PyComparisonOp::Eq, vm)? - { - false - } else { - zelf.args() - .rich_compare_bool(&other.args(), PyComparisonOp::Eq, vm)? - }, - )) - }) - } -} - -impl Hashable for PyGenericAlias { - #[inline] - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { - Ok(zelf.origin.as_object().hash(vm)? ^ zelf.args.as_object().hash(vm)?) - } -} - -impl GetAttr for PyGenericAlias { - fn getattro(zelf: &Py<Self>, attr: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - for exc in ATTR_EXCEPTIONS.iter() { - if *(*exc) == attr.to_string() { - return zelf.as_object().generic_getattr(attr, vm); - } - } - zelf.origin().get_attr(attr, vm) - } -} - -impl Representable for PyGenericAlias { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - zelf.repr(vm) - } -} - -pub fn init(context: &Context) { - let generic_alias_type = &context.types.generic_alias_type; - PyGenericAlias::extend_class(context, generic_alias_type); -} diff --git a/vm/src/builtins/getset.rs b/vm/src/builtins/getset.rs deleted file mode 100644 index 2516fcd5660..00000000000 --- a/vm/src/builtins/getset.rs +++ /dev/null @@ -1,147 +0,0 @@ -/*! Python `attribute` descriptor class. (PyGetSet) - -*/ -use super::PyType; -use crate::{ - class::PyClassImpl, - function::{IntoPyGetterFunc, IntoPySetterFunc, PyGetterFunc, PySetterFunc, PySetterValue}, - types::{Constructor, GetDescriptor, Unconstructible}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "getset_descriptor")] -pub struct PyGetSet { - name: String, - class: &'static Py<PyType>, - getter: Option<PyGetterFunc>, - setter: Option<PySetterFunc>, - // doc: Option<String>, -} - -impl std::fmt::Debug for PyGetSet { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "PyGetSet {{ name: {}, getter: {}, setter: {} }}", - self.name, - if self.getter.is_some() { - "Some" - } else { - "None" - }, - if self.setter.is_some() { - "Some" - } else { - "None" - }, - ) - } -} - -impl PyPayload for PyGetSet { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.getset_type - } -} - -impl GetDescriptor for PyGetSet { - fn descr_get( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - _cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let (zelf, obj) = match Self::_check(&zelf, obj, vm) { - Some(obj) => obj, - None => return Ok(zelf), - }; - if let Some(ref f) = zelf.getter { - f(vm, obj) - } else { - Err(vm.new_attribute_error(format!( - "attribute '{}' of '{}' objects is not readable", - zelf.name, - Self::class(&vm.ctx).name() - ))) - } - } -} - -impl PyGetSet { - pub fn new(name: String, class: &'static Py<PyType>) -> Self { - Self { - name, - class, - getter: None, - setter: None, - } - } - - pub fn with_get<G, X>(mut self, getter: G) -> Self - where - G: IntoPyGetterFunc<X>, - { - self.getter = Some(getter.into_getter()); - self - } - - pub fn with_set<S, X>(mut self, setter: S) -> Self - where - S: IntoPySetterFunc<X>, - { - self.setter = Some(setter.into_setter()); - self - } -} - -#[pyclass(with(GetDescriptor, Constructor))] -impl PyGetSet { - // Descriptor methods - - #[pyslot] - fn descr_set( - zelf: &PyObject, - obj: PyObjectRef, - value: PySetterValue<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let zelf = zelf.try_to_ref::<Self>(vm)?; - if let Some(ref f) = zelf.setter { - f(vm, obj, value) - } else { - Err(vm.new_attribute_error(format!( - "attribute '{}' of '{}' objects is not writable", - zelf.name, - obj.class().name() - ))) - } - } - #[pymethod] - fn __set__( - zelf: PyObjectRef, - obj: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Assign(value), vm) - } - #[pymethod] - fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Delete, vm) - } - - #[pygetset(magic)] - fn name(&self) -> String { - self.name.clone() - } - - #[pygetset(magic)] - fn qualname(&self) -> String { - format!("{}.{}", self.class.slot_name(), self.name.clone()) - } -} -impl Unconstructible for PyGetSet {} - -pub(crate) fn init(context: &Context) { - PyGetSet::extend_class(context, context.types.getset_type); -} diff --git a/vm/src/builtins/int.rs b/vm/src/builtins/int.rs deleted file mode 100644 index 9b25a504fc7..00000000000 --- a/vm/src/builtins/int.rs +++ /dev/null @@ -1,870 +0,0 @@ -use super::{float, PyByteArray, PyBytes, PyStr, PyType, PyTypeRef}; -use crate::{ - builtins::PyStrRef, - bytesinner::PyBytesInner, - class::PyClassImpl, - common::{ - hash, - int::{bigint_to_finite_float, bytes_to_int, true_div}, - }, - convert::{IntoPyException, ToPyObject, ToPyResult}, - function::{ - ArgByteOrder, ArgIntoBool, OptionalArg, OptionalOption, PyArithmeticValue, - PyComparisonValue, - }, - protocol::PyNumberMethods, - types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, - TryFromBorrowedObject, VirtualMachine, -}; -use malachite_bigint::{BigInt, Sign}; -use num_integer::Integer; -use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero}; -use rustpython_format::FormatSpec; -use std::fmt; -use std::ops::{Neg, Not}; - -#[pyclass(module = false, name = "int")] -#[derive(Debug)] -pub struct PyInt { - value: BigInt, -} - -impl fmt::Display for PyInt { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - BigInt::fmt(&self.value, f) - } -} - -pub type PyIntRef = PyRef<PyInt>; - -impl<T> From<T> for PyInt -where - T: Into<BigInt>, -{ - fn from(v: T) -> Self { - Self { value: v.into() } - } -} - -impl PyPayload for PyInt { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.int_type - } - - fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self.value).into() - } -} - -macro_rules! impl_into_pyobject_int { - ($($t:ty)*) => {$( - impl ToPyObject for $t { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self).into() - } - } - )*}; -} - -impl_into_pyobject_int!(isize i8 i16 i32 i64 i128 usize u8 u16 u32 u64 u128 BigInt); - -macro_rules! impl_try_from_object_int { - ($(($t:ty, $to_prim:ident),)*) => {$( - impl<'a> TryFromBorrowedObject<'a> for $t { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - obj.try_value_with(|int: &PyInt| { - int.try_to_primitive(vm) - }, vm) - } - } - )*}; -} - -impl_try_from_object_int!( - (isize, to_isize), - (i8, to_i8), - (i16, to_i16), - (i32, to_i32), - (i64, to_i64), - (i128, to_i128), - (usize, to_usize), - (u8, to_u8), - (u16, to_u16), - (u32, to_u32), - (u64, to_u64), - (u128, to_u128), -); - -fn inner_pow(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { - if int2.is_negative() { - let v1 = try_to_float(int1, vm)?; - let v2 = try_to_float(int2, vm)?; - float::float_pow(v1, v2, vm) - } else { - let value = if let Some(v2) = int2.to_u64() { - return Ok(vm.ctx.new_int(Pow::pow(int1, v2)).into()); - } else if int1.is_one() { - 1 - } else if int1.is_zero() { - 0 - } else if int1 == &BigInt::from(-1) { - if int2.is_odd() { - -1 - } else { - 1 - } - } else { - // missing feature: BigInt exp - // practically, exp over u64 is not possible to calculate anyway - return Ok(vm.ctx.not_implemented()); - }; - Ok(vm.ctx.new_int(value).into()) - } -} - -fn inner_mod(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { - if int2.is_zero() { - Err(vm.new_zero_division_error("integer modulo by zero".to_owned())) - } else { - Ok(vm.ctx.new_int(int1.mod_floor(int2)).into()) - } -} - -fn inner_floordiv(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { - if int2.is_zero() { - Err(vm.new_zero_division_error("integer division by zero".to_owned())) - } else { - Ok(vm.ctx.new_int(int1.div_floor(int2)).into()) - } -} - -fn inner_divmod(int1: &BigInt, int2: &BigInt, vm: &VirtualMachine) -> PyResult { - if int2.is_zero() { - return Err(vm.new_zero_division_error("integer division or modulo by zero".to_owned())); - } - let (div, modulo) = int1.div_mod_floor(int2); - Ok(vm.new_tuple((div, modulo)).into()) -} - -fn inner_lshift(base: &BigInt, bits: &BigInt, vm: &VirtualMachine) -> PyResult { - inner_shift( - base, - bits, - |base, bits| base << bits, - |bits, vm| { - bits.to_usize().ok_or_else(|| { - vm.new_overflow_error("the number is too large to convert to int".to_owned()) - }) - }, - vm, - ) -} - -fn inner_rshift(base: &BigInt, bits: &BigInt, vm: &VirtualMachine) -> PyResult { - inner_shift( - base, - bits, - |base, bits| base >> bits, - |bits, _vm| Ok(bits.to_usize().unwrap_or(usize::MAX)), - vm, - ) -} - -fn inner_shift<F, S>( - base: &BigInt, - bits: &BigInt, - shift_op: F, - shift_bits: S, - vm: &VirtualMachine, -) -> PyResult -where - F: Fn(&BigInt, usize) -> BigInt, - S: Fn(&BigInt, &VirtualMachine) -> PyResult<usize>, -{ - if bits.is_negative() { - Err(vm.new_value_error("negative shift count".to_owned())) - } else if base.is_zero() { - Ok(vm.ctx.new_int(0).into()) - } else { - shift_bits(bits, vm).map(|bits| vm.ctx.new_int(shift_op(base, bits)).into()) - } -} - -fn inner_truediv(i1: &BigInt, i2: &BigInt, vm: &VirtualMachine) -> PyResult { - if i2.is_zero() { - return Err(vm.new_zero_division_error("division by zero".to_owned())); - } - - let float = true_div(i1, i2); - - if float.is_infinite() { - Err(vm.new_exception_msg( - vm.ctx.exceptions.overflow_error.to_owned(), - "integer division result too large for a float".to_owned(), - )) - } else { - Ok(vm.ctx.new_float(float).into()) - } -} - -impl Constructor for PyInt { - type Args = IntOptions; - - fn py_new(cls: PyTypeRef, options: Self::Args, vm: &VirtualMachine) -> PyResult { - let value = if let OptionalArg::Present(val) = options.val_options { - if let OptionalArg::Present(base) = options.base { - let base = base - .try_index(vm)? - .as_bigint() - .to_u32() - .filter(|&v| v == 0 || (2..=36).contains(&v)) - .ok_or_else(|| { - vm.new_value_error("int() base must be >= 2 and <= 36, or 0".to_owned()) - })?; - try_int_radix(&val, base, vm) - } else { - let val = if cls.is(vm.ctx.types.int_type) { - match val.downcast_exact::<PyInt>(vm) { - Ok(i) => { - return Ok(i.into_pyref().into()); - } - Err(val) => val, - } - } else { - val - }; - - val.try_int(vm).map(|x| x.as_bigint().clone()) - } - } else if let OptionalArg::Present(_) = options.base { - Err(vm.new_type_error("int() missing string argument".to_owned())) - } else { - Ok(Zero::zero()) - }?; - - Self::with_value(cls, value, vm).to_pyresult(vm) - } -} - -impl PyInt { - fn with_value<T>(cls: PyTypeRef, value: T, vm: &VirtualMachine) -> PyResult<PyRef<Self>> - where - T: Into<BigInt> + ToPrimitive, - { - if cls.is(vm.ctx.types.int_type) { - Ok(vm.ctx.new_int(value)) - } else if cls.is(vm.ctx.types.bool_type) { - Ok(vm.ctx.new_bool(!value.into().eq(&BigInt::zero()))) - } else { - PyInt::from(value).into_ref_with_type(vm, cls) - } - } - - pub fn as_bigint(&self) -> &BigInt { - &self.value - } - - // _PyLong_AsUnsignedLongMask - pub fn as_u32_mask(&self) -> u32 { - let v = self.as_bigint(); - v.to_u32() - .or_else(|| v.to_i32().map(|i| i as u32)) - .unwrap_or_else(|| { - let mut out = 0u32; - for digit in v.iter_u32_digits() { - out = out.wrapping_shl(32) | digit; - } - match v.sign() { - Sign::Minus => out * -1i32 as u32, - _ => out, - } - }) - } - - pub fn try_to_primitive<'a, I>(&'a self, vm: &VirtualMachine) -> PyResult<I> - where - I: PrimInt + TryFrom<&'a BigInt>, - { - I::try_from(self.as_bigint()).map_err(|_| { - vm.new_overflow_error(format!( - "Python int too large to convert to Rust {}", - std::any::type_name::<I>() - )) - }) - } - - #[inline] - fn int_op<F>(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyArithmeticValue<BigInt> - where - F: Fn(&BigInt, &BigInt) -> BigInt, - { - let r = other - .payload_if_subclass::<PyInt>(vm) - .map(|other| op(&self.value, &other.value)); - PyArithmeticValue::from_option(r) - } - - #[inline] - fn general_op<F>(&self, other: PyObjectRef, op: F, vm: &VirtualMachine) -> PyResult - where - F: Fn(&BigInt, &BigInt) -> PyResult, - { - if let Some(other) = other.payload_if_subclass::<PyInt>(vm) { - op(&self.value, &other.value) - } else { - Ok(vm.ctx.not_implemented()) - } - } -} - -#[pyclass( - flags(BASETYPE), - with(PyRef, Comparable, Hashable, Constructor, AsNumber, Representable) -)] -impl PyInt { - #[pymethod(name = "__radd__")] - #[pymethod(magic)] - fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a + b, vm) - } - - #[pymethod(magic)] - fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a - b, vm) - } - - #[pymethod(magic)] - fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| b - a, vm) - } - - #[pymethod(name = "__rmul__")] - #[pymethod(magic)] - fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a * b, vm) - } - - #[pymethod(magic)] - fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_truediv(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_truediv(b, a, vm), vm) - } - - #[pymethod(magic)] - fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_floordiv(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rfloordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_floordiv(b, a, vm), vm) - } - - #[pymethod(magic)] - fn lshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_lshift(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rlshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_lshift(b, a, vm), vm) - } - - #[pymethod(magic)] - fn rshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_rshift(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rrshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_rshift(b, a, vm), vm) - } - - #[pymethod(name = "__rxor__")] - #[pymethod(magic)] - pub fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a ^ b, vm) - } - - #[pymethod(name = "__ror__")] - #[pymethod(magic)] - pub fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a | b, vm) - } - - #[pymethod(name = "__rand__")] - #[pymethod(magic)] - pub fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyArithmeticValue<BigInt> { - self.int_op(other, |a, b| a & b, vm) - } - - fn modpow(&self, other: PyObjectRef, modulus: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let modulus = match modulus.payload_if_subclass::<PyInt>(vm) { - Some(val) => val.as_bigint(), - None => return Ok(vm.ctx.not_implemented()), - }; - if modulus.is_zero() { - return Err(vm.new_value_error("pow() 3rd argument cannot be 0".to_owned())); - } - - self.general_op( - other, - |a, b| { - let i = if b.is_negative() { - // modular multiplicative inverse - // based on rust-num/num-integer#10, should hopefully be published soon - fn normalize(a: BigInt, n: &BigInt) -> BigInt { - let a = a % n; - if a.is_negative() { - a + n - } else { - a - } - } - fn inverse(a: BigInt, n: &BigInt) -> Option<BigInt> { - use num_integer::*; - let ExtendedGcd { gcd, x: c, .. } = a.extended_gcd(n); - if gcd.is_one() { - Some(normalize(c, n)) - } else { - None - } - } - let a = inverse(a % modulus, modulus).ok_or_else(|| { - vm.new_value_error( - "base is not invertible for the given modulus".to_owned(), - ) - })?; - let b = -b; - a.modpow(&b, modulus) - } else { - a.modpow(b, modulus) - }; - Ok(vm.ctx.new_int(i).into()) - }, - vm, - ) - } - - #[pymethod(magic)] - fn pow( - &self, - other: PyObjectRef, - r#mod: OptionalOption<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - match r#mod.flatten() { - Some(modulus) => self.modpow(other, modulus, vm), - None => self.general_op(other, |a, b| inner_pow(a, b, vm), vm), - } - } - - #[pymethod(magic)] - fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_pow(b, a, vm), vm) - } - - #[pymethod(name = "__mod__")] - fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_mod(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_mod(b, a, vm), vm) - } - - #[pymethod(magic)] - fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_divmod(a, b, vm), vm) - } - - #[pymethod(magic)] - fn rdivmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.general_op(other, |a, b| inner_divmod(b, a, vm), vm) - } - - #[pymethod(magic)] - fn neg(&self) -> BigInt { - -(&self.value) - } - - #[pymethod(magic)] - fn abs(&self) -> BigInt { - self.value.abs() - } - - #[pymethod(magic)] - fn round( - zelf: PyRef<Self>, - precision: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - match precision { - OptionalArg::Missing => (), - OptionalArg::Present(ref value) => { - // Only accept int type ndigits - let _ndigits = value.payload_if_subclass::<PyInt>(vm).ok_or_else(|| { - vm.new_type_error(format!( - "'{}' object cannot be interpreted as an integer", - value.class().name() - )) - })?; - } - } - Ok(zelf) - } - - #[pymethod(magic)] - fn pos(&self) -> BigInt { - self.value.clone() - } - - #[pymethod(magic)] - fn float(&self, vm: &VirtualMachine) -> PyResult<f64> { - try_to_float(&self.value, vm) - } - - #[pymethod(magic)] - fn trunc(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { - zelf.int(vm) - } - - #[pymethod(magic)] - fn floor(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { - zelf.int(vm) - } - - #[pymethod(magic)] - fn ceil(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { - zelf.int(vm) - } - - #[pymethod(magic)] - fn index(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { - zelf.int(vm) - } - - #[pymethod(magic)] - fn invert(&self) -> BigInt { - !(&self.value) - } - - #[pymethod(magic)] - fn format(&self, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<String> { - FormatSpec::parse(spec.as_str()) - .and_then(|format_spec| format_spec.format_int(&self.value)) - .map_err(|err| err.into_pyexception(vm)) - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - !self.value.is_zero() - } - - #[pymethod(magic)] - fn sizeof(&self) -> usize { - std::mem::size_of::<Self>() + (((self.value.bits() + 7) & !7) / 8) as usize - } - - #[pymethod] - fn as_integer_ratio(&self, vm: &VirtualMachine) -> (PyRef<Self>, i32) { - (vm.ctx.new_bigint(&self.value), 1) - } - - #[pymethod] - fn bit_length(&self) -> u64 { - self.value.bits() - } - - #[pymethod] - fn conjugate(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { - zelf.int(vm) - } - - #[pyclassmethod] - fn from_bytes( - cls: PyTypeRef, - args: IntFromByteArgs, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - let signed = args.signed.map_or(false, Into::into); - let value = match (args.byteorder, signed) { - (ArgByteOrder::Big, true) => BigInt::from_signed_bytes_be(args.bytes.as_bytes()), - (ArgByteOrder::Big, false) => BigInt::from_bytes_be(Sign::Plus, args.bytes.as_bytes()), - (ArgByteOrder::Little, true) => BigInt::from_signed_bytes_le(args.bytes.as_bytes()), - (ArgByteOrder::Little, false) => { - BigInt::from_bytes_le(Sign::Plus, args.bytes.as_bytes()) - } - }; - Self::with_value(cls, value, vm) - } - - #[pymethod] - fn to_bytes(&self, args: IntToByteArgs, vm: &VirtualMachine) -> PyResult<PyBytes> { - let signed = args.signed.map_or(false, Into::into); - let byte_len = args.length; - - let value = self.as_bigint(); - match value.sign() { - Sign::Minus if !signed => { - return Err( - vm.new_overflow_error("can't convert negative int to unsigned".to_owned()) - ) - } - Sign::NoSign => return Ok(vec![0u8; byte_len].into()), - _ => {} - } - - let mut origin_bytes = match (args.byteorder, signed) { - (ArgByteOrder::Big, true) => value.to_signed_bytes_be(), - (ArgByteOrder::Big, false) => value.to_bytes_be().1, - (ArgByteOrder::Little, true) => value.to_signed_bytes_le(), - (ArgByteOrder::Little, false) => value.to_bytes_le().1, - }; - - let origin_len = origin_bytes.len(); - if origin_len > byte_len { - return Err(vm.new_overflow_error("int too big to convert".to_owned())); - } - - let mut append_bytes = match value.sign() { - Sign::Minus => vec![255u8; byte_len - origin_len], - _ => vec![0u8; byte_len - origin_len], - }; - - let bytes = match args.byteorder { - ArgByteOrder::Big => { - let mut bytes = append_bytes; - bytes.append(&mut origin_bytes); - bytes - } - ArgByteOrder::Little => { - let mut bytes = origin_bytes; - bytes.append(&mut append_bytes); - bytes - } - }; - Ok(bytes.into()) - } - - #[pygetset] - fn real(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { - zelf.int(vm) - } - - #[pygetset] - fn imag(&self) -> usize { - 0 - } - - #[pygetset] - fn numerator(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRefExact<Self> { - zelf.int(vm) - } - - #[pygetset] - fn denominator(&self) -> usize { - 1 - } - - #[pymethod] - /// Returns the number of ones 1 an int. When the number is < 0, - /// then it returns the number of ones of the absolute value. - fn bit_count(&self) -> u32 { - self.value.iter_u32_digits().map(|n| n.count_ones()).sum() - } - - #[pymethod(magic)] - fn getnewargs(&self, vm: &VirtualMachine) -> PyObjectRef { - (self.value.clone(),).to_pyobject(vm) - } -} - -#[pyclass] -impl PyRef<PyInt> { - #[pymethod(magic)] - fn int(self, vm: &VirtualMachine) -> PyRefExact<PyInt> { - self.into_exact_or(&vm.ctx, |zelf| unsafe { - // TODO: this is actually safe. we need better interface - PyRefExact::new_unchecked(vm.ctx.new_bigint(&zelf.value)) - }) - } -} - -impl Comparable for PyInt { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - let r = other - .payload_if_subclass::<PyInt>(vm) - .map(|other| op.eval_ord(zelf.value.cmp(&other.value))); - Ok(PyComparisonValue::from_option(r)) - } -} - -impl Representable for PyInt { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - Ok(zelf.value.to_string()) - } -} - -impl Hashable for PyInt { - #[inline] - fn hash(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<hash::PyHash> { - Ok(hash::hash_bigint(zelf.as_bigint())) - } -} - -impl AsNumber for PyInt { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyInt::AS_NUMBER; - &AS_NUMBER - } - - #[inline] - fn clone_exact(zelf: &Py<Self>, vm: &VirtualMachine) -> PyRef<Self> { - vm.ctx.new_bigint(&zelf.value) - } -} - -impl PyInt { - pub(super) const AS_NUMBER: PyNumberMethods = PyNumberMethods { - add: Some(|a, b, vm| PyInt::number_op(a, b, |a, b, _vm| a + b, vm)), - subtract: Some(|a, b, vm| PyInt::number_op(a, b, |a, b, _vm| a - b, vm)), - multiply: Some(|a, b, vm| PyInt::number_op(a, b, |a, b, _vm| a * b, vm)), - remainder: Some(|a, b, vm| PyInt::number_op(a, b, inner_mod, vm)), - divmod: Some(|a, b, vm| PyInt::number_op(a, b, inner_divmod, vm)), - power: Some(|a, b, c, vm| { - if let (Some(a), Some(b)) = ( - a.payload::<Self>(), - if b.payload_is::<Self>() { - Some(b) - } else { - None - }, - ) { - if vm.is_none(c) { - a.general_op(b.to_owned(), |a, b| inner_pow(a, b, vm), vm) - } else { - a.modpow(b.to_owned(), c.to_owned(), vm) - } - } else { - Ok(vm.ctx.not_implemented()) - } - }), - negative: Some(|num, vm| (&PyInt::number_downcast(num).value).neg().to_pyresult(vm)), - positive: Some(|num, vm| Ok(PyInt::number_downcast_exact(num, vm).into())), - absolute: Some(|num, vm| PyInt::number_downcast(num).value.abs().to_pyresult(vm)), - boolean: Some(|num, _vm| Ok(PyInt::number_downcast(num).value.is_zero())), - invert: Some(|num, vm| (&PyInt::number_downcast(num).value).not().to_pyresult(vm)), - lshift: Some(|a, b, vm| PyInt::number_op(a, b, inner_lshift, vm)), - rshift: Some(|a, b, vm| PyInt::number_op(a, b, inner_rshift, vm)), - and: Some(|a, b, vm| PyInt::number_op(a, b, |a, b, _vm| a & b, vm)), - xor: Some(|a, b, vm| PyInt::number_op(a, b, |a, b, _vm| a ^ b, vm)), - or: Some(|a, b, vm| PyInt::number_op(a, b, |a, b, _vm| a | b, vm)), - int: Some(|num, vm| Ok(PyInt::number_downcast_exact(num, vm).into())), - float: Some(|num, vm| { - let zelf = PyInt::number_downcast(num); - try_to_float(&zelf.value, vm).map(|x| vm.ctx.new_float(x).into()) - }), - floor_divide: Some(|a, b, vm| PyInt::number_op(a, b, inner_floordiv, vm)), - true_divide: Some(|a, b, vm| PyInt::number_op(a, b, inner_truediv, vm)), - index: Some(|num, vm| Ok(PyInt::number_downcast_exact(num, vm).into())), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - - fn number_op<F, R>(a: &PyObject, b: &PyObject, op: F, vm: &VirtualMachine) -> PyResult - where - F: FnOnce(&BigInt, &BigInt, &VirtualMachine) -> R, - R: ToPyResult, - { - if let (Some(a), Some(b)) = (a.payload::<Self>(), b.payload::<Self>()) { - op(&a.value, &b.value, vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - } -} - -#[derive(FromArgs)] -pub struct IntOptions { - #[pyarg(positional, optional)] - val_options: OptionalArg<PyObjectRef>, - #[pyarg(any, optional)] - base: OptionalArg<PyObjectRef>, -} - -#[derive(FromArgs)] -struct IntFromByteArgs { - bytes: PyBytesInner, - #[pyarg(any, default = "ArgByteOrder::Big")] - byteorder: ArgByteOrder, - #[pyarg(named, optional)] - signed: OptionalArg<ArgIntoBool>, -} - -#[derive(FromArgs)] -struct IntToByteArgs { - #[pyarg(any, default = "1")] - length: usize, - #[pyarg(any, default = "ArgByteOrder::Big")] - byteorder: ArgByteOrder, - #[pyarg(named, optional)] - signed: OptionalArg<ArgIntoBool>, -} - -fn try_int_radix(obj: &PyObject, base: u32, vm: &VirtualMachine) -> PyResult<BigInt> { - debug_assert!(base == 0 || (2..=36).contains(&base)); - - let opt = match_class!(match obj.to_owned() { - string @ PyStr => { - let s = string.as_str(); - bytes_to_int(s.as_bytes(), base) - } - bytes @ PyBytes => { - let bytes = bytes.as_bytes(); - bytes_to_int(bytes, base) - } - bytearray @ PyByteArray => { - let inner = bytearray.borrow_buf(); - bytes_to_int(&inner, base) - } - _ => { - return Err( - vm.new_type_error("int() can't convert non-string with explicit base".to_owned()) - ); - } - }); - match opt { - Some(int) => Ok(int), - None => Err(vm.new_value_error(format!( - "invalid literal for int() with base {}: {}", - base, - obj.repr(vm)?, - ))), - } -} - -// Retrieve inner int value: -pub(crate) fn get_value(obj: &PyObject) -> &BigInt { - &obj.payload::<PyInt>().unwrap().value -} - -pub fn try_to_float(int: &BigInt, vm: &VirtualMachine) -> PyResult<f64> { - bigint_to_finite_float(int) - .ok_or_else(|| vm.new_overflow_error("int too large to convert to float".to_owned())) -} - -pub(crate) fn init(context: &Context) { - PyInt::extend_class(context, context.types.int_type); -} diff --git a/vm/src/builtins/iter.rs b/vm/src/builtins/iter.rs deleted file mode 100644 index 2fed23a5c47..00000000000 --- a/vm/src/builtins/iter.rs +++ /dev/null @@ -1,287 +0,0 @@ -/* - * iterator types - */ - -use super::{PyInt, PyTupleRef, PyType}; -use crate::{ - class::PyClassImpl, - function::ArgCallable, - object::{Traverse, TraverseFn}, - protocol::{PyIterReturn, PySequence, PySequenceMethods}, - types::{IterNext, Iterable, SelfIter}, - Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; -use rustpython_common::{ - lock::{PyMutex, PyRwLock, PyRwLockUpgradableReadGuard}, - static_cell, -}; - -/// Marks status of iterator. -#[derive(Debug, Clone)] -pub enum IterStatus<T> { - /// Iterator hasn't raised StopIteration. - Active(T), - /// Iterator has raised StopIteration. - Exhausted, -} - -unsafe impl<T: Traverse> Traverse for IterStatus<T> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - match self { - IterStatus::Active(ref r) => r.traverse(tracer_fn), - IterStatus::Exhausted => (), - } - } -} - -#[derive(Debug)] -pub struct PositionIterInternal<T> { - pub status: IterStatus<T>, - pub position: usize, -} - -unsafe impl<T: Traverse> Traverse for PositionIterInternal<T> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.status.traverse(tracer_fn) - } -} - -impl<T> PositionIterInternal<T> { - pub fn new(obj: T, position: usize) -> Self { - Self { - status: IterStatus::Active(obj), - position, - } - } - - pub fn set_state<F>(&mut self, state: PyObjectRef, f: F, vm: &VirtualMachine) -> PyResult<()> - where - F: FnOnce(&T, usize) -> usize, - { - if let IterStatus::Active(obj) = &self.status { - if let Some(i) = state.payload::<PyInt>() { - let i = i.try_to_primitive(vm).unwrap_or(0); - self.position = f(obj, i); - Ok(()) - } else { - Err(vm.new_type_error("an integer is required.".to_owned())) - } - } else { - Ok(()) - } - } - - fn _reduce<F>(&self, func: PyObjectRef, f: F, vm: &VirtualMachine) -> PyTupleRef - where - F: FnOnce(&T) -> PyObjectRef, - { - if let IterStatus::Active(obj) = &self.status { - vm.new_tuple((func, (f(obj),), self.position)) - } else { - vm.new_tuple((func, (vm.ctx.new_list(Vec::new()),))) - } - } - - pub fn builtins_iter_reduce<F>(&self, f: F, vm: &VirtualMachine) -> PyTupleRef - where - F: FnOnce(&T) -> PyObjectRef, - { - let iter = builtins_iter(vm).to_owned(); - self._reduce(iter, f, vm) - } - - pub fn builtins_reversed_reduce<F>(&self, f: F, vm: &VirtualMachine) -> PyTupleRef - where - F: FnOnce(&T) -> PyObjectRef, - { - let reversed = builtins_reversed(vm).to_owned(); - self._reduce(reversed, f, vm) - } - - fn _next<F, OP>(&mut self, f: F, op: OP) -> PyResult<PyIterReturn> - where - F: FnOnce(&T, usize) -> PyResult<PyIterReturn>, - OP: FnOnce(&mut Self), - { - if let IterStatus::Active(obj) = &self.status { - let ret = f(obj, self.position); - if let Ok(PyIterReturn::Return(_)) = ret { - op(self); - } else { - self.status = IterStatus::Exhausted; - } - ret - } else { - Ok(PyIterReturn::StopIteration(None)) - } - } - - pub fn next<F>(&mut self, f: F) -> PyResult<PyIterReturn> - where - F: FnOnce(&T, usize) -> PyResult<PyIterReturn>, - { - self._next(f, |zelf| zelf.position += 1) - } - - pub fn rev_next<F>(&mut self, f: F) -> PyResult<PyIterReturn> - where - F: FnOnce(&T, usize) -> PyResult<PyIterReturn>, - { - self._next(f, |zelf| { - if zelf.position == 0 { - zelf.status = IterStatus::Exhausted; - } else { - zelf.position -= 1; - } - }) - } - - pub fn length_hint<F>(&self, f: F) -> usize - where - F: FnOnce(&T) -> usize, - { - if let IterStatus::Active(obj) = &self.status { - f(obj).saturating_sub(self.position) - } else { - 0 - } - } - - pub fn rev_length_hint<F>(&self, f: F) -> usize - where - F: FnOnce(&T) -> usize, - { - if let IterStatus::Active(obj) = &self.status { - if self.position <= f(obj) { - return self.position + 1; - } - } - 0 - } -} - -pub fn builtins_iter(vm: &VirtualMachine) -> &PyObject { - static_cell! { - static INSTANCE: PyObjectRef; - } - INSTANCE.get_or_init(|| vm.builtins.get_attr("iter", vm).unwrap()) -} - -pub fn builtins_reversed(vm: &VirtualMachine) -> &PyObject { - static_cell! { - static INSTANCE: PyObjectRef; - } - INSTANCE.get_or_init(|| vm.builtins.get_attr("reversed", vm).unwrap()) -} - -#[pyclass(module = false, name = "iterator", traverse)] -#[derive(Debug)] -pub struct PySequenceIterator { - // cached sequence methods - #[pytraverse(skip)] - seq_methods: &'static PySequenceMethods, - internal: PyMutex<PositionIterInternal<PyObjectRef>>, -} - -impl PyPayload for PySequenceIterator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.iter_type - } -} - -#[pyclass(with(IterNext, Iterable))] -impl PySequenceIterator { - pub fn new(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { - let seq = PySequence::try_protocol(&obj, vm)?; - Ok(Self { - seq_methods: seq.methods, - internal: PyMutex::new(PositionIterInternal::new(obj, 0)), - }) - } - - #[pymethod(magic)] - fn length_hint(&self, vm: &VirtualMachine) -> PyObjectRef { - let internal = self.internal.lock(); - if let IterStatus::Active(obj) = &internal.status { - let seq = PySequence { - obj, - methods: self.seq_methods, - }; - seq.length(vm) - .map(|x| PyInt::from(x).into_pyobject(vm)) - .unwrap_or_else(|_| vm.ctx.not_implemented()) - } else { - PyInt::from(0).into_pyobject(vm) - } - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal.lock().builtins_iter_reduce(|x| x.clone(), vm) - } - - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.internal.lock().set_state(state, |_, pos| pos, vm) - } -} - -impl SelfIter for PySequenceIterator {} -impl IterNext for PySequenceIterator { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.internal.lock().next(|obj, pos| { - let seq = PySequence { - obj, - methods: zelf.seq_methods, - }; - PyIterReturn::from_getitem_result(seq.get_item(pos as isize, vm), vm) - }) - } -} - -#[pyclass(module = false, name = "callable_iterator", traverse)] -#[derive(Debug)] -pub struct PyCallableIterator { - sentinel: PyObjectRef, - status: PyRwLock<IterStatus<ArgCallable>>, -} - -impl PyPayload for PyCallableIterator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.callable_iterator - } -} - -#[pyclass(with(IterNext, Iterable))] -impl PyCallableIterator { - pub fn new(callable: ArgCallable, sentinel: PyObjectRef) -> Self { - Self { - sentinel, - status: PyRwLock::new(IterStatus::Active(callable)), - } - } -} - -impl SelfIter for PyCallableIterator {} -impl IterNext for PyCallableIterator { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let status = zelf.status.upgradable_read(); - let next = if let IterStatus::Active(callable) = &*status { - let ret = callable.invoke((), vm)?; - if vm.bool_eq(&ret, &zelf.sentinel)? { - *PyRwLockUpgradableReadGuard::upgrade(status) = IterStatus::Exhausted; - PyIterReturn::StopIteration(None) - } else { - PyIterReturn::Return(ret) - } - } else { - PyIterReturn::StopIteration(None) - }; - Ok(next) - } -} - -pub fn init(context: &Context) { - PySequenceIterator::extend_class(context, context.types.iter_type); - PyCallableIterator::extend_class(context, context.types.callable_iterator); -} diff --git a/vm/src/builtins/list.rs b/vm/src/builtins/list.rs deleted file mode 100644 index 03503a0ceaf..00000000000 --- a/vm/src/builtins/list.rs +++ /dev/null @@ -1,630 +0,0 @@ -use super::{PositionIterInternal, PyGenericAlias, PyTupleRef, PyType, PyTypeRef}; -use crate::atomic_func; -use crate::common::lock::{ - PyMappedRwLockReadGuard, PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, -}; -use crate::{ - class::PyClassImpl, - convert::ToPyObject, - function::{ArgSize, FuncArgs, OptionalArg, PyComparisonValue}, - iter::PyExactSizeIterator, - protocol::{PyIterReturn, PyMappingMethods, PySequenceMethods}, - recursion::ReprGuard, - sequence::{MutObjectSequenceOp, OptionalRangeArgs, SequenceExt, SequenceMutExt}, - sliceable::{SequenceIndex, SliceableSequenceMutOp, SliceableSequenceOp}, - types::{ - AsMapping, AsSequence, Comparable, Constructor, Initializer, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, - }, - utils::collection_repr, - vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, -}; -use std::{fmt, ops::DerefMut}; - -#[pyclass(module = false, name = "list", unhashable = true, traverse)] -#[derive(Default)] -pub struct PyList { - elements: PyRwLock<Vec<PyObjectRef>>, -} - -impl fmt::Debug for PyList { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // TODO: implement more detailed, non-recursive Debug formatter - f.write_str("list") - } -} - -impl From<Vec<PyObjectRef>> for PyList { - fn from(elements: Vec<PyObjectRef>) -> Self { - PyList { - elements: PyRwLock::new(elements), - } - } -} - -impl FromIterator<PyObjectRef> for PyList { - fn from_iter<T: IntoIterator<Item = PyObjectRef>>(iter: T) -> Self { - Vec::from_iter(iter).into() - } -} - -impl PyPayload for PyList { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.list_type - } -} - -impl ToPyObject for Vec<PyObjectRef> { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - PyList::new_ref(self, &vm.ctx).into() - } -} - -impl PyList { - pub fn new_ref(elements: Vec<PyObjectRef>, ctx: &Context) -> PyRef<Self> { - PyRef::new_ref(Self::from(elements), ctx.types.list_type.to_owned(), None) - } - - pub fn borrow_vec(&self) -> PyMappedRwLockReadGuard<'_, [PyObjectRef]> { - PyRwLockReadGuard::map(self.elements.read(), |v| &**v) - } - - pub fn borrow_vec_mut(&self) -> PyRwLockWriteGuard<'_, Vec<PyObjectRef>> { - self.elements.write() - } - - fn repeat(&self, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - let elements = &*self.borrow_vec(); - let v = elements.mul(vm, n)?; - Ok(Self::new_ref(v, &vm.ctx)) - } - - fn irepeat(zelf: PyRef<Self>, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.borrow_vec_mut().imul(vm, n)?; - Ok(zelf) - } -} - -#[derive(FromArgs, Default, Traverse)] -pub(crate) struct SortOptions { - #[pyarg(named, default)] - key: Option<PyObjectRef>, - #[pytraverse(skip)] - #[pyarg(named, default = "false")] - reverse: bool, -} - -pub type PyListRef = PyRef<PyList>; - -#[pyclass( - with( - Constructor, - Initializer, - AsMapping, - Iterable, - Comparable, - AsSequence, - Representable - ), - flags(BASETYPE) -)] -impl PyList { - #[pymethod] - pub(crate) fn append(&self, x: PyObjectRef) { - self.borrow_vec_mut().push(x); - } - - #[pymethod] - pub(crate) fn extend(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut new_elements = x.try_to_value(vm)?; - self.borrow_vec_mut().append(&mut new_elements); - Ok(()) - } - - #[pymethod] - pub(crate) fn insert(&self, position: isize, element: PyObjectRef) { - let mut elements = self.borrow_vec_mut(); - let position = elements.saturate_index(position); - elements.insert(position, element); - } - - fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - let other = other.payload_if_subclass::<PyList>(vm).ok_or_else(|| { - vm.new_type_error(format!( - "Cannot add {} and {}", - Self::class(&vm.ctx).name(), - other.class().name() - )) - })?; - let mut elements = self.borrow_vec().to_vec(); - elements.extend(other.borrow_vec().iter().cloned()); - Ok(Self::new_ref(elements, &vm.ctx)) - } - - #[pymethod(magic)] - fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - self.concat(&other, vm) - } - - fn inplace_concat( - zelf: &Py<Self>, - other: &PyObject, - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { - let mut seq = extract_cloned(other, Ok, vm)?; - zelf.borrow_vec_mut().append(&mut seq); - Ok(zelf.to_owned().into()) - } - - #[pymethod(magic)] - fn iadd(zelf: PyRef<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - let mut seq = extract_cloned(&other, Ok, vm)?; - zelf.borrow_vec_mut().append(&mut seq); - Ok(zelf) - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - !self.borrow_vec().is_empty() - } - - #[pymethod] - fn clear(&self) { - let _removed = std::mem::take(self.borrow_vec_mut().deref_mut()); - } - - #[pymethod] - fn copy(&self, vm: &VirtualMachine) -> PyRef<Self> { - Self::new_ref(self.borrow_vec().to_vec(), &vm.ctx) - } - - #[pymethod(magic)] - fn len(&self) -> usize { - self.borrow_vec().len() - } - - #[pymethod(magic)] - fn sizeof(&self) -> usize { - std::mem::size_of::<Self>() - + self.elements.read().capacity() * std::mem::size_of::<PyObjectRef>() - } - - #[pymethod] - fn reverse(&self) { - self.borrow_vec_mut().reverse(); - } - - #[pymethod(magic)] - fn reversed(zelf: PyRef<Self>) -> PyListReverseIterator { - let position = zelf.len().saturating_sub(1); - PyListReverseIterator { - internal: PyMutex::new(PositionIterInternal::new(zelf, position)), - } - } - - fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { - SequenceIndex::Int(i) => self.borrow_vec().getitem_by_index(vm, i), - SequenceIndex::Slice(slice) => self - .borrow_vec() - .getitem_by_slice(vm, slice) - .map(|x| vm.ctx.new_list(x).into()), - } - } - - #[pymethod(magic)] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self._getitem(&needle, vm) - } - - fn _setitem(&self, needle: &PyObject, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { - SequenceIndex::Int(index) => self.borrow_vec_mut().setitem_by_index(vm, index, value), - SequenceIndex::Slice(slice) => { - let sec = extract_cloned(&value, Ok, vm)?; - self.borrow_vec_mut().setitem_by_slice(vm, slice, &sec) - } - } - } - - #[pymethod(magic)] - fn setitem( - &self, - needle: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - self._setitem(&needle, value, vm) - } - - #[pymethod(magic)] - #[pymethod(name = "__rmul__")] - fn mul(&self, n: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - self.repeat(n.into(), vm) - } - - #[pymethod(magic)] - fn imul(zelf: PyRef<Self>, n: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - Self::irepeat(zelf, n.into(), vm) - } - - #[pymethod] - fn count(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - self.mut_count(vm, &needle) - } - - #[pymethod(magic)] - pub(crate) fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self.mut_contains(vm, &needle) - } - - #[pymethod] - fn index( - &self, - needle: PyObjectRef, - range: OptionalRangeArgs, - vm: &VirtualMachine, - ) -> PyResult<usize> { - let (start, stop) = range.saturate(self.len(), vm)?; - let index = self.mut_index_range(vm, &needle, start..stop)?; - if let Some(index) = index.into() { - Ok(index) - } else { - Err(vm.new_value_error(format!("'{}' is not in list", needle.str(vm)?))) - } - } - - #[pymethod] - fn pop(&self, i: OptionalArg<isize>, vm: &VirtualMachine) -> PyResult { - let mut i = i.into_option().unwrap_or(-1); - let mut elements = self.borrow_vec_mut(); - if i < 0 { - i += elements.len() as isize; - } - if elements.is_empty() { - Err(vm.new_index_error("pop from empty list".to_owned())) - } else if i < 0 || i as usize >= elements.len() { - Err(vm.new_index_error("pop index out of range".to_owned())) - } else { - Ok(elements.remove(i as usize)) - } - } - - #[pymethod] - fn remove(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let index = self.mut_index(vm, &needle)?; - - if let Some(index) = index.into() { - // defer delete out of borrow - let is_inside_range = index < self.borrow_vec().len(); - Ok(is_inside_range.then(|| self.borrow_vec_mut().remove(index))) - } else { - Err(vm.new_value_error(format!("'{}' is not in list", needle.str(vm)?))) - } - .map(drop) - } - - fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - match SequenceIndex::try_from_borrowed_object(vm, needle, "list")? { - SequenceIndex::Int(i) => self.borrow_vec_mut().delitem_by_index(vm, i), - SequenceIndex::Slice(slice) => self.borrow_vec_mut().delitem_by_slice(vm, slice), - } - } - - #[pymethod(magic)] - fn delitem(&self, subscript: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self._delitem(&subscript, vm) - } - - #[pymethod] - pub(crate) fn sort(&self, options: SortOptions, vm: &VirtualMachine) -> PyResult<()> { - // replace list contents with [] for duration of sort. - // this prevents keyfunc from messing with the list and makes it easy to - // check if it tries to append elements to it. - let mut elements = std::mem::take(self.borrow_vec_mut().deref_mut()); - let res = do_sort(vm, &mut elements, options.key, options.reverse); - std::mem::swap(self.borrow_vec_mut().deref_mut(), &mut elements); - res?; - - if !elements.is_empty() { - return Err(vm.new_value_error("list modified during sort".to_owned())); - } - - Ok(()) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } -} - -fn extract_cloned<F, R>(obj: &PyObject, mut f: F, vm: &VirtualMachine) -> PyResult<Vec<R>> -where - F: FnMut(PyObjectRef) -> PyResult<R>, -{ - use crate::builtins::PyTuple; - if let Some(tuple) = obj.payload_if_exact::<PyTuple>(vm) { - tuple.iter().map(|x| f(x.clone())).collect() - } else if let Some(list) = obj.payload_if_exact::<PyList>(vm) { - list.borrow_vec().iter().map(|x| f(x.clone())).collect() - } else { - let iter = obj.to_owned().get_iter(vm)?; - let iter = iter.iter::<PyObjectRef>(vm)?; - let len = obj.to_sequence(vm).length_opt(vm).transpose()?.unwrap_or(0); - let mut v = Vec::with_capacity(len); - for x in iter { - v.push(f(x?)?); - } - v.shrink_to_fit(); - Ok(v) - } -} - -impl MutObjectSequenceOp for PyList { - type Guard<'a> = PyMappedRwLockReadGuard<'a, [PyObjectRef]>; - - fn do_get<'a>(index: usize, guard: &'a Self::Guard<'_>) -> Option<&'a PyObjectRef> { - guard.get(index) - } - - fn do_lock(&self) -> Self::Guard<'_> { - self.borrow_vec() - } -} - -impl Constructor for PyList { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyList::default() - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -impl Initializer for PyList { - type Args = OptionalArg<PyObjectRef>; - - fn init(zelf: PyRef<Self>, iterable: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - let mut elements = if let OptionalArg::Present(iterable) = iterable { - iterable.try_to_value(vm)? - } else { - vec![] - }; - std::mem::swap(zelf.borrow_vec_mut().deref_mut(), &mut elements); - Ok(()) - } -} - -impl AsMapping for PyList { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: PyMappingMethods = PyMappingMethods { - length: atomic_func!(|mapping, _vm| Ok(PyList::mapping_downcast(mapping).len())), - subscript: atomic_func!( - |mapping, needle, vm| PyList::mapping_downcast(mapping)._getitem(needle, vm) - ), - ass_subscript: atomic_func!(|mapping, needle, value, vm| { - let zelf = PyList::mapping_downcast(mapping); - if let Some(value) = value { - zelf._setitem(needle, value, vm) - } else { - zelf._delitem(needle, vm) - } - }), - }; - &AS_MAPPING - } -} - -impl AsSequence for PyList { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyList::sequence_downcast(seq).len())), - concat: atomic_func!(|seq, other, vm| { - PyList::sequence_downcast(seq) - .concat(other, vm) - .map(|x| x.into()) - }), - repeat: atomic_func!(|seq, n, vm| { - PyList::sequence_downcast(seq) - .repeat(n, vm) - .map(|x| x.into()) - }), - item: atomic_func!(|seq, i, vm| { - PyList::sequence_downcast(seq) - .borrow_vec() - .getitem_by_index(vm, i) - }), - ass_item: atomic_func!(|seq, i, value, vm| { - let zelf = PyList::sequence_downcast(seq); - if let Some(value) = value { - zelf.borrow_vec_mut().setitem_by_index(vm, i, value) - } else { - zelf.borrow_vec_mut().delitem_by_index(vm, i) - } - }), - contains: atomic_func!(|seq, target, vm| { - let zelf = PyList::sequence_downcast(seq); - zelf.mut_contains(vm, target) - }), - inplace_concat: atomic_func!(|seq, other, vm| { - let zelf = PyList::sequence_downcast(seq); - PyList::inplace_concat(zelf, other, vm) - }), - inplace_repeat: atomic_func!(|seq, n, vm| { - let zelf = PyList::sequence_downcast(seq); - Ok(PyList::irepeat(zelf.to_owned(), n, vm)?.into()) - }), - }; - &AS_SEQUENCE - } -} - -impl Iterable for PyList { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok(PyListIterator { - internal: PyMutex::new(PositionIterInternal::new(zelf, 0)), - } - .into_pyobject(vm)) - } -} - -impl Comparable for PyList { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - if let Some(res) = op.identical_optimization(zelf, other) { - return Ok(res.into()); - } - let other = class_or_notimplemented!(Self, other); - let a = &*zelf.borrow_vec(); - let b = &*other.borrow_vec(); - a.iter() - .richcompare(b.iter(), op, vm) - .map(PyComparisonValue::Implemented) - } -} - -impl Representable for PyList { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let s = if zelf.len() == 0 { - "[]".to_owned() - } else if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - collection_repr(None, "[", "]", zelf.borrow_vec().iter(), vm)? - } else { - "[...]".to_owned() - }; - Ok(s) - } -} - -fn do_sort( - vm: &VirtualMachine, - values: &mut Vec<PyObjectRef>, - key_func: Option<PyObjectRef>, - reverse: bool, -) -> PyResult<()> { - let op = if reverse { - PyComparisonOp::Lt - } else { - PyComparisonOp::Gt - }; - let cmp = |a: &PyObjectRef, b: &PyObjectRef| a.rich_compare_bool(b, op, vm); - - if let Some(ref key_func) = key_func { - let mut items = values - .iter() - .map(|x| Ok((x.clone(), key_func.call((x.clone(),), vm)?))) - .collect::<Result<Vec<_>, _>>()?; - timsort::try_sort_by_gt(&mut items, |a, b| cmp(&a.1, &b.1))?; - *values = items.into_iter().map(|(val, _)| val).collect(); - } else { - timsort::try_sort_by_gt(values, cmp)?; - } - - Ok(()) -} - -#[pyclass(module = false, name = "list_iterator", traverse)] -#[derive(Debug)] -pub struct PyListIterator { - internal: PyMutex<PositionIterInternal<PyListRef>>, -} - -impl PyPayload for PyListIterator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.list_iterator_type - } -} - -#[pyclass(with(Constructor, IterNext, Iterable))] -impl PyListIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().length_hint(|obj| obj.len()) - } - - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.internal - .lock() - .set_state(state, |obj, pos| pos.min(obj.len()), vm) - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) - } -} -impl Unconstructible for PyListIterator {} - -impl SelfIter for PyListIterator {} -impl IterNext for PyListIterator { - fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.internal.lock().next(|list, pos| { - let vec = list.borrow_vec(); - Ok(PyIterReturn::from_result(vec.get(pos).cloned().ok_or(None))) - }) - } -} - -#[pyclass(module = false, name = "list_reverseiterator", traverse)] -#[derive(Debug)] -pub struct PyListReverseIterator { - internal: PyMutex<PositionIterInternal<PyListRef>>, -} - -impl PyPayload for PyListReverseIterator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.list_reverseiterator_type - } -} - -#[pyclass(with(Constructor, IterNext, Iterable))] -impl PyListReverseIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().rev_length_hint(|obj| obj.len()) - } - - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.internal - .lock() - .set_state(state, |obj, pos| pos.min(obj.len()), vm) - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_reversed_reduce(|x| x.clone().into(), vm) - } -} -impl Unconstructible for PyListReverseIterator {} - -impl SelfIter for PyListReverseIterator {} -impl IterNext for PyListReverseIterator { - fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.internal.lock().rev_next(|list, pos| { - let vec = list.borrow_vec(); - Ok(PyIterReturn::from_result(vec.get(pos).cloned().ok_or(None))) - }) - } -} - -pub fn init(context: &Context) { - let list_type = &context.types.list_type; - PyList::extend_class(context, list_type); - - PyListIterator::extend_class(context, context.types.list_iterator_type); - PyListReverseIterator::extend_class(context, context.types.list_reverseiterator_type); -} diff --git a/vm/src/builtins/map.rs b/vm/src/builtins/map.rs deleted file mode 100644 index 44bacf587ed..00000000000 --- a/vm/src/builtins/map.rs +++ /dev/null @@ -1,73 +0,0 @@ -use super::{PyType, PyTypeRef}; -use crate::{ - builtins::PyTupleRef, - class::PyClassImpl, - function::PosArgs, - protocol::{PyIter, PyIterReturn}, - types::{Constructor, IterNext, Iterable, SelfIter}, - Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "map", traverse)] -#[derive(Debug)] -pub struct PyMap { - mapper: PyObjectRef, - iterators: Vec<PyIter>, -} - -impl PyPayload for PyMap { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.map_type - } -} - -impl Constructor for PyMap { - type Args = (PyObjectRef, PosArgs<PyIter>); - - fn py_new(cls: PyTypeRef, (mapper, iterators): Self::Args, vm: &VirtualMachine) -> PyResult { - let iterators = iterators.into_vec(); - PyMap { mapper, iterators } - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -#[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] -impl PyMap { - #[pymethod(magic)] - fn length_hint(&self, vm: &VirtualMachine) -> PyResult<usize> { - self.iterators.iter().try_fold(0, |prev, cur| { - let cur = cur.as_ref().to_owned().length_hint(0, vm)?; - let max = std::cmp::max(prev, cur); - Ok(max) - }) - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef) { - let mut vec = vec![self.mapper.clone()]; - vec.extend(self.iterators.iter().map(|o| o.clone().into())); - (vm.ctx.types.map_type.to_owned(), vm.new_tuple(vec)) - } -} - -impl SelfIter for PyMap {} -impl IterNext for PyMap { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let mut next_objs = Vec::new(); - for iterator in &zelf.iterators { - let item = match iterator.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - next_objs.push(item); - } - - // the mapper itself can raise StopIteration which does stop the map iteration - PyIterReturn::from_pyresult(zelf.mapper.call(next_objs, vm), vm) - } -} - -pub fn init(context: &Context) { - PyMap::extend_class(context, context.types.map_type); -} diff --git a/vm/src/builtins/mappingproxy.rs b/vm/src/builtins/mappingproxy.rs deleted file mode 100644 index a5400ee6de5..00000000000 --- a/vm/src/builtins/mappingproxy.rs +++ /dev/null @@ -1,289 +0,0 @@ -use super::{PyDict, PyDictRef, PyGenericAlias, PyList, PyTuple, PyType, PyTypeRef}; -use crate::{ - atomic_func, - class::PyClassImpl, - convert::ToPyObject, - function::{ArgMapping, OptionalArg, PyComparisonValue}, - object::{Traverse, TraverseFn}, - protocol::{PyMapping, PyMappingMethods, PyNumberMethods, PySequenceMethods}, - types::{ - AsMapping, AsNumber, AsSequence, Comparable, Constructor, Iterable, PyComparisonOp, - Representable, - }, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use once_cell::sync::Lazy; - -#[pyclass(module = false, name = "mappingproxy", traverse)] -#[derive(Debug)] -pub struct PyMappingProxy { - mapping: MappingProxyInner, -} - -#[derive(Debug)] -enum MappingProxyInner { - Class(PyTypeRef), - Mapping(ArgMapping), -} - -unsafe impl Traverse for MappingProxyInner { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - match self { - MappingProxyInner::Class(ref r) => r.traverse(tracer_fn), - MappingProxyInner::Mapping(ref arg) => arg.traverse(tracer_fn), - } - } -} - -impl PyPayload for PyMappingProxy { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.mappingproxy_type - } -} - -impl From<PyTypeRef> for PyMappingProxy { - fn from(dict: PyTypeRef) -> Self { - Self { - mapping: MappingProxyInner::Class(dict), - } - } -} - -impl From<PyDictRef> for PyMappingProxy { - fn from(dict: PyDictRef) -> Self { - Self { - mapping: MappingProxyInner::Mapping(ArgMapping::from_dict_exact(dict)), - } - } -} - -impl Constructor for PyMappingProxy { - type Args = PyObjectRef; - - fn py_new(cls: PyTypeRef, mapping: Self::Args, vm: &VirtualMachine) -> PyResult { - if let Some(methods) = PyMapping::find_methods(&mapping) { - if mapping.payload_if_subclass::<PyList>(vm).is_none() - && mapping.payload_if_subclass::<PyTuple>(vm).is_none() - { - return Self { - mapping: MappingProxyInner::Mapping(ArgMapping::with_methods( - mapping, - unsafe { methods.borrow_static() }, - )), - } - .into_ref_with_type(vm, cls) - .map(Into::into); - } - } - Err(vm.new_type_error(format!( - "mappingproxy() argument must be a mapping, not {}", - mapping.class() - ))) - } -} - -#[pyclass(with( - AsMapping, - Iterable, - Constructor, - AsSequence, - Comparable, - AsNumber, - Representable -))] -impl PyMappingProxy { - fn get_inner(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { - let opt = match &self.mapping { - MappingProxyInner::Class(class) => key - .as_interned_str(vm) - .and_then(|key| class.attributes.read().get(key).cloned()), - MappingProxyInner::Mapping(mapping) => mapping.mapping().subscript(&*key, vm).ok(), - }; - Ok(opt) - } - - #[pymethod] - fn get( - &self, - key: PyObjectRef, - default: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - let obj = self.to_object(vm)?; - Ok(Some(vm.call_method( - &obj, - "get", - (key, default.unwrap_or_none(vm)), - )?)) - } - - #[pymethod(magic)] - pub fn getitem(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.get_inner(key.clone(), vm)? - .ok_or_else(|| vm.new_key_error(key)) - } - - fn _contains(&self, key: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - match &self.mapping { - MappingProxyInner::Class(class) => Ok(key - .as_interned_str(vm) - .map_or(false, |key| class.attributes.read().contains_key(key))), - MappingProxyInner::Mapping(mapping) => mapping.to_sequence(vm).contains(key, vm), - } - } - - #[pymethod(magic)] - pub fn contains(&self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self._contains(&key, vm) - } - - fn to_object(&self, vm: &VirtualMachine) -> PyResult { - Ok(match &self.mapping { - MappingProxyInner::Mapping(d) => d.as_ref().to_owned(), - MappingProxyInner::Class(c) => { - PyDict::from_attributes(c.attributes.read().clone(), vm)?.to_pyobject(vm) - } - }) - } - - #[pymethod] - pub fn items(&self, vm: &VirtualMachine) -> PyResult { - let obj = self.to_object(vm)?; - vm.call_method(&obj, identifier!(vm, items).as_str(), ()) - } - #[pymethod] - pub fn keys(&self, vm: &VirtualMachine) -> PyResult { - let obj = self.to_object(vm)?; - vm.call_method(&obj, identifier!(vm, keys).as_str(), ()) - } - #[pymethod] - pub fn values(&self, vm: &VirtualMachine) -> PyResult { - let obj = self.to_object(vm)?; - vm.call_method(&obj, identifier!(vm, values).as_str(), ()) - } - #[pymethod] - pub fn copy(&self, vm: &VirtualMachine) -> PyResult { - match &self.mapping { - MappingProxyInner::Mapping(d) => vm.call_method(d, identifier!(vm, copy).as_str(), ()), - MappingProxyInner::Class(c) => { - Ok(PyDict::from_attributes(c.attributes.read().clone(), vm)?.to_pyobject(vm)) - } - } - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } - - #[pymethod(magic)] - fn len(&self, vm: &VirtualMachine) -> PyResult<usize> { - let obj = self.to_object(vm)?; - obj.length(vm) - } - - #[pymethod(magic)] - fn reversed(&self, vm: &VirtualMachine) -> PyResult { - vm.call_method( - self.to_object(vm)?.as_object(), - identifier!(vm, __reversed__).as_str(), - (), - ) - } - - #[pymethod(magic)] - fn ior(&self, _args: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!( - "\"'|=' is not supported by {}; use '|' instead\"", - Self::class(&vm.ctx) - ))) - } - - #[pymethod(name = "__ror__")] - #[pymethod(magic)] - fn or(&self, args: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._or(self.copy(vm)?.as_ref(), args.as_ref()) - } -} - -impl Comparable for PyMappingProxy { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - let obj = zelf.to_object(vm)?; - Ok(PyComparisonValue::Implemented( - obj.rich_compare_bool(other, op, vm)?, - )) - } -} - -impl AsMapping for PyMappingProxy { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy<PyMappingMethods> = Lazy::new(|| PyMappingMethods { - length: atomic_func!(|mapping, vm| PyMappingProxy::mapping_downcast(mapping).len(vm)), - subscript: atomic_func!(|mapping, needle, vm| { - PyMappingProxy::mapping_downcast(mapping).getitem(needle.to_owned(), vm) - }), - ..PyMappingMethods::NOT_IMPLEMENTED - }); - &AS_MAPPING - } -} - -impl AsSequence for PyMappingProxy { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - contains: atomic_func!( - |seq, target, vm| PyMappingProxy::sequence_downcast(seq)._contains(target, vm) - ), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -impl AsNumber for PyMappingProxy { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyMappingProxy>() { - a.or(b.to_pyobject(vm), vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - inplace_or: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyMappingProxy>() { - a.ior(b.to_pyobject(vm), vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl Iterable for PyMappingProxy { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - let obj = zelf.to_object(vm)?; - let iter = obj.get_iter(vm)?; - Ok(iter.into()) - } -} - -impl Representable for PyMappingProxy { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let obj = zelf.to_object(vm)?; - Ok(format!("mappingproxy({})", obj.repr(vm)?)) - } -} - -pub fn init(context: &Context) { - PyMappingProxy::extend_class(context, context.types.mappingproxy_type) -} diff --git a/vm/src/builtins/mod.rs b/vm/src/builtins/mod.rs deleted file mode 100644 index ae3b7eea2a7..00000000000 --- a/vm/src/builtins/mod.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! This package contains the python basic/builtin types -//! 7 common PyRef type aliases are exposed - PyBytesRef, PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef, PyTupleRef -//! Do not add more PyRef type aliases. They will be rare enough to use directly PyRef<T>. - -pub(crate) mod asyncgenerator; -pub use asyncgenerator::PyAsyncGen; -pub(crate) mod builtin_func; -pub(crate) mod bytearray; -pub use bytearray::PyByteArray; -pub(crate) mod bytes; -pub use bytes::{PyBytes, PyBytesRef}; -pub(crate) mod classmethod; -pub use classmethod::PyClassMethod; -pub(crate) mod code; -pub use code::PyCode; -pub(crate) mod complex; -pub use complex::PyComplex; -pub(crate) mod coroutine; -pub use coroutine::PyCoroutine; -pub(crate) mod dict; -pub use dict::{PyDict, PyDictRef}; -pub(crate) mod enumerate; -pub use enumerate::PyEnumerate; -pub(crate) mod filter; -pub use filter::PyFilter; -pub(crate) mod float; -pub use float::PyFloat; -pub(crate) mod frame; -pub(crate) mod function; -pub use function::{PyBoundMethod, PyFunction}; -pub(crate) mod generator; -pub use generator::PyGenerator; -pub(crate) mod genericalias; -pub use genericalias::PyGenericAlias; -pub(crate) mod getset; -pub use getset::PyGetSet; -pub(crate) mod int; -pub use int::{PyInt, PyIntRef}; -pub(crate) mod iter; -pub use iter::*; -pub(crate) mod list; -pub use list::{PyList, PyListRef}; -pub(crate) mod map; -pub use map::PyMap; -pub(crate) mod mappingproxy; -pub use mappingproxy::PyMappingProxy; -pub(crate) mod memory; -pub use memory::PyMemoryView; -pub(crate) mod module; -pub use module::{PyModule, PyModuleDef}; -pub(crate) mod namespace; -pub use namespace::PyNamespace; -pub(crate) mod object; -pub use object::PyBaseObject; -pub(crate) mod property; -pub use property::PyProperty; -#[path = "bool.rs"] -pub(crate) mod bool_; -pub use bool_::PyBool; -#[path = "str.rs"] -pub(crate) mod pystr; -pub use pystr::{PyStr, PyStrInterned, PyStrRef}; -#[path = "super.rs"] -pub(crate) mod super_; -pub use super_::PySuper; -#[path = "type.rs"] -pub(crate) mod type_; -pub use type_::{PyType, PyTypeRef}; -pub(crate) mod range; -pub use range::PyRange; -pub(crate) mod set; -pub use set::{PyFrozenSet, PySet}; -pub(crate) mod singletons; -pub use singletons::{PyNone, PyNotImplemented}; -pub(crate) mod slice; -pub use slice::{PyEllipsis, PySlice}; -pub(crate) mod staticmethod; -pub use staticmethod::PyStaticMethod; -pub(crate) mod traceback; -pub use traceback::PyTraceback; -pub(crate) mod tuple; -pub use tuple::{PyTuple, PyTupleRef}; -pub(crate) mod weakproxy; -pub use weakproxy::PyWeakProxy; -pub(crate) mod weakref; -pub use weakref::PyWeak; -pub(crate) mod zip; -pub use zip::PyZip; -#[path = "union.rs"] -pub(crate) mod union_; -pub use union_::PyUnion; -pub(crate) mod descriptor; - -pub use float::try_to_bigint as try_f64_to_bigint; -pub use int::try_to_float as try_bigint_to_f64; - -pub use crate::exceptions::types::*; diff --git a/vm/src/builtins/module.rs b/vm/src/builtins/module.rs deleted file mode 100644 index 58174082d5b..00000000000 --- a/vm/src/builtins/module.rs +++ /dev/null @@ -1,220 +0,0 @@ -use super::{PyDictRef, PyStr, PyStrRef, PyType, PyTypeRef}; -use crate::{ - builtins::{pystr::AsPyStr, PyStrInterned}, - class::PyClassImpl, - convert::ToPyObject, - function::{FuncArgs, PyMethodDef}, - types::{GetAttr, Initializer, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "module")] -#[derive(Debug)] -pub struct PyModuleDef { - // pub index: usize, - pub name: &'static PyStrInterned, - pub doc: Option<&'static PyStrInterned>, - // pub size: isize, - pub methods: &'static [PyMethodDef], - pub slots: PyModuleSlots, - // traverse: traverseproc - // clear: inquiry - // free: freefunc -} - -pub type ModuleCreate = - fn(&VirtualMachine, &PyObject, &'static PyModuleDef) -> PyResult<PyRef<PyModule>>; -pub type ModuleExec = fn(&VirtualMachine, &Py<PyModule>) -> PyResult<()>; - -#[derive(Default)] -pub struct PyModuleSlots { - pub create: Option<ModuleCreate>, - pub exec: Option<ModuleExec>, -} - -impl std::fmt::Debug for PyModuleSlots { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PyModuleSlots") - .field("create", &self.create.is_some()) - .field("exec", &self.exec.is_some()) - .finish() - } -} - -#[allow(clippy::new_without_default)] // avoid Default implementation -#[pyclass(module = false, name = "module")] -#[derive(Debug)] -pub struct PyModule { - // PyObject *md_dict; - pub def: Option<&'static PyModuleDef>, - // state: Any - // weaklist - // for logging purposes after md_dict is cleared - pub name: Option<&'static PyStrInterned>, -} - -impl PyPayload for PyModule { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.module_type - } -} - -#[derive(FromArgs)] -pub struct ModuleInitArgs { - name: PyStrRef, - #[pyarg(any, default)] - doc: Option<PyStrRef>, -} - -impl PyModule { - #[allow(clippy::new_without_default)] - pub fn new() -> Self { - Self { - def: None, - name: None, - } - } - pub fn from_def(def: &'static PyModuleDef) -> Self { - Self { - def: Some(def), - name: Some(def.name), - } - } - pub fn __init_dict_from_def(vm: &VirtualMachine, module: &Py<PyModule>) { - let doc = module.def.unwrap().doc.map(|doc| doc.to_owned()); - module.init_dict(module.name.unwrap(), doc, vm); - } -} - -impl Py<PyModule> { - pub fn __init_methods(&self, vm: &VirtualMachine) -> PyResult<()> { - debug_assert!(self.def.is_some()); - for method in self.def.unwrap().methods { - let func = method - .to_function() - .with_module(self.name.unwrap()) - .into_ref(&vm.ctx); - vm.__module_set_attr(self, vm.ctx.intern_str(method.name), func)?; - } - Ok(()) - } - - fn getattr_inner(&self, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - if let Some(attr) = self.as_object().generic_getattr_opt(name, None, vm)? { - return Ok(attr); - } - if let Ok(getattr) = self.dict().get_item(identifier!(vm, __getattr__), vm) { - return getattr.call((name.to_owned(),), vm); - } - let module_name = if let Some(name) = self.name(vm) { - format!(" '{name}'") - } else { - "".to_owned() - }; - Err(vm.new_attribute_error(format!("module{module_name} has no attribute '{name}'"))) - } - - fn name(&self, vm: &VirtualMachine) -> Option<PyStrRef> { - let name = self - .as_object() - .generic_getattr_opt(identifier!(vm, __name__), None, vm) - .unwrap_or_default()?; - name.downcast::<PyStr>().ok() - } - - // TODO: to be replaced by the commented-out dict method above once dictoffsets land - pub fn dict(&self) -> PyDictRef { - self.as_object().dict().unwrap() - } - // TODO: should be on PyModule, not Py<PyModule> - pub(crate) fn init_dict( - &self, - name: &'static PyStrInterned, - doc: Option<PyStrRef>, - vm: &VirtualMachine, - ) { - let dict = self.dict(); - dict.set_item(identifier!(vm, __name__), name.to_object(), vm) - .expect("Failed to set __name__ on module"); - dict.set_item(identifier!(vm, __doc__), doc.to_pyobject(vm), vm) - .expect("Failed to set __doc__ on module"); - dict.set_item("__package__", vm.ctx.none(), vm) - .expect("Failed to set __package__ on module"); - dict.set_item("__loader__", vm.ctx.none(), vm) - .expect("Failed to set __loader__ on module"); - dict.set_item("__spec__", vm.ctx.none(), vm) - .expect("Failed to set __spec__ on module"); - } - - pub fn get_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult { - let attr_name = attr_name.as_pystr(&vm.ctx); - self.getattr_inner(attr_name, vm) - } - - pub fn set_attr<'a>( - &self, - attr_name: impl AsPyStr<'a>, - attr_value: impl Into<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - self.as_object().set_attr(attr_name, attr_value, vm) - } -} - -#[pyclass(with(GetAttr, Initializer, Representable), flags(BASETYPE, HAS_DICT))] -impl PyModule { - #[pyslot] - fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyModule::new().into_ref_with_type(vm, cls).map(Into::into) - } - - #[pymethod(magic)] - fn dir(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let dict = zelf - .as_object() - .dict() - .ok_or_else(|| vm.new_value_error("module has no dict".to_owned()))?; - let attrs = dict.into_iter().map(|(k, _v)| k).collect(); - Ok(attrs) - } -} - -impl Initializer for PyModule { - type Args = ModuleInitArgs; - - fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - debug_assert!(zelf - .class() - .slots - .flags - .has_feature(crate::types::PyTypeFlags::HAS_DICT)); - zelf.init_dict(vm.ctx.intern_str(args.name.as_str()), args.doc, vm); - Ok(()) - } -} - -impl GetAttr for PyModule { - fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - zelf.getattr_inner(name, vm) - } -} - -impl Representable for PyModule { - #[inline] - fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let importlib = vm.import("_frozen_importlib", None, 0)?; - let module_repr = importlib.get_attr("_module_repr", vm)?; - let repr = module_repr.call((zelf.to_owned(),), vm)?; - repr.downcast() - .map_err(|_| vm.new_type_error("_module_repr did not return a string".into())) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } -} - -pub(crate) fn init(context: &Context) { - PyModule::extend_class(context, context.types.module_type); -} diff --git a/vm/src/builtins/namespace.rs b/vm/src/builtins/namespace.rs deleted file mode 100644 index 4a08dddc9ad..00000000000 --- a/vm/src/builtins/namespace.rs +++ /dev/null @@ -1,120 +0,0 @@ -use super::{tuple::IntoPyTuple, PyTupleRef, PyType, PyTypeRef}; -use crate::{ - builtins::PyDict, - class::PyClassImpl, - function::{FuncArgs, PyComparisonValue}, - recursion::ReprGuard, - types::{Comparable, Constructor, Initializer, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -/// A simple attribute-based namespace. -/// -/// SimpleNamespace(**kwargs) -#[pyclass(module = "types", name = "SimpleNamespace")] -#[derive(Debug)] -pub struct PyNamespace {} - -impl PyPayload for PyNamespace { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.namespace_type - } -} - -impl Constructor for PyNamespace { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - PyNamespace {}.into_ref_with_type(vm, cls).map(Into::into) - } -} - -impl PyNamespace { - pub fn new_ref(ctx: &Context) -> PyRef<Self> { - PyRef::new_ref( - Self {}, - ctx.types.namespace_type.to_owned(), - Some(ctx.new_dict()), - ) - } -} - -#[pyclass( - flags(BASETYPE, HAS_DICT), - with(Constructor, Initializer, Comparable, Representable) -)] -impl PyNamespace { - #[pymethod(magic)] - fn reduce(zelf: PyObjectRef, vm: &VirtualMachine) -> PyTupleRef { - let dict = zelf.as_object().dict().unwrap(); - let obj = zelf.as_object().to_owned(); - let result: (PyObjectRef, PyObjectRef, PyObjectRef) = ( - obj.class().to_owned().into(), - vm.new_tuple(()).into(), - dict.into(), - ); - result.into_pytuple(vm) - } -} - -impl Initializer for PyNamespace { - type Args = FuncArgs; - - fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - if !args.args.is_empty() { - return Err(vm.new_type_error("no positional arguments expected".to_owned())); - } - for (name, value) in args.kwargs.into_iter() { - let name = vm.ctx.new_str(name); - zelf.as_object().set_attr(&name, value, vm)?; - } - Ok(()) - } -} - -impl Comparable for PyNamespace { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - let other = class_or_notimplemented!(Self, other); - let (d1, d2) = ( - zelf.as_object().dict().unwrap(), - other.as_object().dict().unwrap(), - ); - PyDict::cmp(&d1, d2.as_object(), op, vm) - } -} - -impl Representable for PyNamespace { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let o = zelf.as_object(); - let name = if o.class().is(vm.ctx.types.namespace_type) { - "namespace".to_owned() - } else { - o.class().slot_name().to_owned() - }; - - let repr = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let dict = zelf.as_object().dict().unwrap(); - let mut parts = Vec::with_capacity(dict.len()); - for (key, value) in dict { - let k = &key.repr(vm)?; - let key_str = k.as_str(); - let value_repr = value.repr(vm)?; - parts.push(format!("{}={}", &key_str[1..key_str.len() - 1], value_repr)); - } - format!("{}({})", name, parts.join(", ")) - } else { - format!("{name}(...)") - }; - Ok(repr) - } -} - -pub fn init(context: &Context) { - PyNamespace::extend_class(context, context.types.namespace_type); -} diff --git a/vm/src/builtins/object.rs b/vm/src/builtins/object.rs deleted file mode 100644 index efe6aa980f1..00000000000 --- a/vm/src/builtins/object.rs +++ /dev/null @@ -1,377 +0,0 @@ -use super::{PyDictRef, PyList, PyStr, PyStrRef, PyType, PyTypeRef}; -use crate::common::hash::PyHash; -use crate::{ - class::PyClassImpl, - function::{Either, FuncArgs, PyArithmeticValue, PyComparisonValue, PySetterValue}, - types::{Constructor, PyComparisonOp}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; -use itertools::Itertools; - -/// object() -/// -- -/// -/// The base class of the class hierarchy. -/// -/// When called, it accepts no arguments and returns a new featureless -/// instance that has no instance attributes and cannot be given any. -#[pyclass(module = false, name = "object")] -#[derive(Debug)] -pub struct PyBaseObject; - -impl PyPayload for PyBaseObject { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.object_type - } -} - -impl Constructor for PyBaseObject { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - // more or less __new__ operator - let dict = if cls.is(vm.ctx.types.object_type) { - None - } else { - Some(vm.ctx.new_dict()) - }; - - // Ensure that all abstract methods are implemented before instantiating instance. - if let Some(abs_methods) = cls.get_attr(identifier!(vm, __abstractmethods__)) { - if let Some(unimplemented_abstract_method_count) = abs_methods.length_opt(vm) { - let methods: Vec<PyStrRef> = abs_methods.try_to_value(vm)?; - let methods: String = - Itertools::intersperse(methods.iter().map(|name| name.as_str()), ", ") - .collect(); - - let unimplemented_abstract_method_count = unimplemented_abstract_method_count?; - let name = cls.name().to_string(); - - match unimplemented_abstract_method_count { - 0 => {} - 1 => { - return Err(vm.new_type_error(format!( - "Can't instantiate abstract class {} with abstract method {}", - name, methods - ))); - } - 2.. => { - return Err(vm.new_type_error(format!( - "Can't instantiate abstract class {} with abstract methods {}", - name, methods - ))); - } - // TODO: remove `allow` when redox build doesn't complain about it - #[allow(unreachable_patterns)] - _ => unreachable!(), - } - } - } - - Ok(crate::PyRef::new_ref(PyBaseObject, cls, dict).into()) - } -} - -#[pyclass(with(Constructor), flags(BASETYPE))] -impl PyBaseObject { - #[pyslot] - fn slot_richcompare( - zelf: &PyObject, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<Either<PyObjectRef, PyComparisonValue>> { - Self::cmp(zelf, other, op, vm).map(Either::B) - } - - #[inline(always)] - fn cmp( - zelf: &PyObject, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - let res = match op { - PyComparisonOp::Eq => { - if zelf.is(other) { - PyComparisonValue::Implemented(true) - } else { - PyComparisonValue::NotImplemented - } - } - PyComparisonOp::Ne => { - let cmp = zelf - .class() - .mro_find_map(|cls| cls.slots.richcompare.load()) - .unwrap(); - let value = match cmp(zelf, other, PyComparisonOp::Eq, vm)? { - Either::A(obj) => PyArithmeticValue::from_object(vm, obj) - .map(|obj| obj.try_to_bool(vm)) - .transpose()?, - Either::B(value) => value, - }; - value.map(|v| !v) - } - _ => PyComparisonValue::NotImplemented, - }; - Ok(res) - } - - /// Return self==value. - #[pymethod(magic)] - fn eq( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Eq, vm) - } - - /// Return self!=value. - #[pymethod(magic)] - fn ne( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Ne, vm) - } - - /// Return self<value. - #[pymethod(magic)] - fn lt( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Lt, vm) - } - - /// Return self<=value. - #[pymethod(magic)] - fn le( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Le, vm) - } - - /// Return self>=value. - #[pymethod(magic)] - fn ge( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Ge, vm) - } - - /// Return self>value. - #[pymethod(magic)] - fn gt( - zelf: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - Self::cmp(&zelf, &value, PyComparisonOp::Gt, vm) - } - - /// Implement setattr(self, name, value). - #[pymethod] - fn __setattr__( - obj: PyObjectRef, - name: PyStrRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - obj.generic_setattr(&name, PySetterValue::Assign(value), vm) - } - - /// Implement delattr(self, name). - #[pymethod] - fn __delattr__(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - obj.generic_setattr(&name, PySetterValue::Delete, vm) - } - - #[pyslot] - fn slot_setattro( - obj: &PyObject, - attr_name: &Py<PyStr>, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()> { - obj.generic_setattr(attr_name, value, vm) - } - - /// Return str(self). - #[pymethod(magic)] - fn str(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - // FIXME: try tp_repr first and fallback to object.__repr__ - zelf.repr(vm) - } - - #[pyslot] - fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let class = zelf.class(); - match ( - class - .qualname(vm) - .downcast_ref::<PyStr>() - .map(|n| n.as_str()), - class.module(vm).downcast_ref::<PyStr>().map(|m| m.as_str()), - ) { - (None, _) => Err(vm.new_type_error("Unknown qualified name".into())), - (Some(qualname), Some(module)) if module != "builtins" => Ok(PyStr::from(format!( - "<{}.{} object at {:#x}>", - module, - qualname, - zelf.get_id() - )) - .into_ref(&vm.ctx)), - _ => Ok(PyStr::from(format!( - "<{} object at {:#x}>", - class.slot_name(), - zelf.get_id() - )) - .into_ref(&vm.ctx)), - } - } - - /// Return repr(self). - #[pymethod(magic)] - fn repr(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Self::slot_repr(&zelf, vm) - } - - #[pyclassmethod(magic)] - fn subclasshook(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() - } - - #[pyclassmethod(magic)] - fn init_subclass(_cls: PyTypeRef) {} - - #[pymethod(magic)] - pub fn dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyList> { - obj.dir(vm) - } - - #[pymethod(magic)] - fn format(obj: PyObjectRef, format_spec: PyStrRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - if !format_spec.is_empty() { - return Err(vm.new_type_error(format!( - "unsupported format string passed to {}.__format__", - obj.class().name() - ))); - } - obj.str(vm) - } - - #[pyslot] - #[pymethod(magic)] - fn init(_zelf: PyObjectRef, _args: FuncArgs, _vm: &VirtualMachine) -> PyResult<()> { - Ok(()) - } - - #[pygetset(name = "__class__")] - fn get_class(obj: PyObjectRef) -> PyTypeRef { - obj.class().to_owned() - } - - #[pygetset(name = "__class__", setter)] - fn set_class(instance: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if instance.payload_is::<PyBaseObject>() { - match value.downcast::<PyType>() { - Ok(cls) => { - // FIXME(#1979) cls instances might have a payload - instance.set_class(cls, vm); - Ok(()) - } - Err(value) => { - let value_class = value.class(); - let type_repr = &value_class.name(); - Err(vm.new_type_error(format!( - "__class__ must be set to a class, not '{type_repr}' object" - ))) - } - } - } else { - Err(vm.new_type_error( - "__class__ assignment only supported for types without a payload".to_owned(), - )) - } - } - - /// Return getattr(self, name). - #[pyslot] - pub(crate) fn getattro(obj: &PyObject, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - vm_trace!("object.__getattribute__({:?}, {:?})", obj, name); - obj.as_object().generic_getattr(name, vm) - } - - #[pymethod(magic)] - fn getattribute(obj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine) -> PyResult { - Self::getattro(&obj, &name, vm) - } - - #[pymethod(magic)] - fn reduce(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - common_reduce(obj, 0, vm) - } - - #[pymethod(magic)] - fn reduce_ex(obj: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { - let __reduce__ = identifier!(vm, __reduce__); - if let Some(reduce) = vm.get_attribute_opt(obj.clone(), __reduce__)? { - let object_reduce = vm.ctx.types.object_type.get_attr(__reduce__).unwrap(); - let typ_obj: PyObjectRef = obj.class().to_owned().into(); - let class_reduce = typ_obj.get_attr(__reduce__, vm)?; - if !class_reduce.is(&object_reduce) { - return reduce.call((), vm); - } - } - common_reduce(obj, proto, vm) - } - - #[pyslot] - fn slot_hash(zelf: &PyObject, _vm: &VirtualMachine) -> PyResult<PyHash> { - Ok(zelf.get_id() as _) - } - - /// Return hash(self). - #[pymethod(magic)] - fn hash(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyHash> { - Self::slot_hash(&zelf, vm) - } - - #[pymethod(magic)] - fn sizeof(zelf: PyObjectRef) -> usize { - zelf.class().slots.basicsize - } -} - -pub fn object_get_dict(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyDictRef> { - obj.dict() - .ok_or_else(|| vm.new_attribute_error("This object has no __dict__".to_owned())) -} -pub fn object_set_dict(obj: PyObjectRef, dict: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { - obj.set_dict(dict) - .map_err(|_| vm.new_attribute_error("This object has no __dict__".to_owned())) -} - -pub fn init(ctx: &Context) { - PyBaseObject::extend_class(ctx, ctx.types.object_type); -} - -fn common_reduce(obj: PyObjectRef, proto: usize, vm: &VirtualMachine) -> PyResult { - if proto >= 2 { - let reducelib = vm.import("__reducelib", None, 0)?; - let reduce_2 = reducelib.get_attr("reduce_2", vm)?; - reduce_2.call((obj,), vm) - } else { - let copyreg = vm.import("copyreg", None, 0)?; - let reduce_ex = copyreg.get_attr("_reduce_ex", vm)?; - reduce_ex.call((obj, proto), vm) - } -} diff --git a/vm/src/builtins/property.rs b/vm/src/builtins/property.rs deleted file mode 100644 index 61e1ff16921..00000000000 --- a/vm/src/builtins/property.rs +++ /dev/null @@ -1,265 +0,0 @@ -/*! Python `property` descriptor class. - -*/ -use super::{PyStrRef, PyType, PyTypeRef}; -use crate::common::lock::PyRwLock; -use crate::function::{IntoFuncArgs, PosArgs}; -use crate::{ - class::PyClassImpl, - function::{FuncArgs, PySetterValue}, - types::{Constructor, GetDescriptor, Initializer}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "property", traverse)] -#[derive(Debug)] -pub struct PyProperty { - getter: PyRwLock<Option<PyObjectRef>>, - setter: PyRwLock<Option<PyObjectRef>>, - deleter: PyRwLock<Option<PyObjectRef>>, - doc: PyRwLock<Option<PyObjectRef>>, - name: PyRwLock<Option<PyObjectRef>>, -} - -impl PyPayload for PyProperty { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.property_type - } -} - -#[derive(FromArgs)] -pub struct PropertyArgs { - #[pyarg(any, default)] - fget: Option<PyObjectRef>, - #[pyarg(any, default)] - fset: Option<PyObjectRef>, - #[pyarg(any, default)] - fdel: Option<PyObjectRef>, - #[pyarg(any, default)] - doc: Option<PyObjectRef>, - #[pyarg(any, default)] - name: Option<PyStrRef>, -} - -impl GetDescriptor for PyProperty { - fn descr_get( - zelf_obj: PyObjectRef, - obj: Option<PyObjectRef>, - _cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let (zelf, obj) = Self::_unwrap(&zelf_obj, obj, vm)?; - if vm.is_none(&obj) { - Ok(zelf_obj) - } else if let Some(getter) = zelf.getter.read().as_ref() { - getter.call((obj,), vm) - } else { - Err(vm.new_attribute_error("property has no getter".to_string())) - } - } -} - -#[pyclass(with(Constructor, Initializer, GetDescriptor), flags(BASETYPE))] -impl PyProperty { - // Descriptor methods - - #[pyslot] - fn descr_set( - zelf: &PyObject, - obj: PyObjectRef, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()> { - let zelf = zelf.try_to_ref::<Self>(vm)?; - match value { - PySetterValue::Assign(value) => { - if let Some(setter) = zelf.setter.read().as_ref() { - setter.call((obj, value), vm).map(drop) - } else { - Err(vm.new_attribute_error("property has no setter".to_owned())) - } - } - PySetterValue::Delete => { - if let Some(deleter) = zelf.deleter.read().as_ref() { - deleter.call((obj,), vm).map(drop) - } else { - Err(vm.new_attribute_error("property has no deleter".to_owned())) - } - } - } - } - #[pymethod] - fn __set__( - zelf: PyObjectRef, - obj: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Assign(value), vm) - } - #[pymethod] - fn __delete__(zelf: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - Self::descr_set(&zelf, obj, PySetterValue::Delete, vm) - } - - // Access functions - - #[pygetset] - fn fget(&self) -> Option<PyObjectRef> { - self.getter.read().clone() - } - - #[pygetset] - fn fset(&self) -> Option<PyObjectRef> { - self.setter.read().clone() - } - - #[pygetset] - fn fdel(&self) -> Option<PyObjectRef> { - self.deleter.read().clone() - } - - fn doc_getter(&self) -> Option<PyObjectRef> { - self.doc.read().clone() - } - fn doc_setter(&self, value: Option<PyObjectRef>) { - *self.doc.write() = value; - } - - #[pymethod(magic)] - fn set_name(&self, args: PosArgs, vm: &VirtualMachine) -> PyResult<()> { - let func_args = args.into_args(vm); - let func_args_len = func_args.args.len(); - let (_owner, name): (PyObjectRef, PyObjectRef) = func_args.bind(vm).map_err(|_e| { - vm.new_type_error(format!( - "__set_name__() takes 2 positional arguments but {} were given", - func_args_len - )) - })?; - - *self.name.write() = Some(name); - - Ok(()) - } - - // Python builder functions - - #[pymethod] - fn getter( - zelf: PyRef<Self>, - getter: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - PyProperty { - getter: PyRwLock::new(getter.or_else(|| zelf.fget())), - setter: PyRwLock::new(zelf.fset()), - deleter: PyRwLock::new(zelf.fdel()), - doc: PyRwLock::new(None), - name: PyRwLock::new(None), - } - .into_ref_with_type(vm, zelf.class().to_owned()) - } - - #[pymethod] - fn setter( - zelf: PyRef<Self>, - setter: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - PyProperty { - getter: PyRwLock::new(zelf.fget()), - setter: PyRwLock::new(setter.or_else(|| zelf.fset())), - deleter: PyRwLock::new(zelf.fdel()), - doc: PyRwLock::new(None), - name: PyRwLock::new(None), - } - .into_ref_with_type(vm, zelf.class().to_owned()) - } - - #[pymethod] - fn deleter( - zelf: PyRef<Self>, - deleter: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - PyProperty { - getter: PyRwLock::new(zelf.fget()), - setter: PyRwLock::new(zelf.fset()), - deleter: PyRwLock::new(deleter.or_else(|| zelf.fdel())), - doc: PyRwLock::new(None), - name: PyRwLock::new(None), - } - .into_ref_with_type(vm, zelf.class().to_owned()) - } - - #[pygetset(magic)] - fn isabstractmethod(&self, vm: &VirtualMachine) -> PyObjectRef { - let getter_abstract = match self.getter.read().to_owned() { - Some(getter) => getter - .get_attr("__isabstractmethod__", vm) - .unwrap_or_else(|_| vm.ctx.new_bool(false).into()), - _ => vm.ctx.new_bool(false).into(), - }; - let setter_abstract = match self.setter.read().to_owned() { - Some(setter) => setter - .get_attr("__isabstractmethod__", vm) - .unwrap_or_else(|_| vm.ctx.new_bool(false).into()), - _ => vm.ctx.new_bool(false).into(), - }; - vm._or(&setter_abstract, &getter_abstract) - .unwrap_or_else(|_| vm.ctx.new_bool(false).into()) - } - - #[pygetset(magic, setter)] - fn set_isabstractmethod(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if let Some(getter) = self.getter.read().to_owned() { - getter.set_attr("__isabstractmethod__", value, vm)?; - } - Ok(()) - } -} - -impl Constructor for PyProperty { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyProperty { - getter: PyRwLock::new(None), - setter: PyRwLock::new(None), - deleter: PyRwLock::new(None), - doc: PyRwLock::new(None), - name: PyRwLock::new(None), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -impl Initializer for PyProperty { - type Args = PropertyArgs; - - fn init(zelf: PyRef<Self>, args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { - *zelf.getter.write() = args.fget; - *zelf.setter.write() = args.fset; - *zelf.deleter.write() = args.fdel; - *zelf.doc.write() = args.doc; - *zelf.name.write() = args.name.map(|a| a.as_object().to_owned()); - - Ok(()) - } -} - -pub(crate) fn init(context: &Context) { - PyProperty::extend_class(context, context.types.property_type); - - // This is a bit unfortunate, but this instance attribute overlaps with the - // class __doc__ string.. - extend_class!(context, context.types.property_type, { - "__doc__" => context.new_getset( - "__doc__", - context.types.property_type, - PyProperty::doc_getter, - PyProperty::doc_setter, - ), - }); -} diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs deleted file mode 100644 index 29ead029157..00000000000 --- a/vm/src/builtins/set.rs +++ /dev/null @@ -1,1290 +0,0 @@ -/* - * Builtin set type with a sequence of unique items. - */ -use super::{ - builtins_iter, IterStatus, PositionIterInternal, PyDict, PyDictRef, PyGenericAlias, PyTupleRef, - PyType, PyTypeRef, -}; -use crate::{ - atomic_func, - class::PyClassImpl, - common::{ascii, hash::PyHash, lock::PyMutex, rc::PyRc}, - convert::ToPyResult, - dictdatatype::{self, DictSize}, - function::{ArgIterable, FuncArgs, OptionalArg, PosArgs, PyArithmeticValue, PyComparisonValue}, - protocol::{PyIterReturn, PyNumberMethods, PySequenceMethods}, - recursion::ReprGuard, - types::AsNumber, - types::{ - AsSequence, Comparable, Constructor, Hashable, Initializer, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, - }, - utils::collection_repr, - vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, -}; -use once_cell::sync::Lazy; -use std::{fmt, ops::Deref}; - -pub type SetContentType = dictdatatype::Dict<()>; - -#[pyclass(module = false, name = "set", unhashable = true, traverse)] -#[derive(Default)] -pub struct PySet { - pub(super) inner: PySetInner, -} - -impl PySet { - pub fn new_ref(ctx: &Context) -> PyRef<Self> { - // Initialized empty, as calling __hash__ is required for adding each object to the set - // which requires a VM context - this is done in the set code itself. - PyRef::new_ref(Self::default(), ctx.types.set_type.to_owned(), None) - } - - pub fn elements(&self) -> Vec<PyObjectRef> { - self.inner.elements() - } - - fn fold_op( - &self, - others: impl std::iter::Iterator<Item = ArgIterable>, - op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, - vm: &VirtualMachine, - ) -> PyResult<Self> { - Ok(Self { - inner: self.inner.fold_op(others, op, vm)?, - }) - } - - fn op( - &self, - other: AnySet, - op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, - vm: &VirtualMachine, - ) -> PyResult<Self> { - Ok(Self { - inner: self - .inner - .fold_op(std::iter::once(other.into_iterable(vm)?), op, vm)?, - }) - } -} - -#[pyclass(module = false, name = "frozenset", unhashable = true)] -#[derive(Default)] -pub struct PyFrozenSet { - inner: PySetInner, -} - -impl PyFrozenSet { - // Also used by ssl.rs windows. - pub fn from_iter( - vm: &VirtualMachine, - it: impl IntoIterator<Item = PyObjectRef>, - ) -> PyResult<Self> { - let inner = PySetInner::default(); - for elem in it { - inner.add(elem, vm)?; - } - // FIXME: empty set check - Ok(Self { inner }) - } - - pub fn elements(&self) -> Vec<PyObjectRef> { - self.inner.elements() - } - - fn fold_op( - &self, - others: impl std::iter::Iterator<Item = ArgIterable>, - op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, - vm: &VirtualMachine, - ) -> PyResult<Self> { - Ok(Self { - inner: self.inner.fold_op(others, op, vm)?, - }) - } - - fn op( - &self, - other: AnySet, - op: fn(&PySetInner, ArgIterable, &VirtualMachine) -> PyResult<PySetInner>, - vm: &VirtualMachine, - ) -> PyResult<Self> { - Ok(Self { - inner: self - .inner - .fold_op(std::iter::once(other.into_iterable(vm)?), op, vm)?, - }) - } -} - -impl fmt::Debug for PySet { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // TODO: implement more detailed, non-recursive Debug formatter - f.write_str("set") - } -} - -impl fmt::Debug for PyFrozenSet { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // TODO: implement more detailed, non-recursive Debug formatter - f.write_str("PyFrozenSet ")?; - f.debug_set().entries(self.elements().iter()).finish() - } -} - -impl PyPayload for PySet { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.set_type - } -} - -impl PyPayload for PyFrozenSet { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.frozenset_type - } -} - -#[derive(Default, Clone)] -pub(super) struct PySetInner { - content: PyRc<SetContentType>, -} - -unsafe impl crate::object::Traverse for PySetInner { - fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn) { - // FIXME(discord9): Rc means shared ref, so should it be traced? - self.content.traverse(tracer_fn) - } -} - -impl PySetInner { - pub(super) fn from_iter<T>(iter: T, vm: &VirtualMachine) -> PyResult<Self> - where - T: IntoIterator<Item = PyResult<PyObjectRef>>, - { - let set = PySetInner::default(); - for item in iter { - set.add(item?, vm)?; - } - Ok(set) - } - - fn fold_op<O>( - &self, - others: impl std::iter::Iterator<Item = O>, - op: fn(&Self, O, &VirtualMachine) -> PyResult<Self>, - vm: &VirtualMachine, - ) -> PyResult<Self> { - let mut res = self.copy(); - for other in others { - res = op(&res, other, vm)?; - } - Ok(res) - } - - fn len(&self) -> usize { - self.content.len() - } - - fn sizeof(&self) -> usize { - self.content.sizeof() - } - - fn copy(&self) -> PySetInner { - PySetInner { - content: PyRc::new((*self.content).clone()), - } - } - - fn contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - self.retry_op_with_frozenset(needle, vm, |needle, vm| self.content.contains(vm, needle)) - } - - fn compare( - &self, - other: &PySetInner, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<bool> { - if op == PyComparisonOp::Ne { - return self.compare(other, PyComparisonOp::Eq, vm).map(|eq| !eq); - } - if !op.eval_ord(self.len().cmp(&other.len())) { - return Ok(false); - } - let (superset, subset) = if matches!(op, PyComparisonOp::Lt | PyComparisonOp::Le) { - (other, self) - } else { - (self, other) - }; - for key in subset.elements() { - if !superset.contains(&key, vm)? { - return Ok(false); - } - } - Ok(true) - } - - pub(super) fn union(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<PySetInner> { - let set = self.clone(); - for item in other.iter(vm)? { - set.add(item?, vm)?; - } - - Ok(set) - } - - pub(super) fn intersection( - &self, - other: ArgIterable, - vm: &VirtualMachine, - ) -> PyResult<PySetInner> { - let set = PySetInner::default(); - for item in other.iter(vm)? { - let obj = item?; - if self.contains(&obj, vm)? { - set.add(obj, vm)?; - } - } - Ok(set) - } - - pub(super) fn difference( - &self, - other: ArgIterable, - vm: &VirtualMachine, - ) -> PyResult<PySetInner> { - let set = self.copy(); - for item in other.iter(vm)? { - set.content.delete_if_exists(vm, &*item?)?; - } - Ok(set) - } - - pub(super) fn symmetric_difference( - &self, - other: ArgIterable, - vm: &VirtualMachine, - ) -> PyResult<PySetInner> { - let new_inner = self.clone(); - - // We want to remove duplicates in other - let other_set = Self::from_iter(other.iter(vm)?, vm)?; - - for item in other_set.elements() { - new_inner.content.delete_or_insert(vm, &item, ())? - } - - Ok(new_inner) - } - - fn issuperset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - for item in other.iter(vm)? { - if !self.contains(&*item?, vm)? { - return Ok(false); - } - } - Ok(true) - } - - fn issubset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - let other_set = PySetInner::from_iter(other.iter(vm)?, vm)?; - self.compare(&other_set, PyComparisonOp::Le, vm) - } - - pub(super) fn isdisjoint(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - for item in other.iter(vm)? { - if self.contains(&*item?, vm)? { - return Ok(false); - } - } - Ok(true) - } - - fn iter(&self) -> PySetIterator { - PySetIterator { - size: self.content.size(), - internal: PyMutex::new(PositionIterInternal::new(self.content.clone(), 0)), - } - } - - fn repr(&self, class_name: Option<&str>, vm: &VirtualMachine) -> PyResult<String> { - collection_repr(class_name, "{", "}", self.elements().iter(), vm) - } - - fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.content.insert(vm, &*item, ()) - } - - fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.retry_op_with_frozenset(&item, vm, |item, vm| self.content.delete(vm, item)) - } - - fn discard(&self, item: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - self.retry_op_with_frozenset(item, vm, |item, vm| self.content.delete_if_exists(vm, item)) - } - - fn clear(&self) { - self.content.clear() - } - - fn elements(&self) -> Vec<PyObjectRef> { - self.content.keys() - } - - fn pop(&self, vm: &VirtualMachine) -> PyResult { - // TODO: should be pop_front, but that requires rearranging every index - if let Some((key, _)) = self.content.pop_back() { - Ok(key) - } else { - let err_msg = vm.ctx.new_str(ascii!("pop from an empty set")).into(); - Err(vm.new_key_error(err_msg)) - } - } - - fn update( - &self, - others: impl std::iter::Iterator<Item = ArgIterable>, - vm: &VirtualMachine, - ) -> PyResult<()> { - for iterable in others { - for item in iterable.iter(vm)? { - self.add(item?, vm)?; - } - } - Ok(()) - } - - fn update_internal(&self, iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // check AnySet - if let Ok(any_set) = AnySet::try_from_object(vm, iterable.to_owned()) { - self.merge_set(any_set, vm) - // check Dict - } else if let Ok(dict) = iterable.to_owned().downcast_exact::<PyDict>(vm) { - self.merge_dict(dict.into_pyref(), vm) - } else { - // add iterable that is not AnySet or Dict - for item in iterable.try_into_value::<ArgIterable>(vm)?.iter(vm)? { - self.add(item?, vm)?; - } - Ok(()) - } - } - - fn merge_set(&self, any_set: AnySet, vm: &VirtualMachine) -> PyResult<()> { - for item in any_set.as_inner().elements() { - self.add(item, vm)?; - } - Ok(()) - } - - fn merge_dict(&self, dict: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { - for (key, _value) in dict { - self.add(key, vm)?; - } - Ok(()) - } - - fn intersection_update( - &self, - others: impl std::iter::Iterator<Item = ArgIterable>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let mut temp_inner = self.copy(); - self.clear(); - for iterable in others { - for item in iterable.iter(vm)? { - let obj = item?; - if temp_inner.contains(&obj, vm)? { - self.add(obj, vm)?; - } - } - temp_inner = self.copy() - } - Ok(()) - } - - fn difference_update( - &self, - others: impl std::iter::Iterator<Item = ArgIterable>, - vm: &VirtualMachine, - ) -> PyResult<()> { - for iterable in others { - for item in iterable.iter(vm)? { - self.content.delete_if_exists(vm, &*item?)?; - } - } - Ok(()) - } - - fn symmetric_difference_update( - &self, - others: impl std::iter::Iterator<Item = ArgIterable>, - vm: &VirtualMachine, - ) -> PyResult<()> { - for iterable in others { - // We want to remove duplicates in iterable - let iterable_set = Self::from_iter(iterable.iter(vm)?, vm)?; - for item in iterable_set.elements() { - self.content.delete_or_insert(vm, &item, ())?; - } - } - Ok(()) - } - - fn hash(&self, vm: &VirtualMachine) -> PyResult<PyHash> { - crate::utils::hash_iter_unordered(self.elements().iter(), vm) - } - - // Run operation, on failure, if item is a set/set subclass, convert it - // into a frozenset and try the operation again. Propagates original error - // on failure to convert and restores item in KeyError on failure (remove). - fn retry_op_with_frozenset<T, F>( - &self, - item: &PyObject, - vm: &VirtualMachine, - op: F, - ) -> PyResult<T> - where - F: Fn(&PyObject, &VirtualMachine) -> PyResult<T>, - { - op(item, vm).or_else(|original_err| { - item.payload_if_subclass::<PySet>(vm) - // Keep original error around. - .ok_or(original_err) - .and_then(|set| { - op( - &PyFrozenSet { - inner: set.inner.copy(), - } - .into_pyobject(vm), - vm, - ) - // If operation raised KeyError, report original set (set.remove) - .map_err(|op_err| { - if op_err.fast_isinstance(vm.ctx.exceptions.key_error) { - vm.new_key_error(item.to_owned()) - } else { - op_err - } - }) - }) - }) - } -} - -fn extract_set(obj: &PyObject) -> Option<&PySetInner> { - match_class!(match obj { - ref set @ PySet => Some(&set.inner), - ref frozen @ PyFrozenSet => Some(&frozen.inner), - _ => None, - }) -} - -fn reduce_set( - zelf: &PyObject, - vm: &VirtualMachine, -) -> PyResult<(PyTypeRef, PyTupleRef, Option<PyDictRef>)> { - Ok(( - zelf.class().to_owned(), - vm.new_tuple((extract_set(zelf) - .unwrap_or(&PySetInner::default()) - .elements(),)), - zelf.dict(), - )) -} - -#[pyclass( - with( - Constructor, - Initializer, - AsSequence, - Comparable, - Iterable, - AsNumber, - Representable - ), - flags(BASETYPE) -)] -impl PySet { - #[pymethod(magic)] - fn len(&self) -> usize { - self.inner.len() - } - - #[pymethod(magic)] - fn sizeof(&self) -> usize { - std::mem::size_of::<Self>() + self.inner.sizeof() - } - - #[pymethod] - fn copy(&self) -> Self { - Self { - inner: self.inner.copy(), - } - } - - #[pymethod(magic)] - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.contains(&needle, vm) - } - - #[pymethod] - fn union(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { - self.fold_op(others.into_iter(), PySetInner::union, vm) - } - - #[pymethod] - fn intersection(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { - self.fold_op(others.into_iter(), PySetInner::intersection, vm) - } - - #[pymethod] - fn difference(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { - self.fold_op(others.into_iter(), PySetInner::difference, vm) - } - - #[pymethod] - fn symmetric_difference( - &self, - others: PosArgs<ArgIterable>, - vm: &VirtualMachine, - ) -> PyResult<Self> { - self.fold_op(others.into_iter(), PySetInner::symmetric_difference, vm) - } - - #[pymethod] - fn issubset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.issubset(other, vm) - } - - #[pymethod] - fn issuperset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.issuperset(other, vm) - } - - #[pymethod] - fn isdisjoint(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.isdisjoint(other, vm) - } - - #[pymethod(name = "__ror__")] - #[pymethod(magic)] - fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(self.op( - other, - PySetInner::union, - vm, - )?)) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(name = "__rand__")] - #[pymethod(magic)] - fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(self.op( - other, - PySetInner::intersection, - vm, - )?)) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(magic)] - fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(self.op( - other, - PySetInner::difference, - vm, - )?)) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(magic)] - fn rsub( - zelf: PyRef<Self>, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(Self { - inner: other - .as_inner() - .difference(ArgIterable::try_from_object(vm, zelf.into())?, vm)?, - })) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(name = "__rxor__")] - #[pymethod(magic)] - fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(self.op( - other, - PySetInner::symmetric_difference, - vm, - )?)) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod] - pub fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.inner.add(item, vm)?; - Ok(()) - } - - #[pymethod] - fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.inner.remove(item, vm) - } - - #[pymethod] - fn discard(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.inner.discard(&item, vm)?; - Ok(()) - } - - #[pymethod] - fn clear(&self) { - self.inner.clear() - } - - #[pymethod] - fn pop(&self, vm: &VirtualMachine) -> PyResult { - self.inner.pop(vm) - } - - #[pymethod(magic)] - fn ior(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.inner.update(set.into_iterable_iter(vm)?, vm)?; - Ok(zelf) - } - - #[pymethod] - fn update(&self, others: PosArgs<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { - for iterable in others { - self.inner.update_internal(iterable, vm)?; - } - Ok(()) - } - - #[pymethod] - fn intersection_update( - &self, - others: PosArgs<ArgIterable>, - vm: &VirtualMachine, - ) -> PyResult<()> { - self.inner.intersection_update(others.into_iter(), vm)?; - Ok(()) - } - - #[pymethod(magic)] - fn iand(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.inner - .intersection_update(std::iter::once(set.into_iterable(vm)?), vm)?; - Ok(zelf) - } - - #[pymethod] - fn difference_update(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<()> { - self.inner.difference_update(others.into_iter(), vm)?; - Ok(()) - } - - #[pymethod(magic)] - fn isub(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.inner - .difference_update(set.into_iterable_iter(vm)?, vm)?; - Ok(zelf) - } - - #[pymethod] - fn symmetric_difference_update( - &self, - others: PosArgs<ArgIterable>, - vm: &VirtualMachine, - ) -> PyResult<()> { - self.inner - .symmetric_difference_update(others.into_iter(), vm)?; - Ok(()) - } - - #[pymethod(magic)] - fn ixor(zelf: PyRef<Self>, set: AnySet, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - zelf.inner - .symmetric_difference_update(set.into_iterable_iter(vm)?, vm)?; - Ok(zelf) - } - - #[pymethod(magic)] - fn reduce( - zelf: PyRef<Self>, - vm: &VirtualMachine, - ) -> PyResult<(PyTypeRef, PyTupleRef, Option<PyDictRef>)> { - reduce_set(zelf.as_ref(), vm) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } -} - -impl Constructor for PySet { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - PySet::default().into_ref_with_type(vm, cls).map(Into::into) - } -} - -impl Initializer for PySet { - type Args = OptionalArg<PyObjectRef>; - - fn init(zelf: PyRef<Self>, iterable: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - if zelf.len() > 0 { - zelf.clear(); - } - if let OptionalArg::Present(it) = iterable { - zelf.update(PosArgs::new(vec![it]), vm)?; - } - Ok(()) - } -} - -impl AsSequence for PySet { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PySet::sequence_downcast(seq).len())), - contains: atomic_func!(|seq, needle, vm| PySet::sequence_downcast(seq) - .inner - .contains(needle, vm)), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -impl Comparable for PySet { - fn cmp( - zelf: &crate::Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - extract_set(other).map_or(Ok(PyComparisonValue::NotImplemented), |other| { - Ok(zelf.inner.compare(other, op, vm)?.into()) - }) - } -} - -impl Iterable for PySet { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok(zelf.inner.iter().into_pyobject(vm)) - } -} - -impl AsNumber for PySet { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - subtract: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PySet>() { - a.sub(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - and: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PySet>() { - a.and(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - xor: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PySet>() { - a.xor(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - or: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PySet>() { - a.or(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - inplace_subtract: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PySet>() { - PySet::isub(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) - .to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - inplace_and: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PySet>() { - PySet::iand(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) - .to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - inplace_xor: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PySet>() { - PySet::ixor(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) - .to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - inplace_or: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PySet>() { - PySet::ior(a.to_owned(), AnySet::try_from_object(vm, b.to_owned())?, vm) - .to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl Representable for PySet { - #[inline] - fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let class = zelf.class(); - let borrowed_name = class.name(); - let class_name = borrowed_name.deref(); - let s = if zelf.inner.len() == 0 { - format!("{class_name}()") - } else if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let name = if class_name != "set" { - Some(class_name) - } else { - None - }; - zelf.inner.repr(name, vm)? - } else { - format!("{class_name}(...)") - }; - Ok(s) - } -} - -impl Constructor for PyFrozenSet { - type Args = OptionalArg<PyObjectRef>; - - fn py_new(cls: PyTypeRef, iterable: Self::Args, vm: &VirtualMachine) -> PyResult { - let elements = if let OptionalArg::Present(iterable) = iterable { - let iterable = if cls.is(vm.ctx.types.frozenset_type) { - match iterable.downcast_exact::<Self>(vm) { - Ok(fs) => return Ok(fs.into_pyref().into()), - Err(iterable) => iterable, - } - } else { - iterable - }; - iterable.try_to_value(vm)? - } else { - vec![] - }; - - // Return empty fs if iterable passed is empty and only for exact fs types. - if elements.is_empty() && cls.is(vm.ctx.types.frozenset_type) { - Ok(vm.ctx.empty_frozenset.clone().into()) - } else { - Self::from_iter(vm, elements) - .and_then(|o| o.into_ref_with_type(vm, cls).map(Into::into)) - } - } -} - -#[pyclass( - flags(BASETYPE), - with( - Constructor, - AsSequence, - Hashable, - Comparable, - Iterable, - AsNumber, - Representable - ) -)] -impl PyFrozenSet { - #[pymethod(magic)] - fn len(&self) -> usize { - self.inner.len() - } - - #[pymethod(magic)] - fn sizeof(&self) -> usize { - std::mem::size_of::<Self>() + self.inner.sizeof() - } - - #[pymethod] - fn copy(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRef<Self> { - if zelf.class().is(vm.ctx.types.frozenset_type) { - zelf - } else { - Self { - inner: zelf.inner.copy(), - } - .into_ref(&vm.ctx) - } - } - - #[pymethod(magic)] - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.contains(&needle, vm) - } - - #[pymethod] - fn union(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { - self.fold_op(others.into_iter(), PySetInner::union, vm) - } - - #[pymethod] - fn intersection(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { - self.fold_op(others.into_iter(), PySetInner::intersection, vm) - } - - #[pymethod] - fn difference(&self, others: PosArgs<ArgIterable>, vm: &VirtualMachine) -> PyResult<Self> { - self.fold_op(others.into_iter(), PySetInner::difference, vm) - } - - #[pymethod] - fn symmetric_difference( - &self, - others: PosArgs<ArgIterable>, - vm: &VirtualMachine, - ) -> PyResult<Self> { - self.fold_op(others.into_iter(), PySetInner::symmetric_difference, vm) - } - - #[pymethod] - fn issubset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.issubset(other, vm) - } - - #[pymethod] - fn issuperset(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.issuperset(other, vm) - } - - #[pymethod] - fn isdisjoint(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult<bool> { - self.inner.isdisjoint(other, vm) - } - - #[pymethod(name = "__ror__")] - #[pymethod(magic)] - fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(set) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(self.op( - set, - PySetInner::union, - vm, - )?)) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(name = "__rand__")] - #[pymethod(magic)] - fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(self.op( - other, - PySetInner::intersection, - vm, - )?)) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(magic)] - fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(self.op( - other, - PySetInner::difference, - vm, - )?)) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(magic)] - fn rsub( - zelf: PyRef<Self>, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(Self { - inner: other - .as_inner() - .difference(ArgIterable::try_from_object(vm, zelf.into())?, vm)?, - })) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(name = "__rxor__")] - #[pymethod(magic)] - fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyArithmeticValue<Self>> { - if let Ok(other) = AnySet::try_from_object(vm, other) { - Ok(PyArithmeticValue::Implemented(self.op( - other, - PySetInner::symmetric_difference, - vm, - )?)) - } else { - Ok(PyArithmeticValue::NotImplemented) - } - } - - #[pymethod(magic)] - fn reduce( - zelf: PyRef<Self>, - vm: &VirtualMachine, - ) -> PyResult<(PyTypeRef, PyTupleRef, Option<PyDictRef>)> { - reduce_set(zelf.as_ref(), vm) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } -} - -impl AsSequence for PyFrozenSet { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyFrozenSet::sequence_downcast(seq).len())), - contains: atomic_func!(|seq, needle, vm| PyFrozenSet::sequence_downcast(seq) - .inner - .contains(needle, vm)), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -impl Hashable for PyFrozenSet { - #[inline] - fn hash(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - zelf.inner.hash(vm) - } -} - -impl Comparable for PyFrozenSet { - fn cmp( - zelf: &crate::Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - extract_set(other).map_or(Ok(PyComparisonValue::NotImplemented), |other| { - Ok(zelf.inner.compare(other, op, vm)?.into()) - }) - } -} - -impl Iterable for PyFrozenSet { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok(zelf.inner.iter().into_pyobject(vm)) - } -} - -impl AsNumber for PyFrozenSet { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - subtract: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyFrozenSet>() { - a.sub(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - and: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyFrozenSet>() { - a.and(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - xor: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyFrozenSet>() { - a.xor(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - or: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyFrozenSet>() { - a.or(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl Representable for PyFrozenSet { - #[inline] - fn repr_str(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let inner = &zelf.inner; - let class = zelf.class(); - let class_name = class.name(); - let s = if inner.len() == 0 { - format!("{class_name}()") - } else if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - inner.repr(Some(&class_name), vm)? - } else { - format!("{class_name}(...)") - }; - Ok(s) - } -} - -struct AnySet { - object: PyObjectRef, -} - -impl AnySet { - fn into_iterable(self, vm: &VirtualMachine) -> PyResult<ArgIterable> { - self.object.try_into_value(vm) - } - - fn into_iterable_iter( - self, - vm: &VirtualMachine, - ) -> PyResult<impl std::iter::Iterator<Item = ArgIterable>> { - Ok(std::iter::once(self.into_iterable(vm)?)) - } - - fn as_inner(&self) -> &PySetInner { - match_class!(match self.object.as_object() { - ref set @ PySet => &set.inner, - ref frozen @ PyFrozenSet => &frozen.inner, - _ => unreachable!("AnySet is always PySet or PyFrozenSet"), // should not be called. - }) - } -} - -impl TryFromObject for AnySet { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let class = obj.class(); - if class.fast_issubclass(vm.ctx.types.set_type) - || class.fast_issubclass(vm.ctx.types.frozenset_type) - { - Ok(AnySet { object: obj }) - } else { - Err(vm.new_type_error(format!("{class} is not a subtype of set or frozenset"))) - } - } -} - -#[pyclass(module = false, name = "set_iterator")] -pub(crate) struct PySetIterator { - size: DictSize, - internal: PyMutex<PositionIterInternal<PyRc<SetContentType>>>, -} - -impl fmt::Debug for PySetIterator { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // TODO: implement more detailed, non-recursive Debug formatter - f.write_str("set_iterator") - } -} - -impl PyPayload for PySetIterator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.set_iterator_type - } -} - -#[pyclass(with(Constructor, IterNext, Iterable))] -impl PySetIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().length_hint(|_| self.size.entries_size) - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<(PyObjectRef, (PyObjectRef,))> { - let internal = zelf.internal.lock(); - Ok(( - builtins_iter(vm).to_owned(), - (vm.ctx - .new_list(match &internal.status { - IterStatus::Exhausted => vec![], - IterStatus::Active(dict) => { - dict.keys().into_iter().skip(internal.position).collect() - } - }) - .into(),), - )) - } -} -impl Unconstructible for PySetIterator {} - -impl SelfIter for PySetIterator {} -impl IterNext for PySetIterator { - fn next(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let mut internal = zelf.internal.lock(); - let next = if let IterStatus::Active(dict) = &internal.status { - if dict.has_changed_size(&zelf.size) { - internal.status = IterStatus::Exhausted; - return Err(vm.new_runtime_error("set changed size during iteration".to_owned())); - } - match dict.next_entry(internal.position) { - Some((position, key, _)) => { - internal.position = position; - PyIterReturn::Return(key) - } - None => { - internal.status = IterStatus::Exhausted; - PyIterReturn::StopIteration(None) - } - } - } else { - PyIterReturn::StopIteration(None) - }; - Ok(next) - } -} - -pub fn init(context: &Context) { - PySet::extend_class(context, context.types.set_type); - PyFrozenSet::extend_class(context, context.types.frozenset_type); - PySetIterator::extend_class(context, context.types.set_iterator_type); -} diff --git a/vm/src/builtins/singletons.rs b/vm/src/builtins/singletons.rs deleted file mode 100644 index 65b171a2627..00000000000 --- a/vm/src/builtins/singletons.rs +++ /dev/null @@ -1,124 +0,0 @@ -use super::{PyStrRef, PyType, PyTypeRef}; -use crate::{ - class::PyClassImpl, - convert::ToPyObject, - protocol::PyNumberMethods, - types::{AsNumber, Constructor, Representable}, - Context, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "NoneType")] -#[derive(Debug)] -pub struct PyNone; - -impl PyPayload for PyNone { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.none_type - } -} - -// This allows a built-in function to not return a value, mapping to -// Python's behavior of returning `None` in this situation. -impl ToPyObject for () { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.none() - } -} - -impl<T: ToPyObject> ToPyObject for Option<T> { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - Some(x) => x.to_pyobject(vm), - None => vm.ctx.none(), - } - } -} - -impl Constructor for PyNone { - type Args = (); - - fn py_new(_: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.none.clone().into()) - } -} - -#[pyclass(with(Constructor, AsNumber, Representable))] -impl PyNone { - #[pymethod(magic)] - fn bool(&self) -> bool { - false - } -} - -impl Representable for PyNone { - #[inline] - fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Ok(vm.ctx.names.None.to_owned()) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } -} - -impl AsNumber for PyNone { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - boolean: Some(|_number, _vm| Ok(false)), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -#[pyclass(module = false, name = "NotImplementedType")] -#[derive(Debug)] -pub struct PyNotImplemented; - -impl PyPayload for PyNotImplemented { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.not_implemented_type - } -} - -impl Constructor for PyNotImplemented { - type Args = (); - - fn py_new(_: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.not_implemented.clone().into()) - } -} - -#[pyclass(with(Constructor))] -impl PyNotImplemented { - // TODO: As per https://bugs.python.org/issue35712, using NotImplemented - // in boolean contexts will need to raise a DeprecationWarning in 3.9 - // and, eventually, a TypeError. - #[pymethod(magic)] - fn bool(&self) -> bool { - true - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyStrRef { - vm.ctx.names.NotImplemented.to_owned() - } -} - -impl Representable for PyNotImplemented { - #[inline] - fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Ok(vm.ctx.names.NotImplemented.to_owned()) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } -} - -pub fn init(context: &Context) { - PyNone::extend_class(context, context.types.none_type); - PyNotImplemented::extend_class(context, context.types.not_implemented_type); -} diff --git a/vm/src/builtins/slice.rs b/vm/src/builtins/slice.rs deleted file mode 100644 index 5da3649115b..00000000000 --- a/vm/src/builtins/slice.rs +++ /dev/null @@ -1,347 +0,0 @@ -// sliceobject.{h,c} in CPython -// spell-checker:ignore sliceobject -use super::{PyStrRef, PyTupleRef, PyType, PyTypeRef}; -use crate::{ - class::PyClassImpl, - common::hash::{PyHash, PyUHash}, - convert::ToPyObject, - function::{ArgIndex, FuncArgs, OptionalArg, PyComparisonValue}, - sliceable::SaturatedSlice, - types::{Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use malachite_bigint::{BigInt, ToBigInt}; -use num_traits::{One, Signed, Zero}; - -#[pyclass(module = false, name = "slice", unhashable = true, traverse)] -#[derive(Debug)] -pub struct PySlice { - pub start: Option<PyObjectRef>, - pub stop: PyObjectRef, - pub step: Option<PyObjectRef>, -} - -impl PyPayload for PySlice { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.slice_type - } -} - -#[pyclass(with(Comparable, Representable, Hashable))] -impl PySlice { - #[pygetset] - fn start(&self, vm: &VirtualMachine) -> PyObjectRef { - self.start.clone().to_pyobject(vm) - } - - pub(crate) fn start_ref<'a>(&'a self, vm: &'a VirtualMachine) -> &'a PyObject { - match &self.start { - Some(v) => v, - None => vm.ctx.none.as_object(), - } - } - - #[pygetset] - pub(crate) fn stop(&self, _vm: &VirtualMachine) -> PyObjectRef { - self.stop.clone() - } - - #[pygetset] - fn step(&self, vm: &VirtualMachine) -> PyObjectRef { - self.step.clone().to_pyobject(vm) - } - - pub(crate) fn step_ref<'a>(&'a self, vm: &'a VirtualMachine) -> &'a PyObject { - match &self.step { - Some(v) => v, - None => vm.ctx.none.as_object(), - } - } - - pub fn to_saturated(&self, vm: &VirtualMachine) -> PyResult<SaturatedSlice> { - SaturatedSlice::with_slice(self, vm) - } - - #[pyslot] - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let slice: PySlice = match args.args.len() { - 0 => { - return Err( - vm.new_type_error("slice() must have at least one arguments.".to_owned()) - ); - } - 1 => { - let stop = args.bind(vm)?; - PySlice { - start: None, - stop, - step: None, - } - } - _ => { - let (start, stop, step): (PyObjectRef, PyObjectRef, OptionalArg<PyObjectRef>) = - args.bind(vm)?; - PySlice { - start: Some(start), - stop, - step: step.into_option(), - } - } - }; - slice.into_ref_with_type(vm, cls).map(Into::into) - } - - pub(crate) fn inner_indices( - &self, - length: &BigInt, - vm: &VirtualMachine, - ) -> PyResult<(BigInt, BigInt, BigInt)> { - // Calculate step - let step: BigInt; - if vm.is_none(self.step_ref(vm)) { - step = One::one(); - } else { - // Clone the value, not the reference. - let this_step = self.step(vm).try_index(vm)?; - step = this_step.as_bigint().clone(); - - if step.is_zero() { - return Err(vm.new_value_error("slice step cannot be zero.".to_owned())); - } - } - - // For convenience - let backwards = step.is_negative(); - - // Each end of the array - let lower = if backwards { - (-1_i8).to_bigint().unwrap() - } else { - Zero::zero() - }; - - let upper = if backwards { - lower.clone() + length - } else { - length.clone() - }; - - // Calculate start - let mut start: BigInt; - if vm.is_none(self.start_ref(vm)) { - // Default - start = if backwards { - upper.clone() - } else { - lower.clone() - }; - } else { - let this_start = self.start(vm).try_index(vm)?; - start = this_start.as_bigint().clone(); - - if start < Zero::zero() { - // From end of array - start += length; - - if start < lower { - start = lower.clone(); - } - } else if start > upper { - start = upper.clone(); - } - } - - // Calculate Stop - let mut stop: BigInt; - if vm.is_none(&self.stop) { - stop = if backwards { lower } else { upper }; - } else { - let this_stop = self.stop(vm).try_index(vm)?; - stop = this_stop.as_bigint().clone(); - - if stop < Zero::zero() { - // From end of array - stop += length; - if stop < lower { - stop = lower; - } - } else if stop > upper { - stop = upper; - } - } - - Ok((start, stop, step)) - } - - #[pymethod] - fn indices(&self, length: ArgIndex, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - let length = length.as_bigint(); - if length.is_negative() { - return Err(vm.new_value_error("length should not be negative.".to_owned())); - } - let (start, stop, step) = self.inner_indices(length, vm)?; - Ok(vm.new_tuple((start, stop, step))) - } - - #[allow(clippy::type_complexity)] - #[pymethod(magic)] - fn reduce( - zelf: PyRef<Self>, - ) -> PyResult<( - PyTypeRef, - (Option<PyObjectRef>, PyObjectRef, Option<PyObjectRef>), - )> { - Ok(( - zelf.class().to_owned(), - (zelf.start.clone(), zelf.stop.clone(), zelf.step.clone()), - )) - } -} - -impl Hashable for PySlice { - #[inline] - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - const XXPRIME_1: PyUHash = if cfg!(target_pointer_width = "64") { - 11400714785074694791 - } else { - 2654435761 - }; - const XXPRIME_2: PyUHash = if cfg!(target_pointer_width = "64") { - 14029467366897019727 - } else { - 2246822519 - }; - const XXPRIME_5: PyUHash = if cfg!(target_pointer_width = "64") { - 2870177450012600261 - } else { - 374761393 - }; - const ROTATE: u32 = if cfg!(target_pointer_width = "64") { - 31 - } else { - 13 - }; - - let mut acc = XXPRIME_5; - for part in [zelf.start_ref(vm), &zelf.stop, zelf.step_ref(vm)].iter() { - let lane = part.hash(vm)? as PyUHash; - if lane == u64::MAX as PyUHash { - return Ok(-1 as PyHash); - } - acc = acc.wrapping_add(lane.wrapping_mul(XXPRIME_2)); - acc = acc.rotate_left(ROTATE); - acc = acc.wrapping_mul(XXPRIME_1); - } - if acc == u64::MAX as PyUHash { - return Ok(1546275796 as PyHash); - } - Ok(acc as PyHash) - } -} - -impl Comparable for PySlice { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - let other = class_or_notimplemented!(Self, other); - - let ret = match op { - PyComparisonOp::Lt | PyComparisonOp::Le => None - .or_else(|| { - vm.bool_seq_lt(zelf.start_ref(vm), other.start_ref(vm)) - .transpose() - }) - .or_else(|| vm.bool_seq_lt(&zelf.stop, &other.stop).transpose()) - .or_else(|| { - vm.bool_seq_lt(zelf.step_ref(vm), other.step_ref(vm)) - .transpose() - }) - .unwrap_or_else(|| Ok(op == PyComparisonOp::Le))?, - PyComparisonOp::Eq | PyComparisonOp::Ne => { - let eq = vm.identical_or_equal(zelf.start_ref(vm), other.start_ref(vm))? - && vm.identical_or_equal(&zelf.stop, &other.stop)? - && vm.identical_or_equal(zelf.step_ref(vm), other.step_ref(vm))?; - if op == PyComparisonOp::Ne { - !eq - } else { - eq - } - } - PyComparisonOp::Gt | PyComparisonOp::Ge => None - .or_else(|| { - vm.bool_seq_gt(zelf.start_ref(vm), other.start_ref(vm)) - .transpose() - }) - .or_else(|| vm.bool_seq_gt(&zelf.stop, &other.stop).transpose()) - .or_else(|| { - vm.bool_seq_gt(zelf.step_ref(vm), other.step_ref(vm)) - .transpose() - }) - .unwrap_or_else(|| Ok(op == PyComparisonOp::Ge))?, - }; - - Ok(PyComparisonValue::Implemented(ret)) - } -} - -impl Representable for PySlice { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let start_repr = zelf.start_ref(vm).repr(vm)?; - let stop_repr = &zelf.stop.repr(vm)?; - let step_repr = zelf.step_ref(vm).repr(vm)?; - - Ok(format!( - "slice({}, {}, {})", - start_repr.as_str(), - stop_repr.as_str(), - step_repr.as_str() - )) - } -} - -#[pyclass(module = false, name = "EllipsisType")] -#[derive(Debug)] -pub struct PyEllipsis; - -impl PyPayload for PyEllipsis { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.ellipsis_type - } -} - -impl Constructor for PyEllipsis { - type Args = (); - - fn py_new(_cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.ellipsis.clone().into()) - } -} - -#[pyclass(with(Constructor, Representable))] -impl PyEllipsis { - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyStrRef { - vm.ctx.names.Ellipsis.to_owned() - } -} - -impl Representable for PyEllipsis { - #[inline] - fn repr(_zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Ok(vm.ctx.names.Ellipsis.to_owned()) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } -} - -pub fn init(ctx: &Context) { - PySlice::extend_class(ctx, ctx.types.slice_type); - PyEllipsis::extend_class(ctx, ctx.types.ellipsis_type); -} diff --git a/vm/src/builtins/staticmethod.rs b/vm/src/builtins/staticmethod.rs deleted file mode 100644 index 59a5b18b5d4..00000000000 --- a/vm/src/builtins/staticmethod.rs +++ /dev/null @@ -1,168 +0,0 @@ -use super::{PyStr, PyType, PyTypeRef}; -use crate::{ - class::PyClassImpl, - common::lock::PyMutex, - function::FuncArgs, - types::{Callable, Constructor, GetDescriptor, Initializer, Representable}, - Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -#[pyclass(module = false, name = "staticmethod", traverse)] -#[derive(Debug)] -pub struct PyStaticMethod { - pub callable: PyMutex<PyObjectRef>, -} - -impl PyPayload for PyStaticMethod { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.staticmethod_type - } -} - -impl GetDescriptor for PyStaticMethod { - fn descr_get( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - _cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let (zelf, _obj) = Self::_unwrap(&zelf, obj, vm)?; - let x = Ok(zelf.callable.lock().clone()); - x - } -} - -impl From<PyObjectRef> for PyStaticMethod { - fn from(callable: PyObjectRef) -> Self { - Self { - callable: PyMutex::new(callable), - } - } -} - -impl Constructor for PyStaticMethod { - type Args = PyObjectRef; - - fn py_new(cls: PyTypeRef, callable: Self::Args, vm: &VirtualMachine) -> PyResult { - let doc = callable.get_attr("__doc__", vm); - - let result = PyStaticMethod { - callable: PyMutex::new(callable), - } - .into_ref_with_type(vm, cls)?; - let obj = PyObjectRef::from(result); - - if let Ok(doc) = doc { - obj.set_attr("__doc__", doc, vm)?; - } - - Ok(obj) - } -} - -impl PyStaticMethod { - pub fn new_ref(callable: PyObjectRef, ctx: &Context) -> PyRef<Self> { - PyRef::new_ref( - Self { - callable: PyMutex::new(callable), - }, - ctx.types.staticmethod_type.to_owned(), - None, - ) - } -} - -impl Initializer for PyStaticMethod { - type Args = PyObjectRef; - - fn init(zelf: PyRef<Self>, callable: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { - *zelf.callable.lock() = callable; - Ok(()) - } -} - -#[pyclass( - with(Callable, GetDescriptor, Constructor, Initializer, Representable), - flags(BASETYPE, HAS_DICT) -)] -impl PyStaticMethod { - #[pygetset(magic)] - fn func(&self) -> PyObjectRef { - self.callable.lock().clone() - } - - #[pygetset(magic)] - fn wrapped(&self) -> PyObjectRef { - self.callable.lock().clone() - } - - #[pygetset(magic)] - fn module(&self, vm: &VirtualMachine) -> PyResult { - self.callable.lock().get_attr("__module__", vm) - } - - #[pygetset(magic)] - fn qualname(&self, vm: &VirtualMachine) -> PyResult { - self.callable.lock().get_attr("__qualname__", vm) - } - - #[pygetset(magic)] - fn name(&self, vm: &VirtualMachine) -> PyResult { - self.callable.lock().get_attr("__name__", vm) - } - - #[pygetset(magic)] - fn annotations(&self, vm: &VirtualMachine) -> PyResult { - self.callable.lock().get_attr("__annotations__", vm) - } - - #[pygetset(magic)] - fn isabstractmethod(&self, vm: &VirtualMachine) -> PyObjectRef { - match vm.get_attribute_opt(self.callable.lock().clone(), "__isabstractmethod__") { - Ok(Some(is_abstract)) => is_abstract, - _ => vm.ctx.new_bool(false).into(), - } - } - - #[pygetset(magic, setter)] - fn set_isabstractmethod(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.callable - .lock() - .set_attr("__isabstractmethod__", value, vm)?; - Ok(()) - } -} - -impl Callable for PyStaticMethod { - type Args = FuncArgs; - #[inline] - fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let callable = zelf.callable.lock().clone(); - callable.call(args, vm) - } -} - -impl Representable for PyStaticMethod { - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let callable = zelf.callable.lock().repr(vm).unwrap(); - let class = Self::class(&vm.ctx); - - match ( - class - .qualname(vm) - .downcast_ref::<PyStr>() - .map(|n| n.as_str()), - class.module(vm).downcast_ref::<PyStr>().map(|m| m.as_str()), - ) { - (None, _) => Err(vm.new_type_error("Unknown qualified name".into())), - (Some(qualname), Some(module)) if module != "builtins" => { - Ok(format!("<{module}.{qualname}({callable})>")) - } - _ => Ok(format!("<{}({})>", class.slot_name(), callable)), - } - } -} - -pub fn init(context: &Context) { - PyStaticMethod::extend_class(context, context.types.staticmethod_type); -} diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs deleted file mode 100644 index b6015cbe265..00000000000 --- a/vm/src/builtins/str.rs +++ /dev/null @@ -1,1818 +0,0 @@ -use super::{ - int::{PyInt, PyIntRef}, - iter::IterStatus::{self, Exhausted}, - PositionIterInternal, PyBytesRef, PyDict, PyTupleRef, PyType, PyTypeRef, -}; -use crate::{ - anystr::{self, adjust_indices, AnyStr, AnyStrContainer, AnyStrWrapper}, - atomic_func, - class::PyClassImpl, - common::str::{BorrowedStr, PyStrKind, PyStrKindData}, - convert::{IntoPyException, ToPyException, ToPyObject, ToPyResult}, - format::{format, format_map}, - function::{ArgIterable, ArgSize, FuncArgs, OptionalArg, OptionalOption, PyComparisonValue}, - intern::PyInterned, - object::{Traverse, TraverseFn}, - protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, - sequence::SequenceExt, - sliceable::{SequenceIndex, SliceableSequenceOp}, - types::{ - AsMapping, AsNumber, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, - }, - AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, - TryFromBorrowedObject, VirtualMachine, -}; -use ascii::{AsciiStr, AsciiString}; -use bstr::ByteSlice; -use itertools::Itertools; -use num_traits::ToPrimitive; -use once_cell::sync::Lazy; -use rustpython_common::{ - ascii, - atomic::{self, PyAtomic, Radium}, - hash, - lock::PyMutex, -}; -use rustpython_format::{FormatSpec, FormatString, FromTemplate}; -use std::{char, fmt, ops::Range, string::ToString}; -use unic_ucd_bidi::BidiClass; -use unic_ucd_category::GeneralCategory; -use unic_ucd_ident::{is_xid_continue, is_xid_start}; -use unicode_casing::CharExt; - -impl<'a> TryFromBorrowedObject<'a> for String { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - obj.try_value_with(|pystr: &PyStr| Ok(pystr.as_str().to_owned()), vm) - } -} - -impl<'a> TryFromBorrowedObject<'a> for &'a str { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - let pystr: &Py<PyStr> = TryFromBorrowedObject::try_from_borrowed_object(vm, obj)?; - Ok(pystr.as_str()) - } -} - -#[pyclass(module = false, name = "str")] -pub struct PyStr { - bytes: Box<[u8]>, - kind: PyStrKindData, - hash: PyAtomic<hash::PyHash>, -} - -impl fmt::Debug for PyStr { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("PyStr") - .field("value", &self.as_str()) - .field("kind", &self.kind) - .field("hash", &self.hash) - .finish() - } -} - -impl AsRef<str> for PyStr { - fn as_ref(&self) -> &str { - self.as_str() - } -} - -impl AsRef<str> for Py<PyStr> { - fn as_ref(&self) -> &str { - self.as_str() - } -} - -impl AsRef<str> for PyStrRef { - fn as_ref(&self) -> &str { - self.as_str() - } -} - -impl<'a> From<&'a AsciiStr> for PyStr { - fn from(s: &'a AsciiStr) -> Self { - s.to_owned().into() - } -} - -impl From<AsciiString> for PyStr { - fn from(s: AsciiString) -> Self { - unsafe { Self::new_ascii_unchecked(s.into()) } - } -} - -impl<'a> From<&'a str> for PyStr { - fn from(s: &'a str) -> Self { - s.to_owned().into() - } -} - -impl From<String> for PyStr { - fn from(s: String) -> Self { - s.into_boxed_str().into() - } -} - -impl<'a> From<std::borrow::Cow<'a, str>> for PyStr { - fn from(s: std::borrow::Cow<'a, str>) -> Self { - s.into_owned().into() - } -} - -impl From<Box<str>> for PyStr { - #[inline] - fn from(value: Box<str>) -> Self { - // doing the check is ~10x faster for ascii, and is actually only 2% slower worst case for - // non-ascii; see https://github.com/RustPython/RustPython/pull/2586#issuecomment-844611532 - let is_ascii = value.is_ascii(); - let bytes = value.into_boxed_bytes(); - let kind = if is_ascii { - PyStrKind::Ascii - } else { - PyStrKind::Utf8 - } - .new_data(); - Self { - bytes, - kind, - hash: Radium::new(hash::SENTINEL), - } - } -} - -pub type PyStrRef = PyRef<PyStr>; - -impl fmt::Display for PyStr { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self.as_str(), f) - } -} - -pub trait AsPyStr<'a> -where - Self: 'a, -{ - #[allow(clippy::wrong_self_convention)] // to implement on refs - fn as_pystr(self, ctx: &Context) -> &'a Py<PyStr>; -} - -impl<'a> AsPyStr<'a> for &'a Py<PyStr> { - #[inline] - fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { - self - } -} - -impl<'a> AsPyStr<'a> for &'a PyStrRef { - #[inline] - fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { - self - } -} - -impl AsPyStr<'static> for &'static str { - #[inline] - fn as_pystr(self, ctx: &Context) -> &'static Py<PyStr> { - ctx.intern_str(self) - } -} - -impl<'a> AsPyStr<'a> for &'a PyStrInterned { - #[inline] - fn as_pystr(self, _ctx: &Context) -> &'a Py<PyStr> { - self - } -} - -#[pyclass(module = false, name = "str_iterator", traverse = "manual")] -#[derive(Debug)] -pub struct PyStrIterator { - internal: PyMutex<(PositionIterInternal<PyStrRef>, usize)>, -} - -unsafe impl Traverse for PyStrIterator { - fn traverse(&self, tracer: &mut TraverseFn) { - // No need to worry about deadlock, for inner is a PyStr and can't make ref cycle - self.internal.lock().0.traverse(tracer); - } -} - -impl PyPayload for PyStrIterator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.str_iterator_type - } -} - -#[pyclass(with(Constructor, IterNext, Iterable))] -impl PyStrIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().0.length_hint(|obj| obj.char_len()) - } - - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut internal = self.internal.lock(); - internal.1 = usize::MAX; - internal - .0 - .set_state(state, |obj, pos| pos.min(obj.char_len()), vm) - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .0 - .builtins_iter_reduce(|x| x.clone().into(), vm) - } -} -impl Unconstructible for PyStrIterator {} - -impl SelfIter for PyStrIterator {} -impl IterNext for PyStrIterator { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let mut internal = zelf.internal.lock(); - - if let IterStatus::Active(s) = &internal.0.status { - let value = s.as_str(); - - if internal.1 == usize::MAX { - if let Some((offset, ch)) = value.char_indices().nth(internal.0.position) { - internal.0.position += 1; - internal.1 = offset + ch.len_utf8(); - return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); - } - } else if let Some(value) = value.get(internal.1..) { - if let Some(ch) = value.chars().next() { - internal.0.position += 1; - internal.1 += ch.len_utf8(); - return Ok(PyIterReturn::Return(ch.to_pyobject(vm))); - } - } - internal.0.status = Exhausted; - } - Ok(PyIterReturn::StopIteration(None)) - } -} - -#[derive(FromArgs)] -pub struct StrArgs { - #[pyarg(any, optional)] - object: OptionalArg<PyObjectRef>, - #[pyarg(any, optional)] - encoding: OptionalArg<PyStrRef>, - #[pyarg(any, optional)] - errors: OptionalArg<PyStrRef>, -} - -impl Constructor for PyStr { - type Args = StrArgs; - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let string: PyStrRef = match args.object { - OptionalArg::Present(input) => { - if let OptionalArg::Present(enc) = args.encoding { - vm.state.codec_registry.decode_text( - input, - enc.as_str(), - args.errors.into_option(), - vm, - )? - } else { - input.str(vm)? - } - } - OptionalArg::Missing => { - PyStr::from(String::new()).into_ref_with_type(vm, cls.clone())? - } - }; - if string.class().is(&cls) { - Ok(string.into()) - } else { - PyStr::from(string.as_str()) - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } -} - -impl PyStr { - /// # Safety: Given `bytes` must be valid data for given `kind` - pub(crate) unsafe fn new_str_unchecked(bytes: Vec<u8>, kind: PyStrKind) -> Self { - let s = Self { - bytes: bytes.into_boxed_slice(), - kind: kind.new_data(), - hash: Radium::new(hash::SENTINEL), - }; - debug_assert!(matches!(s.kind, PyStrKindData::Ascii) || !s.as_str().is_ascii()); - s - } - - /// # Safety - /// Given `bytes` must be ascii - pub unsafe fn new_ascii_unchecked(bytes: Vec<u8>) -> Self { - Self::new_str_unchecked(bytes, PyStrKind::Ascii) - } - - pub fn new_ref(zelf: impl Into<Self>, ctx: &Context) -> PyRef<Self> { - let zelf = zelf.into(); - PyRef::new_ref(zelf, ctx.types.str_type.to_owned(), None) - } - - fn new_substr(&self, s: String) -> Self { - let kind = if self.kind.kind() == PyStrKind::Ascii || s.is_ascii() { - PyStrKind::Ascii - } else { - PyStrKind::Utf8 - }; - unsafe { - // SAFETY: kind is properly decided for substring - Self::new_str_unchecked(s.into_bytes(), kind) - } - } - - #[inline] - pub fn as_str(&self) -> &str { - unsafe { - // SAFETY: Both PyStrKind::{Ascii, Utf8} are valid utf8 string - std::str::from_utf8_unchecked(&self.bytes) - } - } - - fn char_all<F>(&self, test: F) -> bool - where - F: Fn(char) -> bool, - { - match self.kind.kind() { - PyStrKind::Ascii => self.bytes.iter().all(|&x| test(char::from(x))), - PyStrKind::Utf8 => self.as_str().chars().all(test), - } - } - - fn borrow(&self) -> &BorrowedStr { - unsafe { std::mem::transmute(self) } - } - - fn repeat(zelf: PyRef<Self>, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - if value == 0 && zelf.class().is(vm.ctx.types.str_type) { - // Special case: when some `str` is multiplied by `0`, - // returns the empty `str`. - return Ok(vm.ctx.empty_str.to_owned()); - } - if (value == 1 || zelf.is_empty()) && zelf.class().is(vm.ctx.types.str_type) { - // Special case: when some `str` is multiplied by `1` or is the empty `str`, - // nothing really happens, we need to return an object itself - // with the same `id()` to be compatible with CPython. - // This only works for `str` itself, not its subclasses. - return Ok(zelf); - } - zelf.as_str() - .as_bytes() - .mul(vm, value) - .map(|x| Self::from(unsafe { String::from_utf8_unchecked(x) }).into_ref(&vm.ctx)) - } -} - -#[pyclass( - flags(BASETYPE), - with( - PyRef, - AsMapping, - AsNumber, - AsSequence, - Representable, - Hashable, - Comparable, - Iterable, - Constructor - ) -)] -impl PyStr { - #[pymethod(magic)] - fn add(zelf: PyRef<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if let Some(other) = other.payload::<PyStr>() { - let bytes = zelf.as_str().py_add(other.as_ref()); - Ok(unsafe { - // SAFETY: `kind` is safely decided - let kind = zelf.kind.kind() | other.kind.kind(); - Self::new_str_unchecked(bytes.into_bytes(), kind) - } - .to_pyobject(vm)) - } else if let Some(radd) = vm.get_method(other.clone(), identifier!(vm, __radd__)) { - // hack to get around not distinguishing number add from seq concat - radd?.call((zelf,), vm) - } else { - Err(vm.new_type_error(format!( - "can only concatenate str (not \"{}\") to str", - other.class().name() - ))) - } - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - !self.bytes.is_empty() - } - - fn _contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - if let Some(needle) = needle.payload::<Self>() { - Ok(self.as_str().contains(needle.as_str())) - } else { - Err(vm.new_type_error(format!( - "'in <string>' requires string as left operand, not {}", - needle.class().name() - ))) - } - } - - #[pymethod(magic)] - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self._contains(&needle, vm) - } - - fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "str")? { - SequenceIndex::Int(i) => self.getitem_by_index(vm, i).map(|x| x.to_string()), - SequenceIndex::Slice(slice) => self.getitem_by_slice(vm, slice), - } - .map(|x| self.new_substr(x).into_ref(&vm.ctx).into()) - } - - #[pymethod(magic)] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self._getitem(&needle, vm) - } - - #[inline] - pub(crate) fn hash(&self, vm: &VirtualMachine) -> hash::PyHash { - match self.hash.load(atomic::Ordering::Relaxed) { - hash::SENTINEL => self._compute_hash(vm), - hash => hash, - } - } - #[cold] - fn _compute_hash(&self, vm: &VirtualMachine) -> hash::PyHash { - let hash_val = vm.state.hash_secret.hash_str(self.as_str()); - debug_assert_ne!(hash_val, hash::SENTINEL); - // like with char_len, we don't need a cmpxchg loop, since it'll always be the same value - self.hash.store(hash_val, atomic::Ordering::Relaxed); - hash_val - } - - #[inline] - pub fn byte_len(&self) -> usize { - self.bytes.len() - } - #[inline] - pub fn is_empty(&self) -> bool { - self.bytes.is_empty() - } - - #[pymethod(name = "__len__")] - #[inline] - pub fn char_len(&self) -> usize { - self.borrow().char_len() - } - - #[pymethod(name = "isascii")] - #[inline(always)] - pub fn is_ascii(&self) -> bool { - match self.kind { - PyStrKindData::Ascii => true, - PyStrKindData::Utf8(_) => false, - } - } - - #[pymethod(magic)] - fn sizeof(&self) -> usize { - std::mem::size_of::<Self>() + self.byte_len() * std::mem::size_of::<u8>() - } - - #[pymethod(name = "__rmul__")] - #[pymethod(magic)] - fn mul(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - Self::repeat(zelf, value.into(), vm) - } - - #[inline] - pub(crate) fn repr(&self, vm: &VirtualMachine) -> PyResult<String> { - use crate::literal::escape::UnicodeEscape; - let escape = UnicodeEscape::new_repr(self.as_str()); - escape - .str_repr() - .to_string() - .ok_or_else(|| vm.new_overflow_error("string is too long to generate repr".to_owned())) - } - - #[pymethod] - fn lower(&self) -> String { - match self.kind.kind() { - PyStrKind::Ascii => self.as_str().to_ascii_lowercase(), - PyStrKind::Utf8 => self.as_str().to_lowercase(), - } - } - - // casefold is much more aggressive than lower - #[pymethod] - fn casefold(&self) -> String { - caseless::default_case_fold_str(self.as_str()) - } - - #[pymethod] - fn upper(&self) -> String { - match self.kind.kind() { - PyStrKind::Ascii => self.as_str().to_ascii_uppercase(), - PyStrKind::Utf8 => self.as_str().to_uppercase(), - } - } - - #[pymethod] - fn capitalize(&self) -> String { - let mut chars = self.as_str().chars(); - if let Some(first_char) = chars.next() { - format!( - "{}{}", - first_char.to_uppercase(), - &chars.as_str().to_lowercase(), - ) - } else { - "".to_owned() - } - } - - #[pymethod] - fn split(&self, args: SplitArgs, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let elements = match self.kind.kind() { - PyStrKind::Ascii => self.as_str().py_split( - args, - vm, - |v, s, vm| { - v.as_bytes() - .split_str(s) - .map(|s| { - unsafe { PyStr::new_ascii_unchecked(s.to_owned()) }.to_pyobject(vm) - }) - .collect() - }, - |v, s, n, vm| { - v.as_bytes() - .splitn_str(n, s) - .map(|s| { - unsafe { PyStr::new_ascii_unchecked(s.to_owned()) }.to_pyobject(vm) - }) - .collect() - }, - |v, n, vm| { - v.as_bytes().py_split_whitespace(n, |s| { - unsafe { PyStr::new_ascii_unchecked(s.to_owned()) }.to_pyobject(vm) - }) - }, - ), - PyStrKind::Utf8 => self.as_str().py_split( - args, - vm, - |v, s, vm| v.split(s).map(|s| vm.ctx.new_str(s).into()).collect(), - |v, s, n, vm| v.splitn(n, s).map(|s| vm.ctx.new_str(s).into()).collect(), - |v, n, vm| v.py_split_whitespace(n, |s| vm.ctx.new_str(s).into()), - ), - }?; - Ok(elements) - } - - #[pymethod] - fn rsplit(&self, args: SplitArgs, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let mut elements = self.as_str().py_split( - args, - vm, - |v, s, vm| v.rsplit(s).map(|s| vm.ctx.new_str(s).into()).collect(), - |v, s, n, vm| v.rsplitn(n, s).map(|s| vm.ctx.new_str(s).into()).collect(), - |v, n, vm| v.py_rsplit_whitespace(n, |s| vm.ctx.new_str(s).into()), - )?; - // Unlike Python rsplit, Rust rsplitn returns an iterator that - // starts from the end of the string. - elements.reverse(); - Ok(elements) - } - - #[pymethod] - fn strip(&self, chars: OptionalOption<PyStrRef>) -> String { - self.as_str() - .py_strip( - chars, - |s, chars| s.trim_matches(|c| chars.contains(c)), - |s| s.trim(), - ) - .to_owned() - } - - #[pymethod] - fn lstrip( - zelf: PyRef<Self>, - chars: OptionalOption<PyStrRef>, - vm: &VirtualMachine, - ) -> PyRef<Self> { - let s = zelf.as_str(); - let stripped = s.py_strip( - chars, - |s, chars| s.trim_start_matches(|c| chars.contains(c)), - |s| s.trim_start(), - ); - if s == stripped { - zelf - } else { - vm.ctx.new_str(stripped) - } - } - - #[pymethod] - fn rstrip( - zelf: PyRef<Self>, - chars: OptionalOption<PyStrRef>, - vm: &VirtualMachine, - ) -> PyRef<Self> { - let s = zelf.as_str(); - let stripped = s.py_strip( - chars, - |s, chars| s.trim_end_matches(|c| chars.contains(c)), - |s| s.trim_end(), - ); - if s == stripped { - zelf - } else { - vm.ctx.new_str(stripped) - } - } - - #[pymethod] - fn endswith(&self, options: anystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult<bool> { - let (affix, substr) = - match options.prepare(self.as_str(), self.len(), |s, r| s.get_chars(r)) { - Some(x) => x, - None => return Ok(false), - }; - substr.py_startsendswith( - &affix, - "endswith", - "str", - |s, x: &Py<PyStr>| s.ends_with(x.as_str()), - vm, - ) - } - - #[pymethod] - fn startswith( - &self, - options: anystr::StartsEndsWithArgs, - vm: &VirtualMachine, - ) -> PyResult<bool> { - let (affix, substr) = - match options.prepare(self.as_str(), self.len(), |s, r| s.get_chars(r)) { - Some(x) => x, - None => return Ok(false), - }; - substr.py_startsendswith( - &affix, - "startswith", - "str", - |s, x: &Py<PyStr>| s.starts_with(x.as_str()), - vm, - ) - } - - /// Return a str with the given prefix string removed if present. - /// - /// If the string starts with the prefix string, return string[len(prefix):] - /// Otherwise, return a copy of the original string. - #[pymethod] - fn removeprefix(&self, pref: PyStrRef) -> String { - self.as_str() - .py_removeprefix(pref.as_str(), pref.byte_len(), |s, p| s.starts_with(p)) - .to_owned() - } - - /// Return a str with the given suffix string removed if present. - /// - /// If the string ends with the suffix string, return string[:len(suffix)] - /// Otherwise, return a copy of the original string. - #[pymethod] - fn removesuffix(&self, suffix: PyStrRef) -> String { - self.as_str() - .py_removesuffix(suffix.as_str(), suffix.byte_len(), |s, p| s.ends_with(p)) - .to_owned() - } - - #[pymethod] - fn isalnum(&self) -> bool { - !self.bytes.is_empty() && self.char_all(char::is_alphanumeric) - } - - #[pymethod] - fn isnumeric(&self) -> bool { - !self.bytes.is_empty() && self.char_all(char::is_numeric) - } - - #[pymethod] - fn isdigit(&self) -> bool { - // python's isdigit also checks if exponents are digits, these are the unicode codepoints for exponents - let valid_codepoints: [u16; 10] = [ - 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, - ]; - let s = self.as_str(); - !s.is_empty() - && s.chars() - .filter(|c| !c.is_ascii_digit()) - .all(|c| valid_codepoints.contains(&(c as u16))) - } - - #[pymethod] - fn isdecimal(&self) -> bool { - !self.bytes.is_empty() - && self.char_all(|c| GeneralCategory::of(c) == GeneralCategory::DecimalNumber) - } - - #[pymethod(name = "__mod__")] - fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { - let formatted = self.as_str().py_cformat(values, vm)?; - Ok(formatted) - } - - #[pymethod(magic)] - fn rmod(&self, _values: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() - } - - #[pymethod] - fn format(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<String> { - let format_str = FormatString::from_str(self.as_str()).map_err(|e| e.to_pyexception(vm))?; - format(&format_str, &args, vm) - } - - /// S.format_map(mapping) -> str - /// - /// Return a formatted version of S, using substitutions from mapping. - /// The substitutions are identified by braces ('{' and '}'). - #[pymethod] - fn format_map(&self, mapping: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { - let format_string = - FormatString::from_str(self.as_str()).map_err(|err| err.to_pyexception(vm))?; - format_map(&format_string, &mapping, vm) - } - - #[pymethod(name = "__format__")] - fn __format__(zelf: PyRef<Self>, spec: PyStrRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let spec = spec.as_str(); - if spec.is_empty() { - return if zelf.class().is(vm.ctx.types.str_type) { - Ok(zelf) - } else { - zelf.as_object().str(vm) - }; - } - - let s = FormatSpec::parse(spec) - .and_then(|format_spec| format_spec.format_string(zelf.borrow())) - .map_err(|err| err.into_pyexception(vm))?; - Ok(vm.ctx.new_str(s)) - } - - /// Return a titlecased version of the string where words start with an - /// uppercase character and the remaining characters are lowercase. - #[pymethod] - fn title(&self) -> String { - let mut title = String::with_capacity(self.bytes.len()); - let mut previous_is_cased = false; - for c in self.as_str().chars() { - if c.is_lowercase() { - if !previous_is_cased { - title.extend(c.to_titlecase()); - } else { - title.push(c); - } - previous_is_cased = true; - } else if c.is_uppercase() || c.is_titlecase() { - if previous_is_cased { - title.extend(c.to_lowercase()); - } else { - title.push(c); - } - previous_is_cased = true; - } else { - previous_is_cased = false; - title.push(c); - } - } - title - } - - #[pymethod] - fn swapcase(&self) -> String { - let mut swapped_str = String::with_capacity(self.bytes.len()); - for c in self.as_str().chars() { - // to_uppercase returns an iterator, to_ascii_uppercase returns the char - if c.is_lowercase() { - swapped_str.push(c.to_ascii_uppercase()); - } else if c.is_uppercase() { - swapped_str.push(c.to_ascii_lowercase()); - } else { - swapped_str.push(c); - } - } - swapped_str - } - - #[pymethod] - fn isalpha(&self) -> bool { - !self.bytes.is_empty() && self.char_all(char::is_alphabetic) - } - - #[pymethod] - fn replace(&self, old: PyStrRef, new: PyStrRef, count: OptionalArg<isize>) -> String { - let s = self.as_str(); - match count { - OptionalArg::Present(max_count) if max_count >= 0 => { - if max_count == 0 || (s.is_empty() && !old.is_empty()) { - // nothing to do; return the original bytes - s.to_owned() - } else if s.is_empty() && old.is_empty() { - new.as_str().to_owned() - } else { - s.replacen(old.as_str(), new.as_str(), max_count as usize) - } - } - _ => s.replace(old.as_str(), new.as_str()), - } - } - - /// Return true if all characters in the string are printable or the string is empty, - /// false otherwise. Nonprintable characters are those characters defined in the - /// Unicode character database as `Other` or `Separator`, - /// excepting the ASCII space (0x20) which is considered printable. - /// - /// All characters except those characters defined in the Unicode character - /// database as following categories are considered printable. - /// * Cc (Other, Control) - /// * Cf (Other, Format) - /// * Cs (Other, Surrogate) - /// * Co (Other, Private Use) - /// * Cn (Other, Not Assigned) - /// * Zl Separator, Line ('\u2028', LINE SEPARATOR) - /// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) - /// * Zs (Separator, Space) other than ASCII space('\x20'). - #[pymethod] - fn isprintable(&self) -> bool { - self.char_all(|c| c == '\u{0020}' || rustpython_literal::char::is_printable(c)) - } - - #[pymethod] - fn isspace(&self) -> bool { - use unic_ucd_bidi::bidi_class::abbr_names::*; - !self.bytes.is_empty() - && self.char_all(|c| { - GeneralCategory::of(c) == GeneralCategory::SpaceSeparator - || matches!(BidiClass::of(c), WS | B | S) - }) - } - - // Return true if all cased characters in the string are lowercase and there is at least one cased character, false otherwise. - #[pymethod] - fn islower(&self) -> bool { - match self.kind.kind() { - PyStrKind::Ascii => self.bytes.py_iscase(char::is_lowercase, char::is_uppercase), - PyStrKind::Utf8 => self - .as_str() - .py_iscase(char::is_lowercase, char::is_uppercase), - } - } - - // Return true if all cased characters in the string are uppercase and there is at least one cased character, false otherwise. - #[pymethod] - fn isupper(&self) -> bool { - match self.kind.kind() { - PyStrKind::Ascii => self.bytes.py_iscase(char::is_uppercase, char::is_lowercase), - PyStrKind::Utf8 => self - .as_str() - .py_iscase(char::is_uppercase, char::is_lowercase), - } - } - - #[pymethod] - fn splitlines(&self, args: anystr::SplitLinesArgs, vm: &VirtualMachine) -> Vec<PyObjectRef> { - let into_wrapper = |s: &str| self.new_substr(s.to_owned()).to_pyobject(vm); - let mut elements = Vec::new(); - let mut last_i = 0; - let self_str = self.as_str(); - let mut enumerated = self_str.char_indices().peekable(); - while let Some((i, ch)) = enumerated.next() { - let end_len = match ch { - '\n' => 1, - '\r' => { - let is_rn = enumerated.peek().map_or(false, |(_, ch)| *ch == '\n'); - if is_rn { - let _ = enumerated.next(); - 2 - } else { - 1 - } - } - '\x0b' | '\x0c' | '\x1c' | '\x1d' | '\x1e' | '\u{0085}' | '\u{2028}' - | '\u{2029}' => ch.len_utf8(), - _ => { - continue; - } - }; - let range = if args.keepends { - last_i..i + end_len - } else { - last_i..i - }; - last_i = i + end_len; - elements.push(into_wrapper(&self_str[range])); - } - if last_i != self_str.len() { - elements.push(into_wrapper(&self_str[last_i..])); - } - elements - } - - #[pymethod] - fn join( - zelf: PyRef<Self>, - iterable: ArgIterable<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<PyStrRef> { - let iter = iterable.iter(vm)?; - let joined = match iter.exactly_one() { - Ok(first) => { - let first = first?; - if first.as_object().class().is(vm.ctx.types.str_type) { - return Ok(first); - } else { - first.as_str().to_owned() - } - } - Err(iter) => zelf.as_str().py_join(iter)?, - }; - Ok(vm.ctx.new_str(joined)) - } - - // FIXME: two traversals of str is expensive - #[inline] - fn _to_char_idx(r: &str, byte_idx: usize) -> usize { - r[..byte_idx].chars().count() - } - - #[inline] - fn _find<F>(&self, args: FindArgs, find: F) -> Option<usize> - where - F: Fn(&str, &str) -> Option<usize>, - { - let (sub, range) = args.get_value(self.len()); - self.as_str().py_find(sub.as_str(), range, find) - } - - #[pymethod] - fn find(&self, args: FindArgs) -> isize { - self._find(args, |r, s| Some(Self::_to_char_idx(r, r.find(s)?))) - .map_or(-1, |v| v as isize) - } - - #[pymethod] - fn rfind(&self, args: FindArgs) -> isize { - self._find(args, |r, s| Some(Self::_to_char_idx(r, r.rfind(s)?))) - .map_or(-1, |v| v as isize) - } - - #[pymethod] - fn index(&self, args: FindArgs, vm: &VirtualMachine) -> PyResult<usize> { - self._find(args, |r, s| Some(Self::_to_char_idx(r, r.find(s)?))) - .ok_or_else(|| vm.new_value_error("substring not found".to_owned())) - } - - #[pymethod] - fn rindex(&self, args: FindArgs, vm: &VirtualMachine) -> PyResult<usize> { - self._find(args, |r, s| Some(Self::_to_char_idx(r, r.rfind(s)?))) - .ok_or_else(|| vm.new_value_error("substring not found".to_owned())) - } - - #[pymethod] - fn partition(&self, sep: PyStrRef, vm: &VirtualMachine) -> PyResult { - let (front, has_mid, back) = self.as_str().py_partition( - sep.as_str(), - || self.as_str().splitn(2, sep.as_str()), - vm, - )?; - let partition = ( - self.new_substr(front), - if has_mid { - sep - } else { - vm.ctx.new_str(ascii!("")) - }, - self.new_substr(back), - ); - Ok(partition.to_pyobject(vm)) - } - - #[pymethod] - fn rpartition(&self, sep: PyStrRef, vm: &VirtualMachine) -> PyResult { - let (back, has_mid, front) = self.as_str().py_partition( - sep.as_str(), - || self.as_str().rsplitn(2, sep.as_str()), - vm, - )?; - Ok(( - self.new_substr(front), - if has_mid { - sep - } else { - vm.ctx.new_str(ascii!("")) - }, - self.new_substr(back), - ) - .to_pyobject(vm)) - } - - /// Return `true` if the sequence is ASCII titlecase and the sequence is not - /// empty, `false` otherwise. - #[pymethod] - fn istitle(&self) -> bool { - if self.bytes.is_empty() { - return false; - } - - let mut cased = false; - let mut previous_is_cased = false; - for c in self.as_str().chars() { - if c.is_uppercase() || c.is_titlecase() { - if previous_is_cased { - return false; - } - previous_is_cased = true; - cased = true; - } else if c.is_lowercase() { - if !previous_is_cased { - return false; - } - previous_is_cased = true; - cased = true; - } else { - previous_is_cased = false; - } - } - cased - } - - #[pymethod] - fn count(&self, args: FindArgs) -> usize { - let (needle, range) = args.get_value(self.len()); - self.as_str() - .py_count(needle.as_str(), range, |h, n| h.matches(n).count()) - } - - #[pymethod] - fn zfill(&self, width: isize) -> String { - unsafe { - // SAFETY: this is safe-guaranteed because the original self.as_str() is valid utf8 - String::from_utf8_unchecked(self.as_str().py_zfill(width)) - } - } - - #[inline] - fn _pad( - &self, - width: isize, - fillchar: OptionalArg<PyStrRef>, - pad: fn(&str, usize, char, usize) -> String, - vm: &VirtualMachine, - ) -> PyResult<String> { - let fillchar = fillchar.map_or(Ok(' '), |ref s| { - s.as_str().chars().exactly_one().map_err(|_| { - vm.new_type_error( - "The fill character must be exactly one character long".to_owned(), - ) - }) - })?; - Ok(if self.len() as isize >= width { - String::from(self.as_str()) - } else { - pad(self.as_str(), width as usize, fillchar, self.len()) - }) - } - - #[pymethod] - fn center( - &self, - width: isize, - fillchar: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<String> { - self._pad(width, fillchar, AnyStr::py_center, vm) - } - - #[pymethod] - fn ljust( - &self, - width: isize, - fillchar: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<String> { - self._pad(width, fillchar, AnyStr::py_ljust, vm) - } - - #[pymethod] - fn rjust( - &self, - width: isize, - fillchar: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<String> { - self._pad(width, fillchar, AnyStr::py_rjust, vm) - } - - #[pymethod] - fn expandtabs(&self, args: anystr::ExpandTabsArgs) -> String { - let tab_stop = args.tabsize(); - let mut expanded_str = String::with_capacity(self.byte_len()); - let mut tab_size = tab_stop; - let mut col_count = 0usize; - for ch in self.as_str().chars() { - match ch { - '\t' => { - let num_spaces = tab_size - col_count; - col_count += num_spaces; - let expand = " ".repeat(num_spaces); - expanded_str.push_str(&expand); - } - '\r' | '\n' => { - expanded_str.push(ch); - col_count = 0; - tab_size = 0; - } - _ => { - expanded_str.push(ch); - col_count += 1; - } - } - if col_count >= tab_size { - tab_size += tab_stop; - } - } - expanded_str - } - - #[pymethod] - fn isidentifier(&self) -> bool { - let mut chars = self.as_str().chars(); - let is_identifier_start = chars.next().map_or(false, |c| c == '_' || is_xid_start(c)); - // a string is not an identifier if it has whitespace or starts with a number - is_identifier_start && chars.all(is_xid_continue) - } - - // https://docs.python.org/3/library/stdtypes.html#str.translate - #[pymethod] - fn translate(&self, table: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { - vm.get_method_or_type_error(table.clone(), identifier!(vm, __getitem__), || { - format!("'{}' object is not subscriptable", table.class().name()) - })?; - - let mut translated = String::new(); - for c in self.as_str().chars() { - match table.get_item(&*(c as u32).to_pyobject(vm), vm) { - Ok(value) => { - if let Some(text) = value.payload::<PyStr>() { - translated.push_str(text.as_str()); - } else if let Some(bigint) = value.payload::<PyInt>() { - let ch = bigint - .as_bigint() - .to_u32() - .and_then(std::char::from_u32) - .ok_or_else(|| { - vm.new_value_error( - "character mapping must be in range(0x110000)".to_owned(), - ) - })?; - translated.push(ch); - } else if !vm.is_none(&value) { - return Err(vm.new_type_error( - "character mapping must return integer, None or str".to_owned(), - )); - } - } - _ => translated.push(c), - } - } - Ok(translated) - } - - #[pystaticmethod] - fn maketrans( - dict_or_str: PyObjectRef, - to_str: OptionalArg<PyStrRef>, - none_str: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult { - let new_dict = vm.ctx.new_dict(); - if let OptionalArg::Present(to_str) = to_str { - match dict_or_str.downcast::<PyStr>() { - Ok(from_str) => { - if to_str.len() == from_str.len() { - for (c1, c2) in from_str.as_str().chars().zip(to_str.as_str().chars()) { - new_dict.set_item( - &*vm.new_pyobj(c1 as u32), - vm.new_pyobj(c2 as u32), - vm, - )?; - } - if let OptionalArg::Present(none_str) = none_str { - for c in none_str.as_str().chars() { - new_dict.set_item(&*vm.new_pyobj(c as u32), vm.ctx.none(), vm)?; - } - } - Ok(new_dict.to_pyobject(vm)) - } else { - Err(vm.new_value_error( - "the first two maketrans arguments must have equal length".to_owned(), - )) - } - } - _ => Err(vm.new_type_error( - "first maketrans argument must be a string if there is a second argument" - .to_owned(), - )), - } - } else { - // dict_str must be a dict - match dict_or_str.downcast::<PyDict>() { - Ok(dict) => { - for (key, val) in dict { - // FIXME: ints are key-compatible - if let Some(num) = key.payload::<PyInt>() { - new_dict.set_item( - &*num.as_bigint().to_i32().to_pyobject(vm), - val, - vm, - )?; - } else if let Some(string) = key.payload::<PyStr>() { - if string.len() == 1 { - let num_value = string.as_str().chars().next().unwrap() as u32; - new_dict.set_item(&*num_value.to_pyobject(vm), val, vm)?; - } else { - return Err(vm.new_value_error( - "string keys in translate table must be of length 1".to_owned(), - )); - } - } - } - Ok(new_dict.to_pyobject(vm)) - } - _ => Err(vm.new_value_error( - "if you give only one argument to maketrans it must be a dict".to_owned(), - )), - } - } - } - - #[pymethod] - fn encode(zelf: PyRef<Self>, args: EncodeArgs, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - encode_string(zelf, args.encoding, args.errors, vm) - } - - #[pymethod(magic)] - fn getnewargs(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyObjectRef { - (zelf.as_str(),).to_pyobject(vm) - } -} - -#[pyclass] -impl PyRef<PyStr> { - #[pymethod(magic)] - fn str(self, vm: &VirtualMachine) -> PyRefExact<PyStr> { - self.into_exact_or(&vm.ctx, |zelf| unsafe { - // Creating a copy with same kind is safe - PyStr::new_str_unchecked(zelf.bytes.to_vec(), zelf.kind.kind()).into_exact_ref(&vm.ctx) - }) - } -} - -impl PyStrRef { - pub fn concat_in_place(&mut self, other: &str, vm: &VirtualMachine) { - // TODO: call [A]Rc::get_mut on the str to try to mutate the data in place - if other.is_empty() { - return; - } - let mut s = String::with_capacity(self.byte_len() + other.len()); - s.push_str(self.as_ref()); - s.push_str(other); - *self = PyStr::from(s).into_ref(&vm.ctx); - } -} - -impl Representable for PyStr { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - zelf.repr(vm) - } -} - -impl Hashable for PyStr { - #[inline] - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { - Ok(zelf.hash(vm)) - } -} - -impl Comparable for PyStr { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - _vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - if let Some(res) = op.identical_optimization(zelf, other) { - return Ok(res.into()); - } - let other = class_or_notimplemented!(Self, other); - Ok(op.eval_ord(zelf.as_str().cmp(other.as_str())).into()) - } -} - -impl Iterable for PyStr { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok(PyStrIterator { - internal: PyMutex::new((PositionIterInternal::new(zelf, 0), 0)), - } - .into_pyobject(vm)) - } -} - -impl AsMapping for PyStr { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy<PyMappingMethods> = Lazy::new(|| PyMappingMethods { - length: atomic_func!(|mapping, _vm| Ok(PyStr::mapping_downcast(mapping).len())), - subscript: atomic_func!( - |mapping, needle, vm| PyStr::mapping_downcast(mapping)._getitem(needle, vm) - ), - ..PyMappingMethods::NOT_IMPLEMENTED - }); - &AS_MAPPING - } -} - -impl AsNumber for PyStr { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - remainder: Some(|a, b, vm| { - if let Some(a) = a.downcast_ref::<PyStr>() { - a.modulo(b.to_owned(), vm).to_pyresult(vm) - } else { - Ok(vm.ctx.not_implemented()) - } - }), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl AsSequence for PyStr { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyStr::sequence_downcast(seq).len())), - concat: atomic_func!(|seq, other, vm| { - let zelf = PyStr::sequence_downcast(seq); - PyStr::add(zelf.to_owned(), other.to_owned(), vm) - }), - repeat: atomic_func!(|seq, n, vm| { - let zelf = PyStr::sequence_downcast(seq); - PyStr::repeat(zelf.to_owned(), n, vm).map(|x| x.into()) - }), - item: atomic_func!(|seq, i, vm| { - let zelf = PyStr::sequence_downcast(seq); - zelf.getitem_by_index(vm, i) - .map(|x| zelf.new_substr(x.to_string()).into_ref(&vm.ctx).into()) - }), - contains: atomic_func!( - |seq, needle, vm| PyStr::sequence_downcast(seq)._contains(needle, vm) - ), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -#[derive(FromArgs)] -struct EncodeArgs { - #[pyarg(any, default)] - encoding: Option<PyStrRef>, - #[pyarg(any, default)] - errors: Option<PyStrRef>, -} - -pub(crate) fn encode_string( - s: PyStrRef, - encoding: Option<PyStrRef>, - errors: Option<PyStrRef>, - vm: &VirtualMachine, -) -> PyResult<PyBytesRef> { - let encoding = encoding - .as_ref() - .map_or(crate::codecs::DEFAULT_ENCODING, |s| s.as_str()); - vm.state.codec_registry.encode_text(s, encoding, errors, vm) -} - -impl PyPayload for PyStr { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.str_type - } -} - -impl ToPyObject for String { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self).into() - } -} - -impl ToPyObject for char { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self.to_string()).into() - } -} - -impl ToPyObject for &str { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self).into() - } -} - -impl ToPyObject for &String { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self.clone()).into() - } -} - -impl ToPyObject for &AsciiStr { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self).into() - } -} - -impl ToPyObject for AsciiString { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self).into() - } -} - -type SplitArgs = anystr::SplitArgs<PyStrRef>; - -#[derive(FromArgs)] -pub struct FindArgs { - #[pyarg(positional)] - sub: PyStrRef, - #[pyarg(positional, default)] - start: Option<PyIntRef>, - #[pyarg(positional, default)] - end: Option<PyIntRef>, -} - -impl FindArgs { - fn get_value(self, len: usize) -> (PyStrRef, std::ops::Range<usize>) { - let range = adjust_indices(self.start, self.end, len); - (self.sub, range) - } -} - -pub fn init(ctx: &Context) { - PyStr::extend_class(ctx, ctx.types.str_type); - - PyStrIterator::extend_class(ctx, ctx.types.str_iterator_type); -} - -impl SliceableSequenceOp for PyStr { - type Item = char; - type Sliced = String; - - fn do_get(&self, index: usize) -> Self::Item { - if self.is_ascii() { - self.bytes[index] as char - } else { - self.as_str().chars().nth(index).unwrap() - } - } - - fn do_slice(&self, range: Range<usize>) -> Self::Sliced { - let value = self.as_str(); - if self.is_ascii() { - value[range].to_owned() - } else { - rustpython_common::str::get_chars(value, range).to_owned() - } - } - - fn do_slice_reverse(&self, range: Range<usize>) -> Self::Sliced { - if self.is_ascii() { - // this is an ascii string - let mut v = self.bytes[range].to_vec(); - v.reverse(); - unsafe { - // SAFETY: an ascii string is always utf8 - String::from_utf8_unchecked(v) - } - } else { - let mut s = String::with_capacity(self.bytes.len()); - s.extend( - self.as_str() - .chars() - .rev() - .skip(self.char_len() - range.end) - .take(range.end - range.start), - ); - s - } - } - - fn do_stepped_slice(&self, range: Range<usize>, step: usize) -> Self::Sliced { - if self.is_ascii() { - let v = self.bytes[range].iter().copied().step_by(step).collect(); - unsafe { - // SAFETY: Any subset of ascii string is a valid utf8 string - String::from_utf8_unchecked(v) - } - } else { - let mut s = String::with_capacity(2 * ((range.len() / step) + 1)); - s.extend( - self.as_str() - .chars() - .skip(range.start) - .take(range.end - range.start) - .step_by(step), - ); - s - } - } - - fn do_stepped_slice_reverse(&self, range: Range<usize>, step: usize) -> Self::Sliced { - if self.is_ascii() { - // this is an ascii string - let v: Vec<u8> = self.bytes[range] - .iter() - .rev() - .copied() - .step_by(step) - .collect(); - // TODO: from_utf8_unchecked? - String::from_utf8(v).unwrap() - } else { - // not ascii, so the codepoints have to be at least 2 bytes each - let mut s = String::with_capacity(2 * ((range.len() / step) + 1)); - s.extend( - self.as_str() - .chars() - .rev() - .skip(self.char_len() - range.end) - .take(range.end - range.start) - .step_by(step), - ); - s - } - } - - fn empty() -> Self::Sliced { - String::new() - } - - fn len(&self) -> usize { - self.char_len() - } -} - -impl AsRef<str> for PyRefExact<PyStr> { - fn as_ref(&self) -> &str { - self.as_str() - } -} - -impl AsRef<str> for PyExact<PyStr> { - fn as_ref(&self) -> &str { - self.as_str() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Interpreter; - - #[test] - fn str_title() { - let tests = vec![ - (" Hello ", " hello "), - ("Hello ", "hello "), - ("Hello ", "Hello "), - ("Format This As Title String", "fOrMaT thIs aS titLe String"), - ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), - ("Getint", "getInt"), - ("Greek Ωppercases ...", "greek ωppercases ..."), - ("Greek ῼitlecases ...", "greek ῳitlecases ..."), - ]; - for (title, input) in tests { - assert_eq!(PyStr::from(input).title().as_str(), title); - } - } - - #[test] - fn str_istitle() { - let pos = vec![ - "A", - "A Titlecased Line", - "A\nTitlecased Line", - "A Titlecased, Line", - "Greek Ωppercases ...", - "Greek ῼitlecases ...", - ]; - - for s in pos { - assert!(PyStr::from(s).istitle()); - } - - let neg = vec![ - "", - "a", - "\n", - "Not a capitalized String", - "Not\ta Titlecase String", - "Not--a Titlecase String", - "NOT", - ]; - for s in neg { - assert!(!PyStr::from(s).istitle()); - } - } - - #[test] - fn str_maketrans_and_translate() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { - let table = vm.ctx.new_dict(); - table - .set_item("a", vm.ctx.new_str("🎅").into(), vm) - .unwrap(); - table.set_item("b", vm.ctx.none(), vm).unwrap(); - table - .set_item("c", vm.ctx.new_str(ascii!("xda")).into(), vm) - .unwrap(); - let translated = - PyStr::maketrans(table.into(), OptionalArg::Missing, OptionalArg::Missing, vm) - .unwrap(); - let text = PyStr::from("abc"); - let translated = text.translate(translated, vm).unwrap(); - assert_eq!(translated, "🎅xda".to_owned()); - let translated = text.translate(vm.ctx.new_int(3).into(), vm); - assert_eq!("TypeError", &*translated.unwrap_err().class().name(),); - }) - } -} - -impl AnyStrWrapper for PyStrRef { - type Str = str; - fn as_ref(&self) -> &str { - self.as_str() - } -} - -impl AnyStrContainer<str> for String { - fn new() -> Self { - String::new() - } - - fn with_capacity(capacity: usize) -> Self { - String::with_capacity(capacity) - } - - fn push_str(&mut self, other: &str) { - String::push_str(self, other) - } -} - -impl AnyStr for str { - type Char = char; - type Container = String; - type CharIter<'a> = std::str::Chars<'a>; - type ElementIter<'a> = std::str::Chars<'a>; - - fn element_bytes_len(c: char) -> usize { - c.len_utf8() - } - - fn to_container(&self) -> Self::Container { - self.to_owned() - } - - fn as_bytes(&self) -> &[u8] { - self.as_bytes() - } - - fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error> { - Ok(self) - } - - fn chars(&self) -> Self::CharIter<'_> { - str::chars(self) - } - - fn elements(&self) -> Self::ElementIter<'_> { - str::chars(self) - } - - fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self { - &self[range] - } - - fn get_chars(&self, range: std::ops::Range<usize>) -> &Self { - rustpython_common::str::get_chars(self, range) - } - - fn is_empty(&self) -> bool { - Self::is_empty(self) - } - - fn bytes_len(&self) -> usize { - Self::len(self) - } - - fn py_split_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> - where - F: Fn(&Self) -> PyObjectRef, - { - // CPython split_whitespace - let mut splits = Vec::new(); - let mut last_offset = 0; - let mut count = maxsplit; - for (offset, _) in self.match_indices(|c: char| c.is_ascii_whitespace() || c == '\x0b') { - if last_offset == offset { - last_offset += 1; - continue; - } - if count == 0 { - break; - } - splits.push(convert(&self[last_offset..offset])); - last_offset = offset + 1; - count -= 1; - } - if last_offset != self.len() { - splits.push(convert(&self[last_offset..])); - } - splits - } - - fn py_rsplit_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> - where - F: Fn(&Self) -> PyObjectRef, - { - // CPython rsplit_whitespace - let mut splits = Vec::new(); - let mut last_offset = self.len(); - let mut count = maxsplit; - for (offset, _) in self.rmatch_indices(|c: char| c.is_ascii_whitespace() || c == '\x0b') { - if last_offset == offset + 1 { - last_offset -= 1; - continue; - } - if count == 0 { - break; - } - splits.push(convert(&self[offset + 1..last_offset])); - last_offset = offset; - count -= 1; - } - if last_offset != 0 { - splits.push(convert(&self[..last_offset])); - } - splits - } -} - -/// The unique reference of interned PyStr -/// Always intended to be used as a static reference -pub type PyStrInterned = PyInterned<PyStr>; - -impl PyStrInterned { - #[inline] - pub fn to_exact(&'static self) -> PyRefExact<PyStr> { - unsafe { PyRefExact::new_unchecked(self.to_owned()) } - } -} - -impl std::fmt::Display for PyStrInterned { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.as_str(), f) - } -} - -impl AsRef<str> for PyStrInterned { - #[inline(always)] - fn as_ref(&self) -> &str { - self.as_str() - } -} diff --git a/vm/src/builtins/traceback.rs b/vm/src/builtins/traceback.rs deleted file mode 100644 index 6506e953631..00000000000 --- a/vm/src/builtins/traceback.rs +++ /dev/null @@ -1,85 +0,0 @@ -use rustpython_common::lock::PyMutex; - -use super::PyType; -use crate::{ - class::PyClassImpl, frame::FrameRef, source_code::LineNumber, Context, Py, PyPayload, PyRef, -}; - -#[pyclass(module = false, name = "traceback", traverse)] -#[derive(Debug)] -pub struct PyTraceback { - pub next: PyMutex<Option<PyTracebackRef>>, - pub frame: FrameRef, - #[pytraverse(skip)] - pub lasti: u32, - #[pytraverse(skip)] - pub lineno: LineNumber, -} - -pub type PyTracebackRef = PyRef<PyTraceback>; - -impl PyPayload for PyTraceback { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.traceback_type - } -} - -#[pyclass] -impl PyTraceback { - pub fn new(next: Option<PyRef<Self>>, frame: FrameRef, lasti: u32, lineno: LineNumber) -> Self { - PyTraceback { - next: PyMutex::new(next), - frame, - lasti, - lineno, - } - } - - #[pygetset] - fn tb_frame(&self) -> FrameRef { - self.frame.clone() - } - - #[pygetset] - fn tb_lasti(&self) -> u32 { - self.lasti - } - - #[pygetset] - fn tb_lineno(&self) -> usize { - self.lineno.to_usize() - } - - #[pygetset] - fn tb_next(&self) -> Option<PyRef<Self>> { - self.next.lock().as_ref().cloned() - } - - #[pygetset(setter)] - fn set_tb_next(&self, value: Option<PyRef<Self>>) { - *self.next.lock() = value; - } -} - -impl PyTracebackRef { - pub fn iter(&self) -> impl Iterator<Item = PyTracebackRef> { - std::iter::successors(Some(self.clone()), |tb| tb.next.lock().clone()) - } -} - -pub fn init(context: &Context) { - PyTraceback::extend_class(context, context.types.traceback_type); -} - -#[cfg(feature = "serde")] -impl serde::Serialize for PyTraceback { - fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { - use serde::ser::SerializeStruct; - - let mut struc = s.serialize_struct("PyTraceback", 3)?; - struc.serialize_field("name", self.frame.code.obj_name.as_str())?; - struc.serialize_field("lineno", &self.lineno.get())?; - struc.serialize_field("filename", self.frame.code.source_path.as_str())?; - struc.end() - } -} diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs deleted file mode 100644 index 63cd3e43359..00000000000 --- a/vm/src/builtins/tuple.rs +++ /dev/null @@ -1,560 +0,0 @@ -use super::{PositionIterInternal, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; -use crate::common::{hash::PyHash, lock::PyMutex}; -use crate::object::{Traverse, TraverseFn}; -use crate::{ - atomic_func, - class::PyClassImpl, - convert::{ToPyObject, TransmuteFromObject}, - function::{ArgSize, OptionalArg, PyArithmeticValue, PyComparisonValue}, - iter::PyExactSizeIterator, - protocol::{PyIterReturn, PyMappingMethods, PySequenceMethods}, - recursion::ReprGuard, - sequence::{OptionalRangeArgs, SequenceExt}, - sliceable::{SequenceIndex, SliceableSequenceOp}, - types::{ - AsMapping, AsSequence, Comparable, Constructor, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SelfIter, Unconstructible, - }, - utils::collection_repr, - vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, -}; -use once_cell::sync::Lazy; -use std::{fmt, marker::PhantomData}; - -#[pyclass(module = false, name = "tuple", traverse)] -pub struct PyTuple { - elements: Box<[PyObjectRef]>, -} - -impl fmt::Debug for PyTuple { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // TODO: implement more informational, non-recursive Debug formatter - f.write_str("tuple") - } -} - -impl PyPayload for PyTuple { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.tuple_type - } -} - -pub trait IntoPyTuple { - fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef; -} - -impl IntoPyTuple for () { - fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef { - vm.ctx.empty_tuple.clone() - } -} - -impl IntoPyTuple for Vec<PyObjectRef> { - fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef { - PyTuple::new_ref(self, &vm.ctx) - } -} - -macro_rules! impl_into_pyobj_tuple { - ($(($T:ident, $idx:tt)),+) => { - impl<$($T: ToPyObject),*> IntoPyTuple for ($($T,)*) { - fn into_pytuple(self, vm: &VirtualMachine) -> PyTupleRef { - PyTuple::new_ref(vec![$(self.$idx.to_pyobject(vm)),*], &vm.ctx) - } - } - - impl<$($T: ToPyObject),*> ToPyObject for ($($T,)*) { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - self.into_pytuple(vm).into() - } - } - }; -} - -impl_into_pyobj_tuple!((A, 0)); -impl_into_pyobj_tuple!((A, 0), (B, 1)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2), (D, 3)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4), (F, 5)); -impl_into_pyobj_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4), (F, 5), (G, 6)); - -impl PyTuple { - pub(crate) fn fast_getitem(&self, idx: usize) -> PyObjectRef { - self.elements[idx].clone() - } -} - -pub type PyTupleRef = PyRef<PyTuple>; - -impl Constructor for PyTuple { - type Args = OptionalArg<PyObjectRef>; - - fn py_new(cls: PyTypeRef, iterable: Self::Args, vm: &VirtualMachine) -> PyResult { - let elements = if let OptionalArg::Present(iterable) = iterable { - let iterable = if cls.is(vm.ctx.types.tuple_type) { - match iterable.downcast_exact::<Self>(vm) { - Ok(tuple) => return Ok(tuple.into_pyref().into()), - Err(iterable) => iterable, - } - } else { - iterable - }; - iterable.try_to_value(vm)? - } else { - vec![] - }; - // Return empty tuple only for exact tuple types if the iterable is empty. - if elements.is_empty() && cls.is(vm.ctx.types.tuple_type) { - Ok(vm.ctx.empty_tuple.clone().into()) - } else { - Self { - elements: elements.into_boxed_slice(), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } -} - -impl AsRef<[PyObjectRef]> for PyTuple { - fn as_ref(&self) -> &[PyObjectRef] { - self.as_slice() - } -} - -impl std::ops::Deref for PyTuple { - type Target = [PyObjectRef]; - fn deref(&self) -> &[PyObjectRef] { - self.as_slice() - } -} - -impl<'a> std::iter::IntoIterator for &'a PyTuple { - type Item = &'a PyObjectRef; - type IntoIter = std::slice::Iter<'a, PyObjectRef>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl<'a> std::iter::IntoIterator for &'a Py<PyTuple> { - type Item = &'a PyObjectRef; - type IntoIter = std::slice::Iter<'a, PyObjectRef>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -impl PyTuple { - pub fn new_ref(elements: Vec<PyObjectRef>, ctx: &Context) -> PyRef<Self> { - if elements.is_empty() { - ctx.empty_tuple.clone() - } else { - let elements = elements.into_boxed_slice(); - PyRef::new_ref(Self { elements }, ctx.types.tuple_type.to_owned(), None) - } - } - - /// Creating a new tuple with given boxed slice. - /// NOTE: for usual case, you probably want to use PyTuple::new_ref. - /// Calling this function implies trying micro optimization for non-zero-sized tuple. - pub fn new_unchecked(elements: Box<[PyObjectRef]>) -> Self { - Self { elements } - } - - pub fn as_slice(&self) -> &[PyObjectRef] { - &self.elements - } - - fn repeat(zelf: PyRef<Self>, value: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - Ok(if zelf.elements.is_empty() || value == 0 { - vm.ctx.empty_tuple.clone() - } else if value == 1 && zelf.class().is(vm.ctx.types.tuple_type) { - // Special case: when some `tuple` is multiplied by `1`, - // nothing really happens, we need to return an object itself - // with the same `id()` to be compatible with CPython. - // This only works for `tuple` itself, not its subclasses. - zelf - } else { - let v = zelf.elements.mul(vm, value)?; - let elements = v.into_boxed_slice(); - Self { elements }.into_ref(&vm.ctx) - }) - } -} - -#[pyclass( - flags(BASETYPE), - with( - AsMapping, - AsSequence, - Hashable, - Comparable, - Iterable, - Constructor, - Representable - ) -)] -impl PyTuple { - #[pymethod(magic)] - fn add( - zelf: PyRef<Self>, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyArithmeticValue<PyRef<Self>> { - let added = other.downcast::<Self>().map(|other| { - if other.elements.is_empty() && zelf.class().is(vm.ctx.types.tuple_type) { - zelf - } else if zelf.elements.is_empty() && other.class().is(vm.ctx.types.tuple_type) { - other - } else { - let elements = zelf - .iter() - .chain(other.as_slice()) - .cloned() - .collect::<Box<[_]>>(); - Self { elements }.into_ref(&vm.ctx) - } - }); - PyArithmeticValue::from_option(added.ok()) - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - !self.elements.is_empty() - } - - #[pymethod] - fn count(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - let mut count: usize = 0; - for element in self { - if vm.identical_or_equal(element, &needle)? { - count += 1; - } - } - Ok(count) - } - - #[pymethod(magic)] - #[inline] - pub fn len(&self) -> usize { - self.elements.len() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.elements.is_empty() - } - - #[pymethod(name = "__rmul__")] - #[pymethod(magic)] - fn mul(zelf: PyRef<Self>, value: ArgSize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - Self::repeat(zelf, value.into(), vm) - } - - fn _getitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - match SequenceIndex::try_from_borrowed_object(vm, needle, "tuple")? { - SequenceIndex::Int(i) => self.elements.getitem_by_index(vm, i), - SequenceIndex::Slice(slice) => self - .elements - .getitem_by_slice(vm, slice) - .map(|x| vm.ctx.new_tuple(x).into()), - } - } - - #[pymethod(magic)] - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self._getitem(&needle, vm) - } - - #[pymethod] - fn index( - &self, - needle: PyObjectRef, - range: OptionalRangeArgs, - vm: &VirtualMachine, - ) -> PyResult<usize> { - let (start, stop) = range.saturate(self.len(), vm)?; - for (index, element) in self.elements.iter().enumerate().take(stop).skip(start) { - if vm.identical_or_equal(element, &needle)? { - return Ok(index); - } - } - Err(vm.new_value_error("tuple.index(x): x not in tuple".to_owned())) - } - - fn _contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - for element in self.elements.iter() { - if vm.identical_or_equal(element, needle)? { - return Ok(true); - } - } - Ok(false) - } - - #[pymethod(magic)] - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self._contains(&needle, vm) - } - - #[pymethod(magic)] - fn getnewargs(zelf: PyRef<Self>, vm: &VirtualMachine) -> (PyTupleRef,) { - // the arguments to pass to tuple() is just one tuple - so we'll be doing tuple(tup), which - // should just return tup, or tuplesubclass(tup), which'll copy/validate (e.g. for a - // structseq) - let tup_arg = if zelf.class().is(vm.ctx.types.tuple_type) { - zelf - } else { - PyTuple::new_ref(zelf.elements.clone().into_vec(), &vm.ctx) - }; - (tup_arg,) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } -} - -impl AsMapping for PyTuple { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy<PyMappingMethods> = Lazy::new(|| PyMappingMethods { - length: atomic_func!(|mapping, _vm| Ok(PyTuple::mapping_downcast(mapping).len())), - subscript: atomic_func!( - |mapping, needle, vm| PyTuple::mapping_downcast(mapping)._getitem(needle, vm) - ), - ..PyMappingMethods::NOT_IMPLEMENTED - }); - &AS_MAPPING - } -} - -impl AsSequence for PyTuple { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyTuple::sequence_downcast(seq).len())), - concat: atomic_func!(|seq, other, vm| { - let zelf = PyTuple::sequence_downcast(seq); - match PyTuple::add(zelf.to_owned(), other.to_owned(), vm) { - PyArithmeticValue::Implemented(tuple) => Ok(tuple.into()), - PyArithmeticValue::NotImplemented => Err(vm.new_type_error(format!( - "can only concatenate tuple (not '{}') to tuple", - other.class().name() - ))), - } - }), - repeat: atomic_func!(|seq, n, vm| { - let zelf = PyTuple::sequence_downcast(seq); - PyTuple::repeat(zelf.to_owned(), n, vm).map(|x| x.into()) - }), - item: atomic_func!(|seq, i, vm| { - let zelf = PyTuple::sequence_downcast(seq); - zelf.elements.getitem_by_index(vm, i) - }), - contains: atomic_func!(|seq, needle, vm| { - let zelf = PyTuple::sequence_downcast(seq); - zelf._contains(needle, vm) - }), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -impl Hashable for PyTuple { - #[inline] - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - tuple_hash(zelf.as_slice(), vm) - } -} - -impl Comparable for PyTuple { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - if let Some(res) = op.identical_optimization(zelf, other) { - return Ok(res.into()); - } - let other = class_or_notimplemented!(Self, other); - zelf.iter() - .richcompare(other.iter(), op, vm) - .map(PyComparisonValue::Implemented) - } -} - -impl Iterable for PyTuple { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok(PyTupleIterator { - internal: PyMutex::new(PositionIterInternal::new(zelf, 0)), - } - .into_pyobject(vm)) - } -} - -impl Representable for PyTuple { - #[inline] - fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let s = if zelf.len() == 0 { - vm.ctx.intern_str("()").to_owned() - } else if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let s = if zelf.len() == 1 { - format!("({},)", zelf.elements[0].repr(vm)?) - } else { - collection_repr(None, "(", ")", zelf.elements.iter(), vm)? - }; - vm.ctx.new_str(s) - } else { - vm.ctx.intern_str("(...)").to_owned() - }; - Ok(s) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } -} - -#[pyclass(module = false, name = "tuple_iterator", traverse)] -#[derive(Debug)] -pub(crate) struct PyTupleIterator { - internal: PyMutex<PositionIterInternal<PyTupleRef>>, -} - -impl PyPayload for PyTupleIterator { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.tuple_iterator_type - } -} - -#[pyclass(with(Constructor, IterNext, Iterable))] -impl PyTupleIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().length_hint(|obj| obj.len()) - } - - #[pymethod(magic)] - fn setstate(&self, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.internal - .lock() - .set_state(state, |obj, pos| pos.min(obj.len()), vm) - } - - #[pymethod(magic)] - fn reduce(&self, vm: &VirtualMachine) -> PyTupleRef { - self.internal - .lock() - .builtins_iter_reduce(|x| x.clone().into(), vm) - } -} -impl Unconstructible for PyTupleIterator {} - -impl SelfIter for PyTupleIterator {} -impl IterNext for PyTupleIterator { - fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.internal.lock().next(|tuple, pos| { - Ok(PyIterReturn::from_result( - tuple.get(pos).cloned().ok_or(None), - )) - }) - } -} - -pub(crate) fn init(context: &Context) { - PyTuple::extend_class(context, context.types.tuple_type); - PyTupleIterator::extend_class(context, context.types.tuple_iterator_type); -} - -pub struct PyTupleTyped<T: TransmuteFromObject> { - // SAFETY INVARIANT: T must be repr(transparent) over PyObjectRef, and the - // elements must be logically valid when transmuted to T - tuple: PyTupleRef, - _marker: PhantomData<Vec<T>>, -} - -unsafe impl<T> Traverse for PyTupleTyped<T> -where - T: TransmuteFromObject + Traverse, -{ - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.tuple.traverse(tracer_fn); - } -} - -impl<T: TransmuteFromObject> TryFromObject for PyTupleTyped<T> { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let tuple = PyTupleRef::try_from_object(vm, obj)?; - for elem in &*tuple { - T::check(vm, elem)? - } - // SAFETY: the contract of TransmuteFromObject upholds the variant on `tuple` - Ok(Self { - tuple, - _marker: PhantomData, - }) - } -} - -impl<T: TransmuteFromObject> AsRef<[T]> for PyTupleTyped<T> { - fn as_ref(&self) -> &[T] { - self.as_slice() - } -} - -impl<T: TransmuteFromObject> PyTupleTyped<T> { - #[inline] - pub fn as_slice(&self) -> &[T] { - unsafe { &*(self.tuple.as_slice() as *const [PyObjectRef] as *const [T]) } - } - #[inline] - pub fn len(&self) -> usize { - self.tuple.len() - } - #[inline] - pub fn is_empty(&self) -> bool { - self.tuple.is_empty() - } -} - -impl<T: TransmuteFromObject> Clone for PyTupleTyped<T> { - fn clone(&self) -> Self { - Self { - tuple: self.tuple.clone(), - _marker: PhantomData, - } - } -} - -impl<T: TransmuteFromObject + fmt::Debug> fmt::Debug for PyTupleTyped<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.as_slice().fmt(f) - } -} - -impl<T: TransmuteFromObject> From<PyTupleTyped<T>> for PyTupleRef { - #[inline] - fn from(tup: PyTupleTyped<T>) -> Self { - tup.tuple - } -} - -impl<T: TransmuteFromObject> ToPyObject for PyTupleTyped<T> { - #[inline] - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.tuple.into() - } -} - -pub(super) fn tuple_hash(elements: &[PyObjectRef], vm: &VirtualMachine) -> PyResult<PyHash> { - // TODO: See #3460 for the correct implementation. - // https://github.com/RustPython/RustPython/pull/3460 - crate::utils::hash_iter(elements.iter(), vm) -} diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs deleted file mode 100644 index 6e1ddebb252..00000000000 --- a/vm/src/builtins/type.rs +++ /dev/null @@ -1,1394 +0,0 @@ -use super::{ - mappingproxy::PyMappingProxy, object, union_, PyClassMethod, PyDictRef, PyList, PyStr, - PyStrInterned, PyStrRef, PyTuple, PyTupleRef, PyWeak, -}; -use crate::{ - builtins::{ - descriptor::{ - MemberGetter, MemberKind, MemberSetter, PyDescriptorOwned, PyMemberDef, - PyMemberDescriptor, - }, - function::PyCellRef, - tuple::{IntoPyTuple, PyTupleTyped}, - PyBaseExceptionRef, - }, - class::{PyClassImpl, StaticType}, - common::{ - ascii, - borrow::BorrowedValue, - lock::{PyRwLock, PyRwLockReadGuard}, - }, - convert::ToPyResult, - function::{FuncArgs, KwArgs, OptionalArg, PyMethodDef, PySetterValue}, - identifier, - object::{Traverse, TraverseFn}, - protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, - types::{AsNumber, Callable, GetAttr, PyTypeFlags, PyTypeSlots, Representable, SetAttr}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, -}; -use indexmap::{map::Entry, IndexMap}; -use itertools::Itertools; -use std::{borrow::Borrow, collections::HashSet, fmt, ops::Deref, pin::Pin, ptr::NonNull}; - -#[pyclass(module = false, name = "type", traverse = "manual")] -pub struct PyType { - pub base: Option<PyTypeRef>, - pub bases: Vec<PyTypeRef>, - pub mro: Vec<PyTypeRef>, - pub subclasses: PyRwLock<Vec<PyRef<PyWeak>>>, - pub attributes: PyRwLock<PyAttributes>, - pub slots: PyTypeSlots, - pub heaptype_ext: Option<Pin<Box<HeapTypeExt>>>, -} - -unsafe impl crate::object::Traverse for PyType { - fn traverse(&self, tracer_fn: &mut crate::object::TraverseFn) { - self.base.traverse(tracer_fn); - self.bases.traverse(tracer_fn); - self.mro.traverse(tracer_fn); - self.subclasses.traverse(tracer_fn); - self.attributes - .read_recursive() - .iter() - .map(|(_, v)| v.traverse(tracer_fn)) - .count(); - } -} - -pub struct HeapTypeExt { - pub name: PyRwLock<PyStrRef>, - pub slots: Option<PyTupleTyped<PyStrRef>>, - pub sequence_methods: PySequenceMethods, - pub mapping_methods: PyMappingMethods, -} - -pub struct PointerSlot<T>(NonNull<T>); - -impl<T> PointerSlot<T> { - pub unsafe fn borrow_static(&self) -> &'static T { - self.0.as_ref() - } -} - -impl<T> Clone for PointerSlot<T> { - fn clone(&self) -> Self { - *self - } -} - -impl<T> Copy for PointerSlot<T> {} - -impl<T> From<&'static T> for PointerSlot<T> { - fn from(x: &'static T) -> Self { - Self(NonNull::from(x)) - } -} - -impl<T> AsRef<T> for PointerSlot<T> { - fn as_ref(&self) -> &T { - unsafe { self.0.as_ref() } - } -} - -impl<T> PointerSlot<T> { - pub unsafe fn from_heaptype<F>(typ: &PyType, f: F) -> Option<Self> - where - F: FnOnce(&HeapTypeExt) -> &T, - { - typ.heaptype_ext - .as_ref() - .map(|ext| Self(NonNull::from(f(ext)))) - } -} - -pub type PyTypeRef = PyRef<PyType>; - -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - unsafe impl Send for PyType {} - unsafe impl Sync for PyType {} - } -} - -/// For attributes we do not use a dict, but an IndexMap, which is an Hash Table -/// that maintains order and is compatible with the standard HashMap This is probably -/// faster and only supports strings as keys. -pub type PyAttributes = IndexMap<&'static PyStrInterned, PyObjectRef, ahash::RandomState>; - -unsafe impl Traverse for PyAttributes { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.values().for_each(|v| v.traverse(tracer_fn)); - } -} - -impl fmt::Display for PyType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.name(), f) - } -} - -impl fmt::Debug for PyType { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "[PyType {}]", &self.name()) - } -} - -impl PyPayload for PyType { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.type_type - } -} - -impl PyType { - pub fn new_simple_heap( - name: &str, - base: &PyTypeRef, - ctx: &Context, - ) -> Result<PyRef<Self>, String> { - Self::new_heap( - name, - vec![base.clone()], - Default::default(), - Default::default(), - Self::static_type().to_owned(), - ctx, - ) - } - pub fn new_heap( - name: &str, - bases: Vec<PyRef<Self>>, - attrs: PyAttributes, - slots: PyTypeSlots, - metaclass: PyRef<Self>, - ctx: &Context, - ) -> Result<PyRef<Self>, String> { - // TODO: ensure clean slot name - // assert_eq!(slots.name.borrow(), ""); - - let name = ctx.new_str(name); - let heaptype_ext = HeapTypeExt { - name: PyRwLock::new(name), - slots: None, - sequence_methods: PySequenceMethods::default(), - mapping_methods: PyMappingMethods::default(), - }; - let base = bases[0].clone(); - - Self::new_heap_inner(base, bases, attrs, slots, heaptype_ext, metaclass, ctx) - } - - #[allow(clippy::too_many_arguments)] - fn new_heap_inner( - base: PyRef<Self>, - bases: Vec<PyRef<Self>>, - attrs: PyAttributes, - mut slots: PyTypeSlots, - heaptype_ext: HeapTypeExt, - metaclass: PyRef<Self>, - ctx: &Context, - ) -> Result<PyRef<Self>, String> { - // Check for duplicates in bases. - let mut unique_bases = HashSet::new(); - for base in &bases { - if !unique_bases.insert(base.get_id()) { - return Err(format!("duplicate base class {}", base.name())); - } - } - - let mros = bases - .iter() - .map(|x| x.iter_mro().map(|x| x.to_owned()).collect()) - .collect(); - let mro = linearise_mro(mros)?; - - if base.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { - slots.flags |= PyTypeFlags::HAS_DICT - } - - let new_type = PyRef::new_ref( - PyType { - base: Some(base), - bases, - mro, - subclasses: PyRwLock::default(), - attributes: PyRwLock::new(attrs), - slots, - heaptype_ext: Some(Pin::new(Box::new(heaptype_ext))), - }, - metaclass, - None, - ); - - new_type.init_slots(ctx); - - let weakref_type = super::PyWeak::static_type(); - for base in &new_type.bases { - base.subclasses.write().push( - new_type - .as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) - .unwrap(), - ); - } - - Ok(new_type) - } - - pub fn new_static( - base: PyRef<Self>, - attrs: PyAttributes, - mut slots: PyTypeSlots, - metaclass: PyRef<Self>, - ) -> Result<PyRef<Self>, String> { - if base.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { - slots.flags |= PyTypeFlags::HAS_DICT - } - - let bases = vec![base.clone()]; - let mro = base.iter_mro().map(|x| x.to_owned()).collect(); - - let new_type = PyRef::new_ref( - PyType { - base: Some(base), - bases, - mro, - subclasses: PyRwLock::default(), - attributes: PyRwLock::new(attrs), - slots, - heaptype_ext: None, - }, - metaclass, - None, - ); - - let weakref_type = super::PyWeak::static_type(); - for base in &new_type.bases { - base.subclasses.write().push( - new_type - .as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.to_owned()) - .unwrap(), - ); - } - - Ok(new_type) - } - - pub(crate) fn init_slots(&self, ctx: &Context) { - #[allow(clippy::mutable_key_type)] - let mut slot_name_set = std::collections::HashSet::new(); - - for cls in self.mro.iter() { - for &name in cls.attributes.read().keys() { - if name == identifier!(ctx, __new__) { - continue; - } - if name.as_str().starts_with("__") && name.as_str().ends_with("__") { - slot_name_set.insert(name); - } - } - } - for &name in self.attributes.read().keys() { - if name.as_str().starts_with("__") && name.as_str().ends_with("__") { - slot_name_set.insert(name); - } - } - for attr_name in slot_name_set { - self.update_slot::<true>(attr_name, ctx); - } - } - - pub fn iter_mro(&self) -> impl DoubleEndedIterator<Item = &PyType> { - std::iter::once(self).chain(self.mro.iter().map(|cls| -> &PyType { cls })) - } - - pub(crate) fn mro_find_map<F, R>(&self, f: F) -> Option<R> - where - F: Fn(&Self) -> Option<R>, - { - // the hot path will be primitive types which usually hit the result from itself. - // try std::intrinsics::likely once it is stablized - if let Some(r) = f(self) { - Some(r) - } else { - self.mro.iter().find_map(|cls| f(cls)) - } - } - - // This is used for class initialisation where the vm is not yet available. - pub fn set_str_attr<V: Into<PyObjectRef>>( - &self, - attr_name: &str, - value: V, - ctx: impl AsRef<Context>, - ) { - let ctx = ctx.as_ref(); - let attr_name = ctx.intern_str(attr_name); - self.set_attr(attr_name, value.into()) - } - - pub fn set_attr(&self, attr_name: &'static PyStrInterned, value: PyObjectRef) { - self.attributes.write().insert(attr_name, value); - } - - /// This is the internal get_attr implementation for fast lookup on a class. - pub fn get_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { - flame_guard!(format!("class_get_attr({:?})", attr_name)); - - self.get_direct_attr(attr_name) - .or_else(|| self.get_super_attr(attr_name)) - } - - pub fn get_direct_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { - self.attributes.read().get(attr_name).cloned() - } - - pub fn get_super_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { - self.mro - .iter() - .find_map(|class| class.attributes.read().get(attr_name).cloned()) - } - - // This is the internal has_attr implementation for fast lookup on a class. - pub fn has_attr(&self, attr_name: &'static PyStrInterned) -> bool { - self.attributes.read().contains_key(attr_name) - || self - .mro - .iter() - .any(|c| c.attributes.read().contains_key(attr_name)) - } - - pub fn get_attributes(&self) -> PyAttributes { - // Gather all members here: - let mut attributes = PyAttributes::default(); - - for bc in self.iter_mro().rev() { - for (name, value) in bc.attributes.read().iter() { - attributes.insert(name.to_owned(), value.clone()); - } - } - - attributes - } - - // bound method for every type - pub(crate) fn __new__(zelf: PyRef<PyType>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let (subtype, args): (PyRef<Self>, FuncArgs) = args.bind(vm)?; - if !subtype.fast_issubclass(&zelf) { - return Err(vm.new_type_error(format!( - "{zelf}.__new__({subtype}): {subtype} is not a subtype of {zelf}", - zelf = zelf.name(), - subtype = subtype.name(), - ))); - } - call_slot_new(zelf, subtype, args, vm) - } - - fn name_inner<'a, R: 'a>( - &'a self, - static_f: impl FnOnce(&'static str) -> R, - heap_f: impl FnOnce(&'a HeapTypeExt) -> R, - ) -> R { - if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - static_f(self.slots.name) - } else { - heap_f(self.heaptype_ext.as_ref().unwrap()) - } - } - - pub fn slot_name(&self) -> BorrowedValue<str> { - self.name_inner( - |name| name.into(), - |ext| PyRwLockReadGuard::map(ext.name.read(), |name| name.as_str()).into(), - ) - } - - pub fn name(&self) -> BorrowedValue<str> { - self.name_inner( - |name| name.rsplit_once('.').map_or(name, |(_, name)| name).into(), - |ext| PyRwLockReadGuard::map(ext.name.read(), |name| name.as_str()).into(), - ) - } -} - -impl Py<PyType> { - /// Determines if `subclass` is actually a subclass of `cls`, this doesn't call __subclasscheck__, - /// so only use this if `cls` is known to have not overridden the base __subclasscheck__ magic - /// method. - pub fn fast_issubclass(&self, cls: &impl Borrow<crate::PyObject>) -> bool { - self.as_object().is(cls.borrow()) || self.mro.iter().any(|c| c.is(cls.borrow())) - } - - pub fn iter_mro(&self) -> impl DoubleEndedIterator<Item = &Py<PyType>> { - std::iter::once(self).chain(self.mro.iter().map(|x| x.deref())) - } - - pub fn iter_base_chain(&self) -> impl Iterator<Item = &Py<PyType>> { - std::iter::successors(Some(self), |cls| cls.base.as_deref()) - } - - pub fn extend_methods(&'static self, method_defs: &'static [PyMethodDef], ctx: &Context) { - for method_def in method_defs { - let method = method_def.to_proper_method(self, ctx); - self.set_attr(ctx.intern_str(method_def.name), method); - } - } -} - -#[pyclass( - with(Py, GetAttr, SetAttr, Callable, AsNumber, Representable), - flags(BASETYPE) -)] -impl PyType { - #[pygetset(magic)] - fn bases(&self, vm: &VirtualMachine) -> PyTupleRef { - vm.ctx.new_tuple( - self.bases - .iter() - .map(|x| x.as_object().to_owned()) - .collect(), - ) - } - - #[pygetset(magic)] - fn base(&self) -> Option<PyTypeRef> { - self.base.clone() - } - - #[pygetset(magic)] - fn flags(&self) -> u64 { - self.slots.flags.bits() - } - - #[pygetset] - pub fn __name__(&self, vm: &VirtualMachine) -> PyStrRef { - self.name_inner( - |name| { - vm.ctx - .interned_str(name.rsplit_once('.').map_or(name, |(_, name)| name)) - .unwrap_or_else(|| { - panic!( - "static type name must be already interned but {} is not", - self.slot_name() - ) - }) - .to_owned() - }, - |ext| ext.name.read().clone(), - ) - } - - #[pygetset(magic)] - pub fn qualname(&self, vm: &VirtualMachine) -> PyObjectRef { - self.attributes - .read() - .get(identifier!(vm, __qualname__)) - .cloned() - // We need to exclude this method from going into recursion: - .and_then(|found| { - if found.fast_isinstance(vm.ctx.types.getset_type) { - None - } else { - Some(found) - } - }) - .unwrap_or_else(|| vm.ctx.new_str(self.name().deref()).into()) - } - - #[pygetset(magic, setter)] - fn set_qualname(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - // TODO: we should replace heaptype flag check to immutable flag check - if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - return Err(vm.new_type_error(format!( - "cannot set '__qualname__' attribute of immutable type '{}'", - self.name() - ))); - }; - let value = value.ok_or_else(|| { - vm.new_type_error(format!( - "cannot delete '__qualname__' attribute of immutable type '{}'", - self.name() - )) - })?; - if !value.class().fast_issubclass(vm.ctx.types.str_type) { - return Err(vm.new_type_error(format!( - "can only assign string to {}.__qualname__, not '{}'", - self.name(), - value.class().name() - ))); - } - self.attributes - .write() - .insert(identifier!(vm, __qualname__), value); - Ok(()) - } - - #[pygetset(magic)] - fn annotations(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { - return Err(vm.new_attribute_error(format!( - "type object '{}' has no attribute '__annotations__'", - self.name() - ))); - } - - let __annotations__ = identifier!(vm, __annotations__); - let annotations = self.attributes.read().get(__annotations__).cloned(); - - let annotations = if let Some(annotations) = annotations { - annotations - } else { - let annotations: PyObjectRef = vm.ctx.new_dict().into(); - let removed = self - .attributes - .write() - .insert(__annotations__, annotations.clone()); - debug_assert!(removed.is_none()); - annotations - }; - Ok(annotations) - } - - #[pygetset(magic, setter)] - fn set_annotations(&self, value: Option<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { - if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { - return Err(vm.new_type_error(format!( - "cannot set '__annotations__' attribute of immutable type '{}'", - self.name() - ))); - } - - let __annotations__ = identifier!(vm, __annotations__); - if let Some(value) = value { - self.attributes.write().insert(__annotations__, value); - } else { - self.attributes - .read() - .get(__annotations__) - .cloned() - .ok_or_else(|| { - vm.new_attribute_error(format!( - "'{}' object has no attribute '__annotations__'", - self.name() - )) - })?; - } - - Ok(()) - } - - #[pygetset(magic)] - pub fn module(&self, vm: &VirtualMachine) -> PyObjectRef { - self.attributes - .read() - .get(identifier!(vm, __module__)) - .cloned() - // We need to exclude this method from going into recursion: - .and_then(|found| { - if found.fast_isinstance(vm.ctx.types.getset_type) { - None - } else { - Some(found) - } - }) - .unwrap_or_else(|| vm.ctx.new_str(ascii!("builtins")).into()) - } - - #[pygetset(magic, setter)] - fn set_module(&self, value: PyObjectRef, vm: &VirtualMachine) { - self.attributes - .write() - .insert(identifier!(vm, __module__), value); - } - - #[pyclassmethod(magic)] - fn prepare( - _cls: PyTypeRef, - _name: OptionalArg<PyObjectRef>, - _bases: OptionalArg<PyObjectRef>, - _kwargs: KwArgs, - vm: &VirtualMachine, - ) -> PyDictRef { - vm.ctx.new_dict() - } - - #[pymethod(magic)] - fn subclasses(&self) -> PyList { - let mut subclasses = self.subclasses.write(); - subclasses.retain(|x| x.upgrade().is_some()); - PyList::from( - subclasses - .iter() - .map(|x| x.upgrade().unwrap()) - .collect::<Vec<_>>(), - ) - } - - #[pymethod(magic)] - pub fn ror(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - or_(other, zelf, vm) - } - - #[pymethod(magic)] - pub fn or(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - or_(zelf, other, vm) - } - - #[pyslot] - fn slot_new(metatype: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - vm_trace!("type.__new__ {:?}", args); - - let is_type_type = metatype.is(vm.ctx.types.type_type); - if is_type_type && args.args.len() == 1 && args.kwargs.is_empty() { - return Ok(args.args[0].class().to_owned().into()); - } - - if args.args.len() != 3 { - return Err(vm.new_type_error(if is_type_type { - "type() takes 1 or 3 arguments".to_owned() - } else { - format!( - "type.__new__() takes exactly 3 arguments ({} given)", - args.args.len() - ) - })); - } - - let (name, bases, dict, kwargs): (PyStrRef, PyTupleRef, PyDictRef, KwArgs) = - args.clone().bind(vm)?; - - if name.as_str().as_bytes().contains(&0) { - return Err(vm.new_value_error("type name must not contain null characters".to_owned())); - } - - let (metatype, base, bases) = if bases.is_empty() { - let base = vm.ctx.types.object_type.to_owned(); - (metatype, base.clone(), vec![base]) - } else { - let bases = bases - .iter() - .map(|obj| { - obj.clone().downcast::<PyType>().or_else(|obj| { - if vm - .get_attribute_opt(obj, identifier!(vm, __mro_entries__))? - .is_some() - { - Err(vm.new_type_error( - "type() doesn't support MRO entry resolution; \ - use types.new_class()" - .to_owned(), - )) - } else { - Err(vm.new_type_error("bases must be types".to_owned())) - } - }) - }) - .collect::<PyResult<Vec<_>>>()?; - - // Search the bases for the proper metatype to deal with this: - let winner = calculate_meta_class(metatype.clone(), &bases, vm)?; - let metatype = if !winner.is(&metatype) { - if let Some(ref slot_new) = winner.slots.new.load() { - // Pass it to the winner - return slot_new(winner, args, vm); - } - winner - } else { - metatype - }; - - let base = best_base(&bases, vm)?; - - (metatype, base, bases) - }; - - let mut attributes = dict.to_attributes(vm); - - if let Some(f) = attributes.get_mut(identifier!(vm, __init_subclass__)) { - if f.class().is(vm.ctx.types.function_type) { - *f = PyClassMethod::from(f.clone()).into_pyobject(vm); - } - } - - if let Some(f) = attributes.get_mut(identifier!(vm, __class_getitem__)) { - if f.class().is(vm.ctx.types.function_type) { - *f = PyClassMethod::from(f.clone()).into_pyobject(vm); - } - } - - if let Some(current_frame) = vm.current_frame() { - let entry = attributes.entry(identifier!(vm, __module__)); - if matches!(entry, Entry::Vacant(_)) { - let module_name = vm.unwrap_or_none( - current_frame - .globals - .get_item_opt(identifier!(vm, __name__), vm)?, - ); - entry.or_insert(module_name); - } - } - - attributes - .entry(identifier!(vm, __qualname__)) - .or_insert_with(|| vm.ctx.new_str(name.as_str()).into()); - - // All *classes* should have a dict. Exceptions are *instances* of - // classes that define __slots__ and instances of built-in classes - // (with exceptions, e.g function) - let __dict__ = identifier!(vm, __dict__); - attributes.entry(__dict__).or_insert_with(|| { - vm.ctx - .new_getset( - "__dict__", - vm.ctx.types.object_type, - subtype_get_dict, - subtype_set_dict, - ) - .into() - }); - - // TODO: Flags is currently initialized with HAS_DICT. Should be - // updated when __slots__ are supported (toggling the flag off if - // a class has __slots__ defined). - let heaptype_slots: Option<PyTupleTyped<PyStrRef>> = - if let Some(x) = attributes.get(identifier!(vm, __slots__)) { - Some(if x.to_owned().class().is(vm.ctx.types.str_type) { - PyTupleTyped::<PyStrRef>::try_from_object( - vm, - vec![x.to_owned()].into_pytuple(vm).into(), - )? - } else { - let iter = x.to_owned().get_iter(vm)?; - let elements = { - let mut elements = Vec::new(); - while let PyIterReturn::Return(element) = iter.next(vm)? { - elements.push(element); - } - elements - }; - PyTupleTyped::<PyStrRef>::try_from_object(vm, elements.into_pytuple(vm).into())? - }) - } else { - None - }; - - let base_member_count = base.slots.member_count; - let member_count: usize = - base.slots.member_count + heaptype_slots.as_ref().map(|x| x.len()).unwrap_or(0); - - let flags = PyTypeFlags::heap_type_flags() | PyTypeFlags::HAS_DICT; - let (slots, heaptype_ext) = { - let slots = PyTypeSlots { - member_count, - flags, - ..PyTypeSlots::heap_default() - }; - let heaptype_ext = HeapTypeExt { - name: PyRwLock::new(name), - slots: heaptype_slots.to_owned(), - sequence_methods: PySequenceMethods::default(), - mapping_methods: PyMappingMethods::default(), - }; - (slots, heaptype_ext) - }; - - let typ = Self::new_heap_inner( - base, - bases, - attributes, - slots, - heaptype_ext, - metatype, - &vm.ctx, - ) - .map_err(|e| vm.new_type_error(e))?; - - if let Some(ref slots) = heaptype_slots { - let mut offset = base_member_count; - for member in slots.as_slice() { - let member_def = PyMemberDef { - name: member.to_string(), - kind: MemberKind::ObjectEx, - getter: MemberGetter::Offset(offset), - setter: MemberSetter::Offset(offset), - doc: None, - }; - let member_descriptor: PyRef<PyMemberDescriptor> = - vm.ctx.new_pyref(PyMemberDescriptor { - common: PyDescriptorOwned { - typ: typ.clone(), - name: vm.ctx.intern_str(member.as_str()), - qualname: PyRwLock::new(None), - }, - member: member_def, - }); - - let attr_name = vm.ctx.intern_str(member.to_string()); - if !typ.has_attr(attr_name) { - typ.set_attr(attr_name, member_descriptor.into()); - } - - offset += 1; - } - } - - if let Some(cell) = typ.attributes.write().get(identifier!(vm, __classcell__)) { - let cell = PyCellRef::try_from_object(vm, cell.clone()).map_err(|_| { - vm.new_type_error(format!( - "__classcell__ must be a nonlocal cell, not {}", - cell.class().name() - )) - })?; - cell.set(Some(typ.clone().into())); - }; - - // avoid deadlock - let attributes = typ - .attributes - .read() - .iter() - .filter_map(|(name, obj)| { - vm.get_method(obj.clone(), identifier!(vm, __set_name__)) - .map(|res| res.map(|meth| (obj.clone(), name.to_owned(), meth))) - }) - .collect::<PyResult<Vec<_>>>()?; - for (obj, name, set_name) in attributes { - set_name.call((typ.clone(), name), vm).map_err(|e| { - let err = vm.new_runtime_error(format!( - "Error calling __set_name__ on '{}' instance {} in '{}'", - obj.class().name(), - name, - typ.name() - )); - err.set_cause(Some(e)); - err - })?; - } - - if let Some(init_subclass) = typ.get_super_attr(identifier!(vm, __init_subclass__)) { - let init_subclass = vm - .call_get_descriptor_specific(&init_subclass, None, Some(typ.clone().into())) - .unwrap_or(Ok(init_subclass))?; - init_subclass.call(kwargs, vm)?; - }; - - Ok(typ.into()) - } - - #[pygetset(magic)] - fn dict(zelf: PyRef<Self>) -> PyMappingProxy { - PyMappingProxy::from(zelf) - } - - #[pygetset(magic, setter)] - fn set_dict(&self, _value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - Err(vm.new_not_implemented_error( - "Setting __dict__ attribute on a type isn't yet implemented".to_owned(), - )) - } - - fn check_set_special_type_attr( - &self, - _value: &PyObject, - name: &PyStrInterned, - vm: &VirtualMachine, - ) -> PyResult<()> { - if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { - return Err(vm.new_type_error(format!( - "cannot set '{}' attribute of immutable type '{}'", - name, - self.slot_name() - ))); - } - Ok(()) - } - - #[pygetset(magic, setter)] - fn set_name(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.check_set_special_type_attr(&value, identifier!(vm, __name__), vm)?; - let name = value.downcast::<PyStr>().map_err(|value| { - vm.new_type_error(format!( - "can only assign string to {}.__name__, not '{}'", - self.slot_name(), - value.class().slot_name(), - )) - })?; - if name.as_str().as_bytes().contains(&0) { - return Err(vm.new_value_error("type name must not contain null characters".to_owned())); - } - - *self.heaptype_ext.as_ref().unwrap().name.write() = name; - - Ok(()) - } - - #[pygetset(magic)] - fn text_signature(&self) -> Option<String> { - self.slots - .doc - .and_then(|doc| get_text_signature_from_internal_doc(&self.name(), doc)) - .map(|signature| signature.to_string()) - } -} - -#[pyclass] -impl Py<PyType> { - #[pygetset(name = "__mro__")] - fn get_mro(&self) -> PyTuple { - let elements: Vec<PyObjectRef> = - self.iter_mro().map(|x| x.as_object().to_owned()).collect(); - PyTuple::new_unchecked(elements.into_boxed_slice()) - } - - #[pymethod(magic)] - fn dir(&self) -> PyList { - let attributes: Vec<PyObjectRef> = self - .get_attributes() - .into_iter() - .map(|(k, _)| k.to_object()) - .collect(); - PyList::from(attributes) - } - - #[pymethod(magic)] - fn instancecheck(&self, obj: PyObjectRef) -> bool { - obj.fast_isinstance(self) - } - - #[pymethod(magic)] - fn subclasscheck(&self, subclass: PyTypeRef) -> bool { - subclass.fast_issubclass(self) - } - - #[pyclassmethod(magic)] - fn subclasshook(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented() - } - - #[pymethod] - fn mro(&self) -> Vec<PyObjectRef> { - self.iter_mro().map(|cls| cls.to_owned().into()).collect() - } -} - -const SIGNATURE_END_MARKER: &str = ")\n--\n\n"; -fn get_signature(doc: &str) -> Option<&str> { - doc.find(SIGNATURE_END_MARKER).map(|index| &doc[..=index]) -} - -fn find_signature<'a>(name: &str, doc: &'a str) -> Option<&'a str> { - let name = name.rsplit('.').next().unwrap(); - let doc = doc.strip_prefix(name)?; - doc.starts_with('(').then_some(doc) -} - -pub(crate) fn get_text_signature_from_internal_doc<'a>( - name: &str, - internal_doc: &'a str, -) -> Option<&'a str> { - find_signature(name, internal_doc).and_then(get_signature) -} - -impl GetAttr for PyType { - fn getattro(zelf: &Py<Self>, name_str: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - #[cold] - fn attribute_error( - zelf: &Py<PyType>, - name: &str, - vm: &VirtualMachine, - ) -> PyBaseExceptionRef { - vm.new_attribute_error(format!( - "type object '{}' has no attribute '{}'", - zelf.slot_name(), - name, - )) - } - - let Some(name) = vm.ctx.interned_str(name_str) else { - return Err(attribute_error(zelf, name_str.as_str(), vm)); - }; - vm_trace!("type.__getattribute__({:?}, {:?})", zelf, name); - let mcl = zelf.class(); - let mcl_attr = mcl.get_attr(name); - - if let Some(ref attr) = mcl_attr { - let attr_class = attr.class(); - let has_descr_set = attr_class - .mro_find_map(|cls| cls.slots.descr_set.load()) - .is_some(); - if has_descr_set { - let descr_get = attr_class.mro_find_map(|cls| cls.slots.descr_get.load()); - if let Some(descr_get) = descr_get { - let mcl = mcl.to_owned().into(); - return descr_get(attr.clone(), Some(zelf.to_owned().into()), Some(mcl), vm); - } - } - } - - let zelf_attr = zelf.get_attr(name); - - if let Some(attr) = zelf_attr { - let descr_get = attr.class().mro_find_map(|cls| cls.slots.descr_get.load()); - if let Some(descr_get) = descr_get { - descr_get(attr, None, Some(zelf.to_owned().into()), vm) - } else { - Ok(attr) - } - } else if let Some(attr) = mcl_attr { - vm.call_if_get_descriptor(&attr, zelf.to_owned().into()) - } else { - Err(attribute_error(zelf, name_str.as_str(), vm)) - } - } -} - -impl SetAttr for PyType { - fn setattro( - zelf: &Py<Self>, - attr_name: &Py<PyStr>, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()> { - // TODO: pass PyRefExact instead of &str - let attr_name = vm.ctx.intern_str(attr_name.as_str()); - if let Some(attr) = zelf.get_class_attr(attr_name) { - let descr_set = attr.class().mro_find_map(|cls| cls.slots.descr_set.load()); - if let Some(descriptor) = descr_set { - return descriptor(&attr, zelf.to_owned().into(), value, vm); - } - } - let assign = value.is_assign(); - - if let PySetterValue::Assign(value) = value { - zelf.attributes.write().insert(attr_name, value); - } else { - let prev_value = zelf.attributes.write().remove(attr_name); - if prev_value.is_none() { - return Err(vm.new_exception( - vm.ctx.exceptions.attribute_error.to_owned(), - vec![attr_name.to_object()], - )); - } - } - if attr_name.as_str().starts_with("__") && attr_name.as_str().ends_with("__") { - if assign { - zelf.update_slot::<true>(attr_name, &vm.ctx); - } else { - zelf.update_slot::<false>(attr_name, &vm.ctx); - } - } - Ok(()) - } -} - -impl Callable for PyType { - type Args = FuncArgs; - fn call(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - vm_trace!("type_call: {:?}", zelf); - let obj = call_slot_new(zelf.to_owned(), zelf.to_owned(), args.clone(), vm)?; - - if (zelf.is(vm.ctx.types.type_type) && args.kwargs.is_empty()) || !obj.fast_isinstance(zelf) - { - return Ok(obj); - } - - let init = obj.class().mro_find_map(|cls| cls.slots.init.load()); - if let Some(init_method) = init { - init_method(obj.clone(), args, vm)?; - } - Ok(obj) - } -} - -impl AsNumber for PyType { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| or_(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl Representable for PyType { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let module = zelf.module(vm); - let module = module.downcast_ref::<PyStr>().map(|m| m.as_str()); - - let repr = match module { - Some(module) if module != "builtins" => { - let name = zelf.name(); - format!( - "<class '{}.{}'>", - module, - zelf.qualname(vm) - .downcast_ref::<PyStr>() - .map(|n| n.as_str()) - .unwrap_or_else(|| &name) - ) - } - _ => format!("<class '{}'>", zelf.slot_name()), - }; - Ok(repr) - } -} - -fn find_base_dict_descr(cls: &Py<PyType>, vm: &VirtualMachine) -> Option<PyObjectRef> { - cls.iter_base_chain().skip(1).find_map(|cls| { - // TODO: should actually be some translation of: - // cls.slot_dictoffset != 0 && !cls.flags.contains(HEAPTYPE) - if cls.is(vm.ctx.types.type_type) { - cls.get_attr(identifier!(vm, __dict__)) - } else { - None - } - }) -} - -fn subtype_get_dict(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // TODO: obj.class().as_pyref() need to be supported - let ret = match find_base_dict_descr(obj.class(), vm) { - Some(descr) => vm.call_get_descriptor(&descr, obj).unwrap_or_else(|| { - Err(vm.new_type_error(format!( - "this __dict__ descriptor does not support '{}' objects", - descr.class() - ))) - })?, - None => object::object_get_dict(obj, vm)?.into(), - }; - Ok(ret) -} - -fn subtype_set_dict(obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let cls = obj.class(); - match find_base_dict_descr(cls, vm) { - Some(descr) => { - let descr_set = descr - .class() - .mro_find_map(|cls| cls.slots.descr_set.load()) - .ok_or_else(|| { - vm.new_type_error(format!( - "this __dict__ descriptor does not support '{}' objects", - cls.name() - )) - })?; - descr_set(&descr, obj, PySetterValue::Assign(value), vm) - } - None => { - object::object_set_dict(obj, value.try_into_value(vm)?, vm)?; - Ok(()) - } - } -} - -/* - * The magical type type - */ - -pub(crate) fn init(ctx: &Context) { - PyType::extend_class(ctx, ctx.types.type_type); -} - -pub(crate) fn call_slot_new( - typ: PyTypeRef, - subtype: PyTypeRef, - args: FuncArgs, - vm: &VirtualMachine, -) -> PyResult { - for cls in typ.deref().iter_mro() { - if let Some(slot_new) = cls.slots.new.load() { - return slot_new(subtype, args, vm); - } - } - unreachable!("Should be able to find a new slot somewhere in the mro") -} - -pub(super) fn or_(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if !union_::is_unionable(zelf.clone(), vm) || !union_::is_unionable(other.clone(), vm) { - return vm.ctx.not_implemented(); - } - - let tuple = PyTuple::new_ref(vec![zelf, other], &vm.ctx); - union_::make_union(&tuple, vm) -} - -fn take_next_base(bases: &mut [Vec<PyTypeRef>]) -> Option<PyTypeRef> { - for base in bases.iter() { - let head = base[0].clone(); - if !bases.iter().any(|x| x[1..].iter().any(|x| x.is(&head))) { - // Remove from other heads. - for item in bases.iter_mut() { - if item[0].is(&head) { - item.remove(0); - } - } - - return Some(head); - } - } - - None -} - -fn linearise_mro(mut bases: Vec<Vec<PyTypeRef>>) -> Result<Vec<PyTypeRef>, String> { - vm_trace!("Linearise MRO: {:?}", bases); - // Python requires that the class direct bases are kept in the same order. - // This is called local precedence ordering. - // This means we must verify that for classes A(), B(A) we must reject C(A, B) even though this - // algorithm will allow the mro ordering of [C, B, A, object]. - // To verify this, we make sure non of the direct bases are in the mro of bases after them. - for (i, base_mro) in bases.iter().enumerate() { - let base = &base_mro[0]; // MROs cannot be empty. - for later_mro in &bases[i + 1..] { - // We start at index 1 to skip direct bases. - // This will not catch duplicate bases, but such a thing is already tested for. - if later_mro[1..].iter().any(|cls| cls.is(base)) { - return Err( - "Unable to find mro order which keeps local precedence ordering".to_owned(), - ); - } - } - } - - let mut result = vec![]; - while !bases.is_empty() { - let head = take_next_base(&mut bases).ok_or_else(|| { - // Take the head class of each class here. Now that we have reached the problematic bases. - // Because this failed, we assume the lists cannot be empty. - format!( - "Cannot create a consistent method resolution order (MRO) for bases {}", - bases.iter().map(|x| x.first().unwrap()).format(", ") - ) - })?; - - result.push(head); - - bases.retain(|x| !x.is_empty()); - } - Ok(result) -} - -fn calculate_meta_class( - metatype: PyTypeRef, - bases: &[PyTypeRef], - vm: &VirtualMachine, -) -> PyResult<PyTypeRef> { - // = _PyType_CalculateMetaclass - let mut winner = metatype; - for base in bases { - let base_type = base.class(); - if winner.fast_issubclass(base_type) { - continue; - } else if base_type.fast_issubclass(&winner) { - winner = base_type.to_owned(); - continue; - } - - return Err(vm.new_type_error( - "metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass \ - of the metaclasses of all its bases" - .to_owned(), - )); - } - Ok(winner) -} - -fn best_base(bases: &[PyTypeRef], vm: &VirtualMachine) -> PyResult<PyTypeRef> { - // let mut base = None; - // let mut winner = None; - - for base_i in bases { - // base_proto = PyTuple_GET_ITEM(bases, i); - // if (!PyType_Check(base_proto)) { - // PyErr_SetString( - // PyExc_TypeError, - // "bases must be types"); - // return NULL; - // } - // base_i = (PyTypeObject *)base_proto; - // if (base_i->slot_dict == NULL) { - // if (PyType_Ready(base_i) < 0) - // return NULL; - // } - - if !base_i.slots.flags.has_feature(PyTypeFlags::BASETYPE) { - return Err(vm.new_type_error(format!( - "type '{}' is not an acceptable base type", - base_i.name() - ))); - } - // candidate = solid_base(base_i); - // if (winner == NULL) { - // winner = candidate; - // base = base_i; - // } - // else if (PyType_IsSubtype(winner, candidate)) - // ; - // else if (PyType_IsSubtype(candidate, winner)) { - // winner = candidate; - // base = base_i; - // } - // else { - // PyErr_SetString( - // PyExc_TypeError, - // "multiple bases have " - // "instance lay-out conflict"); - // return NULL; - // } - } - - // FIXME: Ok(base.unwrap()) is expected - Ok(bases[0].clone()) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn map_ids(obj: Result<Vec<PyTypeRef>, String>) -> Result<Vec<usize>, String> { - Ok(obj?.into_iter().map(|x| x.get_id()).collect()) - } - - #[test] - fn test_linearise() { - let context = Context::genesis(); - let object = context.types.object_type.to_owned(); - let type_type = context.types.type_type.to_owned(); - - let a = PyType::new_heap( - "A", - vec![object.clone()], - PyAttributes::default(), - Default::default(), - type_type.clone(), - context, - ) - .unwrap(); - let b = PyType::new_heap( - "B", - vec![object.clone()], - PyAttributes::default(), - Default::default(), - type_type, - context, - ) - .unwrap(); - - assert_eq!( - map_ids(linearise_mro(vec![ - vec![object.clone()], - vec![object.clone()] - ])), - map_ids(Ok(vec![object.clone()])) - ); - assert_eq!( - map_ids(linearise_mro(vec![ - vec![a.clone(), object.clone()], - vec![b.clone(), object.clone()], - ])), - map_ids(Ok(vec![a, b, object])) - ); - } -} diff --git a/vm/src/builtins/union.rs b/vm/src/builtins/union.rs deleted file mode 100644 index 668d87bdcec..00000000000 --- a/vm/src/builtins/union.rs +++ /dev/null @@ -1,293 +0,0 @@ -use super::{genericalias, type_}; -use crate::{ - atomic_func, - builtins::{PyFrozenSet, PyStr, PyTuple, PyTupleRef, PyType}, - class::PyClassImpl, - common::hash, - convert::{ToPyObject, ToPyResult}, - function::PyComparisonValue, - protocol::{PyMappingMethods, PyNumberMethods}, - types::{AsMapping, AsNumber, Comparable, GetAttr, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use once_cell::sync::Lazy; -use std::fmt; - -const CLS_ATTRS: &[&str] = &["__module__"]; - -#[pyclass(module = "types", name = "UnionType", traverse)] -pub struct PyUnion { - args: PyTupleRef, - parameters: PyTupleRef, -} - -impl fmt::Debug for PyUnion { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("UnionObject") - } -} - -impl PyPayload for PyUnion { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.union_type - } -} - -impl PyUnion { - pub fn new(args: PyTupleRef, vm: &VirtualMachine) -> Self { - let parameters = make_parameters(&args, vm); - Self { args, parameters } - } - - fn repr(&self, vm: &VirtualMachine) -> PyResult<String> { - fn repr_item(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<String> { - if obj.is(vm.ctx.types.none_type) { - return Ok("None".to_string()); - } - - if vm - .get_attribute_opt(obj.clone(), identifier!(vm, __origin__))? - .is_some() - && vm - .get_attribute_opt(obj.clone(), identifier!(vm, __args__))? - .is_some() - { - return Ok(obj.repr(vm)?.as_str().to_string()); - } - - match ( - vm.get_attribute_opt(obj.clone(), identifier!(vm, __qualname__))? - .and_then(|o| o.downcast_ref::<PyStr>().map(|n| n.as_str().to_string())), - vm.get_attribute_opt(obj.clone(), identifier!(vm, __module__))? - .and_then(|o| o.downcast_ref::<PyStr>().map(|m| m.as_str().to_string())), - ) { - (None, _) | (_, None) => Ok(obj.repr(vm)?.as_str().to_string()), - (Some(qualname), Some(module)) => Ok(if module == "builtins" { - qualname - } else { - format!("{module}.{qualname}") - }), - } - } - - Ok(self - .args - .iter() - .map(|o| repr_item(o.clone(), vm)) - .collect::<PyResult<Vec<_>>>()? - .join(" | ")) - } -} - -#[pyclass( - flags(BASETYPE), - with(Hashable, Comparable, AsMapping, AsNumber, Representable) -)] -impl PyUnion { - #[pygetset(magic)] - fn parameters(&self) -> PyObjectRef { - self.parameters.clone().into() - } - - #[pygetset(magic)] - fn args(&self) -> PyObjectRef { - self.args.clone().into() - } - - #[pymethod(magic)] - fn instancecheck(zelf: PyRef<Self>, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - if zelf - .args - .iter() - .any(|x| x.class().is(vm.ctx.types.generic_alias_type)) - { - Err(vm.new_type_error( - "isinstance() argument 2 cannot be a parameterized generic".to_owned(), - )) - } else { - obj.is_instance(zelf.args().as_object(), vm) - } - } - - #[pymethod(magic)] - fn subclasscheck(zelf: PyRef<Self>, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - if zelf - .args - .iter() - .any(|x| x.class().is(vm.ctx.types.generic_alias_type)) - { - Err(vm.new_type_error( - "issubclass() argument 2 cannot be a parameterized generic".to_owned(), - )) - } else { - obj.is_subclass(zelf.args().as_object(), vm) - } - } - - #[pymethod(name = "__ror__")] - #[pymethod(magic)] - fn or(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - type_::or_(zelf, other, vm) - } -} - -pub fn is_unionable(obj: PyObjectRef, vm: &VirtualMachine) -> bool { - obj.class().is(vm.ctx.types.none_type) - || obj.payload_if_subclass::<PyType>(vm).is_some() - || obj.class().is(vm.ctx.types.generic_alias_type) - || obj.class().is(vm.ctx.types.union_type) -} - -fn make_parameters(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { - let parameters = genericalias::make_parameters(args, vm); - dedup_and_flatten_args(&parameters, vm) -} - -fn flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { - let mut total_args = 0; - for arg in args { - if let Some(pyref) = arg.downcast_ref::<PyUnion>() { - total_args += pyref.args.len(); - } else { - total_args += 1; - }; - } - - let mut flattened_args = Vec::with_capacity(total_args); - for arg in args { - if let Some(pyref) = arg.downcast_ref::<PyUnion>() { - flattened_args.extend(pyref.args.iter().cloned()); - } else if vm.is_none(arg) { - flattened_args.push(vm.ctx.types.none_type.to_owned().into()); - } else { - flattened_args.push(arg.clone()); - }; - } - - PyTuple::new_ref(flattened_args, &vm.ctx) -} - -fn dedup_and_flatten_args(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { - let args = flatten_args(args, vm); - - let mut new_args: Vec<PyObjectRef> = Vec::with_capacity(args.len()); - for arg in &*args { - if !new_args.iter().any(|param| { - param - .rich_compare_bool(arg, PyComparisonOp::Eq, vm) - .expect("types are always comparable") - }) { - new_args.push(arg.clone()); - } - } - - new_args.shrink_to_fit(); - - PyTuple::new_ref(new_args, &vm.ctx) -} - -pub fn make_union(args: &Py<PyTuple>, vm: &VirtualMachine) -> PyObjectRef { - let args = dedup_and_flatten_args(args, vm); - match args.len() { - 1 => args.fast_getitem(0), - _ => PyUnion::new(args, vm).to_pyobject(vm), - } -} - -impl PyUnion { - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let new_args = genericalias::subs_parameters( - |vm| self.repr(vm), - self.args.clone(), - self.parameters.clone(), - needle, - vm, - )?; - let mut res; - if new_args.len() == 0 { - res = make_union(&new_args, vm); - } else { - res = new_args.fast_getitem(0); - for arg in new_args.iter().skip(1) { - res = vm._or(&res, arg)?; - } - } - - Ok(res) - } -} - -impl AsMapping for PyUnion { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: Lazy<PyMappingMethods> = Lazy::new(|| PyMappingMethods { - subscript: atomic_func!(|mapping, needle, vm| { - PyUnion::mapping_downcast(mapping).getitem(needle.to_owned(), vm) - }), - ..PyMappingMethods::NOT_IMPLEMENTED - }); - &AS_MAPPING - } -} - -impl AsNumber for PyUnion { - fn as_number() -> &'static PyNumberMethods { - static AS_NUMBER: PyNumberMethods = PyNumberMethods { - or: Some(|a, b, vm| PyUnion::or(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), - ..PyNumberMethods::NOT_IMPLEMENTED - }; - &AS_NUMBER - } -} - -impl Comparable for PyUnion { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - op.eq_only(|| { - let other = class_or_notimplemented!(Self, other); - let a = PyFrozenSet::from_iter(vm, zelf.args.into_iter().cloned())?; - let b = PyFrozenSet::from_iter(vm, other.args.into_iter().cloned())?; - Ok(PyComparisonValue::Implemented( - a.into_pyobject(vm).as_object().rich_compare_bool( - b.into_pyobject(vm).as_object(), - PyComparisonOp::Eq, - vm, - )?, - )) - }) - } -} - -impl Hashable for PyUnion { - #[inline] - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<hash::PyHash> { - let set = PyFrozenSet::from_iter(vm, zelf.args.into_iter().cloned())?; - PyFrozenSet::hash(&set.into_ref(&vm.ctx), vm) - } -} - -impl GetAttr for PyUnion { - fn getattro(zelf: &Py<Self>, attr: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - for &exc in CLS_ATTRS { - if *exc == attr.to_string() { - return zelf.as_object().generic_getattr(attr, vm); - } - } - zelf.as_object().get_attr(attr, vm) - } -} - -impl Representable for PyUnion { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - zelf.repr(vm) - } -} - -pub fn init(context: &Context) { - let union_type = &context.types.union_type; - PyUnion::extend_class(context, union_type); -} diff --git a/vm/src/builtins/weakproxy.rs b/vm/src/builtins/weakproxy.rs deleted file mode 100644 index 731d523e226..00000000000 --- a/vm/src/builtins/weakproxy.rs +++ /dev/null @@ -1,241 +0,0 @@ -use super::{PyStr, PyStrRef, PyType, PyTypeRef, PyWeak}; -use crate::{ - atomic_func, - class::PyClassImpl, - common::hash::PyHash, - function::{OptionalArg, PyComparisonValue, PySetterValue}, - protocol::{PyIter, PyIterReturn, PyMappingMethods, PySequenceMethods}, - stdlib::builtins::reversed, - types::{ - AsMapping, AsSequence, Comparable, Constructor, GetAttr, Hashable, IterNext, Iterable, - PyComparisonOp, Representable, SetAttr, - }, - Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use once_cell::sync::Lazy; - -#[pyclass(module = false, name = "weakproxy", unhashable = true, traverse)] -#[derive(Debug)] -pub struct PyWeakProxy { - weak: PyRef<PyWeak>, -} - -impl PyPayload for PyWeakProxy { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.weakproxy_type - } -} - -#[derive(FromArgs)] -pub struct WeakProxyNewArgs { - #[pyarg(positional)] - referent: PyObjectRef, - #[pyarg(positional, optional)] - callback: OptionalArg<PyObjectRef>, -} - -impl Constructor for PyWeakProxy { - type Args = WeakProxyNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { referent, callback }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - // using an internal subclass as the class prevents us from getting the generic weakref, - // which would mess up the weakref count - let weak_cls = WEAK_SUBCLASS.get_or_init(|| { - vm.ctx.new_class( - None, - "__weakproxy", - vm.ctx.types.weakref_type.to_owned(), - super::PyWeak::make_slots(), - ) - }); - // TODO: PyWeakProxy should use the same payload as PyWeak - PyWeakProxy { - weak: referent.downgrade_with_typ(callback.into_option(), weak_cls.clone(), vm)?, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -crate::common::static_cell! { - static WEAK_SUBCLASS: PyTypeRef; -} - -#[pyclass(with( - GetAttr, - SetAttr, - Constructor, - Comparable, - AsSequence, - AsMapping, - Representable, - IterNext -))] -impl PyWeakProxy { - fn try_upgrade(&self, vm: &VirtualMachine) -> PyResult { - self.weak.upgrade().ok_or_else(|| new_reference_error(vm)) - } - - #[pymethod(magic)] - fn str(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> { - self.try_upgrade(vm)?.str(vm) - } - - fn len(&self, vm: &VirtualMachine) -> PyResult<usize> { - self.try_upgrade(vm)?.length(vm) - } - - #[pymethod(magic)] - fn bool(&self, vm: &VirtualMachine) -> PyResult<bool> { - self.try_upgrade(vm)?.is_true(vm) - } - - #[pymethod(magic)] - fn bytes(&self, vm: &VirtualMachine) -> PyResult { - self.try_upgrade(vm)?.bytes(vm) - } - - #[pymethod(magic)] - fn reversed(&self, vm: &VirtualMachine) -> PyResult { - let obj = self.try_upgrade(vm)?; - reversed(obj, vm) - } - #[pymethod(magic)] - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self.try_upgrade(vm)?.to_sequence(vm).contains(&needle, vm) - } - - fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let obj = self.try_upgrade(vm)?; - obj.get_item(&*needle, vm) - } - - fn setitem( - &self, - needle: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let obj = self.try_upgrade(vm)?; - obj.set_item(&*needle, value, vm) - } - - fn delitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let obj = self.try_upgrade(vm)?; - obj.del_item(&*needle, vm) - } -} - -impl Iterable for PyWeakProxy { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - let obj = zelf.try_upgrade(vm)?; - Ok(obj.get_iter(vm)?.into()) - } -} - -impl IterNext for PyWeakProxy { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let obj = zelf.try_upgrade(vm)?; - PyIter::new(obj).next(vm) - } -} - -fn new_reference_error(vm: &VirtualMachine) -> PyRef<super::PyBaseException> { - vm.new_exception_msg( - vm.ctx.exceptions.reference_error.to_owned(), - "weakly-referenced object no longer exists".to_owned(), - ) -} - -impl GetAttr for PyWeakProxy { - // TODO: callbacks - fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - let obj = zelf.try_upgrade(vm)?; - obj.get_attr(name, vm) - } -} - -impl SetAttr for PyWeakProxy { - fn setattro( - zelf: &Py<Self>, - attr_name: &Py<PyStr>, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()> { - let obj = zelf.try_upgrade(vm)?; - obj.call_set_attr(vm, attr_name, value) - } -} - -impl Comparable for PyWeakProxy { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - let obj = zelf.try_upgrade(vm)?; - Ok(PyComparisonValue::Implemented( - obj.rich_compare_bool(other, op, vm)?, - )) - } -} - -impl AsSequence for PyWeakProxy { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods { - length: atomic_func!(|seq, vm| PyWeakProxy::sequence_downcast(seq).len(vm)), - contains: atomic_func!(|seq, needle, vm| { - PyWeakProxy::sequence_downcast(seq).contains(needle.to_owned(), vm) - }), - ..PySequenceMethods::NOT_IMPLEMENTED - }); - &AS_SEQUENCE - } -} - -impl AsMapping for PyWeakProxy { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: PyMappingMethods = PyMappingMethods { - length: atomic_func!(|mapping, vm| PyWeakProxy::mapping_downcast(mapping).len(vm)), - subscript: atomic_func!(|mapping, needle, vm| { - PyWeakProxy::mapping_downcast(mapping).getitem(needle.to_owned(), vm) - }), - ass_subscript: atomic_func!(|mapping, needle, value, vm| { - let zelf = PyWeakProxy::mapping_downcast(mapping); - if let Some(value) = value { - zelf.setitem(needle.to_owned(), value, vm) - } else { - zelf.delitem(needle.to_owned(), vm) - } - }), - }; - &AS_MAPPING - } -} - -impl Representable for PyWeakProxy { - #[inline] - fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - zelf.try_upgrade(vm)?.repr(vm) - } - - #[cold] - fn repr_str(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - unreachable!("use repr instead") - } -} - -pub fn init(context: &Context) { - PyWeakProxy::extend_class(context, context.types.weakproxy_type); -} - -impl Hashable for PyWeakProxy { - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - zelf.try_upgrade(vm)?.hash(vm) - } -} diff --git a/vm/src/builtins/weakref.rs b/vm/src/builtins/weakref.rs deleted file mode 100644 index 1d52225a26c..00000000000 --- a/vm/src/builtins/weakref.rs +++ /dev/null @@ -1,125 +0,0 @@ -use super::{PyGenericAlias, PyType, PyTypeRef}; -use crate::common::{ - atomic::{Ordering, Radium}, - hash::{self, PyHash}, -}; -use crate::{ - class::PyClassImpl, - function::OptionalArg, - types::{Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; - -pub use crate::object::PyWeak; - -#[derive(FromArgs)] -pub struct WeakNewArgs { - #[pyarg(positional)] - referent: PyObjectRef, - #[pyarg(positional, optional)] - callback: OptionalArg<PyObjectRef>, -} - -impl PyPayload for PyWeak { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.weakref_type - } -} - -impl Callable for PyWeak { - type Args = (); - #[inline] - fn call(zelf: &Py<Self>, _: Self::Args, vm: &VirtualMachine) -> PyResult { - Ok(vm.unwrap_or_none(zelf.upgrade())) - } -} - -impl Constructor for PyWeak { - type Args = WeakNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { referent, callback }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let weak = referent.downgrade_with_typ(callback.into_option(), cls, vm)?; - Ok(weak.into()) - } -} - -#[pyclass( - with(Callable, Hashable, Comparable, Constructor, Representable), - flags(BASETYPE) -)] -impl PyWeak { - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } -} - -impl Hashable for PyWeak { - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - let hash = match zelf.hash.load(Ordering::Relaxed) { - hash::SENTINEL => { - let obj = zelf - .upgrade() - .ok_or_else(|| vm.new_type_error("weak object has gone away".to_owned()))?; - let hash = obj.hash(vm)?; - match Radium::compare_exchange( - &zelf.hash, - hash::SENTINEL, - hash::fix_sentinel(hash), - Ordering::Relaxed, - Ordering::Relaxed, - ) { - Ok(_) => hash, - Err(prev_stored) => prev_stored, - } - } - hash => hash, - }; - Ok(hash) - } -} - -impl Comparable for PyWeak { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<crate::function::PyComparisonValue> { - op.eq_only(|| { - let other = class_or_notimplemented!(Self, other); - let both = zelf.upgrade().and_then(|s| other.upgrade().map(|o| (s, o))); - let eq = match both { - Some((a, b)) => vm.bool_eq(&a, &b)?, - None => zelf.is(other), - }; - Ok(eq.into()) - }) - } -} - -impl Representable for PyWeak { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - let id = zelf.get_id(); - let repr = if let Some(o) = zelf.upgrade() { - format!( - "<weakref at {:#x}; to '{}' at {:#x}>", - id, - o.class().name(), - o.get_id(), - ) - } else { - format!("<weakref at {id:#x}; dead>") - }; - Ok(repr) - } -} - -pub fn init(context: &Context) { - PyWeak::extend_class(context, context.types.weakref_type); -} diff --git a/vm/src/byte.rs b/vm/src/byte.rs deleted file mode 100644 index 42455bd27c7..00000000000 --- a/vm/src/byte.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! byte operation APIs -use crate::object::AsObject; -use crate::{PyObject, PyResult, VirtualMachine}; -use num_traits::ToPrimitive; - -pub fn bytes_from_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<Vec<u8>> { - if let Ok(elements) = obj.try_bytes_like(vm, |bytes| bytes.to_vec()) { - return Ok(elements); - } - - if !obj.fast_isinstance(vm.ctx.types.str_type) { - if let Ok(elements) = vm.map_iterable_object(obj, |x| value_from_object(vm, &x)) { - return elements; - } - } - - Err(vm.new_type_error( - "can assign only bytes, buffers, or iterables of ints in range(0, 256)".to_owned(), - )) -} - -pub fn value_from_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult<u8> { - obj.try_index(vm)? - .as_bigint() - .to_u8() - .ok_or_else(|| vm.new_value_error("byte must be in range(0, 256)".to_owned())) -} diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs deleted file mode 100644 index 750344c7810..00000000000 --- a/vm/src/bytesinner.rs +++ /dev/null @@ -1,1236 +0,0 @@ -use crate::{ - anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper}, - builtins::{ - pystr, PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyInt, PyIntRef, PyStr, - PyStrRef, PyTypeRef, - }, - byte::bytes_from_object, - cformat::cformat_bytes, - common::hash, - function::{ArgIterable, Either, OptionalArg, OptionalOption, PyComparisonValue}, - identifier, - literal::escape::Escape, - protocol::PyBuffer, - sequence::{SequenceExt, SequenceMutExt}, - types::PyComparisonOp, - AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, -}; -use bstr::ByteSlice; -use itertools::Itertools; -use malachite_bigint::BigInt; -use num_traits::ToPrimitive; - -#[derive(Debug, Default, Clone)] -pub struct PyBytesInner { - pub(super) elements: Vec<u8>, -} - -impl From<Vec<u8>> for PyBytesInner { - fn from(elements: Vec<u8>) -> PyBytesInner { - Self { elements } - } -} - -impl<'a> TryFromBorrowedObject<'a> for PyBytesInner { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - bytes_from_object(vm, obj).map(Self::from) - } -} - -#[derive(FromArgs)] -pub struct ByteInnerNewOptions { - #[pyarg(any, optional)] - pub source: OptionalArg<PyObjectRef>, - #[pyarg(any, optional)] - pub encoding: OptionalArg<PyStrRef>, - #[pyarg(any, optional)] - pub errors: OptionalArg<PyStrRef>, -} - -impl ByteInnerNewOptions { - fn get_value_from_string( - s: PyStrRef, - encoding: PyStrRef, - errors: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<PyBytesInner> { - let bytes = pystr::encode_string(s, Some(encoding), errors.into_option(), vm)?; - Ok(bytes.as_bytes().to_vec().into()) - } - - fn get_value_from_source(source: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyBytesInner> { - bytes_from_object(vm, &source).map(|x| x.into()) - } - - fn get_value_from_size(size: PyIntRef, vm: &VirtualMachine) -> PyResult<PyBytesInner> { - let size = size.as_bigint().to_isize().ok_or_else(|| { - vm.new_overflow_error("cannot fit 'int' into an index-sized integer".to_owned()) - })?; - let size = if size < 0 { - return Err(vm.new_value_error("negative count".to_owned())); - } else { - size as usize - }; - Ok(vec![0; size].into()) - } - - pub fn get_bytes(self, cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - let inner = match (&self.source, &self.encoding, &self.errors) { - (OptionalArg::Present(obj), OptionalArg::Missing, OptionalArg::Missing) => { - let obj = obj.clone(); - // construct an exact bytes from an exact bytes do not clone - let obj = if cls.is(PyBytes::class(&vm.ctx)) { - match obj.downcast_exact::<PyBytes>(vm) { - Ok(b) => return Ok(b.into_pyref()), - Err(obj) => obj, - } - } else { - obj - }; - - if let Some(bytes_method) = vm.get_method(obj, identifier!(vm, __bytes__)) { - // construct an exact bytes from __bytes__ slot. - // if __bytes__ return a bytes, use the bytes object except we are the subclass of the bytes - let bytes = bytes_method?.call((), vm)?; - let bytes = if cls.is(PyBytes::class(&vm.ctx)) { - match bytes.downcast::<PyBytes>() { - Ok(b) => return Ok(b), - Err(bytes) => bytes, - } - } else { - bytes - }; - Some(PyBytesInner::try_from_borrowed_object(vm, &bytes)) - } else { - None - } - } - _ => None, - } - .unwrap_or_else(|| self.get_bytearray_inner(vm))?; - PyBytes::from(inner).into_ref_with_type(vm, cls) - } - - pub fn get_bytearray_inner(self, vm: &VirtualMachine) -> PyResult<PyBytesInner> { - const STRING_WITHOUT_ENCODING: &str = "string argument without an encoding"; - const ENCODING_WITHOUT_STRING: &str = "encoding without a string argument"; - - match (self.source, self.encoding, self.errors) { - (OptionalArg::Present(obj), OptionalArg::Missing, OptionalArg::Missing) => { - match_class!(match obj { - i @ PyInt => { - Ok(Self::get_value_from_size(i, vm)?) - } - _s @ PyStr => Err(STRING_WITHOUT_ENCODING), - obj => { - Ok(Self::get_value_from_source(obj, vm)?) - } - }) - } - (OptionalArg::Present(obj), OptionalArg::Present(encoding), errors) => { - if let Ok(s) = obj.downcast::<PyStr>() { - Ok(Self::get_value_from_string(s, encoding, errors, vm)?) - } else { - Err(ENCODING_WITHOUT_STRING) - } - } - (OptionalArg::Missing, OptionalArg::Missing, OptionalArg::Missing) => { - Ok(PyBytesInner::default()) - } - (OptionalArg::Missing, OptionalArg::Present(_), _) => Err(ENCODING_WITHOUT_STRING), - (OptionalArg::Missing, _, OptionalArg::Present(_)) => { - Err("errors without a string argument") - } - (OptionalArg::Present(_), OptionalArg::Missing, OptionalArg::Present(_)) => { - Err(STRING_WITHOUT_ENCODING) - } - } - .map_err(|e| vm.new_type_error(e.to_owned())) - } -} - -#[derive(FromArgs)] -pub struct ByteInnerFindOptions { - #[pyarg(positional)] - sub: Either<PyBytesInner, PyIntRef>, - #[pyarg(positional, default)] - start: Option<PyIntRef>, - #[pyarg(positional, default)] - end: Option<PyIntRef>, -} - -impl ByteInnerFindOptions { - pub fn get_value( - self, - len: usize, - vm: &VirtualMachine, - ) -> PyResult<(Vec<u8>, std::ops::Range<usize>)> { - let sub = match self.sub { - Either::A(v) => v.elements.to_vec(), - Either::B(int) => vec![int.as_bigint().byte_or(vm)?], - }; - let range = anystr::adjust_indices(self.start, self.end, len); - Ok((sub, range)) - } -} - -#[derive(FromArgs)] -pub struct ByteInnerPaddingOptions { - #[pyarg(positional)] - width: isize, - #[pyarg(positional, optional)] - fillchar: OptionalArg<PyObjectRef>, -} - -impl ByteInnerPaddingOptions { - fn get_value(self, fn_name: &str, vm: &VirtualMachine) -> PyResult<(isize, u8)> { - let fillchar = if let OptionalArg::Present(v) = self.fillchar { - try_as_bytes(v.clone(), |bytes| bytes.iter().copied().exactly_one().ok()) - .flatten() - .ok_or_else(|| { - vm.new_type_error(format!( - "{}() argument 2 must be a byte string of length 1, not {}", - fn_name, - v.class().name() - )) - })? - } else { - b' ' // default is space - }; - - Ok((self.width, fillchar)) - } -} - -#[derive(FromArgs)] -pub struct ByteInnerTranslateOptions { - #[pyarg(positional)] - table: Option<PyObjectRef>, - #[pyarg(any, optional)] - delete: OptionalArg<PyObjectRef>, -} - -impl ByteInnerTranslateOptions { - pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Vec<u8>, Vec<u8>)> { - let table = self.table.map_or_else( - || Ok((0..=255).collect::<Vec<u8>>()), - |v| { - let bytes = v - .try_into_value::<PyBytesInner>(vm) - .ok() - .filter(|v| v.elements.len() == 256) - .ok_or_else(|| { - vm.new_value_error( - "translation table must be 256 characters long".to_owned(), - ) - })?; - Ok(bytes.elements.to_vec()) - }, - )?; - - let delete = match self.delete { - OptionalArg::Present(byte) => { - let byte: PyBytesInner = byte.try_into_value(vm)?; - byte.elements - } - _ => vec![], - }; - - Ok((table, delete)) - } -} - -pub type ByteInnerSplitOptions = anystr::SplitArgs<PyBytesInner>; - -impl PyBytesInner { - #[inline] - pub fn as_bytes(&self) -> &[u8] { - &self.elements - } - - fn new_repr_overflow_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_overflow_error("bytes object is too large to make repr".to_owned()) - } - - pub fn repr_with_name(&self, class_name: &str, vm: &VirtualMachine) -> PyResult<String> { - const DECORATION_LEN: isize = 2 + 3; // 2 for (), 3 for b"" => bytearray(b"") - let escape = crate::literal::escape::AsciiEscape::new_repr(&self.elements); - let len = escape - .layout() - .len - .and_then(|len| (len as isize).checked_add(DECORATION_LEN + class_name.len() as isize)) - .ok_or_else(|| Self::new_repr_overflow_error(vm))? as usize; - let mut buf = String::with_capacity(len); - buf.push_str(class_name); - buf.push('('); - escape.bytes_repr().write(&mut buf).unwrap(); - buf.push(')'); - debug_assert_eq!(buf.len(), len); - Ok(buf) - } - - pub fn repr_bytes(&self, vm: &VirtualMachine) -> PyResult<String> { - let escape = crate::literal::escape::AsciiEscape::new_repr(&self.elements); - let len = 3 + escape - .layout() - .len - .ok_or_else(|| Self::new_repr_overflow_error(vm))?; - let mut buf = String::with_capacity(len); - escape.bytes_repr().write(&mut buf).unwrap(); - debug_assert_eq!(buf.len(), len); - Ok(buf) - } - - #[inline] - pub fn len(&self) -> usize { - self.elements.len() - } - - #[inline] - pub fn capacity(&self) -> usize { - self.elements.capacity() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.elements.is_empty() - } - - pub fn cmp( - &self, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyComparisonValue { - // TODO: bytes can compare with any object implemented buffer protocol - // but not memoryview, and not equal if compare with unicode str(PyStr) - PyComparisonValue::from_option( - other - .try_bytes_like(vm, |other| op.eval_ord(self.elements.as_slice().cmp(other))) - .ok(), - ) - } - - pub fn hash(&self, vm: &VirtualMachine) -> hash::PyHash { - vm.state.hash_secret.hash_bytes(&self.elements) - } - - pub fn add(&self, other: &[u8]) -> Vec<u8> { - self.elements.py_add(other) - } - - pub fn contains( - &self, - needle: Either<PyBytesInner, PyIntRef>, - vm: &VirtualMachine, - ) -> PyResult<bool> { - Ok(match needle { - Either::A(byte) => self.elements.contains_str(byte.elements.as_slice()), - Either::B(int) => self.elements.contains(&int.as_bigint().byte_or(vm)?), - }) - } - - pub fn isalnum(&self) -> bool { - !self.elements.is_empty() - && self - .elements - .iter() - .all(|x| char::from(*x).is_alphanumeric()) - } - - pub fn isalpha(&self) -> bool { - !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_alphabetic()) - } - - pub fn isascii(&self) -> bool { - self.elements.iter().all(|x| char::from(*x).is_ascii()) - } - - pub fn isdigit(&self) -> bool { - !self.elements.is_empty() - && self - .elements - .iter() - .all(|x| char::from(*x).is_ascii_digit()) - } - - pub fn islower(&self) -> bool { - self.elements - .py_iscase(char::is_lowercase, char::is_uppercase) - } - - pub fn isupper(&self) -> bool { - self.elements - .py_iscase(char::is_uppercase, char::is_lowercase) - } - - pub fn isspace(&self) -> bool { - !self.elements.is_empty() - && self - .elements - .iter() - .all(|x| char::from(*x).is_ascii_whitespace()) - } - - pub fn istitle(&self) -> bool { - if self.elements.is_empty() { - return false; - } - - let mut iter = self.elements.iter().peekable(); - let mut prev_cased = false; - - while let Some(c) = iter.next() { - let current = char::from(*c); - let next = if let Some(k) = iter.peek() { - char::from(**k) - } else if current.is_uppercase() { - return !prev_cased; - } else { - return prev_cased; - }; - - let is_cased = current.to_uppercase().next().unwrap() != current - || current.to_lowercase().next().unwrap() != current; - if (is_cased && next.is_uppercase() && !prev_cased) - || (!is_cased && next.is_lowercase()) - { - return false; - } - - prev_cased = is_cased; - } - - true - } - - pub fn lower(&self) -> Vec<u8> { - self.elements.to_ascii_lowercase() - } - - pub fn upper(&self) -> Vec<u8> { - self.elements.to_ascii_uppercase() - } - - pub fn capitalize(&self) -> Vec<u8> { - let mut new: Vec<u8> = Vec::with_capacity(self.elements.len()); - if let Some((first, second)) = self.elements.split_first() { - new.push(first.to_ascii_uppercase()); - second.iter().for_each(|x| new.push(x.to_ascii_lowercase())); - } - new - } - - pub fn swapcase(&self) -> Vec<u8> { - let mut new: Vec<u8> = Vec::with_capacity(self.elements.len()); - for w in &self.elements { - match w { - 65..=90 => new.push(w.to_ascii_lowercase()), - 97..=122 => new.push(w.to_ascii_uppercase()), - x => new.push(*x), - } - } - new - } - - pub fn hex( - &self, - sep: OptionalArg<Either<PyStrRef, PyBytesRef>>, - bytes_per_sep: OptionalArg<isize>, - vm: &VirtualMachine, - ) -> PyResult<String> { - bytes_to_hex(self.elements.as_slice(), sep, bytes_per_sep, vm) - } - - pub fn fromhex(string: &str, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let mut iter = string.bytes().enumerate(); - let mut bytes: Vec<u8> = Vec::with_capacity(string.len() / 2); - let i = loop { - let (i, b) = match iter.next() { - Some(val) => val, - None => { - return Ok(bytes); - } - }; - - if is_py_ascii_whitespace(b) { - continue; - } - - let top = match b { - b'0'..=b'9' => b - b'0', - b'a'..=b'f' => 10 + b - b'a', - b'A'..=b'F' => 10 + b - b'A', - _ => break i, - }; - - let (i, b) = match iter.next() { - Some(val) => val, - None => break i + 1, - }; - - let bot = match b { - b'0'..=b'9' => b - b'0', - b'a'..=b'f' => 10 + b - b'a', - b'A'..=b'F' => 10 + b - b'A', - _ => break i, - }; - - bytes.push((top << 4) + bot); - }; - - Err(vm.new_value_error(format!( - "non-hexadecimal number found in fromhex() arg at position {i}" - ))) - } - - #[inline] - fn _pad( - &self, - options: ByteInnerPaddingOptions, - pad: fn(&[u8], usize, u8, usize) -> Vec<u8>, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - let (width, fillchar) = options.get_value("center", vm)?; - Ok(if self.len() as isize >= width { - Vec::from(&self.elements[..]) - } else { - pad(&self.elements, width as usize, fillchar, self.len()) - }) - } - - pub fn center( - &self, - options: ByteInnerPaddingOptions, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - self._pad(options, AnyStr::py_center, vm) - } - - pub fn ljust( - &self, - options: ByteInnerPaddingOptions, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - self._pad(options, AnyStr::py_ljust, vm) - } - - pub fn rjust( - &self, - options: ByteInnerPaddingOptions, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - self._pad(options, AnyStr::py_rjust, vm) - } - - pub fn count(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult<usize> { - let (needle, range) = options.get_value(self.elements.len(), vm)?; - Ok(self - .elements - .py_count(needle.as_slice(), range, |h, n| h.find_iter(n).count())) - } - - pub fn join( - &self, - iterable: ArgIterable<PyBytesInner>, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - let iter = iterable.iter(vm)?; - self.elements.py_join(iter) - } - - #[inline] - pub fn find<F>( - &self, - options: ByteInnerFindOptions, - find: F, - vm: &VirtualMachine, - ) -> PyResult<Option<usize>> - where - F: Fn(&[u8], &[u8]) -> Option<usize>, - { - let (needle, range) = options.get_value(self.elements.len(), vm)?; - Ok(self.elements.py_find(&needle, range, find)) - } - - pub fn maketrans( - from: PyBytesInner, - to: PyBytesInner, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - if from.len() != to.len() { - return Err( - vm.new_value_error("the two maketrans arguments must have equal length".to_owned()) - ); - } - let mut res = vec![]; - - for i in 0..=255 { - res.push(if let Some(position) = from.elements.find_byte(i) { - to.elements[position] - } else { - i - }); - } - - Ok(res) - } - - pub fn translate( - &self, - options: ByteInnerTranslateOptions, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - let (table, delete) = options.get_value(vm)?; - - let mut res = if delete.is_empty() { - Vec::with_capacity(self.elements.len()) - } else { - Vec::new() - }; - - for i in &self.elements { - if !delete.contains(i) { - res.push(table[*i as usize]); - } - } - - Ok(res) - } - - pub fn strip(&self, chars: OptionalOption<PyBytesInner>) -> Vec<u8> { - self.elements - .py_strip( - chars, - |s, chars| s.trim_with(|c| chars.contains(&(c as u8))), - |s| s.trim(), - ) - .to_vec() - } - - pub fn lstrip(&self, chars: OptionalOption<PyBytesInner>) -> &[u8] { - self.elements.py_strip( - chars, - |s, chars| s.trim_start_with(|c| chars.contains(&(c as u8))), - |s| s.trim_start(), - ) - } - - pub fn rstrip(&self, chars: OptionalOption<PyBytesInner>) -> &[u8] { - self.elements.py_strip( - chars, - |s, chars| s.trim_end_with(|c| chars.contains(&(c as u8))), - |s| s.trim_end(), - ) - } - - // new in Python 3.9 - pub fn removeprefix(&self, prefix: PyBytesInner) -> Vec<u8> { - self.elements - .py_removeprefix(&prefix.elements, prefix.elements.len(), |s, p| { - s.starts_with(p) - }) - .to_vec() - } - - // new in Python 3.9 - pub fn removesuffix(&self, suffix: PyBytesInner) -> Vec<u8> { - self.elements - .py_removesuffix(&suffix.elements, suffix.elements.len(), |s, p| { - s.ends_with(p) - }) - .to_vec() - } - - pub fn split<F>( - &self, - options: ByteInnerSplitOptions, - convert: F, - vm: &VirtualMachine, - ) -> PyResult<Vec<PyObjectRef>> - where - F: Fn(&[u8], &VirtualMachine) -> PyObjectRef, - { - let elements = self.elements.py_split( - options, - vm, - |v, s, vm| v.split_str(s).map(|v| convert(v, vm)).collect(), - |v, s, n, vm| v.splitn_str(n, s).map(|v| convert(v, vm)).collect(), - |v, n, vm| v.py_split_whitespace(n, |v| convert(v, vm)), - )?; - Ok(elements) - } - - pub fn rsplit<F>( - &self, - options: ByteInnerSplitOptions, - convert: F, - vm: &VirtualMachine, - ) -> PyResult<Vec<PyObjectRef>> - where - F: Fn(&[u8], &VirtualMachine) -> PyObjectRef, - { - let mut elements = self.elements.py_split( - options, - vm, - |v, s, vm| v.rsplit_str(s).map(|v| convert(v, vm)).collect(), - |v, s, n, vm| v.rsplitn_str(n, s).map(|v| convert(v, vm)).collect(), - |v, n, vm| v.py_rsplit_whitespace(n, |v| convert(v, vm)), - )?; - elements.reverse(); - Ok(elements) - } - - pub fn partition( - &self, - sub: &PyBytesInner, - vm: &VirtualMachine, - ) -> PyResult<(Vec<u8>, bool, Vec<u8>)> { - self.elements.py_partition( - &sub.elements, - || self.elements.splitn_str(2, &sub.elements), - vm, - ) - } - - pub fn rpartition( - &self, - sub: &PyBytesInner, - vm: &VirtualMachine, - ) -> PyResult<(Vec<u8>, bool, Vec<u8>)> { - self.elements.py_partition( - &sub.elements, - || self.elements.rsplitn_str(2, &sub.elements), - vm, - ) - } - - pub fn expandtabs(&self, options: anystr::ExpandTabsArgs) -> Vec<u8> { - let tabsize = options.tabsize(); - let mut counter: usize = 0; - let mut res = vec![]; - - if tabsize == 0 { - return self - .elements - .iter() - .copied() - .filter(|x| *x != b'\t') - .collect(); - } - - for i in &self.elements { - if *i == b'\t' { - let len = tabsize - counter % tabsize; - res.extend_from_slice(&vec![b' '; len]); - counter += len; - } else { - res.push(*i); - if *i == b'\r' || *i == b'\n' { - counter = 0; - } else { - counter += 1; - } - } - } - - res - } - - pub fn splitlines<FW, W>(&self, options: anystr::SplitLinesArgs, into_wrapper: FW) -> Vec<W> - where - FW: Fn(&[u8]) -> W, - { - self.elements.py_bytes_splitlines(options, into_wrapper) - } - - pub fn zfill(&self, width: isize) -> Vec<u8> { - self.elements.py_zfill(width) - } - - // len(self)>=1, from="", len(to)>=1, maxcount>=1 - fn replace_interleave(&self, to: PyBytesInner, maxcount: Option<usize>) -> Vec<u8> { - let place_count = self.elements.len() + 1; - let count = maxcount.map_or(place_count, |v| std::cmp::min(v, place_count)) - 1; - let capacity = self.elements.len() + count * to.len(); - let mut result = Vec::with_capacity(capacity); - let to_slice = to.elements.as_slice(); - result.extend_from_slice(to_slice); - for c in &self.elements[..count] { - result.push(*c); - result.extend_from_slice(to_slice); - } - result.extend_from_slice(&self.elements[count..]); - result - } - - fn replace_delete(&self, from: PyBytesInner, maxcount: Option<usize>) -> Vec<u8> { - let count = count_substring(self.elements.as_slice(), from.elements.as_slice(), maxcount); - if count == 0 { - // no matches - return self.elements.clone(); - } - - let result_len = self.len() - (count * from.len()); - debug_assert!(self.len() >= count * from.len()); - - let mut result = Vec::with_capacity(result_len); - let mut last_end = 0; - let mut count = count; - for offset in self.elements.find_iter(&from.elements) { - result.extend_from_slice(&self.elements[last_end..offset]); - last_end = offset + from.len(); - count -= 1; - if count == 0 { - break; - } - } - result.extend_from_slice(&self.elements[last_end..]); - result - } - - pub fn replace_in_place( - &self, - from: PyBytesInner, - to: PyBytesInner, - maxcount: Option<usize>, - ) -> Vec<u8> { - let len = from.len(); - let mut iter = self.elements.find_iter(&from.elements); - - let mut new = if let Some(offset) = iter.next() { - let mut new = self.elements.clone(); - new[offset..offset + len].clone_from_slice(to.elements.as_slice()); - if maxcount == Some(1) { - return new; - } else { - new - } - } else { - return self.elements.clone(); - }; - - let mut count = maxcount.unwrap_or(usize::MAX) - 1; - for offset in iter { - new[offset..offset + len].clone_from_slice(to.elements.as_slice()); - count -= 1; - if count == 0 { - break; - } - } - new - } - - fn replace_general( - &self, - from: PyBytesInner, - to: PyBytesInner, - maxcount: Option<usize>, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - let count = count_substring(self.elements.as_slice(), from.elements.as_slice(), maxcount); - if count == 0 { - // no matches, return unchanged - return Ok(self.elements.clone()); - } - - // Check for overflow - // result_len = self_len + count * (to_len-from_len) - debug_assert!(count > 0); - if to.len() as isize - from.len() as isize - > (isize::MAX - self.elements.len() as isize) / count as isize - { - return Err(vm.new_overflow_error("replace bytes is too long".to_owned())); - } - let result_len = (self.elements.len() as isize - + count as isize * (to.len() as isize - from.len() as isize)) - as usize; - - let mut result = Vec::with_capacity(result_len); - let mut last_end = 0; - let mut count = count; - for offset in self.elements.find_iter(&from.elements) { - result.extend_from_slice(&self.elements[last_end..offset]); - result.extend_from_slice(to.elements.as_slice()); - last_end = offset + from.len(); - count -= 1; - if count == 0 { - break; - } - } - result.extend_from_slice(&self.elements[last_end..]); - Ok(result) - } - - pub fn replace( - &self, - from: PyBytesInner, - to: PyBytesInner, - maxcount: OptionalArg<isize>, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - // stringlib_replace in CPython - let maxcount = match maxcount { - OptionalArg::Present(maxcount) if maxcount >= 0 => { - if maxcount == 0 || (self.elements.is_empty() && !from.is_empty()) { - // nothing to do; return the original bytes - return Ok(self.elements.clone()); - } else if self.elements.is_empty() && from.is_empty() { - return Ok(to.elements); - } - Some(maxcount as usize) - } - _ => None, - }; - - // Handle zero-length special cases - if from.elements.is_empty() { - if to.elements.is_empty() { - // nothing to do; return the original bytes - return Ok(self.elements.clone()); - } - // insert the 'to' bytes everywhere. - // >>> b"Python".replace(b"", b".") - // b'.P.y.t.h.o.n.' - return Ok(self.replace_interleave(to, maxcount)); - } - - // Except for b"".replace(b"", b"A") == b"A" there is no way beyond this - // point for an empty self bytes to generate a non-empty bytes - // Special case so the remaining code always gets a non-empty bytes - if self.elements.is_empty() { - return Ok(self.elements.clone()); - } - - if to.elements.is_empty() { - // delete all occurrences of 'from' bytes - Ok(self.replace_delete(from, maxcount)) - } else if from.len() == to.len() { - // Handle special case where both bytes have the same length - Ok(self.replace_in_place(from, to, maxcount)) - } else { - // Otherwise use the more generic algorithms - self.replace_general(from, to, maxcount, vm) - } - } - - pub fn title(&self) -> Vec<u8> { - let mut res = vec![]; - let mut spaced = true; - - for i in &self.elements { - match i { - 65..=90 | 97..=122 => { - if spaced { - res.push(i.to_ascii_uppercase()); - spaced = false - } else { - res.push(i.to_ascii_lowercase()); - } - } - _ => { - res.push(*i); - spaced = true - } - } - } - - res - } - - pub fn cformat(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - cformat_bytes(vm, self.elements.as_slice(), values) - } - - pub fn mul(&self, n: isize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - self.elements.mul(vm, n) - } - - pub fn imul(&mut self, n: isize, vm: &VirtualMachine) -> PyResult<()> { - self.elements.imul(vm, n) - } - - pub fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let buffer = PyBuffer::try_from_borrowed_object(vm, other)?; - let borrowed = buffer.as_contiguous(); - if let Some(other) = borrowed { - let mut v = Vec::with_capacity(self.elements.len() + other.len()); - v.extend_from_slice(&self.elements); - v.extend_from_slice(&other); - Ok(v) - } else { - let mut v = self.elements.clone(); - buffer.append_to(&mut v); - Ok(v) - } - } -} - -pub fn try_as_bytes<F, R>(obj: PyObjectRef, f: F) -> Option<R> -where - F: Fn(&[u8]) -> R, -{ - match_class!(match obj { - i @ PyBytes => Some(f(i.as_bytes())), - j @ PyByteArray => Some(f(&j.borrow_buf())), - _ => None, - }) -} - -#[inline] -fn count_substring(haystack: &[u8], needle: &[u8], maxcount: Option<usize>) -> usize { - let substrings = haystack.find_iter(needle); - if let Some(maxcount) = maxcount { - std::cmp::min(substrings.take(maxcount).count(), maxcount) - } else { - substrings.count() - } -} - -pub trait ByteOr: ToPrimitive { - fn byte_or(&self, vm: &VirtualMachine) -> PyResult<u8> { - match self.to_u8() { - Some(value) => Ok(value), - None => Err(vm.new_value_error("byte must be in range(0, 256)".to_owned())), - } - } -} - -impl ByteOr for BigInt {} - -impl AnyStrWrapper for PyBytesInner { - type Str = [u8]; - fn as_ref(&self) -> &[u8] { - &self.elements - } -} - -impl AnyStrContainer<[u8]> for Vec<u8> { - fn new() -> Self { - Vec::new() - } - - fn with_capacity(capacity: usize) -> Self { - Vec::with_capacity(capacity) - } - - fn push_str(&mut self, other: &[u8]) { - self.extend(other) - } -} - -const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; - -impl AnyStr for [u8] { - type Char = u8; - type Container = Vec<u8>; - type CharIter<'a> = bstr::Chars<'a>; - type ElementIter<'a> = std::iter::Copied<std::slice::Iter<'a, u8>>; - - fn element_bytes_len(_: u8) -> usize { - 1 - } - - fn to_container(&self) -> Self::Container { - self.to_vec() - } - - fn as_bytes(&self) -> &[u8] { - self - } - - fn as_utf8_str(&self) -> Result<&str, std::str::Utf8Error> { - std::str::from_utf8(self) - } - - fn chars(&self) -> Self::CharIter<'_> { - bstr::ByteSlice::chars(self) - } - - fn elements(&self) -> Self::ElementIter<'_> { - self.iter().copied() - } - - fn get_bytes(&self, range: std::ops::Range<usize>) -> &Self { - &self[range] - } - - fn get_chars(&self, range: std::ops::Range<usize>) -> &Self { - &self[range] - } - - fn is_empty(&self) -> bool { - Self::is_empty(self) - } - - fn bytes_len(&self) -> usize { - Self::len(self) - } - - fn py_split_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> - where - F: Fn(&Self) -> PyObjectRef, - { - let mut splits = Vec::new(); - let mut count = maxsplit; - let mut haystack = self; - while let Some(offset) = haystack.find_byteset(ASCII_WHITESPACES) { - if offset != 0 { - if count == 0 { - break; - } - splits.push(convert(&haystack[..offset])); - count -= 1; - } - haystack = &haystack[offset + 1..]; - } - if !haystack.is_empty() { - splits.push(convert(haystack)); - } - splits - } - - fn py_rsplit_whitespace<F>(&self, maxsplit: isize, convert: F) -> Vec<PyObjectRef> - where - F: Fn(&Self) -> PyObjectRef, - { - let mut splits = Vec::new(); - let mut count = maxsplit; - let mut haystack = self; - while let Some(offset) = haystack.rfind_byteset(ASCII_WHITESPACES) { - if offset + 1 != haystack.len() { - if count == 0 { - break; - } - splits.push(convert(&haystack[offset + 1..])); - count -= 1; - } - haystack = &haystack[..offset]; - } - if !haystack.is_empty() { - splits.push(convert(haystack)); - } - splits - } -} - -#[derive(FromArgs)] -pub struct DecodeArgs { - #[pyarg(any, default)] - encoding: Option<PyStrRef>, - #[pyarg(any, default)] - errors: Option<PyStrRef>, -} - -pub fn bytes_decode( - zelf: PyObjectRef, - args: DecodeArgs, - vm: &VirtualMachine, -) -> PyResult<PyStrRef> { - let DecodeArgs { encoding, errors } = args; - let encoding = encoding - .as_ref() - .map_or(crate::codecs::DEFAULT_ENCODING, |s| s.as_str()); - vm.state - .codec_registry - .decode_text(zelf, encoding, errors, vm) -} - -fn hex_impl_no_sep(bytes: &[u8]) -> String { - let mut buf: Vec<u8> = vec![0; bytes.len() * 2]; - hex::encode_to_slice(bytes, buf.as_mut_slice()).unwrap(); - unsafe { String::from_utf8_unchecked(buf) } -} - -fn hex_impl(bytes: &[u8], sep: u8, bytes_per_sep: isize) -> String { - let len = bytes.len(); - - let buf = if bytes_per_sep < 0 { - let bytes_per_sep = std::cmp::min(len, (-bytes_per_sep) as usize); - let chunks = (len - 1) / bytes_per_sep; - let chunked = chunks * bytes_per_sep; - let unchunked = len - chunked; - let mut buf = vec![0; len * 2 + chunks]; - let mut j = 0; - for i in (0..chunks).map(|i| i * bytes_per_sep) { - hex::encode_to_slice( - &bytes[i..i + bytes_per_sep], - &mut buf[j..j + bytes_per_sep * 2], - ) - .unwrap(); - j += bytes_per_sep * 2; - buf[j] = sep; - j += 1; - } - hex::encode_to_slice(&bytes[chunked..], &mut buf[j..j + unchunked * 2]).unwrap(); - buf - } else { - let bytes_per_sep = std::cmp::min(len, bytes_per_sep as usize); - let chunks = (len - 1) / bytes_per_sep; - let chunked = chunks * bytes_per_sep; - let unchunked = len - chunked; - let mut buf = vec![0; len * 2 + chunks]; - hex::encode_to_slice(&bytes[..unchunked], &mut buf[..unchunked * 2]).unwrap(); - let mut j = unchunked * 2; - for i in (0..chunks).map(|i| i * bytes_per_sep + unchunked) { - buf[j] = sep; - j += 1; - hex::encode_to_slice( - &bytes[i..i + bytes_per_sep], - &mut buf[j..j + bytes_per_sep * 2], - ) - .unwrap(); - j += bytes_per_sep * 2; - } - buf - }; - - unsafe { String::from_utf8_unchecked(buf) } -} - -pub fn bytes_to_hex( - bytes: &[u8], - sep: OptionalArg<Either<PyStrRef, PyBytesRef>>, - bytes_per_sep: OptionalArg<isize>, - vm: &VirtualMachine, -) -> PyResult<String> { - if bytes.is_empty() { - return Ok("".to_owned()); - } - - if let OptionalArg::Present(sep) = sep { - let bytes_per_sep = bytes_per_sep.unwrap_or(1); - if bytes_per_sep == 0 { - return Ok(hex_impl_no_sep(bytes)); - } - - let s_guard; - let b_guard; - let sep = match &sep { - Either::A(s) => { - s_guard = s.as_str(); - s_guard.as_bytes() - } - Either::B(bytes) => { - b_guard = bytes.as_bytes(); - b_guard - } - }; - - if sep.len() != 1 { - return Err(vm.new_value_error("sep must be length 1.".to_owned())); - } - let sep = sep[0]; - if sep > 127 { - return Err(vm.new_value_error("sep must be ASCII.".to_owned())); - } - - Ok(hex_impl(bytes, sep, bytes_per_sep)) - } else { - Ok(hex_impl_no_sep(bytes)) - } -} - -pub const fn is_py_ascii_whitespace(b: u8) -> bool { - matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B') -} diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs deleted file mode 100644 index 2ba7f112385..00000000000 --- a/vm/src/cformat.rs +++ /dev/null @@ -1,475 +0,0 @@ -//! Implementation of Printf-Style string formatting -//! as per the [Python Docs](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting). - -use crate::{ - builtins::{ - try_f64_to_bigint, tuple, PyBaseExceptionRef, PyByteArray, PyBytes, PyFloat, PyInt, PyStr, - }, - function::ArgIntoFloat, - protocol::PyBuffer, - stdlib::builtins, - AsObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine, -}; -use itertools::Itertools; -use num_traits::cast::ToPrimitive; -use rustpython_format::cformat::*; -use std::str::FromStr; - -fn spec_format_bytes( - vm: &VirtualMachine, - spec: &CFormatSpec, - obj: PyObjectRef, -) -> PyResult<Vec<u8>> { - match &spec.format_type { - CFormatType::String(conversion) => match conversion { - // Unlike strings, %r and %a are identical for bytes: the behaviour corresponds to - // %a for strings (not %r) - CFormatConversion::Repr | CFormatConversion::Ascii => { - let b = builtins::ascii(obj, vm)?.into(); - Ok(b) - } - CFormatConversion::Str | CFormatConversion::Bytes => { - if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &obj) { - Ok(buffer.contiguous_or_collect(|bytes| spec.format_bytes(bytes))) - } else { - let bytes = vm - .get_special_method(&obj, identifier!(vm, __bytes__))? - .ok_or_else(|| { - vm.new_type_error(format!( - "%b requires a bytes-like object, or an object that \ - implements __bytes__, not '{}'", - obj.class().name() - )) - })? - .invoke((), vm)?; - let bytes = PyBytes::try_from_borrowed_object(vm, &bytes)?; - Ok(spec.format_bytes(bytes.as_bytes())) - } - } - }, - CFormatType::Number(number_type) => match number_type { - CNumberType::Decimal => match_class!(match &obj { - ref i @ PyInt => { - Ok(spec.format_number(i.as_bigint()).into_bytes()) - } - ref f @ PyFloat => { - Ok(spec - .format_number(&try_f64_to_bigint(f.to_f64(), vm)?) - .into_bytes()) - } - obj => { - if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { - let result = method?.call((), vm)?; - if let Some(i) = result.payload::<PyInt>() { - return Ok(spec.format_number(i.as_bigint()).into_bytes()); - } - } - Err(vm.new_type_error(format!( - "%{} format: a number is required, not {}", - spec.format_char, - obj.class().name() - ))) - } - }), - _ => { - if let Some(i) = obj.payload::<PyInt>() { - Ok(spec.format_number(i.as_bigint()).into_bytes()) - } else { - Err(vm.new_type_error(format!( - "%{} format: an integer is required, not {}", - spec.format_char, - obj.class().name() - ))) - } - } - }, - CFormatType::Float(_) => { - let class = obj.class().to_owned(); - let value = ArgIntoFloat::try_from_object(vm, obj).map_err(|e| { - if e.fast_isinstance(vm.ctx.exceptions.type_error) { - // formatfloat in bytesobject.c generates its own specific exception - // text in this case, mirror it here. - vm.new_type_error(format!("float argument required, not {}", class.name())) - } else { - e - } - })?; - Ok(spec.format_float(value.into()).into_bytes()) - } - CFormatType::Character => { - if let Some(i) = obj.payload::<PyInt>() { - let ch = i - .try_to_primitive::<u8>(vm) - .map_err(|_| vm.new_overflow_error("%c arg not in range(256)".to_owned()))? - as char; - return Ok(spec.format_char(ch).into_bytes()); - } - if let Some(b) = obj.payload::<PyBytes>() { - if b.len() == 1 { - return Ok(spec.format_char(b.as_bytes()[0] as char).into_bytes()); - } - } else if let Some(ba) = obj.payload::<PyByteArray>() { - let buf = ba.borrow_buf(); - if buf.len() == 1 { - return Ok(spec.format_char(buf[0] as char).into_bytes()); - } - } - Err(vm - .new_type_error("%c requires an integer in range(256) or a single byte".to_owned())) - } - } -} - -fn spec_format_string( - vm: &VirtualMachine, - spec: &CFormatSpec, - obj: PyObjectRef, - idx: &usize, -) -> PyResult<String> { - match &spec.format_type { - CFormatType::String(conversion) => { - let result = match conversion { - CFormatConversion::Ascii => builtins::ascii(obj, vm)?.into(), - CFormatConversion::Str => obj.str(vm)?.as_str().to_owned(), - CFormatConversion::Repr => obj.repr(vm)?.as_str().to_owned(), - CFormatConversion::Bytes => { - // idx is the position of the %, we want the position of the b - return Err(vm.new_value_error(format!( - "unsupported format character 'b' (0x62) at index {}", - idx + 1 - ))); - } - }; - Ok(spec.format_string(result)) - } - CFormatType::Number(number_type) => match number_type { - CNumberType::Decimal => match_class!(match &obj { - ref i @ PyInt => { - Ok(spec.format_number(i.as_bigint())) - } - ref f @ PyFloat => { - Ok(spec.format_number(&try_f64_to_bigint(f.to_f64(), vm)?)) - } - obj => { - if let Some(method) = vm.get_method(obj.clone(), identifier!(vm, __int__)) { - let result = method?.call((), vm)?; - if let Some(i) = result.payload::<PyInt>() { - return Ok(spec.format_number(i.as_bigint())); - } - } - Err(vm.new_type_error(format!( - "%{} format: a number is required, not {}", - spec.format_char, - obj.class().name() - ))) - } - }), - _ => { - if let Some(i) = obj.payload::<PyInt>() { - Ok(spec.format_number(i.as_bigint())) - } else { - Err(vm.new_type_error(format!( - "%{} format: an integer is required, not {}", - spec.format_char, - obj.class().name() - ))) - } - } - }, - CFormatType::Float(_) => { - let value = ArgIntoFloat::try_from_object(vm, obj)?; - Ok(spec.format_float(value.into())) - } - CFormatType::Character => { - if let Some(i) = obj.payload::<PyInt>() { - let ch = i - .as_bigint() - .to_u32() - .and_then(std::char::from_u32) - .ok_or_else(|| { - vm.new_overflow_error("%c arg not in range(0x110000)".to_owned()) - })?; - return Ok(spec.format_char(ch)); - } - if let Some(s) = obj.payload::<PyStr>() { - if let Ok(ch) = s.as_str().chars().exactly_one() { - return Ok(spec.format_char(ch)); - } - } - Err(vm.new_type_error("%c requires int or char".to_owned())) - } - } -} - -fn try_update_quantity_from_element( - vm: &VirtualMachine, - element: Option<&PyObjectRef>, -) -> PyResult<CFormatQuantity> { - match element { - Some(width_obj) => { - if let Some(i) = width_obj.payload::<PyInt>() { - let i = i.try_to_primitive::<i32>(vm)?.unsigned_abs(); - Ok(CFormatQuantity::Amount(i as usize)) - } else { - Err(vm.new_type_error("* wants int".to_owned())) - } - } - None => Err(vm.new_type_error("not enough arguments for format string".to_owned())), - } -} - -fn try_conversion_flag_from_tuple( - vm: &VirtualMachine, - element: Option<&PyObjectRef>, -) -> PyResult<CConversionFlags> { - match element { - Some(width_obj) => { - if let Some(i) = width_obj.payload::<PyInt>() { - let i = i.try_to_primitive::<i32>(vm)?; - let flags = if i < 0 { - CConversionFlags::LEFT_ADJUST - } else { - CConversionFlags::from_bits(0).unwrap() - }; - Ok(flags) - } else { - Err(vm.new_type_error("* wants int".to_owned())) - } - } - None => Err(vm.new_type_error("not enough arguments for format string".to_owned())), - } -} - -fn try_update_quantity_from_tuple<'a, I: Iterator<Item = &'a PyObjectRef>>( - vm: &VirtualMachine, - elements: &mut I, - q: &mut Option<CFormatQuantity>, - f: &mut CConversionFlags, -) -> PyResult<()> { - let Some(CFormatQuantity::FromValuesTuple) = q else { - return Ok(()); - }; - let element = elements.next(); - f.insert(try_conversion_flag_from_tuple(vm, element)?); - let quantity = try_update_quantity_from_element(vm, element)?; - *q = Some(quantity); - Ok(()) -} - -fn try_update_precision_from_tuple<'a, I: Iterator<Item = &'a PyObjectRef>>( - vm: &VirtualMachine, - elements: &mut I, - p: &mut Option<CFormatPrecision>, -) -> PyResult<()> { - let Some(CFormatPrecision::Quantity(CFormatQuantity::FromValuesTuple)) = p else { - return Ok(()); - }; - let quantity = try_update_quantity_from_element(vm, elements.next())?; - *p = Some(CFormatPrecision::Quantity(quantity)); - Ok(()) -} - -fn specifier_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_type_error("format requires a mapping".to_owned()) -} - -pub(crate) fn cformat_bytes( - vm: &VirtualMachine, - format_string: &[u8], - values_obj: PyObjectRef, -) -> PyResult<Vec<u8>> { - let mut format = CFormatBytes::parse_from_bytes(format_string) - .map_err(|err| vm.new_value_error(err.to_string()))?; - let (num_specifiers, mapping_required) = format - .check_specifiers() - .ok_or_else(|| specifier_error(vm))?; - - let mut result = vec![]; - - let is_mapping = values_obj.class().has_attr(identifier!(vm, __getitem__)) - && !values_obj.fast_isinstance(vm.ctx.types.tuple_type) - && !values_obj.fast_isinstance(vm.ctx.types.bytes_type) - && !values_obj.fast_isinstance(vm.ctx.types.bytearray_type); - - if num_specifiers == 0 { - // literal only - return if is_mapping - || values_obj - .payload::<tuple::PyTuple>() - .map_or(false, |e| e.is_empty()) - { - for (_, part) in format.iter_mut() { - match part { - CFormatPart::Literal(literal) => result.append(literal), - CFormatPart::Spec(_) => unreachable!(), - } - } - Ok(result) - } else { - Err(vm.new_type_error("not all arguments converted during bytes formatting".to_owned())) - }; - } - - if mapping_required { - // dict - return if is_mapping { - for (_, part) in format.iter_mut() { - match part { - CFormatPart::Literal(literal) => result.append(literal), - CFormatPart::Spec(spec) => { - let value = match &spec.mapping_key { - Some(key) => { - let k = vm.ctx.new_bytes(key.as_str().as_bytes().to_vec()); - values_obj.get_item(k.as_object(), vm)? - } - None => unreachable!(), - }; - let mut part_result = spec_format_bytes(vm, spec, value)?; - result.append(&mut part_result); - } - } - } - Ok(result) - } else { - Err(vm.new_type_error("format requires a mapping".to_owned())) - }; - } - - // tuple - let values = if let Some(tup) = values_obj.payload_if_subclass::<tuple::PyTuple>(vm) { - tup.as_slice() - } else { - std::slice::from_ref(&values_obj) - }; - let mut value_iter = values.iter(); - - for (_, part) in format.iter_mut() { - match part { - CFormatPart::Literal(literal) => result.append(literal), - CFormatPart::Spec(spec) => { - try_update_quantity_from_tuple( - vm, - &mut value_iter, - &mut spec.min_field_width, - &mut spec.flags, - )?; - try_update_precision_from_tuple(vm, &mut value_iter, &mut spec.precision)?; - - let value = match value_iter.next() { - Some(obj) => Ok(obj.clone()), - None => { - Err(vm.new_type_error("not enough arguments for format string".to_owned())) - } - }?; - let mut part_result = spec_format_bytes(vm, spec, value)?; - result.append(&mut part_result); - } - } - } - - // check that all arguments were converted - if value_iter.next().is_some() && !is_mapping { - Err(vm.new_type_error("not all arguments converted during bytes formatting".to_owned())) - } else { - Ok(result) - } -} - -pub(crate) fn cformat_string( - vm: &VirtualMachine, - format_string: &str, - values_obj: PyObjectRef, -) -> PyResult<String> { - let mut format = CFormatString::from_str(format_string) - .map_err(|err| vm.new_value_error(err.to_string()))?; - let (num_specifiers, mapping_required) = format - .check_specifiers() - .ok_or_else(|| specifier_error(vm))?; - - let mut result = String::new(); - - let is_mapping = values_obj.class().has_attr(identifier!(vm, __getitem__)) - && !values_obj.fast_isinstance(vm.ctx.types.tuple_type) - && !values_obj.fast_isinstance(vm.ctx.types.str_type); - - if num_specifiers == 0 { - // literal only - return if is_mapping - || values_obj - .payload::<tuple::PyTuple>() - .map_or(false, |e| e.is_empty()) - { - for (_, part) in format.iter() { - match part { - CFormatPart::Literal(literal) => result.push_str(literal), - CFormatPart::Spec(_) => unreachable!(), - } - } - Ok(result) - } else { - Err(vm - .new_type_error("not all arguments converted during string formatting".to_owned())) - }; - } - - if mapping_required { - // dict - return if is_mapping { - for (idx, part) in format.iter() { - match part { - CFormatPart::Literal(literal) => result.push_str(literal), - CFormatPart::Spec(spec) => { - let value = match &spec.mapping_key { - Some(key) => values_obj.get_item(key.as_str(), vm)?, - None => unreachable!(), - }; - let part_result = spec_format_string(vm, spec, value, idx)?; - result.push_str(&part_result); - } - } - } - Ok(result) - } else { - Err(vm.new_type_error("format requires a mapping".to_owned())) - }; - } - - // tuple - let values = if let Some(tup) = values_obj.payload_if_subclass::<tuple::PyTuple>(vm) { - tup.as_slice() - } else { - std::slice::from_ref(&values_obj) - }; - let mut value_iter = values.iter(); - - for (idx, part) in format.iter_mut() { - match part { - CFormatPart::Literal(literal) => result.push_str(literal), - CFormatPart::Spec(spec) => { - try_update_quantity_from_tuple( - vm, - &mut value_iter, - &mut spec.min_field_width, - &mut spec.flags, - )?; - try_update_precision_from_tuple(vm, &mut value_iter, &mut spec.precision)?; - - let value = match value_iter.next() { - Some(obj) => Ok(obj.clone()), - None => { - Err(vm.new_type_error("not enough arguments for format string".to_owned())) - } - }?; - let part_result = spec_format_string(vm, spec, value, idx)?; - result.push_str(&part_result); - } - } - } - - // check that all arguments were converted - if value_iter.next().is_some() && !is_mapping { - Err(vm.new_type_error("not all arguments converted during string formatting".to_owned())) - } else { - Ok(result) - } -} diff --git a/vm/src/class.rs b/vm/src/class.rs deleted file mode 100644 index a2eea21213d..00000000000 --- a/vm/src/class.rs +++ /dev/null @@ -1,158 +0,0 @@ -//! Utilities to define a new Python class - -use crate::{ - builtins::{PyBaseObject, PyType, PyTypeRef}, - function::PyMethodDef, - identifier, - object::Py, - types::{hash_not_implemented, PyTypeFlags, PyTypeSlots}, - vm::Context, -}; -use rustpython_common::static_cell; - -pub trait StaticType { - // Ideally, saving PyType is better than PyTypeRef - fn static_cell() -> &'static static_cell::StaticCell<PyTypeRef>; - fn static_metaclass() -> &'static Py<PyType> { - PyType::static_type() - } - fn static_baseclass() -> &'static Py<PyType> { - PyBaseObject::static_type() - } - fn static_type() -> &'static Py<PyType> { - Self::static_cell() - .get() - .expect("static type has not been initialized") - } - fn init_manually(typ: PyTypeRef) -> &'static Py<PyType> { - let cell = Self::static_cell(); - cell.set(typ) - .unwrap_or_else(|_| panic!("double initialization from init_manually")); - cell.get().unwrap() - } - fn init_builtin_type() -> &'static Py<PyType> - where - Self: PyClassImpl, - { - let typ = Self::create_static_type(); - let cell = Self::static_cell(); - cell.set(typ) - .unwrap_or_else(|_| panic!("double initialization of {}", Self::NAME)); - cell.get().unwrap() - } - fn create_static_type() -> PyTypeRef - where - Self: PyClassImpl, - { - PyType::new_static( - Self::static_baseclass().to_owned(), - Default::default(), - Self::make_slots(), - Self::static_metaclass().to_owned(), - ) - .unwrap() - } -} - -pub trait PyClassDef { - const NAME: &'static str; - const MODULE_NAME: Option<&'static str>; - const TP_NAME: &'static str; - const DOC: Option<&'static str> = None; - const BASICSIZE: usize; - const UNHASHABLE: bool = false; - - // due to restriction of rust trait system, object.__base__ is None - // but PyBaseObject::Base will be PyBaseObject. - type Base: PyClassDef; -} - -pub trait PyClassImpl: PyClassDef { - const TP_FLAGS: PyTypeFlags = PyTypeFlags::DEFAULT; - - fn extend_class(ctx: &Context, class: &'static Py<PyType>) { - #[cfg(debug_assertions)] - { - assert!(class.slots.flags.is_created_with_flags()); - } - - let _ = ctx.intern_str(Self::NAME); // intern type name - - if Self::TP_FLAGS.has_feature(PyTypeFlags::HAS_DICT) { - let __dict__ = identifier!(ctx, __dict__); - class.set_attr( - __dict__, - ctx.new_getset( - "__dict__", - class, - crate::builtins::object::object_get_dict, - crate::builtins::object::object_set_dict, - ) - .into(), - ); - } - Self::impl_extend_class(ctx, class); - if let Some(doc) = Self::DOC { - class.set_attr(identifier!(ctx, __doc__), ctx.new_str(doc).into()); - } - if let Some(module_name) = Self::MODULE_NAME { - class.set_attr( - identifier!(ctx, __module__), - ctx.new_str(module_name).into(), - ); - } - - if class.slots.new.load().is_some() { - let bound_new = Context::genesis().slot_new_wrapper.build_bound_method( - ctx, - class.to_owned().into(), - class, - ); - class.set_attr(identifier!(ctx, __new__), bound_new.into()); - } - - if class.slots.hash.load().map_or(0, |h| h as usize) == hash_not_implemented as usize { - class.set_attr(ctx.names.__hash__, ctx.none.clone().into()); - } - - class.extend_methods(class.slots.methods, ctx); - } - - fn make_class(ctx: &Context) -> PyTypeRef - where - Self: StaticType, - { - (*Self::static_cell().get_or_init(|| { - let typ = Self::create_static_type(); - Self::extend_class(ctx, unsafe { - // typ will be saved in static_cell - let r: &Py<PyType> = &typ; - &*(r as *const _) - }); - typ - })) - .to_owned() - } - - fn impl_extend_class(ctx: &Context, class: &'static Py<PyType>); - const METHOD_DEFS: &'static [PyMethodDef]; - fn extend_slots(slots: &mut PyTypeSlots); - - fn make_slots() -> PyTypeSlots { - let mut slots = PyTypeSlots { - flags: Self::TP_FLAGS, - name: Self::TP_NAME, - basicsize: Self::BASICSIZE, - doc: Self::DOC, - methods: Self::METHOD_DEFS, - ..Default::default() - }; - - if Self::UNHASHABLE { - slots.hash.store(Some(hash_not_implemented)); - } - - Self::extend_slots(&mut slots); - slots - } -} diff --git a/vm/src/codecs.rs b/vm/src/codecs.rs deleted file mode 100644 index ff7bc489157..00000000000 --- a/vm/src/codecs.rs +++ /dev/null @@ -1,717 +0,0 @@ -use crate::{ - builtins::{PyBaseExceptionRef, PyBytesRef, PyStr, PyStrRef, PyTuple, PyTupleRef}, - common::{ascii, lock::PyRwLock}, - convert::ToPyObject, - function::PyMethodDef, - AsObject, Context, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, -}; -use std::{borrow::Cow, collections::HashMap, fmt::Write, ops::Range}; - -pub struct CodecsRegistry { - inner: PyRwLock<RegistryInner>, -} - -struct RegistryInner { - search_path: Vec<PyObjectRef>, - search_cache: HashMap<String, PyCodec>, - errors: HashMap<String, PyObjectRef>, -} - -pub const DEFAULT_ENCODING: &str = "utf-8"; - -#[derive(Clone)] -#[repr(transparent)] -pub struct PyCodec(PyTupleRef); -impl PyCodec { - #[inline] - pub fn from_tuple(tuple: PyTupleRef) -> Result<Self, PyTupleRef> { - if tuple.len() == 4 { - Ok(PyCodec(tuple)) - } else { - Err(tuple) - } - } - #[inline] - pub fn into_tuple(self) -> PyTupleRef { - self.0 - } - #[inline] - pub fn as_tuple(&self) -> &PyTupleRef { - &self.0 - } - - #[inline] - pub fn get_encode_func(&self) -> &PyObject { - &self.0[0] - } - #[inline] - pub fn get_decode_func(&self) -> &PyObject { - &self.0[1] - } - - pub fn is_text_codec(&self, vm: &VirtualMachine) -> PyResult<bool> { - let is_text = vm.get_attribute_opt(self.0.clone().into(), "_is_text_encoding")?; - is_text.map_or(Ok(true), |is_text| is_text.try_to_bool(vm)) - } - - pub fn encode( - &self, - obj: PyObjectRef, - errors: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult { - let args = match errors { - Some(errors) => vec![obj, errors.into()], - None => vec![obj], - }; - let res = self.get_encode_func().call(args, vm)?; - let res = res - .downcast::<PyTuple>() - .ok() - .filter(|tuple| tuple.len() == 2) - .ok_or_else(|| { - vm.new_type_error("encoder must return a tuple (object, integer)".to_owned()) - })?; - // we don't actually care about the integer - Ok(res[0].clone()) - } - - pub fn decode( - &self, - obj: PyObjectRef, - errors: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult { - let args = match errors { - Some(errors) => vec![obj, errors.into()], - None => vec![obj], - }; - let res = self.get_decode_func().call(args, vm)?; - let res = res - .downcast::<PyTuple>() - .ok() - .filter(|tuple| tuple.len() == 2) - .ok_or_else(|| { - vm.new_type_error("decoder must return a tuple (object,integer)".to_owned()) - })?; - // we don't actually care about the integer - Ok(res[0].clone()) - } - - pub fn get_incremental_encoder( - &self, - errors: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult { - let args = match errors { - Some(e) => vec![e.into()], - None => vec![], - }; - vm.call_method(self.0.as_object(), "incrementalencoder", args) - } - - pub fn get_incremental_decoder( - &self, - errors: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult { - let args = match errors { - Some(e) => vec![e.into()], - None => vec![], - }; - vm.call_method(self.0.as_object(), "incrementaldecoder", args) - } -} - -impl TryFromObject for PyCodec { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - obj.downcast::<PyTuple>() - .ok() - .and_then(|tuple| PyCodec::from_tuple(tuple).ok()) - .ok_or_else(|| { - vm.new_type_error("codec search functions must return 4-tuples".to_owned()) - }) - } -} - -impl ToPyObject for PyCodec { - #[inline] - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.0.into() - } -} - -impl CodecsRegistry { - pub(crate) fn new(ctx: &Context) -> Self { - ::rustpython_vm::common::static_cell! { - static METHODS: Box<[PyMethodDef]>; - } - - let methods = METHODS.get_or_init(|| { - crate::define_methods![ - "strict_errors" => strict_errors as EMPTY, - "ignore_errors" => ignore_errors as EMPTY, - "replace_errors" => replace_errors as EMPTY, - "xmlcharrefreplace_errors" => xmlcharrefreplace_errors as EMPTY, - "backslashreplace_errors" => backslashreplace_errors as EMPTY, - "namereplace_errors" => namereplace_errors as EMPTY, - "surrogatepass_errors" => surrogatepass_errors as EMPTY, - "surrogateescape_errors" => surrogateescape_errors as EMPTY - ] - .into_boxed_slice() - }); - - let errors = [ - ("strict", methods[0].build_function(ctx)), - ("ignore", methods[1].build_function(ctx)), - ("replace", methods[2].build_function(ctx)), - ("xmlcharrefreplace", methods[3].build_function(ctx)), - ("backslashreplace", methods[4].build_function(ctx)), - ("namereplace", methods[5].build_function(ctx)), - ("surrogatepass", methods[6].build_function(ctx)), - ("surrogateescape", methods[7].build_function(ctx)), - ]; - let errors = errors - .into_iter() - .map(|(name, f)| (name.to_owned(), f.into())) - .collect(); - let inner = RegistryInner { - search_path: Vec::new(), - search_cache: HashMap::new(), - errors, - }; - CodecsRegistry { - inner: PyRwLock::new(inner), - } - } - - pub fn register(&self, search_function: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if !search_function.is_callable() { - return Err(vm.new_type_error("argument must be callable".to_owned())); - } - self.inner.write().search_path.push(search_function); - Ok(()) - } - - pub fn unregister(&self, search_function: PyObjectRef) -> PyResult<()> { - let mut inner = self.inner.write(); - // Do nothing if search_path is not created yet or was cleared. - if inner.search_path.is_empty() { - return Ok(()); - } - for (i, item) in inner.search_path.iter().enumerate() { - if item.get_id() == search_function.get_id() { - if !inner.search_cache.is_empty() { - inner.search_cache.clear(); - } - inner.search_path.remove(i); - return Ok(()); - } - } - Ok(()) - } - - pub(crate) fn register_manual(&self, name: &str, codec: PyCodec) -> PyResult<()> { - self.inner - .write() - .search_cache - .insert(name.to_owned(), codec); - Ok(()) - } - - pub fn lookup(&self, encoding: &str, vm: &VirtualMachine) -> PyResult<PyCodec> { - let encoding = normalize_encoding_name(encoding); - let search_path = { - let inner = self.inner.read(); - if let Some(codec) = inner.search_cache.get(encoding.as_ref()) { - // hit cache - return Ok(codec.clone()); - } - inner.search_path.clone() - }; - let encoding = PyStr::from(encoding.into_owned()).into_ref(&vm.ctx); - for func in search_path { - let res = func.call((encoding.clone(),), vm)?; - let res: Option<PyCodec> = res.try_into_value(vm)?; - if let Some(codec) = res { - let mut inner = self.inner.write(); - // someone might have raced us to this, so use theirs - let codec = inner - .search_cache - .entry(encoding.as_str().to_owned()) - .or_insert(codec); - return Ok(codec.clone()); - } - } - Err(vm.new_lookup_error(format!("unknown encoding: {encoding}"))) - } - - fn _lookup_text_encoding( - &self, - encoding: &str, - generic_func: &str, - vm: &VirtualMachine, - ) -> PyResult<PyCodec> { - let codec = self.lookup(encoding, vm)?; - if codec.is_text_codec(vm)? { - Ok(codec) - } else { - Err(vm.new_lookup_error(format!( - "'{encoding}' is not a text encoding; use {generic_func} to handle arbitrary codecs" - ))) - } - } - - pub fn forget(&self, encoding: &str) -> Option<PyCodec> { - let encoding = normalize_encoding_name(encoding); - self.inner.write().search_cache.remove(encoding.as_ref()) - } - - pub fn encode( - &self, - obj: PyObjectRef, - encoding: &str, - errors: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult { - let codec = self.lookup(encoding, vm)?; - codec.encode(obj, errors, vm) - } - - pub fn decode( - &self, - obj: PyObjectRef, - encoding: &str, - errors: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult { - let codec = self.lookup(encoding, vm)?; - codec.decode(obj, errors, vm) - } - - pub fn encode_text( - &self, - obj: PyStrRef, - encoding: &str, - errors: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<PyBytesRef> { - let codec = self._lookup_text_encoding(encoding, "codecs.encode()", vm)?; - codec - .encode(obj.into(), errors, vm)? - .downcast() - .map_err(|obj| { - vm.new_type_error(format!( - "'{}' encoder returned '{}' instead of 'bytes'; use codecs.encode() to \ - encode arbitrary types", - encoding, - obj.class().name(), - )) - }) - } - - pub fn decode_text( - &self, - obj: PyObjectRef, - encoding: &str, - errors: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<PyStrRef> { - let codec = self._lookup_text_encoding(encoding, "codecs.decode()", vm)?; - codec.decode(obj, errors, vm)?.downcast().map_err(|obj| { - vm.new_type_error(format!( - "'{}' decoder returned '{}' instead of 'str'; use codecs.decode() \ - to encode arbitrary types", - encoding, - obj.class().name(), - )) - }) - } - - pub fn register_error(&self, name: String, handler: PyObjectRef) -> Option<PyObjectRef> { - self.inner.write().errors.insert(name, handler) - } - - pub fn lookup_error_opt(&self, name: &str) -> Option<PyObjectRef> { - self.inner.read().errors.get(name).cloned() - } - - pub fn lookup_error(&self, name: &str, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - self.lookup_error_opt(name) - .ok_or_else(|| vm.new_lookup_error(format!("unknown error handler name '{name}'"))) - } -} - -fn normalize_encoding_name(encoding: &str) -> Cow<'_, str> { - if let Some(i) = encoding.find(|c: char| c == ' ' || c.is_ascii_uppercase()) { - let mut out = encoding.as_bytes().to_owned(); - for byte in &mut out[i..] { - if *byte == b' ' { - *byte = b'-'; - } else { - byte.make_ascii_lowercase(); - } - } - String::from_utf8(out).unwrap().into() - } else { - encoding.into() - } -} - -// TODO: exceptions with custom payloads -fn extract_unicode_error_range(err: &PyObject, vm: &VirtualMachine) -> PyResult<Range<usize>> { - let start = err.get_attr("start", vm)?; - let start = start.try_into_value(vm)?; - let end = err.get_attr("end", vm)?; - let end = end.try_into_value(vm)?; - Ok(Range { start, end }) -} - -#[inline] -fn is_decode_err(err: &PyObject, vm: &VirtualMachine) -> bool { - err.fast_isinstance(vm.ctx.exceptions.unicode_decode_error) -} -#[inline] -fn is_encode_ish_err(err: &PyObject, vm: &VirtualMachine) -> bool { - err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) - || err.fast_isinstance(vm.ctx.exceptions.unicode_translate_error) -} - -fn bad_err_type(err: PyObjectRef, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_type_error(format!( - "don't know how to handle {} in error callback", - err.class().name() - )) -} - -fn strict_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let err = err - .downcast() - .unwrap_or_else(|_| vm.new_type_error("codec must pass exception instance".to_owned())); - Err(err) -} - -fn ignore_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { - if is_encode_ish_err(&err, vm) || is_decode_err(&err, vm) { - let range = extract_unicode_error_range(&err, vm)?; - Ok((vm.ctx.new_str(ascii!("")).into(), range.end)) - } else { - Err(bad_err_type(err, vm)) - } -} - -fn replace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { - // char::REPLACEMENT_CHARACTER as a str - let replacement_char = "\u{FFFD}"; - let replace = if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { - "?" - } else if err.fast_isinstance(vm.ctx.exceptions.unicode_decode_error) { - let range = extract_unicode_error_range(&err, vm)?; - return Ok((replacement_char.to_owned(), range.end)); - } else if err.fast_isinstance(vm.ctx.exceptions.unicode_translate_error) { - replacement_char - } else { - return Err(bad_err_type(err, vm)); - }; - let range = extract_unicode_error_range(&err, vm)?; - let replace = replace.repeat(range.end - range.start); - Ok((replace, range.end)) -} - -fn xmlcharrefreplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { - if !is_encode_ish_err(&err, vm) { - return Err(bad_err_type(err, vm)); - } - let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); - let num_chars = range.len(); - // capacity rough guess; assuming that the codepoints are 3 digits in decimal + the &#; - let mut out = String::with_capacity(num_chars * 6); - for c in s_after_start.chars().take(num_chars) { - write!(out, "&#{};", c as u32).unwrap() - } - Ok((out, range.end)) -} - -fn backslashreplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { - if is_decode_err(&err, vm) { - let range = extract_unicode_error_range(&err, vm)?; - let b = PyBytesRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let mut replace = String::with_capacity(4 * range.len()); - for &c in &b[range.clone()] { - write!(replace, "\\x{c:02x}").unwrap(); - } - return Ok((replace, range.end)); - } else if !is_encode_ish_err(&err, vm) { - return Err(bad_err_type(err, vm)); - } - let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); - let num_chars = range.len(); - // minimum 4 output bytes per char: \xNN - let mut out = String::with_capacity(num_chars * 4); - for c in s_after_start.chars().take(num_chars) { - let c = c as u32; - if c >= 0x10000 { - write!(out, "\\U{c:08x}").unwrap(); - } else if c >= 0x100 { - write!(out, "\\u{c:04x}").unwrap(); - } else { - write!(out, "\\x{c:02x}").unwrap(); - } - } - Ok((out, range.end)) -} - -fn namereplace_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(String, usize)> { - if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { - let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = - crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); - let num_chars = range.len(); - let mut out = String::with_capacity(num_chars * 4); - for c in s_after_start.chars().take(num_chars) { - let c_u32 = c as u32; - if let Some(c_name) = unicode_names2::name(c) { - write!(out, "\\N{{{c_name}}}").unwrap(); - } else if c_u32 >= 0x10000 { - write!(out, "\\U{c_u32:08x}").unwrap(); - } else if c_u32 >= 0x100 { - write!(out, "\\u{c_u32:04x}").unwrap(); - } else { - write!(out, "\\x{c_u32:02x}").unwrap(); - } - } - Ok((out, range.end)) - } else { - Err(bad_err_type(err, vm)) - } -} - -#[derive(Eq, PartialEq)] -enum StandardEncoding { - Utf8, - Utf16Be, - Utf16Le, - Utf32Be, - Utf32Le, - Unknown, -} - -fn get_standard_encoding(encoding: &str) -> (usize, StandardEncoding) { - if let Some(encoding) = encoding.to_lowercase().strip_prefix("utf") { - let mut byte_length: usize = 0; - let mut standard_encoding = StandardEncoding::Unknown; - let encoding = encoding - .strip_prefix(|c| ['-', '_'].contains(&c)) - .unwrap_or(encoding); - if encoding == "8" { - byte_length = 3; - standard_encoding = StandardEncoding::Utf8; - } else if let Some(encoding) = encoding.strip_prefix("16") { - byte_length = 2; - if encoding.is_empty() { - if cfg!(target_endian = "little") { - standard_encoding = StandardEncoding::Utf16Le; - } else if cfg!(target_endian = "big") { - standard_encoding = StandardEncoding::Utf16Be; - } - if standard_encoding != StandardEncoding::Unknown { - return (byte_length, standard_encoding); - } - } - let encoding = encoding - .strip_prefix(|c| ['-', '_'].contains(&c)) - .unwrap_or(encoding); - standard_encoding = match encoding { - "be" => StandardEncoding::Utf16Be, - "le" => StandardEncoding::Utf16Le, - _ => StandardEncoding::Unknown, - } - } else if let Some(encoding) = encoding.strip_prefix("32") { - byte_length = 4; - if encoding.is_empty() { - if cfg!(target_endian = "little") { - standard_encoding = StandardEncoding::Utf32Le; - } else if cfg!(target_endian = "big") { - standard_encoding = StandardEncoding::Utf32Be; - } - if standard_encoding != StandardEncoding::Unknown { - return (byte_length, standard_encoding); - } - } - let encoding = encoding - .strip_prefix(|c| ['-', '_'].contains(&c)) - .unwrap_or(encoding); - standard_encoding = match encoding { - "be" => StandardEncoding::Utf32Be, - "le" => StandardEncoding::Utf32Le, - _ => StandardEncoding::Unknown, - } - } - return (byte_length, standard_encoding); - } else if encoding == "CP_UTF8" { - return (3, StandardEncoding::Utf8); - } - (0, StandardEncoding::Unknown) -} - -fn surrogatepass_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { - if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { - let range = extract_unicode_error_range(&err, vm)?; - let s = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_encoding = PyStrRef::try_from_object(vm, err.get_attr("encoding", vm)?)?; - let (_, standard_encoding) = get_standard_encoding(s_encoding.as_str()); - if let StandardEncoding::Unknown = standard_encoding { - // Not supported, fail with original exception - return Err(err.downcast().unwrap()); - } - let s_after_start = - crate::common::str::try_get_chars(s.as_str(), range.start..).unwrap_or(""); - let num_chars = range.len(); - let mut out: Vec<u8> = Vec::with_capacity(num_chars * 4); - for c in s_after_start.chars().take(num_chars).map(|x| x as u32) { - if !(0xd800..=0xdfff).contains(&c) { - // Not a surrogate, fail with original exception - return Err(err.downcast().unwrap()); - } - match standard_encoding { - StandardEncoding::Utf8 => { - out.push((0xe0 | (c >> 12)) as u8); - out.push((0x80 | ((c >> 6) & 0x3f)) as u8); - out.push((0x80 | (c & 0x3f)) as u8); - } - StandardEncoding::Utf16Le => { - out.push(c as u8); - out.push((c >> 8) as u8); - } - StandardEncoding::Utf16Be => { - out.push((c >> 8) as u8); - out.push(c as u8); - } - StandardEncoding::Utf32Le => { - out.push(c as u8); - out.push((c >> 8) as u8); - out.push((c >> 16) as u8); - out.push((c >> 24) as u8); - } - StandardEncoding::Utf32Be => { - out.push((c >> 24) as u8); - out.push((c >> 16) as u8); - out.push((c >> 8) as u8); - out.push(c as u8); - } - StandardEncoding::Unknown => { - unreachable!("NOTE: RUSTPYTHON, should've bailed out earlier") - } - } - } - Ok((vm.ctx.new_bytes(out).into(), range.end)) - } else if is_decode_err(&err, vm) { - let range = extract_unicode_error_range(&err, vm)?; - let s = PyBytesRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_encoding = PyStrRef::try_from_object(vm, err.get_attr("encoding", vm)?)?; - let (byte_length, standard_encoding) = get_standard_encoding(s_encoding.as_str()); - if let StandardEncoding::Unknown = standard_encoding { - // Not supported, fail with original exception - return Err(err.downcast().unwrap()); - } - let mut c: u32 = 0; - // Try decoding a single surrogate character. If there are more, - // let the codec call us again. - let p = &s.as_bytes()[range.start..]; - if p.len() - range.start >= byte_length { - match standard_encoding { - StandardEncoding::Utf8 => { - if (p[0] as u32 & 0xf0) == 0xe0 - && (p[1] as u32 & 0xc0) == 0x80 - && (p[2] as u32 & 0xc0) == 0x80 - { - // it's a three-byte code - c = ((p[0] as u32 & 0x0f) << 12) - + ((p[1] as u32 & 0x3f) << 6) - + (p[2] as u32 & 0x3f); - } - } - StandardEncoding::Utf16Le => { - c = (p[1] as u32) << 8 | p[0] as u32; - } - StandardEncoding::Utf16Be => { - c = (p[0] as u32) << 8 | p[1] as u32; - } - StandardEncoding::Utf32Le => { - c = ((p[3] as u32) << 24) - | ((p[2] as u32) << 16) - | ((p[1] as u32) << 8) - | p[0] as u32; - } - StandardEncoding::Utf32Be => { - c = ((p[0] as u32) << 24) - | ((p[1] as u32) << 16) - | ((p[2] as u32) << 8) - | p[3] as u32; - } - StandardEncoding::Unknown => { - unreachable!("NOTE: RUSTPYTHON, should've bailed out earlier") - } - } - } - // !Py_UNICODE_IS_SURROGATE - if !(0xd800..=0xdfff).contains(&c) { - // Not a surrogate, fail with original exception - return Err(err.downcast().unwrap()); - } - - Ok(( - vm.new_pyobj(format!("\\x{c:x?}")), - range.start + byte_length, - )) - } else { - Err(bad_err_type(err, vm)) - } -} - -fn surrogateescape_errors(err: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyObjectRef, usize)> { - if err.fast_isinstance(vm.ctx.exceptions.unicode_encode_error) { - let range = extract_unicode_error_range(&err, vm)?; - let object = PyStrRef::try_from_object(vm, err.get_attr("object", vm)?)?; - let s_after_start = - crate::common::str::try_get_chars(object.as_str(), range.start..).unwrap_or(""); - let mut out: Vec<u8> = Vec::with_capacity(range.len()); - for ch in s_after_start.chars().take(range.len()) { - let ch = ch as u32; - if !(0xdc80..=0xdcff).contains(&ch) { - // Not a UTF-8b surrogate, fail with original exception - return Err(err.downcast().unwrap()); - } - out.push((ch - 0xdc00) as u8); - } - let out = vm.ctx.new_bytes(out); - Ok((out.into(), range.end)) - } else if is_decode_err(&err, vm) { - let range = extract_unicode_error_range(&err, vm)?; - let object = err.get_attr("object", vm)?; - let object = PyBytesRef::try_from_object(vm, object)?; - let p = &object.as_bytes()[range.clone()]; - let mut consumed = 0; - let mut replace = String::with_capacity(4 * range.len()); - while consumed < 4 && consumed < range.len() { - let c = p[consumed] as u32; - // Refuse to escape ASCII bytes - if c < 128 { - break; - } - write!(replace, "#{}", 0xdc00 + c).unwrap(); - consumed += 1; - } - if consumed == 0 { - return Err(err.downcast().unwrap()); - } - Ok((vm.new_pyobj(replace), range.start + consumed)) - } else { - Err(bad_err_type(err, vm)) - } -} diff --git a/vm/src/compiler.rs b/vm/src/compiler.rs deleted file mode 100644 index 1e35d3b7a6f..00000000000 --- a/vm/src/compiler.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::{builtins::PyBaseExceptionRef, convert::ToPyException, VirtualMachine}; - -#[cfg(feature = "rustpython-codegen")] -pub use rustpython_codegen::CompileOpts; -#[cfg(feature = "rustpython-compiler")] -pub use rustpython_compiler::*; - -#[cfg(not(feature = "rustpython-compiler"))] -pub use rustpython_compiler_core as core; - -#[cfg(all(not(feature = "rustpython-compiler"), feature = "rustpython-parser"))] -pub use rustpython_parser_core as parser; - -#[cfg(not(feature = "rustpython-compiler"))] -mod error { - #[cfg(all(feature = "rustpython-parser", feature = "rustpython-codegen"))] - panic!("Use --features=compiler to enable both parser and codegen"); - - #[derive(Debug, thiserror::Error)] - pub enum CompileErrorType { - #[cfg(feature = "rustpython-codegen")] - #[error(transparent)] - Codegen(#[from] rustpython_codegen::error::CodegenErrorType), - #[cfg(feature = "rustpython-parser")] - #[error(transparent)] - Parse(#[from] rustpython_parser::error::ParseErrorType), - } - - pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>; -} -#[cfg(not(feature = "rustpython-compiler"))] -pub use error::{CompileError, CompileErrorType}; - -impl ToPyException for (CompileError, Option<&str>) { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_syntax_error(&self.0, self.1) - } -} diff --git a/vm/src/convert/try_from.rs b/vm/src/convert/try_from.rs deleted file mode 100644 index 028cc910335..00000000000 --- a/vm/src/convert/try_from.rs +++ /dev/null @@ -1,142 +0,0 @@ -use crate::{ - builtins::PyFloat, - object::{AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult}, - Py, VirtualMachine, -}; -use num_traits::ToPrimitive; - -/// Implemented by any type that can be created from a Python object. -/// -/// Any type that implements `TryFromObject` is automatically `FromArgs`, and -/// so can be accepted as a argument to a built-in function. -pub trait TryFromObject: Sized { - /// Attempt to convert a Python object to a value of this type. - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self>; -} - -/// Rust-side only version of TryFromObject to reduce unnecessary Rc::clone -impl<T: for<'a> TryFromBorrowedObject<'a>> TryFromObject for T { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - TryFromBorrowedObject::try_from_borrowed_object(vm, &obj) - } -} - -impl PyObjectRef { - pub fn try_into_value<T>(self, vm: &VirtualMachine) -> PyResult<T> - where - T: TryFromObject, - { - T::try_from_object(vm, self) - } -} - -impl PyObject { - pub fn try_to_value<'a, T: 'a>(&'a self, vm: &VirtualMachine) -> PyResult<T> - where - T: TryFromBorrowedObject<'a>, - { - T::try_from_borrowed_object(vm, self) - } - - pub fn try_to_ref<'a, T: 'a>(&'a self, vm: &VirtualMachine) -> PyResult<&'a Py<T>> - where - T: PyPayload, - { - self.try_to_value::<&Py<T>>(vm) - } - - pub fn try_value_with<T, F, R>(&self, f: F, vm: &VirtualMachine) -> PyResult<R> - where - T: PyPayload, - F: Fn(&T) -> PyResult<R>, - { - let class = T::class(&vm.ctx); - let py_ref = if self.fast_isinstance(class) { - self.downcast_ref() - .ok_or_else(|| vm.new_downcast_runtime_error(class, self))? - } else { - return Err(vm.new_downcast_type_error(class, self)); - }; - f(py_ref) - } -} - -/// Lower-cost variation of `TryFromObject` -pub trait TryFromBorrowedObject<'a>: Sized -where - Self: 'a, -{ - /// Attempt to convert a Python object to a value of this type. - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self>; -} - -impl<T> TryFromObject for PyRef<T> -where - T: PyPayload, -{ - #[inline] - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let class = T::class(&vm.ctx); - if obj.fast_isinstance(class) { - obj.downcast() - .map_err(|obj| vm.new_downcast_runtime_error(class, &obj)) - } else { - Err(vm.new_downcast_type_error(class, &obj)) - } - } -} - -impl TryFromObject for PyObjectRef { - #[inline] - fn try_from_object(_vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - Ok(obj) - } -} - -impl<T: TryFromObject> TryFromObject for Option<T> { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - if vm.is_none(&obj) { - Ok(None) - } else { - T::try_from_object(vm, obj).map(Some) - } - } -} - -impl<'a, T: 'a + TryFromObject> TryFromBorrowedObject<'a> for Vec<T> { - fn try_from_borrowed_object(vm: &VirtualMachine, value: &'a PyObject) -> PyResult<Self> { - vm.extract_elements_with(value, |obj| T::try_from_object(vm, obj)) - } -} - -impl<'a, T: PyPayload> TryFromBorrowedObject<'a> for &'a Py<T> { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - let class = T::class(&vm.ctx); - if obj.fast_isinstance(class) { - obj.downcast_ref() - .ok_or_else(|| vm.new_downcast_runtime_error(class, &obj)) - } else { - Err(vm.new_downcast_type_error(class, &obj)) - } - } -} - -impl TryFromObject for std::time::Duration { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - use std::time::Duration; - if let Some(float) = obj.payload::<PyFloat>() { - Ok(Duration::from_secs_f64(float.to_f64())) - } else if let Some(int) = obj.try_index_opt(vm) { - let sec = int? - .as_bigint() - .to_u64() - .ok_or_else(|| vm.new_value_error("value out of range".to_owned()))?; - Ok(Duration::from_secs(sec)) - } else { - Err(vm.new_type_error(format!( - "expected an int or float for duration, got {}", - obj.class() - ))) - } - } -} diff --git a/vm/src/coroutine.rs b/vm/src/coroutine.rs deleted file mode 100644 index 6d6b743310a..00000000000 --- a/vm/src/coroutine.rs +++ /dev/null @@ -1,198 +0,0 @@ -use crate::{ - builtins::{PyBaseExceptionRef, PyStrRef}, - common::lock::PyMutex, - frame::{ExecutionResult, FrameRef}, - protocol::PyIterReturn, - AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, -}; -use crossbeam_utils::atomic::AtomicCell; - -impl ExecutionResult { - /// Turn an ExecutionResult into a PyResult that would be returned from a generator or coroutine - fn into_iter_return(self, vm: &VirtualMachine) -> PyIterReturn { - match self { - ExecutionResult::Yield(value) => PyIterReturn::Return(value), - ExecutionResult::Return(value) => { - let arg = if vm.is_none(&value) { - None - } else { - Some(value) - }; - PyIterReturn::StopIteration(arg) - } - } - } -} - -#[derive(Debug)] -pub struct Coro { - frame: FrameRef, - pub closed: AtomicCell<bool>, // TODO: https://github.com/RustPython/RustPython/pull/3183#discussion_r720560652 - running: AtomicCell<bool>, - // code - // _weakreflist - name: PyMutex<PyStrRef>, - // qualname - exception: PyMutex<Option<PyBaseExceptionRef>>, // exc_state -} - -fn gen_name(gen: &PyObject, vm: &VirtualMachine) -> &'static str { - let typ = gen.class(); - if typ.is(vm.ctx.types.coroutine_type) { - "coroutine" - } else if typ.is(vm.ctx.types.async_generator) { - "async generator" - } else { - "generator" - } -} - -impl Coro { - pub fn new(frame: FrameRef, name: PyStrRef) -> Self { - Coro { - frame, - closed: AtomicCell::new(false), - running: AtomicCell::new(false), - exception: PyMutex::default(), - name: PyMutex::new(name), - } - } - - fn maybe_close(&self, res: &PyResult<ExecutionResult>) { - match res { - Ok(ExecutionResult::Return(_)) | Err(_) => self.closed.store(true), - Ok(ExecutionResult::Yield(_)) => {} - } - } - - fn run_with_context<F>( - &self, - gen: &PyObject, - vm: &VirtualMachine, - func: F, - ) -> PyResult<ExecutionResult> - where - F: FnOnce(FrameRef) -> PyResult<ExecutionResult>, - { - if self.running.compare_exchange(false, true).is_err() { - return Err(vm.new_value_error(format!("{} already executing", gen_name(gen, vm)))); - } - - vm.push_exception(self.exception.lock().take()); - - let result = vm.with_frame(self.frame.clone(), func); - - *self.exception.lock() = vm.pop_exception(); - - self.running.store(false); - result - } - - pub fn send( - &self, - gen: &PyObject, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn> { - if self.closed.load() { - return Ok(PyIterReturn::StopIteration(None)); - } - let value = if self.frame.lasti() > 0 { - Some(value) - } else if !vm.is_none(&value) { - return Err(vm.new_type_error(format!( - "can't send non-None value to a just-started {}", - gen_name(gen, vm), - ))); - } else { - None - }; - let result = self.run_with_context(gen, vm, |f| f.resume(value, vm)); - self.maybe_close(&result); - match result { - Ok(exec_res) => Ok(exec_res.into_iter_return(vm)), - Err(e) => { - if e.fast_isinstance(vm.ctx.exceptions.stop_iteration) { - let err = - vm.new_runtime_error(format!("{} raised StopIteration", gen_name(gen, vm))); - err.set_cause(Some(e)); - Err(err) - } else if gen.class().is(vm.ctx.types.async_generator) - && e.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) - { - let err = vm - .new_runtime_error("async generator raised StopAsyncIteration".to_owned()); - err.set_cause(Some(e)); - Err(err) - } else { - Err(e) - } - } - } - } - pub fn throw( - &self, - gen: &PyObject, - exc_type: PyObjectRef, - exc_val: PyObjectRef, - exc_tb: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn> { - if self.closed.load() { - return Err(vm.normalize_exception(exc_type, exc_val, exc_tb)?); - } - let result = self.run_with_context(gen, vm, |f| f.gen_throw(vm, exc_type, exc_val, exc_tb)); - self.maybe_close(&result); - Ok(result?.into_iter_return(vm)) - } - - pub fn close(&self, gen: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if self.closed.load() { - return Ok(()); - } - let result = self.run_with_context(gen, vm, |f| { - f.gen_throw( - vm, - vm.ctx.exceptions.generator_exit.to_owned().into(), - vm.ctx.none(), - vm.ctx.none(), - ) - }); - self.closed.store(true); - match result { - Ok(ExecutionResult::Yield(_)) => { - Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", gen_name(gen, vm)))) - } - Err(e) if !is_gen_exit(&e, vm) => Err(e), - _ => Ok(()), - } - } - - pub fn running(&self) -> bool { - self.running.load() - } - pub fn closed(&self) -> bool { - self.closed.load() - } - pub fn frame(&self) -> FrameRef { - self.frame.clone() - } - pub fn name(&self) -> PyStrRef { - self.name.lock().clone() - } - pub fn set_name(&self, name: PyStrRef) { - *self.name.lock() = name; - } - pub fn repr(&self, gen: &PyObject, id: usize, vm: &VirtualMachine) -> String { - format!( - "<{} object {} at {:#x}>", - gen_name(gen, vm), - self.name.lock(), - id - ) - } -} - -pub fn is_gen_exit(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> bool { - exc.fast_isinstance(vm.ctx.exceptions.generator_exit) -} diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs deleted file mode 100644 index f8aeb3f6da8..00000000000 --- a/vm/src/dictdatatype.rs +++ /dev/null @@ -1,972 +0,0 @@ -//! Ordered dictionary implementation. -//! Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html -//! And: https://www.youtube.com/watch?v=p33CVV29OG8 -//! And: http://code.activestate.com/recipes/578375/ - -use crate::{ - builtins::{PyInt, PyStr, PyStrInterned, PyStrRef}, - convert::ToPyObject, - AsObject, Py, PyExact, PyObject, PyObjectRef, PyRefExact, PyResult, VirtualMachine, -}; -use crate::{ - common::{ - hash, - lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}, - }, - object::{Traverse, TraverseFn}, -}; -use num_traits::ToPrimitive; -use std::{fmt, mem::size_of, ops::ControlFlow}; - -// HashIndex is intended to be same size with hash::PyHash -// but it doesn't mean the values are compatible with actual pyhash value - -/// hash value of an object returned by __hash__ -type HashValue = hash::PyHash; -/// index calculated by resolving collision -type HashIndex = hash::PyHash; -/// index into dict.indices -type IndexIndex = usize; -/// index into dict.entries -type EntryIndex = usize; - -pub struct Dict<T = PyObjectRef> { - inner: PyRwLock<DictInner<T>>, -} - -unsafe impl<T: Traverse> Traverse for Dict<T> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.inner.traverse(tracer_fn); - } -} - -impl<T> fmt::Debug for Dict<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Debug").finish() - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(transparent)] -struct IndexEntry(i64); - -impl IndexEntry { - const FREE: Self = Self(-1); - const DUMMY: Self = Self(-2); - - /// # Safety - /// idx must not be one of FREE or DUMMY - unsafe fn from_index_unchecked(idx: usize) -> Self { - debug_assert!((idx as isize) >= 0); - Self(idx as i64) - } - - fn index(self) -> Option<usize> { - if self.0 >= 0 { - Some(self.0 as usize) - } else { - None - } - } -} - -#[derive(Clone)] -struct DictInner<T> { - used: usize, - filled: usize, - indices: Vec<IndexEntry>, - entries: Vec<Option<DictEntry<T>>>, -} - -unsafe impl<T: Traverse> Traverse for DictInner<T> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.entries - .iter() - .map(|v| { - if let Some(v) = v { - v.key.traverse(tracer_fn); - v.value.traverse(tracer_fn); - } - }) - .count(); - } -} - -impl<T: Clone> Clone for Dict<T> { - fn clone(&self) -> Self { - Self { - inner: PyRwLock::new(self.inner.read().clone()), - } - } -} - -impl<T> Default for Dict<T> { - fn default() -> Self { - Self { - inner: PyRwLock::new(DictInner { - used: 0, - filled: 0, - indices: vec![IndexEntry::FREE; 8], - entries: Vec::new(), - }), - } - } -} - -#[derive(Clone)] -struct DictEntry<T> { - hash: HashValue, - key: PyObjectRef, - index: IndexIndex, - value: T, -} -static_assertions::assert_eq_size!(DictEntry<PyObjectRef>, Option<DictEntry<PyObjectRef>>); - -#[derive(Debug, PartialEq, Eq)] -pub struct DictSize { - indices_size: usize, - pub entries_size: usize, - pub used: usize, - filled: usize, -} - -struct GenIndexes { - idx: HashIndex, - perturb: HashValue, - mask: HashIndex, -} - -impl GenIndexes { - fn new(hash: HashValue, mask: HashIndex) -> Self { - let hash = hash.abs(); - Self { - idx: hash, - perturb: hash, - mask, - } - } - fn next(&mut self) -> usize { - let prev = self.idx; - self.idx = prev - .wrapping_mul(5) - .wrapping_add(self.perturb) - .wrapping_add(1); - self.perturb >>= 5; - (prev & self.mask) as usize - } -} - -impl<T> DictInner<T> { - fn resize(&mut self, new_size: usize) { - let new_size = { - let mut i = 1; - while i < new_size { - i <<= 1; - } - i - }; - self.indices = vec![IndexEntry::FREE; new_size]; - let mask = (new_size - 1) as i64; - for (entry_idx, entry) in self.entries.iter_mut().enumerate() { - if let Some(entry) = entry { - let mut idxs = GenIndexes::new(entry.hash, mask); - loop { - let index_index = idxs.next(); - unsafe { - // Safety: index is always valid here - // index_index is generated by idxs - // entry_idx is saved one - let idx = self.indices.get_unchecked_mut(index_index); - if *idx == IndexEntry::FREE { - *idx = IndexEntry::from_index_unchecked(entry_idx); - entry.index = index_index; - break; - } - } - } - } else { - //removed entry - } - } - self.filled = self.used; - } - - fn unchecked_push( - &mut self, - index: IndexIndex, - hash_value: HashValue, - key: PyObjectRef, - value: T, - index_entry: IndexEntry, - ) { - let entry = DictEntry { - hash: hash_value, - key, - value, - index, - }; - let entry_index = self.entries.len(); - self.entries.push(Some(entry)); - self.indices[index] = unsafe { - // SAFETY: entry_index is self.entries.len(). it never can - // grow to `usize-2` because hash tables cannot full its index - IndexEntry::from_index_unchecked(entry_index) - }; - self.used += 1; - if let IndexEntry::FREE = index_entry { - self.filled += 1; - if let Some(new_size) = self.should_resize() { - self.resize(new_size) - } - } - } - - fn size(&self) -> DictSize { - DictSize { - indices_size: self.indices.len(), - entries_size: self.entries.len(), - used: self.used, - filled: self.filled, - } - } - - #[inline] - fn should_resize(&self) -> Option<usize> { - if self.filled * 3 > self.indices.len() * 2 { - Some(self.used * 2) - } else { - None - } - } - - #[inline] - fn get_entry_checked(&self, idx: EntryIndex, index_index: IndexIndex) -> Option<&DictEntry<T>> { - match self.entries.get(idx) { - Some(Some(entry)) if entry.index == index_index => Some(entry), - _ => None, - } - } -} - -type PopInnerResult<T> = ControlFlow<Option<DictEntry<T>>>; - -impl<T: Clone> Dict<T> { - fn read(&self) -> PyRwLockReadGuard<'_, DictInner<T>> { - self.inner.read() - } - - fn write(&self) -> PyRwLockWriteGuard<'_, DictInner<T>> { - self.inner.write() - } - - /// Store a key - pub fn insert<K>(&self, vm: &VirtualMachine, key: &K, value: T) -> PyResult<()> - where - K: DictKey + ?Sized, - { - let hash = key.key_hash(vm)?; - let _removed = loop { - let (entry_index, index_index) = self.lookup(vm, key, hash, None)?; - let mut inner = self.write(); - if let Some(index) = entry_index.index() { - // Update existing key - if let Some(entry) = inner.entries.get_mut(index) { - let Some(entry) = entry.as_mut() else { - // The dict was changed since we did lookup. Let's try again. - // this is very rare to happen - // (and seems only happen with very high freq gc, and about one time in 10000 iters) - // but still possible - continue; - }; - if entry.index == index_index { - let removed = std::mem::replace(&mut entry.value, value); - // defer dec RC - break Some(removed); - } else { - // stuff shifted around, let's try again - } - } else { - // The dict was changed since we did lookup. Let's try again. - } - } else { - // New key: - inner.unchecked_push(index_index, hash, key.to_pyobject(vm), value, entry_index); - break None; - } - }; - Ok(()) - } - - pub fn contains<K: DictKey + ?Sized>(&self, vm: &VirtualMachine, key: &K) -> PyResult<bool> { - let (entry, _) = self.lookup(vm, key, key.key_hash(vm)?, None)?; - Ok(entry.index().is_some()) - } - - /// Retrieve a key - #[cfg_attr(feature = "flame-it", flame("Dict"))] - pub fn get<K: DictKey + ?Sized>(&self, vm: &VirtualMachine, key: &K) -> PyResult<Option<T>> { - let hash = key.key_hash(vm)?; - self._get_inner(vm, key, hash) - } - - fn _get_inner<K: DictKey + ?Sized>( - &self, - vm: &VirtualMachine, - key: &K, - hash: HashValue, - ) -> PyResult<Option<T>> { - let ret = loop { - let (entry, index_index) = self.lookup(vm, key, hash, None)?; - if let Some(index) = entry.index() { - let inner = self.read(); - if let Some(entry) = inner.get_entry_checked(index, index_index) { - break Some(entry.value.clone()); - } else { - // The dict was changed since we did lookup. Let's try again. - continue; - } - } else { - break None; - } - }; - Ok(ret) - } - - pub fn get_chain<K: DictKey + ?Sized>( - &self, - other: &Self, - vm: &VirtualMachine, - key: &K, - ) -> PyResult<Option<T>> { - let hash = key.key_hash(vm)?; - if let Some(x) = self._get_inner(vm, key, hash)? { - Ok(Some(x)) - } else { - other._get_inner(vm, key, hash) - } - } - - pub fn clear(&self) { - let _removed = { - let mut inner = self.write(); - inner.indices.clear(); - inner.indices.resize(8, IndexEntry::FREE); - inner.used = 0; - inner.filled = 0; - // defer dec rc - std::mem::take(&mut inner.entries) - }; - } - - /// Delete a key - pub fn delete<K>(&self, vm: &VirtualMachine, key: &K) -> PyResult<()> - where - K: DictKey + ?Sized, - { - if self.delete_if_exists(vm, key)? { - Ok(()) - } else { - Err(vm.new_key_error(key.to_pyobject(vm))) - } - } - - pub fn delete_if_exists<K>(&self, vm: &VirtualMachine, key: &K) -> PyResult<bool> - where - K: DictKey + ?Sized, - { - self.delete_if(vm, key, |_| Ok(true)) - } - - /// pred should be VERY CAREFUL about what it does as it is called while - /// the dict's internal mutex is held - pub(crate) fn delete_if<K, F>(&self, vm: &VirtualMachine, key: &K, pred: F) -> PyResult<bool> - where - K: DictKey + ?Sized, - F: Fn(&T) -> PyResult<bool>, - { - let hash = key.key_hash(vm)?; - let deleted = loop { - let lookup = self.lookup(vm, key, hash, None)?; - match self.pop_inner_if(lookup, &pred)? { - ControlFlow::Break(entry) => break entry, - ControlFlow::Continue(()) => continue, - } - }; - Ok(deleted.is_some()) - } - - pub fn delete_or_insert(&self, vm: &VirtualMachine, key: &PyObject, value: T) -> PyResult<()> { - let hash = key.key_hash(vm)?; - let _removed = loop { - let lookup = self.lookup(vm, key, hash, None)?; - let (entry, index_index) = lookup; - if entry.index().is_some() { - match self.pop_inner(lookup) { - ControlFlow::Break(Some(entry)) => break Some(entry), - _ => continue, - } - } else { - let mut inner = self.write(); - inner.unchecked_push(index_index, hash, key.to_owned(), value, entry); - break None; - } - }; - Ok(()) - } - - pub fn setdefault<K, F>(&self, vm: &VirtualMachine, key: &K, default: F) -> PyResult<T> - where - K: DictKey + ?Sized, - F: FnOnce() -> T, - { - let hash = key.key_hash(vm)?; - let res = loop { - let lookup = self.lookup(vm, key, hash, None)?; - let (index_entry, index_index) = lookup; - if let Some(index) = index_entry.index() { - let inner = self.read(); - if let Some(entry) = inner.get_entry_checked(index, index_index) { - break entry.value.clone(); - } else { - // The dict was changed since we did lookup, let's try again. - continue; - } - } else { - let value = default(); - let mut inner = self.write(); - inner.unchecked_push( - index_index, - hash, - key.to_pyobject(vm), - value.clone(), - index_entry, - ); - break value; - } - }; - Ok(res) - } - - #[allow(dead_code)] - pub fn setdefault_entry<K, F>( - &self, - vm: &VirtualMachine, - key: &K, - default: F, - ) -> PyResult<(PyObjectRef, T)> - where - K: DictKey + ?Sized, - F: FnOnce() -> T, - { - let hash = key.key_hash(vm)?; - let res = loop { - let lookup = self.lookup(vm, key, hash, None)?; - let (index_entry, index_index) = lookup; - if let Some(index) = index_entry.index() { - let inner = self.read(); - if let Some(entry) = inner.get_entry_checked(index, index_index) { - break (entry.key.clone(), entry.value.clone()); - } else { - // The dict was changed since we did lookup, let's try again. - continue; - } - } else { - let value = default(); - let key = key.to_pyobject(vm); - let mut inner = self.write(); - let ret = (key.clone(), value.clone()); - inner.unchecked_push(index_index, hash, key, value, index_entry); - break ret; - } - }; - Ok(res) - } - - pub fn len(&self) -> usize { - self.read().used - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn size(&self) -> DictSize { - self.read().size() - } - - pub fn next_entry(&self, mut position: EntryIndex) -> Option<(usize, PyObjectRef, T)> { - let inner = self.read(); - loop { - let entry = inner.entries.get(position)?; - position += 1; - if let Some(entry) = entry { - break Some((position, entry.key.clone(), entry.value.clone())); - } - } - } - - pub fn prev_entry(&self, mut position: EntryIndex) -> Option<(usize, PyObjectRef, T)> { - let inner = self.read(); - loop { - let entry = inner.entries.get(position)?; - position = position.saturating_sub(1); - if let Some(entry) = entry { - break Some((position, entry.key.clone(), entry.value.clone())); - } - } - } - - pub fn len_from_entry_index(&self, position: EntryIndex) -> usize { - self.read().entries.len().saturating_sub(position) - } - - pub fn has_changed_size(&self, old: &DictSize) -> bool { - let current = self.read().size(); - current != *old - } - - pub fn keys(&self) -> Vec<PyObjectRef> { - self.read() - .entries - .iter() - .filter_map(|v| v.as_ref().map(|v| v.key.clone())) - .collect() - } - - /// Lookup the index for the given key. - #[cfg_attr(feature = "flame-it", flame("Dict"))] - fn lookup<K: DictKey + ?Sized>( - &self, - vm: &VirtualMachine, - key: &K, - hash_value: HashValue, - mut lock: Option<PyRwLockReadGuard<DictInner<T>>>, - ) -> PyResult<LookupResult> { - let mut idxs = None; - let mut free_slot = None; - let ret = 'outer: loop { - let (entry_key, ret) = { - let inner = lock.take().unwrap_or_else(|| self.read()); - let idxs = idxs.get_or_insert_with(|| { - GenIndexes::new(hash_value, (inner.indices.len() - 1) as i64) - }); - loop { - let index_index = idxs.next(); - let index_entry = *unsafe { - // Safety: index_index is generated - inner.indices.get_unchecked(index_index) - }; - match index_entry { - IndexEntry::DUMMY => { - if free_slot.is_none() { - free_slot = Some(index_index); - } - } - IndexEntry::FREE => { - let idxs = match free_slot { - Some(free) => (IndexEntry::DUMMY, free), - None => (IndexEntry::FREE, index_index), - }; - return Ok(idxs); - } - idx => { - let entry = unsafe { - // Safety: DUMMY and FREE are already handled above. - // i is always valid and entry always exists. - let i = idx.index().unwrap_unchecked(); - inner.entries.get_unchecked(i).as_ref().unwrap_unchecked() - }; - let ret = (idx, index_index); - if key.key_is(&entry.key) { - break 'outer ret; - } else if entry.hash == hash_value { - break (entry.key.clone(), ret); - } else { - // entry mismatch - } - } - } - // warn!("Perturb value: {}", i); - } - }; - // This comparison needs to be done outside the lock. - if key.key_eq(vm, &entry_key)? { - break 'outer ret; - } else { - // hash collision - } - - // warn!("Perturb value: {}", i); - }; - Ok(ret) - } - - // returns Err(()) if changed since lookup - fn pop_inner(&self, lookup: LookupResult) -> PopInnerResult<T> { - self.pop_inner_if(lookup, |_| Ok::<_, std::convert::Infallible>(true)) - .unwrap_or_else(|x| match x {}) - } - - fn pop_inner_if<E>( - &self, - lookup: LookupResult, - pred: impl Fn(&T) -> Result<bool, E>, - ) -> Result<PopInnerResult<T>, E> { - let (entry_index, index_index) = lookup; - let Some(entry_index) = entry_index.index() else { - return Ok(ControlFlow::Break(None)); - }; - let inner = &mut *self.write(); - let slot = if let Some(slot) = inner.entries.get_mut(entry_index) { - slot - } else { - // The dict was changed since we did lookup. Let's try again. - return Ok(ControlFlow::Continue(())); - }; - match slot { - Some(entry) if entry.index == index_index => { - if !pred(&entry.value)? { - return Ok(ControlFlow::Break(None)); - } - } - // The dict was changed since we did lookup. Let's try again. - _ => return Ok(ControlFlow::Continue(())), - } - *unsafe { - // index_index is result of lookup - inner.indices.get_unchecked_mut(index_index) - } = IndexEntry::DUMMY; - inner.used -= 1; - let removed = slot.take(); - Ok(ControlFlow::Break(removed)) - } - - /// Retrieve and delete a key - pub fn pop<K: DictKey + ?Sized>(&self, vm: &VirtualMachine, key: &K) -> PyResult<Option<T>> { - let hash_value = key.key_hash(vm)?; - let removed = loop { - let lookup = self.lookup(vm, key, hash_value, None)?; - match self.pop_inner(lookup) { - ControlFlow::Break(entry) => break entry.map(|e| e.value), - ControlFlow::Continue(()) => continue, - } - }; - Ok(removed) - } - - pub fn pop_back(&self) -> Option<(PyObjectRef, T)> { - let inner = &mut *self.write(); - let entry = loop { - let entry = inner.entries.pop()?; - if let Some(entry) = entry { - break entry; - } - }; - inner.used -= 1; - *unsafe { - // entry.index always refers valid index - inner.indices.get_unchecked_mut(entry.index) - } = IndexEntry::DUMMY; - Some((entry.key, entry.value)) - } - - pub fn sizeof(&self) -> usize { - let inner = self.read(); - size_of::<Self>() - + size_of::<DictInner<T>>() - + inner.indices.len() * size_of::<i64>() - + inner.entries.len() * size_of::<DictEntry<T>>() - } -} - -type LookupResult = (IndexEntry, IndexIndex); - -/// Types implementing this trait can be used to index -/// the dictionary. Typical usecases are: -/// - PyObjectRef -> arbitrary python type used as key -/// - str -> string reference used as key, this is often used internally -pub trait DictKey { - type Owned: ToPyObject; - fn _to_owned(&self, vm: &VirtualMachine) -> Self::Owned; - fn to_pyobject(&self, vm: &VirtualMachine) -> PyObjectRef { - self._to_owned(vm).to_pyobject(vm) - } - fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue>; - fn key_is(&self, other: &PyObject) -> bool; - fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool>; - fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize>; -} - -/// Implement trait for PyObjectRef such that we can use python objects -/// to index dictionaries. -impl DictKey for PyObject { - type Owned = PyObjectRef; - #[inline(always)] - fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { - self.to_owned() - } - #[inline(always)] - fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { - self.hash(vm) - } - #[inline(always)] - fn key_is(&self, other: &PyObject) -> bool { - self.is(other) - } - #[inline(always)] - fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { - vm.identical_or_equal(self, other_key) - } - #[inline] - fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { - self.try_index(vm)?.try_to_primitive(vm) - } -} - -impl DictKey for Py<PyStr> { - type Owned = PyStrRef; - #[inline(always)] - fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { - self.to_owned() - } - #[inline] - fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { - Ok(self.hash(vm)) - } - #[inline(always)] - fn key_is(&self, other: &PyObject) -> bool { - self.is(other) - } - - fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { - if self.is(other_key) { - Ok(true) - } else if let Some(pystr) = str_exact(other_key, vm) { - Ok(pystr.as_str() == self.as_str()) - } else { - vm.bool_eq(self.as_object(), other_key) - } - } - #[inline(always)] - fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { - self.as_object().key_as_isize(vm) - } -} - -impl DictKey for PyStrInterned { - type Owned = PyRefExact<PyStr>; - #[inline] - fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { - let zelf: &'static PyStrInterned = unsafe { &*(self as *const _) }; - zelf.to_exact() - } - #[inline] - fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { - (**self).key_hash(vm) - } - #[inline] - fn key_is(&self, other: &PyObject) -> bool { - (**self).key_is(other) - } - #[inline] - fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { - (**self).key_eq(vm, other_key) - } - #[inline] - fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { - (**self).key_as_isize(vm) - } -} - -impl DictKey for PyExact<PyStr> { - type Owned = PyRefExact<PyStr>; - #[inline] - fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { - self.to_owned() - } - #[inline(always)] - fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { - (**self).key_hash(vm) - } - #[inline(always)] - fn key_is(&self, other: &PyObject) -> bool { - (**self).key_is(other) - } - #[inline(always)] - fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { - (**self).key_eq(vm, other_key) - } - #[inline(always)] - fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { - (**self).key_as_isize(vm) - } -} - -// AsRef<str> fit this case but not possible in rust 1.46 - -/// Implement trait for the str type, so that we can use strings -/// to index dictionaries. -impl DictKey for str { - type Owned = String; - #[inline(always)] - fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { - self.to_owned() - } - #[inline] - fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { - // follow a similar route as the hashing of PyStrRef - Ok(vm.state.hash_secret.hash_str(self)) - } - #[inline(always)] - fn key_is(&self, _other: &PyObject) -> bool { - // No matter who the other pyobject is, we are never the same thing, since - // we are a str, not a pyobject. - false - } - - fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { - if let Some(pystr) = str_exact(other_key, vm) { - Ok(pystr.as_str() == self) - } else { - // Fall back to PyObjectRef implementation. - let s = vm.ctx.new_str(self); - s.key_eq(vm, other_key) - } - } - - fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { - Err(vm.new_type_error("'str' object cannot be interpreted as an integer".to_owned())) - } -} - -impl DictKey for String { - type Owned = String; - #[inline] - fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { - self.clone() - } - - fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { - self.as_str().key_hash(vm) - } - - fn key_is(&self, other: &PyObject) -> bool { - self.as_str().key_is(other) - } - - fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { - self.as_str().key_eq(vm, other_key) - } - - fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult<isize> { - self.as_str().key_as_isize(vm) - } -} - -impl DictKey for usize { - type Owned = usize; - #[inline] - fn _to_owned(&self, _vm: &VirtualMachine) -> Self::Owned { - *self - } - - fn key_hash(&self, vm: &VirtualMachine) -> PyResult<HashValue> { - Ok(vm.state.hash_secret.hash_value(self)) - } - - fn key_is(&self, _other: &PyObject) -> bool { - false - } - - fn key_eq(&self, vm: &VirtualMachine, other_key: &PyObject) -> PyResult<bool> { - if let Some(int) = other_key.payload_if_exact::<PyInt>(vm) { - if let Some(i) = int.as_bigint().to_usize() { - Ok(i == *self) - } else { - Ok(false) - } - } else { - let int = vm.ctx.new_int(*self); - vm.bool_eq(int.as_ref(), other_key) - } - } - - fn key_as_isize(&self, _vm: &VirtualMachine) -> PyResult<isize> { - Ok(*self as isize) - } -} - -fn str_exact<'a>(obj: &'a PyObject, vm: &VirtualMachine) -> Option<&'a PyStr> { - if obj.class().is(vm.ctx.types.str_type) { - obj.payload::<PyStr>() - } else { - None - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{common::ascii, Interpreter}; - - #[test] - fn test_insert() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { - let dict = Dict::default(); - assert_eq!(0, dict.len()); - - let key1 = vm.new_pyobj(true); - let value1 = vm.new_pyobj(ascii!("abc")); - dict.insert(vm, &*key1, value1).unwrap(); - assert_eq!(1, dict.len()); - - let key2 = vm.new_pyobj(ascii!("x")); - let value2 = vm.new_pyobj(ascii!("def")); - dict.insert(vm, &*key2, value2.clone()).unwrap(); - assert_eq!(2, dict.len()); - - dict.insert(vm, &*key1, value2.clone()).unwrap(); - assert_eq!(2, dict.len()); - - dict.delete(vm, &*key1).unwrap(); - assert_eq!(1, dict.len()); - - dict.insert(vm, &*key1, value2.clone()).unwrap(); - assert_eq!(2, dict.len()); - - assert!(dict.contains(vm, &*key1).unwrap()); - assert!(dict.contains(vm, "x").unwrap()); - - let val = dict.get(vm, "x").unwrap().unwrap(); - vm.bool_eq(&val, &value2) - .expect("retrieved value must be equal to inserted value."); - }) - } - - macro_rules! hash_tests { - ($($name:ident: $example_hash:expr,)*) => { - $( - #[test] - fn $name() { - check_hash_equivalence($example_hash); - } - )* - } - } - - hash_tests! { - test_abc: "abc", - test_x: "x", - } - - fn check_hash_equivalence(text: &str) { - Interpreter::without_stdlib(Default::default()).enter(|vm| { - let value1 = text; - let value2 = vm.new_pyobj(value1.to_owned()); - - let hash1 = value1.key_hash(vm).expect("Hash should not fail."); - let hash2 = value2.key_hash(vm).expect("Hash should not fail."); - assert_eq!(hash1, hash2); - }) - } -} diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs deleted file mode 100644 index d83e7e48d8d..00000000000 --- a/vm/src/exceptions.rs +++ /dev/null @@ -1,1551 +0,0 @@ -use self::types::{PyBaseException, PyBaseExceptionRef}; -use crate::common::lock::PyRwLock; -use crate::object::{Traverse, TraverseFn}; -use crate::{ - builtins::{ - traceback::PyTracebackRef, PyNone, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, - }, - class::{PyClassImpl, StaticType}, - convert::{ToPyException, ToPyObject}, - function::{ArgIterable, FuncArgs, IntoFuncArgs}, - py_io::{self, Write}, - stdlib::sys, - suggestion::offer_suggestions, - types::{Callable, Constructor, Initializer, Representable}, - AsObject, Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, -}; -use crossbeam_utils::atomic::AtomicCell; -use itertools::Itertools; -use std::{ - collections::HashSet, - io::{self, BufRead, BufReader}, -}; - -unsafe impl Traverse for PyBaseException { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.traceback.traverse(tracer_fn); - self.cause.traverse(tracer_fn); - self.context.traverse(tracer_fn); - self.args.traverse(tracer_fn); - } -} - -impl std::fmt::Debug for PyBaseException { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - // TODO: implement more detailed, non-recursive Debug formatter - f.write_str("PyBaseException") - } -} - -impl PyPayload for PyBaseException { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.exceptions.base_exception_type - } -} - -impl VirtualMachine { - // Why `impl VirtualMachine? - // These functions are natively free function in CPython - not methods of PyException - - /// Print exception chain by calling sys.excepthook - pub fn print_exception(&self, exc: PyBaseExceptionRef) { - let vm = self; - let write_fallback = |exc, errstr| { - if let Ok(stderr) = sys::get_stderr(vm) { - let mut stderr = py_io::PyWriter(stderr, vm); - // if this fails stderr might be closed -- ignore it - let _ = writeln!(stderr, "{errstr}"); - let _ = self.write_exception(&mut stderr, exc); - } else { - eprintln!("{errstr}\nlost sys.stderr"); - let _ = self.write_exception(&mut py_io::IoWriter(io::stderr()), exc); - } - }; - if let Ok(excepthook) = vm.sys_module.get_attr("excepthook", vm) { - let (exc_type, exc_val, exc_tb) = vm.split_exception(exc.clone()); - if let Err(eh_exc) = excepthook.call((exc_type, exc_val, exc_tb), vm) { - write_fallback(&eh_exc, "Error in sys.excepthook:"); - write_fallback(&exc, "Original exception was:"); - } - } else { - write_fallback(&exc, "missing sys.excepthook"); - } - } - - pub fn write_exception<W: Write>( - &self, - output: &mut W, - exc: &PyBaseExceptionRef, - ) -> Result<(), W::Error> { - let seen = &mut HashSet::<usize>::new(); - self.write_exception_recursive(output, exc, seen) - } - - fn write_exception_recursive<W: Write>( - &self, - output: &mut W, - exc: &PyBaseExceptionRef, - seen: &mut HashSet<usize>, - ) -> Result<(), W::Error> { - // This function should not be called directly, - // use `wite_exception` as a public interface. - // It is similar to `print_exception_recursive` from `CPython`. - seen.insert(exc.get_id()); - - #[allow(clippy::manual_map)] - if let Some((cause_or_context, msg)) = if let Some(cause) = exc.cause() { - // This can be a special case: `raise e from e`, - // we just ignore it and treat like `raise e` without any extra steps. - Some(( - cause, - "\nThe above exception was the direct cause of the following exception:\n", - )) - } else if let Some(context) = exc.context() { - // This can be a special case: - // e = ValueError('e') - // e.__context__ = e - // In this case, we just ignore - // `__context__` part from going into recursion. - Some(( - context, - "\nDuring handling of the above exception, another exception occurred:\n", - )) - } else { - None - } { - if !seen.contains(&cause_or_context.get_id()) { - self.write_exception_recursive(output, &cause_or_context, seen)?; - writeln!(output, "{msg}")?; - } else { - seen.insert(cause_or_context.get_id()); - } - } - - self.write_exception_inner(output, exc) - } - - /// Print exception with traceback - pub fn write_exception_inner<W: Write>( - &self, - output: &mut W, - exc: &PyBaseExceptionRef, - ) -> Result<(), W::Error> { - let vm = self; - if let Some(tb) = exc.traceback.read().clone() { - writeln!(output, "Traceback (most recent call last):")?; - for tb in tb.iter() { - write_traceback_entry(output, &tb)?; - } - } - - let varargs = exc.args(); - let args_repr = vm.exception_args_as_string(varargs, true); - - let exc_class = exc.class(); - let exc_name = exc_class.name(); - match args_repr.len() { - 0 => write!(output, "{exc_name}"), - 1 => write!(output, "{}: {}", exc_name, args_repr[0]), - _ => write!( - output, - "{}: ({})", - exc_name, - args_repr.into_iter().format(", ") - ), - }?; - - match offer_suggestions(exc, vm) { - Some(suggestions) => writeln!(output, ". Did you mean: '{suggestions}'?"), - None => writeln!(output), - } - } - - fn exception_args_as_string(&self, varargs: PyTupleRef, str_single: bool) -> Vec<PyStrRef> { - let vm = self; - match varargs.len() { - 0 => vec![], - 1 => { - let args0_repr = if str_single { - varargs[0] - .str(vm) - .unwrap_or_else(|_| PyStr::from("<element str() failed>").into_ref(&vm.ctx)) - } else { - varargs[0].repr(vm).unwrap_or_else(|_| { - PyStr::from("<element repr() failed>").into_ref(&vm.ctx) - }) - }; - vec![args0_repr] - } - _ => varargs - .iter() - .map(|vararg| { - vararg.repr(vm).unwrap_or_else(|_| { - PyStr::from("<element repr() failed>").into_ref(&vm.ctx) - }) - }) - .collect(), - } - } - - pub fn split_exception( - &self, - exc: PyBaseExceptionRef, - ) -> (PyObjectRef, PyObjectRef, PyObjectRef) { - let tb = exc.traceback().to_pyobject(self); - let class = exc.class().to_owned(); - (class.into(), exc.into(), tb) - } - - /// Similar to PyErr_NormalizeException in CPython - pub fn normalize_exception( - &self, - exc_type: PyObjectRef, - exc_val: PyObjectRef, - exc_tb: PyObjectRef, - ) -> PyResult<PyBaseExceptionRef> { - let ctor = ExceptionCtor::try_from_object(self, exc_type)?; - let exc = ctor.instantiate_value(exc_val, self)?; - if let Some(tb) = Option::<PyTracebackRef>::try_from_object(self, exc_tb)? { - exc.set_traceback(Some(tb)); - } - Ok(exc) - } - - pub fn invoke_exception( - &self, - cls: PyTypeRef, - args: Vec<PyObjectRef>, - ) -> PyResult<PyBaseExceptionRef> { - // TODO: fast-path built-in exceptions by directly instantiating them? Is that really worth it? - let res = PyType::call(&cls, args.into_args(self), self)?; - PyBaseExceptionRef::try_from_object(self, res) - } -} - -fn print_source_line<W: Write>( - output: &mut W, - filename: &str, - lineno: usize, -) -> Result<(), W::Error> { - // TODO: use io.open() method instead, when available, according to https://github.com/python/cpython/blob/main/Python/traceback.c#L393 - // TODO: support different encodings - let file = match std::fs::File::open(filename) { - Ok(file) => file, - Err(_) => return Ok(()), - }; - let file = BufReader::new(file); - - for (i, line) in file.lines().enumerate() { - if i + 1 == lineno { - if let Ok(line) = line { - // Indented with 4 spaces - writeln!(output, " {}", line.trim_start())?; - } - return Ok(()); - } - } - - Ok(()) -} - -/// Print exception occurrence location from traceback element -fn write_traceback_entry<W: Write>( - output: &mut W, - tb_entry: &PyTracebackRef, -) -> Result<(), W::Error> { - let filename = tb_entry.frame.code.source_path.as_str(); - writeln!( - output, - r##" File "{}", line {}, in {}"##, - filename, tb_entry.lineno, tb_entry.frame.code.obj_name - )?; - print_source_line(output, filename, tb_entry.lineno.to_usize())?; - - Ok(()) -} - -#[derive(Clone)] -pub enum ExceptionCtor { - Class(PyTypeRef), - Instance(PyBaseExceptionRef), -} - -impl TryFromObject for ExceptionCtor { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - obj.downcast::<PyType>() - .and_then(|cls| { - if cls.fast_issubclass(vm.ctx.exceptions.base_exception_type) { - Ok(Self::Class(cls)) - } else { - Err(cls.into()) - } - }) - .or_else(|obj| obj.downcast::<PyBaseException>().map(Self::Instance)) - .map_err(|obj| { - vm.new_type_error(format!( - "exceptions must be classes or instances deriving from BaseException, not {}", - obj.class().name() - )) - }) - } -} - -impl ExceptionCtor { - pub fn instantiate(self, vm: &VirtualMachine) -> PyResult<PyBaseExceptionRef> { - match self { - Self::Class(cls) => vm.invoke_exception(cls, vec![]), - Self::Instance(exc) => Ok(exc), - } - } - - pub fn instantiate_value( - self, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyBaseExceptionRef> { - let exc_inst = value.clone().downcast::<PyBaseException>().ok(); - match (self, exc_inst) { - // both are instances; which would we choose? - (Self::Instance(_exc_a), Some(_exc_b)) => { - Err(vm - .new_type_error("instance exception may not have a separate value".to_owned())) - } - // if the "type" is an instance and the value isn't, use the "type" - (Self::Instance(exc), None) => Ok(exc), - // if the value is an instance of the type, use the instance value - (Self::Class(cls), Some(exc)) if exc.fast_isinstance(&cls) => Ok(exc), - // otherwise; construct an exception of the type using the value as args - (Self::Class(cls), _) => { - let args = match_class!(match value { - PyNone => vec![], - tup @ PyTuple => tup.to_vec(), - exc @ PyBaseException => exc.args().to_vec(), - obj => vec![obj], - }); - vm.invoke_exception(cls, args) - } - } - } -} - -#[derive(Debug, Clone)] -pub struct ExceptionZoo { - pub base_exception_type: &'static Py<PyType>, - pub base_exception_group: &'static Py<PyType>, - pub system_exit: &'static Py<PyType>, - pub keyboard_interrupt: &'static Py<PyType>, - pub generator_exit: &'static Py<PyType>, - pub exception_type: &'static Py<PyType>, - pub stop_iteration: &'static Py<PyType>, - pub stop_async_iteration: &'static Py<PyType>, - pub arithmetic_error: &'static Py<PyType>, - pub floating_point_error: &'static Py<PyType>, - pub overflow_error: &'static Py<PyType>, - pub zero_division_error: &'static Py<PyType>, - pub assertion_error: &'static Py<PyType>, - pub attribute_error: &'static Py<PyType>, - pub buffer_error: &'static Py<PyType>, - pub eof_error: &'static Py<PyType>, - pub import_error: &'static Py<PyType>, - pub module_not_found_error: &'static Py<PyType>, - pub lookup_error: &'static Py<PyType>, - pub index_error: &'static Py<PyType>, - pub key_error: &'static Py<PyType>, - pub memory_error: &'static Py<PyType>, - pub name_error: &'static Py<PyType>, - pub unbound_local_error: &'static Py<PyType>, - pub os_error: &'static Py<PyType>, - pub blocking_io_error: &'static Py<PyType>, - pub child_process_error: &'static Py<PyType>, - pub connection_error: &'static Py<PyType>, - pub broken_pipe_error: &'static Py<PyType>, - pub connection_aborted_error: &'static Py<PyType>, - pub connection_refused_error: &'static Py<PyType>, - pub connection_reset_error: &'static Py<PyType>, - pub file_exists_error: &'static Py<PyType>, - pub file_not_found_error: &'static Py<PyType>, - pub interrupted_error: &'static Py<PyType>, - pub is_a_directory_error: &'static Py<PyType>, - pub not_a_directory_error: &'static Py<PyType>, - pub permission_error: &'static Py<PyType>, - pub process_lookup_error: &'static Py<PyType>, - pub timeout_error: &'static Py<PyType>, - pub reference_error: &'static Py<PyType>, - pub runtime_error: &'static Py<PyType>, - pub not_implemented_error: &'static Py<PyType>, - pub recursion_error: &'static Py<PyType>, - pub syntax_error: &'static Py<PyType>, - pub indentation_error: &'static Py<PyType>, - pub tab_error: &'static Py<PyType>, - pub system_error: &'static Py<PyType>, - pub type_error: &'static Py<PyType>, - pub value_error: &'static Py<PyType>, - pub unicode_error: &'static Py<PyType>, - pub unicode_decode_error: &'static Py<PyType>, - pub unicode_encode_error: &'static Py<PyType>, - pub unicode_translate_error: &'static Py<PyType>, - - #[cfg(feature = "jit")] - pub jit_error: &'static Py<PyType>, - - pub warning: &'static Py<PyType>, - pub deprecation_warning: &'static Py<PyType>, - pub pending_deprecation_warning: &'static Py<PyType>, - pub runtime_warning: &'static Py<PyType>, - pub syntax_warning: &'static Py<PyType>, - pub user_warning: &'static Py<PyType>, - pub future_warning: &'static Py<PyType>, - pub import_warning: &'static Py<PyType>, - pub unicode_warning: &'static Py<PyType>, - pub bytes_warning: &'static Py<PyType>, - pub resource_warning: &'static Py<PyType>, - pub encoding_warning: &'static Py<PyType>, -} - -macro_rules! extend_exception { - ( - $exc_struct:ident, - $ctx:expr, - $class:expr - ) => { - extend_exception!($exc_struct, $ctx, $class, {}); - }; - ( - $exc_struct:ident, - $ctx:expr, - $class:expr, - { $($name:expr => $value:expr),* $(,)* } - ) => { - $exc_struct::extend_class($ctx, $class); - extend_class!($ctx, $class, { - $($name => $value,)* - }); - }; -} - -impl PyBaseException { - pub(crate) fn new(args: Vec<PyObjectRef>, vm: &VirtualMachine) -> PyBaseException { - PyBaseException { - traceback: PyRwLock::new(None), - cause: PyRwLock::new(None), - context: PyRwLock::new(None), - suppress_context: AtomicCell::new(false), - args: PyRwLock::new(PyTuple::new_ref(args, &vm.ctx)), - } - } - - pub fn get_arg(&self, idx: usize) -> Option<PyObjectRef> { - self.args.read().get(idx).cloned() - } -} - -#[pyclass( - with(Constructor, Initializer, Representable), - flags(BASETYPE, HAS_DICT) -)] -impl PyBaseException { - #[pygetset] - pub fn args(&self) -> PyTupleRef { - self.args.read().clone() - } - - #[pygetset(setter)] - fn set_args(&self, args: ArgIterable, vm: &VirtualMachine) -> PyResult<()> { - let args = args.iter(vm)?.collect::<PyResult<Vec<_>>>()?; - *self.args.write() = PyTuple::new_ref(args, &vm.ctx); - Ok(()) - } - - #[pygetset(magic)] - pub fn traceback(&self) -> Option<PyTracebackRef> { - self.traceback.read().clone() - } - - #[pygetset(magic, setter)] - pub fn set_traceback(&self, traceback: Option<PyTracebackRef>) { - *self.traceback.write() = traceback; - } - - #[pygetset(magic)] - pub fn cause(&self) -> Option<PyRef<Self>> { - self.cause.read().clone() - } - - #[pygetset(magic, setter)] - pub fn set_cause(&self, cause: Option<PyRef<Self>>) { - let mut c = self.cause.write(); - self.set_suppress_context(true); - *c = cause; - } - - #[pygetset(magic)] - pub fn context(&self) -> Option<PyRef<Self>> { - self.context.read().clone() - } - - #[pygetset(magic, setter)] - pub fn set_context(&self, context: Option<PyRef<Self>>) { - *self.context.write() = context; - } - - #[pygetset(name = "__suppress_context__")] - pub(super) fn get_suppress_context(&self) -> bool { - self.suppress_context.load() - } - - #[pygetset(name = "__suppress_context__", setter)] - fn set_suppress_context(&self, suppress_context: bool) { - self.suppress_context.store(suppress_context); - } - - #[pymethod] - fn with_traceback(zelf: PyRef<Self>, tb: Option<PyTracebackRef>) -> PyResult<PyRef<Self>> { - *zelf.traceback.write() = tb; - Ok(zelf) - } - - #[pymethod(magic)] - pub(super) fn str(&self, vm: &VirtualMachine) -> PyStrRef { - let str_args = vm.exception_args_as_string(self.args(), true); - match str_args.into_iter().exactly_one() { - Err(i) if i.len() == 0 => vm.ctx.empty_str.to_owned(), - Ok(s) => s, - Err(i) => PyStr::from(format!("({})", i.format(", "))).into_ref(&vm.ctx), - } - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { - if let Some(dict) = zelf.as_object().dict().filter(|x| !x.is_empty()) { - vm.new_tuple((zelf.class().to_owned(), zelf.args(), dict)) - } else { - vm.new_tuple((zelf.class().to_owned(), zelf.args())) - } - } -} - -impl Constructor for PyBaseException { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyBaseException::new(args.args, vm) - .into_ref_with_type(vm, cls) - .map(Into::into) - } -} - -impl Initializer for PyBaseException { - type Args = FuncArgs; - - fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - *zelf.args.write() = PyTuple::new_ref(args.args, &vm.ctx); - Ok(()) - } -} - -impl Representable for PyBaseException { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let repr_args = vm.exception_args_as_string(zelf.args(), false); - let cls = zelf.class(); - Ok(format!("{}({})", cls.name(), repr_args.iter().format(", "))) - } -} - -impl ExceptionZoo { - pub(crate) fn init() -> Self { - use self::types::*; - - let base_exception_type = PyBaseException::init_builtin_type(); - - // Sorted By Hierarchy then alphabetized. - let base_exception_group = PyBaseExceptionGroup::init_builtin_type(); - let system_exit = PySystemExit::init_builtin_type(); - let keyboard_interrupt = PyKeyboardInterrupt::init_builtin_type(); - let generator_exit = PyGeneratorExit::init_builtin_type(); - - let exception_type = PyException::init_builtin_type(); - let stop_iteration = PyStopIteration::init_builtin_type(); - let stop_async_iteration = PyStopAsyncIteration::init_builtin_type(); - let arithmetic_error = PyArithmeticError::init_builtin_type(); - let floating_point_error = PyFloatingPointError::init_builtin_type(); - let overflow_error = PyOverflowError::init_builtin_type(); - let zero_division_error = PyZeroDivisionError::init_builtin_type(); - - let assertion_error = PyAssertionError::init_builtin_type(); - let attribute_error = PyAttributeError::init_builtin_type(); - let buffer_error = PyBufferError::init_builtin_type(); - let eof_error = PyEOFError::init_builtin_type(); - - let import_error = PyImportError::init_builtin_type(); - let module_not_found_error = PyModuleNotFoundError::init_builtin_type(); - - let lookup_error = PyLookupError::init_builtin_type(); - let index_error = PyIndexError::init_builtin_type(); - let key_error = PyKeyError::init_builtin_type(); - - let memory_error = PyMemoryError::init_builtin_type(); - - let name_error = PyNameError::init_builtin_type(); - let unbound_local_error = PyUnboundLocalError::init_builtin_type(); - - // os errors - let os_error = PyOSError::init_builtin_type(); - let blocking_io_error = PyBlockingIOError::init_builtin_type(); - let child_process_error = PyChildProcessError::init_builtin_type(); - - let connection_error = PyConnectionError::init_builtin_type(); - let broken_pipe_error = PyBrokenPipeError::init_builtin_type(); - let connection_aborted_error = PyConnectionAbortedError::init_builtin_type(); - let connection_refused_error = PyConnectionRefusedError::init_builtin_type(); - let connection_reset_error = PyConnectionResetError::init_builtin_type(); - - let file_exists_error = PyFileExistsError::init_builtin_type(); - let file_not_found_error = PyFileNotFoundError::init_builtin_type(); - let interrupted_error = PyInterruptedError::init_builtin_type(); - let is_a_directory_error = PyIsADirectoryError::init_builtin_type(); - let not_a_directory_error = PyNotADirectoryError::init_builtin_type(); - let permission_error = PyPermissionError::init_builtin_type(); - let process_lookup_error = PyProcessLookupError::init_builtin_type(); - let timeout_error = PyTimeoutError::init_builtin_type(); - - let reference_error = PyReferenceError::init_builtin_type(); - - let runtime_error = PyRuntimeError::init_builtin_type(); - let not_implemented_error = PyNotImplementedError::init_builtin_type(); - let recursion_error = PyRecursionError::init_builtin_type(); - - let syntax_error = PySyntaxError::init_builtin_type(); - let indentation_error = PyIndentationError::init_builtin_type(); - let tab_error = PyTabError::init_builtin_type(); - - let system_error = PySystemError::init_builtin_type(); - let type_error = PyTypeError::init_builtin_type(); - let value_error = PyValueError::init_builtin_type(); - let unicode_error = PyUnicodeError::init_builtin_type(); - let unicode_decode_error = PyUnicodeDecodeError::init_builtin_type(); - let unicode_encode_error = PyUnicodeEncodeError::init_builtin_type(); - let unicode_translate_error = PyUnicodeTranslateError::init_builtin_type(); - - #[cfg(feature = "jit")] - let jit_error = PyJitError::init_builtin_type(); - - let warning = PyWarning::init_builtin_type(); - let deprecation_warning = PyDeprecationWarning::init_builtin_type(); - let pending_deprecation_warning = PyPendingDeprecationWarning::init_builtin_type(); - let runtime_warning = PyRuntimeWarning::init_builtin_type(); - let syntax_warning = PySyntaxWarning::init_builtin_type(); - let user_warning = PyUserWarning::init_builtin_type(); - let future_warning = PyFutureWarning::init_builtin_type(); - let import_warning = PyImportWarning::init_builtin_type(); - let unicode_warning = PyUnicodeWarning::init_builtin_type(); - let bytes_warning = PyBytesWarning::init_builtin_type(); - let resource_warning = PyResourceWarning::init_builtin_type(); - let encoding_warning = PyEncodingWarning::init_builtin_type(); - - Self { - base_exception_type, - base_exception_group, - system_exit, - keyboard_interrupt, - generator_exit, - exception_type, - stop_iteration, - stop_async_iteration, - arithmetic_error, - floating_point_error, - overflow_error, - zero_division_error, - assertion_error, - attribute_error, - buffer_error, - eof_error, - import_error, - module_not_found_error, - lookup_error, - index_error, - key_error, - memory_error, - name_error, - unbound_local_error, - os_error, - blocking_io_error, - child_process_error, - connection_error, - broken_pipe_error, - connection_aborted_error, - connection_refused_error, - connection_reset_error, - file_exists_error, - file_not_found_error, - interrupted_error, - is_a_directory_error, - not_a_directory_error, - permission_error, - process_lookup_error, - timeout_error, - reference_error, - runtime_error, - not_implemented_error, - recursion_error, - syntax_error, - indentation_error, - tab_error, - system_error, - type_error, - value_error, - unicode_error, - unicode_decode_error, - unicode_encode_error, - unicode_translate_error, - - #[cfg(feature = "jit")] - jit_error, - - warning, - deprecation_warning, - pending_deprecation_warning, - runtime_warning, - syntax_warning, - user_warning, - future_warning, - import_warning, - unicode_warning, - bytes_warning, - resource_warning, - encoding_warning, - } - } - - // TODO: remove it after fixing `errno` / `winerror` problem - #[allow(clippy::redundant_clone)] - pub fn extend(ctx: &Context) { - use self::types::*; - - let excs = &ctx.exceptions; - - PyBaseException::extend_class(ctx, excs.base_exception_type); - - // Sorted By Hierarchy then alphabetized. - extend_exception!(PyBaseExceptionGroup, ctx, excs.base_exception_group, { - "message" => ctx.new_readonly_getset("message", excs.base_exception_group, make_arg_getter(0)), - "exceptions" => ctx.new_readonly_getset("exceptions", excs.base_exception_group, make_arg_getter(1)), - }); - extend_exception!(PySystemExit, ctx, excs.system_exit, { - "code" => ctx.new_readonly_getset("code", excs.system_exit, system_exit_code), - }); - extend_exception!(PyKeyboardInterrupt, ctx, excs.keyboard_interrupt); - extend_exception!(PyGeneratorExit, ctx, excs.generator_exit); - - extend_exception!(PyException, ctx, excs.exception_type); - - extend_exception!(PyStopIteration, ctx, excs.stop_iteration, { - "value" => ctx.none(), - }); - extend_exception!(PyStopAsyncIteration, ctx, excs.stop_async_iteration); - - extend_exception!(PyArithmeticError, ctx, excs.arithmetic_error); - extend_exception!(PyFloatingPointError, ctx, excs.floating_point_error); - extend_exception!(PyOverflowError, ctx, excs.overflow_error); - extend_exception!(PyZeroDivisionError, ctx, excs.zero_division_error); - - extend_exception!(PyAssertionError, ctx, excs.assertion_error); - extend_exception!(PyAttributeError, ctx, excs.attribute_error, { - "name" => ctx.none(), - "obj" => ctx.none(), - }); - extend_exception!(PyBufferError, ctx, excs.buffer_error); - extend_exception!(PyEOFError, ctx, excs.eof_error); - - extend_exception!(PyImportError, ctx, excs.import_error, { - "msg" => ctx.new_readonly_getset("msg", excs.import_error, make_arg_getter(0)), - "name" => ctx.none(), - "path" => ctx.none(), - }); - extend_exception!(PyModuleNotFoundError, ctx, excs.module_not_found_error); - - extend_exception!(PyLookupError, ctx, excs.lookup_error); - extend_exception!(PyIndexError, ctx, excs.index_error); - - extend_exception!(PyKeyError, ctx, excs.key_error); - - extend_exception!(PyMemoryError, ctx, excs.memory_error); - extend_exception!(PyNameError, ctx, excs.name_error, { - "name" => ctx.none(), - }); - extend_exception!(PyUnboundLocalError, ctx, excs.unbound_local_error); - - // os errors: - let errno_getter = - ctx.new_readonly_getset("errno", excs.os_error, |exc: PyBaseExceptionRef| { - let args = exc.args(); - args.first() - .filter(|_| args.len() > 1 && args.len() <= 5) - .cloned() - }); - let strerror_getter = - ctx.new_readonly_getset("strerror", excs.os_error, |exc: PyBaseExceptionRef| { - let args = exc.args(); - args.get(1) - .filter(|_| args.len() >= 2 && args.len() <= 5) - .cloned() - }); - extend_exception!(PyOSError, ctx, excs.os_error, { - // POSIX exception code - "errno" => errno_getter.clone(), - // exception strerror - "strerror" => strerror_getter.clone(), - // exception filename - "filename" => ctx.none(), - // second exception filename - "filename2" => ctx.none(), - }); - // TODO: this isn't really accurate - #[cfg(windows)] - excs.os_error - .set_str_attr("winerror", errno_getter.clone(), ctx); - - extend_exception!(PyBlockingIOError, ctx, excs.blocking_io_error); - extend_exception!(PyChildProcessError, ctx, excs.child_process_error); - - extend_exception!(PyConnectionError, ctx, excs.connection_error); - extend_exception!(PyBrokenPipeError, ctx, excs.broken_pipe_error); - extend_exception!(PyConnectionAbortedError, ctx, excs.connection_aborted_error); - extend_exception!(PyConnectionRefusedError, ctx, excs.connection_refused_error); - extend_exception!(PyConnectionResetError, ctx, excs.connection_reset_error); - - extend_exception!(PyFileExistsError, ctx, excs.file_exists_error); - extend_exception!(PyFileNotFoundError, ctx, excs.file_not_found_error); - extend_exception!(PyInterruptedError, ctx, excs.interrupted_error); - extend_exception!(PyIsADirectoryError, ctx, excs.is_a_directory_error); - extend_exception!(PyNotADirectoryError, ctx, excs.not_a_directory_error); - extend_exception!(PyPermissionError, ctx, excs.permission_error); - extend_exception!(PyProcessLookupError, ctx, excs.process_lookup_error); - extend_exception!(PyTimeoutError, ctx, excs.timeout_error); - - extend_exception!(PyReferenceError, ctx, excs.reference_error); - extend_exception!(PyRuntimeError, ctx, excs.runtime_error); - extend_exception!(PyNotImplementedError, ctx, excs.not_implemented_error); - extend_exception!(PyRecursionError, ctx, excs.recursion_error); - - extend_exception!(PySyntaxError, ctx, excs.syntax_error, { - "msg" => ctx.new_readonly_getset("msg", excs.syntax_error, make_arg_getter(0)), - // TODO: members - "filename" => ctx.none(), - "lineno" => ctx.none(), - "end_lineno" => ctx.none(), - "offset" => ctx.none(), - "end_offset" => ctx.none(), - "text" => ctx.none(), - }); - extend_exception!(PyIndentationError, ctx, excs.indentation_error); - extend_exception!(PyTabError, ctx, excs.tab_error); - - extend_exception!(PySystemError, ctx, excs.system_error); - extend_exception!(PyTypeError, ctx, excs.type_error); - extend_exception!(PyValueError, ctx, excs.value_error); - extend_exception!(PyUnicodeError, ctx, excs.unicode_error, { - "encoding" => ctx.new_readonly_getset("encoding", excs.unicode_error, make_arg_getter(0)), - "object" => ctx.new_readonly_getset("object", excs.unicode_error, make_arg_getter(1)), - "start" => ctx.new_readonly_getset("start", excs.unicode_error, make_arg_getter(2)), - "end" => ctx.new_readonly_getset("end", excs.unicode_error, make_arg_getter(3)), - "reason" => ctx.new_readonly_getset("reason", excs.unicode_error, make_arg_getter(4)), - }); - extend_exception!(PyUnicodeDecodeError, ctx, excs.unicode_decode_error); - extend_exception!(PyUnicodeEncodeError, ctx, excs.unicode_encode_error); - extend_exception!(PyUnicodeTranslateError, ctx, excs.unicode_translate_error, { - "encoding" => ctx.new_readonly_getset("encoding", excs.unicode_translate_error, none_getter), - "object" => ctx.new_readonly_getset("object", excs.unicode_translate_error, make_arg_getter(0)), - "start" => ctx.new_readonly_getset("start", excs.unicode_translate_error, make_arg_getter(1)), - "end" => ctx.new_readonly_getset("end", excs.unicode_translate_error, make_arg_getter(2)), - "reason" => ctx.new_readonly_getset("reason", excs.unicode_translate_error, make_arg_getter(3)), - }); - - #[cfg(feature = "jit")] - extend_exception!(PyJitError, ctx, excs.jit_error); - - extend_exception!(PyWarning, ctx, excs.warning); - extend_exception!(PyDeprecationWarning, ctx, excs.deprecation_warning); - extend_exception!( - PyPendingDeprecationWarning, - ctx, - excs.pending_deprecation_warning - ); - extend_exception!(PyRuntimeWarning, ctx, excs.runtime_warning); - extend_exception!(PySyntaxWarning, ctx, excs.syntax_warning); - extend_exception!(PyUserWarning, ctx, excs.user_warning); - extend_exception!(PyFutureWarning, ctx, excs.future_warning); - extend_exception!(PyImportWarning, ctx, excs.import_warning); - extend_exception!(PyUnicodeWarning, ctx, excs.unicode_warning); - extend_exception!(PyBytesWarning, ctx, excs.bytes_warning); - extend_exception!(PyResourceWarning, ctx, excs.resource_warning); - extend_exception!(PyEncodingWarning, ctx, excs.encoding_warning); - } -} - -fn none_getter(_obj: PyObjectRef, vm: &VirtualMachine) -> PyRef<PyNone> { - vm.ctx.none.clone() -} - -fn make_arg_getter(idx: usize) -> impl Fn(PyBaseExceptionRef) -> Option<PyObjectRef> { - move |exc| exc.get_arg(idx) -} - -fn system_exit_code(exc: PyBaseExceptionRef) -> Option<PyObjectRef> { - exc.args.read().first().map(|code| { - match_class!(match code { - ref tup @ PyTuple => match tup.as_slice() { - [x] => x.clone(), - _ => code.clone(), - }, - other => other.clone(), - }) - }) -} - -#[cfg(feature = "serde")] -pub struct SerializeException<'vm, 's> { - vm: &'vm VirtualMachine, - exc: &'s PyBaseExceptionRef, -} - -#[cfg(feature = "serde")] -impl<'vm, 's> SerializeException<'vm, 's> { - pub fn new(vm: &'vm VirtualMachine, exc: &'s PyBaseExceptionRef) -> Self { - SerializeException { vm, exc } - } -} - -#[cfg(feature = "serde")] -pub struct SerializeExceptionOwned<'vm> { - vm: &'vm VirtualMachine, - exc: PyBaseExceptionRef, -} - -#[cfg(feature = "serde")] -impl serde::Serialize for SerializeExceptionOwned<'_> { - fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { - let Self { vm, exc } = self; - SerializeException::new(vm, exc).serialize(s) - } -} - -#[cfg(feature = "serde")] -impl serde::Serialize for SerializeException<'_, '_> { - fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { - use serde::ser::*; - - let mut struc = s.serialize_struct("PyBaseException", 7)?; - struc.serialize_field("exc_type", &*self.exc.class().name())?; - let tbs = { - struct Tracebacks(PyTracebackRef); - impl serde::Serialize for Tracebacks { - fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { - let mut s = s.serialize_seq(None)?; - for tb in self.0.iter() { - s.serialize_element(&**tb)?; - } - s.end() - } - } - self.exc.traceback().map(Tracebacks) - }; - struc.serialize_field("traceback", &tbs)?; - struc.serialize_field( - "cause", - &self - .exc - .cause() - .map(|exc| SerializeExceptionOwned { vm: self.vm, exc }), - )?; - struc.serialize_field( - "context", - &self - .exc - .context() - .map(|exc| SerializeExceptionOwned { vm: self.vm, exc }), - )?; - struc.serialize_field("suppress_context", &self.exc.get_suppress_context())?; - - let args = { - struct Args<'vm>(&'vm VirtualMachine, PyTupleRef); - impl serde::Serialize for Args<'_> { - fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> { - s.collect_seq( - self.1 - .iter() - .map(|arg| crate::py_serde::PyObjectSerializer::new(self.0, arg)), - ) - } - } - Args(self.vm, self.exc.args()) - }; - struc.serialize_field("args", &args)?; - - let rendered = { - let mut rendered = String::new(); - self.vm - .write_exception(&mut rendered, self.exc) - .map_err(S::Error::custom)?; - rendered - }; - struc.serialize_field("rendered", &rendered)?; - - struc.end() - } -} - -pub fn cstring_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_value_error("embedded null character".to_owned()) -} - -impl ToPyException for std::ffi::NulError { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - cstring_error(vm) - } -} - -#[cfg(windows)] -impl<C: widestring::UChar> ToPyException for widestring::NulError<C> { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - cstring_error(vm) - } -} - -#[cfg(any(unix, windows, target_os = "wasi"))] -pub(crate) fn raw_os_error_to_exc_type( - errno: i32, - vm: &VirtualMachine, -) -> Option<&'static Py<PyType>> { - use crate::stdlib::errno::errors; - let excs = &vm.ctx.exceptions; - match errno { - errors::EWOULDBLOCK => Some(excs.blocking_io_error), - errors::EALREADY => Some(excs.blocking_io_error), - errors::EINPROGRESS => Some(excs.blocking_io_error), - errors::EPIPE => Some(excs.broken_pipe_error), - #[cfg(not(target_os = "wasi"))] - errors::ESHUTDOWN => Some(excs.broken_pipe_error), - errors::ECHILD => Some(excs.child_process_error), - errors::ECONNABORTED => Some(excs.connection_aborted_error), - errors::ECONNREFUSED => Some(excs.connection_refused_error), - errors::ECONNRESET => Some(excs.connection_reset_error), - errors::EEXIST => Some(excs.file_exists_error), - errors::ENOENT => Some(excs.file_not_found_error), - errors::EISDIR => Some(excs.is_a_directory_error), - errors::ENOTDIR => Some(excs.not_a_directory_error), - errors::EINTR => Some(excs.interrupted_error), - errors::EACCES => Some(excs.permission_error), - errors::EPERM => Some(excs.permission_error), - errors::ESRCH => Some(excs.process_lookup_error), - errors::ETIMEDOUT => Some(excs.timeout_error), - _ => None, - } -} - -#[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] -pub(crate) fn raw_os_error_to_exc_type( - _errno: i32, - _vm: &VirtualMachine, -) -> Option<&'static Py<PyType>> { - None -} - -pub(super) mod types { - use crate::common::lock::PyRwLock; - #[cfg_attr(target_arch = "wasm32", allow(unused_imports))] - use crate::{ - builtins::{ - traceback::PyTracebackRef, tuple::IntoPyTuple, PyInt, PyStrRef, PyTupleRef, PyTypeRef, - }, - convert::ToPyResult, - function::FuncArgs, - types::{Constructor, Initializer}, - AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine, - }; - use crossbeam_utils::atomic::AtomicCell; - use itertools::Itertools; - - // This module is designed to be used as `use builtins::*;`. - // Do not add any pub symbols not included in builtins module. - // `PyBaseExceptionRef` is the only exception. - - pub type PyBaseExceptionRef = PyRef<PyBaseException>; - - // Sorted By Hierarchy then alphabetized. - - #[pyclass(module = false, name = "BaseException", traverse = "manual")] - pub struct PyBaseException { - pub(super) traceback: PyRwLock<Option<PyTracebackRef>>, - pub(super) cause: PyRwLock<Option<PyRef<Self>>>, - pub(super) context: PyRwLock<Option<PyRef<Self>>>, - pub(super) suppress_context: AtomicCell<bool>, - pub(super) args: PyRwLock<PyTupleRef>, - } - - #[pyexception(name, base = "PyBaseException", ctx = "system_exit", impl)] - #[derive(Debug)] - pub struct PySystemExit {} - - #[pyexception(name, base = "PyBaseException", ctx = "base_exception_group", impl)] - #[derive(Debug)] - pub struct PyBaseExceptionGroup {} - - #[pyexception(name, base = "PyBaseException", ctx = "generator_exit", impl)] - #[derive(Debug)] - pub struct PyGeneratorExit {} - - #[pyexception(name, base = "PyBaseException", ctx = "keyboard_interrupt", impl)] - #[derive(Debug)] - pub struct PyKeyboardInterrupt {} - - #[pyexception(name, base = "PyBaseException", ctx = "exception_type", impl)] - #[derive(Debug)] - pub struct PyException {} - - #[pyexception(name, base = "PyException", ctx = "stop_iteration")] - #[derive(Debug)] - pub struct PyStopIteration {} - - #[pyexception] - impl PyStopIteration { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult<()> { - zelf.set_attr("value", vm.unwrap_or_none(args.args.first().cloned()), vm)?; - Ok(()) - } - } - - #[pyexception(name, base = "PyException", ctx = "stop_async_iteration", impl)] - #[derive(Debug)] - pub struct PyStopAsyncIteration {} - - #[pyexception(name, base = "PyException", ctx = "arithmetic_error", impl)] - #[derive(Debug)] - pub struct PyArithmeticError {} - - #[pyexception(name, base = "PyArithmeticError", ctx = "floating_point_error", impl)] - #[derive(Debug)] - pub struct PyFloatingPointError {} - - #[pyexception(name, base = "PyArithmeticError", ctx = "overflow_error", impl)] - #[derive(Debug)] - pub struct PyOverflowError {} - - #[pyexception(name, base = "PyArithmeticError", ctx = "zero_division_error", impl)] - #[derive(Debug)] - pub struct PyZeroDivisionError {} - - #[pyexception(name, base = "PyException", ctx = "assertion_error", impl)] - #[derive(Debug)] - pub struct PyAssertionError {} - - #[pyexception(name, base = "PyException", ctx = "attribute_error", impl)] - #[derive(Debug)] - pub struct PyAttributeError {} - - #[pyexception(name, base = "PyException", ctx = "buffer_error", impl)] - #[derive(Debug)] - pub struct PyBufferError {} - - #[pyexception(name, base = "PyException", ctx = "eof_error", impl)] - #[derive(Debug)] - pub struct PyEOFError {} - - #[pyexception(name, base = "PyException", ctx = "import_error")] - #[derive(Debug)] - pub struct PyImportError {} - - #[pyexception] - impl PyImportError { - #[pyslot] - #[pymethod(name = "__init__")] - pub(crate) fn slot_init( - zelf: PyObjectRef, - args: ::rustpython_vm::function::FuncArgs, - vm: &::rustpython_vm::VirtualMachine, - ) -> ::rustpython_vm::PyResult<()> { - zelf.set_attr( - "name", - vm.unwrap_or_none(args.kwargs.get("name").cloned()), - vm, - )?; - zelf.set_attr( - "path", - vm.unwrap_or_none(args.kwargs.get("path").cloned()), - vm, - )?; - Ok(()) - } - #[pymethod(magic)] - fn reduce(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyTupleRef { - let obj = exc.as_object().to_owned(); - let mut result: Vec<PyObjectRef> = vec![ - obj.class().to_owned().into(), - vm.new_tuple((exc.get_arg(0).unwrap(),)).into(), - ]; - - if let Some(dict) = obj.dict().filter(|x| !x.is_empty()) { - result.push(dict.into()); - } - - result.into_pytuple(vm) - } - } - - #[pyexception(name, base = "PyImportError", ctx = "module_not_found_error", impl)] - #[derive(Debug)] - pub struct PyModuleNotFoundError {} - - #[pyexception(name, base = "PyException", ctx = "lookup_error", impl)] - #[derive(Debug)] - pub struct PyLookupError {} - - #[pyexception(name, base = "PyLookupError", ctx = "index_error", impl)] - #[derive(Debug)] - pub struct PyIndexError {} - - #[pyexception(name, base = "PyLookupError", ctx = "key_error")] - #[derive(Debug)] - pub struct PyKeyError {} - - #[pyexception] - impl PyKeyError { - #[pymethod(magic)] - fn str(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyStrRef { - let args = exc.args(); - if args.len() == 1 { - vm.exception_args_as_string(args, false) - .into_iter() - .exactly_one() - .unwrap() - } else { - exc.str(vm) - } - } - } - - #[pyexception(name, base = "PyException", ctx = "memory_error", impl)] - #[derive(Debug)] - pub struct PyMemoryError {} - - #[pyexception(name, base = "PyException", ctx = "name_error", impl)] - #[derive(Debug)] - pub struct PyNameError {} - - #[pyexception(name, base = "PyNameError", ctx = "unbound_local_error", impl)] - #[derive(Debug)] - pub struct PyUnboundLocalError {} - - #[pyexception(name, base = "PyException", ctx = "os_error")] - #[derive(Debug)] - pub struct PyOSError {} - - // OS Errors: - #[pyexception] - impl PyOSError { - #[cfg(not(target_arch = "wasm32"))] - fn optional_new(args: Vec<PyObjectRef>, vm: &VirtualMachine) -> Option<PyBaseExceptionRef> { - let len = args.len(); - if (2..=5).contains(&len) { - let errno = &args[0]; - errno - .payload_if_subclass::<PyInt>(vm) - .and_then(|errno| errno.try_to_primitive::<i32>(vm).ok()) - .and_then(|errno| super::raw_os_error_to_exc_type(errno, vm)) - .and_then(|typ| vm.invoke_exception(typ.to_owned(), args.to_vec()).ok()) - } else { - None - } - } - #[cfg(not(target_arch = "wasm32"))] - #[pyslot] - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // We need this method, because of how `CPython` copies `init` - // from `BaseException` in `SimpleExtendsException` macro. - // See: `BaseException_new` - if *cls.name() == *vm.ctx.exceptions.os_error.name() { - match Self::optional_new(args.args.to_vec(), vm) { - Some(error) => error.to_pyresult(vm), - None => PyBaseException::slot_new(cls, args, vm), - } - } else { - PyBaseException::slot_new(cls, args, vm) - } - } - #[cfg(target_arch = "wasm32")] - #[pyslot] - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyBaseException::slot_new(cls, args, vm) - } - #[pyslot] - #[pymethod(name = "__init__")] - fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let len = args.args.len(); - let mut new_args = args; - if (3..=5).contains(&len) { - zelf.set_attr("filename", new_args.args[2].clone(), vm)?; - if len == 5 { - zelf.set_attr("filename2", new_args.args[4].clone(), vm)?; - } - - new_args.args.truncate(2); - } - PyBaseException::slot_init(zelf, new_args, vm) - } - - #[pymethod(magic)] - fn str(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let args = exc.args(); - let obj = exc.as_object().to_owned(); - - if args.len() == 2 { - // SAFETY: len() == 2 is checked so get_arg 1 or 2 won't panic - let errno = exc.get_arg(0).unwrap().str(vm)?; - let msg = exc.get_arg(1).unwrap().str(vm)?; - - let s = match obj.get_attr("filename", vm) { - Ok(filename) => match obj.get_attr("filename2", vm) { - Ok(filename2) => format!( - "[Errno {}] {}: '{}' -> '{}'", - errno, - msg, - filename.str(vm)?, - filename2.str(vm)? - ), - Err(_) => format!("[Errno {}] {}: '{}'", errno, msg, filename.str(vm)?), - }, - Err(_) => { - format!("[Errno {errno}] {msg}") - } - }; - Ok(vm.ctx.new_str(s)) - } else { - Ok(exc.str(vm)) - } - } - - #[pymethod(magic)] - fn reduce(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyTupleRef { - let args = exc.args(); - let obj = exc.as_object().to_owned(); - let mut result: Vec<PyObjectRef> = vec![obj.class().to_owned().into()]; - - if args.len() >= 2 && args.len() <= 5 { - // SAFETY: len() == 2 is checked so get_arg 1 or 2 won't panic - let errno = exc.get_arg(0).unwrap(); - let msg = exc.get_arg(1).unwrap(); - - if let Ok(filename) = obj.get_attr("filename", vm) { - if !vm.is_none(&filename) { - let mut args_reduced: Vec<PyObjectRef> = vec![errno, msg, filename]; - - if let Ok(filename2) = obj.get_attr("filename2", vm) { - if !vm.is_none(&filename2) { - args_reduced.push(filename2); - } - } - result.push(args_reduced.into_pytuple(vm).into()); - } else { - result.push(vm.new_tuple((errno, msg)).into()); - } - } else { - result.push(vm.new_tuple((errno, msg)).into()); - } - } else { - result.push(args.into()); - } - - if let Some(dict) = obj.dict().filter(|x| !x.is_empty()) { - result.push(dict.into()); - } - result.into_pytuple(vm) - } - } - - #[pyexception(name, base = "PyOSError", ctx = "blocking_io_error", impl)] - #[derive(Debug)] - pub struct PyBlockingIOError {} - - #[pyexception(name, base = "PyOSError", ctx = "child_process_error", impl)] - #[derive(Debug)] - pub struct PyChildProcessError {} - - #[pyexception(name, base = "PyOSError", ctx = "connection_error", impl)] - #[derive(Debug)] - pub struct PyConnectionError {} - - #[pyexception(name, base = "PyConnectionError", ctx = "broken_pipe_error", impl)] - #[derive(Debug)] - pub struct PyBrokenPipeError {} - - #[pyexception( - name, - base = "PyConnectionError", - ctx = "connection_aborted_error", - impl - )] - #[derive(Debug)] - pub struct PyConnectionAbortedError {} - - #[pyexception( - name, - base = "PyConnectionError", - ctx = "connection_refused_error", - impl - )] - #[derive(Debug)] - pub struct PyConnectionRefusedError {} - - #[pyexception(name, base = "PyConnectionError", ctx = "connection_reset_error", impl)] - #[derive(Debug)] - pub struct PyConnectionResetError {} - - #[pyexception(name, base = "PyOSError", ctx = "file_exists_error", impl)] - #[derive(Debug)] - pub struct PyFileExistsError {} - - #[pyexception(name, base = "PyOSError", ctx = "file_not_found_error", impl)] - #[derive(Debug)] - pub struct PyFileNotFoundError {} - - #[pyexception(name, base = "PyOSError", ctx = "interrupted_error", impl)] - #[derive(Debug)] - pub struct PyInterruptedError {} - - #[pyexception(name, base = "PyOSError", ctx = "is_a_directory_error", impl)] - #[derive(Debug)] - pub struct PyIsADirectoryError {} - - #[pyexception(name, base = "PyOSError", ctx = "not_a_directory_error", impl)] - #[derive(Debug)] - pub struct PyNotADirectoryError {} - - #[pyexception(name, base = "PyOSError", ctx = "permission_error", impl)] - #[derive(Debug)] - pub struct PyPermissionError {} - - #[pyexception(name, base = "PyOSError", ctx = "process_lookup_error", impl)] - #[derive(Debug)] - pub struct PyProcessLookupError {} - - #[pyexception(name, base = "PyOSError", ctx = "timeout_error", impl)] - #[derive(Debug)] - pub struct PyTimeoutError {} - - #[pyexception(name, base = "PyException", ctx = "reference_error", impl)] - #[derive(Debug)] - pub struct PyReferenceError {} - - #[pyexception(name, base = "PyException", ctx = "runtime_error", impl)] - #[derive(Debug)] - pub struct PyRuntimeError {} - - #[pyexception(name, base = "PyRuntimeError", ctx = "not_implemented_error", impl)] - #[derive(Debug)] - pub struct PyNotImplementedError {} - - #[pyexception(name, base = "PyRuntimeError", ctx = "recursion_error", impl)] - #[derive(Debug)] - pub struct PyRecursionError {} - - #[pyexception(name, base = "PyException", ctx = "syntax_error", impl)] - #[derive(Debug)] - pub struct PySyntaxError {} - - #[pyexception(name, base = "PySyntaxError", ctx = "indentation_error", impl)] - #[derive(Debug)] - pub struct PyIndentationError {} - - #[pyexception(name, base = "PyIndentationError", ctx = "tab_error", impl)] - #[derive(Debug)] - pub struct PyTabError {} - - #[pyexception(name, base = "PyException", ctx = "system_error", impl)] - #[derive(Debug)] - pub struct PySystemError {} - - #[pyexception(name, base = "PyException", ctx = "type_error", impl)] - #[derive(Debug)] - pub struct PyTypeError {} - - #[pyexception(name, base = "PyException", ctx = "value_error", impl)] - #[derive(Debug)] - pub struct PyValueError {} - - #[pyexception(name, base = "PyValueError", ctx = "unicode_error", impl)] - #[derive(Debug)] - pub struct PyUnicodeError {} - - #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_decode_error", impl)] - #[derive(Debug)] - pub struct PyUnicodeDecodeError {} - - #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_encode_error", impl)] - #[derive(Debug)] - pub struct PyUnicodeEncodeError {} - - #[pyexception(name, base = "PyUnicodeError", ctx = "unicode_translate_error", impl)] - #[derive(Debug)] - pub struct PyUnicodeTranslateError {} - - /// JIT error. - #[cfg(feature = "jit")] - #[pyexception(name, base = "PyException", ctx = "jit_error", impl)] - #[derive(Debug)] - pub struct PyJitError {} - - // Warnings - #[pyexception(name, base = "PyException", ctx = "warning", impl)] - #[derive(Debug)] - pub struct PyWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "deprecation_warning", impl)] - #[derive(Debug)] - pub struct PyDeprecationWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "pending_deprecation_warning", impl)] - #[derive(Debug)] - pub struct PyPendingDeprecationWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "runtime_warning", impl)] - #[derive(Debug)] - pub struct PyRuntimeWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "syntax_warning", impl)] - #[derive(Debug)] - pub struct PySyntaxWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "user_warning", impl)] - #[derive(Debug)] - pub struct PyUserWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "future_warning", impl)] - #[derive(Debug)] - pub struct PyFutureWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "import_warning", impl)] - #[derive(Debug)] - pub struct PyImportWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "unicode_warning", impl)] - #[derive(Debug)] - pub struct PyUnicodeWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "bytes_warning", impl)] - #[derive(Debug)] - pub struct PyBytesWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "resource_warning", impl)] - #[derive(Debug)] - pub struct PyResourceWarning {} - - #[pyexception(name, base = "PyWarning", ctx = "encoding_warning", impl)] - #[derive(Debug)] - pub struct PyEncodingWarning {} -} diff --git a/vm/src/format.rs b/vm/src/format.rs deleted file mode 100644 index 8109ea00f4a..00000000000 --- a/vm/src/format.rs +++ /dev/null @@ -1,174 +0,0 @@ -use crate::{ - builtins::PyBaseExceptionRef, - convert::{IntoPyException, ToPyException}, - function::FuncArgs, - stdlib::builtins, - PyObject, PyResult, VirtualMachine, -}; - -use rustpython_format::*; - -impl IntoPyException for FormatSpecError { - fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { - match self { - FormatSpecError::DecimalDigitsTooMany => { - vm.new_value_error("Too many decimal digits in format string".to_owned()) - } - FormatSpecError::PrecisionTooBig => vm.new_value_error("Precision too big".to_owned()), - FormatSpecError::InvalidFormatSpecifier => { - vm.new_value_error("Invalid format specifier".to_owned()) - } - FormatSpecError::UnspecifiedFormat(c1, c2) => { - let msg = format!("Cannot specify '{c1}' with '{c2}'."); - vm.new_value_error(msg) - } - FormatSpecError::UnknownFormatCode(c, s) => { - let msg = format!("Unknown format code '{c}' for object of type '{s}'"); - vm.new_value_error(msg) - } - FormatSpecError::PrecisionNotAllowed => { - vm.new_value_error("Precision not allowed in integer format specifier".to_owned()) - } - FormatSpecError::NotAllowed(s) => { - let msg = format!("{s} not allowed with integer format specifier 'c'"); - vm.new_value_error(msg) - } - FormatSpecError::UnableToConvert => { - vm.new_value_error("Unable to convert int to float".to_owned()) - } - FormatSpecError::CodeNotInRange => { - vm.new_overflow_error("%c arg not in range(0x110000)".to_owned()) - } - FormatSpecError::NotImplemented(c, s) => { - let msg = format!("Format code '{c}' for object of type '{s}' not implemented yet"); - vm.new_value_error(msg) - } - } - } -} - -impl ToPyException for FormatParseError { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - match self { - FormatParseError::UnmatchedBracket => { - vm.new_value_error("expected '}' before end of string".to_owned()) - } - _ => vm.new_value_error("Unexpected error parsing format string".to_owned()), - } - } -} - -fn format_internal( - vm: &VirtualMachine, - format: &FormatString, - field_func: &mut impl FnMut(FieldType) -> PyResult, -) -> PyResult<String> { - let mut final_string = String::new(); - for part in &format.format_parts { - let pystr; - let result_string: &str = match part { - FormatPart::Field { - field_name, - conversion_spec, - format_spec, - } => { - let FieldName { field_type, parts } = - FieldName::parse(field_name.as_str()).map_err(|e| e.to_pyexception(vm))?; - - let mut argument = field_func(field_type)?; - - for name_part in parts { - match name_part { - FieldNamePart::Attribute(attribute) => { - argument = argument.get_attr(&vm.ctx.new_str(attribute), vm)?; - } - FieldNamePart::Index(index) => { - argument = argument.get_item(&index, vm)?; - } - FieldNamePart::StringIndex(index) => { - argument = argument.get_item(&index, vm)?; - } - } - } - - let nested_format = - FormatString::from_str(format_spec).map_err(|e| e.to_pyexception(vm))?; - let format_spec = format_internal(vm, &nested_format, field_func)?; - - let argument = match conversion_spec.and_then(FormatConversion::from_char) { - Some(FormatConversion::Str) => argument.str(vm)?.into(), - Some(FormatConversion::Repr) => argument.repr(vm)?.into(), - Some(FormatConversion::Ascii) => { - vm.ctx.new_str(builtins::ascii(argument, vm)?).into() - } - Some(FormatConversion::Bytes) => { - vm.call_method(&argument, identifier!(vm, decode).as_str(), ())? - } - None => argument, - }; - - // FIXME: compiler can intern specs using parser tree. Then this call can be interned_str - pystr = vm.format(&argument, vm.ctx.new_str(format_spec))?; - pystr.as_ref() - } - FormatPart::Literal(literal) => literal, - }; - final_string.push_str(result_string); - } - Ok(final_string) -} - -pub(crate) fn format( - format: &FormatString, - arguments: &FuncArgs, - vm: &VirtualMachine, -) -> PyResult<String> { - let mut auto_argument_index: usize = 0; - let mut seen_index = false; - format_internal(vm, format, &mut |field_type| match field_type { - FieldType::Auto => { - if seen_index { - return Err(vm.new_value_error( - "cannot switch from manual field specification to automatic field numbering" - .to_owned(), - )); - } - auto_argument_index += 1; - arguments - .args - .get(auto_argument_index - 1) - .cloned() - .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) - } - FieldType::Index(index) => { - if auto_argument_index != 0 { - return Err(vm.new_value_error( - "cannot switch from automatic field numbering to manual field specification" - .to_owned(), - )); - } - seen_index = true; - arguments - .args - .get(index) - .cloned() - .ok_or_else(|| vm.new_index_error("tuple index out of range".to_owned())) - } - FieldType::Keyword(keyword) => arguments - .get_optional_kwarg(&keyword) - .ok_or_else(|| vm.new_key_error(vm.ctx.new_str(keyword).into())), - }) -} - -pub(crate) fn format_map( - format: &FormatString, - dict: &PyObject, - vm: &VirtualMachine, -) -> PyResult<String> { - format_internal(vm, format, &mut |field_type| match field_type { - FieldType::Auto | FieldType::Index(_) => { - Err(vm.new_value_error("Format string contains positional fields".to_owned())) - } - FieldType::Keyword(keyword) => dict.get_item(&keyword, vm), - }) -} diff --git a/vm/src/frame.rs b/vm/src/frame.rs deleted file mode 100644 index a0af23336fc..00000000000 --- a/vm/src/frame.rs +++ /dev/null @@ -1,1972 +0,0 @@ -use crate::common::{boxvec::BoxVec, lock::PyMutex}; -use crate::{ - builtins::{ - asyncgenerator::PyAsyncGenWrappedValue, - function::{PyCell, PyCellRef, PyFunction}, - tuple::{PyTuple, PyTupleTyped}, - PyBaseExceptionRef, PyCode, PyCoroutine, PyDict, PyDictRef, PyGenerator, PyList, PySet, - PySlice, PyStr, PyStrInterned, PyStrRef, PyTraceback, PyType, - }, - bytecode, - convert::{IntoObject, ToPyResult}, - coroutine::Coro, - exceptions::ExceptionCtor, - function::{ArgMapping, Either, FuncArgs}, - protocol::{PyIter, PyIterReturn}, - scope::Scope, - source_code::SourceLocation, - stdlib::builtins, - vm::{Context, PyMethod}, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, -}; -use indexmap::IndexMap; -use itertools::Itertools; -#[cfg(feature = "threading")] -use std::sync::atomic; -use std::{fmt, iter::zip}; - -#[derive(Clone, Debug)] -struct Block { - /// The type of block. - typ: BlockType, - /// The level of the value stack when the block was entered. - level: usize, -} - -#[derive(Clone, Debug)] -enum BlockType { - Loop, - TryExcept { - handler: bytecode::Label, - }, - Finally { - handler: bytecode::Label, - }, - - /// Active finally sequence - FinallyHandler { - reason: Option<UnwindReason>, - prev_exc: Option<PyBaseExceptionRef>, - }, - ExceptHandler { - prev_exc: Option<PyBaseExceptionRef>, - }, -} - -pub type FrameRef = PyRef<Frame>; - -/// The reason why we might be unwinding a block. -/// This could be return of function, exception being -/// raised, a break or continue being hit, etc.. -#[derive(Clone, Debug)] -enum UnwindReason { - /// We are returning a value from a return statement. - Returning { value: PyObjectRef }, - - /// We hit an exception, so unwind any try-except and finally blocks. The exception should be - /// on top of the vm exception stack. - Raising { exception: PyBaseExceptionRef }, - - // NoWorries, - /// We are unwinding blocks, since we hit break - Break { target: bytecode::Label }, - - /// We are unwinding blocks since we hit a continue statements. - Continue { target: bytecode::Label }, -} - -#[derive(Debug)] -struct FrameState { - // We need 1 stack per frame - /// The main data frame of the stack machine - stack: BoxVec<PyObjectRef>, - /// Block frames, for controlling loops and exceptions - blocks: Vec<Block>, - /// index of last instruction ran - #[cfg(feature = "threading")] - lasti: u32, -} - -#[cfg(feature = "threading")] -type Lasti = atomic::AtomicU32; -#[cfg(not(feature = "threading"))] -type Lasti = std::cell::Cell<u32>; - -#[pyclass(module = false, name = "frame")] -pub struct Frame { - pub code: PyRef<PyCode>, - - pub fastlocals: PyMutex<Box<[Option<PyObjectRef>]>>, - pub(crate) cells_frees: Box<[PyCellRef]>, - pub locals: ArgMapping, - pub globals: PyDictRef, - pub builtins: PyDictRef, - - // on feature=threading, this is a duplicate of FrameState.lasti, but it's faster to do an - // atomic store than it is to do a fetch_add, for every instruction executed - /// index of last instruction ran - pub lasti: Lasti, - /// tracer function for this frame (usually is None) - pub trace: PyMutex<PyObjectRef>, - state: PyMutex<FrameState>, - - // member - pub trace_lines: PyMutex<bool>, - pub temporary_refs: PyMutex<Vec<PyObjectRef>>, -} - -impl PyPayload for Frame { - fn class(ctx: &Context) -> &'static Py<PyType> { - ctx.types.frame_type - } -} - -// Running a frame can result in one of the below: -pub enum ExecutionResult { - Return(PyObjectRef), - Yield(PyObjectRef), -} - -/// A valid execution result, or an exception -type FrameResult = PyResult<Option<ExecutionResult>>; - -impl Frame { - pub(crate) fn new( - code: PyRef<PyCode>, - scope: Scope, - builtins: PyDictRef, - closure: &[PyCellRef], - vm: &VirtualMachine, - ) -> Frame { - let cells_frees = std::iter::repeat_with(|| PyCell::default().into_ref(&vm.ctx)) - .take(code.cellvars.len()) - .chain(closure.iter().cloned()) - .collect(); - - let state = FrameState { - stack: BoxVec::new(code.max_stackdepth as usize), - blocks: Vec::new(), - #[cfg(feature = "threading")] - lasti: 0, - }; - - Frame { - fastlocals: PyMutex::new(vec![None; code.varnames.len()].into_boxed_slice()), - cells_frees, - locals: scope.locals, - globals: scope.globals, - builtins, - code, - lasti: Lasti::new(0), - state: PyMutex::new(state), - trace: PyMutex::new(vm.ctx.none()), - trace_lines: PyMutex::new(true), - temporary_refs: PyMutex::new(vec![]), - } - } - - pub fn current_location(&self) -> SourceLocation { - self.code.locations[self.lasti() as usize - 1] - } - - pub fn lasti(&self) -> u32 { - #[cfg(feature = "threading")] - { - self.lasti.load(atomic::Ordering::Relaxed) - } - #[cfg(not(feature = "threading"))] - { - self.lasti.get() - } - } - - pub fn locals(&self, vm: &VirtualMachine) -> PyResult<ArgMapping> { - let locals = &self.locals; - let code = &**self.code; - let map = &code.varnames; - let j = std::cmp::min(map.len(), code.varnames.len()); - if !code.varnames.is_empty() { - let fastlocals = self.fastlocals.lock(); - for (&k, v) in zip(&map[..j], &**fastlocals) { - match locals.mapping().ass_subscript(k, v.clone(), vm) { - Ok(()) => {} - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} - Err(e) => return Err(e), - } - } - } - if !code.cellvars.is_empty() || !code.freevars.is_empty() { - let map_to_dict = |keys: &[&PyStrInterned], values: &[PyCellRef]| { - for (&k, v) in zip(keys, values) { - if let Some(value) = v.get() { - locals.mapping().ass_subscript(k, Some(value), vm)?; - } else { - match locals.mapping().ass_subscript(k, None, vm) { - Ok(()) => {} - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => {} - Err(e) => return Err(e), - } - } - } - Ok(()) - }; - map_to_dict(&code.cellvars, &self.cells_frees)?; - if code.flags.contains(bytecode::CodeFlags::IS_OPTIMIZED) { - map_to_dict(&code.freevars, &self.cells_frees[code.cellvars.len()..])?; - } - } - Ok(locals.clone()) - } -} - -impl Py<Frame> { - #[inline(always)] - fn with_exec<R>(&self, f: impl FnOnce(ExecutingFrame) -> R) -> R { - let mut state = self.state.lock(); - let exec = ExecutingFrame { - code: &self.code, - fastlocals: &self.fastlocals, - cells_frees: &self.cells_frees, - locals: &self.locals, - globals: &self.globals, - builtins: &self.builtins, - lasti: &self.lasti, - object: self, - state: &mut state, - }; - f(exec) - } - - // #[cfg_attr(feature = "flame-it", flame("Frame"))] - pub fn run(&self, vm: &VirtualMachine) -> PyResult<ExecutionResult> { - self.with_exec(|mut exec| exec.run(vm)) - } - - pub(crate) fn resume( - &self, - value: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<ExecutionResult> { - self.with_exec(|mut exec| { - if let Some(value) = value { - exec.push_value(value) - } - exec.run(vm) - }) - } - - pub(crate) fn gen_throw( - &self, - vm: &VirtualMachine, - exc_type: PyObjectRef, - exc_val: PyObjectRef, - exc_tb: PyObjectRef, - ) -> PyResult<ExecutionResult> { - self.with_exec(|mut exec| exec.gen_throw(vm, exc_type, exc_val, exc_tb)) - } - - pub fn yield_from_target(&self) -> Option<PyObjectRef> { - self.with_exec(|exec| exec.yield_from_target().map(PyObject::to_owned)) - } - - pub fn is_internal_frame(&self) -> bool { - let code = self.f_code(); - let filename = code.co_filename(); - - filename.as_str().contains("importlib") && filename.as_str().contains("_bootstrap") - } - - pub fn next_external_frame(&self, vm: &VirtualMachine) -> Option<FrameRef> { - self.f_back(vm).map(|mut back| loop { - back = if let Some(back) = back.to_owned().f_back(vm) { - back - } else { - break back; - }; - - if !back.is_internal_frame() { - break back; - } - }) - } -} - -/// An executing frame; essentially just a struct to combine the immutable data outside the mutex -/// with the mutable data inside -struct ExecutingFrame<'a> { - code: &'a PyRef<PyCode>, - fastlocals: &'a PyMutex<Box<[Option<PyObjectRef>]>>, - cells_frees: &'a [PyCellRef], - locals: &'a ArgMapping, - globals: &'a PyDictRef, - builtins: &'a PyDictRef, - object: &'a Py<Frame>, - lasti: &'a Lasti, - state: &'a mut FrameState, -} - -impl fmt::Debug for ExecutingFrame<'_> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("ExecutingFrame") - .field("code", self.code) - // .field("scope", self.scope) - .field("state", self.state) - .finish() - } -} - -impl ExecutingFrame<'_> { - #[inline(always)] - fn update_lasti(&mut self, f: impl FnOnce(&mut u32)) { - #[cfg(feature = "threading")] - { - f(&mut self.state.lasti); - self.lasti - .store(self.state.lasti, atomic::Ordering::Relaxed); - } - #[cfg(not(feature = "threading"))] - { - let mut lasti = self.lasti.get(); - f(&mut lasti); - self.lasti.set(lasti); - } - } - - #[inline(always)] - fn lasti(&self) -> u32 { - #[cfg(feature = "threading")] - { - self.state.lasti - } - #[cfg(not(feature = "threading"))] - { - self.lasti.get() - } - } - - fn run(&mut self, vm: &VirtualMachine) -> PyResult<ExecutionResult> { - flame_guard!(format!("Frame::run({})", self.code.obj_name)); - // Execute until return or exception: - let instrs = &self.code.instructions; - let mut arg_state = bytecode::OpArgState::default(); - loop { - let idx = self.lasti() as usize; - self.update_lasti(|i| *i += 1); - let bytecode::CodeUnit { op, arg } = instrs[idx]; - let arg = arg_state.extend(arg); - let mut do_extend_arg = false; - let result = self.execute_instruction(op, arg, &mut do_extend_arg, vm); - match result { - Ok(None) => {} - Ok(Some(value)) => { - break Ok(value); - } - // Instruction raised an exception - Err(exception) => { - #[cold] - fn handle_exception( - frame: &mut ExecutingFrame, - exception: PyBaseExceptionRef, - idx: usize, - vm: &VirtualMachine, - ) -> FrameResult { - // 1. Extract traceback from exception's '__traceback__' attr. - // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. - // 3. Unwind block stack till appropriate handler is found. - - let loc = frame.code.locations[idx]; - let next = exception.traceback(); - let new_traceback = - PyTraceback::new(next, frame.object.to_owned(), frame.lasti(), loc.row); - vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.row()); - exception.set_traceback(Some(new_traceback.into_ref(&vm.ctx))); - - vm.contextualize_exception(&exception); - - frame.unwind_blocks(vm, UnwindReason::Raising { exception }) - } - - match handle_exception(self, exception, idx, vm) { - Ok(None) => {} - Ok(Some(result)) => break Ok(result), - // TODO: append line number to traceback? - // traceback.append(); - Err(exception) => break Err(exception), - } - } - } - if !do_extend_arg { - arg_state.reset() - } - } - } - - fn yield_from_target(&self) -> Option<&PyObject> { - if let Some(bytecode::CodeUnit { - op: bytecode::Instruction::YieldFrom, - .. - }) = self.code.instructions.get(self.lasti() as usize) - { - Some(self.last_value_ref()) - } else { - None - } - } - - /// Ok(Err(e)) means that an error occurred while calling throw() and the generator should try - /// sending it - fn gen_throw( - &mut self, - vm: &VirtualMachine, - exc_type: PyObjectRef, - exc_val: PyObjectRef, - exc_tb: PyObjectRef, - ) -> PyResult<ExecutionResult> { - if let Some(gen) = self.yield_from_target() { - // borrow checker shenanigans - we only need to use exc_type/val/tb if the following - // variable is Some - let thrower = if let Some(coro) = self.builtin_coro(gen) { - Some(Either::A(coro)) - } else { - vm.get_attribute_opt(gen.to_owned(), "throw")? - .map(Either::B) - }; - if let Some(thrower) = thrower { - let ret = match thrower { - Either::A(coro) => coro - .throw(gen, exc_type, exc_val, exc_tb, vm) - .to_pyresult(vm), // FIXME: - Either::B(meth) => meth.call((exc_type, exc_val, exc_tb), vm), - }; - return ret.map(ExecutionResult::Yield).or_else(|err| { - self.pop_value(); - self.update_lasti(|i| *i += 1); - if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) { - let val = vm.unwrap_or_none(err.get_arg(0)); - self.push_value(val); - self.run(vm) - } else { - let (ty, val, tb) = vm.split_exception(err); - self.gen_throw(vm, ty, val, tb) - } - }); - } - } - let exception = vm.normalize_exception(exc_type, exc_val, exc_tb)?; - match self.unwind_blocks(vm, UnwindReason::Raising { exception }) { - Ok(None) => self.run(vm), - Ok(Some(result)) => Ok(result), - Err(exception) => Err(exception), - } - } - - fn unbound_cell_exception(&self, i: usize, vm: &VirtualMachine) -> PyBaseExceptionRef { - if let Some(&name) = self.code.cellvars.get(i) { - vm.new_exception_msg( - vm.ctx.exceptions.unbound_local_error.to_owned(), - format!("local variable '{name}' referenced before assignment"), - ) - } else { - let name = self.code.freevars[i - self.code.cellvars.len()]; - vm.new_name_error( - format!("free variable '{name}' referenced before assignment in enclosing scope"), - name.to_owned(), - ) - } - } - - /// Execute a single instruction. - #[inline(always)] - fn execute_instruction( - &mut self, - instruction: bytecode::Instruction, - arg: bytecode::OpArg, - extend_arg: &mut bool, - vm: &VirtualMachine, - ) -> FrameResult { - vm.check_signals()?; - - flame_guard!(format!("Frame::execute_instruction({:?})", instruction)); - - #[cfg(feature = "vm-tracing-logging")] - { - trace!("======="); - /* TODO: - for frame in self.frames.iter() { - trace!(" {:?}", frame); - } - */ - trace!(" {:#?}", self); - trace!(" Executing op code: {:?}", instruction); - trace!("======="); - } - - #[cold] - fn name_error(name: &'static PyStrInterned, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) - } - - match instruction { - bytecode::Instruction::LoadConst { idx } => { - self.push_value(self.code.constants[idx.get(arg) as usize].clone().into()); - Ok(None) - } - bytecode::Instruction::ImportName { idx } => { - self.import(vm, Some(self.code.names[idx.get(arg) as usize]))?; - Ok(None) - } - bytecode::Instruction::ImportNameless => { - self.import(vm, None)?; - Ok(None) - } - bytecode::Instruction::ImportStar => { - self.import_star(vm)?; - Ok(None) - } - bytecode::Instruction::ImportFrom { idx } => { - let obj = self.import_from(vm, idx.get(arg))?; - self.push_value(obj); - Ok(None) - } - bytecode::Instruction::LoadFast(idx) => { - #[cold] - fn reference_error( - varname: &'static PyStrInterned, - vm: &VirtualMachine, - ) -> PyBaseExceptionRef { - vm.new_exception_msg( - vm.ctx.exceptions.unbound_local_error.to_owned(), - format!("local variable '{varname}' referenced before assignment",), - ) - } - let idx = idx.get(arg) as usize; - let x = self.fastlocals.lock()[idx] - .clone() - .ok_or_else(|| reference_error(self.code.varnames[idx], vm))?; - self.push_value(x); - Ok(None) - } - bytecode::Instruction::LoadNameAny(idx) => { - let name = self.code.names[idx.get(arg) as usize]; - let result = self.locals.mapping().subscript(name, vm); - match result { - Ok(x) => self.push_value(x), - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - self.push_value(self.load_global_or_builtin(name, vm)?); - } - Err(e) => return Err(e), - } - Ok(None) - } - bytecode::Instruction::LoadGlobal(idx) => { - let name = &self.code.names[idx.get(arg) as usize]; - let x = self.load_global_or_builtin(name, vm)?; - self.push_value(x); - Ok(None) - } - bytecode::Instruction::LoadDeref(i) => { - let i = i.get(arg) as usize; - let x = self.cells_frees[i] - .get() - .ok_or_else(|| self.unbound_cell_exception(i, vm))?; - self.push_value(x); - Ok(None) - } - bytecode::Instruction::LoadClassDeref(i) => { - let i = i.get(arg) as usize; - let name = self.code.freevars[i - self.code.cellvars.len()]; - let value = self.locals.mapping().subscript(name, vm).ok(); - self.push_value(match value { - Some(v) => v, - None => self.cells_frees[i] - .get() - .ok_or_else(|| self.unbound_cell_exception(i, vm))?, - }); - Ok(None) - } - bytecode::Instruction::StoreFast(idx) => { - let value = self.pop_value(); - self.fastlocals.lock()[idx.get(arg) as usize] = Some(value); - Ok(None) - } - bytecode::Instruction::StoreLocal(idx) => { - let name = self.code.names[idx.get(arg) as usize]; - let value = self.pop_value(); - self.locals.mapping().ass_subscript(name, Some(value), vm)?; - Ok(None) - } - bytecode::Instruction::StoreGlobal(idx) => { - let value = self.pop_value(); - self.globals - .set_item(self.code.names[idx.get(arg) as usize], value, vm)?; - Ok(None) - } - bytecode::Instruction::StoreDeref(i) => { - let value = self.pop_value(); - self.cells_frees[i.get(arg) as usize].set(Some(value)); - Ok(None) - } - bytecode::Instruction::DeleteFast(idx) => { - let mut fastlocals = self.fastlocals.lock(); - let idx = idx.get(arg) as usize; - if fastlocals[idx].is_none() { - return Err(vm.new_exception_msg( - vm.ctx.exceptions.unbound_local_error.to_owned(), - format!( - "local variable '{}' referenced before assignment", - self.code.varnames[idx] - ), - )); - } - fastlocals[idx] = None; - Ok(None) - } - bytecode::Instruction::DeleteLocal(idx) => { - let name = self.code.names[idx.get(arg) as usize]; - let res = self.locals.mapping().ass_subscript(name, None, vm); - - match res { - Ok(()) => {} - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - return Err(name_error(name, vm)) - } - Err(e) => return Err(e), - } - Ok(None) - } - bytecode::Instruction::DeleteGlobal(idx) => { - let name = self.code.names[idx.get(arg) as usize]; - match self.globals.del_item(name, vm) { - Ok(()) => {} - Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - return Err(name_error(name, vm)) - } - Err(e) => return Err(e), - } - Ok(None) - } - bytecode::Instruction::DeleteDeref(i) => { - self.cells_frees[i.get(arg) as usize].set(None); - Ok(None) - } - bytecode::Instruction::LoadClosure(i) => { - let value = self.cells_frees[i.get(arg) as usize].clone(); - self.push_value(value.into()); - Ok(None) - } - bytecode::Instruction::Subscript => self.execute_subscript(vm), - bytecode::Instruction::StoreSubscript => self.execute_store_subscript(vm), - bytecode::Instruction::DeleteSubscript => self.execute_delete_subscript(vm), - bytecode::Instruction::Pop => { - // Pop value from stack and ignore. - self.pop_value(); - Ok(None) - } - bytecode::Instruction::Duplicate => { - // Duplicate top of stack - let value = self.last_value(); - self.push_value(value); - Ok(None) - } - bytecode::Instruction::Duplicate2 => { - // Duplicate top 2 of stack - let top = self.last_value(); - let second_to_top = self.nth_value(1).to_owned(); - self.push_value(second_to_top); - self.push_value(top); - Ok(None) - } - // splitting the instructions like this offloads the cost of "dynamic" dispatch (on the - // amount to rotate) to the opcode dispatcher, and generates optimized code for the - // concrete cases we actually have - bytecode::Instruction::Rotate2 => self.execute_rotate(2), - bytecode::Instruction::Rotate3 => self.execute_rotate(3), - bytecode::Instruction::BuildString { size } => { - let s = self - .pop_multiple(size.get(arg) as usize) - .as_slice() - .iter() - .map(|pyobj| pyobj.payload::<PyStr>().unwrap().as_ref()) - .collect::<String>(); - let str_obj = vm.ctx.new_str(s); - self.push_value(str_obj.into()); - Ok(None) - } - bytecode::Instruction::BuildList { size } => { - let elements = self.pop_multiple(size.get(arg) as usize).collect(); - let list_obj = vm.ctx.new_list(elements); - self.push_value(list_obj.into()); - Ok(None) - } - bytecode::Instruction::BuildListUnpack { size } => { - let elements = self.unpack_elements(vm, size.get(arg) as usize)?; - let list_obj = vm.ctx.new_list(elements); - self.push_value(list_obj.into()); - Ok(None) - } - bytecode::Instruction::BuildSet { size } => { - let set = PySet::new_ref(&vm.ctx); - { - for element in self.pop_multiple(size.get(arg) as usize) { - set.add(element, vm)?; - } - } - self.push_value(set.into()); - Ok(None) - } - bytecode::Instruction::BuildSetUnpack { size } => { - let set = PySet::new_ref(&vm.ctx); - { - for element in self.pop_multiple(size.get(arg) as usize) { - vm.map_iterable_object(&element, |x| set.add(x, vm))??; - } - } - self.push_value(set.into()); - Ok(None) - } - bytecode::Instruction::BuildTuple { size } => { - let elements = self.pop_multiple(size.get(arg) as usize).collect(); - let list_obj = vm.ctx.new_tuple(elements); - self.push_value(list_obj.into()); - Ok(None) - } - bytecode::Instruction::BuildTupleUnpack { size } => { - let elements = self.unpack_elements(vm, size.get(arg) as usize)?; - let list_obj = vm.ctx.new_tuple(elements); - self.push_value(list_obj.into()); - Ok(None) - } - bytecode::Instruction::BuildMap { size } => self.execute_build_map(vm, size.get(arg)), - bytecode::Instruction::BuildMapForCall { size } => { - self.execute_build_map_for_call(vm, size.get(arg)) - } - bytecode::Instruction::DictUpdate => { - let other = self.pop_value(); - let dict = self - .last_value_ref() - .downcast_ref::<PyDict>() - .expect("exact dict expected"); - dict.merge_object(other, vm)?; - Ok(None) - } - bytecode::Instruction::BuildSlice { step } => { - self.execute_build_slice(vm, step.get(arg)) - } - bytecode::Instruction::ListAppend { i } => { - let item = self.pop_value(); - let obj = self.nth_value(i.get(arg)); - let list: &Py<PyList> = unsafe { - // SAFETY: trust compiler - obj.downcast_unchecked_ref() - }; - list.append(item); - Ok(None) - } - bytecode::Instruction::SetAdd { i } => { - let item = self.pop_value(); - let obj = self.nth_value(i.get(arg)); - let set: &Py<PySet> = unsafe { - // SAFETY: trust compiler - obj.downcast_unchecked_ref() - }; - set.add(item, vm)?; - Ok(None) - } - bytecode::Instruction::MapAdd { i } => { - let value = self.pop_value(); - let key = self.pop_value(); - let obj = self.nth_value(i.get(arg)); - let dict: &Py<PyDict> = unsafe { - // SAFETY: trust compiler - obj.downcast_unchecked_ref() - }; - dict.set_item(&*key, value, vm)?; - Ok(None) - } - bytecode::Instruction::BinaryOperation { op } => self.execute_binop(vm, op.get(arg)), - bytecode::Instruction::BinaryOperationInplace { op } => { - self.execute_binop_inplace(vm, op.get(arg)) - } - bytecode::Instruction::LoadAttr { idx } => self.load_attr(vm, idx.get(arg)), - bytecode::Instruction::StoreAttr { idx } => self.store_attr(vm, idx.get(arg)), - bytecode::Instruction::DeleteAttr { idx } => self.delete_attr(vm, idx.get(arg)), - bytecode::Instruction::UnaryOperation { op } => self.execute_unop(vm, op.get(arg)), - bytecode::Instruction::TestOperation { op } => self.execute_test(vm, op.get(arg)), - bytecode::Instruction::CompareOperation { op } => self.execute_compare(vm, op.get(arg)), - bytecode::Instruction::ReturnValue => { - let value = self.pop_value(); - self.unwind_blocks(vm, UnwindReason::Returning { value }) - } - bytecode::Instruction::YieldValue => { - let value = self.pop_value(); - let value = if self.code.flags.contains(bytecode::CodeFlags::IS_COROUTINE) { - PyAsyncGenWrappedValue(value).into_pyobject(vm) - } else { - value - }; - Ok(Some(ExecutionResult::Yield(value))) - } - bytecode::Instruction::YieldFrom => self.execute_yield_from(vm), - bytecode::Instruction::SetupAnnotation => self.setup_annotations(vm), - bytecode::Instruction::SetupLoop => { - self.push_block(BlockType::Loop); - Ok(None) - } - bytecode::Instruction::SetupExcept { handler } => { - self.push_block(BlockType::TryExcept { - handler: handler.get(arg), - }); - Ok(None) - } - bytecode::Instruction::SetupFinally { handler } => { - self.push_block(BlockType::Finally { - handler: handler.get(arg), - }); - Ok(None) - } - bytecode::Instruction::EnterFinally => { - self.push_block(BlockType::FinallyHandler { - reason: None, - prev_exc: vm.current_exception(), - }); - Ok(None) - } - bytecode::Instruction::EndFinally => { - // Pop the finally handler from the stack, and recall - // what was the reason we were in this finally clause. - let block = self.pop_block(); - - if let BlockType::FinallyHandler { reason, prev_exc } = block.typ { - vm.set_exception(prev_exc); - if let Some(reason) = reason { - self.unwind_blocks(vm, reason) - } else { - Ok(None) - } - } else { - self.fatal( - "Block type must be finally handler when reaching EndFinally instruction!", - ); - } - } - bytecode::Instruction::SetupWith { end } => { - let context_manager = self.pop_value(); - let error_string = || -> String { - format!( - "'{:.200}' object does not support the context manager protocol", - context_manager.class().name(), - ) - }; - let enter_res = vm - .get_special_method(&context_manager, identifier!(vm, __enter__))? - .ok_or_else(|| vm.new_type_error(error_string()))? - .invoke((), vm)?; - - let exit = context_manager - .get_attr(identifier!(vm, __exit__), vm) - .map_err(|_exc| { - vm.new_type_error({ - format!("'{} (missed __exit__ method)", error_string()) - }) - })?; - self.push_value(exit); - self.push_block(BlockType::Finally { - handler: end.get(arg), - }); - self.push_value(enter_res); - Ok(None) - } - bytecode::Instruction::BeforeAsyncWith => { - let mgr = self.pop_value(); - let error_string = || -> String { - format!( - "'{:.200}' object does not support the asynchronous context manager protocol", - mgr.class().name(), - ) - }; - - let aenter_res = vm - .get_special_method(&mgr, identifier!(vm, __aenter__))? - .ok_or_else(|| vm.new_type_error(error_string()))? - .invoke((), vm)?; - let aexit = mgr - .get_attr(identifier!(vm, __aexit__), vm) - .map_err(|_exc| { - vm.new_type_error({ - format!("'{} (missed __aexit__ method)", error_string()) - }) - })?; - self.push_value(aexit); - self.push_value(aenter_res); - - Ok(None) - } - bytecode::Instruction::SetupAsyncWith { end } => { - let enter_res = self.pop_value(); - self.push_block(BlockType::Finally { - handler: end.get(arg), - }); - self.push_value(enter_res); - Ok(None) - } - bytecode::Instruction::WithCleanupStart => { - let block = self.current_block().unwrap(); - let reason = match block.typ { - BlockType::FinallyHandler { reason, .. } => reason, - _ => self.fatal("WithCleanupStart expects a FinallyHandler block on stack"), - }; - let exc = match reason { - Some(UnwindReason::Raising { exception }) => Some(exception), - _ => None, - }; - - let exit = self.pop_value(); - - let args = if let Some(exc) = exc { - vm.split_exception(exc) - } else { - (vm.ctx.none(), vm.ctx.none(), vm.ctx.none()) - }; - let exit_res = exit.call(args, vm)?; - self.push_value(exit_res); - - Ok(None) - } - bytecode::Instruction::WithCleanupFinish => { - let block = self.pop_block(); - let (reason, prev_exc) = match block.typ { - BlockType::FinallyHandler { reason, prev_exc } => (reason, prev_exc), - _ => self.fatal("WithCleanupFinish expects a FinallyHandler block on stack"), - }; - - let suppress_exception = self.pop_value().try_to_bool(vm)?; - - vm.set_exception(prev_exc); - - if suppress_exception { - Ok(None) - } else if let Some(reason) = reason { - self.unwind_blocks(vm, reason) - } else { - Ok(None) - } - } - bytecode::Instruction::PopBlock => { - self.pop_block(); - Ok(None) - } - bytecode::Instruction::GetIter => { - let iterated_obj = self.pop_value(); - let iter_obj = iterated_obj.get_iter(vm)?; - self.push_value(iter_obj.into()); - Ok(None) - } - bytecode::Instruction::GetAwaitable => { - let awaited_obj = self.pop_value(); - let awaitable = if awaited_obj.payload_is::<PyCoroutine>() { - awaited_obj - } else { - let await_method = vm.get_method_or_type_error( - awaited_obj.clone(), - identifier!(vm, __await__), - || { - format!( - "object {} can't be used in 'await' expression", - awaited_obj.class().name(), - ) - }, - )?; - await_method.call((), vm)? - }; - self.push_value(awaitable); - Ok(None) - } - bytecode::Instruction::GetAIter => { - let aiterable = self.pop_value(); - let aiter = vm.call_special_method(&aiterable, identifier!(vm, __aiter__), ())?; - self.push_value(aiter); - Ok(None) - } - bytecode::Instruction::GetANext => { - let aiter = self.last_value(); - let awaitable = vm.call_special_method(&aiter, identifier!(vm, __anext__), ())?; - let awaitable = if awaitable.payload_is::<PyCoroutine>() { - awaitable - } else { - vm.call_special_method(&awaitable, identifier!(vm, __await__), ())? - }; - self.push_value(awaitable); - Ok(None) - } - bytecode::Instruction::EndAsyncFor => { - let exc = self.pop_value(); - self.pop_value(); // async iterator we were calling __anext__ on - if exc.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) { - vm.take_exception().expect("Should have exception in stack"); - Ok(None) - } else { - Err(exc.downcast().unwrap()) - } - } - bytecode::Instruction::ForIter { target } => self.execute_for_iter(vm, target.get(arg)), - bytecode::Instruction::MakeFunction(flags) => { - self.execute_make_function(vm, flags.get(arg)) - } - bytecode::Instruction::CallFunctionPositional { nargs } => { - let args = self.collect_positional_args(nargs.get(arg)); - self.execute_call(args, vm) - } - bytecode::Instruction::CallFunctionKeyword { nargs } => { - let args = self.collect_keyword_args(nargs.get(arg)); - self.execute_call(args, vm) - } - bytecode::Instruction::CallFunctionEx { has_kwargs } => { - let args = self.collect_ex_args(vm, has_kwargs.get(arg))?; - self.execute_call(args, vm) - } - bytecode::Instruction::LoadMethod { idx } => { - let obj = self.pop_value(); - let method_name = self.code.names[idx.get(arg) as usize]; - let method = PyMethod::get(obj, method_name, vm)?; - let (target, is_method, func) = match method { - PyMethod::Function { target, func } => (target, true, func), - PyMethod::Attribute(val) => (vm.ctx.none(), false, val), - }; - // TODO: figure out a better way to communicate PyMethod::Attribute - CPython uses - // target==NULL, maybe we could use a sentinel value or something? - self.push_value(target); - self.push_value(vm.ctx.new_bool(is_method).into()); - self.push_value(func); - Ok(None) - } - bytecode::Instruction::CallMethodPositional { nargs } => { - let args = self.collect_positional_args(nargs.get(arg)); - self.execute_method_call(args, vm) - } - bytecode::Instruction::CallMethodKeyword { nargs } => { - let args = self.collect_keyword_args(nargs.get(arg)); - self.execute_method_call(args, vm) - } - bytecode::Instruction::CallMethodEx { has_kwargs } => { - let args = self.collect_ex_args(vm, has_kwargs.get(arg))?; - self.execute_method_call(args, vm) - } - bytecode::Instruction::Jump { target } => { - self.jump(target.get(arg)); - Ok(None) - } - bytecode::Instruction::JumpIfTrue { target } => self.jump_if(vm, target.get(arg), true), - bytecode::Instruction::JumpIfFalse { target } => { - self.jump_if(vm, target.get(arg), false) - } - bytecode::Instruction::JumpIfTrueOrPop { target } => { - self.jump_if_or_pop(vm, target.get(arg), true) - } - bytecode::Instruction::JumpIfFalseOrPop { target } => { - self.jump_if_or_pop(vm, target.get(arg), false) - } - - bytecode::Instruction::Raise { kind } => self.execute_raise(vm, kind.get(arg)), - - bytecode::Instruction::Break { target } => self.unwind_blocks( - vm, - UnwindReason::Break { - target: target.get(arg), - }, - ), - bytecode::Instruction::Continue { target } => self.unwind_blocks( - vm, - UnwindReason::Continue { - target: target.get(arg), - }, - ), - bytecode::Instruction::PrintExpr => self.print_expr(vm), - bytecode::Instruction::LoadBuildClass => { - self.push_value(vm.builtins.get_attr(identifier!(vm, __build_class__), vm)?); - Ok(None) - } - bytecode::Instruction::UnpackSequence { size } => { - self.unpack_sequence(size.get(arg), vm) - } - bytecode::Instruction::UnpackEx { args } => { - let args = args.get(arg); - self.execute_unpack_ex(vm, args.before, args.after) - } - bytecode::Instruction::FormatValue { conversion } => { - self.format_value(conversion.get(arg), vm) - } - bytecode::Instruction::PopException {} => { - let block = self.pop_block(); - if let BlockType::ExceptHandler { prev_exc } = block.typ { - vm.set_exception(prev_exc); - Ok(None) - } else { - self.fatal("block type must be ExceptHandler here.") - } - } - bytecode::Instruction::Reverse { amount } => { - let stack_len = self.state.stack.len(); - self.state.stack[stack_len - amount.get(arg) as usize..stack_len].reverse(); - Ok(None) - } - bytecode::Instruction::ExtendedArg => { - *extend_arg = true; - Ok(None) - } - } - } - - #[inline] - fn load_global_or_builtin(&self, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - self.globals - .get_chain(self.builtins, name, vm)? - .ok_or_else(|| { - vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) - }) - } - - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn unpack_elements(&mut self, vm: &VirtualMachine, size: usize) -> PyResult<Vec<PyObjectRef>> { - let mut result = Vec::<PyObjectRef>::new(); - for element in self.pop_multiple(size) { - let items: Vec<_> = element.try_to_value(vm)?; - result.extend(items); - } - Ok(result) - } - - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn import(&mut self, vm: &VirtualMachine, module: Option<&Py<PyStr>>) -> PyResult<()> { - let module = module.unwrap_or(vm.ctx.empty_str); - let from_list = <Option<PyTupleTyped<PyStrRef>>>::try_from_object(vm, self.pop_value())?; - let level = usize::try_from_object(vm, self.pop_value())?; - - let module = vm.import(module, from_list, level)?; - - self.push_value(module); - Ok(()) - } - - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn import_from(&mut self, vm: &VirtualMachine, idx: bytecode::NameIdx) -> PyResult { - let module = self.last_value(); - let name = self.code.names[idx as usize]; - let err = || vm.new_import_error(format!("cannot import name '{name}'"), name.to_owned()); - // Load attribute, and transform any error into import error. - if let Some(obj) = vm.get_attribute_opt(module.clone(), name)? { - return Ok(obj); - } - // fallback to importing '{module.__name__}.{name}' from sys.modules - let mod_name = module - .get_attr(identifier!(vm, __name__), vm) - .map_err(|_| err())?; - let mod_name = mod_name.downcast::<PyStr>().map_err(|_| err())?; - let full_mod_name = format!("{mod_name}.{name}"); - let sys_modules = vm.sys_module.get_attr("modules", vm).map_err(|_| err())?; - sys_modules.get_item(&full_mod_name, vm).map_err(|_| err()) - } - - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn import_star(&mut self, vm: &VirtualMachine) -> PyResult<()> { - let module = self.pop_value(); - - // Grab all the names from the module and put them in the context - if let Some(dict) = module.dict() { - let filter_pred: Box<dyn Fn(&str) -> bool> = - if let Ok(all) = dict.get_item(identifier!(vm, __all__), vm) { - let all: Vec<PyStrRef> = all.try_to_value(vm)?; - let all: Vec<String> = all - .into_iter() - .map(|name| name.as_str().to_owned()) - .collect(); - Box::new(move |name| all.contains(&name.to_owned())) - } else { - Box::new(|name| !name.starts_with('_')) - }; - for (k, v) in dict { - let k = PyStrRef::try_from_object(vm, k)?; - if filter_pred(k.as_str()) { - self.locals.mapping().ass_subscript(&k, Some(v), vm)?; - } - } - } - Ok(()) - } - - /// Unwind blocks. - /// The reason for unwinding gives a hint on what to do when - /// unwinding a block. - /// Optionally returns an exception. - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn unwind_blocks(&mut self, vm: &VirtualMachine, reason: UnwindReason) -> FrameResult { - // First unwind all existing blocks on the block stack: - while let Some(block) = self.current_block() { - match block.typ { - BlockType::Loop => match reason { - UnwindReason::Break { target } => { - self.pop_block(); - self.jump(target); - return Ok(None); - } - UnwindReason::Continue { target } => { - self.jump(target); - return Ok(None); - } - _ => { - self.pop_block(); - } - }, - BlockType::Finally { handler } => { - self.pop_block(); - let prev_exc = vm.current_exception(); - if let UnwindReason::Raising { exception } = &reason { - vm.set_exception(Some(exception.clone())); - } - self.push_block(BlockType::FinallyHandler { - reason: Some(reason), - prev_exc, - }); - self.jump(handler); - return Ok(None); - } - BlockType::TryExcept { handler } => { - self.pop_block(); - if let UnwindReason::Raising { exception } = reason { - self.push_block(BlockType::ExceptHandler { - prev_exc: vm.current_exception(), - }); - vm.contextualize_exception(&exception); - vm.set_exception(Some(exception.clone())); - self.push_value(exception.into()); - self.jump(handler); - return Ok(None); - } - } - BlockType::FinallyHandler { prev_exc, .. } - | BlockType::ExceptHandler { prev_exc } => { - self.pop_block(); - vm.set_exception(prev_exc); - } - } - } - - // We do not have any more blocks to unwind. Inspect the reason we are here: - match reason { - UnwindReason::Raising { exception } => Err(exception), - UnwindReason::Returning { value } => Ok(Some(ExecutionResult::Return(value))), - UnwindReason::Break { .. } | UnwindReason::Continue { .. } => { - self.fatal("break or continue must occur within a loop block.") - } // UnwindReason::NoWorries => Ok(None), - } - } - - #[inline(always)] - fn execute_rotate(&mut self, amount: usize) -> FrameResult { - let i = self.state.stack.len() - amount; - self.state.stack[i..].rotate_right(1); - Ok(None) - } - - fn execute_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { - let b_ref = self.pop_value(); - let a_ref = self.pop_value(); - let value = a_ref.get_item(&*b_ref, vm)?; - self.push_value(value); - Ok(None) - } - - fn execute_store_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { - let idx = self.pop_value(); - let obj = self.pop_value(); - let value = self.pop_value(); - obj.set_item(&*idx, value, vm)?; - Ok(None) - } - - fn execute_delete_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { - let idx = self.pop_value(); - let obj = self.pop_value(); - obj.del_item(&*idx, vm)?; - Ok(None) - } - - fn execute_build_map(&mut self, vm: &VirtualMachine, size: u32) -> FrameResult { - let size = size as usize; - let map_obj = vm.ctx.new_dict(); - for (key, value) in self.pop_multiple(2 * size).tuples() { - map_obj.set_item(&*key, value, vm)?; - } - - self.push_value(map_obj.into()); - Ok(None) - } - - fn execute_build_map_for_call(&mut self, vm: &VirtualMachine, size: u32) -> FrameResult { - let size = size as usize; - let map_obj = vm.ctx.new_dict(); - for obj in self.pop_multiple(size) { - // Take all key-value pairs from the dict: - let dict: PyDictRef = obj.downcast().map_err(|obj| { - vm.new_type_error(format!("'{}' object is not a mapping", obj.class().name())) - })?; - for (key, value) in dict { - if map_obj.contains_key(&*key, vm) { - let key_repr = &key.repr(vm)?; - let msg = format!( - "got multiple values for keyword argument {}", - key_repr.as_str() - ); - return Err(vm.new_type_error(msg)); - } - map_obj.set_item(&*key, value, vm)?; - } - } - - self.push_value(map_obj.into()); - Ok(None) - } - - fn execute_build_slice(&mut self, vm: &VirtualMachine, step: bool) -> FrameResult { - let step = if step { Some(self.pop_value()) } else { None }; - let stop = self.pop_value(); - let start = self.pop_value(); - - let obj = PySlice { - start: Some(start), - stop, - step, - } - .into_ref(&vm.ctx); - self.push_value(obj.into()); - Ok(None) - } - - fn collect_positional_args(&mut self, nargs: u32) -> FuncArgs { - FuncArgs { - args: self.pop_multiple(nargs as usize).collect(), - kwargs: IndexMap::new(), - } - } - - fn collect_keyword_args(&mut self, nargs: u32) -> FuncArgs { - let kwarg_names = self - .pop_value() - .downcast::<PyTuple>() - .expect("kwarg names should be tuple of strings"); - let args = self.pop_multiple(nargs as usize); - - let kwarg_names = kwarg_names - .as_slice() - .iter() - .map(|pyobj| pyobj.payload::<PyStr>().unwrap().as_ref().to_owned()); - FuncArgs::with_kwargs_names(args, kwarg_names) - } - - fn collect_ex_args(&mut self, vm: &VirtualMachine, has_kwargs: bool) -> PyResult<FuncArgs> { - let kwargs = if has_kwargs { - let kw_dict: PyDictRef = self.pop_value().downcast().map_err(|_| { - // TODO: check collections.abc.Mapping - vm.new_type_error("Kwargs must be a dict.".to_owned()) - })?; - let mut kwargs = IndexMap::new(); - for (key, value) in kw_dict.into_iter() { - let key = key - .payload_if_subclass::<PyStr>(vm) - .ok_or_else(|| vm.new_type_error("keywords must be strings".to_owned()))?; - kwargs.insert(key.as_str().to_owned(), value); - } - kwargs - } else { - IndexMap::new() - }; - let args = self.pop_value(); - let args = args.try_to_value(vm)?; - Ok(FuncArgs { args, kwargs }) - } - - #[inline] - fn execute_call(&mut self, args: FuncArgs, vm: &VirtualMachine) -> FrameResult { - let func_ref = self.pop_value(); - let value = func_ref.call(args, vm)?; - self.push_value(value); - Ok(None) - } - - #[inline] - fn execute_method_call(&mut self, args: FuncArgs, vm: &VirtualMachine) -> FrameResult { - let func = self.pop_value(); - let is_method = self.pop_value().is(&vm.ctx.true_value); - let target = self.pop_value(); - - // TODO: It was PyMethod before #4873. Check if it's correct. - let func = if is_method { - if let Some(descr_get) = func.class().mro_find_map(|cls| cls.slots.descr_get.load()) { - let cls = target.class().to_owned().into(); - descr_get(func, Some(target), Some(cls), vm)? - } else { - func - } - } else { - drop(target); // should be None - func - }; - let value = func.call(args, vm)?; - self.push_value(value); - Ok(None) - } - - fn execute_raise(&mut self, vm: &VirtualMachine, kind: bytecode::RaiseKind) -> FrameResult { - let cause = match kind { - bytecode::RaiseKind::RaiseCause => { - let val = self.pop_value(); - Some(if vm.is_none(&val) { - // if the cause arg is none, we clear the cause - None - } else { - // if the cause arg is an exception, we overwrite it - let ctor = ExceptionCtor::try_from_object(vm, val).map_err(|_| { - vm.new_type_error( - "exception causes must derive from BaseException".to_owned(), - ) - })?; - Some(ctor.instantiate(vm)?) - }) - } - // if there's no cause arg, we keep the cause as is - bytecode::RaiseKind::Raise | bytecode::RaiseKind::Reraise => None, - }; - let exception = match kind { - bytecode::RaiseKind::RaiseCause | bytecode::RaiseKind::Raise => { - ExceptionCtor::try_from_object(vm, self.pop_value())?.instantiate(vm)? - } - bytecode::RaiseKind::Reraise => vm - .topmost_exception() - .ok_or_else(|| vm.new_runtime_error("No active exception to reraise".to_owned()))?, - }; - info!("Exception raised: {:?} with cause: {:?}", exception, cause); - if let Some(cause) = cause { - exception.set_cause(cause); - } - Err(exception) - } - - fn builtin_coro<'a>(&self, coro: &'a PyObject) -> Option<&'a Coro> { - match_class!(match coro { - ref g @ PyGenerator => Some(g.as_coro()), - ref c @ PyCoroutine => Some(c.as_coro()), - _ => None, - }) - } - - fn _send( - &self, - gen: &PyObject, - val: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn> { - match self.builtin_coro(gen) { - Some(coro) => coro.send(gen, val, vm), - // FIXME: turn return type to PyResult<PyIterReturn> then ExecutionResult will be simplified - None if vm.is_none(&val) => PyIter::new(gen).next(vm), - None => { - let meth = gen.get_attr("send", vm)?; - PyIterReturn::from_pyresult(meth.call((val,), vm), vm) - } - } - } - - fn execute_yield_from(&mut self, vm: &VirtualMachine) -> FrameResult { - // Value send into iterator: - let val = self.pop_value(); - let coro = self.last_value_ref(); - let result = self._send(coro, val, vm)?; - - // PyIterReturn returned from e.g. gen.__next__() or gen.send() - match result { - PyIterReturn::Return(value) => { - // Set back program counter: - self.update_lasti(|i| *i -= 1); - Ok(Some(ExecutionResult::Yield(value))) - } - PyIterReturn::StopIteration(value) => { - let value = vm.unwrap_or_none(value); - self.pop_value(); - self.push_value(value); - Ok(None) - } - } - } - - fn execute_unpack_ex(&mut self, vm: &VirtualMachine, before: u8, after: u8) -> FrameResult { - let (before, after) = (before as usize, after as usize); - let value = self.pop_value(); - let elements: Vec<_> = value.try_to_value(vm)?; - let min_expected = before + after; - - let middle = elements.len().checked_sub(min_expected).ok_or_else(|| { - vm.new_value_error(format!( - "not enough values to unpack (expected at least {}, got {})", - min_expected, - elements.len() - )) - })?; - - let mut elements = elements; - // Elements on stack from right-to-left: - self.state - .stack - .extend(elements.drain(before + middle..).rev()); - - let middle_elements = elements.drain(before..).collect(); - let t = vm.ctx.new_list(middle_elements); - self.push_value(t.into()); - - // Lastly the first reversed values: - self.state.stack.extend(elements.into_iter().rev()); - - Ok(None) - } - - #[inline] - fn jump(&mut self, label: bytecode::Label) { - let target_pc = label.0; - vm_trace!("jump from {:?} to {:?}", self.lasti(), target_pc); - self.update_lasti(|i| *i = target_pc); - } - - #[inline] - fn jump_if(&mut self, vm: &VirtualMachine, target: bytecode::Label, flag: bool) -> FrameResult { - let obj = self.pop_value(); - let value = obj.try_to_bool(vm)?; - if value == flag { - self.jump(target); - } - Ok(None) - } - - #[inline] - fn jump_if_or_pop( - &mut self, - vm: &VirtualMachine, - target: bytecode::Label, - flag: bool, - ) -> FrameResult { - let obj = self.last_value(); - let value = obj.try_to_bool(vm)?; - if value == flag { - self.jump(target); - } else { - self.pop_value(); - } - Ok(None) - } - - /// The top of stack contains the iterator, lets push it forward - fn execute_for_iter(&mut self, vm: &VirtualMachine, target: bytecode::Label) -> FrameResult { - let top_of_stack = PyIter::new(self.last_value()); - let next_obj = top_of_stack.next(vm); - - // Check the next object: - match next_obj { - Ok(PyIterReturn::Return(value)) => { - self.push_value(value); - Ok(None) - } - Ok(PyIterReturn::StopIteration(_)) => { - // Pop iterator from stack: - self.pop_value(); - - // End of for loop - self.jump(target); - Ok(None) - } - Err(next_error) => { - // Pop iterator from stack: - self.pop_value(); - Err(next_error) - } - } - } - fn execute_make_function( - &mut self, - vm: &VirtualMachine, - flags: bytecode::MakeFunctionFlags, - ) -> FrameResult { - let qualified_name = self - .pop_value() - .downcast::<PyStr>() - .expect("qualified name to be a string"); - let code_obj: PyRef<PyCode> = self - .pop_value() - .downcast() - .expect("Second to top value on the stack must be a code object"); - - let closure = if flags.contains(bytecode::MakeFunctionFlags::CLOSURE) { - Some(PyTupleTyped::try_from_object(vm, self.pop_value()).unwrap()) - } else { - None - }; - - let annotations = if flags.contains(bytecode::MakeFunctionFlags::ANNOTATIONS) { - self.pop_value() - } else { - vm.ctx.new_dict().into() - }; - - let kw_only_defaults = if flags.contains(bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS) { - Some( - self.pop_value() - .downcast::<PyDict>() - .expect("Stack value for keyword only defaults expected to be a dict"), - ) - } else { - None - }; - - let defaults = if flags.contains(bytecode::MakeFunctionFlags::DEFAULTS) { - Some( - self.pop_value() - .downcast::<PyTuple>() - .expect("Stack value for defaults expected to be a tuple"), - ) - } else { - None - }; - - // pop argc arguments - // argument: name, args, globals - // let scope = self.scope.clone(); - let func_obj = PyFunction::new( - code_obj, - self.globals.clone(), - closure, - defaults, - kw_only_defaults, - qualified_name.clone(), - vm.ctx.empty_tuple.clone(), // FIXME: fake implementation - ) - .into_pyobject(vm); - - func_obj.set_attr(identifier!(vm, __doc__), vm.ctx.none(), vm)?; - - let name = qualified_name.as_str().split('.').next_back().unwrap(); - func_obj.set_attr(identifier!(vm, __name__), vm.new_pyobj(name), vm)?; - func_obj.set_attr(identifier!(vm, __qualname__), qualified_name, vm)?; - let module = vm.unwrap_or_none(self.globals.get_item_opt(identifier!(vm, __name__), vm)?); - func_obj.set_attr(identifier!(vm, __module__), module, vm)?; - func_obj.set_attr(identifier!(vm, __annotations__), annotations, vm)?; - - self.push_value(func_obj); - Ok(None) - } - - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn execute_binop(&mut self, vm: &VirtualMachine, op: bytecode::BinaryOperator) -> FrameResult { - let b_ref = &self.pop_value(); - let a_ref = &self.pop_value(); - let value = match op { - bytecode::BinaryOperator::Subtract => vm._sub(a_ref, b_ref), - bytecode::BinaryOperator::Add => vm._add(a_ref, b_ref), - bytecode::BinaryOperator::Multiply => vm._mul(a_ref, b_ref), - bytecode::BinaryOperator::MatrixMultiply => vm._matmul(a_ref, b_ref), - bytecode::BinaryOperator::Power => vm._pow(a_ref, b_ref, vm.ctx.none.as_object()), - bytecode::BinaryOperator::Divide => vm._truediv(a_ref, b_ref), - bytecode::BinaryOperator::FloorDivide => vm._floordiv(a_ref, b_ref), - bytecode::BinaryOperator::Modulo => vm._mod(a_ref, b_ref), - bytecode::BinaryOperator::Lshift => vm._lshift(a_ref, b_ref), - bytecode::BinaryOperator::Rshift => vm._rshift(a_ref, b_ref), - bytecode::BinaryOperator::Xor => vm._xor(a_ref, b_ref), - bytecode::BinaryOperator::Or => vm._or(a_ref, b_ref), - bytecode::BinaryOperator::And => vm._and(a_ref, b_ref), - }?; - - self.push_value(value); - Ok(None) - } - fn execute_binop_inplace( - &mut self, - vm: &VirtualMachine, - op: bytecode::BinaryOperator, - ) -> FrameResult { - let b_ref = &self.pop_value(); - let a_ref = &self.pop_value(); - let value = match op { - bytecode::BinaryOperator::Subtract => vm._isub(a_ref, b_ref), - bytecode::BinaryOperator::Add => vm._iadd(a_ref, b_ref), - bytecode::BinaryOperator::Multiply => vm._imul(a_ref, b_ref), - bytecode::BinaryOperator::MatrixMultiply => vm._imatmul(a_ref, b_ref), - bytecode::BinaryOperator::Power => vm._ipow(a_ref, b_ref, vm.ctx.none.as_object()), - bytecode::BinaryOperator::Divide => vm._itruediv(a_ref, b_ref), - bytecode::BinaryOperator::FloorDivide => vm._ifloordiv(a_ref, b_ref), - bytecode::BinaryOperator::Modulo => vm._imod(a_ref, b_ref), - bytecode::BinaryOperator::Lshift => vm._ilshift(a_ref, b_ref), - bytecode::BinaryOperator::Rshift => vm._irshift(a_ref, b_ref), - bytecode::BinaryOperator::Xor => vm._ixor(a_ref, b_ref), - bytecode::BinaryOperator::Or => vm._ior(a_ref, b_ref), - bytecode::BinaryOperator::And => vm._iand(a_ref, b_ref), - }?; - - self.push_value(value); - Ok(None) - } - - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn execute_unop(&mut self, vm: &VirtualMachine, op: bytecode::UnaryOperator) -> FrameResult { - let a = self.pop_value(); - let value = match op { - bytecode::UnaryOperator::Minus => vm._neg(&a)?, - bytecode::UnaryOperator::Plus => vm._pos(&a)?, - bytecode::UnaryOperator::Invert => vm._invert(&a)?, - bytecode::UnaryOperator::Not => { - let value = a.try_to_bool(vm)?; - vm.ctx.new_bool(!value).into() - } - }; - self.push_value(value); - Ok(None) - } - - #[cold] - fn setup_annotations(&mut self, vm: &VirtualMachine) -> FrameResult { - let __annotations__ = identifier!(vm, __annotations__); - // Try using locals as dict first, if not, fallback to generic method. - let has_annotations = match self - .locals - .clone() - .into_object() - .downcast_exact::<PyDict>(vm) - { - Ok(d) => d.contains_key(__annotations__, vm), - Err(o) => { - let needle = __annotations__.to_object(); - self._in(vm, needle, &o)? - } - }; - if !has_annotations { - self.locals - .as_object() - .set_item(__annotations__, vm.ctx.new_dict().into(), vm)?; - } - Ok(None) - } - - fn print_expr(&mut self, vm: &VirtualMachine) -> FrameResult { - let expr = self.pop_value(); - - let displayhook = vm - .sys_module - .get_attr("displayhook", vm) - .map_err(|_| vm.new_runtime_error("lost sys.displayhook".to_owned()))?; - displayhook.call((expr,), vm)?; - - Ok(None) - } - - fn unpack_sequence(&mut self, size: u32, vm: &VirtualMachine) -> FrameResult { - let value = self.pop_value(); - let elements: Vec<_> = value.try_to_value(vm).map_err(|e| { - if e.class().is(vm.ctx.exceptions.type_error) { - vm.new_type_error(format!( - "cannot unpack non-iterable {} object", - value.class().name() - )) - } else { - e - } - })?; - let msg = match elements.len().cmp(&(size as usize)) { - std::cmp::Ordering::Equal => { - self.state.stack.extend(elements.into_iter().rev()); - return Ok(None); - } - std::cmp::Ordering::Greater => { - format!("too many values to unpack (expected {size})") - } - std::cmp::Ordering::Less => format!( - "not enough values to unpack (expected {}, got {})", - size, - elements.len() - ), - }; - Err(vm.new_value_error(msg)) - } - - fn format_value( - &mut self, - conversion: bytecode::ConversionFlag, - vm: &VirtualMachine, - ) -> FrameResult { - use bytecode::ConversionFlag; - let value = self.pop_value(); - let value = match conversion { - ConversionFlag::Str => value.str(vm)?.into(), - ConversionFlag::Repr => value.repr(vm)?.into(), - ConversionFlag::Ascii => vm.ctx.new_str(builtins::ascii(value, vm)?).into(), - ConversionFlag::None => value, - }; - - let spec = self.pop_value(); - let formatted = vm.format(&value, spec.downcast::<PyStr>().unwrap())?; - self.push_value(formatted.into()); - Ok(None) - } - - fn _in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: &PyObject) -> PyResult<bool> { - let found = vm._contains(haystack, needle)?; - found.try_to_bool(vm) - } - - #[inline(always)] - fn _not_in( - &self, - vm: &VirtualMachine, - needle: PyObjectRef, - haystack: &PyObject, - ) -> PyResult<bool> { - Ok(!self._in(vm, needle, haystack)?) - } - - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn execute_test(&mut self, vm: &VirtualMachine, op: bytecode::TestOperator) -> FrameResult { - let b = self.pop_value(); - let a = self.pop_value(); - let value = match op { - bytecode::TestOperator::Is => a.is(&b), - bytecode::TestOperator::IsNot => !a.is(&b), - bytecode::TestOperator::In => self._in(vm, a, &b)?, - bytecode::TestOperator::NotIn => self._not_in(vm, a, &b)?, - bytecode::TestOperator::ExceptionMatch => a.is_instance(&b, vm)?, - }; - - self.push_value(vm.ctx.new_bool(value).into()); - Ok(None) - } - - #[cfg_attr(feature = "flame-it", flame("Frame"))] - fn execute_compare( - &mut self, - vm: &VirtualMachine, - op: bytecode::ComparisonOperator, - ) -> FrameResult { - let b = self.pop_value(); - let a = self.pop_value(); - let value = a.rich_compare(b, op.into(), vm)?; - self.push_value(value); - Ok(None) - } - - fn load_attr(&mut self, vm: &VirtualMachine, attr: bytecode::NameIdx) -> FrameResult { - let attr_name = self.code.names[attr as usize]; - let parent = self.pop_value(); - let obj = parent.get_attr(attr_name, vm)?; - self.push_value(obj); - Ok(None) - } - - fn store_attr(&mut self, vm: &VirtualMachine, attr: bytecode::NameIdx) -> FrameResult { - let attr_name = self.code.names[attr as usize]; - let parent = self.pop_value(); - let value = self.pop_value(); - parent.set_attr(attr_name, value, vm)?; - Ok(None) - } - - fn delete_attr(&mut self, vm: &VirtualMachine, attr: bytecode::NameIdx) -> FrameResult { - let attr_name = self.code.names[attr as usize]; - let parent = self.pop_value(); - parent.del_attr(attr_name, vm)?; - Ok(None) - } - - fn push_block(&mut self, typ: BlockType) { - self.state.blocks.push(Block { - typ, - level: self.state.stack.len(), - }); - } - - fn pop_block(&mut self) -> Block { - let block = self.state.blocks.pop().expect("No more blocks to pop!"); - self.state.stack.truncate(block.level); - block - } - - #[inline] - fn current_block(&self) -> Option<Block> { - self.state.blocks.last().cloned() - } - - #[inline] - fn push_value(&mut self, obj: PyObjectRef) { - match self.state.stack.try_push(obj) { - Ok(()) => {} - Err(_e) => self.fatal("tried to push value onto stack but overflowed max_stackdepth"), - } - } - - #[inline] - fn pop_value(&mut self) -> PyObjectRef { - match self.state.stack.pop() { - Some(x) => x, - None => self.fatal("tried to pop value but there was nothing on the stack"), - } - } - - fn pop_multiple(&mut self, count: usize) -> crate::common::boxvec::Drain<PyObjectRef> { - let stack_len = self.state.stack.len(); - self.state.stack.drain(stack_len - count..) - } - - #[inline] - fn last_value(&self) -> PyObjectRef { - self.last_value_ref().to_owned() - } - - #[inline] - fn last_value_ref(&self) -> &PyObject { - match &*self.state.stack { - [.., last] => last, - [] => self.fatal("tried to get top of stack but stack is empty"), - } - } - - #[inline] - fn nth_value(&self, depth: u32) -> &PyObject { - let stack = &self.state.stack; - &stack[stack.len() - depth as usize - 1] - } - - #[cold] - #[inline(never)] - fn fatal(&self, msg: &'static str) -> ! { - dbg!(self); - panic!("{}", msg) - } -} - -impl fmt::Debug for Frame { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let state = self.state.lock(); - let stack_str = state.stack.iter().fold(String::new(), |mut s, elem| { - if elem.payload_is::<Frame>() { - s.push_str("\n > {frame}"); - } else { - std::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); - } - s - }); - let block_str = state.blocks.iter().fold(String::new(), |mut s, elem| { - std::fmt::write(&mut s, format_args!("\n > {elem:?}")).unwrap(); - s - }); - // TODO: fix this up - let locals = self.locals.clone(); - write!( - f, - "Frame Object {{ \n Stack:{}\n Blocks:{}\n Locals:{:?}\n}}", - stack_str, - block_str, - locals.into_object() - ) - } -} diff --git a/vm/src/function/argument.rs b/vm/src/function/argument.rs deleted file mode 100644 index bcc711a1e9d..00000000000 --- a/vm/src/function/argument.rs +++ /dev/null @@ -1,582 +0,0 @@ -use crate::{ - builtins::{PyBaseExceptionRef, PyTupleRef, PyTypeRef}, - convert::ToPyObject, - object::{Traverse, TraverseFn}, - AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, -}; -use indexmap::IndexMap; -use itertools::Itertools; -use std::ops::RangeInclusive; - -pub trait IntoFuncArgs: Sized { - fn into_args(self, vm: &VirtualMachine) -> FuncArgs; - fn into_method_args(self, obj: PyObjectRef, vm: &VirtualMachine) -> FuncArgs { - let mut args = self.into_args(vm); - args.prepend_arg(obj); - args - } -} - -impl<T> IntoFuncArgs for T -where - T: Into<FuncArgs>, -{ - fn into_args(self, _vm: &VirtualMachine) -> FuncArgs { - self.into() - } -} - -// A tuple of values that each implement `ToPyObject` represents a sequence of -// arguments that can be bound and passed to a built-in function. -macro_rules! into_func_args_from_tuple { - ($(($n:tt, $T:ident)),*) => { - impl<$($T,)*> IntoFuncArgs for ($($T,)*) - where - $($T: ToPyObject,)* - { - #[inline] - fn into_args(self, vm: &VirtualMachine) -> FuncArgs { - let ($($n,)*) = self; - PosArgs::new(vec![$($n.to_pyobject(vm),)*]).into() - } - - #[inline] - fn into_method_args(self, obj: PyObjectRef, vm: &VirtualMachine) -> FuncArgs { - let ($($n,)*) = self; - PosArgs::new(vec![obj, $($n.to_pyobject(vm),)*]).into() - } - } - }; -} - -into_func_args_from_tuple!((v1, T1)); -into_func_args_from_tuple!((v1, T1), (v2, T2)); -into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3)); -into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4)); -into_func_args_from_tuple!((v1, T1), (v2, T2), (v3, T3), (v4, T4), (v5, T5)); - -/// The `FuncArgs` struct is one of the most used structs then creating -/// a rust function that can be called from python. It holds both positional -/// arguments, as well as keyword arguments passed to the function. -#[derive(Debug, Default, Clone, Traverse)] -pub struct FuncArgs { - pub args: Vec<PyObjectRef>, - // sorted map, according to https://www.python.org/dev/peps/pep-0468/ - pub kwargs: IndexMap<String, PyObjectRef>, -} - -unsafe impl Traverse for IndexMap<String, PyObjectRef> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.values().for_each(|v| v.traverse(tracer_fn)); - } -} - -/// Conversion from vector of python objects to function arguments. -impl<A> From<A> for FuncArgs -where - A: Into<PosArgs>, -{ - fn from(args: A) -> Self { - FuncArgs { - args: args.into().into_vec(), - kwargs: IndexMap::new(), - } - } -} - -impl From<KwArgs> for FuncArgs { - fn from(kwargs: KwArgs) -> Self { - FuncArgs { - args: Vec::new(), - kwargs: kwargs.0, - } - } -} - -impl FromArgs for FuncArgs { - fn from_args(_vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - Ok(std::mem::take(args)) - } -} - -impl FuncArgs { - pub fn new<A, K>(args: A, kwargs: K) -> Self - where - A: Into<PosArgs>, - K: Into<KwArgs>, - { - let PosArgs(args) = args.into(); - let KwArgs(kwargs) = kwargs.into(); - Self { args, kwargs } - } - - pub fn with_kwargs_names<A, KW>(mut args: A, kwarg_names: KW) -> Self - where - A: ExactSizeIterator<Item = PyObjectRef>, - KW: ExactSizeIterator<Item = String>, - { - // last `kwarg_names.len()` elements of args in order of appearance in the call signature - let total_argc = args.len(); - let kwargc = kwarg_names.len(); - let posargc = total_argc - kwargc; - - let posargs = args.by_ref().take(posargc).collect(); - - let kwargs = kwarg_names.zip_eq(args).collect::<IndexMap<_, _>>(); - - FuncArgs { - args: posargs, - kwargs, - } - } - - pub fn prepend_arg(&mut self, item: PyObjectRef) { - self.args.reserve_exact(1); - self.args.insert(0, item) - } - - pub fn shift(&mut self) -> PyObjectRef { - self.args.remove(0) - } - - pub fn get_kwarg(&self, key: &str, default: PyObjectRef) -> PyObjectRef { - self.kwargs - .get(key) - .cloned() - .unwrap_or_else(|| default.clone()) - } - - pub fn get_optional_kwarg(&self, key: &str) -> Option<PyObjectRef> { - self.kwargs.get(key).cloned() - } - - pub fn get_optional_kwarg_with_type( - &self, - key: &str, - ty: PyTypeRef, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - match self.get_optional_kwarg(key) { - Some(kwarg) => { - if kwarg.fast_isinstance(&ty) { - Ok(Some(kwarg)) - } else { - let expected_ty_name = &ty.name(); - let kwarg_class = kwarg.class(); - let actual_ty_name = &kwarg_class.name(); - Err(vm.new_type_error(format!( - "argument of type {expected_ty_name} is required for named parameter `{key}` (got: {actual_ty_name})" - ))) - } - } - None => Ok(None), - } - } - - pub fn take_positional(&mut self) -> Option<PyObjectRef> { - if self.args.is_empty() { - None - } else { - Some(self.args.remove(0)) - } - } - - pub fn take_positional_keyword(&mut self, name: &str) -> Option<PyObjectRef> { - self.take_positional().or_else(|| self.take_keyword(name)) - } - - pub fn take_keyword(&mut self, name: &str) -> Option<PyObjectRef> { - self.kwargs.swap_remove(name) - } - - pub fn remaining_keywords(&mut self) -> impl Iterator<Item = (String, PyObjectRef)> + '_ { - self.kwargs.drain(..) - } - - /// Binds these arguments to their respective values. - /// - /// If there is an insufficient number of arguments, there are leftover - /// arguments after performing the binding, or if an argument is not of - /// the expected type, a TypeError is raised. - /// - /// If the given `FromArgs` includes any conversions, exceptions raised - /// during the conversion will halt the binding and return the error. - pub fn bind<T: FromArgs>(mut self, vm: &VirtualMachine) -> PyResult<T> { - let given_args = self.args.len(); - let bound = T::from_args(vm, &mut self) - .map_err(|e| e.into_exception(T::arity(), given_args, vm))?; - - if !self.args.is_empty() { - Err(vm.new_type_error(format!( - "Expected at most {} arguments ({} given)", - T::arity().end(), - given_args, - ))) - } else if let Some(err) = self.check_kwargs_empty(vm) { - Err(err) - } else { - Ok(bound) - } - } - - pub fn check_kwargs_empty(&self, vm: &VirtualMachine) -> Option<PyBaseExceptionRef> { - self.kwargs - .keys() - .next() - .map(|k| vm.new_type_error(format!("Unexpected keyword argument {k}"))) - } -} - -/// An error encountered while binding arguments to the parameters of a Python -/// function call. -pub enum ArgumentError { - /// The call provided fewer positional arguments than the function requires. - TooFewArgs, - /// The call provided more positional arguments than the function accepts. - TooManyArgs, - /// The function doesn't accept a keyword argument with the given name. - InvalidKeywordArgument(String), - /// The function require a keyword argument with the given name, but one wasn't provided - RequiredKeywordArgument(String), - /// An exception was raised while binding arguments to the function - /// parameters. - Exception(PyBaseExceptionRef), -} - -impl From<PyBaseExceptionRef> for ArgumentError { - fn from(ex: PyBaseExceptionRef) -> Self { - ArgumentError::Exception(ex) - } -} - -impl ArgumentError { - fn into_exception( - self, - arity: RangeInclusive<usize>, - num_given: usize, - vm: &VirtualMachine, - ) -> PyBaseExceptionRef { - match self { - ArgumentError::TooFewArgs => vm.new_type_error(format!( - "Expected at least {} arguments ({} given)", - arity.start(), - num_given - )), - ArgumentError::TooManyArgs => vm.new_type_error(format!( - "Expected at most {} arguments ({} given)", - arity.end(), - num_given - )), - ArgumentError::InvalidKeywordArgument(name) => { - vm.new_type_error(format!("{name} is an invalid keyword argument")) - } - ArgumentError::RequiredKeywordArgument(name) => { - vm.new_type_error(format!("Required keyqord only argument {name}")) - } - ArgumentError::Exception(ex) => ex, - } - } -} - -/// Implemented by any type that can be accepted as a parameter to a built-in -/// function. -/// -pub trait FromArgs: Sized { - /// The range of positional arguments permitted by the function signature. - /// - /// Returns an empty range if not applicable. - fn arity() -> RangeInclusive<usize> { - 0..=0 - } - - /// Extracts this item from the next argument(s). - fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError>; -} - -pub trait FromArgOptional { - type Inner: TryFromObject; - fn from_inner(x: Self::Inner) -> Self; -} -impl<T: TryFromObject> FromArgOptional for OptionalArg<T> { - type Inner = T; - fn from_inner(x: T) -> Self { - Self::Present(x) - } -} -impl<T: TryFromObject> FromArgOptional for T { - type Inner = Self; - fn from_inner(x: Self) -> Self { - x - } -} - -/// A map of keyword arguments to their values. -/// -/// A built-in function with a `KwArgs` parameter is analogous to a Python -/// function with `**kwargs`. All remaining keyword arguments are extracted -/// (and hence the function will permit an arbitrary number of them). -/// -/// `KwArgs` optionally accepts a generic type parameter to allow type checks -/// or conversions of each argument. -/// -/// Note: -/// -/// KwArgs is only for functions that accept arbitrary keyword arguments. For -/// functions that accept only *specific* named arguments, a rust struct with -/// an appropriate FromArgs implementation must be created. -#[derive(Clone)] -pub struct KwArgs<T = PyObjectRef>(IndexMap<String, T>); - -unsafe impl<T> Traverse for KwArgs<T> -where - T: Traverse, -{ - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.0.iter().map(|(_, v)| v.traverse(tracer_fn)).count(); - } -} - -impl<T> KwArgs<T> { - pub fn new(map: IndexMap<String, T>) -> Self { - KwArgs(map) - } - - pub fn pop_kwarg(&mut self, name: &str) -> Option<T> { - self.0.remove(name) - } - - pub fn is_empty(self) -> bool { - self.0.is_empty() - } -} -impl<T> FromIterator<(String, T)> for KwArgs<T> { - fn from_iter<I: IntoIterator<Item = (String, T)>>(iter: I) -> Self { - KwArgs(iter.into_iter().collect()) - } -} -impl<T> Default for KwArgs<T> { - fn default() -> Self { - KwArgs(IndexMap::new()) - } -} - -impl<T> FromArgs for KwArgs<T> -where - T: TryFromObject, -{ - fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - let mut kwargs = IndexMap::new(); - for (name, value) in args.remaining_keywords() { - kwargs.insert(name, value.try_into_value(vm)?); - } - Ok(KwArgs(kwargs)) - } -} - -impl<T> IntoIterator for KwArgs<T> { - type Item = (String, T); - type IntoIter = indexmap::map::IntoIter<String, T>; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -/// A list of positional argument values. -/// -/// A built-in function with a `PosArgs` parameter is analogous to a Python -/// function with `*args`. All remaining positional arguments are extracted -/// (and hence the function will permit an arbitrary number of them). -/// -/// `PosArgs` optionally accepts a generic type parameter to allow type checks -/// or conversions of each argument. -#[derive(Clone)] -pub struct PosArgs<T = PyObjectRef>(Vec<T>); - -unsafe impl<T> Traverse for PosArgs<T> -where - T: Traverse, -{ - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.0.traverse(tracer_fn) - } -} - -impl<T> PosArgs<T> { - pub fn new(args: Vec<T>) -> Self { - Self(args) - } - - pub fn into_vec(self) -> Vec<T> { - self.0 - } - - pub fn iter(&self) -> std::slice::Iter<T> { - self.0.iter() - } -} - -impl<T> From<Vec<T>> for PosArgs<T> { - fn from(v: Vec<T>) -> Self { - Self(v) - } -} - -impl From<()> for PosArgs<PyObjectRef> { - fn from(_args: ()) -> Self { - Self(Vec::new()) - } -} - -impl<T> AsRef<[T]> for PosArgs<T> { - fn as_ref(&self) -> &[T] { - &self.0 - } -} - -impl<T: PyPayload> PosArgs<PyRef<T>> { - pub fn into_tuple(self, vm: &VirtualMachine) -> PyTupleRef { - vm.ctx - .new_tuple(self.0.into_iter().map(Into::into).collect()) - } -} - -impl<T> FromArgs for PosArgs<T> -where - T: TryFromObject, -{ - fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - let mut varargs = Vec::new(); - while let Some(value) = args.take_positional() { - varargs.push(value.try_into_value(vm)?); - } - Ok(PosArgs(varargs)) - } -} - -impl<T> IntoIterator for PosArgs<T> { - type Item = T; - type IntoIter = std::vec::IntoIter<T>; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -impl<T> FromArgs for T -where - T: TryFromObject, -{ - fn arity() -> RangeInclusive<usize> { - 1..=1 - } - - fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - let value = args.take_positional().ok_or(ArgumentError::TooFewArgs)?; - Ok(value.try_into_value(vm)?) - } -} - -/// An argument that may or may not be provided by the caller. -/// -/// This style of argument is not possible in pure Python. -#[derive(Debug, result_like::OptionLike, is_macro::Is)] -pub enum OptionalArg<T = PyObjectRef> { - Present(T), - Missing, -} - -unsafe impl<T> Traverse for OptionalArg<T> -where - T: Traverse, -{ - fn traverse(&self, tracer_fn: &mut TraverseFn) { - match self { - OptionalArg::Present(ref o) => o.traverse(tracer_fn), - OptionalArg::Missing => (), - } - } -} - -impl OptionalArg<PyObjectRef> { - pub fn unwrap_or_none(self, vm: &VirtualMachine) -> PyObjectRef { - self.unwrap_or_else(|| vm.ctx.none()) - } -} - -pub type OptionalOption<T = PyObjectRef> = OptionalArg<Option<T>>; - -impl<T> OptionalOption<T> { - #[inline] - pub fn flatten(self) -> Option<T> { - self.into_option().flatten() - } -} - -impl<T> FromArgs for OptionalArg<T> -where - T: TryFromObject, -{ - fn arity() -> RangeInclusive<usize> { - 0..=1 - } - - fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - let r = if let Some(value) = args.take_positional() { - OptionalArg::Present(value.try_into_value(vm)?) - } else { - OptionalArg::Missing - }; - Ok(r) - } -} - -// For functions that accept no arguments. Implemented explicitly instead of via -// macro below to avoid unused warnings. -impl FromArgs for () { - fn from_args(_vm: &VirtualMachine, _args: &mut FuncArgs) -> Result<Self, ArgumentError> { - Ok(()) - } -} - -// A tuple of types that each implement `FromArgs` represents a sequence of -// arguments that can be bound and passed to a built-in function. -// -// Technically, a tuple can contain tuples, which can contain tuples, and so on, -// so this actually represents a tree of values to be bound from arguments, but -// in practice this is only used for the top-level parameters. -macro_rules! tuple_from_py_func_args { - ($($T:ident),+) => { - impl<$($T),+> FromArgs for ($($T,)+) - where - $($T: FromArgs),+ - { - fn arity() -> RangeInclusive<usize> { - let mut min = 0; - let mut max = 0; - $( - let (start, end) = $T::arity().into_inner(); - min += start; - max += end; - )+ - min..=max - } - - fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - Ok(($($T::from_args(vm, args)?,)+)) - } - } - }; -} - -// Implement `FromArgs` for up to 7-tuples, allowing built-in functions to bind -// up to 7 top-level parameters (note that `PosArgs`, `KwArgs`, nested tuples, etc. -// count as 1, so this should actually be more than enough). -tuple_from_py_func_args!(A); -tuple_from_py_func_args!(A, B); -tuple_from_py_func_args!(A, B, C); -tuple_from_py_func_args!(A, B, C, D); -tuple_from_py_func_args!(A, B, C, D, E); -tuple_from_py_func_args!(A, B, C, D, E, F); -tuple_from_py_func_args!(A, B, C, D, E, F, G); -tuple_from_py_func_args!(A, B, C, D, E, F, G, H); diff --git a/vm/src/function/buffer.rs b/vm/src/function/buffer.rs deleted file mode 100644 index f5d0dd03d63..00000000000 --- a/vm/src/function/buffer.rs +++ /dev/null @@ -1,202 +0,0 @@ -use crate::{ - builtins::{PyStr, PyStrRef}, - common::borrow::{BorrowedValue, BorrowedValueMut}, - protocol::PyBuffer, - AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, - VirtualMachine, -}; - -// Python/getargs.c - -/// any bytes-like object. Like the `y*` format code for `PyArg_Parse` in CPython. -#[derive(Debug, Traverse)] -pub struct ArgBytesLike(PyBuffer); - -impl PyObject { - pub fn try_bytes_like<R>( - &self, - vm: &VirtualMachine, - f: impl FnOnce(&[u8]) -> R, - ) -> PyResult<R> { - let buffer = PyBuffer::try_from_borrowed_object(vm, self)?; - buffer.as_contiguous().map(|x| f(&x)).ok_or_else(|| { - vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned()) - }) - } - - pub fn try_rw_bytes_like<R>( - &self, - vm: &VirtualMachine, - f: impl FnOnce(&mut [u8]) -> R, - ) -> PyResult<R> { - let buffer = PyBuffer::try_from_borrowed_object(vm, self)?; - buffer - .as_contiguous_mut() - .map(|mut x| f(&mut x)) - .ok_or_else(|| { - vm.new_type_error("buffer is not a read-write bytes-like object".to_owned()) - }) - } -} - -impl ArgBytesLike { - pub fn borrow_buf(&self) -> BorrowedValue<'_, [u8]> { - unsafe { self.0.contiguous_unchecked() } - } - - pub fn with_ref<F, R>(&self, f: F) -> R - where - F: FnOnce(&[u8]) -> R, - { - f(&self.borrow_buf()) - } - - pub fn len(&self) -> usize { - self.0.desc.len - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn as_object(&self) -> &PyObject { - &self.0.obj - } -} - -impl From<ArgBytesLike> for PyBuffer { - fn from(buffer: ArgBytesLike) -> Self { - buffer.0 - } -} - -impl<'a> TryFromBorrowedObject<'a> for ArgBytesLike { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; - if buffer.desc.is_contiguous() { - Ok(Self(buffer)) - } else { - Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned())) - } - } -} - -/// A memory buffer, read-write access. Like the `w*` format code for `PyArg_Parse` in CPython. -#[derive(Debug, Traverse)] -pub struct ArgMemoryBuffer(PyBuffer); - -impl ArgMemoryBuffer { - pub fn borrow_buf_mut(&self) -> BorrowedValueMut<'_, [u8]> { - unsafe { self.0.contiguous_mut_unchecked() } - } - - pub fn with_ref<F, R>(&self, f: F) -> R - where - F: FnOnce(&mut [u8]) -> R, - { - f(&mut self.borrow_buf_mut()) - } - - pub fn len(&self) -> usize { - self.0.desc.len - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -impl From<ArgMemoryBuffer> for PyBuffer { - fn from(buffer: ArgMemoryBuffer) -> Self { - buffer.0 - } -} - -impl<'a> TryFromBorrowedObject<'a> for ArgMemoryBuffer { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; - if !buffer.desc.is_contiguous() { - Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned())) - } else if buffer.desc.readonly { - Err(vm.new_type_error("buffer is not a read-write bytes-like object".to_owned())) - } else { - Ok(Self(buffer)) - } - } -} - -/// A text string or bytes-like object. Like the `s*` format code for `PyArg_Parse` in CPython. -pub enum ArgStrOrBytesLike { - Buf(ArgBytesLike), - Str(PyStrRef), -} - -impl ArgStrOrBytesLike { - pub fn as_object(&self) -> &PyObject { - match self { - Self::Buf(b) => b.as_object(), - Self::Str(s) => s.as_object(), - } - } -} - -impl TryFromObject for ArgStrOrBytesLike { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - obj.downcast() - .map(Self::Str) - .or_else(|obj| ArgBytesLike::try_from_object(vm, obj).map(Self::Buf)) - } -} - -impl ArgStrOrBytesLike { - pub fn borrow_bytes(&self) -> BorrowedValue<'_, [u8]> { - match self { - Self::Buf(b) => b.borrow_buf(), - Self::Str(s) => s.as_str().as_bytes().into(), - } - } -} - -#[derive(Debug)] -pub enum ArgAsciiBuffer { - String(PyStrRef), - Buffer(ArgBytesLike), -} - -impl TryFromObject for ArgAsciiBuffer { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - match obj.downcast::<PyStr>() { - Ok(string) => { - if string.as_str().is_ascii() { - Ok(ArgAsciiBuffer::String(string)) - } else { - Err(vm.new_value_error( - "string argument should contain only ASCII characters".to_owned(), - )) - } - } - Err(obj) => ArgBytesLike::try_from_object(vm, obj).map(ArgAsciiBuffer::Buffer), - } - } -} - -impl ArgAsciiBuffer { - pub fn len(&self) -> usize { - match self { - Self::String(s) => s.as_str().len(), - Self::Buffer(buffer) => buffer.len(), - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - #[inline] - pub fn with_ref<R>(&self, f: impl FnOnce(&[u8]) -> R) -> R { - match self { - Self::String(s) => f(s.as_str().as_bytes()), - Self::Buffer(buffer) => buffer.with_ref(f), - } - } -} diff --git a/vm/src/function/fspath.rs b/vm/src/function/fspath.rs deleted file mode 100644 index 41e99b05420..00000000000 --- a/vm/src/function/fspath.rs +++ /dev/null @@ -1,130 +0,0 @@ -use crate::{ - builtins::{PyBytes, PyBytesRef, PyStrRef}, - convert::{IntoPyException, ToPyObject}, - function::PyStr, - protocol::PyBuffer, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, -}; -use std::{ffi::OsStr, path::PathBuf}; - -#[derive(Clone)] -pub enum FsPath { - Str(PyStrRef), - Bytes(PyBytesRef), -} - -impl FsPath { - // PyOS_FSPath in CPython - pub fn try_from(obj: PyObjectRef, check_for_nul: bool, vm: &VirtualMachine) -> PyResult<Self> { - let check_nul = |b: &[u8]| { - if !check_for_nul || memchr::memchr(b'\0', b).is_none() { - Ok(()) - } else { - Err(crate::exceptions::cstring_error(vm)) - } - }; - let match1 = |obj: PyObjectRef| { - let pathlike = match_class!(match obj { - s @ PyStr => { - check_nul(s.as_str().as_bytes())?; - FsPath::Str(s) - } - b @ PyBytes => { - check_nul(&b)?; - FsPath::Bytes(b) - } - obj => return Ok(Err(obj)), - }); - Ok(Ok(pathlike)) - }; - let obj = match match1(obj)? { - Ok(pathlike) => return Ok(pathlike), - Err(obj) => obj, - }; - let method = - vm.get_method_or_type_error(obj.clone(), identifier!(vm, __fspath__), || { - format!( - "should be string, bytes, os.PathLike or integer, not {}", - obj.class().name() - ) - })?; - let result = method.call((), vm)?; - match1(result)?.map_err(|result| { - vm.new_type_error(format!( - "expected {}.__fspath__() to return str or bytes, not {}", - obj.class().name(), - result.class().name(), - )) - }) - } - - pub fn as_os_str(&self, vm: &VirtualMachine) -> PyResult<&OsStr> { - // TODO: FS encodings - match self { - FsPath::Str(s) => Ok(s.as_str().as_ref()), - FsPath::Bytes(b) => Self::bytes_as_osstr(b.as_bytes(), vm), - } - } - - pub fn as_bytes(&self) -> &[u8] { - // TODO: FS encodings - match self { - FsPath::Str(s) => s.as_str().as_bytes(), - FsPath::Bytes(b) => b.as_bytes(), - } - } - - pub fn as_str(&self) -> &str { - match self { - FsPath::Bytes(b) => std::str::from_utf8(b).unwrap(), - FsPath::Str(s) => s.as_str(), - } - } - - pub fn to_path_buf(&self, vm: &VirtualMachine) -> PyResult<PathBuf> { - let path = match self { - FsPath::Str(s) => PathBuf::from(s.as_str()), - FsPath::Bytes(b) => PathBuf::from(Self::bytes_as_osstr(b, vm)?), - }; - Ok(path) - } - - pub fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<std::ffi::CString> { - std::ffi::CString::new(self.as_bytes()).map_err(|e| e.into_pyexception(vm)) - } - - #[cfg(windows)] - pub fn to_widecstring(&self, vm: &VirtualMachine) -> PyResult<widestring::WideCString> { - widestring::WideCString::from_os_str(self.as_os_str(vm)?) - .map_err(|err| err.into_pyexception(vm)) - } - - pub fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { - rustpython_common::os::bytes_as_osstr(b) - .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8".to_owned())) - } -} - -impl ToPyObject for FsPath { - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - match self { - Self::Str(s) => s.into(), - Self::Bytes(b) => b.into(), - } - } -} - -impl TryFromObject for FsPath { - // PyUnicode_FSDecoder in CPython - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let obj = match obj.try_to_value::<PyBuffer>(vm) { - Ok(buffer) => { - let mut bytes = vec![]; - buffer.append_to(&mut bytes); - vm.ctx.new_bytes(bytes).into() - } - Err(_) => obj, - }; - Self::try_from(obj, true, vm) - } -} diff --git a/vm/src/function/getset.rs b/vm/src/function/getset.rs deleted file mode 100644 index 827158e8347..00000000000 --- a/vm/src/function/getset.rs +++ /dev/null @@ -1,244 +0,0 @@ -/*! Python `attribute` descriptor class. (PyGetSet) - -*/ -use crate::{ - convert::ToPyResult, - function::{BorrowedParam, OwnedParam, RefParam}, - object::PyThreadingConstraint, - Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, -}; - -#[derive(result_like::OptionLike, is_macro::Is, Debug)] -pub enum PySetterValue<T = PyObjectRef> { - Assign(T), - Delete, -} - -impl PySetterValue { - pub fn unwrap_or_none(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - Self::Assign(value) => value, - Self::Delete => vm.ctx.none(), - } - } -} - -trait FromPySetterValue -where - Self: Sized, -{ - fn from_setter_value(vm: &VirtualMachine, obj: PySetterValue) -> PyResult<Self>; -} - -impl<T> FromPySetterValue for T -where - T: Sized + TryFromObject, -{ - #[inline] - fn from_setter_value(vm: &VirtualMachine, obj: PySetterValue) -> PyResult<Self> { - let obj = obj.ok_or_else(|| vm.new_type_error("can't delete attribute".to_owned()))?; - T::try_from_object(vm, obj) - } -} - -impl<T> FromPySetterValue for PySetterValue<T> -where - T: Sized + TryFromObject, -{ - #[inline] - fn from_setter_value(vm: &VirtualMachine, obj: PySetterValue) -> PyResult<Self> { - obj.map(|obj| T::try_from_object(vm, obj)).transpose() - } -} - -pub type PyGetterFunc = Box<py_dyn_fn!(dyn Fn(&VirtualMachine, PyObjectRef) -> PyResult)>; -pub type PySetterFunc = - Box<py_dyn_fn!(dyn Fn(&VirtualMachine, PyObjectRef, PySetterValue) -> PyResult<()>)>; - -pub trait IntoPyGetterFunc<T>: PyThreadingConstraint + Sized + 'static { - fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult; - fn into_getter(self) -> PyGetterFunc { - Box::new(move |vm, obj| self.get(obj, vm)) - } -} - -impl<F, T, R> IntoPyGetterFunc<(OwnedParam<T>, R, VirtualMachine)> for F -where - F: Fn(T, &VirtualMachine) -> R + 'static + Send + Sync, - T: TryFromObject, - R: ToPyResult, -{ - fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let obj = T::try_from_object(vm, obj)?; - (self)(obj, vm).to_pyresult(vm) - } -} - -impl<F, S, R> IntoPyGetterFunc<(BorrowedParam<S>, R, VirtualMachine)> for F -where - F: Fn(&Py<S>, &VirtualMachine) -> R + 'static + Send + Sync, - S: PyPayload, - R: ToPyResult, -{ - fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let zelf = PyRef::<S>::try_from_object(vm, obj)?; - (self)(&zelf, vm).to_pyresult(vm) - } -} - -impl<F, S, R> IntoPyGetterFunc<(RefParam<S>, R, VirtualMachine)> for F -where - F: Fn(&S, &VirtualMachine) -> R + 'static + Send + Sync, - S: PyPayload, - R: ToPyResult, -{ - fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let zelf = PyRef::<S>::try_from_object(vm, obj)?; - (self)(&zelf, vm).to_pyresult(vm) - } -} - -impl<F, T, R> IntoPyGetterFunc<(OwnedParam<T>, R)> for F -where - F: Fn(T) -> R + 'static + Send + Sync, - T: TryFromObject, - R: ToPyResult, -{ - fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let obj = T::try_from_object(vm, obj)?; - (self)(obj).to_pyresult(vm) - } -} - -impl<F, S, R> IntoPyGetterFunc<(BorrowedParam<S>, R)> for F -where - F: Fn(&Py<S>) -> R + 'static + Send + Sync, - S: PyPayload, - R: ToPyResult, -{ - fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let zelf = PyRef::<S>::try_from_object(vm, obj)?; - (self)(&zelf).to_pyresult(vm) - } -} - -impl<F, S, R> IntoPyGetterFunc<(RefParam<S>, R)> for F -where - F: Fn(&S) -> R + 'static + Send + Sync, - S: PyPayload, - R: ToPyResult, -{ - fn get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let zelf = PyRef::<S>::try_from_object(vm, obj)?; - (self)(&zelf).to_pyresult(vm) - } -} - -pub trait IntoPyNoResult { - fn into_noresult(self) -> PyResult<()>; -} - -impl IntoPyNoResult for () { - #[inline] - fn into_noresult(self) -> PyResult<()> { - Ok(()) - } -} - -impl IntoPyNoResult for PyResult<()> { - #[inline] - fn into_noresult(self) -> PyResult<()> { - self - } -} - -pub trait IntoPySetterFunc<T>: PyThreadingConstraint + Sized + 'static { - fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()>; - fn into_setter(self) -> PySetterFunc { - Box::new(move |vm, obj, value| self.set(obj, value, vm)) - } -} - -impl<F, T, V, R> IntoPySetterFunc<(OwnedParam<T>, V, R, VirtualMachine)> for F -where - F: Fn(T, V, &VirtualMachine) -> R + 'static + Send + Sync, - T: TryFromObject, - V: FromPySetterValue, - R: IntoPyNoResult, -{ - fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - let obj = T::try_from_object(vm, obj)?; - let value = V::from_setter_value(vm, value)?; - (self)(obj, value, vm).into_noresult() - } -} - -impl<F, S, V, R> IntoPySetterFunc<(BorrowedParam<S>, V, R, VirtualMachine)> for F -where - F: Fn(&Py<S>, V, &VirtualMachine) -> R + 'static + Send + Sync, - S: PyPayload, - V: FromPySetterValue, - R: IntoPyNoResult, -{ - fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - let zelf = PyRef::<S>::try_from_object(vm, obj)?; - let value = V::from_setter_value(vm, value)?; - (self)(&zelf, value, vm).into_noresult() - } -} - -impl<F, S, V, R> IntoPySetterFunc<(RefParam<S>, V, R, VirtualMachine)> for F -where - F: Fn(&S, V, &VirtualMachine) -> R + 'static + Send + Sync, - S: PyPayload, - V: FromPySetterValue, - R: IntoPyNoResult, -{ - fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - let zelf = PyRef::<S>::try_from_object(vm, obj)?; - let value = V::from_setter_value(vm, value)?; - (self)(&zelf, value, vm).into_noresult() - } -} - -impl<F, T, V, R> IntoPySetterFunc<(OwnedParam<T>, V, R)> for F -where - F: Fn(T, V) -> R + 'static + Send + Sync, - T: TryFromObject, - V: FromPySetterValue, - R: IntoPyNoResult, -{ - fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - let obj = T::try_from_object(vm, obj)?; - let value = V::from_setter_value(vm, value)?; - (self)(obj, value).into_noresult() - } -} - -impl<F, S, V, R> IntoPySetterFunc<(BorrowedParam<S>, V, R)> for F -where - F: Fn(&Py<S>, V) -> R + 'static + Send + Sync, - S: PyPayload, - V: FromPySetterValue, - R: IntoPyNoResult, -{ - fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - let zelf = PyRef::<S>::try_from_object(vm, obj)?; - let value = V::from_setter_value(vm, value)?; - (self)(&zelf, value).into_noresult() - } -} - -impl<F, S, V, R> IntoPySetterFunc<(RefParam<S>, V, R)> for F -where - F: Fn(&S, V) -> R + 'static + Send + Sync, - S: PyPayload, - V: FromPySetterValue, - R: IntoPyNoResult, -{ - fn set(&self, obj: PyObjectRef, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { - let zelf = PyRef::<S>::try_from_object(vm, obj)?; - let value = V::from_setter_value(vm, value)?; - (self)(&zelf, value).into_noresult() - } -} diff --git a/vm/src/function/method.rs b/vm/src/function/method.rs deleted file mode 100644 index fa47c16f4a0..00000000000 --- a/vm/src/function/method.rs +++ /dev/null @@ -1,311 +0,0 @@ -use crate::{ - builtins::{ - builtin_func::{PyNativeFunction, PyNativeMethod}, - descriptor::PyMethodDescriptor, - PyType, - }, - function::{IntoPyNativeFn, PyNativeFn}, - Context, Py, PyObjectRef, PyPayload, PyRef, VirtualMachine, -}; - -bitflags::bitflags! { - // METH_XXX flags in CPython - #[derive(Copy, Clone, Debug, PartialEq)] - pub struct PyMethodFlags: u32 { - // const VARARGS = 0x0001; - // const KEYWORDS = 0x0002; - // METH_NOARGS and METH_O must not be combined with the flags above. - // const NOARGS = 0x0004; - // const O = 0x0008; - - // METH_CLASS and METH_STATIC are a little different; these control - // the construction of methods for a class. These cannot be used for - // functions in modules. - const CLASS = 0x0010; - const STATIC = 0x0020; - - // METH_COEXIST allows a method to be entered even though a slot has - // already filled the entry. When defined, the flag allows a separate - // method, "__contains__" for example, to coexist with a defined - // slot like sq_contains. - // const COEXIST = 0x0040; - - // if not Py_LIMITED_API - // const FASTCALL = 0x0080; - - // This bit is preserved for Stackless Python - // const STACKLESS = 0x0100; - - // METH_METHOD means the function stores an - // additional reference to the class that defines it; - // both self and class are passed to it. - // It uses PyCMethodObject instead of PyCFunctionObject. - // May not be combined with METH_NOARGS, METH_O, METH_CLASS or METH_STATIC. - const METHOD = 0x0200; - } -} - -impl PyMethodFlags { - // FIXME: macro temp - pub const EMPTY: Self = Self::empty(); -} - -#[macro_export] -macro_rules! define_methods { - // TODO: more flexible syntax - ($($name:literal => $func:ident as $flags:ident),+) => { - vec![ $( $crate::function::PyMethodDef { - name: $name, - func: $crate::function::IntoPyNativeFn::into_func($func), - flags: $crate::function::PyMethodFlags::$flags, - doc: None, - }),+ ] - }; -} - -#[derive(Clone)] -pub struct PyMethodDef { - pub name: &'static str, // TODO: interned - pub func: &'static PyNativeFn, - pub flags: PyMethodFlags, - pub doc: Option<&'static str>, // TODO: interned -} - -impl PyMethodDef { - #[inline] - pub fn new<Kind>( - name: &'static str, - func: impl IntoPyNativeFn<Kind>, - flags: PyMethodFlags, - doc: Option<&'static str>, - ) -> Self { - Self { - name, - func: func.into_func(), - flags, - doc, - } - } - - #[inline] - pub const fn new_const<Kind>( - name: &'static str, - func: impl IntoPyNativeFn<Kind>, - flags: PyMethodFlags, - doc: Option<&'static str>, - ) -> Self { - Self { - name, - func: super::static_func(func), - flags, - doc, - } - } - - pub fn to_proper_method( - &'static self, - class: &'static Py<PyType>, - ctx: &Context, - ) -> PyObjectRef { - if self.flags.contains(PyMethodFlags::METHOD) { - self.build_method(ctx, class).into() - } else if self.flags.contains(PyMethodFlags::CLASS) { - self.build_classmethod(ctx, class).into() - } else if self.flags.contains(PyMethodFlags::STATIC) { - self.build_staticmethod(ctx, class).into() - } else { - unreachable!(); - } - } - pub fn to_function(&'static self) -> PyNativeFunction { - PyNativeFunction { - zelf: None, - value: self, - module: None, - } - } - pub fn to_method( - &'static self, - class: &'static Py<PyType>, - ctx: &Context, - ) -> PyMethodDescriptor { - PyMethodDescriptor::new(self, class, ctx) - } - pub fn to_bound_method( - &'static self, - obj: PyObjectRef, - class: &'static Py<PyType>, - ) -> PyNativeMethod { - PyNativeMethod { - func: PyNativeFunction { - zelf: Some(obj), - value: self, - module: None, - }, - class, - } - } - pub fn build_function(&'static self, ctx: &Context) -> PyRef<PyNativeFunction> { - self.to_function().into_ref(ctx) - } - pub fn build_bound_function( - &'static self, - ctx: &Context, - obj: PyObjectRef, - ) -> PyRef<PyNativeFunction> { - let function = PyNativeFunction { - zelf: Some(obj), - value: self, - module: None, - }; - PyRef::new_ref( - function, - ctx.types.builtin_function_or_method_type.to_owned(), - None, - ) - } - pub fn build_method( - &'static self, - ctx: &Context, - class: &'static Py<PyType>, - ) -> PyRef<PyMethodDescriptor> { - debug_assert!(self.flags.contains(PyMethodFlags::METHOD)); - PyRef::new_ref( - self.to_method(class, ctx), - ctx.types.method_descriptor_type.to_owned(), - None, - ) - } - pub fn build_bound_method( - &'static self, - ctx: &Context, - obj: PyObjectRef, - class: &'static Py<PyType>, - ) -> PyRef<PyNativeMethod> { - PyRef::new_ref( - self.to_bound_method(obj, class), - ctx.types.builtin_method_type.to_owned(), - None, - ) - } - pub fn build_classmethod( - &'static self, - ctx: &Context, - class: &'static Py<PyType>, - ) -> PyRef<PyMethodDescriptor> { - PyRef::new_ref( - self.to_method(class, ctx), - ctx.types.method_descriptor_type.to_owned(), - None, - ) - } - pub fn build_staticmethod( - &'static self, - ctx: &Context, - class: &'static Py<PyType>, - ) -> PyRef<PyNativeMethod> { - debug_assert!(self.flags.contains(PyMethodFlags::STATIC)); - let func = self.to_function(); - PyNativeMethod { func, class }.into_ref(ctx) - } - - #[doc(hidden)] - pub const fn __const_concat_arrays<const SUM_LEN: usize>( - method_groups: &[&[Self]], - ) -> [Self; SUM_LEN] { - const NULL_METHOD: PyMethodDef = PyMethodDef { - name: "", - func: &|_, _| unreachable!(), - flags: PyMethodFlags::empty(), - doc: None, - }; - let mut all_methods = [NULL_METHOD; SUM_LEN]; - let mut all_idx = 0; - let mut group_idx = 0; - while group_idx < method_groups.len() { - let group = method_groups[group_idx]; - let mut method_idx = 0; - while method_idx < group.len() { - all_methods[all_idx] = group[method_idx].const_copy(); - method_idx += 1; - all_idx += 1; - } - group_idx += 1; - } - all_methods - } - - const fn const_copy(&self) -> Self { - Self { - name: self.name, - func: self.func, - flags: self.flags, - doc: self.doc, - } - } -} - -impl std::fmt::Debug for PyMethodDef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PyMethodDef") - .field("name", &self.name) - .field( - "func", - &(unsafe { std::mem::transmute::<_, [usize; 2]>(self.func)[1] as *const u8 }), - ) - .field("flags", &self.flags) - .field("doc", &self.doc) - .finish() - } -} - -// This is not a part of CPython API. -// But useful to support dynamically generated methods -#[pyclass(name, module = false, ctx = "method_def")] -#[derive(Debug)] -pub struct HeapMethodDef { - method: PyMethodDef, -} - -impl HeapMethodDef { - pub fn new(method: PyMethodDef) -> Self { - Self { method } - } -} - -impl Py<HeapMethodDef> { - pub(crate) unsafe fn method(&self) -> &'static PyMethodDef { - &*(&self.method as *const _) - } - - pub fn build_function(&self, vm: &VirtualMachine) -> PyRef<PyNativeFunction> { - let function = unsafe { self.method() }.to_function(); - let dict = vm.ctx.new_dict(); - dict.set_item("__method_def__", self.to_owned().into(), vm) - .unwrap(); - PyRef::new_ref( - function, - vm.ctx.types.builtin_function_or_method_type.to_owned(), - Some(dict), - ) - } - - pub fn build_method( - &self, - class: &'static Py<PyType>, - vm: &VirtualMachine, - ) -> PyRef<PyMethodDescriptor> { - let function = unsafe { self.method() }.to_method(class, &vm.ctx); - let dict = vm.ctx.new_dict(); - dict.set_item("__method_def__", self.to_owned().into(), vm) - .unwrap(); - PyRef::new_ref( - function, - vm.ctx.types.method_descriptor_type.to_owned(), - Some(dict), - ) - } -} - -#[pyclass] -impl HeapMethodDef {} diff --git a/vm/src/function/mod.rs b/vm/src/function/mod.rs deleted file mode 100644 index a0a20c99600..00000000000 --- a/vm/src/function/mod.rs +++ /dev/null @@ -1,49 +0,0 @@ -mod argument; -mod arithmetic; -mod buffer; -mod builtin; -mod either; -mod fspath; -mod getset; -mod method; -mod number; -mod protocol; - -pub use argument::{ - ArgumentError, FromArgOptional, FromArgs, FuncArgs, IntoFuncArgs, KwArgs, OptionalArg, - OptionalOption, PosArgs, -}; -pub use arithmetic::{PyArithmeticValue, PyComparisonValue}; -pub use buffer::{ArgAsciiBuffer, ArgBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike}; -pub use builtin::{static_func, IntoPyNativeFn, PyNativeFn}; -pub use either::Either; -pub use fspath::FsPath; -pub use getset::PySetterValue; -pub(super) use getset::{IntoPyGetterFunc, IntoPySetterFunc, PyGetterFunc, PySetterFunc}; -pub use method::{HeapMethodDef, PyMethodDef, PyMethodFlags}; -pub use number::{ArgIndex, ArgIntoBool, ArgIntoComplex, ArgIntoFloat, ArgPrimitiveIndex, ArgSize}; -pub use protocol::{ArgCallable, ArgIterable, ArgMapping, ArgSequence}; - -use crate::{builtins::PyStr, convert::TryFromBorrowedObject, PyObject, PyResult, VirtualMachine}; -use builtin::{BorrowedParam, OwnedParam, RefParam}; - -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ArgByteOrder { - Big, - Little, -} - -impl<'a> TryFromBorrowedObject<'a> for ArgByteOrder { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - obj.try_value_with( - |s: &PyStr| match s.as_str() { - "big" => Ok(Self::Big), - "little" => Ok(Self::Little), - _ => { - Err(vm.new_value_error("byteorder must be either 'little' or 'big'".to_owned())) - } - }, - vm, - ) - } -} diff --git a/vm/src/function/number.rs b/vm/src/function/number.rs deleted file mode 100644 index 5f235433953..00000000000 --- a/vm/src/function/number.rs +++ /dev/null @@ -1,198 +0,0 @@ -use super::argument::OptionalArg; -use crate::{builtins::PyIntRef, AsObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine}; -use malachite_bigint::BigInt; -use num_complex::Complex64; -use num_traits::PrimInt; -use std::ops::Deref; - -/// A Python complex-like object. -/// -/// `ArgIntoComplex` implements `FromArgs` so that a built-in function can accept -/// any object that can be transformed into a complex. -/// -/// If the object is not a Python complex object but has a `__complex__()` -/// method, this method will first be called to convert the object into a float. -/// If `__complex__()` is not defined then it falls back to `__float__()`. If -/// `__float__()` is not defined it falls back to `__index__()`. -#[derive(Debug, PartialEq)] -#[repr(transparent)] -pub struct ArgIntoComplex { - value: Complex64, -} - -impl From<ArgIntoComplex> for Complex64 { - fn from(arg: ArgIntoComplex) -> Self { - arg.value - } -} - -impl Deref for ArgIntoComplex { - type Target = Complex64; - - fn deref(&self) -> &Self::Target { - &self.value - } -} - -impl TryFromObject for ArgIntoComplex { - // Equivalent to PyComplex_AsCComplex - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - // We do not care if it was already a complex. - let (value, _) = obj.try_complex(vm)?.ok_or_else(|| { - vm.new_type_error(format!("must be real number, not {}", obj.class().name())) - })?; - Ok(ArgIntoComplex { value }) - } -} - -/// A Python float-like object. -/// -/// `ArgIntoFloat` implements `FromArgs` so that a built-in function can accept -/// any object that can be transformed into a float. -/// -/// If the object is not a Python floating point object but has a `__float__()` -/// method, this method will first be called to convert the object into a float. -/// If `__float__()` is not defined then it falls back to `__index__()`. -#[derive(Debug, PartialEq)] -#[repr(transparent)] -pub struct ArgIntoFloat { - value: f64, -} - -impl ArgIntoFloat { - pub fn vec_into_f64(v: Vec<Self>) -> Vec<f64> { - // TODO: Vec::into_raw_parts once stabilized - let mut v = std::mem::ManuallyDrop::new(v); - let (p, l, c) = (v.as_mut_ptr(), v.len(), v.capacity()); - // SAFETY: IntoPyFloat is repr(transparent) over f64 - unsafe { Vec::from_raw_parts(p.cast(), l, c) } - } -} - -impl From<ArgIntoFloat> for f64 { - fn from(arg: ArgIntoFloat) -> Self { - arg.value - } -} - -impl Deref for ArgIntoFloat { - type Target = f64; - fn deref(&self) -> &Self::Target { - &self.value - } -} - -impl TryFromObject for ArgIntoFloat { - // Equivalent to PyFloat_AsDouble. - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let value = obj.try_float(vm)?.to_f64(); - Ok(ArgIntoFloat { value }) - } -} - -/// A Python bool-like object. -/// -/// `ArgIntoBool` implements `FromArgs` so that a built-in function can accept -/// any object that can be transformed into a boolean. -/// -/// By default an object is considered true unless its class defines either a -/// `__bool__()` method that returns False or a `__len__()` method that returns -/// zero, when called with the object. -#[derive(Debug, Default, PartialEq, Eq)] -pub struct ArgIntoBool { - value: bool, -} - -impl ArgIntoBool { - pub const TRUE: Self = Self { value: true }; - pub const FALSE: Self = Self { value: false }; -} - -impl From<ArgIntoBool> for bool { - fn from(arg: ArgIntoBool) -> Self { - arg.value - } -} - -impl Deref for ArgIntoBool { - type Target = bool; - fn deref(&self) -> &Self::Target { - &self.value - } -} - -impl TryFromObject for ArgIntoBool { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - Ok(Self { - value: obj.try_to_bool(vm)?, - }) - } -} - -// Implement ArgIndex to separate between "true" int and int generated by index -#[derive(Debug, Traverse)] -#[repr(transparent)] -pub struct ArgIndex { - value: PyIntRef, -} - -impl From<ArgIndex> for PyIntRef { - fn from(arg: ArgIndex) -> Self { - arg.value - } -} - -impl Deref for ArgIndex { - type Target = PyIntRef; - - fn deref(&self) -> &Self::Target { - &self.value - } -} - -impl TryFromObject for ArgIndex { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - Ok(Self { - value: obj.try_index(vm)?, - }) - } -} - -#[derive(Debug)] -#[repr(transparent)] -pub struct ArgPrimitiveIndex<T> { - pub value: T, -} - -impl<T> OptionalArg<ArgPrimitiveIndex<T>> { - pub fn into_primitive(self) -> OptionalArg<T> { - self.map(|x| x.value) - } -} - -impl<T> Deref for ArgPrimitiveIndex<T> { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.value - } -} - -impl<T> TryFromObject for ArgPrimitiveIndex<T> -where - T: PrimInt + for<'a> TryFrom<&'a BigInt>, -{ - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - Ok(Self { - value: obj.try_index(vm)?.try_to_primitive(vm)?, - }) - } -} - -pub type ArgSize = ArgPrimitiveIndex<isize>; - -impl From<ArgSize> for isize { - fn from(arg: ArgSize) -> Self { - arg.value - } -} diff --git a/vm/src/function/protocol.rs b/vm/src/function/protocol.rs deleted file mode 100644 index 14aa5ad8412..00000000000 --- a/vm/src/function/protocol.rs +++ /dev/null @@ -1,231 +0,0 @@ -use super::IntoFuncArgs; -use crate::{ - builtins::{iter::PySequenceIterator, PyDict, PyDictRef}, - convert::ToPyObject, - identifier, - object::{Traverse, TraverseFn}, - protocol::{PyIter, PyIterIter, PyMapping, PyMappingMethods}, - types::{AsMapping, GenericMethod}, - AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, -}; -use std::{borrow::Borrow, marker::PhantomData, ops::Deref}; - -#[derive(Clone, Traverse)] -pub struct ArgCallable { - obj: PyObjectRef, - #[pytraverse(skip)] - call: GenericMethod, -} - -impl ArgCallable { - #[inline(always)] - pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { - let args = args.into_args(vm); - (self.call)(&self.obj, args, vm) - } -} - -impl std::fmt::Debug for ArgCallable { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ArgCallable") - .field("obj", &self.obj) - .field("call", &format!("{:08x}", self.call as usize)) - .finish() - } -} - -impl Borrow<PyObject> for ArgCallable { - #[inline(always)] - fn borrow(&self) -> &PyObject { - &self.obj - } -} - -impl AsRef<PyObject> for ArgCallable { - #[inline(always)] - fn as_ref(&self) -> &PyObject { - &self.obj - } -} - -impl From<ArgCallable> for PyObjectRef { - #[inline(always)] - fn from(value: ArgCallable) -> PyObjectRef { - value.obj - } -} - -impl TryFromObject for ArgCallable { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let Some(callable) = obj.to_callable() else { - return Err( - vm.new_type_error(format!("'{}' object is not callable", obj.class().name())) - ); - }; - let call = callable.call; - Ok(ArgCallable { obj, call }) - } -} - -/// An iterable Python object. -/// -/// `ArgIterable` implements `FromArgs` so that a built-in function can accept -/// an object that is required to conform to the Python iterator protocol. -/// -/// ArgIterable can optionally perform type checking and conversions on iterated -/// objects using a generic type parameter that implements `TryFromObject`. -pub struct ArgIterable<T = PyObjectRef> { - iterable: PyObjectRef, - iterfn: Option<crate::types::IterFunc>, - _item: PhantomData<T>, -} - -unsafe impl<T: Traverse> Traverse for ArgIterable<T> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.iterable.traverse(tracer_fn) - } -} - -impl<T> ArgIterable<T> { - /// Returns an iterator over this sequence of objects. - /// - /// This operation may fail if an exception is raised while invoking the - /// `__iter__` method of the iterable object. - pub fn iter<'a>(&self, vm: &'a VirtualMachine) -> PyResult<PyIterIter<'a, T>> { - let iter = PyIter::new(match self.iterfn { - Some(f) => f(self.iterable.clone(), vm)?, - None => PySequenceIterator::new(self.iterable.clone(), vm)?.into_pyobject(vm), - }); - iter.into_iter(vm) - } -} - -impl<T> TryFromObject for ArgIterable<T> -where - T: TryFromObject, -{ - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let iterfn = { - let cls = obj.class(); - let iterfn = cls.mro_find_map(|x| x.slots.iter.load()); - if iterfn.is_none() && !cls.has_attr(identifier!(vm, __getitem__)) { - return Err(vm.new_type_error(format!("'{}' object is not iterable", cls.name()))); - } - iterfn - }; - Ok(Self { - iterable: obj, - iterfn, - _item: PhantomData, - }) - } -} - -#[derive(Debug, Clone, Traverse)] -pub struct ArgMapping { - obj: PyObjectRef, - #[pytraverse(skip)] - methods: &'static PyMappingMethods, -} - -impl ArgMapping { - #[inline] - pub fn with_methods(obj: PyObjectRef, methods: &'static PyMappingMethods) -> Self { - Self { obj, methods } - } - - #[inline(always)] - pub fn from_dict_exact(dict: PyDictRef) -> Self { - Self { - obj: dict.into(), - methods: PyDict::as_mapping(), - } - } - - #[inline(always)] - pub fn mapping(&self) -> PyMapping { - PyMapping { - obj: &self.obj, - methods: self.methods, - } - } -} - -impl Borrow<PyObject> for ArgMapping { - #[inline(always)] - fn borrow(&self) -> &PyObject { - &self.obj - } -} - -impl AsRef<PyObject> for ArgMapping { - #[inline(always)] - fn as_ref(&self) -> &PyObject { - &self.obj - } -} - -impl Deref for ArgMapping { - type Target = PyObject; - #[inline(always)] - fn deref(&self) -> &PyObject { - &self.obj - } -} - -impl From<ArgMapping> for PyObjectRef { - #[inline(always)] - fn from(value: ArgMapping) -> PyObjectRef { - value.obj - } -} - -impl ToPyObject for ArgMapping { - #[inline(always)] - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.obj - } -} - -impl TryFromObject for ArgMapping { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let mapping = PyMapping::try_protocol(&obj, vm)?; - let methods = mapping.methods; - Ok(Self { obj, methods }) - } -} - -// this is not strictly related to PySequence protocol. -#[derive(Clone)] -pub struct ArgSequence<T = PyObjectRef>(Vec<T>); - -unsafe impl<T: Traverse> Traverse for ArgSequence<T> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.0.traverse(tracer_fn); - } -} - -impl<T> ArgSequence<T> { - #[inline(always)] - pub fn into_vec(self) -> Vec<T> { - self.0 - } - #[inline(always)] - pub fn as_slice(&self) -> &[T] { - &self.0 - } -} - -impl<T> std::ops::Deref for ArgSequence<T> { - type Target = [T]; - #[inline(always)] - fn deref(&self) -> &[T] { - self.as_slice() - } -} - -impl<T: TryFromObject> TryFromObject for ArgSequence<T> { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - obj.try_to_value(vm).map(Self) - } -} diff --git a/vm/src/import.rs b/vm/src/import.rs deleted file mode 100644 index 918424296e2..00000000000 --- a/vm/src/import.rs +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Import mechanics - */ -use crate::{ - builtins::{list, traceback::PyTraceback, PyBaseExceptionRef, PyCode}, - scope::Scope, - version::get_git_revision, - vm::{thread, VirtualMachine}, - AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, -}; -use rand::Rng; - -pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult<PyObjectRef> { - flame_guard!("init importlib"); - - // importlib_bootstrap needs these and it inlines checks to sys.modules before calling into - // import machinery, so this should bring some speedup - #[cfg(all(feature = "threading", not(target_os = "wasi")))] - import_builtin(vm, "_thread")?; - import_builtin(vm, "_warnings")?; - import_builtin(vm, "_weakref")?; - - let importlib = thread::enter_vm(vm, || { - let bootstrap = import_frozen(vm, "_frozen_importlib")?; - let install = bootstrap.get_attr("_install", vm)?; - let imp = import_builtin(vm, "_imp")?; - install.call((vm.sys_module.clone(), imp), vm)?; - Ok(bootstrap) - })?; - vm.import_func = importlib.get_attr(identifier!(vm, __import__), vm)?; - Ok(importlib) -} - -pub(crate) fn init_importlib_package(vm: &VirtualMachine, importlib: PyObjectRef) -> PyResult<()> { - thread::enter_vm(vm, || { - flame_guard!("install_external"); - - // same deal as imports above - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - import_builtin(vm, crate::stdlib::os::MODULE_NAME)?; - #[cfg(windows)] - import_builtin(vm, "winreg")?; - import_builtin(vm, "_io")?; - import_builtin(vm, "marshal")?; - - let install_external = importlib.get_attr("_install_external_importers", vm)?; - install_external.call((), vm)?; - // Set pyc magic number to commit hash. Should be changed when bytecode will be more stable. - let importlib_external = vm.import("_frozen_importlib_external", None, 0)?; - let mut magic = get_git_revision().into_bytes(); - magic.truncate(4); - if magic.len() != 4 { - magic = rand::thread_rng().gen::<[u8; 4]>().to_vec(); - } - let magic: PyObjectRef = vm.ctx.new_bytes(magic).into(); - importlib_external.set_attr("MAGIC_NUMBER", magic, vm)?; - let zipimport_res = (|| -> PyResult<()> { - let zipimport = vm.import("zipimport", None, 0)?; - let zipimporter = zipimport.get_attr("zipimporter", vm)?; - let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?; - let path_hooks = list::PyListRef::try_from_object(vm, path_hooks)?; - path_hooks.insert(0, zipimporter); - Ok(()) - })(); - if zipimport_res.is_err() { - warn!("couldn't init zipimport") - } - Ok(()) - }) -} - -pub fn make_frozen(vm: &VirtualMachine, name: &str) -> PyResult<PyRef<PyCode>> { - let frozen = vm.state.frozen.get(name).ok_or_else(|| { - vm.new_import_error( - format!("No such frozen object named {name}"), - vm.ctx.new_str(name), - ) - })?; - Ok(vm.ctx.new_code(frozen.code)) -} - -pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { - let frozen = make_frozen(vm, module_name)?; - let module = import_codeobj(vm, module_name, frozen, false)?; - debug_assert!(module.get_attr(identifier!(vm, __name__), vm).is_ok()); - // TODO: give a correct origname here - module.set_attr("__origname__", vm.ctx.new_str(module_name.to_owned()), vm)?; - Ok(module) -} - -pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { - let make_module_func = vm.state.module_inits.get(module_name).ok_or_else(|| { - vm.new_import_error( - format!("Cannot import builtin module {module_name}"), - vm.ctx.new_str(module_name), - ) - })?; - let module = make_module_func(vm); - let sys_modules = vm.sys_module.get_attr("modules", vm)?; - sys_modules.set_item(module_name, module.as_object().to_owned(), vm)?; - Ok(module.into()) -} - -#[cfg(feature = "rustpython-compiler")] -pub fn import_file( - vm: &VirtualMachine, - module_name: &str, - file_path: String, - content: String, -) -> PyResult { - let code = vm - .compile_with_opts( - &content, - crate::compiler::Mode::Exec, - file_path, - vm.compile_opts(), - ) - .map_err(|err| vm.new_syntax_error(&err, Some(&content)))?; - import_codeobj(vm, module_name, code, true) -} - -pub fn import_codeobj( - vm: &VirtualMachine, - module_name: &str, - code_obj: PyRef<PyCode>, - set_file_attr: bool, -) -> PyResult { - let attrs = vm.ctx.new_dict(); - attrs.set_item( - identifier!(vm, __name__), - vm.ctx.new_str(module_name).into(), - vm, - )?; - if set_file_attr { - attrs.set_item( - identifier!(vm, __file__), - code_obj.source_path.to_object(), - vm, - )?; - } - let module = vm.new_module(module_name, attrs.clone(), None); - - // Store module in cache to prevent infinite loop with mutual importing libs: - let sys_modules = vm.sys_module.get_attr("modules", vm)?; - sys_modules.set_item(module_name, module.clone().into(), vm)?; - - // Execute main code in module: - let scope = Scope::with_builtins(None, attrs, vm); - vm.run_code_obj(code_obj, scope)?; - Ok(module.into()) -} - -fn remove_importlib_frames_inner( - vm: &VirtualMachine, - tb: Option<PyRef<PyTraceback>>, - always_trim: bool, -) -> (Option<PyRef<PyTraceback>>, bool) { - let traceback = if let Some(tb) = tb { - tb - } else { - return (None, false); - }; - - let file_name = traceback.frame.code.source_path.as_str(); - - let (inner_tb, mut now_in_importlib) = - remove_importlib_frames_inner(vm, traceback.next.lock().clone(), always_trim); - if file_name == "_frozen_importlib" || file_name == "_frozen_importlib_external" { - if traceback.frame.code.obj_name.as_str() == "_call_with_frames_removed" { - now_in_importlib = true; - } - if always_trim || now_in_importlib { - return (inner_tb, now_in_importlib); - } - } else { - now_in_importlib = false; - } - - ( - Some( - PyTraceback::new( - inner_tb, - traceback.frame.clone(), - traceback.lasti, - traceback.lineno, - ) - .into_ref(&vm.ctx), - ), - now_in_importlib, - ) -} - -// TODO: This function should do nothing on verbose mode. -// TODO: Fix this function after making PyTraceback.next mutable -pub fn remove_importlib_frames( - vm: &VirtualMachine, - exc: &PyBaseExceptionRef, -) -> PyBaseExceptionRef { - let always_trim = exc.fast_isinstance(vm.ctx.exceptions.import_error); - - if let Some(tb) = exc.traceback() { - let trimmed_tb = remove_importlib_frames_inner(vm, Some(tb), always_trim).0; - exc.set_traceback(trimmed_tb); - } - exc.clone() -} diff --git a/vm/src/intern.rs b/vm/src/intern.rs deleted file mode 100644 index b6197d61f77..00000000000 --- a/vm/src/intern.rs +++ /dev/null @@ -1,308 +0,0 @@ -use crate::{ - builtins::{PyStr, PyStrInterned, PyTypeRef}, - common::lock::PyRwLock, - convert::ToPyObject, - AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, VirtualMachine, -}; -use std::{ - borrow::{Borrow, ToOwned}, - ops::Deref, -}; - -#[derive(Debug)] -pub struct StringPool { - inner: PyRwLock<std::collections::HashSet<CachedPyStrRef, ahash::RandomState>>, -} - -impl Default for StringPool { - fn default() -> Self { - Self { - inner: PyRwLock::new(Default::default()), - } - } -} - -impl Clone for StringPool { - fn clone(&self) -> Self { - Self { - inner: PyRwLock::new(self.inner.read().clone()), - } - } -} - -impl StringPool { - #[inline] - pub unsafe fn intern<S: InternableString>( - &self, - s: S, - typ: PyTypeRef, - ) -> &'static PyStrInterned { - if let Some(found) = self.interned(s.as_ref()) { - return found; - } - - #[cold] - fn miss(zelf: &StringPool, s: PyRefExact<PyStr>) -> &'static PyStrInterned { - let cache = CachedPyStrRef { inner: s }; - let inserted = zelf.inner.write().insert(cache.clone()); - if inserted { - let interned = unsafe { cache.as_interned_str() }; - unsafe { interned.as_object().mark_intern() }; - interned - } else { - unsafe { - zelf.inner - .read() - .get(cache.as_ref()) - .expect("inserted is false") - .as_interned_str() - } - } - } - let str_ref = s.into_pyref_exact(typ); - miss(self, str_ref) - } - - #[inline] - pub fn interned<S: MaybeInternedString + ?Sized>( - &self, - s: &S, - ) -> Option<&'static PyStrInterned> { - if let Some(interned) = s.as_interned() { - return Some(interned); - } - self.inner - .read() - .get(s.as_ref()) - .map(|cached| unsafe { cached.as_interned_str() }) - } -} - -#[derive(Debug, Clone)] -#[repr(transparent)] -pub struct CachedPyStrRef { - inner: PyRefExact<PyStr>, -} - -impl std::hash::Hash for CachedPyStrRef { - fn hash<H: std::hash::Hasher>(&self, state: &mut H) { - self.inner.as_str().hash(state) - } -} - -impl PartialEq for CachedPyStrRef { - fn eq(&self, other: &Self) -> bool { - self.inner.as_str() == other.inner.as_str() - } -} - -impl Eq for CachedPyStrRef {} - -impl std::borrow::Borrow<str> for CachedPyStrRef { - #[inline] - fn borrow(&self) -> &str { - self.inner.as_str() - } -} - -impl AsRef<str> for CachedPyStrRef { - #[inline] - fn as_ref(&self) -> &str { - self.as_str() - } -} - -impl CachedPyStrRef { - /// # Safety - /// the given cache must be alive while returned reference is alive - #[inline] - unsafe fn as_interned_str(&self) -> &'static PyStrInterned { - std::mem::transmute_copy(self) - } - - #[inline] - fn as_str(&self) -> &str { - self.inner.as_str() - } -} - -pub struct PyInterned<T> -where - T: PyPayload, -{ - inner: Py<T>, -} - -impl<T: PyPayload> PyInterned<T> { - #[inline] - pub fn leak(cache: PyRef<T>) -> &'static Self { - unsafe { std::mem::transmute(cache) } - } - - #[inline] - fn as_ptr(&self) -> *const Py<T> { - self as *const _ as *const _ - } - - #[inline] - pub fn to_owned(&'static self) -> PyRef<T> { - unsafe { (*(&self as *const _ as *const PyRef<T>)).clone() } - } - - #[inline] - pub fn to_object(&'static self) -> PyObjectRef { - self.to_owned().into() - } -} - -impl<T: PyPayload> Borrow<PyObject> for PyInterned<T> { - #[inline(always)] - fn borrow(&self) -> &PyObject { - self.inner.borrow() - } -} - -// NOTE: std::hash::Hash of Self and Self::Borrowed *must* be the same -// This is ok only because PyObject doesn't implement Hash -impl<T: PyPayload> std::hash::Hash for PyInterned<T> { - #[inline(always)] - fn hash<H: std::hash::Hasher>(&self, state: &mut H) { - self.get_id().hash(state) - } -} - -impl<T: PyPayload> AsRef<Py<T>> for PyInterned<T> { - #[inline(always)] - fn as_ref(&self) -> &Py<T> { - &self.inner - } -} - -impl<T: PyPayload> Deref for PyInterned<T> { - type Target = Py<T>; - #[inline(always)] - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl<T: PyPayload> PartialEq for PyInterned<T> { - #[inline(always)] - fn eq(&self, other: &Self) -> bool { - std::ptr::eq(self, other) - } -} - -impl<T: PyPayload> Eq for PyInterned<T> {} - -impl<T: PyPayload + std::fmt::Debug> std::fmt::Debug for PyInterned<T> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Debug::fmt(&**self, f)?; - write!(f, "@{:p}", self.as_ptr()) - } -} - -impl<T: PyPayload> ToPyObject for &'static PyInterned<T> { - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.to_owned().into() - } -} - -mod sealed { - use crate::{ - builtins::PyStr, - object::{Py, PyExact, PyRefExact}, - }; - - pub trait SealedInternable {} - - impl SealedInternable for String {} - impl SealedInternable for &str {} - impl SealedInternable for PyRefExact<PyStr> {} - - pub trait SealedMaybeInterned {} - - impl SealedMaybeInterned for str {} - impl SealedMaybeInterned for PyExact<PyStr> {} - impl SealedMaybeInterned for Py<PyStr> {} -} - -/// A sealed marker trait for `DictKey` types that always become an exact instance of `str` -pub trait InternableString -where - Self: sealed::SealedInternable + ToPyObject + AsRef<Self::Interned>, - Self::Interned: MaybeInternedString, -{ - type Interned: ?Sized; - fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr>; -} - -impl InternableString for String { - type Interned = str; - #[inline] - fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr> { - let obj = PyRef::new_ref(PyStr::from(self), str_type, None); - unsafe { PyRefExact::new_unchecked(obj) } - } -} - -impl InternableString for &str { - type Interned = str; - #[inline] - fn into_pyref_exact(self, str_type: PyTypeRef) -> PyRefExact<PyStr> { - self.to_owned().into_pyref_exact(str_type) - } -} - -impl InternableString for PyRefExact<PyStr> { - type Interned = Py<PyStr>; - #[inline] - fn into_pyref_exact(self, _str_type: PyTypeRef) -> PyRefExact<PyStr> { - self - } -} - -pub trait MaybeInternedString: - AsRef<str> + crate::dictdatatype::DictKey + sealed::SealedMaybeInterned -{ - fn as_interned(&self) -> Option<&'static PyStrInterned>; -} - -impl MaybeInternedString for str { - #[inline(always)] - fn as_interned(&self) -> Option<&'static PyStrInterned> { - None - } -} - -impl MaybeInternedString for PyExact<PyStr> { - #[inline(always)] - fn as_interned(&self) -> Option<&'static PyStrInterned> { - None - } -} - -impl MaybeInternedString for Py<PyStr> { - #[inline(always)] - fn as_interned(&self) -> Option<&'static PyStrInterned> { - if self.as_object().is_interned() { - Some(unsafe { std::mem::transmute(self) }) - } else { - None - } - } -} - -impl PyObject { - #[inline] - pub fn as_interned_str(&self, vm: &crate::VirtualMachine) -> Option<&'static PyStrInterned> { - let s: Option<&Py<PyStr>> = self.downcast_ref(); - if self.is_interned() { - s.unwrap().as_interned() - } else if let Some(s) = s { - vm.ctx.interned_str(s.as_str()) - } else { - None - } - } -} diff --git a/vm/src/iter.rs b/vm/src/iter.rs deleted file mode 100644 index 497dc20adcb..00000000000 --- a/vm/src/iter.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::{types::PyComparisonOp, vm::VirtualMachine, PyObjectRef, PyResult}; -use itertools::Itertools; - -pub trait PyExactSizeIterator<'a>: ExactSizeIterator<Item = &'a PyObjectRef> + Sized { - fn eq(self, other: impl PyExactSizeIterator<'a>, vm: &VirtualMachine) -> PyResult<bool> { - let lhs = self; - let rhs = other; - if lhs.len() != rhs.len() { - return Ok(false); - } - for (a, b) in lhs.zip_eq(rhs) { - if !vm.identical_or_equal(a, b)? { - return Ok(false); - } - } - Ok(true) - } - - fn richcompare( - self, - other: impl PyExactSizeIterator<'a>, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<bool> { - let less = match op { - PyComparisonOp::Eq => return PyExactSizeIterator::eq(self, other, vm), - PyComparisonOp::Ne => return PyExactSizeIterator::eq(self, other, vm).map(|eq| !eq), - PyComparisonOp::Lt | PyComparisonOp::Le => true, - PyComparisonOp::Gt | PyComparisonOp::Ge => false, - }; - - let lhs = self; - let rhs = other; - let lhs_len = lhs.len(); - let rhs_len = rhs.len(); - for (a, b) in lhs.zip(rhs) { - if vm.bool_eq(a, b)? { - continue; - } - let ret = if less { - vm.bool_seq_lt(a, b)? - } else { - vm.bool_seq_gt(a, b)? - }; - if let Some(v) = ret { - return Ok(v); - } - } - Ok(op.eval_ord(lhs_len.cmp(&rhs_len))) - } -} - -impl<'a, T> PyExactSizeIterator<'a> for T where T: ExactSizeIterator<Item = &'a PyObjectRef> + Sized {} diff --git a/vm/src/lib.rs b/vm/src/lib.rs deleted file mode 100644 index aae29a9a28f..00000000000 --- a/vm/src/lib.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! This crate contains most python logic. -//! -//! - Compilation -//! - Bytecode -//! - Import mechanics -//! - Base objects - -// to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of -// how `mod` works, but we want this sometimes for pymodule declarations -#![allow(clippy::module_inception)] -// we want to mirror python naming conventions when defining python structs, so that does mean -// uppercase acronyms, e.g. TextIOWrapper instead of TextIoWrapper -#![allow(clippy::upper_case_acronyms)] -#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] -#![doc(html_root_url = "https://docs.rs/rustpython-vm/")] - -#[cfg(feature = "flame-it")] -#[macro_use] -extern crate flamer; - -#[macro_use] -extern crate bitflags; -#[macro_use] -extern crate log; -// extern crate env_logger; - -#[macro_use] -extern crate rustpython_derive; - -extern crate self as rustpython_vm; - -pub use rustpython_derive::*; - -//extern crate eval; use eval::eval::*; -// use py_code_object::{Function, NativeType, PyCodeObject}; - -// This is above everything else so that the defined macros are available everywhere -#[macro_use] -pub(crate) mod macros; - -mod anystr; -pub mod buffer; -pub mod builtins; -pub mod byte; -mod bytesinner; -pub mod cformat; -pub mod class; -mod codecs; -pub mod compiler; -pub mod convert; -mod coroutine; -mod dictdatatype; -#[cfg(feature = "rustpython-compiler")] -pub mod eval; -pub mod exceptions; -pub mod format; -pub mod frame; -pub mod function; -pub mod import; -mod intern; -pub mod iter; -pub mod object; -pub mod prelude; -pub mod protocol; -pub mod py_io; -#[cfg(feature = "serde")] -pub mod py_serde; -pub mod readline; -pub mod recursion; -pub mod scope; -pub mod sequence; -pub mod signal; -pub mod sliceable; -mod source; -pub mod stdlib; -pub mod suggestion; -pub mod types; -pub mod utils; -pub mod version; -pub mod vm; -pub mod warn; -#[cfg(windows)] -pub mod windows; - -pub use self::compiler::parser::source_code; -pub use self::convert::{TryFromBorrowedObject, TryFromObject}; -pub use self::object::{ - AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, - PyResult, PyWeakRef, -}; -pub use self::vm::{Context, Interpreter, Settings, VirtualMachine}; - -pub use rustpython_common as common; -pub use rustpython_compiler_core::{bytecode, frozen}; -pub use rustpython_literal as literal; - -#[doc(hidden)] -pub mod __exports { - pub use paste; -} diff --git a/vm/src/macros.rs b/vm/src/macros.rs deleted file mode 100644 index 4554a65c26e..00000000000 --- a/vm/src/macros.rs +++ /dev/null @@ -1,251 +0,0 @@ -#[macro_export] -macro_rules! extend_module { - ( $vm:expr, $module:expr, { $($name:expr => $value:expr),* $(,)? }) => {{ - $( - $vm.__module_set_attr($module, $vm.ctx.intern_str($name), $value).unwrap(); - )* - }}; -} - -#[macro_export] -macro_rules! py_class { - ( $ctx:expr, $class_name:expr, $class_base:expr, { $($name:tt => $value:expr),* $(,)* }) => { - py_class!($ctx, $class_name, $class_base, $crate::types::PyTypeFlags::BASETYPE, { $($name => $value),* }) - }; - ( $ctx:expr, $class_name:expr, $class_base:expr, $flags:expr, { $($name:tt => $value:expr),* $(,)* }) => { - { - #[allow(unused_mut)] - let mut slots = $crate::types::PyTypeSlots::heap_default(); - slots.flags = $flags; - $($crate::py_class!(@extract_slots($ctx, &mut slots, $name, $value));)* - let py_class = $ctx.new_class(None, $class_name, $class_base, slots); - $($crate::py_class!(@extract_attrs($ctx, &py_class, $name, $value));)* - py_class - } - }; - (@extract_slots($ctx:expr, $slots:expr, (slot $slot_name:ident), $value:expr)) => { - $slots.$slot_name.store(Some($value)); - }; - (@extract_slots($ctx:expr, $class:expr, $name:expr, $value:expr)) => {}; - (@extract_attrs($ctx:expr, $slots:expr, (slot $slot_name:ident), $value:expr)) => {}; - (@extract_attrs($ctx:expr, $class:expr, $name:expr, $value:expr)) => { - $class.set_attr($name, $value); - }; -} - -#[macro_export] -macro_rules! extend_class { - ( $ctx:expr, $class:expr, { $($name:expr => $value:expr),* $(,)* }) => { - $( - $class.set_attr($ctx.intern_str($name), $value.into()); - )* - }; -} - -#[macro_export] -macro_rules! py_namespace { - ( $vm:expr, { $($name:expr => $value:expr),* $(,)* }) => { - { - let namespace = $crate::builtins::PyNamespace::new_ref(&$vm.ctx); - let obj = $crate::object::AsObject::as_object(&namespace); - $( - obj.generic_setattr($vm.ctx.intern_str($name), $crate::function::PySetterValue::Assign($value.into()), $vm).unwrap(); - )* - namespace - } - } -} - -/// Macro to match on the built-in class of a Python object. -/// -/// Like `match`, `match_class!` must be exhaustive, so a default arm without -/// casting is required. -/// -/// # Examples -/// -/// ``` -/// use malachite_bigint::ToBigInt; -/// use num_traits::Zero; -/// -/// use rustpython_vm::match_class; -/// use rustpython_vm::builtins::{PyFloat, PyInt}; -/// use rustpython_vm::{PyPayload}; -/// -/// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { -/// let obj = PyInt::from(0).into_pyobject(vm); -/// assert_eq!( -/// "int", -/// match_class!(match obj { -/// PyInt => "int", -/// PyFloat => "float", -/// _ => "neither", -/// }) -/// ); -/// # }); -/// -/// ``` -/// -/// With a binding to the downcasted type: -/// -/// ``` -/// use malachite_bigint::ToBigInt; -/// use num_traits::Zero; -/// -/// use rustpython_vm::match_class; -/// use rustpython_vm::builtins::{PyFloat, PyInt}; -/// use rustpython_vm::{ PyPayload}; -/// -/// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { -/// let obj = PyInt::from(0).into_pyobject(vm); -/// -/// let int_value = match_class!(match obj { -/// i @ PyInt => i.as_bigint().clone(), -/// f @ PyFloat => f.to_f64().to_bigint().unwrap(), -/// obj => panic!("non-numeric object {:?}", obj), -/// }); -/// -/// assert!(int_value.is_zero()); -/// # }); -/// ``` -#[macro_export] -macro_rules! match_class { - // The default arm. - (match ($obj:expr) { _ => $default:expr $(,)? }) => { - $default - }; - - // The default arm, binding the original object to the specified identifier. - (match ($obj:expr) { $binding:ident => $default:expr $(,)? }) => {{ - #[allow(clippy::redundant_locals)] - let $binding = $obj; - $default - }}; - (match ($obj:expr) { ref $binding:ident => $default:expr $(,)? }) => {{ - #[allow(clippy::redundant_locals)] - let $binding = &$obj; - $default - }}; - - // An arm taken when the object is an instance of the specified built-in - // class and binding the downcasted object to the specified identifier and - // the target expression is a block. - (match ($obj:expr) { $binding:ident @ $class:ty => $expr:block $($rest:tt)* }) => { - $crate::match_class!(match ($obj) { $binding @ $class => ($expr), $($rest)* }) - }; - (match ($obj:expr) { ref $binding:ident @ $class:ty => $expr:block $($rest:tt)* }) => { - $crate::match_class!(match ($obj) { ref $binding @ $class => ($expr), $($rest)* }) - }; - - // An arm taken when the object is an instance of the specified built-in - // class and binding the downcasted object to the specified identifier. - (match ($obj:expr) { $binding:ident @ $class:ty => $expr:expr, $($rest:tt)* }) => { - match $obj.downcast::<$class>() { - Ok($binding) => $expr, - Err(_obj) => $crate::match_class!(match (_obj) { $($rest)* }), - } - }; - (match ($obj:expr) { ref $binding:ident @ $class:ty => $expr:expr, $($rest:tt)* }) => { - match $obj.payload::<$class>() { - ::std::option::Option::Some($binding) => $expr, - ::std::option::Option::None => $crate::match_class!(match ($obj) { $($rest)* }), - } - }; - - // An arm taken when the object is an instance of the specified built-in - // class and the target expression is a block. - (match ($obj:expr) { $class:ty => $expr:block $($rest:tt)* }) => { - $crate::match_class!(match ($obj) { $class => ($expr), $($rest)* }) - }; - - // An arm taken when the object is an instance of the specified built-in - // class. - (match ($obj:expr) { $class:ty => $expr:expr, $($rest:tt)* }) => { - if $obj.payload_is::<$class>() { - $expr - } else { - $crate::match_class!(match ($obj) { $($rest)* }) - } - }; - - // To allow match expressions without parens around the match target - (match $($rest:tt)*) => { - $crate::match_class!(@parse_match () ($($rest)*)) - }; - (@parse_match ($($target:tt)*) ({ $($inner:tt)* })) => { - $crate::match_class!(match ($($target)*) { $($inner)* }) - }; - (@parse_match ($($target:tt)*) ($next:tt $($rest:tt)*)) => { - $crate::match_class!(@parse_match ($($target)* $next) ($($rest)*)) - }; -} - -#[macro_export] -macro_rules! identifier( - ($as_ctx:expr, $name:ident) => { - $as_ctx.as_ref().names.$name - }; -); - -/// Super detailed logging. Might soon overflow your log buffers -/// Default, this logging is discarded, except when a the `vm-tracing-logging` -/// build feature is enabled. -macro_rules! vm_trace { - ($($arg:tt)+) => { - #[cfg(feature = "vm-tracing-logging")] - trace!($($arg)+); - } -} - -macro_rules! flame_guard { - ($name:expr) => { - #[cfg(feature = "flame-it")] - let _guard = ::flame::start_guard($name); - }; -} - -#[macro_export] -macro_rules! class_or_notimplemented { - ($t:ty, $obj:expr) => {{ - let a: &$crate::PyObject = &*$obj; - match $crate::PyObject::downcast_ref::<$t>(&a) { - Some(pyref) => pyref, - None => return Ok($crate::function::PyArithmeticValue::NotImplemented), - } - }}; -} - -#[macro_export] -macro_rules! named_function { - ($ctx:expr, $module:ident, $func:ident) => {{ - #[allow(unused_variables)] // weird lint, something to do with paste probably - let ctx: &$crate::Context = &$ctx; - $crate::__exports::paste::expr! { - ctx.new_method_def( - stringify!($func), - [<$module _ $func>], - ::rustpython_vm::function::PyMethodFlags::empty(), - ) - .to_function() - .with_module(ctx.intern_str(stringify!($module)).into()) - .into_ref(ctx) - } - }}; -} - -// can't use PyThreadingConstraint for stuff like this since it's not an auto trait, and -// therefore we can't add it ad-hoc to a trait object -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - macro_rules! py_dyn_fn { - (dyn Fn($($arg:ty),*$(,)*) -> $ret:ty) => { - dyn Fn($($arg),*) -> $ret + Send + Sync + 'static - }; - } - } else { - macro_rules! py_dyn_fn { - (dyn Fn($($arg:ty),*$(,)*) -> $ret:ty) => { - dyn Fn($($arg),*) -> $ret + 'static - }; - } - } -} diff --git a/vm/src/object/core.rs b/vm/src/object/core.rs deleted file mode 100644 index bda8aaba2f9..00000000000 --- a/vm/src/object/core.rs +++ /dev/null @@ -1,1256 +0,0 @@ -//! Essential types for object models -//! -//! +-------------------------+--------------+---------------+ -//! | Management | Typed | Untyped | -//! +-------------------------+--------------+---------------+ -//! | Interpreter-independent | Py<T> | PyObject | -//! | Reference-counted | PyRef<T> | PyObjectRef | -//! | Weak | PyWeakRef<T> | PyRef<PyWeak> | -//! +-------------------------+--------------+---------------+ -//! -//! PyRef<PyWeak> may looking like to be called as PyObjectWeak by the rule, -//! but not to do to remember it is a PyRef object. -use super::{ - ext::{AsObject, PyRefExact, PyResult}, - payload::PyObjectPayload, - PyAtomicRef, -}; -use crate::object::traverse::{Traverse, TraverseFn}; -use crate::object::traverse_object::PyObjVTable; -use crate::{ - builtins::{PyDictRef, PyType, PyTypeRef}, - common::{ - atomic::{OncePtr, PyAtomic, Radium}, - linked_list::{Link, LinkedList, Pointers}, - lock::{PyMutex, PyMutexGuard, PyRwLock}, - refcount::RefCount, - }, - vm::VirtualMachine, -}; -use itertools::Itertools; -use std::{ - any::TypeId, - borrow::Borrow, - cell::UnsafeCell, - fmt, - marker::PhantomData, - mem::ManuallyDrop, - ops::Deref, - ptr::{self, NonNull}, -}; - -// so, PyObjectRef is basically equivalent to `PyRc<PyInner<dyn PyObjectPayload>>`, except it's -// only one pointer in width rather than 2. We do that by manually creating a vtable, and putting -// a &'static reference to it inside the `PyRc` rather than adjacent to it, like trait objects do. -// This can lead to faster code since there's just less data to pass around, as well as because of -// some weird stuff with trait objects, alignment, and padding. -// -// So, every type has an alignment, which means that if you create a value of it it's location in -// memory has to be a multiple of it's alignment. e.g., a type with alignment 4 (like i32) could be -// at 0xb7befbc0, 0xb7befbc4, or 0xb7befbc8, but not 0xb7befbc2. If you have a struct and there are -// 2 fields whose sizes/alignments don't perfectly fit in with each other, e.g.: -// +-------------+-------------+---------------------------+ -// | u16 | ? | i32 | -// | 0x00 | 0x01 | 0x02 | 0x03 | 0x04 | 0x05 | 0x06 | 0x07 | -// +-------------+-------------+---------------------------+ -// There has to be padding in the space between the 2 fields. But, if that field is a trait object -// (like `dyn PyObjectPayload`) we don't *know* how much padding there is between the `payload` -// field and the previous field. So, Rust has to consult the vtable to know the exact offset of -// `payload` in `PyInner<dyn PyObjectPayload>`, which has a huge performance impact when *every -// single payload access* requires a vtable lookup. Thankfully, we're able to avoid that because of -// the way we use PyObjectRef, in that whenever we want to access the payload we (almost) always -// access it from a generic function. So, rather than doing -// -// - check vtable for payload offset -// - get offset in PyInner struct -// - call as_any() method of PyObjectPayload -// - call downcast_ref() method of Any -// we can just do -// - check vtable that typeid matches -// - pointer cast directly to *const PyInner<T> -// -// and at that point the compiler can know the offset of `payload` for us because **we've given it a -// concrete type to work with before we ever access the `payload` field** - -/// A type to just represent "we've erased the type of this object, cast it before you use it" -#[derive(Debug)] -pub(super) struct Erased; - -pub(super) unsafe fn drop_dealloc_obj<T: PyObjectPayload>(x: *mut PyObject) { - drop(Box::from_raw(x as *mut PyInner<T>)); -} -pub(super) unsafe fn debug_obj<T: PyObjectPayload>( - x: &PyObject, - f: &mut fmt::Formatter, -) -> fmt::Result { - let x = &*(x as *const PyObject as *const PyInner<T>); - fmt::Debug::fmt(x, f) -} - -/// Call `try_trace` on payload -pub(super) unsafe fn try_trace_obj<T: PyObjectPayload>(x: &PyObject, tracer_fn: &mut TraverseFn) { - let x = &*(x as *const PyObject as *const PyInner<T>); - let payload = &x.payload; - payload.try_traverse(tracer_fn) -} - -/// This is an actual python object. It consists of a `typ` which is the -/// python class, and carries some rust payload optionally. This rust -/// payload can be a rust float or rust int in case of float and int objects. -#[repr(C)] -pub(super) struct PyInner<T> { - pub(super) ref_count: RefCount, - // TODO: move typeid into vtable once TypeId::of is const - pub(super) typeid: TypeId, - pub(super) vtable: &'static PyObjVTable, - - pub(super) typ: PyAtomicRef<PyType>, // __class__ member - pub(super) dict: Option<InstanceDict>, - pub(super) weak_list: WeakRefList, - pub(super) slots: Box<[PyRwLock<Option<PyObjectRef>>]>, - - pub(super) payload: T, -} - -impl<T: fmt::Debug> fmt::Debug for PyInner<T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "[PyObject {:?}]", &self.payload) - } -} - -unsafe impl<T: PyObjectPayload> Traverse for Py<T> { - /// DO notice that call `trace` on `Py<T>` means apply `tracer_fn` on `Py<T>`'s children, - /// not like call `trace` on `PyRef<T>` which apply `tracer_fn` on `PyRef<T>` itself - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.0.traverse(tracer_fn) - } -} - -unsafe impl Traverse for PyObject { - /// DO notice that call `trace` on `PyObject` means apply `tracer_fn` on `PyObject`'s children, - /// not like call `trace` on `PyObjectRef` which apply `tracer_fn` on `PyObjectRef` itself - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.0.traverse(tracer_fn) - } -} - -pub(super) struct WeakRefList { - inner: OncePtr<PyMutex<WeakListInner>>, -} - -impl fmt::Debug for WeakRefList { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("WeakRefList").finish_non_exhaustive() - } -} - -struct WeakListInner { - list: LinkedList<WeakLink, Py<PyWeak>>, - generic_weakref: Option<NonNull<Py<PyWeak>>>, - obj: Option<NonNull<PyObject>>, - // one for each live PyWeak with a reference to this, + 1 for the referent object if it's not dead - ref_count: usize, -} - -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - unsafe impl Send for WeakListInner {} - unsafe impl Sync for WeakListInner {} - } -} - -impl WeakRefList { - pub fn new() -> Self { - WeakRefList { - inner: OncePtr::new(), - } - } - - /// returns None if there have never been any weakrefs in this list - fn try_lock(&self) -> Option<PyMutexGuard<'_, WeakListInner>> { - self.inner.get().map(|mu| unsafe { mu.as_ref().lock() }) - } - - fn add( - &self, - obj: &PyObject, - cls: PyTypeRef, - cls_is_weakref: bool, - callback: Option<PyObjectRef>, - dict: Option<PyDictRef>, - ) -> PyRef<PyWeak> { - let is_generic = cls_is_weakref && callback.is_none(); - let inner_ptr = self.inner.get_or_init(|| { - Box::new(PyMutex::new(WeakListInner { - list: LinkedList::default(), - generic_weakref: None, - obj: Some(NonNull::from(obj)), - ref_count: 1, - })) - }); - let mut inner = unsafe { inner_ptr.as_ref().lock() }; - if is_generic { - if let Some(generic_weakref) = inner.generic_weakref { - let generic_weakref = unsafe { generic_weakref.as_ref() }; - if generic_weakref.0.ref_count.get() != 0 { - return generic_weakref.to_owned(); - } - } - } - let obj = PyWeak { - pointers: Pointers::new(), - parent: inner_ptr, - callback: UnsafeCell::new(callback), - hash: Radium::new(crate::common::hash::SENTINEL), - }; - let weak = PyRef::new_ref(obj, cls, dict); - // SAFETY: we don't actually own the PyObjectWeaks inside `list`, and every time we take - // one out of the list we immediately wrap it in ManuallyDrop or forget it - inner.list.push_front(unsafe { ptr::read(&weak) }); - inner.ref_count += 1; - if is_generic { - inner.generic_weakref = Some(NonNull::from(&*weak)); - } - weak - } - - fn clear(&self) { - let to_dealloc = { - let ptr = match self.inner.get() { - Some(ptr) => ptr, - None => return, - }; - let mut inner = unsafe { ptr.as_ref().lock() }; - inner.obj = None; - // TODO: can be an arrayvec - let mut v = Vec::with_capacity(16); - loop { - let inner2 = &mut *inner; - let iter = inner2 - .list - .drain_filter(|_| true) - .filter_map(|wr| { - // we don't have actual ownership of the reference counts in the list. - // but, now we do want ownership (and so incref these *while the lock - // is held*) to avoid weird things if PyWeakObj::drop happens after - // this but before we reach the loop body below - let wr = ManuallyDrop::new(wr); - - if Some(NonNull::from(&**wr)) == inner2.generic_weakref { - inner2.generic_weakref = None - } - - // if strong_count == 0 there's some reentrancy going on. we don't - // want to call the callback - (wr.as_object().strong_count() > 0).then(|| (*wr).clone()) - }) - .take(16); - v.extend(iter); - if v.is_empty() { - break; - } - PyMutexGuard::unlocked(&mut inner, || { - for wr in v.drain(..) { - let cb = unsafe { wr.callback.get().replace(None) }; - if let Some(cb) = cb { - crate::vm::thread::with_vm(&cb, |vm| { - // TODO: handle unraisable exception - let _ = cb.call((wr.clone(),), vm); - }); - } - } - }) - } - inner.ref_count -= 1; - (inner.ref_count == 0).then_some(ptr) - }; - if let Some(ptr) = to_dealloc { - unsafe { WeakRefList::dealloc(ptr) } - } - } - - fn count(&self) -> usize { - self.try_lock() - // we assume the object is still alive (and this is only - // called from PyObject::weak_count so it should be) - .map(|inner| inner.ref_count - 1) - .unwrap_or(0) - } - - unsafe fn dealloc(ptr: NonNull<PyMutex<WeakListInner>>) { - drop(Box::from_raw(ptr.as_ptr())); - } - - fn get_weak_references(&self) -> Vec<PyRef<PyWeak>> { - let inner = match self.try_lock() { - Some(inner) => inner, - None => return vec![], - }; - let mut v = Vec::with_capacity(inner.ref_count - 1); - v.extend(inner.iter().map(|wr| wr.to_owned())); - v - } -} - -impl WeakListInner { - fn iter(&self) -> impl Iterator<Item = &Py<PyWeak>> { - self.list.iter().filter(|wr| wr.0.ref_count.get() > 0) - } -} - -impl Default for WeakRefList { - fn default() -> Self { - Self::new() - } -} - -struct WeakLink; -unsafe impl Link for WeakLink { - type Handle = PyRef<PyWeak>; - - type Target = Py<PyWeak>; - - #[inline(always)] - fn as_raw(handle: &PyRef<PyWeak>) -> NonNull<Self::Target> { - NonNull::from(&**handle) - } - - #[inline(always)] - unsafe fn from_raw(ptr: NonNull<Self::Target>) -> Self::Handle { - PyRef::from_raw(ptr.as_ptr()) - } - - #[inline(always)] - unsafe fn pointers(target: NonNull<Self::Target>) -> NonNull<Pointers<Self::Target>> { - NonNull::new_unchecked(ptr::addr_of_mut!((*target.as_ptr()).0.payload.pointers)) - } -} - -#[pyclass(name = "weakref", module = false)] -#[derive(Debug)] -pub struct PyWeak { - pointers: Pointers<Py<PyWeak>>, - parent: NonNull<PyMutex<WeakListInner>>, - // this is treated as part of parent's mutex - you must hold that lock to access it - callback: UnsafeCell<Option<PyObjectRef>>, - pub(crate) hash: PyAtomic<crate::common::hash::PyHash>, -} - -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - #[allow(clippy::non_send_fields_in_send_ty)] // false positive? - unsafe impl Send for PyWeak {} - unsafe impl Sync for PyWeak {} - } -} - -impl PyWeak { - pub(crate) fn upgrade(&self) -> Option<PyObjectRef> { - let guard = unsafe { self.parent.as_ref().lock() }; - let obj_ptr = guard.obj?; - unsafe { - if !obj_ptr.as_ref().0.ref_count.safe_inc() { - return None; - } - Some(PyObjectRef::from_raw(obj_ptr.as_ptr())) - } - } - - pub(crate) fn is_dead(&self) -> bool { - let guard = unsafe { self.parent.as_ref().lock() }; - guard.obj.is_none() - } - - fn drop_inner(&self) { - let dealloc = { - let mut guard = unsafe { self.parent.as_ref().lock() }; - let offset = memoffset::offset_of!(PyInner<PyWeak>, payload); - let pyinner = (self as *const Self as usize - offset) as *const PyInner<Self>; - let node_ptr = unsafe { NonNull::new_unchecked(pyinner as *mut Py<Self>) }; - // the list doesn't have ownership over its PyRef<PyWeak>! we're being dropped - // right now so that should be obvious!! - std::mem::forget(unsafe { guard.list.remove(node_ptr) }); - guard.ref_count -= 1; - if Some(node_ptr) == guard.generic_weakref { - guard.generic_weakref = None; - } - guard.ref_count == 0 - }; - if dealloc { - unsafe { WeakRefList::dealloc(self.parent) } - } - } -} - -impl Drop for PyWeak { - #[inline(always)] - fn drop(&mut self) { - // we do NOT have actual exclusive access! - // no clue if doing this actually reduces chance of UB - let me: &Self = self; - me.drop_inner(); - } -} - -impl Py<PyWeak> { - #[inline(always)] - pub fn upgrade(&self) -> Option<PyObjectRef> { - PyWeak::upgrade(self) - } -} - -#[derive(Debug)] -pub(super) struct InstanceDict { - pub(super) d: PyRwLock<PyDictRef>, -} - -impl From<PyDictRef> for InstanceDict { - #[inline(always)] - fn from(d: PyDictRef) -> Self { - Self::new(d) - } -} - -impl InstanceDict { - #[inline] - pub fn new(d: PyDictRef) -> Self { - Self { - d: PyRwLock::new(d), - } - } - - #[inline] - pub fn get(&self) -> PyDictRef { - self.d.read().clone() - } - - #[inline] - pub fn set(&self, d: PyDictRef) { - self.replace(d); - } - - #[inline] - pub fn replace(&self, d: PyDictRef) -> PyDictRef { - std::mem::replace(&mut self.d.write(), d) - } -} - -impl<T: PyObjectPayload> PyInner<T> { - fn new(payload: T, typ: PyTypeRef, dict: Option<PyDictRef>) -> Box<Self> { - let member_count = typ.slots.member_count; - Box::new(PyInner { - ref_count: RefCount::new(), - typeid: TypeId::of::<T>(), - vtable: PyObjVTable::of::<T>(), - typ: PyAtomicRef::from(typ), - dict: dict.map(InstanceDict::new), - weak_list: WeakRefList::new(), - payload, - slots: std::iter::repeat_with(|| PyRwLock::new(None)) - .take(member_count) - .collect_vec() - .into_boxed_slice(), - }) - } -} - -/// The `PyObjectRef` is one of the most used types. It is a reference to a -/// python object. A single python object can have multiple references, and -/// this reference counting is accounted for by this type. Use the `.clone()` -/// method to create a new reference and increment the amount of references -/// to the python object by 1. -#[repr(transparent)] -pub struct PyObjectRef { - ptr: NonNull<PyObject>, -} - -impl Clone for PyObjectRef { - #[inline(always)] - fn clone(&self) -> Self { - (**self).to_owned() - } -} - -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - unsafe impl Send for PyObjectRef {} - unsafe impl Sync for PyObjectRef {} - } -} - -#[repr(transparent)] -pub struct PyObject(PyInner<Erased>); - -impl Deref for PyObjectRef { - type Target = PyObject; - #[inline(always)] - fn deref(&self) -> &PyObject { - unsafe { self.ptr.as_ref() } - } -} - -impl ToOwned for PyObject { - type Owned = PyObjectRef; - - #[inline(always)] - fn to_owned(&self) -> Self::Owned { - self.0.ref_count.inc(); - PyObjectRef { - ptr: NonNull::from(self), - } - } -} - -impl PyObjectRef { - #[inline(always)] - pub fn into_raw(self) -> *const PyObject { - let ptr = self.as_raw(); - std::mem::forget(self); - ptr - } - - /// # Safety - /// The raw pointer must have been previously returned from a call to - /// [`PyObjectRef::into_raw`]. The user is responsible for ensuring that the inner data is not - /// dropped more than once due to mishandling the reference count by calling this function - /// too many times. - #[inline(always)] - pub unsafe fn from_raw(ptr: *const PyObject) -> Self { - Self { - ptr: NonNull::new_unchecked(ptr as *mut PyObject), - } - } - - /// Attempt to downcast this reference to a subclass. - /// - /// If the downcast fails, the original ref is returned in as `Err` so - /// another downcast can be attempted without unnecessary cloning. - #[inline(always)] - pub fn downcast<T: PyObjectPayload>(self) -> Result<PyRef<T>, Self> { - if self.payload_is::<T>() { - Ok(unsafe { self.downcast_unchecked() }) - } else { - Err(self) - } - } - - #[inline(always)] - pub fn downcast_ref<T: PyObjectPayload>(&self) -> Option<&Py<T>> { - if self.payload_is::<T>() { - // SAFETY: just checked that the payload is T, and PyRef is repr(transparent) over - // PyObjectRef - Some(unsafe { &*(self as *const PyObjectRef as *const PyRef<T>) }) - } else { - None - } - } - - /// Force to downcast this reference to a subclass. - /// - /// # Safety - /// T must be the exact payload type - #[inline(always)] - pub unsafe fn downcast_unchecked<T: PyObjectPayload>(self) -> PyRef<T> { - // PyRef::from_obj_unchecked(self) - // manual impl to avoid assertion - let obj = ManuallyDrop::new(self); - PyRef { - ptr: obj.ptr.cast(), - } - } - - /// # Safety - /// T must be the exact payload type - #[inline(always)] - pub unsafe fn downcast_unchecked_ref<T: PyObjectPayload>(&self) -> &Py<T> { - debug_assert!(self.payload_is::<T>()); - &*(self as *const PyObjectRef as *const PyRef<T>) - } - - // ideally we'd be able to define these in pyobject.rs, but method visibility rules are weird - - /// Attempt to downcast this reference to the specific class that is associated `T`. - /// - /// If the downcast fails, the original ref is returned in as `Err` so - /// another downcast can be attempted without unnecessary cloning. - #[inline] - pub fn downcast_exact<T: PyObjectPayload + crate::PyPayload>( - self, - vm: &VirtualMachine, - ) -> Result<PyRefExact<T>, Self> { - if self.class().is(T::class(&vm.ctx)) { - // TODO: is this always true? - assert!( - self.payload_is::<T>(), - "obj.__class__ is T::class() but payload is not T" - ); - // SAFETY: just asserted that payload_is::<T>() - Ok(unsafe { PyRefExact::new_unchecked(PyRef::from_obj_unchecked(self)) }) - } else { - Err(self) - } - } -} - -impl PyObject { - #[inline(always)] - fn weak_ref_list(&self) -> Option<&WeakRefList> { - Some(&self.0.weak_list) - } - - pub(crate) fn downgrade_with_weakref_typ_opt( - &self, - callback: Option<PyObjectRef>, - // a reference to weakref_type **specifically** - typ: PyTypeRef, - ) -> Option<PyRef<PyWeak>> { - self.weak_ref_list() - .map(|wrl| wrl.add(self, typ, true, callback, None)) - } - - pub(crate) fn downgrade_with_typ( - &self, - callback: Option<PyObjectRef>, - typ: PyTypeRef, - vm: &VirtualMachine, - ) -> PyResult<PyRef<PyWeak>> { - let dict = if typ - .slots - .flags - .has_feature(crate::types::PyTypeFlags::HAS_DICT) - { - Some(vm.ctx.new_dict()) - } else { - None - }; - let cls_is_weakref = typ.is(vm.ctx.types.weakref_type); - let wrl = self.weak_ref_list().ok_or_else(|| { - vm.new_type_error(format!( - "cannot create weak reference to '{}' object", - self.class().name() - )) - })?; - Ok(wrl.add(self, typ, cls_is_weakref, callback, dict)) - } - - pub fn downgrade( - &self, - callback: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyRef<PyWeak>> { - self.downgrade_with_typ(callback, vm.ctx.types.weakref_type.to_owned(), vm) - } - - pub fn get_weak_references(&self) -> Option<Vec<PyRef<PyWeak>>> { - self.weak_ref_list().map(|wrl| wrl.get_weak_references()) - } - - #[inline(always)] - pub fn payload_is<T: PyObjectPayload>(&self) -> bool { - self.0.typeid == TypeId::of::<T>() - } - - /// Force to return payload as T. - /// - /// # Safety - /// The actual payload type must be T. - #[inline(always)] - pub unsafe fn payload_unchecked<T: PyObjectPayload>(&self) -> &T { - // we cast to a PyInner<T> first because we don't know T's exact offset because of - // varying alignment, but once we get a PyInner<T> the compiler can get it for us - let inner = unsafe { &*(&self.0 as *const PyInner<Erased> as *const PyInner<T>) }; - &inner.payload - } - - #[inline(always)] - pub fn payload<T: PyObjectPayload>(&self) -> Option<&T> { - if self.payload_is::<T>() { - Some(unsafe { self.payload_unchecked() }) - } else { - None - } - } - - #[inline(always)] - pub fn class(&self) -> &Py<PyType> { - self.0.typ.deref() - } - - pub fn set_class(&self, typ: PyTypeRef, vm: &VirtualMachine) { - self.0.typ.swap_to_temporary_refs(typ, vm); - } - - #[inline(always)] - pub fn payload_if_exact<T: PyObjectPayload + crate::PyPayload>( - &self, - vm: &VirtualMachine, - ) -> Option<&T> { - if self.class().is(T::class(&vm.ctx)) { - self.payload() - } else { - None - } - } - - #[inline(always)] - fn instance_dict(&self) -> Option<&InstanceDict> { - self.0.dict.as_ref() - } - - #[inline(always)] - pub fn dict(&self) -> Option<PyDictRef> { - self.instance_dict().map(|d| d.get()) - } - - /// Set the dict field. Returns `Err(dict)` if this object does not have a dict field - /// in the first place. - pub fn set_dict(&self, dict: PyDictRef) -> Result<(), PyDictRef> { - match self.instance_dict() { - Some(d) => { - d.set(dict); - Ok(()) - } - None => Err(dict), - } - } - - #[inline(always)] - pub fn payload_if_subclass<T: crate::PyPayload>(&self, vm: &VirtualMachine) -> Option<&T> { - if self.class().fast_issubclass(T::class(&vm.ctx)) { - self.payload() - } else { - None - } - } - - #[inline(always)] - pub fn downcast_ref<T: PyObjectPayload>(&self) -> Option<&Py<T>> { - if self.payload_is::<T>() { - // SAFETY: just checked that the payload is T, and PyRef is repr(transparent) over - // PyObjectRef - Some(unsafe { self.downcast_unchecked_ref::<T>() }) - } else { - None - } - } - - #[inline(always)] - pub fn downcast_ref_if_exact<T: PyObjectPayload + crate::PyPayload>( - &self, - vm: &VirtualMachine, - ) -> Option<&Py<T>> { - self.class() - .is(T::class(&vm.ctx)) - .then(|| unsafe { self.downcast_unchecked_ref::<T>() }) - } - - /// # Safety - /// T must be the exact payload type - #[inline(always)] - pub unsafe fn downcast_unchecked_ref<T: PyObjectPayload>(&self) -> &Py<T> { - debug_assert!(self.payload_is::<T>()); - &*(self as *const PyObject as *const Py<T>) - } - - #[inline(always)] - pub fn strong_count(&self) -> usize { - self.0.ref_count.get() - } - - #[inline] - pub fn weak_count(&self) -> Option<usize> { - self.weak_ref_list().map(|wrl| wrl.count()) - } - - #[inline(always)] - pub fn as_raw(&self) -> *const PyObject { - self - } - - #[inline(always)] // the outer function is never inlined - fn drop_slow_inner(&self) -> Result<(), ()> { - // __del__ is mostly not implemented - #[inline(never)] - #[cold] - fn call_slot_del( - zelf: &PyObject, - slot_del: fn(&PyObject, &VirtualMachine) -> PyResult<()>, - ) -> Result<(), ()> { - let ret = crate::vm::thread::with_vm(zelf, |vm| { - zelf.0.ref_count.inc(); - if let Err(e) = slot_del(zelf, vm) { - let del_method = zelf.get_class_attr(identifier!(vm, __del__)).unwrap(); - vm.run_unraisable(e, None, del_method); - } - zelf.0.ref_count.dec() - }); - match ret { - // the decref right above set ref_count back to 0 - Some(true) => Ok(()), - // we've been resurrected by __del__ - Some(false) => Err(()), - None => { - warn!("couldn't run __del__ method for object"); - Ok(()) - } - } - } - - // CPython-compatible drop implementation - let del = self.class().mro_find_map(|cls| cls.slots.del.load()); - if let Some(slot_del) = del { - call_slot_del(self, slot_del)?; - } - if let Some(wrl) = self.weak_ref_list() { - wrl.clear(); - } - - Ok(()) - } - - /// Can only be called when ref_count has dropped to zero. `ptr` must be valid - #[inline(never)] - unsafe fn drop_slow(ptr: NonNull<PyObject>) { - if let Err(()) = ptr.as_ref().drop_slow_inner() { - // abort drop for whatever reason - return; - } - let drop_dealloc = ptr.as_ref().0.vtable.drop_dealloc; - // call drop only when there are no references in scope - stacked borrows stuff - drop_dealloc(ptr.as_ptr()) - } - - /// # Safety - /// This call will make the object live forever. - pub(crate) unsafe fn mark_intern(&self) { - self.0.ref_count.leak(); - } - - pub(crate) fn is_interned(&self) -> bool { - self.0.ref_count.is_leaked() - } - - pub(crate) fn get_slot(&self, offset: usize) -> Option<PyObjectRef> { - self.0.slots[offset].read().clone() - } - - pub(crate) fn set_slot(&self, offset: usize, value: Option<PyObjectRef>) { - *self.0.slots[offset].write() = value; - } -} - -impl Borrow<PyObject> for PyObjectRef { - #[inline(always)] - fn borrow(&self) -> &PyObject { - self - } -} - -impl AsRef<PyObject> for PyObjectRef { - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self - } -} - -impl AsRef<PyObject> for PyObject { - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self - } -} - -impl<'a, T: PyObjectPayload> From<&'a Py<T>> for &'a PyObject { - #[inline(always)] - fn from(py_ref: &'a Py<T>) -> Self { - py_ref.as_object() - } -} - -impl Drop for PyObjectRef { - #[inline] - fn drop(&mut self) { - if self.0.ref_count.dec() { - unsafe { PyObject::drop_slow(self.ptr) } - } - } -} - -impl fmt::Debug for PyObject { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // SAFETY: the vtable contains functions that accept payload types that always match up - // with the payload of the object - unsafe { (self.0.vtable.debug)(self, f) } - } -} - -impl fmt::Debug for PyObjectRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.as_object().fmt(f) - } -} - -#[repr(transparent)] -pub struct Py<T: PyObjectPayload>(PyInner<T>); - -impl<T: PyObjectPayload> Py<T> { - pub fn downgrade( - &self, - callback: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyWeakRef<T>> { - Ok(PyWeakRef { - weak: self.as_object().downgrade(callback, vm)?, - _marker: PhantomData, - }) - } -} - -impl<T: PyObjectPayload> ToOwned for Py<T> { - type Owned = PyRef<T>; - - #[inline(always)] - fn to_owned(&self) -> Self::Owned { - self.0.ref_count.inc(); - PyRef { - ptr: NonNull::from(self), - } - } -} - -impl<T: PyObjectPayload> Deref for Py<T> { - type Target = T; - - #[inline(always)] - fn deref(&self) -> &Self::Target { - &self.0.payload - } -} - -impl<T: PyObjectPayload> Borrow<PyObject> for Py<T> { - #[inline(always)] - fn borrow(&self) -> &PyObject { - unsafe { &*(&self.0 as *const PyInner<T> as *const PyObject) } - } -} - -impl<T> AsRef<PyObject> for Py<T> -where - T: PyObjectPayload, -{ - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self.borrow() - } -} - -impl<T: PyObjectPayload> fmt::Debug for Py<T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - (**self).fmt(f) - } -} - -/// A reference to a Python object. -/// -/// Note that a `PyRef<T>` can only deref to a shared / immutable reference. -/// It is the payload type's responsibility to handle (possibly concurrent) -/// mutability with locks or concurrent data structures if required. -/// -/// A `PyRef<T>` can be directly returned from a built-in function to handle -/// situations (such as when implementing in-place methods such as `__iadd__`) -/// where a reference to the same object must be returned. -#[repr(transparent)] -pub struct PyRef<T: PyObjectPayload> { - ptr: NonNull<Py<T>>, -} - -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - unsafe impl<T: PyObjectPayload> Send for PyRef<T> {} - unsafe impl<T: PyObjectPayload> Sync for PyRef<T> {} - } -} - -impl<T: PyObjectPayload> fmt::Debug for PyRef<T> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - (**self).fmt(f) - } -} - -impl<T: PyObjectPayload> Drop for PyRef<T> { - #[inline] - fn drop(&mut self) { - if self.0.ref_count.dec() { - unsafe { PyObject::drop_slow(self.ptr.cast::<PyObject>()) } - } - } -} - -impl<T: PyObjectPayload> Clone for PyRef<T> { - #[inline(always)] - fn clone(&self) -> Self { - (**self).to_owned() - } -} - -impl<T: PyObjectPayload> PyRef<T> { - #[inline(always)] - pub(crate) unsafe fn from_raw(raw: *const Py<T>) -> Self { - Self { - ptr: NonNull::new_unchecked(raw as *mut _), - } - } - - /// Safety: payload type of `obj` must be `T` - #[inline(always)] - unsafe fn from_obj_unchecked(obj: PyObjectRef) -> Self { - debug_assert!(obj.payload_is::<T>()); - let obj = ManuallyDrop::new(obj); - Self { - ptr: obj.ptr.cast(), - } - } - - #[inline(always)] - pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option<PyDictRef>) -> Self { - let inner = Box::into_raw(PyInner::new(payload, typ, dict)); - Self { - ptr: unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) }, - } - } - - pub fn leak(pyref: Self) -> &'static Py<T> { - let ptr = pyref.ptr; - std::mem::forget(pyref); - unsafe { &*ptr.as_ptr() } - } -} - -impl<T> Borrow<PyObject> for PyRef<T> -where - T: PyObjectPayload, -{ - #[inline(always)] - fn borrow(&self) -> &PyObject { - (**self).as_object() - } -} - -impl<T> AsRef<PyObject> for PyRef<T> -where - T: PyObjectPayload, -{ - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self.borrow() - } -} - -impl<T> From<PyRef<T>> for PyObjectRef -where - T: PyObjectPayload, -{ - #[inline] - fn from(value: PyRef<T>) -> Self { - let me = ManuallyDrop::new(value); - PyObjectRef { ptr: me.ptr.cast() } - } -} - -impl<T> Borrow<Py<T>> for PyRef<T> -where - T: PyObjectPayload, -{ - #[inline(always)] - fn borrow(&self) -> &Py<T> { - self - } -} - -impl<T> AsRef<Py<T>> for PyRef<T> -where - T: PyObjectPayload, -{ - #[inline(always)] - fn as_ref(&self) -> &Py<T> { - self - } -} - -impl<T> Deref for PyRef<T> -where - T: PyObjectPayload, -{ - type Target = Py<T>; - - #[inline(always)] - fn deref(&self) -> &Py<T> { - unsafe { self.ptr.as_ref() } - } -} - -#[repr(transparent)] -pub struct PyWeakRef<T: PyObjectPayload> { - weak: PyRef<PyWeak>, - _marker: PhantomData<T>, -} - -impl<T: PyObjectPayload> PyWeakRef<T> { - pub fn upgrade(&self) -> Option<PyRef<T>> { - self.weak - .upgrade() - // SAFETY: PyWeakRef<T> was always created from a PyRef<T>, so the object is T - .map(|obj| unsafe { PyRef::from_obj_unchecked(obj) }) - } -} - -/// Partially initialize a struct, ensuring that all fields are -/// either given values or explicitly left uninitialized -macro_rules! partially_init { - ( - $ty:path {$($init_field:ident: $init_value:expr),*$(,)?}, - Uninit { $($uninit_field:ident),*$(,)? }$(,)? - ) => {{ - // check all the fields are there but *don't* actually run it - - #[allow(clippy::diverging_sub_expression)] // FIXME: better way than using `if false`? - if false { - #[allow(invalid_value, dead_code, unreachable_code)] - let _ = {$ty { - $($init_field: $init_value,)* - $($uninit_field: unreachable!(),)* - }}; - } - let mut m = ::std::mem::MaybeUninit::<$ty>::uninit(); - #[allow(unused_unsafe)] - unsafe { - $(::std::ptr::write(&mut (*m.as_mut_ptr()).$init_field, $init_value);)* - } - m - }}; -} - -pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { - use crate::{builtins::object, class::PyClassImpl}; - use std::mem::MaybeUninit; - - // `type` inherits from `object` - // and both `type` and `object are instances of `type`. - // to produce this circular dependency, we need an unsafe block. - // (and yes, this will never get dropped. TODO?) - let (type_type, object_type) = { - // We cast between these 2 types, so make sure (at compile time) that there's no change in - // layout when we wrap PyInner<PyTypeObj> in MaybeUninit<> - static_assertions::assert_eq_size!(MaybeUninit<PyInner<PyType>>, PyInner<PyType>); - static_assertions::assert_eq_align!(MaybeUninit<PyInner<PyType>>, PyInner<PyType>); - - let type_payload = PyType { - base: None, - bases: vec![], - mro: vec![], - subclasses: PyRwLock::default(), - attributes: PyRwLock::new(Default::default()), - slots: PyType::make_slots(), - heaptype_ext: None, - }; - let object_payload = PyType { - base: None, - bases: vec![], - mro: vec![], - subclasses: PyRwLock::default(), - attributes: PyRwLock::new(Default::default()), - slots: object::PyBaseObject::make_slots(), - heaptype_ext: None, - }; - let type_type_ptr = Box::into_raw(Box::new(partially_init!( - PyInner::<PyType> { - ref_count: RefCount::new(), - typeid: TypeId::of::<PyType>(), - vtable: PyObjVTable::of::<PyType>(), - dict: None, - weak_list: WeakRefList::new(), - payload: type_payload, - slots: Box::new([]), - }, - Uninit { typ } - ))); - let object_type_ptr = Box::into_raw(Box::new(partially_init!( - PyInner::<PyType> { - ref_count: RefCount::new(), - typeid: TypeId::of::<PyType>(), - vtable: PyObjVTable::of::<PyType>(), - dict: None, - weak_list: WeakRefList::new(), - payload: object_payload, - slots: Box::new([]), - }, - Uninit { typ }, - ))); - - let object_type_ptr = object_type_ptr as *mut PyInner<PyType>; - let type_type_ptr = type_type_ptr as *mut PyInner<PyType>; - - unsafe { - (*type_type_ptr).ref_count.inc(); - let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); - ptr::write(&mut (*object_type_ptr).typ, PyAtomicRef::from(type_type)); - (*type_type_ptr).ref_count.inc(); - let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); - ptr::write(&mut (*type_type_ptr).typ, PyAtomicRef::from(type_type)); - - let object_type = PyTypeRef::from_raw(object_type_ptr.cast()); - - (*type_type_ptr).payload.mro = vec![object_type.clone()]; - (*type_type_ptr).payload.bases = vec![object_type.clone()]; - (*type_type_ptr).payload.base = Some(object_type.clone()); - - let type_type = PyTypeRef::from_raw(type_type_ptr.cast()); - - (type_type, object_type) - } - }; - - let weakref_type = PyType { - base: Some(object_type.clone()), - bases: vec![object_type.clone()], - mro: vec![object_type.clone()], - subclasses: PyRwLock::default(), - attributes: PyRwLock::default(), - slots: PyWeak::make_slots(), - heaptype_ext: None, - }; - let weakref_type = PyRef::new_ref(weakref_type, type_type.clone(), None); - - object_type.subclasses.write().push( - type_type - .as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.clone()) - .unwrap(), - ); - - object_type.subclasses.write().push( - weakref_type - .as_object() - .downgrade_with_weakref_typ_opt(None, weakref_type.clone()) - .unwrap(), - ); - - (type_type, object_type, weakref_type) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn miri_test_type_initialization() { - let _ = init_type_hierarchy(); - } - - #[test] - fn miri_test_drop() { - let ctx = crate::Context::genesis(); - let obj = ctx.new_bytes(b"dfghjkl".to_vec()); - drop(obj); - } -} diff --git a/vm/src/object/ext.rs b/vm/src/object/ext.rs deleted file mode 100644 index b8aa544fd9d..00000000000 --- a/vm/src/object/ext.rs +++ /dev/null @@ -1,592 +0,0 @@ -use super::{ - core::{Py, PyObject, PyObjectRef, PyRef}, - payload::{PyObjectPayload, PyPayload}, -}; -use crate::common::{ - atomic::{Ordering, PyAtomic, Radium}, - lock::PyRwLockReadGuard, -}; -use crate::{ - builtins::{PyBaseExceptionRef, PyStrInterned, PyType}, - convert::{IntoPyException, ToPyObject, ToPyResult, TryFromObject}, - vm::Context, - VirtualMachine, -}; -use std::{borrow::Borrow, fmt, marker::PhantomData, ops::Deref, ptr::null_mut}; - -/* Python objects and references. - -Okay, so each python object itself is an class itself (PyObject). Each -python object can have several references to it (PyObjectRef). These -references are Rc (reference counting) rust smart pointers. So when -all references are destroyed, the object itself also can be cleaned up. -Basically reference counting, but then done by rust. - -*/ - -/* - * Good reference: https://github.com/ProgVal/pythonvm-rust/blob/master/src/objects/mod.rs - */ - -/// Use this type for functions which return a python object or an exception. -/// Both the python object and the python exception are `PyObjectRef` types -/// since exceptions are also python objects. -pub type PyResult<T = PyObjectRef> = Result<T, PyBaseExceptionRef>; // A valid value, or an exception - -impl<T: fmt::Display> fmt::Display for PyRef<T> -where - T: PyObjectPayload + fmt::Display, -{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&**self, f) - } -} -impl<T: fmt::Display> fmt::Display for Py<T> -where - T: PyObjectPayload + fmt::Display, -{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&**self, f) - } -} - -#[repr(transparent)] -pub struct PyExact<T: PyObjectPayload> { - inner: Py<T>, -} - -impl<T: PyPayload> PyExact<T> { - /// # Safety - /// Given reference must be exact type of payload T - #[inline(always)] - pub unsafe fn ref_unchecked(r: &Py<T>) -> &Self { - &*(r as *const _ as *const Self) - } -} - -impl<T: PyPayload> Deref for PyExact<T> { - type Target = Py<T>; - #[inline(always)] - fn deref(&self) -> &Py<T> { - &self.inner - } -} - -impl<T: PyObjectPayload> Borrow<PyObject> for PyExact<T> { - #[inline(always)] - fn borrow(&self) -> &PyObject { - self.inner.borrow() - } -} - -impl<T: PyObjectPayload> AsRef<PyObject> for PyExact<T> { - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self.inner.as_ref() - } -} - -impl<T: PyObjectPayload> Borrow<Py<T>> for PyExact<T> { - #[inline(always)] - fn borrow(&self) -> &Py<T> { - &self.inner - } -} - -impl<T: PyObjectPayload> AsRef<Py<T>> for PyExact<T> { - #[inline(always)] - fn as_ref(&self) -> &Py<T> { - &self.inner - } -} - -impl<T: PyPayload> std::borrow::ToOwned for PyExact<T> { - type Owned = PyRefExact<T>; - fn to_owned(&self) -> Self::Owned { - let owned = self.inner.to_owned(); - unsafe { PyRefExact::new_unchecked(owned) } - } -} - -impl<T: PyPayload> PyRef<T> { - pub fn into_exact_or( - self, - ctx: &Context, - f: impl FnOnce(Self) -> PyRefExact<T>, - ) -> PyRefExact<T> { - if self.class().is(T::class(ctx)) { - unsafe { PyRefExact::new_unchecked(self) } - } else { - f(self) - } - } -} - -/// PyRef but guaranteed not to be a subtype instance -#[derive(Debug)] -#[repr(transparent)] -pub struct PyRefExact<T: PyObjectPayload> { - inner: PyRef<T>, -} - -impl<T: PyObjectPayload> PyRefExact<T> { - /// # Safety - /// obj must have exact type for the payload - pub unsafe fn new_unchecked(obj: PyRef<T>) -> Self { - Self { inner: obj } - } - - pub fn into_pyref(self) -> PyRef<T> { - self.inner - } -} - -impl<T: PyObjectPayload> Clone for PyRefExact<T> { - fn clone(&self) -> Self { - let inner = self.inner.clone(); - Self { inner } - } -} - -impl<T: PyPayload> TryFromObject for PyRefExact<T> { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let target_cls = T::class(&vm.ctx); - let cls = obj.class(); - if cls.is(target_cls) { - let obj = obj - .downcast() - .map_err(|obj| vm.new_downcast_runtime_error(target_cls, &obj))?; - Ok(Self { inner: obj }) - } else if cls.fast_issubclass(target_cls) { - Err(vm.new_type_error(format!( - "Expected an exact instance of '{}', not a subclass '{}'", - target_cls.name(), - cls.name(), - ))) - } else { - Err(vm.new_type_error(format!( - "Expected type '{}', not '{}'", - target_cls.name(), - cls.name(), - ))) - } - } -} - -impl<T: PyPayload> Deref for PyRefExact<T> { - type Target = PyExact<T>; - #[inline(always)] - fn deref(&self) -> &PyExact<T> { - unsafe { PyExact::ref_unchecked(self.inner.deref()) } - } -} - -impl<T: PyObjectPayload> Borrow<PyObject> for PyRefExact<T> { - #[inline(always)] - fn borrow(&self) -> &PyObject { - self.inner.borrow() - } -} - -impl<T: PyObjectPayload> AsRef<PyObject> for PyRefExact<T> { - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self.inner.as_ref() - } -} - -impl<T: PyObjectPayload> Borrow<Py<T>> for PyRefExact<T> { - #[inline(always)] - fn borrow(&self) -> &Py<T> { - self.inner.borrow() - } -} - -impl<T: PyObjectPayload> AsRef<Py<T>> for PyRefExact<T> { - #[inline(always)] - fn as_ref(&self) -> &Py<T> { - self.inner.as_ref() - } -} - -impl<T: PyPayload> Borrow<PyExact<T>> for PyRefExact<T> { - #[inline(always)] - fn borrow(&self) -> &PyExact<T> { - self - } -} - -impl<T: PyPayload> AsRef<PyExact<T>> for PyRefExact<T> { - #[inline(always)] - fn as_ref(&self) -> &PyExact<T> { - self - } -} - -impl<T: PyPayload> ToPyObject for PyRefExact<T> { - #[inline(always)] - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.inner.into() - } -} - -pub struct PyAtomicRef<T> { - inner: PyAtomic<*mut u8>, - _phantom: PhantomData<T>, -} - -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - unsafe impl<T: Send + PyObjectPayload> Send for PyAtomicRef<T> {} - unsafe impl<T: Sync + PyObjectPayload> Sync for PyAtomicRef<T> {} - unsafe impl<T: Send + PyObjectPayload> Send for PyAtomicRef<Option<T>> {} - unsafe impl<T: Sync + PyObjectPayload> Sync for PyAtomicRef<Option<T>> {} - unsafe impl Send for PyAtomicRef<PyObject> {} - unsafe impl Sync for PyAtomicRef<PyObject> {} - unsafe impl Send for PyAtomicRef<Option<PyObject>> {} - unsafe impl Sync for PyAtomicRef<Option<PyObject>> {} - } -} - -impl<T: fmt::Debug> fmt::Debug for PyAtomicRef<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "PyAtomicRef(")?; - unsafe { - self.inner - .load(Ordering::Relaxed) - .cast::<T>() - .as_ref() - .fmt(f) - }?; - write!(f, ")") - } -} - -impl<T: PyObjectPayload> From<PyRef<T>> for PyAtomicRef<T> { - fn from(pyref: PyRef<T>) -> Self { - let py = PyRef::leak(pyref); - Self { - inner: Radium::new(py as *const _ as *mut _), - _phantom: Default::default(), - } - } -} - -impl<T: PyObjectPayload> Deref for PyAtomicRef<T> { - type Target = Py<T>; - - fn deref(&self) -> &Self::Target { - unsafe { - self.inner - .load(Ordering::Relaxed) - .cast::<Py<T>>() - .as_ref() - .unwrap_unchecked() - } - } -} - -impl<T: PyObjectPayload> PyAtomicRef<T> { - /// # Safety - /// The caller is responsible to keep the returned PyRef alive - /// until no more reference can be used via PyAtomicRef::deref() - #[must_use] - pub unsafe fn swap(&self, pyref: PyRef<T>) -> PyRef<T> { - let py = PyRef::leak(pyref) as *const Py<T> as *mut _; - let old = Radium::swap(&self.inner, py, Ordering::AcqRel); - PyRef::from_raw(old.cast()) - } - - pub fn swap_to_temporary_refs(&self, pyref: PyRef<T>, vm: &VirtualMachine) { - let old = unsafe { self.swap(pyref) }; - if let Some(frame) = vm.current_frame() { - frame.temporary_refs.lock().push(old.into()); - } - } -} - -impl<T: PyObjectPayload> From<Option<PyRef<T>>> for PyAtomicRef<Option<T>> { - fn from(opt_ref: Option<PyRef<T>>) -> Self { - let val = opt_ref - .map(|x| PyRef::leak(x) as *const Py<T> as *mut _) - .unwrap_or(null_mut()); - Self { - inner: Radium::new(val), - _phantom: Default::default(), - } - } -} - -impl<T: PyObjectPayload> PyAtomicRef<Option<T>> { - pub fn deref(&self) -> Option<&Py<T>> { - unsafe { self.inner.load(Ordering::Relaxed).cast::<Py<T>>().as_ref() } - } - - pub fn to_owned(&self) -> Option<PyRef<T>> { - self.deref().map(|x| x.to_owned()) - } - - /// # Safety - /// The caller is responsible to keep the returned PyRef alive - /// until no more reference can be used via PyAtomicRef::deref() - #[must_use] - pub unsafe fn swap(&self, opt_ref: Option<PyRef<T>>) -> Option<PyRef<T>> { - let val = opt_ref - .map(|x| PyRef::leak(x) as *const Py<T> as *mut _) - .unwrap_or(null_mut()); - let old = Radium::swap(&self.inner, val, Ordering::AcqRel); - unsafe { old.cast::<Py<T>>().as_ref().map(|x| PyRef::from_raw(x)) } - } - - pub fn swap_to_temporary_refs(&self, opt_ref: Option<PyRef<T>>, vm: &VirtualMachine) { - let Some(old) = (unsafe { self.swap(opt_ref) }) else { - return; - }; - if let Some(frame) = vm.current_frame() { - frame.temporary_refs.lock().push(old.into()); - } - } -} - -impl From<PyObjectRef> for PyAtomicRef<PyObject> { - fn from(obj: PyObjectRef) -> Self { - let obj = obj.into_raw(); - Self { - inner: Radium::new(obj as *mut _), - _phantom: Default::default(), - } - } -} - -impl Deref for PyAtomicRef<PyObject> { - type Target = PyObject; - - fn deref(&self) -> &Self::Target { - unsafe { - self.inner - .load(Ordering::Relaxed) - .cast::<PyObject>() - .as_ref() - .unwrap_unchecked() - } - } -} - -impl PyAtomicRef<PyObject> { - /// # Safety - /// The caller is responsible to keep the returned PyRef alive - /// until no more reference can be used via PyAtomicRef::deref() - #[must_use] - pub unsafe fn swap(&self, obj: PyObjectRef) -> PyObjectRef { - let obj = obj.into_raw(); - let old = Radium::swap(&self.inner, obj as *mut _, Ordering::AcqRel); - PyObjectRef::from_raw(old as _) - } - - pub fn swap_to_temporary_refs(&self, obj: PyObjectRef, vm: &VirtualMachine) { - let old = unsafe { self.swap(obj) }; - if let Some(frame) = vm.current_frame() { - frame.temporary_refs.lock().push(old); - } - } -} - -impl From<Option<PyObjectRef>> for PyAtomicRef<Option<PyObject>> { - fn from(obj: Option<PyObjectRef>) -> Self { - let val = obj.map(|x| x.into_raw() as *mut _).unwrap_or(null_mut()); - Self { - inner: Radium::new(val), - _phantom: Default::default(), - } - } -} - -impl PyAtomicRef<Option<PyObject>> { - pub fn deref(&self) -> Option<&PyObject> { - unsafe { - self.inner - .load(Ordering::Relaxed) - .cast::<PyObject>() - .as_ref() - } - } - - pub fn to_owned(&self) -> Option<PyObjectRef> { - self.deref().map(|x| x.to_owned()) - } - - /// # Safety - /// The caller is responsible to keep the returned PyRef alive - /// until no more reference can be used via PyAtomicRef::deref() - #[must_use] - pub unsafe fn swap(&self, obj: Option<PyObjectRef>) -> Option<PyObjectRef> { - let val = obj.map(|x| x.into_raw() as *mut _).unwrap_or(null_mut()); - let old = Radium::swap(&self.inner, val, Ordering::AcqRel); - old.cast::<PyObject>() - .as_ref() - .map(|x| PyObjectRef::from_raw(x)) - } - - pub fn swap_to_temporary_refs(&self, obj: Option<PyObjectRef>, vm: &VirtualMachine) { - let Some(old) = (unsafe { self.swap(obj) }) else { - return; - }; - if let Some(frame) = vm.current_frame() { - frame.temporary_refs.lock().push(old); - } - } -} - -pub trait AsObject -where - Self: Borrow<PyObject>, -{ - #[inline(always)] - fn as_object(&self) -> &PyObject { - self.borrow() - } - - #[inline(always)] - fn get_id(&self) -> usize { - self.as_object().unique_id() - } - - #[inline(always)] - fn is<T>(&self, other: &T) -> bool - where - T: AsObject, - { - self.get_id() == other.get_id() - } - - #[inline(always)] - fn class(&self) -> &Py<PyType> { - self.as_object().class() - } - - fn get_class_attr(&self, attr_name: &'static PyStrInterned) -> Option<PyObjectRef> { - self.class().get_attr(attr_name) - } - - /// Determines if `obj` actually an instance of `cls`, this doesn't call __instancecheck__, so only - /// use this if `cls` is known to have not overridden the base __instancecheck__ magic method. - #[inline] - fn fast_isinstance(&self, cls: &Py<PyType>) -> bool { - self.class().fast_issubclass(cls) - } -} - -impl<T> AsObject for T where T: Borrow<PyObject> {} - -impl PyObject { - #[inline(always)] - fn unique_id(&self) -> usize { - self as *const PyObject as usize - } -} - -// impl<T: ?Sized> Borrow<PyObject> for PyRc<T> { -// #[inline(always)] -// fn borrow(&self) -> &PyObject { -// unsafe { &*(&**self as *const T as *const PyObject) } -// } -// } - -/// A borrow of a reference to a Python object. This avoids having clone the `PyRef<T>`/ -/// `PyObjectRef`, which isn't that cheap as that increments the atomic reference counter. -pub struct PyLease<'a, T: PyObjectPayload> { - inner: PyRwLockReadGuard<'a, PyRef<T>>, -} - -impl<'a, T: PyObjectPayload + PyPayload> PyLease<'a, T> { - #[inline(always)] - pub fn into_owned(self) -> PyRef<T> { - self.inner.clone() - } -} - -impl<'a, T: PyObjectPayload + PyPayload> Borrow<PyObject> for PyLease<'a, T> { - #[inline(always)] - fn borrow(&self) -> &PyObject { - self.inner.as_ref() - } -} - -impl<'a, T: PyObjectPayload + PyPayload> Deref for PyLease<'a, T> { - type Target = PyRef<T>; - #[inline(always)] - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl<'a, T> fmt::Display for PyLease<'a, T> -where - T: PyPayload + fmt::Display, -{ - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Display::fmt(&**self, f) - } -} - -impl<T: PyObjectPayload> ToPyObject for PyRef<T> { - #[inline(always)] - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.into() - } -} - -impl ToPyObject for PyObjectRef { - #[inline(always)] - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self - } -} - -impl ToPyObject for &PyObject { - #[inline(always)] - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.to_owned() - } -} - -// Allows a built-in function to return any built-in object payload without -// explicitly implementing `ToPyObject`. -impl<T> ToPyObject for T -where - T: PyPayload + Sized, -{ - #[inline(always)] - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - PyPayload::into_pyobject(self, vm) - } -} - -impl<T> ToPyResult for T -where - T: ToPyObject, -{ - #[inline(always)] - fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { - Ok(self.to_pyobject(vm)) - } -} - -impl<T, E> ToPyResult for Result<T, E> -where - T: ToPyObject, - E: IntoPyException, -{ - #[inline(always)] - fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { - self.map(|res| T::to_pyobject(res, vm)) - .map_err(|e| E::into_pyexception(e, vm)) - } -} - -impl IntoPyException for PyBaseExceptionRef { - #[inline(always)] - fn into_pyexception(self, _vm: &VirtualMachine) -> PyBaseExceptionRef { - self - } -} diff --git a/vm/src/object/mod.rs b/vm/src/object/mod.rs deleted file mode 100644 index 034523afe5a..00000000000 --- a/vm/src/object/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -mod core; -mod ext; -mod payload; -mod traverse; -mod traverse_object; - -pub use self::core::*; -pub use self::ext::*; -pub use self::payload::*; -pub use traverse::{MaybeTraverse, Traverse, TraverseFn}; diff --git a/vm/src/object/payload.rs b/vm/src/object/payload.rs deleted file mode 100644 index d5f3d6330b5..00000000000 --- a/vm/src/object/payload.rs +++ /dev/null @@ -1,86 +0,0 @@ -use crate::object::{MaybeTraverse, Py, PyObjectRef, PyRef, PyResult}; -use crate::{ - builtins::{PyBaseExceptionRef, PyType, PyTypeRef}, - types::PyTypeFlags, - vm::{Context, VirtualMachine}, - PyRefExact, -}; - -cfg_if::cfg_if! { - if #[cfg(feature = "threading")] { - pub trait PyThreadingConstraint: Send + Sync {} - impl<T: Send + Sync> PyThreadingConstraint for T {} - } else { - pub trait PyThreadingConstraint {} - impl<T> PyThreadingConstraint for T {} - } -} - -pub trait PyPayload: - std::fmt::Debug + MaybeTraverse + PyThreadingConstraint + Sized + 'static -{ - fn class(ctx: &Context) -> &'static Py<PyType>; - - #[inline] - fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - self.into_ref(&vm.ctx).into() - } - - #[inline] - fn _into_ref(self, cls: PyTypeRef, ctx: &Context) -> PyRef<Self> { - let dict = if cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { - Some(ctx.new_dict()) - } else { - None - }; - PyRef::new_ref(self, cls, dict) - } - - #[inline] - fn into_exact_ref(self, ctx: &Context) -> PyRefExact<Self> { - unsafe { - // Self::into_ref() always returns exact typed PyRef - PyRefExact::new_unchecked(self.into_ref(ctx)) - } - } - - #[inline] - fn into_ref(self, ctx: &Context) -> PyRef<Self> { - let cls = Self::class(ctx); - self._into_ref(cls.to_owned(), ctx) - } - - #[inline] - fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult<PyRef<Self>> { - let exact_class = Self::class(&vm.ctx); - if cls.fast_issubclass(exact_class) { - Ok(self._into_ref(cls, &vm.ctx)) - } else { - #[cold] - #[inline(never)] - fn _into_ref_with_type_error( - vm: &VirtualMachine, - cls: &PyTypeRef, - exact_class: &Py<PyType>, - ) -> PyBaseExceptionRef { - vm.new_type_error(format!( - "'{}' is not a subtype of '{}'", - &cls.name(), - exact_class.name() - )) - } - Err(_into_ref_with_type_error(vm, &cls, exact_class)) - } - } -} - -pub trait PyObjectPayload: - std::any::Any + std::fmt::Debug + MaybeTraverse + PyThreadingConstraint + 'static -{ -} - -impl<T: PyPayload + 'static> PyObjectPayload for T {} - -pub trait SlotOffset { - fn offset() -> usize; -} diff --git a/vm/src/object/traverse.rs b/vm/src/object/traverse.rs deleted file mode 100644 index 572bacafcc6..00000000000 --- a/vm/src/object/traverse.rs +++ /dev/null @@ -1,231 +0,0 @@ -use std::ptr::NonNull; - -use rustpython_common::lock::{PyMutex, PyRwLock}; - -use crate::{function::Either, object::PyObjectPayload, AsObject, PyObject, PyObjectRef, PyRef}; - -pub type TraverseFn<'a> = dyn FnMut(&PyObject) + 'a; - -/// This trait is used as a "Optional Trait"(I 'd like to use `Trace?` but it's not allowed yet) for PyObjectPayload type -/// -/// impl for PyObjectPayload, `pyclass` proc macro will handle the actual dispatch if type impl `Trace` -/// Every PyObjectPayload impl `MaybeTrace`, which may or may not be traceable -pub trait MaybeTraverse { - /// if is traceable, will be used by vtable to determine - const IS_TRACE: bool = false; - // if this type is traceable, then call with tracer_fn, default to do nothing - fn try_traverse(&self, traverse_fn: &mut TraverseFn); -} - -/// Type that need traverse it's children should impl `Traverse`(Not `MaybeTraverse`) -/// # Safety -/// impl `traverse()` with caution! Following those guideline so traverse doesn't cause memory error!: -/// - Make sure that every owned object(Every PyObjectRef/PyRef) is called with traverse_fn **at most once**. -/// If some field is not called, the worst results is just memory leak, -/// but if some field is called repeatly, panic and deadlock can happen. -/// -/// - _**DO NOT**_ clone a `PyObjectRef` or `Pyef<T>` in `traverse()` -pub unsafe trait Traverse { - /// impl `traverse()` with caution! Following those guideline so traverse doesn't cause memory error!: - /// - Make sure that every owned object(Every PyObjectRef/PyRef) is called with traverse_fn **at most once**. - /// If some field is not called, the worst results is just memory leak, - /// but if some field is called repeatly, panic and deadlock can happen. - /// - /// - _**DO NOT**_ clone a `PyObjectRef` or `Pyef<T>` in `traverse()` - fn traverse(&self, traverse_fn: &mut TraverseFn); -} - -unsafe impl Traverse for PyObjectRef { - fn traverse(&self, traverse_fn: &mut TraverseFn) { - traverse_fn(self) - } -} - -unsafe impl<T: PyObjectPayload> Traverse for PyRef<T> { - fn traverse(&self, traverse_fn: &mut TraverseFn) { - traverse_fn(self.as_object()) - } -} - -unsafe impl Traverse for () { - fn traverse(&self, _traverse_fn: &mut TraverseFn) {} -} - -unsafe impl<T: Traverse> Traverse for Option<T> { - #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { - if let Some(v) = self { - v.traverse(traverse_fn); - } - } -} - -unsafe impl<T> Traverse for [T] -where - T: Traverse, -{ - #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { - for elem in self { - elem.traverse(traverse_fn); - } - } -} - -unsafe impl<T> Traverse for Box<[T]> -where - T: Traverse, -{ - #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { - for elem in &**self { - elem.traverse(traverse_fn); - } - } -} - -unsafe impl<T> Traverse for Vec<T> -where - T: Traverse, -{ - #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { - for elem in self { - elem.traverse(traverse_fn); - } - } -} - -unsafe impl<T: Traverse> Traverse for PyRwLock<T> { - #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { - // if can't get a lock, this means something else is holding the lock, - // but since gc stopped the world, during gc the lock is always held - // so it is safe to ignore those in gc - if let Some(inner) = self.try_read_recursive() { - inner.traverse(traverse_fn) - } - } -} - -/// Safety: We can't hold lock during traverse it's child because it may cause deadlock. -/// TODO(discord9): check if this is thread-safe to do -/// (Outside of gc phase, only incref/decref will call trace, -/// and refcnt is atomic, so it should be fine?) -unsafe impl<T: Traverse> Traverse for PyMutex<T> { - #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { - let mut chs: Vec<NonNull<PyObject>> = Vec::new(); - if let Some(obj) = self.try_lock() { - obj.traverse(&mut |ch| { - chs.push(NonNull::from(ch)); - }) - } - chs.iter() - .map(|ch| { - // Safety: during gc, this should be fine, because nothing should write during gc's tracing? - let ch = unsafe { ch.as_ref() }; - traverse_fn(ch); - }) - .count(); - } -} - -macro_rules! trace_tuple { - ($(($NAME: ident, $NUM: tt)),*) => { - unsafe impl<$($NAME: Traverse),*> Traverse for ($($NAME),*) { - #[inline] - fn traverse(&self, traverse_fn: &mut TraverseFn) { - $( - self.$NUM.traverse(traverse_fn); - )* - } - } - - }; -} - -unsafe impl<A: Traverse, B: Traverse> Traverse for Either<A, B> { - #[inline] - fn traverse(&self, tracer_fn: &mut TraverseFn) { - match self { - Either::A(a) => a.traverse(tracer_fn), - Either::B(b) => b.traverse(tracer_fn), - } - } -} - -// only tuple with 12 elements or less is supported, -// because long tuple is extremly rare in almost every case -unsafe impl<A: Traverse> Traverse for (A,) { - #[inline] - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.0.traverse(tracer_fn); - } -} -trace_tuple!((A, 0), (B, 1)); -trace_tuple!((A, 0), (B, 1), (C, 2)); -trace_tuple!((A, 0), (B, 1), (C, 2), (D, 3)); -trace_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4)); -trace_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4), (F, 5)); -trace_tuple!((A, 0), (B, 1), (C, 2), (D, 3), (E, 4), (F, 5), (G, 6)); -trace_tuple!( - (A, 0), - (B, 1), - (C, 2), - (D, 3), - (E, 4), - (F, 5), - (G, 6), - (H, 7) -); -trace_tuple!( - (A, 0), - (B, 1), - (C, 2), - (D, 3), - (E, 4), - (F, 5), - (G, 6), - (H, 7), - (I, 8) -); -trace_tuple!( - (A, 0), - (B, 1), - (C, 2), - (D, 3), - (E, 4), - (F, 5), - (G, 6), - (H, 7), - (I, 8), - (J, 9) -); -trace_tuple!( - (A, 0), - (B, 1), - (C, 2), - (D, 3), - (E, 4), - (F, 5), - (G, 6), - (H, 7), - (I, 8), - (J, 9), - (K, 10) -); -trace_tuple!( - (A, 0), - (B, 1), - (C, 2), - (D, 3), - (E, 4), - (F, 5), - (G, 6), - (H, 7), - (I, 8), - (J, 9), - (K, 10), - (L, 11) -); diff --git a/vm/src/object/traverse_object.rs b/vm/src/object/traverse_object.rs deleted file mode 100644 index 38d91b2ac37..00000000000 --- a/vm/src/object/traverse_object.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::{fmt, marker::PhantomData}; - -use crate::{ - object::{ - debug_obj, drop_dealloc_obj, try_trace_obj, Erased, InstanceDict, PyInner, PyObjectPayload, - }, - PyObject, -}; - -use super::{Traverse, TraverseFn}; - -pub(in crate::object) struct PyObjVTable { - pub(in crate::object) drop_dealloc: unsafe fn(*mut PyObject), - pub(in crate::object) debug: unsafe fn(&PyObject, &mut fmt::Formatter) -> fmt::Result, - pub(in crate::object) trace: Option<unsafe fn(&PyObject, &mut TraverseFn)>, -} - -impl PyObjVTable { - pub fn of<T: PyObjectPayload>() -> &'static Self { - struct Helper<T: PyObjectPayload>(PhantomData<T>); - trait VtableHelper { - const VTABLE: PyObjVTable; - } - impl<T: PyObjectPayload> VtableHelper for Helper<T> { - const VTABLE: PyObjVTable = PyObjVTable { - drop_dealloc: drop_dealloc_obj::<T>, - debug: debug_obj::<T>, - trace: { - if T::IS_TRACE { - Some(try_trace_obj::<T>) - } else { - None - } - }, - }; - } - &Helper::<T>::VTABLE - } -} - -unsafe impl Traverse for InstanceDict { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.d.traverse(tracer_fn) - } -} - -unsafe impl Traverse for PyInner<Erased> { - /// Because PyObject hold a `PyInner<Erased>`, so we need to trace it - fn traverse(&self, tracer_fn: &mut TraverseFn) { - // 1. trace `dict` and `slots` field(`typ` can't trace for it's a AtomicRef while is leaked by design) - // 2. call vtable's trace function to trace payload - // self.typ.trace(tracer_fn); - self.dict.traverse(tracer_fn); - // weak_list keeps a *pointer* to a struct for maintaince weak ref, so no ownership, no trace - self.slots.traverse(tracer_fn); - - if let Some(f) = self.vtable.trace { - unsafe { - let zelf = &*(self as *const PyInner<Erased> as *const PyObject); - f(zelf, tracer_fn) - } - }; - } -} - -unsafe impl<T: PyObjectPayload> Traverse for PyInner<T> { - /// Type is known, so we can call `try_trace` directly instead of using erased type vtable - fn traverse(&self, tracer_fn: &mut TraverseFn) { - // 1. trace `dict` and `slots` field(`typ` can't trace for it's a AtomicRef while is leaked by design) - // 2. call corrsponding `try_trace` function to trace payload - // (No need to call vtable's trace function because we already know the type) - // self.typ.trace(tracer_fn); - self.dict.traverse(tracer_fn); - // weak_list keeps a *pointer* to a struct for maintaince weak ref, so no ownership, no trace - self.slots.traverse(tracer_fn); - T::try_traverse(&self.payload, tracer_fn); - } -} diff --git a/vm/src/prelude.rs b/vm/src/prelude.rs deleted file mode 100644 index 0bd0fe88beb..00000000000 --- a/vm/src/prelude.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub use crate::{ - object::{ - AsObject, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, - PyWeakRef, - }, - vm::{Context, Interpreter, Settings, VirtualMachine}, -}; diff --git a/vm/src/protocol/buffer.rs b/vm/src/protocol/buffer.rs deleted file mode 100644 index 820a7c7124e..00000000000 --- a/vm/src/protocol/buffer.rs +++ /dev/null @@ -1,452 +0,0 @@ -//! Buffer protocol -//! https://docs.python.org/3/c-api/buffer.html - -use crate::{ - common::{ - borrow::{BorrowedValue, BorrowedValueMut}, - lock::{MapImmutable, PyMutex, PyMutexGuard}, - }, - object::PyObjectPayload, - sliceable::SequenceIndexOp, - types::{Constructor, Unconstructible}, - Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, -}; -use itertools::Itertools; -use std::{borrow::Cow, fmt::Debug, ops::Range}; - -pub struct BufferMethods { - pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<[u8]>, - pub obj_bytes_mut: fn(&PyBuffer) -> BorrowedValueMut<[u8]>, - pub release: fn(&PyBuffer), - pub retain: fn(&PyBuffer), -} - -impl Debug for BufferMethods { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("BufferMethods") - .field("obj_bytes", &(self.obj_bytes as usize)) - .field("obj_bytes_mut", &(self.obj_bytes_mut as usize)) - .field("release", &(self.release as usize)) - .field("retain", &(self.retain as usize)) - .finish() - } -} - -#[derive(Debug, Clone, Traverse)] -pub struct PyBuffer { - pub obj: PyObjectRef, - #[pytraverse(skip)] - pub desc: BufferDescriptor, - #[pytraverse(skip)] - methods: &'static BufferMethods, -} - -impl PyBuffer { - pub fn new(obj: PyObjectRef, desc: BufferDescriptor, methods: &'static BufferMethods) -> Self { - let zelf = Self { - obj, - desc: desc.validate(), - methods, - }; - zelf.retain(); - zelf - } - - pub fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> { - self.desc - .is_contiguous() - .then(|| unsafe { self.contiguous_unchecked() }) - } - - pub fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> { - (!self.desc.readonly && self.desc.is_contiguous()) - .then(|| unsafe { self.contiguous_mut_unchecked() }) - } - - pub fn from_byte_vector(bytes: Vec<u8>, vm: &VirtualMachine) -> Self { - let bytes_len = bytes.len(); - PyBuffer::new( - PyPayload::into_pyobject(VecBuffer::from(bytes), vm), - BufferDescriptor::simple(bytes_len, true), - &VEC_BUFFER_METHODS, - ) - } - - /// # Safety - /// assume the buffer is contiguous - pub unsafe fn contiguous_unchecked(&self) -> BorrowedValue<[u8]> { - self.obj_bytes() - } - - /// # Safety - /// assume the buffer is contiguous and writable - pub unsafe fn contiguous_mut_unchecked(&self) -> BorrowedValueMut<[u8]> { - self.obj_bytes_mut() - } - - pub fn append_to(&self, buf: &mut Vec<u8>) { - if let Some(bytes) = self.as_contiguous() { - buf.extend_from_slice(&bytes); - } else { - let bytes = &*self.obj_bytes(); - self.desc.for_each_segment(true, |range| { - buf.extend_from_slice(&bytes[range.start as usize..range.end as usize]) - }); - } - } - - pub fn contiguous_or_collect<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R { - let borrowed; - let mut collected; - let v = if let Some(bytes) = self.as_contiguous() { - borrowed = bytes; - &*borrowed - } else { - collected = vec![]; - self.append_to(&mut collected); - &collected - }; - f(v) - } - - pub fn obj_as<T: PyObjectPayload>(&self) -> &Py<T> { - unsafe { self.obj.downcast_unchecked_ref() } - } - - pub fn obj_bytes(&self) -> BorrowedValue<[u8]> { - (self.methods.obj_bytes)(self) - } - - pub fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - (self.methods.obj_bytes_mut)(self) - } - - pub fn release(&self) { - (self.methods.release)(self) - } - - pub fn retain(&self) { - (self.methods.retain)(self) - } - - // drop PyBuffer without calling release - // after this function, the owner should use forget() - // or wrap PyBuffer in the ManaullyDrop to prevent drop() - pub(crate) unsafe fn drop_without_release(&mut self) { - std::ptr::drop_in_place(&mut self.obj); - std::ptr::drop_in_place(&mut self.desc); - } -} - -impl<'a> TryFromBorrowedObject<'a> for PyBuffer { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - let cls = obj.class(); - let as_buffer = cls.mro_find_map(|cls| cls.slots.as_buffer); - if let Some(f) = as_buffer { - return f(obj, vm); - } - Err(vm.new_type_error(format!( - "a bytes-like object is required, not '{}'", - cls.name() - ))) - } -} - -impl Drop for PyBuffer { - fn drop(&mut self) { - self.release(); - } -} - -#[derive(Debug, Clone)] -pub struct BufferDescriptor { - /// product(shape) * itemsize - /// bytes length, but not the length for obj_bytes() even is contiguous - pub len: usize, - pub readonly: bool, - pub itemsize: usize, - pub format: Cow<'static, str>, - /// (shape, stride, suboffset) for each dimension - pub dim_desc: Vec<(usize, isize, isize)>, - // TODO: flags -} - -impl BufferDescriptor { - pub fn simple(bytes_len: usize, readonly: bool) -> Self { - Self { - len: bytes_len, - readonly, - itemsize: 1, - format: Cow::Borrowed("B"), - dim_desc: vec![(bytes_len, 1, 0)], - } - } - - pub fn format( - bytes_len: usize, - readonly: bool, - itemsize: usize, - format: Cow<'static, str>, - ) -> Self { - Self { - len: bytes_len, - readonly, - itemsize, - format, - dim_desc: vec![(bytes_len / itemsize, itemsize as isize, 0)], - } - } - - #[cfg(debug_assertions)] - pub fn validate(self) -> Self { - assert!(self.itemsize != 0); - assert!(self.ndim() != 0); - let mut shape_product = 1; - for (shape, stride, suboffset) in self.dim_desc.iter().cloned() { - shape_product *= shape; - assert!(suboffset >= 0); - assert!(stride != 0); - } - assert!(shape_product * self.itemsize == self.len); - self - } - - #[cfg(not(debug_assertions))] - pub fn validate(self) -> Self { - self - } - - pub fn ndim(&self) -> usize { - self.dim_desc.len() - } - - pub fn is_contiguous(&self) -> bool { - if self.len == 0 { - return true; - } - let mut sd = self.itemsize; - for (shape, stride, _) in self.dim_desc.iter().cloned().rev() { - if shape > 1 && stride != sd as isize { - return false; - } - sd *= shape; - } - true - } - - /// this function do not check the bound - /// panic if indices.len() != ndim - pub fn fast_position(&self, indices: &[usize]) -> isize { - let mut pos = 0; - for (i, (_, stride, suboffset)) in indices - .iter() - .cloned() - .zip_eq(self.dim_desc.iter().cloned()) - { - pos += i as isize * stride + suboffset; - } - pos - } - - /// panic if indices.len() != ndim - pub fn position(&self, indices: &[isize], vm: &VirtualMachine) -> PyResult<isize> { - let mut pos = 0; - for (i, (shape, stride, suboffset)) in indices - .iter() - .cloned() - .zip_eq(self.dim_desc.iter().cloned()) - { - let i = i.wrapped_at(shape).ok_or_else(|| { - vm.new_index_error(format!("index out of bounds on dimension {i}")) - })?; - pos += i as isize * stride + suboffset; - } - Ok(pos) - } - - pub fn for_each_segment<F>(&self, try_conti: bool, mut f: F) - where - F: FnMut(Range<isize>), - { - if self.ndim() == 0 { - f(0..self.itemsize as isize); - return; - } - if try_conti && self.is_last_dim_contiguous() { - self._for_each_segment::<_, true>(0, 0, &mut f); - } else { - self._for_each_segment::<_, false>(0, 0, &mut f); - } - } - - fn _for_each_segment<F, const CONTI: bool>(&self, mut index: isize, dim: usize, f: &mut F) - where - F: FnMut(Range<isize>), - { - let (shape, stride, suboffset) = self.dim_desc[dim]; - if dim + 1 == self.ndim() { - if CONTI { - f(index..index + (shape * self.itemsize) as isize); - } else { - for _ in 0..shape { - let pos = index + suboffset; - f(pos..pos + self.itemsize as isize); - index += stride; - } - } - return; - } - for _ in 0..shape { - self._for_each_segment::<F, CONTI>(index + suboffset, dim + 1, f); - index += stride; - } - } - - /// zip two BufferDescriptor with the same shape - pub fn zip_eq<F>(&self, other: &Self, try_conti: bool, mut f: F) - where - F: FnMut(Range<isize>, Range<isize>) -> bool, - { - if self.ndim() == 0 { - f(0..self.itemsize as isize, 0..other.itemsize as isize); - return; - } - if try_conti && self.is_last_dim_contiguous() { - self._zip_eq::<_, true>(other, 0, 0, 0, &mut f); - } else { - self._zip_eq::<_, false>(other, 0, 0, 0, &mut f); - } - } - - fn _zip_eq<F, const CONTI: bool>( - &self, - other: &Self, - mut a_index: isize, - mut b_index: isize, - dim: usize, - f: &mut F, - ) where - F: FnMut(Range<isize>, Range<isize>) -> bool, - { - let (shape, a_stride, a_suboffset) = self.dim_desc[dim]; - let (_b_shape, b_stride, b_suboffset) = other.dim_desc[dim]; - debug_assert_eq!(shape, _b_shape); - if dim + 1 == self.ndim() { - if CONTI { - if f( - a_index..a_index + (shape * self.itemsize) as isize, - b_index..b_index + (shape * other.itemsize) as isize, - ) { - return; - } - } else { - for _ in 0..shape { - let a_pos = a_index + a_suboffset; - let b_pos = b_index + b_suboffset; - if f( - a_pos..a_pos + self.itemsize as isize, - b_pos..b_pos + other.itemsize as isize, - ) { - return; - } - a_index += a_stride; - b_index += b_stride; - } - } - return; - } - - for _ in 0..shape { - self._zip_eq::<F, CONTI>( - other, - a_index + a_suboffset, - b_index + b_suboffset, - dim + 1, - f, - ); - a_index += a_stride; - b_index += b_stride; - } - } - - fn is_last_dim_contiguous(&self) -> bool { - let (_, stride, suboffset) = self.dim_desc[self.ndim() - 1]; - suboffset == 0 && stride == self.itemsize as isize - } - - pub fn is_zero_in_shape(&self) -> bool { - for (shape, _, _) in self.dim_desc.iter().cloned() { - if shape == 0 { - return true; - } - } - false - } - - // TODO: support fortain order -} - -pub trait BufferResizeGuard { - type Resizable<'a>: 'a - where - Self: 'a; - fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>>; - fn try_resizable(&self, vm: &VirtualMachine) -> PyResult<Self::Resizable<'_>> { - self.try_resizable_opt().ok_or_else(|| { - vm.new_buffer_error("Existing exports of data: object cannot be re-sized".to_owned()) - }) - } -} - -#[pyclass(module = false, name = "vec_buffer")] -#[derive(Debug, PyPayload)] -pub struct VecBuffer { - data: PyMutex<Vec<u8>>, -} - -#[pyclass(flags(BASETYPE), with(Constructor))] -impl VecBuffer { - pub fn take(&self) -> Vec<u8> { - std::mem::take(&mut self.data.lock()) - } -} - -impl From<Vec<u8>> for VecBuffer { - fn from(data: Vec<u8>) -> Self { - Self { - data: PyMutex::new(data), - } - } -} - -impl Unconstructible for VecBuffer {} - -impl PyRef<VecBuffer> { - pub fn into_pybuffer(self, readonly: bool) -> PyBuffer { - let len = self.data.lock().len(); - PyBuffer::new( - self.into(), - BufferDescriptor::simple(len, readonly), - &VEC_BUFFER_METHODS, - ) - } - - pub fn into_pybuffer_with_descriptor(self, desc: BufferDescriptor) -> PyBuffer { - PyBuffer::new(self.into(), desc, &VEC_BUFFER_METHODS) - } -} - -static VEC_BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| { - PyMutexGuard::map_immutable(buffer.obj_as::<VecBuffer>().data.lock(), |x| x.as_slice()) - .into() - }, - obj_bytes_mut: |buffer| { - PyMutexGuard::map(buffer.obj_as::<VecBuffer>().data.lock(), |x| { - x.as_mut_slice() - }) - .into() - }, - release: |_| {}, - retain: |_| {}, -}; diff --git a/vm/src/protocol/callable.rs b/vm/src/protocol/callable.rs deleted file mode 100644 index 8a04e2021e6..00000000000 --- a/vm/src/protocol/callable.rs +++ /dev/null @@ -1,120 +0,0 @@ -use crate::{ - function::{FuncArgs, IntoFuncArgs}, - types::GenericMethod, - {AsObject, PyObject, PyResult, VirtualMachine}, -}; - -impl PyObject { - #[inline] - pub fn to_callable(&self) -> Option<PyCallable<'_>> { - PyCallable::new(self) - } - - #[inline] - pub fn is_callable(&self) -> bool { - self.to_callable().is_some() - } - - /// PyObject_Call*Arg* series - #[inline] - pub fn call(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { - let args = args.into_args(vm); - self.call_with_args(args, vm) - } - - /// PyObject_Call - pub fn call_with_args(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - vm_trace!("Invoke: {:?} {:?}", callable, args); - let Some(callable) = self.to_callable() else { - return Err( - vm.new_type_error(format!("'{}' object is not callable", self.class().name())) - ); - }; - callable.invoke(args, vm) - } -} - -pub struct PyCallable<'a> { - pub obj: &'a PyObject, - pub call: GenericMethod, -} - -impl<'a> PyCallable<'a> { - pub fn new(obj: &'a PyObject) -> Option<Self> { - let call = obj.class().mro_find_map(|cls| cls.slots.call.load())?; - Some(PyCallable { obj, call }) - } - - pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { - let args = args.into_args(vm); - vm.trace_event(TraceEvent::Call)?; - let result = (self.call)(self.obj, args, vm); - vm.trace_event(TraceEvent::Return)?; - result - } -} - -/// Trace events for sys.settrace and sys.setprofile. -enum TraceEvent { - Call, - Return, -} - -impl std::fmt::Display for TraceEvent { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - use TraceEvent::*; - match self { - Call => write!(f, "call"), - Return => write!(f, "return"), - } - } -} - -impl VirtualMachine { - /// Call registered trace function. - #[inline] - fn trace_event(&self, event: TraceEvent) -> PyResult<()> { - if self.use_tracing.get() { - self._trace_event_inner(event) - } else { - Ok(()) - } - } - fn _trace_event_inner(&self, event: TraceEvent) -> PyResult<()> { - let trace_func = self.trace_func.borrow().to_owned(); - let profile_func = self.profile_func.borrow().to_owned(); - if self.is_none(&trace_func) && self.is_none(&profile_func) { - return Ok(()); - } - - let frame_ref = self.current_frame(); - if frame_ref.is_none() { - return Ok(()); - } - - let frame = frame_ref.unwrap().as_object().to_owned(); - let event = self.ctx.new_str(event.to_string()).into(); - let args = vec![frame, event, self.ctx.none()]; - - // temporarily disable tracing, during the call to the - // tracing function itself. - if !self.is_none(&trace_func) { - self.use_tracing.set(false); - let res = trace_func.call(args.clone(), self); - self.use_tracing.set(true); - if res.is_err() { - *self.trace_func.borrow_mut() = self.ctx.none(); - } - } - - if !self.is_none(&profile_func) { - self.use_tracing.set(false); - let res = profile_func.call(args, self); - self.use_tracing.set(true); - if res.is_err() { - *self.profile_func.borrow_mut() = self.ctx.none(); - } - } - Ok(()) - } -} diff --git a/vm/src/protocol/iter.rs b/vm/src/protocol/iter.rs deleted file mode 100644 index 36952916c70..00000000000 --- a/vm/src/protocol/iter.rs +++ /dev/null @@ -1,277 +0,0 @@ -use crate::{ - builtins::iter::PySequenceIterator, - convert::{ToPyObject, ToPyResult}, - object::{Traverse, TraverseFn}, - AsObject, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, -}; -use std::borrow::Borrow; -use std::ops::Deref; - -/// Iterator Protocol -// https://docs.python.org/3/c-api/iter.html -#[derive(Debug, Clone)] -#[repr(transparent)] -pub struct PyIter<O = PyObjectRef>(O) -where - O: Borrow<PyObject>; - -unsafe impl<O: Borrow<PyObject>> Traverse for PyIter<O> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.0.borrow().traverse(tracer_fn); - } -} - -impl PyIter<PyObjectRef> { - pub fn check(obj: &PyObject) -> bool { - obj.class() - .mro_find_map(|x| x.slots.iternext.load()) - .is_some() - } -} - -impl<O> PyIter<O> -where - O: Borrow<PyObject>, -{ - pub fn new(obj: O) -> Self { - Self(obj) - } - pub fn next(&self, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let iternext = { - self.0 - .borrow() - .class() - .mro_find_map(|x| x.slots.iternext.load()) - .ok_or_else(|| { - vm.new_type_error(format!( - "'{}' object is not an iterator", - self.0.borrow().class().name() - )) - })? - }; - iternext(self.0.borrow(), vm) - } - - pub fn iter<'a, 'b, U>( - &'b self, - vm: &'a VirtualMachine, - ) -> PyResult<PyIterIter<'a, U, &'b PyObject>> { - let length_hint = vm.length_hint_opt(self.as_ref().to_owned())?; - Ok(PyIterIter::new(vm, self.0.borrow(), length_hint)) - } - - pub fn iter_without_hint<'a, 'b, U>( - &'b self, - vm: &'a VirtualMachine, - ) -> PyResult<PyIterIter<'a, U, &'b PyObject>> { - Ok(PyIterIter::new(vm, self.0.borrow(), None)) - } -} - -impl PyIter<PyObjectRef> { - /// Returns an iterator over this sequence of objects. - pub fn into_iter<U>(self, vm: &VirtualMachine) -> PyResult<PyIterIter<U, PyObjectRef>> { - let length_hint = vm.length_hint_opt(self.as_object().to_owned())?; - Ok(PyIterIter::new(vm, self.0, length_hint)) - } -} - -impl From<PyIter<PyObjectRef>> for PyObjectRef { - fn from(value: PyIter<PyObjectRef>) -> PyObjectRef { - value.0 - } -} - -impl<O> Borrow<PyObject> for PyIter<O> -where - O: Borrow<PyObject>, -{ - #[inline(always)] - fn borrow(&self) -> &PyObject { - self.0.borrow() - } -} - -impl<O> AsRef<PyObject> for PyIter<O> -where - O: Borrow<PyObject>, -{ - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self.0.borrow() - } -} - -impl<O> Deref for PyIter<O> -where - O: Borrow<PyObject>, -{ - type Target = PyObject; - #[inline(always)] - fn deref(&self) -> &Self::Target { - self.0.borrow() - } -} - -impl ToPyObject for PyIter<PyObjectRef> { - #[inline(always)] - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - self.into() - } -} - -impl TryFromObject for PyIter<PyObjectRef> { - // This helper function is called at multiple places. First, it is called - // in the vm when a for loop is entered. Next, it is used when the builtin - // function 'iter' is called. - fn try_from_object(vm: &VirtualMachine, iter_target: PyObjectRef) -> PyResult<Self> { - let getiter = { - let cls = iter_target.class(); - cls.mro_find_map(|x| x.slots.iter.load()) - }; - if let Some(getiter) = getiter { - let iter = getiter(iter_target, vm)?; - if PyIter::check(&iter) { - Ok(Self(iter)) - } else { - Err(vm.new_type_error(format!( - "iter() returned non-iterator of type '{}'", - iter.class().name() - ))) - } - } else if let Ok(seq_iter) = PySequenceIterator::new(iter_target.clone(), vm) { - Ok(Self(seq_iter.into_pyobject(vm))) - } else { - Err(vm.new_type_error(format!( - "'{}' object is not iterable", - iter_target.class().name() - ))) - } - } -} - -#[derive(result_like::ResultLike)] -pub enum PyIterReturn<T = PyObjectRef> { - Return(T), - StopIteration(Option<PyObjectRef>), -} - -unsafe impl<T: Traverse> Traverse for PyIterReturn<T> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - match self { - PyIterReturn::Return(r) => r.traverse(tracer_fn), - PyIterReturn::StopIteration(Some(obj)) => obj.traverse(tracer_fn), - _ => (), - } - } -} - -impl PyIterReturn { - pub fn from_pyresult(result: PyResult, vm: &VirtualMachine) -> PyResult<Self> { - match result { - Ok(obj) => Ok(Self::Return(obj)), - Err(err) if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) => { - let args = err.get_arg(0); - Ok(Self::StopIteration(args)) - } - Err(err) => Err(err), - } - } - - pub fn from_getitem_result(result: PyResult, vm: &VirtualMachine) -> PyResult<Self> { - match result { - Ok(obj) => Ok(Self::Return(obj)), - Err(err) if err.fast_isinstance(vm.ctx.exceptions.index_error) => { - Ok(Self::StopIteration(None)) - } - Err(err) if err.fast_isinstance(vm.ctx.exceptions.stop_iteration) => { - let args = err.get_arg(0); - Ok(Self::StopIteration(args)) - } - Err(err) => Err(err), - } - } - - pub fn into_async_pyresult(self, vm: &VirtualMachine) -> PyResult { - match self { - Self::Return(obj) => Ok(obj), - Self::StopIteration(v) => Err({ - let args = if let Some(v) = v { vec![v] } else { Vec::new() }; - vm.new_exception(vm.ctx.exceptions.stop_async_iteration.to_owned(), args) - }), - } - } -} - -impl ToPyResult for PyIterReturn { - fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { - match self { - Self::Return(obj) => Ok(obj), - Self::StopIteration(v) => Err(vm.new_stop_iteration(v)), - } - } -} - -impl ToPyResult for PyResult<PyIterReturn> { - fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { - self?.to_pyresult(vm) - } -} - -// Typical rust `Iter` object for `PyIter` -pub struct PyIterIter<'a, T, O = PyObjectRef> -where - O: Borrow<PyObject>, -{ - vm: &'a VirtualMachine, - obj: O, // creating PyIter<O> is zero-cost - length_hint: Option<usize>, - _phantom: std::marker::PhantomData<T>, -} - -unsafe impl<'a, T, O> Traverse for PyIterIter<'a, T, O> -where - O: Traverse + Borrow<PyObject>, -{ - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.obj.traverse(tracer_fn) - } -} - -impl<'a, T, O> PyIterIter<'a, T, O> -where - O: Borrow<PyObject>, -{ - pub fn new(vm: &'a VirtualMachine, obj: O, length_hint: Option<usize>) -> Self { - Self { - vm, - obj, - length_hint, - _phantom: std::marker::PhantomData, - } - } -} - -impl<'a, T, O> Iterator for PyIterIter<'a, T, O> -where - T: TryFromObject, - O: Borrow<PyObject>, -{ - type Item = PyResult<T>; - - fn next(&mut self) -> Option<Self::Item> { - let imp = |next: PyResult<PyIterReturn>| -> PyResult<Option<T>> { - let Some(obj) = next?.into_result().ok() else { - return Ok(None); - }; - Ok(Some(T::try_from_object(self.vm, obj)?)) - }; - let next = PyIter::new(self.obj.borrow()).next(self.vm); - imp(next).transpose() - } - - #[inline] - fn size_hint(&self) -> (usize, Option<usize>) { - (self.length_hint.unwrap_or(0), self.length_hint) - } -} diff --git a/vm/src/protocol/mapping.rs b/vm/src/protocol/mapping.rs deleted file mode 100644 index 11cd03d445f..00000000000 --- a/vm/src/protocol/mapping.rs +++ /dev/null @@ -1,201 +0,0 @@ -use crate::{ - builtins::{ - dict::{PyDictItems, PyDictKeys, PyDictValues}, - type_::PointerSlot, - PyDict, PyStrInterned, - }, - convert::ToPyResult, - object::{Traverse, TraverseFn}, - AsObject, PyObject, PyObjectRef, PyResult, VirtualMachine, -}; -use crossbeam_utils::atomic::AtomicCell; - -// Mapping protocol -// https://docs.python.org/3/c-api/mapping.html - -impl PyObject { - pub fn to_mapping(&self) -> PyMapping<'_> { - PyMapping::from(self) - } -} - -#[allow(clippy::type_complexity)] -#[derive(Default)] -pub struct PyMappingMethods { - pub length: AtomicCell<Option<fn(PyMapping, &VirtualMachine) -> PyResult<usize>>>, - pub subscript: AtomicCell<Option<fn(PyMapping, &PyObject, &VirtualMachine) -> PyResult>>, - pub ass_subscript: AtomicCell< - Option<fn(PyMapping, &PyObject, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>>, - >, -} - -impl std::fmt::Debug for PyMappingMethods { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "mapping methods") - } -} - -impl PyMappingMethods { - fn check(&self) -> bool { - self.subscript.load().is_some() - } - - #[allow(clippy::declare_interior_mutable_const)] - pub const NOT_IMPLEMENTED: PyMappingMethods = PyMappingMethods { - length: AtomicCell::new(None), - subscript: AtomicCell::new(None), - ass_subscript: AtomicCell::new(None), - }; -} - -impl<'a> From<&'a PyObject> for PyMapping<'a> { - fn from(obj: &'a PyObject) -> Self { - static GLOBAL_NOT_IMPLEMENTED: PyMappingMethods = PyMappingMethods::NOT_IMPLEMENTED; - let methods = Self::find_methods(obj) - .map_or(&GLOBAL_NOT_IMPLEMENTED, |x| unsafe { x.borrow_static() }); - Self { obj, methods } - } -} - -#[derive(Copy, Clone)] -pub struct PyMapping<'a> { - pub obj: &'a PyObject, - pub methods: &'static PyMappingMethods, -} - -unsafe impl Traverse for PyMapping<'_> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.obj.traverse(tracer_fn) - } -} - -impl AsRef<PyObject> for PyMapping<'_> { - #[inline(always)] - fn as_ref(&self) -> &PyObject { - self.obj - } -} - -impl<'a> PyMapping<'a> { - pub fn try_protocol(obj: &'a PyObject, vm: &VirtualMachine) -> PyResult<Self> { - if let Some(methods) = Self::find_methods(obj) { - if methods.as_ref().check() { - return Ok(Self { - obj, - methods: unsafe { methods.borrow_static() }, - }); - } - } - - Err(vm.new_type_error(format!("{} is not a mapping object", obj.class()))) - } -} - -impl PyMapping<'_> { - // PyMapping::Check - #[inline] - pub fn check(obj: &PyObject) -> bool { - Self::find_methods(obj).map_or(false, |x| x.as_ref().check()) - } - - pub fn find_methods(obj: &PyObject) -> Option<PointerSlot<PyMappingMethods>> { - obj.class().mro_find_map(|cls| cls.slots.as_mapping.load()) - } - - pub fn length_opt(self, vm: &VirtualMachine) -> Option<PyResult<usize>> { - self.methods.length.load().map(|f| f(self, vm)) - } - - pub fn length(self, vm: &VirtualMachine) -> PyResult<usize> { - self.length_opt(vm).ok_or_else(|| { - vm.new_type_error(format!( - "object of type '{}' has no len() or not a mapping", - self.obj.class() - )) - })? - } - - pub fn subscript(self, needle: &impl AsObject, vm: &VirtualMachine) -> PyResult { - self._subscript(needle.as_object(), vm) - } - - pub fn ass_subscript( - self, - needle: &impl AsObject, - value: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - self._ass_subscript(needle.as_object(), value, vm) - } - - fn _subscript(self, needle: &PyObject, vm: &VirtualMachine) -> PyResult { - let f = - self.methods.subscript.load().ok_or_else(|| { - vm.new_type_error(format!("{} is not a mapping", self.obj.class())) - })?; - f(self, needle, vm) - } - - fn _ass_subscript( - self, - needle: &PyObject, - value: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let f = self.methods.ass_subscript.load().ok_or_else(|| { - vm.new_type_error(format!( - "'{}' object does not support item assignment", - self.obj.class() - )) - })?; - f(self, needle, value, vm) - } - - pub fn keys(self, vm: &VirtualMachine) -> PyResult { - if let Some(dict) = self.obj.downcast_ref_if_exact::<PyDict>(vm) { - PyDictKeys::new(dict.to_owned()).to_pyresult(vm) - } else { - self.method_output_as_list(identifier!(vm, keys), vm) - } - } - - pub fn values(self, vm: &VirtualMachine) -> PyResult { - if let Some(dict) = self.obj.downcast_ref_if_exact::<PyDict>(vm) { - PyDictValues::new(dict.to_owned()).to_pyresult(vm) - } else { - self.method_output_as_list(identifier!(vm, values), vm) - } - } - - pub fn items(self, vm: &VirtualMachine) -> PyResult { - if let Some(dict) = self.obj.downcast_ref_if_exact::<PyDict>(vm) { - PyDictItems::new(dict.to_owned()).to_pyresult(vm) - } else { - self.method_output_as_list(identifier!(vm, items), vm) - } - } - - fn method_output_as_list( - self, - method_name: &'static PyStrInterned, - vm: &VirtualMachine, - ) -> PyResult { - let meth_output = vm.call_method(self.obj, method_name.as_str(), ())?; - if meth_output.is(vm.ctx.types.list_type) { - return Ok(meth_output); - } - - let iter = meth_output.get_iter(vm).map_err(|_| { - vm.new_type_error(format!( - "{}.{}() returned a non-iterable (type {})", - self.obj.class(), - method_name.as_str(), - meth_output.class() - )) - })?; - - // TODO - // PySequence::from(&iter).list(vm).map(|x| x.into()) - vm.ctx.new_list(iter.try_to_value(vm)?).to_pyresult(vm) - } -} diff --git a/vm/src/protocol/mod.rs b/vm/src/protocol/mod.rs deleted file mode 100644 index 989cca73d81..00000000000 --- a/vm/src/protocol/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod buffer; -mod callable; -mod iter; -mod mapping; -mod number; -mod object; -mod sequence; - -pub use buffer::{BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, VecBuffer}; -pub use callable::PyCallable; -pub use iter::{PyIter, PyIterIter, PyIterReturn}; -pub use mapping::{PyMapping, PyMappingMethods}; -pub use number::{ - PyNumber, PyNumberBinaryFunc, PyNumberBinaryOp, PyNumberMethods, PyNumberSlots, - PyNumberTernaryOp, PyNumberUnaryFunc, -}; -pub use sequence::{PySequence, PySequenceMethods}; diff --git a/vm/src/protocol/number.rs b/vm/src/protocol/number.rs deleted file mode 100644 index 0dbb5ca8de0..00000000000 --- a/vm/src/protocol/number.rs +++ /dev/null @@ -1,566 +0,0 @@ -use std::ops::Deref; - -use crossbeam_utils::atomic::AtomicCell; - -use crate::{ - builtins::{int, PyByteArray, PyBytes, PyComplex, PyFloat, PyInt, PyIntRef, PyStr}, - common::int::bytes_to_int, - function::ArgBytesLike, - object::{Traverse, TraverseFn}, - stdlib::warnings, - AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, - VirtualMachine, -}; - -pub type PyNumberUnaryFunc<R = PyObjectRef> = fn(PyNumber, &VirtualMachine) -> PyResult<R>; -pub type PyNumberBinaryFunc = fn(&PyObject, &PyObject, &VirtualMachine) -> PyResult; -pub type PyNumberTernaryFunc = fn(&PyObject, &PyObject, &PyObject, &VirtualMachine) -> PyResult; - -impl PyObject { - #[inline] - pub fn to_number(&self) -> PyNumber { - PyNumber(self) - } - - pub fn try_index_opt(&self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { - if let Some(i) = self.downcast_ref_if_exact::<PyInt>(vm) { - Some(Ok(i.to_owned())) - } else if let Some(i) = self.payload::<PyInt>() { - Some(Ok(vm.ctx.new_bigint(i.as_bigint()))) - } else { - self.to_number().index(vm) - } - } - - #[inline] - pub fn try_index(&self, vm: &VirtualMachine) -> PyResult<PyIntRef> { - self.try_index_opt(vm).transpose()?.ok_or_else(|| { - vm.new_type_error(format!( - "'{}' object cannot be interpreted as an integer", - self.class() - )) - }) - } - - pub fn try_int(&self, vm: &VirtualMachine) -> PyResult<PyIntRef> { - fn try_convert(obj: &PyObject, lit: &[u8], vm: &VirtualMachine) -> PyResult<PyIntRef> { - let base = 10; - let i = bytes_to_int(lit, base).ok_or_else(|| { - let repr = match obj.repr(vm) { - Ok(repr) => repr, - Err(err) => return err, - }; - vm.new_value_error(format!( - "invalid literal for int() with base {}: {}", - base, repr, - )) - })?; - Ok(PyInt::from(i).into_ref(&vm.ctx)) - } - - if let Some(i) = self.downcast_ref_if_exact::<PyInt>(vm) { - Ok(i.to_owned()) - } else if let Some(i) = self.to_number().int(vm).or_else(|| self.try_index_opt(vm)) { - i - } else if let Ok(Some(f)) = vm.get_special_method(self, identifier!(vm, __trunc__)) { - // TODO: Deprecate in 3.11 - // warnings::warn( - // vm.ctx.exceptions.deprecation_warning.clone(), - // "The delegation of int() to __trunc__ is deprecated.".to_owned(), - // 1, - // vm, - // )?; - let ret = f.invoke((), vm)?; - ret.try_index(vm).map_err(|_| { - vm.new_type_error(format!( - "__trunc__ returned non-Integral (type {})", - ret.class() - )) - }) - } else if let Some(s) = self.payload::<PyStr>() { - try_convert(self, s.as_str().as_bytes(), vm) - } else if let Some(bytes) = self.payload::<PyBytes>() { - try_convert(self, bytes, vm) - } else if let Some(bytearray) = self.payload::<PyByteArray>() { - try_convert(self, &bytearray.borrow_buf(), vm) - } else if let Ok(buffer) = ArgBytesLike::try_from_borrowed_object(vm, self) { - // TODO: replace to PyBuffer - try_convert(self, &buffer.borrow_buf(), vm) - } else { - Err(vm.new_type_error(format!( - "int() argument must be a string, a bytes-like object or a real number, not '{}'", - self.class() - ))) - } - } - - pub fn try_float_opt(&self, vm: &VirtualMachine) -> Option<PyResult<PyRef<PyFloat>>> { - if let Some(float) = self.downcast_ref_if_exact::<PyFloat>(vm) { - Some(Ok(float.to_owned())) - } else if let Some(f) = self.to_number().float(vm) { - Some(f) - } else { - self.try_index_opt(vm) - .map(|i| Ok(vm.ctx.new_float(int::try_to_float(i?.as_bigint(), vm)?))) - } - } - - #[inline] - pub fn try_float(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyFloat>> { - self.try_float_opt(vm).ok_or_else(|| { - vm.new_type_error(format!("must be real number, not {}", self.class())) - })? - } -} - -#[derive(Default)] -pub struct PyNumberMethods { - /* Number implementations must check *both* - arguments for proper type and implement the necessary conversions - in the slot functions themselves. */ - pub add: Option<PyNumberBinaryFunc>, - pub subtract: Option<PyNumberBinaryFunc>, - pub multiply: Option<PyNumberBinaryFunc>, - pub remainder: Option<PyNumberBinaryFunc>, - pub divmod: Option<PyNumberBinaryFunc>, - pub power: Option<PyNumberTernaryFunc>, - pub negative: Option<PyNumberUnaryFunc>, - pub positive: Option<PyNumberUnaryFunc>, - pub absolute: Option<PyNumberUnaryFunc>, - pub boolean: Option<PyNumberUnaryFunc<bool>>, - pub invert: Option<PyNumberUnaryFunc>, - pub lshift: Option<PyNumberBinaryFunc>, - pub rshift: Option<PyNumberBinaryFunc>, - pub and: Option<PyNumberBinaryFunc>, - pub xor: Option<PyNumberBinaryFunc>, - pub or: Option<PyNumberBinaryFunc>, - pub int: Option<PyNumberUnaryFunc>, - pub float: Option<PyNumberUnaryFunc>, - - pub inplace_add: Option<PyNumberBinaryFunc>, - pub inplace_subtract: Option<PyNumberBinaryFunc>, - pub inplace_multiply: Option<PyNumberBinaryFunc>, - pub inplace_remainder: Option<PyNumberBinaryFunc>, - pub inplace_power: Option<PyNumberTernaryFunc>, - pub inplace_lshift: Option<PyNumberBinaryFunc>, - pub inplace_rshift: Option<PyNumberBinaryFunc>, - pub inplace_and: Option<PyNumberBinaryFunc>, - pub inplace_xor: Option<PyNumberBinaryFunc>, - pub inplace_or: Option<PyNumberBinaryFunc>, - - pub floor_divide: Option<PyNumberBinaryFunc>, - pub true_divide: Option<PyNumberBinaryFunc>, - pub inplace_floor_divide: Option<PyNumberBinaryFunc>, - pub inplace_true_divide: Option<PyNumberBinaryFunc>, - - pub index: Option<PyNumberUnaryFunc>, - - pub matrix_multiply: Option<PyNumberBinaryFunc>, - pub inplace_matrix_multiply: Option<PyNumberBinaryFunc>, -} - -impl PyNumberMethods { - /// this is NOT a global variable - pub const NOT_IMPLEMENTED: PyNumberMethods = PyNumberMethods { - add: None, - subtract: None, - multiply: None, - remainder: None, - divmod: None, - power: None, - negative: None, - positive: None, - absolute: None, - boolean: None, - invert: None, - lshift: None, - rshift: None, - and: None, - xor: None, - or: None, - int: None, - float: None, - inplace_add: None, - inplace_subtract: None, - inplace_multiply: None, - inplace_remainder: None, - inplace_power: None, - inplace_lshift: None, - inplace_rshift: None, - inplace_and: None, - inplace_xor: None, - inplace_or: None, - floor_divide: None, - true_divide: None, - inplace_floor_divide: None, - inplace_true_divide: None, - index: None, - matrix_multiply: None, - inplace_matrix_multiply: None, - }; - - pub fn not_implemented() -> &'static PyNumberMethods { - static GLOBAL_NOT_IMPLEMENTED: PyNumberMethods = PyNumberMethods::NOT_IMPLEMENTED; - &GLOBAL_NOT_IMPLEMENTED - } -} - -#[derive(Copy, Clone)] -pub enum PyNumberBinaryOp { - Add, - Subtract, - Multiply, - Remainder, - Divmod, - Lshift, - Rshift, - And, - Xor, - Or, - InplaceAdd, - InplaceSubtract, - InplaceMultiply, - InplaceRemainder, - InplaceLshift, - InplaceRshift, - InplaceAnd, - InplaceXor, - InplaceOr, - FloorDivide, - TrueDivide, - InplaceFloorDivide, - InplaceTrueDivide, - MatrixMultiply, - InplaceMatrixMultiply, -} - -#[derive(Copy, Clone)] -pub enum PyNumberTernaryOp { - Power, - InplacePower, -} - -#[derive(Default)] -pub struct PyNumberSlots { - pub add: AtomicCell<Option<PyNumberBinaryFunc>>, - pub subtract: AtomicCell<Option<PyNumberBinaryFunc>>, - pub multiply: AtomicCell<Option<PyNumberBinaryFunc>>, - pub remainder: AtomicCell<Option<PyNumberBinaryFunc>>, - pub divmod: AtomicCell<Option<PyNumberBinaryFunc>>, - pub power: AtomicCell<Option<PyNumberTernaryFunc>>, - pub negative: AtomicCell<Option<PyNumberUnaryFunc>>, - pub positive: AtomicCell<Option<PyNumberUnaryFunc>>, - pub absolute: AtomicCell<Option<PyNumberUnaryFunc>>, - pub boolean: AtomicCell<Option<PyNumberUnaryFunc<bool>>>, - pub invert: AtomicCell<Option<PyNumberUnaryFunc>>, - pub lshift: AtomicCell<Option<PyNumberBinaryFunc>>, - pub rshift: AtomicCell<Option<PyNumberBinaryFunc>>, - pub and: AtomicCell<Option<PyNumberBinaryFunc>>, - pub xor: AtomicCell<Option<PyNumberBinaryFunc>>, - pub or: AtomicCell<Option<PyNumberBinaryFunc>>, - pub int: AtomicCell<Option<PyNumberUnaryFunc>>, - pub float: AtomicCell<Option<PyNumberUnaryFunc>>, - - pub right_add: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_subtract: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_remainder: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_divmod: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_power: AtomicCell<Option<PyNumberTernaryFunc>>, - pub right_lshift: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_rshift: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_and: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_xor: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_or: AtomicCell<Option<PyNumberBinaryFunc>>, - - pub inplace_add: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_subtract: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_remainder: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_power: AtomicCell<Option<PyNumberTernaryFunc>>, - pub inplace_lshift: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_rshift: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_and: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_xor: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_or: AtomicCell<Option<PyNumberBinaryFunc>>, - - pub floor_divide: AtomicCell<Option<PyNumberBinaryFunc>>, - pub true_divide: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_floor_divide: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_true_divide: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_floor_divide: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_true_divide: AtomicCell<Option<PyNumberBinaryFunc>>, - - pub index: AtomicCell<Option<PyNumberUnaryFunc>>, - - pub matrix_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, - pub right_matrix_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, - pub inplace_matrix_multiply: AtomicCell<Option<PyNumberBinaryFunc>>, -} - -impl From<&PyNumberMethods> for PyNumberSlots { - fn from(value: &PyNumberMethods) -> Self { - // right_* functions will use the same left function as PyNumberMethods - // allows both f(self, other) and f(other, self) - Self { - add: AtomicCell::new(value.add), - subtract: AtomicCell::new(value.subtract), - multiply: AtomicCell::new(value.multiply), - remainder: AtomicCell::new(value.remainder), - divmod: AtomicCell::new(value.divmod), - power: AtomicCell::new(value.power), - negative: AtomicCell::new(value.negative), - positive: AtomicCell::new(value.positive), - absolute: AtomicCell::new(value.absolute), - boolean: AtomicCell::new(value.boolean), - invert: AtomicCell::new(value.invert), - lshift: AtomicCell::new(value.lshift), - rshift: AtomicCell::new(value.rshift), - and: AtomicCell::new(value.and), - xor: AtomicCell::new(value.xor), - or: AtomicCell::new(value.or), - int: AtomicCell::new(value.int), - float: AtomicCell::new(value.float), - right_add: AtomicCell::new(value.add), - right_subtract: AtomicCell::new(value.subtract), - right_multiply: AtomicCell::new(value.multiply), - right_remainder: AtomicCell::new(value.remainder), - right_divmod: AtomicCell::new(value.divmod), - right_power: AtomicCell::new(value.power), - right_lshift: AtomicCell::new(value.lshift), - right_rshift: AtomicCell::new(value.rshift), - right_and: AtomicCell::new(value.and), - right_xor: AtomicCell::new(value.xor), - right_or: AtomicCell::new(value.or), - inplace_add: AtomicCell::new(value.inplace_add), - inplace_subtract: AtomicCell::new(value.inplace_subtract), - inplace_multiply: AtomicCell::new(value.inplace_multiply), - inplace_remainder: AtomicCell::new(value.inplace_remainder), - inplace_power: AtomicCell::new(value.inplace_power), - inplace_lshift: AtomicCell::new(value.inplace_lshift), - inplace_rshift: AtomicCell::new(value.inplace_rshift), - inplace_and: AtomicCell::new(value.inplace_and), - inplace_xor: AtomicCell::new(value.inplace_xor), - inplace_or: AtomicCell::new(value.inplace_or), - floor_divide: AtomicCell::new(value.floor_divide), - true_divide: AtomicCell::new(value.true_divide), - right_floor_divide: AtomicCell::new(value.floor_divide), - right_true_divide: AtomicCell::new(value.true_divide), - inplace_floor_divide: AtomicCell::new(value.inplace_floor_divide), - inplace_true_divide: AtomicCell::new(value.inplace_true_divide), - index: AtomicCell::new(value.index), - matrix_multiply: AtomicCell::new(value.matrix_multiply), - right_matrix_multiply: AtomicCell::new(value.matrix_multiply), - inplace_matrix_multiply: AtomicCell::new(value.inplace_matrix_multiply), - } - } -} - -impl PyNumberSlots { - pub fn left_binary_op(&self, op_slot: PyNumberBinaryOp) -> Option<PyNumberBinaryFunc> { - use PyNumberBinaryOp::*; - match op_slot { - Add => self.add.load(), - Subtract => self.subtract.load(), - Multiply => self.multiply.load(), - Remainder => self.remainder.load(), - Divmod => self.divmod.load(), - Lshift => self.lshift.load(), - Rshift => self.rshift.load(), - And => self.and.load(), - Xor => self.xor.load(), - Or => self.or.load(), - InplaceAdd => self.inplace_add.load(), - InplaceSubtract => self.inplace_subtract.load(), - InplaceMultiply => self.inplace_multiply.load(), - InplaceRemainder => self.inplace_remainder.load(), - InplaceLshift => self.inplace_lshift.load(), - InplaceRshift => self.inplace_rshift.load(), - InplaceAnd => self.inplace_and.load(), - InplaceXor => self.inplace_xor.load(), - InplaceOr => self.inplace_or.load(), - FloorDivide => self.floor_divide.load(), - TrueDivide => self.true_divide.load(), - InplaceFloorDivide => self.inplace_floor_divide.load(), - InplaceTrueDivide => self.inplace_true_divide.load(), - MatrixMultiply => self.matrix_multiply.load(), - InplaceMatrixMultiply => self.inplace_matrix_multiply.load(), - } - } - - pub fn right_binary_op(&self, op_slot: PyNumberBinaryOp) -> Option<PyNumberBinaryFunc> { - use PyNumberBinaryOp::*; - match op_slot { - Add => self.right_add.load(), - Subtract => self.right_subtract.load(), - Multiply => self.right_multiply.load(), - Remainder => self.right_remainder.load(), - Divmod => self.right_divmod.load(), - Lshift => self.right_lshift.load(), - Rshift => self.right_rshift.load(), - And => self.right_and.load(), - Xor => self.right_xor.load(), - Or => self.right_or.load(), - FloorDivide => self.right_floor_divide.load(), - TrueDivide => self.right_true_divide.load(), - MatrixMultiply => self.right_matrix_multiply.load(), - _ => None, - } - } - - pub fn left_ternary_op(&self, op_slot: PyNumberTernaryOp) -> Option<PyNumberTernaryFunc> { - use PyNumberTernaryOp::*; - match op_slot { - Power => self.power.load(), - InplacePower => self.inplace_power.load(), - } - } - - pub fn right_ternary_op(&self, op_slot: PyNumberTernaryOp) -> Option<PyNumberTernaryFunc> { - use PyNumberTernaryOp::*; - match op_slot { - Power => self.right_power.load(), - _ => None, - } - } -} -#[derive(Copy, Clone)] -pub struct PyNumber<'a>(&'a PyObject); - -unsafe impl Traverse for PyNumber<'_> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.0.traverse(tracer_fn) - } -} - -impl<'a> Deref for PyNumber<'a> { - type Target = PyObject; - - fn deref(&self) -> &Self::Target { - self.0 - } -} - -impl<'a> PyNumber<'a> { - pub(crate) fn obj(self) -> &'a PyObject { - self.0 - } - - // PyNumber_Check - pub fn check(obj: &PyObject) -> bool { - let methods = &obj.class().slots.as_number; - methods.int.load().is_some() - || methods.index.load().is_some() - || methods.float.load().is_some() - || obj.payload_is::<PyComplex>() - } -} - -impl PyNumber<'_> { - // PyIndex_Check - pub fn is_index(self) -> bool { - self.class().slots.as_number.index.load().is_some() - } - - #[inline] - pub fn int(self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { - self.class().slots.as_number.int.load().map(|f| { - let ret = f(self, vm)?; - - if let Some(ret) = ret.downcast_ref_if_exact::<PyInt>(vm) { - return Ok(ret.to_owned()); - } - - let ret_class = ret.class().to_owned(); - if let Some(ret) = ret.downcast_ref::<PyInt>() { - warnings::warn( - vm.ctx.exceptions.deprecation_warning, - format!( - "__int__ returned non-int (type {}). \ - The ability to return an instance of a strict subclass of int \ - is deprecated, and may be removed in a future version of Python.", - ret_class - ), - 1, - vm, - )?; - - Ok(ret.to_owned()) - } else { - Err(vm.new_type_error(format!( - "{}.__int__ returned non-int(type {})", - self.class(), - ret_class - ))) - } - }) - } - - #[inline] - pub fn index(self, vm: &VirtualMachine) -> Option<PyResult<PyIntRef>> { - self.class().slots.as_number.index.load().map(|f| { - let ret = f(self, vm)?; - - if let Some(ret) = ret.downcast_ref_if_exact::<PyInt>(vm) { - return Ok(ret.to_owned()); - } - - let ret_class = ret.class().to_owned(); - if let Some(ret) = ret.downcast_ref::<PyInt>() { - warnings::warn( - vm.ctx.exceptions.deprecation_warning, - format!( - "__index__ returned non-int (type {}). \ - The ability to return an instance of a strict subclass of int \ - is deprecated, and may be removed in a future version of Python.", - ret_class - ), - 1, - vm, - )?; - - Ok(ret.to_owned()) - } else { - Err(vm.new_type_error(format!( - "{}.__index__ returned non-int(type {})", - self.class(), - ret_class - ))) - } - }) - } - - #[inline] - pub fn float(self, vm: &VirtualMachine) -> Option<PyResult<PyRef<PyFloat>>> { - self.class().slots.as_number.float.load().map(|f| { - let ret = f(self, vm)?; - - if let Some(ret) = ret.downcast_ref_if_exact::<PyFloat>(vm) { - return Ok(ret.to_owned()); - } - - let ret_class = ret.class().to_owned(); - if let Some(ret) = ret.downcast_ref::<PyFloat>() { - warnings::warn( - vm.ctx.exceptions.deprecation_warning, - format!( - "__float__ returned non-float (type {}). \ - The ability to return an instance of a strict subclass of float \ - is deprecated, and may be removed in a future version of Python.", - ret_class - ), - 1, - vm, - )?; - - Ok(ret.to_owned()) - } else { - Err(vm.new_type_error(format!( - "{}.__float__ returned non-float(type {})", - self.class(), - ret_class - ))) - } - }) - } -} diff --git a/vm/src/protocol/object.rs b/vm/src/protocol/object.rs deleted file mode 100644 index dc7f487e855..00000000000 --- a/vm/src/protocol/object.rs +++ /dev/null @@ -1,661 +0,0 @@ -//! Object Protocol -//! https://docs.python.org/3/c-api/object.html - -use crate::{ - builtins::{ - pystr::AsPyStr, PyAsyncGen, PyBytes, PyDict, PyDictRef, PyGenericAlias, PyInt, PyList, - PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, - }, - bytesinner::ByteInnerNewOptions, - common::{hash::PyHash, str::to_ascii}, - convert::{ToPyObject, ToPyResult}, - dictdatatype::DictKey, - function::{Either, OptionalArg, PyArithmeticValue, PySetterValue}, - object::PyPayload, - protocol::{PyIter, PyMapping, PySequence}, - types::{Constructor, PyComparisonOp}, - AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, -}; - -// RustPython doesn't need these items -// PyObject *Py_NotImplemented -// Py_RETURN_NOTIMPLEMENTED - -impl PyObjectRef { - // int PyObject_Print(PyObject *o, FILE *fp, int flags) - - // PyObject *PyObject_GenericGetDict(PyObject *o, void *context) - // int PyObject_GenericSetDict(PyObject *o, PyObject *value, void *context) - - #[inline(always)] - pub fn rich_compare(self, other: Self, opid: PyComparisonOp, vm: &VirtualMachine) -> PyResult { - self._cmp(&other, opid, vm).map(|res| res.to_pyobject(vm)) - } - - pub fn bytes(self, vm: &VirtualMachine) -> PyResult { - let bytes_type = vm.ctx.types.bytes_type; - match self.downcast_exact::<PyInt>(vm) { - Ok(int) => Err(vm.new_downcast_type_error(bytes_type, &int)), - Err(obj) => PyBytes::py_new( - bytes_type.to_owned(), - ByteInnerNewOptions { - source: OptionalArg::Present(obj), - encoding: OptionalArg::Missing, - errors: OptionalArg::Missing, - }, - vm, - ), - } - } - - // const hash_not_implemented: fn(&PyObject, &VirtualMachine) ->PyResult<PyHash> = crate::types::Unhashable::slot_hash; - - pub fn is_true(self, vm: &VirtualMachine) -> PyResult<bool> { - self.try_to_bool(vm) - } - - pub fn not(self, vm: &VirtualMachine) -> PyResult<bool> { - self.is_true(vm).map(|x| !x) - } - - pub fn length_hint(self, defaultvalue: usize, vm: &VirtualMachine) -> PyResult<usize> { - Ok(vm.length_hint_opt(self)?.unwrap_or(defaultvalue)) - } - - // PyObject *PyObject_Dir(PyObject *o) - pub fn dir(self, vm: &VirtualMachine) -> PyResult<PyList> { - let attributes = self.class().get_attributes(); - - let dict = PyDict::from_attributes(attributes, vm)?.into_ref(&vm.ctx); - - if let Some(object_dict) = self.dict() { - vm.call_method( - dict.as_object(), - identifier!(vm, update).as_str(), - (object_dict,), - )?; - } - - let attributes: Vec<_> = dict.into_iter().map(|(k, _v)| k).collect(); - - Ok(PyList::from(attributes)) - } -} - -impl PyObject { - /// Takes an object and returns an iterator for it. - /// This is typically a new iterator but if the argument is an iterator, this - /// returns itself. - pub fn get_iter(&self, vm: &VirtualMachine) -> PyResult<PyIter> { - // PyObject_GetIter - PyIter::try_from_object(vm, self.to_owned()) - } - - // PyObject *PyObject_GetAIter(PyObject *o) - pub fn get_aiter(&self, vm: &VirtualMachine) -> PyResult { - if self.payload_is::<PyAsyncGen>() { - vm.call_special_method(self, identifier!(vm, __aiter__), ()) - } else { - Err(vm.new_type_error("wrong argument type".to_owned())) - } - } - - pub fn has_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult<bool> { - self.get_attr(attr_name, vm).map(|o| !vm.is_none(&o)) - } - - pub fn get_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult { - let attr_name = attr_name.as_pystr(&vm.ctx); - self.get_attr_inner(attr_name, vm) - } - - // get_attribute should be used for full attribute access (usually from user code). - #[cfg_attr(feature = "flame-it", flame("PyObjectRef"))] - #[inline] - pub(crate) fn get_attr_inner(&self, attr_name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - vm_trace!("object.__getattribute__: {:?} {:?}", self, attr_name); - let getattro = self - .class() - .mro_find_map(|cls| cls.slots.getattro.load()) - .unwrap(); - getattro(self, attr_name, vm).map_err(|exc| { - vm.set_attribute_error_context(&exc, self.to_owned(), attr_name.to_owned()); - exc - }) - } - - pub fn call_set_attr( - &self, - vm: &VirtualMachine, - attr_name: &Py<PyStr>, - attr_value: PySetterValue, - ) -> PyResult<()> { - let setattro = { - let cls = self.class(); - cls.mro_find_map(|cls| cls.slots.setattro.load()) - .ok_or_else(|| { - let has_getattr = cls.mro_find_map(|cls| cls.slots.getattro.load()).is_some(); - vm.new_type_error(format!( - "'{}' object has {} attributes ({} {})", - cls.name(), - if has_getattr { "only read-only" } else { "no" }, - if attr_value.is_assign() { - "assign to" - } else { - "del" - }, - attr_name - )) - })? - }; - setattro(self, attr_name, attr_value, vm) - } - - pub fn set_attr<'a>( - &self, - attr_name: impl AsPyStr<'a>, - attr_value: impl Into<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let attr_name = attr_name.as_pystr(&vm.ctx); - let attr_value = attr_value.into(); - self.call_set_attr(vm, attr_name, PySetterValue::Assign(attr_value)) - } - - // int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) - #[cfg_attr(feature = "flame-it", flame)] - pub fn generic_setattr( - &self, - attr_name: &Py<PyStr>, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()> { - vm_trace!("object.__setattr__({:?}, {}, {:?})", self, attr_name, value); - if let Some(attr) = vm - .ctx - .interned_str(attr_name) - .and_then(|attr_name| self.get_class_attr(attr_name)) - { - let descr_set = attr.class().mro_find_map(|cls| cls.slots.descr_set.load()); - if let Some(descriptor) = descr_set { - return descriptor(&attr, self.to_owned(), value, vm); - } - } - - if let Some(dict) = self.dict() { - if let PySetterValue::Assign(value) = value { - dict.set_item(attr_name, value, vm)?; - } else { - dict.del_item(attr_name, vm).map_err(|e| { - if e.fast_isinstance(vm.ctx.exceptions.key_error) { - vm.new_attribute_error(format!( - "'{}' object has no attribute '{}'", - self.class().name(), - attr_name.as_str(), - )) - } else { - e - } - })?; - } - Ok(()) - } else { - Err(vm.new_attribute_error(format!( - "'{}' object has no attribute '{}'", - self.class().name(), - attr_name.as_str(), - ))) - } - } - - pub fn generic_getattr(&self, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - self.generic_getattr_opt(name, None, vm)?.ok_or_else(|| { - vm.new_attribute_error(format!( - "'{}' object has no attribute '{}'", - self.class().name(), - name - )) - }) - } - - /// CPython _PyObject_GenericGetAttrWithDict - pub fn generic_getattr_opt( - &self, - name_str: &Py<PyStr>, - dict: Option<PyDictRef>, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - let name = name_str.as_str(); - let obj_cls = self.class(); - let cls_attr_name = vm.ctx.interned_str(name_str); - let cls_attr = match cls_attr_name.and_then(|name| obj_cls.get_attr(name)) { - Some(descr) => { - let descr_cls = descr.class(); - let descr_get = descr_cls.mro_find_map(|cls| cls.slots.descr_get.load()); - if let Some(descr_get) = descr_get { - if descr_cls - .mro_find_map(|cls| cls.slots.descr_set.load()) - .is_some() - { - let cls = obj_cls.to_owned().into(); - return descr_get(descr, Some(self.to_owned()), Some(cls), vm).map(Some); - } - } - Some((descr, descr_get)) - } - None => None, - }; - - let dict = dict.or_else(|| self.dict()); - - let attr = if let Some(dict) = dict { - dict.get_item_opt(name, vm)? - } else { - None - }; - - if let Some(obj_attr) = attr { - Ok(Some(obj_attr)) - } else if let Some((attr, descr_get)) = cls_attr { - match descr_get { - Some(descr_get) => { - let cls = obj_cls.to_owned().into(); - descr_get(attr, Some(self.to_owned()), Some(cls), vm).map(Some) - } - None => Ok(Some(attr)), - } - } else { - Ok(None) - } - } - - pub fn del_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult<()> { - let attr_name = attr_name.as_pystr(&vm.ctx); - self.call_set_attr(vm, attr_name, PySetterValue::Delete) - } - - // Perform a comparison, raising TypeError when the requested comparison - // operator is not supported. - // see: CPython PyObject_RichCompare - #[inline] // called by ExecutingFrame::execute_compare with const op - fn _cmp( - &self, - other: &Self, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<Either<PyObjectRef, bool>> { - let swapped = op.swapped(); - let call_cmp = |obj: &PyObject, other: &PyObject, op| { - let cmp = obj - .class() - .mro_find_map(|cls| cls.slots.richcompare.load()) - .unwrap(); - let r = match cmp(obj, other, op, vm)? { - Either::A(obj) => PyArithmeticValue::from_object(vm, obj).map(Either::A), - Either::B(arithmetic) => arithmetic.map(Either::B), - }; - Ok(r) - }; - - let mut checked_reverse_op = false; - let is_strict_subclass = { - let self_class = self.class(); - let other_class = other.class(); - !self_class.is(other_class) && other_class.fast_issubclass(self_class) - }; - if is_strict_subclass { - let res = vm.with_recursion("in comparison", || call_cmp(other, self, swapped))?; - checked_reverse_op = true; - if let PyArithmeticValue::Implemented(x) = res { - return Ok(x); - } - } - if let PyArithmeticValue::Implemented(x) = - vm.with_recursion("in comparison", || call_cmp(self, other, op))? - { - return Ok(x); - } - if !checked_reverse_op { - let res = vm.with_recursion("in comparison", || call_cmp(other, self, swapped))?; - if let PyArithmeticValue::Implemented(x) = res { - return Ok(x); - } - } - match op { - PyComparisonOp::Eq => Ok(Either::B(self.is(&other))), - PyComparisonOp::Ne => Ok(Either::B(!self.is(&other))), - _ => Err(vm.new_unsupported_binop_error(self, other, op.operator_token())), - } - } - #[inline(always)] - pub fn rich_compare_bool( - &self, - other: &Self, - opid: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<bool> { - match self._cmp(other, opid, vm)? { - Either::A(obj) => obj.try_to_bool(vm), - Either::B(other) => Ok(other), - } - } - - pub fn repr(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> { - vm.with_recursion("while getting the repr of an object", || { - match self.class().slots.repr.load() { - Some(slot) => slot(self, vm), - None => vm - .call_special_method(self, identifier!(vm, __repr__), ())? - .try_into_value(vm), // TODO: remove magic method call once __repr__ is fully ported to slot - } - }) - } - - pub fn ascii(&self, vm: &VirtualMachine) -> PyResult<ascii::AsciiString> { - let repr = self.repr(vm)?; - let ascii = to_ascii(repr.as_str()); - Ok(ascii) - } - - // Container of the virtual machine state: - pub fn str(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let obj = match self.to_owned().downcast_exact::<PyStr>(vm) { - Ok(s) => return Ok(s.into_pyref()), - Err(obj) => obj, - }; - // TODO: replace to obj.class().slots.str - let str_method = match vm.get_special_method(&obj, identifier!(vm, __str__))? { - Some(str_method) => str_method, - None => return obj.repr(vm), - }; - let s = str_method.invoke((), vm)?; - s.downcast::<PyStr>().map_err(|obj| { - vm.new_type_error(format!( - "__str__ returned non-string (type {})", - obj.class().name() - )) - }) - } - - // Equivalent to check_class. Masks Attribute errors (into TypeErrors) and lets everything - // else go through. - fn check_cls<F>(&self, cls: &PyObject, vm: &VirtualMachine, msg: F) -> PyResult - where - F: Fn() -> String, - { - cls.get_attr(identifier!(vm, __bases__), vm).map_err(|e| { - // Only mask AttributeErrors. - if e.class().is(vm.ctx.exceptions.attribute_error) { - vm.new_type_error(msg()) - } else { - e - } - }) - } - - fn abstract_issubclass(&self, cls: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - let mut derived = self; - let mut first_item: PyObjectRef; - loop { - if derived.is(cls) { - return Ok(true); - } - - let bases = derived.get_attr(identifier!(vm, __bases__), vm)?; - let tuple = PyTupleRef::try_from_object(vm, bases)?; - - let n = tuple.len(); - match n { - 0 => { - return Ok(false); - } - 1 => { - first_item = tuple.fast_getitem(0).clone(); - derived = &first_item; - continue; - } - _ => { - if let Some(i) = (0..n).next() { - let check = vm.with_recursion("in abstract_issubclass", || { - tuple.fast_getitem(i).abstract_issubclass(cls, vm) - })?; - if check { - return Ok(true); - } - } - } - } - - return Ok(false); - } - } - - fn recursive_issubclass(&self, cls: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - if let (Ok(obj), Ok(cls)) = (self.try_to_ref::<PyType>(vm), cls.try_to_ref::<PyType>(vm)) { - Ok(obj.fast_issubclass(cls)) - } else { - self.check_cls(self, vm, || { - format!("issubclass() arg 1 must be a class, not {}", self.class()) - }) - .and(self.check_cls(cls, vm, || { - format!( - "issubclass() arg 2 must be a class, a tuple of classes, or a union, not {}", - cls.class() - ) - })) - .and(self.abstract_issubclass(cls, vm)) - } - } - - /// Determines if `self` is a subclass of `cls`, either directly, indirectly or virtually - /// via the __subclasscheck__ magic method. - pub fn is_subclass(&self, cls: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - if cls.class().is(vm.ctx.types.type_type) { - if self.is(cls) { - return Ok(true); - } - return self.recursive_issubclass(cls, vm); - } - - if let Ok(tuple) = cls.try_to_value::<&Py<PyTuple>>(vm) { - for typ in tuple { - if vm.with_recursion("in __subclasscheck__", || self.is_subclass(typ, vm))? { - return Ok(true); - } - } - return Ok(false); - } - - if let Some(meth) = vm.get_special_method(cls, identifier!(vm, __subclasscheck__))? { - let ret = vm.with_recursion("in __subclasscheck__", || { - meth.invoke((self.to_owned(),), vm) - })?; - return ret.try_to_bool(vm); - } - - self.recursive_issubclass(cls, vm) - } - - fn abstract_isinstance(&self, cls: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - let r = if let Ok(typ) = cls.try_to_ref::<PyType>(vm) { - if self.class().fast_issubclass(typ) { - true - } else if let Ok(icls) = - PyTypeRef::try_from_object(vm, self.get_attr(identifier!(vm, __class__), vm)?) - { - if icls.is(self.class()) { - false - } else { - icls.fast_issubclass(typ) - } - } else { - false - } - } else { - self.check_cls(cls, vm, || { - format!( - "isinstance() arg 2 must be a type or tuple of types, not {}", - cls.class() - ) - })?; - let icls: PyObjectRef = self.get_attr(identifier!(vm, __class__), vm)?; - if vm.is_none(&icls) { - false - } else { - icls.abstract_issubclass(cls, vm)? - } - }; - Ok(r) - } - - /// Determines if `self` is an instance of `cls`, either directly, indirectly or virtually via - /// the __instancecheck__ magic method. - pub fn is_instance(&self, cls: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - // cpython first does an exact check on the type, although documentation doesn't state that - // https://github.com/python/cpython/blob/a24107b04c1277e3c1105f98aff5bfa3a98b33a0/Objects/abstract.c#L2408 - if self.class().is(cls) { - return Ok(true); - } - - if cls.class().is(vm.ctx.types.type_type) { - return self.abstract_isinstance(cls, vm); - } - - if let Ok(tuple) = cls.try_to_ref::<PyTuple>(vm) { - for typ in tuple { - if vm.with_recursion("in __instancecheck__", || self.is_instance(typ, vm))? { - return Ok(true); - } - } - return Ok(false); - } - - if let Some(meth) = vm.get_special_method(cls, identifier!(vm, __instancecheck__))? { - let ret = vm.with_recursion("in __instancecheck__", || { - meth.invoke((self.to_owned(),), vm) - })?; - return ret.try_to_bool(vm); - } - - self.abstract_isinstance(cls, vm) - } - - pub fn hash(&self, vm: &VirtualMachine) -> PyResult<PyHash> { - let hash = self.get_class_attr(identifier!(vm, __hash__)).unwrap(); - if vm.is_none(&hash) { - return Err(vm.new_exception_msg( - vm.ctx.exceptions.type_error.to_owned(), - format!("unhashable type: '{}'", self.class().name()), - )); - } - - let hash = self - .class() - .mro_find_map(|cls| cls.slots.hash.load()) - .unwrap(); - - hash(self, vm) - } - - // type protocol - // PyObject *PyObject_Type(PyObject *o) - pub fn obj_type(&self) -> PyObjectRef { - self.class().to_owned().into() - } - - // int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) - pub fn type_check(&self, typ: PyTypeRef) -> bool { - self.fast_isinstance(&typ) - } - - pub fn length_opt(&self, vm: &VirtualMachine) -> Option<PyResult<usize>> { - self.to_sequence(vm) - .length_opt(vm) - .or_else(|| self.to_mapping().length_opt(vm)) - } - - pub fn length(&self, vm: &VirtualMachine) -> PyResult<usize> { - self.length_opt(vm).ok_or_else(|| { - vm.new_type_error(format!( - "object of type '{}' has no len()", - self.class().name() - )) - })? - } - - pub fn get_item<K: DictKey + ?Sized>(&self, needle: &K, vm: &VirtualMachine) -> PyResult { - if let Some(dict) = self.downcast_ref_if_exact::<PyDict>(vm) { - return dict.get_item(needle, vm); - } - - let needle = needle.to_pyobject(vm); - - if let Ok(mapping) = PyMapping::try_protocol(self, vm) { - mapping.subscript(&needle, vm) - } else if let Ok(seq) = PySequence::try_protocol(self, vm) { - let i = needle.key_as_isize(vm)?; - seq.get_item(i, vm) - } else { - if self.class().fast_issubclass(vm.ctx.types.type_type) { - if self.is(vm.ctx.types.type_type) { - return PyGenericAlias::new(self.class().to_owned(), needle, vm) - .to_pyresult(vm); - } - - if let Some(class_getitem) = - vm.get_attribute_opt(self.to_owned(), identifier!(vm, __class_getitem__))? - { - return class_getitem.call((needle,), vm); - } - } - Err(vm.new_type_error(format!("'{}' object is not subscriptable", self.class()))) - } - } - - pub fn set_item<K: DictKey + ?Sized>( - &self, - needle: &K, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if let Some(dict) = self.downcast_ref_if_exact::<PyDict>(vm) { - return dict.set_item(needle, value, vm); - } - - let mapping = self.to_mapping(); - if let Some(f) = mapping.methods.ass_subscript.load() { - let needle = needle.to_pyobject(vm); - return f(mapping, &needle, Some(value), vm); - } - - let seq = self.to_sequence(vm); - if let Some(f) = seq.methods.ass_item.load() { - let i = needle.key_as_isize(vm)?; - return f(seq, i, Some(value), vm); - } - - Err(vm.new_type_error(format!( - "'{}' does not support item assignment", - self.class() - ))) - } - - pub fn del_item<K: DictKey + ?Sized>(&self, needle: &K, vm: &VirtualMachine) -> PyResult<()> { - if let Some(dict) = self.downcast_ref_if_exact::<PyDict>(vm) { - return dict.del_item(needle, vm); - } - - let mapping = self.to_mapping(); - if let Some(f) = mapping.methods.ass_subscript.load() { - let needle = needle.to_pyobject(vm); - return f(mapping, &needle, None, vm); - } - let seq = self.to_sequence(vm); - if let Some(f) = seq.methods.ass_item.load() { - let i = needle.key_as_isize(vm)?; - return f(seq, i, None, vm); - } - - Err(vm.new_type_error(format!("'{}' does not support item deletion", self.class()))) - } -} diff --git a/vm/src/protocol/sequence.rs b/vm/src/protocol/sequence.rs deleted file mode 100644 index 7c53cc2932d..00000000000 --- a/vm/src/protocol/sequence.rs +++ /dev/null @@ -1,378 +0,0 @@ -use crate::{ - builtins::{type_::PointerSlot, PyList, PyListRef, PySlice, PyTuple, PyTupleRef}, - convert::ToPyObject, - function::PyArithmeticValue, - object::{Traverse, TraverseFn}, - protocol::{PyMapping, PyNumberBinaryOp}, - AsObject, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine, -}; -use crossbeam_utils::atomic::AtomicCell; -use itertools::Itertools; -use std::fmt::Debug; - -// Sequence Protocol -// https://docs.python.org/3/c-api/sequence.html - -impl PyObject { - #[inline] - pub fn to_sequence(&self, vm: &VirtualMachine) -> PySequence<'_> { - static GLOBAL_NOT_IMPLEMENTED: PySequenceMethods = PySequenceMethods::NOT_IMPLEMENTED; - PySequence { - obj: self, - methods: PySequence::find_methods(self, vm) - .map_or(&GLOBAL_NOT_IMPLEMENTED, |x| unsafe { x.borrow_static() }), - } - } -} - -#[allow(clippy::type_complexity)] -#[derive(Default)] -pub struct PySequenceMethods { - pub length: AtomicCell<Option<fn(PySequence, &VirtualMachine) -> PyResult<usize>>>, - pub concat: AtomicCell<Option<fn(PySequence, &PyObject, &VirtualMachine) -> PyResult>>, - pub repeat: AtomicCell<Option<fn(PySequence, isize, &VirtualMachine) -> PyResult>>, - pub item: AtomicCell<Option<fn(PySequence, isize, &VirtualMachine) -> PyResult>>, - pub ass_item: AtomicCell< - Option<fn(PySequence, isize, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>>, - >, - pub contains: AtomicCell<Option<fn(PySequence, &PyObject, &VirtualMachine) -> PyResult<bool>>>, - pub inplace_concat: AtomicCell<Option<fn(PySequence, &PyObject, &VirtualMachine) -> PyResult>>, - pub inplace_repeat: AtomicCell<Option<fn(PySequence, isize, &VirtualMachine) -> PyResult>>, -} - -impl Debug for PySequenceMethods { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Sequence Methods") - } -} - -impl PySequenceMethods { - #[allow(clippy::declare_interior_mutable_const)] - pub const NOT_IMPLEMENTED: PySequenceMethods = PySequenceMethods { - length: AtomicCell::new(None), - concat: AtomicCell::new(None), - repeat: AtomicCell::new(None), - item: AtomicCell::new(None), - ass_item: AtomicCell::new(None), - contains: AtomicCell::new(None), - inplace_concat: AtomicCell::new(None), - inplace_repeat: AtomicCell::new(None), - }; -} - -#[derive(Copy, Clone)] -pub struct PySequence<'a> { - pub obj: &'a PyObject, - pub methods: &'static PySequenceMethods, -} - -unsafe impl Traverse for PySequence<'_> { - fn traverse(&self, tracer_fn: &mut TraverseFn) { - self.obj.traverse(tracer_fn) - } -} - -impl<'a> PySequence<'a> { - #[inline] - pub fn with_methods(obj: &'a PyObject, methods: &'static PySequenceMethods) -> Self { - Self { obj, methods } - } - - pub fn try_protocol(obj: &'a PyObject, vm: &VirtualMachine) -> PyResult<Self> { - let seq = obj.to_sequence(vm); - if seq.check() { - Ok(seq) - } else { - Err(vm.new_type_error(format!("'{}' is not a sequence", obj.class()))) - } - } -} - -impl PySequence<'_> { - pub fn check(&self) -> bool { - self.methods.item.load().is_some() - } - - pub fn find_methods( - obj: &PyObject, - vm: &VirtualMachine, - ) -> Option<PointerSlot<PySequenceMethods>> { - let cls = obj.class(); - // if cls.fast_issubclass(vm.ctx.types.dict_type) { - if cls.is(vm.ctx.types.dict_type) { - return None; - } - cls.mro_find_map(|x| x.slots.as_sequence.load()) - } - - pub fn length_opt(self, vm: &VirtualMachine) -> Option<PyResult<usize>> { - self.methods.length.load().map(|f| f(self, vm)) - } - - pub fn length(self, vm: &VirtualMachine) -> PyResult<usize> { - self.length_opt(vm).ok_or_else(|| { - vm.new_type_error(format!( - "'{}' is not a sequence or has no len()", - self.obj.class() - )) - })? - } - - pub fn concat(self, other: &PyObject, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.concat.load() { - return f(self, other, vm); - } - - // if both arguments apear to be sequences, try fallback to __add__ - if self.check() && other.to_sequence(vm).check() { - let ret = vm.binary_op1(self.obj, other, PyNumberBinaryOp::Add)?; - if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { - return Ok(ret); - } - } - - Err(vm.new_type_error(format!( - "'{}' object can't be concatenated", - self.obj.class() - ))) - } - - pub fn repeat(self, n: isize, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.repeat.load() { - return f(self, n, vm); - } - - // fallback to __mul__ - if self.check() { - let ret = vm.binary_op1(self.obj, &n.to_pyobject(vm), PyNumberBinaryOp::Multiply)?; - if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { - return Ok(ret); - } - } - - Err(vm.new_type_error(format!("'{}' object can't be repeated", self.obj.class()))) - } - - pub fn inplace_concat(self, other: &PyObject, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.inplace_concat.load() { - return f(self, other, vm); - } - if let Some(f) = self.methods.concat.load() { - return f(self, other, vm); - } - - // if both arguments apear to be sequences, try fallback to __iadd__ - if self.check() && other.to_sequence(vm).check() { - let ret = vm._iadd(self.obj, other)?; - if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { - return Ok(ret); - } - } - - Err(vm.new_type_error(format!( - "'{}' object can't be concatenated", - self.obj.class() - ))) - } - - pub fn inplace_repeat(self, n: isize, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.inplace_repeat.load() { - return f(self, n, vm); - } - if let Some(f) = self.methods.repeat.load() { - return f(self, n, vm); - } - - if self.check() { - let ret = vm._imul(self.obj, &n.to_pyobject(vm))?; - if let PyArithmeticValue::Implemented(ret) = PyArithmeticValue::from_object(vm, ret) { - return Ok(ret); - } - } - - Err(vm.new_type_error(format!("'{}' object can't be repeated", self.obj.class()))) - } - - pub fn get_item(self, i: isize, vm: &VirtualMachine) -> PyResult { - if let Some(f) = self.methods.item.load() { - return f(self, i, vm); - } - Err(vm.new_type_error(format!( - "'{}' is not a sequence or does not support indexing", - self.obj.class() - ))) - } - - fn _ass_item(self, i: isize, value: Option<PyObjectRef>, vm: &VirtualMachine) -> PyResult<()> { - if let Some(f) = self.methods.ass_item.load() { - return f(self, i, value, vm); - } - Err(vm.new_type_error(format!( - "'{}' is not a sequence or doesn't support item {}", - self.obj.class(), - if value.is_some() { - "assignment" - } else { - "deletion" - } - ))) - } - - pub fn set_item(self, i: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self._ass_item(i, Some(value), vm) - } - - pub fn del_item(self, i: isize, vm: &VirtualMachine) -> PyResult<()> { - self._ass_item(i, None, vm) - } - - pub fn get_slice(&self, start: isize, stop: isize, vm: &VirtualMachine) -> PyResult { - if let Ok(mapping) = PyMapping::try_protocol(self.obj, vm) { - let slice = PySlice { - start: Some(start.to_pyobject(vm)), - stop: stop.to_pyobject(vm), - step: None, - }; - mapping.subscript(&slice.into_pyobject(vm), vm) - } else { - Err(vm.new_type_error(format!("'{}' object is unsliceable", self.obj.class()))) - } - } - - fn _ass_slice( - &self, - start: isize, - stop: isize, - value: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let mapping = self.obj.to_mapping(); - if let Some(f) = mapping.methods.ass_subscript.load() { - let slice = PySlice { - start: Some(start.to_pyobject(vm)), - stop: stop.to_pyobject(vm), - step: None, - }; - f(mapping, &slice.into_pyobject(vm), value, vm) - } else { - Err(vm.new_type_error(format!( - "'{}' object doesn't support slice {}", - self.obj.class(), - if value.is_some() { - "assignment" - } else { - "deletion" - } - ))) - } - } - - pub fn set_slice( - &self, - start: isize, - stop: isize, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - self._ass_slice(start, stop, Some(value), vm) - } - - pub fn del_slice(&self, start: isize, stop: isize, vm: &VirtualMachine) -> PyResult<()> { - self._ass_slice(start, stop, None, vm) - } - - pub fn tuple(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - if let Some(tuple) = self.obj.downcast_ref_if_exact::<PyTuple>(vm) { - Ok(tuple.to_owned()) - } else if let Some(list) = self.obj.downcast_ref_if_exact::<PyList>(vm) { - Ok(vm.ctx.new_tuple(list.borrow_vec().to_vec())) - } else { - let iter = self.obj.to_owned().get_iter(vm)?; - let iter = iter.iter(vm)?; - Ok(vm.ctx.new_tuple(iter.try_collect()?)) - } - } - - pub fn list(&self, vm: &VirtualMachine) -> PyResult<PyListRef> { - let list = vm.ctx.new_list(self.obj.try_to_value(vm)?); - Ok(list) - } - - pub fn count(&self, target: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { - let mut n = 0; - - let iter = self.obj.to_owned().get_iter(vm)?; - let iter = iter.iter::<PyObjectRef>(vm)?; - - for elem in iter { - let elem = elem?; - if vm.bool_eq(&elem, target)? { - if n == isize::MAX as usize { - return Err(vm.new_overflow_error("index exceeds C integer size".to_string())); - } - n += 1; - } - } - - Ok(n) - } - - pub fn index(&self, target: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { - let mut index: isize = -1; - - let iter = self.obj.to_owned().get_iter(vm)?; - let iter = iter.iter::<PyObjectRef>(vm)?; - - for elem in iter { - if index == isize::MAX { - return Err(vm.new_overflow_error("index exceeds C integer size".to_string())); - } - index += 1; - - let elem = elem?; - if vm.bool_eq(&elem, target)? { - return Ok(index as usize); - } - } - - Err(vm.new_value_error("sequence.index(x): x not in sequence".to_string())) - } - - pub fn extract<F, R>(&self, mut f: F, vm: &VirtualMachine) -> PyResult<Vec<R>> - where - F: FnMut(&PyObject) -> PyResult<R>, - { - if let Some(tuple) = self.obj.payload_if_exact::<PyTuple>(vm) { - tuple.iter().map(|x| f(x.as_ref())).collect() - } else if let Some(list) = self.obj.payload_if_exact::<PyList>(vm) { - list.borrow_vec().iter().map(|x| f(x.as_ref())).collect() - } else { - let iter = self.obj.to_owned().get_iter(vm)?; - let iter = iter.iter::<PyObjectRef>(vm)?; - let len = self.length(vm).unwrap_or(0); - let mut v = Vec::with_capacity(len); - for x in iter { - v.push(f(x?.as_ref())?); - } - v.shrink_to_fit(); - Ok(v) - } - } - - pub fn contains(self, target: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - if let Some(f) = self.methods.contains.load() { - return f(self, target, vm); - } - - let iter = self.obj.to_owned().get_iter(vm)?; - let iter = iter.iter::<PyObjectRef>(vm)?; - - for elem in iter { - let elem = elem?; - if vm.bool_eq(&elem, target)? { - return Ok(true); - } - } - Ok(false) - } -} diff --git a/vm/src/py_io.rs b/vm/src/py_io.rs deleted file mode 100644 index 8091b75afca..00000000000 --- a/vm/src/py_io.rs +++ /dev/null @@ -1,97 +0,0 @@ -use crate::{ - builtins::{PyBaseExceptionRef, PyBytes, PyStr}, - common::ascii, - PyObject, PyObjectRef, PyResult, VirtualMachine, -}; -use std::{fmt, io, ops}; - -pub trait Write { - type Error; - fn write_fmt(&mut self, args: fmt::Arguments) -> Result<(), Self::Error>; -} - -#[repr(transparent)] -pub struct IoWriter<T>(pub T); - -impl<T> IoWriter<T> { - pub fn from_ref(x: &mut T) -> &mut Self { - // SAFETY: IoWriter is repr(transparent) over T - unsafe { &mut *(x as *mut T as *mut Self) } - } -} - -impl<T> ops::Deref for IoWriter<T> { - type Target = T; - fn deref(&self) -> &T { - &self.0 - } -} -impl<T> ops::DerefMut for IoWriter<T> { - fn deref_mut(&mut self) -> &mut T { - &mut self.0 - } -} - -impl<W> Write for IoWriter<W> -where - W: io::Write, -{ - type Error = io::Error; - fn write_fmt(&mut self, args: fmt::Arguments) -> io::Result<()> { - <W as io::Write>::write_fmt(&mut self.0, args) - } -} - -impl Write for String { - type Error = fmt::Error; - fn write_fmt(&mut self, args: fmt::Arguments) -> fmt::Result { - <String as fmt::Write>::write_fmt(self, args) - } -} - -pub struct PyWriter<'vm>(pub PyObjectRef, pub &'vm VirtualMachine); - -impl Write for PyWriter<'_> { - type Error = PyBaseExceptionRef; - fn write_fmt(&mut self, args: fmt::Arguments) -> Result<(), Self::Error> { - let PyWriter(obj, vm) = self; - vm.call_method(obj, "write", (args.to_string(),)).map(drop) - } -} - -pub fn file_readline(obj: &PyObject, size: Option<usize>, vm: &VirtualMachine) -> PyResult { - let args = size.map_or_else(Vec::new, |size| vec![vm.ctx.new_int(size).into()]); - let ret = vm.call_method(obj, "readline", args)?; - let eof_err = || { - vm.new_exception( - vm.ctx.exceptions.eof_error.to_owned(), - vec![vm.ctx.new_str(ascii!("EOF when reading a line")).into()], - ) - }; - let ret = match_class!(match ret { - s @ PyStr => { - let sval = s.as_str(); - if sval.is_empty() { - return Err(eof_err()); - } - if let Some(nonl) = sval.strip_suffix('\n') { - vm.ctx.new_str(nonl).into() - } else { - s.into() - } - } - b @ PyBytes => { - let buf = b.as_bytes(); - if buf.is_empty() { - return Err(eof_err()); - } - if buf.last() == Some(&b'\n') { - vm.ctx.new_bytes(buf[..buf.len() - 1].to_owned()).into() - } else { - b.into() - } - } - _ => return Err(vm.new_type_error("object.readline() returned non-string".to_owned())), - }); - Ok(ret) -} diff --git a/vm/src/readline.rs b/vm/src/readline.rs deleted file mode 100644 index 53647270e18..00000000000 --- a/vm/src/readline.rs +++ /dev/null @@ -1,148 +0,0 @@ -use std::{io, path::Path}; - -type OtherError = Box<dyn std::error::Error>; -type OtherResult<T> = Result<T, OtherError>; - -pub enum ReadlineResult { - Line(String), - Eof, - Interrupt, - Io(std::io::Error), - Other(OtherError), -} - -#[allow(unused)] -mod basic_readline { - use super::*; - - pub trait Helper {} - impl<T> Helper for T {} - - pub struct Readline<H: Helper> { - helper: H, - } - - impl<H: Helper> Readline<H> { - pub fn new(helper: H) -> Self { - Readline { helper } - } - - pub fn load_history(&mut self, _path: &Path) -> OtherResult<()> { - Ok(()) - } - - pub fn save_history(&mut self, _path: &Path) -> OtherResult<()> { - Ok(()) - } - - pub fn add_history_entry(&mut self, _entry: &str) -> OtherResult<()> { - Ok(()) - } - - pub fn readline(&mut self, prompt: &str) -> ReadlineResult { - use std::io::prelude::*; - print!("{prompt}"); - if let Err(e) = io::stdout().flush() { - return ReadlineResult::Io(e); - } - - let next_line = io::stdin().lock().lines().next(); - match next_line { - Some(Ok(line)) => ReadlineResult::Line(line), - None => ReadlineResult::Eof, - Some(Err(e)) if e.kind() == io::ErrorKind::Interrupted => ReadlineResult::Interrupt, - Some(Err(e)) => ReadlineResult::Io(e), - } - } - } -} - -#[cfg(not(target_arch = "wasm32"))] -mod rustyline_readline { - use super::*; - - pub trait Helper: rustyline::Helper {} - impl<T: rustyline::Helper> Helper for T {} - - /// Readline: the REPL - pub struct Readline<H: Helper> { - repl: rustyline::Editor<H, rustyline::history::DefaultHistory>, - } - - impl<H: Helper> Readline<H> { - pub fn new(helper: H) -> Self { - use rustyline::*; - let mut repl = Editor::with_config( - Config::builder() - .completion_type(CompletionType::List) - .tab_stop(8) - .bracketed_paste(false) // multi-line paste - .build(), - ) - .expect("failed to initialize line editor"); - repl.set_helper(Some(helper)); - Readline { repl } - } - - pub fn load_history(&mut self, path: &Path) -> OtherResult<()> { - self.repl.load_history(path)?; - Ok(()) - } - - pub fn save_history(&mut self, path: &Path) -> OtherResult<()> { - if !path.exists() { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - } - self.repl.save_history(path)?; - Ok(()) - } - - pub fn add_history_entry(&mut self, entry: &str) -> OtherResult<()> { - self.repl.add_history_entry(entry)?; - Ok(()) - } - - pub fn readline(&mut self, prompt: &str) -> ReadlineResult { - use rustyline::error::ReadlineError; - loop { - break match self.repl.readline(prompt) { - Ok(line) => ReadlineResult::Line(line), - Err(ReadlineError::Interrupted) => ReadlineResult::Interrupt, - Err(ReadlineError::Eof) => ReadlineResult::Eof, - Err(ReadlineError::Io(e)) => ReadlineResult::Io(e), - Err(ReadlineError::WindowResized) => continue, - Err(e) => ReadlineResult::Other(e.into()), - }; - } - } - } -} - -#[cfg(target_arch = "wasm32")] -use basic_readline as readline_inner; -#[cfg(not(target_arch = "wasm32"))] -use rustyline_readline as readline_inner; - -pub use readline_inner::Helper; - -pub struct Readline<H: Helper>(readline_inner::Readline<H>); - -impl<H: Helper> Readline<H> { - pub fn new(helper: H) -> Self { - Readline(readline_inner::Readline::new(helper)) - } - pub fn load_history(&mut self, path: &Path) -> OtherResult<()> { - self.0.load_history(path) - } - pub fn save_history(&mut self, path: &Path) -> OtherResult<()> { - self.0.save_history(path) - } - pub fn add_history_entry(&mut self, entry: &str) -> OtherResult<()> { - self.0.add_history_entry(entry) - } - pub fn readline(&mut self, prompt: &str) -> ReadlineResult { - self.0.readline(prompt) - } -} diff --git a/vm/src/sequence.rs b/vm/src/sequence.rs deleted file mode 100644 index 614e72d52ee..00000000000 --- a/vm/src/sequence.rs +++ /dev/null @@ -1,156 +0,0 @@ -use crate::{ - builtins::PyIntRef, function::OptionalArg, sliceable::SequenceIndexOp, types::PyComparisonOp, - vm::VirtualMachine, AsObject, PyObject, PyObjectRef, PyResult, -}; -use optional::Optioned; -use std::ops::Range; - -pub trait MutObjectSequenceOp { - type Guard<'a>: 'a; - - fn do_get<'a>(index: usize, guard: &'a Self::Guard<'_>) -> Option<&'a PyObjectRef>; - fn do_lock(&self) -> Self::Guard<'_>; - - fn mut_count(&self, vm: &VirtualMachine, needle: &PyObject) -> PyResult<usize> { - let mut count = 0; - self._mut_iter_equal_skeleton::<_, false>(vm, needle, 0..isize::MAX as usize, || { - count += 1 - })?; - Ok(count) - } - - fn mut_index_range( - &self, - vm: &VirtualMachine, - needle: &PyObject, - range: Range<usize>, - ) -> PyResult<Optioned<usize>> { - self._mut_iter_equal_skeleton::<_, true>(vm, needle, range, || {}) - } - - fn mut_index(&self, vm: &VirtualMachine, needle: &PyObject) -> PyResult<Optioned<usize>> { - self.mut_index_range(vm, needle, 0..isize::MAX as usize) - } - - fn mut_contains(&self, vm: &VirtualMachine, needle: &PyObject) -> PyResult<bool> { - self.mut_index(vm, needle).map(|x| x.is_some()) - } - - fn _mut_iter_equal_skeleton<F, const SHORT: bool>( - &self, - vm: &VirtualMachine, - needle: &PyObject, - range: Range<usize>, - mut f: F, - ) -> PyResult<Optioned<usize>> - where - F: FnMut(), - { - let mut borrower = None; - let mut i = range.start; - - let index = loop { - if i >= range.end { - break Optioned::<usize>::none(); - } - let guard = if let Some(x) = borrower.take() { - x - } else { - self.do_lock() - }; - - let elem = if let Some(x) = Self::do_get(i, &guard) { - x - } else { - break Optioned::<usize>::none(); - }; - - if elem.is(needle) { - f(); - if SHORT { - break Optioned::<usize>::some(i); - } - borrower = Some(guard); - } else { - let elem = elem.clone(); - drop(guard); - - if elem.rich_compare_bool(needle, PyComparisonOp::Eq, vm)? { - f(); - if SHORT { - break Optioned::<usize>::some(i); - } - } - } - i += 1; - }; - - Ok(index) - } -} - -pub trait SequenceExt<T: Clone> -where - Self: AsRef<[T]>, -{ - fn mul(&self, vm: &VirtualMachine, n: isize) -> PyResult<Vec<T>> { - let n = vm.check_repeat_or_overflow_error(self.as_ref().len(), n)?; - let mut v = Vec::with_capacity(n * self.as_ref().len()); - for _ in 0..n { - v.extend_from_slice(self.as_ref()); - } - Ok(v) - } -} - -impl<T: Clone> SequenceExt<T> for [T] {} - -pub trait SequenceMutExt<T: Clone> -where - Self: AsRef<[T]>, -{ - fn as_vec_mut(&mut self) -> &mut Vec<T>; - - fn imul(&mut self, vm: &VirtualMachine, n: isize) -> PyResult<()> { - let n = vm.check_repeat_or_overflow_error(self.as_ref().len(), n)?; - if n == 0 { - self.as_vec_mut().clear(); - } else if n != 1 { - let mut sample = self.as_vec_mut().clone(); - if n != 2 { - self.as_vec_mut().reserve(sample.len() * (n - 1)); - for _ in 0..n - 2 { - self.as_vec_mut().extend_from_slice(&sample); - } - } - self.as_vec_mut().append(&mut sample); - } - Ok(()) - } -} - -impl<T: Clone> SequenceMutExt<T> for Vec<T> { - fn as_vec_mut(&mut self) -> &mut Vec<T> { - self - } -} - -#[derive(FromArgs)] -pub struct OptionalRangeArgs { - #[pyarg(positional, optional)] - start: OptionalArg<PyObjectRef>, - #[pyarg(positional, optional)] - stop: OptionalArg<PyObjectRef>, -} - -impl OptionalRangeArgs { - pub fn saturate(self, len: usize, vm: &VirtualMachine) -> PyResult<(usize, usize)> { - let saturate = |obj: PyObjectRef| -> PyResult<_> { - obj.try_into_value(vm) - .map(|int: PyIntRef| int.as_bigint().saturated_at(len)) - }; - let start = self.start.map_or(Ok(0), saturate)?; - let stop = self.stop.map_or(Ok(len), saturate)?; - Ok((start, stop)) - } -} diff --git a/vm/src/signal.rs b/vm/src/signal.rs deleted file mode 100644 index 7489282de21..00000000000 --- a/vm/src/signal.rs +++ /dev/null @@ -1,125 +0,0 @@ -#![cfg_attr(target_os = "wasi", allow(dead_code))] -use crate::{PyResult, VirtualMachine}; -use std::{ - fmt, - sync::{ - atomic::{AtomicBool, Ordering}, - mpsc, - }, -}; - -pub(crate) const NSIG: usize = 64; -static ANY_TRIGGERED: AtomicBool = AtomicBool::new(false); -// hack to get around const array repeat expressions, rust issue #79270 -#[allow(clippy::declare_interior_mutable_const)] -const ATOMIC_FALSE: AtomicBool = AtomicBool::new(false); -pub(crate) static TRIGGERS: [AtomicBool; NSIG] = [ATOMIC_FALSE; NSIG]; - -#[cfg_attr(feature = "flame-it", flame)] -#[inline(always)] -pub fn check_signals(vm: &VirtualMachine) -> PyResult<()> { - if vm.signal_handlers.is_none() { - return Ok(()); - } - - if !ANY_TRIGGERED.swap(false, Ordering::Acquire) { - return Ok(()); - } - - trigger_signals(vm) -} -#[inline(never)] -#[cold] -fn trigger_signals(vm: &VirtualMachine) -> PyResult<()> { - // unwrap should never fail since we check above - let signal_handlers = vm.signal_handlers.as_ref().unwrap().borrow(); - for (signum, trigger) in TRIGGERS.iter().enumerate().skip(1) { - let triggered = trigger.swap(false, Ordering::Relaxed); - if triggered { - if let Some(handler) = &signal_handlers[signum] { - if let Some(callable) = handler.to_callable() { - callable.invoke((signum, vm.ctx.none()), vm)?; - } - } - } - } - if let Some(signal_rx) = &vm.signal_rx { - for f in signal_rx.rx.try_iter() { - f(vm)?; - } - } - Ok(()) -} - -pub(crate) fn set_triggered() { - ANY_TRIGGERED.store(true, Ordering::Release); -} - -pub fn assert_in_range(signum: i32, vm: &VirtualMachine) -> PyResult<()> { - if (1..NSIG as i32).contains(&signum) { - Ok(()) - } else { - Err(vm.new_value_error("signal number out of range".to_owned())) - } -} - -/// Similar to `PyErr_SetInterruptEx` in CPython -/// -/// Missing signal handler for the given signal number is silently ignored. -#[allow(dead_code)] -#[cfg(not(target_arch = "wasm32"))] -pub fn set_interrupt_ex(signum: i32, vm: &VirtualMachine) -> PyResult<()> { - use crate::stdlib::signal::_signal::{run_signal, SIG_DFL, SIG_IGN}; - assert_in_range(signum, vm)?; - - match signum as usize { - SIG_DFL | SIG_IGN => Ok(()), - _ => { - // interrupt the main thread with given signal number - run_signal(signum); - Ok(()) - } - } -} - -pub type UserSignal = Box<dyn FnOnce(&VirtualMachine) -> PyResult<()> + Send>; - -#[derive(Clone, Debug)] -pub struct UserSignalSender { - tx: mpsc::Sender<UserSignal>, -} - -#[derive(Debug)] -pub struct UserSignalReceiver { - rx: mpsc::Receiver<UserSignal>, -} - -impl UserSignalSender { - pub fn send(&self, sig: UserSignal) -> Result<(), UserSignalSendError> { - self.tx - .send(sig) - .map_err(|mpsc::SendError(sig)| UserSignalSendError(sig))?; - set_triggered(); - Ok(()) - } -} - -pub struct UserSignalSendError(pub UserSignal); - -impl fmt::Debug for UserSignalSendError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("UserSignalSendError") - .finish_non_exhaustive() - } -} - -impl fmt::Display for UserSignalSendError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("sending a signal to a exited vm") - } -} - -pub fn user_signal_channel() -> (UserSignalSender, UserSignalReceiver) { - let (tx, rx) = mpsc::channel(); - (UserSignalSender { tx }, UserSignalReceiver { rx }) -} diff --git a/vm/src/source.rs b/vm/src/source.rs deleted file mode 100644 index 231c2f01c3e..00000000000 --- a/vm/src/source.rs +++ /dev/null @@ -1,20 +0,0 @@ -use crate::source_code::SourceLocation; - -// pub(crate) fn new_location_error( -// index: usize, -// field: &str, -// vm: &VirtualMachine, -// ) -> PyRef<PyBaseException> { -// vm.new_value_error(format!("value {index} is too large for location {field}")) -// } - -pub(crate) struct AtLocation<'a>(pub Option<&'a SourceLocation>); - -impl std::fmt::Display for AtLocation<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let (row, column) = self - .0 - .map_or((0, 0), |l| (l.row.to_usize(), l.column.to_usize())); - write!(f, " at line {row} column {column}",) - } -} diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs deleted file mode 100644 index 50b3153d0cd..00000000000 --- a/vm/src/stdlib/ast.rs +++ /dev/null @@ -1,361 +0,0 @@ -//! `ast` standard module for abstract syntax trees. -//! -//! This module makes use of the parser logic, and translates all ast nodes -//! into python ast.AST objects. - -mod gen; - -use crate::{ - builtins::{self, PyDict, PyModule, PyStrRef, PyType}, - class::{PyClassImpl, StaticType}, - compiler::core::bytecode::OpArgType, - compiler::CompileError, - convert::ToPyException, - source_code::{LinearLocator, OneIndexed, SourceLocation, SourceRange}, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - VirtualMachine, -}; -use num_complex::Complex64; -use num_traits::{ToPrimitive, Zero}; -use rustpython_ast::{self as ast, fold::Fold}; -#[cfg(feature = "rustpython-codegen")] -use rustpython_codegen as codegen; -#[cfg(feature = "rustpython-parser")] -use rustpython_parser as parser; - -#[pymodule] -mod _ast { - use crate::{ - builtins::{PyStrRef, PyTupleRef}, - function::FuncArgs, - AsObject, Context, PyObjectRef, PyPayload, PyResult, VirtualMachine, - }; - #[pyattr] - #[pyclass(module = "_ast", name = "AST")] - #[derive(Debug, PyPayload)] - pub(crate) struct NodeAst; - - #[pyclass(flags(BASETYPE, HAS_DICT))] - impl NodeAst { - #[pyslot] - #[pymethod(magic)] - fn init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let fields = zelf.get_attr("_fields", vm)?; - let fields: Vec<PyStrRef> = fields.try_to_value(vm)?; - let numargs = args.args.len(); - if numargs > fields.len() { - return Err(vm.new_type_error(format!( - "{} constructor takes at most {} positional argument{}", - zelf.class().name(), - fields.len(), - if fields.len() == 1 { "" } else { "s" }, - ))); - } - for (name, arg) in fields.iter().zip(args.args) { - zelf.set_attr(name, arg, vm)?; - } - for (key, value) in args.kwargs { - if let Some(pos) = fields.iter().position(|f| f.as_str() == key) { - if pos < numargs { - return Err(vm.new_type_error(format!( - "{} got multiple values for argument '{}'", - zelf.class().name(), - key - ))); - } - } - zelf.set_attr(vm.ctx.intern_str(key), value, vm)?; - } - Ok(()) - } - - #[pyattr(name = "_fields")] - fn fields(ctx: &Context) -> PyTupleRef { - ctx.empty_tuple.clone() - } - } - - #[pyattr(name = "PyCF_ONLY_AST")] - use super::PY_COMPILE_FLAG_AST_ONLY; -} - -fn get_node_field(vm: &VirtualMachine, obj: &PyObject, field: &'static str, typ: &str) -> PyResult { - vm.get_attribute_opt(obj.to_owned(), field)? - .ok_or_else(|| vm.new_type_error(format!("required field \"{field}\" missing from {typ}"))) -} - -fn get_node_field_opt( - vm: &VirtualMachine, - obj: &PyObject, - field: &'static str, -) -> PyResult<Option<PyObjectRef>> { - Ok(vm - .get_attribute_opt(obj.to_owned(), field)? - .filter(|obj| !vm.is_none(obj))) -} - -trait Node: Sized { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef; - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self>; -} - -impl<T: Node> Node for Vec<T> { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx - .new_list( - self.into_iter() - .map(|node| node.ast_to_object(vm)) - .collect(), - ) - .into() - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - vm.extract_elements_with(&object, |obj| Node::ast_from_object(vm, obj)) - } -} - -impl<T: Node> Node for Box<T> { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - (*self).ast_to_object(vm) - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - T::ast_from_object(vm, object).map(Box::new) - } -} - -impl<T: Node> Node for Option<T> { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - Some(node) => node.ast_to_object(vm), - None => vm.ctx.none(), - } - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - if vm.is_none(&object) { - Ok(None) - } else { - Ok(Some(T::ast_from_object(vm, object)?)) - } - } -} - -fn range_from_object( - vm: &VirtualMachine, - object: PyObjectRef, - name: &str, -) -> PyResult<SourceRange> { - fn make_location(row: u32, column: u32) -> Option<SourceLocation> { - Some(SourceLocation { - row: OneIndexed::new(row)?, - column: OneIndexed::from_zero_indexed(column), - }) - } - let row = ast::Int::ast_from_object(vm, get_node_field(vm, &object, "lineno", name)?)?; - let column = ast::Int::ast_from_object(vm, get_node_field(vm, &object, "col_offset", name)?)?; - let location = make_location(row.to_u32(), column.to_u32()); - let end_row = get_node_field_opt(vm, &object, "end_lineno")? - .map(|obj| ast::Int::ast_from_object(vm, obj)) - .transpose()?; - let end_column = get_node_field_opt(vm, &object, "end_col_offset")? - .map(|obj| ast::Int::ast_from_object(vm, obj)) - .transpose()?; - let end_location = if let (Some(row), Some(column)) = (end_row, end_column) { - make_location(row.to_u32(), column.to_u32()) - } else { - None - }; - let range = SourceRange { - start: location.unwrap_or_default(), - end: end_location, - }; - Ok(range) -} - -fn node_add_location(dict: &Py<PyDict>, range: SourceRange, vm: &VirtualMachine) { - dict.set_item("lineno", vm.ctx.new_int(range.start.row.get()).into(), vm) - .unwrap(); - dict.set_item( - "col_offset", - vm.ctx.new_int(range.start.column.to_zero_indexed()).into(), - vm, - ) - .unwrap(); - if let Some(end_location) = range.end { - dict.set_item( - "end_lineno", - vm.ctx.new_int(end_location.row.get()).into(), - vm, - ) - .unwrap(); - dict.set_item( - "end_col_offset", - vm.ctx.new_int(end_location.column.to_zero_indexed()).into(), - vm, - ) - .unwrap(); - }; -} - -impl Node for ast::String { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_str(self).into() - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - let py_str = PyStrRef::try_from_object(vm, object)?; - Ok(py_str.as_str().to_owned()) - } -} - -impl Node for ast::Identifier { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let id: String = self.into(); - vm.ctx.new_str(id).into() - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - let py_str = PyStrRef::try_from_object(vm, object)?; - Ok(ast::Identifier::new(py_str.as_str())) - } -} - -impl Node for ast::Int { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self.to_u32()).into() - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - let value = object.try_into_value(vm)?; - Ok(ast::Int::new(value)) - } -} - -impl Node for bool { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self as u8).into() - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - i32::try_from_object(vm, object).map(|i| i != 0) - } -} - -impl Node for ast::Constant { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::Constant::None => vm.ctx.none(), - ast::Constant::Bool(b) => vm.ctx.new_bool(b).into(), - ast::Constant::Str(s) => vm.ctx.new_str(s).into(), - ast::Constant::Bytes(b) => vm.ctx.new_bytes(b).into(), - ast::Constant::Int(i) => vm.ctx.new_int(i).into(), - ast::Constant::Tuple(t) => vm - .ctx - .new_tuple(t.into_iter().map(|c| c.ast_to_object(vm)).collect()) - .into(), - ast::Constant::Float(f) => vm.ctx.new_float(f).into(), - ast::Constant::Complex { real, imag } => vm.new_pyobj(Complex64::new(real, imag)), - ast::Constant::Ellipsis => vm.ctx.ellipsis(), - } - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - let constant = match_class!(match object { - ref i @ builtins::int::PyInt => { - let value = i.as_bigint(); - if object.class().is(vm.ctx.types.bool_type) { - ast::Constant::Bool(!value.is_zero()) - } else { - ast::Constant::Int(value.clone()) - } - } - ref f @ builtins::float::PyFloat => ast::Constant::Float(f.to_f64()), - ref c @ builtins::complex::PyComplex => { - let c = c.to_complex(); - ast::Constant::Complex { - real: c.re, - imag: c.im, - } - } - ref s @ builtins::pystr::PyStr => ast::Constant::Str(s.as_str().to_owned()), - ref b @ builtins::bytes::PyBytes => ast::Constant::Bytes(b.as_bytes().to_owned()), - ref t @ builtins::tuple::PyTuple => { - ast::Constant::Tuple( - t.iter() - .map(|elt| Self::ast_from_object(vm, elt.clone())) - .collect::<Result<_, _>>()?, - ) - } - builtins::singletons::PyNone => ast::Constant::None, - builtins::slice::PyEllipsis => ast::Constant::Ellipsis, - obj => - return Err(vm.new_type_error(format!( - "invalid type in Constant: type '{}'", - obj.class().name() - ))), - }); - Ok(constant) - } -} - -impl Node for ast::ConversionFlag { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(self as u8).into() - } - - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - i32::try_from_object(vm, object)? - .to_u32() - .and_then(ast::ConversionFlag::from_op_arg) - .ok_or_else(|| vm.new_value_error("invalid conversion flag".to_owned())) - } -} - -impl Node for ast::located::Arguments { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - self.into_python_arguments().ast_to_object(vm) - } - fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> { - ast::located::PythonArguments::ast_from_object(vm, object) - .map(ast::located::PythonArguments::into_arguments) - } -} - -#[cfg(feature = "rustpython-parser")] -pub(crate) fn parse( - vm: &VirtualMachine, - source: &str, - mode: parser::Mode, -) -> Result<PyObjectRef, CompileError> { - let mut locator = LinearLocator::new(source); - let top = parser::parse(source, mode, "<unknown>").map_err(|e| locator.locate_error(e))?; - let top = locator.fold_mod(top).unwrap(); - Ok(top.ast_to_object(vm)) -} - -#[cfg(feature = "rustpython-codegen")] -pub(crate) fn compile( - vm: &VirtualMachine, - object: PyObjectRef, - filename: &str, - mode: crate::compiler::Mode, -) -> PyResult { - let opts = vm.compile_opts(); - let ast = Node::ast_from_object(vm, object)?; - let code = codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts) - .map_err(|err| (CompileError::from(err), None).to_pyexception(vm))?; // FIXME source - Ok(vm.ctx.new_code(code).into()) -} - -// Required crate visibility for inclusion by gen.rs -pub(crate) use _ast::NodeAst; -// Used by builtins::compile() -pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400; - -pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _ast::make_module(vm); - gen::extend_module_nodes(vm, &module); - module -} diff --git a/vm/src/stdlib/ast/gen.rs b/vm/src/stdlib/ast/gen.rs deleted file mode 100644 index d3969b90246..00000000000 --- a/vm/src/stdlib/ast/gen.rs +++ /dev/null @@ -1,5496 +0,0 @@ -// File automatically generated by ast/asdl_rs.py. - -#![allow(clippy::all)] - -use super::*; -use crate::common::ascii; -#[pyclass(module = "_ast", name = "mod", base = "NodeAst")] -struct NodeMod; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeMod {} -#[pyclass(module = "_ast", name = "Module", base = "NodeMod")] -struct NodeModModule; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeModModule { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("type_ignores")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Interactive", base = "NodeMod")] -struct NodeModInteractive; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeModInteractive { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("body")).into()]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Expression", base = "NodeMod")] -struct NodeModExpression; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeModExpression { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("body")).into()]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "FunctionType", base = "NodeMod")] -struct NodeModFunctionType; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeModFunctionType { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("argtypes")).into(), - ctx.new_str(ascii!("returns")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "stmt", base = "NodeAst")] -struct NodeStmt; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmt {} -#[pyclass(module = "_ast", name = "FunctionDef", base = "NodeStmt")] -struct NodeStmtFunctionDef; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtFunctionDef { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("decorator_list")).into(), - ctx.new_str(ascii!("returns")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ctx.new_str(ascii!("type_params")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AsyncFunctionDef", base = "NodeStmt")] -struct NodeStmtAsyncFunctionDef; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAsyncFunctionDef { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("decorator_list")).into(), - ctx.new_str(ascii!("returns")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ctx.new_str(ascii!("type_params")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "ClassDef", base = "NodeStmt")] -struct NodeStmtClassDef; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtClassDef { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("bases")).into(), - ctx.new_str(ascii!("keywords")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("decorator_list")).into(), - ctx.new_str(ascii!("type_params")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Return", base = "NodeStmt")] -struct NodeStmtReturn; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtReturn { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Delete", base = "NodeStmt")] -struct NodeStmtDelete; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtDelete { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("targets")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Assign", base = "NodeStmt")] -struct NodeStmtAssign; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAssign { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("targets")).into(), - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "TypeAlias", base = "NodeStmt")] -struct NodeStmtTypeAlias; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtTypeAlias { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("type_params")).into(), - ctx.new_str(ascii!("value")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AugAssign", base = "NodeStmt")] -struct NodeStmtAugAssign; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAugAssign { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("op")).into(), - ctx.new_str(ascii!("value")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AnnAssign", base = "NodeStmt")] -struct NodeStmtAnnAssign; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAnnAssign { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("annotation")).into(), - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("simple")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "For", base = "NodeStmt")] -struct NodeStmtFor; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtFor { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("iter")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AsyncFor", base = "NodeStmt")] -struct NodeStmtAsyncFor; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAsyncFor { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("iter")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "While", base = "NodeStmt")] -struct NodeStmtWhile; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtWhile { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("test")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "If", base = "NodeStmt")] -struct NodeStmtIf; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtIf { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("test")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "With", base = "NodeStmt")] -struct NodeStmtWith; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtWith { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("items")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "AsyncWith", base = "NodeStmt")] -struct NodeStmtAsyncWith; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAsyncWith { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("items")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Match", base = "NodeStmt")] -struct NodeStmtMatch; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtMatch { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("subject")).into(), - ctx.new_str(ascii!("cases")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Raise", base = "NodeStmt")] -struct NodeStmtRaise; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtRaise { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("exc")).into(), - ctx.new_str(ascii!("cause")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Try", base = "NodeStmt")] -struct NodeStmtTry; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtTry { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("handlers")).into(), - ctx.new_str(ascii!("orelse")).into(), - ctx.new_str(ascii!("finalbody")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "TryStar", base = "NodeStmt")] -struct NodeStmtTryStar; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtTryStar { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("handlers")).into(), - ctx.new_str(ascii!("orelse")).into(), - ctx.new_str(ascii!("finalbody")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Assert", base = "NodeStmt")] -struct NodeStmtAssert; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtAssert { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("test")).into(), - ctx.new_str(ascii!("msg")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Import", base = "NodeStmt")] -struct NodeStmtImport; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtImport { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "ImportFrom", base = "NodeStmt")] -struct NodeStmtImportFrom; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtImportFrom { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("module")).into(), - ctx.new_str(ascii!("names")).into(), - ctx.new_str(ascii!("level")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Global", base = "NodeStmt")] -struct NodeStmtGlobal; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtGlobal { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Nonlocal", base = "NodeStmt")] -struct NodeStmtNonlocal; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtNonlocal { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("names")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Expr", base = "NodeStmt")] -struct NodeStmtExpr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtExpr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Pass", base = "NodeStmt")] -struct NodeStmtPass; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtPass { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Break", base = "NodeStmt")] -struct NodeStmtBreak; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtBreak { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Continue", base = "NodeStmt")] -struct NodeStmtContinue; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeStmtContinue { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "expr", base = "NodeAst")] -struct NodeExpr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExpr {} -#[pyclass(module = "_ast", name = "BoolOp", base = "NodeExpr")] -struct NodeExprBoolOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprBoolOp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("op")).into(), - ctx.new_str(ascii!("values")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "NamedExpr", base = "NodeExpr")] -struct NodeExprNamedExpr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprNamedExpr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("value")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "BinOp", base = "NodeExpr")] -struct NodeExprBinOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprBinOp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("left")).into(), - ctx.new_str(ascii!("op")).into(), - ctx.new_str(ascii!("right")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "UnaryOp", base = "NodeExpr")] -struct NodeExprUnaryOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprUnaryOp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("op")).into(), - ctx.new_str(ascii!("operand")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Lambda", base = "NodeExpr")] -struct NodeExprLambda; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprLambda { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("body")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "IfExp", base = "NodeExpr")] -struct NodeExprIfExp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprIfExp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("test")).into(), - ctx.new_str(ascii!("body")).into(), - ctx.new_str(ascii!("orelse")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Dict", base = "NodeExpr")] -struct NodeExprDict; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprDict { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("keys")).into(), - ctx.new_str(ascii!("values")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Set", base = "NodeExpr")] -struct NodeExprSet; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprSet { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("elts")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "ListComp", base = "NodeExpr")] -struct NodeExprListComp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprListComp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elt")).into(), - ctx.new_str(ascii!("generators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "SetComp", base = "NodeExpr")] -struct NodeExprSetComp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprSetComp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elt")).into(), - ctx.new_str(ascii!("generators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "DictComp", base = "NodeExpr")] -struct NodeExprDictComp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprDictComp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("key")).into(), - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("generators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "GeneratorExp", base = "NodeExpr")] -struct NodeExprGeneratorExp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprGeneratorExp { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elt")).into(), - ctx.new_str(ascii!("generators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Await", base = "NodeExpr")] -struct NodeExprAwait; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprAwait { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Yield", base = "NodeExpr")] -struct NodeExprYield; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprYield { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "YieldFrom", base = "NodeExpr")] -struct NodeExprYieldFrom; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprYieldFrom { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Compare", base = "NodeExpr")] -struct NodeExprCompare; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprCompare { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("left")).into(), - ctx.new_str(ascii!("ops")).into(), - ctx.new_str(ascii!("comparators")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Call", base = "NodeExpr")] -struct NodeExprCall; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprCall { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("func")).into(), - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("keywords")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "FormattedValue", base = "NodeExpr")] -struct NodeExprFormattedValue; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprFormattedValue { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("conversion")).into(), - ctx.new_str(ascii!("format_spec")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "JoinedStr", base = "NodeExpr")] -struct NodeExprJoinedStr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprJoinedStr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("values")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Constant", base = "NodeExpr")] -struct NodeExprConstant; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprConstant { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("kind")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Attribute", base = "NodeExpr")] -struct NodeExprAttribute; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprAttribute { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("attr")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Subscript", base = "NodeExpr")] -struct NodeExprSubscript; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprSubscript { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("slice")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Starred", base = "NodeExpr")] -struct NodeExprStarred; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprStarred { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("value")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Name", base = "NodeExpr")] -struct NodeExprName; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprName { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("id")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "List", base = "NodeExpr")] -struct NodeExprList; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprList { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elts")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Tuple", base = "NodeExpr")] -struct NodeExprTuple; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprTuple { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("elts")).into(), - ctx.new_str(ascii!("ctx")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "Slice", base = "NodeExpr")] -struct NodeExprSlice; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprSlice { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("lower")).into(), - ctx.new_str(ascii!("upper")).into(), - ctx.new_str(ascii!("step")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "expr_context", base = "NodeAst")] -struct NodeExprContext; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContext {} -#[pyclass(module = "_ast", name = "Load", base = "NodeExprContext")] -struct NodeExprContextLoad; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContextLoad { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Store", base = "NodeExprContext")] -struct NodeExprContextStore; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContextStore { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Del", base = "NodeExprContext")] -struct NodeExprContextDel; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExprContextDel { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "boolop", base = "NodeAst")] -struct NodeBoolOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeBoolOp {} -#[pyclass(module = "_ast", name = "And", base = "NodeBoolOp")] -struct NodeBoolOpAnd; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeBoolOpAnd { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Or", base = "NodeBoolOp")] -struct NodeBoolOpOr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeBoolOpOr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "operator", base = "NodeAst")] -struct NodeOperator; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperator {} -#[pyclass(module = "_ast", name = "Add", base = "NodeOperator")] -struct NodeOperatorAdd; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorAdd { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Sub", base = "NodeOperator")] -struct NodeOperatorSub; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorSub { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Mult", base = "NodeOperator")] -struct NodeOperatorMult; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorMult { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "MatMult", base = "NodeOperator")] -struct NodeOperatorMatMult; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorMatMult { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Div", base = "NodeOperator")] -struct NodeOperatorDiv; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorDiv { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Mod", base = "NodeOperator")] -struct NodeOperatorMod; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorMod { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Pow", base = "NodeOperator")] -struct NodeOperatorPow; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorPow { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "LShift", base = "NodeOperator")] -struct NodeOperatorLShift; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorLShift { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "RShift", base = "NodeOperator")] -struct NodeOperatorRShift; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorRShift { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "BitOr", base = "NodeOperator")] -struct NodeOperatorBitOr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorBitOr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "BitXor", base = "NodeOperator")] -struct NodeOperatorBitXor; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorBitXor { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "BitAnd", base = "NodeOperator")] -struct NodeOperatorBitAnd; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorBitAnd { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "FloorDiv", base = "NodeOperator")] -struct NodeOperatorFloorDiv; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeOperatorFloorDiv { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "unaryop", base = "NodeAst")] -struct NodeUnaryOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOp {} -#[pyclass(module = "_ast", name = "Invert", base = "NodeUnaryOp")] -struct NodeUnaryOpInvert; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOpInvert { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Not", base = "NodeUnaryOp")] -struct NodeUnaryOpNot; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOpNot { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "UAdd", base = "NodeUnaryOp")] -struct NodeUnaryOpUAdd; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOpUAdd { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "USub", base = "NodeUnaryOp")] -struct NodeUnaryOpUSub; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeUnaryOpUSub { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "cmpop", base = "NodeAst")] -struct NodeCmpOp; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOp {} -#[pyclass(module = "_ast", name = "Eq", base = "NodeCmpOp")] -struct NodeCmpOpEq; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpEq { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "NotEq", base = "NodeCmpOp")] -struct NodeCmpOpNotEq; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpNotEq { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Lt", base = "NodeCmpOp")] -struct NodeCmpOpLt; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpLt { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "LtE", base = "NodeCmpOp")] -struct NodeCmpOpLtE; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpLtE { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Gt", base = "NodeCmpOp")] -struct NodeCmpOpGt; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpGt { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "GtE", base = "NodeCmpOp")] -struct NodeCmpOpGtE; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpGtE { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "Is", base = "NodeCmpOp")] -struct NodeCmpOpIs; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpIs { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "IsNot", base = "NodeCmpOp")] -struct NodeCmpOpIsNot; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpIsNot { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "In", base = "NodeCmpOp")] -struct NodeCmpOpIn; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpIn { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "NotIn", base = "NodeCmpOp")] -struct NodeCmpOpNotIn; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeCmpOpNotIn { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![]).into()); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "comprehension", base = "NodeAst")] -struct NodeComprehension; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeComprehension { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("target")).into(), - ctx.new_str(ascii!("iter")).into(), - ctx.new_str(ascii!("ifs")).into(), - ctx.new_str(ascii!("is_async")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "excepthandler", base = "NodeAst")] -struct NodeExceptHandler; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExceptHandler {} -#[pyclass(module = "_ast", name = "ExceptHandler", base = "NodeExceptHandler")] -struct NodeExceptHandlerExceptHandler; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeExceptHandlerExceptHandler { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("type")).into(), - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("body")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "arguments", base = "NodeAst")] -struct NodeArguments; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeArguments { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("posonlyargs")).into(), - ctx.new_str(ascii!("args")).into(), - ctx.new_str(ascii!("vararg")).into(), - ctx.new_str(ascii!("kwonlyargs")).into(), - ctx.new_str(ascii!("kw_defaults")).into(), - ctx.new_str(ascii!("kwarg")).into(), - ctx.new_str(ascii!("defaults")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "arg", base = "NodeAst")] -struct NodeArg; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeArg { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("arg")).into(), - ctx.new_str(ascii!("annotation")).into(), - ctx.new_str(ascii!("type_comment")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "keyword", base = "NodeAst")] -struct NodeKeyword; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeKeyword { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("arg")).into(), - ctx.new_str(ascii!("value")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "alias", base = "NodeAst")] -struct NodeAlias; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeAlias { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("asname")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "withitem", base = "NodeAst")] -struct NodeWithItem; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeWithItem { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("context_expr")).into(), - ctx.new_str(ascii!("optional_vars")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "match_case", base = "NodeAst")] -struct NodeMatchCase; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeMatchCase { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("pattern")).into(), - ctx.new_str(ascii!("guard")).into(), - ctx.new_str(ascii!("body")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "pattern", base = "NodeAst")] -struct NodePattern; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePattern {} -#[pyclass(module = "_ast", name = "MatchValue", base = "NodePattern")] -struct NodePatternMatchValue; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchValue { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchSingleton", base = "NodePattern")] -struct NodePatternMatchSingleton; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchSingleton { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("value")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchSequence", base = "NodePattern")] -struct NodePatternMatchSequence; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchSequence { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("patterns")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchMapping", base = "NodePattern")] -struct NodePatternMatchMapping; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchMapping { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("keys")).into(), - ctx.new_str(ascii!("patterns")).into(), - ctx.new_str(ascii!("rest")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchClass", base = "NodePattern")] -struct NodePatternMatchClass; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchClass { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("cls")).into(), - ctx.new_str(ascii!("patterns")).into(), - ctx.new_str(ascii!("kwd_attrs")).into(), - ctx.new_str(ascii!("kwd_patterns")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchStar", base = "NodePattern")] -struct NodePatternMatchStar; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchStar { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchAs", base = "NodePattern")] -struct NodePatternMatchAs; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchAs { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("pattern")).into(), - ctx.new_str(ascii!("name")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "MatchOr", base = "NodePattern")] -struct NodePatternMatchOr; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodePatternMatchOr { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("patterns")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "type_ignore", base = "NodeAst")] -struct NodeTypeIgnore; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeIgnore {} -#[pyclass(module = "_ast", name = "TypeIgnore", base = "NodeTypeIgnore")] -struct NodeTypeIgnoreTypeIgnore; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeIgnoreTypeIgnore { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("tag")).into(), - ]) - .into(), - ); - class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![]).into()); - } -} -#[pyclass(module = "_ast", name = "type_param", base = "NodeAst")] -struct NodeTypeParam; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParam {} -#[pyclass(module = "_ast", name = "TypeVar", base = "NodeTypeParam")] -struct NodeTypeParamTypeVar; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParamTypeVar { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ - ctx.new_str(ascii!("name")).into(), - ctx.new_str(ascii!("bound")).into(), - ]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "ParamSpec", base = "NodeTypeParam")] -struct NodeTypeParamParamSpec; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParamParamSpec { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} -#[pyclass(module = "_ast", name = "TypeVarTuple", base = "NodeTypeParam")] -struct NodeTypeParamTypeVarTuple; -#[pyclass(flags(HAS_DICT, BASETYPE))] -impl NodeTypeParamTypeVarTuple { - #[extend_class] - fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) { - class.set_attr( - identifier!(ctx, _fields), - ctx.new_tuple(vec![ctx.new_str(ascii!("name")).into()]) - .into(), - ); - class.set_attr( - identifier!(ctx, _attributes), - ctx.new_list(vec![ - ctx.new_str(ascii!("lineno")).into(), - ctx.new_str(ascii!("col_offset")).into(), - ctx.new_str(ascii!("end_lineno")).into(), - ctx.new_str(ascii!("end_col_offset")).into(), - ]) - .into(), - ); - } -} - -// sum -impl Node for ast::located::Mod { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::Mod::Module(cons) => cons.ast_to_object(vm), - ast::located::Mod::Interactive(cons) => cons.ast_to_object(vm), - ast::located::Mod::Expression(cons) => cons.ast_to_object(vm), - ast::located::Mod::FunctionType(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeModModule::static_type()) { - ast::located::Mod::Module(ast::located::ModModule::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeModInteractive::static_type()) { - ast::located::Mod::Interactive(ast::located::ModInteractive::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeModExpression::static_type()) { - ast::located::Mod::Expression(ast::located::ModExpression::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeModFunctionType::static_type()) { - ast::located::Mod::FunctionType(ast::located::ModFunctionType::ast_from_object( - _vm, _object, - )?) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of mod, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::ModModule { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ModModule { - body, - type_ignores, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeModModule::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("type_ignores", type_ignores.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ModModule { - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "Module")?)?, - type_ignores: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_ignores", "Module")?, - )?, - range: Default::default(), - }) - } -} -// constructor -impl Node for ast::located::ModInteractive { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ModInteractive { - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeModInteractive::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ModInteractive { - body: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "body", "Interactive")?, - )?, - range: Default::default(), - }) - } -} -// constructor -impl Node for ast::located::ModExpression { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ModExpression { - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeModExpression::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ModExpression { - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "Expression")?)?, - range: Default::default(), - }) - } -} -// constructor -impl Node for ast::located::ModFunctionType { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ModFunctionType { - argtypes, - returns, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeModFunctionType::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("argtypes", argtypes.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("returns", returns.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ModFunctionType { - argtypes: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "argtypes", "FunctionType")?, - )?, - returns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "returns", "FunctionType")?, - )?, - range: Default::default(), - }) - } -} -// sum -impl Node for ast::located::Stmt { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::Stmt::FunctionDef(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AsyncFunctionDef(cons) => cons.ast_to_object(vm), - ast::located::Stmt::ClassDef(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Return(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Delete(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Assign(cons) => cons.ast_to_object(vm), - ast::located::Stmt::TypeAlias(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AugAssign(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AnnAssign(cons) => cons.ast_to_object(vm), - ast::located::Stmt::For(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AsyncFor(cons) => cons.ast_to_object(vm), - ast::located::Stmt::While(cons) => cons.ast_to_object(vm), - ast::located::Stmt::If(cons) => cons.ast_to_object(vm), - ast::located::Stmt::With(cons) => cons.ast_to_object(vm), - ast::located::Stmt::AsyncWith(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Match(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Raise(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Try(cons) => cons.ast_to_object(vm), - ast::located::Stmt::TryStar(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Assert(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Import(cons) => cons.ast_to_object(vm), - ast::located::Stmt::ImportFrom(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Global(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Nonlocal(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Expr(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Pass(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Break(cons) => cons.ast_to_object(vm), - ast::located::Stmt::Continue(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeStmtFunctionDef::static_type()) { - ast::located::Stmt::FunctionDef(ast::located::StmtFunctionDef::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtAsyncFunctionDef::static_type()) { - ast::located::Stmt::AsyncFunctionDef( - ast::located::StmtAsyncFunctionDef::ast_from_object(_vm, _object)?, - ) - } else if _cls.is(NodeStmtClassDef::static_type()) { - ast::located::Stmt::ClassDef(ast::located::StmtClassDef::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtReturn::static_type()) { - ast::located::Stmt::Return(ast::located::StmtReturn::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtDelete::static_type()) { - ast::located::Stmt::Delete(ast::located::StmtDelete::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtAssign::static_type()) { - ast::located::Stmt::Assign(ast::located::StmtAssign::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtTypeAlias::static_type()) { - ast::located::Stmt::TypeAlias(ast::located::StmtTypeAlias::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtAugAssign::static_type()) { - ast::located::Stmt::AugAssign(ast::located::StmtAugAssign::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtAnnAssign::static_type()) { - ast::located::Stmt::AnnAssign(ast::located::StmtAnnAssign::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtFor::static_type()) { - ast::located::Stmt::For(ast::located::StmtFor::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtAsyncFor::static_type()) { - ast::located::Stmt::AsyncFor(ast::located::StmtAsyncFor::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtWhile::static_type()) { - ast::located::Stmt::While(ast::located::StmtWhile::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtIf::static_type()) { - ast::located::Stmt::If(ast::located::StmtIf::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtWith::static_type()) { - ast::located::Stmt::With(ast::located::StmtWith::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtAsyncWith::static_type()) { - ast::located::Stmt::AsyncWith(ast::located::StmtAsyncWith::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtMatch::static_type()) { - ast::located::Stmt::Match(ast::located::StmtMatch::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtRaise::static_type()) { - ast::located::Stmt::Raise(ast::located::StmtRaise::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtTry::static_type()) { - ast::located::Stmt::Try(ast::located::StmtTry::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtTryStar::static_type()) { - ast::located::Stmt::TryStar(ast::located::StmtTryStar::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtAssert::static_type()) { - ast::located::Stmt::Assert(ast::located::StmtAssert::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtImport::static_type()) { - ast::located::Stmt::Import(ast::located::StmtImport::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtImportFrom::static_type()) { - ast::located::Stmt::ImportFrom(ast::located::StmtImportFrom::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeStmtGlobal::static_type()) { - ast::located::Stmt::Global(ast::located::StmtGlobal::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtNonlocal::static_type()) { - ast::located::Stmt::Nonlocal(ast::located::StmtNonlocal::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtExpr::static_type()) { - ast::located::Stmt::Expr(ast::located::StmtExpr::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtPass::static_type()) { - ast::located::Stmt::Pass(ast::located::StmtPass::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtBreak::static_type()) { - ast::located::Stmt::Break(ast::located::StmtBreak::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeStmtContinue::static_type()) { - ast::located::Stmt::Continue(ast::located::StmtContinue::ast_from_object(_vm, _object)?) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of stmt, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::StmtFunctionDef { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtFunctionDef { - name, - args, - body, - decorator_list, - returns, - type_comment, - type_params, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtFunctionDef::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("decorator_list", decorator_list.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("returns", returns.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_params", type_params.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtFunctionDef { - name: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "name", "FunctionDef")?, - )?, - args: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "args", "FunctionDef")?, - )?, - body: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "body", "FunctionDef")?, - )?, - decorator_list: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "decorator_list", "FunctionDef")?, - )?, - returns: get_node_field_opt(_vm, &_object, "returns")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_params: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_params", "FunctionDef")?, - )?, - range: range_from_object(_vm, _object, "FunctionDef")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAsyncFunctionDef { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAsyncFunctionDef { - name, - args, - body, - decorator_list, - returns, - type_comment, - type_params, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAsyncFunctionDef::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("decorator_list", decorator_list.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("returns", returns.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_params", type_params.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtAsyncFunctionDef { - name: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "name", "AsyncFunctionDef")?, - )?, - args: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "args", "AsyncFunctionDef")?, - )?, - body: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "body", "AsyncFunctionDef")?, - )?, - decorator_list: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "decorator_list", "AsyncFunctionDef")?, - )?, - returns: get_node_field_opt(_vm, &_object, "returns")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_params: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_params", "AsyncFunctionDef")?, - )?, - range: range_from_object(_vm, _object, "AsyncFunctionDef")?, - }) - } -} -// constructor -impl Node for ast::located::StmtClassDef { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtClassDef { - name, - bases, - keywords, - body, - decorator_list, - type_params, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtClassDef::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("bases", bases.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("keywords", keywords.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("decorator_list", decorator_list.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_params", type_params.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtClassDef { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "ClassDef")?)?, - bases: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "bases", "ClassDef")?)?, - keywords: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "keywords", "ClassDef")?, - )?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "ClassDef")?)?, - decorator_list: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "decorator_list", "ClassDef")?, - )?, - type_params: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_params", "ClassDef")?, - )?, - range: range_from_object(_vm, _object, "ClassDef")?, - }) - } -} -// constructor -impl Node for ast::located::StmtReturn { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtReturn { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtReturn::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtReturn { - value: get_node_field_opt(_vm, &_object, "value")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Return")?, - }) - } -} -// constructor -impl Node for ast::located::StmtDelete { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtDelete { - targets, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtDelete::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("targets", targets.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtDelete { - targets: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "targets", "Delete")?, - )?, - range: range_from_object(_vm, _object, "Delete")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAssign { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAssign { - targets, - value, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAssign::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("targets", targets.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtAssign { - targets: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "targets", "Assign")?, - )?, - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Assign")?)?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Assign")?, - }) - } -} -// constructor -impl Node for ast::located::StmtTypeAlias { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtTypeAlias { - name, - type_params, - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtTypeAlias::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("type_params", type_params.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtTypeAlias { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "TypeAlias")?)?, - type_params: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "type_params", "TypeAlias")?, - )?, - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "TypeAlias")?, - )?, - range: range_from_object(_vm, _object, "TypeAlias")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAugAssign { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAugAssign { - target, - op, - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAugAssign::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("op", op.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtAugAssign { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "AugAssign")?, - )?, - op: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "op", "AugAssign")?)?, - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "AugAssign")?, - )?, - range: range_from_object(_vm, _object, "AugAssign")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAnnAssign { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAnnAssign { - target, - annotation, - value, - simple, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAnnAssign::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("annotation", annotation.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("simple", simple.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtAnnAssign { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "AnnAssign")?, - )?, - annotation: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "annotation", "AnnAssign")?, - )?, - value: get_node_field_opt(_vm, &_object, "value")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - simple: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "simple", "AnnAssign")?, - )?, - range: range_from_object(_vm, _object, "AnnAssign")?, - }) - } -} -// constructor -impl Node for ast::located::StmtFor { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtFor { - target, - iter, - body, - orelse, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtFor::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("iter", iter.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtFor { - target: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "target", "For")?)?, - iter: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "iter", "For")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "For")?)?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "For")?)?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "For")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAsyncFor { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAsyncFor { - target, - iter, - body, - orelse, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAsyncFor::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("iter", iter.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtAsyncFor { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "AsyncFor")?, - )?, - iter: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "iter", "AsyncFor")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "AsyncFor")?)?, - orelse: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "orelse", "AsyncFor")?, - )?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "AsyncFor")?, - }) - } -} -// constructor -impl Node for ast::located::StmtWhile { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtWhile { - test, - body, - orelse, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtWhile::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("test", test.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtWhile { - test: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "test", "While")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "While")?)?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "While")?)?, - range: range_from_object(_vm, _object, "While")?, - }) - } -} -// constructor -impl Node for ast::located::StmtIf { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtIf { - test, - body, - orelse, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtIf::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("test", test.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtIf { - test: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "test", "If")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "If")?)?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "If")?)?, - range: range_from_object(_vm, _object, "If")?, - }) - } -} -// constructor -impl Node for ast::located::StmtWith { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtWith { - items, - body, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtWith::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("items", items.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtWith { - items: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "items", "With")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "With")?)?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "With")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAsyncWith { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAsyncWith { - items, - body, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAsyncWith::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("items", items.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtAsyncWith { - items: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "items", "AsyncWith")?, - )?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "AsyncWith")?)?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "AsyncWith")?, - }) - } -} -// constructor -impl Node for ast::located::StmtMatch { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtMatch { - subject, - cases, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtMatch::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("subject", subject.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("cases", cases.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtMatch { - subject: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "subject", "Match")?, - )?, - cases: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "cases", "Match")?)?, - range: range_from_object(_vm, _object, "Match")?, - }) - } -} -// constructor -impl Node for ast::located::StmtRaise { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtRaise { - exc, - cause, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtRaise::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("exc", exc.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("cause", cause.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtRaise { - exc: get_node_field_opt(_vm, &_object, "exc")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - cause: get_node_field_opt(_vm, &_object, "cause")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Raise")?, - }) - } -} -// constructor -impl Node for ast::located::StmtTry { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtTry { - body, - handlers, - orelse, - finalbody, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtTry::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("handlers", handlers.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("finalbody", finalbody.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtTry { - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "Try")?)?, - handlers: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "handlers", "Try")?, - )?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "Try")?)?, - finalbody: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "finalbody", "Try")?, - )?, - range: range_from_object(_vm, _object, "Try")?, - }) - } -} -// constructor -impl Node for ast::located::StmtTryStar { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtTryStar { - body, - handlers, - orelse, - finalbody, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtTryStar::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("handlers", handlers.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("finalbody", finalbody.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtTryStar { - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "TryStar")?)?, - handlers: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "handlers", "TryStar")?, - )?, - orelse: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "orelse", "TryStar")?, - )?, - finalbody: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "finalbody", "TryStar")?, - )?, - range: range_from_object(_vm, _object, "TryStar")?, - }) - } -} -// constructor -impl Node for ast::located::StmtAssert { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtAssert { - test, - msg, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtAssert::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("test", test.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("msg", msg.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtAssert { - test: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "test", "Assert")?)?, - msg: get_node_field_opt(_vm, &_object, "msg")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Assert")?, - }) - } -} -// constructor -impl Node for ast::located::StmtImport { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtImport { - names, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtImport::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("names", names.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtImport { - names: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "names", "Import")?)?, - range: range_from_object(_vm, _object, "Import")?, - }) - } -} -// constructor -impl Node for ast::located::StmtImportFrom { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtImportFrom { - module, - names, - level, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtImportFrom::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("module", module.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("names", names.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("level", level.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtImportFrom { - module: get_node_field_opt(_vm, &_object, "module")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - names: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "names", "ImportFrom")?, - )?, - level: get_node_field_opt(_vm, &_object, "level")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "ImportFrom")?, - }) - } -} -// constructor -impl Node for ast::located::StmtGlobal { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtGlobal { - names, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtGlobal::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("names", names.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtGlobal { - names: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "names", "Global")?)?, - range: range_from_object(_vm, _object, "Global")?, - }) - } -} -// constructor -impl Node for ast::located::StmtNonlocal { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtNonlocal { - names, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtNonlocal::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("names", names.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtNonlocal { - names: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "names", "Nonlocal")?)?, - range: range_from_object(_vm, _object, "Nonlocal")?, - }) - } -} -// constructor -impl Node for ast::located::StmtExpr { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtExpr { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtExpr::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtExpr { - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Expr")?)?, - range: range_from_object(_vm, _object, "Expr")?, - }) - } -} -// constructor -impl Node for ast::located::StmtPass { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtPass { range: _range } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtPass::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtPass { - range: range_from_object(_vm, _object, "Pass")?, - }) - } -} -// constructor -impl Node for ast::located::StmtBreak { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtBreak { range: _range } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtBreak::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtBreak { - range: range_from_object(_vm, _object, "Break")?, - }) - } -} -// constructor -impl Node for ast::located::StmtContinue { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::StmtContinue { range: _range } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeStmtContinue::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::StmtContinue { - range: range_from_object(_vm, _object, "Continue")?, - }) - } -} -// sum -impl Node for ast::located::Expr { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::Expr::BoolOp(cons) => cons.ast_to_object(vm), - ast::located::Expr::NamedExpr(cons) => cons.ast_to_object(vm), - ast::located::Expr::BinOp(cons) => cons.ast_to_object(vm), - ast::located::Expr::UnaryOp(cons) => cons.ast_to_object(vm), - ast::located::Expr::Lambda(cons) => cons.ast_to_object(vm), - ast::located::Expr::IfExp(cons) => cons.ast_to_object(vm), - ast::located::Expr::Dict(cons) => cons.ast_to_object(vm), - ast::located::Expr::Set(cons) => cons.ast_to_object(vm), - ast::located::Expr::ListComp(cons) => cons.ast_to_object(vm), - ast::located::Expr::SetComp(cons) => cons.ast_to_object(vm), - ast::located::Expr::DictComp(cons) => cons.ast_to_object(vm), - ast::located::Expr::GeneratorExp(cons) => cons.ast_to_object(vm), - ast::located::Expr::Await(cons) => cons.ast_to_object(vm), - ast::located::Expr::Yield(cons) => cons.ast_to_object(vm), - ast::located::Expr::YieldFrom(cons) => cons.ast_to_object(vm), - ast::located::Expr::Compare(cons) => cons.ast_to_object(vm), - ast::located::Expr::Call(cons) => cons.ast_to_object(vm), - ast::located::Expr::FormattedValue(cons) => cons.ast_to_object(vm), - ast::located::Expr::JoinedStr(cons) => cons.ast_to_object(vm), - ast::located::Expr::Constant(cons) => cons.ast_to_object(vm), - ast::located::Expr::Attribute(cons) => cons.ast_to_object(vm), - ast::located::Expr::Subscript(cons) => cons.ast_to_object(vm), - ast::located::Expr::Starred(cons) => cons.ast_to_object(vm), - ast::located::Expr::Name(cons) => cons.ast_to_object(vm), - ast::located::Expr::List(cons) => cons.ast_to_object(vm), - ast::located::Expr::Tuple(cons) => cons.ast_to_object(vm), - ast::located::Expr::Slice(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeExprBoolOp::static_type()) { - ast::located::Expr::BoolOp(ast::located::ExprBoolOp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprNamedExpr::static_type()) { - ast::located::Expr::NamedExpr(ast::located::ExprNamedExpr::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprBinOp::static_type()) { - ast::located::Expr::BinOp(ast::located::ExprBinOp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprUnaryOp::static_type()) { - ast::located::Expr::UnaryOp(ast::located::ExprUnaryOp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprLambda::static_type()) { - ast::located::Expr::Lambda(ast::located::ExprLambda::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprIfExp::static_type()) { - ast::located::Expr::IfExp(ast::located::ExprIfExp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprDict::static_type()) { - ast::located::Expr::Dict(ast::located::ExprDict::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprSet::static_type()) { - ast::located::Expr::Set(ast::located::ExprSet::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprListComp::static_type()) { - ast::located::Expr::ListComp(ast::located::ExprListComp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprSetComp::static_type()) { - ast::located::Expr::SetComp(ast::located::ExprSetComp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprDictComp::static_type()) { - ast::located::Expr::DictComp(ast::located::ExprDictComp::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprGeneratorExp::static_type()) { - ast::located::Expr::GeneratorExp(ast::located::ExprGeneratorExp::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprAwait::static_type()) { - ast::located::Expr::Await(ast::located::ExprAwait::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprYield::static_type()) { - ast::located::Expr::Yield(ast::located::ExprYield::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprYieldFrom::static_type()) { - ast::located::Expr::YieldFrom(ast::located::ExprYieldFrom::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprCompare::static_type()) { - ast::located::Expr::Compare(ast::located::ExprCompare::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprCall::static_type()) { - ast::located::Expr::Call(ast::located::ExprCall::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprFormattedValue::static_type()) { - ast::located::Expr::FormattedValue(ast::located::ExprFormattedValue::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprJoinedStr::static_type()) { - ast::located::Expr::JoinedStr(ast::located::ExprJoinedStr::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprConstant::static_type()) { - ast::located::Expr::Constant(ast::located::ExprConstant::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprAttribute::static_type()) { - ast::located::Expr::Attribute(ast::located::ExprAttribute::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprSubscript::static_type()) { - ast::located::Expr::Subscript(ast::located::ExprSubscript::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeExprStarred::static_type()) { - ast::located::Expr::Starred(ast::located::ExprStarred::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprName::static_type()) { - ast::located::Expr::Name(ast::located::ExprName::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprList::static_type()) { - ast::located::Expr::List(ast::located::ExprList::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprTuple::static_type()) { - ast::located::Expr::Tuple(ast::located::ExprTuple::ast_from_object(_vm, _object)?) - } else if _cls.is(NodeExprSlice::static_type()) { - ast::located::Expr::Slice(ast::located::ExprSlice::ast_from_object(_vm, _object)?) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of expr, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::ExprBoolOp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprBoolOp { - op, - values, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprBoolOp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("op", op.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("values", values.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprBoolOp { - op: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "op", "BoolOp")?)?, - values: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "values", "BoolOp")?)?, - range: range_from_object(_vm, _object, "BoolOp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprNamedExpr { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprNamedExpr { - target, - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprNamedExpr::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprNamedExpr { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "NamedExpr")?, - )?, - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "NamedExpr")?, - )?, - range: range_from_object(_vm, _object, "NamedExpr")?, - }) - } -} -// constructor -impl Node for ast::located::ExprBinOp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprBinOp { - left, - op, - right, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprBinOp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("left", left.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("op", op.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("right", right.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprBinOp { - left: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "left", "BinOp")?)?, - op: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "op", "BinOp")?)?, - right: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "right", "BinOp")?)?, - range: range_from_object(_vm, _object, "BinOp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprUnaryOp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprUnaryOp { - op, - operand, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprUnaryOp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("op", op.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("operand", operand.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprUnaryOp { - op: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "op", "UnaryOp")?)?, - operand: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "operand", "UnaryOp")?, - )?, - range: range_from_object(_vm, _object, "UnaryOp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprLambda { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprLambda { - args, - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprLambda::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprLambda { - args: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "args", "Lambda")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "Lambda")?)?, - range: range_from_object(_vm, _object, "Lambda")?, - }) - } -} -// constructor -impl Node for ast::located::ExprIfExp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprIfExp { - test, - body, - orelse, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprIfExp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("test", test.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("orelse", orelse.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprIfExp { - test: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "test", "IfExp")?)?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "IfExp")?)?, - orelse: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "orelse", "IfExp")?)?, - range: range_from_object(_vm, _object, "IfExp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprDict { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprDict { - keys, - values, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprDict::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("keys", keys.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("values", values.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprDict { - keys: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "keys", "Dict")?)?, - values: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "values", "Dict")?)?, - range: range_from_object(_vm, _object, "Dict")?, - }) - } -} -// constructor -impl Node for ast::located::ExprSet { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprSet { - elts, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprSet::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elts", elts.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprSet { - elts: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elts", "Set")?)?, - range: range_from_object(_vm, _object, "Set")?, - }) - } -} -// constructor -impl Node for ast::located::ExprListComp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprListComp { - elt, - generators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprListComp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elt", elt.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("generators", generators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprListComp { - elt: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elt", "ListComp")?)?, - generators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "generators", "ListComp")?, - )?, - range: range_from_object(_vm, _object, "ListComp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprSetComp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprSetComp { - elt, - generators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprSetComp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elt", elt.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("generators", generators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprSetComp { - elt: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elt", "SetComp")?)?, - generators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "generators", "SetComp")?, - )?, - range: range_from_object(_vm, _object, "SetComp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprDictComp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprDictComp { - key, - value, - generators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprDictComp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("key", key.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("generators", generators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprDictComp { - key: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "key", "DictComp")?)?, - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "DictComp")?)?, - generators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "generators", "DictComp")?, - )?, - range: range_from_object(_vm, _object, "DictComp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprGeneratorExp { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprGeneratorExp { - elt, - generators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprGeneratorExp::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elt", elt.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("generators", generators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprGeneratorExp { - elt: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elt", "GeneratorExp")?)?, - generators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "generators", "GeneratorExp")?, - )?, - range: range_from_object(_vm, _object, "GeneratorExp")?, - }) - } -} -// constructor -impl Node for ast::located::ExprAwait { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprAwait { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprAwait::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprAwait { - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Await")?)?, - range: range_from_object(_vm, _object, "Await")?, - }) - } -} -// constructor -impl Node for ast::located::ExprYield { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprYield { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprYield::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprYield { - value: get_node_field_opt(_vm, &_object, "value")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Yield")?, - }) - } -} -// constructor -impl Node for ast::located::ExprYieldFrom { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprYieldFrom { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprYieldFrom::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprYieldFrom { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "YieldFrom")?, - )?, - range: range_from_object(_vm, _object, "YieldFrom")?, - }) - } -} -// constructor -impl Node for ast::located::ExprCompare { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprCompare { - left, - ops, - comparators, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprCompare::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("left", left.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ops", ops.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("comparators", comparators.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprCompare { - left: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "left", "Compare")?)?, - ops: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ops", "Compare")?)?, - comparators: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "comparators", "Compare")?, - )?, - range: range_from_object(_vm, _object, "Compare")?, - }) - } -} -// constructor -impl Node for ast::located::ExprCall { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprCall { - func, - args, - keywords, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprCall::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("func", func.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("keywords", keywords.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprCall { - func: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "func", "Call")?)?, - args: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "args", "Call")?)?, - keywords: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "keywords", "Call")?, - )?, - range: range_from_object(_vm, _object, "Call")?, - }) - } -} -// constructor -impl Node for ast::located::ExprFormattedValue { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprFormattedValue { - value, - conversion, - format_spec, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprFormattedValue::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("conversion", conversion.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("format_spec", format_spec.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprFormattedValue { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "FormattedValue")?, - )?, - conversion: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "conversion", "FormattedValue")?, - )?, - format_spec: get_node_field_opt(_vm, &_object, "format_spec")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "FormattedValue")?, - }) - } -} -// constructor -impl Node for ast::located::ExprJoinedStr { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprJoinedStr { - values, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprJoinedStr::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("values", values.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprJoinedStr { - values: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "values", "JoinedStr")?, - )?, - range: range_from_object(_vm, _object, "JoinedStr")?, - }) - } -} -// constructor -impl Node for ast::located::ExprConstant { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprConstant { - value, - kind, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprConstant::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kind", kind.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprConstant { - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Constant")?)?, - kind: get_node_field_opt(_vm, &_object, "kind")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Constant")?, - }) - } -} -// constructor -impl Node for ast::located::ExprAttribute { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprAttribute { - value, - attr, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprAttribute::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("attr", attr.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprAttribute { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "Attribute")?, - )?, - attr: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "attr", "Attribute")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Attribute")?)?, - range: range_from_object(_vm, _object, "Attribute")?, - }) - } -} -// constructor -impl Node for ast::located::ExprSubscript { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprSubscript { - value, - slice, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprSubscript::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("slice", slice.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprSubscript { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "Subscript")?, - )?, - slice: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "slice", "Subscript")?, - )?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Subscript")?)?, - range: range_from_object(_vm, _object, "Subscript")?, - }) - } -} -// constructor -impl Node for ast::located::ExprStarred { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprStarred { - value, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprStarred::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprStarred { - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "Starred")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Starred")?)?, - range: range_from_object(_vm, _object, "Starred")?, - }) - } -} -// constructor -impl Node for ast::located::ExprName { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprName { - id, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprName::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("id", id.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprName { - id: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "id", "Name")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Name")?)?, - range: range_from_object(_vm, _object, "Name")?, - }) - } -} -// constructor -impl Node for ast::located::ExprList { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprList { - elts, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprList::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elts", elts.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprList { - elts: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elts", "List")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "List")?)?, - range: range_from_object(_vm, _object, "List")?, - }) - } -} -// constructor -impl Node for ast::located::ExprTuple { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprTuple { - elts, - ctx, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprTuple::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("elts", elts.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ctx", ctx.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprTuple { - elts: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "elts", "Tuple")?)?, - ctx: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "ctx", "Tuple")?)?, - range: range_from_object(_vm, _object, "Tuple")?, - }) - } -} -// constructor -impl Node for ast::located::ExprSlice { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExprSlice { - lower, - upper, - step, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeExprSlice::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("lower", lower.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("upper", upper.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("step", step.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExprSlice { - lower: get_node_field_opt(_vm, &_object, "lower")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - upper: get_node_field_opt(_vm, &_object, "upper")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - step: get_node_field_opt(_vm, &_object, "step")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "Slice")?, - }) - } -} -// sum -impl Node for ast::located::ExprContext { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::ExprContext::Load => NodeExprContextLoad::static_type(), - ast::located::ExprContext::Store => NodeExprContextStore::static_type(), - ast::located::ExprContext::Del => NodeExprContextDel::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeExprContextLoad::static_type()) { - ast::located::ExprContext::Load - } else if _cls.is(NodeExprContextStore::static_type()) { - ast::located::ExprContext::Store - } else if _cls.is(NodeExprContextDel::static_type()) { - ast::located::ExprContext::Del - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of expr_context, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// sum -impl Node for ast::located::BoolOp { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::BoolOp::And => NodeBoolOpAnd::static_type(), - ast::located::BoolOp::Or => NodeBoolOpOr::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeBoolOpAnd::static_type()) { - ast::located::BoolOp::And - } else if _cls.is(NodeBoolOpOr::static_type()) { - ast::located::BoolOp::Or - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of boolop, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// sum -impl Node for ast::located::Operator { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::Operator::Add => NodeOperatorAdd::static_type(), - ast::located::Operator::Sub => NodeOperatorSub::static_type(), - ast::located::Operator::Mult => NodeOperatorMult::static_type(), - ast::located::Operator::MatMult => NodeOperatorMatMult::static_type(), - ast::located::Operator::Div => NodeOperatorDiv::static_type(), - ast::located::Operator::Mod => NodeOperatorMod::static_type(), - ast::located::Operator::Pow => NodeOperatorPow::static_type(), - ast::located::Operator::LShift => NodeOperatorLShift::static_type(), - ast::located::Operator::RShift => NodeOperatorRShift::static_type(), - ast::located::Operator::BitOr => NodeOperatorBitOr::static_type(), - ast::located::Operator::BitXor => NodeOperatorBitXor::static_type(), - ast::located::Operator::BitAnd => NodeOperatorBitAnd::static_type(), - ast::located::Operator::FloorDiv => NodeOperatorFloorDiv::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeOperatorAdd::static_type()) { - ast::located::Operator::Add - } else if _cls.is(NodeOperatorSub::static_type()) { - ast::located::Operator::Sub - } else if _cls.is(NodeOperatorMult::static_type()) { - ast::located::Operator::Mult - } else if _cls.is(NodeOperatorMatMult::static_type()) { - ast::located::Operator::MatMult - } else if _cls.is(NodeOperatorDiv::static_type()) { - ast::located::Operator::Div - } else if _cls.is(NodeOperatorMod::static_type()) { - ast::located::Operator::Mod - } else if _cls.is(NodeOperatorPow::static_type()) { - ast::located::Operator::Pow - } else if _cls.is(NodeOperatorLShift::static_type()) { - ast::located::Operator::LShift - } else if _cls.is(NodeOperatorRShift::static_type()) { - ast::located::Operator::RShift - } else if _cls.is(NodeOperatorBitOr::static_type()) { - ast::located::Operator::BitOr - } else if _cls.is(NodeOperatorBitXor::static_type()) { - ast::located::Operator::BitXor - } else if _cls.is(NodeOperatorBitAnd::static_type()) { - ast::located::Operator::BitAnd - } else if _cls.is(NodeOperatorFloorDiv::static_type()) { - ast::located::Operator::FloorDiv - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of operator, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// sum -impl Node for ast::located::UnaryOp { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::UnaryOp::Invert => NodeUnaryOpInvert::static_type(), - ast::located::UnaryOp::Not => NodeUnaryOpNot::static_type(), - ast::located::UnaryOp::UAdd => NodeUnaryOpUAdd::static_type(), - ast::located::UnaryOp::USub => NodeUnaryOpUSub::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeUnaryOpInvert::static_type()) { - ast::located::UnaryOp::Invert - } else if _cls.is(NodeUnaryOpNot::static_type()) { - ast::located::UnaryOp::Not - } else if _cls.is(NodeUnaryOpUAdd::static_type()) { - ast::located::UnaryOp::UAdd - } else if _cls.is(NodeUnaryOpUSub::static_type()) { - ast::located::UnaryOp::USub - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of unaryop, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// sum -impl Node for ast::located::CmpOp { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - let node_type = match self { - ast::located::CmpOp::Eq => NodeCmpOpEq::static_type(), - ast::located::CmpOp::NotEq => NodeCmpOpNotEq::static_type(), - ast::located::CmpOp::Lt => NodeCmpOpLt::static_type(), - ast::located::CmpOp::LtE => NodeCmpOpLtE::static_type(), - ast::located::CmpOp::Gt => NodeCmpOpGt::static_type(), - ast::located::CmpOp::GtE => NodeCmpOpGtE::static_type(), - ast::located::CmpOp::Is => NodeCmpOpIs::static_type(), - ast::located::CmpOp::IsNot => NodeCmpOpIsNot::static_type(), - ast::located::CmpOp::In => NodeCmpOpIn::static_type(), - ast::located::CmpOp::NotIn => NodeCmpOpNotIn::static_type(), - }; - NodeAst - .into_ref_with_type(vm, node_type.to_owned()) - .unwrap() - .into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeCmpOpEq::static_type()) { - ast::located::CmpOp::Eq - } else if _cls.is(NodeCmpOpNotEq::static_type()) { - ast::located::CmpOp::NotEq - } else if _cls.is(NodeCmpOpLt::static_type()) { - ast::located::CmpOp::Lt - } else if _cls.is(NodeCmpOpLtE::static_type()) { - ast::located::CmpOp::LtE - } else if _cls.is(NodeCmpOpGt::static_type()) { - ast::located::CmpOp::Gt - } else if _cls.is(NodeCmpOpGtE::static_type()) { - ast::located::CmpOp::GtE - } else if _cls.is(NodeCmpOpIs::static_type()) { - ast::located::CmpOp::Is - } else if _cls.is(NodeCmpOpIsNot::static_type()) { - ast::located::CmpOp::IsNot - } else if _cls.is(NodeCmpOpIn::static_type()) { - ast::located::CmpOp::In - } else if _cls.is(NodeCmpOpNotIn::static_type()) { - ast::located::CmpOp::NotIn - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of cmpop, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// product -impl Node for ast::located::Comprehension { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::Comprehension { - target, - iter, - ifs, - is_async, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeComprehension::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("target", target.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("iter", iter.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("ifs", ifs.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("is_async", is_async.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::Comprehension { - target: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "target", "comprehension")?, - )?, - iter: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "iter", "comprehension")?, - )?, - ifs: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "ifs", "comprehension")?, - )?, - is_async: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "is_async", "comprehension")?, - )?, - range: Default::default(), - }) - } -} -// sum -impl Node for ast::located::ExceptHandler { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::ExceptHandler::ExceptHandler(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeExceptHandlerExceptHandler::static_type()) { - ast::located::ExceptHandler::ExceptHandler( - ast::located::ExceptHandlerExceptHandler::ast_from_object(_vm, _object)?, - ) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of excepthandler, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::ExceptHandlerExceptHandler { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::ExceptHandlerExceptHandler { - type_, - name, - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type( - _vm, - NodeExceptHandlerExceptHandler::static_type().to_owned(), - ) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("type", type_.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::ExceptHandlerExceptHandler { - type_: get_node_field_opt(_vm, &_object, "type")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - body: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "body", "ExceptHandler")?, - )?, - range: range_from_object(_vm, _object, "ExceptHandler")?, - }) - } -} -// product -impl Node for ast::located::PythonArguments { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PythonArguments { - posonlyargs, - args, - vararg, - kwonlyargs, - kw_defaults, - kwarg, - defaults, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeArguments::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("posonlyargs", posonlyargs.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("args", args.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("vararg", vararg.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kwonlyargs", kwonlyargs.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kw_defaults", kw_defaults.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kwarg", kwarg.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("defaults", defaults.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PythonArguments { - posonlyargs: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "posonlyargs", "arguments")?, - )?, - args: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "args", "arguments")?)?, - vararg: get_node_field_opt(_vm, &_object, "vararg")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - kwonlyargs: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "kwonlyargs", "arguments")?, - )?, - kw_defaults: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "kw_defaults", "arguments")?, - )?, - kwarg: get_node_field_opt(_vm, &_object, "kwarg")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - defaults: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "defaults", "arguments")?, - )?, - range: Default::default(), - }) - } -} -// product -impl Node for ast::located::Arg { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::Arg { - arg, - annotation, - type_comment, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeArg::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("arg", arg.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("annotation", annotation.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("type_comment", type_comment.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::Arg { - arg: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "arg", "arg")?)?, - annotation: get_node_field_opt(_vm, &_object, "annotation")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - type_comment: get_node_field_opt(_vm, &_object, "type_comment")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "arg")?, - }) - } -} -// product -impl Node for ast::located::Keyword { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::Keyword { - arg, - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeKeyword::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("arg", arg.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::Keyword { - arg: get_node_field_opt(_vm, &_object, "arg")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - value: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "value", "keyword")?)?, - range: range_from_object(_vm, _object, "keyword")?, - }) - } -} -// product -impl Node for ast::located::Alias { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::Alias { - name, - asname, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeAlias::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("asname", asname.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::Alias { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "alias")?)?, - asname: get_node_field_opt(_vm, &_object, "asname")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "alias")?, - }) - } -} -// product -impl Node for ast::located::WithItem { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::WithItem { - context_expr, - optional_vars, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeWithItem::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("context_expr", context_expr.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("optional_vars", optional_vars.ast_to_object(_vm), _vm) - .unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::WithItem { - context_expr: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "context_expr", "withitem")?, - )?, - optional_vars: get_node_field_opt(_vm, &_object, "optional_vars")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: Default::default(), - }) - } -} -// product -impl Node for ast::located::MatchCase { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::MatchCase { - pattern, - guard, - body, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeMatchCase::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("pattern", pattern.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("guard", guard.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("body", body.ast_to_object(_vm), _vm).unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::MatchCase { - pattern: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "pattern", "match_case")?, - )?, - guard: get_node_field_opt(_vm, &_object, "guard")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - body: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "body", "match_case")?)?, - range: Default::default(), - }) - } -} -// sum -impl Node for ast::located::Pattern { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::Pattern::MatchValue(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchSingleton(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchSequence(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchMapping(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchClass(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchStar(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchAs(cons) => cons.ast_to_object(vm), - ast::located::Pattern::MatchOr(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodePatternMatchValue::static_type()) { - ast::located::Pattern::MatchValue(ast::located::PatternMatchValue::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchSingleton::static_type()) { - ast::located::Pattern::MatchSingleton( - ast::located::PatternMatchSingleton::ast_from_object(_vm, _object)?, - ) - } else if _cls.is(NodePatternMatchSequence::static_type()) { - ast::located::Pattern::MatchSequence( - ast::located::PatternMatchSequence::ast_from_object(_vm, _object)?, - ) - } else if _cls.is(NodePatternMatchMapping::static_type()) { - ast::located::Pattern::MatchMapping(ast::located::PatternMatchMapping::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchClass::static_type()) { - ast::located::Pattern::MatchClass(ast::located::PatternMatchClass::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchStar::static_type()) { - ast::located::Pattern::MatchStar(ast::located::PatternMatchStar::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchAs::static_type()) { - ast::located::Pattern::MatchAs(ast::located::PatternMatchAs::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodePatternMatchOr::static_type()) { - ast::located::Pattern::MatchOr(ast::located::PatternMatchOr::ast_from_object( - _vm, _object, - )?) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of pattern, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::PatternMatchValue { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchValue { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchValue::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PatternMatchValue { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "MatchValue")?, - )?, - range: range_from_object(_vm, _object, "MatchValue")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchSingleton { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchSingleton { - value, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchSingleton::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("value", value.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PatternMatchSingleton { - value: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "value", "MatchSingleton")?, - )?, - range: range_from_object(_vm, _object, "MatchSingleton")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchSequence { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchSequence { - patterns, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchSequence::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PatternMatchSequence { - patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "patterns", "MatchSequence")?, - )?, - range: range_from_object(_vm, _object, "MatchSequence")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchMapping { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchMapping { - keys, - patterns, - rest, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchMapping::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("keys", keys.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("rest", rest.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PatternMatchMapping { - keys: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "keys", "MatchMapping")?, - )?, - patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "patterns", "MatchMapping")?, - )?, - rest: get_node_field_opt(_vm, &_object, "rest")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "MatchMapping")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchClass { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchClass { - cls, - patterns, - kwd_attrs, - kwd_patterns, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchClass::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("cls", cls.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kwd_attrs", kwd_attrs.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("kwd_patterns", kwd_patterns.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PatternMatchClass { - cls: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "cls", "MatchClass")?)?, - patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "patterns", "MatchClass")?, - )?, - kwd_attrs: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "kwd_attrs", "MatchClass")?, - )?, - kwd_patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "kwd_patterns", "MatchClass")?, - )?, - range: range_from_object(_vm, _object, "MatchClass")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchStar { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchStar { - name, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchStar::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PatternMatchStar { - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "MatchStar")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchAs { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchAs { - pattern, - name, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchAs::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("pattern", pattern.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PatternMatchAs { - pattern: get_node_field_opt(_vm, &_object, "pattern")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - name: get_node_field_opt(_vm, &_object, "name")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "MatchAs")?, - }) - } -} -// constructor -impl Node for ast::located::PatternMatchOr { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::PatternMatchOr { - patterns, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodePatternMatchOr::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("patterns", patterns.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::PatternMatchOr { - patterns: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "patterns", "MatchOr")?, - )?, - range: range_from_object(_vm, _object, "MatchOr")?, - }) - } -} -// sum -impl Node for ast::located::TypeIgnore { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::TypeIgnore::TypeIgnore(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeTypeIgnoreTypeIgnore::static_type()) { - ast::located::TypeIgnore::TypeIgnore( - ast::located::TypeIgnoreTypeIgnore::ast_from_object(_vm, _object)?, - ) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of type_ignore, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::TypeIgnoreTypeIgnore { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::TypeIgnoreTypeIgnore { - lineno, - tag, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeTypeIgnoreTypeIgnore::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("lineno", lineno.ast_to_object(_vm), _vm) - .unwrap(); - dict.set_item("tag", tag.ast_to_object(_vm), _vm).unwrap(); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::TypeIgnoreTypeIgnore { - lineno: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "lineno", "TypeIgnore")?, - )?, - tag: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "tag", "TypeIgnore")?)?, - range: Default::default(), - }) - } -} -// sum -impl Node for ast::located::TypeParam { - fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef { - match self { - ast::located::TypeParam::TypeVar(cons) => cons.ast_to_object(vm), - ast::located::TypeParam::ParamSpec(cons) => cons.ast_to_object(vm), - ast::located::TypeParam::TypeVarTuple(cons) => cons.ast_to_object(vm), - } - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - let _cls = _object.class(); - Ok(if _cls.is(NodeTypeParamTypeVar::static_type()) { - ast::located::TypeParam::TypeVar(ast::located::TypeParamTypeVar::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeTypeParamParamSpec::static_type()) { - ast::located::TypeParam::ParamSpec(ast::located::TypeParamParamSpec::ast_from_object( - _vm, _object, - )?) - } else if _cls.is(NodeTypeParamTypeVarTuple::static_type()) { - ast::located::TypeParam::TypeVarTuple( - ast::located::TypeParamTypeVarTuple::ast_from_object(_vm, _object)?, - ) - } else { - return Err(_vm.new_type_error(format!( - "expected some sort of type_param, but got {}", - _object.repr(_vm)? - ))); - }) - } -} -// constructor -impl Node for ast::located::TypeParamTypeVar { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::TypeParamTypeVar { - name, - bound, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeTypeParamTypeVar::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - dict.set_item("bound", bound.ast_to_object(_vm), _vm) - .unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::TypeParamTypeVar { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "TypeVar")?)?, - bound: get_node_field_opt(_vm, &_object, "bound")? - .map(|obj| Node::ast_from_object(_vm, obj)) - .transpose()?, - range: range_from_object(_vm, _object, "TypeVar")?, - }) - } -} -// constructor -impl Node for ast::located::TypeParamParamSpec { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::TypeParamParamSpec { - name, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeTypeParamParamSpec::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::TypeParamParamSpec { - name: Node::ast_from_object(_vm, get_node_field(_vm, &_object, "name", "ParamSpec")?)?, - range: range_from_object(_vm, _object, "ParamSpec")?, - }) - } -} -// constructor -impl Node for ast::located::TypeParamTypeVarTuple { - fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef { - let ast::located::TypeParamTypeVarTuple { - name, - range: _range, - } = self; - let node = NodeAst - .into_ref_with_type(_vm, NodeTypeParamTypeVarTuple::static_type().to_owned()) - .unwrap(); - let dict = node.as_object().dict().unwrap(); - dict.set_item("name", name.ast_to_object(_vm), _vm).unwrap(); - node_add_location(&dict, _range, _vm); - node.into() - } - fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> { - Ok(ast::located::TypeParamTypeVarTuple { - name: Node::ast_from_object( - _vm, - get_node_field(_vm, &_object, "name", "TypeVarTuple")?, - )?, - range: range_from_object(_vm, _object, "TypeVarTuple")?, - }) - } -} - -pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) { - extend_module!(vm, module, { - "mod" => NodeMod::make_class(&vm.ctx), - "Module" => NodeModModule::make_class(&vm.ctx), - "Interactive" => NodeModInteractive::make_class(&vm.ctx), - "Expression" => NodeModExpression::make_class(&vm.ctx), - "FunctionType" => NodeModFunctionType::make_class(&vm.ctx), - "stmt" => NodeStmt::make_class(&vm.ctx), - "FunctionDef" => NodeStmtFunctionDef::make_class(&vm.ctx), - "AsyncFunctionDef" => NodeStmtAsyncFunctionDef::make_class(&vm.ctx), - "ClassDef" => NodeStmtClassDef::make_class(&vm.ctx), - "Return" => NodeStmtReturn::make_class(&vm.ctx), - "Delete" => NodeStmtDelete::make_class(&vm.ctx), - "Assign" => NodeStmtAssign::make_class(&vm.ctx), - "TypeAlias" => NodeStmtTypeAlias::make_class(&vm.ctx), - "AugAssign" => NodeStmtAugAssign::make_class(&vm.ctx), - "AnnAssign" => NodeStmtAnnAssign::make_class(&vm.ctx), - "For" => NodeStmtFor::make_class(&vm.ctx), - "AsyncFor" => NodeStmtAsyncFor::make_class(&vm.ctx), - "While" => NodeStmtWhile::make_class(&vm.ctx), - "If" => NodeStmtIf::make_class(&vm.ctx), - "With" => NodeStmtWith::make_class(&vm.ctx), - "AsyncWith" => NodeStmtAsyncWith::make_class(&vm.ctx), - "Match" => NodeStmtMatch::make_class(&vm.ctx), - "Raise" => NodeStmtRaise::make_class(&vm.ctx), - "Try" => NodeStmtTry::make_class(&vm.ctx), - "TryStar" => NodeStmtTryStar::make_class(&vm.ctx), - "Assert" => NodeStmtAssert::make_class(&vm.ctx), - "Import" => NodeStmtImport::make_class(&vm.ctx), - "ImportFrom" => NodeStmtImportFrom::make_class(&vm.ctx), - "Global" => NodeStmtGlobal::make_class(&vm.ctx), - "Nonlocal" => NodeStmtNonlocal::make_class(&vm.ctx), - "Expr" => NodeStmtExpr::make_class(&vm.ctx), - "Pass" => NodeStmtPass::make_class(&vm.ctx), - "Break" => NodeStmtBreak::make_class(&vm.ctx), - "Continue" => NodeStmtContinue::make_class(&vm.ctx), - "expr" => NodeExpr::make_class(&vm.ctx), - "BoolOp" => NodeExprBoolOp::make_class(&vm.ctx), - "NamedExpr" => NodeExprNamedExpr::make_class(&vm.ctx), - "BinOp" => NodeExprBinOp::make_class(&vm.ctx), - "UnaryOp" => NodeExprUnaryOp::make_class(&vm.ctx), - "Lambda" => NodeExprLambda::make_class(&vm.ctx), - "IfExp" => NodeExprIfExp::make_class(&vm.ctx), - "Dict" => NodeExprDict::make_class(&vm.ctx), - "Set" => NodeExprSet::make_class(&vm.ctx), - "ListComp" => NodeExprListComp::make_class(&vm.ctx), - "SetComp" => NodeExprSetComp::make_class(&vm.ctx), - "DictComp" => NodeExprDictComp::make_class(&vm.ctx), - "GeneratorExp" => NodeExprGeneratorExp::make_class(&vm.ctx), - "Await" => NodeExprAwait::make_class(&vm.ctx), - "Yield" => NodeExprYield::make_class(&vm.ctx), - "YieldFrom" => NodeExprYieldFrom::make_class(&vm.ctx), - "Compare" => NodeExprCompare::make_class(&vm.ctx), - "Call" => NodeExprCall::make_class(&vm.ctx), - "FormattedValue" => NodeExprFormattedValue::make_class(&vm.ctx), - "JoinedStr" => NodeExprJoinedStr::make_class(&vm.ctx), - "Constant" => NodeExprConstant::make_class(&vm.ctx), - "Attribute" => NodeExprAttribute::make_class(&vm.ctx), - "Subscript" => NodeExprSubscript::make_class(&vm.ctx), - "Starred" => NodeExprStarred::make_class(&vm.ctx), - "Name" => NodeExprName::make_class(&vm.ctx), - "List" => NodeExprList::make_class(&vm.ctx), - "Tuple" => NodeExprTuple::make_class(&vm.ctx), - "Slice" => NodeExprSlice::make_class(&vm.ctx), - "expr_context" => NodeExprContext::make_class(&vm.ctx), - "Load" => NodeExprContextLoad::make_class(&vm.ctx), - "Store" => NodeExprContextStore::make_class(&vm.ctx), - "Del" => NodeExprContextDel::make_class(&vm.ctx), - "boolop" => NodeBoolOp::make_class(&vm.ctx), - "And" => NodeBoolOpAnd::make_class(&vm.ctx), - "Or" => NodeBoolOpOr::make_class(&vm.ctx), - "operator" => NodeOperator::make_class(&vm.ctx), - "Add" => NodeOperatorAdd::make_class(&vm.ctx), - "Sub" => NodeOperatorSub::make_class(&vm.ctx), - "Mult" => NodeOperatorMult::make_class(&vm.ctx), - "MatMult" => NodeOperatorMatMult::make_class(&vm.ctx), - "Div" => NodeOperatorDiv::make_class(&vm.ctx), - "Mod" => NodeOperatorMod::make_class(&vm.ctx), - "Pow" => NodeOperatorPow::make_class(&vm.ctx), - "LShift" => NodeOperatorLShift::make_class(&vm.ctx), - "RShift" => NodeOperatorRShift::make_class(&vm.ctx), - "BitOr" => NodeOperatorBitOr::make_class(&vm.ctx), - "BitXor" => NodeOperatorBitXor::make_class(&vm.ctx), - "BitAnd" => NodeOperatorBitAnd::make_class(&vm.ctx), - "FloorDiv" => NodeOperatorFloorDiv::make_class(&vm.ctx), - "unaryop" => NodeUnaryOp::make_class(&vm.ctx), - "Invert" => NodeUnaryOpInvert::make_class(&vm.ctx), - "Not" => NodeUnaryOpNot::make_class(&vm.ctx), - "UAdd" => NodeUnaryOpUAdd::make_class(&vm.ctx), - "USub" => NodeUnaryOpUSub::make_class(&vm.ctx), - "cmpop" => NodeCmpOp::make_class(&vm.ctx), - "Eq" => NodeCmpOpEq::make_class(&vm.ctx), - "NotEq" => NodeCmpOpNotEq::make_class(&vm.ctx), - "Lt" => NodeCmpOpLt::make_class(&vm.ctx), - "LtE" => NodeCmpOpLtE::make_class(&vm.ctx), - "Gt" => NodeCmpOpGt::make_class(&vm.ctx), - "GtE" => NodeCmpOpGtE::make_class(&vm.ctx), - "Is" => NodeCmpOpIs::make_class(&vm.ctx), - "IsNot" => NodeCmpOpIsNot::make_class(&vm.ctx), - "In" => NodeCmpOpIn::make_class(&vm.ctx), - "NotIn" => NodeCmpOpNotIn::make_class(&vm.ctx), - "comprehension" => NodeComprehension::make_class(&vm.ctx), - "excepthandler" => NodeExceptHandler::make_class(&vm.ctx), - "ExceptHandler" => NodeExceptHandlerExceptHandler::make_class(&vm.ctx), - "arguments" => NodeArguments::make_class(&vm.ctx), - "arg" => NodeArg::make_class(&vm.ctx), - "keyword" => NodeKeyword::make_class(&vm.ctx), - "alias" => NodeAlias::make_class(&vm.ctx), - "withitem" => NodeWithItem::make_class(&vm.ctx), - "match_case" => NodeMatchCase::make_class(&vm.ctx), - "pattern" => NodePattern::make_class(&vm.ctx), - "MatchValue" => NodePatternMatchValue::make_class(&vm.ctx), - "MatchSingleton" => NodePatternMatchSingleton::make_class(&vm.ctx), - "MatchSequence" => NodePatternMatchSequence::make_class(&vm.ctx), - "MatchMapping" => NodePatternMatchMapping::make_class(&vm.ctx), - "MatchClass" => NodePatternMatchClass::make_class(&vm.ctx), - "MatchStar" => NodePatternMatchStar::make_class(&vm.ctx), - "MatchAs" => NodePatternMatchAs::make_class(&vm.ctx), - "MatchOr" => NodePatternMatchOr::make_class(&vm.ctx), - "type_ignore" => NodeTypeIgnore::make_class(&vm.ctx), - "TypeIgnore" => NodeTypeIgnoreTypeIgnore::make_class(&vm.ctx), - "type_param" => NodeTypeParam::make_class(&vm.ctx), - "TypeVar" => NodeTypeParamTypeVar::make_class(&vm.ctx), - "ParamSpec" => NodeTypeParamParamSpec::make_class(&vm.ctx), - "TypeVarTuple" => NodeTypeParamTypeVarTuple::make_class(&vm.ctx), - }) -} diff --git a/vm/src/stdlib/atexit.rs b/vm/src/stdlib/atexit.rs deleted file mode 100644 index dbeda767415..00000000000 --- a/vm/src/stdlib/atexit.rs +++ /dev/null @@ -1,53 +0,0 @@ -pub use atexit::_run_exitfuncs; -pub(crate) use atexit::make_module; - -#[pymodule] -mod atexit { - use crate::{function::FuncArgs, AsObject, PyObjectRef, PyResult, VirtualMachine}; - - #[pyfunction] - fn register(func: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { - vm.state.atexit_funcs.lock().push((func.clone(), args)); - func - } - - #[pyfunction] - fn _clear(vm: &VirtualMachine) { - vm.state.atexit_funcs.lock().clear(); - } - - #[pyfunction] - fn unregister(func: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut funcs = vm.state.atexit_funcs.lock(); - - let mut i = 0; - while i < funcs.len() { - if vm.bool_eq(&funcs[i].0, &func)? { - funcs.remove(i); - } else { - i += 1; - } - } - - Ok(()) - } - - #[pyfunction] - pub fn _run_exitfuncs(vm: &VirtualMachine) { - let funcs: Vec<_> = std::mem::take(&mut *vm.state.atexit_funcs.lock()); - for (func, args) in funcs.into_iter().rev() { - if let Err(e) = func.call(args, vm) { - let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); - vm.run_unraisable(e, Some("Error in atexit._run_exitfuncs".to_owned()), func); - if exit { - break; - } - } - } - } - - #[pyfunction] - fn _ncallbacks(vm: &VirtualMachine) -> usize { - vm.state.atexit_funcs.lock().len() - } -} diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs deleted file mode 100644 index 4cc3a536345..00000000000 --- a/vm/src/stdlib/builtins.rs +++ /dev/null @@ -1,1058 +0,0 @@ -//! Builtin function definitions. -//! -//! Implements the list of [builtin Python functions](https://docs.python.org/3/library/builtins.html). -use crate::{builtins::PyModule, class::PyClassImpl, Py, VirtualMachine}; -pub(crate) use builtins::{__module_def, DOC}; -pub use builtins::{ascii, print, reversed}; - -#[pymodule] -mod builtins { - use crate::{ - builtins::{ - enumerate::PyReverseSequenceIterator, - function::{PyCellRef, PyFunction}, - int::PyIntRef, - iter::PyCallableIterator, - list::{PyList, SortOptions}, - PyByteArray, PyBytes, PyDictRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, - }, - common::{hash::PyHash, str::to_ascii}, - convert::ToPyException, - function::{ - ArgBytesLike, ArgCallable, ArgIndex, ArgIntoBool, ArgIterable, ArgMapping, - ArgStrOrBytesLike, Either, FsPath, FuncArgs, KwArgs, OptionalArg, OptionalOption, - PosArgs, - }, - protocol::{PyIter, PyIterReturn}, - py_io, - readline::{Readline, ReadlineResult}, - stdlib::sys, - types::PyComparisonOp, - AsObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - }; - use num_traits::{Signed, ToPrimitive}; - - #[cfg(not(feature = "rustpython-compiler"))] - const CODEGEN_NOT_SUPPORTED: &str = - "can't compile() to bytecode when the `codegen` feature of rustpython is disabled"; - - #[pyfunction] - fn abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._abs(&x) - } - - #[pyfunction] - fn all(iterable: ArgIterable<ArgIntoBool>, vm: &VirtualMachine) -> PyResult<bool> { - for item in iterable.iter(vm)? { - if !*item? { - return Ok(false); - } - } - Ok(true) - } - - #[pyfunction] - fn any(iterable: ArgIterable<ArgIntoBool>, vm: &VirtualMachine) -> PyResult<bool> { - for item in iterable.iter(vm)? { - if *item? { - return Ok(true); - } - } - Ok(false) - } - - #[pyfunction] - pub fn ascii(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<ascii::AsciiString> { - let repr = obj.repr(vm)?; - let ascii = to_ascii(repr.as_str()); - Ok(ascii) - } - - #[pyfunction] - fn bin(x: PyIntRef) -> String { - let x = x.as_bigint(); - if x.is_negative() { - format!("-0b{:b}", x.abs()) - } else { - format!("0b{x:b}") - } - } - - #[pyfunction] - fn callable(obj: PyObjectRef) -> bool { - obj.is_callable() - } - - #[pyfunction] - fn chr(i: PyIntRef, vm: &VirtualMachine) -> PyResult<String> { - let value = i - .try_to_primitive::<isize>(vm)? - .to_u32() - .and_then(char::from_u32) - .ok_or_else(|| vm.new_value_error("chr() arg not in range(0x110000)".to_owned()))?; - Ok(value.to_string()) - } - - #[derive(FromArgs)] - #[allow(dead_code)] - struct CompileArgs { - source: PyObjectRef, - filename: FsPath, - mode: PyStrRef, - #[pyarg(any, optional)] - flags: OptionalArg<PyIntRef>, - #[pyarg(any, optional)] - dont_inherit: OptionalArg<bool>, - #[pyarg(any, optional)] - optimize: OptionalArg<PyIntRef>, - #[pyarg(any, optional)] - _feature_version: OptionalArg<i32>, - } - - #[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))] - #[pyfunction] - fn compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult { - #[cfg(feature = "rustpython-ast")] - { - use crate::{class::PyClassImpl, stdlib::ast}; - - if args._feature_version.is_present() { - // TODO: add support for _feature_version - } - - let mode_str = args.mode.as_str(); - - if args - .source - .fast_isinstance(&ast::NodeAst::make_class(&vm.ctx)) - { - #[cfg(not(feature = "rustpython-codegen"))] - { - return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())); - } - #[cfg(feature = "rustpython-codegen")] - { - let mode = mode_str - .parse::<crate::compiler::Mode>() - .map_err(|err| vm.new_value_error(err.to_string()))?; - return ast::compile(vm, args.source, args.filename.as_str(), mode); - } - } - - #[cfg(not(feature = "rustpython-parser"))] - { - const PARSER_NOT_SUPPORTED: &str = - "can't compile() source code when the `parser` feature of rustpython is disabled"; - Err(vm.new_type_error(PARSER_NOT_SUPPORTED.to_owned())) - } - #[cfg(feature = "rustpython-parser")] - { - use crate::builtins::PyBytesRef; - use num_traits::Zero; - use rustpython_parser as parser; - - let source = Either::<PyStrRef, PyBytesRef>::try_from_object(vm, args.source)?; - // TODO: compiler::compile should probably get bytes - let source = match &source { - Either::A(string) => string.as_str(), - Either::B(bytes) => std::str::from_utf8(bytes) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?, - }; - - let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; - - if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() { - #[cfg(not(feature = "rustpython-compiler"))] - { - Err(vm.new_value_error(CODEGEN_NOT_SUPPORTED.to_owned())) - } - #[cfg(feature = "rustpython-compiler")] - { - let mode = mode_str - .parse::<crate::compiler::Mode>() - .map_err(|err| vm.new_value_error(err.to_string()))?; - let code = vm - .compile(source, mode, args.filename.as_str().to_owned()) - .map_err(|err| (err, Some(source)).to_pyexception(vm))?; - Ok(code.into()) - } - } else { - let mode = mode_str - .parse::<parser::Mode>() - .map_err(|err| vm.new_value_error(err.to_string()))?; - ast::parse(vm, source, mode).map_err(|e| (e, Some(source)).to_pyexception(vm)) - } - } - } - } - - #[pyfunction] - fn delattr(obj: PyObjectRef, attr: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { - vm.new_type_error(format!( - "attribute name must be string, not '{}'", - attr.class().name() - )) - })?; - obj.del_attr(attr, vm) - } - - #[pyfunction] - fn dir(obj: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<PyList> { - vm.dir(obj.into_option()) - } - - #[pyfunction] - fn divmod(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._divmod(&a, &b) - } - - #[derive(FromArgs)] - struct ScopeArgs { - #[pyarg(any, default)] - globals: Option<PyDictRef>, - #[pyarg(any, default)] - locals: Option<ArgMapping>, - } - - impl ScopeArgs { - fn make_scope(self, vm: &VirtualMachine) -> PyResult<crate::scope::Scope> { - let (globals, locals) = match self.globals { - Some(globals) => { - if !globals.contains_key(identifier!(vm, __builtins__), vm) { - let builtins_dict = vm.builtins.dict().into(); - globals.set_item(identifier!(vm, __builtins__), builtins_dict, vm)?; - } - ( - globals.clone(), - self.locals.unwrap_or_else(|| { - ArgMapping::try_from_object(vm, globals.into()).unwrap() - }), - ) - } - None => ( - vm.current_globals().clone(), - if let Some(locals) = self.locals { - locals - } else { - vm.current_locals()? - }, - ), - }; - - let scope = crate::scope::Scope::with_builtins(Some(locals), globals, vm); - Ok(scope) - } - } - - #[pyfunction] - fn eval( - source: Either<ArgStrOrBytesLike, PyRef<crate::builtins::PyCode>>, - scope: ScopeArgs, - vm: &VirtualMachine, - ) -> PyResult { - // source as string - let code = match source { - Either::A(either) => { - let source: &[u8] = &either.borrow_bytes(); - if source.contains(&0) { - return Err(vm.new_exception_msg( - vm.ctx.exceptions.syntax_error.to_owned(), - "source code string cannot contain null bytes".to_owned(), - )); - } - - let source = std::str::from_utf8(source).map_err(|err| { - let msg = format!( - "(unicode error) 'utf-8' codec can't decode byte 0x{:x?} in position {}: invalid start byte", - source[err.valid_up_to()], - err.valid_up_to() - ); - - vm.new_exception_msg(vm.ctx.exceptions.syntax_error.to_owned(), msg) - })?; - Ok(Either::A(vm.ctx.new_str(source.trim_start()))) - } - Either::B(code) => Ok(Either::B(code)), - }?; - run_code(vm, code, scope, crate::compiler::Mode::Eval, "eval") - } - - #[pyfunction] - fn exec( - source: Either<PyStrRef, PyRef<crate::builtins::PyCode>>, - scope: ScopeArgs, - vm: &VirtualMachine, - ) -> PyResult { - run_code(vm, source, scope, crate::compiler::Mode::Exec, "exec") - } - - fn run_code( - vm: &VirtualMachine, - source: Either<PyStrRef, PyRef<crate::builtins::PyCode>>, - scope: ScopeArgs, - #[allow(unused_variables)] mode: crate::compiler::Mode, - func: &str, - ) -> PyResult { - let scope = scope.make_scope(vm)?; - - // Determine code object: - let code_obj = match source { - #[cfg(feature = "rustpython-compiler")] - Either::A(string) => vm - .compile(string.as_str(), mode, "<string>".to_owned()) - .map_err(|err| vm.new_syntax_error(&err, Some(string.as_str())))?, - #[cfg(not(feature = "rustpython-compiler"))] - Either::A(_) => return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())), - Either::B(code_obj) => code_obj, - }; - - if !code_obj.freevars.is_empty() { - return Err(vm.new_type_error(format!( - "code object passed to {func}() may not contain free variables" - ))); - } - - // Run the code: - vm.run_code_obj(code_obj, scope) - } - - #[pyfunction] - fn format( - value: PyObjectRef, - format_spec: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<PyStrRef> { - vm.format(&value, format_spec.unwrap_or(vm.ctx.new_str(""))) - } - - #[pyfunction] - fn getattr( - obj: PyObjectRef, - attr: PyObjectRef, - default: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { - vm.new_type_error(format!( - "attribute name must be string, not '{}'", - attr.class().name() - )) - })?; - - if let OptionalArg::Present(default) = default { - Ok(vm.get_attribute_opt(obj, attr)?.unwrap_or(default)) - } else { - obj.get_attr(attr, vm) - } - } - - #[pyfunction] - fn globals(vm: &VirtualMachine) -> PyDictRef { - vm.current_globals().clone() - } - - #[pyfunction] - fn hasattr(obj: PyObjectRef, attr: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { - vm.new_type_error(format!( - "attribute name must be string, not '{}'", - attr.class().name() - )) - })?; - Ok(vm.get_attribute_opt(obj, attr)?.is_some()) - } - - #[pyfunction] - fn hash(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyHash> { - obj.hash(vm) - } - - #[pyfunction] - fn breakpoint(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - match vm - .sys_module - .get_attr(vm.ctx.intern_str("breakpointhook"), vm) - { - Ok(hook) => hook.as_ref().call(args, vm), - Err(_) => Err(vm.new_runtime_error("lost sys.breakpointhook".to_owned())), - } - } - - #[pyfunction] - fn hex(number: ArgIndex) -> String { - let n = number.as_bigint(); - format!("{n:#x}") - } - - #[pyfunction] - fn id(obj: PyObjectRef) -> usize { - obj.get_id() - } - - #[pyfunction] - fn input(prompt: OptionalArg<PyStrRef>, vm: &VirtualMachine) -> PyResult { - let stdin = sys::get_stdin(vm)?; - let stdout = sys::get_stdout(vm)?; - let stderr = sys::get_stderr(vm)?; - - let _ = vm.call_method(&stderr, "flush", ()); - - let fd_matches = |obj, expected| { - vm.call_method(obj, "fileno", ()) - .and_then(|o| i64::try_from_object(vm, o)) - .ok() - .map_or(false, |fd| fd == expected) - }; - - // everything is normalish, we can just rely on rustyline to use stdin/stdout - if fd_matches(&stdin, 0) && fd_matches(&stdout, 1) && atty::is(atty::Stream::Stdin) { - let prompt = prompt.as_ref().map_or("", |s| s.as_str()); - let mut readline = Readline::new(()); - match readline.readline(prompt) { - ReadlineResult::Line(s) => Ok(vm.ctx.new_str(s).into()), - ReadlineResult::Eof => { - Err(vm.new_exception_empty(vm.ctx.exceptions.eof_error.to_owned())) - } - ReadlineResult::Interrupt => { - Err(vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned())) - } - ReadlineResult::Io(e) => Err(vm.new_os_error(e.to_string())), - ReadlineResult::Other(e) => Err(vm.new_runtime_error(e.to_string())), - } - } else { - if let OptionalArg::Present(prompt) = prompt { - vm.call_method(&stdout, "write", (prompt,))?; - } - let _ = vm.call_method(&stdout, "flush", ()); - py_io::file_readline(&stdin, None, vm) - } - } - - #[pyfunction] - fn isinstance(obj: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - obj.is_instance(&typ, vm) - } - - #[pyfunction] - fn issubclass(subclass: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - subclass.is_subclass(&typ, vm) - } - - #[pyfunction] - fn iter( - iter_target: PyObjectRef, - sentinel: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyIter> { - if let OptionalArg::Present(sentinel) = sentinel { - let callable = ArgCallable::try_from_object(vm, iter_target)?; - let iterator = PyCallableIterator::new(callable, sentinel) - .into_ref(&vm.ctx) - .into(); - Ok(PyIter::new(iterator)) - } else { - iter_target.get_iter(vm) - } - } - - #[pyfunction] - fn aiter(iter_target: PyObjectRef, vm: &VirtualMachine) -> PyResult { - iter_target.get_aiter(vm) - } - - #[pyfunction] - fn len(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - obj.length(vm) - } - - #[pyfunction] - fn locals(vm: &VirtualMachine) -> PyResult<ArgMapping> { - vm.current_locals() - } - - fn min_or_max( - mut args: FuncArgs, - vm: &VirtualMachine, - func_name: &str, - op: PyComparisonOp, - ) -> PyResult { - let default = args.take_keyword("default"); - let key_func = args.take_keyword("key"); - - if let Some(err) = args.check_kwargs_empty(vm) { - return Err(err); - } - - let candidates = match args.args.len().cmp(&1) { - std::cmp::Ordering::Greater => { - if default.is_some() { - return Err(vm.new_type_error(format!( - "Cannot specify a default for {func_name}() with multiple positional arguments" - ))); - } - args.args - } - std::cmp::Ordering::Equal => args.args[0].try_to_value(vm)?, - std::cmp::Ordering::Less => { - // zero arguments means type error: - return Err( - vm.new_type_error(format!("{func_name} expected at least 1 argument, got 0")) - ); - } - }; - - let mut candidates_iter = candidates.into_iter(); - let mut x = match candidates_iter.next() { - Some(x) => x, - None => { - return default.ok_or_else(|| { - vm.new_value_error(format!("{func_name}() arg is an empty sequence")) - }) - } - }; - - let key_func = key_func.filter(|f| !vm.is_none(f)); - if let Some(ref key_func) = key_func { - let mut x_key = key_func.call((x.clone(),), vm)?; - for y in candidates_iter { - let y_key = key_func.call((y.clone(),), vm)?; - if y_key.rich_compare_bool(&x_key, op, vm)? { - x = y; - x_key = y_key; - } - } - } else { - for y in candidates_iter { - if y.rich_compare_bool(&x, op, vm)? { - x = y; - } - } - } - - Ok(x) - } - - #[pyfunction] - fn max(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - min_or_max(args, vm, "max", PyComparisonOp::Gt) - } - - #[pyfunction] - fn min(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - min_or_max(args, vm, "min", PyComparisonOp::Lt) - } - - #[pyfunction] - fn next( - iterator: PyObjectRef, - default_value: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn> { - if !PyIter::check(&iterator) { - return Err(vm.new_type_error(format!( - "{} object is not an iterator", - iterator.class().name() - ))); - } - PyIter::new(iterator).next(vm).map(|iret| match iret { - PyIterReturn::Return(obj) => PyIterReturn::Return(obj), - PyIterReturn::StopIteration(v) => { - default_value.map_or(PyIterReturn::StopIteration(v), PyIterReturn::Return) - } - }) - } - - #[pyfunction] - fn oct(number: ArgIndex, vm: &VirtualMachine) -> PyResult { - let n = number.as_bigint(); - let s = if n.is_negative() { - format!("-0o{:o}", n.abs()) - } else { - format!("0o{n:o}") - }; - - Ok(vm.ctx.new_str(s).into()) - } - - #[pyfunction] - fn ord(string: Either<ArgBytesLike, PyStrRef>, vm: &VirtualMachine) -> PyResult<u32> { - match string { - Either::A(bytes) => bytes.with_ref(|bytes| { - let bytes_len = bytes.len(); - if bytes_len != 1 { - return Err(vm.new_type_error(format!( - "ord() expected a character, but string of length {bytes_len} found" - ))); - } - Ok(u32::from(bytes[0])) - }), - Either::B(string) => { - let string = string.as_str(); - let string_len = string.chars().count(); - if string_len != 1 { - return Err(vm.new_type_error(format!( - "ord() expected a character, but string of length {string_len} found" - ))); - } - match string.chars().next() { - Some(character) => Ok(character as u32), - None => Err(vm.new_type_error( - "ord() could not guess the integer representing this character".to_owned(), - )), - } - } - } - } - - #[derive(FromArgs)] - struct PowArgs { - base: PyObjectRef, - exp: PyObjectRef, - #[pyarg(any, optional, name = "mod")] - modulus: Option<PyObjectRef>, - } - - #[pyfunction] - fn pow(args: PowArgs, vm: &VirtualMachine) -> PyResult { - let PowArgs { - base: x, - exp: y, - modulus, - } = args; - let modulus = modulus.as_ref().map_or(vm.ctx.none.as_object(), |m| m); - vm._pow(&x, &y, modulus) - } - - #[pyfunction] - pub fn exit(exit_code_arg: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult { - let code = exit_code_arg.unwrap_or_else(|| vm.ctx.new_int(0).into()); - Err(vm.new_exception(vm.ctx.exceptions.system_exit.to_owned(), vec![code])) - } - - #[derive(Debug, Default, FromArgs)] - pub struct PrintOptions { - #[pyarg(named, default)] - sep: Option<PyStrRef>, - #[pyarg(named, default)] - end: Option<PyStrRef>, - #[pyarg(named, default = "ArgIntoBool::FALSE")] - flush: ArgIntoBool, - #[pyarg(named, default)] - file: Option<PyObjectRef>, - } - - #[pyfunction] - pub fn print(objects: PosArgs, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { - let file = match options.file { - Some(f) => f, - None => sys::get_stdout(vm)?, - }; - let write = |obj: PyStrRef| vm.call_method(&file, "write", (obj,)); - - let sep = options - .sep - .unwrap_or_else(|| PyStr::from(" ").into_ref(&vm.ctx)); - - let mut first = true; - for object in objects { - if first { - first = false; - } else { - write(sep.clone())?; - } - - write(object.str(vm)?)?; - } - - let end = options - .end - .unwrap_or_else(|| PyStr::from("\n").into_ref(&vm.ctx)); - write(end)?; - - if *options.flush { - vm.call_method(&file, "flush", ())?; - } - - Ok(()) - } - - #[pyfunction] - fn repr(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - obj.repr(vm) - } - - #[pyfunction] - pub fn reversed(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if let Some(reversed_method) = vm.get_method(obj.clone(), identifier!(vm, __reversed__)) { - reversed_method?.call((), vm) - } else { - vm.get_method_or_type_error(obj.clone(), identifier!(vm, __getitem__), || { - "argument to reversed() must be a sequence".to_owned() - })?; - let len = obj.length(vm)?; - let obj_iterator = PyReverseSequenceIterator::new(obj, len); - Ok(obj_iterator.into_pyobject(vm)) - } - } - - #[derive(FromArgs)] - pub struct RoundArgs { - number: PyObjectRef, - #[pyarg(any, optional)] - ndigits: OptionalOption<PyObjectRef>, - } - - #[pyfunction] - fn round(RoundArgs { number, ndigits }: RoundArgs, vm: &VirtualMachine) -> PyResult { - let meth = vm - .get_special_method(&number, identifier!(vm, __round__))? - .ok_or_else(|| { - vm.new_type_error(format!( - "type {} doesn't define __round__", - number.class().name() - )) - })?; - match ndigits.flatten() { - Some(obj) => { - let ndigits = obj.try_index(vm)?; - meth.invoke((ndigits,), vm) - } - None => { - // without a parameter, the result type is coerced to int - meth.invoke((), vm) - } - } - } - - #[pyfunction] - fn setattr( - obj: PyObjectRef, - attr: PyObjectRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let attr = attr.try_to_ref::<PyStr>(vm).map_err(|_e| { - vm.new_type_error(format!( - "attribute name must be string, not '{}'", - attr.class().name() - )) - })?; - obj.set_attr(attr, value, vm)?; - Ok(()) - } - - // builtin_slice - - #[pyfunction] - fn sorted(iterable: PyObjectRef, opts: SortOptions, vm: &VirtualMachine) -> PyResult<PyList> { - let items: Vec<_> = iterable.try_to_value(vm)?; - let lst = PyList::from(items); - lst.sort(opts, vm)?; - Ok(lst) - } - - #[derive(FromArgs)] - pub struct SumArgs { - #[pyarg(positional)] - iterable: ArgIterable, - #[pyarg(any, optional)] - start: OptionalArg<PyObjectRef>, - } - - #[pyfunction] - fn sum(SumArgs { iterable, start }: SumArgs, vm: &VirtualMachine) -> PyResult { - // Start with zero and add at will: - let mut sum = start - .into_option() - .unwrap_or_else(|| vm.ctx.new_int(0).into()); - - match_class!(match sum { - PyStr => - return Err(vm.new_type_error( - "sum() can't sum strings [use ''.join(seq) instead]".to_owned() - )), - PyBytes => - return Err(vm.new_type_error( - "sum() can't sum bytes [use b''.join(seq) instead]".to_owned() - )), - PyByteArray => - return Err(vm.new_type_error( - "sum() can't sum bytearray [use b''.join(seq) instead]".to_owned() - )), - _ => (), - }); - - for item in iterable.iter(vm)? { - sum = vm._add(&sum, &*item?)?; - } - Ok(sum) - } - - #[pyfunction] - fn __import__(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - vm.import_func.call(args, vm) - } - - #[pyfunction] - fn vars(obj: OptionalArg, vm: &VirtualMachine) -> PyResult { - if let OptionalArg::Present(obj) = obj { - obj.get_attr(identifier!(vm, __dict__), vm).map_err(|_| { - vm.new_type_error("vars() argument must have __dict__ attribute".to_owned()) - }) - } else { - Ok(vm.current_locals()?.into()) - } - } - - #[pyfunction] - pub fn __build_class__( - function: PyRef<PyFunction>, - qualified_name: PyStrRef, - bases: PosArgs, - mut kwargs: KwArgs, - vm: &VirtualMachine, - ) -> PyResult { - let name = qualified_name.as_str().split('.').next_back().unwrap(); - let name_obj = vm.ctx.new_str(name); - - // Update bases. - let mut new_bases: Option<Vec<PyObjectRef>> = None; - let bases = PyTuple::new_ref(bases.into_vec(), &vm.ctx); - for (i, base) in bases.iter().enumerate() { - if base.fast_isinstance(vm.ctx.types.type_type) { - if let Some(bases) = &mut new_bases { - bases.push(base.clone()); - } - continue; - } - let mro_entries = - vm.get_attribute_opt(base.clone(), identifier!(vm, __mro_entries__))?; - let entries = match mro_entries { - Some(meth) => meth.call((bases.clone(),), vm)?, - None => { - if let Some(bases) = &mut new_bases { - bases.push(base.clone()); - } - continue; - } - }; - let entries: PyTupleRef = entries - .downcast() - .map_err(|_| vm.new_type_error("__mro_entries__ must return a tuple".to_owned()))?; - let new_bases = new_bases.get_or_insert_with(|| bases[..i].to_vec()); - new_bases.extend_from_slice(&entries); - } - - let new_bases = new_bases.map(|v| PyTuple::new_ref(v, &vm.ctx)); - let (orig_bases, bases) = match new_bases { - Some(new) => (Some(bases), new), - None => (None, bases), - }; - - // Use downcast_exact to keep ref to old object on error. - let metaclass = kwargs - .pop_kwarg("metaclass") - .map(|metaclass| { - metaclass - .downcast_exact::<PyType>(vm) - .map(|m| m.into_pyref()) - }) - .unwrap_or_else(|| Ok(vm.ctx.types.type_type.to_owned())); - - let (metaclass, meta_name) = match metaclass { - Ok(mut metaclass) => { - for base in bases.iter() { - let base_class = base.class(); - if base_class.fast_issubclass(&metaclass) { - metaclass = base.class().to_owned(); - } else if !metaclass.fast_issubclass(base_class) { - return Err(vm.new_type_error( - "metaclass conflict: the metaclass of a derived class must be a (non-strict) \ - subclass of the metaclasses of all its bases" - .to_owned(), - )); - } - } - let meta_name = metaclass.slot_name(); - (metaclass.to_owned().into(), meta_name.to_owned()) - } - Err(obj) => (obj, "<metaclass>".to_owned()), - }; - - let bases: PyObjectRef = bases.into(); - - // Prepare uses full __getattribute__ resolution chain. - let namespace = vm - .get_attribute_opt(metaclass.clone(), identifier!(vm, __prepare__))? - .map_or(Ok(vm.ctx.new_dict().into()), |prepare| { - let args = - FuncArgs::new(vec![name_obj.clone().into(), bases.clone()], kwargs.clone()); - prepare.call(args, vm) - })?; - - // Accept any PyMapping as namespace. - let namespace = ArgMapping::try_from_object(vm, namespace.clone()).map_err(|_| { - vm.new_type_error(format!( - "{}.__prepare__() must return a mapping, not {}", - meta_name, - namespace.class() - )) - })?; - - let classcell = function.invoke_with_locals(().into(), Some(namespace.clone()), vm)?; - let classcell = <Option<PyCellRef>>::try_from_object(vm, classcell)?; - - if let Some(orig_bases) = orig_bases { - namespace.as_object().set_item( - identifier!(vm, __orig_bases__), - orig_bases.into(), - vm, - )?; - } - - let args = FuncArgs::new(vec![name_obj.into(), bases, namespace.into()], kwargs); - let class = metaclass.call(args, vm)?; - - if let Some(ref classcell) = classcell { - let classcell = classcell.get().ok_or_else(|| { - vm.new_type_error(format!( - "__class__ not set defining {meta_name:?} as {class:?}. Was __classcell__ propagated to type.__new__?" - )) - })?; - - if !classcell.is(&class) { - return Err(vm.new_type_error(format!( - "__class__ set to {classcell:?} defining {meta_name:?} as {class:?}" - ))); - } - } - - Ok(class) - } -} - -pub fn init_module(vm: &VirtualMachine, module: &Py<PyModule>) { - let ctx = &vm.ctx; - - crate::protocol::VecBuffer::make_class(&vm.ctx); - - builtins::extend_module(vm, module).unwrap(); - - let debug_mode: bool = vm.state.settings.optimize == 0; - extend_module!(vm, module, { - "__debug__" => ctx.new_bool(debug_mode), - - "bool" => ctx.types.bool_type.to_owned(), - "bytearray" => ctx.types.bytearray_type.to_owned(), - "bytes" => ctx.types.bytes_type.to_owned(), - "classmethod" => ctx.types.classmethod_type.to_owned(), - "complex" => ctx.types.complex_type.to_owned(), - "dict" => ctx.types.dict_type.to_owned(), - "enumerate" => ctx.types.enumerate_type.to_owned(), - "float" => ctx.types.float_type.to_owned(), - "frozenset" => ctx.types.frozenset_type.to_owned(), - "filter" => ctx.types.filter_type.to_owned(), - "int" => ctx.types.int_type.to_owned(), - "list" => ctx.types.list_type.to_owned(), - "map" => ctx.types.map_type.to_owned(), - "memoryview" => ctx.types.memoryview_type.to_owned(), - "object" => ctx.types.object_type.to_owned(), - "property" => ctx.types.property_type.to_owned(), - "range" => ctx.types.range_type.to_owned(), - "set" => ctx.types.set_type.to_owned(), - "slice" => ctx.types.slice_type.to_owned(), - "staticmethod" => ctx.types.staticmethod_type.to_owned(), - "str" => ctx.types.str_type.to_owned(), - "super" => ctx.types.super_type.to_owned(), - "tuple" => ctx.types.tuple_type.to_owned(), - "type" => ctx.types.type_type.to_owned(), - "zip" => ctx.types.zip_type.to_owned(), - - // Constants - "None" => ctx.none(), - "True" => ctx.new_bool(true), - "False" => ctx.new_bool(false), - "NotImplemented" => ctx.not_implemented(), - "Ellipsis" => vm.ctx.ellipsis.clone(), - - // ordered by exception_hierarchy.txt - // Exceptions: - "BaseException" => ctx.exceptions.base_exception_type.to_owned(), - "BaseExceptionGroup" => ctx.exceptions.base_exception_group.to_owned(), - "SystemExit" => ctx.exceptions.system_exit.to_owned(), - "KeyboardInterrupt" => ctx.exceptions.keyboard_interrupt.to_owned(), - "GeneratorExit" => ctx.exceptions.generator_exit.to_owned(), - "Exception" => ctx.exceptions.exception_type.to_owned(), - "StopIteration" => ctx.exceptions.stop_iteration.to_owned(), - "StopAsyncIteration" => ctx.exceptions.stop_async_iteration.to_owned(), - "ArithmeticError" => ctx.exceptions.arithmetic_error.to_owned(), - "FloatingPointError" => ctx.exceptions.floating_point_error.to_owned(), - "OverflowError" => ctx.exceptions.overflow_error.to_owned(), - "ZeroDivisionError" => ctx.exceptions.zero_division_error.to_owned(), - "AssertionError" => ctx.exceptions.assertion_error.to_owned(), - "AttributeError" => ctx.exceptions.attribute_error.to_owned(), - "BufferError" => ctx.exceptions.buffer_error.to_owned(), - "EOFError" => ctx.exceptions.eof_error.to_owned(), - "ImportError" => ctx.exceptions.import_error.to_owned(), - "ModuleNotFoundError" => ctx.exceptions.module_not_found_error.to_owned(), - "LookupError" => ctx.exceptions.lookup_error.to_owned(), - "IndexError" => ctx.exceptions.index_error.to_owned(), - "KeyError" => ctx.exceptions.key_error.to_owned(), - "MemoryError" => ctx.exceptions.memory_error.to_owned(), - "NameError" => ctx.exceptions.name_error.to_owned(), - "UnboundLocalError" => ctx.exceptions.unbound_local_error.to_owned(), - "OSError" => ctx.exceptions.os_error.to_owned(), - // OSError alias - "IOError" => ctx.exceptions.os_error.to_owned(), - "EnvironmentError" => ctx.exceptions.os_error.to_owned(), - "BlockingIOError" => ctx.exceptions.blocking_io_error.to_owned(), - "ChildProcessError" => ctx.exceptions.child_process_error.to_owned(), - "ConnectionError" => ctx.exceptions.connection_error.to_owned(), - "BrokenPipeError" => ctx.exceptions.broken_pipe_error.to_owned(), - "ConnectionAbortedError" => ctx.exceptions.connection_aborted_error.to_owned(), - "ConnectionRefusedError" => ctx.exceptions.connection_refused_error.to_owned(), - "ConnectionResetError" => ctx.exceptions.connection_reset_error.to_owned(), - "FileExistsError" => ctx.exceptions.file_exists_error.to_owned(), - "FileNotFoundError" => ctx.exceptions.file_not_found_error.to_owned(), - "InterruptedError" => ctx.exceptions.interrupted_error.to_owned(), - "IsADirectoryError" => ctx.exceptions.is_a_directory_error.to_owned(), - "NotADirectoryError" => ctx.exceptions.not_a_directory_error.to_owned(), - "PermissionError" => ctx.exceptions.permission_error.to_owned(), - "ProcessLookupError" => ctx.exceptions.process_lookup_error.to_owned(), - "TimeoutError" => ctx.exceptions.timeout_error.to_owned(), - "ReferenceError" => ctx.exceptions.reference_error.to_owned(), - "RuntimeError" => ctx.exceptions.runtime_error.to_owned(), - "NotImplementedError" => ctx.exceptions.not_implemented_error.to_owned(), - "RecursionError" => ctx.exceptions.recursion_error.to_owned(), - "SyntaxError" => ctx.exceptions.syntax_error.to_owned(), - "IndentationError" => ctx.exceptions.indentation_error.to_owned(), - "TabError" => ctx.exceptions.tab_error.to_owned(), - "SystemError" => ctx.exceptions.system_error.to_owned(), - "TypeError" => ctx.exceptions.type_error.to_owned(), - "ValueError" => ctx.exceptions.value_error.to_owned(), - "UnicodeError" => ctx.exceptions.unicode_error.to_owned(), - "UnicodeDecodeError" => ctx.exceptions.unicode_decode_error.to_owned(), - "UnicodeEncodeError" => ctx.exceptions.unicode_encode_error.to_owned(), - "UnicodeTranslateError" => ctx.exceptions.unicode_translate_error.to_owned(), - - // Warnings - "Warning" => ctx.exceptions.warning.to_owned(), - "DeprecationWarning" => ctx.exceptions.deprecation_warning.to_owned(), - "PendingDeprecationWarning" => ctx.exceptions.pending_deprecation_warning.to_owned(), - "RuntimeWarning" => ctx.exceptions.runtime_warning.to_owned(), - "SyntaxWarning" => ctx.exceptions.syntax_warning.to_owned(), - "UserWarning" => ctx.exceptions.user_warning.to_owned(), - "FutureWarning" => ctx.exceptions.future_warning.to_owned(), - "ImportWarning" => ctx.exceptions.import_warning.to_owned(), - "UnicodeWarning" => ctx.exceptions.unicode_warning.to_owned(), - "BytesWarning" => ctx.exceptions.bytes_warning.to_owned(), - "ResourceWarning" => ctx.exceptions.resource_warning.to_owned(), - "EncodingWarning" => ctx.exceptions.encoding_warning.to_owned(), - }); - - #[cfg(feature = "jit")] - extend_module!(vm, module, { - "JitError" => ctx.exceptions.jit_error.to_owned(), - }); -} diff --git a/vm/src/stdlib/codecs.rs b/vm/src/stdlib/codecs.rs deleted file mode 100644 index 349122fcd38..00000000000 --- a/vm/src/stdlib/codecs.rs +++ /dev/null @@ -1,457 +0,0 @@ -pub(crate) use _codecs::make_module; - -#[pymodule] -mod _codecs { - use crate::common::encodings; - use crate::{ - builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyStr, PyStrRef, PyTuple}, - codecs, - function::{ArgBytesLike, FuncArgs}, - AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, - }; - use std::ops::Range; - - #[pyfunction] - fn register(search_function: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - vm.state.codec_registry.register(search_function, vm) - } - - #[pyfunction] - fn unregister(search_function: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - vm.state.codec_registry.unregister(search_function) - } - - #[pyfunction] - fn lookup(encoding: PyStrRef, vm: &VirtualMachine) -> PyResult { - vm.state - .codec_registry - .lookup(encoding.as_str(), vm) - .map(|codec| codec.into_tuple().into()) - } - - #[derive(FromArgs)] - struct CodeArgs { - obj: PyObjectRef, - #[pyarg(any, optional)] - encoding: Option<PyStrRef>, - #[pyarg(any, optional)] - errors: Option<PyStrRef>, - } - - #[pyfunction] - fn encode(args: CodeArgs, vm: &VirtualMachine) -> PyResult { - let encoding = args - .encoding - .as_ref() - .map_or(codecs::DEFAULT_ENCODING, |s| s.as_str()); - vm.state - .codec_registry - .encode(args.obj, encoding, args.errors, vm) - } - - #[pyfunction] - fn decode(args: CodeArgs, vm: &VirtualMachine) -> PyResult { - let encoding = args - .encoding - .as_ref() - .map_or(codecs::DEFAULT_ENCODING, |s| s.as_str()); - vm.state - .codec_registry - .decode(args.obj, encoding, args.errors, vm) - } - - #[pyfunction] - fn _forget_codec(encoding: PyStrRef, vm: &VirtualMachine) { - vm.state.codec_registry.forget(encoding.as_str()); - } - - #[pyfunction] - fn register_error(name: PyStrRef, handler: PyObjectRef, vm: &VirtualMachine) { - vm.state - .codec_registry - .register_error(name.as_str().to_owned(), handler); - } - - #[pyfunction] - fn lookup_error(name: PyStrRef, vm: &VirtualMachine) -> PyResult { - vm.state.codec_registry.lookup_error(name.as_str(), vm) - } - - struct ErrorsHandler<'a> { - vm: &'a VirtualMachine, - encoding: &'a str, - errors: Option<PyStrRef>, - handler: once_cell::unsync::OnceCell<PyObjectRef>, - } - impl<'a> ErrorsHandler<'a> { - #[inline] - fn new(encoding: &'a str, errors: Option<PyStrRef>, vm: &'a VirtualMachine) -> Self { - ErrorsHandler { - vm, - encoding, - errors, - handler: Default::default(), - } - } - #[inline] - fn handler_func(&self) -> PyResult<&PyObject> { - let vm = self.vm; - Ok(self.handler.get_or_try_init(|| { - let errors = self.errors.as_ref().map_or("strict", |s| s.as_str()); - vm.state.codec_registry.lookup_error(errors, vm) - })?) - } - } - impl encodings::StrBuffer for PyStrRef { - fn is_ascii(&self) -> bool { - PyStr::is_ascii(self) - } - } - impl<'vm> encodings::ErrorHandler for ErrorsHandler<'vm> { - type Error = PyBaseExceptionRef; - type StrBuf = PyStrRef; - type BytesBuf = PyBytesRef; - - fn handle_encode_error( - &self, - data: &str, - char_range: Range<usize>, - reason: &str, - ) -> PyResult<(encodings::EncodeReplace<PyStrRef, PyBytesRef>, usize)> { - let vm = self.vm; - let data_str = vm.ctx.new_str(data).into(); - let encode_exc = vm.new_exception( - vm.ctx.exceptions.unicode_encode_error.to_owned(), - vec![ - vm.ctx.new_str(self.encoding).into(), - data_str, - vm.ctx.new_int(char_range.start).into(), - vm.ctx.new_int(char_range.end).into(), - vm.ctx.new_str(reason).into(), - ], - ); - let res = self.handler_func()?.call((encode_exc,), vm)?; - let tuple_err = || { - vm.new_type_error( - "encoding error handler must return (str/bytes, int) tuple".to_owned(), - ) - }; - let (replace, restart) = match res.payload::<PyTuple>().map(|tup| tup.as_slice()) { - Some([replace, restart]) => (replace.clone(), restart), - _ => return Err(tuple_err()), - }; - let replace = match_class!(match replace { - s @ PyStr => encodings::EncodeReplace::Str(s), - b @ PyBytes => encodings::EncodeReplace::Bytes(b), - _ => return Err(tuple_err()), - }); - let restart = isize::try_from_borrowed_object(vm, restart).map_err(|_| tuple_err())?; - let restart = if restart < 0 { - // will still be out of bounds if it underflows ¯\_(ツ)_/¯ - data.len().wrapping_sub(restart.unsigned_abs()) - } else { - restart as usize - }; - Ok((replace, restart)) - } - - fn handle_decode_error( - &self, - data: &[u8], - byte_range: Range<usize>, - reason: &str, - ) -> PyResult<(PyStrRef, Option<PyBytesRef>, usize)> { - let vm = self.vm; - let data_bytes: PyObjectRef = vm.ctx.new_bytes(data.to_vec()).into(); - let decode_exc = vm.new_exception( - vm.ctx.exceptions.unicode_decode_error.to_owned(), - vec![ - vm.ctx.new_str(self.encoding).into(), - data_bytes.clone(), - vm.ctx.new_int(byte_range.start).into(), - vm.ctx.new_int(byte_range.end).into(), - vm.ctx.new_str(reason).into(), - ], - ); - let handler = self.handler_func()?; - let res = handler.call((decode_exc.clone(),), vm)?; - let new_data = decode_exc - .get_arg(1) - .ok_or_else(|| vm.new_type_error("object attribute not set".to_owned()))?; - let new_data = if new_data.is(&data_bytes) { - None - } else { - let new_data: PyBytesRef = new_data - .downcast() - .map_err(|_| vm.new_type_error("object attribute must be bytes".to_owned()))?; - Some(new_data) - }; - let data = new_data.as_ref().map_or(data, |s| s.as_ref()); - let tuple_err = || { - vm.new_type_error("decoding error handler must return (str, int) tuple".to_owned()) - }; - match res.payload::<PyTuple>().map(|tup| tup.as_slice()) { - Some([replace, restart]) => { - let replace = replace - .downcast_ref::<PyStr>() - .ok_or_else(tuple_err)? - .to_owned(); - let restart = - isize::try_from_borrowed_object(vm, restart).map_err(|_| tuple_err())?; - let restart = if restart < 0 { - // will still be out of bounds if it underflows ¯\_(ツ)_/¯ - data.len().wrapping_sub(restart.unsigned_abs()) - } else { - restart as usize - }; - Ok((replace, new_data, restart)) - } - _ => Err(tuple_err()), - } - } - - fn error_oob_restart(&self, i: usize) -> PyBaseExceptionRef { - self.vm - .new_index_error(format!("position {i} from error handler out of bounds")) - } - - fn error_encoding( - &self, - data: &str, - char_range: Range<usize>, - reason: &str, - ) -> Self::Error { - let vm = self.vm; - vm.new_exception( - vm.ctx.exceptions.unicode_encode_error.to_owned(), - vec![ - vm.ctx.new_str(self.encoding).into(), - vm.ctx.new_str(data).into(), - vm.ctx.new_int(char_range.start).into(), - vm.ctx.new_int(char_range.end).into(), - vm.ctx.new_str(reason).into(), - ], - ) - } - } - - type EncodeResult = PyResult<(Vec<u8>, usize)>; - - #[derive(FromArgs)] - struct EncodeArgs { - #[pyarg(positional)] - s: PyStrRef, - #[pyarg(positional, optional)] - errors: Option<PyStrRef>, - } - - impl EncodeArgs { - #[inline] - fn encode<'a, F>(self, name: &'a str, encode: F, vm: &'a VirtualMachine) -> EncodeResult - where - F: FnOnce(&str, &ErrorsHandler<'a>) -> PyResult<Vec<u8>>, - { - let errors = ErrorsHandler::new(name, self.errors, vm); - let encoded = encode(self.s.as_str(), &errors)?; - Ok((encoded, self.s.char_len())) - } - } - - type DecodeResult = PyResult<(String, usize)>; - - #[derive(FromArgs)] - struct DecodeArgs { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg(positional, optional)] - errors: Option<PyStrRef>, - #[pyarg(positional, default = "false")] - final_decode: bool, - } - - impl DecodeArgs { - #[inline] - fn decode<'a, F>(self, name: &'a str, decode: F, vm: &'a VirtualMachine) -> DecodeResult - where - F: FnOnce(&[u8], &ErrorsHandler<'a>, bool) -> DecodeResult, - { - let data = self.data.borrow_buf(); - let errors = ErrorsHandler::new(name, self.errors, vm); - decode(&data, &errors, self.final_decode) - } - } - - #[derive(FromArgs)] - struct DecodeArgsNoFinal { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg(positional, optional)] - errors: Option<PyStrRef>, - } - - impl DecodeArgsNoFinal { - #[inline] - fn decode<'a, F>(self, name: &'a str, decode: F, vm: &'a VirtualMachine) -> DecodeResult - where - F: FnOnce(&[u8], &ErrorsHandler<'a>) -> DecodeResult, - { - let data = self.data.borrow_buf(); - let errors = ErrorsHandler::new(name, self.errors, vm); - decode(&data, &errors) - } - } - - macro_rules! do_codec { - ($module:ident :: $func:ident, $args: expr, $vm:expr) => {{ - use encodings::$module as codec; - $args.$func(codec::ENCODING_NAME, codec::$func, $vm) - }}; - } - - #[pyfunction] - fn utf_8_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { - do_codec!(utf8::encode, args, vm) - } - - #[pyfunction] - fn utf_8_decode(args: DecodeArgs, vm: &VirtualMachine) -> DecodeResult { - do_codec!(utf8::decode, args, vm) - } - - #[pyfunction] - fn latin_1_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { - if args.s.is_ascii() { - return Ok((args.s.as_str().as_bytes().to_vec(), args.s.byte_len())); - } - do_codec!(latin_1::encode, args, vm) - } - - #[pyfunction] - fn latin_1_decode(args: DecodeArgsNoFinal, vm: &VirtualMachine) -> DecodeResult { - do_codec!(latin_1::decode, args, vm) - } - - #[pyfunction] - fn ascii_encode(args: EncodeArgs, vm: &VirtualMachine) -> EncodeResult { - if args.s.is_ascii() { - return Ok((args.s.as_str().as_bytes().to_vec(), args.s.byte_len())); - } - do_codec!(ascii::encode, args, vm) - } - - #[pyfunction] - fn ascii_decode(args: DecodeArgsNoFinal, vm: &VirtualMachine) -> DecodeResult { - do_codec!(ascii::decode, args, vm) - } - - // TODO: implement these codecs in Rust! - - use crate::common::static_cell::StaticCell; - #[inline] - fn delegate_pycodecs( - cell: &'static StaticCell<PyObjectRef>, - name: &'static str, - args: FuncArgs, - vm: &VirtualMachine, - ) -> PyResult { - let f = cell.get_or_try_init(|| { - let module = vm.import("_pycodecs", None, 0)?; - module.get_attr(name, vm) - })?; - f.call(args, vm) - } - macro_rules! delegate_pycodecs { - ($name:ident, $args:ident, $vm:ident) => {{ - rustpython_common::static_cell!( - static FUNC: PyObjectRef; - ); - delegate_pycodecs(&FUNC, stringify!($name), $args, $vm) - }}; - } - - #[pyfunction] - fn mbcs_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(mbcs_encode, args, vm) - } - #[pyfunction] - fn mbcs_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(mbcs_decode, args, vm) - } - #[pyfunction] - fn readbuffer_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(readbuffer_encode, args, vm) - } - #[pyfunction] - fn escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(escape_encode, args, vm) - } - #[pyfunction] - fn escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(escape_decode, args, vm) - } - #[pyfunction] - fn unicode_escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(unicode_escape_encode, args, vm) - } - #[pyfunction] - fn unicode_escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(unicode_escape_decode, args, vm) - } - #[pyfunction] - fn raw_unicode_escape_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(raw_unicode_escape_encode, args, vm) - } - #[pyfunction] - fn raw_unicode_escape_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(raw_unicode_escape_decode, args, vm) - } - #[pyfunction] - fn utf_7_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_7_encode, args, vm) - } - #[pyfunction] - fn utf_7_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_7_decode, args, vm) - } - #[pyfunction] - fn utf_16_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_encode, args, vm) - } - #[pyfunction] - fn utf_16_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_decode, args, vm) - } - #[pyfunction] - fn charmap_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(charmap_encode, args, vm) - } - #[pyfunction] - fn charmap_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(charmap_decode, args, vm) - } - #[pyfunction] - fn charmap_build(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(charmap_build, args, vm) - } - #[pyfunction] - fn utf_16_le_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_le_encode, args, vm) - } - #[pyfunction] - fn utf_16_le_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_le_decode, args, vm) - } - #[pyfunction] - fn utf_16_be_encode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_be_encode, args, vm) - } - #[pyfunction] - fn utf_16_be_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_be_decode, args, vm) - } - #[pyfunction] - fn utf_16_ex_decode(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - delegate_pycodecs!(utf_16_ex_decode, args, vm) - } - // TODO: utf-32 functions -} diff --git a/vm/src/stdlib/collections.rs b/vm/src/stdlib/collections.rs deleted file mode 100644 index e416792d05e..00000000000 --- a/vm/src/stdlib/collections.rs +++ /dev/null @@ -1,740 +0,0 @@ -pub(crate) use _collections::make_module; - -#[pymodule] -mod _collections { - use crate::{ - atomic_func, - builtins::{ - IterStatus::{Active, Exhausted}, - PositionIterInternal, PyGenericAlias, PyInt, PyTypeRef, - }, - common::lock::{PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}, - function::{FuncArgs, KwArgs, OptionalArg, PyComparisonValue}, - iter::PyExactSizeIterator, - protocol::{PyIterReturn, PySequenceMethods}, - recursion::ReprGuard, - sequence::{MutObjectSequenceOp, OptionalRangeArgs}, - sliceable::SequenceIndexOp, - types::{ - AsSequence, Comparable, Constructor, Initializer, IterNext, Iterable, PyComparisonOp, - Representable, SelfIter, - }, - utils::collection_repr, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - }; - use crossbeam_utils::atomic::AtomicCell; - use std::cmp::max; - use std::collections::VecDeque; - - #[pyattr] - #[pyclass(name = "deque", unhashable = true)] - #[derive(Debug, Default, PyPayload)] - struct PyDeque { - deque: PyRwLock<VecDeque<PyObjectRef>>, - maxlen: Option<usize>, - state: AtomicCell<usize>, // incremented whenever the indices move - } - - type PyDequeRef = PyRef<PyDeque>; - - #[derive(FromArgs)] - struct PyDequeOptions { - #[pyarg(any, optional)] - iterable: OptionalArg<PyObjectRef>, - #[pyarg(any, optional)] - maxlen: OptionalArg<PyObjectRef>, - } - - impl PyDeque { - fn borrow_deque(&self) -> PyRwLockReadGuard<'_, VecDeque<PyObjectRef>> { - self.deque.read() - } - - fn borrow_deque_mut(&self) -> PyRwLockWriteGuard<'_, VecDeque<PyObjectRef>> { - self.deque.write() - } - } - - #[pyclass( - flags(BASETYPE), - with( - Constructor, - Initializer, - AsSequence, - Comparable, - Iterable, - Representable - ) - )] - impl PyDeque { - #[pymethod] - fn append(&self, obj: PyObjectRef) { - self.state.fetch_add(1); - let mut deque = self.borrow_deque_mut(); - if self.maxlen == Some(deque.len()) { - deque.pop_front(); - } - deque.push_back(obj); - } - - #[pymethod] - fn appendleft(&self, obj: PyObjectRef) { - self.state.fetch_add(1); - let mut deque = self.borrow_deque_mut(); - if self.maxlen == Some(deque.len()) { - deque.pop_back(); - } - deque.push_front(obj); - } - - #[pymethod] - fn clear(&self) { - self.state.fetch_add(1); - self.borrow_deque_mut().clear() - } - - #[pymethod(magic)] - #[pymethod] - fn copy(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - Self { - deque: PyRwLock::new(zelf.borrow_deque().clone()), - maxlen: zelf.maxlen, - state: AtomicCell::new(zelf.state.load()), - } - .into_ref_with_type(vm, zelf.class().to_owned()) - } - - #[pymethod] - fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - let start_state = self.state.load(); - let count = self.mut_count(vm, &obj)?; - - if start_state != self.state.load() { - return Err(vm.new_runtime_error("deque mutated during iteration".to_owned())); - } - Ok(count) - } - - #[pymethod] - fn extend(&self, iter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self._extend(&iter, vm) - } - - fn _extend(&self, iter: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - self.state.fetch_add(1); - let max_len = self.maxlen; - let mut elements: Vec<PyObjectRef> = iter.try_to_value(vm)?; - if let Some(max_len) = max_len { - if max_len > elements.len() { - let mut deque = self.borrow_deque_mut(); - let drain_until = deque.len().saturating_sub(max_len - elements.len()); - deque.drain(..drain_until); - } else { - self.borrow_deque_mut().clear(); - elements.drain(..(elements.len() - max_len)); - } - } - self.borrow_deque_mut().extend(elements); - Ok(()) - } - - #[pymethod] - fn extendleft(&self, iter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let max_len = self.maxlen; - let mut elements: Vec<PyObjectRef> = iter.try_to_value(vm)?; - elements.reverse(); - - if let Some(max_len) = max_len { - if max_len > elements.len() { - let mut deque = self.borrow_deque_mut(); - let truncate_until = max_len - elements.len(); - deque.truncate(truncate_until); - } else { - self.borrow_deque_mut().clear(); - elements.truncate(max_len); - } - } - let mut created = VecDeque::from(elements); - let mut borrowed = self.borrow_deque_mut(); - created.append(&mut borrowed); - std::mem::swap(&mut created, &mut borrowed); - Ok(()) - } - - #[pymethod] - fn index( - &self, - needle: PyObjectRef, - range: OptionalRangeArgs, - vm: &VirtualMachine, - ) -> PyResult<usize> { - let start_state = self.state.load(); - - let (start, stop) = range.saturate(self.len(), vm)?; - let index = self.mut_index_range(vm, &needle, start..stop)?; - if start_state != self.state.load() { - Err(vm.new_runtime_error("deque mutated during iteration".to_owned())) - } else if let Some(index) = index.into() { - Ok(index) - } else { - Err(vm.new_value_error( - needle - .repr(vm) - .map(|repr| format!("{repr} is not in deque")) - .unwrap_or_else(|_| String::new()), - )) - } - } - - #[pymethod] - fn insert(&self, idx: i32, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - self.state.fetch_add(1); - let mut deque = self.borrow_deque_mut(); - - if self.maxlen == Some(deque.len()) { - return Err(vm.new_index_error("deque already at its maximum size".to_owned())); - } - - let idx = if idx < 0 { - if -idx as usize > deque.len() { - 0 - } else { - deque.len() - ((-idx) as usize) - } - } else if idx as usize > deque.len() { - deque.len() - } else { - idx as usize - }; - - deque.insert(idx, obj); - - Ok(()) - } - - #[pymethod] - fn pop(&self, vm: &VirtualMachine) -> PyResult { - self.state.fetch_add(1); - self.borrow_deque_mut() - .pop_back() - .ok_or_else(|| vm.new_index_error("pop from an empty deque".to_owned())) - } - - #[pymethod] - fn popleft(&self, vm: &VirtualMachine) -> PyResult { - self.state.fetch_add(1); - self.borrow_deque_mut() - .pop_front() - .ok_or_else(|| vm.new_index_error("pop from an empty deque".to_owned())) - } - - #[pymethod] - fn remove(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let start_state = self.state.load(); - let index = self.mut_index(vm, &obj)?; - - if start_state != self.state.load() { - Err(vm.new_index_error("deque mutated during remove().".to_owned())) - } else if let Some(index) = index.into() { - let mut deque = self.borrow_deque_mut(); - self.state.fetch_add(1); - Ok(deque.remove(index).unwrap()) - } else { - Err(vm.new_value_error("deque.remove(x): x not in deque".to_owned())) - } - } - - #[pymethod] - fn reverse(&self) { - let rev: VecDeque<_> = self.borrow_deque().iter().cloned().rev().collect(); - *self.borrow_deque_mut() = rev; - } - - #[pymethod(magic)] - fn reversed(zelf: PyRef<Self>) -> PyResult<PyReverseDequeIterator> { - Ok(PyReverseDequeIterator { - state: zelf.state.load(), - internal: PyMutex::new(PositionIterInternal::new(zelf, 0)), - }) - } - - #[pymethod] - fn rotate(&self, mid: OptionalArg<isize>) { - self.state.fetch_add(1); - let mut deque = self.borrow_deque_mut(); - if !deque.is_empty() { - let mid = mid.unwrap_or(1) % deque.len() as isize; - if mid.is_negative() { - deque.rotate_left(-mid as usize); - } else { - deque.rotate_right(mid as usize); - } - } - } - - #[pygetset] - fn maxlen(&self) -> Option<usize> { - self.maxlen - } - - #[pymethod(magic)] - fn getitem(&self, idx: isize, vm: &VirtualMachine) -> PyResult { - let deque = self.borrow_deque(); - idx.wrapped_at(deque.len()) - .and_then(|i| deque.get(i).cloned()) - .ok_or_else(|| vm.new_index_error("deque index out of range".to_owned())) - } - - #[pymethod(magic)] - fn setitem(&self, idx: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut deque = self.borrow_deque_mut(); - idx.wrapped_at(deque.len()) - .and_then(|i| deque.get_mut(i)) - .map(|x| *x = value) - .ok_or_else(|| vm.new_index_error("deque index out of range".to_owned())) - } - - #[pymethod(magic)] - fn delitem(&self, idx: isize, vm: &VirtualMachine) -> PyResult<()> { - let mut deque = self.borrow_deque_mut(); - idx.wrapped_at(deque.len()) - .and_then(|i| deque.remove(i).map(drop)) - .ok_or_else(|| vm.new_index_error("deque index out of range".to_owned())) - } - - #[pymethod(magic)] - fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - self._contains(&needle, vm) - } - - fn _contains(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - let start_state = self.state.load(); - let ret = self.mut_contains(vm, needle)?; - if start_state != self.state.load() { - Err(vm.new_runtime_error("deque mutated during iteration".to_owned())) - } else { - Ok(ret) - } - } - - fn _mul(&self, n: isize, vm: &VirtualMachine) -> PyResult<VecDeque<PyObjectRef>> { - let deque = self.borrow_deque(); - let n = vm.check_repeat_or_overflow_error(deque.len(), n)?; - let mul_len = n * deque.len(); - let iter = deque.iter().cycle().take(mul_len); - let skipped = self - .maxlen - .and_then(|maxlen| mul_len.checked_sub(maxlen)) - .unwrap_or(0); - - let deque = iter.skip(skipped).cloned().collect(); - Ok(deque) - } - - #[pymethod(magic)] - #[pymethod(name = "__rmul__")] - fn mul(&self, n: isize, vm: &VirtualMachine) -> PyResult<Self> { - let deque = self._mul(n, vm)?; - Ok(PyDeque { - deque: PyRwLock::new(deque), - maxlen: self.maxlen, - state: AtomicCell::new(0), - }) - } - - #[pymethod(magic)] - fn imul(zelf: PyRef<Self>, n: isize, vm: &VirtualMachine) -> PyResult<PyRef<Self>> { - let mul_deque = zelf._mul(n, vm)?; - *zelf.borrow_deque_mut() = mul_deque; - Ok(zelf) - } - - #[pymethod(magic)] - fn len(&self) -> usize { - self.borrow_deque().len() - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - !self.borrow_deque().is_empty() - } - - #[pymethod(magic)] - fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<Self> { - self.concat(&other, vm) - } - - fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<Self> { - if let Some(o) = other.payload_if_subclass::<PyDeque>(vm) { - let mut deque = self.borrow_deque().clone(); - let elements = o.borrow_deque().clone(); - deque.extend(elements); - - let skipped = self - .maxlen - .and_then(|maxlen| deque.len().checked_sub(maxlen)) - .unwrap_or(0); - deque.drain(..skipped); - - Ok(PyDeque { - deque: PyRwLock::new(deque), - maxlen: self.maxlen, - state: AtomicCell::new(0), - }) - } else { - Err(vm.new_type_error(format!( - "can only concatenate deque (not \"{}\") to deque", - other.class().name() - ))) - } - } - - #[pymethod(magic)] - fn iadd( - zelf: PyRef<Self>, - other: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - zelf.extend(other, vm)?; - Ok(zelf) - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - let cls = zelf.class().to_owned(); - let value = match zelf.maxlen { - Some(v) => vm.new_pyobj((vm.ctx.empty_tuple.clone(), v)), - None => vm.ctx.empty_tuple.clone().into(), - }; - Ok(vm.new_pyobj((cls, value, vm.ctx.none(), PyDequeIterator::new(zelf)))) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } - } - - impl MutObjectSequenceOp for PyDeque { - type Guard<'a> = PyRwLockReadGuard<'a, VecDeque<PyObjectRef>>; - - fn do_get<'a>(index: usize, guard: &'a Self::Guard<'_>) -> Option<&'a PyObjectRef> { - guard.get(index) - } - - fn do_lock(&self) -> Self::Guard<'_> { - self.borrow_deque() - } - } - - impl Constructor for PyDeque { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - PyDeque::default() - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl Initializer for PyDeque { - type Args = PyDequeOptions; - - fn init( - zelf: PyRef<Self>, - PyDequeOptions { iterable, maxlen }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult<()> { - // TODO: This is _basically_ pyobject_to_opt_usize in itertools.rs - // need to move that function elsewhere and refactor usages. - let maxlen = if let Some(obj) = maxlen.into_option() { - if !vm.is_none(&obj) { - let maxlen: isize = obj - .payload::<PyInt>() - .ok_or_else(|| vm.new_type_error("an integer is required.".to_owned()))? - .try_to_primitive(vm)?; - - if maxlen.is_negative() { - return Err(vm.new_value_error("maxlen must be non-negative.".to_owned())); - } - Some(maxlen as usize) - } else { - None - } - } else { - None - }; - - // retrieve elements first to not to make too huge lock - let elements = iterable - .into_option() - .map(|iter| { - let mut elements: Vec<PyObjectRef> = iter.try_to_value(vm)?; - if let Some(maxlen) = maxlen { - elements.drain(..elements.len().saturating_sub(maxlen)); - } - Ok(elements) - }) - .transpose()?; - - // SAFETY: This is hacky part for read-only field - // Because `maxlen` is only mutated from __init__. We can abuse the lock of deque to ensure this is locked enough. - // If we make a single lock of deque not only for extend but also for setting maxlen, it will be safe. - { - let mut deque = zelf.borrow_deque_mut(); - // Clear any previous data present. - deque.clear(); - unsafe { - // `maxlen` is better to be defined as UnsafeCell in common practice, - // but then more type works without any safety benefits - let unsafe_maxlen = - &zelf.maxlen as *const _ as *const std::cell::UnsafeCell<Option<usize>>; - *(*unsafe_maxlen).get() = maxlen; - } - if let Some(elements) = elements { - deque.extend(elements); - } - } - - Ok(()) - } - } - - impl AsSequence for PyDeque { - fn as_sequence() -> &'static PySequenceMethods { - static AS_SEQUENCE: PySequenceMethods = PySequenceMethods { - length: atomic_func!(|seq, _vm| Ok(PyDeque::sequence_downcast(seq).len())), - concat: atomic_func!(|seq, other, vm| { - PyDeque::sequence_downcast(seq) - .concat(other, vm) - .map(|x| x.into_ref(&vm.ctx).into()) - }), - repeat: atomic_func!(|seq, n, vm| { - PyDeque::sequence_downcast(seq) - .mul(n, vm) - .map(|x| x.into_ref(&vm.ctx).into()) - }), - item: atomic_func!(|seq, i, vm| PyDeque::sequence_downcast(seq).getitem(i, vm)), - ass_item: atomic_func!(|seq, i, value, vm| { - let zelf = PyDeque::sequence_downcast(seq); - if let Some(value) = value { - zelf.setitem(i, value, vm) - } else { - zelf.delitem(i, vm) - } - }), - contains: atomic_func!( - |seq, needle, vm| PyDeque::sequence_downcast(seq)._contains(needle, vm) - ), - inplace_concat: atomic_func!(|seq, other, vm| { - let zelf = PyDeque::sequence_downcast(seq); - zelf._extend(other, vm)?; - Ok(zelf.to_owned().into()) - }), - inplace_repeat: atomic_func!(|seq, n, vm| { - let zelf = PyDeque::sequence_downcast(seq); - PyDeque::imul(zelf.to_owned(), n, vm).map(|x| x.into()) - }), - }; - - &AS_SEQUENCE - } - } - - impl Comparable for PyDeque { - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - if let Some(res) = op.identical_optimization(zelf, other) { - return Ok(res.into()); - } - let other = class_or_notimplemented!(Self, other); - let lhs = zelf.borrow_deque(); - let rhs = other.borrow_deque(); - lhs.iter() - .richcompare(rhs.iter(), op, vm) - .map(PyComparisonValue::Implemented) - } - } - - impl Iterable for PyDeque { - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - Ok(PyDequeIterator::new(zelf).into_pyobject(vm)) - } - } - - impl Representable for PyDeque { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let deque = zelf.borrow_deque().clone(); - let class = zelf.class(); - let class_name = class.name(); - let closing_part = zelf - .maxlen - .map(|maxlen| format!("], maxlen={maxlen}")) - .unwrap_or_else(|| "]".to_owned()); - - let s = if zelf.len() == 0 { - format!("{class_name}([{closing_part})") - } else if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - collection_repr(Some(&class_name), "[", &closing_part, deque.iter(), vm)? - } else { - "[...]".to_owned() - }; - - Ok(s) - } - } - - #[pyattr] - #[pyclass(name = "_deque_iterator")] - #[derive(Debug, PyPayload)] - struct PyDequeIterator { - state: usize, - internal: PyMutex<PositionIterInternal<PyDequeRef>>, - } - - #[derive(FromArgs)] - struct DequeIterArgs { - #[pyarg(positional)] - deque: PyDequeRef, - - #[pyarg(positional, optional)] - index: OptionalArg<isize>, - } - - impl Constructor for PyDequeIterator { - type Args = (DequeIterArgs, KwArgs); - - fn py_new( - cls: PyTypeRef, - (DequeIterArgs { deque, index }, _kwargs): Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let iter = PyDequeIterator::new(deque); - if let OptionalArg::Present(index) = index { - let index = max(index, 0) as usize; - iter.internal.lock().position = index; - } - iter.into_ref_with_type(vm, cls).map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyDequeIterator { - pub(crate) fn new(deque: PyDequeRef) -> Self { - PyDequeIterator { - state: deque.state.load(), - internal: PyMutex::new(PositionIterInternal::new(deque, 0)), - } - } - - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().length_hint(|obj| obj.len()) - } - - #[pymethod(magic)] - fn reduce( - zelf: PyRef<Self>, - vm: &VirtualMachine, - ) -> (PyTypeRef, (PyDequeRef, PyObjectRef)) { - let internal = zelf.internal.lock(); - let deque = match &internal.status { - Active(obj) => obj.clone(), - Exhausted => PyDeque::default().into_ref(&vm.ctx), - }; - ( - zelf.class().to_owned(), - (deque, vm.ctx.new_int(internal.position).into()), - ) - } - } - - impl SelfIter for PyDequeIterator {} - impl IterNext for PyDequeIterator { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.internal.lock().next(|deque, pos| { - if zelf.state != deque.state.load() { - return Err(vm.new_runtime_error("Deque mutated during iteration".to_owned())); - } - let deque = deque.borrow_deque(); - Ok(PyIterReturn::from_result( - deque.get(pos).cloned().ok_or(None), - )) - }) - } - } - - #[pyattr] - #[pyclass(name = "_deque_reverse_iterator")] - #[derive(Debug, PyPayload)] - struct PyReverseDequeIterator { - state: usize, - // position is counting from the tail - internal: PyMutex<PositionIterInternal<PyDequeRef>>, - } - - impl Constructor for PyReverseDequeIterator { - type Args = (DequeIterArgs, KwArgs); - - fn py_new( - cls: PyTypeRef, - - (DequeIterArgs { deque, index }, _kwargs): Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let iter = PyDeque::reversed(deque)?; - if let OptionalArg::Present(index) = index { - let index = max(index, 0) as usize; - iter.internal.lock().position = index; - } - iter.into_ref_with_type(vm, cls).map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyReverseDequeIterator { - #[pymethod(magic)] - fn length_hint(&self) -> usize { - self.internal.lock().length_hint(|obj| obj.len()) - } - - #[pymethod(magic)] - fn reduce( - zelf: PyRef<Self>, - vm: &VirtualMachine, - ) -> PyResult<(PyTypeRef, (PyDequeRef, PyObjectRef))> { - let internal = zelf.internal.lock(); - let deque = match &internal.status { - Active(obj) => obj.clone(), - Exhausted => PyDeque::default().into_ref(&vm.ctx), - }; - Ok(( - zelf.class().to_owned(), - (deque, vm.ctx.new_int(internal.position).into()), - )) - } - } - - impl SelfIter for PyReverseDequeIterator {} - impl IterNext for PyReverseDequeIterator { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - zelf.internal.lock().next(|deque, pos| { - if deque.state.load() != zelf.state { - return Err(vm.new_runtime_error("Deque mutated during iteration".to_owned())); - } - let deque = deque.borrow_deque(); - let r = deque - .len() - .checked_sub(pos + 1) - .and_then(|pos| deque.get(pos)) - .cloned(); - Ok(PyIterReturn::from_result(r.ok_or(None))) - }) - } - } -} diff --git a/vm/src/stdlib/functools.rs b/vm/src/stdlib/functools.rs deleted file mode 100644 index d13b9b84f69..00000000000 --- a/vm/src/stdlib/functools.rs +++ /dev/null @@ -1,33 +0,0 @@ -pub(crate) use _functools::make_module; - -#[pymodule] -mod _functools { - use crate::{function::OptionalArg, protocol::PyIter, PyObjectRef, PyResult, VirtualMachine}; - - #[pyfunction] - fn reduce( - function: PyObjectRef, - iterator: PyIter, - start_value: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let mut iter = iterator.iter_without_hint(vm)?; - let start_value = if let OptionalArg::Present(val) = start_value { - val - } else { - iter.next().transpose()?.ok_or_else(|| { - let exc_type = vm.ctx.exceptions.type_error.to_owned(); - vm.new_exception_msg( - exc_type, - "reduce() of empty sequence with no initial value".to_owned(), - ) - })? - }; - - let mut accumulator = start_value; - for next_obj in iter { - accumulator = function.call((accumulator, next_obj?), vm)? - } - Ok(accumulator) - } -} diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs deleted file mode 100644 index f59f8e4e633..00000000000 --- a/vm/src/stdlib/imp.rs +++ /dev/null @@ -1,197 +0,0 @@ -use crate::frozen::FrozenModule; -use crate::{builtins::PyBaseExceptionRef, VirtualMachine}; -pub(crate) use _imp::make_module; - -#[cfg(feature = "threading")] -#[pymodule(sub)] -mod lock { - use crate::{stdlib::thread::RawRMutex, PyResult, VirtualMachine}; - - static IMP_LOCK: RawRMutex = RawRMutex::INIT; - - #[pyfunction] - fn acquire_lock(_vm: &VirtualMachine) { - IMP_LOCK.lock() - } - - #[pyfunction] - fn release_lock(vm: &VirtualMachine) -> PyResult<()> { - if !IMP_LOCK.is_locked() { - Err(vm.new_runtime_error("Global import lock not held".to_owned())) - } else { - unsafe { IMP_LOCK.unlock() }; - Ok(()) - } - } - - #[pyfunction] - fn lock_held(_vm: &VirtualMachine) -> bool { - IMP_LOCK.is_locked() - } -} - -#[cfg(not(feature = "threading"))] -#[pymodule(sub)] -mod lock { - use crate::vm::VirtualMachine; - #[pyfunction] - pub(super) fn acquire_lock(_vm: &VirtualMachine) {} - #[pyfunction] - pub(super) fn release_lock(_vm: &VirtualMachine) {} - #[pyfunction] - pub(super) fn lock_held(_vm: &VirtualMachine) -> bool { - false - } -} - -#[allow(dead_code)] -enum FrozenError { - BadName, // The given module name wasn't valid. - NotFound, // It wasn't in PyImport_FrozenModules. - Disabled, // -X frozen_modules=off (and not essential) - Excluded, // The PyImport_FrozenModules entry has NULL "code" - // (module is present but marked as unimportable, stops search). - Invalid, // The PyImport_FrozenModules entry is bogus - // (eg. does not contain executable code). -} - -impl FrozenError { - fn to_pyexception(&self, mod_name: &str, vm: &VirtualMachine) -> PyBaseExceptionRef { - use FrozenError::*; - let msg = match self { - BadName | NotFound => format!("No such frozen object named {mod_name}"), - Disabled => format!("Frozen modules are disabled and the frozen object named {mod_name} is not essential"), - Excluded => format!("Excluded frozen object named {mod_name}"), - Invalid => format!("Frozen object named {mod_name} is invalid"), - }; - vm.new_import_error(msg, vm.ctx.new_str(mod_name)) - } -} - -// find_frozen in frozen.c -fn find_frozen(name: &str, vm: &VirtualMachine) -> Result<FrozenModule, FrozenError> { - vm.state - .frozen - .get(name) - .copied() - .ok_or(FrozenError::NotFound) -} - -#[pymodule(with(lock))] -mod _imp { - use crate::{ - builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef}, - function::OptionalArg, - import, PyObjectRef, PyRef, PyResult, VirtualMachine, - }; - - #[pyattr] - fn check_hash_based_pycs(vm: &VirtualMachine) -> PyStrRef { - vm.ctx - .new_str(vm.state.settings.check_hash_based_pycs.clone()) - } - - #[pyfunction] - fn extension_suffixes() -> PyResult<Vec<PyObjectRef>> { - Ok(Vec::new()) - } - - #[pyfunction] - fn is_builtin(name: PyStrRef, vm: &VirtualMachine) -> bool { - vm.state.module_inits.contains_key(name.as_str()) - } - - #[pyfunction] - fn is_frozen(name: PyStrRef, vm: &VirtualMachine) -> bool { - vm.state.frozen.contains_key(name.as_str()) - } - - #[pyfunction] - fn create_builtin(spec: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let sys_modules = vm.sys_module.get_attr("modules", vm).unwrap(); - let name: PyStrRef = spec.get_attr("name", vm)?.try_into_value(vm)?; - - let module = if let Ok(module) = sys_modules.get_item(&*name, vm) { - module - } else if let Some(make_module_func) = vm.state.module_inits.get(name.as_str()) { - make_module_func(vm).into() - } else { - vm.ctx.none() - }; - Ok(module) - } - - #[pyfunction] - fn exec_builtin(_mod: PyRef<PyModule>) -> i32 { - // TODO: Should we do something here? - 0 - } - - #[pyfunction] - fn get_frozen_object(name: PyStrRef, vm: &VirtualMachine) -> PyResult<PyRef<PyCode>> { - import::make_frozen(vm, name.as_str()) - } - - #[pyfunction] - fn init_frozen(name: PyStrRef, vm: &VirtualMachine) -> PyResult { - import::import_frozen(vm, name.as_str()) - } - - #[pyfunction] - fn is_frozen_package(name: PyStrRef, vm: &VirtualMachine) -> PyResult<bool> { - super::find_frozen(name.as_str(), vm) - .map(|frozen| frozen.package) - .map_err(|e| e.to_pyexception(name.as_str(), vm)) - } - - #[pyfunction] - fn _override_frozen_modules_for_tests(value: isize, vm: &VirtualMachine) { - vm.state.override_frozen_modules.store(value); - } - - #[pyfunction] - fn _fix_co_filename(_code: PyObjectRef, _path: PyStrRef) { - // TODO: - } - - #[pyfunction] - fn _frozen_module_names(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let names = vm - .state - .frozen - .keys() - .map(|&name| vm.ctx.new_str(name).into()) - .collect(); - Ok(names) - } - - #[allow(clippy::type_complexity)] - #[pyfunction] - fn find_frozen( - name: PyStrRef, - withdata: OptionalArg<bool>, - vm: &VirtualMachine, - ) -> PyResult<Option<(Option<PyRef<PyMemoryView>>, bool, PyStrRef)>> { - use super::FrozenError::*; - - if withdata.into_option().is_some() { - // this is keyword-only argument in CPython - unimplemented!(); - } - - let info = match super::find_frozen(name.as_str(), vm) { - Ok(info) => info, - Err(NotFound | Disabled | BadName) => return Ok(None), - Err(e) => return Err(e.to_pyexception(name.as_str(), vm)), - }; - - let origname = name; // FIXME: origname != name - Ok(Some((None, info.package, origname))) - } - - #[pyfunction] - fn source_hash(key: u64, source: PyBytesRef) -> Vec<u8> { - let hash: u64 = crate::common::hash::keyed_hash(key, source.as_bytes()); - hash.to_le_bytes().to_vec() - } -} diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs deleted file mode 100644 index f6af1f9f946..00000000000 --- a/vm/src/stdlib/io.rs +++ /dev/null @@ -1,4120 +0,0 @@ -/* - * I/O core tools. - */ -cfg_if::cfg_if! { - if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { - use crate::common::crt_fd::Offset; - } else { - type Offset = i64; - } -} - -use crate::{ - builtins::PyBaseExceptionRef, - builtins::PyModule, - convert::{IntoPyException, ToPyException, ToPyObject}, - PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, -}; -pub use _io::io_open as open; - -impl ToPyException for std::io::Error { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - use std::io::ErrorKind; - - let excs = &vm.ctx.exceptions; - #[allow(unreachable_patterns)] // some errors are just aliases of each other - let exc_type = match self.kind() { - ErrorKind::NotFound => excs.file_not_found_error, - ErrorKind::PermissionDenied => excs.permission_error, - ErrorKind::AlreadyExists => excs.file_exists_error, - ErrorKind::WouldBlock => excs.blocking_io_error, - _ => self - .raw_os_error() - .and_then(|errno| crate::exceptions::raw_os_error_to_exc_type(errno, vm)) - .unwrap_or(excs.os_error), - }; - let errno = self.raw_os_error().to_pyobject(vm); - let msg = vm.ctx.new_str(self.to_string()).into(); - vm.new_exception(exc_type.to_owned(), vec![errno, msg]) - } -} - -impl IntoPyException for std::io::Error { - fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { - self.to_pyexception(vm) - } -} - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let ctx = &vm.ctx; - - let module = _io::make_module(vm); - - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - fileio::extend_module(vm, &module).unwrap(); - - let unsupported_operation = _io::UNSUPPORTED_OPERATION - .get_or_init(|| _io::make_unsupportedop(ctx)) - .clone(); - extend_module!(vm, &module, { - "UnsupportedOperation" => unsupported_operation, - "BlockingIOError" => ctx.exceptions.blocking_io_error.to_owned(), - }); - - module -} - -// not used on all platforms -#[allow(unused)] -#[derive(Copy, Clone)] -#[repr(transparent)] -pub struct Fildes(pub i32); - -impl TryFromObject for Fildes { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - use crate::builtins::int; - let int = match obj.downcast::<int::PyInt>() { - Ok(i) => i, - Err(obj) => { - let fileno_meth = vm.get_attribute_opt(obj, "fileno")?.ok_or_else(|| { - vm.new_type_error( - "argument must be an int, or have a fileno() method.".to_owned(), - ) - })?; - fileno_meth - .call((), vm)? - .downcast() - .map_err(|_| vm.new_type_error("fileno() returned a non-integer".to_owned()))? - } - }; - let fd = int.try_to_primitive(vm)?; - if fd < 0 { - return Err(vm.new_value_error(format!( - "file descriptor cannot be a negative integer ({fd})" - ))); - } - Ok(Fildes(fd)) - } -} - -#[pymodule] -mod _io { - use super::*; - use crate::{ - builtins::{ - PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyIntRef, PyMemoryView, PyStr, - PyStrRef, PyType, PyTypeRef, - }, - class::StaticType, - common::lock::{ - PyMappedThreadMutexGuard, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard, - PyThreadMutex, PyThreadMutexGuard, - }, - convert::ToPyObject, - function::{ - ArgBytesLike, ArgIterable, ArgMemoryBuffer, ArgSize, Either, FuncArgs, IntoFuncArgs, - OptionalArg, OptionalOption, PySetterValue, - }, - protocol::{ - BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, VecBuffer, - }, - recursion::ReprGuard, - types::{ - Callable, Constructor, DefaultConstructor, Destructor, Initializer, IterNext, Iterable, - }, - vm::VirtualMachine, - AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, - }; - use bstr::ByteSlice; - use crossbeam_utils::atomic::AtomicCell; - use malachite_bigint::{BigInt, BigUint}; - use num_traits::ToPrimitive; - use std::{ - io::{self, prelude::*, Cursor, SeekFrom}, - ops::Range, - }; - - #[allow(clippy::let_and_return)] - fn validate_whence(whence: i32) -> bool { - let x = (0..=2).contains(&whence); - cfg_if::cfg_if! { - if #[cfg(any(target_os = "dragonfly", target_os = "freebsd", target_os = "linux"))] { - x || matches!(whence, libc::SEEK_DATA | libc::SEEK_HOLE) - } else { - x - } - } - } - - fn ensure_unclosed(file: &PyObject, msg: &str, vm: &VirtualMachine) -> PyResult<()> { - if file.get_attr("closed", vm)?.try_to_bool(vm)? { - Err(vm.new_value_error(msg.to_owned())) - } else { - Ok(()) - } - } - - pub fn new_unsupported_operation(vm: &VirtualMachine, msg: String) -> PyBaseExceptionRef { - vm.new_exception_msg(UNSUPPORTED_OPERATION.get().unwrap().clone(), msg) - } - - fn _unsupported<T>(vm: &VirtualMachine, zelf: &PyObject, operation: &str) -> PyResult<T> { - Err(new_unsupported_operation( - vm, - format!("{}.{}() not supported", zelf.class().name(), operation), - )) - } - - #[derive(FromArgs)] - pub(super) struct OptionalSize { - // In a few functions, the default value is -1 rather than None. - // Make sure the default value doesn't affect compatibility. - #[pyarg(positional, default)] - size: Option<ArgSize>, - } - - impl OptionalSize { - #[allow(clippy::wrong_self_convention)] - pub fn to_usize(self) -> Option<usize> { - self.size?.to_usize() - } - - pub fn try_usize(self, vm: &VirtualMachine) -> PyResult<Option<usize>> { - self.size - .map(|v| { - let v = *v; - if v >= 0 { - Ok(v as usize) - } else { - Err(vm.new_value_error(format!("Negative size value {v}"))) - } - }) - .transpose() - } - } - - fn os_err(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef { - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - { - use crate::convert::ToPyException; - err.to_pyexception(vm) - } - #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] - { - vm.new_os_error(err.to_string()) - } - } - - pub(super) fn io_closed_error(vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_value_error("I/O operation on closed file".to_owned()) - } - - #[pyattr] - const DEFAULT_BUFFER_SIZE: usize = 8 * 1024; - - pub(super) fn seekfrom( - vm: &VirtualMachine, - offset: PyObjectRef, - how: OptionalArg<i32>, - ) -> PyResult<SeekFrom> { - let seek = match how { - OptionalArg::Present(0) | OptionalArg::Missing => { - SeekFrom::Start(offset.try_into_value(vm)?) - } - OptionalArg::Present(1) => SeekFrom::Current(offset.try_into_value(vm)?), - OptionalArg::Present(2) => SeekFrom::End(offset.try_into_value(vm)?), - _ => return Err(vm.new_value_error("invalid value for how".to_owned())), - }; - Ok(seek) - } - - #[derive(Debug)] - struct BufferedIO { - cursor: Cursor<Vec<u8>>, - } - - impl BufferedIO { - fn new(cursor: Cursor<Vec<u8>>) -> BufferedIO { - BufferedIO { cursor } - } - - fn write(&mut self, data: &[u8]) -> Option<u64> { - let length = data.len(); - - match self.cursor.write_all(data) { - Ok(_) => Some(length as u64), - Err(_) => None, - } - } - - //return the entire contents of the underlying - fn getvalue(&self) -> Vec<u8> { - self.cursor.clone().into_inner() - } - - //skip to the jth position - fn seek(&mut self, seek: SeekFrom) -> io::Result<u64> { - self.cursor.seek(seek) - } - - //Read k bytes from the object and return. - fn read(&mut self, bytes: Option<usize>) -> Option<Vec<u8>> { - let pos = self.cursor.position().to_usize()?; - let avail_slice = self.cursor.get_ref().get(pos..)?; - // if we don't specify the number of bytes, or it's too big, give the whole rest of the slice - let n = bytes.map_or_else( - || avail_slice.len(), - |n| std::cmp::min(n, avail_slice.len()), - ); - let b = avail_slice[..n].to_vec(); - self.cursor.set_position((pos + n) as u64); - Some(b) - } - - fn tell(&self) -> u64 { - self.cursor.position() - } - - fn readline(&mut self, size: Option<usize>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - self.read_until(size, b'\n', vm) - } - - fn read_until( - &mut self, - size: Option<usize>, - byte: u8, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - let size = match size { - None => { - let mut buf: Vec<u8> = Vec::new(); - self.cursor - .read_until(byte, &mut buf) - .map_err(|err| os_err(vm, err))?; - return Ok(buf); - } - Some(0) => { - return Ok(Vec::new()); - } - Some(size) => size, - }; - - let available = { - // For Cursor, fill_buf returns all of the remaining data unlike other BufReads which have outer reading source. - // Unless we add other data by write, there will be no more data. - let buf = self.cursor.fill_buf().map_err(|err| os_err(vm, err))?; - if size < buf.len() { - &buf[..size] - } else { - buf - } - }; - let buf = match available.find_byte(byte) { - Some(i) => available[..=i].to_vec(), - _ => available.to_vec(), - }; - self.cursor.consume(buf.len()); - Ok(buf) - } - - fn truncate(&mut self, pos: Option<usize>) -> usize { - let pos = pos.unwrap_or_else(|| self.tell() as usize); - self.cursor.get_mut().truncate(pos); - pos - } - } - - fn file_closed(file: &PyObject, vm: &VirtualMachine) -> PyResult<bool> { - file.get_attr("closed", vm)?.try_to_bool(vm) - } - fn check_closed(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if file_closed(file, vm)? { - Err(io_closed_error(vm)) - } else { - Ok(()) - } - } - - fn check_readable(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if vm.call_method(file, "readable", ())?.try_to_bool(vm)? { - Ok(()) - } else { - Err(new_unsupported_operation( - vm, - "File or stream is not readable".to_owned(), - )) - } - } - - fn check_writable(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if vm.call_method(file, "writable", ())?.try_to_bool(vm)? { - Ok(()) - } else { - Err(new_unsupported_operation( - vm, - "File or stream is not writable.".to_owned(), - )) - } - } - - fn check_seekable(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if vm.call_method(file, "seekable", ())?.try_to_bool(vm)? { - Ok(()) - } else { - Err(new_unsupported_operation( - vm, - "File or stream is not seekable".to_owned(), - )) - } - } - - fn check_decoded(decoded: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - decoded.downcast().map_err(|obj| { - vm.new_type_error(format!( - "decoder should return a string result, not '{}'", - obj.class().name() - )) - }) - } - - #[pyattr] - #[pyclass(name = "_IOBase")] - #[derive(Debug, PyPayload)] - pub struct _IOBase; - - #[pyclass(with(IterNext, Iterable, Destructor), flags(BASETYPE, HAS_DICT))] - impl _IOBase { - #[pymethod] - fn seek( - zelf: PyObjectRef, - _pos: PyObjectRef, - _whence: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - _unsupported(vm, &zelf, "seek") - } - #[pymethod] - fn tell(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm.call_method(&zelf, "seek", (0, 1)) - } - #[pymethod] - fn truncate(zelf: PyObjectRef, _pos: OptionalArg, vm: &VirtualMachine) -> PyResult { - _unsupported(vm, &zelf, "truncate") - } - #[pymethod] - fn fileno(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - _unsupported(vm, &zelf, "truncate") - } - - #[pyattr] - fn __closed(ctx: &Context) -> PyIntRef { - ctx.new_bool(false) - } - - #[pymethod(magic)] - fn enter(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { - check_closed(&instance, vm)?; - Ok(instance) - } - - #[pymethod(magic)] - fn exit(instance: PyObjectRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - vm.call_method(&instance, "close", ())?; - Ok(()) - } - - #[pymethod] - fn flush(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // just check if this is closed; if it isn't, do nothing - check_closed(&instance, vm) - } - - #[pymethod] - fn seekable(_self: PyObjectRef) -> bool { - false - } - #[pymethod] - fn readable(_self: PyObjectRef) -> bool { - false - } - #[pymethod] - fn writable(_self: PyObjectRef) -> bool { - false - } - - #[pymethod] - fn isatty(_self: PyObjectRef) -> bool { - false - } - - #[pygetset] - fn closed(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { - instance.get_attr("__closed", vm) - } - - #[pymethod] - fn close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - iobase_close(&instance, vm) - } - - #[pymethod] - fn readline( - instance: PyObjectRef, - size: OptionalSize, - vm: &VirtualMachine, - ) -> PyResult<Vec<u8>> { - let size = size.to_usize(); - let read = instance.get_attr("read", vm)?; - let mut res = Vec::new(); - while size.map_or(true, |s| res.len() < s) { - let read_res = ArgBytesLike::try_from_object(vm, read.call((1,), vm)?)?; - if read_res.with_ref(|b| b.is_empty()) { - break; - } - read_res.with_ref(|b| res.extend_from_slice(b)); - if res.ends_with(b"\n") { - break; - } - } - Ok(res) - } - - #[pymethod] - fn readlines( - instance: PyObjectRef, - hint: OptionalOption<isize>, - vm: &VirtualMachine, - ) -> PyResult<Vec<PyObjectRef>> { - let hint = hint.flatten().unwrap_or(-1); - if hint <= 0 { - return instance.try_to_value(vm); - } - let hint = hint as usize; - let mut ret = Vec::new(); - let it = ArgIterable::<PyObjectRef>::try_from_object(vm, instance)?; - let mut full_len = 0; - for line in it.iter(vm)? { - let line = line?; - let line_len = line.length(vm)?; - ret.push(line.clone()); - full_len += line_len; - if full_len > hint { - break; - } - } - Ok(ret) - } - - #[pymethod] - fn writelines( - instance: PyObjectRef, - lines: ArgIterable, - vm: &VirtualMachine, - ) -> PyResult<()> { - check_closed(&instance, vm)?; - for line in lines.iter(vm)? { - vm.call_method(&instance, "write", (line?,))?; - } - Ok(()) - } - - #[pymethod(name = "_checkClosed")] - fn check_closed(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - check_closed(&instance, vm) - } - - #[pymethod(name = "_checkReadable")] - fn check_readable(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - check_readable(&instance, vm) - } - - #[pymethod(name = "_checkWritable")] - fn check_writable(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - check_writable(&instance, vm) - } - - #[pymethod(name = "_checkSeekable")] - fn check_seekable(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - check_seekable(&instance, vm) - } - } - - impl Destructor for _IOBase { - fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let _ = vm.call_method(zelf, "close", ()); - Ok(()) - } - - #[cold] - fn del(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<()> { - unreachable!("slot_del is implemented") - } - } - - impl Iterable for _IOBase { - fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - check_closed(&zelf, vm)?; - Ok(zelf) - } - - fn iter(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { - unreachable!("slot_iter is implemented") - } - } - - impl IterNext for _IOBase { - fn slot_iternext(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let line = vm.call_method(zelf, "readline", ())?; - Ok(if !line.clone().try_to_bool(vm)? { - PyIterReturn::StopIteration(None) - } else { - PyIterReturn::Return(line) - }) - } - - fn next(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { - unreachable!("slot_iternext is implemented") - } - } - - pub(super) fn iobase_close(file: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if !file_closed(file, vm)? { - let res = vm.call_method(file, "flush", ()); - file.set_attr("__closed", vm.new_pyobj(true), vm)?; - res?; - } - Ok(()) - } - - #[pyattr] - #[pyclass(name = "_RawIOBase", base = "_IOBase")] - pub(super) struct _RawIOBase; - - #[pyclass(flags(BASETYPE, HAS_DICT))] - impl _RawIOBase { - #[pymethod] - fn read(instance: PyObjectRef, size: OptionalSize, vm: &VirtualMachine) -> PyResult { - if let Some(size) = size.to_usize() { - // FIXME: unnecessary zero-init - let b = PyByteArray::from(vec![0; size]).into_ref(&vm.ctx); - let n = <Option<usize>>::try_from_object( - vm, - vm.call_method(&instance, "readinto", (b.clone(),))?, - )?; - Ok(n.map(|n| { - let mut bytes = b.borrow_buf_mut(); - bytes.truncate(n); - // FIXME: try to use Arc::unwrap on the bytearray to get at the inner buffer - bytes.clone() - }) - .to_pyobject(vm)) - } else { - vm.call_method(&instance, "readall", ()) - } - } - - #[pymethod] - fn readall(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<Option<Vec<u8>>> { - let mut chunks = Vec::new(); - let mut total_len = 0; - loop { - let data = vm.call_method(&instance, "read", (DEFAULT_BUFFER_SIZE,))?; - let data = <Option<PyBytesRef>>::try_from_object(vm, data)?; - match data { - None => { - if chunks.is_empty() { - return Ok(None); - } - break; - } - Some(b) => { - if b.as_bytes().is_empty() { - break; - } - total_len += b.as_bytes().len(); - chunks.push(b) - } - } - } - let mut ret = Vec::with_capacity(total_len); - for b in chunks { - ret.extend_from_slice(b.as_bytes()) - } - Ok(Some(ret)) - } - } - - #[pyattr] - #[pyclass(name = "_BufferedIOBase", base = "_IOBase")] - struct _BufferedIOBase; - - #[pyclass(flags(BASETYPE))] - impl _BufferedIOBase { - #[pymethod] - fn read(zelf: PyObjectRef, _size: OptionalArg, vm: &VirtualMachine) -> PyResult { - _unsupported(vm, &zelf, "read") - } - #[pymethod] - fn read1(zelf: PyObjectRef, _size: OptionalArg, vm: &VirtualMachine) -> PyResult { - _unsupported(vm, &zelf, "read1") - } - fn _readinto( - zelf: PyObjectRef, - bufobj: PyObjectRef, - method: &str, - vm: &VirtualMachine, - ) -> PyResult<usize> { - let b = ArgMemoryBuffer::try_from_borrowed_object(vm, &bufobj)?; - let l = b.len(); - let data = vm.call_method(&zelf, method, (l,))?; - if data.is(&bufobj) { - return Ok(l); - } - let mut buf = b.borrow_buf_mut(); - let data = ArgBytesLike::try_from_object(vm, data)?; - let data = data.borrow_buf(); - match buf.get_mut(..data.len()) { - Some(slice) => { - slice.copy_from_slice(&data); - Ok(data.len()) - } - None => Err(vm.new_value_error( - "readinto: buffer and read data have different lengths".to_owned(), - )), - } - } - #[pymethod] - fn readinto(zelf: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - Self::_readinto(zelf, b, "read", vm) - } - #[pymethod] - fn readinto1(zelf: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - Self::_readinto(zelf, b, "read1", vm) - } - #[pymethod] - fn write(zelf: PyObjectRef, _b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - _unsupported(vm, &zelf, "write") - } - #[pymethod] - fn detach(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - _unsupported(vm, &zelf, "detach") - } - } - - // TextIO Base has no public constructor - #[pyattr] - #[pyclass(name = "_TextIOBase", base = "_IOBase")] - #[derive(Debug, PyPayload)] - struct _TextIOBase; - - #[pyclass(flags(BASETYPE))] - impl _TextIOBase { - #[pygetset] - fn encoding(_zelf: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.none() - } - } - - #[derive(FromArgs, Clone)] - struct BufferSize { - #[pyarg(any, optional)] - buffer_size: OptionalArg<isize>, - } - - bitflags::bitflags! { - #[derive(Copy, Clone, Debug, PartialEq, Default)] - struct BufferedFlags: u8 { - const DETACHED = 1 << 0; - const WRITABLE = 1 << 1; - const READABLE = 1 << 2; - } - } - - #[derive(Debug, Default)] - struct BufferedData { - raw: Option<PyObjectRef>, - flags: BufferedFlags, - abs_pos: Offset, - buffer: Vec<u8>, - pos: Offset, - raw_pos: Offset, - read_end: Offset, - write_pos: Offset, - write_end: Offset, - } - - impl BufferedData { - fn check_init(&self, vm: &VirtualMachine) -> PyResult<&PyObject> { - if let Some(raw) = &self.raw { - Ok(raw) - } else { - let msg = if self.flags.contains(BufferedFlags::DETACHED) { - "raw stream has been detached" - } else { - "I/O operation on uninitialized object" - }; - Err(vm.new_value_error(msg.to_owned())) - } - } - - #[inline] - fn writable(&self) -> bool { - self.flags.contains(BufferedFlags::WRITABLE) - } - #[inline] - fn readable(&self) -> bool { - self.flags.contains(BufferedFlags::READABLE) - } - - #[inline] - fn valid_read(&self) -> bool { - self.readable() && self.read_end != -1 - } - #[inline] - fn valid_write(&self) -> bool { - self.writable() && self.write_end != -1 - } - - #[inline] - fn raw_offset(&self) -> Offset { - if (self.valid_read() || self.valid_write()) && self.raw_pos >= 0 { - self.raw_pos - self.pos - } else { - 0 - } - } - #[inline] - fn readahead(&self) -> Offset { - if self.valid_read() { - self.read_end - self.pos - } else { - 0 - } - } - - fn reset_read(&mut self) { - self.read_end = -1; - } - fn reset_write(&mut self) { - self.write_pos = 0; - self.write_end = -1; - } - - fn flush(&mut self, vm: &VirtualMachine) -> PyResult<()> { - if !self.valid_write() || self.write_pos == self.write_end { - self.reset_write(); - return Ok(()); - } - - let rewind = self.raw_offset() + (self.pos - self.write_pos); - if rewind != 0 { - self.raw_seek(-rewind, 1, vm)?; - self.raw_pos = -rewind; - } - - while self.write_pos < self.write_end { - let n = - self.raw_write(None, self.write_pos as usize..self.write_end as usize, vm)?; - let n = n.ok_or_else(|| { - vm.new_exception_msg( - vm.ctx.exceptions.blocking_io_error.to_owned(), - "write could not complete without blocking".to_owned(), - ) - })?; - self.write_pos += n as Offset; - self.raw_pos = self.write_pos; - vm.check_signals()?; - } - - self.reset_write(); - - Ok(()) - } - - fn flush_rewind(&mut self, vm: &VirtualMachine) -> PyResult<()> { - self.flush(vm)?; - if self.readable() { - let res = self.raw_seek(-self.raw_offset(), 1, vm); - self.reset_read(); - res?; - } - Ok(()) - } - - fn raw_seek(&mut self, pos: Offset, whence: i32, vm: &VirtualMachine) -> PyResult<Offset> { - let ret = vm.call_method(self.check_init(vm)?, "seek", (pos, whence))?; - let offset = get_offset(ret, vm)?; - if offset < 0 { - return Err( - vm.new_os_error(format!("Raw stream returned invalid position {offset}")) - ); - } - self.abs_pos = offset; - Ok(offset) - } - - fn seek(&mut self, target: Offset, whence: i32, vm: &VirtualMachine) -> PyResult<Offset> { - if matches!(whence, 0 | 1) && self.readable() { - let current = self.raw_tell_cache(vm)?; - let available = self.readahead(); - if available > 0 { - let offset = if whence == 0 { - target - (current - self.raw_offset()) - } else { - target - }; - if offset >= -self.pos && offset <= available { - self.pos += offset; - return Ok(current - available + offset); - } - } - } - // raw.get_attr("seek", vm)?.call(args, vm) - if self.writable() { - self.flush(vm)?; - } - let target = if whence == 1 { - target - self.raw_offset() - } else { - target - }; - let res = self.raw_seek(target, whence, vm); - self.raw_pos = -1; - if res.is_ok() && self.readable() { - self.reset_read(); - } - res - } - - fn raw_tell(&mut self, vm: &VirtualMachine) -> PyResult<Offset> { - let ret = vm.call_method(self.check_init(vm)?, "tell", ())?; - let offset = get_offset(ret, vm)?; - if offset < 0 { - return Err( - vm.new_os_error(format!("Raw stream returned invalid position {offset}")) - ); - } - self.abs_pos = offset; - Ok(offset) - } - - fn raw_tell_cache(&mut self, vm: &VirtualMachine) -> PyResult<Offset> { - if self.abs_pos == -1 { - self.raw_tell(vm) - } else { - Ok(self.abs_pos) - } - } - - /// None means non-blocking failed - fn raw_write( - &mut self, - buf: Option<PyBuffer>, - buf_range: Range<usize>, - vm: &VirtualMachine, - ) -> PyResult<Option<usize>> { - let len = buf_range.len(); - let res = if let Some(buf) = buf { - let memobj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?.to_pyobject(vm); - - // TODO: loop if write() raises an interrupt - vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj,))? - } else { - let v = std::mem::take(&mut self.buffer); - let writebuf = VecBuffer::from(v).into_ref(&vm.ctx); - let memobj = PyMemoryView::from_buffer_range( - writebuf.clone().into_pybuffer(true), - buf_range, - vm, - )? - .into_ref(&vm.ctx); - - // TODO: loop if write() raises an interrupt - let res = vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj.clone(),)); - - memobj.release(); - self.buffer = writebuf.take(); - - res? - }; - - if vm.is_none(&res) { - return Ok(None); - } - let n = isize::try_from_object(vm, res)?; - if n < 0 || n as usize > len { - return Err(vm.new_os_error(format!( - "raw write() returned invalid length {n} (should have been between 0 and {len})" - ))); - } - if self.abs_pos != -1 { - self.abs_pos += n as Offset - } - Ok(Some(n as usize)) - } - - fn write(&mut self, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { - if !self.valid_read() && !self.valid_write() { - self.pos = 0; - self.raw_pos = 0; - } - let avail = self.buffer.len() - self.pos as usize; - let buf_len; - { - let buf = obj.borrow_buf(); - buf_len = buf.len(); - if buf.len() <= avail { - self.buffer[self.pos as usize..][..buf.len()].copy_from_slice(&buf); - if !self.valid_write() || self.write_pos > self.pos { - self.write_pos = self.pos - } - self.adjust_position(self.pos + buf.len() as Offset); - if self.pos > self.write_end { - self.write_end = self.pos - } - return Ok(buf.len()); - } - } - - // TODO: something something check if error is BlockingIOError? - let _ = self.flush(vm); - - let offset = self.raw_offset(); - if offset != 0 { - self.raw_seek(-offset, 1, vm)?; - self.raw_pos -= offset; - } - - let mut remaining = buf_len; - let mut written = 0; - let buffer: PyBuffer = obj.into(); - while remaining > self.buffer.len() { - let res = self.raw_write(Some(buffer.clone()), written..buf_len, vm)?; - match res { - Some(n) => { - written += n; - if let Some(r) = remaining.checked_sub(n) { - remaining = r - } else { - break; - } - vm.check_signals()?; - } - None => { - // raw file is non-blocking - if remaining > self.buffer.len() { - // can't buffer everything, buffer what we can and error - let buf = buffer.as_contiguous().unwrap(); - let buffer_len = self.buffer.len(); - self.buffer.copy_from_slice(&buf[written..][..buffer_len]); - self.raw_pos = 0; - let buffer_size = self.buffer.len() as _; - self.adjust_position(buffer_size); - self.write_end = buffer_size; - // TODO: BlockingIOError(errno, msg, written) - // written += self.buffer.len(); - return Err(vm.new_exception_msg( - vm.ctx.exceptions.blocking_io_error.to_owned(), - "write could not complete without blocking".to_owned(), - )); - } else { - break; - } - } - } - } - if self.readable() { - self.reset_read(); - } - if remaining > 0 { - let buf = buffer.as_contiguous().unwrap(); - self.buffer[..remaining].copy_from_slice(&buf[written..][..remaining]); - written += remaining; - } - self.write_pos = 0; - self.write_end = remaining as _; - self.adjust_position(remaining as _); - self.raw_pos = 0; - - Ok(written) - } - - fn active_read_slice(&self) -> &[u8] { - &self.buffer[self.pos as usize..][..self.readahead() as usize] - } - - fn read_fast(&mut self, n: usize) -> Option<Vec<u8>> { - let ret = self.active_read_slice().get(..n)?.to_vec(); - self.pos += n as Offset; - Some(ret) - } - - fn read_generic(&mut self, n: usize, vm: &VirtualMachine) -> PyResult<Option<Vec<u8>>> { - if let Some(fast) = self.read_fast(n) { - return Ok(Some(fast)); - } - - let current_size = self.readahead() as usize; - - let mut out = vec![0u8; n]; - let mut remaining = n; - let mut written = 0; - if current_size > 0 { - let slice = self.active_read_slice(); - out[..slice.len()].copy_from_slice(slice); - remaining -= current_size; - written += current_size; - self.pos += current_size as Offset; - } - if self.writable() { - self.flush_rewind(vm)?; - } - self.reset_read(); - macro_rules! handle_opt_read { - ($x:expr) => { - match ($x, written > 0) { - (Some(0), _) | (None, true) => { - out.truncate(written); - return Ok(Some(out)); - } - (Some(r), _) => r, - (None, _) => return Ok(None), - } - }; - } - while remaining > 0 { - // MINUS_LAST_BLOCK() in CPython - let r = self.buffer.len() * (remaining / self.buffer.len()); - if r == 0 { - break; - } - let r = self.raw_read(Either::A(Some(&mut out)), written..written + r, vm)?; - let r = handle_opt_read!(r); - remaining -= r; - written += r; - } - self.pos = 0; - self.raw_pos = 0; - self.read_end = 0; - - while remaining > 0 && (self.read_end as usize) < self.buffer.len() { - let r = handle_opt_read!(self.fill_buffer(vm)?); - if remaining > r { - out[written..][..r].copy_from_slice(&self.buffer[self.pos as usize..][..r]); - written += r; - self.pos += r as Offset; - remaining -= r; - } else if remaining > 0 { - out[written..][..remaining] - .copy_from_slice(&self.buffer[self.pos as usize..][..remaining]); - written += remaining; - self.pos += remaining as Offset; - remaining = 0; - } - if remaining == 0 { - break; - } - } - - Ok(Some(out)) - } - - fn fill_buffer(&mut self, vm: &VirtualMachine) -> PyResult<Option<usize>> { - let start = if self.valid_read() { - self.read_end as usize - } else { - 0 - }; - let buf_end = self.buffer.len(); - let res = self.raw_read(Either::A(None), start..buf_end, vm)?; - if let Some(n) = res.filter(|n| *n > 0) { - let new_start = (start + n) as Offset; - self.read_end = new_start; - self.raw_pos = new_start; - } - Ok(res) - } - - fn raw_read( - &mut self, - v: Either<Option<&mut Vec<u8>>, PyBuffer>, - buf_range: Range<usize>, - vm: &VirtualMachine, - ) -> PyResult<Option<usize>> { - let len = buf_range.len(); - let res = match v { - Either::A(v) => { - let v = v.unwrap_or(&mut self.buffer); - let readbuf = VecBuffer::from(std::mem::take(v)).into_ref(&vm.ctx); - let memobj = PyMemoryView::from_buffer_range( - readbuf.clone().into_pybuffer(false), - buf_range, - vm, - )? - .into_ref(&vm.ctx); - - // TODO: loop if readinto() raises an interrupt - let res = - vm.call_method(self.raw.as_ref().unwrap(), "readinto", (memobj.clone(),)); - - memobj.release(); - std::mem::swap(v, &mut readbuf.take()); - - res? - } - Either::B(buf) => { - let memobj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?; - // TODO: loop if readinto() raises an interrupt - vm.call_method(self.raw.as_ref().unwrap(), "readinto", (memobj,))? - } - }; - - if vm.is_none(&res) { - return Ok(None); - } - let n = isize::try_from_object(vm, res)?; - if n < 0 || n as usize > len { - return Err(vm.new_os_error(format!( - "raw readinto() returned invalid length {n} (should have been between 0 and {len})" - ))); - } - if n > 0 && self.abs_pos != -1 { - self.abs_pos += n as Offset - } - Ok(Some(n as usize)) - } - - fn read_all(&mut self, vm: &VirtualMachine) -> PyResult<Option<PyBytesRef>> { - let buf = self.active_read_slice(); - let data = if buf.is_empty() { - None - } else { - let b = buf.to_vec(); - self.pos += buf.len() as Offset; - Some(b) - }; - - if self.writable() { - self.flush_rewind(vm)?; - } - - let readall = vm - .get_str_method(self.raw.clone().unwrap(), "readall") - .transpose()?; - if let Some(readall) = readall { - let res = readall.call((), vm)?; - let res = <Option<PyBytesRef>>::try_from_object(vm, res)?; - let ret = if let Some(mut data) = data { - if let Some(bytes) = res { - data.extend_from_slice(bytes.as_bytes()); - } - Some(PyBytes::from(data).into_ref(&vm.ctx)) - } else { - res - }; - return Ok(ret); - } - - let mut chunks = Vec::new(); - - let mut read_size = 0; - loop { - let read_data = vm.call_method(self.raw.as_ref().unwrap(), "read", ())?; - let read_data = <Option<PyBytesRef>>::try_from_object(vm, read_data)?; - - match read_data { - Some(b) if !b.as_bytes().is_empty() => { - let l = b.as_bytes().len(); - read_size += l; - if self.abs_pos != -1 { - self.abs_pos += l as Offset; - } - chunks.push(b); - } - read_data => { - let ret = if data.is_none() && read_size == 0 { - read_data - } else { - let mut data = data.unwrap_or_default(); - data.reserve(read_size); - for bytes in &chunks { - data.extend_from_slice(bytes.as_bytes()) - } - Some(PyBytes::from(data).into_ref(&vm.ctx)) - }; - break Ok(ret); - } - } - } - } - - fn adjust_position(&mut self, new_pos: Offset) { - self.pos = new_pos; - if self.valid_read() && self.read_end < self.pos { - self.read_end = self.pos - } - } - - fn peek(&mut self, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let have = self.readahead(); - let slice = if have > 0 { - &self.buffer[self.pos as usize..][..have as usize] - } else { - self.reset_read(); - let r = self.fill_buffer(vm)?.unwrap_or(0); - self.pos = 0; - &self.buffer[..r] - }; - Ok(slice.to_vec()) - } - - fn readinto_generic( - &mut self, - buf: PyBuffer, - readinto1: bool, - vm: &VirtualMachine, - ) -> PyResult<Option<usize>> { - let mut written = 0; - let n = self.readahead(); - let buf_len; - { - let mut b = buf.as_contiguous_mut().unwrap(); - buf_len = b.len(); - if n > 0 { - if n as usize >= b.len() { - b.copy_from_slice(&self.buffer[self.pos as usize..][..buf_len]); - self.pos += buf_len as Offset; - return Ok(Some(buf_len)); - } - b[..n as usize] - .copy_from_slice(&self.buffer[self.pos as usize..][..n as usize]); - self.pos += n; - written = n as usize; - } - } - if self.writable() { - self.flush_rewind(vm)?; - } - self.reset_read(); - self.pos = 0; - - let mut remaining = buf_len - written; - while remaining > 0 { - let n = if remaining > self.buffer.len() { - self.raw_read(Either::B(buf.clone()), written..written + remaining, vm)? - } else if !(readinto1 && written != 0) { - let n = self.fill_buffer(vm)?; - if let Some(n) = n.filter(|&n| n > 0) { - let n = std::cmp::min(n, remaining); - buf.as_contiguous_mut().unwrap()[written..][..n] - .copy_from_slice(&self.buffer[self.pos as usize..][..n]); - self.pos += n as Offset; - written += n; - remaining -= n; - continue; - } - n - } else { - break; - }; - let n = match n { - Some(0) => break, - None if written > 0 => break, - None => return Ok(None), - Some(n) => n, - }; - - if readinto1 { - written += n; - break; - } - written += n; - remaining -= n; - } - - Ok(Some(written)) - } - } - - pub fn get_offset(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Offset> { - let int = obj.try_index(vm)?; - int.as_bigint().try_into().map_err(|_| { - vm.new_value_error(format!( - "cannot fit '{}' into an offset-sized integer", - obj.class().name() - )) - }) - } - - pub fn repr_fileobj_name(obj: &PyObject, vm: &VirtualMachine) -> PyResult<Option<PyStrRef>> { - let name = match obj.get_attr("name", vm) { - Ok(name) => Some(name), - Err(e) - if e.fast_isinstance(vm.ctx.exceptions.attribute_error) - || e.fast_isinstance(vm.ctx.exceptions.value_error) => - { - None - } - Err(e) => return Err(e), - }; - match name { - Some(name) => { - if let Some(_guard) = ReprGuard::enter(vm, obj) { - name.repr(vm).map(Some) - } else { - Err(vm.new_runtime_error(format!( - "reentrant call inside {}.__repr__", - obj.class().slot_name() - ))) - } - } - None => Ok(None), - } - } - - #[pyclass] - trait BufferedMixin: PyPayload { - const CLASS_NAME: &'static str; - const READABLE: bool; - const WRITABLE: bool; - const SEEKABLE: bool = false; - fn data(&self) -> &PyThreadMutex<BufferedData>; - fn lock(&self, vm: &VirtualMachine) -> PyResult<PyThreadMutexGuard<BufferedData>> { - self.data() - .lock() - .ok_or_else(|| vm.new_runtime_error("reentrant call inside buffered io".to_owned())) - } - - #[pyslot] - fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let zelf: PyRef<Self> = zelf.try_into_value(vm)?; - zelf.__init__(args, vm) - } - - #[pymethod] - fn __init__(&self, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let (raw, BufferSize { buffer_size }): (PyObjectRef, _) = - args.bind(vm).map_err(|e| { - let msg = format!("{}() {}", Self::CLASS_NAME, *e.str(vm)); - vm.new_exception_msg(e.class().to_owned(), msg) - })?; - self.init(raw, BufferSize { buffer_size }, vm) - } - - fn init( - &self, - raw: PyObjectRef, - BufferSize { buffer_size }: BufferSize, - vm: &VirtualMachine, - ) -> PyResult<()> { - let mut data = self.lock(vm)?; - data.raw = None; - data.flags.remove(BufferedFlags::DETACHED); - - let buffer_size = match buffer_size { - OptionalArg::Present(i) if i <= 0 => { - return Err( - vm.new_value_error("buffer size must be strictly positive".to_owned()) - ); - } - OptionalArg::Present(i) => i as usize, - OptionalArg::Missing => DEFAULT_BUFFER_SIZE, - }; - - if Self::SEEKABLE { - check_seekable(&raw, vm)?; - } - if Self::READABLE { - data.flags.insert(BufferedFlags::READABLE); - check_readable(&raw, vm)?; - } - if Self::WRITABLE { - data.flags.insert(BufferedFlags::WRITABLE); - check_writable(&raw, vm)?; - } - - data.buffer = vec![0; buffer_size]; - - if Self::READABLE { - data.reset_read(); - } - if Self::WRITABLE { - data.reset_write(); - } - if Self::SEEKABLE { - data.pos = 0; - } - - data.raw = Some(raw); - - Ok(()) - } - #[pymethod] - fn seek( - &self, - target: PyObjectRef, - whence: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<Offset> { - let whence = whence.unwrap_or(0); - if !validate_whence(whence) { - return Err(vm.new_value_error(format!("whence value {whence} unsupported"))); - } - let mut data = self.lock(vm)?; - let raw = data.check_init(vm)?; - ensure_unclosed(raw, "seek of closed file", vm)?; - check_seekable(raw, vm)?; - let target = get_offset(target, vm)?; - data.seek(target, whence, vm) - } - #[pymethod] - fn tell(&self, vm: &VirtualMachine) -> PyResult<Offset> { - let mut data = self.lock(vm)?; - Ok(data.raw_tell(vm)? - data.raw_offset()) - } - #[pymethod] - fn truncate( - zelf: PyRef<Self>, - pos: OptionalOption<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - let pos = pos.flatten().to_pyobject(vm); - let mut data = zelf.lock(vm)?; - data.check_init(vm)?; - if data.writable() { - data.flush_rewind(vm)?; - } - let res = vm.call_method(data.raw.as_ref().unwrap(), "truncate", (pos,))?; - let _ = data.raw_tell(vm); - Ok(res) - } - #[pymethod] - fn detach(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - vm.call_method(zelf.as_object(), "flush", ())?; - let mut data = zelf.lock(vm)?; - data.flags.insert(BufferedFlags::DETACHED); - data.raw - .take() - .ok_or_else(|| vm.new_value_error("raw stream has been detached".to_owned())) - } - #[pymethod] - fn seekable(&self, vm: &VirtualMachine) -> PyResult { - vm.call_method(self.lock(vm)?.check_init(vm)?, "seekable", ()) - } - #[pygetset] - fn raw(&self, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { - Ok(self.lock(vm)?.raw.clone()) - } - #[pygetset] - fn closed(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("closed", vm) - } - #[pygetset] - fn name(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("name", vm) - } - #[pygetset] - fn mode(&self, vm: &VirtualMachine) -> PyResult { - self.lock(vm)?.check_init(vm)?.get_attr("mode", vm) - } - #[pymethod] - fn fileno(&self, vm: &VirtualMachine) -> PyResult { - vm.call_method(self.lock(vm)?.check_init(vm)?, "fileno", ()) - } - #[pymethod] - fn isatty(&self, vm: &VirtualMachine) -> PyResult { - vm.call_method(self.lock(vm)?.check_init(vm)?, "isatty", ()) - } - - #[pyslot] - fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let name_repr = repr_fileobj_name(zelf, vm)?; - let cls = zelf.class(); - let slot_name = cls.slot_name(); - let repr = if let Some(name_repr) = name_repr { - format!("<{slot_name} name={name_repr}>") - } else { - format!("<{slot_name}>") - }; - Ok(vm.ctx.new_str(repr)) - } - - #[pymethod(magic)] - fn repr(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Self::slot_repr(&zelf, vm) - } - - fn close_strict(&self, vm: &VirtualMachine) -> PyResult { - let mut data = self.lock(vm)?; - let raw = data.check_init(vm)?; - if file_closed(raw, vm)? { - return Ok(vm.ctx.none()); - } - let flush_res = data.flush(vm); - let close_res = vm.call_method(data.raw.as_ref().unwrap(), "close", ()); - exception_chain(flush_res, close_res) - } - - #[pymethod] - fn close(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - { - let data = zelf.lock(vm)?; - let raw = data.check_init(vm)?; - if file_closed(raw, vm)? { - return Ok(vm.ctx.none()); - } - } - let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop); - let data = zelf.lock(vm)?; - let raw = data.raw.as_ref().unwrap(); - let close_res = vm.call_method(raw, "close", ()); - exception_chain(flush_res, close_res) - } - - #[pymethod] - fn readable(&self) -> bool { - Self::READABLE - } - #[pymethod] - fn writable(&self) -> bool { - Self::WRITABLE - } - - // TODO: this should be the default for an equivalent of _PyObject_GetState - #[pymethod(magic)] - fn reduce(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) - } - } - - #[pyclass] - trait BufferedReadable: PyPayload { - type Reader: BufferedMixin; - fn reader(&self) -> &Self::Reader; - #[pymethod] - fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Option<PyBytesRef>> { - let mut data = self.reader().lock(vm)?; - let raw = data.check_init(vm)?; - let n = size.size.map(|s| *s).unwrap_or(-1); - if n < -1 { - return Err(vm.new_value_error("read length must be non-negative or -1".to_owned())); - } - ensure_unclosed(raw, "read of closed file", vm)?; - match n.to_usize() { - Some(n) => data - .read_generic(n, vm) - .map(|x| x.map(|b| PyBytes::from(b).into_ref(&vm.ctx))), - None => data.read_all(vm), - } - } - #[pymethod] - fn peek(&self, _size: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let mut data = self.reader().lock(vm)?; - let raw = data.check_init(vm)?; - ensure_unclosed(raw, "peek of closed file", vm)?; - - if data.writable() { - let _ = data.flush_rewind(vm); - } - data.peek(vm) - } - #[pymethod] - fn read1(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let mut data = self.reader().lock(vm)?; - let raw = data.check_init(vm)?; - ensure_unclosed(raw, "read of closed file", vm)?; - let n = size.to_usize().unwrap_or(data.buffer.len()); - if n == 0 { - return Ok(Vec::new()); - } - let have = data.readahead(); - if have > 0 { - let n = std::cmp::min(have as usize, n); - return Ok(data.read_fast(n).unwrap()); - } - let mut v = vec![0; n]; - data.reset_read(); - let r = data - .raw_read(Either::A(Some(&mut v)), 0..n, vm)? - .unwrap_or(0); - v.truncate(r); - v.shrink_to_fit(); - Ok(v) - } - #[pymethod] - fn readinto(&self, buf: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<Option<usize>> { - let mut data = self.reader().lock(vm)?; - let raw = data.check_init(vm)?; - ensure_unclosed(raw, "readinto of closed file", vm)?; - data.readinto_generic(buf.into(), false, vm) - } - #[pymethod] - fn readinto1(&self, buf: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<Option<usize>> { - let mut data = self.reader().lock(vm)?; - let raw = data.check_init(vm)?; - ensure_unclosed(raw, "readinto of closed file", vm)?; - data.readinto_generic(buf.into(), true, vm) - } - } - - fn exception_chain<T>(e1: PyResult<()>, e2: PyResult<T>) -> PyResult<T> { - match (e1, e2) { - (Err(e1), Err(e)) => { - e.set_context(Some(e1)); - Err(e) - } - (Err(e), Ok(_)) | (Ok(()), Err(e)) => Err(e), - (Ok(()), Ok(close_res)) => Ok(close_res), - } - } - - #[pyattr] - #[pyclass(name = "BufferedReader", base = "_BufferedIOBase")] - #[derive(Debug, Default, PyPayload)] - struct BufferedReader { - data: PyThreadMutex<BufferedData>, - } - impl BufferedMixin for BufferedReader { - const CLASS_NAME: &'static str = "BufferedReader"; - const READABLE: bool = true; - const WRITABLE: bool = false; - fn data(&self) -> &PyThreadMutex<BufferedData> { - &self.data - } - } - impl BufferedReadable for BufferedReader { - type Reader = Self; - fn reader(&self) -> &Self::Reader { - self - } - } - - #[pyclass( - with(DefaultConstructor, BufferedMixin, BufferedReadable), - flags(BASETYPE, HAS_DICT) - )] - impl BufferedReader {} - - impl DefaultConstructor for BufferedReader {} - - #[pyclass] - trait BufferedWritable: PyPayload { - type Writer: BufferedMixin; - fn writer(&self) -> &Self::Writer; - #[pymethod] - fn write(&self, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { - let mut data = self.writer().lock(vm)?; - let raw = data.check_init(vm)?; - ensure_unclosed(raw, "write to closed file", vm)?; - - data.write(obj, vm) - } - #[pymethod] - fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { - let mut data = self.writer().lock(vm)?; - let raw = data.check_init(vm)?; - ensure_unclosed(raw, "flush of closed file", vm)?; - data.flush_rewind(vm) - } - } - - #[pyattr] - #[pyclass(name = "BufferedWriter", base = "_BufferedIOBase")] - #[derive(Debug, Default, PyPayload)] - struct BufferedWriter { - data: PyThreadMutex<BufferedData>, - } - impl BufferedMixin for BufferedWriter { - const CLASS_NAME: &'static str = "BufferedWriter"; - const READABLE: bool = false; - const WRITABLE: bool = true; - fn data(&self) -> &PyThreadMutex<BufferedData> { - &self.data - } - } - impl BufferedWritable for BufferedWriter { - type Writer = Self; - fn writer(&self) -> &Self::Writer { - self - } - } - - #[pyclass( - with(DefaultConstructor, BufferedMixin, BufferedWritable), - flags(BASETYPE, HAS_DICT) - )] - impl BufferedWriter {} - - impl DefaultConstructor for BufferedWriter {} - - #[pyattr] - #[pyclass(name = "BufferedRandom", base = "_BufferedIOBase")] - #[derive(Debug, Default, PyPayload)] - struct BufferedRandom { - data: PyThreadMutex<BufferedData>, - } - impl BufferedMixin for BufferedRandom { - const CLASS_NAME: &'static str = "BufferedRandom"; - const READABLE: bool = true; - const WRITABLE: bool = true; - const SEEKABLE: bool = true; - fn data(&self) -> &PyThreadMutex<BufferedData> { - &self.data - } - } - impl BufferedReadable for BufferedRandom { - type Reader = Self; - fn reader(&self) -> &Self::Reader { - self - } - } - impl BufferedWritable for BufferedRandom { - type Writer = Self; - fn writer(&self) -> &Self::Writer { - self - } - } - - #[pyclass( - with(DefaultConstructor, BufferedMixin, BufferedReadable, BufferedWritable), - flags(BASETYPE, HAS_DICT) - )] - impl BufferedRandom {} - - impl DefaultConstructor for BufferedRandom {} - - #[pyattr] - #[pyclass(name = "BufferedRWPair", base = "_BufferedIOBase")] - #[derive(Debug, Default, PyPayload)] - struct BufferedRWPair { - read: BufferedReader, - write: BufferedWriter, - } - impl BufferedReadable for BufferedRWPair { - type Reader = BufferedReader; - fn reader(&self) -> &Self::Reader { - &self.read - } - } - impl BufferedWritable for BufferedRWPair { - type Writer = BufferedWriter; - fn writer(&self) -> &Self::Writer { - &self.write - } - } - - impl DefaultConstructor for BufferedRWPair {} - - impl Initializer for BufferedRWPair { - type Args = (PyObjectRef, PyObjectRef, BufferSize); - - fn init( - zelf: PyRef<Self>, - (reader, writer, buffer_size): Self::Args, - vm: &VirtualMachine, - ) -> PyResult<()> { - zelf.read.init(reader, buffer_size.clone(), vm)?; - zelf.write.init(writer, buffer_size, vm)?; - Ok(()) - } - } - - #[pyclass( - with(DefaultConstructor, Initializer, BufferedReadable, BufferedWritable), - flags(BASETYPE, HAS_DICT) - )] - impl BufferedRWPair { - #[pymethod] - fn flush(&self, vm: &VirtualMachine) -> PyResult<()> { - self.write.flush(vm) - } - - #[pymethod] - fn readable(&self) -> bool { - true - } - #[pymethod] - fn writable(&self) -> bool { - true - } - - #[pygetset] - fn closed(&self, vm: &VirtualMachine) -> PyResult { - self.write.closed(vm) - } - - #[pymethod] - fn isatty(&self, vm: &VirtualMachine) -> PyResult { - // read.isatty() or write.isatty() - let res = self.read.isatty(vm)?; - if res.clone().try_to_bool(vm)? { - Ok(res) - } else { - self.write.isatty(vm) - } - } - - #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult { - let write_res = self.write.close_strict(vm).map(drop); - let read_res = self.read.close_strict(vm); - exception_chain(write_res, read_res) - } - } - - #[derive(FromArgs)] - struct TextIOWrapperArgs { - buffer: PyObjectRef, - #[pyarg(any, default)] - encoding: Option<PyStrRef>, - #[pyarg(any, default)] - errors: Option<PyStrRef>, - #[pyarg(any, default)] - newline: Newlines, - #[pyarg(any, default = "false")] - line_buffering: bool, - #[pyarg(any, default = "false")] - write_through: bool, - } - - #[derive(Debug, Copy, Clone, Default)] - enum Newlines { - #[default] - Universal, - Passthrough, - Lf, - Cr, - Crlf, - } - - impl Newlines { - /// returns position where the new line starts if found, otherwise position at which to - /// continue the search after more is read into the buffer - fn find_newline(&self, s: &str) -> Result<usize, usize> { - let len = s.len(); - match self { - Newlines::Universal | Newlines::Lf => s.find('\n').map(|p| p + 1).ok_or(len), - Newlines::Passthrough => { - let bytes = s.as_bytes(); - memchr::memchr2(b'\n', b'\r', bytes) - .map(|p| { - let nl_len = - if bytes[p] == b'\r' && bytes.get(p + 1).copied() == Some(b'\n') { - 2 - } else { - 1 - }; - p + nl_len - }) - .ok_or(len) - } - Newlines::Cr => s.find('\n').map(|p| p + 1).ok_or(len), - Newlines::Crlf => { - // s[searched..] == remaining - let mut searched = 0; - let mut remaining = s.as_bytes(); - loop { - match memchr::memchr(b'\r', remaining) { - Some(p) => match remaining.get(p + 1) { - Some(&ch_after_cr) => { - let pos_after = p + 2; - if ch_after_cr == b'\n' { - break Ok(searched + pos_after); - } else { - searched += pos_after; - remaining = &remaining[pos_after..]; - continue; - } - } - None => break Err(searched + p), - }, - None => break Err(len), - } - } - } - } - } - } - - impl TryFromObject for Newlines { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let nl = if vm.is_none(&obj) { - Self::Universal - } else { - let s = obj.downcast::<PyStr>().map_err(|obj| { - vm.new_type_error(format!( - "newline argument must be str or None, not {}", - obj.class().name() - )) - })?; - match s.as_str() { - "" => Self::Passthrough, - "\n" => Self::Lf, - "\r" => Self::Cr, - "\r\n" => Self::Crlf, - _ => return Err(vm.new_value_error(format!("illegal newline value: {s}"))), - } - }; - Ok(nl) - } - } - - /// A length of or index into a UTF-8 string, measured in both chars and bytes - #[derive(Debug, Default, Copy, Clone)] - struct Utf8size { - bytes: usize, - chars: usize, - } - impl Utf8size { - fn len_pystr(s: &PyStr) -> Self { - Utf8size { - bytes: s.byte_len(), - chars: s.char_len(), - } - } - - fn len_str(s: &str) -> Self { - Utf8size { - bytes: s.len(), - chars: s.chars().count(), - } - } - } - impl std::ops::Add for Utf8size { - type Output = Self; - #[inline] - fn add(mut self, rhs: Self) -> Self { - self += rhs; - self - } - } - impl std::ops::AddAssign for Utf8size { - #[inline] - fn add_assign(&mut self, rhs: Self) { - self.bytes += rhs.bytes; - self.chars += rhs.chars; - } - } - impl std::ops::Sub for Utf8size { - type Output = Self; - #[inline] - fn sub(mut self, rhs: Self) -> Self { - self -= rhs; - self - } - } - impl std::ops::SubAssign for Utf8size { - #[inline] - fn sub_assign(&mut self, rhs: Self) { - self.bytes -= rhs.bytes; - self.chars -= rhs.chars; - } - } - - // TODO: implement legit fast-paths for other encodings - type EncodeFunc = fn(PyStrRef) -> PendingWrite; - fn textio_encode_utf8(s: PyStrRef) -> PendingWrite { - PendingWrite::Utf8(s) - } - - #[derive(Debug)] - struct TextIOData { - buffer: PyObjectRef, - encoder: Option<(PyObjectRef, Option<EncodeFunc>)>, - decoder: Option<PyObjectRef>, - encoding: PyStrRef, - errors: PyStrRef, - newline: Newlines, - line_buffering: bool, - write_through: bool, - chunk_size: usize, - seekable: bool, - has_read1: bool, - // these are more state than configuration - pending: PendingWrites, - telling: bool, - snapshot: Option<(i32, PyBytesRef)>, - decoded_chars: Option<PyStrRef>, - // number of characters we've consumed from decoded_chars - decoded_chars_used: Utf8size, - b2cratio: f64, - } - - #[derive(Debug, Default)] - struct PendingWrites { - num_bytes: usize, - data: PendingWritesData, - } - - #[derive(Debug, Default)] - enum PendingWritesData { - #[default] - None, - One(PendingWrite), - Many(Vec<PendingWrite>), - } - - #[derive(Debug)] - enum PendingWrite { - Utf8(PyStrRef), - Bytes(PyBytesRef), - } - - impl PendingWrite { - fn as_bytes(&self) -> &[u8] { - match self { - Self::Utf8(s) => s.as_str().as_bytes(), - Self::Bytes(b) => b.as_bytes(), - } - } - } - - impl PendingWrites { - fn push(&mut self, write: PendingWrite) { - self.num_bytes += write.as_bytes().len(); - self.data = match std::mem::take(&mut self.data) { - PendingWritesData::None => PendingWritesData::One(write), - PendingWritesData::One(write1) => PendingWritesData::Many(vec![write1, write]), - PendingWritesData::Many(mut v) => { - v.push(write); - PendingWritesData::Many(v) - } - } - } - fn take(&mut self, vm: &VirtualMachine) -> PyBytesRef { - let PendingWrites { num_bytes, data } = std::mem::take(self); - if let PendingWritesData::One(PendingWrite::Bytes(b)) = data { - return b; - } - let writes_iter = match data { - PendingWritesData::None => itertools::Either::Left(vec![].into_iter()), - PendingWritesData::One(write) => itertools::Either::Right(std::iter::once(write)), - PendingWritesData::Many(writes) => itertools::Either::Left(writes.into_iter()), - }; - let mut buf = Vec::with_capacity(num_bytes); - writes_iter.for_each(|chunk| buf.extend_from_slice(chunk.as_bytes())); - PyBytes::from(buf).into_ref(&vm.ctx) - } - } - - #[derive(Default, Debug)] - struct TextIOCookie { - start_pos: Offset, - dec_flags: i32, - bytes_to_feed: i32, - chars_to_skip: i32, - need_eof: bool, - // chars_to_skip but utf8 bytes - bytes_to_skip: i32, - } - - impl TextIOCookie { - const START_POS_OFF: usize = 0; - const DEC_FLAGS_OFF: usize = Self::START_POS_OFF + std::mem::size_of::<Offset>(); - const BYTES_TO_FEED_OFF: usize = Self::DEC_FLAGS_OFF + 4; - const CHARS_TO_SKIP_OFF: usize = Self::BYTES_TO_FEED_OFF + 4; - const NEED_EOF_OFF: usize = Self::CHARS_TO_SKIP_OFF + 4; - const BYTES_TO_SKIP_OFF: usize = Self::NEED_EOF_OFF + 1; - const BYTE_LEN: usize = Self::BYTES_TO_SKIP_OFF + 4; - fn parse(cookie: &BigInt) -> Option<Self> { - let (_, mut buf) = cookie.to_bytes_le(); - if buf.len() > Self::BYTE_LEN { - return None; - } - buf.resize(Self::BYTE_LEN, 0); - let buf: &[u8; Self::BYTE_LEN] = buf.as_slice().try_into().unwrap(); - macro_rules! get_field { - ($t:ty, $off:ident) => {{ - <$t>::from_ne_bytes( - buf[Self::$off..][..std::mem::size_of::<$t>()] - .try_into() - .unwrap(), - ) - }}; - } - Some(TextIOCookie { - start_pos: get_field!(Offset, START_POS_OFF), - dec_flags: get_field!(i32, DEC_FLAGS_OFF), - bytes_to_feed: get_field!(i32, BYTES_TO_FEED_OFF), - chars_to_skip: get_field!(i32, CHARS_TO_SKIP_OFF), - need_eof: get_field!(u8, NEED_EOF_OFF) != 0, - bytes_to_skip: get_field!(i32, BYTES_TO_SKIP_OFF), - }) - } - fn build(&self) -> BigInt { - let mut buf = [0; Self::BYTE_LEN]; - macro_rules! set_field { - ($field:expr, $off:ident) => {{ - let field = $field; - buf[Self::$off..][..std::mem::size_of_val(&field)] - .copy_from_slice(&field.to_ne_bytes()) - }}; - } - set_field!(self.start_pos, START_POS_OFF); - set_field!(self.dec_flags, DEC_FLAGS_OFF); - set_field!(self.bytes_to_feed, BYTES_TO_FEED_OFF); - set_field!(self.chars_to_skip, CHARS_TO_SKIP_OFF); - set_field!(self.need_eof as u8, NEED_EOF_OFF); - set_field!(self.bytes_to_skip, BYTES_TO_SKIP_OFF); - BigUint::from_bytes_le(&buf).into() - } - fn set_decoder_state(&self, decoder: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - if self.start_pos == 0 && self.dec_flags == 0 { - vm.call_method(decoder, "reset", ())?; - } else { - vm.call_method( - decoder, - "setstate", - ((vm.ctx.new_bytes(vec![]), self.dec_flags),), - )?; - } - Ok(()) - } - fn num_to_skip(&self) -> Utf8size { - Utf8size { - bytes: self.bytes_to_skip as usize, - chars: self.chars_to_skip as usize, - } - } - fn set_num_to_skip(&mut self, num: Utf8size) { - self.bytes_to_skip = num.bytes as i32; - self.chars_to_skip = num.chars as i32; - } - } - - #[pyattr] - #[pyclass(name = "TextIOWrapper", base = "_TextIOBase")] - #[derive(Debug, Default, PyPayload)] - struct TextIOWrapper { - data: PyThreadMutex<Option<TextIOData>>, - } - - impl DefaultConstructor for TextIOWrapper {} - - impl Initializer for TextIOWrapper { - type Args = TextIOWrapperArgs; - - fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - let mut data = zelf.lock_opt(vm)?; - *data = None; - - let encoding = match args.encoding { - None if vm.state.settings.utf8_mode > 0 => PyStr::from("utf-8").into_ref(&vm.ctx), - Some(enc) if enc.as_str() != "locale" => enc, - _ => { - // None without utf8_mode or "locale" encoding - vm.import("locale", None, 0)? - .get_attr("getencoding", vm)? - .call((), vm)? - .try_into_value(vm)? - } - }; - - let errors = args - .errors - .unwrap_or_else(|| PyStr::from("strict").into_ref(&vm.ctx)); - - let buffer = args.buffer; - - let has_read1 = vm.get_attribute_opt(buffer.clone(), "read1")?.is_some(); - let seekable = vm.call_method(&buffer, "seekable", ())?.try_to_bool(vm)?; - - let codec = vm.state.codec_registry.lookup(encoding.as_str(), vm)?; - - let encoder = if vm.call_method(&buffer, "writable", ())?.try_to_bool(vm)? { - let incremental_encoder = - codec.get_incremental_encoder(Some(errors.clone()), vm)?; - let encoding_name = vm.get_attribute_opt(incremental_encoder.clone(), "name")?; - let encodefunc = encoding_name.and_then(|name| { - let name = name.payload::<PyStr>()?; - match name.as_str() { - "utf-8" => Some(textio_encode_utf8 as EncodeFunc), - _ => None, - } - }); - Some((incremental_encoder, encodefunc)) - } else { - None - }; - - let decoder = if vm.call_method(&buffer, "readable", ())?.try_to_bool(vm)? { - let incremental_decoder = - codec.get_incremental_decoder(Some(errors.clone()), vm)?; - // TODO: wrap in IncrementalNewlineDecoder if newlines == Universal | Passthrough - Some(incremental_decoder) - } else { - None - }; - - *data = Some(TextIOData { - buffer, - encoder, - decoder, - encoding, - errors, - newline: args.newline, - line_buffering: args.line_buffering, - write_through: args.write_through, - chunk_size: 8192, - seekable, - has_read1, - - pending: PendingWrites::default(), - telling: seekable, - snapshot: None, - decoded_chars: None, - decoded_chars_used: Utf8size::default(), - b2cratio: 0.0, - }); - - Ok(()) - } - } - - impl TextIOWrapper { - fn lock_opt( - &self, - vm: &VirtualMachine, - ) -> PyResult<PyThreadMutexGuard<Option<TextIOData>>> { - self.data - .lock() - .ok_or_else(|| vm.new_runtime_error("reentrant call inside textio".to_owned())) - } - - fn lock(&self, vm: &VirtualMachine) -> PyResult<PyMappedThreadMutexGuard<TextIOData>> { - let lock = self.lock_opt(vm)?; - PyThreadMutexGuard::try_map(lock, |x| x.as_mut()) - .map_err(|_| vm.new_value_error("I/O operation on uninitialized object".to_owned())) - } - } - - #[pyclass(with(DefaultConstructor, Initializer), flags(BASETYPE))] - impl TextIOWrapper { - #[pymethod] - fn seekable(&self, vm: &VirtualMachine) -> PyResult { - let textio = self.lock(vm)?; - vm.call_method(&textio.buffer, "seekable", ()) - } - #[pymethod] - fn readable(&self, vm: &VirtualMachine) -> PyResult { - let textio = self.lock(vm)?; - vm.call_method(&textio.buffer, "readable", ()) - } - #[pymethod] - fn writable(&self, vm: &VirtualMachine) -> PyResult { - let textio = self.lock(vm)?; - vm.call_method(&textio.buffer, "writable", ()) - } - - #[pygetset(name = "_CHUNK_SIZE")] - fn chunksize(&self, vm: &VirtualMachine) -> PyResult<usize> { - Ok(self.lock(vm)?.chunk_size) - } - - #[pygetset(setter, name = "_CHUNK_SIZE")] - fn set_chunksize( - &self, - chunk_size: PySetterValue<usize>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let mut textio = self.lock(vm)?; - match chunk_size { - PySetterValue::Assign(chunk_size) => textio.chunk_size = chunk_size, - PySetterValue::Delete => { - Err(vm.new_attribute_error("cannot delete attribute".to_owned()))? - } - }; - // TODO: RUSTPYTHON - // Change chunk_size type, validate it manually and throws ValueError if invalid. - // https://github.com/python/cpython/blob/2e9da8e3522764d09f1d6054a2be567e91a30812/Modules/_io/textio.c#L3124-L3143 - Ok(()) - } - - #[pymethod] - fn seek( - zelf: PyRef<Self>, - cookie: PyObjectRef, - how: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult { - let how = how.unwrap_or(0); - - let reset_encoder = |encoder, start_of_stream| { - if start_of_stream { - vm.call_method(encoder, "reset", ()) - } else { - vm.call_method(encoder, "setstate", (0,)) - } - }; - - let textio = zelf.lock(vm)?; - - if !textio.seekable { - return Err(new_unsupported_operation( - vm, - "underlying stream is not seekable".to_owned(), - )); - } - - let cookie = match how { - // SEEK_SET - 0 => cookie, - // SEEK_CUR - 1 => { - if vm.bool_eq(&cookie, vm.ctx.new_int(0).as_ref())? { - vm.call_method(&textio.buffer, "tell", ())? - } else { - return Err(new_unsupported_operation( - vm, - "can't do nonzero cur-relative seeks".to_owned(), - )); - } - } - // SEEK_END - 2 => { - if vm.bool_eq(&cookie, vm.ctx.new_int(0).as_ref())? { - drop(textio); - vm.call_method(zelf.as_object(), "flush", ())?; - let mut textio = zelf.lock(vm)?; - textio.set_decoded_chars(None); - textio.snapshot = None; - if let Some(decoder) = &textio.decoder { - vm.call_method(decoder, "reset", ())?; - } - let res = vm.call_method(&textio.buffer, "seek", (0, 2))?; - if let Some((encoder, _)) = &textio.encoder { - let start_of_stream = vm.bool_eq(&res, vm.ctx.new_int(0).as_ref())?; - reset_encoder(encoder, start_of_stream)?; - } - return Ok(res); - } else { - return Err(new_unsupported_operation( - vm, - "can't do nonzero end-relative seeks".to_owned(), - )); - } - } - _ => { - return Err( - vm.new_value_error(format!("invalid whence ({how}, should be 0, 1 or 2)")) - ) - } - }; - use crate::types::PyComparisonOp; - if cookie.rich_compare_bool(vm.ctx.new_int(0).as_ref(), PyComparisonOp::Lt, vm)? { - return Err( - vm.new_value_error(format!("negative seek position {}", &cookie.repr(vm)?)) - ); - } - drop(textio); - vm.call_method(zelf.as_object(), "flush", ())?; - let cookie_obj = crate::builtins::PyIntRef::try_from_object(vm, cookie)?; - let cookie = TextIOCookie::parse(cookie_obj.as_bigint()) - .ok_or_else(|| vm.new_value_error("invalid cookie".to_owned()))?; - let mut textio = zelf.lock(vm)?; - vm.call_method(&textio.buffer, "seek", (cookie.start_pos,))?; - textio.set_decoded_chars(None); - textio.snapshot = None; - if let Some(decoder) = &textio.decoder { - cookie.set_decoder_state(decoder, vm)?; - } - if cookie.chars_to_skip != 0 { - let TextIOData { - ref decoder, - ref buffer, - ref mut snapshot, - .. - } = *textio; - let decoder = decoder - .as_ref() - .ok_or_else(|| vm.new_value_error("invalid cookie".to_owned()))?; - let input_chunk = vm.call_method(buffer, "read", (cookie.bytes_to_feed,))?; - let input_chunk: PyBytesRef = input_chunk.downcast().map_err(|obj| { - vm.new_type_error(format!( - "underlying read() should have returned a bytes object, not '{}'", - obj.class().name() - )) - })?; - *snapshot = Some((cookie.dec_flags, input_chunk.clone())); - let decoded = vm.call_method(decoder, "decode", (input_chunk, cookie.need_eof))?; - let decoded = check_decoded(decoded, vm)?; - let pos_is_valid = decoded - .as_str() - .is_char_boundary(cookie.bytes_to_skip as usize); - textio.set_decoded_chars(Some(decoded)); - if !pos_is_valid { - return Err(vm.new_os_error("can't restore logical file position".to_owned())); - } - textio.decoded_chars_used = cookie.num_to_skip(); - } else { - textio.snapshot = Some((cookie.dec_flags, PyBytes::from(vec![]).into_ref(&vm.ctx))) - } - if let Some((encoder, _)) = &textio.encoder { - let start_of_stream = cookie.start_pos == 0 && cookie.dec_flags == 0; - reset_encoder(encoder, start_of_stream)?; - } - Ok(cookie_obj.into()) - } - - #[pymethod] - fn tell(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult { - let mut textio = zelf.lock(vm)?; - if !textio.seekable { - return Err(new_unsupported_operation( - vm, - "underlying stream is not seekable".to_owned(), - )); - } - if !textio.telling { - return Err(vm.new_os_error("telling position disabled by next() call".to_owned())); - } - textio.write_pending(vm)?; - drop(textio); - vm.call_method(zelf.as_object(), "flush", ())?; - let textio = zelf.lock(vm)?; - let pos = vm.call_method(&textio.buffer, "tell", ())?; - let (decoder, (dec_flags, next_input)) = match (&textio.decoder, &textio.snapshot) { - (Some(d), Some(s)) => (d, s), - _ => return Ok(pos), - }; - let pos = Offset::try_from_object(vm, pos)?; - let mut cookie = TextIOCookie { - start_pos: pos - next_input.len() as Offset, - dec_flags: *dec_flags, - ..Default::default() - }; - if textio.decoded_chars_used.bytes == 0 { - return Ok(cookie.build().to_pyobject(vm)); - } - let decoder_getstate = || { - let state = vm.call_method(decoder, "getstate", ())?; - parse_decoder_state(state, vm) - }; - let decoder_decode = |b: &[u8]| { - let decoded = vm.call_method(decoder, "decode", (vm.ctx.new_bytes(b.to_vec()),))?; - let decoded = check_decoded(decoded, vm)?; - Ok(Utf8size::len_pystr(&decoded)) - }; - let saved_state = vm.call_method(decoder, "getstate", ())?; - let mut num_to_skip = textio.decoded_chars_used; - let mut skip_bytes = (textio.b2cratio * num_to_skip.chars as f64) as isize; - let mut skip_back = 1; - while skip_bytes > 0 { - cookie.set_decoder_state(decoder, vm)?; - let input = &next_input.as_bytes()[..skip_bytes as usize]; - let ndecoded = decoder_decode(input)?; - if ndecoded.chars <= num_to_skip.chars { - let (dec_buffer, dec_flags) = decoder_getstate()?; - if dec_buffer.is_empty() { - cookie.dec_flags = dec_flags; - num_to_skip -= ndecoded; - break; - } - skip_bytes -= dec_buffer.len() as isize; - skip_back = 1; - } else { - skip_bytes -= skip_back; - skip_back *= 2; - } - } - if skip_bytes <= 0 { - skip_bytes = 0; - cookie.set_decoder_state(decoder, vm)?; - } - let skip_bytes = skip_bytes as usize; - - cookie.start_pos += skip_bytes as Offset; - cookie.set_num_to_skip(num_to_skip); - - if num_to_skip.chars != 0 { - let mut ndecoded = Utf8size::default(); - let mut input = next_input.as_bytes(); - input = &input[skip_bytes..]; - while !input.is_empty() { - let (byte1, rest) = input.split_at(1); - let n = decoder_decode(byte1)?; - ndecoded += n; - cookie.bytes_to_feed += 1; - let (dec_buffer, dec_flags) = decoder_getstate()?; - if dec_buffer.is_empty() && ndecoded.chars < num_to_skip.chars { - cookie.start_pos += cookie.bytes_to_feed as Offset; - num_to_skip -= ndecoded; - cookie.dec_flags = dec_flags; - cookie.bytes_to_feed = 0; - ndecoded = Utf8size::default(); - } - if ndecoded.chars >= num_to_skip.chars { - break; - } - input = rest; - } - if input.is_empty() { - let decoded = - vm.call_method(decoder, "decode", (vm.ctx.new_bytes(vec![]), true))?; - let decoded = check_decoded(decoded, vm)?; - let final_decoded_chars = ndecoded.chars + decoded.char_len(); - cookie.need_eof = true; - if final_decoded_chars < num_to_skip.chars { - return Err( - vm.new_os_error("can't reconstruct logical file position".to_owned()) - ); - } - } - } - vm.call_method(decoder, "setstate", (saved_state,))?; - cookie.set_num_to_skip(num_to_skip); - Ok(cookie.build().to_pyobject(vm)) - } - - #[pygetset] - fn name(&self, vm: &VirtualMachine) -> PyResult { - let buffer = self.lock(vm)?.buffer.clone(); - buffer.get_attr("name", vm) - } - #[pygetset] - fn encoding(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Ok(self.lock(vm)?.encoding.clone()) - } - #[pygetset] - fn errors(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Ok(self.lock(vm)?.errors.clone()) - } - - #[pymethod] - fn fileno(&self, vm: &VirtualMachine) -> PyResult { - let buffer = self.lock(vm)?.buffer.clone(); - vm.call_method(&buffer, "fileno", ()) - } - - #[pymethod] - fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let mut textio = self.lock(vm)?; - textio.check_closed(vm)?; - let decoder = textio - .decoder - .clone() - .ok_or_else(|| new_unsupported_operation(vm, "not readable".to_owned()))?; - - textio.write_pending(vm)?; - - let s = if let Some(mut remaining) = size.to_usize() { - let mut chunks = Vec::new(); - let mut chunks_bytes = 0; - loop { - if let Some((s, char_len)) = textio.get_decoded_chars(remaining, vm) { - chunks_bytes += s.byte_len(); - chunks.push(s); - remaining = remaining.saturating_sub(char_len); - } - if remaining == 0 { - break; - } - let eof = textio.read_chunk(remaining, vm)?; - if eof { - break; - } - } - if chunks.is_empty() { - vm.ctx.empty_str.to_owned() - } else if chunks.len() == 1 { - chunks.pop().unwrap() - } else { - let mut ret = String::with_capacity(chunks_bytes); - for chunk in chunks { - ret.push_str(chunk.as_str()) - } - PyStr::from(ret).into_ref(&vm.ctx) - } - } else { - let bytes = vm.call_method(&textio.buffer, "read", ())?; - let decoded = vm.call_method(&decoder, "decode", (bytes, true))?; - let decoded = check_decoded(decoded, vm)?; - let ret = textio.take_decoded_chars(Some(decoded), vm); - textio.snapshot = None; - ret - }; - Ok(s) - } - - #[pymethod] - fn write(&self, obj: PyStrRef, vm: &VirtualMachine) -> PyResult<usize> { - let mut textio = self.lock(vm)?; - textio.check_closed(vm)?; - - let (encoder, encodefunc) = textio - .encoder - .as_ref() - .ok_or_else(|| new_unsupported_operation(vm, "not writable".to_owned()))?; - - let char_len = obj.char_len(); - - let data = obj.as_str(); - - let replace_nl = match textio.newline { - Newlines::Cr => Some("\r"), - Newlines::Crlf => Some("\r\n"), - _ => None, - }; - let has_lf = if replace_nl.is_some() || textio.line_buffering { - data.contains('\n') - } else { - false - }; - let flush = textio.line_buffering && (has_lf || data.contains('\r')); - let chunk = if let Some(replace_nl) = replace_nl { - if has_lf { - PyStr::from(data.replace('\n', replace_nl)).into_ref(&vm.ctx) - } else { - obj - } - } else { - obj - }; - let chunk = if let Some(encodefunc) = *encodefunc { - encodefunc(chunk) - } else { - let b = vm.call_method(encoder, "encode", (chunk.clone(),))?; - b.downcast::<PyBytes>() - .map(PendingWrite::Bytes) - .or_else(|obj| { - // TODO: not sure if encode() returning the str it was passed is officially - // supported or just a quirk of how the CPython code is written - if obj.is(&chunk) { - Ok(PendingWrite::Utf8(chunk)) - } else { - Err(vm.new_type_error(format!( - "encoder should return a bytes object, not '{}'", - obj.class().name() - ))) - } - })? - }; - if textio.pending.num_bytes + chunk.as_bytes().len() > textio.chunk_size { - textio.write_pending(vm)?; - } - textio.pending.push(chunk); - if flush || textio.write_through || textio.pending.num_bytes >= textio.chunk_size { - textio.write_pending(vm)?; - } - if flush { - let _ = vm.call_method(&textio.buffer, "flush", ()); - } - - Ok(char_len) - } - - #[pymethod] - fn flush(&self, vm: &VirtualMachine) -> PyResult { - let mut textio = self.lock(vm)?; - textio.check_closed(vm)?; - textio.telling = textio.seekable; - textio.write_pending(vm)?; - vm.call_method(&textio.buffer, "flush", ()) - } - - #[pymethod] - fn isatty(&self, vm: &VirtualMachine) -> PyResult { - let textio = self.lock(vm)?; - textio.check_closed(vm)?; - vm.call_method(&textio.buffer, "isatty", ()) - } - - #[pymethod] - fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let limit = size.to_usize(); - - let mut textio = self.lock(vm)?; - check_closed(&textio.buffer, vm)?; - - textio.write_pending(vm)?; - - #[derive(Clone)] - struct SlicedStr(PyStrRef, Range<usize>); - impl SlicedStr { - #[inline] - fn byte_len(&self) -> usize { - self.1.len() - } - #[inline] - fn char_len(&self) -> usize { - if self.is_full_slice() { - self.0.char_len() - } else { - self.slice().chars().count() - } - } - #[inline] - fn is_full_slice(&self) -> bool { - self.1.len() >= self.0.byte_len() - } - #[inline] - fn slice(&self) -> &str { - &self.0.as_str()[self.1.clone()] - } - #[inline] - fn slice_pystr(self, vm: &VirtualMachine) -> PyStrRef { - if self.is_full_slice() { - self.0 - } else { - // TODO: try to use Arc::get_mut() on the str? - PyStr::from(self.slice()).into_ref(&vm.ctx) - } - } - fn utf8_len(&self) -> Utf8size { - Utf8size { - bytes: self.byte_len(), - chars: self.char_len(), - } - } - } - - let mut start; - let mut endpos; - let mut offset_to_buffer; - let mut chunked = Utf8size::default(); - let mut remaining: Option<SlicedStr> = None; - let mut chunks = Vec::new(); - - let cur_line = 'outer: loop { - let decoded_chars = loop { - match textio.decoded_chars.as_ref() { - Some(s) if !s.is_empty() => break s, - _ => {} - } - let eof = textio.read_chunk(0, vm)?; - if eof { - textio.set_decoded_chars(None); - textio.snapshot = None; - start = Utf8size::default(); - endpos = Utf8size::default(); - offset_to_buffer = Utf8size::default(); - break 'outer None; - } - }; - let line = match remaining.take() { - None => { - start = textio.decoded_chars_used; - offset_to_buffer = Utf8size::default(); - decoded_chars.clone() - } - Some(remaining) => { - assert_eq!(textio.decoded_chars_used.bytes, 0); - offset_to_buffer = remaining.utf8_len(); - let decoded_chars = decoded_chars.as_str(); - let line = if remaining.is_full_slice() { - let mut line = remaining.0; - line.concat_in_place(decoded_chars, vm); - line - } else { - let remaining = remaining.slice(); - let mut s = - String::with_capacity(remaining.len() + decoded_chars.len()); - s.push_str(remaining); - s.push_str(decoded_chars); - PyStr::from(s).into_ref(&vm.ctx) - }; - start = Utf8size::default(); - line - } - }; - let line_from_start = &line.as_str()[start.bytes..]; - let nl_res = textio.newline.find_newline(line_from_start); - match nl_res { - Ok(p) | Err(p) => { - endpos = start + Utf8size::len_str(&line_from_start[..p]); - if let Some(limit) = limit { - // original CPython logic: endpos = start + limit - chunked - if chunked.chars + endpos.chars >= limit { - endpos = start - + Utf8size { - chars: limit - chunked.chars, - bytes: crate::common::str::char_range_end( - line_from_start, - limit - chunked.chars, - ) - .unwrap(), - }; - break Some(line); - } - } - } - } - if nl_res.is_ok() { - break Some(line); - } - if endpos.bytes > start.bytes { - let chunk = SlicedStr(line.clone(), start.bytes..endpos.bytes); - chunked += chunk.utf8_len(); - chunks.push(chunk); - } - let line_len = line.byte_len(); - if endpos.bytes < line_len { - remaining = Some(SlicedStr(line, endpos.bytes..line_len)); - } - textio.set_decoded_chars(None); - }; - - let cur_line = cur_line.map(|line| { - textio.decoded_chars_used = endpos - offset_to_buffer; - SlicedStr(line, start.bytes..endpos.bytes) - }); - // don't need to care about chunked.chars anymore - let mut chunked = chunked.bytes; - if let Some(remaining) = remaining { - chunked += remaining.byte_len(); - chunks.push(remaining); - } - let line = if !chunks.is_empty() { - if let Some(cur_line) = cur_line { - chunked += cur_line.byte_len(); - chunks.push(cur_line); - } - let mut s = String::with_capacity(chunked); - for chunk in chunks { - s.push_str(chunk.slice()) - } - PyStr::from(s).into_ref(&vm.ctx) - } else if let Some(cur_line) = cur_line { - cur_line.slice_pystr(vm) - } else { - vm.ctx.empty_str.to_owned() - }; - Ok(line) - } - - #[pymethod] - fn close(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { - let buffer = zelf.lock(vm)?.buffer.clone(); - if file_closed(&buffer, vm)? { - return Ok(()); - } - let flush_res = vm.call_method(zelf.as_object(), "flush", ()).map(drop); - let close_res = vm.call_method(&buffer, "close", ()).map(drop); - exception_chain(flush_res, close_res) - } - #[pygetset] - fn closed(&self, vm: &VirtualMachine) -> PyResult { - let buffer = self.lock(vm)?.buffer.clone(); - buffer.get_attr("closed", vm) - } - #[pygetset] - fn buffer(&self, vm: &VirtualMachine) -> PyResult { - Ok(self.lock(vm)?.buffer.clone()) - } - - #[pymethod(magic)] - fn reduce(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) - } - } - - fn parse_decoder_state(state: PyObjectRef, vm: &VirtualMachine) -> PyResult<(PyBytesRef, i32)> { - use crate::builtins::{int, PyTuple}; - let state_err = || vm.new_type_error("illegal decoder state".to_owned()); - let state = state.downcast::<PyTuple>().map_err(|_| state_err())?; - match state.as_slice() { - [buf, flags] => { - let buf = buf.clone().downcast::<PyBytes>().map_err(|obj| { - vm.new_type_error(format!( - "illegal decoder state: the first item should be a bytes object, not '{}'", - obj.class().name() - )) - })?; - let flags = flags.payload::<int::PyInt>().ok_or_else(state_err)?; - let flags = flags.try_to_primitive(vm)?; - Ok((buf, flags)) - } - _ => Err(state_err()), - } - } - - impl TextIOData { - fn write_pending(&mut self, vm: &VirtualMachine) -> PyResult<()> { - if self.pending.num_bytes == 0 { - return Ok(()); - } - let data = self.pending.take(vm); - vm.call_method(&self.buffer, "write", (data,))?; - Ok(()) - } - /// returns true on EOF - fn read_chunk(&mut self, size_hint: usize, vm: &VirtualMachine) -> PyResult<bool> { - let decoder = self - .decoder - .as_ref() - .ok_or_else(|| new_unsupported_operation(vm, "not readable".to_owned()))?; - - let dec_state = if self.telling { - let state = vm.call_method(decoder, "getstate", ())?; - Some(parse_decoder_state(state, vm)?) - } else { - None - }; - - let method = if self.has_read1 { "read1" } else { "read" }; - let size_hint = if size_hint > 0 { - (self.b2cratio.max(1.0) * size_hint as f64) as usize - } else { - size_hint - }; - let chunk_size = std::cmp::max(self.chunk_size, size_hint); - let input_chunk = vm.call_method(&self.buffer, method, (chunk_size,))?; - - let buf = ArgBytesLike::try_from_borrowed_object(vm, &input_chunk).map_err(|_| { - vm.new_type_error(format!( - "underlying {}() should have returned a bytes-like object, not '{}'", - method, - input_chunk.class().name() - )) - })?; - let nbytes = buf.borrow_buf().len(); - let eof = nbytes == 0; - let decoded = vm.call_method(decoder, "decode", (input_chunk, eof))?; - let decoded = check_decoded(decoded, vm)?; - - let char_len = decoded.char_len(); - self.b2cratio = if char_len > 0 { - nbytes as f64 / char_len as f64 - } else { - 0.0 - }; - let eof = if char_len > 0 { false } else { eof }; - self.set_decoded_chars(Some(decoded)); - - if let Some((dec_buffer, dec_flags)) = dec_state { - // TODO: inplace append to bytes when refcount == 1 - let mut next_input = dec_buffer.as_bytes().to_vec(); - next_input.extend_from_slice(&buf.borrow_buf()); - self.snapshot = Some((dec_flags, PyBytes::from(next_input).into_ref(&vm.ctx))); - } - - Ok(eof) - } - - fn check_closed(&self, vm: &VirtualMachine) -> PyResult<()> { - check_closed(&self.buffer, vm) - } - - /// returns str, str.char_len() (it might not be cached in the str yet but we calculate it - /// anyway in this method) - fn get_decoded_chars( - &mut self, - n: usize, - vm: &VirtualMachine, - ) -> Option<(PyStrRef, usize)> { - if n == 0 { - return None; - } - let decoded_chars = self.decoded_chars.as_ref()?; - let avail = &decoded_chars.as_str()[self.decoded_chars_used.bytes..]; - if avail.is_empty() { - return None; - } - let avail_chars = decoded_chars.char_len() - self.decoded_chars_used.chars; - let (chars, chars_used) = if n >= avail_chars { - if self.decoded_chars_used.bytes == 0 { - (decoded_chars.clone(), avail_chars) - } else { - (PyStr::from(avail).into_ref(&vm.ctx), avail_chars) - } - } else { - let s = crate::common::str::get_chars(avail, 0..n); - (PyStr::from(s).into_ref(&vm.ctx), n) - }; - self.decoded_chars_used += Utf8size { - bytes: chars.byte_len(), - chars: chars_used, - }; - Some((chars, chars_used)) - } - fn set_decoded_chars(&mut self, s: Option<PyStrRef>) { - self.decoded_chars = s; - self.decoded_chars_used = Utf8size::default(); - } - fn take_decoded_chars( - &mut self, - append: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyStrRef { - let empty_str = || vm.ctx.empty_str.to_owned(); - let chars_pos = std::mem::take(&mut self.decoded_chars_used).bytes; - let decoded_chars = match std::mem::take(&mut self.decoded_chars) { - None => return append.unwrap_or_else(empty_str), - Some(s) if s.is_empty() => return append.unwrap_or_else(empty_str), - Some(s) => s, - }; - let append_len = append.as_ref().map_or(0, |s| s.byte_len()); - if append_len == 0 && chars_pos == 0 { - return decoded_chars; - } - // TODO: in-place editing of `str` when refcount == 1 - let decoded_chars_unused = &decoded_chars.as_str()[chars_pos..]; - let mut s = String::with_capacity(decoded_chars_unused.len() + append_len); - s.push_str(decoded_chars_unused); - if let Some(append) = append { - s.push_str(append.as_str()) - } - PyStr::from(s).into_ref(&vm.ctx) - } - } - - #[pyattr] - #[pyclass(name = "StringIO", base = "_TextIOBase")] - #[derive(Debug, PyPayload)] - struct StringIO { - buffer: PyRwLock<BufferedIO>, - closed: AtomicCell<bool>, - } - - #[derive(FromArgs)] - struct StringIONewArgs { - #[pyarg(positional, optional)] - object: OptionalOption<PyStrRef>, - - // TODO: use this - #[pyarg(any, default)] - #[allow(dead_code)] - newline: Newlines, - } - - impl Constructor for StringIO { - type Args = StringIONewArgs; - - #[allow(unused_variables)] - fn py_new( - cls: PyTypeRef, - Self::Args { object, newline }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let raw_bytes = object - .flatten() - .map_or_else(Vec::new, |v| v.as_str().as_bytes().to_vec()); - - StringIO { - buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), - closed: AtomicCell::new(false), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl StringIO { - fn buffer(&self, vm: &VirtualMachine) -> PyResult<PyRwLockWriteGuard<'_, BufferedIO>> { - if !self.closed.load() { - Ok(self.buffer.write()) - } else { - Err(io_closed_error(vm)) - } - } - } - - #[pyclass(flags(BASETYPE, HAS_DICT), with(Constructor))] - impl StringIO { - #[pymethod] - fn readable(&self) -> bool { - true - } - #[pymethod] - fn writable(&self) -> bool { - true - } - #[pymethod] - fn seekable(&self) -> bool { - true - } - - #[pygetset] - fn closed(&self) -> bool { - self.closed.load() - } - - #[pymethod] - fn close(&self) { - self.closed.store(true); - } - - // write string to underlying vector - #[pymethod] - fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult<u64> { - let bytes = data.as_str().as_bytes(); - self.buffer(vm)? - .write(bytes) - .ok_or_else(|| vm.new_type_error("Error Writing String".to_owned())) - } - - // return the entire contents of the underlying - #[pymethod] - fn getvalue(&self, vm: &VirtualMachine) -> PyResult<String> { - let bytes = self.buffer(vm)?.getvalue(); - String::from_utf8(bytes) - .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned())) - } - - // skip to the jth position - #[pymethod] - fn seek( - &self, - offset: PyObjectRef, - how: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<u64> { - self.buffer(vm)? - .seek(seekfrom(vm, offset, how)?) - .map_err(|err| os_err(vm, err)) - } - - // Read k bytes from the object and return. - // If k is undefined || k == -1, then we read all bytes until the end of the file. - // This also increments the stream position by the value of k - #[pymethod] - fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<String> { - let data = self.buffer(vm)?.read(size.to_usize()).unwrap_or_default(); - - let value = String::from_utf8(data) - .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned()))?; - Ok(value) - } - - #[pymethod] - fn tell(&self, vm: &VirtualMachine) -> PyResult<u64> { - Ok(self.buffer(vm)?.tell()) - } - - #[pymethod] - fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<String> { - // TODO size should correspond to the number of characters, at the moments its the number of - // bytes. - let input = self.buffer(vm)?.readline(size.to_usize(), vm)?; - String::from_utf8(input) - .map_err(|_| vm.new_value_error("Error Retrieving Value".to_owned())) - } - - #[pymethod] - fn truncate(&self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult<usize> { - let mut buffer = self.buffer(vm)?; - let pos = pos.try_usize(vm)?; - Ok(buffer.truncate(pos)) - } - } - - #[pyattr] - #[pyclass(name = "BytesIO", base = "_BufferedIOBase")] - #[derive(Debug, PyPayload)] - struct BytesIO { - buffer: PyRwLock<BufferedIO>, - closed: AtomicCell<bool>, - exports: AtomicCell<usize>, - } - - impl Constructor for BytesIO { - type Args = OptionalArg<Option<PyBytesRef>>; - - fn py_new(cls: PyTypeRef, object: Self::Args, vm: &VirtualMachine) -> PyResult { - let raw_bytes = object - .flatten() - .map_or_else(Vec::new, |input| input.as_bytes().to_vec()); - - BytesIO { - buffer: PyRwLock::new(BufferedIO::new(Cursor::new(raw_bytes))), - closed: AtomicCell::new(false), - exports: AtomicCell::new(0), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl BytesIO { - fn buffer(&self, vm: &VirtualMachine) -> PyResult<PyRwLockWriteGuard<'_, BufferedIO>> { - if !self.closed.load() { - Ok(self.buffer.write()) - } else { - Err(io_closed_error(vm)) - } - } - } - - #[pyclass(flags(BASETYPE, HAS_DICT), with(PyRef, Constructor))] - impl BytesIO { - #[pymethod] - fn readable(&self) -> bool { - true - } - #[pymethod] - fn writable(&self) -> bool { - true - } - #[pymethod] - fn seekable(&self) -> bool { - true - } - - #[pymethod] - fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult<u64> { - let mut buffer = self.try_resizable(vm)?; - data.with_ref(|b| buffer.write(b)) - .ok_or_else(|| vm.new_type_error("Error Writing Bytes".to_owned())) - } - - // Retrieves the entire bytes object value from the underlying buffer - #[pymethod] - fn getvalue(&self, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - let bytes = self.buffer(vm)?.getvalue(); - Ok(vm.ctx.new_bytes(bytes)) - } - - // Takes an integer k (bytes) and returns them from the underlying buffer - // If k is undefined || k == -1, then we read all bytes until the end of the file. - // This also increments the stream position by the value of k - #[pymethod] - #[pymethod(name = "read1")] - fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let buf = self.buffer(vm)?.read(size.to_usize()).unwrap_or_default(); - Ok(buf) - } - - #[pymethod] - fn readinto(&self, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<usize> { - let mut buf = self.buffer(vm)?; - let ret = buf - .cursor - .read(&mut obj.borrow_buf_mut()) - .map_err(|_| vm.new_value_error("Error readinto from Take".to_owned()))?; - - Ok(ret) - } - - //skip to the jth position - #[pymethod] - fn seek( - &self, - offset: PyObjectRef, - how: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<u64> { - self.buffer(vm)? - .seek(seekfrom(vm, offset, how)?) - .map_err(|err| os_err(vm, err)) - } - - #[pymethod] - fn tell(&self, vm: &VirtualMachine) -> PyResult<u64> { - Ok(self.buffer(vm)?.tell()) - } - - #[pymethod] - fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - self.buffer(vm)?.readline(size.to_usize(), vm) - } - - #[pymethod] - fn truncate(&self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult<usize> { - if self.closed.load() { - return Err(io_closed_error(vm)); - } - let mut buffer = self.try_resizable(vm)?; - let pos = pos.try_usize(vm)?; - Ok(buffer.truncate(pos)) - } - - #[pygetset] - fn closed(&self) -> bool { - self.closed.load() - } - - #[pymethod] - fn close(&self, vm: &VirtualMachine) -> PyResult<()> { - drop(self.try_resizable(vm)?); - self.closed.store(true); - Ok(()) - } - } - - #[pyclass] - impl PyRef<BytesIO> { - #[pymethod] - fn getbuffer(self, vm: &VirtualMachine) -> PyResult<PyMemoryView> { - let len = self.buffer.read().cursor.get_ref().len(); - let buffer = PyBuffer::new( - self.into(), - BufferDescriptor::simple(len, false), - &BYTES_IO_BUFFER_METHODS, - ); - let view = PyMemoryView::from_buffer(buffer, vm)?; - Ok(view) - } - } - - static BYTES_IO_BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| { - let zelf = buffer.obj_as::<BytesIO>(); - PyRwLockReadGuard::map(zelf.buffer.read(), |x| x.cursor.get_ref().as_slice()).into() - }, - obj_bytes_mut: |buffer| { - let zelf = buffer.obj_as::<BytesIO>(); - PyRwLockWriteGuard::map(zelf.buffer.write(), |x| x.cursor.get_mut().as_mut_slice()) - .into() - }, - - release: |buffer| { - buffer.obj_as::<BytesIO>().exports.fetch_sub(1); - }, - - retain: |buffer| { - buffer.obj_as::<BytesIO>().exports.fetch_add(1); - }, - }; - - impl BufferResizeGuard for BytesIO { - type Resizable<'a> = PyRwLockWriteGuard<'a, BufferedIO>; - - fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>> { - let w = self.buffer.write(); - (self.exports.load() == 0).then_some(w) - } - } - - #[repr(u8)] - enum FileMode { - Read = b'r', - Write = b'w', - Exclusive = b'x', - Append = b'a', - } - #[repr(u8)] - enum EncodeMode { - Text = b't', - Bytes = b'b', - } - struct Mode { - file: FileMode, - encode: EncodeMode, - plus: bool, - } - impl std::str::FromStr for Mode { - type Err = ParseModeError; - fn from_str(s: &str) -> Result<Self, Self::Err> { - let mut file = None; - let mut encode = None; - let mut plus = false; - macro_rules! set_mode { - ($var:ident, $mode:path, $err:ident) => {{ - match $var { - Some($mode) => return Err(ParseModeError::InvalidMode), - Some(_) => return Err(ParseModeError::$err), - None => $var = Some($mode), - } - }}; - } - - for ch in s.chars() { - match ch { - '+' => { - if plus { - return Err(ParseModeError::InvalidMode); - } - plus = true - } - 't' => set_mode!(encode, EncodeMode::Text, MultipleEncode), - 'b' => set_mode!(encode, EncodeMode::Bytes, MultipleEncode), - 'r' => set_mode!(file, FileMode::Read, MultipleFile), - 'a' => set_mode!(file, FileMode::Append, MultipleFile), - 'w' => set_mode!(file, FileMode::Write, MultipleFile), - 'x' => set_mode!(file, FileMode::Exclusive, MultipleFile), - _ => return Err(ParseModeError::InvalidMode), - } - } - - let file = file.ok_or(ParseModeError::NoFile)?; - let encode = encode.unwrap_or(EncodeMode::Text); - - Ok(Mode { file, encode, plus }) - } - } - impl Mode { - fn rawmode(&self) -> &'static str { - match (&self.file, self.plus) { - (FileMode::Read, true) => "rb+", - (FileMode::Read, false) => "rb", - (FileMode::Write, true) => "wb+", - (FileMode::Write, false) => "wb", - (FileMode::Exclusive, true) => "xb+", - (FileMode::Exclusive, false) => "xb", - (FileMode::Append, true) => "ab+", - (FileMode::Append, false) => "ab", - } - } - } - enum ParseModeError { - InvalidMode, - MultipleFile, - MultipleEncode, - NoFile, - } - impl ParseModeError { - fn error_msg(&self, mode_string: &str) -> String { - match self { - ParseModeError::InvalidMode => format!("invalid mode: '{mode_string}'"), - ParseModeError::MultipleFile => { - "must have exactly one of create/read/write/append mode".to_owned() - } - ParseModeError::MultipleEncode => { - "can't have text and binary mode at once".to_owned() - } - ParseModeError::NoFile => { - "Must have exactly one of create/read/write/append mode and at most one plus" - .to_owned() - } - } - } - } - - #[derive(FromArgs)] - struct IoOpenArgs { - file: PyObjectRef, - #[pyarg(any, optional)] - mode: OptionalArg<PyStrRef>, - #[pyarg(flatten)] - opts: OpenArgs, - } - #[pyfunction] - fn open(args: IoOpenArgs, vm: &VirtualMachine) -> PyResult { - io_open( - args.file, - args.mode.as_ref().into_option().map(|s| s.as_str()), - args.opts, - vm, - ) - } - - #[pyfunction] - fn open_code(file: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // TODO: lifecycle hooks or something? - io_open(file, Some("rb"), OpenArgs::default(), vm) - } - - #[derive(FromArgs)] - pub struct OpenArgs { - #[pyarg(any, default = "-1")] - buffering: isize, - #[pyarg(any, default)] - encoding: Option<PyStrRef>, - #[pyarg(any, default)] - errors: Option<PyStrRef>, - #[pyarg(any, default)] - newline: Option<PyStrRef>, - #[pyarg(any, default = "true")] - closefd: bool, - #[pyarg(any, default)] - opener: Option<PyObjectRef>, - } - impl Default for OpenArgs { - fn default() -> Self { - OpenArgs { - buffering: -1, - encoding: None, - errors: None, - newline: None, - closefd: true, - opener: None, - } - } - } - - pub fn io_open( - file: PyObjectRef, - mode: Option<&str>, - opts: OpenArgs, - vm: &VirtualMachine, - ) -> PyResult { - // mode is optional: 'rt' is the default mode (open from reading text) - let mode_string = mode.unwrap_or("r"); - let mode = mode_string - .parse::<Mode>() - .map_err(|e| vm.new_value_error(e.error_msg(mode_string)))?; - - if let EncodeMode::Bytes = mode.encode { - let msg = if opts.encoding.is_some() { - Some("binary mode doesn't take an encoding argument") - } else if opts.errors.is_some() { - Some("binary mode doesn't take an errors argument") - } else if opts.newline.is_some() { - Some("binary mode doesn't take a newline argument") - } else { - None - }; - if let Some(msg) = msg { - return Err(vm.new_value_error(msg.to_owned())); - } - } - - // check file descriptor validity - #[cfg(unix)] - if let Ok(crate::stdlib::os::OsPathOrFd::Fd(fd)) = file.clone().try_into_value(vm) { - nix::fcntl::fcntl(fd, nix::fcntl::F_GETFD) - .map_err(|_| crate::stdlib::os::errno_err(vm))?; - } - - // Construct a FileIO (subclass of RawIOBase) - // This is subsequently consumed by a Buffered Class. - let file_io_class: &Py<PyType> = { - cfg_if::cfg_if! { - if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { - Some(super::fileio::FileIO::static_type()) - } else { - None - } - } - } - .ok_or_else(|| { - new_unsupported_operation( - vm, - "Couldn't get FileIO, io.open likely isn't supported on your platform".to_owned(), - ) - })?; - let raw = PyType::call( - file_io_class, - (file, mode.rawmode(), opts.closefd, opts.opener).into_args(vm), - vm, - )?; - - let isatty = opts.buffering < 0 && { - let atty = vm.call_method(&raw, "isatty", ())?; - bool::try_from_object(vm, atty)? - }; - - let line_buffering = opts.buffering == 1 || isatty; - - let buffering = if opts.buffering < 0 || opts.buffering == 1 { - DEFAULT_BUFFER_SIZE - } else { - opts.buffering as usize - }; - - if buffering == 0 { - let ret = match mode.encode { - EncodeMode::Text => { - Err(vm.new_value_error("can't have unbuffered text I/O".to_owned())) - } - EncodeMode::Bytes => Ok(raw), - }; - return ret; - } - - let cls = if mode.plus { - BufferedRandom::static_type() - } else if let FileMode::Read = mode.file { - BufferedReader::static_type() - } else { - BufferedWriter::static_type() - }; - let buffered = PyType::call(cls, (raw, buffering).into_args(vm), vm)?; - - match mode.encode { - EncodeMode::Text => { - let tio = TextIOWrapper::static_type(); - let wrapper = PyType::call( - tio, - ( - buffered, - opts.encoding, - opts.errors, - opts.newline, - line_buffering, - ) - .into_args(vm), - vm, - )?; - wrapper.set_attr("mode", vm.new_pyobj(mode_string), vm)?; - Ok(wrapper) - } - EncodeMode::Bytes => Ok(buffered), - } - } - - rustpython_common::static_cell! { - pub(super) static UNSUPPORTED_OPERATION: PyTypeRef; - } - - pub(super) fn make_unsupportedop(ctx: &Context) -> PyTypeRef { - use crate::types::PyTypeSlots; - PyType::new_heap( - "UnsupportedOperation", - vec![ - ctx.exceptions.os_error.to_owned(), - ctx.exceptions.value_error.to_owned(), - ], - Default::default(), - PyTypeSlots::heap_default(), - ctx.types.type_type.to_owned(), - ctx, - ) - .unwrap() - } - - #[pyfunction] - fn text_encoding( - encoding: PyObjectRef, - _stacklevel: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<PyStrRef> { - if vm.is_none(&encoding) { - // TODO: This is `locale` encoding - but we don't have locale encoding yet - return Ok(vm.ctx.new_str("utf-8")); - } - encoding.try_into_value(vm) - } - - #[cfg(test)] - mod tests { - use super::*; - - #[test] - fn test_buffered_read() { - let data = vec![1, 2, 3, 4]; - let bytes = None; - let mut buffered = BufferedIO { - cursor: Cursor::new(data.clone()), - }; - - assert_eq!(buffered.read(bytes).unwrap(), data); - } - - #[test] - fn test_buffered_seek() { - let data = vec![1, 2, 3, 4]; - let count: u64 = 2; - let mut buffered = BufferedIO { - cursor: Cursor::new(data), - }; - - assert_eq!(buffered.seek(SeekFrom::Start(count)).unwrap(), count); - assert_eq!(buffered.read(Some(count as usize)).unwrap(), vec![3, 4]); - } - - #[test] - fn test_buffered_value() { - let data = vec![1, 2, 3, 4]; - let buffered = BufferedIO { - cursor: Cursor::new(data.clone()), - }; - - assert_eq!(buffered.getvalue(), data); - } - } -} - -// disable FileIO on WASM -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] -#[pymodule] -mod fileio { - use super::{Offset, _io::*}; - use crate::{ - builtins::{PyStr, PyStrRef}, - common::crt_fd::Fd, - convert::ToPyException, - function::{ArgBytesLike, ArgMemoryBuffer, OptionalArg, OptionalOption}, - stdlib::os, - types::{DefaultConstructor, Initializer, Representable}, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - }; - use crossbeam_utils::atomic::AtomicCell; - use std::io::{Read, Write}; - - bitflags::bitflags! { - #[derive(Copy, Clone, Debug, PartialEq)] - struct Mode: u8 { - const CREATED = 0b0001; - const READABLE = 0b0010; - const WRITABLE = 0b0100; - const APPENDING = 0b1000; - } - } - - enum ModeError { - Invalid, - BadRwa, - } - impl ModeError { - fn error_msg(&self, mode_str: &str) -> String { - match self { - ModeError::Invalid => format!("invalid mode: {mode_str}"), - ModeError::BadRwa => { - "Must have exactly one of create/read/write/append mode and at most one plus" - .to_owned() - } - } - } - } - - fn compute_mode(mode_str: &str) -> Result<(Mode, i32), ModeError> { - let mut flags = 0; - let mut plus = false; - let mut rwa = false; - let mut mode = Mode::empty(); - for c in mode_str.bytes() { - match c { - b'x' => { - if rwa { - return Err(ModeError::BadRwa); - } - rwa = true; - mode.insert(Mode::WRITABLE | Mode::CREATED); - flags |= libc::O_EXCL | libc::O_CREAT; - } - b'r' => { - if rwa { - return Err(ModeError::BadRwa); - } - rwa = true; - mode.insert(Mode::READABLE); - } - b'w' => { - if rwa { - return Err(ModeError::BadRwa); - } - rwa = true; - mode.insert(Mode::WRITABLE); - flags |= libc::O_CREAT | libc::O_TRUNC; - } - b'a' => { - if rwa { - return Err(ModeError::BadRwa); - } - rwa = true; - mode.insert(Mode::WRITABLE | Mode::APPENDING); - flags |= libc::O_APPEND | libc::O_CREAT; - } - b'+' => { - if plus { - return Err(ModeError::BadRwa); - } - plus = true; - mode.insert(Mode::READABLE | Mode::WRITABLE); - } - b'b' => {} - _ => return Err(ModeError::Invalid), - } - } - - if !rwa { - return Err(ModeError::BadRwa); - } - - if mode.contains(Mode::READABLE | Mode::WRITABLE) { - flags |= libc::O_RDWR - } else if mode.contains(Mode::READABLE) { - flags |= libc::O_RDONLY - } else { - flags |= libc::O_WRONLY - } - - #[cfg(windows)] - { - flags |= libc::O_BINARY | libc::O_NOINHERIT; - } - #[cfg(unix)] - { - flags |= libc::O_CLOEXEC - } - - Ok((mode, flags as _)) - } - - #[pyattr] - #[pyclass(module = "io", name, base = "_RawIOBase")] - #[derive(Debug, PyPayload)] - pub(super) struct FileIO { - fd: AtomicCell<i32>, - closefd: AtomicCell<bool>, - mode: AtomicCell<Mode>, - seekable: AtomicCell<Option<bool>>, - } - - #[derive(FromArgs)] - pub struct FileIOArgs { - #[pyarg(positional)] - name: PyObjectRef, - #[pyarg(any, default)] - mode: Option<PyStrRef>, - #[pyarg(any, default = "true")] - closefd: bool, - #[pyarg(any, default)] - opener: Option<PyObjectRef>, - } - - impl Default for FileIO { - fn default() -> Self { - Self { - fd: AtomicCell::new(-1), - closefd: AtomicCell::new(false), - mode: AtomicCell::new(Mode::empty()), - seekable: AtomicCell::new(None), - } - } - } - - impl DefaultConstructor for FileIO {} - - impl Initializer for FileIO { - type Args = FileIOArgs; - - fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - let mode_obj = args - .mode - .unwrap_or_else(|| PyStr::from("rb").into_ref(&vm.ctx)); - let mode_str = mode_obj.as_str(); - let name = args.name; - let (mode, flags) = - compute_mode(mode_str).map_err(|e| vm.new_value_error(e.error_msg(mode_str)))?; - zelf.mode.store(mode); - let fd = if let Some(opener) = args.opener { - let fd = opener.call((name.clone(), flags), vm)?; - if !fd.fast_isinstance(vm.ctx.types.int_type) { - return Err(vm.new_type_error("expected integer from opener".to_owned())); - } - let fd = i32::try_from_object(vm, fd)?; - if fd < 0 { - return Err(vm.new_value_error(format!("opener returned {fd}"))); - } - fd - } else if let Some(i) = name.payload::<crate::builtins::PyInt>() { - i.try_to_primitive(vm)? - } else { - let path = os::OsPath::try_from_object(vm, name.clone())?; - if !args.closefd { - return Err( - vm.new_value_error("Cannot use closefd=False with file name".to_owned()) - ); - } - os::open(path, flags as _, None, Default::default(), vm)? - }; - - if mode.contains(Mode::APPENDING) { - let _ = os::lseek(fd as _, 0, libc::SEEK_END, vm); - } - - zelf.fd.store(fd); - zelf.closefd.store(args.closefd); - #[cfg(windows)] - crate::stdlib::msvcrt::setmode_binary(fd); - zelf.as_object().set_attr("name", name, vm)?; - Ok(()) - } - } - - impl Representable for FileIO { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let fd = zelf.fd.load(); - if fd < 0 { - return Ok("<_io.FileIO [closed]>".to_owned()); - } - let name_repr = repr_fileobj_name(zelf.as_object(), vm)?; - let mode = zelf.mode(); - let closefd = if zelf.closefd.load() { "True" } else { "False" }; - let repr = if let Some(name_repr) = name_repr { - format!("<_io.FileIO name={name_repr} mode='{mode}' closefd={closefd}>") - } else { - format!("<_io.FileIO fd={fd} mode='{mode}' closefd={closefd}>") - }; - Ok(repr) - } - } - - #[pyclass( - with(DefaultConstructor, Initializer, Representable), - flags(BASETYPE, HAS_DICT) - )] - impl FileIO { - #[pygetset] - fn closed(&self) -> bool { - self.fd.load() < 0 - } - - #[pygetset] - fn closefd(&self) -> bool { - self.closefd.load() - } - - #[pymethod] - fn fileno(&self, vm: &VirtualMachine) -> PyResult<i32> { - let fd = self.fd.load(); - if fd >= 0 { - Ok(fd) - } else { - Err(io_closed_error(vm)) - } - } - - fn get_fd(&self, vm: &VirtualMachine) -> PyResult<Fd> { - self.fileno(vm).map(Fd) - } - - #[pymethod] - fn readable(&self) -> bool { - self.mode.load().contains(Mode::READABLE) - } - #[pymethod] - fn writable(&self) -> bool { - self.mode.load().contains(Mode::WRITABLE) - } - #[pygetset] - fn mode(&self) -> &'static str { - let mode = self.mode.load(); - if mode.contains(Mode::CREATED) { - if mode.contains(Mode::READABLE) { - "xb+" - } else { - "xb" - } - } else if mode.contains(Mode::APPENDING) { - if mode.contains(Mode::READABLE) { - "ab+" - } else { - "ab" - } - } else if mode.contains(Mode::READABLE) { - if mode.contains(Mode::WRITABLE) { - "rb+" - } else { - "rb" - } - } else { - "wb" - } - } - - #[pymethod] - fn read(&self, read_byte: OptionalSize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - if !self.mode.load().contains(Mode::READABLE) { - return Err(new_unsupported_operation( - vm, - "File or stream is not readable".to_owned(), - )); - } - let mut handle = self.get_fd(vm)?; - let bytes = if let Some(read_byte) = read_byte.to_usize() { - let mut bytes = vec![0; read_byte]; - let n = handle - .read(&mut bytes) - .map_err(|err| err.to_pyexception(vm))?; - bytes.truncate(n); - bytes - } else { - let mut bytes = vec![]; - handle - .read_to_end(&mut bytes) - .map_err(|err| err.to_pyexception(vm))?; - bytes - }; - - Ok(bytes) - } - - #[pymethod] - fn readinto(&self, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult<usize> { - if !self.mode.load().contains(Mode::READABLE) { - return Err(new_unsupported_operation( - vm, - "File or stream is not readable".to_owned(), - )); - } - - let handle = self.get_fd(vm)?; - - let mut buf = obj.borrow_buf_mut(); - let mut f = handle.take(buf.len() as _); - let ret = f.read(&mut buf).map_err(|e| e.to_pyexception(vm))?; - - Ok(ret) - } - - #[pymethod] - fn write(&self, obj: ArgBytesLike, vm: &VirtualMachine) -> PyResult<usize> { - if !self.mode.load().contains(Mode::WRITABLE) { - return Err(new_unsupported_operation( - vm, - "File or stream is not writable".to_owned(), - )); - } - - let mut handle = self.get_fd(vm)?; - - let len = obj - .with_ref(|b| handle.write(b)) - .map_err(|err| err.to_pyexception(vm))?; - - //return number of bytes written - Ok(len) - } - - #[pymethod] - fn close(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<()> { - let res = iobase_close(zelf.as_object(), vm); - if !zelf.closefd.load() { - zelf.fd.store(-1); - return res; - } - let fd = zelf.fd.swap(-1); - if fd >= 0 { - Fd(fd).close().map_err(|e| e.to_pyexception(vm))?; - } - res - } - - #[pymethod] - fn seekable(&self, vm: &VirtualMachine) -> PyResult<bool> { - let fd = self.fileno(vm)?; - Ok(self.seekable.load().unwrap_or_else(|| { - let seekable = os::lseek(fd, 0, libc::SEEK_CUR, vm).is_ok(); - self.seekable.store(Some(seekable)); - seekable - })) - } - - #[pymethod] - fn seek( - &self, - offset: PyObjectRef, - how: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<Offset> { - let how = how.unwrap_or(0); - let fd = self.fileno(vm)?; - let offset = get_offset(offset, vm)?; - - os::lseek(fd, offset, how, vm) - } - - #[pymethod] - fn tell(&self, vm: &VirtualMachine) -> PyResult<Offset> { - let fd = self.fileno(vm)?; - os::lseek(fd, 0, libc::SEEK_CUR, vm) - } - - #[pymethod] - fn truncate(&self, len: OptionalOption, vm: &VirtualMachine) -> PyResult<Offset> { - let fd = self.fileno(vm)?; - let len = match len.flatten() { - Some(l) => get_offset(l, vm)?, - None => os::lseek(fd, 0, libc::SEEK_CUR, vm)?, - }; - os::ftruncate(fd, len, vm)?; - Ok(len) - } - - #[pymethod] - fn isatty(&self, vm: &VirtualMachine) -> PyResult<bool> { - let fd = self.fileno(vm)?; - Ok(os::isatty(fd)) - } - - #[pymethod(magic)] - fn reduce(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot pickle '{}' object", zelf.class().name()))) - } - } -} diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs deleted file mode 100644 index 3bc26c0a8f0..00000000000 --- a/vm/src/stdlib/itertools.rs +++ /dev/null @@ -1,1955 +0,0 @@ -pub(crate) use decl::make_module; - -#[pymodule(name = "itertools")] -mod decl { - use crate::stdlib::itertools::decl::int::get_value; - use crate::{ - builtins::{ - int, tuple::IntoPyTuple, PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyTupleRef, - PyTypeRef, - }, - common::{ - lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, - rc::PyRc, - }, - convert::ToPyObject, - function::{ArgCallable, ArgIntoBool, FuncArgs, OptionalArg, OptionalOption, PosArgs}, - identifier, - protocol::{PyIter, PyIterReturn, PyNumber}, - stdlib::sys, - types::{Constructor, IterNext, Iterable, Representable, SelfIter}, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, TryFromObject, - VirtualMachine, - }; - use crossbeam_utils::atomic::AtomicCell; - use num_traits::{Signed, ToPrimitive}; - use std::fmt; - - #[pyattr] - #[pyclass(name = "chain")] - #[derive(Debug, PyPayload)] - struct PyItertoolsChain { - source: PyRwLock<Option<PyIter>>, - active: PyRwLock<Option<PyIter>>, - } - - #[pyclass(with(IterNext, Iterable), flags(BASETYPE, HAS_DICT))] - impl PyItertoolsChain { - #[pyslot] - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let args_list = PyList::from(args.args); - PyItertoolsChain { - source: PyRwLock::new(Some(args_list.to_pyobject(vm).get_iter(vm)?)), - active: PyRwLock::new(None), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - - #[pyclassmethod] - fn from_iterable( - cls: PyTypeRef, - source: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<PyRef<Self>> { - PyItertoolsChain { - source: PyRwLock::new(Some(source.get_iter(vm)?)), - active: PyRwLock::new(None), - } - .into_ref_with_type(vm, cls) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - let source = zelf.source.read().clone(); - let active = zelf.active.read().clone(); - let cls = zelf.class().to_owned(); - let empty_tuple = vm.ctx.empty_tuple.clone(); - let reduced = match source { - Some(source) => match active { - Some(active) => vm.new_tuple((cls, empty_tuple, (source, active))), - None => vm.new_tuple((cls, empty_tuple, (source,))), - }, - None => vm.new_tuple((cls, empty_tuple)), - }; - Ok(reduced) - } - - #[pymethod(magic)] - fn setstate(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.is_empty() { - let msg = String::from("function takes at leat 1 arguments (0 given)"); - return Err(vm.new_type_error(msg)); - } - if args.len() > 2 { - let msg = format!("function takes at most 2 arguments ({} given)", args.len()); - return Err(vm.new_type_error(msg)); - } - let source = &args[0]; - if args.len() == 1 { - if !PyIter::check(source.as_ref()) { - return Err(vm.new_type_error(String::from("Arguments must be iterators."))); - } - *zelf.source.write() = source.to_owned().try_into_value(vm)?; - return Ok(()); - } - let active = &args[1]; - - if !PyIter::check(source.as_ref()) || !PyIter::check(active.as_ref()) { - return Err(vm.new_type_error(String::from("Arguments must be iterators."))); - } - let mut source_lock = zelf.source.write(); - let mut active_lock = zelf.active.write(); - *source_lock = source.to_owned().try_into_value(vm)?; - *active_lock = active.to_owned().try_into_value(vm)?; - Ok(()) - } - } - - impl SelfIter for PyItertoolsChain {} - - impl IterNext for PyItertoolsChain { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let Some(source) = zelf.source.read().clone() else { - return Ok(PyIterReturn::StopIteration(None)); - }; - let next = loop { - let maybe_active = zelf.active.read().clone(); - if let Some(active) = maybe_active { - match active.next(vm) { - Ok(PyIterReturn::Return(ok)) => { - break Ok(PyIterReturn::Return(ok)); - } - Ok(PyIterReturn::StopIteration(_)) => { - *zelf.active.write() = None; - } - Err(err) => { - break Err(err); - } - } - } else { - match source.next(vm) { - Ok(PyIterReturn::Return(ok)) => match ok.get_iter(vm) { - Ok(iter) => { - *zelf.active.write() = Some(iter); - } - Err(err) => { - break Err(err); - } - }, - Ok(PyIterReturn::StopIteration(_)) => { - break Ok(PyIterReturn::StopIteration(None)); - } - Err(err) => { - break Err(err); - } - } - } - }; - match next { - Err(_) | Ok(PyIterReturn::StopIteration(_)) => { - *zelf.source.write() = None; - } - _ => {} - }; - next - } - } - - #[pyattr] - #[pyclass(name = "compress")] - #[derive(Debug, PyPayload)] - struct PyItertoolsCompress { - data: PyIter, - selectors: PyIter, - } - - #[derive(FromArgs)] - struct CompressNewArgs { - #[pyarg(any)] - data: PyIter, - #[pyarg(any)] - selectors: PyIter, - } - - impl Constructor for PyItertoolsCompress { - type Args = CompressNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { data, selectors }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyItertoolsCompress { data, selectors } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsCompress { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>) -> (PyTypeRef, (PyIter, PyIter)) { - ( - zelf.class().to_owned(), - (zelf.data.clone(), zelf.selectors.clone()), - ) - } - } - - impl SelfIter for PyItertoolsCompress {} - - impl IterNext for PyItertoolsCompress { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - loop { - let sel_obj = match zelf.selectors.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - let verdict = sel_obj.clone().try_to_bool(vm)?; - let data_obj = zelf.data.next(vm)?; - - if verdict { - return Ok(data_obj); - } - } - } - } - - #[pyattr] - #[pyclass(name = "count")] - #[derive(Debug, PyPayload)] - struct PyItertoolsCount { - cur: PyRwLock<PyObjectRef>, - step: PyObjectRef, - } - - #[derive(FromArgs)] - struct CountNewArgs { - #[pyarg(positional, optional)] - start: OptionalArg<PyObjectRef>, - - #[pyarg(positional, optional)] - step: OptionalArg<PyObjectRef>, - } - - impl Constructor for PyItertoolsCount { - type Args = CountNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { start, step }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let start = start.into_option().unwrap_or_else(|| vm.new_pyobj(0)); - let step = step.into_option().unwrap_or_else(|| vm.new_pyobj(1)); - if !PyNumber::check(&start) || !PyNumber::check(&step) { - return Err(vm.new_type_error("a number is required".to_owned())); - } - - Self { - cur: PyRwLock::new(start), - step, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor, Representable))] - impl PyItertoolsCount { - // TODO: Implement this - // if (lz->cnt == PY_SSIZE_T_MAX) - // return Py_BuildValue("0(00)", Py_TYPE(lz), lz->long_cnt, lz->long_step); - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef,)) { - (zelf.class().to_owned(), (zelf.cur.read().clone(),)) - } - } - - impl SelfIter for PyItertoolsCount {} - - impl IterNext for PyItertoolsCount { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let mut cur = zelf.cur.write(); - let step = zelf.step.clone(); - let result = cur.clone(); - *cur = vm._iadd(&cur, step.as_object())?; - Ok(PyIterReturn::Return(result.to_pyobject(vm))) - } - } - - impl Representable for PyItertoolsCount { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let cur = format!("{}", zelf.cur.read().clone().repr(vm)?); - let step = &zelf.step; - if vm.bool_eq(step, vm.ctx.new_int(1).as_object())? { - return Ok(format!("count({cur})")); - } - Ok(format!("count({}, {})", cur, step.repr(vm)?)) - } - } - - #[pyattr] - #[pyclass(name = "cycle")] - #[derive(Debug, PyPayload)] - struct PyItertoolsCycle { - iter: PyIter, - saved: PyRwLock<Vec<PyObjectRef>>, - index: AtomicCell<usize>, - } - - impl Constructor for PyItertoolsCycle { - type Args = PyIter; - - fn py_new(cls: PyTypeRef, iter: Self::Args, vm: &VirtualMachine) -> PyResult { - Self { - iter, - saved: PyRwLock::new(Vec::new()), - index: AtomicCell::new(0), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsCycle {} - - impl SelfIter for PyItertoolsCycle {} - - impl IterNext for PyItertoolsCycle { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let item = if let PyIterReturn::Return(item) = zelf.iter.next(vm)? { - zelf.saved.write().push(item.clone()); - item - } else { - let saved = zelf.saved.read(); - if saved.len() == 0 { - return Ok(PyIterReturn::StopIteration(None)); - } - - let last_index = zelf.index.fetch_add(1); - - if last_index >= saved.len() - 1 { - zelf.index.store(0); - } - - saved[last_index].clone() - }; - - Ok(PyIterReturn::Return(item)) - } - } - - #[pyattr] - #[pyclass(name = "repeat")] - #[derive(Debug, PyPayload)] - struct PyItertoolsRepeat { - object: PyObjectRef, - times: Option<PyRwLock<usize>>, - } - - #[derive(FromArgs)] - struct PyRepeatNewArgs { - object: PyObjectRef, - #[pyarg(any, optional)] - times: OptionalArg<PyIntRef>, - } - - impl Constructor for PyItertoolsRepeat { - type Args = PyRepeatNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { object, times }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let times = match times.into_option() { - Some(int) => { - let val: isize = int.try_to_primitive(vm)?; - // times always >= 0. - Some(PyRwLock::new(val.to_usize().unwrap_or(0))) - } - None => None, - }; - PyItertoolsRepeat { object, times } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor, Representable), flags(BASETYPE))] - impl PyItertoolsRepeat { - #[pymethod(magic)] - fn length_hint(&self, vm: &VirtualMachine) -> PyResult<usize> { - // Return TypeError, length_hint picks this up and returns the default. - let times = self - .times - .as_ref() - .ok_or_else(|| vm.new_type_error("length of unsized object.".to_owned()))?; - Ok(*times.read()) - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - let cls = zelf.class().to_owned(); - Ok(match zelf.times { - Some(ref times) => vm.new_tuple((cls, (zelf.object.clone(), *times.read()))), - None => vm.new_tuple((cls, (zelf.object.clone(),))), - }) - } - } - - impl SelfIter for PyItertoolsRepeat {} - - impl IterNext for PyItertoolsRepeat { - fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> { - if let Some(ref times) = zelf.times { - let mut times = times.write(); - if *times == 0 { - return Ok(PyIterReturn::StopIteration(None)); - } - *times -= 1; - } - Ok(PyIterReturn::Return(zelf.object.clone())) - } - } - - impl Representable for PyItertoolsRepeat { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let mut fmt = format!("{}", &zelf.object.repr(vm)?); - if let Some(ref times) = zelf.times { - fmt.push_str(", "); - fmt.push_str(&times.read().to_string()); - } - Ok(format!("repeat({fmt})")) - } - } - - #[pyattr] - #[pyclass(name = "starmap")] - #[derive(Debug, PyPayload)] - struct PyItertoolsStarmap { - function: PyObjectRef, - iterable: PyIter, - } - - #[derive(FromArgs)] - struct StarmapNewArgs { - #[pyarg(positional)] - function: PyObjectRef, - #[pyarg(positional)] - iterable: PyIter, - } - - impl Constructor for PyItertoolsStarmap { - type Args = StarmapNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { function, iterable }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyItertoolsStarmap { function, iterable } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsStarmap { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef, PyIter)) { - ( - zelf.class().to_owned(), - (zelf.function.clone(), zelf.iterable.clone()), - ) - } - } - - impl SelfIter for PyItertoolsStarmap {} - - impl IterNext for PyItertoolsStarmap { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let obj = zelf.iterable.next(vm)?; - let function = &zelf.function; - match obj { - PyIterReturn::Return(obj) => { - let args: Vec<_> = obj.try_to_value(vm)?; - PyIterReturn::from_pyresult(function.call(args, vm), vm) - } - PyIterReturn::StopIteration(v) => Ok(PyIterReturn::StopIteration(v)), - } - } - } - - #[pyattr] - #[pyclass(name = "takewhile")] - #[derive(Debug, PyPayload)] - struct PyItertoolsTakewhile { - predicate: PyObjectRef, - iterable: PyIter, - stop_flag: AtomicCell<bool>, - } - - #[derive(FromArgs)] - struct TakewhileNewArgs { - #[pyarg(positional)] - predicate: PyObjectRef, - #[pyarg(positional)] - iterable: PyIter, - } - - impl Constructor for PyItertoolsTakewhile { - type Args = TakewhileNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { - predicate, - iterable, - }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyItertoolsTakewhile { - predicate, - iterable, - stop_flag: AtomicCell::new(false), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsTakewhile { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { - ( - zelf.class().to_owned(), - (zelf.predicate.clone(), zelf.iterable.clone()), - zelf.stop_flag.load() as _, - ) - } - #[pymethod(magic)] - fn setstate(zelf: PyRef<Self>, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if let Ok(obj) = ArgIntoBool::try_from_object(vm, state) { - zelf.stop_flag.store(*obj); - } - Ok(()) - } - } - - impl SelfIter for PyItertoolsTakewhile {} - - impl IterNext for PyItertoolsTakewhile { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - if zelf.stop_flag.load() { - return Ok(PyIterReturn::StopIteration(None)); - } - - // might be StopIteration or anything else, which is propagated upwards - let obj = match zelf.iterable.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - let predicate = &zelf.predicate; - - let verdict = predicate.call((obj.clone(),), vm)?; - let verdict = verdict.try_to_bool(vm)?; - if verdict { - Ok(PyIterReturn::Return(obj)) - } else { - zelf.stop_flag.store(true); - Ok(PyIterReturn::StopIteration(None)) - } - } - } - - #[pyattr] - #[pyclass(name = "dropwhile")] - #[derive(Debug, PyPayload)] - struct PyItertoolsDropwhile { - predicate: ArgCallable, - iterable: PyIter, - start_flag: AtomicCell<bool>, - } - - #[derive(FromArgs)] - struct DropwhileNewArgs { - #[pyarg(positional)] - predicate: ArgCallable, - #[pyarg(positional)] - iterable: PyIter, - } - - impl Constructor for PyItertoolsDropwhile { - type Args = DropwhileNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { - predicate, - iterable, - }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyItertoolsDropwhile { - predicate, - iterable, - start_flag: AtomicCell::new(false), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsDropwhile { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef, PyIter), u32) { - ( - zelf.class().to_owned(), - (zelf.predicate.clone().into(), zelf.iterable.clone()), - (zelf.start_flag.load() as _), - ) - } - #[pymethod(magic)] - fn setstate(zelf: PyRef<Self>, state: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if let Ok(obj) = ArgIntoBool::try_from_object(vm, state) { - zelf.start_flag.store(*obj); - } - Ok(()) - } - } - - impl SelfIter for PyItertoolsDropwhile {} - - impl IterNext for PyItertoolsDropwhile { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let predicate = &zelf.predicate; - let iterable = &zelf.iterable; - - if !zelf.start_flag.load() { - loop { - let obj = match iterable.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => { - return Ok(PyIterReturn::StopIteration(v)); - } - }; - let pred = predicate.clone(); - let pred_value = pred.invoke((obj.clone(),), vm)?; - if !pred_value.try_to_bool(vm)? { - zelf.start_flag.store(true); - return Ok(PyIterReturn::Return(obj)); - } - } - } - iterable.next(vm) - } - } - - struct GroupByState { - current_value: Option<PyObjectRef>, - current_key: Option<PyObjectRef>, - next_group: bool, - grouper: Option<PyWeakRef<PyItertoolsGrouper>>, - } - - impl fmt::Debug for GroupByState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("GroupByState") - .field("current_value", &self.current_value) - .field("current_key", &self.current_key) - .field("next_group", &self.next_group) - .finish() - } - } - - impl GroupByState { - fn is_current(&self, grouper: &Py<PyItertoolsGrouper>) -> bool { - self.grouper - .as_ref() - .and_then(|g| g.upgrade()) - .map_or(false, |ref current_grouper| grouper.is(current_grouper)) - } - } - - #[pyattr] - #[pyclass(name = "groupby")] - #[derive(PyPayload)] - struct PyItertoolsGroupBy { - iterable: PyIter, - key_func: Option<PyObjectRef>, - state: PyMutex<GroupByState>, - } - - impl fmt::Debug for PyItertoolsGroupBy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PyItertoolsGroupBy") - .field("iterable", &self.iterable) - .field("key_func", &self.key_func) - .field("state", &self.state.lock()) - .finish() - } - } - - #[derive(FromArgs)] - struct GroupByArgs { - iterable: PyIter, - #[pyarg(any, optional)] - key: OptionalOption<PyObjectRef>, - } - - impl Constructor for PyItertoolsGroupBy { - type Args = GroupByArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { iterable, key }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyItertoolsGroupBy { - iterable, - key_func: key.flatten(), - state: PyMutex::new(GroupByState { - current_key: None, - current_value: None, - next_group: false, - grouper: None, - }), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsGroupBy { - pub(super) fn advance( - &self, - vm: &VirtualMachine, - ) -> PyResult<PyIterReturn<(PyObjectRef, PyObjectRef)>> { - let new_value = match self.iterable.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - let new_key = if let Some(ref kf) = self.key_func { - kf.call((new_value.clone(),), vm)? - } else { - new_value.clone() - }; - Ok(PyIterReturn::Return((new_value, new_key))) - } - } - - impl SelfIter for PyItertoolsGroupBy {} - - impl IterNext for PyItertoolsGroupBy { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let mut state = zelf.state.lock(); - state.grouper = None; - - if !state.next_group { - // FIXME: unnecessary clone. current_key always exist until assigning new - let current_key = state.current_key.clone(); - drop(state); - - let (value, key) = if let Some(old_key) = current_key { - loop { - let (value, new_key) = match zelf.advance(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => { - return Ok(PyIterReturn::StopIteration(v)); - } - }; - if !vm.bool_eq(&new_key, &old_key)? { - break (value, new_key); - } - } - } else { - match zelf.advance(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => { - return Ok(PyIterReturn::StopIteration(v)); - } - } - }; - - state = zelf.state.lock(); - state.current_value = Some(value); - state.current_key = Some(key); - } - - state.next_group = false; - - let grouper = PyItertoolsGrouper { - groupby: zelf.to_owned(), - } - .into_ref(&vm.ctx); - - state.grouper = Some(grouper.downgrade(None, vm).unwrap()); - Ok(PyIterReturn::Return( - (state.current_key.as_ref().unwrap().clone(), grouper).to_pyobject(vm), - )) - } - } - - #[pyattr] - #[pyclass(name = "_grouper")] - #[derive(Debug, PyPayload)] - struct PyItertoolsGrouper { - groupby: PyRef<PyItertoolsGroupBy>, - } - - #[pyclass(with(IterNext, Iterable))] - impl PyItertoolsGrouper {} - - impl SelfIter for PyItertoolsGrouper {} - - impl IterNext for PyItertoolsGrouper { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let old_key = { - let mut state = zelf.groupby.state.lock(); - - if !state.is_current(zelf) { - return Ok(PyIterReturn::StopIteration(None)); - } - - // check to see if the value has already been retrieved from the iterator - if let Some(val) = state.current_value.take() { - return Ok(PyIterReturn::Return(val)); - } - - state.current_key.as_ref().unwrap().clone() - }; - let (value, key) = match zelf.groupby.advance(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - if vm.bool_eq(&key, &old_key)? { - Ok(PyIterReturn::Return(value)) - } else { - let mut state = zelf.groupby.state.lock(); - state.current_value = Some(value); - state.current_key = Some(key); - state.next_group = true; - state.grouper = None; - Ok(PyIterReturn::StopIteration(None)) - } - } - } - - #[pyattr] - #[pyclass(name = "islice")] - #[derive(Debug, PyPayload)] - struct PyItertoolsIslice { - iterable: PyIter, - cur: AtomicCell<usize>, - next: AtomicCell<usize>, - stop: Option<usize>, - step: usize, - } - - // Restrict obj to ints with value 0 <= val <= sys.maxsize - // On failure (out of range, non-int object) a ValueError is raised. - fn pyobject_to_opt_usize( - obj: PyObjectRef, - name: &'static str, - vm: &VirtualMachine, - ) -> PyResult<usize> { - let is_int = obj.fast_isinstance(vm.ctx.types.int_type); - if is_int { - let value = int::get_value(&obj).to_usize(); - if let Some(value) = value { - // Only succeeds for values for which 0 <= value <= sys.maxsize - if value <= sys::MAXSIZE as usize { - return Ok(value); - } - } - } - // We don't have an int or value was < 0 or > sys.maxsize - Err(vm.new_value_error(format!( - "{name} argument for islice() must be None or an integer: 0 <= x <= sys.maxsize." - ))) - } - - #[pyclass(with(IterNext, Iterable), flags(BASETYPE))] - impl PyItertoolsIslice { - #[pyslot] - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let (iter, start, stop, step) = match args.args.len() { - 0 | 1 => { - return Err(vm.new_type_error(format!( - "islice expected at least 2 arguments, got {}", - args.args.len() - ))); - } - 2 => { - let (iter, stop): (PyObjectRef, PyObjectRef) = args.bind(vm)?; - (iter, 0usize, stop, 1usize) - } - _ => { - let (iter, start, stop, step) = if args.args.len() == 3 { - let (iter, start, stop): (PyObjectRef, PyObjectRef, PyObjectRef) = - args.bind(vm)?; - (iter, start, stop, 1usize) - } else { - let (iter, start, stop, step): ( - PyObjectRef, - PyObjectRef, - PyObjectRef, - PyObjectRef, - ) = args.bind(vm)?; - - let step = if !vm.is_none(&step) { - pyobject_to_opt_usize(step, "Step", vm)? - } else { - 1usize - }; - (iter, start, stop, step) - }; - let start = if !vm.is_none(&start) { - pyobject_to_opt_usize(start, "Start", vm)? - } else { - 0usize - }; - - (iter, start, stop, step) - } - }; - - let stop = if !vm.is_none(&stop) { - Some(pyobject_to_opt_usize(stop, "Stop", vm)?) - } else { - None - }; - - let iter = iter.get_iter(vm)?; - - PyItertoolsIslice { - iterable: iter, - cur: AtomicCell::new(0), - next: AtomicCell::new(start), - stop, - step, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - let cls = zelf.class().to_owned(); - let itr = zelf.iterable.clone(); - let cur = zelf.cur.take(); - let next = zelf.next.take(); - let step = zelf.step; - match zelf.stop { - Some(stop) => Ok(vm.new_tuple((cls, (itr, next, stop, step), (cur,)))), - _ => Ok(vm.new_tuple((cls, (itr, next, vm.new_pyobj(()), step), (cur,)))), - } - } - - #[pymethod(magic)] - fn setstate(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.len() != 1 { - let msg = format!("function takes exactly 1 argument ({} given)", args.len()); - return Err(vm.new_type_error(msg)); - } - let cur = &args[0]; - if let Ok(cur) = cur.try_to_value(vm) { - zelf.cur.store(cur); - } else { - return Err(vm.new_type_error(String::from("Argument must be usize."))); - } - Ok(()) - } - } - - impl SelfIter for PyItertoolsIslice {} - - impl IterNext for PyItertoolsIslice { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - while zelf.cur.load() < zelf.next.load() { - zelf.iterable.next(vm)?; - zelf.cur.fetch_add(1); - } - - if let Some(stop) = zelf.stop { - if zelf.cur.load() >= stop { - return Ok(PyIterReturn::StopIteration(None)); - } - } - - let obj = match zelf.iterable.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - zelf.cur.fetch_add(1); - - // TODO is this overflow check required? attempts to copy CPython. - let (next, ovf) = zelf.next.load().overflowing_add(zelf.step); - zelf.next.store(if ovf { zelf.stop.unwrap() } else { next }); - - Ok(PyIterReturn::Return(obj)) - } - } - - #[pyattr] - #[pyclass(name = "filterfalse")] - #[derive(Debug, PyPayload)] - struct PyItertoolsFilterFalse { - predicate: PyObjectRef, - iterable: PyIter, - } - - #[derive(FromArgs)] - struct FilterFalseNewArgs { - #[pyarg(positional)] - predicate: PyObjectRef, - #[pyarg(positional)] - iterable: PyIter, - } - - impl Constructor for PyItertoolsFilterFalse { - type Args = FilterFalseNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { - predicate, - iterable, - }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - PyItertoolsFilterFalse { - predicate, - iterable, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor), flags(BASETYPE))] - impl PyItertoolsFilterFalse { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>) -> (PyTypeRef, (PyObjectRef, PyIter)) { - ( - zelf.class().to_owned(), - (zelf.predicate.clone(), zelf.iterable.clone()), - ) - } - } - - impl SelfIter for PyItertoolsFilterFalse {} - - impl IterNext for PyItertoolsFilterFalse { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let predicate = &zelf.predicate; - let iterable = &zelf.iterable; - - loop { - let obj = match iterable.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - let pred_value = if vm.is_none(predicate) { - obj.clone() - } else { - predicate.call((obj.clone(),), vm)? - }; - - if !pred_value.try_to_bool(vm)? { - return Ok(PyIterReturn::Return(obj)); - } - } - } - } - - #[pyattr] - #[pyclass(name = "accumulate")] - #[derive(Debug, PyPayload)] - struct PyItertoolsAccumulate { - iterable: PyIter, - binop: Option<PyObjectRef>, - initial: Option<PyObjectRef>, - acc_value: PyRwLock<Option<PyObjectRef>>, - } - - #[derive(FromArgs)] - struct AccumulateArgs { - iterable: PyIter, - #[pyarg(any, optional)] - func: OptionalOption<PyObjectRef>, - #[pyarg(named, optional)] - initial: OptionalOption<PyObjectRef>, - } - - impl Constructor for PyItertoolsAccumulate { - type Args = AccumulateArgs; - - fn py_new(cls: PyTypeRef, args: AccumulateArgs, vm: &VirtualMachine) -> PyResult { - PyItertoolsAccumulate { - iterable: args.iterable, - binop: args.func.flatten(), - initial: args.initial.flatten(), - acc_value: PyRwLock::new(None), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsAccumulate { - #[pymethod(magic)] - fn setstate(zelf: PyRef<Self>, state: PyObjectRef, _vm: &VirtualMachine) -> PyResult<()> { - *zelf.acc_value.write() = Some(state); - Ok(()) - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { - let class = zelf.class().to_owned(); - let binop = zelf.binop.clone(); - let it = zelf.iterable.clone(); - let acc_value = zelf.acc_value.read().clone(); - if let Some(initial) = &zelf.initial { - let chain_args = PyList::from(vec![initial.clone(), it.to_pyobject(vm)]); - let chain = PyItertoolsChain { - source: PyRwLock::new(Some(chain_args.to_pyobject(vm).get_iter(vm).unwrap())), - active: PyRwLock::new(None), - }; - let tup = vm.new_tuple((chain, binop)); - return vm.new_tuple((class, tup, acc_value)); - } - match acc_value { - Some(obj) if obj.is(&vm.ctx.none) => { - let chain_args = PyList::from(vec![]); - let chain = PyItertoolsChain { - source: PyRwLock::new(Some( - chain_args.to_pyobject(vm).get_iter(vm).unwrap(), - )), - active: PyRwLock::new(None), - } - .into_pyobject(vm); - let acc = Self { - iterable: PyIter::new(chain), - binop, - initial: None, - acc_value: PyRwLock::new(None), - }; - let tup = vm.new_tuple((acc, 1, None::<PyObjectRef>)); - let islice_cls = PyItertoolsIslice::class(&vm.ctx).to_owned(); - return vm.new_tuple((islice_cls, tup)); - } - _ => {} - } - let tup = vm.new_tuple((it, binop)); - vm.new_tuple((class, tup, acc_value)) - } - } - - impl SelfIter for PyItertoolsAccumulate {} - - impl IterNext for PyItertoolsAccumulate { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let iterable = &zelf.iterable; - - let acc_value = zelf.acc_value.read().clone(); - - let next_acc_value = match acc_value { - None => match &zelf.initial { - None => match iterable.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => { - return Ok(PyIterReturn::StopIteration(v)); - } - }, - Some(obj) => obj.clone(), - }, - Some(value) => { - let obj = match iterable.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => { - return Ok(PyIterReturn::StopIteration(v)); - } - }; - match &zelf.binop { - None => vm._add(&value, &obj)?, - Some(op) => op.call((value, obj), vm)?, - } - } - }; - *zelf.acc_value.write() = Some(next_acc_value.clone()); - - Ok(PyIterReturn::Return(next_acc_value)) - } - } - - #[derive(Debug)] - struct PyItertoolsTeeData { - iterable: PyIter, - values: PyRwLock<Vec<PyObjectRef>>, - } - - impl PyItertoolsTeeData { - fn new(iterable: PyIter, _vm: &VirtualMachine) -> PyResult<PyRc<PyItertoolsTeeData>> { - Ok(PyRc::new(PyItertoolsTeeData { - iterable, - values: PyRwLock::new(vec![]), - })) - } - - fn get_item(&self, vm: &VirtualMachine, index: usize) -> PyResult<PyIterReturn> { - if self.values.read().len() == index { - let result = match self.iterable.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - self.values.write().push(result); - } - Ok(PyIterReturn::Return(self.values.read()[index].clone())) - } - } - - #[pyattr] - #[pyclass(name = "tee")] - #[derive(Debug, PyPayload)] - struct PyItertoolsTee { - tee_data: PyRc<PyItertoolsTeeData>, - index: AtomicCell<usize>, - } - - #[derive(FromArgs)] - struct TeeNewArgs { - #[pyarg(positional)] - iterable: PyIter, - #[pyarg(positional, optional)] - n: OptionalArg<usize>, - } - - impl Constructor for PyItertoolsTee { - type Args = TeeNewArgs; - - // TODO: make tee() a function, rename this class to itertools._tee and make - // teedata a python class - fn py_new( - _cls: PyTypeRef, - Self::Args { iterable, n }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let n = n.unwrap_or(2); - - let copyable = if iterable.class().has_attr(identifier!(vm, __copy__)) { - vm.call_special_method(iterable.as_object(), identifier!(vm, __copy__), ())? - } else { - PyItertoolsTee::from_iter(iterable, vm)? - }; - - let mut tee_vec: Vec<PyObjectRef> = Vec::with_capacity(n); - for _ in 0..n { - tee_vec.push(vm.call_special_method(&copyable, identifier!(vm, __copy__), ())?); - } - - Ok(PyTuple::new_ref(tee_vec, &vm.ctx).into()) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsTee { - fn from_iter(iterator: PyIter, vm: &VirtualMachine) -> PyResult { - let class = PyItertoolsTee::class(&vm.ctx); - if iterator.class().is(PyItertoolsTee::class(&vm.ctx)) { - return vm.call_special_method(&iterator, identifier!(vm, __copy__), ()); - } - Ok(PyItertoolsTee { - tee_data: PyItertoolsTeeData::new(iterator, vm)?, - index: AtomicCell::new(0), - } - .into_ref_with_type(vm, class.to_owned())? - .into()) - } - - #[pymethod(magic)] - fn copy(&self) -> Self { - Self { - tee_data: PyRc::clone(&self.tee_data), - index: AtomicCell::new(self.index.load()), - } - } - } - impl SelfIter for PyItertoolsTee {} - impl IterNext for PyItertoolsTee { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let value = match zelf.tee_data.get_item(vm, zelf.index.load())? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - zelf.index.fetch_add(1); - Ok(PyIterReturn::Return(value)) - } - } - - #[pyattr] - #[pyclass(name = "product")] - #[derive(Debug, PyPayload)] - struct PyItertoolsProduct { - pools: Vec<Vec<PyObjectRef>>, - idxs: PyRwLock<Vec<usize>>, - cur: AtomicCell<usize>, - stop: AtomicCell<bool>, - } - - #[derive(FromArgs)] - struct ProductArgs { - #[pyarg(named, optional)] - repeat: OptionalArg<usize>, - } - - impl Constructor for PyItertoolsProduct { - type Args = (PosArgs<PyObjectRef>, ProductArgs); - - fn py_new(cls: PyTypeRef, (iterables, args): Self::Args, vm: &VirtualMachine) -> PyResult { - let repeat = args.repeat.unwrap_or(1); - let mut pools = Vec::new(); - for arg in iterables.iter() { - pools.push(arg.try_to_value(vm)?); - } - let pools = std::iter::repeat(pools) - .take(repeat) - .flatten() - .collect::<Vec<Vec<PyObjectRef>>>(); - - let l = pools.len(); - - PyItertoolsProduct { - pools, - idxs: PyRwLock::new(vec![0; l]), - cur: AtomicCell::new(l.wrapping_sub(1)), - stop: AtomicCell::new(false), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsProduct { - fn update_idxs(&self, mut idxs: PyRwLockWriteGuard<'_, Vec<usize>>) { - if idxs.len() == 0 { - self.stop.store(true); - return; - } - - let cur = self.cur.load(); - let lst_idx = &self.pools[cur].len() - 1; - - if idxs[cur] == lst_idx { - if cur == 0 { - self.stop.store(true); - return; - } - idxs[cur] = 0; - self.cur.fetch_sub(1); - self.update_idxs(idxs); - } else { - idxs[cur] += 1; - self.cur.store(idxs.len() - 1); - } - } - - #[pymethod(magic)] - fn setstate(zelf: PyRef<Self>, state: PyTupleRef, vm: &VirtualMachine) -> PyResult<()> { - let args = state.as_slice(); - if args.len() != zelf.pools.len() { - let msg = "Invalid number of arguments".to_string(); - return Err(vm.new_type_error(msg)); - } - let mut idxs: PyRwLockWriteGuard<'_, Vec<usize>> = zelf.idxs.write(); - idxs.clear(); - for s in 0..args.len() { - let index = get_value(state.get(s).unwrap()).to_usize().unwrap(); - let pool_size = zelf.pools.get(s).unwrap().len(); - if pool_size == 0 { - zelf.stop.store(true); - return Ok(()); - } - if index >= pool_size { - idxs.push(pool_size - 1); - } else { - idxs.push(index); - } - } - zelf.stop.store(false); - Ok(()) - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { - let class = zelf.class().to_owned(); - - if zelf.stop.load() { - return vm.new_tuple((class, (vm.ctx.empty_tuple.clone(),))); - } - - let mut pools: Vec<PyObjectRef> = Vec::new(); - for element in zelf.pools.iter() { - pools.push(element.clone().into_pytuple(vm).into()); - } - - let mut indices: Vec<PyObjectRef> = Vec::new(); - - for item in &zelf.idxs.read()[..] { - indices.push(vm.new_pyobj(*item)); - } - - vm.new_tuple(( - class, - pools.clone().into_pytuple(vm), - indices.into_pytuple(vm), - )) - } - } - - impl SelfIter for PyItertoolsProduct {} - impl IterNext for PyItertoolsProduct { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - // stop signal - if zelf.stop.load() { - return Ok(PyIterReturn::StopIteration(None)); - } - - let pools = &zelf.pools; - - for p in pools { - if p.is_empty() { - return Ok(PyIterReturn::StopIteration(None)); - } - } - - let idxs = zelf.idxs.write(); - let res = vm.ctx.new_tuple( - pools - .iter() - .zip(idxs.iter()) - .map(|(pool, idx)| pool[*idx].clone()) - .collect(), - ); - - zelf.update_idxs(idxs); - - Ok(PyIterReturn::Return(res.into())) - } - } - - #[pyattr] - #[pyclass(name = "combinations")] - #[derive(Debug, PyPayload)] - struct PyItertoolsCombinations { - pool: Vec<PyObjectRef>, - indices: PyRwLock<Vec<usize>>, - result: PyRwLock<Option<Vec<PyObjectRef>>>, - r: AtomicCell<usize>, - exhausted: AtomicCell<bool>, - } - - #[derive(FromArgs)] - struct CombinationsNewArgs { - #[pyarg(any)] - iterable: PyObjectRef, - #[pyarg(any)] - r: PyIntRef, - } - - impl Constructor for PyItertoolsCombinations { - type Args = CombinationsNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { iterable, r }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let pool: Vec<_> = iterable.try_to_value(vm)?; - - let r = r.as_bigint(); - if r.is_negative() { - return Err(vm.new_value_error("r must be non-negative".to_owned())); - } - let r = r.to_usize().unwrap(); - - let n = pool.len(); - - Self { - pool, - indices: PyRwLock::new((0..r).collect()), - result: PyRwLock::new(None), - r: AtomicCell::new(r), - exhausted: AtomicCell::new(r > n), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsCombinations { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef { - let r = zelf.r.load(); - - let class = zelf.class().to_owned(); - - if zelf.exhausted.load() { - return vm.new_tuple(( - class, - vm.new_tuple((vm.ctx.empty_tuple.clone(), vm.ctx.new_int(r))), - )); - } - - let tup = vm.new_tuple((zelf.pool.clone().into_pytuple(vm), vm.ctx.new_int(r))); - - if zelf.result.read().is_none() { - vm.new_tuple((class, tup)) - } else { - let mut indices: Vec<PyObjectRef> = Vec::new(); - - for item in &zelf.indices.read()[..r] { - indices.push(vm.new_pyobj(*item)); - } - - vm.new_tuple((class, tup, indices.into_pytuple(vm))) - } - } - } - - impl SelfIter for PyItertoolsCombinations {} - impl IterNext for PyItertoolsCombinations { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - // stop signal - if zelf.exhausted.load() { - return Ok(PyIterReturn::StopIteration(None)); - } - - let n = zelf.pool.len(); - let r = zelf.r.load(); - - if r == 0 { - zelf.exhausted.store(true); - return Ok(PyIterReturn::Return(vm.new_tuple(()).into())); - } - - let mut result_lock = zelf.result.write(); - let result = if let Some(ref mut result) = *result_lock { - let mut indices = zelf.indices.write(); - - // Scan indices right-to-left until finding one that is not at its maximum (i + n - r). - let mut idx = r as isize - 1; - while idx >= 0 && indices[idx as usize] == idx as usize + n - r { - idx -= 1; - } - - // If no suitable index is found, then the indices are all at - // their maximum value and we're done. - if idx < 0 { - zelf.exhausted.store(true); - return Ok(PyIterReturn::StopIteration(None)); - } else { - // Increment the current index which we know is not at its - // maximum. Then move back to the right setting each index - // to its lowest possible value (one higher than the index - // to its left -- this maintains the sort order invariant). - indices[idx as usize] += 1; - for j in idx as usize + 1..r { - indices[j] = indices[j - 1] + 1; - } - - // Update the result tuple for the new indices - // starting with i, the leftmost index that changed - for i in idx as usize..r { - let index = indices[i]; - let elem = &zelf.pool[index]; - result[i] = elem.to_owned(); - } - - result.to_vec() - } - } else { - let res = zelf.pool[0..r].to_vec(); - *result_lock = Some(res.clone()); - res - }; - - Ok(PyIterReturn::Return(vm.ctx.new_tuple(result).into())) - } - } - - #[pyattr] - #[pyclass(name = "combinations_with_replacement")] - #[derive(Debug, PyPayload)] - struct PyItertoolsCombinationsWithReplacement { - pool: Vec<PyObjectRef>, - indices: PyRwLock<Vec<usize>>, - r: AtomicCell<usize>, - exhausted: AtomicCell<bool>, - } - - impl Constructor for PyItertoolsCombinationsWithReplacement { - type Args = CombinationsNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { iterable, r }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let pool: Vec<_> = iterable.try_to_value(vm)?; - let r = r.as_bigint(); - if r.is_negative() { - return Err(vm.new_value_error("r must be non-negative".to_owned())); - } - let r = r.to_usize().unwrap(); - - let n = pool.len(); - - PyItertoolsCombinationsWithReplacement { - pool, - indices: PyRwLock::new(vec![0; r]), - r: AtomicCell::new(r), - exhausted: AtomicCell::new(n == 0 && r > 0), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsCombinationsWithReplacement {} - - impl SelfIter for PyItertoolsCombinationsWithReplacement {} - - impl IterNext for PyItertoolsCombinationsWithReplacement { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - // stop signal - if zelf.exhausted.load() { - return Ok(PyIterReturn::StopIteration(None)); - } - - let n = zelf.pool.len(); - let r = zelf.r.load(); - - if r == 0 { - zelf.exhausted.store(true); - return Ok(PyIterReturn::Return(vm.new_tuple(()).into())); - } - - let mut indices = zelf.indices.write(); - - let res = vm - .ctx - .new_tuple(indices.iter().map(|&i| zelf.pool[i].clone()).collect()); - - // Scan indices right-to-left until finding one that is not at its maximum (i + n - r). - let mut idx = r as isize - 1; - while idx >= 0 && indices[idx as usize] == n - 1 { - idx -= 1; - } - - // If no suitable index is found, then the indices are all at - // their maximum value and we're done. - if idx < 0 { - zelf.exhausted.store(true); - } else { - let index = indices[idx as usize] + 1; - - // Increment the current index which we know is not at its - // maximum. Then set all to the right to the same value. - for j in idx as usize..r { - indices[j] = index; - } - } - - Ok(PyIterReturn::Return(res.into())) - } - } - - #[pyattr] - #[pyclass(name = "permutations")] - #[derive(Debug, PyPayload)] - struct PyItertoolsPermutations { - pool: Vec<PyObjectRef>, // Collected input iterable - indices: PyRwLock<Vec<usize>>, // One index per element in pool - cycles: PyRwLock<Vec<usize>>, // One rollover counter per element in the result - result: PyRwLock<Option<Vec<usize>>>, // Indexes of the most recently returned result - r: AtomicCell<usize>, // Size of result tuple - exhausted: AtomicCell<bool>, // Set when the iterator is exhausted - } - - #[derive(FromArgs)] - struct PermutationsNewArgs { - #[pyarg(positional)] - iterable: PyObjectRef, - #[pyarg(positional, optional)] - r: OptionalOption<PyObjectRef>, - } - - impl Constructor for PyItertoolsPermutations { - type Args = PermutationsNewArgs; - - fn py_new( - cls: PyTypeRef, - Self::Args { iterable, r }: Self::Args, - vm: &VirtualMachine, - ) -> PyResult { - let pool: Vec<_> = iterable.try_to_value(vm)?; - - let n = pool.len(); - // If r is not provided, r == n. If provided, r must be a positive integer, or None. - // If None, it behaves the same as if it was not provided. - let r = match r.flatten() { - Some(r) => { - let val = r - .payload::<PyInt>() - .ok_or_else(|| vm.new_type_error("Expected int as r".to_owned()))? - .as_bigint(); - - if val.is_negative() { - return Err(vm.new_value_error("r must be non-negative".to_owned())); - } - val.to_usize().unwrap() - } - None => n, - }; - - Self { - pool, - indices: PyRwLock::new((0..n).collect()), - cycles: PyRwLock::new((0..r.min(n)).map(|i| n - i).collect()), - result: PyRwLock::new(None), - r: AtomicCell::new(r), - exhausted: AtomicCell::new(r > n), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsPermutations { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyRef<PyTuple> { - vm.new_tuple(( - zelf.class().to_owned(), - vm.new_tuple((zelf.pool.clone(), vm.ctx.new_int(zelf.r.load()))), - )) - } - } - - impl SelfIter for PyItertoolsPermutations {} - - impl IterNext for PyItertoolsPermutations { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - // stop signal - if zelf.exhausted.load() { - return Ok(PyIterReturn::StopIteration(None)); - } - - let n = zelf.pool.len(); - let r = zelf.r.load(); - - if n == 0 { - zelf.exhausted.store(true); - return Ok(PyIterReturn::Return(vm.new_tuple(()).into())); - } - - let mut result = zelf.result.write(); - - if let Some(ref mut result) = *result { - let mut indices = zelf.indices.write(); - let mut cycles = zelf.cycles.write(); - let mut sentinel = false; - - // Decrement rightmost cycle, moving leftward upon zero rollover - for i in (0..r).rev() { - cycles[i] -= 1; - - if cycles[i] == 0 { - // rotation: indices[i:] = indices[i+1:] + indices[i:i+1] - let index = indices[i]; - for j in i..n - 1 { - indices[j] = indices[j + 1]; - } - indices[n - 1] = index; - cycles[i] = n - i; - } else { - let j = cycles[i]; - indices.swap(i, n - j); - - for k in i..r { - // start with i, the leftmost element that changed - // yield tuple(pool[k] for k in indices[:r]) - result[k] = indices[k]; - } - sentinel = true; - break; - } - } - if !sentinel { - zelf.exhausted.store(true); - return Ok(PyIterReturn::StopIteration(None)); - } - } else { - // On the first pass, initialize result tuple using the indices - *result = Some((0..r).collect()); - } - - Ok(PyIterReturn::Return( - vm.ctx - .new_tuple( - result - .as_ref() - .unwrap() - .iter() - .map(|&i| zelf.pool[i].clone()) - .collect(), - ) - .into(), - )) - } - } - - #[derive(FromArgs)] - struct ZipLongestArgs { - #[pyarg(named, optional)] - fillvalue: OptionalArg<PyObjectRef>, - } - - impl Constructor for PyItertoolsZipLongest { - type Args = (PosArgs<PyIter>, ZipLongestArgs); - - fn py_new(cls: PyTypeRef, (iterators, args): Self::Args, vm: &VirtualMachine) -> PyResult { - let fillvalue = args.fillvalue.unwrap_or_none(vm); - let iterators = iterators.into_vec(); - PyItertoolsZipLongest { - iterators, - fillvalue: PyRwLock::new(fillvalue), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyattr] - #[pyclass(name = "zip_longest")] - #[derive(Debug, PyPayload)] - struct PyItertoolsZipLongest { - iterators: Vec<PyIter>, - fillvalue: PyRwLock<PyObjectRef>, - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsZipLongest { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - let args: Vec<PyObjectRef> = zelf - .iterators - .iter() - .map(|i| i.clone().to_pyobject(vm)) - .collect(); - Ok(vm.new_tuple(( - zelf.class().to_owned(), - vm.new_tuple(args), - zelf.fillvalue.read().to_owned(), - ))) - } - - #[pymethod(magic)] - fn setstate(zelf: PyRef<Self>, state: PyObjectRef, _vm: &VirtualMachine) -> PyResult<()> { - *zelf.fillvalue.write() = state; - Ok(()) - } - } - - impl SelfIter for PyItertoolsZipLongest {} - - impl IterNext for PyItertoolsZipLongest { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - if zelf.iterators.is_empty() { - return Ok(PyIterReturn::StopIteration(None)); - } - let mut result: Vec<PyObjectRef> = Vec::new(); - let mut numactive = zelf.iterators.len(); - - for idx in 0..zelf.iterators.len() { - let next_obj = match zelf.iterators[idx].next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => { - numactive -= 1; - if numactive == 0 { - return Ok(PyIterReturn::StopIteration(v)); - } - zelf.fillvalue.read().clone() - } - }; - result.push(next_obj); - } - Ok(PyIterReturn::Return(vm.ctx.new_tuple(result).into())) - } - } - - #[pyattr] - #[pyclass(name = "pairwise")] - #[derive(Debug, PyPayload)] - struct PyItertoolsPairwise { - iterator: PyIter, - old: PyRwLock<Option<PyObjectRef>>, - } - - impl Constructor for PyItertoolsPairwise { - type Args = PyIter; - - fn py_new(cls: PyTypeRef, iterator: Self::Args, vm: &VirtualMachine) -> PyResult { - PyItertoolsPairwise { - iterator, - old: PyRwLock::new(None), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - #[pyclass(with(IterNext, Iterable, Constructor))] - impl PyItertoolsPairwise {} - - impl SelfIter for PyItertoolsPairwise {} - - impl IterNext for PyItertoolsPairwise { - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let old = match zelf.old.read().clone() { - None => match zelf.iterator.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }, - Some(obj) => obj, - }; - let new = match zelf.iterator.next(vm)? { - PyIterReturn::Return(obj) => obj, - PyIterReturn::StopIteration(v) => return Ok(PyIterReturn::StopIteration(v)), - }; - *zelf.old.write() = Some(new.clone()); - Ok(PyIterReturn::Return(vm.new_tuple((old, new)).into())) - } - } -} diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs deleted file mode 100644 index 35f9b93d5fb..00000000000 --- a/vm/src/stdlib/marshal.rs +++ /dev/null @@ -1,233 +0,0 @@ -pub(crate) use decl::make_module; - -#[pymodule(name = "marshal")] -mod decl { - use crate::builtins::code::{CodeObject, Literal, PyObjBag}; - use crate::class::StaticType; - use crate::{ - builtins::{ - PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat, - PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple, - }, - convert::ToPyObject, - function::{ArgBytesLike, OptionalArg}, - object::AsObject, - protocol::PyBuffer, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, - }; - use malachite_bigint::BigInt; - use num_complex::Complex64; - use num_traits::Zero; - use rustpython_compiler_core::marshal; - - #[pyattr(name = "version")] - use marshal::FORMAT_VERSION; - - pub struct DumpError; - - impl marshal::Dumpable for PyObjectRef { - type Error = DumpError; - type Constant = Literal; - fn with_dump<R>( - &self, - f: impl FnOnce(marshal::DumpableValue<'_, Self>) -> R, - ) -> Result<R, Self::Error> { - use marshal::DumpableValue::*; - if self.is(PyStopIteration::static_type()) { - return Ok(f(StopIter)); - } - let ret = match_class!(match self { - PyNone => f(None), - PyEllipsis => f(Ellipsis), - ref pyint @ PyInt => { - if self.class().is(PyBool::static_type()) { - f(Boolean(!pyint.as_bigint().is_zero())) - } else { - f(Integer(pyint.as_bigint())) - } - } - ref pyfloat @ PyFloat => { - f(Float(pyfloat.to_f64())) - } - ref pycomplex @ PyComplex => { - f(Complex(pycomplex.to_complex64())) - } - ref pystr @ PyStr => { - f(Str(pystr.as_str())) - } - ref pylist @ PyList => { - f(List(&pylist.borrow_vec())) - } - ref pyset @ PySet => { - let elements = pyset.elements(); - f(Set(&elements)) - } - ref pyfrozen @ PyFrozenSet => { - let elements = pyfrozen.elements(); - f(Frozenset(&elements)) - } - ref pytuple @ PyTuple => { - f(Tuple(pytuple.as_slice())) - } - ref pydict @ PyDict => { - let entries = pydict.into_iter().collect::<Vec<_>>(); - f(Dict(&entries)) - } - ref bytes @ PyBytes => { - f(Bytes(bytes.as_bytes())) - } - ref bytes @ PyByteArray => { - f(Bytes(&bytes.borrow_buf())) - } - ref co @ PyCode => { - f(Code(co)) - } - _ => return Err(DumpError), - }); - Ok(ret) - } - } - - #[pyfunction] - fn dumps( - value: PyObjectRef, - _version: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<PyBytes> { - use marshal::Dumpable; - let mut buf = Vec::new(); - value - .with_dump(|val| marshal::serialize_value(&mut buf, val)) - .unwrap_or_else(Err) - .map_err(|DumpError| { - vm.new_not_implemented_error( - "TODO: not implemented yet or marshal unsupported type".to_owned(), - ) - })?; - Ok(PyBytes::from(buf)) - } - - #[pyfunction] - fn dump( - value: PyObjectRef, - f: PyObjectRef, - version: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let dumped = dumps(value, version, vm)?; - vm.call_method(&f, "write", (dumped,))?; - Ok(()) - } - - #[derive(Copy, Clone)] - struct PyMarshalBag<'a>(&'a VirtualMachine); - - impl<'a> marshal::MarshalBag for PyMarshalBag<'a> { - type Value = PyObjectRef; - fn make_bool(&self, value: bool) -> Self::Value { - self.0.ctx.new_bool(value).into() - } - fn make_none(&self) -> Self::Value { - self.0.ctx.none() - } - fn make_ellipsis(&self) -> Self::Value { - self.0.ctx.ellipsis() - } - fn make_float(&self, value: f64) -> Self::Value { - self.0.ctx.new_float(value).into() - } - fn make_complex(&self, value: Complex64) -> Self::Value { - self.0.ctx.new_complex(value).into() - } - fn make_str(&self, value: &str) -> Self::Value { - self.0.ctx.new_str(value).into() - } - fn make_bytes(&self, value: &[u8]) -> Self::Value { - self.0.ctx.new_bytes(value.to_vec()).into() - } - fn make_int(&self, value: BigInt) -> Self::Value { - self.0.ctx.new_int(value).into() - } - fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value { - let elements = elements.collect(); - self.0.ctx.new_tuple(elements).into() - } - fn make_code(&self, code: CodeObject) -> Self::Value { - self.0.ctx.new_code(code).into() - } - fn make_stop_iter(&self) -> Result<Self::Value, marshal::MarshalError> { - Ok(self.0.ctx.exceptions.stop_iteration.to_owned().into()) - } - fn make_list( - &self, - it: impl Iterator<Item = Self::Value>, - ) -> Result<Self::Value, marshal::MarshalError> { - Ok(self.0.ctx.new_list(it.collect()).into()) - } - fn make_set( - &self, - it: impl Iterator<Item = Self::Value>, - ) -> Result<Self::Value, marshal::MarshalError> { - let vm = self.0; - let set = PySet::new_ref(&vm.ctx); - for elem in it { - set.add(elem, vm).unwrap() - } - Ok(set.into()) - } - fn make_frozenset( - &self, - it: impl Iterator<Item = Self::Value>, - ) -> Result<Self::Value, marshal::MarshalError> { - let vm = self.0; - Ok(PyFrozenSet::from_iter(vm, it).unwrap().to_pyobject(vm)) - } - fn make_dict( - &self, - it: impl Iterator<Item = (Self::Value, Self::Value)>, - ) -> Result<Self::Value, marshal::MarshalError> { - let vm = self.0; - let dict = vm.ctx.new_dict(); - for (k, v) in it { - dict.set_item(&*k, v, vm).unwrap() - } - Ok(dict.into()) - } - type ConstantBag = PyObjBag<'a>; - fn constant_bag(self) -> Self::ConstantBag { - PyObjBag(&self.0.ctx) - } - } - - #[pyfunction] - fn loads(pybuffer: PyBuffer, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - let buf = pybuffer.as_contiguous().ok_or_else(|| { - vm.new_buffer_error("Buffer provided to marshal.loads() is not contiguous".to_owned()) - })?; - marshal::deserialize_value(&mut &buf[..], PyMarshalBag(vm)).map_err(|e| match e { - marshal::MarshalError::Eof => vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "marshal data too short".to_owned(), - ), - marshal::MarshalError::InvalidBytecode => { - vm.new_value_error("Couldn't deserialize python bytecode".to_owned()) - } - marshal::MarshalError::InvalidUtf8 => { - vm.new_value_error("invalid utf8 in marshalled string".to_owned()) - } - marshal::MarshalError::InvalidLocation => { - vm.new_value_error("invalid location in marshalled object".to_owned()) - } - marshal::MarshalError::BadType => { - vm.new_value_error("bad marshal data (unknown type code)".to_owned()) - } - }) - } - - #[pyfunction] - fn load(f: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyObjectRef> { - let read_res = vm.call_method(&f, "read", ())?; - let bytes = ArgBytesLike::try_from_object(vm, read_res)?; - loads(PyBuffer::from(bytes), vm) - } -} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs deleted file mode 100644 index 5b177b4b0b2..00000000000 --- a/vm/src/stdlib/mod.rs +++ /dev/null @@ -1,131 +0,0 @@ -#[cfg(feature = "rustpython-ast")] -pub(crate) mod ast; -pub mod atexit; -pub mod builtins; -mod codecs; -mod collections; -pub mod errno; -mod functools; -mod imp; -pub mod io; -mod itertools; -mod marshal; -mod operator; -// TODO: maybe make this an extension module, if we ever get those -// mod re; -mod sre; -mod string; -#[cfg(feature = "rustpython-compiler")] -mod symtable; -mod sysconfigdata; -#[cfg(feature = "threading")] -pub mod thread; -pub mod time; -pub mod warnings; -mod weakref; - -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] -#[macro_use] -pub mod os; -#[cfg(windows)] -pub mod nt; -#[cfg(unix)] -pub mod posix; -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] -#[cfg(not(any(unix, windows)))] -#[path = "posix_compat.rs"] -pub mod posix; - -#[cfg(windows)] -pub(crate) mod msvcrt; -#[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] -mod pwd; -#[cfg(not(target_arch = "wasm32"))] -pub(crate) mod signal; -pub mod sys; -#[cfg(windows)] -mod winapi; -#[cfg(windows)] -mod winreg; - -use crate::{builtins::PyModule, PyRef, VirtualMachine}; -use std::{borrow::Cow, collections::HashMap}; - -pub type StdlibInitFunc = Box<py_dyn_fn!(dyn Fn(&VirtualMachine) -> PyRef<PyModule>)>; -pub type StdlibMap = HashMap<Cow<'static, str>, StdlibInitFunc, ahash::RandomState>; - -pub fn get_module_inits() -> StdlibMap { - macro_rules! modules { - { - $( - #[cfg($cfg:meta)] - { $( $key:expr => $val:expr),* $(,)? } - )* - } => {{ - let modules = [ - $( - $(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)* - )* - ]; - modules.into_iter().collect() - }}; - } - modules! { - #[cfg(all())] - { - "atexit" => atexit::make_module, - "_codecs" => codecs::make_module, - "_collections" => collections::make_module, - "errno" => errno::make_module, - "_functools" => functools::make_module, - "itertools" => itertools::make_module, - "_io" => io::make_module, - "marshal" => marshal::make_module, - "_operator" => operator::make_module, - "_sre" => sre::make_module, - "_string" => string::make_module, - "time" => time::make_module, - "_weakref" => weakref::make_module, - "_imp" => imp::make_module, - "_warnings" => warnings::make_module, - sys::sysconfigdata_name() => sysconfigdata::make_module, - } - // parser related modules: - #[cfg(feature = "rustpython-ast")] - { - "_ast" => ast::make_module, - } - // compiler related modules: - #[cfg(feature = "rustpython-compiler")] - { - "symtable" => symtable::make_module, - } - #[cfg(any(unix, target_os = "wasi"))] - { - "posix" => posix::make_module, - // "fcntl" => fcntl::make_module, - } - // disable some modules on WASM - #[cfg(not(target_arch = "wasm32"))] - { - "_signal" => signal::make_module, - } - #[cfg(feature = "threading")] - { - "_thread" => thread::make_module, - } - // Unix-only - #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] - { - "pwd" => pwd::make_module, - } - // Windows-only - #[cfg(windows)] - { - "nt" => nt::make_module, - "msvcrt" => msvcrt::make_module, - "_winapi" => winapi::make_module, - "winreg" => winreg::make_module, - } - } -} diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs deleted file mode 100644 index 03ddb44f222..00000000000 --- a/vm/src/stdlib/msvcrt.rs +++ /dev/null @@ -1,117 +0,0 @@ -pub use msvcrt::*; - -#[pymodule] -mod msvcrt { - use crate::{ - builtins::{PyBytes, PyStrRef}, - common::suppress_iph, - stdlib::os::errno_err, - PyRef, PyResult, VirtualMachine, - }; - use itertools::Itertools; - use windows_sys::Win32::{ - Foundation::{HANDLE, INVALID_HANDLE_VALUE}, - System::Diagnostics::Debug, - }; - - #[pyattr] - use windows_sys::Win32::System::Diagnostics::Debug::{ - SEM_FAILCRITICALERRORS, SEM_NOALIGNMENTFAULTEXCEPT, SEM_NOGPFAULTERRORBOX, - SEM_NOOPENFILEERRORBOX, - }; - - pub fn setmode_binary(fd: i32) { - unsafe { suppress_iph!(_setmode(fd, libc::O_BINARY)) }; - } - - extern "C" { - fn _getch() -> i32; - fn _getwch() -> u32; - fn _getche() -> i32; - fn _getwche() -> u32; - fn _putch(c: u32) -> i32; - fn _putwch(c: u16) -> u32; - } - - #[pyfunction] - fn getch() -> Vec<u8> { - let c = unsafe { _getch() }; - vec![c as u8] - } - #[pyfunction] - fn getwch() -> String { - let c = unsafe { _getwch() }; - std::char::from_u32(c).unwrap().to_string() - } - #[pyfunction] - fn getche() -> Vec<u8> { - let c = unsafe { _getche() }; - vec![c as u8] - } - #[pyfunction] - fn getwche() -> String { - let c = unsafe { _getwche() }; - std::char::from_u32(c).unwrap().to_string() - } - #[pyfunction] - fn putch(b: PyRef<PyBytes>, vm: &VirtualMachine) -> PyResult<()> { - let &c = b.as_bytes().iter().exactly_one().map_err(|_| { - vm.new_type_error("putch() argument must be a byte string of length 1".to_owned()) - })?; - unsafe { suppress_iph!(_putch(c.into())) }; - Ok(()) - } - #[pyfunction] - fn putwch(s: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - let c = s.as_str().chars().exactly_one().map_err(|_| { - vm.new_type_error("putch() argument must be a string of length 1".to_owned()) - })?; - unsafe { suppress_iph!(_putwch(c as u16)) }; - Ok(()) - } - - extern "C" { - fn _setmode(fd: i32, flags: i32) -> i32; - } - - #[pyfunction] - fn setmode(fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult<i32> { - let flags = unsafe { suppress_iph!(_setmode(fd, flags)) }; - if flags == -1 { - Err(errno_err(vm)) - } else { - Ok(flags) - } - } - - extern "C" { - fn _open_osfhandle(osfhandle: isize, flags: i32) -> i32; - fn _get_osfhandle(fd: i32) -> libc::intptr_t; - } - - #[pyfunction] - fn open_osfhandle(handle: isize, flags: i32, vm: &VirtualMachine) -> PyResult<i32> { - let ret = unsafe { suppress_iph!(_open_osfhandle(handle, flags)) }; - if ret == -1 { - Err(errno_err(vm)) - } else { - Ok(ret) - } - } - - #[pyfunction] - fn get_osfhandle(fd: i32, vm: &VirtualMachine) -> PyResult<isize> { - let ret = unsafe { suppress_iph!(_get_osfhandle(fd)) }; - if ret as HANDLE == INVALID_HANDLE_VALUE { - Err(errno_err(vm)) - } else { - Ok(ret) - } - } - - #[allow(non_snake_case)] - #[pyfunction] - fn SetErrorMode(mode: Debug::THREAD_ERROR_MODE, _: &VirtualMachine) -> u32 { - unsafe { suppress_iph!(Debug::SetErrorMode(mode)) } - } -} diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs deleted file mode 100644 index 206faa82e1a..00000000000 --- a/vm/src/stdlib/nt.rs +++ /dev/null @@ -1,432 +0,0 @@ -use crate::{builtins::PyModule, PyRef, VirtualMachine}; - -pub use module::raw_set_handle_inheritable; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} - -#[pymodule(name = "nt", with(super::os::_os))] -pub(crate) mod module { - #[cfg(target_env = "msvc")] - use crate::builtins::PyListRef; - use crate::{ - builtins::{PyDictRef, PyStrRef, PyTupleRef}, - common::{crt_fd::Fd, os::errno, suppress_iph}, - convert::ToPyException, - function::Either, - function::OptionalArg, - stdlib::os::{ - errno_err, DirFd, FollowSymlinks, OsPath, SupportFunc, TargetIsDirectory, _os, - }, - PyResult, TryFromObject, VirtualMachine, - }; - use libc::intptr_t; - use std::{ - env, fs, io, - mem::MaybeUninit, - os::windows::ffi::{OsStrExt, OsStringExt}, - }; - use windows_sys::Win32::{ - Foundation::{self, INVALID_HANDLE_VALUE}, - Storage::FileSystem, - System::{Console, Threading}, - }; - - #[pyattr] - use libc::{O_BINARY, O_TEMPORARY}; - - #[pyfunction] - pub(super) fn access(path: OsPath, mode: u8, vm: &VirtualMachine) -> PyResult<bool> { - let attr = unsafe { FileSystem::GetFileAttributesW(path.to_widecstring(vm)?.as_ptr()) }; - Ok(attr != FileSystem::INVALID_FILE_ATTRIBUTES - && (mode & 2 == 0 - || attr & FileSystem::FILE_ATTRIBUTE_READONLY == 0 - || attr & FileSystem::FILE_ATTRIBUTE_DIRECTORY != 0)) - } - - #[derive(FromArgs)] - pub(super) struct SymlinkArgs { - src: OsPath, - dst: OsPath, - #[pyarg(flatten)] - target_is_directory: TargetIsDirectory, - #[pyarg(flatten)] - _dir_fd: DirFd<{ _os::SYMLINK_DIR_FD as usize }>, - } - - #[pyfunction] - pub(super) fn symlink(args: SymlinkArgs, vm: &VirtualMachine) -> PyResult<()> { - use std::os::windows::fs as win_fs; - let dir = args.target_is_directory.target_is_directory - || args - .dst - .as_path() - .parent() - .and_then(|dst_parent| dst_parent.join(&args.src).symlink_metadata().ok()) - .map_or(false, |meta| meta.is_dir()); - let res = if dir { - win_fs::symlink_dir(args.src.path, args.dst.path) - } else { - win_fs::symlink_file(args.src.path, args.dst.path) - }; - res.map_err(|err| err.to_pyexception(vm)) - } - - #[pyfunction] - fn set_inheritable(fd: i32, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { - let handle = Fd(fd).to_raw_handle().map_err(|e| e.to_pyexception(vm))?; - set_handle_inheritable(handle as _, inheritable, vm) - } - - #[pyattr] - fn environ(vm: &VirtualMachine) -> PyDictRef { - let environ = vm.ctx.new_dict(); - - for (key, value) in env::vars() { - environ.set_item(&key, vm.new_pyobj(value), vm).unwrap(); - } - environ - } - - #[pyfunction] - fn chmod( - path: OsPath, - dir_fd: DirFd<0>, - mode: u32, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult<()> { - const S_IWRITE: u32 = 128; - let [] = dir_fd.0; - let metadata = if follow_symlinks.0 { - fs::metadata(&path) - } else { - fs::symlink_metadata(&path) - }; - let meta = metadata.map_err(|err| err.to_pyexception(vm))?; - let mut permissions = meta.permissions(); - permissions.set_readonly(mode & S_IWRITE == 0); - fs::set_permissions(&path, permissions).map_err(|err| err.to_pyexception(vm)) - } - - // cwait is available on MSVC only (according to CPython) - #[cfg(target_env = "msvc")] - extern "C" { - fn _cwait(termstat: *mut i32, procHandle: intptr_t, action: i32) -> intptr_t; - } - - #[cfg(target_env = "msvc")] - #[pyfunction] - fn waitpid(pid: intptr_t, opt: i32, vm: &VirtualMachine) -> PyResult<(intptr_t, i32)> { - let mut status = 0; - let pid = unsafe { suppress_iph!(_cwait(&mut status, pid, opt)) }; - if pid == -1 { - Err(errno_err(vm)) - } else { - Ok((pid, status << 8)) - } - } - - #[cfg(target_env = "msvc")] - #[pyfunction] - fn wait(vm: &VirtualMachine) -> PyResult<(intptr_t, i32)> { - waitpid(-1, 0, vm) - } - - #[pyfunction] - fn kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { - let sig = sig as u32; - let pid = pid as u32; - - if sig == Console::CTRL_C_EVENT || sig == Console::CTRL_BREAK_EVENT { - let ret = unsafe { Console::GenerateConsoleCtrlEvent(sig, pid) }; - let res = if ret == 0 { Err(errno_err(vm)) } else { Ok(()) }; - return res; - } - - let h = unsafe { Threading::OpenProcess(Threading::PROCESS_ALL_ACCESS, 0, pid) }; - if h == 0 { - return Err(errno_err(vm)); - } - let ret = unsafe { Threading::TerminateProcess(h, sig) }; - let res = if ret == 0 { Err(errno_err(vm)) } else { Ok(()) }; - unsafe { Foundation::CloseHandle(h) }; - res - } - - #[pyfunction] - fn get_terminal_size( - fd: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<_os::PyTerminalSize> { - let (columns, lines) = { - let stdhandle = match fd { - OptionalArg::Present(0) => Console::STD_INPUT_HANDLE, - OptionalArg::Present(1) | OptionalArg::Missing => Console::STD_OUTPUT_HANDLE, - OptionalArg::Present(2) => Console::STD_ERROR_HANDLE, - _ => return Err(vm.new_value_error("bad file descriptor".to_owned())), - }; - let h = unsafe { Console::GetStdHandle(stdhandle) }; - if h == 0 { - return Err(vm.new_os_error("handle cannot be retrieved".to_owned())); - } - if h == INVALID_HANDLE_VALUE { - return Err(errno_err(vm)); - } - let mut csbi = MaybeUninit::uninit(); - let ret = unsafe { Console::GetConsoleScreenBufferInfo(h, csbi.as_mut_ptr()) }; - let csbi = unsafe { csbi.assume_init() }; - if ret == 0 { - return Err(errno_err(vm)); - } - let w = csbi.srWindow; - ( - (w.Right - w.Left + 1) as usize, - (w.Bottom - w.Top + 1) as usize, - ) - }; - Ok(_os::PyTerminalSize { columns, lines }) - } - - #[cfg(target_env = "msvc")] - extern "C" { - fn _wexecv(cmdname: *const u16, argv: *const *const u16) -> intptr_t; - } - - #[cfg(target_env = "msvc")] - #[pyfunction] - fn execv( - path: PyStrRef, - argv: Either<PyListRef, PyTupleRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - use std::iter::once; - - let make_widestring = - |s: &str| widestring::WideCString::from_os_str(s).map_err(|err| err.to_pyexception(vm)); - - let path = make_widestring(path.as_str())?; - - let argv = vm.extract_elements_with(argv.as_ref(), |obj| { - let arg = PyStrRef::try_from_object(vm, obj)?; - make_widestring(arg.as_str()) - })?; - - let first = argv - .first() - .ok_or_else(|| vm.new_value_error("execv() arg 2 must not be empty".to_owned()))?; - - if first.is_empty() { - return Err( - vm.new_value_error("execv() arg 2 first element cannot be empty".to_owned()) - ); - } - - let argv_execv: Vec<*const u16> = argv - .iter() - .map(|v| v.as_ptr()) - .chain(once(std::ptr::null())) - .collect(); - - if (unsafe { suppress_iph!(_wexecv(path.as_ptr(), argv_execv.as_ptr())) } == -1) { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - - #[pyfunction] - fn _getfinalpathname(path: OsPath, vm: &VirtualMachine) -> PyResult { - let real = path - .as_ref() - .canonicalize() - .map_err(|e| e.to_pyexception(vm))?; - path.mode.process_path(real, vm) - } - - #[pyfunction] - fn _getfullpathname(path: OsPath, vm: &VirtualMachine) -> PyResult { - let wpath = path.to_widecstring(vm)?; - let mut buffer = vec![0u16; Foundation::MAX_PATH as usize]; - let ret = unsafe { - FileSystem::GetFullPathNameW( - wpath.as_ptr(), - buffer.len() as _, - buffer.as_mut_ptr(), - std::ptr::null_mut(), - ) - }; - if ret == 0 { - return Err(errno_err(vm)); - } - if ret as usize > buffer.len() { - buffer.resize(ret as usize, 0); - let ret = unsafe { - FileSystem::GetFullPathNameW( - wpath.as_ptr(), - buffer.len() as _, - buffer.as_mut_ptr(), - std::ptr::null_mut(), - ) - }; - if ret == 0 { - return Err(errno_err(vm)); - } - } - let buffer = widestring::WideCString::from_vec_truncate(buffer); - path.mode.process_path(buffer.to_os_string(), vm) - } - - #[pyfunction] - fn _getvolumepathname(path: OsPath, vm: &VirtualMachine) -> PyResult { - let wide = path.to_widecstring(vm)?; - let buflen = std::cmp::max(wide.len(), Foundation::MAX_PATH as usize); - let mut buffer = vec![0u16; buflen]; - let ret = unsafe { - FileSystem::GetVolumePathNameW(wide.as_ptr(), buffer.as_mut_ptr(), buflen as _) - }; - if ret == 0 { - return Err(errno_err(vm)); - } - let buffer = widestring::WideCString::from_vec_truncate(buffer); - path.mode.process_path(buffer.to_os_string(), vm) - } - - #[pyfunction] - fn _path_splitroot(path: OsPath, vm: &VirtualMachine) -> PyResult<(String, String)> { - let orig: Vec<_> = path.path.encode_wide().collect(); - if orig.is_empty() { - return Ok(("".to_owned(), "".to_owned())); - } - let backslashed: Vec<_> = orig - .iter() - .copied() - .map(|c| if c == b'/' as u16 { b'\\' as u16 } else { c }) - .chain(std::iter::once(0)) // null-terminated - .collect(); - - fn from_utf16(wstr: &[u16], vm: &VirtualMachine) -> PyResult<String> { - String::from_utf16(wstr).map_err(|e| vm.new_unicode_decode_error(e.to_string())) - } - - let wbuf = windows::core::PCWSTR::from_raw(backslashed.as_ptr()); - let (root, path) = match unsafe { windows::Win32::UI::Shell::PathCchSkipRoot(wbuf) } { - Ok(end) => { - assert!(!end.is_null()); - let len: usize = unsafe { end.as_ptr().offset_from(wbuf.as_ptr()) } - .try_into() - .expect("len must be non-negative"); - assert!( - len < backslashed.len(), // backslashed is null-terminated - "path: {:?} {} < {}", - std::path::PathBuf::from(std::ffi::OsString::from_wide(&backslashed)), - len, - backslashed.len() - ); - (from_utf16(&orig[..len], vm)?, from_utf16(&orig[len..], vm)?) - } - Err(_) => ("".to_owned(), from_utf16(&orig, vm)?), - }; - Ok((root, path)) - } - - #[pyfunction] - fn _getdiskusage(path: OsPath, vm: &VirtualMachine) -> PyResult<(u64, u64)> { - use FileSystem::GetDiskFreeSpaceExW; - - let wpath = path.to_widecstring(vm)?; - let mut _free_to_me: u64 = 0; - let mut total: u64 = 0; - let mut free: u64 = 0; - let ret = - unsafe { GetDiskFreeSpaceExW(wpath.as_ptr(), &mut _free_to_me, &mut total, &mut free) }; - if ret != 0 { - return Ok((total, free)); - } - let err = io::Error::last_os_error(); - if err.raw_os_error() == Some(Foundation::ERROR_DIRECTORY as i32) { - if let Some(parent) = path.as_ref().parent() { - let parent = widestring::WideCString::from_os_str(parent).unwrap(); - - let ret = unsafe { - GetDiskFreeSpaceExW(parent.as_ptr(), &mut _free_to_me, &mut total, &mut free) - }; - - return if ret == 0 { - Err(errno_err(vm)) - } else { - Ok((total, free)) - }; - } - } - Err(err.to_pyexception(vm)) - } - - #[pyfunction] - fn get_handle_inheritable(handle: intptr_t, vm: &VirtualMachine) -> PyResult<bool> { - let mut flags = 0; - if unsafe { Foundation::GetHandleInformation(handle as _, &mut flags) } == 0 { - Err(errno_err(vm)) - } else { - Ok(flags & Foundation::HANDLE_FLAG_INHERIT != 0) - } - } - - pub fn raw_set_handle_inheritable(handle: intptr_t, inheritable: bool) -> io::Result<()> { - let flags = if inheritable { - Foundation::HANDLE_FLAG_INHERIT - } else { - 0 - }; - let res = unsafe { - Foundation::SetHandleInformation(handle as _, Foundation::HANDLE_FLAG_INHERIT, flags) - }; - if res == 0 { - Err(errno()) - } else { - Ok(()) - } - } - - #[pyfunction] - fn set_handle_inheritable( - handle: intptr_t, - inheritable: bool, - vm: &VirtualMachine, - ) -> PyResult<()> { - raw_set_handle_inheritable(handle, inheritable).map_err(|e| e.to_pyexception(vm)) - } - - #[pyfunction] - fn mkdir( - path: OsPath, - mode: OptionalArg<i32>, - dir_fd: DirFd<{ _os::MKDIR_DIR_FD as usize }>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let mode = mode.unwrap_or(0o777); - let [] = dir_fd.0; - let _ = mode; - let wide = path.to_widecstring(vm)?; - let res = unsafe { FileSystem::CreateDirectoryW(wide.as_ptr(), std::ptr::null_mut()) }; - if res == 0 { - return Err(errno_err(vm)); - } - Ok(()) - } - - pub(crate) fn support_funcs() -> Vec<SupportFunc> { - Vec::new() - } -} - -pub fn init_winsock() { - static WSA_INIT: parking_lot::Once = parking_lot::Once::new(); - WSA_INIT.call_once(|| unsafe { - let mut wsa_data = std::mem::MaybeUninit::uninit(); - let _ = windows_sys::Win32::Networking::WinSock::WSAStartup(0x0101, wsa_data.as_mut_ptr()); - }) -} diff --git a/vm/src/stdlib/operator.rs b/vm/src/stdlib/operator.rs deleted file mode 100644 index 6035e002aa2..00000000000 --- a/vm/src/stdlib/operator.rs +++ /dev/null @@ -1,605 +0,0 @@ -pub(crate) use _operator::make_module; - -#[pymodule] -mod _operator { - use crate::common::cmp; - use crate::{ - builtins::{PyInt, PyIntRef, PyStr, PyStrRef, PyTupleRef, PyTypeRef}, - function::Either, - function::{ArgBytesLike, FuncArgs, KwArgs, OptionalArg}, - identifier, - protocol::PyIter, - recursion::ReprGuard, - types::{Callable, Constructor, PyComparisonOp, Representable}, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - }; - - #[pyfunction] - fn lt(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - a.rich_compare(b, PyComparisonOp::Lt, vm) - } - - #[pyfunction] - fn le(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - a.rich_compare(b, PyComparisonOp::Le, vm) - } - - #[pyfunction] - fn gt(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - a.rich_compare(b, PyComparisonOp::Gt, vm) - } - - #[pyfunction] - fn ge(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - a.rich_compare(b, PyComparisonOp::Ge, vm) - } - - #[pyfunction] - fn eq(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - a.rich_compare(b, PyComparisonOp::Eq, vm) - } - - #[pyfunction] - fn ne(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - a.rich_compare(b, PyComparisonOp::Ne, vm) - } - - #[pyfunction] - fn not_(a: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - a.try_to_bool(vm).map(|r| !r) - } - - #[pyfunction] - fn truth(a: PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - a.try_to_bool(vm) - } - - #[pyfunction] - fn is_(a: PyObjectRef, b: PyObjectRef) -> PyResult<bool> { - Ok(a.is(&b)) - } - - #[pyfunction] - fn is_not(a: PyObjectRef, b: PyObjectRef) -> PyResult<bool> { - Ok(!a.is(&b)) - } - - #[pyfunction] - fn abs(a: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._abs(&a) - } - - #[pyfunction] - fn add(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._add(&a, &b) - } - - #[pyfunction] - fn and_(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._and(&a, &b) - } - - #[pyfunction] - fn floordiv(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._floordiv(&a, &b) - } - - // Note: Keep track of issue17567. Will need changes in order to strictly match behavior of - // a.__index__ as raised in the issue. Currently, we accept int subclasses. - #[pyfunction] - fn index(a: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyIntRef> { - a.try_index(vm) - } - - #[pyfunction] - fn invert(pos: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._invert(&pos) - } - - #[pyfunction] - fn lshift(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._lshift(&a, &b) - } - - #[pyfunction(name = "mod")] - fn mod_(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._mod(&a, &b) - } - - #[pyfunction] - fn mul(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._mul(&a, &b) - } - - #[pyfunction] - fn matmul(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._matmul(&a, &b) - } - - #[pyfunction] - fn neg(pos: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._neg(&pos) - } - - #[pyfunction] - fn or_(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._or(&a, &b) - } - - #[pyfunction] - fn pos(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._pos(&obj) - } - - #[pyfunction] - fn pow(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._pow(&a, &b, vm.ctx.none.as_object()) - } - - #[pyfunction] - fn rshift(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._rshift(&a, &b) - } - - #[pyfunction] - fn sub(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._sub(&a, &b) - } - - #[pyfunction] - fn truediv(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._truediv(&a, &b) - } - - #[pyfunction] - fn xor(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._xor(&a, &b) - } - - // Sequence based operators - - #[pyfunction] - fn concat(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // Best attempt at checking that a is sequence-like. - if !a.class().has_attr(identifier!(vm, __getitem__)) - || a.fast_isinstance(vm.ctx.types.dict_type) - { - return Err( - vm.new_type_error(format!("{} object can't be concatenated", a.class().name())) - ); - } - vm._add(&a, &b) - } - - #[pyfunction] - fn contains(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._contains(&a, b) - } - - #[pyfunction(name = "countOf")] - fn count_of(a: PyIter, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - let mut count: usize = 0; - for element in a.iter_without_hint::<PyObjectRef>(vm)? { - let element = element?; - if element.is(&b) || vm.bool_eq(&b, &element)? { - count += 1; - } - } - Ok(count) - } - - #[pyfunction] - fn delitem(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - a.del_item(&*b, vm) - } - - #[pyfunction] - fn getitem(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - a.get_item(&*b, vm) - } - - #[pyfunction(name = "indexOf")] - fn index_of(a: PyIter, b: PyObjectRef, vm: &VirtualMachine) -> PyResult<usize> { - for (index, element) in a.iter_without_hint::<PyObjectRef>(vm)?.enumerate() { - let element = element?; - if element.is(&b) || vm.bool_eq(&b, &element)? { - return Ok(index); - } - } - Err(vm.new_value_error("sequence.index(x): x not in sequence".to_owned())) - } - - #[pyfunction] - fn setitem( - a: PyObjectRef, - b: PyObjectRef, - c: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - a.set_item(&*b, c, vm) - } - - #[pyfunction] - fn length_hint(obj: PyObjectRef, default: OptionalArg, vm: &VirtualMachine) -> PyResult<usize> { - let default: usize = default - .map(|v| { - if !v.fast_isinstance(vm.ctx.types.int_type) { - return Err(vm.new_type_error(format!( - "'{}' type cannot be interpreted as an integer", - v.class().name() - ))); - } - v.payload::<PyInt>().unwrap().try_to_primitive(vm) - }) - .unwrap_or(Ok(0))?; - obj.length_hint(default, vm) - } - - // Inplace Operators - - #[pyfunction] - fn iadd(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._iadd(&a, &b) - } - - #[pyfunction] - fn iand(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._iand(&a, &b) - } - - #[pyfunction] - fn iconcat(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - // Best attempt at checking that a is sequence-like. - if !a.class().has_attr(identifier!(vm, __getitem__)) - || a.fast_isinstance(vm.ctx.types.dict_type) - { - return Err( - vm.new_type_error(format!("{} object can't be concatenated", a.class().name())) - ); - } - vm._iadd(&a, &b) - } - - #[pyfunction] - fn ifloordiv(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._ifloordiv(&a, &b) - } - - #[pyfunction] - fn ilshift(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._ilshift(&a, &b) - } - - #[pyfunction] - fn imod(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._imod(&a, &b) - } - - #[pyfunction] - fn imul(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._imul(&a, &b) - } - - #[pyfunction] - fn imatmul(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._imatmul(&a, &b) - } - - #[pyfunction] - fn ior(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._ior(&a, &b) - } - - #[pyfunction] - fn ipow(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._ipow(&a, &b, vm.ctx.none.as_object()) - } - - #[pyfunction] - fn irshift(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._irshift(&a, &b) - } - - #[pyfunction] - fn isub(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._isub(&a, &b) - } - - #[pyfunction] - fn itruediv(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._itruediv(&a, &b) - } - - #[pyfunction] - fn ixor(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm._ixor(&a, &b) - } - - #[pyfunction] - fn _compare_digest( - a: Either<PyStrRef, ArgBytesLike>, - b: Either<PyStrRef, ArgBytesLike>, - vm: &VirtualMachine, - ) -> PyResult<bool> { - let res = match (a, b) { - (Either::A(a), Either::A(b)) => { - if !a.as_str().is_ascii() || !b.as_str().is_ascii() { - return Err(vm.new_type_error( - "comparing strings with non-ASCII characters is not supported".to_owned(), - )); - } - cmp::timing_safe_cmp(a.as_str().as_bytes(), b.as_str().as_bytes()) - } - (Either::B(a), Either::B(b)) => { - a.with_ref(|a| b.with_ref(|b| cmp::timing_safe_cmp(a, b))) - } - _ => { - return Err(vm.new_type_error( - "unsupported operand types(s) or combination of types".to_owned(), - )) - } - }; - Ok(res) - } - - /// attrgetter(attr, ...) --> attrgetter object - /// - /// Return a callable object that fetches the given attribute(s) from its operand. - /// After f = attrgetter('name'), the call f(r) returns r.name. - /// After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). - /// After h = attrgetter('name.first', 'name.last'), the call h(r) returns - /// (r.name.first, r.name.last). - #[pyattr] - #[pyclass(name = "attrgetter")] - #[derive(Debug, PyPayload)] - struct PyAttrGetter { - attrs: Vec<PyStrRef>, - } - - #[pyclass(with(Callable, Constructor, Representable))] - impl PyAttrGetter { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<(PyTypeRef, PyTupleRef)> { - let attrs = vm - .ctx - .new_tuple(zelf.attrs.iter().map(|v| v.clone().into()).collect()); - Ok((zelf.class().to_owned(), attrs)) - } - - // Go through dotted parts of string and call getattr on whatever is returned. - fn get_single_attr( - obj: PyObjectRef, - attr: &Py<PyStr>, - vm: &VirtualMachine, - ) -> PyResult<PyObjectRef> { - let attr_str = attr.as_str(); - let parts = attr_str.split('.').collect::<Vec<_>>(); - if parts.len() == 1 { - return obj.get_attr(attr, vm); - } - let mut obj = obj; - for part in parts { - obj = obj.get_attr(&vm.ctx.new_str(part), vm)?; - } - Ok(obj) - } - } - - impl Constructor for PyAttrGetter { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - let nattr = args.args.len(); - // Check we get no keyword and at least one positional. - if !args.kwargs.is_empty() { - return Err(vm.new_type_error("attrgetter() takes no keyword arguments".to_owned())); - } - if nattr == 0 { - return Err(vm.new_type_error("attrgetter expected 1 argument, got 0.".to_owned())); - } - let mut attrs = Vec::with_capacity(nattr); - for o in args.args { - if let Ok(r) = o.try_into_value(vm) { - attrs.push(r); - } else { - return Err(vm.new_type_error("attribute name must be a string".to_owned())); - } - } - PyAttrGetter { attrs } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl Callable for PyAttrGetter { - type Args = PyObjectRef; - fn call(zelf: &Py<Self>, obj: Self::Args, vm: &VirtualMachine) -> PyResult { - // Handle case where we only have one attribute. - if zelf.attrs.len() == 1 { - return Self::get_single_attr(obj, &zelf.attrs[0], vm); - } - // Build tuple and call get_single on each element in attrs. - let mut results = Vec::with_capacity(zelf.attrs.len()); - for o in &zelf.attrs { - results.push(Self::get_single_attr(obj.clone(), o, vm)?); - } - Ok(vm.ctx.new_tuple(results).into()) - } - } - - impl Representable for PyAttrGetter { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let fmt = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let mut parts = Vec::with_capacity(zelf.attrs.len()); - for part in &zelf.attrs { - parts.push(part.as_object().repr(vm)?.as_str().to_owned()); - } - parts.join(", ") - } else { - "...".to_owned() - }; - Ok(format!("operator.attrgetter({fmt})")) - } - } - - /// itemgetter(item, ...) --> itemgetter object - /// - /// Return a callable object that fetches the given item(s) from its operand. - /// After f = itemgetter(2), the call f(r) returns r[2]. - /// After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) - #[pyattr] - #[pyclass(name = "itemgetter")] - #[derive(Debug, PyPayload)] - struct PyItemGetter { - items: Vec<PyObjectRef>, - } - - #[pyclass(with(Callable, Constructor, Representable))] - impl PyItemGetter { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyObjectRef { - let items = vm.ctx.new_tuple(zelf.items.to_vec()); - vm.new_pyobj((zelf.class().to_owned(), items)) - } - } - impl Constructor for PyItemGetter { - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { - // Check we get no keyword and at least one positional. - if !args.kwargs.is_empty() { - return Err(vm.new_type_error("itemgetter() takes no keyword arguments".to_owned())); - } - if args.args.is_empty() { - return Err(vm.new_type_error("itemgetter expected 1 argument, got 0.".to_owned())); - } - PyItemGetter { items: args.args } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl Callable for PyItemGetter { - type Args = PyObjectRef; - fn call(zelf: &Py<Self>, obj: Self::Args, vm: &VirtualMachine) -> PyResult { - // Handle case where we only have one attribute. - if zelf.items.len() == 1 { - return obj.get_item(&*zelf.items[0], vm); - } - // Build tuple and call get_single on each element in attrs. - let mut results = Vec::with_capacity(zelf.items.len()); - for item in &zelf.items { - results.push(obj.get_item(&**item, vm)?); - } - Ok(vm.ctx.new_tuple(results).into()) - } - } - - impl Representable for PyItemGetter { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let fmt = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let mut items = Vec::with_capacity(zelf.items.len()); - for item in &zelf.items { - items.push(item.repr(vm)?.as_str().to_owned()); - } - items.join(", ") - } else { - "...".to_owned() - }; - Ok(format!("operator.itemgetter({fmt})")) - } - } - - /// methodcaller(name, ...) --> methodcaller object - /// - /// Return a callable object that calls the given method on its operand. - /// After f = methodcaller('name'), the call f(r) returns r.name(). - /// After g = methodcaller('name', 'date', foo=1), the call g(r) returns - /// r.name('date', foo=1). - #[pyattr] - #[pyclass(name = "methodcaller")] - #[derive(Debug, PyPayload)] - struct PyMethodCaller { - name: PyStrRef, - args: FuncArgs, - } - - #[pyclass(with(Callable, Constructor, Representable))] - impl PyMethodCaller { - #[pymethod(magic)] - fn reduce(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyTupleRef> { - // With no kwargs, return (type(obj), (name, *args)) tuple. - if zelf.args.kwargs.is_empty() { - let mut pargs = vec![zelf.name.as_object().to_owned()]; - pargs.append(&mut zelf.args.args.clone()); - Ok(vm.new_tuple((zelf.class().to_owned(), vm.ctx.new_tuple(pargs)))) - } else { - // If we have kwargs, create a partial function that contains them and pass back that - // along with the args. - let partial = vm.import("functools", None, 0)?.get_attr("partial", vm)?; - let args = FuncArgs::new( - vec![zelf.class().to_owned().into(), zelf.name.clone().into()], - KwArgs::new(zelf.args.kwargs.clone()), - ); - let callable = partial.call(args, vm)?; - Ok(vm.new_tuple((callable, vm.ctx.new_tuple(zelf.args.args.clone())))) - } - } - } - - impl Constructor for PyMethodCaller { - type Args = (PyObjectRef, FuncArgs); - - fn py_new(cls: PyTypeRef, (name, args): Self::Args, vm: &VirtualMachine) -> PyResult { - if let Ok(name) = name.try_into_value(vm) { - PyMethodCaller { name, args } - .into_ref_with_type(vm, cls) - .map(Into::into) - } else { - Err(vm.new_type_error("method name must be a string".to_owned())) - } - } - } - - impl Callable for PyMethodCaller { - type Args = PyObjectRef; - - #[inline] - fn call(zelf: &Py<Self>, obj: Self::Args, vm: &VirtualMachine) -> PyResult { - vm.call_method(&obj, zelf.name.as_str(), zelf.args.clone()) - } - } - - impl Representable for PyMethodCaller { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let fmt = if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let args = &zelf.args.args; - let kwargs = &zelf.args.kwargs; - let mut fmt = vec![zelf.name.as_object().repr(vm)?.as_str().to_owned()]; - if !args.is_empty() { - let mut parts = Vec::with_capacity(args.len()); - for v in args { - parts.push(v.repr(vm)?.as_str().to_owned()); - } - fmt.push(parts.join(", ")); - } - // build name=value pairs from KwArgs. - if !kwargs.is_empty() { - let mut parts = Vec::with_capacity(kwargs.len()); - for (key, value) in kwargs { - let value_repr = value.repr(vm)?; - parts.push(format!("{key}={value_repr}")); - } - fmt.push(parts.join(", ")); - } - fmt.join(", ") - } else { - "...".to_owned() - }; - Ok(format!("operator.methodcaller({fmt})")) - } - } -} diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs deleted file mode 100644 index 376c18fb3ad..00000000000 --- a/vm/src/stdlib/os.rs +++ /dev/null @@ -1,1747 +0,0 @@ -use crate::{ - builtins::{PyBaseExceptionRef, PyModule, PySet}, - common::crt_fd::Fd, - convert::IntoPyException, - function::{ArgumentError, FromArgs, FsPath, FuncArgs}, - AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, -}; - -use std::{ - ffi, fs, io, - path::{Path, PathBuf}, -}; - -#[derive(Debug, Copy, Clone)] -pub(super) enum OutputMode { - String, - Bytes, -} - -impl OutputMode { - pub(super) fn process_path(self, path: impl Into<PathBuf>, vm: &VirtualMachine) -> PyResult { - fn inner(mode: OutputMode, path: PathBuf, vm: &VirtualMachine) -> PyResult { - let path_as_string = |p: PathBuf| { - p.into_os_string().into_string().map_err(|_| { - vm.new_unicode_decode_error( - "Can't convert OS path to valid UTF-8 string".into(), - ) - }) - }; - match mode { - OutputMode::String => path_as_string(path).map(|s| vm.ctx.new_str(s).into()), - OutputMode::Bytes => { - #[cfg(any(unix, target_os = "wasi"))] - { - use rustpython_common::os::ffi::OsStringExt; - Ok(vm.ctx.new_bytes(path.into_os_string().into_vec()).into()) - } - #[cfg(windows)] - { - path_as_string(path).map(|s| vm.ctx.new_bytes(s.into_bytes()).into()) - } - } - } - } - inner(self, path.into(), vm) - } -} - -// path_ without allow_fd in CPython -#[derive(Clone)] -pub struct OsPath { - pub path: ffi::OsString, - pub(super) mode: OutputMode, -} - -impl OsPath { - pub fn new_str(path: impl Into<ffi::OsString>) -> Self { - let path = path.into(); - Self { - path, - mode: OutputMode::String, - } - } - - pub(crate) fn from_fspath(fspath: FsPath, vm: &VirtualMachine) -> PyResult<OsPath> { - let path = fspath.as_os_str(vm)?.to_owned(); - let mode = match fspath { - FsPath::Str(_) => OutputMode::String, - FsPath::Bytes(_) => OutputMode::Bytes, - }; - Ok(OsPath { path, mode }) - } - - pub fn as_path(&self) -> &Path { - Path::new(&self.path) - } - - pub fn into_bytes(self) -> Vec<u8> { - self.path.into_encoded_bytes() - } - - pub fn into_cstring(self, vm: &VirtualMachine) -> PyResult<ffi::CString> { - ffi::CString::new(self.into_bytes()).map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(windows)] - pub fn to_widecstring(&self, vm: &VirtualMachine) -> PyResult<widestring::WideCString> { - widestring::WideCString::from_os_str(&self.path).map_err(|err| err.into_pyexception(vm)) - } - - pub fn filename(&self, vm: &VirtualMachine) -> PyResult { - self.mode.process_path(self.path.clone(), vm) - } -} - -pub(super) fn fs_metadata<P: AsRef<Path>>( - path: P, - follow_symlink: bool, -) -> io::Result<fs::Metadata> { - if follow_symlink { - fs::metadata(path.as_ref()) - } else { - fs::symlink_metadata(path.as_ref()) - } -} - -impl AsRef<Path> for OsPath { - fn as_ref(&self) -> &Path { - self.as_path() - } -} - -impl TryFromObject for OsPath { - // TODO: path_converter with allow_fd=0 in CPython - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let fspath = FsPath::try_from(obj, true, vm)?; - Self::from_fspath(fspath, vm) - } -} - -// path_t with allow_fd in CPython -#[derive(Clone)] -pub(crate) enum OsPathOrFd { - Path(OsPath), - Fd(i32), -} - -impl TryFromObject for OsPathOrFd { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let r = match obj.try_index_opt(vm) { - Some(int) => Self::Fd(int?.try_to_primitive(vm)?), - None => Self::Path(obj.try_into_value(vm)?), - }; - Ok(r) - } -} - -impl From<OsPath> for OsPathOrFd { - fn from(path: OsPath) -> Self { - Self::Path(path) - } -} - -impl OsPathOrFd { - pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef { - match self { - OsPathOrFd::Path(path) => path.filename(vm).unwrap_or_else(|_| vm.ctx.none()), - OsPathOrFd::Fd(fd) => vm.ctx.new_int(*fd).into(), - } - } -} - -#[cfg(unix)] -impl IntoPyException for nix::Error { - fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { - io::Error::from(self).into_pyexception(vm) - } -} - -// TODO: preserve the input `PyObjectRef` of filename and filename2 (Failing check `self.assertIs(err.filename, name, str(func)`) -pub struct IOErrorBuilder { - error: io::Error, - filename: Option<OsPathOrFd>, - filename2: Option<OsPathOrFd>, -} - -impl IOErrorBuilder { - pub fn new(error: io::Error) -> Self { - Self { - error, - filename: None, - filename2: None, - } - } - pub(crate) fn filename(mut self, filename: impl Into<OsPathOrFd>) -> Self { - let filename = filename.into(); - self.filename.replace(filename); - self - } - pub(crate) fn filename2(mut self, filename: impl Into<OsPathOrFd>) -> Self { - let filename = filename.into(); - self.filename2.replace(filename); - self - } -} - -impl IntoPyException for IOErrorBuilder { - fn into_pyexception(self, vm: &VirtualMachine) -> PyBaseExceptionRef { - let excp = self.error.into_pyexception(vm); - - if let Some(filename) = self.filename { - excp.as_object() - .set_attr("filename", filename.filename(vm), vm) - .unwrap(); - } - if let Some(filename2) = self.filename2 { - excp.as_object() - .set_attr("filename2", filename2.filename(vm), vm) - .unwrap(); - } - excp - } -} - -/// Convert the error stored in the `errno` variable into an Exception -#[inline] -pub fn errno_err(vm: &VirtualMachine) -> PyBaseExceptionRef { - crate::common::os::errno().into_pyexception(vm) -} - -#[allow(dead_code)] -#[derive(FromArgs, Default)] -pub struct TargetIsDirectory { - #[pyarg(any, default = "false")] - pub(crate) target_is_directory: bool, -} - -cfg_if::cfg_if! { - if #[cfg(all(any(unix, target_os = "wasi"), not(target_os = "redox")))] { - use libc::AT_FDCWD; - } else { - const AT_FDCWD: i32 = -100; - } -} -const DEFAULT_DIR_FD: Fd = Fd(AT_FDCWD); - -// XXX: AVAILABLE should be a bool, but we can't yet have it as a bool and just cast it to usize -#[derive(Copy, Clone)] -pub struct DirFd<const AVAILABLE: usize>(pub(crate) [Fd; AVAILABLE]); - -impl<const AVAILABLE: usize> Default for DirFd<AVAILABLE> { - fn default() -> Self { - Self([DEFAULT_DIR_FD; AVAILABLE]) - } -} - -// not used on all platforms -#[allow(unused)] -impl DirFd<1> { - #[inline(always)] - pub(crate) fn fd_opt(&self) -> Option<Fd> { - self.get_opt().map(Fd) - } - - #[inline] - pub(crate) fn get_opt(&self) -> Option<i32> { - let fd = self.fd(); - if fd == DEFAULT_DIR_FD { - None - } else { - Some(fd.0) - } - } - - #[inline(always)] - pub(crate) fn fd(&self) -> Fd { - self.0[0] - } -} - -impl<const AVAILABLE: usize> FromArgs for DirFd<AVAILABLE> { - fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> { - let fd = match args.take_keyword("dir_fd") { - Some(o) if vm.is_none(&o) => DEFAULT_DIR_FD, - None => DEFAULT_DIR_FD, - Some(o) => { - let fd = o.try_index_opt(vm).unwrap_or_else(|| { - Err(vm.new_type_error(format!( - "argument should be integer or None, not {}", - o.class().name() - ))) - })?; - let fd = fd.try_to_primitive(vm)?; - Fd(fd) - } - }; - if AVAILABLE == 0 && fd != DEFAULT_DIR_FD { - return Err(vm - .new_not_implemented_error("dir_fd unavailable on this platform".to_owned()) - .into()); - } - Ok(Self([fd; AVAILABLE])) - } -} - -#[derive(FromArgs)] -pub(super) struct FollowSymlinks( - #[pyarg(named, name = "follow_symlinks", default = "true")] pub bool, -); - -fn bytes_as_osstr<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a ffi::OsStr> { - rustpython_common::os::bytes_as_osstr(b) - .map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8".to_owned())) -} - -#[pymodule(sub)] -pub(super) mod _os { - use super::{ - errno_err, DirFd, FollowSymlinks, IOErrorBuilder, OsPath, OsPathOrFd, OutputMode, - SupportFunc, - }; - use crate::{ - builtins::{ - PyBytesRef, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, - }, - common::crt_fd::{Fd, Offset}, - common::lock::{OnceCell, PyRwLock}, - common::suppress_iph, - convert::{IntoPyException, ToPyObject}, - function::{ArgBytesLike, Either, FsPath, FuncArgs, OptionalArg}, - protocol::PyIterReturn, - recursion::ReprGuard, - types::{IterNext, Iterable, PyStructSequence, Representable, SelfIter}, - vm::VirtualMachine, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, - }; - use crossbeam_utils::atomic::AtomicCell; - use itertools::Itertools; - use std::{ - env, ffi, fs, - fs::OpenOptions, - io::{self, Read, Write}, - path::PathBuf, - time::{Duration, SystemTime}, - }; - - const OPEN_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); - pub(crate) const MKDIR_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); - const STAT_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); - const UTIME_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); - pub(crate) const SYMLINK_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); - - #[pyattr] - use libc::{ - O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY, SEEK_CUR, SEEK_END, - SEEK_SET, - }; - - #[pyattr] - pub(crate) const F_OK: u8 = 0; - #[pyattr] - pub(crate) const R_OK: u8 = 1 << 2; - #[pyattr] - pub(crate) const W_OK: u8 = 1 << 1; - #[pyattr] - pub(crate) const X_OK: u8 = 1 << 0; - - #[pyfunction] - fn close(fileno: i32, vm: &VirtualMachine) -> PyResult<()> { - Fd(fileno).close().map_err(|e| e.into_pyexception(vm)) - } - - #[pyfunction] - fn closerange(fd_low: i32, fd_high: i32) { - for fileno in fd_low..fd_high { - let _ = Fd(fileno).close(); - } - } - - #[cfg(any(unix, windows, target_os = "wasi"))] - #[derive(FromArgs)] - struct OpenArgs { - path: OsPath, - flags: i32, - #[pyarg(any, default)] - mode: Option<i32>, - #[pyarg(flatten)] - dir_fd: DirFd<{ OPEN_DIR_FD as usize }>, - } - - #[pyfunction] - fn open(args: OpenArgs, vm: &VirtualMachine) -> PyResult<i32> { - os_open(args.path, args.flags, args.mode, args.dir_fd, vm) - } - - #[cfg(any(unix, windows, target_os = "wasi"))] - pub(crate) fn os_open( - name: OsPath, - flags: i32, - mode: Option<i32>, - dir_fd: DirFd<{ OPEN_DIR_FD as usize }>, - vm: &VirtualMachine, - ) -> PyResult<i32> { - let mode = mode.unwrap_or(0o777); - #[cfg(windows)] - let fd = { - let [] = dir_fd.0; - let name = name.to_widecstring(vm)?; - let flags = flags | libc::O_NOINHERIT; - Fd::wopen(&name, flags, mode) - }; - #[cfg(not(windows))] - let fd = { - let name = name.clone().into_cstring(vm)?; - #[cfg(not(target_os = "wasi"))] - let flags = flags | libc::O_CLOEXEC; - #[cfg(not(target_os = "redox"))] - if let Some(dir_fd) = dir_fd.fd_opt() { - dir_fd.openat(&name, flags, mode) - } else { - Fd::open(&name, flags, mode) - } - #[cfg(target_os = "redox")] - { - let [] = dir_fd.0; - Fd::open(&name, flags, mode) - } - }; - fd.map(|fd| fd.0) - .map_err(|e| IOErrorBuilder::new(e).filename(name).into_pyexception(vm)) - } - - #[pyfunction] - fn fsync(fd: i32, vm: &VirtualMachine) -> PyResult<()> { - Fd(fd).fsync().map_err(|err| err.into_pyexception(vm)) - } - - #[pyfunction] - fn read(fd: i32, n: usize, vm: &VirtualMachine) -> PyResult<PyBytesRef> { - let mut buffer = vec![0u8; n]; - let mut file = Fd(fd); - let n = file - .read(&mut buffer) - .map_err(|err| err.into_pyexception(vm))?; - buffer.truncate(n); - - Ok(vm.ctx.new_bytes(buffer)) - } - - #[pyfunction] - fn write(fd: i32, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult { - let mut file = Fd(fd); - let written = data - .with_ref(|b| file.write(b)) - .map_err(|err| err.into_pyexception(vm))?; - - Ok(vm.ctx.new_int(written).into()) - } - - #[pyfunction] - #[pyfunction(name = "unlink")] - fn remove(path: OsPath, dir_fd: DirFd<0>, vm: &VirtualMachine) -> PyResult<()> { - let [] = dir_fd.0; - let is_junction = cfg!(windows) - && fs::metadata(&path).map_or(false, |meta| meta.file_type().is_dir()) - && fs::symlink_metadata(&path).map_or(false, |meta| meta.file_type().is_symlink()); - let res = if is_junction { - fs::remove_dir(&path) - } else { - fs::remove_file(&path) - }; - res.map_err(|e| IOErrorBuilder::new(e).filename(path).into_pyexception(vm)) - } - - #[cfg(not(windows))] - #[pyfunction] - fn mkdir( - path: OsPath, - mode: OptionalArg<i32>, - dir_fd: DirFd<{ MKDIR_DIR_FD as usize }>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let mode = mode.unwrap_or(0o777); - let path = path.into_cstring(vm)?; - #[cfg(not(target_os = "redox"))] - if let Some(fd) = dir_fd.get_opt() { - let res = unsafe { libc::mkdirat(fd, path.as_ptr(), mode as _) }; - let res = if res < 0 { Err(errno_err(vm)) } else { Ok(()) }; - return res; - } - #[cfg(target_os = "redox")] - let [] = dir_fd.0; - let res = unsafe { libc::mkdir(path.as_ptr(), mode as _) }; - if res < 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - - #[pyfunction] - fn mkdirs(path: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - fs::create_dir_all(path.as_str()).map_err(|err| err.into_pyexception(vm)) - } - - #[pyfunction] - fn rmdir(path: OsPath, dir_fd: DirFd<0>, vm: &VirtualMachine) -> PyResult<()> { - let [] = dir_fd.0; - fs::remove_dir(&path) - .map_err(|e| IOErrorBuilder::new(e).filename(path).into_pyexception(vm)) - } - - const LISTDIR_FD: bool = cfg!(all(unix, not(target_os = "redox"))); - - #[pyfunction] - fn listdir(path: OptionalArg<OsPathOrFd>, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let path = path.unwrap_or_else(|| OsPathOrFd::Path(OsPath::new_str("."))); - let list = match path { - OsPathOrFd::Path(path) => { - let dir_iter = fs::read_dir(&path).map_err(|err| err.into_pyexception(vm))?; - dir_iter - .map(|entry| match entry { - Ok(entry_path) => path.mode.process_path(entry_path.file_name(), vm), - Err(e) => Err(IOErrorBuilder::new(e) - .filename(path.clone()) - .into_pyexception(vm)), - }) - .collect::<PyResult<_>>()? - } - OsPathOrFd::Fd(fno) => { - #[cfg(not(all(unix, not(target_os = "redox"))))] - { - let _ = fno; - return Err(vm.new_not_implemented_error( - "can't pass fd to listdir on this platform".to_owned(), - )); - } - #[cfg(all(unix, not(target_os = "redox")))] - { - use rustpython_common::os::ffi::OsStrExt; - let new_fd = nix::unistd::dup(fno).map_err(|e| e.into_pyexception(vm))?; - let mut dir = - nix::dir::Dir::from_fd(new_fd).map_err(|e| e.into_pyexception(vm))?; - dir.iter() - .filter_map(|entry| { - entry - .map_err(|e| e.into_pyexception(vm)) - .and_then(|entry| { - let fname = entry.file_name().to_bytes(); - Ok(match fname { - b"." | b".." => None, - _ => Some( - OutputMode::String - .process_path(ffi::OsStr::from_bytes(fname), vm)?, - ), - }) - }) - .transpose() - }) - .collect::<PyResult<_>>()? - } - } - }; - Ok(list) - } - - fn pyref_as_str<'a>( - obj: &'a Either<PyStrRef, PyBytesRef>, - vm: &VirtualMachine, - ) -> PyResult<&'a str> { - Ok(match obj { - Either::A(ref s) => s.as_str(), - Either::B(ref b) => super::bytes_as_osstr(b.as_bytes(), vm)? - .to_str() - .ok_or_else(|| { - vm.new_unicode_decode_error("can't decode bytes for utf-8".to_owned()) - })?, - }) - } - - #[pyfunction] - fn putenv( - key: Either<PyStrRef, PyBytesRef>, - value: Either<PyStrRef, PyBytesRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let key = pyref_as_str(&key, vm)?; - let value = pyref_as_str(&value, vm)?; - if key.contains('\0') || value.contains('\0') { - return Err(vm.new_value_error("embedded null byte".to_string())); - } - if key.is_empty() || key.contains('=') { - return Err(vm.new_value_error("illegal environment variable name".to_string())); - } - env::set_var(key, value); - Ok(()) - } - - #[pyfunction] - fn unsetenv(key: Either<PyStrRef, PyBytesRef>, vm: &VirtualMachine) -> PyResult<()> { - let key = pyref_as_str(&key, vm)?; - if key.contains('\0') { - return Err(vm.new_value_error("embedded null byte".to_string())); - } - if key.is_empty() || key.contains('=') { - return Err(vm.new_value_error("illegal environment variable name".to_string())); - } - env::remove_var(key); - Ok(()) - } - - #[pyfunction] - fn readlink(path: OsPath, dir_fd: DirFd<0>, vm: &VirtualMachine) -> PyResult { - let mode = path.mode; - let [] = dir_fd.0; - let path = fs::read_link(&path) - .map_err(|err| IOErrorBuilder::new(err).filename(path).into_pyexception(vm))?; - mode.process_path(path, vm) - } - - #[pyattr] - #[pyclass(name)] - #[derive(Debug, PyPayload)] - struct DirEntry { - file_name: std::ffi::OsString, - pathval: PathBuf, - file_type: io::Result<fs::FileType>, - mode: OutputMode, - stat: OnceCell<PyObjectRef>, - lstat: OnceCell<PyObjectRef>, - #[cfg(unix)] - ino: AtomicCell<u64>, - #[cfg(not(unix))] - ino: AtomicCell<Option<u64>>, - } - - #[pyclass(with(Representable))] - impl DirEntry { - #[pygetset] - fn name(&self, vm: &VirtualMachine) -> PyResult { - self.mode.process_path(&self.file_name, vm) - } - - #[pygetset] - fn path(&self, vm: &VirtualMachine) -> PyResult { - self.mode.process_path(&self.pathval, vm) - } - - fn perform_on_metadata( - &self, - follow_symlinks: FollowSymlinks, - action: fn(fs::Metadata) -> bool, - vm: &VirtualMachine, - ) -> PyResult<bool> { - match super::fs_metadata(&self.pathval, follow_symlinks.0) { - Ok(meta) => Ok(action(meta)), - Err(e) => { - // FileNotFoundError is caught and not raised - if e.kind() == io::ErrorKind::NotFound { - Ok(false) - } else { - Err(e.into_pyexception(vm)) - } - } - } - } - - #[pymethod] - fn is_dir(&self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult<bool> { - self.perform_on_metadata( - follow_symlinks, - |meta: fs::Metadata| -> bool { meta.is_dir() }, - vm, - ) - } - - #[pymethod] - fn is_file(&self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult<bool> { - self.perform_on_metadata( - follow_symlinks, - |meta: fs::Metadata| -> bool { meta.is_file() }, - vm, - ) - } - - #[pymethod] - fn is_symlink(&self, vm: &VirtualMachine) -> PyResult<bool> { - Ok(self - .file_type - .as_ref() - .map_err(|err| err.into_pyexception(vm))? - .is_symlink()) - } - - #[pymethod] - fn stat( - &self, - dir_fd: DirFd<{ STAT_DIR_FD as usize }>, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult { - let do_stat = |follow_symlinks| { - stat( - OsPath { - path: self.pathval.as_os_str().to_owned(), - mode: OutputMode::String, - } - .into(), - dir_fd, - FollowSymlinks(follow_symlinks), - vm, - ) - }; - let lstat = || self.lstat.get_or_try_init(|| do_stat(false)); - let stat = if follow_symlinks.0 { - // if follow_symlinks == true and we aren't a symlink, cache both stat and lstat - self.stat.get_or_try_init(|| { - if self.is_symlink(vm)? { - do_stat(true) - } else { - lstat().map(Clone::clone) - } - })? - } else { - lstat()? - }; - Ok(stat.clone()) - } - - #[cfg(not(unix))] - #[pymethod] - fn inode(&self, vm: &VirtualMachine) -> PyResult<u64> { - match self.ino.load() { - Some(ino) => Ok(ino), - None => { - let stat = stat_inner( - OsPath { - path: self.pathval.as_os_str().to_owned(), - mode: OutputMode::String, - } - .into(), - DirFd::default(), - FollowSymlinks(false), - ) - .map_err(|e| e.into_pyexception(vm))? - .ok_or_else(|| crate::exceptions::cstring_error(vm))?; - // Err(T) means other thread set `ino` at the mean time which is safe to ignore - let _ = self.ino.compare_exchange(None, Some(stat.st_ino)); - Ok(stat.st_ino) - } - } - } - - #[cfg(unix)] - #[pymethod] - fn inode(&self, _vm: &VirtualMachine) -> PyResult<u64> { - Ok(self.ino.load()) - } - - #[pymethod(magic)] - fn fspath(&self, vm: &VirtualMachine) -> PyResult { - self.path(vm) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } - } - - impl Representable for DirEntry { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let name = match zelf.as_object().get_attr("name", vm) { - Ok(name) => Some(name), - Err(e) - if e.fast_isinstance(vm.ctx.exceptions.attribute_error) - || e.fast_isinstance(vm.ctx.exceptions.value_error) => - { - None - } - Err(e) => return Err(e), - }; - if let Some(name) = name { - if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { - let repr = name.repr(vm)?; - Ok(format!("<{} {}>", zelf.class(), repr)) - } else { - Err(vm.new_runtime_error(format!( - "reentrant call inside {}.__repr__", - zelf.class() - ))) - } - } else { - Ok(format!("<{}>", zelf.class())) - } - } - } - - #[pyattr] - #[pyclass(name = "ScandirIter")] - #[derive(Debug, PyPayload)] - struct ScandirIterator { - entries: PyRwLock<Option<fs::ReadDir>>, - mode: OutputMode, - } - - #[pyclass(with(IterNext, Iterable))] - impl ScandirIterator { - #[pymethod] - fn close(&self) { - let entryref: &mut Option<fs::ReadDir> = &mut self.entries.write(); - let _dropped = entryref.take(); - } - - #[pymethod(magic)] - fn enter(zelf: PyRef<Self>) -> PyRef<Self> { - zelf - } - - #[pymethod(magic)] - fn exit(zelf: PyRef<Self>, _args: FuncArgs) { - zelf.close() - } - } - impl SelfIter for ScandirIterator {} - impl IterNext for ScandirIterator { - fn next(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let entryref: &mut Option<fs::ReadDir> = &mut zelf.entries.write(); - - match entryref { - None => Ok(PyIterReturn::StopIteration(None)), - Some(inner) => match inner.next() { - Some(entry) => match entry { - Ok(entry) => { - #[cfg(unix)] - let ino = { - use std::os::unix::fs::DirEntryExt; - entry.ino() - }; - #[cfg(not(unix))] - let ino = None; - - Ok(PyIterReturn::Return( - DirEntry { - file_name: entry.file_name(), - pathval: entry.path(), - file_type: entry.file_type(), - mode: zelf.mode, - lstat: OnceCell::new(), - stat: OnceCell::new(), - ino: AtomicCell::new(ino), - } - .into_ref(&vm.ctx) - .into(), - )) - } - Err(err) => Err(err.into_pyexception(vm)), - }, - None => { - let _dropped = entryref.take(); - Ok(PyIterReturn::StopIteration(None)) - } - }, - } - } - } - - #[pyfunction] - fn scandir(path: OptionalArg<OsPath>, vm: &VirtualMachine) -> PyResult { - let path = path.unwrap_or_else(|| OsPath::new_str(".")); - let entries = fs::read_dir(path.path).map_err(|err| err.into_pyexception(vm))?; - Ok(ScandirIterator { - entries: PyRwLock::new(Some(entries)), - mode: path.mode, - } - .into_ref(&vm.ctx) - .into()) - } - - #[pyattr] - #[pyclass(module = "os", name = "stat_result")] - #[derive(Debug, PyStructSequence, FromArgs)] - struct StatResult { - pub st_mode: PyIntRef, - pub st_ino: PyIntRef, - pub st_dev: PyIntRef, - pub st_nlink: PyIntRef, - pub st_uid: PyIntRef, - pub st_gid: PyIntRef, - pub st_size: PyIntRef, - // TODO: unnamed structsequence fields - #[pyarg(positional, default)] - pub __st_atime_int: libc::time_t, - #[pyarg(positional, default)] - pub __st_mtime_int: libc::time_t, - #[pyarg(positional, default)] - pub __st_ctime_int: libc::time_t, - #[pyarg(any, default)] - pub st_atime: f64, - #[pyarg(any, default)] - pub st_mtime: f64, - #[pyarg(any, default)] - pub st_ctime: f64, - #[pyarg(any, default)] - pub st_atime_ns: i128, - #[pyarg(any, default)] - pub st_mtime_ns: i128, - #[pyarg(any, default)] - pub st_ctime_ns: i128, - } - - #[pyclass(with(PyStructSequence))] - impl StatResult { - fn from_stat(stat: &StatStruct, vm: &VirtualMachine) -> Self { - let (atime, mtime, ctime); - #[cfg(any(unix, windows))] - #[cfg(not(target_os = "netbsd"))] - { - atime = (stat.st_atime, stat.st_atime_nsec); - mtime = (stat.st_mtime, stat.st_mtime_nsec); - ctime = (stat.st_ctime, stat.st_ctime_nsec); - } - #[cfg(target_os = "netbsd")] - { - atime = (stat.st_atime, stat.st_atimensec); - mtime = (stat.st_mtime, stat.st_mtimensec); - ctime = (stat.st_ctime, stat.st_ctimensec); - } - #[cfg(target_os = "wasi")] - { - atime = (stat.st_atim.tv_sec, stat.st_atim.tv_nsec); - mtime = (stat.st_mtim.tv_sec, stat.st_mtim.tv_nsec); - ctime = (stat.st_ctim.tv_sec, stat.st_ctim.tv_nsec); - } - - const NANOS_PER_SEC: u32 = 1_000_000_000; - let to_f64 = |(s, ns)| (s as f64) + (ns as f64) / (NANOS_PER_SEC as f64); - let to_ns = |(s, ns)| s as i128 * NANOS_PER_SEC as i128 + ns as i128; - StatResult { - st_mode: vm.ctx.new_pyref(stat.st_mode), - st_ino: vm.ctx.new_pyref(stat.st_ino), - st_dev: vm.ctx.new_pyref(stat.st_dev), - st_nlink: vm.ctx.new_pyref(stat.st_nlink), - st_uid: vm.ctx.new_pyref(stat.st_uid), - st_gid: vm.ctx.new_pyref(stat.st_gid), - st_size: vm.ctx.new_pyref(stat.st_size), - __st_atime_int: atime.0, - __st_mtime_int: mtime.0, - __st_ctime_int: ctime.0, - st_atime: to_f64(atime), - st_mtime: to_f64(mtime), - st_ctime: to_f64(ctime), - st_atime_ns: to_ns(atime), - st_mtime_ns: to_ns(mtime), - st_ctime_ns: to_ns(ctime), - } - } - - #[pyslot] - fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let flatten_args = |r: &[PyObjectRef]| { - let mut vec_args = Vec::from(r); - loop { - if let Ok(obj) = vec_args.iter().exactly_one() { - match obj.payload::<PyTuple>() { - Some(t) => { - vec_args = Vec::from(t.as_slice()); - } - None => { - return vec_args; - } - } - } else { - return vec_args; - } - } - }; - - let args: FuncArgs = flatten_args(&args.args).into(); - - let stat: StatResult = args.bind(vm)?; - Ok(stat.to_pyobject(vm)) - } - } - - #[cfg(not(windows))] - use libc::stat as StatStruct; - - #[cfg(windows)] - struct StatStruct { - st_dev: libc::c_ulong, - st_ino: u64, - st_mode: libc::c_ushort, - st_nlink: i32, - st_uid: i32, - st_gid: i32, - st_size: u64, - st_atime: libc::time_t, - st_atime_nsec: i32, - st_mtime: libc::time_t, - st_mtime_nsec: i32, - st_ctime: libc::time_t, - st_ctime_nsec: i32, - } - - #[cfg(windows)] - fn meta_to_stat(meta: &fs::Metadata) -> io::Result<StatStruct> { - let st_mode = { - // Based on CPython fileutils.c' attributes_to_mode - let mut m = 0; - if meta.is_dir() { - m |= libc::S_IFDIR | 0o111; /* IFEXEC for user,group,other */ - } else { - m |= libc::S_IFREG; - } - if meta.permissions().readonly() { - m |= 0o444; - } else { - m |= 0o666; - } - m as _ - }; - let (atime, mtime, ctime) = (meta.accessed()?, meta.modified()?, meta.created()?); - let sec = |systime: SystemTime| match systime.duration_since(SystemTime::UNIX_EPOCH) { - Ok(d) => d.as_secs() as libc::time_t, - Err(e) => -(e.duration().as_secs() as libc::time_t), - }; - let nsec = |systime: SystemTime| match systime.duration_since(SystemTime::UNIX_EPOCH) { - Ok(d) => d.subsec_nanos() as i32, - Err(e) => -(e.duration().subsec_nanos() as i32), - }; - Ok(StatStruct { - st_dev: 0, - st_ino: 0, - st_mode, - st_nlink: 0, - st_uid: 0, - st_gid: 0, - st_size: meta.len(), - st_atime: sec(atime), - st_mtime: sec(mtime), - st_ctime: sec(ctime), - st_atime_nsec: nsec(atime), - st_mtime_nsec: nsec(mtime), - st_ctime_nsec: nsec(ctime), - }) - } - - #[cfg(windows)] - fn stat_inner( - file: OsPathOrFd, - dir_fd: DirFd<{ STAT_DIR_FD as usize }>, - follow_symlinks: FollowSymlinks, - ) -> io::Result<Option<StatStruct>> { - // TODO: replicate CPython's win32_xstat - let [] = dir_fd.0; - let meta = match file { - OsPathOrFd::Path(path) => super::fs_metadata(path, follow_symlinks.0)?, - OsPathOrFd::Fd(fno) => { - use std::os::windows::io::FromRawHandle; - let handle = Fd(fno).to_raw_handle()?; - let file = - std::mem::ManuallyDrop::new(unsafe { std::fs::File::from_raw_handle(handle) }); - file.metadata()? - } - }; - meta_to_stat(&meta).map(Some) - } - - #[cfg(not(windows))] - fn stat_inner( - file: OsPathOrFd, - dir_fd: DirFd<{ STAT_DIR_FD as usize }>, - follow_symlinks: FollowSymlinks, - ) -> io::Result<Option<StatStruct>> { - let mut stat = std::mem::MaybeUninit::uninit(); - let ret = match file { - OsPathOrFd::Path(path) => { - use rustpython_common::os::ffi::OsStrExt; - let path = path.as_ref().as_os_str().as_bytes(); - let path = match ffi::CString::new(path) { - Ok(x) => x, - Err(_) => return Ok(None), - }; - - #[cfg(not(target_os = "redox"))] - let fstatat_ret = dir_fd.get_opt().map(|dir_fd| { - let flags = if follow_symlinks.0 { - 0 - } else { - libc::AT_SYMLINK_NOFOLLOW - }; - unsafe { libc::fstatat(dir_fd, path.as_ptr(), stat.as_mut_ptr(), flags) } - }); - #[cfg(target_os = "redox")] - let ([], fstatat_ret) = (dir_fd.0, None); - - fstatat_ret.unwrap_or_else(|| { - if follow_symlinks.0 { - unsafe { libc::stat(path.as_ptr(), stat.as_mut_ptr()) } - } else { - unsafe { libc::lstat(path.as_ptr(), stat.as_mut_ptr()) } - } - }) - } - OsPathOrFd::Fd(fd) => unsafe { libc::fstat(fd, stat.as_mut_ptr()) }, - }; - if ret < 0 { - return Err(io::Error::last_os_error()); - } - Ok(Some(unsafe { stat.assume_init() })) - } - - #[pyfunction] - #[pyfunction(name = "fstat")] - fn stat( - file: OsPathOrFd, - dir_fd: DirFd<{ STAT_DIR_FD as usize }>, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult { - let stat = stat_inner(file.clone(), dir_fd, follow_symlinks) - .map_err(|e| IOErrorBuilder::new(e).filename(file).into_pyexception(vm))? - .ok_or_else(|| crate::exceptions::cstring_error(vm))?; - Ok(StatResult::from_stat(&stat, vm).to_pyobject(vm)) - } - - #[pyfunction] - fn lstat( - file: OsPathOrFd, - dir_fd: DirFd<{ STAT_DIR_FD as usize }>, - vm: &VirtualMachine, - ) -> PyResult { - stat(file, dir_fd, FollowSymlinks(false), vm) - } - - fn curdir_inner(vm: &VirtualMachine) -> PyResult<PathBuf> { - env::current_dir().map_err(|err| err.into_pyexception(vm)) - } - - #[pyfunction] - fn getcwd(vm: &VirtualMachine) -> PyResult { - OutputMode::String.process_path(curdir_inner(vm)?, vm) - } - - #[pyfunction] - fn getcwdb(vm: &VirtualMachine) -> PyResult { - OutputMode::Bytes.process_path(curdir_inner(vm)?, vm) - } - - #[pyfunction] - fn chdir(path: OsPath, vm: &VirtualMachine) -> PyResult<()> { - env::set_current_dir(&path.path) - .map_err(|err| IOErrorBuilder::new(err).filename(path).into_pyexception(vm)) - } - - #[pyfunction] - fn fspath(path: PyObjectRef, vm: &VirtualMachine) -> PyResult<FsPath> { - FsPath::try_from(path, false, vm) - } - - #[pyfunction] - #[pyfunction(name = "replace")] - fn rename(src: OsPath, dst: OsPath, vm: &VirtualMachine) -> PyResult<()> { - fs::rename(&src.path, &dst.path).map_err(|err| { - IOErrorBuilder::new(err) - .filename(src) - .filename2(dst) - .into_pyexception(vm) - }) - } - - #[pyfunction] - fn getpid(vm: &VirtualMachine) -> PyObjectRef { - let pid = std::process::id(); - vm.ctx.new_int(pid).into() - } - - #[pyfunction] - fn cpu_count(vm: &VirtualMachine) -> PyObjectRef { - let cpu_count = num_cpus::get(); - vm.ctx.new_int(cpu_count).into() - } - - #[pyfunction] - fn _exit(code: i32) { - std::process::exit(code) - } - - #[pyfunction] - fn abort() { - extern "C" { - fn abort(); - } - unsafe { abort() } - } - - #[pyfunction] - fn urandom(size: isize, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - if size < 0 { - return Err(vm.new_value_error("negative argument not allowed".to_owned())); - } - let mut buf = vec![0u8; size as usize]; - getrandom::getrandom(&mut buf).map_err(|e| match e.raw_os_error() { - Some(errno) => io::Error::from_raw_os_error(errno).into_pyexception(vm), - None => vm.new_os_error("Getting random failed".to_owned()), - })?; - Ok(buf) - } - - #[pyfunction] - pub fn isatty(fd: i32) -> bool { - unsafe { suppress_iph!(libc::isatty(fd)) != 0 } - } - - #[pyfunction] - pub fn lseek(fd: i32, position: Offset, how: i32, vm: &VirtualMachine) -> PyResult<Offset> { - #[cfg(not(windows))] - let res = unsafe { suppress_iph!(libc::lseek(fd, position, how)) }; - #[cfg(windows)] - let res = unsafe { - use windows_sys::Win32::Storage::FileSystem; - let handle = Fd(fd).to_raw_handle().map_err(|e| e.into_pyexception(vm))?; - let mut distance_to_move: [i32; 2] = std::mem::transmute(position); - let ret = FileSystem::SetFilePointer( - handle as _, - distance_to_move[0], - &mut distance_to_move[1], - how as _, - ); - if ret == FileSystem::INVALID_SET_FILE_POINTER { - -1 - } else { - distance_to_move[0] = ret as _; - std::mem::transmute(distance_to_move) - } - }; - if res < 0 { - Err(errno_err(vm)) - } else { - Ok(res) - } - } - - #[pyfunction] - fn link(src: OsPath, dst: OsPath, vm: &VirtualMachine) -> PyResult<()> { - fs::hard_link(&src.path, &dst.path).map_err(|err| { - IOErrorBuilder::new(err) - .filename(src) - .filename2(dst) - .into_pyexception(vm) - }) - } - - #[derive(FromArgs)] - struct UtimeArgs { - path: OsPath, - #[pyarg(any, default)] - times: Option<PyTupleRef>, - #[pyarg(named, default)] - ns: Option<PyTupleRef>, - #[pyarg(flatten)] - dir_fd: DirFd<{ UTIME_DIR_FD as usize }>, - #[pyarg(flatten)] - follow_symlinks: FollowSymlinks, - } - - #[pyfunction] - fn utime(args: UtimeArgs, vm: &VirtualMachine) -> PyResult<()> { - let parse_tup = |tup: &PyTuple| -> Option<(PyObjectRef, PyObjectRef)> { - if tup.len() != 2 { - None - } else { - Some((tup[0].clone(), tup[1].clone())) - } - }; - let (acc, modif) = match (args.times, args.ns) { - (Some(t), None) => { - let (a, m) = parse_tup(&t).ok_or_else(|| { - vm.new_type_error( - "utime: 'times' must be either a tuple of two ints or None".to_owned(), - ) - })?; - (a.try_into_value(vm)?, m.try_into_value(vm)?) - } - (None, Some(ns)) => { - let (a, m) = parse_tup(&ns).ok_or_else(|| { - vm.new_type_error("utime: 'ns' must be a tuple of two ints".to_owned()) - })?; - let ns_in_sec: PyObjectRef = vm.ctx.new_int(1_000_000_000).into(); - let ns_to_dur = |obj: PyObjectRef| { - let divmod = vm._divmod(&obj, &ns_in_sec)?; - let (div, rem) = - divmod - .payload::<PyTuple>() - .and_then(parse_tup) - .ok_or_else(|| { - vm.new_type_error(format!( - "{}.__divmod__() must return a 2-tuple, not {}", - obj.class().name(), - divmod.class().name() - )) - })?; - let secs = div.try_index(vm)?.try_to_primitive(vm)?; - let ns = rem.try_index(vm)?.try_to_primitive(vm)?; - Ok(Duration::new(secs, ns)) - }; - // TODO: do validation to make sure this doesn't.. underflow? - (ns_to_dur(a)?, ns_to_dur(m)?) - } - (None, None) => { - let now = SystemTime::now(); - let now = now.duration_since(SystemTime::UNIX_EPOCH).unwrap(); - (now, now) - } - (Some(_), Some(_)) => { - return Err(vm.new_value_error( - "utime: you may specify either 'times' or 'ns' but not both".to_owned(), - )) - } - }; - utime_impl(args.path, acc, modif, args.dir_fd, args.follow_symlinks, vm) - } - - fn utime_impl( - path: OsPath, - acc: Duration, - modif: Duration, - dir_fd: DirFd<{ UTIME_DIR_FD as usize }>, - _follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult<()> { - #[cfg(any(target_os = "wasi", unix))] - { - #[cfg(not(target_os = "redox"))] - { - let path = path.into_cstring(vm)?; - - let ts = |d: Duration| libc::timespec { - tv_sec: d.as_secs() as _, - tv_nsec: d.subsec_nanos() as _, - }; - let times = [ts(acc), ts(modif)]; - - let ret = unsafe { - libc::utimensat( - dir_fd.fd().0, - path.as_ptr(), - times.as_ptr(), - if _follow_symlinks.0 { - 0 - } else { - libc::AT_SYMLINK_NOFOLLOW - }, - ) - }; - if ret < 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - #[cfg(target_os = "redox")] - { - let [] = dir_fd.0; - - let tv = |d: Duration| libc::timeval { - tv_sec: d.as_secs() as _, - tv_usec: d.as_micros() as _, - }; - nix::sys::stat::utimes(path.as_ref(), &tv(acc).into(), &tv(modif).into()) - .map_err(|err| err.into_pyexception(vm)) - } - } - #[cfg(windows)] - { - use std::{fs::OpenOptions, os::windows::prelude::*}; - type DWORD = u32; - use windows_sys::Win32::{Foundation::FILETIME, Storage::FileSystem}; - - let [] = dir_fd.0; - - let ft = |d: Duration| { - let intervals = - ((d.as_secs() as i64 + 11644473600) * 10_000_000) + (d.as_nanos() as i64 / 100); - FILETIME { - dwLowDateTime: intervals as DWORD, - dwHighDateTime: (intervals >> 32) as DWORD, - } - }; - - let acc = ft(acc); - let modif = ft(modif); - - let f = OpenOptions::new() - .write(true) - .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS) - .open(path) - .map_err(|err| err.into_pyexception(vm))?; - - let ret = unsafe { - FileSystem::SetFileTime(f.as_raw_handle() as _, std::ptr::null(), &acc, &modif) - }; - - if ret == 0 { - Err(io::Error::last_os_error().into_pyexception(vm)) - } else { - Ok(()) - } - } - } - - #[cfg(all(any(unix, windows), not(target_os = "redox")))] - #[pyattr] - #[pyclass(module = "os", name = "times_result")] - #[derive(Debug, PyStructSequence)] - struct TimesResult { - pub user: f64, - pub system: f64, - pub children_user: f64, - pub children_system: f64, - pub elapsed: f64, - } - - #[cfg(all(any(unix, windows), not(target_os = "redox")))] - #[pyclass(with(PyStructSequence))] - impl TimesResult {} - - #[cfg(all(any(unix, windows), not(target_os = "redox")))] - #[pyfunction] - fn times(vm: &VirtualMachine) -> PyResult { - #[cfg(windows)] - { - use std::mem::MaybeUninit; - use windows_sys::Win32::{Foundation::FILETIME, System::Threading}; - - let mut _create = MaybeUninit::<FILETIME>::uninit(); - let mut _exit = MaybeUninit::<FILETIME>::uninit(); - let mut kernel = MaybeUninit::<FILETIME>::uninit(); - let mut user = MaybeUninit::<FILETIME>::uninit(); - - unsafe { - let h_proc = Threading::GetCurrentProcess(); - Threading::GetProcessTimes( - h_proc, - _create.as_mut_ptr(), - _exit.as_mut_ptr(), - kernel.as_mut_ptr(), - user.as_mut_ptr(), - ); - } - - let kernel = unsafe { kernel.assume_init() }; - let user = unsafe { user.assume_init() }; - - let times_result = TimesResult { - user: user.dwHighDateTime as f64 * 429.4967296 + user.dwLowDateTime as f64 * 1e-7, - system: kernel.dwHighDateTime as f64 * 429.4967296 - + kernel.dwLowDateTime as f64 * 1e-7, - children_user: 0.0, - children_system: 0.0, - elapsed: 0.0, - }; - - Ok(times_result.to_pyobject(vm)) - } - #[cfg(unix)] - { - let mut t = libc::tms { - tms_utime: 0, - tms_stime: 0, - tms_cutime: 0, - tms_cstime: 0, - }; - - let tick_for_second = unsafe { libc::sysconf(libc::_SC_CLK_TCK) } as f64; - let c = unsafe { libc::times(&mut t as *mut _) }; - - // XXX: The signedness of `clock_t` varies from platform to platform. - if c == (-1i8) as libc::clock_t { - return Err(vm.new_os_error("Fail to get times".to_string())); - } - - let times_result = TimesResult { - user: t.tms_utime as f64 / tick_for_second, - system: t.tms_stime as f64 / tick_for_second, - children_user: t.tms_cutime as f64 / tick_for_second, - children_system: t.tms_cstime as f64 / tick_for_second, - elapsed: c as f64 / tick_for_second, - }; - - Ok(times_result.to_pyobject(vm)) - } - } - - #[cfg(target_os = "linux")] - #[derive(FromArgs)] - struct CopyFileRangeArgs { - #[pyarg(positional)] - src: i32, - #[pyarg(positional)] - dst: i32, - #[pyarg(positional)] - count: i64, - #[pyarg(any, default)] - offset_src: Option<Offset>, - #[pyarg(any, default)] - offset_dst: Option<Offset>, - } - - #[cfg(target_os = "linux")] - #[pyfunction] - fn copy_file_range(args: CopyFileRangeArgs, vm: &VirtualMachine) -> PyResult<usize> { - let p_offset_src = args.offset_src.as_ref().map_or_else(std::ptr::null, |x| x); - let p_offset_dst = args.offset_dst.as_ref().map_or_else(std::ptr::null, |x| x); - let count: usize = args - .count - .try_into() - .map_err(|_| vm.new_value_error("count should >= 0".to_string()))?; - - // The flags argument is provided to allow - // for future extensions and currently must be to 0. - let flags = 0u32; - - // Safety: p_offset_src and p_offset_dst is a unique pointer for offset_src and offset_dst respectively, - // and will only be freed after this function ends. - // - // Why not use `libc::copy_file_range`: On `musl-libc`, `libc::copy_file_range` is not provided. Therefore - // we use syscalls directly instead. - let ret = unsafe { - libc::syscall( - libc::SYS_copy_file_range, - args.src, - p_offset_src as *mut i64, - args.dst, - p_offset_dst as *mut i64, - count, - flags, - ) - }; - - usize::try_from(ret).map_err(|_| errno_err(vm)) - } - - #[pyfunction] - fn strerror(e: i32) -> String { - unsafe { ffi::CStr::from_ptr(libc::strerror(e)) } - .to_string_lossy() - .into_owned() - } - - #[pyfunction] - pub fn ftruncate(fd: i32, length: Offset, vm: &VirtualMachine) -> PyResult<()> { - Fd(fd).ftruncate(length).map_err(|e| e.into_pyexception(vm)) - } - - #[pyfunction] - fn truncate(path: PyObjectRef, length: Offset, vm: &VirtualMachine) -> PyResult<()> { - if let Ok(fd) = path.try_to_value(vm) { - return ftruncate(fd, length, vm); - } - let path = OsPath::try_from_object(vm, path)?; - // TODO: just call libc::truncate() on POSIX - let f = OpenOptions::new() - .write(true) - .open(path) - .map_err(|e| e.into_pyexception(vm))?; - f.set_len(length as u64) - .map_err(|e| e.into_pyexception(vm))?; - drop(f); - Ok(()) - } - - #[cfg(all(unix, not(any(target_os = "redox", target_os = "android"))))] - #[pyfunction] - fn getloadavg(vm: &VirtualMachine) -> PyResult<(f64, f64, f64)> { - let mut loadavg = [0f64; 3]; - - // Safety: loadavg is on stack and only write by `getloadavg` and are freed - // after this function ends. - unsafe { - if libc::getloadavg(&mut loadavg[0] as *mut f64, 3) != 3 { - return Err(vm.new_os_error("Load averages are unobtainable".to_string())); - } - } - - Ok((loadavg[0], loadavg[1], loadavg[2])) - } - - #[cfg(any(unix, windows))] - #[pyfunction] - fn waitstatus_to_exitcode(status: i32, vm: &VirtualMachine) -> PyResult<i32> { - let status = u32::try_from(status) - .map_err(|_| vm.new_value_error(format!("invalid WEXITSTATUS: {status}")))?; - - cfg_if::cfg_if! { - if #[cfg(not(windows))] { - let status = status as libc::c_int; - if libc::WIFEXITED(status) { - return Ok(libc::WEXITSTATUS(status)); - } - - if libc::WIFSIGNALED(status) { - return Ok(-libc::WTERMSIG(status)); - } - - Err(vm.new_value_error(format!("Invalid wait status: {status}"))) - } else { - i32::try_from(status.rotate_right(8)) - .map_err(|_| vm.new_value_error(format!("invalid wait status: {status}"))) - } - } - } - - #[pyfunction] - fn device_encoding(fd: i32, _vm: &VirtualMachine) -> PyResult<Option<String>> { - if !isatty(fd) { - return Ok(None); - } - - cfg_if::cfg_if! { - if #[cfg(any(target_os = "android", target_os = "redox"))] { - Ok(Some("UTF-8".to_owned())) - } else if #[cfg(windows)] { - use windows_sys::Win32::System::Console; - let cp = match fd { - 0 => unsafe { Console::GetConsoleCP() }, - 1 | 2 => unsafe { Console::GetConsoleOutputCP() }, - _ => 0, - }; - - Ok(Some(format!("cp{cp}"))) - } else { - let encoding = unsafe { - let encoding = libc::nl_langinfo(libc::CODESET); - if encoding.is_null() || encoding.read() == '\0' as libc::c_char { - "UTF-8".to_owned() - } else { - ffi::CStr::from_ptr(encoding).to_string_lossy().into_owned() - } - }; - - Ok(Some(encoding)) - } - } - } - - #[pyattr] - #[pyclass(module = "os", name = "terminal_size")] - #[derive(PyStructSequence)] - #[allow(dead_code)] - pub(crate) struct PyTerminalSize { - pub columns: usize, - pub lines: usize, - } - #[pyclass(with(PyStructSequence))] - impl PyTerminalSize {} - - #[pyattr] - #[pyclass(module = "os", name = "uname_result")] - #[derive(Debug, PyStructSequence)] - pub(crate) struct UnameResult { - pub sysname: String, - pub nodename: String, - pub release: String, - pub version: String, - pub machine: String, - } - - #[pyclass(with(PyStructSequence))] - impl UnameResult {} - - pub(super) fn support_funcs() -> Vec<SupportFunc> { - let mut supports = super::platform::module::support_funcs(); - supports.extend(vec![ - SupportFunc::new("open", Some(false), Some(OPEN_DIR_FD), Some(false)), - SupportFunc::new("access", Some(false), Some(false), None), - SupportFunc::new("chdir", None, Some(false), Some(false)), - // chflags Some, None Some - SupportFunc::new("listdir", Some(LISTDIR_FD), Some(false), Some(false)), - SupportFunc::new("mkdir", Some(false), Some(MKDIR_DIR_FD), Some(false)), - // mkfifo Some Some None - // mknod Some Some None - SupportFunc::new("readlink", Some(false), None, Some(false)), - SupportFunc::new("remove", Some(false), None, Some(false)), - SupportFunc::new("unlink", Some(false), None, Some(false)), - SupportFunc::new("rename", Some(false), None, Some(false)), - SupportFunc::new("replace", Some(false), None, Some(false)), // TODO: Fix replace - SupportFunc::new("rmdir", Some(false), None, Some(false)), - SupportFunc::new("scandir", None, Some(false), Some(false)), - SupportFunc::new("stat", Some(true), Some(STAT_DIR_FD), Some(true)), - SupportFunc::new("fstat", Some(true), Some(STAT_DIR_FD), Some(true)), - SupportFunc::new("symlink", Some(false), Some(SYMLINK_DIR_FD), Some(false)), - SupportFunc::new("truncate", Some(true), Some(false), Some(false)), - SupportFunc::new( - "utime", - Some(false), - Some(UTIME_DIR_FD), - Some(cfg!(all(unix, not(target_os = "redox")))), - ), - ]); - supports - } -} -pub(crate) use _os::{ftruncate, isatty, lseek}; - -pub(crate) struct SupportFunc { - name: &'static str, - // realistically, each of these is just a bool of "is this function in the supports_* set". - // However, None marks that the function maybe _should_ support fd/dir_fd/follow_symlinks, but - // we haven't implemented it yet. - fd: Option<bool>, - dir_fd: Option<bool>, - follow_symlinks: Option<bool>, -} - -impl SupportFunc { - pub(crate) fn new( - name: &'static str, - fd: Option<bool>, - dir_fd: Option<bool>, - follow_symlinks: Option<bool>, - ) -> Self { - Self { - name, - fd, - dir_fd, - follow_symlinks, - } - } -} - -pub fn extend_module(vm: &VirtualMachine, module: &Py<PyModule>) { - let support_funcs = _os::support_funcs(); - let supports_fd = PySet::default().into_ref(&vm.ctx); - let supports_dir_fd = PySet::default().into_ref(&vm.ctx); - let supports_follow_symlinks = PySet::default().into_ref(&vm.ctx); - for support in support_funcs { - let func_obj = module.get_attr(support.name, vm).unwrap(); - if support.fd.unwrap_or(false) { - supports_fd.clone().add(func_obj.clone(), vm).unwrap(); - } - if support.dir_fd.unwrap_or(false) { - supports_dir_fd.clone().add(func_obj.clone(), vm).unwrap(); - } - if support.follow_symlinks.unwrap_or(false) { - supports_follow_symlinks.clone().add(func_obj, vm).unwrap(); - } - } - - extend_module!(vm, module, { - "supports_fd" => supports_fd, - "supports_dir_fd" => supports_dir_fd, - "supports_follow_symlinks" => supports_follow_symlinks, - "error" => vm.ctx.exceptions.os_error.to_owned(), - }); -} -pub(crate) use _os::os_open as open; - -#[cfg(not(windows))] -use super::posix as platform; - -#[cfg(windows)] -use super::nt as platform; - -pub(crate) use platform::module::MODULE_NAME; diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs deleted file mode 100644 index 1adf0006efd..00000000000 --- a/vm/src/stdlib/posix.rs +++ /dev/null @@ -1,2125 +0,0 @@ -use crate::{builtins::PyModule, PyRef, VirtualMachine}; -use std::os::unix::io::RawFd; - -pub fn raw_set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> { - use nix::fcntl; - let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); - let mut new_flags = flags; - new_flags.set(fcntl::FdFlag::FD_CLOEXEC, !inheritable); - if flags != new_flags { - fcntl::fcntl(fd, fcntl::FcntlArg::F_SETFD(new_flags))?; - } - Ok(()) -} - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} - -#[pymodule(name = "posix", with(super::os::_os))] -pub mod module { - use crate::{ - builtins::{PyDictRef, PyInt, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, - convert::{IntoPyException, ToPyObject, TryFromObject}, - function::{Either, KwArgs, OptionalArg}, - stdlib::os::{ - errno_err, DirFd, FollowSymlinks, OsPath, OsPathOrFd, SupportFunc, TargetIsDirectory, - _os, fs_metadata, IOErrorBuilder, - }, - types::{Constructor, Representable}, - utils::ToCString, - AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine, - }; - use bitflags::bitflags; - use nix::{ - fcntl, - unistd::{self, Gid, Pid, Uid}, - }; - use std::{ - env, - ffi::{CStr, CString}, - fs, io, - os::unix::io::RawFd, - }; - use strum_macros::{EnumIter, EnumString}; - - #[pyattr] - use libc::{PRIO_PGRP, PRIO_PROCESS, PRIO_USER}; - - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "macos" - ))] - #[pyattr] - use libc::{SEEK_DATA, SEEK_HOLE}; - - #[cfg(not(any(target_os = "redox", target_os = "freebsd")))] - #[pyattr] - use libc::O_DSYNC; - #[pyattr] - use libc::{O_CLOEXEC, O_NONBLOCK, WNOHANG}; - #[cfg(target_os = "macos")] - #[pyattr] - use libc::{O_EVTONLY, O_FSYNC, O_NOFOLLOW_ANY, O_SYMLINK}; - #[cfg(not(target_os = "redox"))] - #[pyattr] - use libc::{O_NDELAY, O_NOCTTY}; - - #[pyattr] - use libc::{RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NOW}; - - #[cfg(target_os = "linux")] - #[pyattr] - use libc::{GRND_NONBLOCK, GRND_RANDOM}; - - #[pyattr] - const EX_OK: i8 = exitcode::OK as i8; - #[pyattr] - const EX_USAGE: i8 = exitcode::USAGE as i8; - #[pyattr] - const EX_DATAERR: i8 = exitcode::DATAERR as i8; - #[pyattr] - const EX_NOINPUT: i8 = exitcode::NOINPUT as i8; - #[pyattr] - const EX_NOUSER: i8 = exitcode::NOUSER as i8; - #[pyattr] - const EX_NOHOST: i8 = exitcode::NOHOST as i8; - #[pyattr] - const EX_UNAVAILABLE: i8 = exitcode::UNAVAILABLE as i8; - #[pyattr] - const EX_SOFTWARE: i8 = exitcode::SOFTWARE as i8; - #[pyattr] - const EX_OSERR: i8 = exitcode::OSERR as i8; - #[pyattr] - const EX_OSFILE: i8 = exitcode::OSFILE as i8; - #[pyattr] - const EX_CANTCREAT: i8 = exitcode::CANTCREAT as i8; - #[pyattr] - const EX_IOERR: i8 = exitcode::IOERR as i8; - #[pyattr] - const EX_TEMPFAIL: i8 = exitcode::TEMPFAIL as i8; - #[pyattr] - const EX_PROTOCOL: i8 = exitcode::PROTOCOL as i8; - #[pyattr] - const EX_NOPERM: i8 = exitcode::NOPERM as i8; - #[pyattr] - const EX_CONFIG: i8 = exitcode::CONFIG as i8; - - #[cfg(any( - target_os = "macos", - target_os = "linux", - target_os = "android", - target_os = "freebsd", - target_os = "dragonfly", - target_os = "netbsd", - target_os = "macos" - ))] - #[pyattr] - const SCHED_RR: i32 = libc::SCHED_RR; - #[cfg(any( - target_os = "macos", - target_os = "linux", - target_os = "android", - target_os = "freebsd", - target_os = "dragonfly", - target_os = "netbsd", - target_os = "macos" - ))] - #[pyattr] - const SCHED_FIFO: i32 = libc::SCHED_FIFO; - #[cfg(any( - target_os = "macos", - target_os = "linux", - target_os = "freebsd", - target_os = "dragonfly", - target_os = "netbsd", - target_os = "macos" - ))] - #[pyattr] - const SCHED_OTHER: i32 = libc::SCHED_OTHER; - #[cfg(any(target_os = "linux", target_os = "android"))] - #[pyattr] - const SCHED_IDLE: i32 = libc::SCHED_IDLE; - #[cfg(any(target_os = "linux", target_os = "android"))] - #[pyattr] - const SCHED_BATCH: i32 = libc::SCHED_BATCH; - - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - #[pyattr] - const POSIX_SPAWN_OPEN: i32 = PosixSpawnFileActionIdentifier::Open as i32; - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - #[pyattr] - const POSIX_SPAWN_CLOSE: i32 = PosixSpawnFileActionIdentifier::Close as i32; - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - #[pyattr] - const POSIX_SPAWN_DUP2: i32 = PosixSpawnFileActionIdentifier::Dup2 as i32; - - #[cfg(target_os = "macos")] - #[pyattr] - const _COPYFILE_DATA: u32 = 1 << 3; - - // Flags for os_access - bitflags! { - #[derive(Copy, Clone, Debug, PartialEq)] - pub struct AccessFlags: u8 { - const F_OK = _os::F_OK; - const R_OK = _os::R_OK; - const W_OK = _os::W_OK; - const X_OK = _os::X_OK; - } - } - - struct Permissions { - is_readable: bool, - is_writable: bool, - is_executable: bool, - } - - fn get_permissions(mode: u32) -> Permissions { - Permissions { - is_readable: mode & 4 != 0, - is_writable: mode & 2 != 0, - is_executable: mode & 1 != 0, - } - } - - fn get_right_permission( - mode: u32, - file_owner: Uid, - file_group: Gid, - ) -> nix::Result<Permissions> { - let owner_mode = (mode & 0o700) >> 6; - let owner_permissions = get_permissions(owner_mode); - - let group_mode = (mode & 0o070) >> 3; - let group_permissions = get_permissions(group_mode); - - let others_mode = mode & 0o007; - let others_permissions = get_permissions(others_mode); - - let user_id = nix::unistd::getuid(); - let groups_ids = getgroups_impl()?; - - if file_owner == user_id { - Ok(owner_permissions) - } else if groups_ids.contains(&file_group) { - Ok(group_permissions) - } else { - Ok(others_permissions) - } - } - - #[cfg(any(target_os = "macos", target_os = "ios"))] - fn getgroups_impl() -> nix::Result<Vec<Gid>> { - use libc::{c_int, gid_t}; - use nix::errno::Errno; - use std::ptr; - let ret = unsafe { libc::getgroups(0, ptr::null_mut()) }; - let mut groups = Vec::<Gid>::with_capacity(Errno::result(ret)? as usize); - let ret = unsafe { - libc::getgroups( - groups.capacity() as c_int, - groups.as_mut_ptr() as *mut gid_t, - ) - }; - - Errno::result(ret).map(|s| { - unsafe { groups.set_len(s as usize) }; - groups - }) - } - - #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "redox")))] - use nix::unistd::getgroups as getgroups_impl; - - #[cfg(target_os = "redox")] - fn getgroups_impl() -> nix::Result<Vec<Gid>> { - Err(nix::Error::EOPNOTSUPP) - } - - #[pyfunction] - fn getgroups(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let group_ids = getgroups_impl().map_err(|e| e.into_pyexception(vm))?; - Ok(group_ids - .into_iter() - .map(|gid| vm.ctx.new_int(gid.as_raw()).into()) - .collect()) - } - - #[pyfunction] - pub(super) fn access(path: OsPath, mode: u8, vm: &VirtualMachine) -> PyResult<bool> { - use std::os::unix::fs::MetadataExt; - - let flags = AccessFlags::from_bits(mode).ok_or_else(|| { - vm.new_value_error( - "One of the flags is wrong, there are only 4 possibilities F_OK, R_OK, W_OK and X_OK" - .to_owned(), - ) - })?; - - let metadata = fs::metadata(path.path); - - // if it's only checking for F_OK - if flags == AccessFlags::F_OK { - return Ok(metadata.is_ok()); - } - - let metadata = metadata.map_err(|err| err.into_pyexception(vm))?; - - let user_id = metadata.uid(); - let group_id = metadata.gid(); - let mode = metadata.mode(); - - let perm = get_right_permission(mode, Uid::from_raw(user_id), Gid::from_raw(group_id)) - .map_err(|err| err.into_pyexception(vm))?; - - let r_ok = !flags.contains(AccessFlags::R_OK) || perm.is_readable; - let w_ok = !flags.contains(AccessFlags::W_OK) || perm.is_writable; - let x_ok = !flags.contains(AccessFlags::X_OK) || perm.is_executable; - - Ok(r_ok && w_ok && x_ok) - } - - #[pyattr] - fn environ(vm: &VirtualMachine) -> PyDictRef { - use rustpython_common::os::ffi::OsStringExt; - - let environ = vm.ctx.new_dict(); - for (key, value) in env::vars_os() { - let key: PyObjectRef = vm.ctx.new_bytes(key.into_vec()).into(); - let value: PyObjectRef = vm.ctx.new_bytes(value.into_vec()).into(); - environ.set_item(&*key, value, vm).unwrap(); - } - - environ - } - - #[derive(FromArgs)] - pub(super) struct SymlinkArgs { - src: OsPath, - dst: OsPath, - #[pyarg(flatten)] - _target_is_directory: TargetIsDirectory, - #[pyarg(flatten)] - dir_fd: DirFd<{ _os::SYMLINK_DIR_FD as usize }>, - } - - #[pyfunction] - pub(super) fn symlink(args: SymlinkArgs, vm: &VirtualMachine) -> PyResult<()> { - let src = args.src.into_cstring(vm)?; - let dst = args.dst.into_cstring(vm)?; - #[cfg(not(target_os = "redox"))] - { - nix::unistd::symlinkat(&*src, args.dir_fd.get_opt(), &*dst) - .map_err(|err| err.into_pyexception(vm)) - } - #[cfg(target_os = "redox")] - { - let [] = args.dir_fd.0; - let res = unsafe { libc::symlink(src.as_ptr(), dst.as_ptr()) }; - if res < 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn fchdir(fd: RawFd, vm: &VirtualMachine) -> PyResult<()> { - nix::unistd::fchdir(fd).map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn chroot(path: OsPath, vm: &VirtualMachine) -> PyResult<()> { - use crate::stdlib::os::IOErrorBuilder; - - nix::unistd::chroot(&*path.path).map_err(|err| { - // Use `From<nix::Error> for io::Error` when it is available - IOErrorBuilder::new(io::Error::from_raw_os_error(err as i32)) - .filename(path) - .into_pyexception(vm) - }) - } - - // As of now, redox does not seems to support chown command (cf. https://gitlab.redox-os.org/redox-os/coreutils , last checked on 05/07/2020) - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn chown( - path: OsPathOrFd, - uid: isize, - gid: isize, - dir_fd: DirFd<1>, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult<()> { - let uid = if uid >= 0 { - Some(nix::unistd::Uid::from_raw(uid as u32)) - } else if uid == -1 { - None - } else { - return Err(vm.new_os_error(String::from("Specified uid is not valid."))); - }; - - let gid = if gid >= 0 { - Some(nix::unistd::Gid::from_raw(gid as u32)) - } else if gid == -1 { - None - } else { - return Err(vm.new_os_error(String::from("Specified gid is not valid."))); - }; - - let flag = if follow_symlinks.0 { - nix::unistd::FchownatFlags::FollowSymlink - } else { - nix::unistd::FchownatFlags::NoFollowSymlink - }; - - let dir_fd = dir_fd.get_opt(); - match path { - OsPathOrFd::Path(ref p) => { - nix::unistd::fchownat(dir_fd, p.path.as_os_str(), uid, gid, flag) - } - OsPathOrFd::Fd(fd) => nix::unistd::fchown(fd, uid, gid), - } - .map_err(|err| { - // Use `From<nix::Error> for io::Error` when it is available - IOErrorBuilder::new(io::Error::from_raw_os_error(err as i32)) - .filename(path) - .into_pyexception(vm) - }) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn lchown(path: OsPath, uid: isize, gid: isize, vm: &VirtualMachine) -> PyResult<()> { - chown( - OsPathOrFd::Path(path), - uid, - gid, - DirFd::default(), - FollowSymlinks(false), - vm, - ) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn fchown(fd: i32, uid: isize, gid: isize, vm: &VirtualMachine) -> PyResult<()> { - chown( - OsPathOrFd::Fd(fd), - uid, - gid, - DirFd::default(), - FollowSymlinks(true), - vm, - ) - } - - #[derive(FromArgs)] - struct RegisterAtForkArgs { - #[pyarg(named, optional)] - before: OptionalArg<PyObjectRef>, - #[pyarg(named, optional)] - after_in_parent: OptionalArg<PyObjectRef>, - #[pyarg(named, optional)] - after_in_child: OptionalArg<PyObjectRef>, - } - - impl RegisterAtForkArgs { - fn into_validated( - self, - vm: &VirtualMachine, - ) -> PyResult<( - Option<PyObjectRef>, - Option<PyObjectRef>, - Option<PyObjectRef>, - )> { - fn into_option( - arg: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - match arg { - OptionalArg::Present(obj) => { - if !obj.is_callable() { - return Err(vm.new_type_error("Args must be callable".to_owned())); - } - Ok(Some(obj)) - } - OptionalArg::Missing => Ok(None), - } - } - let before = into_option(self.before, vm)?; - let after_in_parent = into_option(self.after_in_parent, vm)?; - let after_in_child = into_option(self.after_in_child, vm)?; - if before.is_none() && after_in_parent.is_none() && after_in_child.is_none() { - return Err(vm.new_type_error("At least one arg must be present".to_owned())); - } - Ok((before, after_in_parent, after_in_child)) - } - } - - #[pyfunction] - fn register_at_fork( - args: RegisterAtForkArgs, - _ignored: KwArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { - let (before, after_in_parent, after_in_child) = args.into_validated(vm)?; - - if let Some(before) = before { - vm.state.before_forkers.lock().push(before); - } - if let Some(after_in_parent) = after_in_parent { - vm.state.after_forkers_parent.lock().push(after_in_parent); - } - if let Some(after_in_child) = after_in_child { - vm.state.after_forkers_child.lock().push(after_in_child); - } - Ok(()) - } - - fn run_at_forkers(mut funcs: Vec<PyObjectRef>, reversed: bool, vm: &VirtualMachine) { - if !funcs.is_empty() { - if reversed { - funcs.reverse(); - } - for func in funcs.into_iter() { - if let Err(e) = func.call((), vm) { - let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); - vm.run_unraisable(e, Some("Exception ignored in".to_owned()), func); - if exit { - // Do nothing! - } - } - } - } - } - - fn py_os_before_fork(vm: &VirtualMachine) { - let before_forkers: Vec<PyObjectRef> = vm.state.before_forkers.lock().clone(); - // functions must be executed in reversed order as they are registered - // only for before_forkers, refer: test_register_at_fork in test_posix - - run_at_forkers(before_forkers, true, vm); - } - - fn py_os_after_fork_child(vm: &VirtualMachine) { - let after_forkers_child: Vec<PyObjectRef> = vm.state.after_forkers_child.lock().clone(); - run_at_forkers(after_forkers_child, false, vm); - } - - fn py_os_after_fork_parent(vm: &VirtualMachine) { - let after_forkers_parent: Vec<PyObjectRef> = vm.state.after_forkers_parent.lock().clone(); - run_at_forkers(after_forkers_parent, false, vm); - } - - #[pyfunction] - fn fork(vm: &VirtualMachine) -> i32 { - let pid: i32; - py_os_before_fork(vm); - unsafe { - pid = libc::fork(); - } - if pid == 0 { - py_os_after_fork_child(vm); - } else { - py_os_after_fork_parent(vm); - } - pid - } - - #[cfg(not(target_os = "redox"))] - const MKNOD_DIR_FD: bool = cfg!(not(target_vendor = "apple")); - - #[cfg(not(target_os = "redox"))] - #[derive(FromArgs)] - struct MknodArgs { - #[pyarg(any)] - path: OsPath, - #[pyarg(any)] - mode: libc::mode_t, - #[pyarg(any)] - device: libc::dev_t, - #[pyarg(flatten)] - dir_fd: DirFd<{ MKNOD_DIR_FD as usize }>, - } - - #[cfg(not(target_os = "redox"))] - impl MknodArgs { - fn _mknod(self, vm: &VirtualMachine) -> PyResult<i32> { - Ok(unsafe { - libc::mknod( - self.path.clone().into_cstring(vm)?.as_ptr(), - self.mode, - self.device, - ) - }) - } - #[cfg(not(target_vendor = "apple"))] - fn mknod(self, vm: &VirtualMachine) -> PyResult<()> { - let ret = match self.dir_fd.get_opt() { - None => self._mknod(vm)?, - Some(non_default_fd) => unsafe { - libc::mknodat( - non_default_fd, - self.path.clone().into_cstring(vm)?.as_ptr(), - self.mode, - self.device, - ) - }, - }; - if ret != 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - #[cfg(target_vendor = "apple")] - fn mknod(self, vm: &VirtualMachine) -> PyResult<()> { - let [] = self.dir_fd.0; - let ret = self._mknod(vm)?; - if ret != 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn mknod(args: MknodArgs, vm: &VirtualMachine) -> PyResult<()> { - args.mknod(vm) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn nice(increment: i32, vm: &VirtualMachine) -> PyResult<i32> { - use nix::errno::{errno, Errno}; - Errno::clear(); - let res = unsafe { libc::nice(increment) }; - if res == -1 && errno() != 0 { - Err(errno_err(vm)) - } else { - Ok(res) - } - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn sched_get_priority_max(policy: i32, vm: &VirtualMachine) -> PyResult<i32> { - let max = unsafe { libc::sched_get_priority_max(policy) }; - if max == -1 { - Err(errno_err(vm)) - } else { - Ok(max) - } - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn sched_get_priority_min(policy: i32, vm: &VirtualMachine) -> PyResult<i32> { - let min = unsafe { libc::sched_get_priority_min(policy) }; - if min == -1 { - Err(errno_err(vm)) - } else { - Ok(min) - } - } - - #[pyfunction] - fn sched_yield(vm: &VirtualMachine) -> PyResult<()> { - nix::sched::sched_yield().map_err(|e| e.into_pyexception(vm)) - } - - #[pyattr] - #[pyclass(name = "sched_param")] - #[derive(Debug, PyPayload)] - struct SchedParam { - sched_priority: PyObjectRef, - } - - impl TryFromObject for SchedParam { - fn try_from_object(_vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - Ok(SchedParam { - sched_priority: obj, - }) - } - } - - #[pyclass(with(Constructor, Representable))] - impl SchedParam { - #[pygetset] - fn sched_priority(&self, vm: &VirtualMachine) -> PyObjectRef { - self.sched_priority.clone().to_pyobject(vm) - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[cfg(not(target_env = "musl"))] - fn try_to_libc(&self, vm: &VirtualMachine) -> PyResult<libc::sched_param> { - use crate::AsObject; - let priority_class = self.sched_priority.class(); - let priority_type = priority_class.name(); - let priority = self.sched_priority.clone(); - let value = priority.downcast::<PyInt>().map_err(|_| { - vm.new_type_error(format!("an integer is required (got type {priority_type})")) - })?; - let sched_priority = value.try_to_primitive(vm)?; - Ok(libc::sched_param { sched_priority }) - } - } - - #[derive(FromArgs)] - pub struct SchedParamArg { - sched_priority: PyObjectRef, - } - impl Constructor for SchedParam { - type Args = SchedParamArg; - fn py_new(cls: PyTypeRef, arg: Self::Args, vm: &VirtualMachine) -> PyResult { - SchedParam { - sched_priority: arg.sched_priority, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl Representable for SchedParam { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let sched_priority_repr = zelf.sched_priority.repr(vm)?; - Ok(format!( - "posix.sched_param(sched_priority = {})", - sched_priority_repr.as_str() - )) - } - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[pyfunction] - fn sched_getscheduler(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult<i32> { - let policy = unsafe { libc::sched_getscheduler(pid) }; - if policy == -1 { - Err(errno_err(vm)) - } else { - Ok(policy) - } - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[derive(FromArgs)] - struct SchedSetschedulerArgs { - #[pyarg(positional)] - pid: i32, - #[pyarg(positional)] - policy: i32, - #[pyarg(positional)] - sched_param_obj: crate::PyRef<SchedParam>, - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[cfg(not(target_env = "musl"))] - #[pyfunction] - fn sched_setscheduler(args: SchedSetschedulerArgs, vm: &VirtualMachine) -> PyResult<i32> { - let libc_sched_param = args.sched_param_obj.try_to_libc(vm)?; - let policy = unsafe { libc::sched_setscheduler(args.pid, args.policy, &libc_sched_param) }; - if policy == -1 { - Err(errno_err(vm)) - } else { - Ok(policy) - } - } - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[pyfunction] - fn sched_getparam(pid: libc::pid_t, vm: &VirtualMachine) -> PyResult<SchedParam> { - let param = unsafe { - let mut param = std::mem::MaybeUninit::uninit(); - if -1 == libc::sched_getparam(pid, param.as_mut_ptr()) { - return Err(errno_err(vm)); - } - param.assume_init() - }; - Ok(SchedParam { - sched_priority: param.sched_priority.to_pyobject(vm), - }) - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[derive(FromArgs)] - struct SchedSetParamArgs { - #[pyarg(positional)] - pid: i32, - #[pyarg(positional)] - sched_param_obj: crate::PyRef<SchedParam>, - } - - #[cfg(any( - target_os = "linux", - target_os = "netbsd", - target_os = "freebsd", - target_os = "android" - ))] - #[cfg(not(target_env = "musl"))] - #[pyfunction] - fn sched_setparam(args: SchedSetParamArgs, vm: &VirtualMachine) -> PyResult<i32> { - let libc_sched_param = args.sched_param_obj.try_to_libc(vm)?; - let ret = unsafe { libc::sched_setparam(args.pid, &libc_sched_param) }; - if ret == -1 { - Err(errno_err(vm)) - } else { - Ok(ret) - } - } - - #[pyfunction] - fn get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult<bool> { - let flags = fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD); - match flags { - Ok(ret) => Ok((ret & libc::FD_CLOEXEC) == 0), - Err(err) => Err(err.into_pyexception(vm)), - } - } - - #[pyfunction] - fn set_inheritable(fd: i32, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { - super::raw_set_inheritable(fd, inheritable).map_err(|err| err.into_pyexception(vm)) - } - - #[pyfunction] - fn get_blocking(fd: RawFd, vm: &VirtualMachine) -> PyResult<bool> { - let flags = fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFL); - match flags { - Ok(ret) => Ok((ret & libc::O_NONBLOCK) == 0), - Err(err) => Err(err.into_pyexception(vm)), - } - } - - #[pyfunction] - fn set_blocking(fd: RawFd, blocking: bool, vm: &VirtualMachine) -> PyResult<()> { - let _set_flag = || { - use nix::fcntl::{fcntl, FcntlArg, OFlag}; - - let flags = OFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFL)?); - let mut new_flags = flags; - new_flags.set(OFlag::from_bits_truncate(libc::O_NONBLOCK), !blocking); - if flags != new_flags { - fcntl(fd, FcntlArg::F_SETFL(new_flags))?; - } - Ok(()) - }; - _set_flag().map_err(|err: nix::Error| err.into_pyexception(vm)) - } - - #[pyfunction] - fn pipe(vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { - use nix::unistd::close; - use nix::unistd::pipe; - let (rfd, wfd) = pipe().map_err(|err| err.into_pyexception(vm))?; - set_inheritable(rfd, false, vm) - .and_then(|_| set_inheritable(wfd, false, vm)) - .map_err(|err| { - let _ = close(rfd); - let _ = close(wfd); - err - })?; - Ok((rfd, wfd)) - } - - // cfg from nix - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "emscripten", - target_os = "freebsd", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd" - ))] - #[pyfunction] - fn pipe2(flags: libc::c_int, vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { - let oflags = fcntl::OFlag::from_bits_truncate(flags); - nix::unistd::pipe2(oflags).map_err(|err| err.into_pyexception(vm)) - } - - #[pyfunction] - fn system(command: PyStrRef, vm: &VirtualMachine) -> PyResult<i32> { - let cstr = command.to_cstring(vm)?; - let x = unsafe { libc::system(cstr.as_ptr()) }; - Ok(x) - } - - fn _chmod( - path: OsPath, - dir_fd: DirFd<0>, - mode: u32, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult<()> { - let [] = dir_fd.0; - let err_path = path.clone(); - let body = move || { - use std::os::unix::fs::PermissionsExt; - let meta = fs_metadata(&path, follow_symlinks.0)?; - let mut permissions = meta.permissions(); - permissions.set_mode(mode); - fs::set_permissions(&path, permissions) - }; - body().map_err(|err| { - IOErrorBuilder::new(err) - .filename(err_path) - .into_pyexception(vm) - }) - } - - #[cfg(not(target_os = "redox"))] - fn _fchmod(fd: RawFd, mode: u32, vm: &VirtualMachine) -> PyResult<()> { - nix::sys::stat::fchmod( - fd, - nix::sys::stat::Mode::from_bits(mode as libc::mode_t).unwrap(), - ) - .map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn chmod( - path: OsPathOrFd, - dir_fd: DirFd<0>, - mode: u32, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult<()> { - match path { - OsPathOrFd::Path(path) => _chmod(path, dir_fd, mode, follow_symlinks, vm), - OsPathOrFd::Fd(fd) => _fchmod(fd, mode, vm), - } - } - - #[cfg(target_os = "redox")] - #[pyfunction] - fn chmod( - path: OsPath, - dir_fd: DirFd<0>, - mode: u32, - follow_symlinks: FollowSymlinks, - vm: &VirtualMachine, - ) -> PyResult<()> { - _chmod(path, dir_fd, mode, follow_symlinks, vm) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn fchmod(fd: RawFd, mode: u32, vm: &VirtualMachine) -> PyResult<()> { - _fchmod(fd, mode, vm) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn lchmod(path: OsPath, mode: u32, vm: &VirtualMachine) -> PyResult<()> { - _chmod(path, DirFd::default(), mode, FollowSymlinks(false), vm) - } - - #[pyfunction] - fn execv( - path: OsPath, - argv: Either<PyListRef, PyTupleRef>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let path = path.into_cstring(vm)?; - - let argv = vm.extract_elements_with(argv.as_ref(), |obj| { - PyStrRef::try_from_object(vm, obj)?.to_cstring(vm) - })?; - let argv: Vec<&CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); - - let first = argv - .first() - .ok_or_else(|| vm.new_value_error("execv() arg 2 must not be empty".to_owned()))?; - if first.to_bytes().is_empty() { - return Err( - vm.new_value_error("execv() arg 2 first element cannot be empty".to_owned()) - ); - } - - unistd::execv(&path, &argv) - .map(|_ok| ()) - .map_err(|err| err.into_pyexception(vm)) - } - - #[pyfunction] - fn execve( - path: OsPath, - argv: Either<PyListRef, PyTupleRef>, - env: PyDictRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let path = path.into_cstring(vm)?; - - let argv = vm.extract_elements_with(argv.as_ref(), |obj| { - PyStrRef::try_from_object(vm, obj)?.to_cstring(vm) - })?; - let argv: Vec<&CStr> = argv.iter().map(|entry| entry.as_c_str()).collect(); - - let first = argv - .first() - .ok_or_else(|| vm.new_value_error("execve() arg 2 must not be empty".to_owned()))?; - - if first.to_bytes().is_empty() { - return Err( - vm.new_value_error("execve() arg 2 first element cannot be empty".to_owned()) - ); - } - - let env = env - .into_iter() - .map(|(k, v)| -> PyResult<_> { - let (key, value) = ( - OsPath::try_from_object(vm, k)?.into_bytes(), - OsPath::try_from_object(vm, v)?.into_bytes(), - ); - - if memchr::memchr(b'=', &key).is_some() { - return Err(vm.new_value_error("illegal environment variable name".to_owned())); - } - - let mut entry = key; - entry.push(b'='); - entry.extend_from_slice(&value); - - CString::new(entry).map_err(|err| err.into_pyexception(vm)) - }) - .collect::<Result<Vec<_>, _>>()?; - - let env: Vec<&CStr> = env.iter().map(|entry| entry.as_c_str()).collect(); - - unistd::execve(&path, &argv, &env).map_err(|err| err.into_pyexception(vm))?; - Ok(()) - } - - #[pyfunction] - fn getppid(vm: &VirtualMachine) -> PyObjectRef { - let ppid = unistd::getppid().as_raw(); - vm.ctx.new_int(ppid).into() - } - - #[pyfunction] - fn getgid(vm: &VirtualMachine) -> PyObjectRef { - let gid = unistd::getgid().as_raw(); - vm.ctx.new_int(gid).into() - } - - #[pyfunction] - fn getegid(vm: &VirtualMachine) -> PyObjectRef { - let egid = unistd::getegid().as_raw(); - vm.ctx.new_int(egid).into() - } - - #[pyfunction] - fn getpgid(pid: u32, vm: &VirtualMachine) -> PyResult { - let pgid = - unistd::getpgid(Some(Pid::from_raw(pid as i32))).map_err(|e| e.into_pyexception(vm))?; - Ok(vm.new_pyobj(pgid.as_raw())) - } - - #[pyfunction] - fn getpgrp(vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.new_int(unistd::getpgrp().as_raw()).into() - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn getsid(pid: u32, vm: &VirtualMachine) -> PyResult { - let sid = - unistd::getsid(Some(Pid::from_raw(pid as i32))).map_err(|e| e.into_pyexception(vm))?; - Ok(vm.new_pyobj(sid.as_raw())) - } - - #[pyfunction] - fn getuid(vm: &VirtualMachine) -> PyObjectRef { - let uid = unistd::getuid().as_raw(); - vm.ctx.new_int(uid).into() - } - - #[pyfunction] - fn geteuid(vm: &VirtualMachine) -> PyObjectRef { - let euid = unistd::geteuid().as_raw(); - vm.ctx.new_int(euid).into() - } - - #[pyfunction] - fn setgid(gid: Option<Gid>, vm: &VirtualMachine) -> PyResult<()> { - let gid = gid.ok_or_else(|| vm.new_os_error("Invalid argument".to_string()))?; - unistd::setgid(gid).map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn setegid(egid: Option<Gid>, vm: &VirtualMachine) -> PyResult<()> { - let egid = egid.ok_or_else(|| vm.new_os_error("Invalid argument".to_string()))?; - unistd::setegid(egid).map_err(|err| err.into_pyexception(vm)) - } - - #[pyfunction] - fn setpgid(pid: u32, pgid: u32, vm: &VirtualMachine) -> PyResult<()> { - unistd::setpgid(Pid::from_raw(pid as i32), Pid::from_raw(pgid as i32)) - .map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn setsid(vm: &VirtualMachine) -> PyResult<()> { - unistd::setsid() - .map(|_ok| ()) - .map_err(|err| err.into_pyexception(vm)) - } - - fn try_from_id(vm: &VirtualMachine, obj: PyObjectRef, typ_name: &str) -> PyResult<Option<u32>> { - use std::cmp::Ordering; - let i = obj - .try_to_ref::<PyInt>(vm) - .map_err(|_| { - vm.new_type_error(format!( - "an integer is required (got type {})", - obj.class().name() - )) - })? - .try_to_primitive::<i64>(vm)?; - - match i.cmp(&-1) { - Ordering::Greater => Ok(Some(i.try_into().map_err(|_| { - vm.new_overflow_error(format!("{typ_name} is larger than maximum")) - })?)), - Ordering::Less => { - Err(vm.new_overflow_error(format!("{typ_name} is less than minimum"))) - } - Ordering::Equal => Ok(None), // -1 means does not change the value - } - } - - impl TryFromObject for Option<Uid> { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - Ok(try_from_id(vm, obj, "uid")?.map(Uid::from_raw)) - } - } - - impl TryFromObject for Option<Gid> { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - Ok(try_from_id(vm, obj, "gid")?.map(Gid::from_raw)) - } - } - - #[pyfunction] - fn setuid(uid: Option<Uid>, vm: &VirtualMachine) -> PyResult<()> { - let uid = uid.ok_or_else(|| vm.new_os_error("Invalid argument".to_string()))?; - unistd::setuid(uid).map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn seteuid(euid: Option<Uid>, vm: &VirtualMachine) -> PyResult<()> { - let euid = euid.ok_or_else(|| vm.new_os_error("Invalid argument".to_string()))?; - unistd::seteuid(euid).map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn setreuid(ruid: Option<Uid>, euid: Option<Uid>, vm: &VirtualMachine) -> PyResult<()> { - if let Some(ruid) = ruid { - unistd::setuid(ruid).map_err(|err| err.into_pyexception(vm))?; - } - if let Some(euid) = euid { - unistd::seteuid(euid).map_err(|err| err.into_pyexception(vm))?; - } - Ok(()) - } - - // cfg from nix - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" - ))] - #[pyfunction] - fn setresuid( - ruid: Option<Uid>, - euid: Option<Uid>, - suid: Option<Uid>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let unwrap_or_unchanged = - |u: Option<Uid>| u.unwrap_or_else(|| Uid::from_raw(libc::uid_t::MAX)); - unistd::setresuid( - unwrap_or_unchanged(ruid), - unwrap_or_unchanged(euid), - unwrap_or_unchanged(suid), - ) - .map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn openpty(vm: &VirtualMachine) -> PyResult<(i32, i32)> { - let r = nix::pty::openpty(None, None).map_err(|err| err.into_pyexception(vm))?; - for fd in &[r.master, r.slave] { - super::raw_set_inheritable(*fd, false).map_err(|e| e.into_pyexception(vm))?; - } - Ok((r.master, r.slave)) - } - - #[pyfunction] - fn ttyname(fd: i32, vm: &VirtualMachine) -> PyResult { - let name = unistd::ttyname(fd).map_err(|e| e.into_pyexception(vm))?; - let name = name.into_os_string().into_string().unwrap(); - Ok(vm.ctx.new_str(name).into()) - } - - #[pyfunction] - fn umask(mask: libc::mode_t) -> libc::mode_t { - unsafe { libc::umask(mask) } - } - - #[pyfunction] - fn uname(vm: &VirtualMachine) -> PyResult<_os::UnameResult> { - let info = uname::uname().map_err(|err| err.into_pyexception(vm))?; - Ok(_os::UnameResult { - sysname: info.sysname, - nodename: info.nodename, - release: info.release, - version: info.version, - machine: info.machine, - }) - } - - #[pyfunction] - fn sync() { - #[cfg(not(any(target_os = "redox", target_os = "android")))] - unsafe { - libc::sync(); - } - } - - // cfg from nix - #[cfg(any(target_os = "android", target_os = "linux", target_os = "openbsd"))] - #[pyfunction] - fn getresuid(vm: &VirtualMachine) -> PyResult<(u32, u32, u32)> { - let ret = unistd::getresuid().map_err(|e| e.into_pyexception(vm))?; - Ok(( - ret.real.as_raw(), - ret.effective.as_raw(), - ret.saved.as_raw(), - )) - } - - // cfg from nix - #[cfg(any(target_os = "android", target_os = "linux", target_os = "openbsd"))] - #[pyfunction] - fn getresgid(vm: &VirtualMachine) -> PyResult<(u32, u32, u32)> { - let ret = unistd::getresgid().map_err(|e| e.into_pyexception(vm))?; - Ok(( - ret.real.as_raw(), - ret.effective.as_raw(), - ret.saved.as_raw(), - )) - } - - // cfg from nix - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" - ))] - #[pyfunction] - fn setresgid( - rgid: Option<Gid>, - egid: Option<Gid>, - sgid: Option<Gid>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let unwrap_or_unchanged = - |u: Option<Gid>| u.unwrap_or_else(|| Gid::from_raw(libc::gid_t::MAX)); - unistd::setresgid( - unwrap_or_unchanged(rgid), - unwrap_or_unchanged(egid), - unwrap_or_unchanged(sgid), - ) - .map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn setregid(rgid: Option<Gid>, egid: Option<Gid>, vm: &VirtualMachine) -> PyResult<()> { - if let Some(rgid) = rgid { - unistd::setgid(rgid).map_err(|err| err.into_pyexception(vm))?; - } - if let Some(egid) = egid { - unistd::setegid(egid).map_err(|err| err.into_pyexception(vm))?; - } - Ok(()) - } - - // cfg from nix - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" - ))] - #[pyfunction] - fn initgroups(user_name: PyStrRef, gid: Option<Gid>, vm: &VirtualMachine) -> PyResult<()> { - let user = CString::new(user_name.as_str()).unwrap(); - let gid = gid.ok_or_else(|| vm.new_os_error("Invalid argument".to_string()))?; - unistd::initgroups(&user, gid).map_err(|err| err.into_pyexception(vm)) - } - - // cfg from nix - #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "redox")))] - #[pyfunction] - fn setgroups( - group_ids: crate::function::ArgIterable<Option<Gid>>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let gids = group_ids - .iter(vm)? - .collect::<Result<Option<Vec<_>>, _>>()? - .ok_or_else(|| vm.new_os_error("Invalid argument".to_string()))?; - let ret = unistd::setgroups(&gids); - ret.map_err(|err| err.into_pyexception(vm)) - } - - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - fn envp_from_dict( - env: crate::function::ArgMapping, - vm: &VirtualMachine, - ) -> PyResult<Vec<CString>> { - let keys = env.mapping().keys(vm)?; - let values = env.mapping().values(vm)?; - - let keys = PyListRef::try_from_object(vm, keys) - .map_err(|_| vm.new_type_error("env.keys() is not a list".to_owned()))? - .borrow_vec() - .to_vec(); - let values = PyListRef::try_from_object(vm, values) - .map_err(|_| vm.new_type_error("env.values() is not a list".to_owned()))? - .borrow_vec() - .to_vec(); - - keys.into_iter() - .zip(values) - .map(|(k, v)| { - let k = OsPath::try_from_object(vm, k)?.into_bytes(); - let v = OsPath::try_from_object(vm, v)?.into_bytes(); - if k.contains(&0) { - return Err( - vm.new_value_error("envp dict key cannot contain a nul byte".to_owned()) - ); - } - if k.contains(&b'=') { - return Err(vm.new_value_error( - "envp dict key cannot contain a '=' character".to_owned(), - )); - } - if v.contains(&0) { - return Err( - vm.new_value_error("envp dict value cannot contain a nul byte".to_owned()) - ); - } - let mut env = k; - env.push(b'='); - env.extend(v); - Ok(unsafe { CString::from_vec_unchecked(env) }) - }) - .collect() - } - - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - #[derive(FromArgs)] - pub(super) struct PosixSpawnArgs { - #[pyarg(positional)] - path: OsPath, - #[pyarg(positional)] - args: crate::function::ArgIterable<OsPath>, - #[pyarg(positional)] - env: crate::function::ArgMapping, - #[pyarg(named, default)] - file_actions: Option<crate::function::ArgIterable<PyTupleRef>>, - #[pyarg(named, default)] - setsigdef: Option<crate::function::ArgIterable<i32>>, - } - - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - #[derive(num_enum::IntoPrimitive, num_enum::TryFromPrimitive)] - #[repr(i32)] - enum PosixSpawnFileActionIdentifier { - Open, - Close, - Dup2, - } - - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - impl PosixSpawnArgs { - fn spawn(self, spawnp: bool, vm: &VirtualMachine) -> PyResult<libc::pid_t> { - use crate::TryFromBorrowedObject; - - let path = self - .path - .clone() - .into_cstring(vm) - .map_err(|_| vm.new_value_error("path should not have nul bytes".to_owned()))?; - - let mut file_actions = unsafe { - let mut fa = std::mem::MaybeUninit::uninit(); - assert!(libc::posix_spawn_file_actions_init(fa.as_mut_ptr()) == 0); - fa.assume_init() - }; - if let Some(it) = self.file_actions { - for action in it.iter(vm)? { - let action = action?; - let (id, args) = action.split_first().ok_or_else(|| { - vm.new_type_error( - "Each file_actions element must be a non-empty tuple".to_owned(), - ) - })?; - let id = i32::try_from_borrowed_object(vm, id)?; - let id = PosixSpawnFileActionIdentifier::try_from(id).map_err(|_| { - vm.new_type_error("Unknown file_actions identifier".to_owned()) - })?; - let args: crate::function::FuncArgs = args.to_vec().into(); - let ret = match id { - PosixSpawnFileActionIdentifier::Open => { - let (fd, path, oflag, mode): (_, OsPath, _, _) = args.bind(vm)?; - let path = CString::new(path.into_bytes()).map_err(|_| { - vm.new_value_error( - "POSIX_SPAWN_OPEN path should not have nul bytes".to_owned(), - ) - })?; - unsafe { - libc::posix_spawn_file_actions_addopen( - &mut file_actions, - fd, - path.as_ptr(), - oflag, - mode, - ) - } - } - PosixSpawnFileActionIdentifier::Close => { - let (fd,) = args.bind(vm)?; - unsafe { - libc::posix_spawn_file_actions_addclose(&mut file_actions, fd) - } - } - PosixSpawnFileActionIdentifier::Dup2 => { - let (fd, newfd) = args.bind(vm)?; - unsafe { - libc::posix_spawn_file_actions_adddup2(&mut file_actions, fd, newfd) - } - } - }; - if ret != 0 { - return Err(IOErrorBuilder::new(std::io::Error::from_raw_os_error(ret)) - .filename(self.path) - .into_pyexception(vm)); - } - } - } - - let mut attrp = unsafe { - let mut sa = std::mem::MaybeUninit::uninit(); - assert!(libc::posix_spawnattr_init(sa.as_mut_ptr()) == 0); - sa.assume_init() - }; - if let Some(sigs) = self.setsigdef { - use nix::sys::signal; - let mut set = signal::SigSet::empty(); - for sig in sigs.iter(vm)? { - let sig = sig?; - let sig = signal::Signal::try_from(sig).map_err(|_| { - vm.new_value_error(format!("signal number {sig} out of range")) - })?; - set.add(sig); - } - assert!( - unsafe { libc::posix_spawnattr_setsigdefault(&mut attrp, set.as_ref()) } == 0 - ); - } - - let mut args: Vec<CString> = self - .args - .iter(vm)? - .map(|res| { - CString::new(res?.into_bytes()).map_err(|_| { - vm.new_value_error("path should not have nul bytes".to_owned()) - }) - }) - .collect::<Result<_, _>>()?; - let argv: Vec<*mut libc::c_char> = args - .iter_mut() - .map(|s| s.as_ptr() as _) - .chain(std::iter::once(std::ptr::null_mut())) - .collect(); - let mut env = envp_from_dict(self.env, vm)?; - let envp: Vec<*mut libc::c_char> = env - .iter_mut() - .map(|s| s.as_ptr() as _) - .chain(std::iter::once(std::ptr::null_mut())) - .collect(); - - let mut pid = 0; - let ret = unsafe { - if spawnp { - libc::posix_spawnp( - &mut pid, - path.as_ptr(), - &file_actions, - &attrp, - argv.as_ptr(), - envp.as_ptr(), - ) - } else { - libc::posix_spawn( - &mut pid, - path.as_ptr(), - &file_actions, - &attrp, - argv.as_ptr(), - envp.as_ptr(), - ) - } - }; - - if ret == 0 { - Ok(pid) - } else { - Err(IOErrorBuilder::new(std::io::Error::from_raw_os_error(ret)) - .filename(self.path) - .into_pyexception(vm)) - } - } - } - - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - #[pyfunction] - fn posix_spawn(args: PosixSpawnArgs, vm: &VirtualMachine) -> PyResult<libc::pid_t> { - args.spawn(false, vm) - } - #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] - #[pyfunction] - fn posix_spawnp(args: PosixSpawnArgs, vm: &VirtualMachine) -> PyResult<libc::pid_t> { - args.spawn(true, vm) - } - - #[pyfunction(name = "WIFSIGNALED")] - fn wifsignaled(status: i32) -> bool { - libc::WIFSIGNALED(status) - } - #[pyfunction(name = "WIFSTOPPED")] - fn wifstopped(status: i32) -> bool { - libc::WIFSTOPPED(status) - } - #[pyfunction(name = "WIFEXITED")] - fn wifexited(status: i32) -> bool { - libc::WIFEXITED(status) - } - #[pyfunction(name = "WTERMSIG")] - fn wtermsig(status: i32) -> i32 { - libc::WTERMSIG(status) - } - #[pyfunction(name = "WSTOPSIG")] - fn wstopsig(status: i32) -> i32 { - libc::WSTOPSIG(status) - } - #[pyfunction(name = "WEXITSTATUS")] - fn wexitstatus(status: i32) -> i32 { - libc::WEXITSTATUS(status) - } - - #[pyfunction] - fn waitpid(pid: libc::pid_t, opt: i32, vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { - let mut status = 0; - let pid = unsafe { libc::waitpid(pid, &mut status, opt) }; - let pid = nix::Error::result(pid).map_err(|err| err.into_pyexception(vm))?; - Ok((pid, status)) - } - #[pyfunction] - fn wait(vm: &VirtualMachine) -> PyResult<(libc::pid_t, i32)> { - waitpid(-1, 0, vm) - } - - #[pyfunction] - fn kill(pid: i32, sig: isize, vm: &VirtualMachine) -> PyResult<()> { - { - let ret = unsafe { libc::kill(pid, sig as i32) }; - if ret == -1 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - } - - #[pyfunction] - fn get_terminal_size( - fd: OptionalArg<i32>, - vm: &VirtualMachine, - ) -> PyResult<_os::PyTerminalSize> { - let (columns, lines) = { - nix::ioctl_read_bad!(winsz, libc::TIOCGWINSZ, libc::winsize); - let mut w = libc::winsize { - ws_row: 0, - ws_col: 0, - ws_xpixel: 0, - ws_ypixel: 0, - }; - unsafe { winsz(fd.unwrap_or(libc::STDOUT_FILENO), &mut w) } - .map_err(|err| err.into_pyexception(vm))?; - (w.ws_col.into(), w.ws_row.into()) - }; - Ok(_os::PyTerminalSize { columns, lines }) - } - - // from libstd: - // https://github.com/rust-lang/rust/blob/daecab3a784f28082df90cebb204998051f3557d/src/libstd/sys/unix/fs.rs#L1251 - #[cfg(target_os = "macos")] - extern "C" { - fn fcopyfile( - in_fd: libc::c_int, - out_fd: libc::c_int, - state: *mut libc::c_void, // copyfile_state_t (unused) - flags: u32, // copyfile_flags_t - ) -> libc::c_int; - } - - #[cfg(target_os = "macos")] - #[pyfunction] - fn _fcopyfile(in_fd: i32, out_fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult<()> { - let ret = unsafe { fcopyfile(in_fd, out_fd, std::ptr::null_mut(), flags as u32) }; - if ret < 0 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - - #[pyfunction] - fn dup(fd: i32, vm: &VirtualMachine) -> PyResult<i32> { - let fd = nix::unistd::dup(fd).map_err(|e| e.into_pyexception(vm))?; - super::raw_set_inheritable(fd, false) - .map(|()| fd) - .map_err(|e| { - let _ = nix::unistd::close(fd); - e.into_pyexception(vm) - }) - } - - #[derive(FromArgs)] - struct Dup2Args { - #[pyarg(positional)] - fd: i32, - #[pyarg(positional)] - fd2: i32, - #[pyarg(any, default = "true")] - inheritable: bool, - } - - #[pyfunction] - fn dup2(args: Dup2Args, vm: &VirtualMachine) -> PyResult<i32> { - let fd = nix::unistd::dup2(args.fd, args.fd2).map_err(|e| e.into_pyexception(vm))?; - if !args.inheritable { - super::raw_set_inheritable(fd, false).map_err(|e| { - let _ = nix::unistd::close(fd); - e.into_pyexception(vm) - })? - } - Ok(fd) - } - - pub(crate) fn support_funcs() -> Vec<SupportFunc> { - vec![ - SupportFunc::new("chmod", Some(false), Some(false), Some(false)), - #[cfg(not(target_os = "redox"))] - SupportFunc::new("chroot", Some(false), None, None), - #[cfg(not(target_os = "redox"))] - SupportFunc::new("chown", Some(true), Some(true), Some(true)), - #[cfg(not(target_os = "redox"))] - SupportFunc::new("lchown", None, None, None), - #[cfg(not(target_os = "redox"))] - SupportFunc::new("fchown", Some(true), None, Some(true)), - #[cfg(not(target_os = "redox"))] - SupportFunc::new("mknod", Some(true), Some(MKNOD_DIR_FD), Some(false)), - SupportFunc::new("umask", Some(false), Some(false), Some(false)), - SupportFunc::new("execv", None, None, None), - SupportFunc::new("pathconf", Some(true), None, None), - ] - } - - #[pyfunction] - fn getlogin(vm: &VirtualMachine) -> PyResult<String> { - // Get a pointer to the login name string. The string is statically - // allocated and might be overwritten on subsequent calls to this - // function or to `cuserid()`. See man getlogin(3) for more information. - let ptr = unsafe { libc::getlogin() }; - if ptr.is_null() { - return Err(vm.new_os_error("unable to determine login name".to_owned())); - } - let slice = unsafe { CStr::from_ptr(ptr) }; - slice - .to_str() - .map(|s| s.to_owned()) - .map_err(|e| vm.new_unicode_decode_error(format!("unable to decode login name: {e}"))) - } - - // cfg from nix - #[cfg(any( - target_os = "android", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" - ))] - #[pyfunction] - fn getgrouplist(user: PyStrRef, group: u32, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let user = CString::new(user.as_str()).unwrap(); - let gid = Gid::from_raw(group); - let group_ids = unistd::getgrouplist(&user, gid).map_err(|err| err.into_pyexception(vm))?; - Ok(group_ids - .into_iter() - .map(|gid| vm.new_pyobj(gid.as_raw())) - .collect()) - } - - #[cfg(not(target_os = "redox"))] - cfg_if::cfg_if! { - if #[cfg(all(target_os = "linux", target_env = "gnu"))] { - type PriorityWhichType = libc::__priority_which_t; - } else { - type PriorityWhichType = libc::c_int; - } - } - #[cfg(not(target_os = "redox"))] - cfg_if::cfg_if! { - if #[cfg(target_os = "freebsd")] { - type PriorityWhoType = i32; - } else { - type PriorityWhoType = u32; - } - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn getpriority( - which: PriorityWhichType, - who: PriorityWhoType, - vm: &VirtualMachine, - ) -> PyResult { - use nix::errno::{errno, Errno}; - Errno::clear(); - let retval = unsafe { libc::getpriority(which, who) }; - if errno() != 0 { - Err(errno_err(vm)) - } else { - Ok(vm.ctx.new_int(retval).into()) - } - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn setpriority( - which: PriorityWhichType, - who: PriorityWhoType, - priority: i32, - vm: &VirtualMachine, - ) -> PyResult<()> { - let retval = unsafe { libc::setpriority(which, who, priority) }; - if retval == -1 { - Err(errno_err(vm)) - } else { - Ok(()) - } - } - - struct ConfName(i32); - - impl TryFromObject for ConfName { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let i = match obj.downcast::<PyInt>() { - Ok(int) => int.try_to_primitive(vm)?, - Err(obj) => { - let s = PyStrRef::try_from_object(vm, obj)?; - s.as_str().parse::<PathconfVar>().map_err(|_| { - vm.new_value_error("unrecognized configuration name".to_string()) - })? as i32 - } - }; - Ok(Self(i)) - } - } - - // Copy from [nix::unistd::PathconfVar](https://docs.rs/nix/0.21.0/nix/unistd/enum.PathconfVar.html) - // Change enum name to fit python doc - #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, EnumIter, EnumString)] - #[repr(i32)] - #[allow(non_camel_case_types)] - pub enum PathconfVar { - #[cfg(any( - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_os = "redox" - ))] - /// Minimum number of bits needed to represent, as a signed integer value, - /// the maximum size of a regular file allowed in the specified directory. - PC_FILESIZEBITS = libc::_PC_FILESIZEBITS, - /// Maximum number of links to a single file. - PC_LINK_MAX = libc::_PC_LINK_MAX, - /// Maximum number of bytes in a terminal canonical input line. - PC_MAX_CANON = libc::_PC_MAX_CANON, - /// Minimum number of bytes for which space is available in a terminal input - /// queue; therefore, the maximum number of bytes a conforming application - /// may require to be typed as input before reading them. - PC_MAX_INPUT = libc::_PC_MAX_INPUT, - /// Maximum number of bytes in a filename (not including the terminating - /// null of a filename string). - PC_NAME_MAX = libc::_PC_NAME_MAX, - /// Maximum number of bytes the implementation will store as a pathname in a - /// user-supplied buffer of unspecified size, including the terminating null - /// character. Minimum number the implementation will accept as the maximum - /// number of bytes in a pathname. - PC_PATH_MAX = libc::_PC_PATH_MAX, - /// Maximum number of bytes that is guaranteed to be atomic when writing to - /// a pipe. - PC_PIPE_BUF = libc::_PC_PIPE_BUF, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "illumos", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_os = "redox", - target_os = "solaris" - ))] - /// Symbolic links can be created. - PC_2_SYMLINKS = libc::_PC_2_SYMLINKS, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd", - target_os = "redox" - ))] - /// Minimum number of bytes of storage actually allocated for any portion of - /// a file. - PC_ALLOC_SIZE_MIN = libc::_PC_ALLOC_SIZE_MIN, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd" - ))] - /// Recommended increment for file transfer sizes between the - /// `POSIX_REC_MIN_XFER_SIZE` and `POSIX_REC_MAX_XFER_SIZE` values. - PC_REC_INCR_XFER_SIZE = libc::_PC_REC_INCR_XFER_SIZE, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd", - target_os = "redox" - ))] - /// Maximum recommended file transfer size. - PC_REC_MAX_XFER_SIZE = libc::_PC_REC_MAX_XFER_SIZE, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd", - target_os = "redox" - ))] - /// Minimum recommended file transfer size. - PC_REC_MIN_XFER_SIZE = libc::_PC_REC_MIN_XFER_SIZE, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "openbsd", - target_os = "redox" - ))] - /// Recommended file transfer buffer alignment. - PC_REC_XFER_ALIGN = libc::_PC_REC_XFER_ALIGN, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "illumos", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_os = "redox", - target_os = "solaris" - ))] - /// Maximum number of bytes in a symbolic link. - PC_SYMLINK_MAX = libc::_PC_SYMLINK_MAX, - /// The use of `chown` and `fchown` is restricted to a process with - /// appropriate privileges, and to changing the group ID of a file only to - /// the effective group ID of the process or to one of its supplementary - /// group IDs. - PC_CHOWN_RESTRICTED = libc::_PC_CHOWN_RESTRICTED, - /// Pathname components longer than {NAME_MAX} generate an error. - PC_NO_TRUNC = libc::_PC_NO_TRUNC, - /// This symbol shall be defined to be the value of a character that shall - /// disable terminal special character handling. - PC_VDISABLE = libc::_PC_VDISABLE, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "illumos", - target_os = "linux", - target_os = "openbsd", - target_os = "redox", - target_os = "solaris" - ))] - /// Asynchronous input or output operations may be performed for the - /// associated file. - PC_ASYNC_IO = libc::_PC_ASYNC_IO, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "illumos", - target_os = "linux", - target_os = "openbsd", - target_os = "redox", - target_os = "solaris" - ))] - /// Prioritized input or output operations may be performed for the - /// associated file. - PC_PRIO_IO = libc::_PC_PRIO_IO, - #[cfg(any( - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "illumos", - target_os = "linux", - target_os = "netbsd", - target_os = "openbsd", - target_os = "redox", - target_os = "solaris" - ))] - /// Synchronized input or output operations may be performed for the - /// associated file. - PC_SYNC_IO = libc::_PC_SYNC_IO, - #[cfg(any(target_os = "dragonfly", target_os = "openbsd"))] - /// The resolution in nanoseconds for all file timestamps. - PC_TIMESTAMP_RESOLUTION = libc::_PC_TIMESTAMP_RESOLUTION, - } - - #[cfg(unix)] - #[pyfunction] - fn pathconf( - path: OsPathOrFd, - ConfName(name): ConfName, - vm: &VirtualMachine, - ) -> PyResult<Option<libc::c_long>> { - use nix::errno::{self, Errno}; - - Errno::clear(); - debug_assert_eq!(errno::errno(), 0); - let raw = match path { - OsPathOrFd::Path(path) => { - let path = CString::new(path.into_bytes()) - .map_err(|_| vm.new_value_error("embedded null character".to_owned()))?; - unsafe { libc::pathconf(path.as_ptr(), name) } - } - OsPathOrFd::Fd(fd) => unsafe { libc::fpathconf(fd, name) }, - }; - - if raw == -1 { - if errno::errno() == 0 { - Ok(None) - } else { - Err(io::Error::from(Errno::last()).into_pyexception(vm)) - } - } else { - Ok(Some(raw)) - } - } - - #[pyfunction] - fn fpathconf(fd: i32, name: ConfName, vm: &VirtualMachine) -> PyResult<Option<libc::c_long>> { - pathconf(OsPathOrFd::Fd(fd), name, vm) - } - - #[pyattr] - fn pathconf_names(vm: &VirtualMachine) -> PyDictRef { - use strum::IntoEnumIterator; - let pathname = vm.ctx.new_dict(); - for variant in PathconfVar::iter() { - // get the name of variant as a string to use as the dictionary key - let key = vm.ctx.new_str(format!("{:?}", variant)); - // get the enum from the string and convert it to an integer for the dictionary value - let value = vm.ctx.new_int(variant as u8); - pathname - .set_item(&*key, value.into(), vm) - .expect("dict set_item unexpectedly failed"); - } - pathname - } - - #[cfg(any(target_os = "linux", target_os = "macos"))] - #[derive(FromArgs)] - struct SendFileArgs { - out_fd: i32, - in_fd: i32, - offset: crate::common::crt_fd::Offset, - count: i64, - #[cfg(target_os = "macos")] - #[pyarg(any, optional)] - headers: OptionalArg<PyObjectRef>, - #[cfg(target_os = "macos")] - #[pyarg(any, optional)] - trailers: OptionalArg<PyObjectRef>, - #[cfg(target_os = "macos")] - #[allow(dead_code)] - #[pyarg(any, default)] - // TODO: not implemented - flags: OptionalArg<i32>, - } - - #[cfg(target_os = "linux")] - #[pyfunction] - fn sendfile(args: SendFileArgs, vm: &VirtualMachine) -> PyResult { - let mut file_offset = args.offset; - - let res = nix::sys::sendfile::sendfile( - args.out_fd, - args.in_fd, - Some(&mut file_offset), - args.count as usize, - ) - .map_err(|err| err.into_pyexception(vm))?; - Ok(vm.ctx.new_int(res as u64).into()) - } - - #[cfg(target_os = "macos")] - fn _extract_vec_bytes( - x: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult<Option<Vec<crate::function::ArgBytesLike>>> { - x.into_option() - .map(|x| { - let v: Vec<crate::function::ArgBytesLike> = x.try_to_value(vm)?; - Ok(if v.is_empty() { None } else { Some(v) }) - }) - .transpose() - .map(Option::flatten) - } - - #[cfg(target_os = "macos")] - #[pyfunction] - fn sendfile(args: SendFileArgs, vm: &VirtualMachine) -> PyResult { - let headers = _extract_vec_bytes(args.headers, vm)?; - let count = headers - .as_ref() - .map(|v| v.iter().map(|s| s.len()).sum()) - .unwrap_or(0) as i64 - + args.count; - - let headers = headers - .as_ref() - .map(|v| v.iter().map(|b| b.borrow_buf()).collect::<Vec<_>>()); - let headers = headers - .as_ref() - .map(|v| v.iter().map(|borrowed| &**borrowed).collect::<Vec<_>>()); - let headers = headers.as_deref(); - - let trailers = _extract_vec_bytes(args.trailers, vm)?; - let trailers = trailers - .as_ref() - .map(|v| v.iter().map(|b| b.borrow_buf()).collect::<Vec<_>>()); - let trailers = trailers - .as_ref() - .map(|v| v.iter().map(|borrowed| &**borrowed).collect::<Vec<_>>()); - let trailers = trailers.as_deref(); - - let (res, written) = nix::sys::sendfile::sendfile( - args.in_fd, - args.out_fd, - args.offset, - Some(count), - headers, - trailers, - ); - res.map_err(|err| err.into_pyexception(vm))?; - Ok(vm.ctx.new_int(written as u64).into()) - } - - #[cfg(target_os = "linux")] - unsafe fn sys_getrandom(buf: *mut libc::c_void, buflen: usize, flags: u32) -> isize { - libc::syscall(libc::SYS_getrandom, buf, buflen, flags as usize) as _ - } - - #[cfg(target_os = "linux")] - #[pyfunction] - fn getrandom(size: isize, flags: OptionalArg<u32>, vm: &VirtualMachine) -> PyResult<Vec<u8>> { - let size = usize::try_from(size) - .map_err(|_| vm.new_os_error(format!("Invalid argument for size: {size}")))?; - let mut buf = Vec::with_capacity(size); - unsafe { - let len = sys_getrandom( - buf.as_mut_ptr() as *mut libc::c_void, - size, - flags.unwrap_or(0), - ) - .try_into() - .map_err(|_| errno_err(vm))?; - buf.set_len(len); - } - Ok(buf) - } -} diff --git a/vm/src/stdlib/posix_compat.rs b/vm/src/stdlib/posix_compat.rs deleted file mode 100644 index 3f939dce10e..00000000000 --- a/vm/src/stdlib/posix_compat.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! `posix` compatible module for `not(any(unix, windows))` -use crate::{builtins::PyModule, PyRef, VirtualMachine}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = module::make_module(vm); - super::os::extend_module(vm, &module); - module -} - -#[pymodule(name = "posix", with(super::os::_os))] -pub(crate) mod module { - use crate::{ - builtins::PyStrRef, - stdlib::os::{DirFd, OsPath, SupportFunc, TargetIsDirectory, _os}, - PyObjectRef, PyResult, VirtualMachine, - }; - use std::env; - - #[pyfunction] - pub(super) fn access(_path: PyStrRef, _mode: u8, vm: &VirtualMachine) -> PyResult<bool> { - os_unimpl("os.access", vm) - } - - #[derive(FromArgs)] - #[allow(unused)] - pub(super) struct SymlinkArgs { - src: OsPath, - dst: OsPath, - #[pyarg(flatten)] - _target_is_directory: TargetIsDirectory, - #[pyarg(flatten)] - _dir_fd: DirFd<{ _os::SYMLINK_DIR_FD as usize }>, - } - - #[pyfunction] - pub(super) fn symlink(_args: SymlinkArgs, vm: &VirtualMachine) -> PyResult<()> { - os_unimpl("os.symlink", vm) - } - - #[cfg(target_os = "wasi")] - #[pyattr] - fn environ(vm: &VirtualMachine) -> crate::builtins::PyDictRef { - use rustpython_common::os::ffi::OsStringExt; - - let environ = vm.ctx.new_dict(); - for (key, value) in env::vars_os() { - let key: PyObjectRef = vm.ctx.new_bytes(key.into_vec()).into(); - let value: PyObjectRef = vm.ctx.new_bytes(value.into_vec()).into(); - environ.set_item(&*key, value, vm).unwrap(); - } - - environ - } - - #[allow(dead_code)] - fn os_unimpl<T>(func: &str, vm: &VirtualMachine) -> PyResult<T> { - Err(vm.new_os_error(format!("{} is not supported on this platform", func))) - } - - pub(crate) fn support_funcs() -> Vec<SupportFunc> { - Vec::new() - } -} diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs deleted file mode 100644 index f5d1e50a262..00000000000 --- a/vm/src/stdlib/pwd.rs +++ /dev/null @@ -1,109 +0,0 @@ -pub(crate) use pwd::make_module; - -#[pymodule] -mod pwd { - use crate::{ - builtins::{PyIntRef, PyStrRef}, - convert::{IntoPyException, ToPyObject}, - exceptions, - types::PyStructSequence, - PyObjectRef, PyResult, VirtualMachine, - }; - use nix::unistd::{self, User}; - use std::ptr::NonNull; - - #[pyattr] - #[pyclass(module = "pwd", name = "struct_passwd")] - #[derive(PyStructSequence)] - struct Passwd { - pw_name: String, - pw_passwd: String, - pw_uid: u32, - pw_gid: u32, - pw_gecos: String, - pw_dir: String, - pw_shell: String, - } - #[pyclass(with(PyStructSequence))] - impl Passwd {} - - impl From<User> for Passwd { - fn from(user: User) -> Self { - // this is just a pain... - let cstr_lossy = |s: std::ffi::CString| { - s.into_string() - .unwrap_or_else(|e| e.into_cstring().to_string_lossy().into_owned()) - }; - let pathbuf_lossy = |p: std::path::PathBuf| { - p.into_os_string() - .into_string() - .unwrap_or_else(|s| s.to_string_lossy().into_owned()) - }; - Passwd { - pw_name: user.name, - pw_passwd: cstr_lossy(user.passwd), - pw_uid: user.uid.as_raw(), - pw_gid: user.gid.as_raw(), - pw_gecos: cstr_lossy(user.gecos), - pw_dir: pathbuf_lossy(user.dir), - pw_shell: pathbuf_lossy(user.shell), - } - } - } - - #[pyfunction] - fn getpwnam(name: PyStrRef, vm: &VirtualMachine) -> PyResult<Passwd> { - let pw_name = name.as_str(); - if pw_name.contains('\0') { - return Err(exceptions::cstring_error(vm)); - } - let user = User::from_name(name.as_str()).map_err(|err| err.into_pyexception(vm))?; - let user = user.ok_or_else(|| { - vm.new_key_error( - vm.ctx - .new_str(format!("getpwnam(): name not found: {pw_name}")) - .into(), - ) - })?; - Ok(Passwd::from(user)) - } - - #[pyfunction] - fn getpwuid(uid: PyIntRef, vm: &VirtualMachine) -> PyResult<Passwd> { - let uid_t = libc::uid_t::try_from(uid.as_bigint()) - .map(unistd::Uid::from_raw) - .ok(); - let user = uid_t - .map(User::from_uid) - .transpose() - .map_err(|err| err.into_pyexception(vm))? - .flatten(); - let user = user.ok_or_else(|| { - vm.new_key_error( - vm.ctx - .new_str(format!("getpwuid(): uid not found: {}", uid.as_bigint())) - .into(), - ) - })?; - Ok(Passwd::from(user)) - } - - // TODO: maybe merge this functionality into nix? - #[pyfunction] - fn getpwall(vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - // setpwent, getpwent, etc are not thread safe. Could use fgetpwent_r, but this is easier - static GETPWALL: parking_lot::Mutex<()> = parking_lot::const_mutex(()); - let _guard = GETPWALL.lock(); - let mut list = Vec::new(); - - unsafe { libc::setpwent() }; - while let Some(ptr) = NonNull::new(unsafe { libc::getpwent() }) { - let user = User::from(unsafe { ptr.as_ref() }); - let passwd = Passwd::from(user).to_pyobject(vm); - list.push(passwd); - } - unsafe { libc::endpwent() }; - - Ok(list) - } -} diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs deleted file mode 100644 index 4a698a76336..00000000000 --- a/vm/src/stdlib/signal.rs +++ /dev/null @@ -1,281 +0,0 @@ -use crate::{builtins::PyModule, PyRef, VirtualMachine}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _signal::make_module(vm); - - _signal::init_signal_handlers(&module, vm); - - module -} - -#[pymodule] -pub(crate) mod _signal { - use crate::{ - builtins::PyModule, - convert::{IntoPyException, TryFromBorrowedObject}, - signal, Py, PyObjectRef, PyResult, VirtualMachine, - }; - use std::sync::atomic::{self, Ordering}; - - cfg_if::cfg_if! { - if #[cfg(windows)] { - type WakeupFd = libc::SOCKET; - const INVALID_WAKEUP: WakeupFd = (-1isize) as usize; - static WAKEUP: atomic::AtomicUsize = atomic::AtomicUsize::new(INVALID_WAKEUP); - // windows doesn't use the same fds for files and sockets like windows does, so we need - // this to know whether to send() or write() - static WAKEUP_IS_SOCKET: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - } else { - type WakeupFd = i32; - const INVALID_WAKEUP: WakeupFd = -1; - static WAKEUP: atomic::AtomicI32 = atomic::AtomicI32::new(INVALID_WAKEUP); - } - } - - #[cfg(unix)] - pub use nix::unistd::alarm as sig_alarm; - - #[cfg(not(windows))] - pub use libc::SIG_ERR; - - #[cfg(not(windows))] - #[pyattr] - pub use libc::{SIG_DFL, SIG_IGN}; - - #[cfg(windows)] - #[pyattr] - pub const SIG_DFL: libc::sighandler_t = 0; - #[cfg(windows)] - #[pyattr] - pub const SIG_IGN: libc::sighandler_t = 1; - #[cfg(windows)] - pub const SIG_ERR: libc::sighandler_t = !0; - - #[cfg(all(unix, not(target_os = "redox")))] - extern "C" { - fn siginterrupt(sig: i32, flag: i32) -> i32; - } - - #[pyattr] - use crate::signal::NSIG; - - #[pyattr] - pub use libc::{SIGABRT, SIGFPE, SIGILL, SIGINT, SIGSEGV, SIGTERM}; - - #[cfg(unix)] - #[pyattr] - use libc::{ - SIGALRM, SIGBUS, SIGCHLD, SIGCONT, SIGHUP, SIGIO, SIGKILL, SIGPIPE, SIGPROF, SIGQUIT, - SIGSTOP, SIGSYS, SIGTRAP, SIGTSTP, SIGTTIN, SIGTTOU, SIGURG, SIGUSR1, SIGUSR2, SIGVTALRM, - SIGWINCH, SIGXCPU, SIGXFSZ, - }; - - #[cfg(unix)] - #[cfg(not(any( - target_vendor = "apple", - target_os = "openbsd", - target_os = "freebsd", - target_os = "netbsd" - )))] - #[pyattr] - use libc::{SIGPWR, SIGSTKFLT}; - - pub(super) fn init_signal_handlers(module: &Py<PyModule>, vm: &VirtualMachine) { - let sig_dfl = vm.new_pyobj(SIG_DFL as u8); - let sig_ign = vm.new_pyobj(SIG_IGN as u8); - - for signum in 1..NSIG { - let handler = unsafe { libc::signal(signum as i32, SIG_IGN) }; - if handler != SIG_ERR { - unsafe { libc::signal(signum as i32, handler) }; - } - let py_handler = if handler == SIG_DFL { - Some(sig_dfl.clone()) - } else if handler == SIG_IGN { - Some(sig_ign.clone()) - } else { - None - }; - vm.signal_handlers.as_deref().unwrap().borrow_mut()[signum] = py_handler; - } - - let int_handler = module - .get_attr("default_int_handler", vm) - .expect("_signal does not have this attr?"); - if !vm.state.settings.no_sig_int { - signal(libc::SIGINT, int_handler, vm).expect("Failed to set sigint handler"); - } - } - - #[pyfunction] - pub fn signal( - signalnum: i32, - handler: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - signal::assert_in_range(signalnum, vm)?; - let signal_handlers = vm - .signal_handlers - .as_deref() - .ok_or_else(|| vm.new_value_error("signal only works in main thread".to_owned()))?; - - let sig_handler = - match usize::try_from_borrowed_object(vm, &handler).ok() { - Some(SIG_DFL) => SIG_DFL, - Some(SIG_IGN) => SIG_IGN, - None if handler.is_callable() => run_signal as libc::sighandler_t, - _ => return Err(vm.new_type_error( - "signal handler must be signal.SIG_IGN, signal.SIG_DFL, or a callable object" - .to_owned(), - )), - }; - signal::check_signals(vm)?; - - let old = unsafe { libc::signal(signalnum, sig_handler) }; - if old == SIG_ERR { - return Err(vm.new_os_error("Failed to set signal".to_owned())); - } - #[cfg(all(unix, not(target_os = "redox")))] - unsafe { - siginterrupt(signalnum, 1); - } - - let old_handler = std::mem::replace( - &mut signal_handlers.borrow_mut()[signalnum as usize], - Some(handler), - ); - Ok(old_handler) - } - - #[pyfunction] - fn getsignal(signalnum: i32, vm: &VirtualMachine) -> PyResult { - signal::assert_in_range(signalnum, vm)?; - let signal_handlers = vm - .signal_handlers - .as_deref() - .ok_or_else(|| vm.new_value_error("getsignal only works in main thread".to_owned()))?; - let handler = signal_handlers.borrow()[signalnum as usize] - .clone() - .unwrap_or_else(|| vm.ctx.none()); - Ok(handler) - } - - #[cfg(unix)] - #[pyfunction] - fn alarm(time: u32) -> u32 { - let prev_time = if time == 0 { - sig_alarm::cancel() - } else { - sig_alarm::set(time) - }; - prev_time.unwrap_or(0) - } - - #[pyfunction] - fn default_int_handler( - _signum: PyObjectRef, - _arg: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { - Err(vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned())) - } - - #[derive(FromArgs)] - struct SetWakeupFdArgs { - fd: WakeupFd, - #[pyarg(named, default = "true")] - warn_on_full_buffer: bool, - } - - #[pyfunction] - fn set_wakeup_fd(args: SetWakeupFdArgs, vm: &VirtualMachine) -> PyResult<WakeupFd> { - // TODO: implement warn_on_full_buffer - let _ = args.warn_on_full_buffer; - let fd = args.fd; - - if vm.signal_handlers.is_none() { - return Err(vm.new_value_error("signal only works in main thread".to_owned())); - } - - #[cfg(windows)] - let is_socket = if fd != INVALID_WAKEUP { - use windows_sys::Win32::Networking::WinSock; - - crate::stdlib::nt::init_winsock(); - let mut res = 0i32; - let mut res_size = std::mem::size_of::<i32>() as i32; - let res = unsafe { - WinSock::getsockopt( - fd, - WinSock::SOL_SOCKET, - WinSock::SO_ERROR, - &mut res as *mut i32 as *mut _, - &mut res_size, - ) - }; - // if getsockopt succeeded, fd is for sure a socket - let is_socket = res == 0; - if !is_socket { - let err = std::io::Error::last_os_error(); - // if getsockopt failed for some other reason, throw - if err.raw_os_error() != Some(WinSock::WSAENOTSOCK) { - return Err(err.into_pyexception(vm)); - } - } - is_socket - } else { - false - }; - #[cfg(not(windows))] - if fd != INVALID_WAKEUP { - use nix::fcntl; - let oflags = fcntl::fcntl(fd, fcntl::F_GETFL).map_err(|e| e.into_pyexception(vm))?; - let nonblock = - fcntl::OFlag::from_bits_truncate(oflags).contains(fcntl::OFlag::O_NONBLOCK); - if !nonblock { - return Err(vm.new_value_error(format!("the fd {fd} must be in non-blocking mode"))); - } - } - - let old_fd = WAKEUP.swap(fd, Ordering::Relaxed); - #[cfg(windows)] - WAKEUP_IS_SOCKET.store(is_socket, Ordering::Relaxed); - - Ok(old_fd) - } - - #[cfg(all(unix, not(target_os = "redox")))] - #[pyfunction(name = "siginterrupt")] - fn py_siginterrupt(signum: i32, flag: i32, vm: &VirtualMachine) -> PyResult<()> { - signal::assert_in_range(signum, vm)?; - let res = unsafe { siginterrupt(signum, flag) }; - if res < 0 { - Err(crate::stdlib::os::errno_err(vm)) - } else { - Ok(()) - } - } - - pub extern "C" fn run_signal(signum: i32) { - signal::TRIGGERS[signum as usize].store(true, Ordering::Relaxed); - signal::set_triggered(); - let wakeup_fd = WAKEUP.load(Ordering::Relaxed); - if wakeup_fd != INVALID_WAKEUP { - let sigbyte = signum as u8; - #[cfg(windows)] - if WAKEUP_IS_SOCKET.load(Ordering::Relaxed) { - let _res = unsafe { - windows_sys::Win32::Networking::WinSock::send( - wakeup_fd, - &sigbyte as *const u8 as *const _, - 1, - 0, - ) - }; - return; - } - let _res = unsafe { libc::write(wakeup_fd as _, &sigbyte as *const u8 as *const _, 1) }; - // TODO: handle _res < 1, support warn_on_full_buffer - } - } -} diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs deleted file mode 100644 index 93ecd7c24e8..00000000000 --- a/vm/src/stdlib/sre.rs +++ /dev/null @@ -1,833 +0,0 @@ -pub(crate) use _sre::make_module; - -#[pymodule] -mod _sre { - use crate::{ - atomic_func, - builtins::{ - PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyStr, PyStrRef, PyTuple, - PyTupleRef, PyTypeRef, - }, - common::{ascii, hash::PyHash}, - convert::ToPyObject, - function::{ArgCallable, OptionalArg, PosArgs, PyComparisonValue}, - protocol::{PyBuffer, PyMappingMethods}, - stdlib::sys, - types::{AsMapping, Comparable, Hashable, Representable}, - Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, - TryFromObject, VirtualMachine, - }; - use core::str; - use crossbeam_utils::atomic::AtomicCell; - use itertools::Itertools; - use num_traits::ToPrimitive; - use sre_engine::{ - constants::SreFlag, - engine::{lower_ascii, lower_unicode, upper_unicode, Request, SearchIter, State, StrDrive}, - }; - - #[pyattr] - pub use sre_engine::{constants::SRE_MAGIC as MAGIC, CODESIZE, MAXGROUPS, MAXREPEAT}; - - #[pyfunction] - fn getcodesize() -> usize { - CODESIZE - } - #[pyfunction] - fn ascii_iscased(ch: i32) -> bool { - (ch >= b'a' as i32 && ch <= b'z' as i32) || (ch >= b'A' as i32 && ch <= b'Z' as i32) - } - #[pyfunction] - fn unicode_iscased(ch: i32) -> bool { - let ch = ch as u32; - ch != lower_unicode(ch) || ch != upper_unicode(ch) - } - #[pyfunction] - fn ascii_tolower(ch: i32) -> i32 { - lower_ascii(ch as u32) as i32 - } - #[pyfunction] - fn unicode_tolower(ch: i32) -> i32 { - lower_unicode(ch as u32) as i32 - } - - trait SreStr: StrDrive { - fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef; - - fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request<Self> { - Request::new(self, start, end, &pattern.code, false) - } - } - - impl SreStr for &[u8] { - fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx - .new_bytes(self.iter().take(end).skip(start).cloned().collect()) - .into() - } - } - - impl SreStr for &str { - fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef { - vm.ctx - .new_str(self.chars().take(end).skip(start).collect::<String>()) - .into() - } - } - - #[pyfunction] - fn compile( - pattern: PyObjectRef, - flags: u16, - code: PyObjectRef, - groups: usize, - groupindex: PyDictRef, - indexgroup: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<Pattern> { - // FIXME: - // pattern could only be None if called by re.Scanner - // re.Scanner has no official API and in CPython's implement - // isbytes will be hanging (-1) - // here is just a hack to let re.Scanner works only with str not bytes - let isbytes = !vm.is_none(&pattern) && !pattern.payload_is::<PyStr>(); - let code = code.try_to_value(vm)?; - Ok(Pattern { - pattern, - flags: SreFlag::from_bits_truncate(flags), - code, - groups, - groupindex, - indexgroup: indexgroup.try_to_value(vm)?, - isbytes, - }) - } - - #[derive(FromArgs)] - struct StringArgs { - string: PyObjectRef, - #[pyarg(any, default = "0")] - pos: usize, - #[pyarg(any, default = "sys::MAXSIZE as usize")] - endpos: usize, - } - - #[derive(FromArgs)] - struct SubArgs { - // repl: Either<ArgCallable, PyStrRef>, - repl: PyObjectRef, - string: PyObjectRef, - #[pyarg(any, default = "0")] - count: usize, - } - - #[derive(FromArgs)] - struct SplitArgs { - string: PyObjectRef, - #[pyarg(any, default = "0")] - maxsplit: isize, - } - - #[pyattr] - #[pyclass(name = "Pattern")] - #[derive(Debug, PyPayload)] - pub(crate) struct Pattern { - pub pattern: PyObjectRef, - pub flags: SreFlag, - pub code: Vec<u32>, - pub groups: usize, - pub groupindex: PyDictRef, - pub indexgroup: Vec<Option<PyStrRef>>, - pub isbytes: bool, - } - - macro_rules! with_sre_str { - ($pattern:expr, $string:expr, $vm:expr, $f:expr) => { - if $pattern.isbytes { - Pattern::with_bytes($string, $vm, $f) - } else { - Pattern::with_str($string, $vm, $f) - } - }; - } - - #[pyclass(with(Hashable, Comparable, Representable))] - impl Pattern { - fn with_str<F, R>(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult<R> - where - F: FnOnce(&str) -> PyResult<R>, - { - let string = string - .payload::<PyStr>() - .ok_or_else(|| vm.new_type_error("expected string".to_owned()))?; - f(string.as_str()) - } - - fn with_bytes<F, R>(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult<R> - where - F: FnOnce(&[u8]) -> PyResult<R>, - { - PyBuffer::try_from_borrowed_object(vm, string)?.contiguous_or_collect(f) - } - - #[pymethod(name = "match")] - fn pymatch( - zelf: PyRef<Pattern>, - string_args: StringArgs, - vm: &VirtualMachine, - ) -> PyResult<Option<PyRef<Match>>> { - let StringArgs { - string, - pos, - endpos, - } = string_args; - with_sre_str!(zelf, &string.clone(), vm, |x| { - let req = x.create_request(&zelf, pos, endpos); - let mut state = State::default(); - state.pymatch(req); - Ok(state - .has_matched - .then(|| Match::new(&state, zelf.clone(), string).into_ref(&vm.ctx))) - }) - } - - #[pymethod] - fn fullmatch( - zelf: PyRef<Pattern>, - string_args: StringArgs, - vm: &VirtualMachine, - ) -> PyResult<Option<PyRef<Match>>> { - with_sre_str!(zelf, &string_args.string.clone(), vm, |x| { - let mut req = x.create_request(&zelf, string_args.pos, string_args.endpos); - req.match_all = true; - let mut state = State::default(); - state.pymatch(req); - Ok(state.has_matched.then(|| { - Match::new(&state, zelf.clone(), string_args.string).into_ref(&vm.ctx) - })) - }) - } - - #[pymethod] - fn search( - zelf: PyRef<Pattern>, - string_args: StringArgs, - vm: &VirtualMachine, - ) -> PyResult<Option<PyRef<Match>>> { - with_sre_str!(zelf, &string_args.string.clone(), vm, |x| { - let req = x.create_request(&zelf, string_args.pos, string_args.endpos); - let mut state = State::default(); - state.search(req); - Ok(state.has_matched.then(|| { - Match::new(&state, zelf.clone(), string_args.string).into_ref(&vm.ctx) - })) - }) - } - - #[pymethod] - fn findall( - zelf: PyRef<Pattern>, - string_args: StringArgs, - vm: &VirtualMachine, - ) -> PyResult<Vec<PyObjectRef>> { - with_sre_str!(zelf, &string_args.string, vm, |s| { - let req = s.create_request(&zelf, string_args.pos, string_args.endpos); - let state = State::default(); - let mut matchlist: Vec<PyObjectRef> = Vec::new(); - let mut iter = SearchIter { req, state }; - - while iter.next().is_some() { - let m = Match::new(&iter.state, zelf.clone(), string_args.string.clone()); - - let item = if zelf.groups == 0 || zelf.groups == 1 { - m.get_slice(zelf.groups, s, vm) - .unwrap_or_else(|| vm.ctx.none()) - } else { - m.groups(OptionalArg::Present(vm.ctx.new_str(ascii!("")).into()), vm)? - .into() - }; - - matchlist.push(item); - } - - Ok(matchlist) - }) - } - - #[pymethod] - fn finditer( - zelf: PyRef<Pattern>, - string_args: StringArgs, - vm: &VirtualMachine, - ) -> PyResult<PyCallableIterator> { - let scanner = SreScanner { - pattern: zelf, - string: string_args.string, - start: AtomicCell::new(string_args.pos), - end: string_args.endpos, - must_advance: AtomicCell::new(false), - } - .into_ref(&vm.ctx); - let search = vm.get_str_method(scanner.into(), "search").unwrap()?; - let search = ArgCallable::try_from_object(vm, search)?; - let iterator = PyCallableIterator::new(search, vm.ctx.none()); - Ok(iterator) - } - - #[pymethod] - fn scanner( - zelf: PyRef<Pattern>, - string_args: StringArgs, - vm: &VirtualMachine, - ) -> PyRef<SreScanner> { - SreScanner { - pattern: zelf, - string: string_args.string, - start: AtomicCell::new(string_args.pos), - end: string_args.endpos, - must_advance: AtomicCell::new(false), - } - .into_ref(&vm.ctx) - } - - #[pymethod] - fn sub(zelf: PyRef<Pattern>, sub_args: SubArgs, vm: &VirtualMachine) -> PyResult { - Self::subx(zelf, sub_args, false, vm) - } - #[pymethod] - fn subn(zelf: PyRef<Pattern>, sub_args: SubArgs, vm: &VirtualMachine) -> PyResult { - Self::subx(zelf, sub_args, true, vm) - } - - #[pymethod] - fn split( - zelf: PyRef<Pattern>, - split_args: SplitArgs, - vm: &VirtualMachine, - ) -> PyResult<Vec<PyObjectRef>> { - with_sre_str!(zelf, &split_args.string, vm, |s| { - let req = s.create_request(&zelf, 0, usize::MAX); - let state = State::default(); - let mut splitlist: Vec<PyObjectRef> = Vec::new(); - let mut iter = SearchIter { req, state }; - let mut n = 0; - let mut last = 0; - - while (split_args.maxsplit == 0 || n < split_args.maxsplit) && iter.next().is_some() - { - /* get segment before this match */ - splitlist.push(s.slice(last, iter.state.start, vm)); - - let m = Match::new(&iter.state, zelf.clone(), split_args.string.clone()); - - // add groups (if any) - for i in 1..=zelf.groups { - splitlist.push(m.get_slice(i, s, vm).unwrap_or_else(|| vm.ctx.none())); - } - - n += 1; - last = iter.state.string_position; - } - - // get segment following last match (even if empty) - splitlist.push(req.string.slice(last, s.count(), vm)); - - Ok(splitlist) - }) - } - - #[pygetset] - fn flags(&self) -> u16 { - self.flags.bits() - } - #[pygetset] - fn groupindex(&self) -> PyDictRef { - self.groupindex.clone() - } - #[pygetset] - fn groups(&self) -> usize { - self.groups - } - #[pygetset] - fn pattern(&self) -> PyObjectRef { - self.pattern.clone() - } - - fn subx( - zelf: PyRef<Pattern>, - sub_args: SubArgs, - subn: bool, - vm: &VirtualMachine, - ) -> PyResult { - let SubArgs { - repl, - string, - count, - } = sub_args; - - let (is_callable, filter) = if repl.is_callable() { - (true, repl) - } else { - let is_template = if zelf.isbytes { - Self::with_bytes(&repl, vm, |x| Ok(x.contains(&b'\\'))) - } else { - Self::with_str(&repl, vm, |x| Ok(x.contains('\\'))) - }?; - if is_template { - let re = vm.import("re", None, 0)?; - let func = re.get_attr("_subx", vm)?; - let filter = func.call((zelf.clone(), repl), vm)?; - (filter.is_callable(), filter) - } else { - (false, repl) - } - }; - - with_sre_str!(zelf, &string, vm, |s| { - let req = s.create_request(&zelf, 0, usize::MAX); - let state = State::default(); - let mut sublist: Vec<PyObjectRef> = Vec::new(); - let mut iter = SearchIter { req, state }; - let mut n = 0; - let mut last_pos = 0; - - while (count == 0 || n < count) && iter.next().is_some() { - if last_pos < iter.state.start { - /* get segment before this match */ - sublist.push(s.slice(last_pos, iter.state.start, vm)); - } - - if is_callable { - let m = Match::new(&iter.state, zelf.clone(), string.clone()); - let ret = filter.call((m.into_ref(&vm.ctx),), vm)?; - sublist.push(ret); - } else { - sublist.push(filter.clone()); - } - - last_pos = iter.state.string_position; - n += 1; - } - - /* get segment following last match */ - sublist.push(s.slice(last_pos, iter.req.end, vm)); - - let list = PyList::from(sublist).into_pyobject(vm); - - let join_type: PyObjectRef = if zelf.isbytes { - vm.ctx.new_bytes(vec![]).into() - } else { - vm.ctx.new_str(ascii!("")).into() - }; - let ret = vm.call_method(&join_type, "join", (list,))?; - - Ok(if subn { (ret, n).to_pyobject(vm) } else { ret }) - }) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } - } - - impl Hashable for Pattern { - fn hash(zelf: &crate::Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> { - let hash = zelf.pattern.hash(vm)?; - let (_, code, _) = unsafe { zelf.code.align_to::<u8>() }; - let hash = hash ^ vm.state.hash_secret.hash_bytes(code); - let hash = hash ^ (zelf.flags.bits() as PyHash); - let hash = hash ^ (zelf.isbytes as i64); - Ok(hash) - } - } - - impl Comparable for Pattern { - fn cmp( - zelf: &crate::Py<Self>, - other: &PyObject, - op: crate::types::PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue> { - if let Some(res) = op.identical_optimization(zelf, other) { - return Ok(res.into()); - } - op.eq_only(|| { - if let Some(other) = other.downcast_ref::<Pattern>() { - Ok(PyComparisonValue::Implemented( - zelf.flags == other.flags - && zelf.isbytes == other.isbytes - && zelf.code == other.code - && vm.bool_eq(&zelf.pattern, &other.pattern)?, - )) - } else { - Ok(PyComparisonValue::NotImplemented) - } - }) - } - } - - impl Representable for Pattern { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - let flag_names = [ - ("re.TEMPLATE", SreFlag::TEMPLATE), - ("re.IGNORECASE", SreFlag::IGNORECASE), - ("re.LOCALE", SreFlag::LOCALE), - ("re.MULTILINE", SreFlag::MULTILINE), - ("re.DOTALL", SreFlag::DOTALL), - ("re.UNICODE", SreFlag::UNICODE), - ("re.VERBOSE", SreFlag::VERBOSE), - ("re.DEBUG", SreFlag::DEBUG), - ("re.ASCII", SreFlag::ASCII), - ]; - - /* Omit re.UNICODE for valid string patterns. */ - let mut flags = zelf.flags; - if !zelf.isbytes - && (flags & (SreFlag::LOCALE | SreFlag::UNICODE | SreFlag::ASCII)) - == SreFlag::UNICODE - { - flags &= !SreFlag::UNICODE; - } - - let flags = flag_names - .iter() - .filter(|(_, flag)| flags.contains(*flag)) - .map(|(name, _)| name) - .join("|"); - - let pattern = zelf.pattern.repr(vm)?; - let truncated: String; - let s = if pattern.char_len() > 200 { - truncated = pattern.as_str().chars().take(200).collect(); - &truncated - } else { - pattern.as_str() - }; - - if flags.is_empty() { - Ok(format!("re.compile({s})")) - } else { - Ok(format!("re.compile({s}, {flags})")) - } - } - } - - #[pyattr] - #[pyclass(name = "Match")] - #[derive(Debug, PyPayload)] - pub(crate) struct Match { - string: PyObjectRef, - pattern: PyRef<Pattern>, - pos: usize, - endpos: usize, - lastindex: isize, - regs: Vec<(isize, isize)>, - } - - #[pyclass(with(AsMapping, Representable))] - impl Match { - pub(crate) fn new<S: StrDrive>( - state: &State<S>, - pattern: PyRef<Pattern>, - string: PyObjectRef, - ) -> Self { - let mut regs = vec![(state.start as isize, state.string_position as isize)]; - for group in 0..pattern.groups { - let mark_index = 2 * group; - if mark_index + 1 < state.marks.len() { - let start = state.marks[mark_index]; - let end = state.marks[mark_index + 1]; - if start.is_some() && end.is_some() { - regs.push((start.unpack() as isize, end.unpack() as isize)); - continue; - } - } - regs.push((-1, -1)); - } - Self { - string, - pattern, - pos: state.start, - endpos: state.string_position, - lastindex: state.marks.last_index(), - regs, - } - } - - #[pygetset] - fn pos(&self) -> usize { - self.pos - } - #[pygetset] - fn endpos(&self) -> usize { - self.endpos - } - #[pygetset] - fn lastindex(&self) -> Option<isize> { - if self.lastindex >= 0 { - Some(self.lastindex) - } else { - None - } - } - #[pygetset] - fn lastgroup(&self) -> Option<PyStrRef> { - let i = self.lastindex.to_usize()?; - self.pattern.indexgroup.get(i)?.clone() - } - #[pygetset] - fn re(&self) -> PyRef<Pattern> { - self.pattern.clone() - } - #[pygetset] - fn string(&self) -> PyObjectRef { - self.string.clone() - } - #[pygetset] - fn regs(&self, vm: &VirtualMachine) -> PyTupleRef { - PyTuple::new_ref( - self.regs.iter().map(|&x| x.to_pyobject(vm)).collect(), - &vm.ctx, - ) - } - - #[pymethod] - fn start(&self, group: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<isize> { - self.span(group, vm).map(|x| x.0) - } - #[pymethod] - fn end(&self, group: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult<isize> { - self.span(group, vm).map(|x| x.1) - } - #[pymethod] - fn span( - &self, - group: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<(isize, isize)> { - let index = group.map_or(Ok(0), |group| { - self.get_index(group, vm) - .ok_or_else(|| vm.new_index_error("no such group".to_owned())) - })?; - Ok(self.regs[index]) - } - - #[pymethod] - fn expand(zelf: PyRef<Match>, template: PyStrRef, vm: &VirtualMachine) -> PyResult { - let re = vm.import("re", None, 0)?; - let func = re.get_attr("_expand", vm)?; - func.call((zelf.pattern.clone(), zelf, template), vm) - } - - #[pymethod] - fn group(&self, args: PosArgs<PyObjectRef>, vm: &VirtualMachine) -> PyResult { - with_sre_str!(self.pattern, &self.string, vm, |str_drive| { - let args = args.into_vec(); - if args.is_empty() { - return Ok(self.get_slice(0, str_drive, vm).unwrap().to_pyobject(vm)); - } - let mut v: Vec<PyObjectRef> = args - .into_iter() - .map(|x| { - self.get_index(x, vm) - .ok_or_else(|| vm.new_index_error("no such group".to_owned())) - .map(|index| { - self.get_slice(index, str_drive, vm) - .map(|x| x.to_pyobject(vm)) - .unwrap_or_else(|| vm.ctx.none()) - }) - }) - .try_collect()?; - if v.len() == 1 { - Ok(v.pop().unwrap()) - } else { - Ok(vm.ctx.new_tuple(v).into()) - } - }) - } - - #[pymethod(magic)] - fn getitem( - &self, - group: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<Option<PyObjectRef>> { - with_sre_str!(self.pattern, &self.string, vm, |str_drive| { - let i = self - .get_index(group, vm) - .ok_or_else(|| vm.new_index_error("no such group".to_owned()))?; - Ok(self.get_slice(i, str_drive, vm)) - }) - } - - #[pymethod] - fn groups( - &self, - default: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyTupleRef> { - let default = default.unwrap_or_else(|| vm.ctx.none()); - - with_sre_str!(self.pattern, &self.string, vm, |str_drive| { - let v: Vec<PyObjectRef> = (1..self.regs.len()) - .map(|i| { - self.get_slice(i, str_drive, vm) - .map(|s| s.to_pyobject(vm)) - .unwrap_or_else(|| default.clone()) - }) - .collect(); - Ok(PyTuple::new_ref(v, &vm.ctx)) - }) - } - - #[pymethod] - fn groupdict( - &self, - default: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<PyDictRef> { - let default = default.unwrap_or_else(|| vm.ctx.none()); - - with_sre_str!(self.pattern, &self.string, vm, |str_drive| { - let dict = vm.ctx.new_dict(); - - for (key, index) in self.pattern.groupindex.clone() { - let value = self - .get_index(index, vm) - .and_then(|x| self.get_slice(x, str_drive, vm)) - .map(|x| x.to_pyobject(vm)) - .unwrap_or_else(|| default.clone()); - dict.set_item(&*key, value, vm)?; - } - Ok(dict) - }) - } - - fn get_index(&self, group: PyObjectRef, vm: &VirtualMachine) -> Option<usize> { - let i = if let Ok(i) = group.try_index(vm) { - i - } else { - self.pattern - .groupindex - .get_item_opt(&*group, vm) - .ok()?? - .downcast::<PyInt>() - .ok()? - }; - let i = i.as_bigint().to_isize()?; - if i >= 0 && i as usize <= self.pattern.groups { - Some(i as usize) - } else { - None - } - } - - fn get_slice<S: SreStr>( - &self, - index: usize, - str_drive: S, - vm: &VirtualMachine, - ) -> Option<PyObjectRef> { - let (start, end) = self.regs[index]; - if start < 0 || end < 0 { - return None; - } - Some(str_drive.slice(start as usize, end as usize, vm)) - } - - #[pyclassmethod(magic)] - fn class_getitem(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { - PyGenericAlias::new(cls, args, vm) - } - } - - impl AsMapping for Match { - fn as_mapping() -> &'static PyMappingMethods { - static AS_MAPPING: once_cell::sync::Lazy<PyMappingMethods> = - once_cell::sync::Lazy::new(|| PyMappingMethods { - subscript: atomic_func!(|mapping, needle, vm| { - Match::mapping_downcast(mapping) - .getitem(needle.to_owned(), vm) - .map(|x| x.to_pyobject(vm)) - }), - ..PyMappingMethods::NOT_IMPLEMENTED - }); - &AS_MAPPING - } - } - - impl Representable for Match { - #[inline] - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> { - with_sre_str!(zelf.pattern, &zelf.string, vm, |str_drive| { - Ok(format!( - "<re.Match object; span=({}, {}), match={}>", - zelf.regs[0].0, - zelf.regs[0].1, - zelf.get_slice(0, str_drive, vm).unwrap().repr(vm)? - )) - }) - } - } - - #[pyattr] - #[pyclass(name = "SRE_Scanner")] - #[derive(Debug, PyPayload)] - struct SreScanner { - pattern: PyRef<Pattern>, - string: PyObjectRef, - start: AtomicCell<usize>, - end: usize, - must_advance: AtomicCell<bool>, - } - - #[pyclass] - impl SreScanner { - #[pygetset] - fn pattern(&self) -> PyRef<Pattern> { - self.pattern.clone() - } - - #[pymethod(name = "match")] - fn pymatch(&self, vm: &VirtualMachine) -> PyResult<Option<PyRef<Match>>> { - with_sre_str!(self.pattern, &self.string.clone(), vm, |s| { - let mut req = s.create_request(&self.pattern, self.start.load(), self.end); - let mut state = State::default(); - req.must_advance = self.must_advance.load(); - state.pymatch(req); - - self.must_advance - .store(state.string_position == state.start); - self.start.store(state.string_position); - - Ok(state.has_matched.then(|| { - Match::new(&state, self.pattern.clone(), self.string.clone()).into_ref(&vm.ctx) - })) - }) - } - - #[pymethod] - fn search(&self, vm: &VirtualMachine) -> PyResult<Option<PyRef<Match>>> { - if self.start.load() > self.end { - return Ok(None); - } - with_sre_str!(self.pattern, &self.string.clone(), vm, |s| { - let mut req = s.create_request(&self.pattern, self.start.load(), self.end); - let mut state = State::default(); - req.must_advance = self.must_advance.load(); - - state.search(req); - - self.must_advance - .store(state.string_position == state.start); - self.start.store(state.string_position); - - Ok(state.has_matched.then(|| { - Match::new(&state, self.pattern.clone(), self.string.clone()).into_ref(&vm.ctx) - })) - }) - } - } -} diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs deleted file mode 100644 index cedff92d961..00000000000 --- a/vm/src/stdlib/string.rs +++ /dev/null @@ -1,98 +0,0 @@ -/* String builtin module - */ - -pub(crate) use _string::make_module; - -#[pymodule] -mod _string { - use crate::common::ascii; - use crate::{ - builtins::{PyList, PyStrRef}, - convert::ToPyException, - convert::ToPyObject, - PyObjectRef, PyResult, VirtualMachine, - }; - use rustpython_format::{ - FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate, - }; - use std::mem; - - fn create_format_part( - literal: String, - field_name: Option<String>, - format_spec: Option<String>, - conversion_spec: Option<char>, - vm: &VirtualMachine, - ) -> PyObjectRef { - let tuple = ( - literal, - field_name, - format_spec, - conversion_spec.map(|c| c.to_string()), - ); - tuple.to_pyobject(vm) - } - - #[pyfunction] - fn formatter_parser(text: PyStrRef, vm: &VirtualMachine) -> PyResult<PyList> { - let format_string = - FormatString::from_str(text.as_str()).map_err(|e| e.to_pyexception(vm))?; - - let mut result = Vec::new(); - let mut literal = String::new(); - for part in format_string.format_parts { - match part { - FormatPart::Field { - field_name, - conversion_spec, - format_spec, - } => { - result.push(create_format_part( - mem::take(&mut literal), - Some(field_name), - Some(format_spec), - conversion_spec, - vm, - )); - } - FormatPart::Literal(text) => literal.push_str(&text), - } - } - if !literal.is_empty() { - result.push(create_format_part( - mem::take(&mut literal), - None, - None, - None, - vm, - )); - } - Ok(result.into()) - } - - #[pyfunction] - fn formatter_field_name_split( - text: PyStrRef, - vm: &VirtualMachine, - ) -> PyResult<(PyObjectRef, PyList)> { - let field_name = FieldName::parse(text.as_str()).map_err(|e| e.to_pyexception(vm))?; - - let first = match field_name.field_type { - FieldType::Auto => vm.ctx.new_str(ascii!("")).into(), - FieldType::Index(index) => index.to_pyobject(vm), - FieldType::Keyword(attribute) => attribute.to_pyobject(vm), - }; - - let rest = field_name - .parts - .iter() - .map(|p| match p { - FieldNamePart::Attribute(attribute) => (true, attribute).to_pyobject(vm), - FieldNamePart::StringIndex(index) => (false, index).to_pyobject(vm), - FieldNamePart::Index(index) => (false, *index).to_pyobject(vm), - }) - .collect(); - - Ok((first, rest)) - } -} diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs deleted file mode 100644 index 10d79e9a8b5..00000000000 --- a/vm/src/stdlib/symtable.rs +++ /dev/null @@ -1,255 +0,0 @@ -pub(crate) use symtable::make_module; - -#[pymodule] -mod symtable { - use crate::{ - builtins::PyStrRef, compiler, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - }; - use rustpython_codegen::symboltable::{ - Symbol, SymbolFlags, SymbolScope, SymbolTable, SymbolTableType, - }; - use std::fmt; - - #[pyfunction] - fn symtable( - source: PyStrRef, - filename: PyStrRef, - mode: PyStrRef, - vm: &VirtualMachine, - ) -> PyResult<PyRef<PySymbolTable>> { - let mode = mode - .as_str() - .parse::<compiler::Mode>() - .map_err(|err| vm.new_value_error(err.to_string()))?; - - let symtable = compiler::compile_symtable(source.as_str(), mode, filename.as_str()) - .map_err(|err| vm.new_syntax_error(&err, Some(source.as_str())))?; - - let py_symbol_table = to_py_symbol_table(symtable); - Ok(py_symbol_table.into_ref(&vm.ctx)) - } - - fn to_py_symbol_table(symtable: SymbolTable) -> PySymbolTable { - PySymbolTable { symtable } - } - - #[pyattr] - #[pyclass(name = "SymbolTable")] - #[derive(PyPayload)] - struct PySymbolTable { - symtable: SymbolTable, - } - - impl fmt::Debug for PySymbolTable { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "SymbolTable()") - } - } - - #[pyclass] - impl PySymbolTable { - #[pymethod] - fn get_name(&self) -> String { - self.symtable.name.clone() - } - - #[pymethod] - fn get_type(&self) -> String { - self.symtable.typ.to_string() - } - - #[pymethod] - fn get_lineno(&self) -> u32 { - self.symtable.line_number - } - - #[pymethod] - fn is_nested(&self) -> bool { - self.symtable.is_nested - } - - #[pymethod] - fn is_optimized(&self) -> bool { - self.symtable.typ == SymbolTableType::Function - } - - #[pymethod] - fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<PyRef<PySymbol>> { - let name = name.as_str(); - if let Some(symbol) = self.symtable.symbols.get(name) { - Ok(PySymbol { - symbol: symbol.clone(), - namespaces: self - .symtable - .sub_tables - .iter() - .filter(|table| table.name == name) - .cloned() - .collect(), - is_top_scope: self.symtable.name == "top", - } - .into_ref(&vm.ctx)) - } else { - Err(vm.new_key_error(vm.ctx.new_str(format!("lookup {name} failed")).into())) - } - } - - #[pymethod] - fn get_identifiers(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let symbols = self - .symtable - .symbols - .keys() - .map(|s| vm.ctx.new_str(s.as_str()).into()) - .collect(); - Ok(symbols) - } - - #[pymethod] - fn get_symbols(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let symbols = self - .symtable - .symbols - .values() - .map(|s| { - (PySymbol { - symbol: s.clone(), - namespaces: self - .symtable - .sub_tables - .iter() - .filter(|&table| table.name == s.name) - .cloned() - .collect(), - is_top_scope: self.symtable.name == "top", - }) - .into_ref(&vm.ctx) - .into() - }) - .collect(); - Ok(symbols) - } - - #[pymethod] - fn has_children(&self) -> bool { - !self.symtable.sub_tables.is_empty() - } - - #[pymethod] - fn get_children(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let children = self - .symtable - .sub_tables - .iter() - .map(|t| to_py_symbol_table(t.clone()).into_pyobject(vm)) - .collect(); - Ok(children) - } - } - - #[pyattr] - #[pyclass(name = "Symbol")] - #[derive(PyPayload)] - struct PySymbol { - symbol: Symbol, - namespaces: Vec<SymbolTable>, - is_top_scope: bool, - } - - impl fmt::Debug for PySymbol { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Symbol()") - } - } - - #[pyclass] - impl PySymbol { - #[pymethod] - fn get_name(&self) -> String { - self.symbol.name.clone() - } - - #[pymethod] - fn is_global(&self) -> bool { - self.symbol.is_global() || (self.is_top_scope && self.symbol.is_bound()) - } - - #[pymethod] - fn is_declared_global(&self) -> bool { - matches!(self.symbol.scope, SymbolScope::GlobalExplicit) - } - - #[pymethod] - fn is_local(&self) -> bool { - self.symbol.is_local() || (self.is_top_scope && self.symbol.is_bound()) - } - - #[pymethod] - fn is_imported(&self) -> bool { - self.symbol.flags.contains(SymbolFlags::IMPORTED) - } - - #[pymethod] - fn is_nested(&self) -> bool { - // TODO - false - } - - #[pymethod] - fn is_nonlocal(&self) -> bool { - self.symbol.flags.contains(SymbolFlags::NONLOCAL) - } - - #[pymethod] - fn is_referenced(&self) -> bool { - self.symbol.flags.contains(SymbolFlags::REFERENCED) - } - - #[pymethod] - fn is_assigned(&self) -> bool { - self.symbol.flags.contains(SymbolFlags::ASSIGNED) - } - - #[pymethod] - fn is_parameter(&self) -> bool { - self.symbol.flags.contains(SymbolFlags::PARAMETER) - } - - #[pymethod] - fn is_free(&self) -> bool { - matches!(self.symbol.scope, SymbolScope::Free) - } - - #[pymethod] - fn is_namespace(&self) -> bool { - !self.namespaces.is_empty() - } - - #[pymethod] - fn is_annotated(&self) -> bool { - self.symbol.flags.contains(SymbolFlags::ANNOTATED) - } - - #[pymethod] - fn get_namespaces(&self, vm: &VirtualMachine) -> PyResult<Vec<PyObjectRef>> { - let namespaces = self - .namespaces - .iter() - .map(|table| to_py_symbol_table(table.clone()).into_pyobject(vm)) - .collect(); - Ok(namespaces) - } - - #[pymethod] - fn get_namespace(&self, vm: &VirtualMachine) -> PyResult { - if self.namespaces.len() != 1 { - return Err( - vm.new_value_error("namespace is bound to multiple namespaces".to_owned()) - ); - } - Ok(to_py_symbol_table(self.namespaces.first().unwrap().clone()) - .into_ref(&vm.ctx) - .into()) - } - } -} diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs deleted file mode 100644 index 4c8b6a060f8..00000000000 --- a/vm/src/stdlib/sys.rs +++ /dev/null @@ -1,1005 +0,0 @@ -use crate::{builtins::PyModule, convert::ToPyObject, Py, PyResult, VirtualMachine}; - -pub(crate) use sys::{UnraisableHookArgs, __module_def, DOC, MAXSIZE, MULTIARCH}; - -#[pymodule] -mod sys { - use crate::{ - builtins::{PyDictRef, PyNamespace, PyStr, PyStrRef, PyTupleRef, PyTypeRef}, - common::{ - ascii, - hash::{PyHash, PyUHash}, - }, - frame::FrameRef, - function::{FuncArgs, OptionalArg, PosArgs}, - stdlib::builtins, - stdlib::warnings::warn, - types::PyStructSequence, - version, - vm::{Settings, VirtualMachine}, - AsObject, PyObject, PyObjectRef, PyRef, PyRefExact, PyResult, - }; - use num_traits::ToPrimitive; - use std::{ - env::{self, VarError}, - path, - sync::atomic::Ordering, - }; - - // not the same as CPython (e.g. rust's x86_x64-unknown-linux-gnu is just x86_64-linux-gnu) - // but hopefully that's just an implementation detail? TODO: copy CPython's multiarch exactly, - // https://github.com/python/cpython/blob/3.8/configure.ac#L725 - pub(crate) const MULTIARCH: &str = env!("RUSTPYTHON_TARGET_TRIPLE"); - - #[pyattr(name = "_rustpython_debugbuild")] - const RUSTPYTHON_DEBUGBUILD: bool = cfg!(debug_assertions); - - #[pyattr(name = "abiflags")] - pub(crate) const ABIFLAGS: &str = ""; - #[pyattr(name = "api_version")] - const API_VERSION: u32 = 0x0; // what C api? - #[pyattr(name = "copyright")] - const COPYRIGHT: &str = "Copyright (c) 2019 RustPython Team"; - #[pyattr(name = "float_repr_style")] - const FLOAT_REPR_STYLE: &str = "short"; - #[pyattr(name = "_framework")] - const FRAMEWORK: &str = ""; - #[pyattr(name = "hexversion")] - const HEXVERSION: usize = version::VERSION_HEX; - #[pyattr(name = "maxsize")] - pub(crate) const MAXSIZE: isize = isize::MAX; - #[pyattr(name = "maxunicode")] - const MAXUNICODE: u32 = std::char::MAX as u32; - #[pyattr(name = "platform")] - pub(crate) const PLATFORM: &str = { - cfg_if::cfg_if! { - if #[cfg(any(target_os = "linux", target_os = "android"))] { - // Android is linux as well. see https://bugs.python.org/issue32637 - "linux" - } else if #[cfg(target_os = "macos")] { - "darwin" - } else if #[cfg(windows)] { - "win32" - } else if #[cfg(target_os = "wasi")] { - "wasi" - } else { - "unknown" - } - } - }; - #[pyattr(name = "ps1")] - const PS1: &str = ">>>>> "; - #[pyattr(name = "ps2")] - const PS2: &str = "..... "; - - #[cfg(windows)] - #[pyattr(name = "_vpath")] - const VPATH: Option<&'static str> = None; // TODO: actual VPATH value - - #[cfg(windows)] - #[pyattr(name = "dllhandle")] - const DLLHANDLE: usize = 0; - - #[pyattr] - fn default_prefix(_vm: &VirtualMachine) -> &'static str { - // TODO: the windows one doesn't really make sense - if cfg!(windows) { - "C:" - } else { - "/usr/local" - } - } - #[pyattr] - fn prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_PREFIX").unwrap_or_else(|| default_prefix(vm)) - } - #[pyattr] - fn base_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| prefix(vm)) - } - #[pyattr] - fn exec_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| prefix(vm)) - } - #[pyattr] - fn base_exec_prefix(vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_BASEPREFIX").unwrap_or_else(|| exec_prefix(vm)) - } - #[pyattr] - fn platlibdir(_vm: &VirtualMachine) -> &'static str { - option_env!("RUSTPYTHON_PLATLIBDIR").unwrap_or("lib") - } - - // alphabetical order with segments of pyattr and others - - #[pyattr] - fn argv(vm: &VirtualMachine) -> Vec<PyObjectRef> { - vm.state - .settings - .argv - .iter() - .map(|arg| vm.ctx.new_str(arg.clone()).into()) - .collect() - } - - #[pyattr] - fn builtin_module_names(vm: &VirtualMachine) -> PyTupleRef { - let mut module_names: Vec<_> = vm.state.module_inits.keys().cloned().collect(); - module_names.push("sys".into()); - module_names.push("builtins".into()); - module_names.sort(); - vm.ctx.new_tuple( - module_names - .into_iter() - .map(|n| vm.ctx.new_str(n).into()) - .collect(), - ) - } - - #[pyattr] - fn byteorder(vm: &VirtualMachine) -> PyStrRef { - // https://doc.rust-lang.org/reference/conditional-compilation.html#target_endian - vm.ctx - .intern_str(if cfg!(target_endian = "little") { - "little" - } else if cfg!(target_endian = "big") { - "big" - } else { - "unknown" - }) - .to_owned() - } - - #[pyattr] - fn _base_executable(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - if let Ok(var) = env::var("__PYVENV_LAUNCHER__") { - ctx.new_str(var).into() - } else { - executable(vm) - } - } - - #[pyattr] - fn dont_write_bytecode(vm: &VirtualMachine) -> bool { - vm.state.settings.dont_write_bytecode - } - - #[pyattr] - fn executable(vm: &VirtualMachine) -> PyObjectRef { - let ctx = &vm.ctx; - #[cfg(not(target_arch = "wasm32"))] - { - if let Some(exec_path) = env::args_os().next() { - if let Ok(path) = which::which(exec_path) { - return ctx - .new_str( - path.into_os_string() - .into_string() - .unwrap_or_else(|p| p.to_string_lossy().into_owned()), - ) - .into(); - } - } - } - if let Some(exec_path) = env::args().next() { - let path = path::Path::new(&exec_path); - if !path.exists() { - return ctx.new_str(ascii!("")).into(); - } - if path.is_absolute() { - return ctx.new_str(exec_path).into(); - } - if let Ok(dir) = env::current_dir() { - if let Ok(dir) = dir.into_os_string().into_string() { - return ctx - .new_str(format!( - "{}/{}", - dir, - exec_path.strip_prefix("./").unwrap_or(&exec_path) - )) - .into(); - } - } - } - ctx.none() - } - - #[pyattr] - fn _git(vm: &VirtualMachine) -> PyTupleRef { - vm.new_tuple(( - ascii!("RustPython"), - version::get_git_identifier(), - version::get_git_revision(), - )) - } - - #[pyattr] - fn implementation(vm: &VirtualMachine) -> PyRef<PyNamespace> { - // TODO: Add crate version to this namespace - let ctx = &vm.ctx; - py_namespace!(vm, { - "name" => ctx.new_str(ascii!("rustpython")), - "cache_tag" => ctx.new_str(ascii!("rustpython-01")), - "_multiarch" => ctx.new_str(MULTIARCH.to_owned()), - "version" => version_info(vm), - "hexversion" => ctx.new_int(version::VERSION_HEX), - }) - } - - #[pyattr] - fn meta_path(_vm: &VirtualMachine) -> Vec<PyObjectRef> { - Vec::new() - } - - #[pyattr] - fn orig_argv(vm: &VirtualMachine) -> Vec<PyObjectRef> { - env::args().map(|arg| vm.ctx.new_str(arg).into()).collect() - } - - #[pyattr] - fn path(vm: &VirtualMachine) -> Vec<PyObjectRef> { - vm.state - .settings - .path_list - .iter() - .map(|path| vm.ctx.new_str(path.clone()).into()) - .collect() - } - - #[pyattr] - fn path_hooks(_vm: &VirtualMachine) -> Vec<PyObjectRef> { - Vec::new() - } - - #[pyattr] - fn path_importer_cache(vm: &VirtualMachine) -> PyDictRef { - vm.ctx.new_dict() - } - - #[pyattr] - fn pycache_prefix(vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.none() - } - - #[pyattr] - fn version(_vm: &VirtualMachine) -> String { - version::get_version() - } - - #[cfg(windows)] - #[pyattr] - fn winver(_vm: &VirtualMachine) -> String { - // Note: This is Python DLL version in CPython, but we arbitrary fill it for compatibility - version::get_winver_number() - } - - #[pyattr] - fn _xoptions(vm: &VirtualMachine) -> PyDictRef { - let ctx = &vm.ctx; - let xopts = ctx.new_dict(); - for (key, value) in &vm.state.settings.xopts { - let value = value.as_ref().map_or_else( - || ctx.new_bool(true).into(), - |s| ctx.new_str(s.clone()).into(), - ); - xopts.set_item(&**key, value, vm).unwrap(); - } - xopts - } - - #[pyattr] - fn warnoptions(vm: &VirtualMachine) -> Vec<PyObjectRef> { - vm.state - .settings - .warnopts - .iter() - .map(|s| vm.ctx.new_str(s.clone()).into()) - .collect() - } - - #[pyfunction] - fn audit(_args: FuncArgs) { - // TODO: sys.audit implementation - } - - #[pyfunction] - fn exit(code: OptionalArg<PyObjectRef>, vm: &VirtualMachine) -> PyResult { - let code = code.unwrap_or_none(vm); - Err(vm.new_exception(vm.ctx.exceptions.system_exit.to_owned(), vec![code])) - } - - #[pyfunction(name = "__displayhook__")] - #[pyfunction] - fn displayhook(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - // Save non-None values as "_" - if vm.is_none(&obj) { - return Ok(()); - } - // set to none to avoid recursion while printing - vm.builtins.set_attr("_", vm.ctx.none(), vm)?; - // TODO: catch encoding errors - let repr = obj.repr(vm)?.into(); - builtins::print(PosArgs::new(vec![repr]), Default::default(), vm)?; - vm.builtins.set_attr("_", obj, vm)?; - Ok(()) - } - - #[pyfunction(name = "__excepthook__")] - #[pyfunction] - fn excepthook( - exc_type: PyObjectRef, - exc_val: PyObjectRef, - exc_tb: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - let exc = vm.normalize_exception(exc_type, exc_val, exc_tb)?; - let stderr = super::get_stderr(vm)?; - vm.write_exception(&mut crate::py_io::PyWriter(stderr, vm), &exc) - } - - #[pyfunction(name = "__breakpointhook__")] - #[pyfunction] - pub fn breakpointhook(args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let env_var = std::env::var("PYTHONBREAKPOINT") - .and_then(|env_var| { - if env_var.is_empty() { - Err(VarError::NotPresent) - } else { - Ok(env_var) - } - }) - .unwrap_or_else(|_| "pdb.set_trace".to_owned()); - - if env_var.eq("0") { - return Ok(vm.ctx.none()); - }; - - let print_unimportable_module_warn = || { - warn( - vm.ctx.exceptions.runtime_warning, - format!("Ignoring unimportable $PYTHONBREAKPOINT: \"{env_var}\"",), - 0, - vm, - ) - .unwrap(); - Ok(vm.ctx.none()) - }; - - let last = match env_var.rsplit_once('.') { - Some((_, last)) => last, - None if !env_var.is_empty() => env_var.as_str(), - _ => return print_unimportable_module_warn(), - }; - - let (module_path, attr_name) = if last == env_var { - ("builtins", env_var.as_str()) - } else { - (&env_var[..(env_var.len() - last.len() - 1)], last) - }; - - let module = match vm.import(&vm.ctx.new_str(module_path), None, 0) { - Ok(module) => module, - Err(_) => { - return print_unimportable_module_warn(); - } - }; - - match vm.get_attribute_opt(module, &vm.ctx.new_str(attr_name)) { - Ok(Some(hook)) => hook.as_ref().call(args, vm), - _ => print_unimportable_module_warn(), - } - } - - #[pyfunction] - fn exc_info(vm: &VirtualMachine) -> (PyObjectRef, PyObjectRef, PyObjectRef) { - match vm.topmost_exception() { - Some(exception) => vm.split_exception(exception), - None => (vm.ctx.none(), vm.ctx.none(), vm.ctx.none()), - } - } - - #[pyattr] - fn flags(vm: &VirtualMachine) -> PyTupleRef { - Flags::from_settings(&vm.state.settings).into_struct_sequence(vm) - } - - #[pyattr] - fn float_info(vm: &VirtualMachine) -> PyTupleRef { - PyFloatInfo::INFO.into_struct_sequence(vm) - } - - #[pyfunction] - fn getdefaultencoding() -> &'static str { - crate::codecs::DEFAULT_ENCODING - } - - #[pyfunction] - fn getrefcount(obj: PyObjectRef) -> usize { - obj.strong_count() - } - - #[pyfunction] - fn getrecursionlimit(vm: &VirtualMachine) -> usize { - vm.recursion_limit.get() - } - - #[derive(FromArgs)] - struct GetsizeofArgs { - obj: PyObjectRef, - #[pyarg(any, optional)] - default: Option<PyObjectRef>, - } - - #[pyfunction] - fn getsizeof(args: GetsizeofArgs, vm: &VirtualMachine) -> PyResult { - let sizeof = || -> PyResult<usize> { - let res = vm.call_special_method(&args.obj, identifier!(vm, __sizeof__), ())?; - let res = res.try_index(vm)?.try_to_primitive::<usize>(vm)?; - Ok(res + std::mem::size_of::<PyObject>()) - }; - sizeof() - .map(|x| vm.ctx.new_int(x).into()) - .or_else(|err| args.default.ok_or(err)) - } - - #[pyfunction] - fn getfilesystemencoding(_vm: &VirtualMachine) -> String { - // TODO: implement non-utf-8 mode. - "utf-8".to_owned() - } - - #[cfg(not(windows))] - #[pyfunction] - fn getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { - "surrogateescape".to_owned() - } - - #[cfg(windows)] - #[pyfunction] - fn getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { - "surrogatepass".to_owned() - } - - #[pyfunction] - fn getprofile(vm: &VirtualMachine) -> PyObjectRef { - vm.profile_func.borrow().clone() - } - - #[pyfunction] - fn _getframe(offset: OptionalArg<usize>, vm: &VirtualMachine) -> PyResult<FrameRef> { - let offset = offset.into_option().unwrap_or(0); - if offset > vm.frames.borrow().len() - 1 { - return Err(vm.new_value_error("call stack is not deep enough".to_owned())); - } - let idx = vm.frames.borrow().len() - offset - 1; - let frame = &vm.frames.borrow()[idx]; - Ok(frame.clone()) - } - - #[pyfunction] - fn gettrace(vm: &VirtualMachine) -> PyObjectRef { - vm.trace_func.borrow().clone() - } - - #[cfg(windows)] - #[pyfunction] - fn getwindowsversion(vm: &VirtualMachine) -> PyResult<crate::builtins::tuple::PyTupleRef> { - use std::ffi::OsString; - use std::os::windows::ffi::OsStringExt; - use windows_sys::Win32::System::SystemInformation::{ - GetVersionExW, OSVERSIONINFOEXW, OSVERSIONINFOW, - }; - - let mut version: OSVERSIONINFOEXW = unsafe { std::mem::zeroed() }; - version.dwOSVersionInfoSize = std::mem::size_of::<OSVERSIONINFOEXW>() as u32; - let result = unsafe { - let osvi = &mut version as *mut OSVERSIONINFOEXW as *mut OSVERSIONINFOW; - // SAFETY: GetVersionExW accepts a pointer of OSVERSIONINFOW, but windows-sys crate's type currently doesn't allow to do so. - // https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getversionexw#parameters - GetVersionExW(osvi) - }; - - if result == 0 { - return Err(vm.new_os_error("failed to get windows version".to_owned())); - } - - let service_pack = { - let (last, _) = version - .szCSDVersion - .iter() - .take_while(|&x| x != &0) - .enumerate() - .last() - .unwrap_or((0, &0)); - let sp = OsString::from_wide(&version.szCSDVersion[..last]); - sp.into_string() - .map_err(|_| vm.new_os_error("service pack is not ASCII".to_owned()))? - }; - Ok(WindowsVersion { - major: version.dwMajorVersion, - minor: version.dwMinorVersion, - build: version.dwBuildNumber, - platform: version.dwPlatformId, - service_pack, - service_pack_major: version.wServicePackMajor, - service_pack_minor: version.wServicePackMinor, - suite_mask: version.wSuiteMask, - product_type: version.wProductType, - platform_version: ( - version.dwMajorVersion, - version.dwMinorVersion, - version.dwBuildNumber, - ), // TODO Provide accurate version, like CPython impl - } - .into_struct_sequence(vm)) - } - - fn _unraisablehook(unraisable: UnraisableHookArgs, vm: &VirtualMachine) -> PyResult<()> { - use super::PyStderr; - - let stderr = PyStderr(vm); - if !vm.is_none(&unraisable.object) { - if !vm.is_none(&unraisable.err_msg) { - write!(stderr, "{}: ", unraisable.err_msg.str(vm)?); - } else { - write!(stderr, "Exception ignored in: "); - } - // exception in del will be ignored but printed - let repr = &unraisable.object.repr(vm); - let str = match repr { - Ok(v) => v.to_string(), - Err(_) => format!( - "<object {} repr() failed>", - unraisable.object.class().name() - ), - }; - writeln!(stderr, "{str}"); - } else if !vm.is_none(&unraisable.err_msg) { - writeln!(stderr, "{}:", unraisable.err_msg.str(vm)?); - } - - // TODO: print received unraisable.exc_traceback - let tb_module = vm.import("traceback", None, 0)?; - let print_stack = tb_module.get_attr("print_stack", vm)?; - print_stack.call((), vm)?; - - if vm.is_none(unraisable.exc_type.as_object()) { - // TODO: early return, but with what error? - } - assert!(unraisable - .exc_type - .fast_issubclass(vm.ctx.exceptions.base_exception_type)); - - // TODO: print module name and qualname - - if !vm.is_none(&unraisable.exc_value) { - write!(stderr, "{}: ", unraisable.exc_type); - if let Ok(str) = unraisable.exc_value.str(vm) { - write!(stderr, "{}", str.as_str()); - } else { - write!(stderr, "<exception str() failed>"); - } - } - writeln!(stderr); - // TODO: call file.flush() - - Ok(()) - } - - #[pyattr] - #[pyfunction(name = "__unraisablehook__")] - fn unraisablehook(unraisable: UnraisableHookArgs, vm: &VirtualMachine) { - if let Err(e) = _unraisablehook(unraisable, vm) { - let stderr = super::PyStderr(vm); - writeln!( - stderr, - "{}", - e.as_object() - .repr(vm) - .unwrap_or_else(|_| vm.ctx.empty_str.to_owned()) - .as_str() - ); - } - } - - #[pyattr] - fn hash_info(vm: &VirtualMachine) -> PyTupleRef { - PyHashInfo::INFO.into_struct_sequence(vm) - } - - #[pyfunction] - fn intern(s: PyRefExact<PyStr>, vm: &VirtualMachine) -> PyRef<PyStr> { - vm.ctx.intern_str(s).to_owned() - } - - #[pyattr] - fn int_info(vm: &VirtualMachine) -> PyTupleRef { - PyIntInfo::INFO.into_struct_sequence(vm) - } - - #[pyfunction] - fn get_int_max_str_digits(vm: &VirtualMachine) -> usize { - vm.state.int_max_str_digits.load() - } - - #[pyfunction] - fn set_int_max_str_digits(maxdigits: usize, vm: &VirtualMachine) -> PyResult<()> { - let threshold = PyIntInfo::INFO.str_digits_check_threshold; - if maxdigits == 0 || maxdigits >= threshold { - vm.state.int_max_str_digits.store(maxdigits); - Ok(()) - } else { - let error = format!("maxdigits must be 0 or larger than {:?}", threshold); - Err(vm.new_value_error(error)) - } - } - - #[pyfunction] - fn is_finalizing(vm: &VirtualMachine) -> bool { - vm.state.finalizing.load(Ordering::Acquire) - } - - #[pyfunction] - fn setprofile(profilefunc: PyObjectRef, vm: &VirtualMachine) { - vm.profile_func.replace(profilefunc); - update_use_tracing(vm); - } - - #[pyfunction] - fn setrecursionlimit(recursion_limit: i32, vm: &VirtualMachine) -> PyResult<()> { - let recursion_limit = recursion_limit - .to_usize() - .filter(|&u| u >= 1) - .ok_or_else(|| { - vm.new_value_error( - "recursion limit must be greater than or equal to one".to_owned(), - ) - })?; - let recursion_depth = vm.current_recursion_depth(); - - if recursion_limit > recursion_depth { - vm.recursion_limit.set(recursion_limit); - Ok(()) - } else { - Err(vm.new_recursion_error(format!( - "cannot set the recursion limit to {recursion_limit} at the recursion depth {recursion_depth}: the limit is too low" - ))) - } - } - - #[pyfunction] - fn settrace(tracefunc: PyObjectRef, vm: &VirtualMachine) { - vm.trace_func.replace(tracefunc); - update_use_tracing(vm); - } - - #[cfg(feature = "threading")] - #[pyattr] - fn thread_info(vm: &VirtualMachine) -> PyTupleRef { - PyThreadInfo::INFO.into_struct_sequence(vm) - } - - #[pyattr] - fn version_info(vm: &VirtualMachine) -> PyTupleRef { - VersionInfo::VERSION.into_struct_sequence(vm) - } - - fn update_use_tracing(vm: &VirtualMachine) { - let trace_is_none = vm.is_none(&vm.trace_func.borrow()); - let profile_is_none = vm.is_none(&vm.profile_func.borrow()); - let tracing = !(trace_is_none && profile_is_none); - vm.use_tracing.set(tracing); - } - - /// sys.flags - /// - /// Flags provided through command line arguments or environment vars. - #[pyclass(no_attr, name = "flags", module = "sys")] - #[derive(Debug, PyStructSequence)] - pub(super) struct Flags { - /// -d - debug: u8, - /// -i - inspect: u8, - /// -i - interactive: u8, - /// -O or -OO - optimize: u8, - /// -B - dont_write_bytecode: u8, - /// -s - no_user_site: u8, - /// -S - no_site: u8, - /// -E - ignore_environment: u8, - /// -v - verbose: u8, - /// -b - bytes_warning: u64, - /// -q - quiet: u8, - /// -R - hash_randomization: u8, - /// -I - isolated: u8, - /// -X dev - dev_mode: bool, - /// -X utf8 - utf8_mode: u8, - /// -X int_max_str_digits=number - int_max_str_digits: i64, - /// -P, `PYTHONSAFEPATH` - safe_path: bool, - /// -X warn_default_encoding, PYTHONWARNDEFAULTENCODING - warn_default_encoding: u8, - } - - #[pyclass(with(PyStructSequence))] - impl Flags { - fn from_settings(settings: &Settings) -> Self { - Self { - debug: settings.debug as u8, - inspect: settings.inspect as u8, - interactive: settings.interactive as u8, - optimize: settings.optimize, - dont_write_bytecode: settings.dont_write_bytecode as u8, - no_user_site: settings.no_user_site as u8, - no_site: settings.no_site as u8, - ignore_environment: settings.ignore_environment as u8, - verbose: settings.verbose, - bytes_warning: settings.bytes_warning, - quiet: settings.quiet as u8, - hash_randomization: settings.hash_seed.is_none() as u8, - isolated: settings.isolated as u8, - dev_mode: settings.dev_mode, - utf8_mode: settings.utf8_mode, - int_max_str_digits: settings.int_max_str_digits, - safe_path: settings.safe_path, - warn_default_encoding: settings.warn_default_encoding as u8, - } - } - - #[pyslot] - fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("cannot create 'sys.flags' instances".to_owned())) - } - } - - #[cfg(feature = "threading")] - #[pyclass(no_attr, name = "thread_info")] - #[derive(PyStructSequence)] - pub(super) struct PyThreadInfo { - name: Option<&'static str>, - lock: Option<&'static str>, - version: Option<&'static str>, - } - - #[cfg(feature = "threading")] - #[pyclass(with(PyStructSequence))] - impl PyThreadInfo { - const INFO: Self = PyThreadInfo { - name: crate::stdlib::thread::_thread::PYTHREAD_NAME, - // As I know, there's only way to use lock as "Mutex" in Rust - // with satisfying python document spec. - lock: Some("mutex+cond"), - version: None, - }; - } - - #[pyclass(no_attr, name = "float_info")] - #[derive(PyStructSequence)] - pub(super) struct PyFloatInfo { - max: f64, - max_exp: i32, - max_10_exp: i32, - min: f64, - min_exp: i32, - min_10_exp: i32, - dig: u32, - mant_dig: u32, - epsilon: f64, - radix: u32, - rounds: i32, - } - - #[pyclass(with(PyStructSequence))] - impl PyFloatInfo { - const INFO: Self = PyFloatInfo { - max: f64::MAX, - max_exp: f64::MAX_EXP, - max_10_exp: f64::MAX_10_EXP, - min: f64::MIN_POSITIVE, - min_exp: f64::MIN_EXP, - min_10_exp: f64::MIN_10_EXP, - dig: f64::DIGITS, - mant_dig: f64::MANTISSA_DIGITS, - epsilon: f64::EPSILON, - radix: f64::RADIX, - rounds: 1, // FE_TONEAREST - }; - } - - #[pyclass(no_attr, name = "hash_info")] - #[derive(PyStructSequence)] - pub(super) struct PyHashInfo { - width: usize, - modulus: PyUHash, - inf: PyHash, - nan: PyHash, - imag: PyHash, - algorithm: &'static str, - hash_bits: usize, - seed_bits: usize, - cutoff: usize, - } - - #[pyclass(with(PyStructSequence))] - impl PyHashInfo { - const INFO: Self = { - use rustpython_common::hash::*; - PyHashInfo { - width: std::mem::size_of::<PyHash>() * 8, - modulus: MODULUS, - inf: INF, - nan: NAN, - imag: IMAG, - algorithm: ALGO, - hash_bits: HASH_BITS, - seed_bits: SEED_BITS, - cutoff: 0, // no small string optimizations - } - }; - } - - #[pyclass(no_attr, name = "int_info")] - #[derive(PyStructSequence)] - pub(super) struct PyIntInfo { - bits_per_digit: usize, - sizeof_digit: usize, - default_max_str_digits: usize, - str_digits_check_threshold: usize, - } - - #[pyclass(with(PyStructSequence))] - impl PyIntInfo { - const INFO: Self = PyIntInfo { - bits_per_digit: 30, //? - sizeof_digit: std::mem::size_of::<u32>(), - default_max_str_digits: 4300, - str_digits_check_threshold: 640, - }; - } - - #[pyclass(no_attr, name = "version_info")] - #[derive(Default, Debug, PyStructSequence)] - pub struct VersionInfo { - major: usize, - minor: usize, - micro: usize, - releaselevel: &'static str, - serial: usize, - } - - #[pyclass(with(PyStructSequence))] - impl VersionInfo { - pub const VERSION: VersionInfo = VersionInfo { - major: version::MAJOR, - minor: version::MINOR, - micro: version::MICRO, - releaselevel: version::RELEASELEVEL, - serial: version::SERIAL, - }; - #[pyslot] - fn slot_new( - _cls: crate::builtins::type_::PyTypeRef, - _args: crate::function::FuncArgs, - vm: &crate::VirtualMachine, - ) -> crate::PyResult { - Err(vm.new_type_error("cannot create 'sys.version_info' instances".to_owned())) - } - } - - #[cfg(windows)] - #[pyclass(no_attr, name = "getwindowsversion")] - #[derive(Default, Debug, PyStructSequence)] - pub(super) struct WindowsVersion { - major: u32, - minor: u32, - build: u32, - platform: u32, - service_pack: String, - service_pack_major: u16, - service_pack_minor: u16, - suite_mask: u16, - product_type: u8, - platform_version: (u32, u32, u32), - } - - #[cfg(windows)] - #[pyclass(with(PyStructSequence))] - impl WindowsVersion {} - - #[pyclass(no_attr, name = "UnraisableHookArgs")] - #[derive(Debug, PyStructSequence, TryIntoPyStructSequence)] - pub struct UnraisableHookArgs { - pub exc_type: PyTypeRef, - pub exc_value: PyObjectRef, - pub exc_traceback: PyObjectRef, - pub err_msg: PyObjectRef, - pub object: PyObjectRef, - } - - #[pyclass(with(PyStructSequence))] - impl UnraisableHookArgs {} -} - -pub(crate) fn init_module(vm: &VirtualMachine, module: &Py<PyModule>, builtins: &Py<PyModule>) { - sys::extend_module(vm, module).unwrap(); - - let modules = vm.ctx.new_dict(); - modules - .set_item("sys", module.to_owned().into(), vm) - .unwrap(); - modules - .set_item("builtins", builtins.to_owned().into(), vm) - .unwrap(); - extend_module!(vm, module, { - "__doc__" => sys::DOC.to_owned().to_pyobject(vm), - "modules" => modules, - }); -} - -/// Similar to PySys_WriteStderr in CPython. -/// -/// # Usage -/// -/// ```rust,ignore -/// writeln!(sys::PyStderr(vm), "foo bar baz :)"); -/// ``` -/// -/// Unlike writing to a `std::io::Write` with the `write[ln]!()` macro, there's no error condition here; -/// this is intended to be a replacement for the `eprint[ln]!()` macro, so `write!()`-ing to PyStderr just -/// returns `()`. -pub struct PyStderr<'vm>(pub &'vm VirtualMachine); - -impl PyStderr<'_> { - pub fn write_fmt(&self, args: std::fmt::Arguments<'_>) { - use crate::py_io::Write; - - let vm = self.0; - if let Ok(stderr) = get_stderr(vm) { - let mut stderr = crate::py_io::PyWriter(stderr, vm); - if let Ok(()) = stderr.write_fmt(args) { - return; - } - } - eprint!("{args}") - } -} - -pub fn get_stdin(vm: &VirtualMachine) -> PyResult { - vm.sys_module - .get_attr("stdin", vm) - .map_err(|_| vm.new_runtime_error("lost sys.stdin".to_owned())) -} -pub fn get_stdout(vm: &VirtualMachine) -> PyResult { - vm.sys_module - .get_attr("stdout", vm) - .map_err(|_| vm.new_runtime_error("lost sys.stdout".to_owned())) -} -pub fn get_stderr(vm: &VirtualMachine) -> PyResult { - vm.sys_module - .get_attr("stderr", vm) - .map_err(|_| vm.new_runtime_error("lost sys.stderr".to_owned())) -} - -pub(crate) fn sysconfigdata_name() -> String { - format!( - "_sysconfigdata_{}_{}_{}", - sys::ABIFLAGS, - sys::PLATFORM, - sys::MULTIARCH - ) -} diff --git a/vm/src/stdlib/sysconfigdata.rs b/vm/src/stdlib/sysconfigdata.rs deleted file mode 100644 index 929227ac11f..00000000000 --- a/vm/src/stdlib/sysconfigdata.rs +++ /dev/null @@ -1,25 +0,0 @@ -pub(crate) use _sysconfigdata::make_module; - -#[pymodule] -pub(crate) mod _sysconfigdata { - use crate::{builtins::PyDictRef, convert::ToPyObject, stdlib::sys::MULTIARCH, VirtualMachine}; - - #[pyattr] - fn build_time_vars(vm: &VirtualMachine) -> PyDictRef { - let vars = vm.ctx.new_dict(); - macro_rules! sysvars { - ($($key:literal => $value:expr),*$(,)?) => {{ - $(vars.set_item($key, $value.to_pyobject(vm), vm).unwrap();)* - }}; - } - sysvars! { - // fake shared module extension - "EXT_SUFFIX" => format!(".rustpython-{MULTIARCH}"), - "MULTIARCH" => MULTIARCH, - // enough for tests to stop expecting urandom() to fail after restricting file resources - "HAVE_GETRANDOM" => 1, - } - include!(concat!(env!("OUT_DIR"), "/env_vars.rs")); - vars - } -} diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs deleted file mode 100644 index 63e66474d15..00000000000 --- a/vm/src/stdlib/thread.rs +++ /dev/null @@ -1,444 +0,0 @@ -//! Implementation of the _thread module -#[cfg_attr(target_arch = "wasm32", allow(unused_imports))] -pub(crate) use _thread::{make_module, RawRMutex}; - -#[pymodule] -pub(crate) mod _thread { - use crate::{ - builtins::{PyDictRef, PyStr, PyTupleRef, PyTypeRef}, - convert::ToPyException, - function::{ArgCallable, Either, FuncArgs, KwArgs, OptionalArg, PySetterValue}, - types::{Constructor, GetAttr, Representable, SetAttr}, - AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine, - }; - use crossbeam_utils::atomic::AtomicCell; - use parking_lot::{ - lock_api::{RawMutex as RawMutexT, RawMutexTimed, RawReentrantMutex}, - RawMutex, RawThreadId, - }; - use std::{cell::RefCell, fmt, thread, time::Duration}; - use thread_local::ThreadLocal; - - // PYTHREAD_NAME: show current thread name - pub const PYTHREAD_NAME: Option<&str> = { - cfg_if::cfg_if! { - if #[cfg(windows)] { - Some("nt") - } else if #[cfg(unix)] { - Some("pthread") - } else if #[cfg(any(target_os = "solaris", target_os = "illumos"))] { - Some("solaris") - } else { - None - } - } - }; - - // TIMEOUT_MAX_IN_MICROSECONDS is a value in microseconds - #[cfg(not(target_os = "windows"))] - const TIMEOUT_MAX_IN_MICROSECONDS: i64 = i64::MAX / 1_000; - - #[cfg(target_os = "windows")] - const TIMEOUT_MAX_IN_MICROSECONDS: i64 = 0xffffffff * 1_000; - - // this is a value in seconds - #[pyattr] - const TIMEOUT_MAX: f64 = (TIMEOUT_MAX_IN_MICROSECONDS / 1_000_000) as f64; - - #[pyattr] - fn error(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.exceptions.runtime_error.to_owned() - } - - #[derive(FromArgs)] - struct AcquireArgs { - #[pyarg(any, default = "true")] - blocking: bool, - #[pyarg(any, default = "Either::A(-1.0)")] - timeout: Either<f64, i64>, - } - - macro_rules! acquire_lock_impl { - ($mu:expr, $args:expr, $vm:expr) => {{ - let (mu, args, vm) = ($mu, $args, $vm); - let timeout = match args.timeout { - Either::A(f) => f, - Either::B(i) => i as f64, - }; - match args.blocking { - true if timeout == -1.0 => { - mu.lock(); - Ok(true) - } - true if timeout < 0.0 => { - Err(vm.new_value_error("timeout value must be positive".to_owned())) - } - true => { - // modified from std::time::Duration::from_secs_f64 to avoid a panic. - // TODO: put this in the Duration::try_from_object impl, maybe? - let nanos = timeout * 1_000_000_000.0; - if timeout > TIMEOUT_MAX as f64 || nanos < 0.0 || !nanos.is_finite() { - return Err(vm.new_overflow_error( - "timestamp too large to convert to Rust Duration".to_owned(), - )); - } - - Ok(mu.try_lock_for(Duration::from_secs_f64(timeout))) - } - false if timeout != -1.0 => Err(vm - .new_value_error("can't specify a timeout for a non-blocking call".to_owned())), - false => Ok(mu.try_lock()), - } - }}; - } - macro_rules! repr_lock_impl { - ($zelf:expr) => {{ - let status = if $zelf.mu.is_locked() { - "locked" - } else { - "unlocked" - }; - Ok(format!( - "<{} {} object at {:#x}>", - status, - $zelf.class().name(), - $zelf.get_id() - )) - }}; - } - - #[pyattr(name = "LockType")] - #[pyclass(module = "thread", name = "lock")] - #[derive(PyPayload)] - struct Lock { - mu: RawMutex, - } - - impl fmt::Debug for Lock { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad("Lock") - } - } - - #[pyclass(with(Constructor, Representable))] - impl Lock { - #[pymethod] - #[pymethod(name = "acquire_lock")] - #[pymethod(name = "__enter__")] - fn acquire(&self, args: AcquireArgs, vm: &VirtualMachine) -> PyResult<bool> { - acquire_lock_impl!(&self.mu, args, vm) - } - #[pymethod] - #[pymethod(name = "release_lock")] - fn release(&self, vm: &VirtualMachine) -> PyResult<()> { - if !self.mu.is_locked() { - return Err(vm.new_runtime_error("release unlocked lock".to_owned())); - } - unsafe { self.mu.unlock() }; - Ok(()) - } - - #[pymethod] - fn _at_fork_reinit(&self, _vm: &VirtualMachine) -> PyResult<()> { - if self.mu.is_locked() { - unsafe { - self.mu.unlock(); - }; - } - // Casting to AtomicCell is as unsafe as CPython code. - // Using AtomicCell will prevent compiler optimizer move it to somewhere later unsafe place. - // It will be not under the cell anymore after init call. - - let new_mut = RawMutex::INIT; - unsafe { - let old_mutex: &AtomicCell<RawMutex> = std::mem::transmute(&self.mu); - old_mutex.swap(new_mut); - } - - Ok(()) - } - - #[pymethod(magic)] - fn exit(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - self.release(vm) - } - - #[pymethod] - fn locked(&self) -> bool { - self.mu.is_locked() - } - } - - impl Constructor for Lock { - type Args = FuncArgs; - fn py_new(_cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("cannot create '_thread.lock' instances".to_owned())) - } - } - - impl Representable for Lock { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - repr_lock_impl!(zelf) - } - } - - pub type RawRMutex = RawReentrantMutex<RawMutex, RawThreadId>; - #[pyattr] - #[pyclass(module = "thread", name = "RLock")] - #[derive(PyPayload)] - struct RLock { - mu: RawRMutex, - } - - impl fmt::Debug for RLock { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad("RLock") - } - } - - #[pyclass(with(Representable))] - impl RLock { - #[pyslot] - fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - RLock { - mu: RawRMutex::INIT, - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - - #[pymethod] - #[pymethod(name = "acquire_lock")] - #[pymethod(name = "__enter__")] - fn acquire(&self, args: AcquireArgs, vm: &VirtualMachine) -> PyResult<bool> { - acquire_lock_impl!(&self.mu, args, vm) - } - #[pymethod] - #[pymethod(name = "release_lock")] - fn release(&self, vm: &VirtualMachine) -> PyResult<()> { - if !self.mu.is_locked() { - return Err(vm.new_runtime_error("release unlocked lock".to_owned())); - } - unsafe { self.mu.unlock() }; - Ok(()) - } - - #[pymethod] - fn _at_fork_reinit(&self, _vm: &VirtualMachine) -> PyResult<()> { - if self.mu.is_locked() { - unsafe { - self.mu.unlock(); - }; - } - let new_mut = RawRMutex::INIT; - - let old_mutex: AtomicCell<&RawRMutex> = AtomicCell::new(&self.mu); - old_mutex.swap(&new_mut); - - Ok(()) - } - - #[pymethod] - fn _is_owned(&self) -> bool { - self.mu.is_owned_by_current_thread() - } - - #[pymethod(magic)] - fn exit(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - self.release(vm) - } - } - - impl Representable for RLock { - #[inline] - fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> { - repr_lock_impl!(zelf) - } - } - - #[pyfunction] - fn get_ident() -> u64 { - thread_to_id(&thread::current()) - } - - fn thread_to_id(t: &thread::Thread) -> u64 { - use std::hash::{Hash, Hasher}; - struct U64Hash { - v: Option<u64>, - } - impl Hasher for U64Hash { - fn write(&mut self, _: &[u8]) { - unreachable!() - } - fn write_u64(&mut self, i: u64) { - self.v = Some(i); - } - fn finish(&self) -> u64 { - self.v.expect("should have written a u64") - } - } - // TODO: use id.as_u64() once it's stable, until then, ThreadId is just a wrapper - // around NonZeroU64, so this should work (?) - let mut h = U64Hash { v: None }; - t.id().hash(&mut h); - h.finish() - } - - #[pyfunction] - fn allocate_lock() -> Lock { - Lock { mu: RawMutex::INIT } - } - - #[pyfunction] - fn start_new_thread( - func: ArgCallable, - args: PyTupleRef, - kwargs: OptionalArg<PyDictRef>, - vm: &VirtualMachine, - ) -> PyResult<u64> { - let args = FuncArgs::new( - args.to_vec(), - kwargs - .map_or_else(Default::default, |k| k.to_attributes(vm)) - .into_iter() - .map(|(k, v)| (k.as_str().to_owned(), v)) - .collect::<KwArgs>(), - ); - let mut thread_builder = thread::Builder::new(); - let stacksize = vm.state.stacksize.load(); - if stacksize != 0 { - thread_builder = thread_builder.stack_size(stacksize); - } - thread_builder - .spawn( - vm.new_thread() - .make_spawn_func(move |vm| run_thread(func, args, vm)), - ) - .map(|handle| { - vm.state.thread_count.fetch_add(1); - thread_to_id(handle.thread()) - }) - .map_err(|err| err.to_pyexception(vm)) - } - - fn run_thread(func: ArgCallable, args: FuncArgs, vm: &VirtualMachine) { - match func.invoke(args, vm) { - Ok(_obj) => {} - Err(e) if e.fast_isinstance(vm.ctx.exceptions.system_exit) => {} - Err(exc) => { - vm.run_unraisable( - exc, - Some("Exception ignored in thread started by".to_owned()), - func.into(), - ); - } - } - SENTINELS.with(|sents| { - for lock in sents.replace(Default::default()) { - if lock.mu.is_locked() { - unsafe { lock.mu.unlock() }; - } - } - }); - vm.state.thread_count.fetch_sub(1); - } - - #[cfg(not(target_arch = "wasm32"))] - #[pyfunction] - fn interrupt_main(signum: OptionalArg<i32>, vm: &VirtualMachine) -> PyResult<()> { - crate::signal::set_interrupt_ex(signum.unwrap_or(libc::SIGINT), vm) - } - - #[pyfunction] - fn exit(vm: &VirtualMachine) -> PyResult { - Err(vm.new_exception_empty(vm.ctx.exceptions.system_exit.to_owned())) - } - - thread_local!(static SENTINELS: RefCell<Vec<PyRef<Lock>>> = RefCell::default()); - - #[pyfunction] - fn _set_sentinel(vm: &VirtualMachine) -> PyRef<Lock> { - let lock = Lock { mu: RawMutex::INIT }.into_ref(&vm.ctx); - SENTINELS.with(|sents| sents.borrow_mut().push(lock.clone())); - lock - } - - #[pyfunction] - fn stack_size(size: OptionalArg<usize>, vm: &VirtualMachine) -> usize { - let size = size.unwrap_or(0); - // TODO: do validation on this to make sure it's not too small - vm.state.stacksize.swap(size) - } - - #[pyfunction] - fn _count(vm: &VirtualMachine) -> usize { - vm.state.thread_count.load() - } - - #[pyattr] - #[pyclass(module = "thread", name = "_local")] - #[derive(Debug, PyPayload)] - struct Local { - data: ThreadLocal<PyDictRef>, - } - - #[pyclass(with(GetAttr, SetAttr), flags(BASETYPE))] - impl Local { - fn ldict(&self, vm: &VirtualMachine) -> PyDictRef { - self.data.get_or(|| vm.ctx.new_dict()).clone() - } - - #[pyslot] - fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Local { - data: ThreadLocal::new(), - } - .into_ref_with_type(vm, cls) - .map(Into::into) - } - } - - impl GetAttr for Local { - fn getattro(zelf: &Py<Self>, attr: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - let ldict = zelf.ldict(vm); - if attr.as_str() == "__dict__" { - Ok(ldict.into()) - } else { - zelf.as_object() - .generic_getattr_opt(attr, Some(ldict), vm)? - .ok_or_else(|| { - vm.new_attribute_error(format!( - "{} has no attribute '{}'", - zelf.class().name(), - attr - )) - }) - } - } - } - - impl SetAttr for Local { - fn setattro( - zelf: &Py<Self>, - attr: &Py<PyStr>, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()> { - if attr.as_str() == "__dict__" { - Err(vm.new_attribute_error(format!( - "{} attribute '__dict__' is read-only", - zelf.class().name() - ))) - } else { - let dict = zelf.ldict(vm); - if let PySetterValue::Assign(value) = value { - dict.set_item(attr, value, vm)?; - } else { - dict.del_item(attr, vm)?; - } - Ok(()) - } - } - } -} diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs deleted file mode 100644 index 9717a69665f..00000000000 --- a/vm/src/stdlib/time.rs +++ /dev/null @@ -1,800 +0,0 @@ -//! The python `time` module. - -// See also: -// https://docs.python.org/3/library/time.html -pub use time::*; - -#[pymodule(name = "time", with(platform))] -mod time { - use crate::{ - builtins::{PyStrRef, PyTypeRef}, - function::{Either, FuncArgs, OptionalArg}, - types::PyStructSequence, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, - }; - use chrono::{ - naive::{NaiveDate, NaiveDateTime, NaiveTime}, - Datelike, Timelike, - }; - use std::time::Duration; - - #[allow(dead_code)] - pub(super) const SEC_TO_MS: i64 = 1000; - #[allow(dead_code)] - pub(super) const MS_TO_US: i64 = 1000; - #[allow(dead_code)] - pub(super) const SEC_TO_US: i64 = SEC_TO_MS * MS_TO_US; - #[allow(dead_code)] - pub(super) const US_TO_NS: i64 = 1000; - #[allow(dead_code)] - pub(super) const MS_TO_NS: i64 = MS_TO_US * US_TO_NS; - #[allow(dead_code)] - pub(super) const SEC_TO_NS: i64 = SEC_TO_MS * MS_TO_NS; - #[allow(dead_code)] - pub(super) const NS_TO_MS: i64 = 1000 * 1000; - #[allow(dead_code)] - pub(super) const NS_TO_US: i64 = 1000; - - fn duration_since_system_now(vm: &VirtualMachine) -> PyResult<Duration> { - use std::time::{SystemTime, UNIX_EPOCH}; - - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|e| vm.new_value_error(format!("Time error: {e:?}"))) - } - - // TODO: implement proper monotonic time for wasm/wasi. - #[cfg(not(any(unix, windows)))] - fn get_monotonic_time(vm: &VirtualMachine) -> PyResult<Duration> { - duration_since_system_now(vm) - } - - // TODO: implement proper perf time for wasm/wasi. - #[cfg(not(any(unix, windows)))] - fn get_perf_time(vm: &VirtualMachine) -> PyResult<Duration> { - duration_since_system_now(vm) - } - - #[cfg(not(unix))] - #[pyfunction] - fn sleep(dur: Duration) { - std::thread::sleep(dur); - } - - #[cfg(not(target_os = "wasi"))] - #[pyfunction] - fn time_ns(vm: &VirtualMachine) -> PyResult<u64> { - Ok(duration_since_system_now(vm)?.as_nanos() as u64) - } - - #[pyfunction] - pub fn time(vm: &VirtualMachine) -> PyResult<f64> { - _time(vm) - } - - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - fn _time(vm: &VirtualMachine) -> PyResult<f64> { - Ok(duration_since_system_now(vm)?.as_secs_f64()) - } - - #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] - fn _time(_vm: &VirtualMachine) -> PyResult<f64> { - use wasm_bindgen::prelude::*; - #[wasm_bindgen] - extern "C" { - type Date; - #[wasm_bindgen(static_method_of = Date)] - fn now() -> f64; - } - // Date.now returns unix time in milliseconds, we want it in seconds - Ok(Date::now() / 1000.0) - } - - #[pyfunction] - fn monotonic(vm: &VirtualMachine) -> PyResult<f64> { - Ok(get_monotonic_time(vm)?.as_secs_f64()) - } - - #[pyfunction] - fn monotonic_ns(vm: &VirtualMachine) -> PyResult<u128> { - Ok(get_monotonic_time(vm)?.as_nanos()) - } - - #[pyfunction] - fn perf_counter(vm: &VirtualMachine) -> PyResult<f64> { - Ok(get_perf_time(vm)?.as_secs_f64()) - } - - #[pyfunction] - fn perf_counter_ns(vm: &VirtualMachine) -> PyResult<u128> { - Ok(get_perf_time(vm)?.as_nanos()) - } - - fn pyobj_to_naive_date_time( - value: Either<f64, i64>, - vm: &VirtualMachine, - ) -> PyResult<NaiveDateTime> { - let timestamp = match value { - Either::A(float) => { - let secs = float.trunc() as i64; - let nsecs = (float.fract() * 1e9) as u32; - NaiveDateTime::from_timestamp_opt(secs, nsecs) - } - Either::B(int) => NaiveDateTime::from_timestamp_opt(int, 0), - }; - timestamp.ok_or_else(|| { - vm.new_overflow_error("timestamp out of range for platform time_t".to_owned()) - }) - } - - impl OptionalArg<Either<f64, i64>> { - /// Construct a localtime from the optional seconds, or get the current local time. - fn naive_or_local(self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { - Ok(match self { - OptionalArg::Present(secs) => pyobj_to_naive_date_time(secs, vm)?, - OptionalArg::Missing => chrono::offset::Local::now().naive_local(), - }) - } - - fn naive_or_utc(self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { - Ok(match self { - OptionalArg::Present(secs) => pyobj_to_naive_date_time(secs, vm)?, - OptionalArg::Missing => chrono::offset::Utc::now().naive_utc(), - }) - } - } - - impl OptionalArg<PyStructTime> { - fn naive_or_local(self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { - Ok(match self { - OptionalArg::Present(t) => t.to_date_time(vm)?, - OptionalArg::Missing => chrono::offset::Local::now().naive_local(), - }) - } - } - - /// https://docs.python.org/3/library/time.html?highlight=gmtime#time.gmtime - #[pyfunction] - fn gmtime(secs: OptionalArg<Either<f64, i64>>, vm: &VirtualMachine) -> PyResult<PyStructTime> { - let instant = secs.naive_or_utc(vm)?; - Ok(PyStructTime::new(vm, instant, 0)) - } - - #[pyfunction] - fn localtime( - secs: OptionalArg<Either<f64, i64>>, - vm: &VirtualMachine, - ) -> PyResult<PyStructTime> { - let instant = secs.naive_or_local(vm)?; - // TODO: isdst flag must be valid value here - // https://docs.python.org/3/library/time.html#time.localtime - Ok(PyStructTime::new(vm, instant, -1)) - } - - #[pyfunction] - fn mktime(t: PyStructTime, vm: &VirtualMachine) -> PyResult<f64> { - let datetime = t.to_date_time(vm)?; - let seconds_since_epoch = datetime.timestamp() as f64; - Ok(seconds_since_epoch) - } - - const CFMT: &str = "%a %b %e %H:%M:%S %Y"; - - #[pyfunction] - fn asctime(t: OptionalArg<PyStructTime>, vm: &VirtualMachine) -> PyResult { - let instant = t.naive_or_local(vm)?; - let formatted_time = instant.format(CFMT).to_string(); - Ok(vm.ctx.new_str(formatted_time).into()) - } - - #[pyfunction] - fn ctime(secs: OptionalArg<Either<f64, i64>>, vm: &VirtualMachine) -> PyResult<String> { - let instant = secs.naive_or_local(vm)?; - Ok(instant.format(CFMT).to_string()) - } - - #[pyfunction] - fn strftime(format: PyStrRef, t: OptionalArg<PyStructTime>, vm: &VirtualMachine) -> PyResult { - use std::fmt::Write; - - let instant = t.naive_or_local(vm)?; - let mut formatted_time = String::new(); - - /* - * chrono doesn't support all formats and it - * raises an error if unsupported format is supplied. - * If error happens, we set result as input arg. - */ - write!(&mut formatted_time, "{}", instant.format(format.as_str())) - .unwrap_or_else(|_| formatted_time = format.to_string()); - Ok(vm.ctx.new_str(formatted_time).into()) - } - - #[pyfunction] - fn strptime( - string: PyStrRef, - format: OptionalArg<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<PyStructTime> { - let format = format.as_ref().map_or("%a %b %H:%M:%S %Y", |s| s.as_str()); - let instant = NaiveDateTime::parse_from_str(string.as_str(), format) - .map_err(|e| vm.new_value_error(format!("Parse error: {e:?}")))?; - Ok(PyStructTime::new(vm, instant, -1)) - } - - #[cfg(not(any( - windows, - target_vendor = "apple", - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "fuchsia", - target_os = "emscripten", - )))] - fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { - Err(vm.new_not_implemented_error("thread time unsupported in this system".to_owned())) - } - - #[pyfunction] - fn thread_time(vm: &VirtualMachine) -> PyResult<f64> { - Ok(get_thread_time(vm)?.as_secs_f64()) - } - - #[pyfunction] - fn thread_time_ns(vm: &VirtualMachine) -> PyResult<u64> { - Ok(get_thread_time(vm)?.as_nanos() as u64) - } - - #[cfg(any(windows, all(target_arch = "wasm32", target_arch = "emscripten")))] - pub(super) fn time_muldiv(ticks: i64, mul: i64, div: i64) -> u64 { - let intpart = ticks / div; - let ticks = ticks % div; - let remaining = (ticks * mul) / div; - (intpart * mul + remaining) as u64 - } - - #[cfg(all(target_arch = "wasm32", target_os = "emscripten"))] - fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { - let t: libc::tms = unsafe { - let mut t = std::mem::MaybeUninit::uninit(); - if libc::times(t.as_mut_ptr()) == -1 { - return Err(vm.new_os_error("Failed to get clock time".to_owned())); - } - t.assume_init() - }; - let freq = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; - - Ok(Duration::from_nanos( - time_muldiv(t.tms_utime, SEC_TO_NS, freq) + time_muldiv(t.tms_stime, SEC_TO_NS, freq), - )) - } - - // same as the get_process_time impl for most unixes - #[cfg(all(target_arch = "wasm32", target_os = "wasi"))] - pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { - let time: libc::timespec = unsafe { - let mut time = std::mem::MaybeUninit::uninit(); - if libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, time.as_mut_ptr()) == -1 { - return Err(vm.new_os_error("Failed to get clock time".to_owned())); - } - time.assume_init() - }; - Ok(Duration::new(time.tv_sec as u64, time.tv_nsec as u32)) - } - - #[cfg(not(any( - windows, - target_os = "macos", - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "linux", - target_os = "illumos", - target_os = "netbsd", - target_os = "solaris", - target_os = "openbsd", - target_os = "redox", - all(target_arch = "wasm32", not(target_os = "unknown")) - )))] - fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { - Err(vm.new_not_implemented_error("process time unsupported in this system".to_owned())) - } - - #[pyfunction] - fn process_time(vm: &VirtualMachine) -> PyResult<f64> { - Ok(get_process_time(vm)?.as_secs_f64()) - } - - #[pyfunction] - fn process_time_ns(vm: &VirtualMachine) -> PyResult<u64> { - Ok(get_process_time(vm)?.as_nanos() as u64) - } - - #[pyattr] - #[pyclass(name = "struct_time")] - #[derive(PyStructSequence, TryIntoPyStructSequence)] - #[allow(dead_code)] - struct PyStructTime { - tm_year: PyObjectRef, - tm_mon: PyObjectRef, - tm_mday: PyObjectRef, - tm_hour: PyObjectRef, - tm_min: PyObjectRef, - tm_sec: PyObjectRef, - tm_wday: PyObjectRef, - tm_yday: PyObjectRef, - tm_isdst: PyObjectRef, - } - - impl std::fmt::Debug for PyStructTime { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "struct_time()") - } - } - - #[pyclass(with(PyStructSequence))] - impl PyStructTime { - fn new(vm: &VirtualMachine, tm: NaiveDateTime, isdst: i32) -> Self { - PyStructTime { - tm_year: vm.ctx.new_int(tm.year()).into(), - tm_mon: vm.ctx.new_int(tm.month()).into(), - tm_mday: vm.ctx.new_int(tm.day()).into(), - tm_hour: vm.ctx.new_int(tm.hour()).into(), - tm_min: vm.ctx.new_int(tm.minute()).into(), - tm_sec: vm.ctx.new_int(tm.second()).into(), - tm_wday: vm.ctx.new_int(tm.weekday().num_days_from_monday()).into(), - tm_yday: vm.ctx.new_int(tm.ordinal()).into(), - tm_isdst: vm.ctx.new_int(isdst).into(), - } - } - - fn to_date_time(&self, vm: &VirtualMachine) -> PyResult<NaiveDateTime> { - let invalid_overflow = - || vm.new_overflow_error("mktime argument out of range".to_owned()); - let invalid_value = || vm.new_value_error("invalid struct_time parameter".to_owned()); - - macro_rules! field { - ($field:ident) => { - self.$field.clone().try_into_value(vm)? - }; - } - let dt = NaiveDateTime::new( - NaiveDate::from_ymd_opt(field!(tm_year), field!(tm_mon), field!(tm_mday)) - .ok_or_else(invalid_value)?, - NaiveTime::from_hms_opt(field!(tm_hour), field!(tm_min), field!(tm_sec)) - .ok_or_else(invalid_overflow)?, - ); - Ok(dt) - } - - #[pyslot] - fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - // cls is ignorable because this is not a basetype - let seq = args.bind(vm)?; - Ok(vm.new_pyobj(Self::try_from_object(vm, seq)?)) - } - } - - #[allow(unused_imports)] - use super::platform::*; -} - -#[cfg(unix)] -#[pymodule(sub)] -mod platform { - #[allow(unused_imports)] - use super::{SEC_TO_NS, US_TO_NS}; - #[cfg_attr(target_os = "macos", allow(unused_imports))] - use crate::{ - builtins::{PyNamespace, PyStrRef}, - convert::IntoPyException, - PyObject, PyRef, PyResult, TryFromBorrowedObject, VirtualMachine, - }; - use nix::{sys::time::TimeSpec, time::ClockId}; - use std::time::Duration; - - #[cfg(target_os = "solaris")] - #[pyattr] - use libc::CLOCK_HIGHRES; - #[cfg(not(any( - target_os = "illumos", - target_os = "netbsd", - target_os = "solaris", - target_os = "openbsd", - )))] - #[pyattr] - use libc::CLOCK_PROCESS_CPUTIME_ID; - #[cfg(not(any( - target_os = "illumos", - target_os = "netbsd", - target_os = "solaris", - target_os = "openbsd", - target_os = "redox", - )))] - #[pyattr] - use libc::CLOCK_THREAD_CPUTIME_ID; - #[cfg(target_os = "linux")] - #[pyattr] - use libc::{CLOCK_BOOTTIME, CLOCK_MONOTONIC_RAW, CLOCK_TAI}; - #[pyattr] - use libc::{CLOCK_MONOTONIC, CLOCK_REALTIME}; - #[cfg(any(target_os = "freebsd", target_os = "openbsd", target_os = "dragonfly"))] - #[pyattr] - use libc::{CLOCK_PROF, CLOCK_UPTIME}; - - impl<'a> TryFromBorrowedObject<'a> for ClockId { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> { - obj.try_to_value(vm).map(ClockId::from_raw) - } - } - - fn get_clock_time(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<Duration> { - let ts = nix::time::clock_gettime(clk_id).map_err(|e| e.into_pyexception(vm))?; - Ok(ts.into()) - } - - #[pyfunction] - fn clock_gettime(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<f64> { - get_clock_time(clk_id, vm).map(|d| d.as_secs_f64()) - } - - #[pyfunction] - fn clock_gettime_ns(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<u128> { - get_clock_time(clk_id, vm).map(|d| d.as_nanos()) - } - - #[cfg(not(target_os = "redox"))] - #[pyfunction] - fn clock_getres(clk_id: ClockId, vm: &VirtualMachine) -> PyResult<f64> { - let ts = nix::time::clock_getres(clk_id).map_err(|e| e.into_pyexception(vm))?; - Ok(Duration::from(ts).as_secs_f64()) - } - - #[cfg(not(target_os = "redox"))] - #[cfg(not(target_vendor = "apple"))] - fn set_clock_time(clk_id: ClockId, timespec: TimeSpec, vm: &VirtualMachine) -> PyResult<()> { - nix::time::clock_settime(clk_id, timespec).map_err(|e| e.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[cfg(target_os = "macos")] - fn set_clock_time(clk_id: ClockId, timespec: TimeSpec, vm: &VirtualMachine) -> PyResult<()> { - // idk why nix disables clock_settime on macos - let ret = unsafe { libc::clock_settime(clk_id.as_raw(), timespec.as_ref()) }; - nix::Error::result(ret) - .map(drop) - .map_err(|e| e.into_pyexception(vm)) - } - - #[cfg(not(target_os = "redox"))] - #[cfg(any(not(target_vendor = "apple"), target_os = "macos"))] - #[pyfunction] - fn clock_settime(clk_id: ClockId, time: Duration, vm: &VirtualMachine) -> PyResult<()> { - set_clock_time(clk_id, time.into(), vm) - } - - #[cfg(not(target_os = "redox"))] - #[cfg(any(not(target_vendor = "apple"), target_os = "macos"))] - #[pyfunction] - fn clock_settime_ns(clk_id: ClockId, time: libc::time_t, vm: &VirtualMachine) -> PyResult<()> { - let ts = Duration::from_nanos(time as _).into(); - set_clock_time(clk_id, ts, vm) - } - - // Requires all CLOCK constants available and clock_getres - #[cfg(any( - target_os = "macos", - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "emscripten", - target_os = "linux", - ))] - #[pyfunction] - fn get_clock_info(name: PyStrRef, vm: &VirtualMachine) -> PyResult<PyRef<PyNamespace>> { - let (adj, imp, mono, res) = match name.as_ref() { - "monotonic" | "perf_counter" => ( - false, - "time.clock_gettime(CLOCK_MONOTONIC)", - true, - clock_getres(ClockId::CLOCK_MONOTONIC, vm)?, - ), - "process_time" => ( - false, - "time.clock_gettime(CLOCK_PROCESS_CPUTIME_ID)", - true, - clock_getres(ClockId::CLOCK_PROCESS_CPUTIME_ID, vm)?, - ), - "thread_time" => ( - false, - "time.clock_gettime(CLOCK_THREAD_CPUTIME_ID)", - true, - clock_getres(ClockId::CLOCK_THREAD_CPUTIME_ID, vm)?, - ), - "time" => ( - true, - "time.clock_gettime(CLOCK_REALTIME)", - false, - clock_getres(ClockId::CLOCK_REALTIME, vm)?, - ), - _ => return Err(vm.new_value_error("unknown clock".to_owned())), - }; - - Ok(py_namespace!(vm, { - "implementation" => vm.new_pyobj(imp), - "monotonic" => vm.ctx.new_bool(mono), - "adjustable" => vm.ctx.new_bool(adj), - "resolution" => vm.ctx.new_float(res), - })) - } - - #[cfg(not(any( - target_os = "macos", - target_os = "android", - target_os = "dragonfly", - target_os = "freebsd", - target_os = "fuchsia", - target_os = "emscripten", - target_os = "linux", - )))] - #[pyfunction] - fn get_clock_info(_name: PyStrRef, vm: &VirtualMachine) -> PyResult<PyRef<PyNamespace>> { - Err(vm.new_not_implemented_error("get_clock_info unsupported on this system".to_owned())) - } - - pub(super) fn get_monotonic_time(vm: &VirtualMachine) -> PyResult<Duration> { - get_clock_time(ClockId::CLOCK_MONOTONIC, vm) - } - - pub(super) fn get_perf_time(vm: &VirtualMachine) -> PyResult<Duration> { - get_clock_time(ClockId::CLOCK_MONOTONIC, vm) - } - - #[pyfunction] - fn sleep(dur: Duration, vm: &VirtualMachine) -> PyResult<()> { - // this is basically std::thread::sleep, but that catches interrupts and we don't want to; - - let ts = TimeSpec::from(dur); - let res = unsafe { libc::nanosleep(ts.as_ref(), std::ptr::null_mut()) }; - let interrupted = res == -1 && nix::errno::errno() == libc::EINTR; - - if interrupted { - vm.check_signals()?; - } - - Ok(()) - } - - #[cfg(not(any( - target_os = "illumos", - target_os = "netbsd", - target_os = "openbsd", - target_os = "redox" - )))] - pub(super) fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { - get_clock_time(ClockId::CLOCK_THREAD_CPUTIME_ID, vm) - } - - #[cfg(target_os = "solaris")] - pub(super) fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { - Ok(Duration::from_nanos(unsafe { libc::gethrvtime() })) - } - - #[cfg(not(any( - target_os = "illumos", - target_os = "netbsd", - target_os = "solaris", - target_os = "openbsd", - )))] - pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { - get_clock_time(ClockId::CLOCK_PROCESS_CPUTIME_ID, vm) - } - - #[cfg(any( - target_os = "illumos", - target_os = "netbsd", - target_os = "solaris", - target_os = "openbsd", - ))] - pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { - use nix::sys::resource::{getrusage, UsageWho}; - fn from_timeval(tv: libc::timeval, vm: &VirtualMachine) -> PyResult<i64> { - (|tv: libc::timeval| { - let t = tv.tv_sec.checked_mul(SEC_TO_NS)?; - let u = (tv.tv_usec as i64).checked_mul(US_TO_NS)?; - t.checked_add(u) - })(tv) - .ok_or_else(|| { - vm.new_overflow_error("timestamp too large to convert to i64".to_owned()) - }) - } - let ru = getrusage(UsageWho::RUSAGE_SELF).map_err(|e| e.into_pyexception(vm))?; - let utime = from_timeval(ru.user_time().into(), vm)?; - let stime = from_timeval(ru.system_time().into(), vm)?; - - Ok(Duration::from_nanos((utime + stime) as u64)) - } -} - -#[cfg(windows)] -#[pymodule] -mod platform { - use super::{time_muldiv, MS_TO_NS, SEC_TO_NS}; - use crate::{ - builtins::{PyNamespace, PyStrRef}, - stdlib::os::errno_err, - PyRef, PyResult, VirtualMachine, - }; - use std::time::Duration; - use windows_sys::Win32::{ - Foundation::FILETIME, - System::Performance::{QueryPerformanceCounter, QueryPerformanceFrequency}, - System::SystemInformation::{GetSystemTimeAdjustment, GetTickCount64}, - System::Threading::{GetCurrentProcess, GetCurrentThread, GetProcessTimes, GetThreadTimes}, - }; - - fn u64_from_filetime(time: FILETIME) -> u64 { - let large: [u32; 2] = [time.dwLowDateTime, time.dwHighDateTime]; - unsafe { std::mem::transmute(large) } - } - - fn win_perf_counter_frequency(vm: &VirtualMachine) -> PyResult<i64> { - let frequency = unsafe { - let mut freq = std::mem::MaybeUninit::uninit(); - if QueryPerformanceFrequency(freq.as_mut_ptr()) == 0 { - return Err(errno_err(vm)); - } - freq.assume_init() - }; - - if frequency < 1 { - Err(vm.new_runtime_error("invalid QueryPerformanceFrequency".to_owned())) - } else if frequency > i64::MAX / SEC_TO_NS { - Err(vm.new_overflow_error("QueryPerformanceFrequency is too large".to_owned())) - } else { - Ok(frequency) - } - } - - fn global_frequency(vm: &VirtualMachine) -> PyResult<i64> { - rustpython_common::static_cell! { - static FREQUENCY: PyResult<i64>; - }; - FREQUENCY - .get_or_init(|| win_perf_counter_frequency(vm)) - .clone() - } - - pub(super) fn get_perf_time(vm: &VirtualMachine) -> PyResult<Duration> { - let ticks = unsafe { - let mut performance_count = std::mem::MaybeUninit::uninit(); - QueryPerformanceCounter(performance_count.as_mut_ptr()); - performance_count.assume_init() - }; - - Ok(Duration::from_nanos(time_muldiv( - ticks, - SEC_TO_NS, - global_frequency(vm)?, - ))) - } - - fn get_system_time_adjustment(vm: &VirtualMachine) -> PyResult<u32> { - let mut _time_adjustment = std::mem::MaybeUninit::uninit(); - let mut time_increment = std::mem::MaybeUninit::uninit(); - let mut _is_time_adjustment_disabled = std::mem::MaybeUninit::uninit(); - let time_increment = unsafe { - if GetSystemTimeAdjustment( - _time_adjustment.as_mut_ptr(), - time_increment.as_mut_ptr(), - _is_time_adjustment_disabled.as_mut_ptr(), - ) == 0 - { - return Err(errno_err(vm)); - } - time_increment.assume_init() - }; - Ok(time_increment) - } - - pub(super) fn get_monotonic_time(vm: &VirtualMachine) -> PyResult<Duration> { - let ticks = unsafe { GetTickCount64() }; - - Ok(Duration::from_nanos( - (ticks as i64).checked_mul(MS_TO_NS).ok_or_else(|| { - vm.new_overflow_error("timestamp too large to convert to i64".to_owned()) - })? as u64, - )) - } - - #[pyfunction] - fn get_clock_info(name: PyStrRef, vm: &VirtualMachine) -> PyResult<PyRef<PyNamespace>> { - let (adj, imp, mono, res) = match name.as_ref() { - "monotonic" => ( - false, - "GetTickCount64()", - true, - get_system_time_adjustment(vm)? as f64 * 1e-7, - ), - "perf_counter" => ( - false, - "QueryPerformanceCounter()", - true, - 1.0 / (global_frequency(vm)? as f64), - ), - "process_time" => (false, "GetProcessTimes()", true, 1e-7), - "thread_time" => (false, "GetThreadTimes()", true, 1e-7), - "time" => ( - true, - "GetSystemTimeAsFileTime()", - false, - get_system_time_adjustment(vm)? as f64 * 1e-7, - ), - _ => return Err(vm.new_value_error("unknown clock".to_owned())), - }; - - Ok(py_namespace!(vm, { - "implementation" => vm.new_pyobj(imp), - "monotonic" => vm.ctx.new_bool(mono), - "adjustable" => vm.ctx.new_bool(adj), - "resolution" => vm.ctx.new_float(res), - })) - } - - pub(super) fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> { - let (kernel_time, user_time) = unsafe { - let mut _creation_time = std::mem::MaybeUninit::uninit(); - let mut _exit_time = std::mem::MaybeUninit::uninit(); - let mut kernel_time = std::mem::MaybeUninit::uninit(); - let mut user_time = std::mem::MaybeUninit::uninit(); - - let thread = GetCurrentThread(); - if GetThreadTimes( - thread, - _creation_time.as_mut_ptr(), - _exit_time.as_mut_ptr(), - kernel_time.as_mut_ptr(), - user_time.as_mut_ptr(), - ) == 0 - { - return Err(vm.new_os_error("Failed to get clock time".to_owned())); - } - (kernel_time.assume_init(), user_time.assume_init()) - }; - let k_time = u64_from_filetime(kernel_time); - let u_time = u64_from_filetime(user_time); - Ok(Duration::from_nanos((k_time + u_time) * 100)) - } - - pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> { - let (kernel_time, user_time) = unsafe { - let mut _creation_time = std::mem::MaybeUninit::uninit(); - let mut _exit_time = std::mem::MaybeUninit::uninit(); - let mut kernel_time = std::mem::MaybeUninit::uninit(); - let mut user_time = std::mem::MaybeUninit::uninit(); - - let process = GetCurrentProcess(); - if GetProcessTimes( - process, - _creation_time.as_mut_ptr(), - _exit_time.as_mut_ptr(), - kernel_time.as_mut_ptr(), - user_time.as_mut_ptr(), - ) == 0 - { - return Err(vm.new_os_error("Failed to get clock time".to_owned())); - } - (kernel_time.assume_init(), user_time.assume_init()) - }; - let k_time = u64_from_filetime(kernel_time); - let u_time = u64_from_filetime(user_time); - Ok(Duration::from_nanos((k_time + u_time) * 100)) - } -} - -// mostly for wasm32 -#[cfg(not(any(unix, windows)))] -#[pymodule(sub)] -mod platform {} diff --git a/vm/src/stdlib/warnings.rs b/vm/src/stdlib/warnings.rs deleted file mode 100644 index ef72490f146..00000000000 --- a/vm/src/stdlib/warnings.rs +++ /dev/null @@ -1,49 +0,0 @@ -pub(crate) use _warnings::make_module; - -use crate::{builtins::PyType, Py, PyResult, VirtualMachine}; - -pub fn warn( - category: &Py<PyType>, - message: String, - stack_level: usize, - vm: &VirtualMachine, -) -> PyResult<()> { - // TODO: use rust warnings module - if let Ok(module) = vm.import("warnings", None, 0) { - if let Ok(func) = module.get_attr("warn", vm) { - let _ = func.call((message, category.to_owned(), stack_level), vm); - } - } - Ok(()) -} - -#[pymodule] -mod _warnings { - use crate::{ - builtins::{PyStrRef, PyTypeRef}, - function::OptionalArg, - PyResult, VirtualMachine, - }; - - #[derive(FromArgs)] - struct WarnArgs { - #[pyarg(positional)] - message: PyStrRef, - #[pyarg(any, optional)] - category: OptionalArg<PyTypeRef>, - #[pyarg(any, optional)] - stacklevel: OptionalArg<u32>, - } - - #[pyfunction] - fn warn(args: WarnArgs, vm: &VirtualMachine) -> PyResult<()> { - let level = args.stacklevel.unwrap_or(1); - crate::warn::warn( - args.message, - args.category.into_option(), - level as isize, - None, - vm, - ) - } -} diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs deleted file mode 100644 index 3ef0de61553..00000000000 --- a/vm/src/stdlib/weakref.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Implementation in line with the python `weakref` module. -//! -//! See also: -//! - [python weakref module](https://docs.python.org/3/library/weakref.html) -//! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) -//! -pub(crate) use _weakref::make_module; - -#[pymodule] -mod _weakref { - use crate::{ - builtins::{PyDictRef, PyTypeRef, PyWeak}, - PyObjectRef, PyResult, VirtualMachine, - }; - - #[pyattr(name = "ref")] - fn ref_(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.types.weakref_type.to_owned() - } - #[pyattr] - fn proxy(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.types.weakproxy_type.to_owned() - } - #[pyattr(name = "ReferenceType")] - fn reference_type(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.types.weakref_type.to_owned() - } - #[pyattr(name = "ProxyType")] - fn proxy_type(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.types.weakproxy_type.to_owned() - } - #[pyattr(name = "CallableProxyType")] - fn callable_proxy_type(vm: &VirtualMachine) -> PyTypeRef { - vm.ctx.types.weakproxy_type.to_owned() - } - - #[pyfunction] - fn getweakrefcount(obj: PyObjectRef) -> usize { - obj.weak_count().unwrap_or(0) - } - - #[pyfunction] - fn getweakrefs(obj: PyObjectRef) -> Vec<PyObjectRef> { - match obj.get_weak_references() { - Some(v) => v.into_iter().map(Into::into).collect(), - None => vec![], - } - } - - #[pyfunction] - fn _remove_dead_weakref( - dict: PyDictRef, - key: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - dict._as_dict_inner() - .delete_if(vm, &*key, |wr| { - let wr = wr - .payload::<PyWeak>() - .ok_or_else(|| vm.new_type_error("not a weakref".to_owned()))?; - Ok(wr.is_dead()) - }) - .map(drop) - } -} diff --git a/vm/src/stdlib/winapi.rs b/vm/src/stdlib/winapi.rs deleted file mode 100644 index ed88fabceb5..00000000000 --- a/vm/src/stdlib/winapi.rs +++ /dev/null @@ -1,424 +0,0 @@ -#![allow(non_snake_case)] -pub(crate) use _winapi::make_module; - -#[pymodule] -mod _winapi { - use crate::{ - builtins::PyStrRef, - common::windows::ToWideString, - convert::{ToPyException, ToPyResult}, - function::{ArgMapping, ArgSequence, OptionalArg}, - stdlib::os::errno_err, - windows::WindowsSysResult, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, - }; - use std::ptr::{null, null_mut}; - use windows::{ - core::PCWSTR, - Win32::Foundation::{HANDLE, HINSTANCE, MAX_PATH}, - }; - use windows_sys::Win32::Foundation::{BOOL, HANDLE as RAW_HANDLE}; - - #[pyattr] - use windows_sys::Win32::{ - Foundation::{ - DUPLICATE_CLOSE_SOURCE, DUPLICATE_SAME_ACCESS, ERROR_ALREADY_EXISTS, ERROR_BROKEN_PIPE, - ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_NETNAME_DELETED, ERROR_NO_DATA, - ERROR_NO_SYSTEM_RESOURCES, ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, - ERROR_PIPE_CONNECTED, ERROR_SEM_TIMEOUT, GENERIC_READ, GENERIC_WRITE, STILL_ACTIVE, - WAIT_ABANDONED, WAIT_ABANDONED_0, WAIT_OBJECT_0, WAIT_TIMEOUT, - }, - Storage::FileSystem::{ - FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OVERLAPPED, FILE_GENERIC_READ, - FILE_GENERIC_WRITE, FILE_TYPE_CHAR, FILE_TYPE_DISK, FILE_TYPE_PIPE, FILE_TYPE_REMOTE, - FILE_TYPE_UNKNOWN, OPEN_EXISTING, PIPE_ACCESS_DUPLEX, PIPE_ACCESS_INBOUND, SYNCHRONIZE, - }, - System::{ - Console::{STD_ERROR_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE}, - Memory::{ - FILE_MAP_ALL_ACCESS, MEM_COMMIT, MEM_FREE, MEM_IMAGE, MEM_MAPPED, MEM_PRIVATE, - MEM_RESERVE, PAGE_EXECUTE, PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE, - PAGE_EXECUTE_WRITECOPY, PAGE_GUARD, PAGE_NOACCESS, PAGE_NOCACHE, PAGE_READONLY, - PAGE_READWRITE, PAGE_WRITECOMBINE, PAGE_WRITECOPY, SEC_COMMIT, SEC_IMAGE, - SEC_LARGE_PAGES, SEC_NOCACHE, SEC_RESERVE, SEC_WRITECOMBINE, - }, - Pipes::{ - PIPE_READMODE_MESSAGE, PIPE_TYPE_MESSAGE, PIPE_UNLIMITED_INSTANCES, PIPE_WAIT, - }, - SystemServices::LOCALE_NAME_MAX_LENGTH, - Threading::{ - ABOVE_NORMAL_PRIORITY_CLASS, BELOW_NORMAL_PRIORITY_CLASS, - CREATE_BREAKAWAY_FROM_JOB, CREATE_DEFAULT_ERROR_MODE, CREATE_NEW_CONSOLE, - CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, DETACHED_PROCESS, HIGH_PRIORITY_CLASS, - IDLE_PRIORITY_CLASS, INFINITE, NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, - REALTIME_PRIORITY_CLASS, STARTF_USESHOWWINDOW, STARTF_USESTDHANDLES, - }, - }, - UI::WindowsAndMessaging::SW_HIDE, - }; - - #[pyfunction] - fn CloseHandle(handle: HANDLE) -> WindowsSysResult<BOOL> { - WindowsSysResult(unsafe { windows_sys::Win32::Foundation::CloseHandle(handle.0) }) - } - - #[pyfunction] - fn GetStdHandle( - std_handle: windows_sys::Win32::System::Console::STD_HANDLE, - ) -> WindowsSysResult<RAW_HANDLE> { - WindowsSysResult(unsafe { windows_sys::Win32::System::Console::GetStdHandle(std_handle) }) - } - - #[pyfunction] - fn CreatePipe( - _pipe_attrs: PyObjectRef, - size: u32, - vm: &VirtualMachine, - ) -> PyResult<(HANDLE, HANDLE)> { - let (read, write) = unsafe { - let mut read = std::mem::MaybeUninit::<isize>::uninit(); - let mut write = std::mem::MaybeUninit::<isize>::uninit(); - WindowsSysResult(windows_sys::Win32::System::Pipes::CreatePipe( - read.as_mut_ptr(), - write.as_mut_ptr(), - std::ptr::null(), - size, - )) - .to_pyresult(vm)?; - (read.assume_init(), write.assume_init()) - }; - Ok((HANDLE(read), HANDLE(write))) - } - - #[pyfunction] - fn DuplicateHandle( - (src_process, src): (HANDLE, HANDLE), - target_process: HANDLE, - access: u32, - inherit: BOOL, - options: OptionalArg<u32>, - vm: &VirtualMachine, - ) -> PyResult<HANDLE> { - let target = unsafe { - let mut target = std::mem::MaybeUninit::<isize>::uninit(); - WindowsSysResult(windows_sys::Win32::Foundation::DuplicateHandle( - src_process.0, - src.0, - target_process.0, - target.as_mut_ptr(), - access, - inherit, - options.unwrap_or(0), - )) - .to_pyresult(vm)?; - target.assume_init() - }; - Ok(HANDLE(target)) - } - - #[pyfunction] - fn GetCurrentProcess() -> HANDLE { - unsafe { windows::Win32::System::Threading::GetCurrentProcess() } - } - - #[pyfunction] - fn GetFileType( - h: HANDLE, - vm: &VirtualMachine, - ) -> PyResult<windows_sys::Win32::Storage::FileSystem::FILE_TYPE> { - let file_type = unsafe { windows_sys::Win32::Storage::FileSystem::GetFileType(h.0) }; - if file_type == 0 && unsafe { windows_sys::Win32::Foundation::GetLastError() } != 0 { - Err(errno_err(vm)) - } else { - Ok(file_type) - } - } - - #[derive(FromArgs)] - struct CreateProcessArgs { - #[pyarg(positional)] - name: Option<PyStrRef>, - #[pyarg(positional)] - command_line: Option<PyStrRef>, - #[pyarg(positional)] - _proc_attrs: PyObjectRef, - #[pyarg(positional)] - _thread_attrs: PyObjectRef, - #[pyarg(positional)] - inherit_handles: i32, - #[pyarg(positional)] - creation_flags: u32, - #[pyarg(positional)] - env_mapping: Option<ArgMapping>, - #[pyarg(positional)] - current_dir: Option<PyStrRef>, - #[pyarg(positional)] - startup_info: PyObjectRef, - } - - #[pyfunction] - fn CreateProcess( - args: CreateProcessArgs, - vm: &VirtualMachine, - ) -> PyResult<(HANDLE, HANDLE, u32, u32)> { - let mut si: windows_sys::Win32::System::Threading::STARTUPINFOEXW = - unsafe { std::mem::zeroed() }; - si.StartupInfo.cb = std::mem::size_of_val(&si) as _; - - macro_rules! si_attr { - ($attr:ident, $t:ty) => {{ - si.StartupInfo.$attr = <Option<$t>>::try_from_object( - vm, - args.startup_info.get_attr(stringify!($attr), vm)?, - )? - .unwrap_or(0) as _ - }}; - ($attr:ident) => {{ - si.StartupInfo.$attr = <Option<_>>::try_from_object( - vm, - args.startup_info.get_attr(stringify!($attr), vm)?, - )? - .unwrap_or(0) - }}; - } - si_attr!(dwFlags); - si_attr!(wShowWindow); - si_attr!(hStdInput, usize); - si_attr!(hStdOutput, usize); - si_attr!(hStdError, usize); - - let mut env = args - .env_mapping - .map(|m| getenvironment(m, vm)) - .transpose()?; - let env = env.as_mut().map_or_else(null_mut, |v| v.as_mut_ptr()); - - let mut attrlist = - getattributelist(args.startup_info.get_attr("lpAttributeList", vm)?, vm)?; - si.lpAttributeList = attrlist - .as_mut() - .map_or_else(null_mut, |l| l.attrlist.as_mut_ptr() as _); - - let wstr = |s: PyStrRef| { - let ws = widestring::WideCString::from_str(s.as_str()) - .map_err(|err| err.to_pyexception(vm))?; - Ok(ws.into_vec_with_nul()) - }; - - let app_name = args.name.map(wstr).transpose()?; - let app_name = app_name.as_ref().map_or_else(null, |w| w.as_ptr()); - - let mut command_line = args.command_line.map(wstr).transpose()?; - let command_line = command_line - .as_mut() - .map_or_else(null_mut, |w| w.as_mut_ptr()); - - let mut current_dir = args.current_dir.map(wstr).transpose()?; - let current_dir = current_dir - .as_mut() - .map_or_else(null_mut, |w| w.as_mut_ptr()); - - let procinfo = unsafe { - let mut procinfo = std::mem::MaybeUninit::uninit(); - WindowsSysResult(windows_sys::Win32::System::Threading::CreateProcessW( - app_name, - command_line, - std::ptr::null(), - std::ptr::null(), - args.inherit_handles, - args.creation_flags - | windows_sys::Win32::System::Threading::EXTENDED_STARTUPINFO_PRESENT - | windows_sys::Win32::System::Threading::CREATE_UNICODE_ENVIRONMENT, - env as _, - current_dir, - &mut si as *mut _ as *mut _, - procinfo.as_mut_ptr(), - )) - .into_pyresult(vm)?; - procinfo.assume_init() - }; - - Ok(( - HANDLE(procinfo.hProcess), - HANDLE(procinfo.hThread), - procinfo.dwProcessId, - procinfo.dwThreadId, - )) - } - - fn getenvironment(env: ArgMapping, vm: &VirtualMachine) -> PyResult<Vec<u16>> { - let keys = env.mapping().keys(vm)?; - let values = env.mapping().values(vm)?; - - let keys = ArgSequence::try_from_object(vm, keys)?.into_vec(); - let values = ArgSequence::try_from_object(vm, values)?.into_vec(); - - if keys.len() != values.len() { - return Err( - vm.new_runtime_error("environment changed size during iteration".to_owned()) - ); - } - - let mut out = widestring::WideString::new(); - for (k, v) in keys.into_iter().zip(values.into_iter()) { - let k = PyStrRef::try_from_object(vm, k)?; - let k = k.as_str(); - let v = PyStrRef::try_from_object(vm, v)?; - let v = v.as_str(); - if k.contains('\0') || v.contains('\0') { - return Err(crate::exceptions::cstring_error(vm)); - } - if k.is_empty() || k[1..].contains('=') { - return Err(vm.new_value_error("illegal environment variable name".to_owned())); - } - out.push_str(k); - out.push_str("="); - out.push_str(v); - out.push_str("\0"); - } - out.push_str("\0"); - Ok(out.into_vec()) - } - - struct AttrList { - handlelist: Option<Vec<usize>>, - attrlist: Vec<u8>, - } - impl Drop for AttrList { - fn drop(&mut self) { - unsafe { - windows_sys::Win32::System::Threading::DeleteProcThreadAttributeList( - self.attrlist.as_mut_ptr() as *mut _, - ) - }; - } - } - - fn getattributelist(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Option<AttrList>> { - <Option<ArgMapping>>::try_from_object(vm, obj)? - .map(|mapping| { - let handlelist = mapping - .as_ref() - .get_item("handle_list", vm) - .ok() - .and_then(|obj| { - <Option<ArgSequence<usize>>>::try_from_object(vm, obj) - .map(|s| match s { - Some(s) if !s.is_empty() => Some(s.into_vec()), - _ => None, - }) - .transpose() - }) - .transpose()?; - - let attr_count = handlelist.is_some() as u32; - let (result, mut size) = unsafe { - let mut size = std::mem::MaybeUninit::uninit(); - let result = WindowsSysResult( - windows_sys::Win32::System::Threading::InitializeProcThreadAttributeList( - std::ptr::null_mut(), - attr_count, - 0, - size.as_mut_ptr(), - ), - ); - (result, size.assume_init()) - }; - if !result.is_err() - || unsafe { windows_sys::Win32::Foundation::GetLastError() } - != windows_sys::Win32::Foundation::ERROR_INSUFFICIENT_BUFFER - { - return Err(errno_err(vm)); - } - let mut attrlist = vec![0u8; size]; - WindowsSysResult(unsafe { - windows_sys::Win32::System::Threading::InitializeProcThreadAttributeList( - attrlist.as_mut_ptr() as *mut _, - attr_count, - 0, - &mut size, - ) - }) - .into_pyresult(vm)?; - let mut attrs = AttrList { - handlelist, - attrlist, - }; - if let Some(ref mut handlelist) = attrs.handlelist { - WindowsSysResult(unsafe { - windows_sys::Win32::System::Threading::UpdateProcThreadAttribute( - attrs.attrlist.as_mut_ptr() as _, - 0, - (2 & 0xffff) | 0x20000, // PROC_THREAD_ATTRIBUTE_HANDLE_LIST - handlelist.as_mut_ptr() as _, - (handlelist.len() * std::mem::size_of::<HANDLE>()) as _, - std::ptr::null_mut(), - std::ptr::null(), - ) - }) - .into_pyresult(vm)?; - } - Ok(attrs) - }) - .transpose() - } - - #[pyfunction] - fn WaitForSingleObject(h: HANDLE, ms: u32, vm: &VirtualMachine) -> PyResult<u32> { - let ret = unsafe { windows_sys::Win32::System::Threading::WaitForSingleObject(h.0, ms) }; - if ret == windows_sys::Win32::Foundation::WAIT_FAILED { - Err(errno_err(vm)) - } else { - Ok(ret) - } - } - - #[pyfunction] - fn GetExitCodeProcess(h: HANDLE, vm: &VirtualMachine) -> PyResult<u32> { - unsafe { - let mut ec = std::mem::MaybeUninit::uninit(); - WindowsSysResult(windows_sys::Win32::System::Threading::GetExitCodeProcess( - h.0, - ec.as_mut_ptr(), - )) - .to_pyresult(vm)?; - Ok(ec.assume_init()) - } - } - - #[pyfunction] - fn TerminateProcess(h: HANDLE, exit_code: u32) -> WindowsSysResult<BOOL> { - WindowsSysResult(unsafe { - windows_sys::Win32::System::Threading::TerminateProcess(h.0, exit_code) - }) - } - - // TODO: ctypes.LibraryLoader.LoadLibrary - #[allow(dead_code)] - fn LoadLibrary(path: PyStrRef, vm: &VirtualMachine) -> PyResult<isize> { - let path = path.as_str().to_wides_with_nul(); - let handle = unsafe { - windows::Win32::System::LibraryLoader::LoadLibraryW(PCWSTR::from_raw(path.as_ptr())) - .unwrap() - }; - if handle.is_invalid() { - return Err(vm.new_runtime_error("LoadLibrary failed".to_owned())); - } - Ok(handle.0) - } - - #[pyfunction] - fn GetModuleFileName(handle: isize, vm: &VirtualMachine) -> PyResult<String> { - let mut path: Vec<u16> = vec![0; MAX_PATH as usize]; - let handle = HINSTANCE(handle); - - let length = - unsafe { windows::Win32::System::LibraryLoader::GetModuleFileNameW(handle, &mut path) }; - if length == 0 { - return Err(vm.new_runtime_error("GetModuleFileName failed".to_owned())); - } - - let (path, _) = path.split_at(length as usize); - Ok(String::from_utf16(path).unwrap()) - } -} diff --git a/vm/src/stdlib/winreg.rs b/vm/src/stdlib/winreg.rs deleted file mode 100644 index b992de9cb98..00000000000 --- a/vm/src/stdlib/winreg.rs +++ /dev/null @@ -1,340 +0,0 @@ -#![allow(non_snake_case)] - -use crate::{builtins::PyModule, PyRef, VirtualMachine}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = winreg::make_module(vm); - - macro_rules! add_constants { - ($($name:ident),*$(,)?) => { - extend_module!(vm, &module, { - $((stringify!($name)) => vm.new_pyobj(::winreg::enums::$name as usize)),* - }) - }; - } - - add_constants!( - HKEY_CLASSES_ROOT, - HKEY_CURRENT_USER, - HKEY_LOCAL_MACHINE, - HKEY_USERS, - HKEY_PERFORMANCE_DATA, - HKEY_CURRENT_CONFIG, - HKEY_DYN_DATA, - ); - module -} - -#[pymodule] -mod winreg { - use crate::common::lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; - use crate::{ - builtins::PyStrRef, convert::ToPyException, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromObject, VirtualMachine, - }; - use ::winreg::{enums::RegType, RegKey, RegValue}; - use std::{ffi::OsStr, io}; - use windows_sys::Win32::Foundation; - - // access rights - #[pyattr] - pub use windows_sys::Win32::System::Registry::{ - KEY_ALL_ACCESS, KEY_CREATE_LINK, KEY_CREATE_SUB_KEY, KEY_ENUMERATE_SUB_KEYS, KEY_EXECUTE, - KEY_NOTIFY, KEY_QUERY_VALUE, KEY_READ, KEY_SET_VALUE, KEY_WOW64_32KEY, KEY_WOW64_64KEY, - KEY_WRITE, - }; - // value types - #[pyattr] - pub use windows_sys::Win32::System::Registry::{ - REG_BINARY, REG_DWORD, REG_DWORD_BIG_ENDIAN, REG_DWORD_LITTLE_ENDIAN, REG_EXPAND_SZ, - REG_FULL_RESOURCE_DESCRIPTOR, REG_LINK, REG_MULTI_SZ, REG_NONE, REG_QWORD, - REG_QWORD_LITTLE_ENDIAN, REG_RESOURCE_LIST, REG_RESOURCE_REQUIREMENTS_LIST, REG_SZ, - }; - - #[pyattr] - #[pyclass(module = "winreg", name = "HKEYType")] - #[derive(Debug, PyPayload)] - struct PyHkey { - key: PyRwLock<RegKey>, - } - type PyHkeyRef = PyRef<PyHkey>; - - // TODO: fix this - unsafe impl Sync for PyHkey {} - - impl PyHkey { - fn new(key: RegKey) -> Self { - Self { - key: PyRwLock::new(key), - } - } - - fn key(&self) -> PyRwLockReadGuard<'_, RegKey> { - self.key.read() - } - - fn key_mut(&self) -> PyRwLockWriteGuard<'_, RegKey> { - self.key.write() - } - } - - #[pyclass] - impl PyHkey { - #[pymethod] - fn Close(&self) { - let null_key = RegKey::predef(0 as ::winreg::HKEY); - let key = std::mem::replace(&mut *self.key_mut(), null_key); - drop(key); - } - #[pymethod] - fn Detach(&self) -> usize { - let null_key = RegKey::predef(0 as ::winreg::HKEY); - let key = std::mem::replace(&mut *self.key_mut(), null_key); - let handle = key.raw_handle(); - std::mem::forget(key); - handle as usize - } - - #[pymethod(magic)] - fn bool(&self) -> bool { - !self.key().raw_handle().is_null() - } - #[pymethod(magic)] - fn enter(zelf: PyRef<Self>) -> PyRef<Self> { - zelf - } - #[pymethod(magic)] - fn exit(&self, _cls: PyObjectRef, _exc: PyObjectRef, _tb: PyObjectRef) { - self.Close(); - } - } - - enum Hkey { - PyHkey(PyHkeyRef), - Constant(::winreg::HKEY), - } - impl TryFromObject for Hkey { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - obj.downcast().map(Self::PyHkey).or_else(|o| { - usize::try_from_object(vm, o).map(|i| Self::Constant(i as ::winreg::HKEY)) - }) - } - } - impl Hkey { - fn with_key<R>(&self, f: impl FnOnce(&RegKey) -> R) -> R { - match self { - Self::PyHkey(py) => f(&py.key()), - Self::Constant(hkey) => { - let k = RegKey::predef(*hkey); - let res = f(&k); - std::mem::forget(k); - res - } - } - } - fn into_key(self) -> RegKey { - let k = match self { - Self::PyHkey(py) => py.key().raw_handle(), - Self::Constant(k) => k, - }; - RegKey::predef(k) - } - } - - #[derive(FromArgs)] - struct OpenKeyArgs { - key: Hkey, - sub_key: Option<PyStrRef>, - #[pyarg(any, default = "0")] - reserved: i32, - #[pyarg(any, default = "::winreg::enums::KEY_READ")] - access: u32, - } - - #[pyfunction(name = "OpenKeyEx")] - #[pyfunction] - fn OpenKey(args: OpenKeyArgs, vm: &VirtualMachine) -> PyResult<PyHkey> { - let OpenKeyArgs { - key, - sub_key, - reserved, - access, - } = args; - - if reserved != 0 { - // RegKey::open_subkey* doesn't have a reserved param, so this'll do - return Err(vm.new_value_error("reserved param must be 0".to_owned())); - } - - let sub_key = sub_key.as_ref().map_or("", |s| s.as_str()); - let key = key - .with_key(|k| k.open_subkey_with_flags(sub_key, access)) - .map_err(|e| e.to_pyexception(vm))?; - - Ok(PyHkey::new(key)) - } - - #[pyfunction] - fn QueryValue(key: Hkey, subkey: Option<PyStrRef>, vm: &VirtualMachine) -> PyResult<String> { - let subkey = subkey.as_ref().map_or("", |s| s.as_str()); - key.with_key(|k| k.get_value(subkey)) - .map_err(|e| e.to_pyexception(vm)) - } - - #[pyfunction] - fn QueryValueEx( - key: Hkey, - subkey: Option<PyStrRef>, - vm: &VirtualMachine, - ) -> PyResult<(PyObjectRef, usize)> { - let subkey = subkey.as_ref().map_or("", |s| s.as_str()); - let regval = key - .with_key(|k| k.get_raw_value(subkey)) - .map_err(|e| e.to_pyexception(vm))?; - #[allow(clippy::redundant_clone)] - let ty = regval.vtype.clone() as usize; - Ok((reg_to_py(regval, vm)?, ty)) - } - - #[pyfunction] - fn EnumKey(key: Hkey, index: u32, vm: &VirtualMachine) -> PyResult<String> { - key.with_key(|k| k.enum_keys().nth(index as usize)) - .unwrap_or_else(|| { - Err(io::Error::from_raw_os_error( - Foundation::ERROR_NO_MORE_ITEMS as i32, - )) - }) - .map_err(|e| e.to_pyexception(vm)) - } - - #[pyfunction] - fn EnumValue( - key: Hkey, - index: u32, - vm: &VirtualMachine, - ) -> PyResult<(String, PyObjectRef, usize)> { - let (name, value) = key - .with_key(|k| k.enum_values().nth(index as usize)) - .unwrap_or_else(|| { - Err(io::Error::from_raw_os_error( - Foundation::ERROR_NO_MORE_ITEMS as i32, - )) - }) - .map_err(|e| e.to_pyexception(vm))?; - #[allow(clippy::redundant_clone)] - let ty = value.vtype.clone() as usize; - Ok((name, reg_to_py(value, vm)?, ty)) - } - - #[pyfunction] - fn CloseKey(key: Hkey) { - match key { - Hkey::PyHkey(py) => py.Close(), - Hkey::Constant(hkey) => drop(RegKey::predef(hkey)), - } - } - - #[pyfunction] - fn CreateKey(key: Hkey, subkey: Option<PyStrRef>, vm: &VirtualMachine) -> PyResult<PyHkey> { - let k = match subkey { - Some(subkey) => { - let (k, _disp) = key - .with_key(|k| k.create_subkey(subkey.as_str())) - .map_err(|e| e.to_pyexception(vm))?; - k - } - None => key.into_key(), - }; - Ok(PyHkey::new(k)) - } - - #[pyfunction] - fn SetValue( - key: Hkey, - subkey: Option<PyStrRef>, - typ: u32, - value: PyStrRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - if typ != REG_SZ { - return Err(vm.new_type_error("type must be winreg.REG_SZ".to_owned())); - } - let subkey = subkey.as_ref().map_or("", |s| s.as_str()); - key.with_key(|k| k.set_value(subkey, &OsStr::new(value.as_str()))) - .map_err(|e| e.to_pyexception(vm)) - } - - #[pyfunction] - fn DeleteKey(key: Hkey, subkey: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - key.with_key(|k| k.delete_subkey(subkey.as_str())) - .map_err(|e| e.to_pyexception(vm)) - } - - fn reg_to_py(value: RegValue, vm: &VirtualMachine) -> PyResult { - macro_rules! bytes_to_int { - ($int:ident, $f:ident, $name:ident) => {{ - let i = if value.bytes.is_empty() { - Ok(0 as $int) - } else { - (&*value.bytes).try_into().map($int::$f).map_err(|_| { - vm.new_value_error(format!("{} value is wrong length", stringify!(name))) - }) - }; - i.map(|i| vm.ctx.new_int(i).into()) - }}; - } - let bytes_to_wide = |b: &[u8]| -> Option<&[u16]> { - if b.len() % 2 == 0 { - Some(unsafe { std::slice::from_raw_parts(b.as_ptr().cast(), b.len() / 2) }) - } else { - None - } - }; - match value.vtype { - RegType::REG_DWORD => bytes_to_int!(u32, from_ne_bytes, REG_DWORD), - RegType::REG_DWORD_BIG_ENDIAN => { - bytes_to_int!(u32, from_be_bytes, REG_DWORD_BIG_ENDIAN) - } - RegType::REG_QWORD => bytes_to_int!(u64, from_ne_bytes, REG_DWORD), - // RegType::REG_QWORD_BIG_ENDIAN => bytes_to_int!(u64, from_be_bytes, REG_DWORD_BIG_ENDIAN), - RegType::REG_SZ | RegType::REG_EXPAND_SZ => { - let wide_slice = bytes_to_wide(&value.bytes).ok_or_else(|| { - vm.new_value_error("REG_SZ string doesn't have an even byte length".to_owned()) - })?; - let nul_pos = wide_slice - .iter() - .position(|w| *w == 0) - .unwrap_or(wide_slice.len()); - let s = String::from_utf16_lossy(&wide_slice[..nul_pos]); - Ok(vm.ctx.new_str(s).into()) - } - RegType::REG_MULTI_SZ => { - if value.bytes.is_empty() { - return Ok(vm.ctx.new_list(vec![]).into()); - } - let wide_slice = bytes_to_wide(&value.bytes).ok_or_else(|| { - vm.new_value_error( - "REG_MULTI_SZ string doesn't have an even byte length".to_owned(), - ) - })?; - let wide_slice = if let Some((0, rest)) = wide_slice.split_last() { - rest - } else { - wide_slice - }; - let strings = wide_slice - .split(|c| *c == 0) - .map(|s| vm.new_pyobj(String::from_utf16_lossy(s))) - .collect(); - Ok(vm.ctx.new_list(strings).into()) - } - _ => { - if value.bytes.is_empty() { - Ok(vm.ctx.none()) - } else { - Ok(vm.ctx.new_bytes(value.bytes).into()) - } - } - } - } -} diff --git a/vm/src/suggestion.rs b/vm/src/suggestion.rs deleted file mode 100644 index d46630a6515..00000000000 --- a/vm/src/suggestion.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::{ - builtins::{PyStr, PyStrRef}, - exceptions::types::PyBaseExceptionRef, - sliceable::SliceableSequenceOp, - AsObject, Py, PyObjectRef, VirtualMachine, -}; -use rustpython_common::str::levenshtein::{levenshtein_distance, MOVE_COST}; -use std::iter::ExactSizeIterator; - -const MAX_CANDIDATE_ITEMS: usize = 750; - -fn calculate_suggestions<'a>( - dir_iter: impl ExactSizeIterator<Item = &'a PyObjectRef>, - name: &PyObjectRef, -) -> Option<PyStrRef> { - if dir_iter.len() >= MAX_CANDIDATE_ITEMS { - return None; - } - - let mut suggestion: Option<&Py<PyStr>> = None; - let mut suggestion_distance = usize::MAX; - let name = name.downcast_ref::<PyStr>()?; - - for item in dir_iter { - let item_name = item.downcast_ref::<PyStr>()?; - if name.as_str() == item_name.as_str() { - continue; - } - // No more than 1/3 of the characters should need changed - let max_distance = usize::min( - (name.len() + item_name.len() + 3) * MOVE_COST / 6, - suggestion_distance - 1, - ); - let current_distance = - levenshtein_distance(name.as_str(), item_name.as_str(), max_distance); - if current_distance > max_distance { - continue; - } - if suggestion.is_none() || current_distance < suggestion_distance { - suggestion = Some(item_name); - suggestion_distance = current_distance; - } - } - suggestion.map(|r| r.to_owned()) -} - -pub fn offer_suggestions(exc: &PyBaseExceptionRef, vm: &VirtualMachine) -> Option<PyStrRef> { - if exc.class().is(vm.ctx.exceptions.attribute_error) { - let name = exc.as_object().get_attr("name", vm).unwrap(); - let obj = exc.as_object().get_attr("obj", vm).unwrap(); - - calculate_suggestions(vm.dir(Some(obj)).ok()?.borrow_vec().iter(), &name) - } else if exc.class().is(vm.ctx.exceptions.name_error) { - let name = exc.as_object().get_attr("name", vm).unwrap(); - let mut tb = exc.traceback()?; - for traceback in tb.iter() { - tb = traceback; - } - - let varnames = tb.frame.code.clone().co_varnames(vm); - if let Some(suggestions) = calculate_suggestions(varnames.iter(), &name) { - return Some(suggestions); - }; - - let globals: Vec<_> = tb.frame.globals.as_object().try_to_value(vm).ok()?; - if let Some(suggestions) = calculate_suggestions(globals.iter(), &name) { - return Some(suggestions); - }; - - let builtins: Vec<_> = tb.frame.builtins.as_object().try_to_value(vm).ok()?; - calculate_suggestions(builtins.iter(), &name) - } else { - None - } -} diff --git a/vm/src/types/mod.rs b/vm/src/types/mod.rs deleted file mode 100644 index 56d925bfaf1..00000000000 --- a/vm/src/types/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod slot; -mod structseq; -mod zoo; - -pub use slot::*; -pub use structseq::PyStructSequence; -pub(crate) use zoo::TypeZoo; diff --git a/vm/src/types/slot.rs b/vm/src/types/slot.rs deleted file mode 100644 index 132a0f68c47..00000000000 --- a/vm/src/types/slot.rs +++ /dev/null @@ -1,1309 +0,0 @@ -use crate::{ - builtins::{type_::PointerSlot, PyInt, PyStr, PyStrInterned, PyStrRef, PyType, PyTypeRef}, - bytecode::ComparisonOperator, - common::hash::PyHash, - convert::{ToPyObject, ToPyResult}, - function::{ - Either, FromArgs, FuncArgs, OptionalArg, PyComparisonValue, PyMethodDef, PySetterValue, - }, - identifier, - protocol::{ - PyBuffer, PyIterReturn, PyMapping, PyMappingMethods, PyNumber, PyNumberMethods, - PyNumberSlots, PySequence, PySequenceMethods, - }, - vm::Context, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; -use crossbeam_utils::atomic::AtomicCell; -use num_traits::{Signed, ToPrimitive}; -use std::{borrow::Borrow, cmp::Ordering, ops::Deref}; - -#[macro_export] -macro_rules! atomic_func { - ($x:expr) => { - crossbeam_utils::atomic::AtomicCell::new(Some($x)) - }; -} - -// The corresponding field in CPython is `tp_` prefixed. -// e.g. name -> tp_name -#[derive(Default)] -#[non_exhaustive] -pub struct PyTypeSlots { - /// # Safety - /// For static types, always safe. - /// For heap types, `__name__` must alive - pub(crate) name: &'static str, // tp_name with <module>.<class> for print, not class name - - pub basicsize: usize, - // tp_itemsize - - // Methods to implement standard operations - - // Method suites for standard classes - pub as_number: PyNumberSlots, - pub as_sequence: AtomicCell<Option<PointerSlot<PySequenceMethods>>>, - pub as_mapping: AtomicCell<Option<PointerSlot<PyMappingMethods>>>, - - // More standard operations (here for binary compatibility) - pub hash: AtomicCell<Option<HashFunc>>, - pub call: AtomicCell<Option<GenericMethod>>, - // tp_str - pub repr: AtomicCell<Option<StringifyFunc>>, - pub getattro: AtomicCell<Option<GetattroFunc>>, - pub setattro: AtomicCell<Option<SetattroFunc>>, - - // Functions to access object as input/output buffer - pub as_buffer: Option<AsBufferFunc>, - - // Assigned meaning in release 2.1 - // rich comparisons - pub richcompare: AtomicCell<Option<RichCompareFunc>>, - - // Iterators - pub iter: AtomicCell<Option<IterFunc>>, - pub iternext: AtomicCell<Option<IterNextFunc>>, - - pub methods: &'static [PyMethodDef], - - // Flags to define presence of optional/expanded features - pub flags: PyTypeFlags, - - // tp_doc - pub doc: Option<&'static str>, - - // Strong reference on a heap type, borrowed reference on a static type - // tp_base - // tp_dict - pub descr_get: AtomicCell<Option<DescrGetFunc>>, - pub descr_set: AtomicCell<Option<DescrSetFunc>>, - // tp_dictoffset - pub init: AtomicCell<Option<InitFunc>>, - // tp_alloc - pub new: AtomicCell<Option<NewFunc>>, - // tp_free - // tp_is_gc - // tp_bases - // tp_mro - // tp_cache - // tp_subclasses - // tp_weaklist - pub del: AtomicCell<Option<DelFunc>>, - - // The count of tp_members. - pub member_count: usize, -} - -impl PyTypeSlots { - pub fn new(name: &'static str, flags: PyTypeFlags) -> Self { - Self { - name, - flags, - ..Default::default() - } - } - - pub fn heap_default() -> Self { - Self { - // init: AtomicCell::new(Some(init_wrapper)), - ..Default::default() - } - } -} - -impl std::fmt::Debug for PyTypeSlots { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("PyTypeSlots") - } -} - -bitflags! { - #[derive(Copy, Clone, Debug, PartialEq)] - #[non_exhaustive] - pub struct PyTypeFlags: u64 { - const IMMUTABLETYPE = 1 << 8; - const HEAPTYPE = 1 << 9; - const BASETYPE = 1 << 10; - const METHOD_DESCRIPTOR = 1 << 17; - const HAS_DICT = 1 << 40; - - #[cfg(debug_assertions)] - const _CREATED_WITH_FLAGS = 1 << 63; - } -} - -impl PyTypeFlags { - // Default used for both built-in and normal classes: empty, for now. - // CPython default: Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | Py_TPFLAGS_HAVE_VERSION_TAG - pub const DEFAULT: Self = Self::empty(); - - // CPython: See initialization of flags in type_new. - /// Used for types created in Python. Subclassable and are a - /// heaptype. - pub const fn heap_type_flags() -> Self { - match Self::from_bits(Self::DEFAULT.bits() | Self::HEAPTYPE.bits() | Self::BASETYPE.bits()) - { - Some(flags) => flags, - None => unreachable!(), - } - } - - pub fn has_feature(self, flag: Self) -> bool { - self.contains(flag) - } - - #[cfg(debug_assertions)] - pub fn is_created_with_flags(self) -> bool { - self.contains(Self::_CREATED_WITH_FLAGS) - } -} - -impl Default for PyTypeFlags { - fn default() -> Self { - Self::DEFAULT - } -} - -pub(crate) type GenericMethod = fn(&PyObject, FuncArgs, &VirtualMachine) -> PyResult; -pub(crate) type HashFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyHash>; -// CallFunc = GenericMethod -pub(crate) type StringifyFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyStrRef>; -pub(crate) type GetattroFunc = fn(&PyObject, &Py<PyStr>, &VirtualMachine) -> PyResult; -pub(crate) type SetattroFunc = - fn(&PyObject, &Py<PyStr>, PySetterValue, &VirtualMachine) -> PyResult<()>; -pub(crate) type AsBufferFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyBuffer>; -pub(crate) type RichCompareFunc = fn( - &PyObject, - &PyObject, - PyComparisonOp, - &VirtualMachine, -) -> PyResult<Either<PyObjectRef, PyComparisonValue>>; -pub(crate) type IterFunc = fn(PyObjectRef, &VirtualMachine) -> PyResult; -pub(crate) type IterNextFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyIterReturn>; -pub(crate) type DescrGetFunc = - fn(PyObjectRef, Option<PyObjectRef>, Option<PyObjectRef>, &VirtualMachine) -> PyResult; -pub(crate) type DescrSetFunc = - fn(&PyObject, PyObjectRef, PySetterValue, &VirtualMachine) -> PyResult<()>; -pub(crate) type NewFunc = fn(PyTypeRef, FuncArgs, &VirtualMachine) -> PyResult; -pub(crate) type InitFunc = fn(PyObjectRef, FuncArgs, &VirtualMachine) -> PyResult<()>; -pub(crate) type DelFunc = fn(&PyObject, &VirtualMachine) -> PyResult<()>; - -// slot_sq_length -pub(crate) fn len_wrapper(obj: &PyObject, vm: &VirtualMachine) -> PyResult<usize> { - let ret = vm.call_special_method(obj, identifier!(vm, __len__), ())?; - let len = ret.payload::<PyInt>().ok_or_else(|| { - vm.new_type_error(format!( - "'{}' object cannot be interpreted as an integer", - ret.class() - )) - })?; - let len = len.as_bigint(); - if len.is_negative() { - return Err(vm.new_value_error("__len__() should return >= 0".to_owned())); - } - let len = len.to_isize().ok_or_else(|| { - vm.new_overflow_error("cannot fit 'int' into an index-sized integer".to_owned()) - })?; - Ok(len as usize) -} - -macro_rules! number_unary_op_wrapper { - ($name:ident) => { - |a, vm| vm.call_special_method(a.deref(), identifier!(vm, $name), ()) - }; -} -macro_rules! number_binary_op_wrapper { - ($name:ident) => { - |a, b, vm| vm.call_special_method(a, identifier!(vm, $name), (b.to_owned(),)) - }; -} -macro_rules! number_binary_right_op_wrapper { - ($name:ident) => { - |a, b, vm| vm.call_special_method(b, identifier!(vm, $name), (a.to_owned(),)) - }; -} -fn getitem_wrapper<K: ToPyObject>(obj: &PyObject, needle: K, vm: &VirtualMachine) -> PyResult { - vm.call_special_method(obj, identifier!(vm, __getitem__), (needle,)) -} - -fn setitem_wrapper<K: ToPyObject>( - obj: &PyObject, - needle: K, - value: Option<PyObjectRef>, - vm: &VirtualMachine, -) -> PyResult<()> { - match value { - Some(value) => vm.call_special_method(obj, identifier!(vm, __setitem__), (needle, value)), - None => vm.call_special_method(obj, identifier!(vm, __delitem__), (needle,)), - } - .map(drop) -} - -fn repr_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let ret = vm.call_special_method(zelf, identifier!(vm, __repr__), ())?; - ret.downcast::<PyStr>().map_err(|obj| { - vm.new_type_error(format!( - "__repr__ returned non-string (type {})", - obj.class() - )) - }) -} - -fn hash_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyHash> { - let hash_obj = vm.call_special_method(zelf, identifier!(vm, __hash__), ())?; - let py_int = hash_obj - .payload_if_subclass::<PyInt>(vm) - .ok_or_else(|| vm.new_type_error("__hash__ method should return an integer".to_owned()))?; - Ok(rustpython_common::hash::hash_bigint(py_int.as_bigint())) -} - -/// Marks a type as unhashable. Similar to PyObject_HashNotImplemented in CPython -pub fn hash_not_implemented(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyHash> { - Err(vm.new_type_error(format!("unhashable type: {}", zelf.class().name()))) -} - -fn call_wrapper(zelf: &PyObject, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - vm.call_special_method(zelf, identifier!(vm, __call__), args) -} - -fn getattro_wrapper(zelf: &PyObject, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - let __getattribute__ = identifier!(vm, __getattribute__); - let __getattr__ = identifier!(vm, __getattr__); - match vm.call_special_method(zelf, __getattribute__, (name.to_owned(),)) { - Ok(r) => Ok(r), - Err(_) if zelf.class().has_attr(__getattr__) => { - vm.call_special_method(zelf, __getattr__, (name.to_owned(),)) - } - Err(e) => Err(e), - } -} - -fn setattro_wrapper( - zelf: &PyObject, - name: &Py<PyStr>, - value: PySetterValue, - vm: &VirtualMachine, -) -> PyResult<()> { - let name = name.to_owned(); - match value { - PySetterValue::Assign(value) => { - vm.call_special_method(zelf, identifier!(vm, __setattr__), (name, value))?; - } - PySetterValue::Delete => { - vm.call_special_method(zelf, identifier!(vm, __delattr__), (name,))?; - } - }; - Ok(()) -} - -pub(crate) fn richcompare_wrapper( - zelf: &PyObject, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, -) -> PyResult<Either<PyObjectRef, PyComparisonValue>> { - vm.call_special_method(zelf, op.method_name(&vm.ctx), (other.to_owned(),)) - .map(Either::A) -} - -fn iter_wrapper(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm.call_special_method(&zelf, identifier!(vm, __iter__), ()) -} - -// PyObject_SelfIter in CPython -fn self_iter(zelf: PyObjectRef, _vm: &VirtualMachine) -> PyResult { - Ok(zelf) -} - -fn iternext_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - PyIterReturn::from_pyresult( - vm.call_special_method(zelf, identifier!(vm, __next__), ()), - vm, - ) -} - -fn descr_get_wrapper( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - cls: Option<PyObjectRef>, - vm: &VirtualMachine, -) -> PyResult { - vm.call_special_method(&zelf, identifier!(vm, __get__), (obj, cls)) -} - -fn descr_set_wrapper( - zelf: &PyObject, - obj: PyObjectRef, - value: PySetterValue, - vm: &VirtualMachine, -) -> PyResult<()> { - match value { - PySetterValue::Assign(val) => { - vm.call_special_method(zelf, identifier!(vm, __set__), (obj, val)) - } - PySetterValue::Delete => vm.call_special_method(zelf, identifier!(vm, __delete__), (obj,)), - } - .map(drop) -} - -fn init_wrapper(obj: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let res = vm.call_special_method(&obj, identifier!(vm, __init__), args)?; - if !vm.is_none(&res) { - return Err(vm.new_type_error("__init__ must return None".to_owned())); - } - Ok(()) -} - -pub(crate) fn new_wrapper(cls: PyTypeRef, mut args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let new = cls.get_attr(identifier!(vm, __new__)).unwrap(); - args.prepend_arg(cls.into()); - new.call(args, vm) -} - -fn del_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - vm.call_special_method(zelf, identifier!(vm, __del__), ())?; - Ok(()) -} - -impl PyType { - pub(crate) fn update_slot<const ADD: bool>(&self, name: &'static PyStrInterned, ctx: &Context) { - debug_assert!(name.as_str().starts_with("__")); - debug_assert!(name.as_str().ends_with("__")); - - macro_rules! toggle_slot { - ($name:ident, $func:expr) => {{ - self.slots.$name.store(if ADD { Some($func) } else { None }); - }}; - } - - macro_rules! toggle_subslot { - ($group:ident, $name:ident, $func:expr) => { - self.slots - .$group - .$name - .store(if ADD { Some($func) } else { None }); - }; - } - - macro_rules! update_slot { - ($name:ident, $func:expr) => {{ - self.slots.$name.store(Some($func)); - }}; - } - - macro_rules! update_pointer_slot { - ($name:ident, $pointed:ident) => {{ - self.slots - .$name - .store(unsafe { PointerSlot::from_heaptype(self, |ext| &ext.$pointed) }); - }}; - } - - macro_rules! toggle_ext_func { - ($n1:ident, $n2:ident, $func:expr) => {{ - self.heaptype_ext.as_ref().unwrap().$n1.$n2.store(if ADD { - Some($func) - } else { - None - }); - }}; - } - - match name { - _ if name == identifier!(ctx, __len__) => { - // update_slot!(as_mapping, slot_as_mapping); - toggle_ext_func!(sequence_methods, length, |seq, vm| len_wrapper(seq.obj, vm)); - update_pointer_slot!(as_sequence, sequence_methods); - toggle_ext_func!(mapping_methods, length, |mapping, vm| len_wrapper( - mapping.obj, - vm - )); - update_pointer_slot!(as_mapping, mapping_methods); - } - _ if name == identifier!(ctx, __getitem__) => { - // update_slot!(as_mapping, slot_as_mapping); - toggle_ext_func!(sequence_methods, item, |seq, i, vm| getitem_wrapper( - seq.obj, i, vm - )); - update_pointer_slot!(as_sequence, sequence_methods); - toggle_ext_func!(mapping_methods, subscript, |mapping, key, vm| { - getitem_wrapper(mapping.obj, key, vm) - }); - update_pointer_slot!(as_mapping, mapping_methods); - } - _ if name == identifier!(ctx, __setitem__) || name == identifier!(ctx, __delitem__) => { - // update_slot!(as_mapping, slot_as_mapping); - toggle_ext_func!(sequence_methods, ass_item, |seq, i, value, vm| { - setitem_wrapper(seq.obj, i, value, vm) - }); - update_pointer_slot!(as_sequence, sequence_methods); - toggle_ext_func!(mapping_methods, ass_subscript, |mapping, key, value, vm| { - setitem_wrapper(mapping.obj, key, value, vm) - }); - update_pointer_slot!(as_mapping, mapping_methods); - } - _ if name == identifier!(ctx, __repr__) => { - update_slot!(repr, repr_wrapper); - } - _ if name == identifier!(ctx, __hash__) => { - let is_unhashable = self - .attributes - .read() - .get(identifier!(ctx, __hash__)) - .map_or(false, |a| a.is(&ctx.none)); - let wrapper = if is_unhashable { - hash_not_implemented - } else { - hash_wrapper - }; - toggle_slot!(hash, wrapper); - } - _ if name == identifier!(ctx, __call__) => { - toggle_slot!(call, call_wrapper); - } - _ if name == identifier!(ctx, __getattr__) - || name == identifier!(ctx, __getattribute__) => - { - update_slot!(getattro, getattro_wrapper); - } - _ if name == identifier!(ctx, __setattr__) || name == identifier!(ctx, __delattr__) => { - update_slot!(setattro, setattro_wrapper); - } - _ if name == identifier!(ctx, __eq__) - || name == identifier!(ctx, __ne__) - || name == identifier!(ctx, __le__) - || name == identifier!(ctx, __lt__) - || name == identifier!(ctx, __ge__) - || name == identifier!(ctx, __gt__) => - { - update_slot!(richcompare, richcompare_wrapper); - } - _ if name == identifier!(ctx, __iter__) => { - toggle_slot!(iter, iter_wrapper); - } - _ if name == identifier!(ctx, __next__) => { - toggle_slot!(iternext, iternext_wrapper); - } - _ if name == identifier!(ctx, __get__) => { - toggle_slot!(descr_get, descr_get_wrapper); - } - _ if name == identifier!(ctx, __set__) || name == identifier!(ctx, __delete__) => { - update_slot!(descr_set, descr_set_wrapper); - } - _ if name == identifier!(ctx, __init__) => { - toggle_slot!(init, init_wrapper); - } - _ if name == identifier!(ctx, __new__) => { - toggle_slot!(new, new_wrapper); - } - _ if name == identifier!(ctx, __del__) => { - toggle_slot!(del, del_wrapper); - } - _ if name == identifier!(ctx, __int__) => { - toggle_subslot!(as_number, int, number_unary_op_wrapper!(__int__)); - } - _ if name == identifier!(ctx, __index__) => { - toggle_subslot!(as_number, index, number_unary_op_wrapper!(__index__)); - } - _ if name == identifier!(ctx, __float__) => { - toggle_subslot!(as_number, float, number_unary_op_wrapper!(__float__)); - } - _ if name == identifier!(ctx, __add__) => { - toggle_subslot!(as_number, add, number_binary_op_wrapper!(__add__)); - } - _ if name == identifier!(ctx, __radd__) => { - toggle_subslot!( - as_number, - right_add, - number_binary_right_op_wrapper!(__radd__) - ); - } - _ if name == identifier!(ctx, __iadd__) => { - toggle_subslot!(as_number, inplace_add, number_binary_op_wrapper!(__iadd__)); - } - _ if name == identifier!(ctx, __sub__) => { - toggle_subslot!(as_number, subtract, number_binary_op_wrapper!(__sub__)); - } - _ if name == identifier!(ctx, __rsub__) => { - toggle_subslot!( - as_number, - right_subtract, - number_binary_right_op_wrapper!(__rsub__) - ); - } - _ if name == identifier!(ctx, __isub__) => { - toggle_subslot!( - as_number, - inplace_subtract, - number_binary_op_wrapper!(__isub__) - ); - } - _ if name == identifier!(ctx, __mul__) => { - toggle_subslot!(as_number, multiply, number_binary_op_wrapper!(__mul__)); - } - _ if name == identifier!(ctx, __rmul__) => { - toggle_subslot!( - as_number, - right_multiply, - number_binary_right_op_wrapper!(__rmul__) - ); - } - _ if name == identifier!(ctx, __imul__) => { - toggle_subslot!( - as_number, - inplace_multiply, - number_binary_op_wrapper!(__imul__) - ); - } - _ if name == identifier!(ctx, __mod__) => { - toggle_subslot!(as_number, remainder, number_binary_op_wrapper!(__mod__)); - } - _ if name == identifier!(ctx, __rmod__) => { - toggle_subslot!( - as_number, - right_remainder, - number_binary_right_op_wrapper!(__rmod__) - ); - } - _ if name == identifier!(ctx, __imod__) => { - toggle_subslot!( - as_number, - inplace_remainder, - number_binary_op_wrapper!(__imod__) - ); - } - _ if name == identifier!(ctx, __divmod__) => { - toggle_subslot!(as_number, divmod, number_binary_op_wrapper!(__divmod__)); - } - _ if name == identifier!(ctx, __rdivmod__) => { - toggle_subslot!( - as_number, - right_divmod, - number_binary_right_op_wrapper!(__rdivmod__) - ); - } - _ if name == identifier!(ctx, __pow__) => { - toggle_subslot!(as_number, power, |a, b, c, vm| { - let args = if vm.is_none(c) { - vec![b.to_owned()] - } else { - vec![b.to_owned(), c.to_owned()] - }; - vm.call_special_method(a, identifier!(vm, __pow__), args) - }); - } - _ if name == identifier!(ctx, __rpow__) => { - toggle_subslot!(as_number, right_power, |a, b, c, vm| { - let args = if vm.is_none(c) { - vec![a.to_owned()] - } else { - vec![a.to_owned(), c.to_owned()] - }; - vm.call_special_method(b, identifier!(vm, __rpow__), args) - }); - } - _ if name == identifier!(ctx, __ipow__) => { - toggle_subslot!(as_number, inplace_power, |a, b, _, vm| { - vm.call_special_method(a, identifier!(vm, __ipow__), (b.to_owned(),)) - }); - } - _ if name == identifier!(ctx, __lshift__) => { - toggle_subslot!(as_number, lshift, number_binary_op_wrapper!(__lshift__)); - } - _ if name == identifier!(ctx, __rlshift__) => { - toggle_subslot!( - as_number, - right_lshift, - number_binary_right_op_wrapper!(__rlshift__) - ); - } - _ if name == identifier!(ctx, __ilshift__) => { - toggle_subslot!( - as_number, - inplace_lshift, - number_binary_op_wrapper!(__ilshift__) - ); - } - _ if name == identifier!(ctx, __rshift__) => { - toggle_subslot!(as_number, rshift, number_binary_op_wrapper!(__rshift__)); - } - _ if name == identifier!(ctx, __rrshift__) => { - toggle_subslot!( - as_number, - right_rshift, - number_binary_right_op_wrapper!(__rrshift__) - ); - } - _ if name == identifier!(ctx, __irshift__) => { - toggle_subslot!( - as_number, - inplace_rshift, - number_binary_op_wrapper!(__irshift__) - ); - } - _ if name == identifier!(ctx, __and__) => { - toggle_subslot!(as_number, and, number_binary_op_wrapper!(__and__)); - } - _ if name == identifier!(ctx, __rand__) => { - toggle_subslot!( - as_number, - right_and, - number_binary_right_op_wrapper!(__rand__) - ); - } - _ if name == identifier!(ctx, __iand__) => { - toggle_subslot!(as_number, inplace_and, number_binary_op_wrapper!(__iand__)); - } - _ if name == identifier!(ctx, __xor__) => { - toggle_subslot!(as_number, xor, number_binary_op_wrapper!(__xor__)); - } - _ if name == identifier!(ctx, __rxor__) => { - toggle_subslot!( - as_number, - right_xor, - number_binary_right_op_wrapper!(__rxor__) - ); - } - _ if name == identifier!(ctx, __ixor__) => { - toggle_subslot!(as_number, inplace_xor, number_binary_op_wrapper!(__ixor__)); - } - _ if name == identifier!(ctx, __or__) => { - toggle_subslot!(as_number, or, number_binary_op_wrapper!(__or__)); - } - _ if name == identifier!(ctx, __ror__) => { - toggle_subslot!( - as_number, - right_or, - number_binary_right_op_wrapper!(__ror__) - ); - } - _ if name == identifier!(ctx, __ior__) => { - toggle_subslot!(as_number, inplace_or, number_binary_op_wrapper!(__ior__)); - } - _ if name == identifier!(ctx, __floordiv__) => { - toggle_subslot!( - as_number, - floor_divide, - number_binary_op_wrapper!(__floordiv__) - ); - } - _ if name == identifier!(ctx, __rfloordiv__) => { - toggle_subslot!( - as_number, - right_floor_divide, - number_binary_right_op_wrapper!(__rfloordiv__) - ); - } - _ if name == identifier!(ctx, __ifloordiv__) => { - toggle_subslot!( - as_number, - inplace_floor_divide, - number_binary_op_wrapper!(__ifloordiv__) - ); - } - _ if name == identifier!(ctx, __truediv__) => { - toggle_subslot!( - as_number, - true_divide, - number_binary_op_wrapper!(__truediv__) - ); - } - _ if name == identifier!(ctx, __rtruediv__) => { - toggle_subslot!( - as_number, - right_true_divide, - number_binary_right_op_wrapper!(__rtruediv__) - ); - } - _ if name == identifier!(ctx, __itruediv__) => { - toggle_subslot!( - as_number, - inplace_true_divide, - number_binary_op_wrapper!(__itruediv__) - ); - } - _ if name == identifier!(ctx, __matmul__) => { - toggle_subslot!( - as_number, - matrix_multiply, - number_binary_op_wrapper!(__matmul__) - ); - } - _ if name == identifier!(ctx, __rmatmul__) => { - toggle_subslot!( - as_number, - right_matrix_multiply, - number_binary_right_op_wrapper!(__rmatmul__) - ); - } - _ if name == identifier!(ctx, __imatmul__) => { - toggle_subslot!( - as_number, - inplace_matrix_multiply, - number_binary_op_wrapper!(__imatmul__) - ); - } - _ => {} - } - } -} - -#[pyclass] -pub trait Constructor: PyPayload { - type Args: FromArgs; - - #[inline] - #[pyslot] - fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let args: Self::Args = args.bind(vm)?; - Self::py_new(cls, args, vm) - } - - fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult; -} - -#[pyclass] -pub trait DefaultConstructor: PyPayload + Default { - #[inline] - #[pyslot] - fn slot_new(cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Self::default().into_ref_with_type(vm, cls).map(Into::into) - } -} - -/// For types that cannot be instantiated through Python code. -pub trait Unconstructible: PyPayload {} - -impl<T> Constructor for T -where - T: Unconstructible, -{ - type Args = FuncArgs; - - fn py_new(cls: PyTypeRef, _args: Self::Args, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error(format!("cannot create {} instances", cls.slot_name()))) - } -} - -#[pyclass] -pub trait Initializer: PyPayload { - type Args: FromArgs; - - #[pyslot] - #[inline] - fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let zelf = zelf.try_into_value(vm)?; - let args: Self::Args = args.bind(vm)?; - Self::init(zelf, args, vm) - } - - #[pymethod] - #[inline] - fn __init__(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()> { - Self::init(zelf, args, vm) - } - - fn init(zelf: PyRef<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult<()>; -} - -#[pyclass] -pub trait Destructor: PyPayload { - #[inline] // for __del__ - #[pyslot] - fn slot_del(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let zelf = zelf - .downcast_ref() - .ok_or_else(|| vm.new_type_error("unexpected payload for __del__".to_owned()))?; - Self::del(zelf, vm) - } - - #[pymethod] - fn __del__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - Self::slot_del(&zelf, vm) - } - - fn del(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<()>; -} - -#[pyclass] -pub trait Callable: PyPayload { - type Args: FromArgs; - - #[inline] - #[pyslot] - fn slot_call(zelf: &PyObject, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let zelf = zelf.downcast_ref().ok_or_else(|| { - let repr = zelf.repr(vm); - let help = if let Ok(repr) = repr.as_ref() { - repr.as_str().to_owned() - } else { - zelf.class().name().to_owned() - }; - vm.new_type_error(format!("unexpected payload for __call__ of {help}")) - })?; - let args = args.bind(vm)?; - Self::call(zelf, args, vm) - } - - #[inline] - #[pymethod] - fn __call__(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Self::slot_call(&zelf, args.bind(vm)?, vm) - } - fn call(zelf: &Py<Self>, args: Self::Args, vm: &VirtualMachine) -> PyResult; -} - -#[pyclass] -pub trait GetDescriptor: PyPayload { - #[pyslot] - fn descr_get( - zelf: PyObjectRef, - obj: Option<PyObjectRef>, - cls: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult; - - #[inline] - #[pymethod(magic)] - fn get( - zelf: PyObjectRef, - obj: PyObjectRef, - cls: OptionalArg<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult { - Self::descr_get(zelf, Some(obj), cls.into_option(), vm) - } - - #[inline] - fn _as_pyref<'a>(zelf: &'a PyObject, vm: &VirtualMachine) -> PyResult<&'a Py<Self>> { - zelf.try_to_value(vm) - } - - #[inline] - fn _unwrap<'a>( - zelf: &'a PyObject, - obj: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> PyResult<(&'a Py<Self>, PyObjectRef)> { - let zelf = Self::_as_pyref(zelf, vm)?; - let obj = vm.unwrap_or_none(obj); - Ok((zelf, obj)) - } - - #[inline] - fn _check<'a>( - zelf: &'a PyObject, - obj: Option<PyObjectRef>, - vm: &VirtualMachine, - ) -> Option<(&'a Py<Self>, PyObjectRef)> { - // CPython descr_check - let obj = obj?; - // if (!PyObject_TypeCheck(obj, descr->d_type)) { - // PyErr_Format(PyExc_TypeError, - // "descriptor '%V' for '%.100s' objects " - // "doesn't apply to a '%.100s' object", - // descr_name((PyDescrObject *)descr), "?", - // descr->d_type->slot_name, - // obj->ob_type->slot_name); - // *pres = NULL; - // return 1; - // } else { - Some((Self::_as_pyref(zelf, vm).unwrap(), obj)) - } - - #[inline] - fn _cls_is(cls: &Option<PyObjectRef>, other: &impl Borrow<PyObject>) -> bool { - cls.as_ref().map_or(false, |cls| other.borrow().is(cls)) - } -} - -#[pyclass] -pub trait Hashable: PyPayload { - #[inline] - #[pyslot] - fn slot_hash(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyHash> { - let zelf = zelf - .downcast_ref() - .ok_or_else(|| vm.new_type_error("unexpected payload for __hash__".to_owned()))?; - Self::hash(zelf, vm) - } - - #[inline] - #[pymethod] - fn __hash__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyHash> { - Self::slot_hash(&zelf, vm) - } - - fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash>; -} - -#[pyclass] -pub trait Representable: PyPayload { - #[inline] - #[pyslot] - fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let zelf = zelf - .downcast_ref() - .ok_or_else(|| vm.new_type_error("unexpected payload for __repr__".to_owned()))?; - Self::repr(zelf, vm) - } - - #[inline] - #[pymethod] - fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> { - Self::slot_repr(&zelf, vm) - } - - #[inline] - fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> { - let repr = Self::repr_str(zelf, vm)?; - Ok(vm.ctx.new_str(repr)) - } - - fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String>; -} - -#[pyclass] -pub trait Comparable: PyPayload { - #[inline] - #[pyslot] - fn slot_richcompare( - zelf: &PyObject, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<Either<PyObjectRef, PyComparisonValue>> { - let zelf = zelf.downcast_ref().ok_or_else(|| { - vm.new_type_error(format!( - "unexpected payload for {}", - op.method_name(&vm.ctx).as_str() - )) - })?; - Self::cmp(zelf, other, op, vm).map(Either::B) - } - - fn cmp( - zelf: &Py<Self>, - other: &PyObject, - op: PyComparisonOp, - vm: &VirtualMachine, - ) -> PyResult<PyComparisonValue>; - - #[inline] - #[pymethod(magic)] - fn eq(zelf: &Py<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyComparisonValue> { - Self::cmp(zelf, &other, PyComparisonOp::Eq, vm) - } - #[inline] - #[pymethod(magic)] - fn ne(zelf: &Py<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyComparisonValue> { - Self::cmp(zelf, &other, PyComparisonOp::Ne, vm) - } - #[inline] - #[pymethod(magic)] - fn lt(zelf: &Py<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyComparisonValue> { - Self::cmp(zelf, &other, PyComparisonOp::Lt, vm) - } - #[inline] - #[pymethod(magic)] - fn le(zelf: &Py<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyComparisonValue> { - Self::cmp(zelf, &other, PyComparisonOp::Le, vm) - } - #[inline] - #[pymethod(magic)] - fn ge(zelf: &Py<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyComparisonValue> { - Self::cmp(zelf, &other, PyComparisonOp::Ge, vm) - } - #[inline] - #[pymethod(magic)] - fn gt(zelf: &Py<Self>, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyComparisonValue> { - Self::cmp(zelf, &other, PyComparisonOp::Gt, vm) - } -} - -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -#[repr(transparent)] -pub struct PyComparisonOp(ComparisonOperator); - -impl From<ComparisonOperator> for PyComparisonOp { - fn from(op: ComparisonOperator) -> Self { - Self(op) - } -} - -#[allow(non_upper_case_globals)] -impl PyComparisonOp { - pub const Lt: Self = Self(ComparisonOperator::Less); - pub const Gt: Self = Self(ComparisonOperator::Greater); - pub const Ne: Self = Self(ComparisonOperator::NotEqual); - pub const Eq: Self = Self(ComparisonOperator::Equal); - pub const Le: Self = Self(ComparisonOperator::LessOrEqual); - pub const Ge: Self = Self(ComparisonOperator::GreaterOrEqual); -} - -impl PyComparisonOp { - pub fn eq_only( - self, - f: impl FnOnce() -> PyResult<PyComparisonValue>, - ) -> PyResult<PyComparisonValue> { - match self { - Self::Eq => f(), - Self::Ne => f().map(|x| x.map(|eq| !eq)), - _ => Ok(PyComparisonValue::NotImplemented), - } - } - - pub fn eval_ord(self, ord: Ordering) -> bool { - let bit = match ord { - Ordering::Less => Self::Lt, - Ordering::Equal => Self::Eq, - Ordering::Greater => Self::Gt, - }; - self.0 as u8 & bit.0 as u8 != 0 - } - - pub fn swapped(self) -> Self { - match self { - Self::Lt => Self::Gt, - Self::Le => Self::Ge, - Self::Eq => Self::Eq, - Self::Ne => Self::Ne, - Self::Ge => Self::Le, - Self::Gt => Self::Lt, - } - } - - pub fn method_name(self, ctx: &Context) -> &'static PyStrInterned { - match self { - Self::Lt => identifier!(ctx, __lt__), - Self::Le => identifier!(ctx, __le__), - Self::Eq => identifier!(ctx, __eq__), - Self::Ne => identifier!(ctx, __ne__), - Self::Ge => identifier!(ctx, __ge__), - Self::Gt => identifier!(ctx, __gt__), - } - } - - pub fn operator_token(self) -> &'static str { - match self { - Self::Lt => "<", - Self::Le => "<=", - Self::Eq => "==", - Self::Ne => "!=", - Self::Ge => ">=", - Self::Gt => ">", - } - } - - /// Returns an appropriate return value for the comparison when a and b are the same object, if an - /// appropriate return value exists. - #[inline] - pub fn identical_optimization( - self, - a: &impl Borrow<PyObject>, - b: &impl Borrow<PyObject>, - ) -> Option<bool> { - self.map_eq(|| a.borrow().is(b.borrow())) - } - - /// Returns `Some(true)` when self is `Eq` and `f()` returns true. Returns `Some(false)` when self - /// is `Ne` and `f()` returns true. Otherwise returns `None`. - #[inline] - pub fn map_eq(self, f: impl FnOnce() -> bool) -> Option<bool> { - let eq = match self { - Self::Eq => true, - Self::Ne => false, - _ => return None, - }; - f().then_some(eq) - } -} - -#[pyclass] -pub trait GetAttr: PyPayload { - #[pyslot] - fn slot_getattro(obj: &PyObject, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult { - let zelf = obj.downcast_ref().ok_or_else(|| { - vm.new_type_error("unexpected payload for __getattribute__".to_owned()) - })?; - Self::getattro(zelf, name, vm) - } - - fn getattro(zelf: &Py<Self>, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult; - - #[inline] - #[pymethod(magic)] - fn getattribute(zelf: PyRef<Self>, name: PyStrRef, vm: &VirtualMachine) -> PyResult { - Self::getattro(&zelf, &name, vm) - } -} - -#[pyclass] -pub trait SetAttr: PyPayload { - #[pyslot] - #[inline] - fn slot_setattro( - obj: &PyObject, - name: &Py<PyStr>, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()> { - let zelf = obj - .downcast_ref::<Self>() - .ok_or_else(|| vm.new_type_error("unexpected payload for __setattr__".to_owned()))?; - Self::setattro(zelf, name, value, vm) - } - - fn setattro( - zelf: &Py<Self>, - name: &Py<PyStr>, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()>; - - #[inline] - #[pymethod(magic)] - fn setattr( - zelf: PyRef<Self>, - name: PyStrRef, - value: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - Self::setattro(&zelf, &name, PySetterValue::Assign(value), vm) - } - - #[inline] - #[pymethod(magic)] - fn delattr(zelf: PyRef<Self>, name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { - Self::setattro(&zelf, &name, PySetterValue::Delete, vm) - } -} - -#[pyclass] -pub trait AsBuffer: PyPayload { - // TODO: `flags` parameter - #[inline] - #[pyslot] - fn slot_as_buffer(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyBuffer> { - let zelf = zelf - .downcast_ref() - .ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer".to_owned()))?; - Self::as_buffer(zelf, vm) - } - - fn as_buffer(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyBuffer>; -} - -#[pyclass] -pub trait AsMapping: PyPayload { - #[pyslot] - fn as_mapping() -> &'static PyMappingMethods; - - #[inline] - fn mapping_downcast(mapping: PyMapping) -> &Py<Self> { - unsafe { mapping.obj.downcast_unchecked_ref() } - } -} - -#[pyclass] -pub trait AsSequence: PyPayload { - #[pyslot] - fn as_sequence() -> &'static PySequenceMethods; - - #[inline] - fn sequence_downcast(seq: PySequence) -> &Py<Self> { - unsafe { seq.obj.downcast_unchecked_ref() } - } -} - -#[pyclass] -pub trait AsNumber: PyPayload { - #[pyslot] - fn as_number() -> &'static PyNumberMethods; - - fn clone_exact(_zelf: &Py<Self>, _vm: &VirtualMachine) -> PyRef<Self> { - // not all AsNumber requires this implementation. - unimplemented!() - } - - #[inline] - fn number_downcast(num: PyNumber) -> &Py<Self> { - unsafe { num.obj().downcast_unchecked_ref() } - } - - #[inline] - fn number_downcast_exact(num: PyNumber, vm: &VirtualMachine) -> PyRef<Self> { - if let Some(zelf) = num.downcast_ref_if_exact::<Self>(vm) { - zelf.to_owned() - } else { - Self::clone_exact(Self::number_downcast(num), vm) - } - } -} - -#[pyclass] -pub trait Iterable: PyPayload { - #[pyslot] - fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let zelf = zelf - .downcast() - .map_err(|_| vm.new_type_error("unexpected payload for __iter__".to_owned()))?; - Self::iter(zelf, vm) - } - - #[pymethod] - fn __iter__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Self::slot_iter(zelf, vm) - } - - fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult; - - fn extend_slots(_slots: &mut PyTypeSlots) {} -} - -// `Iterator` fits better, but to avoid confusion with rust std::iter::Iterator -#[pyclass(with(Iterable))] -pub trait IterNext: PyPayload + Iterable { - #[pyslot] - fn slot_iternext(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyIterReturn> { - let zelf = zelf - .downcast_ref() - .ok_or_else(|| vm.new_type_error("unexpected payload for __next__".to_owned()))?; - Self::next(zelf, vm) - } - - fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn>; - - #[inline] - #[pymethod] - fn __next__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - Self::slot_iternext(&zelf, vm).to_pyresult(vm) - } -} - -pub trait SelfIter: PyPayload {} - -impl<T> Iterable for T -where - T: SelfIter, -{ - #[cold] - fn slot_iter(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let repr = zelf.repr(vm)?; - unreachable!("slot must be overriden for {}", repr.as_str()); - } - - fn __iter__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self_iter(zelf, vm) - } - - #[cold] - fn iter(_zelf: PyRef<Self>, _vm: &VirtualMachine) -> PyResult { - unreachable!("slot_iter is implemented"); - } - - fn extend_slots(slots: &mut PyTypeSlots) { - let prev = slots.iter.swap(Some(self_iter)); - debug_assert!(prev.is_some()); // slot_iter would be set - } -} diff --git a/vm/src/types/structseq.rs b/vm/src/types/structseq.rs deleted file mode 100644 index 516e2085af6..00000000000 --- a/vm/src/types/structseq.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::{ - builtins::{PyTuple, PyTupleRef, PyType}, - class::{PyClassImpl, StaticType}, - vm::Context, - AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, -}; - -#[pyclass] -pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { - const FIELD_NAMES: &'static [&'static str]; - - fn into_tuple(self, vm: &VirtualMachine) -> PyTuple; - - fn into_struct_sequence(self, vm: &VirtualMachine) -> PyTupleRef { - self.into_tuple(vm) - .into_ref_with_type(vm, Self::static_type().to_owned()) - .unwrap() - } - - fn try_elements_from<const FIELD_LEN: usize>( - obj: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<[PyObjectRef; FIELD_LEN]> { - let typ = Self::static_type(); - // if !obj.fast_isinstance(typ) { - // return Err(vm.new_type_error(format!( - // "{} is not a subclass of {}", - // obj.class().name(), - // typ.name(), - // ))); - // } - let seq: Vec<PyObjectRef> = obj.try_into_value(vm)?; - let seq: [PyObjectRef; FIELD_LEN] = seq.try_into().map_err(|_| { - vm.new_type_error(format!( - "{} takes a sequence of length {}", - typ.name(), - FIELD_LEN - )) - })?; - Ok(seq) - } - - #[pymethod(magic)] - fn repr(zelf: PyRef<PyTuple>, vm: &VirtualMachine) -> PyResult<String> { - let format_field = |(value, name): (&PyObjectRef, _)| { - let s = value.repr(vm)?; - Ok(format!("{name}={s}")) - }; - let (body, suffix) = if let Some(_guard) = - rustpython_vm::recursion::ReprGuard::enter(vm, zelf.as_object()) - { - if Self::FIELD_NAMES.len() == 1 { - let value = zelf.first().unwrap(); - let formatted = format_field((value, Self::FIELD_NAMES[0]))?; - (formatted, ",") - } else { - let fields: PyResult<Vec<_>> = zelf - .iter() - .zip(Self::FIELD_NAMES.iter().copied()) - .map(format_field) - .collect(); - (fields?.join(", "), "") - } - } else { - (String::new(), "...") - }; - Ok(format!("{}({}{})", Self::TP_NAME, body, suffix)) - } - - #[pymethod(magic)] - fn reduce(zelf: PyRef<PyTuple>, vm: &VirtualMachine) -> PyTupleRef { - vm.new_tuple((zelf.class().to_owned(), (vm.ctx.new_tuple(zelf.to_vec()),))) - } - - #[extend_class] - fn extend_pyclass(ctx: &Context, class: &'static Py<PyType>) { - for (i, &name) in Self::FIELD_NAMES.iter().enumerate() { - // cast i to a u8 so there's less to store in the getter closure. - // Hopefully there's not struct sequences with >=256 elements :P - let i = i as u8; - class.set_attr( - ctx.intern_str(name), - ctx.new_readonly_getset(name, class, move |zelf: &PyTuple| { - zelf.fast_getitem(i.into()) - }) - .into(), - ); - } - - class.set_attr( - identifier!(ctx, __match_args__), - ctx.new_tuple( - Self::FIELD_NAMES - .iter() - .map(|&name| ctx.new_str(name).into()) - .collect::<Vec<_>>(), - ) - .into(), - ); - } -} diff --git a/vm/src/utils.rs b/vm/src/utils.rs deleted file mode 100644 index ab45343f850..00000000000 --- a/vm/src/utils.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::{ - builtins::PyStr, - convert::{ToPyException, ToPyObject}, - PyObjectRef, PyResult, VirtualMachine, -}; - -pub fn hash_iter<'a, I: IntoIterator<Item = &'a PyObjectRef>>( - iter: I, - vm: &VirtualMachine, -) -> PyResult<rustpython_common::hash::PyHash> { - vm.state.hash_secret.hash_iter(iter, |obj| obj.hash(vm)) -} - -pub fn hash_iter_unordered<'a, I: IntoIterator<Item = &'a PyObjectRef>>( - iter: I, - vm: &VirtualMachine, -) -> PyResult<rustpython_common::hash::PyHash> { - rustpython_common::hash::hash_iter_unordered(iter, |obj| obj.hash(vm)) -} - -impl ToPyObject for std::convert::Infallible { - fn to_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { - match self {} - } -} - -pub trait ToCString { - fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<std::ffi::CString>; -} - -impl ToCString for &str { - fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<std::ffi::CString> { - std::ffi::CString::new(*self).map_err(|err| err.to_pyexception(vm)) - } -} - -impl ToCString for PyStr { - fn to_cstring(&self, vm: &VirtualMachine) -> PyResult<std::ffi::CString> { - std::ffi::CString::new(self.as_ref()).map_err(|err| err.to_pyexception(vm)) - } -} - -pub(crate) fn collection_repr<'a, I>( - class_name: Option<&str>, - prefix: &str, - suffix: &str, - iter: I, - vm: &VirtualMachine, -) -> PyResult<String> -where - I: std::iter::Iterator<Item = &'a PyObjectRef>, -{ - let mut repr = String::new(); - if let Some(name) = class_name { - repr.push_str(name); - repr.push('('); - } - repr.push_str(prefix); - { - let mut parts_iter = iter.map(|o| o.repr(vm)); - repr.push_str( - parts_iter - .next() - .transpose()? - .expect("this is not called for empty collection") - .as_str(), - ); - for part in parts_iter { - repr.push_str(", "); - repr.push_str(part?.as_str()); - } - } - repr.push_str(suffix); - if class_name.is_some() { - repr.push(')'); - } - - Ok(repr) -} diff --git a/vm/src/version.rs b/vm/src/version.rs deleted file mode 100644 index 9a75f71142c..00000000000 --- a/vm/src/version.rs +++ /dev/null @@ -1,113 +0,0 @@ -/* Several function to retrieve version information. - */ - -use chrono::{prelude::DateTime, Local}; -use std::time::{Duration, UNIX_EPOCH}; - -// = 3.12.0alpha -pub const MAJOR: usize = 3; -pub const MINOR: usize = 12; -pub const MICRO: usize = 0; -pub const RELEASELEVEL: &str = "alpha"; -pub const RELEASELEVEL_N: usize = 0xA; -pub const SERIAL: usize = 0; - -pub const VERSION_HEX: usize = - (MAJOR << 24) | (MINOR << 16) | (MICRO << 8) | (RELEASELEVEL_N << 4) | SERIAL; - -pub fn get_version() -> String { - format!( - "{:.80} ({:.80}) \n[{:.80}]", // \n is PyPy convention - get_version_number(), - get_build_info(), - get_compiler() - ) -} - -pub fn get_version_number() -> String { - format!("{MAJOR}.{MINOR}.{MICRO}{RELEASELEVEL}") -} - -pub fn get_winver_number() -> String { - format!("{MAJOR}.{MINOR}") -} - -pub fn get_compiler() -> String { - format!("rustc {}", env!("RUSTC_VERSION")) -} - -pub fn get_build_info() -> String { - // See: https://reproducible-builds.org/docs/timestamps/ - let git_revision = get_git_revision(); - let separator = if git_revision.is_empty() { "" } else { ":" }; - - let git_identifier = get_git_identifier(); - - format!( - "{id}{sep}{revision}, {date:.20}, {time:.9}", - id = if git_identifier.is_empty() { - "default".to_owned() - } else { - git_identifier - }, - sep = separator, - revision = git_revision, - date = get_git_date(), - time = get_git_time(), - ) -} - -pub fn get_git_revision() -> String { - option_env!("RUSTPYTHON_GIT_HASH").unwrap_or("").to_owned() -} - -pub fn get_git_tag() -> String { - option_env!("RUSTPYTHON_GIT_TAG").unwrap_or("").to_owned() -} - -pub fn get_git_branch() -> String { - option_env!("RUSTPYTHON_GIT_BRANCH") - .unwrap_or("") - .to_owned() -} - -pub fn get_git_identifier() -> String { - let git_tag = get_git_tag(); - let git_branch = get_git_branch(); - - if git_tag.is_empty() || git_tag == "undefined" { - git_branch - } else { - git_tag - } -} - -fn get_git_timestamp_datetime() -> DateTime<Local> { - let timestamp = option_env!("RUSTPYTHON_GIT_TIMESTAMP") - .unwrap_or("") - .to_owned(); - let timestamp = timestamp.parse::<u64>().unwrap_or(0); - - let datetime = UNIX_EPOCH + Duration::from_secs(timestamp); - - datetime.into() -} - -pub fn get_git_date() -> String { - let datetime = get_git_timestamp_datetime(); - - datetime.format("%b %e %Y").to_string() -} - -pub fn get_git_time() -> String { - let datetime = get_git_timestamp_datetime(); - - datetime.format("%H:%M:%S").to_string() -} - -pub fn get_git_datetime() -> String { - let date = get_git_date(); - let time = get_git_time(); - - format!("{date} {time}") -} diff --git a/vm/src/vm/compile.rs b/vm/src/vm/compile.rs deleted file mode 100644 index c44158f2098..00000000000 --- a/vm/src/vm/compile.rs +++ /dev/null @@ -1,106 +0,0 @@ -use crate::{ - builtins::{PyCode, PyDictRef}, - compiler::{self, CompileError, CompileOpts}, - convert::TryFromObject, - scope::Scope, - AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine, -}; - -impl VirtualMachine { - pub fn compile( - &self, - source: &str, - mode: compiler::Mode, - source_path: String, - ) -> Result<PyRef<PyCode>, CompileError> { - self.compile_with_opts(source, mode, source_path, self.compile_opts()) - } - - pub fn compile_with_opts( - &self, - source: &str, - mode: compiler::Mode, - source_path: String, - opts: CompileOpts, - ) -> Result<PyRef<PyCode>, CompileError> { - compiler::compile(source, mode, source_path, opts).map(|code| self.ctx.new_code(code)) - } - - pub fn run_script(&self, scope: Scope, path: &str) -> PyResult<()> { - if get_importer(path, self)?.is_some() { - self.insert_sys_path(self.new_pyobj(path))?; - let runpy = self.import("runpy", None, 0)?; - let run_module_as_main = runpy.get_attr("_run_module_as_main", self)?; - run_module_as_main.call((identifier!(self, __main__).to_owned(), false), self)?; - return Ok(()); - } - - let dir = std::path::Path::new(path) - .parent() - .unwrap() - .to_str() - .unwrap(); - self.insert_sys_path(self.new_pyobj(dir))?; - - match std::fs::read_to_string(path) { - Ok(source) => { - self.run_code_string(scope, &source, path.to_owned())?; - } - Err(err) => { - error!("Failed reading file '{}': {}", path, err); - // TODO: Need to change to ExitCode or Termination - std::process::exit(1); - } - } - Ok(()) - } - - pub fn run_code_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult { - let code_obj = self - .compile(source, compiler::Mode::Exec, source_path.clone()) - .map_err(|err| self.new_syntax_error(&err, Some(source)))?; - // trace!("Code object: {:?}", code_obj.borrow()); - scope.globals.set_item( - identifier!(self, __file__), - self.new_pyobj(source_path), - self, - )?; - self.run_code_obj(code_obj, scope) - } - - pub fn run_block_expr(&self, scope: Scope, source: &str) -> PyResult { - let code_obj = self - .compile(source, compiler::Mode::BlockExpr, "<embedded>".to_owned()) - .map_err(|err| self.new_syntax_error(&err, Some(source)))?; - // trace!("Code object: {:?}", code_obj.borrow()); - self.run_code_obj(code_obj, scope) - } -} - -fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> { - let path_importer_cache = vm.sys_module.get_attr("path_importer_cache", vm)?; - let path_importer_cache = PyDictRef::try_from_object(vm, path_importer_cache)?; - if let Some(importer) = path_importer_cache.get_item_opt(path, vm)? { - return Ok(Some(importer)); - } - let path = vm.ctx.new_str(path); - let path_hooks = vm.sys_module.get_attr("path_hooks", vm)?; - let mut importer = None; - let path_hooks: Vec<PyObjectRef> = path_hooks.try_into_value(vm)?; - for path_hook in path_hooks { - match path_hook.call((path.clone(),), vm) { - Ok(imp) => { - importer = Some(imp); - break; - } - Err(e) if e.fast_isinstance(vm.ctx.exceptions.import_error) => continue, - Err(e) => return Err(e), - } - } - Ok(if let Some(imp) = importer { - let imp = path_importer_cache.get_or_insert(vm, path.into(), || imp.clone())?; - Some(imp) - } else { - None - }) -} diff --git a/vm/src/vm/context.rs b/vm/src/vm/context.rs deleted file mode 100644 index 4181a65896c..00000000000 --- a/vm/src/vm/context.rs +++ /dev/null @@ -1,585 +0,0 @@ -use crate::{ - builtins::{ - bytes, - code::{self, PyCode}, - descriptor::{ - MemberGetter, MemberKind, MemberSetter, MemberSetterFunc, PyDescriptorOwned, - PyMemberDef, PyMemberDescriptor, - }, - getset::PyGetSet, - object, pystr, - type_::PyAttributes, - PyBaseException, PyBytes, PyComplex, PyDict, PyDictRef, PyEllipsis, PyFloat, PyFrozenSet, - PyInt, PyIntRef, PyList, PyListRef, PyNone, PyNotImplemented, PyStr, PyStrInterned, - PyTuple, PyTupleRef, PyType, PyTypeRef, - }, - class::{PyClassImpl, StaticType}, - common::rc::PyRc, - exceptions, - function::{ - HeapMethodDef, IntoPyGetterFunc, IntoPyNativeFn, IntoPySetterFunc, PyMethodDef, - PyMethodFlags, - }, - intern::{InternableString, MaybeInternedString, StringPool}, - object::{Py, PyObjectPayload, PyObjectRef, PyPayload, PyRef}, - types::{PyTypeFlags, PyTypeSlots, TypeZoo}, - PyResult, VirtualMachine, -}; -use malachite_bigint::BigInt; -use num_complex::Complex64; -use num_traits::ToPrimitive; -use rustpython_common::lock::PyRwLock; - -#[derive(Debug)] -pub struct Context { - pub true_value: PyIntRef, - pub false_value: PyIntRef, - pub none: PyRef<PyNone>, - pub empty_tuple: PyTupleRef, - pub empty_frozenset: PyRef<PyFrozenSet>, - pub empty_str: &'static PyStrInterned, - pub empty_bytes: PyRef<PyBytes>, - pub ellipsis: PyRef<PyEllipsis>, - pub not_implemented: PyRef<PyNotImplemented>, - - pub types: TypeZoo, - pub exceptions: exceptions::ExceptionZoo, - pub int_cache_pool: Vec<PyIntRef>, - // there should only be exact objects of str in here, no non-str objects and no subclasses - pub(crate) string_pool: StringPool, - pub(crate) slot_new_wrapper: PyMethodDef, - pub names: ConstName, -} - -macro_rules! declare_const_name { - ($($name:ident,)*) => { - #[derive(Debug, Clone, Copy)] - #[allow(non_snake_case)] - pub struct ConstName { - $(pub $name: &'static PyStrInterned,)* - } - - impl ConstName { - unsafe fn new(pool: &StringPool, typ: &PyTypeRef) -> Self { - Self { - $($name: pool.intern(stringify!($name), typ.clone()),)* - } - } - } - } -} - -declare_const_name! { - True, - False, - None, - NotImplemented, - Ellipsis, - - // magic methods - __abs__, - __abstractmethods__, - __add__, - __aenter__, - __aexit__, - __aiter__, - __alloc__, - __all__, - __and__, - __anext__, - __annotations__, - __args__, - __await__, - __bases__, - __bool__, - __build_class__, - __builtins__, - __bytes__, - __call__, - __ceil__, - __cformat__, - __class__, - __classcell__, - __class_getitem__, - __complex__, - __contains__, - __copy__, - __deepcopy__, - __del__, - __delattr__, - __delete__, - __delitem__, - __dict__, - __dir__, - __div__, - __divmod__, - __doc__, - __enter__, - __eq__, - __exit__, - __file__, - __float__, - __floor__, - __floordiv__, - __format__, - __fspath__, - __ge__, - __get__, - __getattr__, - __getattribute__, - __getformat__, - __getitem__, - __getnewargs__, - __gt__, - __hash__, - __iadd__, - __iand__, - __idiv__, - __ifloordiv__, - __ilshift__, - __imatmul__, - __imod__, - __import__, - __imul__, - __index__, - __init__, - __init_subclass__, - __instancecheck__, - __int__, - __invert__, - __ior__, - __ipow__, - __irshift__, - __isub__, - __iter__, - __itruediv__, - __ixor__, - __jit__, // RustPython dialect - __le__, - __len__, - __length_hint__, - __lshift__, - __lt__, - __main__, - __match_args__, - __matmul__, - __missing__, - __mod__, - __module__, - __mro_entries__, - __mul__, - __name__, - __ne__, - __neg__, - __new__, - __next__, - __or__, - __orig_bases__, - __orig_class__, - __origin__, - __parameters__, - __pos__, - __pow__, - __prepare__, - __qualname__, - __radd__, - __rand__, - __rdiv__, - __rdivmod__, - __reduce__, - __reduce_ex__, - __repr__, - __reversed__, - __rfloordiv__, - __rlshift__, - __rmatmul__, - __rmod__, - __rmul__, - __ror__, - __round__, - __rpow__, - __rrshift__, - __rshift__, - __rsub__, - __rtruediv__, - __rxor__, - __set__, - __setattr__, - __setitem__, - __setstate__, - __set_name__, - __slots__, - __str__, - __sub__, - __subclasscheck__, - __subclasshook__, - __subclasses__, - __sizeof__, - __truediv__, - __trunc__, - __xor__, - - // common names - _attributes, - _fields, - _showwarnmsg, - decode, - encode, - keys, - items, - values, - version, - update, - copy, - flush, - close, - WarningMessage, -} - -// Basic objects: -impl Context { - pub const INT_CACHE_POOL_RANGE: std::ops::RangeInclusive<i32> = (-5)..=256; - const INT_CACHE_POOL_MIN: i32 = *Self::INT_CACHE_POOL_RANGE.start(); - - pub fn genesis() -> &'static PyRc<Self> { - rustpython_common::static_cell! { - static CONTEXT: PyRc<Context>; - } - CONTEXT.get_or_init(|| PyRc::new(Self::init_genesis())) - } - - fn init_genesis() -> Self { - flame_guard!("init Context"); - let types = TypeZoo::init(); - let exceptions = exceptions::ExceptionZoo::init(); - - #[inline] - fn create_object<T: PyObjectPayload + PyPayload>( - payload: T, - cls: &'static Py<PyType>, - ) -> PyRef<T> { - PyRef::new_ref(payload, cls.to_owned(), None) - } - - let none = create_object(PyNone, PyNone::static_type()); - let ellipsis = create_object(PyEllipsis, PyEllipsis::static_type()); - let not_implemented = create_object(PyNotImplemented, PyNotImplemented::static_type()); - - let int_cache_pool = Self::INT_CACHE_POOL_RANGE - .map(|v| { - PyRef::new_ref( - PyInt::from(BigInt::from(v)), - types.int_type.to_owned(), - None, - ) - }) - .collect(); - - let true_value = create_object(PyInt::from(1), types.bool_type); - let false_value = create_object(PyInt::from(0), types.bool_type); - - let empty_tuple = create_object( - PyTuple::new_unchecked(Vec::new().into_boxed_slice()), - types.tuple_type, - ); - let empty_frozenset = PyRef::new_ref( - PyFrozenSet::default(), - types.frozenset_type.to_owned(), - None, - ); - - let string_pool = StringPool::default(); - let names = unsafe { ConstName::new(&string_pool, &types.str_type.to_owned()) }; - - let slot_new_wrapper = PyMethodDef { - name: names.__new__.as_str(), - func: PyType::__new__.into_func(), - flags: PyMethodFlags::METHOD, - doc: None, - }; - - let empty_str = unsafe { string_pool.intern("", types.str_type.to_owned()) }; - let empty_bytes = create_object(PyBytes::from(Vec::new()), types.bytes_type); - Context { - true_value, - false_value, - none, - empty_tuple, - empty_frozenset, - empty_str, - empty_bytes, - - ellipsis, - not_implemented, - - types, - exceptions, - int_cache_pool, - string_pool, - slot_new_wrapper, - names, - } - } - - pub fn intern_str<S: InternableString>(&self, s: S) -> &'static PyStrInterned { - unsafe { self.string_pool.intern(s, self.types.str_type.to_owned()) } - } - - pub fn interned_str<S: MaybeInternedString + ?Sized>( - &self, - s: &S, - ) -> Option<&'static PyStrInterned> { - self.string_pool.interned(s) - } - - #[inline(always)] - pub fn none(&self) -> PyObjectRef { - self.none.clone().into() - } - - #[inline(always)] - pub fn ellipsis(&self) -> PyObjectRef { - self.ellipsis.clone().into() - } - - #[inline(always)] - pub fn not_implemented(&self) -> PyObjectRef { - self.not_implemented.clone().into() - } - - // universal pyref constructor - pub fn new_pyref<T, P>(&self, value: T) -> PyRef<P> - where - T: Into<P>, - P: PyPayload, - { - value.into().into_ref(self) - } - - // shortcuts for common type - - #[inline] - pub fn new_int<T: Into<BigInt> + ToPrimitive>(&self, i: T) -> PyIntRef { - if let Some(i) = i.to_i32() { - if Self::INT_CACHE_POOL_RANGE.contains(&i) { - let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; - return self.int_cache_pool[inner_idx].clone(); - } - } - PyInt::from(i).into_ref(self) - } - - #[inline] - pub fn new_bigint(&self, i: &BigInt) -> PyIntRef { - if let Some(i) = i.to_i32() { - if Self::INT_CACHE_POOL_RANGE.contains(&i) { - let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; - return self.int_cache_pool[inner_idx].clone(); - } - } - PyInt::from(i.clone()).into_ref(self) - } - - #[inline] - pub fn new_float(&self, value: f64) -> PyRef<PyFloat> { - PyFloat::from(value).into_ref(self) - } - - #[inline] - pub fn new_complex(&self, value: Complex64) -> PyRef<PyComplex> { - PyComplex::from(value).into_ref(self) - } - - #[inline] - pub fn new_str(&self, s: impl Into<pystr::PyStr>) -> PyRef<PyStr> { - pystr::PyStr::new_ref(s, self) - } - - pub fn interned_or_new_str<S, M>(&self, s: S) -> PyRef<PyStr> - where - S: Into<PyStr> + AsRef<M>, - M: MaybeInternedString, - { - match self.interned_str(s.as_ref()) { - Some(s) => s.to_owned(), - None => self.new_str(s), - } - } - - #[inline] - pub fn new_bytes(&self, data: Vec<u8>) -> PyRef<bytes::PyBytes> { - bytes::PyBytes::new_ref(data, self) - } - - #[inline(always)] - pub fn new_bool(&self, b: bool) -> PyIntRef { - let value = if b { - &self.true_value - } else { - &self.false_value - }; - value.clone() - } - - #[inline(always)] - pub fn new_tuple(&self, elements: Vec<PyObjectRef>) -> PyTupleRef { - PyTuple::new_ref(elements, self) - } - - #[inline(always)] - pub fn new_list(&self, elements: Vec<PyObjectRef>) -> PyListRef { - PyList::new_ref(elements, self) - } - - #[inline(always)] - pub fn new_dict(&self) -> PyDictRef { - PyDict::new_ref(self) - } - - pub fn new_class( - &self, - module: Option<&str>, - name: &str, - base: PyTypeRef, - slots: PyTypeSlots, - ) -> PyTypeRef { - let mut attrs = PyAttributes::default(); - if let Some(module) = module { - attrs.insert(identifier!(self, __module__), self.new_str(module).into()); - }; - PyType::new_heap( - name, - vec![base], - attrs, - slots, - self.types.type_type.to_owned(), - self, - ) - .unwrap() - } - - pub fn new_exception_type( - &self, - module: &str, - name: &str, - bases: Option<Vec<PyTypeRef>>, - ) -> PyTypeRef { - let bases = if let Some(bases) = bases { - bases - } else { - vec![self.exceptions.exception_type.to_owned()] - }; - let mut attrs = PyAttributes::default(); - attrs.insert(identifier!(self, __module__), self.new_str(module).into()); - - let interned_name = self.intern_str(name); - PyType::new_heap( - name, - bases, - attrs, - PyTypeSlots { - name: interned_name.as_str(), - ..PyBaseException::make_slots() - }, - self.types.type_type.to_owned(), - self, - ) - .unwrap() - } - - pub fn new_method_def<F, FKind>( - &self, - name: &'static str, - f: F, - flags: PyMethodFlags, - doc: Option<&'static str>, - ) -> PyRef<HeapMethodDef> - where - F: IntoPyNativeFn<FKind>, - { - let def = PyMethodDef { - name, - func: f.into_func(), - flags, - doc, - }; - let payload = HeapMethodDef::new(def); - PyRef::new_ref(payload, self.types.method_def.to_owned(), None) - } - - #[inline] - pub fn new_member( - &self, - name: &str, - member_kind: MemberKind, - getter: fn(&VirtualMachine, PyObjectRef) -> PyResult, - setter: MemberSetterFunc, - class: &'static Py<PyType>, - ) -> PyRef<PyMemberDescriptor> { - let member_def = PyMemberDef { - name: name.to_owned(), - kind: member_kind, - getter: MemberGetter::Getter(getter), - setter: MemberSetter::Setter(setter), - doc: None, - }; - let member_descriptor = PyMemberDescriptor { - common: PyDescriptorOwned { - typ: class.to_owned(), - name: self.intern_str(name), - qualname: PyRwLock::new(None), - }, - member: member_def, - }; - member_descriptor.into_ref(self) - } - - pub fn new_readonly_getset<F, T>( - &self, - name: impl Into<String>, - class: &'static Py<PyType>, - f: F, - ) -> PyRef<PyGetSet> - where - F: IntoPyGetterFunc<T>, - { - let name = name.into(); - let getset = PyGetSet::new(name, class).with_get(f); - PyRef::new_ref(getset, self.types.getset_type.to_owned(), None) - } - - pub fn new_getset<G, S, T, U>( - &self, - name: impl Into<String>, - class: &'static Py<PyType>, - g: G, - s: S, - ) -> PyRef<PyGetSet> - where - G: IntoPyGetterFunc<T>, - S: IntoPySetterFunc<U>, - { - let name = name.into(); - let getset = PyGetSet::new(name, class).with_get(g).with_set(s); - PyRef::new_ref(getset, self.types.getset_type.to_owned(), None) - } - - pub fn new_base_object(&self, class: PyTypeRef, dict: Option<PyDictRef>) -> PyObjectRef { - debug_assert_eq!( - class.slots.flags.contains(PyTypeFlags::HAS_DICT), - dict.is_some() - ); - PyRef::new_ref(object::PyBaseObject, class, dict).into() - } - - pub fn new_code(&self, code: impl code::IntoCodeObject) -> PyRef<PyCode> { - let code = code.into_code_object(self); - PyRef::new_ref(PyCode { code }, self.types.code_type.to_owned(), None) - } -} - -impl AsRef<Context> for Context { - fn as_ref(&self) -> &Self { - self - } -} diff --git a/vm/src/vm/interpreter.rs b/vm/src/vm/interpreter.rs deleted file mode 100644 index 9fcd2ea1d2e..00000000000 --- a/vm/src/vm/interpreter.rs +++ /dev/null @@ -1,133 +0,0 @@ -use super::{setting::Settings, thread, Context, VirtualMachine}; -use crate::{ - stdlib::{atexit, sys}, - PyResult, -}; -use std::sync::atomic::Ordering; - -/// The general interface for the VM -/// -/// # Examples -/// Runs a simple embedded hello world program. -/// ``` -/// use rustpython_vm::Interpreter; -/// use rustpython_vm::compiler::Mode; -/// Interpreter::without_stdlib(Default::default()).enter(|vm| { -/// let scope = vm.new_scope_with_builtins(); -/// let source = r#"print("Hello World!")"#; -/// let code_obj = vm.compile( -/// source, -/// Mode::Exec, -/// "<embedded>".to_owned(), -/// ).map_err(|err| vm.new_syntax_error(&err, Some(source))).unwrap(); -/// vm.run_code_obj(code_obj, scope).unwrap(); -/// }); -/// ``` -pub struct Interpreter { - vm: VirtualMachine, -} - -impl Interpreter { - /// This is a bare unit to build up an interpreter without the standard library. - /// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterConfig`. - /// To create an interpreter without the `rustpython` crate, but only with `rustpython-vm`, - /// try to build one from the source code of `InterpreterConfig`. It will not be a one-liner but it also will not be too hard. - pub fn without_stdlib(settings: Settings) -> Self { - Self::with_init(settings, |_| {}) - } - - /// Create with initialize function taking mutable vm reference. - /// ``` - /// use rustpython_vm::Interpreter; - /// Interpreter::with_init(Default::default(), |vm| { - /// // put this line to add stdlib to the vm - /// // vm.add_native_modules(rustpython_stdlib::get_module_inits()); - /// }).enter(|vm| { - /// vm.run_code_string(vm.new_scope_with_builtins(), "print(1)", "<...>".to_owned()); - /// }); - /// ``` - pub fn with_init<F>(settings: Settings, init: F) -> Self - where - F: FnOnce(&mut VirtualMachine), - { - let ctx = Context::genesis(); - crate::types::TypeZoo::extend(ctx); - crate::exceptions::ExceptionZoo::extend(ctx); - let mut vm = VirtualMachine::new(settings, ctx.clone()); - init(&mut vm); - vm.initialize(); - Self { vm } - } - - pub fn enter<F, R>(&self, f: F) -> R - where - F: FnOnce(&VirtualMachine) -> R, - { - thread::enter_vm(&self.vm, || f(&self.vm)) - } - - pub fn run<F, R>(self, f: F) -> u8 - where - F: FnOnce(&VirtualMachine) -> PyResult<R>, - { - self.enter(|vm| { - let res = f(vm); - flush_std(vm); - - // See if any exception leaked out: - let exit_code = res - .map(|_| 0) - .map_err(|exc| vm.handle_exit_exception(exc)) - .unwrap_or_else(|code| code); - - atexit::_run_exitfuncs(vm); - - vm.state.finalizing.store(true, Ordering::Release); - - flush_std(vm); - - exit_code - }) - } -} - -pub(crate) fn flush_std(vm: &VirtualMachine) { - if let Ok(stdout) = sys::get_stdout(vm) { - let _ = vm.call_method(&stdout, identifier!(vm, flush).as_str(), ()); - } - if let Ok(stderr) = sys::get_stderr(vm) { - let _ = vm.call_method(&stderr, identifier!(vm, flush).as_str(), ()); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - builtins::{int, PyStr}, - PyObjectRef, - }; - use malachite_bigint::ToBigInt; - - #[test] - fn test_add_py_integers() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { - let a: PyObjectRef = vm.ctx.new_int(33_i32).into(); - let b: PyObjectRef = vm.ctx.new_int(12_i32).into(); - let res = vm._add(&a, &b).unwrap(); - let value = int::get_value(&res); - assert_eq!(*value, 45_i32.to_bigint().unwrap()); - }) - } - - #[test] - fn test_multiply_str() { - Interpreter::without_stdlib(Default::default()).enter(|vm| { - let a = vm.new_pyobj(crate::common::ascii!("Hello ")); - let b = vm.new_pyobj(4_i32); - let res = vm._mul(&a, &b).unwrap(); - let value = res.payload::<PyStr>().unwrap(); - assert_eq!(value.as_ref(), "Hello Hello Hello Hello ") - }) - } -} diff --git a/vm/src/vm/method.rs b/vm/src/vm/method.rs deleted file mode 100644 index 258b1e9473e..00000000000 --- a/vm/src/vm/method.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! This module will be replaced once #3100 is done -//! Do not expose this type to outside of this crate - -use super::VirtualMachine; -use crate::{ - builtins::{PyBaseObject, PyStr, PyStrInterned}, - function::IntoFuncArgs, - object::{AsObject, Py, PyObject, PyObjectRef, PyResult}, - types::PyTypeFlags, -}; - -#[derive(Debug)] -pub enum PyMethod { - Function { - target: PyObjectRef, - func: PyObjectRef, - }, - Attribute(PyObjectRef), -} - -impl PyMethod { - pub fn get(obj: PyObjectRef, name: &Py<PyStr>, vm: &VirtualMachine) -> PyResult<Self> { - let cls = obj.class(); - let getattro = cls.mro_find_map(|cls| cls.slots.getattro.load()).unwrap(); - if getattro as usize != PyBaseObject::getattro as usize { - return obj.get_attr(name, vm).map(Self::Attribute); - } - - // any correct method name is always interned already. - let interned_name = vm.ctx.interned_str(name); - let mut is_method = false; - - let cls_attr = match interned_name.and_then(|name| cls.get_attr(name)) { - Some(descr) => { - let descr_cls = descr.class(); - let descr_get = if descr_cls - .slots - .flags - .has_feature(PyTypeFlags::METHOD_DESCRIPTOR) - { - is_method = true; - None - } else { - let descr_get = descr_cls.mro_find_map(|cls| cls.slots.descr_get.load()); - if let Some(descr_get) = descr_get { - if descr_cls - .mro_find_map(|cls| cls.slots.descr_set.load()) - .is_some() - { - let cls = cls.to_owned().into(); - return descr_get(descr, Some(obj), Some(cls), vm).map(Self::Attribute); - } - } - descr_get - }; - Some((descr, descr_get)) - } - None => None, - }; - - if let Some(dict) = obj.dict() { - if let Some(attr) = dict.get_item_opt(name, vm)? { - return Ok(Self::Attribute(attr)); - } - } - - if let Some((attr, descr_get)) = cls_attr { - match descr_get { - None if is_method => Ok(Self::Function { - target: obj, - func: attr, - }), - Some(descr_get) => { - let cls = cls.to_owned().into(); - descr_get(attr, Some(obj), Some(cls), vm).map(Self::Attribute) - } - None => Ok(Self::Attribute(attr)), - } - } else if let Some(getter) = cls.get_attr(identifier!(vm, __getattr__)) { - getter.call((obj, name.to_owned()), vm).map(Self::Attribute) - } else { - let exc = vm.new_attribute_error(format!( - "'{}' object has no attribute '{}'", - cls.name(), - name - )); - vm.set_attribute_error_context(&exc, obj.clone(), name.to_owned()); - Err(exc) - } - } - - pub(crate) fn get_special<const DIRECT: bool>( - obj: &PyObject, - name: &'static PyStrInterned, - vm: &VirtualMachine, - ) -> PyResult<Option<Self>> { - let obj_cls = obj.class(); - let attr = if DIRECT { - obj_cls.get_direct_attr(name) - } else { - obj_cls.get_attr(name) - }; - let func = match attr { - Some(f) => f, - None => { - return Ok(None); - } - }; - let meth = if func - .class() - .slots - .flags - .has_feature(PyTypeFlags::METHOD_DESCRIPTOR) - { - Self::Function { - target: obj.to_owned(), - func, - } - } else { - let obj_cls = obj_cls.to_owned().into(); - let attr = vm - .call_get_descriptor_specific(&func, Some(obj.to_owned()), Some(obj_cls)) - .unwrap_or(Ok(func))?; - Self::Attribute(attr) - }; - Ok(Some(meth)) - } - - pub fn invoke(self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { - let (func, args) = match self { - PyMethod::Function { target, func } => (func, args.into_method_args(target, vm)), - PyMethod::Attribute(func) => (func, args.into_args(vm)), - }; - func.call(args, vm) - } - - #[allow(dead_code)] - pub fn invoke_ref(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { - let (func, args) = match self { - PyMethod::Function { target, func } => { - (func, args.into_method_args(target.clone(), vm)) - } - PyMethod::Attribute(func) => (func, args.into_args(vm)), - }; - func.call(args, vm) - } -} diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs deleted file mode 100644 index e943ab7fabb..00000000000 --- a/vm/src/vm/mod.rs +++ /dev/null @@ -1,935 +0,0 @@ -//! Implement virtual machine to run instructions. -//! -//! See also: -//! <https://github.com/ProgVal/pythonvm-rust/blob/master/src/processor/mod.rs> - -#[cfg(feature = "rustpython-compiler")] -mod compile; -mod context; -mod interpreter; -mod method; -mod setting; -pub mod thread; -mod vm_new; -mod vm_object; -mod vm_ops; - -use crate::{ - builtins::{ - code::PyCode, - pystr::AsPyStr, - tuple::{PyTuple, PyTupleTyped}, - PyBaseExceptionRef, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, PyStrRef, - PyTypeRef, - }, - codecs::CodecsRegistry, - common::{hash::HashSecret, lock::PyMutex, rc::PyRc}, - convert::ToPyObject, - frame::{ExecutionResult, Frame, FrameRef}, - frozen::FrozenModule, - function::{ArgMapping, FuncArgs, PySetterValue}, - import, - protocol::PyIterIter, - scope::Scope, - signal, stdlib, - warn::WarningsState, - AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, -}; -use crossbeam_utils::atomic::AtomicCell; -#[cfg(unix)] -use nix::{ - sys::signal::{kill, sigaction, SaFlags, SigAction, SigSet, Signal::SIGINT}, - unistd::getpid, -}; -use std::sync::atomic::AtomicBool; -use std::{ - borrow::Cow, - cell::{Cell, Ref, RefCell}, - collections::{HashMap, HashSet}, -}; - -pub use context::Context; -pub use interpreter::Interpreter; -pub(crate) use method::PyMethod; -pub use setting::Settings; - -// Objects are live when they are on stack, or referenced by a name (for now) - -/// Top level container of a python virtual machine. In theory you could -/// create more instances of this struct and have them operate fully isolated. -/// -/// To construct this, please refer to the [`Interpreter`](Interpreter) -pub struct VirtualMachine { - pub builtins: PyRef<PyModule>, - pub sys_module: PyRef<PyModule>, - pub ctx: PyRc<Context>, - pub frames: RefCell<Vec<FrameRef>>, - pub wasm_id: Option<String>, - exceptions: RefCell<ExceptionStack>, - pub import_func: PyObjectRef, - pub profile_func: RefCell<PyObjectRef>, - pub trace_func: RefCell<PyObjectRef>, - pub use_tracing: Cell<bool>, - pub recursion_limit: Cell<usize>, - pub(crate) signal_handlers: Option<Box<RefCell<[Option<PyObjectRef>; signal::NSIG]>>>, - pub(crate) signal_rx: Option<signal::UserSignalReceiver>, - pub repr_guards: RefCell<HashSet<usize>>, - pub state: PyRc<PyGlobalState>, - pub initialized: bool, - recursion_depth: Cell<usize>, -} - -#[derive(Debug, Default)] -struct ExceptionStack { - exc: Option<PyBaseExceptionRef>, - prev: Option<Box<ExceptionStack>>, -} - -pub struct PyGlobalState { - pub settings: Settings, - pub module_inits: stdlib::StdlibMap, - pub frozen: HashMap<&'static str, FrozenModule, ahash::RandomState>, - pub stacksize: AtomicCell<usize>, - pub thread_count: AtomicCell<usize>, - pub hash_secret: HashSecret, - pub atexit_funcs: PyMutex<Vec<(PyObjectRef, FuncArgs)>>, - pub codec_registry: CodecsRegistry, - pub finalizing: AtomicBool, - pub warnings: WarningsState, - pub override_frozen_modules: AtomicCell<isize>, - pub before_forkers: PyMutex<Vec<PyObjectRef>>, - pub after_forkers_child: PyMutex<Vec<PyObjectRef>>, - pub after_forkers_parent: PyMutex<Vec<PyObjectRef>>, - pub int_max_str_digits: AtomicCell<usize>, -} - -pub fn process_hash_secret_seed() -> u32 { - use once_cell::sync::OnceCell; - static SEED: OnceCell<u32> = OnceCell::new(); - *SEED.get_or_init(rand::random) -} - -impl VirtualMachine { - /// Create a new `VirtualMachine` structure. - fn new(settings: Settings, ctx: PyRc<Context>) -> VirtualMachine { - flame_guard!("new VirtualMachine"); - - // make a new module without access to the vm; doesn't - // set __spec__, __loader__, etc. attributes - let new_module = |def| { - PyRef::new_ref( - PyModule::from_def(def), - ctx.types.module_type.to_owned(), - Some(ctx.new_dict()), - ) - }; - - // Hard-core modules: - let builtins = new_module(stdlib::builtins::__module_def(&ctx)); - let sys_module = new_module(stdlib::sys::__module_def(&ctx)); - - let import_func = ctx.none(); - let profile_func = RefCell::new(ctx.none()); - let trace_func = RefCell::new(ctx.none()); - // hack to get around const array repeat expressions, rust issue #79270 - const NONE: Option<PyObjectRef> = None; - // putting it in a const optimizes better, prevents linear initialization of the array - #[allow(clippy::declare_interior_mutable_const)] - const SIGNAL_HANDLERS: RefCell<[Option<PyObjectRef>; signal::NSIG]> = - RefCell::new([NONE; signal::NSIG]); - let signal_handlers = Some(Box::new(SIGNAL_HANDLERS)); - - let module_inits = stdlib::get_module_inits(); - - let seed = match settings.hash_seed { - Some(seed) => seed, - None => process_hash_secret_seed(), - }; - let hash_secret = HashSecret::new(seed); - - let codec_registry = CodecsRegistry::new(&ctx); - - let warnings = WarningsState::init_state(&ctx); - - let int_max_str_digits = AtomicCell::new(match settings.int_max_str_digits { - -1 => 4300, - other => other, - } as usize); - let mut vm = VirtualMachine { - builtins, - sys_module, - ctx, - frames: RefCell::new(vec![]), - wasm_id: None, - exceptions: RefCell::default(), - import_func, - profile_func, - trace_func, - use_tracing: Cell::new(false), - recursion_limit: Cell::new(if cfg!(debug_assertions) { 256 } else { 1000 }), - signal_handlers, - signal_rx: None, - repr_guards: RefCell::default(), - state: PyRc::new(PyGlobalState { - settings, - module_inits, - frozen: HashMap::default(), - stacksize: AtomicCell::new(0), - thread_count: AtomicCell::new(0), - hash_secret, - atexit_funcs: PyMutex::default(), - codec_registry, - finalizing: AtomicBool::new(false), - warnings, - override_frozen_modules: AtomicCell::new(0), - before_forkers: PyMutex::default(), - after_forkers_child: PyMutex::default(), - after_forkers_parent: PyMutex::default(), - int_max_str_digits, - }), - initialized: false, - recursion_depth: Cell::new(0), - }; - - if vm.state.hash_secret.hash_str("") - != vm - .ctx - .interned_str("") - .expect("empty str must be interned") - .hash(&vm) - { - panic!("Interpreters in same process must share the hash seed"); - } - - let frozen = core_frozen_inits().collect(); - PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen; - - vm.builtins.init_dict( - vm.ctx.intern_str("builtins"), - Some(vm.ctx.intern_str(stdlib::builtins::DOC.unwrap()).to_owned()), - &vm, - ); - vm.sys_module.init_dict( - vm.ctx.intern_str("sys"), - Some(vm.ctx.intern_str(stdlib::sys::DOC.unwrap()).to_owned()), - &vm, - ); - // let name = vm.sys_module.get_attr("__name__", &vm).unwrap(); - vm - } - - /// set up the encodings search function - /// init_importlib must be called before this call - #[cfg(feature = "encodings")] - fn import_encodings(&mut self) -> PyResult<()> { - self.import("encodings", None, 0).map_err(|import_err| { - let rustpythonpath_env = std::env::var("RUSTPYTHONPATH").ok(); - let pythonpath_env = std::env::var("PYTHONPATH").ok(); - let env_set = rustpythonpath_env.as_ref().is_some() || pythonpath_env.as_ref().is_some(); - let path_contains_env = self.state.settings.path_list.iter().any(|s| { - Some(s.as_str()) == rustpythonpath_env.as_deref() || Some(s.as_str()) == pythonpath_env.as_deref() - }); - - let guide_message = if !env_set { - "Neither RUSTPYTHONPATH nor PYTHONPATH is set. Try setting one of them to the stdlib directory." - } else if path_contains_env { - "RUSTPYTHONPATH or PYTHONPATH is set, but it doesn't contain the encodings library. If you are customizing the RustPython vm/interpreter, try adding the stdlib directory to the path. If you are developing the RustPython interpreter, it might be a bug during development." - } else { - "RUSTPYTHONPATH or PYTHONPATH is set, but it wasn't loaded to `Settings::path_list`. If you are going to customize the RustPython vm/interpreter, those environment variables are not loaded in the Settings struct by default. Please try creating a customized instance of the Settings struct. If you are developing the RustPython interpreter, it might be a bug during development." - }; - - let msg = format!( - "RustPython could not import the encodings module. It usually means something went wrong. Please carefully read the following messages and follow the steps.\n\ - \n\ - {guide_message}\n\ - If you don't have access to a consistent external environment (e.g. targeting wasm, embedding \ - rustpython in another application), try enabling the `freeze-stdlib` feature.\n\ - If this is intended and you want to exclude the encodings module from your interpreter, please remove the `encodings` feature from `rustpython-vm` crate." - ); - - let err = self.new_runtime_error(msg); - err.set_cause(Some(import_err)); - err - })?; - Ok(()) - } - - fn import_utf8_encodings(&mut self) -> PyResult<()> { - import::import_frozen(self, "codecs")?; - // FIXME: See corresponding part of `core_frozen_inits` - // let encoding_module_name = if cfg!(feature = "freeze-stdlib") { - // "encodings.utf_8" - // } else { - // "encodings_utf_8" - // }; - let encoding_module_name = "encodings_utf_8"; - let encoding_module = import::import_frozen(self, encoding_module_name)?; - let getregentry = encoding_module.get_attr("getregentry", self)?; - let codec_info = getregentry.call((), self)?; - self.state - .codec_registry - .register_manual("utf-8", codec_info.try_into_value(self)?)?; - Ok(()) - } - - fn initialize(&mut self) { - flame_guard!("init VirtualMachine"); - - if self.initialized { - panic!("Double Initialize Error"); - } - - stdlib::builtins::init_module(self, &self.builtins); - stdlib::sys::init_module(self, &self.sys_module, &self.builtins); - - let mut essential_init = || -> PyResult { - #[cfg(not(target_arch = "wasm32"))] - import::import_builtin(self, "_signal")?; - #[cfg(any(feature = "parser", feature = "compiler"))] - import::import_builtin(self, "_ast")?; - #[cfg(not(feature = "threading"))] - import::import_frozen(self, "_thread")?; - let importlib = import::init_importlib_base(self)?; - self.import_utf8_encodings()?; - - #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - { - // this isn't fully compatible with CPython; it imports "io" and sets - // builtins.open to io.OpenWrapper, but this is easier, since it doesn't - // require the Python stdlib to be present - let io = import::import_builtin(self, "_io")?; - let set_stdio = |name, fd, mode: &str| { - let stdio = crate::stdlib::io::open( - self.ctx.new_int(fd).into(), - Some(mode), - Default::default(), - self, - )?; - let dunder_name = self.ctx.intern_str(format!("__{name}__")); - self.sys_module.set_attr( - dunder_name, // e.g. __stdin__ - stdio.clone(), - self, - )?; - self.sys_module.set_attr(name, stdio, self)?; - Ok(()) - }; - set_stdio("stdin", 0, "r")?; - set_stdio("stdout", 1, "w")?; - set_stdio("stderr", 2, "w")?; - - let io_open = io.get_attr("open", self)?; - self.builtins.set_attr("open", io_open, self)?; - } - - Ok(importlib) - }; - - let res = essential_init(); - let importlib = self.expect_pyresult(res, "essential initialization failed"); - - if self.state.settings.allow_external_library && cfg!(feature = "rustpython-compiler") { - if let Err(e) = import::init_importlib_package(self, importlib) { - eprintln!("importlib initialization failed. This is critical for many complicated packages."); - self.print_exception(e); - } - } - - #[cfg(feature = "encodings")] - if cfg!(feature = "freeze-stdlib") || !self.state.settings.path_list.is_empty() { - if let Err(e) = self.import_encodings() { - eprintln!( - "encodings initialization failed. Only utf-8 encoding will be supported." - ); - self.print_exception(e); - } - } else { - // Here may not be the best place to give general `path_list` advice, - // but bare rustpython_vm::VirtualMachine users skipped proper settings must hit here while properly setup vm never enters here. - eprintln!( - "feature `encodings` is enabled but `settings.path_list` is empty. \ - Please add the library path to `settings.path_list`. If you intended to disable the entire standard library (including the `encodings` feature), please also make sure to disable the `encodings` feature.\n\ - Tip: You may also want to add `\"\"` to `settings.path_list` in order to enable importing from the current working directory." - ); - } - - self.initialized = true; - } - - fn state_mut(&mut self) -> &mut PyGlobalState { - PyRc::get_mut(&mut self.state) - .expect("there should not be multiple threads while a user has a mut ref to a vm") - } - - /// Can only be used in the initialization closure passed to [`Interpreter::with_init`] - pub fn add_native_module<S>(&mut self, name: S, module: stdlib::StdlibInitFunc) - where - S: Into<Cow<'static, str>>, - { - self.state_mut().module_inits.insert(name.into(), module); - } - - pub fn add_native_modules<I>(&mut self, iter: I) - where - I: IntoIterator<Item = (Cow<'static, str>, stdlib::StdlibInitFunc)>, - { - self.state_mut().module_inits.extend(iter); - } - - /// Can only be used in the initialization closure passed to [`Interpreter::with_init`] - pub fn add_frozen<I>(&mut self, frozen: I) - where - I: IntoIterator<Item = (&'static str, FrozenModule)>, - { - self.state_mut().frozen.extend(frozen); - } - - /// Set the custom signal channel for the interpreter - pub fn set_user_signal_channel(&mut self, signal_rx: signal::UserSignalReceiver) { - self.signal_rx = Some(signal_rx); - } - - pub fn run_code_obj(&self, code: PyRef<PyCode>, scope: Scope) -> PyResult { - let frame = Frame::new(code, scope, self.builtins.dict(), &[], self).into_ref(&self.ctx); - self.run_frame(frame) - } - - #[cold] - pub fn run_unraisable(&self, e: PyBaseExceptionRef, msg: Option<String>, object: PyObjectRef) { - let sys_module = self.import("sys", None, 0).unwrap(); - let unraisablehook = sys_module.get_attr("unraisablehook", self).unwrap(); - - let exc_type = e.class().to_owned(); - let exc_traceback = e.traceback().to_pyobject(self); // TODO: actual traceback - let exc_value = e.into(); - let args = stdlib::sys::UnraisableHookArgs { - exc_type, - exc_value, - exc_traceback, - err_msg: self.new_pyobj(msg), - object, - }; - if let Err(e) = unraisablehook.call((args,), self) { - println!("{}", e.as_object().repr(self).unwrap().as_str()); - } - } - - #[inline(always)] - pub fn run_frame(&self, frame: FrameRef) -> PyResult { - match self.with_frame(frame, |f| f.run(self))? { - ExecutionResult::Return(value) => Ok(value), - _ => panic!("Got unexpected result from function"), - } - } - - pub fn current_recursion_depth(&self) -> usize { - self.recursion_depth.get() - } - - /// Used to run the body of a (possibly) recursive function. It will raise a - /// RecursionError if recursive functions are nested far too many times, - /// preventing a stack overflow. - pub fn with_recursion<R, F: FnOnce() -> PyResult<R>>(&self, _where: &str, f: F) -> PyResult<R> { - self.check_recursive_call(_where)?; - self.recursion_depth.set(self.recursion_depth.get() + 1); - let result = f(); - self.recursion_depth.set(self.recursion_depth.get() - 1); - result - } - - pub fn with_frame<R, F: FnOnce(FrameRef) -> PyResult<R>>( - &self, - frame: FrameRef, - f: F, - ) -> PyResult<R> { - self.with_recursion("", || { - self.frames.borrow_mut().push(frame.clone()); - let result = f(frame); - // defer dec frame - let _popped = self.frames.borrow_mut().pop(); - result - }) - } - - /// Returns a basic CompileOpts instance with options accurate to the vm. Used - /// as the CompileOpts for `vm.compile()`. - #[cfg(feature = "rustpython-codegen")] - pub fn compile_opts(&self) -> crate::compiler::CompileOpts { - crate::compiler::CompileOpts { - optimize: self.state.settings.optimize, - } - } - - // To be called right before raising the recursion depth. - fn check_recursive_call(&self, _where: &str) -> PyResult<()> { - if self.recursion_depth.get() >= self.recursion_limit.get() { - Err(self.new_recursion_error(format!("maximum recursion depth exceeded {_where}"))) - } else { - Ok(()) - } - } - - pub fn current_frame(&self) -> Option<Ref<FrameRef>> { - let frames = self.frames.borrow(); - if frames.is_empty() { - None - } else { - Some(Ref::map(self.frames.borrow(), |frames| { - frames.last().unwrap() - })) - } - } - - pub fn current_locals(&self) -> PyResult<ArgMapping> { - self.current_frame() - .expect("called current_locals but no frames on the stack") - .locals(self) - } - - pub fn current_globals(&self) -> Ref<PyDictRef> { - let frame = self - .current_frame() - .expect("called current_globals but no frames on the stack"); - Ref::map(frame, |f| &f.globals) - } - - pub fn try_class(&self, module: &'static str, class: &'static str) -> PyResult<PyTypeRef> { - let class = self - .import(module, None, 0)? - .get_attr(class, self)? - .downcast() - .expect("not a class"); - Ok(class) - } - - pub fn class(&self, module: &'static str, class: &'static str) -> PyTypeRef { - let module = self - .import(module, None, 0) - .unwrap_or_else(|_| panic!("unable to import {module}")); - - let class = module - .get_attr(class, self) - .unwrap_or_else(|_| panic!("module {module:?} has no class {class}")); - class.downcast().expect("not a class") - } - - #[inline] - pub fn import<'a>( - &self, - module_name: impl AsPyStr<'a>, - from_list: Option<PyTupleTyped<PyStrRef>>, - level: usize, - ) -> PyResult { - let module_name = module_name.as_pystr(&self.ctx); - self.import_inner(module_name, from_list, level) - } - - fn import_inner( - &self, - module: &Py<PyStr>, - from_list: Option<PyTupleTyped<PyStrRef>>, - level: usize, - ) -> PyResult { - // if the import inputs seem weird, e.g a package import or something, rather than just - // a straight `import ident` - let weird = module.as_str().contains('.') - || level != 0 - || from_list.as_ref().map_or(false, |x| !x.is_empty()); - - let cached_module = if weird { - None - } else { - let sys_modules = self.sys_module.get_attr("modules", self)?; - sys_modules.get_item(module, self).ok() - }; - - match cached_module { - Some(cached_module) => { - if self.is_none(&cached_module) { - Err(self.new_import_error( - format!("import of {module} halted; None in sys.modules"), - module.to_owned(), - )) - } else { - Ok(cached_module) - } - } - None => { - let import_func = self - .builtins - .get_attr(identifier!(self, __import__), self) - .map_err(|_| { - self.new_import_error("__import__ not found".to_owned(), module.to_owned()) - })?; - - let (locals, globals) = if let Some(frame) = self.current_frame() { - (Some(frame.locals.clone()), Some(frame.globals.clone())) - } else { - (None, None) - }; - let from_list = match from_list { - Some(tup) => tup.to_pyobject(self), - None => self.new_tuple(()).into(), - }; - import_func - .call((module.to_owned(), globals, locals, from_list, level), self) - .map_err(|exc| import::remove_importlib_frames(self, &exc)) - } - } - } - - pub fn extract_elements_with<T, F>(&self, value: &PyObject, func: F) -> PyResult<Vec<T>> - where - F: Fn(PyObjectRef) -> PyResult<T>, - { - // Extract elements from item, if possible: - let cls = value.class(); - let list_borrow; - let slice = if cls.is(self.ctx.types.tuple_type) { - value.payload::<PyTuple>().unwrap().as_slice() - } else if cls.is(self.ctx.types.list_type) { - list_borrow = value.payload::<PyList>().unwrap().borrow_vec(); - &list_borrow - } else { - return self.map_pyiter(value, func); - }; - slice.iter().map(|obj| func(obj.clone())).collect() - } - - pub fn map_iterable_object<F, R>(&self, obj: &PyObject, mut f: F) -> PyResult<PyResult<Vec<R>>> - where - F: FnMut(PyObjectRef) -> PyResult<R>, - { - match_class!(match obj { - ref l @ PyList => { - let mut i: usize = 0; - let mut results = Vec::with_capacity(l.borrow_vec().len()); - loop { - let elem = { - let elements = &*l.borrow_vec(); - if i >= elements.len() { - results.shrink_to_fit(); - return Ok(Ok(results)); - } else { - elements[i].clone() - } - // free the lock - }; - match f(elem) { - Ok(result) => results.push(result), - Err(err) => return Ok(Err(err)), - } - i += 1; - } - } - ref t @ PyTuple => Ok(t.iter().cloned().map(f).collect()), - // TODO: put internal iterable type - obj => { - Ok(self.map_pyiter(obj, f)) - } - }) - } - - fn map_pyiter<F, R>(&self, value: &PyObject, mut f: F) -> PyResult<Vec<R>> - where - F: FnMut(PyObjectRef) -> PyResult<R>, - { - let iter = value.to_owned().get_iter(self)?; - let cap = match self.length_hint_opt(value.to_owned()) { - Err(e) if e.class().is(self.ctx.exceptions.runtime_error) => return Err(e), - Ok(Some(value)) => Some(value), - // Use a power of 2 as a default capacity. - _ => None, - }; - // TODO: fix extend to do this check (?), see test_extend in Lib/test/list_tests.py, - // https://github.com/python/cpython/blob/v3.9.0/Objects/listobject.c#L922-L928 - if let Some(cap) = cap { - if cap >= isize::max_value() as usize { - return Ok(Vec::new()); - } - } - - let mut results = PyIterIter::new(self, iter.as_ref(), cap) - .map(|element| f(element?)) - .collect::<PyResult<Vec<_>>>()?; - results.shrink_to_fit(); - Ok(results) - } - - pub fn get_attribute_opt<'a>( - &self, - obj: PyObjectRef, - attr_name: impl AsPyStr<'a>, - ) -> PyResult<Option<PyObjectRef>> { - let attr_name = attr_name.as_pystr(&self.ctx); - match obj.get_attr_inner(attr_name, self) { - Ok(attr) => Ok(Some(attr)), - Err(e) if e.fast_isinstance(self.ctx.exceptions.attribute_error) => Ok(None), - Err(e) => Err(e), - } - } - - pub fn set_attribute_error_context( - &self, - exc: &PyBaseExceptionRef, - obj: PyObjectRef, - name: PyStrRef, - ) { - if exc.class().is(self.ctx.exceptions.attribute_error) { - let exc = exc.as_object(); - exc.set_attr("name", name, self).unwrap(); - exc.set_attr("obj", obj, self).unwrap(); - } - } - - // get_method should be used for internal access to magic methods (by-passing - // the full getattribute look-up. - pub fn get_method_or_type_error<F>( - &self, - obj: PyObjectRef, - method_name: &'static PyStrInterned, - err_msg: F, - ) -> PyResult - where - F: FnOnce() -> String, - { - let method = obj - .class() - .get_attr(method_name) - .ok_or_else(|| self.new_type_error(err_msg()))?; - self.call_if_get_descriptor(&method, obj) - } - - // TODO: remove + transfer over to get_special_method - pub(crate) fn get_method( - &self, - obj: PyObjectRef, - method_name: &'static PyStrInterned, - ) -> Option<PyResult> { - let method = obj.get_class_attr(method_name)?; - Some(self.call_if_get_descriptor(&method, obj)) - } - - pub(crate) fn get_str_method(&self, obj: PyObjectRef, method_name: &str) -> Option<PyResult> { - let method_name = self.ctx.interned_str(method_name)?; - self.get_method(obj, method_name) - } - - #[inline] - /// Checks for triggered signals and calls the appropriate handlers. A no-op on - /// platforms where signals are not supported. - pub fn check_signals(&self) -> PyResult<()> { - #[cfg(not(target_arch = "wasm32"))] - { - crate::signal::check_signals(self) - } - #[cfg(target_arch = "wasm32")] - { - Ok(()) - } - } - - pub(crate) fn push_exception(&self, exc: Option<PyBaseExceptionRef>) { - let mut excs = self.exceptions.borrow_mut(); - let prev = std::mem::take(&mut *excs); - excs.prev = Some(Box::new(prev)); - excs.exc = exc - } - - pub(crate) fn pop_exception(&self) -> Option<PyBaseExceptionRef> { - let mut excs = self.exceptions.borrow_mut(); - let cur = std::mem::take(&mut *excs); - *excs = *cur.prev.expect("pop_exception() without nested exc stack"); - cur.exc - } - - pub(crate) fn take_exception(&self) -> Option<PyBaseExceptionRef> { - self.exceptions.borrow_mut().exc.take() - } - - pub(crate) fn current_exception(&self) -> Option<PyBaseExceptionRef> { - self.exceptions.borrow().exc.clone() - } - - pub(crate) fn set_exception(&self, exc: Option<PyBaseExceptionRef>) { - // don't be holding the RefCell guard while __del__ is called - let prev = std::mem::replace(&mut self.exceptions.borrow_mut().exc, exc); - drop(prev); - } - - pub(crate) fn contextualize_exception(&self, exception: &PyBaseExceptionRef) { - if let Some(context_exc) = self.topmost_exception() { - if !context_exc.is(exception) { - let mut o = context_exc.clone(); - while let Some(context) = o.context() { - if context.is(exception) { - o.set_context(None); - break; - } - o = context; - } - exception.set_context(Some(context_exc)) - } - } - } - - pub(crate) fn topmost_exception(&self) -> Option<PyBaseExceptionRef> { - let excs = self.exceptions.borrow(); - let mut cur = &*excs; - loop { - if let Some(exc) = &cur.exc { - return Some(exc.clone()); - } - cur = cur.prev.as_deref()?; - } - } - - pub fn handle_exit_exception(&self, exc: PyBaseExceptionRef) -> u8 { - if exc.fast_isinstance(self.ctx.exceptions.system_exit) { - let args = exc.args(); - let msg = match args.as_slice() { - [] => return 0, - [arg] => match_class!(match arg { - ref i @ PyInt => { - use num_traits::cast::ToPrimitive; - return i.as_bigint().to_u8().unwrap_or(0); - } - arg => { - if self.is_none(arg) { - return 0; - } else { - arg.str(self).ok() - } - } - }), - _ => args.as_object().repr(self).ok(), - }; - if let Some(msg) = msg { - let stderr = stdlib::sys::PyStderr(self); - writeln!(stderr, "{msg}"); - } - 1 - } else if exc.fast_isinstance(self.ctx.exceptions.keyboard_interrupt) { - #[allow(clippy::if_same_then_else)] - { - self.print_exception(exc); - #[cfg(unix)] - { - let action = SigAction::new( - nix::sys::signal::SigHandler::SigDfl, - SaFlags::SA_ONSTACK, - SigSet::empty(), - ); - let result = unsafe { sigaction(SIGINT, &action) }; - if result.is_ok() { - interpreter::flush_std(self); - kill(getpid(), SIGINT).expect("Expect to be killed."); - } - - (libc::SIGINT as u8) + 128u8 - } - #[cfg(not(unix))] - { - 1 - } - } - } else { - self.print_exception(exc); - 1 - } - } - - #[doc(hidden)] - pub fn __module_set_attr( - &self, - module: &Py<PyModule>, - attr_name: &'static PyStrInterned, - attr_value: impl Into<PyObjectRef>, - ) -> PyResult<()> { - let val = attr_value.into(); - module - .as_object() - .generic_setattr(attr_name, PySetterValue::Assign(val), self) - } - - pub fn insert_sys_path(&self, obj: PyObjectRef) -> PyResult<()> { - let sys_path = self.sys_module.get_attr("path", self).unwrap(); - self.call_method(&sys_path, "insert", (0, obj))?; - Ok(()) - } - - pub fn run_module(&self, module: &str) -> PyResult<()> { - let runpy = self.import("runpy", None, 0)?; - let run_module_as_main = runpy.get_attr("_run_module_as_main", self)?; - run_module_as_main.call((module,), self)?; - Ok(()) - } -} - -impl AsRef<Context> for VirtualMachine { - fn as_ref(&self) -> &Context { - &self.ctx - } -} - -fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> { - let iter = std::iter::empty(); - macro_rules! ext_modules { - ($iter:ident, $($t:tt)*) => { - let $iter = $iter.chain(py_freeze!($($t)*)); - }; - } - - // keep as example but use file one now - // ext_modules!( - // iter, - // source = "initialized = True; print(\"Hello world!\")\n", - // module_name = "__hello__", - // ); - - // Python modules that the vm calls into, but are not actually part of the stdlib. They could - // in theory be implemented in Rust, but are easiest to do in Python for one reason or another. - // Includes _importlib_bootstrap and _importlib_bootstrap_external - ext_modules!( - iter, - dir = "./Lib/python_builtins", - crate_name = "rustpython_compiler_core" - ); - - // core stdlib Python modules that the vm calls into, but are still used in Python - // application code, e.g. copyreg - // FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules. - // We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterConfig::new().init_stdlib().interpreter()` - // #[cfg(not(feature = "freeze-stdlib"))] - ext_modules!( - iter, - dir = "./Lib/core_modules", - crate_name = "rustpython_compiler_core" - ); - - iter -} - -#[test] -fn test_nested_frozen() { - use rustpython_vm as vm; - - vm::Interpreter::with_init(Default::default(), |vm| { - // vm.add_native_modules(rustpython_stdlib::get_module_inits()); - vm.add_frozen(rustpython_vm::py_freeze!(dir = "../extra_tests/snippets")); - }) - .enter(|vm| { - let scope = vm.new_scope_with_builtins(); - - let source = "from dir_module.dir_module_inner import value2"; - let code_obj = vm - .compile(source, vm::compiler::Mode::Exec, "<embedded>".to_owned()) - .map_err(|err| vm.new_syntax_error(&err, Some(source))) - .unwrap(); - - if let Err(e) = vm.run_code_obj(code_obj, scope) { - vm.print_exception(e); - panic!(); - } - }) -} diff --git a/vm/src/vm/setting.rs b/vm/src/vm/setting.rs deleted file mode 100644 index a30c75560c8..00000000000 --- a/vm/src/vm/setting.rs +++ /dev/null @@ -1,134 +0,0 @@ -#[cfg(feature = "flame-it")] -use std::ffi::OsString; - -/// Struct containing all kind of settings for the python vm. -#[non_exhaustive] -pub struct Settings { - /// -d command line switch - pub debug: bool, - - /// -i - pub inspect: bool, - - /// -i, with no script - pub interactive: bool, - - /// -O optimization switch counter - pub optimize: u8, - - /// Not set SIGINT handler(i.e. for embedded mode) - pub no_sig_int: bool, - - /// -s - pub no_user_site: bool, - - /// -S - pub no_site: bool, - - /// -E - pub ignore_environment: bool, - - /// verbosity level (-v switch) - pub verbose: u8, - - /// -q - pub quiet: bool, - - /// -B - pub dont_write_bytecode: bool, - - /// -P - pub safe_path: bool, - - /// -b - pub bytes_warning: u64, - - /// -Xfoo[=bar] - pub xopts: Vec<(String, Option<String>)>, - - /// -X int_max_str_digits - pub int_max_str_digits: i64, - - /// -I - pub isolated: bool, - - /// -Xdev - pub dev_mode: bool, - - /// -X warn_default_encoding, PYTHONWARNDEFAULTENCODING - pub warn_default_encoding: bool, - - /// -Wfoo - pub warnopts: Vec<String>, - - /// Environment PYTHONPATH and RUSTPYTHONPATH: - pub path_list: Vec<String>, - - /// sys.argv - pub argv: Vec<String>, - - /// PYTHONHASHSEED=x - pub hash_seed: Option<u32>, - - /// -u, PYTHONUNBUFFERED=x - // TODO: use this; can TextIOWrapper even work with a non-buffered? - pub stdio_unbuffered: bool, - - /// --check-hash-based-pycs - pub check_hash_based_pycs: String, - - /// false for wasm. Not a command-line option - pub allow_external_library: bool, - - pub utf8_mode: u8, - - #[cfg(feature = "flame-it")] - pub profile_output: Option<OsString>, - #[cfg(feature = "flame-it")] - pub profile_format: Option<String>, -} - -impl Settings { - pub fn with_path(mut self, path: String) -> Self { - self.path_list.push(path); - self - } -} - -/// Sensible default settings. -impl Default for Settings { - fn default() -> Self { - Settings { - debug: false, - inspect: false, - interactive: false, - optimize: 0, - no_sig_int: false, - no_user_site: false, - no_site: false, - ignore_environment: false, - verbose: 0, - quiet: false, - dont_write_bytecode: false, - safe_path: false, - bytes_warning: 0, - xopts: vec![], - isolated: false, - dev_mode: false, - warn_default_encoding: false, - warnopts: vec![], - path_list: vec![], - argv: vec![], - hash_seed: None, - stdio_unbuffered: false, - check_hash_based_pycs: "default".to_owned(), - allow_external_library: cfg!(feature = "importlib"), - utf8_mode: 1, - int_max_str_digits: -1, - #[cfg(feature = "flame-it")] - profile_output: None, - #[cfg(feature = "flame-it")] - profile_format: None, - } - } -} diff --git a/vm/src/vm/thread.rs b/vm/src/vm/thread.rs deleted file mode 100644 index 4460c39cd72..00000000000 --- a/vm/src/vm/thread.rs +++ /dev/null @@ -1,167 +0,0 @@ -use crate::{AsObject, PyObject, VirtualMachine}; -use itertools::Itertools; -use std::{ - cell::RefCell, - ptr::{null, NonNull}, - thread_local, -}; - -thread_local! { - pub(super) static VM_STACK: RefCell<Vec<NonNull<VirtualMachine>>> = Vec::with_capacity(1).into(); - static VM_CURRENT: RefCell<*const VirtualMachine> = null::<VirtualMachine>().into(); -} - -pub fn with_current_vm<R>(f: impl FnOnce(&VirtualMachine) -> R) -> R { - VM_CURRENT.with(|x| unsafe { - f(x.clone() - .into_inner() - .as_ref() - .expect("call with_current_vm() but VM_CURRENT is null")) - }) -} - -pub fn enter_vm<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { - VM_STACK.with(|vms| { - vms.borrow_mut().push(vm.into()); - let prev = VM_CURRENT.with(|current| current.replace(vm)); - let ret = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); - vms.borrow_mut().pop(); - VM_CURRENT.with(|current| current.replace(prev)); - ret.unwrap_or_else(|e| std::panic::resume_unwind(e)) - }) -} - -pub fn with_vm<F, R>(obj: &PyObject, f: F) -> Option<R> -where - F: Fn(&VirtualMachine) -> R, -{ - let vm_owns_obj = |intp: NonNull<VirtualMachine>| { - // SAFETY: all references in VM_STACK should be valid - let vm = unsafe { intp.as_ref() }; - obj.fast_isinstance(vm.ctx.types.object_type) - }; - VM_STACK.with(|vms| { - let intp = match vms.borrow().iter().copied().exactly_one() { - Ok(x) => { - debug_assert!(vm_owns_obj(x)); - x - } - Err(mut others) => others.find(|x| vm_owns_obj(*x))?, - }; - // SAFETY: all references in VM_STACK should be valid, and should not be changed or moved - // at least until this function returns and the stack unwinds to an enter_vm() call - let vm = unsafe { intp.as_ref() }; - let prev = VM_CURRENT.with(|current| current.replace(vm)); - let ret = f(vm); - VM_CURRENT.with(|current| current.replace(prev)); - Some(ret) - }) -} - -#[must_use = "ThreadedVirtualMachine does nothing unless you move it to another thread and call .run()"] -#[cfg(feature = "threading")] -pub struct ThreadedVirtualMachine { - pub(super) vm: VirtualMachine, -} - -#[cfg(feature = "threading")] -impl ThreadedVirtualMachine { - /// Create a `FnOnce()` that can easily be passed to a function like [`std::thread::Builder::spawn`] - /// - /// # Note - /// - /// If you return a `PyObjectRef` (or a type that contains one) from `F`, and don't `join()` - /// on the thread this `FnOnce` runs in, there is a possibility that that thread will panic - /// as `PyObjectRef`'s `Drop` implementation tries to run the `__del__` destructor of a - /// Python object but finds that it's not in the context of any vm. - pub fn make_spawn_func<F, R>(self, f: F) -> impl FnOnce() -> R - where - F: FnOnce(&VirtualMachine) -> R, - { - move || self.run(f) - } - - /// Run a function in this thread context - /// - /// # Note - /// - /// If you return a `PyObjectRef` (or a type that contains one) from `F`, and don't return the object - /// to the parent thread and then `join()` on the `JoinHandle` (or similar), there is a possibility that - /// the current thread will panic as `PyObjectRef`'s `Drop` implementation tries to run the `__del__` - /// destructor of a python object but finds that it's not in the context of any vm. - pub fn run<F, R>(self, f: F) -> R - where - F: FnOnce(&VirtualMachine) -> R, - { - let vm = &self.vm; - enter_vm(vm, || f(vm)) - } -} - -impl VirtualMachine { - /// Start a new thread with access to the same interpreter. - /// - /// # Note - /// - /// If you return a `PyObjectRef` (or a type that contains one) from `F`, and don't `join()` - /// on the thread, there is a possibility that that thread will panic as `PyObjectRef`'s `Drop` - /// implementation tries to run the `__del__` destructor of a python object but finds that it's - /// not in the context of any vm. - #[cfg(feature = "threading")] - pub fn start_thread<F, R>(&self, f: F) -> std::thread::JoinHandle<R> - where - F: FnOnce(&VirtualMachine) -> R, - F: Send + 'static, - R: Send + 'static, - { - let thread = self.new_thread(); - std::thread::spawn(|| thread.run(f)) - } - - /// Create a new VM thread that can be passed to a function like [`std::thread::spawn`] - /// to use the same interpreter on a different thread. Note that if you just want to - /// use this with `thread::spawn`, you can use - /// [`vm.start_thread()`](`VirtualMachine::start_thread`) as a convenience. - /// - /// # Usage - /// - /// ``` - /// # rustpython_vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { - /// use std::thread::Builder; - /// let handle = Builder::new() - /// .name("my thread :)".into()) - /// .spawn(vm.new_thread().make_spawn_func(|vm| vm.ctx.none())) - /// .expect("couldn't spawn thread"); - /// let returned_obj = handle.join().expect("thread panicked"); - /// assert!(vm.is_none(&returned_obj)); - /// # }) - /// ``` - /// - /// Note: this function is safe, but running the returned ThreadedVirtualMachine in the same - /// thread context (i.e. with the same thread-local storage) doesn't have any - /// specific guaranteed behavior. - #[cfg(feature = "threading")] - pub fn new_thread(&self) -> ThreadedVirtualMachine { - use std::cell::Cell; - let vm = VirtualMachine { - builtins: self.builtins.clone(), - sys_module: self.sys_module.clone(), - ctx: self.ctx.clone(), - frames: RefCell::new(vec![]), - wasm_id: self.wasm_id.clone(), - exceptions: RefCell::default(), - import_func: self.import_func.clone(), - profile_func: RefCell::new(self.ctx.none()), - trace_func: RefCell::new(self.ctx.none()), - use_tracing: Cell::new(false), - recursion_limit: self.recursion_limit.clone(), - signal_handlers: None, - signal_rx: None, - repr_guards: RefCell::default(), - state: self.state.clone(), - initialized: self.initialized, - recursion_depth: Cell::new(0), - }; - ThreadedVirtualMachine { vm } - } -} diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs deleted file mode 100644 index 9cf6931d905..00000000000 --- a/vm/src/vm/vm_new.rs +++ /dev/null @@ -1,417 +0,0 @@ -use crate::{ - builtins::{ - builtin_func::PyNativeFunction, - descriptor::PyMethodDescriptor, - tuple::{IntoPyTuple, PyTupleRef}, - PyBaseException, PyBaseExceptionRef, PyDictRef, PyModule, PyStrRef, PyType, PyTypeRef, - }, - convert::ToPyObject, - function::{IntoPyNativeFn, PyMethodFlags}, - scope::Scope, - source::AtLocation, - source_code::SourceLocation, - vm::VirtualMachine, - AsObject, Py, PyObject, PyObjectRef, PyRef, -}; - -/// Collection of object creation helpers -impl VirtualMachine { - /// Create a new python object - pub fn new_pyobj(&self, value: impl ToPyObject) -> PyObjectRef { - value.to_pyobject(self) - } - - pub fn new_tuple(&self, value: impl IntoPyTuple) -> PyTupleRef { - value.into_pytuple(self) - } - - pub fn new_module( - &self, - name: &str, - dict: PyDictRef, - doc: Option<PyStrRef>, - ) -> PyRef<PyModule> { - let module = PyRef::new_ref( - PyModule::new(), - self.ctx.types.module_type.to_owned(), - Some(dict), - ); - module.init_dict(self.ctx.intern_str(name), doc, self); - module - } - - pub fn new_scope_with_builtins(&self) -> Scope { - Scope::with_builtins(None, self.ctx.new_dict(), self) - } - - pub fn new_function<F, FKind>(&self, name: &'static str, f: F) -> PyRef<PyNativeFunction> - where - F: IntoPyNativeFn<FKind>, - { - let def = self - .ctx - .new_method_def(name, f, PyMethodFlags::empty(), None); - def.build_function(self) - } - - pub fn new_method<F, FKind>( - &self, - name: &'static str, - class: &'static Py<PyType>, - f: F, - ) -> PyRef<PyMethodDescriptor> - where - F: IntoPyNativeFn<FKind>, - { - let def = self - .ctx - .new_method_def(name, f, PyMethodFlags::METHOD, None); - def.build_method(class, self) - } - - /// Instantiate an exception with arguments. - /// This function should only be used with builtin exception types; if a user-defined exception - /// type is passed in, it may not be fully initialized; try using - /// [`vm.invoke_exception()`][Self::invoke_exception] or - /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. - pub fn new_exception(&self, exc_type: PyTypeRef, args: Vec<PyObjectRef>) -> PyBaseExceptionRef { - // TODO: add repr of args into logging? - - PyRef::new_ref( - // TODO: this constructor might be invalid, because multiple - // exception (even builtin ones) are using custom constructors, - // see `OSError` as an example: - PyBaseException::new(args, self), - exc_type, - Some(self.ctx.new_dict()), - ) - } - - /// Instantiate an exception with no arguments. - /// This function should only be used with builtin exception types; if a user-defined exception - /// type is passed in, it may not be fully initialized; try using - /// [`vm.invoke_exception()`][Self::invoke_exception] or - /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. - pub fn new_exception_empty(&self, exc_type: PyTypeRef) -> PyBaseExceptionRef { - self.new_exception(exc_type, vec![]) - } - - /// Instantiate an exception with `msg` as the only argument. - /// This function should only be used with builtin exception types; if a user-defined exception - /// type is passed in, it may not be fully initialized; try using - /// [`vm.invoke_exception()`][Self::invoke_exception] or - /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. - pub fn new_exception_msg(&self, exc_type: PyTypeRef, msg: String) -> PyBaseExceptionRef { - self.new_exception(exc_type, vec![self.ctx.new_str(msg).into()]) - } - - /// Instantiate an exception with `msg` as the only argument and `dict` for object - /// This function should only be used with builtin exception types; if a user-defined exception - /// type is passed in, it may not be fully initialized; try using - /// [`vm.invoke_exception()`][Self::invoke_exception] or - /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. - pub fn new_exception_msg_dict( - &self, - exc_type: PyTypeRef, - msg: String, - dict: PyDictRef, - ) -> PyBaseExceptionRef { - PyRef::new_ref( - // TODO: this constructor might be invalid, because multiple - // exception (even builtin ones) are using custom constructors, - // see `OSError` as an example: - PyBaseException::new(vec![self.ctx.new_str(msg).into()], self), - exc_type, - Some(dict), - ) - } - - pub fn new_lookup_error(&self, msg: String) -> PyBaseExceptionRef { - let lookup_error = self.ctx.exceptions.lookup_error.to_owned(); - self.new_exception_msg(lookup_error, msg) - } - - pub fn new_attribute_error(&self, msg: String) -> PyBaseExceptionRef { - let attribute_error = self.ctx.exceptions.attribute_error.to_owned(); - self.new_exception_msg(attribute_error, msg) - } - - pub fn new_type_error(&self, msg: String) -> PyBaseExceptionRef { - let type_error = self.ctx.exceptions.type_error.to_owned(); - self.new_exception_msg(type_error, msg) - } - - pub fn new_name_error(&self, msg: String, name: PyStrRef) -> PyBaseExceptionRef { - let name_error_type = self.ctx.exceptions.name_error.to_owned(); - let name_error = self.new_exception_msg(name_error_type, msg); - name_error.as_object().set_attr("name", name, self).unwrap(); - name_error - } - - pub fn new_unsupported_unary_error(&self, a: &PyObject, op: &str) -> PyBaseExceptionRef { - self.new_type_error(format!( - "bad operand type for {}: '{}'", - op, - a.class().name() - )) - } - - pub fn new_unsupported_binop_error( - &self, - a: &PyObject, - b: &PyObject, - op: &str, - ) -> PyBaseExceptionRef { - self.new_type_error(format!( - "'{}' not supported between instances of '{}' and '{}'", - op, - a.class().name(), - b.class().name() - )) - } - - pub fn new_unsupported_ternop_error( - &self, - a: &PyObject, - b: &PyObject, - c: &PyObject, - op: &str, - ) -> PyBaseExceptionRef { - self.new_type_error(format!( - "Unsupported operand types for '{}': '{}', '{}', and '{}'", - op, - a.class().name(), - b.class().name(), - c.class().name() - )) - } - - pub fn new_os_error(&self, msg: String) -> PyBaseExceptionRef { - let os_error = self.ctx.exceptions.os_error.to_owned(); - self.new_exception_msg(os_error, msg) - } - - pub fn new_system_error(&self, msg: String) -> PyBaseExceptionRef { - let sys_error = self.ctx.exceptions.system_error.to_owned(); - self.new_exception_msg(sys_error, msg) - } - - pub fn new_unicode_decode_error(&self, msg: String) -> PyBaseExceptionRef { - let unicode_decode_error = self.ctx.exceptions.unicode_decode_error.to_owned(); - self.new_exception_msg(unicode_decode_error, msg) - } - - pub fn new_unicode_encode_error(&self, msg: String) -> PyBaseExceptionRef { - let unicode_encode_error = self.ctx.exceptions.unicode_encode_error.to_owned(); - self.new_exception_msg(unicode_encode_error, msg) - } - - /// Create a new python ValueError object. Useful for raising errors from - /// python functions implemented in rust. - pub fn new_value_error(&self, msg: String) -> PyBaseExceptionRef { - let value_error = self.ctx.exceptions.value_error.to_owned(); - self.new_exception_msg(value_error, msg) - } - - pub fn new_buffer_error(&self, msg: String) -> PyBaseExceptionRef { - let buffer_error = self.ctx.exceptions.buffer_error.to_owned(); - self.new_exception_msg(buffer_error, msg) - } - - // TODO: don't take ownership should make the success path faster - pub fn new_key_error(&self, obj: PyObjectRef) -> PyBaseExceptionRef { - let key_error = self.ctx.exceptions.key_error.to_owned(); - self.new_exception(key_error, vec![obj]) - } - - pub fn new_index_error(&self, msg: String) -> PyBaseExceptionRef { - let index_error = self.ctx.exceptions.index_error.to_owned(); - self.new_exception_msg(index_error, msg) - } - - pub fn new_not_implemented_error(&self, msg: String) -> PyBaseExceptionRef { - let not_implemented_error = self.ctx.exceptions.not_implemented_error.to_owned(); - self.new_exception_msg(not_implemented_error, msg) - } - - pub fn new_recursion_error(&self, msg: String) -> PyBaseExceptionRef { - let recursion_error = self.ctx.exceptions.recursion_error.to_owned(); - self.new_exception_msg(recursion_error, msg) - } - - pub fn new_zero_division_error(&self, msg: String) -> PyBaseExceptionRef { - let zero_division_error = self.ctx.exceptions.zero_division_error.to_owned(); - self.new_exception_msg(zero_division_error, msg) - } - - pub fn new_overflow_error(&self, msg: String) -> PyBaseExceptionRef { - let overflow_error = self.ctx.exceptions.overflow_error.to_owned(); - self.new_exception_msg(overflow_error, msg) - } - - #[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))] - pub fn new_syntax_error( - &self, - error: &crate::compiler::CompileError, - source: Option<&str>, - ) -> PyBaseExceptionRef { - let syntax_error_type = match &error.error { - #[cfg(feature = "rustpython-parser")] - crate::compiler::CompileErrorType::Parse(p) if p.is_indentation_error() => { - self.ctx.exceptions.indentation_error - } - #[cfg(feature = "rustpython-parser")] - crate::compiler::CompileErrorType::Parse(p) if p.is_tab_error() => { - self.ctx.exceptions.tab_error - } - _ => self.ctx.exceptions.syntax_error, - } - .to_owned(); - - // TODO: replace to SourceCode - fn get_statement(source: &str, loc: Option<SourceLocation>) -> Option<String> { - let line = source - .split('\n') - .nth(loc?.row.to_zero_indexed_usize())? - .to_owned(); - Some(line + "\n") - } - - let statement = if let Some(source) = source { - get_statement(source, error.location) - } else { - None - }; - - fn fmt( - error: &crate::compiler::CompileError, - statement: Option<&str>, - f: &mut impl std::fmt::Write, - ) -> std::fmt::Result { - let loc = error.location; - if let Some(ref stmt) = statement { - // visualize the error when location and statement are provided - write!( - f, - "{error}{at_location}\n{stmt}{arrow:>pad$}", - error = error.error, - at_location = AtLocation(loc.as_ref()), - pad = loc.map_or(0, |loc| loc.column.to_usize()), - arrow = "^" - ) - } else { - write!( - f, - "{error}{at_location}", - error = error.error, - at_location = AtLocation(loc.as_ref()), - ) - } - } - - let mut msg = String::new(); - fmt(error, statement.as_deref(), &mut msg).unwrap(); - - let syntax_error = self.new_exception_msg(syntax_error_type, msg); - let (lineno, offset) = error.python_location(); - let lineno = self.ctx.new_int(lineno); - let offset = self.ctx.new_int(offset); - syntax_error - .as_object() - .set_attr("lineno", lineno, self) - .unwrap(); - syntax_error - .as_object() - .set_attr("offset", offset, self) - .unwrap(); - - syntax_error - .as_object() - .set_attr("text", statement.to_pyobject(self), self) - .unwrap(); - syntax_error - .as_object() - .set_attr( - "filename", - self.ctx.new_str(error.source_path.clone()), - self, - ) - .unwrap(); - syntax_error - } - - pub fn new_import_error(&self, msg: String, name: PyStrRef) -> PyBaseExceptionRef { - let import_error = self.ctx.exceptions.import_error.to_owned(); - let exc = self.new_exception_msg(import_error, msg); - exc.as_object().set_attr("name", name, self).unwrap(); - exc - } - - pub fn new_runtime_error(&self, msg: String) -> PyBaseExceptionRef { - let runtime_error = self.ctx.exceptions.runtime_error.to_owned(); - self.new_exception_msg(runtime_error, msg) - } - - pub fn new_memory_error(&self, msg: String) -> PyBaseExceptionRef { - let memory_error_type = self.ctx.exceptions.memory_error.to_owned(); - self.new_exception_msg(memory_error_type, msg) - } - - pub fn new_stop_iteration(&self, value: Option<PyObjectRef>) -> PyBaseExceptionRef { - let dict = self.ctx.new_dict(); - let args = if let Some(value) = value { - // manually set `value` attribute like StopIteration.__init__ - dict.set_item("value", value.clone(), self) - .expect("dict.__setitem__ never fails"); - vec![value] - } else { - Vec::new() - }; - - PyRef::new_ref( - PyBaseException::new(args, self), - self.ctx.exceptions.stop_iteration.to_owned(), - Some(dict), - ) - } - - fn new_downcast_error( - &self, - msg: &'static str, - error_type: &'static Py<PyType>, - class: &Py<PyType>, - obj: &PyObject, // the impl Borrow allows to pass PyObjectRef or &PyObject - ) -> PyBaseExceptionRef { - let actual_class = obj.class(); - let actual_type = &*actual_class.name(); - let expected_type = &*class.name(); - let msg = format!("Expected {msg} '{expected_type}' but '{actual_type}' found"); - self.new_exception_msg(error_type.to_owned(), msg) - } - - pub(crate) fn new_downcast_runtime_error( - &self, - class: &Py<PyType>, - obj: &impl AsObject, - ) -> PyBaseExceptionRef { - self.new_downcast_error( - "payload", - self.ctx.exceptions.runtime_error, - class, - obj.as_object(), - ) - } - - pub(crate) fn new_downcast_type_error( - &self, - class: &Py<PyType>, - obj: &impl AsObject, - ) -> PyBaseExceptionRef { - self.new_downcast_error( - "type", - self.ctx.exceptions.type_error, - class, - obj.as_object(), - ) - } -} diff --git a/vm/src/vm/vm_object.rs b/vm/src/vm/vm_object.rs deleted file mode 100644 index a957ed66ca4..00000000000 --- a/vm/src/vm/vm_object.rs +++ /dev/null @@ -1,158 +0,0 @@ -use super::PyMethod; -use crate::{ - builtins::{pystr::AsPyStr, PyBaseExceptionRef, PyList, PyStrInterned}, - function::IntoFuncArgs, - identifier, - object::{AsObject, PyObject, PyObjectRef, PyResult}, - vm::VirtualMachine, -}; - -/// PyObject support -impl VirtualMachine { - #[track_caller] - #[cold] - fn _py_panic_failed(&self, exc: PyBaseExceptionRef, msg: &str) -> ! { - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - { - let show_backtrace = - std::env::var_os("RUST_BACKTRACE").map_or(cfg!(target_os = "wasi"), |v| &v != "0"); - let after = if show_backtrace { - self.print_exception(exc); - "exception backtrace above" - } else { - "run with RUST_BACKTRACE=1 to see Python backtrace" - }; - panic!("{msg}; {after}") - } - #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] - { - use wasm_bindgen::prelude::*; - #[wasm_bindgen] - extern "C" { - #[wasm_bindgen(js_namespace = console)] - fn error(s: &str); - } - let mut s = String::new(); - self.write_exception(&mut s, &exc).unwrap(); - error(&s); - panic!("{}; exception backtrace above", msg) - } - } - - #[track_caller] - pub fn unwrap_pyresult<T>(&self, result: PyResult<T>) -> T { - match result { - Ok(x) => x, - Err(exc) => { - self._py_panic_failed(exc, "called `vm.unwrap_pyresult()` on an `Err` value") - } - } - } - #[track_caller] - pub fn expect_pyresult<T>(&self, result: PyResult<T>, msg: &str) -> T { - match result { - Ok(x) => x, - Err(exc) => self._py_panic_failed(exc, msg), - } - } - - /// Test whether a python object is `None`. - pub fn is_none(&self, obj: &PyObject) -> bool { - obj.is(&self.ctx.none) - } - pub fn option_if_none(&self, obj: PyObjectRef) -> Option<PyObjectRef> { - if self.is_none(&obj) { - None - } else { - Some(obj) - } - } - pub fn unwrap_or_none(&self, obj: Option<PyObjectRef>) -> PyObjectRef { - obj.unwrap_or_else(|| self.ctx.none()) - } - - pub fn call_get_descriptor_specific( - &self, - descr: &PyObject, - obj: Option<PyObjectRef>, - cls: Option<PyObjectRef>, - ) -> Option<PyResult> { - let descr_get = descr - .class() - .mro_find_map(|cls| cls.slots.descr_get.load())?; - Some(descr_get(descr.to_owned(), obj, cls, self)) - } - - pub fn call_get_descriptor(&self, descr: &PyObject, obj: PyObjectRef) -> Option<PyResult> { - let cls = obj.class().to_owned().into(); - self.call_get_descriptor_specific(descr, Some(obj), Some(cls)) - } - - pub fn call_if_get_descriptor(&self, attr: &PyObject, obj: PyObjectRef) -> PyResult { - self.call_get_descriptor(attr, obj) - .unwrap_or_else(|| Ok(attr.to_owned())) - } - - #[inline] - pub fn call_method<T>(&self, obj: &PyObject, method_name: &str, args: T) -> PyResult - where - T: IntoFuncArgs, - { - flame_guard!(format!("call_method({:?})", method_name)); - - let dynamic_name; - let name = match self.ctx.interned_str(method_name) { - Some(name) => name.as_pystr(&self.ctx), - None => { - dynamic_name = self.ctx.new_str(method_name); - &dynamic_name - } - }; - PyMethod::get(obj.to_owned(), name, self)?.invoke(args, self) - } - - pub fn dir(&self, obj: Option<PyObjectRef>) -> PyResult<PyList> { - let seq = match obj { - Some(obj) => self - .get_special_method(&obj, identifier!(self, __dir__))? - .ok_or_else(|| self.new_type_error("object does not provide __dir__".to_owned()))? - .invoke((), self)?, - None => self.call_method( - self.current_locals()?.as_object(), - identifier!(self, keys).as_str(), - (), - )?, - }; - let items: Vec<_> = seq.try_to_value(self)?; - let lst = PyList::from(items); - lst.sort(Default::default(), self)?; - Ok(lst) - } - - #[inline] - pub(crate) fn get_special_method( - &self, - obj: &PyObject, - method: &'static PyStrInterned, - ) -> PyResult<Option<PyMethod>> { - PyMethod::get_special::<false>(obj, method, self) - } - - /// NOT PUBLIC API - #[doc(hidden)] - pub fn call_special_method( - &self, - obj: &PyObject, - method: &'static PyStrInterned, - args: impl IntoFuncArgs, - ) -> PyResult { - self.get_special_method(obj, method)? - .ok_or_else(|| self.new_attribute_error(method.as_str().to_owned()))? - .invoke(args, self) - } - - #[deprecated(note = "in favor of `obj.call(args, vm)`")] - pub fn invoke(&self, obj: &impl AsObject, args: impl IntoFuncArgs) -> PyResult { - obj.as_object().call(args, self) - } -} diff --git a/vm/src/warn.rs b/vm/src/warn.rs deleted file mode 100644 index d0acccbf29b..00000000000 --- a/vm/src/warn.rs +++ /dev/null @@ -1,415 +0,0 @@ -use crate::{ - builtins::{ - PyDict, PyDictRef, PyListRef, PyStr, PyStrInterned, PyStrRef, PyTuple, PyTupleRef, - PyTypeRef, - }, - convert::{IntoObject, TryFromObject}, - types::PyComparisonOp, - AsObject, Context, Py, PyObjectRef, PyResult, VirtualMachine, -}; - -pub struct WarningsState { - filters: PyListRef, - _once_registry: PyDictRef, - default_action: PyStrRef, - filters_version: usize, -} - -impl WarningsState { - fn create_filter(ctx: &Context) -> PyListRef { - ctx.new_list(vec![ctx - .new_tuple(vec![ - ctx.new_str("__main__").into(), - ctx.types.none_type.as_object().to_owned(), - ctx.exceptions.warning.as_object().to_owned(), - ctx.new_str("ACTION").into(), - ctx.new_int(0).into(), - ]) - .into()]) - } - - pub fn init_state(ctx: &Context) -> WarningsState { - WarningsState { - filters: Self::create_filter(ctx), - _once_registry: PyDict::new_ref(ctx), - default_action: ctx.new_str("default"), - filters_version: 0, - } - } -} - -fn check_matched(obj: &PyObjectRef, arg: &PyObjectRef, vm: &VirtualMachine) -> PyResult<bool> { - if obj.class().is(vm.ctx.types.none_type) { - return Ok(true); - } - - if obj.rich_compare_bool(arg, PyComparisonOp::Eq, vm)? { - return Ok(false); - } - - let result = obj.call((arg.to_owned(),), vm); - Ok(result.is_ok()) -} - -fn get_warnings_attr( - vm: &VirtualMachine, - attr_name: &'static PyStrInterned, - try_import: bool, -) -> PyResult<Option<PyObjectRef>> { - let module = if try_import - && !vm - .state - .finalizing - .load(std::sync::atomic::Ordering::SeqCst) - { - match vm.import("warnings", None, 0) { - Ok(module) => module, - Err(_) => return Ok(None), - } - } else { - // TODO: finalizing support - return Ok(None); - }; - Ok(Some(module.get_attr(attr_name, vm)?)) -} - -pub fn warn( - message: PyStrRef, - category: Option<PyTypeRef>, - stack_level: isize, - source: Option<PyObjectRef>, - vm: &VirtualMachine, -) -> PyResult<()> { - let (filename, lineno, module, registry) = setup_context(stack_level, vm)?; - warn_explicit( - category, message, filename, lineno, module, registry, None, source, vm, - ) -} - -fn get_default_action(vm: &VirtualMachine) -> PyResult<PyObjectRef> { - Ok(vm.state.warnings.default_action.clone().into()) - // .map_err(|_| { - // vm.new_value_error(format!( - // "_warnings.defaultaction must be a string, not '{}'", - // vm.state.warnings.default_action - // )) - // }) -} - -fn get_filter( - category: PyObjectRef, - text: PyObjectRef, - lineno: usize, - module: PyObjectRef, - mut _item: PyTupleRef, - vm: &VirtualMachine, -) -> PyResult { - let filters = vm.state.warnings.filters.as_object().to_owned(); - - let filters: PyListRef = filters - .try_into_value(vm) - .map_err(|_| vm.new_value_error("_warnings.filters must be a list".to_string()))?; - - /* WarningsState.filters could change while we are iterating over it. */ - for i in 0..filters.borrow_vec().len() { - let tmp_item = if let Some(tmp_item) = filters.borrow_vec().get(i).cloned() { - let tmp_item = PyTupleRef::try_from_object(vm, tmp_item)?; - (tmp_item.len() == 5).then_some(tmp_item) - } else { - None - } - .ok_or_else(|| vm.new_value_error(format!("_warnings.filters item {i} isn't a 5-tuple")))?; - - /* Python code: action, msg, cat, mod, ln = item */ - let action = if let Some(action) = tmp_item.first() { - action.str(vm).map(|action| action.into_object()) - } else { - Err(vm.new_type_error("action must be a string".to_string())) - }; - - let good_msg = if let Some(msg) = tmp_item.get(1) { - check_matched(msg, &text, vm)? - } else { - false - }; - - let is_subclass = if let Some(cat) = tmp_item.get(2) { - category.fast_isinstance(cat.class()) - } else { - false - }; - - let good_mod = if let Some(item_mod) = tmp_item.get(3) { - check_matched(item_mod, &module, vm)? - } else { - false - }; - - let ln = tmp_item.get(4).map_or(0, |ln_obj| { - ln_obj.try_int(vm).map_or(0, |ln| ln.as_u32_mask() as _) - }); - - if good_msg && good_mod && is_subclass && (ln == 0 || lineno == ln) { - _item = tmp_item; - return action; - } - } - - get_default_action(vm) -} - -fn already_warned( - registry: PyObjectRef, - key: PyObjectRef, - should_set: bool, - vm: &VirtualMachine, -) -> PyResult<bool> { - let version_obj = registry.get_item(identifier!(&vm.ctx, version), vm).ok(); - let filters_version = vm.ctx.new_int(vm.state.warnings.filters_version).into(); - - match version_obj { - Some(version_obj) - if version_obj.try_int(vm).is_ok() || version_obj.is(&filters_version) => - { - let already_warned = registry.get_item(key.as_ref(), vm)?; - if already_warned.is_true(vm)? { - return Ok(true); - } - } - _ => { - let registry = registry.dict(); - if let Some(registry) = registry.as_ref() { - registry.clear(); - let r = registry.set_item("version", filters_version, vm); - if r.is_err() { - return Ok(false); - } - } - } - } - - /* This warning wasn't found in the registry, set it. */ - if !should_set { - return Ok(false); - } - - let item = vm.ctx.true_value.clone().into(); - let _ = registry.set_item(key.as_ref(), item, vm); // ignore set error - Ok(true) -} - -fn normalize_module(filename: &Py<PyStr>, vm: &VirtualMachine) -> Option<PyObjectRef> { - let obj = match filename.char_len() { - 0 => vm.new_pyobj("<unknown>"), - len if len >= 3 && filename.as_str().ends_with(".py") => { - vm.new_pyobj(&filename.as_str()[..len - 3]) - } - _ => filename.as_object().to_owned(), - }; - Some(obj) -} - -#[allow(clippy::too_many_arguments)] -fn warn_explicit( - category: Option<PyTypeRef>, - message: PyStrRef, - filename: PyStrRef, - lineno: usize, - module: Option<PyObjectRef>, - registry: PyObjectRef, - source_line: Option<PyObjectRef>, - source: Option<PyObjectRef>, - vm: &VirtualMachine, -) -> PyResult<()> { - let registry: PyObjectRef = registry - .try_into_value(vm) - .map_err(|_| vm.new_type_error("'registry' must be a dict or None".to_owned()))?; - - // Normalize module. - let module = match module.or_else(|| normalize_module(&filename, vm)) { - Some(module) => module, - None => return Ok(()), - }; - - // Normalize message. - let text = message.as_str(); - - let category = if let Some(category) = category { - if !category.fast_issubclass(vm.ctx.exceptions.warning) { - return Err(vm.new_type_error(format!( - "category must be a Warning subclass, not '{}'", - category.class().name() - ))); - } - category - } else { - vm.ctx.exceptions.user_warning.to_owned() - }; - - let category = if message.fast_isinstance(vm.ctx.exceptions.warning) { - message.class().to_owned() - } else { - category - }; - - // Create key. - let key = PyTuple::new_ref( - vec![ - vm.ctx.new_int(3).into(), - vm.ctx.new_str(text).into(), - category.as_object().to_owned(), - vm.ctx.new_int(lineno).into(), - ], - &vm.ctx, - ); - - if !vm.is_none(registry.as_object()) && already_warned(registry, key.into_object(), false, vm)? - { - return Ok(()); - } - - let item = vm.ctx.new_tuple(vec![]); - let action = get_filter( - category.as_object().to_owned(), - vm.ctx.new_str(text).into(), - lineno, - module, - item, - vm, - )?; - - if action.str(vm)?.as_str().eq("error") { - return Err(vm.new_type_error(message.to_string())); - } - - if action.str(vm)?.as_str().eq("ignore") { - return Ok(()); - } - - call_show_warning( - // t_state, - category, - message, - filename, - lineno, // lineno_obj, - source_line, - source, - vm, - ) -} - -fn call_show_warning( - category: PyTypeRef, - message: PyStrRef, - filename: PyStrRef, - lineno: usize, - source_line: Option<PyObjectRef>, - source: Option<PyObjectRef>, - vm: &VirtualMachine, -) -> PyResult<()> { - let Some(show_fn) = - get_warnings_attr(vm, identifier!(&vm.ctx, _showwarnmsg), source.is_some())? - else { - return show_warning(filename, lineno, message, category, source_line, vm); - }; - if !show_fn.is_callable() { - return Err( - vm.new_type_error("warnings._showwarnmsg() must be set to a callable".to_owned()) - ); - } - let Some(warnmsg_cls) = get_warnings_attr(vm, identifier!(&vm.ctx, WarningMessage), false)? - else { - return Err(vm.new_type_error("unable to get warnings.WarningMessage".to_owned())); - }; - - let msg = warnmsg_cls.call( - vec![ - message.into(), - category.into(), - filename.into(), - vm.new_pyobj(lineno), - vm.ctx.none(), - vm.ctx.none(), - vm.unwrap_or_none(source), - ], - vm, - )?; - show_fn.call((msg,), vm)?; - Ok(()) -} - -fn show_warning( - _filename: PyStrRef, - _lineno: usize, - text: PyStrRef, - category: PyTypeRef, - _source_line: Option<PyObjectRef>, - vm: &VirtualMachine, -) -> PyResult<()> { - let stderr = crate::stdlib::sys::PyStderr(vm); - writeln!(stderr, "{}: {}", category.name(), text.as_str(),); - Ok(()) -} - -/// filename, module, and registry are new refs, globals is borrowed -/// Returns `Ok` on success, or `Err` on error (no new refs) -fn setup_context( - mut stack_level: isize, - vm: &VirtualMachine, -) -> PyResult< - // filename, lineno, module, registry - (PyStrRef, usize, Option<PyObjectRef>, PyObjectRef), -> { - let __warningregistry__ = "__warningregistry__"; - let __name__ = "__name__"; - - let mut f = vm.current_frame().as_deref().cloned(); - - // Stack level comparisons to Python code is off by one as there is no - // warnings-related stack level to avoid. - if stack_level <= 0 || f.as_ref().map_or(false, |frame| frame.is_internal_frame()) { - loop { - stack_level -= 1; - if stack_level <= 0 { - break; - } - if let Some(tmp) = f { - f = tmp.f_back(vm); - } else { - break; - } - } - } else { - loop { - stack_level -= 1; - if stack_level <= 0 { - break; - } - if let Some(tmp) = f { - f = tmp.next_external_frame(vm); - } else { - break; - } - } - } - - let (globals, filename, lineno) = if let Some(f) = f { - (f.globals.clone(), f.code.source_path, f.f_lineno()) - } else { - (vm.current_globals().clone(), vm.ctx.intern_str("sys"), 1) - }; - - let registry = if let Ok(registry) = globals.get_item(__warningregistry__, vm) { - registry - } else { - let registry = PyDict::new_ref(&vm.ctx); - globals.set_item(__warningregistry__, registry.clone().into(), vm)?; - registry.into() - }; - - // Setup module. - let module = globals - .get_item(__name__, vm) - .unwrap_or_else(|_| vm.new_pyobj("<string>")); - Ok((filename.to_owned(), lineno, Some(module), registry)) -} diff --git a/vm/src/windows.rs b/vm/src/windows.rs deleted file mode 100644 index e749241c07f..00000000000 --- a/vm/src/windows.rs +++ /dev/null @@ -1,68 +0,0 @@ -use crate::{ - convert::{ToPyObject, ToPyResult}, - stdlib::os::errno_err, - PyObjectRef, PyResult, TryFromObject, VirtualMachine, -}; -use windows::Win32::Foundation::HANDLE; -use windows_sys::Win32::Foundation::{BOOL, HANDLE as RAW_HANDLE, INVALID_HANDLE_VALUE}; - -pub(crate) trait WindowsSysResultValue { - type Ok: ToPyObject; - fn is_err(&self) -> bool; - fn into_ok(self) -> Self::Ok; -} - -impl WindowsSysResultValue for RAW_HANDLE { - type Ok = HANDLE; - fn is_err(&self) -> bool { - *self == INVALID_HANDLE_VALUE - } - fn into_ok(self) -> Self::Ok { - HANDLE(self) - } -} - -impl WindowsSysResultValue for BOOL { - type Ok = (); - fn is_err(&self) -> bool { - *self == 0 - } - fn into_ok(self) -> Self::Ok {} -} - -pub(crate) struct WindowsSysResult<T>(pub T); - -impl<T: WindowsSysResultValue> WindowsSysResult<T> { - pub fn is_err(&self) -> bool { - self.0.is_err() - } - pub fn into_pyresult(self, vm: &VirtualMachine) -> PyResult<T::Ok> { - if self.is_err() { - Err(errno_err(vm)) - } else { - Ok(self.0.into_ok()) - } - } -} - -impl<T: WindowsSysResultValue> ToPyResult for WindowsSysResult<T> { - fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { - let ok = self.into_pyresult(vm)?; - Ok(ok.to_pyobject(vm)) - } -} - -type HandleInt = usize; // TODO: change to isize when fully ported to windows-rs - -impl TryFromObject for HANDLE { - fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> { - let handle = HandleInt::try_from_object(vm, obj)?; - Ok(HANDLE(handle as isize)) - } -} - -impl ToPyObject for HANDLE { - fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { - (self.0 as HandleInt).to_pyobject(vm) - } -} diff --git a/wapm.toml b/wapm.toml index 98bf96bed12..b116cb84161 100644 --- a/wapm.toml +++ b/wapm.toml @@ -1,6 +1,6 @@ [package] name = "rustpython" -version = "0.3.0" +version = "0.5.0" description = "A Python-3 (CPython >= 3.5.0) Interpreter written in Rust 🐍 😱 🤘" license-file = "LICENSE" readme = "README.md" @@ -8,7 +8,7 @@ repository = "https://github.com/RustPython/RustPython" [[module]] name = "rustpython" -source = "target/wasm32-wasi/release/rustpython.wasm" +source = "target/wasm32-wasip1/release/rustpython.wasm" abi = "wasi" [[command]] diff --git a/wasm/demo/.envrc b/wasm/demo/.envrc new file mode 100644 index 00000000000..928b937d323 --- /dev/null +++ b/wasm/demo/.envrc @@ -0,0 +1,2 @@ +export NODE_OPTIONS=--openssl-legacy-provider +export PATH=$PATH:`pwd`/../../geckodriver diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json new file mode 100644 index 00000000000..287e0b6ffeb --- /dev/null +++ b/wasm/demo/package-lock.json @@ -0,0 +1,5728 @@ +{ + "name": "app", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "app", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@codemirror/lang-python": "^6.1.6", + "@xterm/addon-fit": "^0.10.0", + "@xterm/xterm": "^5.3.0", + "codemirror": "^6.0.1", + "upgrade": "^1.1.0", + "xterm-readline": "^1.1.2" + }, + "devDependencies": { + "@wasm-tool/wasm-pack-plugin": "^1.7.0", + "css-loader": "^7.1.2", + "html-webpack-plugin": "^5.6.3", + "mini-css-extract-plugin": "^2.9.2", + "serve": "^14.2.5", + "webpack": "^5.105.0", + "webpack-cli": "^6.0.1", + "webpack-dev-server": "^5.2.1" + } + }, + "node_modules/@codemirror/autocomplete": { + "version": "6.18.6", + "resolved": "https://registry.npmjs.org/@codemirror/autocomplete/-/autocomplete-6.18.6.tgz", + "integrity": "sha512-PHHBXFomUs5DF+9tCOM/UoW6XQ4R44lLNNhRaW9PKPTU0D7lIjRg3ElxaJnTwsl/oHiR93WSXDBrekhoUGCPtg==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.17.0", + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@codemirror/commands": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@codemirror/commands/-/commands-6.8.0.tgz", + "integrity": "sha512-q8VPEFaEP4ikSlt6ZxjB3zW72+7osfAYW9i8Zu943uqbKuz6utc1+F170hyLUCUltXORjQXRyYQNfkckzA/bPQ==", + "license": "MIT", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@codemirror/state": "^6.4.0", + "@codemirror/view": "^6.27.0", + "@lezer/common": "^1.1.0" + } + }, + "node_modules/@codemirror/lang-python": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/@codemirror/lang-python/-/lang-python-6.1.7.tgz", + "integrity": "sha512-mZnFTsL4lW5p9ch8uKNKeRU3xGGxr1QpESLilfON2E3fQzOa/OygEMkaDvERvXDJWJA9U9oN/D4w0ZuUzNO4+g==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.3.2", + "@codemirror/language": "^6.8.0", + "@codemirror/state": "^6.0.0", + "@lezer/common": "^1.2.1", + "@lezer/python": "^1.1.4" + } + }, + "node_modules/@codemirror/language": { + "version": "6.10.8", + "resolved": "https://registry.npmjs.org/@codemirror/language/-/language-6.10.8.tgz", + "integrity": "sha512-wcP8XPPhDH2vTqf181U8MbZnW+tDyPYy0UzVOa+oHORjyT+mhhom9vBd7dApJwoDz9Nb/a8kHjJIsuA/t8vNFw==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.23.0", + "@lezer/common": "^1.1.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "node_modules/@codemirror/lint": { + "version": "6.8.4", + "resolved": "https://registry.npmjs.org/@codemirror/lint/-/lint-6.8.4.tgz", + "integrity": "sha512-u4q7PnZlJUojeRe8FJa/njJcMctISGgPQ4PnWsd9268R4ZTtU+tfFYmwkBvgcrK2+QQ8tYFVALVb5fVJykKc5A==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.35.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/search": { + "version": "6.5.10", + "resolved": "https://registry.npmjs.org/@codemirror/search/-/search-6.5.10.tgz", + "integrity": "sha512-RMdPdmsrUf53pb2VwflKGHEe1XVM07hI7vV2ntgw1dmqhimpatSJKva4VA9h4TLUDOD4EIF02201oZurpnEFsg==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "crelt": "^1.0.5" + } + }, + "node_modules/@codemirror/state": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.2.tgz", + "integrity": "sha512-FVqsPqtPWKVVL3dPSxy8wEF/ymIEuVzF1PK3VbUgrxXpJUSHQWWZz4JMToquRxnkw+36LTamCZG2iua2Ptq0fA==", + "license": "MIT", + "dependencies": { + "@marijn/find-cluster-break": "^1.0.0" + } + }, + "node_modules/@codemirror/view": { + "version": "6.36.3", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.36.3.tgz", + "integrity": "sha512-N2bilM47QWC8Hnx0rMdDxO2x2ImJ1FvZWXubwKgjeoOrWwEiFrtpA7SFHcuZ+o2Ze2VzbkgbzWVj4+V18LVkeg==", + "license": "MIT", + "dependencies": { + "@codemirror/state": "^6.5.0", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "node_modules/@discoveryjs/json-ext": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.6.3.tgz", + "integrity": "sha512-4B4OijXeVNOPZlYA2oEwWOTkzyltLao+xbotHQeqN++Rv27Y6s818+n2Qkp8q+Fxhn0t/5lA5X1Mxktud8eayQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.17.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", + "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz", + "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@jsonjoy.com/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/json-pack": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-1.1.1.tgz", + "integrity": "sha512-osjeBqMJ2lb/j/M8NCPjs1ylqWIcTRTycIhVB5pt6LgzgeRSb0YRZ7j9RfA8wIUrsr/medIuhVyonXRZWLyfdw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/base64": "^1.1.1", + "@jsonjoy.com/util": "^1.1.2", + "hyperdyperid": "^1.2.0", + "thingies": "^1.20.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/util": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.5.0.tgz", + "integrity": "sha512-ojoNsrIuPI9g6o8UxhraZQSyF2ByJanAY4cTFbc8Mf2AXEF4aQRGY1dJxyJpuyav8r9FGflEt/Ff3u5Nt6YMPA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", + "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@lezer/common": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.2.3.tgz", + "integrity": "sha512-w7ojc8ejBqr2REPsWxJjrMFsA/ysDCFICn8zEOR9mrqzOu2amhITYuLD8ag6XZf0CFXDrhKqw7+tW8cX66NaDA==", + "license": "MIT" + }, + "node_modules/@lezer/highlight": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.2.1.tgz", + "integrity": "sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@lezer/lr": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.4.2.tgz", + "integrity": "sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@lezer/python": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@lezer/python/-/python-1.1.15.tgz", + "integrity": "sha512-aVQ43m2zk4FZYedCqL0KHPEUsqZOrmAvRhkhHlVPnDD1HODDyyQv5BRIuod4DadkgBEZd53vQOtXTonNbEgjrQ==", + "license": "MIT", + "dependencies": { + "@lezer/common": "^1.2.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, + "node_modules/@marijn/find-cluster-break": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz", + "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==", + "license": "MIT" + }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/bonjour": { + "version": "3.5.13", + "resolved": "https://registry.npmjs.org/@types/bonjour/-/bonjour-3.5.13.tgz", + "integrity": "sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect-history-api-fallback": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.5.4.tgz", + "integrity": "sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express-serve-static-core": "*", + "@types/node": "*" + } + }, + "node_modules/@types/eslint": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz", + "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "*", + "@types/json-schema": "*" + } + }, + "node_modules/@types/eslint-scope": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", + "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.6.tgz", + "integrity": "sha512-3xhRnjJPkULekpSzgtoNYYcTWgEZkp4myc+Saevii5JPnHNvHMRlBSHDbs7Bh1iPPoVTERHEZXyhyLbMEsExsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/express/node_modules/@types/express-serve-static-core": { + "version": "4.19.6", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", + "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/html-minifier-terser": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", + "integrity": "sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/http-proxy": { + "version": "1.17.16", + "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.16.tgz", + "integrity": "sha512-sdWoUajOB1cd0A8cRRQ1cfyWNbmFKLAqBB89Y8x5iYyG/mkJHc0YUH8pdWBy2omi9qtCpiIgGjuwO0dQST2l5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.13.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.8.tgz", + "integrity": "sha512-G3EfaZS+iOGYWLLRCEAXdWK9my08oHNZ+FHluRiggIYJPOXzhOiDgpVCUHaUvyIC5/fj7C/p637jdzC666AOKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/@types/node-forge": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.11.tgz", + "integrity": "sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/qs": { + "version": "6.9.18", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.18.tgz", + "integrity": "sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/retry": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", + "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-index": { + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.4.tgz", + "integrity": "sha512-qLpGZ/c2fhSs5gnYsQxtDEq3Oy8SXPClIXkW5ghvAvsNuVSA8k+gCONcUCS/UjLEYvYps+e8uBtfgXgvhwfNug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, + "node_modules/@types/sockjs": { + "version": "0.3.36", + "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.36.tgz", + "integrity": "sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/ws": { + "version": "8.5.14", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.14.tgz", + "integrity": "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@wasm-tool/wasm-pack-plugin": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@wasm-tool/wasm-pack-plugin/-/wasm-pack-plugin-1.7.0.tgz", + "integrity": "sha512-WikzYsw7nTd5CZxH75h7NxM/FLJAgqfWt+/gk3EL3wYKxiIlpMIYPja+sHQl3ARiicIYy4BDfxkbAVjRYlouTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^2.4.1", + "command-exists": "^1.2.7", + "watchpack": "^2.1.1", + "which": "^2.0.2" + } + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz", + "integrity": "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/helper-numbers": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.13.2.tgz", + "integrity": "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.13.2.tgz", + "integrity": "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.14.1.tgz", + "integrity": "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.13.2.tgz", + "integrity": "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.13.2", + "@webassemblyjs/helper-api-error": "1.13.2", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.13.2.tgz", + "integrity": "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.14.1.tgz", + "integrity": "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/wasm-gen": "1.14.1" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.13.2.tgz", + "integrity": "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.13.2.tgz", + "integrity": "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.13.2.tgz", + "integrity": "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.14.1.tgz", + "integrity": "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/helper-wasm-section": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-opt": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1", + "@webassemblyjs/wast-printer": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.14.1.tgz", + "integrity": "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.14.1.tgz", + "integrity": "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz", + "integrity": "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-api-error": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.14.1.tgz", + "integrity": "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webpack-cli/configtest": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-3.0.1.tgz", + "integrity": "sha512-u8d0pJ5YFgneF/GuvEiDA61Tf1VDomHHYMjv/wc9XzYj7nopltpG96nXN5dJRstxZhcNpV1g+nT6CydO7pHbjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12.0" + }, + "peerDependencies": { + "webpack": "^5.82.0", + "webpack-cli": "6.x.x" + } + }, + "node_modules/@webpack-cli/info": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-3.0.1.tgz", + "integrity": "sha512-coEmDzc2u/ffMvuW9aCjoRzNSPDl/XLuhPdlFRpT9tZHmJ/039az33CE7uH+8s0uL1j5ZNtfdv0HkfaKRBGJsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12.0" + }, + "peerDependencies": { + "webpack": "^5.82.0", + "webpack-cli": "6.x.x" + } + }, + "node_modules/@webpack-cli/serve": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-3.0.1.tgz", + "integrity": "sha512-sbgw03xQaCLiT6gcY/6u3qBDn01CWw/nbaXl3gTdTFuJJ75Gffv3E3DBpgvY2fkkrdS1fpjaXNOmJlnbtKauKg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12.0" + }, + "peerDependencies": { + "webpack": "^5.82.0", + "webpack-cli": "6.x.x" + }, + "peerDependenciesMeta": { + "webpack-dev-server": { + "optional": true + } + } + }, + "node_modules/@xterm/addon-fit": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.10.0.tgz", + "integrity": "sha512-UFYkDm4HUahf2lnEyHvio51TNGiLK66mqP2JoATy7hRZeXaGMRDr00JiSF7m63vR5WKATF605yEggJKsw0JpMQ==", + "license": "MIT", + "peerDependencies": { + "@xterm/xterm": "^5.0.0" + } + }, + "node_modules/@xterm/xterm": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz", + "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==", + "license": "MIT" + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@zeit/schemas": { + "version": "2.36.0", + "resolved": "https://registry.npmjs.org/@zeit/schemas/-/schemas-2.36.0.tgz", + "integrity": "sha512-7kjMwcChYEzMKjeex9ZFXkt1AyNov9R5HZtjBKVsmVpw7pa7ZtlCGvCBC2vnnXctaYN+aRI61HjIqeetZW5ROg==", + "dev": true, + "license": "MIT" + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-import-phases": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/acorn-import-phases/-/acorn-import-phases-1.0.4.tgz", + "integrity": "sha512-wKmbr/DDiIXzEOiWrTTUcDm24kQ2vGfZQvM2fwg2vXqR5uW6aapr7ObPtj1th32b9u90/Pf4AItvdTh42fBmVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" + }, + "peerDependencies": { + "acorn": "^8.14.0" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/ansi-align": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.1.tgz", + "integrity": "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.1.0" + } + }, + "node_modules/ansi-align/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/ansi-align/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-html-community": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/ansi-html-community/-/ansi-html-community-0.0.8.tgz", + "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==", + "dev": true, + "engines": [ + "node >= 0.8.0" + ], + "license": "Apache-2.0", + "bin": { + "ansi-html": "bin/ansi-html" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arch": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", + "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true, + "license": "MIT" + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.9.19", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz", + "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/batch": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", + "dev": true, + "license": "MIT" + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/bonjour-service": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz", + "integrity": "sha512-3YuAUiSkWykd+2Azjgyxei8OWf8thdn8AITIog2M4UICzoqfjlqr64WIjEXZllf/W6vK1goqleSR6brGomxQqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "multicast-dns": "^7.2.5" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true, + "license": "ISC" + }, + "node_modules/boxen": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-7.0.0.tgz", + "integrity": "sha512-j//dBVuyacJbvW+tvZ9HuH03fZ46QcaKvvhZickZqtB271DxJ7SNRSNxrV/dZX0085m7hISRZWbzWlJvx/rHSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-align": "^3.0.1", + "camelcase": "^7.0.0", + "chalk": "^5.0.1", + "cli-boxes": "^3.0.0", + "string-width": "^5.1.2", + "type-fest": "^2.13.0", + "widest-line": "^4.0.1", + "wrap-ansi": "^8.0.1" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/boxen/node_modules/chalk": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", + "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/bundle-name": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", + "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/bytes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.3.tgz", + "integrity": "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "get-intrinsic": "^1.2.6" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/camel-case": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-4.1.2.tgz", + "integrity": "sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pascal-case": "^3.1.2", + "tslib": "^2.0.3" + } + }, + "node_modules/camelcase": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-7.0.1.tgz", + "integrity": "sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001768", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001768.tgz", + "integrity": "sha512-qY3aDRZC5nWPgHUgIB84WL+nySuo19wk0VJpp/XI9T34lrvkyhRvNVOFJOp2kxClQhiFBu+TaUSudf6oa3vkSA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/chalk-template/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/chalk-template/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/chalk-template/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chrome-trace-event": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.4.tgz", + "integrity": "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0" + } + }, + "node_modules/clean-css": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-5.3.3.tgz", + "integrity": "sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "source-map": "~0.6.0" + }, + "engines": { + "node": ">= 10.0" + } + }, + "node_modules/cli-boxes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-3.0.0.tgz", + "integrity": "sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/clipboardy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/clipboardy/-/clipboardy-3.0.0.tgz", + "integrity": "sha512-Su+uU5sr1jkUy1sGRpLKjKrvEOVXgSgiSInwa/qeID6aJ07yh+5NWc3h2QfjHjBnfX4LhtFcuAWKUsJ3r+fjbg==", + "dev": true, + "license": "MIT", + "dependencies": { + "arch": "^2.2.0", + "execa": "^5.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/codemirror": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/codemirror/-/codemirror-6.0.1.tgz", + "integrity": "sha512-J8j+nZ+CdWmIeFIGXEFbFPtpiYacFMDR8GlHK3IyHQJMCaVRfGx9NT+Hxivv1ckLWPvNdZqndbr/7lVhrf/Svg==", + "license": "MIT", + "dependencies": { + "@codemirror/autocomplete": "^6.0.0", + "@codemirror/commands": "^6.0.0", + "@codemirror/language": "^6.0.0", + "@codemirror/lint": "^6.0.0", + "@codemirror/search": "^6.0.0", + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/command-exists": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", + "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.1.0", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/compression/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/connect-history-api-fallback": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz", + "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/crelt": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz", + "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css-loader": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-7.1.2.tgz", + "integrity": "sha512-6WvYYn7l/XEGN8Xu2vWFt9nVzrCn39vKyTEFf/ExEyoksJjjSZV/0/35XPlMbpnr6VGhZIUg5yJrL8tGfes/FA==", + "dev": true, + "license": "MIT", + "dependencies": { + "icss-utils": "^5.1.0", + "postcss": "^8.4.33", + "postcss-modules-extract-imports": "^3.1.0", + "postcss-modules-local-by-default": "^4.0.5", + "postcss-modules-scope": "^3.2.0", + "postcss-modules-values": "^4.0.0", + "postcss-value-parser": "^4.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "webpack": "^5.27.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/css-select": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz", + "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", + "domutils": "^2.8.0", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/default-browser": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.2.1.tgz", + "integrity": "sha512-WY/3TUME0x3KPYdRRxEJJvXRHV4PyPoUsxtZa78lwItwRQRHhd2U9xOscaT/YTf8uCXIAjeJOFBVEh/7FtD8Xg==", + "dev": true, + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.0.tgz", + "integrity": "sha512-A6p/pu/6fyBcA1TRz/GqWYPViplrftcW2gZC9q79ngNCKAeR/X3gcEdXQHl4KNXV+3wgIJ1CPkJQ3IHM6lcsyA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", + "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true, + "license": "MIT" + }, + "node_modules/dns-packet": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz", + "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dom-converter": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", + "integrity": "sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==", + "dev": true, + "license": "MIT", + "dependencies": { + "utila": "~0.4" + } + }, + "node_modules/dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "dev": true, + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz", + "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true, + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.286", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.286.tgz", + "integrity": "sha512-9tfDXhJ4RKFNerfjdCcZfufu49vg620741MNs26a9+bhLThdB+plgMeou98CAaHu/WATj2iHOOHTp1hWtABj2A==", + "dev": true, + "license": "ISC" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.19.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz", + "integrity": "sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "dev": true, + "license": "BSD-2-Clause", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/envinfo": { + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.14.0.tgz", + "integrity": "sha512-CO40UI41xDQzhLB1hWyqUKgFhs250pNcGbyGKe1l/e4FSaI/+YE4IMG76GDt0In67WLPACIITC+sOi08x4wIvg==", + "dev": true, + "license": "MIT", + "bin": { + "envinfo": "dist/cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", + "integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/express/node_modules/qs": { + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/express/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fastest-levenshtein": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", + "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4.9.1" + } + }, + "node_modules/faye-websocket": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", + "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "websocket-driver": ">=0.5.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true, + "license": "BSD-3-Clause", + "bin": { + "flat": "cli.js" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/handle-thing": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", + "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", + "dev": true, + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/hpack.js": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", + "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.1", + "obuf": "^1.0.0", + "readable-stream": "^2.0.1", + "wbuf": "^1.1.0" + } + }, + "node_modules/hpack.js/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/hpack.js/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/html-minifier-terser": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", + "integrity": "sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "camel-case": "^4.1.2", + "clean-css": "^5.2.2", + "commander": "^8.3.0", + "he": "^1.2.0", + "param-case": "^3.0.4", + "relateurl": "^0.2.7", + "terser": "^5.10.0" + }, + "bin": { + "html-minifier-terser": "cli.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/html-webpack-plugin": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-5.6.3.tgz", + "integrity": "sha512-QSf1yjtSAsmf7rYBV7XX86uua4W/vkhIt0xNXKbsi2foEeW7vjJQz4bhnpL3xH+l1ryl1680uNv968Z+X6jSYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/html-minifier-terser": "^6.0.0", + "html-minifier-terser": "^6.0.2", + "lodash": "^4.17.21", + "pretty-error": "^4.0.0", + "tapable": "^2.0.0" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/html-webpack-plugin" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "webpack": "^5.20.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/htmlparser2": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-6.1.0.tgz", + "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.0.0", + "domutils": "^2.5.2", + "entities": "^2.0.0" + } + }, + "node_modules/http-deceiver": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", + "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-parser-js": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.9.tgz", + "integrity": "sha512-n1XsPy3rXVxlqxVioEWdC+0+M+SQw0DpJynwtOPo1X+ZlvdzTLtDBIJJlDQTnwZIFJrZSzSGmIOUdP8tu+SgLw==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-proxy-middleware": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", + "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-proxy": "^1.17.8", + "http-proxy": "^1.18.1", + "is-glob": "^4.0.1", + "is-plain-obj": "^3.0.0", + "micromatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "@types/express": "^4.17.13" + }, + "peerDependenciesMeta": { + "@types/express": { + "optional": true + } + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/hyperdyperid": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hyperdyperid/-/hyperdyperid-1.2.0.tgz", + "integrity": "sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.18" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/icss-utils": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", + "integrity": "sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, + "node_modules/interpret": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-3.1.1.tgz", + "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/ipaddr.js": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", + "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "dev": true, + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-inside-container/node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "dev": true, + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-network-error": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.1.0.tgz", + "integrity": "sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", + "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-port-reachable": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-port-reachable/-/is-port-reachable-4.0.0.tgz", + "integrity": "sha512-9UoipoxYmSk6Xy7QFgRv2HDyaysmgSG75TFQs6S+3pDM7ZhKTF/bskZV+0UlABHzKjNVhPjYCLfeZUEg1wXxig==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/jest-worker/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/launch-editor": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/launch-editor/-/launch-editor-2.10.0.tgz", + "integrity": "sha512-D7dBRJo/qcGX9xlvt/6wUYzQxjh5G1RvZPgPv8vi4KRU99DVQL/oW7tnVOCCTm2HGeo3C5HvGE5Yrh6UBoZ0vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picocolors": "^1.0.0", + "shell-quote": "^1.8.1" + } + }, + "node_modules/loader-runner": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.1.tgz", + "integrity": "sha512-IWqP2SCPhyVFTBtRcgMHdzlf9ul25NwaFx4wCEH/KjAXuuHY4yNjvPXsBokp8jCB936PyWRaPKUNh8NvylLp2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.11.5" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.23", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", + "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "dev": true, + "license": "MIT" + }, + "node_modules/lower-case": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", + "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.3" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/memfs": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/memfs/-/memfs-4.17.0.tgz", + "integrity": "sha512-4eirfZ7thblFmqFjywlTmuWVSvccHAJbn1r8qQLzmTO11qcqpohOjmY2mFce6x7x7WtskzRqApPD0hv+Oa74jg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/json-pack": "^1.0.3", + "@jsonjoy.com/util": "^1.3.0", + "tree-dump": "^1.0.1", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">= 4.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/mini-css-extract-plugin": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.9.2.tgz", + "integrity": "sha512-GJuACcS//jtq4kCtd5ii/M0SZf7OZRH+BxdqXZHaJfb8TJiVl+NgQRPwiYt2EuqeSkNydn/7vP+bcE27C5mb9w==", + "dev": true, + "license": "MIT", + "dependencies": { + "schema-utils": "^4.0.0", + "tapable": "^2.2.1" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "dev": true, + "license": "ISC" + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true, + "license": "MIT" + }, + "node_modules/multicast-dns": { + "version": "7.2.5", + "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-7.2.5.tgz", + "integrity": "sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==", + "dev": true, + "license": "MIT", + "dependencies": { + "dns-packet": "^5.2.2", + "thunky": "^1.0.2" + }, + "bin": { + "multicast-dns": "cli.js" + } + }, + "node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true, + "license": "MIT" + }, + "node_modules/no-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", + "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "lower-case": "^2.0.2", + "tslib": "^2.0.3" + } + }, + "node_modules/node-forge": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.2.tgz", + "integrity": "sha512-6xKiQ+cph9KImrRh0VsjH2d8/GXA4FIMlgU4B757iI1ApvcyA9VlouP0yZJha01V+huImO+kKMU7ih+2+E14fw==", + "dev": true, + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "dev": true, + "license": "MIT" + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", + "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/open/-/open-10.1.0.tgz", + "integrity": "sha512-mnkeQ1qP5Ue2wd+aivTD3NHd/lZ96Lu0jgf0pwktLPtx6cTZiH7tyeGRRHs0zX0rbrahXPnXlUnbeXyaBBuIaw==", + "dev": true, + "license": "MIT", + "dependencies": { + "default-browser": "^5.2.1", + "define-lazy-prop": "^3.0.0", + "is-inside-container": "^1.0.0", + "is-wsl": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open/node_modules/is-wsl": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz", + "integrity": "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-retry": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", + "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.2", + "is-network-error": "^1.0.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/param-case": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", + "integrity": "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "dot-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/pascal-case": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz", + "integrity": "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", + "dev": true, + "license": "(WTFPL OR MIT)" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-to-regexp": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-3.3.0.tgz", + "integrity": "sha512-qyCH421YQPS2WFDxDjftfc1ZR5WKQzVzqsp4n9M2kQhVOo/ByahFoUNJfl58kOcEGfQ//7weFTDhm+ss8Ecxgw==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/postcss": { + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", + "integrity": "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.8", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-modules-extract-imports": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.1.0.tgz", + "integrity": "sha512-k3kNe0aNFQDAZGbin48pL2VNidTF0w4/eASDsxlyspobzU3wZQLOGj7L9gfRe0Jo9/4uud09DsjFNH7winGv8Q==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-local-by-default": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.2.0.tgz", + "integrity": "sha512-5kcJm/zk+GJDSfw+V/42fJ5fhjL5YbFDl8nVdXkJPLLW+Vf9mTD5Xe0wqIaDnLuL2U6cDNpTr+UQ+v2HWIBhzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "icss-utils": "^5.0.0", + "postcss-selector-parser": "^7.0.0", + "postcss-value-parser": "^4.1.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-scope": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-3.2.1.tgz", + "integrity": "sha512-m9jZstCVaqGjTAuny8MdgE88scJnCiQSlSrOWcTQgM2t32UBe+MUmFSO5t7VMSfAf/FJKImAxBav8ooCHJXCJA==", + "dev": true, + "license": "ISC", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-values": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz", + "integrity": "sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "icss-utils": "^5.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-selector-parser": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", + "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/pretty-error": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pretty-error/-/pretty-error-4.0.0.tgz", + "integrity": "sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash": "^4.17.20", + "renderkid": "^3.0.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-addr/node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/range-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/registry-auth-token": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.3.2.tgz", + "integrity": "sha512-JL39c60XlzCVgNrO+qq68FoNb56w/m7JYvGR2jT5iR1xBrUA3Mfx5Twk5rqTThPmQKMWydGmq8oFtDlxfrmxnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "dev": true, + "license": "MIT", + "dependencies": { + "rc": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/relateurl": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz", + "integrity": "sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/renderkid": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-3.0.0.tgz", + "integrity": "sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "css-select": "^4.1.3", + "dom-converter": "^0.2.0", + "htmlparser2": "^6.1.0", + "lodash": "^4.17.21", + "strip-ansi": "^6.0.1" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/run-applescript": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.0.0.tgz", + "integrity": "sha512-9by4Ij99JUr/MCFBUkDKLWK3G9HVXmabKz9U5MlIAIuvuzkiOicRYs8XJLxX+xahD+mLiiCYDqF9dKAgtzKP1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/schema-utils": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.3.tgz", + "integrity": "sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.9.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/select-hose": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", + "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==", + "dev": true, + "license": "MIT" + }, + "node_modules/selfsigned": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.4.1.tgz", + "integrity": "sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node-forge": "^1.3.0", + "node-forge": "^1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/send/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serialize-javascript": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/serve": { + "version": "14.2.5", + "resolved": "https://registry.npmjs.org/serve/-/serve-14.2.5.tgz", + "integrity": "sha512-Qn/qMkzCcMFVPb60E/hQy+iRLpiU8PamOfOSYoAHmmF+fFFmpPpqa6Oci2iWYpTdOUM3VF+TINud7CfbQnsZbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@zeit/schemas": "2.36.0", + "ajv": "8.12.0", + "arg": "5.0.2", + "boxen": "7.0.0", + "chalk": "5.0.1", + "chalk-template": "0.4.0", + "clipboardy": "3.0.0", + "compression": "1.8.1", + "is-port-reachable": "4.0.0", + "serve-handler": "6.1.6", + "update-check": "1.5.4" + }, + "bin": { + "serve": "build/main.js" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/serve-handler": { + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/serve-handler/-/serve-handler-6.1.6.tgz", + "integrity": "sha512-x5RL9Y2p5+Sh3D38Fh9i/iQ5ZK+e4xuXRd/pGbM4D13tgo/MGwbttUk8emytcr1YYzBYs+apnUngBDFYfpjPuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.0.0", + "content-disposition": "0.5.2", + "mime-types": "2.1.18", + "minimatch": "3.1.2", + "path-is-inside": "1.0.2", + "path-to-regexp": "3.3.0", + "range-parser": "1.2.0" + } + }, + "node_modules/serve-handler/node_modules/mime-db": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz", + "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-handler/node_modules/mime-types": { + "version": "2.1.18", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz", + "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "~1.33.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", + "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "~1.3.4", + "batch": "0.6.1", + "debug": "2.6.9", + "escape-html": "~1.0.3", + "http-errors": "~1.6.2", + "mime-types": "~2.1.17", + "parseurl": "~1.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-index/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true, + "license": "ISC" + }, + "node_modules/serve-index/node_modules/setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/serve-index/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve/node_modules/ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/serve/node_modules/chalk": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.0.1.tgz", + "integrity": "sha512-Fo07WOYGqMfCWHOzSXOt2CxDbC6skS/jO9ynEcmpANMoPrD+W1r1K6Vx7iNm+AQmETU1Xr2t+n8nzkV9t6xh3w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true, + "license": "ISC" + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "license": "MIT", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.2.tgz", + "integrity": "sha512-AzqKpGKjrj7EM6rKVQEPpB288oCfnrEIuyoT9cyF4nmGa7V8Zk6f7RRqYisX8X9m+Q7bd632aZW4ky7EhbQztA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/sockjs": { + "version": "0.3.24", + "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", + "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "faye-websocket": "^0.11.3", + "uuid": "^8.3.2", + "websocket-driver": "^0.7.4" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/spdy": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz", + "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.0", + "handle-thing": "^2.0.0", + "http-deceiver": "^1.2.7", + "select-hose": "^2.0.0", + "spdy-transport": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/spdy-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", + "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.0", + "detect-node": "^2.0.4", + "hpack.js": "^2.1.6", + "obuf": "^1.1.2", + "readable-stream": "^3.0.6", + "wbuf": "^1.7.3" + } + }, + "node_modules/spdy-transport/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/spdy-transport/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/spdy/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/spdy/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/style-mod": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.2.tgz", + "integrity": "sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==", + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tapable": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", + "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/terser": { + "version": "5.39.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.39.0.tgz", + "integrity": "sha512-LBAhFyLho16harJoWMg/nZsQYgTrg5jXOn2nCYjRUcZZEdE3qa2zb8QEDRUGVZBW4rlazf2fxkg8tztybTaqWw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.8.2", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "5.3.16", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz", + "integrity": "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.0", + "serialize-javascript": "^6.0.2", + "terser": "^5.31.1" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/terser/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/thingies": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/thingies/-/thingies-1.21.0.tgz", + "integrity": "sha512-hsqsJsFMsV+aD4s3CWKk85ep/3I9XzYV/IXaSouJMYIoDlgyi11cBhsqYe9/geRfB0YIikBQg6raRaM+nIMP9g==", + "dev": true, + "license": "Unlicense", + "engines": { + "node": ">=10.18" + }, + "peerDependencies": { + "tslib": "^2" + } + }, + "node_modules/thunky": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", + "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==", + "dev": true, + "license": "MIT" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tree-dump": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/tree-dump/-/tree-dump-1.0.2.tgz", + "integrity": "sha512-dpev9ABuLWdEubk+cIaI9cHwRNNDjkBBLXTwI4UCUFdQ5xXKqNXoK4FEciw/vxf+NQ7Cb7sGUyeUtORvHIdRXQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true, + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/update-check": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/update-check/-/update-check-1.5.4.tgz", + "integrity": "sha512-5YHsflzHP4t1G+8WGPlvKbJEbAJGCgw+Em+dGR1KmBUbr1J36SJBqlHLjR7oob7sco5hWHGQVcr9B2poIVDDTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "registry-auth-token": "3.3.2", + "registry-url": "3.1.0" + } + }, + "node_modules/upgrade": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/upgrade/-/upgrade-1.1.0.tgz", + "integrity": "sha512-NtkVvqVCqsJo5U3mYRum2Tw6uCltOxfIJ/AfTZeTmw6U39IB5X23xF+kRZ9aiPaORqeiQQ7Q209/ibhOvxzwHA==", + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/utila": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/utila/-/utila-0.4.0.tgz", + "integrity": "sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==", + "dev": true, + "license": "MIT" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "license": "MIT" + }, + "node_modules/watchpack": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz", + "integrity": "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/wbuf": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", + "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimalistic-assert": "^1.0.0" + } + }, + "node_modules/webpack": { + "version": "5.105.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", + "integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/eslint-scope": "^3.7.7", + "@types/estree": "^1.0.8", + "@types/json-schema": "^7.0.15", + "@webassemblyjs/ast": "^1.14.1", + "@webassemblyjs/wasm-edit": "^1.14.1", + "@webassemblyjs/wasm-parser": "^1.14.1", + "acorn": "^8.15.0", + "acorn-import-phases": "^1.0.3", + "browserslist": "^4.28.1", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.19.0", + "es-module-lexer": "^2.0.0", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.11", + "json-parse-even-better-errors": "^2.3.1", + "loader-runner": "^4.3.1", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^4.3.3", + "tapable": "^2.3.0", + "terser-webpack-plugin": "^5.3.16", + "watchpack": "^2.5.1", + "webpack-sources": "^3.3.3" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependenciesMeta": { + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-cli": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-6.0.1.tgz", + "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@discoveryjs/json-ext": "^0.6.1", + "@webpack-cli/configtest": "^3.0.1", + "@webpack-cli/info": "^3.0.1", + "@webpack-cli/serve": "^3.0.1", + "colorette": "^2.0.14", + "commander": "^12.1.0", + "cross-spawn": "^7.0.3", + "envinfo": "^7.14.0", + "fastest-levenshtein": "^1.0.12", + "import-local": "^3.0.2", + "interpret": "^3.1.1", + "rechoir": "^0.8.0", + "webpack-merge": "^6.0.1" + }, + "bin": { + "webpack-cli": "bin/cli.js" + }, + "engines": { + "node": ">=18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.82.0" + }, + "peerDependenciesMeta": { + "webpack-bundle-analyzer": { + "optional": true + }, + "webpack-dev-server": { + "optional": true + } + } + }, + "node_modules/webpack-cli/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/webpack-dev-middleware": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-7.4.2.tgz", + "integrity": "sha512-xOO8n6eggxnwYpy1NlzUKpvrjfJTvae5/D6WOK0S2LSo7vjmo5gCM1DbLUmFqrMTJP+W/0YZNctm7jasWvLuBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "colorette": "^2.0.10", + "memfs": "^4.6.0", + "mime-types": "^2.1.31", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "schema-utils": "^4.0.0" + }, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + }, + "peerDependenciesMeta": { + "webpack": { + "optional": true + } + } + }, + "node_modules/webpack-dev-middleware/node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/webpack-dev-server": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.1.tgz", + "integrity": "sha512-ml/0HIj9NLpVKOMq+SuBPLHcmbG+TGIjXRHsYfZwocUBIqEvws8NnS/V9AFQ5FKP+tgn5adwVwRrTEpGL33QFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/bonjour": "^3.5.13", + "@types/connect-history-api-fallback": "^1.5.4", + "@types/express": "^4.17.21", + "@types/express-serve-static-core": "^4.17.21", + "@types/serve-index": "^1.9.4", + "@types/serve-static": "^1.15.5", + "@types/sockjs": "^0.3.36", + "@types/ws": "^8.5.10", + "ansi-html-community": "^0.0.8", + "bonjour-service": "^1.2.1", + "chokidar": "^3.6.0", + "colorette": "^2.0.10", + "compression": "^1.7.4", + "connect-history-api-fallback": "^2.0.0", + "express": "^4.21.2", + "graceful-fs": "^4.2.6", + "http-proxy-middleware": "^2.0.7", + "ipaddr.js": "^2.1.0", + "launch-editor": "^2.6.1", + "open": "^10.0.3", + "p-retry": "^6.2.0", + "schema-utils": "^4.2.0", + "selfsigned": "^2.4.1", + "serve-index": "^1.9.1", + "sockjs": "^0.3.24", + "spdy": "^4.0.2", + "webpack-dev-middleware": "^7.4.2", + "ws": "^8.18.0" + }, + "bin": { + "webpack-dev-server": "bin/webpack-dev-server.js" + }, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + }, + "peerDependenciesMeta": { + "webpack": { + "optional": true + }, + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-dev-server/node_modules/@types/express-serve-static-core": { + "version": "4.19.6", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", + "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/webpack-merge": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz", + "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "clone-deep": "^4.0.1", + "flat": "^5.0.2", + "wildcard": "^2.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/webpack-sources": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.3.3.tgz", + "integrity": "sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/websocket-driver": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz", + "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "http-parser-js": ">=0.5.1", + "safe-buffer": ">=5.1.0", + "websocket-extensions": ">=0.1.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/widest-line": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-4.0.1.tgz", + "integrity": "sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^5.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wildcard": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.1.tgz", + "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xterm-readline": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/xterm-readline/-/xterm-readline-1.1.2.tgz", + "integrity": "sha512-1+W2nVuQvCYz9OUYwFBiolrSQUui51aDDyacKXt4PuxeBHqzvabQEJ2kwdBDzsmOjz5BwlDTAjJmYpH2OGqLFA==", + "license": "MIT", + "dependencies": { + "string-width": "^4" + }, + "peerDependencies": { + "@xterm/xterm": "^5.5.0" + } + }, + "node_modules/xterm-readline/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/xterm-readline/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/wasm/demo/package.json b/wasm/demo/package.json index c8751325087..7954e8cd866 100644 --- a/wasm/demo/package.json +++ b/wasm/demo/package.json @@ -4,27 +4,28 @@ "description": "Bindings to the RustPython library for WebAssembly", "main": "index.js", "dependencies": { - "codemirror": "^5.42.0", - "local-echo": "^0.2.0", - "xterm": "^3.8.0" + "@codemirror/lang-python": "^6.1.6", + "@xterm/addon-fit": "^0.10.0", + "@xterm/xterm": "^5.3.0", + "codemirror": "^6.0.1", + "upgrade": "^1.1.0", + "xterm-readline": "^1.1.2" }, "devDependencies": { - "@wasm-tool/wasm-pack-plugin": "^1.1.0", - "clean-webpack-plugin": "^3.0.0", - "css-loader": "^3.4.1", - "html-webpack-plugin": "^3.2.0", - "mini-css-extract-plugin": "^0.9.0", - "raw-loader": "^4.0.0", - "serve": "^11.0.2", - "webpack": "^4.16.3", - "webpack-cli": "^3.1.0", - "webpack-dev-server": "^3.1.5" + "@wasm-tool/wasm-pack-plugin": "^1.7.0", + "css-loader": "^7.1.2", + "html-webpack-plugin": "^5.6.3", + "mini-css-extract-plugin": "^2.9.2", + "serve": "^14.2.5", + "webpack": "^5.105.0", + "webpack-cli": "^6.0.1", + "webpack-dev-server": "^5.2.1" }, "scripts": { - "dev": "webpack-dev-server -d", + "dev": "webpack serve", "build": "webpack", "dist": "webpack --mode production", - "test": "webpack --mode production && cd ../tests && pytest" + "test": "webpack --mode production && cd ../tests && pytest -v" }, "repository": { "type": "git", diff --git a/wasm/demo/snippets/asyncbrowser.py b/wasm/demo/snippets/asyncbrowser.py index 5cd2f7b0a0d..979c1d389d0 100644 --- a/wasm/demo/snippets/asyncbrowser.py +++ b/wasm/demo/snippets/asyncbrowser.py @@ -1,5 +1,6 @@ -import browser import asyncweb +import browser + async def main(delay): url = f"https://httpbin.org/delay/{delay}" diff --git a/wasm/demo/snippets/fetch.py b/wasm/demo/snippets/fetch.py index f507057b223..80e1775c76a 100644 --- a/wasm/demo/snippets/fetch.py +++ b/wasm/demo/snippets/fetch.py @@ -1,12 +1,12 @@ from browser import fetch + def fetch_handler(res): print(f"headers: {res['headers']}") + fetch( "https://httpbin.org/get", response_format="json", - headers={ - "X-Header-Thing": "rustpython is neat!" - }, + headers={"X-Header-Thing": "rustpython is neat!"}, ).then(fetch_handler, lambda err: print(f"error: {err}")) diff --git a/wasm/demo/snippets/import_pypi.py b/wasm/demo/snippets/import_pypi.py index cbe837cd058..547fe2365e6 100644 --- a/wasm/demo/snippets/import_pypi.py +++ b/wasm/demo/snippets/import_pypi.py @@ -1,20 +1,20 @@ +# make sys.modules['os'] a dumb version of the os module, which has posixpath +# available as os.path as well as a few other utilities, but will raise an +# OSError for anything that actually requires an OS +import _dummy_os import asyncweb import whlimport whlimport.setup() -# make sys.modules['os'] a dumb version of the os module, which has posixpath -# available as os.path as well as a few other utilities, but will raise an -# OSError for anything that actually requires an OS -import _dummy_os -_dummy_os._shim() @asyncweb.main async def main(): await whlimport.load_package("pygments") import pygments - import pygments.lexers import pygments.formatters.html + import pygments.lexers + lexer = pygments.lexers.get_lexer_by_name("python") fmter = pygments.formatters.html.HtmlFormatter(noclasses=True, style="default") print(pygments.highlight("print('hi, mom!')", lexer, fmter)) diff --git a/wasm/demo/snippets/mandelbrot.py b/wasm/demo/snippets/mandelbrot.py index b4010c75396..ea4fade56dc 100644 --- a/wasm/demo/snippets/mandelbrot.py +++ b/wasm/demo/snippets/mandelbrot.py @@ -1,6 +1,7 @@ w = 50.0 h = 50.0 + def mandel(): """Print a mandelbrot fractal to the console, yielding after each character is printed""" y = 0.0 @@ -20,9 +21,9 @@ def mandel(): i += 1 if Tr + Ti <= 4: - print('*', end='') + print("*", end="") else: - print('·', end='') + print("·", end="") x += 1 yield @@ -31,14 +32,24 @@ def mandel(): y += 1 yield + # run the mandelbrot -try: from browser import request_animation_frame -except: request_animation_frame = None +try: + from browser import request_animation_frame +except: + request_animation_frame = None gen = mandel() + + def gen_cb(_time=None): - for _ in range(4): gen.__next__() + for _ in range(4): + gen.__next__() request_animation_frame(gen_cb) -if request_animation_frame: gen_cb() -else: any(gen) + + +if request_animation_frame: + gen_cb() +else: + any(gen) diff --git a/wasm/demo/src/index.ejs b/wasm/demo/src/index.ejs index 02f2f52f5cf..d261b110e5a 100644 --- a/wasm/demo/src/index.ejs +++ b/wasm/demo/src/index.ejs @@ -14,7 +14,6 @@ browser's devtools and play with <code>rp.pyEval('1 + 1')</code> </p> <div id="code-wrapper"> - <textarea id="code"><%= defaultSnippet %></textarea> <select id="snippets"> <% for (const name of snippets) { %> <option @@ -77,7 +76,7 @@ <a href="https://github.com/RustPython/RustPython"> <img style="position: absolute; top: 0; right: 0; border: 0;" - src="https://s3.amazonaws.com/github/ribbons/forkme_right_green_007200.png" + src="https://github.blog/wp-content/uploads/2008/12/forkme_right_green_007200.png" alt="Fork me on GitHub" /> </a> diff --git a/wasm/demo/src/index.js b/wasm/demo/src/index.js index aa1619eb2f9..0b568fa1d9e 100644 --- a/wasm/demo/src/index.js +++ b/wasm/demo/src/index.js @@ -1,44 +1,54 @@ import './style.css'; -import 'xterm/lib/xterm.css'; -import CodeMirror from 'codemirror'; -import 'codemirror/mode/python/python'; -import 'codemirror/addon/comment/comment'; -import 'codemirror/lib/codemirror.css'; -import { Terminal } from 'xterm'; -import LocalEchoController from 'local-echo'; +import '@xterm/xterm/css/xterm.css'; +import { EditorView, basicSetup } from 'codemirror'; +import { keymap } from '@codemirror/view'; +import { indentUnit } from '@codemirror/language'; +import { indentWithTab } from '@codemirror/commands'; +import { python } from '@codemirror/lang-python'; +import { Terminal } from '@xterm/xterm'; +import { FitAddon } from '@xterm/addon-fit'; +import { Readline } from 'xterm-readline'; let rp; // A dependency graph that contains any wasm must be imported asynchronously. import('rustpython') - .then((rustpy) => { - rp = rustpy; + .then((rustpython) => { + rp = rustpython; // so people can play around with it - window.rp = rustpy; + window.rp = rustpython; onReady(); }) .catch((e) => { console.error('Error importing `rustpython`:', e); - document.getElementById('error').textContent = e; + let errorDetails = e.toString(); + if (window.__RUSTPYTHON_ERROR) { + errorDetails += '\nRustPython Error: ' + window.__RUSTPYTHON_ERROR; + } + if (window.__RUSTPYTHON_ERROR_STACK) { + errorDetails += '\nStack: ' + window.__RUSTPYTHON_ERROR_STACK; + } + document.getElementById('error').textContent = errorDetails; }); -const editor = CodeMirror.fromTextArea(document.getElementById('code'), { - extraKeys: { - 'Ctrl-Enter': runCodeFromTextarea, - 'Cmd-Enter': runCodeFromTextarea, - 'Shift-Tab': 'indentLess', - 'Ctrl-/': 'toggleComment', - 'Cmd-/': 'toggleComment', - Tab: (editor) => { - var spaces = Array(editor.getOption('indentUnit') + 1).join(' '); - editor.replaceSelection(spaces); - }, - }, - lineNumbers: true, - mode: 'text/x-python', - indentUnit: 4, - autofocus: true, +const fixedHeightEditor = EditorView.theme({ + '&': { height: '100%' }, + '.cm-scroller': { overflow: 'auto' }, }); +const editor = new EditorView({ + parent: document.getElementById('code-wrapper'), + extensions: [ + basicSetup, + python(), + keymap.of( + { key: 'Ctrl-Enter', mac: 'Cmd-Enter', run: runCodeFromTextarea }, + indentWithTab, + ), + indentUnit.of(' '), + fixedHeightEditor, + ], +}); +editor.focus(); const consoleElement = document.getElementById('console'); const errorElement = document.getElementById('error'); @@ -48,7 +58,7 @@ function runCodeFromTextarea() { consoleElement.value = ''; errorElement.textContent = ''; - const code = editor.getValue(); + const code = editor.state.doc.toString(); try { rp.pyExec(code, { stdout: (output) => { @@ -78,18 +88,25 @@ function updateSnippet() { // the require here creates a webpack context; it's fine to use it // dynamically. // https://webpack.js.org/guides/dependency-management/ - const { default: snippet } = require( - `raw-loader!../snippets/${selected}.py`, - ); + const snippet = require(`../snippets/${selected}.py?raw`); - editor.setValue(snippet); - runCodeFromTextarea(); + editor.dispatch({ + changes: { from: 0, to: editor.state.doc.length, insert: snippet }, + }); +} +function updateSnippetAndRun() { + updateSnippet(); + requestAnimationFrame(runCodeFromTextarea); } +updateSnippet(); const term = new Terminal(); +const readline = new Readline(); +const fitAddon = new FitAddon(); +term.loadAddon(readline); +term.loadAddon(fitAddon); term.open(document.getElementById('terminal')); - -const localEcho = new LocalEchoController(term); +fitAddon.fit(); let terminalVM; @@ -107,40 +124,33 @@ finally: } async function readPrompts() { - let continuing = false; + let continuing = ''; while (true) { - const ps1 = getPrompt('ps1'); - const ps2 = getPrompt('ps2'); - let input; + let input = await readline.read(getPrompt(continuing ? 'ps2' : 'ps1')); + if (input.endsWith('\n')) input = input.slice(0, -1); if (continuing) { - const prom = localEcho.read(ps2, ps2); - localEcho._activePrompt.prompt = ps1; - localEcho._input = localEcho.history.entries.pop() + '\n'; - localEcho._cursor = localEcho._input.length; - localEcho._active = true; - input = await prom; - if (!input.endsWith('\n')) continue; - } else { - input = await localEcho.read(ps1, ps2); + input = continuing += '\n' + input; + if (!continuing.endsWith('\n')) continue; } try { + console.log([input]); terminalVM.execSingle(input); } catch (err) { if (err.canContinue) { - continuing = true; + continuing = input; continue; } else if (err instanceof WebAssembly.RuntimeError) { err = window.__RUSTPYTHON_ERROR || err; } - localEcho.println(err); + readline.print('' + err); } - continuing = false; + continuing = ''; } } function onReady() { - snippets.addEventListener('change', updateSnippet); + snippets.addEventListener('change', updateSnippetAndRun); document .getElementById('run-btn') .addEventListener('click', runCodeFromTextarea); @@ -148,7 +158,7 @@ function onReady() { runCodeFromTextarea(); terminalVM = rp.vmStore.init('term_vm'); - terminalVM.setStdout((data) => localEcho.print(data)); + terminalVM.setStdout((data) => readline.print(data)); readPrompts().catch((err) => console.error(err)); // so that the test knows that we're ready diff --git a/wasm/demo/src/style.css b/wasm/demo/src/style.css index 771894e4910..f9892745acc 100644 --- a/wasm/demo/src/style.css +++ b/wasm/demo/src/style.css @@ -3,7 +3,7 @@ textarea { resize: vertical; } -#code, +#code-wrapper, #console { height: 30vh; width: calc(100% - 3px); diff --git a/wasm/demo/webpack.config.js b/wasm/demo/webpack.config.js index e20b4772411..6c798d2ca4f 100644 --- a/wasm/demo/webpack.config.js +++ b/wasm/demo/webpack.config.js @@ -1,7 +1,6 @@ const HtmlWebpackPlugin = require('html-webpack-plugin'); const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const WasmPackPlugin = require('@wasm-tool/wasm-pack-plugin'); -const { CleanWebpackPlugin } = require('clean-webpack-plugin'); const path = require('path'); const fs = require('fs'); @@ -12,13 +11,14 @@ module.exports = (env = {}) => { output: { path: path.join(__dirname, 'dist'), filename: 'index.js', + clean: true, }, mode: 'development', resolve: { alias: { rustpython: path.resolve( __dirname, - env.rustpythonPkg || '../lib/pkg', + env.rustpythonPkg || '../../crates/wasm/pkg', ), }, }, @@ -28,10 +28,17 @@ module.exports = (env = {}) => { test: /\.css$/, use: [MiniCssExtractPlugin.loader, 'css-loader'], }, + { + resourceQuery: '?raw', + type: 'asset/source', + }, + { + test: /\.wasm$/, + type: 'webassembly/async', + }, ], }, plugins: [ - new CleanWebpackPlugin(), new HtmlWebpackPlugin({ filename: 'index.html', template: 'src/index.ejs', @@ -51,11 +58,14 @@ module.exports = (env = {}) => { filename: 'styles.css', }), ], + experiments: { + asyncWebAssembly: true, + }, }; if (!env.noWasmPack) { config.plugins.push( new WasmPackPlugin({ - crateDirectory: path.join(__dirname, '../lib'), + crateDirectory: path.join(__dirname, '../../crates/wasm'), }), ); } diff --git a/wasm/example/package.json b/wasm/example/package.json index 5653448b258..40b70e0b75d 100644 --- a/wasm/example/package.json +++ b/wasm/example/package.json @@ -6,7 +6,7 @@ }, "devDependencies": { "raw-loader": "1.0.0", - "webpack": "4.28.2", + "webpack": "5.104.1", "webpack-cli": "^3.1.2" }, "scripts": { diff --git a/wasm/example/src/main.py b/wasm/example/src/main.py index 5447d078afc..855b1e2ce49 100644 --- a/wasm/example/src/main.py +++ b/wasm/example/src/main.py @@ -1,12 +1,14 @@ -from browser import fetch, alert +from browser import alert, fetch + def fetch_handler(repos): star_sum = 0 for repo in repos: - star_sum += repo['stars'] - alert(f'Average github trending star count: {star_sum / len(repos)}') + star_sum += repo["stars"] + alert(f"Average github trending star count: {star_sum / len(repos)}") + fetch( - 'https://github-trending-api.now.sh/repositories', - response_format='json', -).then(fetch_handler, lambda err: alert(f"Error: {err}")) \ No newline at end of file + "https://github-trending-api.now.sh/repositories", + response_format="json", +).then(fetch_handler, lambda err: alert(f"Error: {err}")) diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml deleted file mode 100644 index 309607e70fe..00000000000 --- a/wasm/lib/Cargo.toml +++ /dev/null @@ -1,46 +0,0 @@ -[package] -name = "rustpython_wasm" -version = "0.3.0" -authors = ["RustPython Team"] -license = "MIT" -description = "A Python-3 (CPython >= 3.5.0) Interpreter written in Rust, compiled to WASM" -repository = "https://github.com/RustPython/RustPython/tree/main/wasm/lib" -edition = "2021" - -[lib] -crate-type = ["cdylib", "rlib"] - -[features] -default = ["freeze-stdlib"] -freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib/freeze-stdlib", "rustpython-stdlib"] -no-start-func = [] - -[dependencies] -rustpython-common = { workspace = true } -rustpython-pylib = { workspace = true, optional = true } -rustpython-stdlib = { workspace = true, default-features = false, optional = true } -# make sure no threading! otherwise wasm build will fail -rustpython-vm = { workspace = true, features = ["compiler", "encodings", "serde"] } - -rustpython-parser = { workspace = true } - -serde = { workspace = true } - -console_error_panic_hook = "0.1" -js-sys = "0.3" -serde-wasm-bindgen = "0.3.1" -wasm-bindgen = "0.2.80" -wasm-bindgen-futures = "0.4" -web-sys = { version = "0.3", features = [ - "console", - "Document", - "Element", - "Window", - "Headers", - "Request", - "RequestInit", - "Response" -] } - -[package.metadata.wasm-pack.profile.release] -wasm-opt = false#["-O1"] diff --git a/wasm/lib/README.md b/wasm/lib/README.md deleted file mode 100644 index b5f0e5df108..00000000000 --- a/wasm/lib/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# RustPython - -A Python-3 (CPython >= 3.8.0) Interpreter written in Rust. - -[![Build Status](https://travis-ci.org/RustPython/RustPython.svg?branch=main)](https://travis-ci.org/RustPython/RustPython) -[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) - -# WARNING: this project is still in a pre-alpha state! - -**Using this in a production project is inadvisable. Please only do so if you understand the risks.** - -## Usage - -#### Check out our [online demo](https://rustpython.github.io/demo/) running on WebAssembly. - -## Goals - -- Full Python-3 environment entirely in Rust (not CPython bindings) -- A clean implementation without compatibility hacks - -## Quick Documentation - -```js -pyEval(code, options?); -``` - -`code`: `string`: The Python code to run - -`options`: - -- `vars?`: `{ [key: string]: any }`: Variables passed to the VM that can be - accessed in Python with the variable `js_vars`. Functions do work, and - receive the Python kwargs as the `this` argument. -- `stdout?`: `"console" | ((out: string) => void) | null`: A function to replace the - native print function, and it will be `console.log` when giving `undefined` - or "console", and it will be a dumb function when giving null. - -## License - -This project is licensed under the MIT license. diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs deleted file mode 100644 index 134567fba62..00000000000 --- a/wasm/lib/src/convert.rs +++ /dev/null @@ -1,271 +0,0 @@ -use crate::js_module; -use crate::vm_class::{stored_vm_from_wasm, WASMVirtualMachine}; -use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array}; -use rustpython_parser::ParseErrorType; -use rustpython_vm::{ - builtins::PyBaseExceptionRef, - compiler::{CompileError, CompileErrorType}, - exceptions, - function::{ArgBytesLike, FuncArgs}, - py_serde, AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, -}; -use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; - -#[wasm_bindgen(inline_js = r" -export class PyError extends Error { - constructor(info) { - const msg = info.args[0]; - if (typeof msg === 'string') super(msg); - else super(); - this.info = info; - } - get name() { return this.info.exc_type; } - get traceback() { return this.info.traceback; } - toString() { return this.info.rendered; } -} -")] -extern "C" { - pub type PyError; - #[wasm_bindgen(constructor)] - fn new(info: JsValue) -> PyError; -} - -pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyBaseExceptionRef) -> JsValue { - let jserr = vm.try_class("_js", "JSError").ok(); - let js_arg = if jserr.map_or(false, |jserr| py_err.fast_isinstance(&jserr)) { - py_err.get_arg(0) - } else { - None - }; - let js_arg = js_arg - .as_ref() - .and_then(|x| x.payload::<js_module::PyJsValue>()); - match js_arg { - Some(val) => val.value.clone(), - None => { - let res = - serde_wasm_bindgen::to_value(&exceptions::SerializeException::new(vm, py_err)); - match res { - Ok(err_info) => PyError::new(err_info).into(), - Err(e) => e.into(), - } - } - } -} - -pub fn js_py_typeerror(vm: &VirtualMachine, js_err: JsValue) -> PyBaseExceptionRef { - let msg = js_err.unchecked_into::<js_sys::Error>().to_string(); - vm.new_type_error(msg.into()) -} - -pub fn js_err_to_py_err(vm: &VirtualMachine, js_err: &JsValue) -> PyBaseExceptionRef { - match js_err.dyn_ref::<js_sys::Error>() { - Some(err) => { - let exc_type = match String::from(err.name()).as_str() { - "TypeError" => vm.ctx.exceptions.type_error, - "ReferenceError" => vm.ctx.exceptions.name_error, - "SyntaxError" => vm.ctx.exceptions.syntax_error, - _ => vm.ctx.exceptions.exception_type, - } - .to_owned(); - vm.new_exception_msg(exc_type, err.message().into()) - } - None => vm.new_exception_msg( - vm.ctx.exceptions.exception_type.to_owned(), - format!("{js_err:?}"), - ), - } -} - -pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { - if let Some(ref wasm_id) = vm.wasm_id { - if py_obj.fast_isinstance(vm.ctx.types.function_type) { - let wasm_vm = WASMVirtualMachine { - id: wasm_id.clone(), - }; - let weak_py_obj = wasm_vm.push_held_rc(py_obj).unwrap().unwrap(); - - let closure = move |args: Option<Box<[JsValue]>>, - kwargs: Option<Object>| - -> Result<JsValue, JsValue> { - let py_obj = match wasm_vm.assert_valid() { - Ok(_) => weak_py_obj - .upgrade() - .expect("weak_py_obj to be valid if VM is valid"), - Err(err) => { - return Err(err); - } - }; - stored_vm_from_wasm(&wasm_vm).interp.enter(move |vm| { - let args = match args { - Some(args) => Vec::from(args) - .into_iter() - .map(|arg| js_to_py(vm, arg)) - .collect::<Vec<_>>(), - None => Vec::new(), - }; - let mut py_func_args = FuncArgs::from(args); - if let Some(ref kwargs) = kwargs { - for pair in object_entries(kwargs) { - let (key, val) = pair?; - py_func_args - .kwargs - .insert(js_sys::JsString::from(key).into(), js_to_py(vm, val)); - } - } - let result = py_obj.call(py_func_args, vm); - pyresult_to_jsresult(vm, result) - }) - }; - let closure = Closure::wrap(Box::new(closure) - as Box< - dyn FnMut(Option<Box<[JsValue]>>, Option<Object>) -> Result<JsValue, JsValue>, - >); - let func = closure.as_ref().clone(); - - // stores pretty much nothing, it's fine to leak this because if it gets dropped - // the error message is worse - closure.forget(); - - return func; - } - } - // the browser module might not be injected - if vm.try_class("_js", "Promise").is_ok() { - if let Some(py_prom) = py_obj.payload::<js_module::PyPromise>() { - return py_prom.as_js(vm).into(); - } - } - - if let Ok(bytes) = ArgBytesLike::try_from_borrowed_object(vm, &py_obj) { - bytes.with_ref(|bytes| unsafe { - // `Uint8Array::view` is an `unsafe fn` because it provides - // a direct view into the WASM linear memory; if you were to allocate - // something with Rust that view would probably become invalid. It's safe - // because we then copy the array using `Uint8Array::slice`. - let view = Uint8Array::view(bytes); - view.slice(0, bytes.len() as u32).into() - }) - } else { - py_serde::serialize(vm, &py_obj, &serde_wasm_bindgen::Serializer::new()) - .unwrap_or(JsValue::UNDEFINED) - } -} - -pub fn object_entries(obj: &Object) -> impl Iterator<Item = Result<(JsValue, JsValue), JsValue>> { - Object::entries(obj).values().into_iter().map(|pair| { - pair.map(|pair| { - let key = Reflect::get(&pair, &"0".into()).unwrap(); - let val = Reflect::get(&pair, &"1".into()).unwrap(); - (key, val) - }) - }) -} - -pub fn pyresult_to_jsresult(vm: &VirtualMachine, result: PyResult) -> Result<JsValue, JsValue> { - result - .map(|value| py_to_js(vm, value)) - .map_err(|err| py_err_to_js_err(vm, &err)) -} - -pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { - if js_val.is_object() { - if let Some(promise) = js_val.dyn_ref::<Promise>() { - // the browser module might not be injected - if vm.try_class("browser", "Promise").is_ok() { - return js_module::PyPromise::new(promise.clone()) - .into_ref(&vm.ctx) - .into(); - } - } - if Array::is_array(&js_val) { - let js_arr: Array = js_val.into(); - let elems = js_arr - .values() - .into_iter() - .map(|val| js_to_py(vm, val.expect("Iteration over array failed"))) - .collect(); - vm.ctx.new_list(elems).into() - } else if ArrayBuffer::is_view(&js_val) || js_val.is_instance_of::<ArrayBuffer>() { - // unchecked_ref because if it's not an ArrayBuffer it could either be a TypedArray - // or a DataView, but they all have a `buffer` property - let u8_array = js_sys::Uint8Array::new( - &js_val - .dyn_ref::<ArrayBuffer>() - .cloned() - .unwrap_or_else(|| js_val.unchecked_ref::<Uint8Array>().buffer()), - ); - let mut vec = vec![0; u8_array.length() as usize]; - u8_array.copy_to(&mut vec); - vm.ctx.new_bytes(vec).into() - } else { - let dict = vm.ctx.new_dict(); - for pair in object_entries(&Object::from(js_val)) { - let (key, val) = pair.expect("iteration over object to not fail"); - let py_val = js_to_py(vm, val); - dict.set_item( - String::from(js_sys::JsString::from(key)).as_str(), - py_val, - vm, - ) - .unwrap(); - } - dict.into() - } - } else if js_val.is_function() { - let func = js_sys::Function::from(js_val); - vm.new_function( - vm.ctx.intern_str(String::from(func.name())).as_str(), - move |args: FuncArgs, vm: &VirtualMachine| -> PyResult { - let this = Object::new(); - for (k, v) in args.kwargs { - Reflect::set(&this, &k.into(), &py_to_js(vm, v)) - .expect("property to be settable"); - } - let js_args = args - .args - .into_iter() - .map(|v| py_to_js(vm, v)) - .collect::<Array>(); - func.apply(&this, &js_args) - .map(|val| js_to_py(vm, val)) - .map_err(|err| js_err_to_py_err(vm, &err)) - }, - ) - .into() - } else if let Some(err) = js_val.dyn_ref::<js_sys::Error>() { - js_err_to_py_err(vm, err).into() - } else if js_val.is_undefined() { - // Because `JSON.stringify(undefined)` returns undefined - vm.ctx.none() - } else { - py_serde::deserialize(vm, serde_wasm_bindgen::Deserializer::from(js_val)) - .unwrap_or_else(|_| vm.ctx.none()) - } -} - -pub fn syntax_err(err: CompileError) -> SyntaxError { - let js_err = SyntaxError::new(&format!("Error parsing Python code: {err}")); - let _ = Reflect::set( - &js_err, - &"row".into(), - &(err.location.unwrap().row.get()).into(), - ); - let _ = Reflect::set( - &js_err, - &"col".into(), - &(err.location.unwrap().column.get()).into(), - ); - let can_continue = matches!(&err.error, CompileErrorType::Parse(ParseErrorType::Eof)); - let _ = Reflect::set(&js_err, &"canContinue".into(), &can_continue.into()); - js_err -} - -pub trait PyResultExt<T> { - fn into_js(self, vm: &VirtualMachine) -> Result<T, JsValue>; -} -impl<T> PyResultExt<T> for PyResult<T> { - fn into_js(self, vm: &VirtualMachine) -> Result<T, JsValue> { - self.map_err(|err| py_err_to_js_err(vm, &err)) - } -} diff --git a/wasm/lib/src/lib.rs b/wasm/lib/src/lib.rs deleted file mode 100644 index 85546c78d33..00000000000 --- a/wasm/lib/src/lib.rs +++ /dev/null @@ -1,135 +0,0 @@ -pub mod browser_module; -pub mod convert; -pub mod js_module; -pub mod vm_class; -pub mod wasm_builtins; - -#[macro_use] -extern crate rustpython_vm; - -use js_sys::{Reflect, WebAssembly::RuntimeError}; -use std::panic; -pub use vm_class::add_init_func; -pub(crate) use vm_class::weak_vm; -use wasm_bindgen::prelude::*; - -/// Sets error info on the window object, and prints the backtrace to console -pub fn panic_hook(info: &panic::PanicInfo) { - // If something errors, just ignore it; we don't want to panic in the panic hook - let try_set_info = || { - let msg = &info.to_string(); - let window = match web_sys::window() { - Some(win) => win, - None => return, - }; - let _ = Reflect::set(&window, &"__RUSTPYTHON_ERROR_MSG".into(), &msg.into()); - let error = RuntimeError::new(msg); - let _ = Reflect::set(&window, &"__RUSTPYTHON_ERROR".into(), &error); - let stack = match Reflect::get(&error, &"stack".into()) { - Ok(stack) => stack, - Err(_) => return, - }; - let _ = Reflect::set(&window, &"__RUSTPYTHON_ERROR_STACK".into(), &stack); - }; - try_set_info(); - console_error_panic_hook::hook(info); -} - -#[doc(hidden)] -#[cfg(not(feature = "no-start-func"))] -#[wasm_bindgen(start)] -pub fn _setup_console_error() { - std::panic::set_hook(Box::new(panic_hook)); -} - -pub mod eval { - use crate::vm_class::VMStore; - use js_sys::{Object, Reflect, TypeError}; - use rustpython_vm::compiler::Mode; - use wasm_bindgen::prelude::*; - - const PY_EVAL_VM_ID: &str = "__py_eval_vm"; - - fn run_py(source: &str, options: Option<Object>, mode: Mode) -> Result<JsValue, JsValue> { - let vm = VMStore::init(PY_EVAL_VM_ID.into(), Some(true)); - let options = options.unwrap_or_default(); - let js_vars = { - let prop = Reflect::get(&options, &"vars".into())?; - if prop.is_undefined() { - None - } else if prop.is_object() { - Some(Object::from(prop)) - } else { - return Err(TypeError::new("vars must be an object").into()); - } - }; - - vm.set_stdout(Reflect::get(&options, &"stdout".into())?)?; - - if let Some(js_vars) = js_vars { - vm.add_to_scope("js_vars".into(), js_vars.into())?; - } - vm.run(source, mode, None) - } - - /// Evaluate Python code - /// - /// ```js - /// var result = pyEval(code, options?); - /// ``` - /// - /// `code`: `string`: The Python code to run in eval mode - /// - /// `options`: - /// - /// - `vars?`: `{ [key: string]: any }`: Variables passed to the VM that can be - /// accessed in Python with the variable `js_vars`. Functions do work, and - /// receive the Python kwargs as the `this` argument. - /// - `stdout?`: `"console" | ((out: string) => void) | null`: A function to replace the - /// native print native print function, and it will be `console.log` when giving - /// `undefined` or "console", and it will be a dumb function when giving null. - #[wasm_bindgen(js_name = pyEval)] - pub fn eval_py(source: &str, options: Option<Object>) -> Result<JsValue, JsValue> { - run_py(source, options, Mode::Eval) - } - - /// Evaluate Python code - /// - /// ```js - /// pyExec(code, options?); - /// ``` - /// - /// `code`: `string`: The Python code to run in exec mode - /// - /// `options`: The options are the same as eval mode - #[wasm_bindgen(js_name = pyExec)] - pub fn exec_py(source: &str, options: Option<Object>) -> Result<(), JsValue> { - run_py(source, options, Mode::Exec).map(drop) - } - - /// Evaluate Python code - /// - /// ```js - /// var result = pyExecSingle(code, options?); - /// ``` - /// - /// `code`: `string`: The Python code to run in exec single mode - /// - /// `options`: The options are the same as eval mode - #[wasm_bindgen(js_name = pyExecSingle)] - pub fn exec_single_py(source: &str, options: Option<Object>) -> Result<JsValue, JsValue> { - run_py(source, options, Mode::Single) - } -} - -/// A module containing all the wasm-bindgen exports that rustpython_wasm has -/// Re-export as `pub use rustpython_wasm::exports::*;` in the root of your crate if you want your -/// wasm module to mimic rustpython_wasm's API -pub mod exports { - pub use crate::convert::PyError; - pub use crate::eval::{eval_py, exec_py, exec_single_py}; - pub use crate::vm_class::{VMStore, WASMVirtualMachine}; -} - -#[doc(hidden)] -pub use exports::*; diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs deleted file mode 100644 index 0c62bb32e7e..00000000000 --- a/wasm/lib/src/vm_class.rs +++ /dev/null @@ -1,363 +0,0 @@ -use crate::{ - browser_module::setup_browser_module, - convert::{self, PyResultExt}, - js_module, wasm_builtins, -}; -use js_sys::{Object, TypeError}; -use rustpython_vm::{ - builtins::{PyModule, PyWeak}, - compiler::Mode, - scope::Scope, - Interpreter, PyObjectRef, PyPayload, PyRef, PyResult, Settings, VirtualMachine, -}; -use std::{ - cell::RefCell, - collections::HashMap, - rc::{Rc, Weak}, -}; -use wasm_bindgen::prelude::*; - -pub(crate) struct StoredVirtualMachine { - pub interp: Interpreter, - pub scope: Scope, - /// you can put a Rc in here, keep it as a Weak, and it'll be held only for - /// as long as the StoredVM is alive - held_objects: RefCell<Vec<PyObjectRef>>, -} - -#[pymodule] -mod _window {} - -fn init_window_module(vm: &VirtualMachine) -> PyRef<PyModule> { - let module = _window::make_module(vm); - - extend_module!(vm, &module, { - "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(&vm.ctx), - }); - - module -} - -impl StoredVirtualMachine { - fn new(id: String, inject_browser_module: bool) -> StoredVirtualMachine { - let mut scope = None; - let mut settings = Settings::default(); - settings.allow_external_library = false; - let interp = Interpreter::with_init(settings, |vm| { - #[cfg(feature = "freeze-stdlib")] - vm.add_native_modules(rustpython_stdlib::get_module_inits()); - - #[cfg(feature = "freeze-stdlib")] - vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); - - vm.wasm_id = Some(id); - - js_module::setup_js_module(vm); - if inject_browser_module { - vm.add_native_module("_window".to_owned(), Box::new(init_window_module)); - setup_browser_module(vm); - } - - VM_INIT_FUNCS.with(|cell| { - for f in cell.borrow().iter() { - f(vm) - } - }); - - scope = Some(vm.new_scope_with_builtins()); - }); - - StoredVirtualMachine { - interp, - scope: scope.unwrap(), - held_objects: RefCell::new(Vec::new()), - } - } -} - -/// Add a hook to add builtins or frozen modules to the RustPython VirtualMachine while it's -/// initializing. -pub fn add_init_func(f: fn(&mut VirtualMachine)) { - VM_INIT_FUNCS.with(|cell| cell.borrow_mut().push(f)) -} - -// It's fine that it's thread local, since WASM doesn't even have threads yet. thread_local! -// probably gets compiled down to a normal-ish static variable, like Atomic* types do: -// https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions -thread_local! { - static STORED_VMS: RefCell<HashMap<String, Rc<StoredVirtualMachine>>> = RefCell::default(); - static VM_INIT_FUNCS: RefCell<Vec<fn(&mut VirtualMachine)>> = RefCell::default(); -} - -pub fn get_vm_id(vm: &VirtualMachine) -> &str { - vm.wasm_id - .as_ref() - .expect("VirtualMachine inside of WASM crate should have wasm_id set") -} -pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc<StoredVirtualMachine> { - STORED_VMS.with(|cell| { - cell.borrow() - .get(&wasm_vm.id) - .expect("VirtualMachine is not valid") - .clone() - }) -} -pub(crate) fn weak_vm(vm: &VirtualMachine) -> Weak<StoredVirtualMachine> { - let id = get_vm_id(vm); - STORED_VMS - .with(|cell| Rc::downgrade(cell.borrow().get(id).expect("VirtualMachine is not valid"))) -} - -#[wasm_bindgen(js_name = vmStore)] -pub struct VMStore; - -#[wasm_bindgen(js_class = vmStore)] -impl VMStore { - pub fn init(id: String, inject_browser_module: Option<bool>) -> WASMVirtualMachine { - STORED_VMS.with(|cell| { - let mut vms = cell.borrow_mut(); - if !vms.contains_key(&id) { - let stored_vm = - StoredVirtualMachine::new(id.clone(), inject_browser_module.unwrap_or(true)); - vms.insert(id.clone(), Rc::new(stored_vm)); - } - }); - WASMVirtualMachine { id } - } - - pub(crate) fn _get(id: String) -> Option<WASMVirtualMachine> { - STORED_VMS.with(|cell| { - let vms = cell.borrow(); - if vms.contains_key(&id) { - Some(WASMVirtualMachine { id }) - } else { - None - } - }) - } - - pub fn get(id: String) -> JsValue { - match Self::_get(id) { - Some(wasm_vm) => wasm_vm.into(), - None => JsValue::UNDEFINED, - } - } - - pub fn destroy(id: String) { - STORED_VMS.with(|cell| { - use std::collections::hash_map::Entry; - match cell.borrow_mut().entry(id) { - Entry::Occupied(o) => { - let (_k, stored_vm) = o.remove_entry(); - // for f in stored_vm.drop_handlers.iter() { - // f(); - // } - // deallocate the VM - drop(stored_vm); - } - Entry::Vacant(_v) => {} - } - }); - } - - pub fn ids() -> Vec<JsValue> { - STORED_VMS.with(|cell| cell.borrow().keys().map(|k| k.into()).collect()) - } -} - -#[wasm_bindgen(js_name = VirtualMachine)] -#[derive(Clone)] -pub struct WASMVirtualMachine { - pub(crate) id: String, -} - -#[wasm_bindgen(js_class = VirtualMachine)] -impl WASMVirtualMachine { - pub(crate) fn with_unchecked<F, R>(&self, f: F) -> R - where - F: FnOnce(&StoredVirtualMachine) -> R, - { - let stored_vm = STORED_VMS.with(|cell| { - let mut vms = cell.borrow_mut(); - vms.get_mut(&self.id).unwrap().clone() - }); - f(&stored_vm) - } - - pub(crate) fn with<F, R>(&self, f: F) -> Result<R, JsValue> - where - F: FnOnce(&StoredVirtualMachine) -> R, - { - self.assert_valid()?; - Ok(self.with_unchecked(f)) - } - - pub(crate) fn with_vm<F, R>(&self, f: F) -> Result<R, JsValue> - where - F: FnOnce(&VirtualMachine, &StoredVirtualMachine) -> R, - { - self.with(|stored| stored.interp.enter(|vm| f(vm, stored))) - } - - pub fn valid(&self) -> bool { - STORED_VMS.with(|cell| cell.borrow().contains_key(&self.id)) - } - - pub(crate) fn push_held_rc( - &self, - obj: PyObjectRef, - ) -> Result<PyResult<PyRef<PyWeak>>, JsValue> { - self.with_vm(|vm, stored_vm| { - let weak = obj.downgrade(None, vm)?; - stored_vm.held_objects.borrow_mut().push(obj); - Ok(weak) - }) - } - - pub fn assert_valid(&self) -> Result<(), JsValue> { - if self.valid() { - Ok(()) - } else { - Err(TypeError::new( - "Invalid VirtualMachine, this VM was destroyed while this reference was still held", - ) - .into()) - } - } - - pub fn destroy(&self) -> Result<(), JsValue> { - self.assert_valid()?; - VMStore::destroy(self.id.clone()); - Ok(()) - } - - #[wasm_bindgen(js_name = addToScope)] - pub fn add_to_scope(&self, name: String, value: JsValue) -> Result<(), JsValue> { - self.with_vm(move |vm, StoredVirtualMachine { ref scope, .. }| { - let value = convert::js_to_py(vm, value); - scope.globals.set_item(&name, value, vm).into_js(vm) - })? - } - - #[wasm_bindgen(js_name = setStdout)] - pub fn set_stdout(&self, stdout: JsValue) -> Result<(), JsValue> { - self.with_vm(|vm, _| { - fn error() -> JsValue { - TypeError::new("Unknown stdout option, please pass a function or 'console'").into() - } - use wasm_builtins::make_stdout_object; - let stdout: PyObjectRef = if let Some(s) = stdout.as_string() { - match s.as_str() { - "console" => make_stdout_object(vm, wasm_builtins::sys_stdout_write_console), - _ => return Err(error()), - } - } else if stdout.is_function() { - let func = js_sys::Function::from(stdout); - make_stdout_object(vm, move |data, vm| { - func.call1(&JsValue::UNDEFINED, &data.into()) - .map_err(|err| convert::js_py_typeerror(vm, err))?; - Ok(()) - }) - } else if stdout.is_null() { - make_stdout_object(vm, |_, _| Ok(())) - } else if stdout.is_undefined() { - make_stdout_object(vm, wasm_builtins::sys_stdout_write_console) - } else { - return Err(error()); - }; - vm.sys_module.set_attr("stdout", stdout, vm).unwrap(); - Ok(()) - })? - } - - #[wasm_bindgen(js_name = injectModule)] - pub fn inject_module( - &self, - name: String, - source: &str, - imports: Option<Object>, - ) -> Result<(), JsValue> { - self.with_vm(|vm, _| { - let code = vm - .compile(source, Mode::Exec, name.clone()) - .map_err(convert::syntax_err)?; - let attrs = vm.ctx.new_dict(); - attrs - .set_item("__name__", vm.new_pyobj(name.as_str()), vm) - .into_js(vm)?; - - if let Some(imports) = imports { - for entry in convert::object_entries(&imports) { - let (key, value) = entry?; - let key: String = Object::from(key).to_string().into(); - attrs - .set_item(key.as_str(), convert::js_to_py(vm, value), vm) - .into_js(vm)?; - } - } - - vm.run_code_obj(code, Scope::new(None, attrs.clone())) - .into_js(vm)?; - - let module = vm.new_module(&name, attrs, None); - - let sys_modules = vm.sys_module.get_attr("modules", vm).into_js(vm)?; - sys_modules.set_item(&name, module.into(), vm).into_js(vm)?; - - Ok(()) - })? - } - - #[wasm_bindgen(js_name = injectJSModule)] - pub fn inject_js_module(&self, name: String, module: Object) -> Result<(), JsValue> { - self.with_vm(|vm, _| { - let py_module = vm.new_module(&name, vm.ctx.new_dict(), None); - for entry in convert::object_entries(&module) { - let (key, value) = entry?; - let key = Object::from(key).to_string(); - extend_module!(vm, &py_module, { - String::from(key) => convert::js_to_py(vm, value), - }); - } - - let sys_modules = vm.sys_module.get_attr("modules", vm).into_js(vm)?; - sys_modules - .set_item(&name, py_module.into(), vm) - .into_js(vm)?; - - Ok(()) - })? - } - - pub(crate) fn run( - &self, - source: &str, - mode: Mode, - source_path: Option<String>, - ) -> Result<JsValue, JsValue> { - self.with_vm(|vm, StoredVirtualMachine { ref scope, .. }| { - let source_path = source_path.unwrap_or_else(|| "<wasm>".to_owned()); - let code = vm.compile(source, mode, source_path); - let code = code.map_err(convert::syntax_err)?; - let result = vm.run_code_obj(code, scope.clone()); - convert::pyresult_to_jsresult(vm, result) - })? - } - - pub fn exec(&self, source: &str, source_path: Option<String>) -> Result<JsValue, JsValue> { - self.run(source, Mode::Exec, source_path) - } - - pub fn eval(&self, source: &str, source_path: Option<String>) -> Result<JsValue, JsValue> { - self.run(source, Mode::Eval, source_path) - } - - #[wasm_bindgen(js_name = execSingle)] - pub fn exec_single( - &self, - source: &str, - source_path: Option<String>, - ) -> Result<JsValue, JsValue> { - self.run(source, Mode::Single, source_path) - } -} diff --git a/wasm/notebook/README.md b/wasm/notebook/README.md index 0e4d07d0e69..f9a07e9eb58 100644 --- a/wasm/notebook/README.md +++ b/wasm/notebook/README.md @@ -10,10 +10,10 @@ You can use the notebook to experiment with using Python and Javascript in the b The main use case is for scientific communication where you can have: -- text or thesis in markdown, -- math with Tex, -- a model or analysis written in python, -- a user interface and interactive visualization with JS. +- text or thesis in markdown, +- math with Tex, +- a model or analysis written in python, +- a user interface and interactive visualization with JS. The Notebook loads python in your browser (so you don't have to install it) then let yous play with those languages. @@ -25,14 +25,14 @@ To read more about the reasoning behind certain features, check the blog on [htt Sample notebooks are under `snippets` -- `snippets/python-markdown-math.txt`: python, markdown and math -- `snippets/python-js.txt`, adds javascript -- `snippets/python-js-css-md/` adds styling with css in separate, more organized files. +- `snippets/python-markdown-math.txt`: python, markdown and math +- `snippets/python-js.txt`, adds javascript +- `snippets/python-js-css-md/` adds styling with css in separate, more organized files. ## How to use -- Run locally with `npm run dev` -- Build with `npm run dist` +- Run locally with `npm run dev` +- Build with `npm run dist` ## JS API @@ -96,6 +96,6 @@ assert adder(5) == 9 ## Wish list / TO DO -- Collaborative peer-to-peer editing with WebRTC. Think Google Doc or Etherpad editing but for code in the browser -- `%%load` command for dynamically adding javascript libraries or css framework -- Clean up and organize the code. Seriously rethink if we want to make it more than a toy. +- Collaborative peer-to-peer editing with WebRTC. Think Google Doc or Etherpad editing but for code in the browser +- `%%load` command for dynamically adding javascript libraries or css framework +- Clean up and organize the code. Seriously rethink if we want to make it more than a toy. diff --git a/wasm/notebook/package.json b/wasm/notebook/package.json index 2a730258cde..64517331c49 100644 --- a/wasm/notebook/package.json +++ b/wasm/notebook/package.json @@ -12,19 +12,17 @@ "xterm": "^3.8.0" }, "devDependencies": { - "@wasm-tool/wasm-pack-plugin": "^1.1.0", - "clean-webpack-plugin": "^3.0.0", - "css-loader": "^3.4.1", - "html-webpack-plugin": "^3.2.0", - "mini-css-extract-plugin": "^0.9.0", - "raw-loader": "^4.0.0", - "serve": "^11.0.2", - "webpack": "^4.16.3", - "webpack-cli": "^3.1.0", - "webpack-dev-server": "^3.1.5" + "@wasm-tool/wasm-pack-plugin": "^1.7.0", + "css-loader": "^7.1.2", + "html-webpack-plugin": "^5.6.3", + "lezer-loader": "^0.3.0", + "mini-css-extract-plugin": "^2.9.2", + "webpack": "^5.97.1", + "webpack-cli": "^6.0.1", + "webpack-dev-server": "^5.2.0" }, "scripts": { - "dev": "webpack-dev-server -d", + "dev": "webpack serve", "build": "webpack", "dist": "webpack --mode production", "test": "webpack --mode production && cd ../tests && pytest" diff --git a/wasm/notebook/src/index.js b/wasm/notebook/src/index.js index 422bc4d0d65..64b058a9aca 100644 --- a/wasm/notebook/src/index.js +++ b/wasm/notebook/src/index.js @@ -34,10 +34,10 @@ let rp; // A dependency graph that contains any wasm must be imported asynchronously. import('rustpython') - .then((rustpy) => { - rp = rustpy; + .then((rustpython) => { + rp = rustpython; // so people can play around with it - window.rp = rustpy; + window.rp = rustpython; onReady(); }) .catch((e) => { diff --git a/wasm/notebook/webpack.config.js b/wasm/notebook/webpack.config.js index 9fda3cf4aad..a3e3f8ef713 100644 --- a/wasm/notebook/webpack.config.js +++ b/wasm/notebook/webpack.config.js @@ -1,7 +1,6 @@ const HtmlWebpackPlugin = require('html-webpack-plugin'); const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const WasmPackPlugin = require('@wasm-tool/wasm-pack-plugin'); -const { CleanWebpackPlugin } = require('clean-webpack-plugin'); const path = require('path'); const fs = require('fs'); @@ -12,13 +11,14 @@ module.exports = (env = {}) => { output: { path: path.join(__dirname, 'dist'), filename: 'index.js', + clean: true, }, mode: 'development', resolve: { alias: { rustpython: path.resolve( __dirname, - env.rustpythonPkg || '../lib/pkg', + env.rustpythonPkg || '../../crates/wasm/pkg', ), }, }, @@ -30,15 +30,14 @@ module.exports = (env = {}) => { }, { test: /\.(woff(2)?|ttf)$/, - use: { - loader: 'file-loader', - options: { name: 'fonts/[name].[ext]' }, + type: 'asset/resource', + generator: { + filename: 'fonts/[name].[ext]', }, }, ], }, plugins: [ - new CleanWebpackPlugin(), new HtmlWebpackPlugin({ filename: 'index.html', template: 'src/index.ejs', @@ -58,6 +57,9 @@ module.exports = (env = {}) => { filename: 'styles.css', }), ], + experiments: { + asyncWebAssembly: true, + }, }; if (!env.noWasmPack) { config.plugins.push( diff --git a/wasm/tests/conftest.py b/wasm/tests/conftest.py index ecb5b0007b1..c7555c9958d 100644 --- a/wasm/tests/conftest.py +++ b/wasm/tests/conftest.py @@ -1,10 +1,11 @@ -import subprocess +import atexit import os -import time import socket -import atexit -import pytest +import subprocess import sys +import time + +import pytest PORT = 8080 @@ -41,7 +42,7 @@ def pytest_sessionfinish(session): # From https://gist.github.com/butla/2d9a4c0f35ea47b7452156c96a4e7b12 -def wait_for_port(port, host="0.0.0.0", timeout=5.0): +def wait_for_port(port, host="localhost", timeout=5.0): """Wait until a port starts accepting TCP connections. Args: port (int): Port number. @@ -65,11 +66,11 @@ def wait_for_port(port, host="0.0.0.0", timeout=5.0): from selenium import webdriver -from selenium.webdriver.firefox.options import Options +from selenium.common.exceptions import JavascriptException from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.firefox.options import Options from selenium.webdriver.support import expected_conditions as EC -from selenium.common.exceptions import JavascriptException +from selenium.webdriver.support.ui import WebDriverWait class Driver(webdriver.Firefox): @@ -94,7 +95,7 @@ def wdriver(request): options.add_argument("-headless") driver = Driver(options=options) try: - driver.get(f"http://0.0.0.0:{PORT}") + driver.get(f"http://localhost:{PORT}") WebDriverWait(driver, 5).until( EC.presence_of_element_located((By.ID, "rp_loaded")) ) @@ -102,6 +103,12 @@ def wdriver(request): driver._print_panic() driver.quit() raise + except Exception as e: + print(f"Error waiting for page to load: {e}") + # Check the page source to see what's loaded + print("Page source:", driver.page_source[:500]) + driver.quit() + raise yield driver diff --git a/wasm/tests/requirements.txt b/wasm/tests/requirements.txt index 52dc2039123..a474c5f74b7 100644 --- a/wasm/tests/requirements.txt +++ b/wasm/tests/requirements.txt @@ -1,3 +1,3 @@ pytest -selenium +selenium==4.36.0 certifi